6056 files changed, 402592 insertions, 139974 deletions

diff --git a/Documentation/00-INDEX b/Documentation/00-INDEX
index 5bd4b07c2f90..c8a8eb1a2b11 100644
--- a/Documentation/00-INDEX
+++ b/Documentation/00-INDEX
@@ -152,8 +152,6 @@ driver-model/
 	- directory with info about Linux driver model.
 early-userspace/
 	- info about initramfs, klibc, and userspace early during boot.
-edac.txt
-	- information on EDAC - Error Detection And Correction
 efi-stub.txt
 	- How to use the EFI boot stub to bypass GRUB or elilo on EFI systems.
 eisa.txt
diff --git a/Documentation/ABI/stable/sysfs-devices b/Documentation/ABI/stable/sysfs-devices
index df449d79b563..35c457f8ce73 100644
--- a/Documentation/ABI/stable/sysfs-devices
+++ b/Documentation/ABI/stable/sysfs-devices
@@ -8,3 +8,17 @@ Description:
 		Any device associated with a device-tree node will have
 		an of_path symlink pointing to the corresponding device
 		node in /sys/firmware/devicetree/
+
+What:		/sys/devices/*/devspec
+Date:		October 2016
+Contact:	Device Tree mailing list <devicetree@vger.kernel.org>
+Description:
+		If CONFIG_OF is enabled, then this file is present. When
+		read, it returns full name of the device node.
+
+What:		/sys/devices/*/obppath
+Date:		October 2016
+Contact:	Device Tree mailing list <devicetree@vger.kernel.org>
+Description:
+		If CONFIG_OF is enabled, then this file is present. When
+		read, it returns full name of the device node.
diff --git a/Documentation/ABI/testing/sysfs-block b/Documentation/ABI/testing/sysfs-block
index 71d184dbb70d..2da04ce6aeef 100644
--- a/Documentation/ABI/testing/sysfs-block
+++ b/Documentation/ABI/testing/sysfs-block
@@ -235,3 +235,45 @@ Description:
 		write_same_max_bytes is 0, write same is not supported
 		by the device.
 
+What:		/sys/block/<disk>/queue/write_zeroes_max_bytes
+Date:		November 2016
+Contact:	Chaitanya Kulkarni <chaitanya.kulkarni@wdc.com>
+Description:
+		Devices that support write zeroes operation in which a
+		single request can be issued to zero out the range of
+		contiguous blocks on storage without having any payload
+		in the request. This can be used to optimize writing zeroes
+		to the devices. write_zeroes_max_bytes indicates how many
+		bytes can be written in a single write zeroes command. If
+		write_zeroes_max_bytes is 0, write zeroes is not supported
+		by the device.
+
+What:		/sys/block/<disk>/queue/zoned
+Date:		September 2016
+Contact:	Damien Le Moal <damien.lemoal@hgst.com>
+Description:
+		zoned indicates if the device is a zoned block device
+		and the zone model of the device if it is indeed zoned.
+		The possible values indicated by zoned are "none" for
+		regular block devices and "host-aware" or "host-managed"
+		for zoned block devices. The characteristics of
+		host-aware and host-managed zoned block devices are
+		described in the ZBC (Zoned Block Commands) and ZAC
+		(Zoned Device ATA Command Set) standards. These standards
+		also define the "drive-managed" zone model. However,
+		since drive-managed zoned block devices do not support
+		zone commands, they will be treated as regular block
+		devices and zoned will report "none".
+
+What:		/sys/block/<disk>/queue/chunk_sectors
+Date:		September 2016
+Contact:	Hannes Reinecke <hare@suse.com>
+Description:
+		chunk_sectors has different meaning depending on the type
+		of the disk. For a RAID device (dm-raid), chunk_sectors
+		indicates the size in 512B sectors of the RAID volume
+		stripe segment. For a zoned block device, either
+		host-aware or host-managed, chunk_sectors indicates the
+		size of 512B sectors of the zones of the device, with
+		the eventual exception of the last zone of the device
+		which may be smaller.
diff --git a/Documentation/ABI/testing/sysfs-bus-fsl-mc b/Documentation/ABI/testing/sysfs-bus-fsl-mc
new file mode 100644
index 000000000000..80256b8b4f26
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-fsl-mc
@@ -0,0 +1,21 @@
+What:		/sys/bus/fsl-mc/drivers/.../bind
+Date:		December 2016
+Contact:	stuart.yoder@nxp.com
+Description:
+		Writing a device location to this file will cause
+		the driver to attempt to bind to the device found at
+		this location. The format for the location is Object.Id
+		and is the same as found in /sys/bus/fsl-mc/devices/.
+                For example:
+		# echo dpni.2 > /sys/bus/fsl-mc/drivers/fsl_dpaa2_eth/bind
+
+What:		/sys/bus/fsl-mc/drivers/.../unbind
+Date:		December 2016
+Contact:	stuart.yoder@nxp.com
+Description:
+		Writing a device location to this file will cause the
+		driver to attempt to unbind from the device found at
+		this location. The format for the location is Object.Id
+		and is the same as found in /sys/bus/fsl-mc/devices/.
+                For example:
+		# echo dpni.2 > /sys/bus/fsl-mc/drivers/fsl_dpaa2_eth/unbind
diff --git a/Documentation/ABI/testing/sysfs-bus-iio b/Documentation/ABI/testing/sysfs-bus-iio
index fee35c00cc4e..b8f220f978dd 100644
--- a/Documentation/ABI/testing/sysfs-bus-iio
+++ b/Documentation/ABI/testing/sysfs-bus-iio
@@ -329,6 +329,7 @@ What:		/sys/bus/iio/devices/iio:deviceX/in_pressure_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_humidityrelative_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_velocity_sqrt(x^2+y^2+z^2)_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_illuminance_scale
+What:		/sys/bus/iio/devices/iio:deviceX/in_countY_scale
 KernelVersion:	2.6.35
 Contact:	linux-iio@vger.kernel.org
 Description:
@@ -1579,3 +1580,20 @@ Contact:	linux-iio@vger.kernel.org
 Description:
 		Raw (unscaled no offset etc.) electric conductivity reading that
 		can be processed to siemens per meter.
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_countY_raw
+KernelVersion:	4.9
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Raw counter device counts from channel Y. For quadrature
+		counters, multiplication by an available [Y]_scale results in
+		the counts of a single quadrature signal phase from channel Y.
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_indexY_raw
+KernelVersion:	4.9
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Raw counter device index value from channel Y. This attribute
+		provides an absolute positional reference (e.g. a pulse once per
+		revolution) which may be used to home positional systems as
+		required.
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-adc-envelope-detector b/Documentation/ABI/testing/sysfs-bus-iio-adc-envelope-detector
new file mode 100644
index 000000000000..2071f9bcfaa5
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-iio-adc-envelope-detector
@@ -0,0 +1,36 @@
+What:		/sys/bus/iio/devices/iio:deviceX/in_altvoltageY_invert
+Date:		October 2016
+KernelVersion:	4.9
+Contact:	Peter Rosin <peda@axentia.se>
+Description:
+		The DAC is used to find the peak level of an alternating
+		voltage input signal by a binary search using the output
+		of a comparator wired to an interrupt pin. Like so:
+		                           _
+		                          | \
+		     input +------>-------|+ \
+		                          |   \
+		            .-------.     |    }---.
+		            |       |     |   /    |
+		            |    dac|-->--|- /     |
+		            |       |     |_/      |
+		            |       |              |
+		            |       |              |
+		            |    irq|------<-------'
+		            |       |
+		            '-------'
+		The boolean invert attribute (0/1) should be set when the
+		input signal is centered around the maximum value of the
+		dac instead of zero. The envelope detector will search
+		from below in this case and will also invert the result.
+		The edge/level of the interrupt is also switched to its
+		opposite value.
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_altvoltageY_compare_interval
+Date:		October 2016
+KernelVersion:	4.9
+Contact:	Peter Rosin <peda@axentia.se>
+Description:
+		Number of milliseconds to wait for the comparator in each
+		step of the binary search for the input peak level. Needs
+		to relate to the frequency of the input signal.
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-counter-104-quad-8 b/Documentation/ABI/testing/sysfs-bus-iio-counter-104-quad-8
new file mode 100644
index 000000000000..ba676520b953
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-iio-counter-104-quad-8
@@ -0,0 +1,125 @@
+What:		/sys/bus/iio/devices/iio:deviceX/in_count_count_direction_available
+What:		/sys/bus/iio/devices/iio:deviceX/in_count_count_mode_available
+What:		/sys/bus/iio/devices/iio:deviceX/in_count_noise_error_available
+What:		/sys/bus/iio/devices/iio:deviceX/in_count_quadrature_mode_available
+What:		/sys/bus/iio/devices/iio:deviceX/in_index_index_polarity_available
+What:		/sys/bus/iio/devices/iio:deviceX/in_index_synchronous_mode_available
+KernelVersion:	4.9
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Discrete set of available values for the respective counter
+		configuration are listed in this file.
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_countY_count_direction
+KernelVersion:	4.9
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Read-only attribute that indicates whether the counter for
+		channel Y is counting up or down.
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_countY_count_mode
+KernelVersion:	4.9
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Count mode for channel Y. Four count modes are available:
+		normal, range limit, non-recycle, and modulo-n. The preset value
+		for channel Y is used by the count mode where required.
+
+		Normal:
+			Counting is continuous in either direction.
+
+		Range Limit:
+			An upper or lower limit is set, mimicking limit switches
+			in the mechanical counterpart. The upper limit is set to
+			the preset value, while the lower limit is set to 0. The
+			counter freezes at count = preset when counting up, and
+			at count = 0 when counting down. At either of these
+			limits, the counting is resumed only when the count
+			direction is reversed.
+
+		Non-recycle:
+			Counter is disabled whenever a 24-bit count overflow or
+			underflow takes place. The counter is re-enabled when a
+			new count value is loaded to the counter via a preset
+			operation or write to raw.
+
+		Modulo-N:
+			A count boundary is set between 0 and the preset value.
+			The counter is reset to 0 at count = preset when
+			counting up, while the counter is set to the preset
+			value at count = 0 when counting down; the counter does
+			not freeze at the bundary points, but counts
+			continuously throughout.
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_countY_noise_error
+KernelVersion:	4.9
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Read-only attribute that indicates whether excessive noise is
+		present at the channel Y count inputs in quadrature clock mode;
+		irrelevant in non-quadrature clock mode.
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_countY_preset
+KernelVersion:	4.9
+Contact:	linux-iio@vger.kernel.org
+Description:
+		If the counter device supports preset registers, the preset
+		count for channel Y is provided by this attribute.
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_countY_quadrature_mode
+KernelVersion:	4.9
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Configure channel Y counter for non-quadrature or quadrature
+		clock mode. Selecting non-quadrature clock mode will disable
+		synchronous load mode. In quadrature clock mode, the channel Y
+		scale attribute selects the encoder phase division (scale of 1
+		selects full-cycle, scale of 0.5 selects half-cycle, scale of
+		0.25 selects quarter-cycle) processed by the channel Y counter.
+
+		Non-quadrature:
+			The filter and decoder circuit are bypassed. Encoder A
+			input serves as the count input and B as the UP/DOWN
+			direction control input, with B = 1 selecting UP Count
+			mode and B = 0 selecting Down Count mode.
+
+		Quadrature:
+			Encoder A and B inputs are digitally filtered and
+			decoded for UP/DN clock.
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_countY_set_to_preset_on_index
+KernelVersion:	4.9
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Whether to set channel Y counter with channel Y preset value
+		when channel Y index input is active, or continuously count.
+		Valid attribute values are boolean.
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_indexY_index_polarity
+KernelVersion:	4.9
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Active level of channel Y index input; irrelevant in
+		non-synchronous load mode.
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_indexY_synchronous_mode
+KernelVersion:	4.9
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Configure channel Y counter for non-synchronous or synchronous
+		load mode. Synchronous load mode cannot be selected in
+		non-quadrature clock mode.
+
+		Non-synchronous:
+			A logic low level is the active level at this index
+			input. The index function (as enabled via
+			set_to_preset_on_index) is performed directly on the
+			active level of the index input.
+
+		Synchronous:
+			Intended for interfacing with encoder Index output in
+			quadrature clock mode. The active level is configured
+			via index_polarity. The index function (as enabled via
+			set_to_preset_on_index) is performed synchronously with
+			the quadrature clock on the active level of the index
+			input.
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-cros-ec b/Documentation/ABI/testing/sysfs-bus-iio-cros-ec
new file mode 100644
index 000000000000..297b9720f024
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-iio-cros-ec
@@ -0,0 +1,18 @@
+What:		/sys/bus/iio/devices/iio:deviceX/calibrate
+Date:		July 2015
+KernelVersion:	4.7
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Writing '1' will perform a FOC (Fast Online Calibration). The
+                corresponding calibration offsets can be read from *_calibbias
+                entries.
+
+What:		/sys/bus/iio/devices/iio:deviceX/location
+Date:		July 2015
+KernelVersion:	4.7
+Contact:	linux-iio@vger.kernel.org
+Description:
+		This attribute returns a string with the physical location where
+                the motion sensor is placed. For example, in a laptop a motion
+                sensor can be located on the base or on the lid. Current valid
+		values are 'base' and 'lid'.
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-dac-dpot-dac b/Documentation/ABI/testing/sysfs-bus-iio-dac-dpot-dac
new file mode 100644
index 000000000000..580e93f373f6
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-iio-dac-dpot-dac
@@ -0,0 +1,8 @@
+What:		/sys/bus/iio/devices/iio:deviceX/out_voltageY_raw_available
+Date:		October 2016
+KernelVersion:	4.9
+Contact:	Peter Rosin <peda@axentia.se>
+Description:
+		The range of available values represented as the minimum value,
+		the step and the maximum value, all enclosed in square brackets.
+		Example: [0 1 256]
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-light-isl29018 b/Documentation/ABI/testing/sysfs-bus-iio-light-isl29018
new file mode 100644
index 000000000000..f0ce0a0476ea
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-iio-light-isl29018
@@ -0,0 +1,19 @@
+What:		/sys/bus/iio/devices/iio:deviceX/proximity_on_chip_ambient_infrared_suppression
+Date:		January 2011
+KernelVersion:	2.6.37
+Contact:	linux-iio@vger.kernel.org
+Description:
+		From ISL29018 Data Sheet (FN6619.4, Oct 8, 2012) regarding the
+		infrared suppression:
+
+		Scheme 0, makes full n (4, 8, 12, 16) bits (unsigned) proximity
+		detection. The range of Scheme 0 proximity count is from 0 to
+		2^n. Logic 1 of this bit, Scheme 1, makes n-1 (3, 7, 11, 15)
+		bits (2's complementary) proximity_less_ambient detection. The
+		range of Scheme 1 proximity count is from -2^(n-1) to 2^(n-1).
+		The sign bit is extended for resolutions less than 16. While
+		Scheme 0 has wider dynamic range, Scheme 1 proximity detection
+		is less affected by the ambient IR noise variation.
+
+		0 Sensing IR from LED and ambient
+		1 Sensing IR from LED with ambient IR rejection
diff --git a/drivers/staging/iio/Documentation/sysfs-bus-iio-light-tsl2583 b/Documentation/ABI/testing/sysfs-bus-iio-light-tsl2583
index 660781df409f..a2e19964e87e 100644
--- a/drivers/staging/iio/Documentation/sysfs-bus-iio-light-tsl2583
+++ b/Documentation/ABI/testing/sysfs-bus-iio-light-tsl2583
@@ -1,18 +1,18 @@
-What:		/sys/bus/iio/devices/device[n]/lux_table
+What:		/sys/bus/iio/devices/device[n]/in_illuminance_calibrate
 KernelVersion:	2.6.37
 Contact:	linux-iio@vger.kernel.org
 Description:
-		This property gets/sets the table of coefficients
-		used in calculating illuminance in lux.
+		This property causes an internal calibration of the als gain trim
+		value which is later used in calculating illuminance in lux.
 
-What:		/sys/bus/iio/devices/device[n]/illuminance0_calibrate
+What:		/sys/bus/iio/devices/device[n]/in_illuminance_lux_table
 KernelVersion:	2.6.37
 Contact:	linux-iio@vger.kernel.org
 Description:
-		This property causes an internal calibration of the als gain trim
-		value which is later used in calculating illuminance in lux.
+		This property gets/sets the table of coefficients
+		used in calculating illuminance in lux.
 
-What:		/sys/bus/iio/devices/device[n]/illuminance0_input_target
+What:		/sys/bus/iio/devices/device[n]/in_illuminance_input_target
 KernelVersion:	2.6.37
 Contact:	linux-iio@vger.kernel.org
 Description:
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-potentiometer-mcp4531 b/Documentation/ABI/testing/sysfs-bus-iio-potentiometer-mcp4531
new file mode 100644
index 000000000000..2a91fbe394fc
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-iio-potentiometer-mcp4531
@@ -0,0 +1,8 @@
+What:		/sys/bus/iio/devices/iio:deviceX/out_resistance_raw_available
+Date:		October 2016
+KernelVersion:	4.9
+Contact:	Peter Rosin <peda@axentia.se>
+Description:
+		The range of available values represented as the minimum value,
+		the step and the maximum value, all enclosed in square brackets.
+		Example: [0 1 256]
diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci
index b3bc50f650ee..5a1732b78707 100644
--- a/Documentation/ABI/testing/sysfs-bus-pci
+++ b/Documentation/ABI/testing/sysfs-bus-pci
@@ -294,3 +294,10 @@ Description:
 		a firmware bug to the system vendor.  Writing to this file
 		taints the kernel with TAINT_FIRMWARE_WORKAROUND, which
 		reduces the supportability of your system.
+
+What:		/sys/bus/pci/devices/.../revision
+Date:		November 2016
+Contact:	Emil Velikov <emil.l.velikov@gmail.com>
+Description:
+		This file contains the revision field of the the PCI device.
+		The value comes from device config space. The file is read only.
diff --git a/Documentation/ABI/testing/sysfs-bus-vfio-mdev b/Documentation/ABI/testing/sysfs-bus-vfio-mdev
new file mode 100644
index 000000000000..452dbe39270e
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-vfio-mdev
@@ -0,0 +1,111 @@
+What:           /sys/.../<device>/mdev_supported_types/
+Date:           October 2016
+Contact:        Kirti Wankhede <kwankhede@nvidia.com>
+Description:
+                This directory contains list of directories of currently
+		supported mediated device types and their details for
+		<device>. Supported type attributes are defined by the
+		vendor driver who registers with Mediated device framework.
+		Each supported type is a directory whose name is created
+		by adding the device driver string as a prefix to the
+		string provided by the vendor driver.
+
+What:           /sys/.../<device>/mdev_supported_types/<type-id>/
+Date:           October 2016
+Contact:        Kirti Wankhede <kwankhede@nvidia.com>
+Description:
+                This directory gives details of supported type, like name,
+		description, available_instances, device_api etc.
+		'device_api' and 'available_instances' are mandatory
+		attributes to be provided by vendor driver. 'name',
+		'description' and other vendor driver specific attributes
+		are optional.
+
+What:           /sys/.../mdev_supported_types/<type-id>/create
+Date:           October 2016
+Contact:        Kirti Wankhede <kwankhede@nvidia.com>
+Description:
+		Writing UUID to this file will create mediated device of
+		type <type-id> for parent device <device>. This is a
+		write-only file.
+		For example:
+		# echo "83b8f4f2-509f-382f-3c1e-e6bfe0fa1001" >	\
+		       /sys/devices/foo/mdev_supported_types/foo-1/create
+
+What:           /sys/.../mdev_supported_types/<type-id>/devices/
+Date:           October 2016
+Contact:        Kirti Wankhede <kwankhede@nvidia.com>
+Description:
+		This directory contains symbolic links pointing to mdev
+		devices sysfs entries which are created of this <type-id>.
+
+What:           /sys/.../mdev_supported_types/<type-id>/available_instances
+Date:           October 2016
+Contact:        Kirti Wankhede <kwankhede@nvidia.com>
+Description:
+		Reading this attribute will show the number of mediated
+		devices of type <type-id> that can be created. This is a
+		readonly file.
+Users:
+		Userspace applications interested in creating mediated
+		device of that type. Userspace application should check
+		the number of available instances could be created before
+		creating mediated device of this type.
+
+What:           /sys/.../mdev_supported_types/<type-id>/device_api
+Date:           October 2016
+Contact:        Kirti Wankhede <kwankhede@nvidia.com>
+Description:
+		Reading this attribute will show VFIO device API supported
+		by this type. For example, "vfio-pci" for a PCI device,
+		"vfio-platform" for platform device.
+
+What:           /sys/.../mdev_supported_types/<type-id>/name
+Date:           October 2016
+Contact:        Kirti Wankhede <kwankhede@nvidia.com>
+Description:
+		Reading this attribute will show human readable name of the
+		mediated device that will get created of type <type-id>.
+		This is optional attribute. For example: "Grid M60-0Q"
+Users:
+		Userspace applications interested in knowing the name of
+		a particular <type-id> that can help in understanding the
+		type of mediated device.
+
+What:           /sys/.../mdev_supported_types/<type-id>/description
+Date:           October 2016
+Contact:        Kirti Wankhede <kwankhede@nvidia.com>
+Description:
+		Reading this attribute will show description of the type of
+		mediated device that will get created of type <type-id>.
+		This is optional attribute. For example:
+		"2 heads, 512M FB, 2560x1600 maximum resolution"
+Users:
+		Userspace applications interested in knowing the details of
+		a particular <type-id> that can help in understanding the
+		features provided by that type of mediated device.
+
+What:           /sys/.../<device>/<UUID>/
+Date:           October 2016
+Contact:        Kirti Wankhede <kwankhede@nvidia.com>
+Description:
+		This directory represents device directory of mediated
+		device. It contains all the attributes related to mediated
+		device.
+
+What:           /sys/.../<device>/<UUID>/mdev_type
+Date:           October 2016
+Contact:        Kirti Wankhede <kwankhede@nvidia.com>
+Description:
+		This is symbolic link pointing to supported type, <type-id>
+		directory of which this mediated device is created.
+
+What:           /sys/.../<device>/<UUID>/remove
+Date:           October 2016
+Contact:        Kirti Wankhede <kwankhede@nvidia.com>
+Description:
+		Writing '1' to this file destroys the mediated device. The
+		vendor driver can fail the remove() callback if that device
+		is active and the vendor driver doesn't support hot unplug.
+		Example:
+		# echo 1 > /sys/bus/mdev/devices/<UUID>/remove
diff --git a/Documentation/ABI/testing/sysfs-class-fpga-bridge b/Documentation/ABI/testing/sysfs-class-fpga-bridge
new file mode 100644
index 000000000000..312ae2c579d8
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-class-fpga-bridge
@@ -0,0 +1,11 @@
+What:		/sys/class/fpga_bridge/<bridge>/name
+Date:		January 2016
+KernelVersion:	4.5
+Contact:	Alan Tull <atull@opensource.altera.com>
+Description:	Name of low level FPGA bridge driver.
+
+What:		/sys/class/fpga_bridge/<bridge>/state
+Date:		January 2016
+KernelVersion:	4.5
+Contact:	Alan Tull <atull@opensource.altera.com>
+Description:	Show bridge state as "enabled" or "disabled"
diff --git a/Documentation/ABI/testing/sysfs-class-led b/Documentation/ABI/testing/sysfs-class-led
index 86ace287d48b..491cdeedc195 100644
--- a/Documentation/ABI/testing/sysfs-class-led
+++ b/Documentation/ABI/testing/sysfs-class-led
@@ -4,16 +4,24 @@ KernelVersion:	2.6.17
 Contact:	Richard Purdie <rpurdie@rpsys.net>
 Description:
 		Set the brightness of the LED. Most LEDs don't
-		have hardware brightness support so will just be turned on for
+		have hardware brightness support, so will just be turned on for
 		non-zero brightness settings. The value is between 0 and
 		/sys/class/leds/<led>/max_brightness.
 
+		Writing 0 to this file clears active trigger.
+
+		Writing non-zero to this file while trigger is active changes the
+		top brightness trigger is going to use.
+
 What:		/sys/class/leds/<led>/max_brightness
 Date:		March 2006
 KernelVersion:	2.6.17
 Contact:	Richard Purdie <rpurdie@rpsys.net>
 Description:
-		Maximum brightness level for this led, default is 255 (LED_FULL).
+		Maximum brightness level for this LED, default is 255 (LED_FULL).
+
+		If the LED does not support different brightness levels, this
+		should be 1.
 
 What:		/sys/class/leds/<led>/trigger
 Date:		March 2006
@@ -21,7 +29,7 @@ KernelVersion:	2.6.17
 Contact:	Richard Purdie <rpurdie@rpsys.net>
 Description:
 		Set the trigger for this LED. A trigger is a kernel based source
-		of led events.
+		of LED events.
 		You can change triggers in a similar manner to the way an IO
 		scheduler is chosen. Trigger specific parameters can appear in
 		/sys/class/leds/<led> once a given trigger is selected. For
diff --git a/Documentation/ABI/testing/sysfs-class-mei b/Documentation/ABI/testing/sysfs-class-mei
index 80d9888a8ece..5096a82f4cde 100644
--- a/Documentation/ABI/testing/sysfs-class-mei
+++ b/Documentation/ABI/testing/sysfs-class-mei
@@ -29,3 +29,19 @@ Description:	Display fw status registers content
 		Also number of registers varies between 1 and 6
 		depending on generation.
 
+What:		/sys/class/mei/meiN/hbm_ver
+Date:		Aug 2016
+KernelVersion:	4.9
+Contact:	Tomas Winkler <tomas.winkler@intel.com>
+Description:	Display the negotiated HBM protocol version.
+
+		The HBM protocol version negotiated
+		between the driver and the device.
+
+What:		/sys/class/mei/meiN/hbm_ver_drv
+Date:		Aug 2016
+KernelVersion:	4.9
+Contact:	Tomas Winkler <tomas.winkler@intel.com>
+Description:	Display the driver HBM protocol version.
+
+		The HBM protocol version supported by the driver.
diff --git a/Documentation/ABI/testing/sysfs-class-remoteproc b/Documentation/ABI/testing/sysfs-class-remoteproc
new file mode 100644
index 000000000000..d188afebc8ba
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-class-remoteproc
@@ -0,0 +1,50 @@
+What:		/sys/class/remoteproc/.../firmware
+Date:		October 2016
+Contact:	Matt Redfearn <matt.redfearn@imgtec.com>
+Description:	Remote processor firmware
+
+		Reports the name of the firmware currently loaded to the
+		remote processor.
+
+		To change the running firmware, ensure the remote processor is
+		stopped (using /sys/class/remoteproc/.../state) and write a new filename.
+
+What:		/sys/class/remoteproc/.../state
+Date:		October 2016
+Contact:	Matt Redfearn <matt.redfearn@imgtec.com>
+Description:	Remote processor state
+
+		Reports the state of the remote processor, which will be one of:
+
+		"offline"
+		"suspended"
+		"running"
+		"crashed"
+		"invalid"
+
+		"offline" means the remote processor is powered off.
+
+		"suspended" means that the remote processor is suspended and
+		must be woken to receive messages.
+
+		"running" is the normal state of an available remote processor
+
+		"crashed" indicates that a problem/crash has been detected on
+		the remote processor.
+
+		"invalid" is returned if the remote processor is in an
+		unknown state.
+
+		Writing this file controls the state of the remote processor.
+		The following states can be written:
+
+		"start"
+		"stop"
+
+		Writing "start" will attempt to start the processor running the
+		firmware indicated by, or written to,
+		/sys/class/remoteproc/.../firmware. The remote processor should
+		transition to "running" state.
+
+		Writing "stop" will attempt to halt the remote processor and
+		return it to the "offline" state.
diff --git a/Documentation/ABI/testing/sysfs-devices-deferred_probe b/Documentation/ABI/testing/sysfs-devices-deferred_probe
new file mode 100644
index 000000000000..58553d7a321f
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-devices-deferred_probe
@@ -0,0 +1,12 @@
+What:		/sys/devices/.../deferred_probe
+Date:		August 2016
+Contact:	Ben Hutchings <ben.hutchings@codethink.co.uk>
+Description:
+		The /sys/devices/.../deferred_probe attribute is
+		present for all devices.  If a driver detects during
+		probing a device that a related device is not yet
+		ready, it may defer probing of the first device.  The
+		kernel will retry probing the first device after any
+		other device is successfully probed.  This attribute
+		reads as 1 if probing of this device is currently
+		deferred, or 0 otherwise.
diff --git a/Documentation/ABI/testing/sysfs-platform-phy-rcar-gen3-usb2 b/Documentation/ABI/testing/sysfs-platform-phy-rcar-gen3-usb2
new file mode 100644
index 000000000000..6212697bbf6f
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-platform-phy-rcar-gen3-usb2
@@ -0,0 +1,15 @@
+What:		/sys/devices/platform/<phy-name>/role
+Date:		October 2016
+KernelVersion:	4.10
+Contact:	Yoshihiro Shimoda <yoshihiro.shimoda.uh@renesas.com>
+Description:
+		This file can be read and write.
+		The file can show/change the phy mode for role swap of usb.
+
+		Write the following strings to change the mode:
+		 "host" - switching mode from peripheral to host.
+		 "peripheral" - switching mode from host to peripheral.
+
+		Read the file, then it shows the following strings:
+		 "host" - The mode is host now.
+		 "peripheral" - The mode is peripheral now.
diff --git a/Documentation/ABI/testing/sysfs-platform-sst-atom b/Documentation/ABI/testing/sysfs-platform-sst-atom
new file mode 100644
index 000000000000..0d07c0395660
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-platform-sst-atom
@@ -0,0 +1,17 @@
+What:		/sys/devices/platform/8086%x:00/firmware_version
+Date:		November 2016
+KernelVersion:	4.10
+Contact:	"Sebastien Guiriec" <sebastien.guiriec@intel.com>
+Description:
+		LPE Firmware version for SST driver on all atom
+		plaforms (BYT/CHT/Merrifield/BSW).
+		If the FW has never been loaded it will display:
+			"FW not yet loaded"
+		If FW has been loaded it will display:
+			"v01.aa.bb.cc"
+		aa: Major version is reflecting SoC version:
+			0d: BYT FW
+			0b: BSW FW
+			07: Merrifield FW
+		bb: Minor version
+		cc: Build version
diff --git a/Documentation/ABI/testing/sysfs-power b/Documentation/ABI/testing/sysfs-power
index 50b368d490b5..f523e5a3ac33 100644
--- a/Documentation/ABI/testing/sysfs-power
+++ b/Documentation/ABI/testing/sysfs-power
@@ -7,30 +7,35 @@ Description:
 		subsystem.
 
 What:		/sys/power/state
-Date:		May 2014
+Date:		November 2016
 Contact:	Rafael J. Wysocki <rjw@rjwysocki.net>
 Description:
 		The /sys/power/state file controls system sleep states.
 		Reading from this file returns the available sleep state
-		labels, which may be "mem", "standby", "freeze" and "disk"
-		(hibernation).  The meanings of the first three labels depend on
-		the relative_sleep_states command line argument as follows:
-		 1) relative_sleep_states = 1
-		    "mem", "standby", "freeze" represent non-hibernation sleep
-		    states from the deepest ("mem", always present) to the
-		    shallowest ("freeze").  "standby" and "freeze" may or may
-		    not be present depending on the capabilities of the
-		    platform.  "freeze" can only be present if "standby" is
-		    present.
-		 2) relative_sleep_states = 0 (default)
-		    "mem" - "suspend-to-RAM", present if supported.
-		    "standby" - "power-on suspend", present if supported.
-		    "freeze" - "suspend-to-idle", always present.
-
-		Writing to this file one of these strings causes the system to
-		transition into the corresponding state, if available.  See
-		Documentation/power/states.txt for a description of what
-		"suspend-to-RAM", "power-on suspend" and "suspend-to-idle" mean.
+		labels, which may be "mem" (suspend), "standby" (power-on
+		suspend), "freeze" (suspend-to-idle) and "disk" (hibernation).
+
+		Writing one of the above strings to this file causes the system
+		to transition into the corresponding state, if available.
+
+		See Documentation/power/states.txt for more information.
+
+What:		/sys/power/mem_sleep
+Date:		November 2016
+Contact:	Rafael J. Wysocki <rjw@rjwysocki.net>
+Description:
+		The /sys/power/mem_sleep file controls the operating mode of
+		system suspend.  Reading from it returns the available modes
+		as "s2idle" (always present), "shallow" and "deep" (present if
+		supported).  The mode that will be used on subsequent attempts
+		to suspend the system (by writing "mem" to the /sys/power/state
+		file described above) is enclosed in square brackets.
+
+		Writing one of the above strings to this file causes the mode
+		represented by it to be used on subsequent attempts to suspend
+		the system.
+
+		See Documentation/power/states.txt for more information.
 
 What:		/sys/power/disk
 Date:		September 2006
diff --git a/Documentation/Changes b/Documentation/Changes
new file mode 120000
index 000000000000..7564ae1682ba
--- /dev/null
+++ b/Documentation/Changes
@@ -0,0 +1 @@
+process/changes.rst
\ No newline at end of file
diff --git a/Documentation/IPMI.txt b/Documentation/IPMI.txt
index c0d8788e75d3..72292308d0f5 100644
--- a/Documentation/IPMI.txt
+++ b/Documentation/IPMI.txt
@@ -111,6 +111,8 @@ ipmi_ssif - A driver for accessing BMCs on the SMBus. It uses the
 I2C kernel driver's SMBus interfaces to send and receive IPMI messages
 over the SMBus.
 
+ipmi_powernv - A driver for access BMCs on POWERNV systems.
+
 ipmi_watchdog - IPMI requires systems to have a very capable watchdog
 timer.  This driver implements the standard Linux watchdog timer
 interface on top of the IPMI message handler.
@@ -118,17 +120,15 @@ interface on top of the IPMI message handler.
 ipmi_poweroff - Some systems support the ability to be turned off via
 IPMI commands.
 
-These are all individually selectable via configuration options.
+bt-bmc - This is not part of the main driver, but instead a driver for
+accessing a BMC-side interface of a BT interface.  It is used on BMCs
+running Linux to provide an interface to the host.
 
-Note that the KCS-only interface has been removed.  The af_ipmi driver
-is no longer supported and has been removed because it was impossible
-to do 32 bit emulation on 64-bit kernels with it.
+These are all individually selectable via configuration options.
 
 Much documentation for the interface is in the include files.  The
 IPMI include files are:
 
-net/af_ipmi.h - Contains the socket interface.
-
 linux/ipmi.h - Contains the user interface and IOCTL interface for IPMI.
 
 linux/ipmi_smi.h - Contains the interface for system management interfaces
@@ -245,6 +245,16 @@ addressed (because some boards actually have multiple BMCs on them)
 and the user should not have to care what type of SMI is below them.
 
 
+Watching For Interfaces
+
+When your code comes up, the IPMI driver may or may not have detected
+if IPMI devices exist.  So you might have to defer your setup until
+the device is detected, or you might be able to do it immediately.
+To handle this, and to allow for discovery, you register an SMI
+watcher with ipmi_smi_watcher_register() to iterate over interfaces
+and tell you when they come and go.
+
+
 Creating the User
 
 To user the message handler, you must first create a user using
@@ -263,7 +273,7 @@ closing the device automatically destroys the user.
 
 Messaging
 
-To send a message from kernel-land, the ipmi_request() call does
+To send a message from kernel-land, the ipmi_request_settime() call does
 pretty much all message handling.  Most of the parameter are
 self-explanatory.  However, it takes a "msgid" parameter.  This is NOT
 the sequence number of messages.  It is simply a long value that is
@@ -352,11 +362,12 @@ that for more details.
 The SI Driver
 -------------
 
-The SI driver allows up to 4 KCS or SMIC interfaces to be configured
-in the system.  By default, scan the ACPI tables for interfaces, and
-if it doesn't find any the driver will attempt to register one KCS
-interface at the spec-specified I/O port 0xca2 without interrupts.
-You can change this at module load time (for a module) with:
+The SI driver allows KCS, BT, and SMIC interfaces to be configured
+in the system.  It discovers interfaces through a host of different
+methods, depending on the system.
+
+You can specify up to four interfaces on the module load line and
+control some module parameters:
 
   modprobe ipmi_si.o type=<type1>,<type2>....
        ports=<port1>,<port2>... addrs=<addr1>,<addr2>...
@@ -367,7 +378,7 @@ You can change this at module load time (for a module) with:
        force_kipmid=<enable1>,<enable2>,...
        kipmid_max_busy_us=<ustime1>,<ustime2>,...
        unload_when_empty=[0|1]
-       trydefaults=[0|1] trydmi=[0|1] tryacpi=[0|1]
+       trydmi=[0|1] tryacpi=[0|1]
        tryplatform=[0|1] trypci=[0|1]
 
 Each of these except try... items is a list, the first item for the
@@ -386,10 +397,6 @@ use the I/O port given as the device address.
 If you specify irqs as non-zero for an interface, the driver will
 attempt to use the given interrupt for the device.
 
-trydefaults sets whether the standard IPMI interface at 0xca2 and
-any interfaces specified by ACPE are tried.  By default, the driver
-tries it, set this value to zero to turn this off.
-
 The other try... items disable discovery by their corresponding
 names.  These are all enabled by default, set them to zero to disable
 them.  The tryplatform disables openfirmware.
@@ -434,7 +441,7 @@ kernel command line as:
 
   ipmi_si.type=<type1>,<type2>...
        ipmi_si.ports=<port1>,<port2>... ipmi_si.addrs=<addr1>,<addr2>...
-       ipmi_si.irqs=<irq1>,<irq2>... ipmi_si.trydefaults=[0|1]
+       ipmi_si.irqs=<irq1>,<irq2>...
        ipmi_si.regspacings=<sp1>,<sp2>,...
        ipmi_si.regsizes=<size1>,<size2>,...
        ipmi_si.regshifts=<shift1>,<shift2>,...
@@ -444,11 +451,6 @@ kernel command line as:
 
 It works the same as the module parameters of the same names.
 
-By default, the driver will attempt to detect any device specified by
-ACPI, and if none of those then a KCS device at the spec-specified
-0xca2.  If you want to turn this off, set the "trydefaults" option to
-false.
-
 If your IPMI interface does not support interrupts and is a KCS or
 SMIC interface, the IPMI driver will start a kernel thread for the
 interface to help speed things up.  This is a low-priority kernel
@@ -500,7 +502,8 @@ at module load time (for a module) with:
 	addr=<i2caddr1>[,<i2caddr2>[,...]]
 	adapter=<adapter1>[,<adapter2>[...]]
 	dbg=<flags1>,<flags2>...
-        slave_addrs=<addr1>,<addr2>,...
+	slave_addrs=<addr1>,<addr2>,...
+	tryacpi=[0|1] trydmi=[0|1]
 	[dbg_probe=1]
 
 The addresses are normal I2C addresses.  The adapter is the string
@@ -513,6 +516,9 @@ spaces in kernel parameters.
 The debug flags are bit flags for each BMC found, they are:
 IPMI messages: 1, driver state: 2, timing: 4, I2C probe: 8
 
+The tryxxx parameters can be used to disable detecting interfaces
+from various sources.
+
 Setting dbg_probe to 1 will enable debugging of the probing and
 detection process for BMCs on the SMBusses.
 
@@ -535,7 +541,8 @@ kernel command line as:
 	ipmi_ssif.adapter=<adapter1>[,<adapter2>[...]]
 	ipmi_ssif.dbg=<flags1>[,<flags2>[...]]
 	ipmi_ssif.dbg_probe=1
-        ipmi_ssif.slave_addrs=<addr1>[,<addr2>[...]]
+	ipmi_ssif.slave_addrs=<addr1>[,<addr2>[...]]
+	ipmi_ssif.tryacpi=[0|1] ipmi_ssif.trydmi=[0|1]
 
 These are the same options as on the module command line.
 
diff --git a/Documentation/acpi/DSD-properties-rules.txt b/Documentation/acpi/DSD-properties-rules.txt
new file mode 100644
index 000000000000..3e4862bdad98
--- /dev/null
+++ b/Documentation/acpi/DSD-properties-rules.txt
@@ -0,0 +1,97 @@
+_DSD Device Properties Usage Rules
+----------------------------------
+
+Properties, Property Sets and Property Subsets
+----------------------------------------------
+
+The _DSD (Device Specific Data) configuration object, introduced in ACPI 5.1,
+allows any type of device configuration data to be provided via the ACPI
+namespace.  In principle, the format of the data may be arbitrary, but it has to
+be identified by a UUID which must be recognized by the driver processing the
+_DSD output.  However, there are generic UUIDs defined for _DSD recognized by
+the ACPI subsystem in the Linux kernel which automatically processes the data
+packages associated with them and makes those data available to device drivers
+as "device properties".
+
+A device property is a data item consisting of a string key and a value (of a
+specific type) associated with it.
+
+In the ACPI _DSD context it is an element of the sub-package following the
+generic Device Properties UUID in the _DSD return package as specified in the
+Device Properties UUID definition document [1].
+
+It also may be regarded as the definition of a key and the associated data type
+that can be returned by _DSD in the Device Properties UUID sub-package for a
+given device.
+
+A property set is a collection of properties applicable to a hardware entity
+like a device.  In the ACPI _DSD context it is the set of all properties that
+can be returned in the Device Properties UUID sub-package for the device in
+question.
+
+Property subsets are nested collections of properties.  Each of them is
+associated with an additional key (name) allowing the subset to be referred
+to as a whole (and to be treated as a separate entity).  The canonical
+representation of property subsets is via the mechanism specified in the
+Hierarchical Properties Extension UUID definition document [2].
+
+Property sets may be hierarchical.  That is, a property set may contain
+multiple property subsets that each may contain property subsets of its
+own and so on.
+
+General Validity Rule for Property Sets
+---------------------------------------
+
+Valid property sets must follow the guidance given by the Device Properties UUID
+definition document [1].
+
+_DSD properties are intended to be used in addition to, and not instead of, the
+existing mechanisms defined by the ACPI specification.  Therefore, as a rule,
+they should only be used if the ACPI specification does not make direct
+provisions for handling the underlying use case.  It generally is invalid to
+return property sets which do not follow that rule from _DSD in data packages
+associated with the Device Properties UUID.
+
+Additional Considerations
+-------------------------
+
+There are cases in which, even if the general rule given above is followed in
+principle, the property set may still not be regarded as a valid one.
+
+For example, that applies to device properties which may cause kernel code
+(either a device driver or a library/subsystem) to access hardware in a way
+possibly leading to a conflict with AML methods in the ACPI namespace.  In
+particular, that may happen if the kernel code uses device properties to
+manipulate hardware normally controlled by ACPI methods related to power
+management, like _PSx and _DSW (for device objects) or _ON and _OFF (for power
+resource objects), or by ACPI device disabling/enabling methods, like _DIS and
+_SRS.
+
+In all cases in which kernel code may do something that will confuse AML as a
+result of using device properties, the device properties in question are not
+suitable for the ACPI environment and consequently they cannot belong to a valid
+property set.
+
+Property Sets and Device Tree Bindings
+--------------------------------------
+
+It often is useful to make _DSD return property sets that follow Device Tree
+bindings.
+
+In those cases, however, the above validity considerations must be taken into
+account in the first place and returning invalid property sets from _DSD must be
+avoided.  For this reason, it may not be possible to make _DSD return a property
+set following the given DT binding literally and completely.  Still, for the
+sake of code re-use, it may make sense to provide as much of the configuration
+data as possible in the form of device properties and complement that with an
+ACPI-specific mechanism suitable for the use case at hand.
+
+In any case, property sets following DT bindings literally should not be
+expected to automatically work in the ACPI environment regardless of their
+contents.
+
+References
+----------
+
+[1] http://www.uefi.org/sites/default/files/resources/_DSD-device-properties-UUID.pdf
+[2] http://www.uefi.org/sites/default/files/resources/_DSD-hierarchical-data-extension-UUID-v1.1.pdf
diff --git a/Documentation/acpi/enumeration.txt b/Documentation/acpi/enumeration.txt
index a91ec5af52df..209a5eba6b87 100644
--- a/Documentation/acpi/enumeration.txt
+++ b/Documentation/acpi/enumeration.txt
@@ -415,3 +415,12 @@ the "compatible" property in the _DSD or a _CID as long as one of their
 ancestors provides a _DSD with a valid "compatible" property.  Such device
 objects are then simply regarded as additional "blocks" providing hierarchical
 configuration information to the driver of the composite ancestor device.
+
+However, PRP0001 can only be returned from either _HID or _CID of a device
+object if all of the properties returned by the _DSD associated with it (either
+the _DSD of the device object itself or the _DSD of its ancestor in the
+"composite device" case described above) can be used in the ACPI environment.
+Otherwise, the _DSD itself is regarded as invalid and therefore the "compatible"
+property returned by it is meaningless.
+
+Refer to DSD-properties-rules.txt for more information.
diff --git a/Documentation/acpi/gpio-properties.txt b/Documentation/acpi/gpio-properties.txt
index 5aafe0b351a1..2aff0349facd 100644
--- a/Documentation/acpi/gpio-properties.txt
+++ b/Documentation/acpi/gpio-properties.txt
@@ -51,6 +51,68 @@ it to 1 marks the GPIO as active low.
 In our Bluetooth example the "reset-gpios" refers to the second GpioIo()
 resource, second pin in that resource with the GPIO number of 31.
 
+It is possible to leave holes in the array of GPIOs. This is useful in
+cases like with SPI host controllers where some chip selects may be
+implemented as GPIOs and some as native signals. For example a SPI host
+controller can have chip selects 0 and 2 implemented as GPIOs and 1 as
+native:
+
+  Package () {
+      "cs-gpios",
+      Package () {
+          ^GPIO, 19, 0, 0, // chip select 0: GPIO
+          0,               // chip select 1: native signal
+          ^GPIO, 20, 0, 0, // chip select 2: GPIO
+      }
+  }
+
+Other supported properties
+--------------------------
+
+Following Device Tree compatible device properties are also supported by
+_DSD device properties for GPIO controllers:
+
+- gpio-hog
+- output-high
+- output-low
+- input
+- line-name
+
+Example:
+
+  Name (_DSD, Package () {
+      // _DSD Hierarchical Properties Extension UUID
+      ToUUID("dbb8e3e6-5886-4ba6-8795-1319f52a966b"),
+      Package () {
+          Package () {"hog-gpio8", "G8PU"}
+      }
+  })
+
+  Name (G8PU, Package () {
+      ToUUID("daffd814-6eba-4d8c-8a91-bc9bbf4aa301"),
+      Package () {
+          Package () {"gpio-hog", 1},
+          Package () {"gpios", Package () {8, 0}},
+          Package () {"output-high", 1},
+          Package () {"line-name", "gpio8-pullup"},
+      }
+  })
+
+- gpio-line-names
+
+Example:
+
+  Package () {
+      "gpio-line-names",
+      Package () {
+          "SPI0_CS_N", "EXP2_INT", "MUX6_IO", "UART0_RXD", "MUX7_IO",
+          "LVL_C_A1", "MUX0_IO", "SPI1_MISO"
+      }
+  }
+
+See Documentation/devicetree/bindings/gpio/gpio.txt for more information
+about these properties.
+
 ACPI GPIO Mappings Provided by Drivers
 --------------------------------------
 
diff --git a/Documentation/acpi/osi.txt b/Documentation/acpi/osi.txt
new file mode 100644
index 000000000000..50cde0ceb9b0
--- /dev/null
+++ b/Documentation/acpi/osi.txt
@@ -0,0 +1,187 @@
+ACPI _OSI and _REV methods
+--------------------------
+
+An ACPI BIOS can use the "Operating System Interfaces" method (_OSI)
+to find out what the operating system supports. Eg. If BIOS
+AML code includes _OSI("XYZ"), the kernel's AML interpreter
+can evaluate that method, look to see if it supports 'XYZ'
+and answer YES or NO to the BIOS.
+
+The ACPI _REV method returns the "Revision of the ACPI specification
+that OSPM supports"
+
+This document explains how and why the BIOS and Linux should use these methods.
+It also explains how and why they are widely misused.
+
+How to use _OSI
+---------------
+
+Linux runs on two groups of machines -- those that are tested by the OEM
+to be compatible with Linux, and those that were never tested with Linux,
+but where Linux was installed to replace the original OS (Windows or OSX).
+
+The larger group is the systems tested to run only Windows.  Not only that,
+but many were tested to run with just one specific version of Windows.
+So even though the BIOS may use _OSI to query what version of Windows is running,
+only a single path through the BIOS has actually been tested.
+Experience shows that taking untested paths through the BIOS
+exposes Linux to an entire category of BIOS bugs.
+For this reason, Linux _OSI defaults must continue to claim compatibility
+with all versions of Windows.
+
+But Linux isn't actually compatible with Windows, and the Linux community
+has also been hurt with regressions when Linux adds the latest version of
+Windows to its list of _OSI strings.  So it is possible that additional strings
+will be more thoroughly vetted before shipping upstream in the future.
+But it is likely that they will all eventually be added.
+
+What should an OEM do if they want to support Linux and Windows
+using the same BIOS image?  Often they need to do something different
+for Linux to deal with how Linux is different from Windows.
+Here the BIOS should ask exactly what it wants to know:
+
+_OSI("Linux-OEM-my_interface_name")
+where 'OEM' is needed if this is an OEM-specific hook,
+and 'my_interface_name' describes the hook, which could be a
+quirk, a bug, or a bug-fix.
+
+In addition, the OEM should send a patch to upstream Linux
+via the linux-acpi@vger.kernel.org mailing list.  When that patch
+is checked into Linux, the OS will answer "YES" when the BIOS
+on the OEM's system uses _OSI to ask if the interface is supported
+by the OS.  Linux distributors can back-port that patch for Linux
+pre-installs, and it will be included by all distributions that
+re-base to upstream.  If the distribution can not update the kernel binary,
+they can also add an acpi_osi=Linux-OEM-my_interface_name
+cmdline parameter to the boot loader, as needed.
+
+If the string refers to a feature where the upstream kernel
+eventually grows support, a patch should be sent to remove
+the string when that support is added to the kernel.
+
+That was easy.  Read on, to find out how to do it wrong.
+
+Before _OSI, there was _OS
+--------------------------
+
+ACPI 1.0 specified "_OS" as an
+"object that evaluates to a string that identifies the operating system."
+
+The ACPI BIOS flow would include an evaluation of _OS, and the AML
+interpreter in the kernel would return to it a string identifying the OS:
+
+Windows 98, SE: "Microsoft Windows"
+Windows ME: "Microsoft WindowsME:Millenium Edition"
+Windows NT: "Microsoft Windows NT"
+
+The idea was on a platform tasked with running multiple OS's,
+the BIOS could use _OS to enable devices that an OS
+might support, or enable quirks or bug workarounds
+necessary to make the platform compatible with that pre-existing OS.
+
+But _OS had fundamental problems.  First, the BIOS needed to know the name
+of every possible version of the OS that would run on it, and needed to know
+all the quirks of those OS's.  Certainly it would make more sense
+for the BIOS to ask *specific* things of the OS, such
+"do you support a specific interface", and thus in ACPI 3.0,
+_OSI was born to replace _OS.
+
+_OS was abandoned, though even today, many BIOS look for
+_OS "Microsoft Windows NT", though it seems somewhat far-fetched
+that anybody would install those old operating systems
+over what came with the machine.
+
+Linux answers "Microsoft Windows NT" to please that BIOS idiom.
+That is the *only* viable strategy, as that is what modern Windows does,
+and so doing otherwise could steer the BIOS down an untested path.
+
+_OSI is born, and immediately misused
+--------------------------------------
+
+With _OSI, the *BIOS* provides the string describing an interface,
+and asks the OS: "YES/NO, are you compatible with this interface?"
+
+eg. _OSI("3.0 Thermal Model") would return TRUE if the OS knows how
+to deal with the thermal extensions made to the ACPI 3.0 specification.
+An old OS that doesn't know about those extensions would answer FALSE,
+and a new OS may be able to return TRUE.
+
+For an OS-specific interface, the ACPI spec said that the BIOS and the OS
+were to agree on a string of the form such as "Windows-interface_name".
+
+But two bad things happened.  First, the Windows ecosystem used _OSI
+not as designed, but as a direct replacement for _OS -- identifying
+the OS version, rather than an OS supported interface.  Indeed, right
+from the start, the ACPI 3.0 spec itself codified this misuse
+in example code using _OSI("Windows 2001").
+
+This misuse was adopted and continues today.
+
+Linux had no choice but to also return TRUE to _OSI("Windows 2001")
+and its successors.  To do otherwise would virtually guarantee breaking
+a BIOS that has been tested only with that _OSI returning TRUE.
+
+This strategy is problematic, as Linux is never completely compatible with
+the latest version of Windows, and sometimes it takes more than a year
+to iron out incompatibilities.
+
+Not to be out-done, the Linux community made things worse by returning TRUE
+to _OSI("Linux").  Doing so is even worse than the Windows misuse
+of _OSI, as "Linux" does not even contain any version information.
+_OSI("Linux") led to some BIOS' malfunctioning due to BIOS writer's
+using it in untested BIOS flows.  But some OEM's used _OSI("Linux")
+in tested flows to support real Linux features.  In 2009, Linux
+removed _OSI("Linux"), and added a cmdline parameter to restore it
+for legacy systems still needed it.  Further a BIOS_BUG warning prints
+for all BIOS's that invoke it.
+
+No BIOS should use _OSI("Linux").
+
+The result is a strategy for Linux to maximize compatibility with
+ACPI BIOS that are tested on Windows machines.  There is a real risk
+of over-stating that compatibility; but the alternative has often been
+catastrophic failure resulting from the BIOS taking paths that
+were never validated under *any* OS.
+
+Do not use _REV
+---------------
+
+Since _OSI("Linux") went away, some BIOS writers used _REV
+to support Linux and Windows differences in the same BIOS.
+
+_REV was defined in ACPI 1.0 to return the version of ACPI
+supported by the OS and the OS AML interpreter.
+
+Modern Windows returns _REV = 2.  Linux used ACPI_CA_SUPPORT_LEVEL,
+which would increment, based on the version of the spec supported.
+
+Unfortunately, _REV was also misused.  eg. some BIOS would check
+for _REV = 3, and do something for Linux, but when Linux returned
+_REV = 4, that support broke.
+
+In response to this problem, Linux returns _REV = 2 always,
+from mid-2015 onward.  The ACPI specification will also be updated
+to reflect that _REV is deprecated, and always returns 2.
+
+Apple Mac and _OSI("Darwin")
+----------------------------
+
+On Apple's Mac platforms, the ACPI BIOS invokes _OSI("Darwin")
+to determine if the machine is running Apple OSX.
+
+Like Linux's _OSI("*Windows*") strategy, Linux defaults to
+answering YES to _OSI("Darwin") to enable full access
+to the hardware and validated BIOS paths seen by OSX.
+Just like on Windows-tested platforms, this strategy has risks.
+
+Starting in Linux-3.18, the kernel answered YES to _OSI("Darwin")
+for the purpose of enabling Mac Thunderbolt support.  Further,
+if the kernel noticed _OSI("Darwin") being invoked, it additionally
+disabled all _OSI("*Windows*") to keep poorly written Mac BIOS
+from going down untested combinations of paths.
+
+The Linux-3.18 change in default caused power regressions on Mac
+laptops, and the 3.18 implementation did not allow changing
+the default via cmdline "acpi_osi=!Darwin".  Linux-4.7 fixed
+the ability to use acpi_osi=!Darwin as a workaround, and
+we hope to see Mac Thunderbolt power management support in Linux-4.11.
diff --git a/Documentation/admin-guide/index.rst b/Documentation/admin-guide/index.rst
index 2681cbd24cdd..8ddae4e4299a 100644
--- a/Documentation/admin-guide/index.rst
+++ b/Documentation/admin-guide/index.rst
@@ -59,6 +59,7 @@ configure specific aspects of kernel behavior to your liking.
    binfmt-misc
    mono
    java
+   ras
 
 .. only::  subproject and html
 
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 62d68b2056de..be2d6d0a03a4 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -1560,6 +1560,12 @@
 		       disable
 		         Do not enable intel_pstate as the default
 		         scaling driver for the supported processors
+		       passive
+			 Use intel_pstate as a scaling driver, but configure it
+			 to work with generic cpufreq governors (instead of
+			 enabling its internal governor).  This mode cannot be
+			 used along with the hardware-managed P-states (HWP)
+			 feature.
 		       force
 			 Enable intel_pstate on systems that prohibit it by default
 			 in favor of acpi-cpufreq. Forcing the intel_pstate driver
@@ -1580,6 +1586,9 @@
 			Description Table, specifies preferred power management
 			profile as "Enterprise Server" or "Performance Server",
 			then this feature is turned on by default.
+		per_cpu_perf_limits
+			Allow per-logical-CPU P-State performance control limits using
+			cpufreq sysfs interface
 
 	intremap=	[X86-64, Intel-IOMMU]
 			on	enable Interrupt Remapping (default)
@@ -2122,6 +2131,12 @@
 			memory contents and reserves bad memory
 			regions that are detected.
 
+	mem_sleep_default=	[SUSPEND] Default system suspend mode:
+			s2idle  - Suspend-To-Idle
+			shallow - Power-On Suspend or equivalent (if supported)
+			deep    - Suspend-To-RAM or equivalent (if supported)
+			See Documentation/power/states.txt.
+
 	meye.*=		[HW] Set MotionEye Camera parameters
 			See Documentation/video4linux/meye.txt.
 
@@ -3475,13 +3490,6 @@
 			[KNL, SMP] Set scheduler's default relax_domain_level.
 			See Documentation/cgroup-v1/cpusets.txt.
 
-	relative_sleep_states=
-			[SUSPEND] Use sleep state labeling where the deepest
-			state available other than hibernation is always "mem".
-			Format: { "0" | "1" }
-			0 -- Traditional sleep state labels.
-			1 -- Relative sleep state labels.
-
 	reserve=	[KNL,BUGS] Force the kernel to ignore some iomem area
 
 	reservetop=	[X86-32]
diff --git a/Documentation/admin-guide/ras.rst b/Documentation/admin-guide/ras.rst
new file mode 100644
index 000000000000..d71340e86c27
--- /dev/null
+++ b/Documentation/admin-guide/ras.rst
@@ -0,0 +1,1190 @@
+.. include:: <isonum.txt>
+
+============================================
+Reliability, Availability and Serviceability
+============================================
+
+RAS concepts
+************
+
+Reliability, Availability and Serviceability (RAS) is a concept used on
+servers meant to measure their robusteness.
+
+Reliability
+  is the probability that a system will produce correct outputs.
+
+  * Generally measured as Mean Time Between Failures (MTBF)
+  * Enhanced by features that help to avoid, detect and repair hardware faults
+
+Availability
+  is the probability that a system is operational at a given time
+
+  * Generally measured as a percentage of downtime per a period of time
+  * Often uses mechanisms to detect and correct hardware faults in
+    runtime;
+
+Serviceability (or maintainability)
+  is the simplicity and speed with which a system can be repaired or
+  maintained
+
+  * Generally measured on Mean Time Between Repair (MTBR)
+
+Improving RAS
+-------------
+
+In order to reduce systems downtime, a system should be capable of detecting
+hardware errors, and, when possible correcting them in runtime. It should
+also provide mechanisms to detect hardware degradation, in order to warn
+the system administrator to take the action of replacing a component before
+it causes data loss or system downtime.
+
+Among the monitoring measures, the most usual ones include:
+
+* CPU – detect errors at instruction execution and at L1/L2/L3 caches;
+* Memory – add error correction logic (ECC) to detect and correct errors;
+* I/O – add CRC checksums for tranfered data;
+* Storage – RAID, journal file systems, checksums,
+  Self-Monitoring, Analysis and Reporting Technology (SMART).
+
+By monitoring the number of occurrences of error detections, it is possible
+to identify if the probability of hardware errors is increasing, and, on such
+case, do a preventive maintainance to replace a degrated component while
+those errors are correctable.
+
+Types of errors
+---------------
+
+Most mechanisms used on modern systems use use technologies like Hamming
+Codes that allow error correction when the number of errors on a bit packet
+is below a threshold. If the number of errors is above, those mechanisms
+can indicate with a high degree of confidence that an error happened, but
+they can't correct.
+
+Also, sometimes an error occur on a component that it is not used. For
+example, a part of the memory that it is not currently allocated.
+
+That defines some categories of errors:
+
+* **Correctable Error (CE)** - the error detection mechanism detected and
+  corrected the error. Such errors are usually not fatal, although some
+  Kernel mechanisms allow the system administrator to consider them as fatal.
+
+* **Uncorrected Error (UE)** - the amount of errors happened above the error
+  correction threshold, and the system was unable to auto-correct.
+
+* **Fatal Error** - when an UE error happens on a critical component of the
+  system (for example, a piece of the Kernel got corrupted by an UE), the
+  only reliable way to avoid data corruption is to hang or reboot the machine.
+
+* **Non-fatal Error** - when an UE error happens on an unused component,
+  like a CPU in power down state or an unused memory bank, the system may
+  still run, eventually replacing the affected hardware by a hot spare,
+  if available.
+
+  Also, when an error happens on an userspace process, it is also possible to
+  kill such process and let userspace restart it.
+
+The mechanism for handling non-fatal errors is usually complex and may
+require the help of some userspace application, in order to apply the
+policy desired by the system administrator.
+
+Identifying a bad hardware component
+------------------------------------
+
+Just detecting a hardware flaw is usually not enough, as the system needs
+to pinpoint to the minimal replaceable unit (MRU) that should be exchanged
+to make the hardware reliable again.
+
+So, it requires not only error logging facilities, but also mechanisms that
+will translate the error message to the silkscreen or component label for
+the MRU.
+
+Typically, it is very complex for memory, as modern CPUs interlace memory
+from different memory modules, in order to provide a better performance. The
+DMI BIOS usually have a list of memory module labels, with can be obtained
+using the ``dmidecode`` tool. For example, on a desktop machine, it shows::
+
+	Memory Device
+		Total Width: 64 bits
+		Data Width: 64 bits
+		Size: 16384 MB
+		Form Factor: SODIMM
+		Set: None
+		Locator: ChannelA-DIMM0
+		Bank Locator: BANK 0
+		Type: DDR4
+		Type Detail: Synchronous
+		Speed: 2133 MHz
+		Rank: 2
+		Configured Clock Speed: 2133 MHz
+
+On the above example, a DDR4 SO-DIMM memory module is located at the
+system's memory labeled as "BANK 0", as given by the *bank locator* field.
+Please notice that, on such system, the *total width* is equal to the
+*data witdh*. It means that such memory module doesn't have error
+detection/correction mechanisms.
+
+Unfortunately, not all systems use the same field to specify the memory
+bank. On this example, from an older server, ``dmidecode`` shows::
+
+	Memory Device
+		Array Handle: 0x1000
+		Error Information Handle: Not Provided
+		Total Width: 72 bits
+		Data Width: 64 bits
+		Size: 8192 MB
+		Form Factor: DIMM
+		Set: 1
+		Locator: DIMM_A1
+		Bank Locator: Not Specified
+		Type: DDR3
+		Type Detail: Synchronous Registered (Buffered)
+		Speed: 1600 MHz
+		Rank: 2
+		Configured Clock Speed: 1600 MHz
+
+There, the DDR3 RDIMM memory module is located at the system's memory labeled
+as "DIMM_A1", as given by the *locator* field. Please notice that this
+memory module has 64 bits of *data witdh* and 72 bits of *total width*. So,
+it has 8 extra bits to be used by error detection and correction mechanisms.
+Such kind of memory is called Error-correcting code memory (ECC memory).
+
+To make things even worse, it is not uncommon that systems with different
+labels on their system's board to use exactly the same BIOS, meaning that
+the labels provided by the BIOS won't match the real ones.
+
+ECC memory
+----------
+
+As mentioned on the previous section, ECC memory has extra bits to be
+used for error correction. So, on 64 bit systems, a memory module
+has 64 bits of *data width*, and 74 bits of *total width*. So, there are
+8 bits extra bits to be used for the error detection and correction
+mechanisms. Those extra bits are called *syndrome*\ [#f1]_\ [#f2]_.
+
+So, when the cpu requests the memory controller to write a word with
+*data width*, the memory controller calculates the *syndrome* in real time,
+using Hamming code, or some other error correction code, like SECDED+,
+producing a code with *total width* size. Such code is then written
+on the memory modules.
+
+At read, the *total width* bits code is converted back, using the same
+ECC code used on write, producing a word with *data width* and a *syndrome*.
+The word with *data width* is sent to the CPU, even when errors happen.
+
+The memory controller also looks at the *syndrome* in order to check if
+there was an error, and if the ECC code was able to fix such error.
+If the error was corrected, a Corrected Error (CE) happened. If not, an
+Uncorrected Error (UE) happened.
+
+The information about the CE/UE errors is stored on some special registers
+at the memory controller and can be accessed by reading such registers,
+either by BIOS, by some special CPUs or by Linux EDAC driver. On x86 64
+bit CPUs, such errors can also be retrieved via the Machine Check
+Architecture (MCA)\ [#f3]_.
+
+.. [#f1] Please notice that several memory controllers allow operation on a
+  mode called "Lock-Step", where it groups two memory modules together,
+  doing 128-bit reads/writes. That gives 16 bits for error correction, with
+  significatively improves the error correction mechanism, at the expense
+  that, when an error happens, there's no way to know what memory module is
+  to blame. So, it has to blame both memory modules.
+
+.. [#f2] Some memory controllers also allow using memory in mirror mode.
+  On such mode, the same data is written to two memory modules. At read,
+  the system checks both memory modules, in order to check if both provide
+  identical data. On such configuration, when an error happens, there's no
+  way to know what memory module is to blame. So, it has to blame both
+  memory modules (or 4 memory modules, if the system is also on Lock-step
+  mode).
+
+.. [#f3] For more details about the Machine Check Architecture (MCA),
+  please read Documentation/x86/x86_64/machinecheck at the Kernel tree.
+
+EDAC - Error Detection And Correction
+*************************************
+
+.. note::
+
+   "bluesmoke" was the name for this device driver subsystem when it
+   was "out-of-tree" and maintained at http://bluesmoke.sourceforge.net.
+   That site is mostly archaic now and can be used only for historical
+   purposes.
+
+   When the subsystem was pushed upstream for the first time, on
+   Kernel 2.6.16, for the first time, it was renamed to ``EDAC``.
+
+Purpose
+-------
+
+The ``edac`` kernel module's goal is to detect and report hardware errors
+that occur within the computer system running under linux.
+
+Memory
+------
+
+Memory Correctable Errors (CE) and Uncorrectable Errors (UE) are the
+primary errors being harvested. These types of errors are harvested by
+the ``edac_mc`` device.
+
+Detecting CE events, then harvesting those events and reporting them,
+**can** but must not necessarily be a predictor of future UE events. With
+CE events only, the system can and will continue to operate as no data
+has been damaged yet.
+
+However, preventive maintenance and proactive part replacement of memory
+modules exhibiting CEs can reduce the likelihood of the dreaded UE events
+and system panics.
+
+Other hardware elements
+-----------------------
+
+A new feature for EDAC, the ``edac_device`` class of device, was added in
+the 2.6.23 version of the kernel.
+
+This new device type allows for non-memory type of ECC hardware detectors
+to have their states harvested and presented to userspace via the sysfs
+interface.
+
+Some architectures have ECC detectors for L1, L2 and L3 caches,
+along with DMA engines, fabric switches, main data path switches,
+interconnections, and various other hardware data paths. If the hardware
+reports it, then a edac_device device probably can be constructed to
+harvest and present that to userspace.
+
+
+PCI bus scanning
+----------------
+
+In addition, PCI devices are scanned for PCI Bus Parity and SERR Errors
+in order to determine if errors are occurring during data transfers.
+
+The presence of PCI Parity errors must be examined with a grain of salt.
+There are several add-in adapters that do **not** follow the PCI specification
+with regards to Parity generation and reporting. The specification says
+the vendor should tie the parity status bits to 0 if they do not intend
+to generate parity.  Some vendors do not do this, and thus the parity bit
+can "float" giving false positives.
+
+There is a PCI device attribute located in sysfs that is checked by
+the EDAC PCI scanning code. If that attribute is set, PCI parity/error
+scanning is skipped for that device. The attribute is::
+
+	broken_parity_status
+
+and is located in ``/sys/devices/pci<XXX>/0000:XX:YY.Z`` directories for
+PCI devices.
+
+
+Versioning
+----------
+
+EDAC is composed of a "core" module (``edac_core.ko``) and several Memory
+Controller (MC) driver modules. On a given system, the CORE is loaded
+and one MC driver will be loaded. Both the CORE and the MC driver (or
+``edac_device`` driver) have individual versions that reflect current
+release level of their respective modules.
+
+Thus, to "report" on what version a system is running, one must report
+both the CORE's and the MC driver's versions.
+
+
+Loading
+-------
+
+If ``edac`` was statically linked with the kernel then no loading
+is necessary. If ``edac`` was built as modules then simply modprobe
+the ``edac`` pieces that you need. You should be able to modprobe
+hardware-specific modules and have the dependencies load the necessary
+core modules.
+
+Example::
+
+	$ modprobe amd76x_edac
+
+loads both the ``amd76x_edac.ko`` memory controller module and the
+``edac_mc.ko`` core module.
+
+
+Sysfs interface
+---------------
+
+EDAC presents a ``sysfs`` interface for control and reporting purposes. It
+lives in the /sys/devices/system/edac directory.
+
+Within this directory there currently reside 2 components:
+
+	======= ==============================
+	mc	memory controller(s) system
+	pci	PCI control and status system
+	======= ==============================
+
+
+
+Memory Controller (mc) Model
+----------------------------
+
+Each ``mc`` device controls a set of memory modules [#f4]_. These modules
+are laid out in a Chip-Select Row (``csrowX``) and Channel table (``chX``).
+There can be multiple csrows and multiple channels.
+
+.. [#f4] Nowadays, the term DIMM (Dual In-line Memory Module) is widely
+  used to refer to a memory module, although there are other memory
+  packaging alternatives, like SO-DIMM, SIMM, etc. Along this document,
+  and inside the EDAC system, the term "dimm" is used for all memory
+  modules, even when they use a different kind of packaging.
+
+Memory controllers allow for several csrows, with 8 csrows being a
+typical value. Yet, the actual number of csrows depends on the layout of
+a given motherboard, memory controller and memory module characteristics.
+
+Dual channels allow for dual data length (e. g. 128 bits, on 64 bit systems)
+data transfers to/from the CPU from/to memory. Some newer chipsets allow
+for more than 2 channels, like Fully Buffered DIMMs (FB-DIMMs) memory
+controllers. The following example will assume 2 channels:
+
+	+------------+-----------------------+
+	| Chip       |       Channels        |
+	| Select     +-----------+-----------+
+	| rows       |  ``ch0``  |  ``ch1``  |
+	+============+===========+===========+
+	| ``csrow0`` |  DIMM_A0  |  DIMM_B0  |
+	+------------+           |           |
+	| ``csrow1`` |           |           |
+	+------------+-----------+-----------+
+	| ``csrow2`` |  DIMM_A1  | DIMM_B1   |
+	+------------+           |           |
+	| ``csrow3`` |           |           |
+	+------------+-----------+-----------+
+
+In the above example, there are 4 physical slots on the motherboard
+for memory DIMMs:
+
+	+---------+---------+
+	| DIMM_A0 | DIMM_B0 |
+	+---------+---------+
+	| DIMM_A1 | DIMM_B1 |
+	+---------+---------+
+
+Labels for these slots are usually silk-screened on the motherboard.
+Slots labeled ``A`` are channel 0 in this example. Slots labeled ``B`` are
+channel 1. Notice that there are two csrows possible on a physical DIMM.
+These csrows are allocated their csrow assignment based on the slot into
+which the memory DIMM is placed. Thus, when 1 DIMM is placed in each
+Channel, the csrows cross both DIMMs.
+
+Memory DIMMs come single or dual "ranked". A rank is a populated csrow.
+Thus, 2 single ranked DIMMs, placed in slots DIMM_A0 and DIMM_B0 above
+will have just one csrow (csrow0). csrow1 will be empty. On the other
+hand, when 2 dual ranked DIMMs are similarly placed, then both csrow0
+and csrow1 will be populated. The pattern repeats itself for csrow2 and
+csrow3.
+
+The representation of the above is reflected in the directory
+tree in EDAC's sysfs interface. Starting in directory
+``/sys/devices/system/edac/mc``, each memory controller will be
+represented by its own ``mcX`` directory, where ``X`` is the
+index of the MC::
+
+	..../edac/mc/
+		   |
+		   |->mc0
+		   |->mc1
+		   |->mc2
+		   ....
+
+Under each ``mcX`` directory each ``csrowX`` is again represented by a
+``csrowX``, where ``X`` is the csrow index::
+
+	.../mc/mc0/
+		|
+		|->csrow0
+		|->csrow2
+		|->csrow3
+		....
+
+Notice that there is no csrow1, which indicates that csrow0 is composed
+of a single ranked DIMMs. This should also apply in both Channels, in
+order to have dual-channel mode be operational. Since both csrow2 and
+csrow3 are populated, this indicates a dual ranked set of DIMMs for
+channels 0 and 1.
+
+Within each of the ``mcX`` and ``csrowX`` directories are several EDAC
+control and attribute files.
+
+``mcX`` directories
+-------------------
+
+In ``mcX`` directories are EDAC control and attribute files for
+this ``X`` instance of the memory controllers.
+
+For a description of the sysfs API, please see:
+
+	Documentation/ABI/testing/sysfs-devices-edac
+
+
+``dimmX`` or ``rankX`` directories
+----------------------------------
+
+The recommended way to use the EDAC subsystem is to look at the information
+provided by the ``dimmX`` or ``rankX`` directories [#f5]_.
+
+A typical EDAC system has the following structure under
+``/sys/devices/system/edac/``\ [#f6]_::
+
+	/sys/devices/system/edac/
+	├── mc
+	│   ├── mc0
+	│   │   ├── ce_count
+	│   │   ├── ce_noinfo_count
+	│   │   ├── dimm0
+	│   │   │   ├── dimm_dev_type
+	│   │   │   ├── dimm_edac_mode
+	│   │   │   ├── dimm_label
+	│   │   │   ├── dimm_location
+	│   │   │   ├── dimm_mem_type
+	│   │   │   ├── size
+	│   │   │   └── uevent
+	│   │   ├── max_location
+	│   │   ├── mc_name
+	│   │   ├── reset_counters
+	│   │   ├── seconds_since_reset
+	│   │   ├── size_mb
+	│   │   ├── ue_count
+	│   │   ├── ue_noinfo_count
+	│   │   └── uevent
+	│   ├── mc1
+	│   │   ├── ce_count
+	│   │   ├── ce_noinfo_count
+	│   │   ├── dimm0
+	│   │   │   ├── dimm_dev_type
+	│   │   │   ├── dimm_edac_mode
+	│   │   │   ├── dimm_label
+	│   │   │   ├── dimm_location
+	│   │   │   ├── dimm_mem_type
+	│   │   │   ├── size
+	│   │   │   └── uevent
+	│   │   ├── max_location
+	│   │   ├── mc_name
+	│   │   ├── reset_counters
+	│   │   ├── seconds_since_reset
+	│   │   ├── size_mb
+	│   │   ├── ue_count
+	│   │   ├── ue_noinfo_count
+	│   │   └── uevent
+	│   └── uevent
+	└── uevent
+
+In the ``dimmX`` directories are EDAC control and attribute files for
+this ``X`` memory module:
+
+- ``size`` - Total memory managed by this csrow attribute file
+
+	This attribute file displays, in count of megabytes, the memory
+	that this csrow contains.
+
+- ``dimm_dev_type``  - Device type attribute file
+
+	This attribute file will display what type of DRAM device is
+	being utilized on this DIMM.
+	Examples:
+
+		- x1
+		- x2
+		- x4
+		- x8
+
+- ``dimm_edac_mode`` - EDAC Mode of operation attribute file
+
+	This attribute file will display what type of Error detection
+	and correction is being utilized.
+
+- ``dimm_label`` - memory module label control file
+
+	This control file allows this DIMM to have a label assigned
+	to it. With this label in the module, when errors occur
+	the output can provide the DIMM label in the system log.
+	This becomes vital for panic events to isolate the
+	cause of the UE event.
+
+	DIMM Labels must be assigned after booting, with information
+	that correctly identifies the physical slot with its
+	silk screen label. This information is currently very
+	motherboard specific and determination of this information
+	must occur in userland at this time.
+
+- ``dimm_location`` - location of the memory module
+
+	The location can have up to 3 levels, and describe how the
+	memory controller identifies the location of a memory module.
+	Depending on the type of memory and memory controller, it
+	can be:
+
+		- *csrow* and *channel* - used when the memory controller
+		  doesn't identify a single DIMM - e. g. in ``rankX`` dir;
+		- *branch*, *channel*, *slot* - typically used on FB-DIMM memory
+		  controllers;
+		- *channel*, *slot* - used on Nehalem and newer Intel drivers.
+
+- ``dimm_mem_type`` - Memory Type attribute file
+
+	This attribute file will display what type of memory is currently
+	on this csrow. Normally, either buffered or unbuffered memory.
+	Examples:
+
+		- Registered-DDR
+		- Unbuffered-DDR
+
+.. [#f5] On some systems, the memory controller doesn't have any logic
+  to identify the memory module. On such systems, the directory is called ``rankX`` and works on a similar way as the ``csrowX`` directories.
+  On modern Intel memory controllers, the memory controller identifies the
+  memory modules directly. On such systems, the directory is called ``dimmX``.
+
+.. [#f6] There are also some ``power`` directories and ``subsystem``
+  symlinks inside the sysfs mapping that are automatically created by
+  the sysfs subsystem. Currently, they serve no purpose.
+
+``csrowX`` directories
+----------------------
+
+When CONFIG_EDAC_LEGACY_SYSFS is enabled, sysfs will contain the ``csrowX``
+directories. As this API doesn't work properly for Rambus, FB-DIMMs and
+modern Intel Memory Controllers, this is being deprecated in favor of
+``dimmX`` directories.
+
+In the ``csrowX`` directories are EDAC control and attribute files for
+this ``X`` instance of csrow:
+
+
+- ``ue_count`` - Total Uncorrectable Errors count attribute file
+
+	This attribute file displays the total count of uncorrectable
+	errors that have occurred on this csrow. If panic_on_ue is set
+	this counter will not have a chance to increment, since EDAC
+	will panic the system.
+
+
+- ``ce_count`` - Total Correctable Errors count attribute file
+
+	This attribute file displays the total count of correctable
+	errors that have occurred on this csrow. This count is very
+	important to examine. CEs provide early indications that a
+	DIMM is beginning to fail. This count field should be
+	monitored for non-zero values and report such information
+	to the system administrator.
+
+
+- ``size_mb`` - Total memory managed by this csrow attribute file
+
+	This attribute file displays, in count of megabytes, the memory
+	that this csrow contains.
+
+
+- ``mem_type`` - Memory Type attribute file
+
+	This attribute file will display what type of memory is currently
+	on this csrow. Normally, either buffered or unbuffered memory.
+	Examples:
+
+		- Registered-DDR
+		- Unbuffered-DDR
+
+
+- ``edac_mode`` - EDAC Mode of operation attribute file
+
+	This attribute file will display what type of Error detection
+	and correction is being utilized.
+
+
+- ``dev_type`` - Device type attribute file
+
+	This attribute file will display what type of DRAM device is
+	being utilized on this DIMM.
+	Examples:
+
+		- x1
+		- x2
+		- x4
+		- x8
+
+
+- ``ch0_ce_count`` - Channel 0 CE Count attribute file
+
+	This attribute file will display the count of CEs on this
+	DIMM located in channel 0.
+
+
+- ``ch0_ue_count`` - Channel 0 UE Count attribute file
+
+	This attribute file will display the count of UEs on this
+	DIMM located in channel 0.
+
+
+- ``ch0_dimm_label`` - Channel 0 DIMM Label control file
+
+
+	This control file allows this DIMM to have a label assigned
+	to it. With this label in the module, when errors occur
+	the output can provide the DIMM label in the system log.
+	This becomes vital for panic events to isolate the
+	cause of the UE event.
+
+	DIMM Labels must be assigned after booting, with information
+	that correctly identifies the physical slot with its
+	silk screen label. This information is currently very
+	motherboard specific and determination of this information
+	must occur in userland at this time.
+
+
+- ``ch1_ce_count`` - Channel 1 CE Count attribute file
+
+
+	This attribute file will display the count of CEs on this
+	DIMM located in channel 1.
+
+
+- ``ch1_ue_count`` - Channel 1 UE Count attribute file
+
+
+	This attribute file will display the count of UEs on this
+	DIMM located in channel 0.
+
+
+- ``ch1_dimm_label`` - Channel 1 DIMM Label control file
+
+	This control file allows this DIMM to have a label assigned
+	to it. With this label in the module, when errors occur
+	the output can provide the DIMM label in the system log.
+	This becomes vital for panic events to isolate the
+	cause of the UE event.
+
+	DIMM Labels must be assigned after booting, with information
+	that correctly identifies the physical slot with its
+	silk screen label. This information is currently very
+	motherboard specific and determination of this information
+	must occur in userland at this time.
+
+
+System Logging
+--------------
+
+If logging for UEs and CEs is enabled, then system logs will contain
+information indicating that errors have been detected::
+
+  EDAC MC0: CE page 0x283, offset 0xce0, grain 8, syndrome 0x6ec3, row 0, channel 1 "DIMM_B1": amd76x_edac
+  EDAC MC0: CE page 0x1e5, offset 0xfb0, grain 8, syndrome 0xb741, row 0, channel 1 "DIMM_B1": amd76x_edac
+
+
+The structure of the message is:
+
+	+---------------------------------------+-------------+
+	| Content                               + Example     |
+	+=======================================+=============+
+	| The memory controller                 | MC0         |
+	+---------------------------------------+-------------+
+	| Error type                            | CE          |
+	+---------------------------------------+-------------+
+	| Memory page                           | 0x283       |
+	+---------------------------------------+-------------+
+	| Offset in the page                    | 0xce0       |
+	+---------------------------------------+-------------+
+	| The byte granularity                  | grain 8     |
+	| or resolution of the error            |             |
+	+---------------------------------------+-------------+
+	| The error syndrome                    | 0xb741      |
+	+---------------------------------------+-------------+
+	| Memory row                            | row 0       +
+	+---------------------------------------+-------------+
+	| Memory channel                        | channel 1   |
+	+---------------------------------------+-------------+
+	| DIMM label, if set prior              | DIMM B1     |
+	+---------------------------------------+-------------+
+	| And then an optional, driver-specific |             |
+	| message that may have additional      |             |
+	| information.                          |             |
+	+---------------------------------------+-------------+
+
+Both UEs and CEs with no info will lack all but memory controller, error
+type, a notice of "no info" and then an optional, driver-specific error
+message.
+
+
+PCI Bus Parity Detection
+------------------------
+
+On Header Type 00 devices, the primary status is looked at for any
+parity error regardless of whether parity is enabled on the device or
+not. (The spec indicates parity is generated in some cases). On Header
+Type 01 bridges, the secondary status register is also looked at to see
+if parity occurred on the bus on the other side of the bridge.
+
+
+Sysfs configuration
+-------------------
+
+Under ``/sys/devices/system/edac/pci`` are control and attribute files as
+follows:
+
+
+- ``check_pci_parity`` - Enable/Disable PCI Parity checking control file
+
+	This control file enables or disables the PCI Bus Parity scanning
+	operation. Writing a 1 to this file enables the scanning. Writing
+	a 0 to this file disables the scanning.
+
+	Enable::
+
+		echo "1" >/sys/devices/system/edac/pci/check_pci_parity
+
+	Disable::
+
+		echo "0" >/sys/devices/system/edac/pci/check_pci_parity
+
+
+- ``pci_parity_count`` - Parity Count
+
+	This attribute file will display the number of parity errors that
+	have been detected.
+
+
+Module parameters
+-----------------
+
+- ``edac_mc_panic_on_ue`` - Panic on UE control file
+
+	An uncorrectable error will cause a machine panic.  This is usually
+	desirable.  It is a bad idea to continue when an uncorrectable error
+	occurs - it is indeterminate what was uncorrected and the operating
+	system context might be so mangled that continuing will lead to further
+	corruption. If the kernel has MCE configured, then EDAC will never
+	notice the UE.
+
+	LOAD TIME::
+
+		module/kernel parameter: edac_mc_panic_on_ue=[0|1]
+
+	RUN TIME::
+
+		echo "1" > /sys/module/edac_core/parameters/edac_mc_panic_on_ue
+
+
+- ``edac_mc_log_ue`` - Log UE control file
+
+
+	Generate kernel messages describing uncorrectable errors.  These errors
+	are reported through the system message log system.  UE statistics
+	will be accumulated even when UE logging is disabled.
+
+	LOAD TIME::
+
+		module/kernel parameter: edac_mc_log_ue=[0|1]
+
+	RUN TIME::
+
+		echo "1" > /sys/module/edac_core/parameters/edac_mc_log_ue
+
+
+- ``edac_mc_log_ce`` - Log CE control file
+
+
+	Generate kernel messages describing correctable errors.  These
+	errors are reported through the system message log system.
+	CE statistics will be accumulated even when CE logging is disabled.
+
+	LOAD TIME::
+
+		module/kernel parameter: edac_mc_log_ce=[0|1]
+
+	RUN TIME::
+
+		echo "1" > /sys/module/edac_core/parameters/edac_mc_log_ce
+
+
+- ``edac_mc_poll_msec`` - Polling period control file
+
+
+	The time period, in milliseconds, for polling for error information.
+	Too small a value wastes resources.  Too large a value might delay
+	necessary handling of errors and might loose valuable information for
+	locating the error.  1000 milliseconds (once each second) is the current
+	default. Systems which require all the bandwidth they can get, may
+	increase this.
+
+	LOAD TIME::
+
+		module/kernel parameter: edac_mc_poll_msec=[0|1]
+
+	RUN TIME::
+
+		echo "1000" > /sys/module/edac_core/parameters/edac_mc_poll_msec
+
+
+- ``panic_on_pci_parity`` - Panic on PCI PARITY Error
+
+
+	This control file enables or disables panicking when a parity
+	error has been detected.
+
+
+	module/kernel parameter::
+
+			edac_panic_on_pci_pe=[0|1]
+
+	Enable::
+
+		echo "1" > /sys/module/edac_core/parameters/edac_panic_on_pci_pe
+
+	Disable::
+
+		echo "0" > /sys/module/edac_core/parameters/edac_panic_on_pci_pe
+
+
+
+EDAC device type
+----------------
+
+In the header file, edac_pci.h, there is a series of edac_device structures
+and APIs for the EDAC_DEVICE.
+
+User space access to an edac_device is through the sysfs interface.
+
+At the location ``/sys/devices/system/edac`` (sysfs) new edac_device devices
+will appear.
+
+There is a three level tree beneath the above ``edac`` directory. For example,
+the ``test_device_edac`` device (found at the http://bluesmoke.sourceforget.net
+website) installs itself as::
+
+	/sys/devices/system/edac/test-instance
+
+in this directory are various controls, a symlink and one or more ``instance``
+directories.
+
+The standard default controls are:
+
+	==============	=======================================================
+	log_ce		boolean to log CE events
+	log_ue		boolean to log UE events
+	panic_on_ue	boolean to ``panic`` the system if an UE is encountered
+			(default off, can be set true via startup script)
+	poll_msec	time period between POLL cycles for events
+	==============	=======================================================
+
+The test_device_edac device adds at least one of its own custom control:
+
+	==============	==================================================
+	test_bits	which in the current test driver does nothing but
+			show how it is installed. A ported driver can
+			add one or more such controls and/or attributes
+			for specific uses.
+			One out-of-tree driver uses controls here to allow
+			for ERROR INJECTION operations to hardware
+			injection registers
+	==============	==================================================
+
+The symlink points to the 'struct dev' that is registered for this edac_device.
+
+Instances
+---------
+
+One or more instance directories are present. For the ``test_device_edac``
+case:
+
+	+----------------+
+	| test-instance0 |
+	+----------------+
+
+
+In this directory there are two default counter attributes, which are totals of
+counter in deeper subdirectories.
+
+	==============	====================================
+	ce_count	total of CE events of subdirectories
+	ue_count	total of UE events of subdirectories
+	==============	====================================
+
+Blocks
+------
+
+At the lowest directory level is the ``block`` directory. There can be 0, 1
+or more blocks specified in each instance:
+
+	+-------------+
+	| test-block0 |
+	+-------------+
+
+In this directory the default attributes are:
+
+	==============	================================================
+	ce_count	which is counter of CE events for this ``block``
+			of hardware being monitored
+	ue_count	which is counter of UE events for this ``block``
+			of hardware being monitored
+	==============	================================================
+
+
+The ``test_device_edac`` device adds 4 attributes and 1 control:
+
+	================== ====================================================
+	test-block-bits-0	for every POLL cycle this counter
+				is incremented
+	test-block-bits-1	every 10 cycles, this counter is bumped once,
+				and test-block-bits-0 is set to 0
+	test-block-bits-2	every 100 cycles, this counter is bumped once,
+				and test-block-bits-1 is set to 0
+	test-block-bits-3	every 1000 cycles, this counter is bumped once,
+				and test-block-bits-2 is set to 0
+	================== ====================================================
+
+
+	================== ====================================================
+	reset-counters		writing ANY thing to this control will
+				reset all the above counters.
+	================== ====================================================
+
+
+Use of the ``test_device_edac`` driver should enable any others to create their own
+unique drivers for their hardware systems.
+
+The ``test_device_edac`` sample driver is located at the
+http://bluesmoke.sourceforge.net project site for EDAC.
+
+
+Usage of EDAC APIs on Nehalem and newer Intel CPUs
+--------------------------------------------------
+
+On older Intel architectures, the memory controller was part of the North
+Bridge chipset. Nehalem, Sandy Bridge, Ivy Bridge, Haswell, Sky Lake and
+newer Intel architectures integrated an enhanced version of the memory
+controller (MC) inside the CPUs.
+
+This chapter will cover the differences of the enhanced memory controllers
+found on newer Intel CPUs, such as ``i7core_edac``, ``sb_edac`` and
+``sbx_edac`` drivers.
+
+.. note::
+
+   The Xeon E7 processor families use a separate chip for the memory
+   controller, called Intel Scalable Memory Buffer. This section doesn't
+   apply for such families.
+
+1) There is one Memory Controller per Quick Patch Interconnect
+   (QPI). At the driver, the term "socket" means one QPI. This is
+   associated with a physical CPU socket.
+
+   Each MC have 3 physical read channels, 3 physical write channels and
+   3 logic channels. The driver currently sees it as just 3 channels.
+   Each channel can have up to 3 DIMMs.
+
+   The minimum known unity is DIMMs. There are no information about csrows.
+   As EDAC API maps the minimum unity is csrows, the driver sequentially
+   maps channel/DIMM into different csrows.
+
+   For example, supposing the following layout::
+
+	Ch0 phy rd0, wr0 (0x063f4031): 2 ranks, UDIMMs
+	  dimm 0 1024 Mb offset: 0, bank: 8, rank: 1, row: 0x4000, col: 0x400
+	  dimm 1 1024 Mb offset: 4, bank: 8, rank: 1, row: 0x4000, col: 0x400
+        Ch1 phy rd1, wr1 (0x063f4031): 2 ranks, UDIMMs
+	  dimm 0 1024 Mb offset: 0, bank: 8, rank: 1, row: 0x4000, col: 0x400
+	Ch2 phy rd3, wr3 (0x063f4031): 2 ranks, UDIMMs
+	  dimm 0 1024 Mb offset: 0, bank: 8, rank: 1, row: 0x4000, col: 0x400
+
+   The driver will map it as::
+
+	csrow0: channel 0, dimm0
+	csrow1: channel 0, dimm1
+	csrow2: channel 1, dimm0
+	csrow3: channel 2, dimm0
+
+   exports one DIMM per csrow.
+
+   Each QPI is exported as a different memory controller.
+
+2) The MC has the ability to inject errors to test drivers. The drivers
+   implement this functionality via some error injection nodes:
+
+   For injecting a memory error, there are some sysfs nodes, under
+   ``/sys/devices/system/edac/mc/mc?/``:
+
+   - ``inject_addrmatch/*``:
+      Controls the error injection mask register. It is possible to specify
+      several characteristics of the address to match an error code::
+
+         dimm = the affected dimm. Numbers are relative to a channel;
+         rank = the memory rank;
+         channel = the channel that will generate an error;
+         bank = the affected bank;
+         page = the page address;
+         column (or col) = the address column.
+
+      each of the above values can be set to "any" to match any valid value.
+
+      At driver init, all values are set to any.
+
+      For example, to generate an error at rank 1 of dimm 2, for any channel,
+      any bank, any page, any column::
+
+		echo 2 >/sys/devices/system/edac/mc/mc0/inject_addrmatch/dimm
+		echo 1 >/sys/devices/system/edac/mc/mc0/inject_addrmatch/rank
+
+	To return to the default behaviour of matching any, you can do::
+
+		echo any >/sys/devices/system/edac/mc/mc0/inject_addrmatch/dimm
+		echo any >/sys/devices/system/edac/mc/mc0/inject_addrmatch/rank
+
+   - ``inject_eccmask``:
+          specifies what bits will have troubles,
+
+   - ``inject_section``:
+       specifies what ECC cache section will get the error::
+
+		3 for both
+		2 for the highest
+		1 for the lowest
+
+   - ``inject_type``:
+       specifies the type of error, being a combination of the following bits::
+
+		bit 0 - repeat
+		bit 1 - ecc
+		bit 2 - parity
+
+   - ``inject_enable``:
+       starts the error generation when something different than 0 is written.
+
+   All inject vars can be read. root permission is needed for write.
+
+   Datasheet states that the error will only be generated after a write on an
+   address that matches inject_addrmatch. It seems, however, that reading will
+   also produce an error.
+
+   For example, the following code will generate an error for any write access
+   at socket 0, on any DIMM/address on channel 2::
+
+	echo 2 >/sys/devices/system/edac/mc/mc0/inject_addrmatch/channel
+	echo 2 >/sys/devices/system/edac/mc/mc0/inject_type
+	echo 64 >/sys/devices/system/edac/mc/mc0/inject_eccmask
+	echo 3 >/sys/devices/system/edac/mc/mc0/inject_section
+	echo 1 >/sys/devices/system/edac/mc/mc0/inject_enable
+	dd if=/dev/mem of=/dev/null seek=16k bs=4k count=1 >& /dev/null
+
+   For socket 1, it is needed to replace "mc0" by "mc1" at the above
+   commands.
+
+   The generated error message will look like::
+
+	EDAC MC0: UE row 0, channel-a= 0 channel-b= 0 labels "-": NON_FATAL (addr = 0x0075b980, socket=0, Dimm=0, Channel=2, syndrome=0x00000040, count=1, Err=8c0000400001009f:4000080482 (read error: read ECC error))
+
+3) Corrected Error memory register counters
+
+   Those newer MCs have some registers to count memory errors. The driver
+   uses those registers to report Corrected Errors on devices with Registered
+   DIMMs.
+
+   However, those counters don't work with Unregistered DIMM. As the chipset
+   offers some counters that also work with UDIMMs (but with a worse level of
+   granularity than the default ones), the driver exposes those registers for
+   UDIMM memories.
+
+   They can be read by looking at the contents of ``all_channel_counts/``::
+
+     $ for i in /sys/devices/system/edac/mc/mc0/all_channel_counts/*; do echo $i; cat $i; done
+	/sys/devices/system/edac/mc/mc0/all_channel_counts/udimm0
+	0
+	/sys/devices/system/edac/mc/mc0/all_channel_counts/udimm1
+	0
+	/sys/devices/system/edac/mc/mc0/all_channel_counts/udimm2
+	0
+
+   What happens here is that errors on different csrows, but at the same
+   dimm number will increment the same counter.
+   So, in this memory mapping::
+
+	csrow0: channel 0, dimm0
+	csrow1: channel 0, dimm1
+	csrow2: channel 1, dimm0
+	csrow3: channel 2, dimm0
+
+   The hardware will increment udimm0 for an error at the first dimm at either
+   csrow0, csrow2  or csrow3;
+
+   The hardware will increment udimm1 for an error at the second dimm at either
+   csrow0, csrow2  or csrow3;
+
+   The hardware will increment udimm2 for an error at the third dimm at either
+   csrow0, csrow2  or csrow3;
+
+4) Standard error counters
+
+   The standard error counters are generated when an mcelog error is received
+   by the driver. Since, with UDIMM, this is counted by software, it is
+   possible that some errors could be lost. With RDIMM's, they display the
+   contents of the registers
+
+Reference documents used on ``amd64_edac``
+------------------------------------------
+
+``amd64_edac`` module is based on the following documents
+(available from http://support.amd.com/en-us/search/tech-docs):
+
+1. :Title:  BIOS and Kernel Developer's Guide for AMD Athlon 64 and AMD
+	   Opteron Processors
+   :AMD publication #: 26094
+   :Revision: 3.26
+   :Link: http://support.amd.com/TechDocs/26094.PDF
+
+2. :Title:  BIOS and Kernel Developer's Guide for AMD NPT Family 0Fh
+	   Processors
+   :AMD publication #: 32559
+   :Revision: 3.00
+   :Issue Date: May 2006
+   :Link: http://support.amd.com/TechDocs/32559.pdf
+
+3. :Title:  BIOS and Kernel Developer's Guide (BKDG) For AMD Family 10h
+	   Processors
+   :AMD publication #: 31116
+   :Revision: 3.00
+   :Issue Date: September 07, 2007
+   :Link: http://support.amd.com/TechDocs/31116.pdf
+
+4. :Title: BIOS and Kernel Developer's Guide (BKDG) for AMD Family 15h
+	  Models 30h-3Fh Processors
+   :AMD publication #: 49125
+   :Revision: 3.06
+   :Issue Date: 2/12/2015 (latest release)
+   :Link: http://support.amd.com/TechDocs/49125_15h_Models_30h-3Fh_BKDG.pdf
+
+5. :Title: BIOS and Kernel Developer's Guide (BKDG) for AMD Family 15h
+	  Models 60h-6Fh Processors
+   :AMD publication #: 50742
+   :Revision: 3.01
+   :Issue Date: 7/23/2015 (latest release)
+   :Link: http://support.amd.com/TechDocs/50742_15h_Models_60h-6Fh_BKDG.pdf
+
+6. :Title: BIOS and Kernel Developer's Guide (BKDG) for AMD Family 16h
+	  Models 00h-0Fh Processors
+   :AMD publication #: 48751
+   :Revision: 3.03
+   :Issue Date: 2/23/2015 (latest release)
+   :Link: http://support.amd.com/TechDocs/48751_16h_bkdg.pdf
+
+Credits
+=======
+
+* Written by Doug Thompson <dougthompson@xmission.com>
+
+  - 7 Dec 2005
+  - 17 Jul 2007	Updated
+
+* |copy| Mauro Carvalho Chehab
+
+  - 05 Aug 2009	Nehalem interface
+  - 26 Oct 2016 Converted to ReST and cleanups at the Nehalem section
+
+* EDAC authors/maintainers:
+
+  - Doug Thompson, Dave Jiang, Dave Peterson et al,
+  - Mauro Carvalho Chehab
+  - Borislav Petkov
+  - original author: Thayne Harbaugh
diff --git a/Documentation/admin-guide/vga-softcursor.rst b/Documentation/admin-guide/vga-softcursor.rst
index a663a745cff4..f52175457e60 100644
--- a/Documentation/admin-guide/vga-softcursor.rst
+++ b/Documentation/admin-guide/vga-softcursor.rst
@@ -4,15 +4,13 @@ Software cursor for VGA
 by Pavel Machek <pavel@atrey.karlin.mff.cuni.cz>
 and Martin Mares <mj@atrey.karlin.mff.cuni.cz>
 
-Linux now has some ability to manipulate cursor appearance. Normally, you
-can set the size of hardware cursor (and also work around some ugly bugs in
-those miserable Trident cards [#f1]_. You can now play a few new tricks:
-you can make your cursor look
-
-like a non-blinking red block, make it inverse background of the character it's
-over or to highlight that character and still choose whether the original
-hardware cursor should remain visible or not.  There may be other things I have
-never thought of.
+Linux now has some ability to manipulate cursor appearance.  Normally,
+you can set the size of hardware cursor.  You can now play a few new
+tricks: you can make your cursor look like a non-blinking red block,
+make it inverse background of the character it's over or to highlight
+that character and still choose whether the original hardware cursor
+should remain visible or not.  There may be other things I have never
+thought of.
 
 The cursor appearance is controlled by a ``<ESC>[?1;2;3c`` escape sequence
 where 1, 2 and 3 are parameters described below. If you omit any of them,
@@ -48,8 +46,6 @@ third parameter
 	Bit setting takes place before bit toggling, so you can simply clear a
 	bit by including it in both the set mask and the toggle mask.
 
-.. [#f1] see ``#define TRIDENT_GLITCH`` in ``drivers/video/vgacon.c``.
-
 Examples
 --------
 
diff --git a/Documentation/arm/stm32/overview.txt b/Documentation/arm/stm32/overview.txt
index 09aed5588d7c..a03b0357c017 100644
--- a/Documentation/arm/stm32/overview.txt
+++ b/Documentation/arm/stm32/overview.txt
@@ -5,7 +5,8 @@ Introduction
 ------------
 
   The STMicroelectronics family of Cortex-M based MCUs are supported by the
-  'STM32' platform of ARM Linux. Currently only the STM32F429 is supported.
+  'STM32' platform of ARM Linux. Currently only the STM32F429 (Cortex-M4)
+  and STM32F746 (Cortex-M7) are supported.
 
 
 Configuration
diff --git a/Documentation/arm/stm32/stm32f746-overview.txt b/Documentation/arm/stm32/stm32f746-overview.txt
new file mode 100644
index 000000000000..cffd2b1ccd6f
--- /dev/null
+++ b/Documentation/arm/stm32/stm32f746-overview.txt
@@ -0,0 +1,34 @@
+			STM32F746 Overview
+			==================
+
+  Introduction
+  ------------
+	The STM32F746 is a Cortex-M7 MCU aimed at various applications.
+	It features:
+	- Cortex-M7 core running up to @216MHz
+	- 1MB internal flash, 320KBytes internal RAM (+4KB of backup SRAM)
+	- FMC controller to connect SDRAM, NOR and NAND memories
+	- Dual mode QSPI
+	- SD/MMC/SDIO support
+	- Ethernet controller
+	- USB OTFG FS & HS controllers
+	- I2C, SPI, CAN busses support
+	- Several 16 & 32 bits general purpose timers
+	- Serial Audio interface
+	- LCD controller
+	- HDMI-CEC
+	- SPDIFRX
+
+  Resources
+  ---------
+	Datasheet and reference manual are publicly available on ST website:
+	- http://www.st.com/content/st_com/en/products/microcontrollers/stm32-32-bit-arm-cortex-mcus/stm32f7-series/stm32f7x6/stm32f746ng.html
+
+  Document Author
+  ---------------
+	Alexandre Torgue <alexandre.torgue@st.com>
+
+
+
+
+
diff --git a/Documentation/block/biodoc.txt b/Documentation/block/biodoc.txt
index 918e1e0d0e78..01ddeaf64b0f 100644
--- a/Documentation/block/biodoc.txt
+++ b/Documentation/block/biodoc.txt
@@ -348,7 +348,7 @@ Drivers can now specify a request prepare function (q->prep_rq_fn) that the
 block layer would invoke to pre-build device commands for a given request,
 or perform other preparatory processing for the request. This is routine is
 called by elv_next_request(), i.e. typically just before servicing a request.
-(The prepare function would not be called for requests that have REQ_DONTPREP
+(The prepare function would not be called for requests that have RQF_DONTPREP
 enabled)
 
 Aside:
@@ -553,8 +553,8 @@ struct request {
 	struct request_list *rl;
 }
 	
-See the rq_flag_bits definitions for an explanation of the various flags
-available. Some bits are used by the block layer or i/o scheduler.
+See the req_ops and req_flag_bits definitions for an explanation of the various
+flags available. Some bits are used by the block layer or i/o scheduler.
 	
 The behaviour of the various sector counts are almost the same as before,
 except that since we have multi-segment bios, current_nr_sectors refers
diff --git a/Documentation/block/cfq-iosched.txt b/Documentation/block/cfq-iosched.txt
index 1e4f835a659d..895bd3813115 100644
--- a/Documentation/block/cfq-iosched.txt
+++ b/Documentation/block/cfq-iosched.txt
@@ -240,11 +240,11 @@ All cfq queues doing synchronous sequential IO go on to sync-idle tree.
 On this tree we idle on each queue individually.
 
 All synchronous non-sequential queues go on sync-noidle tree. Also any
-request which are marked with REQ_NOIDLE go on this service tree. On this
-tree we do not idle on individual queues instead idle on the whole group
-of queues or the tree. So if there are 4 queues waiting for IO to dispatch
-we will idle only once last queue has dispatched the IO and there is
-no more IO on this service tree.
+synchronous write request which is not marked with REQ_IDLE goes on this
+service tree. On this tree we do not idle on individual queues instead idle
+on the whole group of queues or the tree. So if there are 4 queues waiting
+for IO to dispatch we will idle only once last queue has dispatched the IO
+and there is no more IO on this service tree.
 
 All async writes go on async service tree. There is no idling on async
 queues.
@@ -257,17 +257,17 @@ tree idling provides isolation with buffered write queues on async tree.
 
 FAQ
 ===
-Q1. Why to idle at all on queues marked with REQ_NOIDLE.
+Q1. Why to idle at all on queues not marked with REQ_IDLE.
 
-A1. We only do tree idle (all queues on sync-noidle tree) on queues marked
-    with REQ_NOIDLE. This helps in providing isolation with all the sync-idle
+A1. We only do tree idle (all queues on sync-noidle tree) on queues not marked
+    with REQ_IDLE. This helps in providing isolation with all the sync-idle
     queues. Otherwise in presence of many sequential readers, other
     synchronous IO might not get fair share of disk.
 
     For example, if there are 10 sequential readers doing IO and they get
-    100ms each. If a REQ_NOIDLE request comes in, it will be scheduled
-    roughly after 1 second. If after completion of REQ_NOIDLE request we
-    do not idle, and after a couple of milli seconds a another REQ_NOIDLE
+    100ms each. If a !REQ_IDLE request comes in, it will be scheduled
+    roughly after 1 second. If after completion of !REQ_IDLE request we
+    do not idle, and after a couple of milli seconds a another !REQ_IDLE
     request comes in, again it will be scheduled after 1second. Repeat it
     and notice how a workload can lose its disk share and suffer due to
     multiple sequential readers.
@@ -276,16 +276,16 @@ A1. We only do tree idle (all queues on sync-noidle tree) on queues marked
     context of fsync, and later some journaling data is written. Journaling
     data comes in only after fsync has finished its IO (atleast for ext4
     that seemed to be the case). Now if one decides not to idle on fsync
-    thread due to REQ_NOIDLE, then next journaling write will not get
+    thread due to !REQ_IDLE, then next journaling write will not get
     scheduled for another second. A process doing small fsync, will suffer
     badly in presence of multiple sequential readers.
 
-    Hence doing tree idling on threads using REQ_NOIDLE flag on requests
+    Hence doing tree idling on threads using !REQ_IDLE flag on requests
     provides isolation from multiple sequential readers and at the same
     time we do not idle on individual threads.
 
-Q2. When to specify REQ_NOIDLE
-A2. I would think whenever one is doing synchronous write and not expecting
+Q2. When to specify REQ_IDLE
+A2. I would think whenever one is doing synchronous write and expecting
     more writes to be dispatched from same context soon, should be able
-    to specify REQ_NOIDLE on writes and that probably should work well for
+    to specify REQ_IDLE on writes and that probably should work well for
     most of the cases.
diff --git a/Documentation/block/null_blk.txt b/Documentation/block/null_blk.txt
index d8880ca30af4..3140dbd860d8 100644
--- a/Documentation/block/null_blk.txt
+++ b/Documentation/block/null_blk.txt
@@ -72,4 +72,4 @@ use_per_node_hctx=[0/1]: Default: 0
      queue for each CPU node in the system.
 
 use_lightnvm=[0/1]: Default: 0
-  Register device with LightNVM. Requires blk-mq to be used.
+  Register device with LightNVM. Requires blk-mq and CONFIG_NVM to be enabled.
diff --git a/Documentation/block/queue-sysfs.txt b/Documentation/block/queue-sysfs.txt
index 2a3904030dea..51642159aedb 100644
--- a/Documentation/block/queue-sysfs.txt
+++ b/Documentation/block/queue-sysfs.txt
@@ -58,6 +58,20 @@ When read, this file shows the total number of block IO polls and how
 many returned success.  Writing '0' to this file will disable polling
 for this device.  Writing any non-zero value will enable this feature.
 
+io_poll_delay (RW)
+------------------
+If polling is enabled, this controls what kind of polling will be
+performed. It defaults to -1, which is classic polling. In this mode,
+the CPU will repeatedly ask for completions without giving up any time.
+If set to 0, a hybrid polling mode is used, where the kernel will attempt
+to make an educated guess at when the IO will complete. Based on this
+guess, the kernel will put the process issuing IO to sleep for an amount
+of time, before entering a classic poll loop. This mode might be a
+little slower than pure classic polling, but it will be more efficient.
+If set to a value larger than 0, the kernel will put the process issuing
+IO to sleep for this amont of microseconds before entering classic
+polling.
+
 iostats (RW)
 -------------
 This file is used to control (on/off) the iostats accounting of the
@@ -169,5 +183,14 @@ This is the number of bytes the device can write in a single write-same
 command.  A value of '0' means write-same is not supported by this
 device.
 
+wb_lat_usec (RW)
+----------------
+If the device is registered for writeback throttling, then this file shows
+the target minimum read latency. If this latency is exceeded in a given
+window of time (see wb_window_usec), then the writeback throttling will start
+scaling back writes. Writing a value of '0' to this file disables the
+feature. Writing a value of '-1' to this file resets the value to the
+default setting.
+
 
 Jens Axboe <jens.axboe@oracle.com>, February 2009
diff --git a/Documentation/cpu-freq/cpufreq-stats.txt b/Documentation/cpu-freq/cpufreq-stats.txt
index 8d9773f23550..3c355f6ad834 100644
--- a/Documentation/cpu-freq/cpufreq-stats.txt
+++ b/Documentation/cpu-freq/cpufreq-stats.txt
@@ -44,11 +44,17 @@ the stats driver insertion.
 total 0
 drwxr-xr-x  2 root root    0 May 14 16:06 .
 drwxr-xr-x  3 root root    0 May 14 15:58 ..
+--w-------  1 root root 4096 May 14 16:06 reset
 -r--r--r--  1 root root 4096 May 14 16:06 time_in_state
 -r--r--r--  1 root root 4096 May 14 16:06 total_trans
 -r--r--r--  1 root root 4096 May 14 16:06 trans_table
 --------------------------------------------------------------------------------
 
+-  reset
+Write-only attribute that can be used to reset the stat counters. This can be
+useful for evaluating system behaviour under different governors without the
+need for a reboot.
+
 -  time_in_state
 This gives the amount of time spent in each of the frequencies supported by
 this CPU. The cat output will have "<frequency> <time>" pair in each line, which
diff --git a/Documentation/cpu-freq/intel-pstate.txt b/Documentation/cpu-freq/intel-pstate.txt
index e6bd1e6512a5..1953994ef5e6 100644
--- a/Documentation/cpu-freq/intel-pstate.txt
+++ b/Documentation/cpu-freq/intel-pstate.txt
@@ -48,7 +48,7 @@ In addition to the frequency-controlling interfaces provided by the cpufreq
 core, the driver provides its own sysfs files to control the P-State selection.
 These files have been added to /sys/devices/system/cpu/intel_pstate/.
 Any changes made to these files are applicable to all CPUs (even in a
-multi-package system).
+multi-package system, Refer to later section on placing "Per-CPU limits").
 
       max_perf_pct: Limits the maximum P-State that will be requested by
       the driver. It states it as a percentage of the available performance. The
@@ -120,13 +120,57 @@ frequency is fictional for Intel Core processors. Even if the scaling
 driver selects a single P-State, the actual frequency the processor
 will run at is selected by the processor itself.
 
+Per-CPU limits
+
+The kernel command line option "intel_pstate=per_cpu_perf_limits" forces
+the intel_pstate driver to use per-CPU performance limits.  When it is set,
+the sysfs control interface described above is subject to limitations.
+- The following controls are not available for both read and write
+	/sys/devices/system/cpu/intel_pstate/max_perf_pct
+	/sys/devices/system/cpu/intel_pstate/min_perf_pct
+- The following controls can be used to set performance limits, as far as the
+architecture of the processor permits:
+	/sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq
+	/sys/devices/system/cpu/cpu*/cpufreq/scaling_min_freq
+	/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
+- User can still observe turbo percent and number of P-States from
+	/sys/devices/system/cpu/intel_pstate/turbo_pct
+	/sys/devices/system/cpu/intel_pstate/num_pstates
+- User can read write system wide turbo status
+	/sys/devices/system/cpu/no_turbo
+
+Support of energy performance hints
+It is possible to provide hints to the HWP algorithms in the processor
+to be more performance centric to more energy centric. When the driver
+is using HWP, two additional cpufreq sysfs attributes are presented for
+each logical CPU.
+These attributes are:
+	- energy_performance_available_preferences
+	- energy_performance_preference
+
+To get list of supported hints:
+$ cat energy_performance_available_preferences
+    default performance balance_performance balance_power power
+
+The current preference can be read or changed via cpufreq sysfs
+attribute "energy_performance_preference". Reading from this attribute
+will display current effective setting. User can write any of the valid
+preference string to this attribute. User can always restore to power-on
+default by writing "default".
+
+Since threads can migrate to different CPUs, this is possible that the
+new CPU may have different energy performance preference than the previous
+one. To avoid such issues, either threads can be pinned to specific CPUs
+or set the same energy performance preference value to all CPUs.
+
 Tuning Intel P-State driver
 
-When HWP mode is not used, debugfs files have also been added to allow the
-tuning of the internal governor algorithm. These files are located at
-/sys/kernel/debug/pstate_snb/. The algorithm uses a PID (Proportional
-Integral Derivative) controller. The PID tunable parameters are:
+When the performance can be tuned using PID (Proportional Integral
+Derivative) controller, debugfs files are provided for adjusting performance.
+They are presented under:
+/sys/kernel/debug/pstate_snb/
 
+The PID tunable parameters are:
       deadband
       d_gain_pct
       i_gain_pct
diff --git a/Documentation/crypto/api-intro.txt b/Documentation/crypto/api-intro.txt
index beda682e8d77..45d943fcae5b 100644
--- a/Documentation/crypto/api-intro.txt
+++ b/Documentation/crypto/api-intro.txt
@@ -44,12 +44,9 @@ one block while the former can operate on an arbitrary amount of data,
 subject to block size requirements (i.e., non-stream ciphers can only
 process multiples of blocks).
 
-Support for hardware crypto devices via an asynchronous interface is
-under development.
-
 Here's an example of how to use the API:
 
-	#include <crypto/ahash.h>
+	#include <crypto/hash.h>
 	#include <linux/err.h>
 	#include <linux/scatterlist.h>
 	
diff --git a/Documentation/dev-tools/sparse.rst b/Documentation/dev-tools/sparse.rst
index 8c250e8a2105..78aa00a604a0 100644
--- a/Documentation/dev-tools/sparse.rst
+++ b/Documentation/dev-tools/sparse.rst
@@ -51,13 +51,6 @@ sure that bitwise types don't get mixed up (little-endian vs big-endian
 vs cpu-endian vs whatever), and there the constant "0" really _is_
 special.
 
-__bitwise__ - to be used for relatively compact stuff (gfp_t, etc.) that
-is mostly warning-free and is supposed to stay that way.  Warnings will
-be generated without __CHECK_ENDIAN__.
-
-__bitwise - noisy stuff; in particular, __le*/__be* are that.  We really
-don't want to drown in noise unless we'd explicitly asked for it.
-
 Using sparse for lock checking
 ------------------------------
 
@@ -109,9 +102,4 @@ be recompiled or not.  The latter is a fast way to check the whole tree if you
 have already built it.
 
 The optional make variable CF can be used to pass arguments to sparse.  The
-build system passes -Wbitwise to sparse automatically.  To perform endianness
-checks, you may define __CHECK_ENDIAN__::
-
-        make C=2 CF="-D__CHECK_ENDIAN__"
-
-These checks are disabled by default as they generate a host of warnings.
+build system passes -Wbitwise to sparse automatically.
diff --git a/Documentation/device-mapper/delay.txt b/Documentation/device-mapper/delay.txt
index a07b5927f4a8..4b1d22a44ce4 100644
--- a/Documentation/device-mapper/delay.txt
+++ b/Documentation/device-mapper/delay.txt
@@ -16,12 +16,12 @@ Example scripts
 [[
 #!/bin/sh
 # Create device delaying rw operation for 500ms
-echo "0 `blockdev --getsize $1` delay $1 0 500" | dmsetup create delayed
+echo "0 `blockdev --getsz $1` delay $1 0 500" | dmsetup create delayed
 ]]
 
 [[
 #!/bin/sh
 # Create device delaying only write operation for 500ms and
 # splitting reads and writes to different devices $1 $2
-echo "0 `blockdev --getsize $1` delay $1 0 0 $2 0 500" | dmsetup create delayed
+echo "0 `blockdev --getsz $1` delay $1 0 0 $2 0 500" | dmsetup create delayed
 ]]
diff --git a/Documentation/device-mapper/dm-crypt.txt b/Documentation/device-mapper/dm-crypt.txt
index 692171fe9da0..ff1f87bf26e8 100644
--- a/Documentation/device-mapper/dm-crypt.txt
+++ b/Documentation/device-mapper/dm-crypt.txt
@@ -21,13 +21,30 @@ Parameters: <cipher> <key> <iv_offset> <device path> \
     /proc/crypto contains supported crypto modes
 
 <key>
-    Key used for encryption. It is encoded as a hexadecimal number.
+    Key used for encryption. It is encoded either as a hexadecimal number
+    or it can be passed as <key_string> prefixed with single colon
+    character (':') for keys residing in kernel keyring service.
     You can only use key sizes that are valid for the selected cipher
     in combination with the selected iv mode.
     Note that for some iv modes the key string can contain additional
     keys (for example IV seed) so the key contains more parts concatenated
     into a single string.
 
+<key_string>
+    The kernel keyring key is identified by string in following format:
+    <key_size>:<key_type>:<key_description>.
+
+<key_size>
+    The encryption key size in bytes. The kernel key payload size must match
+    the value passed in <key_size>.
+
+<key_type>
+    Either 'logon' or 'user' kernel key type.
+
+<key_description>
+    The kernel keyring key description crypt target should look for
+    when loading key of <key_type>.
+
 <keycount>
     Multi-key compatibility mode. You can define <keycount> keys and
     then sectors are encrypted according to their offsets (sector 0 uses key0;
@@ -85,7 +102,13 @@ https://gitlab.com/cryptsetup/cryptsetup
 [[
 #!/bin/sh
 # Create a crypt device using dmsetup
-dmsetup create crypt1 --table "0 `blockdev --getsize $1` crypt aes-cbc-essiv:sha256 babebabebabebabebabebabebabebabe 0 $1 0"
+dmsetup create crypt1 --table "0 `blockdev --getsz $1` crypt aes-cbc-essiv:sha256 babebabebabebabebabebabebabebabe 0 $1 0"
+]]
+
+[[
+#!/bin/sh
+# Create a crypt device using dmsetup when encryption key is stored in keyring service
+dmsetup create crypt2 --table "0 `blockdev --getsize $1` crypt aes-cbc-essiv:sha256 :32:logon:my_prefix:my_key 0 $1 0"
 ]]
 
 [[
diff --git a/Documentation/device-mapper/dm-raid.txt b/Documentation/device-mapper/dm-raid.txt
index 9bd531aa2279..5e3786fd9ea7 100644
--- a/Documentation/device-mapper/dm-raid.txt
+++ b/Documentation/device-mapper/dm-raid.txt
@@ -242,6 +242,10 @@ recovery.  Here is a fuller description of the individual fields:
 			in RAID1/10 or wrong parity values found in RAID4/5/6.
 			This value is valid only after a "check" of the array
 			is performed.  A healthy array has a 'mismatch_cnt' of 0.
+	<data_offset>   The current data offset to the start of the user data on
+			each component device of a raid set (see the respective
+			raid parameter to support out-of-place reshaping).
+
 
 Message Interface
 -----------------
diff --git a/Documentation/device-mapper/linear.txt b/Documentation/device-mapper/linear.txt
index d5307d380a45..7cb98d89d3f8 100644
--- a/Documentation/device-mapper/linear.txt
+++ b/Documentation/device-mapper/linear.txt
@@ -16,15 +16,15 @@ Example scripts
 [[
 #!/bin/sh
 # Create an identity mapping for a device
-echo "0 `blockdev --getsize $1` linear $1 0" | dmsetup create identity
+echo "0 `blockdev --getsz $1` linear $1 0" | dmsetup create identity
 ]]
 
 
 [[
 #!/bin/sh
 # Join 2 devices together
-size1=`blockdev --getsize $1`
-size2=`blockdev --getsize $2`
+size1=`blockdev --getsz $1`
+size2=`blockdev --getsz $2`
 echo "0 $size1 linear $1 0
 $size1 $size2 linear $2 0" | dmsetup create joined
 ]]
@@ -44,7 +44,7 @@ if (!defined($dev)) {
         die("Please specify a device.\n");
 }
 
-my $dev_size = `blockdev --getsize $dev`;
+my $dev_size = `blockdev --getsz $dev`;
 my $extents = int($dev_size / $extent_size) -
               (($dev_size % $extent_size) ? 1 : 0);
 
diff --git a/Documentation/device-mapper/striped.txt b/Documentation/device-mapper/striped.txt
index 45f3b91ea4c3..07ec492cceee 100644
--- a/Documentation/device-mapper/striped.txt
+++ b/Documentation/device-mapper/striped.txt
@@ -37,9 +37,9 @@ if (!$num_devs) {
         die("Specify at least one device\n");
 }
 
-$min_dev_size = `blockdev --getsize $devs[0]`;
+$min_dev_size = `blockdev --getsz $devs[0]`;
 for ($i = 1; $i < $num_devs; $i++) {
-        my $this_size = `blockdev --getsize $devs[$i]`;
+        my $this_size = `blockdev --getsz $devs[$i]`;
         $min_dev_size = ($min_dev_size < $this_size) ?
                         $min_dev_size : $this_size;
 }
diff --git a/Documentation/device-mapper/switch.txt b/Documentation/device-mapper/switch.txt
index 424835e57f27..5bd4831db4a8 100644
--- a/Documentation/device-mapper/switch.txt
+++ b/Documentation/device-mapper/switch.txt
@@ -123,7 +123,7 @@ Assume that you have volumes vg1/switch0 vg1/switch1 vg1/switch2 with
 the same size.
 
 Create a switch device with 64kB region size:
-    dmsetup create switch --table "0 `blockdev --getsize /dev/vg1/switch0`
+    dmsetup create switch --table "0 `blockdev --getsz /dev/vg1/switch0`
 	switch 3 128 0 /dev/vg1/switch0 0 /dev/vg1/switch1 0 /dev/vg1/switch2 0"
 
 Set mappings for the first 7 entries to point to devices switch0, switch1,
diff --git a/Documentation/devicetree/bindings/arm/amlogic,scpi.txt b/Documentation/devicetree/bindings/arm/amlogic,scpi.txt
new file mode 100644
index 000000000000..7b9a861e9306
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/amlogic,scpi.txt
@@ -0,0 +1,20 @@
+System Control and Power Interface (SCPI) Message Protocol
+(in addition to the standard binding in [0])
+----------------------------------------------------------
+Required properties
+
+- compatible : should be "amlogic,meson-gxbb-scpi"
+
+AMLOGIC SRAM and Shared Memory for SCPI
+------------------------------------
+
+Required properties:
+- compatible : should be "amlogic,meson-gxbb-sram"
+
+Each sub-node represents the reserved area for SCPI.
+
+Required sub-node properties:
+- compatible : should be "amlogic,meson-gxbb-scp-shmem" for SRAM based shared
+		memory on Amlogic GXBB SoC.
+
+[0] Documentation/devicetree/bindings/arm/arm,scpi.txt
diff --git a/Documentation/devicetree/bindings/arm/amlogic.txt b/Documentation/devicetree/bindings/arm/amlogic.txt
index fcc6f6c10803..9b2b41ab6817 100644
--- a/Documentation/devicetree/bindings/arm/amlogic.txt
+++ b/Documentation/devicetree/bindings/arm/amlogic.txt
@@ -17,6 +17,18 @@ Boards with the Amlogic Meson GXBaby SoC shall have the following properties:
   Required root node property:
     compatible: "amlogic,meson-gxbb";
 
+Boards with the Amlogic Meson GXL S905X SoC shall have the following properties:
+  Required root node property:
+    compatible: "amlogic,s905x", "amlogic,meson-gxl";
+
+Boards with the Amlogic Meson GXL S905D SoC shall have the following properties:
+  Required root node property:
+    compatible: "amlogic,s905d", "amlogic,meson-gxl";
+
+Boards with the Amlogic Meson GXM S912 SoC shall have the following properties:
+  Required root node property:
+    compatible: "amlogic,s912", "amlogic,meson-gxm";
+
 Board compatible values:
   - "geniatech,atv1200" (Meson6)
   - "minix,neo-x8" (Meson8)
@@ -28,3 +40,10 @@ Board compatible values:
   - "hardkernel,odroid-c2" (Meson gxbb)
   - "amlogic,p200" (Meson gxbb)
   - "amlogic,p201" (Meson gxbb)
+  - "amlogic,p212" (Meson gxl s905x)
+  - "amlogic,p230" (Meson gxl s905d)
+  - "amlogic,p231" (Meson gxl s905d)
+  - "amlogic,q200" (Meson gxm s912)
+  - "amlogic,q201" (Meson gxm s912)
+  - "nexbox,a95x" (Meson gxbb or Meson gxl s905x)
+  - "nexbox,a1" (Meson gxm s912)
diff --git a/Documentation/devicetree/bindings/arm/arm,scpi.txt b/Documentation/devicetree/bindings/arm/arm,scpi.txt
index faa4b44572e3..401831973638 100644
--- a/Documentation/devicetree/bindings/arm/arm,scpi.txt
+++ b/Documentation/devicetree/bindings/arm/arm,scpi.txt
@@ -7,7 +7,10 @@ by Linux to initiate various system control and power operations.
 
 Required properties:
 
-- compatible : should be "arm,scpi"
+- compatible : should be
+	* "arm,scpi" : For implementations complying to SCPI v1.0 or above
+	* "arm,scpi-pre-1.0" : For implementations complying to all
+		unversioned releases prior to SCPI v1.0
 - mboxes: List of phandle and mailbox channel specifiers
 	  All the channels reserved by remote SCP firmware for use by
 	  SCPI message protocol should be specified in any order
@@ -59,18 +62,14 @@ SRAM and Shared Memory for SCPI
 A small area of SRAM is reserved for SCPI communication between application
 processors and SCP.
 
-Required properties:
-- compatible : should be "arm,juno-sram-ns" for Non-secure SRAM on Juno
-
-The rest of the properties should follow the generic mmio-sram description
-found in ../../sram/sram.txt
+The properties should follow the generic mmio-sram description found in [3]
 
 Each sub-node represents the reserved area for SCPI.
 
 Required sub-node properties:
 - reg : The base offset and size of the reserved area with the SRAM
-- compatible : should be "arm,juno-scp-shmem" for Non-secure SRAM based
-	       shared memory on Juno platforms
+- compatible : should be "arm,scp-shmem" for Non-secure SRAM based
+	       shared memory
 
 Sensor bindings for the sensors based on SCPI Message Protocol
 --------------------------------------------------------------
@@ -81,11 +80,9 @@ Required properties:
 - #thermal-sensor-cells: should be set to 1. This property follows the
 			 thermal device tree bindings[2].
 
-			 Valid cell values are raw identifiers (Sensor
-			 ID) as used by the firmware. Refer to
-			 platform documentation for your
-			 implementation for the IDs to use. For Juno
-			 R0 and Juno R1 refer to [3].
+			 Valid cell values are raw identifiers (Sensor ID)
+			 as used by the firmware. Refer to  platform details
+			 for your implementation for the IDs to use.
 
 Power domain bindings for the power domains based on SCPI Message Protocol
 ------------------------------------------------------------
@@ -112,7 +109,7 @@ Required properties:
 [0] http://infocenter.arm.com/help/topic/com.arm.doc.dui0922b/index.html
 [1] Documentation/devicetree/bindings/clock/clock-bindings.txt
 [2] Documentation/devicetree/bindings/thermal/thermal.txt
-[3] http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.dui0922b/apas03s22.html
+[3] Documentation/devicetree/bindings/sram/sram.txt
 [4] Documentation/devicetree/bindings/power/power_domain.txt
 
 Example:
diff --git a/Documentation/devicetree/bindings/arm/arm-boards b/Documentation/devicetree/bindings/arm/arm-boards
index ab318a56fca2..b6e810c2781a 100644
--- a/Documentation/devicetree/bindings/arm/arm-boards
+++ b/Documentation/devicetree/bindings/arm/arm-boards
@@ -148,11 +148,12 @@ Example:
 
 /dts-v1/;
 #include <dt-bindings/interrupt-controller/irq.h>
-#include "skeleton.dtsi"
 
 / {
 	model = "ARM RealView PB1176 with device tree";
 	compatible = "arm,realview-pb1176";
+	#address-cells = <1>;
+	#size-cells = <1>;
 
 	soc {
 		#address-cells = <1>;
diff --git a/Documentation/devicetree/bindings/arm/atmel-at91.txt b/Documentation/devicetree/bindings/arm/atmel-at91.txt
index e1f5ad855f14..29737b9b616e 100644
--- a/Documentation/devicetree/bindings/arm/atmel-at91.txt
+++ b/Documentation/devicetree/bindings/arm/atmel-at91.txt
@@ -225,3 +225,19 @@ required properties:
 		compatible = "atmel,sama5d3-sfr", "syscon";
 		reg = <0xf0038000 0x60>;
 	};
+
+Security Module (SECUMOD)
+
+The Security Module macrocell provides all necessary secure functions to avoid
+voltage, temperature, frequency and mechanical attacks on the chip. It also
+embeds secure memories that can be scrambled
+
+required properties:
+- compatible: Should be "atmel,<chip>-secumod", "syscon".
+  <chip> can be "sama5d2".
+- reg: Should contain registers location and length
+
+	secumod@fc040000 {
+		compatible = "atmel,sama5d2-secumod", "syscon";
+		reg = <0xfc040000 0x100>;
+	};
diff --git a/Documentation/devicetree/bindings/arm/bcm/ns2.txt b/Documentation/devicetree/bindings/arm/bcm/brcm,ns2.txt
index 35f056f4a1c3..35f056f4a1c3 100644
--- a/Documentation/devicetree/bindings/arm/bcm/ns2.txt
+++ b/Documentation/devicetree/bindings/arm/bcm/brcm,ns2.txt
diff --git a/Documentation/devicetree/bindings/arm/cpu-capacity.txt b/Documentation/devicetree/bindings/arm/cpu-capacity.txt
new file mode 100644
index 000000000000..7809fbe0cdb7
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/cpu-capacity.txt
@@ -0,0 +1,236 @@
+==========================================
+ARM CPUs capacity bindings
+==========================================
+
+==========================================
+1 - Introduction
+==========================================
+
+ARM systems may be configured to have cpus with different power/performance
+characteristics within the same chip. In this case, additional information has
+to be made available to the kernel for it to be aware of such differences and
+take decisions accordingly.
+
+==========================================
+2 - CPU capacity definition
+==========================================
+
+CPU capacity is a number that provides the scheduler information about CPUs
+heterogeneity. Such heterogeneity can come from micro-architectural differences
+(e.g., ARM big.LITTLE systems) or maximum frequency at which CPUs can run
+(e.g., SMP systems with multiple frequency domains). Heterogeneity in this
+context is about differing performance characteristics; this binding tries to
+capture a first-order approximation of the relative performance of CPUs.
+
+CPU capacities are obtained by running a suitable benchmark. This binding makes
+no guarantees on the validity or suitability of any particular benchmark, the
+final capacity should, however, be:
+
+* A "single-threaded" or CPU affine benchmark
+* Divided by the running frequency of the CPU executing the benchmark
+* Not subject to dynamic frequency scaling of the CPU
+
+For the time being we however advise usage of the Dhrystone benchmark. What
+above thus becomes:
+
+CPU capacities are obtained by running the Dhrystone benchmark on each CPU at
+max frequency (with caches enabled). The obtained DMIPS score is then divided
+by the frequency (in MHz) at which the benchmark has been run, so that
+DMIPS/MHz are obtained.  Such values are then normalized w.r.t. the highest
+score obtained in the system.
+
+==========================================
+3 - capacity-dmips-mhz
+==========================================
+
+capacity-dmips-mhz is an optional cpu node [1] property: u32 value
+representing CPU capacity expressed in normalized DMIPS/MHz. At boot time, the
+maximum frequency available to the cpu is then used to calculate the capacity
+value internally used by the kernel.
+
+capacity-dmips-mhz property is all-or-nothing: if it is specified for a cpu
+node, it has to be specified for every other cpu nodes, or the system will
+fall back to the default capacity value for every CPU. If cpufreq is not
+available, final capacities are calculated by directly using capacity-dmips-
+mhz values (normalized w.r.t. the highest value found while parsing the DT).
+
+===========================================
+4 - Examples
+===========================================
+
+Example 1 (ARM 64-bit, 6-cpu system, two clusters):
+capacities-dmips-mhz are scaled w.r.t. 1024 (cpu@0 and cpu@1)
+supposing cluster0@max-freq=1100 and custer1@max-freq=850,
+final capacities are 1024 for cluster0 and 446 for cluster1
+
+cpus {
+	#address-cells = <2>;
+	#size-cells = <0>;
+
+	cpu-map {
+		cluster0 {
+			core0 {
+				cpu = <&A57_0>;
+			};
+			core1 {
+				cpu = <&A57_1>;
+			};
+		};
+
+		cluster1 {
+			core0 {
+				cpu = <&A53_0>;
+			};
+			core1 {
+				cpu = <&A53_1>;
+			};
+			core2 {
+				cpu = <&A53_2>;
+			};
+			core3 {
+				cpu = <&A53_3>;
+			};
+		};
+	};
+
+	idle-states {
+		entry-method = "arm,psci";
+
+		CPU_SLEEP_0: cpu-sleep-0 {
+			compatible = "arm,idle-state";
+			arm,psci-suspend-param = <0x0010000>;
+			local-timer-stop;
+			entry-latency-us = <100>;
+			exit-latency-us = <250>;
+			min-residency-us = <150>;
+		};
+
+		CLUSTER_SLEEP_0: cluster-sleep-0 {
+			compatible = "arm,idle-state";
+			arm,psci-suspend-param = <0x1010000>;
+			local-timer-stop;
+			entry-latency-us = <800>;
+			exit-latency-us = <700>;
+			min-residency-us = <2500>;
+		};
+	};
+
+	A57_0: cpu@0 {
+		compatible = "arm,cortex-a57","arm,armv8";
+		reg = <0x0 0x0>;
+		device_type = "cpu";
+		enable-method = "psci";
+		next-level-cache = <&A57_L2>;
+		clocks = <&scpi_dvfs 0>;
+		cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+		capacity-dmips-mhz = <1024>;
+	};
+
+	A57_1: cpu@1 {
+		compatible = "arm,cortex-a57","arm,armv8";
+		reg = <0x0 0x1>;
+		device_type = "cpu";
+		enable-method = "psci";
+		next-level-cache = <&A57_L2>;
+		clocks = <&scpi_dvfs 0>;
+		cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+		capacity-dmips-mhz = <1024>;
+	};
+
+	A53_0: cpu@100 {
+		compatible = "arm,cortex-a53","arm,armv8";
+		reg = <0x0 0x100>;
+		device_type = "cpu";
+		enable-method = "psci";
+		next-level-cache = <&A53_L2>;
+		clocks = <&scpi_dvfs 1>;
+		cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+		capacity-dmips-mhz = <578>;
+	};
+
+	A53_1: cpu@101 {
+		compatible = "arm,cortex-a53","arm,armv8";
+		reg = <0x0 0x101>;
+		device_type = "cpu";
+		enable-method = "psci";
+		next-level-cache = <&A53_L2>;
+		clocks = <&scpi_dvfs 1>;
+		cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+		capacity-dmips-mhz = <578>;
+	};
+
+	A53_2: cpu@102 {
+		compatible = "arm,cortex-a53","arm,armv8";
+		reg = <0x0 0x102>;
+		device_type = "cpu";
+		enable-method = "psci";
+		next-level-cache = <&A53_L2>;
+		clocks = <&scpi_dvfs 1>;
+		cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+		capacity-dmips-mhz = <578>;
+	};
+
+	A53_3: cpu@103 {
+		compatible = "arm,cortex-a53","arm,armv8";
+		reg = <0x0 0x103>;
+		device_type = "cpu";
+		enable-method = "psci";
+		next-level-cache = <&A53_L2>;
+		clocks = <&scpi_dvfs 1>;
+		cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+		capacity-dmips-mhz = <578>;
+	};
+
+	A57_L2: l2-cache0 {
+		compatible = "cache";
+	};
+
+	A53_L2: l2-cache1 {
+		compatible = "cache";
+	};
+};
+
+Example 2 (ARM 32-bit, 4-cpu system, two clusters,
+	   cpus 0,1@1GHz, cpus 2,3@500MHz):
+capacities-dmips-mhz are scaled w.r.t. 2 (cpu@0 and cpu@1), this means that first
+cpu@0 and cpu@1 are twice fast than cpu@2 and cpu@3 (at the same frequency)
+
+cpus {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	cpu0: cpu@0 {
+		device_type = "cpu";
+		compatible = "arm,cortex-a15";
+		reg = <0>;
+		capacity-dmips-mhz = <2>;
+	};
+
+	cpu1: cpu@1 {
+		device_type = "cpu";
+		compatible = "arm,cortex-a15";
+		reg = <1>;
+		capacity-dmips-mhz = <2>;
+	};
+
+	cpu2: cpu@2 {
+		device_type = "cpu";
+		compatible = "arm,cortex-a15";
+		reg = <0x100>;
+		capacity-dmips-mhz = <1>;
+	};
+
+	cpu3: cpu@3 {
+		device_type = "cpu";
+		compatible = "arm,cortex-a15";
+		reg = <0x101>;
+		capacity-dmips-mhz = <1>;
+	};
+};
+
+===========================================
+5 - References
+===========================================
+
+[1] ARM Linux Kernel documentation - CPUs bindings
+    Documentation/devicetree/bindings/arm/cpus.txt
diff --git a/Documentation/devicetree/bindings/arm/cpus.txt b/Documentation/devicetree/bindings/arm/cpus.txt
index e6782d50cbcd..a1bcfeed5f24 100644
--- a/Documentation/devicetree/bindings/arm/cpus.txt
+++ b/Documentation/devicetree/bindings/arm/cpus.txt
@@ -178,6 +178,7 @@ nodes to be present and contain the properties described below.
 			    "marvell,pj4b"
 			    "marvell,sheeva-v5"
 			    "nvidia,tegra132-denver"
+			    "nvidia,tegra186-denver"
 			    "qcom,krait"
 			    "qcom,kryo"
 			    "qcom,scorpion"
@@ -241,6 +242,14 @@ nodes to be present and contain the properties described below.
 			# List of phandles to idle state nodes supported
 			  by this cpu [3].
 
+	- capacity-dmips-mhz
+		Usage: Optional
+		Value type: <u32>
+		Definition:
+			# u32 value representing CPU capacity [3] in
+			  DMIPS/MHz, relative to highest capacity-dmips-mhz
+			  in the system.
+
 	- rockchip,pmu
 		Usage: optional for systems that have an "enable-method"
 		       property value of "rockchip,rk3066-smp"
@@ -464,3 +473,5 @@ cpus {
 [2] arm/msm/qcom,kpss-acc.txt
 [3] ARM Linux kernel documentation - idle states bindings
     Documentation/devicetree/bindings/arm/idle-states.txt
+[3] ARM Linux kernel documentation - cpu capacity bindings
+    Documentation/devicetree/bindings/arm/cpu-capacity.txt
diff --git a/Documentation/devicetree/bindings/arm/fsl.txt b/Documentation/devicetree/bindings/arm/fsl.txt
index dbbc0952021c..d6ee9c6e1dbb 100644
--- a/Documentation/devicetree/bindings/arm/fsl.txt
+++ b/Documentation/devicetree/bindings/arm/fsl.txt
@@ -97,7 +97,7 @@ Freescale LS1021A Platform Device Tree Bindings
 Required root node compatible properties:
   - compatible = "fsl,ls1021a";
 
-Freescale LS1021A SoC-specific Device Tree Bindings
+Freescale SoC-specific Device Tree Bindings
 -------------------------------------------
 
 Freescale SCFG
@@ -105,7 +105,11 @@ Freescale SCFG
 configuration and status registers for the chip. Such as getting PEX port
 status.
   Required properties:
-  - compatible: should be "fsl,ls1021a-scfg"
+  - compatible: Should contain a chip-specific compatible string,
+	Chip-specific strings are of the form "fsl,<chip>-scfg",
+	The following <chip>s are known to be supported:
+	ls1021a, ls1043a, ls1046a, ls2080a.
+
   - reg: should contain base address and length of SCFG memory-mapped registers
 
 Example:
@@ -119,7 +123,11 @@ Freescale DCFG
 configuration and status for the device. Such as setting the secondary
 core start address and release the secondary core from holdoff and startup.
   Required properties:
-  - compatible: should be "fsl,ls1021a-dcfg"
+  - compatible: Should contain a chip-specific compatible string,
+	Chip-specific strings are of the form "fsl,<chip>-dcfg",
+	The following <chip>s are known to be supported:
+	ls1021a, ls1043a, ls1046a, ls2080a.
+
   - reg : should contain base address and length of DCFG memory-mapped registers
 
 Example:
@@ -131,6 +139,10 @@ Example:
 Freescale ARMv8 based Layerscape SoC family Device Tree Bindings
 ----------------------------------------------------------------
 
+LS1043A SoC
+Required root node properties:
+    - compatible = "fsl,ls1043a";
+
 LS1043A ARMv8 based RDB Board
 Required root node properties:
     - compatible = "fsl,ls1043a-rdb", "fsl,ls1043a";
@@ -139,6 +151,22 @@ LS1043A ARMv8 based QDS Board
 Required root node properties:
     - compatible = "fsl,ls1043a-qds", "fsl,ls1043a";
 
+LS1046A SoC
+Required root node properties:
+    - compatible = "fsl,ls1046a";
+
+LS1046A ARMv8 based QDS Board
+Required root node properties:
+    - compatible = "fsl,ls1046a-qds", "fsl,ls1046a";
+
+LS1046A ARMv8 based RDB Board
+Required root node properties:
+    - compatible = "fsl,ls1046a-rdb", "fsl,ls1046a";
+
+LS2080A SoC
+Required root node properties:
+    - compatible = "fsl,ls2080a";
+
 LS2080A ARMv8 based Simulator model
 Required root node properties:
     - compatible = "fsl,ls2080a-simu", "fsl,ls2080a";
diff --git a/Documentation/devicetree/bindings/arm/hisilicon/hisilicon.txt b/Documentation/devicetree/bindings/arm/hisilicon/hisilicon.txt
index 3f81575aa6be..7df79a715611 100644
--- a/Documentation/devicetree/bindings/arm/hisilicon/hisilicon.txt
+++ b/Documentation/devicetree/bindings/arm/hisilicon/hisilicon.txt
@@ -28,6 +28,10 @@ HiP06 D03 Board
 Required root node properties:
 	- compatible = "hisilicon,hip06-d03";
 
+HiP07 D05 Board
+Required root node properties:
+	- compatible = "hisilicon,hip07-d05";
+
 Hisilicon system controller
 
 Required properties:
diff --git a/Documentation/devicetree/bindings/arm/juno,scpi.txt b/Documentation/devicetree/bindings/arm/juno,scpi.txt
new file mode 100644
index 000000000000..2ace8696bbee
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/juno,scpi.txt
@@ -0,0 +1,26 @@
+System Control and Power Interface (SCPI) Message Protocol
+(in addition to the standard binding in [0])
+
+Juno SRAM and Shared Memory for SCPI
+------------------------------------
+
+Required properties:
+- compatible : should be "arm,juno-sram-ns" for Non-secure SRAM
+
+Each sub-node represents the reserved area for SCPI.
+
+Required sub-node properties:
+- reg : The base offset and size of the reserved area with the SRAM
+- compatible : should be "arm,juno-scp-shmem" for Non-secure SRAM based
+	       shared memory on Juno platforms
+
+Sensor bindings for the sensors based on SCPI Message Protocol
+--------------------------------------------------------------
+Required properties:
+- compatible : should be "arm,scpi-sensors".
+- #thermal-sensor-cells: should be set to 1.
+			 For Juno R0 and Juno R1 refer to [1] for the
+			 sensor identifiers
+
+[0] Documentation/devicetree/bindings/arm/arm,scpi.txt
+[1] http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.dui0922b/apas03s22.html
diff --git a/Documentation/devicetree/bindings/arm/keystone/ti,sci.txt b/Documentation/devicetree/bindings/arm/keystone/ti,sci.txt
new file mode 100644
index 000000000000..31f5f9a104cc
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/keystone/ti,sci.txt
@@ -0,0 +1,81 @@
+Texas Instruments System Control Interface (TI-SCI) Message Protocol
+--------------------------------------------------------------------
+
+Texas Instrument's processors including those belonging to Keystone generation
+of processors have separate hardware entity which is now responsible for the
+management of the System on Chip (SoC) system. These include various system
+level functions as well.
+
+An example of such an SoC is K2G, which contains the system control hardware
+block called Power Management Micro Controller (PMMC). This hardware block is
+initialized early into boot process and provides services to Operating Systems
+on multiple processors including ones running Linux.
+
+See http://processors.wiki.ti.com/index.php/TISCI for protocol definition.
+
+TI-SCI controller Device Node:
+=============================
+
+The TI-SCI node describes the Texas Instrument's System Controller entity node.
+This parent node may optionally have additional children nodes which describe
+specific functionality such as clocks, power domain, reset or additional
+functionality as may be required for the SoC. This hierarchy also describes the
+relationship between the TI-SCI parent node to the child node.
+
+Required properties:
+-------------------
+- compatible: should be "ti,k2g-sci"
+- mbox-names:
+	"rx" - Mailbox corresponding to receive path
+	"tx" - Mailbox corresponding to transmit path
+
+- mboxes: Mailboxes corresponding to the mbox-names. Each value of the mboxes
+	  property should contain a phandle to the mailbox controller device
+	  node and an args specifier that will be the phandle to the intended
+	  sub-mailbox child node to be used for communication.
+
+See Documentation/devicetree/bindings/mailbox/mailbox.txt for more details
+about the generic mailbox controller and client driver bindings. Also see
+Documentation/devicetree/bindings/mailbox/ti,message-manager.txt for typical
+controller that is used to communicate with this System controllers.
+
+Optional Properties:
+-------------------
+- reg-names:
+	debug_messages - Map the Debug message region
+- reg:  register space corresponding to the debug_messages
+- ti,system-reboot-controller: If system reboot can be triggered by SoC reboot
+
+Example (K2G):
+-------------
+	pmmc: pmmc {
+		compatible = "ti,k2g-sci";
+		mbox-names = "rx", "tx";
+		mboxes= <&msgmgr &msgmgr_proxy_pmmc_rx>,
+			<&msgmgr &msgmgr_proxy_pmmc_tx>;
+		reg-names = "debug_messages";
+		reg = <0x02921800 0x800>;
+	};
+
+
+TI-SCI Client Device Node:
+=========================
+
+Client nodes are maintained as children of the relevant TI-SCI device node.
+
+Example (K2G):
+-------------
+	pmmc: pmmc {
+		compatible = "ti,k2g-sci";
+		...
+
+		my_clk_node: clk_node {
+			...
+			...
+		};
+
+		my_pd_node: pd_node {
+			...
+			...
+		};
+	};
diff --git a/Documentation/devicetree/bindings/arm/omap/omap.txt b/Documentation/devicetree/bindings/arm/omap/omap.txt
index f53e2ee65e35..05f95c3ed7d4 100644
--- a/Documentation/devicetree/bindings/arm/omap/omap.txt
+++ b/Documentation/devicetree/bindings/arm/omap/omap.txt
@@ -86,6 +86,9 @@ SoCs:
 - DRA722
   compatible = "ti,dra722", "ti,dra72", "ti,dra7"
 
+- DRA718
+  compatible = "ti,dra718", "ti,dra722", "ti,dra72", "ti,dra7"
+
 - AM5728
   compatible = "ti,am5728", "ti,dra742", "ti,dra74", "ti,dra7"
 
@@ -175,12 +178,18 @@ Boards:
 - AM5728 IDK
   compatible = "ti,am5728-idk", "ti,am5728", "ti,dra742", "ti,dra74", "ti,dra7"
 
+- AM5718 IDK
+  compatible = "ti,am5718-idk", "ti,am5718", "ti,dra7"
+
 - DRA742 EVM:  Software Development Board for DRA742
   compatible = "ti,dra7-evm", "ti,dra742", "ti,dra74", "ti,dra7"
 
 - DRA722 EVM: Software Development Board for DRA722
   compatible = "ti,dra72-evm", "ti,dra722", "ti,dra72", "ti,dra7"
 
+- DRA718 EVM: Software Development Board for DRA718
+  compatible = "ti,dra718-evm", "ti,dra718", "ti,dra722", "ti,dra72", "ti,dra7"
+
 - DM3730 Logic PD Torpedo + Wireless: Commercial System on Module with WiFi and Bluetooth
   compatible = "logicpd,dm3730-torpedo-devkit", "ti,omap3630", "ti,omap3"
 
diff --git a/Documentation/devicetree/bindings/arm/oxnas.txt b/Documentation/devicetree/bindings/arm/oxnas.txt
index b9e49711ba05..ac64e60f99f1 100644
--- a/Documentation/devicetree/bindings/arm/oxnas.txt
+++ b/Documentation/devicetree/bindings/arm/oxnas.txt
@@ -5,5 +5,10 @@ Boards with the OX810SE SoC shall have the following properties:
   Required root node property:
     compatible: "oxsemi,ox810se"
 
+Boards with the OX820 SoC shall have the following properties:
+  Required root node property:
+    compatible: "oxsemi,ox820"
+
 Board compatible values:
   - "wd,mbwe" (OX810SE)
+  - "cloudengines,pogoplugv3" (OX820)
diff --git a/Documentation/devicetree/bindings/arm/qcom.txt b/Documentation/devicetree/bindings/arm/qcom.txt
index 3e24518c6678..028d16e72186 100644
--- a/Documentation/devicetree/bindings/arm/qcom.txt
+++ b/Documentation/devicetree/bindings/arm/qcom.txt
@@ -21,7 +21,10 @@ The 'SoC' element must be one of the following strings:
 	apq8096
 	msm8916
 	msm8974
+	msm8992
+	msm8994
 	msm8996
+	mdm9615
 
 The 'board' element must be one of the following strings:
 
diff --git a/Documentation/devicetree/bindings/arm/rockchip.txt b/Documentation/devicetree/bindings/arm/rockchip.txt
index 55f388f954de..cc4ace6397ab 100644
--- a/Documentation/devicetree/bindings/arm/rockchip.txt
+++ b/Documentation/devicetree/bindings/arm/rockchip.txt
@@ -25,6 +25,10 @@ Rockchip platforms device tree bindings
     Required root node properties:
       - compatible = "radxa,rock2-square", "rockchip,rk3288";
 
+- Rikomagic MK808 v1 board:
+    Required root node properties:
+      - compatible = "rikomagic,mk808", "rockchip,rk3066a";
+
 - Firefly Firefly-RK3288 board:
     Required root node properties:
       - compatible = "firefly,firefly-rk3288", "rockchip,rk3288";
@@ -99,6 +103,18 @@ Rockchip platforms device tree bindings
     Required root node properties:
       - compatible = "mqmaker,miqi", "rockchip,rk3288";
 
+- Rockchip PX3 Evaluation board:
+    Required root node properties:
+      - compatible = "rockchip,px3-evb", "rockchip,px3", "rockchip,rk3188";
+
+- Rockchip PX5 Evaluation board:
+    Required root node properties:
+      - compatible = "rockchip,px5-evb", "rockchip,px5", "rockchip,rk3368";
+
+- Rockchip RK1108 Evaluation board
+    Required root node properties:
+      - compatible = "rockchip,rk1108-evb", "rockchip,rk1108";
+
 - Rockchip RK3368 evb:
     Required root node properties:
       - compatible = "rockchip,rk3368-evb-act8846", "rockchip,rk3368";
diff --git a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.txt b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.txt
index 0ea7f14ef294..3c551894f621 100644
--- a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.txt
+++ b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.txt
@@ -15,6 +15,8 @@ Required root node properties:
 	- "samsung,xyref5260"	- for Exynos5260-based Samsung board.
 	- "samsung,smdk5410"	- for Exynos5410-based Samsung SMDK5410 eval board.
 	- "samsung,smdk5420"	- for Exynos5420-based Samsung SMDK5420 eval board.
+	- "samsung,tm2"		- for Exynos5433-based Samsung TM2 board.
+	- "samsung,tm2e"	- for Exynos5433-based Samsung TM2E board.
 	- "samsung,sd5v1"	- for Exynos5440-based Samsung board.
 	- "samsung,ssdk5440"	- for Exynos5440-based Samsung board.
 
@@ -22,6 +24,9 @@ Required root node properties:
   * FriendlyARM
 	- "friendlyarm,tiny4412"  - for Exynos4412-based FriendlyARM
 				    TINY4412 board.
+  * TOPEET
+	- "topeet,itop4412-elite" - for Exynos4412-based TOPEET
+                                    Elite base board.
 
   * Google
 	- "google,pi"		- for Exynos5800-based Google Peach Pi
diff --git a/Documentation/devicetree/bindings/arm/shmobile.txt b/Documentation/devicetree/bindings/arm/shmobile.txt
index 2f0b7169f132..253bf9b86690 100644
--- a/Documentation/devicetree/bindings/arm/shmobile.txt
+++ b/Documentation/devicetree/bindings/arm/shmobile.txt
@@ -13,6 +13,10 @@ SoCs:
     compatible = "renesas,r8a73a4"
   - R-Mobile A1 (R8A77400)
     compatible = "renesas,r8a7740"
+  - RZ/G1M (R8A77430)
+    compatible = "renesas,r8a7743"
+  - RZ/G1E (R8A77450)
+    compatible = "renesas,r8a7745"
   - R-Car M1A (R8A77781)
     compatible = "renesas,r8a7778"
   - R-Car H1 (R8A77790)
@@ -35,7 +39,7 @@ SoCs:
 
 Boards:
 
-  - Alt
+  - Alt (RTP0RC7794SEB00010S)
     compatible = "renesas,alt", "renesas,r8a7794"
   - APE6-EVM
     compatible = "renesas,ape6evm", "renesas,r8a73a4"
@@ -47,9 +51,9 @@ Boards:
     compatible = "renesas,bockw", "renesas,r8a7778"
   - Genmai (RTK772100BC00000BR)
     compatible = "renesas,genmai", "renesas,r7s72100"
-  - Gose
+  - Gose (RTP0RC7793SEB00010S)
     compatible = "renesas,gose", "renesas,r8a7793"
-  - H3ULCB (RTP0RC7795SKB00010S)
+  - H3ULCB (R-Car Starter Kit Premier, RTP0RC7795SKB00010S)
     compatible = "renesas,h3ulcb", "renesas,r8a7795";
   - Henninger
     compatible = "renesas,henninger", "renesas,r8a7791"
@@ -61,7 +65,9 @@ Boards:
     compatible = "renesas,kzm9g", "renesas,sh73a0"
   - Lager (RTP0RC7790SEB00010S)
     compatible = "renesas,lager", "renesas,r8a7790"
-  - Marzen
+  - M3ULCB (R-Car Starter Kit Pro, RTP0RC7796SKB00010S)
+    compatible = "renesas,m3ulcb", "renesas,r8a7796";
+  - Marzen (R0P7779A00010S)
     compatible = "renesas,marzen", "renesas,r8a7779"
   - Porter (M2-LCDP)
     compatible = "renesas,porter", "renesas,r8a7791"
@@ -73,5 +79,27 @@ Boards:
     compatible = "renesas,salvator-x", "renesas,r8a7796";
   - SILK (RTP0RC7794LCB00011S)
     compatible = "renesas,silk", "renesas,r8a7794"
+  - SK-RZG1E (YR8A77450S000BE)
+    compatible = "renesas,sk-rzg1e", "renesas,r8a7745"
+  - SK-RZG1M (YR8A77430S000BE)
+    compatible = "renesas,sk-rzg1m", "renesas,r8a7743"
   - Wheat
     compatible = "renesas,wheat", "renesas,r8a7792"
+
+
+Most Renesas ARM SoCs have a Product Register that allows to retrieve SoC
+product and revision information.  If present, a device node for this register
+should be added.
+
+Required properties:
+  - compatible: Must be "renesas,prr".
+  - reg: Base address and length of the register block.
+
+
+Examples
+--------
+
+	prr: chipid@ff000044 {
+		compatible = "renesas,prr";
+		reg = <0 0xff000044 0 4>;
+	};
diff --git a/Documentation/devicetree/bindings/arm/sunxi.txt b/Documentation/devicetree/bindings/arm/sunxi.txt
index 3975d0a0e4c2..4d6467cc2aa2 100644
--- a/Documentation/devicetree/bindings/arm/sunxi.txt
+++ b/Documentation/devicetree/bindings/arm/sunxi.txt
@@ -14,4 +14,5 @@ using one of the following compatible strings:
   allwinner,sun8i-a83t
   allwinner,sun8i-h3
   allwinner,sun9i-a80
+  allwinner,sun50i-a64
   nextthing,gr8
diff --git a/Documentation/devicetree/bindings/arm/swir.txt b/Documentation/devicetree/bindings/arm/swir.txt
new file mode 100644
index 000000000000..042be73a95d3
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/swir.txt
@@ -0,0 +1,12 @@
+Sierra Wireless Modules device tree bindings
+--------------------------------------------
+
+Supported Modules :
+ - WP8548 : Includes MDM9615 and PM8018 in a module
+
+Sierra Wireless modules shall have the following properties :
+  Required root node property
+   - compatible: "swir,wp8548" for the WP8548 CF3 Module
+
+Board compatible values:
+  - "swir,mangoh-green-wp8548" for the mangOH green board with the WP8548 module
diff --git a/Documentation/devicetree/bindings/ata/ahci-fsl-qoriq.txt b/Documentation/devicetree/bindings/ata/ahci-fsl-qoriq.txt
index 032a7606b862..fc33ca01e9ba 100644
--- a/Documentation/devicetree/bindings/ata/ahci-fsl-qoriq.txt
+++ b/Documentation/devicetree/bindings/ata/ahci-fsl-qoriq.txt
@@ -3,7 +3,7 @@ Binding for Freescale QorIQ AHCI SATA Controller
 Required properties:
   - reg: Physical base address and size of the controller's register area.
   - compatible: Compatibility string. Must be 'fsl,<chip>-ahci', where
-    chip could be ls1021a, ls2080a, ls1043a etc.
+    chip could be ls1021a, ls1043a, ls1046a, ls2080a etc.
   - clocks: Input clock specifier. Refer to common clock bindings.
   - interrupts: Interrupt specifier. Refer to interrupt binding.
 
diff --git a/Documentation/devicetree/bindings/ata/ahci-st.txt b/Documentation/devicetree/bindings/ata/ahci-st.txt
index e1d01df8e3c1..909c9935360d 100644
--- a/Documentation/devicetree/bindings/ata/ahci-st.txt
+++ b/Documentation/devicetree/bindings/ata/ahci-st.txt
@@ -18,21 +18,6 @@ Optional properties:
 
 Example:
 
-	/* Example for stih416 */
-	sata0: sata@fe380000 {
-		compatible	= "st,ahci";
-		reg		= <0xfe380000 0x1000>;
-		interrupts	= <GIC_SPI 157 IRQ_TYPE_NONE>;
-		interrupt-names	= "hostc";
-		phys		= <&phy_port0 PHY_TYPE_SATA>;
-		phy-names	= "ahci_phy";
-		resets		= <&powerdown STIH416_SATA0_POWERDOWN>,
-				  <&softreset STIH416_SATA0_SOFTRESET>;
-		reset-names	= "pwr-dwn", "sw-rst";
-		clocks		= <&clk_s_a0_ls CLK_ICN_REG>;
-		clock-names	= "ahci_clk";
-	};
-
 	/* Example for stih407 family silicon */
 	sata0: sata@9b20000 {
 		compatible	= "st,ahci";
diff --git a/Documentation/devicetree/bindings/bus/nvidia,tegra20-gmi.txt b/Documentation/devicetree/bindings/bus/nvidia,tegra20-gmi.txt
new file mode 100644
index 000000000000..83b0e54f727c
--- /dev/null
+++ b/Documentation/devicetree/bindings/bus/nvidia,tegra20-gmi.txt
@@ -0,0 +1,132 @@
+Device tree bindings for NVIDIA Tegra Generic Memory Interface bus
+
+The Generic Memory Interface bus enables memory transfers between internal and
+external memory. Can be used to attach various high speed devices such as
+synchronous/asynchronous NOR, FPGA, UARTS and more.
+
+The actual devices are instantiated from the child nodes of a GMI node.
+
+Required properties:
+ - compatible : Should contain one of the following:
+        For Tegra20 must contain "nvidia,tegra20-gmi".
+        For Tegra30 must contain "nvidia,tegra30-gmi".
+ - reg: Should contain GMI controller registers location and length.
+ - clocks: Must contain an entry for each entry in clock-names.
+ - clock-names: Must include the following entries: "gmi"
+ - resets : Must contain an entry for each entry in reset-names.
+ - reset-names : Must include the following entries: "gmi"
+ - #address-cells: The number of cells used to represent physical base
+   addresses in the GMI address space. Should be 2.
+ - #size-cells: The number of cells used to represent the size of an address
+   range in the GMI address space. Should be 1.
+ - ranges: Must be set up to reflect the memory layout with three integer values
+   for each chip-select line in use (only one entry is supported, see below
+   comments):
+   <cs-number> <offset> <physical address of mapping> <size>
+
+Note that the GMI controller does not have any internal chip-select address
+decoding, because of that chip-selects either need to be managed via software
+or by employing external chip-select decoding logic.
+
+If external chip-select logic is used to support multiple devices it is assumed
+that the devices use the same timing and so are probably the same type. It also
+assumes that they can fit in the 256MB address range. In this case only one
+child device is supported which represents the active chip-select line, see
+examples for more insight.
+
+The chip-select number is decoded from the child nodes second address cell of
+'ranges' property, if 'ranges' property is not present or empty chip-select will
+then be decoded from the first cell of the 'reg' property.
+
+Optional child cs node properties:
+
+ - nvidia,snor-data-width-32bit: Use 32bit data-bus, default is 16bit.
+ - nvidia,snor-mux-mode: Enable address/data MUX mode.
+ - nvidia,snor-rdy-active-before-data: Assert RDY signal one cycle before data.
+   If omitted it will be asserted with data.
+ - nvidia,snor-rdy-active-high: RDY signal is active high
+ - nvidia,snor-adv-active-high: ADV signal is active high
+ - nvidia,snor-oe-active-high: WE/OE signal is active high
+ - nvidia,snor-cs-active-high: CS signal is active high
+
+  Note that there is some special handling for the timing values.
+  From Tegra TRM:
+  Programming 0 means 1 clock cycle: actual cycle = programmed cycle + 1
+
+ - nvidia,snor-muxed-width: Number of cycles MUX address/data asserted on the
+   bus. Valid values are 0-15, default is 1
+ - nvidia,snor-hold-width: Number of cycles CE stays asserted after the
+   de-assertion of WR_N (in case of SLAVE/MASTER Request) or OE_N
+   (in case of MASTER Request). Valid values are 0-15, default is 1
+ - nvidia,snor-adv-width: Number of cycles during which ADV stays asserted.
+   Valid values are 0-15, default is 1.
+ - nvidia,snor-ce-width: Number of cycles before CE is asserted.
+   Valid values are 0-15, default is 4
+ - nvidia,snor-we-width: Number of cycles during which WE stays asserted.
+   Valid values are 0-15, default is 1
+ - nvidia,snor-oe-width: Number of cycles during which OE stays asserted.
+   Valid values are 0-255, default is 1
+ - nvidia,snor-wait-width: Number of cycles before READY is asserted.
+   Valid values are 0-255, default is 3
+
+Example with two SJA1000 CAN controllers connected to the GMI bus. We wrap the
+controllers with a simple-bus node since they are all connected to the same
+chip-select (CS4), in this example external address decoding is provided:
+
+gmi@70090000 {
+	compatible = "nvidia,tegra20-gmi";
+	reg = <0x70009000 0x1000>;
+	#address-cells = <2>;
+	#size-cells = <1>;
+	clocks = <&tegra_car TEGRA20_CLK_NOR>;
+	clock-names = "gmi";
+	resets = <&tegra_car 42>;
+	reset-names = "gmi";
+	ranges = <4 0 0xd0000000 0xfffffff>;
+
+	status = "okay";
+
+	bus@4,0 {
+		compatible = "simple-bus";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 4 0 0x40100>;
+
+		nvidia,snor-mux-mode;
+		nvidia,snor-adv-active-high;
+
+		can@0 {
+			reg = <0 0x100>;
+			...
+		};
+
+		can@40000 {
+			reg = <0x40000 0x100>;
+			...
+		};
+	};
+};
+
+Example with one SJA1000 CAN controller connected to the GMI bus
+on CS4:
+
+gmi@70090000 {
+	compatible = "nvidia,tegra20-gmi";
+	reg = <0x70009000 0x1000>;
+	#address-cells = <2>;
+	#size-cells = <1>;
+	clocks = <&tegra_car TEGRA20_CLK_NOR>;
+	clock-names = "gmi";
+	resets = <&tegra_car 42>;
+	reset-names = "gmi";
+	ranges = <4 0 0xd0000000 0xfffffff>;
+
+	status = "okay";
+
+	can@4,0 {
+		reg = <4 0 0x100>;
+		nvidia,snor-mux-mode;
+		nvidia,snor-adv-active-high;
+		...
+	};
+};
diff --git a/Documentation/devicetree/bindings/bus/ti,da850-mstpri.txt b/Documentation/devicetree/bindings/bus/ti,da850-mstpri.txt
new file mode 100644
index 000000000000..72daefc6b4a1
--- /dev/null
+++ b/Documentation/devicetree/bindings/bus/ti,da850-mstpri.txt
@@ -0,0 +1,20 @@
+* Device tree bindings for Texas Instruments da8xx master peripheral
+  priority driver
+
+DA8XX SoCs feature a set of registers allowing to change the priority of all
+peripherals classified as masters.
+
+Documentation:
+OMAP-L138 (DA850) - http://www.ti.com/lit/ug/spruh82c/spruh82c.pdf
+
+Required properties:
+
+- compatible:		"ti,da850-mstpri" - for da850 based boards
+- reg:			offset and length of the mstpri registers
+
+Example for da850-lcdk is shown below.
+
+mstpri {
+	compatible = "ti,da850-mstpri";
+	reg = <0x14110 0x0c>;
+};
diff --git a/Documentation/devicetree/bindings/clock/exynos5433-clock.txt b/Documentation/devicetree/bindings/clock/exynos5433-clock.txt
index 63379b04e052..1dc80f8811fe 100644
--- a/Documentation/devicetree/bindings/clock/exynos5433-clock.txt
+++ b/Documentation/devicetree/bindings/clock/exynos5433-clock.txt
@@ -79,7 +79,7 @@ Required Properties:
 	Input clocks for fsys clock controller:
 		- oscclk
 		- sclk_ufs_mphy
-		- div_aclk_fsys_200
+		- aclk_fsys_200
 		- sclk_pcie_100_fsys
 		- sclk_ufsunipro_fsys
 		- sclk_mmc2_fsys
@@ -104,6 +104,10 @@ Required Properties:
 		- sclk_decon_tv_vclk_disp
 		- aclk_disp_333
 
+	Input clocks for audio clock controller:
+		- oscclk
+		- fout_aud_pll
+
 	Input clocks for bus0 clock controller:
 		- aclk_bus0_400
 
@@ -235,7 +239,7 @@ Example 2: Examples of clock controller nodes are listed below.
 
 		clock-names = "oscclk",
 			"sclk_ufs_mphy",
-			"div_aclk_fsys_200",
+			"aclk_fsys_200",
 			"sclk_pcie_100_fsys",
 			"sclk_ufsunipro_fsys",
 			"sclk_mmc2_fsys",
@@ -245,7 +249,7 @@ Example 2: Examples of clock controller nodes are listed below.
 			"sclk_usbdrd30_fsys";
 		clocks = <&xxti>,
 		       <&cmu_cpif CLK_SCLK_UFS_MPHY>,
-		       <&cmu_top CLK_DIV_ACLK_FSYS_200>,
+		       <&cmu_top CLK_ACLK_FSYS_200>,
 		       <&cmu_top CLK_SCLK_PCIE_100_FSYS>,
 		       <&cmu_top CLK_SCLK_UFSUNIPRO_FSYS>,
 		       <&cmu_top CLK_SCLK_MMC2_FSYS>,
@@ -297,6 +301,9 @@ Example 2: Examples of clock controller nodes are listed below.
 		compatible = "samsung,exynos5433-cmu-aud";
 		reg = <0x114c0000 0x0b04>;
 		#clock-cells = <1>;
+
+		clock-names = "oscclk", "fout_aud_pll";
+		clocks = <&xxti>, <&cmu_top CLK_FOUT_AUD_PLL>;
 	};
 
 	cmu_bus0: clock-controller@13600000 {
diff --git a/Documentation/devicetree/bindings/clock/hi3519-crg.txt b/Documentation/devicetree/bindings/clock/hisi-crg.txt
index acd1f235d548..cc60b3d423f3 100644
--- a/Documentation/devicetree/bindings/clock/hi3519-crg.txt
+++ b/Documentation/devicetree/bindings/clock/hisi-crg.txt
@@ -1,7 +1,7 @@
-* Hisilicon Hi3519 Clock and Reset Generator(CRG)
+* HiSilicon Clock and Reset Generator(CRG)
 
-The Hi3519 CRG module provides clock and reset signals to various
-controllers within the SoC.
+The CRG module provides clock and reset signals to various
+modules within the SoC.
 
 This binding uses the following bindings:
     Documentation/devicetree/bindings/clock/clock-bindings.txt
@@ -10,7 +10,11 @@ This binding uses the following bindings:
 Required Properties:
 
 - compatible: should be one of the following.
-  - "hisilicon,hi3519-crg" - controller compatible with Hi3519 SoC.
+  - "hisilicon,hi3516cv300-crg"
+  - "hisilicon,hi3516cv300-sysctrl"
+  - "hisilicon,hi3519-crg"
+  - "hisilicon,hi3798cv200-crg"
+  - "hisilicon,hi3798cv200-sysctrl"
 
 - reg: physical base address of the controller and length of memory mapped
   region.
diff --git a/Documentation/devicetree/bindings/clock/imx31-clock.txt b/Documentation/devicetree/bindings/clock/imx31-clock.txt
index 19df842c694f..8163d565f697 100644
--- a/Documentation/devicetree/bindings/clock/imx31-clock.txt
+++ b/Documentation/devicetree/bindings/clock/imx31-clock.txt
@@ -77,7 +77,7 @@ Examples:
 clks: ccm@53f80000{
 	compatible = "fsl,imx31-ccm";
 	reg = <0x53f80000 0x4000>;
-	interrupts = <0 31 0x04 0 53 0x04>;
+	interrupts = <31>, <53>;
 	#clock-cells = <1>;
 };
 
diff --git a/Documentation/devicetree/bindings/clock/oxnas,stdclk.txt b/Documentation/devicetree/bindings/clock/oxnas,stdclk.txt
index 208cca6ac4ec..b652f3fb7796 100644
--- a/Documentation/devicetree/bindings/clock/oxnas,stdclk.txt
+++ b/Documentation/devicetree/bindings/clock/oxnas,stdclk.txt
@@ -5,22 +5,15 @@ Please also refer to clock-bindings.txt in this directory for common clock
 bindings usage.
 
 Required properties:
-- compatible: Should be "oxsemi,ox810se-stdclk"
+- compatible: For OX810SE, should be "oxsemi,ox810se-stdclk"
+	      For OX820, should be "oxsemi,ox820-stdclk"
 - #clock-cells: 1, see below
 
 Parent node should have the following properties :
-- compatible: Should be "oxsemi,ox810se-sys-ctrl", "syscon", "simple-mfd"
-
-For OX810SE, the clock indices are :
- - 0: LEON
- - 1: DMA_SGDMA
- - 2: CIPHER
- - 3: SATA
- - 4: AUDIO
- - 5: USBMPH
- - 6: ETHA
- - 7: PCIA
- - 8: NAND
+- compatible: For OX810SE, should be
+		"oxsemi,ox810se-sys-ctrl", "syscon", "simple-mfd"
+	      For OX820, should be
+		"oxsemi,ox820-sys-ctrl", "syscon", "simple-mfd"
 
 example:
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc.txt b/Documentation/devicetree/bindings/clock/qcom,gcc.txt
index 3cf44217068e..5b4dfc1ea54f 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc.txt
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc.txt
@@ -14,6 +14,7 @@ Required properties :
 			"qcom,gcc-msm8974"
 			"qcom,gcc-msm8974pro"
 			"qcom,gcc-msm8974pro-ac"
+			"qcom,gcc-msm8994"
 			"qcom,gcc-msm8996"
 			"qcom,gcc-mdm9615"
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmcc.txt b/Documentation/devicetree/bindings/clock/qcom,rpmcc.txt
new file mode 100644
index 000000000000..87d3714b956a
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,rpmcc.txt
@@ -0,0 +1,37 @@
+Qualcomm RPM Clock Controller Binding
+------------------------------------------------
+The RPM is a dedicated hardware engine for managing the shared
+SoC resources in order to keep the lowest power profile. It
+communicates with other hardware subsystems via shared memory
+and accepts clock requests, aggregates the requests and turns
+the clocks on/off or scales them on demand.
+
+Required properties :
+- compatible : shall contain only one of the following. The generic
+               compatible "qcom,rpmcc" should be also included.
+
+			"qcom,rpmcc-msm8916", "qcom,rpmcc"
+			"qcom,rpmcc-apq8064", "qcom,rpmcc"
+
+- #clock-cells : shall contain 1
+
+Example:
+	smd {
+		compatible = "qcom,smd";
+
+		rpm {
+			interrupts = <0 168 1>;
+			qcom,ipc = <&apcs 8 0>;
+			qcom,smd-edge = <15>;
+
+			rpm_requests {
+				compatible = "qcom,rpm-msm8916";
+				qcom,smd-channels = "rpm_requests";
+
+				rpmcc: clock-controller {
+					compatible = "qcom,rpmcc-msm8916", "qcom,rpmcc";
+					#clock-cells = <1>;
+				};
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/clock/qoriq-clock.txt b/Documentation/devicetree/bindings/clock/qoriq-clock.txt
index 16a3ec433119..df9cb5ac5f72 100644
--- a/Documentation/devicetree/bindings/clock/qoriq-clock.txt
+++ b/Documentation/devicetree/bindings/clock/qoriq-clock.txt
@@ -32,6 +32,9 @@ Required properties:
 	* "fsl,b4420-clockgen"
 	* "fsl,b4860-clockgen"
 	* "fsl,ls1021a-clockgen"
+	* "fsl,ls1043a-clockgen"
+	* "fsl,ls1046a-clockgen"
+	* "fsl,ls2080a-clockgen"
 	Chassis-version clock strings include:
 	* "fsl,qoriq-clockgen-1.0": for chassis 1.0 clocks
 	* "fsl,qoriq-clockgen-2.0": for chassis 2.0 clocks
diff --git a/Documentation/devicetree/bindings/clock/renesas,cpg-mssr.txt b/Documentation/devicetree/bindings/clock/renesas,cpg-mssr.txt
index 394d725ac7e0..c46919412953 100644
--- a/Documentation/devicetree/bindings/clock/renesas,cpg-mssr.txt
+++ b/Documentation/devicetree/bindings/clock/renesas,cpg-mssr.txt
@@ -13,6 +13,8 @@ They provide the following functionalities:
 
 Required Properties:
   - compatible: Must be one of:
+      - "renesas,r8a7743-cpg-mssr" for the r8a7743 SoC (RZ/G1M)
+      - "renesas,r8a7745-cpg-mssr" for the r8a7745 SoC (RZ/G1E)
       - "renesas,r8a7795-cpg-mssr" for the r8a7795 SoC (R-Car H3)
       - "renesas,r8a7796-cpg-mssr" for the r8a7796 SoC (R-Car M3-W)
 
@@ -22,8 +24,9 @@ Required Properties:
   - clocks: References to external parent clocks, one entry for each entry in
     clock-names
   - clock-names: List of external parent clock names. Valid names are:
-      - "extal" (r8a7795, r8a7796)
+      - "extal" (r8a7743, r8a7745, r8a7795, r8a7796)
       - "extalr" (r8a7795, r8a7796)
+      - "usb_extal" (r8a7743, r8a7745)
 
   - #clock-cells: Must be 2
       - For CPG core clocks, the two clock specifier cells must be "CPG_CORE"
diff --git a/Documentation/devicetree/bindings/clock/rockchip,rk1108-cru.txt b/Documentation/devicetree/bindings/clock/rockchip,rk1108-cru.txt
new file mode 100644
index 000000000000..4da126116cf0
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/rockchip,rk1108-cru.txt
@@ -0,0 +1,59 @@
+* Rockchip RK1108 Clock and Reset Unit
+
+The RK1108 clock controller generates and supplies clock to various
+controllers within the SoC and also implements a reset controller for SoC
+peripherals.
+
+Required Properties:
+
+- compatible: should be "rockchip,rk1108-cru"
+- reg: physical base address of the controller and length of memory mapped
+  region.
+- #clock-cells: should be 1.
+- #reset-cells: should be 1.
+
+Optional Properties:
+
+- rockchip,grf: phandle to the syscon managing the "general register files"
+  If missing pll rates are not changeable, due to the missing pll lock status.
+
+Each clock is assigned an identifier and client nodes can use this identifier
+to specify the clock which they consume. All available clocks are defined as
+preprocessor macros in the dt-bindings/clock/rk1108-cru.h headers and can be
+used in device tree sources. Similar macros exist for the reset sources in
+these files.
+
+External clocks:
+
+There are several clocks that are generated outside the SoC. It is expected
+that they are defined using standard clock bindings with following
+clock-output-names:
+ - "xin24m" - crystal input - required,
+ - "ext_vip" - external VIP clock - optional
+ - "ext_i2s" - external I2S clock - optional
+ - "ext_gmac" - external GMAC clock - optional
+ - "hdmiphy" - external clock input derived from HDMI PHY - optional
+ - "usbphy" - external clock input derived from USB PHY - optional
+
+Example: Clock controller node:
+
+	cru: cru@20200000 {
+		compatible = "rockchip,rk1108-cru";
+		reg = <0x20200000 0x1000>;
+		rockchip,grf = <&grf>;
+
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+	};
+
+Example: UART controller node that consumes the clock generated by the clock
+  controller:
+
+	uart0: serial@10230000 {
+		compatible = "rockchip,rk1108-uart", "snps,dw-apb-uart";
+		reg = <0x10230000 0x100>;
+		interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		clocks = <&cru SCLK_UART0>;
+	};
diff --git a/Documentation/devicetree/bindings/clock/st,stm32-rcc.txt b/Documentation/devicetree/bindings/clock/st,stm32-rcc.txt
index c209de6cfadb..0532d815dae3 100644
--- a/Documentation/devicetree/bindings/clock/st,stm32-rcc.txt
+++ b/Documentation/devicetree/bindings/clock/st,stm32-rcc.txt
@@ -7,7 +7,9 @@ Please refer to clock-bindings.txt for common clock controller binding usage.
 Please also refer to reset.txt for common reset controller binding usage.
 
 Required properties:
-- compatible: Should be "st,stm32f42xx-rcc"
+- compatible: Should be:
+  "st,stm32f42xx-rcc"
+  "st,stm32f469-rcc"
 - reg: should be register base and length as documented in the
   datasheet
 - #reset-cells: 1, see below
diff --git a/Documentation/devicetree/bindings/clock/sunxi-ccu.txt b/Documentation/devicetree/bindings/clock/sunxi-ccu.txt
index 3868458a5feb..74d44a4273f2 100644
--- a/Documentation/devicetree/bindings/clock/sunxi-ccu.txt
+++ b/Documentation/devicetree/bindings/clock/sunxi-ccu.txt
@@ -7,6 +7,7 @@ Required properties :
 		- "allwinner,sun8i-a23-ccu"
 		- "allwinner,sun8i-a33-ccu"
 		- "allwinner,sun8i-h3-ccu"
+		- "allwinner,sun50i-a64-ccu"
 
 - reg: Must contain the registers base address and length
 - clocks: phandle to the oscillators feeding the CCU. Two are needed:
diff --git a/Documentation/devicetree/bindings/cpufreq/brcm,stb-avs-cpu-freq.txt b/Documentation/devicetree/bindings/cpufreq/brcm,stb-avs-cpu-freq.txt
new file mode 100644
index 000000000000..af2385795d78
--- /dev/null
+++ b/Documentation/devicetree/bindings/cpufreq/brcm,stb-avs-cpu-freq.txt
@@ -0,0 +1,78 @@
+Broadcom AVS mail box and interrupt register bindings
+=====================================================
+
+A total of three DT nodes are required. One node (brcm,avs-cpu-data-mem)
+references the mailbox register used to communicate with the AVS CPU[1]. The
+second node (brcm,avs-cpu-l2-intr) is required to trigger an interrupt on
+the AVS CPU. The interrupt tells the AVS CPU that it needs to process a
+command sent to it by a driver. Interrupting the AVS CPU is mandatory for
+commands to be processed.
+
+The interface also requires a reference to the AVS host interrupt controller,
+so a driver can react to interrupts generated by the AVS CPU whenever a command
+has been processed. See [2] for more information on the brcm,l2-intc node.
+
+[1] The AVS CPU is an independent co-processor that runs proprietary
+firmware. On some SoCs, this firmware supports DFS and DVFS in addition to
+Adaptive Voltage Scaling.
+
+[2] Documentation/devicetree/bindings/interrupt-controller/brcm,l2-intc.txt
+
+
+Node brcm,avs-cpu-data-mem
+--------------------------
+
+Required properties:
+- compatible: must include: brcm,avs-cpu-data-mem and
+              should include: one of brcm,bcm7271-avs-cpu-data-mem or
+                              brcm,bcm7268-avs-cpu-data-mem
+- reg: Specifies base physical address and size of the registers.
+- interrupts: The interrupt that the AVS CPU will use to interrupt the host
+              when a command completed.
+- interrupt-parent: The interrupt controller the above interrupt is routed
+                    through.
+- interrupt-names: The name of the interrupt used to interrupt the host.
+
+Optional properties:
+- None
+
+Node brcm,avs-cpu-l2-intr
+-------------------------
+
+Required properties:
+- compatible: must include: brcm,avs-cpu-l2-intr and
+              should include: one of brcm,bcm7271-avs-cpu-l2-intr or
+                              brcm,bcm7268-avs-cpu-l2-intr
+- reg: Specifies base physical address and size of the registers.
+
+Optional properties:
+- None
+
+
+Example
+=======
+
+	avs_host_l2_intc: interrupt-controller@f04d1200 {
+		#interrupt-cells = <1>;
+		compatible = "brcm,l2-intc";
+		interrupt-parent = <&intc>;
+		reg = <0xf04d1200 0x48>;
+		interrupt-controller;
+		interrupts = <0x0 0x19 0x0>;
+		interrupt-names = "avs";
+	};
+
+	avs-cpu-data-mem@f04c4000 {
+		compatible = "brcm,bcm7271-avs-cpu-data-mem",
+				"brcm,avs-cpu-data-mem";
+		reg = <0xf04c4000 0x60>;
+		interrupts = <0x1a>;
+		interrupt-parent = <&avs_host_l2_intc>;
+		interrupt-names = "sw_intr";
+	};
+
+	avs-cpu-l2-intr@f04d1100 {
+		compatible = "brcm,bcm7271-avs-cpu-l2-intr",
+				"brcm,avs-cpu-l2-intr";
+		reg = <0xf04d1100 0x10>;
+	};
diff --git a/Documentation/devicetree/bindings/crypto/fsl-sec4.txt b/Documentation/devicetree/bindings/crypto/fsl-sec4.txt
index adeca34c5a33..10a425f451fc 100644
--- a/Documentation/devicetree/bindings/crypto/fsl-sec4.txt
+++ b/Documentation/devicetree/bindings/crypto/fsl-sec4.txt
@@ -123,6 +123,9 @@ PROPERTIES
 
 
 EXAMPLE
+
+iMX6QDL/SX requires four clocks
+
 	crypto@300000 {
 		compatible = "fsl,sec-v4.0";
 		fsl,sec-era = <2>;
@@ -139,6 +142,23 @@ EXAMPLE
 		clock-names = "mem", "aclk", "ipg", "emi_slow";
 	};
 
+
+iMX6UL does only require three clocks
+
+	crypto: caam@2140000 {
+		compatible = "fsl,sec-v4.0";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		reg = <0x2140000 0x3c000>;
+		ranges = <0 0x2140000 0x3c000>;
+		interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>;
+
+		clocks = <&clks IMX6UL_CLK_CAAM_MEM>,
+			 <&clks IMX6UL_CLK_CAAM_ACLK>,
+			 <&clks IMX6UL_CLK_CAAM_IPG>;
+		clock-names = "mem", "aclk", "ipg";
+	};
+
 =====================================================================
 Job Ring (JR) Node
 
diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.txt b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.txt
new file mode 100644
index 000000000000..00f74bad1e95
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.txt
@@ -0,0 +1,112 @@
+Amlogic Meson Display Controller
+================================
+
+The Amlogic Meson Display controller is composed of several components
+that are going to be documented below:
+
+DMC|---------------VPU (Video Processing Unit)----------------|------HHI------|
+   | vd1   _______     _____________    _________________     |               |
+D  |-------|      |----|            |   |                |    |   HDMI PLL    |
+D  | vd2   | VIU  |    | Video Post |   | Video Encoders |<---|-----VCLK      |
+R  |-------|      |----| Processing |   |                |    |               |
+   | osd2  |      |    |            |---| Enci ----------|----|-----VDAC------|
+R  |-------| CSC  |----| Scalers    |   | Encp ----------|----|----HDMI-TX----|
+A  | osd1  |      |    | Blenders   |   | Encl ----------|----|---------------|
+M  |-------|______|----|____________|   |________________|    |               |
+___|__________________________________________________________|_______________|
+
+
+VIU: Video Input Unit
+---------------------
+
+The Video Input Unit is in charge of the pixel scanout from the DDR memory.
+It fetches the frames addresses, stride and parameters from the "Canvas" memory.
+This part is also in charge of the CSC (Colorspace Conversion).
+It can handle 2 OSD Planes and 2 Video Planes.
+
+VPP: Video Post Processing
+--------------------------
+
+The Video Post Processing is in charge of the scaling and blending of the
+various planes into a single pixel stream.
+There is a special "pre-blending" used by the video planes with a dedicated
+scaler and a "post-blending" to merge with the OSD Planes.
+The OSD planes also have a dedicated scaler for one of the OSD.
+
+VENC: Video Encoders
+--------------------
+
+The VENC is composed of the multiple pixel encoders :
+ - ENCI : Interlace Video encoder for CVBS and Interlace HDMI
+ - ENCP : Progressive Video Encoder for HDMI
+ - ENCL : LCD LVDS Encoder
+The VENC Unit gets a Pixel Clocks (VCLK) from a dedicated HDMI PLL and clock
+tree and provides the scanout clock to the VPP and VIU.
+The ENCI is connected to a single VDAC for Composite Output.
+The ENCI and ENCP are connected to an on-chip HDMI Transceiver.
+
+Device Tree Bindings:
+---------------------
+
+VPU: Video Processing Unit
+--------------------------
+
+Required properties:
+- compatible: value should be different for each SoC family as :
+	- GXBB (S905) : "amlogic,meson-gxbb-vpu"
+	- GXL (S905X, S905D) : "amlogic,meson-gxl-vpu"
+	- GXM (S912) : "amlogic,meson-gxm-vpu"
+	followed by the common "amlogic,meson-gx-vpu"
+- reg: base address and size of he following memory-mapped regions :
+	- vpu
+	- hhi
+	- dmc
+- reg-names: should contain the names of the previous memory regions
+- interrupts: should contain the VENC Vsync interrupt number
+
+Required nodes:
+
+The connections to the VPU output video ports are modeled using the OF graph
+bindings specified in Documentation/devicetree/bindings/graph.txt.
+
+The following table lists for each supported model the port number
+corresponding to each VPU output.
+
+		Port 0		Port 1
+-----------------------------------------
+ S905 (GXBB)	CVBS VDAC	HDMI-TX
+ S905X (GXL)	CVBS VDAC	HDMI-TX
+ S905D (GXL)	CVBS VDAC	HDMI-TX
+ S912 (GXM)	CVBS VDAC	HDMI-TX
+
+Example:
+
+tv-connector {
+	compatible = "composite-video-connector";
+
+	port {
+		tv_connector_in: endpoint {
+			remote-endpoint = <&cvbs_vdac_out>;
+		};
+	};
+};
+
+vpu: vpu@d0100000 {
+	compatible = "amlogic,meson-gxbb-vpu";
+	reg = <0x0 0xd0100000 0x0 0x100000>,
+	      <0x0 0xc883c000 0x0 0x1000>,
+	      <0x0 0xc8838000 0x0 0x1000>;
+	reg-names = "vpu", "hhi", "dmc";
+	interrupts = <GIC_SPI 3 IRQ_TYPE_EDGE_RISING>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	/* CVBS VDAC output port */
+	port@0 {
+		reg = <0>;
+
+		cvbs_vdac_out: endpoint {
+			remote-endpoint = <&tv_connector_in>;
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/display/brcm,bcm-vc4.txt b/Documentation/devicetree/bindings/display/brcm,bcm-vc4.txt
index a5ea451e67fc..e2768703ac2b 100644
--- a/Documentation/devicetree/bindings/display/brcm,bcm-vc4.txt
+++ b/Documentation/devicetree/bindings/display/brcm,bcm-vc4.txt
@@ -43,6 +43,13 @@ Required properties for DPI:
 - port:		Port node with a single endpoint connecting to the panel
 		  device, as defined in [1]
 
+Required properties for VEC:
+- compatible:	Should be "brcm,bcm2835-vec"
+- reg:		Physical base address and length of the registers
+- clocks:	The core clock the unit runs on
+- interrupts:	The interrupt number
+		  See bindings/interrupt-controller/brcm,bcm2835-armctrl-ic.txt
+
 Required properties for V3D:
 - compatible:	Should be "brcm,bcm2835-v3d"
 - reg:		Physical base address and length of the V3D's registers
@@ -92,6 +99,13 @@ dpi: dpi@7e208000 {
 	};
 };
 
+vec: vec@7e806000 {
+	compatible = "brcm,bcm2835-vec";
+	reg = <0x7e806000 0x1000>;
+	clocks = <&clocks BCM2835_CLOCK_VEC>;
+	interrupts = <2 27>;
+};
+
 v3d: v3d@7ec00000 {
 	compatible = "brcm,bcm2835-v3d";
 	reg = <0x7ec00000 0x1000>;
diff --git a/Documentation/devicetree/bindings/display/bridge/dumb-vga-dac.txt b/Documentation/devicetree/bindings/display/bridge/dumb-vga-dac.txt
index 003bc246a270..164cbb15f04c 100644
--- a/Documentation/devicetree/bindings/display/bridge/dumb-vga-dac.txt
+++ b/Documentation/devicetree/bindings/display/bridge/dumb-vga-dac.txt
@@ -16,6 +16,8 @@ graph bindings specified in Documentation/devicetree/bindings/graph.txt.
 - Video port 0 for RGB input
 - Video port 1 for VGA output
 
+Optional properties:
+- vdd-supply: Power supply for DAC
 
 Example
 -------
diff --git a/Documentation/devicetree/bindings/display/bridge/dw_hdmi.txt b/Documentation/devicetree/bindings/display/bridge/dw_hdmi.txt
index dc1452f0d5d8..5e9a84d6e5f1 100644
--- a/Documentation/devicetree/bindings/display/bridge/dw_hdmi.txt
+++ b/Documentation/devicetree/bindings/display/bridge/dw_hdmi.txt
@@ -19,7 +19,9 @@ Required properties:
 
 Optional properties
 - reg-io-width: the width of the reg:1,4, default set to 1 if not present
-- ddc-i2c-bus: phandle of an I2C controller used for DDC EDID probing
+- ddc-i2c-bus: phandle of an I2C controller used for DDC EDID probing,
+  if the property is omitted, a functionally reduced I2C bus
+  controller on DW HDMI is probed
 - clocks, clock-names: phandle to the HDMI CEC clock, name should be "cec"
 
 Example:
diff --git a/Documentation/devicetree/bindings/display/ti/ti,tfp410.txt b/Documentation/devicetree/bindings/display/bridge/ti,tfp410.txt
index 2cbe32a3d0bb..54d7e31525ec 100644
--- a/Documentation/devicetree/bindings/display/ti/ti,tfp410.txt
+++ b/Documentation/devicetree/bindings/display/bridge/ti,tfp410.txt
@@ -6,10 +6,15 @@ Required properties:
 
 Optional properties:
 - powerdown-gpios: power-down gpio
+- reg: I2C address. If and only if present the device node
+       should be placed into the i2c controller node where the
+       tfp410 i2c is connected to.
 
 Required nodes:
-- Video port 0 for DPI input
-- Video port 1 for DVI output
+- Video port 0 for DPI input [1].
+- Video port 1 for DVI output [1].
+
+[1]: Documentation/devicetree/bindings/media/video-interfaces.txt
 
 Example
 -------
diff --git a/Documentation/devicetree/bindings/display/ht16k33.txt b/Documentation/devicetree/bindings/display/ht16k33.txt
new file mode 100644
index 000000000000..8e5b30b87754
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/ht16k33.txt
@@ -0,0 +1,42 @@
+Holtek ht16k33 RAM mapping 16*8 LED controller driver with keyscan
+-------------------------------------------------------------------------------
+
+Required properties:
+- compatible:		"holtek,ht16k33"
+- reg:			I2C slave address of the chip.
+- interrupt-parent:	A phandle pointing to the interrupt controller
+			serving the interrupt for this chip.
+- interrupts:		Interrupt specification for the key pressed interrupt.
+- refresh-rate-hz:	Display update interval in HZ.
+- debounce-delay-ms:	Debouncing interval time in milliseconds.
+- linux,keymap: 	The keymap for keys as described in the binding
+			document (devicetree/bindings/input/matrix-keymap.txt).
+
+Optional properties:
+- linux,no-autorepeat:	Disable keyrepeat.
+- default-brightness-level: Initial brightness level [0-15] (default: 15).
+
+Example:
+
+&i2c1 {
+	ht16k33: ht16k33@70 {
+		compatible = "holtek,ht16k33";
+		reg = <0x70>;
+		refresh-rate-hz = <20>;
+		debounce-delay-ms = <50>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <5 (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_EDGE_RISING)>;
+		linux,keymap = <
+			MATRIX_KEY(2, 0, KEY_F6)
+			MATRIX_KEY(3, 0, KEY_F8)
+			MATRIX_KEY(4, 0, KEY_F10)
+			MATRIX_KEY(5, 0, KEY_F4)
+			MATRIX_KEY(6, 0, KEY_F2)
+			MATRIX_KEY(2, 1, KEY_F5)
+			MATRIX_KEY(3, 1, KEY_F7)
+			MATRIX_KEY(4, 1, KEY_F9)
+			MATRIX_KEY(5, 1, KEY_F3)
+			MATRIX_KEY(6, 1, KEY_F1)
+		>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/display/mxsfb.txt b/Documentation/devicetree/bindings/display/mxsfb.txt
index 96ec5179c8a0..472e1ea6c591 100644
--- a/Documentation/devicetree/bindings/display/mxsfb.txt
+++ b/Documentation/devicetree/bindings/display/mxsfb.txt
@@ -1,20 +1,57 @@
 * Freescale MXS LCD Interface (LCDIF)
 
+New bindings:
+=============
 Required properties:
-- compatible: Should be "fsl,<chip>-lcdif".  Supported chips include
-  imx23 and imx28.
-- reg: Address and length of the register set for lcdif
-- interrupts: Should contain lcdif interrupts
-- display : phandle to display node (see below for details)
+- compatible:	Should be "fsl,imx23-lcdif" for i.MX23.
+		Should be "fsl,imx28-lcdif" for i.MX28.
+		Should be "fsl,imx6sx-lcdif" for i.MX6SX.
+- reg:		Address and length of the register set for LCDIF
+- interrupts:	Should contain LCDIF interrupt
+- clocks:	A list of phandle + clock-specifier pairs, one for each
+		entry in 'clock-names'.
+- clock-names:	A list of clock names. For MXSFB it should contain:
+    - "pix" for the LCDIF block clock
+    - (MX6SX-only) "axi", "disp_axi" for the bus interface clock
+
+Required sub-nodes:
+  - port: The connection to an encoder chip.
+
+Example:
+
+	lcdif1: display-controller@2220000 {
+		compatible = "fsl,imx6sx-lcdif", "fsl,imx28-lcdif";
+		reg = <0x02220000 0x4000>;
+		interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&clks IMX6SX_CLK_LCDIF1_PIX>,
+			 <&clks IMX6SX_CLK_LCDIF_APB>,
+			 <&clks IMX6SX_CLK_DISPLAY_AXI>;
+		clock-names = "pix", "axi", "disp_axi";
+
+		port {
+			parallel_out: endpoint {
+				remote-endpoint = <&panel_in_parallel>;
+			};
+		};
+	};
+
+Deprecated bindings:
+====================
+Required properties:
+- compatible:	Should be "fsl,imx23-lcdif" for i.MX23.
+		Should be "fsl,imx28-lcdif" for i.MX28.
+- reg:		Address and length of the register set for LCDIF
+- interrupts:	Should contain LCDIF interrupts
+- display:	phandle to display node (see below for details)
 
 * display node
 
 Required properties:
-- bits-per-pixel : <16> for RGB565, <32> for RGB888/666.
-- bus-width : number of data lines.  Could be <8>, <16>, <18> or <24>.
+- bits-per-pixel:	<16> for RGB565, <32> for RGB888/666.
+- bus-width:		number of data lines.  Could be <8>, <16>, <18> or <24>.
 
 Required sub-node:
-- display-timings : Refer to binding doc display-timing.txt for details.
+- display-timings:	Refer to binding doc display-timing.txt for details.
 
 Examples:
 
diff --git a/Documentation/devicetree/bindings/display/panel/auo,g133han01.txt b/Documentation/devicetree/bindings/display/panel/auo,g133han01.txt
new file mode 100644
index 000000000000..3afc76747824
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/auo,g133han01.txt
@@ -0,0 +1,7 @@
+AU Optronics Corporation 13.3" FHD (1920x1080) TFT LCD panel
+
+Required properties:
+- compatible: should be "auo,g133han01"
+
+This binding is compatible with the simple-panel binding, which is specified
+in simple-panel.txt in this directory.
diff --git a/Documentation/devicetree/bindings/display/panel/auo,g185han01.txt b/Documentation/devicetree/bindings/display/panel/auo,g185han01.txt
new file mode 100644
index 000000000000..ed657c2141d4
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/auo,g185han01.txt
@@ -0,0 +1,7 @@
+AU Optronics Corporation 18.5" FHD (1920x1080) TFT LCD panel
+
+Required properties:
+- compatible: should be "auo,g185han01"
+
+This binding is compatible with the simple-panel binding, which is specified
+in simple-panel.txt in this directory.
diff --git a/Documentation/devicetree/bindings/display/panel/auo,t215hvn01.txt b/Documentation/devicetree/bindings/display/panel/auo,t215hvn01.txt
new file mode 100644
index 000000000000..cbd9da3f03b1
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/auo,t215hvn01.txt
@@ -0,0 +1,7 @@
+AU Optronics Corporation 21.5" FHD (1920x1080) color TFT LCD panel
+
+Required properties:
+- compatible: should be "auo,t215hvn01"
+
+This binding is compatible with the simple-panel binding, which is specified
+in simple-panel.txt in this directory.
diff --git a/Documentation/devicetree/bindings/display/panel/chunghwa,claa070wp03xg.txt b/Documentation/devicetree/bindings/display/panel/chunghwa,claa070wp03xg.txt
new file mode 100644
index 000000000000..dd22685d2adc
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/chunghwa,claa070wp03xg.txt
@@ -0,0 +1,7 @@
+Chunghwa Picture Tubes Ltd. 7" WXGA TFT LCD panel
+
+Required properties:
+- compatible: should be "chunghwa,claa070wp03xg"
+
+This binding is compatible with the simple-panel binding, which is specified
+in simple-panel.txt in this directory.
diff --git a/Documentation/devicetree/bindings/display/panel/display-timing.txt b/Documentation/devicetree/bindings/display/panel/display-timing.txt
index e1d4a0b59612..81a75893d1b8 100644
--- a/Documentation/devicetree/bindings/display/panel/display-timing.txt
+++ b/Documentation/devicetree/bindings/display/panel/display-timing.txt
@@ -32,6 +32,14 @@ optional properties:
 			- active low  = drive pixel data on falling edge/
 					sample data on rising edge
 			- ignored     = ignored
+ - syncclk-active: with
+			- active high = drive sync on rising edge/
+					sample sync on falling edge of pixel
+					clock
+			- active low  = drive sync on falling edge/
+					sample sync on rising edge of pixel
+					clock
+			- omitted     = same configuration as pixelclk-active
  - interlaced (bool): boolean to enable interlaced mode
  - doublescan (bool): boolean to enable doublescan mode
  - doubleclk (bool): boolean to enable doubleclock mode
diff --git a/Documentation/devicetree/bindings/display/panel/nvd,9128.txt b/Documentation/devicetree/bindings/display/panel/nvd,9128.txt
new file mode 100644
index 000000000000..17bcd017c678
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/nvd,9128.txt
@@ -0,0 +1,7 @@
+New Vision Display 7.0" 800 RGB x 480 TFT LCD panel
+
+Required properties:
+- compatible: should be "nvd,9128"
+
+This binding is compatible with the simple-panel binding, which is specified
+in simple-panel.txt in this directory.
diff --git a/Documentation/devicetree/bindings/display/panel/sharp,lq150x1lg11.txt b/Documentation/devicetree/bindings/display/panel/sharp,lq150x1lg11.txt
new file mode 100644
index 000000000000..0f57c3143506
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/sharp,lq150x1lg11.txt
@@ -0,0 +1,36 @@
+Sharp 15" LQ150X1LG11 XGA TFT LCD panel
+
+Required properties:
+- compatible: should be "sharp,lq150x1lg11"
+- power-supply: regulator to provide the VCC supply voltage (3.3 volts)
+
+Optional properties:
+- backlight: phandle of the backlight device
+- rlud-gpios: a single GPIO for the RL/UD (rotate 180 degrees) pin.
+- sellvds-gpios: a single GPIO for the SELLVDS pin.
+
+If rlud-gpios and/or sellvds-gpios are not specified, the RL/UD and/or SELLVDS
+pins are assumed to be handled appropriately by the hardware.
+
+Example:
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pwm 0 100000>;                      /* VBR */
+
+		brightness-levels = <0 20 40 60 80 100>;
+		default-brightness-level = <2>;
+
+		power-supply = <&vdd_12v_reg>;               /* VDD */
+		enable-gpios = <&gpio 42 GPIO_ACTIVE_HIGH>;  /* XSTABY */
+	};
+
+	panel {
+		compatible = "sharp,lq150x1lg11";
+
+		power-supply = <&vcc_3v3_reg>;               /* VCC */
+
+		backlight = <&backlight>;
+		rlud-gpios = <&gpio 17 GPIO_ACTIVE_HIGH>;    /* RL/UD */
+		sellvds-gpios = <&gpio 18 GPIO_ACTIVE_HIGH>; /* SELLVDS */
+	};
diff --git a/Documentation/devicetree/bindings/display/renesas,du.txt b/Documentation/devicetree/bindings/display/renesas,du.txt
index 0d30e42e40be..1a02f099a0ff 100644
--- a/Documentation/devicetree/bindings/display/renesas,du.txt
+++ b/Documentation/devicetree/bindings/display/renesas,du.txt
@@ -6,9 +6,11 @@ Required Properties:
     - "renesas,du-r8a7779" for R8A7779 (R-Car H1) compatible DU
     - "renesas,du-r8a7790" for R8A7790 (R-Car H2) compatible DU
     - "renesas,du-r8a7791" for R8A7791 (R-Car M2-W) compatible DU
+    - "renesas,du-r8a7792" for R8A7792 (R-Car V2H) compatible DU
     - "renesas,du-r8a7793" for R8A7793 (R-Car M2-N) compatible DU
     - "renesas,du-r8a7794" for R8A7794 (R-Car E2) compatible DU
     - "renesas,du-r8a7795" for R8A7795 (R-Car H3) compatible DU
+    - "renesas,du-r8a7796" for R8A7796 (R-Car M3-W) compatible DU
 
   - reg: A list of base address and length of each memory resource, one for
     each entry in the reg-names property.
@@ -25,10 +27,10 @@ Required Properties:
   - clock-names: Name of the clocks. This property is model-dependent.
     - R8A7779 uses a single functional clock. The clock doesn't need to be
       named.
-    - R8A779[01345] use one functional clock per channel and one clock per LVDS
-      encoder (if available). The functional clocks must be named "du.x" with
-      "x" being the channel numerical index. The LVDS clocks must be named
-      "lvds.x" with "x" being the LVDS encoder numerical index.
+    - R8A779[0123456] use one functional clock per channel and one clock per
+      LVDS encoder (if available). The functional clocks must be named "du.x"
+      with "x" being the channel numerical index. The LVDS clocks must be
+      named "lvds.x" with "x" being the LVDS encoder numerical index.
     - In addition to the functional and encoder clocks, all DU versions also
       support externally supplied pixel clocks. Those clocks are optional.
       When supplied they must be named "dclkin.x" with "x" being the input
@@ -47,9 +49,11 @@ corresponding to each DU output.
  R8A7779 (H1)	DPAD 0		DPAD 1		-		-
  R8A7790 (H2)	DPAD		LVDS 0		LVDS 1		-
  R8A7791 (M2-W)	DPAD		LVDS 0		-		-
+ R8A7792 (V2H)	DPAD 0		DPAD 1		-		-
  R8A7793 (M2-N)	DPAD		LVDS 0		-		-
  R8A7794 (E2)	DPAD 0		DPAD 1		-		-
  R8A7795 (H3)	DPAD		HDMI 0		HDMI 1		LVDS
+ R8A7796 (M3-W)	DPAD		HDMI		LVDS		-
 
 
 Example: R8A7790 (R-Car H2) DU
diff --git a/Documentation/devicetree/bindings/display/sunxi/sun4i-drm.txt b/Documentation/devicetree/bindings/display/sunxi/sun4i-drm.txt
index b95696d748c7..b82c00449468 100644
--- a/Documentation/devicetree/bindings/display/sunxi/sun4i-drm.txt
+++ b/Documentation/devicetree/bindings/display/sunxi/sun4i-drm.txt
@@ -28,6 +28,8 @@ The TCON acts as a timing controller for RGB, LVDS and TV interfaces.
 Required properties:
  - compatible: value must be either:
    * allwinner,sun5i-a13-tcon
+   * allwinner,sun6i-a31-tcon
+   * allwinner,sun6i-a31s-tcon
    * allwinner,sun8i-a33-tcon
  - reg: base address and size of memory-mapped region
  - interrupts: interrupt associated to this IP
@@ -50,7 +52,7 @@ Required properties:
   second the block connected to the TCON channel 1 (usually the TV
   encoder)
 
-On the A13, there is one more clock required:
+On SoCs other than the A33, there is one more clock required:
    - 'tcon-ch1': The clock driving the TCON channel 1
 
 DRC
@@ -64,6 +66,8 @@ adaptive backlight control.
 
 Required properties:
   - compatible: value must be one of:
+    * allwinner,sun6i-a31-drc
+    * allwinner,sun6i-a31s-drc
     * allwinner,sun8i-a33-drc
   - reg: base address and size of the memory-mapped region.
   - interrupts: interrupt associated to this IP
@@ -87,6 +91,7 @@ system.
 Required properties:
   - compatible: value must be one of:
     * allwinner,sun5i-a13-display-backend
+    * allwinner,sun6i-a31-display-backend
     * allwinner,sun8i-a33-display-backend
   - reg: base address and size of the memory-mapped region.
   - clocks: phandles to the clocks feeding the frontend and backend
@@ -117,6 +122,7 @@ deinterlacing and color space conversion.
 Required properties:
   - compatible: value must be one of:
     * allwinner,sun5i-a13-display-frontend
+    * allwinner,sun6i-a31-display-frontend
     * allwinner,sun8i-a33-display-frontend
   - reg: base address and size of the memory-mapped region.
   - interrupts: interrupt associated to this IP
@@ -142,6 +148,8 @@ extra node.
 Required properties:
   - compatible: value must be one of:
     * allwinner,sun5i-a13-display-engine
+    * allwinner,sun6i-a31-display-engine
+    * allwinner,sun6i-a31s-display-engine
     * allwinner,sun8i-a33-display-engine
 
   - allwinner,pipelines: list of phandle to the display engine
diff --git a/Documentation/devicetree/bindings/display/tilcdc/tilcdc.txt b/Documentation/devicetree/bindings/display/tilcdc/tilcdc.txt
index a83abd79c55c..6fddb4f4f71a 100644
--- a/Documentation/devicetree/bindings/display/tilcdc/tilcdc.txt
+++ b/Documentation/devicetree/bindings/display/tilcdc/tilcdc.txt
@@ -1,7 +1,9 @@
 Device-Tree bindings for tilcdc DRM driver
 
 Required properties:
- - compatible: value should be "ti,am33xx-tilcdc".
+ - compatible: value should be one of the following:
+    - "ti,am33xx-tilcdc" for AM335x based boards
+    - "ti,da850-tilcdc" for DA850/AM18x/OMAP-L138 based boards
  - interrupts: the interrupt number
  - reg: base address and size of the LCDC device
 
@@ -51,7 +53,7 @@ Optional nodes:
 Example:
 
 	fb: fb@4830e000 {
-		compatible = "ti,am33xx-tilcdc";
+		compatible = "ti,am33xx-tilcdc", "ti,da850-tilcdc";
 		reg = <0x4830e000 0x1000>;
 		interrupt-parent = <&intc>;
 		interrupts = <36>;
diff --git a/Documentation/devicetree/bindings/display/zte,vou.txt b/Documentation/devicetree/bindings/display/zte,vou.txt
new file mode 100644
index 000000000000..740e5bd2e4f7
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/zte,vou.txt
@@ -0,0 +1,84 @@
+ZTE VOU Display Controller
+
+This is a display controller found on ZTE ZX296718 SoC.  It includes multiple
+Graphic Layer (GL) and Video Layer (VL), two Mixers/Channels, and a few blocks
+handling scaling, color space conversion etc.  VOU also integrates the support
+for typical output devices, like HDMI, TV Encoder, VGA, and RGB LCD.
+
+* Master VOU node
+
+It must be the parent node of all the sub-device nodes.
+
+Required properties:
+ - compatible: should be "zte,zx296718-vou"
+ - #address-cells: should be <1>
+ - #size-cells: should be <1>
+ - ranges: list of address translations between VOU and sub-devices
+
+* VOU DPC device
+
+Required properties:
+ - compatible: should be "zte,zx296718-dpc"
+ - reg: Physical base address and length of DPC register regions, one for each
+   entry in 'reg-names'
+ - reg-names: The names of register regions. The following regions are required:
+	"osd"
+	"timing_ctrl"
+	"dtrc"
+	"vou_ctrl"
+	"otfppu"
+ - interrupts: VOU DPC interrupt number to CPU
+ - clocks: A list of phandle + clock-specifier pairs, one for each entry
+   in 'clock-names'
+ - clock-names: A list of clock names.  The following clocks are required:
+	"aclk"
+	"ppu_wclk"
+	"main_wclk"
+	"aux_wclk"
+
+* HDMI output device
+
+Required properties:
+ - compatible: should be "zte,zx296718-hdmi"
+ - reg: Physical base address and length of the HDMI device IO region
+ - interrupts : HDMI interrupt number to CPU
+ - clocks: A list of phandle + clock-specifier pairs, one for each entry
+   in 'clock-names'
+ - clock-names: A list of clock names.  The following clocks are required:
+	"osc_cec"
+	"osc_clk"
+	"xclk"
+
+Example:
+
+vou: vou@1440000 {
+	compatible = "zte,zx296718-vou";
+	#address-cells = <1>;
+	#size-cells = <1>;
+	ranges = <0 0x1440000 0x10000>;
+
+	dpc: dpc@0 {
+		compatible = "zte,zx296718-dpc";
+		reg = <0x0000 0x1000>, <0x1000 0x1000>,
+		      <0x5000 0x1000>, <0x6000 0x1000>,
+		      <0xa000 0x1000>;
+		reg-names = "osd", "timing_ctrl",
+			    "dtrc", "vou_ctrl",
+			    "otfppu";
+		interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&topcrm VOU_ACLK>, <&topcrm VOU_PPU_WCLK>,
+			 <&topcrm VOU_MAIN_WCLK>, <&topcrm VOU_AUX_WCLK>;
+		clock-names = "aclk", "ppu_wclk",
+			      "main_wclk", "aux_wclk";
+	};
+
+	hdmi: hdmi@c000 {
+		compatible = "zte,zx296718-hdmi";
+		reg = <0xc000 0x4000>;
+		interrupts = <GIC_SPI 82 IRQ_TYPE_EDGE_RISING>;
+		clocks = <&topcrm HDMI_OSC_CEC>,
+			 <&topcrm HDMI_OSC_CLK>,
+			 <&topcrm HDMI_XCLK>;
+		clock-names = "osc_cec", "osc_clk", "xclk";
+	};
+};
diff --git a/Documentation/devicetree/bindings/dma/nbpfaxi.txt b/Documentation/devicetree/bindings/dma/nbpfaxi.txt
index d5e2522b9ec1..d2e1e62e346a 100644
--- a/Documentation/devicetree/bindings/dma/nbpfaxi.txt
+++ b/Documentation/devicetree/bindings/dma/nbpfaxi.txt
@@ -23,6 +23,14 @@ Required properties
 #define NBPF_SLAVE_RQ_LEVEL	4
 
 Optional properties:
+- max-burst-mem-read:	limit burst size for memory reads
+  (DMA_MEM_TO_MEM/DMA_MEM_TO_DEV) to this value, specified in bytes, rather
+  than using the maximum burst size allowed by the hardware's buffer size.
+- max-burst-mem-write:	limit burst size for memory writes
+  (DMA_DEV_TO_MEM/DMA_MEM_TO_MEM) to this value, specified in bytes, rather
+  than using the maximum burst size allowed by the hardware's buffer size.
+  If both max-burst-mem-read and max-burst-mem-write are set, DMA_MEM_TO_MEM
+  will use the lower value.
 
 You can use dma-channels and dma-requests as described in dma.txt, although they
 won't be used, this information is derived from the compatibility string.
diff --git a/Documentation/devicetree/bindings/dma/qcom_hidma_mgmt.txt b/Documentation/devicetree/bindings/dma/qcom_hidma_mgmt.txt
index fd5618bd8fbc..55492c264d17 100644
--- a/Documentation/devicetree/bindings/dma/qcom_hidma_mgmt.txt
+++ b/Documentation/devicetree/bindings/dma/qcom_hidma_mgmt.txt
@@ -5,13 +5,13 @@ memcpy and memset capabilities. It has been designed for virtualized
 environments.
 
 Each HIDMA HW instance consists of multiple DMA channels. These channels
-share the same bandwidth. The bandwidth utilization can be parititioned
+share the same bandwidth. The bandwidth utilization can be partitioned
 among channels based on the priority and weight assignments.
 
 There are only two priority levels and 15 weigh assignments possible.
 
 Other parameters here determine how much of the system bus this HIDMA
-instance can use like maximum read/write request and and number of bytes to
+instance can use like maximum read/write request and number of bytes to
 read/write in a single burst.
 
 Main node required properties:
@@ -47,12 +47,18 @@ When the OS is not in control of the management interface (i.e. it's a guest),
 the channel nodes appear on their own, not under a management node.
 
 Required properties:
-- compatible: must contain "qcom,hidma-1.0"
+- compatible: must contain "qcom,hidma-1.0" for initial HW or "qcom,hidma-1.1"
+for MSI capable HW.
 - reg: Addresses for the transfer and event channel
 - interrupts: Should contain the event interrupt
 - desc-count: Number of asynchronous requests this channel can handle
 - iommus: required a iommu node
 
+Optional properties for MSI:
+- msi-parent : See the generic MSI binding described in
+ devicetree/bindings/interrupt-controller/msi.txt for a description of the
+ msi-parent property.
+
 Example:
 
 Hypervisor OS configuration:
diff --git a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt
index 5f2ce669789a..3316a9c2e638 100644
--- a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt
+++ b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.txt
@@ -24,6 +24,7 @@ Required Properties:
 		- "renesas,dmac-r8a7793" (R-Car M2-N)
 		- "renesas,dmac-r8a7794" (R-Car E2)
 		- "renesas,dmac-r8a7795" (R-Car H3)
+		- "renesas,dmac-r8a7796" (R-Car M3-W)
 
 - reg: base address and length of the registers block for the DMAC
 
diff --git a/Documentation/devicetree/bindings/dma/snps-dma.txt b/Documentation/devicetree/bindings/dma/snps-dma.txt
index 0f5583293c9c..4775c66f4508 100644
--- a/Documentation/devicetree/bindings/dma/snps-dma.txt
+++ b/Documentation/devicetree/bindings/dma/snps-dma.txt
@@ -27,6 +27,8 @@ Optional properties:
   that services interrupts for this device
 - is_private: The device channels should be marked as private and not for by the
   general purpose DMA channel allocator. False if not passed.
+- multi-block: Multi block transfers supported by hardware. Array property with
+  one cell per channel. 0: not supported, 1 (default): supported.
 
 Example:
 
diff --git a/Documentation/devicetree/bindings/dma/st_fdma.txt b/Documentation/devicetree/bindings/dma/st_fdma.txt
new file mode 100644
index 000000000000..495d853c569b
--- /dev/null
+++ b/Documentation/devicetree/bindings/dma/st_fdma.txt
@@ -0,0 +1,87 @@
+* STMicroelectronics Flexible Direct Memory Access Device Tree bindings
+
+The FDMA is a general-purpose direct memory access controller capable of
+supporting 16 independent DMA channels. It accepts up to 32 DMA requests.
+The FDMA is based on a Slim processor which requires a firmware.
+
+* FDMA Controller
+
+Required properties:
+- compatible	: Should be one of
+		 - st,stih407-fdma-mpe31-11, "st,slim-rproc";
+		 - st,stih407-fdma-mpe31-12, "st,slim-rproc";
+		 - st,stih407-fdma-mpe31-13, "st,slim-rproc";
+- reg		: Should contain an entry for each name in reg-names
+- reg-names	: Must contain "slimcore", "dmem", "peripherals", "imem" entries
+- interrupts	: Should contain one interrupt shared by all channels
+- dma-channels	: Number of channels supported by the controller
+- #dma-cells	: Must be <3>. See DMA client section below
+- clocks	: Must contain an entry for each clock
+See: Documentation/devicetree/bindings/clock/clock-bindings.txt
+
+
+Example:
+
+	fdma0: dma-controller@8e20000 {
+		compatible = "st,stih407-fdma-mpe31-11", "st,slim-rproc";
+		reg = <0x8e20000 0x8000>,
+		      <0x8e30000 0x3000>,
+		      <0x8e37000 0x1000>,
+		      <0x8e38000 0x8000>;
+		reg-names = "slimcore", "dmem", "peripherals", "imem";
+		clocks = <&clk_s_c0_flexgen CLK_FDMA>,
+			 <&clk_s_c0_flexgen CLK_EXT2F_A9>,
+			 <&clk_s_c0_flexgen CLK_EXT2F_A9>,
+			 <&clk_s_c0_flexgen CLK_EXT2F_A9>;
+		interrupts = <GIC_SPI 5 IRQ_TYPE_NONE>;
+		dma-channels = <16>;
+		#dma-cells = <3>;
+	};
+
+* DMA client
+
+Required properties:
+- dmas: Comma separated list of dma channel requests
+- dma-names: Names of the aforementioned requested channels
+
+Each dmas request consists of 4 cells:
+1. A phandle pointing to the FDMA controller
+2. The request line number
+3. A 32bit mask specifying (see include/linux/platform_data/dma-st-fdma.h)
+ -bit 2-0: Holdoff value, dreq will be masked for
+	0x0: 0-0.5us
+	0x1: 0.5-1us
+	0x2: 1-1.5us
+ -bit 17: data swap
+	0x0: disabled
+	0x1: enabled
+ -bit 21: Increment Address
+	0x0: no address increment between transfers
+	0x1: increment address between transfers
+ -bit 22: 2 STBus Initiator Coprocessor interface
+	0x0: high priority port
+	0x1: low priority port
+4. transfers type
+ 0 free running
+ 1 paced
+
+Example:
+
+	sti_uni_player2: sti-uni-player@2 {
+		compatible = "st,sti-uni-player";
+		status = "disabled";
+		#sound-dai-cells = <0>;
+		st,syscfg = <&syscfg_core>;
+		clocks = <&clk_s_d0_flexgen CLK_PCM_2>;
+		assigned-clocks = <&clk_s_d0_flexgen CLK_PCM_2>;
+		assigned-clock-parents = <&clk_s_d0_quadfs 2>;
+		assigned-clock-rates = <50000000>;
+		reg = <0x8D82000 0x158>;
+		interrupts = <GIC_SPI 86 IRQ_TYPE_NONE>;
+		dmas = <&fdma0 4 0 1>;
+		dai-name = "Uni Player #1 (DAC)";
+		dma-names = "tx";
+		st,uniperiph-id = <2>;
+		st,version = <5>;
+		st,mode = "PCM";
+	};
diff --git a/Documentation/devicetree/bindings/extcon/extcon-usb-gpio.txt b/Documentation/devicetree/bindings/extcon/extcon-usb-gpio.txt
index af0b903de293..dfc14f71e81f 100644
--- a/Documentation/devicetree/bindings/extcon/extcon-usb-gpio.txt
+++ b/Documentation/devicetree/bindings/extcon/extcon-usb-gpio.txt
@@ -5,7 +5,10 @@ connected to a GPIO pin.
 
 Required properties:
 - compatible: Should be "linux,extcon-usb-gpio"
+
+Either one of id-gpio or vbus-gpio must be present. Both can be present as well.
 - id-gpio: gpio for USB ID pin. See gpio binding.
+- vbus-gpio: gpio for USB VBUS pin.
 
 Example: Examples of extcon-usb-gpio node in dra7-evm.dts as listed below:
 	extcon_usb1 {
diff --git a/Documentation/devicetree/bindings/firmware/nvidia,tegra186-bpmp.txt b/Documentation/devicetree/bindings/firmware/nvidia,tegra186-bpmp.txt
new file mode 100644
index 000000000000..e821e16ad65b
--- /dev/null
+++ b/Documentation/devicetree/bindings/firmware/nvidia,tegra186-bpmp.txt
@@ -0,0 +1,108 @@
+NVIDIA Tegra Boot and Power Management Processor (BPMP)
+
+The BPMP is a specific processor in Tegra chip, which is designed for
+booting process handling and offloading the power management, clock
+management, and reset control tasks from the CPU. The binding document
+defines the resources that would be used by the BPMP firmware driver,
+which can create the interprocessor communication (IPC) between the CPU
+and BPMP.
+
+Required properties:
+- name : Should be bpmp
+- compatible
+    Array of strings
+    One of:
+    - "nvidia,tegra186-bpmp"
+- mboxes : The phandle of mailbox controller and the mailbox specifier.
+- shmem : List of the phandle of the TX and RX shared memory area that
+	  the IPC between CPU and BPMP is based on.
+- #clock-cells : Should be 1.
+- #power-domain-cells : Should be 1.
+- #reset-cells : Should be 1.
+
+This node is a mailbox consumer. See the following files for details of
+the mailbox subsystem, and the specifiers implemented by the relevant
+provider(s):
+
+- .../mailbox/mailbox.txt
+- .../mailbox/nvidia,tegra186-hsp.txt
+
+This node is a clock, power domain, and reset provider. See the following
+files for general documentation of those features, and the specifiers
+implemented by this node:
+
+- .../clock/clock-bindings.txt
+- <dt-bindings/clock/tegra186-clock.h>
+- ../power/power_domain.txt
+- <dt-bindings/power/tegra186-powergate.h>
+- .../reset/reset.txt
+- <dt-bindings/reset/tegra186-reset.h>
+
+The BPMP implements some services which must be represented by separate nodes.
+For example, it can provide access to certain I2C controllers, and the I2C
+bindings represent each I2C controller as a device tree node. Such nodes should
+be nested directly inside the main BPMP node.
+
+Software can determine whether a child node of the BPMP node represents a device
+by checking for a compatible property. Any node with a compatible property
+represents a device that can be instantiated. Nodes without a compatible
+property may be used to provide configuration information regarding the BPMP
+itself, although no such configuration nodes are currently defined by this
+binding.
+
+The BPMP firmware defines no single global name-/numbering-space for such
+services. Put another way, the numbering scheme for I2C buses is distinct from
+the numbering scheme for any other service the BPMP may provide (e.g. a future
+hypothetical SPI bus service). As such, child device nodes will have no reg
+property, and the BPMP node will have no #address-cells or #size-cells property.
+
+The shared memory bindings for BPMP
+-----------------------------------
+
+The shared memory area for the IPC TX and RX between CPU and BPMP are
+predefined and work on top of sysram, which is an SRAM inside the chip.
+
+See ".../sram/sram.txt" for the bindings.
+
+Example:
+
+hsp_top0: hsp@03c00000 {
+	...
+	#mbox-cells = <2>;
+};
+
+sysram@30000000 {
+	compatible = "nvidia,tegra186-sysram", "mmio-sram";
+	reg = <0x0 0x30000000 0x0 0x50000>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges = <0 0x0 0x0 0x30000000 0x0 0x50000>;
+
+	cpu_bpmp_tx: shmem@4e000 {
+		compatible = "nvidia,tegra186-bpmp-shmem";
+		reg = <0x0 0x4e000 0x0 0x1000>;
+		label = "cpu-bpmp-tx";
+		pool;
+	};
+
+	cpu_bpmp_rx: shmem@4f000 {
+		compatible = "nvidia,tegra186-bpmp-shmem";
+		reg = <0x0 0x4f000 0x0 0x1000>;
+		label = "cpu-bpmp-rx";
+		pool;
+	};
+};
+
+bpmp {
+	compatible = "nvidia,tegra186-bpmp";
+	mboxes = <&hsp_top0 TEGRA_HSP_MBOX_TYPE_DB TEGRA_HSP_DB_MASTER_BPMP>;
+	shmem = <&cpu_bpmp_tx &cpu_bpmp_rx>;
+	#clock-cells = <1>;
+	#power-domain-cells = <1>;
+	#reset-cells = <1>;
+
+	i2c {
+		compatible = "...";
+		...
+	};
+};
diff --git a/Documentation/devicetree/bindings/firmware/qcom,scm.txt b/Documentation/devicetree/bindings/firmware/qcom,scm.txt
index 3b4436e56865..20f26fbce875 100644
--- a/Documentation/devicetree/bindings/firmware/qcom,scm.txt
+++ b/Documentation/devicetree/bindings/firmware/qcom,scm.txt
@@ -10,8 +10,10 @@ Required properties:
  * "qcom,scm-apq8064" for APQ8064 platforms
  * "qcom,scm-msm8660" for MSM8660 platforms
  * "qcom,scm-msm8690" for MSM8690 platforms
+ * "qcom,scm-msm8996" for MSM8996 platforms
  * "qcom,scm" for later processors (MSM8916, APQ8084, MSM8974, etc)
 - clocks: One to three clocks may be required based on compatible.
+ * No clock required for "qcom,scm-msm8996"
  * Only core clock required for "qcom,scm-apq8064", "qcom,scm-msm8660", and "qcom,scm-msm8960"
  * Core, iface, and bus clocks required for "qcom,scm"
 - clock-names: Must contain "core" for the core clock, "iface" for the interface
diff --git a/Documentation/devicetree/bindings/fpga/altera-fpga2sdram-bridge.txt b/Documentation/devicetree/bindings/fpga/altera-fpga2sdram-bridge.txt
new file mode 100644
index 000000000000..817a8d4bf903
--- /dev/null
+++ b/Documentation/devicetree/bindings/fpga/altera-fpga2sdram-bridge.txt
@@ -0,0 +1,16 @@
+Altera FPGA To SDRAM Bridge Driver
+
+Required properties:
+- compatible		: Should contain "altr,socfpga-fpga2sdram-bridge"
+
+Optional properties:
+- bridge-enable		: 0 if driver should disable bridge at startup
+			  1 if driver should enable bridge at startup
+			  Default is to leave bridge in current state.
+
+Example:
+	fpga_bridge3: fpga-bridge@ffc25080 {
+		compatible = "altr,socfpga-fpga2sdram-bridge";
+		reg = <0xffc25080 0x4>;
+		bridge-enable = <0>;
+	};
diff --git a/Documentation/devicetree/bindings/fpga/altera-freeze-bridge.txt b/Documentation/devicetree/bindings/fpga/altera-freeze-bridge.txt
new file mode 100644
index 000000000000..f8e288c71b2d
--- /dev/null
+++ b/Documentation/devicetree/bindings/fpga/altera-freeze-bridge.txt
@@ -0,0 +1,23 @@
+Altera Freeze Bridge Controller Driver
+
+The Altera Freeze Bridge Controller manages one or more freeze bridges.
+The controller can freeze/disable the bridges which prevents signal
+changes from passing through the bridge.  The controller can also
+unfreeze/enable the bridges which allows traffic to pass through the
+bridge normally.
+
+Required properties:
+- compatible		: Should contain "altr,freeze-bridge-controller"
+- regs			: base address and size for freeze bridge module
+
+Optional properties:
+- bridge-enable		: 0 if driver should disable bridge at startup
+			  1 if driver should enable bridge at startup
+			  Default is to leave bridge in current state.
+
+Example:
+	freeze-controller@100000450 {
+		compatible = "altr,freeze-bridge-controller";
+		regs = <0x1000 0x10>;
+		bridge-enable = <0>;
+	};
diff --git a/Documentation/devicetree/bindings/fpga/altera-hps2fpga-bridge.txt b/Documentation/devicetree/bindings/fpga/altera-hps2fpga-bridge.txt
new file mode 100644
index 000000000000..6406f9337eeb
--- /dev/null
+++ b/Documentation/devicetree/bindings/fpga/altera-hps2fpga-bridge.txt
@@ -0,0 +1,39 @@
+Altera FPGA/HPS Bridge Driver
+
+Required properties:
+- regs		: base address and size for AXI bridge module
+- compatible	: Should contain one of:
+		  "altr,socfpga-lwhps2fpga-bridge",
+		  "altr,socfpga-hps2fpga-bridge", or
+		  "altr,socfpga-fpga2hps-bridge"
+- resets	: Phandle and reset specifier for this bridge's reset
+- clocks	: Clocks used by this module.
+
+Optional properties:
+- bridge-enable	: 0 if driver should disable bridge at startup.
+		  1 if driver should enable bridge at startup.
+		  Default is to leave bridge in its current state.
+
+Example:
+	fpga_bridge0: fpga-bridge@ff400000 {
+		compatible = "altr,socfpga-lwhps2fpga-bridge";
+		reg = <0xff400000 0x100000>;
+		resets = <&rst LWHPS2FPGA_RESET>;
+		clocks = <&l4_main_clk>;
+		bridge-enable = <0>;
+	};
+
+	fpga_bridge1: fpga-bridge@ff500000 {
+		compatible = "altr,socfpga-hps2fpga-bridge";
+		reg = <0xff500000 0x10000>;
+		resets = <&rst HPS2FPGA_RESET>;
+		clocks = <&l4_main_clk>;
+		bridge-enable = <1>;
+	};
+
+	fpga_bridge2: fpga-bridge@ff600000 {
+		compatible = "altr,socfpga-fpga2hps-bridge";
+		reg = <0xff600000 0x100000>;
+		resets = <&rst FPGA2HPS_RESET>;
+		clocks = <&l4_main_clk>;
+	};
diff --git a/Documentation/devicetree/bindings/fpga/altera-socfpga-a10-fpga-mgr.txt b/Documentation/devicetree/bindings/fpga/altera-socfpga-a10-fpga-mgr.txt
new file mode 100644
index 000000000000..2fd8e7a84734
--- /dev/null
+++ b/Documentation/devicetree/bindings/fpga/altera-socfpga-a10-fpga-mgr.txt
@@ -0,0 +1,19 @@
+Altera SOCFPGA Arria10 FPGA Manager
+
+Required properties:
+- compatible : should contain "altr,socfpga-a10-fpga-mgr"
+- reg        : base address and size for memory mapped io.
+               - The first index is for FPGA manager register access.
+               - The second index is for writing FPGA configuration data.
+- resets     : Phandle and reset specifier for the device's reset.
+- clocks     : Clocks used by the device.
+
+Example:
+
+	fpga_mgr: fpga-mgr@ffd03000 {
+		compatible = "altr,socfpga-a10-fpga-mgr";
+		reg = <0xffd03000 0x100
+		       0xffcfe400 0x20>;
+		clocks = <&l4_mp_clk>;
+		resets = <&rst FPGAMGR_RESET>;
+	};
diff --git a/Documentation/devicetree/bindings/fpga/fpga-region.txt b/Documentation/devicetree/bindings/fpga/fpga-region.txt
new file mode 100644
index 000000000000..3b32ba15a717
--- /dev/null
+++ b/Documentation/devicetree/bindings/fpga/fpga-region.txt
@@ -0,0 +1,494 @@
+FPGA Region Device Tree Binding
+
+Alan Tull 2016
+
+ CONTENTS
+ - Introduction
+ - Terminology
+ - Sequence
+ - FPGA Region
+ - Supported Use Models
+ - Device Tree Examples
+ - Constraints
+
+
+Introduction
+============
+
+FPGA Regions represent FPGA's and partial reconfiguration regions of FPGA's in
+the Device Tree.  FPGA Regions provide a way to program FPGAs under device tree
+control.
+
+This device tree binding document hits some of the high points of FPGA usage and
+attempts to include terminology used by both major FPGA manufacturers.  This
+document isn't a replacement for any manufacturers specifications for FPGA
+usage.
+
+
+Terminology
+===========
+
+Full Reconfiguration
+ * The entire FPGA is programmed.
+
+Partial Reconfiguration (PR)
+ * A section of an FPGA is reprogrammed while the rest of the FPGA is not
+   affected.
+ * Not all FPGA's support PR.
+
+Partial Reconfiguration Region (PRR)
+ * Also called a "reconfigurable partition"
+ * A PRR is a specific section of a FPGA reserved for reconfiguration.
+ * A base (or static) FPGA image may create a set of PRR's that later may
+   be independently reprogrammed many times.
+ * The size and specific location of each PRR is fixed.
+ * The connections at the edge of each PRR are fixed.  The image that is loaded
+   into a PRR must fit and must use a subset of the region's connections.
+ * The busses within the FPGA are split such that each region gets its own
+   branch that may be gated independently.
+
+Persona
+ * Also called a "partial bit stream"
+ * An FPGA image that is designed to be loaded into a PRR.  There may be
+   any number of personas designed to fit into a PRR, but only one at at time
+   may be loaded.
+ * A persona may create more regions.
+
+FPGA Bridge
+ * FPGA Bridges gate bus signals between a host and FPGA.
+ * FPGA Bridges should be disabled while the FPGA is being programmed to
+   prevent spurious signals on the cpu bus and to the soft logic.
+ * FPGA bridges may be actual hardware or soft logic on an FPGA.
+ * During Full Reconfiguration, hardware bridges between the host and FPGA
+   will be disabled.
+ * During Partial Reconfiguration of a specific region, that region's bridge
+   will be used to gate the busses.  Traffic to other regions is not affected.
+ * In some implementations, the FPGA Manager transparantly handles gating the
+   buses, eliminating the need to show the hardware FPGA bridges in the
+   device tree.
+ * An FPGA image may create a set of reprogrammable regions, each having its
+   own bridge and its own split of the busses in the FPGA.
+
+FPGA Manager
+ * An FPGA Manager is a hardware block that programs an FPGA under the control
+   of a host processor.
+
+Base Image
+ * Also called the "static image"
+ * An FPGA image that is designed to do full reconfiguration of the FPGA.
+ * A base image may set up a set of partial reconfiguration regions that may
+   later be reprogrammed.
+
+    ----------------       ----------------------------------
+    |  Host CPU    |       |             FPGA               |
+    |              |       |                                |
+    |          ----|       |       -----------    --------  |
+    |          | H |       |   |==>| Bridge0 |<==>| PRR0 |  |
+    |          | W |       |   |   -----------    --------  |
+    |          |   |       |   |                            |
+    |          | B |<=====>|<==|   -----------    --------  |
+    |          | R |       |   |==>| Bridge1 |<==>| PRR1 |  |
+    |          | I |       |   |   -----------    --------  |
+    |          | D |       |   |                            |
+    |          | G |       |   |   -----------    --------  |
+    |          | E |       |   |==>| Bridge2 |<==>| PRR2 |  |
+    |          ----|       |       -----------    --------  |
+    |              |       |                                |
+    ----------------       ----------------------------------
+
+Figure 1: An FPGA set up with a base image that created three regions.  Each
+region (PRR0-2) gets its own split of the busses that is independently gated by
+a soft logic bridge (Bridge0-2) in the FPGA.  The contents of each PRR can be
+reprogrammed independently while the rest of the system continues to function.
+
+
+Sequence
+========
+
+When a DT overlay that targets a FPGA Region is applied, the FPGA Region will
+do the following:
+
+ 1. Disable appropriate FPGA bridges.
+ 2. Program the FPGA using the FPGA manager.
+ 3. Enable the FPGA bridges.
+ 4. The Device Tree overlay is accepted into the live tree.
+ 5. Child devices are populated.
+
+When the overlay is removed, the child nodes will be removed and the FPGA Region
+will disable the bridges.
+
+
+FPGA Region
+===========
+
+FPGA Regions represent FPGA's and FPGA PR regions in the device tree.  An FPGA
+Region brings together the elements needed to program on a running system and
+add the child devices:
+
+ * FPGA Manager
+ * FPGA Bridges
+ * image-specific information needed to to the programming.
+ * child nodes
+
+The intended use is that a Device Tree overlay (DTO) can be used to reprogram an
+FPGA while an operating system is running.
+
+An FPGA Region that exists in the live Device Tree reflects the current state.
+If the live tree shows a "firmware-name" property or child nodes under a FPGA
+Region, the FPGA already has been programmed.  A DTO that targets a FPGA Region
+and adds the "firmware-name" property is taken as a request to reprogram the
+FPGA.  After reprogramming is successful, the overlay is accepted into the live
+tree.
+
+The base FPGA Region in the device tree represents the FPGA and supports full
+reconfiguration.  It must include a phandle to an FPGA Manager.  The base
+FPGA region will be the child of one of the hardware bridges (the bridge that
+allows register access) between the cpu and the FPGA.  If there are more than
+one bridge to control during FPGA programming, the region will also contain a
+list of phandles to the additional hardware FPGA Bridges.
+
+For partial reconfiguration (PR), each PR region will have an FPGA Region.
+These FPGA regions are children of FPGA bridges which are then children of the
+base FPGA region.  The "Full Reconfiguration to add PRR's" example below shows
+this.
+
+If an FPGA Region does not specify a FPGA Manager, it will inherit the FPGA
+Manager specified by its ancestor FPGA Region.  This supports both the case
+where the same FPGA Manager is used for all of a FPGA as well the case where
+a different FPGA Manager is used for each region.
+
+FPGA Regions do not inherit their ancestor FPGA regions' bridges.  This prevents
+shutting down bridges that are upstream from the other active regions while one
+region is getting reconfigured (see Figure 1 above).  During PR, the FPGA's
+hardware bridges remain enabled.  The PR regions' bridges will be FPGA bridges
+within the static image of the FPGA.
+
+Required properties:
+- compatible : should contain "fpga-region"
+- fpga-mgr : should contain a phandle to an FPGA Manager.  Child FPGA Regions
+	inherit this property from their ancestor regions.  A fpga-mgr property
+	in a region will override any inherited FPGA manager.
+- #address-cells, #size-cells, ranges : must be present to handle address space
+	mapping for child nodes.
+
+Optional properties:
+- firmware-name : should contain the name of an FPGA image file located on the
+	firmware search path.  If this property shows up in a live device tree
+	it indicates that the FPGA has already been programmed with this image.
+	If this property is in an overlay targeting a FPGA region, it is a
+	request to program the FPGA with that image.
+- fpga-bridges : should contain a list of phandles to FPGA Bridges that must be
+	controlled during FPGA programming along with the parent FPGA bridge.
+	This property is optional if the FPGA Manager handles the bridges.
+        If the fpga-region is  the child of a fpga-bridge, the list should not
+        contain the parent bridge.
+- partial-fpga-config : boolean, set if partial reconfiguration is to be done,
+	otherwise full reconfiguration is done.
+- external-fpga-config : boolean, set if the FPGA has already been configured
+	prior to OS boot up.
+- region-unfreeze-timeout-us : The maximum time in microseconds to wait for
+	bridges to successfully become enabled after the region has been
+	programmed.
+- region-freeze-timeout-us : The maximum time in microseconds to wait for
+	bridges to successfully become disabled before the region has been
+	programmed.
+- child nodes : devices in the FPGA after programming.
+
+In the example below, when an overlay is applied targeting fpga-region0,
+fpga_mgr is used to program the FPGA.  Two bridges are controlled during
+programming: the parent fpga_bridge0 and fpga_bridge1.  Because the region is
+the child of fpga_bridge0, only fpga_bridge1 needs to be specified in the
+fpga-bridges property.  During programming, these bridges are disabled, the
+firmware specified in the overlay is loaded to the FPGA using the FPGA manager
+specified in the region.  If FPGA programming succeeds, the bridges are
+reenabled and the overlay makes it into the live device tree.  The child devices
+are then populated.  If FPGA programming fails, the bridges are left disabled
+and the overlay is rejected.  The overlay's ranges property maps the lwhps
+bridge's region (0xff200000) and the hps bridge's region (0xc0000000) for use by
+the two child devices.
+
+Example:
+Base tree contains:
+
+	fpga_mgr: fpga-mgr@ff706000 {
+		compatible = "altr,socfpga-fpga-mgr";
+		reg = <0xff706000 0x1000
+		       0xffb90000 0x20>;
+		interrupts = <0 175 4>;
+	};
+
+	fpga_bridge0: fpga-bridge@ff400000 {
+		compatible = "altr,socfpga-lwhps2fpga-bridge";
+		reg = <0xff400000 0x100000>;
+		resets = <&rst LWHPS2FPGA_RESET>;
+		clocks = <&l4_main_clk>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		fpga_region0: fpga-region0 {
+			compatible = "fpga-region";
+			fpga-mgr = <&fpga_mgr>;
+		};
+	};
+
+	fpga_bridge1: fpga-bridge@ff500000 {
+		compatible = "altr,socfpga-hps2fpga-bridge";
+		reg = <0xff500000 0x10000>;
+		resets = <&rst HPS2FPGA_RESET>;
+		clocks = <&l4_main_clk>;
+	};
+
+Overlay contains:
+
+/dts-v1/ /plugin/;
+/ {
+	fragment@0 {
+		target = <&fpga_region0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		__overlay__ {
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			firmware-name = "soc_system.rbf";
+			fpga-bridges = <&fpga_bridge1>;
+			ranges = <0x20000 0xff200000 0x100000>,
+				 <0x0 0xc0000000 0x20000000>;
+
+			gpio@10040 {
+				compatible = "altr,pio-1.0";
+				reg = <0x10040 0x20>;
+				altr,gpio-bank-width = <4>;
+				#gpio-cells = <2>;
+				clocks = <2>;
+				gpio-controller;
+			};
+
+			onchip-memory {
+				device_type = "memory";
+				compatible = "altr,onchipmem-15.1";
+				reg = <0x0 0x10000>;
+			};
+		};
+	};
+};
+
+
+Supported Use Models
+====================
+
+In all cases the live DT must have the FPGA Manager, FPGA Bridges (if any), and
+a FPGA Region.  The target of the Device Tree Overlay is the FPGA Region.  Some
+uses are specific to a FPGA device.
+
+ * No FPGA Bridges
+   In this case, the FPGA Manager which programs the FPGA also handles the
+   bridges behind the scenes.  No FPGA Bridge devices are needed for full
+   reconfiguration.
+
+ * Full reconfiguration with hardware bridges
+   In this case, there are hardware bridges between the processor and FPGA that
+   need to be controlled during full reconfiguration.  Before the overlay is
+   applied, the live DT must include the FPGA Manager, FPGA Bridges, and a
+   FPGA Region.  The FPGA Region is the child of the bridge that allows
+   register access to the FPGA.  Additional bridges may be listed in a
+   fpga-bridges property in the FPGA region or in the device tree overlay.
+
+ * Partial reconfiguration with bridges in the FPGA
+   In this case, the FPGA will have one or more PRR's that may be programmed
+   separately while the rest of the FPGA can remain active.  To manage this,
+   bridges need to exist in the FPGA that can gate the buses going to each FPGA
+   region while the buses are enabled for other sections.  Before any partial
+   reconfiguration can be done, a base FPGA image must be loaded which includes
+   PRR's with FPGA bridges.  The device tree should have a FPGA region for each
+   PRR.
+
+Device Tree Examples
+====================
+
+The intention of this section is to give some simple examples, focusing on
+the placement of the elements detailed above, especially:
+ * FPGA Manager
+ * FPGA Bridges
+ * FPGA Region
+ * ranges
+ * target-path or target
+
+For the purposes of this section, I'm dividing the Device Tree into two parts,
+each with its own requirements.  The two parts are:
+ * The live DT prior to the overlay being added
+ * The DT overlay
+
+The live Device Tree must contain an FPGA Region, an FPGA Manager, and any FPGA
+Bridges.  The FPGA Region's "fpga-mgr" property specifies the manager by phandle
+to handle programming the FPGA.  If the FPGA Region is the child of another FPGA
+Region, the parent's FPGA Manager is used.  If FPGA Bridges need to be involved,
+they are specified in the FPGA Region by the "fpga-bridges" property.  During
+FPGA programming, the FPGA Region will disable the bridges that are in its
+"fpga-bridges" list and will re-enable them after FPGA programming has
+succeeded.
+
+The Device Tree Overlay will contain:
+ * "target-path" or "target"
+   The insertion point where the the contents of the overlay will go into the
+   live tree.  target-path is a full path, while target is a phandle.
+ * "ranges"
+    The address space mapping from processor to FPGA bus(ses).
+ * "firmware-name"
+   Specifies the name of the FPGA image file on the firmware search
+   path.  The search path is described in the firmware class documentation.
+ * "partial-fpga-config"
+   This binding is a boolean and should be present if partial reconfiguration
+   is to be done.
+ * child nodes corresponding to hardware that will be loaded in this region of
+   the FPGA.
+
+Device Tree Example: Full Reconfiguration without Bridges
+=========================================================
+
+Live Device Tree contains:
+	fpga_mgr0: fpga-mgr@f8007000 {
+		compatible = "xlnx,zynq-devcfg-1.0";
+		reg = <0xf8007000 0x100>;
+		interrupt-parent = <&intc>;
+		interrupts = <0 8 4>;
+		clocks = <&clkc 12>;
+		clock-names = "ref_clk";
+		syscon = <&slcr>;
+	};
+
+	fpga_region0: fpga-region0 {
+		compatible = "fpga-region";
+		fpga-mgr = <&fpga_mgr0>;
+		#address-cells = <0x1>;
+		#size-cells = <0x1>;
+		ranges;
+	};
+
+DT Overlay contains:
+/dts-v1/ /plugin/;
+/ {
+fragment@0 {
+	target = <&fpga_region0>;
+	#address-cells = <1>;
+	#size-cells = <1>;
+	__overlay__ {
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		firmware-name = "zynq-gpio.bin";
+
+		gpio1: gpio@40000000 {
+			compatible = "xlnx,xps-gpio-1.00.a";
+			reg = <0x40000000 0x10000>;
+			gpio-controller;
+			#gpio-cells = <0x2>;
+			xlnx,gpio-width= <0x6>;
+		};
+	};
+};
+
+Device Tree Example: Full Reconfiguration to add PRR's
+======================================================
+
+The base FPGA Region is specified similar to the first example above.
+
+This example programs the FPGA to have two regions that can later be partially
+configured.  Each region has its own bridge in the FPGA fabric.
+
+DT Overlay contains:
+/dts-v1/ /plugin/;
+/ {
+	fragment@0 {
+		target = <&fpga_region0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		__overlay__ {
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			firmware-name = "base.rbf";
+
+			fpga-bridge@4400 {
+				compatible = "altr,freeze-bridge";
+				reg = <0x4400 0x10>;
+
+				fpga_region1: fpga-region1 {
+					compatible = "fpga-region";
+					#address-cells = <0x1>;
+					#size-cells = <0x1>;
+					ranges;
+				};
+			};
+
+			fpga-bridge@4420 {
+				compatible = "altr,freeze-bridge";
+				reg = <0x4420 0x10>;
+
+				fpga_region2: fpga-region2 {
+					compatible = "fpga-region";
+					#address-cells = <0x1>;
+					#size-cells = <0x1>;
+					ranges;
+				};
+			};
+		};
+	};
+};
+
+Device Tree Example: Partial Reconfiguration
+============================================
+
+This example reprograms one of the PRR's set up in the previous example.
+
+The sequence that occurs when this overlay is similar to the above, the only
+differences are that the FPGA is partially reconfigured due to the
+"partial-fpga-config" boolean and the only bridge that is controlled during
+programming is the FPGA based bridge of fpga_region1.
+
+/dts-v1/ /plugin/;
+/ {
+	fragment@0 {
+		target = <&fpga_region1>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		__overlay__ {
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			firmware-name = "soc_image2.rbf";
+			partial-fpga-config;
+
+			gpio@10040 {
+				compatible = "altr,pio-1.0";
+				reg = <0x10040 0x20>;
+				clocks = <0x2>;
+				altr,gpio-bank-width = <0x4>;
+				resetvalue = <0x0>;
+				#gpio-cells = <0x2>;
+				gpio-controller;
+			};
+		};
+	};
+};
+
+Constraints
+===========
+
+It is beyond the scope of this document to fully describe all the FPGA design
+constraints required to make partial reconfiguration work[1] [2] [3], but a few
+deserve quick mention.
+
+A persona must have boundary connections that line up with those of the partion
+or region it is designed to go into.
+
+During programming, transactions through those connections must be stopped and
+the connections must be held at a fixed logic level.  This can be achieved by
+FPGA Bridges that exist on the FPGA fabric prior to the partial reconfiguration.
+
+--
+[1] www.altera.com/content/dam/altera-www/global/en_US/pdfs/literature/ug/ug_partrecon.pdf
+[2] tspace.library.utoronto.ca/bitstream/1807/67932/1/Byma_Stuart_A_201411_MAS_thesis.pdf
+[3] http://www.xilinx.com/support/documentation/sw_manuals/xilinx14_1/ug702.pdf
diff --git a/Documentation/devicetree/bindings/gpio/gpio-sx150x.txt b/Documentation/devicetree/bindings/gpio/gpio-sx150x.txt
deleted file mode 100644
index c809acb9c71b..000000000000
--- a/Documentation/devicetree/bindings/gpio/gpio-sx150x.txt
+++ /dev/null
@@ -1,41 +0,0 @@
-SEMTECH SX150x GPIO expander bindings
-
-
-Required properties:
-
-- compatible: should be "semtech,sx1506q",
-			"semtech,sx1508q",
-			"semtech,sx1509q",
-			"semtech,sx1502q".
-
-- reg: The I2C slave address for this device.
-
-- interrupt-parent: phandle of the parent interrupt controller.
-
-- interrupts: Interrupt specifier for the controllers interrupt.
-
-- #gpio-cells: Should be 2. The first cell is the GPIO number and the
-		second cell is used to specify optional parameters:
-		bit 0: polarity (0: normal, 1: inverted)
-
-- gpio-controller: Marks the device as a GPIO controller.
-
-- interrupt-controller: Marks the device as a interrupt controller.
-
-The GPIO expander can optionally be used as an interrupt controller, in
-which case it uses the default two cell specifier as described in
-Documentation/devicetree/bindings/interrupt-controller/interrupts.txt.
-
-Example:
-
-	i2c_gpio_expander@20{
-		#gpio-cells = <2>;
-		#interrupt-cells = <2>;
-		compatible = "semtech,sx1506q";
-		reg = <0x20>;
-		interrupt-parent = <&gpio_1>;
-		interrupts = <16 0>;
-
-		gpio-controller;
-		interrupt-controller;
-	};
diff --git a/Documentation/devicetree/bindings/gpio/gpio_oxnas.txt b/Documentation/devicetree/bindings/gpio/gpio_oxnas.txt
index 928ed4f43907..966514744df4 100644
--- a/Documentation/devicetree/bindings/gpio/gpio_oxnas.txt
+++ b/Documentation/devicetree/bindings/gpio/gpio_oxnas.txt
@@ -3,7 +3,7 @@
 Please refer to gpio.txt for generic information regarding GPIO bindings.
 
 Required properties:
- - compatible: "oxsemi,ox810se-gpio"
+ - compatible: "oxsemi,ox810se-gpio" or "oxsemi,ox820-gpio"
  - reg: Base address and length for the device.
  - interrupts: The port interrupt shared by all pins.
  - gpio-controller: Marks the port as GPIO controller.
diff --git a/Documentation/devicetree/bindings/gpio/mrvl-gpio.txt b/Documentation/devicetree/bindings/gpio/mrvl-gpio.txt
index c3d016532d8e..30fd2201b3d4 100644
--- a/Documentation/devicetree/bindings/gpio/mrvl-gpio.txt
+++ b/Documentation/devicetree/bindings/gpio/mrvl-gpio.txt
@@ -17,7 +17,9 @@ Required properties:
 - #interrupt-cells: Specifies the number of cells needed to encode an
   interrupt source.
 - gpio-controller : Marks the device node as a gpio controller.
-- #gpio-cells : Should be one.  It is the pin number.
+- #gpio-cells : Should be two.  The first cell is the pin number and
+  the second cell is used to specify flags. See gpio.txt for possible
+  values.
 
 Example for a MMP platform:
 
@@ -27,7 +29,7 @@ Example for a MMP platform:
 		interrupts = <49>;
 		interrupt-names = "gpio_mux";
 		gpio-controller;
-		#gpio-cells = <1>;
+		#gpio-cells = <2>;
 		interrupt-controller;
 		#interrupt-cells = <1>;
       };
diff --git a/Documentation/devicetree/bindings/hwmon/mcp3021.txt b/Documentation/devicetree/bindings/hwmon/mcp3021.txt
new file mode 100644
index 000000000000..294318ba6914
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/mcp3021.txt
@@ -0,0 +1,21 @@
+mcp3021 properties
+
+Required properties:
+- compatible: Must be one of the following:
+	- "microchip,mcp3021" for mcp3021
+	- "microchip,mcp3221" for mcp3221
+- reg: I2C address
+
+Optional properties:
+
+- reference-voltage-microvolt
+	Reference voltage in microvolt (uV)
+
+Example:
+
+mcp3021@4d {
+	compatible = "microchip,mcp3021";
+	reg = <0x4d>;
+
+	reference-voltage-microvolt = <4500000>; /* 4.5 V */
+};
diff --git a/Documentation/devicetree/bindings/hwmon/tmp108.txt b/Documentation/devicetree/bindings/hwmon/tmp108.txt
new file mode 100644
index 000000000000..8c4b10df86d9
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/tmp108.txt
@@ -0,0 +1,14 @@
+TMP108 temperature sensor
+-------------------------
+
+This device supports I2C only.
+
+Requires node properties:
+- compatible : "ti,tmp108"
+- reg : the I2C address of the device. This is 0x48, 0x49, 0x4a, or 0x4b.
+
+Example:
+	tmp108@48 {
+		compatible = "ti,tmp108";
+		reg = <0x48>;
+	};
diff --git a/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.txt b/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.txt
new file mode 100644
index 000000000000..70c054a9a997
--- /dev/null
+++ b/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.txt
@@ -0,0 +1,20 @@
+* Freescale Low Power Inter IC (LPI2C) for i.MX
+
+Required properties:
+- compatible :
+  - "fsl,imx7ulp-lpi2c" for LPI2C compatible with the one integrated on i.MX7ULP soc
+  - "fsl,imx8dv-lpi2c" for LPI2C compatible with the one integrated on i.MX8DV soc
+- reg : address and length of the lpi2c master registers
+- interrupt-parent : core interrupt controller
+- interrupts : lpi2c interrupt
+- clocks : lpi2c clock specifier
+
+Examples:
+
+lpi2c7: lpi2c7@40A50000 {
+	compatible = "fsl,imx8dv-lpi2c";
+	reg = <0x40A50000 0x10000>;
+	interrupt-parent = <&intc>;
+	interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
+	clocks = <&clks IMX7ULP_CLK_LPI2C7>;
+};
diff --git a/Documentation/devicetree/bindings/i2c/i2c-pxa.txt b/Documentation/devicetree/bindings/i2c/i2c-pxa.txt
index 12b78ac507e9..d30f0b11d853 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-pxa.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-pxa.txt
@@ -7,6 +7,7 @@ Required properties :
    compatible processor, e.g. pxa168, pxa910, mmp2, mmp3.
    For the pxa2xx/pxa3xx, an additional node "mrvl,pxa-i2c" is required
    as shown in the example below.
+   For the Armada 3700, the compatible should be "marvell,armada-3700-i2c".
 
 Recommended properties :
 
diff --git a/Documentation/devicetree/bindings/i2c/i2c-rcar.txt b/Documentation/devicetree/bindings/i2c/i2c-rcar.txt
index 239632a0d709..2b8bd33dbf8d 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-rcar.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-rcar.txt
@@ -1,17 +1,25 @@
 I2C for R-Car platforms
 
 Required properties:
-- compatible: Must be one of
-	"renesas,i2c-rcar"
-	"renesas,i2c-r8a7778"
-	"renesas,i2c-r8a7779"
-	"renesas,i2c-r8a7790"
-	"renesas,i2c-r8a7791"
-	"renesas,i2c-r8a7792"
-	"renesas,i2c-r8a7793"
-	"renesas,i2c-r8a7794"
-	"renesas,i2c-r8a7795"
-	"renesas,i2c-r8a7796"
+- compatible:
+	"renesas,i2c-r8a7778" if the device is a part of a R8A7778 SoC.
+	"renesas,i2c-r8a7779" if the device is a part of a R8A7779 SoC.
+	"renesas,i2c-r8a7790" if the device is a part of a R8A7790 SoC.
+	"renesas,i2c-r8a7791" if the device is a part of a R8A7791 SoC.
+	"renesas,i2c-r8a7792" if the device is a part of a R8A7792 SoC.
+	"renesas,i2c-r8a7793" if the device is a part of a R8A7793 SoC.
+	"renesas,i2c-r8a7794" if the device is a part of a R8A7794 SoC.
+	"renesas,i2c-r8a7795" if the device is a part of a R8A7795 SoC.
+	"renesas,i2c-r8a7796" if the device is a part of a R8A7796 SoC.
+	"renesas,rcar-gen1-i2c" for a generic R-Car Gen1 compatible device.
+	"renesas,rcar-gen2-i2c" for a generic R-Car Gen2 compatible device.
+	"renesas,rcar-gen3-i2c" for a generic R-Car Gen3 compatible device.
+	"renesas,i2c-rcar" (deprecated)
+
+	When compatible with the generic version, nodes must list the
+	SoC-specific version corresponding to the platform first followed
+	by the generic version.
+
 - reg: physical base address of the controller and length of memory mapped
   region.
 - interrupts: interrupt specifier.
@@ -33,7 +41,7 @@ Examples :
 i2c0: i2c@e6508000 {
 	#address-cells = <1>;
 	#size-cells = <0>;
-	compatible = "renesas,i2c-r8a7791";
+	compatible = "renesas,i2c-r8a7791", "renesas,rcar-gen2-i2c";
 	reg = <0 0xe6508000 0 0x40>;
 	interrupts = <0 287 IRQ_TYPE_LEVEL_HIGH>;
 	clocks = <&mstp9_clks R8A7791_CLK_I2C0>;
diff --git a/Documentation/devicetree/bindings/i2c/i2c-sh_mobile.txt b/Documentation/devicetree/bindings/i2c/i2c-sh_mobile.txt
index 214f94c25d37..7716acc55dec 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-sh_mobile.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-sh_mobile.txt
@@ -1,8 +1,7 @@
 Device tree configuration for Renesas IIC (sh_mobile) driver
 
 Required properties:
-- compatible      : "renesas,iic-<soctype>". "renesas,rmobile-iic" as fallback
-		     Examples with soctypes are:
+- compatible      :
 			- "renesas,iic-r8a73a4" (R-Mobile APE6)
 			- "renesas,iic-r8a7740" (R-Mobile A1)
 			- "renesas,iic-r8a7790" (R-Car H2)
@@ -12,6 +11,17 @@ Required properties:
 			- "renesas,iic-r8a7794" (R-Car E2)
 			- "renesas,iic-r8a7795" (R-Car H3)
 			- "renesas,iic-sh73a0" (SH-Mobile AG5)
+			- "renesas,rcar-gen2-iic" (generic R-Car Gen2 compatible device)
+			- "renesas,rcar-gen3-iic" (generic R-Car Gen3 compatible device)
+			- "renesas,rmobile-iic" (generic device)
+
+			When compatible with a generic R-Car version, nodes
+			must list the SoC-specific version corresponding to
+			the platform first followed by the generic R-Car
+			version.
+
+			renesas,rmobile-iic must always follow.
+
 - reg             : address start and address range size of device
 - interrupts      : interrupt of device
 - clocks          : clock for device
@@ -31,7 +41,8 @@ Pinctrl properties might be needed, too. See there.
 Example:
 
 	iic0: i2c@e6500000 {
-		compatible = "renesas,iic-r8a7790", "renesas,rmobile-iic";
+		compatible = "renesas,iic-r8a7790", "renesas,rcar-gen2-iic",
+			     "renesas,rmobile-iic";
 		reg = <0 0xe6500000 0 0x425>;
 		interrupts = <0 174 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp3_clks R8A7790_CLK_IIC0>;
diff --git a/Documentation/devicetree/bindings/i2c/trivial-devices.txt b/Documentation/devicetree/bindings/i2c/trivial-devices.txt
index fbbad6446741..cdd7b48826c3 100644
--- a/Documentation/devicetree/bindings/i2c/trivial-devices.txt
+++ b/Documentation/devicetree/bindings/i2c/trivial-devices.txt
@@ -39,11 +39,13 @@ dallas,ds75		Digital Thermometer and Thermostat
 dlg,da9053		DA9053: flexible system level PMIC with multicore support
 dlg,da9063		DA9063: system PMIC for quad-core application processors
 domintech,dmard09	DMARD09: 3-axis Accelerometer
+domintech,dmard10	DMARD10: 3-axis Accelerometer
 epson,rx8010		I2C-BUS INTERFACE REAL TIME CLOCK MODULE
 epson,rx8025		High-Stability. I2C-Bus INTERFACE REAL TIME CLOCK MODULE
 epson,rx8581		I2C-BUS INTERFACE REAL TIME CLOCK MODULE
 fsl,mag3110		MAG3110: Xtrinsic High Accuracy, 3D Magnetometer
 fsl,mc13892		MC13892: Power Management Integrated Circuit (PMIC) for i.MX35/51
+fsl,mma7660		MMA7660FC: 3-Axis Orientation/Motion Detection Sensor
 fsl,mma8450		MMA8450Q: Xtrinsic Low-power, 3-axis Xtrinsic Accelerometer
 fsl,mpl3115		MPL3115: Absolute Digital Pressure Sensor
 fsl,mpr121		MPR121: Proximity Capacitive Touch Sensor Controller
@@ -57,6 +59,7 @@ maxim,max1237		Low-Power, 4-/12-Channel, 2-Wire Serial, 12-Bit ADCs
 maxim,max6625		9-Bit/12-Bit Temperature Sensors with I²C-Compatible Serial Interface
 mc,rv3029c2		Real Time Clock Module with I2C-Bus
 mcube,mc3230		mCube 3-axis 8-bit digital accelerometer
+memsic,mxc6225		MEMSIC 2-axis 8-bit digital accelerometer
 microchip,mcp4531-502	Microchip 7-bit Single I2C Digital Potentiometer (5k)
 microchip,mcp4531-103	Microchip 7-bit Single I2C Digital Potentiometer (10k)
 microchip,mcp4531-503	Microchip 7-bit Single I2C Digital Potentiometer (50k)
@@ -121,6 +124,11 @@ microchip,mcp4662-502	Microchip 8-bit Dual I2C Digital Potentiometer with NV Mem
 microchip,mcp4662-103	Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (10k)
 microchip,mcp4662-503	Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (50k)
 microchip,mcp4662-104	Microchip 8-bit Dual I2C Digital Potentiometer with NV Memory (100k)
+microchip,tc654		PWM Fan Speed Controller With Fan Fault Detection
+microchip,tc655		PWM Fan Speed Controller With Fan Fault Detection
+miramems,da226		MiraMEMS DA226 2-axis 14-bit digital accelerometer
+miramems,da280		MiraMEMS DA280 3-axis 14-bit digital accelerometer
+miramems,da311		MiraMEMS DA311 3-axis 12-bit digital accelerometer
 national,lm63		Temperature sensor with integrated fan control
 national,lm75		I2C TEMP SENSOR
 national,lm80		Serial Interface ACPI-Compatible Microprocessor System Hardware Monitor
@@ -130,6 +138,8 @@ nuvoton,npct501		i2c trusted platform module (TPM)
 nuvoton,npct601		i2c trusted platform module (TPM2)
 nxp,pca9556		Octal SMBus and I2C registered interface
 nxp,pca9557		8-bit I2C-bus and SMBus I/O port with reset
+nxp,pcf2127		Real-time clock
+nxp,pcf2129		Real-time clock
 nxp,pcf8563		Real-time clock/calendar
 nxp,pcf85063		Tiny Real-Time Clock
 oki,ml86v7667		OKI ML86V7667 video decoder
@@ -146,6 +156,7 @@ ricoh,rv5c387a		I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC
 samsung,24ad0xd1	S524AD0XF1 (128K/256K-bit Serial EEPROM for Low Power)
 sgx,vz89x		SGX Sensortech VZ89X Sensors
 sii,s35390a		2-wire CMOS real-time clock
+silabs,si7020		Relative Humidity and Temperature Sensors
 skyworks,sky81452	Skyworks SKY81452: Six-Channel White LED Driver with Touch Panel Bias Supply
 st,24c256		i2c serial eeprom  (24cxx)
 st,m41t00		Serial real-time clock (RTC)
@@ -158,4 +169,5 @@ ti,tsc2003		I2C Touch-Screen Controller
 ti,tmp102		Low Power Digital Temperature Sensor with SMBUS/Two Wire Serial Interface
 ti,tmp103		Low Power Digital Temperature Sensor with SMBUS/Two Wire Serial Interface
 ti,tmp275		Digital Temperature Sensor
+winbond,w83793		Winbond/Nuvoton H/W Monitor
 winbond,wpct301		i2c trusted platform module (TPM)
diff --git a/Documentation/devicetree/bindings/iio/adc/envelope-detector.txt b/Documentation/devicetree/bindings/iio/adc/envelope-detector.txt
new file mode 100644
index 000000000000..27544bdd4478
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/envelope-detector.txt
@@ -0,0 +1,54 @@
+Bindings for ADC envelope detector using a DAC and a comparator
+
+The DAC is used to find the peak level of an alternating voltage input
+signal by a binary search using the output of a comparator wired to
+an interrupt pin. Like so:
+                          _
+                         | \
+    input +------>-------|+ \
+                         |   \
+           .-------.     |    }---.
+           |       |     |   /    |
+           |    dac|-->--|- /     |
+           |       |     |_/      |
+           |       |              |
+           |       |              |
+           |    irq|------<-------'
+           |       |
+           '-------'
+
+Required properties:
+- compatible: Should be "axentia,tse850-envelope-detector"
+- io-channels: Channel node of the dac to be used for comparator input.
+- io-channel-names: Should be "dac".
+- interrupt specification for one client interrupt,
+  see ../../interrupt-controller/interrupts.txt for details.
+- interrupt-names: Should be "comp".
+
+Example:
+
+	&i2c {
+		dpot: mcp4651-104@28 {
+			compatible = "microchip,mcp4651-104";
+			reg = <0x28>;
+			#io-channel-cells = <1>;
+		};
+	};
+
+	dac: dac {
+		compatible = "dpot-dac";
+		vref-supply = <&reg_3v3>;
+		io-channels = <&dpot 0>;
+		io-channel-names = "dpot";
+		#io-channel-cells = <1>;
+	};
+
+	envelope-detector {
+		compatible = "axentia,tse850-envelope-detector";
+		io-channels = <&dac 0>;
+		io-channel-names = "dac";
+
+		interrupt-parent = <&gpio>;
+		interrupts = <3 IRQ_TYPE_EDGE_FALLING>;
+		interrupt-names = "comp";
+	};
diff --git a/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.txt b/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.txt
new file mode 100644
index 000000000000..49ed82e89870
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.txt
@@ -0,0 +1,83 @@
+STMicroelectronics STM32 ADC device driver
+
+STM32 ADC is a successive approximation analog-to-digital converter.
+It has several multiplexed input channels. Conversions can be performed
+in single, continuous, scan or discontinuous mode. Result of the ADC is
+stored in a left-aligned or right-aligned 32-bit data register.
+Conversions can be launched in software or using hardware triggers.
+
+The analog watchdog feature allows the application to detect if the input
+voltage goes beyond the user-defined, higher or lower thresholds.
+
+Each STM32 ADC block can have up to 3 ADC instances.
+
+Each instance supports two contexts to manage conversions, each one has its
+own configurable sequence and trigger:
+- regular conversion can be done in sequence, running in background
+- injected conversions have higher priority, and so have the ability to
+  interrupt regular conversion sequence (either triggered in SW or HW).
+  Regular sequence is resumed, in case it has been interrupted.
+
+Contents of a stm32 adc root node:
+-----------------------------------
+Required properties:
+- compatible: Should be "st,stm32f4-adc-core".
+- reg: Offset and length of the ADC block register set.
+- interrupts: Must contain the interrupt for ADC block.
+- clocks: Clock for the analog circuitry (common to all ADCs).
+- clock-names: Must be "adc".
+- interrupt-controller: Identifies the controller node as interrupt-parent
+- vref-supply: Phandle to the vref input analog reference voltage.
+- #interrupt-cells = <1>;
+- #address-cells = <1>;
+- #size-cells = <0>;
+
+Optional properties:
+- A pinctrl state named "default" for each ADC channel may be defined to set
+  inX ADC pins in mode of operation for analog input on external pin.
+
+Contents of a stm32 adc child node:
+-----------------------------------
+An ADC block node should contain at least one subnode, representing an
+ADC instance available on the machine.
+
+Required properties:
+- compatible: Should be "st,stm32f4-adc".
+- reg: Offset of ADC instance in ADC block (e.g. may be 0x0, 0x100, 0x200).
+- clocks: Input clock private to this ADC instance.
+- interrupt-parent: Phandle to the parent interrupt controller.
+- interrupts: IRQ Line for the ADC (e.g. may be 0 for adc@0, 1 for adc@100 or
+  2 for adc@200).
+- st,adc-channels: List of single-ended channels muxed for this ADC.
+  It can have up to 16 channels, numbered from 0 to 15 (resp. for in0..in15).
+- #io-channel-cells = <1>: See the IIO bindings section "IIO consumers" in
+  Documentation/devicetree/bindings/iio/iio-bindings.txt
+
+Example:
+	adc: adc@40012000 {
+		compatible = "st,stm32f4-adc-core";
+		reg = <0x40012000 0x400>;
+		interrupts = <18>;
+		clocks = <&rcc 0 168>;
+		clock-names = "adc";
+		vref-supply = <&reg_vref>;
+		interrupt-controller;
+		pinctrl-names = "default";
+		pinctrl-0 = <&adc3_in8_pin>;
+
+		#interrupt-cells = <1>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		adc@0 {
+			compatible = "st,stm32f4-adc";
+			#io-channel-cells = <1>;
+			reg = <0x0>;
+			clocks = <&rcc 0 168>;
+			interrupt-parent = <&adc>;
+			interrupts = <0>;
+			st,adc-channels = <8>;
+		};
+		...
+		other adc child nodes follow...
+	};
diff --git a/Documentation/devicetree/bindings/iio/adc/ti-adc161s626.txt b/Documentation/devicetree/bindings/iio/adc/ti-adc161s626.txt
index 9ed2315781e4..3d25011f0c99 100644
--- a/Documentation/devicetree/bindings/iio/adc/ti-adc161s626.txt
+++ b/Documentation/devicetree/bindings/iio/adc/ti-adc161s626.txt
@@ -3,6 +3,7 @@
 Required properties:
  - compatible: Should be "ti,adc141s626" or "ti,adc161s626"
  - reg: spi chip select number for the device
+ - vdda-supply: supply voltage to VDDA pin
 
 Recommended properties:
  - spi-max-frequency: Definition as per
@@ -11,6 +12,7 @@ Recommended properties:
 Example:
 adc@0 {
 	compatible = "ti,adc161s626";
+	vdda-supply = <&vdda_fixed>;
 	reg = <0>;
 	spi-max-frequency = <4300000>;
 };
diff --git a/Documentation/devicetree/bindings/iio/dac/dpot-dac.txt b/Documentation/devicetree/bindings/iio/dac/dpot-dac.txt
new file mode 100644
index 000000000000..fdf47a01bfef
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/dac/dpot-dac.txt
@@ -0,0 +1,41 @@
+Bindings for DAC emulation using a digital potentiometer
+
+It is assumed that the dpot is used as a voltage divider between the
+current dpot wiper setting and the maximum resistance of the dpot. The
+divided voltage is provided by a vref regulator.
+
+                  .------.
+   .-----------.  |      |
+   | vref      |--'    .---.
+   | regulator |--.    |   |
+   '-----------'  |    | d |
+                  |    | p |
+                  |    | o |  wiper
+                  |    | t |<---------+
+                  |    |   |
+                  |    '---'       dac output voltage
+                  |      |
+                  '------+------------+
+
+Required properties:
+- compatible: Should be "dpot-dac"
+- vref-supply: The regulator supplying the voltage divider.
+- io-channels: Channel node of the dpot to be used for the voltage division.
+- io-channel-names: Should be "dpot".
+
+Example:
+
+	&i2c {
+		dpot: mcp4651-503@28 {
+			compatible = "microchip,mcp4651-503";
+			reg = <0x28>;
+			#io-channel-cells = <1>;
+		};
+	};
+
+	dac {
+		compatible = "dpot-dac";
+		vref-supply = <&reg_3v3>;
+		io-channels = <&dpot 0>;
+		io-channel-names = "dpot";
+	};
diff --git a/Documentation/devicetree/bindings/iio/dac/mcp4725.txt b/Documentation/devicetree/bindings/iio/dac/mcp4725.txt
new file mode 100644
index 000000000000..1bc6c093fbfe
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/dac/mcp4725.txt
@@ -0,0 +1,35 @@
+Microchip mcp4725 and mcp4726 DAC device driver
+
+Required properties:
+	- compatible: Must be "microchip,mcp4725" or "microchip,mcp4726"
+	- reg: Should contain the DAC I2C address
+	- vdd-supply: Phandle to the Vdd power supply. This supply is used as a
+	  voltage reference on mcp4725. It is used as a voltage reference on
+	  mcp4726 if there is no vref-supply specified.
+
+Optional properties (valid only for mcp4726):
+	- vref-supply: Optional phandle to the Vref power supply. Vref pin is
+	  used as a voltage reference when this supply is specified.
+	- microchip,vref-buffered: Boolean to enable buffering of the external
+	  Vref pin. This boolean is not valid without the vref-supply. Quoting
+	  the datasheet: This is offered in cases where the reference voltage
+	  does not have the current capability not to drop its voltage when
+	  connected to the internal resistor ladder circuit.
+
+Examples:
+
+	/* simple mcp4725 */
+	mcp4725@60 {
+		compatible = "microchip,mcp4725";
+		reg = <0x60>;
+		vdd-supply = <&vdac_vdd>;
+	};
+
+	/* mcp4726 with the buffered external reference voltage */
+	mcp4726@60 {
+		compatible = "microchip,mcp4726";
+		reg = <0x60>;
+		vdd-supply = <&vdac_vdd>;
+		vref-supply = <&vdac_vref>;
+		microchip,vref-buffered;
+	};
diff --git a/Documentation/devicetree/bindings/iio/gyroscope/invensense,mpu3050.txt b/Documentation/devicetree/bindings/iio/gyroscope/invensense,mpu3050.txt
new file mode 100644
index 000000000000..b0d3b59966bc
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/gyroscope/invensense,mpu3050.txt
@@ -0,0 +1,46 @@
+Invensense MPU-3050 Gyroscope device tree bindings
+
+Required properties:
+  - compatible : should be "invensense,mpu3050"
+  - reg : the I2C address of the sensor
+
+Optional properties:
+  - interrupt-parent : should be the phandle for the interrupt controller
+  - interrupts : interrupt mapping for the trigger interrupt from the
+    internal oscillator. The following IRQ modes are supported:
+    IRQ_TYPE_EDGE_RISING, IRQ_TYPE_EDGE_FALLING, IRQ_TYPE_LEVEL_HIGH and
+    IRQ_TYPE_LEVEL_LOW. The driver should detect and configure the hardware
+    for the desired interrupt type.
+  - vdd-supply : supply regulator for the main power voltage.
+  - vlogic-supply : supply regulator for the signal voltage.
+  - mount-matrix : see iio/mount-matrix.txt
+
+Optional subnodes:
+  - The MPU-3050 will pass through and forward the I2C signals from the
+    incoming I2C bus, alternatively drive traffic to a slave device (usually
+    an accelerometer) on its own initiative. Therefore is supports a subnode
+    i2c gate node. For details see: i2c/i2c-gate.txt
+
+Example:
+
+mpu3050@68 {
+	compatible = "invensense,mpu3050";
+	reg = <0x68>;
+	interrupt-parent = <&foo>;
+	interrupts = <12 IRQ_TYPE_EDGE_FALLING>;
+	vdd-supply = <&bar>;
+	vlogic-supply = <&baz>;
+
+	/* External I2C interface */
+	i2c-gate {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		fnord@18 {
+			compatible = "fnord";
+			reg = <0x18>;
+			interrupt-parent = <&foo>;
+			interrupts = <13 IRQ_TYPE_EDGE_FALLING>;
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/iio/humidity/hts221.txt b/Documentation/devicetree/bindings/iio/humidity/hts221.txt
new file mode 100644
index 000000000000..b20ab9c12080
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/humidity/hts221.txt
@@ -0,0 +1,22 @@
+* HTS221 STM humidity + temperature sensor
+
+Required properties:
+- compatible: should be "st,hts221"
+- reg: i2c address of the sensor / spi cs line
+
+Optional properties:
+- interrupt-parent: should be the phandle for the interrupt controller
+- interrupts: interrupt mapping for IRQ. It should be configured with
+  flags IRQ_TYPE_LEVEL_HIGH or IRQ_TYPE_EDGE_RISING.
+
+  Refer to interrupt-controller/interrupts.txt for generic interrupt
+  client node bindings.
+
+Example:
+
+hts221@5f {
+	compatible = "st,hts221";
+	reg = <0x5f>;
+	interrupt-parent = <&gpio0>;
+	interrupts = <0 IRQ_TYPE_EDGE_RISING>;
+};
diff --git a/Documentation/devicetree/bindings/iio/light/isl29018.txt b/Documentation/devicetree/bindings/iio/light/isl29018.txt
new file mode 100644
index 000000000000..425ab459e209
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/light/isl29018.txt
@@ -0,0 +1,28 @@
+* ISL 29018/29023/29035 I2C ALS, Proximity, and Infrared sensor
+
+Required properties:
+
+  - compatible: Should be one of
+		"isil,isl29018"
+		"isil,isl29023"
+		"isil,isl29035"
+  - reg: the I2C address of the device
+
+Optional properties:
+
+  - interrupt-parent: should be the phandle for the interrupt controller
+  - interrupts: the sole interrupt generated by the device
+
+  Refer to interrupt-controller/interrupts.txt for generic interrupt client
+  node bindings.
+
+  - vcc-supply: phandle to the regulator that provides power to the sensor.
+
+Example:
+
+isl29018@44 {
+	compatible = "isil,isl29018";
+	reg = <0x44>;
+	interrupt-parent = <&gpio>;
+	interrupts = <TEGRA_GPIO(Z, 2) IRQ_TYPE_LEVEL_HIGH>;
+};
diff --git a/Documentation/devicetree/bindings/iio/light/tsl2583.txt b/Documentation/devicetree/bindings/iio/light/tsl2583.txt
new file mode 100644
index 000000000000..8e2066c83f70
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/light/tsl2583.txt
@@ -0,0 +1,26 @@
+* TAOS TSL 2580/2581/2583 ALS sensor
+
+Required properties:
+
+  - compatible: Should be one of
+		"amstaos,tsl2580"
+		"amstaos,tsl2581"
+		"amstaos,tsl2583"
+  - reg: the I2C address of the device
+
+Optional properties:
+
+  - interrupt-parent: should be the phandle for the interrupt controller
+  - interrupts: the sole interrupt generated by the device
+
+  Refer to interrupt-controller/interrupts.txt for generic interrupt client
+  node bindings.
+
+  - vcc-supply: phandle to the regulator that provides power to the sensor.
+
+Example:
+
+tsl2581@29 {
+	compatible = "amstaos,tsl2581";
+	reg = <0x29>;
+};
diff --git a/Documentation/devicetree/bindings/iio/potentiostat/lmp91000.txt b/Documentation/devicetree/bindings/iio/potentiostat/lmp91000.txt
new file mode 100644
index 000000000000..b9b621e94cd7
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/potentiostat/lmp91000.txt
@@ -0,0 +1,30 @@
+* Texas Instruments LMP91000 potentiostat
+
+http://www.ti.com/lit/ds/symlink/lmp91000.pdf
+
+Required properties:
+
+  - compatible: should be "ti,lmp91000"
+  - reg: the I2C address of the device
+  - io-channels: the phandle of the iio provider
+
+  - ti,external-tia-resistor: if the property ti,tia-gain-ohm is not defined this
+    needs to be set to signal that an external resistor value is being used.
+
+Optional properties:
+
+  - ti,tia-gain-ohm: ohm value of the internal resistor for the transimpedance
+    amplifier. Must be 2750, 3500, 7000, 14000, 35000, 120000, or 350000 ohms.
+
+  - ti,rload-ohm: ohm value of the internal resistor load applied to the gas
+    sensor. Must be 10, 33, 50, or 100 (default) ohms.
+
+Example:
+
+lmp91000@48 {
+	compatible = "ti,lmp91000";
+	reg = <0x48>;
+	ti,tia-gain-ohm = <7500>;
+	ti,rload = <100>;
+	io-channels = <&adc>;
+};
diff --git a/Documentation/devicetree/bindings/iio/st-sensors.txt b/Documentation/devicetree/bindings/iio/st-sensors.txt
index e41fe340162b..c040c9ad1889 100644
--- a/Documentation/devicetree/bindings/iio/st-sensors.txt
+++ b/Documentation/devicetree/bindings/iio/st-sensors.txt
@@ -42,6 +42,7 @@ Accelerometers:
 - st,lsm303agr-accel
 - st,lis2dh12-accel
 - st,h3lis331dl-accel
+- st,lng2dm-accel
 
 Gyroscopes:
 - st,l3g4200d-gyro
diff --git a/Documentation/devicetree/bindings/leds/pca963x.txt b/Documentation/devicetree/bindings/leds/pca963x.txt
index dafbe9931c2b..dfbdb123a9bf 100644
--- a/Documentation/devicetree/bindings/leds/pca963x.txt
+++ b/Documentation/devicetree/bindings/leds/pca963x.txt
@@ -7,6 +7,9 @@ Optional properties:
 - nxp,totem-pole : use totem pole (push-pull) instead of open-drain (pca9632 defaults
   to open-drain, newer chips to totem pole)
 - nxp,hw-blink : use hardware blinking instead of software blinking
+- nxp,period-scale : In some configurations, the chip blinks faster than expected.
+		     This parameter provides a scaling ratio (fixed point, decimal divided
+		     by 1000) to compensate, e.g. 1300=1.3x and 750=0.75x.
 
 Each led is represented as a sub-node of the nxp,pca963x device.
 
diff --git a/Documentation/devicetree/bindings/mailbox/brcm,bcm2835-mbox.txt b/Documentation/devicetree/bindings/mailbox/brcm,bcm2835-mbox.txt
index e893615ef635..b48d7d30012c 100644
--- a/Documentation/devicetree/bindings/mailbox/brcm,bcm2835-mbox.txt
+++ b/Documentation/devicetree/bindings/mailbox/brcm,bcm2835-mbox.txt
@@ -12,7 +12,7 @@ Required properties:
 
 Example:
 
-mailbox: mailbox@7e00b800 {
+mailbox: mailbox@7e00b880 {
 	compatible = "brcm,bcm2835-mbox";
 	reg = <0x7e00b880 0x40>;
 	interrupts = <0 1>;
diff --git a/Documentation/devicetree/bindings/mailbox/nvidia,tegra186-hsp.txt b/Documentation/devicetree/bindings/mailbox/nvidia,tegra186-hsp.txt
new file mode 100644
index 000000000000..b99d25fc2f26
--- /dev/null
+++ b/Documentation/devicetree/bindings/mailbox/nvidia,tegra186-hsp.txt
@@ -0,0 +1,52 @@
+NVIDIA Tegra Hardware Synchronization Primitives (HSP)
+
+The HSP modules are used for the processors to share resources and communicate
+together. It provides a set of hardware synchronization primitives for
+interprocessor communication. So the interprocessor communication (IPC)
+protocols can use hardware synchronization primitives, when operating between
+two processors not in an SMP relationship.
+
+The features that HSP supported are shared mailboxes, shared semaphores,
+arbitrated semaphores and doorbells.
+
+Required properties:
+- name : Should be hsp
+- compatible
+    Array of strings.
+    one of:
+    - "nvidia,tegra186-hsp"
+- reg : Offset and length of the register set for the device.
+- interrupt-names
+    Array of strings.
+    Contains a list of names for the interrupts described by the interrupt
+    property. May contain the following entries, in any order:
+    - "doorbell"
+    Users of this binding MUST look up entries in the interrupt property
+    by name, using this interrupt-names property to do so.
+- interrupts
+    Array of interrupt specifiers.
+    Must contain one entry per entry in the interrupt-names property,
+    in a matching order.
+- #mbox-cells : Should be 2.
+
+The mbox specifier of the "mboxes" property in the client node should
+contain two data. The first one should be the HSP type and the second
+one should be the ID that the client is going to use. Those information
+can be found in the following file.
+
+- <dt-bindings/mailbox/tegra186-hsp.h>.
+
+Example:
+
+hsp_top0: hsp@3c00000 {
+	compatible = "nvidia,tegra186-hsp";
+	reg = <0x0 0x03c00000 0x0 0xa0000>;
+	interrupts = <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>;
+	interrupt-names = "doorbell";
+	#mbox-cells = <2>;
+};
+
+client {
+	...
+	mboxes = <&hsp_top0 TEGRA_HSP_MBOX_TYPE_DB TEGRA_HSP_DB_MASTER_XXX>;
+};
diff --git a/Documentation/devicetree/bindings/media/renesas,fcp.txt b/Documentation/devicetree/bindings/media/renesas,fcp.txt
index 27f9b8e459ac..3ec91803ba58 100644
--- a/Documentation/devicetree/bindings/media/renesas,fcp.txt
+++ b/Documentation/devicetree/bindings/media/renesas,fcp.txt
@@ -11,15 +11,9 @@ are paired with. These DT bindings currently support the FCPV and FCPF.
 
  - compatible: Must be one or more of the following
 
-   - "renesas,r8a7795-fcpv" for R8A7795 (R-Car H3) compatible 'FCP for VSP'
-   - "renesas,r8a7795-fcpf" for R8A7795 (R-Car H3) compatible 'FCP for FDP'
    - "renesas,fcpv" for generic compatible 'FCP for VSP'
    - "renesas,fcpf" for generic compatible 'FCP for FDP'
 
-   When compatible with the generic version, nodes must list the
-   SoC-specific version corresponding to the platform first, followed by the
-   family-specific and/or generic versions.
-
  - reg: the register base and size for the device registers
  - clocks: Reference to the functional clock
 
@@ -32,7 +26,7 @@ Device node example
 -------------------
 
 	fcpvd1: fcp@fea2f000 {
-		compatible = "renesas,r8a7795-fcpv", "renesas,fcpv";
+		compatible = "renesas,fcpv";
 		reg = <0 0xfea2f000 0 0x200>;
 		clocks = <&cpg CPG_MOD 602>;
 		power-domains = <&sysc R8A7795_PD_A3VP>;
diff --git a/Documentation/devicetree/bindings/memory-controllers/ti-da8xx-ddrctl.txt b/Documentation/devicetree/bindings/memory-controllers/ti-da8xx-ddrctl.txt
new file mode 100644
index 000000000000..ec1dd408d573
--- /dev/null
+++ b/Documentation/devicetree/bindings/memory-controllers/ti-da8xx-ddrctl.txt
@@ -0,0 +1,20 @@
+* Device tree bindings for Texas Instruments da8xx DDR2/mDDR memory controller
+
+The DDR2/mDDR memory controller present on Texas Instruments da8xx SoCs features
+a set of registers which allow to tweak the controller's behavior.
+
+Documentation:
+OMAP-L138 (DA850) - http://www.ti.com/lit/ug/spruh82c/spruh82c.pdf
+
+Required properties:
+
+- compatible:		"ti,da850-ddr-controller" - for da850 SoC based boards
+- reg:			a tuple containing the base address of the memory
+			controller and the size of the memory area to map
+
+Example for da850 shown below.
+
+ddrctl {
+	compatible = "ti,da850-ddr-controller";
+	reg = <0xb0000000 0xe8>;
+};
diff --git a/Documentation/devicetree/bindings/mfd/lp873x.txt b/Documentation/devicetree/bindings/mfd/lp873x.txt
index 52766c2035f7..ae9cf39bd101 100644
--- a/Documentation/devicetree/bindings/mfd/lp873x.txt
+++ b/Documentation/devicetree/bindings/mfd/lp873x.txt
@@ -7,6 +7,9 @@ Required properties:
   - #gpio-cells:	Should be two.  The first cell is the pin number and
 			the second cell is used to specify flags.
 			See ../gpio/gpio.txt for more information.
+  - xxx-in-supply:	Phandle to parent supply node of each regulator
+			populated under regulators node. xxx can be
+			buck0, buck1, ldo0 or ldo1.
   - regulators:	List of child nodes that specify the regulator
 			initialization data.
 Example:
@@ -17,6 +20,11 @@ pmic: lp8733@60 {
 	gpio-controller;
 	#gpio-cells = <2>;
 
+	buck0-in-supply = <&vsys_3v3>;
+	buck1-in-supply = <&vsys_3v3>;
+	ldo0-in-supply = <&vsys_3v3>;
+	ldo1-in-supply = <&vsys_3v3>;
+
 	regulators {
 		lp8733_buck0: buck0 {
 			regulator-name = "lp8733-buck0";
diff --git a/Documentation/devicetree/bindings/mfd/max77620.txt b/Documentation/devicetree/bindings/mfd/max77620.txt
index 2ad44f7e4880..9c16d51cc15b 100644
--- a/Documentation/devicetree/bindings/mfd/max77620.txt
+++ b/Documentation/devicetree/bindings/mfd/max77620.txt
@@ -106,6 +106,18 @@ Here supported time periods by device in microseconds are as follows:
 MAX77620 supports 40, 80, 160, 320, 640, 1280, 2560 and 5120 microseconds.
 MAX20024 supports 20, 40, 80, 160, 320, 640, 1280 and 2540 microseconds.
 
+-maxim,power-ok-control: configure map power ok bit
+			1: Enables POK(Power OK) to control nRST_IO and GPIO1
+			POK function.
+			0: Disables POK control.
+			if property missing, do not configure MPOK bit.
+			If POK mapping is enabled for GPIO1/nRST_IO then,
+			GPIO1/nRST_IO pins are HIGH only if all rails
+			that have POK control enabled are HIGH.
+			If any of the rails goes down(which are enabled for POK
+			control) then, GPIO1/nRST_IO goes LOW.
+			this property is valid for max20024 only.
+
 For DT binding details of different sub modules like GPIO, pincontrol,
 regulator, power, please refer respective device-tree binding document
 under their respective sub-system directories.
diff --git a/Documentation/devicetree/bindings/mfd/tps65086.txt b/Documentation/devicetree/bindings/mfd/tps65086.txt
index d3705612a846..9cfa886fe99f 100644
--- a/Documentation/devicetree/bindings/mfd/tps65086.txt
+++ b/Documentation/devicetree/bindings/mfd/tps65086.txt
@@ -23,7 +23,7 @@ Required properties:
                             defined below.
 
 Optional regulator properties:
- - ti,regulator-step-size-25mv	: This is applicable for buck[1,2,6], set this
+ - ti,regulator-step-size-25mv	: This is applicable for buck[1-6], set this
 				    if the regulator is factory set with a 25mv
 				    step voltage mapping.
  - ti,regulator-decay		: This is applicable for buck[1-6], set this if
diff --git a/Documentation/devicetree/bindings/mmc/amlogic,meson-gx.txt b/Documentation/devicetree/bindings/mmc/amlogic,meson-gx.txt
new file mode 100644
index 000000000000..7f95ec400863
--- /dev/null
+++ b/Documentation/devicetree/bindings/mmc/amlogic,meson-gx.txt
@@ -0,0 +1,32 @@
+Amlogic SD / eMMC controller for S905/GXBB family SoCs
+
+The MMC 5.1 compliant host controller on Amlogic provides the
+interface for SD, eMMC and SDIO devices.
+
+This file documents the properties in addition to those available in
+the MMC core bindings, documented by mmc.txt.
+
+Required properties:
+- compatible : contains one of:
+  - "amlogic,meson-gx-mmc"
+  - "amlogic,meson-gxbb-mmc"
+  - "amlogic,meson-gxl-mmc"
+  - "amlogic,meson-gxm-mmc"
+- clocks     : A list of phandle + clock-specifier pairs for the clocks listed in clock-names.
+- clock-names: Should contain the following:
+	"core" - Main peripheral bus clock
+	"clkin0" - Parent clock of internal mux
+	"clkin1" - Other parent clock of internal mux
+  The driver has an interal mux clock which switches between clkin0 and clkin1 depending on the
+  clock rate requested by the MMC core.
+
+Example:
+
+	sd_emmc_a: mmc@70000 {
+		compatible = "amlogic,meson-gxbb-mmc";
+		reg = <0x0 0x70000 0x0 0x2000>;
+		interrupts = < GIC_SPI 216 IRQ_TYPE_EDGE_RISING>;
+		clocks = <&clkc CLKID_SD_EMMC_A>, <&xtal>, <&clkc CLKID_FCLK_DIV2>;
+		clock-names = "core", "clkin0", "clkin1";
+		pinctrl-0 = <&emmc_pins>;
+	};
diff --git a/Documentation/devicetree/bindings/mmc/brcm,sdhci-iproc.txt b/Documentation/devicetree/bindings/mmc/brcm,sdhci-iproc.txt
index be56d2bd474a..954561d09a8e 100644
--- a/Documentation/devicetree/bindings/mmc/brcm,sdhci-iproc.txt
+++ b/Documentation/devicetree/bindings/mmc/brcm,sdhci-iproc.txt
@@ -7,6 +7,15 @@ Required properties:
 - compatible : Should be one of the following
 	       "brcm,bcm2835-sdhci"
 	       "brcm,sdhci-iproc-cygnus"
+	       "brcm,sdhci-iproc"
+
+Use brcm2835-sdhci for Rasperry PI.
+
+Use sdhci-iproc-cygnus for Broadcom SDHCI Controllers
+restricted to 32bit host accesses to SDHCI registers.
+
+Use sdhci-iproc for Broadcom SDHCI Controllers that allow standard
+8, 16, 32-bit host access to SDHCI register.
 
 - clocks : The clock feeding the SDHCI controller.
 
diff --git a/Documentation/devicetree/bindings/mmc/renesas,mmcif.txt b/Documentation/devicetree/bindings/mmc/renesas,mmcif.txt
index ff611fa66871..e4ba92aa035e 100644
--- a/Documentation/devicetree/bindings/mmc/renesas,mmcif.txt
+++ b/Documentation/devicetree/bindings/mmc/renesas,mmcif.txt
@@ -8,11 +8,14 @@ Required properties:
 
 - compatible: should be "renesas,mmcif-<soctype>", "renesas,sh-mmcif" as a
   fallback. Examples with <soctype> are:
+	- "renesas,mmcif-r8a73a4" for the MMCIF found in r8a73a4 SoCs
 	- "renesas,mmcif-r8a7740" for the MMCIF found in r8a7740 SoCs
+	- "renesas,mmcif-r8a7778" for the MMCIF found in r8a7778 SoCs
 	- "renesas,mmcif-r8a7790" for the MMCIF found in r8a7790 SoCs
 	- "renesas,mmcif-r8a7791" for the MMCIF found in r8a7791 SoCs
 	- "renesas,mmcif-r8a7793" for the MMCIF found in r8a7793 SoCs
 	- "renesas,mmcif-r8a7794" for the MMCIF found in r8a7794 SoCs
+	- "renesas,mmcif-sh73a0" for the MMCIF found in sh73a0 SoCs
 
 - clocks: reference to the functional clock
 
diff --git a/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.txt b/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.txt
index 07184e8f894e..ea9c1c9607f6 100644
--- a/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.txt
+++ b/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.txt
@@ -13,6 +13,7 @@ Required Properties:
 	- "rockchip,rk2928-dw-mshc": for Rockchip RK2928 and following,
 							before RK3288
 	- "rockchip,rk3288-dw-mshc": for Rockchip RK3288
+	- "rockchip,rk1108-dw-mshc", "rockchip,rk3288-dw-mshc": for Rockchip RK1108
 	- "rockchip,rk3036-dw-mshc", "rockchip,rk3288-dw-mshc": for Rockchip RK3036
 	- "rockchip,rk3368-dw-mshc", "rockchip,rk3288-dw-mshc": for Rockchip RK3368
 	- "rockchip,rk3399-dw-mshc", "rockchip,rk3288-dw-mshc": for Rockchip RK3399
diff --git a/Documentation/devicetree/bindings/mmc/sdhci-cadence.txt b/Documentation/devicetree/bindings/mmc/sdhci-cadence.txt
new file mode 100644
index 000000000000..750374fc9d94
--- /dev/null
+++ b/Documentation/devicetree/bindings/mmc/sdhci-cadence.txt
@@ -0,0 +1,30 @@
+* Cadence SD/SDIO/eMMC Host Controller
+
+Required properties:
+- compatible: should be "cdns,sd4hc".
+- reg: offset and length of the register set for the device.
+- interrupts: a single interrupt specifier.
+- clocks: phandle to the input clock.
+
+Optional properties:
+For eMMC configuration, supported speed modes are not indicated by the SDHCI
+Capabilities Register.  Instead, the following properties should be specified
+if supported.  See mmc.txt for details.
+- mmc-ddr-1_8v
+- mmc-ddr-1_2v
+- mmc-hs200-1_8v
+- mmc-hs200-1_2v
+- mmc-hs400-1_8v
+- mmc-hs400-1_2v
+
+Example:
+	emmc: sdhci@5a000000 {
+		compatible = "cdns,sd4hc";
+		reg = <0x5a000000 0x400>;
+		interrupts = <0 78 4>;
+		clocks = <&clk 4>;
+		bus-width = <8>;
+		mmc-ddr-1_8v;
+		mmc-hs200-1_8v;
+		mmc-hs400-1_8v;
+	};
diff --git a/Documentation/devicetree/bindings/mmc/sdhci-msm.txt b/Documentation/devicetree/bindings/mmc/sdhci-msm.txt
index 485483a63d8c..0576264eab5e 100644
--- a/Documentation/devicetree/bindings/mmc/sdhci-msm.txt
+++ b/Documentation/devicetree/bindings/mmc/sdhci-msm.txt
@@ -17,6 +17,7 @@ Required properties:
 	"iface" - Main peripheral bus clock (PCLK/HCLK - AHB Bus clock) (required)
 	"core"	- SDC MMC clock (MCLK) (required)
 	"bus"	- SDCC bus voter clock (optional)
+	"xo"	- TCXO clock (optional)
 
 Example:
 
diff --git a/Documentation/devicetree/bindings/mmc/sdhci.txt b/Documentation/devicetree/bindings/mmc/sdhci.txt
new file mode 100644
index 000000000000..1c95a1a555c3
--- /dev/null
+++ b/Documentation/devicetree/bindings/mmc/sdhci.txt
@@ -0,0 +1,13 @@
+The properties specific for SD host controllers. For properties shared by MMC
+host controllers refer to the mmc[1] bindings.
+
+  [1] Documentation/devicetree/bindings/mmc/mmc.txt
+
+Optional properties:
+- sdhci-caps-mask: The sdhci capabilities register is incorrect. This 64bit
+  property corresponds to the bits in the sdhci capabilty register. If the bit
+  is on in the mask then the bit is incorrect in the register and should be
+  turned off, before applying sdhci-caps.
+- sdhci-caps: The sdhci capabilities register is incorrect. This 64bit
+  property corresponds to the bits in the sdhci capability register. If the
+  bit is on in the property then the bit should be turned on.
diff --git a/Documentation/devicetree/bindings/mmc/synopsys-dw-mshc.txt b/Documentation/devicetree/bindings/mmc/synopsys-dw-mshc.txt
index bfa461aaac99..7fd17c3da116 100644
--- a/Documentation/devicetree/bindings/mmc/synopsys-dw-mshc.txt
+++ b/Documentation/devicetree/bindings/mmc/synopsys-dw-mshc.txt
@@ -59,8 +59,9 @@ Optional properties:
   is specified and the ciu clock is specified then we'll try to set the ciu
   clock to this at probe time.
 
-* clock-freq-min-max: Minimum and Maximum clock frequency for card output
+* clock-freq-min-max (DEPRECATED): Minimum and Maximum clock frequency for card output
   clock(cclk_out). If it's not specified, max is 200MHZ and min is 400KHz by default.
+	  (Use the "max-frequency" instead of "clock-freq-min-max".)
 
 * num-slots: specifies the number of slots supported by the controller.
   The number of physical slots actually used could be equal or less than the
@@ -74,11 +75,6 @@ Optional properties:
 * card-detect-delay: Delay in milli-seconds before detecting card after card
   insert event. The default value is 0.
 
-* supports-highspeed (DEPRECATED): Enables support for high speed cards (up to 50MHz)
-			   (use "cap-mmc-highspeed" or "cap-sd-highspeed" instead)
-
-* broken-cd: as documented in mmc core bindings.
-
 * vmmc-supply: The phandle to the regulator to use for vmmc.  If this is
   specified we'll defer probe until we can find this regulator.
 
diff --git a/Documentation/devicetree/bindings/mmc/tmio_mmc.txt b/Documentation/devicetree/bindings/mmc/tmio_mmc.txt
index 13df9c2399c3..a1650edfd2b7 100644
--- a/Documentation/devicetree/bindings/mmc/tmio_mmc.txt
+++ b/Documentation/devicetree/bindings/mmc/tmio_mmc.txt
@@ -11,8 +11,8 @@ optional bindings can be used.
 
 Required properties:
 - compatible:	"renesas,sdhi-shmobile" - a generic sh-mobile SDHI unit
-		"renesas,sdhi-sh7372" - SDHI IP on SH7372 SoC
 		"renesas,sdhi-sh73a0" - SDHI IP on SH73A0 SoC
+		"renesas,sdhi-r7s72100" - SDHI IP on R7S72100 SoC
 		"renesas,sdhi-r8a73a4" - SDHI IP on R8A73A4 SoC
 		"renesas,sdhi-r8a7740" - SDHI IP on R8A7740 SoC
 		"renesas,sdhi-r8a7778" - SDHI IP on R8A7778 SoC
diff --git a/Documentation/devicetree/bindings/nvmem/brcm,ocotp.txt b/Documentation/devicetree/bindings/nvmem/brcm,ocotp.txt
new file mode 100644
index 000000000000..6462e12d8de6
--- /dev/null
+++ b/Documentation/devicetree/bindings/nvmem/brcm,ocotp.txt
@@ -0,0 +1,17 @@
+Broadcom OTP memory controller
+
+Required Properties:
+- compatible: "brcm,ocotp" for the first generation Broadcom OTPC which is used
+  in Cygnus and supports 32 bit read/write. Use "brcm,ocotp-v2" for the second
+  generation Broadcom OTPC which is used in SoC's such as Stingray and supports
+  64-bit read/write.
+- reg: Base address of the OTP controller.
+- brcm,ocotp-size: Amount of memory available, in 32 bit words
+
+Example:
+
+otp: otp@0301c800 {
+	compatible = "brcm,ocotp";
+	reg = <0x0301c800 0x2c>;
+	brcm,ocotp-size = <2048>;
+};
diff --git a/Documentation/devicetree/bindings/nvmem/lpc1850-otp.txt b/Documentation/devicetree/bindings/nvmem/lpc1850-otp.txt
new file mode 100644
index 000000000000..853b6a754644
--- /dev/null
+++ b/Documentation/devicetree/bindings/nvmem/lpc1850-otp.txt
@@ -0,0 +1,20 @@
+* NXP LPC18xx OTP memory
+
+Internal OTP (One Time Programmable) memory for NXP LPC18xx/43xx devices.
+
+Required properties:
+  - compatible: Should be "nxp,lpc1850-otp"
+  - reg: Must contain an entry with the physical base address and length
+    for each entry in reg-names.
+  - address-cells: must be set to 1.
+  - size-cells: must be set to 1.
+
+See nvmem.txt for more information.
+
+Example:
+  otp: otp@40045000 {
+    compatible = "nxp,lpc1850-otp";
+    reg = <0x40045000 0x1000>;
+    #address-cells = <1>;
+    #size-cells = <1>;
+  };
diff --git a/Documentation/devicetree/bindings/opp/opp.txt b/Documentation/devicetree/bindings/opp/opp.txt
index ee91cbdd95ee..9f5ca4457b5f 100644
--- a/Documentation/devicetree/bindings/opp/opp.txt
+++ b/Documentation/devicetree/bindings/opp/opp.txt
@@ -86,8 +86,14 @@ Optional properties:
   Single entry is for target voltage and three entries are for <target min max>
   voltages.
 
-  Entries for multiple regulators must be present in the same order as
-  regulators are specified in device's DT node.
+  Entries for multiple regulators shall be provided in the same field separated
+  by angular brackets <>. The OPP binding doesn't provide any provisions to
+  relate the values to their power supplies or the order in which the supplies
+  need to be configured and that is left for the implementation specific
+  binding.
+
+  Entries for all regulators shall be of the same size, i.e. either all use a
+  single value or triplets.
 
 - opp-microvolt-<name>: Named opp-microvolt property. This is exactly similar to
   the above opp-microvolt property, but allows multiple voltage ranges to be
@@ -104,10 +110,13 @@ Optional properties:
 
   Should only be set if opp-microvolt is set for the OPP.
 
-  Entries for multiple regulators must be present in the same order as
-  regulators are specified in device's DT node. If this property isn't required
-  for few regulators, then this should be marked as zero for them. If it isn't
-  required for any regulator, then this property need not be present.
+  Entries for multiple regulators shall be provided in the same field separated
+  by angular brackets <>. If current values aren't required for a regulator,
+  then it shall be filled with 0. If current values aren't required for any of
+  the regulators, then this field is not required. The OPP binding doesn't
+  provide any provisions to relate the values to their power supplies or the
+  order in which the supplies need to be configured and that is left for the
+  implementation specific binding.
 
 - opp-microamp-<name>: Named opp-microamp property. Similar to
   opp-microvolt-<name> property, but for microamp instead.
@@ -386,10 +395,12 @@ Example 4: Handling multiple regulators
 / {
 	cpus {
 		cpu@0 {
-			compatible = "arm,cortex-a7";
+			compatible = "vendor,cpu-type";
 			...
 
-			cpu-supply = <&cpu_supply0>, <&cpu_supply1>, <&cpu_supply2>;
+			vcc0-supply = <&cpu_supply0>;
+			vcc1-supply = <&cpu_supply1>;
+			vcc2-supply = <&cpu_supply2>;
 			operating-points-v2 = <&cpu0_opp_table>;
 		};
 	};
diff --git a/Documentation/devicetree/bindings/pci/brcm,iproc-pcie.txt b/Documentation/devicetree/bindings/pci/brcm,iproc-pcie.txt
index 01b88f4e0d5b..b8e48b4762b2 100644
--- a/Documentation/devicetree/bindings/pci/brcm,iproc-pcie.txt
+++ b/Documentation/devicetree/bindings/pci/brcm,iproc-pcie.txt
@@ -1,10 +1,17 @@
 * Broadcom iProc PCIe controller with the platform bus interface
 
 Required properties:
-- compatible: Must be "brcm,iproc-pcie" for PAXB, or "brcm,iproc-pcie-paxc"
-  for PAXC.  PAXB-based root complex is used for external endpoint devices.
-  PAXC-based root complex is connected to emulated endpoint devices
-  internal to the ASIC
+- compatible:
+      "brcm,iproc-pcie" for the first generation of PAXB based controller,
+used in SoCs including NSP, Cygnus, NS2, and Pegasus
+      "brcm,iproc-pcie-paxb-v2" for the second generation of PAXB-based
+controllers, used in Stingray
+      "brcm,iproc-pcie-paxc" for the first generation of PAXC based
+controller, used in NS2
+      "brcm,iproc-pcie-paxc-v2" for the second generation of PAXC based
+controller, used in Stingray
+  PAXB-based root complex is used for external endpoint devices. PAXC-based
+root complex is connected to emulated endpoint devices internal to the ASIC
 - reg: base address and length of the PCIe controller I/O register space
 - #interrupt-cells: set to <1>
 - interrupt-map-mask and interrupt-map, standard PCI properties to define the
@@ -19,6 +26,10 @@ Required properties:
 Optional properties:
 - phys: phandle of the PCIe PHY device
 - phy-names: must be "pcie-phy"
+- dma-coherent: present if DMA operations are coherent
+- dma-ranges: Some PAXB-based root complexes do not have inbound mapping done
+  by the ASIC after power on reset.  In this case, SW is required to configure
+the mapping, based on inbound memory regions specified by this property.
 
 - brcm,pcie-ob: Some iProc SoCs do not have the outbound address mapping done
 by the ASIC after power on reset. In this case, SW needs to configure it
@@ -29,11 +40,6 @@ effective:
 Required:
 - brcm,pcie-ob-axi-offset: The offset from the AXI address to the internal
 address used by the iProc PCIe core (not the PCIe address)
-- brcm,pcie-ob-window-size: The outbound address mapping window size (in MB)
-
-Optional:
-- brcm,pcie-ob-oarr-size: Some iProc SoCs need the OARR size bit to be set to
-increase the outbound window size
 
 MSI support (optional):
 
@@ -41,10 +47,19 @@ For older platforms without MSI integrated in the GIC, iProc PCIe core provides
 an event queue based MSI support.  The iProc MSI uses host memories to store
 MSI posted writes in the event queues
 
-- msi-parent: Link to the device node of the MSI controller.  On newer iProc
-platforms, the MSI controller may be gicv2m or gicv3-its.  On older iProc
-platforms without MSI support in its interrupt controller, one may use the
-event queue based MSI support integrated within the iProc PCIe core.
+On newer iProc platforms, gicv2m or gicv3-its based MSI support should be used
+
+- msi-map: Maps a Requester ID to an MSI controller and associated MSI
+sideband data
+
+- msi-parent: Link to the device node of the MSI controller, used when no MSI
+sideband data is passed between the iProc PCIe controller and the MSI
+controller
+
+Refer to the following binding documents for more detailed description on
+the use of 'msi-map' and 'msi-parent':
+  Documentation/devicetree/bindings/pci/pci-msi.txt
+  Documentation/devicetree/bindings/interrupt-controller/msi.txt
 
 When the iProc event queue based MSI is used, one needs to define the
 following properties in the MSI device node:
@@ -80,9 +95,7 @@ Example:
 		phy-names = "pcie-phy";
 
 		brcm,pcie-ob;
-		brcm,pcie-ob-oarr-size;
 		brcm,pcie-ob-axi-offset = <0x00000000>;
-		brcm,pcie-ob-window-size = <256>;
 
 		msi-parent = <&msi0>;
 
diff --git a/Documentation/devicetree/bindings/pci/layerscape-pci.txt b/Documentation/devicetree/bindings/pci/layerscape-pci.txt
index 41e9f55a1467..ee1c72d5162e 100644
--- a/Documentation/devicetree/bindings/pci/layerscape-pci.txt
+++ b/Documentation/devicetree/bindings/pci/layerscape-pci.txt
@@ -15,6 +15,7 @@ Required properties:
 - compatible: should contain the platform identifier such as:
         "fsl,ls1021a-pcie", "snps,dw-pcie"
         "fsl,ls2080a-pcie", "fsl,ls2085a-pcie", "snps,dw-pcie"
+        "fsl,ls1046a-pcie"
 - reg: base addresses and lengths of the PCIe controller
 - interrupts: A list of interrupt outputs of the controller. Must contain an
   entry for each entry in the interrupt-names property.
diff --git a/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt b/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt
index b8cc395fffea..982a74ea6df9 100644
--- a/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt
+++ b/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt
@@ -110,6 +110,20 @@ Power supplies for Tegra124:
   - avdd-pll-erefe-supply: Power supply for PLLE (shared with USB3). Must
     supply 1.05 V.
 
+Power supplies for Tegra210:
+- Required:
+  - avdd-pll-uerefe-supply: Power supply for PLLE (shared with USB3). Must
+    supply 1.05 V.
+  - hvddio-pex-supply: High-voltage supply for PCIe I/O and PCIe output
+    clocks. Must supply 1.8 V.
+  - dvddio-pex-supply: Power supply for digital PCIe I/O. Must supply 1.05 V.
+  - dvdd-pex-pll-supply: Power supply for dedicated (internal) PCIe PLL. Must
+    supply 1.05 V.
+  - hvdd-pex-pll-e-supply: High-voltage supply for PLLE (shared with USB3).
+    Must supply 3.3 V.
+  - vddio-pex-ctl-supply: Power supply for PCIe control I/O partition. Must
+    supply 1.8 V.
+
 Root ports are defined as subnodes of the PCIe controller node.
 
 Required properties:
@@ -436,3 +450,99 @@ Board DTS:
 			status = "okay";
 		};
 	};
+
+Tegra210:
+---------
+
+SoC DTSI:
+
+	pcie-controller@01003000 {
+		compatible = "nvidia,tegra210-pcie";
+		device_type = "pci";
+		reg = <0x0 0x01003000 0x0 0x00000800   /* PADS registers */
+		       0x0 0x01003800 0x0 0x00000800   /* AFI registers */
+		       0x0 0x02000000 0x0 0x10000000>; /* configuration space */
+		reg-names = "pads", "afi", "cs";
+		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
+
+		bus-range = <0x00 0xff>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		ranges = <0x82000000 0 0x01000000 0x0 0x01000000 0 0x00001000   /* port 0 configuration space */
+			  0x82000000 0 0x01001000 0x0 0x01001000 0 0x00001000   /* port 1 configuration space */
+			  0x81000000 0 0x0        0x0 0x12000000 0 0x00010000   /* downstream I/O (64 KiB) */
+			  0x82000000 0 0x13000000 0x0 0x13000000 0 0x0d000000   /* non-prefetchable memory (208 MiB) */
+			  0xc2000000 0 0x20000000 0x0 0x20000000 0 0x20000000>; /* prefetchable memory (512 MiB) */
+
+		clocks = <&tegra_car TEGRA210_CLK_PCIE>,
+			 <&tegra_car TEGRA210_CLK_AFI>,
+			 <&tegra_car TEGRA210_CLK_PLL_E>,
+			 <&tegra_car TEGRA210_CLK_CML0>;
+		clock-names = "pex", "afi", "pll_e", "cml";
+		resets = <&tegra_car 70>,
+			 <&tegra_car 72>,
+			 <&tegra_car 74>;
+		reset-names = "pex", "afi", "pcie_x";
+		status = "disabled";
+
+		pci@1,0 {
+			device_type = "pci";
+			assigned-addresses = <0x82000800 0 0x01000000 0 0x1000>;
+			reg = <0x000800 0 0 0 0>;
+			status = "disabled";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			nvidia,num-lanes = <4>;
+		};
+
+		pci@2,0 {
+			device_type = "pci";
+			assigned-addresses = <0x82001000 0 0x01001000 0 0x1000>;
+			reg = <0x001000 0 0 0 0>;
+			status = "disabled";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			nvidia,num-lanes = <1>;
+		};
+	};
+
+Board DTS:
+
+	pcie-controller@01003000 {
+		status = "okay";
+
+		avdd-pll-uerefe-supply = <&avdd_1v05_pll>;
+		hvddio-pex-supply = <&vdd_1v8>;
+		dvddio-pex-supply = <&vdd_pex_1v05>;
+		dvdd-pex-pll-supply = <&vdd_pex_1v05>;
+		hvdd-pex-pll-e-supply = <&vdd_1v8>;
+		vddio-pex-ctl-supply = <&vdd_1v8>;
+
+		pci@1,0 {
+			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-0}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-1}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-2}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-3}>;
+			phy-names = "pcie-0", "pcie-1", "pcie-2", "pcie-3";
+			status = "okay";
+		};
+
+		pci@2,0 {
+			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-4}>;
+			phy-names = "pcie-0";
+			status = "okay";
+		};
+	};
diff --git a/Documentation/devicetree/bindings/pci/pci.txt b/Documentation/devicetree/bindings/pci/pci.txt
index 08dcfad09f8d..50f9e2ca5b13 100644
--- a/Documentation/devicetree/bindings/pci/pci.txt
+++ b/Documentation/devicetree/bindings/pci/pci.txt
@@ -18,3 +18,9 @@ driver implementation may support the following properties:
    host bridges in the system, otherwise potentially conflicting domain numbers
    may be assigned to root buses behind different host bridges.  The domain
    number for each host bridge in the system must be unique.
+- max-link-speed:
+   If present this property specifies PCI gen for link capability.  Host
+   drivers could add this as a strategy to avoid unnecessary operation for
+   unsupported link speed, for instance, trying to do training for
+   unsupported link speed, etc.  Must be '4' for gen4, '3' for gen3, '2'
+   for gen2, and '1' for gen1. Any other values are invalid.
diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie.txt b/Documentation/devicetree/bindings/pci/qcom,pcie.txt
index 4059a6f89bc1..e15f9b19901f 100644
--- a/Documentation/devicetree/bindings/pci/qcom,pcie.txt
+++ b/Documentation/devicetree/bindings/pci/qcom,pcie.txt
@@ -7,6 +7,7 @@
 			- "qcom,pcie-ipq8064" for ipq8064
 			- "qcom,pcie-apq8064" for apq8064
 			- "qcom,pcie-apq8084" for apq8084
+			- "qcom,pcie-msm8996" for msm8996 or apq8096
 
 - reg:
 	Usage: required
@@ -92,6 +93,17 @@
 			- "aux"		Auxiliary (AUX) clock
 			- "bus_master"	Master AXI clock
 			- "bus_slave"	Slave AXI clock
+
+- clock-names:
+	Usage: required for msm8996/apq8096
+	Value type: <stringlist>
+	Definition: Should contain the following entries
+			- "pipe"	Pipe Clock driving internal logic
+			- "aux"		Auxiliary (AUX) clock
+			- "cfg"		Configuration clock
+			- "bus_master"	Master AXI clock
+			- "bus_slave"	Slave AXI clock
+
 - resets:
 	Usage: required
 	Value type: <prop-encoded-array>
@@ -115,7 +127,7 @@
 			- "core" Core reset
 
 - power-domains:
-	Usage: required for apq8084
+	Usage: required for apq8084 and msm8996/apq8096
 	Value type: <prop-encoded-array>
 	Definition: A phandle and power domain specifier pair to the
 		    power domain which is responsible for collapsing
diff --git a/Documentation/devicetree/bindings/pci/rcar-pci.txt b/Documentation/devicetree/bindings/pci/rcar-pci.txt
index 6cf99690eef9..eee518db90b9 100644
--- a/Documentation/devicetree/bindings/pci/rcar-pci.txt
+++ b/Documentation/devicetree/bindings/pci/rcar-pci.txt
@@ -7,6 +7,7 @@ compatible: "renesas,pcie-r8a7779" for the R8A7779 SoC;
 	    "renesas,pcie-r8a7793" for the R8A7793 SoC;
 	    "renesas,pcie-r8a7795" for the R8A7795 SoC;
 	    "renesas,pcie-rcar-gen2" for a generic R-Car Gen2 compatible device.
+	    "renesas,pcie-rcar-gen3" for a generic R-Car Gen3 compatible device.
 
 	    When compatible with the generic version, nodes must list the
 	    SoC-specific version corresponding to the platform first
diff --git a/Documentation/devicetree/bindings/phy/meson-usb2-phy.txt b/Documentation/devicetree/bindings/phy/meson8b-usb2-phy.txt
index 9da5ea234154..5fa73b9d20f5 100644
--- a/Documentation/devicetree/bindings/phy/meson-usb2-phy.txt
+++ b/Documentation/devicetree/bindings/phy/meson8b-usb2-phy.txt
@@ -1,4 +1,4 @@
-* Amlogic USB2 PHY
+* Amlogic Meson8b and GXBB USB2 PHY
 
 Required properties:
 - compatible:	Depending on the platform this should be one of:
@@ -16,10 +16,10 @@ Optional properties:
 
 Example:
 
-usb0_phy: usb_phy@0 {
+usb0_phy: usb-phy@c0000000 {
 	compatible = "amlogic,meson-gxbb-usb2-phy";
 	#phy-cells = <0>;
-	reg = <0x0 0x0 0x0 0x20>;
+	reg = <0x0 0xc0000000 0x0 0x20>;
 	resets = <&reset RESET_USB_OTG>;
 	clocks = <&clkc CLKID_USB>, <&clkc CLKID_USB0>;
 	clock-names = "usb_general", "usb";
diff --git a/Documentation/devicetree/bindings/pinctrl/allwinner,sunxi-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/allwinner,sunxi-pinctrl.txt
index 1685821eea41..de1378b4efad 100644
--- a/Documentation/devicetree/bindings/pinctrl/allwinner,sunxi-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/allwinner,sunxi-pinctrl.txt
@@ -28,6 +28,20 @@ Required properties:
 - reg: Should contain the register physical address and length for the
   pin controller.
 
+- clocks: phandle to the clocks feeding the pin controller:
+  - "apb": the gated APB parent clock
+  - "hosc": the high frequency oscillator in the system
+  - "losc": the low frequency oscillator in the system
+
+Note: For backward compatibility reasons, the hosc and losc clocks are only
+required if you need to use the optional input-debounce property. Any new
+device tree should set them.
+
+Optional properties:
+  - input-debounce: Array of debouncing periods in microseconds. One period per
+    irq bank found in the controller. 0 if no setup required.
+
+
 Please refer to pinctrl-bindings.txt in this directory for details of the
 common pinctrl bindings used by client devices.
 
@@ -37,6 +51,22 @@ pins it needs, and how they should be configured, with regard to muxer
 configuration, drive strength and pullups. If one of these options is
 not set, its actual value will be unspecified.
 
+This driver supports the generic pin multiplexing and configuration
+bindings. For details on each properties, you can refer to
+./pinctrl-bindings.txt.
+
+Required sub-node properties:
+  - pins
+  - function
+
+Optional sub-node properties:
+  - bias-disable
+  - bias-pull-up
+  - bias-pull-down
+  - drive-strength
+
+*** Deprecated pin configuration and multiplexing binding
+
 Required subnode-properties:
 
 - allwinner,pins: List of strings containing the pin name.
diff --git a/Documentation/devicetree/bindings/pinctrl/atmel,at91-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/atmel,at91-pinctrl.txt
index b7a93e80a302..9a8a45d9d8ab 100644
--- a/Documentation/devicetree/bindings/pinctrl/atmel,at91-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/atmel,at91-pinctrl.txt
@@ -98,6 +98,8 @@ DRIVE_STRENGTH (3 << 5): indicate the drive strength of the pin using the
 				01 - Low
 				10 - Medium
 				11 - High
+OUTPUT		(1 << 7): indicate this pin need to be configured as an output.
+OUTPUT_VAL	(1 << 8): output val (1 = high, 0 = low)
 DEBOUNCE	(1 << 16): indicate this pin needs debounce.
 DEBOUNCE_VAL	(0x3fff << 17): debounce value.
 
diff --git a/Documentation/devicetree/bindings/pinctrl/meson,pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/meson,pinctrl.txt
index fe7fe0b03cfb..2392557ede27 100644
--- a/Documentation/devicetree/bindings/pinctrl/meson,pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/meson,pinctrl.txt
@@ -7,6 +7,8 @@ Required properties for the root node:
 		      "amlogic,meson8b-aobus-pinctrl"
 		      "amlogic,meson-gxbb-periphs-pinctrl"
 		      "amlogic,meson-gxbb-aobus-pinctrl"
+		      "amlogic,meson-gxl-periphs-pinctrl"
+		      "amlogic,meson-gxl-aobus-pinctrl"
  - reg: address and size of registers controlling irq functionality
 
 === GPIO sub-nodes ===
diff --git a/Documentation/devicetree/bindings/pinctrl/oxnas,pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/oxnas,pinctrl.txt
index d6074321f730..09e81a95bbfd 100644
--- a/Documentation/devicetree/bindings/pinctrl/oxnas,pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/oxnas,pinctrl.txt
@@ -9,7 +9,7 @@ used for a specific device or function. This node represents configurations of
 pins, optional function, and optional mux related configuration.
 
 Required properties for pin controller node:
- - compatible: "oxsemi,ox810se-pinctrl"
+ - compatible: "oxsemi,ox810se-pinctrl" or "oxsemi,ox820-pinctrl"
  - oxsemi,sys-ctrl: a phandle to the system controller syscon node
 
 Required properties for pin configuration sub-nodes:
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt
index b73c96d24f59..bf3f7b014724 100644
--- a/Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt
@@ -97,6 +97,11 @@ For example:
 	};
 
 == Pin controller devices ==
+Required properties: See the pin controller driver specific documentation
+
+Optional properties:
+#pinctrl-cells:	Number of pin control cells in addition to the index within the
+		pin controller device instance
 
 Pin controller devices should contain the pin configuration nodes that client
 devices reference.
@@ -119,7 +124,8 @@ For example:
 
 The contents of each of those pin configuration child nodes is defined
 entirely by the binding for the individual pin controller device. There
-exists no common standard for this content.
+exists no common standard for this content. The pinctrl framework only
+provides generic helper bindings that the pin controller driver can use.
 
 The pin configuration nodes need not be direct children of the pin controller
 device; they may be grandchildren, for example. Whether this is legal, and
@@ -156,6 +162,42 @@ state_2_node_a {
 	pins = "mfio29", "mfio30";
 };
 
+Optionally an altenative binding can be used if more suitable depending on the
+pin controller hardware. For hardaware where there is a large number of identical
+pin controller instances, naming each pin and function can easily become
+unmaintainable. This is especially the case if the same controller is used for
+different pins and functions depending on the SoC revision and packaging.
+
+For cases like this, the pin controller driver may use pinctrl-pin-array helper
+binding with a hardware based index and a number of pin configuration values:
+
+pincontroller {
+	... /* Standard DT properties for the device itself elided */
+	#pinctrl-cells = <2>;
+
+	state_0_node_a {
+		pinctrl-pin-array = <
+			0 A_DELAY_PS(0) G_DELAY_PS(120)
+			4 A_DELAY_PS(0) G_DELAY_PS(360)
+			...
+		>;
+	};
+	...
+};
+
+Above #pinctrl-cells specifies the number of value cells in addition to the
+index of the registers. This is similar to the interrupts-extended binding with
+one exception. There is no need to specify the phandle for each entry as that
+is already known as the defined pins are always children of the pin controller
+node. Further having the phandle pointing to another pin controller would not
+currently work as the pinctrl framework uses named modes to group pins for each
+pin control device.
+
+The index for pinctrl-pin-array must relate to the hardware for the pinctrl
+registers, and must not be a virtual index of pin instances. The reason for
+this is to avoid mapping of the index in the dts files and the pin controller
+driver as it can change.
+
 == Generic pin configuration node content ==
 
 Many data items that are represented in a pin configuration node are common
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-palmas.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-palmas.txt
index caf297bee1fb..c28d4eb83b76 100644
--- a/Documentation/devicetree/bindings/pinctrl/pinctrl-palmas.txt
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-palmas.txt
@@ -35,6 +35,15 @@ Optional properties:
 - ti,palmas-enable-dvfs2: Enable DVFS2. Configure pins for DVFS2 mode.
 	Selection primary or secondary function associated to GPADC_START
 	and SYSEN2 pin/pad for DVFS2 interface
+- ti,palmas-override-powerhold: This is applicable for PMICs for which
+	GPIO7 is configured in POWERHOLD mode which has higher priority
+	over DEV_ON bit and keeps the PMIC supplies on even after the DEV_ON
+	bit is turned off. This property enables driver to over ride the
+	POWERHOLD value to GPIO7 so as to turn off the PMIC in power off
+	scenarios. So for GPIO7 if ti,palmas-override-powerhold is set
+	then the GPIO_7 field should never be muxed to anything else.
+	It should be set to POWERHOLD by default and only in case of
+	power off scenarios the driver will over ride the mux value.
 
 This binding uses the following generic properties as defined in
 pinctrl-bindings.txt:
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-single.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-single.txt
index 66dcaa9efd74..e705acd3612c 100644
--- a/Documentation/devicetree/bindings/pinctrl/pinctrl-single.txt
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-single.txt
@@ -7,6 +7,9 @@ Required properties:
 
 - reg : offset and length of the register set for the mux registers
 
+- #pinctrl-cells : number of cells in addition to the index, set to 1
+  for pinctrl-single,pins and 2 for pinctrl-single,bits
+
 - pinctrl-single,register-width : pinmux register access width in bits
 
 - pinctrl-single,function-mask : mask of allowed pinmux function bits
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-sx150x.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-sx150x.txt
new file mode 100644
index 000000000000..bf76867168e9
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-sx150x.txt
@@ -0,0 +1,74 @@
+SEMTECH SX150x GPIO expander bindings
+
+Please refer to pinctrl-bindings.txt, ../gpio/gpio.txt, and
+../interrupt-controller/interrupts.txt for generic information regarding
+pin controller, GPIO, and interrupt bindings.
+
+Required properties:
+- compatible: should be one of :
+			"semtech,sx1501q",
+			"semtech,sx1502q",
+			"semtech,sx1503q",
+			"semtech,sx1504q",
+			"semtech,sx1505q",
+			"semtech,sx1506q",
+			"semtech,sx1507q",
+			"semtech,sx1508q",
+			"semtech,sx1509q".
+
+- reg: The I2C slave address for this device.
+
+- #gpio-cells: Should be 2. The first cell is the GPIO number and the
+		second cell is used to specify optional parameters:
+		bit 0: polarity (0: normal, 1: inverted)
+
+- gpio-controller: Marks the device as a GPIO controller.
+
+Optional properties :
+- interrupt-parent: phandle of the parent interrupt controller.
+
+- interrupts: Interrupt specifier for the controllers interrupt.
+
+- interrupt-controller: Marks the device as a interrupt controller.
+
+- semtech,probe-reset: Will trigger a reset of the GPIO expander on probe,
+		only for sx1507q, sx1508q and sx1509q
+
+The GPIO expander can optionally be used as an interrupt controller, in
+which case it uses the default two cell specifier.
+
+Required properties for pin configuration sub-nodes:
+ - pins: List of pins to which the configuration applies.
+
+Optional properties for pin configuration sub-nodes:
+----------------------------------------------------
+ - bias-disable: disable any pin bias, except the OSCIO pin
+ - bias-pull-up: pull up the pin, except the OSCIO pin
+ - bias-pull-down: pull down the pin, except the OSCIO pin
+ - bias-pull-pin-default: use pin-default pull state, except the OSCIO pin
+ - drive-push-pull: drive actively high and low
+ - drive-open-drain: drive with open drain only for sx1507q, sx1508q and sx1509q and except the OSCIO pin
+ - output-low: set the pin to output mode with low level
+ - output-high: set the pin to output mode with high level
+
+Example:
+
+	i2c0gpio-expander@20{
+		#gpio-cells = <2>;
+		#interrupt-cells = <2>;
+		compatible = "semtech,sx1506q";
+		reg = <0x20>;
+		interrupt-parent = <&gpio_1>;
+		interrupts = <16 0>;
+
+		gpio-controller;
+		interrupt-controller;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&gpio1_cfg_pins>;
+
+		gpio1_cfg_pins: gpio1-cfg {
+			pins = "gpio1";
+			bias-pull-up;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,msm8994-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/qcom,msm8994-pinctrl.txt
new file mode 100644
index 000000000000..13cd629f896e
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,msm8994-pinctrl.txt
@@ -0,0 +1,177 @@
+Qualcomm MSM8994 TLMM block
+
+This binding describes the Top Level Mode Multiplexer block found in the
+MSM8994 platform.
+
+- compatible:
+	Usage: required
+	Value type: <string>
+	Definition: Should contain one of:
+		    "qcom,msm8992-pinctrl",
+		    "qcom,msm8994-pinctrl".
+
+- reg:
+	Usage: required
+	Value type: <prop-encoded-array>
+	Definition: the base address and size of the TLMM register space.
+
+- interrupts:
+	Usage: required
+	Value type: <prop-encoded-array>
+	Definition: should specify the TLMM summary IRQ.
+
+- interrupt-controller:
+	Usage: required
+	Value type: <none>
+	Definition: identifies this node as an interrupt controller
+
+- #interrupt-cells:
+	Usage: required
+	Value type: <u32>
+	Definition: must be 2. Specifying the pin number and flags, as defined
+		    in <dt-bindings/interrupt-controller/irq.h>
+
+- gpio-controller:
+	Usage: required
+	Value type: <none>
+	Definition: identifies this node as a gpio controller
+
+- #gpio-cells:
+	Usage: required
+	Value type: <u32>
+	Definition: must be 2. Specifying the pin number and flags, as defined
+		    in <dt-bindings/gpio/gpio.h>
+
+Please refer to ../gpio/gpio.txt and ../interrupt-controller/interrupts.txt for
+a general description of GPIO and interrupt bindings.
+
+Please refer to pinctrl-bindings.txt in this directory for details of the
+common pinctrl bindings used by client devices, including the meaning of the
+phrase "pin configuration node".
+
+The pin configuration nodes act as a container for an arbitrary number of
+subnodes. Each of these subnodes represents some desired configuration for a
+pin, a group, or a list of pins or groups. This configuration can include the
+mux function to select on those pin(s)/group(s), and various pin configuration
+parameters, such as pull-up, drive strength, etc.
+
+
+PIN CONFIGURATION NODES:
+
+The name of each subnode is not important; all subnodes should be enumerated
+and processed purely based on their content.
+
+Each subnode only affects those parameters that are explicitly listed. In
+other words, a subnode that lists a mux function but no pin configuration
+parameters implies no information about any pin configuration parameters.
+Similarly, a pin subnode that describes a pullup parameter implies no
+information about e.g. the mux function.
+
+
+The following generic properties as defined in pinctrl-bindings.txt are valid
+to specify in a pin configuration subnode:
+
+- pins:
+	Usage: required
+	Value type: <string-array>
+	Definition: List of gpio pins affected by the properties specified in
+		    this subnode.
+
+		    Valid pins are:
+		      gpio0-gpio145
+		        Supports mux, bias and drive-strength
+
+		      sdc1_clk, sdc1_cmd, sdc1_data sdc1_rclk, sdc2_clk,
+		      sdc2_cmd, sdc2_data
+		        Supports bias and drive-strength
+
+- function:
+	Usage: required
+	Value type: <string>
+	Definition: Specify the alternative function to be configured for the
+		    specified pins. Functions are only valid for gpio pins.
+		    Valid values are:
+
+		    audio_ref_clk,  blsp_i2c1, blsp_i2c2, blsp_i2c3, blsp_i2c4, blsp_i2c5,
+		    blsp_i2c6, blsp_i2c7, blsp_i2c8, blsp_i2c9, blsp_i2c10, blsp_i2c11,
+		    blsp_i2c12, blsp_spi1, blsp_spi1_cs1, blsp_spi1_cs2, blsp_spi1_cs3,
+		    blsp_spi2, blsp_spi2_cs1, blsp_spi2_cs2, blsp_spi2_cs3, blsp_spi3,
+		    blsp_spi4, blsp_spi5, blsp_spi6, blsp_spi7, blsp_spi8, blsp_spi9,
+		    blsp_spi10, blsp_spi10_cs1, blsp_spi10_cs2, blsp_spi10_cs3, blsp_spi11,
+		    blsp_spi12, blsp_uart1, blsp_uart2, blsp_uart3, blsp_uart4, blsp_uart5,
+		    blsp_uart6, blsp_uart7, blsp_uart8, blsp_uart9, blsp_uart10, blsp_uart11,
+		    blsp_uart12, blsp_uim1, blsp_uim2, blsp_uim3, blsp_uim4, blsp_uim5,
+		    blsp_uim6, blsp_uim7, blsp_uim8, blsp_uim9, blsp_uim10, blsp_uim11,
+		    blsp_uim12, blsp11_i2c_scl_b, blsp11_i2c_sda_b, blsp11_uart_rx_b,
+		    blsp11_uart_tx_b, cam_mclk0, cam_mclk1, cam_mclk2, cam_mclk3,
+		    cci_async_in0, cci_async_in1, cci_async_in2, cci_i2c0, cci_i2c1,
+		    cci_timer0, cci_timer1, cci_timer2, cci_timer3, cci_timer4,
+		    gcc_gp1_clk_a, gcc_gp1_clk_b, gcc_gp2_clk_a, gcc_gp2_clk_b, gcc_gp3_clk_a,
+		    gcc_gp3_clk_b, gp_mn, gp_pdm0, gp_pdm1, gp_pdm2, gp0_clk,
+		    gp1_clk, gps_tx, gsm_tx, hdmi_cec, hdmi_ddc, hdmi_hpd, hdmi_rcv,
+		    mdp_vsync, mss_lte, nav_pps, nav_tsync, qdss_cti_trig_in_a,
+		    qdss_cti_trig_in_b, qdss_cti_trig_in_c, qdss_cti_trig_in_d,
+		    qdss_cti_trig_out_a, qdss_cti_trig_out_b, qdss_cti_trig_out_c,
+		    qdss_cti_trig_out_d, qdss_traceclk_a, qdss_traceclk_b, qdss_tracectl_a,
+		    qdss_tracectl_b, qdss_tracedata_a, qdss_tracedata_b, qua_mi2s, pci_e0,
+		    pci_e1, pri_mi2s, sdc4, sec_mi2s, slimbus, spkr_i2s, ter_mi2s, tsif1,
+		    tsif2, uim_batt_alarm, uim1, uim2, uim3, uim4, gpio
+
+- bias-disable:
+	Usage: optional
+	Value type: <none>
+	Definition: The specified pins should be configued as no pull.
+
+- bias-pull-down:
+	Usage: optional
+	Value type: <none>
+	Definition: The specified pins should be configued as pull down.
+
+- bias-pull-up:
+	Usage: optional
+	Value type: <none>
+	Definition: The specified pins should be configued as pull up.
+
+- output-high:
+	Usage: optional
+	Value type: <none>
+	Definition: The specified pins are configured in output mode, driven
+		    high.
+		    Not valid for sdc pins.
+
+- output-low:
+	Usage: optional
+	Value type: <none>
+	Definition: The specified pins are configured in output mode, driven
+		    low.
+		    Not valid for sdc pins.
+
+- drive-strength:
+	Usage: optional
+	Value type: <u32>
+	Definition: Selects the drive strength for the specified pins, in mA.
+		    Valid values are: 2, 4, 6, 8, 10, 12, 14 and 16
+
+Example:
+
+	msmgpio: pinctrl@fd510000 {
+		compatible = "qcom,msm8994-pinctrl";
+		reg = <0xfd510000 0x4000>;
+		interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+
+		blsp1_uart2_default: blsp1_uart2_default {
+			pinmux {
+				pins = "gpio4", "gpio5";
+				function = "blsp_uart2";
+			};
+			pinconf {
+				pins = "gpio4", "gpio5";
+				drive-strength = <16>;
+				bias-disable;
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.txt
index c68b9554561f..4722bc61a1a2 100644
--- a/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.txt
@@ -19,10 +19,11 @@ The pins are grouped into up to 5 individual pin banks which need to be
 defined as gpio sub-nodes of the pinmux controller.
 
 Required properties for iomux controller:
-  - compatible: one of "rockchip,rk2928-pinctrl", "rockchip,rk3066a-pinctrl"
-		       "rockchip,rk3066b-pinctrl", "rockchip,rk3188-pinctrl"
-		       "rockchip,rk3228-pinctrl", "rockchip,rk3288-pinctrl"
-		       "rockchip,rk3368-pinctrl", "rockchip,rk3399-pinctrl"
+  - compatible: one of "rockchip,rk1108-pinctrl", "rockchip,rk2928-pinctrl"
+		       "rockchip,rk3066a-pinctrl", "rockchip,rk3066b-pinctrl"
+		       "rockchip,rk3188-pinctrl", "rockchip,rk3228-pinctrl"
+		       "rockchip,rk3288-pinctrl", "rockchip,rk3368-pinctrl"
+		       "rockchip,rk3399-pinctrl"
   - rockchip,grf: phandle referencing a syscon providing the
 	 "general register files"
 
diff --git a/Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt
index d49e22d2a8b5..1baf19eecabf 100644
--- a/Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt
@@ -19,11 +19,30 @@ Required Properties:
   - "samsung,exynos5260-pinctrl": for Exynos5260 compatible pin-controller.
   - "samsung,exynos5410-pinctrl": for Exynos5410 compatible pin-controller.
   - "samsung,exynos5420-pinctrl": for Exynos5420 compatible pin-controller.
+  - "samsung,exynos5433-pinctrl": for Exynos5433 compatible pin-controller.
   - "samsung,exynos7-pinctrl": for Exynos7 compatible pin-controller.
 
 - reg: Base address of the pin controller hardware module and length of
   the address space it occupies.
 
+  - reg: Second base address of the pin controller if the specific registers
+  of the pin controller are separated into the different base address.
+
+	Eg: GPF[1-5] of Exynos5433 are separated into the two base address.
+	- First base address is for GPAx and GPF[1-5] external interrupt
+	  registers.
+	- Second base address is for GPF[1-5] pinctrl registers.
+
+	pinctrl_0: pinctrl@10580000 {
+		compatible = "samsung,exynos5433-pinctrl";
+		reg = <0x10580000 0x1a20>, <0x11090000 0x100>;
+
+		wakeup-interrupt-controller {
+			compatible = "samsung,exynos7-wakeup-eint";
+			interrupts = <0 16 0>;
+		};
+	};
+
 - Pin banks as child nodes: Pin banks of the controller are represented by child
   nodes of the controller node. Bank name is taken from name of the node. Each
   bank node must contain following properties:
diff --git a/Documentation/devicetree/bindings/pinctrl/ti,da850-pupd.txt b/Documentation/devicetree/bindings/pinctrl/ti,da850-pupd.txt
new file mode 100644
index 000000000000..7f2980567c9f
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/ti,da850-pupd.txt
@@ -0,0 +1,55 @@
+* Pin configuration for TI DA850/OMAP-L138/AM18x
+
+These SoCs have a separate controller for setting bias (internal pullup/down).
+Bias can only be selected for groups rather than individual pins.
+
+Required Properties:
+
+  - compatible: Must be "ti,da850-pupd"
+  - reg: Base address and length of the memory resource used by the pullup/down
+    controller hardware module.
+
+The controller node also acts as a container for pin group configuration nodes.
+The names of these groups are ignored.
+
+Pin Group Node Properties:
+
+- groups: An array of strings, each string containing the name of a pin group.
+          Valid names are "cp0".."cp31".
+
+The pin configuration parameters use the generic pinconf bindings defined in
+pinctrl-bindings.txt in this directory. The supported parameters are
+bias-disable, bias-pull-up, bias-pull-down.
+
+
+Example
+-------
+
+In common dtsi file:
+
+	pinconf: pin-controller@22c00c {
+		compatible = "ti,da850-pupd";
+		reg = <0x22c00c 0x8>;
+	};
+
+In board-specific file:
+
+	&pinconf {
+		pinctrl-0 = <&pinconf_bias_groups>;
+		pinctrl-names = "default";
+
+		pinconf_bias_groups: bias-groups {
+			pull-up {
+				groups = "cp30", "cp31";
+				bias-pull-up;
+			};
+			pull-down {
+				groups = "cp29", "cp28";
+				bias-pull-down;
+			};
+			disable {
+				groups = "cp27", "cp26";
+				bias-disable;
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/power/domain-idle-state.txt b/Documentation/devicetree/bindings/power/domain-idle-state.txt
new file mode 100644
index 000000000000..eefc7ed22ca2
--- /dev/null
+++ b/Documentation/devicetree/bindings/power/domain-idle-state.txt
@@ -0,0 +1,33 @@
+PM Domain Idle State Node:
+
+A domain idle state node represents the state parameters that will be used to
+select the state when there are no active components in the domain.
+
+The state node has the following parameters -
+
+- compatible:
+	Usage: Required
+	Value type: <string>
+	Definition: Must be "domain-idle-state".
+
+- entry-latency-us
+	Usage: Required
+	Value type: <prop-encoded-array>
+	Definition: u32 value representing worst case latency in
+		    microseconds required to enter the idle state.
+		    The exit-latency-us duration may be guaranteed
+		    only after entry-latency-us has passed.
+
+- exit-latency-us
+	Usage: Required
+	Value type: <prop-encoded-array>
+	Definition: u32 value representing worst case latency
+		    in microseconds required to exit the idle state.
+
+- min-residency-us
+	Usage: Required
+	Value type: <prop-encoded-array>
+	Definition: u32 value representing minimum residency duration
+		    in microseconds after which the idle state will yield
+		    power benefits after overcoming the overhead in entering
+i		    the idle state.
diff --git a/Documentation/devicetree/bindings/power/power_domain.txt b/Documentation/devicetree/bindings/power/power_domain.txt
index 025b5e7df61c..723e1ad937da 100644
--- a/Documentation/devicetree/bindings/power/power_domain.txt
+++ b/Documentation/devicetree/bindings/power/power_domain.txt
@@ -29,6 +29,15 @@ Optional properties:
    specified by this binding. More details about power domain specifier are
    available in the next section.
 
+- domain-idle-states : A phandle of an idle-state that shall be soaked into a
+                generic domain power state. The idle state definitions are
+                compatible with domain-idle-state specified in [1].
+  The domain-idle-state property reflects the idle state of this PM domain and
+  not the idle states of the devices or sub-domains in the PM domain. Devices
+  and sub-domains have their own idle-states independent of the parent
+  domain's idle states. In the absence of this property, the domain would be
+  considered as capable of being powered-on or powered-off.
+
 Example:
 
 	power: power-controller@12340000 {
@@ -59,6 +68,38 @@ The nodes above define two power controllers: 'parent' and 'child'.
 Domains created by the 'child' power controller are subdomains of '0' power
 domain provided by the 'parent' power controller.
 
+Example 3:
+	parent: power-controller@12340000 {
+		compatible = "foo,power-controller";
+		reg = <0x12340000 0x1000>;
+		#power-domain-cells = <0>;
+		domain-idle-states = <&DOMAIN_RET>, <&DOMAIN_PWR_DN>;
+	};
+
+	child: power-controller@12341000 {
+		compatible = "foo,power-controller";
+		reg = <0x12341000 0x1000>;
+		power-domains = <&parent 0>;
+		#power-domain-cells = <0>;
+		domain-idle-states = <&DOMAIN_PWR_DN>;
+	};
+
+	DOMAIN_RET: state@0 {
+		compatible = "domain-idle-state";
+		reg = <0x0>;
+		entry-latency-us = <1000>;
+		exit-latency-us = <2000>;
+		min-residency-us = <10000>;
+	};
+
+	DOMAIN_PWR_DN: state@1 {
+		compatible = "domain-idle-state";
+		reg = <0x1>;
+		entry-latency-us = <5000>;
+		exit-latency-us = <8000>;
+		min-residency-us = <7000>;
+	};
+
 ==PM domain consumers==
 
 Required properties:
@@ -76,3 +117,5 @@ Example:
 The node above defines a typical PM domain consumer device, which is located
 inside a PM domain with index 0 of a power controller represented by a node
 with the label "power".
+
+[1]. Documentation/devicetree/bindings/power/domain-idle-state.txt
diff --git a/Documentation/devicetree/bindings/power/renesas,rcar-sysc.txt b/Documentation/devicetree/bindings/power/renesas,rcar-sysc.txt
index 0725fb37a973..d91715bc8d52 100644
--- a/Documentation/devicetree/bindings/power/renesas,rcar-sysc.txt
+++ b/Documentation/devicetree/bindings/power/renesas,rcar-sysc.txt
@@ -1,12 +1,14 @@
-DT bindings for the Renesas R-Car System Controller
+DT bindings for the Renesas R-Car (RZ/G) System Controller
 
 == System Controller Node ==
 
-The R-Car System Controller provides power management for the CPU cores and
-various coprocessors.
+The R-Car (RZ/G) System Controller provides power management for the CPU cores
+and various coprocessors.
 
 Required properties:
   - compatible: Must contain exactly one of the following:
+      - "renesas,r8a7743-sysc" (RZ/G1M)
+      - "renesas,r8a7745-sysc" (RZ/G1E)
       - "renesas,r8a7779-sysc" (R-Car H1)
       - "renesas,r8a7790-sysc" (R-Car H2)
       - "renesas,r8a7791-sysc" (R-Car M2-W)
diff --git a/Documentation/devicetree/bindings/pwm/pwm-bcm2835.txt b/Documentation/devicetree/bindings/pwm/pwm-bcm2835.txt
index fb6fb31bc4c4..cf573e85b11d 100644
--- a/Documentation/devicetree/bindings/pwm/pwm-bcm2835.txt
+++ b/Documentation/devicetree/bindings/pwm/pwm-bcm2835.txt
@@ -3,7 +3,7 @@ BCM2835 PWM controller (Raspberry Pi controller)
 Required properties:
 - compatible: should be "brcm,bcm2835-pwm"
 - reg: physical base address and length of the controller's registers
-- clock: This clock defines the base clock frequency of the PWM hardware
+- clocks: This clock defines the base clock frequency of the PWM hardware
   system, the period and the duty_cycle of the PWM signal is a multiple of
   the base period.
 - #pwm-cells: Should be 2. See pwm.txt in this directory for a description of
diff --git a/Documentation/devicetree/bindings/pwm/pwm-hibvt.txt b/Documentation/devicetree/bindings/pwm/pwm-hibvt.txt
new file mode 100644
index 000000000000..fa7849d67836
--- /dev/null
+++ b/Documentation/devicetree/bindings/pwm/pwm-hibvt.txt
@@ -0,0 +1,21 @@
+Hisilicon PWM controller
+
+Required properties:
+-compatible: should contain one SoC specific compatible string
+ The SoC specific strings supported including:
+	"hisilicon,hi3516cv300-pwm"
+	"hisilicon,hi3519v100-pwm"
+- reg: physical base address and length of the controller's registers.
+- clocks: phandle and clock specifier of the PWM reference clock.
+- resets: phandle and reset specifier for the PWM controller reset.
+- #pwm-cells: Should be 3. See pwm.txt in this directory for a description of
+  the cells format.
+
+Example:
+	pwm: pwm@12130000 {
+		compatible = "hisilicon,hi3516cv300-pwm";
+		reg = <0x12130000 0x10000>;
+		clocks = <&crg_ctrl HI3516CV300_PWM_CLK>;
+		resets = <&crg_ctrl 0x38 0>;
+		#pwm-cells = <3>;
+	};
diff --git a/Documentation/devicetree/bindings/regulator/pwm-regulator.txt b/Documentation/devicetree/bindings/regulator/pwm-regulator.txt
index 3aeba9f86ed8..bf85aa9ad6a7 100644
--- a/Documentation/devicetree/bindings/regulator/pwm-regulator.txt
+++ b/Documentation/devicetree/bindings/regulator/pwm-regulator.txt
@@ -59,7 +59,7 @@ Any property defined as part of the core regulator binding can also be used.
 
 Continuous Voltage With Enable GPIO Example:
 	pwm_regulator {
-		compatible = "pwm-regulator;
+		compatible = "pwm-regulator";
 		pwms = <&pwm1 0 8448 0>;
 		enable-gpios = <&gpio0 23 GPIO_ACTIVE_HIGH>;
 		regulator-min-microvolt = <1016000>;
@@ -76,7 +76,7 @@ Continuous Voltage With Enable GPIO Example:
 
 Voltage Table Example:
 	pwm_regulator {
-		compatible = "pwm-regulator;
+		compatible = "pwm-regulator";
 		pwms = <&pwm1 0 8448 0>;
 		regulator-min-microvolt = <1016000>;
 		regulator-max-microvolt = <1114000>;
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt
new file mode 100644
index 000000000000..b85885a298d8
--- /dev/null
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt
@@ -0,0 +1,98 @@
+Qualcomm ADSP Peripheral Image Loader
+
+This document defines the binding for a component that loads and boots firmware
+on the Qualcomm ADSP Hexagon core.
+
+- compatible:
+	Usage: required
+	Value type: <string>
+	Definition: must be one of:
+		    "qcom,msm8974-adsp-pil"
+		    "qcom,msm8996-adsp-pil"
+
+- interrupts-extended:
+	Usage: required
+	Value type: <prop-encoded-array>
+	Definition: must list the watchdog, fatal IRQs ready, handover and
+		    stop-ack IRQs
+
+- interrupt-names:
+	Usage: required
+	Value type: <stringlist>
+	Definition: must be "wdog", "fatal", "ready", "handover", "stop-ack"
+
+- clocks:
+	Usage: required
+	Value type: <prop-encoded-array>
+	Definition: reference to the xo clock to be held on behalf of the
+		    booting Hexagon core
+
+- clock-names:
+	Usage: required
+	Value type: <stringlist>
+	Definition: must be "xo"
+
+- cx-supply:
+	Usage: required
+	Value type: <phandle>
+	Definition: reference to the regulator to be held on behalf of the
+		    booting Hexagon core
+
+- memory-region:
+	Usage: required
+	Value type: <phandle>
+	Definition: reference to the reserved-memory for the ADSP
+
+- qcom,smem-states:
+	Usage: required
+	Value type: <phandle>
+	Definition: reference to the smem state for requesting the ADSP to
+		    shut down
+
+- qcom,smem-state-names:
+	Usage: required
+	Value type: <stringlist>
+	Definition: must be "stop"
+
+
+= SUBNODES
+The adsp node may have an subnode named "smd-edge" that describes the SMD edge,
+channels and devices related to the ADSP.  See ../soc/qcom/qcom,smd.txt for
+details on how to describe the SMD edge.
+
+
+= EXAMPLE
+The following example describes the resources needed to boot control the
+ADSP, as it is found on MSM8974 boards.
+
+	adsp {
+		compatible = "qcom,msm8974-adsp-pil";
+
+		interrupts-extended = <&intc 0 162 IRQ_TYPE_EDGE_RISING>,
+				      <&adsp_smp2p_in 0 IRQ_TYPE_EDGE_RISING>,
+				      <&adsp_smp2p_in 1 IRQ_TYPE_EDGE_RISING>,
+				      <&adsp_smp2p_in 2 IRQ_TYPE_EDGE_RISING>,
+				      <&adsp_smp2p_in 3 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "wdog",
+				  "fatal",
+				  "ready",
+				  "handover",
+				  "stop-ack";
+
+		clocks = <&rpmcc RPM_CXO_CLK>;
+		clock-names = "xo";
+
+		cx-supply = <&pm8841_s2>;
+
+		memory-region = <&adsp_region>;
+
+		qcom,smem-states = <&adsp_smp2p_out 0>;
+		qcom,smem-state-names = "stop";
+
+		smd-edge {
+			interrupts = <0 156 IRQ_TYPE_EDGE_RISING>;
+
+			qcom,ipc = <&apcs 8 8>;
+			qcom,smd-edge = <1>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt b/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt
index 0d2361ebe3d7..d420f84ddfb0 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt
@@ -60,8 +60,8 @@ on the Qualcomm WCNSS core.
 		    see ../reserved-memory/reserved-memory.txt
 
 = SUBNODES
-A single subnode of the WCNSS PIL describes the attached rf module and its
-resource dependencies.
+A required subnode of the WCNSS PIL is used to describe the attached rf module
+and its resource dependencies. It is described by the following properties:
 
 - compatible:
 	Usage: required
@@ -90,6 +90,11 @@ resource dependencies.
 	Definition: reference to the regulators to be held on behalf of the
 		    booting of the WCNSS core
 
+
+The wcnss node can also have an subnode named "smd-edge" that describes the SMD
+edge, channels and devices related to the WCNSS.
+See ../soc/qcom/qcom,smd.txt for details on how to describe the SMD edge.
+
 = EXAMPLE
 The following example describes the resources needed to boot control the WCNSS,
 with attached WCN3680, as it is commonly found on MSM8974 boards.
@@ -129,4 +134,25 @@ pronto@fb204000 {
 		vddpa-supply = <&pm8941_l19>;
 		vdddig-supply = <&pm8941_s3>;
 	};
+
+	smd-edge {
+		interrupts = <0 142 1>;
+
+		qcom,ipc = <&apcs 8 17>;
+		qcom,smd-edge = <6>;
+		qcom,remote-pid = <4>;
+
+		label = "pronto";
+
+		wcnss {
+			compatible = "qcom,wcnss";
+			qcom,smd-channels = "WCNSS_CTRL";
+
+			qcom,mmio = <&pronto>;
+
+			bt {
+				compatible = "qcom,wcnss-bt";
+			};
+		};
+	};
 };
diff --git a/Documentation/devicetree/bindings/reserved-memory/ramoops.txt b/Documentation/devicetree/bindings/reserved-memory/ramoops.txt
index e81f821a2135..0eba562fe5c6 100644
--- a/Documentation/devicetree/bindings/reserved-memory/ramoops.txt
+++ b/Documentation/devicetree/bindings/reserved-memory/ramoops.txt
@@ -46,3 +46,6 @@ Optional properties:
   (defaults to buffered mappings)
 
 - no-dump-oops: if present, only dump panics (defaults to panics and oops)
+
+- flags: if present, pass ramoops behavioral flags (defaults to 0,
+  see include/linux/pstore_ram.h RAMOOPS_FLAG_* for flag values).
diff --git a/Documentation/devicetree/bindings/reset/oxnas,reset.txt b/Documentation/devicetree/bindings/reset/oxnas,reset.txt
index 6f06db930030..d27ccb5d04fc 100644
--- a/Documentation/devicetree/bindings/reset/oxnas,reset.txt
+++ b/Documentation/devicetree/bindings/reset/oxnas,reset.txt
@@ -5,45 +5,19 @@ Please also refer to reset.txt in this directory for common reset
 controller binding usage.
 
 Required properties:
-- compatible: Should be "oxsemi,ox810se-reset"
+- compatible: For OX810SE, should be "oxsemi,ox810se-reset"
+	      For OX820, should be "oxsemi,ox820-reset"
 - #reset-cells: 1, see below
 
 Parent node should have the following properties :
-- compatible: Should be "oxsemi,ox810se-sys-ctrl", "syscon", "simple-mfd"
+- compatible: For OX810SE, should be :
+			"oxsemi,ox810se-sys-ctrl", "syscon", "simple-mfd"
+	      For OX820, should be :
+			"oxsemi,ox820-sys-ctrl", "syscon", "simple-mfd"
 
-For OX810SE, the indices are :
- - 0 : ARM
- - 1 : COPRO
- - 2 : Reserved
- - 3 : Reserved
- - 4 : USBHS
- - 5 : USBHSPHY
- - 6 : MAC
- - 7 : PCI
- - 8 : DMA
- - 9 : DPE
- - 10 : DDR
- - 11 : SATA
- - 12 : SATA_LINK
- - 13 : SATA_PHY
- - 14 : Reserved
- - 15 : NAND
- - 16 : GPIO
- - 17 : UART1
- - 18 : UART2
- - 19 : MISC
- - 20 : I2S
- - 21 : AHB_MON
- - 22 : UART3
- - 23 : UART4
- - 24 : SGDMA
- - 25 : Reserved
- - 26 : Reserved
- - 27 : Reserved
- - 28 : Reserved
- - 29 : Reserved
- - 30 : Reserved
- - 31 : BUS
+Reset indices are in dt-bindings include files :
+- For OX810SE: include/dt-bindings/reset/oxsemi,ox810se.h
+- For OX820: include/dt-bindings/reset/oxsemi,ox820.h
 
 example:
 
diff --git a/Documentation/devicetree/bindings/reset/renesas,rst.txt b/Documentation/devicetree/bindings/reset/renesas,rst.txt
new file mode 100644
index 000000000000..fe5e0f37b3c9
--- /dev/null
+++ b/Documentation/devicetree/bindings/reset/renesas,rst.txt
@@ -0,0 +1,37 @@
+DT bindings for the Renesas R-Car and RZ/G Reset Controllers
+
+The R-Car and RZ/G Reset Controllers provide reset control, and implement the
+following functions:
+  - Latching of the levels on mode pins when PRESET# is negated,
+  - Mode monitoring register,
+  - Reset control of peripheral devices (on R-Car Gen1),
+  - Watchdog timer (on R-Car Gen1),
+  - Register-based reset control and boot address registers for the various CPU
+    cores (on R-Car Gen2 and Gen3, and on RZ/G).
+
+
+Required properties:
+  - compatible: Should be
+		  - "renesas,<soctype>-reset-wdt" for R-Car Gen1,
+		  - "renesas,<soctype>-rst" for R-Car Gen2 and Gen3, and RZ/G
+		Examples with soctypes are:
+		  - "renesas,r8a7743-rst" (RZ/G1M)
+		  - "renesas,r8a7745-rst" (RZ/G1E)
+		  - "renesas,r8a7778-reset-wdt" (R-Car M1A)
+		  - "renesas,r8a7779-reset-wdt" (R-Car H1)
+		  - "renesas,r8a7790-rst" (R-Car H2)
+		  - "renesas,r8a7791-rst" (R-Car M2-W)
+		  - "renesas,r8a7792-rst" (R-Car V2H
+		  - "renesas,r8a7793-rst" (R-Car M2-N)
+		  - "renesas,r8a7794-rst" (R-Car E2)
+		  - "renesas,r8a7795-rst" (R-Car H3)
+		  - "renesas,r8a7796-rst" (R-Car M3-W)
+  - reg: Address start and address range for the device.
+
+
+Example:
+
+	rst: reset-controller@e6160000 {
+		compatible = "renesas,r8a7795-rst";
+		reg = <0 0xe6160000 0 0x0200>;
+	};
diff --git a/Documentation/devicetree/bindings/reset/st,sti-powerdown.txt b/Documentation/devicetree/bindings/reset/st,sti-powerdown.txt
index 1cfd21d1dfa1..92527138bc93 100644
--- a/Documentation/devicetree/bindings/reset/st,sti-powerdown.txt
+++ b/Documentation/devicetree/bindings/reset/st,sti-powerdown.txt
@@ -16,15 +16,14 @@ Please refer to reset.txt in this directory for common reset
 controller binding usage.
 
 Required properties:
-- compatible: Should be "st,<chip>-powerdown"
-	ex: "st,stih415-powerdown", "st,stih416-powerdown"
+- compatible: Should be "st,stih407-powerdown"
 - #reset-cells: 1, see below
 
 example:
 
 	powerdown: powerdown-controller {
+		compatible = "st,stih407-powerdown";
 		#reset-cells = <1>;
-		compatible = "st,stih415-powerdown";
 	};
 
 
@@ -37,11 +36,10 @@ index specifying which channel to use, as described in reset.txt
 
 example:
 
-	usb1: usb@fe200000 {
-		resets	= <&powerdown STIH41X_USB1_POWERDOWN>;
+	st_dwc3: dwc3@8f94000 {
+		resets          = <&powerdown STIH407_USB3_POWERDOWN>,
 	};
 
 Macro definitions for the supported reset channels can be found in:
 
-include/dt-bindings/reset/stih415-resets.h
-include/dt-bindings/reset/stih416-resets.h
+include/dt-bindings/reset/stih407-resets.h
diff --git a/Documentation/devicetree/bindings/reset/st,sti-softreset.txt b/Documentation/devicetree/bindings/reset/st,sti-softreset.txt
index 891a2fd85ed6..a21658f18fe6 100644
--- a/Documentation/devicetree/bindings/reset/st,sti-softreset.txt
+++ b/Documentation/devicetree/bindings/reset/st,sti-softreset.txt
@@ -15,15 +15,14 @@ Please refer to reset.txt in this directory for common reset
 controller binding usage.
 
 Required properties:
-- compatible: Should be "st,<chip>-softreset" example:
-	"st,stih415-softreset" or "st,stih416-softreset";
+- compatible: Should be st,stih407-softreset";
 - #reset-cells: 1, see below
 
 example:
 
 	softreset: softreset-controller {
 		#reset-cells = <1>;
-		compatible = "st,stih415-softreset";
+		compatible = "st,stih407-softreset";
 	};
 
 
@@ -42,5 +41,4 @@ example:
 
 Macro definitions for the supported reset channels can be found in:
 
-include/dt-bindings/reset/stih415-resets.h
-include/dt-bindings/reset/stih416-resets.h
+include/dt-bindings/reset/stih407-resets.h
diff --git a/Documentation/devicetree/bindings/rng/omap_rng.txt b/Documentation/devicetree/bindings/rng/omap_rng.txt
index 6a62acd86953..471477299ece 100644
--- a/Documentation/devicetree/bindings/rng/omap_rng.txt
+++ b/Documentation/devicetree/bindings/rng/omap_rng.txt
@@ -1,4 +1,4 @@
-OMAP SoC HWRNG Module
+OMAP SoC and Inside-Secure HWRNG Module
 
 Required properties:
 
@@ -6,11 +6,13 @@ Required properties:
   RNG versions:
   - "ti,omap2-rng" for OMAP2.
   - "ti,omap4-rng" for OMAP4, OMAP5 and AM33XX.
+  - "inside-secure,safexcel-eip76" for SoCs with EIP76 IP block
   Note that these two versions are incompatible.
 - ti,hwmods: Name of the hwmod associated with the RNG module
 - reg : Offset and length of the register set for the module
 - interrupts : the interrupt number for the RNG module.
-		Only used for "ti,omap4-rng".
+		Used for "ti,omap4-rng" and "inside-secure,safexcel-eip76"
+- clocks: the trng clock source
 
 Example:
 /* AM335x */
@@ -20,3 +22,11 @@ rng: rng@48310000 {
 	reg = <0x48310000 0x2000>;
 	interrupts = <111>;
 };
+
+/* SafeXcel IP-76 */
+trng: rng@f2760000 {
+	compatible = "inside-secure,safexcel-eip76";
+	reg = <0xf2760000 0x7d>;
+	interrupts = <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>;
+	clocks = <&cpm_syscon0 1 25>;
+};
diff --git a/Documentation/devicetree/bindings/scsi/hisilicon-sas.txt b/Documentation/devicetree/bindings/scsi/hisilicon-sas.txt
index bf2411f366e5..2a42a323fa1a 100644
--- a/Documentation/devicetree/bindings/scsi/hisilicon-sas.txt
+++ b/Documentation/devicetree/bindings/scsi/hisilicon-sas.txt
@@ -6,6 +6,7 @@ Main node required properties:
   - compatible : value should be as follows:
 	(a) "hisilicon,hip05-sas-v1" for v1 hw in hip05 chipset
 	(b) "hisilicon,hip06-sas-v2" for v2 hw in hip06 chipset
+	(c) "hisilicon,hip07-sas-v2" for v2 hw in hip07 chipset
   - sas-addr : array of 8 bytes for host SAS address
   - reg : Address and length of the SAS register
   - hisilicon,sas-syscon: phandle of syscon used for sas control
diff --git a/Documentation/devicetree/bindings/security/tpm/ibmvtpm.txt b/Documentation/devicetree/bindings/security/tpm/ibmvtpm.txt
new file mode 100644
index 000000000000..d89f99971368
--- /dev/null
+++ b/Documentation/devicetree/bindings/security/tpm/ibmvtpm.txt
@@ -0,0 +1,41 @@
+* Device Tree Bindings for IBM Virtual Trusted Platform Module(vtpm)
+
+Required properties:
+
+- compatible            : property name that conveys the platform architecture
+                          identifiers, as 'IBM,vtpm'
+- device_type           : specifies type of virtual device
+- interrupts            : property specifying the interrupt source number and
+                          sense code associated with this virtual I/O Adapters
+- ibm,my-drc-index      : integer index for the connector between the device
+                          and its parent - present only if Dynamic
+                          Reconfiguration(DR) Connector is enabled
+- ibm,#dma-address-cells: specifies the number of cells that are used to
+                          encode the physical address field of dma-window
+                          properties
+- ibm,#dma-size-cells   : specifies the number of cells that are used to
+                          encode the size field of dma-window properties
+- ibm,my-dma-window     : specifies DMA window associated with this virtual
+                          IOA
+- ibm,loc-code          : specifies the unique and persistent location code
+                          associated with this virtual I/O Adapters
+- linux,sml-base        : 64-bit base address of the reserved memory allocated
+                          for the firmware event log
+- linux,sml-size        : size of the memory allocated for the firmware event log
+
+Example (IBM Virtual Trusted Platform Module)
+---------------------------------------------
+
+                vtpm@30000003 {
+                        ibm,#dma-size-cells = <0x2>;
+                        compatible = "IBM,vtpm";
+                        device_type = "IBM,vtpm";
+                        ibm,my-drc-index = <0x30000003>;
+                        ibm,#dma-address-cells = <0x2>;
+                        linux,sml-base = <0xc60e 0x0>;
+                        interrupts = <0xa0003 0x0>;
+                        ibm,my-dma-window = <0x10000003 0x0 0x0 0x0 0x10000000>;
+                        ibm,loc-code = "U8286.41A.10082DV-V3-C3";
+                        reg = <0x30000003>;
+                        linux,sml-size = <0xbce10200>;
+                };
diff --git a/Documentation/devicetree/bindings/security/tpm/tpm-i2c.txt b/Documentation/devicetree/bindings/security/tpm/tpm-i2c.txt
new file mode 100644
index 000000000000..8cb638b7e89c
--- /dev/null
+++ b/Documentation/devicetree/bindings/security/tpm/tpm-i2c.txt
@@ -0,0 +1,21 @@
+* Device Tree Bindings for I2C based Trusted Platform Module(TPM)
+
+Required properties:
+
+- compatible     : 'manufacturer,model', eg. nuvoton,npct650
+- label          : human readable string describing the device, eg. "tpm"
+- linux,sml-base : 64-bit base address of the reserved memory allocated for
+                   the firmware event log
+- linux,sml-size : size of the memory allocated for the firmware event log
+
+Example (for OpenPower Systems with Nuvoton TPM 2.0 on I2C)
+----------------------------------------------------------
+
+tpm@57 {
+	reg = <0x57>;
+	label = "tpm";
+	compatible = "nuvoton,npct650", "nuvoton,npct601";
+	linux,sml-base = <0x7f 0xfd450000>;
+	linux,sml-size = <0x10000>;
+	status = "okay";
+};
diff --git a/Documentation/devicetree/bindings/security/tpm/tpm_tis_mmio.txt b/Documentation/devicetree/bindings/security/tpm/tpm_tis_mmio.txt
new file mode 100644
index 000000000000..41d740545189
--- /dev/null
+++ b/Documentation/devicetree/bindings/security/tpm/tpm_tis_mmio.txt
@@ -0,0 +1,25 @@
+Trusted Computing Group MMIO Trusted Platform Module
+
+The TCG defines multi vendor standard for accessing a TPM chip, this
+is the standard protocol defined to access the TPM via MMIO. Typically
+this interface will be implemented over Intel's LPC bus.
+
+Refer to the 'TCG PC Client Specific TPM Interface Specification (TIS)' TCG
+publication for the specification.
+
+Required properties:
+
+- compatible: should contain a string below for the chip, followed by
+              "tcg,tpm-tis-mmio". Valid chip strings are:
+	          * "atmel,at97sc3204"
+- reg: The location of the MMIO registers, should be at least 0x5000 bytes
+- interrupt-parent/interrupts: An optional interrupt indicating command completion.
+
+Example:
+
+	tpm_tis@90000 {
+				compatible = "atmel,at97sc3204", "tcg,tpm-tis-mmio";
+				reg = <0x90000 0x5000>;
+				interrupt-parent = <&EIC0>;
+				interrupts = <1 2>;
+	};
diff --git a/Documentation/devicetree/bindings/serial/nvidia,tegra20-hsuart.txt b/Documentation/devicetree/bindings/serial/nvidia,tegra20-hsuart.txt
index 845850caf088..c93a2d1c1a65 100644
--- a/Documentation/devicetree/bindings/serial/nvidia,tegra20-hsuart.txt
+++ b/Documentation/devicetree/bindings/serial/nvidia,tegra20-hsuart.txt
@@ -10,7 +10,7 @@ Required properties:
   See ../reset/reset.txt for details.
 - reset-names : Must include the following entries:
   - serial
-- dmas : Must contain an entry for each entry in clock-names.
+- dmas : Must contain an entry for each entry in dma-names.
   See ../dma/dma.txt for details.
 - dma-names : Must include the following entries:
   - rx
diff --git a/Documentation/devicetree/bindings/powerpc/fsl/guts.txt b/Documentation/devicetree/bindings/soc/fsl/guts.txt
index b71b2039e112..07adca914d3d 100644
--- a/Documentation/devicetree/bindings/powerpc/fsl/guts.txt
+++ b/Documentation/devicetree/bindings/soc/fsl/guts.txt
@@ -25,6 +25,9 @@ Recommended properties:
  - fsl,liodn-bits : Indicates the number of defined bits in the LIODN
    registers, for those SOCs that have a PAMU device.
 
+ - little-endian : Indicates that the global utilities block is little
+   endian. The default is big endian.
+
 Examples:
 	global-utilities@e0000 {	/* global utilities block */
 		compatible = "fsl,mpc8548-guts";
diff --git a/Documentation/devicetree/bindings/soc/mediatek/scpsys.txt b/Documentation/devicetree/bindings/soc/mediatek/scpsys.txt
index e8f15e34027f..16fe94d7783c 100644
--- a/Documentation/devicetree/bindings/soc/mediatek/scpsys.txt
+++ b/Documentation/devicetree/bindings/soc/mediatek/scpsys.txt
@@ -9,17 +9,20 @@ domain control.
 
 The driver implements the Generic PM domain bindings described in
 power/power_domain.txt. It provides the power domains defined in
-include/dt-bindings/power/mt8173-power.h.
+include/dt-bindings/power/mt8173-power.h and mt2701-power.h.
 
 Required properties:
-- compatible: Must be "mediatek,mt8173-scpsys"
+- compatible: Should be one of:
+	- "mediatek,mt2701-scpsys"
+	- "mediatek,mt8173-scpsys"
 - #power-domain-cells: Must be 1
 - reg: Address range of the SCPSYS unit
 - infracfg: must contain a phandle to the infracfg controller
 - clock, clock-names: clocks according to the common clock binding.
-                      The clocks needed "mm", "mfg", "venc" and "venc_lt".
-		      These are the clocks which hardware needs to be enabled
-		      before enabling certain power domains.
+                      These are clocks which hardware needs to be
+                      enabled before enabling certain power domains.
+	Required clocks for MT2701: "mm", "mfg", "ethif"
+	Required clocks for MT8173: "mm", "mfg", "venc", "venc_lt"
 
 Optional properties:
 - vdec-supply: Power supply for the vdec power domain
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,smd.txt b/Documentation/devicetree/bindings/soc/qcom/qcom,smd.txt
index 97d9b3e1bf39..ea1dc75ec9ea 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,smd.txt
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,smd.txt
@@ -43,6 +43,13 @@ The edge is described by the following properties:
 	Definition: the identifier for the remote processor as known by the rest
 		    of the system.
 
+- label:
+	Usage: optional
+	Value type: <string>
+	Definition: name of the edge, used for debugging and identification
+		    purposes. The node name will be used if this is not
+		    present.
+
 = SMD DEVICES
 
 In turn, subnodes of the "edges" represent devices tied to SMD channels on that
diff --git a/Documentation/devicetree/bindings/sound/axentia,tse850-pcm5142.txt b/Documentation/devicetree/bindings/sound/axentia,tse850-pcm5142.txt
new file mode 100644
index 000000000000..5b9b38f578bb
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/axentia,tse850-pcm5142.txt
@@ -0,0 +1,88 @@
+Devicetree bindings for the Axentia TSE-850 audio complex
+
+Required properties:
+  - compatible: "axentia,tse850-pcm5142"
+  - axentia,ssc-controller: The phandle of the atmel SSC controller used as
+    cpu dai.
+  - axentia,audio-codec: The phandle of the PCM5142 codec.
+  - axentia,add-gpios: gpio specifier that controls the mixer.
+  - axentia,loop1-gpios: gpio specifier that controls loop relays on channel 1.
+  - axentia,loop2-gpios: gpio specifier that controls loop relays on channel 2.
+  - axentia,ana-supply: Regulator that supplies the output amplifier. Must
+    support voltages in the 2V - 20V range, in 1V steps.
+
+The schematics explaining the gpios are as follows:
+
+               loop1 relays
+   IN1 +---o  +------------+  o---+ OUT1
+            \                /
+             +              +
+             |   /          |
+             +--o  +--.     |
+             |  add   |     |
+             |        V     |
+             |      .---.   |
+   DAC +----------->|Sum|---+
+             |      '---'   |
+             |              |
+             +              +
+
+   IN2 +---o--+------------+--o---+ OUT2
+               loop2 relays
+
+The 'loop1' gpio pin controlls two relays, which are either in loop position,
+meaning that input and output are directly connected, or they are in mixer
+position, meaning that the signal is passed through the 'Sum' mixer. Similarly
+for 'loop2'.
+
+In the above, the 'loop1' relays are inactive, thus feeding IN1 to the mixer
+(if 'add' is active) and feeding the mixer output to OUT1. The 'loop2' relays
+are active, short-cutting the TSE-850 from channel 2. IN1, IN2, OUT1 and OUT2
+are TSE-850 connectors and DAC is the PCB name of the (filtered) output from
+the PCM5142 codec.
+
+Example:
+
+	&i2c {
+		codec: pcm5142@4c {
+			compatible = "ti,pcm5142";
+
+			reg = <0x4c>;
+
+			AVDD-supply = <&reg_3v3>;
+			DVDD-supply = <&reg_3v3>;
+			CPVDD-supply = <&reg_3v3>;
+
+			clocks = <&sck>;
+
+			pll-in = <3>;
+			pll-out = <6>;
+		};
+	};
+
+	ana: ana-reg {
+		compatible = "pwm-regulator";
+
+		regulator-name = "ANA";
+
+		pwms = <&pwm0 2 1000 PWM_POLARITY_INVERTED>;
+		pwm-dutycycle-unit = <1000>;
+		pwm-dutycycle-range = <100 1000>;
+
+		regulator-min-microvolt = <2000000>;
+		regulator-max-microvolt = <20000000>;
+		regulator-ramp-delay = <1000>;
+	};
+
+	sound {
+		compatible = "axentia,tse850-pcm5142";
+
+		axentia,ssc-controller = <&ssc0>;
+		axentia,audio-codec = <&codec>;
+
+		axentia,add-gpios = <&pioA 8 GPIO_ACTIVE_LOW>;
+		axentia,loop1-gpios = <&pioA 10 GPIO_ACTIVE_LOW>;
+		axentia,loop2-gpios = <&pioA 11 GPIO_ACTIVE_LOW>;
+
+		axentia,ana-supply = <&ana>;
+	};
diff --git a/Documentation/devicetree/bindings/sound/cs35l34.txt b/Documentation/devicetree/bindings/sound/cs35l34.txt
new file mode 100644
index 000000000000..b218ead2e68e
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/cs35l34.txt
@@ -0,0 +1,64 @@
+CS35L34 Speaker Amplifier
+
+Required properties:
+
+  - compatible : "cirrus,cs35l34"
+
+  - reg : the I2C address of the device for I2C.
+
+  - VA-supply, VP-supply : power supplies for the device,
+    as covered in
+    Documentation/devicetree/bindings/regulator/regulator.txt.
+
+  - cirrus,boost-vtge-millivolt : Boost Voltage Value.  Configures the boost
+    converter's output voltage in mV. The range is from VP to 8V with
+    increments of 100mV.
+
+  - cirrus,boost-nanohenry: Inductor value for boost converter. The value is
+    in nH and they can be values of 1000nH, 1100nH, 1200nH, 1500nH, and 2200nH.
+
+Optional properties:
+
+  - reset-gpios: GPIO used to reset the amplifier.
+
+  - interrupt-parent : Specifies the phandle of the interrupt controller to
+    which the IRQs from CS35L34 are delivered to.
+  - interrupts : IRQ line info CS35L34.
+    (See Documentation/devicetree/bindings/interrupt-controller/interrupts.txt
+    for further information relating to interrupt properties)
+
+  - cirrus,boost-peak-milliamp : Boost converter peak current limit in mA. The
+    range starts at 1200mA and goes to a maximum of 3840mA with increments of
+    80mA. The default value is 2480mA.
+
+  - cirrus,i2s-sdinloc : ADSP SDIN I2S channel location. Indicates whether the
+    received mono data is in the left or right portion of the I2S frame
+    according to the AD0 pin or directly via this configuration.
+    0x0 (Default) = Selected by AD0 input (if AD0 = LOW, use left channel),
+    0x2 = Left,
+    0x1 = Selected by the inversion of the AD0 input (if AD0 = LOW, use right
+    channel),
+    0x3 = Right.
+
+  - cirrus,gain-zc-disable: Boolean property. If set, the gain change will take
+    effect without waiting for a zero cross.
+
+  - cirrus,tdm-rising-edge: Boolean property. If set, data is on the rising edge of
+    SCLK. Otherwise, data is on the falling edge of SCLK.
+
+
+Example:
+
+cs35l34: cs35l34@40 {
+	compatible = "cirrus,cs35l34";
+	reg = <0x40>;
+
+	interrupt-parent = <&gpio8>;
+	interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
+
+	reset-gpios = <&gpio 10 0>;
+
+	cirrus,boost-vtge-milltvolt = <8000>; /* 8V */
+	cirrus,boost-ind-nanohenry = <1000>; /* 1uH */
+	cirrus,boost-peak-milliamp = <3000>; /* 3A */
+};
diff --git a/Documentation/devicetree/bindings/sound/cs42l42.txt b/Documentation/devicetree/bindings/sound/cs42l42.txt
new file mode 100644
index 000000000000..9a2c5e2423d5
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/cs42l42.txt
@@ -0,0 +1,110 @@
+CS42L42 audio CODEC
+
+Required properties:
+
+  - compatible : "cirrus,cs42l42"
+
+  - reg : the I2C address of the device for I2C.
+
+  - VP-supply, VCP-supply, VD_FILT-supply, VL-supply, VA-supply :
+  power supplies for the device, as covered in
+  Documentation/devicetree/bindings/regulator/regulator.txt.
+
+Optional properties:
+
+  - reset-gpios : a GPIO spec for the reset pin. If specified, it will be
+  deasserted before communication to the codec starts.
+
+  - interrupt-parent : Specifies the phandle of the interrupt controller to
+  which the IRQs from CS42L42 are delivered to.
+
+  - interrupts : IRQ line info CS42L42.
+  (See Documentation/devicetree/bindings/interrupt-controller/interrupts.txt
+  for further information relating to interrupt properties)
+
+  - cirrus,ts-inv : Boolean property. For jacks that invert the tip sense
+  polarity. Normal jacks will short tip sense pin to HS1 when headphones are
+  plugged in and leave tip sense floating when not plugged in. Inverting jacks
+  short tip sense when unplugged and float when plugged in.
+
+  0 = (Default) Non-inverted
+  1 = Inverted
+
+  - cirrus,ts-dbnc-rise : Debounce the rising edge of TIP_SENSE_PLUG. With no
+  debounce, the tip sense pin might be noisy on a plug event.
+
+  0 - 0ms,
+  1 - 125ms,
+  2 - 250ms,
+  3 - 500ms,
+  4 - 750ms,
+  5 - (Default) 1s,
+  6 - 1.25s,
+  7 - 1.5s,
+
+  - cirrus,ts-dbnc-fall : Debounce the falling edge of TIP_SENSE_UNPLUG.
+  With no debounce, the tip sense pin might be noisy on an unplug event.
+
+  0 - 0ms,
+  1 - 125ms,
+  2 - 250ms,
+  3 - 500ms,
+  4 - 750ms,
+  5 - (Default) 1s,
+  6 - 1.25s,
+  7 - 1.5s,
+
+  - cirrus,btn-det-init-dbnce : This sets how long the driver sleeps after
+  enabling button detection interrupts. After auto-detection and before
+  servicing button interrupts, the HS bias needs time to settle. If you
+  don't wait, there is possibility for erroneous button interrupt.
+
+  0ms - 200ms,
+  Default = 100ms
+
+  - cirrus,btn-det-event-dbnce : This sets how long the driver delays after
+  receiving a button press interrupt. With level detect interrupts, you want
+  to wait a small amount of time to make sure the button press is making a
+  clean connection with the bias resistors.
+
+  0ms - 20ms,
+  Default = 10ms
+
+  - cirrus,bias-lvls : For a level-detect headset button scheme, each button
+  will bias the mic pin to a certain voltage. To determine which button was
+  pressed, the driver will compare this biased voltage to sequential,
+  decreasing voltages and will stop when a comparator is tripped,
+  indicating a comparator voltage < bias voltage. This value represents a
+  percentage of the internally generated HS bias voltage. For different
+  hardware setups, a designer might want to tweak this. This is an array of
+  descending values for the comparator voltage.
+
+  Array of 4 values
+  Each 0-63
+  < x1 x2 x3 x4 >
+  Default = < 15 8 4 1>
+
+
+Example:
+
+cs42l42: cs42l42@48 {
+	compatible = "cirrus,cs42l42";
+	reg = <0x48>;
+	VA-supply = <&dummy_vreg>;
+	VP-supply = <&dummy_vreg>;
+	VCP-supply = <&dummy_vreg>;
+	VD_FILT-supply = <&dummy_vreg>;
+	VL-supply = <&dummy_vreg>;
+
+	reset-gpios = <&axi_gpio_0 1 0>;
+	interrupt-parent = <&gpio0>;
+	interrupts = <55 8>
+
+	cirrus,ts-inv = <0x00>;
+	cirrus,ts-dbnc-rise = <0x05>;
+	cirrus,ts-dbnc-fall = <0x00>;
+	cirrus,btn-det-init-dbnce = <100>;
+	cirrus,btn-det-event-dbnce = <10>;
+	cirrus,bias-lvls = <0x0F 0x08 0x04 0x01>;
+	cirrus,hs-bias-ramp-rate = <0x02>;
+};
\ No newline at end of file
diff --git a/Documentation/devicetree/bindings/sound/davinci-mcbsp.txt b/Documentation/devicetree/bindings/sound/davinci-mcbsp.txt
index 55b53e1fd72c..e0b6165c9cfc 100644
--- a/Documentation/devicetree/bindings/sound/davinci-mcbsp.txt
+++ b/Documentation/devicetree/bindings/sound/davinci-mcbsp.txt
@@ -43,7 +43,7 @@ mcbsp0: mcbsp@1d10000 {
 		<0x00310000 0x1000>;
 	reg-names = "mpu", "dat";
 	interrupts = <97 98>;
-	interrupts-names = "rx", "tx";
+	interrupt-names = "rx", "tx";
 	dmas = <&edma0 3 1
 		&edma0 2 1>;
 	dma-names = "tx", "rx";
diff --git a/Documentation/devicetree/bindings/sound/qcom,apq8016-sbc.txt b/Documentation/devicetree/bindings/sound/qcom,apq8016-sbc.txt
index d9d8635ff94c..6a4aadc4ce06 100644
--- a/Documentation/devicetree/bindings/sound/qcom,apq8016-sbc.txt
+++ b/Documentation/devicetree/bindings/sound/qcom,apq8016-sbc.txt
@@ -44,8 +44,7 @@ Required dai-link subnodes:
 Required CPU/CODEC subnodes properties:
 
 -link-name		: Name of the dai link.
--sound-dai		: phandle and port of CPU/CODEC
--capture-dai		: phandle and port of CPU/CODEC
+-sound-dai		: phandle/s and port of CPU/CODEC
 
 Example:
 
@@ -73,7 +72,7 @@ sound: sound {
 			sound-dai = <&lpass MI2S_PRIMARY>;
 		};
 		codec {
-			sound-dai = <&wcd_codec 0>;
+			sound-dai = <&lpass_codec 0>, <&wcd_codec 0>;
 		};
 	};
 
diff --git a/Documentation/devicetree/bindings/sound/qcom,msm8916-wcd-analog.txt b/Documentation/devicetree/bindings/sound/qcom,msm8916-wcd-analog.txt
new file mode 100644
index 000000000000..ccb401cfef9d
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/qcom,msm8916-wcd-analog.txt
@@ -0,0 +1,85 @@
+msm8916 analog audio CODEC
+
+Bindings for codec Analog IP which is integrated in pmic pm8916,
+
+## Bindings for codec core on pmic:
+
+Required properties
+ - compatible = "qcom,pm8916-wcd-analog-codec";
+ - reg: represents the slave base address provided to the peripheral.
+ - interrupt-parent : The parent interrupt controller.
+ - interrupts: List of interrupts in given SPMI peripheral.
+ - interrupt-names: Names specified to above list of interrupts in same
+		    order. List of supported interrupt names are:
+  "cdc_spk_cnp_int" - Speaker click and pop interrupt.
+  "cdc_spk_clip_int" - Speaker clip interrupt.
+  "cdc_spk_ocp_int" - Speaker over current protect interrupt.
+  "mbhc_ins_rem_det1" - jack insert removal detect interrupt 1.
+  "mbhc_but_rel_det" - button release interrupt.
+  "mbhc_but_press_det" - button press event
+  "mbhc_ins_rem_det" - jack insert removal detect interrupt.
+  "mbhc_switch_int"	- multi button headset interrupt.
+  "cdc_ear_ocp_int" - Earphone over current protect interrupt.
+  "cdc_hphr_ocp_int" - Headphone R over current protect interrupt.
+  "cdc_hphl_ocp_det" - Headphone L over current protect interrupt.
+  "cdc_ear_cnp_int" - earphone cnp interrupt.
+  "cdc_hphr_cnp_int" - hphr click and pop interrupt.
+  "cdc_hphl_cnp_int" - hphl click and pop interrupt.
+
+ - clocks: Handle to mclk.
+ - clock-names: should be "mclk"
+ - vdd-cdc-io-supply: phandle to VDD_CDC_IO regulator DT node.
+ - vdd-cdc-tx-rx-cx-supply: phandle to VDD_CDC_TX/RX/CX regulator DT node.
+ - vdd-micbias-supply: phandle of VDD_MICBIAS supply's regulator DT node.
+
+Optional Properties:
+- qcom,micbias1-ext-cap: boolean, present if micbias1 has external capacitor
+			 connected.
+- qcom,micbias2-ext-cap: boolean, present if micbias2 has external capacitor
+			 connected.
+
+Example:
+
+spmi_bus {
+	...
+	audio-codec@f000{
+		compatible = "qcom,pm8916-wcd-analog-codec";
+		reg = <0xf000 0x200>;
+		reg-names = "pmic-codec-core";
+		clocks = <&gcc GCC_CODEC_DIGCODEC_CLK>;
+		clock-names = "mclk";
+		interrupt-parent = <&spmi_bus>;
+		interrupts = <0x1 0xf0 0x0 IRQ_TYPE_NONE>,
+			     <0x1 0xf0 0x1 IRQ_TYPE_NONE>,
+			     <0x1 0xf0 0x2 IRQ_TYPE_NONE>,
+			     <0x1 0xf0 0x3 IRQ_TYPE_NONE>,
+			     <0x1 0xf0 0x4 IRQ_TYPE_NONE>,
+			     <0x1 0xf0 0x5 IRQ_TYPE_NONE>,
+			     <0x1 0xf0 0x6 IRQ_TYPE_NONE>,
+			     <0x1 0xf0 0x7 IRQ_TYPE_NONE>,
+			     <0x1 0xf1 0x0 IRQ_TYPE_NONE>,
+			     <0x1 0xf1 0x1 IRQ_TYPE_NONE>,
+			     <0x1 0xf1 0x2 IRQ_TYPE_NONE>,
+			     <0x1 0xf1 0x3 IRQ_TYPE_NONE>,
+			     <0x1 0xf1 0x4 IRQ_TYPE_NONE>,
+			     <0x1 0xf1 0x5 IRQ_TYPE_NONE>;
+		interrupt-names = "cdc_spk_cnp_int",
+				  "cdc_spk_clip_int",
+				  "cdc_spk_ocp_int",
+				  "mbhc_ins_rem_det1",
+				  "mbhc_but_rel_det",
+				  "mbhc_but_press_det",
+				  "mbhc_ins_rem_det",
+				  "mbhc_switch_int",
+				  "cdc_ear_ocp_int",
+				  "cdc_hphr_ocp_int",
+				  "cdc_hphl_ocp_det",
+				  "cdc_ear_cnp_int",
+				  "cdc_hphr_cnp_int",
+				  "cdc_hphl_cnp_int";
+	               VDD-CDC-IO-supply = <&pm8916_l5>;
+	               VDD-CDC-TX-RX-CX-supply = <&pm8916_l5>;
+	               VDD-MICBIAS-supply = <&pm8916_l13>;
+	               #sound-dai-cells = <1>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/sound/qcom,msm8916-wcd-digital.txt b/Documentation/devicetree/bindings/sound/qcom,msm8916-wcd-digital.txt
new file mode 100644
index 000000000000..1c8e4cb25176
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/qcom,msm8916-wcd-digital.txt
@@ -0,0 +1,20 @@
+msm8916 digital audio CODEC
+
+## Bindings for codec core in lpass:
+
+Required properties
+ - compatible = "qcom,msm8916-wcd-digital-codec";
+ - reg: address space for lpass codec.
+ - clocks: Handle to mclk and ahbclk
+ - clock-names: should be "mclk", "ahbix-clk".
+
+Example:
+
+audio-codec@771c000{
+	compatible = "qcom,msm8916-wcd-digital-codec";
+	reg = <0x0771c000 0x400>;
+	clocks = <&gcc GCC_ULTAUDIO_AHBFABRIC_IXFABRIC_CLK>,
+		 <&gcc GCC_CODEC_DIGCODEC_CLK>;
+	clock-names = "ahbix-clk", "mclk";
+	#sound-dai-cells = <1>;
+};
diff --git a/Documentation/devicetree/bindings/sound/rt5514.txt b/Documentation/devicetree/bindings/sound/rt5514.txt
index 9cabfc18cb57..929ca6756b02 100644
--- a/Documentation/devicetree/bindings/sound/rt5514.txt
+++ b/Documentation/devicetree/bindings/sound/rt5514.txt
@@ -13,6 +13,9 @@ Optional properties:
 - clocks: The phandle of the master clock to the CODEC
 - clock-names: Should be "mclk"
 
+- realtek,dmic-init-delay-ms
+  Set the DMIC initial delay (ms) to wait it ready.
+
 Pins on the device (for linking into audio routes) for RT5514:
 
   * DMIC1L
diff --git a/Documentation/devicetree/bindings/sound/rt5663.txt b/Documentation/devicetree/bindings/sound/rt5663.txt
index 7d3c974c6e2e..70eaeaed2b18 100644
--- a/Documentation/devicetree/bindings/sound/rt5663.txt
+++ b/Documentation/devicetree/bindings/sound/rt5663.txt
@@ -1,10 +1,10 @@
-RT5663/RT5668 audio CODEC
+RT5663 audio CODEC
 
 This device supports I2C only.
 
 Required properties:
 
-- compatible : One of "realtek,rt5663" or "realtek,rt5668".
+- compatible : "realtek,rt5663".
 
 - reg : The I2C address of the device.
 
@@ -12,7 +12,7 @@ Required properties:
 
 Optional properties:
 
-Pins on the device (for linking into audio routes) for RT5663/RT5668:
+Pins on the device (for linking into audio routes) for RT5663:
 
   * IN1P
   * IN1N
diff --git a/Documentation/devicetree/bindings/sound/rt5665.txt b/Documentation/devicetree/bindings/sound/rt5665.txt
new file mode 100755
index 000000000000..419c89219681
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/rt5665.txt
@@ -0,0 +1,68 @@
+RT5665/RT5666/RT5668 audio CODEC
+
+This device supports I2C only.
+
+Required properties:
+
+- compatible : One of "realtek,rt5665", "realtek,rt5666" or "realtek,rt5668".
+
+- reg : The I2C address of the device.
+
+- interrupts : The CODEC's interrupt output.
+
+Optional properties:
+
+- realtek,in1-differential
+- realtek,in2-differential
+- realtek,in3-differential
+- realtek,in4-differential
+  Boolean. Indicate MIC1/2/3/4 input are differential, rather than single-ended.
+
+- realtek,dmic1-data-pin
+  0: dmic1 is not used
+  1: using GPIO4 pin as dmic1 data pin
+  2: using IN2N pin as dmic2 data pin
+
+- realtek,dmic2-data-pin
+  0: dmic2 is not used
+  1: using GPIO5 pin as dmic2 data pin
+  2: using IN2P pin as dmic2 data pin
+
+- realtek,jd-src
+  0: No JD is used
+  1: using JD1 as JD source
+
+- realtek,ldo1-en-gpios : The GPIO that controls the CODEC's LDO1_EN pin.
+
+Pins on the device (for linking into audio routes) for RT5659/RT5658:
+
+  * DMIC L1
+  * DMIC R1
+  * DMIC L2
+  * DMIC R2
+  * IN1P
+  * IN1N
+  * IN2P
+  * IN2N
+  * IN3P
+  * IN3N
+  * IN4P
+  * IN4N
+  * HPOL
+  * HPOR
+  * LOUTL
+  * LOUTR
+  * MONOOUT
+  * PDML
+  * PDMR
+
+Example:
+
+rt5659 {
+	compatible = "realtek,rt5665";
+	reg = <0x1b>;
+	interrupt-parent = <&gpio>;
+	interrupts = <TEGRA_GPIO(W, 3) GPIO_ACTIVE_HIGH>;
+	realtek,ldo1-en-gpios =
+		<&gpio TEGRA_GPIO(V, 3) GPIO_ACTIVE_HIGH>;
+};
diff --git a/Documentation/devicetree/bindings/sound/samsung,tm2-audio.txt b/Documentation/devicetree/bindings/sound/samsung,tm2-audio.txt
new file mode 100644
index 000000000000..94442e5673b3
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/samsung,tm2-audio.txt
@@ -0,0 +1,38 @@
+Samsung Exynos5433 TM2(E) audio complex with WM5110 codec
+
+Required properties:
+
+ - compatible		 : "samsung,tm2-audio"
+ - model		 : the user-visible name of this sound complex
+ - audio-codec		 : the phandle of the wm5110 audio codec node,
+			   as described in ../mfd/arizona.txt
+ - i2s-controller	 : the phandle of the I2S controller
+ - audio-amplifier	 : the phandle of the MAX98504 amplifier
+ - samsung,audio-routing : a list of the connections between audio components;
+			   each entry is a pair of strings, the first being the
+			   connection's sink, the second being the connection's
+			   source; valid names for sources and sinks are the
+			   WM5110's and MAX98504's pins and the jacks on the
+			   board: HP, SPK, Main Mic, Sub Mic, Third Mic,
+			   Headset Mic
+ - mic-bias-gpios	 : GPIO pin that enables the Main Mic bias regulator
+
+
+Example:
+
+sound {
+	compatible = "samsung,tm2-audio";
+	audio-codec = <&wm5110>;
+	i2s-controller = <&i2s0>;
+	audio-amplifier = <&max98504>;
+	mic-bias-gpios = <&gpr3 2 0>;
+	model = "wm5110";
+	samsung,audio-routing =
+		"HP", "HPOUT1L",
+		"HP", "HPOUT1R",
+		"SPK", "SPKOUT",
+		"SPKOUT", "HPOUT2L",
+		"SPKOUT", "HPOUT2R",
+		"Main Mic", "MICBIAS2",
+		"IN1R", "Main Mic";
+};
diff --git a/Documentation/devicetree/bindings/sound/sun4i-codec.txt b/Documentation/devicetree/bindings/sound/sun4i-codec.txt
index 0dce690f78f5..3033bd8aab0f 100644
--- a/Documentation/devicetree/bindings/sound/sun4i-codec.txt
+++ b/Documentation/devicetree/bindings/sound/sun4i-codec.txt
@@ -1,8 +1,12 @@
 * Allwinner A10 Codec
 
 Required properties:
-- compatible: must be either "allwinner,sun4i-a10-codec" or
-  "allwinner,sun7i-a20-codec"
+- compatible: must be one of the following compatibles:
+		- "allwinner,sun4i-a10-codec"
+		- "allwinner,sun6i-a31-codec"
+		- "allwinner,sun7i-a20-codec"
+		- "allwinner,sun8i-a23-codec"
+		- "allwinner,sun8i-h3-codec"
 - reg: must contain the registers location and length
 - interrupts: must contain the codec interrupt
 - dmas: DMA channels for tx and rx dma. See the DMA client binding,
@@ -17,6 +21,43 @@ Required properties:
 Optional properties:
 - allwinner,pa-gpios: gpio to enable external amplifier
 
+Required properties for the following compatibles:
+		- "allwinner,sun6i-a31-codec"
+		- "allwinner,sun8i-a23-codec"
+		- "allwinner,sun8i-h3-codec"
+- resets: phandle to the reset control for this device
+- allwinner,audio-routing: A list of the connections between audio components.
+			   Each entry is a pair of strings, the first being the
+			   connection's sink, the second being the connection's
+			   source. Valid names include:
+
+			   Audio pins on the SoC:
+			   "HP"
+			   "HPCOM"
+			   "LINEIN"
+			   "LINEOUT"	(not on sun8i-a23)
+			   "MIC1"
+			   "MIC2"
+			   "MIC3"	(sun6i-a31 only)
+
+			   Microphone biases from the SoC:
+			   "HBIAS"
+			   "MBIAS"
+
+			   Board connectors:
+			   "Headphone"
+			   "Headset Mic"
+			   "Line In"
+			   "Line Out"
+			   "Mic"
+			   "Speaker"
+
+Required properties for the following compatibles:
+		- "allwinner,sun8i-a23-codec"
+		- "allwinner,sun8i-h3-codec"
+- allwinner,codec-analog-controls: A phandle to the codec analog controls
+				   block in the PRCM.
+
 Example:
 codec: codec@01c22c00 {
 	#sound-dai-cells = <0>;
@@ -28,3 +69,23 @@ codec: codec@01c22c00 {
 	dmas = <&dma 0 19>, <&dma 0 19>;
 	dma-names = "rx", "tx";
 };
+
+codec: codec@01c22c00 {
+	#sound-dai-cells = <0>;
+	compatible = "allwinner,sun6i-a31-codec";
+	reg = <0x01c22c00 0x98>;
+	interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+	clocks = <&ccu CLK_APB1_CODEC>, <&ccu CLK_CODEC>;
+	clock-names = "apb", "codec";
+	resets = <&ccu RST_APB1_CODEC>;
+	dmas = <&dma 15>, <&dma 15>;
+	dma-names = "rx", "tx";
+	allwinner,audio-routing =
+		"Headphone", "HP",
+		"Speaker", "LINEOUT",
+		"LINEIN", "Line In",
+		"MIC1",	"MBIAS",
+		"MIC1", "Mic",
+		"MIC2", "HBIAS",
+		"MIC2", "Headset Mic";
+};
diff --git a/Documentation/devicetree/bindings/sound/sun8i-codec-analog.txt b/Documentation/devicetree/bindings/sound/sun8i-codec-analog.txt
new file mode 100644
index 000000000000..779b735781ba
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/sun8i-codec-analog.txt
@@ -0,0 +1,16 @@
+* Allwinner Codec Analog Controls
+
+Required properties:
+- compatible: must be one of the following compatibles:
+		- "allwinner,sun8i-a23-codec-analog"
+		- "allwinner,sun8i-h3-codec-analog"
+
+Required properties if not a sub-node of the PRCM node:
+- reg: must contain the registers location and length
+
+Example:
+prcm: prcm@01f01400 {
+	codec_analog: codec-analog {
+		compatible = "allwinner,sun8i-a23-codec-analog";
+	};
+};
diff --git a/Documentation/devicetree/bindings/sound/tlv320aic31xx.txt b/Documentation/devicetree/bindings/sound/tlv320aic31xx.txt
index 9340d2ddcc54..6fbba562eaa7 100644
--- a/Documentation/devicetree/bindings/sound/tlv320aic31xx.txt
+++ b/Documentation/devicetree/bindings/sound/tlv320aic31xx.txt
@@ -12,6 +12,7 @@ Required properties:
     "ti,tlv320aic3120" - TLV320AIC3120 (mono speaker amp, MiniDSP)
     "ti,tlv320aic3111" - TLV320AIC3111 (stereo speaker amp, MiniDSP)
     "ti,tlv320dac3100" - TLV320DAC3100 (no ADC, mono speaker amp, no MiniDSP)
+    "ti,tlv320dac3101" - TLV320DAC3101 (no ADC, stereo speaker amp, no MiniDSP)
 
 - reg - <int> -  I2C slave address
 - HPVDD-supply, SPRVDD-supply, SPLVDD-supply, AVDD-supply, IOVDD-supply,
diff --git a/Documentation/devicetree/bindings/sound/wm8580.txt b/Documentation/devicetree/bindings/sound/wm8580.txt
index 7d9821f348da..78fce9b14954 100644
--- a/Documentation/devicetree/bindings/sound/wm8580.txt
+++ b/Documentation/devicetree/bindings/sound/wm8580.txt
@@ -1,10 +1,10 @@
-WM8580 audio CODEC
+WM8580 and WM8581 audio CODEC
 
 This device supports I2C only.
 
 Required properties:
 
-  - compatible : "wlf,wm8580"
+  - compatible : "wlf,wm8580", "wlf,wm8581"
 
   - reg : the I2C address of the device.
 
diff --git a/Documentation/devicetree/bindings/spi/sh-msiof.txt b/Documentation/devicetree/bindings/spi/sh-msiof.txt
index aa005c1d10d9..da6614c63796 100644
--- a/Documentation/devicetree/bindings/spi/sh-msiof.txt
+++ b/Documentation/devicetree/bindings/spi/sh-msiof.txt
@@ -10,6 +10,7 @@ Required properties:
 			 "renesas,msiof-r8a7792" (R-Car V2H)
 			 "renesas,msiof-r8a7793" (R-Car M2-N)
 			 "renesas,msiof-r8a7794" (R-Car E2)
+			 "renesas,msiof-r8a7796" (R-Car M3-W)
 			 "renesas,msiof-sh73a0" (SH-Mobile AG5)
 - reg                  : A list of offsets and lengths of the register sets for
 			 the device.
diff --git a/Documentation/devicetree/bindings/spi/spi-armada-3700.txt b/Documentation/devicetree/bindings/spi/spi-armada-3700.txt
new file mode 100644
index 000000000000..1564aa8c02cd
--- /dev/null
+++ b/Documentation/devicetree/bindings/spi/spi-armada-3700.txt
@@ -0,0 +1,25 @@
+* Marvell Armada 3700 SPI Controller
+
+Required Properties:
+
+- compatible: should be "marvell,armada-3700-spi"
+- reg: physical base address of the controller and length of memory mapped
+       region.
+- interrupts: The interrupt number. The interrupt specifier format depends on
+	      the interrupt controller and of its driver.
+- clocks: Must contain the clock source, usually from the North Bridge clocks.
+- num-cs: The number of chip selects that is supported by this SPI Controller
+- #address-cells: should be 1.
+- #size-cells: should be 0.
+
+Example:
+
+	spi0: spi@10600 {
+		compatible = "marvell,armada-3700-spi";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x10600 0x5d>;
+		clocks = <&nb_perih_clk 7>;
+		interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>;
+		num-cs = <4>;
+	};
diff --git a/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.txt b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.txt
new file mode 100644
index 000000000000..225ace1d0c65
--- /dev/null
+++ b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.txt
@@ -0,0 +1,19 @@
+* Freescale Low Power SPI (LPSPI) for i.MX
+
+Required properties:
+- compatible :
+  - "fsl,imx7ulp-spi" for LPSPI compatible with the one integrated on i.MX7ULP soc
+- reg : address and length of the lpspi master registers
+- interrupt-parent : core interrupt controller
+- interrupts : lpspi interrupt
+- clocks : lpspi clock specifier
+
+Examples:
+
+lpspi2: lpspi@40290000 {
+	compatible = "fsl,imx7ulp-spi";
+	reg = <0x40290000 0x10000>;
+	interrupt-parent = <&intc>;
+	interrupts = <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>;
+	clocks = <&clks IMX7ULP_CLK_LPSPI2>;
+};
diff --git a/Documentation/devicetree/bindings/spi/spi-sun6i.txt b/Documentation/devicetree/bindings/spi/spi-sun6i.txt
index 21de73db6a05..2ec99b86b622 100644
--- a/Documentation/devicetree/bindings/spi/spi-sun6i.txt
+++ b/Documentation/devicetree/bindings/spi/spi-sun6i.txt
@@ -1,7 +1,7 @@
-Allwinner A31 SPI controller
+Allwinner A31/H3 SPI controller
 
 Required properties:
-- compatible: Should be "allwinner,sun6i-a31-spi".
+- compatible: Should be "allwinner,sun6i-a31-spi" or "allwinner,sun8i-h3-spi".
 - reg: Should contain register location and length.
 - interrupts: Should contain interrupt.
 - clocks: phandle to the clocks feeding the SPI controller. Two are
@@ -12,6 +12,11 @@ Required properties:
 - resets: phandle to the reset controller asserting this device in
           reset
 
+Optional properties:
+- dmas: DMA specifiers for rx and tx dma. See the DMA client binding,
+	Documentation/devicetree/bindings/dma/dma.txt
+- dma-names: DMA request names should include "rx" and "tx" if present.
+
 Example:
 
 spi1: spi@01c69000 {
@@ -22,3 +27,19 @@ spi1: spi@01c69000 {
 	clock-names = "ahb", "mod";
 	resets = <&ahb1_rst 21>;
 };
+
+spi0: spi@01c68000 {
+	compatible = "allwinner,sun8i-h3-spi";
+	reg = <0x01c68000 0x1000>;
+	interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
+	clocks = <&ccu CLK_BUS_SPI0>, <&ccu CLK_SPI0>;
+	clock-names = "ahb", "mod";
+	dmas = <&dma 23>, <&dma 23>;
+	dma-names = "rx", "tx";
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi0_pins>;
+	resets = <&ccu RST_BUS_SPI0>;
+	status = "disabled";
+	#address-cells = <1>;
+	#size-cells = <0>;
+};
diff --git a/Documentation/devicetree/bindings/sram/sram.txt b/Documentation/devicetree/bindings/sram/sram.txt
index add48f09015e..068c2c03c38f 100644
--- a/Documentation/devicetree/bindings/sram/sram.txt
+++ b/Documentation/devicetree/bindings/sram/sram.txt
@@ -4,7 +4,7 @@ Simple IO memory regions to be managed by the genalloc API.
 
 Required properties:
 
-- compatible : mmio-sram
+- compatible : mmio-sram or atmel,sama5d2-securam
 
 - reg : SRAM iomem address range
 
diff --git a/Documentation/devicetree/bindings/thermal/brcm,bcm2835-thermal.txt b/Documentation/devicetree/bindings/thermal/brcm,bcm2835-thermal.txt
new file mode 100644
index 000000000000..474531d2b2c5
--- /dev/null
+++ b/Documentation/devicetree/bindings/thermal/brcm,bcm2835-thermal.txt
@@ -0,0 +1,17 @@
+Binding for Thermal Sensor driver for BCM2835 SoCs.
+
+Required parameters:
+-------------------
+
+compatible: 	should be one of: "brcm,bcm2835-thermal",
+		"brcm,bcm2836-thermal" or "brcm,bcm2837-thermal"
+reg:		Address range of the thermal registers.
+clocks: 	Phandle of the clock used by the thermal sensor.
+
+Example:
+
+thermal: thermal@7e212000 {
+	compatible = "brcm,bcm2835-thermal";
+	reg = <0x7e212000 0x8>;
+	clocks = <&clocks BCM2835_CLOCK_TSENS>;
+};
diff --git a/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt b/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt
index 08efe6bc2193..43003aec94bd 100644
--- a/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/rockchip-thermal.txt
@@ -22,10 +22,13 @@ Required properties:
 	      TSADC controller.
 - pinctrl-2 : The "sleep" pinctrl state, it will be in for suspend.
 - #thermal-sensor-cells : Should be 1. See ./thermal.txt for a description.
+
+Optional properties:
 - rockchip,hw-tshut-temp : The hardware-controlled shutdown temperature value.
 - rockchip,hw-tshut-mode : The hardware-controlled shutdown mode 0:CRU 1:GPIO.
 - rockchip,hw-tshut-polarity : The hardware-controlled active polarity 0:LOW
 			       1:HIGH.
+- rockchip,grf : The phandle of the syscon node for the general register file.
 
 Exiample:
 tsadc: tsadc@ff280000 {
diff --git a/Documentation/devicetree/bindings/thermal/st-thermal.txt b/Documentation/devicetree/bindings/thermal/st-thermal.txt
index 3b9251b4a145..a2f939137e35 100644
--- a/Documentation/devicetree/bindings/thermal/st-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/st-thermal.txt
@@ -3,17 +3,8 @@ Binding for Thermal Sensor driver for STMicroelectronics STi series of SoCs.
 Required parameters:
 -------------------
 
-compatible : 	st,<SoC>-<module>-thermal; should be one of:
-		  "st,stih415-sas-thermal",
-		  "st,stih415-mpe-thermal",
-		  "st,stih416-sas-thermal"
-		  "st,stih416-mpe-thermal"
-		  "st,stid127-thermal" or
-		  "st,stih407-thermal"
-		according to the SoC type (stih415, stih416, stid127, stih407)
-		and module type (sas or mpe). On stid127 & stih407 there is only
-		one die/module, so there is no module type in the compatible
-		string.
+compatible : 	Should be "st,stih407-thermal"
+
 clock-names : 	Should be "thermal".
 		  See: Documentation/devicetree/bindings/resource-names.txt
 clocks : 	Phandle of the clock used by the thermal sensor.
@@ -25,18 +16,17 @@ Optional parameters:
 reg : 		For non-sysconf based sensors, this should be the physical base
 		address and length of the sensor's registers.
 interrupts :	Standard way to define interrupt number.
-		Interrupt is mandatory to be defined when compatible is
-		"stih416-mpe-thermal".
 		  NB: For thermal sensor's for which no interrupt has been
 		  defined, a polling delay of 1000ms will be used to read the
 		  temperature from device.
 
 Example:
 
-	temp1@fdfe8000 {
-		compatible	= "st,stih416-mpe-thermal";
-		reg		= <0xfdfe8000 0x10>;
-		clock-names	= "thermal";
-		clocks		= <&clk_m_mpethsens>;
-		interrupts	= <GIC_SPI 23 IRQ_TYPE_NONE>;
+	temp0@91a0000 {
+		compatible = "st,stih407-thermal";
+		reg = <0x91a0000 0x28>;
+		clock-names = "thermal";
+		clocks = <&CLK_SYSIN>;
+		interrupts = <GIC_SPI 205 IRQ_TYPE_EDGE_RISING>;
+		st,passive_cooling_temp = <110>;
 	};
diff --git a/Documentation/devicetree/bindings/timer/ezchip,nps400-timer0.txt b/Documentation/devicetree/bindings/timer/ezchip,nps400-timer0.txt
new file mode 100644
index 000000000000..e3cfce8fecc5
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/ezchip,nps400-timer0.txt
@@ -0,0 +1,17 @@
+NPS Network Processor
+
+Required properties:
+
+- compatible :	should be "ezchip,nps400-timer0"
+
+Clocks required for compatible = "ezchip,nps400-timer0":
+- interrupts : The interrupt of the first timer
+- clocks : Must contain a single entry describing the clock input
+
+Example:
+
+timer {
+	compatible = "ezchip,nps400-timer0";
+	interrupts = <3>;
+	clocks = <&sysclk>;
+};
diff --git a/Documentation/devicetree/bindings/timer/ezchip,nps400-timer.txt b/Documentation/devicetree/bindings/timer/ezchip,nps400-timer1.txt
index c8c03d700382..c0ab4190b8fb 100644
--- a/Documentation/devicetree/bindings/timer/ezchip,nps400-timer.txt
+++ b/Documentation/devicetree/bindings/timer/ezchip,nps400-timer1.txt
@@ -2,14 +2,14 @@ NPS Network Processor
 
 Required properties:
 
-- compatible :	should be "ezchip,nps400-timer"
+- compatible :	should be "ezchip,nps400-timer1"
 
-Clocks required for compatible = "ezchip,nps400-timer":
+Clocks required for compatible = "ezchip,nps400-timer1":
 - clocks : Must contain a single entry describing the clock input
 
 Example:
 
 timer {
-	compatible = "ezchip,nps400-timer";
+	compatible = "ezchip,nps400-timer1";
 	clocks = <&sysclk>;
 };
diff --git a/Documentation/devicetree/bindings/ufs/ufs-qcom.txt b/Documentation/devicetree/bindings/ufs/ufs-qcom.txt
index 070baf4d7d97..b6b5130e5f65 100644
--- a/Documentation/devicetree/bindings/ufs/ufs-qcom.txt
+++ b/Documentation/devicetree/bindings/ufs/ufs-qcom.txt
@@ -7,8 +7,11 @@ To bind UFS PHY with UFS host controller, the controller node should
 contain a phandle reference to UFS PHY node.
 
 Required properties:
-- compatible        : compatible list, contains "qcom,ufs-phy-qmp-20nm"
-		      or "qcom,ufs-phy-qmp-14nm" according to the relevant phy in use.
+- compatible        : compatible list, contains one of the following -
+			"qcom,ufs-phy-qmp-20nm" for 20nm ufs phy,
+			"qcom,ufs-phy-qmp-14nm" for legacy 14nm ufs phy,
+			"qcom,msm8996-ufs-phy-qmp-14nm" for 14nm ufs phy
+			 present on MSM8996 chipset.
 - reg               : should contain PHY register address space (mandatory),
 - reg-names         : indicates various resources passed to driver (via reg proptery) by name.
                       Required "reg-names" is "phy_mem".
diff --git a/Documentation/devicetree/bindings/usb/atmel-usb.txt b/Documentation/devicetree/bindings/usb/atmel-usb.txt
index f4262ed60582..ad8ea56a9ed3 100644
--- a/Documentation/devicetree/bindings/usb/atmel-usb.txt
+++ b/Documentation/devicetree/bindings/usb/atmel-usb.txt
@@ -6,9 +6,9 @@ Required properties:
  - compatible: Should be "atmel,at91rm9200-ohci" for USB controllers
    used in host mode.
  - reg: Address and length of the register set for the device
- - interrupts: Should contain ehci interrupt
+ - interrupts: Should contain ohci interrupt
  - clocks: Should reference the peripheral, host and system clocks
- - clock-names: Should contains two strings
+ - clock-names: Should contain three strings
 		"ohci_clk" for the peripheral clock
 		"hclk" for the host clock
 		"uhpck" for the system clock
@@ -35,7 +35,7 @@ Required properties:
  - reg: Address and length of the register set for the device
  - interrupts: Should contain ehci interrupt
  - clocks: Should reference the peripheral and the UTMI clocks
- - clock-names: Should contains two strings
+ - clock-names: Should contain two strings
 		"ehci_clk" for the peripheral clock
 		"usb_clk" for the UTMI clock
 
@@ -58,7 +58,7 @@ Required properties:
  - reg: Address and length of the register set for the device
  - interrupts: Should contain macb interrupt
  - clocks: Should reference the peripheral and the AHB clocks
- - clock-names: Should contains two strings
+ - clock-names: Should contain two strings
 		"pclk" for the peripheral clock
 		"hclk" for the AHB clock
 
@@ -85,7 +85,7 @@ Required properties:
  - reg: Address and length of the register set for the device
  - interrupts: Should contain usba interrupt
  - clocks: Should reference the peripheral and host clocks
- - clock-names: Should contains two strings
+ - clock-names: Should contain two strings
 		"pclk" for the peripheral clock
 		"hclk" for the host clock
  - ep childnode: To specify the number of endpoints and their properties.
diff --git a/Documentation/devicetree/bindings/usb/da8xx-usb.txt b/Documentation/devicetree/bindings/usb/da8xx-usb.txt
new file mode 100644
index 000000000000..ccb844aba7d4
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/da8xx-usb.txt
@@ -0,0 +1,43 @@
+TI DA8xx MUSB
+~~~~~~~~~~~~~
+For DA8xx/OMAP-L1x/AM17xx/AM18xx platforms.
+
+Required properties:
+~~~~~~~~~~~~~~~~~~~~
+ - compatible : Should be set to "ti,da830-musb".
+
+ - reg: Offset and length of the USB controller register set.
+
+ - interrupts: The USB interrupt number.
+
+ - interrupt-names: Should be set to "mc".
+
+ - dr_mode: The USB operation mode. Should be one of "host", "peripheral" or "otg".
+
+ - phys: Phandle for the PHY device
+
+ - phy-names: Should be "usb-phy"
+
+Optional properties:
+~~~~~~~~~~~~~~~~~~~~
+ - vbus-supply: Phandle to a regulator providing the USB bus power.
+
+Example:
+	usb_phy: usb-phy {
+		compatible = "ti,da830-usb-phy";
+		#phy-cells = <0>;
+		status = "okay";
+	};
+	usb0: usb@200000 {
+		compatible = "ti,da830-musb";
+		reg =   <0x00200000 0x10000>;
+		interrupts = <58>;
+		interrupt-names = "mc";
+
+		dr_mode = "host";
+		vbus-supply = <&usb_vbus>;
+		phys = <&usb_phy 0>;
+		phy-names = "usb-phy";
+
+		status = "okay";
+	};
diff --git a/Documentation/devicetree/bindings/usb/dwc2.txt b/Documentation/devicetree/bindings/usb/dwc2.txt
index 2c30a5479069..6c7c2bce6d0c 100644
--- a/Documentation/devicetree/bindings/usb/dwc2.txt
+++ b/Documentation/devicetree/bindings/usb/dwc2.txt
@@ -12,6 +12,7 @@ Required properties:
   - "lantiq,xrx200-usb": The DWC2 USB controller instance in Lantiq XRX SoCs;
   - "amlogic,meson8b-usb": The DWC2 USB controller instance in Amlogic Meson8b SoCs;
   - "amlogic,meson-gxbb-usb": The DWC2 USB controller instance in Amlogic S905 SoCs;
+  - "amcc,dwc-otg": The DWC2 USB controller instance in AMCC Canyonlands 460EX SoCs;
   - snps,dwc2: A generic DWC2 USB controller with default parameters.
 - reg : Should contain 1 register range (address and length)
 - interrupts : Should contain 1 interrupt
@@ -25,11 +26,13 @@ Optional properties:
 Refer to phy/phy-bindings.txt for generic phy consumer properties
 - dr_mode: shall be one of "host", "peripheral" and "otg"
   Refer to usb/generic.txt
-- g-use-dma: enable dma usage in gadget driver.
 - g-rx-fifo-size: size of rx fifo size in gadget mode.
 - g-np-tx-fifo-size: size of non-periodic tx fifo size in gadget mode.
 - g-tx-fifo-size: size of periodic tx fifo per endpoint (except ep0) in gadget mode.
 
+Deprecated properties:
+- g-use-dma: gadget DMA mode is automatically detected
+
 Example:
 
         usb@101c0000 {
diff --git a/Documentation/devicetree/bindings/usb/mt8173-mtu3.txt b/Documentation/devicetree/bindings/usb/mt8173-mtu3.txt
new file mode 100644
index 000000000000..e049d199bf0d
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/mt8173-mtu3.txt
@@ -0,0 +1,87 @@
+The device node for Mediatek USB3.0 DRD controller
+
+Required properties:
+ - compatible : should be "mediatek,mt8173-mtu3"
+ - reg : specifies physical base address and size of the registers
+ - reg-names: should be "mac" for device IP and "ippc" for IP port control
+ - interrupts : interrupt used by the device IP
+ - power-domains : a phandle to USB power domain node to control USB's
+	mtcmos
+ - vusb33-supply : regulator of USB avdd3.3v
+ - clocks : a list of phandle + clock-specifier pairs, one for each
+	entry in clock-names
+ - clock-names : must contain "sys_ck" for clock of controller;
+	"wakeup_deb_p0" and "wakeup_deb_p1" are optional, they are
+	depends on "mediatek,enable-wakeup"
+ - phys : a list of phandle + phy specifier pairs
+ - dr_mode : should be one of "host", "peripheral" or "otg",
+	refer to usb/generic.txt
+
+Optional properties:
+ - #address-cells, #size-cells : should be '2' if the device has sub-nodes
+	with 'reg' property
+ - ranges : allows valid 1:1 translation between child's address space and
+	parent's address space
+ - extcon : external connector for vbus and idpin changes detection, needed
+	when supports dual-role mode.
+ - vbus-supply : reference to the VBUS regulator, needed when supports
+	dual-role mode.
+ - pinctl-names : a pinctrl state named "default" must be defined,
+	"id_float" and "id_ground" are optinal which depends on
+	"mediatek,enable-manual-drd"
+ - pinctrl-0 : pin control group
+	See: Documentation/devicetree/bindings/pinctrl/pinctrl-binding.txt
+
+ - maximum-speed : valid arguments are "super-speed", "high-speed" and
+	"full-speed"; refer to usb/generic.txt
+ - enable-manual-drd : supports manual dual-role switch via debugfs; usually
+	used when receptacle is TYPE-A and also wants to support dual-role
+	mode.
+ - mediatek,enable-wakeup : supports ip sleep wakeup used by host mode
+ - mediatek,syscon-wakeup : phandle to syscon used to access USB wakeup
+	control register, it depends on "mediatek,enable-wakeup".
+
+Sub-nodes:
+The xhci should be added as subnode to mtu3 as shown in the following example
+if host mode is enabled. The DT binding details of xhci can be found in:
+Documentation/devicetree/bindings/usb/mt8173-xhci.txt
+
+Example:
+ssusb: usb@11271000 {
+	compatible = "mediatek,mt8173-mtu3";
+	reg = <0 0x11271000 0 0x3000>,
+	      <0 0x11280700 0 0x0100>;
+	reg-names = "mac", "ippc";
+	interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_LOW>;
+	phys = <&phy_port0 PHY_TYPE_USB3>,
+	       <&phy_port1 PHY_TYPE_USB2>;
+	power-domains = <&scpsys MT8173_POWER_DOMAIN_USB>;
+	clocks = <&topckgen CLK_TOP_USB30_SEL>,
+		 <&pericfg CLK_PERI_USB0>,
+		 <&pericfg CLK_PERI_USB1>;
+	clock-names = "sys_ck",
+		      "wakeup_deb_p0",
+		      "wakeup_deb_p1";
+	vusb33-supply = <&mt6397_vusb_reg>;
+	vbus-supply = <&usb_p0_vbus>;
+	extcon = <&extcon_usb>;
+	dr_mode = "otg";
+	mediatek,enable-wakeup;
+	mediatek,syscon-wakeup = <&pericfg>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+	ranges;
+	status = "disabled";
+
+	usb_host: xhci@11270000 {
+		compatible = "mediatek,mt8173-xhci";
+		reg = <0 0x11270000 0 0x1000>;
+		reg-names = "mac";
+		interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_LOW>;
+		power-domains = <&scpsys MT8173_POWER_DOMAIN_USB>;
+		clocks = <&topckgen CLK_TOP_USB30_SEL>;
+		clock-names = "sys_ck";
+		vusb33-supply = <&mt6397_vusb_reg>;
+		status = "disabled";
+	};
+};
diff --git a/Documentation/devicetree/bindings/usb/mt8173-xhci.txt b/Documentation/devicetree/bindings/usb/mt8173-xhci.txt
index b3a7ffa48852..2a930bd52b94 100644
--- a/Documentation/devicetree/bindings/usb/mt8173-xhci.txt
+++ b/Documentation/devicetree/bindings/usb/mt8173-xhci.txt
@@ -2,10 +2,18 @@ MT8173 xHCI
 
 The device node for Mediatek SOC USB3.0 host controller
 
+There are two scenarios: the first one only supports xHCI driver;
+the second one supports dual-role mode, and the host is based on xHCI
+driver. Take account of backward compatibility, we divide bindings
+into two parts.
+
+1st: only supports xHCI driver
+------------------------------------------------------------------------
+
 Required properties:
  - compatible : should contain "mediatek,mt8173-xhci"
- - reg : specifies physical base address and size of the registers,
-	the first one for MAC, the second for IPPC
+ - reg : specifies physical base address and size of the registers
+ - reg-names: should be "mac" for xHCI MAC and "ippc" for IP port control
  - interrupts : interrupt used by the controller
  - power-domains : a phandle to USB power domain node to control USB's
 	mtcmos
@@ -27,12 +35,16 @@ Optional properties:
 	control register, it depends on "mediatek,wakeup-src".
  - vbus-supply : reference to the VBUS regulator;
  - usb3-lpm-capable : supports USB3.0 LPM
+ - pinctrl-names : a pinctrl state named "default" must be defined
+ - pinctrl-0 : pin control group
+	See: Documentation/devicetree/bindings/pinctrl/pinctrl-binding.txt
 
 Example:
 usb30: usb@11270000 {
 	compatible = "mediatek,mt8173-xhci";
 	reg = <0 0x11270000 0 0x1000>,
 	      <0 0x11280700 0 0x0100>;
+	reg-names = "mac", "ippc";
 	interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_LOW>;
 	power-domains = <&scpsys MT8173_POWER_DOMAIN_USB>;
 	clocks = <&topckgen CLK_TOP_USB30_SEL>,
@@ -49,3 +61,41 @@ usb30: usb@11270000 {
 	mediatek,syscon-wakeup = <&pericfg>;
 	mediatek,wakeup-src = <1>;
 };
+
+2nd: dual-role mode with xHCI driver
+------------------------------------------------------------------------
+
+In the case, xhci is added as subnode to mtu3. An example and the DT binding
+details of mtu3 can be found in:
+Documentation/devicetree/bindings/usb/mtu3.txt
+
+Required properties:
+ - compatible : should contain "mediatek,mt8173-xhci"
+ - reg : specifies physical base address and size of the registers
+ - reg-names: should be "mac" for xHCI MAC
+ - interrupts : interrupt used by the host controller
+ - power-domains : a phandle to USB power domain node to control USB's
+	mtcmos
+ - vusb33-supply : regulator of USB avdd3.3v
+
+ - clocks : a list of phandle + clock-specifier pairs, one for each
+	entry in clock-names
+ - clock-names : must be
+	"sys_ck": for clock of xHCI MAC
+
+Optional properties:
+ - vbus-supply : reference to the VBUS regulator;
+ - usb3-lpm-capable : supports USB3.0 LPM
+
+Example:
+usb30: usb@11270000 {
+	compatible = "mediatek,mt8173-xhci";
+	reg = <0 0x11270000 0 0x1000>;
+	reg-names = "mac";
+	interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_LOW>;
+	power-domains = <&scpsys MT8173_POWER_DOMAIN_USB>;
+	clocks = <&topckgen CLK_TOP_USB30_SEL>;
+	clock-names = "sys_ck";
+	vusb33-supply = <&mt6397_vusb_reg>;
+	usb3-lpm-capable;
+};
diff --git a/Documentation/devicetree/bindings/usb/ohci-da8xx.txt b/Documentation/devicetree/bindings/usb/ohci-da8xx.txt
new file mode 100644
index 000000000000..2dc8f67eda39
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/ohci-da8xx.txt
@@ -0,0 +1,23 @@
+DA8XX USB OHCI controller
+
+Required properties:
+
+ - compatible: Should be "ti,da830-ohci"
+ - reg:        Should contain one register range i.e. start and length
+ - interrupts: Description of the interrupt line
+ - phys:       Phandle for the PHY device
+ - phy-names:  Should be "usb-phy"
+
+Optional properties:
+ - vbus-supply: phandle of regulator that controls vbus power / over-current
+
+Example:
+
+ohci: usb@0225000 {
+        compatible = "ti,da830-ohci";
+        reg = <0x225000 0x1000>;
+        interrupts = <59>;
+        phys = <&usb_phy 1>;
+        phy-names = "usb-phy";
+        vbus-supply = <&reg_usb_ohci>;
+};
diff --git a/Documentation/devicetree/bindings/usb/s3c2410-usb.txt b/Documentation/devicetree/bindings/usb/s3c2410-usb.txt
new file mode 100644
index 000000000000..e45b38ce2986
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/s3c2410-usb.txt
@@ -0,0 +1,22 @@
+Samsung S3C2410 and compatible SoC USB controller
+
+OHCI
+
+Required properties:
+ - compatible: should be "samsung,s3c2410-ohci" for USB host controller
+ - reg: address and lenght of the controller memory mapped region
+ - interrupts: interrupt number for the USB OHCI controller
+ - clocks: Should reference the bus and host clocks
+ - clock-names: Should contain two strings
+		"usb-bus-host" for the USB bus clock
+		"usb-host" for the USB host clock
+
+Example:
+
+usb0: ohci@49000000 {
+	compatible = "samsung,s3c2410-ohci";
+	reg = <0x49000000 0x100>;
+	interrupts = <0 0 26 3>;
+	clocks = <&clocks UCLK>, <&clocks HCLK_USBH>;
+	clock-names = "usb-bus-host", "usb-host";
+};
diff --git a/Documentation/devicetree/bindings/usb/usb-xhci.txt b/Documentation/devicetree/bindings/usb/usb-xhci.txt
index 966885c636d0..0b7d8576001c 100644
--- a/Documentation/devicetree/bindings/usb/usb-xhci.txt
+++ b/Documentation/devicetree/bindings/usb/usb-xhci.txt
@@ -11,6 +11,7 @@ Required properties:
     - "renesas,xhci-r8a7791" for r8a7791 SoC
     - "renesas,xhci-r8a7793" for r8a7793 SoC
     - "renesas,xhci-r8a7795" for r8a7795 SoC
+    - "renesas,xhci-r8a7796" for r8a7796 SoC
     - "renesas,rcar-gen2-xhci" for a generic R-Car Gen2 compatible device
     - "renesas,rcar-gen3-xhci" for a generic R-Car Gen3 compatible device
     - "xhci-platform" (deprecated)
diff --git a/Documentation/devicetree/bindings/vendor-prefixes.txt b/Documentation/devicetree/bindings/vendor-prefixes.txt
index f0a48ea78659..16d3b5e7f5d1 100644
--- a/Documentation/devicetree/bindings/vendor-prefixes.txt
+++ b/Documentation/devicetree/bindings/vendor-prefixes.txt
@@ -24,9 +24,11 @@ ampire	Ampire Co., Ltd.
 ams	AMS AG
 amstaos	AMS-Taos Inc.
 analogix	Analogix Semiconductor, Inc.
+andestech	Andes Technology Corporation
 apm	Applied Micro Circuits Corporation (APM)
 aptina	Aptina Imaging
 arasan	Arasan Chip Systems
+aries	Aries Embedded GmbH
 arm	ARM Ltd.
 armadeus	ARMadeus Systems SARL
 arrow	Arrow Electronics
@@ -39,6 +41,7 @@ auo	AU Optronics Corporation
 auvidea Auvidea GmbH
 avago	Avago Technologies
 avic	Shanghai AVIC Optoelectronics Co., Ltd.
+axentia	Axentia Technologies AB
 axis	Axis Communications AB
 boe	BOE Technology Group Co., Ltd.
 bosch	Bosch Sensortec GmbH
@@ -67,6 +70,7 @@ creative	Creative Technology Ltd
 crystalfontz	Crystalfontz America, Inc.
 cubietech	Cubietech, Ltd.
 cypress	Cypress Semiconductor Corporation
+cznic	CZ.NIC, z.s.p.o.
 dallas	Maxim Integrated Products (formerly Dallas Semiconductor)
 davicom	DAVICOM Semiconductor, Inc.
 delta	Delta Electronics, Inc.
@@ -126,6 +130,7 @@ hitex	Hitex Development Tools
 holt	Holt Integrated Circuits, Inc.
 honeywell	Honeywell
 hp	Hewlett Packard
+holtek	Holtek Semiconductor, Inc.
 i2se	I2SE GmbH
 ibm	International Business Machines (IBM)
 idt	Integrated Device Technologies, Inc.
@@ -135,6 +140,7 @@ infineon Infineon Technologies
 inforce	Inforce Computing
 ingenic	Ingenic Semiconductor
 innolux	Innolux Corporation
+inside-secure	INSIDE Secure
 intel	Intel Corporation
 intercontrol	Inter Control Group
 invensense	InvenSense Inc.
@@ -158,18 +164,22 @@ lg	LG Corporation
 linux	Linux-specific binding
 lltc	Linear Technology Corporation
 lsi	LSI Corp. (LSI Logic)
+macnica	Macnica Americas
 marvell	Marvell Technology Group Ltd.
 maxim	Maxim Integrated Products
+mcube	mCube
 meas	Measurement Specialties
 mediatek	MediaTek Inc.
 melexis	Melexis N.V.
 melfas	MELFAS Inc.
+memsic	MEMSIC Inc.
 merrii	Merrii Technology Co., Ltd.
 micrel	Micrel Inc.
 microchip	Microchip Technology Inc.
 microcrystal	Micro Crystal AG
 micron	Micron Technology Inc.
 minix	MINIX Technology Ltd.
+miramems	MiraMEMS Sensing Technology Co., Ltd.
 mitsubishi	Mitsubishi Electric Corporation
 mosaixtech	Mosaix Technologies, Inc.
 moxa	Moxa
@@ -180,19 +190,24 @@ mti	Imagination Technologies Ltd. (formerly MIPS Technologies Inc.)
 mundoreader	Mundo Reader S.L.
 murata	Murata Manufacturing Co., Ltd.
 mxicy	Macronix International Co., Ltd.
+myir	MYIR Tech Limited
 national	National Semiconductor
 nec	NEC LCD Technologies, Ltd.
 neonode		Neonode Inc.
 netgear	NETGEAR
 netlogic	Broadcom Corporation (formerly NetLogic Microsystems)
 netxeon		Shenzhen Netxeon Technology CO., LTD
+nexbox	Nexbox
 newhaven	Newhaven Display International
+ni	National Instruments
 nintendo	Nintendo
 nokia	Nokia
 nuvoton	Nuvoton Technology Corporation
+nvd	New Vision Display
 nvidia	NVIDIA
 nxp	NXP Semiconductors
 okaya	Okaya Electric America, Inc.
+oki	Oki Electric Industry Co., Ltd.
 olimex	OLIMEX Ltd.
 onion	Onion Corporation
 onnn	ON Semiconductor Corp.
@@ -208,6 +223,7 @@ parade	Parade Technologies Inc.
 pericom	Pericom Technology Inc.
 phytec	PHYTEC Messtechnik GmbH
 picochip	Picochip Ltd
+pine64	Pine64
 pixcir  PIXCIR MICROELECTRONICS Co., Ltd
 plathome	Plat'Home Co., Ltd.
 plda	PLDA
@@ -229,8 +245,10 @@ realtek Realtek Semiconductor Corp.
 renesas	Renesas Electronics Corporation
 richtek	Richtek Technology Corporation
 ricoh	Ricoh Co. Ltd.
+rikomagic	Rikomagic Tech Corp. Ltd
 rockchip	Fuzhou Rockchip Electronics Co., Ltd
 samsung	Samsung Semiconductor
+samtec	Samtec/Softing company
 sandisk	Sandisk Corporation
 sbs	Smart Battery System
 schindler	Schindler
@@ -275,6 +293,7 @@ tcg	Trusted Computing Group
 tcl	Toby Churchill Ltd.
 technexion	TechNexion
 technologic	Technologic Systems
+terasic	Terasic Inc.
 thine	THine Electronics, Inc.
 ti	Texas Instruments
 tlm	Trusted Logic Mobility
@@ -289,6 +308,7 @@ tronfy	Tronfy
 tronsmart	Tronsmart
 truly	Truly Semiconductors Limited
 tyan	Tyan Computer Corporation
+udoo	Udoo
 uniwest	United Western Technologies Corp (UniWest)
 upisemi	uPI Semiconductor Corp.
 urt	United Radiant Technology Corporation
diff --git a/Documentation/devicetree/bindings/video/bridge/sil-sii8620.txt b/Documentation/devicetree/bindings/video/bridge/sil-sii8620.txt
new file mode 100644
index 000000000000..9409d9c6a260
--- /dev/null
+++ b/Documentation/devicetree/bindings/video/bridge/sil-sii8620.txt
@@ -0,0 +1,33 @@
+Silicon Image SiI8620 HDMI/MHL bridge bindings
+
+Required properties:
+	- compatible: "sil,sii8620"
+	- reg: i2c address of the bridge
+	- cvcc10-supply: Digital Core Supply Voltage (1.0V)
+	- iovcc18-supply: I/O Supply Voltage (1.8V)
+	- interrupts, interrupt-parent: interrupt specifier of INT pin
+	- reset-gpios: gpio specifier of RESET pin
+	- clocks, clock-names: specification and name of "xtal" clock
+	- video interfaces: Device node can contain video interface port
+			    node for HDMI encoder according to [1].
+
+[1]: Documentation/devicetree/bindings/media/video-interfaces.txt
+
+Example:
+	sii8620@39 {
+		reg = <0x39>;
+		compatible = "sil,sii8620";
+		cvcc10-supply = <&ldo36_reg>;
+		iovcc18-supply = <&ldo34_reg>;
+		interrupt-parent = <&gpf0>;
+		interrupts = <2 0>;
+		reset-gpio = <&gpv7 0 0>;
+		clocks = <&pmu_system_controller 0>;
+		clock-names = "xtal";
+
+		port {
+			mhl_to_hdmi: endpoint {
+				remote-endpoint = <&hdmi_to_mhl>;
+			};
+		};
+	};
diff --git a/Documentation/dmaengine/client.txt b/Documentation/dmaengine/client.txt
index 9e33189745f0..c72b4563de10 100644
--- a/Documentation/dmaengine/client.txt
+++ b/Documentation/dmaengine/client.txt
@@ -37,8 +37,8 @@ The slave DMA usage consists of following steps:
 2. Set slave and controller specific parameters
 
    Next step is always to pass some specific information to the DMA
-   driver.  Most of the generic information which a slave DMA can use
-   is in struct dma_slave_config.  This allows the clients to specify
+   driver. Most of the generic information which a slave DMA can use
+   is in struct dma_slave_config. This allows the clients to specify
    DMA direction, DMA addresses, bus widths, DMA burst lengths etc
    for the peripheral.
 
@@ -52,7 +52,7 @@ The slave DMA usage consists of following steps:
 				  struct dma_slave_config *config)
 
    Please see the dma_slave_config structure definition in dmaengine.h
-   for a detailed explanation of the struct members.  Please note
+   for a detailed explanation of the struct members. Please note
    that the 'direction' member will be going away as it duplicates the
    direction given in the prepare call.
 
@@ -101,7 +101,7 @@ The slave DMA usage consists of following steps:
 	desc = dmaengine_prep_slave_sg(chan, sgl, nr_sg, direction, flags);
 
    Once a descriptor has been obtained, the callback information can be
-   added and the descriptor must then be submitted.  Some DMA engine
+   added and the descriptor must then be submitted. Some DMA engine
    drivers may hold a spinlock between a successful preparation and
    submission so it is important that these two operations are closely
    paired.
@@ -138,7 +138,7 @@ The slave DMA usage consists of following steps:
    activity via other DMA engine calls not covered in this document.
 
    dmaengine_submit() will not start the DMA operation, it merely adds
-   it to the pending queue.  For this, see step 5, dma_async_issue_pending.
+   it to the pending queue. For this, see step 5, dma_async_issue_pending.
 
 5. Issue pending DMA requests and wait for callback notification
 
@@ -184,13 +184,13 @@ Further APIs:
 
 3. int dmaengine_resume(struct dma_chan *chan)
 
-   Resume a previously paused DMA channel.  It is invalid to resume a
+   Resume a previously paused DMA channel. It is invalid to resume a
    channel which is not currently paused.
 
 4. enum dma_status dma_async_is_tx_complete(struct dma_chan *chan,
         dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used)
 
-   This can be used to check the status of the channel.  Please see
+   This can be used to check the status of the channel. Please see
    the documentation in include/linux/dmaengine.h for a more complete
    description of this API.
 
@@ -200,7 +200,7 @@ Further APIs:
 
    Note:
 	Not all DMA engine drivers can return reliable information for
-	a running DMA channel.  It is recommended that DMA engine users
+	a running DMA channel. It is recommended that DMA engine users
 	pause or stop (via dmaengine_terminate_all()) the channel before
 	using this API.
 
diff --git a/Documentation/dmaengine/dmatest.txt b/Documentation/dmaengine/dmatest.txt
index dd77a81bdb80..fb683c72dea8 100644
--- a/Documentation/dmaengine/dmatest.txt
+++ b/Documentation/dmaengine/dmatest.txt
@@ -34,7 +34,7 @@ command:
 	% ls -1 /sys/class/dma/
 
 Once started a message like "dmatest: Started 1 threads using dma0chan0" is
-emitted.  After that only test failure messages are reported until the test
+emitted. After that only test failure messages are reported until the test
 stops.
 
 Note that running a new test will not stop any in progress test.
@@ -43,11 +43,11 @@ The following command returns the state of the test.
 	% cat /sys/module/dmatest/parameters/run
 
 To wait for test completion userpace can poll 'run' until it is false, or use
-the wait parameter.  Specifying 'wait=1' when loading the module causes module
+the wait parameter. Specifying 'wait=1' when loading the module causes module
 initialization to pause until a test run has completed, while reading
 /sys/module/dmatest/parameters/wait waits for any running test to complete
-before returning.  For example, the following scripts wait for 42 tests
-to complete before exiting.  Note that if 'iterations' is set to 'infinite' then
+before returning. For example, the following scripts wait for 42 tests
+to complete before exiting. Note that if 'iterations' is set to 'infinite' then
 waiting is disabled.
 
 Example:
@@ -81,7 +81,7 @@ Example of output:
 
 The message format is unified across the different types of errors. A number in
 the parens represents additional information, e.g. error code, error counter,
-or status.  A test thread also emits a summary line at completion listing the
+or status. A test thread also emits a summary line at completion listing the
 number of tests executed, number that failed, and a result code.
 
 Example:
diff --git a/Documentation/dmaengine/provider.txt b/Documentation/dmaengine/provider.txt
index c4fd47540b31..e33bc1c8ed2c 100644
--- a/Documentation/dmaengine/provider.txt
+++ b/Documentation/dmaengine/provider.txt
@@ -384,7 +384,7 @@ where to put them)
     - The descriptor should be prepared for reuse by invoking
       dmaengine_desc_set_reuse() which will set DMA_CTRL_REUSE.
     - dmaengine_desc_set_reuse() will succeed only when channel support
-      reusable descriptor as exhibited by capablities
+      reusable descriptor as exhibited by capabilities
     - As a consequence, if a device driver wants to skip the dma_map_sg() and
       dma_unmap_sg() in between 2 transfers, because the DMA'd data wasn't used,
       it can resubmit the transfer right after its completion.
diff --git a/Documentation/dmaengine/pxa_dma.txt b/Documentation/dmaengine/pxa_dma.txt
index 413ef9cfaa4d..0736d44b5438 100644
--- a/Documentation/dmaengine/pxa_dma.txt
+++ b/Documentation/dmaengine/pxa_dma.txt
@@ -29,7 +29,7 @@ Constraints
 
   d) Bandwidth guarantee
      The PXA architecture has 4 levels of DMAs priorities : high, normal, low.
-     The high prorities get twice as much bandwidth as the normal, which get twice
+     The high priorities get twice as much bandwidth as the normal, which get twice
      as much as the low priorities.
      A driver should be able to request a priority, especially the real-time
      ones such as pxa_camera with (big) throughputs.
diff --git a/Documentation/dontdiff b/Documentation/dontdiff
index 5385cba941d2..a23edccd2059 100644
--- a/Documentation/dontdiff
+++ b/Documentation/dontdiff
@@ -139,7 +139,6 @@ hpet_example
 hugepage-mmap
 hugepage-shm
 ihex2fw
-ikconfig.h*
 inat-tables.c
 initramfs_list
 int16.c
diff --git a/Documentation/driver-api/edac.rst b/Documentation/driver-api/edac.rst
new file mode 100644
index 000000000000..b8c742aa0a71
--- /dev/null
+++ b/Documentation/driver-api/edac.rst
@@ -0,0 +1,178 @@
+Error Detection And Correction (EDAC) Devices
+=============================================
+
+Main Concepts used at the EDAC subsystem
+----------------------------------------
+
+There are several things to be aware of that aren't at all obvious, like
+*sockets, *socket sets*, *banks*, *rows*, *chip-select rows*, *channels*,
+etc...
+
+These are some of the many terms that are thrown about that don't always
+mean what people think they mean (Inconceivable!).  In the interest of
+creating a common ground for discussion, terms and their definitions
+will be established.
+
+* Memory devices
+
+The individual DRAM chips on a memory stick.  These devices commonly
+output 4 and 8 bits each (x4, x8). Grouping several of these in parallel
+provides the number of bits that the memory controller expects:
+typically 72 bits, in order to provide 64 bits + 8 bits of ECC data.
+
+* Memory Stick
+
+A printed circuit board that aggregates multiple memory devices in
+parallel.  In general, this is the Field Replaceable Unit (FRU) which
+gets replaced, in the case of excessive errors. Most often it is also
+called DIMM (Dual Inline Memory Module).
+
+* Memory Socket
+
+A physical connector on the motherboard that accepts a single memory
+stick. Also called as "slot" on several datasheets.
+
+* Channel
+
+A memory controller channel, responsible to communicate with a group of
+DIMMs. Each channel has its own independent control (command) and data
+bus, and can be used independently or grouped with other channels.
+
+* Branch
+
+It is typically the highest hierarchy on a Fully-Buffered DIMM memory
+controller. Typically, it contains two channels. Two channels at the
+same branch can be used in single mode or in lockstep mode. When
+lockstep is enabled, the cacheline is doubled, but it generally brings
+some performance penalty. Also, it is generally not possible to point to
+just one memory stick when an error occurs, as the error correction code
+is calculated using two DIMMs instead of one. Due to that, it is capable
+of correcting more errors than on single mode.
+
+* Single-channel
+
+The data accessed by the memory controller is contained into one dimm
+only. E. g. if the data is 64 bits-wide, the data flows to the CPU using
+one 64 bits parallel access. Typically used with SDR, DDR, DDR2 and DDR3
+memories. FB-DIMM and RAMBUS use a different concept for channel, so
+this concept doesn't apply there.
+
+* Double-channel
+
+The data size accessed by the memory controller is interlaced into two
+dimms, accessed at the same time. E. g. if the DIMM is 64 bits-wide (72
+bits with ECC), the data flows to the CPU using a 128 bits parallel
+access.
+
+* Chip-select row
+
+This is the name of the DRAM signal used to select the DRAM ranks to be
+accessed. Common chip-select rows for single channel are 64 bits, for
+dual channel 128 bits. It may not be visible by the memory controller,
+as some DIMM types have a memory buffer that can hide direct access to
+it from the Memory Controller.
+
+* Single-Ranked stick
+
+A Single-ranked stick has 1 chip-select row of memory. Motherboards
+commonly drive two chip-select pins to a memory stick. A single-ranked
+stick, will occupy only one of those rows. The other will be unused.
+
+.. _doubleranked:
+
+* Double-Ranked stick
+
+A double-ranked stick has two chip-select rows which access different
+sets of memory devices.  The two rows cannot be accessed concurrently.
+
+* Double-sided stick
+
+**DEPRECATED TERM**, see :ref:`Double-Ranked stick <doubleranked>`.
+
+A double-sided stick has two chip-select rows which access different sets
+of memory devices. The two rows cannot be accessed concurrently.
+"Double-sided" is irrespective of the memory devices being mounted on
+both sides of the memory stick.
+
+* Socket set
+
+All of the memory sticks that are required for a single memory access or
+all of the memory sticks spanned by a chip-select row.  A single socket
+set has two chip-select rows and if double-sided sticks are used these
+will occupy those chip-select rows.
+
+* Bank
+
+This term is avoided because it is unclear when needing to distinguish
+between chip-select rows and socket sets.
+
+
+Memory Controllers
+------------------
+
+Most of the EDAC core is focused on doing Memory Controller error detection.
+The :c:func:`edac_mc_alloc`. It uses internally the struct ``mem_ctl_info``
+to describe the memory controllers, with is an opaque struct for the EDAC
+drivers. Only the EDAC core is allowed to touch it.
+
+.. kernel-doc:: include/linux/edac.h
+
+.. kernel-doc:: drivers/edac/edac_mc.h
+
+PCI Controllers
+---------------
+
+The EDAC subsystem provides a mechanism to handle PCI controllers by calling
+the :c:func:`edac_pci_alloc_ctl_info`. It will use the struct
+:c:type:`edac_pci_ctl_info` to describe the PCI controllers.
+
+.. kernel-doc:: drivers/edac/edac_pci.h
+
+EDAC Blocks
+-----------
+
+The EDAC subsystem also provides a generic mechanism to report errors on
+other parts of the hardware via :c:func:`edac_device_alloc_ctl_info` function.
+
+The structures :c:type:`edac_dev_sysfs_block_attribute`,
+:c:type:`edac_device_block`, :c:type:`edac_device_instance` and
+:c:type:`edac_device_ctl_info` provide a generic or abstract 'edac_device'
+representation at sysfs.
+
+This set of structures and the code that implements the APIs for the same, provide for registering EDAC type devices which are NOT standard memory or
+PCI, like:
+
+- CPU caches (L1 and L2)
+- DMA engines
+- Core CPU switches
+- Fabric switch units
+- PCIe interface controllers
+- other EDAC/ECC type devices that can be monitored for
+  errors, etc.
+
+It allows for a 2 level set of hierarchy.
+
+For example, a cache could be composed of L1, L2 and L3 levels of cache.
+Each CPU core would have its own L1 cache, while sharing L2 and maybe L3
+caches. On such case, those can be represented via the following sysfs
+nodes::
+
+	/sys/devices/system/edac/..
+
+	pci/		<existing pci directory (if available)>
+	mc/		<existing memory device directory>
+	cpu/cpu0/..	<L1 and L2 block directory>
+		/L1-cache/ce_count
+			 /ue_count
+		/L2-cache/ce_count
+			 /ue_count
+	cpu/cpu1/..	<L1 and L2 block directory>
+		/L1-cache/ce_count
+			 /ue_count
+		/L2-cache/ce_count
+			 /ue_count
+	...
+
+	the L1 and L2 directories would be "edac_device_block's"
+
+.. kernel-doc:: drivers/edac/edac_device.h
diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst
index a528178a54a5..5475a2807e7a 100644
--- a/Documentation/driver-api/index.rst
+++ b/Documentation/driver-api/index.rst
@@ -26,6 +26,7 @@ available subsections can be seen below.
    spi
    i2c
    hsi
+   edac
    miscellaneous
    vme
    80211/index
diff --git a/Documentation/edac.txt b/Documentation/edac.txt
deleted file mode 100644
index f89cfd85ae13..000000000000
--- a/Documentation/edac.txt
+++ /dev/null
@@ -1,812 +0,0 @@
-EDAC - Error Detection And Correction
-=====================================
-
-"bluesmoke" was the name for this device driver when it
-was "out-of-tree" and maintained at sourceforge.net -
-bluesmoke.sourceforge.net. That site is mostly archaic now and can be
-used only for historical purposes.
-
-When the subsystem was pushed into 2.6.16 for the first time, it was
-renamed to 'EDAC'.
-
-PURPOSE
--------
-
-The 'edac' kernel module's goal is to detect and report hardware errors
-that occur within the computer system running under linux.
-
-MEMORY
-------
-
-Memory Correctable Errors (CE) and Uncorrectable Errors (UE) are the
-primary errors being harvested. These types of errors are harvested by
-the 'edac_mc' device.
-
-Detecting CE events, then harvesting those events and reporting them,
-*can* but must not necessarily be a predictor of future UE events. With
-CE events only, the system can and will continue to operate as no data
-has been damaged yet.
-
-However, preventive maintenance and proactive part replacement of memory
-DIMMs exhibiting CEs can reduce the likelihood of the dreaded UE events
-and system panics.
-
-OTHER HARDWARE ELEMENTS
------------------------
-
-A new feature for EDAC, the edac_device class of device, was added in
-the 2.6.23 version of the kernel.
-
-This new device type allows for non-memory type of ECC hardware detectors
-to have their states harvested and presented to userspace via the sysfs
-interface.
-
-Some architectures have ECC detectors for L1, L2 and L3 caches,
-along with DMA engines, fabric switches, main data path switches,
-interconnections, and various other hardware data paths. If the hardware
-reports it, then a edac_device device probably can be constructed to
-harvest and present that to userspace.
-
-
-PCI BUS SCANNING
-----------------
-
-In addition, PCI devices are scanned for PCI Bus Parity and SERR Errors
-in order to determine if errors are occurring during data transfers.
-
-The presence of PCI Parity errors must be examined with a grain of salt.
-There are several add-in adapters that do *not* follow the PCI specification
-with regards to Parity generation and reporting. The specification says
-the vendor should tie the parity status bits to 0 if they do not intend
-to generate parity.  Some vendors do not do this, and thus the parity bit
-can "float" giving false positives.
-
-There is a PCI device attribute located in sysfs that is checked by
-the EDAC PCI scanning code. If that attribute is set, PCI parity/error
-scanning is skipped for that device. The attribute is:
-
-	broken_parity_status
-
-and is located in /sys/devices/pci<XXX>/0000:XX:YY.Z directories for
-PCI devices.
-
-
-VERSIONING
-----------
-
-EDAC is composed of a "core" module (edac_core.ko) and several Memory
-Controller (MC) driver modules. On a given system, the CORE is loaded
-and one MC driver will be loaded. Both the CORE and the MC driver (or
-edac_device driver) have individual versions that reflect current
-release level of their respective modules.
-
-Thus, to "report" on what version a system is running, one must report
-both the CORE's and the MC driver's versions.
-
-
-LOADING
--------
-
-If 'edac' was statically linked with the kernel then no loading
-is necessary. If 'edac' was built as modules then simply modprobe
-the 'edac' pieces that you need. You should be able to modprobe
-hardware-specific modules and have the dependencies load the necessary
-core modules.
-
-Example:
-
-$> modprobe amd76x_edac
-
-loads both the amd76x_edac.ko memory controller module and the edac_mc.ko
-core module.
-
-
-SYSFS INTERFACE
----------------
-
-EDAC presents a 'sysfs' interface for control and reporting purposes. It
-lives in the /sys/devices/system/edac directory.
-
-Within this directory there currently reside 2 components:
-
-	mc	memory controller(s) system
-	pci	PCI control and status system
-
-
-
-Memory Controller (mc) Model
-----------------------------
-
-Each 'mc' device controls a set of DIMM memory modules. These modules
-are laid out in a Chip-Select Row (csrowX) and Channel table (chX).
-There can be multiple csrows and multiple channels.
-
-Memory controllers allow for several csrows, with 8 csrows being a
-typical value. Yet, the actual number of csrows depends on the layout of
-a given motherboard, memory controller and DIMM characteristics.
-
-Dual channels allows for 128 bit data transfers to/from the CPU from/to
-memory. Some newer chipsets allow for more than 2 channels, like Fully
-Buffered DIMMs (FB-DIMMs). The following example will assume 2 channels:
-
-
-		Channel 0	Channel 1
-	===================================
-	csrow0	| DIMM_A0	| DIMM_B0 |
-	csrow1	| DIMM_A0	| DIMM_B0 |
-	===================================
-
-	===================================
-	csrow2	| DIMM_A1	| DIMM_B1 |
-	csrow3	| DIMM_A1	| DIMM_B1 |
-	===================================
-
-In the above example table there are 4 physical slots on the motherboard
-for memory DIMMs:
-
-	DIMM_A0
-	DIMM_B0
-	DIMM_A1
-	DIMM_B1
-
-Labels for these slots are usually silk-screened on the motherboard.
-Slots labeled 'A' are channel 0 in this example. Slots labeled 'B' are
-channel 1. Notice that there are two csrows possible on a physical DIMM.
-These csrows are allocated their csrow assignment based on the slot into
-which the memory DIMM is placed. Thus, when 1 DIMM is placed in each
-Channel, the csrows cross both DIMMs.
-
-Memory DIMMs come single or dual "ranked". A rank is a populated csrow.
-Thus, 2 single ranked DIMMs, placed in slots DIMM_A0 and DIMM_B0 above
-will have 1 csrow, csrow0. csrow1 will be empty. On the other hand,
-when 2 dual ranked DIMMs are similarly placed, then both csrow0 and
-csrow1 will be populated. The pattern repeats itself for csrow2 and
-csrow3.
-
-The representation of the above is reflected in the directory
-tree in EDAC's sysfs interface. Starting in directory
-/sys/devices/system/edac/mc each memory controller will be represented
-by its own 'mcX' directory, where 'X' is the index of the MC.
-
-
-	..../edac/mc/
-		   |
-		   |->mc0
-		   |->mc1
-		   |->mc2
-		   ....
-
-Under each 'mcX' directory each 'csrowX' is again represented by a
-'csrowX', where 'X' is the csrow index:
-
-
-	.../mc/mc0/
-		|
-		|->csrow0
-		|->csrow2
-		|->csrow3
-		....
-
-Notice that there is no csrow1, which indicates that csrow0 is composed
-of a single ranked DIMMs. This should also apply in both Channels, in
-order to have dual-channel mode be operational. Since both csrow2 and
-csrow3 are populated, this indicates a dual ranked set of DIMMs for
-channels 0 and 1.
-
-
-Within each of the 'mcX' and 'csrowX' directories are several EDAC
-control and attribute files.
-
-
-'mcX' directories
------------------
-
-In 'mcX' directories are EDAC control and attribute files for
-this 'X' instance of the memory controllers.
-
-For a description of the sysfs API, please see:
-	Documentation/ABI/testing/sysfs-devices-edac
-
-
-
-'csrowX' directories
---------------------
-
-When CONFIG_EDAC_LEGACY_SYSFS is enabled, sysfs will contain the csrowX
-directories. As this API doesn't work properly for Rambus, FB-DIMMs and
-modern Intel Memory Controllers, this is being deprecated in favor of
-dimmX directories.
-
-In the 'csrowX' directories are EDAC control and attribute files for
-this 'X' instance of csrow:
-
-
-Total Uncorrectable Errors count attribute file:
-
-	'ue_count'
-
-	This attribute file displays the total count of uncorrectable
-	errors that have occurred on this csrow. If panic_on_ue is set
-	this counter will not have a chance to increment, since EDAC
-	will panic the system.
-
-
-Total Correctable Errors count attribute file:
-
-	'ce_count'
-
-	This attribute file displays the total count of correctable
-	errors that have occurred on this csrow. This count is very
-	important to examine. CEs provide early indications that a
-	DIMM is beginning to fail. This count field should be
-	monitored for non-zero values and report such information
-	to the system administrator.
-
-
-Total memory managed by this csrow attribute file:
-
-	'size_mb'
-
-	This attribute file displays, in count of megabytes, the memory
-	that this csrow contains.
-
-
-Memory Type attribute file:
-
-	'mem_type'
-
-	This attribute file will display what type of memory is currently
-	on this csrow. Normally, either buffered or unbuffered memory.
-	Examples:
-		Registered-DDR
-		Unbuffered-DDR
-
-
-EDAC Mode of operation attribute file:
-
-	'edac_mode'
-
-	This attribute file will display what type of Error detection
-	and correction is being utilized.
-
-
-Device type attribute file:
-
-	'dev_type'
-
-	This attribute file will display what type of DRAM device is
-	being utilized on this DIMM.
-	Examples:
-		x1
-		x2
-		x4
-		x8
-
-
-Channel 0 CE Count attribute file:
-
-	'ch0_ce_count'
-
-	This attribute file will display the count of CEs on this
-	DIMM located in channel 0.
-
-
-Channel 0 UE Count attribute file:
-
-	'ch0_ue_count'
-
-	This attribute file will display the count of UEs on this
-	DIMM located in channel 0.
-
-
-Channel 0 DIMM Label control file:
-
-	'ch0_dimm_label'
-
-	This control file allows this DIMM to have a label assigned
-	to it. With this label in the module, when errors occur
-	the output can provide the DIMM label in the system log.
-	This becomes vital for panic events to isolate the
-	cause of the UE event.
-
-	DIMM Labels must be assigned after booting, with information
-	that correctly identifies the physical slot with its
-	silk screen label. This information is currently very
-	motherboard specific and determination of this information
-	must occur in userland at this time.
-
-
-Channel 1 CE Count attribute file:
-
-	'ch1_ce_count'
-
-	This attribute file will display the count of CEs on this
-	DIMM located in channel 1.
-
-
-Channel 1 UE Count attribute file:
-
-	'ch1_ue_count'
-
-	This attribute file will display the count of UEs on this
-	DIMM located in channel 0.
-
-
-Channel 1 DIMM Label control file:
-
-	'ch1_dimm_label'
-
-	This control file allows this DIMM to have a label assigned
-	to it. With this label in the module, when errors occur
-	the output can provide the DIMM label in the system log.
-	This becomes vital for panic events to isolate the
-	cause of the UE event.
-
-	DIMM Labels must be assigned after booting, with information
-	that correctly identifies the physical slot with its
-	silk screen label. This information is currently very
-	motherboard specific and determination of this information
-	must occur in userland at this time.
-
-
-
-SYSTEM LOGGING
---------------
-
-If logging for UEs and CEs is enabled, then system logs will contain
-information indicating that errors have been detected:
-
-EDAC MC0: CE page 0x283, offset 0xce0, grain 8, syndrome 0x6ec3, row 0,
-channel 1 "DIMM_B1": amd76x_edac
-
-EDAC MC0: CE page 0x1e5, offset 0xfb0, grain 8, syndrome 0xb741, row 0,
-channel 1 "DIMM_B1": amd76x_edac
-
-
-The structure of the message is:
-	the memory controller			(MC0)
-	Error type				(CE)
-	memory page				(0x283)
-	offset in the page			(0xce0)
-	the byte granularity 			(grain 8)
-		or resolution of the error
-	the error syndrome			(0xb741)
-	memory row				(row 0)
-	memory channel				(channel 1)
-	DIMM label, if set prior		(DIMM B1
-	and then an optional, driver-specific message that may
-		have additional information.
-
-Both UEs and CEs with no info will lack all but memory controller, error
-type, a notice of "no info" and then an optional, driver-specific error
-message.
-
-
-PCI Bus Parity Detection
-------------------------
-
-On Header Type 00 devices, the primary status is looked at for any
-parity error regardless of whether parity is enabled on the device or
-not. (The spec indicates parity is generated in some cases). On Header
-Type 01 bridges, the secondary status register is also looked at to see
-if parity occurred on the bus on the other side of the bridge.
-
-
-SYSFS CONFIGURATION
--------------------
-
-Under /sys/devices/system/edac/pci are control and attribute files as follows:
-
-
-Enable/Disable PCI Parity checking control file:
-
-	'check_pci_parity'
-
-
-	This control file enables or disables the PCI Bus Parity scanning
-	operation. Writing a 1 to this file enables the scanning. Writing
-	a 0 to this file disables the scanning.
-
-	Enable:
-	echo "1" >/sys/devices/system/edac/pci/check_pci_parity
-
-	Disable:
-	echo "0" >/sys/devices/system/edac/pci/check_pci_parity
-
-
-Parity Count:
-
-	'pci_parity_count'
-
-	This attribute file will display the number of parity errors that
-	have been detected.
-
-
-
-MODULE PARAMETERS
------------------
-
-Panic on UE control file:
-
-	'edac_mc_panic_on_ue'
-
-	An uncorrectable error will cause a machine panic.  This is usually
-	desirable.  It is a bad idea to continue when an uncorrectable error
-	occurs - it is indeterminate what was uncorrected and the operating
-	system context might be so mangled that continuing will lead to further
-	corruption. If the kernel has MCE configured, then EDAC will never
-	notice the UE.
-
-	LOAD TIME: module/kernel parameter: edac_mc_panic_on_ue=[0|1]
-
-	RUN TIME:  echo "1" > /sys/module/edac_core/parameters/edac_mc_panic_on_ue
-
-
-Log UE control file:
-
-	'edac_mc_log_ue'
-
-	Generate kernel messages describing uncorrectable errors.  These errors
-	are reported through the system message log system.  UE statistics
-	will be accumulated even when UE logging is disabled.
-
-	LOAD TIME: module/kernel parameter: edac_mc_log_ue=[0|1]
-
-	RUN TIME: echo "1" > /sys/module/edac_core/parameters/edac_mc_log_ue
-
-
-Log CE control file:
-
-	'edac_mc_log_ce'
-
-	Generate kernel messages describing correctable errors.  These
-	errors are reported through the system message log system.
-	CE statistics will be accumulated even when CE logging is disabled.
-
-	LOAD TIME: module/kernel parameter: edac_mc_log_ce=[0|1]
-
-	RUN TIME: echo "1" > /sys/module/edac_core/parameters/edac_mc_log_ce
-
-
-Polling period control file:
-
-	'edac_mc_poll_msec'
-
-	The time period, in milliseconds, for polling for error information.
-	Too small a value wastes resources.  Too large a value might delay
-	necessary handling of errors and might loose valuable information for
-	locating the error.  1000 milliseconds (once each second) is the current
-	default. Systems which require all the bandwidth they can get, may
-	increase this.
-
-	LOAD TIME: module/kernel parameter: edac_mc_poll_msec=[0|1]
-
-	RUN TIME: echo "1000" > /sys/module/edac_core/parameters/edac_mc_poll_msec
-
-
-Panic on PCI PARITY Error:
-
-	'panic_on_pci_parity'
-
-
-	This control file enables or disables panicking when a parity
-	error has been detected.
-
-
-	module/kernel parameter: edac_panic_on_pci_pe=[0|1]
-
-	Enable:
-	echo "1" > /sys/module/edac_core/parameters/edac_panic_on_pci_pe
-
-	Disable:
-	echo "0" > /sys/module/edac_core/parameters/edac_panic_on_pci_pe
-
-
-
-EDAC device type
-----------------
-
-In the header file, edac_core.h, there is a series of edac_device structures
-and APIs for the EDAC_DEVICE.
-
-User space access to an edac_device is through the sysfs interface.
-
-At the location /sys/devices/system/edac (sysfs) new edac_device devices will
-appear.
-
-There is a three level tree beneath the above 'edac' directory. For example,
-the 'test_device_edac' device (found at the bluesmoke.sourceforget.net website)
-installs itself as:
-
-	/sys/devices/systm/edac/test-instance
-
-in this directory are various controls, a symlink and one or more 'instance'
-directories.
-
-The standard default controls are:
-
-	log_ce		boolean to log CE events
-	log_ue		boolean to log UE events
-	panic_on_ue	boolean to 'panic' the system if an UE is encountered
-			(default off, can be set true via startup script)
-	poll_msec	time period between POLL cycles for events
-
-The test_device_edac device adds at least one of its own custom control:
-
-	test_bits	which in the current test driver does nothing but
-			show how it is installed. A ported driver can
-			add one or more such controls and/or attributes
-			for specific uses.
-			One out-of-tree driver uses controls here to allow
-			for ERROR INJECTION operations to hardware
-			injection registers
-
-The symlink points to the 'struct dev' that is registered for this edac_device.
-
-INSTANCES
----------
-
-One or more instance directories are present. For the 'test_device_edac' case:
-
-	test-instance0
-
-
-In this directory there are two default counter attributes, which are totals of
-counter in deeper subdirectories.
-
-	ce_count	total of CE events of subdirectories
-	ue_count	total of UE events of subdirectories
-
-BLOCKS
-------
-
-At the lowest directory level is the 'block' directory. There can be 0, 1
-or more blocks specified in each instance.
-
-	test-block0
-
-
-In this directory the default attributes are:
-
-	ce_count	which is counter of CE events for this 'block'
-			of hardware being monitored
-	ue_count	which is counter of UE events for this 'block'
-			of hardware being monitored
-
-
-The 'test_device_edac' device adds 4 attributes and 1 control:
-
-	test-block-bits-0	for every POLL cycle this counter
-				is incremented
-	test-block-bits-1	every 10 cycles, this counter is bumped once,
-				and test-block-bits-0 is set to 0
-	test-block-bits-2	every 100 cycles, this counter is bumped once,
-				and test-block-bits-1 is set to 0
-	test-block-bits-3	every 1000 cycles, this counter is bumped once,
-				and test-block-bits-2 is set to 0
-
-
-	reset-counters		writing ANY thing to this control will
-				reset all the above counters.
-
-
-Use of the 'test_device_edac' driver should enable any others to create their own
-unique drivers for their hardware systems.
-
-The 'test_device_edac' sample driver is located at the
-bluesmoke.sourceforge.net project site for EDAC.
-
-
-NEHALEM USAGE OF EDAC APIs
---------------------------
-
-This chapter documents some EXPERIMENTAL mappings for EDAC API to handle
-Nehalem EDAC driver. They will likely be changed on future versions
-of the driver.
-
-Due to the way Nehalem exports Memory Controller data, some adjustments
-were done at i7core_edac driver. This chapter will cover those differences
-
-1) On Nehalem, there is one Memory Controller per Quick Patch Interconnect
-   (QPI). At the driver, the term "socket" means one QPI. This is
-   associated with a physical CPU socket.
-
-   Each MC have 3 physical read channels, 3 physical write channels and
-   3 logic channels. The driver currently sees it as just 3 channels.
-   Each channel can have up to 3 DIMMs.
-
-   The minimum known unity is DIMMs. There are no information about csrows.
-   As EDAC API maps the minimum unity is csrows, the driver sequentially
-   maps channel/dimm into different csrows.
-
-   For example, supposing the following layout:
-	Ch0 phy rd0, wr0 (0x063f4031): 2 ranks, UDIMMs
-	  dimm 0 1024 Mb offset: 0, bank: 8, rank: 1, row: 0x4000, col: 0x400
-	  dimm 1 1024 Mb offset: 4, bank: 8, rank: 1, row: 0x4000, col: 0x400
-        Ch1 phy rd1, wr1 (0x063f4031): 2 ranks, UDIMMs
-	  dimm 0 1024 Mb offset: 0, bank: 8, rank: 1, row: 0x4000, col: 0x400
-	Ch2 phy rd3, wr3 (0x063f4031): 2 ranks, UDIMMs
-	  dimm 0 1024 Mb offset: 0, bank: 8, rank: 1, row: 0x4000, col: 0x400
-   The driver will map it as:
-	csrow0: channel 0, dimm0
-	csrow1: channel 0, dimm1
-	csrow2: channel 1, dimm0
-	csrow3: channel 2, dimm0
-
-exports one
-   DIMM per csrow.
-
-   Each QPI is exported as a different memory controller.
-
-2) Nehalem MC has the ability to generate errors. The driver implements this
-   functionality via some error injection nodes:
-
-   For injecting a memory error, there are some sysfs nodes, under
-   /sys/devices/system/edac/mc/mc?/:
-
-   inject_addrmatch/*:
-      Controls the error injection mask register. It is possible to specify
-      several characteristics of the address to match an error code:
-         dimm = the affected dimm. Numbers are relative to a channel;
-         rank = the memory rank;
-         channel = the channel that will generate an error;
-         bank = the affected bank;
-         page = the page address;
-         column (or col) = the address column.
-      each of the above values can be set to "any" to match any valid value.
-
-      At driver init, all values are set to any.
-
-      For example, to generate an error at rank 1 of dimm 2, for any channel,
-      any bank, any page, any column:
-		echo 2 >/sys/devices/system/edac/mc/mc0/inject_addrmatch/dimm
-		echo 1 >/sys/devices/system/edac/mc/mc0/inject_addrmatch/rank
-
-	To return to the default behaviour of matching any, you can do:
-		echo any >/sys/devices/system/edac/mc/mc0/inject_addrmatch/dimm
-		echo any >/sys/devices/system/edac/mc/mc0/inject_addrmatch/rank
-
-   inject_eccmask:
-       specifies what bits will have troubles,
-
-   inject_section:
-       specifies what ECC cache section will get the error:
-		3 for both
-		2 for the highest
-		1 for the lowest
-
-   inject_type:
-       specifies the type of error, being a combination of the following bits:
-		bit 0 - repeat
-		bit 1 - ecc
-		bit 2 - parity
-
-       inject_enable starts the error generation when something different
-       than 0 is written.
-
-   All inject vars can be read. root permission is needed for write.
-
-   Datasheet states that the error will only be generated after a write on an
-   address that matches inject_addrmatch. It seems, however, that reading will
-   also produce an error.
-
-   For example, the following code will generate an error for any write access
-   at socket 0, on any DIMM/address on channel 2:
-
-   echo 2 >/sys/devices/system/edac/mc/mc0/inject_addrmatch/channel
-   echo 2 >/sys/devices/system/edac/mc/mc0/inject_type
-   echo 64 >/sys/devices/system/edac/mc/mc0/inject_eccmask
-   echo 3 >/sys/devices/system/edac/mc/mc0/inject_section
-   echo 1 >/sys/devices/system/edac/mc/mc0/inject_enable
-   dd if=/dev/mem of=/dev/null seek=16k bs=4k count=1 >& /dev/null
-
-   For socket 1, it is needed to replace "mc0" by "mc1" at the above
-   commands.
-
-   The generated error message will look like:
-
-   EDAC MC0: UE row 0, channel-a= 0 channel-b= 0 labels "-": NON_FATAL (addr = 0x0075b980, socket=0, Dimm=0, Channel=2, syndrome=0x00000040, count=1, Err=8c0000400001009f:4000080482 (read error: read ECC error))
-
-3) Nehalem specific Corrected Error memory counters
-
-   Nehalem have some registers to count memory errors. The driver uses those
-   registers to report Corrected Errors on devices with Registered Dimms.
-
-   However, those counters don't work with Unregistered Dimms. As the chipset
-   offers some counters that also work with UDIMMS (but with a worse level of
-   granularity than the default ones), the driver exposes those registers for
-   UDIMM memories.
-
-   They can be read by looking at the contents of all_channel_counts/
-
-   $ for i in /sys/devices/system/edac/mc/mc0/all_channel_counts/*; do echo $i; cat $i; done
-	/sys/devices/system/edac/mc/mc0/all_channel_counts/udimm0
-	0
-	/sys/devices/system/edac/mc/mc0/all_channel_counts/udimm1
-	0
-	/sys/devices/system/edac/mc/mc0/all_channel_counts/udimm2
-	0
-
-   What happens here is that errors on different csrows, but at the same
-   dimm number will increment the same counter.
-   So, in this memory mapping:
-	csrow0: channel 0, dimm0
-	csrow1: channel 0, dimm1
-	csrow2: channel 1, dimm0
-	csrow3: channel 2, dimm0
-   The hardware will increment udimm0 for an error at the first dimm at either
-	csrow0, csrow2  or csrow3;
-   The hardware will increment udimm1 for an error at the second dimm at either
-	csrow0, csrow2  or csrow3;
-   The hardware will increment udimm2 for an error at the third dimm at either
-	csrow0, csrow2  or csrow3;
-
-4) Standard error counters
-
-   The standard error counters are generated when an mcelog error is received
-   by the driver. Since, with udimm, this is counted by software, it is
-   possible that some errors could be lost. With rdimm's, they display the
-   contents of the registers
-
-AMD64_EDAC REFERENCE DOCUMENTS USED
------------------------------------
-amd64_edac module is based on the following documents
-(available from http://support.amd.com/en-us/search/tech-docs):
-
-1. Title:  BIOS and Kernel Developer's Guide for AMD Athlon 64 and AMD
-	   Opteron Processors
-   AMD publication #: 26094
-   Revision: 3.26
-   Link: http://support.amd.com/TechDocs/26094.PDF
-
-2. Title:  BIOS and Kernel Developer's Guide for AMD NPT Family 0Fh
-	   Processors
-   AMD publication #: 32559
-   Revision: 3.00
-   Issue Date: May 2006
-   Link: http://support.amd.com/TechDocs/32559.pdf
-
-3. Title:  BIOS and Kernel Developer's Guide (BKDG) For AMD Family 10h
-	   Processors
-   AMD publication #: 31116
-   Revision: 3.00
-   Issue Date: September 07, 2007
-   Link: http://support.amd.com/TechDocs/31116.pdf
-
-4. Title: BIOS and Kernel Developer's Guide (BKDG) for AMD Family 15h
-	  Models 30h-3Fh Processors
-   AMD publication #: 49125
-   Revision: 3.06
-   Issue Date: 2/12/2015 (latest release)
-   Link: http://support.amd.com/TechDocs/49125_15h_Models_30h-3Fh_BKDG.pdf
-
-5. Title: BIOS and Kernel Developer's Guide (BKDG) for AMD Family 15h
-	  Models 60h-6Fh Processors
-   AMD publication #: 50742
-   Revision: 3.01
-   Issue Date: 7/23/2015 (latest release)
-   Link: http://support.amd.com/TechDocs/50742_15h_Models_60h-6Fh_BKDG.pdf
-
-6. Title: BIOS and Kernel Developer's Guide (BKDG) for AMD Family 16h
-	  Models 00h-0Fh Processors
-   AMD publication #: 48751
-   Revision: 3.03
-   Issue Date: 2/23/2015 (latest release)
-   Link: http://support.amd.com/TechDocs/48751_16h_bkdg.pdf
-
-CREDITS:
-========
-
-Written by Doug Thompson <dougthompson@xmission.com>
-7 Dec 2005
-17 Jul 2007	Updated
-
-(c) Mauro Carvalho Chehab
-05 Aug 2009	Nehalem interface
-
-EDAC authors/maintainers:
-
-	Doug Thompson, Dave Jiang, Dave Peterson et al,
-	Mauro Carvalho Chehab
-	Borislav Petkov
-	original author: Thayne Harbaugh
diff --git a/Documentation/filesystems/Locking b/Documentation/filesystems/Locking
index 1b5f15653b1b..69e2387ca278 100644
--- a/Documentation/filesystems/Locking
+++ b/Documentation/filesystems/Locking
@@ -556,7 +556,7 @@ till "end_pgoff". ->map_pages() is called with page table locked and must
 not block.  If it's not possible to reach a page without blocking,
 filesystem should skip it. Filesystem should use do_set_pte() to setup
 page table entry. Pointer to entry associated with the page is passed in
-"pte" field in fault_env structure. Pointers to entries for other offsets
+"pte" field in vm_fault structure. Pointers to entries for other offsets
 should be calculated relative to "pte".
 
 	->page_mkwrite() is called when a previously read-only pte is
diff --git a/Documentation/filesystems/configfs/configfs.txt b/Documentation/filesystems/configfs/configfs.txt
index 8ec9136aae56..3828e85345ae 100644
--- a/Documentation/filesystems/configfs/configfs.txt
+++ b/Documentation/filesystems/configfs/configfs.txt
@@ -174,7 +174,7 @@ among other things.  For that, it needs a type.
 		void (*release)(struct config_item *);
 		int (*allow_link)(struct config_item *src,
 				  struct config_item *target);
-		int (*drop_link)(struct config_item *src,
+		void (*drop_link)(struct config_item *src,
 				 struct config_item *target);
 	};
 
diff --git a/Documentation/filesystems/dax.txt b/Documentation/filesystems/dax.txt
index 23d18b8a49d5..a7e6e14aeb08 100644
--- a/Documentation/filesystems/dax.txt
+++ b/Documentation/filesystems/dax.txt
@@ -58,22 +58,22 @@ Implementation Tips for Filesystem Writers
 Filesystem support consists of
 - adding support to mark inodes as being DAX by setting the S_DAX flag in
   i_flags
-- implementing the direct_IO address space operation, and calling
-  dax_do_io() instead of blockdev_direct_IO() if S_DAX is set
+- implementing ->read_iter and ->write_iter operations which use dax_iomap_rw()
+  when inode has S_DAX flag set
 - implementing an mmap file operation for DAX files which sets the
   VM_MIXEDMAP and VM_HUGEPAGE flags on the VMA, and setting the vm_ops to
-  include handlers for fault, pmd_fault and page_mkwrite (which should
-  probably call dax_fault(), dax_pmd_fault() and dax_mkwrite(), passing the
-  appropriate get_block() callback)
-- calling dax_truncate_page() instead of block_truncate_page() for DAX files
-- calling dax_zero_page_range() instead of zero_user() for DAX files
+  include handlers for fault, pmd_fault, page_mkwrite, pfn_mkwrite. These
+  handlers should probably call dax_iomap_fault() (for fault and page_mkwrite
+  handlers), dax_iomap_pmd_fault(), dax_pfn_mkwrite() passing the appropriate
+  iomap operations.
+- calling iomap_zero_range() passing appropriate iomap operations instead of
+  block_truncate_page() for DAX files
 - ensuring that there is sufficient locking between reads, writes,
   truncates and page faults
 
-The get_block() callback passed to the DAX functions may return
-uninitialised extents.  If it does, it must ensure that simultaneous
-calls to get_block() (for example by a page-fault racing with a read()
-or a write()) work correctly.
+The iomap handlers for allocating blocks must make sure that allocated blocks
+are zeroed out and converted to written extents before being returned to avoid
+exposure of uninitialized data through mmap.
 
 These filesystems may be used for inspiration:
 - ext2: see Documentation/filesystems/ext2.txt
diff --git a/Documentation/filesystems/ext4.txt b/Documentation/filesystems/ext4.txt
index 6c0108eb0137..3698ed3146e3 100644
--- a/Documentation/filesystems/ext4.txt
+++ b/Documentation/filesystems/ext4.txt
@@ -351,14 +351,13 @@ nouid32			Disables 32-bit UIDs and GIDs.  This is for
 			interoperability  with  older kernels which only
 			store and expect 16-bit values.
 
-block_validity		This options allows to enables/disables the in-kernel
+block_validity(*)	These options enable or disable the in-kernel
 noblock_validity	facility for tracking filesystem metadata blocks
-			within internal data structures. This allows multi-
-			block allocator and other routines to quickly locate
-			extents which might overlap with filesystem metadata
-			blocks. This option is intended for debugging
-			purposes and since it negatively affects the
-			performance, it is off by default.
+			within internal data structures.  This allows multi-
+			block allocator and other routines to notice
+			bugs or corrupted allocation bitmaps which cause
+			blocks to be allocated which overlap with
+			filesystem metadata blocks.
 
 dioread_lock		Controls whether or not ext4 should use the DIO read
 dioread_nolock		locking. If the dioread_nolock option is specified
diff --git a/Documentation/filesystems/sysfs-pci.txt b/Documentation/filesystems/sysfs-pci.txt
index 74eaac26f8b8..6ea1ceda6f52 100644
--- a/Documentation/filesystems/sysfs-pci.txt
+++ b/Documentation/filesystems/sysfs-pci.txt
@@ -17,6 +17,7 @@ that support it.  For example, a given bus might look like this:
      |   |-- resource0
      |   |-- resource1
      |   |-- resource2
+     |   |-- revision
      |   |-- rom
      |   |-- subsystem_device
      |   |-- subsystem_vendor
@@ -41,6 +42,7 @@ files, each with their own function.
        resource		   PCI resource host addresses (ascii, ro)
        resource0..N	   PCI resource N, if present (binary, mmap, rw[1])
        resource0_wc..N_wc  PCI WC map resource N, if prefetchable (binary, mmap)
+       revision		   PCI revision (ascii, ro)
        rom		   PCI ROM resource, if present (binary, ro)
        subsystem_device	   PCI subsystem device (ascii, ro)
        subsystem_vendor	   PCI subsystem vendor (ascii, ro)
diff --git a/Documentation/filesystems/xfs.txt b/Documentation/filesystems/xfs.txt
index c2d44e6e117b..3b9b5c149f32 100644
--- a/Documentation/filesystems/xfs.txt
+++ b/Documentation/filesystems/xfs.txt
@@ -51,13 +51,6 @@ default behaviour.
 	CRC enabled filesystems always use the attr2 format, and so
 	will reject the noattr2 mount option if it is set.
 
-  barrier (*)
-  nobarrier
-	Enables/disables the use of block layer write barriers for
-	writes into the journal and for data integrity operations.
-	This allows for drive level write caching to be enabled, for
-	devices that support write barriers.
-
   discard
   nodiscard (*)
 	Enable/disable the issuing of commands to let the block
@@ -228,7 +221,10 @@ default behaviour.
 Deprecated Mount Options
 ========================
 
-None at present.
+  Name				Removal Schedule
+  ----				----------------
+  barrier			no earlier than v4.15
+  nobarrier			no earlier than v4.15
 
 
 Removed Mount Options
diff --git a/Documentation/fpga/fpga-mgr.txt b/Documentation/fpga/fpga-mgr.txt
index ce3e84fa9023..86ee5078fd03 100644
--- a/Documentation/fpga/fpga-mgr.txt
+++ b/Documentation/fpga/fpga-mgr.txt
@@ -18,31 +18,37 @@ API Functions:
 To program the FPGA from a file or from a buffer:
 -------------------------------------------------
 
-	int fpga_mgr_buf_load(struct fpga_manager *mgr, u32 flags,
+	int fpga_mgr_buf_load(struct fpga_manager *mgr,
+			      struct fpga_image_info *info,
 		              const char *buf, size_t count);
 
 Load the FPGA from an image which exists as a buffer in memory.
 
-	int fpga_mgr_firmware_load(struct fpga_manager *mgr, u32 flags,
+	int fpga_mgr_firmware_load(struct fpga_manager *mgr,
+				   struct fpga_image_info *info,
 		                   const char *image_name);
 
 Load the FPGA from an image which exists as a file.  The image file must be on
-the firmware search path (see the firmware class documentation).
-
-For both these functions, flags == 0 for normal full reconfiguration or
-FPGA_MGR_PARTIAL_RECONFIG for partial reconfiguration.  If successful, the FPGA
-ends up in operating mode.  Return 0 on success or a negative error code.
+the firmware search path (see the firmware class documentation).  If successful,
+the FPGA ends up in operating mode.  Return 0 on success or a negative error
+code.
 
+A FPGA design contained in a FPGA image file will likely have particulars that
+affect how the image is programmed to the FPGA.  These are contained in struct
+fpga_image_info.  Currently the only such particular is a single flag bit
+indicating whether the image is for full or partial reconfiguration.
 
 To get/put a reference to a FPGA manager:
 -----------------------------------------
 
 	struct fpga_manager *of_fpga_mgr_get(struct device_node *node);
+	struct fpga_manager *fpga_mgr_get(struct device *dev);
+
+Given a DT node or device, get an exclusive reference to a FPGA manager.
 
 	void fpga_mgr_put(struct fpga_manager *mgr);
 
-Given a DT node, get an exclusive reference to a FPGA manager or release
-the reference.
+Release the reference.
 
 
 To register or unregister the low level FPGA-specific driver:
@@ -70,8 +76,11 @@ struct device_node *mgr_node = ...
 char *buf = ...
 int count = ...
 
+/* struct with information about the FPGA image to program. */
+struct fpga_image_info info;
+
 /* flags indicates whether to do full or partial reconfiguration */
-int flags = 0;
+info.flags = 0;
 
 int ret;
 
@@ -79,7 +88,7 @@ int ret;
 struct fpga_manager *mgr = of_fpga_mgr_get(mgr_node);
 
 /* Load the buffer to the FPGA */
-ret = fpga_mgr_buf_load(mgr, flags, buf, count);
+ret = fpga_mgr_buf_load(mgr, &info, buf, count);
 
 /* Release the FPGA manager */
 fpga_mgr_put(mgr);
@@ -96,8 +105,11 @@ struct device_node *mgr_node = ...
 /* FPGA image is in this file which is in the firmware search path */
 const char *path = "fpga-image-9.rbf"
 
+/* struct with information about the FPGA image to program. */
+struct fpga_image_info info;
+
 /* flags indicates whether to do full or partial reconfiguration */
-int flags = 0;
+info.flags = 0;
 
 int ret;
 
@@ -105,7 +117,7 @@ int ret;
 struct fpga_manager *mgr = of_fpga_mgr_get(mgr_node);
 
 /* Get the firmware image (path) and load it to the FPGA */
-ret = fpga_mgr_firmware_load(mgr, flags, path);
+ret = fpga_mgr_firmware_load(mgr, &info, path);
 
 /* Release the FPGA manager */
 fpga_mgr_put(mgr);
@@ -157,7 +169,10 @@ The programming sequence is:
  2. .write (may be called once or multiple times)
  3. .write_complete
 
-The .write_init function will prepare the FPGA to receive the image data.
+The .write_init function will prepare the FPGA to receive the image data.  The
+buffer passed into .write_init will be atmost .initial_header_size bytes long,
+if the whole bitstream is not immediately available then the core code will
+buffer up at least this much before starting.
 
 The .write function writes a buffer to the FPGA. The buffer may be contain the
 whole FPGA image or may be a smaller chunk of an FPGA image.  In the latter
diff --git a/Documentation/gpio/driver.txt b/Documentation/gpio/driver.txt
index 368d5a294d89..747c721776ed 100644
--- a/Documentation/gpio/driver.txt
+++ b/Documentation/gpio/driver.txt
@@ -175,8 +175,8 @@ The IRQ portions of the GPIO block are implemented using an irqchip, using
 the header <linux/irq.h>. So basically such a driver is utilizing two sub-
 systems simultaneously: gpio and irq.
 
-RT_FULL: GPIO driver should not use spinlock_t or any sleepable APIs
-(like PM runtime) as part of its irq_chip implementation on -RT.
+RT_FULL: a realtime compliant GPIO driver should not use spinlock_t or any
+sleepable APIs (like PM runtime) as part of its irq_chip implementation.
 - spinlock_t should be replaced with raw_spinlock_t [1].
 - If sleepable APIs have to be used, these can be done from the .irq_bus_lock()
   and .irq_bus_unlock() callbacks, as these are the only slowpath callbacks
@@ -185,33 +185,32 @@ RT_FULL: GPIO driver should not use spinlock_t or any sleepable APIs
 GPIO irqchips usually fall in one of two categories:
 
 * CHAINED GPIO irqchips: these are usually the type that is embedded on
-  an SoC. This means that there is a fast IRQ handler for the GPIOs that
+  an SoC. This means that there is a fast IRQ flow handler for the GPIOs that
   gets called in a chain from the parent IRQ handler, most typically the
-  system interrupt controller. This means the GPIO irqchip is registered
-  using irq_set_chained_handler() or the corresponding
-  gpiochip_set_chained_irqchip() helper function, and the GPIO irqchip
-  handler will be called immediately from the parent irqchip, while
-  holding the IRQs disabled. The GPIO irqchip will then end up calling
-  something like this sequence in its interrupt handler:
-
-  static irqreturn_t tc3589x_gpio_irq(int irq, void *data)
+  system interrupt controller. This means that the GPIO irqchip handler will
+  be called immediately from the parent irqchip, while holding the IRQs
+  disabled. The GPIO irqchip will then end up calling something like this
+  sequence in its interrupt handler:
+
+  static irqreturn_t foo_gpio_irq(int irq, void *data)
       chained_irq_enter(...);
       generic_handle_irq(...);
       chained_irq_exit(...);
 
   Chained GPIO irqchips typically can NOT set the .can_sleep flag on
-  struct gpio_chip, as everything happens directly in the callbacks.
+  struct gpio_chip, as everything happens directly in the callbacks: no
+  slow bus traffic like I2C can be used.
 
   RT_FULL: Note, chained IRQ handlers will not be forced threaded on -RT.
   As result, spinlock_t or any sleepable APIs (like PM runtime) can't be used
   in chained IRQ handler.
-  if required (and if it can't be converted to the nested threaded GPIO irqchip)
-  - chained IRQ handler can be converted to generic irq handler and this way
-  it will be threaded IRQ handler on -RT and hard IRQ handler on non-RT
+  If required (and if it can't be converted to the nested threaded GPIO irqchip)
+  a chained IRQ handler can be converted to generic irq handler and this way
+  it will be a threaded IRQ handler on -RT and a hard IRQ handler on non-RT
   (for example, see [3]).
   Know W/A: The generic_handle_irq() is expected to be called with IRQ disabled,
-  so IRQ core will complain if it will be called from IRQ handler which is
-  forced thread. The "fake?" raw lock can be used to W/A this problem:
+  so the IRQ core will complain if it is called from an IRQ handler which is
+  forced to a thread. The "fake?" raw lock can be used to W/A this problem:
 
 	raw_spinlock_t wa_lock;
 	static irqreturn_t omap_gpio_irq_handler(int irq, void *gpiobank)
@@ -243,7 +242,7 @@ GPIO irqchips usually fall in one of two categories:
   by the driver. The hallmark of this driver is to call something like
   this in its interrupt handler:
 
-  static irqreturn_t tc3589x_gpio_irq(int irq, void *data)
+  static irqreturn_t foo_gpio_irq(int irq, void *data)
       ...
       handle_nested_irq(irq);
 
@@ -256,23 +255,31 @@ associated irqdomain and resource allocation callbacks, the gpiolib has
 some helpers that can be enabled by selecting the GPIOLIB_IRQCHIP Kconfig
 symbol:
 
-* gpiochip_irqchip_add(): adds an irqchip to a gpiochip. It will pass
+* gpiochip_irqchip_add(): adds a chained irqchip to a gpiochip. It will pass
   the struct gpio_chip* for the chip to all IRQ callbacks, so the callbacks
   need to embed the gpio_chip in its state container and obtain a pointer
   to the container using container_of().
   (See Documentation/driver-model/design-patterns.txt)
 
-  If there is a need to exclude certain GPIOs from the IRQ domain, one can
-  set .irq_need_valid_mask of the gpiochip before gpiochip_add_data() is
-  called. This allocates .irq_valid_mask with as many bits set as there are
-  GPIOs in the chip. Drivers can exclude GPIOs by clearing bits from this
-  mask. The mask must be filled in before gpiochip_irqchip_add() is called.
+* gpiochip_irqchip_add_nested(): adds a nested irqchip to a gpiochip.
+  Apart from that it works exactly like the chained irqchip.
 
 * gpiochip_set_chained_irqchip(): sets up a chained irq handler for a
   gpio_chip from a parent IRQ and passes the struct gpio_chip* as handler
   data. (Notice handler data, since the irqchip data is likely used by the
-  parent irqchip!) This is for the chained type of chip. This is also used
-  to set up a nested irqchip if NULL is passed as handler.
+  parent irqchip!).
+
+* gpiochip_set_nested_irqchip(): sets up a nested irq handler for a
+  gpio_chip from a parent IRQ. As the parent IRQ has usually been
+  explicitly requested by the driver, this does very little more than
+  mark all the child IRQs as having the other IRQ as parent.
+
+If there is a need to exclude certain GPIOs from the IRQ domain, you can
+set .irq_need_valid_mask of the gpiochip before gpiochip_add_data() is
+called. This allocates an .irq_valid_mask with as many bits set as there
+are GPIOs in the chip. Drivers can exclude GPIOs by clearing bits from this
+mask. The mask must be filled in before gpiochip_irqchip_add() or
+gpiochip_irqchip_add_nested() is called.
 
 To use the helpers please keep the following in mind:
 
@@ -323,6 +330,9 @@ When implementing an irqchip inside a GPIO driver, these two functions should
 typically be called in the .startup() and .shutdown() callbacks from the
 irqchip.
 
+When using the gpiolib irqchip helpers, these callback are automatically
+assigned.
+
 Real-Time compliance for GPIO IRQ chips
 ---------------------------------------
 
diff --git a/Documentation/gpu/drm-internals.rst b/Documentation/gpu/drm-internals.rst
index 37284bcc7764..e35920db1f4c 100644
--- a/Documentation/gpu/drm-internals.rst
+++ b/Documentation/gpu/drm-internals.rst
@@ -143,6 +143,9 @@ Device Instance and Driver Handling
 .. kernel-doc:: drivers/gpu/drm/drm_drv.c
    :export:
 
+.. kernel-doc:: include/drm/drm_drv.h
+   :internal:
+
 Driver Load
 -----------
 
@@ -350,6 +353,23 @@ how the ioctl is allowed to be called.
 .. kernel-doc:: drivers/gpu/drm/drm_ioctl.c
    :export:
 
+
+Misc Utilities
+==============
+
+Printer
+-------
+
+.. kernel-doc:: include/drm/drm_print.h
+   :doc: print
+
+.. kernel-doc:: include/drm/drm_print.h
+   :internal:
+
+.. kernel-doc:: drivers/gpu/drm/drm_print.c
+   :export:
+
+
 Legacy Support Code
 ===================
 
diff --git a/Documentation/gpu/drm-kms-helpers.rst b/Documentation/gpu/drm-kms-helpers.rst
index bb4254d19cbb..03040aa14fe8 100644
--- a/Documentation/gpu/drm-kms-helpers.rst
+++ b/Documentation/gpu/drm-kms-helpers.rst
@@ -63,6 +63,9 @@ Atomic State Reset and Initialization
 .. kernel-doc:: drivers/gpu/drm/drm_atomic_helper.c
    :doc: atomic state reset and initialization
 
+Helper Functions Reference
+--------------------------
+
 .. kernel-doc:: include/drm/drm_atomic_helper.h
    :internal:
 
@@ -261,14 +264,6 @@ Plane Helper Reference
 .. kernel-doc:: drivers/gpu/drm/drm_plane_helper.c
    :export:
 
-Tile group
-==========
-
-# FIXME: This should probably be moved into a property documentation section
-
-.. kernel-doc:: drivers/gpu/drm/drm_crtc.c
-   :doc: Tile group
-
 Auxiliary Modeset Helpers
 =========================
 
diff --git a/Documentation/gpu/drm-kms.rst b/Documentation/gpu/drm-kms.rst
index 38af5d1cc59f..0c9abdc0ee31 100644
--- a/Documentation/gpu/drm-kms.rst
+++ b/Documentation/gpu/drm-kms.rst
@@ -15,25 +15,24 @@ be setup by initializing the following fields.
 -  struct drm_mode_config_funcs \*funcs;
    Mode setting functions.
 
-Modeset Base Object Abstraction
-===============================
+Mode Configuration
 
-.. kernel-doc:: include/drm/drm_mode_object.h
-   :internal:
+KMS Core Structures and Functions
+=================================
 
-.. kernel-doc:: drivers/gpu/drm/drm_mode_object.c
+.. kernel-doc:: drivers/gpu/drm/drm_mode_config.c
    :export:
 
-KMS Data Structures
-===================
-
-.. kernel-doc:: include/drm/drm_crtc.h
+.. kernel-doc:: include/drm/drm_mode_config.h
    :internal:
 
-KMS API Functions
-=================
+Modeset Base Object Abstraction
+===============================
 
-.. kernel-doc:: drivers/gpu/drm/drm_crtc.c
+.. kernel-doc:: include/drm/drm_mode_object.h
+   :internal:
+
+.. kernel-doc:: drivers/gpu/drm/drm_mode_object.c
    :export:
 
 Atomic Mode Setting Function Reference
@@ -45,6 +44,15 @@ Atomic Mode Setting Function Reference
 .. kernel-doc:: include/drm/drm_atomic.h
    :internal:
 
+CRTC Abstraction
+================
+
+.. kernel-doc:: drivers/gpu/drm/drm_crtc.c
+   :export:
+
+.. kernel-doc:: include/drm/drm_crtc.h
+   :internal:
+
 Frame Buffer Abstraction
 ========================
 
@@ -63,52 +71,17 @@ Frame Buffer Functions Reference
 DRM Format Handling
 ===================
 
+.. kernel-doc:: include/drm/drm_fourcc.h
+   :internal:
+
 .. kernel-doc:: drivers/gpu/drm/drm_fourcc.c
    :export:
 
 Dumb Buffer Objects
 ===================
 
-The KMS API doesn't standardize backing storage object creation and
-leaves it to driver-specific ioctls. Furthermore actually creating a
-buffer object even for GEM-based drivers is done through a
-driver-specific ioctl - GEM only has a common userspace interface for
-sharing and destroying objects. While not an issue for full-fledged
-graphics stacks that include device-specific userspace components (in
-libdrm for instance), this limit makes DRM-based early boot graphics
-unnecessarily complex.
-
-Dumb objects partly alleviate the problem by providing a standard API to
-create dumb buffers suitable for scanout, which can then be used to
-create KMS frame buffers.
-
-To support dumb objects drivers must implement the dumb_create,
-dumb_destroy and dumb_map_offset operations.
-
--  int (\*dumb_create)(struct drm_file \*file_priv, struct
-   drm_device \*dev, struct drm_mode_create_dumb \*args);
-   The dumb_create operation creates a driver object (GEM or TTM
-   handle) suitable for scanout based on the width, height and depth
-   from the struct :c:type:`struct drm_mode_create_dumb
-   <drm_mode_create_dumb>` argument. It fills the argument's
-   handle, pitch and size fields with a handle for the newly created
-   object and its line pitch and size in bytes.
-
--  int (\*dumb_destroy)(struct drm_file \*file_priv, struct
-   drm_device \*dev, uint32_t handle);
-   The dumb_destroy operation destroys a dumb object created by
-   dumb_create.
-
--  int (\*dumb_map_offset)(struct drm_file \*file_priv, struct
-   drm_device \*dev, uint32_t handle, uint64_t \*offset);
-   The dumb_map_offset operation associates an mmap fake offset with
-   the object given by the handle and returns it. Drivers must use the
-   :c:func:`drm_gem_create_mmap_offset()` function to associate
-   the fake offset as described in ?.
-
-Note that dumb objects may not be used for gpu acceleration, as has been
-attempted on some ARM embedded platforms. Such drivers really must have
-a hardware-specific ioctl to allocate suitable buffer objects.
+.. kernel-doc:: drivers/gpu/drm/drm_dumb_buffers.c
+   :doc: overview
 
 Plane Abstraction
 =================
@@ -287,6 +260,12 @@ Property Types and Blob Property Support
 .. kernel-doc:: drivers/gpu/drm/drm_property.c
    :export:
 
+Standard Connector Properties
+-----------------------------
+
+.. kernel-doc:: drivers/gpu/drm/drm_connector.c
+   :doc: standard connector properties
+
 Plane Composition Properties
 ----------------------------
 
@@ -308,6 +287,18 @@ Color Management Properties
 .. kernel-doc:: drivers/gpu/drm/drm_color_mgmt.c
    :export:
 
+Tile Group Property
+-------------------
+
+.. kernel-doc:: drivers/gpu/drm/drm_connector.c
+   :doc: Tile group
+
+Explicit Fencing Properties
+---------------------------
+
+.. kernel-doc:: drivers/gpu/drm/drm_atomic.c
+   :doc: explicit fencing properties
+
 Existing KMS Properties
 -----------------------
 
diff --git a/Documentation/gpu/drm-uapi.rst b/Documentation/gpu/drm-uapi.rst
index 1ba301cebe16..de3ac9f90f8f 100644
--- a/Documentation/gpu/drm-uapi.rst
+++ b/Documentation/gpu/drm-uapi.rst
@@ -216,3 +216,9 @@ interfaces. Especially since all hardware-acceleration interfaces to
 userspace are driver specific for efficiency and other reasons these
 interfaces can be rather substantial. Hence every driver has its own
 chapter.
+
+Testing and validation
+======================
+
+.. kernel-doc:: drivers/gpu/drm/drm_debugfs_crc.c
+   :doc: CRC ABI
diff --git a/Documentation/gpu/i915.rst b/Documentation/gpu/i915.rst
index 87aaffc22920..117d2ab7a5f7 100644
--- a/Documentation/gpu/i915.rst
+++ b/Documentation/gpu/i915.rst
@@ -49,6 +49,15 @@ Intel GVT-g Guest Support(vGPU)
 .. kernel-doc:: drivers/gpu/drm/i915/i915_vgpu.c
    :internal:
 
+Intel GVT-g Host Support(vGPU device model)
+-------------------------------------------
+
+.. kernel-doc:: drivers/gpu/drm/i915/intel_gvt.c
+   :doc: Intel GVT-g host support
+
+.. kernel-doc:: drivers/gpu/drm/i915/intel_gvt.c
+   :internal:
+
 Display Hardware Handling
 =========================
 
@@ -180,7 +189,7 @@ Display Refresh Rate Switching (DRRS)
 DPIO
 ----
 
-.. kernel-doc:: drivers/gpu/drm/i915/i915_reg.h
+.. kernel-doc:: drivers/gpu/drm/i915/intel_dpio_phy.c
    :doc: DPIO
 
 CSR firmware support for DMC
@@ -249,19 +258,19 @@ Global GTT views
 GTT Fences and Swizzling
 ------------------------
 
-.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_fence.c
+.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_fence_reg.c
    :internal:
 
 Global GTT Fence Handling
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_fence.c
+.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_fence_reg.c
    :doc: fence register handling
 
 Hardware Tiling and Swizzling Details
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_fence.c
+.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_fence_reg.c
    :doc: tiling swizzling details
 
 Object Tiling IOCTLs
diff --git a/Documentation/hwmon/hwmon-kernel-api.txt b/Documentation/hwmon/hwmon-kernel-api.txt
index ef9d74947f5c..2505ae67e2b6 100644
--- a/Documentation/hwmon/hwmon-kernel-api.txt
+++ b/Documentation/hwmon/hwmon-kernel-api.txt
@@ -23,7 +23,6 @@ Each hardware monitoring driver must #include <linux/hwmon.h> and, in most
 cases, <linux/hwmon-sysfs.h>. linux/hwmon.h declares the following
 register/unregister functions:
 
-struct device *hwmon_device_register(struct device *dev);
 struct device *
 hwmon_device_register_with_groups(struct device *dev, const char *name,
 				  void *drvdata,
@@ -38,36 +37,31 @@ struct device *
 hwmon_device_register_with_info(struct device *dev,
 				const char *name, void *drvdata,
 				const struct hwmon_chip_info *info,
-				const struct attribute_group **groups);
+				const struct attribute_group **extra_groups);
 
 struct device *
 devm_hwmon_device_register_with_info(struct device *dev,
-				     const char *name,
-				     void *drvdata,
-				     const struct hwmon_chip_info *info,
-				     const struct attribute_group **groups);
+				const char *name,
+				void *drvdata,
+				const struct hwmon_chip_info *info,
+				const struct attribute_group **extra_groups);
 
 void hwmon_device_unregister(struct device *dev);
 void devm_hwmon_device_unregister(struct device *dev);
 
-hwmon_device_register registers a hardware monitoring device. The parameter
-of this function is a pointer to the parent device.
-This function returns a pointer to the newly created hardware monitoring device
-or PTR_ERR for failure. If this registration function is used, hardware
-monitoring sysfs attributes are expected to have been created and attached to
-the parent device prior to calling hwmon_device_register. A name attribute must
-have been created by the caller.
-
-hwmon_device_register_with_groups is similar to hwmon_device_register. However,
-it has additional parameters. The name parameter is a pointer to the hwmon
-device name. The registration function wil create a name sysfs attribute
-pointing to this name. The drvdata parameter is the pointer to the local
-driver data.  hwmon_device_register_with_groups will attach this pointer
-to the newly allocated hwmon device. The pointer can be retrieved by the driver
-using dev_get_drvdata() on the hwmon device pointer. The groups parameter is
+hwmon_device_register_with_groups registers a hardware monitoring device.
+The first parameter of this function is a pointer to the parent device.
+The name parameter is a pointer to the hwmon device name. The registration
+function wil create a name sysfs attribute pointing to this name.
+The drvdata parameter is the pointer to the local driver data.
+hwmon_device_register_with_groups will attach this pointer to the newly
+allocated hwmon device. The pointer can be retrieved by the driver using
+dev_get_drvdata() on the hwmon device pointer. The groups parameter is
 a pointer to a list of sysfs attribute groups. The list must be NULL terminated.
 hwmon_device_register_with_groups creates the hwmon device with name attribute
 as well as all sysfs attributes attached to the hwmon device.
+This function returns a pointer to the newly created hardware monitoring device
+or PTR_ERR for failure.
 
 devm_hwmon_device_register_with_groups is similar to
 hwmon_device_register_with_groups. However, it is device managed, meaning the
@@ -87,13 +81,13 @@ hwmon_device_unregister deregisters a registered hardware monitoring device.
 The parameter of this function is the pointer to the registered hardware
 monitoring device structure. This function must be called from the driver
 remove function if the hardware monitoring device was registered with
-hwmon_device_register, hwmon_device_register_with_groups, or
-hwmon_device_register_with_info.
+hwmon_device_register_with_groups or hwmon_device_register_with_info.
 
 devm_hwmon_device_unregister does not normally have to be called. It is only
 needed for error handling, and only needed if the driver probe fails after
-the call to devm_hwmon_device_register_with_groups and if the automatic
-(device managed) removal would be too late.
+the call to devm_hwmon_device_register_with_groups or
+hwmon_device_register_with_info and if the automatic (device managed)
+removal would be too late.
 
 Using devm_hwmon_device_register_with_info()
 --------------------------------------------
@@ -106,9 +100,9 @@ const char *name	Device name
 void *drvdata		Driver private data
 const struct hwmon_chip_info *info
 			Pointer to chip description.
-const struct attribute_group **groups
-			Null-terminated list of additional sysfs attribute
-			groups.
+const struct attribute_group **extra_groups
+			Null-terminated list of additional non-standard
+			sysfs attribute groups.
 
 This function returns a pointer to the created hardware monitoring device
 on success and a negative error code for failure.
@@ -160,7 +154,7 @@ It contains following fields:
 * type: The hardware monitoring sensor type.
   Supported sensor types are
   * hwmon_chip		A virtual sensor type, used to describe attributes
-			which apply to the entire chip.
+  *			which are not bound to a specific input or output
   * hwmon_temp		Temperature sensor
   * hwmon_in		Voltage sensor
   * hwmon_curr		Current sensor
@@ -293,9 +287,9 @@ Driver-provided sysfs attributes
 
 If the hardware monitoring device is registered with
 hwmon_device_register_with_info or devm_hwmon_device_register_with_info,
-it is most likely not necessary to provide sysfs attributes. Only non-standard
-sysfs attributes need to be provided when one of those registration functions
-is used.
+it is most likely not necessary to provide sysfs attributes. Only additional
+non-standard sysfs attributes need to be provided when one of those registration
+functions is used.
 
 The header file linux/hwmon-sysfs.h provides a number of useful macros to
 declare and use hardware monitoring sysfs attributes.
diff --git a/Documentation/hwmon/tc654 b/Documentation/hwmon/tc654
new file mode 100644
index 000000000000..91a2843f5f98
--- /dev/null
+++ b/Documentation/hwmon/tc654
@@ -0,0 +1,31 @@
+Kernel driver tc654
+===================
+
+Supported chips:
+  * Microship TC654 and TC655
+    Prefix: 'tc654'
+    Datasheet: http://ww1.microchip.com/downloads/en/DeviceDoc/20001734C.pdf
+
+Authors:
+        Chris Packham <chris.packham@alliedtelesis.co.nz>
+        Masahiko Iwamoto <iwamoto@allied-telesis.co.jp>
+
+Description
+-----------
+This driver implements support for the Microchip TC654 and TC655.
+
+The TC654 uses the 2-wire interface compatible with the SMBUS 2.0
+specification. The TC654 has two (2) inputs for measuring fan RPM and
+one (1) PWM output which can be used for fan control.
+
+Configuration Notes
+-------------------
+Ordinarily the pwm1_mode ABI is used for controlling the pwm output
+mode.  However, for this chip the output is always pwm, and the
+pwm1_mode determines if the pwm output is controlled via the pwm1 value
+or via the Vin analog input.
+
+
+Setting pwm1_mode to 1 will cause the pwm output to be driven based on
+the pwm1 value. Setting pwm1_mode to 0 will cause the pwm output to be
+driven based on the Vin input.
diff --git a/Documentation/hwmon/tmp108 b/Documentation/hwmon/tmp108
new file mode 100644
index 000000000000..25802df23010
--- /dev/null
+++ b/Documentation/hwmon/tmp108
@@ -0,0 +1,36 @@
+Kernel driver tmp108
+====================
+
+Supported chips:
+  * Texas Instruments TMP108
+    Prefix: 'tmp108'
+    Addresses scanned: none
+    Datasheet: http://www.ti.com/product/tmp108
+
+Author:
+	John Muir <john@jmuir.com>
+
+Description
+-----------
+
+The Texas Instruments TMP108 implements one temperature sensor. An alert pin
+can be set when temperatures exceed minimum or maximum values plus or minus a
+hysteresis value. (This driver does not support interrupts for the alert pin,
+and the device runs in comparator mode.)
+
+The sensor is accurate to 0.75C over the range of -25 to +85 C, and to 1.0
+degree from -40 to +125 C. Resolution of the sensor is 0.0625 degree. The
+operating temperature has a minimum of -55 C and a maximum of +150 C.
+Hysteresis values can be set to 0, 1, 2, or 4C.
+
+The TMP108 has a programmable update rate that can select between 8, 4, 1, and
+0.5 Hz.
+
+By default the TMP108 reads the temperature continuously. To conserve power,
+the TMP108 has a one-shot mode where the device is normally shut-down. When a
+one shot is requested the temperature is read, the result can be retrieved,
+and then the device is shut down automatically. (This driver only supports
+continuous mode.)
+
+The driver provides the common sysfs-interface for temperatures (see
+Documentation/hwmon/sysfs-interface under Temperatures).
diff --git a/Documentation/i2c/busses/i2c-mlxcpld b/Documentation/i2c/busses/i2c-mlxcpld
new file mode 100644
index 000000000000..4e46c440b38d
--- /dev/null
+++ b/Documentation/i2c/busses/i2c-mlxcpld
@@ -0,0 +1,47 @@
+Driver i2c-mlxcpld
+
+Author: Michael Shych <michaelsh@mellanox.com>
+
+This is the Mellanox I2C controller logic, implemented in Lattice CPLD
+device.
+Device supports:
+ - Master mode.
+ - One physical bus.
+ - Polling mode.
+
+This controller is equipped within the next Mellanox systems:
+"msx6710", "msx6720", "msb7700", "msn2700", "msx1410", "msn2410", "msb7800",
+"msn2740", "msn2100".
+
+The next transaction types are supported:
+ - Receive Byte/Block.
+ - Send Byte/Block.
+ - Read Byte/Block.
+ - Write Byte/Block.
+
+Registers:
+CTRL		0x1 - control reg.
+			Resets all the registers.
+HALF_CYC	0x4 - cycle reg.
+			Configure the width of I2C SCL half clock cycle (in 4 LPC_CLK
+			units).
+I2C_HOLD	0x5 - hold reg.
+			OE (output enable) is delayed by value set to this register
+			(in LPC_CLK units)
+CMD			0x6 - command reg.
+			Bit 0, 0 = write, 1 = read.
+			Bits [7:1] - the 7bit Address of the I2C device.
+			It should be written last as it triggers an I2C transaction.
+NUM_DATA	0x7 - data size reg.
+			Number of data bytes to write in read transaction
+NUM_ADDR	0x8 - address reg.
+			Number of address bytes to write in read transaction.
+STATUS		0x9 - status reg.
+			Bit 0 - transaction is completed.
+			Bit 4 - ACK/NACK.
+DATAx		0xa - 0x54  - 68 bytes data buffer regs.
+			For write transaction address is specified in four first bytes
+			(DATA1 - DATA4), data starting from DATA4.
+			For read transactions address is sent in a separate transaction and
+			specified in the four first bytes (DATA0 - DATA3). Data is read
+			starting from DATA0.
diff --git a/Documentation/i2c/smbus-protocol b/Documentation/i2c/smbus-protocol
index 14d4ec1be245..092d474f5843 100644
--- a/Documentation/i2c/smbus-protocol
+++ b/Documentation/i2c/smbus-protocol
@@ -200,10 +200,14 @@ alerting device's address.
 [S] [HostAddr] [Wr] A [DevAddr] A [DataLow] A [DataHigh] A [P]
 
 This is implemented in the following way in the Linux kernel:
-* I2C bus drivers which support SMBus Host Notify should call
-  i2c_setup_smbus_host_notify() to setup SMBus Host Notify support.
-* I2C drivers for devices which can trigger SMBus Host Notify should implement
-  the optional alert() callback.
+* I2C bus drivers which support SMBus Host Notify should report
+  I2C_FUNC_SMBUS_HOST_NOTIFY.
+* I2C bus drivers trigger SMBus Host Notify by a call to
+  i2c_handle_smbus_host_notify().
+* I2C drivers for devices which can trigger SMBus Host Notify will have
+  client->irq assigned to a Host Notify IRQ if noone else specified an other.
+
+There is currently no way to retrieve the data parameter from the client.
 
 
 Packet Error Checking (PEC)
diff --git a/Documentation/laptops/thinkpad-acpi.txt b/Documentation/laptops/thinkpad-acpi.txt
index 72a150d8f3df..ba2e7d254842 100644
--- a/Documentation/laptops/thinkpad-acpi.txt
+++ b/Documentation/laptops/thinkpad-acpi.txt
@@ -540,6 +540,7 @@ Events that are propagated by the driver to userspace:
 0x6022		ALARM: a sensor is extremely hot
 0x6030		System thermal table changed
 0x6040		Nvidia Optimus/AC adapter related (TO BE VERIFIED)
+0x60C0		X1 Yoga 2016, Tablet mode status changed
 
 Battery nearly empty alarms are a last resort attempt to get the
 operating system to hibernate or shutdown cleanly (0x2313), or shutdown
diff --git a/Documentation/leds/leds-lp5523.txt b/Documentation/leds/leds-lp5523.txt
index 0dbbd279c9b9..0961a060fc4d 100644
--- a/Documentation/leds/leds-lp5523.txt
+++ b/Documentation/leds/leds-lp5523.txt
@@ -34,8 +34,8 @@ There are two ways to run LED patterns.
   Control interface for the engines:
   x is 1 .. 3
   enginex_mode : disabled, load, run
-  enginex_load : microcode load (visible only in load mode)
-  enginex_leds : led mux control (visible only in load mode)
+  enginex_load : microcode load
+  enginex_leds : led mux control
 
   cd /sys/class/leds/lp5523:channel2/device
   echo "load" > engine3_mode
diff --git a/Documentation/leds/uleds.txt b/Documentation/leds/uleds.txt
new file mode 100644
index 000000000000..13e375a580f9
--- /dev/null
+++ b/Documentation/leds/uleds.txt
@@ -0,0 +1,36 @@
+Userspace LEDs
+==============
+
+The uleds driver supports userspace LEDs. This can be useful for testing
+triggers and can also be used to implement virtual LEDs.
+
+
+Usage
+=====
+
+When the driver is loaded, a character device is created at /dev/uleds. To
+create a new LED class device, open /dev/uleds and write a uleds_user_dev
+structure to it (found in kernel public header file linux/uleds.h).
+
+    #define LED_MAX_NAME_SIZE 64
+
+    struct uleds_user_dev {
+        char name[LED_MAX_NAME_SIZE];
+    };
+
+A new LED class device will be created with the name given. The name can be
+any valid sysfs device node name, but consider using the LED class naming
+convention of "devicename:color:function".
+
+The current brightness is found by reading a single byte from the character
+device. Values are unsigned: 0 to 255. Reading will block until the brightness
+changes. The device node can also be polled to notify when the brightness value
+changes.
+
+The LED class device will be removed when the open file handle to /dev/uleds
+is closed.
+
+Multiple LED class devices are created by opening additional file handles to
+/dev/uleds.
+
+See tools/leds/uledmon.c for an example userspace program.
diff --git a/Documentation/livepatch/livepatch.txt b/Documentation/livepatch/livepatch.txt
index 6c43f6ebee8d..f5967316deb9 100644
--- a/Documentation/livepatch/livepatch.txt
+++ b/Documentation/livepatch/livepatch.txt
@@ -87,7 +87,7 @@ The theory about how to apply functions a safe way is rather complex.
 The aim is to define a so-called consistency model. It attempts to define
 conditions when the new implementation could be used so that the system
 stays consistent. The theory is not yet finished. See the discussion at
-http://thread.gmane.org/gmane.linux.kernel/1823033/focus=1828189
+https://lkml.kernel.org/r/20141107140458.GA21774@suse.cz
 
 The current consistency model is very simple. It guarantees that either
 the old or the new function is called. But various functions get redirected
diff --git a/Documentation/power/devices.txt b/Documentation/power/devices.txt
index 8ba6625fdd63..73ddea39a9ce 100644
--- a/Documentation/power/devices.txt
+++ b/Documentation/power/devices.txt
@@ -607,7 +607,9 @@ individually.  Instead, a set of devices sharing a power resource can be put
 into a low-power state together at the same time by turning off the shared
 power resource.  Of course, they also need to be put into the full-power state
 together, by turning the shared power resource on.  A set of devices with this
-property is often referred to as a power domain.
+property is often referred to as a power domain. A power domain may also be
+nested inside another power domain. The nested domain is referred to as the
+sub-domain of the parent domain.
 
 Support for power domains is provided through the pm_domain field of struct
 device.  This field is a pointer to an object of type struct dev_pm_domain,
@@ -629,6 +631,16 @@ support for power domains into subsystem-level callbacks, for example by
 modifying the platform bus type.  Other platforms need not implement it or take
 it into account in any way.
 
+Devices may be defined as IRQ-safe which indicates to the PM core that their
+runtime PM callbacks may be invoked with disabled interrupts (see
+Documentation/power/runtime_pm.txt for more information).  If an IRQ-safe
+device belongs to a PM domain, the runtime PM of the domain will be
+disallowed, unless the domain itself is defined as IRQ-safe. However, it
+makes sense to define a PM domain as IRQ-safe only if all the devices in it
+are IRQ-safe. Moreover, if an IRQ-safe domain has a parent domain, the runtime
+PM of the parent is only allowed if the parent itself is IRQ-safe too with the
+additional restriction that all child domains of an IRQ-safe parent must also
+be IRQ-safe.
 
 Device Low Power (suspend) States
 ---------------------------------
diff --git a/Documentation/power/states.txt b/Documentation/power/states.txt
index 50f3ef9177c1..8a39ce45d8a0 100644
--- a/Documentation/power/states.txt
+++ b/Documentation/power/states.txt
@@ -8,25 +8,43 @@ for each state.
 
 The states are represented by strings that can be read or written to the
 /sys/power/state file.  Those strings may be "mem", "standby", "freeze" and
-"disk", where the last one always represents hibernation (Suspend-To-Disk) and
-the meaning of the remaining ones depends on the relative_sleep_states command
-line argument.
-
-For relative_sleep_states=1, the strings "mem", "standby" and "freeze" label the
-available non-hibernation sleep states from the deepest to the shallowest,
-respectively.  In that case, "mem" is always present in /sys/power/state,
-because there is at least one non-hibernation sleep state in every system.  If
-the given system supports two non-hibernation sleep states, "standby" is present
-in /sys/power/state in addition to "mem".  If the system supports three
-non-hibernation sleep states, "freeze" will be present in /sys/power/state in
-addition to "mem" and "standby".
-
-For relative_sleep_states=0, which is the default, the following descriptions
-apply.
-
-state:		Suspend-To-Idle
+"disk", where the last three always represent Power-On Suspend (if supported),
+Suspend-To-Idle and hibernation (Suspend-To-Disk), respectively.
+
+The meaning of the "mem" string is controlled by the /sys/power/mem_sleep file.
+It contains strings representing the available modes of system suspend that may
+be triggered by writing "mem" to /sys/power/state.  These modes are "s2idle"
+(Suspend-To-Idle), "shallow" (Power-On Suspend) and "deep" (Suspend-To-RAM).
+The "s2idle" mode is always available, while the other ones are only available
+if supported by the platform (if not supported, the strings representing them
+are not present in /sys/power/mem_sleep).  The string representing the suspend
+mode to be used subsequently is enclosed in square brackets.  Writing one of
+the other strings present in /sys/power/mem_sleep to it causes the suspend mode
+to be used subsequently to change to the one represented by that string.
+
+Consequently, there are two ways to cause the system to go into the
+Suspend-To-Idle sleep state.  The first one is to write "freeze" directly to
+/sys/power/state.  The second one is to write "s2idle" to /sys/power/mem_sleep
+and then to wrtie "mem" to /sys/power/state.  Similarly, there are two ways
+to cause the system to go into the Power-On Suspend sleep state (the strings to
+write to the control files in that case are "standby" or "shallow" and "mem",
+respectively) if that state is supported by the platform.  In turn, there is
+only one way to cause the system to go into the Suspend-To-RAM state (write
+"deep" into /sys/power/mem_sleep and "mem" into /sys/power/state).
+
+The default suspend mode (ie. the one to be used without writing anything into
+/sys/power/mem_sleep) is either "deep" (if Suspend-To-RAM is supported) or
+"s2idle", but it can be overridden by the value of the "mem_sleep_default"
+parameter in the kernel command line.  On some ACPI-based systems, depending on
+the information in the FADT, the default may be "s2idle" even if Suspend-To-RAM
+is supported.
+
+The properties of all of the sleep states are described below.
+
+
+State:		Suspend-To-Idle
 ACPI state:	S0
-Label:		"freeze"
+Label:		"s2idle" ("freeze")
 
 This state is a generic, pure software, light-weight, system sleep state.
 It allows more energy to be saved relative to runtime idle by freezing user
@@ -35,13 +53,13 @@ lower-power than available at run time), such that the processors can
 spend more time in their idle states.
 
 This state can be used for platforms without Power-On Suspend/Suspend-to-RAM
-support, or it can be used in addition to Suspend-to-RAM (memory sleep)
-to provide reduced resume latency.  It is always supported.
+support, or it can be used in addition to Suspend-to-RAM to provide reduced
+resume latency.  It is always supported.
 
 
 State:		Standby / Power-On Suspend
 ACPI State:	S1
-Label:		"standby"
+Label:		"shallow" ("standby")
 
 This state, if supported, offers moderate, though real, power savings, while
 providing a relatively low-latency transition back to a working system.  No
@@ -58,7 +76,7 @@ state.
 
 State:		Suspend-to-RAM
 ACPI State:	S3
-Label:		"mem"
+Label:		"deep"
 
 This state, if supported, offers significant power savings as everything in the
 system is put into a low-power state, except for memory, which should be placed
diff --git a/Documentation/security/keys-trusted-encrypted.txt b/Documentation/security/keys-trusted-encrypted.txt
index 324ddf5223b3..b20a993a32af 100644
--- a/Documentation/security/keys-trusted-encrypted.txt
+++ b/Documentation/security/keys-trusted-encrypted.txt
@@ -32,8 +32,6 @@ Usage:
                      (40 ascii zeros)
        blobauth=     ascii hex auth for sealed data default 0x00...
                      (40 ascii zeros)
-       blobauth=     ascii hex auth for sealed data default 0x00...
-                     (40 ascii zeros)
        pcrinfo=	     ascii hex of PCR_INFO or PCR_INFO_LONG (no default)
        pcrlock=	     pcr number to be extended to "lock" blob
        migratable=   0|1 indicating permission to reseal to new PCR values,
diff --git a/Documentation/sync_file.txt b/Documentation/sync_file.txt
index b63a68531afd..269681a6faec 100644
--- a/Documentation/sync_file.txt
+++ b/Documentation/sync_file.txt
@@ -6,7 +6,7 @@
 
 This document serves as a guide for device drivers writers on what the
 sync_file API is, and how drivers can support it. Sync file is the carrier of
-the fences(struct fence) that are needed to synchronize between drivers or
+the fences(struct dma_fence) that are needed to synchronize between drivers or
 across process boundaries.
 
 The sync_file API is meant to be used to send and receive fence information
@@ -32,9 +32,9 @@ in-fences and out-fences
 Sync files can go either to or from userspace. When a sync_file is sent from
 the driver to userspace we call the fences it contains 'out-fences'. They are
 related to a buffer that the driver is processing or is going to process, so
-the driver creates an out-fence to be able to notify, through fence_signal(),
-when it has finished using (or processing) that buffer. Out-fences are fences
-that the driver creates.
+the driver creates an out-fence to be able to notify, through
+dma_fence_signal(), when it has finished using (or processing) that buffer.
+Out-fences are fences that the driver creates.
 
 On the other hand if the driver receives fence(s) through a sync_file from
 userspace we call these fence(s) 'in-fences'. Receiveing in-fences means that
@@ -47,7 +47,7 @@ Creating Sync Files
 When a driver needs to send an out-fence userspace it creates a sync_file.
 
 Interface:
-	struct sync_file *sync_file_create(struct fence *fence);
+	struct sync_file *sync_file_create(struct dma_fence *fence);
 
 The caller pass the out-fence and gets back the sync_file. That is just the
 first step, next it needs to install an fd on sync_file->file. So it gets an
@@ -72,11 +72,11 @@ of the Sync File to the kernel. The kernel can then retrieve the fences
 from it.
 
 Interface:
-	struct fence *sync_file_get_fence(int fd);
+	struct dma_fence *sync_file_get_fence(int fd);
 
 
 The returned reference is owned by the caller and must be disposed of
-afterwards using fence_put(). In case of error, a NULL is returned instead.
+afterwards using dma_fence_put(). In case of error, a NULL is returned instead.
 
 References:
 [1] struct sync_file in include/linux/sync_file.h
diff --git a/Documentation/trace/events.txt b/Documentation/trace/events.txt
index 08d74d75150d..2cc08d4a326e 100644
--- a/Documentation/trace/events.txt
+++ b/Documentation/trace/events.txt
@@ -189,16 +189,13 @@ And for string fields they are:
 
 ==, !=, ~
 
-The glob (~) only accepts a wild card character (*) at the start and or
-end of the string. For example:
+The glob (~) accepts a wild card character (*,?) and character classes
+([). For example:
 
   prev_comm ~ "*sh"
   prev_comm ~ "sh*"
   prev_comm ~ "*sh*"
-
-But does not allow for it to be within the string:
-
-  prev_comm ~ "ba*sh"   <-- is invalid
+  prev_comm ~ "ba*sh"
 
 5.2 Setting filters
 -------------------
diff --git a/Documentation/trace/ftrace.txt b/Documentation/trace/ftrace.txt
index 5596e2d71d6d..006f47c7d913 100644
--- a/Documentation/trace/ftrace.txt
+++ b/Documentation/trace/ftrace.txt
@@ -416,6 +416,12 @@ of ftrace. Here is a list of some of the key files:
 
 		trace_fd = open("trace_marker", WR_ONLY);
 
+  trace_marker_raw:
+
+	This is similar to trace_marker above, but is meant for for binary data
+	to be written to it, where a tool can be used to parse the data
+	from trace_pipe_raw.
+
   uprobe_events:
  
 	Add dynamic tracepoints in programs.
@@ -2238,16 +2244,13 @@ hrtimer_interrupt
 sys_nanosleep
 
 
-Perhaps this is not enough. The filters also allow simple wild
-cards. Only the following are currently available
+Perhaps this is not enough. The filters also allow glob(7) matching.
 
   <match>*  - will match functions that begin with <match>
   *<match>  - will match functions that end with <match>
   *<match>* - will match functions that have <match> in it
-
-These are the only wild cards which are supported.
-
-  <match>*<match> will not work.
+  <match1>*<match2> - will match functions that begin with
+                      <match1> and end with <match2>
 
 Note: It is better to use quotes to enclose the wild cards,
       otherwise the shell may expand the parameters into names
diff --git a/Documentation/trace/intel_th.txt b/Documentation/trace/intel_th.txt
index f7fc5ba5df8d..f92070e7dde0 100644
--- a/Documentation/trace/intel_th.txt
+++ b/Documentation/trace/intel_th.txt
@@ -97,3 +97,25 @@ $ echo 0 > /sys/bus/intel_th/devices/0-msc0/active
 # and now you can collect the trace from the device node:
 
 $ cat /dev/intel_th0/msc0 > my_stp_trace
+
+Host Debugger Mode
+==================
+
+It is possible to configure the Trace Hub and control its trace
+capture from a remote debug host, which should be connected via one of
+the hardware debugging interfaces, which will then be used to both
+control Intel Trace Hub and transfer its trace data to the debug host.
+
+The driver needs to be told that such an arrangement is taking place
+so that it does not touch any capture/port configuration and avoids
+conflicting with the debug host's configuration accesses. The only
+activity that the driver will perform in this mode is collecting
+software traces to the Software Trace Hub (an stm class device). The
+user is still responsible for setting up adequate master/channel
+mappings that the decoder on the receiving end would recognize.
+
+In order to enable the host mode, set the 'host_mode' parameter of the
+'intel_th' kernel module to 'y'. None of the virtual output devices
+will show up on the intel_th bus. Also, trace configuration and
+capture controlling attribute groups of the 'gth' device will not be
+exposed. The 'sth' device will operate as usual.
diff --git a/Documentation/trace/stm.txt b/Documentation/trace/stm.txt
index ea035f9dbfd7..11cff47eecce 100644
--- a/Documentation/trace/stm.txt
+++ b/Documentation/trace/stm.txt
@@ -69,12 +69,43 @@ stm device's channel mmio region is 64 bytes and hardware page size is
 width==64, you should be able to mmap() one page on this file
 descriptor and obtain direct access to an mmio region for 64 channels.
 
+Examples of STM devices are Intel(R) Trace Hub [1] and Coresight STM
+[2].
+
+stm_source
+==========
+
 For kernel-based trace sources, there is "stm_source" device
 class. Devices of this class can be connected and disconnected to/from
-stm devices at runtime via a sysfs attribute.
+stm devices at runtime via a sysfs attribute called "stm_source_link"
+by writing the name of the desired stm device there, for example:
 
-Examples of STM devices are Intel(R) Trace Hub [1] and Coresight STM
-[2].
+$ echo dummy_stm.0 > /sys/class/stm_source/console/stm_source_link
+
+For examples on how to use stm_source interface in the kernel, refer
+to stm_console or stm_heartbeat drivers.
+
+Each stm_source device will need to assume a master and a range of
+channels, depending on how many channels it requires. These are
+allocated for the device according to the policy configuration. If
+there's a node in the root of the policy directory that matches the
+stm_source device's name (for example, "console"), this node will be
+used to allocate master and channel numbers. If there's no such policy
+node, the stm core will pick the first contiguous chunk of channels
+within the first available master. Note that the node must exist
+before the stm_source device is connected to its stm device.
+
+stm_console
+===========
+
+One implementation of this interface also used in the example above is
+the "stm_console" driver, which basically provides a one-way console
+for kernel messages over an stm device.
+
+To configure the master/channel pair that will be assigned to this
+console in the STP stream, create a "console" policy entry (see the
+beginning of this text on how to do that). When initialized, it will
+consume one channel.
 
 [1] https://software.intel.com/sites/default/files/managed/d3/3c/intel-th-developer-manual.pdf
 [2] http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0444b/index.html
diff --git a/Documentation/translations/zh_CN/sparse.txt b/Documentation/translations/zh_CN/sparse.txt
index cc144e581515..e41dc940e162 100644
--- a/Documentation/translations/zh_CN/sparse.txt
+++ b/Documentation/translations/zh_CN/sparse.txt
@@ -92,9 +92,4 @@ DaveJ 把每小时自动生成的 git 源码树 tar 包放在以下地址：
 如果你已经编译了内核，用后一种方式可以很快地检查整个源码树。
 
 make 的可选变量 CHECKFLAGS 可以用来向 sparse 工具传递参数。编译系统会自
-动向 sparse 工具传递 -Wbitwise 参数。你可以定义 __CHECK_ENDIAN__ 来进行
-大小尾检查。
-
-	make C=2 CHECKFLAGS="-D__CHECK_ENDIAN__"
-
-这些检查默认都是被关闭的，因为他们通常会产生大量的警告。
+动向 sparse 工具传递 -Wbitwise 参数。
diff --git a/Documentation/vfio-mediated-device.txt b/Documentation/vfio-mediated-device.txt
new file mode 100644
index 000000000000..b38afec35edc
--- /dev/null
+++ b/Documentation/vfio-mediated-device.txt
@@ -0,0 +1,398 @@
+/*
+ * VFIO Mediated devices
+ *
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ *     Author: Neo Jia <cjia@nvidia.com>
+ *             Kirti Wankhede <kwankhede@nvidia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+Virtual Function I/O (VFIO) Mediated devices[1]
+===============================================
+
+The number of use cases for virtualizing DMA devices that do not have built-in
+SR_IOV capability is increasing. Previously, to virtualize such devices,
+developers had to create their own management interfaces and APIs, and then
+integrate them with user space software. To simplify integration with user space
+software, we have identified common requirements and a unified management
+interface for such devices.
+
+The VFIO driver framework provides unified APIs for direct device access. It is
+an IOMMU/device-agnostic framework for exposing direct device access to user
+space in a secure, IOMMU-protected environment. This framework is used for
+multiple devices, such as GPUs, network adapters, and compute accelerators. With
+direct device access, virtual machines or user space applications have direct
+access to the physical device. This framework is reused for mediated devices.
+
+The mediated core driver provides a common interface for mediated device
+management that can be used by drivers of different devices. This module
+provides a generic interface to perform these operations:
+
+* Create and destroy a mediated device
+* Add a mediated device to and remove it from a mediated bus driver
+* Add a mediated device to and remove it from an IOMMU group
+
+The mediated core driver also provides an interface to register a bus driver.
+For example, the mediated VFIO mdev driver is designed for mediated devices and
+supports VFIO APIs. The mediated bus driver adds a mediated device to and
+removes it from a VFIO group.
+
+The following high-level block diagram shows the main components and interfaces
+in the VFIO mediated driver framework. The diagram shows NVIDIA, Intel, and IBM
+devices as examples, as these devices are the first devices to use this module.
+
+     +---------------+
+     |               |
+     | +-----------+ |  mdev_register_driver() +--------------+
+     | |           | +<------------------------+              |
+     | |  mdev     | |                         |              |
+     | |  bus      | +------------------------>+ vfio_mdev.ko |<-> VFIO user
+     | |  driver   | |     probe()/remove()    |              |    APIs
+     | |           | |                         +--------------+
+     | +-----------+ |
+     |               |
+     |  MDEV CORE    |
+     |   MODULE      |
+     |   mdev.ko     |
+     | +-----------+ |  mdev_register_device() +--------------+
+     | |           | +<------------------------+              |
+     | |           | |                         |  nvidia.ko   |<-> physical
+     | |           | +------------------------>+              |    device
+     | |           | |        callbacks        +--------------+
+     | | Physical  | |
+     | |  device   | |  mdev_register_device() +--------------+
+     | | interface | |<------------------------+              |
+     | |           | |                         |  i915.ko     |<-> physical
+     | |           | +------------------------>+              |    device
+     | |           | |        callbacks        +--------------+
+     | |           | |
+     | |           | |  mdev_register_device() +--------------+
+     | |           | +<------------------------+              |
+     | |           | |                         | ccw_device.ko|<-> physical
+     | |           | +------------------------>+              |    device
+     | |           | |        callbacks        +--------------+
+     | +-----------+ |
+     +---------------+
+
+
+Registration Interfaces
+=======================
+
+The mediated core driver provides the following types of registration
+interfaces:
+
+* Registration interface for a mediated bus driver
+* Physical device driver interface
+
+Registration Interface for a Mediated Bus Driver
+------------------------------------------------
+
+The registration interface for a mediated bus driver provides the following
+structure to represent a mediated device's driver:
+
+     /*
+      * struct mdev_driver [2] - Mediated device's driver
+      * @name: driver name
+      * @probe: called when new device created
+      * @remove: called when device removed
+      * @driver: device driver structure
+      */
+     struct mdev_driver {
+	     const char *name;
+	     int  (*probe)  (struct device *dev);
+	     void (*remove) (struct device *dev);
+	     struct device_driver    driver;
+     };
+
+A mediated bus driver for mdev should use this structure in the function calls
+to register and unregister itself with the core driver:
+
+* Register:
+
+  extern int  mdev_register_driver(struct mdev_driver *drv,
+				   struct module *owner);
+
+* Unregister:
+
+  extern void mdev_unregister_driver(struct mdev_driver *drv);
+
+The mediated bus driver is responsible for adding mediated devices to the VFIO
+group when devices are bound to the driver and removing mediated devices from
+the VFIO when devices are unbound from the driver.
+
+
+Physical Device Driver Interface
+--------------------------------
+
+The physical device driver interface provides the parent_ops[3] structure to
+define the APIs to manage work in the mediated core driver that is related to
+the physical device.
+
+The structures in the parent_ops structure are as follows:
+
+* dev_attr_groups: attributes of the parent device
+* mdev_attr_groups: attributes of the mediated device
+* supported_config: attributes to define supported configurations
+
+The functions in the parent_ops structure are as follows:
+
+* create: allocate basic resources in a driver for a mediated device
+* remove: free resources in a driver when a mediated device is destroyed
+
+The callbacks in the parent_ops structure are as follows:
+
+* open: open callback of mediated device
+* close: close callback of mediated device
+* ioctl: ioctl callback of mediated device
+* read : read emulation callback
+* write: write emulation callback
+* mmap: mmap emulation callback
+
+A driver should use the parent_ops structure in the function call to register
+itself with the mdev core driver:
+
+extern int  mdev_register_device(struct device *dev,
+                                 const struct parent_ops *ops);
+
+However, the parent_ops structure is not required in the function call that a
+driver should use to unregister itself with the mdev core driver:
+
+extern void mdev_unregister_device(struct device *dev);
+
+
+Mediated Device Management Interface Through sysfs
+==================================================
+
+The management interface through sysfs enables user space software, such as
+libvirt, to query and configure mediated devices in a hardware-agnostic fashion.
+This management interface provides flexibility to the underlying physical
+device's driver to support features such as:
+
+* Mediated device hot plug
+* Multiple mediated devices in a single virtual machine
+* Multiple mediated devices from different physical devices
+
+Links in the mdev_bus Class Directory
+-------------------------------------
+The /sys/class/mdev_bus/ directory contains links to devices that are registered
+with the mdev core driver.
+
+Directories and files under the sysfs for Each Physical Device
+--------------------------------------------------------------
+
+|- [parent physical device]
+|--- Vendor-specific-attributes [optional]
+|--- [mdev_supported_types]
+|     |--- [<type-id>]
+|     |   |--- create
+|     |   |--- name
+|     |   |--- available_instances
+|     |   |--- device_api
+|     |   |--- description
+|     |   |--- [devices]
+|     |--- [<type-id>]
+|     |   |--- create
+|     |   |--- name
+|     |   |--- available_instances
+|     |   |--- device_api
+|     |   |--- description
+|     |   |--- [devices]
+|     |--- [<type-id>]
+|          |--- create
+|          |--- name
+|          |--- available_instances
+|          |--- device_api
+|          |--- description
+|          |--- [devices]
+
+* [mdev_supported_types]
+
+  The list of currently supported mediated device types and their details.
+
+  [<type-id>], device_api, and available_instances are mandatory attributes
+  that should be provided by vendor driver.
+
+* [<type-id>]
+
+  The [<type-id>] name is created by adding the the device driver string as a
+  prefix to the string provided by the vendor driver. This format of this name
+  is as follows:
+
+	sprintf(buf, "%s-%s", dev_driver_string(parent->dev), group->name);
+
+* device_api
+
+  This attribute should show which device API is being created, for example,
+  "vfio-pci" for a PCI device.
+
+* available_instances
+
+  This attribute should show the number of devices of type <type-id> that can be
+  created.
+
+* [device]
+
+  This directory contains links to the devices of type <type-id> that have been
+created.
+
+* name
+
+  This attribute should show human readable name. This is optional attribute.
+
+* description
+
+  This attribute should show brief features/description of the type. This is
+  optional attribute.
+
+Directories and Files Under the sysfs for Each mdev Device
+----------------------------------------------------------
+
+|- [parent phy device]
+|--- [$MDEV_UUID]
+         |--- remove
+         |--- mdev_type {link to its type}
+         |--- vendor-specific-attributes [optional]
+
+* remove (write only)
+Writing '1' to the 'remove' file destroys the mdev device. The vendor driver can
+fail the remove() callback if that device is active and the vendor driver
+doesn't support hot unplug.
+
+Example:
+	# echo 1 > /sys/bus/mdev/devices/$mdev_UUID/remove
+
+Mediated device Hot plug:
+------------------------
+
+Mediated devices can be created and assigned at runtime. The procedure to hot
+plug a mediated device is the same as the procedure to hot plug a PCI device.
+
+Translation APIs for Mediated Devices
+=====================================
+
+The following APIs are provided for translating user pfn to host pfn in a VFIO
+driver:
+
+extern int vfio_pin_pages(struct device *dev, unsigned long *user_pfn,
+                          int npage, int prot, unsigned long *phys_pfn);
+
+extern int vfio_unpin_pages(struct device *dev, unsigned long *user_pfn,
+                            int npage);
+
+These functions call back into the back-end IOMMU module by using the pin_pages
+and unpin_pages callbacks of the struct vfio_iommu_driver_ops[4]. Currently
+these callbacks are supported in the TYPE1 IOMMU module. To enable them for
+other IOMMU backend modules, such as PPC64 sPAPR module, they need to provide
+these two callback functions.
+
+Using the Sample Code
+=====================
+
+mtty.c in samples/vfio-mdev/ directory is a sample driver program to
+demonstrate how to use the mediated device framework.
+
+The sample driver creates an mdev device that simulates a serial port over a PCI
+card.
+
+1. Build and load the mtty.ko module.
+
+   This step creates a dummy device, /sys/devices/virtual/mtty/mtty/
+
+   Files in this device directory in sysfs are similar to the following:
+
+   # tree /sys/devices/virtual/mtty/mtty/
+      /sys/devices/virtual/mtty/mtty/
+      |-- mdev_supported_types
+      |   |-- mtty-1
+      |   |   |-- available_instances
+      |   |   |-- create
+      |   |   |-- device_api
+      |   |   |-- devices
+      |   |   `-- name
+      |   `-- mtty-2
+      |       |-- available_instances
+      |       |-- create
+      |       |-- device_api
+      |       |-- devices
+      |       `-- name
+      |-- mtty_dev
+      |   `-- sample_mtty_dev
+      |-- power
+      |   |-- autosuspend_delay_ms
+      |   |-- control
+      |   |-- runtime_active_time
+      |   |-- runtime_status
+      |   `-- runtime_suspended_time
+      |-- subsystem -> ../../../../class/mtty
+      `-- uevent
+
+2. Create a mediated device by using the dummy device that you created in the
+   previous step.
+
+   # echo "83b8f4f2-509f-382f-3c1e-e6bfe0fa1001" >	\
+              /sys/devices/virtual/mtty/mtty/mdev_supported_types/mtty-2/create
+
+3. Add parameters to qemu-kvm.
+
+   -device vfio-pci,\
+    sysfsdev=/sys/bus/mdev/devices/83b8f4f2-509f-382f-3c1e-e6bfe0fa1001
+
+4. Boot the VM.
+
+   In the Linux guest VM, with no hardware on the host, the device appears
+   as  follows:
+
+   # lspci -s 00:05.0 -xxvv
+   00:05.0 Serial controller: Device 4348:3253 (rev 10) (prog-if 02 [16550])
+           Subsystem: Device 4348:3253
+           Physical Slot: 5
+           Control: I/O+ Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr-
+   Stepping- SERR- FastB2B- DisINTx-
+           Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=medium >TAbort-
+   <TAbort- <MAbort- >SERR- <PERR- INTx-
+           Interrupt: pin A routed to IRQ 10
+           Region 0: I/O ports at c150 [size=8]
+           Region 1: I/O ports at c158 [size=8]
+           Kernel driver in use: serial
+   00: 48 43 53 32 01 00 00 02 10 02 00 07 00 00 00 00
+   10: 51 c1 00 00 59 c1 00 00 00 00 00 00 00 00 00 00
+   20: 00 00 00 00 00 00 00 00 00 00 00 00 48 43 53 32
+   30: 00 00 00 00 00 00 00 00 00 00 00 00 0a 01 00 00
+
+   In the Linux guest VM, dmesg output for the device is as follows:
+
+   serial 0000:00:05.0: PCI INT A -> Link[LNKA] -> GSI 10 (level, high) -> IRQ
+10
+   0000:00:05.0: ttyS1 at I/O 0xc150 (irq = 10) is a 16550A
+   0000:00:05.0: ttyS2 at I/O 0xc158 (irq = 10) is a 16550A
+
+
+5. In the Linux guest VM, check the serial ports.
+
+   # setserial -g /dev/ttyS*
+   /dev/ttyS0, UART: 16550A, Port: 0x03f8, IRQ: 4
+   /dev/ttyS1, UART: 16550A, Port: 0xc150, IRQ: 10
+   /dev/ttyS2, UART: 16550A, Port: 0xc158, IRQ: 10
+
+6. Using a minicom or any terminal enulation program, open port /dev/ttyS1 or
+   /dev/ttyS2 with hardware flow control disabled.
+
+7. Type data on the minicom terminal or send data to the terminal emulation
+   program and read the data.
+
+   Data is loop backed from hosts mtty driver.
+
+8. Destroy the mediated device that you created.
+
+   # echo 1 > /sys/bus/mdev/devices/83b8f4f2-509f-382f-3c1e-e6bfe0fa1001/remove
+
+References
+==========
+
+[1] See Documentation/vfio.txt for more information on VFIO.
+[2] struct mdev_driver in include/linux/mdev.h
+[3] struct parent_ops in include/linux/mdev.h
+[4] struct vfio_iommu_driver_ops in include/linux/vfio.h
diff --git a/Documentation/virtual/kvm/00-INDEX b/Documentation/virtual/kvm/00-INDEX
index fee9f2bf9c64..69fe1a8b7ad1 100644
--- a/Documentation/virtual/kvm/00-INDEX
+++ b/Documentation/virtual/kvm/00-INDEX
@@ -6,6 +6,8 @@ cpuid.txt
 	- KVM-specific cpuid leaves (x86).
 devices/
 	- KVM_CAP_DEVICE_CTRL userspace API.
+halt-polling.txt
+	- notes on halt-polling
 hypercalls.txt
 	- KVM hypercalls.
 locking.txt
diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
index 6bbceb9a3a19..03145b7cafaa 100644
--- a/Documentation/virtual/kvm/api.txt
+++ b/Documentation/virtual/kvm/api.txt
@@ -2034,6 +2034,8 @@ registers, find a list below:
   PPC   | KVM_REG_PPC_WORT              | 64
   PPC	| KVM_REG_PPC_SPRG9             | 64
   PPC	| KVM_REG_PPC_DBSR              | 32
+  PPC   | KVM_REG_PPC_TIDR              | 64
+  PPC   | KVM_REG_PPC_PSSCR             | 64
   PPC   | KVM_REG_PPC_TM_GPR0           | 64
           ...
   PPC   | KVM_REG_PPC_TM_GPR31          | 64
@@ -2050,6 +2052,7 @@ registers, find a list below:
   PPC   | KVM_REG_PPC_TM_VSCR           | 32
   PPC   | KVM_REG_PPC_TM_DSCR           | 64
   PPC   | KVM_REG_PPC_TM_TAR            | 64
+  PPC   | KVM_REG_PPC_TM_XER            | 64
         |                               |
   MIPS  | KVM_REG_MIPS_R0               | 64
           ...
@@ -2209,7 +2212,7 @@ after pausing the vcpu, but before it is resumed.
 4.71 KVM_SIGNAL_MSI
 
 Capability: KVM_CAP_SIGNAL_MSI
-Architectures: x86 arm64
+Architectures: x86 arm arm64
 Type: vm ioctl
 Parameters: struct kvm_msi (in)
 Returns: >0 on delivery, 0 if guest blocked the MSI, and -1 on error
diff --git a/Documentation/virtual/kvm/halt-polling.txt b/Documentation/virtual/kvm/halt-polling.txt
new file mode 100644
index 000000000000..4a8418318769
--- /dev/null
+++ b/Documentation/virtual/kvm/halt-polling.txt
@@ -0,0 +1,127 @@
+The KVM halt polling system
+===========================
+
+The KVM halt polling system provides a feature within KVM whereby the latency
+of a guest can, under some circumstances, be reduced by polling in the host
+for some time period after the guest has elected to no longer run by cedeing.
+That is, when a guest vcpu has ceded, or in the case of powerpc when all of the
+vcpus of a single vcore have ceded, the host kernel polls for wakeup conditions
+before giving up the cpu to the scheduler in order to let something else run.
+
+Polling provides a latency advantage in cases where the guest can be run again
+very quickly by at least saving us a trip through the scheduler, normally on
+the order of a few micro-seconds, although performance benefits are workload
+dependant. In the event that no wakeup source arrives during the polling
+interval or some other task on the runqueue is runnable the scheduler is
+invoked. Thus halt polling is especially useful on workloads with very short
+wakeup periods where the time spent halt polling is minimised and the time
+savings of not invoking the scheduler are distinguishable.
+
+The generic halt polling code is implemented in:
+
+	virt/kvm/kvm_main.c: kvm_vcpu_block()
+
+The powerpc kvm-hv specific case is implemented in:
+
+	arch/powerpc/kvm/book3s_hv.c: kvmppc_vcore_blocked()
+
+Halt Polling Interval
+=====================
+
+The maximum time for which to poll before invoking the scheduler, referred to
+as the halt polling interval, is increased and decreased based on the perceived
+effectiveness of the polling in an attempt to limit pointless polling.
+This value is stored in either the vcpu struct:
+
+	kvm_vcpu->halt_poll_ns
+
+or in the case of powerpc kvm-hv, in the vcore struct:
+
+	kvmppc_vcore->halt_poll_ns
+
+Thus this is a per vcpu (or vcore) value.
+
+During polling if a wakeup source is received within the halt polling interval,
+the interval is left unchanged. In the event that a wakeup source isn't
+received during the polling interval (and thus schedule is invoked) there are
+two options, either the polling interval and total block time[0] were less than
+the global max polling interval (see module params below), or the total block
+time was greater than the global max polling interval.
+
+In the event that both the polling interval and total block time were less than
+the global max polling interval then the polling interval can be increased in
+the hope that next time during the longer polling interval the wake up source
+will be received while the host is polling and the latency benefits will be
+received. The polling interval is grown in the function grow_halt_poll_ns() and
+is multiplied by the module parameter halt_poll_ns_grow.
+
+In the event that the total block time was greater than the global max polling
+interval then the host will never poll for long enough (limited by the global
+max) to wakeup during the polling interval so it may as well be shrunk in order
+to avoid pointless polling. The polling interval is shrunk in the function
+shrink_halt_poll_ns() and is divided by the module parameter
+halt_poll_ns_shrink, or set to 0 iff halt_poll_ns_shrink == 0.
+
+It is worth noting that this adjustment process attempts to hone in on some
+steady state polling interval but will only really do a good job for wakeups
+which come at an approximately constant rate, otherwise there will be constant
+adjustment of the polling interval.
+
+[0] total block time: the time between when the halt polling function is
+		      invoked and a wakeup source received (irrespective of
+		      whether the scheduler is invoked within that function).
+
+Module Parameters
+=================
+
+The kvm module has 3 tuneable module parameters to adjust the global max
+polling interval as well as the rate at which the polling interval is grown and
+shrunk. These variables are defined in include/linux/kvm_host.h and as module
+parameters in virt/kvm/kvm_main.c, or arch/powerpc/kvm/book3s_hv.c in the
+powerpc kvm-hv case.
+
+Module Parameter    |	     Description	      |	     Default Value
+--------------------------------------------------------------------------------
+halt_poll_ns	    | The global max polling interval | KVM_HALT_POLL_NS_DEFAULT
+		    | which defines the ceiling value |
+		    | of the polling interval for     | (per arch value)
+		    | each vcpu. 		      |
+--------------------------------------------------------------------------------
+halt_poll_ns_grow   | The value by which the halt     |	2
+		    | polling interval is multiplied  |
+		    | in the grow_halt_poll_ns()      |
+		    | function.			      |
+--------------------------------------------------------------------------------
+halt_poll_ns_shrink | The value by which the halt     |	0
+		    | polling interval is divided in  |
+		    | the shrink_halt_poll_ns()	      |
+		    | function.			      |
+--------------------------------------------------------------------------------
+
+These module parameters can be set from the debugfs files in:
+
+	/sys/module/kvm/parameters/
+
+Note: that these module parameters are system wide values and are not able to
+      be tuned on a per vm basis.
+
+Further Notes
+=============
+
+- Care should be taken when setting the halt_poll_ns module parameter as a
+large value has the potential to drive the cpu usage to 100% on a machine which
+would be almost entirely idle otherwise. This is because even if a guest has
+wakeups during which very little work is done and which are quite far apart, if
+the period is shorter than the global max polling interval (halt_poll_ns) then
+the host will always poll for the entire block time and thus cpu utilisation
+will go to 100%.
+
+- Halt polling essentially presents a trade off between power usage and latency
+and the module parameters should be used to tune the affinity for this. Idle
+cpu time is essentially converted to host kernel time with the aim of decreasing
+latency when entering the guest.
+
+- Halt polling will only be conducted by the host when no other tasks are
+runnable on that cpu, otherwise the polling will cease immediately and
+schedule will be invoked to allow that other task to run. Thus this doesn't
+allow a guest to denial of service the cpu.
diff --git a/MAINTAINERS b/MAINTAINERS
index 592fdbaa283e..6a671ce87762 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -260,6 +260,12 @@ L:	linux-gpio@vger.kernel.org
 S:	Maintained
 F:	drivers/gpio/gpio-104-idio-16.c
 
+ACCES 104-QUAD-8 IIO DRIVER
+M:	William Breathitt Gray <vilhelm.gray@gmail.com>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	drivers/iio/counter/104-quad-8.c
+
 ACENIC DRIVER
 M:	Jes Sorensen <jes@trained-monkey.org>
 L:	linux-acenic@sunsite.dk
@@ -534,6 +540,7 @@ S:	Supported
 F:	fs/afs/
 F:	include/net/af_rxrpc.h
 F:	net/rxrpc/af_rxrpc.c
+W:	https://www.infradead.org/~dhowells/kafs/
 
 AGPGART DRIVER
 M:	David Airlie <airlied@linux.ie>
@@ -803,7 +810,7 @@ S:	Supported
 F:	drivers/iio/*/ad*
 X:	drivers/iio/*/adjd*
 F:	drivers/staging/iio/*/ad*
-F:	staging/iio/trigger/iio-trig-bfin-timer.c
+F:	drivers/staging/iio/trigger/iio-trig-bfin-timer.c
 
 ANALOG DEVICES INC DMA DRIVERS
 M:	Lars-Peter Clausen <lars@metafoo.de>
@@ -1035,6 +1042,7 @@ L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
 N:	sun[x456789]i
 F:	arch/arm/boot/dts/ntc-gr8*
+F:	arch/arm64/boot/dts/allwinner/
 
 ARM/Allwinner SoC Clock Support
 M:	Emilio López <emilio@elopez.com.ar>
@@ -1052,6 +1060,7 @@ F:	arch/arm/mach-meson/
 F:	arch/arm/boot/dts/meson*
 F:	arch/arm64/boot/dts/amlogic/
 F: 	drivers/pinctrl/meson/
+F:	drivers/mmc/host/meson*
 N:	meson
 
 ARM/Annapurna Labs ALPINE ARCHITECTURE
@@ -1495,8 +1504,9 @@ L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 L:	linux-oxnas@lists.tuxfamily.org (moderated for non-subscribers)
 S:	Maintained
 F:	arch/arm/mach-oxnas/
-F:	arch/arm/boot/dts/oxnas*
+F:	arch/arm/boot/dts/ox8*.dtsi
 F:	arch/arm/boot/dts/wd-mbwe.dts
+F:	arch/arm/boot/dts/cloudengines-pogoplug-series-3.dts
 N:	oxnas
 
 ARM/Mediatek RTC DRIVER
@@ -1617,6 +1627,7 @@ F:	arch/arm/mach-qcom/
 F:	arch/arm64/boot/dts/qcom/*
 F:	drivers/i2c/busses/i2c-qup.c
 F:	drivers/clk/qcom/
+F:	drivers/pinctrl/qcom/
 F:	drivers/soc/qcom/
 F:	drivers/spi/spi-qup.c
 F:	drivers/tty/serial/msm_serial.h
@@ -1791,16 +1802,16 @@ F:	drivers/char/hw_random/st-rng.c
 F:	drivers/clocksource/arm_global_timer.c
 F:	drivers/clocksource/clksrc_st_lpc.c
 F:	drivers/cpufreq/sti-cpufreq.c
+F:	drivers/dma/st_fdma*
 F:	drivers/i2c/busses/i2c-st.c
 F:	drivers/media/rc/st_rc.c
 F:	drivers/media/platform/sti/c8sectpfe/
 F:	drivers/mmc/host/sdhci-st.c
 F:	drivers/phy/phy-miphy28lp.c
-F:	drivers/phy/phy-miphy365x.c
 F:	drivers/phy/phy-stih407-usb.c
-F:	drivers/phy/phy-stih41x-usb.c
 F:	drivers/pinctrl/pinctrl-st.c
 F:	drivers/remoteproc/st_remoteproc.c
+F:	drivers/remoteproc/st_slim_rproc.c
 F:	drivers/reset/sti/
 F:	drivers/rtc/rtc-st-lpc.c
 F:	drivers/tty/serial/st-asc.c
@@ -1809,6 +1820,7 @@ F:	drivers/usb/host/ehci-st.c
 F:	drivers/usb/host/ohci-st.c
 F:	drivers/watchdog/st_lpc_wdt.c
 F:	drivers/ata/ahci_st.c
+F:	include/linux/remoteproc/st_slim_rproc.h
 
 ARM/STM32 ARCHITECTURE
 M:	Maxime Coquelin <mcoquelin.stm32@gmail.com>
@@ -2334,6 +2346,13 @@ F:	include/uapi/linux/ax25.h
 F:	include/net/ax25.h
 F:	net/ax25/
 
+AXENTIA ASOC DRIVERS
+M:	Peter Rosin <peda@axentia.se>
+L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
+S:	Maintained
+F:	Documentation/devicetree/bindings/sound/axentia,*
+F:	sound/soc/atmel/tse850-pcm5142.c
+
 AZ6007 DVB DRIVER
 M:	Mauro Carvalho Chehab <mchehab@s-opensource.com>
 M:	Mauro Carvalho Chehab <mchehab@kernel.org>
@@ -2608,6 +2627,7 @@ L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/rpi/linux-rpi.git
 S:	Maintained
 N:	bcm2835
+F:	drivers/staging/vc04_services
 
 BROADCOM BCM47XX MIPS ARCHITECTURE
 M:	Hauke Mehrtens <hauke@hauke-m.de>
@@ -2760,6 +2780,14 @@ L:	bcm-kernel-feedback-list@broadcom.com
 S:	Maintained
 F:	drivers/mtd/nand/brcmnand/
 
+BROADCOM STB AVS CPUFREQ DRIVER
+M:	Markus Mayer <mmayer@broadcom.com>
+M:	bcm-kernel-feedback-list@broadcom.com
+L:	linux-pm@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/cpufreq/brcm,stb-avs-cpu-freq.txt
+F:	drivers/cpufreq/brcmstb*
+
 BROADCOM SPECIFIC AMBA DRIVER (BCMA)
 M:	Rafał Miłecki <zajec5@gmail.com>
 L:	linux-wireless@vger.kernel.org
@@ -2774,7 +2802,7 @@ S:	Supported
 F:	drivers/net/ethernet/broadcom/bcmsysport.*
 
 BROADCOM VULCAN ARM64 SOC
-M:	Jayachandran C. <jchandra@broadcom.com>
+M:	Jayachandran C. <c.jayachandran@gmail.com>
 M:	bcm-kernel-feedback-list@broadcom.com
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
@@ -3048,6 +3076,12 @@ F:	drivers/usb/host/whci/
 F:	drivers/usb/wusbcore/
 F:	include/linux/usb/wusb*
 
+HT16K33 LED CONTROLLER DRIVER
+M:	Robin van der Gracht <robin@protonic.nl>
+S:	Maintained
+F:	drivers/auxdisplay/ht16k33.c
+F:	Documentation/devicetree/bindings/display/ht16k33.txt
+
 CFAG12864B LCD DRIVER
 M:	Miguel Ojeda Sandonis <miguel.ojeda.sandonis@gmail.com>
 W:	http://miguelojeda.es/auxdisplay.htm
@@ -3167,15 +3201,15 @@ S:	Supported
 F:	drivers/clocksource
 
 CISCO FCOE HBA DRIVER
-M:	Hiral Patel <hiralpat@cisco.com>
-M:	Suma Ramars <sramars@cisco.com>
-M:	Brian Uchino <buchino@cisco.com>
+M:	Satish Kharat <satishkh@cisco.com>
+M:	Sesidhar Baddela <sebaddel@cisco.com>
+M:	Karan Tilak Kumar <kartilak@cisco.com>
 L:	linux-scsi@vger.kernel.org
 S:	Supported
 F:	drivers/scsi/fnic/
 
 CISCO SCSI HBA DRIVER
-M:	Narsimhulu Musini <nmusini@cisco.com>
+M:	Karan Tilak Kumar <kartilak@cisco.com>
 M:	Sesidhar Baddela <sebaddel@cisco.com>
 L:	linux-scsi@vger.kernel.org
 S:	Supported
@@ -3352,6 +3386,7 @@ L:	linux-pm@vger.kernel.org
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git
 T:	git git://git.linaro.org/people/vireshk/linux.git (For ARM Updates)
+B:	https://bugzilla.kernel.org
 F:	Documentation/cpu-freq/
 F:	drivers/cpufreq/
 F:	include/linux/cpufreq.h
@@ -3391,6 +3426,7 @@ M:	Daniel Lezcano <daniel.lezcano@linaro.org>
 L:	linux-pm@vger.kernel.org
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git
+B:	https://bugzilla.kernel.org
 F:	drivers/cpuidle/*
 F:	include/linux/cpuidle.h
 
@@ -3436,6 +3472,7 @@ F:	arch/*/crypto/
 F:	crypto/
 F:	drivers/crypto/
 F:	include/crypto/
+F:	include/linux/crypto*
 
 CRYPTOGRAPHIC RANDOM NUMBER GENERATOR
 M:	Neil Horman <nhorman@tuxdriver.com>
@@ -3929,7 +3966,7 @@ F:	include/linux/dma-buf*
 F:	include/linux/reservation.h
 F:	include/linux/*fence.h
 F:	Documentation/dma-buf-sharing.txt
-T:	git git://git.linaro.org/people/sumitsemwal/linux-dma-buf.git
+T:	git git://anongit.freedesktop.org/drm/drm-misc
 
 SYNC FILE FRAMEWORK
 M:	Sumit Semwal <sumit.semwal@linaro.org>
@@ -3937,10 +3974,12 @@ R:	Gustavo Padovan <gustavo@padovan.org>
 S:	Maintained
 L:	linux-media@vger.kernel.org
 L:	dri-devel@lists.freedesktop.org
-F:	drivers/dma-buf/sync_file.c
+F:	drivers/dma-buf/sync_*
+F:	drivers/dma-buf/sw_sync.c
 F:	include/linux/sync_file.h
+F:	include/uapi/linux/sync_file.h
 F:	Documentation/sync_file.txt
-T:	git git://git.linaro.org/people/sumitsemwal/linux-dma-buf.git
+T:	git git://anongit.freedesktop.org/drm/drm-misc
 
 DMA GENERIC OFFLOAD ENGINE SUBSYSTEM
 M:	Vinod Koul <vinod.koul@intel.com>
@@ -4040,11 +4079,30 @@ F:	Documentation/gpu/
 F:	include/drm/
 F:	include/uapi/drm/
 
+DRM DRIVERS AND MISC GPU PATCHES
+M:	Daniel Vetter <daniel.vetter@intel.com>
+M:	Jani Nikula <jani.nikula@linux.intel.com>
+M:	Sean Paul <seanpaul@chromium.org>
+W:	https://01.org/linuxgraphics/gfx-docs/maintainer-tools/drm-misc.html
+S:	Maintained
+T:	git git://anongit.freedesktop.org/drm/drm-misc
+F:	Documentation/gpu/
+F:	drivers/gpu/vga/
+F:	drivers/gpu/drm/*
+F:	include/drm/drm*
+F:	include/uapi/drm/drm*
+
 DRM DRIVER FOR AST SERVER GRAPHICS CHIPS
 M:	Dave Airlie <airlied@redhat.com>
 S:	Odd Fixes
 F:	drivers/gpu/drm/ast/
 
+DRM DRIVERS FOR BRIDGE CHIPS
+M:	Archit Taneja <architt@codeaurora.org>
+S:	Maintained
+T:	git git://anongit.freedesktop.org/drm/drm-misc
+F:	drivers/gpu/drm/bridge/
+
 DRM DRIVER FOR BOCHS VIRTUAL GPU
 M:	Gerd Hoffmann <kraxel@redhat.com>
 S:	Odd Fixes
@@ -4080,7 +4138,6 @@ INTEL DRM DRIVERS (excluding Poulsbo, Moorestown and derivative chipsets)
 M:	Daniel Vetter <daniel.vetter@intel.com>
 M:	Jani Nikula <jani.nikula@linux.intel.com>
 L:	intel-gfx@lists.freedesktop.org
-L:	dri-devel@lists.freedesktop.org
 W:	https://01.org/linuxgraphics/
 B:	https://01.org/linuxgraphics/documentation/how-report-bugs
 C:	irc://chat.freenode.net/intel-gfx
@@ -4092,6 +4149,16 @@ F:	include/drm/i915*
 F:	include/uapi/drm/i915_drm.h
 F:	Documentation/gpu/i915.rst
 
+INTEL GVT-g DRIVERS (Intel GPU Virtualization)
+M:      Zhenyu Wang <zhenyuw@linux.intel.com>
+M:      Zhi Wang <zhi.a.wang@intel.com>
+L:      igvt-g-dev@lists.01.org
+L:      intel-gfx@lists.freedesktop.org
+W:      https://01.org/igvt-g
+T:      git https://github.com/01org/gvt-linux.git
+S:      Supported
+F:      drivers/gpu/drm/i915/gvt/
+
 DRM DRIVERS FOR ATMEL HLCDC
 M:	Boris Brezillon <boris.brezillon@free-electrons.com>
 L:	dri-devel@lists.freedesktop.org
@@ -4106,6 +4173,15 @@ S:	Supported
 F:	drivers/gpu/drm/sun4i/
 F:	Documentation/devicetree/bindings/display/sunxi/sun4i-drm.txt
 
+DRM DRIVERS FOR AMLOGIC SOCS
+M:	Neil Armstrong <narmstrong@baylibre.com>
+L:	dri-devel@lists.freedesktop.org
+L:	linux-amlogic@lists.infradead.org
+W:	http://linux-meson.com/
+S:	Supported
+F:	drivers/gpu/drm/meson/
+F:	Documentation/devicetree/bindings/display/amlogic,meson-vpu.txt
+
 DRM DRIVERS FOR EXYNOS
 M:	Inki Dae <inki.dae@samsung.com>
 M:	Joonyoung Shim <jy0922.shim@samsung.com>
@@ -4145,6 +4221,7 @@ F:	drivers/gpu/drm/gma500/
 
 DRM DRIVERS FOR HISILICON
 M:	Xinliang Liu <z.liuxinliang@hisilicon.com>
+M:	Rongrong Zou <zourongrong@gmail.com>
 R:	Xinwei Kong <kong.kongxinwei@hisilicon.com>
 R:	Chen Feng <puck.chen@hisilicon.com>
 L:	dri-devel@lists.freedesktop.org
@@ -4269,6 +4346,7 @@ DRM DRIVERS FOR VIVANTE GPU IP
 M:	Lucas Stach <l.stach@pengutronix.de>
 R:	Russell King <linux+etnaviv@armlinux.org.uk>
 R:	Christian Gmeiner <christian.gmeiner@gmail.com>
+L:	etnaviv@lists.freedesktop.org
 L:	dri-devel@lists.freedesktop.org
 S:	Maintained
 F:	drivers/gpu/drm/etnaviv/
@@ -4309,6 +4387,13 @@ S:	Maintained
 F:	drivers/gpu/drm/tilcdc/
 F:	Documentation/devicetree/bindings/display/tilcdc/
 
+DRM DRIVERS FOR ZTE ZX
+M:	Shawn Guo <shawnguo@kernel.org>
+L:	dri-devel@lists.freedesktop.org
+S:	Maintained
+F:	drivers/gpu/drm/zte/
+F:	Documentation/devicetree/bindings/display/zte,vou.txt
+
 DSBR100 USB FM RADIO DRIVER
 M:	Alexey Klimov <klimov.linux@gmail.com>
 L:	linux-media@vger.kernel.org
@@ -4503,7 +4588,8 @@ L:	linux-edac@vger.kernel.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/bp/bp.git for-next
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/mchehab/linux-edac.git linux_next
 S:	Supported
-F:	Documentation/edac.txt
+F:	Documentation/admin-guide/ras.rst
+F:	Documentation/driver-api/edac.rst
 F:	drivers/edac/
 F:	include/linux/edac.h
 
@@ -4712,11 +4798,11 @@ M:	David Woodhouse <dwmw2@infradead.org>
 L:	linux-embedded@vger.kernel.org
 S:	Maintained
 
-EMULEX/AVAGO LPFC FC/FCOE SCSI DRIVER
-M:	James Smart <james.smart@avagotech.com>
-M:	Dick Kennedy <dick.kennedy@avagotech.com>
+EMULEX/BROADCOM LPFC FC/FCOE SCSI DRIVER
+M:	James Smart <james.smart@broadcom.com>
+M:	Dick Kennedy <dick.kennedy@broadcom.com>
 L:	linux-scsi@vger.kernel.org
-W:	http://www.avagotech.com
+W:	http://www.broadcom.com
 S:	Supported
 F:	drivers/scsi/lpfc/
 
@@ -4974,7 +5060,9 @@ K:	fmc_d.*register
 FPGA MANAGER FRAMEWORK
 M:	Alan Tull <atull@opensource.altera.com>
 R:	Moritz Fischer <moritz.fischer@ettus.com>
+L:	linux-fpga@vger.kernel.org
 S:	Maintained
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/atull/linux-fpga.git
 F:	drivers/fpga/
 F:	include/linux/fpga/fpga-mgr.h
 W:	http://www.rocketboards.org
@@ -4992,10 +5080,9 @@ F:	drivers/net/wan/dlci.c
 F:	drivers/net/wan/sdla.c
 
 FRAMEBUFFER LAYER
-M:	Tomi Valkeinen <tomi.valkeinen@ti.com>
 L:	linux-fbdev@vger.kernel.org
 Q:	http://patchwork.kernel.org/project/linux-fbdev/list/
-S:	Maintained
+S:	Orphan
 F:	Documentation/fb/
 F:	drivers/video/
 F:	include/video/
@@ -5003,6 +5090,14 @@ F:	include/linux/fb.h
 F:	include/uapi/video/
 F:	include/uapi/linux/fb.h
 
+FREESCALE CAAM (Cryptographic Acceleration and Assurance Module) DRIVER
+M:	Horia Geantă <horia.geanta@nxp.com>
+M:	Dan Douglass <dan.douglass@nxp.com>
+L:	linux-crypto@vger.kernel.org
+S:	Maintained
+F:	drivers/crypto/caam/
+F:	Documentation/devicetree/bindings/crypto/fsl-sec4.txt
+
 FREESCALE DIU FRAMEBUFFER DRIVER
 M:	Timur Tabi <timur@tabi.org>
 L:	linux-fbdev@vger.kernel.org
@@ -5068,9 +5163,18 @@ S:	Maintained
 F:	drivers/net/ethernet/freescale/fman
 F:	Documentation/devicetree/bindings/powerpc/fsl/fman.txt
 
+FREESCALE SOC DRIVERS
+M:	Scott Wood <oss@buserror.net>
+L:	linuxppc-dev@lists.ozlabs.org
+L:	linux-arm-kernel@lists.infradead.org
+S:	Maintained
+F:	drivers/soc/fsl/
+F:	include/linux/fsl/
+
 FREESCALE QUICC ENGINE LIBRARY
+M:	Qiang Zhao <qiang.zhao@nxp.com>
 L:	linuxppc-dev@lists.ozlabs.org
-S:	Orphan
+S:	Maintained
 F:	drivers/soc/fsl/qe/
 F:	include/soc/fsl/*qe*.h
 F:	include/soc/fsl/*ucc*.h
@@ -5122,13 +5226,6 @@ F:	sound/soc/fsl/fsl*
 F:	sound/soc/fsl/imx*
 F:	sound/soc/fsl/mpc8610_hpcd.c
 
-FREESCALE QORIQ MANAGEMENT COMPLEX DRIVER
-M:	"J. German Rivera" <German.Rivera@freescale.com>
-M:	Stuart Yoder <stuart.yoder@nxp.com>
-L:	linux-kernel@vger.kernel.org
-S:	Maintained
-F:	drivers/staging/fsl-mc/
-
 FREEVXFS FILESYSTEM
 M:	Christoph Hellwig <hch@infradead.org>
 W:	ftp://ftp.openlinux.org/pub/people/hch/vxfs
@@ -5162,6 +5259,7 @@ F:	include/linux/fscache*.h
 FS-CRYPTO: FILE SYSTEM LEVEL ENCRYPTION SUPPORT
 M:	Theodore Y. Ts'o <tytso@mit.edu>
 M:	Jaegeuk Kim <jaegeuk@kernel.org>
+L:	linux-fsdevel@vger.kernel.org
 S:	Supported
 F:	fs/crypto/
 F:	include/linux/fscrypto.h
@@ -5226,6 +5324,7 @@ L:	kernel-hardening@lists.openwall.com
 S:	Maintained
 F:	scripts/gcc-plugins/
 F:	scripts/gcc-plugin.sh
+F:	scripts/Makefile.gcc-plugins
 F:	Documentation/gcc-plugins.txt
 
 GCOV BASED KERNEL PROFILING
@@ -5637,7 +5736,6 @@ F:	drivers/watchdog/hpwdt.c
 
 HEWLETT-PACKARD SMART ARRAY RAID DRIVER (hpsa)
 M:	Don Brace <don.brace@microsemi.com>
-L:	iss_storagedev@hp.com
 L:	esc.storagedev@microsemi.com
 L:	linux-scsi@vger.kernel.org
 S:	Supported
@@ -5648,7 +5746,6 @@ F:	include/uapi/linux/cciss*.h
 
 HEWLETT-PACKARD SMART CISS RAID DRIVER (cciss)
 M:	Don Brace <don.brace@microsemi.com>
-L:	iss_storagedev@hp.com
 L:	esc.storagedev@microsemi.com
 L:	linux-scsi@vger.kernel.org
 S:	Supported
@@ -5869,6 +5966,7 @@ F:	drivers/input/serio/hyperv-keyboard.c
 F:	drivers/pci/host/pci-hyperv.c
 F:	drivers/net/hyperv/
 F:	drivers/scsi/storvsc_drv.c
+F:	drivers/uio/uio_hv_generic.c
 F:	drivers/video/fbdev/hyperv_fb.c
 F:	include/linux/hyperv.h
 F:	tools/hv/
@@ -6144,6 +6242,22 @@ L:	linux-media@vger.kernel.org
 S:	Maintained
 F:	drivers/media/rc/iguanair.c
 
+IIO DIGITAL POTENTIOMETER DAC
+M:	Peter Rosin <peda@axentia.se>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	Documentation/ABI/testing/sysfs-bus-iio-dac-dpot-dac
+F:	Documentation/devicetree/bindings/iio/dac/dpot-dac.txt
+F:	drivers/iio/dac/dpot-dac.c
+
+IIO ENVELOPE DETECTOR
+M:	Peter Rosin <peda@axentia.se>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	Documentation/ABI/testing/sysfs-bus-iio-adc-envelope-detector
+F:	Documentation/devicetree/bindings/iio/adc/envelope-detector.txt
+F:	drivers/iio/adc/envelope-detector.c
+
 IIO SUBSYSTEM AND DRIVERS
 M:	Jonathan Cameron <jic23@kernel.org>
 R:	Hartmut Knaack <knaack.h@gmx.de>
@@ -6301,9 +6415,11 @@ S:	Maintained
 F:	drivers/platform/x86/intel-vbtn.c
 
 INTEL IDLE DRIVER
+M:	Jacob Pan <jacob.jun.pan@linux.intel.com>
 M:	Len Brown <lenb@kernel.org>
 L:	linux-pm@vger.kernel.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git
+B:	https://bugzilla.kernel.org
 S:	Supported
 F:	drivers/idle/intel_idle.c
 
@@ -6401,10 +6517,7 @@ F:	drivers/net/ethernet/intel/*/
 
 INTEL RDMA RNIC DRIVER
 M:     Faisal Latif <faisal.latif@intel.com>
-R:     Chien Tin Tung <chien.tin.tung@intel.com>
-R:     Mustafa Ismail <mustafa.ismail@intel.com>
-R:     Shiraz Saleem <shiraz.saleem@intel.com>
-R:     Tatyana Nikolova <tatyana.e.nikolova@intel.com>
+M:     Shiraz Saleem <shiraz.saleem@intel.com>
 L:     linux-rdma@vger.kernel.org
 S:     Supported
 F:     drivers/infiniband/hw/i40iw/
@@ -6523,6 +6636,13 @@ S:	Maintained
 F:	arch/x86/include/asm/pmc_core.h
 F:	drivers/platform/x86/intel_pmc_core*
 
+INVENSENSE MPU-3050 GYROSCOPE DRIVER
+M:	Linus Walleij <linus.walleij@linaro.org>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	drivers/iio/gyro/mpu3050*
+F:	Documentation/devicetree/bindings/iio/gyroscope/inv,mpu3050.txt
+
 IOC3 ETHERNET DRIVER
 M:	Ralf Baechle <ralf@linux-mips.org>
 L:	linux-mips@linux-mips.org
@@ -7730,6 +7850,7 @@ MCP4531 MICROCHIP DIGITAL POTENTIOMETER DRIVER
 M:	Peter Rosin <peda@axentia.se>
 L:	linux-iio@vger.kernel.org
 S:	Maintained
+F:	Documentation/ABI/testing/sysfs-bus-iio-potentiometer-mcp4531
 F:	drivers/iio/potentiometer/mcp4531.c
 
 MEASUREMENT COMPUTING CIO-DAC IIO DRIVER
@@ -7888,12 +8009,12 @@ S:	Maintained
 F:	drivers/net/wireless/mediatek/mt7601u/
 
 MEGARAID SCSI/SAS DRIVERS
-M:	Kashyap Desai <kashyap.desai@avagotech.com>
-M:	Sumit Saxena <sumit.saxena@avagotech.com>
-M:	Uday Lingala <uday.lingala@avagotech.com>
-L:	megaraidlinux.pdl@avagotech.com
+M:	Kashyap Desai <kashyap.desai@broadcom.com>
+M:	Sumit Saxena <sumit.saxena@broadcom.com>
+M:	Shivasharan S <shivasharan.srikanteshwara@broadcom.com>
+L:	megaraidlinux.pdl@broadcom.com
 L:	linux-scsi@vger.kernel.org
-W:	http://www.lsi.com
+W:	http://www.avagotech.com/support/
 S:	Maintained
 F:	Documentation/scsi/megaraid.txt
 F:	drivers/scsi/megaraid.*
@@ -7931,6 +8052,15 @@ W:	http://www.mellanox.com
 Q:	http://patchwork.ozlabs.org/project/netdev/list/
 F:	drivers/net/ethernet/mellanox/mlxsw/
 
+MELLANOX MLXCPLD I2C AND MUX DRIVER
+M:	Vadim Pasternak <vadimp@mellanox.com>
+M:	Michael Shych <michaelsh@mellanox.com>
+L:	linux-i2c@vger.kernel.org
+S:	Supported
+F:	drivers/i2c/busses/i2c-mlxcpld.c
+F:	drivers/i2c/muxes/i2c-mux-mlxcpld.c
+F:	Documentation/i2c/busses/i2c-mlxcpld
+
 MELLANOX MLXCPLD LED DRIVER
 M:	Vadim Pasternak <vadimp@mellanox.com>
 L:	linux-leds@vger.kernel.org
@@ -7944,6 +8074,13 @@ L:      platform-driver-x86@vger.kernel.org
 S:      Supported
 F:      arch/x86/platform/mellanox/mlx-platform.c
 
+MELLANOX MLX CPLD HOTPLUG DRIVER
+M:	Vadim Pasternak <vadimp@mellanox.com>
+L:	platform-driver-x86@vger.kernel.org
+S:	Supported
+F:	drivers/platform/x86/mlxcpld-hotplug.c
+F:	include/linux/platform_data/mlxcpld-hotplug.h
+
 SOFT-ROCE DRIVER (rxe)
 M:	Moni Shoua <monis@mellanox.com>
 L:	linux-rdma@vger.kernel.org
@@ -8321,6 +8458,12 @@ T:	git git://linuxtv.org/mkrufky/tuners.git
 S:	Maintained
 F:	drivers/media/tuners/mxl5007t.*
 
+MXSFB DRM DRIVER
+M:	Marek Vasut <marex@denx.de>
+S:	Supported
+F:	drivers/gpu/drm/mxsfb/
+F:	Documentation/devicetree/bindings/display/mxsfb-drm.txt
+
 MYRICOM MYRI-10G 10GbE DRIVER (MYRI10GE)
 M:	Hyong-Youb Kim <hykim@myri.com>
 L:	netdev@vger.kernel.org
@@ -8367,7 +8510,6 @@ F:	drivers/scsi/arm/oak.c
 F:	drivers/scsi/atari_scsi.*
 F:	drivers/scsi/dmx3191d.c
 F:	drivers/scsi/g_NCR5380.*
-F:	drivers/scsi/g_NCR5380_mmio.c
 F:	drivers/scsi/mac_scsi.*
 F:	drivers/scsi/sun3_scsi.*
 F:	drivers/scsi/sun3_scsi_vme.c
@@ -8726,6 +8868,16 @@ L:	linux-nvme@lists.infradead.org
 S:	Supported
 F:	drivers/nvme/target/
 
+NVM EXPRESS FC TRANSPORT DRIVERS
+M:	James Smart <james.smart@broadcom.com>
+L:	linux-nvme@lists.infradead.org
+S:	Supported
+F:	include/linux/nvme-fc.h
+F:	include/linux/nvme-fc-driver.h
+F:	drivers/nvme/host/fc.c
+F:	drivers/nvme/target/fc.c
+F:	drivers/nvme/target/fcloop.c
+
 NVMEM FRAMEWORK
 M:	Srinivas Kandagatla <srinivas.kandagatla@linaro.org>
 M:	Maxime Ripard <maxime.ripard@free-electrons.com>
@@ -8788,6 +8940,7 @@ F:	drivers/regulator/tps65217-regulator.c
 F:	drivers/regulator/tps65218-regulator.c
 F:	drivers/regulator/tps65910-regulator.c
 F:	drivers/regulator/twl-regulator.c
+F:	drivers/regulator/twl6030-regulator.c
 F:	include/linux/i2c-omap.h
 
 OMAP DEVICE TREE SUPPORT
@@ -9144,7 +9297,7 @@ F:      drivers/misc/panel.c
 
 PARALLEL PORT SUBSYSTEM
 M:	Sudip Mukherjee <sudipm.mukherjee@gmail.com>
-M:	Sudip Mukherjee <sudip@vectorindia.org>
+M:	Sudip Mukherjee <sudip.mukherjee@codethink.co.uk>
 L:	linux-parport@lists.infradead.org (subscribers-only)
 S:	Maintained
 F:	drivers/parport/
@@ -9615,8 +9768,8 @@ F:      arch/mips/boot/dts/pistachio/
 F:      arch/mips/configs/pistachio*_defconfig
 
 PKTCDVD DRIVER
-M:	Jiri Kosina <jikos@kernel.org>
-S:	Maintained
+S:	Orphan
+M:	linux-block@vger.kernel.org
 F:	drivers/block/pktcdvd.c
 F:	include/linux/pktcdvd.h
 F:	include/uapi/linux/pktcdvd.h
@@ -9851,7 +10004,7 @@ F:	drivers/media/usb/pwc/*
 
 PWM FAN DRIVER
 M:	Kamil Debski <kamil@wypas.org>
-M:	Lukasz Majewski <l.majewski@samsung.com>
+M:	Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
 L:	linux-hwmon@vger.kernel.org
 S:	Supported
 F:	Documentation/devicetree/bindings/hwmon/pwm-fan.txt
@@ -9993,6 +10146,12 @@ F:	fs/qnx4/
 F:	include/uapi/linux/qnx4_fs.h
 F:	include/uapi/linux/qnxtypes.h
 
+QORIQ DPAA2 FSL-MC BUS DRIVER
+M:	Stuart Yoder <stuart.yoder@nxp.com>
+L:	linux-kernel@vger.kernel.org
+S:	Maintained
+F:	drivers/staging/fsl-mc/
+
 QT1010 MEDIA DRIVER
 M:	Antti Palosaari <crope@iki.fi>
 L:	linux-media@vger.kernel.org
@@ -10455,7 +10614,7 @@ F:	arch/s390/pci/
 F:	drivers/pci/hotplug/s390_pci_hpc.c
 
 S390 ZCRYPT DRIVER
-M:	Ingo Tuchscherer <ingo.tuchscherer@de.ibm.com>
+M:	Harald Freudenberger <freude@de.ibm.com>
 L:	linux-s390@vger.kernel.org
 W:	http://www.ibm.com/developerworks/linux/linux390/
 S:	Supported
@@ -10622,7 +10781,7 @@ L:	netdev@vger.kernel.org
 F:	drivers/net/ethernet/samsung/sxgbe/
 
 SAMSUNG THERMAL DRIVER
-M:	Lukasz Majewski <l.majewski@samsung.com>
+M:	Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
 L:	linux-pm@vger.kernel.org
 L:	linux-samsung-soc@vger.kernel.org
 S:	Supported
@@ -10754,6 +10913,11 @@ W:	http://www.sunplus.com
 S:	Supported
 F:	arch/score/
 
+SCR24X CHIP CARD INTERFACE DRIVER
+M:	Lubomir Rintel <lkundrak@v3.sk>
+S:	Supported
+F:	drivers/char/pcmcia/scr24x_cs.c
+
 SYSTEM CONTROL & POWER INTERFACE (SCPI) Message Protocol drivers
 M:	Sudeep Holla <sudeep.holla@arm.com>
 L:	linux-arm-kernel@lists.infradead.org
@@ -10962,7 +11126,6 @@ F:	drivers/net/ethernet/emulex/benet/
 EMULEX ONECONNECT ROCE DRIVER
 M:	Selvin Xavier <selvin.xavier@avagotech.com>
 M:	Devesh Sharma <devesh.sharma@avagotech.com>
-M:	Mitesh Ahuja <mitesh.ahuja@avagotech.com>
 L:	linux-rdma@vger.kernel.org
 W:	http://www.emulex.com
 S:	Supported
@@ -11157,7 +11320,7 @@ F:	include/media/i2c/ov2659.h
 SILICON MOTION SM712 FRAME BUFFER DRIVER
 M:	Sudip Mukherjee <sudipm.mukherjee@gmail.com>
 M:	Teddy Wang <teddy.wang@siliconmotion.com>
-M:	Sudip Mukherjee <sudip@vectorindia.org>
+M:	Sudip Mukherjee <sudip.mukherjee@codethink.co.uk>
 L:	linux-fbdev@vger.kernel.org
 S:	Maintained
 F:	drivers/video/fbdev/sm712*
@@ -11585,17 +11748,11 @@ F:	drivers/staging/rtl8712/
 STAGING - SILICON MOTION SM750 FRAME BUFFER DRIVER
 M:	Sudip Mukherjee <sudipm.mukherjee@gmail.com>
 M:	Teddy Wang <teddy.wang@siliconmotion.com>
-M:	Sudip Mukherjee <sudip@vectorindia.org>
+M:	Sudip Mukherjee <sudip.mukherjee@codethink.co.uk>
 L:	linux-fbdev@vger.kernel.org
 S:	Maintained
 F:	drivers/staging/sm750fb/
 
-STAGING - SLICOSS
-M:	Lior Dotan <liodot@gmail.com>
-M:	Christopher Harrer <charrer@alacritech.com>
-S:	Odd Fixes
-F:	drivers/staging/slicoss/
-
 STAGING - SPEAKUP CONSOLE SPEECH DRIVER
 M:	William Hubbs <w.d.hubbs@gmail.com>
 M:	Chris Brannon <chris@the-brannons.com>
@@ -11705,6 +11862,7 @@ S:	Supported
 F:	arch/arc/
 F:	Documentation/devicetree/bindings/arc/*
 F:	Documentation/devicetree/bindings/interrupt-controller/snps,arc*
+F:	drivers/clocksource/arc_timer.c
 F:	drivers/tty/serial/arc_uart.c
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git
 
@@ -11965,6 +12123,16 @@ S:	Maintained
 F:	arch/xtensa/
 F:	drivers/irqchip/irq-xtensa-*
 
+Texas Instruments' System Control Interface (TISCI) Protocol Driver
+M:	Nishanth Menon <nm@ti.com>
+M:	Tero Kristo <t-kristo@ti.com>
+M:	Santosh Shilimkar <ssantosh@kernel.org>
+L:	linux-arm-kernel@lists.infradead.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/arm/keystone/ti,sci.txt
+F:	drivers/firmware/ti_sci*
+F:	include/linux/soc/ti/ti_sci_protocol.h
+
 THANKO'S RAREMONO AM/FM/SW RADIO RECEIVER USB DRIVER
 M:	Hans Verkuil <hverkuil@xs4all.nl>
 L:	linux-media@vger.kernel.org
@@ -12395,6 +12563,12 @@ S:	Maintained
 F:	Documentation/filesystems/udf.txt
 F:	fs/udf/
 
+UDRAW TABLET
+M:	Bastien Nocera <hadess@hadess.net>
+L:	linux-input@vger.kernel.org
+S:	Maintained
+F:	drivers/hid/hid-udraw.c
+
 UFS FILESYSTEM
 M:	Evgeniy Dushistov <dushistov@mail.ru>
 S:	Maintained
@@ -12451,7 +12625,8 @@ F:	Documentation/scsi/ufs.txt
 F:	drivers/scsi/ufs/
 
 UNIVERSAL FLASH STORAGE HOST CONTROLLER DRIVER DWC HOOKS
-M:	Joao Pinto <Joao.Pinto@synopsys.com>
+M:	Manjunath M Bettegowda <manjumb@synopsys.com>
+M:	Prabu Thangamuthu <prabut@synopsys.com>
 L:	linux-scsi@vger.kernel.org
 S:	Supported
 F:	drivers/scsi/ufs/*dwc*
@@ -12809,6 +12984,15 @@ F:	drivers/vfio/
 F:	include/linux/vfio.h
 F:	include/uapi/linux/vfio.h
 
+VFIO MEDIATED DEVICE DRIVERS
+M:	Kirti Wankhede <kwankhede@nvidia.com>
+L:	kvm@vger.kernel.org
+S:	Maintained
+F:	Documentation/vfio-mediated-device.txt
+F:	drivers/vfio/mdev/
+F:	include/linux/mdev.h
+F:	samples/vfio-mdev/
+
 VFIO PLATFORM DRIVER
 M:	Baptiste Reynal <b.reynal@virtualopensystems.com>
 L:	kvm@vger.kernel.org
@@ -12863,6 +13047,7 @@ F:	drivers/net/virtio_net.c
 F:	drivers/block/virtio_blk.c
 F:	include/linux/virtio_*.h
 F:	include/uapi/linux/virtio_*.h
+F:	drivers/crypto/virtio/
 
 VIRTIO DRIVERS FOR S390
 M:	Christian Borntraeger <borntraeger@de.ibm.com>
@@ -12899,6 +13084,14 @@ S:	Maintained
 F:	drivers/virtio/virtio_input.c
 F:	include/uapi/linux/virtio_input.h
 
+VIRTIO CRYPTO DRIVER
+M:  Gonglei <arei.gonglei@huawei.com>
+L:  virtualization@lists.linux-foundation.org
+L:  linux-crypto@vger.kernel.org
+S:  Maintained
+F:  drivers/crypto/virtio/
+F:  include/uapi/linux/virtio_crypto.h
+
 VIA RHINE NETWORK DRIVER
 S:	Orphan
 F:	drivers/net/ethernet/via/via-rhine.c
@@ -13003,6 +13196,13 @@ S:	Maintained
 F:	drivers/scsi/vmw_pvscsi.c
 F:	drivers/scsi/vmw_pvscsi.h
 
+VMWARE PVRDMA DRIVER
+M:	Adit Ranadive <aditr@vmware.com>
+M:	VMware PV-Drivers <pv-drivers@vmware.com>
+L:	linux-rdma@vger.kernel.org
+S:	Maintained
+F:	drivers/infiniband/hw/vmw_pvrdma/
+
 VOLTAGE AND CURRENT REGULATOR FRAMEWORK
 M:	Liam Girdwood <lgirdwood@gmail.com>
 M:	Mark Brown <broonie@kernel.org>
@@ -13250,7 +13450,6 @@ F:	drivers/media/tuners/tuner-xc2028.*
 
 XEN HYPERVISOR INTERFACE
 M:	Boris Ostrovsky <boris.ostrovsky@oracle.com>
-M:	David Vrabel <david.vrabel@citrix.com>
 M:	Juergen Gross <jgross@suse.com>
 L:	xen-devel@lists.xenproject.org (moderated for non-subscribers)
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git
diff --git a/arch/Kconfig b/arch/Kconfig
index 835d55d52104..19483aea4bbc 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -364,8 +364,9 @@ menuconfig GCC_PLUGINS
 	  See Documentation/gcc-plugins.txt for details.
 
 config GCC_PLUGIN_CYC_COMPLEXITY
-	bool "Compute the cyclomatic complexity of a function"
+	bool "Compute the cyclomatic complexity of a function" if EXPERT
 	depends on GCC_PLUGINS
+	depends on !COMPILE_TEST
 	help
 	  The complexity M of a function's control flow graph is defined as:
 	   M = E - N + 2P
@@ -375,6 +376,10 @@ config GCC_PLUGIN_CYC_COMPLEXITY
 	  N = the number of nodes
 	  P = the number of connected components (exit nodes).
 
+	  Enabling this plugin reports the complexity to stderr during the
+	  build. It mainly serves as a simple example of how to create a
+	  gcc plugin for the kernel.
+
 config GCC_PLUGIN_SANCOV
 	bool
 	depends on GCC_PLUGINS
diff --git a/arch/alpha/kernel/ptrace.c b/arch/alpha/kernel/ptrace.c
index 940dfb406591..04abdec7f496 100644
--- a/arch/alpha/kernel/ptrace.c
+++ b/arch/alpha/kernel/ptrace.c
@@ -283,7 +283,7 @@ long arch_ptrace(struct task_struct *child, long request,
 	/* When I and D space are separate, these will need to be fixed.  */
 	case PTRACE_PEEKTEXT: /* read word at location addr. */
 	case PTRACE_PEEKDATA:
-		copied = access_process_vm(child, addr, &tmp, sizeof(tmp),
+		copied = ptrace_access_vm(child, addr, &tmp, sizeof(tmp),
 				FOLL_FORCE);
 		ret = -EIO;
 		if (copied != sizeof(tmp))
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig
index bd204bfa29ed..ab12723d39a0 100644
--- a/arch/arc/Kconfig
+++ b/arch/arc/Kconfig
@@ -8,9 +8,9 @@
 
 config ARC
 	def_bool y
+	select ARC_TIMERS
 	select ARCH_SUPPORTS_ATOMIC_RMW if ARC_HAS_LLSC
 	select BUILDTIME_EXTABLE_SORT
-	select CLKSRC_OF
 	select CLONE_BACKWARDS
 	select COMMON_CLK
 	select GENERIC_ATOMIC64 if !ISA_ARCV2 || !(ARC_HAS_LL64 && ARC_HAS_LLSC)
@@ -115,6 +115,7 @@ config ISA_ARCOMPACT
 
 config ISA_ARCV2
 	bool "ARC ISA v2"
+	select ARC_TIMERS_64BIT
 	help
 	  ISA for the Next Generation ARC-HS cores
 
@@ -410,16 +411,6 @@ config ARC_HAS_DIV_REM
 	bool "Insn: div, divu, rem, remu"
 	default y
 
-config ARC_HAS_RTC
-	bool "Local 64-bit r/o cycle counter"
-	default n
-	depends on !SMP
-
-config ARC_HAS_GFRC
-	bool "SMP synchronized 64-bit cycle counter"
-	default y
-	depends on SMP
-
 config ARC_NUMBER_OF_INTERRUPTS
 	int "Number of interrupts"
 	range 8 240
diff --git a/arch/arc/boot/dts/abilis_tb10x.dtsi b/arch/arc/boot/dts/abilis_tb10x.dtsi
index de53f5c3251c..3121536b25a3 100644
--- a/arch/arc/boot/dts/abilis_tb10x.dtsi
+++ b/arch/arc/boot/dts/abilis_tb10x.dtsi
@@ -129,6 +129,7 @@
 			data-width = <4>;
 			clocks = <&ahb_clk>;
 			clock-names = "hclk";
+			multi-block = <1 1 1 1 1 1>;
 		};
 
 		i2c0: i2c@FF120000 {
diff --git a/arch/arc/boot/dts/axs101.dts b/arch/arc/boot/dts/axs101.dts
index d9b9b9dcfc4c..70aec7d6ca60 100644
--- a/arch/arc/boot/dts/axs101.dts
+++ b/arch/arc/boot/dts/axs101.dts
@@ -17,6 +17,6 @@
 	compatible = "snps,axs101", "snps,arc-sdp";
 
 	chosen {
-		bootargs = "earlycon=uart8250,mmio32,0xe0022000,115200n8 console=tty0 console=ttyS3,115200n8 consoleblank=0";
+		bootargs = "earlycon=uart8250,mmio32,0xe0022000,115200n8 console=tty0 console=ttyS3,115200n8 consoleblank=0 video=1280x720@60";
 	};
 };
diff --git a/arch/arc/boot/dts/axs103_idu.dts b/arch/arc/boot/dts/axs103_idu.dts
index 070c29782216..5c843d9b4ac8 100644
--- a/arch/arc/boot/dts/axs103_idu.dts
+++ b/arch/arc/boot/dts/axs103_idu.dts
@@ -20,6 +20,6 @@
 	compatible = "snps,axs103", "snps,arc-sdp";
 
 	chosen {
-		bootargs = "earlycon=uart8250,mmio32,0xe0022000,115200n8 console=ttyS3,115200n8 debug print-fatal-signals=1";
+		bootargs = "earlycon=uart8250,mmio32,0xe0022000,115200n8 console=tty0 console=ttyS3,115200n8 print-fatal-signals=1 consoleblank=0 video=1280x720@60";
 	};
 };
diff --git a/arch/arc/boot/dts/zebu_hs.dts b/arch/arc/boot/dts/haps_hs.dts
index 1c1324e84965..1c1324e84965 100644
--- a/arch/arc/boot/dts/zebu_hs.dts
+++ b/arch/arc/boot/dts/haps_hs.dts
diff --git a/arch/arc/boot/dts/zebu_hs_idu.dts b/arch/arc/boot/dts/haps_hs_idu.dts
index 65204b4c0f13..65204b4c0f13 100644
--- a/arch/arc/boot/dts/zebu_hs_idu.dts
+++ b/arch/arc/boot/dts/haps_hs_idu.dts
diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig
index 0a0eaf09aac7..6980b966a364 100644
--- a/arch/arc/configs/axs101_defconfig
+++ b/arch/arc/configs/axs101_defconfig
@@ -75,9 +75,11 @@ CONFIG_I2C=y
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_DESIGNWARE_PLATFORM=y
 # CONFIG_HWMON is not set
+CONFIG_DRM=m
+CONFIG_DRM_I2C_ADV7511=m
+CONFIG_DRM_ARCPGU=m
 CONFIG_FB=y
 CONFIG_FRAMEBUFFER_CONSOLE=y
-CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y
 CONFIG_LOGO=y
 # CONFIG_LOGO_LINUX_MONO is not set
 # CONFIG_LOGO_LINUX_VGA16 is not set
diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig
index 110874705085..30a3d4cf53d2 100644
--- a/arch/arc/configs/axs103_smp_defconfig
+++ b/arch/arc/configs/axs103_smp_defconfig
@@ -77,9 +77,11 @@ CONFIG_I2C=y
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_DESIGNWARE_PLATFORM=y
 # CONFIG_HWMON is not set
+CONFIG_DRM=m
+CONFIG_DRM_I2C_ADV7511=m
+CONFIG_DRM_ARCPGU=m
 CONFIG_FB=y
 CONFIG_FRAMEBUFFER_CONSOLE=y
-CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y
 CONFIG_LOGO=y
 # CONFIG_LOGO_LINUX_MONO is not set
 # CONFIG_LOGO_LINUX_VGA16 is not set
diff --git a/arch/arc/configs/zebu_hs_defconfig b/arch/arc/configs/haps_hs_defconfig
index 9f6166be7145..57b3e599322f 100644
--- a/arch/arc/configs/zebu_hs_defconfig
+++ b/arch/arc/configs/haps_hs_defconfig
@@ -23,7 +23,7 @@ CONFIG_MODULES=y
 # CONFIG_IOSCHED_CFQ is not set
 CONFIG_ARC_PLAT_SIM=y
 CONFIG_ISA_ARCV2=y
-CONFIG_ARC_BUILTIN_DTB_NAME="zebu_hs"
+CONFIG_ARC_BUILTIN_DTB_NAME="haps_hs"
 CONFIG_PREEMPT=y
 # CONFIG_COMPACTION is not set
 CONFIG_NET=y
diff --git a/arch/arc/configs/zebu_hs_smp_defconfig b/arch/arc/configs/haps_hs_smp_defconfig
index 44e9693f4257..f85985adebb2 100644
--- a/arch/arc/configs/zebu_hs_smp_defconfig
+++ b/arch/arc/configs/haps_hs_smp_defconfig
@@ -26,7 +26,7 @@ CONFIG_MODULES=y
 CONFIG_ARC_PLAT_SIM=y
 CONFIG_ISA_ARCV2=y
 CONFIG_SMP=y
-CONFIG_ARC_BUILTIN_DTB_NAME="zebu_hs_idu"
+CONFIG_ARC_BUILTIN_DTB_NAME="haps_hs_idu"
 CONFIG_PREEMPT=y
 # CONFIG_COMPACTION is not set
 CONFIG_NET=y
diff --git a/arch/arc/configs/nsimosci_hs_smp_defconfig b/arch/arc/configs/nsimosci_hs_smp_defconfig
index 6da71ba253a9..155add7761ed 100644
--- a/arch/arc/configs/nsimosci_hs_smp_defconfig
+++ b/arch/arc/configs/nsimosci_hs_smp_defconfig
@@ -21,7 +21,7 @@ CONFIG_MODULES=y
 CONFIG_ARC_PLAT_SIM=y
 CONFIG_ISA_ARCV2=y
 CONFIG_SMP=y
-# CONFIG_ARC_HAS_GFRC is not set
+# CONFIG_ARC_TIMERS_64BIT is not set
 CONFIG_ARC_BUILTIN_DTB_NAME="nsimosci_hs_idu"
 CONFIG_PREEMPT=y
 # CONFIG_COMPACTION is not set
diff --git a/arch/arc/configs/vdk_hs38_smp_defconfig b/arch/arc/configs/vdk_hs38_smp_defconfig
index 969b206d6c67..573028f19de7 100644
--- a/arch/arc/configs/vdk_hs38_smp_defconfig
+++ b/arch/arc/configs/vdk_hs38_smp_defconfig
@@ -15,7 +15,7 @@ CONFIG_ARC_PLAT_AXS10X=y
 CONFIG_AXS103=y
 CONFIG_ISA_ARCV2=y
 CONFIG_SMP=y
-# CONFIG_ARC_HAS_GFRC is not set
+# CONFIG_ARC_TIMERS_64BIT is not set
 CONFIG_ARC_UBOOT_SUPPORT=y
 CONFIG_ARC_BUILTIN_DTB_NAME="vdk_hs38_smp"
 CONFIG_PREEMPT=y
diff --git a/arch/arc/include/asm/arcregs.h b/arch/arc/include/asm/arcregs.h
index 1bd24ec3e350..da41a54ea2d7 100644
--- a/arch/arc/include/asm/arcregs.h
+++ b/arch/arc/include/asm/arcregs.h
@@ -20,7 +20,6 @@
 #define ARC_REG_FP_V2_BCR	0xc8	/* ARCv2 FPU */
 #define ARC_REG_SLC_BCR		0xce
 #define ARC_REG_DCCM_BUILD	0x74	/* DCCM size (common) */
-#define ARC_REG_TIMERS_BCR	0x75
 #define ARC_REG_AP_BCR		0x76
 #define ARC_REG_ICCM_BUILD	0x78	/* ICCM size (common) */
 #define ARC_REG_XY_MEM_BCR	0x79
@@ -112,90 +111,7 @@
 
 #ifndef __ASSEMBLY__
 
-/*
- ******************************************************************
- *      Inline ASM macros to read/write AUX Regs
- *      Essentially invocation of lr/sr insns from "C"
- */
-
-#if 1
-
-#define read_aux_reg(reg)	__builtin_arc_lr(reg)
-
-/* gcc builtin sr needs reg param to be long immediate */
-#define write_aux_reg(reg_immed, val)		\
-		__builtin_arc_sr((unsigned int)(val), reg_immed)
-
-#else
-
-#define read_aux_reg(reg)		\
-({					\
-	unsigned int __ret;		\
-	__asm__ __volatile__(		\
-	"	lr    %0, [%1]"		\
-	: "=r"(__ret)			\
-	: "i"(reg));			\
-	__ret;				\
-})
-
-/*
- * Aux Reg address is specified as long immediate by caller
- * e.g.
- *    write_aux_reg(0x69, some_val);
- * This generates tightest code.
- */
-#define write_aux_reg(reg_imm, val)	\
-({					\
-	__asm__ __volatile__(		\
-	"	sr   %0, [%1]	\n"	\
-	:				\
-	: "ir"(val), "i"(reg_imm));	\
-})
-
-/*
- * Aux Reg address is specified in a variable
- *  * e.g.
- *      reg_num = 0x69
- *      write_aux_reg2(reg_num, some_val);
- * This has to generate glue code to load the reg num from
- *  memory to a reg hence not recommended.
- */
-#define write_aux_reg2(reg_in_var, val)		\
-({						\
-	unsigned int tmp;			\
-						\
-	__asm__ __volatile__(			\
-	"	ld   %0, [%2]	\n\t"		\
-	"	sr   %1, [%0]	\n\t"		\
-	: "=&r"(tmp)				\
-	: "r"(val), "memory"(&reg_in_var));	\
-})
-
-#endif
-
-#define READ_BCR(reg, into)				\
-{							\
-	unsigned int tmp;				\
-	tmp = read_aux_reg(reg);			\
-	if (sizeof(tmp) == sizeof(into)) {		\
-		into = *((typeof(into) *)&tmp);		\
-	} else {					\
-		extern void bogus_undefined(void);	\
-		bogus_undefined();			\
-	}						\
-}
-
-#define WRITE_AUX(reg, into)				\
-{							\
-	unsigned int tmp;				\
-	if (sizeof(tmp) == sizeof(into)) {		\
-		tmp = (*(unsigned int *)&(into));	\
-		write_aux_reg(reg, tmp);		\
-	} else  {					\
-		extern void bogus_undefined(void);	\
-		bogus_undefined();			\
-	}						\
-}
+#include <soc/arc/aux.h>
 
 /* Helpers */
 #define TO_KB(bytes)		((bytes) >> 10)
@@ -291,13 +207,7 @@ struct bcr_fp_arcv2 {
 #endif
 };
 
-struct bcr_timer {
-#ifdef CONFIG_CPU_BIG_ENDIAN
-	unsigned int pad2:15, rtsc:1, pad1:5, rtc:1, t1:1, t0:1, ver:8;
-#else
-	unsigned int ver:8, t0:1, t1:1, rtc:1, pad1:5, rtsc:1, pad2:15;
-#endif
-};
+#include <soc/arc/timers.h>
 
 struct bcr_bpu_arcompact {
 #ifdef CONFIG_CPU_BIG_ENDIAN
diff --git a/arch/arc/kernel/Makefile b/arch/arc/kernel/Makefile
index cfcdedf52ff8..8942c5c3b4c5 100644
--- a/arch/arc/kernel/Makefile
+++ b/arch/arc/kernel/Makefile
@@ -8,7 +8,7 @@
 # Pass UTS_MACHINE for user_regset definition
 CFLAGS_ptrace.o		+= -DUTS_MACHINE='"$(UTS_MACHINE)"'
 
-obj-y	:= arcksyms.o setup.o irq.o time.o reset.o ptrace.o process.o devtree.o
+obj-y	:= arcksyms.o setup.o irq.o reset.o ptrace.o process.o devtree.o
 obj-y	+= signal.o traps.o sys.o troubleshoot.o stacktrace.o disasm.o
 obj-$(CONFIG_ISA_ARCOMPACT)		+= entry-compact.o intc-compact.o
 obj-$(CONFIG_ISA_ARCV2)			+= entry-arcv2.o intc-arcv2.o
diff --git a/arch/arc/kernel/mcip.c b/arch/arc/kernel/mcip.c
index f39142acc89e..560c4afc2af4 100644
--- a/arch/arc/kernel/mcip.c
+++ b/arch/arc/kernel/mcip.c
@@ -11,8 +11,8 @@
 #include <linux/smp.h>
 #include <linux/irq.h>
 #include <linux/spinlock.h>
+#include <soc/arc/mcip.h>
 #include <asm/irqflags-arcv2.h>
-#include <asm/mcip.h>
 #include <asm/setup.h>
 
 static DEFINE_RAW_SPINLOCK(mcip_lock);
diff --git a/arch/arc/kernel/setup.c b/arch/arc/kernel/setup.c
index 0385df77a697..3093fa898a23 100644
--- a/arch/arc/kernel/setup.c
+++ b/arch/arc/kernel/setup.c
@@ -10,6 +10,8 @@
 #include <linux/fs.h>
 #include <linux/delay.h>
 #include <linux/root_dev.h>
+#include <linux/clk-provider.h>
+#include <linux/clocksource.h>
 #include <linux/console.h>
 #include <linux/module.h>
 #include <linux/cpu.h>
@@ -234,11 +236,11 @@ static char *arc_cpu_mumbojumbo(int cpu_id, char *buf, int len)
 		       is_isa_arcompact() ? "ARCompact" : "ARCv2",
 		       IS_AVAIL1(cpu->isa.be, "[Big-Endian]"));
 
-	n += scnprintf(buf + n, len - n, "Timers\t\t: %s%s%s%s\nISA Extn\t: ",
+	n += scnprintf(buf + n, len - n, "Timers\t\t: %s%s%s%s%s%s\nISA Extn\t: ",
 		       IS_AVAIL1(cpu->extn.timer0, "Timer0 "),
 		       IS_AVAIL1(cpu->extn.timer1, "Timer1 "),
-		       IS_AVAIL2(cpu->extn.rtc, "Local-64-bit-Ctr ",
-				 CONFIG_ARC_HAS_RTC));
+		       IS_AVAIL2(cpu->extn.rtc, "RTC [UP 64-bit] ", CONFIG_ARC_TIMERS_64BIT),
+		       IS_AVAIL2(cpu->extn.gfrc, "GFRC [SMP 64-bit] ", CONFIG_ARC_TIMERS_64BIT));
 
 	n += i = scnprintf(buf + n, len - n, "%s%s%s%s%s",
 			   IS_AVAIL2(cpu->isa.atomic, "atomic ", CONFIG_ARC_HAS_LLSC),
@@ -449,6 +451,15 @@ void __init setup_arch(char **cmdline_p)
 	arc_unwind_init();
 }
 
+/*
+ * Called from start_kernel() - boot CPU only
+ */
+void __init time_init(void)
+{
+	of_clk_init(NULL);
+	clocksource_probe();
+}
+
 static int __init customize_machine(void)
 {
 	if (machine_desc->init_machine)
diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c
index cd8aad8226dd..08450a1a5b5f 100644
--- a/arch/arc/mm/dma.c
+++ b/arch/arc/mm/dma.c
@@ -158,7 +158,10 @@ static dma_addr_t arc_dma_map_page(struct device *dev, struct page *page,
 		unsigned long attrs)
 {
 	phys_addr_t paddr = page_to_phys(page) + offset;
-	_dma_cache_sync(paddr, size, dir);
+
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		_dma_cache_sync(paddr, size, dir);
+
 	return plat_phys_to_dma(dev, paddr);
 }
 
diff --git a/arch/arc/plat-axs10x/axs10x.c b/arch/arc/plat-axs10x/axs10x.c
index 86548701023c..38ff349d7f2a 100644
--- a/arch/arc/plat-axs10x/axs10x.c
+++ b/arch/arc/plat-axs10x/axs10x.c
@@ -21,7 +21,7 @@
 #include <asm/asm-offsets.h>
 #include <asm/io.h>
 #include <asm/mach_desc.h>
-#include <asm/mcip.h>
+#include <soc/arc/mcip.h>
 
 #define AXS_MB_CGU		0xE0010000
 #define AXS_MB_CREG		0xE0011000
diff --git a/arch/arc/plat-eznps/include/plat/ctop.h b/arch/arc/plat-eznps/include/plat/ctop.h
index 9d6718c1a199..ee2e32df5e90 100644
--- a/arch/arc/plat-eznps/include/plat/ctop.h
+++ b/arch/arc/plat-eznps/include/plat/ctop.h
@@ -46,9 +46,7 @@
 #define CTOP_AUX_UDMC				(CTOP_AUX_BASE + 0x300)
 
 /* EZchip core instructions */
-#define CTOP_INST_HWSCHD_OFF_R3			0x3B6F00BF
 #define CTOP_INST_HWSCHD_OFF_R4			0x3C6F00BF
-#define CTOP_INST_HWSCHD_RESTORE_R3		0x3E6F70C3
 #define CTOP_INST_HWSCHD_RESTORE_R4		0x3E6F7103
 #define CTOP_INST_SCHD_RW			0x3E6F7004
 #define CTOP_INST_SCHD_RD			0x3E6F7084
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index caef68429b08..5fab553fd03a 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -888,6 +888,11 @@ config MACH_STM32F429
 	depends on ARCH_STM32
 	default y
 
+config MACH_STM32F746
+	bool "STMicrolectronics STM32F746"
+	depends on ARCH_STM32
+	default y
+
 config ARCH_MPS2
 	bool "ARM MPS2 platform"
 	depends on ARM_SINGLE_ARMV7M
diff --git a/arch/arm/Makefile b/arch/arm/Makefile
index 6be9ee148b78..ab30cc634d02 100644
--- a/arch/arm/Makefile
+++ b/arch/arm/Makefile
@@ -191,6 +191,7 @@ machine-$(CONFIG_ARCH_MXS)		+= mxs
 machine-$(CONFIG_ARCH_NETX)		+= netx
 machine-$(CONFIG_ARCH_NOMADIK)		+= nomadik
 machine-$(CONFIG_ARCH_NSPIRE)		+= nspire
+machine-$(CONFIG_ARCH_OXNAS)		+= oxnas
 machine-$(CONFIG_ARCH_OMAP1)		+= omap1
 machine-$(CONFIG_ARCH_OMAP2PLUS)	+= omap2
 machine-$(CONFIG_ARCH_ORION5X)		+= orion5x
@@ -311,8 +312,11 @@ all:	$(KBUILD_IMAGE) $(KBUILD_DTBS)
 
 boot := arch/arm/boot
 
+archheaders:
+	$(Q)$(MAKE) $(build)=arch/arm/tools uapi
+
 archprepare:
-	$(Q)$(MAKE) $(build)=arch/arm/tools include/generated/mach-types.h
+	$(Q)$(MAKE) $(build)=arch/arm/tools kapi
 
 # Convert bzImage to zImage
 bzImage: zImage
diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile
index c558ba75cbcc..cccdbcb557b6 100644
--- a/arch/arm/boot/dts/Makefile
+++ b/arch/arm/boot/dts/Makefile
@@ -75,6 +75,7 @@ dtb-$(CONFIG_ARCH_BCM_5301X) += \
 	bcm4708-asus-rt-ac56u.dtb \
 	bcm4708-asus-rt-ac68u.dtb \
 	bcm4708-buffalo-wzr-1750dhp.dtb \
+	bcm4708-luxul-xap-1510.dtb \
 	bcm4708-luxul-xwc-1000.dtb \
 	bcm4708-netgear-r6250.dtb \
 	bcm4708-netgear-r6300-v2.dtb \
@@ -86,11 +87,16 @@ dtb-$(CONFIG_ARCH_BCM_5301X) += \
 	bcm4709-buffalo-wxr-1900dhp.dtb \
 	bcm4709-netgear-r7000.dtb \
 	bcm4709-netgear-r8000.dtb \
+	bcm4709-tplink-archer-c9-v1.dtb \
 	bcm47094-dlink-dir-885l.dtb \
+	bcm47094-luxul-xwr-3100.dtb \
+	bcm47094-netgear-r8500.dtb \
 	bcm94708.dtb \
 	bcm94709.dtb \
 	bcm953012er.dtb \
 	bcm953012k.dtb
+dtb-$(CONFIG_ARCH_BCM_53573) += \
+	bcm47189-tenda-ac9.dtb
 dtb-$(CONFIG_ARCH_BCM_63XX) += \
 	bcm963138dvt.dtb
 dtb-$(CONFIG_ARCH_BCM_CYGNUS) += \
@@ -136,6 +142,7 @@ dtb-$(CONFIG_ARCH_EXYNOS4) += \
 	exynos4210-smdkv310.dtb \
 	exynos4210-trats.dtb \
 	exynos4210-universal_c210.dtb \
+	exynos4412-itop-elite.dtb \
 	exynos4412-odroidu3.dtb \
 	exynos4412-odroidx.dtb \
 	exynos4412-odroidx2.dtb \
@@ -330,6 +337,7 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6dl-aristainetos_7.dtb \
 	imx6dl-aristainetos2_4.dtb \
 	imx6dl-aristainetos2_7.dtb \
+	imx6dl-colibri-eval-v3.dtb \
 	imx6dl-cubox-i.dtb \
 	imx6dl-dfi-fs700-m60.dtb \
 	imx6dl-gw51xx.dtb \
@@ -340,6 +348,7 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6dl-gw552x.dtb \
 	imx6dl-gw553x.dtb \
 	imx6dl-hummingboard.dtb \
+	imx6dl-icore.dtb \
 	imx6dl-nit6xlite.dtb \
 	imx6dl-nitrogen6x.dtb \
 	imx6dl-phytec-pbab01.dtb \
@@ -381,10 +390,12 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6q-gw553x.dtb \
 	imx6q-h100.dtb \
 	imx6q-hummingboard.dtb \
+	imx6q-icore.dtb \
 	imx6q-icore-rqs.dtb \
 	imx6q-marsboard.dtb \
 	imx6q-nitrogen6x.dtb \
 	imx6q-nitrogen6_max.dtb \
+	imx6q-nitrogen6_som2.dtb \
 	imx6q-novena.dtb \
 	imx6q-phytec-pbab01.dtb \
 	imx6q-rex-pro.dtb \
@@ -416,14 +427,19 @@ dtb-$(CONFIG_SOC_IMX6SX) += \
 	imx6sx-sabreauto.dtb \
 	imx6sx-sdb-reva.dtb \
 	imx6sx-sdb-sai.dtb \
-	imx6sx-sdb.dtb
+	imx6sx-sdb.dtb \
+	imx6sx-udoo-neo-basic.dtb \
+	imx6sx-udoo-neo-extended.dtb \
+	imx6sx-udoo-neo-full.dtb
 dtb-$(CONFIG_SOC_IMX6UL) += \
 	imx6ul-14x14-evk.dtb \
 	imx6ul-geam-kit.dtb \
+	imx6ul-liteboard.dtb \
 	imx6ul-pico-hobbit.dtb \
 	imx6ul-tx6ul-0010.dtb \
 	imx6ul-tx6ul-0011.dtb \
-	imx6ul-tx6ul-mainboard.dtb
+	imx6ul-tx6ul-mainboard.dtb \
+	imx6ull-14x14-evk.dtb
 dtb-$(CONFIG_SOC_IMX7D) += \
 	imx7d-cl-som-imx7.dtb \
 	imx7d-colibri-eval-v3.dtb \
@@ -561,6 +577,7 @@ dtb-$(CONFIG_SOC_AM33XX) += \
 	am335x-sl50.dtb \
 	am335x-wega-rdk.dtb
 dtb-$(CONFIG_ARCH_OMAP4) += \
+	omap4-droid4-xt894.dtb \
 	omap4-duovero-parlor.dtb \
 	omap4-kc1.dtb \
 	omap4-panda.dtb \
@@ -588,15 +605,18 @@ dtb-$(CONFIG_SOC_DRA7XX) += \
 	am57xx-cl-som-am57x.dtb \
 	am57xx-sbc-am57x.dtb \
 	am572x-idk.dtb \
+	am571x-idk.dtb \
 	dra7-evm.dtb \
 	dra72-evm.dtb \
-	dra72-evm-revc.dtb
+	dra72-evm-revc.dtb \
+	dra71-evm.dtb
 dtb-$(CONFIG_ARCH_ORION5X) += \
 	orion5x-kuroboxpro.dtb \
 	orion5x-lacie-d2-network.dtb \
 	orion5x-lacie-ethernet-disk-mini-v2.dtb \
 	orion5x-linkstation-lsgl.dtb \
 	orion5x-linkstation-lswtgl.dtb \
+	orion5x-lschl.dtb \
 	orion5x-lswsgl.dtb \
 	orion5x-maxtor-shared-storage-2.dtb \
 	orion5x-netgear-wnr854t.dtb \
@@ -604,7 +624,8 @@ dtb-$(CONFIG_ARCH_ORION5X) += \
 dtb-$(CONFIG_ARCH_PRIMA2) += \
 	prima2-evb.dtb
 dtb-$(CONFIG_ARCH_OXNAS) += \
-	wd-mbwe.dtb
+	wd-mbwe.dtb \
+	cloudengines-pogoplug-series-3.dtb
 dtb-$(CONFIG_ARCH_QCOM) += \
 	qcom-apq8060-dragonboard.dtb \
 	qcom-apq8064-arrow-sd-600eval.dtb \
@@ -620,7 +641,8 @@ dtb-$(CONFIG_ARCH_QCOM) += \
 	qcom-msm8660-surf.dtb \
 	qcom-msm8960-cdp.dtb \
 	qcom-msm8974-lge-nexus5-hammerhead.dtb \
-	qcom-msm8974-sony-xperia-honami.dtb
+	qcom-msm8974-sony-xperia-honami.dtb \
+	qcom-mdm9615-wp8548-mangoh-green.dtb
 dtb-$(CONFIG_ARCH_REALVIEW) += \
 	arm-realview-pb1176.dtb \
 	arm-realview-pb11mp.dtb \
@@ -635,11 +657,14 @@ dtb-$(CONFIG_ARCH_REALVIEW) += \
 	arm-realview-pba8.dtb \
 	arm-realview-pbx-a9.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += \
+	rk1108-evb.dtb \
 	rk3036-evb.dtb \
 	rk3036-kylin.dtb \
 	rk3066a-bqcurie2.dtb \
 	rk3066a-marsboard.dtb \
+	rk3066a-mk808.dtb \
 	rk3066a-rayeager.dtb \
+	rk3188-px3-evb.dtb \
 	rk3188-radxarock.dtb \
 	rk3228-evb.dtb \
 	rk3229-evb.dtb \
@@ -677,6 +702,8 @@ dtb-$(CONFIG_ARCH_SHMOBILE_MULTI) += \
 	r7s72100-rskrza1.dtb \
 	r8a73a4-ape6evm.dtb \
 	r8a7740-armadillo800eva.dtb \
+	r8a7743-sk-rzg1m.dtb \
+	r8a7745-sk-rzg1e.dtb \
 	r8a7778-bockw.dtb \
 	r8a7779-marzen.dtb \
 	r8a7790-lager.dtb \
@@ -690,12 +717,14 @@ dtb-$(CONFIG_ARCH_SHMOBILE_MULTI) += \
 	sh73a0-kzm9g.dtb
 dtb-$(CONFIG_ARCH_SOCFPGA) += \
 	socfpga_arria5_socdk.dtb \
+	socfpga_arria10_socdk_qspi.dtb \
 	socfpga_arria10_socdk_sdmmc.dtb \
 	socfpga_cyclone5_mcvevk.dtb \
 	socfpga_cyclone5_socdk.dtb \
 	socfpga_cyclone5_de0_sockit.dtb \
 	socfpga_cyclone5_sockit.dtb \
 	socfpga_cyclone5_socrates.dtb \
+	socfpga_cyclone5_sodia.dtb \
 	socfpga_cyclone5_vining_fpga.dtb \
 	socfpga_vt.dtb
 dtb-$(CONFIG_ARCH_SPEAR13XX) += \
@@ -712,16 +741,12 @@ dtb-$(CONFIG_ARCH_STI) += \
 	stih407-b2120.dtb \
 	stih410-b2120.dtb \
 	stih410-b2260.dtb \
-	stih415-b2000.dtb \
-	stih415-b2020.dtb \
-	stih416-b2000.dtb \
-	stih416-b2020.dtb \
-	stih416-b2020e.dtb \
 	stih418-b2199.dtb
 dtb-$(CONFIG_ARCH_STM32)+= \
 	stm32f429-disco.dtb \
 	stm32f469-disco.dtb \
-	stm32429i-eval.dtb
+	stm32429i-eval.dtb \
+	stm32746g-eval.dtb
 dtb-$(CONFIG_MACH_SUN4I) += \
 	sun4i-a10-a1000.dtb \
 	sun4i-a10-ba10-tvbox.dtb \
@@ -760,6 +785,7 @@ dtb-$(CONFIG_MACH_SUN5I) += \
 	sun5i-a13-olinuxino-micro.dtb \
 	sun5i-a13-q8-tablet.dtb \
 	sun5i-a13-utoo-p66.dtb \
+	sun5i-gr8-chip-pro.dtb \
 	sun5i-gr8-evb.dtb \
 	sun5i-r8-chip.dtb
 dtb-$(CONFIG_MACH_SUN6I) += \
@@ -897,6 +923,7 @@ dtb-$(CONFIG_ARCH_VT8500) += \
 	wm8750-apc8750.dtb \
 	wm8850-w70v2.dtb
 dtb-$(CONFIG_ARCH_ZYNQ) += \
+	zynq-microzed.dtb \
 	zynq-parallella.dtb \
 	zynq-zc702.dtb \
 	zynq-zc706.dtb \
@@ -920,6 +947,7 @@ dtb-$(CONFIG_MACH_ARMADA_38X) += \
 	armada-385-db-ap.dtb \
 	armada-385-linksys-caiman.dtb \
 	armada-385-linksys-cobra.dtb \
+	armada-385-turris-omnia.dtb \
 	armada-388-clearfog.dtb \
 	armada-388-db.dtb \
 	armada-388-gp.dtb \
diff --git a/arch/arm/boot/dts/am335x-baltos-ir2110.dts b/arch/arm/boot/dts/am335x-baltos-ir2110.dts
index a9a97307d66c..501c7527121b 100644
--- a/arch/arm/boot/dts/am335x-baltos-ir2110.dts
+++ b/arch/arm/boot/dts/am335x-baltos-ir2110.dts
@@ -54,16 +54,22 @@
 	dr_mode = "host";
 };
 
+&davinci_mdio {
+	phy0: ethernet-phy@0 {
+		reg = <1>;
+	};
+};
+
 &cpsw_emac0 {
-	phy_id = <&davinci_mdio>, <1>;
 	phy-mode = "rmii";
 	dual_emac_res_vlan = <1>;
+	phy-handle = <&phy0>;
 };
 
 &cpsw_emac1 {
-	phy_id = <&davinci_mdio>, <7>;
 	phy-mode = "rgmii-txid";
 	dual_emac_res_vlan = <2>;
+	phy-handle = <&phy1>;
 };
 
 &phy_sel {
diff --git a/arch/arm/boot/dts/am335x-baltos-ir3220.dts b/arch/arm/boot/dts/am335x-baltos-ir3220.dts
index fe002a17c04b..19f53b8569e1 100644
--- a/arch/arm/boot/dts/am335x-baltos-ir3220.dts
+++ b/arch/arm/boot/dts/am335x-baltos-ir3220.dts
@@ -109,9 +109,9 @@
 };
 
 &cpsw_emac1 {
-	phy_id = <&davinci_mdio>, <7>;
 	phy-mode = "rgmii-txid";
 	dual_emac_res_vlan = <2>;
+	phy-handle = <&phy1>;
 };
 
 &phy_sel {
diff --git a/arch/arm/boot/dts/am335x-baltos-ir5221.dts b/arch/arm/boot/dts/am335x-baltos-ir5221.dts
index d0faa7b8c5da..2b9d7f4db23f 100644
--- a/arch/arm/boot/dts/am335x-baltos-ir5221.dts
+++ b/arch/arm/boot/dts/am335x-baltos-ir5221.dts
@@ -114,7 +114,7 @@
 
 &usb1 {
 	status = "okay";
-	dr_mode = "otg";
+	dr_mode = "host";
 };
 
 &cpsw_emac0 {
@@ -127,9 +127,9 @@
 };
 
 &cpsw_emac1 {
-	phy_id = <&davinci_mdio>, <7>;
 	phy-mode = "rgmii-txid";
 	dual_emac_res_vlan = <2>;
+	phy-handle = <&phy1>;
 };
 
 &phy_sel {
diff --git a/arch/arm/boot/dts/am335x-baltos.dtsi b/arch/arm/boot/dts/am335x-baltos.dtsi
index dd45d172a892..efb5eae290a8 100644
--- a/arch/arm/boot/dts/am335x-baltos.dtsi
+++ b/arch/arm/boot/dts/am335x-baltos.dtsi
@@ -364,11 +364,14 @@
 };
 
 &davinci_mdio {
+	status = "okay";
 	pinctrl-names = "default", "sleep";
 	pinctrl-0 = <&davinci_mdio_default>;
 	pinctrl-1 = <&davinci_mdio_sleep>;
 
-	status = "okay";
+	phy1: ethernet-phy@1 {
+		reg = <7>;
+	};
 };
 
 &mmc1 {
@@ -406,3 +409,7 @@
 &gpio0 {
 	ti,no-reset-on-init;
 };
+
+&gpio3 {
+	ti,no-reset-on-init;
+};
diff --git a/arch/arm/boot/dts/am335x-bone-common.dtsi b/arch/arm/boot/dts/am335x-bone-common.dtsi
index 007b5e5a51a9..dc561d505bbe 100644
--- a/arch/arm/boot/dts/am335x-bone-common.dtsi
+++ b/arch/arm/boot/dts/am335x-bone-common.dtsi
@@ -6,6 +6,8 @@
  * published by the Free Software Foundation.
  */
 
+#include <dt-bindings/mfd/tps65217.h>
+
 / {
 	cpus {
 		cpu@0 {
@@ -310,8 +312,23 @@
 	 * by the hardware problems. (Tip: double-check by performing a current
 	 * measurement after shutdown: it should be less than 1 mA.)
 	 */
+
+	interrupts = <7>; /* NMI */
+	interrupt-parent = <&intc>;
+
 	ti,pmic-shutdown-controller;
 
+	charger {
+		interrupts = <TPS65217_IRQ_AC>, <TPS65217_IRQ_USB>;
+		interrupts-names = "AC", "USB";
+		status = "okay";
+	};
+
+	pwrbutton {
+		interrupts = <TPS65217_IRQ_PB>;
+		status = "okay";
+	};
+
 	regulators {
 		dcdc1_reg: regulator@0 {
 			regulator-name = "vdds_dpr";
@@ -393,3 +410,8 @@
 &sham {
 	status = "okay";
 };
+
+&rtc {
+	clocks = <&clk_32768_ck>, <&clkdiv32k_ick>;
+	clock-names = "ext-clk", "int-clk";
+};
diff --git a/arch/arm/boot/dts/am335x-boneblack.dts b/arch/arm/boot/dts/am335x-boneblack.dts
index 6bbb1fee0868..db00d8ef7b19 100644
--- a/arch/arm/boot/dts/am335x-boneblack.dts
+++ b/arch/arm/boot/dts/am335x-boneblack.dts
@@ -79,6 +79,14 @@
 
 &lcdc {
 	status = "okay";
+
+	/* If you want to get 24 bit RGB and 16 BGR mode instead of
+	 * current 16 bit RGB and 24 BGR modes, set the propety
+	 * below to "crossed" and uncomment the video-ports -property
+	 * in tda19988 node.
+	 */
+	blue-and-red-wiring = "straight";
+
 	port {
 		lcdc_0: endpoint@0 {
 			remote-endpoint = <&hdmi_0>;
@@ -95,6 +103,9 @@
 		pinctrl-0 = <&nxp_hdmi_bonelt_pins>;
 		pinctrl-1 = <&nxp_hdmi_bonelt_off_pins>;
 
+		/* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */
+		/* video-ports = <0x234501>; */
+
 		#sound-dai-cells = <0>;
 		audio-ports = <	TDA998x_I2S	0x03>;
 
diff --git a/arch/arm/boot/dts/am335x-evm.dts b/arch/arm/boot/dts/am335x-evm.dts
index e82432c79f85..c2186ec2834b 100644
--- a/arch/arm/boot/dts/am335x-evm.dts
+++ b/arch/arm/boot/dts/am335x-evm.dts
@@ -783,3 +783,8 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&dcan1_pins_default>;
 };
+
+&rtc {
+	clocks = <&clk_32768_ck>, <&clkdiv32k_ick>;
+	clock-names = "ext-clk", "int-clk";
+};
diff --git a/arch/arm/boot/dts/am335x-evmsk.dts b/arch/arm/boot/dts/am335x-evmsk.dts
index 975c36e332a2..e2548d1ce753 100644
--- a/arch/arm/boot/dts/am335x-evmsk.dts
+++ b/arch/arm/boot/dts/am335x-evmsk.dts
@@ -715,3 +715,8 @@
 
 	blue-and-red-wiring = "crossed";
 };
+
+&rtc {
+	clocks = <&clk_32768_ck>, <&clkdiv32k_ick>;
+	clock-names = "ext-clk", "int-clk";
+};
diff --git a/arch/arm/boot/dts/am335x-icev2.dts b/arch/arm/boot/dts/am335x-icev2.dts
index 85e04c205542..1463df3b5b19 100644
--- a/arch/arm/boot/dts/am335x-icev2.dts
+++ b/arch/arm/boot/dts/am335x-icev2.dts
@@ -43,52 +43,52 @@
 		enable-active-high;
 	};
 
-	leds0 {
+	leds-iio {
+		status = "disabled";
 		compatible = "gpio-leds";
-
-		led0 {
+		led-out0 {
 			label = "out0";
 			gpios = <&tpic2810 0 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
 
-		led1 {
+		led-out1 {
 			label = "out1";
 			gpios = <&tpic2810 1 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
 
-		led2 {
+		led-out2 {
 			label = "out2";
 			gpios = <&tpic2810 2 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
 
-		led3 {
+		led-out3 {
 			label = "out3";
 			gpios = <&tpic2810 3 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
 
-		led4 {
+		led-out4 {
 			label = "out4";
 			gpios = <&tpic2810 4 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
 
-		led5 {
+		led-out5 {
 			label = "out5";
 			gpios = <&tpic2810 5 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
 
-		led6 {
+		led-out6 {
 			label = "out6";
 			gpios = <&tpic2810 6 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
 		};
 
-		led7 {
+		led-out7 {
 			label = "out7";
 			gpios = <&tpic2810 7 GPIO_ACTIVE_HIGH>;
 			default-state = "off";
@@ -187,6 +187,8 @@
 			AM33XX_IOPAD(0x954, PIN_INPUT_PULLUP | MUX_MODE0) /* (B17) spi0_d0.spi0_d0 */
 			AM33XX_IOPAD(0x958, PIN_INPUT_PULLUP | MUX_MODE0) /* (B16) spi0_d1.spi0_d1 */
 			AM33XX_IOPAD(0x95c, PIN_INPUT_PULLUP | MUX_MODE0) /* (A16) spi0_cs0.spi0_cs0 */
+			AM33XX_IOPAD(0x960, PIN_INPUT_PULLUP | MUX_MODE0) /* (C15) spi0_cs1.spi0_cs1 */
+			AM33XX_IOPAD(0x9a0, PIN_INPUT_PULLUP | MUX_MODE7) /* (B12) mcasp0_aclkr.gpio3[18] */
 		>;
 	};
 
@@ -224,6 +226,31 @@
 	};
 };
 
+&spi0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi0_pins_default>;
+
+	sn65hvs882@1 {
+		compatible = "pisosr-gpio";
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		load-gpios = <&gpio3 18 GPIO_ACTIVE_LOW>;
+
+		reg = <1>;
+		spi-max-frequency = <1000000>;
+		spi-cpol;
+	};
+};
+
+&tscadc {
+	status = "okay";
+	adc {
+		ti,adc-channels = <1 2 3 4 5 6 7>;
+	};
+};
+
 #include "tps65910.dtsi"
 
 &tps {
diff --git a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi
index 194d884c9de1..64c8aa9057a3 100644
--- a/arch/arm/boot/dts/am33xx.dtsi
+++ b/arch/arm/boot/dts/am33xx.dtsi
@@ -130,6 +130,7 @@
 				reg = <0x210000 0x2000>;
 				#address-cells = <1>;
 				#size-cells = <1>;
+				#pinctrl-cells = <1>;
 				ranges = <0 0x210000 0x2000>;
 
 				am33xx_pinmux: pinmux@800 {
@@ -137,6 +138,7 @@
 					reg = <0x800 0x238>;
 					#address-cells = <1>;
 					#size-cells = <0>;
+					#pinctrl-cells = <1>;
 					pinctrl-single,register-width = <32>;
 					pinctrl-single,function-mask = <0x7f>;
 				};
@@ -505,6 +507,8 @@
 			interrupts = <75
 				      76>;
 			ti,hwmods = "rtc";
+			clocks = <&clkdiv32k_ick>;
+			clock-names = "int-clk";
 		};
 
 		spi0: spi@48030000 {
@@ -855,6 +859,8 @@
 			interrupts = <16>;
 			ti,hwmods = "adc_tsc";
 			status = "disabled";
+			dmas = <&edma 53 0>, <&edma 57 0>;
+			dma-names = "fifo0", "fifo1";
 
 			tsc {
 				compatible = "ti,am3359-tsc";
diff --git a/arch/arm/boot/dts/am3517.dtsi b/arch/arm/boot/dts/am3517.dtsi
index 0db19d39d24c..9fe545dbfa89 100644
--- a/arch/arm/boot/dts/am3517.dtsi
+++ b/arch/arm/boot/dts/am3517.dtsi
@@ -66,6 +66,7 @@
 			reg = <0x480025d8 0x24>;
 			#address-cells = <1>;
 			#size-cells = <0>;
+			#pinctrl-cells = <1>;
 			#interrupt-cells = <1>;
 			interrupt-controller;
 			pinctrl-single,register-width = <16>;
diff --git a/arch/arm/boot/dts/am4372.dtsi b/arch/arm/boot/dts/am4372.dtsi
index a275fa956813..ac55f93fc91e 100644
--- a/arch/arm/boot/dts/am4372.dtsi
+++ b/arch/arm/boot/dts/am4372.dtsi
@@ -189,6 +189,7 @@
 					reg = <0x800 0x31c>;
 					#address-cells = <1>;
 					#size-cells = <0>;
+					#pinctrl-cells = <1>;
 					#interrupt-cells = <1>;
 					interrupt-controller;
 					pinctrl-single,register-width = <32>;
@@ -871,6 +872,8 @@
 			clocks = <&adc_tsc_fck>;
 			clock-names = "fck";
 			status = "disabled";
+			dmas = <&edma 53 0>, <&edma 57 0>;
+			dma-names = "fifo0", "fifo1";
 
 			tsc {
 				compatible = "ti,am3359-tsc";
diff --git a/arch/arm/boot/dts/am437x-idk-evm.dts b/arch/arm/boot/dts/am437x-idk-evm.dts
index 25ce611c6568..b76a7c0264a5 100644
--- a/arch/arm/boot/dts/am437x-idk-evm.dts
+++ b/arch/arm/boot/dts/am437x-idk-evm.dts
@@ -117,6 +117,58 @@
 		compatible = "fixed-clock";
 		clock-frequency = <32768>;
 	};
+
+	leds-iio {
+		status = "disabled";
+		compatible = "gpio-leds";
+		led-out0 {
+			label = "out0";
+			gpios = <&tpic2810 0 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out1 {
+			label = "out1";
+			gpios = <&tpic2810 1 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out2 {
+			label = "out2";
+			gpios = <&tpic2810 2 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out3 {
+			label = "out3";
+			gpios = <&tpic2810 3 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out4 {
+			label = "out4";
+			gpios = <&tpic2810 4 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out5 {
+			label = "out5";
+			gpios = <&tpic2810 5 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out6 {
+			label = "out6";
+			gpios = <&tpic2810 6 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out7 {
+			label = "out7";
+			gpios = <&tpic2810 7 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
 };
 
 &am43xx_pinmux {
@@ -178,6 +230,24 @@
 		>;
 	};
 
+	spi1_pins_default: spi1_pins_default {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x908, PIN_INPUT | MUX_MODE2)	/* mii1_col.spi1_sclk */
+			AM4372_IOPAD(0x910, PIN_INPUT | MUX_MODE2)	/* mii1_rx_er.spi1_d1 */
+			AM4372_IOPAD(0x944, PIN_OUTPUT | MUX_MODE2)	/* rmii1_ref_clk.spi1_cs0 */
+			AM4372_IOPAD(0x90c, PIN_OUTPUT | MUX_MODE7)	/* mii1_crs.gpio3_1 */
+		>;
+	};
+
+	spi1_pins_sleep: spi1_pins_sleep {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x908, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x910, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x944, PIN_INPUT_PULLDOWN | MUX_MODE7)
+			AM4372_IOPAD(0x90c, PIN_INPUT_PULLDOWN | MUX_MODE7)
+		>;
+	};
+
 	ecap0_pins_default: backlight_pins_default {
 		pinctrl-single,pins = <
 			AM4372_IOPAD(0x964, PIN_OUTPUT | MUX_MODE0) /* ecap0_in_pwm0_out.ecap0_in_pwm0_out */
@@ -290,6 +360,33 @@
 	pinctrl-0 = <&i2c2_pins_default>;
 	pinctrl-1 = <&i2c2_pins_sleep>;
 	clock-frequency = <100000>;
+
+	tpic2810: tpic2810@60 {
+		compatible = "ti,tpic2810";
+		reg = <0x60>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
+
+&spi1 {
+	status = "okay";
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&spi1_pins_default>;
+	pinctrl-1 = <&spi1_pins_sleep>;
+	ti,pindir-d0-out-d1-in;
+
+	sn65hvs882: sn65hvs882@0 {
+		compatible = "pisosr-gpio";
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		load-gpios = <&gpio3 1 GPIO_ACTIVE_LOW>;
+
+		reg = <0>;
+		spi-max-frequency = <1000000>;
+		spi-cpol;
+	};
 };
 
 &epwmss0 {
@@ -310,6 +407,10 @@
 	status = "okay";
 };
 
+&gpio3 {
+	status = "okay";
+};
+
 &gpio4 {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/am571x-idk.dts b/arch/arm/boot/dts/am571x-idk.dts
new file mode 100644
index 000000000000..d6e43e5184c1
--- /dev/null
+++ b/arch/arm/boot/dts/am571x-idk.dts
@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2015-2016 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+/dts-v1/;
+
+#include "dra72x.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include "am57xx-idk-common.dtsi"
+
+/ {
+	model = "TI AM5718 IDK";
+	compatible = "ti,am5718-idk", "ti,am5718", "ti,dra7";
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x0 0x40000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		cpu0-led {
+			label = "status0:red:cpu0";
+			gpios = <&gpio2 25 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "cpu0";
+		};
+
+		usr0-led {
+			label = "status0:green:usr";
+			gpios = <&gpio2 26 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		heartbeat-led {
+			label = "status0:blue:heartbeat";
+			gpios = <&gpio2 27 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "heartbeat";
+		};
+
+		usr1-led {
+			label = "status1:red:usr";
+			gpios = <&gpio2 28 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		usr2-led {
+			label = "status1:green:usr";
+			gpios = <&gpio2 21 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		mmc0-led {
+			label = "status1:blue:mmc0";
+			gpios = <&gpio2 19 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "mmc0";
+		};
+	};
+
+	extcon_usb2: extcon_usb2 {
+	     compatible = "linux,extcon-usb-gpio";
+	     id-gpio = <&gpio5 7 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&mmc1 {
+	status = "okay";
+	vmmc-supply = <&ldo1_reg>;
+	bus-width = <4>;
+	cd-gpios = <&gpio6 27 0>; /* gpio 219 */
+};
+
+&omap_dwc3_2 {
+	extcon = <&extcon_usb2>;
+};
diff --git a/arch/arm/boot/dts/am572x-idk.dts b/arch/arm/boot/dts/am572x-idk.dts
index 87bbc66f0f21..27d9149cedba 100644
--- a/arch/arm/boot/dts/am572x-idk.dts
+++ b/arch/arm/boot/dts/am572x-idk.dts
@@ -83,3 +83,7 @@
 	bus-width = <4>;
 	cd-gpios = <&gpio6 27 0>; /* gpio 219 */
 };
+
+&sn65hvs882 {
+	load-gpios = <&gpio3 19 GPIO_ACTIVE_LOW>;
+};
diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi
index 6df7829a2c15..78bee26361f1 100644
--- a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi
+++ b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi
@@ -204,6 +204,7 @@
 		interrupt-controller;
 
 		ti,system-power-controller;
+		ti,palmas-override-powerhold;
 
 		tps659038_pmic {
 			compatible = "ti,tps659038-pmic";
diff --git a/arch/arm/boot/dts/am57xx-idk-common.dtsi b/arch/arm/boot/dts/am57xx-idk-common.dtsi
index 03cec62260e1..555ae21f2b9a 100644
--- a/arch/arm/boot/dts/am57xx-idk-common.dtsi
+++ b/arch/arm/boot/dts/am57xx-idk-common.dtsi
@@ -43,6 +43,58 @@
 		regulator-always-on;
 		regulator-boot-on;
 	};
+
+	leds-iio {
+		status = "disabled";
+		compatible = "gpio-leds";
+		led-out0 {
+			label = "out0";
+			gpios = <&tpic2810 0 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out1 {
+			label = "out1";
+			gpios = <&tpic2810 1 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out2 {
+			label = "out2";
+			gpios = <&tpic2810 2 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out3 {
+			label = "out3";
+			gpios = <&tpic2810 3 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out4 {
+			label = "out4";
+			gpios = <&tpic2810 4 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out5 {
+			label = "out5";
+			gpios = <&tpic2810 5 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out6 {
+			label = "out6";
+			gpios = <&tpic2810 6 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-out7 {
+			label = "out7";
+			gpios = <&tpic2810 7 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
 };
 
 &i2c1 {
@@ -57,6 +109,7 @@
 		#interrupt-cells = <2>;
 		interrupt-controller;
 		ti,system-power-controller;
+		ti,palmas-override-powerhold;
 
 		tps659038_pmic {
 			compatible = "ti,tps659038-pmic";
@@ -253,6 +306,28 @@
 	};
 };
 
+&mcspi3 {
+	status = "okay";
+	ti,pindir-d0-out-d1-in;
+
+	sn65hvs882: sn65hvs882@0 {
+		compatible = "pisosr-gpio";
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		reg = <0>;
+		spi-max-frequency = <1000000>;
+		spi-cpol;
+	};
+
+	tpic2810: tpic2810@60 {
+		compatible = "ti,tpic2810";
+		reg = <0x60>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
+
 &uart3 {
 	status = "okay";
 	interrupts-extended = <&crossbar_mpu GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH
diff --git a/arch/arm/boot/dts/armada-370-db.dts b/arch/arm/boot/dts/armada-370-db.dts
index 033fa63544f7..a9419f8e17e8 100644
--- a/arch/arm/boot/dts/armada-370-db.dts
+++ b/arch/arm/boot/dts/armada-370-db.dts
@@ -67,7 +67,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x40000000>; /* 1 GB */
 	};
@@ -86,18 +86,6 @@
 				status = "okay";
 			};
 
-			mdio {
-				pinctrl-0 = <&mdio_pins>;
-				pinctrl-names = "default";
-				phy0: ethernet-phy@0 {
-					reg = <0>;
-				};
-
-				phy1: ethernet-phy@1 {
-					reg = <1>;
-				};
-			};
-
 			ethernet@70000 {
 				pinctrl-0 = <&ge0_rgmii_pins>;
 				pinctrl-names = "default";
@@ -182,24 +170,6 @@
 				};
 			};
 		};
-
-		pcie-controller {
-			status = "okay";
-			/*
-			 * The two PCIe units are accessible through
-			 * both standard PCIe slots and mini-PCIe
-			 * slots on the board.
-			 */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-
-			pcie@2,0 {
-				/* Port 1, Lane 0 */
-				status = "okay";
-			};
-		};
 	};
 
 	sound {
@@ -261,6 +231,37 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+	/*
+	 * The two PCIe units are accessible through
+	 * both standard PCIe slots and mini-PCIe
+	 * slots on the board.
+	 */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+
+	pcie@2,0 {
+		/* Port 1, Lane 0 */
+		status = "okay";
+	};
+};
+
+&mdio {
+	pinctrl-0 = <&mdio_pins>;
+	pinctrl-names = "default";
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+	};
+
+	phy1: ethernet-phy@1 {
+		reg = <1>;
+	};
+};
+
+
 &spi0 {
 	pinctrl-0 = <&spi0_pins2>;
 	pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/armada-370-dlink-dns327l.dts b/arch/arm/boot/dts/armada-370-dlink-dns327l.dts
index e2a363b1dd8a..aeedc463daa6 100644
--- a/arch/arm/boot/dts/armada-370-dlink-dns327l.dts
+++ b/arch/arm/boot/dts/armada-370-dlink-dns327l.dts
@@ -62,7 +62,7 @@
 		stdout-path = &uart0;
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x20000000>; /* 512 MiB */
 	};
@@ -72,20 +72,6 @@
 			MBUS_ID(0x01, 0xe0) 0 0xfff00000 0x100000
 			MBUS_ID(0x09, 0x01) 0 0xf1100000 0x10000>;
 
-		pcie-controller {
-			status = "okay";
-
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-
-			pcie@2,0 {
-				/* Port 1, Lane 0 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 			sata@a0000 {
 				nr-ports = <2>;
@@ -262,6 +248,20 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+
+	pcie@2,0 {
+		/* Port 1, Lane 0 */
+		status = "okay";
+	};
+};
+
 &pinctrl {
 	sata_l_white_pin: sata-l-white-pin {
 		marvell,pins = "mpp57";
diff --git a/arch/arm/boot/dts/armada-370-mirabox.dts b/arch/arm/boot/dts/armada-370-mirabox.dts
index d5e19cd4d256..a1425409e570 100644
--- a/arch/arm/boot/dts/armada-370-mirabox.dts
+++ b/arch/arm/boot/dts/armada-370-mirabox.dts
@@ -54,7 +54,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x20000000>; /* 512 MB */
 	};
@@ -64,22 +64,6 @@
 			  MBUS_ID(0x01, 0xe0) 0 0xfff00000 0x100000
 			  MBUS_ID(0x09, 0x01) 0 0xf1100000 0x10000>;
 
-		pcie-controller {
-			status = "okay";
-
-			/* Internal mini-PCIe connector */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-
-			/* Connected on the PCB to a USB 3.0 XHCI controller */
-			pcie@2,0 {
-				/* Port 1, Lane 0 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 			serial@12000 {
 				status = "okay";
@@ -113,17 +97,6 @@
 				};
 			};
 
-			mdio {
-				pinctrl-0 = <&mdio_pins>;
-				pinctrl-names = "default";
-				phy0: ethernet-phy@0 {
-					reg = <0>;
-				};
-
-				phy1: ethernet-phy@1 {
-					reg = <1>;
-				};
-			};
 			ethernet@70000 {
 				pinctrl-0 = <&ge0_rgmii_pins>;
 				pinctrl-names = "default";
@@ -197,6 +170,34 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+
+	/* Internal mini-PCIe connector */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+
+	/* Connected on the PCB to a USB 3.0 XHCI controller */
+	pcie@2,0 {
+		/* Port 1, Lane 0 */
+		status = "okay";
+	};
+};
+
+&mdio {
+	pinctrl-0 = <&mdio_pins>;
+	pinctrl-names = "default";
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+	};
+
+	phy1: ethernet-phy@1 {
+		reg = <1>;
+	};
+};
+
 &pinctrl {
 	pwr_led_pin: pwr-led-pin {
 		marvell,pins = "mpp63";
diff --git a/arch/arm/boot/dts/armada-370-netgear-rn102.dts b/arch/arm/boot/dts/armada-370-netgear-rn102.dts
index 39181b3fa90d..6bd9265f1062 100644
--- a/arch/arm/boot/dts/armada-370-netgear-rn102.dts
+++ b/arch/arm/boot/dts/armada-370-netgear-rn102.dts
@@ -56,7 +56,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x20000000>; /* 512 MB */
 	};
@@ -66,22 +66,6 @@
 			  MBUS_ID(0x01, 0xe0) 0 0xfff00000 0x100000
 			  MBUS_ID(0x09, 0x01) 0 0xf1100000 0x10000>;
 
-		pcie-controller {
-			status = "okay";
-
-			/* Connected to Marvell 88SE9170 SATA controller */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-
-			/* Connected to FL1009 USB 3.0 controller */
-			pcie@2,0 {
-				/* Port 1, Lane 0 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 
 			/* RTC is provided by Intersil ISL12057 I2C RTC chip */
@@ -99,14 +83,6 @@
 				status = "okay";
 			};
 
-			mdio {
-				pinctrl-0 = <&mdio_pins>;
-				pinctrl-names = "default";
-				phy0: ethernet-phy@0 { /* Marvell 88E1318 */
-					reg = <0>;
-				};
-			};
-
 			ethernet@74000 {
 				pinctrl-0 = <&ge1_rgmii_pins>;
 				pinctrl-names = "default";
@@ -120,8 +96,11 @@
 			};
 
 			i2c@11000 {
-				compatible = "marvell,mv64xxx-i2c";
 				clock-frequency = <100000>;
+
+				pinctrl-0 = <&i2c0_pins>;
+				pinctrl-names = "default";
+
 				status = "okay";
 
 				isl12057: isl12057@68 {
@@ -257,6 +236,30 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+
+	/* Connected to Marvell 88SE9170 SATA controller */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+
+	/* Connected to FL1009 USB 3.0 controller */
+	pcie@2,0 {
+		/* Port 1, Lane 0 */
+		status = "okay";
+	};
+};
+
+&mdio {
+	pinctrl-0 = <&mdio_pins>;
+	pinctrl-names = "default";
+	phy0: ethernet-phy@0 { /* Marvell 88E1318 */
+		reg = <0>;
+	};
+};
+
 &pinctrl {
 	power_led_pin: power-led-pin {
 		marvell,pins = "mpp57";
diff --git a/arch/arm/boot/dts/armada-370-netgear-rn104.dts b/arch/arm/boot/dts/armada-370-netgear-rn104.dts
index 11565752b9f6..c84ab5bf1e18 100644
--- a/arch/arm/boot/dts/armada-370-netgear-rn104.dts
+++ b/arch/arm/boot/dts/armada-370-netgear-rn104.dts
@@ -56,7 +56,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x20000000>; /* 512 MB */
 	};
@@ -66,22 +66,6 @@
 			  MBUS_ID(0x01, 0xe0) 0 0xfff00000 0x100000
 			  MBUS_ID(0x09, 0x01) 0 0xf1100000 0x10000>;
 
-		pcie-controller {
-			status = "okay";
-
-			/* Connected to FL1009 USB 3.0 controller */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-
-			/* Connected to Marvell 88SE9215 SATA controller */
-			pcie@2,0 {
-				/* Port 1, Lane 0 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 
 			/* RTC is provided by Intersil ISL12057 I2C RTC chip */
@@ -93,18 +77,6 @@
 				status = "okay";
 			};
 
-			mdio {
-				pinctrl-0 = <&mdio_pins>;
-				pinctrl-names = "default";
-				phy0: ethernet-phy@0 { /* Marvell 88E1318 */
-					reg = <0>;
-				};
-
-				phy1: ethernet-phy@1 { /* Marvell 88E1318 */
-					reg = <1>;
-				};
-			};
-
 			ethernet@70000 {
 				pinctrl-0 = <&ge0_rgmii_pins>;
 				pinctrl-names = "default";
@@ -126,8 +98,11 @@
 			};
 
 			i2c@11000 {
-				compatible = "marvell,mv64xxx-i2c";
 				clock-frequency = <100000>;
+
+				pinctrl-0 = <&i2c0_pins>;
+				pinctrl-names = "default";
+
 				status = "okay";
 
 				isl12057: isl12057@68 {
@@ -279,6 +254,34 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+
+	/* Connected to FL1009 USB 3.0 controller */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+
+	/* Connected to Marvell 88SE9215 SATA controller */
+	pcie@2,0 {
+		/* Port 1, Lane 0 */
+		status = "okay";
+	};
+};
+
+&mdio {
+	pinctrl-0 = <&mdio_pins>;
+	pinctrl-names = "default";
+	phy0: ethernet-phy@0 { /* Marvell 88E1318 */
+		reg = <0>;
+	};
+
+	phy1: ethernet-phy@1 { /* Marvell 88E1318 */
+		reg = <1>;
+	};
+};
+
 &pinctrl {
 	poweroff: poweroff {
 		marvell,pins = "mpp60";
diff --git a/arch/arm/boot/dts/armada-370-rd.dts b/arch/arm/boot/dts/armada-370-rd.dts
index fbef730e8d37..c3fd6e49212f 100644
--- a/arch/arm/boot/dts/armada-370-rd.dts
+++ b/arch/arm/boot/dts/armada-370-rd.dts
@@ -67,7 +67,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x20000000>; /* 512 MB */
 	};
@@ -77,22 +77,6 @@
 			  MBUS_ID(0x01, 0xe0) 0 0xfff00000 0x100000
 			  MBUS_ID(0x09, 0x01) 0 0xf1100000 0x10000>;
 
-		pcie-controller {
-			status = "okay";
-
-			/* Internal mini-PCIe connector */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-
-			/* Internal mini-PCIe connector */
-			pcie@2,0 {
-				/* Port 1, Lane 0 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 			serial@12000 {
 				status = "okay";
@@ -102,14 +86,6 @@
 				status = "okay";
 			};
 
-			mdio {
-				pinctrl-0 = <&mdio_pins>;
-				pinctrl-names = "default";
-				phy0: ethernet-phy@0 {
-					reg = <0>;
-				};
-			};
-
 			ethernet@70000 {
 				status = "okay";
 				phy = <&phy0>;
@@ -146,7 +122,7 @@
 				compatible = "gpio-keys";
 				#address-cells = <1>;
 				#size-cells = <0>;
-				button@1 {
+				button {
 					label = "Software Button";
 					linux,code = <KEY_POWER>;
 					gpios = <&gpio0 6 GPIO_ACTIVE_LOW>;
@@ -196,7 +172,7 @@
 		};
 	};
 
-	dsa@0 {
+	dsa {
 		compatible = "marvell,dsa";
 		#address-cells = <2>;
 		#size-cells = <0>;
@@ -235,7 +211,32 @@
 			};
 		};
 	 };
- };
+};
+
+&pciec {
+	status = "okay";
+
+	/* Internal mini-PCIe connector */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+
+	/* Internal mini-PCIe connector */
+	pcie@2,0 {
+		/* Port 1, Lane 0 */
+		status = "okay";
+	};
+};
+
+&mdio {
+	pinctrl-0 = <&mdio_pins>;
+	pinctrl-names = "default";
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+	};
+};
+
 
 &pinctrl {
 	fan_pins: fan-pins {
diff --git a/arch/arm/boot/dts/armada-370-seagate-nas-4bay.dts b/arch/arm/boot/dts/armada-370-seagate-nas-4bay.dts
index ae2e1fe50ef6..eb6af53b4954 100644
--- a/arch/arm/boot/dts/armada-370-seagate-nas-4bay.dts
+++ b/arch/arm/boot/dts/armada-370-seagate-nas-4bay.dts
@@ -28,20 +28,7 @@
 	compatible = "seagate,dart-4", "marvell,armada370", "marvell,armada-370-xp";
 
 	soc {
-		pcie-controller {
-			/* SATA AHCI controller 88SE9170 */
-			pcie@1,0 {
-				status = "okay";
-			};
-		};
-
 		internal-regs {
-			mdio {
-				phy1: ethernet-phy@1 {
-					reg = <1>;
-				};
-			};
-
 			ethernet@74000 {
 				status = "okay";
 				pinctrl-0 = <&ge1_rgmii_pins>;
@@ -131,3 +118,17 @@
 			  1300 0>;
 	};
 };
+
+&pciec {
+	/* SATA AHCI controller 88SE9170 */
+	pcie@1,0 {
+		status = "okay";
+	};
+};
+
+&mdio {
+	phy1: ethernet-phy@1 {
+		reg = <1>;
+	};
+};
+
diff --git a/arch/arm/boot/dts/armada-370-seagate-nas-xbay.dtsi b/arch/arm/boot/dts/armada-370-seagate-nas-xbay.dtsi
index 3036e25c5992..e9a5b952afc0 100644
--- a/arch/arm/boot/dts/armada-370-seagate-nas-xbay.dtsi
+++ b/arch/arm/boot/dts/armada-370-seagate-nas-xbay.dtsi
@@ -23,7 +23,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x20000000>; /* 512 MB */
 	};
@@ -32,15 +32,6 @@
 		ranges = <MBUS_ID(0xf0, 0x01) 0 0xf1000000 0x100000
 			  MBUS_ID(0x01, 0xe0) 0 0xfff00000 0x100000>;
 
-		pcie-controller {
-			status = "okay";
-
-			/* USB 3.0 bridge ASM1042A */
-			pcie@2,0 {
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 			serial@12000 {
 				status = "okay";
@@ -51,15 +42,6 @@
 				status = "okay";
 			};
 
-			mdio {
-				pinctrl-0 = <&mdio_pins>;
-				pinctrl-names = "default";
-
-				phy0: ethernet-phy@0 {
-					reg = <0>;
-				};
-			};
-
 			ethernet@70000 {
 				status = "okay";
 				pinctrl-0 = <&ge0_rgmii_pins>;
@@ -159,19 +141,19 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		button@1 {
+		power {
 			label = "Power button";
 			linux,code = <KEY_POWER>;
 			gpios = <&gpio1 19 GPIO_ACTIVE_LOW>;
 			debounce-interval = <100>;
 		};
-		button@2 {
+		backup {
 			label = "Backup button";
 			linux,code = <KEY_OPTION>;
 			gpios = <&gpio0 31 GPIO_ACTIVE_LOW>;
 			debounce-interval = <100>;
 		};
-		button@3 {
+		reset {
 			label = "Reset Button";
 			linux,code = <KEY_RESTART>;
 			gpios = <&gpio1 23 GPIO_ACTIVE_LOW>;
@@ -208,6 +190,25 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+
+	/* USB 3.0 bridge ASM1042A */
+	pcie@2,0 {
+		status = "okay";
+	};
+};
+
+
+&mdio {
+	pinctrl-0 = <&mdio_pins>;
+	pinctrl-names = "default";
+
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+	};
+};
+
 &pinctrl {
 	pinctrl-0 = <&hdd0_led_sata_pin>, <&hdd1_led_sata_pin>;
 	pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/armada-370-seagate-personal-cloud.dtsi b/arch/arm/boot/dts/armada-370-seagate-personal-cloud.dtsi
index 01cded310cbc..d079a89ee5a2 100644
--- a/arch/arm/boot/dts/armada-370-seagate-personal-cloud.dtsi
+++ b/arch/arm/boot/dts/armada-370-seagate-personal-cloud.dtsi
@@ -24,7 +24,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x20000000>; /* 512 MB */
 	};
@@ -33,15 +33,6 @@
 		ranges = <MBUS_ID(0xf0, 0x01) 0 0xf1000000 0x100000
 			  MBUS_ID(0x01, 0xe0) 0 0xfff00000 0x100000>;
 
-		pcie-controller {
-			status = "okay";
-
-			/* USB 3.0 Bridge ASM1042A */
-			pcie@1,0 {
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 			coherency-fabric@20200 {
 				broken-idle;
@@ -51,15 +42,6 @@
 				status = "okay";
 			};
 
-			mdio {
-				pinctrl-0 = <&mdio_pins>;
-				pinctrl-names = "default";
-
-				phy0: ethernet-phy@0 {
-					reg = <0>;
-				};
-			};
-
 			ethernet@74000 {
 				status = "okay";
 				pinctrl-0 = <&ge1_rgmii_pins>;
@@ -107,19 +89,19 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		button@1 {
+		power {
 			label = "Power button";
 			linux,code = <KEY_POWER>;
 			gpios = <&gpio1 19 GPIO_ACTIVE_HIGH>;
 			debounce-interval = <100>;
 		};
-		button@2 {
+		reset {
 			label = "Reset Button";
 			linux,code = <KEY_RESTART>;
 			gpios = <&gpio1 23 GPIO_ACTIVE_LOW>;
 			debounce-interval = <100>;
 		};
-		button@3 {
+		button {
 			label = "USB VBUS error";
 			linux,code = <KEY_UNKNOWN>;
 			gpios = <&gpio1 21 GPIO_ACTIVE_LOW>;
@@ -143,6 +125,24 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+
+	/* USB 3.0 Bridge ASM1042A */
+	pcie@1,0 {
+		status = "okay";
+	};
+};
+
+&mdio {
+	pinctrl-0 = <&mdio_pins>;
+	pinctrl-names = "default";
+
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+	};
+};
+
 &pinctrl {
 	pinctrl-0 = <&sata_led_pin>;
 	pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/armada-370-synology-ds213j.dts b/arch/arm/boot/dts/armada-370-synology-ds213j.dts
index a9cc42776874..99f9de229ea8 100644
--- a/arch/arm/boot/dts/armada-370-synology-ds213j.dts
+++ b/arch/arm/boot/dts/armada-370-synology-ds213j.dts
@@ -70,7 +70,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x20000000>; /* 512 MB */
 	};
@@ -127,12 +127,6 @@
 				status = "okay";
 			};
 
-			mdio {
-				phy1: ethernet-phy@1 { /* Marvell 88E1512 */
-					reg = <1>;
-				};
-			};
-
 			ethernet@70000 {
 			       status = "okay";
 			       phy = <&phy1>;
@@ -192,7 +186,7 @@
 		pinctrl-0 = <&sata1_pwr_pin &sata2_pwr_pin>;
 		pinctrl-names = "default";
 
-		sata1_regulator: sata1-regulator {
+		sata1_regulator: sata1-regulator@1 {
 			compatible = "regulator-fixed";
 			reg = <1>;
 			regulator-name = "SATA1 Power";
@@ -205,7 +199,7 @@
 			gpio = <&gpio1 5 GPIO_ACTIVE_HIGH>;
 		};
 
-		sata2_regulator: sata2-regulator {
+		sata2_regulator: sata2-regulator@2 {
 			compatible = "regulator-fixed";
 			reg = <2>;
 			regulator-name = "SATA2 Power";
@@ -220,6 +214,12 @@
 	};
 };
 
+&mdio {
+	phy1: ethernet-phy@1 { /* Marvell 88E1512 */
+		reg = <1>;
+	};
+};
+
 &pinctrl {
 	disk1_led_pin: disk1-led-pin {
 		marvell,pins = "mpp31";
diff --git a/arch/arm/boot/dts/armada-370-xp.dtsi b/arch/arm/boot/dts/armada-370-xp.dtsi
index 3ccedc9dffb2..b0520bdeea27 100644
--- a/arch/arm/boot/dts/armada-370-xp.dtsi
+++ b/arch/arm/boot/dts/armada-370-xp.dtsi
@@ -50,8 +50,6 @@
  * 370 and Armada XP SoC.
  */
 
-/include/ "skeleton64.dtsi"
-
 #define MBUS_ID(target,attributes) (((target) << 24) | ((attributes) << 16))
 
 / {
@@ -86,7 +84,7 @@
 		pcie-mem-aperture = <0xf8000000 0x7e00000>;
 		pcie-io-aperture  = <0xffe00000 0x100000>;
 
-		devbus-bootcs {
+		devbus_bootcs: devbus-bootcs {
 			compatible = "marvell,mvebu-devbus";
 			reg = <MBUS_ID(0xf0, 0x01) 0x10400 0x8>;
 			ranges = <0 MBUS_ID(0x01, 0x2f) 0 0xffffffff>;
@@ -96,7 +94,7 @@
 			status = "disabled";
 		};
 
-		devbus-cs0 {
+		devbus_cs0: devbus-cs0 {
 			compatible = "marvell,mvebu-devbus";
 			reg = <MBUS_ID(0xf0, 0x01) 0x10408 0x8>;
 			ranges = <0 MBUS_ID(0x01, 0x3e) 0 0xffffffff>;
@@ -106,7 +104,7 @@
 			status = "disabled";
 		};
 
-		devbus-cs1 {
+		devbus_cs1: devbus-cs1 {
 			compatible = "marvell,mvebu-devbus";
 			reg = <MBUS_ID(0xf0, 0x01) 0x10410 0x8>;
 			ranges = <0 MBUS_ID(0x01, 0x3d) 0 0xffffffff>;
@@ -116,7 +114,7 @@
 			status = "disabled";
 		};
 
-		devbus-cs2 {
+		devbus_cs2: devbus-cs2 {
 			compatible = "marvell,mvebu-devbus";
 			reg = <MBUS_ID(0xf0, 0x01) 0x10418 0x8>;
 			ranges = <0 MBUS_ID(0x01, 0x3b) 0 0xffffffff>;
@@ -126,7 +124,7 @@
 			status = "disabled";
 		};
 
-		devbus-cs3 {
+		devbus_cs3: devbus-cs3 {
 			compatible = "marvell,mvebu-devbus";
 			reg = <MBUS_ID(0xf0, 0x01) 0x10420 0x8>;
 			ranges = <0 MBUS_ID(0x01, 0x37) 0 0xffffffff>;
@@ -142,7 +140,7 @@
 			#size-cells = <1>;
 			ranges = <0 MBUS_ID(0xf0, 0x01) 0 0x100000>;
 
-			rtc@10300 {
+			rtc: rtc@10300 {
 				compatible = "marvell,orion-rtc";
 				reg = <0x10300 0x20>;
 				interrupts = <50>;
@@ -214,33 +212,38 @@
 				msi-controller;
 			};
 
-			coherency-fabric@20200 {
+			coherencyfab: coherency-fabric@20200 {
 				compatible = "marvell,coherency-fabric";
 				reg = <0x20200 0xb0>, <0x21010 0x1c>;
 			};
 
-			timer@20300 {
+			timer: timer@20300 {
 				reg = <0x20300 0x30>, <0x21040 0x30>;
 				interrupts = <37>, <38>, <39>, <40>, <5>, <6>;
 			};
 
-			watchdog@20300 {
+			watchdog: watchdog@20300 {
 				reg = <0x20300 0x34>, <0x20704 0x4>;
 			};
 
-			pmsu@22000 {
+			cpurst: cpurst@20800 {
+				compatible = "marvell,armada-370-cpu-reset";
+				reg = <0x20800 0x8>;
+			};
+
+			pmsu: pmsu@22000 {
 				compatible = "marvell,armada-370-pmsu";
 				reg = <0x22000 0x1000>;
 			};
 
-			usb@50000 {
+			usb0: usb@50000 {
 				compatible = "marvell,orion-ehci";
 				reg = <0x50000 0x500>;
 				interrupts = <45>;
 				status = "disabled";
 			};
 
-			usb@51000 {
+			usb1: usb@51000 {
 				compatible = "marvell,orion-ehci";
 				reg = <0x51000 0x500>;
 				interrupts = <46>;
@@ -254,7 +257,7 @@
 				status = "disabled";
 			};
 
-			mdio: mdio {
+			mdio: mdio@72004 {
 				#address-cells = <1>;
 				#size-cells = <0>;
 				compatible = "marvell,orion-mdio";
@@ -269,7 +272,7 @@
 				status = "disabled";
 			};
 
-			sata@a0000 {
+			sata: sata@a0000 {
 				compatible = "marvell,armada-370-sata";
 				reg = <0xa0000 0x5000>;
 				interrupts = <55>;
@@ -278,7 +281,7 @@
 				status = "disabled";
 			};
 
-			nand@d0000 {
+			nand: nand@d0000 {
 				compatible = "marvell,armada370-nand";
 				reg = <0xd0000 0x54>;
 				#address-cells = <1>;
@@ -288,7 +291,7 @@
 				status = "disabled";
 			};
 
-			mvsdio@d4000 {
+			sdio: mvsdio@d4000 {
 				compatible = "marvell,orion-sdio";
 				reg = <0xd4000 0x200>;
 				interrupts = <54>;
diff --git a/arch/arm/boot/dts/armada-370.dtsi b/arch/arm/boot/dts/armada-370.dtsi
index b4258105e91f..b704bcc597f7 100644
--- a/arch/arm/boot/dts/armada-370.dtsi
+++ b/arch/arm/boot/dts/armada-370.dtsi
@@ -50,9 +50,11 @@
  */
 
 #include "armada-370-xp.dtsi"
-/include/ "skeleton.dtsi"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	model = "Marvell Armada 370 family SoC";
 	compatible = "marvell,armada370", "marvell,armada-370-xp";
 
@@ -70,7 +72,7 @@
 			reg = <MBUS_ID(0x01, 0xe0) 0 0x100000>;
 		};
 
-		pcie-controller {
+		pciec: pcie-controller@82000000 {
 			compatible = "marvell,armada-370-pcie";
 			status = "disabled";
 			device_type = "pci";
@@ -89,7 +91,7 @@
 				0x82000000 0x2 0     MBUS_ID(0x08, 0xe8) 0       1 0 /* Port 1.0 MEM */
 				0x81000000 0x2 0     MBUS_ID(0x08, 0xe0) 0       1 0 /* Port 1.0 IO  */>;
 
-			pcie@1,0 {
+			pcie0: pcie@1,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x40000 0 0x2000>;
 				reg = <0x0800 0 0 0 0>;
@@ -106,7 +108,7 @@
 				status = "disabled";
 			};
 
-			pcie@2,0 {
+			pcie2: pcie@2,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82002800 0 0x80000 0 0x2000>;
 				reg = <0x1000 0 0 0 0>;
@@ -125,7 +127,7 @@
 		};
 
 		internal-regs {
-			L2: l2-cache {
+			L2: l2-cache@8000 {
 				compatible = "marvell,aurora-outer-cache";
 				reg = <0x08000 0x1000>;
 				cache-id-part = <0x100>;
@@ -134,14 +136,6 @@
 				wt-override;
 			};
 
-			i2c0: i2c@11000 {
-				reg = <0x11000 0x20>;
-			};
-
-			i2c1: i2c@11100 {
-				reg = <0x11100 0x20>;
-			};
-
 			gpio0: gpio@18100 {
 				compatible = "marvell,orion-gpio";
 				reg = <0x18100 0x40>;
@@ -175,22 +169,8 @@
 				interrupts = <91>;
 			};
 
-			/*
-			 * Default UART pinctrl setting without RTS/CTS, can
-			 * be overwritten on board level if a different
-			 * configuration is used.
-			 */
-			uart0: serial@12000 {
-				pinctrl-0 = <&uart0_pins>;
-				pinctrl-names = "default";
-			};
-
-			uart1: serial@12100 {
-				pinctrl-0 = <&uart1_pins>;
-				pinctrl-names = "default";
-			};
 
-			system-controller@18200 {
+			systemc: system-controller@18200 {
 				compatible = "marvell,armada-370-xp-system-controller";
 				reg = <0x18200 0x100>;
 			};
@@ -208,37 +188,18 @@
 				#clock-cells = <1>;
 			};
 
-			thermal@18300 {
+			thermal: thermal@18300 {
 				compatible = "marvell,armada370-thermal";
 				reg = <0x18300 0x4
 					0x18304 0x4>;
 				status = "okay";
 			};
 
-			sscg@18330 {
+			sscg: sscg@18330 {
 				reg = <0x18330 0x4>;
 			};
 
-			interrupt-controller@20a00 {
-				reg = <0x20a00 0x1d0>, <0x21870 0x58>;
-			};
-
-			timer@20300 {
-				compatible = "marvell,armada-370-timer";
-				clocks = <&coreclk 2>;
-			};
-
-			watchdog@20300 {
-				compatible = "marvell,armada-370-wdt";
-				clocks = <&coreclk 2>;
-			};
-
-			cpurst@20800 {
-				compatible = "marvell,armada-370-cpu-reset";
-				reg = <0x20800 0x8>;
-			};
-
-			cpu-config@21000 {
+			cpuconf: cpu-config@21000 {
 				compatible = "marvell,armada-370-cpu-config";
 				reg = <0x21000 0x8>;
 			};
@@ -253,15 +214,7 @@
 				status = "disabled";
 			};
 
-			usb@50000 {
-				clocks = <&coreclk 0>;
-			};
-
-			usb@51000 {
-				clocks = <&coreclk 0>;
-			};
-
-			xor@60800 {
+			xor0: xor@60800 {
 				compatible = "marvell,orion-xor";
 				reg = <0x60800 0x100
 				       0x60A00 0x100>;
@@ -280,7 +233,7 @@
 				};
 			};
 
-			xor@60900 {
+			xor1: xor@60900 {
 				compatible = "marvell,orion-xor";
 				reg = <0x60900 0x100
 				       0x60b00 0x100>;
@@ -299,15 +252,7 @@
 				};
 			};
 
-			ethernet@70000 {
-				compatible = "marvell,armada-370-neta";
-			};
-
-			ethernet@74000 {
-				compatible = "marvell,armada-370-neta";
-			};
-
-			crypto@90000 {
+			cesa: crypto@90000 {
 				compatible = "marvell,armada-370-crypto";
 				reg = <0x90000 0x10000>;
 				reg-names = "regs";
@@ -342,6 +287,59 @@
 	};
 };
 
+/*
+ * Default UART pinctrl setting without RTS/CTS, can be overwritten on
+ * board level if a different configuration is used.
+ */
+
+&uart0 {
+	pinctrl-0 = <&uart0_pins>;
+	pinctrl-names = "default";
+};
+
+&uart1 {
+	pinctrl-0 = <&uart1_pins>;
+	pinctrl-names = "default";
+};
+
+&i2c0 {
+	reg = <0x11000 0x20>;
+};
+
+&i2c1 {
+	reg = <0x11100 0x20>;
+};
+
+&mpic {
+	reg = <0x20a00 0x1d0>, <0x21870 0x58>;
+};
+
+&timer {
+	compatible = "marvell,armada-370-timer";
+	clocks = <&coreclk 2>;
+};
+
+&watchdog {
+	compatible = "marvell,armada-370-wdt";
+	clocks = <&coreclk 2>;
+};
+
+&usb0 {
+	clocks = <&coreclk 0>;
+};
+
+&usb1 {
+	clocks = <&coreclk 0>;
+};
+
+&eth0 {
+	compatible = "marvell,armada-370-neta";
+};
+
+&eth1 {
+	compatible = "marvell,armada-370-neta";
+};
+
 &pinctrl {
 	compatible = "marvell,mv88f6710-pinctrl";
 
diff --git a/arch/arm/boot/dts/armada-375-db.dts b/arch/arm/boot/dts/armada-375-db.dts
index cded5f0a262d..ef45cbeb3e7d 100644
--- a/arch/arm/boot/dts/armada-375-db.dts
+++ b/arch/arm/boot/dts/armada-375-db.dts
@@ -58,7 +58,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x40000000>; /* 1 GB */
 	};
@@ -69,138 +69,141 @@
 			  MBUS_ID(0x09, 0x09) 0 0xf1100000 0x10000
 			  MBUS_ID(0x09, 0x05) 0 0xf1110000 0x10000>;
 
-		internal-regs {
-			spi@10600 {
-				pinctrl-0 = <&spi0_pins>;
-				pinctrl-names = "default";
-				/*
-				 * SPI conflicts with NAND, so we disable it
-				 * here, and select NAND as the enabled device
-				 * by default.
-				 */
-				status = "disabled";
-
-				spi-flash@0 {
-					#address-cells = <1>;
-					#size-cells = <1>;
-					compatible = "n25q128a13", "jedec,spi-nor";
-					reg = <0>; /* Chip select 0 */
-					spi-max-frequency = <108000000>;
-				};
-			};
-
-			i2c@11000 {
-				status = "okay";
-				clock-frequency = <100000>;
-				pinctrl-0 = <&i2c0_pins>;
-				pinctrl-names = "default";
-			};
-
-			i2c@11100 {
-				status = "okay";
-				clock-frequency = <100000>;
-				pinctrl-0 = <&i2c1_pins>;
-				pinctrl-names = "default";
-			};
-
-			serial@12000 {
-				status = "okay";
-			};
-
-			pinctrl {
-				sdio_st_pins: sdio-st-pins {
-					marvell,pins = "mpp44", "mpp45";
-					marvell,function = "gpio";
-				};
-			};
-
-			sata@a0000 {
-				status = "okay";
-				nr-ports = <2>;
-			};
-
-			nand: nand@d0000 {
-				pinctrl-0 = <&nand_pins>;
-				pinctrl-names = "default";
-				status = "okay";
-				num-cs = <1>;
-				marvell,nand-keep-config;
-				marvell,nand-enable-arbiter;
-				nand-on-flash-bbt;
-				nand-ecc-strength = <4>;
-				nand-ecc-step-size = <512>;
-
-				partition@0 {
-					label = "U-Boot";
-					reg = <0 0x800000>;
-				};
-				partition@800000 {
-					label = "Linux";
-					reg = <0x800000 0x800000>;
-				};
-				partition@1000000 {
-					label = "Filesystem";
-					reg = <0x1000000 0x3f000000>;
-				};
-			};
-
-			usb@54000 {
-				status = "okay";
-			};
-
-			usb3@58000 {
-				status = "okay";
-			};
-
-			mvsdio@d4000 {
-				pinctrl-0 = <&sdio_pins &sdio_st_pins>;
-				pinctrl-names = "default";
-				status = "okay";
-				cd-gpios = <&gpio1 12 GPIO_ACTIVE_HIGH>;
-				wp-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>;
-			};
-
-			mdio {
-				phy0: ethernet-phy@0 {
-					reg = <0>;
-				};
-
-				phy3: ethernet-phy@3 {
-					reg = <3>;
-				};
-			};
-
-			ethernet@f0000 {
-				status = "okay";
-
-				eth0@c4000 {
-					status = "okay";
-					phy = <&phy0>;
-					phy-mode = "rgmii-id";
-				};
-
-				eth1@c5000 {
-					status = "okay";
-					phy = <&phy3>;
-					phy-mode = "gmii";
-				};
-			};
-		};
-
-		pcie-controller {
-			status = "okay";
-			/*
-			 * The two PCIe units are accessible through
-			 * standard PCIe slots on the board.
-			 */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-			pcie@2,0 {
-				/* Port 1, Lane 0 */
-				status = "okay";
-			};
-		};
 	};
 };
+&pciec {
+	status = "okay";
+};
+
+/*
+ * The two PCIe units are accessible through
+ * standard PCIe slots on the board.
+ */
+&pcie0 {
+	/* Port 0, Lane 0 */
+	status = "okay";
+};
+
+&pcie1 {
+	/* Port 1, Lane 0 */
+	status = "okay";
+};
+
+
+&spi0 {
+	pinctrl-0 = <&spi0_pins>;
+	pinctrl-names = "default";
+
+	/*
+	 * SPI conflicts with NAND, so we disable it here, and
+	 * select NAND as the enabled device by default.
+	 */
+
+	status = "disabled";
+
+	spi-flash@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "n25q128a13", "jedec,spi-nor";
+		reg = <0>; /* Chip select 0 */
+		spi-max-frequency = <108000000>;
+	};
+};
+
+&i2c0 {
+	status = "okay";
+	clock-frequency = <100000>;
+	pinctrl-0 = <&i2c0_pins>;
+	pinctrl-names = "default";
+};
+
+&i2c1 {
+	status = "okay";
+	clock-frequency = <100000>;
+	pinctrl-0 = <&i2c1_pins>;
+	pinctrl-names = "default";
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&pinctrl {
+	sdio_st_pins: sdio-st-pins {
+		marvell,pins = "mpp44", "mpp45";
+		marvell,function = "gpio";
+	};
+};
+
+&sata {
+	status = "okay";
+	nr-ports = <2>;
+};
+
+&nand {
+	pinctrl-0 = <&nand_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+	num-cs = <1>;
+	marvell,nand-keep-config;
+	marvell,nand-enable-arbiter;
+	nand-on-flash-bbt;
+	nand-ecc-strength = <4>;
+	nand-ecc-step-size = <512>;
+
+	partition@0 {
+		label = "U-Boot";
+		reg = <0 0x800000>;
+	};
+	partition@800000 {
+		label = "Linux";
+		reg = <0x800000 0x800000>;
+	};
+	partition@1000000 {
+		label = "Filesystem";
+		reg = <0x1000000 0x3f000000>;
+	};
+};
+
+&usb1 {
+	status = "okay";
+};
+
+&usb2 {
+	status = "okay";
+};
+
+&sdio {
+	pinctrl-0 = <&sdio_pins &sdio_st_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+	cd-gpios = <&gpio1 12 GPIO_ACTIVE_HIGH>;
+	wp-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>;
+};
+
+&mdio {
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+	};
+
+	phy3: ethernet-phy@3 {
+		reg = <3>;
+	};
+};
+
+&ethernet {
+	status = "okay";
+};
+
+
+&eth0 {
+	status = "okay";
+	phy = <&phy0>;
+	phy-mode = "rgmii-id";
+};
+
+&eth1 {
+	status = "okay";
+	phy = <&phy3>;
+	phy-mode = "gmii";
+};
diff --git a/arch/arm/boot/dts/armada-375.dtsi b/arch/arm/boot/dts/armada-375.dtsi
index cc952cf8ec30..f515591e8733 100644
--- a/arch/arm/boot/dts/armada-375.dtsi
+++ b/arch/arm/boot/dts/armada-375.dtsi
@@ -45,7 +45,6 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "skeleton.dtsi"
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/phy/phy.h>
@@ -53,6 +52,9 @@
 #define MBUS_ID(target,attributes) (((target) << 24) | ((attributes) << 16))
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	model = "Marvell Armada 375 family SoC";
 	compatible = "marvell,armada375";
 
@@ -65,7 +67,7 @@
 	};
 
 	clocks {
-		/* 2 GHz fixed main PLL */
+		/* 1 GHz fixed main PLL */
 		mainpll: mainpll {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
@@ -84,12 +86,12 @@
 		#size-cells = <0>;
 		enable-method = "marvell,armada-375-smp";
 
-		cpu@0 {
+		cpu0: cpu@0 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <0>;
 		};
-		cpu@1 {
+		cpu1: cpu@1 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <1>;
@@ -115,7 +117,7 @@
 			reg = <MBUS_ID(0x01, 0x1d) 0 0x100000>;
 		};
 
-		devbus-bootcs {
+		devbus_bootcs: devbus-bootcs {
 			compatible = "marvell,mvebu-devbus";
 			reg = <MBUS_ID(0xf0, 0x01) 0x10400 0x8>;
 			ranges = <0 MBUS_ID(0x01, 0x2f) 0 0xffffffff>;
@@ -125,7 +127,7 @@
 			status = "disabled";
 		};
 
-		devbus-cs0 {
+		devbus_cs0: devbus-cs0 {
 			compatible = "marvell,mvebu-devbus";
 			reg = <MBUS_ID(0xf0, 0x01) 0x10408 0x8>;
 			ranges = <0 MBUS_ID(0x01, 0x3e) 0 0xffffffff>;
@@ -135,7 +137,7 @@
 			status = "disabled";
 		};
 
-		devbus-cs1 {
+		devbus_cs1: devbus-cs1 {
 			compatible = "marvell,mvebu-devbus";
 			reg = <MBUS_ID(0xf0, 0x01) 0x10410 0x8>;
 			ranges = <0 MBUS_ID(0x01, 0x3d) 0 0xffffffff>;
@@ -145,7 +147,7 @@
 			status = "disabled";
 		};
 
-		devbus-cs2 {
+		devbus_cs2: devbus-cs2 {
 			compatible = "marvell,mvebu-devbus";
 			reg = <MBUS_ID(0xf0, 0x01) 0x10418 0x8>;
 			ranges = <0 MBUS_ID(0x01, 0x3b) 0 0xffffffff>;
@@ -155,7 +157,7 @@
 			status = "disabled";
 		};
 
-		devbus-cs3 {
+		devbus_cs3: devbus-cs3 {
 			compatible = "marvell,mvebu-devbus";
 			reg = <MBUS_ID(0xf0, 0x01) 0x10420 0x8>;
 			ranges = <0 MBUS_ID(0x01, 0x37) 0 0xffffffff>;
@@ -182,12 +184,12 @@
 				prefetch-data = <1>;
 			};
 
-			scu@c000 {
+			scu: scu@c000 {
 				compatible = "arm,cortex-a9-scu";
 				reg = <0xc000 0x58>;
 			};
 
-			timer@c600 {
+			timer0: timer@c600 {
 				compatible = "arm,cortex-a9-twd-timer";
 				reg = <0xc600 0x20>;
 				interrupts = <GIC_PPI 13 (IRQ_TYPE_EDGE_RISING | GIC_CPU_MASK_SIMPLE(2))>;
@@ -203,7 +205,7 @@
 				      <0xc100 0x100>;
 			};
 
-			mdio {
+			mdio: mdio@c0054 {
 				#address-cells = <1>;
 				#size-cells = <0>;
 				compatible = "marvell,orion-mdio";
@@ -212,7 +214,7 @@
 			};
 
 			/* Network controller */
-			ethernet@f0000 {
+			ethernet: ethernet@f0000 {
 				compatible = "marvell,armada-375-pp2";
 				reg = <0xf0000 0xa000>, /* Packet Processor regs */
 				      <0xc0000 0x3060>, /* LMS regs */
@@ -222,20 +224,20 @@
 				clock-names = "pp_clk", "gop_clk";
 				status = "disabled";
 
-				eth0: eth0@c4000 {
+				eth0: eth0 {
 					interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
 					port-id = <0>;
 					status = "disabled";
 				};
 
-				eth1: eth1@c5000 {
+				eth1: eth1 {
 					interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
 					port-id = <1>;
 					status = "disabled";
 				};
 			};
 
-			rtc@10300 {
+			rtc: rtc@10300 {
 				compatible = "marvell,orion-rtc";
 				reg = <0x10300 0x20>;
 				interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
@@ -307,7 +309,7 @@
 				status = "disabled";
 			};
 
-			pinctrl {
+			pinctrl: pinctrl@18000 {
 				compatible = "marvell,mv88f6720-pinctrl";
 				reg = <0x18000 0x24>;
 
@@ -382,7 +384,7 @@
 				interrupts = <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>;
 			};
 
-			system-controller@18200 {
+			systemc: system-controller@18200 {
 				compatible = "marvell,armada-375-system-controller";
 				reg = <0x18200 0x100>;
 			};
@@ -415,7 +417,7 @@
 				interrupts = <GIC_PPI 15 IRQ_TYPE_LEVEL_HIGH>;
 			};
 
-			timer@20300 {
+			timer1: timer@20300 {
 				compatible = "marvell,armada-375-timer", "marvell,armada-370-timer";
 				reg = <0x20300 0x30>, <0x21040 0x30>;
 				interrupts-extended = <&gic  GIC_SPI  8 IRQ_TYPE_LEVEL_HIGH>,
@@ -428,24 +430,24 @@
 				clock-names = "nbclk", "fixed";
 			};
 
-			watchdog@20300 {
+			watchdog: watchdog@20300 {
 				compatible = "marvell,armada-375-wdt";
 				reg = <0x20300 0x34>, <0x20704 0x4>, <0x18254 0x4>;
 				clocks = <&coreclk 0>, <&refclk>;
 				clock-names = "nbclk", "fixed";
 			};
 
-			cpurst@20800 {
+			cpurst: cpurst@20800 {
 				compatible = "marvell,armada-370-cpu-reset";
 				reg = <0x20800 0x10>;
 			};
 
-			coherency-fabric@21010 {
+			coherencyfab: coherency-fabric@21010 {
 				compatible = "marvell,armada-375-coherency-fabric";
 				reg = <0x21010 0x1c>;
 			};
 
-			usb@50000 {
+			usb0: usb@50000 {
 				compatible = "marvell,orion-ehci";
 				reg = <0x50000 0x500>;
 				interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
@@ -455,7 +457,7 @@
 				status = "disabled";
 			};
 
-			usb@54000 {
+			usb1: usb@54000 {
 				compatible = "marvell,orion-ehci";
 				reg = <0x54000 0x500>;
 				interrupts = <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>;
@@ -463,7 +465,7 @@
 				status = "disabled";
 			};
 
-			usb3@58000 {
+			usb2: usb3@58000 {
 				compatible = "marvell,armada-375-xhci";
 				reg = <0x58000 0x20000>,<0x5b880 0x80>;
 				interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
@@ -473,7 +475,7 @@
 				status = "disabled";
 			};
 
-			xor@60800 {
+			xor0: xor@60800 {
 				compatible = "marvell,orion-xor";
 				reg = <0x60800 0x100
 				       0x60A00 0x100>;
@@ -493,7 +495,7 @@
 				};
 			};
 
-			xor@60900 {
+			xor1: xor@60900 {
 				compatible = "marvell,orion-xor";
 				reg = <0x60900 0x100
 				       0x60b00 0x100>;
@@ -513,7 +515,7 @@
 				};
 			};
 
-			crypto@90000 {
+			cesa: crypto@90000 {
 				compatible = "marvell,armada-375-crypto";
 				reg = <0x90000 0x10000>;
 				reg-names = "regs";
@@ -528,7 +530,7 @@
 				marvell,crypto-sram-size = <0x800>;
 			};
 
-			sata@a0000 {
+			sata: sata@a0000 {
 				compatible = "marvell,armada-370-sata";
 				reg = <0xa0000 0x5000>;
 				interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
@@ -537,7 +539,7 @@
 				status = "disabled";
 			};
 
-			nand@d0000 {
+			nand: nand@d0000 {
 				compatible = "marvell,armada370-nand";
 				reg = <0xd0000 0x54>;
 				#address-cells = <1>;
@@ -547,7 +549,7 @@
 				status = "disabled";
 			};
 
-			mvsdio@d4000 {
+			sdio: mvsdio@d4000 {
 				compatible = "marvell,orion-sdio";
 				reg = <0xd4000 0x200>;
 				interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
@@ -559,7 +561,7 @@
 				status = "disabled";
 			};
 
-			thermal@e8078 {
+			thermal: thermal@e8078 {
 				compatible = "marvell,armada375-thermal";
 				reg = <0xe8078 0x4>, <0xe807c 0x8>;
 				status = "okay";
@@ -580,7 +582,7 @@
 			};
 		};
 
-		pcie-controller {
+		pciec: pcie-controller@82000000 {
 			compatible = "marvell,armada-370-pcie";
 			status = "disabled";
 			device_type = "pci";
@@ -599,7 +601,7 @@
 				0x82000000 0x2 0       MBUS_ID(0x04, 0xd8) 0 1 0 /* Port 1 MEM */
 				0x81000000 0x2 0       MBUS_ID(0x04, 0xd0) 0 1 0 /* Port 1 IO  */>;
 
-			pcie@1,0 {
+			pcie0: pcie@1,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x40000 0 0x2000>;
 				reg = <0x0800 0 0 0 0>;
@@ -616,7 +618,7 @@
 				status = "disabled";
 			};
 
-			pcie@2,0 {
+			pcie1: pcie@2,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x44000 0 0x2000>;
 				reg = <0x1000 0 0 0 0>;
diff --git a/arch/arm/boot/dts/armada-385-turris-omnia.dts b/arch/arm/boot/dts/armada-385-turris-omnia.dts
new file mode 100644
index 000000000000..ab49acb2d452
--- /dev/null
+++ b/arch/arm/boot/dts/armada-385-turris-omnia.dts
@@ -0,0 +1,340 @@
+/*
+ * Device Tree file for the Turris Omnia
+ *
+ * Copyright (C) 2016 Uwe Kleine-König <uwe@kleine-koenig.org>
+ * Copyright (C) 2016 Tomas Hlavacek <tmshlvkc@gmail.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is licensed under the terms of the GNU General Public
+ *     License version 2.  This program is licensed "as is" without
+ *     any warranty of any kind, whether express or implied.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * Schematic available at https://www.turris.cz/doc/_media/rtrom01-schema.pdf
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include "armada-385.dtsi"
+
+/ {
+	model = "Turris Omnia";
+	compatible = "cznic,turris-omnia", "marvell,armada385", "marvell,armada380";
+
+	chosen {
+		stdout-path = &uart0;
+	};
+
+	memory {
+		device_type = "memory";
+		reg = <0x00000000 0x40000000>; /* 1024 MB */
+	};
+
+	soc {
+		ranges = <MBUS_ID(0xf0, 0x01) 0 0xf1000000 0x100000
+			  MBUS_ID(0x01, 0x1d) 0 0xfff00000 0x100000
+			  MBUS_ID(0x09, 0x19) 0 0xf1100000 0x10000
+			  MBUS_ID(0x09, 0x15) 0 0xf1110000 0x10000>;
+
+		internal-regs {
+
+			/* USB part of the PCIe2/USB 2.0 port */
+			usb@58000 {
+				status = "okay";
+			};
+
+			sata@a8000 {
+				status = "okay";
+			};
+
+			sdhci@d8000 {
+				pinctrl-names = "default";
+				pinctrl-0 = <&sdhci_pins>;
+				status = "okay";
+
+				bus-width = <8>;
+				no-1-8-v;
+				non-removable;
+			};
+
+			usb3@f0000 {
+				status = "okay";
+			};
+
+			usb3@f8000 {
+				status = "okay";
+			};
+		};
+
+		pcie-controller {
+			status = "okay";
+
+			pcie@1,0 {
+				/* Port 0, Lane 0 */
+				status = "okay";
+			};
+
+			pcie@2,0 {
+				/* Port 1, Lane 0 */
+				status = "okay";
+			};
+
+			pcie@3,0 {
+				/* Port 2, Lane 0 */
+				status = "okay";
+			};
+		};
+	};
+};
+
+/* Connected to 88E6176 switch, port 6 */
+&eth0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&ge0_rgmii_pins>;
+	status = "okay";
+	phy-mode = "rgmii-id";
+
+	fixed-link {
+		speed = <1000>;
+		full-duplex;
+	};
+};
+
+/* Connected to 88E6176 switch, port 5 */
+&eth1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&ge1_rgmii_pins>;
+	status = "okay";
+	phy-mode = "rgmii-id";
+
+	fixed-link {
+		speed = <1000>;
+		full-duplex;
+	};
+};
+
+/* WAN port */
+&eth2 {
+	status = "okay";
+	phy-mode = "sgmii";
+	phy = <&phy1>;
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_pins>;
+	status = "okay";
+
+	i2cmux@70 {
+		compatible = "nxp,pca9547";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x70>;
+		status = "okay";
+
+		i2c@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+
+			/* STM32F0 command interface at address 0x2a */
+			/* leds device (in STM32F0) at address 0x2b */
+
+			eeprom@54 {
+				compatible = "at,24c64";
+				reg = <0x54>;
+
+				/* The EEPROM contains data for bootloader.
+				 * Contents:
+				 * 	struct omnia_eeprom {
+				 * 		u32 magic; (=0x0341a034 in LE)
+				 *		u32 ramsize; (in GiB)
+				 * 		char regdomain[4];
+				 * 		u32 crc32;
+				 * 	};
+				 */
+			};
+		};
+
+		i2c@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+
+			/* routed to PCIe0/mSATA connector (CN7A) */
+		};
+
+		i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <2>;
+
+			/* routed to PCIe1/USB2 connector (CN61A) */
+		};
+
+		i2c@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <3>;
+
+			/* routed to PCIe2 connector (CN62A) */
+		};
+
+		i2c@4 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <4>;
+
+			/* routed to SFP+ */
+		};
+
+		i2c@5 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <5>;
+
+			/* ATSHA204A at address 0x64 */
+		};
+
+		i2c@6 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <6>;
+
+			/* exposed on pin header */
+		};
+
+		i2c@7 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <7>;
+
+			pcawan: gpio@71 {
+				/*
+				 * GPIO expander for SFP+ signals and
+				 * and phy irq
+				 */
+				compatible = "nxp,pca9538";
+				reg = <0x71>;
+
+				pinctrl-names = "default";
+				pinctrl-0 = <&pcawan_pins>;
+
+				interrupt-parent = <&gpio1>;
+				interrupts = <14 IRQ_TYPE_LEVEL_LOW>;
+
+				gpio-controller;
+				#gpio-cells = <2>;
+			};
+		};
+	};
+};
+
+&mdio {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mdio_pins>;
+	status = "okay";
+
+	phy1: phy@1 {
+		status = "okay";
+		compatible = "ethernet-phy-id0141.0DD1", "ethernet-phy-ieee802.3-c22";
+		reg = <1>;
+
+		/* irq is connected to &pcawan pin 7 */
+	};
+
+	/* Switch MV88E7176 at address 0x10 */
+};
+
+&pinctrl {
+	pcawan_pins: pcawan-pins {
+		marvell,pins = "mpp46";
+		marvell,function = "gpio";
+	};
+
+	spi0cs0_pins: spi0cs0-pins {
+		marvell,pins = "mpp25";
+		marvell,function = "spi0";
+	};
+
+	spi0cs1_pins: spi0cs1-pins {
+		marvell,pins = "mpp26";
+		marvell,function = "spi0";
+	};
+};
+
+&spi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi0_pins &spi0cs0_pins>;
+	status = "okay";
+
+	spi-nor@0 {
+		compatible = "spansion,s25fl164k", "jedec,spi-nor";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		reg = <0>;
+		spi-max-frequency = <40000000>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				reg = <0x0 0x00100000>;
+				label = "U-Boot";
+			};
+
+			partition@100000 {
+				reg = <0x00100000 0x00700000>;
+				label = "Rescue system";
+			};
+		};
+	};
+
+	/* MISO, MOSI, SCLK and CS1 are routed to pin header CN11 */
+};
+
+&uart0 {
+	/* Pin header CN10 */
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pins>;
+	status = "okay";
+};
+
+&uart1 {
+	/* Pin header CN11 */
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart1_pins>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/armada-38x.dtsi b/arch/arm/boot/dts/armada-38x.dtsi
index 2d7668848c5a..7450e9fea45d 100644
--- a/arch/arm/boot/dts/armada-38x.dtsi
+++ b/arch/arm/boot/dts/armada-38x.dtsi
@@ -661,7 +661,7 @@
 	};
 
 	clocks {
-		/* 2 GHz fixed main PLL */
+		/* 1 GHz fixed main PLL */
 		mainpll: mainpll {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
diff --git a/arch/arm/boot/dts/armada-39x.dtsi b/arch/arm/boot/dts/armada-39x.dtsi
index 34cba87f9200..de171baffcf6 100644
--- a/arch/arm/boot/dts/armada-39x.dtsi
+++ b/arch/arm/boot/dts/armada-39x.dtsi
@@ -573,7 +573,7 @@
 	};
 
 	clocks {
-		/* 2 GHz fixed main PLL */
+		/* 1 GHz fixed main PLL */
 		mainpll: mainpll {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
diff --git a/arch/arm/boot/dts/armada-xp-axpwifiap.dts b/arch/arm/boot/dts/armada-xp-axpwifiap.dts
index ce152719bc28..1e1fc4fccbad 100644
--- a/arch/arm/boot/dts/armada-xp-axpwifiap.dts
+++ b/arch/arm/boot/dts/armada-xp-axpwifiap.dts
@@ -62,7 +62,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x00000000 0x00000000 0x40000000>; /* 1GB */
 	};
@@ -73,28 +73,6 @@
 			  MBUS_ID(0x09, 0x09) 0 0 0xf1100000 0x10000
 			  MBUS_ID(0x09, 0x05) 0 0 0xf1110000 0x10000>;
 
-		pcie-controller {
-			status = "okay";
-
-			/* First mini-PCIe port */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-
-			/* Second mini-PCIe port */
-			pcie@2,0 {
-				/* Port 0, Lane 1 */
-				status = "okay";
-			};
-
-			/* Renesas uPD720202 USB 3.0 controller */
-			pcie@3,0 {
-				/* Port 0, Lane 3 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 			/* UART0 */
 			serial@12000 {
@@ -111,16 +89,6 @@
 				status = "okay";
 			};
 
-			mdio {
-				phy0: ethernet-phy@0 {
-					reg = <0>;
-				};
-
-				phy1: ethernet-phy@1 {
-					reg = <1>;
-				};
-			};
-
 			ethernet@70000 {
 				pinctrl-0 = <&ge0_rgmii_pins>;
 				pinctrl-names = "default";
@@ -145,7 +113,7 @@
 		pinctrl-0 = <&keys_pin>;
 		pinctrl-names = "default";
 
-		button@1 {
+		reset {
 			label = "Factory Reset Button";
 			linux,code = <KEY_SETUP>;
 			gpios = <&gpio1 1 GPIO_ACTIVE_LOW>;
@@ -153,6 +121,38 @@
 	};
 };
 
+&mdio {
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+	};
+
+	phy1: ethernet-phy@1 {
+		reg = <1>;
+	};
+};
+
+&pciec {
+	status = "okay";
+
+	/* First mini-PCIe port */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+
+	/* Second mini-PCIe port */
+	pcie@2,0 {
+		/* Port 0, Lane 1 */
+		status = "okay";
+	};
+
+	/* Renesas uPD720202 USB 3.0 controller */
+	pcie@3,0 {
+		/* Port 0, Lane 3 */
+		status = "okay";
+	};
+};
+
 &pinctrl {
 	pinctrl-0 = <&phy_int_pin>;
 	pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/armada-xp-db.dts b/arch/arm/boot/dts/armada-xp-db.dts
index 075120bc3ec4..44a724d39dbe 100644
--- a/arch/arm/boot/dts/armada-xp-db.dts
+++ b/arch/arm/boot/dts/armada-xp-db.dts
@@ -67,7 +67,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0 0x00000000 0 0x80000000>; /* 2 GB */
 	};
@@ -108,39 +108,6 @@
 			};
 		};
 
-		pcie-controller {
-			status = "okay";
-
-			/*
-			 * All 6 slots are physically present as
-			 * standard PCIe slots on the board.
-			 */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-			pcie@2,0 {
-				/* Port 0, Lane 1 */
-				status = "okay";
-			};
-			pcie@3,0 {
-				/* Port 0, Lane 2 */
-				status = "okay";
-			};
-			pcie@4,0 {
-				/* Port 0, Lane 3 */
-				status = "okay";
-			};
-			pcie@9,0 {
-				/* Port 2, Lane 0 */
-				status = "okay";
-			};
-			pcie@10,0 {
-				/* Port 3, Lane 0 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 			serial@12000 {
 				status = "okay";
@@ -160,24 +127,6 @@
 				status = "okay";
 			};
 
-			mdio {
-				phy0: ethernet-phy@0 {
-					reg = <0>;
-				};
-
-				phy1: ethernet-phy@1 {
-					reg = <1>;
-				};
-
-				phy2: ethernet-phy@2 {
-					reg = <25>;
-				};
-
-				phy3: ethernet-phy@3 {
-					reg = <27>;
-				};
-			};
-
 			ethernet@70000 {
 				status = "okay";
 				phy = <&phy0>;
@@ -266,6 +215,57 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+
+	/*
+	 * All 6 slots are physically present as
+	 * standard PCIe slots on the board.
+	 */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+	pcie@2,0 {
+		/* Port 0, Lane 1 */
+		status = "okay";
+	};
+	pcie@3,0 {
+		/* Port 0, Lane 2 */
+		status = "okay";
+	};
+	pcie@4,0 {
+		/* Port 0, Lane 3 */
+		status = "okay";
+	};
+	pcie@9,0 {
+		/* Port 2, Lane 0 */
+		status = "okay";
+	};
+	pcie@10,0 {
+		/* Port 3, Lane 0 */
+		status = "okay";
+	};
+};
+
+&mdio {
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+	};
+
+	phy1: ethernet-phy@1 {
+		reg = <1>;
+	};
+
+	phy2: ethernet-phy@2 {
+		reg = <25>;
+	};
+
+	phy3: ethernet-phy@3 {
+		reg = <27>;
+	};
+};
+
 &spi0 {
 	status = "okay";
 
diff --git a/arch/arm/boot/dts/armada-xp-gp.dts b/arch/arm/boot/dts/armada-xp-gp.dts
index 190e4eccb180..72cb8fa377e3 100644
--- a/arch/arm/boot/dts/armada-xp-gp.dts
+++ b/arch/arm/boot/dts/armada-xp-gp.dts
@@ -68,7 +68,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		/*
                  * 8 GB of plug-in RAM modules by default.The amount
@@ -127,27 +127,6 @@
 			};
 		};
 
-		pcie-controller {
-			status = "okay";
-
-			/*
-			 * The 3 slots are physically present as
-			 * standard PCIe slots on the board.
-			 */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-			pcie@9,0 {
-				/* Port 2, Lane 0 */
-				status = "okay";
-			};
-			pcie@10,0 {
-				/* Port 3, Lane 0 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 			serial@12000 {
 				status = "okay";
@@ -175,24 +154,6 @@
 				status = "okay";
 			};
 
-			mdio {
-				phy0: ethernet-phy@0 {
-					reg = <16>;
-				};
-
-				phy1: ethernet-phy@1 {
-					reg = <17>;
-				};
-
-				phy2: ethernet-phy@2 {
-					reg = <18>;
-				};
-
-				phy3: ethernet-phy@3 {
-					reg = <19>;
-				};
-			};
-
 			ethernet@70000 {
 				status = "okay";
 				phy = <&phy0>;
@@ -251,6 +212,45 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+
+	/*
+	 * The 3 slots are physically present as
+	 * standard PCIe slots on the board.
+	 */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+	pcie@9,0 {
+		/* Port 2, Lane 0 */
+		status = "okay";
+	};
+	pcie@10,0 {
+		/* Port 3, Lane 0 */
+		status = "okay";
+	};
+};
+
+&mdio {
+	phy0: ethernet-phy@0 {
+		reg = <16>;
+	};
+
+	phy1: ethernet-phy@1 {
+		reg = <17>;
+	};
+
+	phy2: ethernet-phy@2 {
+		reg = <18>;
+	};
+
+	phy3: ethernet-phy@3 {
+		reg = <19>;
+	};
+};
+
 &spi0 {
 	status = "okay";
 
diff --git a/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts b/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts
index 8af463f26ea1..d848ae9007db 100644
--- a/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts
+++ b/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts
@@ -57,7 +57,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0 0x00000000 0 0x20000000>; /* 512MB */
 	};
@@ -68,37 +68,11 @@
 			MBUS_ID(0x09, 0x09) 0 0 0xf1100000 0x10000
 			MBUS_ID(0x09, 0x05) 0 0 0xf1110000 0x10000>;
 
-		pcie-controller {
-			status = "okay";
-
-			/* Quad port sata: Marvell 88SX7042 */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-
-			/* USB 3.0 xHCI controller: NEC D720200F1 */
-			pcie@5,0 {
-				/* Port 1, Lane 0 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 			serial@12000 {
 				status = "okay";
 			};
 
-			mdio {
-				phy0: ethernet-phy@0 { /* Marvell 88E1318 */
-					reg = <0>;
-				};
-
-				phy1: ethernet-phy@1 { /* Marvell 88E1318 */
-					reg = <1>;
-				};
-			};
-
 			ethernet@70000 {
 				pinctrl-0 = <&ge0_rgmii_pins>;
 				pinctrl-names = "default";
@@ -295,6 +269,31 @@
 		gpios = <&gpio0 24 GPIO_ACTIVE_HIGH>;
 	};
 };
+&pciec {
+	status = "okay";
+
+	/* Quad port sata: Marvell 88SX7042 */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+
+	/* USB 3.0 xHCI controller: NEC D720200F1 */
+	pcie@5,0 {
+		/* Port 1, Lane 0 */
+		status = "okay";
+	};
+};
+
+&mdio {
+	phy0: ethernet-phy@0 { /* Marvell 88E1318 */
+		reg = <0>;
+	};
+
+	phy1: ethernet-phy@1 { /* Marvell 88E1318 */
+		reg = <1>;
+	};
+};
 
 &pinctrl {
 	poweroff_pin: poweroff-pin {
diff --git a/arch/arm/boot/dts/armada-xp-linksys-mamba.dts b/arch/arm/boot/dts/armada-xp-linksys-mamba.dts
index 076f27f22c3b..83ac884c0f8a 100644
--- a/arch/arm/boot/dts/armada-xp-linksys-mamba.dts
+++ b/arch/arm/boot/dts/armada-xp-linksys-mamba.dts
@@ -62,7 +62,7 @@
 		stdout-path = &uart0;
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x00000000 0x00000000 0x10000000>; /* 256MB */
 	};
@@ -73,28 +73,6 @@
 			  MBUS_ID(0x09, 0x09) 0 0 0xf1100000 0x10000
 			  MBUS_ID(0x09, 0x05) 0 0 0xf1110000 0x10000>;
 
-		pcie-controller {
-			status = "okay";
-
-			/* Etron EJ168 USB 3.0 controller */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-
-			/* First mini-PCIe port */
-			pcie@2,0 {
-				/* Port 0, Lane 1 */
-				status = "okay";
-			};
-
-			/* Second mini-PCIe port */
-			pcie@3,0 {
-				/* Port 0, Lane 3 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 
 			rtc@10300 {
@@ -289,13 +267,13 @@
 		pinctrl-0 = <&keys_pin>;
 		pinctrl-names = "default";
 
-		button@1 {
+		wps {
 			label = "WPS";
 			linux,code = <KEY_WPS_BUTTON>;
 			gpios = <&gpio1 0 GPIO_ACTIVE_LOW>;
 		};
 
-		button@2 {
+		reset {
 			label = "Factory Reset Button";
 			linux,code = <KEY_RESTART>;
 			gpios = <&gpio1 1 GPIO_ACTIVE_LOW>;
@@ -323,7 +301,7 @@
 				      4500 1>;
 	};
 
-	dsa@0 {
+	dsa {
 		compatible = "marvell,dsa";
 		#address-cells = <2>;
 		#size-cells = <0>;
@@ -369,6 +347,28 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+
+	/* Etron EJ168 USB 3.0 controller */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+
+	/* First mini-PCIe port */
+	pcie@2,0 {
+		/* Port 0, Lane 1 */
+		status = "okay";
+	};
+
+	/* Second mini-PCIe port */
+	pcie@3,0 {
+		/* Port 0, Lane 3 */
+		status = "okay";
+	};
+};
+
 &pinctrl {
 
 	keys_pin: keys-pin {
diff --git a/arch/arm/boot/dts/armada-xp-matrix.dts b/arch/arm/boot/dts/armada-xp-matrix.dts
index 6522b04f4a8e..16277380e714 100644
--- a/arch/arm/boot/dts/armada-xp-matrix.dts
+++ b/arch/arm/boot/dts/armada-xp-matrix.dts
@@ -55,7 +55,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		/*
 		 * This board has 4 GB of RAM, but the last 256 MB of
@@ -99,18 +99,18 @@
 				};
 			};
 
-			pcie-controller {
-				status = "okay";
-
-				pcie@1,0 {
-					/* Port 0, Lane 0 */
-					status = "okay";
-				};
-			};
-
 			usb@50000 {
 				status = "okay";
 			};
 		};
 	};
 };
+
+&pciec {
+	status = "okay";
+
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+};
diff --git a/arch/arm/boot/dts/armada-xp-mv78230.dtsi b/arch/arm/boot/dts/armada-xp-mv78230.dtsi
index 6e6d0f04bf2b..05c164b5786d 100644
--- a/arch/arm/boot/dts/armada-xp-mv78230.dtsi
+++ b/arch/arm/boot/dts/armada-xp-mv78230.dtsi
@@ -86,7 +86,7 @@
 		 * configured as x4 or quad x1 lanes. One unit is
 		 * x1 only.
 		 */
-		pcie-controller {
+		pciec: pcie-controller@82000000 {
 			compatible = "marvell,armada-xp-pcie";
 			status = "disabled";
 			device_type = "pci";
@@ -114,7 +114,7 @@
 				0x82000000 0x5 0       MBUS_ID(0x08, 0xe8) 0 1 0 /* Port 1.0 MEM */
 				0x81000000 0x5 0       MBUS_ID(0x08, 0xe0) 0 1 0 /* Port 1.0 IO  */>;
 
-			pcie@1,0 {
+			pcie1: pcie@1,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x40000 0 0x2000>;
 				reg = <0x0800 0 0 0 0>;
@@ -131,7 +131,7 @@
 				status = "disabled";
 			};
 
-			pcie@2,0 {
+			pcie2: pcie@2,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x44000 0 0x2000>;
 				reg = <0x1000 0 0 0 0>;
@@ -148,7 +148,7 @@
 				status = "disabled";
 			};
 
-			pcie@3,0 {
+			pcie3: pcie@3,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x48000 0 0x2000>;
 				reg = <0x1800 0 0 0 0>;
@@ -165,7 +165,7 @@
 				status = "disabled";
 			};
 
-			pcie@4,0 {
+			pcie4: pcie@4,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x4c000 0 0x2000>;
 				reg = <0x2000 0 0 0 0>;
@@ -182,7 +182,7 @@
 				status = "disabled";
 			};
 
-			pcie@5,0 {
+			pcie5: pcie@5,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x80000 0 0x2000>;
 				reg = <0x2800 0 0 0 0>;
diff --git a/arch/arm/boot/dts/armada-xp-mv78260.dtsi b/arch/arm/boot/dts/armada-xp-mv78260.dtsi
index c5fdc99f0dbe..07894b0d3e59 100644
--- a/arch/arm/boot/dts/armada-xp-mv78260.dtsi
+++ b/arch/arm/boot/dts/armada-xp-mv78260.dtsi
@@ -87,7 +87,7 @@
 		 * configured as x4 or quad x1 lanes. One unit is
 		 * x4 only.
 		 */
-		pcie-controller {
+		pciec: pcie-controller@82000000 {
 			compatible = "marvell,armada-xp-pcie";
 			status = "disabled";
 			device_type = "pci";
@@ -129,7 +129,7 @@
 				0x82000000 0x9 0     MBUS_ID(0x04, 0xf8) 0 1 0 /* Port 2.0 MEM */
 				0x81000000 0x9 0     MBUS_ID(0x04, 0xf0) 0 1 0 /* Port 2.0 IO  */>;
 
-			pcie@1,0 {
+			pcie1: pcie@1,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x40000 0 0x2000>;
 				reg = <0x0800 0 0 0 0>;
@@ -146,7 +146,7 @@
 				status = "disabled";
 			};
 
-			pcie@2,0 {
+			pcie2: pcie@2,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x44000 0 0x2000>;
 				reg = <0x1000 0 0 0 0>;
@@ -163,7 +163,7 @@
 				status = "disabled";
 			};
 
-			pcie@3,0 {
+			pcie3: pcie@3,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x48000 0 0x2000>;
 				reg = <0x1800 0 0 0 0>;
@@ -180,7 +180,7 @@
 				status = "disabled";
 			};
 
-			pcie@4,0 {
+			pcie4: pcie@4,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x4c000 0 0x2000>;
 				reg = <0x2000 0 0 0 0>;
@@ -197,7 +197,7 @@
 				status = "disabled";
 			};
 
-			pcie@5,0 {
+			pcie5: pcie@5,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x80000 0 0x2000>;
 				reg = <0x2800 0 0 0 0>;
@@ -214,7 +214,7 @@
 				status = "disabled";
 			};
 
-			pcie@6,0 {
+			pcie6: pcie@6,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x84000 0 0x2000>;
 				reg = <0x3000 0 0 0 0>;
@@ -231,7 +231,7 @@
 				status = "disabled";
 			};
 
-			pcie@7,0 {
+			pcie7: pcie@7,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x88000 0 0x2000>;
 				reg = <0x3800 0 0 0 0>;
@@ -248,7 +248,7 @@
 				status = "disabled";
 			};
 
-			pcie@8,0 {
+			pcie8: pcie@8,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x8c000 0 0x2000>;
 				reg = <0x4000 0 0 0 0>;
@@ -265,7 +265,7 @@
 				status = "disabled";
 			};
 
-			pcie@9,0 {
+			pcie9: pcie@9,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x42000 0 0x2000>;
 				reg = <0x4800 0 0 0 0>;
diff --git a/arch/arm/boot/dts/armada-xp-mv78460.dtsi b/arch/arm/boot/dts/armada-xp-mv78460.dtsi
index 0e24f1a38540..775bee53ce86 100644
--- a/arch/arm/boot/dts/armada-xp-mv78460.dtsi
+++ b/arch/arm/boot/dts/armada-xp-mv78460.dtsi
@@ -104,7 +104,7 @@
 		 * configured as x4 or quad x1 lanes. Two units are
 		 * x4/x1.
 		 */
-		pcie-controller {
+		pciec: pcie-controller@82000000 {
 			compatible = "marvell,armada-xp-pcie";
 			status = "disabled";
 			device_type = "pci";
@@ -150,7 +150,7 @@
 				0x82000000 0xa 0     MBUS_ID(0x08, 0xf8) 0 1 0 /* Port 3.0 MEM */
 				0x81000000 0xa 0     MBUS_ID(0x08, 0xf0) 0 1 0 /* Port 3.0 IO  */>;
 
-			pcie@1,0 {
+			pcie1: pcie@1,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82000800 0 0x40000 0 0x2000>;
 				reg = <0x0800 0 0 0 0>;
@@ -167,7 +167,7 @@
 				status = "disabled";
 			};
 
-			pcie@2,0 {
+			pcie2: pcie@2,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82001000 0 0x44000 0 0x2000>;
 				reg = <0x1000 0 0 0 0>;
@@ -184,7 +184,7 @@
 				status = "disabled";
 			};
 
-			pcie@3,0 {
+			pcie3: pcie@3,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82001800 0 0x48000 0 0x2000>;
 				reg = <0x1800 0 0 0 0>;
@@ -201,7 +201,7 @@
 				status = "disabled";
 			};
 
-			pcie@4,0 {
+			pcie4: pcie@4,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82002000 0 0x4c000 0 0x2000>;
 				reg = <0x2000 0 0 0 0>;
@@ -218,7 +218,7 @@
 				status = "disabled";
 			};
 
-			pcie@5,0 {
+			pcie5: pcie@5,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82002800 0 0x80000 0 0x2000>;
 				reg = <0x2800 0 0 0 0>;
@@ -235,7 +235,7 @@
 				status = "disabled";
 			};
 
-			pcie@6,0 {
+			pcie6: pcie@6,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82003000 0 0x84000 0 0x2000>;
 				reg = <0x3000 0 0 0 0>;
@@ -252,7 +252,7 @@
 				status = "disabled";
 			};
 
-			pcie@7,0 {
+			pcie7: pcie@7,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82003800 0 0x88000 0 0x2000>;
 				reg = <0x3800 0 0 0 0>;
@@ -269,7 +269,7 @@
 				status = "disabled";
 			};
 
-			pcie@8,0 {
+			pcie8: pcie@8,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82004000 0 0x8c000 0 0x2000>;
 				reg = <0x4000 0 0 0 0>;
@@ -286,7 +286,7 @@
 				status = "disabled";
 			};
 
-			pcie@9,0 {
+			pcie9: pcie@9,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82004800 0 0x42000 0 0x2000>;
 				reg = <0x4800 0 0 0 0>;
@@ -303,7 +303,7 @@
 				status = "disabled";
 			};
 
-			pcie@10,0 {
+			pcie10: pcie@10,0 {
 				device_type = "pci";
 				assigned-addresses = <0x82005000 0 0x82000 0 0x2000>;
 				reg = <0x5000 0 0 0 0>;
diff --git a/arch/arm/boot/dts/armada-xp-netgear-rn2120.dts b/arch/arm/boot/dts/armada-xp-netgear-rn2120.dts
index d19f44c70925..a2f0e789465d 100644
--- a/arch/arm/boot/dts/armada-xp-netgear-rn2120.dts
+++ b/arch/arm/boot/dts/armada-xp-netgear-rn2120.dts
@@ -56,7 +56,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0 0x00000000 0 0x80000000>; /* 2GB */
 	};
@@ -67,28 +67,6 @@
 			  MBUS_ID(0x09, 0x09) 0 0 0xf1100000 0x10000
 			  MBUS_ID(0x09, 0x05) 0 0 0xf1110000 0x10000>;
 
-		pcie-controller {
-			status = "okay";
-
-			/* Connected to first Marvell 88SE9170 SATA controller */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-
-			/* Connected to second Marvell 88SE9170 SATA controller */
-			pcie@2,0 {
-				/* Port 0, Lane 1 */
-				status = "okay";
-			};
-
-			/* Connected to Fresco Logic FL1009 USB 3.0 controller */
-			pcie@5,0 {
-				/* Port 1, Lane 0 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 
 			/* RTC is provided by Intersil ISL12057 I2C RTC chip */
@@ -97,7 +75,6 @@
 			};
 
 			i2c@11000 {
-				compatible = "marvell,mv64xxx-i2c";
 				clock-frequency = <400000>;
 				status = "okay";
 
@@ -154,23 +131,19 @@
 				status = "okay";
 			};
 
-			mdio {
-				phy0: ethernet-phy@0 { /* Marvell 88E1318 */
-					reg = <0>;
-				};
-
-				phy1: ethernet-phy@1 { /* Marvell 88E1318 */
-					reg = <1>;
-				};
-			};
-
 			ethernet@70000 {
+				pinctrl-0 = <&ge0_rgmii_pins>;
+				pinctrl-names = "default";
+
 				status = "okay";
 				phy = <&phy0>;
 				phy-mode = "rgmii-id";
 			};
 
 			ethernet@74000 {
+				pinctrl-0 = <&ge1_rgmii_pins>;
+				pinctrl-names = "default";
+
 				status = "okay";
 				phy = <&phy1>;
 				phy-mode = "rgmii-id";
@@ -295,6 +268,39 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+
+	/* Connected to first Marvell 88SE9170 SATA controller */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+
+	/* Connected to second Marvell 88SE9170 SATA controller */
+	pcie@2,0 {
+		/* Port 0, Lane 1 */
+		status = "okay";
+	};
+
+	/* Connected to Fresco Logic FL1009 USB 3.0 controller */
+	pcie@5,0 {
+		/* Port 1, Lane 0 */
+		status = "okay";
+	};
+};
+
+&mdio {
+	phy0: ethernet-phy@0 { /* Marvell 88E1318 */
+		reg = <0>;
+	};
+
+	phy1: ethernet-phy@1 { /* Marvell 88E1318 */
+		reg = <1>;
+	};
+};
+
+
 &pinctrl {
 	poweroff: poweroff {
 		marvell,pins = "mpp42";
diff --git a/arch/arm/boot/dts/armada-xp-openblocks-ax3-4.dts b/arch/arm/boot/dts/armada-xp-openblocks-ax3-4.dts
index ed3b889d16ce..b577c9fb03a4 100644
--- a/arch/arm/boot/dts/armada-xp-openblocks-ax3-4.dts
+++ b/arch/arm/boot/dts/armada-xp-openblocks-ax3-4.dts
@@ -57,7 +57,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0 0x00000000 0 0x40000000>; /* 1 GB soldered on */
 	};
@@ -98,15 +98,6 @@
 			};
 		};
 
-		pcie-controller {
-			status = "okay";
-			/* Internal mini-PCIe connector */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 			rtc@10300 {
 				/* No crystal connected to the internal RTC */
@@ -148,31 +139,13 @@
 				#address-cells = <1>;
 				#size-cells = <0>;
 
-				button@1 {
+				init {
 					label = "Init Button";
 					linux,code = <KEY_POWER>;
 					gpios = <&gpio1 28 GPIO_ACTIVE_HIGH>;
 				};
 			};
 
-			mdio {
-				phy0: ethernet-phy@0 {
-					reg = <0>;
-				};
-
-				phy1: ethernet-phy@1 {
-					reg = <1>;
-				};
-
-				phy2: ethernet-phy@2 {
-					reg = <2>;
-				};
-
-				phy3: ethernet-phy@3 {
-					reg = <3>;
-				};
-			};
-
 			ethernet@70000 {
 				status = "okay";
 				phy = <&phy0>;
@@ -240,6 +213,33 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+	/* Internal mini-PCIe connector */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+};
+
+&mdio {
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+	};
+
+	phy1: ethernet-phy@1 {
+		reg = <1>;
+	};
+
+	phy2: ethernet-phy@2 {
+		reg = <2>;
+	};
+
+	phy3: ethernet-phy@3 {
+		reg = <3>;
+	};
+};
+
 &pinctrl {
 	led_pins: led-pins-0 {
 		marvell,pins = "mpp49", "mpp51", "mpp53";
diff --git a/arch/arm/boot/dts/armada-xp-synology-ds414.dts b/arch/arm/boot/dts/armada-xp-synology-ds414.dts
index ae286736b90a..e803da03146a 100644
--- a/arch/arm/boot/dts/armada-xp-synology-ds414.dts
+++ b/arch/arm/boot/dts/armada-xp-synology-ds414.dts
@@ -70,7 +70,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0 0x00000000 0 0x40000000>; /* 1GB */
 	};
@@ -81,28 +81,6 @@
 			  MBUS_ID(0x09, 0x09) 0 0 0xf1100000 0x10000
 			  MBUS_ID(0x09, 0x05) 0 0 0xf1110000 0x10000>;
 
-		pcie-controller {
-			status = "okay";
-
-			/*
-			 * Connected to Marvell 88SX7042 SATA-II controller
-			 * handling the four disks.
-			 */
-			pcie@1,0 {
-				/* Port 0, Lane 0 */
-				status = "okay";
-			};
-
-			/*
-			 * Connected to EtronTech EJ168A XHCI controller
-			 * providing the two rear USB 3.0 ports.
-			 */
-			pcie@5,0 {
-				/* Port 1, Lane 0 */
-				status = "okay";
-			};
-		};
-
 		internal-regs {
 
 			/* RTC is provided by Seiko S-35390A below */
@@ -150,16 +128,6 @@
 				status = "okay";
 			};
 
-			mdio {
-				phy0: ethernet-phy@0 { /* Marvell 88E1512 */
-					reg = <0>;
-				};
-
-				phy1: ethernet-phy@1 { /* Marvell 88E1512 */
-					reg = <1>;
-				};
-			};
-
 			ethernet@70000 {
 				status = "okay";
 				pinctrl-0 = <&ge0_rgmii_pins>;
@@ -186,7 +154,7 @@
 			     &sata3_pwr_pin &sata4_pwr_pin>;
 		pinctrl-names = "default";
 
-		sata1_regulator: sata1-regulator {
+		sata1_regulator: sata1-regulator@1 {
 			compatible = "regulator-fixed";
 			reg = <1>;
 			regulator-name = "SATA1 Power";
@@ -199,7 +167,7 @@
 			gpio = <&gpio1 10 GPIO_ACTIVE_HIGH>;
 		};
 
-		sata2_regulator: sata2-regulator {
+		sata2_regulator: sata2-regulator@2 {
 			compatible = "regulator-fixed";
 			reg = <2>;
 			regulator-name = "SATA2 Power";
@@ -212,7 +180,7 @@
 			gpio = <&gpio1 12 GPIO_ACTIVE_HIGH>;
 		};
 
-		sata3_regulator: sata3-regulator {
+		sata3_regulator: sata3-regulator@3 {
 			compatible = "regulator-fixed";
 			reg = <3>;
 			regulator-name = "SATA3 Power";
@@ -225,7 +193,7 @@
 			gpio = <&gpio1 13 GPIO_ACTIVE_HIGH>;
 		};
 
-		sata4_regulator: sata4-regulator {
+		sata4_regulator: sata4-regulator@4 {
 			compatible = "regulator-fixed";
 			reg = <4>;
 			regulator-name = "SATA4 Power";
@@ -240,6 +208,39 @@
 	};
 };
 
+&pciec {
+	status = "okay";
+
+	/*
+	 * Connected to Marvell 88SX7042 SATA-II controller
+	 * handling the four disks.
+	 */
+	pcie@1,0 {
+		/* Port 0, Lane 0 */
+		status = "okay";
+	};
+
+	/*
+	 * Connected to EtronTech EJ168A XHCI controller
+	 * providing the two rear USB 3.0 ports.
+	 */
+	pcie@5,0 {
+		/* Port 1, Lane 0 */
+		status = "okay";
+	};
+};
+
+
+&mdio {
+	phy0: ethernet-phy@0 { /* Marvell 88E1512 */
+		reg = <0>;
+	};
+
+	phy1: ethernet-phy@1 { /* Marvell 88E1512 */
+		reg = <1>;
+	};
+};
+
 &pinctrl {
 	sata1_pwr_pin: sata1-pwr-pin {
 		marvell,pins = "mpp42";
diff --git a/arch/arm/boot/dts/armada-xp.dtsi b/arch/arm/boot/dts/armada-xp.dtsi
index 4a5f99e65b51..5274e4ff5d62 100644
--- a/arch/arm/boot/dts/armada-xp.dtsi
+++ b/arch/arm/boot/dts/armada-xp.dtsi
@@ -53,6 +53,9 @@
 #include "armada-370-xp.dtsi"
 
 / {
+	#address-cells = <2>;
+	#size-cells = <2>;
+
 	model = "Marvell Armada XP family SoC";
 	compatible = "marvell,armadaxp", "marvell,armada-370-xp";
 
@@ -75,7 +78,7 @@
 				reg = <0x1400 0x500>;
 			};
 
-			L2: l2-cache {
+			L2: l2-cache@8000 {
 				compatible = "marvell,aurora-system-cache";
 				reg = <0x08000 0x1000>;
 				cache-id-part = <0x100>;
@@ -84,16 +87,6 @@
 				wt-override;
 			};
 
-			i2c0: i2c@11000 {
-				compatible = "marvell,mv78230-i2c", "marvell,mv64xxx-i2c";
-				reg = <0x11000 0x100>;
-			};
-
-			i2c1: i2c@11100 {
-				compatible = "marvell,mv78230-i2c", "marvell,mv64xxx-i2c";
-				reg = <0x11100 0x100>;
-			};
-
 			uart2: serial@12200 {
 				compatible = "snps,dw-apb-uart";
 				pinctrl-0 = <&uart2_pins>;
@@ -118,7 +111,7 @@
 				status = "disabled";
 			};
 
-			system-controller@18200 {
+			systemc: system-controller@18200 {
 				compatible = "marvell,armada-370-xp-system-controller";
 				reg = <0x18200 0x500>;
 			};
@@ -136,7 +129,7 @@
 				#clock-cells = <1>;
 			};
 
-			thermal@182b0 {
+			thermal: thermal@182b0 {
 				compatible = "marvell,armadaxp-thermal";
 				reg = <0x182b0 0x4
 					0x184d0 0x4>;
@@ -150,27 +143,6 @@
 				clocks = <&coreclk 1>;
 			};
 
-			interrupt-controller@20a00 {
-			      reg = <0x20a00 0x2d0>, <0x21070 0x58>;
-			};
-
-			timer@20300 {
-				compatible = "marvell,armada-xp-timer";
-				clocks = <&coreclk 2>, <&refclk>;
-				clock-names = "nbclk", "fixed";
-			};
-
-			watchdog@20300 {
-				compatible = "marvell,armada-xp-wdt";
-				clocks = <&coreclk 2>, <&refclk>;
-				clock-names = "nbclk", "fixed";
-			};
-
-			cpurst@20800 {
-				compatible = "marvell,armada-370-cpu-reset";
-				reg = <0x20800 0x20>;
-			};
-
 			cpu-config@21000 {
 				compatible = "marvell,armada-xp-cpu-config";
 				reg = <0x21000 0x8>;
@@ -184,15 +156,7 @@
 				status = "disabled";
 			};
 
-			usb@50000 {
-				clocks = <&gateclk 18>;
-			};
-
-			usb@51000 {
-				clocks = <&gateclk 19>;
-			};
-
-			usb@52000 {
+			usb2: usb@52000 {
 				compatible = "marvell,orion-ehci";
 				reg = <0x52000 0x500>;
 				interrupts = <47>;
@@ -200,7 +164,7 @@
 				status = "disabled";
 			};
 
-			xor@60900 {
+			xor1: xor@60900 {
 				compatible = "marvell,orion-xor";
 				reg = <0x60900 0x100
 				       0x60b00 0x100>;
@@ -228,7 +192,7 @@
 				compatible = "marvell,armada-xp-neta";
 			};
 
-			crypto@90000 {
+			cesa: crypto@90000 {
 				compatible = "marvell,armada-xp-crypto";
 				reg = <0x90000 0x10000>;
 				reg-names = "regs";
@@ -248,7 +212,7 @@
 				status = "disabled";
 			};
 
-			xor@f0900 {
+			xor0: xor@f0900 {
 				compatible = "marvell,orion-xor";
 				reg = <0xF0900 0x100
 				       0xF0B00 0x100>;
@@ -309,6 +273,44 @@
 	};
 };
 
+&i2c0 {
+	compatible = "marvell,mv78230-i2c", "marvell,mv64xxx-i2c";
+	reg = <0x11000 0x100>;
+};
+
+&i2c1 {
+	compatible = "marvell,mv78230-i2c", "marvell,mv64xxx-i2c";
+	reg = <0x11100 0x100>;
+};
+
+&mpic {
+	reg = <0x20a00 0x2d0>, <0x21070 0x58>;
+};
+
+&timer {
+	compatible = "marvell,armada-xp-timer";
+	clocks = <&coreclk 2>, <&refclk>;
+	clock-names = "nbclk", "fixed";
+};
+
+&watchdog {
+	compatible = "marvell,armada-xp-wdt";
+	clocks = <&coreclk 2>, <&refclk>;
+	clock-names = "nbclk", "fixed";
+};
+
+&cpurst {
+	reg = <0x20800 0x20>;
+};
+
+&usb0 {
+	clocks = <&gateclk 18>;
+};
+
+&usb1 {
+	clocks = <&gateclk 19>;
+};
+
 &pinctrl {
 	ge0_gmii_pins: ge0-gmii-pins {
 		marvell,pins =
diff --git a/arch/arm/boot/dts/artpec6-devboard.dts b/arch/arm/boot/dts/artpec6-devboard.dts
index f823ed382ac7..9dfe845694cf 100644
--- a/arch/arm/boot/dts/artpec6-devboard.dts
+++ b/arch/arm/boot/dts/artpec6-devboard.dts
@@ -46,6 +46,10 @@
 	status = "okay";
 };
 
+&pcie {
+	status = "okay";
+};
+
 &ethernet {
 	status = "okay";
 
diff --git a/arch/arm/boot/dts/artpec6.dtsi b/arch/arm/boot/dts/artpec6.dtsi
index 3489019cc0dc..767cbe8d8557 100644
--- a/arch/arm/boot/dts/artpec6.dtsi
+++ b/arch/arm/boot/dts/artpec6.dtsi
@@ -67,7 +67,7 @@
 		};
 	};
 
-	syscon {
+	syscon: syscon@f8000000 {
 		compatible = "axis,artpec6-syscon", "syscon";
 		reg = <0xf8000000 0x48>;
 	};
@@ -154,6 +154,33 @@
 		interrupt-parent = <&intc>;
 	};
 
+	pcie: pcie@f8050000 {
+		compatible = "axis,artpec6-pcie", "snps,dw-pcie";
+		reg = <0xf8050000 0x2000
+		       0xf8040000 0x1000
+		       0xc0000000 0x2000>;
+		reg-names = "dbi", "phy", "config";
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+			  /* downstream I/O */
+		ranges = <0x81000000 0 0 0xc0002000 0 0x00010000
+			  /* non-prefetchable memory */
+			  0x82000000 0 0xc0012000 0xc0012000 0 0x1ffee000>;
+		num-lanes = <2>;
+		bus-range = <0x00 0xff>;
+		interrupts = <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "msi";
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0x7>;
+		interrupt-map = <0 0 0 1 &intc GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>,
+				<0 0 0 2 &intc GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>,
+				<0 0 0 3 &intc GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>,
+				<0 0 0 4 &intc GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>;
+		axis,syscon-pcie = <&syscon>;
+		status = "disabled";
+	};
+
 	amba@0 {
 		compatible = "simple-bus";
 		#address-cells = <0x1>;
diff --git a/arch/arm/boot/dts/at91-sama5d4_ma5d4.dtsi b/arch/arm/boot/dts/at91-sama5d4_ma5d4.dtsi
index a92c6e0ca854..b5a5a91bc2ef 100644
--- a/arch/arm/boot/dts/at91-sama5d4_ma5d4.dtsi
+++ b/arch/arm/boot/dts/at91-sama5d4_ma5d4.dtsi
@@ -12,8 +12,8 @@
 #include "sama5d4.dtsi"
 
 / {
-	model = "DENX MA5D4";
-	compatible = "denx,ma5d4", "atmel,sama5d4", "atmel,sama5";
+	model = "Aries/DENX MA5D4";
+	compatible = "aries,ma5d4", "denx,ma5d4", "atmel,sama5d4", "atmel,sama5";
 
 	memory {
 		reg = <0x20000000 0x10000000>;
diff --git a/arch/arm/boot/dts/at91-sama5d4_ma5d4evk.dts b/arch/arm/boot/dts/at91-sama5d4_ma5d4evk.dts
index eac4ea2744cc..84be29f38dae 100644
--- a/arch/arm/boot/dts/at91-sama5d4_ma5d4evk.dts
+++ b/arch/arm/boot/dts/at91-sama5d4_ma5d4evk.dts
@@ -13,8 +13,8 @@
 #include "at91-sama5d4_ma5d4.dtsi"
 
 / {
-	model = "DENX MA5D4EVK";
-	compatible = "denx,ma5d4evk", "atmel,sama5d4", "atmel,sama5";
+	model = "Aries/DENX MA5D4EVK";
+	compatible = "aries,ma5d4evk", "denx,ma5d4evk", "atmel,sama5d4", "atmel,sama5";
 
 	chosen {
 		stdout-path = "serial3:115200n8";
diff --git a/arch/arm/boot/dts/at91rm9200.dtsi b/arch/arm/boot/dts/at91rm9200.dtsi
index 4e913c2ccb79..f057e0b15a6f 100644
--- a/arch/arm/boot/dts/at91rm9200.dtsi
+++ b/arch/arm/boot/dts/at91rm9200.dtsi
@@ -481,8 +481,8 @@
 				dbgu {
 					pinctrl_dbgu: dbgu-0 {
 						atmel,pins =
-							<AT91_PIOA 30 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PA30 periph A */
-							 AT91_PIOA 31 AT91_PERIPH_A AT91_PINCTRL_PULL_UP>;	/* PA31 periph with pullup */
+							<AT91_PIOA 30 AT91_PERIPH_A AT91_PINCTRL_PULL_UP
+							 AT91_PIOA 31 AT91_PERIPH_A AT91_PINCTRL_NONE>;
 					};
 				};
 
diff --git a/arch/arm/boot/dts/at91sam9260.dtsi b/arch/arm/boot/dts/at91sam9260.dtsi
index a3e363d79122..9e035b21e1b6 100644
--- a/arch/arm/boot/dts/at91sam9260.dtsi
+++ b/arch/arm/boot/dts/at91sam9260.dtsi
@@ -412,8 +412,8 @@
 				dbgu {
 					pinctrl_dbgu: dbgu-0 {
 						atmel,pins =
-							<AT91_PIOB 14 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB14 periph A */
-							 AT91_PIOB 15 AT91_PERIPH_A AT91_PINCTRL_PULL_UP>;	/* PB15 periph with pullup */
+							<AT91_PIOB 14 AT91_PERIPH_A AT91_PINCTRL_PULL_UP
+							 AT91_PIOB 15 AT91_PERIPH_A AT91_PINCTRL_NONE>;
 					};
 				};
 
diff --git a/arch/arm/boot/dts/at91sam9260ek.dts b/arch/arm/boot/dts/at91sam9260ek.dts
index 2c87f58448e7..b2578feceb08 100644
--- a/arch/arm/boot/dts/at91sam9260ek.dts
+++ b/arch/arm/boot/dts/at91sam9260ek.dts
@@ -174,14 +174,14 @@
 			label = "Button 3";
 			gpios = <&pioA 30 GPIO_ACTIVE_LOW>;
 			linux,code = <0x103>;
-			gpio-key,wakeup;
+			wakeup-source;
 		};
 
 		btn4 {
 			label = "Button 4";
 			gpios = <&pioA 31 GPIO_ACTIVE_LOW>;
 			linux,code = <0x104>;
-			gpio-key,wakeup;
+			wakeup-source;
 		};
 	};
 
diff --git a/arch/arm/boot/dts/at91sam9261.dtsi b/arch/arm/boot/dts/at91sam9261.dtsi
index 32752d7883f1..3fe77c38bd0d 100644
--- a/arch/arm/boot/dts/at91sam9261.dtsi
+++ b/arch/arm/boot/dts/at91sam9261.dtsi
@@ -302,8 +302,8 @@
 				dbgu {
 					pinctrl_dbgu: dbgu-0 {
 						atmel,pins =
-							<AT91_PIOA 9  AT91_PERIPH_A AT91_PINCTRL_NONE>,
-							<AT91_PIOA 10 AT91_PERIPH_A AT91_PINCTRL_PULL_UP>;
+							<AT91_PIOA 9  AT91_PERIPH_A AT91_PINCTRL_PULL_UP>,
+							<AT91_PIOA 10 AT91_PERIPH_A AT91_PINCTRL_NONE>;
 					};
 				};
 
diff --git a/arch/arm/boot/dts/at91sam9263.dtsi b/arch/arm/boot/dts/at91sam9263.dtsi
index aeb1a36373f4..a1888f6d892b 100644
--- a/arch/arm/boot/dts/at91sam9263.dtsi
+++ b/arch/arm/boot/dts/at91sam9263.dtsi
@@ -412,8 +412,8 @@
 				dbgu {
 					pinctrl_dbgu: dbgu-0 {
 						atmel,pins =
-							<AT91_PIOC 30 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PC30 periph A */
-							 AT91_PIOC 31 AT91_PERIPH_A AT91_PINCTRL_PULL_UP>;	/* PC31 periph with pullup */
+							<AT91_PIOC 30 AT91_PERIPH_A AT91_PINCTRL_PULL_UP
+							 AT91_PIOC 31 AT91_PERIPH_A AT91_PINCTRL_NONE>;
 					};
 				};
 
diff --git a/arch/arm/boot/dts/at91sam9g45.dtsi b/arch/arm/boot/dts/at91sam9g45.dtsi
index b3501ae2a3bd..e567d5fd3f9d 100644
--- a/arch/arm/boot/dts/at91sam9g45.dtsi
+++ b/arch/arm/boot/dts/at91sam9g45.dtsi
@@ -478,8 +478,8 @@
 				dbgu {
 					pinctrl_dbgu: dbgu-0 {
 						atmel,pins =
-							<AT91_PIOB 12 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB12 periph A */
-							 AT91_PIOB 13 AT91_PERIPH_A AT91_PINCTRL_NONE>;	/* PB13 periph A */
+							<AT91_PIOB 12 AT91_PERIPH_A AT91_PINCTRL_PULL_UP
+							 AT91_PIOB 13 AT91_PERIPH_A AT91_PINCTRL_NONE>;
 					};
 				};
 
diff --git a/arch/arm/boot/dts/at91sam9n12.dtsi b/arch/arm/boot/dts/at91sam9n12.dtsi
index 3b3eb3edcb47..f43d7695352d 100644
--- a/arch/arm/boot/dts/at91sam9n12.dtsi
+++ b/arch/arm/boot/dts/at91sam9n12.dtsi
@@ -500,8 +500,8 @@
 				dbgu {
 					pinctrl_dbgu: dbgu-0 {
 						atmel,pins =
-							<AT91_PIOA 9 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PA9 periph A */
-							 AT91_PIOA 10 AT91_PERIPH_A AT91_PINCTRL_PULL_UP>;	/* PA10 periph with pullup */
+							<AT91_PIOA 9 AT91_PERIPH_A AT91_PINCTRL_PULL_UP
+							 AT91_PIOA 10 AT91_PERIPH_A AT91_PINCTRL_NONE>;
 					};
 				};
 
diff --git a/arch/arm/boot/dts/at91sam9rl.dtsi b/arch/arm/boot/dts/at91sam9rl.dtsi
index 70adf940d98c..f4c129a98f17 100644
--- a/arch/arm/boot/dts/at91sam9rl.dtsi
+++ b/arch/arm/boot/dts/at91sam9rl.dtsi
@@ -438,8 +438,8 @@
 				dbgu {
 					pinctrl_dbgu: dbgu-0 {
 						atmel,pins =
-							<AT91_PIOA 21 AT91_PERIPH_A AT91_PINCTRL_NONE>,
-							<AT91_PIOA 22 AT91_PERIPH_A AT91_PINCTRL_PULL_UP>;
+							<AT91_PIOA 21 AT91_PERIPH_A AT91_PINCTRL_PULL_UP>,
+							<AT91_PIOA 22 AT91_PERIPH_A AT91_PINCTRL_NONE>;
 					};
 				};
 
diff --git a/arch/arm/boot/dts/at91sam9x5.dtsi b/arch/arm/boot/dts/at91sam9x5.dtsi
index ed4e4bd8a8f1..f66bae925705 100644
--- a/arch/arm/boot/dts/at91sam9x5.dtsi
+++ b/arch/arm/boot/dts/at91sam9x5.dtsi
@@ -460,8 +460,8 @@
 				dbgu {
 					pinctrl_dbgu: dbgu-0 {
 						atmel,pins =
-							<AT91_PIOA 9 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PA9 periph A */
-							 AT91_PIOA 10 AT91_PERIPH_A AT91_PINCTRL_PULL_UP>;	/* PA10 periph A with pullup */
+							<AT91_PIOA 9 AT91_PERIPH_A AT91_PINCTRL_PULL_UP
+							 AT91_PIOA 10 AT91_PERIPH_A AT91_PINCTRL_NONE>;
 					};
 				};
 
diff --git a/arch/arm/boot/dts/bcm-cygnus.dtsi b/arch/arm/boot/dts/bcm-cygnus.dtsi
index fabc9f36c408..8833a4c3cd96 100644
--- a/arch/arm/boot/dts/bcm-cygnus.dtsi
+++ b/arch/arm/boot/dts/bcm-cygnus.dtsi
@@ -91,6 +91,13 @@
 		#address-cells = <1>;
 		#size-cells = <1>;
 
+		otp: otp@0301c800 {
+			compatible = "brcm,ocotp";
+			reg = <0x0301c800 0x2c>;
+			brcm,ocotp-size = <2048>;
+			status = "disabled";
+		};
+
 		pcie_phy: phy@0301d0a0 {
 			compatible = "brcm,cygnus-pcie-phy";
 			reg = <0x0301d0a0 0x14>;
@@ -108,12 +115,21 @@
 			};
 		};
 
-		pinctrl: pinctrl@0x0301d0c8 {
+		pinctrl: pinctrl@0301d0c8 {
 			compatible = "brcm,cygnus-pinmux";
 			reg = <0x0301d0c8 0x30>,
 			      <0x0301d24c 0x2c>;
 		};
 
+		mailbox: mailbox@03024024 {
+			compatible = "brcm,iproc-mailbox";
+			reg = <0x03024024 0x40>;
+			interrupts = <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
+			#interrupt-cells = <1>;
+			interrupt-controller;
+			#mbox-cells = <1>;
+		};
+
 		gpio_crmu: gpio@03024800 {
 			compatible = "brcm,cygnus-crmu-gpio";
 			reg = <0x03024800 0x50>,
@@ -121,6 +137,9 @@
 			ngpios = <6>;
 			#gpio-cells = <2>;
 			gpio-controller;
+			interrupt-controller;
+			interrupt-parent = <&mailbox>;
+			interrupts = <0>;
 		};
 
 		i2c0: i2c@18008000 {
diff --git a/arch/arm/boot/dts/bcm-nsp.dtsi b/arch/arm/boot/dts/bcm-nsp.dtsi
index 7c9e0fae9bb9..b6142bda661e 100644
--- a/arch/arm/boot/dts/bcm-nsp.dtsi
+++ b/arch/arm/boot/dts/bcm-nsp.dtsi
@@ -160,7 +160,7 @@
 
 	axi {
 		compatible = "simple-bus";
-		ranges = <0x00000000 0x18000000 0x0011ba08>;
+		ranges = <0x00000000 0x18000000 0x0011c40a>;
 		#address-cells = <1>;
 		#size-cells = <1>;
 
@@ -241,6 +241,16 @@
 			brcm,nand-has-wp;
 		};
 
+		gpiob: gpio@30000 {
+			compatible = "brcm,iproc-nsp-gpio", "brcm,iproc-gpio";
+			reg = <0x30000 0x50>;
+			#gpio-cells = <2>;
+			gpio-controller;
+			ngpios = <4>;
+			interrupt-controller;
+			interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
 		pwm: pwm@31000 {
 			compatible = "brcm,iproc-pwm";
 			reg = <0x31000 0x28>;
@@ -254,6 +264,35 @@
 			reg = <0x33000 0x14>;
 		};
 
+		qspi: qspi@27200 {
+			compatible = "brcm,spi-bcm-qspi", "brcm,spi-nsp-qspi";
+			reg = <0x027200 0x184>,
+			      <0x027000 0x124>,
+			      <0x11c408 0x004>,
+			      <0x0273a0 0x01c>;
+			reg-names = "mspi", "bspi", "intr_regs",
+				    "intr_status_reg";
+			interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "spi_lr_fullness_reached",
+					  "spi_lr_session_aborted",
+					  "spi_lr_impatient",
+					  "spi_lr_session_done",
+					  "spi_lr_overhead",
+					  "mspi_done",
+					  "mspi_halted";
+			clocks = <&iprocmed>;
+			clock-names = "iprocmed";
+			num-cs = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
 		ccbtimer0: timer@34000 {
 			compatible = "arm,sp804";
 			reg = <0x34000 0x1000>;
diff --git a/arch/arm/boot/dts/bcm2835-rpi-a-plus.dts b/arch/arm/boot/dts/bcm2835-rpi-a-plus.dts
index f7f9db355d98..d0704540db6b 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-a-plus.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-a-plus.dts
@@ -22,7 +22,72 @@
 };
 
 &gpio {
-	pinctrl-0 = <&gpioout &alt0 &i2s_alt0 &alt3>;
+	/*
+	 * This is based on the unreleased schematic for the Model A+.
+	 *
+	 * Legend:
+	 * "NC" = not connected (no rail from the SoC)
+	 * "FOO" = GPIO line named "FOO" on the schematic
+	 * "FOO_N" = GPIO line named "FOO" on schematic, active low
+	 */
+	gpio-line-names = "SDA0",
+			  "SCL0",
+			  "SDA1",
+			  "SCL1",
+			  "GPIO_GCLK",
+			  "GPIO5",
+			  "GPIO6",
+			  "SPI_CE1_N",
+			  "SPI_CE0_N",
+			  "SPI_MISO",
+			  "SPI_MOSI",
+			  "SPI_SCLK",
+			  "GPIO12",
+			  "GPIO13",
+			  /* Serial port */
+			  "TXD0",
+			  "RXD0",
+			  "GPIO16",
+			  "GPIO17",
+			  "GPIO18",
+			  "GPIO19",
+			  "GPIO20",
+			  "GPIO21",
+			  "GPIO22",
+			  "GPIO23",
+			  "GPIO24",
+			  "GPIO25",
+			  "GPIO26",
+			  "GPIO27",
+			  "SDA0",
+			  "SCL0",
+			  "NC", /* GPIO30 */
+			  "NC", /* GPIO31 */
+			  "CAM_GPIO1", /* GPIO32 */
+			  "NC", /* GPIO33 */
+			  "NC", /* GPIO34 */
+			  "PWR_LOW_N", /* GPIO35 */
+			  "NC", /* GPIO36 */
+			  "NC", /* GPIO37 */
+			  "USB_LIMIT", /* GPIO38 */
+			  "NC", /* GPIO39 */
+			  "PWM0_OUT", /* GPIO40 */
+			  "CAM_GPIO0", /* GPIO41 */
+			  "NC", /* GPIO42 */
+			  "NC", /* GPIO43 */
+			  "NC", /* GPIO44 */
+			  "PWM1_OUT", /* GPIO45 */
+			  "HDMI_HPD_N",
+			  "STATUS_LED",
+			  /* Used by SD Card */
+			  "SD_CLK_R",
+			  "SD_CMD_R",
+			  "SD_DATA0_R",
+			  "SD_DATA1_R",
+			  "SD_DATA2_R",
+			  "SD_DATA3_R";
+
+	pinctrl-0 = <&gpioout &alt0 &i2s_alt0>;
 
 	/* I2S interface */
 	i2s_alt0: i2s_alt0 {
diff --git a/arch/arm/boot/dts/bcm2835-rpi-a.dts b/arch/arm/boot/dts/bcm2835-rpi-a.dts
index 8be102f5d826..46d078e29017 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-a.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-a.dts
@@ -15,7 +15,74 @@
 };
 
 &gpio {
-	pinctrl-0 = <&gpioout &alt0 &i2s_alt2 &alt3>;
+	/*
+	 * Taken from Raspberry-Pi-Rev-1.0-Model-AB-Schematics.pdf
+	 * RPI00021 sheet 02
+	 *
+	 * Legend:
+	 * "NC" = not connected (no rail from the SoC)
+	 * "FOO" = GPIO line named "FOO" on the schematic
+	 * "FOO_N" = GPIO line named "FOO" on schematic, active low
+	 */
+	gpio-line-names = "SDA0",
+			  "SCL0",
+			  "SDA1",
+			  "SCL1",
+			  "GPIO_GCLK",
+			  "CAM_GPIO1",
+			  "LAN_RUN",
+			  "SPI_CE1_N",
+			  "SPI_CE0_N",
+			  "SPI_MISO",
+			  "SPI_MOSI",
+			  "SPI_SCLK",
+			  "NC", /* GPIO12 */
+			  "NC", /* GPIO13 */
+			  /* Serial port */
+			  "TXD0",
+			  "RXD0",
+			  "STATUS_LED_N",
+			  "GPIO17",
+			  "GPIO18",
+			  "NC", /* GPIO19 */
+			  "NC", /* GPIO20 */
+			  "GPIO21",
+			  "GPIO22",
+			  "GPIO23",
+			  "GPIO24",
+			  "GPIO25",
+			  "NC", /* GPIO26 */
+			  "CAM_GPIO0",
+			  /* Binary number representing build/revision */
+			  "CONFIG0",
+			  "CONFIG1",
+			  "CONFIG2",
+			  "CONFIG3",
+			  "NC", /* GPIO32 */
+			  "NC", /* GPIO33 */
+			  "NC", /* GPIO34 */
+			  "NC", /* GPIO35 */
+			  "NC", /* GPIO36 */
+			  "NC", /* GPIO37 */
+			  "NC", /* GPIO38 */
+			  "NC", /* GPIO39 */
+			  "PWM0_OUT",
+			  "NC", /* GPIO41 */
+			  "NC", /* GPIO42 */
+			  "NC", /* GPIO43 */
+			  "NC", /* GPIO44 */
+			  "PWM1_OUT",
+			  "HDMI_HPD_P",
+			  "SD_CARD_DET",
+			  /* Used by SD Card */
+			  "SD_CLK_R",
+			  "SD_CMD_R",
+			  "SD_DATA0_R",
+			  "SD_DATA1_R",
+			  "SD_DATA2_R",
+			  "SD_DATA3_R";
+
+	pinctrl-0 = <&gpioout &alt0 &i2s_alt2>;
 
 	/* I2S interface */
 	i2s_alt2: i2s_alt2 {
diff --git a/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts b/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts
index 35cde65c975e..432088ebb0a1 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts
@@ -23,7 +23,73 @@
 };
 
 &gpio {
-	pinctrl-0 = <&gpioout &alt0 &i2s_alt0 &alt3>;
+	/*
+	 * Taken from Raspberry-Pi-B-Plus-V1.2-Schematics.pdf
+	 * RPI-BPLUS sheet 1
+	 *
+	 * Legend:
+	 * "NC" = not connected (no rail from the SoC)
+	 * "FOO" = GPIO line named "FOO" on the schematic
+	 * "FOO_N" = GPIO line named "FOO" on schematic, active low
+	 */
+	gpio-line-names = "SDA0",
+			  "SCL0",
+			  "SDA1",
+			  "SCL1",
+			  "GPIO_GCLK",
+			  "GPIO5",
+			  "GPIO6",
+			  "SPI_CE1_N",
+			  "SPI_CE0_N",
+			  "SPI_MISO",
+			  "SPI_MOSI",
+			  "SPI_SCLK",
+			  "GPIO12",
+			  "GPIO13",
+			  /* Serial port */
+			  "TXD0",
+			  "RXD0",
+			  "GPIO16",
+			  "GPIO17",
+			  "GPIO18",
+			  "GPIO19",
+			  "GPIO20",
+			  "GPIO21",
+			  "GPIO22",
+			  "GPIO23",
+			  "GPIO24",
+			  "GPIO25",
+			  "GPIO26",
+			  "GPIO27",
+			  "SDA0",
+			  "SCL0",
+			  "NC", /* GPIO30 */
+			  "LAN_RUN", /* GPIO31 */
+			  "CAM_GPIO1", /* GPIO32 */
+			  "NC", /* GPIO33 */
+			  "NC", /* GPIO34 */
+			  "PWR_LOW_N", /* GPIO35 */
+			  "NC", /* GPIO36 */
+			  "NC", /* GPIO37 */
+			  "USB_LIMIT", /* GPIO38 */
+			  "NC", /* GPIO39 */
+			  "PWM0_OUT", /* GPIO40 */
+			  "CAM_GPIO0", /* GPIO41 */
+			  "NC", /* GPIO42 */
+			  "NC", /* GPIO43 */
+			  "ETHCLK", /* GPIO44 */
+			  "PWM1_OUT", /* GPIO45 */
+			  "HDMI_HPD_N",
+			  "STATUS_LED",
+			  /* Used by SD Card */
+			  "SD_CLK_R",
+			  "SD_CMD_R",
+			  "SD_DATA0_R",
+			  "SD_DATA1_R",
+			  "SD_DATA2_R",
+			  "SD_DATA3_R";
+
+	pinctrl-0 = <&gpioout &alt0 &i2s_alt0>;
 
 	/* I2S interface */
 	i2s_alt0: i2s_alt0 {
diff --git a/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts b/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts
index 84df85ea6296..4133bc2cd9be 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts
@@ -16,7 +16,73 @@
 };
 
 &gpio {
-	pinctrl-0 = <&gpioout &alt0 &i2s_alt2 &alt3>;
+	/*
+	 * Taken from Raspberry-Pi-Rev-2.0-Model-AB-Schematics.pdf
+	 * RPI00022 sheet 02
+	 *
+	 * Legend:
+	 * "NC" = not connected (no rail from the SoC)
+	 * "FOO" = GPIO line named "FOO" on the schematic
+	 * "FOO_N" = GPIO line named "FOO" on schematic, active low
+	 */
+	gpio-line-names = "SDA0",
+			  "SCL0",
+			  "SDA1",
+			  "SCL1",
+			  "GPIO_GCLK",
+			  "CAM_CLK",
+			  "LAN_RUN",
+			  "SPI_CE1_N",
+			  "SPI_CE0_N",
+			  "SPI_MISO",
+			  "SPI_MOSI",
+			  "SPI_SCLK",
+			  "NC", /* GPIO12 */
+			  "NC", /* GPIO13 */
+			  /* Serial port */
+			  "TXD0",
+			  "RXD0",
+			  "STATUS_LED_N",
+			  "GPIO17",
+			  "GPIO18",
+			  "NC", /* GPIO19 */
+			  "NC", /* GPIO20 */
+			  "CAM_GPIO",
+			  "GPIO22",
+			  "GPIO23",
+			  "GPIO24",
+			  "GPIO25",
+			  "NC", /* GPIO26 */
+			  "GPIO27",
+			  "GPIO28",
+			  "GPIO29",
+			  "GPIO30",
+			  "GPIO31",
+			  "NC", /* GPIO32 */
+			  "NC", /* GPIO33 */
+			  "NC", /* GPIO34 */
+			  "NC", /* GPIO35 */
+			  "NC", /* GPIO36 */
+			  "NC", /* GPIO37 */
+			  "NC", /* GPIO38 */
+			  "NC", /* GPIO39 */
+			  "PWM0_OUT",
+			  "NC", /* GPIO41 */
+			  "NC", /* GPIO42 */
+			  "NC", /* GPIO43 */
+			  "NC", /* GPIO44 */
+			  "PWM1_OUT",
+			  "HDMI_HPD_P",
+			  "SD_CARD_DET",
+			  /* Used by SD Card */
+			  "SD_CLK_R",
+			  "SD_CMD_R",
+			  "SD_DATA0_R",
+			  "SD_DATA1_R",
+			  "SD_DATA2_R",
+			  "SD_DATA3_R";
+
+	pinctrl-0 = <&gpioout &alt0 &i2s_alt2>;
 
 	/* I2S interface */
 	i2s_alt2: i2s_alt2 {
diff --git a/arch/arm/boot/dts/bcm2835-rpi-b.dts b/arch/arm/boot/dts/bcm2835-rpi-b.dts
index 8e626a80fe24..4d56fe3006b0 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-b.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-b.dts
@@ -16,7 +16,74 @@
 };
 
 &gpio {
-	pinctrl-0 = <&gpioout &alt0 &alt3>;
+	/*
+	 * Taken from Raspberry-Pi-Rev-1.0-Model-AB-Schematics.pdf
+	 * RPI00021 sheet 02
+	 *
+	 * Legend:
+	 * "NC" = not connected (no rail from the SoC)
+	 * "FOO" = GPIO line named "FOO" on the schematic
+	 * "FOO_N" = GPIO line named "FOO" on schematic, active low
+	 */
+	gpio-line-names = "SDA0",
+			  "SCL0",
+			  "SDA1",
+			  "SCL1",
+			  "GPIO_GCLK",
+			  "CAM_GPIO1",
+			  "LAN_RUN",
+			  "SPI_CE1_N",
+			  "SPI_CE0_N",
+			  "SPI_MISO",
+			  "SPI_MOSI",
+			  "SPI_SCLK",
+			  "NC", /* GPIO12 */
+			  "NC", /* GPIO13 */
+			  /* Serial port */
+			  "TXD0",
+			  "RXD0",
+			  "STATUS_LED_N",
+			  "GPIO17",
+			  "GPIO18",
+			  "NC", /* GPIO19 */
+			  "NC", /* GPIO20 */
+			  "GPIO21",
+			  "GPIO22",
+			  "GPIO23",
+			  "GPIO24",
+			  "GPIO25",
+			  "NC", /* GPIO26 */
+			  "CAM_GPIO0",
+			  /* Binary number representing build/revision */
+			  "CONFIG0",
+			  "CONFIG1",
+			  "CONFIG2",
+			  "CONFIG3",
+			  "NC", /* GPIO32 */
+			  "NC", /* GPIO33 */
+			  "NC", /* GPIO34 */
+			  "NC", /* GPIO35 */
+			  "NC", /* GPIO36 */
+			  "NC", /* GPIO37 */
+			  "NC", /* GPIO38 */
+			  "NC", /* GPIO39 */
+			  "PWM0_OUT",
+			  "NC", /* GPIO41 */
+			  "NC", /* GPIO42 */
+			  "NC", /* GPIO43 */
+			  "NC", /* GPIO44 */
+			  "PWM1_OUT",
+			  "HDMI_HPD_P",
+			  "SD_CARD_DET",
+			  /* Used by SD Card */
+			  "SD_CLK_R",
+			  "SD_CMD_R",
+			  "SD_DATA0_R",
+			  "SD_DATA1_R",
+			  "SD_DATA2_R",
+			  "SD_DATA3_R";
+
+	pinctrl-0 = <&gpioout &alt0>;
 };
 
 &hdmi {
diff --git a/arch/arm/boot/dts/bcm2835-rpi-zero.dts b/arch/arm/boot/dts/bcm2835-rpi-zero.dts
index 60e359fafc5b..cc8b832c4c78 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-zero.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-zero.dts
@@ -26,7 +26,72 @@
 };
 
 &gpio {
-	pinctrl-0 = <&gpioout &alt0 &i2s_alt0 &alt3>;
+	/*
+	 * This is based on the official GPU firmware DT blob.
+	 *
+	 * Legend:
+	 * "NC" = not connected (no rail from the SoC)
+	 * "FOO" = GPIO line named "FOO" on the schematic
+	 * "FOO_N" = GPIO line named "FOO" on schematic, active low
+	 */
+	gpio-line-names = "SDA0",
+			  "SCL0",
+			  "SDA1",
+			  "SCL1",
+			  "GPIO_GCLK",
+			  "GPIO5",
+			  "GPIO6",
+			  "SPI_CE1_N",
+			  "SPI_CE0_N",
+			  "SPI_MISO",
+			  "SPI_MOSI",
+			  "SPI_SCLK",
+			  "GPIO12",
+			  "GPIO13",
+			  /* Serial port */
+			  "TXD0",
+			  "RXD0",
+			  "GPIO16",
+			  "GPIO17",
+			  "GPIO18",
+			  "GPIO19",
+			  "GPIO20",
+			  "GPIO21",
+			  "GPIO22",
+			  "GPIO23",
+			  "GPIO24",
+			  "GPIO25",
+			  "GPIO26",
+			  "GPIO27",
+			  "SDA0",
+			  "SCL0",
+			  "NC", /* GPIO30 */
+			  "NC", /* GPIO31 */
+			  "CAM_GPIO1", /* GPIO32 */
+			  "NC", /* GPIO33 */
+			  "NC", /* GPIO34 */
+			  "NC", /* GPIO35 */
+			  "NC", /* GPIO36 */
+			  "NC", /* GPIO37 */
+			  "NC", /* GPIO38 */
+			  "NC", /* GPIO39 */
+			  "NC", /* GPIO40 */
+			  "CAM_GPIO0", /* GPIO41 */
+			  "NC", /* GPIO42 */
+			  "NC", /* GPIO43 */
+			  "NC", /* GPIO44 */
+			  "NC", /* GPIO45 */
+			  "HDMI_HPD_N",
+			  "STATUS_LED_N",
+			  /* Used by SD Card */
+			  "SD_CLK_R",
+			  "SD_CMD_R",
+			  "SD_DATA0_R",
+			  "SD_DATA1_R",
+			  "SD_DATA2_R",
+			  "SD_DATA3_R";
+
+	pinctrl-0 = <&gpioout &alt0 &i2s_alt0>;
 
 	/* I2S interface */
 	i2s_alt0: i2s_alt0 {
diff --git a/arch/arm/boot/dts/bcm2835-rpi.dtsi b/arch/arm/boot/dts/bcm2835-rpi.dtsi
index e9b47b2bbc33..6ddf7dfe3f72 100644
--- a/arch/arm/boot/dts/bcm2835-rpi.dtsi
+++ b/arch/arm/boot/dts/bcm2835-rpi.dtsi
@@ -39,22 +39,21 @@
 	};
 
 	alt0: alt0 {
-		brcm,pins = <0 1 2 3 4 5 7 8 9 10 11 14 15 40 45>;
+		brcm,pins = <4 5 7 8 9 10 11 14 15>;
 		brcm,function = <BCM2835_FSEL_ALT0>;
 	};
-
-	alt3: alt3 {
-		brcm,pins = <48 49 50 51 52 53>;
-		brcm,function = <BCM2835_FSEL_ALT3>;
-	};
 };
 
 &i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_gpio0>;
 	status = "okay";
 	clock-frequency = <100000>;
 };
 
 &i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_gpio2>;
 	status = "okay";
 	clock-frequency = <100000>;
 };
@@ -64,11 +63,15 @@
 };
 
 &sdhci {
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_gpio48>;
 	status = "okay";
 	bus-width = <4>;
 };
 
 &pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm0_gpio40 &pwm1_gpio45>;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/bcm2835.dtsi b/arch/arm/boot/dts/bcm2835.dtsi
index a78759e73710..0890d97e674d 100644
--- a/arch/arm/boot/dts/bcm2835.dtsi
+++ b/arch/arm/boot/dts/bcm2835.dtsi
@@ -23,3 +23,9 @@
 		};
 	};
 };
+
+/* enable thermal sensor with the correct compatible property set */
+&thermal {
+	compatible = "brcm,bcm2835-thermal";
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/bcm2836-rpi-2-b.dts b/arch/arm/boot/dts/bcm2836-rpi-2-b.dts
index 39dccf62ac96..bf19e8cfb9e6 100644
--- a/arch/arm/boot/dts/bcm2836-rpi-2-b.dts
+++ b/arch/arm/boot/dts/bcm2836-rpi-2-b.dts
@@ -27,7 +27,7 @@
 };
 
 &gpio {
-	pinctrl-0 = <&gpioout &alt0 &i2s_alt0 &alt3>;
+	pinctrl-0 = <&gpioout &alt0 &i2s_alt0>;
 
 	/* I2S interface */
 	i2s_alt0: i2s_alt0 {
diff --git a/arch/arm/boot/dts/bcm2836.dtsi b/arch/arm/boot/dts/bcm2836.dtsi
index 9d0651d8f373..519a44f5d25a 100644
--- a/arch/arm/boot/dts/bcm2836.dtsi
+++ b/arch/arm/boot/dts/bcm2836.dtsi
@@ -76,3 +76,9 @@
 	interrupt-parent = <&local_intc>;
 	interrupts = <8>;
 };
+
+/* enable thermal sensor with the correct compatible property set */
+&thermal {
+	compatible = "brcm,bcm2836-thermal";
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/bcm283x.dtsi b/arch/arm/boot/dts/bcm283x.dtsi
index 46d46d894a44..9a44da190897 100644
--- a/arch/arm/boot/dts/bcm283x.dtsi
+++ b/arch/arm/boot/dts/bcm283x.dtsi
@@ -104,7 +104,7 @@
 			reg = <0x7e104000 0x10>;
 		};
 
-		mailbox: mailbox@7e00b800 {
+		mailbox: mailbox@7e00b880 {
 			compatible = "brcm,bcm2835-mbox";
 			reg = <0x7e00b880 0x40>;
 			interrupts = <0 1>;
@@ -132,6 +132,209 @@
 
 			interrupt-controller;
 			#interrupt-cells = <2>;
+
+			/* Defines pin muxing groups according to
+			 * BCM2835-ARM-Peripherals.pdf page 102.
+			 *
+			 * While each pin can have its mux selected
+			 * for various functions individually, some
+			 * groups only make sense to switch to a
+			 * particular function together.
+			 */
+			dpi_gpio0: dpi_gpio0 {
+				brcm,pins = <0 1 2 3 4 5 6 7 8 9 10 11
+					     12 13 14 15 16 17 18 19
+					     20 21 22 23 24 25 26 27>;
+				brcm,function = <BCM2835_FSEL_ALT2>;
+			};
+			emmc_gpio22: emmc_gpio22 {
+				brcm,pins = <22 23 24 25 26 27>;
+				brcm,function = <BCM2835_FSEL_ALT3>;
+			};
+			emmc_gpio34: emmc_gpio34 {
+				brcm,pins = <34 35 36 37 38 39>;
+				brcm,function = <BCM2835_FSEL_ALT3>;
+				brcm,pull = <BCM2835_PUD_OFF
+					     BCM2835_PUD_UP
+					     BCM2835_PUD_UP
+					     BCM2835_PUD_UP
+					     BCM2835_PUD_UP
+					     BCM2835_PUD_UP>;
+			};
+			emmc_gpio48: emmc_gpio48 {
+				brcm,pins = <48 49 50 51 52 53>;
+				brcm,function = <BCM2835_FSEL_ALT3>;
+			};
+
+			gpclk0_gpio4: gpclk0_gpio4 {
+				brcm,pins = <4>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			gpclk1_gpio5: gpclk1_gpio5 {
+				brcm,pins = <5>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			gpclk1_gpio42: gpclk1_gpio42 {
+				brcm,pins = <42>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			gpclk1_gpio44: gpclk1_gpio44 {
+				brcm,pins = <44>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			gpclk2_gpio6: gpclk2_gpio6 {
+				brcm,pins = <6>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			gpclk2_gpio43: gpclk2_gpio43 {
+				brcm,pins = <43>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+
+			i2c0_gpio0: i2c0_gpio0 {
+				brcm,pins = <0 1>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			i2c0_gpio32: i2c0_gpio32 {
+				brcm,pins = <32 34>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			i2c0_gpio44: i2c0_gpio44 {
+				brcm,pins = <44 45>;
+				brcm,function = <BCM2835_FSEL_ALT1>;
+			};
+			i2c1_gpio2: i2c1_gpio2 {
+				brcm,pins = <2 3>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			i2c1_gpio44: i2c1_gpio44 {
+				brcm,pins = <44 45>;
+				brcm,function = <BCM2835_FSEL_ALT2>;
+			};
+			i2c_slave_gpio18: i2c_slave_gpio18 {
+				brcm,pins = <18 19 20 21>;
+				brcm,function = <BCM2835_FSEL_ALT3>;
+			};
+
+			jtag_gpio4: jtag_gpio4 {
+				brcm,pins = <4 5 6 12 13>;
+				brcm,function = <BCM2835_FSEL_ALT4>;
+			};
+			jtag_gpio22: jtag_gpio22 {
+				brcm,pins = <22 23 24 25 26 27>;
+				brcm,function = <BCM2835_FSEL_ALT4>;
+			};
+
+			pcm_gpio18: pcm_gpio18 {
+				brcm,pins = <18 19 20 21>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			pcm_gpio28: pcm_gpio28 {
+				brcm,pins = <28 29 30 31>;
+				brcm,function = <BCM2835_FSEL_ALT2>;
+			};
+
+			pwm0_gpio12: pwm0_gpio12 {
+				brcm,pins = <12>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			pwm0_gpio18: pwm0_gpio18 {
+				brcm,pins = <18>;
+				brcm,function = <BCM2835_FSEL_ALT5>;
+			};
+			pwm0_gpio40: pwm0_gpio40 {
+				brcm,pins = <40>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			pwm1_gpio13: pwm1_gpio13 {
+				brcm,pins = <13>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			pwm1_gpio19: pwm1_gpio19 {
+				brcm,pins = <19>;
+				brcm,function = <BCM2835_FSEL_ALT5>;
+			};
+			pwm1_gpio41: pwm1_gpio41 {
+				brcm,pins = <41>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			pwm1_gpio45: pwm1_gpio45 {
+				brcm,pins = <45>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+
+			sdhost_gpio48: sdhost_gpio48 {
+				brcm,pins = <48 49 50 51 52 53>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+
+			spi0_gpio7: spi0_gpio7 {
+				brcm,pins = <7 8 9 10 11>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			spi0_gpio35: spi0_gpio35 {
+				brcm,pins = <35 36 37 38 39>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			spi1_gpio16: spi1_gpio16 {
+				brcm,pins = <16 17 18 19 20 21>;
+				brcm,function = <BCM2835_FSEL_ALT4>;
+			};
+			spi2_gpio40: spi2_gpio40 {
+				brcm,pins = <40 41 42 43 44 45>;
+				brcm,function = <BCM2835_FSEL_ALT4>;
+			};
+
+			uart0_gpio14: uart0_gpio14 {
+				brcm,pins = <14 15>;
+				brcm,function = <BCM2835_FSEL_ALT0>;
+			};
+			/* Separate from the uart0_gpio14 group
+			 * because it conflicts with spi1_gpio16, and
+			 * people often run uart0 on the two pins
+			 * without flow contrl.
+			 */
+			uart0_ctsrts_gpio16: uart0_ctsrts_gpio16 {
+				brcm,pins = <16 17>;
+				brcm,function = <BCM2835_FSEL_ALT3>;
+			};
+			uart0_gpio30: uart0_gpio30 {
+				brcm,pins = <30 31>;
+				brcm,function = <BCM2835_FSEL_ALT3>;
+			};
+			uart0_ctsrts_gpio32: uart0_ctsrts_gpio32 {
+				brcm,pins = <32 33>;
+				brcm,function = <BCM2835_FSEL_ALT3>;
+			};
+
+			uart1_gpio14: uart1_gpio14 {
+				brcm,pins = <14 15>;
+				brcm,function = <BCM2835_FSEL_ALT5>;
+			};
+			uart1_ctsrts_gpio16: uart1_ctsrts_gpio16 {
+				brcm,pins = <16 17>;
+				brcm,function = <BCM2835_FSEL_ALT5>;
+			};
+			uart1_gpio32: uart1_gpio32 {
+				brcm,pins = <32 33>;
+				brcm,function = <BCM2835_FSEL_ALT5>;
+			};
+			uart1_ctsrts_gpio30: uart1_ctsrts_gpio30 {
+				brcm,pins = <30 31>;
+				brcm,function = <BCM2835_FSEL_ALT5>;
+			};
+			uart1_gpio36: uart1_gpio36 {
+				brcm,pins = <36 37 38 39>;
+				brcm,function = <BCM2835_FSEL_ALT2>;
+			};
+			uart1_gpio40: uart1_gpio40 {
+				brcm,pins = <40 41>;
+				brcm,function = <BCM2835_FSEL_ALT5>;
+			};
+			uart1_ctsrts_gpio42: uart1_ctsrts_gpio42 {
+				brcm,pins = <42 43>;
+				brcm,function = <BCM2835_FSEL_ALT5>;
+			};
 		};
 
 		uart0: serial@7e201000 {
@@ -187,6 +390,13 @@
 			interrupts = <2 14>; /* pwa1 */
 		};
 
+		thermal: thermal@7e212000 {
+			compatible = "brcm,bcm2835-thermal";
+			reg = <0x7e212000 0x8>;
+			clocks = <&clocks BCM2835_CLOCK_TSENS>;
+			status = "disabled";
+		};
+
 		aux: aux@0x7e215000 {
 			compatible = "brcm,bcm2835-aux";
 			#clock-cells = <1>;
diff --git a/arch/arm/boot/dts/bcm4708-luxul-xap-1510.dts b/arch/arm/boot/dts/bcm4708-luxul-xap-1510.dts
new file mode 100644
index 000000000000..35e6ed6a3ef7
--- /dev/null
+++ b/arch/arm/boot/dts/bcm4708-luxul-xap-1510.dts
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2016 Luxul Inc.
+ *
+ * Licensed under the ISC license.
+ */
+
+/dts-v1/;
+
+#include "bcm4708.dtsi"
+
+/ {
+	compatible = "luxul,xap-1510v1", "brcm,bcm4708";
+	model = "Luxul XAP-1510 V1";
+
+	chosen {
+		bootargs = "console=ttyS0,115200 earlycon";
+	};
+
+	memory {
+		reg = <0x00000000 0x08000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		5ghz {
+			label = "bcm53xx:blue:5ghz";
+			gpios = <&chipcommon 13 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "none";
+		};
+
+		2ghz {
+			label = "bcm53xx:blue:2ghz";
+			gpios = <&chipcommon 14 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "none";
+		};
+
+		status {
+			label = "bcm53xx:green:status";
+			gpios = <&chipcommon 15 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "timer";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		restart {
+			label = "Reset";
+			linux,code = <KEY_RESTART>;
+			gpios = <&chipcommon 11 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&spi_nor {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts b/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts
index 8ade7def2e8a..eac0f52e5ebd 100644
--- a/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts
+++ b/arch/arm/boot/dts/bcm4709-asus-rt-ac87u.dts
@@ -9,7 +9,7 @@
 
 /dts-v1/;
 
-#include "bcm4708.dtsi"
+#include "bcm4709.dtsi"
 #include "bcm5301x-nand-cs0-bch8.dtsi"
 
 / {
diff --git a/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts b/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts
index 0653e7ef248c..aab39c9864da 100644
--- a/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts
+++ b/arch/arm/boot/dts/bcm4709-buffalo-wxr-1900dhp.dts
@@ -9,7 +9,7 @@
 
 /dts-v1/;
 
-#include "bcm4708.dtsi"
+#include "bcm4709.dtsi"
 #include "bcm5301x-nand-cs0-bch8.dtsi"
 
 / {
diff --git a/arch/arm/boot/dts/bcm4709-netgear-r7000.dts b/arch/arm/boot/dts/bcm4709-netgear-r7000.dts
index a22ed144040b..fd38d2aa3521 100644
--- a/arch/arm/boot/dts/bcm4709-netgear-r7000.dts
+++ b/arch/arm/boot/dts/bcm4709-netgear-r7000.dts
@@ -9,7 +9,7 @@
 
 /dts-v1/;
 
-#include "bcm4708.dtsi"
+#include "bcm4709.dtsi"
 #include "bcm5301x-nand-cs0-bch8.dtsi"
 
 / {
diff --git a/arch/arm/boot/dts/bcm4709-netgear-r8000.dts b/arch/arm/boot/dts/bcm4709-netgear-r8000.dts
index ca181516c28a..92f8a7219e98 100644
--- a/arch/arm/boot/dts/bcm4709-netgear-r8000.dts
+++ b/arch/arm/boot/dts/bcm4709-netgear-r8000.dts
@@ -9,7 +9,7 @@
 
 /dts-v1/;
 
-#include "bcm4708.dtsi"
+#include "bcm4709.dtsi"
 #include "bcm5301x-nand-cs0-bch8.dtsi"
 
 / {
@@ -107,6 +107,10 @@
 	};
 };
 
+&uart0 {
+	status = "okay";
+};
+
 &usb2 {
 	vcc-gpio = <&chipcommon 0 GPIO_ACTIVE_HIGH>;
 };
diff --git a/arch/arm/boot/dts/bcm4709-tplink-archer-c9-v1.dts b/arch/arm/boot/dts/bcm4709-tplink-archer-c9-v1.dts
new file mode 100644
index 000000000000..9a92c24ac2d8
--- /dev/null
+++ b/arch/arm/boot/dts/bcm4709-tplink-archer-c9-v1.dts
@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) 2016 Rafał Miłecki <rafal@milecki.pl>
+ *
+ * Licensed under the ISC license.
+ */
+
+/dts-v1/;
+
+#include "bcm4709.dtsi"
+
+/ {
+	compatible = "tplink,archer-c9-v1", "brcm,bcm4709", "brcm,bcm4708";
+	model = "TP-LINK Archer C9 V1";
+
+	chosen {
+		bootargs = "console=ttyS0,115200 earlycon";
+	};
+
+	memory {
+		reg = <0x00000000 0x08000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		lan {
+			label = "bcm53xx:blue:lan";
+			gpios = <&chipcommon 1 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-off";
+		};
+
+		wps {
+			label = "bcm53xx:blue:wps";
+			gpios = <&chipcommon 2 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-off";
+		};
+
+		2ghz {
+			label = "bcm53xx:blue:2ghz";
+			gpios = <&chipcommon 4 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-off";
+		};
+
+		5ghz {
+			label = "bcm53xx:blue:5ghz";
+			gpios = <&chipcommon 5 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-off";
+		};
+
+		usb3 {
+			label = "bcm53xx:blue:usb3";
+			gpios = <&chipcommon 6 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-off";
+		};
+
+		usb2 {
+			label = "bcm53xx:blue:usb2";
+			gpios = <&chipcommon 7 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-off";
+		};
+
+		wan-blue {
+			label = "bcm53xx:blue:wan";
+			gpios = <&chipcommon 14 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-off";
+		};
+
+		wan-amber {
+			label = "bcm53xx:amber:wan";
+			gpios = <&chipcommon 15 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-off";
+		};
+
+		power {
+			label = "bcm53xx:blue:power";
+			gpios = <&chipcommon 18 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-on";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		wps {
+			label = "WPS";
+			linux,code = <KEY_WPS_BUTTON>;
+			gpios = <&chipcommon 0 GPIO_ACTIVE_LOW>;
+		};
+
+		restart {
+			label = "Reset";
+			linux,code = <KEY_RESTART>;
+			gpios = <&chipcommon 3 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&usb2 {
+	vcc-gpio = <&chipcommon 13 GPIO_ACTIVE_HIGH>;
+};
+
+&usb3 {
+	vcc-gpio = <&chipcommon 12 GPIO_ACTIVE_HIGH>;
+};
+
+&spi_nor {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/bcm4709.dtsi b/arch/arm/boot/dts/bcm4709.dtsi
new file mode 100644
index 000000000000..f03976597a6d
--- /dev/null
+++ b/arch/arm/boot/dts/bcm4709.dtsi
@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016 Rafał Miłecki <rafal@milecki.pl>
+ *
+ * Licensed under the ISC license.
+ */
+
+#include "bcm4708.dtsi"
+
+&uart0 {
+	clock-frequency = <125000000>;
+};
diff --git a/arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts b/arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts
index c8c0b3616935..661348dbb7ce 100644
--- a/arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts
+++ b/arch/arm/boot/dts/bcm47094-dlink-dir-885l.dts
@@ -9,7 +9,7 @@
 
 /dts-v1/;
 
-#include "bcm4708.dtsi"
+#include "bcm47094.dtsi"
 #include "bcm5301x-nand-cs0-bch1.dtsi"
 
 / {
@@ -107,7 +107,6 @@
 
 &uart0 {
 	status = "okay";
-	clock-frequency = <125000000>;
 };
 
 &usb3 {
diff --git a/arch/arm/boot/dts/bcm47094-luxul-xwr-3100.dts b/arch/arm/boot/dts/bcm47094-luxul-xwr-3100.dts
new file mode 100644
index 000000000000..169b35fe5651
--- /dev/null
+++ b/arch/arm/boot/dts/bcm47094-luxul-xwr-3100.dts
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2016 Luxul Inc.
+ *
+ * Licensed under the ISC license.
+ */
+
+/dts-v1/;
+
+#include "bcm47094.dtsi"
+#include "bcm5301x-nand-cs0-bch4.dtsi"
+
+/ {
+	compatible = "luxul,xwr-3100v1", "brcm,bcm47094", "brcm,bcm4708";
+	model = "Luxul XWR-3100 V1";
+
+	chosen {
+		bootargs = "console=ttyS0,115200 earlycon";
+	};
+
+	memory {
+		reg = <0x00000000 0x08000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		power	{
+			label = "bcm53xx:green:power";
+			gpios = <&chipcommon 0 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-on";
+		};
+
+		lan3	{
+			label = "bcm53xx:green:lan1";
+			gpios = <&chipcommon 1 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+
+		lan4	{
+			label = "bcm53xx:green:lan0";
+			gpios = <&chipcommon 2 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+
+		wan	{
+			label = "bcm53xx:green:wan";
+			gpios = <&chipcommon 3 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+
+		lan1	{
+			label = "bcm53xx:green:lan3";
+			gpios = <&chipcommon 4 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+
+		lan2	{
+			label = "bcm53xx:green:lan2";
+			gpios = <&chipcommon 6 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+
+		usb3	{
+			label = "bcm53xx:green:usb3";
+			gpios = <&chipcommon 8 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+
+		status	{
+			label = "bcm53xx:green:status";
+			gpios = <&chipcommon 10 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "timer";
+		};
+
+		2ghz {
+			label = "bcm53xx:green:2ghz";
+			gpios = <&chipcommon 13 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+
+		5ghz {
+			label = "bcm53xx:green:5ghz";
+			gpios = <&chipcommon 14 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		restart {
+			label = "Reset";
+			linux,code = <KEY_RESTART>;
+			gpios = <&chipcommon 17 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&usb3 {
+	vcc-gpio = <&chipcommon 18 GPIO_ACTIVE_HIGH>;
+};
+
+&spi_nor {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/bcm47094-netgear-r8500.dts b/arch/arm/boot/dts/bcm47094-netgear-r8500.dts
new file mode 100644
index 000000000000..521b4155de60
--- /dev/null
+++ b/arch/arm/boot/dts/bcm47094-netgear-r8500.dts
@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2016 Rafał Miłecki <rafal@milecki.pl>
+ *
+ * Licensed under the ISC license.
+ */
+
+/dts-v1/;
+
+#include "bcm47094.dtsi"
+#include "bcm5301x-nand-cs0-bch8.dtsi"
+
+/ {
+	compatible = "netgear,r8500", "brcm,bcm47094", "brcm,bcm4708";
+	model = "Netgear R8500";
+
+	chosen {
+		bootargs = "console=ttyS0,115200";
+	};
+
+	memory {
+		reg = <0x00000000 0x08000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		power0 {
+			label = "bcm53xx:white:power";
+			gpios = <&chipcommon 2 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-on";
+		};
+
+		power1 {
+			label = "bcm53xx:amber:power";
+			gpios = <&chipcommon 3 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+
+		5ghz-1 {
+			label = "bcm53xx:white:5ghz-1";
+			gpios = <&chipcommon 11 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+
+		5ghz-2 {
+			label = "bcm53xx:white:5ghz-2";
+			gpios = <&chipcommon 12 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+
+		2ghz {
+			label = "bcm53xx:white:2ghz";
+			gpios = <&chipcommon 13 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+
+		usb2 {
+			label = "bcm53xx:white:usb2";
+			gpios = <&chipcommon 17 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+
+		usb3 {
+			label = "bcm53xx:white:usb3";
+			gpios = <&chipcommon 18 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		brightness {
+			label = "Backlight";
+			linux,code = <KEY_BRIGHTNESS_ZERO>;
+			gpios = <&chipcommon 1 GPIO_ACTIVE_LOW>;
+		};
+
+		restart {
+			label = "Reset";
+			linux,code = <KEY_RESTART>;
+			gpios = <&chipcommon 10 GPIO_ACTIVE_LOW>;
+		};
+
+		wps {
+			label = "WPS";
+			linux,code = <KEY_WPS_BUTTON>;
+			gpios = <&chipcommon 14 GPIO_ACTIVE_LOW>;
+		};
+
+		rfkill {
+			label = "WiFi";
+			linux,code = <KEY_RFKILL>;
+			gpios = <&chipcommon 20 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/bcm47094.dtsi b/arch/arm/boot/dts/bcm47094.dtsi
new file mode 100644
index 000000000000..4f09aa0114e6
--- /dev/null
+++ b/arch/arm/boot/dts/bcm47094.dtsi
@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) 2016 Rafał Miłecki <rafal@milecki.pl>
+ *
+ * Licensed under the ISC license.
+ */
+
+#include "bcm4708.dtsi"
+
+/ {
+	usb3_phy: usb3-phy {
+		compatible = "brcm,ns-bx-usb3-phy";
+	};
+};
+
+&uart0 {
+	clock-frequency = <125000000>;
+};
diff --git a/arch/arm/boot/dts/bcm47189-tenda-ac9.dts b/arch/arm/boot/dts/bcm47189-tenda-ac9.dts
new file mode 100644
index 000000000000..4403ae8790c2
--- /dev/null
+++ b/arch/arm/boot/dts/bcm47189-tenda-ac9.dts
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2016 Rafał Miłecki <rafal@milecki.pl>
+ *
+ * Licensed under the ISC license.
+ */
+
+/dts-v1/;
+
+#include "bcm53573.dtsi"
+
+/ {
+	compatible = "tenda,ac9", "brcm,bcm47189", "brcm,bcm53573";
+	model = "Tenda AC9";
+
+	chosen {
+		bootargs = "console=ttyS0,115200 earlycon";
+	};
+
+	memory {
+		reg = <0x00000000 0x08000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		usb {
+			label = "bcm53xx:blue:usb";
+			gpios = <&chipcommon 1 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-off";
+		};
+
+		wps {
+			label = "bcm53xx:blue:wps";
+			gpios = <&chipcommon 10 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-off";
+		};
+
+		5ghz {
+			label = "bcm53xx:blue:5ghz";
+			gpios = <&chipcommon 11 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-off";
+		};
+
+		system {
+			label = "bcm53xx:blue:system";
+			gpios = <&chipcommon 15 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "timer";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		rfkill {
+			label = "WiFi";
+			linux,code = <KEY_RFKILL>;
+			gpios = <&chipcommon 3 GPIO_ACTIVE_LOW>;
+		};
+
+		restart {
+			label = "Reset";
+			linux,code = <KEY_RESTART>;
+			gpios = <&chipcommon 7 GPIO_ACTIVE_LOW>;
+		};
+
+		wps {
+			label = "WPS";
+			linux,code = <KEY_WPS_BUTTON>;
+			gpios = <&chipcommon 9 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/bcm5301x-nand-cs0-bch4.dtsi b/arch/arm/boot/dts/bcm5301x-nand-cs0-bch4.dtsi
new file mode 100644
index 000000000000..b4e875df9528
--- /dev/null
+++ b/arch/arm/boot/dts/bcm5301x-nand-cs0-bch4.dtsi
@@ -0,0 +1,13 @@
+/*
+ * Copyright 2016 Luxul Inc.
+ *
+ * Licensed under the ISC license.
+ */
+
+#include "bcm5301x-nand-cs0.dtsi"
+
+&nandcs {
+	nand-ecc-algo = "bch";
+	nand-ecc-strength = <4>;
+	nand-ecc-step-size = <512>;
+};
diff --git a/arch/arm/boot/dts/bcm5301x.dtsi b/arch/arm/boot/dts/bcm5301x.dtsi
index ae4b3880616d..f09a2bb08979 100644
--- a/arch/arm/boot/dts/bcm5301x.dtsi
+++ b/arch/arm/boot/dts/bcm5301x.dtsi
@@ -149,6 +149,13 @@
 		clock-names = "phy-ref-clk";
 	};
 
+	usb3_phy: usb3-phy {
+		compatible = "brcm,ns-ax-usb3-phy";
+		reg = <0x18105000 0x1000>, <0x18003000 0x1000>;
+		reg-names = "dmp", "ccb-mii";
+		#phy-cells = <0>;
+	};
+
 	axi@18000000 {
 		compatible = "brcm,bus-axi";
 		reg = <0x18000000 0x1000>;
diff --git a/arch/arm/boot/dts/bcm53573.dtsi b/arch/arm/boot/dts/bcm53573.dtsi
new file mode 100644
index 000000000000..e2c496a96c32
--- /dev/null
+++ b/arch/arm/boot/dts/bcm53573.dtsi
@@ -0,0 +1,159 @@
+/*
+ * Copyright (C) 2016 Rafał Miłecki <rafal@milecki.pl>
+ *
+ * Licensed under the ISC license.
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include "skeleton.dtsi"
+
+/ {
+	interrupt-parent = <&gic>;
+
+	chosen {
+		stdout-path = &uart0;
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a7";
+			reg = <0x0>;
+		};
+	};
+
+	mpcore {
+		compatible = "simple-bus";
+		ranges = <0x00000000 0x18310000 0x00008000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		gic: interrupt-controller@1000 {
+			compatible = "arm,cortex-a7-gic";
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0x1000 0x1000>,
+			      <0x2000 0x0100>;
+		};
+	};
+
+	clocks {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		alp: oscillator {
+			#clock-cells = <0>;
+			compatible = "fixed-clock";
+			clock-frequency = <40000000>;
+		};
+	};
+
+	axi@18000000 {
+		compatible = "brcm,bus-axi";
+		reg = <0x18000000 0x1000>;
+		ranges = <0x00000000 0x18000000 0x00100000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0x000fffff 0xffff>;
+		interrupt-map =
+			/* ChipCommon */
+			<0x00000000 0 &gic GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+
+			/* IEEE 802.11 0 */
+			<0x00001000 0 &gic GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+
+			/* PCIe Controller 0 */
+			<0x00002000 0 &gic GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+			<0x00002000 1 &gic GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+			<0x00002000 2 &gic GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+			<0x00002000 3 &gic GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+			<0x00002000 4 &gic GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+			<0x00002000 5 &gic GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+
+			/* USB 2.0 Controller */
+			<0x00004000 0 &gic GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+
+			/* Ethernet Controller 0 */
+			<0x00005000 0 &gic GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+
+			/* IEEE 802.11 1 */
+			<0x0000a000 0 &gic GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>,
+
+			/* Ethernet Controller 1 */
+			<0x0000b000 0 &gic GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
+
+		chipcommon: chipcommon@0 {
+			compatible = "simple-bus";
+			reg = <0x00000000 0x1000>;
+			ranges;
+
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+
+			uart0: serial@0300 {
+				compatible = "ns16550a";
+				reg = <0x0300 0x100>;
+				interrupt-parent = <&gic>;
+				interrupts = <GIC_PPI 16 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&alp>;
+				status = "okay";
+			};
+		};
+
+		usb2: usb2@4000 {
+			reg = <0x4000 0x1000>;
+			ranges;
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			ehci: ehci@4000 {
+				compatible = "generic-ehci";
+				reg = <0x4000 0x1000>;
+				interrupt-parent = <&gic>;
+				interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>;
+			};
+
+			ohci: ohci@d000 {
+				#usb-cells = <0>;
+
+				compatible = "generic-ohci";
+				reg = <0xd000 0x1000>;
+				interrupt-parent = <&gic>;
+				interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>;
+			};
+		};
+
+		gmac0: ethernet@5000 {
+			reg = <0x5000 0x1000>;
+		};
+
+		gmac1: ethernet@b000 {
+			reg = <0xb000 0x1000>;
+		};
+
+		pmu@12000 {
+			compatible = "simple-mfd", "syscon";
+			reg = <0x00012000 0x00001000>;
+
+			ilp: ilp {
+				compatible = "brcm,bcm53573-ilp";
+				clocks = <&alp>;
+				#clock-cells = <0>;
+				clock-output-names = "ilp";
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/bcm958625k.dts b/arch/arm/boot/dts/bcm958625k.dts
index 05c5f98c8782..59d96fb91583 100644
--- a/arch/arm/boot/dts/bcm958625k.dts
+++ b/arch/arm/boot/dts/bcm958625k.dts
@@ -139,3 +139,37 @@
 		groups = "nand_grp";
 	};
 };
+
+&qspi {
+	bspi-sel = <0>;
+	flash: m25p80@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "m25p80";
+		reg = <0x0>;
+		spi-max-frequency = <12500000>;
+		m25p,fast-read;
+		spi-cpol;
+		spi-cpha;
+
+		partition@0 {
+			label = "boot";
+			reg = <0x00000000 0x000a0000>;
+		};
+
+		partition@a0000 {
+			label = "env";
+			reg = <0x000a0000 0x00060000>;
+		};
+
+		partition@100000 {
+			label = "system";
+			reg = <0x00100000 0x00600000>;
+		};
+
+		partition@700000 {
+			label = "rootfs";
+			reg = <0x00700000 0x01900000>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/berlin2q-marvell-dmp.dts b/arch/arm/boot/dts/berlin2q-marvell-dmp.dts
index f485308840ab..57aa5f8a7c77 100644
--- a/arch/arm/boot/dts/berlin2q-marvell-dmp.dts
+++ b/arch/arm/boot/dts/berlin2q-marvell-dmp.dts
@@ -48,7 +48,7 @@
 		reg = <0x00000000 0x80000000>;
 	};
 
-	choosen {
+	chosen {
 		bootargs = "earlyprintk";
 		stdout-path = "serial0:115200n8";
 	};
@@ -58,7 +58,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		reg_usb0_vbus: regulator@0 {
+		reg_usb0_vbus: regulator_usb0 {
 			compatible = "regulator-fixed";
 			regulator-name = "usb0_vbus";
 			regulator-min-microvolt = <5000000>;
@@ -67,7 +67,7 @@
 			enable-active-high;
 		};
 
-		reg_usb1_vbus: regulator@1 {
+		reg_usb1_vbus: regulator_usb1 {
 			compatible = "regulator-fixed";
 			regulator-name = "usb1_vbus";
 			regulator-min-microvolt = <5000000>;
@@ -76,7 +76,7 @@
 			enable-active-high;
 		};
 
-		reg_usb2_vbus: regulator@2 {
+		reg_usb2_vbus: regulator_usb2 {
 			compatible = "regulator-fixed";
 			regulator-name = "usb2_vbus";
 			regulator-min-microvolt = <5000000>;
@@ -85,7 +85,7 @@
 			enable-active-high;
 		};
 
-		reg_sdio1_vmmc: regulator@3 {
+		reg_sdio1_vmmc: regulator_sdio1_vmmc {
 			compatible = "regulator-fixed";
 			regulator-min-microvolt = <3300000>;
 			regulator-max-microvolt = <3300000>;
@@ -95,7 +95,7 @@
 			gpio = <&portb 21 GPIO_ACTIVE_HIGH>;
 		};
 
-		reg_sdio1_vqmmc: regulator@4 {
+		reg_sdio1_vqmmc: regulator_sido1_vqmmc {
 			compatible = "regulator-gpio";
 			regulator-min-microvolt = <1800000>;
 			regulator-max-microvolt = <3300000>;
diff --git a/arch/arm/boot/dts/cloudengines-pogoplug-series-3.dts b/arch/arm/boot/dts/cloudengines-pogoplug-series-3.dts
new file mode 100644
index 000000000000..bfde32e37123
--- /dev/null
+++ b/arch/arm/boot/dts/cloudengines-pogoplug-series-3.dts
@@ -0,0 +1,94 @@
+/*
+ * cloudengines-pogoplug-series-3.dtsi - Device tree file for Cloud Engines PogoPlug Series 3
+ *
+ * Copyright (C) 2016 Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * Licensed under GPLv2 or later
+ */
+
+/dts-v1/;
+#include "ox820.dtsi"
+
+/ {
+	model = "Cloud Engines PogoPlug Series 3";
+
+	compatible = "cloudengines,pogoplugv3", "oxsemi,ox820";
+
+	chosen {
+		bootargs = "earlyprintk";
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory {
+		/* 128Mbytes DDR */
+		reg = <0x60000000 0x8000000>;
+	};
+
+	aliases {
+		serial0 = &uart0;
+		gpio0 = &gpio0;
+		gpio1 = &gpio1;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		blue {
+			label = "pogoplug:blue";
+			gpios = <&gpio0 2 0>;
+			default-state = "keep";
+		};
+
+		orange {
+			label = "pogoplug:orange";
+			gpios = <&gpio1 16 1>;
+			default-state = "keep";
+		};
+
+		green {
+			label = "pogoplug:green";
+			gpios = <&gpio1 17 1>;
+			default-state = "keep";
+		};
+	};
+};
+
+&uart0 {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart0>;
+};
+
+&nandc {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_nand>;
+
+	nand@0 {
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		nand-ecc-mode = "soft";
+		nand-ecc-algo = "hamming";
+
+		partition@0 {
+			label = "boot";
+			reg = <0x00000000 0x00e00000>;
+			read-only;
+		};
+
+		partition@e00000 {
+			label = "ubi";
+			reg = <0x00e00000 0x07200000>;
+		};
+	};
+};
+
+&etha {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_etha_mdio>;
+};
diff --git a/arch/arm/boot/dts/da850-lcdk.dts b/arch/arm/boot/dts/da850-lcdk.dts
index 7b8ab21fed6c..afcb4821deb1 100644
--- a/arch/arm/boot/dts/da850-lcdk.dts
+++ b/arch/arm/boot/dts/da850-lcdk.dts
@@ -13,6 +13,7 @@
 
 	aliases {
 		serial2 = &serial2;
+		ethernet0 = &eth0;
 	};
 
 	chosen {
@@ -122,7 +123,7 @@
 	bus-width = <4>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc0_pins>;
-	cd-gpios = <&gpio 64 GPIO_ACTIVE_HIGH>;
+	cd-gpios = <&gpio 64 GPIO_ACTIVE_LOW>;
 	status = "okay";
 };
 
@@ -158,6 +159,14 @@
 	rx-num-evt = <32>;
 };
 
+&usb_phy {
+	status = "okay";
+};
+
+&usb0 {
+	status = "okay";
+};
+
 &aemif {
 	pinctrl-names = "default";
 	pinctrl-0 = <&nand_pins>;
@@ -219,3 +228,11 @@
 		};
 	};
 };
+
+&prictrl {
+	status = "okay";
+};
+
+&memctrl {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/da850.dtsi b/arch/arm/boot/dts/da850.dtsi
index f79e1b91c680..104155d12c2f 100644
--- a/arch/arm/boot/dts/da850.dtsi
+++ b/arch/arm/boot/dts/da850.dtsi
@@ -36,6 +36,7 @@
 			reg = <0x14120 0x50>;
 			#address-cells = <1>;
 			#size-cells = <0>;
+			#pinctrl-cells = <2>;
 			pinctrl-single,bit-per-mux;
 			pinctrl-single,register-width = <32>;
 			pinctrl-single,function-mask = <0xf>;
@@ -186,8 +187,44 @@
 					0xc 0x88888888 0xffffffff
 				>;
 			};
+			lcd_pins: pinmux_lcd_pins {
+				pinctrl-single,bits = <
+					/*
+					 * LCD_D[2], LCD_D[3], LCD_D[4], LCD_D[5],
+					 * LCD_D[6], LCD_D[7]
+					 */
+					0x40 0x22222200 0xffffff00
+					/*
+					 * LCD_D[10], LCD_D[11], LCD_D[12], LCD_D[13],
+					 * LCD_D[14], LCD_D[15], LCD_D[0], LCD_D[1]
+					 */
+					0x44 0x22222222 0xffffffff
+					/* LCD_D[8], LCD_D[9] */
+					0x48 0x00000022 0x000000ff
+
+					/* LCD_PCLK */
+					0x48 0x02000000 0x0f000000
+					/* LCD_AC_ENB_CS, LCD_VSYNC, LCD_HSYNC */
+					0x4c 0x02000022 0x0f0000ff
+				>;
+			};
 
 		};
+		prictrl: priority-controller@14110 {
+			compatible = "ti,da850-mstpri";
+			reg = <0x14110 0x0c>;
+			status = "disabled";
+		};
+		cfgchip: chip-controller@1417c {
+			compatible = "ti,da830-cfgchip", "syscon", "simple-mfd";
+			reg = <0x1417c 0x14>;
+
+			usb_phy: usb-phy {
+				compatible = "ti,da830-usb-phy";
+				#phy-cells = <1>;
+				status = "disabled";
+			};
+		};
 		edma0: edma@0 {
 			compatible = "ti,edma3-tpcc";
 			/* eDMA3 CC0: 0x01c0 0000 - 0x01c0 7fff */
@@ -280,6 +317,8 @@
 		mmc0: mmc@40000 {
 			compatible = "ti,da830-mmc";
 			reg = <0x40000 0x1000>;
+			cap-sd-highspeed;
+			cap-mmc-highspeed;
 			interrupts = <16>;
 			dmas = <&edma0 16 0>, <&edma0 17 0>;
 			dma-names = "rx", "tx";
@@ -288,6 +327,8 @@
 		mmc1: mmc@21b000 {
 			compatible = "ti,da830-mmc";
 			reg = <0x21b000 0x1000>;
+			cap-sd-highspeed;
+			cap-mmc-highspeed;
 			interrupts = <72>;
 			dmas = <&edma1 28 0>, <&edma1 29 0>;
 			dma-names = "rx", "tx";
@@ -336,6 +377,8 @@
 			num-cs = <6>;
 			ti,davinci-spi-intr-line = <1>;
 			interrupts = <20>;
+			dmas = <&edma0 14 0>, <&edma0 15 0>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 		spi1: spi@30e000 {
@@ -350,6 +393,16 @@
 			dma-names = "rx", "tx";
 			status = "disabled";
 		};
+		usb0: usb@200000 {
+			compatible = "ti,da830-musb";
+			reg = <0x200000 0x10000>;
+			interrupts = <58>;
+			interrupt-names = "mc";
+			dr_mode = "otg";
+			phys = <&usb_phy 0>;
+			phy-names = "usb-phy";
+			status = "disabled";
+		};
 		mdio: mdio@224000 {
 			compatible = "ti,davinci_mdio";
 			#address-cells = <1>;
@@ -386,6 +439,11 @@
 			ti,davinci-gpio-unbanked = <0>;
 			status = "disabled";
 		};
+		pinconf: pin-controller@22c00c {
+			compatible = "ti,da850-pupd";
+			reg = <0x22c00c 0x8>;
+			status = "disabled";
+		};
 
 		mcasp0: mcasp@100000 {
 			compatible = "ti,da830-mcasp-audio";
@@ -399,6 +457,13 @@
 				<&edma0 0 1>;
 			dma-names = "tx", "rx";
 		};
+
+		display: display@213000 {
+			compatible = "ti,da850-tilcdc";
+			reg = <0x213000 0x1000>;
+			interrupts = <52>;
+			status = "disabled";
+		};
 	};
 	aemif: aemif@68000000 {
 		compatible = "ti,da850-aemif";
@@ -410,4 +475,9 @@
 			  1 0 0x68000000 0x00008000>;
 		status = "disabled";
 	};
+	memctrl: memory-controller@b0000000 {
+		compatible = "ti,da850-ddr-controller";
+		reg = <0xb0000000 0xe8>;
+		status = "disabled";
+	};
 };
diff --git a/arch/arm/boot/dts/dm814x.dtsi b/arch/arm/boot/dts/dm814x.dtsi
index ff90a6ce6bdc..1facc5f12cef 100644
--- a/arch/arm/boot/dts/dm814x.dtsi
+++ b/arch/arm/boot/dts/dm814x.dtsi
@@ -373,6 +373,7 @@
 					reg = <0x800 0x438>;
 					#address-cells = <1>;
 					#size-cells = <0>;
+					#pinctrl-cells = <1>;
 					pinctrl-single,register-width = <32>;
 					pinctrl-single,function-mask = <0x307ff>;
 				};
diff --git a/arch/arm/boot/dts/dm816x.dtsi b/arch/arm/boot/dts/dm816x.dtsi
index f1e0f771ff29..61dd2f6b02bc 100644
--- a/arch/arm/boot/dts/dm816x.dtsi
+++ b/arch/arm/boot/dts/dm816x.dtsi
@@ -83,6 +83,7 @@
 			reg = <0x48140000 0x21000>;
 			#address-cells = <1>;
 			#size-cells = <1>;
+			#pinctrl-cells = <1>;
 			ranges = <0 0x48140000 0x21000>;
 
 			dm816x_pinmux: pinmux@800 {
@@ -90,6 +91,7 @@
 				reg = <0x800 0x50a>;
 				#address-cells = <1>;
 				#size-cells = <0>;
+				#pinctrl-cells = <1>;
 				pinctrl-single,register-width = <16>;
 				pinctrl-single,function-mask = <0xf>;
 			};
diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi
index d4fcd68f6349..addb7530cfbe 100644
--- a/arch/arm/boot/dts/dra7.dtsi
+++ b/arch/arm/boot/dts/dra7.dtsi
@@ -171,6 +171,7 @@
 					reg = <0x1400 0x0468>;
 					#address-cells = <1>;
 					#size-cells = <0>;
+					#pinctrl-cells = <1>;
 					#interrupt-cells = <1>;
 					interrupt-controller;
 					pinctrl-single,register-width = <32>;
diff --git a/arch/arm/boot/dts/dra71-evm.dts b/arch/arm/boot/dts/dra71-evm.dts
new file mode 100644
index 000000000000..2b9a5a8d69ad
--- /dev/null
+++ b/arch/arm/boot/dts/dra71-evm.dts
@@ -0,0 +1,230 @@
+/*
+ * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include "dra72-evm-common.dtsi"
+#include <dt-bindings/net/ti-dp83867.h>
+
+/ {
+	compatible = "ti,dra718-evm", "ti,dra718", "ti,dra722", "ti,dra72", "ti,dra7";
+	model = "TI DRA718 EVM";
+
+	memory {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x0 0x80000000>; /* 2GB */
+	};
+
+	vpo_sd_1v8_3v3: gpio-regulator-TPS74801 {
+		compatible = "regulator-gpio";
+
+		regulator-name = "vddshv8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3000000>;
+		regulator-boot-on;
+		vin-supply = <&evm_5v0>;
+
+		gpios = <&gpio7 11 GPIO_ACTIVE_HIGH>;
+		states = <1800000 0x0
+			  3000000 0x1>;
+	};
+
+	poweroff: gpio-poweroff {
+		compatible = "gpio-poweroff";
+		gpios = <&gpio7 30 GPIO_ACTIVE_HIGH>;
+		input;
+	};
+};
+
+&i2c1 {
+	status = "okay";
+	clock-frequency = <400000>;
+
+	lp8733: lp8733@60 {
+		compatible = "ti,lp8733";
+		reg = <0x60>;
+
+		buck0-in-supply =<&vsys_3v3>;
+		buck1-in-supply =<&vsys_3v3>;
+		ldo0-in-supply =<&evm_5v0>;
+		ldo1-in-supply =<&evm_5v0>;
+
+		lp8733_regulators: regulators {
+			lp8733_buck0_reg: buck0 {
+				/* FB_B0 -> LP8733-BUCK1 - VPO_S1_AVS - VDD_CORE_AVS (core, mpu, gpu) */
+				regulator-name = "lp8733-buck0";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <1250000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			lp8733_buck1_reg: buck1 {
+				/* FB_B1 -> LP8733-BUCK2 - VPO_S2_AVS - VDD_DSP_AVS (DSP/eve/iva) */
+				regulator-name = "lp8733-buck1";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <1250000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			lp8733_ldo0_reg: ldo0 {
+				/* LDO0 -> LP8733-LDO1 - VPO_L1_3V3 - VDDSHV8 (optional) */
+				regulator-name = "lp8733-ldo0";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			lp8733_ldo1_reg: ldo1 {
+				/* LDO1 -> LP8733-LDO2 - VPO_L2_3V3 - VDDA_USB3V3 */
+				regulator-name = "lp8733-ldo1";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+
+	lp8732: lp8732@61 {
+		compatible = "ti,lp8732";
+		reg = <0x61>;
+
+		buck0-in-supply =<&vsys_3v3>;
+		buck1-in-supply =<&vsys_3v3>;
+		ldo0-in-supply =<&vsys_3v3>;
+		ldo1-in-supply =<&vsys_3v3>;
+
+		lp8732_regulators: regulators {
+			lp8732_buck0_reg: buck0 {
+				/* FB_B0 -> LP8732-BUCK1 - VPO_S3_1V8 - VDDS_1V8 */
+				regulator-name = "lp8732-buck0";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			lp8732_buck1_reg: buck1 {
+				/* FB_B1 -> LP8732-BUCK2 - VPO_S4_DDR - VDD_DDR_1V35 */
+				regulator-name = "lp8732-buck1";
+				regulator-min-microvolt = <1350000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			lp8732_ldo0_reg: ldo0 {
+				/* LDO0 -> LP8732-LDO1 - VPO_L3_1V8 - VDA_1V8_PLL */
+				regulator-name = "lp8732-ldo0";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			lp8732_ldo1_reg: ldo1 {
+				/* LDO1 -> LP8732-LDO2 - VPO_L4_1V8 - VDA_1V8_PHY */
+				regulator-name = "lp8732-ldo1";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+
+&pcf_gpio_21 {
+	interrupt-parent = <&gpio7>;
+	interrupts = <31 IRQ_TYPE_EDGE_FALLING>;
+};
+
+&pcf_hdmi {
+	p0 {
+		/*
+		 * PM_OEn to High: Disable routing I2C3 to PM_I2C
+		 * With this PM_SEL(p3) should not matter
+		 */
+		gpio-hog;
+		gpios = <0 GPIO_ACTIVE_LOW>;
+		output-high;
+		line-name = "pm_oe_n";
+	};
+};
+
+&mmc1 {
+	vmmc_aux-supply = <&vpo_sd_1v8_3v3>;
+};
+
+&mac {
+	mode-gpios = <&pcf_gpio_21 4 GPIO_ACTIVE_LOW>,
+		     <&pcf_hdmi 9 GPIO_ACTIVE_LOW>,	/* P11 */
+		     <&pcf_hdmi 10 GPIO_ACTIVE_LOW>;	/* P12 */
+	dual_emac;
+};
+
+&cpsw_emac0 {
+	phy_id = <&davinci_mdio>, <2>;
+	phy-mode = "rgmii-id";
+	dual_emac_res_vlan = <1>;
+};
+
+&cpsw_emac1 {
+	phy_id = <&davinci_mdio>, <3>;
+	phy-mode = "rgmii-id";
+	dual_emac_res_vlan = <2>;
+};
+
+&davinci_mdio {
+	dp83867_0: ethernet-phy@2 {
+		reg = <2>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+		ti,tx-internal-delay = <DP83867_RGMIIDCTL_250_PS>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_8_B_NIB>;
+		ti,impedance-control = <0x1f>;
+	};
+
+	dp83867_1: ethernet-phy@3 {
+		reg = <3>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+		ti,tx-internal-delay = <DP83867_RGMIIDCTL_250_PS>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_8_B_NIB>;
+		ti,impedance-control = <0x1f>;
+	};
+};
+
+/* No Sata on this device */
+&sata_phy {
+	status = "disabled";
+};
+
+&sata {
+	status = "disabled";
+};
+
+/* No RTC on this device */
+&rtc {
+	status = "disabled";
+};
+
+&usb2_phy1 {
+	phy-supply = <&lp8733_ldo1_reg>;
+};
+
+&usb2_phy2 {
+	phy-supply = <&lp8733_ldo1_reg>;
+};
+
+&dss {
+	/* Supplied by VDA_1V8_PLL */
+	vdda_video-supply = <&lp8732_ldo0_reg>;
+};
+
+&hdmi {
+	/* Supplied by VDA_1V8_PHY */
+	vdda_video-supply = <&lp8732_ldo1_reg>;
+};
diff --git a/arch/arm/boot/dts/dra72-evm-common.dtsi b/arch/arm/boot/dts/dra72-evm-common.dtsi
index c94d8d64710d..e50fbeea96e0 100644
--- a/arch/arm/boot/dts/dra72-evm-common.dtsi
+++ b/arch/arm/boot/dts/dra72-evm-common.dtsi
@@ -18,11 +18,49 @@
 		display0 = &hdmi0;
 	};
 
+	evm_12v0: fixedregulator-evm12v0 {
+		/* main supply */
+		compatible = "regulator-fixed";
+		regulator-name = "evm_12v0";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	evm_5v0: fixedregulator-evm5v0 {
+		/* Output 1 of TPS43351QDAPRQ1 on dra72-evm */
+		/* Output 1 of LM5140QRWGTQ1 on dra71-evm */
+		compatible = "regulator-fixed";
+		regulator-name = "evm_5v0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&evm_12v0>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vsys_3v3: fixedregulator-vsys3v3 {
+		/* Output 2 of TPS43351QDAPRQ1 on dra72-evm */
+		/* Output 2 of LM5140QRWGTQ1 on dra71-evm */
+		compatible = "regulator-fixed";
+		regulator-name = "vsys_3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&evm_12v0>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
 	evm_3v3_sw: fixedregulator-evm_3v3 {
+		/* TPS22965DSG */
 		compatible = "regulator-fixed";
 		regulator-name = "evm_3v3";
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vsys_3v3>;
+		regulator-always-on;
+		regulator-boot-on;
 	};
 
 	aic_dvdd: fixedregulator-aic_dvdd {
@@ -39,6 +77,7 @@
 		regulator-name = "evm_3v3_sd";
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
+		vin-supply = <&evm_3v3_sw>;
 		enable-active-high;
 		gpio = <&pcf_gpio_21 5 GPIO_ACTIVE_HIGH>;
 	};
@@ -69,9 +108,6 @@
 	tpd12s015: encoder {
 		compatible = "ti,tpd12s015";
 
-		pinctrl-names = "default";
-		pinctrl-0 = <&tpd12s015_pins>;
-
 		gpios = <&pcf_hdmi 4 GPIO_ACTIVE_HIGH>,	/* P4, CT CP HPD */
 			<&pcf_hdmi 5 GPIO_ACTIVE_HIGH>,	/* P5, LS OE */
 			<&gpio7 12 GPIO_ACTIVE_HIGH>;	/* gpio7_12/sp1_cs2, HPD */
@@ -134,72 +170,6 @@
 };
 
 &dra7_pmx_core {
-	i2c1_pins: pinmux_i2c1_pins {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x3800, PIN_INPUT | MUX_MODE0) /* i2c1_sda.i2c1_sda */
-			DRA7XX_CORE_IOPAD(0x3804, PIN_INPUT | MUX_MODE0) /* i2c1_scl.i2c1_scl */
-		>;
-	};
-
-	i2c5_pins: pinmux_i2c5_pins {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x36b4, PIN_INPUT | MUX_MODE10) /* mcasp1_axr0.i2c5_sda */
-			DRA7XX_CORE_IOPAD(0x36b8, PIN_INPUT | MUX_MODE10) /* mcasp1_axr1.i2c5_scl */
-		>;
-	};
-
-	i2c5_pins: pinmux_i2c5_pins {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x36b4, PIN_INPUT | MUX_MODE10) /* mcasp1_axr0.i2c5_sda */
-			DRA7XX_CORE_IOPAD(0x36b8, PIN_INPUT | MUX_MODE10) /* mcasp1_axr1.i2c5_scl */
-		>;
-	};
-
-	nand_default: nand_default {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x3400, PIN_INPUT  | MUX_MODE0) /* gpmc_ad0 */
-			DRA7XX_CORE_IOPAD(0x3404, PIN_INPUT  | MUX_MODE0) /* gpmc_ad1 */
-			DRA7XX_CORE_IOPAD(0x3408, PIN_INPUT  | MUX_MODE0) /* gpmc_ad2 */
-			DRA7XX_CORE_IOPAD(0x340c, PIN_INPUT  | MUX_MODE0) /* gpmc_ad3 */
-			DRA7XX_CORE_IOPAD(0x3410, PIN_INPUT  | MUX_MODE0) /* gpmc_ad4 */
-			DRA7XX_CORE_IOPAD(0x3414, PIN_INPUT  | MUX_MODE0) /* gpmc_ad5 */
-			DRA7XX_CORE_IOPAD(0x3418, PIN_INPUT  | MUX_MODE0) /* gpmc_ad6 */
-			DRA7XX_CORE_IOPAD(0x341c, PIN_INPUT  | MUX_MODE0) /* gpmc_ad7 */
-			DRA7XX_CORE_IOPAD(0x3420, PIN_INPUT  | MUX_MODE0) /* gpmc_ad8 */
-			DRA7XX_CORE_IOPAD(0x3424, PIN_INPUT  | MUX_MODE0) /* gpmc_ad9 */
-			DRA7XX_CORE_IOPAD(0x3428, PIN_INPUT  | MUX_MODE0) /* gpmc_ad10 */
-			DRA7XX_CORE_IOPAD(0x342c, PIN_INPUT  | MUX_MODE0) /* gpmc_ad11 */
-			DRA7XX_CORE_IOPAD(0x3430, PIN_INPUT  | MUX_MODE0) /* gpmc_ad12 */
-			DRA7XX_CORE_IOPAD(0x3434, PIN_INPUT  | MUX_MODE0) /* gpmc_ad13 */
-			DRA7XX_CORE_IOPAD(0x3438, PIN_INPUT  | MUX_MODE0) /* gpmc_ad14 */
-			DRA7XX_CORE_IOPAD(0x343c, PIN_INPUT  | MUX_MODE0) /* gpmc_ad15 */
-			DRA7XX_CORE_IOPAD(0x34b4, PIN_OUTPUT | MUX_MODE0) /* gpmc_cs0 */
-			DRA7XX_CORE_IOPAD(0x34c4, PIN_OUTPUT | MUX_MODE0) /* gpmc_advn_ale */
-			DRA7XX_CORE_IOPAD(0x34cc, PIN_OUTPUT | MUX_MODE0) /* gpmc_wen */
-			DRA7XX_CORE_IOPAD(0x34c8, PIN_OUTPUT | MUX_MODE0) /* gpmc_oen_ren */
-			DRA7XX_CORE_IOPAD(0x34d0, PIN_OUTPUT | MUX_MODE0) /* gpmc_ben0 */
-			DRA7XX_CORE_IOPAD(0x34d8, PIN_INPUT  | MUX_MODE0) /* gpmc_wait0 */
-		>;
-	};
-
-	usb1_pins: pinmux_usb1_pins {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x3680, PIN_INPUT_SLEW | MUX_MODE0) /* usb1_drvvbus */
-		>;
-	};
-
-	usb2_pins: pinmux_usb2_pins {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x3684, PIN_INPUT_SLEW | MUX_MODE0) /* usb2_drvvbus */
-		>;
-	};
-
-	tps65917_pins_default: tps65917_pins_default {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x3824, PIN_INPUT_PULLUP | MUX_MODE1) /* wakeup3.sys_nirq1 */
-		>;
-	};
-
 	mmc1_pins_default: mmc1_pins_default {
 		pinctrl-single,pins = <
 			DRA7XX_CORE_IOPAD(0x376c, PIN_INPUT | MUX_MODE14)	/* mmc1sdcd.gpio219 */
@@ -240,161 +210,12 @@
 			DRA7XX_CORE_IOPAD(0x3818, MUX_MODE15 | PULL_UP)	/* wakeup0.off */
 		>;
 	};
-
-	hdmi_pins: pinmux_hdmi_pins {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x3808, PIN_INPUT | MUX_MODE1) /* i2c2_sda.hdmi1_ddc_scl */
-			DRA7XX_CORE_IOPAD(0x380c, PIN_INPUT | MUX_MODE1) /* i2c2_scl.hdmi1_ddc_sda */
-		>;
-	};
-
-	tpd12s015_pins: pinmux_tpd12s015_pins {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x37b8, PIN_INPUT_PULLDOWN | MUX_MODE14) /* gpio7_12 HPD */
-		>;
-	};
-
-	atl_pins: pinmux_atl_pins {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x3698, PIN_OUTPUT | MUX_MODE5)	/* xref_clk1.atl_clk1 */
-			DRA7XX_CORE_IOPAD(0x369c, PIN_OUTPUT | MUX_MODE5)	/* xref_clk2.atl_clk2 */
-		>;
-	};
-
-	mcasp3_pins: pinmux_mcasp3_pins {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x3724, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mcasp3_aclkx */
-			DRA7XX_CORE_IOPAD(0x3728, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mcasp3_fsx */
-			DRA7XX_CORE_IOPAD(0x372c, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* mcasp3_axr0 */
-			DRA7XX_CORE_IOPAD(0x3730, PIN_INPUT_PULLDOWN | MUX_MODE0)	/* mcasp3_axr1 */
-		>;
-	};
-
-	mcasp3_sleep_pins: pinmux_mcasp3_sleep_pins {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x3724, PIN_INPUT_PULLDOWN | MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x3728, PIN_INPUT_PULLDOWN | MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x372c, PIN_INPUT_PULLDOWN | MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x3730, PIN_INPUT_PULLDOWN | MUX_MODE15)
-		>;
-	};
 };
 
 &i2c1 {
 	status = "okay";
-	pinctrl-names = "default";
-	pinctrl-0 = <&i2c1_pins>;
 	clock-frequency = <400000>;
 
-	tps65917: tps65917@58 {
-		compatible = "ti,tps65917";
-		reg = <0x58>;
-
-		pinctrl-names = "default";
-		pinctrl-0 = <&tps65917_pins_default>;
-
-		interrupts = <GIC_SPI 2 IRQ_TYPE_NONE>;  /* IRQ_SYS_1N */
-		interrupt-controller;
-		#interrupt-cells = <2>;
-
-		ti,system-power-controller;
-
-		tps65917_pmic {
-			compatible = "ti,tps65917-pmic";
-
-			tps65917_regulators: regulators {
-				smps1_reg: smps1 {
-					/* VDD_MPU */
-					regulator-name = "smps1";
-					regulator-min-microvolt = <850000>;
-					regulator-max-microvolt = <1250000>;
-					regulator-always-on;
-					regulator-boot-on;
-				};
-
-				smps2_reg: smps2 {
-					/* VDD_CORE */
-					regulator-name = "smps2";
-					regulator-min-microvolt = <850000>;
-					regulator-max-microvolt = <1150000>;
-					regulator-boot-on;
-					regulator-always-on;
-				};
-
-				smps3_reg: smps3 {
-					/* VDD_GPU IVA DSPEVE */
-					regulator-name = "smps3";
-					regulator-min-microvolt = <850000>;
-					regulator-max-microvolt = <1250000>;
-					regulator-boot-on;
-					regulator-always-on;
-				};
-
-				smps4_reg: smps4 {
-					/* VDDS1V8 */
-					regulator-name = "smps4";
-					regulator-min-microvolt = <1800000>;
-					regulator-max-microvolt = <1800000>;
-					regulator-always-on;
-					regulator-boot-on;
-				};
-
-				smps5_reg: smps5 {
-					/* VDD_DDR */
-					regulator-name = "smps5";
-					regulator-min-microvolt = <1350000>;
-					regulator-max-microvolt = <1350000>;
-					regulator-boot-on;
-					regulator-always-on;
-				};
-
-				ldo1_reg: ldo1 {
-					/* LDO1_OUT --> SDIO  */
-					regulator-name = "ldo1";
-					regulator-min-microvolt = <1800000>;
-					regulator-max-microvolt = <3300000>;
-					regulator-always-on;
-					regulator-boot-on;
-					regulator-allow-bypass;
-				};
-
-				ldo3_reg: ldo3 {
-					/* VDDA_1V8_PHY */
-					regulator-name = "ldo3";
-					regulator-min-microvolt = <1800000>;
-					regulator-max-microvolt = <1800000>;
-					regulator-boot-on;
-					regulator-always-on;
-				};
-
-				ldo5_reg: ldo5 {
-					/* VDDA_1V8_PLL */
-					regulator-name = "ldo5";
-					regulator-min-microvolt = <1800000>;
-					regulator-max-microvolt = <1800000>;
-					regulator-always-on;
-					regulator-boot-on;
-				};
-
-				ldo4_reg: ldo4 {
-					/* VDDA_3V_USB: VDDA_USBHS33 */
-					regulator-name = "ldo4";
-					regulator-min-microvolt = <3300000>;
-					regulator-max-microvolt = <3300000>;
-					regulator-boot-on;
-				};
-			};
-		};
-
-		tps65917_power_button {
-			compatible = "ti,palmas-pwrbutton";
-			interrupt-parent = <&tps65917>;
-			interrupts = <1 IRQ_TYPE_NONE>;
-			wakeup-source;
-			ti,palmas-long-press-seconds = <6>;
-		};
-	};
-
 	pcf_gpio_21: gpio@21 {
 		compatible = "ti,pcf8575", "nxp,pcf8575";
 		reg = <0x21>;
@@ -423,8 +244,6 @@
 
 &i2c5 {
 	status = "okay";
-	pinctrl-names = "default";
-	pinctrl-0 = <&i2c5_pins>;
 	clock-frequency = <400000>;
 
 	pcf_hdmi: pcf8575@26 {
@@ -462,8 +281,6 @@
 
 &gpmc {
 	status = "okay";
-	pinctrl-names = "default";
-	pinctrl-0 = <&nand_default>;
 	ranges = <0 0 0x08000000 0x01000000>;	/* minimum GPMC partition = 16MB */
 	nand@0,0 {
 		/* To use NAND, DIP switch SW5 must be set like so:
@@ -548,14 +365,6 @@
 	};
 };
 
-&usb2_phy1 {
-	phy-supply = <&ldo4_reg>;
-};
-
-&usb2_phy2 {
-	phy-supply = <&ldo4_reg>;
-};
-
 &omap_dwc3_1 {
 	extcon = <&extcon_usb1>;
 };
@@ -566,14 +375,10 @@
 
 &usb1 {
 	dr_mode = "peripheral";
-	pinctrl-names = "default";
-	pinctrl-0 = <&usb1_pins>;
 };
 
 &usb2 {
 	dr_mode = "host";
-	pinctrl-names = "default";
-	pinctrl-0 = <&usb2_pins>;
 };
 
 &mmc1 {
@@ -581,7 +386,6 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc1_pins_default>;
 	vmmc-supply = <&evm_3v3_sd>;
-	vmmc_aux-supply = <&ldo1_reg>;
 	bus-width = <4>;
 	/*
 	 * SDCD signal is not being used here - using the fact that GPIO mode
@@ -603,71 +407,8 @@
 	max-frequency = <192000000>;
 };
 
-&dra7_pmx_core {
-	cpsw_default: cpsw_default {
-		pinctrl-single,pins = <
-			/* Slave 2 */
-			DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT | MUX_MODE3)	/* vin2a_d12.rgmii1_txc */
-			DRA7XX_CORE_IOPAD(0x359c, PIN_OUTPUT | MUX_MODE3)	/* vin2a_d13.rgmii1_tctl */
-			DRA7XX_CORE_IOPAD(0x35a0, PIN_OUTPUT | MUX_MODE3)	/* vin2a_d14.rgmii1_td3 */
-			DRA7XX_CORE_IOPAD(0x35a4, PIN_OUTPUT | MUX_MODE3)	/* vin2a_d15.rgmii1_td2 */
-			DRA7XX_CORE_IOPAD(0x35a8, PIN_OUTPUT | MUX_MODE3)	/* vin2a_d16.rgmii1_td1 */
-			DRA7XX_CORE_IOPAD(0x35ac, PIN_OUTPUT | MUX_MODE3)	/* vin2a_d17.rgmii1_td0 */
-			DRA7XX_CORE_IOPAD(0x35b0, PIN_INPUT | MUX_MODE3)	/* vin2a_d18.rgmii1_rclk */
-			DRA7XX_CORE_IOPAD(0x35b4, PIN_INPUT | MUX_MODE3)	/* vin2a_d19.rgmii1_rctl */
-			DRA7XX_CORE_IOPAD(0x35b8, PIN_INPUT | MUX_MODE3)	/* vin2a_d20.rgmii1_rd3 */
-			DRA7XX_CORE_IOPAD(0x35bc, PIN_INPUT | MUX_MODE3)	/* vin2a_d21.rgmii1_rd2 */
-			DRA7XX_CORE_IOPAD(0x35c0, PIN_INPUT | MUX_MODE3)	/* vin2a_d22.rgmii1_rd1 */
-			DRA7XX_CORE_IOPAD(0x35c4, PIN_INPUT | MUX_MODE3)	/* vin2a_d23.rgmii1_rd0 */
-		>;
-
-	};
-
-	cpsw_sleep: cpsw_sleep {
-		pinctrl-single,pins = <
-			/* Slave 2 */
-			DRA7XX_CORE_IOPAD(0x3598, MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x359c, MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x35a0, MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x35a4, MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x35a8, MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x35ac, MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x35b0, MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x35b4, MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x35b8, MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x35bc, MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x35c0, MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x35c4, MUX_MODE15)
-		>;
-	};
-
-	davinci_mdio_default: davinci_mdio_default {
-		pinctrl-single,pins = <
-			/* MDIO */
-			DRA7XX_CORE_IOPAD(0x363c, PIN_OUTPUT_PULLUP | MUX_MODE0)	/* mdio_d.mdio_d */
-			DRA7XX_CORE_IOPAD(0x3640, PIN_INPUT_PULLUP | MUX_MODE0)	/* mdio_clk.mdio_clk */
-		>;
-	};
-
-	davinci_mdio_sleep: davinci_mdio_sleep {
-		pinctrl-single,pins = <
-			DRA7XX_CORE_IOPAD(0x363c, MUX_MODE15)
-			DRA7XX_CORE_IOPAD(0x3640, MUX_MODE15)
-		>;
-	};
-};
-
 &mac {
 	status = "okay";
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&cpsw_default>;
-	pinctrl-1 = <&cpsw_sleep>;
-};
-
-&davinci_mdio {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&davinci_mdio_default>;
-	pinctrl-1 = <&davinci_mdio_sleep>;
 };
 
 &dcan1 {
@@ -741,16 +482,11 @@
 
 &dss {
 	status = "ok";
-
-	vdda_video-supply = <&ldo5_reg>;
 };
 
 &hdmi {
 	status = "ok";
 
-	pinctrl-names = "default";
-	pinctrl-0 = <&hdmi_pins>;
-
 	port {
 		hdmi_out: endpoint {
 			remote-endpoint = <&tpd12s015_in>;
@@ -759,9 +495,6 @@
 };
 
 &atl {
-	pinctrl-names = "default";
-	pinctrl-0 = <&atl_pins>;
-
 	assigned-clocks = <&abe_dpll_sys_clk_mux>,
 			  <&atl_gfclk_mux>,
 			  <&dpll_abe_ck>,
@@ -780,9 +513,6 @@
 
 &mcasp3 {
 	#sound-dai-cells = <0>;
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&mcasp3_pins>;
-	pinctrl-1 = <&mcasp3_sleep_pins>;
 
 	assigned-clocks = <&mcasp3_ahclkx_mux>;
 	assigned-clock-parents = <&atl_clkin2_ck>;
diff --git a/arch/arm/boot/dts/dra72-evm-revc.dts b/arch/arm/boot/dts/dra72-evm-revc.dts
index 3b23b32e1b30..c3d939c9666c 100644
--- a/arch/arm/boot/dts/dra72-evm-revc.dts
+++ b/arch/arm/boot/dts/dra72-evm-revc.dts
@@ -17,17 +17,22 @@
 	};
 };
 
-&tps65917_regulators {
-	ldo2_reg: ldo2 {
-		/* LDO2_OUT --> VDDA_1V8_PHY2 */
-		regulator-name = "ldo2";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-		regulator-always-on;
-		regulator-boot-on;
+&i2c1 {
+	tps65917: tps65917@58 {
+		reg = <0x58>;
+
+		interrupts = <GIC_SPI 2 IRQ_TYPE_NONE>;  /* IRQ_SYS_1N */
 	};
 };
 
+#include "dra72-evm-tps65917.dtsi"
+
+&ldo2_reg {
+	/* LDO2_OUT --> VDDA_1V8_PHY2 */
+	regulator-always-on;
+	regulator-boot-on;
+};
+
 &hdmi {
 	vdda-supply = <&ldo2_reg>;
 };
diff --git a/arch/arm/boot/dts/dra72-evm-tps65917.dtsi b/arch/arm/boot/dts/dra72-evm-tps65917.dtsi
new file mode 100644
index 000000000000..ee6dac44edf1
--- /dev/null
+++ b/arch/arm/boot/dts/dra72-evm-tps65917.dtsi
@@ -0,0 +1,134 @@
+/*
+ * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/*
+ * Integrated Power Management Chip
+ * http://www.ti.com/lit/ds/symlink/tps65917-q1.pdf
+ */
+
+&tps65917 {
+	compatible = "ti,tps65917";
+
+	interrupt-controller;
+	#interrupt-cells = <2>;
+
+	ti,system-power-controller;
+
+	tps65917_pmic {
+		compatible = "ti,tps65917-pmic";
+
+		smps1-in-supply = <&vsys_3v3>;
+		smps2-in-supply = <&vsys_3v3>;
+		smps3-in-supply = <&vsys_3v3>;
+		smps4-in-supply = <&vsys_3v3>;
+		smps5-in-supply = <&vsys_3v3>;
+		ldo1-in-supply = <&vsys_3v3>;
+		ldo2-in-supply = <&vsys_3v3>;
+		ldo3-in-supply = <&vsys_3v3>;
+		ldo4-in-supply = <&evm_5v0>;
+		ldo5-in-supply = <&vsys_3v3>;
+
+		tps65917_regulators: regulators {
+			smps1_reg: smps1 {
+				/* VDD_MPU */
+				regulator-name = "smps1";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <1250000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			smps2_reg: smps2 {
+				/* VDD_CORE */
+				regulator-name = "smps2";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <1150000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			smps3_reg: smps3 {
+				/* VDD_GPU IVA DSPEVE */
+				regulator-name = "smps3";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <1250000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			smps4_reg: smps4 {
+				/* VDDS1V8 */
+				regulator-name = "smps4";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			smps5_reg: smps5 {
+				/* VDD_DDR */
+				regulator-name = "smps5";
+				regulator-min-microvolt = <1350000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo1_reg: ldo1 {
+				/* LDO1_OUT --> SDIO  */
+				regulator-name = "ldo1";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-allow-bypass;
+			};
+
+			ldo2_reg: ldo2 {
+				regulator-name = "ldo2";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-allow-bypass;
+			};
+
+			ldo3_reg: ldo3 {
+				/* VDDA_1V8_PHY */
+				regulator-name = "ldo3";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo5_reg: ldo5 {
+				/* VDDA_1V8_PLL */
+				regulator-name = "ldo5";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			ldo4_reg: ldo4 {
+				/* VDDA_3V_USB: VDDA_USBHS33 */
+				regulator-name = "ldo4";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+			};
+		};
+	};
+
+	tps65917_power_button {
+		compatible = "ti,palmas-pwrbutton";
+		interrupt-parent = <&tps65917>;
+		interrupts = <1 IRQ_TYPE_NONE>;
+		wakeup-source;
+		ti,palmas-long-press-seconds = <6>;
+	};
+};
diff --git a/arch/arm/boot/dts/dra72-evm.dts b/arch/arm/boot/dts/dra72-evm.dts
index e3a9b6985693..cd9c4ff12654 100644
--- a/arch/arm/boot/dts/dra72-evm.dts
+++ b/arch/arm/boot/dts/dra72-evm.dts
@@ -15,16 +15,16 @@
 	};
 };
 
-&tps65917_regulators {
-	ldo2_reg: ldo2 {
-		/* LDO2_OUT --> TP1017 (UNUSED)  */
-		regulator-name = "ldo2";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <3300000>;
-		regulator-allow-bypass;
+&i2c1 {
+	tps65917: tps65917@58 {
+		reg = <0x58>;
+
+		interrupts = <GIC_SPI 2 IRQ_TYPE_NONE>;  /* IRQ_SYS_1N */
 	};
 };
 
+#include "dra72-evm-tps65917.dtsi"
+
 &hdmi {
 	vdda-supply = <&ldo3_reg>;
 };
diff --git a/arch/arm/boot/dts/emev2.dtsi b/arch/arm/boot/dts/emev2.dtsi
index cd119400f440..0124faf175c8 100644
--- a/arch/arm/boot/dts/emev2.dtsi
+++ b/arch/arm/boot/dts/emev2.dtsi
@@ -8,13 +8,14 @@
  * kind, whether express or implied.
  */
 
-#include "skeleton.dtsi"
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 
 / {
 	compatible = "renesas,emev2";
 	interrupt-parent = <&gic>;
+	#address-cells = <1>;
+	#size-cells = <1>;
 
 	aliases {
 		gpio0 = &gpio0;
diff --git a/arch/arm/boot/dts/exynos3250-artik5-eval.dts b/arch/arm/boot/dts/exynos3250-artik5-eval.dts
index be4d6aa379f3..4bd2ee87124e 100644
--- a/arch/arm/boot/dts/exynos3250-artik5-eval.dts
+++ b/arch/arm/boot/dts/exynos3250-artik5-eval.dts
@@ -28,7 +28,7 @@
 	vqmmc-supply = <&ldo3_reg>;
 	card-detect-delay = <200>;
 	clock-frequency = <100000000>;
-	clock-freq-min-max = <400000 100000000>;
+	max-frequency = <100000000>;
 	samsung,dw-mshc-ciu-div = <1>;
 	samsung,dw-mshc-sdr-timing = <0 1>;
 	samsung,dw-mshc-ddr-timing = <1 2>;
diff --git a/arch/arm/boot/dts/exynos3250-artik5.dtsi b/arch/arm/boot/dts/exynos3250-artik5.dtsi
index a70819b1b739..59c89d7662a8 100644
--- a/arch/arm/boot/dts/exynos3250-artik5.dtsi
+++ b/arch/arm/boot/dts/exynos3250-artik5.dtsi
@@ -310,7 +310,7 @@
 	card-detect-delay = <200>;
 	vmmc-supply = <&ldo12_reg>;
 	clock-frequency = <100000000>;
-	clock-freq-min-max = <400000 100000000>;
+	max-frequency = <100000000>;
 	samsung,dw-mshc-ciu-div = <1>;
 	samsung,dw-mshc-sdr-timing = <0 1>;
 	samsung,dw-mshc-ddr-timing = <1 2>;
diff --git a/arch/arm/boot/dts/exynos3250-monk.dts b/arch/arm/boot/dts/exynos3250-monk.dts
index 66f04f6ba6bb..cccfe4b791d1 100644
--- a/arch/arm/boot/dts/exynos3250-monk.dts
+++ b/arch/arm/boot/dts/exynos3250-monk.dts
@@ -435,7 +435,7 @@
 	card-detect-delay = <200>;
 	vmmc-supply = <&vemmc_reg>;
 	clock-frequency = <100000000>;
-	clock-freq-min-max = <400000 100000000>;
+	max-frequency = <100000000>;
 	samsung,dw-mshc-ciu-div = <1>;
 	samsung,dw-mshc-sdr-timing = <0 1>;
 	samsung,dw-mshc-ddr-timing = <1 2>;
diff --git a/arch/arm/boot/dts/exynos3250-pinctrl.dtsi b/arch/arm/boot/dts/exynos3250-pinctrl.dtsi
index ec331169c3d9..a149f148e659 100644
--- a/arch/arm/boot/dts/exynos3250-pinctrl.dtsi
+++ b/arch/arm/boot/dts/exynos3250-pinctrl.dtsi
@@ -362,8 +362,14 @@
 
 		interrupt-controller;
 		interrupt-parent = <&gic>;
-		interrupts = <0 32 0>, <0 33 0>, <0 34 0>, <0 35 0>,
-				<0 36 0>, <0 37 0>, <0 38 0>, <0 39 0>;
+		interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
 		#interrupt-cells = <2>;
 	};
 
@@ -373,8 +379,14 @@
 
 		interrupt-controller;
 		interrupt-parent = <&gic>;
-		interrupts = <0 40 0>, <0 41 0>, <0 42 0>, <0 43 0>,
-				<0 44 0>, <0 45 0>, <0 46 0>, <0 47 0>;
+		interrupts = <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
 		#interrupt-cells = <2>;
 	};
 
diff --git a/arch/arm/boot/dts/exynos3250-rinato.dts b/arch/arm/boot/dts/exynos3250-rinato.dts
index 3967ee5f7752..548413e23c47 100644
--- a/arch/arm/boot/dts/exynos3250-rinato.dts
+++ b/arch/arm/boot/dts/exynos3250-rinato.dts
@@ -649,7 +649,7 @@
 	card-detect-delay = <200>;
 	vmmc-supply = <&ldo12_reg>;
 	clock-frequency = <100000000>;
-	clock-freq-min-max = <400000 100000000>;
+	max-frequency = <100000000>;
 	samsung,dw-mshc-ciu-div = <1>;
 	samsung,dw-mshc-sdr-timing = <0 1>;
 	samsung,dw-mshc-ddr-timing = <1 2>;
diff --git a/arch/arm/boot/dts/exynos3250.dtsi b/arch/arm/boot/dts/exynos3250.dtsi
index e9d2556c0dfd..ba17ee1eb749 100644
--- a/arch/arm/boot/dts/exynos3250.dtsi
+++ b/arch/arm/boot/dts/exynos3250.dtsi
@@ -20,6 +20,8 @@
 #include "exynos4-cpu-thermal.dtsi"
 #include "exynos-syscon-restart.dtsi"
 #include <dt-bindings/clock/exynos3250.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/interrupt-controller/irq.h>
 
 / {
 	compatible = "samsung,exynos3250";
@@ -211,7 +213,8 @@
 		rtc: rtc@10070000 {
 			compatible = "samsung,s3c6410-rtc";
 			reg = <0x10070000 0x100>;
-			interrupts = <0 73 0>, <0 74 0>;
+			interrupts = <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-parent = <&pmu_system_controller>;
 			status = "disabled";
 		};
@@ -219,7 +222,7 @@
 		tmu: tmu@100C0000 {
 			compatible = "samsung,exynos3250-tmu";
 			reg = <0x100C0000 0x100>;
-			interrupts = <0 216 0>;
+			interrupts = <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_TMU_APBIF>;
 			clock-names = "tmu_apbif";
 			#include "exynos4412-tmu-sensor-conf.dtsi"
@@ -234,14 +237,21 @@
 			      <0x10482000 0x1000>,
 			      <0x10484000 0x2000>,
 			      <0x10486000 0x2000>;
-			interrupts = <1 9 0xf04>;
+			interrupts = <GIC_PPI 9
+					(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
 		};
 
 		mct@10050000 {
 			compatible = "samsung,exynos4210-mct";
 			reg = <0x10050000 0x800>;
-			interrupts = <0 218 0>, <0 219 0>, <0 220 0>, <0 221 0>,
-				     <0 223 0>, <0 226 0>, <0 227 0>, <0 228 0>;
+			interrupts = <GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 223 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 228 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_FIN_PLL>, <&cmu CLK_MCT>;
 			clock-names = "fin_pll", "mct";
 		};
@@ -249,24 +259,24 @@
 		pinctrl_1: pinctrl@11000000 {
 			compatible = "samsung,exynos3250-pinctrl";
 			reg = <0x11000000 0x1000>;
-			interrupts = <0 225 0>;
+			interrupts = <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>;
 
 			wakeup-interrupt-controller {
 				compatible = "samsung,exynos4210-wakeup-eint";
-				interrupts = <0 48 0>;
+				interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>;
 			};
 		};
 
 		pinctrl_0: pinctrl@11400000 {
 			compatible = "samsung,exynos3250-pinctrl";
 			reg = <0x11400000 0x1000>;
-			interrupts = <0 240 0>;
+			interrupts = <GIC_SPI 240 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		jpeg: codec@11830000 {
 			compatible = "samsung,exynos3250-jpeg";
 			reg = <0x11830000 0x1000>;
-			interrupts = <0 171 0>;
+			interrupts = <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_JPEG>, <&cmu CLK_SCLK_JPEG>;
 			clock-names = "jpeg", "sclk";
 			power-domains = <&pd_cam>;
@@ -280,7 +290,8 @@
 		sysmmu_jpeg: sysmmu@11A60000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x11a60000 0x1000>;
-			interrupts = <0 156 0>, <0 161 0>;
+			interrupts = <GIC_SPI 156 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "sysmmu", "master";
 			clocks = <&cmu CLK_SMMUJPEG>, <&cmu CLK_JPEG>;
 			power-domains = <&pd_cam>;
@@ -291,7 +302,9 @@
 			compatible = "samsung,exynos3250-fimd";
 			reg = <0x11c00000 0x30000>;
 			interrupt-names = "fifo", "vsync", "lcd_sys";
-			interrupts = <0 84 0>, <0 85 0>, <0 86 0>;
+			interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_SCLK_FIMD0>, <&cmu CLK_FIMD0>;
 			clock-names = "sclk_fimd", "fimd";
 			power-domains = <&pd_lcd0>;
@@ -303,7 +316,7 @@
 		dsi_0: dsi@11C80000 {
 			compatible = "samsung,exynos3250-mipi-dsi";
 			reg = <0x11C80000 0x10000>;
-			interrupts = <0 83 0>;
+			interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
 			samsung,phy-type = <0>;
 			power-domains = <&pd_lcd0>;
 			phys = <&mipi_phy 1>;
@@ -318,7 +331,8 @@
 		sysmmu_fimd0: sysmmu@11E20000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x11e20000 0x1000>;
-			interrupts = <0 80 0>, <0 81 0>;
+			interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "sysmmu", "master";
 			clocks = <&cmu CLK_SMMUFIMD0>, <&cmu CLK_FIMD0>;
 			power-domains = <&pd_lcd0>;
@@ -328,7 +342,7 @@
 		hsotg: hsotg@12480000 {
 			compatible = "snps,dwc2";
 			reg = <0x12480000 0x20000>;
-			interrupts = <0 141 0>;
+			interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_USBOTG>;
 			clock-names = "otg";
 			phys = <&exynos_usbphy 0>;
@@ -339,7 +353,7 @@
 		mshc_0: mshc@12510000 {
 			compatible = "samsung,exynos5420-dw-mshc";
 			reg = <0x12510000 0x1000>;
-			interrupts = <0 142 0>;
+			interrupts = <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_SDMMC0>, <&cmu CLK_SCLK_MMC0>;
 			clock-names = "biu", "ciu";
 			fifo-depth = <0x80>;
@@ -351,7 +365,7 @@
 		mshc_1: mshc@12520000 {
 			compatible = "samsung,exynos5420-dw-mshc";
 			reg = <0x12520000 0x1000>;
-			interrupts = <0 143 0>;
+			interrupts = <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_SDMMC1>, <&cmu CLK_SCLK_MMC1>;
 			clock-names = "biu", "ciu";
 			fifo-depth = <0x80>;
@@ -363,7 +377,7 @@
 		mshc_2: mshc@12530000 {
 			compatible = "samsung,exynos5250-dw-mshc";
 			reg = <0x12530000 0x1000>;
-			interrupts = <0 144 0>;
+			interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_SDMMC2>, <&cmu CLK_SCLK_MMC2>;
 			clock-names = "biu", "ciu";
 			fifo-depth = <0x80>;
@@ -391,7 +405,7 @@
 			pdma0: pdma@12680000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x12680000 0x1000>;
-				interrupts = <0 138 0>;
+				interrupts = <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&cmu CLK_PDMA0>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -402,7 +416,7 @@
 			pdma1: pdma@12690000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x12690000 0x1000>;
-				interrupts = <0 139 0>;
+				interrupts = <GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&cmu CLK_PDMA1>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -415,7 +429,7 @@
 			compatible = "samsung,exynos3250-adc",
 				     "samsung,exynos-adc-v2";
 			reg = <0x126C0000 0x100>;
-			interrupts = <0 137 0>;
+			interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "adc", "sclk";
 			clocks = <&cmu CLK_TSADC>, <&cmu CLK_SCLK_TSADC>;
 			#io-channel-cells = <1>;
@@ -427,7 +441,7 @@
 		mfc: codec@13400000 {
 			compatible = "samsung,mfc-v7";
 			reg = <0x13400000 0x10000>;
-			interrupts = <0 102 0>;
+			interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "mfc", "sclk_mfc";
 			clocks = <&cmu CLK_MFC>, <&cmu CLK_SCLK_MFC>;
 			power-domains = <&pd_mfc>;
@@ -437,7 +451,8 @@
 		sysmmu_mfc: sysmmu@13620000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x13620000 0x1000>;
-			interrupts = <0 96 0>, <0 98 0>;
+			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "sysmmu", "master";
 			clocks = <&cmu CLK_SMMUMFC_L>, <&cmu CLK_MFC>;
 			power-domains = <&pd_mfc>;
@@ -447,7 +462,7 @@
 		serial_0: serial@13800000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x13800000 0x100>;
-			interrupts = <0 109 0>;
+			interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_UART0>, <&cmu CLK_SCLK_UART0>;
 			clock-names = "uart", "clk_uart_baud0";
 			pinctrl-names = "default";
@@ -458,7 +473,7 @@
 		serial_1: serial@13810000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x13810000 0x100>;
-			interrupts = <0 110 0>;
+			interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_UART1>, <&cmu CLK_SCLK_UART1>;
 			clock-names = "uart", "clk_uart_baud0";
 			pinctrl-names = "default";
@@ -469,7 +484,7 @@
 		serial_2: serial@13820000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x13820000 0x100>;
-			interrupts = <0 111 0>;
+			interrupts = <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_UART2>, <&cmu CLK_SCLK_UART2>;
 			clock-names = "uart", "clk_uart_baud0";
 			pinctrl-names = "default";
@@ -482,7 +497,7 @@
 			#size-cells = <0>;
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x13860000 0x100>;
-			interrupts = <0 113 0>;
+			interrupts = <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_I2C0>;
 			clock-names = "i2c";
 			pinctrl-names = "default";
@@ -495,7 +510,7 @@
 			#size-cells = <0>;
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x13870000 0x100>;
-			interrupts = <0 114 0>;
+			interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_I2C1>;
 			clock-names = "i2c";
 			pinctrl-names = "default";
@@ -508,7 +523,7 @@
 			#size-cells = <0>;
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x13880000 0x100>;
-			interrupts = <0 115 0>;
+			interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_I2C2>;
 			clock-names = "i2c";
 			pinctrl-names = "default";
@@ -521,7 +536,7 @@
 			#size-cells = <0>;
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x13890000 0x100>;
-			interrupts = <0 116 0>;
+			interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_I2C3>;
 			clock-names = "i2c";
 			pinctrl-names = "default";
@@ -534,7 +549,7 @@
 			#size-cells = <0>;
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x138A0000 0x100>;
-			interrupts = <0 117 0>;
+			interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_I2C4>;
 			clock-names = "i2c";
 			pinctrl-names = "default";
@@ -547,7 +562,7 @@
 			#size-cells = <0>;
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x138B0000 0x100>;
-			interrupts = <0 118 0>;
+			interrupts = <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_I2C5>;
 			clock-names = "i2c";
 			pinctrl-names = "default";
@@ -560,7 +575,7 @@
 			#size-cells = <0>;
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x138C0000 0x100>;
-			interrupts = <0 119 0>;
+			interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_I2C6>;
 			clock-names = "i2c";
 			pinctrl-names = "default";
@@ -573,7 +588,7 @@
 			#size-cells = <0>;
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x138D0000 0x100>;
-			interrupts = <0 120 0>;
+			interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_I2C7>;
 			clock-names = "i2c";
 			pinctrl-names = "default";
@@ -584,7 +599,7 @@
 		spi_0: spi@13920000 {
 			compatible = "samsung,exynos4210-spi";
 			reg = <0x13920000 0x100>;
-			interrupts = <0 121 0>;
+			interrupts = <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&pdma0 7>, <&pdma0 6>;
 			dma-names = "tx", "rx";
 			#address-cells = <1>;
@@ -600,7 +615,7 @@
 		spi_1: spi@13930000 {
 			compatible = "samsung,exynos4210-spi";
 			reg = <0x13930000 0x100>;
-			interrupts = <0 122 0>;
+			interrupts = <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&pdma1 7>, <&pdma1 6>;
 			dma-names = "tx", "rx";
 			#address-cells = <1>;
@@ -616,7 +631,7 @@
 		i2s2: i2s@13970000 {
 			compatible = "samsung,s3c6410-i2s";
 			reg = <0x13970000 0x100>;
-			interrupts = <0 126 0>;
+			interrupts = <GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu CLK_I2S>, <&cmu CLK_SCLK_I2S>;
 			clock-names = "iis", "i2s_opclk0";
 			dmas = <&pdma0 14>, <&pdma0 13>;
@@ -629,15 +644,19 @@
 		pwm: pwm@139D0000 {
 			compatible = "samsung,exynos4210-pwm";
 			reg = <0x139D0000 0x1000>;
-			interrupts = <0 104 0>, <0 105 0>, <0 106 0>,
-				     <0 107 0>, <0 108 0>;
+			interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
 			#pwm-cells = <3>;
 			status = "disabled";
 		};
 
 		pmu {
 			compatible = "arm,cortex-a7-pmu";
-			interrupts = <0 18 0>, <0 19 0>;
+			interrupts = <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		ppmu_dmc0: ppmu_dmc0@106a0000 {
diff --git a/arch/arm/boot/dts/exynos4.dtsi b/arch/arm/boot/dts/exynos4.dtsi
index 5f034eb5a5e2..c64737baa45e 100644
--- a/arch/arm/boot/dts/exynos4.dtsi
+++ b/arch/arm/boot/dts/exynos4.dtsi
@@ -21,6 +21,8 @@
 
 #include <dt-bindings/clock/exynos4.h>
 #include <dt-bindings/clock/exynos-audss-clk.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/interrupt-controller/irq.h>
 #include "exynos-syscon-restart.dtsi"
 
 / {
@@ -78,6 +80,11 @@
 		reg = <0x10000000 0x100>;
 	};
 
+	scu: snoop-control-unit@10500000 {
+		compatible = "arm,cortex-a9-scu";
+		reg = <0x10500000 0x2000>;
+	};
+
 	memory-controller@12570000 {
 		compatible = "samsung,exynos4210-srom";
 		reg = <0x12570000 0x14>;
@@ -168,7 +175,7 @@
 	dsi_0: dsi@11C80000 {
 		compatible = "samsung,exynos4210-mipi-dsi";
 		reg = <0x11C80000 0x10000>;
-		interrupts = <0 79 0>;
+		interrupts = <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>;
 		power-domains = <&pd_lcd0>;
 		phys = <&mipi_phy 1>;
 		phy-names = "dsim";
@@ -191,7 +198,7 @@
 		fimc_0: fimc@11800000 {
 			compatible = "samsung,exynos4210-fimc";
 			reg = <0x11800000 0x1000>;
-			interrupts = <0 84 0>;
+			interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_FIMC0>, <&clock CLK_SCLK_FIMC0>;
 			clock-names = "fimc", "sclk_fimc";
 			power-domains = <&pd_cam>;
@@ -203,7 +210,7 @@
 		fimc_1: fimc@11810000 {
 			compatible = "samsung,exynos4210-fimc";
 			reg = <0x11810000 0x1000>;
-			interrupts = <0 85 0>;
+			interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_FIMC1>, <&clock CLK_SCLK_FIMC1>;
 			clock-names = "fimc", "sclk_fimc";
 			power-domains = <&pd_cam>;
@@ -215,7 +222,7 @@
 		fimc_2: fimc@11820000 {
 			compatible = "samsung,exynos4210-fimc";
 			reg = <0x11820000 0x1000>;
-			interrupts = <0 86 0>;
+			interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_FIMC2>, <&clock CLK_SCLK_FIMC2>;
 			clock-names = "fimc", "sclk_fimc";
 			power-domains = <&pd_cam>;
@@ -227,7 +234,7 @@
 		fimc_3: fimc@11830000 {
 			compatible = "samsung,exynos4210-fimc";
 			reg = <0x11830000 0x1000>;
-			interrupts = <0 87 0>;
+			interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_FIMC3>, <&clock CLK_SCLK_FIMC3>;
 			clock-names = "fimc", "sclk_fimc";
 			power-domains = <&pd_cam>;
@@ -239,7 +246,7 @@
 		csis_0: csis@11880000 {
 			compatible = "samsung,exynos4210-csis";
 			reg = <0x11880000 0x4000>;
-			interrupts = <0 78 0>;
+			interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_CSIS0>, <&clock CLK_SCLK_CSIS0>;
 			clock-names = "csis", "sclk_csis";
 			bus-width = <4>;
@@ -254,7 +261,7 @@
 		csis_1: csis@11890000 {
 			compatible = "samsung,exynos4210-csis";
 			reg = <0x11890000 0x4000>;
-			interrupts = <0 80 0>;
+			interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_CSIS1>, <&clock CLK_SCLK_CSIS1>;
 			clock-names = "csis", "sclk_csis";
 			bus-width = <2>;
@@ -270,7 +277,7 @@
 	watchdog: watchdog@10060000 {
 		compatible = "samsung,s3c2410-wdt";
 		reg = <0x10060000 0x100>;
-		interrupts = <0 43 0>;
+		interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_WDT>;
 		clock-names = "watchdog";
 		status = "disabled";
@@ -280,7 +287,8 @@
 		compatible = "samsung,s3c6410-rtc";
 		reg = <0x10070000 0x100>;
 		interrupt-parent = <&pmu_system_controller>;
-		interrupts = <0 44 0>, <0 45 0>;
+		interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_RTC>;
 		clock-names = "rtc";
 		status = "disabled";
@@ -289,7 +297,7 @@
 	keypad: keypad@100A0000 {
 		compatible = "samsung,s5pv210-keypad";
 		reg = <0x100A0000 0x100>;
-		interrupts = <0 109 0>;
+		interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_KEYIF>;
 		clock-names = "keypad";
 		status = "disabled";
@@ -298,7 +306,7 @@
 	sdhci_0: sdhci@12510000 {
 		compatible = "samsung,exynos4210-sdhci";
 		reg = <0x12510000 0x100>;
-		interrupts = <0 73 0>;
+		interrupts = <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_SDMMC0>, <&clock CLK_SCLK_MMC0>;
 		clock-names = "hsmmc", "mmc_busclk.2";
 		status = "disabled";
@@ -307,7 +315,7 @@
 	sdhci_1: sdhci@12520000 {
 		compatible = "samsung,exynos4210-sdhci";
 		reg = <0x12520000 0x100>;
-		interrupts = <0 74 0>;
+		interrupts = <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_SDMMC1>, <&clock CLK_SCLK_MMC1>;
 		clock-names = "hsmmc", "mmc_busclk.2";
 		status = "disabled";
@@ -316,7 +324,7 @@
 	sdhci_2: sdhci@12530000 {
 		compatible = "samsung,exynos4210-sdhci";
 		reg = <0x12530000 0x100>;
-		interrupts = <0 75 0>;
+		interrupts = <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_SDMMC2>, <&clock CLK_SCLK_MMC2>;
 		clock-names = "hsmmc", "mmc_busclk.2";
 		status = "disabled";
@@ -325,7 +333,7 @@
 	sdhci_3: sdhci@12540000 {
 		compatible = "samsung,exynos4210-sdhci";
 		reg = <0x12540000 0x100>;
-		interrupts = <0 76 0>;
+		interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_SDMMC3>, <&clock CLK_SCLK_MMC3>;
 		clock-names = "hsmmc", "mmc_busclk.2";
 		status = "disabled";
@@ -344,7 +352,7 @@
 	hsotg: hsotg@12480000 {
 		compatible = "samsung,s3c6400-hsotg";
 		reg = <0x12480000 0x20000>;
-		interrupts = <0 71 0>;
+		interrupts = <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_USB_DEVICE>;
 		clock-names = "otg";
 		phys = <&exynos_usbphy 0>;
@@ -355,7 +363,7 @@
 	ehci: ehci@12580000 {
 		compatible = "samsung,exynos4210-ehci";
 		reg = <0x12580000 0x100>;
-		interrupts = <0 70 0>;
+		interrupts = <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_USB_HOST>;
 		clock-names = "usbhost";
 		status = "disabled";
@@ -381,7 +389,7 @@
 	ohci: ohci@12590000 {
 		compatible = "samsung,exynos4210-ohci";
 		reg = <0x12590000 0x100>;
-		interrupts = <0 70 0>;
+		interrupts = <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_USB_HOST>;
 		clock-names = "usbhost";
 		status = "disabled";
@@ -423,7 +431,7 @@
 	mfc: codec@13400000 {
 		compatible = "samsung,mfc-v5";
 		reg = <0x13400000 0x10000>;
-		interrupts = <0 94 0>;
+		interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
 		power-domains = <&pd_mfc>;
 		clocks = <&clock CLK_MFC>, <&clock CLK_SCLK_MFC>;
 		clock-names = "mfc", "sclk_mfc";
@@ -434,7 +442,7 @@
 	serial_0: serial@13800000 {
 		compatible = "samsung,exynos4210-uart";
 		reg = <0x13800000 0x100>;
-		interrupts = <0 52 0>;
+		interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_UART0>, <&clock CLK_SCLK_UART0>;
 		clock-names = "uart", "clk_uart_baud0";
 		dmas = <&pdma0 15>, <&pdma0 16>;
@@ -445,7 +453,7 @@
 	serial_1: serial@13810000 {
 		compatible = "samsung,exynos4210-uart";
 		reg = <0x13810000 0x100>;
-		interrupts = <0 53 0>;
+		interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_UART1>, <&clock CLK_SCLK_UART1>;
 		clock-names = "uart", "clk_uart_baud0";
 		dmas = <&pdma1 15>, <&pdma1 16>;
@@ -456,7 +464,7 @@
 	serial_2: serial@13820000 {
 		compatible = "samsung,exynos4210-uart";
 		reg = <0x13820000 0x100>;
-		interrupts = <0 54 0>;
+		interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_UART2>, <&clock CLK_SCLK_UART2>;
 		clock-names = "uart", "clk_uart_baud0";
 		dmas = <&pdma0 17>, <&pdma0 18>;
@@ -467,7 +475,7 @@
 	serial_3: serial@13830000 {
 		compatible = "samsung,exynos4210-uart";
 		reg = <0x13830000 0x100>;
-		interrupts = <0 55 0>;
+		interrupts = <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_UART3>, <&clock CLK_SCLK_UART3>;
 		clock-names = "uart", "clk_uart_baud0";
 		dmas = <&pdma1 17>, <&pdma1 18>;
@@ -480,7 +488,7 @@
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-i2c";
 		reg = <0x13860000 0x100>;
-		interrupts = <0 58 0>;
+		interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_I2C0>;
 		clock-names = "i2c";
 		pinctrl-names = "default";
@@ -493,7 +501,7 @@
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-i2c";
 		reg = <0x13870000 0x100>;
-		interrupts = <0 59 0>;
+		interrupts = <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_I2C1>;
 		clock-names = "i2c";
 		pinctrl-names = "default";
@@ -506,7 +514,7 @@
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-i2c";
 		reg = <0x13880000 0x100>;
-		interrupts = <0 60 0>;
+		interrupts = <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_I2C2>;
 		clock-names = "i2c";
 		pinctrl-names = "default";
@@ -519,7 +527,7 @@
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-i2c";
 		reg = <0x13890000 0x100>;
-		interrupts = <0 61 0>;
+		interrupts = <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_I2C3>;
 		clock-names = "i2c";
 		pinctrl-names = "default";
@@ -532,7 +540,7 @@
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-i2c";
 		reg = <0x138A0000 0x100>;
-		interrupts = <0 62 0>;
+		interrupts = <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_I2C4>;
 		clock-names = "i2c";
 		pinctrl-names = "default";
@@ -545,7 +553,7 @@
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-i2c";
 		reg = <0x138B0000 0x100>;
-		interrupts = <0 63 0>;
+		interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_I2C5>;
 		clock-names = "i2c";
 		pinctrl-names = "default";
@@ -558,7 +566,7 @@
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-i2c";
 		reg = <0x138C0000 0x100>;
-		interrupts = <0 64 0>;
+		interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_I2C6>;
 		clock-names = "i2c";
 		pinctrl-names = "default";
@@ -571,7 +579,7 @@
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-i2c";
 		reg = <0x138D0000 0x100>;
-		interrupts = <0 65 0>;
+		interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_I2C7>;
 		clock-names = "i2c";
 		pinctrl-names = "default";
@@ -584,7 +592,7 @@
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-hdmiphy-i2c";
 		reg = <0x138E0000 0x100>;
-		interrupts = <0 93 0>;
+		interrupts = <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_I2C_HDMI>;
 		clock-names = "i2c";
 		status = "disabled";
@@ -598,7 +606,7 @@
 	spi_0: spi@13920000 {
 		compatible = "samsung,exynos4210-spi";
 		reg = <0x13920000 0x100>;
-		interrupts = <0 66 0>;
+		interrupts = <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>;
 		dmas = <&pdma0 7>, <&pdma0 6>;
 		dma-names = "tx", "rx";
 		#address-cells = <1>;
@@ -613,7 +621,7 @@
 	spi_1: spi@13930000 {
 		compatible = "samsung,exynos4210-spi";
 		reg = <0x13930000 0x100>;
-		interrupts = <0 67 0>;
+		interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
 		dmas = <&pdma1 7>, <&pdma1 6>;
 		dma-names = "tx", "rx";
 		#address-cells = <1>;
@@ -628,7 +636,7 @@
 	spi_2: spi@13940000 {
 		compatible = "samsung,exynos4210-spi";
 		reg = <0x13940000 0x100>;
-		interrupts = <0 68 0>;
+		interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>;
 		dmas = <&pdma0 9>, <&pdma0 8>;
 		dma-names = "tx", "rx";
 		#address-cells = <1>;
@@ -643,7 +651,11 @@
 	pwm: pwm@139D0000 {
 		compatible = "samsung,exynos4210-pwm";
 		reg = <0x139D0000 0x1000>;
-		interrupts = <0 37 0>, <0 38 0>, <0 39 0>, <0 40 0>, <0 41 0>;
+		interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_PWM>;
 		clock-names = "timers";
 		#pwm-cells = <3>;
@@ -660,7 +672,7 @@
 		pdma0: pdma@12680000 {
 			compatible = "arm,pl330", "arm,primecell";
 			reg = <0x12680000 0x1000>;
-			interrupts = <0 35 0>;
+			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_PDMA0>;
 			clock-names = "apb_pclk";
 			#dma-cells = <1>;
@@ -671,7 +683,7 @@
 		pdma1: pdma@12690000 {
 			compatible = "arm,pl330", "arm,primecell";
 			reg = <0x12690000 0x1000>;
-			interrupts = <0 36 0>;
+			interrupts = <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_PDMA1>;
 			clock-names = "apb_pclk";
 			#dma-cells = <1>;
@@ -682,7 +694,7 @@
 		mdma1: mdma@12850000 {
 			compatible = "arm,pl330", "arm,primecell";
 			reg = <0x12850000 0x1000>;
-			interrupts = <0 34 0>;
+			interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_MDMA>;
 			clock-names = "apb_pclk";
 			#dma-cells = <1>;
@@ -712,7 +724,7 @@
 	jpeg_codec: jpeg-codec@11840000 {
 		compatible = "samsung,exynos4210-jpeg";
 		reg = <0x11840000 0x1000>;
-		interrupts = <0 88 0>;
+		interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_JPEG>;
 		clock-names = "jpeg";
 		power-domains = <&pd_cam>;
@@ -722,7 +734,7 @@
 	rotator: rotator@12810000 {
 		compatible = "samsung,exynos4210-rotator";
 		reg = <0x12810000 0x64>;
-		interrupts = <0 83 0>;
+		interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_ROTATOR>;
 		clock-names = "rotator";
 		iommus = <&sysmmu_rotator>;
@@ -731,7 +743,7 @@
 	hdmi: hdmi@12D00000 {
 		compatible = "samsung,exynos4210-hdmi";
 		reg = <0x12D00000 0x70000>;
-		interrupts = <0 92 0>;
+		interrupts = <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>;
 		clock-names = "hdmi", "sclk_hdmi", "sclk_pixel", "sclk_hdmiphy",
 			"mout_hdmi";
 		clocks = <&clock CLK_HDMI>, <&clock CLK_SCLK_HDMI>,
@@ -746,7 +758,7 @@
 	hdmicec: cec@100B0000 {
 		compatible = "samsung,s5p-cec";
 		reg = <0x100B0000 0x200>;
-		interrupts = <0 114 0>;
+		interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_HDMI_CEC>;
 		clock-names = "hdmicec";
 		samsung,syscon-phandle = <&pmu_system_controller>;
@@ -757,7 +769,7 @@
 
 	mixer: mixer@12C10000 {
 		compatible = "samsung,exynos4210-mixer";
-		interrupts = <0 91 0>;
+		interrupts = <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>;
 		reg = <0x12C10000 0x2100>, <0x12c00000 0x300>;
 		power-domains = <&pd_tv>;
 		iommus = <&sysmmu_tv>;
@@ -984,7 +996,7 @@
 	sss: sss@10830000 {
 		compatible = "samsung,exynos4210-secss";
 		reg = <0x10830000 0x300>;
-		interrupts = <0 112 0>;
+		interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_SSS>;
 		clock-names = "secss";
 	};
diff --git a/arch/arm/boot/dts/exynos4210-pinctrl.dtsi b/arch/arm/boot/dts/exynos4210-pinctrl.dtsi
index d9b6d25e4abe..f280954b260a 100644
--- a/arch/arm/boot/dts/exynos4210-pinctrl.dtsi
+++ b/arch/arm/boot/dts/exynos4210-pinctrl.dtsi
@@ -537,8 +537,14 @@
 
 			interrupt-controller;
 			interrupt-parent = <&gic>;
-			interrupts = <0 16 0>, <0 17 0>, <0 18 0>, <0 19 0>,
-				     <0 20 0>, <0 21 0>, <0 22 0>, <0 23 0>;
+			interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
 			#interrupt-cells = <2>;
 		};
 
@@ -548,8 +554,14 @@
 
 			interrupt-controller;
 			interrupt-parent = <&gic>;
-			interrupts = <0 24 0>, <0 25 0>, <0 26 0>, <0 27 0>,
-				     <0 28 0>, <0 29 0>, <0 30 0>, <0 31 0>;
+			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
 			#interrupt-cells = <2>;
 		};
 
diff --git a/arch/arm/boot/dts/exynos4210.dtsi b/arch/arm/boot/dts/exynos4210.dtsi
index 2d9b02967105..7f3a18c8f60f 100644
--- a/arch/arm/boot/dts/exynos4210.dtsi
+++ b/arch/arm/boot/dts/exynos4210.dtsi
@@ -109,12 +109,12 @@
 			#interrupt-cells = <1>;
 			#address-cells = <0>;
 			#size-cells = <0>;
-			interrupt-map = <0 &gic 0 57 0>,
-					<1 &gic 0 69 0>,
+			interrupt-map = <0 &gic 0 57 IRQ_TYPE_LEVEL_HIGH>,
+					<1 &gic 0 69 IRQ_TYPE_LEVEL_HIGH>,
 					<2 &combiner 12 6>,
 					<3 &combiner 12 7>,
-					<4 &gic 0 42 0>,
-					<5 &gic 0 48 0>;
+					<4 &gic 0 42 IRQ_TYPE_LEVEL_HIGH>,
+					<5 &gic 0 48 IRQ_TYPE_LEVEL_HIGH>;
 		};
 	};
 
@@ -127,18 +127,18 @@
 	pinctrl_0: pinctrl@11400000 {
 		compatible = "samsung,exynos4210-pinctrl";
 		reg = <0x11400000 0x1000>;
-		interrupts = <0 47 0>;
+		interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
 	pinctrl_1: pinctrl@11000000 {
 		compatible = "samsung,exynos4210-pinctrl";
 		reg = <0x11000000 0x1000>;
-		interrupts = <0 46 0>;
+		interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
 
 		wakup_eint: wakeup-interrupt-controller {
 			compatible = "samsung,exynos4210-wakeup-eint";
 			interrupt-parent = <&gic>;
-			interrupts = <0 32 0>;
+			interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
 		};
 	};
 
@@ -182,7 +182,7 @@
 	g2d: g2d@12800000 {
 		compatible = "samsung,s5pv210-g2d";
 		reg = <0x12800000 0x1000>;
-		interrupts = <0 89 0>;
+		interrupts = <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_SCLK_FIMG2D>, <&clock CLK_G2D>;
 		clock-names = "sclk_fimg2d", "fimg2d";
 		power-domains = <&pd_lcd0>;
@@ -424,10 +424,22 @@
 
 &combiner {
 	samsung,combiner-nr = <16>;
-	interrupts = <0 0 0>, <0 1 0>, <0 2 0>, <0 3 0>,
-		     <0 4 0>, <0 5 0>, <0 6 0>, <0 7 0>,
-		     <0 8 0>, <0 9 0>, <0 10 0>, <0 11 0>,
-		     <0 12 0>, <0 13 0>, <0 14 0>, <0 15 0>;
+	interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
 };
 
 &mdma1 {
diff --git a/arch/arm/boot/dts/exynos4412-itop-elite.dts b/arch/arm/boot/dts/exynos4412-itop-elite.dts
new file mode 100644
index 000000000000..76d87f397178
--- /dev/null
+++ b/arch/arm/boot/dts/exynos4412-itop-elite.dts
@@ -0,0 +1,240 @@
+/*
+ * TOPEET's Exynos4412 based itop board device tree source
+ *
+ * Copyright (c) 2016 SUMOMO Computer Association
+ *			https://www.sumomo.mobi
+ *			Randy Li <ayaka@soulik.info>
+ *
+ * Device tree source file for TOPEET iTop Exynos 4412 core board
+ * which is based on Samsung's Exynos4412 SoC.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/dts-v1/;
+#include <dt-bindings/pwm/pwm.h>
+#include <dt-bindings/sound/samsung-i2s.h>
+#include "exynos4412-itop-scp-core.dtsi"
+
+/ {
+	model = "TOPEET iTop 4412 Elite board based on Exynos4412";
+	compatible = "topeet,itop4412-elite", "samsung,exynos4412", "samsung,exynos4";
+
+	chosen {
+		bootargs = "root=/dev/mmcblk0p2 rw rootfstype=ext4 rootdelay=1 rootwait";
+		stdout-path = "serial2:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led2 {
+			label = "red:system";
+			gpios = <&gpx1 0 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "heartbeat";
+		};
+
+		led3 {
+			label = "red:user";
+			gpios = <&gpk1 1 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		home {
+			label = "GPIO Key Home";
+			linux,code = <KEY_HOME>;
+			gpios = <&gpx1 1 GPIO_ACTIVE_LOW>;
+		};
+
+		back {
+			label = "GPIO Key Back";
+			linux,code = <KEY_BACK>;
+			gpios = <&gpx1 2 GPIO_ACTIVE_LOW>;
+		};
+
+		sleep {
+			label = "GPIO Key Sleep";
+			linux,code = <KEY_POWER>;
+			gpios = <&gpx3 3 GPIO_ACTIVE_LOW>;
+		};
+
+		vol-up {
+			label = "GPIO Key Vol+";
+			linux,code = <KEY_UP>;
+			gpios = <&gpx2 1 GPIO_ACTIVE_LOW>;
+		};
+
+		vol-down {
+			label = "GPIO Key Vol-";
+			linux,code = <KEY_DOWN>;
+			gpios = <&gpx2 0 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "wm-sound";
+
+		assigned-clocks = <&clock_audss EXYNOS_MOUT_AUDSS>,
+				<&clock_audss EXYNOS_MOUT_I2S>,
+				<&clock_audss EXYNOS_DOUT_SRP>,
+				<&clock_audss EXYNOS_DOUT_AUD_BUS>;
+		assigned-clock-parents = <&clock CLK_FOUT_EPLL>,
+				<&clock_audss EXYNOS_MOUT_AUDSS>;
+		assigned-clock-rates = <0>,
+				<0>,
+				<112896000>,
+				<11289600>;
+
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&link0_codec>;
+		simple-audio-card,frame-master = <&link0_codec>;
+
+		simple-audio-card,widgets =
+			"Microphone", "Mic Jack",
+			"Line", "Line In",
+			"Line", "Line Out",
+			"Speaker", "Speaker",
+			"Headphone", "Headphone Jack";
+		simple-audio-card,routing =
+			"Headphone Jack", "HP_L",
+			"Headphone Jack", "HP_R",
+			"Speaker", "SPK_LP",
+			"Speaker", "SPK_LN",
+			"Speaker", "SPK_RP",
+			"Speaker", "SPK_RN",
+			"LINPUT1", "Mic Jack",
+			"LINPUT3", "Mic Jack",
+			"RINPUT1", "Mic Jack",
+			"RINPUT2", "Mic Jack";
+
+		simple-audio-card,cpu {
+			sound-dai = <&i2s0 0>;
+		};
+
+		link0_codec: simple-audio-card,codec {
+			sound-dai = <&codec>;
+			clocks = <&i2s0 CLK_I2S_CDCLK>;
+			system-clock-frequency = <11289600>;
+		};
+	};
+
+	beep {
+		compatible = "pwm-beeper";
+		pwms = <&pwm 0 4000000 PWM_POLARITY_INVERTED>;
+	};
+
+	camera: camera {
+		pinctrl-0 = <&cam_port_a_clk_active>;
+		pinctrl-names = "default";
+		status = "okay";
+		assigned-clocks = <&clock CLK_MOUT_CAM0>;
+		assigned-clock-parents = <&clock CLK_XUSBXTI>;
+	};
+};
+
+&adc {
+	vdd-supply = <&ldo3_reg>;
+	status = "okay";
+};
+
+&ehci {
+	status = "okay";
+	/* In order to reset USB ethernet */
+	samsung,vbus-gpio = <&gpc0 1 GPIO_ACTIVE_HIGH>;
+
+	port@0 {
+		status = "okay";
+	};
+
+	port@2 {
+		status = "okay";
+	};
+};
+
+&exynos_usbphy {
+	status = "okay";
+};
+
+&fimc_0 {
+	status = "okay";
+	assigned-clocks = <&clock CLK_MOUT_FIMC0>,
+			<&clock CLK_SCLK_FIMC0>;
+	assigned-clock-parents = <&clock CLK_MOUT_MPLL_USER_T>;
+	assigned-clock-rates = <0>, <176000000>;
+};
+
+&hsotg {
+	dr_mode = "peripheral";
+	status = "okay";
+};
+
+&i2c_4 {
+	samsung,i2c-sda-delay = <100>;
+	samsung,i2c-slave-addr = <0x10>;
+	samsung,i2c-max-bus-freq = <100000>;
+	pinctrl-0 = <&i2c4_bus>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	codec: wm8960@1a {
+		compatible = "wlf,wm8960";
+		reg = <0x1a>;
+		clocks = <&pmu_system_controller 0>;
+		clock-names = "MCLK1";
+		wlf,shared-lrclk;
+		#sound-dai-cells = <0>;
+	};
+};
+
+&i2s0 {
+	pinctrl-0 = <&i2s0_bus>;
+	pinctrl-names = "default";
+	status = "okay";
+	clocks = <&clock_audss EXYNOS_I2S_BUS>,
+		 <&clock_audss EXYNOS_DOUT_AUD_BUS>,
+		 <&clock_audss EXYNOS_SCLK_I2S>;
+	clock-names = "iis", "i2s_opclk0", "i2s_opclk1";
+};
+
+&pinctrl_1 {
+	ether-reset {
+		samsung,pins = "gpc0-1";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_INPUT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
+	};
+};
+
+&pwm {
+	status = "okay";
+	pinctrl-0 = <&pwm0_out>;
+	pinctrl-names = "default";
+	samsung,pwm-outputs = <0>;
+};
+
+&sdhci_2 {
+	bus-width = <4>;
+	pinctrl-0 = <&sd2_clk &sd2_cmd &sd2_bus4>;
+	pinctrl-names = "default";
+	cd-gpio = <&gpx0 7 GPIO_ACTIVE_LOW>;
+	cap-sd-highspeed;
+	vmmc-supply = <&ldo23_reg>;
+	vqmmc-supply = <&ldo17_reg>;
+	status = "okay";
+};
+
+&serial_1 {
+	status = "okay";
+};
+
+&serial_2 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/exynos4412-itop-scp-core.dtsi b/arch/arm/boot/dts/exynos4412-itop-scp-core.dtsi
new file mode 100644
index 000000000000..a36cd36a26b8
--- /dev/null
+++ b/arch/arm/boot/dts/exynos4412-itop-scp-core.dtsi
@@ -0,0 +1,501 @@
+/*
+ * TOPEET's Exynos4412 based itop board device tree source
+ *
+ * Copyright (c) 2016 SUMOMO Computer Association
+ *			https://www.sumomo.mobi
+ *			Randy Li <ayaka@soulik.info>
+ *
+ * Device tree source file for TOPEET iTop Exynos 4412 SCP package core
+ * board which is based on Samsung's Exynos4412 SoC.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <dt-bindings/clock/samsung,s2mps11.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include "exynos4412.dtsi"
+#include "exynos4412-ppmu-common.dtsi"
+#include "exynos-mfc-reserved-memory.dtsi"
+
+/ {
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0x40000000 0x40000000>;
+	};
+
+	firmware@0203F000 {
+		compatible = "samsung,secure-firmware";
+		reg = <0x0203F000 0x1000>;
+	};
+
+	fixed-rate-clocks {
+		xxti {
+			compatible = "samsung,clock-xxti";
+			clock-frequency = <0>;
+		};
+
+		xusbxti {
+			compatible = "samsung,clock-xusbxti";
+			clock-frequency = <24000000>;
+		};
+	};
+
+	thermal-zones {
+		cpu_thermal: cpu-thermal {
+			cooling-maps {
+				map0 {
+				     /* Corresponds to 800MHz at freq_table */
+				     cooling-device = <&cpu0 7 7>;
+				};
+				map1 {
+				     /* Corresponds to 200MHz at freq_table */
+				     cooling-device = <&cpu0 13 13>;
+			       };
+		       };
+		};
+	};
+
+	usb-hub {
+		compatible = "smsc,usb3503a";
+		reset-gpios = <&gpm2 4 GPIO_ACTIVE_LOW>;
+		connect-gpios = <&gpm3 3 GPIO_ACTIVE_HIGH>;
+		intn-gpios = <&gpx2 3 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&hsic_reset>;
+	};
+};
+
+&bus_dmc {
+	devfreq-events = <&ppmu_dmc0_3>, <&ppmu_dmc1_3>;
+	vdd-supply = <&buck1_reg>;
+	status = "okay";
+};
+
+&bus_acp {
+	devfreq = <&bus_dmc>;
+	status = "okay";
+};
+
+&bus_c2c {
+	devfreq = <&bus_dmc>;
+	status = "okay";
+};
+
+&bus_leftbus {
+	devfreq-events = <&ppmu_leftbus_3>, <&ppmu_rightbus_3>;
+	vdd-supply = <&buck3_reg>;
+	status = "okay";
+};
+
+&bus_rightbus {
+	devfreq = <&bus_leftbus>;
+	status = "okay";
+};
+
+&bus_fsys {
+	devfreq = <&bus_leftbus>;
+	status = "okay";
+};
+
+&bus_peri {
+	devfreq = <&bus_leftbus>;
+	status = "okay";
+};
+
+&bus_mfc {
+	devfreq = <&bus_leftbus>;
+	status = "okay";
+};
+
+&cpu0 {
+	cpu0-supply = <&buck2_reg>;
+};
+
+&hsotg {
+	vusb_d-supply = <&ldo15_reg>;
+	vusb_a-supply = <&ldo12_reg>;
+};
+
+&i2c_1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	samsung,i2c-sda-delay = <100>;
+	samsung,i2c-max-bus-freq = <400000>;
+	pinctrl-0 = <&i2c1_bus>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	s5m8767: s5m8767-pmic@66 {
+		compatible = "samsung,s5m8767-pmic";
+		reg = <0x66>;
+
+		s5m8767,pmic-buck-default-dvs-idx = <3>;
+
+		s5m8767,pmic-buck-dvs-gpios = <&gpb 5 GPIO_ACTIVE_HIGH>,
+						 <&gpb 6 GPIO_ACTIVE_HIGH>,
+						 <&gpb 7 GPIO_ACTIVE_HIGH>;
+
+		s5m8767,pmic-buck-ds-gpios = <&gpm3 5 GPIO_ACTIVE_HIGH>,
+						<&gpm3 6 GPIO_ACTIVE_HIGH>,
+						<&gpm3 7 GPIO_ACTIVE_HIGH>;
+
+		/* VDD_ARM */
+		s5m8767,pmic-buck2-dvs-voltage = <1356250>, <1300000>,
+						 <1243750>, <1118750>,
+						 <1068750>, <1012500>,
+						 <956250>, <900000>;
+		/* VDD_INT */
+		s5m8767,pmic-buck3-dvs-voltage = <1000000>, <1000000>,
+						 <925000>, <925000>,
+						 <887500>, <887500>,
+						 <850000>, <850000>;
+		/* VDD_G3D */
+		s5m8767,pmic-buck4-dvs-voltage = <1081250>, <1081250>,
+						 <1025000>, <950000>,
+						 <918750>, <900000>,
+						 <875000>, <831250>;
+
+		regulators {
+			ldo1_reg: LDO1 {
+				regulator-name = "VDD_ALIVE";
+				regulator-min-microvolt = <1100000>;
+				regulator-max-microvolt = <1100000>;
+				regulator-always-on;
+				regulator-boot-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			/* SCP uses 1.5v, POP uses 1.2v */
+			ldo2_reg: LDO2 {
+				regulator-name = "VDDQ_M12";
+				regulator-min-microvolt = <1500000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-always-on;
+				regulator-boot-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo3_reg: LDO3 {
+				regulator-name = "VDDIOAP_18";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo4_reg: LDO4 {
+				regulator-name = "VDDQ_PRE";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo5_reg: LDO5 {
+				regulator-name = "VDD_LDO5";
+				op_mode = <0>; /* Always off Mode */
+			};
+
+			ldo6_reg: LDO6 {
+				regulator-name = "VDD10_MPLL";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo7_reg: LDO7 {
+				regulator-name = "VDD10_XPLL";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo8_reg: LDO8 {
+				regulator-name = "VDD10_MIPI";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo9_reg: LDO9 {
+				regulator-name = "VDD33_LCD";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo10_reg: LDO10 {
+				regulator-name = "VDD18_MIPI";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo11_reg: LDO11 {
+				regulator-name = "VDD18_ABB1";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo12_reg: LDO12 {
+				regulator-name = "VDD33_UOTG";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo13_reg: LDO13 {
+				regulator-name = "VDDIOPERI_18";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo14_reg: LDO14 {
+				regulator-name = "VDD18_ABB02";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo15_reg: LDO15 {
+				regulator-name = "VDD10_USH";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo16_reg: LDO16 {
+				regulator-name = "VDD18_HSIC";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo17_reg: LDO17 {
+				regulator-name = "VDDIOAP_MMC012_28";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			/* Used by HSIC */
+			ldo18_reg: LDO18 {
+				regulator-name = "VDDIOPERI_28";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo19_reg: LDO19 {
+				regulator-name = "VDD_LDO19";
+				op_mode = <0>; /* Always off Mode */
+			};
+
+			ldo20_reg: LDO20 {
+				regulator-name = "VDD28_CAM";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <2800000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo21_reg: LDO21 {
+				regulator-name = "VDD28_AF";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <2800000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo22_reg: LDO22 {
+				regulator-name = "VDDA28_2M";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo23_reg: LDO23 {
+				regulator-name = "VDD28_TF";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo24_reg: LDO24 {
+				regulator-name = "VDD33_A31";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo25_reg: LDO25 {
+				regulator-name = "VDD18_CAM";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo26_reg: LDO26 {
+				regulator-name = "VDD18_A31";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo27_reg: LDO27 {
+				regulator-name = "GPS_1V8";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			ldo28_reg: LDO28 {
+				regulator-name = "DVDD12";
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			buck1_reg: BUCK1 {
+				regulator-name = "vdd_mif";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt	= <1100000>;
+				regulator-always-on;
+				regulator-boot-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			buck2_reg: BUCK2 {
+				regulator-name = "vdd_arm";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt	= <1456250>;
+				regulator-always-on;
+				regulator-boot-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			buck3_reg: BUCK3 {
+				regulator-name = "vdd_int";
+				regulator-min-microvolt = <875000>;
+				regulator-max-microvolt	= <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			buck4_reg: BUCK4 {
+				regulator-name = "vdd_g3d";
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt	= <1500000>;
+				regulator-always-on;
+				regulator-boot-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			buck5_reg: BUCK5 {
+				regulator-name = "vdd_m12";
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt	= <1500000>;
+				regulator-always-on;
+				regulator-boot-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			buck6_reg: BUCK6 {
+				regulator-name = "vdd12_5m";
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt	= <1500000>;
+				regulator-always-on;
+				regulator-boot-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			buck7_reg: BUCK7 {
+				regulator-name = "pvdd_buck7";
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt	= <2000000>;
+				regulator-boot-on;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			buck8_reg: BUCK8 {
+				regulator-name = "pvdd_buck8";
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt	= <1500000>;
+				regulator-boot-on;
+				regulator-always-on;
+				op_mode = <1>; /* Normal Mode */
+			};
+
+			buck9_reg: BUCK9 {
+				regulator-name = "vddf28_emmc";
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt	= <3000000>;
+				op_mode = <1>; /* Normal Mode */
+			};
+		};
+
+		s5m8767_osc: clocks {
+			#clock-cells = <1>;
+			clock-output-names = "s5m8767_ap",
+					"s5m8767_cp", "s5m8767_bt";
+		};
+
+	};
+};
+
+&mfc {
+	status = "okay";
+};
+
+&mshc_0 {
+	pinctrl-0 = <&sd4_clk &sd4_cmd &sd4_bus4 &sd4_bus8>;
+	pinctrl-names = "default";
+	status = "okay";
+	vmmc-supply = <&buck9_reg>;
+	num-slots = <1>;
+	broken-cd;
+	card-detect-delay = <200>;
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <2 3>;
+	samsung,dw-mshc-ddr-timing = <1 2>;
+	bus-width = <8>;
+	cap-mmc-highspeed;
+};
+
+&pinctrl_1 {
+	hsic_reset: hsic-reset {
+		samsung,pins = "gpm2-4";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_INPUT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
+	};
+};
+
+&rtc {
+	status = "okay";
+	clocks = <&clock CLK_RTC>, <&s5m8767_osc S2MPS11_CLK_AP>;
+	clock-names = "rtc", "rtc_src";
+};
+
+&tmu {
+	vtmu-supply = <&ldo16_reg>;
+	status = "okay";
+};
+
+&watchdog {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/exynos4412-odroidx.dts b/arch/arm/boot/dts/exynos4412-odroidx.dts
index 61906b35ea7a..153a75fe6e24 100644
--- a/arch/arm/boot/dts/exynos4412-odroidx.dts
+++ b/arch/arm/boot/dts/exynos4412-odroidx.dts
@@ -64,6 +64,11 @@
 	};
 };
 
+&adc {
+	vdd-supply = <&ldo10_reg>;
+	status = "okay";
+};
+
 /* VDDQ for MSHC (eMMC card) */
 &buck8_reg {
 	regulator-name = "BUCK8_VDDQ_MMC4_2.8V";
diff --git a/arch/arm/boot/dts/exynos4415-pinctrl.dtsi b/arch/arm/boot/dts/exynos4415-pinctrl.dtsi
deleted file mode 100644
index 76cfd872ead3..000000000000
--- a/arch/arm/boot/dts/exynos4415-pinctrl.dtsi
+++ /dev/null
@@ -1,575 +0,0 @@
-/*
- * Samsung's Exynos4415 SoCs pin-mux and pin-config device tree source
- *
- * Copyright (c) 2014 Samsung Electronics Co., Ltd.
- *
- * Samsung's Exynos4415 SoCs pin-mux and pin-config optiosn are listed as device
- * tree nodes are listed in this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
-
-#include <dt-bindings/pinctrl/samsung.h>
-
-&pinctrl_0 {
-	gpa0: gpa0 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpa1: gpa1 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpb: gpb {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpc0: gpc0 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpc1: gpc1 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpd0: gpd0 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpd1: gpd1 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpf0: gpf0 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpf1: gpf1 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpf2: gpf2 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	uart0_data: uart0-data {
-		samsung,pins = "gpa0-0", "gpa0-1";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	uart0_fctl: uart0-fctl {
-		samsung,pins = "gpa0-2", "gpa0-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	uart1_data: uart1-data {
-		samsung,pins = "gpa0-4", "gpa0-5";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	uart1_fctl: uart1-fctl {
-		samsung,pins = "gpa0-6", "gpa0-7";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	uart2_data: uart2-data {
-		samsung,pins = "gpa1-0", "gpa1-1";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	uart2_fctl: uart2-fctl {
-		samsung,pins = "gpa1-2", "gpa1-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	uart3_data: uart3-data {
-		samsung,pins = "gpa1-4", "gpa1-5";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	i2c2_bus: i2c2-bus {
-		samsung,pins = "gpa0-6", "gpa0-7";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	i2c3_bus: i2c3-bus {
-		samsung,pins = "gpa1-2", "gpa1-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	spi0_bus: spi0-bus {
-		samsung,pins = "gpb-0", "gpb-2", "gpb-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	i2c4_bus: i2c4-bus {
-		samsung,pins = "gpb-0", "gpb-1";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	spi1_bus: spi1-bus {
-		samsung,pins = "gpb-4", "gpb-6", "gpb-7";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	i2c5_bus: i2c5-bus {
-		samsung,pins = "gpb-2", "gpb-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	i2s1_bus: i2s1-bus {
-		samsung,pins = "gpc0-0", "gpc0-1", "gpc0-2", "gpc0-3",
-				"gpc0-4";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	i2s2_bus: i2s2-bus {
-		samsung,pins = "gpc1-0", "gpc1-1", "gpc1-2", "gpc1-3",
-				"gpc1-4";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	pcm2_bus: pcm2-bus {
-		samsung,pins = "gpc1-0", "gpc1-1", "gpc1-2", "gpc1-3",
-				"gpc1-4";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	i2c6_bus: i2c6-bus {
-		samsung,pins = "gpc1-3", "gpc1-4";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_4>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	spi2_bus: spi2-bus {
-		samsung,pins = "gpc1-1", "gpc1-3", "gpc1-4";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_5>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	pwm0_out: pwm0-out {
-		samsung,pins = "gpd0-0";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	pwm1_out: pwm1-out {
-		samsung,pins = "gpd0-1";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	pwm2_out: pwm2-out {
-		samsung,pins = "gpd0-2";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	pwm3_out: pwm3-out {
-		samsung,pins = "gpd0-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	i2c7_bus: i2c7-bus {
-		samsung,pins = "gpd0-2", "gpd0-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	i2c0_bus: i2c0-bus {
-		samsung,pins = "gpd1-0", "gpd1-1";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	i2c1_bus: i2c1-bus {
-		samsung,pins = "gpd1-2", "gpd1-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-};
-
-&pinctrl_1 {
-	gpk0: gpk0 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpk1: gpk1 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpk2: gpk2 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpk3: gpk3 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpl0: gpl0 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpm0: gpm0 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpm1: gpm1 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpm2: gpm2 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpm3: gpm3 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpm4: gpm4 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpx0: gpx0 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		interrupt-parent = <&gic>;
-		interrupts = <0 32 0>, <0 33 0>, <0 34 0>, <0 35 0>,
-				<0 36 0>, <0 37 0>, <0 38 0>, <0 39 0>;
-		#interrupt-cells = <2>;
-	};
-
-	gpx1: gpx1 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		interrupt-parent = <&gic>;
-		interrupts = <0 40 0>, <0 41 0>, <0 42 0>, <0 43 0>,
-				<0 44 0>, <0 45 0>, <0 46 0>, <0 47 0>;
-		#interrupt-cells = <2>;
-	};
-
-	gpx2: gpx2 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	gpx3: gpx3 {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	sd0_clk: sd0-clk {
-		samsung,pins = "gpk0-0";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd0_cmd: sd0-cmd {
-		samsung,pins = "gpk0-1";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd0_cd: sd0-cd {
-		samsung,pins = "gpk0-2";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd0_rdqs: sd0-rdqs {
-		samsung,pins = "gpk0-7";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd0_bus1: sd0-bus-width1 {
-		samsung,pins = "gpk0-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd0_bus4: sd0-bus-width4 {
-		samsung,pins = "gpk0-4", "gpk0-5", "gpk0-6";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd0_bus8: sd0-bus-width8 {
-		samsung,pins = "gpl0-0", "gpl0-1", "gpl0-2", "gpl0-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd1_clk: sd1-clk {
-		samsung,pins = "gpk1-0";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd1_cmd: sd1-cmd {
-		samsung,pins = "gpk1-1";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd1_cd: sd1-cd {
-		samsung,pins = "gpk1-2";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd1_bus1: sd1-bus-width1 {
-		samsung,pins = "gpk1-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd1_bus4: sd1-bus-width4 {
-		samsung,pins = "gpk1-4", "gpk1-5", "gpk1-6";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd2_clk: sd2-clk {
-		samsung,pins = "gpk2-0";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd2_cmd: sd2-cmd {
-		samsung,pins = "gpk2-1";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd2_cd: sd2-cd {
-		samsung,pins = "gpk2-2";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd2_bus1: sd2-bus-width1 {
-		samsung,pins = "gpk2-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	sd2_bus4: sd2-bus-width4 {
-		samsung,pins = "gpk2-4", "gpk2-5", "gpk2-6";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	cam_port_b_io: cam-port-b-io {
-		samsung,pins = "gpm0-0", "gpm0-1", "gpm0-2", "gpm0-3",
-				"gpm0-4", "gpm0-5", "gpm0-6", "gpm0-7",
-				"gpm1-0", "gpm1-1", "gpm2-0", "gpm2-1";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	cam_port_b_clk_active: cam-port-b-clk-active {
-		samsung,pins = "gpm2-2";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV4>;
-	};
-
-	cam_port_b_clk_idle: cam-port-b-clk-idle {
-		samsung,pins = "gpm2-2";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_INPUT>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	fimc_is_i2c0: fimc-is-i2c0 {
-		samsung,pins = "gpm4-0", "gpm4-1";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	fimc_is_i2c1: fimc-is-i2c1 {
-		samsung,pins = "gpm4-2", "gpm4-3";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-
-	fimc_is_uart: fimc-is-uart {
-		samsung,pins = "gpm3-5", "gpm3-7";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_3>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-};
-
-&pinctrl_2 {
-	gpz: gpz {
-		gpio-controller;
-		#gpio-cells = <2>;
-
-		interrupt-controller;
-		#interrupt-cells = <2>;
-	};
-
-	i2s0_bus: i2s0-bus {
-		samsung,pins = "gpz-0", "gpz-1", "gpz-2", "gpz-3",
-				"gpz-4", "gpz-5", "gpz-6";
-		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
-		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
-		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
-	};
-};
diff --git a/arch/arm/boot/dts/exynos4415.dtsi b/arch/arm/boot/dts/exynos4415.dtsi
deleted file mode 100644
index 3c40f8a956dd..000000000000
--- a/arch/arm/boot/dts/exynos4415.dtsi
+++ /dev/null
@@ -1,650 +0,0 @@
-/*
- * Samsung's Exynos4415 SoC device tree source
- *
- * Copyright (c) 2014 Samsung Electronics Co., Ltd.
- *
- * Samsung's Exynos4415 SoC device nodes are listed in this file. Exynos4415
- * based board files can include this file and provide values for board
- * specific bindings.
- *
- * Note: This file does not include device nodes for all the controllers in
- * Exynos4415 SoC. As device tree coverage for Exynos4415 increases, additional
- * nodes can be added to this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <dt-bindings/clock/exynos4415.h>
-#include <dt-bindings/clock/exynos-audss-clk.h>
-
-/ {
-	compatible = "samsung,exynos4415";
-	interrupt-parent = <&gic>;
-	#address-cells = <1>;
-	#size-cells = <1>;
-
-	aliases {
-		pinctrl0 = &pinctrl_0;
-		pinctrl1 = &pinctrl_1;
-		pinctrl2 = &pinctrl_2;
-		mshc0 = &mshc_0;
-		mshc1 = &mshc_1;
-		mshc2 = &mshc_2;
-		spi0 = &spi_0;
-		spi1 = &spi_1;
-		spi2 = &spi_2;
-		i2c0 = &i2c_0;
-		i2c1 = &i2c_1;
-		i2c2 = &i2c_2;
-		i2c3 = &i2c_3;
-		i2c4 = &i2c_4;
-		i2c5 = &i2c_5;
-		i2c6 = &i2c_6;
-		i2c7 = &i2c_7;
-	};
-
-	cpus {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		cpu0: cpu@a00 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a9";
-			reg = <0xa00>;
-			clock-frequency = <1600000000>;
-		};
-
-		cpu1: cpu@a01 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a9";
-			reg = <0xa01>;
-			clock-frequency = <1600000000>;
-		};
-
-		cpu2: cpu@a02 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a9";
-			reg = <0xa02>;
-			clock-frequency = <1600000000>;
-		};
-
-		cpu3: cpu@a03 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a9";
-			reg = <0xa03>;
-			clock-frequency = <1600000000>;
-		};
-	};
-
-	soc: soc {
-		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges;
-
-		sysram@02020000 {
-			compatible = "mmio-sram";
-			reg = <0x02020000 0x50000>;
-			#address-cells = <1>;
-			#size-cells = <1>;
-			ranges = <0 0x02020000 0x50000>;
-
-			smp-sysram@0 {
-				compatible = "samsung,exynos4210-sysram";
-				reg = <0x0 0x1000>;
-			};
-
-			smp-sysram@4f000 {
-				compatible = "samsung,exynos4210-sysram-ns";
-				reg = <0x4f000 0x1000>;
-			};
-		};
-
-		pinctrl_2: pinctrl@03860000 {
-			compatible = "samsung,exynos4415-pinctrl";
-			reg = <0x03860000 0x1000>;
-			interrupts = <0 242 0>;
-		};
-
-		chipid@10000000 {
-			compatible = "samsung,exynos4210-chipid";
-			reg = <0x10000000 0x100>;
-		};
-
-		sysreg_system_controller: syscon@10010000 {
-			compatible = "samsung,exynos4-sysreg", "syscon";
-			reg = <0x10010000 0x400>;
-		};
-
-		pmu_system_controller: system-controller@10020000 {
-			compatible = "samsung,exynos4415-pmu", "syscon";
-			reg = <0x10020000 0x4000>;
-		};
-
-		mipi_phy: video-phy@10020710 {
-			compatible = "samsung,s5pv210-mipi-video-phy";
-			#phy-cells = <1>;
-			syscon = <&pmu_system_controller>;
-		};
-
-		pd_cam: cam-power-domain@10024000 {
-			compatible = "samsung,exynos4210-pd";
-			reg = <0x10024000 0x20>;
-			#power-domain-cells = <0>;
-		};
-
-		pd_tv: tv-power-domain@10024020 {
-			compatible = "samsung,exynos4210-pd";
-			reg = <0x10024020 0x20>;
-			#power-domain-cells = <0>;
-		};
-
-		pd_mfc: mfc-power-domain@10024040 {
-			compatible = "samsung,exynos4210-pd";
-			reg = <0x10024040 0x20>;
-			#power-domain-cells = <0>;
-		};
-
-		pd_g3d: g3d-power-domain@10024060 {
-			compatible = "samsung,exynos4210-pd";
-			reg = <0x10024060 0x20>;
-			#power-domain-cells = <0>;
-		};
-
-		pd_lcd0: lcd0-power-domain@10024080 {
-			compatible = "samsung,exynos4210-pd";
-			reg = <0x10024080 0x20>;
-			#power-domain-cells = <0>;
-		};
-
-		pd_isp0: isp0-power-domain@100240A0 {
-			compatible = "samsung,exynos4210-pd";
-			reg = <0x100240A0 0x20>;
-			#power-domain-cells = <0>;
-		};
-
-		pd_isp1: isp1-power-domain@100240E0 {
-			compatible = "samsung,exynos4210-pd";
-			reg = <0x100240E0 0x20>;
-			#power-domain-cells = <0>;
-		};
-
-		cmu: clock-controller@10030000 {
-			compatible = "samsung,exynos4415-cmu";
-			reg = <0x10030000 0x18000>;
-			#clock-cells = <1>;
-		};
-
-		rtc: rtc@10070000 {
-			compatible = "samsung,s3c6410-rtc";
-			reg = <0x10070000 0x100>;
-			interrupts = <0 73 0>, <0 74 0>;
-			status = "disabled";
-		};
-
-		mct@10050000 {
-			compatible = "samsung,exynos4210-mct";
-			reg = <0x10050000 0x800>;
-			interrupts = <0 218 0>, <0 219 0>, <0 220 0>, <0 221 0>,
-				     <0 223 0>, <0 226 0>, <0 227 0>, <0 228 0>;
-			clocks = <&cmu CLK_FIN_PLL>, <&cmu CLK_MCT>;
-			clock-names = "fin_pll", "mct";
-		};
-
-		gic: interrupt-controller@10481000 {
-			compatible = "arm,cortex-a9-gic";
-			#interrupt-cells = <3>;
-			interrupt-controller;
-			reg = <0x10481000 0x1000>,
-			      <0x10482000 0x1000>,
-			      <0x10484000 0x2000>,
-			      <0x10486000 0x2000>;
-			interrupts = <1 9 0xf04>;
-		};
-
-		l2c: l2-cache-controller@10502000 {
-			compatible = "arm,pl310-cache";
-			reg = <0x10502000 0x1000>;
-			cache-unified;
-			cache-level = <2>;
-			arm,tag-latency = <2 2 1>;
-			arm,data-latency = <3 2 1>;
-			arm,double-linefill = <1>;
-			arm,double-linefill-incr = <0>;
-			arm,double-linefill-wrap = <1>;
-			arm,prefetch-drop = <1>;
-			arm,prefetch-offset = <7>;
-		};
-
-		cmu_dmc: clock-controller@105C0000 {
-			compatible = "samsung,exynos4415-cmu-dmc";
-			reg = <0x105C0000 0x3000>;
-			#clock-cells = <1>;
-		};
-
-		pinctrl_1: pinctrl@11000000 {
-			compatible = "samsung,exynos4415-pinctrl";
-			reg = <0x11000000 0x1000>;
-			interrupts = <0 225 0>;
-
-			wakeup-interrupt-controller {
-				compatible = "samsung,exynos4210-wakeup-eint";
-				interrupt-parent = <&gic>;
-				interrupts = <0 48 0>;
-			};
-		};
-
-		pinctrl_0: pinctrl@11400000 {
-			compatible = "samsung,exynos4415-pinctrl";
-			reg = <0x11400000 0x1000>;
-			interrupts = <0 240 0>;
-		};
-
-		fimd: fimd@11C00000 {
-			compatible = "samsung,exynos4415-fimd";
-			reg = <0x11C00000 0x30000>;
-			interrupt-names = "fifo", "vsync", "lcd_sys";
-			interrupts = <0 84 0>, <0 85 0>, <0 86 0>;
-			clocks = <&cmu CLK_SCLK_FIMD0>, <&cmu CLK_FIMD0>;
-			clock-names = "sclk_fimd", "fimd";
-			samsung,power-domain = <&pd_lcd0>;
-			iommus = <&sysmmu_fimd0>;
-			samsung,sysreg = <&sysreg_system_controller>;
-			status = "disabled";
-		};
-
-		dsi_0: dsi@11C80000 {
-			compatible = "samsung,exynos4415-mipi-dsi";
-			reg = <0x11C80000 0x10000>;
-			interrupts = <0 83 0>;
-			samsung,phy-type = <0>;
-			samsung,power-domain = <&pd_lcd0>;
-			phys = <&mipi_phy 1>;
-			phy-names = "dsim";
-			clocks = <&cmu CLK_DSIM0>, <&cmu CLK_SCLK_MIPI0>;
-			clock-names = "bus_clk", "pll_clk";
-			#address-cells = <1>;
-			#size-cells = <0>;
-			status = "disabled";
-		};
-
-		sysmmu_fimd0: sysmmu@11E20000 {
-			compatible = "samsung,exynos-sysmmu";
-			reg = <0x11e20000 0x1000>;
-			interrupts = <0 80 0>, <0 81 0>;
-			clock-names = "sysmmu", "master";
-			clocks = <&cmu CLK_SMMUFIMD0>, <&cmu CLK_FIMD0>;
-			power-domains = <&pd_lcd0>;
-			#iommu-cells = <0>;
-		};
-
-		hsotg: hsotg@12480000 {
-			compatible = "samsung,s3c6400-hsotg";
-			reg = <0x12480000 0x20000>;
-			interrupts = <0 141 0>;
-			clocks = <&cmu CLK_USBDEVICE>;
-			clock-names = "otg";
-			phys = <&exynos_usbphy 0>;
-			phy-names = "usb2-phy";
-			status = "disabled";
-		};
-
-		mshc_0: mshc@12510000 {
-			compatible = "samsung,exynos5250-dw-mshc";
-			reg = <0x12510000 0x1000>;
-			interrupts = <0 142 0>;
-			clocks = <&cmu CLK_SDMMC0>, <&cmu CLK_SCLK_MMC0>;
-			clock-names = "biu", "ciu";
-			fifo-depth = <0x80>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			status = "disabled";
-		};
-
-		mshc_1: mshc@12520000 {
-			compatible = "samsung,exynos5250-dw-mshc";
-			reg = <0x12520000 0x1000>;
-			interrupts = <0 143 0>;
-			clocks = <&cmu CLK_SDMMC1>, <&cmu CLK_SCLK_MMC1>;
-			clock-names = "biu", "ciu";
-			fifo-depth = <0x80>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			status = "disabled";
-		};
-
-		mshc_2: mshc@12530000 {
-			compatible = "samsung,exynos5250-dw-mshc";
-			reg = <0x12530000 0x1000>;
-			interrupts = <0 144 0>;
-			clocks = <&cmu CLK_SDMMC2>, <&cmu CLK_SCLK_MMC2>;
-			clock-names = "biu", "ciu";
-			fifo-depth = <0x80>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			status = "disabled";
-		};
-
-		ehci: ehci@12580000 {
-			compatible = "samsung,exynos4210-ehci";
-			reg = <0x12580000 0x100>;
-			interrupts = <0 140 0>;
-			clocks = <&cmu CLK_USBHOST>;
-			clock-names = "usbhost";
-			status = "disabled";
-			#address-cells = <1>;
-			#size-cells = <0>;
-			port@0 {
-				reg = <0>;
-				phys = <&exynos_usbphy 1>;
-				status = "disabled";
-			};
-			port@1 {
-				reg = <1>;
-				phys = <&exynos_usbphy 2>;
-				status = "disabled";
-			};
-			port@2 {
-				reg = <2>;
-				phys = <&exynos_usbphy 3>;
-				status = "disabled";
-			};
-		};
-
-		ohci: ohci@12590000 {
-			compatible = "samsung,exynos4210-ohci";
-			reg = <0x12590000 0x100>;
-			interrupts = <0 140 0>;
-			clocks = <&cmu CLK_USBHOST>;
-			clock-names = "usbhost";
-			status = "disabled";
-			#address-cells = <1>;
-			#size-cells = <0>;
-			port@0 {
-				reg = <0>;
-				phys = <&exynos_usbphy 1>;
-				status = "disabled";
-			};
-		};
-
-		exynos_usbphy: exynos-usbphy@125B0000 {
-			compatible = "samsung,exynos4x12-usb2-phy";
-			reg = <0x125B0000 0x100>;
-			samsung,pmureg-phandle = <&pmu_system_controller>;
-			samsung,sysreg-phandle = <&sysreg_system_controller>;
-			clocks = <&cmu CLK_USBDEVICE>, <&xusbxti>;
-			clock-names = "phy", "ref";
-			#phy-cells = <1>;
-			status = "disabled";
-		};
-
-		amba {
-			compatible = "simple-bus";
-			#address-cells = <1>;
-			#size-cells = <1>;
-			interrupt-parent = <&gic>;
-			ranges;
-
-			pdma0: pdma@12680000 {
-				compatible = "arm,pl330", "arm,primecell";
-				reg = <0x12680000 0x1000>;
-				interrupts = <0 138 0>;
-				clocks = <&cmu CLK_PDMA0>;
-				clock-names = "apb_pclk";
-				#dma-cells = <1>;
-				#dma-channels = <8>;
-				#dma-requests = <32>;
-			};
-
-			pdma1: pdma@12690000 {
-				compatible = "arm,pl330", "arm,primecell";
-				reg = <0x12690000 0x1000>;
-				interrupts = <0 139 0>;
-				clocks = <&cmu CLK_PDMA1>;
-				clock-names = "apb_pclk";
-				#dma-cells = <1>;
-				#dma-channels = <8>;
-				#dma-requests = <32>;
-			};
-		};
-
-		adc: adc@126C0000 {
-			compatible = "samsung,exynos3250-adc",
-				     "samsung,exynos-adc-v2";
-			reg = <0x126C0000 0x100>, <0x10020718 0x4>;
-			interrupts = <0 137 0>;
-			clock-names = "adc", "sclk";
-			clocks = <&cmu CLK_TSADC>, <&cmu CLK_SCLK_TSADC>;
-			#io-channel-cells = <1>;
-			io-channel-ranges;
-			status = "disabled";
-		};
-
-		serial_0: serial@13800000 {
-			compatible = "samsung,exynos4210-uart";
-			reg = <0x13800000 0x100>;
-			interrupts = <0 109 0>;
-			clocks = <&cmu CLK_UART0>, <&cmu CLK_SCLK_UART0>;
-			clock-names = "uart", "clk_uart_baud0";
-			status = "disabled";
-		};
-
-		serial_1: serial@13810000 {
-			compatible = "samsung,exynos4210-uart";
-			reg = <0x13810000 0x100>;
-			interrupts = <0 110 0>;
-			clocks = <&cmu CLK_UART1>, <&cmu CLK_SCLK_UART1>;
-			clock-names = "uart", "clk_uart_baud0";
-			status = "disabled";
-		};
-
-		serial_2: serial@13820000 {
-			compatible = "samsung,exynos4210-uart";
-			reg = <0x13820000 0x100>;
-			interrupts = <0 111 0>;
-			clocks = <&cmu CLK_UART2>, <&cmu CLK_SCLK_UART2>;
-			clock-names = "uart", "clk_uart_baud0";
-			status = "disabled";
-		};
-
-		serial_3: serial@13830000 {
-			compatible = "samsung,exynos4210-uart";
-			reg = <0x13830000 0x100>;
-			interrupts = <0 112 0>;
-			clocks = <&cmu CLK_UART3>, <&cmu CLK_SCLK_UART3>;
-			clock-names = "uart", "clk_uart_baud0";
-			status = "disabled";
-		};
-
-		i2c_0: i2c@13860000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "samsung,s3c2440-i2c";
-			reg = <0x13860000 0x100>;
-			interrupts = <0 113 0>;
-			clocks = <&cmu CLK_I2C0>;
-			clock-names = "i2c";
-			pinctrl-names = "default";
-			pinctrl-0 = <&i2c0_bus>;
-			status = "disabled";
-		};
-
-		i2c_1: i2c@13870000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "samsung,s3c2440-i2c";
-			reg = <0x13870000 0x100>;
-			interrupts = <0 114 0>;
-			clocks = <&cmu CLK_I2C1>;
-			clock-names = "i2c";
-			pinctrl-names = "default";
-			pinctrl-0 = <&i2c1_bus>;
-			status = "disabled";
-		};
-
-		i2c_2: i2c@13880000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "samsung,s3c2440-i2c";
-			reg = <0x13880000 0x100>;
-			interrupts = <0 115 0>;
-			clocks = <&cmu CLK_I2C2>;
-			clock-names = "i2c";
-			pinctrl-names = "default";
-			pinctrl-0 = <&i2c2_bus>;
-			status = "disabled";
-		};
-
-		i2c_3: i2c@13890000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "samsung,s3c2440-i2c";
-			reg = <0x13890000 0x100>;
-			interrupts = <0 116 0>;
-			clocks = <&cmu CLK_I2C3>;
-			clock-names = "i2c";
-			pinctrl-names = "default";
-			pinctrl-0 = <&i2c3_bus>;
-			status = "disabled";
-		};
-
-		i2c_4: i2c@138A0000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "samsung,s3c2440-i2c";
-			reg = <0x138A0000 0x100>;
-			interrupts = <0 117 0>;
-			clocks = <&cmu CLK_I2C4>;
-			clock-names = "i2c";
-			pinctrl-names = "default";
-			pinctrl-0 = <&i2c4_bus>;
-			status = "disabled";
-		};
-
-		i2c_5: i2c@138B0000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "samsung,s3c2440-i2c";
-			reg = <0x138B0000 0x100>;
-			interrupts = <0 118 0>;
-			clocks = <&cmu CLK_I2C5>;
-			clock-names = "i2c";
-			pinctrl-names = "default";
-			pinctrl-0 = <&i2c5_bus>;
-			status = "disabled";
-		};
-
-		i2c_6: i2c@138C0000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "samsung,s3c2440-i2c";
-			reg = <0x138C0000 0x100>;
-			interrupts = <0 119 0>;
-			clocks = <&cmu CLK_I2C6>;
-			clock-names = "i2c";
-			pinctrl-names = "default";
-			pinctrl-0 = <&i2c6_bus>;
-			status = "disabled";
-		};
-
-		i2c_7: i2c@138D0000 {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			compatible = "samsung,s3c2440-i2c";
-			reg = <0x138D0000 0x100>;
-			interrupts = <0 120 0>;
-			clocks = <&cmu CLK_I2C7>;
-			clock-names = "i2c";
-			pinctrl-names = "default";
-			pinctrl-0 = <&i2c7_bus>;
-			status = "disabled";
-		};
-
-		spi_0: spi@13920000 {
-			compatible = "samsung,exynos4210-spi";
-			reg = <0x13920000 0x100>;
-			interrupts = <0 121 0>;
-			dmas = <&pdma0 7>, <&pdma0 6>;
-			dma-names = "tx", "rx";
-			#address-cells = <1>;
-			#size-cells = <0>;
-			clocks = <&cmu CLK_SPI0>, <&cmu CLK_SCLK_SPI0>;
-			clock-names = "spi", "spi_busclk0";
-			samsung,spi-src-clk = <0>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&spi0_bus>;
-			status = "disabled";
-		};
-
-		spi_1: spi@13930000 {
-			compatible = "samsung,exynos4210-spi";
-			reg = <0x13930000 0x100>;
-			interrupts = <0 122 0>;
-			dmas = <&pdma1 7>, <&pdma1 6>;
-			dma-names = "tx", "rx";
-			#address-cells = <1>;
-			#size-cells = <0>;
-			clocks = <&cmu CLK_SPI1>, <&cmu CLK_SCLK_SPI1>;
-			clock-names = "spi", "spi_busclk0";
-			samsung,spi-src-clk = <0>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&spi1_bus>;
-			status = "disabled";
-		};
-
-		spi_2: spi@13940000 {
-			compatible = "samsung,exynos4210-spi";
-			reg = <0x13940000 0x100>;
-			interrupts = <0 123 0>;
-			dmas = <&pdma0 9>, <&pdma0 8>;
-			dma-names = "tx", "rx";
-			#address-cells = <1>;
-			#size-cells = <0>;
-			clocks = <&cmu CLK_SPI2>, <&cmu CLK_SCLK_SPI2>;
-			clock-names = "spi", "spi_busclk0";
-			samsung,spi-src-clk = <0>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&spi2_bus>;
-			status = "disabled";
-		};
-
-		clock_audss: clock-controller@03810000 {
-			compatible = "samsung,exynos4210-audss-clock";
-			reg = <0x03810000 0x0C>;
-			#clock-cells = <1>;
-		};
-
-		i2s0: i2s@3830000 {
-			compatible = "samsung,s5pv210-i2s";
-			reg = <0x03830000 0x100>;
-			interrupts = <0 124 0>;
-			clocks = <&clock_audss EXYNOS_I2S_BUS>,
-				<&clock_audss EXYNOS_SCLK_I2S>;
-			clock-names = "iis", "i2s_opclk0";
-			dmas = <&pdma1 10>, <&pdma1 9>, <&pdma1 8>;
-			dma-names = "tx", "rx", "tx-sec";
-			pinctrl-names = "default";
-			pinctrl-0 = <&i2s0_bus>;
-			samsung,idma-addr = <0x03000000>;
-			status = "disabled";
-		};
-
-		pwm: pwm@139D0000 {
-			compatible = "samsung,exynos4210-pwm";
-			reg = <0x139D0000 0x1000>;
-			interrupts = <0 104 0>, <0 105 0>, <0 106 0>,
-				     <0 107 0>, <0 108 0>;
-			#pwm-cells = <3>;
-			status = "disabled";
-		};
-
-		pmu {
-			compatible = "arm,cortex-a9-pmu";
-			interrupts = <0 18 0>, <0 19 0>, <0 20 0>, <0 21 0>;
-		};
-	};
-};
-
-#include "exynos4415-pinctrl.dtsi"
diff --git a/arch/arm/boot/dts/exynos4x12-pinctrl.dtsi b/arch/arm/boot/dts/exynos4x12-pinctrl.dtsi
index a56bf9b1a412..2f866f6e5838 100644
--- a/arch/arm/boot/dts/exynos4x12-pinctrl.dtsi
+++ b/arch/arm/boot/dts/exynos4x12-pinctrl.dtsi
@@ -572,8 +572,14 @@
 
 			interrupt-controller;
 			interrupt-parent = <&gic>;
-			interrupts = <0 16 0>, <0 17 0>, <0 18 0>, <0 19 0>,
-				     <0 20 0>, <0 21 0>, <0 22 0>, <0 23 0>;
+			interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
 			#interrupt-cells = <2>;
 		};
 
@@ -583,8 +589,14 @@
 
 			interrupt-controller;
 			interrupt-parent = <&gic>;
-			interrupts = <0 24 0>, <0 25 0>, <0 26 0>, <0 27 0>,
-				     <0 28 0>, <0 29 0>, <0 30 0>, <0 31 0>;
+			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
 			#interrupt-cells = <2>;
 		};
 
diff --git a/arch/arm/boot/dts/exynos4x12.dtsi b/arch/arm/boot/dts/exynos4x12.dtsi
index 3394bdcf10ae..85a7122658f1 100644
--- a/arch/arm/boot/dts/exynos4x12.dtsi
+++ b/arch/arm/boot/dts/exynos4x12.dtsi
@@ -88,11 +88,11 @@
 			#interrupt-cells = <1>;
 			#address-cells = <0>;
 			#size-cells = <0>;
-			interrupt-map = <0 &gic 0 57 0>,
+			interrupt-map = <0 &gic 0 57 IRQ_TYPE_LEVEL_HIGH>,
 					<1 &combiner 12 5>,
 					<2 &combiner 12 6>,
 					<3 &combiner 12 7>,
-					<4 &gic 1 12 0>;
+					<4 &gic 1 12 IRQ_TYPE_LEVEL_HIGH>;
 		};
 	};
 
@@ -112,7 +112,7 @@
 	g2d: g2d@10800000 {
 		compatible = "samsung,exynos4212-g2d";
 		reg = <0x10800000 0x1000>;
-		interrupts = <0 89 0>;
+		interrupts = <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_SCLK_FIMG2D>, <&clock CLK_G2D>;
 		clock-names = "sclk_fimg2d", "fimg2d";
 		iommus = <&sysmmu_g2d>;
@@ -127,7 +127,7 @@
 		fimc_lite_0: fimc-lite@12390000 {
 			compatible = "samsung,exynos4212-fimc-lite";
 			reg = <0x12390000 0x1000>;
-			interrupts = <0 105 0>;
+			interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&pd_isp>;
 			clocks = <&clock CLK_FIMC_LITE0>;
 			clock-names = "flite";
@@ -138,7 +138,7 @@
 		fimc_lite_1: fimc-lite@123A0000 {
 			compatible = "samsung,exynos4212-fimc-lite";
 			reg = <0x123A0000 0x1000>;
-			interrupts = <0 106 0>;
+			interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&pd_isp>;
 			clocks = <&clock CLK_FIMC_LITE1>;
 			clock-names = "flite";
@@ -147,9 +147,10 @@
 		};
 
 		fimc_is: fimc-is@12000000 {
-			compatible = "samsung,exynos4212-fimc-is", "simple-bus";
+			compatible = "samsung,exynos4212-fimc-is";
 			reg = <0x12000000 0x260000>;
-			interrupts = <0 90 0>, <0 95 0>;
+			interrupts = <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&pd_isp>;
 			clocks = <&clock CLK_FIMC_LITE0>,
 				 <&clock CLK_FIMC_LITE1>, <&clock CLK_PPMUISPX>,
@@ -200,7 +201,7 @@
 	mshc_0: mmc@12550000 {
 		compatible = "samsung,exynos4412-dw-mshc";
 		reg = <0x12550000 0x1000>;
-		interrupts = <0 77 0>;
+		interrupts = <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>;
 		#address-cells = <1>;
 		#size-cells = <0>;
 		fifo-depth = <0x80>;
@@ -461,11 +462,26 @@
 };
 
 &combiner {
-	interrupts = <0 0 0>, <0 1 0>, <0 2 0>, <0 3 0>,
-		     <0 4 0>, <0 5 0>, <0 6 0>, <0 7 0>,
-		     <0 8 0>, <0 9 0>, <0 10 0>, <0 11 0>,
-		     <0 12 0>, <0 13 0>, <0 14 0>, <0 15 0>,
-		     <0 107 0>, <0 108 0>, <0 48 0>, <0 42 0>;
+	interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 };
 
 &exynos_usbphy {
@@ -529,18 +545,18 @@
 &pinctrl_0 {
 	compatible = "samsung,exynos4x12-pinctrl";
 	reg = <0x11400000 0x1000>;
-	interrupts = <0 47 0>;
+	interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
 };
 
 &pinctrl_1 {
 	compatible = "samsung,exynos4x12-pinctrl";
 	reg = <0x11000000 0x1000>;
-	interrupts = <0 46 0>;
+	interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
 
 	wakup_eint: wakeup-interrupt-controller {
 		compatible = "samsung,exynos4210-wakeup-eint";
 		interrupt-parent = <&gic>;
-		interrupts = <0 32 0>;
+		interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
 	};
 };
 
@@ -554,7 +570,7 @@
 &pinctrl_3 {
 	compatible = "samsung,exynos4x12-pinctrl";
 	reg = <0x106E0000 0x1000>;
-	interrupts = <0 72 0>;
+	interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
 };
 
 &pmu_system_controller {
diff --git a/arch/arm/boot/dts/exynos5.dtsi b/arch/arm/boot/dts/exynos5.dtsi
index 8f06609879f5..7fd870ee5093 100644
--- a/arch/arm/boot/dts/exynos5.dtsi
+++ b/arch/arm/boot/dts/exynos5.dtsi
@@ -13,6 +13,8 @@
  * published by the Free Software Foundation.
  */
 
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/interrupt-controller/irq.h>
 #include "exynos-syscon-restart.dtsi"
 
 / {
@@ -53,14 +55,38 @@
 			interrupt-controller;
 			samsung,combiner-nr = <32>;
 			reg = <0x10440000 0x1000>;
-			interrupts =	<0 0 0>, <0 1 0>, <0 2 0>, <0 3 0>,
-					<0 4 0>, <0 5 0>, <0 6 0>, <0 7 0>,
-					<0 8 0>, <0 9 0>, <0 10 0>, <0 11 0>,
-					<0 12 0>, <0 13 0>, <0 14 0>, <0 15 0>,
-					<0 16 0>, <0 17 0>, <0 18 0>, <0 19 0>,
-					<0 20 0>, <0 21 0>, <0 22 0>, <0 23 0>,
-					<0 24 0>, <0 25 0>, <0 26 0>, <0 27 0>,
-					<0 28 0>, <0 29 0>, <0 30 0>, <0 31 0>;
+			interrupts = <0 0 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 1 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 2 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 3 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 4 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 5 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 6 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 7 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 8 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 9 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 10 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 11 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 12 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 13 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 14 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 15 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 16 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 17 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 18 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 19 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 20 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 21 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 22 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 23 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 24 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 25 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 26 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 27 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 28 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 29 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 30 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 31 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		gic: interrupt-controller@10481000 {
@@ -71,7 +97,8 @@
 				<0x10482000 0x1000>,
 				<0x10484000 0x2000>,
 				<0x10486000 0x2000>;
-			interrupts = <1 9 0xf04>;
+			interrupts = <GIC_PPI 9
+					(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
 		};
 
 		sysreg_system_controller: syscon@10050000 {
@@ -82,31 +109,31 @@
 		serial_0: serial@12C00000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C00000 0x100>;
-			interrupts = <0 51 0>;
+			interrupts = <0 51 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		serial_1: serial@12C10000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C10000 0x100>;
-			interrupts = <0 52 0>;
+			interrupts = <0 52 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		serial_2: serial@12C20000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C20000 0x100>;
-			interrupts = <0 53 0>;
+			interrupts = <0 53 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		serial_3: serial@12C30000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C30000 0x100>;
-			interrupts = <0 54 0>;
+			interrupts = <0 54 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		i2c_0: i2c@12C60000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12C60000 0x100>;
-			interrupts = <0 56 0>;
+			interrupts = <0 56 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			samsung,sysreg-phandle = <&sysreg_system_controller>;
@@ -116,7 +143,7 @@
 		i2c_1: i2c@12C70000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12C70000 0x100>;
-			interrupts = <0 57 0>;
+			interrupts = <0 57 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			samsung,sysreg-phandle = <&sysreg_system_controller>;
@@ -126,7 +153,7 @@
 		i2c_2: i2c@12C80000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12C80000 0x100>;
-			interrupts = <0 58 0>;
+			interrupts = <0 58 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			samsung,sysreg-phandle = <&sysreg_system_controller>;
@@ -136,7 +163,7 @@
 		i2c_3: i2c@12C90000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12C90000 0x100>;
-			interrupts = <0 59 0>;
+			interrupts = <0 59 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			samsung,sysreg-phandle = <&sysreg_system_controller>;
@@ -153,7 +180,8 @@
 		rtc: rtc@101E0000 {
 			compatible = "samsung,s3c6410-rtc";
 			reg = <0x101E0000 0x100>;
-			interrupts = <0 43 0>, <0 44 0>;
+			interrupts = <0 43 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 44 IRQ_TYPE_LEVEL_HIGH>;
 			status = "disabled";
 		};
 
diff --git a/arch/arm/boot/dts/exynos5250-snow-common.dtsi b/arch/arm/boot/dts/exynos5250-snow-common.dtsi
index d5d51916bb74..8f3a80430748 100644
--- a/arch/arm/boot/dts/exynos5250-snow-common.dtsi
+++ b/arch/arm/boot/dts/exynos5250-snow-common.dtsi
@@ -523,6 +523,7 @@
 	status = "okay";
 };
 
+/* eMMC flash */
 &mmc_0 {
 	status = "okay";
 	num-slots = <1>;
@@ -536,6 +537,7 @@
 	cap-mmc-highspeed;
 };
 
+/* uSD card */
 &mmc_2 {
 	status = "okay";
 	num-slots = <1>;
@@ -553,6 +555,8 @@
 /*
  * On Snow we've got SIP WiFi and so can keep drive strengths low to
  * reduce EMI.
+ *
+ * WiFi SDIO module
  */
 &mmc_3 {
 	status = "okay";
diff --git a/arch/arm/boot/dts/exynos5250.dtsi b/arch/arm/boot/dts/exynos5250.dtsi
index f7357d99b47c..b6d7444d8585 100644
--- a/arch/arm/boot/dts/exynos5250.dtsi
+++ b/arch/arm/boot/dts/exynos5250.dtsi
@@ -181,8 +181,8 @@
 						<0x1 0 &combiner 23 4>,
 						<0x2 0 &combiner 25 2>,
 						<0x3 0 &combiner 25 3>,
-						<0x4 0 &gic 0 120 0>,
-						<0x5 0 &gic 0 121 0>;
+						<0x4 0 &gic 0 120 IRQ_TYPE_LEVEL_HIGH>,
+						<0x5 0 &gic 0 121 IRQ_TYPE_LEVEL_HIGH>;
 			};
 		};
 
@@ -195,31 +195,31 @@
 		pinctrl_0: pinctrl@11400000 {
 			compatible = "samsung,exynos5250-pinctrl";
 			reg = <0x11400000 0x1000>;
-			interrupts = <0 46 0>;
+			interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
 
 			wakup_eint: wakeup-interrupt-controller {
 				compatible = "samsung,exynos4210-wakeup-eint";
 				interrupt-parent = <&gic>;
-				interrupts = <0 32 0>;
+				interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
 			};
 		};
 
 		pinctrl_1: pinctrl@13400000 {
 			compatible = "samsung,exynos5250-pinctrl";
 			reg = <0x13400000 0x1000>;
-			interrupts = <0 45 0>;
+			interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_2: pinctrl@10d10000 {
 			compatible = "samsung,exynos5250-pinctrl";
 			reg = <0x10d10000 0x1000>;
-			interrupts = <0 50 0>;
+			interrupts = <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_3: pinctrl@03860000 {
 			compatible = "samsung,exynos5250-pinctrl";
 			reg = <0x03860000 0x1000>;
-			interrupts = <0 47 0>;
+			interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pmu_system_controller: system-controller@10040000 {
@@ -236,7 +236,7 @@
 		watchdog@101D0000 {
 			compatible = "samsung,exynos5250-wdt";
 			reg = <0x101D0000 0x100>;
-			interrupts = <0 42 0>;
+			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_WDT>;
 			clock-names = "watchdog";
 			samsung,syscon-phandle = <&pmu_system_controller>;
@@ -245,7 +245,7 @@
 		g2d@10850000 {
 			compatible = "samsung,exynos5250-g2d";
 			reg = <0x10850000 0x1000>;
-			interrupts = <0 91 0>;
+			interrupts = <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_G2D>;
 			clock-names = "fimg2d";
 			iommus = <&sysmmu_g2d>;
@@ -254,7 +254,7 @@
 		mfc: codec@11000000 {
 			compatible = "samsung,mfc-v6";
 			reg = <0x11000000 0x10000>;
-			interrupts = <0 96 0>;
+			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&pd_mfc>;
 			clocks = <&clock CLK_MFC>;
 			clock-names = "mfc";
@@ -265,7 +265,7 @@
 		rotator: rotator@11C00000 {
 			compatible = "samsung,exynos5250-rotator";
 			reg = <0x11C00000 0x64>;
-			interrupts = <0 84 0>;
+			interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_ROTATOR>;
 			clock-names = "rotator";
 			iommus = <&sysmmu_rotator>;
@@ -274,7 +274,7 @@
 		tmu: tmu@10060000 {
 			compatible = "samsung,exynos5250-tmu";
 			reg = <0x10060000 0x100>;
-			interrupts = <0 65 0>;
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_TMU>;
 			clock-names = "tmu_apbif";
 			#include "exynos4412-tmu-sensor-conf.dtsi"
@@ -284,7 +284,7 @@
 			compatible = "snps,dwc-ahci";
 			samsung,sata-freq = <66>;
 			reg = <0x122F0000 0x1ff>;
-			interrupts = <0 115 0>;
+			interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_SATA>, <&clock CLK_SCLK_SATA>;
 			clock-names = "sata", "sclk_sata";
 			phys = <&sata_phy>;
@@ -306,7 +306,7 @@
 		i2c_4: i2c@12CA0000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12CA0000 0x100>;
-			interrupts = <0 60 0>;
+			interrupts = <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			clocks = <&clock CLK_I2C4>;
@@ -319,7 +319,7 @@
 		i2c_5: i2c@12CB0000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12CB0000 0x100>;
-			interrupts = <0 61 0>;
+			interrupts = <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			clocks = <&clock CLK_I2C5>;
@@ -332,7 +332,7 @@
 		i2c_6: i2c@12CC0000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12CC0000 0x100>;
-			interrupts = <0 62 0>;
+			interrupts = <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			clocks = <&clock CLK_I2C6>;
@@ -345,7 +345,7 @@
 		i2c_7: i2c@12CD0000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12CD0000 0x100>;
-			interrupts = <0 63 0>;
+			interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			clocks = <&clock CLK_I2C7>;
@@ -358,7 +358,7 @@
 		i2c_8: i2c@12CE0000 {
 			compatible = "samsung,s3c2440-hdmiphy-i2c";
 			reg = <0x12CE0000 0x1000>;
-			interrupts = <0 64 0>;
+			interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			clocks = <&clock CLK_I2C_HDMI>;
@@ -380,7 +380,7 @@
 			compatible = "samsung,exynos4210-spi";
 			status = "disabled";
 			reg = <0x12d20000 0x100>;
-			interrupts = <0 66 0>;
+			interrupts = <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&pdma0 5
 				&pdma0 4>;
 			dma-names = "tx", "rx";
@@ -396,7 +396,7 @@
 			compatible = "samsung,exynos4210-spi";
 			status = "disabled";
 			reg = <0x12d30000 0x100>;
-			interrupts = <0 67 0>;
+			interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&pdma1 5
 				&pdma1 4>;
 			dma-names = "tx", "rx";
@@ -412,7 +412,7 @@
 			compatible = "samsung,exynos4210-spi";
 			status = "disabled";
 			reg = <0x12d40000 0x100>;
-			interrupts = <0 68 0>;
+			interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&pdma0 7
 				&pdma0 6>;
 			dma-names = "tx", "rx";
@@ -426,7 +426,7 @@
 
 		mmc_0: mmc@12200000 {
 			compatible = "samsung,exynos5250-dw-mshc";
-			interrupts = <0 75 0>;
+			interrupts = <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x12200000 0x1000>;
@@ -438,7 +438,7 @@
 
 		mmc_1: mmc@12210000 {
 			compatible = "samsung,exynos5250-dw-mshc";
-			interrupts = <0 76 0>;
+			interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x12210000 0x1000>;
@@ -450,7 +450,7 @@
 
 		mmc_2: mmc@12220000 {
 			compatible = "samsung,exynos5250-dw-mshc";
-			interrupts = <0 77 0>;
+			interrupts = <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x12220000 0x1000>;
@@ -463,7 +463,7 @@
 		mmc_3: mmc@12230000 {
 			compatible = "samsung,exynos5250-dw-mshc";
 			reg = <0x12230000 0x1000>;
-			interrupts = <0 78 0>;
+			interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			clocks = <&clock CLK_SDMMC3>, <&clock CLK_SCLK_MMC3>;
@@ -526,7 +526,7 @@
 			usbdrd_dwc3: dwc3@12000000 {
 				compatible = "synopsys,dwc3";
 				reg = <0x12000000 0x10000>;
-				interrupts = <0 72 0>;
+				interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
 				phys = <&usbdrd_phy 0>, <&usbdrd_phy 1>;
 				phy-names = "usb2-phy", "usb3-phy";
 			};
@@ -544,7 +544,7 @@
 		ehci: usb@12110000 {
 			compatible = "samsung,exynos4210-ehci";
 			reg = <0x12110000 0x100>;
-			interrupts = <0 71 0>;
+			interrupts = <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
 
 			clocks = <&clock CLK_USB2>;
 			clock-names = "usbhost";
@@ -559,7 +559,7 @@
 		ohci: usb@12120000 {
 			compatible = "samsung,exynos4210-ohci";
 			reg = <0x12120000 0x100>;
-			interrupts = <0 71 0>;
+			interrupts = <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
 
 			clocks = <&clock CLK_USB2>;
 			clock-names = "usbhost";
@@ -591,7 +591,7 @@
 			pdma0: pdma@121A0000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x121A0000 0x1000>;
-				interrupts = <0 34 0>;
+				interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_PDMA0>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -602,7 +602,7 @@
 			pdma1: pdma@121B0000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x121B0000 0x1000>;
-				interrupts = <0 35 0>;
+				interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_PDMA1>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -613,7 +613,7 @@
 			mdma0: mdma@10800000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x10800000 0x1000>;
-				interrupts = <0 33 0>;
+				interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_MDMA0>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -624,7 +624,7 @@
 			mdma1: mdma@11C10000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x11C10000 0x1000>;
-				interrupts = <0 124 0>;
+				interrupts = <GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_MDMA1>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -636,7 +636,7 @@
 		gsc_0:  gsc@13e00000 {
 			compatible = "samsung,exynos5-gsc";
 			reg = <0x13e00000 0x1000>;
-			interrupts = <0 85 0>;
+			interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&pd_gsc>;
 			clocks = <&clock CLK_GSCL0>;
 			clock-names = "gscl";
@@ -646,7 +646,7 @@
 		gsc_1:  gsc@13e10000 {
 			compatible = "samsung,exynos5-gsc";
 			reg = <0x13e10000 0x1000>;
-			interrupts = <0 86 0>;
+			interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&pd_gsc>;
 			clocks = <&clock CLK_GSCL1>;
 			clock-names = "gscl";
@@ -656,7 +656,7 @@
 		gsc_2:  gsc@13e20000 {
 			compatible = "samsung,exynos5-gsc";
 			reg = <0x13e20000 0x1000>;
-			interrupts = <0 87 0>;
+			interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&pd_gsc>;
 			clocks = <&clock CLK_GSCL2>;
 			clock-names = "gscl";
@@ -666,7 +666,7 @@
 		gsc_3:  gsc@13e30000 {
 			compatible = "samsung,exynos5-gsc";
 			reg = <0x13e30000 0x1000>;
-			interrupts = <0 88 0>;
+			interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&pd_gsc>;
 			clocks = <&clock CLK_GSCL3>;
 			clock-names = "gscl";
@@ -677,7 +677,7 @@
 			compatible = "samsung,exynos4212-hdmi";
 			reg = <0x14530000 0x70000>;
 			power-domains = <&pd_disp1>;
-			interrupts = <0 95 0>;
+			interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_HDMI>, <&clock CLK_SCLK_HDMI>,
 				 <&clock CLK_SCLK_PIXEL>, <&clock CLK_SCLK_HDMIPHY>,
 				 <&clock CLK_MOUT_HDMI>;
@@ -690,7 +690,7 @@
 			compatible = "samsung,exynos5250-mixer";
 			reg = <0x14450000 0x10000>;
 			power-domains = <&pd_disp1>;
-			interrupts = <0 94 0>;
+			interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_MIXER>, <&clock CLK_HDMI>,
 				 <&clock CLK_SCLK_HDMI>;
 			clock-names = "mixer", "hdmi", "sclk_hdmi";
@@ -706,7 +706,7 @@
 		adc: adc@12D10000 {
 			compatible = "samsung,exynos-adc-v1";
 			reg = <0x12D10000 0x100>;
-			interrupts = <0 106 0>;
+			interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_ADC>;
 			clock-names = "adc";
 			#io-channel-cells = <1>;
@@ -718,7 +718,7 @@
 		sss@10830000 {
 			compatible = "samsung,exynos4210-secss";
 			reg = <0x10830000 0x300>;
-			interrupts = <0 112 0>;
+			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_SSS>;
 			clock-names = "secss";
 		};
diff --git a/arch/arm/boot/dts/exynos5260.dtsi b/arch/arm/boot/dts/exynos5260.dtsi
index a86a4898d077..5818718618b1 100644
--- a/arch/arm/boot/dts/exynos5260.dtsi
+++ b/arch/arm/boot/dts/exynos5260.dtsi
@@ -10,6 +10,8 @@
 */
 
 #include <dt-bindings/clock/exynos5260-clk.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/interrupt-controller/irq.h>
 
 / {
 	compatible = "samsung,exynos5260", "samsung,exynos5";
@@ -168,7 +170,8 @@
 				<0x10482000 0x1000>,
 				<0x10484000 0x2000>,
 				<0x10486000 0x2000>;
-			interrupts = <1 9 0xf04>;
+			interrupts = <GIC_PPI 9
+					(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
 		};
 
 		chipid: chipid@10000000 {
@@ -181,10 +184,18 @@
 			reg = <0x100B0000 0x1000>;
 			clocks = <&fin_pll>, <&clock_peri PERI_CLK_MCT>;
 			clock-names = "fin_pll", "mct";
-			interrupts = <0 104 0>, <0 105 0>, <0 106 0>,
-					<0 107 0>, <0 122 0>, <0 123 0>,
-					<0 124 0>, <0 125 0>, <0 126 0>,
-					<0 127 0>, <0 128 0>, <0 129 0>;
+			interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 128 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 129 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		cci: cci@10F00000 {
@@ -210,25 +221,25 @@
 		pinctrl_0: pinctrl@11600000 {
 			compatible = "samsung,exynos5260-pinctrl";
 			reg = <0x11600000 0x1000>;
-			interrupts = <0 79 0>;
+			interrupts = <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>;
 
 			wakeup-interrupt-controller {
 				compatible = "samsung,exynos4210-wakeup-eint";
 				interrupt-parent = <&gic>;
-				interrupts = <0 32 0>;
+				interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
 			};
 		};
 
 		pinctrl_1: pinctrl@12290000 {
 			compatible = "samsung,exynos5260-pinctrl";
 			reg = <0x12290000 0x1000>;
-			interrupts = <0 157 0>;
+			interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_2: pinctrl@128B0000 {
 			compatible = "samsung,exynos5260-pinctrl";
 			reg = <0x128B0000 0x1000>;
-			interrupts = <0 243 0>;
+			interrupts = <GIC_SPI 243 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pmu_system_controller: system-controller@10D50000 {
@@ -239,7 +250,7 @@
 		uart0: serial@12C00000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C00000 0x100>;
-			interrupts = <0 146 0>;
+			interrupts = <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peri PERI_CLK_UART0>, <&clock_peri PERI_SCLK_UART0>;
 			clock-names = "uart", "clk_uart_baud0";
 			status = "disabled";
@@ -248,7 +259,7 @@
 		uart1: serial@12C10000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C10000 0x100>;
-			interrupts = <0 147 0>;
+			interrupts = <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peri PERI_CLK_UART1>, <&clock_peri PERI_SCLK_UART1>;
 			clock-names = "uart", "clk_uart_baud0";
 			status = "disabled";
@@ -257,7 +268,7 @@
 		uart2: serial@12C20000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C20000 0x100>;
-			interrupts = <0 148 0>;
+			interrupts = <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peri PERI_CLK_UART2>, <&clock_peri PERI_SCLK_UART2>;
 			clock-names = "uart", "clk_uart_baud0";
 			status = "disabled";
@@ -266,7 +277,7 @@
 		uart3: serial@12860000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12860000 0x100>;
-			interrupts = <0 145 0>;
+			interrupts = <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_aud AUD_CLK_AUD_UART>, <&clock_aud AUD_SCLK_AUD_UART>;
 			clock-names = "uart", "clk_uart_baud0";
 			status = "disabled";
@@ -275,7 +286,7 @@
 		mmc_0: mmc@12140000 {
 			compatible = "samsung,exynos5250-dw-mshc";
 			reg = <0x12140000 0x2000>;
-			interrupts = <0 156 0>;
+			interrupts = <GIC_SPI 156 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			clocks = <&clock_fsys FSYS_CLK_MMC0>, <&clock_top TOP_SCLK_MMC0>;
@@ -287,7 +298,7 @@
 		mmc_1: mmc@12150000 {
 			compatible = "samsung,exynos5250-dw-mshc";
 			reg = <0x12150000 0x2000>;
-			interrupts = <0 158 0>;
+			interrupts = <GIC_SPI 158 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			clocks = <&clock_fsys FSYS_CLK_MMC1>, <&clock_top TOP_SCLK_MMC1>;
@@ -299,7 +310,7 @@
 		mmc_2: mmc@12160000 {
 			compatible = "samsung,exynos5250-dw-mshc";
 			reg = <0x12160000 0x2000>;
-			interrupts = <0 159 0>;
+			interrupts = <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			clocks = <&clock_fsys FSYS_CLK_MMC2>, <&clock_top TOP_SCLK_MMC2>;
diff --git a/arch/arm/boot/dts/exynos5410-odroidxu.dts b/arch/arm/boot/dts/exynos5410-odroidxu.dts
index 3c271cb4b2be..c4de1353e5df 100644
--- a/arch/arm/boot/dts/exynos5410-odroidxu.dts
+++ b/arch/arm/boot/dts/exynos5410-odroidxu.dts
@@ -15,6 +15,7 @@
 #include <dt-bindings/clock/maxim,max77802.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/sound/samsung-i2s.h>
 #include "exynos54xx-odroidxu-leds.dtsi"
 
 / {
@@ -57,6 +58,61 @@
 		compatible = "samsung,secure-firmware";
 		reg = <0x02073000 0x1000>;
 	};
+
+	sound: sound {
+		compatible = "simple-audio-card";
+
+		simple-audio-card,name = "Odroid-XU";
+		simple-audio-card,widgets =
+			"Headphone", "Headphone Jack",
+			"Speakers", "Speakers";
+		simple-audio-card,routing =
+			"Headphone Jack", "HPL",
+			"Headphone Jack", "HPR",
+			"Headphone Jack", "MICBIAS",
+			"IN1", "Headphone Jack",
+			"Speakers", "SPKL",
+			"Speakers", "SPKR";
+
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&link0_codec>;
+		simple-audio-card,frame-master = <&link0_codec>;
+
+		simple-audio-card,cpu {
+			sound-dai = <&audi2s0 0>;
+			system-clock-frequency = <19200000>;
+		};
+
+		link0_codec: simple-audio-card,codec {
+			sound-dai = <&max98090>;
+			clocks = <&audi2s0 CLK_I2S_CDCLK>;
+		};
+	};
+};
+
+&audi2s0 {
+	status = "okay";
+};
+
+&clock {
+	clocks = <&fin_pll>;
+	assigned-clocks = <&clock CLK_FOUT_EPLL>;
+	assigned-clock-rates = <192000000>;
+};
+
+&clock_audss {
+	assigned-clocks = <&clock_audss EXYNOS_MOUT_AUDSS>,
+			<&clock_audss EXYNOS_MOUT_I2S>,
+			<&clock_audss EXYNOS_DOUT_SRP>,
+			<&clock_audss EXYNOS_DOUT_AUD_BUS>;
+
+	assigned-clock-parents = <&clock CLK_FOUT_EPLL>,
+			<&clock_audss EXYNOS_MOUT_AUDSS>;
+
+	assigned-clock-rates =  <0>,
+				<0>,
+				<96000000>,
+				<19200000>;
 };
 
 &cpu0_thermal {
@@ -440,6 +496,19 @@
 	};
 };
 
+&i2c_1 {
+	status = "okay";
+	max98090: max98090@10 {
+		compatible = "maxim,max98090";
+		reg = <0x10>;
+		interrupt-parent = <&gpj3>;
+		interrupts = <0 IRQ_TYPE_NONE>;
+		clocks = <&audi2s0 CLK_I2S_CDCLK>;
+		clock-names = "mclk";
+		#sound-dai-cells = <0>;
+	};
+};
+
 &mmc_0 {
 	status = "okay";
 	mmc-pwrseq = <&emmc_pwrseq>;
diff --git a/arch/arm/boot/dts/exynos5410-pinctrl.dtsi b/arch/arm/boot/dts/exynos5410-pinctrl.dtsi
index a083d23fdee3..ff46a1c27182 100644
--- a/arch/arm/boot/dts/exynos5410-pinctrl.dtsi
+++ b/arch/arm/boot/dts/exynos5410-pinctrl.dtsi
@@ -615,4 +615,13 @@
 		interrupt-controller;
 		#interrupt-cells = <2>;
 	};
+
+	audi2s0_bus: audi2s0-bus {
+		samsung,pins = "gpz-0", "gpz-1", "gpz-2", "gpz-3",
+				"gpz-4";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
 };
diff --git a/arch/arm/boot/dts/exynos5410.dtsi b/arch/arm/boot/dts/exynos5410.dtsi
index 137f48464f8b..2b6adafe18e2 100644
--- a/arch/arm/boot/dts/exynos5410.dtsi
+++ b/arch/arm/boot/dts/exynos5410.dtsi
@@ -16,6 +16,7 @@
 #include "exynos54xx.dtsi"
 #include "exynos-syscon-restart.dtsi"
 #include <dt-bindings/clock/exynos5410.h>
+#include <dt-bindings/clock/exynos-audss-clk.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 
 / {
@@ -82,10 +83,18 @@
 			#clock-cells = <1>;
 		};
 
+		clock_audss: audss-clock-controller@3810000 {
+			compatible = "samsung,exynos5410-audss-clock";
+			reg = <0x03810000 0x0C>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>, <&clock CLK_FOUT_EPLL>;
+			clock-names = "pll_ref", "pll_in";
+		};
+
 		tmu_cpu0: tmu@10060000 {
 			compatible = "samsung,exynos5420-tmu";
 			reg = <0x10060000 0x100>;
-			interrupts = <GIC_SPI 65 0>;
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_TMU>;
 			clock-names = "tmu_apbif";
 			#include "exynos4412-tmu-sensor-conf.dtsi"
@@ -94,7 +103,7 @@
 		tmu_cpu1: tmu@10064000 {
 			compatible = "samsung,exynos5420-tmu";
 			reg = <0x10064000 0x100>;
-			interrupts = <GIC_SPI 183 0>;
+			interrupts = <GIC_SPI 183 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_TMU>;
 			clock-names = "tmu_apbif";
 			#include "exynos4412-tmu-sensor-conf.dtsi"
@@ -103,7 +112,7 @@
 		tmu_cpu2: tmu@10068000 {
 			compatible = "samsung,exynos5420-tmu";
 			reg = <0x10068000 0x100>;
-			interrupts = <GIC_SPI 184 0>;
+			interrupts = <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_TMU>;
 			clock-names = "tmu_apbif";
 			#include "exynos4412-tmu-sensor-conf.dtsi"
@@ -112,7 +121,7 @@
 		tmu_cpu3: tmu@1006c000 {
 			compatible = "samsung,exynos5420-tmu";
 			reg = <0x1006c000 0x100>;
-			interrupts = <GIC_SPI 185 0>;
+			interrupts = <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_TMU>;
 			clock-names = "tmu_apbif";
 			#include "exynos4412-tmu-sensor-conf.dtsi"
@@ -121,7 +130,7 @@
 		mmc_0: mmc@12200000 {
 			compatible = "samsung,exynos5250-dw-mshc";
 			reg = <0x12200000 0x1000>;
-			interrupts = <0 75 0>;
+			interrupts = <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			clocks = <&clock CLK_MMC0>, <&clock CLK_SCLK_MMC0>;
@@ -133,7 +142,7 @@
 		mmc_1: mmc@12210000 {
 			compatible = "samsung,exynos5250-dw-mshc";
 			reg = <0x12210000 0x1000>;
-			interrupts = <0 76 0>;
+			interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			clocks = <&clock CLK_MMC1>, <&clock CLK_SCLK_MMC1>;
@@ -145,7 +154,7 @@
 		mmc_2: mmc@12220000 {
 			compatible = "samsung,exynos5250-dw-mshc";
 			reg = <0x12220000 0x1000>;
-			interrupts = <0 77 0>;
+			interrupts = <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			clocks = <&clock CLK_MMC2>, <&clock CLK_SCLK_MMC2>;
@@ -157,31 +166,81 @@
 		pinctrl_0: pinctrl@13400000 {
 			compatible = "samsung,exynos5410-pinctrl";
 			reg = <0x13400000 0x1000>;
-			interrupts = <0 45 0>;
+			interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
 
 			wakeup-interrupt-controller {
 				compatible = "samsung,exynos4210-wakeup-eint";
 				interrupt-parent = <&gic>;
-				interrupts = <0 32 0>;
+				interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
 			};
 		};
 
 		pinctrl_1: pinctrl@14000000 {
 			compatible = "samsung,exynos5410-pinctrl";
 			reg = <0x14000000 0x1000>;
-			interrupts = <0 46 0>;
+			interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_2: pinctrl@10d10000 {
 			compatible = "samsung,exynos5410-pinctrl";
 			reg = <0x10d10000 0x1000>;
-			interrupts = <0 50 0>;
+			interrupts = <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_3: pinctrl@03860000 {
 			compatible = "samsung,exynos5410-pinctrl";
 			reg = <0x03860000 0x1000>;
-			interrupts = <0 47 0>;
+			interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		amba {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "simple-bus";
+			interrupt-parent = <&gic>;
+			ranges;
+
+			pdma0: pdma@12680000 {
+				compatible = "arm,pl330", "arm,primecell";
+				reg = <0x121A0000 0x1000>;
+				interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clock CLK_PDMA0>;
+				clock-names = "apb_pclk";
+				#dma-cells = <1>;
+				#dma-channels = <8>;
+				#dma-requests = <32>;
+			};
+
+			pdma1: pdma@12690000 {
+				compatible = "arm,pl330", "arm,primecell";
+				reg = <0x121B0000 0x1000>;
+				interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clock CLK_PDMA1>;
+				clock-names = "apb_pclk";
+				#dma-cells = <1>;
+				#dma-channels = <8>;
+				#dma-requests = <32>;
+			};
+		};
+
+		audi2s0: i2s@03830000 {
+			compatible = "samsung,exynos5420-i2s";
+			reg = <0x03830000 0x100>;
+			dmas = <&pdma0 10
+				&pdma0 9
+				&pdma0 8>;
+			dma-names = "tx", "rx", "tx-sec";
+			clocks = <&clock_audss EXYNOS_I2S_BUS>,
+				<&clock_audss EXYNOS_I2S_BUS>,
+				<&clock_audss EXYNOS_SCLK_I2S>;
+			clock-names = "iis", "i2s_opclk0", "i2s_opclk1";
+			#clock-cells = <1>;
+			clock-output-names = "i2s_cdclk0";
+			#sound-dai-cells = <1>;
+			samsung,idma-addr = <0x03000000>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&audi2s0_bus>;
+			status = "disabled";
 		};
 	};
 
@@ -329,7 +388,7 @@
 };
 
 &usbdrd_dwc3_1 {
-	interrupts = <GIC_SPI 200 0>;
+	interrupts = <GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH>;
 };
 
 &usbdrd_phy1 {
diff --git a/arch/arm/boot/dts/exynos5420-peach-pit.dts b/arch/arm/boot/dts/exynos5420-peach-pit.dts
index ec4a00f1ce01..1f964ec35c5e 100644
--- a/arch/arm/boot/dts/exynos5420-peach-pit.dts
+++ b/arch/arm/boot/dts/exynos5420-peach-pit.dts
@@ -697,6 +697,7 @@
 	status = "okay";
 };
 
+/* eMMC flash */
 &mmc_0 {
 	status = "okay";
 	num-slots = <1>;
@@ -714,6 +715,7 @@
 	bus-width = <8>;
 };
 
+/* WiFi SDIO module */
 &mmc_1 {
 	status = "okay";
 	num-slots = <1>;
@@ -733,6 +735,7 @@
 	vqmmc-supply = <&buck10_reg>;
 };
 
+/* uSD card */
 &mmc_2 {
 	status = "okay";
 	num-slots = <1>;
diff --git a/arch/arm/boot/dts/exynos5420.dtsi b/arch/arm/boot/dts/exynos5420.dtsi
index 00c4cfa54839..906a1a42a7ea 100644
--- a/arch/arm/boot/dts/exynos5420.dtsi
+++ b/arch/arm/boot/dts/exynos5420.dtsi
@@ -193,7 +193,7 @@
 		mfc: codec@11000000 {
 			compatible = "samsung,mfc-v7";
 			reg = <0x11000000 0x10000>;
-			interrupts = <0 96 0>;
+			interrupts = <0 96 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_MFC>;
 			clock-names = "mfc";
 			power-domains = <&mfc_pd>;
@@ -203,7 +203,7 @@
 
 		mmc_0: mmc@12200000 {
 			compatible = "samsung,exynos5420-dw-mshc-smu";
-			interrupts = <0 75 0>;
+			interrupts = <0 75 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x12200000 0x2000>;
@@ -215,7 +215,7 @@
 
 		mmc_1: mmc@12210000 {
 			compatible = "samsung,exynos5420-dw-mshc-smu";
-			interrupts = <0 76 0>;
+			interrupts = <0 76 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x12210000 0x2000>;
@@ -227,7 +227,7 @@
 
 		mmc_2: mmc@12220000 {
 			compatible = "samsung,exynos5420-dw-mshc";
-			interrupts = <0 77 0>;
+			interrupts = <0 77 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x12220000 0x1000>;
@@ -320,37 +320,37 @@
 		pinctrl_0: pinctrl@13400000 {
 			compatible = "samsung,exynos5420-pinctrl";
 			reg = <0x13400000 0x1000>;
-			interrupts = <0 45 0>;
+			interrupts = <0 45 IRQ_TYPE_LEVEL_HIGH>;
 
 			wakeup-interrupt-controller {
 				compatible = "samsung,exynos4210-wakeup-eint";
 				interrupt-parent = <&gic>;
-				interrupts = <0 32 0>;
+				interrupts = <0 32 IRQ_TYPE_LEVEL_HIGH>;
 			};
 		};
 
 		pinctrl_1: pinctrl@13410000 {
 			compatible = "samsung,exynos5420-pinctrl";
 			reg = <0x13410000 0x1000>;
-			interrupts = <0 78 0>;
+			interrupts = <0 78 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_2: pinctrl@14000000 {
 			compatible = "samsung,exynos5420-pinctrl";
 			reg = <0x14000000 0x1000>;
-			interrupts = <0 46 0>;
+			interrupts = <0 46 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_3: pinctrl@14010000 {
 			compatible = "samsung,exynos5420-pinctrl";
 			reg = <0x14010000 0x1000>;
-			interrupts = <0 50 0>;
+			interrupts = <0 50 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_4: pinctrl@03860000 {
 			compatible = "samsung,exynos5420-pinctrl";
 			reg = <0x03860000 0x1000>;
-			interrupts = <0 47 0>;
+			interrupts = <0 47 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		amba {
@@ -363,7 +363,7 @@
 			adma: adma@03880000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x03880000 0x1000>;
-				interrupts = <0 110 0>;
+				interrupts = <0 110 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock_audss EXYNOS_ADMA>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -374,7 +374,7 @@
 			pdma0: pdma@121A0000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x121A0000 0x1000>;
-				interrupts = <0 34 0>;
+				interrupts = <0 34 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_PDMA0>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -385,7 +385,7 @@
 			pdma1: pdma@121B0000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x121B0000 0x1000>;
-				interrupts = <0 35 0>;
+				interrupts = <0 35 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_PDMA1>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -396,7 +396,7 @@
 			mdma0: mdma@10800000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x10800000 0x1000>;
-				interrupts = <0 33 0>;
+				interrupts = <0 33 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_MDMA0>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -407,7 +407,7 @@
 			mdma1: mdma@11C10000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x11C10000 0x1000>;
-				interrupts = <0 124 0>;
+				interrupts = <0 124 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_MDMA1>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -479,7 +479,7 @@
 		spi_0: spi@12d20000 {
 			compatible = "samsung,exynos4210-spi";
 			reg = <0x12d20000 0x100>;
-			interrupts = <0 68 0>;
+			interrupts = <0 68 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&pdma0 5
 				&pdma0 4>;
 			dma-names = "tx", "rx";
@@ -495,7 +495,7 @@
 		spi_1: spi@12d30000 {
 			compatible = "samsung,exynos4210-spi";
 			reg = <0x12d30000 0x100>;
-			interrupts = <0 69 0>;
+			interrupts = <0 69 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&pdma1 5
 				&pdma1 4>;
 			dma-names = "tx", "rx";
@@ -511,7 +511,7 @@
 		spi_2: spi@12d40000 {
 			compatible = "samsung,exynos4210-spi";
 			reg = <0x12d40000 0x100>;
-			interrupts = <0 70 0>;
+			interrupts = <0 70 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&pdma0 7
 				&pdma0 6>;
 			dma-names = "tx", "rx";
@@ -539,7 +539,7 @@
 		dsi@14500000 {
 			compatible = "samsung,exynos5410-mipi-dsi";
 			reg = <0x14500000 0x10000>;
-			interrupts = <0 82 0>;
+			interrupts = <0 82 IRQ_TYPE_LEVEL_HIGH>;
 			phys = <&mipi_phy 1>;
 			phy-names = "dsim";
 			clocks = <&clock CLK_DSIM1>, <&clock CLK_SCLK_MIPI1>;
@@ -552,7 +552,7 @@
 		adc: adc@12D10000 {
 			compatible = "samsung,exynos-adc-v2";
 			reg = <0x12D10000 0x100>;
-			interrupts = <0 106 0>;
+			interrupts = <0 106 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_TSADC>;
 			clock-names = "adc";
 			#io-channel-cells = <1>;
@@ -564,7 +564,7 @@
 		hsi2c_8: i2c@12E00000 {
 			compatible = "samsung,exynos5250-hsi2c";
 			reg = <0x12E00000 0x1000>;
-			interrupts = <0 87 0>;
+			interrupts = <0 87 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -577,7 +577,7 @@
 		hsi2c_9: i2c@12E10000 {
 			compatible = "samsung,exynos5250-hsi2c";
 			reg = <0x12E10000 0x1000>;
-			interrupts = <0 88 0>;
+			interrupts = <0 88 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -590,7 +590,7 @@
 		hsi2c_10: i2c@12E20000 {
 			compatible = "samsung,exynos5250-hsi2c";
 			reg = <0x12E20000 0x1000>;
-			interrupts = <0 203 0>;
+			interrupts = <0 203 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -603,7 +603,7 @@
 		hdmi: hdmi@14530000 {
 			compatible = "samsung,exynos5420-hdmi";
 			reg = <0x14530000 0x70000>;
-			interrupts = <0 95 0>;
+			interrupts = <0 95 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_HDMI>, <&clock CLK_SCLK_HDMI>,
 				 <&clock CLK_DOUT_PIXEL>, <&clock CLK_SCLK_HDMIPHY>,
 				 <&clock CLK_MOUT_HDMI>;
@@ -622,7 +622,7 @@
 		mixer: mixer@14450000 {
 			compatible = "samsung,exynos5420-mixer";
 			reg = <0x14450000 0x10000>;
-			interrupts = <0 94 0>;
+			interrupts = <0 94 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_MIXER>, <&clock CLK_HDMI>,
 				 <&clock CLK_SCLK_HDMI>;
 			clock-names = "mixer", "hdmi", "sclk_hdmi";
@@ -633,7 +633,7 @@
 		rotator: rotator@11C00000 {
 			compatible = "samsung,exynos5250-rotator";
 			reg = <0x11C00000 0x64>;
-			interrupts = <0 84 0>;
+			interrupts = <0 84 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_ROTATOR>;
 			clock-names = "rotator";
 			iommus = <&sysmmu_rotator>;
@@ -642,7 +642,7 @@
 		gsc_0: video-scaler@13e00000 {
 			compatible = "samsung,exynos5-gsc";
 			reg = <0x13e00000 0x1000>;
-			interrupts = <0 85 0>;
+			interrupts = <0 85 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_GSCL0>;
 			clock-names = "gscl";
 			power-domains = <&gsc_pd>;
@@ -652,7 +652,7 @@
 		gsc_1: video-scaler@13e10000 {
 			compatible = "samsung,exynos5-gsc";
 			reg = <0x13e10000 0x1000>;
-			interrupts = <0 86 0>;
+			interrupts = <0 86 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_GSCL1>;
 			clock-names = "gscl";
 			power-domains = <&gsc_pd>;
@@ -662,7 +662,7 @@
 		jpeg_0: jpeg@11F50000 {
 			compatible = "samsung,exynos5420-jpeg";
 			reg = <0x11F50000 0x1000>;
-			interrupts = <0 89 0>;
+			interrupts = <0 89 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "jpeg";
 			clocks = <&clock CLK_JPEG>;
 			iommus = <&sysmmu_jpeg0>;
@@ -671,7 +671,7 @@
 		jpeg_1: jpeg@11F60000 {
 			compatible = "samsung,exynos5420-jpeg";
 			reg = <0x11F60000 0x1000>;
-			interrupts = <0 168 0>;
+			interrupts = <0 168 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "jpeg";
 			clocks = <&clock CLK_JPEG2>;
 			iommus = <&sysmmu_jpeg1>;
@@ -691,7 +691,7 @@
 		tmu_cpu0: tmu@10060000 {
 			compatible = "samsung,exynos5420-tmu";
 			reg = <0x10060000 0x100>;
-			interrupts = <0 65 0>;
+			interrupts = <0 65 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_TMU>;
 			clock-names = "tmu_apbif";
 			#include "exynos4412-tmu-sensor-conf.dtsi"
@@ -700,7 +700,7 @@
 		tmu_cpu1: tmu@10064000 {
 			compatible = "samsung,exynos5420-tmu";
 			reg = <0x10064000 0x100>;
-			interrupts = <0 183 0>;
+			interrupts = <0 183 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_TMU>;
 			clock-names = "tmu_apbif";
 			#include "exynos4412-tmu-sensor-conf.dtsi"
@@ -709,7 +709,7 @@
 		tmu_cpu2: tmu@10068000 {
 			compatible = "samsung,exynos5420-tmu-ext-triminfo";
 			reg = <0x10068000 0x100>, <0x1006c000 0x4>;
-			interrupts = <0 184 0>;
+			interrupts = <0 184 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_TMU>, <&clock CLK_TMU>;
 			clock-names = "tmu_apbif", "tmu_triminfo_apbif";
 			#include "exynos4412-tmu-sensor-conf.dtsi"
@@ -718,7 +718,7 @@
 		tmu_cpu3: tmu@1006c000 {
 			compatible = "samsung,exynos5420-tmu-ext-triminfo";
 			reg = <0x1006c000 0x100>, <0x100a0000 0x4>;
-			interrupts = <0 185 0>;
+			interrupts = <0 185 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_TMU>, <&clock CLK_TMU_GPU>;
 			clock-names = "tmu_apbif", "tmu_triminfo_apbif";
 			#include "exynos4412-tmu-sensor-conf.dtsi"
@@ -727,7 +727,7 @@
 		tmu_gpu: tmu@100a0000 {
 			compatible = "samsung,exynos5420-tmu-ext-triminfo";
 			reg = <0x100a0000 0x100>, <0x10068000 0x4>;
-			interrupts = <0 215 0>;
+			interrupts = <0 215 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock CLK_TMU_GPU>, <&clock CLK_TMU>;
 			clock-names = "tmu_apbif", "tmu_triminfo_apbif";
 			#include "exynos4412-tmu-sensor-conf.dtsi"
@@ -799,7 +799,7 @@
 		sysmmu_scaler1r: sysmmu@0x12890000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x12890000 0x1000>;
-			interrupts = <0 186 0>;
+			interrupts = <0 186 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "sysmmu", "master";
 			clocks = <&clock CLK_SMMU_MSCL1>, <&clock CLK_MSCL1>;
 			#iommu-cells = <0>;
@@ -808,7 +808,7 @@
 		sysmmu_scaler2r: sysmmu@0x128A0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x128A0000 0x1000>;
-			interrupts = <0 188 0>;
+			interrupts = <0 188 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "sysmmu", "master";
 			clocks = <&clock CLK_SMMU_MSCL2>, <&clock CLK_MSCL2>;
 			#iommu-cells = <0>;
@@ -867,7 +867,7 @@
 		sysmmu_jpeg1: sysmmu@0x11F20000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x11F20000 0x1000>;
-			interrupts = <0 169 0>;
+			interrupts = <0 169 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "sysmmu", "master";
 			clocks = <&clock CLK_SMMU_JPEG2>, <&clock CLK_JPEG2>;
 			#iommu-cells = <0>;
@@ -1445,7 +1445,7 @@
 };
 
 &usbdrd_dwc3_1 {
-	interrupts = <GIC_SPI 73 0>;
+	interrupts = <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
 };
 
 &usbdrd_phy1 {
diff --git a/arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi b/arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi
index 246d298557f5..05b9afdd6757 100644
--- a/arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi
+++ b/arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi
@@ -147,6 +147,11 @@
 	};
 };
 
+&adc {
+	vdd-supply = <&ldo4_reg>;
+	status = "okay";
+};
+
 &bus_wcore {
 	devfreq-events = <&nocp_mem0_0>, <&nocp_mem0_1>,
 			<&nocp_mem1_0>, <&nocp_mem1_1>;
@@ -293,6 +298,12 @@
 				regulator-max-microvolt = <1800000>;
 			};
 
+			ldo4_reg: LDO4 {
+				regulator-name = "vdd_adc";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
 			ldo5_reg: LDO5 {
 				regulator-name = "vdd_ldo5";
 				regulator-min-microvolt = <1800000>;
@@ -499,7 +510,6 @@
 &mmc_0 {
 	status = "okay";
 	mmc-pwrseq = <&emmc_pwrseq>;
-	cd-gpios = <&gpc0 2 GPIO_ACTIVE_LOW>;
 	card-detect-delay = <200>;
 	samsung,dw-mshc-ciu-div = <3>;
 	samsung,dw-mshc-sdr-timing = <0 4>;
diff --git a/arch/arm/boot/dts/exynos5440.dtsi b/arch/arm/boot/dts/exynos5440.dtsi
index e6bffd13cedd..2a2e570bbee6 100644
--- a/arch/arm/boot/dts/exynos5440.dtsi
+++ b/arch/arm/boot/dts/exynos5440.dtsi
@@ -10,6 +10,8 @@
 */
 
 #include <dt-bindings/clock/exynos5440.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/interrupt-controller/irq.h>
 
 / {
 	compatible = "samsung,exynos5440", "samsung,exynos5";
@@ -41,7 +43,8 @@
 			<0x2E2000 0x1000>,
 			<0x2E4000 0x2000>,
 			<0x2E6000 0x2000>;
-		interrupts = <1 9 0xf04>;
+		interrupts = <GIC_PPI 9
+				(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
 	};
 
 	cpus {
@@ -72,26 +75,26 @@
 
 	arm-pmu {
 		compatible = "arm,cortex-a15-pmu", "arm,cortex-a9-pmu";
-		interrupts = <0 52 4>,
-			     <0 53 4>,
-			     <0 54 4>,
-			     <0 55 4>;
+		interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
 	timer {
 		compatible = "arm,cortex-a15-timer",
 			     "arm,armv7-timer";
-		interrupts = <1 13 0xf08>,
-			     <1 14 0xf08>,
-			     <1 11 0xf08>,
-			     <1 10 0xf08>;
+		interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
 		clock-frequency = <50000000>;
 	};
 
 	cpufreq@160000 {
 		compatible = "samsung,exynos5440-cpufreq";
 		reg = <0x160000 0x1000>;
-		interrupts = <0 57 0>;
+		interrupts = <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>;
 		operating-points = <
 				/* KHz	  uV */
 				1500000 1100000
@@ -108,7 +111,7 @@
 	serial_0: serial@B0000 {
 		compatible = "samsung,exynos4210-uart";
 		reg = <0xB0000 0x1000>;
-		interrupts = <0 2 0>;
+		interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_B_125>, <&clock CLK_B_125>;
 		clock-names = "uart", "clk_uart_baud0";
 	};
@@ -116,7 +119,7 @@
 	serial_1: serial@C0000 {
 		compatible = "samsung,exynos4210-uart";
 		reg = <0xC0000 0x1000>;
-		interrupts = <0 3 0>;
+		interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_B_125>, <&clock CLK_B_125>;
 		clock-names = "uart", "clk_uart_baud0";
 	};
@@ -124,7 +127,7 @@
 	spi_0: spi@D0000 {
 		compatible = "samsung,exynos5440-spi";
 		reg = <0xD0000 0x100>;
-		interrupts = <0 4 0>;
+		interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>;
 		#address-cells = <1>;
 		#size-cells = <0>;
 		samsung,spi-src-clk = <0>;
@@ -136,8 +139,14 @@
 	pin_ctrl: pinctrl@E0000 {
 		compatible = "samsung,exynos5440-pinctrl";
 		reg = <0xE0000 0x1000>;
-		interrupts = <0 37 0>, <0 38 0>, <0 39 0>, <0 40 0>,
-			     <0 41 0>, <0 42 0>, <0 43 0>, <0 44 0>;
+		interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-controller;
 		#interrupt-cells = <2>;
 		#gpio-cells = <2>;
@@ -162,7 +171,7 @@
 	i2c@F0000 {
 		compatible = "samsung,exynos5440-i2c";
 		reg = <0xF0000 0x1000>;
-		interrupts = <0 5 0>;
+		interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
 		#address-cells = <1>;
 		#size-cells = <0>;
 		clocks = <&clock CLK_B_125>;
@@ -172,7 +181,7 @@
 	i2c@100000 {
 		compatible = "samsung,exynos5440-i2c";
 		reg = <0x100000 0x1000>;
-		interrupts = <0 6 0>;
+		interrupts = <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
 		#address-cells = <1>;
 		#size-cells = <0>;
 		clocks = <&clock CLK_B_125>;
@@ -182,16 +191,16 @@
 	watchdog@110000 {
 		compatible = "samsung,s3c2410-wdt";
 		reg = <0x110000 0x1000>;
-		interrupts = <0 1 0>;
+		interrupts = <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_B_125>;
 		clock-names = "watchdog";
 	};
 
 	gmac: ethernet@00230000 {
-		compatible = "snps,dwmac-3.70a";
+		compatible = "snps,dwmac-3.70a", "snps,dwmac";
 		reg = <0x00230000 0x8000>;
 		interrupt-parent = <&gic>;
-		interrupts = <0 31 4>;
+		interrupts = <GIC_SPI 31 4>;
 		interrupt-names = "macirq";
 		phy-mode = "sgmii";
 		clocks = <&clock CLK_GMAC0>;
@@ -209,7 +218,8 @@
 	rtc@130000 {
 		compatible = "samsung,s3c6410-rtc";
 		reg = <0x130000 0x1000>;
-		interrupts = <0 17 0>, <0 16 0>;
+		interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_B_125>;
 		clock-names = "rtc";
 	};
@@ -217,7 +227,7 @@
 	tmuctrl_0: tmuctrl@160118 {
 		compatible = "samsung,exynos5440-tmu";
 		reg = <0x160118 0x230>, <0x160368 0x10>;
-		interrupts = <0 58 0>;
+		interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_B_125>;
 		clock-names = "tmu_apbif";
 		#include "exynos5440-tmu-sensor-conf.dtsi"
@@ -226,7 +236,7 @@
 	tmuctrl_1: tmuctrl@16011C {
 		compatible = "samsung,exynos5440-tmu";
 		reg = <0x16011C 0x230>, <0x160368 0x10>;
-		interrupts = <0 58 0>;
+		interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_B_125>;
 		clock-names = "tmu_apbif";
 		#include "exynos5440-tmu-sensor-conf.dtsi"
@@ -235,7 +245,7 @@
 	tmuctrl_2: tmuctrl@160120 {
 		compatible = "samsung,exynos5440-tmu";
 		reg = <0x160120 0x230>, <0x160368 0x10>;
-		interrupts = <0 58 0>;
+		interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_B_125>;
 		clock-names = "tmu_apbif";
 		#include "exynos5440-tmu-sensor-conf.dtsi"
@@ -259,7 +269,7 @@
 	sata@210000 {
 		compatible = "snps,exynos5440-ahci";
 		reg = <0x210000 0x10000>;
-		interrupts = <0 30 0>;
+		interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_SATA>;
 		clock-names = "sata";
 	};
@@ -267,7 +277,7 @@
 	ohci@220000 {
 		compatible = "samsung,exynos5440-ohci";
 		reg = <0x220000 0x1000>;
-		interrupts = <0 29 0>;
+		interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_USB>;
 		clock-names = "usbhost";
 	};
@@ -275,7 +285,7 @@
 	ehci@221000 {
 		compatible = "samsung,exynos5440-ehci";
 		reg = <0x221000 0x1000>;
-		interrupts = <0 29 0>;
+		interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_USB>;
 		clock-names = "usbhost";
 	};
@@ -285,7 +295,9 @@
 		reg = <0x290000 0x1000
 			0x270000 0x1000
 			0x271000 0x40>;
-		interrupts = <0 20 0>, <0 21 0>, <0 22 0>;
+		interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_PR0_250_O>, <&clock CLK_PB0_250_O>;
 		clock-names = "pcie", "pcie_bus";
 		#address-cells = <3>;
@@ -306,7 +318,9 @@
 		reg = <0x2a0000 0x1000
 			0x272000 0x1000
 			0x271040 0x40>;
-		interrupts = <0 23 0>, <0 24 0>, <0 25 0>;
+		interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clock CLK_PR1_250_O>, <&clock CLK_PB0_250_O>;
 		clock-names = "pcie", "pcie_bus";
 		#address-cells = <3>;
diff --git a/arch/arm/boot/dts/exynos54xx.dtsi b/arch/arm/boot/dts/exynos54xx.dtsi
index 9d31cdce1959..0389e8a10d0b 100644
--- a/arch/arm/boot/dts/exynos54xx.dtsi
+++ b/arch/arm/boot/dts/exynos54xx.dtsi
@@ -62,34 +62,34 @@
 						<1 &combiner 23 4>,
 						<2 &combiner 25 2>,
 						<3 &combiner 25 3>,
-						<4 &gic 0 120 0>,
-						<5 &gic 0 121 0>,
-						<6 &gic 0 122 0>,
-						<7 &gic 0 123 0>,
-						<8 &gic 0 128 0>,
-						<9 &gic 0 129 0>,
-						<10 &gic 0 130 0>,
-						<11 &gic 0 131 0>;
+						<4 &gic 0 120 IRQ_TYPE_LEVEL_HIGH>,
+						<5 &gic 0 121 IRQ_TYPE_LEVEL_HIGH>,
+						<6 &gic 0 122 IRQ_TYPE_LEVEL_HIGH>,
+						<7 &gic 0 123 IRQ_TYPE_LEVEL_HIGH>,
+						<8 &gic 0 128 IRQ_TYPE_LEVEL_HIGH>,
+						<9 &gic 0 129 IRQ_TYPE_LEVEL_HIGH>,
+						<10 &gic 0 130 IRQ_TYPE_LEVEL_HIGH>,
+						<11 &gic 0 131 IRQ_TYPE_LEVEL_HIGH>;
 			};
 		};
 
 		watchdog: watchdog@101d0000 {
 			compatible = "samsung,exynos5420-wdt";
 			reg = <0x101d0000 0x100>;
-			interrupts = <0 42 0>;
+			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		sss: sss@10830000 {
 			compatible = "samsung,exynos4210-secss";
 			reg = <0x10830000 0x300>;
-			interrupts = <0 112 0>;
+			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		/* i2c_0-3 are defined in exynos5.dtsi */
 		hsi2c_4: i2c@12ca0000 {
 			compatible = "samsung,exynos5250-hsi2c";
 			reg = <0x12ca0000 0x1000>;
-			interrupts = <0 60 0>;
+			interrupts = <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			status = "disabled";
@@ -98,7 +98,7 @@
 		hsi2c_5: i2c@12cb0000 {
 			compatible = "samsung,exynos5250-hsi2c";
 			reg = <0x12cb0000 0x1000>;
-			interrupts = <0 61 0>;
+			interrupts = <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			status = "disabled";
@@ -107,7 +107,7 @@
 		hsi2c_6: i2c@12cc0000 {
 			compatible = "samsung,exynos5250-hsi2c";
 			reg = <0x12cc0000 0x1000>;
-			interrupts = <0 62 0>;
+			interrupts = <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			status = "disabled";
@@ -116,7 +116,7 @@
 		hsi2c_7: i2c@12cd0000 {
 			compatible = "samsung,exynos5250-hsi2c";
 			reg = <0x12cd0000 0x1000>;
-			interrupts = <0 63 0>;
+			interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			status = "disabled";
@@ -131,7 +131,7 @@
 			usbdrd_dwc3_0: dwc3@12000000 {
 				compatible = "snps,dwc3";
 				reg = <0x12000000 0x10000>;
-				interrupts = <0 72 0>;
+				interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
 				phys = <&usbdrd_phy0 0>, <&usbdrd_phy0 1>;
 				phy-names = "usb2-phy", "usb3-phy";
 			};
@@ -166,7 +166,7 @@
 		usbhost2: usb@12110000 {
 			compatible = "samsung,exynos4210-ehci";
 			reg = <0x12110000 0x100>;
-			interrupts = <0 71 0>;
+			interrupts = <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -179,7 +179,7 @@
 		usbhost1: usb@12120000 {
 			compatible = "samsung,exynos4210-ohci";
 			reg = <0x12120000 0x100>;
-			interrupts = <0 71 0>;
+			interrupts = <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
 
 			#address-cells = <1>;
 			#size-cells = <0>;
diff --git a/arch/arm/boot/dts/exynos5800-peach-pi.dts b/arch/arm/boot/dts/exynos5800-peach-pi.dts
index 01f466816fea..f9ff7f07ae0c 100644
--- a/arch/arm/boot/dts/exynos5800-peach-pi.dts
+++ b/arch/arm/boot/dts/exynos5800-peach-pi.dts
@@ -665,6 +665,7 @@
 	status = "okay";
 };
 
+/* eMMC flash */
 &mmc_0 {
 	status = "okay";
 	num-slots = <1>;
@@ -683,6 +684,7 @@
 	bus-width = <8>;
 };
 
+/* WiFi SDIO module */
 &mmc_1 {
 	status = "okay";
 	num-slots = <1>;
@@ -702,6 +704,7 @@
 	vqmmc-supply = <&buck10_reg>;
 };
 
+/* uSD card */
 &mmc_2 {
 	status = "okay";
 	num-slots = <1>;
diff --git a/arch/arm/boot/dts/hi3620.dtsi b/arch/arm/boot/dts/hi3620.dtsi
index c85d07e6db61..541d70094544 100644
--- a/arch/arm/boot/dts/hi3620.dtsi
+++ b/arch/arm/boot/dts/hi3620.dtsi
@@ -11,10 +11,12 @@
  * publishhed by the Free Software Foundation.
  */
 
-#include "skeleton.dtsi"
 #include <dt-bindings/clock/hi3620-clock.h>
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		serial0 = &uart0;
 		serial1 = &uart1;
@@ -537,6 +539,7 @@
 			reg = <0x803000 0x188>;
 			#address-cells = <1>;
 			#size-cells = <1>;
+			#pinctrl-cells = <1>;
 			#gpio-range-cells = <3>;
 			ranges;
 
@@ -558,6 +561,7 @@
 			reg = <0x803800 0x2dc>;
 			#address-cells = <1>;
 			#size-cells = <1>;
+			#pinctrl-cells = <1>;
 			ranges;
 
 			pinctrl-single,register-width = <32>;
diff --git a/arch/arm/boot/dts/hip01.dtsi b/arch/arm/boot/dts/hip01.dtsi
index 4e9562f806a2..9d5fd5cfefa6 100644
--- a/arch/arm/boot/dts/hip01.dtsi
+++ b/arch/arm/boot/dts/hip01.dtsi
@@ -11,8 +11,6 @@
  * published by the Free Software Foundation.
  */
 
-#include "skeleton.dtsi"
-
 / {
 	interrupt-parent = <&gic>;
 	#address-cells = <1>;
diff --git a/arch/arm/boot/dts/hisi-x5hd2.dtsi b/arch/arm/boot/dts/hisi-x5hd2.dtsi
index 0da76c5ff6d7..c02e092fad8b 100644
--- a/arch/arm/boot/dts/hisi-x5hd2.dtsi
+++ b/arch/arm/boot/dts/hisi-x5hd2.dtsi
@@ -7,10 +7,12 @@
  * publishhed by the Free Software Foundation.
  */
 
-#include "skeleton.dtsi"
 #include <dt-bindings/clock/hix5hd2-clock.h>
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		serial0 = &uart0;
 	};
diff --git a/arch/arm/boot/dts/imx1.dtsi b/arch/arm/boot/dts/imx1.dtsi
index 22f5d1db5b31..b792eee3899b 100644
--- a/arch/arm/boot/dts/imx1.dtsi
+++ b/arch/arm/boot/dts/imx1.dtsi
@@ -9,7 +9,6 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
-#include "skeleton.dtsi"
 #include "imx1-pinfunc.h"
 
 #include <dt-bindings/clock/imx1-clock.h>
@@ -17,6 +16,9 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		gpio0 = &gpio1;
 		gpio1 = &gpio2;
diff --git a/arch/arm/boot/dts/imx23.dtsi b/arch/arm/boot/dts/imx23.dtsi
index 440ee9a4a158..ac2a9da62b6c 100644
--- a/arch/arm/boot/dts/imx23.dtsi
+++ b/arch/arm/boot/dts/imx23.dtsi
@@ -9,10 +9,12 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
-#include "skeleton.dtsi"
 #include "imx23-pinfunc.h"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	interrupt-parent = <&icoll>;
 
 	aliases {
@@ -464,7 +466,7 @@
 				reg = <0x80038000 0x2000>;
 				status = "disabled";
 			};
-                };
+		};
 
 		apbx@80040000 {
 			compatible = "simple-bus";
diff --git a/arch/arm/boot/dts/imx25.dtsi b/arch/arm/boot/dts/imx25.dtsi
index af6af8741fe5..831d09a28155 100644
--- a/arch/arm/boot/dts/imx25.dtsi
+++ b/arch/arm/boot/dts/imx25.dtsi
@@ -9,10 +9,12 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
-#include "skeleton.dtsi"
 #include "imx25-pinfunc.h"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		ethernet0 = &fec;
 		gpio0 = &gpio1;
diff --git a/arch/arm/boot/dts/imx27.dtsi b/arch/arm/boot/dts/imx27.dtsi
index f818ea483aeb..9d8b5969ee3b 100644
--- a/arch/arm/boot/dts/imx27.dtsi
+++ b/arch/arm/boot/dts/imx27.dtsi
@@ -9,7 +9,6 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
-#include "skeleton.dtsi"
 #include "imx27-pinfunc.h"
 
 #include <dt-bindings/clock/imx27-clock.h>
@@ -18,6 +17,9 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		ethernet0 = &fec;
 		gpio0 = &gpio1;
diff --git a/arch/arm/boot/dts/imx28-m28.dtsi b/arch/arm/boot/dts/imx28-m28.dtsi
index 214bb1506b53..a69856e41ba4 100644
--- a/arch/arm/boot/dts/imx28-m28.dtsi
+++ b/arch/arm/boot/dts/imx28-m28.dtsi
@@ -12,8 +12,8 @@
 #include "imx28.dtsi"
 
 / {
-	model = "DENX M28";
-	compatible = "denx,m28", "fsl,imx28";
+	model = "Aries/DENX M28";
+	compatible = "aries,m28", "denx,m28", "fsl,imx28";
 
 	memory {
 		reg = <0x40000000 0x08000000>;
diff --git a/arch/arm/boot/dts/imx28-m28evk.dts b/arch/arm/boot/dts/imx28-m28evk.dts
index 8d04e57039bc..dbfb8aab505f 100644
--- a/arch/arm/boot/dts/imx28-m28evk.dts
+++ b/arch/arm/boot/dts/imx28-m28evk.dts
@@ -13,8 +13,8 @@
 #include "imx28-m28.dtsi"
 
 / {
-	model = "DENX M28EVK";
-	compatible = "denx,m28evk", "fsl,imx28";
+	model = "Aries/DENX M28EVK";
+	compatible = "aries,m28evk", "denx,m28evk", "fsl,imx28";
 
 	apb@80000000 {
 		apbh@80000000 {
diff --git a/arch/arm/boot/dts/imx28.dtsi b/arch/arm/boot/dts/imx28.dtsi
index 0ad893bf5f43..3aabf65a6a52 100644
--- a/arch/arm/boot/dts/imx28.dtsi
+++ b/arch/arm/boot/dts/imx28.dtsi
@@ -10,10 +10,12 @@
  */
 
 #include <dt-bindings/gpio/gpio.h>
-#include "skeleton.dtsi"
 #include "imx28-pinfunc.h"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	interrupt-parent = <&icoll>;
 
 	aliases {
diff --git a/arch/arm/boot/dts/imx31.dtsi b/arch/arm/boot/dts/imx31.dtsi
index 1ce7ae94e7ad..685916e3d8a1 100644
--- a/arch/arm/boot/dts/imx31.dtsi
+++ b/arch/arm/boot/dts/imx31.dtsi
@@ -9,9 +9,10 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
-#include "skeleton.dtsi"
-
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		serial0 = &uart1;
 		serial1 = &uart2;
@@ -118,13 +119,6 @@
 				interrupts = <19>;
 				clocks = <&clks 25>;
 			};
-
-			clks: ccm@53f80000{
-				compatible = "fsl,imx31-ccm";
-				reg = <0x53f80000 0x4000>;
-				interrupts = <0 31 0x04 0 53 0x04>;
-				#clock-cells = <1>;
-			};
 		};
 
 		aips@53f00000 { /* AIPS2 */
@@ -134,6 +128,13 @@
 			reg = <0x53f00000 0x100000>;
 			ranges;
 
+			clks: ccm@53f80000{
+				compatible = "fsl,imx31-ccm";
+				reg = <0x53f80000 0x4000>;
+				interrupts = <31>, <53>;
+				#clock-cells = <1>;
+			};
+
 			gpt: timer@53f90000 {
 				compatible = "fsl,imx31-gpt";
 				reg = <0x53f90000 0x4000>;
diff --git a/arch/arm/boot/dts/imx35.dtsi b/arch/arm/boot/dts/imx35.dtsi
index f812d586c5ce..9f40e6229189 100644
--- a/arch/arm/boot/dts/imx35.dtsi
+++ b/arch/arm/boot/dts/imx35.dtsi
@@ -8,10 +8,12 @@
  * Free Software Foundation.
  */
 
-#include "skeleton.dtsi"
 #include "imx35-pinfunc.h"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		ethernet0 = &fec;
 		gpio0 = &gpio1;
diff --git a/arch/arm/boot/dts/imx50.dtsi b/arch/arm/boot/dts/imx50.dtsi
index 8fe8beeb68a4..fe0221e4cbf7 100644
--- a/arch/arm/boot/dts/imx50.dtsi
+++ b/arch/arm/boot/dts/imx50.dtsi
@@ -11,11 +11,13 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
-#include "skeleton.dtsi"
 #include "imx50-pinfunc.h"
 #include <dt-bindings/clock/imx5-clock.h>
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		ethernet0 = &fec;
 		gpio0 = &gpio1;
@@ -103,8 +105,8 @@
 					reg = <0x50004000 0x4000>;
 					interrupts = <1>;
 					clocks = <&clks IMX5_CLK_ESDHC1_IPG_GATE>,
-					         <&clks IMX5_CLK_DUMMY>,
-					         <&clks IMX5_CLK_ESDHC1_PER_GATE>;
+						 <&clks IMX5_CLK_DUMMY>,
+						 <&clks IMX5_CLK_ESDHC1_PER_GATE>;
 					clock-names = "ipg", "ahb", "per";
 					bus-width = <4>;
 					status = "disabled";
@@ -115,8 +117,8 @@
 					reg = <0x50008000 0x4000>;
 					interrupts = <2>;
 					clocks = <&clks IMX5_CLK_ESDHC2_IPG_GATE>,
-					         <&clks IMX5_CLK_DUMMY>,
-					         <&clks IMX5_CLK_ESDHC2_PER_GATE>;
+						 <&clks IMX5_CLK_DUMMY>,
+						 <&clks IMX5_CLK_ESDHC2_PER_GATE>;
 					clock-names = "ipg", "ahb", "per";
 					bus-width = <4>;
 					status = "disabled";
@@ -127,7 +129,7 @@
 					reg = <0x5000c000 0x4000>;
 					interrupts = <33>;
 					clocks = <&clks IMX5_CLK_UART3_IPG_GATE>,
-					         <&clks IMX5_CLK_UART3_PER_GATE>;
+						 <&clks IMX5_CLK_UART3_PER_GATE>;
 					clock-names = "ipg", "per";
 					status = "disabled";
 				};
@@ -139,7 +141,7 @@
 					reg = <0x50010000 0x4000>;
 					interrupts = <36>;
 					clocks = <&clks IMX5_CLK_ECSPI1_IPG_GATE>,
-					         <&clks IMX5_CLK_ECSPI1_PER_GATE>;
+						 <&clks IMX5_CLK_ECSPI1_PER_GATE>;
 					clock-names = "ipg", "per";
 					status = "disabled";
 				};
@@ -164,8 +166,8 @@
 					reg = <0x50020000 0x4000>;
 					interrupts = <3>;
 					clocks = <&clks IMX5_CLK_ESDHC3_IPG_GATE>,
-					         <&clks IMX5_CLK_DUMMY>,
-					         <&clks IMX5_CLK_ESDHC3_PER_GATE>;
+						 <&clks IMX5_CLK_DUMMY>,
+						 <&clks IMX5_CLK_ESDHC3_PER_GATE>;
 					clock-names = "ipg", "ahb", "per";
 					bus-width = <4>;
 					status = "disabled";
@@ -176,8 +178,8 @@
 					reg = <0x50024000 0x4000>;
 					interrupts = <4>;
 					clocks = <&clks IMX5_CLK_ESDHC4_IPG_GATE>,
-					         <&clks IMX5_CLK_DUMMY>,
-					         <&clks IMX5_CLK_ESDHC4_PER_GATE>;
+						 <&clks IMX5_CLK_DUMMY>,
+						 <&clks IMX5_CLK_ESDHC4_PER_GATE>;
 					clock-names = "ipg", "ahb", "per";
 					bus-width = <4>;
 					status = "disabled";
@@ -279,7 +281,7 @@
 				reg = <0x53fa0000 0x4000>;
 				interrupts = <39>;
 				clocks = <&clks IMX5_CLK_GPT_IPG_GATE>,
-				         <&clks IMX5_CLK_GPT_HF_GATE>;
+					 <&clks IMX5_CLK_GPT_HF_GATE>;
 				clock-names = "ipg", "per";
 			};
 
@@ -298,7 +300,7 @@
 				compatible = "fsl,imx50-pwm", "fsl,imx27-pwm";
 				reg = <0x53fb4000 0x4000>;
 				clocks = <&clks IMX5_CLK_PWM1_IPG_GATE>,
-				         <&clks IMX5_CLK_PWM1_HF_GATE>;
+					 <&clks IMX5_CLK_PWM1_HF_GATE>;
 				clock-names = "ipg", "per";
 				interrupts = <61>;
 			};
@@ -308,7 +310,7 @@
 				compatible = "fsl,imx50-pwm", "fsl,imx27-pwm";
 				reg = <0x53fb8000 0x4000>;
 				clocks = <&clks IMX5_CLK_PWM2_IPG_GATE>,
-				         <&clks IMX5_CLK_PWM2_HF_GATE>;
+					 <&clks IMX5_CLK_PWM2_HF_GATE>;
 				clock-names = "ipg", "per";
 				interrupts = <94>;
 			};
@@ -318,7 +320,7 @@
 				reg = <0x53fbc000 0x4000>;
 				interrupts = <31>;
 				clocks = <&clks IMX5_CLK_UART1_IPG_GATE>,
-				         <&clks IMX5_CLK_UART1_PER_GATE>;
+					 <&clks IMX5_CLK_UART1_PER_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -328,7 +330,7 @@
 				reg = <0x53fc0000 0x4000>;
 				interrupts = <32>;
 				clocks = <&clks IMX5_CLK_UART2_IPG_GATE>,
-				         <&clks IMX5_CLK_UART2_PER_GATE>;
+					 <&clks IMX5_CLK_UART2_PER_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -383,7 +385,7 @@
 				reg = <0x53ff0000 0x4000>;
 				interrupts = <13>;
 				clocks = <&clks IMX5_CLK_UART4_IPG_GATE>,
-				         <&clks IMX5_CLK_UART4_PER_GATE>;
+					 <&clks IMX5_CLK_UART4_PER_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -401,7 +403,7 @@
 				reg = <0x63f90000 0x4000>;
 				interrupts = <86>;
 				clocks = <&clks IMX5_CLK_UART5_IPG_GATE>,
-				         <&clks IMX5_CLK_UART5_PER_GATE>;
+					 <&clks IMX5_CLK_UART5_PER_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -420,7 +422,7 @@
 				reg = <0x63fac000 0x4000>;
 				interrupts = <37>;
 				clocks = <&clks IMX5_CLK_ECSPI2_IPG_GATE>,
-				         <&clks IMX5_CLK_ECSPI2_PER_GATE>;
+					 <&clks IMX5_CLK_ECSPI2_PER_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -430,7 +432,7 @@
 				reg = <0x63fb0000 0x4000>;
 				interrupts = <6>;
 				clocks = <&clks IMX5_CLK_SDMA_GATE>,
-				         <&clks IMX5_CLK_SDMA_GATE>;
+					 <&clks IMX5_CLK_SDMA_GATE>;
 				clock-names = "ipg", "ahb";
 				fsl,sdma-ram-script-name = "imx/sdma/sdma-imx50.bin";
 			};
@@ -442,7 +444,7 @@
 				reg = <0x63fc0000 0x4000>;
 				interrupts = <38>;
 				clocks = <&clks IMX5_CLK_CSPI_IPG_GATE>,
-				         <&clks IMX5_CLK_CSPI_IPG_GATE>;
+					 <&clks IMX5_CLK_CSPI_IPG_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -492,8 +494,8 @@
 				reg = <0x63fec000 0x4000>;
 				interrupts = <87>;
 				clocks = <&clks IMX5_CLK_FEC_GATE>,
-				         <&clks IMX5_CLK_FEC_GATE>,
-				         <&clks IMX5_CLK_FEC_GATE>;
+					 <&clks IMX5_CLK_FEC_GATE>,
+					 <&clks IMX5_CLK_FEC_GATE>;
 				clock-names = "ipg", "ahb", "ptp";
 				status = "disabled";
 			};
diff --git a/arch/arm/boot/dts/imx51.dtsi b/arch/arm/boot/dts/imx51.dtsi
index f46fe9bf0bcb..33526cade735 100644
--- a/arch/arm/boot/dts/imx51.dtsi
+++ b/arch/arm/boot/dts/imx51.dtsi
@@ -10,7 +10,6 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
-#include "skeleton.dtsi"
 #include "imx51-pinfunc.h"
 #include <dt-bindings/clock/imx5-clock.h>
 #include <dt-bindings/gpio/gpio.h>
@@ -18,6 +17,9 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		ethernet0 = &fec;
 		gpio0 = &gpio1;
@@ -130,8 +132,8 @@
 			reg = <0x40000000 0x20000000>;
 			interrupts = <11 10>;
 			clocks = <&clks IMX5_CLK_IPU_GATE>,
-			         <&clks IMX5_CLK_IPU_DI0_GATE>,
-			         <&clks IMX5_CLK_IPU_DI1_GATE>;
+				 <&clks IMX5_CLK_IPU_DI0_GATE>,
+				 <&clks IMX5_CLK_IPU_DI1_GATE>;
 			clock-names = "bus", "di0", "di1";
 			resets = <&src 2>;
 
@@ -169,8 +171,8 @@
 					reg = <0x70004000 0x4000>;
 					interrupts = <1>;
 					clocks = <&clks IMX5_CLK_ESDHC1_IPG_GATE>,
-					         <&clks IMX5_CLK_DUMMY>,
-					         <&clks IMX5_CLK_ESDHC1_PER_GATE>;
+						 <&clks IMX5_CLK_DUMMY>,
+						 <&clks IMX5_CLK_ESDHC1_PER_GATE>;
 					clock-names = "ipg", "ahb", "per";
 					status = "disabled";
 				};
@@ -180,8 +182,8 @@
 					reg = <0x70008000 0x4000>;
 					interrupts = <2>;
 					clocks = <&clks IMX5_CLK_ESDHC2_IPG_GATE>,
-					         <&clks IMX5_CLK_DUMMY>,
-					         <&clks IMX5_CLK_ESDHC2_PER_GATE>;
+						 <&clks IMX5_CLK_DUMMY>,
+						 <&clks IMX5_CLK_ESDHC2_PER_GATE>;
 					clock-names = "ipg", "ahb", "per";
 					bus-width = <4>;
 					status = "disabled";
@@ -192,7 +194,7 @@
 					reg = <0x7000c000 0x4000>;
 					interrupts = <33>;
 					clocks = <&clks IMX5_CLK_UART3_IPG_GATE>,
-					         <&clks IMX5_CLK_UART3_PER_GATE>;
+						 <&clks IMX5_CLK_UART3_PER_GATE>;
 					clock-names = "ipg", "per";
 					status = "disabled";
 				};
@@ -204,7 +206,7 @@
 					reg = <0x70010000 0x4000>;
 					interrupts = <36>;
 					clocks = <&clks IMX5_CLK_ECSPI1_IPG_GATE>,
-					         <&clks IMX5_CLK_ECSPI1_PER_GATE>;
+						 <&clks IMX5_CLK_ECSPI1_PER_GATE>;
 					clock-names = "ipg", "per";
 					status = "disabled";
 				};
@@ -229,8 +231,8 @@
 					reg = <0x70020000 0x4000>;
 					interrupts = <3>;
 					clocks = <&clks IMX5_CLK_ESDHC3_IPG_GATE>,
-					         <&clks IMX5_CLK_DUMMY>,
-					         <&clks IMX5_CLK_ESDHC3_PER_GATE>;
+						 <&clks IMX5_CLK_DUMMY>,
+						 <&clks IMX5_CLK_ESDHC3_PER_GATE>;
 					clock-names = "ipg", "ahb", "per";
 					bus-width = <4>;
 					status = "disabled";
@@ -241,8 +243,8 @@
 					reg = <0x70024000 0x4000>;
 					interrupts = <4>;
 					clocks = <&clks IMX5_CLK_ESDHC4_IPG_GATE>,
-					         <&clks IMX5_CLK_DUMMY>,
-					         <&clks IMX5_CLK_ESDHC4_PER_GATE>;
+						 <&clks IMX5_CLK_DUMMY>,
+						 <&clks IMX5_CLK_ESDHC4_PER_GATE>;
 					clock-names = "ipg", "ahb", "per";
 					bus-width = <4>;
 					status = "disabled";
@@ -364,7 +366,7 @@
 				reg = <0x73fa0000 0x4000>;
 				interrupts = <39>;
 				clocks = <&clks IMX5_CLK_GPT_IPG_GATE>,
-				         <&clks IMX5_CLK_GPT_HF_GATE>;
+					 <&clks IMX5_CLK_GPT_HF_GATE>;
 				clock-names = "ipg", "per";
 			};
 
@@ -378,7 +380,7 @@
 				compatible = "fsl,imx51-pwm", "fsl,imx27-pwm";
 				reg = <0x73fb4000 0x4000>;
 				clocks = <&clks IMX5_CLK_PWM1_IPG_GATE>,
-				         <&clks IMX5_CLK_PWM1_HF_GATE>;
+					 <&clks IMX5_CLK_PWM1_HF_GATE>;
 				clock-names = "ipg", "per";
 				interrupts = <61>;
 			};
@@ -388,7 +390,7 @@
 				compatible = "fsl,imx51-pwm", "fsl,imx27-pwm";
 				reg = <0x73fb8000 0x4000>;
 				clocks = <&clks IMX5_CLK_PWM2_IPG_GATE>,
-				         <&clks IMX5_CLK_PWM2_HF_GATE>;
+					 <&clks IMX5_CLK_PWM2_HF_GATE>;
 				clock-names = "ipg", "per";
 				interrupts = <94>;
 			};
@@ -398,7 +400,7 @@
 				reg = <0x73fbc000 0x4000>;
 				interrupts = <31>;
 				clocks = <&clks IMX5_CLK_UART1_IPG_GATE>,
-				         <&clks IMX5_CLK_UART1_PER_GATE>;
+					 <&clks IMX5_CLK_UART1_PER_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -408,7 +410,7 @@
 				reg = <0x73fc0000 0x4000>;
 				interrupts = <32>;
 				clocks = <&clks IMX5_CLK_UART2_IPG_GATE>,
-				         <&clks IMX5_CLK_UART2_PER_GATE>;
+					 <&clks IMX5_CLK_UART2_PER_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -456,7 +458,7 @@
 				reg = <0x83fac000 0x4000>;
 				interrupts = <37>;
 				clocks = <&clks IMX5_CLK_ECSPI2_IPG_GATE>,
-				         <&clks IMX5_CLK_ECSPI2_PER_GATE>;
+					 <&clks IMX5_CLK_ECSPI2_PER_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -466,7 +468,7 @@
 				reg = <0x83fb0000 0x4000>;
 				interrupts = <6>;
 				clocks = <&clks IMX5_CLK_SDMA_GATE>,
-				         <&clks IMX5_CLK_SDMA_GATE>;
+					 <&clks IMX5_CLK_SDMA_GATE>;
 				clock-names = "ipg", "ahb";
 				#dma-cells = <3>;
 				fsl,sdma-ram-script-name = "imx/sdma/sdma-imx51.bin";
@@ -479,7 +481,7 @@
 				reg = <0x83fc0000 0x4000>;
 				interrupts = <38>;
 				clocks = <&clks IMX5_CLK_CSPI_IPG_GATE>,
-				         <&clks IMX5_CLK_CSPI_IPG_GATE>;
+					 <&clks IMX5_CLK_CSPI_IPG_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -582,8 +584,8 @@
 				reg = <0x83fec000 0x4000>;
 				interrupts = <87>;
 				clocks = <&clks IMX5_CLK_FEC_GATE>,
-				         <&clks IMX5_CLK_FEC_GATE>,
-				         <&clks IMX5_CLK_FEC_GATE>;
+					 <&clks IMX5_CLK_FEC_GATE>,
+					 <&clks IMX5_CLK_FEC_GATE>;
 				clock-names = "ipg", "ahb", "ptp";
 				status = "disabled";
 			};
diff --git a/arch/arm/boot/dts/imx53-m53.dtsi b/arch/arm/boot/dts/imx53-m53.dtsi
index d259f57bfd98..ec390aa562c3 100644
--- a/arch/arm/boot/dts/imx53-m53.dtsi
+++ b/arch/arm/boot/dts/imx53-m53.dtsi
@@ -12,8 +12,8 @@
 #include "imx53.dtsi"
 
 / {
-	model = "DENX M53";
-	compatible = "denx,imx53-m53", "fsl,imx53";
+	model = "Aries/DENX M53";
+	compatible = "aries,imx53-m53", "denx,imx53-m53", "fsl,imx53";
 
 	memory {
 		reg = <0x70000000 0x20000000>,
diff --git a/arch/arm/boot/dts/imx53-m53evk.dts b/arch/arm/boot/dts/imx53-m53evk.dts
index dcee1e0f968f..4347a321c782 100644
--- a/arch/arm/boot/dts/imx53-m53evk.dts
+++ b/arch/arm/boot/dts/imx53-m53evk.dts
@@ -13,8 +13,8 @@
 #include "imx53-m53.dtsi"
 
 / {
-	model = "DENX M53EVK";
-	compatible = "denx,imx53-m53evk", "fsl,imx53";
+	model = "Aries/DENX M53EVK";
+	compatible = "aries,imx53-m53evk", "denx,imx53-m53evk", "fsl,imx53";
 
 	display1: display@di1 {
 		compatible = "fsl,imx-parallel-display";
diff --git a/arch/arm/boot/dts/imx53.dtsi b/arch/arm/boot/dts/imx53.dtsi
index 0777b41cdfe8..ca51dc03e327 100644
--- a/arch/arm/boot/dts/imx53.dtsi
+++ b/arch/arm/boot/dts/imx53.dtsi
@@ -10,7 +10,6 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
-#include "skeleton.dtsi"
 #include "imx53-pinfunc.h"
 #include <dt-bindings/clock/imx5-clock.h>
 #include <dt-bindings/gpio/gpio.h>
@@ -18,6 +17,9 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		ethernet0 = &fec;
 		gpio0 = &gpio1;
@@ -131,8 +133,8 @@
 			reg = <0x18000000 0x08000000>;
 			interrupts = <11 10>;
 			clocks = <&clks IMX5_CLK_IPU_GATE>,
-			         <&clks IMX5_CLK_IPU_DI0_GATE>,
-			         <&clks IMX5_CLK_IPU_DI1_GATE>;
+				 <&clks IMX5_CLK_IPU_DI0_GATE>,
+				 <&clks IMX5_CLK_IPU_DI1_GATE>;
 			clock-names = "bus", "di0", "di1";
 			resets = <&src 2>;
 
@@ -199,8 +201,8 @@
 					reg = <0x50004000 0x4000>;
 					interrupts = <1>;
 					clocks = <&clks IMX5_CLK_ESDHC1_IPG_GATE>,
-					         <&clks IMX5_CLK_DUMMY>,
-					         <&clks IMX5_CLK_ESDHC1_PER_GATE>;
+						 <&clks IMX5_CLK_DUMMY>,
+						 <&clks IMX5_CLK_ESDHC1_PER_GATE>;
 					clock-names = "ipg", "ahb", "per";
 					bus-width = <4>;
 					status = "disabled";
@@ -211,8 +213,8 @@
 					reg = <0x50008000 0x4000>;
 					interrupts = <2>;
 					clocks = <&clks IMX5_CLK_ESDHC2_IPG_GATE>,
-					         <&clks IMX5_CLK_DUMMY>,
-					         <&clks IMX5_CLK_ESDHC2_PER_GATE>;
+						 <&clks IMX5_CLK_DUMMY>,
+						 <&clks IMX5_CLK_ESDHC2_PER_GATE>;
 					clock-names = "ipg", "ahb", "per";
 					bus-width = <4>;
 					status = "disabled";
@@ -223,7 +225,7 @@
 					reg = <0x5000c000 0x4000>;
 					interrupts = <33>;
 					clocks = <&clks IMX5_CLK_UART3_IPG_GATE>,
-					         <&clks IMX5_CLK_UART3_PER_GATE>;
+						 <&clks IMX5_CLK_UART3_PER_GATE>;
 					clock-names = "ipg", "per";
 					dmas = <&sdma 42 4 0>, <&sdma 43 4 0>;
 					dma-names = "rx", "tx";
@@ -237,7 +239,7 @@
 					reg = <0x50010000 0x4000>;
 					interrupts = <36>;
 					clocks = <&clks IMX5_CLK_ECSPI1_IPG_GATE>,
-					         <&clks IMX5_CLK_ECSPI1_PER_GATE>;
+						 <&clks IMX5_CLK_ECSPI1_PER_GATE>;
 					clock-names = "ipg", "per";
 					status = "disabled";
 				};
@@ -264,8 +266,8 @@
 					reg = <0x50020000 0x4000>;
 					interrupts = <3>;
 					clocks = <&clks IMX5_CLK_ESDHC3_IPG_GATE>,
-					         <&clks IMX5_CLK_DUMMY>,
-					         <&clks IMX5_CLK_ESDHC3_PER_GATE>;
+						 <&clks IMX5_CLK_DUMMY>,
+						 <&clks IMX5_CLK_ESDHC3_PER_GATE>;
 					clock-names = "ipg", "ahb", "per";
 					bus-width = <4>;
 					status = "disabled";
@@ -276,8 +278,8 @@
 					reg = <0x50024000 0x4000>;
 					interrupts = <4>;
 					clocks = <&clks IMX5_CLK_ESDHC4_IPG_GATE>,
-					         <&clks IMX5_CLK_DUMMY>,
-					         <&clks IMX5_CLK_ESDHC4_PER_GATE>;
+						 <&clks IMX5_CLK_DUMMY>,
+						 <&clks IMX5_CLK_ESDHC4_PER_GATE>;
 					clock-names = "ipg", "ahb", "per";
 					bus-width = <4>;
 					status = "disabled";
@@ -419,7 +421,7 @@
 				reg = <0x53fa0000 0x4000>;
 				interrupts = <39>;
 				clocks = <&clks IMX5_CLK_GPT_IPG_GATE>,
-				         <&clks IMX5_CLK_GPT_HF_GATE>;
+					 <&clks IMX5_CLK_GPT_HF_GATE>;
 				clock-names = "ipg", "per";
 			};
 
@@ -440,11 +442,11 @@
 				reg = <0x53fa8008 0x4>;
 				gpr = <&gpr>;
 				clocks = <&clks IMX5_CLK_LDB_DI0_SEL>,
-				         <&clks IMX5_CLK_LDB_DI1_SEL>,
-				         <&clks IMX5_CLK_IPU_DI0_SEL>,
-				         <&clks IMX5_CLK_IPU_DI1_SEL>,
-				         <&clks IMX5_CLK_LDB_DI0_GATE>,
-				         <&clks IMX5_CLK_LDB_DI1_GATE>;
+					 <&clks IMX5_CLK_LDB_DI1_SEL>,
+					 <&clks IMX5_CLK_IPU_DI0_SEL>,
+					 <&clks IMX5_CLK_IPU_DI1_SEL>,
+					 <&clks IMX5_CLK_LDB_DI0_GATE>,
+					 <&clks IMX5_CLK_LDB_DI1_GATE>;
 				clock-names = "di0_pll", "di1_pll",
 					      "di0_sel", "di1_sel",
 					      "di0", "di1";
@@ -486,7 +488,7 @@
 				compatible = "fsl,imx53-pwm", "fsl,imx27-pwm";
 				reg = <0x53fb4000 0x4000>;
 				clocks = <&clks IMX5_CLK_PWM1_IPG_GATE>,
-				         <&clks IMX5_CLK_PWM1_HF_GATE>;
+					 <&clks IMX5_CLK_PWM1_HF_GATE>;
 				clock-names = "ipg", "per";
 				interrupts = <61>;
 			};
@@ -496,7 +498,7 @@
 				compatible = "fsl,imx53-pwm", "fsl,imx27-pwm";
 				reg = <0x53fb8000 0x4000>;
 				clocks = <&clks IMX5_CLK_PWM2_IPG_GATE>,
-				         <&clks IMX5_CLK_PWM2_HF_GATE>;
+					 <&clks IMX5_CLK_PWM2_HF_GATE>;
 				clock-names = "ipg", "per";
 				interrupts = <94>;
 			};
@@ -506,7 +508,7 @@
 				reg = <0x53fbc000 0x4000>;
 				interrupts = <31>;
 				clocks = <&clks IMX5_CLK_UART1_IPG_GATE>,
-				         <&clks IMX5_CLK_UART1_PER_GATE>;
+					 <&clks IMX5_CLK_UART1_PER_GATE>;
 				clock-names = "ipg", "per";
 				dmas = <&sdma 18 4 0>, <&sdma 19 4 0>;
 				dma-names = "rx", "tx";
@@ -518,7 +520,7 @@
 				reg = <0x53fc0000 0x4000>;
 				interrupts = <32>;
 				clocks = <&clks IMX5_CLK_UART2_IPG_GATE>,
-				         <&clks IMX5_CLK_UART2_PER_GATE>;
+					 <&clks IMX5_CLK_UART2_PER_GATE>;
 				clock-names = "ipg", "per";
 				dmas = <&sdma 12 4 0>, <&sdma 13 4 0>;
 				dma-names = "rx", "tx";
@@ -530,7 +532,7 @@
 				reg = <0x53fc8000 0x4000>;
 				interrupts = <82>;
 				clocks = <&clks IMX5_CLK_CAN1_IPG_GATE>,
-				         <&clks IMX5_CLK_CAN1_SERIAL_GATE>;
+					 <&clks IMX5_CLK_CAN1_SERIAL_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -540,7 +542,7 @@
 				reg = <0x53fcc000 0x4000>;
 				interrupts = <83>;
 				clocks = <&clks IMX5_CLK_CAN2_IPG_GATE>,
-				         <&clks IMX5_CLK_CAN2_SERIAL_GATE>;
+					 <&clks IMX5_CLK_CAN2_SERIAL_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -603,7 +605,7 @@
 				reg = <0x53ff0000 0x4000>;
 				interrupts = <13>;
 				clocks = <&clks IMX5_CLK_UART4_IPG_GATE>,
-				         <&clks IMX5_CLK_UART4_PER_GATE>;
+					 <&clks IMX5_CLK_UART4_PER_GATE>;
 				clock-names = "ipg", "per";
 				dmas = <&sdma 2 4 0>, <&sdma 3 4 0>;
 				dma-names = "rx", "tx";
@@ -635,7 +637,7 @@
 				reg = <0x63f90000 0x4000>;
 				interrupts = <86>;
 				clocks = <&clks IMX5_CLK_UART5_IPG_GATE>,
-				         <&clks IMX5_CLK_UART5_PER_GATE>;
+					 <&clks IMX5_CLK_UART5_PER_GATE>;
 				clock-names = "ipg", "per";
 				dmas = <&sdma 16 4 0>, <&sdma 17 4 0>;
 				dma-names = "rx", "tx";
@@ -656,7 +658,7 @@
 				reg = <0x63fac000 0x4000>;
 				interrupts = <37>;
 				clocks = <&clks IMX5_CLK_ECSPI2_IPG_GATE>,
-				         <&clks IMX5_CLK_ECSPI2_PER_GATE>;
+					 <&clks IMX5_CLK_ECSPI2_PER_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -666,7 +668,7 @@
 				reg = <0x63fb0000 0x4000>;
 				interrupts = <6>;
 				clocks = <&clks IMX5_CLK_SDMA_GATE>,
-				         <&clks IMX5_CLK_SDMA_GATE>;
+					 <&clks IMX5_CLK_SDMA_GATE>;
 				clock-names = "ipg", "ahb";
 				#dma-cells = <3>;
 				fsl,sdma-ram-script-name = "imx/sdma/sdma-imx53.bin";
@@ -679,7 +681,7 @@
 				reg = <0x63fc0000 0x4000>;
 				interrupts = <38>;
 				clocks = <&clks IMX5_CLK_CSPI_IPG_GATE>,
-				         <&clks IMX5_CLK_CSPI_IPG_GATE>;
+					 <&clks IMX5_CLK_CSPI_IPG_GATE>;
 				clock-names = "ipg", "per";
 				status = "disabled";
 			};
@@ -755,8 +757,8 @@
 				reg = <0x63fec000 0x4000>;
 				interrupts = <87>;
 				clocks = <&clks IMX5_CLK_FEC_GATE>,
-				         <&clks IMX5_CLK_FEC_GATE>,
-				         <&clks IMX5_CLK_FEC_GATE>;
+					 <&clks IMX5_CLK_FEC_GATE>,
+					 <&clks IMX5_CLK_FEC_GATE>;
 				clock-names = "ipg", "ahb", "ptp";
 				status = "disabled";
 			};
@@ -766,7 +768,7 @@
 				reg = <0x63ff0000 0x1000>;
 				interrupts = <92>;
 				clocks = <&clks IMX5_CLK_TVE_GATE>,
-				         <&clks IMX5_CLK_IPU_DI1_SEL>;
+					 <&clks IMX5_CLK_IPU_DI1_SEL>;
 				clock-names = "tve", "di_sel";
 				status = "disabled";
 
@@ -782,7 +784,7 @@
 				reg = <0x63ff4000 0x1000>;
 				interrupts = <9>;
 				clocks = <&clks IMX5_CLK_VPU_REFERENCE_GATE>,
-				         <&clks IMX5_CLK_VPU_GATE>;
+					 <&clks IMX5_CLK_VPU_GATE>;
 				clock-names = "per", "ahb";
 				resets = <&src 1>;
 				iram = <&ocram>;
@@ -793,7 +795,7 @@
 				reg = <0x63ff8000 0x4000>;
 				interrupts = <19 20>;
 				clocks = <&clks IMX5_CLK_SAHARA_IPG_GATE>,
-				         <&clks IMX5_CLK_SAHARA_IPG_GATE>;
+					 <&clks IMX5_CLK_SAHARA_IPG_GATE>;
 				clock-names = "ipg", "ahb";
 			};
 		};
diff --git a/arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts b/arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts
new file mode 100644
index 000000000000..e0c21727866d
--- /dev/null
+++ b/arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts
@@ -0,0 +1,253 @@
+/*
+ * Copyright 2014-2016 Toradex AG
+ * Copyright 2012 Freescale Semiconductor, Inc.
+ * Copyright 2011 Linaro Ltd.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include "imx6dl.dtsi"
+#include "imx6qdl-colibri.dtsi"
+
+/ {
+	model = "Toradex Colibri iMX6DL/S on Colibri Evaluation Board V3";
+	compatible = "toradex,colibri_imx6dl-eval-v3", "toradex,colibri_imx6dl",
+		     "fsl,imx6dl";
+
+	aliases {
+		i2c0 = &i2c2;
+		i2c1 = &i2c3;
+	};
+
+	aliases {
+		rtc0 = &rtc_i2c;
+		rtc1 = &snvs_rtc;
+	};
+
+	clocks {
+		/* Fixed crystal dedicated to mcp251x */
+		clk16m: clk@1 {
+			compatible = "fixed-clock";
+			reg = <1>;
+			#clock-cells = <0>;
+			clock-frequency = <16000000>;
+			clock-output-names = "clk16m";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_keys>;
+
+		wakeup {
+			label = "Wake-Up";
+			gpios = <&gpio2 22 GPIO_ACTIVE_HIGH>; /* SODIMM 45 */
+			linux,code = <KEY_WAKEUP>;
+			debounce-interval = <10>;
+			wakeup-source;
+		};
+	};
+
+	lcd_display: display@di0 {
+		compatible = "fsl,imx-parallel-display";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		interface-pix-fmt = "bgr666";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_ipu1_lcdif>;
+		status = "okay";
+
+		port@0 {
+			reg = <0>;
+
+			lcd_display_in: endpoint {
+				remote-endpoint = <&ipu1_di0_disp0>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+
+			lcd_display_out: endpoint {
+				remote-endpoint = <&lcd_panel_in>;
+			};
+		};
+	};
+
+	panel: panel {
+		/*
+		 * edt,et057090dhu: EDT 5.7" LCD TFT
+		 * edt,et070080dh6: EDT 7.0" LCD TFT
+		 */
+		compatible = "edt,et057090dhu";
+		backlight = <&backlight>;
+
+		port {
+			lcd_panel_in: endpoint {
+				remote-endpoint = <&lcd_display_out>;
+			};
+		};
+	};
+};
+
+&backlight {
+	brightness-levels = <0 127 191 223 239 247 251 255>;
+	default-brightness-level = <1>;
+	status = "okay";
+};
+
+/* Colibri SSP */
+&ecspi4 {
+	status = "okay";
+
+	mcp251x0: mcp251x@1 {
+		compatible = "microchip,mcp2515";
+		reg = <0>;
+		clocks = <&clk16m>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <27 0x2>;
+		spi-max-frequency = <10000000>;
+		status = "okay";
+	};
+};
+
+&hdmi {
+	status = "okay";
+};
+
+/*
+ * Colibri I2C: I2C3_SDA/SCL on SODIMM 194/196 (e.g. RTC on carrier board)
+ */
+&i2c3 {
+	status = "okay";
+
+	/* M41T0M6 real time clock on carrier board */
+	rtc_i2c: rtc@68 {
+		compatible = "st,m41t00";
+		reg = <0x68>;
+	};
+};
+
+&ipu1_di0_disp0 {
+	remote-endpoint = <&lcd_display_in>;
+};
+
+&pwm1 {
+	status = "okay";
+};
+
+&pwm2 {
+	status = "okay";
+};
+
+&pwm3 {
+	status = "okay";
+};
+
+&pwm4 {
+	status = "okay";
+};
+
+&reg_usb_host_vbus {
+	status = "okay";
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&uart3 {
+	status = "okay";
+};
+
+&usbh1 {
+	vbus-supply = <&reg_usb_host_vbus>;
+	status = "okay";
+};
+
+&usbotg {
+	status = "okay";
+};
+
+/* Colibri MMC */
+&usdhc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_mmc_cd>;
+	cd-gpios = <&gpio2 5 GPIO_ACTIVE_LOW>; /* MMCD */
+	status = "okay";
+};
+
+&weim {
+	status = "okay";
+
+	/* weim memory map: 32MB on CS0, 32MB on CS1, 32MB on CS2 */
+	ranges = <0 0 0x08000000 0x02000000
+		  1 0 0x0a000000 0x02000000
+		  2 0 0x0c000000 0x02000000>;
+
+	/* SRAM on Colibri nEXT_CS0 */
+	sram@0,0 {
+		compatible = "cypress,cy7c1019dv33-10zsxi, mtd-ram";
+		reg = <0 0 0x00010000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		bank-width = <2>;
+		fsl,weim-cs-timing = <0x00010081 0x00000000 0x04000000
+				      0x00000000 0x04000040 0x00000000>;
+	};
+
+	/* SRAM on Colibri nEXT_CS1 */
+	sram@1,0 {
+		compatible = "cypress,cy7c1019dv33-10zsxi, mtd-ram";
+		reg = <1 0 0x00010000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		bank-width = <2>;
+		fsl,weim-cs-timing = <0x00010081 0x00000000 0x04000000
+				      0x00000000 0x04000040 0x00000000>;
+	};
+};
diff --git a/arch/arm/boot/dts/imx6dl-icore.dts b/arch/arm/boot/dts/imx6dl-icore.dts
new file mode 100644
index 000000000000..aec332c14af1
--- /dev/null
+++ b/arch/arm/boot/dts/imx6dl-icore.dts
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2016 Amarula Solutions B.V.
+ * Copyright (C) 2016 Engicam S.r.l.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "imx6dl.dtsi"
+#include "imx6qdl-icore.dtsi"
+
+/ {
+	model = "Engicam i.CoreM6 DualLite/Solo Starter Kit";
+	compatible = "engicam,imx6-icore", "fsl,imx6dl";
+};
+
+&can1 {
+	status = "okay";
+};
+
+&can2 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6dl-riotboard.dts b/arch/arm/boot/dts/imx6dl-riotboard.dts
index 75d73437adf7..2cb72824e800 100644
--- a/arch/arm/boot/dts/imx6dl-riotboard.dts
+++ b/arch/arm/boot/dts/imx6dl-riotboard.dts
@@ -390,7 +390,7 @@
 				MX6QDL_PAD_RGMII_RD3__RGMII_RD3		0x1b030		/* AR8035 pin strapping: MODE#3: pull up */
 				MX6QDL_PAD_RGMII_RX_CTL__RGMII_RX_CTL	0x130b0		/* AR8035 pin strapping: MODE#0: pull down */
 				MX6QDL_PAD_GPIO_16__ENET_REF_CLK	0x4001b0a8	/* GPIO16 -> AR8035 25MHz */
-			        MX6QDL_PAD_EIM_D31__GPIO3_IO31		0x130b0		/* RGMII_nRST */
+				MX6QDL_PAD_EIM_D31__GPIO3_IO31		0x130b0		/* RGMII_nRST */
 				MX6QDL_PAD_ENET_TX_EN__GPIO1_IO28	0x180b0		/* AR8035 interrupt */
 				MX6QDL_PAD_GPIO_6__ENET_IRQ		0x000b1
 			>;
diff --git a/arch/arm/boot/dts/imx6dl-tx6dl-comtft.dts b/arch/arm/boot/dts/imx6dl-tx6dl-comtft.dts
index 063fe7510da5..aac42ac465b6 100644
--- a/arch/arm/boot/dts/imx6dl-tx6dl-comtft.dts
+++ b/arch/arm/boot/dts/imx6dl-tx6dl-comtft.dts
@@ -105,7 +105,7 @@
 				pixelclk-active = <1>;
 			};
 		};
-        };
+	};
 };
 
 &can1 {
diff --git a/arch/arm/boot/dts/imx6dl-tx6u-801x.dts b/arch/arm/boot/dts/imx6dl-tx6u-801x.dts
index b7a72840b7f0..d1f1298ec55a 100644
--- a/arch/arm/boot/dts/imx6dl-tx6u-801x.dts
+++ b/arch/arm/boot/dts/imx6dl-tx6u-801x.dts
@@ -199,7 +199,7 @@
 				pixelclk-active = <0>;
 			};
 		};
-        };
+	};
 };
 
 &ipu1_di0_disp0 {
diff --git a/arch/arm/boot/dts/imx6q-apalis-ixora.dts b/arch/arm/boot/dts/imx6q-apalis-ixora.dts
index 207b85b91ada..0ea75f7b6039 100644
--- a/arch/arm/boot/dts/imx6q-apalis-ixora.dts
+++ b/arch/arm/boot/dts/imx6q-apalis-ixora.dts
@@ -147,28 +147,6 @@
 			gpios = <&gpio2 2 GPIO_ACTIVE_HIGH>;
 		};
 	};
-
-	pwmleds {
-		compatible = "pwm-leds";
-
-		ledpwm1 {
-			label = "PWM1";
-			pwms = <&pwm1 0 50000>;
-			max-brightness = <255>;
-		};
-
-		ledpwm2 {
-			label = "PWM2";
-			pwms = <&pwm2 0 50000>;
-			max-brightness = <255>;
-		};
-
-		ledpwm3 {
-			label = "PWM3";
-			pwms = <&pwm3 0 50000>;
-			max-brightness = <255>;
-		};
-	};
 };
 
 &backlight {
diff --git a/arch/arm/boot/dts/imx6q-b650v3.dts b/arch/arm/boot/dts/imx6q-b650v3.dts
index d85388725426..1dcaee23ed9c 100644
--- a/arch/arm/boot/dts/imx6q-b650v3.dts
+++ b/arch/arm/boot/dts/imx6q-b650v3.dts
@@ -98,3 +98,9 @@
 		line-name = "PCA9539-P05";
 	};
 };
+
+&usbphy1 {
+	fsl,tx-cal-45-dn-ohms = <55>;
+	fsl,tx-cal-45-dp-ohms = <55>;
+	fsl,tx-d-cal = <100>;
+};
diff --git a/arch/arm/boot/dts/imx6q-cm-fx6.dts b/arch/arm/boot/dts/imx6q-cm-fx6.dts
index 59bc5a4dce17..a150bca84daa 100644
--- a/arch/arm/boot/dts/imx6q-cm-fx6.dts
+++ b/arch/arm/boot/dts/imx6q-cm-fx6.dts
@@ -183,7 +183,6 @@
 			MX6QDL_PAD_ENET_REF_CLK__ENET_TX_CLK	0x1b0b0
 			MX6QDL_PAD_ENET_MDIO__ENET_MDIO		0x1b0b0
 			MX6QDL_PAD_ENET_MDC__ENET_MDC		0x1b0b0
-			MX6QDL_PAD_GPIO_16__ENET_REF_CLK	0x4001b0a8
 		>;
 	};
 
diff --git a/arch/arm/boot/dts/imx6q-evi.dts b/arch/arm/boot/dts/imx6q-evi.dts
index 6de21ff47c3a..7c7c1a855ece 100644
--- a/arch/arm/boot/dts/imx6q-evi.dts
+++ b/arch/arm/boot/dts/imx6q-evi.dts
@@ -232,10 +232,7 @@
 };
 
 &weim {
-	#address-cells = <2>;
-	#size-cells = <1>;
 	ranges = <0 0 0x08000000 0x08000000>;
-	fsl,weim-cs-gpr = <&gpr>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_weimfpga &pinctrl_weimcs>;
 	status = "okay";
diff --git a/arch/arm/boot/dts/imx6q-icore.dts b/arch/arm/boot/dts/imx6q-icore.dts
new file mode 100644
index 000000000000..025f54350c28
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-icore.dts
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2016 Amarula Solutions B.V.
+ * Copyright (C) 2016 Engicam S.r.l.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "imx6q.dtsi"
+#include "imx6qdl-icore.dtsi"
+
+/ {
+	model = "Engicam i.CoreM6 Quad/Dual Starter Kit";
+	compatible = "engicam,imx6-icore", "fsl,imx6q";
+};
+
+&can1 {
+	status = "okay";
+};
+
+&can2 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6q-nitrogen6_som2.dts b/arch/arm/boot/dts/imx6q-nitrogen6_som2.dts
new file mode 100644
index 000000000000..cf4feefe02c5
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-nitrogen6_som2.dts
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2016 Boundary Devices, Inc.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6q.dtsi"
+#include "imx6qdl-nitrogen6_som2.dtsi"
+
+/ {
+	model = "Boundary Devices i.MX6 Quad Nitrogen6_SOM2 Board";
+	compatible = "boundary,imx6q-nitrogen6_som2", "fsl,imx6q";
+};
+
+&sata {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6q-novena.dts b/arch/arm/boot/dts/imx6q-novena.dts
index 1723e89e3acc..758bca96786f 100644
--- a/arch/arm/boot/dts/imx6q-novena.dts
+++ b/arch/arm/boot/dts/imx6q-novena.dts
@@ -451,6 +451,10 @@
 	status = "okay";
 };
 
+&pwm1 {
+	status = "okay";
+};
+
 &sata {
 	target-supply = <&reg_sata>;
 	fsl,transmit-level-mV = <1025>;
diff --git a/arch/arm/boot/dts/imx6q-phytec-pbab01.dts b/arch/arm/boot/dts/imx6q-phytec-pbab01.dts
index c139ac0ebe15..1f4771304da8 100644
--- a/arch/arm/boot/dts/imx6q-phytec-pbab01.dts
+++ b/arch/arm/boot/dts/imx6q-phytec-pbab01.dts
@@ -23,5 +23,5 @@
 };
 
 &sata {
-        status = "okay";
+	status = "okay";
 };
diff --git a/arch/arm/boot/dts/imx6q-tx6q-1010-comtft.dts b/arch/arm/boot/dts/imx6q-tx6q-1010-comtft.dts
index 65e95ae7509a..71746edc2ee9 100644
--- a/arch/arm/boot/dts/imx6q-tx6q-1010-comtft.dts
+++ b/arch/arm/boot/dts/imx6q-tx6q-1010-comtft.dts
@@ -105,7 +105,7 @@
 				pixelclk-active = <1>;
 			};
 		};
-        };
+	};
 };
 
 &can1 {
diff --git a/arch/arm/boot/dts/imx6q-tx6q-1010.dts b/arch/arm/boot/dts/imx6q-tx6q-1010.dts
index 20cd0e7b3e21..f9cd21a41a79 100644
--- a/arch/arm/boot/dts/imx6q-tx6q-1010.dts
+++ b/arch/arm/boot/dts/imx6q-tx6q-1010.dts
@@ -199,7 +199,7 @@
 				pixelclk-active = <0>;
 			};
 		};
-        };
+	};
 };
 
 &ipu1_di0_disp0 {
diff --git a/arch/arm/boot/dts/imx6q-tx6q-1020-comtft.dts b/arch/arm/boot/dts/imx6q-tx6q-1020-comtft.dts
index 9ed243b704ff..959ff3fb7304 100644
--- a/arch/arm/boot/dts/imx6q-tx6q-1020-comtft.dts
+++ b/arch/arm/boot/dts/imx6q-tx6q-1020-comtft.dts
@@ -105,7 +105,7 @@
 				pixelclk-active = <1>;
 			};
 		};
-        };
+	};
 };
 
 &can1 {
diff --git a/arch/arm/boot/dts/imx6q-tx6q-1020.dts b/arch/arm/boot/dts/imx6q-tx6q-1020.dts
index 347b531d3763..b49133d25d80 100644
--- a/arch/arm/boot/dts/imx6q-tx6q-1020.dts
+++ b/arch/arm/boot/dts/imx6q-tx6q-1020.dts
@@ -199,7 +199,7 @@
 				pixelclk-active = <0>;
 			};
 		};
-        };
+	};
 };
 
 &ds1339 {
diff --git a/arch/arm/boot/dts/imx6q-utilite-pro.dts b/arch/arm/boot/dts/imx6q-utilite-pro.dts
index 61990630a748..22009947cebc 100644
--- a/arch/arm/boot/dts/imx6q-utilite-pro.dts
+++ b/arch/arm/boot/dts/imx6q-utilite-pro.dts
@@ -68,7 +68,41 @@
 			label = "Power Button";
 			gpios = <&gpio1 29 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_POWER>;
-			gpio-key,wakeup;
+			wakeup-source;
+		};
+	};
+
+	i2cmux {
+		compatible = "i2c-mux-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_i2c1mux>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		mux-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>;
+		i2c-parent = <&i2c1>;
+
+		i2c@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			eeprom@50 {
+				compatible = "at24,24c02";
+				reg = <0x50>;
+				pagesize = <16>;
+			};
+
+			em3027: rtc@56 {
+				compatible = "emmicro,em3027";
+				reg = <0x56>;
+			};
+		};
+
+		i2c_dvi_ddc: i2c@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 		};
 	};
 };
@@ -82,17 +116,6 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_i2c1>;
 	status = "okay";
-
-	eeprom@50 {
-		compatible = "at24,24c02";
-		reg = <0x50>;
-		pagesize = <16>;
-	};
-
-	em3027: rtc@56 {
-		compatible = "emmicro,em3027";
-		reg = <0x56>;
-	};
 };
 
 &i2c2 {
@@ -115,6 +138,12 @@
 		>;
 	};
 
+	pinctrl_i2c1mux: i2c1muxgrp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_2__GPIO1_IO02 0x1b0b0
+		>;
+	};
+
 	pinctrl_i2c2: i2c2grp {
 		fsl,pins = <
 			MX6QDL_PAD_KEY_COL3__I2C2_SCL 0x4001b8b1
diff --git a/arch/arm/boot/dts/imx6qdl-apalis.dtsi b/arch/arm/boot/dts/imx6qdl-apalis.dtsi
index 99e323b57261..8c8a049eb3d0 100644
--- a/arch/arm/boot/dts/imx6qdl-apalis.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-apalis.dtsi
@@ -49,7 +49,10 @@
 
 	backlight: backlight {
 		compatible = "pwm-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_bl_on>;
 		pwms = <&pwm4 0 5000000>;
+		enable-gpios = <&gpio3 13 GPIO_ACTIVE_HIGH>;
 		status = "disabled";
 	};
 
@@ -620,6 +623,12 @@
 		>;
 	};
 
+	pinctrl_gpio_bl_on: gpioblon {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_DA13__GPIO3_IO13 0x1b0b0
+		>;
+	};
+
 	pinctrl_gpio_keys: gpio1io04grp {
 		fsl,pins = <
 			/* Power button */
diff --git a/arch/arm/boot/dts/imx6qdl-apf6dev.dtsi b/arch/arm/boot/dts/imx6qdl-apf6dev.dtsi
index edbce222c782..5e7792d6bf58 100644
--- a/arch/arm/boot/dts/imx6qdl-apf6dev.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-apf6dev.dtsi
@@ -347,13 +347,13 @@
 			fsl,pins = <
 				MX6QDL_PAD_DI0_PIN4__GPIO4_IO20		0x100b1
 				MX6QDL_PAD_DISP0_DAT18__GPIO5_IO12	0x100b1
-				MX6QDL_PAD_DISP0_DAT19__GPIO5_IO13 	0x100b1
-				MX6QDL_PAD_DISP0_DAT20__GPIO5_IO14 	0x100b1
-				MX6QDL_PAD_DISP0_DAT21__GPIO5_IO15 	0x100b1
-				MX6QDL_PAD_DISP0_DAT22__GPIO5_IO16 	0x100b1
-				MX6QDL_PAD_DISP0_DAT23__GPIO5_IO17 	0x100b1
-				MX6QDL_PAD_CSI0_PIXCLK__GPIO5_IO18 	0x100b1
-				MX6QDL_PAD_CSI0_VSYNC__GPIO5_IO21  	0x100b1
+				MX6QDL_PAD_DISP0_DAT19__GPIO5_IO13	0x100b1
+				MX6QDL_PAD_DISP0_DAT20__GPIO5_IO14	0x100b1
+				MX6QDL_PAD_DISP0_DAT21__GPIO5_IO15	0x100b1
+				MX6QDL_PAD_DISP0_DAT22__GPIO5_IO16	0x100b1
+				MX6QDL_PAD_DISP0_DAT23__GPIO5_IO17	0x100b1
+				MX6QDL_PAD_CSI0_PIXCLK__GPIO5_IO18	0x100b1
+				MX6QDL_PAD_CSI0_VSYNC__GPIO5_IO21	0x100b1
 			>;
 		};
 
diff --git a/arch/arm/boot/dts/imx6qdl-colibri.dtsi b/arch/arm/boot/dts/imx6qdl-colibri.dtsi
new file mode 100644
index 000000000000..e6faa653f91a
--- /dev/null
+++ b/arch/arm/boot/dts/imx6qdl-colibri.dtsi
@@ -0,0 +1,890 @@
+/*
+ * Copyright 2014-2016 Toradex AG
+ * Copyright 2012 Freescale Semiconductor, Inc.
+ * Copyright 2011 Linaro Ltd.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "Toradex Colibri iMX6DL/S Module";
+	compatible = "toradex,colibri_imx6dl", "fsl,imx6dl";
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_bl_on>;
+		pwms = <&pwm3 0 5000000>;
+		enable-gpios = <&gpio3 26 GPIO_ACTIVE_HIGH>; /* Colibri BL_ON */
+		status = "disabled";
+	};
+
+	reg_1p8v: regulator-1p8v {
+		compatible = "regulator-fixed";
+		regulator-name = "1P8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+	};
+
+	reg_2p5v: regulator-2p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "2P5V";
+		regulator-min-microvolt = <2500000>;
+		regulator-max-microvolt = <2500000>;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "3P3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_usb_host_vbus: regulator-usb-host-vbus {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_regulator_usbh_pwr>;
+		regulator-name = "usb_host_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio3 31 GPIO_ACTIVE_HIGH>; /* USBH_PEN */
+		status = "disabled";
+	};
+
+	sound {
+		compatible = "fsl,imx-audio-sgtl5000";
+		model = "imx6dl-colibri-sgtl5000";
+		ssi-controller = <&ssi1>;
+		audio-codec = <&codec>;
+		audio-routing =
+			"Headphone Jack", "HP_OUT",
+			"LINE_IN", "Line In Jack",
+			"MIC_IN", "Mic Jack",
+			"Mic Jack", "Mic Bias";
+		mux-int-port = <1>;
+		mux-ext-port = <5>;
+	};
+
+	/* Optional S/PDIF in on SODIMM 88 and out on SODIMM 90, 137 or 168 */
+	sound_spdif: sound-spdif {
+		compatible = "fsl,imx-audio-spdif";
+		model = "imx-spdif";
+		spdif-controller = <&spdif>;
+		spdif-in;
+		spdif-out;
+		status = "disabled";
+	};
+};
+
+&audmux {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_audmux &pinctrl_mic_gnd>;
+	status = "okay";
+};
+
+/* Optional on SODIMM 55/63 */
+&can1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+	status = "disabled";
+};
+
+/* Optional on SODIMM 178/188 */
+&can2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	status = "disabled";
+};
+
+/* Colibri SSP */
+&ecspi4 {
+	fsl,spi-num-chipselects = <1>;
+	cs-gpios = <&gpio5 2 GPIO_ACTIVE_HIGH>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi4>;
+	status = "disabled";
+};
+
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet>;
+	phy-mode = "rmii";
+	status = "okay";
+};
+
+&hdmi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hdmi_ddc>;
+	status = "disabled";
+};
+
+/*
+ * PWR_I2C: power I2C to audio codec, PMIC, temperature sensor and
+ * touch screen controller
+ */
+&i2c2 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	status = "okay";
+
+	pmic: pfuze100@08 {
+		compatible = "fsl,pfuze100";
+		reg = <0x08>;
+
+		regulators {
+			sw1a_reg: sw1ab {
+				regulator-min-microvolt = <300000>;
+				regulator-max-microvolt = <1875000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <6250>;
+			};
+
+			sw1c_reg: sw1c {
+				regulator-min-microvolt = <300000>;
+				regulator-max-microvolt = <1875000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <6250>;
+			};
+
+			sw3a_reg: sw3a {
+				regulator-min-microvolt = <400000>;
+				regulator-max-microvolt = <1975000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			swbst_reg: swbst {
+				regulator-min-microvolt = <5000000>;
+				regulator-max-microvolt = <5150000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			snvs_reg: vsnvs {
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vref_reg: vrefddr {
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* vgen1: unused */
+
+			vgen2_reg: vgen2 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1550000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* vgen3: unused */
+
+			vgen4_reg: vgen4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vgen5_reg: vgen5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vgen6_reg: vgen6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+	};
+
+	codec: sgtl5000@0a {
+		compatible = "fsl,sgtl5000";
+		reg = <0x0a>;
+		clocks = <&clks IMX6QDL_CLK_CKO>;
+		VDDA-supply = <&reg_2p5v>;
+		VDDIO-supply = <&reg_3p3v>;
+	};
+
+	/* STMPE811 touch screen controller */
+	stmpe811@41 {
+		compatible = "st,stmpe811";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_touch_int>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x41>;
+		interrupts = <20 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-parent = <&gpio6>;
+		interrupt-controller;
+		id = <0>;
+		blocks = <0x5>;
+		irq-trigger = <0x1>;
+
+		stmpe_touchscreen {
+			compatible = "st,stmpe-ts";
+			reg = <0>;
+			/* 3.25 MHz ADC clock speed */
+			st,adc-freq = <1>;
+			/* 8 sample average control */
+			st,ave-ctrl = <3>;
+			/* 7 length fractional part in z */
+			st,fraction-z = <7>;
+			/*
+			 * 50 mA typical 80 mA max touchscreen drivers
+			 * current limit value
+			 */
+			st,i-drive = <1>;
+			/* 12-bit ADC */
+			st,mod-12b = <1>;
+			/* internal ADC reference */
+			st,ref-sel = <0>;
+			/* ADC converstion time: 80 clocks */
+			st,sample-time = <4>;
+			/* 1 ms panel driver settling time */
+			st,settling = <3>;
+			/* 5 ms touch detect interrupt delay */
+			st,touch-det-delay = <5>;
+		};
+	};
+};
+
+/*
+ * I2C3_SDA/SCL on SODIMM 194/196 (e.g. RTC on carrier board)
+ */
+&i2c3 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default", "recovery";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	pinctrl-1 = <&pinctrl_i2c3_recovery>;
+	scl-gpios = <&gpio1 3 GPIO_ACTIVE_HIGH>;
+	sda-gpios = <&gpio1 6 GPIO_ACTIVE_HIGH>;
+	status = "disabled";
+};
+
+/* Colibri PWM<B> */
+&pwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm1>;
+	status = "disabled";
+};
+
+/* Colibri PWM<D> */
+&pwm2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm2>;
+	status = "disabled";
+};
+
+/* Colibri PWM<A> */
+&pwm3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm3>;
+	status = "disabled";
+};
+
+/* Colibri PWM<C> */
+&pwm4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm4>;
+	status = "disabled";
+};
+
+/* Optional S/PDIF out on SODIMM 137 */
+&spdif {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spdif>;
+	status = "disabled";
+};
+
+&ssi1 {
+	status = "okay";
+};
+
+/* Colibri UART_A */
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1_dte &pinctrl_uart1_ctrl>;
+	fsl,dte-mode;
+	uart-has-rtscts;
+	status = "disabled";
+};
+
+/* Colibri UART_B */
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2_dte>;
+	fsl,dte-mode;
+	uart-has-rtscts;
+	status = "disabled";
+};
+
+/* Colibri UART_C */
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3_dte>;
+	fsl,dte-mode;
+	status = "disabled";
+};
+
+&usbotg {
+	pinctrl-names = "default";
+	disable-over-current;
+	dr_mode = "peripheral";
+	status = "disabled";
+};
+
+/* Colibri MMC */
+&usdhc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	vqmmc-supply = <&reg_3p3v>;
+	bus-width = <4>;
+	voltage-ranges = <3300 3300>;
+	status = "disabled";
+};
+
+/* eMMC */
+&usdhc3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	vqmmc-supply = <&reg_3p3v>;
+	bus-width = <8>;
+	voltage-ranges = <3300 3300>;
+	non-removable;
+	status = "okay";
+};
+
+&weim {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_weim_sram  &pinctrl_weim_cs0
+		     &pinctrl_weim_cs1   &pinctrl_weim_cs2
+		     &pinctrl_weim_rdnwr &pinctrl_weim_npwe>;
+	#address-cells = <2>;
+	#size-cells = <1>;
+	status = "disabled";
+};
+
+&iomuxc {
+	pinctrl_audmux: audmuxgrp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL0__AUD5_TXC	0x130b0
+			MX6QDL_PAD_KEY_ROW0__AUD5_TXD	0x130b0
+			MX6QDL_PAD_KEY_COL1__AUD5_TXFS	0x130b0
+			MX6QDL_PAD_KEY_ROW1__AUD5_RXD	0x130b0
+			/* SGTL5000 sys_mclk */
+			MX6QDL_PAD_GPIO_0__CCM_CLKO1	0x000b0
+		>;
+	};
+
+	pinctrl_cam_mclk: cammclkgrp {
+		fsl,pins = <
+			/* Parallel Camera CAM sys_mclk */
+			MX6QDL_PAD_NANDF_CS2__CCM_CLKO2	0x00b0
+		>;
+	};
+
+	pinctrl_ecspi4: ecspi4grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D22__ECSPI4_MISO	0x100b1
+			MX6QDL_PAD_EIM_D28__ECSPI4_MOSI	0x100b1
+			MX6QDL_PAD_EIM_D21__ECSPI4_SCLK 0x100b1
+			/* SPI CS */
+			MX6QDL_PAD_EIM_A25__GPIO5_IO02	0x000b1
+		>;
+	};
+
+	pinctrl_enet: enetgrp {
+		fsl,pins = <
+			MX6QDL_PAD_ENET_MDC__ENET_MDC		0x1b0b0
+			MX6QDL_PAD_ENET_MDIO__ENET_MDIO		0x1b0b0
+			MX6QDL_PAD_ENET_RXD0__ENET_RX_DATA0	0x1b0b0
+			MX6QDL_PAD_ENET_RXD1__ENET_RX_DATA1	0x1b0b0
+			MX6QDL_PAD_ENET_RX_ER__ENET_RX_ER	0x1b0b0
+			MX6QDL_PAD_ENET_TX_EN__ENET_TX_EN	0x1b0b0
+			MX6QDL_PAD_ENET_TXD0__ENET_TX_DATA0	0x1b0b0
+			MX6QDL_PAD_ENET_TXD1__ENET_TX_DATA1	0x1b0b0
+			MX6QDL_PAD_ENET_CRS_DV__ENET_RX_EN	0x1b0b0
+			MX6QDL_PAD_GPIO_16__ENET_REF_CLK     ((1<<30) | 0x1b0b0)
+		>;
+	};
+
+	pinctrl_flexcan1: flexcan1grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_7__FLEXCAN1_TX		0x1b0b0
+			MX6QDL_PAD_GPIO_8__FLEXCAN1_RX		0x1b0b0
+		>;
+	};
+
+	pinctrl_flexcan2: flexcan2grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL4__FLEXCAN2_TX	0x1b0b0
+			MX6QDL_PAD_KEY_ROW4__FLEXCAN2_RX	0x1b0b0
+		>;
+	};
+
+	pinctrl_gpio_bl_on: gpioblon {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D26__GPIO3_IO26		0x1b0b0
+		>;
+	};
+
+	pinctrl_gpio_keys: gpiokeys {
+		fsl,pins = <
+			/* Power button */
+			MX6QDL_PAD_EIM_A16__GPIO2_IO22		0x1b0b0
+		>;
+	};
+
+	pinctrl_hdmi_ddc: hdmiddcgrp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL3__HDMI_TX_DDC_SCL 0x4001b8b1
+			MX6QDL_PAD_KEY_ROW3__HDMI_TX_DDC_SDA 0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_EB2__I2C2_SCL 0x4001b8b1
+			MX6QDL_PAD_EIM_D16__I2C2_SDA 0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_3__I2C3_SCL 0x4001b8b1
+			MX6QDL_PAD_GPIO_6__I2C3_SDA 0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c3_recovery: i2c3recoverygrp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_3__GPIO1_IO03 0x4001b8b1
+			MX6QDL_PAD_GPIO_6__GPIO1_IO06 0x4001b8b1
+		>;
+	};
+
+	pinctrl_ipu1_csi0: ipu1csi0grp { /* Parallel Camera */
+		fsl,pins = <
+			MX6QDL_PAD_EIM_A17__IPU1_CSI1_DATA12	0xb0b1
+			MX6QDL_PAD_EIM_A18__IPU1_CSI1_DATA13	0xb0b1
+			MX6QDL_PAD_EIM_A19__IPU1_CSI1_DATA14	0xb0b1
+			MX6QDL_PAD_EIM_A20__IPU1_CSI1_DATA15	0xb0b1
+			MX6QDL_PAD_EIM_A21__IPU1_CSI1_DATA16	0xb0b1
+			MX6QDL_PAD_EIM_A22__IPU1_CSI1_DATA17	0xb0b1
+			MX6QDL_PAD_EIM_A23__IPU1_CSI1_DATA18	0xb0b1
+			MX6QDL_PAD_EIM_A24__IPU1_CSI1_DATA19	0xb0b1
+			MX6QDL_PAD_EIM_D17__IPU1_CSI1_PIXCLK	0xb0b1
+			MX6QDL_PAD_EIM_EB3__IPU1_CSI1_HSYNC	0xb0b1
+			MX6QDL_PAD_EIM_D29__IPU1_CSI1_VSYNC	0xb0b1
+			/* Disable PWM pins on camera interface */
+			MX6QDL_PAD_SD4_DAT1__GPIO2_IO09		0x40
+			MX6QDL_PAD_GPIO_1__GPIO1_IO01		0x40
+		>;
+	};
+
+	pinctrl_ipu1_lcdif: ipu1lcdifgrp {
+		fsl,pins = <
+			MX6QDL_PAD_DI0_DISP_CLK__IPU1_DI0_DISP_CLK	0xa1
+			MX6QDL_PAD_DI0_PIN15__IPU1_DI0_PIN15		0xa1
+			MX6QDL_PAD_DI0_PIN2__IPU1_DI0_PIN02		0xa1
+			MX6QDL_PAD_DI0_PIN3__IPU1_DI0_PIN03		0xa1
+			MX6QDL_PAD_DISP0_DAT0__IPU1_DISP0_DATA00	0xa1
+			MX6QDL_PAD_DISP0_DAT1__IPU1_DISP0_DATA01	0xa1
+			MX6QDL_PAD_DISP0_DAT2__IPU1_DISP0_DATA02	0xa1
+			MX6QDL_PAD_DISP0_DAT3__IPU1_DISP0_DATA03	0xa1
+			MX6QDL_PAD_DISP0_DAT4__IPU1_DISP0_DATA04	0xa1
+			MX6QDL_PAD_DISP0_DAT5__IPU1_DISP0_DATA05	0xa1
+			MX6QDL_PAD_DISP0_DAT6__IPU1_DISP0_DATA06	0xa1
+			MX6QDL_PAD_DISP0_DAT7__IPU1_DISP0_DATA07	0xa1
+			MX6QDL_PAD_DISP0_DAT8__IPU1_DISP0_DATA08	0xa1
+			MX6QDL_PAD_DISP0_DAT9__IPU1_DISP0_DATA09	0xa1
+			MX6QDL_PAD_DISP0_DAT10__IPU1_DISP0_DATA10	0xa1
+			MX6QDL_PAD_DISP0_DAT11__IPU1_DISP0_DATA11	0xa1
+			MX6QDL_PAD_DISP0_DAT12__IPU1_DISP0_DATA12	0xa1
+			MX6QDL_PAD_DISP0_DAT13__IPU1_DISP0_DATA13	0xa1
+			MX6QDL_PAD_DISP0_DAT14__IPU1_DISP0_DATA14	0xa1
+			MX6QDL_PAD_DISP0_DAT15__IPU1_DISP0_DATA15	0xa1
+			MX6QDL_PAD_DISP0_DAT16__IPU1_DISP0_DATA16	0xa1
+			MX6QDL_PAD_DISP0_DAT17__IPU1_DISP0_DATA17	0xa1
+		>;
+	};
+
+	pinctrl_mic_gnd: gpiomicgnd {
+		fsl,pins = <
+			/* Controls Mic GND, PU or '1' pull Mic GND to GND */
+			MX6QDL_PAD_RGMII_TD1__GPIO6_IO21 0x1b0b0
+		>;
+	};
+
+	pinctrl_mmc_cd: gpiommccd {
+		fsl,pins = <
+			MX6QDL_PAD_NANDF_D5__GPIO2_IO05	0x80000000
+		>;
+	};
+
+	pinctrl_pwm1: pwm1grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_9__PWM1_OUT	0x1b0b1
+		>;
+	};
+
+	pinctrl_pwm2: pwm2grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_1__PWM2_OUT	0x1b0b1
+			MX6QDL_PAD_EIM_A21__GPIO2_IO17	0x00040
+		>;
+	};
+
+	pinctrl_pwm3: pwm3grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD4_DAT1__PWM3_OUT	0x1b0b1
+			MX6QDL_PAD_EIM_A22__GPIO2_IO16	0x00040
+		>;
+	};
+
+	pinctrl_pwm4: pwm4grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD4_DAT2__PWM4_OUT	0x1b0b1
+		>;
+	};
+
+	pinctrl_regulator_usbh_pwr: gpioregusbhpwrgrp {
+		fsl,pins = <
+			/* USBH_EN */
+			MX6QDL_PAD_EIM_D31__GPIO3_IO31	0x0f058
+		>;
+	};
+
+	pinctrl_spdif: spdifgrp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_17__SPDIF_OUT 0x1b0b0
+		>;
+	};
+
+	pinctrl_touch_int: gpiotouchintgrp {
+		fsl,pins = <
+			/* STMPE811 interrupt */
+			MX6QDL_PAD_RGMII_TD0__GPIO6_IO20 0x1b0b0
+		>;
+	};
+
+	pinctrl_uart1_dce: uart1dcegrp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT10__UART1_TX_DATA 0x1b0b1
+			MX6QDL_PAD_CSI0_DAT11__UART1_RX_DATA 0x1b0b1
+		>;
+	};
+
+	/* DTE mode */
+	pinctrl_uart1_dte: uart1dtegrp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT10__UART1_RX_DATA 0x1b0b1
+			MX6QDL_PAD_CSI0_DAT11__UART1_TX_DATA 0x1b0b1
+			MX6QDL_PAD_EIM_D19__UART1_RTS_B	0x1b0b1
+			MX6QDL_PAD_EIM_D20__UART1_CTS_B 0x1b0b1
+		>;
+	};
+
+	/* Additional DTR, DSR, DCD */
+	pinctrl_uart1_ctrl: uart1ctrlgrp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D23__UART1_DCD_B 0x1b0b0
+			MX6QDL_PAD_EIM_D24__UART1_DTR_B 0x1b0b0
+			MX6QDL_PAD_EIM_D25__UART1_DSR_B 0x1b0b0
+		>;
+	};
+
+	pinctrl_uart2_dte: uart2dtegrp {
+		fsl,pins = <
+			MX6QDL_PAD_SD4_DAT4__UART2_TX_DATA	0x1b0b1
+			MX6QDL_PAD_SD4_DAT7__UART2_RX_DATA	0x1b0b1
+			MX6QDL_PAD_SD4_DAT6__UART2_RTS_B	0x1b0b1
+			MX6QDL_PAD_SD4_DAT5__UART2_CTS_B	0x1b0b1
+		>;
+	};
+
+	pinctrl_uart3_dte: uart3dtegrp {
+		fsl,pins = <
+			MX6QDL_PAD_SD4_CLK__UART3_TX_DATA	0x1b0b1
+			MX6QDL_PAD_SD4_CMD__UART3_RX_DATA	0x1b0b1
+		>;
+	};
+
+	pinctrl_usbc_det: usbcdetgrp {
+		fsl,pins = <
+			/* USBC_DET */
+			MX6QDL_PAD_GPIO_17__GPIO7_IO12		0x1b0b0
+			/* USBC_DET_EN */
+			MX6QDL_PAD_RGMII_TX_CTL__GPIO6_IO26	0x0f058
+			/* USBC_DET_OVERWRITE */
+			MX6QDL_PAD_RGMII_RXC__GPIO6_IO30	0x0f058
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD1_CMD__SD1_CMD	0x17071
+			MX6QDL_PAD_SD1_CLK__SD1_CLK	0x10071
+			MX6QDL_PAD_SD1_DAT0__SD1_DATA0	0x17071
+			MX6QDL_PAD_SD1_DAT1__SD1_DATA1	0x17071
+			MX6QDL_PAD_SD1_DAT2__SD1_DATA2	0x17071
+			MX6QDL_PAD_SD1_DAT3__SD1_DATA3	0x17071
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_CMD__SD3_CMD	0x17059
+			MX6QDL_PAD_SD3_CLK__SD3_CLK	0x10059
+			MX6QDL_PAD_SD3_DAT0__SD3_DATA0	0x17059
+			MX6QDL_PAD_SD3_DAT1__SD3_DATA1	0x17059
+			MX6QDL_PAD_SD3_DAT2__SD3_DATA2	0x17059
+			MX6QDL_PAD_SD3_DAT3__SD3_DATA3	0x17059
+			MX6QDL_PAD_SD3_DAT4__SD3_DATA4	0x17059
+			MX6QDL_PAD_SD3_DAT5__SD3_DATA5	0x17059
+			MX6QDL_PAD_SD3_DAT6__SD3_DATA6	0x17059
+			MX6QDL_PAD_SD3_DAT7__SD3_DATA7	0x17059
+			/* eMMC reset */
+			MX6QDL_PAD_SD3_RST__SD3_RESET	0x17059
+		>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3100mhzgrp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_CMD__SD3_CMD	0x170b9
+			MX6QDL_PAD_SD3_CLK__SD3_CLK	0x100b9
+			MX6QDL_PAD_SD3_DAT0__SD3_DATA0	0x170b9
+			MX6QDL_PAD_SD3_DAT1__SD3_DATA1	0x170b9
+			MX6QDL_PAD_SD3_DAT2__SD3_DATA2	0x170b9
+			MX6QDL_PAD_SD3_DAT3__SD3_DATA3	0x170b9
+			MX6QDL_PAD_SD3_DAT4__SD3_DATA4	0x170b9
+			MX6QDL_PAD_SD3_DAT5__SD3_DATA5	0x170b9
+			MX6QDL_PAD_SD3_DAT6__SD3_DATA6	0x170b9
+			MX6QDL_PAD_SD3_DAT7__SD3_DATA7	0x170b9
+			/* eMMC reset */
+			MX6QDL_PAD_SD3_RST__SD3_RESET	0x170b9
+		>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3200mhzgrp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_CMD__SD3_CMD	0x170f9
+			MX6QDL_PAD_SD3_CLK__SD3_CLK	0x100f9
+			MX6QDL_PAD_SD3_DAT0__SD3_DATA0	0x170f9
+			MX6QDL_PAD_SD3_DAT1__SD3_DATA1	0x170f9
+			MX6QDL_PAD_SD3_DAT2__SD3_DATA2	0x170f9
+			MX6QDL_PAD_SD3_DAT3__SD3_DATA3	0x170f9
+			MX6QDL_PAD_SD3_DAT4__SD3_DATA4	0x170f9
+			MX6QDL_PAD_SD3_DAT5__SD3_DATA5	0x170f9
+			MX6QDL_PAD_SD3_DAT6__SD3_DATA6	0x170f9
+			MX6QDL_PAD_SD3_DAT7__SD3_DATA7	0x170f9
+			/* eMMC reset */
+			MX6QDL_PAD_SD3_RST__SD3_RESET	0x170f9
+		>;
+	};
+
+	pinctrl_weim_cs0: weimcs0grp {
+		fsl,pins = <
+			/* nEXT_CS0 */
+			MX6QDL_PAD_EIM_CS0__EIM_CS0_B	0xb0b1
+		>;
+	};
+
+	pinctrl_weim_cs1: weimcs1grp {
+		fsl,pins = <
+			/* nEXT_CS1 */
+			MX6QDL_PAD_EIM_CS1__EIM_CS1_B	0xb0b1
+		>;
+	};
+
+	pinctrl_weim_cs2: weimcs2grp {
+		fsl,pins = <
+			/* nEXT_CS2 */
+			MX6QDL_PAD_SD2_DAT1__EIM_CS2_B	0xb0b1
+		>;
+	};
+
+	pinctrl_weim_sram: weimsramgrp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_OE__EIM_OE_B		0xb0b1
+			MX6QDL_PAD_EIM_RW__EIM_RW		0xb0b1
+			/* Data */
+			MX6QDL_PAD_CSI0_DATA_EN__EIM_DATA00	0x1b0b0
+			MX6QDL_PAD_CSI0_VSYNC__EIM_DATA01	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT4__EIM_DATA02	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT5__EIM_DATA03	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT6__EIM_DATA04	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT7__EIM_DATA05	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT8__EIM_DATA06	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT9__EIM_DATA07	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT12__EIM_DATA08	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT13__EIM_DATA09	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT14__EIM_DATA10	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT15__EIM_DATA11	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT16__EIM_DATA12	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT17__EIM_DATA13	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT18__EIM_DATA14	0x1b0b0
+			MX6QDL_PAD_CSI0_DAT19__EIM_DATA15	0x1b0b0
+			/* Address */
+			MX6QDL_PAD_EIM_DA15__EIM_AD15		0xb0b1
+			MX6QDL_PAD_EIM_DA14__EIM_AD14		0xb0b1
+			MX6QDL_PAD_EIM_DA13__EIM_AD13		0xb0b1
+			MX6QDL_PAD_EIM_DA12__EIM_AD12		0xb0b1
+			MX6QDL_PAD_EIM_DA11__EIM_AD11		0xb0b1
+			MX6QDL_PAD_EIM_DA10__EIM_AD10		0xb0b1
+			MX6QDL_PAD_EIM_DA9__EIM_AD09		0xb0b1
+			MX6QDL_PAD_EIM_DA8__EIM_AD08		0xb0b1
+			MX6QDL_PAD_EIM_DA7__EIM_AD07		0xb0b1
+			MX6QDL_PAD_EIM_DA6__EIM_AD06		0xb0b1
+			MX6QDL_PAD_EIM_DA5__EIM_AD05		0xb0b1
+			MX6QDL_PAD_EIM_DA4__EIM_AD04		0xb0b1
+			MX6QDL_PAD_EIM_DA3__EIM_AD03		0xb0b1
+			MX6QDL_PAD_EIM_DA2__EIM_AD02		0xb0b1
+			MX6QDL_PAD_EIM_DA1__EIM_AD01		0xb0b1
+			MX6QDL_PAD_EIM_DA0__EIM_AD00		0xb0b1
+		>;
+	};
+
+	pinctrl_weim_rdnwr: weimrdnwr {
+		fsl,pins = <
+			MX6QDL_PAD_SD2_CLK__GPIO1_IO10		0x0040
+			MX6QDL_PAD_RGMII_TD3__GPIO6_IO23	0x130b0
+		>;
+	};
+
+	pinctrl_weim_npwe: weimnpwe {
+		fsl,pins = <
+			MX6QDL_PAD_SD2_DAT3__GPIO1_IO12		0x0040
+			MX6QDL_PAD_RGMII_TD2__GPIO6_IO22	0x130b0
+		>;
+	};
+
+	/* ADDRESS[16:18] [25] used as GPIO */
+	pinctrl_weim_gpio_1: weimgpio-1 {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_ROW4__GPIO4_IO15		0x1b0b0
+			MX6QDL_PAD_KEY_ROW2__GPIO4_IO11		0x1b0b0
+			MX6QDL_PAD_KEY_COL2__GPIO4_IO10		0x1b0b0
+			MX6QDL_PAD_DISP0_DAT23__GPIO5_IO17	0x1b0b0
+			MX6QDL_PAD_DISP0_DAT22__GPIO5_IO16	0x1b0b0
+			MX6QDL_PAD_DISP0_DAT21__GPIO5_IO15	0x1b0b0
+			MX6QDL_PAD_DISP0_DAT20__GPIO5_IO14	0x1b0b0
+			MX6QDL_PAD_DISP0_DAT19__GPIO5_IO13	0x1b0b0
+			MX6QDL_PAD_DISP0_DAT18__GPIO5_IO12	0x1b0b0
+			MX6QDL_PAD_NANDF_D1__GPIO2_IO01		0x1b0b0
+		>;
+	};
+
+	/* ADDRESS[19:24] used as GPIO */
+	pinctrl_weim_gpio_2: weimgpio-2 {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_ROW2__GPIO4_IO11		0x1b0b0
+			MX6QDL_PAD_KEY_COL2__GPIO4_IO10		0x1b0b0
+			MX6QDL_PAD_DISP0_DAT23__GPIO5_IO17	0x1b0b0
+			MX6QDL_PAD_DISP0_DAT22__GPIO5_IO16	0x1b0b0
+			MX6QDL_PAD_DISP0_DAT21__GPIO5_IO15	0x1b0b0
+			MX6QDL_PAD_DISP0_DAT20__GPIO5_IO14	0x1b0b0
+			MX6QDL_PAD_DISP0_DAT19__GPIO5_IO13	0x1b0b0
+			MX6QDL_PAD_DISP0_DAT18__GPIO5_IO12	0x1b0b0
+			MX6QDL_PAD_NANDF_D1__GPIO2_IO01		0x1b0b0
+		>;
+	};
+
+	/* DATA[16:31] used as GPIO */
+	pinctrl_weim_gpio_3: weimgpio-3 {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_LBA__GPIO2_IO27		0x1b0b0
+			MX6QDL_PAD_EIM_BCLK__GPIO6_IO31		0x1b0b0
+			MX6QDL_PAD_NANDF_CS3__GPIO6_IO16	0x1b0b0
+			MX6QDL_PAD_NANDF_CS1__GPIO6_IO14	0x1b0b0
+			MX6QDL_PAD_NANDF_RB0__GPIO6_IO10	0x1b0b0
+			MX6QDL_PAD_NANDF_ALE__GPIO6_IO08	0x1b0b0
+			MX6QDL_PAD_NANDF_WP_B__GPIO6_IO09	0x1b0b0
+			MX6QDL_PAD_NANDF_CS0__GPIO6_IO11	0x1b0b0
+			MX6QDL_PAD_NANDF_CLE__GPIO6_IO07	0x1b0b0
+			MX6QDL_PAD_GPIO_19__GPIO4_IO05		0x1b0b0
+			MX6QDL_PAD_CSI0_MCLK__GPIO5_IO19	0x1b0b0
+			MX6QDL_PAD_CSI0_PIXCLK__GPIO5_IO18	0x1b0b0
+			MX6QDL_PAD_GPIO_4__GPIO1_IO04		0x1b0b0
+			MX6QDL_PAD_GPIO_5__GPIO1_IO05		0x1b0b0
+			MX6QDL_PAD_GPIO_2__GPIO1_IO02		0x1b0b0
+		>;
+	};
+
+	/* DQM[0:3] used as GPIO */
+	pinctrl_weim_gpio_4: weimgpio-4 {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_EB0__GPIO2_IO28		0x1b0b0
+			MX6QDL_PAD_EIM_EB1__GPIO2_IO29		0x1b0b0
+			MX6QDL_PAD_SD2_DAT2__GPIO1_IO13		0x1b0b0
+			MX6QDL_PAD_NANDF_D0__GPIO2_IO00		0x1b0b0
+		>;
+	};
+
+	/* RDY used as GPIO */
+	pinctrl_weim_gpio_5: weimgpio-5 {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_WAIT__GPIO5_IO00		0x1b0b0
+		>;
+	};
+
+	/* ADDRESS[16] DATA[30] used as GPIO */
+	pinctrl_weim_gpio_6: weimgpio-6 {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_ROW4__GPIO4_IO15		0x1b0b0
+			MX6QDL_PAD_KEY_COL4__GPIO4_IO14		0x1b0b0
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi b/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi
index a7100f99123e..54aca3a07ce4 100644
--- a/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi
@@ -153,9 +153,9 @@
 
 &clks {
 	assigned-clocks = <&clks IMX6QDL_CLK_LDB_DI0_SEL>,
-	                  <&clks IMX6QDL_CLK_LDB_DI1_SEL>;
+			  <&clks IMX6QDL_CLK_LDB_DI1_SEL>;
 	assigned-clock-parents = <&clks IMX6QDL_CLK_PLL3_USB_OTG>,
-	                  <&clks IMX6QDL_CLK_PLL3_USB_OTG>;
+				 <&clks IMX6QDL_CLK_PLL3_USB_OTG>;
 };
 
 &ecspi3 {
diff --git a/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi b/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi
index 8953eba0573d..88e5cb3b6be9 100644
--- a/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi
@@ -154,9 +154,9 @@
 
 &clks {
 	assigned-clocks = <&clks IMX6QDL_CLK_LDB_DI0_SEL>,
-	                  <&clks IMX6QDL_CLK_LDB_DI1_SEL>;
+			  <&clks IMX6QDL_CLK_LDB_DI1_SEL>;
 	assigned-clock-parents = <&clks IMX6QDL_CLK_PLL3_USB_OTG>,
-	                  <&clks IMX6QDL_CLK_PLL3_USB_OTG>;
+				 <&clks IMX6QDL_CLK_PLL3_USB_OTG>;
 };
 
 &fec {
diff --git a/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi b/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi
index 6ac41c7ed32e..1753ab720b0b 100644
--- a/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi
@@ -144,9 +144,9 @@
 
 &clks {
 	assigned-clocks = <&clks IMX6QDL_CLK_LDB_DI0_SEL>,
-	                  <&clks IMX6QDL_CLK_LDB_DI1_SEL>;
+			  <&clks IMX6QDL_CLK_LDB_DI1_SEL>;
 	assigned-clock-parents = <&clks IMX6QDL_CLK_PLL3_USB_OTG>,
-	                  <&clks IMX6QDL_CLK_PLL3_USB_OTG>;
+				 <&clks IMX6QDL_CLK_PLL3_USB_OTG>;
 };
 
 &fec {
diff --git a/arch/arm/boot/dts/imx6qdl-gw552x.dtsi b/arch/arm/boot/dts/imx6qdl-gw552x.dtsi
index 805e23674a94..ee83161f674b 100644
--- a/arch/arm/boot/dts/imx6qdl-gw552x.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-gw552x.dtsi
@@ -291,7 +291,7 @@
 				MX6QDL_PAD_KEY_COL1__UART5_TX_DATA	0x1b0b1
 				MX6QDL_PAD_KEY_ROW1__UART5_RX_DATA	0x1b0b1
 			>;
-                };
+		};
 
 		pinctrl_wdog: wdoggrp {
 			fsl,pins = <
diff --git a/arch/arm/boot/dts/imx6qdl-icore.dtsi b/arch/arm/boot/dts/imx6qdl-icore.dtsi
new file mode 100644
index 000000000000..023839a02dd0
--- /dev/null
+++ b/arch/arm/boot/dts/imx6qdl-icore.dtsi
@@ -0,0 +1,265 @@
+/*
+ * Copyright (C) 2016 Amarula Solutions B.V.
+ * Copyright (C) 2016 Engicam S.r.l.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/ {
+	memory {
+		reg = <0x10000000 0x80000000>;
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "3P3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_usb_h1_vbus: regulator-usb-h1-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_h1_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_usb_otg_vbus: regulator-usb-otg-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_otg_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	rmii_clk: clock-rmii-clk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <25000000>;  /* 25MHz for example */
+	};
+};
+
+&can1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+	xceiver-supply = <&reg_3p3v>;
+};
+
+&can2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	xceiver-supply = <&reg_3p3v>;
+};
+
+&clks {
+	assigned-clocks = <&clks IMX6QDL_CLK_LVDS2_SEL>;
+	assigned-clock-parents = <&clks IMX6QDL_CLK_OSC>;
+};
+
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet>;
+	phy-reset-gpios = <&gpio7 12 GPIO_ACTIVE_LOW>;
+	clocks = <&clks IMX6QDL_CLK_ENET>, <&clks IMX6QDL_CLK_ENET>, <&rmii_clk>;
+	phy-mode = "rmii";
+	status = "okay";
+};
+
+&gpmi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	nand-on-flash-bbt;
+	status = "okay";
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	status = "okay";
+};
+
+&i2c2 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	status = "okay";
+};
+
+&i2c3 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart4>;
+	status = "okay";
+};
+
+&usbh1 {
+	vbus-supply = <&reg_usb_h1_vbus>;
+	disable-over-current;
+	status = "okay";
+};
+
+&usbotg {
+	vbus-supply = <&reg_usb_otg_vbus>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usbotg>;
+	disable-over-current;
+	status = "okay";
+};
+
+&usdhc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	cd-gpios = <&gpio1 1 GPIO_ACTIVE_LOW>;
+	no-1-8-v;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_enet: enetgrp {
+		fsl,pins = <
+			MX6QDL_PAD_ENET_CRS_DV__ENET_RX_EN	0x1b0b0
+			MX6QDL_PAD_GPIO_16__ENET_REF_CLK	0x1b0b1
+			MX6QDL_PAD_ENET_TX_EN__ENET_TX_EN	0x1b0b0
+			MX6QDL_PAD_ENET_RXD1__ENET_RX_DATA1	0x1b0b0
+			MX6QDL_PAD_ENET_RXD0__ENET_RX_DATA0	0x1b0b0
+			MX6QDL_PAD_ENET_TXD1__ENET_TX_DATA1	0x1b0b0
+			MX6QDL_PAD_ENET_TXD0__ENET_TX_DATA0	0x1b0b0
+			MX6QDL_PAD_ENET_MDC__ENET_MDC		0x1b0b0
+			MX6QDL_PAD_ENET_MDIO__ENET_MDIO		0x1b0b0
+			MX6QDL_PAD_ENET_REF_CLK__GPIO1_IO23	0x1b0b0
+			MX6QDL_PAD_GPIO_17__GPIO7_IO12		0x1b0b0
+		>;
+	};
+
+	pinctrl_flexcan1: flexcan1grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_ROW2__FLEXCAN1_RX 0x1b020
+			MX6QDL_PAD_KEY_COL2__FLEXCAN1_TX 0x1b020
+		>;
+	};
+
+	pinctrl_flexcan2: flexcan2grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL4__FLEXCAN2_TX 0x1b020
+			MX6QDL_PAD_KEY_ROW4__FLEXCAN2_RX 0x1b020
+		>;
+	};
+
+	pinctrl_gpmi_nand: gpmi-nand {
+		fsl,pins = <
+			MX6QDL_PAD_NANDF_CLE__NAND_CLE     0xb0b1
+			MX6QDL_PAD_NANDF_ALE__NAND_ALE     0xb0b1
+			MX6QDL_PAD_NANDF_WP_B__NAND_WP_B   0xb0b1
+			MX6QDL_PAD_NANDF_RB0__NAND_READY_B 0xb000
+			MX6QDL_PAD_NANDF_CS0__NAND_CE0_B   0xb0b1
+			MX6QDL_PAD_NANDF_CS1__NAND_CE1_B   0xb0b1
+			MX6QDL_PAD_SD4_CMD__NAND_RE_B      0xb0b1
+			MX6QDL_PAD_SD4_CLK__NAND_WE_B      0xb0b1
+			MX6QDL_PAD_NANDF_D0__NAND_DATA00   0xb0b1
+			MX6QDL_PAD_NANDF_D1__NAND_DATA01   0xb0b1
+			MX6QDL_PAD_NANDF_D2__NAND_DATA02   0xb0b1
+			MX6QDL_PAD_NANDF_D3__NAND_DATA03   0xb0b1
+			MX6QDL_PAD_NANDF_D4__NAND_DATA04   0xb0b1
+			MX6QDL_PAD_NANDF_D5__NAND_DATA05   0xb0b1
+			MX6QDL_PAD_NANDF_D6__NAND_DATA06   0xb0b1
+			MX6QDL_PAD_NANDF_D7__NAND_DATA07   0xb0b1
+			MX6QDL_PAD_SD4_DAT0__NAND_DQS      0x00b1
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D21__I2C1_SCL 0x4001b8b1
+			MX6QDL_PAD_EIM_D28__I2C1_SDA 0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_EB2__I2C2_SCL  0x4001b8b1
+			MX6QDL_PAD_KEY_ROW3__I2C2_SDA 0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_5__I2C3_SCL  0x4001b8b1
+			MX6QDL_PAD_EIM_D18__I2C3_SDA 0x4001b8b1
+			MX6QDL_PAD_GPIO_0__CCM_CLKO1	0x130b0
+		>;
+	};
+
+	pinctrl_uart4: uart4grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL0__UART4_TX_DATA 0x1b0b1
+			MX6QDL_PAD_KEY_ROW0__UART4_RX_DATA 0x1b0b1
+		>;
+	};
+
+	pinctrl_usbotg: usbotggrp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_1__USB_OTG_ID 0x17059
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD1_CMD__SD1_CMD    0x17070
+			MX6QDL_PAD_SD1_CLK__SD1_CLK    0x10070
+			MX6QDL_PAD_SD1_DAT0__SD1_DATA0 0x17070
+			MX6QDL_PAD_SD1_DAT1__SD1_DATA1 0x17070
+			MX6QDL_PAD_SD1_DAT2__SD1_DATA2 0x17070
+			MX6QDL_PAD_SD1_DAT3__SD1_DATA3 0x17070
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/imx6qdl-nit6xlite.dtsi b/arch/arm/boot/dts/imx6qdl-nit6xlite.dtsi
index 880bd782a5b7..63acd54f5278 100644
--- a/arch/arm/boot/dts/imx6qdl-nit6xlite.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-nit6xlite.dtsi
@@ -97,15 +97,6 @@
 		};
 	};
 
-	bt_rfkill {
-		compatible = "rfkill-gpio";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_bt_rfkill>;
-		gpios = <&gpio6 8 GPIO_ACTIVE_HIGH>;
-		name = "bt_rfkill";
-		type = <2>;
-	};
-
 	gpio-keys {
 		compatible = "gpio-keys";
 		pinctrl-names = "default";
@@ -160,7 +151,7 @@
 		};
 	};
 
-	backlight_lcd {
+	backlight-lcd {
 		compatible = "pwm-backlight";
 		pwms = <&pwm1 0 5000000>;
 		brightness-levels = <0 4 8 16 32 64 128 255>;
@@ -169,7 +160,7 @@
 		status = "okay";
 	};
 
-	backlight_lvds0: backlight_lvds0 {
+	backlight_lvds0: backlight-lvds0 {
 		compatible = "pwm-backlight";
 		pwms = <&pwm4 0 5000000>;
 		brightness-levels = <0 4 8 16 32 64 128 255>;
@@ -178,7 +169,7 @@
 		status = "okay";
 	};
 
-	panel_lvds0 {
+	panel-lvds0 {
 		compatible = "hannstar,hsd100pxn1";
 		backlight = <&backlight_lvds0>;
 
@@ -328,19 +319,6 @@
 			>;
 		};
 
-		pinctrl_bt_rfkill: bt_rfkillgrp {
-			fsl,pins = <
-				/* BT wake */
-				MX6QDL_PAD_NANDF_D2__GPIO2_IO02		0x1b0b0
-				/* BT reset */
-				MX6QDL_PAD_NANDF_ALE__GPIO6_IO08	0x0b0b0
-				/* BT reg en */
-				MX6QDL_PAD_NANDF_CS2__GPIO6_IO15	0x1b0b0
-				/* BT host wake irq */
-				MX6QDL_PAD_NANDF_CS3__GPIO6_IO16	0x100b0
-			>;
-		};
-
 		pinctrl_ecspi1: ecspi1grp {
 			fsl,pins = <
 				MX6QDL_PAD_EIM_D17__ECSPI1_MISO		0x100b1
@@ -374,7 +352,7 @@
 			>;
 		};
 
-		pinctrl_gpio_keys: gpio_keysgrp {
+		pinctrl_gpio_keys: gpio-keysgrp {
 			fsl,pins = <
 				/* Home Button: J14 pin 5 */
 				MX6QDL_PAD_GPIO_18__GPIO7_IO13		0x1b0b0
@@ -457,7 +435,7 @@
 			>;
 		};
 
-		pinctrl_wlan_vmmc: wlan_vmmcgrp {
+		pinctrl_wlan_vmmc: wlan-vmmcgrp {
 			fsl,pins = <
 				MX6QDL_PAD_NANDF_CLE__GPIO6_IO07	0x030b0
 			>;
diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi
index b0b3220a1fd9..34887a10c5f1 100644
--- a/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi
@@ -229,7 +229,7 @@
 		};
 	};
 
-	backlight_lcd: backlight_lcd {
+	backlight_lcd: backlight-lcd {
 		compatible = "pwm-backlight";
 		pwms = <&pwm1 0 5000000>;
 		brightness-levels = <0 4 8 16 32 64 128 255>;
@@ -238,7 +238,7 @@
 		status = "okay";
 	};
 
-	backlight_lvds0: backlight_lvds0 {
+	backlight_lvds0: backlight-lvds0 {
 		compatible = "pwm-backlight";
 		pwms = <&pwm4 0 5000000>;
 		brightness-levels = <0 4 8 16 32 64 128 255>;
@@ -247,7 +247,7 @@
 		status = "okay";
 	};
 
-	backlight_lvds1: backlight_lvds1 {
+	backlight_lvds1: backlight-lvds1 {
 		compatible = "pwm-backlight";
 		pwms = <&pwm2 0 5000000>;
 		brightness-levels = <0 4 8 16 32 64 128 255>;
@@ -282,7 +282,7 @@
 		};
 	};
 
-	panel_lcd {
+	panel-lcd {
 		compatible = "okaya,rs800480t-7x0gp";
 		backlight = <&backlight_lcd>;
 
@@ -293,7 +293,7 @@
 		};
 	};
 
-	panel_lvds0 {
+	panel-lvds0 {
 		compatible = "hannstar,hsd100pxn1";
 		backlight = <&backlight_lvds0>;
 
@@ -304,7 +304,7 @@
 		};
 	};
 
-	panel_lvds1 {
+	panel-lvds1 {
 		compatible = "hannstar,hsd100pxn1";
 		backlight = <&backlight_lvds1>;
 
@@ -447,7 +447,7 @@
 };
 
 &iomuxc {
-	imx6q-nitrogen6_max {
+	imx6q-nitrogen6-max {
 		pinctrl_audmux: audmuxgrp {
 			fsl,pins = <
 				MX6QDL_PAD_CSI0_DAT7__AUD3_RXD		0x130b0
@@ -504,7 +504,7 @@
 			>;
 		};
 
-		pinctrl_gpio_keys: gpio_keysgrp {
+		pinctrl_gpio_keys: gpio-keysgrp {
 			fsl,pins = <
 				/* Power Button */
 				MX6QDL_PAD_NANDF_D3__GPIO2_IO03		0x1b0b0
@@ -720,7 +720,7 @@
 			>;
 		};
 
-		pinctrl_wlan_vmmc: wlan_vmmcgrp {
+		pinctrl_wlan_vmmc: wlan-vmmcgrp {
 			fsl,pins = <
 				MX6QDL_PAD_NANDF_CS0__GPIO6_IO11	0x100b0
 				MX6QDL_PAD_NANDF_CS2__GPIO6_IO15	0x000b0
diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi
new file mode 100644
index 000000000000..d80f21abea62
--- /dev/null
+++ b/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi
@@ -0,0 +1,770 @@
+/*
+ * Copyright 2016 Boundary Devices, Inc.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/ {
+	chosen {
+		stdout-path = &uart2;
+	};
+
+	memory {
+		reg = <0x10000000 0x40000000>;
+	};
+
+	backlight_lcd: backlight-lcd {
+		compatible = "pwm-backlight";
+		pwms = <&pwm1 0 5000000>;
+		brightness-levels = <0 4 8 16 32 64 128 255>;
+		default-brightness-level = <7>;
+		power-supply = <&reg_3p3v>;
+		status = "okay";
+	};
+
+	backlight_lvds0: backlight-lvds0 {
+		compatible = "pwm-backlight";
+		pwms = <&pwm4 0 5000000>;
+		brightness-levels = <0 4 8 16 32 64 128 255>;
+		default-brightness-level = <7>;
+		power-supply = <&reg_3p3v>;
+		status = "okay";
+	};
+
+	backlight_lvds1: backlight-lvds1 {
+		compatible = "gpio-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_backlight_lvds1>;
+		gpios = <&gpio2 31 GPIO_ACTIVE_HIGH>;
+		default-on;
+		status = "okay";
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_keys>;
+
+		power {
+			label = "Power Button";
+			gpios = <&gpio2 3 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_POWER>;
+			wakeup-source;
+		};
+
+		menu {
+			label = "Menu";
+			gpios = <&gpio2 1 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_MENU>;
+		};
+
+		home {
+			label = "Home";
+			gpios = <&gpio2 4 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_HOME>;
+		};
+
+		back {
+			label = "Back";
+			gpios = <&gpio2 2 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_BACK>;
+		};
+
+		volume-up {
+			label = "Volume Up";
+			gpios = <&gpio7 13 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+		};
+
+		volume-down {
+			label = "Volume Down";
+			gpios = <&gpio7 1 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEDOWN>;
+		};
+	};
+
+	lcd_display: display@di0 {
+		compatible = "fsl,imx-parallel-display";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		interface-pix-fmt = "bgr666";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_j15>;
+		status = "okay";
+
+		port@0 {
+			reg = <0>;
+
+			lcd_display_in: endpoint {
+				remote-endpoint = <&ipu1_di0_disp0>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+
+			lcd_display_out: endpoint {
+				remote-endpoint = <&lcd_panel_in>;
+			};
+		};
+	};
+
+	panel-lcd {
+		compatible = "okaya,rs800480t-7x0gp";
+		backlight = <&backlight_lcd>;
+
+		port {
+			lcd_panel_in: endpoint {
+				remote-endpoint = <&lcd_display_out>;
+			};
+		};
+	};
+
+	panel-lvds0 {
+		compatible = "hannstar,hsd100pxn1";
+		backlight = <&backlight_lvds0>;
+
+		port {
+			panel_in_lvds0: endpoint {
+				remote-endpoint = <&lvds0_out>;
+			};
+		};
+	};
+
+	panel-lvds1 {
+		compatible = "hannstar,hsd100pxn1";
+		backlight = <&backlight_lvds1>;
+
+		port {
+			panel_in_lvds1: endpoint {
+				remote-endpoint = <&lvds1_out>;
+			};
+		};
+	};
+
+	reg_1p8v: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "1P8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+	};
+
+	reg_2p5v: regulator-2v5 {
+		compatible = "regulator-fixed";
+		regulator-name = "2P5V";
+		regulator-min-microvolt = <2500000>;
+		regulator-max-microvolt = <2500000>;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "3P3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_can_xcvr: regulator-can-xcvr {
+		compatible = "regulator-fixed";
+		regulator-name = "CAN XCVR";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_can_xcvr>;
+		gpio = <&gpio1 2 GPIO_ACTIVE_LOW>;
+	};
+
+	reg_usb_h1_vbus: regulator-usb-h1-vbus {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usbh1>;
+		regulator-name = "usb_h1_vbus";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio7 12 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
+	reg_usb_otg_vbus: regulator-usb-otg-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_otg_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio3 22 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_wlan_vmmc: regulator-wlan-vmmc {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_wlan_vmmc>;
+		regulator-name = "reg_wlan_vmmc";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio6 15 GPIO_ACTIVE_HIGH>;
+		startup-delay-us = <70000>;
+		enable-active-high;
+	};
+
+	sound {
+		compatible = "fsl,imx6q-nitrogen6_som2-sgtl5000",
+			     "fsl,imx-audio-sgtl5000";
+		model = "imx6q-nitrogen6_som2-sgtl5000";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_sgtl5000>;
+		ssi-controller = <&ssi1>;
+		audio-codec = <&codec>;
+		audio-routing =
+			"MIC_IN", "Mic Jack",
+			"Mic Jack", "Mic Bias",
+			"Headphone Jack", "HP_OUT";
+		mux-int-port = <1>;
+		mux-ext-port = <3>;
+	};
+};
+
+&audmux {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_audmux>;
+	status = "okay";
+};
+
+&can1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_can1>;
+	xceiver-supply = <&reg_can_xcvr>;
+	status = "okay";
+};
+
+&clks {
+	assigned-clocks = <&clks IMX6QDL_CLK_LDB_DI0_SEL>,
+			  <&clks IMX6QDL_CLK_LDB_DI1_SEL>;
+	assigned-clock-parents = <&clks IMX6QDL_CLK_PLL3_USB_OTG>,
+				 <&clks IMX6QDL_CLK_PLL3_USB_OTG>;
+};
+
+&ecspi1 {
+	fsl,spi-num-chipselects = <1>;
+	cs-gpios = <&gpio3 19 GPIO_ACTIVE_HIGH>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi1>;
+	status = "okay";
+
+	flash: m25p80@0 {
+		compatible = "microchip,sst25vf016b";
+		spi-max-frequency = <20000000>;
+		reg = <0>;
+	};
+};
+
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet>;
+	phy-mode = "rgmii";
+	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
+			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
+	fsl,err006687-workaround-present;
+	status = "okay";
+};
+
+&hdmi {
+	ddc-i2c-bus = <&i2c2>;
+	status = "okay";
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	status = "okay";
+
+	codec: sgtl5000@0a {
+		compatible = "fsl,sgtl5000";
+		reg = <0x0a>;
+		clocks = <&clks IMX6QDL_CLK_CKO>;
+		VDDA-supply = <&reg_2p5v>;
+		VDDIO-supply = <&reg_3p3v>;
+	};
+
+	rtc@68 {
+		compatible = "st,rv4162";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_rv4162>;
+		reg = <0x68>;
+		interrupts-extended = <&gpio6 7 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
+
+&i2c2 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	status = "okay";
+};
+
+&i2c3 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	status = "okay";
+
+	touchscreen@04 {
+		compatible = "eeti,egalax_ts";
+		reg = <0x04>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <9 IRQ_TYPE_EDGE_FALLING>;
+		wakeup-gpios = <&gpio1 9 GPIO_ACTIVE_LOW>;
+	};
+
+	touchscreen@38 {
+		compatible = "edt,edt-ft5x06";
+		reg = <0x38>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <9 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&iomuxc {
+	pinctrl_audmux: audmuxgrp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT7__AUD3_RXD		0x130b0
+			MX6QDL_PAD_CSI0_DAT4__AUD3_TXC		0x130b0
+			MX6QDL_PAD_CSI0_DAT5__AUD3_TXD		0x110b0
+			MX6QDL_PAD_CSI0_DAT6__AUD3_TXFS		0x130b0
+		>;
+	};
+
+	pinctrl_backlight_lvds1: backlight-lvds1grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_EB3__GPIO2_IO31		0x0b0b0
+		>;
+	};
+
+	pinctrl_can1: can1grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL2__FLEXCAN1_TX	0x1b0b0
+			MX6QDL_PAD_KEY_ROW2__FLEXCAN1_RX	0x1b0b0
+		>;
+	};
+
+	pinctrl_can_xcvr: can-xcvrgrp {
+		fsl,pins = <
+			/* Flexcan XCVR enable */
+			MX6QDL_PAD_GPIO_2__GPIO1_IO02		0x0b0b0
+		>;
+	};
+
+	pinctrl_ecspi1: ecspi1grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D17__ECSPI1_MISO		0x100b1
+			MX6QDL_PAD_EIM_D18__ECSPI1_MOSI		0x100b1
+			MX6QDL_PAD_EIM_D16__ECSPI1_SCLK		0x100b1
+			MX6QDL_PAD_EIM_D19__GPIO3_IO19		0x000b1
+		>;
+	};
+
+	pinctrl_enet: enetgrp {
+		fsl,pins = <
+			MX6QDL_PAD_ENET_MDIO__ENET_MDIO		0x1b0b0
+			MX6QDL_PAD_ENET_MDC__ENET_MDC		0x1b0b0
+			MX6QDL_PAD_RGMII_TXC__RGMII_TXC		0x100b0
+			MX6QDL_PAD_RGMII_TD0__RGMII_TD0		0x100b0
+			MX6QDL_PAD_RGMII_TD1__RGMII_TD1		0x100b0
+			MX6QDL_PAD_RGMII_TD2__RGMII_TD2		0x100b0
+			MX6QDL_PAD_RGMII_TD3__RGMII_TD3		0x100b0
+			MX6QDL_PAD_RGMII_TX_CTL__RGMII_TX_CTL	0x100b0
+			MX6QDL_PAD_ENET_REF_CLK__ENET_TX_CLK	0x100b0
+			MX6QDL_PAD_RGMII_RXC__RGMII_RXC		0x1b0b0
+			MX6QDL_PAD_RGMII_RD0__RGMII_RD0		0x130b0
+			MX6QDL_PAD_RGMII_RD1__RGMII_RD1		0x1b0b0
+			MX6QDL_PAD_RGMII_RD2__RGMII_RD2		0x130b0
+			MX6QDL_PAD_RGMII_RD3__RGMII_RD3		0x1b0b0
+			MX6QDL_PAD_RGMII_RX_CTL__RGMII_RX_CTL	0x130b0
+			MX6QDL_PAD_ENET_RXD0__GPIO1_IO27	0x030b0
+			MX6QDL_PAD_ENET_TX_EN__GPIO1_IO28	0x1b0b0
+			MX6QDL_PAD_GPIO_6__ENET_IRQ		0x000b1
+		>;
+	};
+
+	pinctrl_gpio_keys: gpio-keysgrp {
+		fsl,pins = <
+			/* Power Button */
+			MX6QDL_PAD_NANDF_D3__GPIO2_IO03		0x1b0b0
+			/* Menu Button */
+			MX6QDL_PAD_NANDF_D1__GPIO2_IO01		0x1b0b0
+			/* Home Button */
+			MX6QDL_PAD_NANDF_D4__GPIO2_IO04		0x1b0b0
+			/* Back Button */
+			MX6QDL_PAD_NANDF_D2__GPIO2_IO02		0x1b0b0
+			/* Volume Up Button */
+			MX6QDL_PAD_GPIO_18__GPIO7_IO13		0x1b0b0
+			/* Volume Down Button */
+			MX6QDL_PAD_SD3_DAT4__GPIO7_IO01		0x1b0b0
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D21__I2C1_SCL	0x4001b8b1
+			MX6QDL_PAD_EIM_D28__I2C1_SDA	0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL3__I2C2_SCL	0x4001b8b1
+			MX6QDL_PAD_KEY_ROW3__I2C2_SDA	0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_5__I2C3_SCL	0x4001b8b1
+			MX6QDL_PAD_GPIO_16__I2C3_SDA	0x4001b8b1
+			MX6QDL_PAD_GPIO_9__GPIO1_IO09	0x1b0b0
+		>;
+	};
+
+	pinctrl_i2c3mux: i2c3muxgrp {
+		fsl,pins = <
+			/* PCIe I2C enable */
+			MX6QDL_PAD_EIM_OE__GPIO2_IO25	0x000b0
+		>;
+	};
+
+	pinctrl_j15: j15grp {
+		fsl,pins = <
+			MX6QDL_PAD_DI0_DISP_CLK__IPU1_DI0_DISP_CLK 0x10
+			MX6QDL_PAD_DI0_PIN15__IPU1_DI0_PIN15       0x10
+			MX6QDL_PAD_DI0_PIN2__IPU1_DI0_PIN02        0x10
+			MX6QDL_PAD_DI0_PIN3__IPU1_DI0_PIN03        0x10
+			MX6QDL_PAD_DISP0_DAT0__IPU1_DISP0_DATA00   0x10
+			MX6QDL_PAD_DISP0_DAT1__IPU1_DISP0_DATA01   0x10
+			MX6QDL_PAD_DISP0_DAT2__IPU1_DISP0_DATA02   0x10
+			MX6QDL_PAD_DISP0_DAT3__IPU1_DISP0_DATA03   0x10
+			MX6QDL_PAD_DISP0_DAT4__IPU1_DISP0_DATA04   0x10
+			MX6QDL_PAD_DISP0_DAT5__IPU1_DISP0_DATA05   0x10
+			MX6QDL_PAD_DISP0_DAT6__IPU1_DISP0_DATA06   0x10
+			MX6QDL_PAD_DISP0_DAT7__IPU1_DISP0_DATA07   0x10
+			MX6QDL_PAD_DISP0_DAT8__IPU1_DISP0_DATA08   0x10
+			MX6QDL_PAD_DISP0_DAT9__IPU1_DISP0_DATA09   0x10
+			MX6QDL_PAD_DISP0_DAT10__IPU1_DISP0_DATA10  0x10
+			MX6QDL_PAD_DISP0_DAT11__IPU1_DISP0_DATA11  0x10
+			MX6QDL_PAD_DISP0_DAT12__IPU1_DISP0_DATA12  0x10
+			MX6QDL_PAD_DISP0_DAT13__IPU1_DISP0_DATA13  0x10
+			MX6QDL_PAD_DISP0_DAT14__IPU1_DISP0_DATA14  0x10
+			MX6QDL_PAD_DISP0_DAT15__IPU1_DISP0_DATA15  0x10
+			MX6QDL_PAD_DISP0_DAT16__IPU1_DISP0_DATA16  0x10
+			MX6QDL_PAD_DISP0_DAT17__IPU1_DISP0_DATA17  0x10
+			MX6QDL_PAD_DISP0_DAT18__IPU1_DISP0_DATA18  0x10
+			MX6QDL_PAD_DISP0_DAT19__IPU1_DISP0_DATA19  0x10
+			MX6QDL_PAD_DISP0_DAT20__IPU1_DISP0_DATA20  0x10
+			MX6QDL_PAD_DISP0_DAT21__IPU1_DISP0_DATA21  0x10
+			MX6QDL_PAD_DISP0_DAT22__IPU1_DISP0_DATA22  0x10
+			MX6QDL_PAD_DISP0_DAT23__IPU1_DISP0_DATA23  0x10
+		>;
+	};
+
+	pinctrl_pcie: pciegrp {
+		fsl,pins = <
+			/* PCIe reset */
+			MX6QDL_PAD_EIM_BCLK__GPIO6_IO31	0x030b0
+			MX6QDL_PAD_EIM_DA4__GPIO3_IO04	0x030b0
+		>;
+	};
+
+	pinctrl_pwm1: pwm1grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD1_DAT3__PWM1_OUT	0x030b1
+		>;
+	};
+
+	pinctrl_pwm3: pwm3grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD1_DAT1__PWM3_OUT	0x030b1
+		>;
+	};
+
+	pinctrl_pwm4: pwm4grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD1_CMD__PWM4_OUT	0x030b1
+		>;
+	};
+
+	pinctrl_rv4162: rv4162grp {
+		fsl,pins = <
+			MX6QDL_PAD_NANDF_CLE__GPIO6_IO07	0x1b0b0
+		>;
+	};
+
+	pinctrl_sgtl5000: sgtl5000grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_0__CCM_CLKO1		0x000b0
+			MX6QDL_PAD_EIM_D29__GPIO3_IO29		0x130b0
+			MX6QDL_PAD_EIM_DA2__GPIO3_IO02		0x130b0
+			MX6QDL_PAD_ENET_RX_ER__GPIO1_IO24	0x130b0
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_DAT7__UART1_TX_DATA	0x1b0b1
+			MX6QDL_PAD_SD3_DAT6__UART1_RX_DATA	0x1b0b1
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D26__UART2_TX_DATA	0x1b0b1
+			MX6QDL_PAD_EIM_D27__UART2_RX_DATA	0x1b0b1
+		>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D24__UART3_TX_DATA	0x1b0b1
+			MX6QDL_PAD_EIM_D25__UART3_RX_DATA	0x1b0b1
+			MX6QDL_PAD_EIM_D23__UART3_CTS_B		0x1b0b1
+			MX6QDL_PAD_EIM_D31__UART3_RTS_B		0x1b0b1
+		>;
+	};
+
+	pinctrl_usbh1: usbh1grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_17__GPIO7_IO12		0x030b0
+		>;
+	};
+
+	pinctrl_usbotg: usbotggrp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_1__USB_OTG_ID		0x17059
+			MX6QDL_PAD_KEY_COL4__USB_OTG_OC		0x1b0b0
+			/* power enable, high active */
+			MX6QDL_PAD_EIM_D22__GPIO3_IO22		0x030b0
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD2_CLK__SD2_CLK		0x10071
+			MX6QDL_PAD_SD2_CMD__SD2_CMD		0x17071
+			MX6QDL_PAD_SD2_DAT0__SD2_DATA0		0x17071
+			MX6QDL_PAD_SD2_DAT1__SD2_DATA1		0x17071
+			MX6QDL_PAD_SD2_DAT2__SD2_DATA2		0x17071
+			MX6QDL_PAD_SD2_DAT3__SD2_DATA3		0x17071
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_CLK__SD3_CLK		0x10071
+			MX6QDL_PAD_SD3_CMD__SD3_CMD		0x17071
+			MX6QDL_PAD_SD3_DAT0__SD3_DATA0		0x17071
+			MX6QDL_PAD_SD3_DAT1__SD3_DATA1		0x17071
+			MX6QDL_PAD_SD3_DAT2__SD3_DATA2		0x17071
+			MX6QDL_PAD_SD3_DAT3__SD3_DATA3		0x17071
+			MX6QDL_PAD_SD3_DAT5__GPIO7_IO00		0x1b0b0
+		>;
+	};
+
+	pinctrl_usdhc4: usdhc4grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD4_CMD__SD4_CMD		0x17059
+			MX6QDL_PAD_SD4_CLK__SD4_CLK		0x10059
+			MX6QDL_PAD_SD4_DAT0__SD4_DATA0		0x17059
+			MX6QDL_PAD_SD4_DAT1__SD4_DATA1		0x17059
+			MX6QDL_PAD_SD4_DAT2__SD4_DATA2		0x17059
+			MX6QDL_PAD_SD4_DAT3__SD4_DATA3		0x17059
+			MX6QDL_PAD_SD4_DAT4__SD4_DATA4		0x17059
+			MX6QDL_PAD_SD4_DAT5__SD4_DATA5		0x17059
+			MX6QDL_PAD_SD4_DAT6__SD4_DATA6		0x17059
+			MX6QDL_PAD_SD4_DAT7__SD4_DATA7		0x17059
+		>;
+	};
+
+	pinctrl_wlan_vmmc: wlan-vmmcgrp {
+		fsl,pins = <
+			MX6QDL_PAD_NANDF_CS1__GPIO6_IO14	0x100b0
+			MX6QDL_PAD_NANDF_CS2__GPIO6_IO15	0x030b0
+			MX6QDL_PAD_NANDF_CS3__GPIO6_IO16	0x030b0
+			MX6QDL_PAD_SD1_CLK__OSC32K_32K_OUT	0x000b0
+		>;
+	};
+};
+
+&ipu1_di0_disp0 {
+	remote-endpoint = <&lcd_display_in>;
+};
+
+&ldb {
+	status = "okay";
+
+	lvds-channel@0 {
+		fsl,data-mapping = "spwg";
+		fsl,data-width = <18>;
+		status = "okay";
+
+		port@4 {
+			reg = <4>;
+
+			lvds0_out: endpoint {
+				remote-endpoint = <&panel_in_lvds0>;
+			};
+		};
+	};
+
+	lvds-channel@1 {
+		fsl,data-mapping = "spwg";
+		fsl,data-width = <18>;
+		status = "okay";
+
+		port@4 {
+			reg = <4>;
+
+			lvds1_out: endpoint {
+				remote-endpoint = <&panel_in_lvds1>;
+			};
+		};
+	};
+};
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pcie>;
+	reset-gpio = <&gpio6 31 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&pwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm1>;
+	status = "okay";
+};
+
+&pwm3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm3>;
+	status = "okay";
+};
+
+&pwm4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm4>;
+	status = "okay";
+};
+
+&ssi1 {
+	status = "okay";
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	status = "okay";
+};
+
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&usbh1 {
+	vbus-supply = <&reg_usb_h1_vbus>;
+	status = "okay";
+};
+
+&usbotg {
+	vbus-supply = <&reg_usb_otg_vbus>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usbotg>;
+	disable-over-current;
+	status = "okay";
+};
+
+&usdhc2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc2>;
+	bus-width = <4>;
+	non-removable;
+	vmmc-supply = <&reg_wlan_vmmc>;
+	cap-power-off-card;
+	keep-power-in-suspend;
+	status = "okay";
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+	wlcore: wlcore@2 {
+		compatible = "ti,wl1271";
+		reg = <2>;
+		interrupt-parent = <&gpio6>;
+		interrupts = <14 IRQ_TYPE_LEVEL_HIGH>;
+		ref-clock-frequency = <38400000>;
+	};
+};
+
+&usdhc3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	cd-gpios = <&gpio7 0 GPIO_ACTIVE_LOW>;
+	bus-width = <4>;
+	vmmc-supply = <&reg_3p3v>;
+	status = "okay";
+};
+
+&usdhc4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc4>;
+	bus-width = <8>;
+	non-removable;
+	vmmc-supply = <&reg_1p8v>;
+	keep-power-in-suspend;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi
index db868bc42c0f..e476d01959ea 100644
--- a/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi
@@ -167,7 +167,7 @@
 		mux-ext-port = <3>;
 	};
 
-	backlight_lcd: backlight_lcd {
+	backlight_lcd: backlight-lcd {
 		compatible = "pwm-backlight";
 		pwms = <&pwm1 0 5000000>;
 		brightness-levels = <0 4 8 16 32 64 128 255>;
@@ -176,7 +176,7 @@
 		status = "okay";
 	};
 
-	backlight_lvds: backlight_lvds {
+	backlight_lvds: backlight-lvds {
 		compatible = "pwm-backlight";
 		pwms = <&pwm4 0 5000000>;
 		brightness-levels = <0 4 8 16 32 64 128 255>;
@@ -211,7 +211,7 @@
 		};
 	};
 
-	lcd_panel {
+	panel-lcd {
 		compatible = "okaya,rs800480t-7x0gp";
 		backlight = <&backlight_lcd>;
 
@@ -222,7 +222,7 @@
 		};
 	};
 
-	panel {
+	panel-lvds0 {
 		compatible = "hannstar,hsd100pxn1";
 		backlight = <&backlight_lvds>;
 
@@ -413,7 +413,7 @@
 			>;
 		};
 
-		pinctrl_gpio_keys: gpio_keysgrp {
+		pinctrl_gpio_keys: gpio-keysgrp {
 			fsl,pins = <
 				/* Power Button */
 				MX6QDL_PAD_NANDF_D3__GPIO2_IO03		0x1b0b0
@@ -561,7 +561,7 @@
 			>;
 		};
 
-		pinctrl_wlan_vmmc: wlan_vmmcgrp {
+		pinctrl_wlan_vmmc: wlan-vmmcgrp {
 			fsl,pins = <
 				MX6QDL_PAD_NANDF_CS0__GPIO6_IO11	0x100b0
 				MX6QDL_PAD_NANDF_CS2__GPIO6_IO15	0x000b0
diff --git a/arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi b/arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi
index e0280cac2484..e9801a26f3b4 100644
--- a/arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-phytec-pfla02.dtsi
@@ -427,10 +427,10 @@
 };
 
 &usdhc3 {
-        pinctrl-names = "default";
-        pinctrl-0 = <&pinctrl_usdhc3
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc3
 		     &pinctrl_usdhc3_cdwp>;
 	cd-gpios = <&gpio1 27 GPIO_ACTIVE_LOW>;
 	wp-gpios = <&gpio1 29 GPIO_ACTIVE_HIGH>;
-        status = "disabled";
+	status = "disabled";
 };
diff --git a/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi b/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi
index e000e6f12bf5..52390ba83e81 100644
--- a/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi
@@ -283,7 +283,7 @@
 		VD-supply = <&reg_audio>;
 		VLS-supply = <&reg_audio>;
 		VLC-supply = <&reg_audio>;
-        };
+	};
 
 };
 
@@ -613,8 +613,6 @@
 &weim {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_weim_nor &pinctrl_weim_cs0>;
-	#address-cells = <2>;
-	#size-cells = <1>;
 	ranges = <0 0 0x08000000 0x08000000>;
 	status = "disabled"; /* pin conflict with SPI NOR */
 
diff --git a/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi b/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi
index 81dd6cd1937d..1f9076e271e4 100644
--- a/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi
@@ -153,7 +153,7 @@
 		mux-ext-port = <4>;
 	};
 
-	backlight_lcd: backlight_lcd {
+	backlight_lcd: backlight-lcd {
 		compatible = "pwm-backlight";
 		pwms = <&pwm1 0 5000000>;
 		brightness-levels = <0 4 8 16 32 64 128 255>;
@@ -162,7 +162,7 @@
 		status = "okay";
 	};
 
-	backlight_lvds: backlight_lvds {
+	backlight_lvds: backlight-lvds {
 		compatible = "pwm-backlight";
 		pwms = <&pwm4 0 5000000>;
 		brightness-levels = <0 4 8 16 32 64 128 255>;
@@ -197,7 +197,7 @@
 		};
 	};
 
-	lcd_panel {
+	panel-lcd {
 		compatible = "okaya,rs800480t-7x0gp";
 		backlight = <&backlight_lcd>;
 
@@ -208,7 +208,7 @@
 		};
 	};
 
-	panel {
+	panel-lvds0 {
 		compatible = "hannstar,hsd100pxn1";
 		backlight = <&backlight_lvds>;
 
@@ -378,7 +378,7 @@
 			>;
 		};
 
-		pinctrl_gpio_keys: gpio_keysgrp {
+		pinctrl_gpio_keys: gpio-keysgrp {
 			fsl,pins = <
 				/* Power Button */
 				MX6QDL_PAD_NANDF_D3__GPIO2_IO03		0x1b0b0
diff --git a/arch/arm/boot/dts/imx6qdl-sabresd.dtsi b/arch/arm/boot/dts/imx6qdl-sabresd.dtsi
index 8e9e0d98db2f..55ef53571fdd 100644
--- a/arch/arm/boot/dts/imx6qdl-sabresd.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-sabresd.dtsi
@@ -129,8 +129,8 @@
 		pinctrl-0 = <&pinctrl_gpio_leds>;
 
 		red {
-		        gpios = <&gpio1 2 0>;
-		        default-state = "on";
+			gpios = <&gpio1 2 0>;
+			default-state = "on";
 		};
 	};
 
diff --git a/arch/arm/boot/dts/imx6qdl-tx6.dtsi b/arch/arm/boot/dts/imx6qdl-tx6.dtsi
index ac9529f85593..2bf2e623ac1e 100644
--- a/arch/arm/boot/dts/imx6qdl-tx6.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-tx6.dtsi
@@ -429,8 +429,8 @@
 	pinctrl_edt_ft5x06: edt-ft5x06grp {
 		fsl,pins = <
 			MX6QDL_PAD_NANDF_CS2__GPIO6_IO15	0x1b0b0 /* Interrupt */
-			MX6QDL_PAD_EIM_A16__GPIO2_IO22  	0x1b0b0 /* Reset */
-			MX6QDL_PAD_EIM_A17__GPIO2_IO21  	0x1b0b0 /* Wake */
+			MX6QDL_PAD_EIM_A16__GPIO2_IO22		0x1b0b0 /* Reset */
+			MX6QDL_PAD_EIM_A17__GPIO2_IO21		0x1b0b0 /* Wake */
 		>;
 	};
 
@@ -481,21 +481,21 @@
 
 	pinctrl_gpmi_nand: gpminandgrp {
 		fsl,pins = <
-			MX6QDL_PAD_NANDF_CLE__NAND_CLE    	0x0b0b1
-			MX6QDL_PAD_NANDF_ALE__NAND_ALE    	0x0b0b1
-			MX6QDL_PAD_NANDF_WP_B__NAND_WP_B  	0x0b0b1
+			MX6QDL_PAD_NANDF_CLE__NAND_CLE		0x0b0b1
+			MX6QDL_PAD_NANDF_ALE__NAND_ALE		0x0b0b1
+			MX6QDL_PAD_NANDF_WP_B__NAND_WP_B	0x0b0b1
 			MX6QDL_PAD_NANDF_RB0__NAND_READY_B	0x0b000
-			MX6QDL_PAD_NANDF_CS0__NAND_CE0_B  	0x0b0b1
-			MX6QDL_PAD_SD4_CMD__NAND_RE_B     	0x0b0b1
-			MX6QDL_PAD_SD4_CLK__NAND_WE_B     	0x0b0b1
-			MX6QDL_PAD_NANDF_D0__NAND_DATA00  	0x0b0b1
-			MX6QDL_PAD_NANDF_D1__NAND_DATA01  	0x0b0b1
-			MX6QDL_PAD_NANDF_D2__NAND_DATA02  	0x0b0b1
-			MX6QDL_PAD_NANDF_D3__NAND_DATA03  	0x0b0b1
-			MX6QDL_PAD_NANDF_D4__NAND_DATA04  	0x0b0b1
-			MX6QDL_PAD_NANDF_D5__NAND_DATA05  	0x0b0b1
-			MX6QDL_PAD_NANDF_D6__NAND_DATA06  	0x0b0b1
-			MX6QDL_PAD_NANDF_D7__NAND_DATA07  	0x0b0b1
+			MX6QDL_PAD_NANDF_CS0__NAND_CE0_B	0x0b0b1
+			MX6QDL_PAD_SD4_CMD__NAND_RE_B		0x0b0b1
+			MX6QDL_PAD_SD4_CLK__NAND_WE_B		0x0b0b1
+			MX6QDL_PAD_NANDF_D0__NAND_DATA00	0x0b0b1
+			MX6QDL_PAD_NANDF_D1__NAND_DATA01	0x0b0b1
+			MX6QDL_PAD_NANDF_D2__NAND_DATA02	0x0b0b1
+			MX6QDL_PAD_NANDF_D3__NAND_DATA03	0x0b0b1
+			MX6QDL_PAD_NANDF_D4__NAND_DATA04	0x0b0b1
+			MX6QDL_PAD_NANDF_D5__NAND_DATA05	0x0b0b1
+			MX6QDL_PAD_NANDF_D6__NAND_DATA06	0x0b0b1
+			MX6QDL_PAD_NANDF_D7__NAND_DATA07	0x0b0b1
 		>;
 	};
 
diff --git a/arch/arm/boot/dts/imx6qdl-wandboard-revb1.dtsi b/arch/arm/boot/dts/imx6qdl-wandboard-revb1.dtsi
index ef7fa62b9898..a32089132263 100644
--- a/arch/arm/boot/dts/imx6qdl-wandboard-revb1.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-wandboard-revb1.dtsi
@@ -28,7 +28,7 @@
 				MX6QDL_PAD_EIM_D29__GPIO3_IO29		0x80000000	/* RGMII_nRST */
 				MX6QDL_PAD_EIM_DA13__GPIO3_IO13		0x80000000	/* BT_ON */
 				MX6QDL_PAD_EIM_DA14__GPIO3_IO14		0x80000000	/* BT_WAKE */
-				MX6QDL_PAD_EIM_DA15__GPIO3_IO15		0x80000000	/* BT_HOST_WAKE */				
+				MX6QDL_PAD_EIM_DA15__GPIO3_IO15		0x80000000	/* BT_HOST_WAKE */
 			>;
 		};
 	};
diff --git a/arch/arm/boot/dts/imx6qdl-wandboard.dtsi b/arch/arm/boot/dts/imx6qdl-wandboard.dtsi
index 2b9c2be436f9..82dc5744ae19 100644
--- a/arch/arm/boot/dts/imx6qdl-wandboard.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-wandboard.dtsi
@@ -129,8 +129,8 @@
 
 		pinctrl_i2c1: i2c1grp {
 			fsl,pins = <
-				MX6QDL_PAD_EIM_D21__I2C1_SCL 		0x4001b8b1
-				MX6QDL_PAD_EIM_D28__I2C1_SDA 		0x4001b8b1
+				MX6QDL_PAD_EIM_D21__I2C1_SCL		0x4001b8b1
+				MX6QDL_PAD_EIM_D28__I2C1_SDA		0x4001b8b1
 			>;
 		};
 
diff --git a/arch/arm/boot/dts/imx6qdl.dtsi b/arch/arm/boot/dts/imx6qdl.dtsi
index b13b0b2db881..53e6e63cbb02 100644
--- a/arch/arm/boot/dts/imx6qdl.dtsi
+++ b/arch/arm/boot/dts/imx6qdl.dtsi
@@ -13,9 +13,10 @@
 #include <dt-bindings/clock/imx6qdl-clock.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 
-#include "skeleton.dtsi"
-
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		ethernet0 = &fec;
 		can0 = &can1;
@@ -204,9 +205,9 @@
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
 			interrupt-map = <0 0 0 1 &gpc GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
-			                <0 0 0 2 &gpc GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>,
-			                <0 0 0 3 &gpc GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>,
-			                <0 0 0 4 &gpc GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
+					<0 0 0 2 &gpc GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &gpc GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &gpc GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clks IMX6QDL_CLK_PCIE_AXI>,
 				 <&clks IMX6QDL_CLK_LVDS1_GATE>,
 				 <&clks IMX6QDL_CLK_PCIE_REF_125M>;
@@ -1092,10 +1093,13 @@
 			};
 
 			weim: weim@021b8000 {
+				#address-cells = <2>;
+				#size-cells = <1>;
 				compatible = "fsl,imx6q-weim";
 				reg = <0x021b8000 0x4000>;
 				interrupts = <0 14 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clks IMX6QDL_CLK_EIM_SLOW>;
+				fsl,weim-cs-gpr = <&gpr>;
 			};
 
 			ocotp: ocotp@021bc000 {
diff --git a/arch/arm/boot/dts/imx6qp.dtsi b/arch/arm/boot/dts/imx6qp.dtsi
index 886dbf2eca49..0d4977ab7d29 100644
--- a/arch/arm/boot/dts/imx6qp.dtsi
+++ b/arch/arm/boot/dts/imx6qp.dtsi
@@ -85,5 +85,22 @@
 		pcie: pcie@0x01000000 {
 			compatible = "fsl,imx6qp-pcie", "snps,dw-pcie";
 		};
+
+		aips-bus@02100000 {
+			mmdc0: mmdc@021b0000 { /* MMDC0 */
+				compatible = "fsl,imx6qp-mmdc", "fsl,imx6q-mmdc";
+				reg = <0x021b0000 0x4000>;
+			};
+		};
 	};
 };
+
+&ldb {
+	clocks = <&clks IMX6QDL_CLK_LDB_DI0_SEL>, <&clks IMX6QDL_CLK_LDB_DI1_SEL>,
+		 <&clks IMX6QDL_CLK_IPU1_DI0_SEL>, <&clks IMX6QDL_CLK_IPU1_DI1_SEL>,
+		 <&clks IMX6QDL_CLK_IPU2_DI0_SEL>, <&clks IMX6QDL_CLK_IPU2_DI1_SEL>,
+		 <&clks IMX6QDL_CLK_LDB_DI0_PODF>, <&clks IMX6QDL_CLK_LDB_DI1_PODF>;
+	clock-names = "di0_pll", "di1_pll",
+		      "di0_sel", "di1_sel", "di2_sel", "di3_sel",
+		      "di0", "di1";
+};
diff --git a/arch/arm/boot/dts/imx6sl.dtsi b/arch/arm/boot/dts/imx6sl.dtsi
index 02378db3f5fc..4fd6de29f07d 100644
--- a/arch/arm/boot/dts/imx6sl.dtsi
+++ b/arch/arm/boot/dts/imx6sl.dtsi
@@ -8,11 +8,13 @@
  */
 
 #include <dt-bindings/interrupt-controller/irq.h>
-#include "skeleton.dtsi"
 #include "imx6sl-pinfunc.h"
 #include <dt-bindings/clock/imx6sl-clock.h>
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		ethernet0 = &fec;
 		gpio0 = &gpio1;
@@ -893,8 +895,11 @@
 			};
 
 			weim: weim@021b8000 {
+				#address-cells = <2>;
+				#size-cells = <1>;
 				reg = <0x021b8000 0x4000>;
 				interrupts = <0 14 IRQ_TYPE_LEVEL_HIGH>;
+				fsl,weim-cs-gpr = <&gpr>;
 			};
 
 			ocotp: ocotp@021bc000 {
diff --git a/arch/arm/boot/dts/imx6sx-sdb.dtsi b/arch/arm/boot/dts/imx6sx-sdb.dtsi
index 9d70cfd40aff..da815527a7f8 100644
--- a/arch/arm/boot/dts/imx6sx-sdb.dtsi
+++ b/arch/arm/boot/dts/imx6sx-sdb.dtsi
@@ -192,10 +192,10 @@
 };
 
 &i2c4 {
-        clock-frequency = <100000>;
-        pinctrl-names = "default";
-        pinctrl-0 = <&pinctrl_i2c4>;
-        status = "okay";
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c4>;
+	status = "okay";
 
 	codec: wm8962@1a {
 		compatible = "wlf,wm8962";
@@ -290,6 +290,14 @@
 	status = "okay";
 };
 
+&usbphy1 {
+	fsl,tx-d-cal = <106>;
+};
+
+&usbphy2 {
+	fsl,tx-d-cal = <106>;
+};
+
 &usdhc2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_usdhc2>;
diff --git a/arch/arm/boot/dts/imx6sx-udoo-neo-basic.dts b/arch/arm/boot/dts/imx6sx-udoo-neo-basic.dts
new file mode 100644
index 000000000000..0c1fc1a8f913
--- /dev/null
+++ b/arch/arm/boot/dts/imx6sx-udoo-neo-basic.dts
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2016 Andreas Färber
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "imx6sx-udoo-neo.dtsi"
+
+/ {
+	model = "UDOO Neo Basic";
+	compatible = "udoo,neobasic", "fsl,imx6sx";
+
+	memory {
+		reg = <0x80000000 0x20000000>;
+	};
+};
+
+&fec1 {
+	phy-handle = <&ethphy1>;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy1: ethernet-phy@0 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <0>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/imx6sx-udoo-neo-extended.dts b/arch/arm/boot/dts/imx6sx-udoo-neo-extended.dts
new file mode 100644
index 000000000000..5d6c2274ee2b
--- /dev/null
+++ b/arch/arm/boot/dts/imx6sx-udoo-neo-extended.dts
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2016 Andreas Färber
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "imx6sx-udoo-neo.dtsi"
+
+/ {
+	model = "UDOO Neo Extended";
+	compatible = "udoo,neoextended", "fsl,imx6sx";
+
+	memory {
+		reg = <0x80000000 0x40000000>;
+	};
+};
diff --git a/arch/arm/boot/dts/imx6sx-udoo-neo-full.dts b/arch/arm/boot/dts/imx6sx-udoo-neo-full.dts
new file mode 100644
index 000000000000..653ceb29e28b
--- /dev/null
+++ b/arch/arm/boot/dts/imx6sx-udoo-neo-full.dts
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2016 Andreas Färber
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "imx6sx-udoo-neo.dtsi"
+
+/ {
+	model = "UDOO Neo Full";
+	compatible = "udoo,neofull", "fsl,imx6sx";
+
+	memory {
+		reg = <0x80000000 0x40000000>;
+	};
+};
+
+&fec1 {
+	phy-handle = <&ethphy1>;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy1: ethernet-phy@0 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <0>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/imx6sx-udoo-neo.dtsi b/arch/arm/boot/dts/imx6sx-udoo-neo.dtsi
new file mode 100644
index 000000000000..2b65d26f4396
--- /dev/null
+++ b/arch/arm/boot/dts/imx6sx-udoo-neo.dtsi
@@ -0,0 +1,293 @@
+/*
+ * Copyright (c) 2016 Andreas Färber
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "imx6sx.dtsi"
+
+/ {
+	compatible = "fsl,imx6sx";
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		red {
+			label = "udoo-neo:red:mmc";
+			gpios = <&gpio6 0 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "mmc0";
+		};
+
+		orange {
+			label = "udoo-neo:orange:user";
+			gpios = <&gpio4 6 GPIO_ACTIVE_HIGH>;
+			default-state = "keep";
+		};
+	};
+
+	reg_sdio_pwr: regulator-sdio-pwr {
+		compatible = "regulator-fixed";
+		gpio = <&gpio6 1 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-name = "SDIO_PWR";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+	};
+};
+
+&cpu0 {
+	arm-supply = <&sw1a_reg>;
+	soc-supply = <&sw1c_reg>;
+};
+
+&fec1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet1>;
+	phy-mode = "rmii";
+	phy-reset-gpios = <&gpio2 1 GPIO_ACTIVE_LOW>;
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	clock-frequency = <100000>;
+	status = "okay";
+
+	pmic: pmic@08 {
+		compatible = "fsl,pfuze3000";
+		reg = <0x08>;
+
+		regulators {
+			sw1a_reg: sw1a {
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1475000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <6250>;
+			};
+
+			sw1c_reg: sw1b {
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1475000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <6250>;
+			};
+
+			sw2_reg: sw2 {
+				regulator-min-microvolt = <1500000>;
+				regulator-max-microvolt = <1850000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			sw3a_reg: sw3 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <1650000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			swbst_reg: swbst {
+				regulator-min-microvolt = <5000000>;
+				regulator-max-microvolt = <5150000>;
+			};
+
+			snvs_reg: vsnvs {
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vref_reg: vrefddr {
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vgen1_reg: vldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			vgen2_reg: vldo2 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1550000>;
+			};
+
+			vgen3_reg: vccsd {
+				regulator-min-microvolt = <2850000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			vgen4_reg: v33 {
+				regulator-min-microvolt = <2850000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			vgen5_reg: vldo3 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			vgen6_reg: vldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&iomuxc {
+	pinctrl_enet1: enet1grp {
+		fsl,pins =
+			<MX6SX_PAD_ENET1_CRS__GPIO2_IO_1	0xa0b1>,
+			<MX6SX_PAD_ENET1_MDC__ENET1_MDC		0xa0b1>,
+			<MX6SX_PAD_ENET1_MDIO__ENET1_MDIO	0xa0b1>,
+			<MX6SX_PAD_RGMII1_TD0__ENET1_TX_DATA_0	0xa0b1>,
+			<MX6SX_PAD_RGMII1_TD1__ENET1_TX_DATA_1	0xa0b1>,
+			<MX6SX_PAD_RGMII1_TX_CTL__ENET1_TX_EN	0xa0b1>,
+
+			<MX6SX_PAD_ENET1_TX_CLK__ENET1_REF_CLK1	0x3081>,
+			<MX6SX_PAD_ENET2_TX_CLK__GPIO2_IO_9	0x3081>,
+			<MX6SX_PAD_RGMII1_RD0__ENET1_RX_DATA_0	0x3081>,
+			<MX6SX_PAD_RGMII1_RD1__ENET1_RX_DATA_1	0x3081>,
+			<MX6SX_PAD_RGMII1_RX_CTL__ENET1_RX_EN	0x3081>,
+			<MX6SX_PAD_RGMII1_RXC__ENET1_RX_ER	0x3081>,
+
+			<MX6SX_PAD_ENET2_RX_CLK__ENET2_REF_CLK_25M	0x91>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins =
+			<MX6SX_PAD_GPIO1_IO00__I2C1_SCL		0x4001b8b1>,
+			<MX6SX_PAD_GPIO1_IO01__I2C1_SDA		0x4001b8b1>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins =
+			<MX6SX_PAD_GPIO1_IO04__UART1_TX		0x1b0b1>,
+			<MX6SX_PAD_GPIO1_IO05__UART1_RX		0x1b0b1>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins =
+			<MX6SX_PAD_GPIO1_IO06__UART2_TX		0x1b0b1>,
+			<MX6SX_PAD_GPIO1_IO07__UART2_RX		0x1b0b1>;
+	};
+
+	pinctrl_uart5: uart5grp {
+		fsl,pins =
+			<MX6SX_PAD_SD4_DATA4__UART5_RX		0x1b0b1>,
+			<MX6SX_PAD_SD4_DATA5__UART5_TX		0x1b0b1>;
+	};
+
+	pinctrl_uart6: uart6grp {
+		fsl,pins =
+			<MX6SX_PAD_CSI_DATA00__UART6_RI_B	0x1b0b1>,
+			<MX6SX_PAD_CSI_DATA01__UART6_DSR_B	0x1b0b1>,
+			<MX6SX_PAD_CSI_DATA02__UART6_DTR_B	0x1b0b1>,
+			<MX6SX_PAD_CSI_DATA03__UART6_DCD_B	0x1b0b1>,
+			<MX6SX_PAD_CSI_DATA04__UART6_RX		0x1b0b1>,
+			<MX6SX_PAD_CSI_DATA05__UART6_TX		0x1b0b1>,
+			<MX6SX_PAD_CSI_DATA06__UART6_RTS_B	0x1b0b1>,
+			<MX6SX_PAD_CSI_DATA07__UART6_CTS_B	0x1b0b1>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins =
+			<MX6SX_PAD_SD2_CMD__USDHC2_CMD		0x17059>,
+			<MX6SX_PAD_SD2_CLK__USDHC2_CLK		0x10059>,
+			<MX6SX_PAD_SD2_DATA0__USDHC2_DATA0	0x17059>,
+			<MX6SX_PAD_SD2_DATA1__USDHC2_DATA1	0x17059>,
+			<MX6SX_PAD_SD2_DATA2__USDHC2_DATA2	0x17059>,
+			<MX6SX_PAD_SD2_DATA3__USDHC2_DATA3	0x17059>,
+			<MX6SX_PAD_SD1_DATA0__GPIO6_IO_2	0x17059>; /* CD */
+	};
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+/* Cortex-M4 serial */
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	status = "disabled";
+};
+
+/* Arduino serial */
+&uart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart5>;
+	status = "disabled";
+};
+
+&uart6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart6>;
+	uart-has-rtscts;
+	status = "disabled";
+};
+
+&usdhc2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc2>;
+	vmmc-supply = <&reg_sdio_pwr>;
+	bus-width = <4>;
+	cd-gpios = <&gpio6 2 GPIO_ACTIVE_LOW>;
+	no-1-8-v;
+	keep-power-in-suspend;
+	wakeup-source;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6sx.dtsi b/arch/arm/boot/dts/imx6sx.dtsi
index 1a473e83efbf..076a30f9bcae 100644
--- a/arch/arm/boot/dts/imx6sx.dtsi
+++ b/arch/arm/boot/dts/imx6sx.dtsi
@@ -11,9 +11,11 @@
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include "imx6sx-pinfunc.h"
-#include "skeleton.dtsi"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		can0 = &flexcan1;
 		can1 = &flexcan2;
@@ -858,7 +860,7 @@
 				fsl,num-tx-queues=<3>;
 				fsl,num-rx-queues=<3>;
 				status = "disabled";
-                        };
+			};
 
 			mlb: mlb@0218c000 {
 				reg = <0x0218c000 0x4000>;
@@ -968,10 +970,13 @@
 			};
 
 			weim: weim@021b8000 {
+				#address-cells = <2>;
+				#size-cells = <1>;
 				compatible = "fsl,imx6sx-weim", "fsl,imx6q-weim";
 				reg = <0x021b8000 0x4000>;
 				interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clks IMX6SX_CLK_EIM_SLOW>;
+				fsl,weim-cs-gpr = <&gpr>;
 			};
 
 			ocotp: ocotp@021bc000 {
@@ -1143,7 +1148,7 @@
 				lcdif1: lcdif@02220000 {
 					compatible = "fsl,imx6sx-lcdif", "fsl,imx28-lcdif";
 					reg = <0x02220000 0x4000>;
-					interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+					interrupts = <GIC_SPI 5 IRQ_TYPE_EDGE_RISING>;
 					clocks = <&clks IMX6SX_CLK_LCDIF1_PIX>,
 						 <&clks IMX6SX_CLK_LCDIF_APB>,
 						 <&clks IMX6SX_CLK_DISPLAY_AXI>;
@@ -1154,7 +1159,7 @@
 				lcdif2: lcdif@02224000 {
 					compatible = "fsl,imx6sx-lcdif", "fsl,imx28-lcdif";
 					reg = <0x02224000 0x4000>;
-					interrupts = <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
+					interrupts = <GIC_SPI 6 IRQ_TYPE_EDGE_RISING>;
 					clocks = <&clks IMX6SX_CLK_LCDIF2_PIX>,
 						 <&clks IMX6SX_CLK_LCDIF_APB>,
 						 <&clks IMX6SX_CLK_DISPLAY_AXI>;
@@ -1181,7 +1186,7 @@
 				fsl,adck-max-frequency = <30000000>, <40000000>,
 							 <20000000>;
 				status = "disabled";
-                        };
+			};
 
 			adc2: adc@02284000 {
 				compatible = "fsl,imx6sx-adc", "fsl,vf610-adc";
@@ -1192,7 +1197,7 @@
 				fsl,adck-max-frequency = <30000000>, <40000000>,
 							 <20000000>;
 				status = "disabled";
-                        };
+			};
 
 			wdog3: wdog@02288000 {
 				compatible = "fsl,imx6sx-wdt", "fsl,imx21-wdt";
diff --git a/arch/arm/boot/dts/imx6ul-14x14-evk.dts b/arch/arm/boot/dts/imx6ul-14x14-evk.dts
index e281d5087d4a..00f98e5bfcaf 100644
--- a/arch/arm/boot/dts/imx6ul-14x14-evk.dts
+++ b/arch/arm/boot/dts/imx6ul-14x14-evk.dts
@@ -225,7 +225,7 @@
 };
 
 &usbotg1 {
-	dr_mode = "peripheral";
+	dr_mode = "otg";
 	status = "okay";
 };
 
@@ -235,6 +235,14 @@
 	status = "okay";
 };
 
+&usbphy1 {
+	fsl,tx-d-cal = <106>;
+};
+
+&usbphy2 {
+	fsl,tx-d-cal = <106>;
+};
+
 &usdhc1 {
 	pinctrl-names = "default", "state_100mhz", "state_200mhz";
 	pinctrl-0 = <&pinctrl_usdhc1>;
diff --git a/arch/arm/boot/dts/imx6ul-liteboard.dts b/arch/arm/boot/dts/imx6ul-liteboard.dts
new file mode 100644
index 000000000000..6e04cb9202f4
--- /dev/null
+++ b/arch/arm/boot/dts/imx6ul-liteboard.dts
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2016 Grinn
+ *
+ * Author: Marcin Niestroj <m.niestroj@grinn-global.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "imx6ul-litesom.dtsi"
+
+/ {
+	model = "Grinn i.MX6UL liteBoard";
+	compatible = "grinn,imx6ul-liteboard", "grinn,imx6ul-litesom",
+		     "fsl,imx6ul";
+
+	chosen {
+		stdout-path = &uart1;
+	};
+
+	reg_usb_otg1_vbus: regulator-usb-otg1-vbus {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usb_otg1_vbus>;
+		regulator-name = "usb_otg1_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio2 8 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&iomuxc {
+	pinctrl_enet1: enet1grp {
+		fsl,pins = <
+			MX6UL_PAD_GPIO1_IO07__ENET1_MDC		0x1b0b0
+			MX6UL_PAD_GPIO1_IO06__ENET1_MDIO	0x1b0b0
+			MX6UL_PAD_ENET1_RX_EN__ENET1_RX_EN	0x1b0b0
+			MX6UL_PAD_ENET1_RX_ER__ENET1_RX_ER	0x1b0b0
+			MX6UL_PAD_ENET1_RX_DATA0__ENET1_RDATA00 0x1b0b0
+			MX6UL_PAD_ENET1_RX_DATA1__ENET1_RDATA01 0x1b0b0
+			MX6UL_PAD_ENET1_TX_EN__ENET1_TX_EN	0x1b0b0
+			MX6UL_PAD_ENET1_TX_DATA0__ENET1_TDATA00 0x1b0b0
+			MX6UL_PAD_ENET1_TX_DATA1__ENET1_TDATA01 0x1b0b0
+			MX6UL_PAD_ENET1_TX_CLK__ENET1_REF_CLK1	0x4001b031
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX6UL_PAD_UART1_TX_DATA__UART1_DCE_TX	0x1b0b1
+			MX6UL_PAD_UART1_RX_DATA__UART1_DCE_RX	0x1b0b1
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX6UL_PAD_UART1_RTS_B__GPIO1_IO19	0x17059
+			MX6UL_PAD_SD1_CMD__USDHC1_CMD		0x17059
+			MX6UL_PAD_SD1_CLK__USDHC1_CLK		0x10071
+			MX6UL_PAD_SD1_DATA0__USDHC1_DATA0	0x17059
+			MX6UL_PAD_SD1_DATA1__USDHC1_DATA1	0x17059
+			MX6UL_PAD_SD1_DATA2__USDHC1_DATA2	0x17059
+			MX6UL_PAD_SD1_DATA3__USDHC1_DATA3	0x17059
+		>;
+	};
+
+	pinctrl_usb_otg1_vbus: usb-otg1-vbus {
+		fsl,pins = <
+			MX6UL_PAD_ENET2_RX_DATA0__GPIO2_IO08	0x79
+		>;
+	};
+};
+
+&fec1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet1>;
+	phy-mode = "rmii";
+	phy-handle = <&ethphy0>;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@0 {
+			reg = <0>;
+		};
+	};
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+&usbotg1 {
+	vbus-supply = <&reg_usb_otg1_vbus>;
+	dr_mode = "host";
+	status = "okay";
+};
+
+&usdhc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	cd-gpios = <&gpio1 19 GPIO_ACTIVE_LOW>;
+	no-1-8-v;
+	keep-power-in-suspend;
+	wakeup-source;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6ul-litesom.dtsi b/arch/arm/boot/dts/imx6ul-litesom.dtsi
new file mode 100644
index 000000000000..461292d33417
--- /dev/null
+++ b/arch/arm/boot/dts/imx6ul-litesom.dtsi
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2016 Grinn
+ *
+ * Author: Marcin Niestroj <m.niestroj@grinn-global.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "imx6ul.dtsi"
+
+/ {
+	model = "Grinn i.MX6UL liteSOM";
+	compatible = "grinn,imx6ul-litesom", "fsl,imx6ul";
+
+	memory {
+		reg = <0x80000000 0x20000000>;
+	};
+};
+
+&iomuxc {
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX6UL_PAD_NAND_RE_B__USDHC2_CLK	    0x10069
+			MX6UL_PAD_NAND_WE_B__USDHC2_CMD	    0x17059
+			MX6UL_PAD_NAND_DATA00__USDHC2_DATA0 0x17059
+			MX6UL_PAD_NAND_DATA01__USDHC2_DATA1 0x17059
+			MX6UL_PAD_NAND_DATA02__USDHC2_DATA2 0x17059
+			MX6UL_PAD_NAND_DATA03__USDHC2_DATA3 0x17059
+			MX6UL_PAD_NAND_DATA04__USDHC2_DATA4 0x17059
+			MX6UL_PAD_NAND_DATA05__USDHC2_DATA5 0x17059
+			MX6UL_PAD_NAND_DATA06__USDHC2_DATA6 0x17059
+			MX6UL_PAD_NAND_DATA07__USDHC2_DATA7 0x17059
+			MX6UL_PAD_NAND_ALE__USDHC2_RESET_B  0x17059
+		>;
+	};
+};
+
+&usdhc2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc2>;
+	no-1-8-v;
+	non-removable;
+	keep-power-in-suspend;
+	wakeup-source;
+	bus-width = <8>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6ul.dtsi b/arch/arm/boot/dts/imx6ul.dtsi
index c5c05fdccc78..39845a7e0463 100644
--- a/arch/arm/boot/dts/imx6ul.dtsi
+++ b/arch/arm/boot/dts/imx6ul.dtsi
@@ -11,9 +11,11 @@
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include "imx6ul-pinfunc.h"
-#include "skeleton.dtsi"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		ethernet0 = &fec1;
 		ethernet1 = &fec2;
diff --git a/arch/arm/boot/dts/imx6ull-14x14-evk.dts b/arch/arm/boot/dts/imx6ull-14x14-evk.dts
new file mode 100644
index 000000000000..db5bc076e1cc
--- /dev/null
+++ b/arch/arm/boot/dts/imx6ull-14x14-evk.dts
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2016 Freescale Semiconductor, Inc.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "imx6ul-14x14-evk.dts"
+
+/ {
+	model = "Freescale i.MX6 UlltraLite 14x14 EVK Board";
+	compatible = "fsl,imx6ull-14x14-evk", "fsl,imx6ull";
+};
+
+&clks {
+	assigned-clocks = <&clks IMX6UL_CLK_PLL3_PFD2>;
+	assigned-clock-rates = <320000000>;
+};
diff --git a/arch/arm/boot/dts/imx6ull-pinfunc.h b/arch/arm/boot/dts/imx6ull-pinfunc.h
new file mode 100644
index 000000000000..118202336691
--- /dev/null
+++ b/arch/arm/boot/dts/imx6ull-pinfunc.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2016 Freescale Semiconductor, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __DTS_IMX6ULL_PINFUNC_H
+#define __DTS_IMX6ULL_PINFUNC_H
+
+#include "imx6ul-pinfunc.h"
+/*
+ * The pin function ID is a tuple of
+ * <mux_reg conf_reg input_reg mux_mode input_val>
+ */
+#define MX6ULL_PAD_ENET2_RX_DATA0__EPDC_SDDO08                    0x00E4 0x0370 0x0000 0x9 0x0
+#define MX6ULL_PAD_ENET2_RX_DATA1__EPDC_SDDO09                    0x00E8 0x0374 0x0000 0x9 0x0
+#define MX6ULL_PAD_ENET2_RX_EN__EPDC_SDDO10                       0x00EC 0x0378 0x0000 0x9 0x0
+#define MX6ULL_PAD_ENET2_TX_DATA0__EPDC_SDDO11                    0x00F0 0x037C 0x0000 0x9 0x0
+#define MX6ULL_PAD_ENET2_TX_DATA1__EPDC_SDDO12                    0x00F4 0x0380 0x0000 0x9 0x0
+#define MX6ULL_PAD_ENET2_TX_EN__EPDC_SDDO13                       0x00F8 0x0384 0x0000 0x9 0x0
+#define MX6ULL_PAD_ENET2_TX_CLK__EPDC_SDDO14                      0x00FC 0x0388 0x0000 0x9 0x0
+#define MX6ULL_PAD_ENET2_RX_ER__EPDC_SDDO15                       0x0100 0x038C 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_CLK__EPDC_SDCLK                            0x0104 0x0390 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_ENABLE__EPDC_SDLE                          0x0108 0x0394 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_HSYNC__EPDC_SDOE                           0x010C 0x0398 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_VSYNC__EPDC_SDCE0                          0x0110 0x039C 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_RESET__EPDC_GDOE                           0x0114 0x03A0 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA00__EPDC_SDDO00                        0x0118 0x03A4 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA01__EPDC_SDDO01                        0x011C 0x03A8 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA02__EPDC_SDDO02                        0x0120 0x03AC 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA03__EPDC_SDDO03                        0x0124 0x03B0 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA04__EPDC_SDDO04                        0x0128 0x03B4 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA05__EPDC_SDDO05                        0x012C 0x03B8 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA06__EPDC_SDDO06                        0x0130 0x03BC 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA07__EPDC_SDDO07                        0x0134 0x03C0 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA14__EPDC_SDSHR                         0x0150 0x03DC 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA15__EPDC_GDRL                          0x0154 0x03E0 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA16__EPDC_GDCLK                         0x0158 0x03E4 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA17__EPDC_GDSP                          0x015C 0x03E8 0x0000 0x9 0x0
+#define MX6ULL_PAD_LCD_DATA21__EPDC_SDCE1                         0x016C 0x03F8 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_MCLK__ESAI_TX3_RX2                         0x01D4 0x0460 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_PIXCLK__ESAI_TX2_RX3                       0x01D8 0x0464 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_VSYNC__ESAI_TX4_RX1                        0x01DC 0x0468 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_HSYNC__ESAI_TX1                            0x01E0 0x046C 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_DATA00__ESAI_TX_HF_CLK                     0x01E4 0x0470 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_DATA01__ESAI_RX_HF_CLK                     0x01E8 0x0474 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_DATA02__ESAI_RX_FS                         0x01EC 0x0478 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_DATA03__ESAI_RX_CLK                        0x01F0 0x047C 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_DATA04__ESAI_TX_FS                         0x01F4 0x0480 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_DATA05__ESAI_TX_CLK                        0x01F8 0x0484 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_DATA06__ESAI_TX5_RX0                       0x01FC 0x0488 0x0000 0x9 0x0
+#define MX6ULL_PAD_CSI_DATA07__ESAI_T0                            0x0200 0x048C 0x0000 0x9 0x0
+
+#endif /* __DTS_IMX6ULL_PINFUNC_H */
diff --git a/arch/arm/boot/dts/imx6ull.dtsi b/arch/arm/boot/dts/imx6ull.dtsi
new file mode 100644
index 000000000000..dee8ab8135e1
--- /dev/null
+++ b/arch/arm/boot/dts/imx6ull.dtsi
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2016 Freescale Semiconductor, Inc.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "imx6ul.dtsi"
+#include "imx6ull-pinfunc.h"
diff --git a/arch/arm/boot/dts/imx7d-pinfunc.h b/arch/arm/boot/dts/imx7d-pinfunc.h
index 3f9f0d9c8094..7bc3c00e56c6 100644
--- a/arch/arm/boot/dts/imx7d-pinfunc.h
+++ b/arch/arm/boot/dts/imx7d-pinfunc.h
@@ -43,26 +43,30 @@
 #define MX7D_PAD_GPIO1_IO04__GPIO1_IO4                            0x0010 0x0040 0x0000 0x0 0x0
 #define MX7D_PAD_GPIO1_IO04__USB_OTG1_OC                          0x0010 0x0040 0x072C 0x1 0x1
 #define MX7D_PAD_GPIO1_IO04__FLEXTIMER1_CH4                       0x0010 0x0040 0x0594 0x2 0x1
-#define MX7D_PAD_GPIO1_IO04__UART5_CTS_B                          0x0010 0x0040 0x0710 0x3 0x4
+#define MX7D_PAD_GPIO1_IO04__UART5_DCE_CTS                        0x0010 0x0040 0x0000 0x3 0x0
+#define MX7D_PAD_GPIO1_IO04__UART5_DTE_RTS                        0x0010 0x0040 0x0710 0x3 0x4
 #define MX7D_PAD_GPIO1_IO04__I2C1_SCL                             0x0010 0x0040 0x05D4 0x4 0x2
 #define MX7D_PAD_GPIO1_IO04__OBSERVE3_OUT                         0x0010 0x0040 0x0000 0x6 0x0
 #define MX7D_PAD_GPIO1_IO05__GPIO1_IO5                            0x0014 0x0044 0x0000 0x0 0x0
 #define MX7D_PAD_GPIO1_IO05__USB_OTG1_PWR                         0x0014 0x0044 0x0000 0x1 0x0
 #define MX7D_PAD_GPIO1_IO05__FLEXTIMER1_CH5                       0x0014 0x0044 0x0598 0x2 0x1
-#define MX7D_PAD_GPIO1_IO05__UART5_RTS_B                          0x0014 0x0044 0x0710 0x3 0x5
+#define MX7D_PAD_GPIO1_IO05__UART5_DCE_RTS                        0x0014 0x0044 0x0710 0x3 0x5
+#define MX7D_PAD_GPIO1_IO05__UART5_DTE_CTS                        0x0014 0x0044 0x0000 0x3 0x0
 #define MX7D_PAD_GPIO1_IO05__I2C1_SDA                             0x0014 0x0044 0x05D8 0x4 0x2
 #define MX7D_PAD_GPIO1_IO05__OBSERVE4_OUT                         0x0014 0x0044 0x0000 0x6 0x0
 #define MX7D_PAD_GPIO1_IO06__GPIO1_IO6                            0x0018 0x0048 0x0000 0x0 0x0
 #define MX7D_PAD_GPIO1_IO06__USB_OTG2_OC                          0x0018 0x0048 0x0728 0x1 0x1
 #define MX7D_PAD_GPIO1_IO06__FLEXTIMER1_CH6                       0x0018 0x0048 0x059C 0x2 0x1
-#define MX7D_PAD_GPIO1_IO06__UART5_RX_DATA                        0x0018 0x0048 0x0714 0x3 0x4
+#define MX7D_PAD_GPIO1_IO06__UART5_DCE_RX                         0x0018 0x0048 0x0714 0x3 0x4
+#define MX7D_PAD_GPIO1_IO06__UART5_DTE_TX                         0x0018 0x0048 0x0000 0x3 0x0
 #define MX7D_PAD_GPIO1_IO06__I2C2_SCL                             0x0018 0x0048 0x05DC 0x4 0x2
 #define MX7D_PAD_GPIO1_IO06__CCM_WAIT                             0x0018 0x0048 0x0000 0x5 0x0
 #define MX7D_PAD_GPIO1_IO06__KPP_ROW4                             0x0018 0x0048 0x0624 0x6 0x1
 #define MX7D_PAD_GPIO1_IO07__GPIO1_IO7                            0x001C 0x004C 0x0000 0x0 0x0
 #define MX7D_PAD_GPIO1_IO07__USB_OTG2_PWR                         0x001C 0x004C 0x0000 0x1 0x0
 #define MX7D_PAD_GPIO1_IO07__FLEXTIMER1_CH7                       0x001C 0x004C 0x05A0 0x2 0x1
-#define MX7D_PAD_GPIO1_IO07__UART5_TX_DATA                        0x001C 0x004C 0x0714 0x3 0x5
+#define MX7D_PAD_GPIO1_IO07__UART5_DCE_TX                         0x001C 0x004C 0x0000 0x3 0x0
+#define MX7D_PAD_GPIO1_IO07__UART5_DTE_RX                         0x001C 0x004C 0x0714 0x3 0x5
 #define MX7D_PAD_GPIO1_IO07__I2C2_SDA                             0x001C 0x004C 0x05E0 0x4 0x2
 #define MX7D_PAD_GPIO1_IO07__CCM_STOP                             0x001C 0x004C 0x0000 0x5 0x0
 #define MX7D_PAD_GPIO1_IO07__KPP_COL4                             0x001C 0x004C 0x0604 0x6 0x1
diff --git a/arch/arm/boot/dts/imx7s.dtsi b/arch/arm/boot/dts/imx7s.dtsi
index 2b6cb05bc01a..8ff2cbdd8f0d 100644
--- a/arch/arm/boot/dts/imx7s.dtsi
+++ b/arch/arm/boot/dts/imx7s.dtsi
@@ -46,9 +46,11 @@
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include "imx7d-pinfunc.h"
-#include "skeleton.dtsi"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	aliases {
 		gpio0 = &gpio1;
 		gpio1 = &gpio2;
diff --git a/arch/arm/boot/dts/integratorap.dts b/arch/arm/boot/dts/integratorap.dts
index 6f16d09dc5a4..e8b249f92fb3 100644
--- a/arch/arm/boot/dts/integratorap.dts
+++ b/arch/arm/boot/dts/integratorap.dts
@@ -10,6 +10,41 @@
 	compatible = "arm,integrator-ap";
 	dma-ranges = <0x80000000 0x0 0x80000000>;
 
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu@0 {
+			device_type = "cpu";
+			/*
+			 * Since the board has pluggable CPU modules, we
+			 * cannot define a proper compatible here. Let the
+			 * boot loader fill in the apropriate compatible
+			 * string if necessary.
+			 */
+			/* compatible = "arm,arm926ej-s"; */
+			reg = <0>;
+			/*
+			 * The documentation in ARM DUI 0138E page 3-12 states
+			 * that the maximum frequency for this clock is 200 MHz
+			 * but painful trial-and-error has proved to me that it
+			 * is actually just hanging the system above 71 MHz.
+			 * Sad but true.
+			 */
+					 /* kHz     uV   */
+			operating-points = <71000  0
+					    66000  0
+					    60000  0
+					    48000  0
+					    36000  0
+					    24000  0
+					    12000  0>;
+			clocks = <&cmosc>;
+			clock-names = "cpu";
+			clock-latency = <1000000>; /* 1 ms */
+		};
+	};
+
 	aliases {
 		arm,timer-primary = &timer2;
 		arm,timer-secondary = &timer1;
diff --git a/arch/arm/boot/dts/integratorcp.dts b/arch/arm/boot/dts/integratorcp.dts
index 1b5e4b006b72..97f38b57a702 100644
--- a/arch/arm/boot/dts/integratorcp.dts
+++ b/arch/arm/boot/dts/integratorcp.dts
@@ -13,6 +13,32 @@
 		bootargs = "root=/dev/ram0 console=ttyAMA0,38400n8 earlyprintk";
 	};
 
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu@0 {
+			device_type = "cpu";
+			/*
+			 * Since the board has pluggable CPU modules, we
+			 * cannot define a proper compatible here. Let the
+			 * boot loader fill in the apropriate compatible
+			 * string if necessary.
+			 */
+			/* compatible = "arm,arm920t"; */
+			reg = <0>;
+			/*
+			 * TBD comment.
+			 */
+					 /* kHz     uV   */
+			operating-points = <50000  0
+					    48000  0>;
+			clocks = <&cmcore>;
+			clock-names = "cpu";
+			clock-latency = <1000000>; /* 1 ms */
+		};
+	};
+
 	/*
 	 * The Integrator/CP overall clocking architecture can be found in
 	 * ARM DUI 0184B page 7-28 "Integrator/CP922T system clocks" which
diff --git a/arch/arm/boot/dts/keystone-k2g.dtsi b/arch/arm/boot/dts/keystone-k2g.dtsi
index 2919c5190653..63c7cf0c6b6d 100644
--- a/arch/arm/boot/dts/keystone-k2g.dtsi
+++ b/arch/arm/boot/dts/keystone-k2g.dtsi
@@ -72,6 +72,7 @@
 	soc {
 		#address-cells = <1>;
 		#size-cells = <1>;
+		#pinctrl-cells = <1>;
 		compatible = "ti,keystone","simple-bus";
 		ranges = <0x0 0x0 0x0 0xc0000000>;
 		dma-ranges = <0x80000000 0x8 0x00000000 0x80000000>;
diff --git a/arch/arm/boot/dts/keystone-k2l.dtsi b/arch/arm/boot/dts/keystone-k2l.dtsi
index 2ee3d0ac2816..0c5e74e79ba2 100644
--- a/arch/arm/boot/dts/keystone-k2l.dtsi
+++ b/arch/arm/boot/dts/keystone-k2l.dtsi
@@ -59,6 +59,7 @@
 			reg = <0x02620690 0xc>;
 			#address-cells = <1>;
 			#size-cells = <0>;
+			#pinctrl-cells = <2>;
 			pinctrl-single,bit-per-mux;
 			pinctrl-single,register-width = <32>;
 			pinctrl-single,function-mask = <0x1>;
diff --git a/arch/arm/boot/dts/kirkwood-topkick.dts b/arch/arm/boot/dts/kirkwood-topkick.dts
index 1e9a72100a45..330aada6d33f 100644
--- a/arch/arm/boot/dts/kirkwood-topkick.dts
+++ b/arch/arm/boot/dts/kirkwood-topkick.dts
@@ -4,7 +4,7 @@
 #include "kirkwood-6282.dtsi"
 
 / {
-	model = "Univeral Scientific Industrial Co. Topkick-1281P2";
+	model = "Universal Scientific Industrial Co. Topkick-1281P2";
 	compatible = "usi,topkick-1281P2", "usi,topkick", "marvell,kirkwood-88f6282", "marvell,kirkwood";
 
 	memory {
diff --git a/arch/arm/boot/dts/lpc32xx.dtsi b/arch/arm/boot/dts/lpc32xx.dtsi
index b5841fab51c1..d81fe433e3c8 100644
--- a/arch/arm/boot/dts/lpc32xx.dtsi
+++ b/arch/arm/boot/dts/lpc32xx.dtsi
@@ -479,6 +479,8 @@
 				compatible = "nxp,lpc3220-pwm";
 				reg = <0x4005C000 0x4>;
 				clocks = <&clk LPC32XX_CLK_PWM1>;
+				assigned-clocks = <&clk LPC32XX_CLK_PWM1>;
+				assigned-clock-parents = <&clk LPC32XX_CLK_PERIPH>;
 				status = "disabled";
 			};
 
@@ -486,6 +488,8 @@
 				compatible = "nxp,lpc3220-pwm";
 				reg = <0x4005C004 0x4>;
 				clocks = <&clk LPC32XX_CLK_PWM2>;
+				assigned-clocks = <&clk LPC32XX_CLK_PWM2>;
+				assigned-clock-parents = <&clk LPC32XX_CLK_PERIPH>;
 				status = "disabled";
 			};
 
diff --git a/arch/arm/boot/dts/ls1021a.dtsi b/arch/arm/boot/dts/ls1021a.dtsi
index 368e21934285..282d854f4342 100644
--- a/arch/arm/boot/dts/ls1021a.dtsi
+++ b/arch/arm/boot/dts/ls1021a.dtsi
@@ -47,6 +47,7 @@
 
 #include "skeleton64.dtsi"
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/thermal/thermal.h>
 
 / {
 	compatible = "fsl,ls1021a";
@@ -70,14 +71,15 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		cpu@f00 {
+		cpu0: cpu@f00 {
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			reg = <0xf00>;
 			clocks = <&cluster1_clk>;
+			#cooling-cells = <2>;
 		};
 
-		cpu@f01 {
+		cpu1: cpu@f01 {
 			compatible = "arm,cortex-a7";
 			device_type = "cpu";
 			reg = <0xf01>;
@@ -251,6 +253,84 @@
 			};
 		};
 
+		tmu: tmu@1f00000 {
+			compatible = "fsl,qoriq-tmu";
+			reg = <0x0 0x1f00000 0x0 0x10000>;
+			interrupts = <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
+			fsl,tmu-range = <0xb0000 0xa0026 0x80048 0x30061>;
+			fsl,tmu-calibration = <0x00000000 0x0000000f
+					       0x00000001 0x00000017
+					       0x00000002 0x0000001e
+					       0x00000003 0x00000026
+					       0x00000004 0x0000002e
+					       0x00000005 0x00000035
+					       0x00000006 0x0000003d
+					       0x00000007 0x00000044
+					       0x00000008 0x0000004c
+					       0x00000009 0x00000053
+					       0x0000000a 0x0000005b
+					       0x0000000b 0x00000064
+
+					       0x00010000 0x00000011
+					       0x00010001 0x0000001c
+					       0x00010002 0x00000024
+					       0x00010003 0x0000002b
+					       0x00010004 0x00000034
+					       0x00010005 0x00000039
+					       0x00010006 0x00000042
+					       0x00010007 0x0000004c
+					       0x00010008 0x00000051
+					       0x00010009 0x0000005a
+					       0x0001000a 0x00000063
+
+					       0x00020000 0x00000013
+					       0x00020001 0x00000019
+					       0x00020002 0x00000024
+					       0x00020003 0x0000002c
+					       0x00020004 0x00000035
+					       0x00020005 0x0000003d
+					       0x00020006 0x00000046
+					       0x00020007 0x00000050
+					       0x00020008 0x00000059
+
+					       0x00030000 0x00000002
+					       0x00030001 0x0000000d
+					       0x00030002 0x00000019
+					       0x00030003 0x00000024>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		thermal-zones {
+			cpu_thermal: cpu-thermal {
+				polling-delay-passive = <1000>;
+				polling-delay = <5000>;
+
+				thermal-sensors = <&tmu 0>;
+
+				trips {
+					cpu_alert: cpu-alert {
+						temperature = <85000>;
+						hysteresis = <2000>;
+						type = "passive";
+					};
+					cpu_crit: cpu-crit {
+						temperature = <95000>;
+						hysteresis = <2000>;
+						type = "critical";
+					};
+				};
+
+				cooling-maps {
+					map0 {
+						trip = <&cpu_alert>;
+						cooling-device =
+							<&cpu0 THERMAL_NO_LIMIT
+							THERMAL_NO_LIMIT>;
+					};
+				};
+			};
+		};
+
 		dspi0: dspi@2100000 {
 			compatible = "fsl,ls1021a-v1.0-dspi";
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/mps2-an385.dts b/arch/arm/boot/dts/mps2-an385.dts
index 31c374d72a6f..aebbebfc25d1 100644
--- a/arch/arm/boot/dts/mps2-an385.dts
+++ b/arch/arm/boot/dts/mps2-an385.dts
@@ -59,7 +59,7 @@
 		stdout-path = "serial0:9600n8";
 	};
 
-	memory {
+	memory@21000000 {
 		device_type = "memory";
 		reg = <0x21000000 0x1000000>;
 	};
diff --git a/arch/arm/boot/dts/mps2-an399.dts b/arch/arm/boot/dts/mps2-an399.dts
index 5e7e5ca2edbf..349abf70b2a5 100644
--- a/arch/arm/boot/dts/mps2-an399.dts
+++ b/arch/arm/boot/dts/mps2-an399.dts
@@ -59,7 +59,7 @@
 		stdout-path = "serial0:9600n8";
 	};
 
-	memory {
+	memory@60000000 {
 		device_type = "memory";
 		reg = <0x60000000 0x1000000>;
 	};
diff --git a/arch/arm/boot/dts/mps2.dtsi b/arch/arm/boot/dts/mps2.dtsi
index efb8a03cb970..23467390558d 100644
--- a/arch/arm/boot/dts/mps2.dtsi
+++ b/arch/arm/boot/dts/mps2.dtsi
@@ -42,10 +42,12 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "skeleton.dtsi"
 #include "armv7-m.dtsi"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	oscclk0: clk-osc0 {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
diff --git a/arch/arm/boot/dts/mt2701.dtsi b/arch/arm/boot/dts/mt2701.dtsi
index 18596a2c58a1..7eab6f4c4665 100644
--- a/arch/arm/boot/dts/mt2701.dtsi
+++ b/arch/arm/boot/dts/mt2701.dtsi
@@ -12,8 +12,10 @@
  * GNU General Public License for more details.
  */
 
+#include <dt-bindings/clock/mt2701-clk.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/reset/mt2701-resets.h>
 #include "skeleton64.dtsi"
 #include "mt2701-pinfunc.h"
 
@@ -71,10 +73,18 @@
 		#clock-cells = <0>;
 	};
 
-	uart_clk: dummy26m {
+	clk26m: oscillator@0 {
 		compatible = "fixed-clock";
+		#clock-cells = <0>;
 		clock-frequency = <26000000>;
+		clock-output-names = "clk26m";
+	};
+
+	rtc32k: oscillator@1 {
+		compatible = "fixed-clock";
 		#clock-cells = <0>;
+		clock-frequency = <32000>;
+		clock-output-names = "rtc32k";
 	};
 
 	timer {
@@ -104,6 +114,26 @@
 		reg = <0 0x10005000 0 0x1000>;
 	};
 
+	topckgen: syscon@10000000 {
+		compatible = "mediatek,mt2701-topckgen", "syscon";
+		reg = <0 0x10000000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
+	infracfg: syscon@10001000 {
+		compatible = "mediatek,mt2701-infracfg", "syscon";
+		reg = <0 0x10001000 0 0x1000>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+	};
+
+	pericfg: syscon@10003000 {
+		compatible = "mediatek,mt2701-pericfg", "syscon";
+		reg = <0 0x10003000 0 0x1000>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+	};
+
 	watchdog: watchdog@10007000 {
 		compatible = "mediatek,mt2701-wdt",
 			     "mediatek,mt6589-wdt";
@@ -128,6 +158,12 @@
 		reg = <0 0x10200100 0 0x1c>;
 	};
 
+	apmixedsys: syscon@10209000 {
+		compatible = "mediatek,mt2701-apmixedsys", "syscon";
+		reg = <0 0x10209000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
 	gic: interrupt-controller@10211000 {
 		compatible = "arm,cortex-a7-gic";
 		interrupt-controller;
@@ -144,7 +180,8 @@
 			     "mediatek,mt6577-uart";
 		reg = <0 0x11002000 0 0x400>;
 		interrupts = <GIC_SPI 51 IRQ_TYPE_LEVEL_LOW>;
-		clocks = <&uart_clk>;
+		clocks = <&pericfg CLK_PERI_UART0_SEL>, <&pericfg CLK_PERI_UART0>;
+		clock-names = "baud", "bus";
 		status = "disabled";
 	};
 
@@ -153,7 +190,8 @@
 			     "mediatek,mt6577-uart";
 		reg = <0 0x11003000 0 0x400>;
 		interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_LOW>;
-		clocks = <&uart_clk>;
+		clocks = <&pericfg CLK_PERI_UART1_SEL>, <&pericfg CLK_PERI_UART1>;
+		clock-names = "baud", "bus";
 		status = "disabled";
 	};
 
@@ -162,7 +200,8 @@
 			     "mediatek,mt6577-uart";
 		reg = <0 0x11004000 0 0x400>;
 		interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_LOW>;
-		clocks = <&uart_clk>;
+		clocks = <&pericfg CLK_PERI_UART2_SEL>, <&pericfg CLK_PERI_UART2>;
+		clock-names = "baud", "bus";
 		status = "disabled";
 	};
 
@@ -171,7 +210,8 @@
 			     "mediatek,mt6577-uart";
 		reg = <0 0x11005000 0 0x400>;
 		interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_LOW>;
-		clocks = <&uart_clk>;
+		clocks = <&pericfg CLK_PERI_UART3_SEL>, <&pericfg CLK_PERI_UART3>;
+		clock-names = "baud", "bus";
 		status = "disabled";
 	};
 };
diff --git a/arch/arm/boot/dts/omap2420.dtsi b/arch/arm/boot/dts/omap2420.dtsi
index fb712b9aa874..aba542d63d6d 100644
--- a/arch/arm/boot/dts/omap2420.dtsi
+++ b/arch/arm/boot/dts/omap2420.dtsi
@@ -38,6 +38,7 @@
 				reg = <0x0 0x1000>;
 				#address-cells = <1>;
 				#size-cells = <1>;
+				#pinctrl-cells = <1>;
 				ranges = <0 0x0 0x1000>;
 
 				omap2420_pmx: pinmux@30 {
@@ -46,6 +47,7 @@
 					reg = <0x30 0x0113>;
 					#address-cells = <1>;
 					#size-cells = <0>;
+					#pinctrl-cells = <1>;
 					pinctrl-single,register-width = <8>;
 					pinctrl-single,function-mask = <0x3f>;
 				};
diff --git a/arch/arm/boot/dts/omap2430.dtsi b/arch/arm/boot/dts/omap2430.dtsi
index 455aaea407dd..84635eeb99cd 100644
--- a/arch/arm/boot/dts/omap2430.dtsi
+++ b/arch/arm/boot/dts/omap2430.dtsi
@@ -38,6 +38,7 @@
 				reg = <0x2000 0x1000>;
 				#address-cells = <1>;
 				#size-cells = <1>;
+				#pinctrl-cells = <1>;
 				ranges = <0 0x2000 0x1000>;
 
 				omap2430_pmx: pinmux@30 {
@@ -46,6 +47,7 @@
 					reg = <0x30 0x0154>;
 					#address-cells = <1>;
 					#size-cells = <0>;
+					#pinctrl-cells = <1>;
 					pinctrl-single,register-width = <8>;
 					pinctrl-single,function-mask = <0x3f>;
 				};
diff --git a/arch/arm/boot/dts/omap3.dtsi b/arch/arm/boot/dts/omap3.dtsi
index 353d818ce5a6..ecf5eb584c75 100644
--- a/arch/arm/boot/dts/omap3.dtsi
+++ b/arch/arm/boot/dts/omap3.dtsi
@@ -106,6 +106,7 @@
 					reg = <0x30 0x238>;
 					#address-cells = <1>;
 					#size-cells = <0>;
+					#pinctrl-cells = <1>;
 					#interrupt-cells = <1>;
 					interrupt-controller;
 					pinctrl-single,register-width = <16>;
@@ -145,6 +146,7 @@
 					reg = <0xa00 0x5c>;
 					#address-cells = <1>;
 					#size-cells = <0>;
+					#pinctrl-cells = <1>;
 					#interrupt-cells = <1>;
 					interrupt-controller;
 					pinctrl-single,register-width = <16>;
diff --git a/arch/arm/boot/dts/omap34xx.dtsi b/arch/arm/boot/dts/omap34xx.dtsi
index e41c52d3b113..834fdf13601f 100644
--- a/arch/arm/boot/dts/omap34xx.dtsi
+++ b/arch/arm/boot/dts/omap34xx.dtsi
@@ -34,6 +34,7 @@
 			reg = <0x480025d8 0x24>;
 			#address-cells = <1>;
 			#size-cells = <0>;
+			#pinctrl-cells = <1>;
 			#interrupt-cells = <1>;
 			interrupt-controller;
 			pinctrl-single,register-width = <16>;
diff --git a/arch/arm/boot/dts/omap36xx.dtsi b/arch/arm/boot/dts/omap36xx.dtsi
index 718fa88407cd..d1a3e56b50ce 100644
--- a/arch/arm/boot/dts/omap36xx.dtsi
+++ b/arch/arm/boot/dts/omap36xx.dtsi
@@ -66,6 +66,7 @@
 			reg = <0x480025a0 0x5c>;
 			#address-cells = <1>;
 			#size-cells = <0>;
+			#pinctrl-cells = <1>;
 			#interrupt-cells = <1>;
 			interrupt-controller;
 			pinctrl-single,register-width = <16>;
diff --git a/arch/arm/boot/dts/omap4-droid4-xt894.dts b/arch/arm/boot/dts/omap4-droid4-xt894.dts
new file mode 100644
index 000000000000..f3ccb4ceed9e
--- /dev/null
+++ b/arch/arm/boot/dts/omap4-droid4-xt894.dts
@@ -0,0 +1,188 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+/dts-v1/;
+
+#include "omap443x.dtsi"
+
+/ {
+	model = "Motorola Droid 4 XT894";
+	compatible = "motorola,droid4", "ti,omap4430", "ti,omap4";
+
+	chosen {
+		stdout-path = &uart3;
+	};
+
+	/*
+	 * We seem to have only 1021 MB accessible, 1021 - 1022 is locked,
+	 * then 1023 - 1024 seems to contain mbm. For SRAM, see the notes
+	 * below about SRAM and L3_ICLK2 being unused by default,
+	 */
+	memory {
+		device_type = "memory";
+		reg = <0x80000000 0x3fd00000>;	/* 1021 MB */
+	};
+
+	/* CPCAP really supports 1650000 to 3400000 range */
+	vmmc: regulator-mmc {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmc";
+		regulator-min-microvolt = <3000000>;
+		regulator-max-microvolt = <3000000>;
+		regulator-always-on;
+	};
+
+	/* CPCAP really supports 3000000 to 3100000 range */
+	vemmc: regulator-emmc {
+		compatible = "regulator-fixed";
+		regulator-name = "vemmc";
+		regulator-min-microvolt = <3000000>;
+		regulator-max-microvolt = <3000000>;
+		regulator-always-on;
+	};
+
+	/* CPCAP really supports 1650000 to 1950000 range */
+	wl12xx_vmmc: regulator-wl12xx {
+		compatible = "regulator-fixed";
+		regulator-name = "vwl1271";
+		regulator-min-microvolt = <1650000>;
+		regulator-max-microvolt = <1650000>;
+		gpio = <&gpio3 30 GPIO_ACTIVE_HIGH>;	/* gpio94 */
+		startup-delay-us = <70000>;
+		enable-active-high;
+	};
+};
+
+/* L3_2 interconnect is unused, SRAM, GPMC and L3_ICLK2 disabled */
+&gpmc {
+	status = "disabled";
+};
+
+&mmc1 {
+	vmmc-supply = <&vmmc>;
+	bus-width = <4>;
+	cd-gpios = <&gpio4 10 GPIO_ACTIVE_LOW>;	/* gpio106 */
+};
+
+&mmc2 {
+	vmmc-supply = <&vemmc>;
+	bus-width = <8>;
+	non-removable;
+};
+
+&mmc3 {
+	vmmc-supply = <&wl12xx_vmmc>;
+	interrupts-extended = <&wakeupgen GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH
+			       &omap4_pmx_core 0xde>;
+
+	non-removable;
+	bus-width = <4>;
+	cap-power-off-card;
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+	wlcore: wlcore@2 {
+		compatible = "ti,wl1283";
+		reg = <2>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <4 IRQ_TYPE_LEVEL_HIGH>; /* gpio100 */
+		ref-clock-frequency = <26000000>;
+		tcxo-clock-frequency = <26000000>;
+	};
+};
+
+/* L3_2 interconnect is unused, SRAM, GPMC and L3_ICLK2 disabled */
+&ocmcram {
+	status = "disabled";
+};
+
+&omap4_pmx_core {
+	usb_gpio_mux_sel1: pinmux_usb_gpio_mux_sel1_pins {
+		/* gpio_60 */
+		pinctrl-single,pins = <
+		OMAP4_IOPAD(0x088, PIN_OUTPUT | MUX_MODE3)
+		>;
+	};
+
+	usb_ulpi_pins: pinmux_usb_ulpi_pins {
+		pinctrl-single,pins = <
+		OMAP4_IOPAD(0x196, MUX_MODE7)
+		OMAP4_IOPAD(0x198, MUX_MODE7)
+		OMAP4_IOPAD(0x1b2, PIN_INPUT_PULLUP | MUX_MODE0)
+		OMAP4_IOPAD(0x1b4, PIN_INPUT_PULLUP | MUX_MODE0)
+		OMAP4_IOPAD(0x1b6, PIN_INPUT_PULLUP | MUX_MODE0)
+		OMAP4_IOPAD(0x1b8, PIN_INPUT_PULLUP | MUX_MODE0)
+		OMAP4_IOPAD(0x1ba, PIN_INPUT_PULLUP | MUX_MODE0)
+		OMAP4_IOPAD(0x1bc, PIN_INPUT_PULLUP | MUX_MODE0)
+		OMAP4_IOPAD(0x1be, PIN_INPUT_PULLUP | MUX_MODE0)
+		OMAP4_IOPAD(0x1c0, PIN_INPUT_PULLUP | MUX_MODE0)
+		OMAP4_IOPAD(0x1c2, PIN_INPUT_PULLUP | MUX_MODE0)
+		OMAP4_IOPAD(0x1c4, PIN_INPUT_PULLUP | MUX_MODE0)
+		OMAP4_IOPAD(0x1c6, PIN_INPUT_PULLUP | MUX_MODE0)
+		OMAP4_IOPAD(0x1c8, PIN_INPUT_PULLUP | MUX_MODE0)
+		>;
+	};
+
+	/* usb0_otg_dp and usb0_otg_dm */
+	usb_utmi_pins: pinmux_usb_utmi_pins {
+		pinctrl-single,pins = <
+		OMAP4_IOPAD(0x196, PIN_INPUT | MUX_MODE0)
+		OMAP4_IOPAD(0x198, PIN_INPUT | MUX_MODE0)
+		OMAP4_IOPAD(0x1b2, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1b4, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1b6, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1b8, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1ba, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1bc, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1be, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1c0, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1c2, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1c4, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1c6, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1c8, PIN_INPUT_PULLUP | MUX_MODE7)
+		>;
+	};
+
+	/* uart3_tx_irtx and uart3_rx_irrx */
+	uart3_pins: pinmux_uart3_pins {
+		pinctrl-single,pins = <
+		OMAP4_IOPAD(0x196, MUX_MODE7)
+		OMAP4_IOPAD(0x198, MUX_MODE7)
+		OMAP4_IOPAD(0x1b2, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1b4, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1b6, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1b8, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1ba, MUX_MODE2)
+		OMAP4_IOPAD(0x1bc, PIN_INPUT | MUX_MODE2)
+		OMAP4_IOPAD(0x1be, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1c0, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1c2, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1c4, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1c6, PIN_INPUT_PULLUP | MUX_MODE7)
+		OMAP4_IOPAD(0x1c8, PIN_INPUT_PULLUP | MUX_MODE7)
+		>;
+	};
+};
+
+&omap4_pmx_wkup {
+	usb_gpio_mux_sel2: pinmux_usb_gpio_mux_sel2_pins {
+		/* gpio_wk0 */
+		pinctrl-single,pins = <
+		OMAP4_IOPAD(0x040, PIN_OUTPUT_PULLDOWN | MUX_MODE3)
+		>;
+	};
+};
+
+&uart3 {
+	interrupts-extended = <&wakeupgen GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH
+			       &omap4_pmx_core 0x17c>;
+};
+
+/* Internal UTMI+ PHY used for OTG, CPCAP ULPI PHY for detection and charger */
+&usb_otg_hs {
+	interface-type = <1>;
+	mode = <3>;
+	power = <50>;
+};
diff --git a/arch/arm/boot/dts/omap4.dtsi b/arch/arm/boot/dts/omap4.dtsi
index 0ced079b7ae3..8087456b5fbe 100644
--- a/arch/arm/boot/dts/omap4.dtsi
+++ b/arch/arm/boot/dts/omap4.dtsi
@@ -184,6 +184,7 @@
 					reg = <0x40 0x0196>;
 					#address-cells = <1>;
 					#size-cells = <0>;
+					#pinctrl-cells = <1>;
 					#interrupt-cells = <1>;
 					interrupt-controller;
 					pinctrl-single,register-width = <16>;
@@ -256,6 +257,7 @@
 					reg = <0x1e040 0x0038>;
 					#address-cells = <1>;
 					#size-cells = <0>;
+					#pinctrl-cells = <1>;
 					#interrupt-cells = <1>;
 					interrupt-controller;
 					pinctrl-single,register-width = <16>;
diff --git a/arch/arm/boot/dts/omap5-uevm.dts b/arch/arm/boot/dts/omap5-uevm.dts
index 53d31a87b44b..a8c72611fbe3 100644
--- a/arch/arm/boot/dts/omap5-uevm.dts
+++ b/arch/arm/boot/dts/omap5-uevm.dts
@@ -27,12 +27,98 @@
 			default-state = "off";
 		};
 	};
+
+	evm_keys {
+		compatible = "gpio-keys";
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&evm_keys_pins>;
+
+		#address-cells = <7>;
+		#size-cells = <0>;
+
+		btn1 {
+			label = "BTN1";
+			linux,code = <169>;
+			gpios = <&gpio3 19 GPIO_ACTIVE_LOW>;	/* gpio3_83 */
+			wakeup-source;
+			autorepeat;
+			debounce_interval = <50>;
+		};
+	};
+
+	evm_leds {
+		compatible = "gpio-leds";
+
+		led1 {
+			label = "omap5:red:led";
+			gpios = <&gpio9 17 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "mmc0";
+			default-state = "off";
+		};
+
+		led2 {
+			label = "omap5:green:led";
+			gpios = <&gpio9 18 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "mmc1";
+			default-state = "off";
+		};
+
+		led3 {
+			label = "omap5:blue:led";
+			gpios = <&gpio9 19 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "mmc2";
+			default-state = "off";
+		};
+
+		led4 {
+			label = "omap5:green:led1";
+			gpios = <&gpio9 2 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+			default-state = "off";
+		};
+
+		led5 {
+			label = "omap5:green:led2";
+			gpios = <&gpio9 3 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-on";
+			default-state = "off";
+		};
+
+		led6 {
+			label = "omap5:green:led3";
+			gpios = <&gpio9 4 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+			default-state = "off";
+		};
+
+		led7 {
+			label = "omap5:green:led4";
+			gpios = <&gpio9 5 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-on";
+			default-state = "off";
+		};
+
+		led8 {
+			label = "omap5:green:led5";
+			gpios = <&gpio9 6 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+			default-state = "off";
+		};
+	};
 };
 
 &hdmi {
 	vdda-supply = <&ldo4_reg>;
 };
 
+&i2c1 {
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+	};
+};
+
 &i2c5 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c5_pins>;
@@ -48,6 +134,12 @@
 };
 
 &omap5_pmx_core {
+	evm_keys_pins: pinmux_evm_keys_gpio_pins {
+		pinctrl-single,pins = <
+			OMAP5_IOPAD(0x0b6, PIN_INPUT | MUX_MODE6)	/* gpio3_83 */
+		>;
+	};
+
 	i2c5_pins: pinmux_i2c5_pins {
 		pinctrl-single,pins = <
 			OMAP5_IOPAD(0x1c6, PIN_INPUT | MUX_MODE0)		/* i2c5_scl */
diff --git a/arch/arm/boot/dts/omap5.dtsi b/arch/arm/boot/dts/omap5.dtsi
index 25262118ec3d..968c67a49dbd 100644
--- a/arch/arm/boot/dts/omap5.dtsi
+++ b/arch/arm/boot/dts/omap5.dtsi
@@ -171,6 +171,7 @@
 					reg = <0x40 0x01b6>;
 					#address-cells = <1>;
 					#size-cells = <0>;
+					#pinctrl-cells = <1>;
 					#interrupt-cells = <1>;
 					interrupt-controller;
 					pinctrl-single,register-width = <16>;
@@ -270,6 +271,7 @@
 				reg = <0xc840 0x003c>;
 				#address-cells = <1>;
 				#size-cells = <0>;
+				#pinctrl-cells = <1>;
 				#interrupt-cells = <1>;
 				interrupt-controller;
 				pinctrl-single,register-width = <16>;
diff --git a/arch/arm/boot/dts/orion5x-lschl.dts b/arch/arm/boot/dts/orion5x-lschl.dts
new file mode 100644
index 000000000000..947409252845
--- /dev/null
+++ b/arch/arm/boot/dts/orion5x-lschl.dts
@@ -0,0 +1,171 @@
+/*
+ * Device Tree file for Buffalo Linkstation LS-CHLv3
+ *
+ * Copyright (C) 2016 Ash Hughes <ashley.hughes@blueyonder.co.uk>
+ * Copyright (C) 2015, 2016
+ * Roger Shimizu <rogershimizu@gmail.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "orion5x-linkstation.dtsi"
+#include "mvebu-linkstation-gpio-simple.dtsi"
+#include "mvebu-linkstation-fan.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "Buffalo Linkstation Live v3 (LS-CHL)";
+	compatible = "buffalo,lschl", "marvell,orion5x-88f5182", "marvell,orion5x";
+
+	memory { /* 128 MB */
+		device_type = "memory";
+		reg = <0x00000000 0x8000000>;
+	};
+
+	gpio_keys {
+		func {
+			label = "Function Button";
+			linux,code = <KEY_OPTION>;
+			gpios = <&gpio0 15 GPIO_ACTIVE_LOW>;
+		};
+
+		power-on-switch {
+			gpios = <&gpio0 8 GPIO_ACTIVE_LOW>;
+		};
+
+		power-auto-switch {
+			gpios = <&gpio0 10 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	gpio_leds {
+		pinctrl-0 = <&pmx_led_power &pmx_led_alarm &pmx_led_info &pmx_led_func>;
+		blue-power-led {
+			gpios = <&gpio0 0 GPIO_ACTIVE_LOW>;
+		};
+
+		red-alarm-led {
+			gpios = <&gpio0 2 GPIO_ACTIVE_LOW>;
+		};
+
+		amber-info-led {
+			gpios = <&gpio0 3 GPIO_ACTIVE_LOW>;
+		};
+
+		func {
+			label = "lschl:func:blue:top";
+			gpios = <&gpio0 17 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	gpio_fan {
+		gpios = <&gpio0 14 GPIO_ACTIVE_LOW
+			 &gpio0 16 GPIO_ACTIVE_LOW>;
+
+		alarm-gpios = <&gpio0 6 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&pinctrl {
+	pmx_led_power: pmx-leds {
+		marvell,pins = "mpp0";
+		marvell,function = "gpio";
+	};
+
+	pmx_power_hdd: pmx-power-hdd {
+		marvell,pins = "mpp1";
+		marvell,function = "gpio";
+	};
+
+	pmx_led_alarm: pmx-leds {
+		marvell,pins = "mpp2";
+		marvell,function = "gpio";
+	};
+
+	pmx_led_info: pmx-leds {
+		marvell,pins = "mpp3";
+		marvell,function = "gpio";
+	};
+
+	pmx_fan_lock: pmx-fan-lock {
+		marvell,pins = "mpp6";
+		marvell,function = "gpio";
+	};
+
+	pmx_power_switch: pmx-power-switch {
+		marvell,pins = "mpp8", "mpp10", "mpp15";
+		marvell,function = "gpio";
+	};
+
+	pmx_power_usb: pmx-power-usb {
+		marvell,pins = "mpp9";
+		marvell,function = "gpio";
+	};
+
+	pmx_fan_high: pmx-fan-high {
+		marvell,pins = "mpp14";
+		marvell,function = "gpio";
+	};
+
+	pmx_fan_low: pmx-fan-low {
+		marvell,pins = "mpp16";
+		marvell,function = "gpio";
+	};
+
+	pmx_led_func: pmx-leds {
+		marvell,pins = "mpp17";
+		marvell,function = "gpio";
+	};
+
+	pmx_sw_init: pmx-sw-init {
+		marvell,pins = "mpp7";
+		marvell,function = "gpio";
+	};
+};
+
+&hdd_power {
+	gpios = <&gpio0 1 GPIO_ACTIVE_HIGH>;
+};
+
+&usb_power {
+	gpios = <&gpio0 9 GPIO_ACTIVE_HIGH>;
+};
+
diff --git a/arch/arm/boot/dts/ox820.dtsi b/arch/arm/boot/dts/ox820.dtsi
new file mode 100644
index 000000000000..e40f282a023a
--- /dev/null
+++ b/arch/arm/boot/dts/ox820.dtsi
@@ -0,0 +1,296 @@
+/*
+ * ox820.dtsi - Device tree file for Oxford Semiconductor OX820 SoC
+ *
+ * Copyright (C) 2016 Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * Licensed under GPLv2 or later
+ */
+
+/include/ "skeleton.dtsi"
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	compatible = "oxsemi,ox820";
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		enable-method = "oxsemi,ox820-smp";
+
+		cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,arm11mpcore";
+			clocks = <&armclk>;
+			reg = <0>;
+		};
+
+		cpu@1 {
+			device_type = "cpu";
+			compatible = "arm,arm11mpcore";
+			clocks = <&armclk>;
+			reg = <1>;
+		};
+	};
+
+	memory {
+		/* Max 512MB @ 0x60000000 */
+		reg = <0x60000000 0x20000000>;
+	};
+
+	clocks {
+		osc: oscillator {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <25000000>;
+		};
+
+		gmacclk: gmacclk {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <125000000>;
+		};
+
+		sysclk: sysclk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+			clock-div = <4>;
+			clock-mult = <1>;
+			clocks = <&osc>;
+		};
+
+		plla: plla {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <850000000>;
+		};
+
+		armclk: armclk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+			clock-div = <2>;
+			clock-mult = <1>;
+			clocks = <&plla>;
+		};
+	};
+
+	soc {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "simple-bus";
+		ranges;
+		interrupt-parent = <&gic>;
+
+		nandc: nand-controller@41000000 {
+			compatible = "oxsemi,ox820-nand";
+			reg = <0x41000000 0x100000>;
+			clocks = <&stdclk 11>;
+			resets = <&reset 15>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		etha: ethernet@40400000 {
+			compatible = "oxsemi,ox820-dwmac", "snps,dwmac";
+			reg = <0x40400000 0x2000>;
+			interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "macirq", "eth_wake_irq";
+			mac-address = [000000000000]; /* Filled in by U-Boot */
+			phy-mode = "rgmii";
+
+			clocks = <&stdclk 9>, <&gmacclk>;
+			clock-names = "gmac", "stmmaceth";
+			resets = <&reset 6>;
+
+			/* Regmap for sys registers */
+			oxsemi,sys-ctrl = <&sys>;
+
+			status = "disabled";
+		};
+
+		apb-bridge@44000000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "simple-bus";
+			ranges = <0 0x44000000 0x1000000>;
+
+			pinctrl: pinctrl {
+				compatible = "oxsemi,ox820-pinctrl";
+
+				/* Regmap for sys registers */
+				oxsemi,sys-ctrl = <&sys>;
+
+				pinctrl_uart0: uart0 {
+					uart0 {
+						pins = "gpio30", "gpio31";
+						function = "fct5";
+					};
+				};
+
+				pinctrl_uart0_modem: uart0_modem {
+					uart0_modem_a {
+						pins = "gpio24", "gpio24", "gpio26", "gpio27";
+						function = "fct4";
+					};
+					uart0_modem_b {
+						pins = "gpio28", "gpio29";
+						function = "fct5";
+					};
+				};
+
+				pinctrl_uart1: uart1 {
+					uart1 {
+						pins = "gpio7", "gpio8";
+						function = "fct4";
+					};
+				};
+
+				pinctrl_uart1_modem: uart1_modem {
+					uart1_modem {
+						pins = "gpio5", "gpio6", "gpio40", "gpio41", "gpio42", "gpio43";
+						function = "fct4";
+					};
+				};
+
+				pinctrl_etha_mdio: etha_mdio {
+					etha_mdio {
+						pins = "gpio3", "gpio4";
+						function = "fct1";
+					};
+				};
+
+				pinctrl_nand: nand {
+					nand {
+						pins = "gpio12", "gpio13", "gpio14", "gpio15",
+						     "gpio16", "gpio17", "gpio18", "gpio19",
+						     "gpio20", "gpio21", "gpio22", "gpio23",
+						     "gpio24";
+						function = "fct1";
+					};
+				};
+			};
+
+			gpio0: gpio@000000 {
+				compatible = "oxsemi,ox820-gpio";
+				reg = <0x000000 0x100000>;
+				interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
+				#gpio-cells = <2>;
+				gpio-controller;
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				ngpios = <32>;
+				oxsemi,gpio-bank = <0>;
+				gpio-ranges = <&pinctrl 0 0 32>;
+			};
+
+			gpio1: gpio@100000 {
+				compatible = "oxsemi,ox820-gpio";
+				reg = <0x100000 0x100000>;
+				interrupts = <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
+				#gpio-cells = <2>;
+				gpio-controller;
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				ngpios = <18>;
+				oxsemi,gpio-bank = <1>;
+				gpio-ranges = <&pinctrl 0 32 18>;
+			};
+
+			uart0: serial@200000 {
+			       compatible = "ns16550a";
+			       reg = <0x200000 0x100000>;
+			       interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
+			       reg-shift = <0>;
+			       fifo-size = <16>;
+			       reg-io-width = <1>;
+			       current-speed = <115200>;
+			       no-loopback-test;
+			       status = "disabled";
+			       clocks = <&sysclk>;
+			       resets = <&reset 17>;
+			};
+
+			uart1: serial@300000 {
+			       compatible = "ns16550a";
+			       reg = <0x200000 0x100000>;
+			       interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
+			       reg-shift = <0>;
+			       fifo-size = <16>;
+			       reg-io-width = <1>;
+			       current-speed = <115200>;
+			       no-loopback-test;
+			       status = "disabled";
+			       clocks = <&sysclk>;
+			       resets = <&reset 18>;
+			};
+
+			rps@400000 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+				compatible = "simple-bus";
+				ranges = <0 0x400000 0x100000>;
+
+				intc: interrupt-controller@0 {
+					compatible = "oxsemi,ox820-rps-irq", "oxsemi,ox810se-rps-irq";
+					interrupt-controller;
+					reg = <0 0x200>;
+					interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+					#interrupt-cells = <1>;
+					valid-mask = <0xFFFFFFFF>;
+					clear-mask = <0>;
+				};
+
+				timer0: timer@200 {
+					compatible = "oxsemi,ox820-rps-timer";
+					reg = <0x200 0x40>;
+					clocks = <&sysclk>;
+					interrupt-parent = <&intc>;
+					interrupts = <4>;
+				};
+			};
+
+			sys: sys-ctrl@e00000 {
+				compatible = "oxsemi,ox820-sys-ctrl", "syscon", "simple-mfd";
+				reg = <0xe00000 0x200000>;
+
+				reset: reset-controller {
+					compatible = "oxsemi,ox820-reset", "oxsemi,ox810se-reset";
+					#reset-cells = <1>;
+				};
+
+				stdclk: stdclk {
+					compatible = "oxsemi,ox820-stdclk", "oxsemi,ox810se-stdclk";
+					#clock-cells = <1>;
+				};
+			};
+		};
+
+		apb-bridge@47000000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "simple-bus";
+			ranges = <0 0x47000000 0x1000000>;
+
+			scu: scu@0 {
+				compatible = "arm,arm11mp-scu";
+				reg = <0x0 0x100>;
+			};
+
+			local-timer@600 {
+				compatible = "arm,arm11mp-twd-timer";
+				reg = <0x600 0x20>;
+				interrupts = <GIC_PPI 13 (GIC_CPU_MASK_RAW(3)|IRQ_TYPE_LEVEL_HIGH)>;
+				clocks = <&armclk>;
+			};
+
+			gic: gic@1000 {
+				compatible = "arm,arm11mp-gic";
+				interrupt-controller;
+				#interrupt-cells = <3>;
+				reg = <0x1000 0x1000>,
+				      <0x100 0x500>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/pxa25x.dtsi b/arch/arm/boot/dts/pxa25x.dtsi
new file mode 100644
index 000000000000..f9f4726396a0
--- /dev/null
+++ b/arch/arm/boot/dts/pxa25x.dtsi
@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2016 Robert Jarzmik <robert.jarzmik@free.fr>
+ *
+ * The code contained herein is licensed under the GNU General Public
+ * License. You may obtain a copy of the GNU General Public License
+ * Version 2 or later at the following locations:
+ *
+ * http://www.opensource.org/licenses/gpl-license.html
+ * http://www.gnu.org/copyleft/gpl.html
+ */
+#include "pxa2xx.dtsi"
+#include "dt-bindings/clock/pxa-clock.h"
+
+/ {
+	model = "Marvell PXA25x family SoC";
+	compatible = "marvell,pxa250";
+
+	clocks {
+	       /*
+		* The muxing of external clocks/internal dividers for osc* clock
+		* sources has been hidden under the carpet by now.
+		*/
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		clks: pxa2xx_clks@41300004 {
+			compatible = "marvell,pxa250-core-clocks";
+			#clock-cells = <1>;
+			status = "okay";
+		};
+
+		/* timer oscillator */
+		clktimer: oscillator {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency  = <3686400>;
+			clock-output-names = "ostimer";
+		};
+	};
+
+	pxabus {
+		pdma: dma-controller@40000000 {
+			compatible = "marvell,pdma-1.0";
+			reg = <0x40000000 0x10000>;
+			interrupts = <25>;
+			#dma-channels = <16>;
+			#dma-cells = <2>;
+			#dma-requests = <40>;
+			status = "okay";
+		};
+
+		pxairq: interrupt-controller@40d00000 {
+			marvell,intc-priority;
+			marvell,intc-nr-irqs = <32>;
+		};
+
+		pinctrl: pinctrl@40e00000 {
+			reg = <0x40e00054 0x20 0x40e0000c 0xc 0x40e0010c 4
+			       0x40f00020 0x10>;
+			compatible = "marvell,pxa25x-pinctrl";
+		};
+
+		gpio: gpio@40e00000 {
+			compatible = "intel,pxa25x-gpio";
+			gpio-ranges = <&pinctrl 0 0 84>;
+			clocks = <&clks CLK_NONE>;
+		};
+
+		pwm0: pwm@40b00000 {
+			compatible = "marvell,pxa250-pwm";
+			reg = <0x40b00000 0x10>;
+			#pwm-cells = <1>;
+			clocks = <&clks CLK_PWM0>;
+		};
+
+		pwm1: pwm@40b00010 {
+			compatible = "marvell,pxa250-pwm";
+			reg = <0x40b00010 0x10>;
+			#pwm-cells = <1>;
+			clocks = <&clks CLK_PWM1>;
+		};
+	};
+
+	timer@40a00000 {
+		compatible = "marvell,pxa-timer";
+		reg = <0x40a00000 0x20>;
+		interrupts = <26>;
+		clocks = <&clktimer>;
+		status = "okay";
+	};
+
+	pxa250_opp_table: opp_table0 {
+		compatible = "operating-points-v2";
+
+		opp@99532800 {
+			opp-hz = /bits/ 64 <99532800>;
+			opp-microvolt = <1000000 950000 1650000>;
+			clock-latency-ns = <20>;
+		};
+		opp@199065600 {
+			opp-hz = /bits/ 64 <199065600>;
+			opp-microvolt = <1000000 950000 1650000>;
+			clock-latency-ns = <20>;
+		};
+		opp@298598400 {
+			opp-hz = /bits/ 64 <298598400>;
+			opp-microvolt = <1100000 1045000 1650000>;
+			clock-latency-ns = <20>;
+		};
+		opp@398131200 {
+			opp-hz = /bits/ 64 <398131200>;
+			opp-microvolt = <1300000 1235000 1650000>;
+			clock-latency-ns = <20>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/pxa27x.dtsi b/arch/arm/boot/dts/pxa27x.dtsi
index 9e73dc6b3ed3..e0fab48ba6fa 100644
--- a/arch/arm/boot/dts/pxa27x.dtsi
+++ b/arch/arm/boot/dts/pxa27x.dtsi
@@ -137,4 +137,44 @@
 		clocks = <&clks CLK_OSTIMER>;
 		status = "okay";
 	};
+
+	pxa270_opp_table: opp_table0 {
+		compatible = "operating-points-v2";
+
+		opp@104000000 {
+			opp-hz = /bits/ 64 <104000000>;
+			opp-microvolt = <900000 900000 1705000>;
+			clock-latency-ns = <20>;
+		};
+		opp@156000000 {
+			opp-hz = /bits/ 64 <156000000>;
+			opp-microvolt = <1000000 1000000 1705000>;
+			clock-latency-ns = <20>;
+		};
+		opp@208000000 {
+			opp-hz = /bits/ 64 <208000000>;
+			opp-microvolt = <1180000 1180000 1705000>;
+			clock-latency-ns = <20>;
+		};
+		opp@312000000 {
+			opp-hz = /bits/ 64 <312000000>;
+			opp-microvolt = <1250000 1250000 1705000>;
+			clock-latency-ns = <20>;
+		};
+		opp@416000000 {
+			opp-hz = /bits/ 64 <416000000>;
+			opp-microvolt = <1350000 1350000 1705000>;
+			clock-latency-ns = <20>;
+		};
+		opp@520000000 {
+			opp-hz = /bits/ 64 <520000000>;
+			opp-microvolt = <1450000 1450000 1705000>;
+			clock-latency-ns = <20>;
+		};
+		opp@624000000 {
+			opp-hz = /bits/ 64 <624000000>;
+			opp-microvolt = <1550000 1550000 1705000>;
+			clock-latency-ns = <20>;
+		};
+	};
 };
diff --git a/arch/arm/boot/dts/pxa2xx.dtsi b/arch/arm/boot/dts/pxa2xx.dtsi
index 3ff077ca4400..e4ebcde17837 100644
--- a/arch/arm/boot/dts/pxa2xx.dtsi
+++ b/arch/arm/boot/dts/pxa2xx.dtsi
@@ -54,8 +54,8 @@
 			reg = <0x40e00000 0x10000>;
 			gpio-controller;
 			#gpio-cells = <0x2>;
-			interrupts = <10>;
-			interrupt-names = "gpio_mux";
+			interrupts = <8>, <9>, <10>;
+			interrupt-names = "gpio0", "gpio1", "gpio_mux";
 			interrupt-controller;
 			#interrupt-cells = <0x2>;
 			ranges;
diff --git a/arch/arm/boot/dts/pxa3xx.dtsi b/arch/arm/boot/dts/pxa3xx.dtsi
index 9d6f3aacedb7..7a0cc4ea819a 100644
--- a/arch/arm/boot/dts/pxa3xx.dtsi
+++ b/arch/arm/boot/dts/pxa3xx.dtsi
@@ -138,6 +138,7 @@
 			reg = <0x40e10000 0xffff>;
 			#address-cells = <1>;
 			#size-cells = <0>;
+			#pinctrl-cells = <1>;
 			pinctrl-single,register-width = <32>;
 			pinctrl-single,function-mask = <0x7>;
 		};
diff --git a/arch/arm/boot/dts/qcom-apq8060-dragonboard.dts b/arch/arm/boot/dts/qcom-apq8060-dragonboard.dts
index 6c0038398ef2..4b8872cc8bf9 100644
--- a/arch/arm/boot/dts/qcom-apq8060-dragonboard.dts
+++ b/arch/arm/boot/dts/qcom-apq8060-dragonboard.dts
@@ -51,6 +51,29 @@
 			regulator-boot-on;
 		};
 
+		/* GPIO controlled ethernet power regulator */
+		dragon_veth: xc622a331mrg {
+			compatible = "regulator-fixed";
+			regulator-name = "XC6222A331MR-G";
+			regulator-min-microvolt = <3300000>;
+			regulator-max-microvolt = <3300000>;
+			vin-supply = <&vph>;
+			gpio = <&pm8058_gpio 40 GPIO_ACTIVE_HIGH>;
+			enable-active-high;
+			pinctrl-names = "default";
+			pinctrl-0 = <&dragon_veth_gpios>;
+			regulator-always-on;
+		};
+
+		/* VDDvario fixed regulator */
+		dragon_vario: nds332p {
+			compatible = "regulator-fixed";
+			regulator-name = "NDS332P";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			vin-supply = <&pm8058_s3>;
+		};
+
 		/* This is a levelshifter for SDCC5 */
 		dragon_vio_txb: txb0104rgyr {
 			compatible = "regulator-fixed";
@@ -167,6 +190,36 @@
 					bias-pull-up;
 				};
 			};
+
+			dragon_ebi2_pins: ebi2 {
+				/*
+				 * Pins used by EBI2 on the Dragonboard, actually only
+				 * CS2 is used by a real peripheral. CS0 is just
+				 * routed to a test point.
+				 */
+				mux0 {
+					pins =
+					    /* "gpio39", CS1A_N this is not good to mux */
+					    "gpio40", /* CS2A_N */
+					    "gpio134"; /* CS0_N testpoint TP29 */
+					function = "ebi2cs";
+				};
+				mux1 {
+					pins =
+					    /* EBI2_ADDR_7 downto EBI2_ADDR_0 address bus */
+					    "gpio123", "gpio124", "gpio125", "gpio126",
+					    "gpio127", "gpio128", "gpio129", "gpio130",
+					    /* EBI2_DATA_15 downto EBI2_DATA_0 data bus */
+					    "gpio135", "gpio136", "gpio137", "gpio138",
+					    "gpio139", "gpio140", "gpio141", "gpio142",
+					    "gpio143", "gpio144", "gpio145", "gpio146",
+					    "gpio147", "gpio148", "gpio149", "gpio150",
+					    "gpio151", /* EBI2_OE_N */
+					    "gpio153", /* EBI2_ADV */
+					    "gpio157"; /* EBI2_WE_N */
+					function = "ebi2";
+				};
+			};
 		};
 
 		qcom,ssbi@500000 {
@@ -201,6 +254,15 @@
 				};
 
 				gpio@150 {
+					dragon_ethernet_gpios: ethernet-gpios {
+						pinconf {
+							pins = "gpio7";
+							function = "normal";
+							input-enable;
+							bias-disable;
+							power-source = <PM8058_GPIO_S3>;
+						};
+					};
 					dragon_bmp085_gpios: bmp085-gpios {
 						pinconf {
 							pins = "gpio16";
@@ -238,6 +300,14 @@
 							power-source = <PM8058_GPIO_S3>;
 						};
 					};
+					dragon_veth_gpios: veth-gpios {
+						pinconf {
+							pins = "gpio40";
+							function = "normal";
+							bias-disable;
+							drive-push-pull;
+						};
+					};
 				};
 
 				led@48 {
@@ -322,6 +392,55 @@
 			};
 		};
 
+		external-bus@1a100000 {
+			/* The EBI2 will instantiate first, then populate its children */
+			status = "ok";
+			pinctrl-names = "default";
+			pinctrl-0 = <&dragon_ebi2_pins>;
+
+			/*
+			 * An on-board SMSC LAN9221 chip for "debug ethernet",
+			 * which is actually just an ordinary ethernet on the
+			 * EBI2. This has a 25MHz chrystal next to it, so no
+			 * clocking is needed.
+			 */
+			ethernet-ebi2@2,0 {
+				compatible = "smsc,lan9221", "smsc,lan9115";
+				reg = <2 0x0 0x100>;
+				/*
+				 * GPIO7 has interrupt 198 on the PM8058
+				 * The second interrupt is the PME interrupt
+				 * for network wakeup, connected to the TLMM.
+				 */
+				interrupts-extended = <&pmicintc 198 IRQ_TYPE_EDGE_FALLING>,
+						    <&tlmm 29 IRQ_TYPE_EDGE_RISING>;
+				reset-gpios = <&tlmm 30 GPIO_ACTIVE_LOW>;
+				vdd33a-supply = <&dragon_veth>;
+				vddvario-supply = <&dragon_vario>;
+				pinctrl-names = "default";
+				pinctrl-0 = <&dragon_ethernet_gpios>;
+				phy-mode = "mii";
+				reg-io-width = <2>;
+				smsc,force-external-phy;
+				/* IRQ on edge falling = active low */
+				smsc,irq-active-low;
+				smsc,irq-push-pull;
+
+				/*
+				 * SLOW chipselect config
+				 * Delay 9 cycles (140ns@64MHz) between SMSC
+				 * LAN9221 Ethernet controller reads and writes
+				 * on CS2.
+				 */
+				qcom,xmem-recovery-cycles = <0>;
+				qcom,xmem-write-hold-cycles = <3>;
+				qcom,xmem-write-delta-cycles = <31>;
+				qcom,xmem-read-delta-cycles = <28>;
+				qcom,xmem-write-wait-cycles = <9>;
+				qcom,xmem-read-wait-cycles = <9>;
+			};
+		};
+
 		rpm@104000 {
 			/*
 			 * Set up of the PMIC RPM regulators for this board
diff --git a/arch/arm/boot/dts/qcom-apq8064-asus-nexus7-flo.dts b/arch/arm/boot/dts/qcom-apq8064-asus-nexus7-flo.dts
index b72e09506448..e39440a86739 100644
--- a/arch/arm/boot/dts/qcom-apq8064-asus-nexus7-flo.dts
+++ b/arch/arm/boot/dts/qcom-apq8064-asus-nexus7-flo.dts
@@ -15,6 +15,20 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		ramoops@88d00000{
+			compatible = "ramoops";
+			reg = <0x88d00000 0x100000>;
+			record-size     = <0x00020000>;
+			console-size    = <0x00020000>;
+			ftrace-size     = <0x00020000>;
+		};
+	};
+
 	ext_3p3v: regulator-fixed@1 {
 		compatible = "regulator-fixed";
 		regulator-min-microvolt = <3300000>;
@@ -99,6 +113,7 @@
 				l2 {
 					regulator-min-microvolt = <1200000>;
 					regulator-max-microvolt = <1200000>;
+					regulator-always-on;
 				};
 
 				/* msm_otg-HSUSB_3p3 */
@@ -133,13 +148,14 @@
 					regulator-min-microvolt = <3000000>;
 					regulator-max-microvolt = <3000000>;
 					bias-pull-down;
+					regulator-always-on;
 				};
 
 				/* pwm_power for backlight */
 				l17 {
 					regulator-min-microvolt = <3000000>;
-					regulator-max-microvolt = <3600000>;
-					bias-pull-down;
+					regulator-max-microvolt = <3000000>;
+					regulator-always-on;
 				};
 
 				/* camera, qdsp6 */
@@ -184,6 +200,63 @@
 			};
 		};
 
+		mdp@5100000 {
+			status = "okay";
+			ports {
+				port@1 {
+					mdp_dsi1_out: endpoint {
+						remote-endpoint = <&dsi0_in>;
+					};
+				};
+			};
+		};
+
+		dsi0: mdss_dsi@4700000 {
+			status = "okay";
+			vdda-supply = <&pm8921_l2>;/*VDD_MIPI1 to 4*/
+			vdd-supply = <&pm8921_l8>;
+			vddio-supply = <&pm8921_lvs7>;
+			avdd-supply = <&pm8921_l11>;
+			vcss-supply = <&ext_3p3v>;
+
+			panel@0 {
+				reg = <0>;
+				compatible = "jdi,lt070me05000";
+
+				vddp-supply = <&pm8921_l17>;
+				iovcc-supply = <&pm8921_lvs7>;
+
+				enable-gpios = <&pm8921_gpio 36 GPIO_ACTIVE_HIGH>;
+				reset-gpios = <&tlmm_pinmux 54 GPIO_ACTIVE_LOW>;
+				dcdc-en-gpios = <&pm8921_gpio 23 GPIO_ACTIVE_HIGH>;
+
+				port {
+					panel_in: endpoint {
+						remote-endpoint = <&dsi0_out>;
+					};
+				};
+			};
+			ports {
+				port@0 {
+					dsi0_in: endpoint {
+						remote-endpoint = <&mdp_dsi1_out>;
+					};
+				};
+
+				port@1 {
+					dsi0_out: endpoint {
+						remote-endpoint = <&panel_in>;
+						data-lanes = <0 1 2 3>;
+					};
+				};
+			};
+		};
+
+		dsi-phy@4700200 {
+			status = "okay";
+			vddio-supply = <&pm8921_lvs7>;/*VDD_PLL2_1 to 7*/
+		};
+
 		gsbi@16200000 {
 			status = "okay";
 			qcom,mode = <GSBI_PROT_I2C>;
diff --git a/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts b/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts
index 2eeb0904eaa7..3d37cab3b9a9 100644
--- a/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts
+++ b/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts
@@ -43,6 +43,17 @@
 		};
 	};
 
+	hdmi-out {
+		compatible = "hdmi-connector";
+		type = "d";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&hdmi_out>;
+			};
+		};
+	};
+
 	soc {
 		pinctrl@800000 {
 			card_detect: card_detect {
@@ -64,6 +75,25 @@
 					bias-disable;
 				};
 			};
+
+			hdmi_pinctrl: hdmi-pinctrl {
+				mux {
+					pins = "gpio70", "gpio71", "gpio72";
+					function = "hdmi";
+				};
+
+				pinconf_ddc {
+					pins = "gpio70", "gpio71";
+					bias-pull-up;
+					drive-strength = <2>;
+				};
+
+				pinconf_hpd {
+					pins = "gpio72";
+					bias-pull-down;
+					drive-strength = <16>;
+				};
+			};
 		};
 
 		rpm@108000 {
@@ -329,5 +359,49 @@
 				mmc-pwrseq = <&sdcc4_pwrseq>;
 			};
 		};
+
+		hdmi-tx@4a00000 {
+			status = "okay";
+
+			core-vdda-supply = <&pm8921_hdmi_switch>;
+			hdmi-mux-supply = <&ext_3p3v>;
+
+			hpd-gpios = <&tlmm_pinmux 72 GPIO_ACTIVE_HIGH>;
+
+			pinctrl-names = "default";
+			pinctrl-0 = <&hdmi_pinctrl>;
+
+			ports {
+				port@0 {
+					endpoint {
+						remote-endpoint = <&mdp_dtv_out>;
+					};
+				};
+
+				port@1 {
+					endpoint {
+						remote-endpoint = <&hdmi_con>;
+					};
+				};
+			};
+		};
+
+		hdmi-phy@4a00400 {
+			status = "okay";
+
+			core-vdda-supply = <&pm8921_hdmi_switch>;
+		};
+
+		mdp@5100000 {
+			status = "okay";
+
+			ports {
+				port@3 {
+					endpoint {
+						remote-endpoint = <&hdmi_in>;
+					};
+				};
+			};
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/qcom-apq8064.dtsi b/arch/arm/boot/dts/qcom-apq8064.dtsi
index 1dbe697b2e90..268bd470c865 100644
--- a/arch/arm/boot/dts/qcom-apq8064.dtsi
+++ b/arch/arm/boot/dts/qcom-apq8064.dtsi
@@ -1060,6 +1060,231 @@
 			reg = <0x1a400000 0x100>;
 		};
 
+		gpu: adreno-3xx@4300000 {
+			compatible = "qcom,adreno-3xx";
+			reg = <0x04300000 0x20000>;
+			reg-names = "kgsl_3d0_reg_memory";
+			interrupts = <GIC_SPI 80 0>;
+			interrupt-names = "kgsl_3d0_irq";
+			clock-names =
+			    "core_clk",
+			    "iface_clk",
+			    "mem_clk",
+			    "mem_iface_clk";
+			clocks =
+			    <&mmcc GFX3D_CLK>,
+			    <&mmcc GFX3D_AHB_CLK>,
+			    <&mmcc GFX3D_AXI_CLK>,
+			    <&mmcc MMSS_IMEM_AHB_CLK>;
+			qcom,chipid = <0x03020002>;
+
+			iommus = <&gfx3d 0
+				  &gfx3d 1
+				  &gfx3d 2
+				  &gfx3d 3
+				  &gfx3d 4
+				  &gfx3d 5
+				  &gfx3d 6
+				  &gfx3d 7
+				  &gfx3d 8
+				  &gfx3d 9
+				  &gfx3d 10
+				  &gfx3d 11
+				  &gfx3d 12
+				  &gfx3d 13
+				  &gfx3d 14
+				  &gfx3d 15
+				  &gfx3d 16
+				  &gfx3d 17
+				  &gfx3d 18
+				  &gfx3d 19
+				  &gfx3d 20
+				  &gfx3d 21
+				  &gfx3d 22
+				  &gfx3d 23
+				  &gfx3d 24
+				  &gfx3d 25
+				  &gfx3d 26
+				  &gfx3d 27
+				  &gfx3d 28
+				  &gfx3d 29
+				  &gfx3d 30
+				  &gfx3d 31
+				  &gfx3d1 0
+				  &gfx3d1 1
+				  &gfx3d1 2
+				  &gfx3d1 3
+				  &gfx3d1 4
+				  &gfx3d1 5
+				  &gfx3d1 6
+				  &gfx3d1 7
+				  &gfx3d1 8
+				  &gfx3d1 9
+				  &gfx3d1 10
+				  &gfx3d1 11
+				  &gfx3d1 12
+				  &gfx3d1 13
+				  &gfx3d1 14
+				  &gfx3d1 15
+				  &gfx3d1 16
+				  &gfx3d1 17
+				  &gfx3d1 18
+				  &gfx3d1 19
+				  &gfx3d1 20
+				  &gfx3d1 21
+				  &gfx3d1 22
+				  &gfx3d1 23
+				  &gfx3d1 24
+				  &gfx3d1 25
+				  &gfx3d1 26
+				  &gfx3d1 27
+				  &gfx3d1 28
+				  &gfx3d1 29
+				  &gfx3d1 30
+				  &gfx3d1 31>;
+
+			qcom,gpu-pwrlevels {
+				compatible = "qcom,gpu-pwrlevels";
+				qcom,gpu-pwrlevel@0 {
+					qcom,gpu-freq = <450000000>;
+				};
+				qcom,gpu-pwrlevel@1 {
+					qcom,gpu-freq = <27000000>;
+				};
+			};
+		};
+
+		mmss_sfpb: syscon@5700000 {
+			compatible = "syscon";
+			reg = <0x5700000 0x70>;
+		};
+
+		dsi0: mdss_dsi@4700000 {
+			compatible = "qcom,mdss-dsi-ctrl";
+			label = "MDSS DSI CTRL->0";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <GIC_SPI 82 0>;
+			reg = <0x04700000 0x200>;
+			reg-names = "dsi_ctrl";
+
+			clocks = <&mmcc DSI_M_AHB_CLK>,
+				<&mmcc DSI_S_AHB_CLK>,
+				<&mmcc AMP_AHB_CLK>,
+				<&mmcc DSI_CLK>,
+				<&mmcc DSI1_BYTE_CLK>,
+				<&mmcc DSI_PIXEL_CLK>,
+				<&mmcc DSI1_ESC_CLK>;
+			clock-names = "iface_clk", "bus_clk", "core_mmss_clk",
+					"src_clk", "byte_clk", "pixel_clk",
+					"core_clk";
+
+			assigned-clocks = <&mmcc DSI1_BYTE_SRC>,
+					<&mmcc DSI1_ESC_SRC>,
+					<&mmcc DSI_SRC>,
+					<&mmcc DSI_PIXEL_SRC>;
+			assigned-clock-parents = <&dsi0_phy 0>,
+						<&dsi0_phy 0>,
+						<&dsi0_phy 1>,
+						<&dsi0_phy 1>;
+			syscon-sfpb = <&mmss_sfpb>;
+			phys = <&dsi0_phy>;
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					dsi0_in: endpoint {
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					dsi0_out: endpoint {
+					};
+				};
+			};
+		};
+
+
+		dsi0_phy: dsi-phy@4700200 {
+			compatible = "qcom,dsi-phy-28nm-8960";
+			#clock-cells = <1>;
+
+			reg = <0x04700200 0x100>,
+				<0x04700300 0x200>,
+				<0x04700500 0x5c>;
+			reg-names = "dsi_pll", "dsi_phy", "dsi_phy_regulator";
+			clock-names = "iface_clk";
+			clocks = <&mmcc DSI_M_AHB_CLK>;
+		};
+
+
+		mdp_port0: iommu@7500000 {
+			compatible = "qcom,apq8064-iommu";
+			#iommu-cells = <1>;
+			clock-names =
+			    "smmu_pclk",
+			    "iommu_clk";
+			clocks =
+			    <&mmcc SMMU_AHB_CLK>,
+			    <&mmcc MDP_AXI_CLK>;
+			reg = <0x07500000 0x100000>;
+			interrupts =
+			    <GIC_SPI 63 0>,
+			    <GIC_SPI 64 0>;
+			qcom,ncb = <2>;
+		};
+
+		mdp_port1: iommu@7600000 {
+			compatible = "qcom,apq8064-iommu";
+			#iommu-cells = <1>;
+			clock-names =
+			    "smmu_pclk",
+			    "iommu_clk";
+			clocks =
+			    <&mmcc SMMU_AHB_CLK>,
+			    <&mmcc MDP_AXI_CLK>;
+			reg = <0x07600000 0x100000>;
+			interrupts =
+			    <GIC_SPI 61 0>,
+			    <GIC_SPI 62 0>;
+			qcom,ncb = <2>;
+		};
+
+		gfx3d: iommu@7c00000 {
+			compatible = "qcom,apq8064-iommu";
+			#iommu-cells = <1>;
+			clock-names =
+			    "smmu_pclk",
+			    "iommu_clk";
+			clocks =
+			    <&mmcc SMMU_AHB_CLK>,
+			    <&mmcc GFX3D_AXI_CLK>;
+			reg = <0x07c00000 0x100000>;
+			interrupts =
+			    <GIC_SPI 69 0>,
+			    <GIC_SPI 70 0>;
+			qcom,ncb = <3>;
+		};
+
+		gfx3d1: iommu@7d00000 {
+			compatible = "qcom,apq8064-iommu";
+			#iommu-cells = <1>;
+			clock-names =
+			    "smmu_pclk",
+			    "iommu_clk";
+			clocks =
+			    <&mmcc SMMU_AHB_CLK>,
+			    <&mmcc GFX3D_AXI_CLK>;
+			reg = <0x07d00000 0x100000>;
+			interrupts =
+			    <GIC_SPI 210 0>,
+			    <GIC_SPI 211 0>;
+			qcom,ncb = <3>;
+		};
+
 		pcie: pci@1b500000 {
 			compatible = "qcom,pcie-apq8064", "snps,dw-pcie";
 			reg = <0x1b500000 0x1000
@@ -1095,6 +1320,102 @@
 			reset-names = "axi", "ahb", "por", "pci", "phy";
 			status = "disabled";
 		};
+
+		hdmi: hdmi-tx@4a00000 {
+			compatible = "qcom,hdmi-tx-8960";
+			reg = <0x04a00000 0x2f0>;
+			reg-names = "core_physical";
+			interrupts = <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mmcc HDMI_APP_CLK>,
+				 <&mmcc HDMI_M_AHB_CLK>,
+				 <&mmcc HDMI_S_AHB_CLK>;
+			clock-names = "core_clk",
+				      "master_iface_clk",
+				      "slave_iface_clk";
+
+			phys = <&hdmi_phy>;
+			phy-names = "hdmi-phy";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					hdmi_in: endpoint {
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					hdmi_out: endpoint {
+					};
+				};
+			};
+		};
+
+		hdmi_phy: hdmi-phy@4a00400 {
+			compatible = "qcom,hdmi-phy-8960";
+			reg = <0x4a00400 0x60>,
+			      <0x4a00500 0x100>;
+			reg-names = "hdmi_phy",
+				    "hdmi_pll";
+
+			clocks = <&mmcc HDMI_S_AHB_CLK>;
+			clock-names = "slave_iface_clk";
+		};
+
+		mdp: mdp@5100000 {
+			compatible = "qcom,mdp4";
+			reg = <0x05100000 0xf0000>;
+			interrupts = <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mmcc MDP_CLK>,
+				 <&mmcc MDP_AHB_CLK>,
+				 <&mmcc MDP_AXI_CLK>,
+				 <&mmcc MDP_LUT_CLK>,
+				 <&mmcc HDMI_TV_CLK>,
+				 <&mmcc MDP_TV_CLK>;
+			clock-names = "core_clk",
+				      "iface_clk",
+				      "bus_clk",
+				      "lut_clk",
+				      "hdmi_clk",
+				      "tv_clk";
+
+			iommus = <&mdp_port0 0
+				  &mdp_port0 2
+				  &mdp_port1 0
+				  &mdp_port1 2>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					mdp_lvds_out: endpoint {
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					mdp_dsi1_out: endpoint {
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+					mdp_dsi2_out: endpoint {
+					};
+				};
+
+				port@3 {
+					reg = <3>;
+					mdp_dtv_out: endpoint {
+					};
+				};
+			};
+		};
 	};
 };
 #include "qcom-apq8064-pins.dtsi"
diff --git a/arch/arm/boot/dts/qcom-apq8084.dtsi b/arch/arm/boot/dts/qcom-apq8084.dtsi
index 39eb7a4ed16a..80d48867107f 100644
--- a/arch/arm/boot/dts/qcom-apq8084.dtsi
+++ b/arch/arm/boot/dts/qcom-apq8084.dtsi
@@ -182,13 +182,13 @@
 	};
 
 	clocks {
-		xo_board {
+		xo_board: xo_board {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <19200000>;
 		};
 
-		sleep_clk {
+		sleep_clk: sleep_clk {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <32768>;
@@ -416,8 +416,10 @@
 			reg-names = "hc_mem", "core_mem";
 			interrupts = <0 123 0>, <0 138 0>;
 			interrupt-names = "hc_irq", "pwr_irq";
-			clocks = <&gcc GCC_SDCC1_APPS_CLK>, <&gcc GCC_SDCC1_AHB_CLK>;
-			clock-names = "core", "iface";
+			clocks = <&gcc GCC_SDCC1_APPS_CLK>,
+				 <&gcc GCC_SDCC1_AHB_CLK>,
+				 <&xo_board>;
+			clock-names = "core", "iface", "xo";
 			status = "disabled";
 		};
 
@@ -427,8 +429,10 @@
 			reg-names = "hc_mem", "core_mem";
 			interrupts = <0 125 0>, <0 221 0>;
 			interrupt-names = "hc_irq", "pwr_irq";
-			clocks = <&gcc GCC_SDCC2_APPS_CLK>, <&gcc GCC_SDCC2_AHB_CLK>;
-			clock-names = "core", "iface";
+			clocks = <&gcc GCC_SDCC2_APPS_CLK>,
+				 <&gcc GCC_SDCC2_AHB_CLK>,
+				 <&xo_board>;
+			clock-names = "core", "iface", "xo";
 			status = "disabled";
 		};
 
diff --git a/arch/arm/boot/dts/qcom-mdm9615-wp8548-mangoh-green.dts b/arch/arm/boot/dts/qcom-mdm9615-wp8548-mangoh-green.dts
new file mode 100644
index 000000000000..26160c324802
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-mdm9615-wp8548-mangoh-green.dts
@@ -0,0 +1,281 @@
+/*
+ * Device Tree Source for mangOH Green Board with WP8548 Module
+ *
+ * Copyright (C) 2016 BayLibre, SAS.
+ * Author : Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <dt-bindings/input/input.h>
+
+#include "qcom-mdm9615-wp8548.dtsi"
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "MangOH Green with WP8548 Module";
+	compatible = "swir,mangoh-green-wp8548", "swir,wp8548", "qcom,mdm9615";
+
+	aliases {
+		spi0 = &gsbi3_spi;
+		serial0 = &gsbi4_serial;
+		serial1 = &gsbi5_serial;
+		i2c0 = &gsbi5_i2c;
+		mmc0 = &sdcc1;
+	};
+
+	chosen {
+		stdout-path = "serial1:115200n8";
+	};
+};
+
+&msmgpio {
+	/* MangOH GPIO Mapping :
+	 * - 2 : GPIOEXP_INT2
+	 * - 7 : IOT1_GPIO2
+	 * - 8 : IOT0_GPIO4
+	 * - 13: IOT0_GPIO3
+	 * - 21: IOT1_GPIO4
+	 * - 22: IOT2_GPIO1
+	 * - 23: IOT2_GPIO2
+	 * - 24: IOT2_GPIO3
+	 * - 25: IOT1_GPIO1
+	 * - 32: IOT1_GPIO3
+	 * - 33: IOT0_GPIO2
+	 * - 42: IOT0_GPIO1 and SD Card Detect
+	 */
+
+	gpioext1_pins: gpioext1_pins {
+		pins {
+			pins = "gpio2";
+			function = "gpio";
+			input-enable;
+			bias-disable;
+		};
+	};
+
+	sdc_cd_pins: sdc_cd_pins {
+		pins {
+			pins = "gpio42";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+};
+
+&gsbi3_spi {
+	spi@0 {
+		compatible = "swir,mangoh-iotport-spi", "spidev";
+		spi-max-frequency = <24000000>;
+		reg = <0>;
+	};
+};
+
+&gsbi5_i2c {
+	mux@71 {
+		compatible = "nxp,pca9548";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x71>;
+
+		i2c_iot0: i2c@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+		};
+
+		i2c_iot1: i2c@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+		};
+
+		i2c_iot2: i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <2>;
+		};
+
+		i2c@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <3>;
+
+			usbhub: hub@8 {
+				compatible = "smsc,usb3503a";
+				reg = <0x8>;
+				connect-gpios = <&gpioext2 1 GPIO_ACTIVE_HIGH>;
+				intn-gpios = <&gpioext2 0 GPIO_ACTIVE_LOW>;
+				initial-mode = <1>;
+			};
+		};
+
+		i2c@4 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <4>;
+
+			gpioext0: gpio@3e {
+				/* GPIO Expander 0 Mapping :
+				 * - 0: ARDUINO_RESET_Level shift
+				 * - 1: BattChrgr_PG_N
+				 * - 2: BattGauge_GPIO
+				 * - 3: LED_ON (out active high)
+				 * - 4: ATmega_reset_GPIO
+				 * - 5: X
+				 * - 6: PCM_ANALOG_SELECT (out active high)
+				 * - 7: X
+				 * - 8: Board_rev_res1 (in)
+				 * - 9: Board_rev_res2 (in)
+				 * - 10: UART_EXP1_ENn (out active low / pull-down)
+				 * - 11: UART_EXP1_IN (out pull-down)
+				 * - 12: UART_EXP2_IN (out pull-down)
+				 * - 13: SDIO_SEL (out pull-down)
+				 * - 14: SPI_EXP1_ENn (out active low / pull-down)
+				 * - 15: SPI_EXP1_IN (out pull-down)
+				 */
+				#gpio-cells = <2>;
+				#interrupt-cells = <2>;
+				compatible = "semtech,sx1509q";
+				reg = <0x3e>;
+				interrupt-parent = <&gpioext1>;
+				interrupts = <0 IRQ_TYPE_EDGE_FALLING>;
+
+				probe-reset;
+
+				gpio-controller;
+				interrupt-controller;
+			};
+		};
+
+		i2c@5 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <5>;
+
+			gpioext1: gpio@3f {
+				/* GPIO Expander 1 Mapping :
+				 * - 0: GPIOEXP_INT1
+				 * - 1: Battery detect
+				 * - 2: GPIO_SCF3_RESET
+				 * - 3: LED_CARD_DETECT_IOT0 (in)
+				 * - 4: LED_CARD_DETECT_IOT1 (in)
+				 * - 5: LED_CARD_DETECT_IOT2 (in)
+				 * - 6: UIM2_PWM_SELECT
+				 * - 7: UIM2_M2_S_SELECT
+				 * - 8: TP900
+				 * - 9: SENSOR_INT1 (in)
+				 * - 10: SENSOR_INT2 (in)
+				 * - 11: CARD_DETECT_IOT0 (in pull-up)
+				 * - 12: CARD_DETECT_IOT2 (in pull-up)
+				 * - 13: CARD_DETECT_IOT1 (in pull-up)
+				 * - 14: GPIOEXP_INT3 (in active low / pull-up)
+				 * - 15: BattChrgr_INT_N
+				 */
+				pinctrl-0 = <&gpioext1_pins>;
+				pinctrl-names = "default";
+
+				#gpio-cells = <2>;
+				#interrupt-cells = <2>;
+				compatible = "semtech,sx1509q";
+				reg = <0x3f>;
+				interrupt-parent = <&msmgpio>;
+				interrupts = <0 IRQ_TYPE_EDGE_FALLING>;
+
+				probe-reset;
+
+				gpio-controller;
+				interrupt-controller;
+			};
+		};
+
+		i2c@6 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <6>;
+
+			gpioext2: gpio@70 {
+				/* GPIO Expander 2 Mapping :
+				 * - 0: USB_HUB_INTn
+				 * - 1: HUB_CONNECT
+				 * - 2: GPIO_IOT2_RESET (out active low / pull-up)
+				 * - 3: GPIO_IOT1_RESET (out active low / pull-up)
+				 * - 4: GPIO_IOT0_RESET (out active low / pull-up)
+				 * - 5: TP901
+				 * - 6: TP902
+				 * - 7: TP903
+				 * - 8: UART_EXP2_ENn (out active low / pull-down)
+				 * - 9: PCM_EXP1_ENn (out active low)
+				 * - 10: PCM_EXP1_SEL (out)
+				 * - 11: ARD_FTDI
+				 * - 12: TP904
+				 * - 13: TP905
+				 * - 14: TP906
+				 * - 15: RS232_Enable (out active high / pull-up)
+				 */
+				#gpio-cells = <2>;
+				#interrupt-cells = <2>;
+				compatible = "semtech,sx1509q";
+				reg = <0x70>;
+				interrupt-parent = <&gpioext1>;
+				interrupts = <14 IRQ_TYPE_EDGE_FALLING>;
+
+				probe-reset;
+
+				gpio-controller;
+				interrupt-controller;
+			};
+		};
+
+		i2c@7 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <7>;
+		};
+	};
+};
+
+&sdcc1 {
+	pinctrl-0 = <&sdc_cd_pins>;
+	pinctrl-names = "default";
+	disable-wp;
+	cd-gpios = <&msmgpio 42 GPIO_ACTIVE_LOW>; /* Active low CD */
+};
diff --git a/arch/arm/boot/dts/qcom-mdm9615-wp8548.dtsi b/arch/arm/boot/dts/qcom-mdm9615-wp8548.dtsi
new file mode 100644
index 000000000000..7869898e392d
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-mdm9615-wp8548.dtsi
@@ -0,0 +1,170 @@
+/*
+ * Device Tree Source for Sierra Wireless WP8548 Module
+ *
+ * Copyright (C) 2016 BayLibre, SAS.
+ * Author : Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "qcom-mdm9615.dtsi"
+
+/ {
+	model = "Sierra Wireless WP8548 Module";
+	compatible = "swir,wp8548", "qcom,mdm9615";
+
+	memory {
+		reg = <0x48000000 0x7F00000>;
+	};
+};
+
+&msmgpio {
+	pinctrl-0 = <&reset_out_pins>;
+	pinctrl-names = "default";
+
+	gsbi3_pins: gsbi3_pins {
+		mux {
+			pins = "gpio8", "gpio9", "gpio10", "gpio11";
+			function = "gsbi3";
+			drive-strength = <8>;
+			bias-disable;
+		};
+	};
+
+	gsbi4_pins: gsbi4_pins {
+		mux {
+			pins = "gpio12", "gpio13", "gpio14", "gpio15";
+			function = "gsbi4";
+			drive-strength = <8>;
+			bias-disable;
+		};
+	};
+
+	gsbi5_i2c_pins: gsbi5_i2c_pins {
+		pin16 {
+			pins = "gpio16";
+			function = "gsbi5_i2c";
+			drive-strength = <8>;
+			bias-disable;
+		};
+
+		pin17 {
+			pins = "gpio17";
+			function = "gsbi5_i2c";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	gsbi5_uart_pins: gsbi5_uart_pins {
+		mux {
+			pins = "gpio18", "gpio19";
+			function = "gsbi5_uart";
+			drive-strength = <8>;
+			bias-disable;
+		};
+	};
+
+	reset_out_pins: reset_out_pins {
+		pins {
+			pins = "gpio66";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+			output-high;
+		};
+	};
+};
+
+&pmicgpio {
+	usb_vbus_5v_pins: usb_vbus_5v_pins {
+		pins = "gpio4";
+		function = "normal";
+		output-high;
+		bias-disable;
+		qcom,drive-strength = <1>;
+		power-source = <2>;
+	};
+};
+
+&gsbi3 {
+	status = "ok";
+	qcom,mode = <GSBI_PROT_SPI>;
+};
+
+&gsbi3_spi {
+	status = "ok";
+	pinctrl-0 = <&gsbi3_pins>;
+	pinctrl-names = "default";
+	assigned-clocks = <&gcc GSBI3_QUP_CLK>;
+	assigned-clock-rates = <24000000>;
+};
+
+&gsbi4 {
+	status = "ok";
+	qcom,mode = <GSBI_PROT_UART_W_FC>;
+};
+
+&gsbi4_serial {
+	status = "ok";
+	pinctrl-0 = <&gsbi4_pins>;
+	pinctrl-names = "default";
+};
+
+&gsbi5 {
+	status = "ok";
+	qcom,mode = <GSBI_PROT_I2C_UART>;
+};
+
+&gsbi5_i2c {
+	status = "ok";
+	clock-frequency = <200000>;
+	pinctrl-0 = <&gsbi5_i2c_pins>;
+	pinctrl-names = "default";
+};
+
+&gsbi5_serial {
+	status = "ok";
+	pinctrl-0 = <&gsbi5_uart_pins>;
+	pinctrl-names = "default";
+};
+
+&sdcc1 {
+	status = "ok";
+};
diff --git a/arch/arm/boot/dts/qcom-mdm9615.dtsi b/arch/arm/boot/dts/qcom-mdm9615.dtsi
new file mode 100644
index 000000000000..5ae4ec59e6ea
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-mdm9615.dtsi
@@ -0,0 +1,557 @@
+/*
+ * Device Tree Source for Qualcomm MDM9615 SoC
+ *
+ * Copyright (C) 2016 BayLibre, SAS.
+ * Author : Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+/include/ "skeleton.dtsi"
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/qcom,gcc-mdm9615.h>
+#include <dt-bindings/reset/qcom,gcc-mdm9615.h>
+#include <dt-bindings/mfd/qcom-rpm.h>
+#include <dt-bindings/soc/qcom,gsbi.h>
+
+/ {
+	model = "Qualcomm MDM9615";
+	compatible = "qcom,mdm9615";
+	interrupt-parent = <&intc>;
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			compatible = "arm,cortex-a5";
+			device_type = "cpu";
+			next-level-cache = <&L2>;
+		};
+	};
+
+	cpu-pmu {
+		compatible = "arm,cortex-a5-pmu";
+		interrupts = <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_HIGH)>;
+	};
+
+	clocks {
+		cxo_board {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <19200000>;
+		};
+	};
+
+	regulators {
+		vsdcc_fixed: vsdcc-regulator {
+			compatible = "regulator-fixed";
+			regulator-name = "SDCC Power";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <2700000>;
+			regulator-always-on;
+		};
+	};
+
+	soc: soc {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+		compatible = "simple-bus";
+
+		L2: l2-cache@2040000 {
+			compatible = "arm,pl310-cache";
+			reg = <0x02040000 0x1000>;
+			arm,data-latency = <2 2 0>;
+			cache-unified;
+			cache-level = <2>;
+		};
+
+		intc: interrupt-controller@2000000 {
+			compatible = "qcom,msm-qgic2";
+			interrupt-controller;
+			#interrupt-cells = <3>;
+			reg = <0x02000000 0x1000>,
+			      <0x02002000 0x1000>;
+		};
+
+		timer@200a000 {
+			compatible = "qcom,kpss-timer", "qcom,msm-timer";
+			interrupts = <GIC_PPI 1 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_EDGE_RISING)>,
+				     <GIC_PPI 2 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_EDGE_RISING)>,
+				     <GIC_PPI 3 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_EDGE_RISING)>;
+			reg = <0x0200a000 0x100>;
+			clock-frequency = <27000000>,
+					  <32768>;
+			cpu-offset = <0x80000>;
+		};
+
+		msmgpio: pinctrl@800000 {
+			compatible = "qcom,mdm9615-pinctrl";
+			gpio-controller;
+			#gpio-cells = <2>;
+			interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			reg = <0x800000 0x4000>;
+		};
+
+		gcc: clock-controller@900000 {
+			compatible = "qcom,gcc-mdm9615";
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			reg = <0x900000 0x4000>;
+		};
+
+		lcc: clock-controller@28000000 {
+			compatible = "qcom,lcc-mdm9615";
+			reg = <0x28000000 0x1000>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		l2cc: clock-controller@2011000 {
+			compatible = "syscon";
+			reg = <0x02011000 0x1000>;
+		};
+
+		rng@1a500000 {
+			compatible = "qcom,prng";
+			reg = <0x1a500000 0x200>;
+			clocks = <&gcc PRNG_CLK>;
+			clock-names = "core";
+			assigned-clocks = <&gcc PRNG_CLK>;
+			assigned-clock-rates = <32000000>;
+		};
+
+		gsbi2: gsbi@16100000 {
+			compatible = "qcom,gsbi-v1.0.0";
+			cell-index = <2>;
+			reg = <0x16100000 0x100>;
+			clocks = <&gcc GSBI2_H_CLK>;
+			clock-names = "iface";
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			gsbi2_i2c: i2c@16180000 {
+				compatible = "qcom,i2c-qup-v1.1.1";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x16180000 0x1000>;
+				interrupts = <GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GSBI2_QUP_CLK>, <&gcc GSBI2_H_CLK>;
+				clock-names = "core", "iface";
+				status = "disabled";
+			};
+		};
+
+		gsbi3: gsbi@16200000 {
+			compatible = "qcom,gsbi-v1.0.0";
+			cell-index = <3>;
+			reg = <0x16200000 0x100>;
+			clocks = <&gcc GSBI3_H_CLK>;
+			clock-names = "iface";
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			gsbi3_spi: spi@16280000 {
+				compatible = "qcom,spi-qup-v1.1.1";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x16280000 0x1000>;
+				interrupts = <GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>;
+				spi-max-frequency = <24000000>;
+
+				clocks = <&gcc GSBI3_QUP_CLK>, <&gcc GSBI3_H_CLK>;
+				clock-names = "core", "iface";
+				status = "disabled";
+			};
+		};
+
+		gsbi4: gsbi@16300000 {
+			compatible = "qcom,gsbi-v1.0.0";
+			cell-index = <4>;
+			reg = <0x16300000 0x100>;
+			clocks = <&gcc GSBI4_H_CLK>;
+			clock-names = "iface";
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			syscon-tcsr = <&tcsr>;
+
+			gsbi4_serial: serial@16340000 {
+				compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm";
+				reg = <0x16340000 0x1000>,
+				      <0x16300000 0x1000>;
+				interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GSBI4_UART_CLK>, <&gcc GSBI4_H_CLK>;
+				clock-names = "core", "iface";
+				status = "disabled";
+			};
+		};
+
+		gsbi5: gsbi@16400000 {
+			compatible = "qcom,gsbi-v1.0.0";
+			cell-index = <5>;
+			reg = <0x16400000 0x100>;
+			clocks = <&gcc GSBI5_H_CLK>;
+			clock-names = "iface";
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			syscon-tcsr = <&tcsr>;
+
+			gsbi5_i2c: i2c@16480000 {
+				compatible = "qcom,i2c-qup-v1.1.1";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x16480000 0x1000>;
+				interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
+
+				/* QUP clock is not initialized, set rate */
+				assigned-clocks = <&gcc GSBI5_QUP_CLK>;
+				assigned-clock-rates = <24000000>;
+
+				clocks = <&gcc GSBI5_QUP_CLK>, <&gcc GSBI5_H_CLK>;
+				clock-names = "core", "iface";
+				status = "disabled";
+			};
+
+			gsbi5_serial: serial@16440000 {
+				compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm";
+				reg = <0x16440000 0x1000>,
+				      <0x16400000 0x1000>;
+				interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GSBI5_UART_CLK>, <&gcc GSBI5_H_CLK>;
+				clock-names = "core", "iface";
+				status = "disabled";
+			};
+		};
+
+		qcom,ssbi@500000 {
+			compatible = "qcom,ssbi";
+			reg = <0x500000 0x1000>;
+			qcom,controller-type = "pmic-arbiter";
+
+			pmicintc: pmic@0 {
+				compatible = "qcom,pm8018", "qcom,pm8921";
+				interrupts = <GIC_PPI 226 IRQ_TYPE_LEVEL_HIGH>;
+				#interrupt-cells = <2>;
+				interrupt-controller;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				pwrkey@1c {
+					compatible = "qcom,pm8018-pwrkey", "qcom,pm8921-pwrkey";
+					reg = <0x1c>;
+					interrupt-parent = <&pmicintc>;
+					interrupts = <50 IRQ_TYPE_EDGE_RISING>,
+						     <51 IRQ_TYPE_EDGE_RISING>;
+					debounce = <15625>;
+					pull-up;
+				};
+
+				pmicmpp: mpp@50 {
+					compatible = "qcom,pm8018-mpp", "qcom,ssbi-mpp";
+					interrupt-parent = <&pmicintc>;
+					interrupts = <24 IRQ_TYPE_NONE>,
+						     <25 IRQ_TYPE_NONE>,
+						     <26 IRQ_TYPE_NONE>,
+						     <27 IRQ_TYPE_NONE>,
+						     <28 IRQ_TYPE_NONE>,
+						     <29 IRQ_TYPE_NONE>;
+					reg = <0x50>;
+					gpio-controller;
+					#gpio-cells = <2>;
+				};
+
+				rtc@11d {
+					compatible = "qcom,pm8018-rtc", "qcom,pm8921-rtc";
+					interrupt-parent = <&pmicintc>;
+					interrupts = <39 IRQ_TYPE_EDGE_RISING>;
+					reg = <0x11d>;
+					allow-set-time;
+				};
+
+				pmicgpio: gpio@150 {
+					compatible = "qcom,pm8018-gpio", "qcom,ssbi-gpio";
+					interrupt-parent = <&pmicintc>;
+					interrupts = <24 IRQ_TYPE_NONE>,
+						     <25 IRQ_TYPE_NONE>,
+						     <26 IRQ_TYPE_NONE>,
+						     <27 IRQ_TYPE_NONE>,
+						     <28 IRQ_TYPE_NONE>,
+						     <29 IRQ_TYPE_NONE>;
+					gpio-controller;
+					#gpio-cells = <2>;
+				};
+			};
+		};
+
+		sdcc1bam: dma@12182000{
+			compatible = "qcom,bam-v1.3.0";
+			reg = <0x12182000 0x8000>;
+			interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc SDC1_H_CLK>;
+			clock-names = "bam_clk";
+			#dma-cells = <1>;
+			qcom,ee = <0>;
+		};
+
+		sdcc2bam: dma@12142000{
+			compatible = "qcom,bam-v1.3.0";
+			reg = <0x12142000 0x8000>;
+			interrupts = <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc SDC2_H_CLK>;
+			clock-names = "bam_clk";
+			#dma-cells = <1>;
+			qcom,ee = <0>;
+		};
+
+		amba {
+			compatible = "arm,amba-bus";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			sdcc1: sdcc@12180000 {
+				status = "disabled";
+				compatible = "arm,pl18x", "arm,primecell";
+				arm,primecell-periphid = <0x00051180>;
+				reg = <0x12180000 0x2000>;
+				interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names	= "cmd_irq";
+				clocks = <&gcc SDC1_CLK>, <&gcc SDC1_H_CLK>;
+				clock-names = "mclk", "apb_pclk";
+				bus-width = <8>;
+				max-frequency = <48000000>;
+				cap-sd-highspeed;
+				cap-mmc-highspeed;
+				vmmc-supply = <&vsdcc_fixed>;
+				dmas = <&sdcc1bam 2>, <&sdcc1bam 1>;
+				dma-names = "tx", "rx";
+				assigned-clocks = <&gcc SDC1_CLK>;
+				assigned-clock-rates = <400000>;
+			};
+
+			sdcc2: sdcc@12140000 {
+				compatible = "arm,pl18x", "arm,primecell";
+				arm,primecell-periphid = <0x00051180>;
+				status = "disabled";
+				reg = <0x12140000 0x2000>;
+				interrupts = <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names	= "cmd_irq";
+				clocks = <&gcc SDC2_CLK>, <&gcc SDC2_H_CLK>;
+				clock-names = "mclk", "apb_pclk";
+				bus-width = <4>;
+				cap-sd-highspeed;
+				cap-mmc-highspeed;
+				max-frequency = <48000000>;
+				no-1-8-v;
+				vmmc-supply = <&vsdcc_fixed>;
+				dmas = <&sdcc2bam 2>, <&sdcc2bam 1>;
+				dma-names = "tx", "rx";
+				assigned-clocks = <&gcc SDC2_CLK>;
+				assigned-clock-rates = <400000>;
+			};
+		};
+
+		tcsr: syscon@1a400000 {
+			compatible = "qcom,tcsr-mdm9615", "syscon";
+			reg = <0x1a400000 0x100>;
+		};
+
+		rpm: rpm@108000 {
+			compatible = "qcom,rpm-mdm9615";
+			reg = <0x108000 0x1000>;
+
+			qcom,ipc = <&l2cc 0x8 2>;
+
+			interrupts = <GIC_SPI 19 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 21 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 22 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names	= "ack", "err", "wakeup";
+
+			regulators {
+				compatible = "qcom,rpm-pm8018-regulators";
+
+				vin_lvs1-supply = <&pm8018_s3>;
+
+				vdd_l7-supply = <&pm8018_s4>;
+				vdd_l8-supply = <&pm8018_s3>;
+				vdd_l9_l10_l11_l12-supply = <&pm8018_s5>;
+
+				/* Buck SMPS */
+				pm8018_s1: s1 {
+					regulator-min-microvolt = <500000>;
+					regulator-max-microvolt = <1150000>;
+					qcom,switch-mode-frequency = <1600000>;
+					bias-pull-down;
+				};
+
+				pm8018_s2: s2 {
+					regulator-min-microvolt = <1225000>;
+					regulator-max-microvolt = <1300000>;
+					qcom,switch-mode-frequency = <1600000>;
+					bias-pull-down;
+				};
+
+				pm8018_s3: s3 {
+					regulator-always-on;
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					qcom,switch-mode-frequency = <1600000>;
+					bias-pull-down;
+				};
+
+				pm8018_s4: s4 {
+					regulator-min-microvolt = <2100000>;
+					regulator-max-microvolt = <2200000>;
+					qcom,switch-mode-frequency = <1600000>;
+					bias-pull-down;
+				};
+
+				pm8018_s5: s5 {
+					regulator-always-on;
+					regulator-min-microvolt = <1350000>;
+					regulator-max-microvolt = <1350000>;
+					qcom,switch-mode-frequency = <1600000>;
+					bias-pull-down;
+				};
+
+				/* PMOS LDO */
+				pm8018_l2: l2 {
+					regulator-always-on;
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					bias-pull-down;
+				};
+
+				pm8018_l3: l3 {
+					regulator-always-on;
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <1800000>;
+					bias-pull-down;
+				};
+
+				pm8018_l4: l4 {
+					regulator-min-microvolt = <3300000>;
+					regulator-max-microvolt = <3300000>;
+					bias-pull-down;
+				};
+
+				pm8018_l5: l5 {
+					regulator-min-microvolt = <2850000>;
+					regulator-max-microvolt = <2850000>;
+					bias-pull-down;
+				};
+
+				pm8018_l6: l6 {
+					regulator-min-microvolt = <1800000>;
+					regulator-max-microvolt = <2850000>;
+					bias-pull-down;
+				};
+
+				pm8018_l7: l7 {
+					regulator-min-microvolt = <1850000>;
+					regulator-max-microvolt = <1900000>;
+					bias-pull-down;
+				};
+
+				pm8018_l8: l8 {
+					regulator-min-microvolt = <1200000>;
+					regulator-max-microvolt = <1200000>;
+					bias-pull-down;
+				};
+
+				pm8018_l9: l9 {
+					regulator-min-microvolt = <750000>;
+					regulator-max-microvolt = <1150000>;
+					bias-pull-down;
+				};
+
+				pm8018_l10: l10 {
+					regulator-min-microvolt = <1050000>;
+					regulator-max-microvolt = <1050000>;
+					bias-pull-down;
+				};
+
+				pm8018_l11: l11 {
+					regulator-min-microvolt = <1050000>;
+					regulator-max-microvolt = <1050000>;
+					bias-pull-down;
+				};
+
+				pm8018_l12: l12 {
+					regulator-min-microvolt = <1050000>;
+					regulator-max-microvolt = <1050000>;
+					bias-pull-down;
+				};
+
+				pm8018_l13: l13 {
+					regulator-min-microvolt = <1850000>;
+					regulator-max-microvolt = <2950000>;
+					bias-pull-down;
+				};
+
+				pm8018_l14: l14 {
+					regulator-min-microvolt = <2850000>;
+					regulator-max-microvolt = <2850000>;
+					bias-pull-down;
+				};
+
+				/* Low Voltage Switch */
+				pm8018_lvs1: lvs1 {
+					bias-pull-down;
+				};
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/qcom-msm8660.dtsi b/arch/arm/boot/dts/qcom-msm8660.dtsi
index 8c65e0d82559..4d828f810746 100644
--- a/arch/arm/boot/dts/qcom-msm8660.dtsi
+++ b/arch/arm/boot/dts/qcom-msm8660.dtsi
@@ -141,6 +141,23 @@
 			};
 		};
 
+		external-bus@1a100000 {
+			compatible = "qcom,msm8660-ebi2";
+			#address-cells = <2>;
+			#size-cells = <1>;
+			ranges = <0 0x0 0x1a800000 0x00800000>,
+				 <1 0x0 0x1b000000 0x00800000>,
+				 <2 0x0 0x1b800000 0x00800000>,
+				 <3 0x0 0x1d000000 0x08000000>,
+				 <4 0x0 0x1c800000 0x00800000>,
+				 <5 0x0 0x1c000000 0x00800000>;
+			reg = <0x1a100000 0x1000>, <0x1a110000 0x1000>;
+			reg-names = "ebi2", "xmem";
+			clocks = <&gcc EBI2_2X_CLK>, <&gcc EBI2_CLK>;
+			clock-names = "ebi2x", "ebi2";
+			status = "disabled";
+		};
+
 		qcom,ssbi@500000 {
 			compatible = "qcom,ssbi";
 			reg = <0x500000 0x1000>;
diff --git a/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts b/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts
index c0fb4a698c56..382bcc3231a9 100644
--- a/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts
+++ b/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts
@@ -224,6 +224,35 @@
 		status = "ok";
 	};
 
+	pinctrl@fd510000 {
+		sdhc1_pin_a: sdhc1-pin-active {
+			clk {
+				pins = "sdc1_clk";
+				drive-strength = <16>;
+				bias-disable;
+			};
+
+			cmd-data {
+				pins = "sdc1_cmd", "sdc1_data";
+				drive-strength = <10>;
+				bias-pull-up;
+			};
+		};
+	};
+
+	sdhci@f9824900 {
+		status = "ok";
+
+		vmmc-supply = <&pm8941_l20>;
+		vqmmc-supply = <&pm8941_s3>;
+
+		bus-width = <8>;
+		non-removable;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&sdhc1_pin_a>;
+	};
+
 	gpio-keys {
 		compatible = "gpio-keys";
 		input-name = "gpio-keys";
diff --git a/arch/arm/boot/dts/qcom-msm8974.dtsi b/arch/arm/boot/dts/qcom-msm8974.dtsi
index d2109475bdfd..49d579f28865 100644
--- a/arch/arm/boot/dts/qcom-msm8974.dtsi
+++ b/arch/arm/boot/dts/qcom-msm8974.dtsi
@@ -220,13 +220,13 @@
 	};
 
 	clocks {
-		xo_board {
+		xo_board: xo_board {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <19200000>;
 		};
 
-		sleep_clk {
+		sleep_clk: sleep_clk {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <32768>;
@@ -558,8 +558,10 @@
 			reg-names = "hc_mem", "core_mem";
 			interrupts = <0 123 0>, <0 138 0>;
 			interrupt-names = "hc_irq", "pwr_irq";
-			clocks = <&gcc GCC_SDCC1_APPS_CLK>, <&gcc GCC_SDCC1_AHB_CLK>;
-			clock-names = "core", "iface";
+			clocks = <&gcc GCC_SDCC1_APPS_CLK>,
+				 <&gcc GCC_SDCC1_AHB_CLK>,
+				 <&xo_board>;
+			clock-names = "core", "iface", "xo";
 			status = "disabled";
 		};
 
@@ -569,8 +571,10 @@
 			reg-names = "hc_mem", "core_mem";
 			interrupts = <0 125 0>, <0 221 0>;
 			interrupt-names = "hc_irq", "pwr_irq";
-			clocks = <&gcc GCC_SDCC2_APPS_CLK>, <&gcc GCC_SDCC2_AHB_CLK>;
-			clock-names = "core", "iface";
+			clocks = <&gcc GCC_SDCC2_APPS_CLK>,
+				 <&gcc GCC_SDCC2_AHB_CLK>,
+				 <&xo_board>;
+			clock-names = "core", "iface", "xo";
 			status = "disabled";
 		};
 
diff --git a/arch/arm/boot/dts/r7s72100-rskrza1.dts b/arch/arm/boot/dts/r7s72100-rskrza1.dts
index e5dea5bb4032..dd4418195ca6 100644
--- a/arch/arm/boot/dts/r7s72100-rskrza1.dts
+++ b/arch/arm/boot/dts/r7s72100-rskrza1.dts
@@ -56,6 +56,11 @@
 	};
 };
 
+&sdhi1 {
+	bus-width = <4>;
+	status = "okay";
+};
+
 &scif2 {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/r7s72100.dtsi b/arch/arm/boot/dts/r7s72100.dtsi
index fb9ef9ca120e..3dd427d68c83 100644
--- a/arch/arm/boot/dts/r7s72100.dtsi
+++ b/arch/arm/boot/dts/r7s72100.dtsi
@@ -117,6 +117,15 @@
 			clock-output-names = "ether";
 		};
 
+		mstp8_clks: mstp8_clks@fcfe0434 {
+			#clock-cells = <1>;
+			compatible = "renesas,r7s72100-mstp-clocks", "renesas,cpg-mstp-clocks";
+			reg = <0xfcfe0434 4>;
+			clocks = <&p1_clk>;
+			clock-indices = <R7S72100_CLK_MMCIF>;
+			clock-output-names = "mmcif";
+		};
+
 		mstp9_clks: mstp9_clks@fcfe0438 {
 			#clock-cells = <1>;
 			compatible = "renesas,r7s72100-mstp-clocks", "renesas,cpg-mstp-clocks";
@@ -140,6 +149,14 @@
 			>;
 			clock-output-names = "spi0", "spi1", "spi2", "spi3", "spi4";
 		};
+		mstp12_clks: mstp12_clks@fcfe0444 {
+			#clock-cells = <1>;
+			compatible = "renesas,r7s72100-mstp-clocks", "renesas,cpg-mstp-clocks";
+			reg = <0xfcfe0444 4>;
+			clocks = <&p1_clk>, <&p1_clk>;
+			clock-indices = <R7S72100_CLK_SDHI1 R7S72100_CLK_SDHI0>;
+			clock-output-names = "sdhi1", "sdhi0";
+		};
 	};
 
 	cpus {
@@ -441,4 +458,42 @@
 		#size-cells = <0>;
 		status = "disabled";
 	};
+
+	mmcif: mmc@e804c800 {
+		compatible = "renesas,mmcif-r7s72100", "renesas,sh-mmcif";
+		reg = <0xe804c800 0x80>;
+		interrupts = <GIC_SPI 268 IRQ_TYPE_LEVEL_HIGH
+			      GIC_SPI 269 IRQ_TYPE_LEVEL_HIGH
+			      GIC_SPI 267 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&mstp8_clks R7S72100_CLK_MMCIF>;
+		reg-io-width = <4>;
+		bus-width = <8>;
+		status = "disabled";
+	};
+
+	sdhi0: sd@e804e000 {
+		compatible = "renesas,sdhi-r7s72100";
+		reg = <0xe804e000 0x100>;
+		interrupts = <GIC_SPI 270 IRQ_TYPE_LEVEL_HIGH
+			      GIC_SPI 271 IRQ_TYPE_LEVEL_HIGH
+			      GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>;
+
+		clocks = <&mstp12_clks R7S72100_CLK_SDHI0>;
+		cap-sd-highspeed;
+		cap-sdio-irq;
+		status = "disabled";
+	};
+
+	sdhi1: sd@e804e800 {
+		compatible = "renesas,sdhi-r7s72100";
+		reg = <0xe804e800 0x100>;
+		interrupts = <GIC_SPI 273 IRQ_TYPE_LEVEL_HIGH
+			      GIC_SPI 274 IRQ_TYPE_LEVEL_HIGH
+			      GIC_SPI 275 IRQ_TYPE_LEVEL_HIGH>;
+
+		clocks = <&mstp12_clks R7S72100_CLK_SDHI1>;
+		cap-sd-highspeed;
+		cap-sdio-irq;
+		status = "disabled";
+	};
 };
diff --git a/arch/arm/boot/dts/r8a73a4.dtsi b/arch/arm/boot/dts/r8a73a4.dtsi
index ca8672778fe0..53183ffe04c1 100644
--- a/arch/arm/boot/dts/r8a73a4.dtsi
+++ b/arch/arm/boot/dts/r8a73a4.dtsi
@@ -751,6 +751,11 @@
 		};
 	};
 
+	prr: chipid@ff000044 {
+		compatible = "renesas,prr";
+		reg = <0 0xff000044 0 4>;
+	};
+
 	sysc: system-controller@e6180000 {
 		compatible = "renesas,sysc-r8a73a4", "renesas,sysc-rmobile";
 		reg = <0 0xe6180000 0 0x8000>, <0 0xe6188000 0 0x8000>;
diff --git a/arch/arm/boot/dts/r8a7740.dtsi b/arch/arm/boot/dts/r8a7740.dtsi
index 159e04eb1b9e..34159a8349de 100644
--- a/arch/arm/boot/dts/r8a7740.dtsi
+++ b/arch/arm/boot/dts/r8a7740.dtsi
@@ -8,8 +8,6 @@
  * kind, whether express or implied.
  */
 
-/include/ "skeleton.dtsi"
-
 #include <dt-bindings/clock/r8a7740-clock.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -17,6 +15,8 @@
 / {
 	compatible = "renesas,r8a7740";
 	interrupt-parent = <&gic>;
+	#address-cells = <1>;
+	#size-cells = <1>;
 
 	cpus {
 		#address-cells = <1>;
diff --git a/arch/arm/boot/dts/r8a7743-sk-rzg1m.dts b/arch/arm/boot/dts/r8a7743-sk-rzg1m.dts
new file mode 100644
index 000000000000..3a22538208f2
--- /dev/null
+++ b/arch/arm/boot/dts/r8a7743-sk-rzg1m.dts
@@ -0,0 +1,57 @@
+/*
+ * Device Tree Source for the SK-RZG1M board
+ *
+ * Copyright (C) 2016 Cogent Embedded, Inc.
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2. This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied.
+ */
+
+/dts-v1/;
+#include "r8a7743.dtsi"
+
+/ {
+	model = "SK-RZG1M";
+	compatible = "renesas,sk-rzg1m", "renesas,r8a7743";
+
+	aliases {
+		serial0 = &scif0;
+	};
+
+	chosen {
+		bootargs = "ignore_loglevel rw root=/dev/nfs ip=dhcp";
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0 0x40000000 0 0x40000000>;
+	};
+
+	memory@200000000 {
+		device_type = "memory";
+		reg = <2 0x00000000 0 0x40000000>;
+	};
+};
+
+&extal_clk {
+	clock-frequency = <20000000>;
+};
+
+&scif0 {
+	status = "okay";
+};
+
+&ether {
+	phy-handle = <&phy1>;
+	renesas,ether-link-active-low;
+	status = "okay";
+
+	phy1: ethernet-phy@1 {
+		reg = <1>;
+		interrupt-parent = <&irqc>;
+		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+		micrel,led-mode = <1>;
+	};
+};
diff --git a/arch/arm/boot/dts/r8a7743.dtsi b/arch/arm/boot/dts/r8a7743.dtsi
new file mode 100644
index 000000000000..216cb1f37f87
--- /dev/null
+++ b/arch/arm/boot/dts/r8a7743.dtsi
@@ -0,0 +1,476 @@
+/*
+ * Device Tree Source for the r8a7743 SoC
+ *
+ * Copyright (C) 2016 Cogent Embedded Inc.
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2. This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied.
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/r8a7743-cpg-mssr.h>
+#include <dt-bindings/power/r8a7743-sysc.h>
+
+/ {
+	compatible = "renesas,r8a7743";
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a15";
+			reg = <0>;
+			clock-frequency = <1500000000>;
+			clocks = <&cpg CPG_CORE R8A7743_CLK_Z>;
+			power-domains = <&sysc R8A7743_PD_CA15_CPU0>;
+			next-level-cache = <&L2_CA15>;
+		};
+
+		L2_CA15: cache-controller@0 {
+			compatible = "cache";
+			reg = <0>;
+			cache-unified;
+			cache-level = <2>;
+			power-domains = <&sysc R8A7743_PD_CA15_SCU>;
+		};
+	};
+
+	soc {
+		compatible = "simple-bus";
+		interrupt-parent = <&gic>;
+
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		gic: interrupt-controller@f1001000 {
+			compatible = "arm,gic-400";
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0 0xf1001000 0 0x1000>,
+			      <0 0xf1002000 0 0x1000>,
+			      <0 0xf1004000 0 0x2000>,
+			      <0 0xf1006000 0 0x2000>;
+			interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(2) |
+						 IRQ_TYPE_LEVEL_HIGH)>;
+		};
+
+		irqc: interrupt-controller@e61c0000 {
+			compatible = "renesas,irqc-r8a7743", "renesas,irqc";
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			reg = <0 0xe61c0000 0 0x200>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 407>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+		};
+
+		timer {
+			compatible = "arm,armv7-timer";
+			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) |
+						  IRQ_TYPE_LEVEL_LOW)>,
+				     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) |
+						  IRQ_TYPE_LEVEL_LOW)>,
+				     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) |
+						  IRQ_TYPE_LEVEL_LOW)>,
+				     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) |
+						  IRQ_TYPE_LEVEL_LOW)>;
+		};
+
+		cpg: clock-controller@e6150000 {
+			compatible = "renesas,r8a7743-cpg-mssr";
+			reg = <0 0xe6150000 0 0x1000>;
+			clocks = <&extal_clk>, <&usb_extal_clk>;
+			clock-names = "extal", "usb_extal";
+			#clock-cells = <2>;
+			#power-domain-cells = <0>;
+		};
+
+		sysc: system-controller@e6180000 {
+			compatible = "renesas,r8a7743-sysc";
+			reg = <0 0xe6180000 0 0x200>;
+			#power-domain-cells = <1>;
+		};
+
+		rst: reset-controller@e6160000 {
+			compatible = "renesas,r8a7743-rst";
+			reg = <0 0xe6160000 0 0x100>;
+		};
+
+		dmac0: dma-controller@e6700000 {
+			compatible = "renesas,dmac-r8a7743",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe6700000 0 0x20000>;
+			interrupts = <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 204 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 213 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14";
+			clocks = <&cpg CPG_MOD 219>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			#dma-cells = <1>;
+			dma-channels = <15>;
+		};
+
+		dmac1: dma-controller@e6720000 {
+			compatible = "renesas,dmac-r8a7743",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe6720000 0 0x20000>;
+			interrupts = <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 310 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 311 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 313 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 315 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 316 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 317 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14";
+			clocks = <&cpg CPG_MOD 218>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			#dma-cells = <1>;
+			dma-channels = <15>;
+		};
+
+		scifa0: serial@e6c40000 {
+			compatible = "renesas,scifa-r8a7743",
+				     "renesas,rcar-gen2-scifa", "renesas,scifa";
+			reg = <0 0xe6c40000 0 0x40>;
+			interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 204>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x21>, <&dmac0 0x22>,
+			       <&dmac1 0x21>, <&dmac1 0x22>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifa1: serial@e6c50000 {
+			compatible = "renesas,scifa-r8a7743",
+				     "renesas,rcar-gen2-scifa", "renesas,scifa";
+			reg = <0 0xe6c50000 0 0x40>;
+			interrupts = <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 203>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x25>, <&dmac0 0x26>,
+			       <&dmac1 0x25>, <&dmac1 0x26>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifa2: serial@e6c60000 {
+			compatible = "renesas,scifa-r8a7743",
+				     "renesas,rcar-gen2-scifa", "renesas,scifa";
+			reg = <0 0xe6c60000 0 0x40>;
+			interrupts = <GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 202>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x27>, <&dmac0 0x28>,
+			       <&dmac1 0x27>, <&dmac1 0x28>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifa3: serial@e6c70000 {
+			compatible = "renesas,scifa-r8a7743",
+				     "renesas,rcar-gen2-scifa", "renesas,scifa";
+			reg = <0 0xe6c70000 0 0x40>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 1106>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x1b>, <&dmac0 0x1c>,
+			       <&dmac1 0x1b>, <&dmac1 0x1c>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifa4: serial@e6c78000 {
+			compatible = "renesas,scifa-r8a7743",
+				     "renesas,rcar-gen2-scifa", "renesas,scifa";
+			reg = <0 0xe6c78000 0 0x40>;
+			interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 1107>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x1f>, <&dmac0 0x20>,
+			       <&dmac1 0x1f>, <&dmac1 0x20>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifa5: serial@e6c80000 {
+			compatible = "renesas,scifa-r8a7743",
+				     "renesas,rcar-gen2-scifa", "renesas,scifa";
+			reg = <0 0xe6c80000 0 0x40>;
+			interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 1108>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x23>, <&dmac0 0x24>,
+			       <&dmac1 0x23>, <&dmac1 0x24>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifb0: serial@e6c20000 {
+			compatible = "renesas,scifb-r8a7743",
+				     "renesas,rcar-gen2-scifb", "renesas,scifb";
+			reg = <0 0xe6c20000 0 0x100>;
+			interrupts = <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 206>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x3d>, <&dmac0 0x3e>,
+		       <&dmac1 0x3d>, <&dmac1 0x3e>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifb1: serial@e6c30000 {
+			compatible = "renesas,scifb-r8a7743",
+				     "renesas,rcar-gen2-scifb", "renesas,scifb";
+			reg = <0 0xe6c30000 0 0x100>;
+			interrupts = <GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 207>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x19>, <&dmac0 0x1a>,
+			       <&dmac1 0x19>, <&dmac1 0x1a>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifb2: serial@e6ce0000 {
+			compatible = "renesas,scifb-r8a7743",
+				     "renesas,rcar-gen2-scifb", "renesas,scifb";
+			reg = <0 0xe6ce0000 0 0x100>;
+			interrupts = <GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 216>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x1d>, <&dmac0 0x1e>,
+			       <&dmac1 0x1d>, <&dmac1 0x1e>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif0: serial@e6e60000 {
+			compatible = "renesas,scif-r8a7743",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6e60000 0 0x40>;
+			interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 721>,
+				 <&cpg CPG_CORE R8A7743_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x29>, <&dmac0 0x2a>,
+			       <&dmac1 0x29>, <&dmac1 0x2a>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif1: serial@e6e68000 {
+			compatible = "renesas,scif-r8a7743",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6e68000 0 0x40>;
+			interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 720>,
+				 <&cpg CPG_CORE R8A7743_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x2d>, <&dmac0 0x2e>,
+			       <&dmac1 0x2d>, <&dmac1 0x2e>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif2: serial@e6e58000 {
+			compatible = "renesas,scif-r8a7743",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6e58000 0 0x40>;
+			interrupts = <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 719>,
+				 <&cpg CPG_CORE R8A7743_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x2b>, <&dmac0 0x2c>,
+			       <&dmac1 0x2b>, <&dmac1 0x2c>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif3: serial@e6ea8000 {
+			compatible = "renesas,scif-r8a7743",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6ea8000 0 0x40>;
+			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 718>,
+				 <&cpg CPG_CORE R8A7743_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x2f>, <&dmac0 0x30>,
+			       <&dmac1 0x2f>, <&dmac1 0x30>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif4: serial@e6ee0000 {
+			compatible = "renesas,scif-r8a7743",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6ee0000 0 0x40>;
+			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 715>,
+				 <&cpg CPG_CORE R8A7743_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0xfb>, <&dmac0 0xfc>,
+			       <&dmac1 0xfb>, <&dmac1 0xfc>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif5: serial@e6ee8000 {
+			compatible = "renesas,scif-r8a7743",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6ee8000 0 0x40>;
+			interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 714>,
+				 <&cpg CPG_CORE R8A7743_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0xfd>, <&dmac0 0xfe>,
+			       <&dmac1 0xfd>, <&dmac1 0xfe>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		hscif0: serial@e62c0000 {
+			compatible = "renesas,hscif-r8a7743",
+				     "renesas,rcar-gen2-hscif", "renesas,hscif";
+			reg = <0 0xe62c0000 0 0x60>;
+			interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 717>,
+				 <&cpg CPG_CORE R8A7743_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x39>, <&dmac0 0x3a>,
+			       <&dmac1 0x39>, <&dmac1 0x3a>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		hscif1: serial@e62c8000 {
+			compatible = "renesas,hscif-r8a7743",
+				     "renesas,rcar-gen2-hscif", "renesas,hscif";
+			reg = <0 0xe62c8000 0 0x60>;
+			interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 716>,
+				 <&cpg CPG_CORE R8A7743_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x4d>, <&dmac0 0x4e>,
+			       <&dmac1 0x4d>, <&dmac1 0x4e>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		hscif2: serial@e62d0000 {
+			compatible = "renesas,hscif-r8a7743",
+				     "renesas,rcar-gen2-hscif", "renesas,hscif";
+			reg = <0 0xe62d0000 0 0x60>;
+			interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 713>,
+				 <&cpg CPG_CORE R8A7743_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x3b>, <&dmac0 0x3c>,
+			       <&dmac1 0x3b>, <&dmac1 0x3c>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		ether: ethernet@ee700000 {
+			compatible = "renesas,ether-r8a7743";
+			reg = <0 0xee700000 0 0x400>;
+			interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 813>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			phy-mode = "rmii";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+	};
+
+	/* External root clock */
+	extal_clk: extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+
+	/* External USB clock - can be overridden by the board */
+	usb_extal_clk: usb_extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <48000000>;
+	};
+
+	/* External SCIF clock */
+	scif_clk: scif {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+};
diff --git a/arch/arm/boot/dts/r8a7745-sk-rzg1e.dts b/arch/arm/boot/dts/r8a7745-sk-rzg1e.dts
new file mode 100644
index 000000000000..97840b340197
--- /dev/null
+++ b/arch/arm/boot/dts/r8a7745-sk-rzg1e.dts
@@ -0,0 +1,52 @@
+/*
+ * Device Tree Source for the SK-RZG1E board
+ *
+ * Copyright (C) 2016 Cogent Embedded, Inc.
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2. This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied.
+ */
+
+/dts-v1/;
+#include "r8a7745.dtsi"
+
+/ {
+	model = "SK-RZG1E";
+	compatible = "renesas,sk-rzg1e", "renesas,r8a7745";
+
+	aliases {
+		serial0 = &scif2;
+	};
+
+	chosen {
+		bootargs = "ignore_loglevel rw root=/dev/nfs ip=dhcp";
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0 0x40000000 0 0x40000000>;
+	};
+};
+
+&extal_clk {
+	clock-frequency = <20000000>;
+};
+
+&scif2 {
+	status = "okay";
+};
+
+&ether {
+	phy-handle = <&phy1>;
+	renesas,ether-link-active-low;
+	status = "okay";
+
+	phy1: ethernet-phy@1 {
+		reg = <1>;
+		interrupt-parent = <&irqc>;
+		interrupts = <8 IRQ_TYPE_LEVEL_LOW>;
+		micrel,led-mode = <1>;
+	};
+};
diff --git a/arch/arm/boot/dts/r8a7745.dtsi b/arch/arm/boot/dts/r8a7745.dtsi
new file mode 100644
index 000000000000..0b2e2f37150f
--- /dev/null
+++ b/arch/arm/boot/dts/r8a7745.dtsi
@@ -0,0 +1,476 @@
+/*
+ * Device Tree Source for the r8a7745 SoC
+ *
+ * Copyright (C) 2016 Cogent Embedded Inc.
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2. This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied.
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/r8a7745-cpg-mssr.h>
+#include <dt-bindings/power/r8a7745-sysc.h>
+
+/ {
+	compatible = "renesas,r8a7745";
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a7";
+			reg = <0>;
+			clock-frequency = <1000000000>;
+			clocks = <&cpg CPG_CORE R8A7745_CLK_Z2>;
+			power-domains = <&sysc R8A7745_PD_CA7_CPU0>;
+			next-level-cache = <&L2_CA7>;
+		};
+
+		L2_CA7: cache-controller@0 {
+			compatible = "cache";
+			reg = <0>;
+			cache-unified;
+			cache-level = <2>;
+			power-domains = <&sysc R8A7745_PD_CA7_SCU>;
+		};
+	};
+
+	soc {
+		compatible = "simple-bus";
+		interrupt-parent = <&gic>;
+
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		gic: interrupt-controller@f1001000 {
+			compatible = "arm,gic-400";
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0 0xf1001000 0 0x1000>,
+			      <0 0xf1002000 0 0x1000>,
+			      <0 0xf1004000 0 0x2000>,
+			      <0 0xf1006000 0 0x2000>;
+			interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(2) |
+						 IRQ_TYPE_LEVEL_HIGH)>;
+		};
+
+		irqc: interrupt-controller@e61c0000 {
+			compatible = "renesas,irqc-r8a7745", "renesas,irqc";
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			reg = <0 0xe61c0000 0 0x200>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 407>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+		};
+
+		timer {
+			compatible = "arm,armv7-timer";
+			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) |
+						  IRQ_TYPE_LEVEL_LOW)>,
+				     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) |
+						  IRQ_TYPE_LEVEL_LOW)>,
+				     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) |
+						  IRQ_TYPE_LEVEL_LOW)>,
+				     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) |
+						  IRQ_TYPE_LEVEL_LOW)>;
+		};
+
+		cpg: clock-controller@e6150000 {
+			compatible = "renesas,r8a7745-cpg-mssr";
+			reg = <0 0xe6150000 0 0x1000>;
+			clocks = <&extal_clk>, <&usb_extal_clk>;
+			clock-names = "extal", "usb_extal";
+			#clock-cells = <2>;
+			#power-domain-cells = <0>;
+		};
+
+		sysc: system-controller@e6180000 {
+			compatible = "renesas,r8a7745-sysc";
+			reg = <0 0xe6180000 0 0x200>;
+			#power-domain-cells = <1>;
+		};
+
+		rst: reset-controller@e6160000 {
+			compatible = "renesas,r8a7745-rst";
+			reg = <0 0xe6160000 0 0x100>;
+		};
+
+		dmac0: dma-controller@e6700000 {
+			compatible = "renesas,dmac-r8a7745",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe6700000 0 0x20000>;
+			interrupts = <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 204 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 213 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14";
+			clocks = <&cpg CPG_MOD 219>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			#dma-cells = <1>;
+			dma-channels = <15>;
+		};
+
+		dmac1: dma-controller@e6720000 {
+			compatible = "renesas,dmac-r8a7745",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe6720000 0 0x20000>;
+			interrupts = <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 310 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 311 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 313 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 315 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 316 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 317 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14";
+			clocks = <&cpg CPG_MOD 218>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			#dma-cells = <1>;
+			dma-channels = <15>;
+		};
+
+		scifa0: serial@e6c40000 {
+			compatible = "renesas,scifa-r8a7745",
+				     "renesas,rcar-gen2-scifa", "renesas,scifa";
+			reg = <0 0xe6c40000 0 0x40>;
+			interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 204>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x21>, <&dmac0 0x22>,
+			       <&dmac1 0x21>, <&dmac1 0x22>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifa1: serial@e6c50000 {
+			compatible = "renesas,scifa-r8a7745",
+				     "renesas,rcar-gen2-scifa", "renesas,scifa";
+			reg = <0 0xe6c50000 0 0x40>;
+			interrupts = <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 203>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x25>, <&dmac0 0x26>,
+			       <&dmac1 0x25>, <&dmac1 0x26>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifa2: serial@e6c60000 {
+			compatible = "renesas,scifa-r8a7745",
+				     "renesas,rcar-gen2-scifa", "renesas,scifa";
+			reg = <0 0xe6c60000 0 0x40>;
+			interrupts = <GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 202>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x27>, <&dmac0 0x28>,
+			       <&dmac1 0x27>, <&dmac1 0x28>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifa3: serial@e6c70000 {
+			compatible = "renesas,scifa-r8a7745",
+				     "renesas,rcar-gen2-scifa", "renesas,scifa";
+			reg = <0 0xe6c70000 0 0x40>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 1106>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x1b>, <&dmac0 0x1c>,
+			       <&dmac1 0x1b>, <&dmac1 0x1c>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifa4: serial@e6c78000 {
+			compatible = "renesas,scifa-r8a7745",
+				     "renesas,rcar-gen2-scifa", "renesas,scifa";
+			reg = <0 0xe6c78000 0 0x40>;
+			interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 1107>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x1f>, <&dmac0 0x20>,
+			       <&dmac1 0x1f>, <&dmac1 0x20>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifa5: serial@e6c80000 {
+			compatible = "renesas,scifa-r8a7745",
+				     "renesas,rcar-gen2-scifa", "renesas,scifa";
+			reg = <0 0xe6c80000 0 0x40>;
+			interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 1108>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x23>, <&dmac0 0x24>,
+			       <&dmac1 0x23>, <&dmac1 0x24>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifb0: serial@e6c20000 {
+			compatible = "renesas,scifb-r8a7745",
+				     "renesas,rcar-gen2-scifb", "renesas,scifb";
+			reg = <0 0xe6c20000 0 0x100>;
+			interrupts = <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 206>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x3d>, <&dmac0 0x3e>,
+		       <&dmac1 0x3d>, <&dmac1 0x3e>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifb1: serial@e6c30000 {
+			compatible = "renesas,scifb-r8a7745",
+				     "renesas,rcar-gen2-scifb", "renesas,scifb";
+			reg = <0 0xe6c30000 0 0x100>;
+			interrupts = <GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 207>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x19>, <&dmac0 0x1a>,
+			       <&dmac1 0x19>, <&dmac1 0x1a>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scifb2: serial@e6ce0000 {
+			compatible = "renesas,scifb-r8a7745",
+				     "renesas,rcar-gen2-scifb", "renesas,scifb";
+			reg = <0 0xe6ce0000 0 0x100>;
+			interrupts = <GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 216>;
+			clock-names = "fck";
+			dmas = <&dmac0 0x1d>, <&dmac0 0x1e>,
+			       <&dmac1 0x1d>, <&dmac1 0x1e>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif0: serial@e6e60000 {
+			compatible = "renesas,scif-r8a7745",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6e60000 0 0x40>;
+			interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 721>,
+				 <&cpg CPG_CORE R8A7745_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x29>, <&dmac0 0x2a>,
+			       <&dmac1 0x29>, <&dmac1 0x2a>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif1: serial@e6e68000 {
+			compatible = "renesas,scif-r8a7745",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6e68000 0 0x40>;
+			interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 720>,
+				 <&cpg CPG_CORE R8A7745_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x2d>, <&dmac0 0x2e>,
+			       <&dmac1 0x2d>, <&dmac1 0x2e>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif2: serial@e6e58000 {
+			compatible = "renesas,scif-r8a7745",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6e58000 0 0x40>;
+			interrupts = <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 719>,
+				 <&cpg CPG_CORE R8A7745_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x2b>, <&dmac0 0x2c>,
+			       <&dmac1 0x2b>, <&dmac1 0x2c>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif3: serial@e6ea8000 {
+			compatible = "renesas,scif-r8a7745",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6ea8000 0 0x40>;
+			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 718>,
+				 <&cpg CPG_CORE R8A7745_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x2f>, <&dmac0 0x30>,
+			       <&dmac1 0x2f>, <&dmac1 0x30>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif4: serial@e6ee0000 {
+			compatible = "renesas,scif-r8a7745",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6ee0000 0 0x40>;
+			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 715>,
+				 <&cpg CPG_CORE R8A7745_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0xfb>, <&dmac0 0xfc>,
+			       <&dmac1 0xfb>, <&dmac1 0xfc>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		scif5: serial@e6ee8000 {
+			compatible = "renesas,scif-r8a7745",
+				     "renesas,rcar-gen2-scif", "renesas,scif";
+			reg = <0 0xe6ee8000 0 0x40>;
+			interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 714>,
+				 <&cpg CPG_CORE R8A7745_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0xfd>, <&dmac0 0xfe>,
+			       <&dmac1 0xfd>, <&dmac1 0xfe>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		hscif0: serial@e62c0000 {
+			compatible = "renesas,hscif-r8a7745",
+				     "renesas,rcar-gen2-hscif", "renesas,hscif";
+			reg = <0 0xe62c0000 0 0x60>;
+			interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 717>,
+				 <&cpg CPG_CORE R8A7745_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x39>, <&dmac0 0x3a>,
+			       <&dmac1 0x39>, <&dmac1 0x3a>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		hscif1: serial@e62c8000 {
+			compatible = "renesas,hscif-r8a7745",
+				     "renesas,rcar-gen2-hscif", "renesas,hscif";
+			reg = <0 0xe62c8000 0 0x60>;
+			interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 716>,
+				 <&cpg CPG_CORE R8A7745_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x4d>, <&dmac0 0x4e>,
+			       <&dmac1 0x4d>, <&dmac1 0x4e>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		hscif2: serial@e62d0000 {
+			compatible = "renesas,hscif-r8a7745",
+				     "renesas,rcar-gen2-hscif", "renesas,hscif";
+			reg = <0 0xe62d0000 0 0x60>;
+			interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 713>,
+				 <&cpg CPG_CORE R8A7745_CLK_ZS>, <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac0 0x3b>, <&dmac0 0x3c>,
+			       <&dmac1 0x3b>, <&dmac1 0x3c>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		ether: ethernet@ee700000 {
+			compatible = "renesas,ether-r8a7745";
+			reg = <0 0xee700000 0 0x400>;
+			interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 813>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			phy-mode = "rmii";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+	};
+
+	/* External root clock */
+	extal_clk: extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+
+	/* External USB clock - can be overridden by the board */
+	usb_extal_clk: usb_extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <48000000>;
+	};
+
+	/* External SCIF clock */
+	scif_clk: scif {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+};
diff --git a/arch/arm/boot/dts/r8a7778.dtsi b/arch/arm/boot/dts/r8a7778.dtsi
index e571d66ea0fe..d0db998effc8 100644
--- a/arch/arm/boot/dts/r8a7778.dtsi
+++ b/arch/arm/boot/dts/r8a7778.dtsi
@@ -14,8 +14,6 @@
  * kind, whether express or implied.
  */
 
-/include/ "skeleton.dtsi"
-
 #include <dt-bindings/clock/r8a7778-clock.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -23,6 +21,8 @@
 / {
 	compatible = "renesas,r8a7778";
 	interrupt-parent = <&gic>;
+	#address-cells = <1>;
+	#size-cells = <1>;
 
 	cpus {
 		#address-cells = <1>;
@@ -626,4 +626,9 @@
 				"sru-src6", "sru-src7", "sru-src8";
 		};
 	};
+
+	rst: reset-controller@ffcc0000 {
+		compatible = "renesas,r8a7778-reset-wdt";
+		reg = <0xffcc0000 0x40>;
+	};
 };
diff --git a/arch/arm/boot/dts/r8a7779-marzen.dts b/arch/arm/boot/dts/r8a7779-marzen.dts
index 541678df90a9..676151b70185 100644
--- a/arch/arm/boot/dts/r8a7779-marzen.dts
+++ b/arch/arm/boot/dts/r8a7779-marzen.dts
@@ -170,7 +170,7 @@
 
 	du_pins: du {
 		du0 {
-			groups = "du0_rgb888", "du0_sync_1", "du0_clk_out_0";
+			groups = "du0_rgb888", "du0_sync_1", "du0_clk_out_0", "du0_clk_in";
 			function = "du0";
 		};
 		du1 {
diff --git a/arch/arm/boot/dts/r8a7779.dtsi b/arch/arm/boot/dts/r8a7779.dtsi
index b9bbcce69dfb..55a7c1e37c57 100644
--- a/arch/arm/boot/dts/r8a7779.dtsi
+++ b/arch/arm/boot/dts/r8a7779.dtsi
@@ -9,8 +9,6 @@
  * kind, whether express or implied.
  */
 
-/include/ "skeleton.dtsi"
-
 #include <dt-bindings/clock/r8a7779-clock.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -19,6 +17,8 @@
 / {
 	compatible = "renesas,r8a7779";
 	interrupt-parent = <&gic>;
+	#address-cells = <1>;
+	#size-cells = <1>;
 
 	cpus {
 		#address-cells = <1>;
@@ -420,7 +420,7 @@
 
 	du: display@fff80000 {
 		compatible = "renesas,du-r8a7779";
-		reg = <0 0xfff80000 0 0x40000>;
+		reg = <0xfff80000 0x40000>;
 		interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp1_clks R8A7779_CLK_DU>;
 		power-domains = <&sysc R8A7779_PD_ALWAYS_ON>;
@@ -590,6 +590,16 @@
 		};
 	};
 
+	prr: chipid@ff000044 {
+		compatible = "renesas,prr";
+		reg = <0xff000044 4>;
+	};
+
+	rst: reset-controller@ffcc0000 {
+		compatible = "renesas,r8a7779-reset-wdt";
+		reg = <0xffcc0000 0x48>;
+	};
+
 	sysc: system-controller@ffd85000 {
 		compatible = "renesas,r8a7779-sysc";
 		reg = <0xffd85000 0x0200>;
diff --git a/arch/arm/boot/dts/r8a7790-lager.dts b/arch/arm/boot/dts/r8a7790-lager.dts
index 52b56fcaddf2..bd512c86e852 100644
--- a/arch/arm/boot/dts/r8a7790-lager.dts
+++ b/arch/arm/boot/dts/r8a7790-lager.dts
@@ -50,7 +50,9 @@
 	aliases {
 		serial0 = &scif0;
 		serial1 = &scifa1;
-		i2c8 = "i2cexio";
+		i2c8 = &gpioi2c1;
+		i2c10 = &i2cexio0;
+		i2c11 = &i2cexio1;
 	};
 
 	chosen {
@@ -231,12 +233,23 @@
 		};
 	};
 
+	hdmi-in {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&adv7612_in>;
+			};
+		};
+	};
+
 	hdmi-out {
 		compatible = "hdmi-connector";
 		type = "a";
 
 		port {
-			hdmi_con: endpoint {
+			hdmi_con_out: endpoint {
 				remote-endpoint = <&adv7511_out>;
 			};
 		};
@@ -254,6 +267,17 @@
 		clock-frequency = <148500000>;
 	};
 
+	gpioi2c1: i2c-8 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "i2c-gpio";
+		status = "disabled";
+		gpios = <&gpio1 17 GPIO_ACTIVE_HIGH /* sda */
+			 &gpio1 16 GPIO_ACTIVE_HIGH /* scl */
+			>;
+		i2c-gpio,delay-us = <5>;
+	};
+
 	/*
 	 * IIC0/I2C0 is routed to EXIO connector A, pins 114 (SCL) + 116 (SDA) only.
 	 * We use the I2C demuxer, so the desired IP core can be selected at runtime
@@ -262,11 +286,26 @@
 	 * bus with IIC3 on pins 110 (SCL) + 112 (SDA), select I2C0 at runtime, and
 	 * instantiate the slave device at runtime according to the documentation.
 	 * You can then communicate with the slave via IIC3.
+	 *
+	 * IIC0/I2C0 does not appear to support fallback to GPIO.
 	 */
-	i2cexio: i2c-8 {
+	i2cexio0: i2c-10 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&iic0>, <&i2c0>;
-		i2c-bus-name = "i2c-exio";
+		i2c-bus-name = "i2c-exio0";
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	/*
+	 * IIC1/I2C1 is routed to EXIO connector A, pins 78 (SCL) + 80 (SDA).
+	 * This is similar to the arangement described for i2cexio0 (above)
+	 * with a fallback to GPIO also provided.
+	 */
+	i2cexio1: i2c-11 {
+		compatible = "i2c-demux-pinctrl";
+		i2c-parent = <&iic1>, <&i2c1>, <&gpioi2c1>;
+		i2c-bus-name = "i2c-exio1";
 		#address-cells = <1>;
 		#size-cells = <0>;
 	};
@@ -392,6 +431,11 @@
 		function = "iic0";
 	};
 
+	i2c1_pins: i2c1 {
+		groups = "i2c1";
+		function = "i2c1";
+	};
+
 	iic1_pins: iic1 {
 		groups = "iic1";
 		function = "iic1";
@@ -427,6 +471,11 @@
 		function = "usb2";
 	};
 
+	vin0_pins: vin0 {
+		groups = "vin0_data24", "vin0_sync", "vin0_clkenb", "vin0_clk";
+		function = "vin0";
+	};
+
 	vin1_pins: vin1 {
 		groups = "vin1_data8", "vin1_clk";
 		function = "vin1";
@@ -559,6 +608,7 @@
 	vqmmc-supply = <&vccq_sdhi0>;
 	cd-gpios = <&gpio3 6 GPIO_ACTIVE_LOW>;
 	sd-uhs-sdr50;
+	sd-uhs-sdr104;
 	status = "okay";
 };
 
@@ -580,18 +630,22 @@
 
 &i2c0	{
 	pinctrl-0 = <&i2c0_pins>;
-	pinctrl-names = "i2c-exio";
+	pinctrl-names = "i2c-exio0";
 };
 
 &iic0	{
 	pinctrl-0 = <&iic0_pins>;
-	pinctrl-names = "i2c-exio";
+	pinctrl-names = "i2c-exio0";
+};
+
+&i2c1	{
+	pinctrl-0 = <&i2c1_pins>;
+	pinctrl-names = "i2c-exio1";
 };
 
 &iic1	{
-	status = "okay";
 	pinctrl-0 = <&iic1_pins>;
-	pinctrl-names = "default";
+	pinctrl-names = "i2c-exio1";
 };
 
 &iic2	{
@@ -646,7 +700,34 @@
 			port@1 {
 				reg = <1>;
 				adv7511_out: endpoint {
-					remote-endpoint = <&hdmi_con>;
+					remote-endpoint = <&hdmi_con_out>;
+				};
+			};
+		};
+	};
+
+	hdmi-in@4c {
+		compatible = "adi,adv7612";
+		reg = <0x4c>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <20 IRQ_TYPE_LEVEL_LOW>;
+		default-input = <0>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				adv7612_in: endpoint {
+					remote-endpoint = <&hdmi_con_in>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+				adv7612_out: endpoint {
+					remote-endpoint = <&vin0ep2>;
 				};
 			};
 		};
@@ -722,6 +803,25 @@
 	status = "okay";
 };
 
+/* HDMI video input */
+&vin0 {
+	pinctrl-0 = <&vin0_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	port {
+		vin0ep2: endpoint {
+			remote-endpoint = <&adv7612_out>;
+			bus-width = <24>;
+			hsync-active = <0>;
+			vsync-active = <0>;
+			pclk-sample = <1>;
+			data-active = <1>;
+		};
+	};
+};
+
 /* composite video input */
 &vin1 {
 	pinctrl-0 = <&vin1_pins>;
diff --git a/arch/arm/boot/dts/r8a7790.dtsi b/arch/arm/boot/dts/r8a7790.dtsi
index 351fcc2f87df..0c8900d4b824 100644
--- a/arch/arm/boot/dts/r8a7790.dtsi
+++ b/arch/arm/boot/dts/r8a7790.dtsi
@@ -711,7 +711,7 @@
 	scifb0: serial@e6c20000 {
 		compatible = "renesas,scifb-r8a7790",
 			     "renesas,rcar-gen2-scifb", "renesas,scifb";
-		reg = <0 0xe6c20000 0 64>;
+		reg = <0 0xe6c20000 0 0x100>;
 		interrupts = <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp2_clks R8A7790_CLK_SCIFB0>;
 		clock-names = "fck";
@@ -725,7 +725,7 @@
 	scifb1: serial@e6c30000 {
 		compatible = "renesas,scifb-r8a7790",
 			     "renesas,rcar-gen2-scifb", "renesas,scifb";
-		reg = <0 0xe6c30000 0 64>;
+		reg = <0 0xe6c30000 0 0x100>;
 		interrupts = <GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp2_clks R8A7790_CLK_SCIFB1>;
 		clock-names = "fck";
@@ -739,7 +739,7 @@
 	scifb2: serial@e6ce0000 {
 		compatible = "renesas,scifb-r8a7790",
 			     "renesas,rcar-gen2-scifb", "renesas,scifb";
-		reg = <0 0xe6ce0000 0 64>;
+		reg = <0 0xe6ce0000 0 0x100>;
 		interrupts = <GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp2_clks R8A7790_CLK_SCIFB2>;
 		clock-names = "fck";
@@ -1471,6 +1471,16 @@
 		};
 	};
 
+	prr: chipid@ff000044 {
+		compatible = "renesas,prr";
+		reg = <0 0xff000044 0 4>;
+	};
+
+	rst: reset-controller@e6160000 {
+		compatible = "renesas,r8a7790-rst";
+		reg = <0 0xe6160000 0 0x0100>;
+	};
+
 	sysc: system-controller@e6180000 {
 		compatible = "renesas,r8a7790-sysc";
 		reg = <0 0xe6180000 0 0x0200>;
diff --git a/arch/arm/boot/dts/r8a7791-koelsch.dts b/arch/arm/boot/dts/r8a7791-koelsch.dts
index f8a7d090fd01..5405d337d744 100644
--- a/arch/arm/boot/dts/r8a7791-koelsch.dts
+++ b/arch/arm/boot/dts/r8a7791-koelsch.dts
@@ -50,6 +50,8 @@
 	aliases {
 		serial0 = &scif0;
 		serial1 = &scif1;
+		i2c9 = &gpioi2c1;
+		i2c12 = &i2cexio1;
 	};
 
 	chosen {
@@ -265,12 +267,23 @@
 		};
 	};
 
+	hdmi-in {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&adv7612_in>;
+			};
+		};
+	};
+
 	hdmi-out {
 		compatible = "hdmi-connector";
 		type = "a";
 
 		port {
-			hdmi_con: endpoint {
+			hdmi_con_out: endpoint {
 				remote-endpoint = <&adv7511_out>;
 			};
 		};
@@ -287,6 +300,29 @@
 		#clock-cells = <0>;
 		clock-frequency = <148500000>;
 	};
+
+	gpioi2c1: i2c-9 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "i2c-gpio";
+		status = "disabled";
+		gpios = <&gpio7 16 GPIO_ACTIVE_HIGH /* sda */
+			 &gpio7 15 GPIO_ACTIVE_HIGH /* scl */
+			>;
+		i2c-gpio,delay-us = <5>;
+	};
+
+	/*
+	 * I2C1 is routed to EXIO connector B, pins 64 (SCL) + 66 (SDA).
+	 * A fallback to GPIO is provided.
+	 */
+	i2cexio1: i2c-12 {
+		compatible = "i2c-demux-pinctrl";
+		i2c-parent = <&i2c1>, <&gpioi2c1>;
+		i2c-bus-name = "i2c-exio1";
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
 };
 
 &du {
@@ -322,6 +358,11 @@
 	pinctrl-0 = <&scif_clk_pins>;
 	pinctrl-names = "default";
 
+	i2c1_pins: i2c1 {
+		groups = "i2c1";
+		function = "i2c1";
+	};
+
 	i2c2_pins: i2c2 {
 		groups = "i2c2";
 		function = "i2c2";
@@ -360,16 +401,37 @@
 	sdhi0_pins: sd0 {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
+		power-source = <3300>;
+	};
+
+	sdhi0_pins_uhs: sd0_uhs {
+		groups = "sdhi0_data4", "sdhi0_ctrl";
+		function = "sdhi0";
+		power-source = <1800>;
 	};
 
 	sdhi1_pins: sd1 {
 		groups = "sdhi1_data4", "sdhi1_ctrl";
 		function = "sdhi1";
+		power-source = <3300>;
+	};
+
+	sdhi1_pins_uhs: sd1_uhs {
+		groups = "sdhi1_data4", "sdhi1_ctrl";
+		function = "sdhi1";
+		power-source = <1800>;
 	};
 
 	sdhi2_pins: sd2 {
 		groups = "sdhi2_data4", "sdhi2_ctrl";
 		function = "sdhi2";
+		power-source = <3300>;
+	};
+
+	sdhi2_pins_uhs: sd2_uhs {
+		groups = "sdhi2_data4", "sdhi2_ctrl";
+		function = "sdhi2";
+		power-source = <1800>;
 	};
 
 	qspi_pins: qspi {
@@ -393,6 +455,11 @@
 		function = "usb1";
 	};
 
+	vin0_pins: vin0 {
+		groups = "vin0_data24", "vin0_sync", "vin0_clkenb", "vin0_clk";
+		function = "vin0";
+	};
+
 	vin1_pins: vin1 {
 		groups = "vin1_data8", "vin1_clk";
 		function = "vin1";
@@ -454,33 +521,40 @@
 
 &sdhi0 {
 	pinctrl-0 = <&sdhi0_pins>;
-	pinctrl-names = "default";
+	pinctrl-1 = <&sdhi0_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
 
 	vmmc-supply = <&vcc_sdhi0>;
 	vqmmc-supply = <&vccq_sdhi0>;
 	cd-gpios = <&gpio6 6 GPIO_ACTIVE_LOW>;
 	wp-gpios = <&gpio6 7 GPIO_ACTIVE_HIGH>;
+	sd-uhs-sdr50;
+	sd-uhs-sdr104;
 	status = "okay";
 };
 
 &sdhi1 {
 	pinctrl-0 = <&sdhi1_pins>;
-	pinctrl-names = "default";
+	pinctrl-1 = <&sdhi1_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
 
 	vmmc-supply = <&vcc_sdhi1>;
 	vqmmc-supply = <&vccq_sdhi1>;
 	cd-gpios = <&gpio6 14 GPIO_ACTIVE_LOW>;
 	wp-gpios = <&gpio6 15 GPIO_ACTIVE_HIGH>;
+	sd-uhs-sdr50;
 	status = "okay";
 };
 
 &sdhi2 {
 	pinctrl-0 = <&sdhi2_pins>;
-	pinctrl-names = "default";
+	pinctrl-1 = <&sdhi2_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
 
 	vmmc-supply = <&vcc_sdhi2>;
 	vqmmc-supply = <&vccq_sdhi2>;
 	cd-gpios = <&gpio6 22 GPIO_ACTIVE_LOW>;
+	sd-uhs-sdr50;
 	status = "okay";
 };
 
@@ -538,6 +612,11 @@
 	};
 };
 
+&i2c1 {
+	pinctrl-0 = <&i2c1_pins>;
+	pinctrl-names = "i2c-exio1";
+};
+
 &i2c2 {
 	pinctrl-0 = <&i2c2_pins>;
 	pinctrl-names = "default";
@@ -590,7 +669,34 @@
 			port@1 {
 				reg = <1>;
 				adv7511_out: endpoint {
-					remote-endpoint = <&hdmi_con>;
+					remote-endpoint = <&hdmi_con_out>;
+				};
+			};
+		};
+	};
+
+	hdmi-in@4c {
+		compatible = "adi,adv7612";
+		reg = <0x4c>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
+		default-input = <0>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				adv7612_in: endpoint {
+					remote-endpoint = <&hdmi_con_in>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+				adv7612_out: endpoint {
+					remote-endpoint = <&vin0ep2>;
 				};
 			};
 		};
@@ -672,6 +778,27 @@
 	cpu0-supply = <&vdd_dvfs>;
 };
 
+/* HDMI video input */
+&vin0 {
+	status = "okay";
+	pinctrl-0 = <&vin0_pins>;
+	pinctrl-names = "default";
+
+	port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		vin0ep2: endpoint {
+			remote-endpoint = <&adv7612_out>;
+			bus-width = <24>;
+			hsync-active = <0>;
+			vsync-active = <0>;
+			pclk-sample = <1>;
+			data-active = <1>;
+		};
+	};
+};
+
 /* composite video input */
 &vin1 {
 	status = "okay";
diff --git a/arch/arm/boot/dts/r8a7791.dtsi b/arch/arm/boot/dts/r8a7791.dtsi
index 162b55c665a3..87214668d70f 100644
--- a/arch/arm/boot/dts/r8a7791.dtsi
+++ b/arch/arm/boot/dts/r8a7791.dtsi
@@ -584,6 +584,7 @@
 		dmas = <&dmac0 0xcd>, <&dmac0 0xce>,
 		       <&dmac1 0xcd>, <&dmac1 0xce>;
 		dma-names = "tx", "rx", "tx", "rx";
+		max-frequency = <195000000>;
 		power-domains = <&sysc R8A7791_PD_ALWAYS_ON>;
 		status = "disabled";
 	};
@@ -596,6 +597,7 @@
 		dmas = <&dmac0 0xc1>, <&dmac0 0xc2>,
 		       <&dmac1 0xc1>, <&dmac1 0xc2>;
 		dma-names = "tx", "rx", "tx", "rx";
+		max-frequency = <97500000>;
 		power-domains = <&sysc R8A7791_PD_ALWAYS_ON>;
 		status = "disabled";
 	};
@@ -608,6 +610,7 @@
 		dmas = <&dmac0 0xd3>, <&dmac0 0xd4>,
 		       <&dmac1 0xd3>, <&dmac1 0xd4>;
 		dma-names = "tx", "rx", "tx", "rx";
+		max-frequency = <97500000>;
 		power-domains = <&sysc R8A7791_PD_ALWAYS_ON>;
 		status = "disabled";
 	};
@@ -699,7 +702,7 @@
 	scifb0: serial@e6c20000 {
 		compatible = "renesas,scifb-r8a7791",
 			     "renesas,rcar-gen2-scifb", "renesas,scifb";
-		reg = <0 0xe6c20000 0 64>;
+		reg = <0 0xe6c20000 0 0x100>;
 		interrupts = <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp2_clks R8A7791_CLK_SCIFB0>;
 		clock-names = "fck";
@@ -713,7 +716,7 @@
 	scifb1: serial@e6c30000 {
 		compatible = "renesas,scifb-r8a7791",
 			     "renesas,rcar-gen2-scifb", "renesas,scifb";
-		reg = <0 0xe6c30000 0 64>;
+		reg = <0 0xe6c30000 0 0x100>;
 		interrupts = <GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp2_clks R8A7791_CLK_SCIFB1>;
 		clock-names = "fck";
@@ -727,7 +730,7 @@
 	scifb2: serial@e6ce0000 {
 		compatible = "renesas,scifb-r8a7791",
 			     "renesas,rcar-gen2-scifb", "renesas,scifb";
-		reg = <0 0xe6ce0000 0 64>;
+		reg = <0 0xe6ce0000 0 0x100>;
 		interrupts = <GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp2_clks R8A7791_CLK_SCIFB2>;
 		clock-names = "fck";
@@ -1482,6 +1485,16 @@
 		};
 	};
 
+	rst: reset-controller@e6160000 {
+		compatible = "renesas,r8a7791-rst";
+		reg = <0 0xe6160000 0 0x0100>;
+	};
+
+	prr: chipid@ff000044 {
+		compatible = "renesas,prr";
+		reg = <0 0xff000044 0 4>;
+	};
+
 	sysc: system-controller@e6180000 {
 		compatible = "renesas,r8a7791-sysc";
 		reg = <0 0xe6180000 0 0x0200>;
diff --git a/arch/arm/boot/dts/r8a7792-wheat.dts b/arch/arm/boot/dts/r8a7792-wheat.dts
index 6dbb94114a93..c24f26fdab1f 100644
--- a/arch/arm/boot/dts/r8a7792-wheat.dts
+++ b/arch/arm/boot/dts/r8a7792-wheat.dts
@@ -86,6 +86,34 @@
 		gpio = <&gpio11 12 GPIO_ACTIVE_HIGH>;
 		enable-active-high;
 	};
+
+	hdmi-out0 {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con0: endpoint {
+				remote-endpoint = <&adv7513_0_out>;
+			};
+		};
+	};
+
+	hdmi-out1 {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con1: endpoint {
+				remote-endpoint = <&adv7513_1_out>;
+			};
+		};
+	};
+
+	osc2_clk: osc2 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <74250000>;
+	};
 };
 
 &extal_clk {
@@ -128,6 +156,16 @@
 		groups = "qspi_ctrl", "qspi_data4";
 		function = "qspi";
 	};
+
+	du0_pins: du0 {
+		groups = "du0_rgb888", "du0_sync", "du0_disp";
+		function = "du0";
+	};
+
+	du1_pins: du1 {
+		groups = "du1_rgb666", "du1_sync", "du1_disp";
+		function = "du1";
+	};
 };
 
 &scif0 {
@@ -197,3 +235,91 @@
 		};
 	};
 };
+
+&i2c4 {
+	status = "okay";
+	clock-frequency = <400000>;
+
+	hdmi@3d {
+		compatible = "adi,adv7513";
+		reg = <0x3d>;
+
+		adi,input-depth = <8>;
+		adi,input-colorspace = "rgb";
+		adi,input-clock = "1x";
+		adi,input-style = <1>;
+		adi,input-justification = "evenly";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				adv7513_0_in: endpoint {
+					remote-endpoint = <&du_out_rgb0>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				adv7513_0_out: endpoint {
+					remote-endpoint = <&hdmi_con0>;
+				};
+			};
+		};
+	};
+
+	hdmi@39 {
+		compatible = "adi,adv7513";
+		reg = <0x39>;
+
+		adi,input-depth = <8>;
+		adi,input-colorspace = "rgb";
+		adi,input-clock = "1x";
+		adi,input-style = <1>;
+		adi,input-justification = "evenly";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				adv7513_1_in: endpoint {
+					remote-endpoint = <&du_out_rgb1>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				adv7513_1_out: endpoint {
+					remote-endpoint = <&hdmi_con1>;
+				};
+			};
+		};
+	};
+};
+
+&du {
+	pinctrl-0 = <&du0_pins &du1_pins>;
+	pinctrl-names = "default";
+
+	clocks = <&mstp7_clks R8A7792_CLK_DU0>, <&mstp7_clks R8A7792_CLK_DU1>,
+		 <&osc2_clk>;
+	clock-names = "du.0", "du.1", "dclkin.0";
+	status = "okay";
+
+	ports {
+		port@0 {
+			endpoint {
+				remote-endpoint = <&adv7513_0_in>;
+			};
+		};
+		port@1 {
+			endpoint {
+				remote-endpoint = <&adv7513_1_in>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/r8a7792.dtsi b/arch/arm/boot/dts/r8a7792.dtsi
index 713141d38b3e..6ced3c1ec377 100644
--- a/arch/arm/boot/dts/r8a7792.dtsi
+++ b/arch/arm/boot/dts/r8a7792.dtsi
@@ -26,6 +26,8 @@
 		i2c4 = &i2c4;
 		i2c5 = &i2c5;
 		spi0 = &qspi;
+		spi1 = &msiof0;
+		spi2 = &msiof1;
 		vin0 = &vin0;
 		vin1 = &vin1;
 		vin2 = &vin2;
@@ -118,6 +120,16 @@
 				      IRQ_TYPE_LEVEL_LOW)>;
 		};
 
+		rst: reset-controller@e6160000 {
+			compatible = "renesas,r8a7792-rst";
+			reg = <0 0xe6160000 0 0x0100>;
+		};
+
+		prr: chipid@ff000044 {
+			compatible = "renesas,prr";
+			reg = <0 0xff000044 0 4>;
+		};
+
 		sysc: system-controller@e6180000 {
 			compatible = "renesas,r8a7792-sysc";
 			reg = <0 0xe6180000 0 0x0200>;
@@ -572,6 +584,34 @@
 			status = "disabled";
 		};
 
+		msiof0: spi@e6e20000 {
+			compatible = "renesas,msiof-r8a7792";
+			reg = <0 0xe6e20000 0 0x0064>;
+			interrupts = <GIC_SPI 156 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp0_clks R8A7792_CLK_MSIOF0>;
+			dmas = <&dmac0 0x51>, <&dmac0 0x52>,
+			       <&dmac1 0x51>, <&dmac1 0x52>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7792_PD_ALWAYS_ON>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof1: spi@e6e10000 {
+			compatible = "renesas,msiof-r8a7792";
+			reg = <0 0xe6e10000 0 0x0064>;
+			interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&mstp2_clks R8A7792_CLK_MSIOF1>;
+			dmas = <&dmac0 0x55>, <&dmac0 0x56>,
+			       <&dmac1 0x55>, <&dmac1 0x56>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7792_PD_ALWAYS_ON>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		du: display@feb00000 {
 			compatible = "renesas,du-r8a7792";
 			reg = <0 0xfeb00000 0 0x40000>;
@@ -763,6 +803,13 @@
 			clock-div = <48>;
 			clock-mult = <1>;
 		};
+		mp_clk: mp {
+			compatible = "fixed-factor-clock";
+			clocks = <&pll1_div2_clk>;
+			#clock-cells = <0>;
+			clock-div = <15>;
+			clock-mult = <1>;
+		};
 		m2_clk: m2 {
 			compatible = "fixed-factor-clock";
 			clocks = <&cpg_clocks R8A7792_CLK_PLL1>;
@@ -793,6 +840,15 @@
 		};
 
 		/* Gate clocks */
+		mstp0_clks: mstp0_clks@e6150130 {
+			compatible = "renesas,r8a7792-mstp-clocks",
+				     "renesas,cpg-mstp-clocks";
+			reg = <0 0xe6150130 0 4>, <0 0xe6150030 0 4>;
+			clocks = <&mp_clk>;
+			#clock-cells = <1>;
+			clock-indices = <R8A7792_CLK_MSIOF0>;
+			clock-output-names = "msiof0";
+		};
 		mstp1_clks: mstp1_clks@e6150134 {
 			compatible = "renesas,r8a7792-mstp-clocks",
 				     "renesas,cpg-mstp-clocks";
@@ -811,12 +867,13 @@
 			compatible = "renesas,r8a7792-mstp-clocks",
 				     "renesas,cpg-mstp-clocks";
 			reg = <0 0xe6150138 0 4>, <0 0xe6150040 0 4>;
-			clocks = <&zs_clk>, <&zs_clk>;
+			clocks = <&mp_clk>, <&zs_clk>, <&zs_clk>;
 			#clock-cells = <1>;
 			clock-indices = <
+				R8A7792_CLK_MSIOF1
 				R8A7792_CLK_SYS_DMAC1 R8A7792_CLK_SYS_DMAC0
 			>;
-			clock-output-names = "sys-dmac1", "sys-dmac0";
+			clock-output-names = "msiof1", "sys-dmac1", "sys-dmac0";
 		};
 		mstp3_clks: mstp3_clks@e615013c {
 			compatible = "renesas,r8a7792-mstp-clocks",
diff --git a/arch/arm/boot/dts/r8a7793-gose.dts b/arch/arm/boot/dts/r8a7793-gose.dts
index 90af18600124..dc311eba4444 100644
--- a/arch/arm/boot/dts/r8a7793-gose.dts
+++ b/arch/arm/boot/dts/r8a7793-gose.dts
@@ -346,18 +346,18 @@
 	};
 
 	sdhi0_pins: sd0 {
-		renesas,groups = "sdhi0_data4", "sdhi0_ctrl";
-		renesas,function = "sdhi0";
+		groups = "sdhi0_data4", "sdhi0_ctrl";
+		function = "sdhi0";
 	};
 
 	sdhi1_pins: sd1 {
-		renesas,groups = "sdhi1_data4", "sdhi1_ctrl";
-		renesas,function = "sdhi1";
+		groups = "sdhi1_data4", "sdhi1_ctrl";
+		function = "sdhi1";
 	};
 
 	sdhi2_pins: sd2 {
-		renesas,groups = "sdhi2_data4", "sdhi2_ctrl";
-		renesas,function = "sdhi2";
+		groups = "sdhi2_data4", "sdhi2_ctrl";
+		function = "sdhi2";
 	};
 
 	qspi_pins: qspi {
diff --git a/arch/arm/boot/dts/r8a7793.dtsi b/arch/arm/boot/dts/r8a7793.dtsi
index 8d02aacf2892..2fb527ca0b15 100644
--- a/arch/arm/boot/dts/r8a7793.dtsi
+++ b/arch/arm/boot/dts/r8a7793.dtsi
@@ -666,7 +666,7 @@
 	scifb0: serial@e6c20000 {
 		compatible = "renesas,scifb-r8a7793",
 			     "renesas,rcar-gen2-scifb", "renesas,scifb";
-		reg = <0 0xe6c20000 0 64>;
+		reg = <0 0xe6c20000 0 0x100>;
 		interrupts = <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp2_clks R8A7793_CLK_SCIFB0>;
 		clock-names = "fck";
@@ -680,7 +680,7 @@
 	scifb1: serial@e6c30000 {
 		compatible = "renesas,scifb-r8a7793",
 			     "renesas,rcar-gen2-scifb", "renesas,scifb";
-		reg = <0 0xe6c30000 0 64>;
+		reg = <0 0xe6c30000 0 0x100>;
 		interrupts = <GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp2_clks R8A7793_CLK_SCIFB1>;
 		clock-names = "fck";
@@ -694,7 +694,7 @@
 	scifb2: serial@e6ce0000 {
 		compatible = "renesas,scifb-r8a7793",
 			     "renesas,rcar-gen2-scifb", "renesas,scifb";
-		reg = <0 0xe6ce0000 0 64>;
+		reg = <0 0xe6ce0000 0 0x100>;
 		interrupts = <GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp2_clks R8A7793_CLK_SCIFB2>;
 		clock-names = "fck";
@@ -852,6 +852,33 @@
 		status = "disabled";
 	};
 
+	vin0: video@e6ef0000 {
+		compatible = "renesas,vin-r8a7793", "renesas,rcar-gen2-vin";
+		reg = <0 0xe6ef0000 0 0x1000>;
+		interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&mstp8_clks R8A7793_CLK_VIN0>;
+		power-domains = <&sysc R8A7793_PD_ALWAYS_ON>;
+		status = "disabled";
+	};
+
+	vin1: video@e6ef1000 {
+		compatible = "renesas,vin-r8a7793", "renesas,rcar-gen2-vin";
+		reg = <0 0xe6ef1000 0 0x1000>;
+		interrupts = <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&mstp8_clks R8A7793_CLK_VIN1>;
+		power-domains = <&sysc R8A7793_PD_ALWAYS_ON>;
+		status = "disabled";
+	};
+
+	vin2: video@e6ef2000 {
+		compatible = "renesas,vin-r8a7793", "renesas,rcar-gen2-vin";
+		reg = <0 0xe6ef2000 0 0x1000>;
+		interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&mstp8_clks R8A7793_CLK_VIN2>;
+		power-domains = <&sysc R8A7793_PD_ALWAYS_ON>;
+		status = "disabled";
+	};
+
 	qspi: spi@e6b10000 {
 		compatible = "renesas,qspi-r8a7793", "renesas,qspi";
 		reg = <0 0xe6b10000 0 0x2c>;
@@ -1279,6 +1306,16 @@
 		};
 	};
 
+	rst: reset-controller@e6160000 {
+		compatible = "renesas,r8a7793-rst";
+		reg = <0 0xe6160000 0 0x0100>;
+	};
+
+	prr: chipid@ff000044 {
+		compatible = "renesas,prr";
+		reg = <0 0xff000044 0 4>;
+	};
+
 	sysc: system-controller@e6180000 {
 		compatible = "renesas,r8a7793-sysc";
 		reg = <0 0xe6180000 0 0x0200>;
diff --git a/arch/arm/boot/dts/r8a7794-alt.dts b/arch/arm/boot/dts/r8a7794-alt.dts
index 8d1b35afaf82..569e3f0e97a5 100644
--- a/arch/arm/boot/dts/r8a7794-alt.dts
+++ b/arch/arm/boot/dts/r8a7794-alt.dts
@@ -18,6 +18,8 @@
 
 	aliases {
 		serial0 = &scif2;
+		i2c10 = &gpioi2c4;
+		i2c12 = &i2cexio4;
 	};
 
 	chosen {
@@ -135,6 +137,29 @@
 		#clock-cells = <0>;
 		clock-frequency = <148500000>;
 	};
+
+	gpioi2c4: i2c-10 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "i2c-gpio";
+		status = "disabled";
+		gpios = <&gpio4 9 GPIO_ACTIVE_HIGH /* sda */
+			 &gpio4 8 GPIO_ACTIVE_HIGH /* scl */
+			>;
+		i2c-gpio,delay-us = <5>;
+	};
+
+	/*
+	 * I2C4 is routed to EXIO connector B, pins 73 (SCL) + 74 (SDA).
+	 * A fallback to GPIO is provided.
+	 */
+	i2cexio4: i2c-14 {
+		compatible = "i2c-demux-pinctrl";
+		i2c-parent = <&i2c4>, <&gpioi2c4>;
+		i2c-bus-name = "i2c-exio4";
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
 };
 
 &du {
@@ -165,8 +190,8 @@
 	pinctrl-names = "default";
 
 	du_pins: du {
-		groups = "du1_rgb666", "du1_sync", "du1_disp", "du1_dotclkout0";
-		function = "du";
+		groups = "du1_rgb666", "du1_sync", "du1_disp", "du1_clk0_out";
+		function = "du1";
 	};
 
 	scif2_pins: scif2 {
@@ -194,6 +219,11 @@
 		function = "i2c1";
 	};
 
+	i2c4_pins: i2c4 {
+		groups = "i2c4";
+		function = "i2c4";
+	};
+
 	vin0_pins: vin0 {
 		groups = "vin0_data8", "vin0_clk";
 		function = "vin0";
@@ -207,11 +237,25 @@
 	sdhi0_pins: sd0 {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
+		power-source = <3300>;
+	};
+
+	sdhi0_pins_uhs: sd0_uhs {
+		groups = "sdhi0_data4", "sdhi0_ctrl";
+		function = "sdhi0";
+		power-source = <1800>;
 	};
 
 	sdhi1_pins: sd1 {
 		groups = "sdhi1_data4", "sdhi1_ctrl";
 		function = "sdhi1";
+		power-source = <3300>;
+	};
+
+	sdhi1_pins_uhs: sd1_uhs {
+		groups = "sdhi1_data4", "sdhi1_ctrl";
+		function = "sdhi1";
+		power-source = <1800>;
 	};
 };
 
@@ -255,23 +299,28 @@
 
 &sdhi0 {
 	pinctrl-0 = <&sdhi0_pins>;
-	pinctrl-names = "default";
+	pinctrl-1 = <&sdhi0_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
 
 	vmmc-supply = <&vcc_sdhi0>;
 	vqmmc-supply = <&vccq_sdhi0>;
 	cd-gpios = <&gpio6 6 GPIO_ACTIVE_LOW>;
 	wp-gpios = <&gpio6 7 GPIO_ACTIVE_LOW>;
+	sd-uhs-sdr50;
+	sd-uhs-sdr104;
 	status = "okay";
 };
 
 &sdhi1 {
 	pinctrl-0 = <&sdhi1_pins>;
-	pinctrl-names = "default";
+	pinctrl-1 = <&sdhi1_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
 
 	vmmc-supply = <&vcc_sdhi1>;
 	vqmmc-supply = <&vccq_sdhi1>;
 	cd-gpios = <&gpio6 14 GPIO_ACTIVE_LOW>;
 	wp-gpios = <&gpio6 15 GPIO_ACTIVE_LOW>;
+	sd-uhs-sdr50;
 	status = "okay";
 };
 
@@ -296,6 +345,11 @@
 	};
 };
 
+&i2c4 {
+	pinctrl-0 = <&i2c4_pins>;
+	pinctrl-names = "i2c-exio4";
+};
+
 &vin0 {
 	status = "okay";
 	pinctrl-0 = <&vin0_pins>;
diff --git a/arch/arm/boot/dts/r8a7794.dtsi b/arch/arm/boot/dts/r8a7794.dtsi
index 9365580a194f..fb576dba748c 100644
--- a/arch/arm/boot/dts/r8a7794.dtsi
+++ b/arch/arm/boot/dts/r8a7794.dtsi
@@ -319,7 +319,7 @@
 				  "ch12";
 		clocks = <&mstp5_clks R8A7794_CLK_AUDIO_DMAC0>;
 		clock-names = "fck";
-		power-domains = <&cpg_clocks>;
+		power-domains = <&sysc R8A7794_PD_ALWAYS_ON>;
 		#dma-cells = <1>;
 		dma-channels = <13>;
 	};
@@ -411,7 +411,7 @@
 	scifb0: serial@e6c20000 {
 		compatible = "renesas,scifb-r8a7794",
 			     "renesas,rcar-gen2-scifb", "renesas,scifb";
-		reg = <0 0xe6c20000 0 64>;
+		reg = <0 0xe6c20000 0 0x100>;
 		interrupts = <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp2_clks R8A7794_CLK_SCIFB0>;
 		clock-names = "fck";
@@ -425,7 +425,7 @@
 	scifb1: serial@e6c30000 {
 		compatible = "renesas,scifb-r8a7794",
 			     "renesas,rcar-gen2-scifb", "renesas,scifb";
-		reg = <0 0xe6c30000 0 64>;
+		reg = <0 0xe6c30000 0 0x100>;
 		interrupts = <GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp2_clks R8A7794_CLK_SCIFB1>;
 		clock-names = "fck";
@@ -439,7 +439,7 @@
 	scifb2: serial@e6ce0000 {
 		compatible = "renesas,scifb-r8a7794",
 			     "renesas,rcar-gen2-scifb", "renesas,scifb";
-		reg = <0 0xe6ce0000 0 64>;
+		reg = <0 0xe6ce0000 0 0x100>;
 		interrupts = <GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp2_clks R8A7794_CLK_SCIFB2>;
 		clock-names = "fck";
@@ -731,6 +731,7 @@
 		dmas = <&dmac0 0xcd>, <&dmac0 0xce>,
 		       <&dmac1 0xcd>, <&dmac1 0xce>;
 		dma-names = "tx", "rx", "tx", "rx";
+		max-frequency = <195000000>;
 		power-domains = <&sysc R8A7794_PD_ALWAYS_ON>;
 		status = "disabled";
 	};
@@ -743,6 +744,7 @@
 		dmas = <&dmac0 0xc1>, <&dmac0 0xc2>,
 		       <&dmac1 0xc1>, <&dmac1 0xc2>;
 		dma-names = "tx", "rx", "tx", "rx";
+		max-frequency = <97500000>;
 		power-domains = <&sysc R8A7794_PD_ALWAYS_ON>;
 		status = "disabled";
 	};
@@ -755,6 +757,7 @@
 		dmas = <&dmac0 0xd3>, <&dmac0 0xd4>,
 		       <&dmac1 0xd3>, <&dmac1 0xd4>;
 		dma-names = "tx", "rx", "tx", "rx";
+		max-frequency = <97500000>;
 		power-domains = <&sysc R8A7794_PD_ALWAYS_ON>;
 		status = "disabled";
 	};
@@ -1025,8 +1028,7 @@
 			clocks = <&extal_clk &usb_extal_clk>;
 			#clock-cells = <1>;
 			clock-output-names = "main", "pll0", "pll1", "pll3",
-					     "lb", "qspi", "sdh", "sd0", "z",
-					     "rcan";
+					     "lb", "qspi", "sdh", "sd0", "rcan";
 			#power-domain-cells = <0>;
 		};
 		/* Variable factor clocks */
@@ -1260,7 +1262,7 @@
 		mstp7_clks: mstp7_clks@e615014c {
 			compatible = "renesas,r8a7794-mstp-clocks", "renesas,cpg-mstp-clocks";
 			reg = <0 0xe615014c 0 4>, <0 0xe61501c4 0 4>;
-			clocks = <&mp_clk>, <&mp_clk>,
+			clocks = <&mp_clk>, <&hp_clk>,
 				 <&zs_clk>, <&p_clk>, <&p_clk>, <&zs_clk>,
 				 <&zs_clk>, <&p_clk>, <&p_clk>, <&p_clk>, <&p_clk>,
 				 <&zx_clk>;
@@ -1375,6 +1377,16 @@
 		};
 	};
 
+	rst: reset-controller@e6160000 {
+		compatible = "renesas,r8a7794-rst";
+		reg = <0 0xe6160000 0 0x0100>;
+	};
+
+	prr: chipid@ff000044 {
+		compatible = "renesas,prr";
+		reg = <0 0xff000044 0 4>;
+	};
+
 	sysc: system-controller@e6180000 {
 		compatible = "renesas,r8a7794-sysc";
 		reg = <0 0xe6180000 0 0x0200>;
@@ -1483,67 +1495,67 @@
 			      "mix.0", "mix.1",
 			      "dvc.0", "dvc.1",
 			      "clk_a", "clk_b", "clk_c", "clk_i";
-		power-domains = <&cpg_clocks>;
+		power-domains = <&sysc R8A7794_PD_ALWAYS_ON>;
 
 		status = "disabled";
 
 		rcar_sound,dvc {
-			dvc0: dvc@0 {
+			dvc0: dvc-0 {
 				dmas = <&audma0 0xbc>;
 				dma-names = "tx";
 			};
-			dvc1: dvc@1 {
+			dvc1: dvc-1 {
 				dmas = <&audma0 0xbe>;
 				dma-names = "tx";
 			};
 		};
 
 		rcar_sound,mix {
-			mix0: mix@0 { };
-			mix1: mix@1 { };
+			mix0: mix-0 { };
+			mix1: mix-1 { };
 		};
 
 		rcar_sound,ctu {
-			ctu00: ctu@0 { };
-			ctu01: ctu@1 { };
-			ctu02: ctu@2 { };
-			ctu03: ctu@3 { };
-			ctu10: ctu@4 { };
-			ctu11: ctu@5 { };
-			ctu12: ctu@6 { };
-			ctu13: ctu@7 { };
+			ctu00: ctu-0 { };
+			ctu01: ctu-1 { };
+			ctu02: ctu-2 { };
+			ctu03: ctu-3 { };
+			ctu10: ctu-4 { };
+			ctu11: ctu-5 { };
+			ctu12: ctu-6 { };
+			ctu13: ctu-7 { };
 		};
 
 		rcar_sound,src {
-			src@0 {
+			src-0 {
 				status = "disabled";
 			};
-			src1: src@1 {
+			src1: src-1 {
 				interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x87>, <&audma0 0x9c>;
 				dma-names = "rx", "tx";
 			};
-			src2: src@2 {
+			src2: src-2 {
 				interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x89>, <&audma0 0x9e>;
 				dma-names = "rx", "tx";
 			};
-			src3: src@3 {
+			src3: src-3 {
 				interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x8b>, <&audma0 0xa0>;
 				dma-names = "rx", "tx";
 			};
-			src4: src@4 {
+			src4: src-4 {
 				interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x8d>, <&audma0 0xb0>;
 				dma-names = "rx", "tx";
 			};
-			src5: src@5 {
+			src5: src-5 {
 				interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x8f>, <&audma0 0xb2>;
 				dma-names = "rx", "tx";
 			};
-			src6: src@6 {
+			src6: src-6 {
 				interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x91>, <&audma0 0xb4>;
 				dma-names = "rx", "tx";
@@ -1551,61 +1563,61 @@
 		};
 
 		rcar_sound,ssi {
-			ssi0: ssi@0 {
+			ssi0: ssi-0 {
 				interrupts = <GIC_SPI 370 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x01>, <&audma0 0x02>,
 				       <&audma0 0x15>, <&audma0 0x16>;
 				dma-names = "rx", "tx", "rxu", "txu";
 			};
-			ssi1: ssi@1 {
+			ssi1: ssi-1 {
 				interrupts = <GIC_SPI 371 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x03>, <&audma0 0x04>,
 				       <&audma0 0x49>, <&audma0 0x4a>;
 				dma-names = "rx", "tx", "rxu", "txu";
 			};
-			ssi2: ssi@2 {
+			ssi2: ssi-2 {
 				interrupts = <GIC_SPI 372 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x05>, <&audma0 0x06>,
 				       <&audma0 0x63>, <&audma0 0x64>;
 				dma-names = "rx", "tx", "rxu", "txu";
 			};
-			ssi3: ssi@3 {
+			ssi3: ssi-3 {
 				interrupts = <GIC_SPI 373 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x07>, <&audma0 0x08>,
 				       <&audma0 0x6f>, <&audma0 0x70>;
 				dma-names = "rx", "tx", "rxu", "txu";
 			};
-			ssi4: ssi@4 {
+			ssi4: ssi-4 {
 				interrupts = <GIC_SPI 374 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x09>, <&audma0 0x0a>,
 				       <&audma0 0x71>, <&audma0 0x72>;
 				dma-names = "rx", "tx", "rxu", "txu";
 			};
-			ssi5: ssi@5 {
+			ssi5: ssi-5 {
 				interrupts = <GIC_SPI 375 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x0b>, <&audma0 0x0c>,
 				       <&audma0 0x73>, <&audma0 0x74>;
 				dma-names = "rx", "tx", "rxu", "txu";
 			};
-			ssi6: ssi@6 {
+			ssi6: ssi-6 {
 				interrupts = <GIC_SPI 376 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x0d>, <&audma0 0x0e>,
 				       <&audma0 0x75>, <&audma0 0x76>;
 				dma-names = "rx", "tx", "rxu", "txu";
 			};
-			ssi7: ssi@7 {
+			ssi7: ssi-7 {
 				interrupts = <GIC_SPI 377 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x0f>, <&audma0 0x10>,
 				       <&audma0 0x79>, <&audma0 0x7a>;
 				dma-names = "rx", "tx", "rxu", "txu";
 			};
-			ssi8: ssi@8 {
+			ssi8: ssi-8 {
 				interrupts = <GIC_SPI 378 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x11>, <&audma0 0x12>,
 				       <&audma0 0x7b>, <&audma0 0x7c>;
 				dma-names = "rx", "tx", "rxu", "txu";
 			};
-			ssi9: ssi@9 {
+			ssi9: ssi-9 {
 				interrupts = <GIC_SPI 379 IRQ_TYPE_LEVEL_HIGH>;
 				dmas = <&audma0 0x13>, <&audma0 0x14>,
 				       <&audma0 0x7d>, <&audma0 0x7e>;
diff --git a/arch/arm/boot/dts/rk1108-evb.dts b/arch/arm/boot/dts/rk1108-evb.dts
new file mode 100644
index 000000000000..3956cff4ca79
--- /dev/null
+++ b/arch/arm/boot/dts/rk1108-evb.dts
@@ -0,0 +1,69 @@
+/*
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ *  Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "rk1108.dtsi"
+
+/ {
+	model = "Rockchip RK1108 Evaluation board";
+	compatible = "rockchip,rk1108-evb", "rockchip,rk1108";
+
+	memory@60000000 {
+		device_type = "memory";
+		reg = <0x60000000 0x08000000>;
+	};
+
+	chosen {
+		stdout-path = "serial2:1500000n8";
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&uart2 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/rk1108.dtsi b/arch/arm/boot/dts/rk1108.dtsi
new file mode 100644
index 000000000000..d7700235e0f5
--- /dev/null
+++ b/arch/arm/boot/dts/rk1108.dtsi
@@ -0,0 +1,452 @@
+/*
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/rk1108-cru.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	compatible = "rockchip,rk1108";
+
+	interrupt-parent = <&gic>;
+
+	aliases {
+		serial0 = &uart0;
+		serial1 = &uart1;
+		serial2 = &uart2;
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@f00 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a7";
+			reg = <0xf00>;
+		};
+	};
+
+	arm-pmu {
+		compatible = "arm,cortex-a7-pmu";
+		interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	timer {
+		compatible = "arm,armv7-timer";
+		interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_HIGH)>,
+			     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_HIGH)>;
+		clock-frequency = <24000000>;
+	};
+
+	xin24m: oscillator {
+		compatible = "fixed-clock";
+		clock-frequency = <24000000>;
+		clock-output-names = "xin24m";
+		#clock-cells = <0>;
+	};
+
+	amba {
+		compatible = "simple-bus";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		pdma: pdma@102a0000 {
+			compatible = "arm,pl330", "arm,primecell";
+			reg = <0x102a0000 0x4000>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells = <1>;
+			arm,pl330-broken-no-flushp;
+			clocks = <&cru ACLK_DMAC>;
+			clock-names = "apb_pclk";
+		};
+	};
+
+	bus_intmem@10080000 {
+		compatible = "mmio-sram";
+		reg = <0x10080000 0x2000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x10080000 0x2000>;
+	};
+
+	uart2: serial@10210000 {
+		compatible = "rockchip,rk1108-uart", "snps,dw-apb-uart";
+		reg = <0x10210000 0x100>;
+		interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		clock-frequency = <24000000>;
+		clocks = <&cru SCLK_UART2>, <&cru PCLK_UART2>;
+		clock-names = "baudclk", "apb_pclk";
+		pinctrl-names = "default";
+		pinctrl-0 = <&uart2m0_xfer>;
+		status = "disabled";
+	};
+
+	uart1: serial@10220000 {
+		compatible = "rockchip,rk1108-uart", "snps,dw-apb-uart";
+		reg = <0x10220000 0x100>;
+		interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		clock-frequency = <24000000>;
+		clocks = <&cru SCLK_UART1>, <&cru PCLK_UART1>;
+		clock-names = "baudclk", "apb_pclk";
+		pinctrl-names = "default";
+		pinctrl-0 = <&uart1_xfer>;
+		status = "disabled";
+	};
+
+	uart0: serial@10230000 {
+		compatible = "rockchip,rk1108-uart", "snps,dw-apb-uart";
+		reg = <0x10230000 0x100>;
+		interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		clock-frequency = <24000000>;
+		clocks = <&cru SCLK_UART0>, <&cru PCLK_UART0>;
+		clock-names = "baudclk", "apb_pclk";
+		pinctrl-names = "default";
+		pinctrl-0 = <&uart0_xfer &uart0_cts &uart0_rts>;
+		status = "disabled";
+	};
+
+	grf: syscon@10300000 {
+		compatible = "rockchip,rk1108-grf", "syscon";
+		reg = <0x10300000 0x1000>;
+	};
+
+	pmugrf: syscon@20060000 {
+		compatible = "rockchip,rk1108-pmugrf", "syscon";
+		reg = <0x20060000 0x1000>;
+	};
+
+	cru: clock-controller@20200000 {
+		compatible = "rockchip,rk1108-cru";
+		reg = <0x20200000 0x1000>;
+		rockchip,grf = <&grf>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+	};
+
+	emmc: dwmmc@30110000 {
+		compatible = "rockchip,rk1108-dw-mshc", "rockchip,rk3288-dw-mshc";
+		clock-freq-min-max = <400000 150000000>;
+		clocks = <&cru HCLK_EMMC>, <&cru SCLK_EMMC>,
+			 <&cru SCLK_EMMC_DRV>, <&cru SCLK_EMMC_SAMPLE>;
+		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
+		fifo-depth = <0x100>;
+		interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+		reg = <0x30110000 0x4000>;
+		status = "disabled";
+	};
+
+	sdio: dwmmc@30120000 {
+		compatible = "rockchip,rk1108-dw-mshc", "rockchip,rk3288-dw-mshc";
+		clock-freq-min-max = <400000 150000000>;
+		clocks = <&cru HCLK_SDIO>, <&cru SCLK_SDIO>,
+			 <&cru SCLK_SDIO_DRV>, <&cru SCLK_SDIO_SAMPLE>;
+		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
+		fifo-depth = <0x100>;
+		interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+		reg = <0x30120000 0x4000>;
+		status = "disabled";
+	};
+
+	sdmmc: dwmmc@30130000 {
+		compatible = "rockchip,rk1108-dw-mshc", "rockchip,rk3288-dw-mshc";
+		clock-freq-min-max = <400000 100000000>;
+		clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>,
+			 <&cru SCLK_SDMMC_DRV>, <&cru SCLK_SDMMC_SAMPLE>;
+		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
+		fifo-depth = <0x100>;
+		interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+		reg = <0x30130000 0x4000>;
+		status = "disabled";
+	};
+
+	gic: interrupt-controller@32010000 {
+		compatible = "arm,gic-400";
+		interrupt-controller;
+		#interrupt-cells = <3>;
+		#address-cells = <0>;
+
+		reg = <0x32011000 0x1000>,
+		      <0x32012000 0x1000>,
+		      <0x32014000 0x2000>,
+		      <0x32016000 0x2000>;
+		interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(1) | IRQ_TYPE_LEVEL_HIGH)>;
+	};
+
+	pinctrl: pinctrl {
+		compatible = "rockchip,rk1108-pinctrl";
+		rockchip,grf = <&grf>;
+		rockchip,pmu = <&pmugrf>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		gpio0: gpio0@20030000 {
+			compatible = "rockchip,gpio-bank";
+			reg = <0x20030000 0x100>;
+			interrupts = <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&xin24m>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio1: gpio1@10310000 {
+			compatible = "rockchip,gpio-bank";
+			reg = <0x10310000 0x100>;
+			interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&xin24m>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio2: gpio2@10320000 {
+			compatible = "rockchip,gpio-bank";
+			reg = <0x10320000 0x100>;
+			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&xin24m>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio3: gpio3@10330000 {
+			compatible = "rockchip,gpio-bank";
+			reg = <0x10330000 0x100>;
+			interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&xin24m>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		pcfg_pull_up: pcfg-pull-up {
+			bias-pull-up;
+		};
+
+		pcfg_pull_down: pcfg-pull-down {
+			bias-pull-down;
+		};
+
+		pcfg_pull_none: pcfg-pull-none {
+			bias-disable;
+		};
+
+		pcfg_pull_none_drv_8ma: pcfg-pull-none-drv-8ma {
+			drive-strength = <8>;
+		};
+
+		pcfg_pull_none_drv_12ma: pcfg-pull-none-drv-12ma {
+			drive-strength = <12>;
+		};
+
+		pcfg_pull_up_drv_8ma: pcfg-pull-up-drv-8ma {
+			bias-pull-up;
+			drive-strength = <8>;
+		};
+
+		pcfg_pull_none_drv_4ma: pcfg-pull-none-drv-4ma {
+			drive-strength = <4>;
+		};
+
+		pcfg_pull_up_drv_4ma: pcfg-pull-up-drv-4ma {
+			bias-pull-up;
+			drive-strength = <4>;
+		};
+
+		pcfg_output_high: pcfg-output-high {
+			output-high;
+		};
+
+		pcfg_output_low: pcfg-output-low {
+			output-low;
+		};
+
+		pcfg_input_high: pcfg-input-high {
+			bias-pull-up;
+			input-enable;
+		};
+
+		i2c1 {
+			i2c1_xfer: i2c1-xfer {
+				rockchip,pins = <2 RK_PD3 RK_FUNC_1 &pcfg_pull_up>,
+						<2 RK_PD4 RK_FUNC_1 &pcfg_pull_up>;
+			};
+		};
+
+		i2c2m1 {
+			i2c2m1_xfer: i2c2m1-xfer {
+				rockchip,pins = <0 RK_PC2 RK_FUNC_2 &pcfg_pull_none>,
+						<0 RK_PC6 RK_FUNC_3 &pcfg_pull_none>;
+			};
+
+			i2c2m1_gpio: i2c2m1-gpio {
+				rockchip,pins = <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>,
+						<0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>;
+			};
+		};
+
+		i2c2m05v {
+			i2c2m05v_xfer: i2c2m05v-xfer {
+				rockchip,pins = <1 RK_PD5 RK_FUNC_2 &pcfg_pull_none>,
+						<1 RK_PD4 RK_FUNC_2 &pcfg_pull_none>;
+			};
+
+			i2c2m05v_gpio: i2c2m05v-gpio {
+				rockchip,pins = <1 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>,
+						<1 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>;
+			};
+		};
+
+		i2c3 {
+			i2c3_xfer: i2c3-xfer {
+				rockchip,pins = <0 RK_PB6 RK_FUNC_1 &pcfg_pull_none>,
+						<0 RK_PC4 RK_FUNC_2 &pcfg_pull_none>;
+			};
+		};
+
+		sdmmc {
+			sdmmc_clk: sdmmc-clk {
+				rockchip,pins = <3 RK_PC4 RK_FUNC_1 &pcfg_pull_none_drv_4ma>;
+			};
+
+			sdmmc_cmd: sdmmc-cmd {
+				rockchip,pins = <3 RK_PC5 RK_FUNC_1 &pcfg_pull_up_drv_4ma>;
+			};
+
+			sdmmc_cd: sdmmc-cd {
+				rockchip,pins = <0 RK_PA1 RK_FUNC_1 &pcfg_pull_up_drv_4ma>;
+			};
+
+			sdmmc_bus1: sdmmc-bus1 {
+				rockchip,pins = <3 RK_PC3 RK_FUNC_1 &pcfg_pull_up_drv_4ma>;
+			};
+
+			sdmmc_bus4: sdmmc-bus4 {
+				rockchip,pins = <3 RK_PC3 RK_FUNC_1 &pcfg_pull_up_drv_4ma>,
+						<3 RK_PC2 RK_FUNC_1 &pcfg_pull_up_drv_4ma>,
+						<3 RK_PC1 RK_FUNC_1 &pcfg_pull_up_drv_4ma>,
+						<3 RK_PC0 RK_FUNC_1 &pcfg_pull_up_drv_4ma>;
+			};
+		};
+
+		uart0 {
+			uart0_xfer: uart0-xfer {
+				rockchip,pins = <3 RK_PA6 RK_FUNC_1 &pcfg_pull_up>,
+						<3 RK_PA5 RK_FUNC_1 &pcfg_pull_none>;
+			};
+
+			uart0_cts: uart0-cts {
+				rockchip,pins = <3 RK_PA4 RK_FUNC_1 &pcfg_pull_none>;
+			};
+
+			uart0_rts: uart0-rts {
+				rockchip,pins = <3 RK_PA3 RK_FUNC_1 &pcfg_pull_none>;
+			};
+
+			uart0_rts_gpio: uart0-rts-gpio {
+				rockchip,pins = <3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>;
+			};
+		};
+
+		uart1 {
+			uart1_xfer: uart1-xfer {
+				rockchip,pins = <1 RK_PD3 RK_FUNC_1 &pcfg_pull_up>,
+						<1 RK_PD2 RK_FUNC_1 &pcfg_pull_none>;
+			};
+
+			uart1_cts: uart1-cts {
+				rockchip,pins = <1 RK_PD0 RK_FUNC_1 &pcfg_pull_none>;
+			};
+
+			uart1_rts: uart1-rts {
+				rockchip,pins = <1 RK_PD1 RK_FUNC_1 &pcfg_pull_none>;
+			};
+		};
+
+		uart2m0 {
+			uart2m0_xfer: uart2m0-xfer {
+				rockchip,pins = <2 RK_PD2 RK_FUNC_1 &pcfg_pull_up>,
+						<2 RK_PD1 RK_FUNC_1 &pcfg_pull_none>;
+			};
+		};
+
+		uart2m1 {
+			uart2m1_xfer: uart2m1-xfer {
+				rockchip,pins = <3 RK_PC3 RK_FUNC_2 &pcfg_pull_up>,
+						<3 RK_PC2 RK_FUNC_2 &pcfg_pull_none>;
+			};
+		};
+
+		uart2_5v {
+			uart2_5v_cts: uart2_5v-cts {
+				rockchip,pins = <1 RK_PD4 RK_FUNC_1 &pcfg_pull_none>;
+			};
+
+			uart2_5v_rts: uart2_5v-rts {
+				rockchip,pins = <1 RK_PD5 RK_FUNC_1 &pcfg_pull_none>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/rk3036-evb.dts b/arch/arm/boot/dts/rk3036-evb.dts
index 8db9e9b197a2..2f5f15524fba 100644
--- a/arch/arm/boot/dts/rk3036-evb.dts
+++ b/arch/arm/boot/dts/rk3036-evb.dts
@@ -46,7 +46,7 @@
 	model = "Rockchip RK3036 Evaluation board";
 	compatible = "rockchip,rk3036-evb", "rockchip,rk3036";
 
-	memory {
+	memory@60000000 {
 		device_type = "memory";
 		reg = <0x60000000 0x40000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3036-kylin.dts b/arch/arm/boot/dts/rk3036-kylin.dts
index 1df1557a46c3..3de958ec29c0 100644
--- a/arch/arm/boot/dts/rk3036-kylin.dts
+++ b/arch/arm/boot/dts/rk3036-kylin.dts
@@ -46,7 +46,7 @@
 	model = "Rockchip RK3036 KylinBoard";
 	compatible = "rockchip,rk3036-kylin", "rockchip,rk3036";
 
-	memory {
+	memory@60000000 {
 		device_type = "memory";
 		reg = <0x60000000 0x20000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3036.dtsi b/arch/arm/boot/dts/rk3036.dtsi
index a935523a1eb8..4ed49a243e5c 100644
--- a/arch/arm/boot/dts/rk3036.dtsi
+++ b/arch/arm/boot/dts/rk3036.dtsi
@@ -44,9 +44,11 @@
 #include <dt-bindings/pinctrl/rockchip.h>
 #include <dt-bindings/clock/rk3036-cru.h>
 #include <dt-bindings/soc/rockchip,boot-mode.h>
-#include "skeleton.dtsi"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	compatible = "rockchip,rk3036";
 
 	interrupt-parent = <&gic>;
@@ -204,7 +206,6 @@
 		g-np-tx-fifo-size = <16>;
 		g-rx-fifo-size = <275>;
 		g-tx-fifo-size = <256 128 128 64 64 32>;
-		g-use-dma;
 		status = "disabled";
 	};
 
@@ -244,7 +245,7 @@
 		compatible = "rockchip,rk3036-dw-mshc", "rockchip,rk3288-dw-mshc";
 		reg = <0x10214000 0x4000>;
 		clock-frequency = <37500000>;
-		clock-freq-min-max = <400000 37500000>;
+		max-frequency = <37500000>;
 		clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>;
 		clock-names = "biu", "ciu";
 		fifo-depth = <0x100>;
@@ -255,7 +256,7 @@
 	sdio: dwmmc@10218000 {
 		compatible = "rockchip,rk3036-dw-mshc", "rockchip,rk3288-dw-mshc";
 		reg = <0x10218000 0x4000>;
-		clock-freq-min-max = <400000 37500000>;
+		max-frequency = <37500000>;
 		clocks = <&cru HCLK_SDIO>, <&cru SCLK_SDIO>,
 			 <&cru SCLK_SDIO_DRV>, <&cru SCLK_SDIO_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
@@ -271,7 +272,7 @@
 		bus-width = <8>;
 		cap-mmc-highspeed;
 		clock-frequency = <37500000>;
-		clock-freq-min-max = <400000 37500000>;
+		max-frequency = <37500000>;
 		clocks = <&cru HCLK_EMMC>, <&cru SCLK_EMMC>,
 			 <&cru SCLK_EMMC_DRV>, <&cru SCLK_EMMC_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
diff --git a/arch/arm/boot/dts/rk3066a-bqcurie2.dts b/arch/arm/boot/dts/rk3066a-bqcurie2.dts
index bc674ee206ec..c0d8b5446ba7 100644
--- a/arch/arm/boot/dts/rk3066a-bqcurie2.dts
+++ b/arch/arm/boot/dts/rk3066a-bqcurie2.dts
@@ -49,7 +49,7 @@
 	model = "bq Curie 2";
 	compatible = "mundoreader,bq-curie2", "rockchip,rk3066a";
 
-	memory {
+	memory@60000000 {
 		device_type = "memory";
 		reg = <0x60000000 0x40000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3066a-marsboard.dts b/arch/arm/boot/dts/rk3066a-marsboard.dts
index a2b763e949b4..0a54c4beff8d 100644
--- a/arch/arm/boot/dts/rk3066a-marsboard.dts
+++ b/arch/arm/boot/dts/rk3066a-marsboard.dts
@@ -47,7 +47,7 @@
 	model = "MarsBoard RK3066";
 	compatible = "haoyu,marsboard-rk3066", "rockchip,rk3066a";
 
-	memory {
+	memory@60000000 {
 		device_type = "memory";
 		reg = <0x60000000 0x40000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3066a-mk808.dts b/arch/arm/boot/dts/rk3066a-mk808.dts
new file mode 100644
index 000000000000..658eb7ddeaf5
--- /dev/null
+++ b/arch/arm/boot/dts/rk3066a-mk808.dts
@@ -0,0 +1,195 @@
+/*
+ * Copyright (c) 2016 Paweł Jarosz <paweljarosz3691@gmail.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+#include "rk3066a.dtsi"
+
+/ {
+	model = "Rikomagic MK808";
+	compatible = "rikomagic,mk808", "rockchip,rk3066a";
+
+	chosen {
+		stdout-path = "serial2:115200n8";
+	};
+
+	memory@60000000 {
+		reg = <0x60000000 0x40000000>;
+		device_type = "memory";
+	};
+
+	gpio-leds {
+		compatible = "gpio-leds";
+
+		blue {
+			label = "mk808:blue:power";
+			gpios = <&gpio0 3 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "default-on";
+		};
+	};
+
+	vcc_io: vcc-io {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_io";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	vcc_host: usb-host-regulator {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio0 6 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&host_drv>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-name = "host-pwr";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		startup-delay-us = <100000>;
+		vin-supply = <&vcc_io>;
+	};
+
+	vcc_otg: usb-otg-regulator {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio0 5 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&otg_drv>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-name = "vcc_otg";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		startup-delay-us = <100000>;
+		vin-supply = <&vcc_io>;
+	};
+
+	vcc_sd: sdmmc-regulator {
+		compatible = "regulator-fixed";
+		gpio = <&gpio3 7 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&sdmmc_pwr>;
+		pinctrl-names = "default";
+		regulator-name = "vcc_sd";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		startup-delay-us = <100000>;
+		vin-supply = <&vcc_io>;
+	};
+
+	vcc_wifi: sdio-regulator {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio3 24 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&wifi_pwr>;
+		pinctrl-names = "default";
+		regulator-name = "vcc_wifi";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		startup-delay-us = <100000>;
+		vin-supply = <&vcc_io>;
+	};
+};
+
+&mmc0 {
+	bus-width = <4>;
+	cap-mmc-highspeed;
+	cap-sd-highspeed;
+	num-slots = <1>;
+	vmmc-supply = <&vcc_sd>;
+	status = "okay";
+};
+
+&mmc1 {
+	bus-width = <4>;
+	disable-wp;
+	non-removable;
+	num-slots = <1>;
+	pinctrl-0 = <&sd1_clk &sd1_cmd &sd1_bus4>;
+	pinctrl-names = "default";
+	vmmc-supply = <&vcc_wifi>;
+	status = "okay";
+};
+
+&pinctrl {
+	usb-host {
+		host_drv: host-drv {
+			rockchip,pins = <RK_GPIO0 6 RK_FUNC_GPIO &pcfg_pull_default>;
+		};
+	};
+
+	usb-otg {
+		otg_drv: otg-drv {
+			rockchip,pins = <RK_GPIO0 5 RK_FUNC_GPIO &pcfg_pull_default>;
+		};
+	};
+
+	sdmmc {
+		sdmmc_pwr: sdmmc-pwr {
+			rockchip,pins = <RK_GPIO3 7 RK_FUNC_GPIO &pcfg_pull_default>;
+		};
+	};
+
+	sdio {
+		wifi_pwr: wifi-pwr {
+			rockchip,pins = <RK_GPIO3 24 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&usb_host {
+	status = "okay";
+};
+
+&usb_otg {
+	status = "okay";
+};
+
+&usbphy {
+	status = "okay";
+};
+
+&wdt {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/rk3066a-rayeager.dts b/arch/arm/boot/dts/rk3066a-rayeager.dts
index 6e7f2187a0e3..82465b644443 100644
--- a/arch/arm/boot/dts/rk3066a-rayeager.dts
+++ b/arch/arm/boot/dts/rk3066a-rayeager.dts
@@ -48,7 +48,7 @@
 	model = "Rayeager PX2";
 	compatible = "chipspark,rayeager-px2", "rockchip,rk3066a";
 
-	memory {
+	memory@60000000 {
 		device_type = "memory";
 		reg = <0x60000000 0x40000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3066a.dtsi b/arch/arm/boot/dts/rk3066a.dtsi
index 0d0dae3a1694..e498c362b9e7 100644
--- a/arch/arm/boot/dts/rk3066a.dtsi
+++ b/arch/arm/boot/dts/rk3066a.dtsi
@@ -151,6 +151,14 @@
 
 		#clock-cells = <1>;
 		#reset-cells = <1>;
+		assigned-clocks = <&cru PLL_CPLL>, <&cru PLL_GPLL>,
+				  <&cru ACLK_CPU>, <&cru HCLK_CPU>,
+				  <&cru PCLK_CPU>, <&cru ACLK_PERI>,
+				  <&cru HCLK_PERI>, <&cru PCLK_PERI>;
+		assigned-clock-rates = <400000000>, <594000000>,
+				       <300000000>, <150000000>,
+				       <75000000>, <300000000>,
+				       <150000000>, <75000000>;
 	};
 
 	timer@2000e000 {
@@ -162,7 +170,7 @@
 	};
 
 	efuse: efuse@20010000 {
-		compatible = "rockchip,rockchip-efuse";
+		compatible = "rockchip,rk3066a-efuse";
 		reg = <0x20010000 0x4000>;
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -197,7 +205,7 @@
 		clock-names = "saradc", "apb_pclk";
 		interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
 		#io-channel-cells = <1>;
-		resets = <&cru SRST_SARADC>;
+		resets = <&cru SRST_TSADC>;
 		reset-names = "saradc-apb";
 		status = "disabled";
 	};
@@ -628,15 +636,26 @@
 };
 
 &mmc0 {
+	clock-frequency = <50000000>;
+	dmas = <&dmac2 1>;
+	dma-names = "rx-tx";
+	max-frequency = <50000000>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_cd &sd0_bus4>;
 };
 
 &mmc1 {
+	dmas = <&dmac2 3>;
+	dma-names = "rx-tx";
 	pinctrl-names = "default";
 	pinctrl-0 = <&sd1_clk &sd1_cmd &sd1_cd &sd1_bus4>;
 };
 
+&emmc {
+	dmas = <&dmac2 4>;
+	dma-names = "rx-tx";
+};
+
 &pwm0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pwm0_out>;
@@ -668,21 +687,29 @@
 };
 
 &uart0 {
+	dmas = <&dmac1_s 0>, <&dmac1_s 1>;
+	dma-names = "tx", "rx";
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_xfer>;
 };
 
 &uart1 {
+	dmas = <&dmac1_s 2>, <&dmac1_s 3>;
+	dma-names = "tx", "rx";
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart1_xfer>;
 };
 
 &uart2 {
+	dmas = <&dmac2 6>, <&dmac2 7>;
+	dma-names = "tx", "rx";
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart2_xfer>;
 };
 
 &uart3 {
+	dmas = <&dmac2 8>, <&dmac2 9>;
+	dma-names = "tx", "rx";
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart3_xfer>;
 };
diff --git a/arch/arm/boot/dts/rk3188-px3-evb.dts b/arch/arm/boot/dts/rk3188-px3-evb.dts
new file mode 100644
index 000000000000..df727bafd6dc
--- /dev/null
+++ b/arch/arm/boot/dts/rk3188-px3-evb.dts
@@ -0,0 +1,328 @@
+/*
+ * Copyright (c) 2016 Andy Yan <andy.yan@rock-chips.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+#include <dt-bindings/input/input.h>
+#include "rk3188.dtsi"
+
+/ {
+	model = "Rockchip PX3-EVB";
+	compatible = "rockchip,px3-evb", "rockchip,px3", "rockchip,rk3188";
+
+	chosen {
+		stdout-path = "serial2:115200n8";
+	};
+
+	memory@60000000 {
+		reg = <0x60000000 0x80000000>;
+		device_type = "memory";
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		autorepeat;
+
+		power {
+			gpios = <&gpio0 4 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_POWER>;
+			label = "GPIO Key Power";
+			linux,input-type = <1>;
+			wakeup-source;
+			debounce-interval = <100>;
+		};
+	};
+
+	vcc_sys: vsys-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vsys";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-boot-on;
+	};
+};
+
+&cpu0 {
+	cpu0-supply = <&vdd_cpu>;
+};
+
+&emmc {
+	bus-width = <8>;
+	cap-mmc-highspeed;
+	disable-wp;
+	non-removable;
+	num-slots = <1>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_clk>, <&emmc_cmd>, <&emmc_rst>;
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+
+	accelerometer@18 {
+		compatible = "bosch,bma250";
+		reg = <0x18>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
+
+&i2c1 {
+	status = "okay";
+	clock-frequency = <400000>;
+
+	rk808: pmic@1c {
+		compatible = "rockchip,rk818";
+		reg = <0x1c>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		rockchip,system-power-controller;
+		wakeup-source;
+		#clock-cells = <1>;
+		clock-output-names = "xin32k", "rk808-clkout2";
+
+		vcc1-supply = <&vcc_sys>;
+		vcc2-supply = <&vcc_sys>;
+		vcc3-supply = <&vcc_sys>;
+		vcc4-supply = <&vcc_sys>;
+		vcc6-supply = <&vcc_sys>;
+		vcc7-supply = <&vcc_sys>;
+		vcc8-supply = <&vcc_io>;
+		vcc9-supply = <&vcc_io>;
+
+		regulators {
+			vdd_cpu: DCDC_REG1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-name = "vdd_arm";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_gpu: DCDC_REG2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <1250000>;
+				regulator-name = "vdd_gpu";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1000000>;
+				};
+			};
+
+			vcc_ddr: DCDC_REG3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vcc_ddr";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcc_io: DCDC_REG4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc_io";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vcc_cif: LDO_REG1 {
+				 regulator-min-microvolt = <3300000>;
+				 regulator-max-microvolt = <3300000>;
+				 regulator-name = "vcc_cif";
+			};
+
+			vcc_jetta33: LDO_REG2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc_jetta33";
+			};
+
+			vdd_10: LDO_REG3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-name = "vdd_10";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1000000>;
+				};
+			};
+
+			lvds_12: LDO_REG4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "lvds_12";
+			};
+
+			lvds_25: LDO_REG5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "lvds_25";
+			};
+
+			cif_18: LDO_REG6 {
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-name = "cif_18";
+			};
+
+			vcc_sd: LDO_REG7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc_sd";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			wl_18: LDO_REG8 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "wl_18";
+			};
+
+			lcd_33: SWITCH_REG1 {
+				regulator-name = "lcd_33";
+			};
+		};
+	};
+
+};
+
+&i2c2 {
+	gsl1680: touchscreen@40 {
+		compatible = "silead,gsl1680";
+		reg = <0x40>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <15 IRQ_TYPE_EDGE_FALLING>;
+		power-gpios = <&gpio0 14 GPIO_ACTIVE_HIGH>;
+		touchscreen-size-x = <800>;
+		touchscreen-size-y = <1280>;
+		silead,max-fingers = <5>;
+	};
+};
+
+&mmc0 {
+	num-slots = <1>;
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&sd0_clk>, <&sd0_cmd>, <&sd0_cd>, <&sd0_bus4>;
+	vmmc-supply = <&vcc_sd>;
+
+	bus-width = <4>;
+	cap-mmc-highspeed;
+	cap-sd-highspeed;
+	disable-wp;
+};
+
+&pinctrl {
+	pcfg_output_low: pcfg-output-low {
+		output-low;
+	};
+
+	usb {
+		host_vbus_drv: host-vbus-drv {
+			rockchip,pins = <0 3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+		otg_vbus_drv: otg-vbus-drv {
+			rockchip,pins = <2 31 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&pwm1 {
+	status = "okay";
+};
+
+&pwm2 {
+	status = "okay";
+};
+
+&pwm3 {
+	status = "okay";
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&uart3 {
+	status = "okay";
+};
+
+&usbphy {
+	status = "okay";
+};
+
+&usb_host {
+	status = "okay";
+};
+
+&usb_otg {
+	status = "okay";
+};
+
+&wdt {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/rk3188-radxarock.dts b/arch/arm/boot/dts/rk3188-radxarock.dts
index 1da46d138029..5e8a235ed02d 100644
--- a/arch/arm/boot/dts/rk3188-radxarock.dts
+++ b/arch/arm/boot/dts/rk3188-radxarock.dts
@@ -48,7 +48,7 @@
 	model = "Radxa Rock";
 	compatible = "radxa,rock", "rockchip,rk3188";
 
-	memory {
+	memory@60000000 {
 		device_type = "memory";
 		reg = <0x60000000 0x80000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3188.dtsi b/arch/arm/boot/dts/rk3188.dtsi
index 31f81b265cef..869e189331ec 100644
--- a/arch/arm/boot/dts/rk3188.dtsi
+++ b/arch/arm/boot/dts/rk3188.dtsi
@@ -147,7 +147,7 @@
 	};
 
 	efuse: efuse@20010000 {
-		compatible = "rockchip,rockchip-efuse";
+		compatible = "rockchip,rk3188-efuse";
 		reg = <0x20010000 0x4000>;
 		#address-cells = <1>;
 		#size-cells = <1>;
diff --git a/arch/arm/boot/dts/rk3228-evb.dts b/arch/arm/boot/dts/rk3228-evb.dts
index 904668e2e666..58834330a5ba 100644
--- a/arch/arm/boot/dts/rk3228-evb.dts
+++ b/arch/arm/boot/dts/rk3228-evb.dts
@@ -46,7 +46,7 @@
 	model = "Rockchip RK3228 Evaluation board";
 	compatible = "rockchip,rk3228-evb", "rockchip,rk3228";
 
-	memory {
+	memory@60000000 {
 		device_type = "memory";
 		reg = <0x60000000 0x40000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3229-evb.dts b/arch/arm/boot/dts/rk3229-evb.dts
index b6a12035a6bb..dcdd0cee619e 100644
--- a/arch/arm/boot/dts/rk3229-evb.dts
+++ b/arch/arm/boot/dts/rk3229-evb.dts
@@ -46,7 +46,7 @@
 	model = "Rockchip RK3229 Evaluation board";
 	compatible = "rockchip,rk3229-evb", "rockchip,rk3229";
 
-	memory {
+	memory@60000000 {
 		device_type = "memory";
 		reg = <0x60000000 0x40000000>;
 	};
diff --git a/arch/arm/boot/dts/rk322x.dtsi b/arch/arm/boot/dts/rk322x.dtsi
index 9e6bf0e311bb..9d3aee5abc15 100644
--- a/arch/arm/boot/dts/rk322x.dtsi
+++ b/arch/arm/boot/dts/rk322x.dtsi
@@ -44,9 +44,11 @@
 #include <dt-bindings/pinctrl/rockchip.h>
 #include <dt-bindings/clock/rk3228-cru.h>
 #include <dt-bindings/thermal/thermal.h>
-#include "skeleton.dtsi"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	interrupt-parent = <&gic>;
 
 	aliases {
@@ -402,7 +404,7 @@
 		reg = <0x30020000 0x4000>;
 		interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
 		clock-frequency = <37500000>;
-		clock-freq-min-max = <400000 37500000>;
+		max-frequency = <37500000>;
 		clocks = <&cru HCLK_EMMC>, <&cru SCLK_EMMC>,
 			 <&cru SCLK_EMMC_DRV>, <&cru SCLK_EMMC_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
diff --git a/arch/arm/boot/dts/rk3288-evb.dtsi b/arch/arm/boot/dts/rk3288-evb.dtsi
index d59208b5eb6c..bf7ccfad3260 100644
--- a/arch/arm/boot/dts/rk3288-evb.dtsi
+++ b/arch/arm/boot/dts/rk3288-evb.dtsi
@@ -43,7 +43,7 @@
 #include "rk3288.dtsi"
 
 / {
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x0 0x80000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3288-fennec.dts b/arch/arm/boot/dts/rk3288-fennec.dts
index 2e3c34135ed8..805c0d26770b 100644
--- a/arch/arm/boot/dts/rk3288-fennec.dts
+++ b/arch/arm/boot/dts/rk3288-fennec.dts
@@ -46,7 +46,7 @@
 	model = "Rockchip RK3288 Fennec Board";
 	compatible = "rockchip,rk3288-fennec", "rockchip,rk3288";
 
-	memory {
+	memory@0 {
 		reg = <0x0 0x80000000>;
 		device_type = "memory";
 	};
diff --git a/arch/arm/boot/dts/rk3288-firefly-reload-core.dtsi b/arch/arm/boot/dts/rk3288-firefly-reload-core.dtsi
index ec418c99de95..d242588bae0d 100644
--- a/arch/arm/boot/dts/rk3288-firefly-reload-core.dtsi
+++ b/arch/arm/boot/dts/rk3288-firefly-reload-core.dtsi
@@ -45,7 +45,7 @@
 #include "rk3288.dtsi"
 
 / {
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0 0x80000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3288-firefly.dtsi b/arch/arm/boot/dts/rk3288-firefly.dtsi
index 114c90fb65e2..44935af1fb0e 100644
--- a/arch/arm/boot/dts/rk3288-firefly.dtsi
+++ b/arch/arm/boot/dts/rk3288-firefly.dtsi
@@ -44,7 +44,7 @@
 #include "rk3288.dtsi"
 
 / {
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0 0x80000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3288-miqi.dts b/arch/arm/boot/dts/rk3288-miqi.dts
index 24488421f0f0..441d450fd151 100644
--- a/arch/arm/boot/dts/rk3288-miqi.dts
+++ b/arch/arm/boot/dts/rk3288-miqi.dts
@@ -52,7 +52,7 @@
 		stdout-path = "serial2:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0 0x80000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3288-popmetal.dts b/arch/arm/boot/dts/rk3288-popmetal.dts
index 56dd377d5658..bc6d10054f6a 100644
--- a/arch/arm/boot/dts/rk3288-popmetal.dts
+++ b/arch/arm/boot/dts/rk3288-popmetal.dts
@@ -48,7 +48,7 @@
 	model = "PopMetal-RK3288";
 	compatible = "chipspark,popmetal-rk3288", "rockchip,rk3288";
 
-	memory{
+	memory@0 {
 		device_type = "memory";
 		reg = <0 0x80000000>;
 	};
@@ -68,7 +68,7 @@
 		pinctrl-0 = <&pwrbtn>;
 
 		power {
-			gpios = <&gpio0 5 GPIO_ACTIVE_LOW>;
+			gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_POWER>;
 			label = "GPIO Key Power";
 			linux,input-type = <1>;
@@ -79,7 +79,7 @@
 
 	ir: ir-receiver {
 		compatible = "gpio-ir-receiver";
-		gpios = <&gpio0 6 GPIO_ACTIVE_LOW>;
+		gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_LOW>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&ir_int>;
 	};
@@ -94,7 +94,7 @@
 
 	vcc_sd: sdmmc-regulator {
 		compatible = "regulator-fixed";
-		gpio = <&gpio7 11 GPIO_ACTIVE_LOW>;
+		gpio = <&gpio7 RK_PB3 GPIO_ACTIVE_LOW>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&sdmmc_pwr>;
 		regulator-name = "vcc_sd";
@@ -128,7 +128,7 @@
 	vcc28_dvp: vcc28-dvp-regulator {
 		compatible = "regulator-fixed";
 		enable-active-high;
-		gpio = <&gpio0 17 GPIO_ACTIVE_HIGH>;
+		gpio = <&gpio0 RK_PC1 GPIO_ACTIVE_HIGH>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&dvp_pwr>;
 		regulator-name = "vcc28_dvp";
@@ -147,6 +147,8 @@
 	bus-width = <8>;
 	cap-mmc-highspeed;
 	disable-wp;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
 	non-removable;
 	num-slots = <1>;
 	pinctrl-names = "default";
@@ -165,6 +167,10 @@
 	num-slots = <1>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>;
+	sd-uhs-sdr12;
+	sd-uhs-sdr25;
+	sd-uhs-sdr50;
+	sd-uhs-sdr104;
 	vmmc-supply = <&vcc_sd>;
 	vqmmc-supply = <&vccio_sd>;
 	status = "okay";
@@ -174,7 +180,7 @@
 	phy-supply = <&vcc_lan>;
 	phy-mode = "rgmii";
 	clock_in_out = "input";
-	snps,reset-gpio = <&gpio4 7 0>;
+	snps,reset-gpio = <&gpio4 RK_PB0 0>;
 	snps,reset-active-low;
 	snps,reset-delays-us = <0 10000 1000000>;
 	assigned-clocks = <&cru SCLK_MAC>;
@@ -280,7 +286,7 @@
 			vccio_sd: LDO_REG2 {
 				regulator-always-on;
 				regulator-boot-on;
-				regulator-min-microvolt = <3300000>;
+				regulator-min-microvolt = <1800000>;
 				regulator-max-microvolt = <3300000>;
 				regulator-name = "vccio_sd";
 				regulator-state-mem {
@@ -443,43 +449,43 @@
 &pinctrl {
 	ak8963 {
 		comp_int: comp-int {
-			rockchip,pins = <8 1 RK_FUNC_GPIO &pcfg_pull_up>;
+			rockchip,pins = <8 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>;
 		};
 	};
 
 	buttons {
 		pwrbtn: pwrbtn {
-			rockchip,pins = <0 5 RK_FUNC_GPIO &pcfg_pull_up>;
+			rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>;
 		};
 	};
 
 	dvp {
 		dvp_pwr: dvp-pwr {
-			rockchip,pins = <0 17 RK_FUNC_GPIO &pcfg_pull_none>;
+			rockchip,pins = <0 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 	};
 
 	ir {
 		ir_int: ir-int {
-			rockchip,pins = <0 6 RK_FUNC_GPIO &pcfg_pull_up>;
+			rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>;
 		};
 	};
 
 	mma8452 {
 		gsensor_int: gsensor-int {
-			rockchip,pins = <8 0 RK_FUNC_GPIO &pcfg_pull_up>;
+			rockchip,pins = <8 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>;
 		};
 	};
 
 	pmic {
 		pmic_int: pmic-int {
-			rockchip,pins = <RK_GPIO0 4 RK_FUNC_GPIO &pcfg_pull_up>;
+			rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>;
 		};
 	};
 
 	sdmmc {
 		sdmmc_pwr: sdmmc-pwr {
-			rockchip,pins = <7 11 RK_FUNC_GPIO &pcfg_pull_none>;
+			rockchip,pins = <7 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/rk3288-r89.dts b/arch/arm/boot/dts/rk3288-r89.dts
index 4b8a8adb243c..04faa72dbd95 100644
--- a/arch/arm/boot/dts/rk3288-r89.dts
+++ b/arch/arm/boot/dts/rk3288-r89.dts
@@ -48,7 +48,7 @@
 / {
 	compatible = "netxeon,r89", "rockchip,rk3288";
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x0 0x80000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3288-rock2-som.dtsi b/arch/arm/boot/dts/rk3288-rock2-som.dtsi
index bb1f01e037ba..b25ba806d5ee 100644
--- a/arch/arm/boot/dts/rk3288-rock2-som.dtsi
+++ b/arch/arm/boot/dts/rk3288-rock2-som.dtsi
@@ -42,7 +42,7 @@
 #include "rk3288.dtsi"
 
 / {
-	memory {
+	memory@0 {
 		reg = <0x0 0x80000000>;
 		device_type = "memory";
 	};
diff --git a/arch/arm/boot/dts/rk3288-veyron.dtsi b/arch/arm/boot/dts/rk3288-veyron.dtsi
index 3dd2cca48c11..2251d28e9d2a 100644
--- a/arch/arm/boot/dts/rk3288-veyron.dtsi
+++ b/arch/arm/boot/dts/rk3288-veyron.dtsi
@@ -47,7 +47,7 @@
 #include "rk3288.dtsi"
 
 / {
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x0 0x80000000>;
 	};
diff --git a/arch/arm/boot/dts/rk3288.dtsi b/arch/arm/boot/dts/rk3288.dtsi
index 17ec2e2d7a60..4fad13368a7b 100644
--- a/arch/arm/boot/dts/rk3288.dtsi
+++ b/arch/arm/boot/dts/rk3288.dtsi
@@ -46,9 +46,11 @@
 #include <dt-bindings/thermal/thermal.h>
 #include <dt-bindings/power/rk3288-power.h>
 #include <dt-bindings/soc/rockchip,boot-mode.h>
-#include "skeleton.dtsi"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	compatible = "rockchip,rk3288";
 
 	interrupt-parent = <&gic>;
@@ -227,7 +229,7 @@
 
 	sdmmc: dwmmc@ff0c0000 {
 		compatible = "rockchip,rk3288-dw-mshc";
-		clock-freq-min-max = <400000 150000000>;
+		max-frequency = <150000000>;
 		clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>,
 			 <&cru SCLK_SDMMC_DRV>, <&cru SCLK_SDMMC_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
@@ -239,7 +241,7 @@
 
 	sdio0: dwmmc@ff0d0000 {
 		compatible = "rockchip,rk3288-dw-mshc";
-		clock-freq-min-max = <400000 150000000>;
+		max-frequency = <150000000>;
 		clocks = <&cru HCLK_SDIO0>, <&cru SCLK_SDIO0>,
 			 <&cru SCLK_SDIO0_DRV>, <&cru SCLK_SDIO0_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
@@ -251,7 +253,7 @@
 
 	sdio1: dwmmc@ff0e0000 {
 		compatible = "rockchip,rk3288-dw-mshc";
-		clock-freq-min-max = <400000 150000000>;
+		max-frequency = <150000000>;
 		clocks = <&cru HCLK_SDIO1>, <&cru SCLK_SDIO1>,
 			 <&cru SCLK_SDIO1_DRV>, <&cru SCLK_SDIO1_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
@@ -263,7 +265,7 @@
 
 	emmc: dwmmc@ff0f0000 {
 		compatible = "rockchip,rk3288-dw-mshc";
-		clock-freq-min-max = <400000 150000000>;
+		max-frequency = <150000000>;
 		clocks = <&cru HCLK_EMMC>, <&cru SCLK_EMMC>,
 			 <&cru SCLK_EMMC_DRV>, <&cru SCLK_EMMC_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
@@ -596,7 +598,6 @@
 		g-np-tx-fifo-size = <16>;
 		g-rx-fifo-size = <275>;
 		g-tx-fifo-size = <256 128 128 64 64 32>;
-		g-use-dma;
 		phys = <&usbphy0>;
 		phy-names = "usb2-phy";
 		status = "disabled";
@@ -1116,7 +1117,7 @@
 	};
 
 	efuse: efuse@ffb40000 {
-		compatible = "rockchip,rockchip-efuse";
+		compatible = "rockchip,rk3288-efuse";
 		reg = <0xffb40000 0x20>;
 		#address-cells = <1>;
 		#size-cells = <1>;
diff --git a/arch/arm/boot/dts/rk3xxx.dtsi b/arch/arm/boot/dts/rk3xxx.dtsi
index e15beb3c671e..0b45811cf28b 100644
--- a/arch/arm/boot/dts/rk3xxx.dtsi
+++ b/arch/arm/boot/dts/rk3xxx.dtsi
@@ -44,9 +44,11 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/soc/rockchip,boot-mode.h>
-#include "skeleton.dtsi"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
 	interrupt-parent = <&gic>;
 
 	aliases {
@@ -181,7 +183,6 @@
 		g-np-tx-fifo-size = <16>;
 		g-rx-fifo-size = <275>;
 		g-tx-fifo-size = <256 128 128 64 64 32>;
-		g-use-dma;
 		phys = <&usbphy0>;
 		phy-names = "usb2-phy";
 		status = "disabled";
diff --git a/arch/arm/boot/dts/sama5d2.dtsi b/arch/arm/boot/dts/sama5d2.dtsi
index 7173ec9059a1..ceb9783ff7e1 100644
--- a/arch/arm/boot/dts/sama5d2.dtsi
+++ b/arch/arm/boot/dts/sama5d2.dtsi
@@ -735,6 +735,11 @@
 						atmel,clk-output-range = <0 83000000>;
 					};
 
+					securam_clk: securam_clk {
+						#clock-cells = <0>;
+						reg = <51>;
+					};
+
 					i2s0_clk: i2s0_clk {
 						#clock-cells = <0>;
 						reg = <54>;
@@ -1030,6 +1035,7 @@
 				#address-cells = <1>;
 				#size-cells = <0>;
 				clocks = <&twi0_clk>;
+				atmel,fifo-size = <16>;
 				status = "disabled";
 			};
 
@@ -1058,6 +1064,15 @@
 				status = "disabled";
 			};
 
+			securam: sram@f8044000 {
+				compatible = "atmel,sama5d2-securam", "mmio-sram";
+				reg = <0xf8044000 0x1420>;
+				clocks = <&securam_clk>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 0xf8044000 0x1420>;
+			};
+
 			rstc@f8048000 {
 				compatible = "atmel,sama5d3-rstc";
 				reg = <0xf8048000 0x10>;
@@ -1088,30 +1103,12 @@
 				status = "disabled";
 			};
 
-			sckc@f8048050 {
-				compatible = "atmel,at91sam9x5-sckc";
+			clk32k: sckc@f8048050 {
+				compatible = "atmel,sama5d4-sckc";
 				reg = <0xf8048050 0x4>;
 
-				slow_rc_osc: slow_rc_osc {
-					compatible = "atmel,at91sam9x5-clk-slow-rc-osc";
-					#clock-cells = <0>;
-					clock-frequency = <32768>;
-					clock-accuracy = <250000000>;
-					atmel,startup-time-usec = <75>;
-				};
-
-				slow_osc: slow_osc {
-					compatible = "atmel,at91sam9x5-clk-slow-osc";
-					#clock-cells = <0>;
-					clocks = <&slow_xtal>;
-					atmel,startup-time-usec = <1200000>;
-				};
-
-				clk32k: slowck {
-					compatible = "atmel,at91sam9x5-clk-slow";
-					#clock-cells = <0>;
-					clocks = <&slow_rc_osc &slow_osc>;
-				};
+				clocks = <&slow_xtal>;
+				#clock-cells = <0>;
 			};
 
 			rtc@f80480b0 {
@@ -1231,6 +1228,7 @@
 				#address-cells = <1>;
 				#size-cells = <0>;
 				clocks = <&twi1_clk>;
+				atmel,fifo-size = <16>;
 				status = "disabled";
 			};
 
@@ -1260,6 +1258,11 @@
 				clocks = <&pioA_clk>;
 			};
 
+			secumod@fc040000 {
+				compatible = "atmel,sama5d2-secumod", "syscon";
+				reg = <0xfc040000 0x100>;
+			};
+
 			tdes@fc044000 {
 				compatible = "atmel,at91sam9g46-tdes";
 				reg = <0xfc044000 0x100>;
diff --git a/arch/arm/boot/dts/sama5d3.dtsi b/arch/arm/boot/dts/sama5d3.dtsi
index 4c84d333fc7e..b06448ba6649 100644
--- a/arch/arm/boot/dts/sama5d3.dtsi
+++ b/arch/arm/boot/dts/sama5d3.dtsi
@@ -549,8 +549,8 @@
 				dbgu {
 					pinctrl_dbgu: dbgu-0 {
 						atmel,pins =
-							<AT91_PIOB 30 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB30 periph A */
-							 AT91_PIOB 31 AT91_PERIPH_A AT91_PINCTRL_PULL_UP>;	/* PB31 periph A with pullup */
+							<AT91_PIOB 30 AT91_PERIPH_A AT91_PINCTRL_PULL_UP
+							 AT91_PIOB 31 AT91_PERIPH_A AT91_PINCTRL_NONE>;
 					};
 				};
 
diff --git a/arch/arm/boot/dts/sama5d4.dtsi b/arch/arm/boot/dts/sama5d4.dtsi
index 65e725fb5679..4f60c1b7b137 100644
--- a/arch/arm/boot/dts/sama5d4.dtsi
+++ b/arch/arm/boot/dts/sama5d4.dtsi
@@ -1314,30 +1314,11 @@
 				status = "disabled";
 			};
 
-			sckc@fc068650 {
-				compatible = "atmel,at91sam9x5-sckc";
+			clk32k: sckc@fc068650 {
+				compatible = "atmel,sama5d4-sckc";
 				reg = <0xfc068650 0x4>;
-
-				slow_rc_osc: slow_rc_osc {
-					compatible = "atmel,at91sam9x5-clk-slow-rc-osc";
-					#clock-cells = <0>;
-					clock-frequency = <32768>;
-					clock-accuracy = <250000000>;
-					atmel,startup-time-usec = <75>;
-				};
-
-				slow_osc: slow_osc {
-					compatible = "atmel,at91sam9x5-clk-slow-osc";
-					#clock-cells = <0>;
-					clocks = <&slow_xtal>;
-					atmel,startup-time-usec = <1200000>;
-				};
-
-				clk32k: slowck {
-					compatible = "atmel,at91sam9x5-clk-slow";
-					#clock-cells = <0>;
-					clocks = <&slow_rc_osc &slow_osc>;
-				};
+				#clock-cells = <0>;
+				clocks = <&slow_xtal>;
 			};
 
 			rtc@fc0686b0 {
@@ -1461,8 +1442,8 @@
 				dbgu {
 					pinctrl_dbgu: dbgu-0 {
 						atmel,pins =
-							<AT91_PIOB 24 AT91_PERIPH_A AT91_PINCTRL_NONE>,     /* conflicts with D14 and TDI */
-							<AT91_PIOB 25 AT91_PERIPH_A AT91_PINCTRL_PULL_UP>;  /* conflicts with D15 and TDO */
+							<AT91_PIOB 24 AT91_PERIPH_A AT91_PINCTRL_PULL_UP	/* conflicts with D14 and TDI */
+							 AT91_PIOB 25 AT91_PERIPH_A AT91_PINCTRL_NONE>;		/* conflicts with D15 and TDO */
 					};
 				};
 
diff --git a/arch/arm/boot/dts/sh73a0.dtsi b/arch/arm/boot/dts/sh73a0.dtsi
index 032fe2f14b16..e1267590b575 100644
--- a/arch/arm/boot/dts/sh73a0.dtsi
+++ b/arch/arm/boot/dts/sh73a0.dtsi
@@ -8,8 +8,6 @@
  * kind, whether express or implied.
  */
 
-/include/ "skeleton.dtsi"
-
 #include <dt-bindings/clock/sh73a0-clock.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -17,6 +15,8 @@
 / {
 	compatible = "renesas,sh73a0";
 	interrupt-parent = <&gic>;
+	#address-cells = <1>;
+	#size-cells = <1>;
 
 	cpus {
 		#address-cells = <1>;
diff --git a/arch/arm/boot/dts/socfpga.dtsi b/arch/arm/boot/dts/socfpga.dtsi
index 9f48141270b8..da689659131f 100644
--- a/arch/arm/boot/dts/socfpga.dtsi
+++ b/arch/arm/boot/dts/socfpga.dtsi
@@ -686,6 +686,12 @@
 			arm,data-latency = <2 1 1>;
 			prefetch-data = <1>;
 			prefetch-instr = <1>;
+			arm,shared-override;
+			arm,double-linefill = <1>;
+			arm,double-linefill-incr = <0>;
+			arm,double-linefill-wrap = <1>;
+			arm,prefetch-drop = <0>;
+			arm,prefetch-offset = <7>;
 		};
 
 		mmc: dwmmc0@ff704000 {
@@ -700,11 +706,38 @@
 			status = "disabled";
 		};
 
+		nand0: nand@ff900000 {
+			#address-cells = <0x1>;
+			#size-cells = <0x1>;
+			compatible = "denali,denali-nand-dt";
+			reg = <0xff900000 0x100000>,
+			      <0xffb80000 0x10000>;
+			reg-names = "nand_data", "denali_reg";
+			interrupts = <0x0 0x90 0x4>;
+			dma-mask = <0xffffffff>;
+			clocks = <&nand_clk>;
+			status = "disabled";
+		};
+
 		ocram: sram@ffff0000 {
 			compatible = "mmio-sram";
 			reg = <0xffff0000 0x10000>;
 		};
 
+		qspi: spi@ff705000 {
+			compatible = "cdns,qspi-nor";
+                        #address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0xff705000 0x1000>,
+			      <0xffa00000 0x1000>;
+			interrupts = <0 151 4>;
+			cdns,fifo-depth = <128>;
+			cdns,fifo-width = <4>;
+			cdns,trigger-address = <0x00000000>;
+			clocks = <&qspi_clk>;
+			status = "disabled";
+		};
+
 		rst: rstmgr@ffd05000 {
 			#reset-cells = <1>;
 			compatible = "altr,rst-mgr";
diff --git a/arch/arm/boot/dts/socfpga_arria10.dtsi b/arch/arm/boot/dts/socfpga_arria10.dtsi
index f520cbff5e1c..551c636a4f01 100644
--- a/arch/arm/boot/dts/socfpga_arria10.dtsi
+++ b/arch/arm/boot/dts/socfpga_arria10.dtsi
@@ -562,6 +562,21 @@
 			status = "disabled";
 		};
 
+		spi1: spi@ffda5000 {
+			compatible = "snps,dw-apb-ssi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0xffda5000 0x100>;
+			interrupts = <0 102 4>;
+			num-chipselect = <4>;
+			bus-num = <0>;
+			/*32bit_access;*/
+			tx-dma-channel = <&pdma 16>;
+			rx-dma-channel = <&pdma 17>;
+			clocks = <&spi_m_clk>;
+			status = "disabled";
+		};
+
 		sdr: sdr@ffc25000 {
 			compatible = "syscon";
 			reg = <0xffcfb100 0x80>;
@@ -573,6 +588,9 @@
 			interrupts = <0 18 IRQ_TYPE_LEVEL_HIGH>;
 			cache-unified;
 			cache-level = <2>;
+			prefetch-data = <1>;
+			prefetch-instr = <1>;
+			arm,shared-override;
 		};
 
 		mmc: dwmmc0@ff808000 {
@@ -657,6 +675,20 @@
 			};
 		};
 
+		qspi: spi@ff809000 {
+			compatible = "cdns,qspi-nor";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0xff809000 0x100>,
+			      <0xffa00000 0x100000>;
+			interrupts = <0 100 IRQ_TYPE_LEVEL_HIGH>;
+			cdns,fifo-depth = <128>;
+			cdns,fifo-width = <4>;
+			cdns,trigger-address = <0x00000000>;
+			clocks = <&qspi_clk>;
+			status = "disabled";
+		};
+
 		rst: rstmgr@ffd05000 {
 			#reset-cells = <1>;
 			compatible = "altr,rst-mgr";
diff --git a/arch/arm/boot/dts/socfpga_arria10_socdk.dtsi b/arch/arm/boot/dts/socfpga_arria10_socdk.dtsi
index 8e3a4adc389f..eb00ae37f316 100644
--- a/arch/arm/boot/dts/socfpga_arria10_socdk.dtsi
+++ b/arch/arm/boot/dts/socfpga_arria10_socdk.dtsi
@@ -36,6 +36,30 @@
 		reg = <0x0 0x40000000>; /* 1GB */
 	};
 
+	a10leds {
+		compatible = "gpio-leds";
+
+		a10sr_led0 {
+			label = "a10sr-led0";
+			gpios = <&a10sr_gpio 0 1>;
+		};
+
+		a10sr_led1 {
+			label = "a10sr-led1";
+			gpios = <&a10sr_gpio 1 1>;
+		};
+
+		a10sr_led2 {
+			label = "a10sr-led2";
+			gpios = <&a10sr_gpio 2 1>;
+		};
+
+		a10sr_led3 {
+			label = "a10sr-led3";
+			gpios = <&a10sr_gpio 3 1>;
+		};
+	};
+
 	soc {
 		clkmgr@ffd04000 {
 			clocks {
@@ -75,6 +99,31 @@
 	status = "okay";
 };
 
+&gpio1 {
+	status = "okay";
+};
+
+&spi1 {
+	status = "okay";
+
+	resource-manager@0 {
+		compatible = "altr,a10sr";
+		reg = <0>;
+		spi-max-frequency = <100000>;
+		/* low-level active IRQ at GPIO1_5 */
+		interrupt-parent = <&portb>;
+		interrupts = <5 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+
+		a10sr_gpio: gpio-controller {
+			compatible = "altr,a10sr-gpio";
+			gpio-controller;
+			#gpio-cells = <2>;
+		};
+	};
+};
+
 &i2c1 {
 	speed-mode = <0>;
 	status = "okay";
diff --git a/arch/arm/boot/dts/socfpga_arria10_socdk_qspi.dts b/arch/arm/boot/dts/socfpga_arria10_socdk_qspi.dts
new file mode 100644
index 000000000000..beb2fc6b9eb6
--- /dev/null
+++ b/arch/arm/boot/dts/socfpga_arria10_socdk_qspi.dts
@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2016 Intel. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+#include "socfpga_arria10_socdk.dtsi"
+
+&qspi {
+	status = "okay";
+
+	flash0: n25q00@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "n25q00aa";
+		reg = <0>;
+		spi-max-frequency = <100000000>;
+
+		m25p,fast-read;
+		cdns,page-size = <256>;
+		cdns,block-size = <16>;
+		cdns,read-delay = <4>;
+		cdns,tshsl-ns = <50>;
+		cdns,tsd2d-ns = <50>;
+		cdns,tchsh-ns = <4>;
+		cdns,tslch-ns = <4>;
+
+		partition@qspi-boot {
+			label = "Boot and fpga data";
+			reg = <0x0 0x2720000>;
+		};
+
+		partition@qspi-rootfs {
+			label = "Root Filesystem - JFFS2";
+			reg = <0x2720000 0x58E0000>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/socfpga_arria5_socdk.dts b/arch/arm/boot/dts/socfpga_arria5_socdk.dts
index 3c8867862b0d..f739ead074a2 100644
--- a/arch/arm/boot/dts/socfpga_arria5_socdk.dts
+++ b/arch/arm/boot/dts/socfpga_arria5_socdk.dts
@@ -82,6 +82,39 @@
 	status = "okay";
 };
 
+&qspi {
+	status = "okay";
+
+	flash: flash@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "n25q256a";
+		reg = <0>;
+		spi-max-frequency = <100000000>;
+
+		m25p,fast-read;
+		cdns,page-size = <256>;
+		cdns,block-size = <16>;
+		cdns,read-delay = <4>;
+		cdns,tshsl-ns = <50>;
+		cdns,tsd2d-ns = <50>;
+		cdns,tchsh-ns = <4>;
+		cdns,tslch-ns = <4>;
+
+		partition@qspi-boot {
+			/* 8MB for raw data. */
+			label = "Flash 0 Raw Data";
+			reg = <0x0 0x800000>;
+		};
+
+		partition@qspi-rootfs {
+			/* 120MB for jffs2 data. */
+			label = "Flash 0 jffs2 Filesystem";
+			reg = <0x800000 0x7800000>;
+		};
+	};
+};
+
 &usb1 {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/socfpga_cyclone5_de0_sockit.dts b/arch/arm/boot/dts/socfpga_cyclone5_de0_sockit.dts
index afea3645ada4..5ecd2ef405e3 100644
--- a/arch/arm/boot/dts/socfpga_cyclone5_de0_sockit.dts
+++ b/arch/arm/boot/dts/socfpga_cyclone5_de0_sockit.dts
@@ -18,7 +18,7 @@
 
 / {
 	model = "Terasic DE-0(Atlas)";
-	compatible = "altr,socfpga-cyclone5", "altr,socfpga";
+	compatible = "terasic,de0-atlas", "altr,socfpga-cyclone5", "altr,socfpga";
 
 	chosen {
 		bootargs = "earlyprintk";
diff --git a/arch/arm/boot/dts/socfpga_cyclone5_mcv.dtsi b/arch/arm/boot/dts/socfpga_cyclone5_mcv.dtsi
index f86f9c060d7a..6ad3b1eb9b86 100644
--- a/arch/arm/boot/dts/socfpga_cyclone5_mcv.dtsi
+++ b/arch/arm/boot/dts/socfpga_cyclone5_mcv.dtsi
@@ -18,7 +18,7 @@
 #include "socfpga_cyclone5.dtsi"
 
 / {
-	model = "DENX MCV";
+	model = "Aries/DENX MCV";
 	compatible = "altr,socfpga-cyclone5", "altr,socfpga";
 
 	memory {
diff --git a/arch/arm/boot/dts/socfpga_cyclone5_mcvevk.dts b/arch/arm/boot/dts/socfpga_cyclone5_mcvevk.dts
index 7186a29b8b86..e5a98e5696ca 100644
--- a/arch/arm/boot/dts/socfpga_cyclone5_mcvevk.dts
+++ b/arch/arm/boot/dts/socfpga_cyclone5_mcvevk.dts
@@ -18,8 +18,8 @@
 #include "socfpga_cyclone5_mcv.dtsi"
 
 / {
-	model = "DENX MCV EVK";
-	compatible = "altr,socfpga-cyclone5", "altr,socfpga";
+	model = "Aries/DENX MCV EVK";
+	compatible = "denx,mcvevk", "altr,socfpga-cyclone5", "altr,socfpga";
 
 	aliases {
 		ethernet0 = &gmac0;
diff --git a/arch/arm/boot/dts/socfpga_cyclone5_socdk.dts b/arch/arm/boot/dts/socfpga_cyclone5_socdk.dts
index 15e43f43f244..6306d008f01b 100644
--- a/arch/arm/boot/dts/socfpga_cyclone5_socdk.dts
+++ b/arch/arm/boot/dts/socfpga_cyclone5_socdk.dts
@@ -19,7 +19,7 @@
 
 / {
 	model = "Altera SOCFPGA Cyclone V SoC Development Kit";
-	compatible = "altr,socfpga-cyclone5", "altr,socfpga";
+	compatible = "altr,socfpga-cyclone5-socdk", "altr,socfpga-cyclone5", "altr,socfpga";
 
 	chosen {
 		bootargs = "earlyprintk";
@@ -87,6 +87,39 @@
 	status = "okay";
 };
 
+&qspi {
+	status = "okay";
+
+	flash0: n25q00@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "n25q00";
+		reg = <0>;	/* chip select */
+		spi-max-frequency = <100000000>;
+
+		m25p,fast-read;
+		cdns,page-size = <256>;
+		cdns,block-size = <16>;
+		cdns,read-delay = <4>;
+		cdns,tshsl-ns = <50>;
+		cdns,tsd2d-ns = <50>;
+		cdns,tchsh-ns = <4>;
+		cdns,tslch-ns = <4>;
+
+		partition@qspi-boot {
+			/* 8MB for raw data. */
+			label = "Flash 0 Raw Data";
+			reg = <0x0 0x800000>;
+		};
+
+		partition@qspi-rootfs {
+			/* 120MB for jffs2 data. */
+			label = "Flash 0 jffs2 Filesystem";
+			reg = <0x800000 0x7800000>;
+		};
+	};
+};
+
 &usb1 {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/socfpga_cyclone5_sockit.dts b/arch/arm/boot/dts/socfpga_cyclone5_sockit.dts
index 02e22f554ef0..a0c90b3bdfd1 100644
--- a/arch/arm/boot/dts/socfpga_cyclone5_sockit.dts
+++ b/arch/arm/boot/dts/socfpga_cyclone5_sockit.dts
@@ -19,7 +19,7 @@
 
 / {
 	model = "Terasic SoCkit";
-	compatible = "altr,socfpga-cyclone5", "altr,socfpga";
+	compatible = "terasic,socfpga-cyclone5-sockit", "altr,socfpga-cyclone5", "altr,socfpga";
 
 	chosen {
 		bootargs = "earlyprintk";
@@ -175,6 +175,27 @@
 	status = "okay";
 };
 
+&qspi {
+	status = "okay";
+
+	flash: flash@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "n25q00";
+		reg = <0>;
+		spi-max-frequency = <100000000>;
+
+		m25p,fast-read;
+		cdns,page-size = <256>;
+		cdns,block-size = <16>;
+		cdns,read-delay = <4>;
+		cdns,tshsl-ns = <50>;
+		cdns,tsd2d-ns = <50>;
+		cdns,tchsh-ns = <4>;
+		cdns,tslch-ns = <4>;
+	};
+};
+
 &usb1 {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/socfpga_cyclone5_socrates.dts b/arch/arm/boot/dts/socfpga_cyclone5_socrates.dts
index d79853775061..c3d52f27b21e 100644
--- a/arch/arm/boot/dts/socfpga_cyclone5_socrates.dts
+++ b/arch/arm/boot/dts/socfpga_cyclone5_socrates.dts
@@ -80,3 +80,22 @@
 &mmc {
 	status = "okay";
 };
+
+&qspi {
+	status = "okay";
+
+	flash: flash@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "n25q256a";
+		reg = <0>;
+		spi-max-frequency = <100000000>;
+		m25p,fast-read;
+		cdns,read-delay = <4>;
+		cdns,tshsl-ns = <50>;
+		cdns,tsd2d-ns = <50>;
+		cdns,tchsh-ns = <4>;
+		cdns,tslch-ns = <4>;
+		status = "okay";
+	};
+};
diff --git a/arch/arm/boot/dts/socfpga_cyclone5_sodia.dts b/arch/arm/boot/dts/socfpga_cyclone5_sodia.dts
new file mode 100644
index 000000000000..5b7e3c27e6e9
--- /dev/null
+++ b/arch/arm/boot/dts/socfpga_cyclone5_sodia.dts
@@ -0,0 +1,123 @@
+/*
+ *  Copyright (C) 2016 Nobuhiro Iwamatsu <iwamatsu@nigauri.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "socfpga_cyclone5.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/ {
+	model = "Altera SOCFPGA Cyclone V SoC Macnica Sodia board";
+	compatible = "macnica,sodia", "altr,socfpga-cyclone5", "altr,socfpga";
+
+	chosen {
+		bootargs = "earlyprintk";
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory {
+		name = "memory";
+		device_type = "memory";
+		reg = <0x0 0x40000000>;
+	};
+
+	aliases {
+		ethernet0 = &gmac1;
+	};
+
+	regulator_3_3v: 3-3-v-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	leds: gpio-leds {
+		compatible = "gpio-leds";
+
+		hps_led0 {
+			label = "hps:green:led0";
+			gpios = <&portb 12 GPIO_ACTIVE_LOW>;
+		};
+
+		hps_led1 {
+			label = "hps:green:led1";
+			gpios = <&portb 13 GPIO_ACTIVE_LOW>;
+		};
+
+		hps_led2 {
+			label = "hps:green:led2";
+			gpios = <&portb 14 GPIO_ACTIVE_LOW>;
+		};
+
+		hps_led3 {
+			label = "hps:green:led3";
+			gpios = <&portb 15 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&gmac1 {
+	status = "okay";
+	phy-mode = "rgmii";
+	phy = <&phy0>;
+
+	mdio0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		phy0: ethernet-phy@0 {
+			reg = <0>;
+			rxd0-skew-ps = <0>;
+			rxd1-skew-ps = <0>;
+			rxd2-skew-ps = <0>;
+			rxd3-skew-ps = <0>;
+			rxdv-skew-ps = <0>;
+			rxc-skew-ps = <3000>;
+			txen-skew-ps = <0>;
+			txc-skew-ps = <3000>;
+		};
+	};
+};
+
+&gpio1 {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+
+	eeprom@51 {
+		compatible = "atmel,24c32";
+		reg = <0x51>;
+		pagesize = <32>;
+	};
+
+	rtc@68 {
+		compatible = "dallas,ds1339";
+		reg = <0x68>;
+	};
+};
+
+&mmc0 {
+	cd-gpios = <&portb 18 0>;
+	vmmc-supply = <&regulator_3_3v>;
+	vqmmc-supply = <&regulator_3_3v>;
+	status = "okay";
+};
+
+&usb1 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts b/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts
index b844473601d2..363ee62457fe 100644
--- a/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts
+++ b/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts
@@ -51,7 +51,7 @@
 
 / {
 	model = "samtec VIN|ING FPGA";
-	compatible = "altr,socfpga-cyclone5", "altr,socfpga";
+	compatible = "samtec,vining", "altr,socfpga-cyclone5", "altr,socfpga";
 
 	chosen {
 		bootargs = "console=ttyS0,115200";
diff --git a/arch/arm/boot/dts/spear13xx.dtsi b/arch/arm/boot/dts/spear13xx.dtsi
index 449acf0d8272..17ea0abcdbd7 100644
--- a/arch/arm/boot/dts/spear13xx.dtsi
+++ b/arch/arm/boot/dts/spear13xx.dtsi
@@ -118,6 +118,7 @@
 			block_size = <0xfff>;
 			dma-masters = <2>;
 			data-width = <8 8>;
+			multi-block = <1 1 1 1 1 1 1 1>;
 		};
 
 		dma@eb000000 {
@@ -134,6 +135,7 @@
 			chan_priority = <1>;
 			block_size = <0xfff>;
 			data-width = <8 8>;
+			multi-block = <1 1 1 1 1 1 1 1>;
 		};
 
 		fsmc: flash@b0000000 {
diff --git a/arch/arm/boot/dts/stih407-clock.dtsi b/arch/arm/boot/dts/stih407-clock.dtsi
index 13029c03d7c6..34c119a66f14 100644
--- a/arch/arm/boot/dts/stih407-clock.dtsi
+++ b/arch/arm/boot/dts/stih407-clock.dtsi
@@ -101,6 +101,7 @@
 				clocks = <&clk_sysin>;
 
 				clock-output-names = "clk-s-a0-pll-ofd-0";
+				clock-critical = <0>; /* clk-s-a0-pll-ofd-0 */
 			};
 
 			clk_s_a0_flexgen: clk-s-a0-flexgen {
@@ -112,6 +113,7 @@
 					 <&clk_sysin>;
 
 				clock-output-names = "clk-ic-lmi0";
+				clock-critical = <CLK_IC_LMI0>;
 			};
 		};
 
@@ -126,6 +128,7 @@
 					     "clk-s-c0-fs0-ch1",
 					     "clk-s-c0-fs0-ch2",
 					     "clk-s-c0-fs0-ch3";
+			clock-critical = <0>; /* clk-s-c0-fs0-ch0 */
 		};
 
 		clk_s_c0: clockgen-c@09103000 {
@@ -139,6 +142,7 @@
 				clocks = <&clk_sysin>;
 
 				clock-output-names = "clk-s-c0-pll0-odf-0";
+				clock-critical = <0>; /* clk-s-c0-pll0-odf-0 */
 			};
 
 			clk_s_c0_pll1: clk-s-c0-pll1 {
@@ -194,6 +198,12 @@
 						     "clk-main-disp",
 						     "clk-aux-disp",
 						     "clk-compo-dvp";
+				clock-critical = <CLK_PROC_STFE>,
+						 <CLK_ICN_CPU>,
+						 <CLK_TX_ICN_DMU>,
+						 <CLK_EXT2F_A9>,
+						 <CLK_ICN_LMI>,
+						 <CLK_ICN_SBC>;
 			};
 		};
 
diff --git a/arch/arm/boot/dts/stih407-family.dtsi b/arch/arm/boot/dts/stih407-family.dtsi
index 8f79b4147bba..c8b2944e304a 100644
--- a/arch/arm/boot/dts/stih407-family.dtsi
+++ b/arch/arm/boot/dts/stih407-family.dtsi
@@ -916,7 +916,7 @@
 		};
 
 		sti_uni_player0: sti-uni-player@8d80000 {
-			compatible = "st,sti-uni-player";
+			compatible = "st,stih407-uni-player-hdmi";
 			#sound-dai-cells = <0>;
 			st,syscfg = <&syscfg_core>;
 			clocks = <&clk_s_d0_flexgen CLK_PCM_0>;
@@ -926,17 +926,13 @@
 			reg = <0x8d80000 0x158>;
 			interrupts = <GIC_SPI 84 IRQ_TYPE_NONE>;
 			dmas = <&fdma0 2 0 1>;
-			dai-name = "Uni Player #0 (HDMI)";
 			dma-names = "tx";
-			st,uniperiph-id = <0>;
-			st,version = <5>;
-			st,mode = "HDMI";
 
 			status		= "disabled";
 		};
 
 		sti_uni_player1: sti-uni-player@8d81000 {
-			compatible = "st,sti-uni-player";
+			compatible = "st,stih407-uni-player-pcm-out";
 			#sound-dai-cells = <0>;
 			st,syscfg = <&syscfg_core>;
 			clocks = <&clk_s_d0_flexgen CLK_PCM_1>;
@@ -946,17 +942,13 @@
 			reg = <0x8d81000 0x158>;
 			interrupts = <GIC_SPI 85 IRQ_TYPE_NONE>;
 			dmas = <&fdma0 3 0 1>;
-			dai-name = "Uni Player #1 (PIO)";
 			dma-names = "tx";
-			st,uniperiph-id = <1>;
-			st,version = <5>;
-			st,mode = "PCM";
 
 			status = "disabled";
 		};
 
 		sti_uni_player2: sti-uni-player@8d82000 {
-			compatible = "st,sti-uni-player";
+			compatible = "st,stih407-uni-player-dac";
 			#sound-dai-cells = <0>;
 			st,syscfg = <&syscfg_core>;
 			clocks = <&clk_s_d0_flexgen CLK_PCM_2>;
@@ -966,17 +958,13 @@
 			reg = <0x8d82000 0x158>;
 			interrupts = <GIC_SPI 86 IRQ_TYPE_NONE>;
 			dmas = <&fdma0 4 0 1>;
-			dai-name = "Uni Player #1 (DAC)";
 			dma-names = "tx";
-			st,uniperiph-id = <2>;
-			st,version = <5>;
-			st,mode = "PCM";
 
 			status = "disabled";
 		};
 
 		sti_uni_player3: sti-uni-player@8d85000 {
-			compatible = "st,sti-uni-player";
+			compatible = "st,stih407-uni-player-spdif";
 			#sound-dai-cells = <0>;
 			st,syscfg = <&syscfg_core>;
 			clocks = <&clk_s_d0_flexgen CLK_SPDIFF>;
@@ -987,38 +975,30 @@
 			interrupts = <GIC_SPI 89 IRQ_TYPE_NONE>;
 			dmas = <&fdma0 7 0 1>;
 			dma-names = "tx";
-			dai-name = "Uni Player #1 (PIO)";
-			st,uniperiph-id = <3>;
-			st,version = <5>;
-			st,mode = "SPDIF";
 
 			status = "disabled";
 		};
 
 		sti_uni_reader0: sti-uni-reader@8d83000 {
-			compatible = "st,sti-uni-reader";
+			compatible = "st,stih407-uni-reader-pcm_in";
 			#sound-dai-cells = <0>;
 			st,syscfg = <&syscfg_core>;
 			reg = <0x8d83000 0x158>;
 			interrupts = <GIC_SPI 87 IRQ_TYPE_NONE>;
 			dmas = <&fdma0 5 0 1>;
 			dma-names = "rx";
-			dai-name = "Uni Reader #0 (PCM IN)";
-			st,version = <3>;
 
 			status = "disabled";
 		};
 
 		sti_uni_reader1: sti-uni-reader@8d84000 {
-			compatible = "st,sti-uni-reader";
+			compatible = "st,stih407-uni-reader-hdmi";
 			#sound-dai-cells = <0>;
 			st,syscfg = <&syscfg_core>;
 			reg = <0x8d84000 0x158>;
 			interrupts = <GIC_SPI 88 IRQ_TYPE_NONE>;
 			dmas = <&fdma0 6 0 1>;
 			dma-names = "rx";
-			dai-name = "Uni Reader #1 (HDMI RX)";
-			st,version = <3>;
 
 			status = "disabled";
 		};
diff --git a/arch/arm/boot/dts/stih407-pinctrl.dtsi b/arch/arm/boot/dts/stih407-pinctrl.dtsi
index c325cc059ae4..daab16b5ae64 100644
--- a/arch/arm/boot/dts/stih407-pinctrl.dtsi
+++ b/arch/arm/boot/dts/stih407-pinctrl.dtsi
@@ -1157,7 +1157,7 @@
 			reg = <0x0923f080 0x4>;
 			reg-names = "irqmux";
 			interrupts = <GIC_SPI 192 IRQ_TYPE_NONE>;
-			interrupts-names = "irqmux";
+			interrupt-names = "irqmux";
 			ranges = <0 0x09230000 0x3000>;
 
 			pio40: gpio@09230000 {
diff --git a/arch/arm/boot/dts/stih407.dtsi b/arch/arm/boot/dts/stih407.dtsi
index 291ffacbd2e0..fa149837df14 100644
--- a/arch/arm/boot/dts/stih407.dtsi
+++ b/arch/arm/boot/dts/stih407.dtsi
@@ -102,7 +102,7 @@
 							 <&clk_s_d2_quadfs 0>;
 			};
 
-			sti-hdmi@8d04000 {
+			sti_hdmi: sti-hdmi@8d04000 {
 				compatible = "st,stih407-hdmi";
 				reg = <0x8d04000 0x1000>;
 				reg-names = "hdmi-reg";
diff --git a/arch/arm/boot/dts/stih410-b2260.dts b/arch/arm/boot/dts/stih410-b2260.dts
index 7fb507fcba7e..06b0696cb6b8 100644
--- a/arch/arm/boot/dts/stih410-b2260.dts
+++ b/arch/arm/boot/dts/stih410-b2260.dts
@@ -165,6 +165,9 @@
 			status = "okay";
 		};
 
+		sti_uni_player0: sti-uni-player@8d80000 {
+			status = "okay";
+		};
 		/* SSC11 to HDMI */
 		hdmiddc: i2c@9541000 {
 			/* HDMI V1.3a supports Standard mode only */
@@ -174,9 +177,22 @@
 			status = "okay";
 		};
 
-		sti-display-subsystem {
-			sti_hdmi: sti-hdmi@8d04000 {
-				status = "okay";
+		sound {
+			compatible = "simple-audio-card";
+			simple-audio-card,name = "STI-B2260";
+			status = "okay";
+
+			simple-audio-card,dai-link@0 {
+				/* DAC */
+				format = "i2s";
+				mclk-fs = <128>;
+				cpu {
+					sound-dai = <&sti_uni_player0>;
+				};
+
+				codec {
+					sound-dai = <&sti_hdmi>;
+				};
 			};
 		};
 
diff --git a/arch/arm/boot/dts/stih410-clock.dtsi b/arch/arm/boot/dts/stih410-clock.dtsi
index 8598effd6c01..07c8ef9d77f6 100644
--- a/arch/arm/boot/dts/stih410-clock.dtsi
+++ b/arch/arm/boot/dts/stih410-clock.dtsi
@@ -208,7 +208,8 @@
 						     "clk-clust-hades",
 						     "clk-hwpe-hades",
 						     "clk-fc-hades";
-				clock-critical = <CLK_ICN_CPU>,
+				clock-critical = <CLK_PROC_STFE>,
+						 <CLK_ICN_CPU>,
 						 <CLK_TX_ICN_DMU>,
 						 <CLK_EXT2F_A9>,
 						 <CLK_ICN_LMI>,
diff --git a/arch/arm/boot/dts/stih410.dtsi b/arch/arm/boot/dts/stih410.dtsi
index a3ef7341c051..281a12424cf6 100644
--- a/arch/arm/boot/dts/stih410.dtsi
+++ b/arch/arm/boot/dts/stih410.dtsi
@@ -193,7 +193,7 @@
 							 <&clk_s_d2_quadfs 0>;
 			};
 
-			sti-hdmi@8d04000 {
+			sti_hdmi: sti-hdmi@8d04000 {
 				compatible = "st,stih407-hdmi";
 				reg = <0x8d04000 0x1000>;
 				reg-names = "hdmi-reg";
diff --git a/arch/arm/boot/dts/stih415-b2000.dts b/arch/arm/boot/dts/stih415-b2000.dts
deleted file mode 100644
index bdfbd3765db2..000000000000
--- a/arch/arm/boot/dts/stih415-b2000.dts
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics (R&D) Limited.
- * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-/dts-v1/;
-#include "stih415.dtsi"
-#include "stih41x-b2000.dtsi"
-/ {
-	model = "STiH415 B2000 Board";
-	compatible = "st,stih415-b2000", "st,stih415";
-};
diff --git a/arch/arm/boot/dts/stih415-b2020.dts b/arch/arm/boot/dts/stih415-b2020.dts
deleted file mode 100644
index 71903a87bd31..000000000000
--- a/arch/arm/boot/dts/stih415-b2020.dts
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics (R&D) Limited.
- * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-/dts-v1/;
-#include "stih415.dtsi"
-#include "stih41x-b2020.dtsi"
-/ {
-	model = "STiH415 B2020 Board";
-	compatible = "st,stih415-b2020", "st,stih415";
-};
diff --git a/arch/arm/boot/dts/stih415-clock.dtsi b/arch/arm/boot/dts/stih415-clock.dtsi
deleted file mode 100644
index 3ee34514bc4b..000000000000
--- a/arch/arm/boot/dts/stih415-clock.dtsi
+++ /dev/null
@@ -1,533 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics (R&D) Limited
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <dt-bindings/clock/stih415-clks.h>
-
-/ {
-	clocks {
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges;
-
-		/*
-		 * Fixed 30MHz oscillator input to SoC
-		 */
-		clk_sysin: clk-sysin {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-			clock-frequency = <30000000>;
-		};
-
-		/*
-		 * ClockGenAs on SASG1
-		 */
-		clockgen-a@fee62000 {
-			reg = <0xfee62000 0xb48>;
-
-			clk_s_a0_pll: clk-s-a0-pll {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-plls-c65";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-s-a0-pll0-hs",
-						     "clk-s-a0-pll0-ls",
-						     "clk-s-a0-pll1";
-			};
-
-			clk_s_a0_osc_prediv: clk-s-a0-osc-prediv {
-				#clock-cells = <0>;
-				compatible = "st,clkgena-prediv-c65",
-					     "st,clkgena-prediv";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-s-a0-osc-prediv";
-			};
-
-			clk_s_a0_hs: clk-s-a0-hs {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c65-hs",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_s_a0_osc_prediv>,
-					 <&clk_s_a0_pll 0>, /* PLL0 HS */
-					 <&clk_s_a0_pll 2>; /* PLL1 */
-
-				clock-output-names = "clk-s-fdma-0",
-						     "clk-s-fdma-1",
-						     ""; /* clk-s-jit-sense */
-						     /* Fourth output unused */
-			};
-
-			clk_s_a0_ls: clk-s-a0-ls {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c65-ls",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_s_a0_osc_prediv>,
-					 <&clk_s_a0_pll 1>, /* PLL0 LS */
-					 <&clk_s_a0_pll 2>; /* PLL1 */
-
-				clock-output-names = "clk-s-icn-reg-0",
-						     "clk-s-icn-if-0",
-						     "clk-s-icn-reg-lp-0",
-						     "clk-s-emiss",
-						     "clk-s-eth1-phy",
-						     "clk-s-mii-ref-out";
-						 /* Remaining outputs unused */
-			};
-		};
-
-		clockgen-a@fee81000 {
-			reg = <0xfee81000 0xb48>;
-
-			clk_s_a1_pll: clk-s-a1-pll {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-plls-c65";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-s-a1-pll0-hs",
-						     "clk-s-a1-pll0-ls",
-						     "clk-s-a1-pll1";
-			};
-
-			clk_s_a1_osc_prediv: clk-s-a1-osc-prediv {
-				#clock-cells = <0>;
-				compatible = "st,clkgena-prediv-c65",
-					     "st,clkgena-prediv";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-s-a1-osc-prediv";
-			};
-
-			clk_s_a1_hs: clk-s-a1-hs {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c65-hs",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_s_a1_osc_prediv>,
-					 <&clk_s_a1_pll 0>, /* PLL0 HS */
-					 <&clk_s_a1_pll 2>; /* PLL1 */
-
-				clock-output-names = "", /* Reserved */
-						     "", /* Reserved */
-						     "clk-s-stac-phy",
-						     "clk-s-vtac-tx-phy";
-			};
-
-			clk_s_a1_ls: clk-s-a1-ls {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c65-ls",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_s_a1_osc_prediv>,
-					 <&clk_s_a1_pll 1>, /* PLL0 LS */
-					 <&clk_s_a1_pll 2>; /* PLL1 */
-
-				clock-output-names = "clk-s-icn-if-2",
-						     "clk-s-card-mmc",
-						     "clk-s-icn-if-1",
-						     "clk-s-gmac0-phy",
-						     "clk-s-nand-ctrl",
-						     "", /* Reserved */
-						     "clk-s-mii0-ref-out",
-						     ""; /* clk-s-stac-sys */
-						 /* Remaining outputs unused */
-			};
-		};
-
-		/*
-		 * ClockGenAs on MPE41
-		 */
-		clockgen-a@fde12000 {
-			reg = <0xfde12000 0xb50>;
-
-			clk_m_a0_pll0: clk-m-a0-pll0 {
-				#clock-cells = <1>;
-				compatible = "st,plls-c32-a1x-0", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a0-pll0-phi0",
-						     "clk-m-a0-pll0-phi1",
-						     "clk-m-a0-pll0-phi2",
-						     "clk-m-a0-pll0-phi3";
-			};
-
-			clk_m_a0_pll1: clk-m-a0-pll1 {
-				#clock-cells = <1>;
-				compatible = "st,plls-c32-a1x-1", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a0-pll1-phi0",
-						     "clk-m-a0-pll1-phi1",
-						     "clk-m-a0-pll1-phi2",
-						     "clk-m-a0-pll1-phi3";
-			};
-
-			clk_m_a0_osc_prediv: clk-m-a0-osc-prediv {
-				#clock-cells = <0>;
-				compatible = "st,clkgena-prediv-c32",
-					     "st,clkgena-prediv";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a0-osc-prediv";
-			};
-
-			clk_m_a0_div0: clk-m-a0-div0 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf0",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a0_osc_prediv>,
-					 <&clk_m_a0_pll0 0>, /* PLL0 PHI0 */
-					 <&clk_m_a0_pll1 0>; /* PLL1 PHI0 */
-
-				clock-output-names = "clk-m-apb-pm", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "clk-m-pp-dmu-0",
-						     "clk-m-pp-dmu-1",
-						     "clk-m-icm-disp",
-						     ""; /* Unused */
-			};
-
-			clk_m_a0_div1: clk-m-a0-div1 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf1",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a0_osc_prediv>,
-					 <&clk_m_a0_pll0 1>, /* PLL0 PHI1 */
-					 <&clk_m_a0_pll1 1>; /* PLL1 PHI1 */
-
-				clock-output-names = "", /* Unused */
-						     "", /* Unused */
-						     "clk-m-a9-ext2f",
-						     "clk-m-st40rt",
-						     "clk-m-st231-dmu-0",
-						     "clk-m-st231-dmu-1",
-						     "clk-m-st231-aud",
-						     "clk-m-st231-gp-0";
-			};
-
-			clk_m_a0_div2: clk-m-a0-div2 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf2",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a0_osc_prediv>,
-					 <&clk_m_a0_pll0 2>, /* PLL0 PHI2 */
-					 <&clk_m_a0_pll1 2>; /* PLL1 PHI2 */
-
-				clock-output-names = "clk-m-st231-gp-1",
-						     "clk-m-icn-cpu",
-						     "clk-m-icn-stac",
-						     "clk-m-icn-dmu-0",
-						     "clk-m-icn-dmu-1",
-						     "", /* Unused */
-						     "", /* Unused */
-						     ""; /* Unused */
-			};
-
-			clk_m_a0_div3: clk-m-a0-div3 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf3",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a0_osc_prediv>,
-					 <&clk_m_a0_pll0 3>, /* PLL0 PHI3 */
-					 <&clk_m_a0_pll1 3>; /* PLL1 PHI3 */
-
-				clock-output-names = "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "clk-m-icn-eram",
-						     "clk-m-a9-trace";
-			};
-		};
-
-		clockgen-a@fd6db000 {
-			reg = <0xfd6db000 0xb50>;
-
-			clk_m_a1_pll0: clk-m-a1-pll0 {
-				#clock-cells = <1>;
-				compatible = "st,plls-c32-a1x-0", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a1-pll0-phi0",
-						     "clk-m-a1-pll0-phi1",
-						     "clk-m-a1-pll0-phi2",
-						     "clk-m-a1-pll0-phi3";
-			};
-
-			clk_m_a1_pll1: clk-m-a1-pll1 {
-				#clock-cells = <1>;
-				compatible = "st,plls-c32-a1x-1", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a1-pll1-phi0",
-						     "clk-m-a1-pll1-phi1",
-						     "clk-m-a1-pll1-phi2",
-						     "clk-m-a1-pll1-phi3";
-			};
-
-			clk_m_a1_osc_prediv: clk-m-a1-osc-prediv {
-				#clock-cells = <0>;
-				compatible = "st,clkgena-prediv-c32",
-					     "st,clkgena-prediv";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a1-osc-prediv";
-			};
-
-			clk_m_a1_div0: clk-m-a1-div0 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf0",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a1_osc_prediv>,
-					 <&clk_m_a1_pll0 0>, /* PLL0 PHI0 */
-					 <&clk_m_a1_pll1 0>; /* PLL1 PHI0 */
-
-				clock-output-names = "clk-m-fdma-12",
-						     "clk-m-fdma-10",
-						     "clk-m-fdma-11",
-						     "clk-m-hva-lmi",
-						     "clk-m-proc-sc",
-						     "clk-m-tp",
-						     "clk-m-icn-gpu",
-						     "clk-m-icn-vdp-0";
-			};
-
-			clk_m_a1_div1: clk-m-a1-div1 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf1",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a1_osc_prediv>,
-					 <&clk_m_a1_pll0 1>, /* PLL0 PHI1 */
-					 <&clk_m_a1_pll1 1>; /* PLL1 PHI1 */
-
-				clock-output-names = "clk-m-icn-vdp-1",
-						     "clk-m-icn-vdp-2",
-						     "clk-m-icn-vdp-3",
-						     "clk-m-prv-t1-bus",
-						     "clk-m-icn-vdp-4",
-						     "clk-m-icn-reg-10",
-						     "", /* Unused */
-						     ""; /* clk-m-icn-st231 */
-			};
-
-			clk_m_a1_div2: clk-m-a1-div2 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf2",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a1_osc_prediv>,
-					 <&clk_m_a1_pll0 2>, /* PLL0 PHI2 */
-					 <&clk_m_a1_pll1 2>; /* PLL1 PHI2 */
-
-				clock-output-names = "clk-m-fvdp-proc-alt",
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     ""; /* Unused */
-			};
-
-			clk_m_a1_div3: clk-m-a1-div3 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf3",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a1_osc_prediv>,
-					 <&clk_m_a1_pll0 3>, /* PLL0 PHI3 */
-					 <&clk_m_a1_pll1 3>; /* PLL1 PHI3 */
-
-				clock-output-names = "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     ""; /* Unused */
-			};
-		};
-
-		clk_m_a9_ext2f_div2: clk-m-a9-ext2f-div2 {
-			#clock-cells = <0>;
-			compatible = "fixed-factor-clock";
-			clocks = <&clk_m_a0_div1 2>;
-			clock-div = <2>;
-			clock-mult = <1>;
-		};
-
-		clockgen-a@fd345000 {
-			reg = <0xfd345000 0xb50>;
-
-			clk_m_a2_pll0: clk-m-a2-pll0 {
-				#clock-cells = <1>;
-				compatible = "st,plls-c32-a1x-0", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a2-pll0-phi0",
-						     "clk-m-a2-pll0-phi1",
-						     "clk-m-a2-pll0-phi2",
-						     "clk-m-a2-pll0-phi3";
-			};
-
-			clk_m_a2_pll1: clk-m-a2-pll1 {
-				#clock-cells = <1>;
-				compatible = "st,plls-c32-a1x-1", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a2-pll1-phi0",
-						     "clk-m-a2-pll1-phi1",
-						     "clk-m-a2-pll1-phi2",
-						     "clk-m-a2-pll1-phi3";
-			};
-
-			clk_m_a2_osc_prediv: clk-m-a2-osc-prediv {
-				#clock-cells = <0>;
-				compatible = "st,clkgena-prediv-c32",
-					     "st,clkgena-prediv";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a2-osc-prediv";
-			};
-
-			clk_m_a2_div0: clk-m-a2-div0 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf0",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a2_osc_prediv>,
-					 <&clk_m_a2_pll0 0>, /* PLL0 PHI0 */
-					 <&clk_m_a2_pll1 0>; /* PLL1 PHI0 */
-
-				clock-output-names = "clk-m-vtac-main-phy",
-						     "clk-m-vtac-aux-phy",
-						     "clk-m-stac-phy",
-						     "clk-m-stac-sys",
-						     "", /* clk-m-mpestac-pg */
-						     "", /* clk-m-mpestac-wc */
-						     "", /* clk-m-mpevtacaux-pg*/
-						     ""; /* clk-m-mpevtacmain-pg*/
-			};
-
-			clk_m_a2_div1: clk-m-a2-div1 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf1",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a2_osc_prediv>,
-					 <&clk_m_a2_pll0 1>, /* PLL0 PHI1 */
-					 <&clk_m_a2_pll1 1>; /* PLL1 PHI1 */
-
-				clock-output-names = "", /* clk-m-mpevtacrx0-wc */
-						     "", /* clk-m-mpevtacrx1-wc */
-						     "clk-m-compo-main",
-						     "clk-m-compo-aux",
-						     "clk-m-bdisp-0",
-						     "clk-m-bdisp-1",
-						     "clk-m-icn-bdisp-0",
-						     "clk-m-icn-bdisp-1";
-			};
-
-			clk_m_a2_div2: clk-m-a2-div2 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf2",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a2_osc_prediv>,
-					 <&clk_m_a2_pll0 2>, /* PLL0 PHI2 */
-					 <&clk_m_a2_pll1 2>; /* PLL1 PHI2 */
-
-				clock-output-names = "", /* clk-m-icn-hqvdp0 */
-						     "", /* clk-m-icn-hqvdp1 */
-						     "clk-m-icn-compo",
-						     "", /* clk-m-icn-vdpaux */
-						     "clk-m-icn-ts",
-						     "clk-m-icn-reg-lp-10",
-						     "clk-m-dcephy-impctrl",
-						     ""; /* Unused */
-			};
-
-			clk_m_a2_div3: clk-m-a2-div3 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf3",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a2_osc_prediv>,
-					 <&clk_m_a2_pll0 3>, /* PLL0 PHI3 */
-					 <&clk_m_a2_pll1 3>; /* PLL1 PHI3 */
-
-				clock-output-names = ""; /* Unused */
-						/* Remaining outputs unused */
-			};
-		};
-
-		/*
-		 * A9 PLL
-		 */
-		clockgen-a9@fdde00d8 {
-			reg = <0xfdde00d8 0x70>;
-
-			clockgen_a9_pll: clockgen-a9-pll {
-				#clock-cells = <1>;
-				compatible = "st,stih415-plls-c32-a9", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-				clock-output-names = "clockgen-a9-pll-odf";
-			};
-		};
-
-		/*
-		 * ARM CPU related clocks
-		 */
-		clk_m_a9: clk-m-a9@fdde00d8 {
-			#clock-cells = <0>;
-			compatible = "st,stih415-clkgen-a9-mux", "st,clkgen-mux";
-			reg = <0xfdde00d8 0x4>;
-			clocks = <&clockgen_a9_pll 0>,
-				 <&clockgen_a9_pll 0>,
-				 <&clk_m_a0_div1 2>,
-				 <&clk_m_a9_ext2f_div2>;
-		};
-
-		/*
-		 * ARM Peripheral clock for timers
-		 */
-		arm_periph_clk: clk-m-a9-periphs {
-			#clock-cells = <0>;
-			compatible = "fixed-factor-clock";
-			clocks = <&clk_m_a9>;
-			clock-div = <2>;
-			clock-mult = <1>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/stih415-pinctrl.dtsi b/arch/arm/boot/dts/stih415-pinctrl.dtsi
deleted file mode 100644
index bd028ce98b61..000000000000
--- a/arch/arm/boot/dts/stih415-pinctrl.dtsi
+++ /dev/null
@@ -1,545 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics (R&D) Limited.
- * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-#include "st-pincfg.h"
-#include <dt-bindings/interrupt-controller/arm-gic.h>
-/ {
-
-	aliases {
-		gpio0	= &pio0;
-		gpio1	= &pio1;
-		gpio2	= &pio2;
-		gpio3	= &pio3;
-		gpio4	= &pio4;
-		gpio5	= &pio5;
-		gpio6	= &pio6;
-		gpio7	= &pio7;
-		gpio8	= &pio8;
-		gpio9	= &pio9;
-		gpio10	= &pio10;
-		gpio11	= &pio11;
-		gpio12	= &pio12;
-		gpio13	= &pio13;
-		gpio14	= &pio14;
-		gpio15	= &pio15;
-		gpio16	= &pio16;
-		gpio17	= &pio17;
-		gpio18	= &pio18;
-		gpio19	= &pio100;
-		gpio20	= &pio101;
-		gpio21	= &pio102;
-		gpio22	= &pio103;
-		gpio23	= &pio104;
-		gpio24	= &pio105;
-		gpio25	= &pio106;
-		gpio26	= &pio107;
-	};
-
-	soc {
-		pin-controller-sbc {
-			#address-cells	= <1>;
-			#size-cells	= <1>;
-			compatible	= "st,stih415-sbc-pinctrl";
-			st,syscfg	= <&syscfg_sbc>;
-			reg 		= <0xfe61f080 0x4>;
-			reg-names	= "irqmux";
-			interrupts 	= <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names	= "irqmux";
-			ranges 		= <0 0xfe610000 0x5000>;
-
-			pio0: gpio@fe610000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0 0x100>;
-				st,bank-name	= "PIO0";
-			};
-			pio1: gpio@fe611000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x1000 0x100>;
-				st,bank-name	= "PIO1";
-			};
-			pio2: gpio@fe612000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x2000 0x100>;
-				st,bank-name	= "PIO2";
-			};
-			pio3: gpio@fe613000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x3000 0x100>;
-				st,bank-name	= "PIO3";
-			};
-			pio4: gpio@fe614000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x4000 0x100>;
-				st,bank-name	= "PIO4";
-			};
-
-			sbc_serial1 {
-				pinctrl_sbc_serial1:sbc_serial1 {
-					st,pins {
-						tx	= <&pio2 6 ALT3 OUT>;
-						rx	= <&pio2 7 ALT3 IN>;
-					};
-				};
-			};
-
-			keyscan {
-				pinctrl_keyscan: keyscan {
-					st,pins {
-						keyin0 = <&pio0 2 ALT2 IN>;
-						keyin1 = <&pio0 3 ALT2 IN>;
-						keyin2 = <&pio0 4 ALT2 IN>;
-						keyin3 = <&pio2 6 ALT2 IN>;
-
-						keyout0 = <&pio1 6 ALT2 OUT>;
-						keyout1 = <&pio1 7 ALT2 OUT>;
-						keyout2 = <&pio0 6 ALT2 OUT>;
-						keyout3 = <&pio2 7 ALT2 OUT>;
-					};
-				};
-			};
-
-			sbc_i2c0 {
-				pinctrl_sbc_i2c0_default: sbc_i2c0-default {
-					st,pins {
-						sda = <&pio4 6 ALT1 BIDIR>;
-						scl = <&pio4 5 ALT1 BIDIR>;
-					};
-				};
-			};
-
-			sbc_i2c1 {
-				pinctrl_sbc_i2c1_default: sbc_i2c1-default {
-					st,pins {
-						sda = <&pio3 2 ALT2 BIDIR>;
-						scl = <&pio3 1 ALT2 BIDIR>;
-					};
-				};
-			};
-
-			rc{
-				pinctrl_ir: ir0 {
-					st,pins {
-						ir = <&pio4 0 ALT2 IN>;
-					};
-				};
-			};
-
-			gmac1 {
-				pinctrl_mii1: mii1 {
-						st,pins {
-						 txd0   = <&pio0 0 ALT1 OUT  SE_NICLK_IO	0	CLK_A>;
-						 txd1   = <&pio0 1 ALT1 OUT  SE_NICLK_IO	0	CLK_A>;
-						 txd2   = <&pio0 2 ALT1 OUT  SE_NICLK_IO	0	CLK_A>;
-						 txd3   = <&pio0 3 ALT1 OUT  SE_NICLK_IO	0	CLK_A>;
-						 txer   = <&pio0 4 ALT1 OUT  SE_NICLK_IO	0	CLK_A>;
-						 txen   = <&pio0 5 ALT1 OUT  SE_NICLK_IO	0	CLK_A>;
-						 txclk  = <&pio0 6 ALT1 IN   NICLK	0	CLK_A>;
-						 col    = <&pio0 7 ALT1 IN   BYPASS	1000>;
-						 mdio   = <&pio1 0 ALT1 OUT  BYPASS	0>;
-						 mdc    = <&pio1 1 ALT1 OUT  NICLK	0	CLK_A>;
-						 crs    = <&pio1 2 ALT1 IN   BYPASS	1000>;
-						 mdint  = <&pio1 3 ALT1 IN   BYPASS	0>;
-						 rxd0   = <&pio1 4 ALT1 IN   SE_NICLK_IO	0	CLK_A>;
-						 rxd1   = <&pio1 5 ALT1 IN   SE_NICLK_IO	0	CLK_A>;
-						 rxd2   = <&pio1 6 ALT1 IN   SE_NICLK_IO	0	CLK_A>;
-						 rxd3   = <&pio1 7 ALT1 IN   SE_NICLK_IO	0	CLK_A>;
-						 rxdv   = <&pio2 0 ALT1 IN   SE_NICLK_IO	0	CLK_A>;
-						 rx_er  = <&pio2 1 ALT1 IN   SE_NICLK_IO	0	CLK_A>;
-						 rxclk  = <&pio2 2 ALT1 IN   NICLK	0	CLK_A>;
-						 phyclk = <&pio2 3 ALT1 IN   NICLK	1000	CLK_A>;
-					};
-				};
-
-				pinctrl_rgmii1: rgmii1-0 {
-					st,pins {
-						 txd0 =	 <&pio0 0 ALT1 OUT DE_IO	1000	CLK_A>;
-						 txd1 =	 <&pio0 1 ALT1 OUT DE_IO	1000	CLK_A>;
-						 txd2 =	 <&pio0 2 ALT1 OUT DE_IO	1000	CLK_A>;
-						 txd3 =	 <&pio0 3 ALT1 OUT DE_IO	1000	CLK_A>;
-						 txen =	 <&pio0 5 ALT1 OUT DE_IO	0	CLK_A>;
-						 txclk = <&pio0 6 ALT1 IN	NICLK	0	CLK_A>;
-						 mdio =	 <&pio1 0 ALT1 OUT	BYPASS	0>;
-						 mdc =	 <&pio1 1 ALT1 OUT	NICLK	0	CLK_A>;
-						 rxd0 =	 <&pio1 4 ALT1 IN DE_IO	0	CLK_A>;
-						 rxd1 =	 <&pio1 5 ALT1 IN DE_IO	0	CLK_A>;
-						 rxd2 =	 <&pio1 6 ALT1 IN DE_IO	0	CLK_A>;
-						 rxd3 =	 <&pio1 7 ALT1 IN DE_IO	0	CLK_A>;
-
-						 rxdv =	  <&pio2 0 ALT1 IN DE_IO	500	CLK_A>;
-						 rxclk =  <&pio2 2 ALT1 IN	NICLK	0	CLK_A>;
-						 phyclk = <&pio2 3 ALT4 OUT	NICLK	0	CLK_B>;
-
-						 clk125= <&pio3 7 ALT4 IN 	NICLK	0	CLK_A>;
-					};
-				};
-			};
-		};
-
-		pin-controller-front {
-			#address-cells	= <1>;
-			#size-cells	= <1>;
-			compatible	= "st,stih415-front-pinctrl";
-			st,syscfg	= <&syscfg_front>;
-			reg 		= <0xfee0f080 0x4>;
-			reg-names	= "irqmux";
-			interrupts 	= <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names	= "irqmux";
-			ranges		= <0 0xfee00000 0x8000>;
-
-			pio5: gpio@fee00000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0 0x100>;
-				st,bank-name	= "PIO5";
-			};
-			pio6: gpio@fee01000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x1000 0x100>;
-				st,bank-name	= "PIO6";
-			};
-			pio7: gpio@fee02000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x2000 0x100>;
-				st,bank-name	= "PIO7";
-			};
-			pio8: gpio@fee03000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x3000 0x100>;
-				st,bank-name	= "PIO8";
-			};
-			pio9: gpio@fee04000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x4000 0x100>;
-				st,bank-name	= "PIO9";
-			};
-			pio10: gpio@fee05000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x5000 0x100>;
-				st,bank-name	= "PIO10";
-			};
-			pio11: gpio@fee06000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x6000 0x100>;
-				st,bank-name	= "PIO11";
-			};
-			pio12: gpio@fee07000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x7000 0x100>;
-				st,bank-name	= "PIO12";
-			};
-
-			i2c0 {
-				pinctrl_i2c0_default: i2c0-default {
-					st,pins {
-						sda = <&pio9 3 ALT1 BIDIR>;
-						scl = <&pio9 2 ALT1 BIDIR>;
-					};
-				};
-			};
-
-			i2c1 {
-				pinctrl_i2c1_default: i2c1-default {
-					st,pins {
-						sda = <&pio12 1 ALT1 BIDIR>;
-						scl = <&pio12 0 ALT1 BIDIR>;
-					};
-				};
-			};
-		};
-
-		pin-controller-rear {
-			#address-cells	= <1>;
-			#size-cells	= <1>;
-			compatible	= "st,stih415-rear-pinctrl";
-			st,syscfg	= <&syscfg_rear>;
-			reg 		= <0xfe82f080 0x4>;
-			reg-names	= "irqmux";
-			interrupts 	= <GIC_SPI 182 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names	= "irqmux";
-			ranges		= <0 0xfe820000 0x8000>;
-
-			pio13: gpio@fe820000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0 0x100>;
-				st,bank-name	= "PIO13";
-			};
-			pio14: gpio@fe821000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x1000 0x100>;
-				st,bank-name	= "PIO14";
-			};
-			pio15: gpio@fe822000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x2000 0x100>;
-				st,bank-name	= "PIO15";
-			};
-			pio16: gpio@fe823000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x3000 0x100>;
-				st,bank-name	= "PIO16";
-			};
-			pio17: gpio@fe824000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x4000 0x100>;
-				st,bank-name	= "PIO17";
-			};
-			pio18: gpio@fe825000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x5000 0x100>;
-				st,bank-name	= "PIO18";
-			};
-
-			serial2 {
-				pinctrl_serial2: serial2-0 {
-					st,pins {
-						tx	= <&pio17 4 ALT2 OUT>;
-						rx	= <&pio17 5 ALT2 IN>;
-					};
-				};
-			};
-
-			gmac0{
-				pinctrl_mii0: mii0 {
-					st,pins {
-					 mdint =	<&pio13 6 ALT2	IN	BYPASS		0>;
-					 txen =		<&pio13 7 ALT2	OUT	SE_NICLK_IO	0	CLK_A>;
-
-					 txd0 =		<&pio14 0 ALT2	OUT	SE_NICLK_IO	0	CLK_A>;
-					 txd1 =		<&pio14 1 ALT2	OUT	SE_NICLK_IO	0	CLK_A>;
-					 txd2 =		<&pio14 2 ALT2	OUT	SE_NICLK_IO	0	CLK_B>;
-					 txd3 =		<&pio14 3 ALT2	OUT	SE_NICLK_IO	0	CLK_B>;
-
-					 txclk =	<&pio15 0 ALT2	IN	NICLK		0	CLK_A>;
-					 txer =		<&pio15 1 ALT2	OUT	SE_NICLK_IO	0	CLK_A>;
-					 crs =		<&pio15 2 ALT2	IN	BYPASS		1000>;
-					 col =		<&pio15 3 ALT2	IN	BYPASS		1000>;
-					 mdio  =        <&pio15 4 ALT2	OUT	BYPASS 	3000>;
-					 mdc   =        <&pio15 5 ALT2	OUT     NICLK  	0    	CLK_B>;
-
-					 rxd0 =		<&pio16 0 ALT2	IN	SE_NICLK_IO	0	CLK_A>;
-					 rxd1 =		<&pio16 1 ALT2	IN	SE_NICLK_IO	0	CLK_A>;
-					 rxd2 =		<&pio16 2 ALT2	IN	SE_NICLK_IO	0	CLK_A>;
-					 rxd3 =		<&pio16 3 ALT2	IN	SE_NICLK_IO	0	CLK_A>;
-					 rxdv =		<&pio15 6 ALT2	IN	SE_NICLK_IO	0	CLK_A>;
-					 rx_er =	<&pio15 7 ALT2	IN	SE_NICLK_IO	0	CLK_A>;
-					 rxclk =	<&pio17 0 ALT2	IN	NICLK		0	CLK_A>;
-					 phyclk =	<&pio13 5 ALT2	OUT	NICLK	1000	CLK_A>;
-
-					};
-				};
-
-			pinctrl_gmii0: gmii0 {
-				st,pins {
-					 mdint =	<&pio13 6	ALT2 IN		BYPASS	0>;
-					 mdio  =        <&pio15 4 	ALT2 OUT	BYPASS 	3000>;
-					 mdc   =        <&pio15 5 	ALT2 OUT    	NICLK  	0    	CLK_B>;
-					 txen =		<&pio13 7	ALT2 OUT	SE_NICLK_IO	3000	CLK_A>;
-
-					 txd0 =		<&pio14 0	ALT2 OUT	SE_NICLK_IO	3000	CLK_A>;
-					 txd1 =		<&pio14 1	ALT2 OUT	SE_NICLK_IO	3000	CLK_A>;
-					 txd2 =		<&pio14 2	ALT2 OUT	SE_NICLK_IO	3000	CLK_B>;
-					 txd3 =		<&pio14 3	ALT2 OUT	SE_NICLK_IO	3000	CLK_B>;
-					 txd4 =		<&pio14 4	ALT2 OUT	SE_NICLK_IO	3000	CLK_B>;
-					 txd5 =		<&pio14 5	ALT2 OUT	SE_NICLK_IO	3000	CLK_B>;
-					 txd6 =		<&pio14 6	ALT2 OUT	SE_NICLK_IO	3000	CLK_B>;
-					 txd7 =		<&pio14 7	ALT2 OUT	SE_NICLK_IO	3000	CLK_B>;
-
-					 txclk =	<&pio15 0	ALT2 IN		NICLK	0	CLK_A>;
-					 txer =		<&pio15 1	ALT2 OUT 	SE_NICLK_IO	3000	CLK_A>;
-					 crs =		<&pio15 2	ALT2 IN		BYPASS	1000>;
-					 col =		<&pio15 3	ALT2 IN		BYPASS	1000>;
-					 rxdv =		<&pio15 6	ALT2 IN		SE_NICLK_IO	1500	CLK_A>;
-					 rx_er =	<&pio15 7	ALT2 IN		SE_NICLK_IO	1500	CLK_A>;
-
-					 rxd0 =		<&pio16 0	ALT2 IN		SE_NICLK_IO	1500	CLK_A>;
-					 rxd1 =		<&pio16 1	ALT2 IN		SE_NICLK_IO	1500	CLK_A>;
-					 rxd2 =		<&pio16 2	ALT2 IN		SE_NICLK_IO	1500	CLK_A>;
-					 rxd3 =		<&pio16 3	ALT2 IN		SE_NICLK_IO	1500	CLK_A>;
-					 rxd4 =		<&pio16 4	ALT2 IN		SE_NICLK_IO	1500	CLK_A>;
-					 rxd5 =		<&pio16 5	ALT2 IN		SE_NICLK_IO	1500	CLK_A>;
-					 rxd6 =		<&pio16 6	ALT2 IN		SE_NICLK_IO	1500	CLK_A>;
-					 rxd7 =		<&pio16 7	ALT2 IN		SE_NICLK_IO	1500	CLK_A>;
-
-					 rxclk =	<&pio17 0	ALT2 IN	NICLK	0	CLK_A>;
-					 clk125 =	<&pio17 6	ALT1 IN	NICLK	0	CLK_A>;
-                                         phyclk =       <&pio13 5       ALT4 OUT NICLK   0       CLK_B>;
-
-
-					};
-				};
-			};
-
-			mmc0 {
-				pinctrl_mmc0: mmc0 {
-					st,pins {
-						mmcclk = <&pio13 4 ALT4 BIDIR_PU NICLK 0 CLK_B>;
-						data0  = <&pio14 4 ALT4 BIDIR_PU BYPASS 0>;
-						data1  = <&pio14 5 ALT4 BIDIR_PU BYPASS 0>;
-						data2  = <&pio14 6 ALT4 BIDIR_PU BYPASS 0>;
-						data3  = <&pio14 7 ALT4 BIDIR_PU BYPASS 0>;
-						cmd    = <&pio15 1 ALT4 BIDIR_PU BYPASS 0>;
-						wp     = <&pio15 3 ALT4 IN>;
-						data4  = <&pio16 4 ALT4 BIDIR_PU BYPASS 0>;
-						data5  = <&pio16 5 ALT4 BIDIR_PU BYPASS 0>;
-						data6  = <&pio16 6 ALT4 BIDIR_PU BYPASS 0>;
-						data7  = <&pio16 7 ALT4 BIDIR_PU BYPASS 0>;
-						pwr    = <&pio17 1 ALT4 OUT>;
-						cd     = <&pio17 2 ALT4 IN>;
-						led    = <&pio17 3 ALT4 OUT>;
-					};
-				};
-			};
-		};
-
-		pin-controller-left {
-			#address-cells	= <1>;
-			#size-cells	= <1>;
-			compatible	= "st,stih415-left-pinctrl";
-			st,syscfg	= <&syscfg_left>;
-			reg 		= <0xfd6bf080 0x4>;
-			reg-names	= "irqmux";
-			interrupts 	= <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names	= "irqmux";
-			ranges		= <0 0xfd6b0000 0x3000>;
-
-			pio100: gpio@fd6b0000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0 0x100>;
-				st,bank-name	= "PIO100";
-			};
-			pio101: gpio@fd6b1000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x1000 0x100>;
-				st,bank-name	= "PIO101";
-			};
-			pio102: gpio@fd6b2000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x2000 0x100>;
-				st,bank-name	= "PIO102";
-			};
-		};
-
-		pin-controller-right {
-			#address-cells	= <1>;
-			#size-cells	= <1>;
-			compatible	= "st,stih415-right-pinctrl";
-			st,syscfg	= <&syscfg_right>;
-			reg 		= <0xfd33f080 0x4>;
-			reg-names	= "irqmux";
-			interrupts 	= <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names	= "irqmux";
-			ranges		= <0 0xfd330000 0x5000>;
-
-			pio103: gpio@fd330000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0 0x100>;
-				st,bank-name	= "PIO103";
-			};
-			pio104: gpio@fd331000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x1000 0x100>;
-				st,bank-name	= "PIO104";
-			};
-			pio105: gpio@fd332000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x2000 0x100>;
-				st,bank-name	= "PIO105";
-			};
-			pio106: gpio@fd333000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x3000 0x100>;
-				st,bank-name	= "PIO106";
-			};
-			pio107: gpio@fd334000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x4000 0x100>;
-				st,bank-name	= "PIO107";
-			};
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/stih415.dtsi b/arch/arm/boot/dts/stih415.dtsi
deleted file mode 100644
index 12427e651e5e..000000000000
--- a/arch/arm/boot/dts/stih415.dtsi
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics (R&D) Limited.
- * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-#include "stih41x.dtsi"
-#include "stih415-clock.dtsi"
-#include "stih415-pinctrl.dtsi"
-#include <dt-bindings/interrupt-controller/arm-gic.h>
-#include <dt-bindings/reset/stih415-resets.h>
-/ {
-
-	L2: cache-controller {
-		compatible = "arm,pl310-cache";
-		reg = <0xfffe2000 0x1000>;
-		arm,data-latency = <3 2 2>;
-		arm,tag-latency = <1 1 1>;
-		cache-unified;
-		cache-level = <2>;
-	};
-
-	soc {
-		#address-cells = <1>;
-		#size-cells = <1>;
-		interrupt-parent = <&intc>;
-		ranges;
-		compatible	= "simple-bus";
-
-		powerdown: powerdown-controller {
-			#reset-cells = <1>;
-			compatible = "st,stih415-powerdown";
-		};
-
-		softreset: softreset-controller {
-			#reset-cells = <1>;
-			compatible = "st,stih415-softreset";
-		};
-
-		syscfg_sbc: sbc-syscfg@fe600000{
-			compatible      = "st,stih415-sbc-syscfg", "syscon";
-			reg		= <0xfe600000 0xb4>;
-		};
-
-		syscfg_front: front-syscfg@fee10000{
-			compatible      = "st,stih415-front-syscfg", "syscon";
-			reg		= <0xfee10000 0x194>;
-		};
-
-		syscfg_rear: rear-syscfg@fe830000{
-			compatible      = "st,stih415-rear-syscfg", "syscon";
-			reg		= <0xfe830000 0x190>;
-		};
-
-		/* MPE syscfgs */
-		syscfg_left: left-syscfg@fd690000{
-			compatible      = "st,stih415-left-syscfg", "syscon";
-			reg		= <0xfd690000 0x78>;
-		};
-
-		syscfg_right: right-syscfg@fd320000{
-			compatible      = "st,stih415-right-syscfg", "syscon";
-			reg		= <0xfd320000 0x180>;
-		};
-
-		syscfg_system: system-syscfg@fdde0000  {
-			compatible      = "st,stih415-system-syscfg", "syscon";
-			reg		= <0xfdde0000 0x15c>;
-		};
-
-		syscfg_lpm: lpm-syscfg@fe4b5100{
-			compatible      = "st,stih415-lpm-syscfg", "syscon";
-			reg		= <0xfe4b5100 0x08>;
-		};
-
-		serial2: serial@fed32000 {
-			compatible	= "st,asc";
-			status 		= "disabled";
-			reg		= <0xfed32000 0x2c>;
-			interrupts	= <0 197 0>;
-			pinctrl-names 	= "default";
-			pinctrl-0 	= <&pinctrl_serial2>;
-			clocks		= <&clk_s_a0_ls CLK_ICN_REG>;
-		};
-
-		/* SBC comms block ASCs in SASG1 */
-		sbc_serial1: serial@fe531000 {
-			compatible	= "st,asc";
-			status 		= "disabled";
-			reg		= <0xfe531000 0x2c>;
-			interrupts	= <0 210 0>;
-			clocks		= <&clk_sysin>;
-			pinctrl-names 	= "default";
-			pinctrl-0	= <&pinctrl_sbc_serial1>;
-		};
-
-		i2c@fed40000 {
-			compatible	= "st,comms-ssc4-i2c";
-			reg		= <0xfed40000 0x110>;
-			interrupts	= <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>;
-			clocks		= <&clk_s_a0_ls CLK_ICN_REG>;
-			clock-names	= "ssc";
-			clock-frequency = <400000>;
-			pinctrl-names	= "default";
-			pinctrl-0	= <&pinctrl_i2c0_default>;
-
-			status		= "disabled";
-		};
-
-		i2c@fed41000 {
-			compatible	= "st,comms-ssc4-i2c";
-			reg		= <0xfed41000 0x110>;
-			interrupts	= <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
-			clocks		= <&clk_s_a0_ls CLK_ICN_REG>;
-			clock-names	= "ssc";
-			clock-frequency = <400000>;
-			pinctrl-names	= "default";
-			pinctrl-0	= <&pinctrl_i2c1_default>;
-
-			status		= "disabled";
-		};
-
-		i2c@fe540000 {
-			compatible	= "st,comms-ssc4-i2c";
-			reg		= <0xfe540000 0x110>;
-			interrupts	= <GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH>;
-			clocks		= <&clk_sysin>;
-			clock-names	= "ssc";
-			clock-frequency = <400000>;
-			pinctrl-names	= "default";
-			pinctrl-0	= <&pinctrl_sbc_i2c0_default>;
-
-			status		= "disabled";
-		};
-
-		i2c@fe541000 {
-			compatible	= "st,comms-ssc4-i2c";
-			reg		= <0xfe541000 0x110>;
-			interrupts	= <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>;
-			clocks		= <&clk_sysin>;
-			clock-names	= "ssc";
-			clock-frequency = <400000>;
-			pinctrl-names	= "default";
-			pinctrl-0	= <&pinctrl_sbc_i2c1_default>;
-
-			status		= "disabled";
-		};
-
-		ethernet0: dwmac@fe810000 {
-			device_type 	= "network";
-			compatible	= "st,stih415-dwmac", "snps,dwmac", "snps,dwmac-3.610";
-			status 		= "disabled";
-
-			reg		= <0xfe810000 0x8000>;
-			reg-names	= "stmmaceth";
-
-			interrupts 	= <0 147 0>, <0 148 0>, <0 149 0>;
-			interrupt-names = "macirq", "eth_wake_irq", "eth_lpi";
-			resets			= <&softreset STIH415_ETH0_SOFTRESET>;
-			reset-names		= "stmmaceth";
-
-			snps,pbl 	= <32>;
-			snps,mixed-burst;
-			snps,force_sf_dma_mode;
-
-			st,syscon	= <&syscfg_rear 0x148>;
-
-			pinctrl-names 	= "default";
-			pinctrl-0	= <&pinctrl_mii0>;
-			clock-names	= "stmmaceth", "sti-ethclk";
-			clocks		= <&clk_s_a1_ls CLK_ICN_IF_2>, <&clk_s_a1_ls CLK_GMAC0_PHY>;
-		};
-
-		ethernet1: dwmac@fef08000 {
-			device_type = "network";
-			compatible	= "st,stih415-dwmac", "snps,dwmac", "snps,dwmac-3.610";
-			status 		= "disabled";
-			reg		= <0xfef08000 0x8000>;
-			reg-names	= "stmmaceth";
-			interrupts 	= <0 150 0>, <0 151 0>, <0 152 0>;
-			interrupt-names = "macirq", "eth_wake_irq", "eth_lpi";
-
-			snps,pbl	= <32>;
-			snps,mixed-burst;
-			snps,force_sf_dma_mode;
-
-			st,syscon		= <&syscfg_sbc 0x74>;
-
-			resets			= <&softreset STIH415_ETH1_SOFTRESET>;
-			reset-names		= "stmmaceth";
-			pinctrl-names 	= "default";
-			pinctrl-0	= <&pinctrl_mii1>;
-			clock-names	= "stmmaceth", "sti-ethclk";
-			clocks		= <&clk_s_a0_ls CLK_ICN_REG>, <&clk_s_a0_ls CLK_ETH1_PHY>;
-		};
-
-		rc: rc@fe518000 {
-			compatible	= "st,comms-irb";
-			reg		= <0xfe518000 0x234>;
-			interrupts	=  <0 203 0>;
-			clocks		= <&clk_sysin>;
-			rx-mode		= "infrared";
-			pinctrl-names 	= "default";
-			pinctrl-0	= <&pinctrl_ir>;
-			resets		= <&softreset STIH415_IRB_SOFTRESET>;
-		};
-
-		keyscan: keyscan@fe4b0000 {
-			compatible = "st,sti-keyscan";
-			status = "disabled";
-			reg = <0xfe4b0000 0x2000>;
-			interrupts = <GIC_SPI 212 IRQ_TYPE_NONE>;
-			clocks = <&clk_sysin>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_keyscan>;
-			resets	= <&powerdown STIH415_KEYSCAN_POWERDOWN>,
-				  <&softreset STIH415_KEYSCAN_SOFTRESET>;
-		};
-
-		mmc0: sdhci@fe81e000 {
-			compatible      = "st,sdhci";
-			status          = "disabled";
-			reg             = <0xfe81e000 0x1000>;
-			interrupts      = <GIC_SPI 145 IRQ_TYPE_NONE>;
-			interrupt-names = "mmcirq";
-			pinctrl-names   = "default";
-			pinctrl-0       = <&pinctrl_mmc0>;
-			clock-names     = "mmc";
-			clocks          = <&clk_s_a1_ls 1>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/stih416-b2000.dts b/arch/arm/boot/dts/stih416-b2000.dts
deleted file mode 100644
index 488e80a5d69d..000000000000
--- a/arch/arm/boot/dts/stih416-b2000.dts
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics (R&D) Limited.
- * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-/dts-v1/;
-#include "stih416.dtsi"
-#include "stih41x-b2000.dtsi"
-/ {
-	model = "STiH416 B2000";
-	compatible = "st,stih416-b2000", "st,stih416";
-};
diff --git a/arch/arm/boot/dts/stih416-b2020.dts b/arch/arm/boot/dts/stih416-b2020.dts
deleted file mode 100644
index 200a81844765..000000000000
--- a/arch/arm/boot/dts/stih416-b2020.dts
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics (R&D) Limited.
- * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-/dts-v1/;
-#include "stih416.dtsi"
-#include "stih41x-b2020.dtsi"
-/ {
-	model = "STiH416 B2020";
-	compatible = "st,stih416-b2020", "st,stih416";
-
-	soc {
-		mmc1: sdhci@fe81f000 {
-			status       = "okay";
-			bus-width    = <8>;
-			non-removable;
-		};
-
-		miphy365x_phy: phy@fe382000 {
-			phy_port0: port@fe382000 {
-				st,sata-gen = <3>;
-			};
-
-			phy_port1: port@fe38a000 {
-				st,pcie-tx-pol-inv;
-			};
-		};
-
-		sata0: sata@fe380000{
-			status = "okay";
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/stih416-b2020e.dts b/arch/arm/boot/dts/stih416-b2020e.dts
deleted file mode 100644
index de320cd067de..000000000000
--- a/arch/arm/boot/dts/stih416-b2020e.dts
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (C) 2014 STMicroelectronics (R&D) Limited.
- * Author: Lee Jones <lee.jones@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-/dts-v1/;
-#include "stih416.dtsi"
-#include "stih41x-b2020.dtsi"
-#include <dt-bindings/gpio/gpio.h>
-/ {
-	model = "STiH416 B2020 REV-E";
-	compatible = "st,stih416-b2020", "st,stih416";
-
-	soc {
-		leds {
-			compatible = "gpio-leds";
-			red {
-				label			= "Front Panel LED";
-				gpios			= <&pio4 1 GPIO_ACTIVE_HIGH>;
-				linux,default-trigger	= "heartbeat";
-			};
-			green {
-				gpios			= <&pio1 3 GPIO_ACTIVE_HIGH>;
-				default-state 		= "off";
-			};
-		};
-
-		ethernet1: dwmac@fef08000 {
-			snps,reset-gpio = <&pio0 7>;
-		};
-
-		mmc1: sdhci@fe81f000 {
-			status       = "okay";
-			bus-width    = <8>;
-			non-removable;
-		};
-
-		miphy365x_phy: phy@fe382000 {
-			phy_port0: port@fe382000 {
-				st,sata-gen = <3>;
-			};
-
-			phy_port1: port@fe38a000 {
-				st,pcie-tx-pol-inv;
-			};
-		};
-
-		sata0: sata@fe380000{
-			status = "okay";
-		};
-
-		/* SAS PWM Module */
-		pwm0: pwm@fed10000 {
-			status		= "okay";
-		};
-
-		/* SBC PWM Module */
-		pwm1: pwm@fe510000 {
-			status		= "okay";
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/stih416-clock.dtsi b/arch/arm/boot/dts/stih416-clock.dtsi
deleted file mode 100644
index 5b4fb838cddb..000000000000
--- a/arch/arm/boot/dts/stih416-clock.dtsi
+++ /dev/null
@@ -1,756 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics R&D Limited
- * <stlinux-devel@stlinux.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <dt-bindings/clock/stih416-clks.h>
-
-/ {
-	clocks {
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges;
-
-		/*
-		 * Fixed 30MHz oscillator inputs to SoC
-		 */
-		clk_sysin: clk-sysin {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-			clock-frequency = <30000000>;
-		};
-
-		/*
-		 * ClockGenAs on SASG2
-		 */
-		clockgen-a@fee62000 {
-			reg = <0xfee62000 0xb48>;
-
-			clk_s_a0_pll: clk-s-a0-pll {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-plls-c65";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-s-a0-pll0-hs",
-						     "clk-s-a0-pll0-ls",
-						     "clk-s-a0-pll1";
-			};
-
-			clk_s_a0_osc_prediv: clk-s-a0-osc-prediv {
-				#clock-cells = <0>;
-				compatible = "st,clkgena-prediv-c65",
-					     "st,clkgena-prediv";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-s-a0-osc-prediv";
-			};
-
-			clk_s_a0_hs: clk-s-a0-hs {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c65-hs",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_s_a0_osc_prediv>,
-					 <&clk_s_a0_pll 0>, /* PLL0 HS */
-					 <&clk_s_a0_pll 2>; /* PLL1 */
-
-				clock-output-names = "clk-s-fdma-0",
-						     "clk-s-fdma-1",
-						     ""; /* clk-s-jit-sense */
-						     /* Fourth output unused */
-			};
-
-			clk_s_a0_ls: clk-s-a0-ls {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c65-ls",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_s_a0_osc_prediv>,
-					 <&clk_s_a0_pll 1>, /* PLL0 LS */
-					 <&clk_s_a0_pll 2>; /* PLL1 */
-
-				clock-output-names = "clk-s-icn-reg-0",
-						     "clk-s-icn-if-0",
-						     "clk-s-icn-reg-lp-0",
-						     "clk-s-emiss",
-						     "clk-s-eth1-phy",
-						     "clk-s-mii-ref-out";
-						     /* Remaining outputs unused */
-			};
-		};
-
-		clockgen-a@fee81000 {
-			reg = <0xfee81000 0xb48>;
-
-			clk_s_a1_pll: clk-s-a1-pll {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-plls-c65";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-s-a1-pll0-hs",
-						     "clk-s-a1-pll0-ls",
-						     "clk-s-a1-pll1";
-			};
-
-			clk_s_a1_osc_prediv: clk-s-a1-osc-prediv {
-				#clock-cells = <0>;
-				compatible = "st,clkgena-prediv-c65",
-					     "st,clkgena-prediv";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-s-a1-osc-prediv";
-			};
-
-			clk_s_a1_hs: clk-s-a1-hs {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c65-hs",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_s_a1_osc_prediv>,
-					 <&clk_s_a1_pll 0>, /* PLL0 HS */
-					 <&clk_s_a1_pll 2>; /* PLL1 */
-
-				clock-output-names = "", /* Reserved */
-						     "", /* Reserved */
-						     "clk-s-stac-phy",
-						     "clk-s-vtac-tx-phy";
-			};
-
-			clk_s_a1_ls: clk-s-a1-ls {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c65-ls",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_s_a1_osc_prediv>,
-					 <&clk_s_a1_pll 1>, /* PLL0 LS */
-					 <&clk_s_a1_pll 2>; /* PLL1 */
-
-				clock-output-names = "clk-s-icn-if-2",
-						     "clk-s-card-mmc-0",
-						     "clk-s-icn-if-1",
-						     "clk-s-gmac0-phy",
-						     "clk-s-nand-ctrl",
-						     "", /* Reserved */
-						     "clk-s-mii0-ref-out",
-						     "clk-s-stac-sys",
-						     "clk-s-card-mmc-1";
-						     /* Remaining outputs unused */
-			};
-		};
-
-		/*
-		 * ClockGenAs on MPE42
-		 */
-		clockgen-a@fde12000 {
-			reg = <0xfde12000 0xb50>;
-
-			clk_m_a0_pll0: clk-m-a0-pll0 {
-				#clock-cells = <1>;
-				compatible = "st,plls-c32-a1x-0", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a0-pll0-phi0",
-						     "clk-m-a0-pll0-phi1",
-						     "clk-m-a0-pll0-phi2",
-						     "clk-m-a0-pll0-phi3";
-			};
-
-			clk_m_a0_pll1: clk-m-a0-pll1 {
-				#clock-cells = <1>;
-				compatible = "st,plls-c32-a1x-1", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a0-pll1-phi0",
-						     "clk-m-a0-pll1-phi1",
-						     "clk-m-a0-pll1-phi2",
-						     "clk-m-a0-pll1-phi3";
-			};
-
-			clk_m_a0_osc_prediv: clk-m-a0-osc-prediv {
-				#clock-cells = <0>;
-				compatible = "st,clkgena-prediv-c32",
-					     "st,clkgena-prediv";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a0-osc-prediv";
-			};
-
-			clk_m_a0_div0: clk-m-a0-div0 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf0",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a0_osc_prediv>,
-					 <&clk_m_a0_pll0 0>, /* PLL0 PHI0 */
-					 <&clk_m_a0_pll1 0>; /* PLL1 PHI0 */
-
-				clock-output-names = "", /* Unused */
-						     "", /* Unused */
-						     "clk-m-fdma-12",
-						     "", /* Unused */
-						     "clk-m-pp-dmu-0",
-						     "clk-m-pp-dmu-1",
-						     "clk-m-icm-lmi",
-						     "clk-m-vid-dmu-0";
-			};
-
-			clk_m_a0_div1: clk-m-a0-div1 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf1",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a0_osc_prediv>,
-					 <&clk_m_a0_pll0 1>, /* PLL0 PHI1 */
-					 <&clk_m_a0_pll1 1>; /* PLL1 PHI1 */
-
-				clock-output-names = "clk-m-vid-dmu-1",
-						     "", /* Unused */
-						     "clk-m-a9-ext2f",
-						     "clk-m-st40rt",
-						     "clk-m-st231-dmu-0",
-						     "clk-m-st231-dmu-1",
-						     "clk-m-st231-aud",
-						     "clk-m-st231-gp-0";
-			};
-
-			clk_m_a0_div2: clk-m-a0-div2 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf2",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a0_osc_prediv>,
-					 <&clk_m_a0_pll0 2>, /* PLL0 PHI2 */
-					 <&clk_m_a0_pll1 2>; /* PLL1 PHI2 */
-
-				clock-output-names = "clk-m-st231-gp-1",
-						     "clk-m-icn-cpu",
-						     "clk-m-icn-stac",
-						     "clk-m-tx-icn-dmu-0",
-						     "clk-m-tx-icn-dmu-1",
-						     "clk-m-tx-icn-ts",
-						     "clk-m-icn-vdp-0",
-						     "clk-m-icn-vdp-1";
-			};
-
-			clk_m_a0_div3: clk-m-a0-div3 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf3",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a0_osc_prediv>,
-					 <&clk_m_a0_pll0 3>, /* PLL0 PHI3 */
-					 <&clk_m_a0_pll1 3>; /* PLL1 PHI3 */
-
-				clock-output-names = "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "clk-m-icn-vp8",
-						     "", /* Unused */
-						     "clk-m-icn-reg-11",
-						     "clk-m-a9-trace";
-			};
-		};
-
-		clockgen-a@fd6db000 {
-			reg = <0xfd6db000 0xb50>;
-
-			clk_m_a1_pll0: clk-m-a1-pll0 {
-				#clock-cells = <1>;
-				compatible = "st,plls-c32-a1x-0", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a1-pll0-phi0",
-						     "clk-m-a1-pll0-phi1",
-						     "clk-m-a1-pll0-phi2",
-						     "clk-m-a1-pll0-phi3";
-			};
-
-			clk_m_a1_pll1: clk-m-a1-pll1 {
-				#clock-cells = <1>;
-				compatible = "st,plls-c32-a1x-1", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a1-pll1-phi0",
-						     "clk-m-a1-pll1-phi1",
-						     "clk-m-a1-pll1-phi2",
-						     "clk-m-a1-pll1-phi3";
-			};
-
-			clk_m_a1_osc_prediv: clk-m-a1-osc-prediv {
-				#clock-cells = <0>;
-				compatible = "st,clkgena-prediv-c32",
-					     "st,clkgena-prediv";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a1-osc-prediv";
-			};
-
-			clk_m_a1_div0: clk-m-a1-div0 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf0",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a1_osc_prediv>,
-					 <&clk_m_a1_pll0 0>, /* PLL0 PHI0 */
-					 <&clk_m_a1_pll1 0>; /* PLL1 PHI0 */
-
-				clock-output-names = "", /* Unused */
-						     "clk-m-fdma-10",
-						     "clk-m-fdma-11",
-						     "clk-m-hva-alt",
-						     "clk-m-proc-sc",
-						     "clk-m-tp",
-						     "clk-m-rx-icn-dmu-0",
-						     "clk-m-rx-icn-dmu-1";
-			};
-
-			clk_m_a1_div1: clk-m-a1-div1 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf1",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a1_osc_prediv>,
-					 <&clk_m_a1_pll0 1>, /* PLL0 PHI1 */
-					 <&clk_m_a1_pll1 1>; /* PLL1 PHI1 */
-
-				clock-output-names = "clk-m-rx-icn-ts",
-						     "clk-m-rx-icn-vdp-0",
-						     "", /* Unused */
-						     "clk-m-prv-t1-bus",
-						     "clk-m-icn-reg-12",
-						     "clk-m-icn-reg-10",
-						     "", /* Unused */
-						     "clk-m-icn-st231";
-			};
-
-			clk_m_a1_div2: clk-m-a1-div2 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf2",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a1_osc_prediv>,
-					 <&clk_m_a1_pll0 2>, /* PLL0 PHI2 */
-					 <&clk_m_a1_pll1 2>; /* PLL1 PHI2 */
-
-				clock-output-names = "clk-m-fvdp-proc-alt",
-						     "clk-m-icn-reg-13",
-						     "clk-m-tx-icn-gpu",
-						     "clk-m-rx-icn-gpu",
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* clk-m-apb-pm-12 */
-						     ""; /* Unused */
-			};
-
-			clk_m_a1_div3: clk-m-a1-div3 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf3",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a1_osc_prediv>,
-					 <&clk_m_a1_pll0 3>, /* PLL0 PHI3 */
-					 <&clk_m_a1_pll1 3>; /* PLL1 PHI3 */
-
-				clock-output-names = "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     "", /* Unused */
-						     ""; /* clk-m-gpu-alt */
-			};
-		};
-
-		clk_m_a9_ext2f_div2: clk-m-a9-ext2f-div2 {
-			#clock-cells = <0>;
-			compatible = "fixed-factor-clock";
-			clocks = <&clk_m_a0_div1 2>;
-			clock-div = <2>;
-			clock-mult = <1>;
-		};
-
-		clockgen-a@fd345000 {
-			reg = <0xfd345000 0xb50>;
-
-			clk_m_a2_pll0: clk-m-a2-pll0 {
-				#clock-cells = <1>;
-				compatible = "st,plls-c32-a1x-0", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a2-pll0-phi0",
-						     "clk-m-a2-pll0-phi1",
-						     "clk-m-a2-pll0-phi2",
-						     "clk-m-a2-pll0-phi3";
-			};
-
-			clk_m_a2_pll1: clk-m-a2-pll1 {
-				#clock-cells = <1>;
-				compatible = "st,plls-c32-a1x-1", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a2-pll1-phi0",
-						     "clk-m-a2-pll1-phi1",
-						     "clk-m-a2-pll1-phi2",
-						     "clk-m-a2-pll1-phi3";
-			};
-
-			clk_m_a2_osc_prediv: clk-m-a2-osc-prediv {
-				#clock-cells = <0>;
-				compatible = "st,clkgena-prediv-c32",
-					     "st,clkgena-prediv";
-
-				clocks = <&clk_sysin>;
-
-				clock-output-names = "clk-m-a2-osc-prediv";
-			};
-
-			clk_m_a2_div0: clk-m-a2-div0 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf0",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a2_osc_prediv>,
-					 <&clk_m_a2_pll0 0>, /* PLL0 PHI0 */
-					 <&clk_m_a2_pll1 0>; /* PLL1 PHI0 */
-
-				clock-output-names = "clk-m-vtac-main-phy",
-						     "clk-m-vtac-aux-phy",
-						     "clk-m-stac-phy",
-						     "clk-m-stac-sys",
-						     "", /* clk-m-mpestac-pg */
-						     "", /* clk-m-mpestac-wc */
-						     "", /* clk-m-mpevtacaux-pg*/
-						     ""; /* clk-m-mpevtacmain-pg*/
-			};
-
-			clk_m_a2_div1: clk-m-a2-div1 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf1",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a2_osc_prediv>,
-					 <&clk_m_a2_pll0 1>, /* PLL0 PHI1 */
-					 <&clk_m_a2_pll1 1>; /* PLL1 PHI1 */
-
-				clock-output-names = "", /* clk-m-mpevtacrx0-wc */
-						     "", /* clk-m-mpevtacrx1-wc */
-						     "clk-m-compo-main",
-						     "clk-m-compo-aux",
-						     "clk-m-bdisp-0",
-						     "clk-m-bdisp-1",
-						     "clk-m-icn-bdisp",
-						     "clk-m-icn-compo";
-			};
-
-			clk_m_a2_div2: clk-m-a2-div2 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf2",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a2_osc_prediv>,
-					 <&clk_m_a2_pll0 2>, /* PLL0 PHI2 */
-					 <&clk_m_a2_pll1 2>; /* PLL1 PHI2 */
-
-				clock-output-names = "clk-m-icn-vdp-2",
-						     "", /* Unused */
-						     "clk-m-icn-reg-14",
-						     "clk-m-mdtp",
-						     "clk-m-jpegdec",
-						     "", /* Unused */
-						     "clk-m-dcephy-impctrl",
-						     ""; /* Unused */
-			};
-
-			clk_m_a2_div3: clk-m-a2-div3 {
-				#clock-cells = <1>;
-				compatible = "st,clkgena-divmux-c32-odf3",
-					     "st,clkgena-divmux";
-
-				clocks = <&clk_m_a2_osc_prediv>,
-					 <&clk_m_a2_pll0 3>, /* PLL0 PHI3 */
-					 <&clk_m_a2_pll1 3>; /* PLL1 PHI3 */
-
-				clock-output-names = "", /* Unused */
-						     ""; /* clk-m-apb-pm-11 */
-						     /* Remaining outputs unused */
-			};
-		};
-
-		/*
-		 * A9 PLL
-		 */
-		clockgen-a9@fdde08b0 {
-			reg = <0xfdde08b0 0x70>;
-
-			clockgen_a9_pll: clockgen-a9-pll {
-				#clock-cells = <1>;
-				compatible = "st,stih416-plls-c32-a9", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-				clock-output-names = "clockgen-a9-pll-odf";
-			};
-		};
-
-		/*
-		 * ARM CPU related clocks
-		 */
-		clk_m_a9: clk-m-a9@fdde08ac {
-			#clock-cells = <0>;
-			compatible = "st,stih416-clkgen-a9-mux", "st,clkgen-mux";
-			reg = <0xfdde08ac 0x4>;
-			clocks = <&clockgen_a9_pll 0>,
-				 <&clockgen_a9_pll 0>,
-				 <&clk_m_a0_div1 2>,
-				 <&clk_m_a9_ext2f_div2>;
-		};
-
-		/*
-		 * ARM Peripheral clock for timers
-		 */
-		arm_periph_clk: clk-m-a9-periphs {
-			#clock-cells = <0>;
-			compatible = "fixed-factor-clock";
-			clocks = <&clk_m_a9>;
-			clock-div = <2>;
-			clock-mult = <1>;
-		};
-
-		/*
-		 * Frequency synthesizers on the SASG2
-		 */
-		clockgen_b0: clockgen-b0@fee108b4 {
-			#clock-cells = <1>;
-			compatible = "st,stih416-quadfs216", "st,quadfs";
-			reg = <0xfee108b4 0x44>;
-
-			clocks = <&clk_sysin>;
-			clock-output-names = "clk-s-usb48",
-					     "clk-s-dss",
-					     "clk-s-stfe-frc-2",
-					     "clk-s-thsens-scard";
-		};
-
-		clockgen_b1: clockgen-b1@fe8308c4 {
-			#clock-cells = <1>;
-			compatible = "st,stih416-quadfs216", "st,quadfs";
-			reg = <0xfe8308c4 0x44>;
-
-			clocks = <&clk_sysin>;
-			clock-output-names = "clk-s-pcm-0",
-					     "clk-s-pcm-1",
-					     "clk-s-pcm-2",
-					     "clk-s-pcm-3";
-		};
-
-		clockgen_c: clockgen-c@fe8307d0 {
-			#clock-cells = <1>;
-			compatible = "st,stih416-quadfs432", "st,quadfs";
-			reg = <0xfe8307d0 0x44>;
-
-			clocks = <&clk_sysin>;
-			clock-output-names = "clk-s-c-fs0-ch0",
-					     "clk-s-c-vcc-sd",
-					     "clk-s-c-fs0-ch2";
-		};
-
-		clk_s_vcc_hd: clk-s-vcc-hd@fe8308b8 {
-			#clock-cells = <0>;
-			compatible = "st,stih416-clkgenc-vcc-hd", "st,clkgen-mux";
-			reg = <0xfe8308b8 0x4>; /* SYSCFG2558 */
-
-			clocks = <&clk_sysin>,
-				 <&clockgen_c 0>;
-		};
-
-		/*
-		 * Add a dummy clock for the HDMI PHY for the VCC input mux
-		 */
-		clk_s_tmds_fromphy: clk-s-tmds-fromphy {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-			clock-frequency = <0>;
-		};
-
-		clockgen_c_vcc: clockgen-c-vcc@fe8308ac {
-			#clock-cells = <1>;
-			compatible = "st,stih416-clkgenc", "st,clkgen-vcc";
-			reg = <0xfe8308ac 0xc>; /* SYSCFG2555,2556,2557 */
-
-			clocks = <&clk_s_vcc_hd>,
-				 <&clockgen_c 1>,
-				 <&clk_s_tmds_fromphy>,
-				 <&clockgen_c 2>;
-
-			clock-output-names  = "clk-s-pix-hdmi",
-					      "clk-s-pix-dvo",
-					      "clk-s-out-dvo",
-					      "clk-s-pix-hd",
-					      "clk-s-hddac",
-					      "clk-s-denc",
-					      "clk-s-sddac",
-					      "clk-s-pix-main",
-					      "clk-s-pix-aux",
-					      "clk-s-stfe-frc-0",
-					      "clk-s-ref-mcru",
-					      "clk-s-slave-mcru",
-					      "clk-s-tmds-hdmi",
-					      "clk-s-hdmi-reject-pll",
-					      "clk-s-thsens";
-		};
-
-		clockgen_d: clockgen-d@fee107e0 {
-			#clock-cells = <1>;
-			compatible = "st,stih416-quadfs216", "st,quadfs";
-			reg = <0xfee107e0 0x44>;
-
-			clocks = <&clk_sysin>;
-			clock-output-names = "clk-s-ccsc",
-					     "clk-s-stfe-frc-1",
-					     "clk-s-tsout-1",
-					     "clk-s-mchi";
-		};
-
-		/*
-		 * Frequency synthesizers on the MPE42
-		 */
-		clockgen_e: clockgen-e@fd3208bc {
-			#clock-cells = <1>;
-			compatible = "st,stih416-quadfs660-E", "st,quadfs";
-			reg = <0xfd3208bc 0xb0>;
-
-			clocks = <&clk_sysin>;
-			clock-output-names = "clk-m-pix-mdtp-0",
-					     "clk-m-pix-mdtp-1",
-					     "clk-m-pix-mdtp-2",
-					     "clk-m-mpelpc";
-		};
-
-		clockgen_f: clockgen-f@fd320878 {
-			#clock-cells = <1>;
-			compatible = "st,stih416-quadfs660-F", "st,quadfs";
-			reg = <0xfd320878 0xf0>;
-
-			clocks = <&clk_sysin>;
-			clock-output-names = "clk-m-main-vidfs",
-					     "clk-m-hva-fs",
-					     "clk-m-fvdp-vcpu",
-					     "clk-m-fvdp-proc-fs";
-		};
-
-		clk_m_fvdp_proc: clk-m-fvdp-proc@fd320910 {
-			#clock-cells = <0>;
-			compatible = "st,stih416-clkgenf-vcc-fvdp", "st,clkgen-mux";
-			reg = <0xfd320910 0x4>; /* SYSCFG8580 */
-
-			clocks = <&clk_m_a1_div2 0>,
-				 <&clockgen_f 3>;
-		};
-
-		clk_m_hva: clk-m-hva@fd690868 {
-			#clock-cells = <0>;
-			compatible = "st,stih416-clkgenf-vcc-hva", "st,clkgen-mux";
-			reg = <0xfd690868 0x4>; /* SYSCFG9538 */
-
-			clocks = <&clockgen_f 1>,
-				 <&clk_m_a1_div0 3>;
-		};
-
-		clk_m_f_vcc_hd: clk-m-f-vcc-hd@fd32086c {
-			#clock-cells = <0>;
-			compatible = "st,stih416-clkgenf-vcc-hd", "st,clkgen-mux";
-			reg = <0xfd32086c 0x4>; /* SYSCFG8539 */
-
-			clocks = <&clockgen_c_vcc 7>,
-				 <&clockgen_f 0>;
-		};
-
-		clk_m_f_vcc_sd: clk-m-f-vcc-sd@fd32086c {
-			#clock-cells = <0>;
-			compatible = "st,stih416-clkgenf-vcc-sd", "st,clkgen-mux";
-			reg = <0xfd32086c 0x4>; /* SYSCFG8539 */
-
-			clocks = <&clockgen_c_vcc 8>,
-				 <&clockgen_f 1>;
-		};
-
-		/*
-		 * Add a dummy clock for the HDMIRx external signal clock
-		 */
-		clk_m_pix_hdmirx_sas: clk-m-pix-hdmirx-sas {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-			clock-frequency = <0>;
-		};
-
-		clockgen_f_vcc: clockgen-f-vcc@fd32086c {
-			#clock-cells = <1>;
-			compatible = "st,stih416-clkgenf", "st,clkgen-vcc";
-			reg = <0xfd32086c 0xc>; /* SYSCFG8539,8540,8541 */
-
-			clocks = <&clk_m_f_vcc_hd>,
-				 <&clk_m_f_vcc_sd>,
-				 <&clockgen_f 0>,
-				 <&clk_m_pix_hdmirx_sas>;
-
-			clock-output-names  = "clk-m-pix-main-pipe",
-					      "clk-m-pix-aux-pipe",
-					      "clk-m-pix-main-cru",
-					      "clk-m-pix-aux-cru",
-					      "clk-m-xfer-be-compo",
-					      "clk-m-xfer-pip-compo",
-					      "clk-m-xfer-aux-compo",
-					      "clk-m-vsens",
-					      "clk-m-pix-hdmirx-0",
-					      "clk-m-pix-hdmirx-1";
-		};
-
-		/*
-		 * DDR PLL
-		 */
-		clockgen-ddr@0xfdde07d8 {
-			reg = <0xfdde07d8 0x110>;
-
-			clockgen_ddr_pll: clockgen-ddr-pll {
-				#clock-cells = <1>;
-				compatible = "st,stih416-plls-c32-ddr", "st,clkgen-plls-c32";
-
-				clocks = <&clk_sysin>;
-				clock-output-names = "clockgen-ddr0",
-						     "clockgen-ddr1";
-			};
-		};
-
-		/*
-		 * GPU PLL
-		 */
-		clockgen-gpu@fd68ff00 {
-			reg = <0xfd68ff00 0x910>;
-
-			clockgen_gpu_pll: clockgen-gpu-pll {
-				#clock-cells = <1>;
-				compatible = "st,stih416-gpu-pll-c32", "st,clkgengpu-pll-c32";
-
-				clocks = <&clk_sysin>;
-				clock-output-names = "clockgen-gpu-pll";
-			};
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/stih416-pinctrl.dtsi b/arch/arm/boot/dts/stih416-pinctrl.dtsi
deleted file mode 100644
index 9c97f7e651a0..000000000000
--- a/arch/arm/boot/dts/stih416-pinctrl.dtsi
+++ /dev/null
@@ -1,692 +0,0 @@
-
-/*
- * Copyright (C) 2013 STMicroelectronics Limited.
- * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-#include "st-pincfg.h"
-#include <dt-bindings/interrupt-controller/arm-gic.h>
-/ {
-
-	aliases {
-		gpio0	= &pio0;
-		gpio1	= &pio1;
-		gpio2	= &pio2;
-		gpio3	= &pio3;
-		gpio4	= &pio4;
-		gpio5	= &pio40;
-		gpio6	= &pio5;
-		gpio7	= &pio6;
-		gpio8	= &pio7;
-		gpio9	= &pio8;
-		gpio10	= &pio9;
-		gpio11	= &pio10;
-		gpio12	= &pio11;
-		gpio13	= &pio12;
-		gpio14	= &pio30;
-		gpio15	= &pio31;
-		gpio16	= &pio13;
-		gpio17	= &pio14;
-		gpio18	= &pio15;
-		gpio19	= &pio16;
-		gpio20	= &pio17;
-		gpio21	= &pio18;
-		gpio22	= &pio100;
-		gpio23	= &pio101;
-		gpio24	= &pio102;
-		gpio25	= &pio103;
-		gpio26	= &pio104;
-		gpio27	= &pio105;
-		gpio28	= &pio106;
-		gpio29	= &pio107;
-	};
-
-	soc {
-		pin-controller-sbc {
-			#address-cells	= <1>;
-			#size-cells	= <1>;
-			compatible	= "st,stih416-sbc-pinctrl";
-			st,syscfg	= <&syscfg_sbc>;
-			reg 		= <0xfe61f080 0x4>;
-			reg-names	= "irqmux";
-			interrupts 	= <GIC_SPI 182 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names	= "irqmux";
-			ranges		= <0 0xfe610000 0x6000>;
-
-			pio0: gpio@fe610000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0 0x100>;
-				st,bank-name	= "PIO0";
-			};
-			pio1: gpio@fe611000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x1000 0x100>;
-				st,bank-name	= "PIO1";
-			};
-			pio2: gpio@fe612000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x2000 0x100>;
-				st,bank-name	= "PIO2";
-			};
-			pio3: gpio@fe613000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x3000 0x100>;
-				st,bank-name	= "PIO3";
-			};
-			pio4: gpio@fe614000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x4000 0x100>;
-				st,bank-name	= "PIO4";
-			};
-			pio40: gpio@fe615000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x5000 0x100>;
-				st,bank-name	= "PIO40";
-				st,retime-pin-mask = <0x7f>;
-			};
-
-			rc{
-				pinctrl_ir: ir0 {
-					st,pins {
-						ir = <&pio4 0 ALT2 IN>;
-					};
-				};
-			};
-			sbc_serial1 {
-				pinctrl_sbc_serial1: sbc_serial1 {
-					st,pins {
-						tx	= <&pio2 6 ALT3 OUT>;
-						rx	= <&pio2 7 ALT3 IN>;
-					};
-				};
-			};
-
-			keyscan {
-				pinctrl_keyscan: keyscan {
-					st,pins {
-						keyin0 = <&pio0 2 ALT2 IN>;
-						keyin1 = <&pio0 3 ALT2 IN>;
-						keyin2 = <&pio0 4 ALT2 IN>;
-						keyin3 = <&pio2 6 ALT2 IN>;
-
-						keyout0 = <&pio1 6 ALT2 OUT>;
-						keyout1 = <&pio1 7 ALT2 OUT>;
-						keyout2 = <&pio0 6 ALT2 OUT>;
-						keyout3 = <&pio2 7 ALT2 OUT>;
-					};
-				};
-			};
-
-			sbc_i2c0 {
-				pinctrl_sbc_i2c0_default: sbc_i2c0-default {
-					st,pins {
-						sda = <&pio4 6 ALT1 BIDIR>;
-						scl = <&pio4 5 ALT1 BIDIR>;
-					};
-				};
-			};
-
-			usb {
-				pinctrl_usb3: usb3 {
-					st,pins {
-						oc-detect = <&pio40 0 ALT1 IN>;
-						pwr-enable = <&pio40 1 ALT1 OUT>;
-					};
-				};
-			};
-
-			sbc_i2c1 {
-				pinctrl_sbc_i2c1_default: sbc_i2c1-default {
-					st,pins {
-						sda = <&pio3 2 ALT2 BIDIR>;
-						scl = <&pio3 1 ALT2 BIDIR>;
-					};
-				};
-			};
-
-			gmac1 {
-				pinctrl_mii1: mii1 {
-					st,pins {
-						txd0 = <&pio0 0 ALT1 OUT SE_NICLK_IO 0 CLK_A>;
-						txd1 = <&pio0 1 ALT1 OUT SE_NICLK_IO 0 CLK_A>;
-						txd2 = <&pio0 2 ALT1 OUT SE_NICLK_IO 0 CLK_A>;
-						txd3 = <&pio0 3 ALT1 OUT SE_NICLK_IO 0 CLK_A>;
-						txer = <&pio0 4 ALT1 OUT SE_NICLK_IO 0 CLK_A>;
-						txen = <&pio0 5 ALT1 OUT SE_NICLK_IO 0 CLK_A>;
-						txclk = <&pio0 6 ALT1 IN NICLK 0 CLK_A>;
-						col =   <&pio0 7 ALT1 IN BYPASS 1000>;
-
-						mdio =  <&pio1 0 ALT1 OUT BYPASS 1500>;
-						mdc =   <&pio1 1 ALT1 OUT NICLK 0 CLK_A>;
-						crs =   <&pio1 2 ALT1 IN BYPASS 1000>;
-						mdint = <&pio1 3 ALT1 IN BYPASS 0>;
-						rxd0 =  <&pio1 4 ALT1 IN SE_NICLK_IO 0 CLK_A>;
-						rxd1 =  <&pio1 5 ALT1 IN SE_NICLK_IO 0 CLK_A>;
-						rxd2 =  <&pio1 6 ALT1 IN SE_NICLK_IO 0 CLK_A>;
-						rxd3 =  <&pio1 7 ALT1 IN SE_NICLK_IO 0 CLK_A>;
-
-						rxdv =  <&pio2 0 ALT1 IN SE_NICLK_IO 0 CLK_A>;
-						rx_er = <&pio2 1 ALT1 IN SE_NICLK_IO 0 CLK_A>;
-						rxclk = <&pio2 2 ALT1 IN NICLK 0 CLK_A>;
-						phyclk = <&pio2 3 ALT1 OUT NICLK 0 CLK_A>;
-					};
-				};
-				pinctrl_rgmii1: rgmii1-0 {
-					st,pins {
-						txd0 =  <&pio0 0 ALT1 OUT DE_IO 500 CLK_A>;
-						txd1 =  <&pio0 1 ALT1 OUT DE_IO 500 CLK_A>;
-						txd2 =  <&pio0 2 ALT1 OUT DE_IO 500 CLK_A>;
-						txd3 =  <&pio0 3 ALT1 OUT DE_IO 500 CLK_A>;
-						txen =  <&pio0 5 ALT1 OUT DE_IO 0   CLK_A>;
-						txclk = <&pio0 6 ALT1 IN  NICLK 0   CLK_A>;
-
-						mdio = <&pio1 0 ALT1 OUT BYPASS 0>;
-						mdc  = <&pio1 1 ALT1 OUT NICLK  0 CLK_A>;
-						rxd0 = <&pio1 4 ALT1 IN DE_IO 500 CLK_A>;
-						rxd1 = <&pio1 5 ALT1 IN DE_IO 500 CLK_A>;
-						rxd2 = <&pio1 6 ALT1 IN DE_IO 500 CLK_A>;
-						rxd3 = <&pio1 7 ALT1 IN DE_IO 500 CLK_A>;
-
-						rxdv   = <&pio2 0 ALT1 IN  DE_IO 500 CLK_A>;
-						rxclk  = <&pio2 2 ALT1 IN  NICLK 0   CLK_A>;
-						phyclk = <&pio2 3 ALT4 OUT NICLK 0   CLK_B>;
-
-						clk125= <&pio3 7 ALT4 IN NICLK 0 CLK_A>;
-					};
-				};
-			};
-
-			pwm1 {
-				pinctrl_pwm1_chan0_default: pwm1-0-default {
-					st,pins {
-						pwm-out    = <&pio3 0 ALT1 OUT>;
-						pwm-capturein = <&pio3 2 ALT1 IN>;
-
-					};
-				};
-				pinctrl_pwm1_chan1_default: pwm1-1-default {
-					st,pins {
-						pwm-out    = <&pio4 4 ALT1 OUT>;
-						pwm-capturein = <&pio4 3 ALT1 IN>;
-					};
-				};
-				pinctrl_pwm1_chan2_default: pwm1-2-default {
-					st,pins {
-						pwm-out    = <&pio4 6 ALT3 OUT>;
-					};
-				};
-				pinctrl_pwm1_chan3_default: pwm1-3-default {
-					st,pins {
-						pwm-out    = <&pio4 7 ALT3 OUT>;
-					};
-				};
-			};
-		};
-
-		pin-controller-front {
-			#address-cells	= <1>;
-			#size-cells	= <1>;
-			compatible	= "st,stih416-front-pinctrl";
-			st,syscfg	= <&syscfg_front>;
-			reg 		= <0xfee0f080 0x4>;
-			reg-names	= "irqmux";
-			interrupts 	= <GIC_SPI 183 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names	= "irqmux";
-			ranges		= <0 0xfee00000 0x10000>;
-
-			pio5: gpio@fee00000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0 0x100>;
-				st,bank-name	= "PIO5";
-			};
-			pio6: gpio@fee01000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x1000 0x100>;
-				st,bank-name	= "PIO6";
-			};
-			pio7: gpio@fee02000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x2000 0x100>;
-				st,bank-name	= "PIO7";
-			};
-			pio8: gpio@fee03000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x3000 0x100>;
-				st,bank-name	= "PIO8";
-			};
-			pio9: gpio@fee04000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x4000 0x100>;
-				st,bank-name	= "PIO9";
-			};
-			pio10: gpio@fee05000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x5000 0x100>;
-				st,bank-name	= "PIO10";
-			};
-			pio11: gpio@fee06000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x6000 0x100>;
-				st,bank-name	= "PIO11";
-			};
-			pio12: gpio@fee07000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x7000 0x100>;
-				st,bank-name	= "PIO12";
-			};
-			pio30: gpio@fee08000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x8000 0x100>;
-				st,bank-name	= "PIO30";
-			};
-			pio31: gpio@fee09000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x9000 0x100>;
-				st,bank-name	= "PIO31";
-			};
-
-			pwm0 {
-				pinctrl_pwm0_chan0_default: pwm0-0-default {
-					st,pins {
-						pwm-out    = <&pio9 7 ALT2 OUT>;
-						pwm-capturein = <&pio9 6 ALT2 IN>;
-					};
-				};
-			};
-
-			serial2-oe {
-				pinctrl_serial2_oe: serial2-1 {
-					st,pins {
-						output-enable	= <&pio11 3 ALT2 OUT>;
-					};
-				};
-			};
-
-			i2c0 {
-				pinctrl_i2c0_default: i2c0-default {
-					st,pins {
-						sda = <&pio9 3 ALT1 BIDIR>;
-						scl = <&pio9 2 ALT1 BIDIR>;
-					};
-				};
-			};
-
-			usb {
-				pinctrl_usb0: usb0 {
-					st,pins {
-						oc-detect = <&pio9 4 ALT1 IN>;
-						pwr-enable = <&pio9 5 ALT1 OUT>;
-					};
-				};
-			};
-
-
-			i2c1 {
-				pinctrl_i2c1_default: i2c1-default {
-					st,pins {
-						sda = <&pio12 1 ALT1 BIDIR>;
-						scl = <&pio12 0 ALT1 BIDIR>;
-					};
-				};
-			};
-
-			fsm {
-				pinctrl_fsm: fsm {
-					st,pins {
-						spi-fsm-clk  = <&pio12 2 ALT1 OUT>;
-						spi-fsm-cs   = <&pio12 3 ALT1 OUT>;
-						spi-fsm-mosi = <&pio12 4 ALT1 OUT>;
-						spi-fsm-miso = <&pio12 5 ALT1 IN>;
-						spi-fsm-hol  = <&pio12 6 ALT1 OUT>;
-						spi-fsm-wp   = <&pio12 7 ALT1 OUT>;
-					};
-				};
-			};
-		};
-
-		pin-controller-rear {
-			#address-cells	= <1>;
-			#size-cells	= <1>;
-			compatible	= "st,stih416-rear-pinctrl";
-			st,syscfg	= <&syscfg_rear>;
-			reg 		= <0xfe82f080 0x4>;
-			reg-names	= "irqmux";
-			interrupts 	= <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names	= "irqmux";
-			ranges 		= <0 0xfe820000 0x6000>;
-
-			pio13: gpio@fe820000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0 0x100>;
-				st,bank-name	= "PIO13";
-			};
-			pio14: gpio@fe821000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x1000 0x100>;
-				st,bank-name	= "PIO14";
-			};
-			pio15: gpio@fe822000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x2000 0x100>;
-				st,bank-name	= "PIO15";
-			};
-			pio16: gpio@fe823000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x3000 0x100>;
-				st,bank-name	= "PIO16";
-			};
-			pio17: gpio@fe824000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x4000 0x100>;
-				st,bank-name	= "PIO17";
-			};
-			pio18: gpio@fe825000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x5000 0x100>;
-				st,bank-name	= "PIO18";
-				st,retime-pin-mask = <0xf>;
-			};
-
-			serial2 {
-				pinctrl_serial2: serial2-0 {
-					st,pins {
-						tx	= <&pio17 4 ALT2 OUT>;
-						rx	= <&pio17 5 ALT2 IN>;
-					};
-				};
-			};
-
-			gmac0 {
-				pinctrl_mii0: mii0 {
-					st,pins {
-						mdint = <&pio13 6 ALT2 IN  BYPASS      0>;
-						txen =  <&pio13 7 ALT2 OUT SE_NICLK_IO 0 CLK_A>;
-						txd0 =  <&pio14 0 ALT2 OUT SE_NICLK_IO 0 CLK_A>;
-						txd1 =  <&pio14 1 ALT2 OUT SE_NICLK_IO 0 CLK_A>;
-						txd2 =  <&pio14 2 ALT2 OUT SE_NICLK_IO 0 CLK_B>;
-						txd3 =  <&pio14 3 ALT2 OUT SE_NICLK_IO 0 CLK_B>;
-
-						txclk = <&pio15 0 ALT2 IN  NICLK       0 CLK_A>;
-						txer =  <&pio15 1 ALT2 OUT SE_NICLK_IO 0 CLK_A>;
-						crs = <&pio15 2 ALT2 IN  BYPASS 1000>;
-						col = <&pio15 3 ALT2 IN  BYPASS 1000>;
-						mdio= <&pio15 4 ALT2 OUT BYPASS 1500>;
-						mdc = <&pio15 5 ALT2 OUT NICLK  0    CLK_B>;
-
-						rxd0 =  <&pio16 0 ALT2 IN SE_NICLK_IO 0 CLK_A>;
-						rxd1 =  <&pio16 1 ALT2 IN SE_NICLK_IO 0 CLK_A>;
-						rxd2 =  <&pio16 2 ALT2 IN SE_NICLK_IO 0 CLK_A>;
-						rxd3 =  <&pio16 3 ALT2 IN SE_NICLK_IO 0 CLK_A>;
-						rxdv =  <&pio15 6 ALT2 IN SE_NICLK_IO 0 CLK_A>;
-						rx_er = <&pio15 7 ALT2 IN SE_NICLK_IO 0 CLK_A>;
-						rxclk = <&pio17 0 ALT2 IN NICLK 0 CLK_A>;
-						phyclk = <&pio13 5 ALT2 OUT NICLK 0 CLK_B>;
-					};
-				};
-
-				pinctrl_gmii0: gmii0 {
-					st,pins {
-						};
-				};
-				pinctrl_rgmii0: rgmii0 {
-					st,pins {
-						 phyclk = <&pio13  5 ALT4 OUT NICLK 0 CLK_B>;
-						 txen = <&pio13 7 ALT2 OUT DE_IO 0 CLK_A>;
-						 txd0  = <&pio14 0 ALT2 OUT DE_IO 500 CLK_A>;
-						 txd1  = <&pio14 1 ALT2 OUT DE_IO 500 CLK_A>;
-						 txd2  = <&pio14 2 ALT2 OUT DE_IO 500 CLK_B>;
-						 txd3  = <&pio14 3 ALT2 OUT DE_IO 500 CLK_B>;
-						 txclk = <&pio15 0 ALT2 IN NICLK 0 CLK_A>;
-
-						 mdio = <&pio15 4 ALT2 OUT BYPASS 0>;
-						 mdc = <&pio15 5 ALT2 OUT NICLK 0 CLK_B>;
-
-						 rxdv = <&pio15 6 ALT2 IN DE_IO 500 CLK_A>;
-						 rxd0 =<&pio16 0 ALT2 IN DE_IO	500 CLK_A>;
-						 rxd1 =<&pio16 1 ALT2 IN DE_IO	500 CLK_A>;
-						 rxd2 =<&pio16 2 ALT2 IN DE_IO	500 CLK_A>;
-						 rxd3  =<&pio16 3 ALT2 IN DE_IO 500 CLK_A>;
-						 rxclk =<&pio17 0 ALT2 IN NICLK 0 CLK_A>;
-
-						 clk125=<&pio17 6 ALT1 IN NICLK 0 CLK_A>;
-					};
-				};
-			};
-
-			mmc0 {
-				pinctrl_mmc0: mmc0 {
-					st,pins {
-						mmcclk  = <&pio13 4 ALT4 BIDIR_PU NICLK 0 CLK_B>;
-						data0   = <&pio14 4 ALT4 BIDIR_PU BYPASS 0>;
-						data1   = <&pio14 5 ALT4 BIDIR_PU BYPASS 0>;
-						data2   = <&pio14 6 ALT4 BIDIR_PU BYPASS 0>;
-						data3   = <&pio14 7 ALT4 BIDIR_PU BYPASS 0>;
-						cmd     = <&pio15 1 ALT4 BIDIR_PU BYPASS 0>;
-						wp      = <&pio15 3 ALT4 IN>;
-						data4   = <&pio16 4 ALT4 BIDIR_PU BYPASS 0>;
-						data5   = <&pio16 5 ALT4 BIDIR_PU BYPASS 0>;
-						data6   = <&pio16 6 ALT4 BIDIR_PU BYPASS 0>;
-						data7   = <&pio16 7 ALT4 BIDIR_PU BYPASS 0>;
-						pwr     = <&pio17 1 ALT4 OUT>;
-						cd      = <&pio17 2 ALT4 IN>;
-						led     = <&pio17 3 ALT4 OUT>;
-					};
-				};
-			};
-			mmc1 {
-				pinctrl_mmc1: mmc1 {
-					st,pins {
-						mmcclk  = <&pio15 0 ALT3 BIDIR_PU NICLK 0 CLK_B>;
-						data0   = <&pio13 7 ALT3 BIDIR_PU BYPASS 0>;
-						data1   = <&pio14 1 ALT3 BIDIR_PU BYPASS 0>;
-						data2   = <&pio14 2 ALT3 BIDIR_PU BYPASS 0>;
-						data3   = <&pio14 3 ALT3 BIDIR_PU BYPASS 0>;
-						cmd     = <&pio15 4 ALT3 BIDIR_PU BYPASS 0>;
-						data4   = <&pio15 6 ALT3 BIDIR_PU BYPASS 0>;
-						data5   = <&pio15 7 ALT3 BIDIR_PU BYPASS 0>;
-						data6   = <&pio16 0 ALT3 BIDIR_PU BYPASS 0>;
-						data7   = <&pio16 1 ALT3 BIDIR_PU BYPASS 0>;
-						pwr     = <&pio16 2 ALT3 OUT>;
-						nreset  = <&pio13 6 ALT3 OUT>;
-					};
-				};
-			};
-
-			usb {
-				pinctrl_usb1: usb1 {
-					st,pins {
-						oc-detect = <&pio18 0 ALT1 IN>;
-						pwr-enable = <&pio18 1 ALT1 OUT>;
-					};
-				};
-				pinctrl_usb2: usb2 {
-					st,pins {
-						oc-detect = <&pio18 2 ALT1 IN>;
-						pwr-enable = <&pio18 3 ALT1 OUT>;
-					};
-				};
-			};
-
-			pwm0 {
-				pinctrl_pwm0_chan1_default: pwm0-1-default {
-					st,pins {
-						pwm-out    = <&pio13 2 ALT2 OUT>;
-						pwm-capturein = <&pio13 1 ALT2 IN>;
-					};
-				};
-				pinctrl_pwm0_chan2_default: pwm0-2-default {
-					st,pins {
-						pwm-out    = <&pio15 2 ALT4 OUT>;
-					};
-				};
-				pinctrl_pwm0_chan3_default: pwm0-3-default {
-					st,pins {
-						pwm-out    = <&pio17 4 ALT1 OUT>;
-					};
-				};
-			};
-
-		};
-
-		pin-controller-fvdp-fe {
-			#address-cells	= <1>;
-			#size-cells	= <1>;
-			compatible	= "st,stih416-fvdp-fe-pinctrl";
-			st,syscfg	= <&syscfg_fvdp_fe>;
-			reg 		= <0xfd6bf080 0x4>;
-			reg-names	= "irqmux";
-			interrupts 	= <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names	= "irqmux";
-			ranges		= <0 0xfd6b0000 0x3000>;
-
-			pio100: gpio@fd6b0000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0 0x100>;
-				st,bank-name	= "PIO100";
-			};
-			pio101: gpio@fd6b1000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x1000 0x100>;
-				st,bank-name	= "PIO101";
-			};
-			pio102: gpio@fd6b2000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x2000 0x100>;
-				st,bank-name	= "PIO102";
-			};
-		};
-
-		pin-controller-fvdp-lite {
-			#address-cells	= <1>;
-			#size-cells	= <1>;
-			compatible	= "st,stih416-fvdp-lite-pinctrl";
-			st,syscfg		= <&syscfg_fvdp_lite>;
-			reg 		= <0xfd33f080 0x4>;
-			reg-names	= "irqmux";
-			interrupts 	= <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names	= "irqmux";
-			ranges			= <0 0xfd330000 0x5000>;
-
-			pio103: gpio@fd330000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0 0x100>;
-				st,bank-name	= "PIO103";
-			};
-			pio104: gpio@fd331000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x1000 0x100>;
-				st,bank-name	= "PIO104";
-			};
-			pio105: gpio@fd332000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x2000 0x100>;
-				st,bank-name	= "PIO105";
-			};
-			pio106: gpio@fd333000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x3000 0x100>;
-				st,bank-name	= "PIO106";
-			};
-
-			pio107: gpio@fd334000 {
-				gpio-controller;
-				#gpio-cells	= <2>;
-				interrupt-controller;
-				#interrupt-cells = <2>;
-				reg		= <0x4000 0x100>;
-				st,bank-name	= "PIO107";
-				st,retime-pin-mask = <0xf>;
-			};
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/stih416.dtsi b/arch/arm/boot/dts/stih416.dtsi
deleted file mode 100644
index fe1f9cf770e4..000000000000
--- a/arch/arm/boot/dts/stih416.dtsi
+++ /dev/null
@@ -1,517 +0,0 @@
-/*
- * Copyright (C) 2012 STMicroelectronics Limited.
- * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-#include "stih41x.dtsi"
-#include "stih416-clock.dtsi"
-#include "stih416-pinctrl.dtsi"
-
-#include <dt-bindings/phy/phy.h>
-#include <dt-bindings/interrupt-controller/arm-gic.h>
-#include <dt-bindings/reset/stih416-resets.h>
-#include <dt-bindings/interrupt-controller/irq-st.h>
-/ {
-	L2: cache-controller {
-		compatible = "arm,pl310-cache";
-		reg = <0xfffe2000 0x1000>;
-		arm,data-latency = <3 3 3>;
-		arm,tag-latency = <2 2 2>;
-		cache-unified;
-		cache-level = <2>;
-	};
-
-	arm-pmu {
-		compatible = "arm,cortex-a9-pmu";
-		interrupt-parent = <&intc>;
-		interrupts = <GIC_PPI 15 IRQ_TYPE_LEVEL_HIGH>;
-	};
-
-	soc {
-		#address-cells = <1>;
-		#size-cells = <1>;
-		interrupt-parent = <&intc>;
-		ranges;
-		compatible	= "simple-bus";
-
-		restart {
-			compatible = "st,stih416-restart";
-			st,syscfg = <&syscfg_sbc>;
-			status = "okay";
-		};
-
-		powerdown: powerdown-controller {
-			#reset-cells = <1>;
-			compatible = "st,stih416-powerdown";
-		};
-
-		softreset: softreset-controller {
-			#reset-cells = <1>;
-			compatible = "st,stih416-softreset";
-		};
-
-		syscfg_sbc:sbc-syscfg@fe600000{
-			compatible	= "st,stih416-sbc-syscfg", "syscon";
-			reg		= <0xfe600000 0x1000>;
-		};
-
-		syscfg_front:front-syscfg@fee10000{
-			compatible	= "st,stih416-front-syscfg", "syscon";
-			reg		= <0xfee10000 0x1000>;
-		};
-
-		syscfg_rear:rear-syscfg@fe830000{
-			compatible	= "st,stih416-rear-syscfg", "syscon";
-			reg		= <0xfe830000 0x1000>;
-		};
-
-		/* MPE */
-		syscfg_fvdp_fe:fvdp-fe-syscfg@fddf0000{
-			compatible	= "st,stih416-fvdp-fe-syscfg", "syscon";
-			reg		= <0xfddf0000 0x1000>;
-		};
-
-		syscfg_fvdp_lite:fvdp-lite-syscfg@fd6a0000{
-			compatible	= "st,stih416-fvdp-lite-syscfg", "syscon";
-			reg		= <0xfd6a0000 0x1000>;
-		};
-
-		syscfg_cpu:cpu-syscfg@fdde0000{
-			compatible	= "st,stih416-cpu-syscfg", "syscon";
-			reg		= <0xfdde0000 0x1000>;
-		};
-
-		syscfg_compo:compo-syscfg@fd320000{
-			compatible	= "st,stih416-compo-syscfg", "syscon";
-			reg		= <0xfd320000 0x1000>;
-		};
-
-		syscfg_transport:transport-syscfg@fd690000{
-			compatible	= "st,stih416-transport-syscfg", "syscon";
-			reg		= <0xfd690000 0x1000>;
-		};
-
-		syscfg_lpm:lpm-syscfg@fe4b5100{
-			compatible	= "st,stih416-lpm-syscfg", "syscon";
-			reg		= <0xfe4b5100 0x8>;
-		};
-
-		irq-syscfg {
-			compatible    = "st,stih416-irq-syscfg";
-			st,syscfg     = <&syscfg_cpu>;
-			st,irq-device = <ST_IRQ_SYSCFG_PMU_0>,
-					<ST_IRQ_SYSCFG_PMU_1>;
-			st,fiq-device = <ST_IRQ_SYSCFG_DISABLED>,
-					<ST_IRQ_SYSCFG_DISABLED>;
-		};
-
-		serial2: serial@fed32000{
-			compatible	= "st,asc";
-			status 		= "disabled";
-			reg		= <0xfed32000 0x2c>;
-			interrupts	= <0 197 0>;
-			clocks 		= <&clk_s_a0_ls CLK_ICN_REG>;
-			pinctrl-names 	= "default";
-			pinctrl-0 	= <&pinctrl_serial2 &pinctrl_serial2_oe>;
-		};
-
-		/* SBC_UART1 */
-		sbc_serial1: serial@fe531000 {
-			compatible	= "st,asc";
-			status 		= "disabled";
-			reg		= <0xfe531000 0x2c>;
-			interrupts	= <0 210 0>;
-			pinctrl-names 	= "default";
-			pinctrl-0 	= <&pinctrl_sbc_serial1>;
-			clocks		= <&clk_sysin>;
-		};
-
-		i2c@fed40000 {
-			compatible	= "st,comms-ssc4-i2c";
-			reg		= <0xfed40000 0x110>;
-			interrupts	= <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>;
-			clocks 		= <&clk_s_a0_ls CLK_ICN_REG>;
-			clock-names	= "ssc";
-			clock-frequency = <400000>;
-			pinctrl-names	= "default";
-			pinctrl-0	= <&pinctrl_i2c0_default>;
-
-			status		= "disabled";
-		};
-
-		i2c@fed41000 {
-			compatible	= "st,comms-ssc4-i2c";
-			reg		= <0xfed41000 0x110>;
-			interrupts	= <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
-			clocks 		= <&clk_s_a0_ls CLK_ICN_REG>;
-			clock-names	= "ssc";
-			clock-frequency = <400000>;
-			pinctrl-names	= "default";
-			pinctrl-0	= <&pinctrl_i2c1_default>;
-
-			status		= "disabled";
-		};
-
-		i2c@fe540000 {
-			compatible	= "st,comms-ssc4-i2c";
-			reg		= <0xfe540000 0x110>;
-			interrupts	= <GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH>;
-			clocks		= <&clk_sysin>;
-			clock-names	= "ssc";
-			clock-frequency = <400000>;
-			pinctrl-names	= "default";
-			pinctrl-0	= <&pinctrl_sbc_i2c0_default>;
-
-			status		= "disabled";
-		};
-
-		i2c@fe541000 {
-			compatible	= "st,comms-ssc4-i2c";
-			reg		= <0xfe541000 0x110>;
-			interrupts	= <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>;
-			clocks		= <&clk_sysin>;
-			clock-names	= "ssc";
-			clock-frequency = <400000>;
-			pinctrl-names	= "default";
-			pinctrl-0	= <&pinctrl_sbc_i2c1_default>;
-
-			status		= "disabled";
-		};
-
-		ethernet0: dwmac@fe810000 {
-			device_type 	= "network";
-			compatible	= "st,stih416-dwmac", "snps,dwmac", "snps,dwmac-3.710";
-			status 		= "disabled";
-			reg		= <0xfe810000 0x8000>;
-			reg-names	= "stmmaceth";
-
-			interrupts = <0 133 0>, <0 134 0>, <0 135 0>;
-			interrupt-names = "macirq", "eth_wake_irq", "eth_lpi";
-
-			snps,pbl 	= <32>;
-			snps,mixed-burst;
-
-			st,syscon		= <&syscfg_rear 0x8bc>;
-			resets			= <&softreset STIH416_ETH0_SOFTRESET>;
-			reset-names		= "stmmaceth";
-			pinctrl-names 	= "default";
-			pinctrl-0	= <&pinctrl_mii0>;
-			clock-names	= "stmmaceth", "sti-ethclk";
-			clocks		= <&clk_s_a1_ls CLK_ICN_IF_2>, <&clk_s_a1_ls CLK_GMAC0_PHY>;
-		};
-
-		ethernet1: dwmac@fef08000 {
-			device_type = "network";
-			compatible		= "st,stih416-dwmac", "snps,dwmac", "snps,dwmac-3.710";
-			status 		= "disabled";
-			reg		= <0xfef08000 0x8000>;
-			reg-names	= "stmmaceth";
-			interrupts = <0 136 0>, <0 137 0>, <0 138 0>;
-			interrupt-names = "macirq", "eth_wake_irq", "eth_lpi";
-
-			snps,pbl	= <32>;
-			snps,mixed-burst;
-
-			st,syscon	= <&syscfg_sbc 0x7f0>;
-
-			resets		= <&softreset STIH416_ETH1_SOFTRESET>;
-			reset-names	= "stmmaceth";
-			pinctrl-names 	= "default";
-			pinctrl-0	= <&pinctrl_mii1>;
-			clock-names	= "stmmaceth", "sti-ethclk";
-			clocks		= <&clk_s_a0_ls CLK_ICN_REG>, <&clk_s_a0_ls CLK_ETH1_PHY>;
-		};
-
-		rc: rc@fe518000 {
-			compatible	= "st,comms-irb";
-			reg		= <0xfe518000 0x234>;
-			interrupts	=  <0 203 0>;
-			rx-mode         = "infrared";
-			clocks		= <&clk_sysin>;
-			pinctrl-names 	= "default";
-			pinctrl-0	= <&pinctrl_ir>;
-			resets		= <&softreset STIH416_IRB_SOFTRESET>;
-		};
-
-		/* FSM */
-		spifsm: spifsm@fe902000 {
-			compatible	   = "st,spi-fsm";
-			reg		   = <0xfe902000 0x1000>;
-			pinctrl-0	   = <&pinctrl_fsm>;
-
-			st,syscfg	   = <&syscfg_rear>;
-			st,boot-device-reg = <0x958>;
-			st,boot-device-spi = <0x1a>;
-
-			status = "disabled";
-		};
-
-		keyscan: keyscan@fe4b0000 {
-			compatible = "st,sti-keyscan";
-			status = "disabled";
-			reg = <0xfe4b0000 0x2000>;
-			interrupts = <GIC_SPI 212 IRQ_TYPE_NONE>;
-			clocks = <&clk_sysin>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_keyscan>;
-			resets	= <&powerdown STIH416_KEYSCAN_POWERDOWN>,
-				  <&softreset STIH416_KEYSCAN_SOFTRESET>;
-		};
-
-		temp0 {
-			compatible = "st,stih416-sas-thermal";
-			clock-names = "thermal";
-			clocks = <&clockgen_c_vcc 14>;
-
-			status = "okay";
-		};
-
-		temp1@fdfe8000 {
-			compatible = "st,stih416-mpe-thermal";
-			reg = <0xfdfe8000 0x10>;
-			clocks = <&clockgen_e 3>;
-			clock-names = "thermal";
-			interrupts = <GIC_SPI 23 IRQ_TYPE_EDGE_RISING>;
-
-			status = "okay";
-		};
-
-		mmc0: sdhci@fe81e000 {
-			compatible	= "st,sdhci";
-			status		= "disabled";
-			reg		= <0xfe81e000 0x1000>;
-			interrupts	= <GIC_SPI 127 IRQ_TYPE_NONE>;
-			interrupt-names	= "mmcirq";
-			pinctrl-names	= "default";
-			pinctrl-0	= <&pinctrl_mmc0>;
-			clock-names	= "mmc";
-			clocks		= <&clk_s_a1_ls 1>;
-		};
-
-		mmc1: sdhci@fe81f000 {
-			compatible	= "st,sdhci";
-			status		= "disabled";
-			reg		= <0xfe81f000 0x1000>;
-			interrupts	= <GIC_SPI 128 IRQ_TYPE_NONE>;
-			interrupt-names	= "mmcirq";
-			pinctrl-names	= "default";
-			pinctrl-0	= <&pinctrl_mmc1>;
-			clock-names	= "mmc";
-			clocks		= <&clk_s_a1_ls 8>;
-		};
-
-		miphy365x_phy: phy@fe382000 {
-			compatible      = "st,miphy365x-phy";
-			st,syscfg	= <&syscfg_rear 0x824 0x828>;
-			#address-cells	= <1>;
-			#size-cells	= <1>;
-			ranges;
-
-			phy_port0: port@fe382000 {
-				#phy-cells = <1>;
-				reg = <0xfe382000 0x100>, <0xfe394000 0x100>;
-				reg-names = "sata", "pcie";
-			};
-
-			phy_port1: port@fe38a000 {
-				#phy-cells = <1>;
-				reg = <0xfe38a000 0x100>, <0xfe804000 0x100>;
-				reg-names = "sata", "pcie";
-			};
-		};
-
-		sata0: sata@fe380000 {
-			compatible      = "st,sti-ahci";
-			reg             = <0xfe380000 0x1000>;
-			interrupts      = <GIC_SPI 157 IRQ_TYPE_NONE>;
-			interrupt-names = "hostc";
-			phys	        = <&phy_port0 PHY_TYPE_SATA>;
-			phy-names       = "sata-phy";
-			resets	        = <&powerdown STIH416_SATA0_POWERDOWN>,
-					  <&softreset STIH416_SATA0_SOFTRESET>;
-			reset-names     = "pwr-dwn", "sw-rst";
-			clock-names     = "ahci_clk";
-			clocks	        = <&clk_s_a0_ls CLK_ICN_REG>;
-
-			status	        = "disabled";
-		};
-
-		usb2_phy: phy@0 {
-			compatible = "st,stih416-usb-phy";
-			#phy-cells = <0>;
-			st,syscfg = <&syscfg_rear>;
-			clocks = <&clk_sysin>;
-			clock-names = "osc_phy";
-		};
-
-		ehci0: usb@fe1ffe00 {
-			compatible = "st,st-ehci-300x";
-			reg = <0xfe1ffe00 0x100>;
-			interrupts = <GIC_SPI 148 IRQ_TYPE_NONE>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_usb0>;
-			clocks = <&clk_s_a1_ls 0>,
-				 <&clockgen_b0 0>;
-			clock-names = "ic", "clk48";
-			phys = <&usb2_phy>;
-			phy-names = "usb";
-			resets = <&powerdown STIH416_USB0_POWERDOWN>,
-				 <&softreset STIH416_USB0_SOFTRESET>;
-			reset-names = "power", "softreset";
-		};
-
-		ohci0: usb@fe1ffc00 {
-			compatible = "st,st-ohci-300x";
-			reg = <0xfe1ffc00 0x100>;
-			interrupts = <GIC_SPI 149 IRQ_TYPE_NONE>;
-			clocks = <&clk_s_a1_ls 0>,
-				 <&clockgen_b0 0>;
-			clock-names = "ic", "clk48";
-			phys = <&usb2_phy>;
-			phy-names = "usb";
-			status = "okay";
-			resets = <&powerdown STIH416_USB0_POWERDOWN>,
-				 <&softreset STIH416_USB0_SOFTRESET>;
-			reset-names = "power", "softreset";
-		};
-
-		ehci1: usb@fe203e00 {
-			compatible = "st,st-ehci-300x";
-			reg = <0xfe203e00 0x100>;
-			interrupts = <GIC_SPI 150 IRQ_TYPE_NONE>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_usb1>;
-			clocks = <&clk_s_a1_ls 0>,
-				 <&clockgen_b0 0>;
-			clock-names = "ic", "clk48";
-			phys = <&usb2_phy>;
-			phy-names = "usb";
-			resets = <&powerdown STIH416_USB1_POWERDOWN>,
-				 <&softreset STIH416_USB1_SOFTRESET>;
-			reset-names = "power", "softreset";
-		};
-
-		ohci1: usb@fe203c00 {
-			compatible = "st,st-ohci-300x";
-			reg = <0xfe203c00 0x100>;
-			interrupts = <GIC_SPI 151 IRQ_TYPE_NONE>;
-			clocks = <&clk_s_a1_ls 0>,
-				 <&clockgen_b0 0>;
-			clock-names = "ic", "clk48";
-			phys = <&usb2_phy>;
-			phy-names = "usb";
-			resets = <&powerdown STIH416_USB1_POWERDOWN>,
-				 <&softreset STIH416_USB1_SOFTRESET>;
-			reset-names = "power", "softreset";
-		};
-
-		ehci2: usb@fe303e00 {
-			compatible = "st,st-ehci-300x";
-			reg = <0xfe303e00 0x100>;
-			interrupts = <GIC_SPI 152 IRQ_TYPE_NONE>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_usb2>;
-			clocks = <&clk_s_a1_ls 0>,
-				 <&clockgen_b0 0>;
-			clock-names = "ic", "clk48";
-			phys = <&usb2_phy>;
-			phy-names = "usb";
-			resets = <&powerdown STIH416_USB2_POWERDOWN>,
-				 <&softreset STIH416_USB2_SOFTRESET>;
-			reset-names = "power", "softreset";
-		};
-
-		ohci2: usb@fe303c00 {
-			compatible = "st,st-ohci-300x";
-			reg = <0xfe303c00 0x100>;
-			interrupts = <GIC_SPI 153 IRQ_TYPE_NONE>;
-			clocks = <&clk_s_a1_ls 0>,
-				 <&clockgen_b0 0>;
-			clock-names = "ic", "clk48";
-			phys = <&usb2_phy>;
-			phy-names = "usb";
-			resets = <&powerdown STIH416_USB2_POWERDOWN>,
-				 <&softreset STIH416_USB2_SOFTRESET>;
-			reset-names = "power", "softreset";
-		};
-
-		ehci3: usb@fe343e00 {
-			compatible = "st,st-ehci-300x";
-			reg = <0xfe343e00 0x100>;
-			interrupts = <GIC_SPI 154 IRQ_TYPE_NONE>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_usb3>;
-			clocks = <&clk_s_a1_ls 0>,
-				 <&clockgen_b0 0>;
-			clock-names = "ic", "clk48";
-			phys = <&usb2_phy>;
-			phy-names = "usb";
-			resets = <&powerdown STIH416_USB3_POWERDOWN>,
-				 <&softreset STIH416_USB3_SOFTRESET>;
-			reset-names = "power", "softreset";
-		};
-
-		ohci3: usb@fe343c00 {
-			compatible = "st,st-ohci-300x";
-			reg = <0xfe343c00 0x100>;
-			interrupts = <GIC_SPI 155 IRQ_TYPE_NONE>;
-			clocks = <&clk_s_a1_ls 0>,
-				 <&clockgen_b0 0>;
-			clock-names = "ic", "clk48";
-			phys = <&usb2_phy>;
-			phy-names = "usb";
-			resets = <&powerdown STIH416_USB3_POWERDOWN>,
-				 <&softreset STIH416_USB3_SOFTRESET>;
-			reset-names = "power", "softreset";
-		};
-
-		/* SAS PWM Module */
-		pwm0: pwm@fed10000 {
-			compatible	= "st,sti-pwm";
-			status		= "disabled";
-			#pwm-cells	= <2>;
-			reg		= <0xfed10000 0x68>;
-			interrupts      = <GIC_SPI 200 IRQ_TYPE_NONE>;
-
-			pinctrl-names	= "default";
-			pinctrl-0 = 	<&pinctrl_pwm0_chan0_default
-					&pinctrl_pwm0_chan1_default
-					&pinctrl_pwm0_chan2_default
-					&pinctrl_pwm0_chan3_default>;
-
-			clock-names	= "pwm", "capture";
-			clocks		= <&clk_sysin>, <&clk_s_a0_ls CLK_ICN_REG>;
-
-			st,pwm-num-chan = <4>;
-			st,capture-num-chan = <2>;
-		};
-
-		/* SBC PWM Module */
-		pwm1: pwm@fe510000 {
-			compatible	= "st,sti-pwm";
-			status		= "disabled";
-			#pwm-cells	= <2>;
-			reg		= <0xfe510000 0x68>;
-			interrupts      = <GIC_SPI 202 IRQ_TYPE_NONE>;
-
-			pinctrl-names	= "default";
-			pinctrl-0	= <&pinctrl_pwm1_chan0_default
-					/*
-					 * Shared with SBC_OBS_NOTRST.  Don't
-					 * enable unless you really know what
-					 * you're doing.
-					 *
-					 * &pinctrl_pwm1_chan1_default
-					 */
-					&pinctrl_pwm1_chan2_default
-					&pinctrl_pwm1_chan3_default>;
-
-			clock-names	= "pwm";
-			clocks		= <&clk_sysin>;
-			st,pwm-num-chan = <3>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/stih41x-b2000.dtsi b/arch/arm/boot/dts/stih41x-b2000.dtsi
deleted file mode 100644
index 9bfa0674b452..000000000000
--- a/arch/arm/boot/dts/stih41x-b2000.dtsi
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics (R&D) Limited.
- * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-#include <dt-bindings/input/input.h>
-#include <dt-bindings/gpio/gpio.h>
-
-/ {
-
-	memory{
-		device_type = "memory";
-		reg = <0x60000000 0x40000000>;
-	};
-
-	chosen {
-		bootargs = "console=ttyAS0,115200 clk_ignore_unused";
-		linux,stdout-path = &serial2;
-	};
-
-	aliases {
-		ttyAS0 = &serial2;
-		ethernet0 = &ethernet0;
-		ethernet1 = &ethernet1;
-	};
-
-	soc {
-		serial2: serial@fed32000 {
-			status = "okay";
-		};
-
-		leds {
-			compatible	= "gpio-leds";
-			fp_led {
-				label	= "Front Panel LED";
-				gpios	= <&pio105 7 GPIO_ACTIVE_HIGH>;
-				linux,default-trigger	= "heartbeat";
-			};
-		};
-
-		/* HDMI Tx I2C */
-		i2c@fed41000 {
-			/* HDMI V1.3a supports Standard mode only */
-			clock-frequency = <100000>;
-			i2c-min-scl-pulse-width-us = <0>;
-			i2c-min-sda-pulse-width-us = <5>;
-
-			status = "okay";
-		};
-
-		ethernet0: dwmac@fe810000 {
-			status			= "okay";
-			phy-mode		= "mii";
-			pinctrl-0		= <&pinctrl_mii0>;
-
-			snps,reset-gpio 	= <&pio106 2>;
-			snps,reset-active-low;
-			snps,reset-delays-us 	= <0 10000 10000>;
-		};
-
-		ethernet1: dwmac@fef08000 {
-			status			= "disabled";
-			phy-mode		= "mii";
-			st,tx-retime-src	= "txclk";
-
-			snps,reset-gpio 	= <&pio4 7>;
-			snps,reset-active-low;
-			snps,reset-delays-us 	= <0 10000 10000>;
-		};
-
-		keyscan: keyscan@fe4b0000 {
-			keypad,num-rows = <4>;
-			keypad,num-columns = <4>;
-			st,debounce-us = <5000>;
-			linux,keymap = < MATRIX_KEY(0x00, 0x00, KEY_F13)
-					 MATRIX_KEY(0x00, 0x01, KEY_F9)
-					 MATRIX_KEY(0x00, 0x02, KEY_F5)
-					 MATRIX_KEY(0x00, 0x03, KEY_F1)
-					 MATRIX_KEY(0x01, 0x00, KEY_F14)
-					 MATRIX_KEY(0x01, 0x01, KEY_F10)
-					 MATRIX_KEY(0x01, 0x02, KEY_F6)
-					 MATRIX_KEY(0x01, 0x03, KEY_F2)
-					 MATRIX_KEY(0x02, 0x00, KEY_F15)
-					 MATRIX_KEY(0x02, 0x01, KEY_F11)
-					 MATRIX_KEY(0x02, 0x02, KEY_F7)
-					 MATRIX_KEY(0x02, 0x03, KEY_F3)
-					 MATRIX_KEY(0x03, 0x00, KEY_F16)
-					 MATRIX_KEY(0x03, 0x01, KEY_F12)
-					 MATRIX_KEY(0x03, 0x02, KEY_F8)
-					 MATRIX_KEY(0x03, 0x03, KEY_F4) >;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/stih41x-b2020.dtsi b/arch/arm/boot/dts/stih41x-b2020.dtsi
deleted file mode 100644
index 322e0e95176c..000000000000
--- a/arch/arm/boot/dts/stih41x-b2020.dtsi
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics (R&D) Limited.
- * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-#include "stih41x-b2020x.dtsi"
-#include <dt-bindings/gpio/gpio.h>
-/ {
-	memory{
-		device_type = "memory";
-		reg = <0x40000000 0x80000000>;
-	};
-
-	chosen {
-		bootargs = "console=ttyAS0,115200 clk_ignore_unused";
-		linux,stdout-path = &sbc_serial1;
-	};
-
-	aliases {
-		ttyAS0 = &sbc_serial1;
-		ethernet1 = &ethernet1;
-	};
-	soc {
-		sbc_serial1: serial@fe531000 {
-			status = "okay";
-		};
-
-		leds {
-			compatible	= "gpio-leds";
-			red {
-				label	= "Front Panel LED";
-				gpios	= <&pio4 1 GPIO_ACTIVE_HIGH>;
-				linux,default-trigger	= "heartbeat";
-			};
-			green {
-				gpios	= <&pio4 7 GPIO_ACTIVE_HIGH>;
-				default-state = "off";
-			};
-		};
-
-		i2c@fed40000 {
-			status = "okay";
-		};
-
-		/* HDMI Tx I2C */
-		i2c@fed41000 {
-			/* HDMI V1.3a supports Standard mode only */
-			clock-frequency = <100000>;
-			i2c-min-scl-pulse-width-us = <0>;
-			i2c-min-sda-pulse-width-us = <5>;
-
-			status = "okay";
-		};
-
-		i2c@fe540000 {
-			status = "okay";
-		};
-
-		i2c@fe541000 {
-			status = "okay";
-		};
-
-		ethernet1: dwmac@fef08000 {
-			status			= "okay";
-			phy-mode		= "rgmii-id";
-			max-speed		= <1000>;
-			st,tx-retime-src	= "clk_125";
-			snps,reset-gpio 	= <&pio3 0>;
-			snps,reset-active-low;
-			snps,reset-delays-us 	= <0 10000 10000>;
-
-			pinctrl-0	= <&pinctrl_rgmii1>;
-		};
-
-		mmc0: sdhci@fe81e000 {
-			bus-width = <8>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/stih41x-b2020x.dtsi b/arch/arm/boot/dts/stih41x-b2020x.dtsi
deleted file mode 100644
index f797a0607382..000000000000
--- a/arch/arm/boot/dts/stih41x-b2020x.dtsi
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics (R&D) Limited.
- * Author: Lee Jones <lee.jones@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-/ {
-	soc {
-		mmc0: sdhci@fe81e000 {
-			status = "okay";
-		};
-
-		spifsm: spifsm@fe902000 {
-			#address-cells = <1>;
-			#size-cells    = <1>;
-
-			status = "okay";
-
-			partition@0 {
-				label = "SerialFlash1";
-				reg   = <0x00000000 0x00500000>;
-			};
-
-			partition@500000 {
-				label = "SerialFlash2";
-				reg   = <0x00500000 0x00b00000>;
-			};
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/stih41x.dtsi b/arch/arm/boot/dts/stih41x.dtsi
deleted file mode 100644
index 5cb0e63376b5..000000000000
--- a/arch/arm/boot/dts/stih41x.dtsi
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (C) 2014 STMicroelectronics Limited.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * publishhed by the Free Software Foundation.
- */
-/ {
-	#address-cells = <1>;
-	#size-cells = <1>;
-
-	cpus {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		cpu@0 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a9";
-			reg = <0>;
-		};
-		cpu@1 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a9";
-			reg = <1>;
-		};
-	};
-
-	intc: interrupt-controller@fffe1000 {
-		compatible = "arm,cortex-a9-gic";
-		#interrupt-cells = <3>;
-		interrupt-controller;
-		reg = <0xfffe1000 0x1000>,
-		      <0xfffe0100 0x100>;
-	};
-
-	scu@fffe0000 {
-		compatible = "arm,cortex-a9-scu";
-		reg = <0xfffe0000 0x1000>;
-	};
-
-	timer@fffe0200 {
-		interrupt-parent = <&intc>;
-		compatible = "arm,cortex-a9-global-timer";
-		reg = <0xfffe0200 0x100>;
-		interrupts = <1 11 0x04>;
-		clocks = <&arm_periph_clk>;
-	};
-};
diff --git a/arch/arm/boot/dts/stihxxx-b2120.dtsi b/arch/arm/boot/dts/stihxxx-b2120.dtsi
index ed2b7a99ecff..4b8f62f89664 100644
--- a/arch/arm/boot/dts/stihxxx-b2120.dtsi
+++ b/arch/arm/boot/dts/stihxxx-b2120.dtsi
@@ -135,6 +135,10 @@
 			};
 		};
 
+		sti_uni_player0: sti-uni-player@8d80000 {
+			status = "okay";
+		};
+
 		sti_uni_player2: sti-uni-player@8d82000 {
 			status = "okay";
 		};
@@ -151,13 +155,26 @@
 
 		sound {
 			compatible = "simple-audio-card";
-			simple-audio-card,name = "sti audio card";
+			simple-audio-card,name = "STI-B2120";
 			status = "okay";
 
 			simple-audio-card,dai-link@0 {
+				/* HDMI */
+				format = "i2s";
+				mclk-fs = <128>;
+				cpu {
+					sound-dai = <&sti_uni_player0>;
+				};
+
+				codec {
+					sound-dai = <&sti_hdmi>;
+				};
+			};
+			simple-audio-card,dai-link@1 {
 				/* DAC */
 				format = "i2s";
 				mclk-fs = <256>;
+				frame-inversion = <1>;
 				cpu {
 					sound-dai = <&sti_uni_player2>;
 				};
@@ -166,7 +183,7 @@
 					sound-dai = <&sti_sasg_codec 1>;
 				};
 			};
-			simple-audio-card,dai-link@1 {
+			simple-audio-card,dai-link@2 {
 				/* SPDIF */
 				format = "left_j";
 				mclk-fs = <128>;
diff --git a/arch/arm/boot/dts/stm32429i-eval.dts b/arch/arm/boot/dts/stm32429i-eval.dts
index 6bfc5959dac3..5436e880e28f 100644
--- a/arch/arm/boot/dts/stm32429i-eval.dts
+++ b/arch/arm/boot/dts/stm32429i-eval.dts
@@ -47,6 +47,7 @@
 
 /dts-v1/;
 #include "stm32f429.dtsi"
+#include <dt-bindings/input/input.h>
 
 / {
 	model = "STMicroelectronics STM32429i-EVAL board";
@@ -65,6 +66,10 @@
 		serial0 = &usart1;
 	};
 
+	soc {
+		dma-ranges = <0xc0000000 0x0 0x10000000>;
+	};
+
 	leds {
 		compatible = "gpio-leds";
 		green {
@@ -82,6 +87,23 @@
 		};
 	};
 
+	gpio_keys {
+		compatible = "gpio-keys";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		autorepeat;
+		button@0 {
+			label = "Wake up";
+			linux,code = <KEY_WAKEUP>;
+			gpios = <&gpioa 0 0>;
+		};
+		button@1 {
+			label = "Tamper";
+			linux,code = <KEY_RESTART>;
+			gpios = <&gpioc 13 0>;
+		};
+	};
+
 	usbotg_hs_phy: usbphy {
 		#phy-cells = <0>;
 		compatible = "usb-nop-xceiv";
@@ -94,11 +116,12 @@
 	clock-frequency = <25000000>;
 };
 
-&ethernet0 {
+&mac {
 	status = "okay";
-	pinctrl-0	= <&ethernet0_mii>;
+	pinctrl-0	= <&ethernet_mii>;
 	pinctrl-names	= "default";
-	phy-mode	= "mii-id";
+	phy-mode	= "mii";
+	phy-handle	= <&phy1>;
 	mdio0 {
 		#address-cells = <1>;
 		#size-cells = <0>;
diff --git a/arch/arm/boot/dts/stm32746g-eval.dts b/arch/arm/boot/dts/stm32746g-eval.dts
new file mode 100644
index 000000000000..aa03fac1ec55
--- /dev/null
+++ b/arch/arm/boot/dts/stm32746g-eval.dts
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2015 - Maxime Coquelin <mcoquelin.stm32@gmail.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+#include "stm32f746.dtsi"
+#include <dt-bindings/input/input.h>
+
+/ {
+	model = "STMicroelectronics STM32746g-EVAL board";
+	compatible = "st,stm32746g-eval", "st,stm32f746";
+
+	chosen {
+		bootargs = "root=/dev/ram";
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory {
+		reg = <0xc0000000 0x2000000>;
+	};
+
+	aliases {
+		serial0 = &usart1;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		green {
+			gpios = <&gpiof 10 1>;
+			linux,default-trigger = "heartbeat";
+		};
+		red {
+			gpios = <&gpiob 7 1>;
+		};
+	};
+
+	gpio_keys {
+		compatible = "gpio-keys";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		autorepeat;
+		button@0 {
+			label = "Wake up";
+			linux,code = <KEY_WAKEUP>;
+			gpios = <&gpioc 13 0>;
+		};
+	};
+};
+
+&clk_hse {
+	clock-frequency = <25000000>;
+};
+
+&usart1 {
+	pinctrl-0 = <&usart1_pins_a>;
+	pinctrl-names = "default";
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/stm32f429-disco.dts b/arch/arm/boot/dts/stm32f429-disco.dts
index 01408073dd53..7d0415e80668 100644
--- a/arch/arm/boot/dts/stm32f429-disco.dts
+++ b/arch/arm/boot/dts/stm32f429-disco.dts
@@ -47,6 +47,7 @@
 
 /dts-v1/;
 #include "stm32f429.dtsi"
+#include <dt-bindings/input/input.h>
 
 / {
 	model = "STMicroelectronics STM32F429i-DISCO board";
@@ -75,6 +76,18 @@
 			linux,default-trigger = "heartbeat";
 		};
 	};
+
+	gpio_keys {
+		compatible = "gpio-keys";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		autorepeat;
+		button@0 {
+			label = "User";
+			linux,code = <KEY_HOME>;
+			gpios = <&gpioa 0 0>;
+		};
+	};
 };
 
 &clk_hse {
diff --git a/arch/arm/boot/dts/stm32f429.dtsi b/arch/arm/boot/dts/stm32f429.dtsi
index 336ee4fb587d..e4dae0eda3cd 100644
--- a/arch/arm/boot/dts/stm32f429.dtsi
+++ b/arch/arm/boot/dts/stm32f429.dtsi
@@ -56,11 +56,21 @@
 			compatible = "fixed-clock";
 			clock-frequency = <0>;
 		};
+
+		clk-lse {
+			#clock-cells = <0>;
+			compatible = "fixed-clock";
+			clock-frequency = <32768>;
+		};
+
+		clk-lsi {
+			#clock-cells = <0>;
+			compatible = "fixed-clock";
+			clock-frequency = <32000>;
+		};
 	};
 
 	soc {
-		dma-ranges = <0xc0000000 0x0 0x10000000>;
-
 		timer2: timer@40000000 {
 			compatible = "st,stm32-timer";
 			reg = <0x40000000 0x400>;
@@ -122,6 +132,9 @@
 			interrupts = <39>;
 			clocks = <&rcc 0 146>;
 			status = "disabled";
+			dmas = <&dma1 1 4 0x400 0x0>,
+			       <&dma1 3 4 0x400 0x0>;
+			dma-names = "rx", "tx";
 		};
 
 		usart4: serial@40004c00 {
@@ -162,6 +175,9 @@
 			interrupts = <37>;
 			clocks = <&rcc 0 164>;
 			status = "disabled";
+			dmas = <&dma2 2 4 0x400 0x0>,
+			       <&dma2 7 4 0x400 0x0>;
+			dma-names = "rx", "tx";
 		};
 
 		usart6: serial@40011400 {
@@ -185,11 +201,18 @@
 			interrupts = <1>, <2>, <3>, <6>, <7>, <8>, <9>, <10>, <23>, <40>, <41>, <42>, <62>, <76>;
 		};
 
+		pwrcfg: power-config@40007000 {
+			compatible = "syscon";
+			reg = <0x40007000 0x400>;
+		};
+
 		pin-controller {
 			#address-cells = <1>;
 			#size-cells = <1>;
 			compatible = "st,stm32f429-pinctrl";
 			ranges = <0 0x40020000 0x3000>;
+			interrupt-parent = <&exti>;
+			st,syscfg = <&syscfg 0x8>;
 			pins-are-numbered;
 
 			gpioa: gpio@40020000 {
@@ -313,7 +336,7 @@
 				};
 			};
 
-			ethernet0_mii: mii@0 {
+			ethernet_mii: mii@0 {
 				pins {
 					pinmux = <STM32F429_PG13_FUNC_ETH_MII_TXD0_ETH_RMII_TXD0>,
 						 <STM32F429_PG14_FUNC_ETH_MII_TXD1_ETH_RMII_TXD1>,
@@ -340,6 +363,7 @@
 			compatible = "st,stm32f42xx-rcc", "st,stm32-rcc";
 			reg = <0x40023800 0x400>;
 			clocks = <&clk_hse>;
+			st,syscfg = <&pwrcfg>;
 		};
 
 		dma1: dma-controller@40026000 {
@@ -373,24 +397,22 @@
 			st,mem2mem;
 		};
 
-		ethernet0: dwmac@40028000 {
+		mac: ethernet@40028000 {
 			compatible = "st,stm32-dwmac", "snps,dwmac-3.50a";
 			reg = <0x40028000 0x8000>;
 			reg-names = "stmmaceth";
-			interrupts = <61>, <62>;
-			interrupt-names = "macirq", "eth_wake_irq";
-			clock-names = "stmmaceth", "tx-clk", "rx-clk";
+			interrupts = <61>;
+			interrupt-names = "macirq";
+			clock-names = "stmmaceth", "mac-clk-tx", "mac-clk-rx";
 			clocks = <&rcc 0 25>, <&rcc 0 26>, <&rcc 0 27>;
 			st,syscon = <&syscfg 0x4>;
 			snps,pbl = <8>;
 			snps,mixed-burst;
-			dma-ranges;
 			status = "disabled";
 		};
 
 		usbotg_hs: usb@40040000 {
 			compatible = "snps,dwc2";
-			dma-ranges;
 			reg = <0x40040000 0x40000>;
 			interrupts = <77>;
 			clocks = <&rcc 0 29>;
diff --git a/arch/arm/boot/dts/stm32f469-disco.dts b/arch/arm/boot/dts/stm32f469-disco.dts
index e911af836471..8877c00ce8e8 100644
--- a/arch/arm/boot/dts/stm32f469-disco.dts
+++ b/arch/arm/boot/dts/stm32f469-disco.dts
@@ -64,6 +64,14 @@
 	aliases {
 		serial0 = &usart3;
 	};
+
+	soc {
+		dma-ranges = <0xc0000000 0x0 0x10000000>;
+	};
+};
+
+&rcc {
+	compatible = "st,stm32f469-rcc", "st,stm32f42xx-rcc", "st,stm32-rcc";
 };
 
 &clk_hse {
diff --git a/arch/arm/boot/dts/stm32f746.dtsi b/arch/arm/boot/dts/stm32f746.dtsi
new file mode 100644
index 000000000000..f321ffe87144
--- /dev/null
+++ b/arch/arm/boot/dts/stm32f746.dtsi
@@ -0,0 +1,304 @@
+/*
+ * Copyright 2015 - Maxime Coquelin <mcoquelin.stm32@gmail.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "skeleton.dtsi"
+#include "armv7-m.dtsi"
+#include <dt-bindings/pinctrl/stm32f746-pinfunc.h>
+
+/ {
+	clocks {
+		clk_hse: clk-hse {
+			#clock-cells = <0>;
+			compatible = "fixed-clock";
+			clock-frequency = <0>;
+		};
+	};
+
+	soc {
+		timer2: timer@40000000 {
+			compatible = "st,stm32-timer";
+			reg = <0x40000000 0x400>;
+			interrupts = <28>;
+			clocks = <&rcc 0 128>;
+			status = "disabled";
+		};
+
+		timer3: timer@40000400 {
+			compatible = "st,stm32-timer";
+			reg = <0x40000400 0x400>;
+			interrupts = <29>;
+			clocks = <&rcc 0 129>;
+			status = "disabled";
+		};
+
+		timer4: timer@40000800 {
+			compatible = "st,stm32-timer";
+			reg = <0x40000800 0x400>;
+			interrupts = <30>;
+			clocks = <&rcc 0 130>;
+			status = "disabled";
+		};
+
+		timer5: timer@40000c00 {
+			compatible = "st,stm32-timer";
+			reg = <0x40000c00 0x400>;
+			interrupts = <50>;
+			clocks = <&rcc 0 131>;
+		};
+
+		timer6: timer@40001000 {
+			compatible = "st,stm32-timer";
+			reg = <0x40001000 0x400>;
+			interrupts = <54>;
+			clocks = <&rcc 0 132>;
+			status = "disabled";
+		};
+
+		timer7: timer@40001400 {
+			compatible = "st,stm32-timer";
+			reg = <0x40001400 0x400>;
+			interrupts = <55>;
+			clocks = <&rcc 0 133>;
+			status = "disabled";
+		};
+
+		usart2: serial@40004400 {
+			compatible = "st,stm32f7-usart", "st,stm32f7-uart";
+			reg = <0x40004400 0x400>;
+			interrupts = <38>;
+			clocks =  <&rcc 0 145>;
+			status = "disabled";
+		};
+
+		usart3: serial@40004800 {
+			compatible = "st,stm32f7-usart", "st,stm32f7-uart";
+			reg = <0x40004800 0x400>;
+			interrupts = <39>;
+			clocks = <&rcc 0 146>;
+			status = "disabled";
+		};
+
+		usart4: serial@40004c00 {
+			compatible = "st,stm32f7-uart";
+			reg = <0x40004c00 0x400>;
+			interrupts = <52>;
+			clocks = <&rcc 0 147>;
+			status = "disabled";
+		};
+
+		usart5: serial@40005000 {
+			compatible = "st,stm32f7-uart";
+			reg = <0x40005000 0x400>;
+			interrupts = <53>;
+			clocks = <&rcc 0 148>;
+			status = "disabled";
+		};
+
+		usart7: serial@40007800 {
+			compatible = "st,stm32f7-usart", "st,stm32f7-uart";
+			reg = <0x40007800 0x400>;
+			interrupts = <82>;
+			clocks = <&rcc 0 158>;
+			status = "disabled";
+		};
+
+		usart8: serial@40007c00 {
+			compatible = "st,stm32f7-usart", "st,stm32f7-uart";
+			reg = <0x40007c00 0x400>;
+			interrupts = <83>;
+			clocks = <&rcc 0 159>;
+			status = "disabled";
+		};
+
+		usart1: serial@40011000 {
+			compatible = "st,stm32f7-usart", "st,stm32f7-uart";
+			reg = <0x40011000 0x400>;
+			interrupts = <37>;
+			clocks = <&rcc 0 164>;
+			status = "disabled";
+		};
+
+		usart6: serial@40011400 {
+			compatible = "st,stm32f7-usart", "st,stm32f7-uart";
+			reg = <0x40011400 0x400>;
+			interrupts = <71>;
+			clocks = <&rcc 0 165>;
+			status = "disabled";
+		};
+
+		syscfg: system-config@40013800 {
+			compatible = "syscon";
+			reg = <0x40013800 0x400>;
+		};
+
+		exti: interrupt-controller@40013c00 {
+			compatible = "st,stm32-exti";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			reg = <0x40013C00 0x400>;
+			interrupts = <1>, <2>, <3>, <6>, <7>, <8>, <9>, <10>, <23>, <40>, <41>, <42>, <62>, <76>;
+		};
+
+		pin-controller {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "st,stm32f746-pinctrl";
+			ranges = <0 0x40020000 0x3000>;
+			interrupt-parent = <&exti>;
+			st,syscfg = <&syscfg 0x8>;
+			pins-are-numbered;
+
+			gpioa: gpio@40020000 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				reg = <0x0 0x400>;
+				clocks = <&rcc 0 256>;
+				st,bank-name = "GPIOA";
+			};
+
+			gpiob: gpio@40020400 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				reg = <0x400 0x400>;
+				clocks = <&rcc 0 257>;
+				st,bank-name = "GPIOB";
+			};
+
+			gpioc: gpio@40020800 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				reg = <0x800 0x400>;
+				clocks = <&rcc 0 258>;
+				st,bank-name = "GPIOC";
+			};
+
+			gpiod: gpio@40020c00 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				reg = <0xc00 0x400>;
+				clocks = <&rcc 0 259>;
+				st,bank-name = "GPIOD";
+			};
+
+			gpioe: gpio@40021000 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				reg = <0x1000 0x400>;
+				clocks = <&rcc 0 260>;
+				st,bank-name = "GPIOE";
+			};
+
+			gpiof: gpio@40021400 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				reg = <0x1400 0x400>;
+				clocks = <&rcc 0 261>;
+				st,bank-name = "GPIOF";
+			};
+
+			gpiog: gpio@40021800 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				reg = <0x1800 0x400>;
+				clocks = <&rcc 0 262>;
+				st,bank-name = "GPIOG";
+			};
+
+			gpioh: gpio@40021c00 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				reg = <0x1c00 0x400>;
+				clocks = <&rcc 0 263>;
+				st,bank-name = "GPIOH";
+			};
+
+			gpioi: gpio@40022000 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				reg = <0x2000 0x400>;
+				clocks = <&rcc 0 264>;
+				st,bank-name = "GPIOI";
+			};
+
+			gpioj: gpio@40022400 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				reg = <0x2400 0x400>;
+				clocks = <&rcc 0 265>;
+				st,bank-name = "GPIOJ";
+			};
+
+			gpiok: gpio@40022800 {
+				gpio-controller;
+				#gpio-cells = <2>;
+				reg = <0x2800 0x400>;
+				clocks = <&rcc 0 266>;
+				st,bank-name = "GPIOK";
+			};
+
+			usart1_pins_a: usart1@0 {
+				pins1 {
+					pinmux = <STM32F746_PA9_FUNC_USART1_TX>;
+					bias-disable;
+					drive-push-pull;
+					slew-rate = <0>;
+				};
+				pins2 {
+					pinmux = <STM32F746_PA10_FUNC_USART1_RX>;
+					bias-disable;
+				};
+			};
+		};
+
+		rcc: rcc@40023800 {
+			#clock-cells = <2>;
+			compatible = "st,stm32f42xx-rcc", "st,stm32-rcc";
+			reg = <0x40023800 0x400>;
+			clocks = <&clk_hse>;
+		};
+	};
+};
+
+&systick {
+	clocks = <&rcc 1 0>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/sun4i-a10.dtsi b/arch/arm/boot/dts/sun4i-a10.dtsi
index 7e7dfc2b43db..b14a4281058d 100644
--- a/arch/arm/boot/dts/sun4i-a10.dtsi
+++ b/arch/arm/boot/dts/sun4i-a10.dtsi
@@ -967,7 +967,8 @@
 			compatible = "allwinner,sun4i-a10-pinctrl";
 			reg = <0x01c20800 0x400>;
 			interrupts = <28>;
-			clocks = <&apb0_gates 5>;
+			clocks = <&apb0_gates 5>, <&osc24M>, <&osc32k>;
+			clock-names = "apb", "hosc", "losc";
 			gpio-controller;
 			interrupt-controller;
 			#interrupt-cells = <3>;
diff --git a/arch/arm/boot/dts/sun5i-a10s-olinuxino-micro.dts b/arch/arm/boot/dts/sun5i-a10s-olinuxino-micro.dts
index aef91476f9ae..0684d7930d65 100644
--- a/arch/arm/boot/dts/sun5i-a10s-olinuxino-micro.dts
+++ b/arch/arm/boot/dts/sun5i-a10s-olinuxino-micro.dts
@@ -250,8 +250,8 @@
 
 &spi2 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&spi2_pins_a>,
-		    <&spi2_cs0_pins_a>;
+	pinctrl-0 = <&spi2_pins_b>,
+		    <&spi2_cs0_pins_b>;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/sun5i-a10s.dtsi b/arch/arm/boot/dts/sun5i-a10s.dtsi
index c41a2ba34dde..7aa8c7aa0153 100644
--- a/arch/arm/boot/dts/sun5i-a10s.dtsi
+++ b/arch/arm/boot/dts/sun5i-a10s.dtsi
@@ -243,14 +243,14 @@
 		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 	};
 
-	spi2_pins_a: spi2@0 {
+	spi2_pins_b: spi2@1 {
 		allwinner,pins = "PB12", "PB13", "PB14";
 		allwinner,function = "spi2";
 		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
 		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 	};
 
-	spi2_cs0_pins_a: spi2_cs0@0 {
+	spi2_cs0_pins_b: spi2_cs0@1 {
 		allwinner,pins = "PB11";
 		allwinner,function = "spi2";
 		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
diff --git a/arch/arm/boot/dts/sun5i-a13-olinuxino.dts b/arch/arm/boot/dts/sun5i-a13-olinuxino.dts
index b3c234c65ea1..bb7210e0e4a9 100644
--- a/arch/arm/boot/dts/sun5i-a13-olinuxino.dts
+++ b/arch/arm/boot/dts/sun5i-a13-olinuxino.dts
@@ -72,6 +72,47 @@
 			default-state = "on";
 		};
 	};
+
+	bridge {
+		compatible = "dumb-vga-dac";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				vga_bridge_in: endpoint {
+					remote-endpoint = <&tcon0_out_vga>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				vga_bridge_out: endpoint {
+					remote-endpoint = <&vga_con_in>;
+				};
+			};
+		};
+	};
+
+	vga {
+		compatible = "vga-connector";
+
+		port {
+			vga_con_in: endpoint {
+				remote-endpoint = <&vga_bridge_out>;
+			};
+		};
+	};
+};
+
+&be0 {
+	status = "okay";
 };
 
 &ehci0 {
@@ -211,6 +252,19 @@
 	status = "okay";
 };
 
+&tcon0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&lcd_rgb666_pins>;
+	status = "okay";
+};
+
+&tcon0_out {
+	tcon0_out_vga: endpoint@0 {
+		reg = <0>;
+		remote-endpoint = <&vga_bridge_in>;
+	};
+};
+
 &uart1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart1_pins_b>;
diff --git a/arch/arm/boot/dts/sun5i-a13-utoo-p66.dts b/arch/arm/boot/dts/sun5i-a13-utoo-p66.dts
index a8b0bcc04514..3d7ff10a48e9 100644
--- a/arch/arm/boot/dts/sun5i-a13-utoo-p66.dts
+++ b/arch/arm/boot/dts/sun5i-a13-utoo-p66.dts
@@ -83,22 +83,6 @@
 	allwinner,pins = "PG3";
 };
 
-&i2c1 {
-	icn8318: touchscreen@40 {
-		compatible = "chipone,icn8318";
-		reg = <0x40>;
-		interrupt-parent = <&pio>;
-		interrupts = <6 9 IRQ_TYPE_EDGE_FALLING>; /* EINT9 (PG9) */
-		pinctrl-names = "default";
-		pinctrl-0 = <&ts_wake_pin_p66>;
-		wake-gpios = <&pio 1 3 GPIO_ACTIVE_HIGH>; /* PB3 */
-		touchscreen-size-x = <800>;
-		touchscreen-size-y = <480>;
-		touchscreen-inverted-x;
-		touchscreen-swapped-x-y;
-	};
-};
-
 &mmc2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc2_pins_a>;
@@ -121,20 +105,26 @@
 		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
 		allwinner,pull = <SUN4I_PINCTRL_PULL_UP>;
 	};
-
-	ts_wake_pin_p66: ts_wake_pin@0 {
-		allwinner,pins = "PB3";
-		allwinner,function = "gpio_out";
-		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
-		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
-	};
-
 };
 
 &reg_usb0_vbus {
 	gpio = <&pio 1 4 GPIO_ACTIVE_HIGH>; /* PB4 */
 };
 
+&touchscreen {
+	compatible = "chipone,icn8318";
+	reg = <0x40>;
+	/* The P66 uses a different EINT then the reference design */
+	interrupts = <6 9 IRQ_TYPE_EDGE_FALLING>; /* EINT9 (PG9) */
+	/* The icn8318 binding expects wake-gpios instead of power-gpios */
+	wake-gpios = <&pio 1 3 GPIO_ACTIVE_HIGH>; /* PB3 */
+	touchscreen-size-x = <800>;
+	touchscreen-size-y = <480>;
+	touchscreen-inverted-x;
+	touchscreen-swapped-x-y;
+	status = "okay";
+};
+
 &uart1 {
 	/* The P66 uses the uart pins as gpios */
 	status = "disabled";
diff --git a/arch/arm/boot/dts/sun5i-gr8-chip-pro.dts b/arch/arm/boot/dts/sun5i-gr8-chip-pro.dts
new file mode 100644
index 000000000000..92a2dc6250a5
--- /dev/null
+++ b/arch/arm/boot/dts/sun5i-gr8-chip-pro.dts
@@ -0,0 +1,266 @@
+/*
+ * Copyright 2016 Free Electrons
+ * Copyright 2016 NextThing Co
+ *
+ * Maxime Ripard <maxime.ripard@free-electrons.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+#include "sun5i-gr8.dtsi"
+#include "sunxi-common-regulators.dtsi"
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	model = "NextThing C.H.I.P. Pro";
+	compatible = "nextthing,chip-pro", "nextthing,gr8";
+
+	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		serial0 = &uart1;
+		serial1 = &uart2;
+		serial2 = &uart3;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		status {
+			label = "chip-pro:white:status";
+			gpios = <&axp_gpio 2 GPIO_ACTIVE_HIGH>;
+			default-state = "on";
+		};
+	};
+
+	mmc0_pwrseq: mmc0_pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_reg_on_pin_chip_pro>;
+		reset-gpios = <&pio 1 10 GPIO_ACTIVE_LOW>; /* PB10 */
+	};
+};
+
+&codec {
+	status = "okay";
+};
+
+&ehci0 {
+	status = "okay";
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_pins_a>;
+	status = "okay";
+
+	axp209: pmic@34 {
+		reg = <0x34>;
+
+		/*
+		* The interrupt is routed through the "External Fast
+		* Interrupt Request" pin (ball G13 of the module)
+		* directly to the main interrupt controller, without
+		* any other controller interfering.
+		*/
+		interrupts = <0>;
+	};
+};
+
+#include "axp209.dtsi"
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins_a>;
+	status = "disabled";
+};
+
+&i2s0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2s0_mclk_pins_a>, <&i2s0_data_pins_a>;
+	status = "disabled";
+};
+
+&mmc0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc0_pins_a>;
+	vmmc-supply = <&reg_vcc3v3>;
+	mmc-pwrseq = <&mmc0_pwrseq>;
+	bus-width = <4>;
+	non-removable;
+	status = "okay";
+};
+
+&nfc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&nand_pins_a &nand_cs0_pins_a &nand_rb0_pins_a>;
+	status = "okay";
+
+	nand@0 {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		reg = <0>;
+		allwinner,rb = <0>;
+		nand-ecc-mode = "hw";
+	};
+};
+
+&ohci0 {
+	status = "okay";
+};
+
+&otg_sram {
+	status = "okay";
+};
+
+&pio {
+	usb0_id_pin_chip_pro: usb0-id-pin@0 {
+		allwinner,pins = "PG2";
+		allwinner,function = "gpio_in";
+		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+	};
+
+	wifi_reg_on_pin_chip_pro: wifi-reg-on-pin@0 {
+		allwinner,pins = "PB10";
+		allwinner,function = "gpio_out";
+		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+	};
+};
+
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm0_pins_a>, <&pwm1_pins>;
+	status = "disabled";
+};
+
+&reg_dcdc2 {
+	regulator-min-microvolt = <1000000>;
+	regulator-max-microvolt = <1400000>;
+	regulator-name = "vdd-cpu";
+	regulator-always-on;
+};
+
+&reg_dcdc3 {
+	regulator-min-microvolt = <1000000>;
+	regulator-max-microvolt = <1300000>;
+	regulator-name = "vdd-sys";
+	regulator-always-on;
+};
+
+&reg_ldo1 {
+	regulator-name = "vdd-rtc";
+};
+
+&reg_ldo2 {
+	regulator-min-microvolt = <2700000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "avcc";
+	regulator-always-on;
+};
+
+/*
+ * Both LDO3 and LDO4 are used in parallel to power up the
+ * WiFi/BT chip.
+ */
+&reg_ldo3 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-wifi-1";
+	regulator-always-on;
+};
+
+&reg_ldo4 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-wifi-2";
+	regulator-always-on;
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart1_pins_a>, <&uart1_cts_rts_pins_a>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_pins_a>, <&uart2_cts_rts_pins_a>;
+	status = "disabled";
+};
+
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart3_pins_a>, <&uart3_cts_rts_pins_a>;
+	status = "okay";
+};
+
+&usb_otg {
+	/*
+	 * The CHIP Pro doesn't have a controllable VBUS, nor does it
+	 * have any 5v rail on the board itself.
+	 *
+	 * If one wants to use it as a true OTG port, it should be
+	 * done in the baseboard, and its DT / overlay will add it.
+	 */
+	dr_mode = "otg";
+	status = "okay";
+};
+
+&usb_power_supply {
+	status = "okay";
+};
+
+&usbphy {
+	pinctrl-names = "default";
+	pinctrl-0 = <&usb0_id_pin_chip_pro>;
+	usb0_id_det-gpio = <&pio 6 2 GPIO_ACTIVE_HIGH>; /* PG2 */
+	usb0_vbus_power-supply = <&usb_power_supply>;
+	usb1_vbus-supply = <&reg_vcc5v0>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/sun5i-gr8-evb.dts b/arch/arm/boot/dts/sun5i-gr8-evb.dts
index 714381fd64d7..030605aa8065 100644
--- a/arch/arm/boot/dts/sun5i-gr8-evb.dts
+++ b/arch/arm/boot/dts/sun5i-gr8-evb.dts
@@ -75,6 +75,39 @@
 		brightness-levels = <0 10 20 30 40 50 60 70 80 90 100>;
 		default-brightness-level = <8>;
 	};
+
+	sound-analog {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "gr8-evb-wm8978";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,mclk-fs = <512>;
+
+		simple-audio-card,cpu {
+			sound-dai = <&i2s0>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&wm8978>;
+		};
+	};
+
+	sound-spdif {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "On-board SPDIF";
+
+		simple-audio-card,cpu {
+			sound-dai = <&spdif>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&spdif_out>;
+		};
+	};
+
+	spdif_out: spdif-out {
+		#sound-dai-cells = <0>;
+		compatible = "linux,spdif-dit";
+	};
 };
 
 &be0 {
diff --git a/arch/arm/boot/dts/sun5i-gr8.dtsi b/arch/arm/boot/dts/sun5i-gr8.dtsi
index ca54e03ef366..ea86d4d58db6 100644
--- a/arch/arm/boot/dts/sun5i-gr8.dtsi
+++ b/arch/arm/boot/dts/sun5i-gr8.dtsi
@@ -792,7 +792,7 @@
 			};
 
 			i2s0_mclk_pins_a: i2s0-mclk@0 {
-				allwinner,pins = "PB6", "PB7", "PB8", "PB9";
+				allwinner,pins = "PB5";
 				allwinner,function = "i2s0";
 				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
 				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
@@ -854,6 +854,13 @@
 				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 			};
 
+			pwm1_pins: pwm1 {
+				allwinner,pins = "PG13";
+				allwinner,function = "pwm1";
+				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
 			spdif_tx_pins_a: spdif@0 {
 				allwinner,pins = "PB10";
 				allwinner,function = "spdif";
@@ -874,6 +881,34 @@
 				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
 				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 			};
+
+			uart2_pins_a: uart2@1 {
+				allwinner,pins = "PD2", "PD3";
+				allwinner,function = "uart2";
+				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
+			uart2_cts_rts_pins_a: uart2-cts-rts@0 {
+				allwinner,pins = "PD4", "PD5";
+				allwinner,function = "uart2";
+				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
+			uart3_pins_a: uart3@1 {
+				allwinner,pins = "PG9", "PG10";
+				allwinner,function = "uart3";
+				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
+			uart3_cts_rts_pins_a: uart3-cts-rts@0 {
+				allwinner,pins = "PG11", "PG12";
+				allwinner,function = "uart3";
+				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
 		};
 
 		pwm: pwm@01c20e00 {
@@ -978,6 +1013,16 @@
 			status = "disabled";
 		};
 
+		uart3: serial@01c28c00 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x01c28c00 0x400>;
+			interrupts = <4>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			clocks = <&apb1_gates 19>;
+			status = "disabled";
+		};
+
 		i2c0: i2c@01c2ac00 {
 			compatible = "allwinner,sun4i-a10-i2c";
 			reg = <0x01c2ac00 0x400>;
diff --git a/arch/arm/boot/dts/sun5i-r8-chip.dts b/arch/arm/boot/dts/sun5i-r8-chip.dts
index b68a12374b35..c6da5ad37152 100644
--- a/arch/arm/boot/dts/sun5i-r8-chip.dts
+++ b/arch/arm/boot/dts/sun5i-r8-chip.dts
@@ -56,9 +56,11 @@
 
 	aliases {
 		i2c0 = &i2c0;
+		i2c1 = &i2c1;
 		i2c2 = &i2c2;
 		serial0 = &uart1;
 		serial1 = &uart3;
+		spi0 = &spi2;
 	};
 
 	chosen {
@@ -74,6 +76,20 @@
 			default-state = "on";
 		};
 	};
+
+	mmc0_pwrseq: mmc0_pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-names = "default";
+		pinctrl-0 = <&chip_wifi_reg_on_pin>;
+		reset-gpios = <&pio 2 19 GPIO_ACTIVE_LOW>; /* PC19 */
+	};
+
+	onewire {
+		compatible = "w1-gpio";
+		gpios = <&pio 3 2 GPIO_ACTIVE_HIGH>; /* PD2 */
+		pinctrl-names = "default";
+		pinctrl-0 = <&chip_w1_pin>;
+	};
 };
 
 &be0 {
@@ -112,6 +128,12 @@
 
 #include "axp209.dtsi"
 
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins_a>;
+	status = "disabled";
+};
+
 &i2c2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c2_pins_a>;
@@ -131,10 +153,15 @@
 	};
 };
 
+&mmc0_pins_a {
+	allwinner,pull = <SUN4I_PINCTRL_PULL_UP>;
+};
+
 &mmc0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc0_pins_a>;
 	vmmc-supply = <&reg_vcc3v3>;
+	mmc-pwrseq = <&mmc0_pwrseq>;
 	bus-width = <4>;
 	non-removable;
 	status = "okay";
@@ -156,12 +183,26 @@
 		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 	};
 
+	chip_wifi_reg_on_pin: chip_wifi_reg_on_pin@0 {
+	        allwinner,pins = "PC19";
+	        allwinner,function = "gpio_out";
+	        allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+	        allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+	};
+
 	chip_id_det_pin: chip_id_det_pin@0 {
 		allwinner,pins = "PG2";
 		allwinner,function = "gpio_in";
 		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
 		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 	};
+
+	chip_w1_pin: chip_w1_pin@0 {
+		allwinner,pins = "PD2";
+		allwinner,function = "gpio_in";
+	        allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+	        allwinner,pull = <SUN4I_PINCTRL_PULL_UP>;
+	};
 };
 
 &reg_dcdc2 {
@@ -189,6 +230,28 @@
 	regulator-always-on;
 };
 
+/*
+ * Both LDO3 and LDO4 are used in parallel to power up the WiFi/BT
+ * Chip.
+ *
+ * If those are not enabled, the SDIO part will not enumerate, and
+ * since there's no way currently to pass DT infos to an SDIO device,
+ * we cannot really do better than this ugly hack for now.
+ */
+&reg_ldo3 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-wifi-1";
+	regulator-always-on;
+};
+
+&reg_ldo4 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-wifi-2";
+	regulator-always-on;
+};
+
 &reg_ldo5 {
 	regulator-min-microvolt = <1800000>;
 	regulator-max-microvolt = <1800000>;
@@ -202,6 +265,12 @@
 	status = "okay";
 };
 
+&spi2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi2_pins_a>;
+	status = "disabled";
+};
+
 &tcon0 {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/sun5i-reference-design-tablet.dtsi b/arch/arm/boot/dts/sun5i-reference-design-tablet.dtsi
index 20cc940f5f91..82f87cdcd164 100644
--- a/arch/arm/boot/dts/sun5i-reference-design-tablet.dtsi
+++ b/arch/arm/boot/dts/sun5i-reference-design-tablet.dtsi
@@ -41,6 +41,7 @@
  */
 #include "sunxi-reference-design-tablet.dtsi"
 
+#include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/pwm/pwm.h>
 
 / {
@@ -84,6 +85,23 @@
 };
 
 &i2c1 {
+	/*
+	 * The gsl1680 is rated at 400KHz and it will not work reliable at
+	 * 100KHz, this has been confirmed on multiple different q8 tablets.
+	 * All other devices on this bus are also rated for 400KHz.
+	 */
+	clock-frequency = <400000>;
+
+	touchscreen: touchscreen {
+		interrupt-parent = <&pio>;
+		interrupts = <6 11 IRQ_TYPE_EDGE_FALLING>; /* EINT11 (PG11) */
+		pinctrl-names = "default";
+		pinctrl-0 = <&ts_power_pin>;
+		power-gpios = <&pio 1 3 GPIO_ACTIVE_HIGH>; /* PB3 */
+		/* Tablet dts must provide reg and compatible */
+		status = "disabled";
+	};
+
 	pcf8563: rtc@51 {
 		compatible = "nxp,pcf8563";
 		reg = <0x51>;
@@ -125,6 +143,13 @@
 		allwinner,pull = <SUN4I_PINCTRL_PULL_UP>;
 	};
 
+	ts_power_pin: ts_power_pin {
+		pins = "PB3";
+		function = "gpio_out";
+		drive-strength = <10>;
+		bias-disable;
+	};
+
 	usb0_vbus_detect_pin: usb0_vbus_detect_pin@0 {
 		allwinner,pins = "PG1";
 		allwinner,function = "gpio_in";
diff --git a/arch/arm/boot/dts/sun5i.dtsi b/arch/arm/boot/dts/sun5i.dtsi
index e374f4fc8073..b0fca4ef4dae 100644
--- a/arch/arm/boot/dts/sun5i.dtsi
+++ b/arch/arm/boot/dts/sun5i.dtsi
@@ -547,7 +547,8 @@
 		pio: pinctrl@01c20800 {
 			reg = <0x01c20800 0x400>;
 			interrupts = <28>;
-			clocks = <&apb0_gates 5>;
+			clocks = <&apb0_gates 5>, <&osc24M>, <&osc32k>;
+			clock-names = "apb", "hosc", "losc";
 			gpio-controller;
 			interrupt-controller;
 			#interrupt-cells = <3>;
@@ -574,6 +575,16 @@
 				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 			};
 
+			lcd_rgb565_pins: lcd_rgb565@0 {
+				allwinner,pins = "PD3", "PD4", "PD5", "PD6", "PD7",
+						 "PD10", "PD11", "PD12", "PD13", "PD14", "PD15",
+						 "PD19", "PD20", "PD21", "PD22", "PD23",
+						 "PD24", "PD25", "PD26", "PD27";
+				allwinner,function = "lcd0";
+				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
 			mmc0_pins_a: mmc0@0 {
 				allwinner,pins = "PF0", "PF1", "PF2", "PF3",
 						 "PF4", "PF5";
@@ -591,6 +602,20 @@
 				allwinner,pull = <SUN4I_PINCTRL_PULL_UP>;
 			};
 
+			spi2_pins_a: spi2@0 {
+				allwinner,pins = "PE1", "PE2", "PE3";
+				allwinner,function = "spi2";
+				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
+			spi2_cs0_pins_a: spi2-cs0@0 {
+				allwinner,pins = "PE0";
+				allwinner,function = "spi2";
+				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
 			uart3_pins_a: uart3@0 {
 				allwinner,pins = "PG9", "PG10";
 				allwinner,function = "uart3";
diff --git a/arch/arm/boot/dts/sun6i-a31-hummingbird.dts b/arch/arm/boot/dts/sun6i-a31-hummingbird.dts
index 9a74637f677f..735914f6ae44 100644
--- a/arch/arm/boot/dts/sun6i-a31-hummingbird.dts
+++ b/arch/arm/boot/dts/sun6i-a31-hummingbird.dts
@@ -63,12 +63,79 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	vga-connector {
+		compatible = "vga-connector";
+
+		port {
+			vga_con_in: endpoint {
+				remote-endpoint = <&vga_dac_out>;
+			};
+		};
+	};
+
+	vga-dac {
+		compatible = "dumb-vga-dac";
+		vdd-supply = <&reg_vga_3v3>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+
+				vga_dac_in: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&tcon0_out_vga>;
+				};
+			};
+
+			port@1 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <1>;
+
+				vga_dac_out: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&vga_con_in>;
+				};
+			};
+		};
+	};
+
+	reg_vga_3v3: vga_3v3_regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vga-3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		enable-active-high;
+		gpio = <&pio 7 25 GPIO_ACTIVE_HIGH>; /* PH25 */
+	};
+
 	wifi_pwrseq: wifi_pwrseq {
 		compatible = "mmc-pwrseq-simple";
 		reset-gpios = <&pio 6 10 GPIO_ACTIVE_LOW>; /* PG10 */
 	};
 };
 
+&codec {
+	allwinner,audio-routing =
+		"Headphone", "HP",
+		"Speaker", "LINEOUT",
+		"LINEIN", "Line In",
+		"MIC1", "Mic",
+		"MIC2", "Headset Mic",
+		"Mic",	"MBIAS",
+		"Headset Mic", "HBIAS";
+	allwinner,pa-gpios = <&pio 7 22 GPIO_ACTIVE_HIGH>; /* PH22 */
+	status = "okay";
+};
+
 &cpu0 {
 	cpu-supply = <&reg_dcdc3>;
 };
@@ -245,6 +312,19 @@
 	status = "okay";
 };
 
+&tcon0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&lcd0_rgb888_pins>;
+	status = "okay";
+};
+
+&tcon0_out {
+	tcon0_out_vga: endpoint@0 {
+		reg = <0>;
+		remote-endpoint = <&vga_dac_in>;
+	};
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_pins_a>;
diff --git a/arch/arm/boot/dts/sun6i-a31.dtsi b/arch/arm/boot/dts/sun6i-a31.dtsi
index ce1960453a0b..2b26175d55d1 100644
--- a/arch/arm/boot/dts/sun6i-a31.dtsi
+++ b/arch/arm/boot/dts/sun6i-a31.dtsi
@@ -231,6 +231,11 @@
 		};
 	};
 
+	de: display-engine {
+		compatible = "allwinner,sun6i-a31-display-engine";
+		allwinner,pipelines = <&fe0>;
+	};
+
 	soc@01c00000 {
 		compatible = "simple-bus";
 		#address-cells = <1>;
@@ -246,6 +251,44 @@
 			#dma-cells = <1>;
 		};
 
+		tcon0: lcd-controller@01c0c000 {
+			compatible = "allwinner,sun6i-a31-tcon";
+			reg = <0x01c0c000 0x1000>;
+			interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
+			resets = <&ccu RST_AHB1_LCD0>;
+			reset-names = "lcd";
+			clocks = <&ccu CLK_AHB1_LCD0>,
+				 <&ccu CLK_LCD0_CH0>,
+				 <&ccu CLK_LCD0_CH1>;
+			clock-names = "ahb",
+				      "tcon-ch0",
+				      "tcon-ch1";
+			clock-output-names = "tcon0-pixel-clock";
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				tcon0_in: port@0 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0>;
+
+					tcon0_in_drc0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&drc0_out_tcon0>;
+					};
+				};
+
+				tcon0_out: port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <1>;
+				};
+			};
+		};
+
 		mmc0: mmc@01c0f000 {
 			compatible = "allwinner,sun7i-a20-mmc";
 			reg = <0x01c0f000 0x1000>;
@@ -428,19 +471,55 @@
 				     <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&ccu CLK_APB1_PIO>;
+			clocks = <&ccu CLK_APB1_PIO>, <&osc24M>, <&osc32k>;
+			clock-names = "apb", "hosc", "losc";
 			gpio-controller;
 			interrupt-controller;
 			#interrupt-cells = <3>;
 			#gpio-cells = <3>;
 
-			uart0_pins_a: uart0@0 {
-				allwinner,pins = "PH20", "PH21";
-				allwinner,function = "uart0";
+			gmac_pins_gmii_a: gmac_gmii@0 {
+				allwinner,pins = "PA0", "PA1", "PA2", "PA3",
+						"PA4", "PA5", "PA6", "PA7",
+						"PA8", "PA9", "PA10", "PA11",
+						"PA12", "PA13", "PA14",	"PA15",
+						"PA16", "PA17", "PA18", "PA19",
+						"PA20", "PA21", "PA22", "PA23",
+						"PA24", "PA25", "PA26", "PA27";
+				allwinner,function = "gmac";
+				/*
+				 * data lines in GMII mode run at 125MHz and
+				 * might need a higher signal drive strength
+				 */
+				allwinner,drive = <SUN4I_PINCTRL_30_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
+			gmac_pins_mii_a: gmac_mii@0 {
+				allwinner,pins = "PA0", "PA1", "PA2", "PA3",
+						"PA8", "PA9", "PA11",
+						"PA12", "PA13", "PA14", "PA19",
+						"PA20", "PA21", "PA22", "PA23",
+						"PA24", "PA26", "PA27";
+				allwinner,function = "gmac";
 				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
 				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 			};
 
+			gmac_pins_rgmii_a: gmac_rgmii@0 {
+				allwinner,pins = "PA0", "PA1", "PA2", "PA3",
+						"PA9", "PA10", "PA11",
+						"PA12", "PA13", "PA14", "PA19",
+						"PA20", "PA25", "PA26", "PA27";
+				allwinner,function = "gmac";
+				/*
+				 * data lines in RGMII mode use DDR mode
+				 * and need a higher signal drive strength
+				 */
+				allwinner,drive = <SUN4I_PINCTRL_40_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
 			i2c0_pins_a: i2c0@0 {
 				allwinner,pins = "PH14", "PH15";
 				allwinner,function = "i2c0";
@@ -462,6 +541,19 @@
 				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 			};
 
+			lcd0_rgb888_pins: lcd0_rgb888 {
+				allwinner,pins = "PD0", "PD1", "PD2", "PD3",
+						 "PD4", "PD5", "PD6", "PD7",
+						 "PD8", "PD9", "PD10", "PD11",
+						 "PD12", "PD13", "PD14", "PD15",
+						 "PD16", "PD17", "PD18", "PD19",
+						 "PD20", "PD21", "PD22", "PD23",
+						 "PD24", "PD25", "PD26", "PD27";
+				allwinner,function = "lcd0";
+				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
 			mmc0_pins_a: mmc0@0 {
 				allwinner,pins = "PF0", "PF1", "PF2",
 						 "PF3", "PF4", "PF5";
@@ -506,47 +598,12 @@
 				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 			};
 
-			gmac_pins_mii_a: gmac_mii@0 {
-				allwinner,pins = "PA0", "PA1", "PA2", "PA3",
-						"PA8", "PA9", "PA11",
-						"PA12", "PA13", "PA14", "PA19",
-						"PA20", "PA21", "PA22", "PA23",
-						"PA24", "PA26", "PA27";
-				allwinner,function = "gmac";
+			uart0_pins_a: uart0@0 {
+				allwinner,pins = "PH20", "PH21";
+				allwinner,function = "uart0";
 				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
 				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 			};
-
-			gmac_pins_gmii_a: gmac_gmii@0 {
-				allwinner,pins = "PA0", "PA1", "PA2", "PA3",
-						"PA4", "PA5", "PA6", "PA7",
-						"PA8", "PA9", "PA10", "PA11",
-						"PA12", "PA13", "PA14",	"PA15",
-						"PA16", "PA17", "PA18", "PA19",
-						"PA20", "PA21", "PA22", "PA23",
-						"PA24", "PA25", "PA26", "PA27";
-				allwinner,function = "gmac";
-				/*
-				 * data lines in GMII mode run at 125MHz and
-				 * might need a higher signal drive strength
-				 */
-				allwinner,drive = <SUN4I_PINCTRL_30_MA>;
-				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
-			};
-
-			gmac_pins_rgmii_a: gmac_rgmii@0 {
-				allwinner,pins = "PA0", "PA1", "PA2", "PA3",
-						"PA9", "PA10", "PA11",
-						"PA12", "PA13", "PA14", "PA19",
-						"PA20", "PA25", "PA26", "PA27";
-				allwinner,function = "gmac";
-				/*
-				 * data lines in RGMII mode use DDR mode
-				 * and need a higher signal drive strength
-				 */
-				allwinner,drive = <SUN4I_PINCTRL_40_MA>;
-				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
-			};
 		};
 
 		timer@01c20c00 {
@@ -728,6 +785,19 @@
 			reset-names = "ahb";
 		};
 
+		codec: codec@01c22c00 {
+			#sound-dai-cells = <0>;
+			compatible = "allwinner,sun6i-a31-codec";
+			reg = <0x01c22c00 0x400>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_APB1_CODEC>, <&ccu CLK_CODEC>;
+			clock-names = "apb", "codec";
+			resets = <&ccu RST_APB1_CODEC>;
+			dmas = <&dma 15>, <&dma 15>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
 		timer@01c60000 {
 			compatible = "allwinner,sun6i-a31-hstimer",
 				     "allwinner,sun7i-a20-hstimer";
@@ -799,6 +869,115 @@
 			interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
 		};
 
+		fe0: display-frontend@01e00000 {
+			compatible = "allwinner,sun6i-a31-display-frontend";
+			reg = <0x01e00000 0x20000>;
+			interrupts = <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_AHB1_FE0>, <&ccu CLK_FE0>,
+				 <&ccu CLK_DRAM_FE0>;
+			clock-names = "ahb", "mod",
+				      "ram";
+			resets = <&ccu RST_AHB1_FE0>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				fe0_out: port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <1>;
+
+					fe0_out_be0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&be0_in_fe0>;
+					};
+				};
+			};
+		};
+
+		be0: display-backend@01e60000 {
+			compatible = "allwinner,sun6i-a31-display-backend";
+			reg = <0x01e60000 0x10000>;
+			interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_AHB1_BE0>, <&ccu CLK_BE0>,
+				 <&ccu CLK_DRAM_BE0>;
+			clock-names = "ahb", "mod",
+				      "ram";
+			resets = <&ccu RST_AHB1_BE0>;
+
+			assigned-clocks = <&ccu CLK_BE0>;
+			assigned-clock-rates = <300000000>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				be0_in: port@0 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0>;
+
+					be0_in_fe0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&fe0_out_be0>;
+					};
+				};
+
+				be0_out: port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <1>;
+
+					be0_out_drc0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&drc0_in_be0>;
+					};
+				};
+			};
+		};
+
+		drc0: drc@01e70000 {
+			compatible = "allwinner,sun6i-a31-drc";
+			reg = <0x01e70000 0x10000>;
+			interrupts = <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_AHB1_DRC0>, <&ccu CLK_IEP_DRC0>,
+				 <&ccu CLK_DRAM_DRC0>;
+			clock-names = "ahb", "mod",
+				      "ram";
+			resets = <&ccu RST_AHB1_DRC0>;
+
+			assigned-clocks = <&ccu CLK_IEP_DRC0>;
+			assigned-clock-rates = <300000000>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				drc0_in: port@0 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0>;
+
+					drc0_in_be0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&be0_out_drc0>;
+					};
+				};
+
+				drc0_out: port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <1>;
+
+					drc0_out_tcon0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&tcon0_in_drc0>;
+					};
+				};
+			};
+		};
+
 		rtc: rtc@01f00000 {
 			compatible = "allwinner,sun6i-a31-rtc";
 			reg = <0x01f00000 0x54>;
@@ -886,7 +1065,8 @@
 			reg = <0x01f02c00 0x400>;
 			interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&apb0_gates 0>;
+			clocks = <&apb0_gates 0>, <&osc24M>, <&osc32k>;
+			clock-names = "apb", "hosc", "losc";
 			resets = <&apb0_rst 0>;
 			gpio-controller;
 			interrupt-controller;
diff --git a/arch/arm/boot/dts/sun6i-a31s-sina31s.dts b/arch/arm/boot/dts/sun6i-a31s-sina31s.dts
index 6ead2f5c847a..c35ec112f5a0 100644
--- a/arch/arm/boot/dts/sun6i-a31s-sina31s.dts
+++ b/arch/arm/boot/dts/sun6i-a31s-sina31s.dts
@@ -65,6 +65,14 @@
 	};
 };
 
+&codec {
+	allwinner,audio-routing =
+		"Line Out", "LINEOUT",
+		"MIC1", "Mic",
+		"Mic",	"MBIAS";
+	status = "okay";
+};
+
 &ehci0 {
 	/* USB 2.0 4 port hub IC */
 	status = "okay";
diff --git a/arch/arm/boot/dts/sun6i-a31s.dtsi b/arch/arm/boot/dts/sun6i-a31s.dtsi
index c17a32771b98..97e2c51d0aea 100644
--- a/arch/arm/boot/dts/sun6i-a31s.dtsi
+++ b/arch/arm/boot/dts/sun6i-a31s.dtsi
@@ -48,6 +48,14 @@
 
 #include "sun6i-a31.dtsi"
 
+&de {
+	compatible = "allwinner,sun6i-a31s-display-engine";
+};
+
 &pio {
 	compatible = "allwinner,sun6i-a31s-pinctrl";
 };
+
+&tcon0 {
+	compatible = "allwinner,sun6i-a31s-tcon";
+};
diff --git a/arch/arm/boot/dts/sun7i-a20-bananapi-m1-plus.dts b/arch/arm/boot/dts/sun7i-a20-bananapi-m1-plus.dts
index ba5bca0fe997..532f1a160560 100644
--- a/arch/arm/boot/dts/sun7i-a20-bananapi-m1-plus.dts
+++ b/arch/arm/boot/dts/sun7i-a20-bananapi-m1-plus.dts
@@ -105,6 +105,10 @@
 	status = "okay";
 };
 
+&cpu0 {
+	cpu-supply = <&reg_dcdc2>;
+};
+
 &ehci0 {
 	status = "okay";
 };
@@ -132,16 +136,14 @@
 	status = "okay";
 
 	axp209: pmic@34 {
-		compatible = "x-powers,axp209";
 		reg = <0x34>;
 		interrupt-parent = <&nmi_intc>;
 		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
-
-		interrupt-controller;
-		#interrupt-cells = <1>;
 	};
 };
 
+#include "axp209.dtsi"
+
 &ir0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&ir0_rx_pins_a>;
@@ -167,7 +169,7 @@
 	mmc-pwrseq = <&mmc3_pwrseq>;
 	bus-width = <4>;
 	non-removable;
-	enable-sdio-wakeup;
+	wakeup-source;
 	status = "okay";
 
 	brcmf: bcrmf@1 {
@@ -192,6 +194,10 @@
 	status = "okay";
 };
 
+&otg_sram {
+	status = "okay";
+};
+
 &pio {
 	gmac_power_pin_bpi_m1p: gmac_power_pin@0 {
 		allwinner,pins = "PH23";
@@ -222,8 +228,54 @@
 	};
 };
 
+&reg_dcdc2 {
+	regulator-always-on;
+	regulator-min-microvolt = <1000000>;
+	regulator-max-microvolt = <1400000>;
+	regulator-name = "vdd-cpu";
+};
+
+&reg_dcdc3 {
+	regulator-always-on;
+	regulator-min-microvolt = <1000000>;
+	regulator-max-microvolt = <1400000>;
+	regulator-name = "vdd-int-dll";
+};
+
+&reg_ldo1 {
+	regulator-name = "vdd-rtc";
+};
+
+&reg_ldo2 {
+	regulator-always-on;
+	regulator-min-microvolt = <3000000>;
+	regulator-max-microvolt = <3000000>;
+	regulator-name = "avcc";
+};
+
+&reg_usb0_vbus {
+	status = "okay";
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_pins_a>;
 	status = "okay";
 };
+
+&usb_otg {
+	dr_mode = "otg";
+	status = "okay";
+};
+
+&usb_power_supply {
+	status = "okay";
+};
+
+&usbphy {
+	usb0_id_det-gpios = <&pio 7 4 GPIO_ACTIVE_HIGH>; /* PH4 */
+	usb0_vbus_power-supply = <&usb_power_supply>;
+	usb0_vbus-supply = <&reg_usb0_vbus>;
+	/* VBUS on usb host ports are tied to DC5V and therefore always on */
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/sun7i-a20-olimex-som-evb.dts b/arch/arm/boot/dts/sun7i-a20-olimex-som-evb.dts
index 23aacce4d6c7..134e0c1b129d 100644
--- a/arch/arm/boot/dts/sun7i-a20-olimex-som-evb.dts
+++ b/arch/arm/boot/dts/sun7i-a20-olimex-som-evb.dts
@@ -88,6 +88,10 @@
 	status = "okay";
 };
 
+&cpu0 {
+	cpu-supply = <&reg_dcdc2>;
+};
+
 &codec {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/sun7i-a20.dtsi b/arch/arm/boot/dts/sun7i-a20.dtsi
index 94cf5a1c7172..f7db067b0de0 100644
--- a/arch/arm/boot/dts/sun7i-a20.dtsi
+++ b/arch/arm/boot/dts/sun7i-a20.dtsi
@@ -1085,7 +1085,8 @@
 			compatible = "allwinner,sun7i-a20-pinctrl";
 			reg = <0x01c20800 0x400>;
 			interrupts = <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&apb0_gates 5>;
+			clocks = <&apb0_gates 5>, <&osc24M>, <&osc32k>;
+			clock-names = "apb", "hosc", "losc";
 			gpio-controller;
 			interrupt-controller;
 			#interrupt-cells = <3>;
diff --git a/arch/arm/boot/dts/sun8i-a23-a33.dtsi b/arch/arm/boot/dts/sun8i-a23-a33.dtsi
index 300a1bd5a6ec..e4991a78ad73 100644
--- a/arch/arm/boot/dts/sun8i-a23-a33.dtsi
+++ b/arch/arm/boot/dts/sun8i-a23-a33.dtsi
@@ -266,7 +266,8 @@
 			/* compatible gets set in SoC specific dtsi file */
 			reg = <0x01c20800 0x400>;
 			/* interrupts get set in SoC specific dtsi file */
-			clocks = <&ccu CLK_BUS_PIO>;
+			clocks = <&ccu CLK_BUS_PIO>, <&osc24M>, <&osc32k>;
+			clock-names = "apb", "hosc", "losc";
 			gpio-controller;
 			interrupt-controller;
 			#interrupt-cells = <3>;
@@ -575,7 +576,8 @@
 			compatible = "allwinner,sun8i-a23-r-pinctrl";
 			reg = <0x01f02c00 0x400>;
 			interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&apb0_gates 0>;
+			clocks = <&apb0_gates 0>, <&osc24M>, <&osc32k>;
+			clock-names = "apb", "hosc", "losc";
 			resets = <&apb0_rst 0>;
 			gpio-controller;
 			interrupt-controller;
diff --git a/arch/arm/boot/dts/sun8i-a23-polaroid-mid2407pxe03.dts b/arch/arm/boot/dts/sun8i-a23-polaroid-mid2407pxe03.dts
index a86cbedda34c..21bb291b9568 100644
--- a/arch/arm/boot/dts/sun8i-a23-polaroid-mid2407pxe03.dts
+++ b/arch/arm/boot/dts/sun8i-a23-polaroid-mid2407pxe03.dts
@@ -98,13 +98,6 @@
 	};
 };
 
-&reg_ldo_io1 {
-	regulator-min-microvolt = <3300000>;
-	regulator-max-microvolt = <3300000>;
-	regulator-name = "vcc-touchscreen";
-	status = "okay";
-};
-
 &touchscreen {
 	reg = <0x40>;
 	compatible = "silead,gsl1680";
diff --git a/arch/arm/boot/dts/sun8i-a33-sinlinx-sina33.dts b/arch/arm/boot/dts/sun8i-a33-sinlinx-sina33.dts
index fef6abc0a703..71bb9418c5f9 100644
--- a/arch/arm/boot/dts/sun8i-a33-sinlinx-sina33.dts
+++ b/arch/arm/boot/dts/sun8i-a33-sinlinx-sina33.dts
@@ -213,6 +213,11 @@
 	status = "okay";
 };
 
+&usb_otg {
+	dr_mode = "peripheral";
+	status = "okay";
+};
+
 &usbphy {
 	status = "okay";
 	usb1_vbus-supply = <&reg_vcc5v0>; /* USB1 VBUS is always on */
diff --git a/arch/arm/boot/dts/sun8i-h3-nanopi-m1.dts b/arch/arm/boot/dts/sun8i-h3-nanopi-m1.dts
new file mode 100644
index 000000000000..ec63d104b404
--- /dev/null
+++ b/arch/arm/boot/dts/sun8i-h3-nanopi-m1.dts
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2016 Milo Kim <woogyom.kim@gmail.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "sun8i-h3-nanopi.dtsi"
+
+/ {
+	model = "FriendlyArm NanoPi M1";
+	compatible = "friendlyarm,nanopi-m1", "allwinner,sun8i-h3";
+};
+
+&ehci1 {
+	status = "okay";
+};
+
+&ehci2 {
+	status = "okay";
+};
+
+&ohci1 {
+	status = "okay";
+};
+
+&ohci2 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/sun8i-h3-nanopi-neo.dts b/arch/arm/boot/dts/sun8i-h3-nanopi-neo.dts
index 3d64cafc1e90..8d2cc6e9a03f 100644
--- a/arch/arm/boot/dts/sun8i-h3-nanopi-neo.dts
+++ b/arch/arm/boot/dts/sun8i-h3-nanopi-neo.dts
@@ -40,86 +40,9 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
-/dts-v1/;
-#include "sun8i-h3.dtsi"
-#include "sunxi-common-regulators.dtsi"
-
-#include <dt-bindings/gpio/gpio.h>
-#include <dt-bindings/pinctrl/sun4i-a10.h>
+#include "sun8i-h3-nanopi.dtsi"
 
 / {
 	model = "FriendlyARM NanoPi NEO";
 	compatible = "friendlyarm,nanopi-neo", "allwinner,sun8i-h3";
-
-	aliases {
-		serial0 = &uart0;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-
-	leds {
-		compatible = "gpio-leds";
-		pinctrl-names = "default";
-		pinctrl-0 = <&leds_opc>, <&leds_r_opc>;
-
-		pwr {
-			label = "nanopi:green:pwr";
-			gpios = <&r_pio 0 10 GPIO_ACTIVE_HIGH>; /* PL10 */
-			default-state = "on";
-		};
-
-		status {
-			label = "nanopi:blue:status";
-			gpios = <&pio 0 10 GPIO_ACTIVE_HIGH>; /* PA10 */
-		};
-	};
-};
-
-&ehci3 {
-	status = "okay";
-};
-
-&mmc0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&mmc0_pins_a>, <&mmc0_cd_pin>;
-	vmmc-supply = <&reg_vcc3v3>;
-	bus-width = <4>;
-	cd-gpios = <&pio 5 6 GPIO_ACTIVE_HIGH>; /* PF6 */
-	cd-inverted;
-	status = "okay";
-};
-
-&ohci3 {
-	status = "okay";
-};
-
-&pio {
-	leds_opc: led-pins {
-		allwinner,pins = "PA10";
-		allwinner,function = "gpio_out";
-		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
-		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
-	};
-};
-
-&r_pio {
-	leds_r_opc: led-pins {
-		allwinner,pins = "PL10";
-		allwinner,function = "gpio_out";
-		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
-		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
-	};
-};
-
-&uart0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&uart0_pins_a>;
-	status = "okay";
-};
-
-&usbphy {
-	/* USB VBUS is always on */
-	status = "okay";
 };
diff --git a/arch/arm/boot/dts/sun8i-h3-nanopi.dtsi b/arch/arm/boot/dts/sun8i-h3-nanopi.dtsi
new file mode 100644
index 000000000000..8038aa29a5a7
--- /dev/null
+++ b/arch/arm/boot/dts/sun8i-h3-nanopi.dtsi
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2016 James Pettigrew <james@innovum.com.au>
+ * Copyright (C) 2016 Milo Kim <woogyom.kim@gmail.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+#include "sun8i-h3.dtsi"
+#include "sunxi-common-regulators.dtsi"
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/pinctrl/sun4i-a10.h>
+
+/ {
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&leds_npi>, <&leds_r_npi>;
+
+		status {
+			label = "nanopi:blue:status";
+			gpios = <&pio 0 10 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+
+		pwr {
+			label = "nanopi:green:pwr";
+			gpios = <&r_pio 0 10 GPIO_ACTIVE_HIGH>;
+			default-state = "on";
+		};
+	};
+
+	r_gpio_keys {
+		compatible = "gpio-keys";
+		input-name = "k1";
+		pinctrl-names = "default";
+		pinctrl-0 = <&sw_r_npi>;
+
+		k1@0 {
+			label = "k1";
+			linux,code = <KEY_POWER>;
+			gpios = <&r_pio 0 3 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&ehci3 {
+	status = "okay";
+};
+
+&mmc0 {
+	bus-width = <4>;
+	cd-gpios = <&pio 5 6 GPIO_ACTIVE_HIGH>;
+	cd-inverted;
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc0_pins_a>, <&mmc0_cd_pin>;
+	status = "okay";
+	vmmc-supply = <&reg_vcc3v3>;
+};
+
+&ohci3 {
+	status = "okay";
+};
+
+&pio {
+	leds_npi: led_pins@0 {
+		allwinner,pins = "PA10";
+		allwinner,function = "gpio_out";
+		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+	};
+};
+
+&r_pio {
+	leds_r_npi: led_pins@0 {
+		allwinner,pins = "PL10";
+		allwinner,function = "gpio_out";
+		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+	};
+
+	sw_r_npi: key_pins@0 {
+		allwinner,pins = "PL3";
+		allwinner,function = "gpio_in";
+		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pins_a>;
+	status = "okay";
+};
+
+&usbphy {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/sun8i-h3.dtsi b/arch/arm/boot/dts/sun8i-h3.dtsi
index f4ba088b225e..6c14a6f72820 100644
--- a/arch/arm/boot/dts/sun8i-h3.dtsi
+++ b/arch/arm/boot/dts/sun8i-h3.dtsi
@@ -321,7 +321,8 @@
 			reg = <0x01c20800 0x400>;
 			interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&ccu CLK_BUS_PIO>;
+			clocks = <&ccu CLK_BUS_PIO>, <&osc24M>, <&osc32k>;
+			clock-names = "apb", "hosc", "losc";
 			gpio-controller;
 			#gpio-cells = <3>;
 			interrupt-controller;
@@ -381,6 +382,20 @@
 				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 			};
 
+			spi0_pins: spi0 {
+				allwinner,pins = "PC0", "PC1", "PC2", "PC3";
+				allwinner,function = "spi0";
+				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
+			spi1_pins: spi1 {
+				allwinner,pins = "PA15", "PA16", "PA14", "PA13";
+				allwinner,function = "spi1";
+				allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
 			uart0_pins_a: uart0@0 {
 				allwinner,pins = "PA4", "PA5";
 				allwinner,function = "uart0";
@@ -425,6 +440,38 @@
 			clocks = <&osc24M>;
 		};
 
+		spi0: spi@01c68000 {
+			compatible = "allwinner,sun8i-h3-spi";
+			reg = <0x01c68000 0x1000>;
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_SPI0>, <&ccu CLK_SPI0>;
+			clock-names = "ahb", "mod";
+			dmas = <&dma 23>, <&dma 23>;
+			dma-names = "rx", "tx";
+			pinctrl-names = "default";
+			pinctrl-0 = <&spi0_pins>;
+			resets = <&ccu RST_BUS_SPI0>;
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		spi1: spi@01c69000 {
+			compatible = "allwinner,sun8i-h3-spi";
+			reg = <0x01c69000 0x1000>;
+			interrupts = <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_SPI1>, <&ccu CLK_SPI1>;
+			clock-names = "ahb", "mod";
+			dmas = <&dma 24>, <&dma 24>;
+			dma-names = "rx", "tx";
+			pinctrl-names = "default";
+			pinctrl-0 = <&spi1_pins>;
+			resets = <&ccu RST_BUS_SPI1>;
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
 		wdt0: watchdog@01c20ca0 {
 			compatible = "allwinner,sun6i-a31-wdt";
 			reg = <0x01c20ca0 0x20>;
@@ -568,7 +615,8 @@
 			compatible = "allwinner,sun8i-h3-r-pinctrl";
 			reg = <0x01f02c00 0x400>;
 			interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&apb0_gates 0>;
+			clocks = <&apb0_gates 0>, <&osc24M>, <&osc32k>;
+			clock-names = "apb", "hosc", "losc";
 			resets = <&apb0_reset 0>;
 			gpio-controller;
 			#gpio-cells = <3>;
diff --git a/arch/arm/boot/dts/sun8i-reference-design-tablet.dtsi b/arch/arm/boot/dts/sun8i-reference-design-tablet.dtsi
index 08cd00143635..69bc0cd26ca7 100644
--- a/arch/arm/boot/dts/sun8i-reference-design-tablet.dtsi
+++ b/arch/arm/boot/dts/sun8i-reference-design-tablet.dtsi
@@ -209,6 +209,13 @@
 	status = "okay";
 };
 
+&reg_ldo_io1 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-touchscreen";
+	status = "okay";
+};
+
 &reg_rtc_ldo {
 	regulator-name = "vcc-rtc";
 };
diff --git a/arch/arm/boot/dts/sun9i-a80-cubieboard4.dts b/arch/arm/boot/dts/sun9i-a80-cubieboard4.dts
index 439847acd41e..67b02fe7f11c 100644
--- a/arch/arm/boot/dts/sun9i-a80-cubieboard4.dts
+++ b/arch/arm/boot/dts/sun9i-a80-cubieboard4.dts
@@ -76,6 +76,14 @@
 			gpios = <&pio 7 6 GPIO_ACTIVE_HIGH>; /* PH6 */
 		};
 	};
+
+	wifi_pwrseq: wifi_pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		clocks = <&ac100_rtc 1>;
+		clock-names = "ext_clock";
+		/* enables internal regulator and de-asserts reset */
+		reset-gpios = <&r_pio 0 2 GPIO_ACTIVE_LOW>; /* PL2 WL-PMU-EN */
+	};
 };
 
 &mmc0 {
@@ -88,6 +96,21 @@
 	status = "okay";
 };
 
+&mmc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc1_pins>, <&wifi_en_pin_cubieboard4>;
+	vmmc-supply = <&reg_dldo1>;
+	vqmmc-supply = <&reg_cldo3>;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	bus-width = <4>;
+	non-removable;
+	status = "okay";
+};
+
+&mmc1_pins {
+	allwinner,pull = <SUN4I_PINCTRL_PULL_UP>;
+};
+
 &mmc2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc2_8bit_pins>;
@@ -128,6 +151,15 @@
 	status = "okay";
 };
 
+&r_pio {
+	wifi_en_pin_cubieboard4: wifi_en_pin@0 {
+		allwinner,pins = "PL2";
+		allwinner,function = "gpio_out";
+		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+	};
+};
+
 &r_rsb {
 	status = "okay";
 
diff --git a/arch/arm/boot/dts/sun9i-a80-optimus.dts b/arch/arm/boot/dts/sun9i-a80-optimus.dts
index ceb6ef15d669..7e036b2be762 100644
--- a/arch/arm/boot/dts/sun9i-a80-optimus.dts
+++ b/arch/arm/boot/dts/sun9i-a80-optimus.dts
@@ -105,6 +105,14 @@
 		enable-active-high;
 		gpio = <&pio 7 5 GPIO_ACTIVE_HIGH>; /* PH5 */
 	};
+
+	wifi_pwrseq: wifi_pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		clocks = <&ac100_rtc 1>;
+		clock-names = "ext_clock";
+		/* enables internal regulator and de-asserts reset */
+		reset-gpios = <&r_pio 0 2 GPIO_ACTIVE_LOW>; /* PL2 WL-PMU-EN */
+	};
 };
 
 &ehci0 {
@@ -130,6 +138,21 @@
 	status = "okay";
 };
 
+&mmc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc1_pins>, <&wifi_en_pin_optimus>;
+	vmmc-supply = <&reg_dldo1>;
+	vqmmc-supply = <&reg_cldo3>;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	bus-width = <4>;
+	non-removable;
+	status = "okay";
+};
+
+&mmc1_pins {
+	allwinner,pull = <SUN4I_PINCTRL_PULL_UP>;
+};
+
 &mmc2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc2_8bit_pins>;
@@ -199,6 +222,13 @@
 		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
 		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 	};
+
+	wifi_en_pin_optimus: wifi_en_pin@0 {
+		allwinner,pins = "PL2";
+		allwinner,function = "gpio_out";
+		allwinner,drive = <SUN4I_PINCTRL_10_MA>;
+		allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+	};
 };
 
 &r_rsb {
diff --git a/arch/arm/boot/dts/sun9i-a80.dtsi b/arch/arm/boot/dts/sun9i-a80.dtsi
index 3c5214cbe4e6..979ad1aacfb1 100644
--- a/arch/arm/boot/dts/sun9i-a80.dtsi
+++ b/arch/arm/boot/dts/sun9i-a80.dtsi
@@ -678,7 +678,8 @@
 				     <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&apb0_gates 5>;
+			clocks = <&apb0_gates 5>, <&osc24M>, <&osc32k>;
+			clock-names = "apb", "hosc", "losc";
 			gpio-controller;
 			interrupt-controller;
 			#interrupt-cells = <3>;
@@ -700,6 +701,14 @@
 				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
 			};
 
+			mmc1_pins: mmc1 {
+				allwinner,pins = "PG0", "PG1" ,"PG2", "PG3",
+						 "PG4", "PG5";
+				allwinner,function = "mmc1";
+				allwinner,drive = <SUN4I_PINCTRL_30_MA>;
+				allwinner,pull = <SUN4I_PINCTRL_NO_PULL>;
+			};
+
 			mmc2_8bit_pins: mmc2_8bit {
 				allwinner,pins = "PC6", "PC7", "PC8", "PC9",
 						 "PC10", "PC11", "PC12",
@@ -894,7 +903,8 @@
 			reg = <0x08002c00 0x400>;
 			interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&apbs_gates 0>;
+			clocks = <&apbs_gates 0>, <&osc24M>, <&osc32k>;
+			clock-names = "apb", "hosc", "losc";
 			resets = <&apbs_rst 0>;
 			gpio-controller;
 			interrupt-controller;
diff --git a/arch/arm/boot/dts/tegra124-apalis.dtsi b/arch/arm/boot/dts/tegra124-apalis.dtsi
index e7a73db17613..0819721dda59 100644
--- a/arch/arm/boot/dts/tegra124-apalis.dtsi
+++ b/arch/arm/boot/dts/tegra124-apalis.dtsi
@@ -1595,7 +1595,7 @@
 		clock-frequency = <400000>;
 
 		/* SGTL5000 audio codec */
-		sgtl5000: codec@0a {
+		sgtl5000: codec@a {
 			compatible = "fsl,sgtl5000";
 			reg = <0x0a>;
 			VDDA-supply = <&reg_3v3>;
diff --git a/arch/arm/boot/dts/tegra124-nyan.dtsi b/arch/arm/boot/dts/tegra124-nyan.dtsi
index 271505e0715f..eabfa655a3cd 100644
--- a/arch/arm/boot/dts/tegra124-nyan.dtsi
+++ b/arch/arm/boot/dts/tegra124-nyan.dtsi
@@ -42,6 +42,12 @@
 		};
 	};
 
+	gpu@0,57000000 {
+		status = "okay";
+
+		vdd-supply = <&vdd_gpu>;
+	};
+
 	serial@70006000 {
 		/* Debug connector on the bottom of the board near SD card. */
 		status = "okay";
@@ -214,7 +220,7 @@
 					regulator-always-on;
 				};
 
-				sd6 {
+				vdd_gpu: sd6 {
 					regulator-name = "+VDD_GPU_AP";
 					regulator-min-microvolt = <650000>;
 					regulator-max-microvolt = <1200000>;
diff --git a/arch/arm/boot/dts/tegra20.dtsi b/arch/arm/boot/dts/tegra20.dtsi
index 2207c08e3fa3..e8807503f87c 100644
--- a/arch/arm/boot/dts/tegra20.dtsi
+++ b/arch/arm/boot/dts/tegra20.dtsi
@@ -376,6 +376,19 @@
 		status = "disabled";
 	};
 
+	gmi@70009000 {
+		compatible = "nvidia,tegra20-gmi";
+		reg = <0x70009000 0x1000>;
+		#address-cells = <2>;
+		#size-cells = <1>;
+		ranges = <0 0 0xd0000000 0xfffffff>;
+		clocks = <&tegra_car TEGRA20_CLK_NOR>;
+		clock-names = "gmi";
+		resets = <&tegra_car 42>;
+		reset-names = "gmi";
+		status = "disabled";
+	};
+
 	pwm: pwm@7000a000 {
 		compatible = "nvidia,tegra20-pwm";
 		reg = <0x7000a000 0x100>;
diff --git a/arch/arm/boot/dts/tegra30-apalis.dtsi b/arch/arm/boot/dts/tegra30-apalis.dtsi
index 192b95177aac..f6c7c3e958ac 100644
--- a/arch/arm/boot/dts/tegra30-apalis.dtsi
+++ b/arch/arm/boot/dts/tegra30-apalis.dtsi
@@ -48,6 +48,24 @@
 		pinctrl-0 = <&state_default>;
 
 		state_default: pinmux {
+			/* Analogue Audio (On-module) */
+			clk1_out_pw4 {
+				nvidia,pins = "clk1_out_pw4";
+				nvidia,function = "extperiph1";
+				nvidia,pull = <TEGRA_PIN_PULL_NONE>;
+				nvidia,tristate = <TEGRA_PIN_DISABLE>;
+				nvidia,enable-input = <TEGRA_PIN_DISABLE>;
+			};
+			dap3_fs_pp0 {
+				nvidia,pins =	"dap3_fs_pp0",
+						"dap3_sclk_pp3",
+						"dap3_din_pp1",
+						"dap3_dout_pp2";
+				nvidia,function = "i2s2";
+				nvidia,pull = <TEGRA_PIN_PULL_NONE>;
+				nvidia,tristate = <TEGRA_PIN_DISABLE>;
+			};
+
 			/* Apalis BKL1_ON */
 			pv2 {
 				nvidia,pins = "pv2";
@@ -429,6 +447,15 @@
 		status = "okay";
 		clock-frequency = <100000>;
 
+		/* SGTL5000 audio codec */
+		sgtl5000: codec@a {
+			compatible = "fsl,sgtl5000";
+			reg = <0x0a>;
+			VDDA-supply = <&sys_3v3_reg>;
+			VDDIO-supply = <&sys_3v3_reg>;
+			clocks = <&tegra_car TEGRA30_CLK_EXTERN1>;
+		};
+
 		pmic: tps65911@2d {
 			compatible = "ti,tps65911";
 			reg = <0x2d>;
@@ -660,6 +687,12 @@
 		nvidia,sys-clock-req-active-high;
 	};
 
+	ahub@70080000 {
+		i2s@70080500 {
+			status = "okay";
+		};
+	};
+
 	/* eMMC */
 	sdhci@78000600 {
 		status = "okay";
@@ -733,4 +766,20 @@
 			regulator-always-on;
 		};
 	};
+
+	sound {
+		compatible = "toradex,tegra-audio-sgtl5000-apalis_t30",
+			     "nvidia,tegra-audio-sgtl5000";
+		nvidia,model = "Toradex Apalis T30";
+		nvidia,audio-routing =
+			"Headphone Jack", "HP_OUT",
+			"LINE_IN", "Line In Jack",
+			"MIC_IN", "Mic Jack";
+		nvidia,i2s-controller = <&tegra_i2s2>;
+		nvidia,audio-codec = <&sgtl5000>;
+		clocks = <&tegra_car TEGRA30_CLK_PLL_A>,
+			 <&tegra_car TEGRA30_CLK_PLL_A_OUT0>,
+			 <&tegra_car TEGRA30_CLK_EXTERN1>;
+		clock-names = "pll_a", "pll_a_out0", "mclk";
+	};
 };
diff --git a/arch/arm/boot/dts/tegra30-colibri.dtsi b/arch/arm/boot/dts/tegra30-colibri.dtsi
index a265534cd314..5360d638eedc 100644
--- a/arch/arm/boot/dts/tegra30-colibri.dtsi
+++ b/arch/arm/boot/dts/tegra30-colibri.dtsi
@@ -29,6 +29,24 @@
 		pinctrl-0 = <&state_default>;
 
 		state_default: pinmux {
+			/* Analogue Audio (On-module) */
+			clk1_out_pw4 {
+				nvidia,pins = "clk1_out_pw4";
+				nvidia,function = "extperiph1";
+				nvidia,pull = <TEGRA_PIN_PULL_NONE>;
+				nvidia,tristate = <TEGRA_PIN_DISABLE>;
+				nvidia,enable-input = <TEGRA_PIN_DISABLE>;
+			};
+			dap3_fs_pp0 {
+				nvidia,pins =	"dap3_fs_pp0",
+						"dap3_sclk_pp3",
+						"dap3_din_pp1",
+						"dap3_dout_pp2";
+				nvidia,function = "i2s2";
+				nvidia,pull = <TEGRA_PIN_PULL_NONE>;
+				nvidia,tristate = <TEGRA_PIN_DISABLE>;
+			};
+
 			/* Colibri BL_ON */
 			pv2 {
 				nvidia,pins = "pv2";
@@ -207,6 +225,15 @@
 		status = "okay";
 		clock-frequency = <100000>;
 
+		/* SGTL5000 audio codec */
+		sgtl5000: codec@a {
+			compatible = "fsl,sgtl5000";
+			reg = <0x0a>;
+			VDDA-supply = <&sys_3v3_reg>;
+			VDDIO-supply = <&sys_3v3_reg>;
+			clocks = <&tegra_car TEGRA30_CLK_EXTERN1>;
+		};
+
 		pmic: tps65911@2d {
 			compatible = "ti,tps65911";
 			reg = <0x2d>;
@@ -396,6 +423,12 @@
 		nvidia,sys-clock-req-active-high;
 	};
 
+	ahub@70080000 {
+		i2s@70080500 {
+			status = "okay";
+		};
+	};
+
 	/* eMMC */
 	sdhci@78000600 {
 		status = "okay";
@@ -471,4 +504,20 @@
 			regulator-always-on;
 		};
 	};
+
+	sound {
+		compatible = "toradex,tegra-audio-sgtl5000-colibri_t30",
+			     "nvidia,tegra-audio-sgtl5000";
+		nvidia,model = "Toradex Colibri T30";
+		nvidia,audio-routing =
+			"Headphone Jack", "HP_OUT",
+			"LINE_IN", "Line In Jack",
+			"MIC_IN", "Mic Jack";
+		nvidia,i2s-controller = <&tegra_i2s2>;
+		nvidia,audio-codec = <&sgtl5000>;
+		clocks = <&tegra_car TEGRA30_CLK_PLL_A>,
+			 <&tegra_car TEGRA30_CLK_PLL_A_OUT0>,
+			 <&tegra_car TEGRA30_CLK_EXTERN1>;
+		clock-names = "pll_a", "pll_a_out0", "mclk";
+	};
 };
diff --git a/arch/arm/boot/dts/tegra30.dtsi b/arch/arm/boot/dts/tegra30.dtsi
index 5030065cbdfe..bbb1c002e7f1 100644
--- a/arch/arm/boot/dts/tegra30.dtsi
+++ b/arch/arm/boot/dts/tegra30.dtsi
@@ -439,6 +439,19 @@
 		status = "disabled";
 	};
 
+	gmi@70009000 {
+		compatible = "nvidia,tegra30-gmi";
+		reg = <0x70009000 0x1000>;
+		#address-cells = <2>;
+		#size-cells = <1>;
+		ranges = <0 0 0x48000000 0x7ffffff>;
+		clocks = <&tegra_car TEGRA30_CLK_NOR>;
+		clock-names = "gmi";
+		resets = <&tegra_car 42>;
+		reset-names = "gmi";
+		status = "disabled";
+	};
+
 	pwm: pwm@7000a000 {
 		compatible = "nvidia,tegra30-pwm", "nvidia,tegra20-pwm";
 		reg = <0x7000a000 0x100>;
diff --git a/arch/arm/boot/dts/tps65217.dtsi b/arch/arm/boot/dts/tps65217.dtsi
index a63272422d76..02de56b55823 100644
--- a/arch/arm/boot/dts/tps65217.dtsi
+++ b/arch/arm/boot/dts/tps65217.dtsi
@@ -13,6 +13,18 @@
 
 &tps {
 	compatible = "ti,tps65217";
+	interrupt-controller;
+	#interrupt-cells = <1>;
+
+	charger {
+		compatible = "ti,tps65217-charger";
+		status = "disabled";
+	};
+
+	pwrbutton {
+		compatible = "ti,tps65217-pwrbutton";
+		status = "disabled";
+	};
 
 	regulators {
 		#address-cells = <1>;
diff --git a/arch/arm/boot/dts/uniphier-common32.dtsi b/arch/arm/boot/dts/uniphier-common32.dtsi
deleted file mode 100644
index 8c8a85176b64..000000000000
--- a/arch/arm/boot/dts/uniphier-common32.dtsi
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Device Tree Source commonly used by UniPhier ARM SoCs
- *
- * Copyright (C) 2015-2016 Socionext Inc.
- *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This file is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/include/ "skeleton.dtsi"
-
-/ {
-	psci {
-		compatible = "arm,psci-0.2";
-		method = "smc";
-	};
-
-	clocks {
-		refclk: ref {
-			#clock-cells = <0>;
-			compatible = "fixed-clock";
-		};
-	};
-
-	soc: soc {
-		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges;
-		interrupt-parent = <&intc>;
-
-		serial0: serial@54006800 {
-			compatible = "socionext,uniphier-uart";
-			status = "disabled";
-			reg = <0x54006800 0x40>;
-			interrupts = <0 33 4>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_uart0>;
-			clocks = <&peri_clk 0>;
-		};
-
-		serial1: serial@54006900 {
-			compatible = "socionext,uniphier-uart";
-			status = "disabled";
-			reg = <0x54006900 0x40>;
-			interrupts = <0 35 4>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_uart1>;
-			clocks = <&peri_clk 1>;
-		};
-
-		serial2: serial@54006a00 {
-			compatible = "socionext,uniphier-uart";
-			status = "disabled";
-			reg = <0x54006a00 0x40>;
-			interrupts = <0 37 4>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_uart2>;
-			clocks = <&peri_clk 2>;
-		};
-
-		serial3: serial@54006b00 {
-			compatible = "socionext,uniphier-uart";
-			status = "disabled";
-			reg = <0x54006b00 0x40>;
-			interrupts = <0 177 4>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_uart3>;
-			clocks = <&peri_clk 3>;
-		};
-
-		system_bus: system-bus@58c00000 {
-			compatible = "socionext,uniphier-system-bus";
-			status = "disabled";
-			reg = <0x58c00000 0x400>;
-			#address-cells = <2>;
-			#size-cells = <1>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_system_bus>;
-		};
-
-		smpctrl@59800000 {
-			compatible = "socionext,uniphier-smpctrl";
-			reg = <0x59801000 0x400>;
-		};
-
-		mioctrl@59810000 {
-			compatible = "socionext,uniphier-mioctrl",
-				     "simple-mfd", "syscon";
-			reg = <0x59810000 0x800>;
-
-			mio_clk: clock {
-				#clock-cells = <1>;
-			};
-
-			mio_rst: reset {
-				#reset-cells = <1>;
-			};
-		};
-
-		perictrl@59820000 {
-			compatible = "socionext,uniphier-perictrl",
-				     "simple-mfd", "syscon";
-			reg = <0x59820000 0x200>;
-
-			peri_clk: clock {
-				#clock-cells = <1>;
-			};
-
-			peri_rst: reset {
-				#reset-cells = <1>;
-			};
-		};
-
-		timer@60000200 {
-			compatible = "arm,cortex-a9-global-timer";
-			reg = <0x60000200 0x20>;
-			interrupts = <1 11 0x104>;
-			clocks = <&arm_timer_clk>;
-		};
-
-		timer@60000600 {
-			compatible = "arm,cortex-a9-twd-timer";
-			reg = <0x60000600 0x20>;
-			interrupts = <1 13 0x104>;
-			clocks = <&arm_timer_clk>;
-		};
-
-		intc: interrupt-controller@60001000 {
-			compatible = "arm,cortex-a9-gic";
-			reg = <0x60001000 0x1000>,
-			      <0x60000100 0x100>;
-			#interrupt-cells = <3>;
-			interrupt-controller;
-		};
-
-		soc-glue@5f800000 {
-			compatible = "socionext,uniphier-soc-glue",
-				     "simple-mfd", "syscon";
-			reg = <0x5f800000 0x2000>;
-
-			pinctrl: pinctrl {
-				/* specify compatible in each SoC DTSI */
-			};
-		};
-
-		sysctrl@61840000 {
-			compatible = "socionext,uniphier-sysctrl",
-				     "simple-mfd", "syscon";
-			reg = <0x61840000 0x4000>;
-
-			sys_clk: clock {
-				#clock-cells = <1>;
-			};
-
-			sys_rst: reset {
-				#reset-cells = <1>;
-			};
-		};
-	};
-};
-
-/include/ "uniphier-pinctrl.dtsi"
diff --git a/arch/arm/boot/dts/uniphier-ld4.dtsi b/arch/arm/boot/dts/uniphier-ld4.dtsi
index 95f342c9d9c1..a7c494d7c43a 100644
--- a/arch/arm/boot/dts/uniphier-ld4.dtsi
+++ b/arch/arm/boot/dts/uniphier-ld4.dtsi
@@ -43,7 +43,7 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
-/include/ "uniphier-common32.dtsi"
+/include/ "skeleton.dtsi"
 
 / {
 	compatible = "socionext,uniphier-ld4";
@@ -61,147 +61,267 @@
 		};
 	};
 
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
 	clocks {
+		refclk: ref {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <24576000>;
+		};
+
 		arm_timer_clk: arm_timer_clk {
 			#clock-cells = <0>;
 			compatible = "fixed-clock";
 			clock-frequency = <50000000>;
 		};
 	};
-};
-
-&soc {
-	l2: l2-cache@500c0000 {
-		compatible = "socionext,uniphier-system-cache";
-		reg = <0x500c0000 0x2000>, <0x503c0100 0x4>, <0x506c0000 0x400>;
-		interrupts = <0 174 4>, <0 175 4>;
-		cache-unified;
-		cache-size = <(512 * 1024)>;
-		cache-sets = <256>;
-		cache-line-size = <128>;
-		cache-level = <2>;
-	};
 
-	i2c0: i2c@58400000 {
-		compatible = "socionext,uniphier-i2c";
-		status = "disabled";
-		reg = <0x58400000 0x40>;
+	soc {
+		compatible = "simple-bus";
 		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 41 1>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c0>;
-		clocks = <&peri_clk 4>;
-		clock-frequency = <100000>;
-	};
+		#size-cells = <1>;
+		ranges;
+		interrupt-parent = <&intc>;
 
-	i2c1: i2c@58480000 {
-		compatible = "socionext,uniphier-i2c";
-		status = "disabled";
-		reg = <0x58480000 0x40>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 42 1>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c1>;
-		clocks = <&peri_clk 5>;
-		clock-frequency = <100000>;
-	};
+		l2: l2-cache@500c0000 {
+			compatible = "socionext,uniphier-system-cache";
+			reg = <0x500c0000 0x2000>, <0x503c0100 0x4>,
+			      <0x506c0000 0x400>;
+			interrupts = <0 174 4>, <0 175 4>;
+			cache-unified;
+			cache-size = <(512 * 1024)>;
+			cache-sets = <256>;
+			cache-line-size = <128>;
+			cache-level = <2>;
+		};
 
-	/* chip-internal connection for DMD */
-	i2c2: i2c@58500000 {
-		compatible = "socionext,uniphier-i2c";
-		reg = <0x58500000 0x40>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 43 1>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c2>;
-		clocks = <&peri_clk 6>;
-		clock-frequency = <400000>;
-	};
+		serial0: serial@54006800 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006800 0x40>;
+			interrupts = <0 33 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart0>;
+			clocks = <&peri_clk 0>;
+		};
 
-	i2c3: i2c@58580000 {
-		compatible = "socionext,uniphier-i2c";
-		status = "disabled";
-		reg = <0x58580000 0x40>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 44 1>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c3>;
-		clocks = <&peri_clk 7>;
-		clock-frequency = <100000>;
-	};
+		serial1: serial@54006900 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006900 0x40>;
+			interrupts = <0 35 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart1>;
+			clocks = <&peri_clk 1>;
+		};
 
-	usb0: usb@5a800100 {
-		compatible = "socionext,uniphier-ehci", "generic-ehci";
-		status = "disabled";
-		reg = <0x5a800100 0x100>;
-		interrupts = <0 80 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_usb0>;
-		clocks = <&mio_clk 7>, <&mio_clk 8>, <&mio_clk 12>;
-		resets = <&mio_rst 7>, <&mio_rst 8>, <&mio_rst 12>, <&sys_rst 8>;
-	};
+		serial2: serial@54006a00 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006a00 0x40>;
+			interrupts = <0 37 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart2>;
+			clocks = <&peri_clk 2>;
+		};
 
-	usb1: usb@5a810100 {
-		compatible = "socionext,uniphier-ehci", "generic-ehci";
-		status = "disabled";
-		reg = <0x5a810100 0x100>;
-		interrupts = <0 81 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_usb1>;
-		clocks = <&mio_clk 7>, <&mio_clk 9>, <&mio_clk 13>;
-		resets = <&mio_rst 7>, <&mio_rst 9>, <&mio_rst 13>, <&sys_rst 8>;
-	};
+		serial3: serial@54006b00 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006b00 0x40>;
+			interrupts = <0 29 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart3>;
+			clocks = <&peri_clk 3>;
+		};
 
-	usb2: usb@5a820100 {
-		compatible = "socionext,uniphier-ehci", "generic-ehci";
-		status = "disabled";
-		reg = <0x5a820100 0x100>;
-		interrupts = <0 82 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_usb2>;
-		clocks = <&mio_clk 7>, <&mio_clk 10>, <&mio_clk 14>;
-		resets = <&mio_rst 7>, <&mio_rst 10>, <&mio_rst 14>, <&sys_rst 8>;
-	};
+		i2c0: i2c@58400000 {
+			compatible = "socionext,uniphier-i2c";
+			status = "disabled";
+			reg = <0x58400000 0x40>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 41 1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c0>;
+			clocks = <&peri_clk 4>;
+			clock-frequency = <100000>;
+		};
 
-};
+		i2c1: i2c@58480000 {
+			compatible = "socionext,uniphier-i2c";
+			status = "disabled";
+			reg = <0x58480000 0x40>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 42 1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c1>;
+			clocks = <&peri_clk 5>;
+			clock-frequency = <100000>;
+		};
 
-&refclk {
-	clock-frequency = <24576000>;
-};
+		/* chip-internal connection for DMD */
+		i2c2: i2c@58500000 {
+			compatible = "socionext,uniphier-i2c";
+			reg = <0x58500000 0x40>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 43 1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c2>;
+			clocks = <&peri_clk 6>;
+			clock-frequency = <400000>;
+		};
 
-&serial3 {
-	interrupts = <0 29 4>;
-};
+		i2c3: i2c@58580000 {
+			compatible = "socionext,uniphier-i2c";
+			status = "disabled";
+			reg = <0x58580000 0x40>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 44 1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c3>;
+			clocks = <&peri_clk 7>;
+			clock-frequency = <100000>;
+		};
 
-&mio_clk {
-	compatible = "socionext,uniphier-ld4-mio-clock";
-};
+		system_bus: system-bus@58c00000 {
+			compatible = "socionext,uniphier-system-bus";
+			status = "disabled";
+			reg = <0x58c00000 0x400>;
+			#address-cells = <2>;
+			#size-cells = <1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_system_bus>;
+		};
 
-&mio_rst {
-	compatible = "socionext,uniphier-ld4-mio-reset";
-	resets = <&sys_rst 7>;
-};
+		smpctrl@59800000 {
+			compatible = "socionext,uniphier-smpctrl";
+			reg = <0x59801000 0x400>;
+		};
 
-&peri_clk {
-	compatible = "socionext,uniphier-ld4-peri-clock";
-};
+		mioctrl@59810000 {
+			compatible = "socionext,uniphier-ld4-mioctrl",
+				     "simple-mfd", "syscon";
+			reg = <0x59810000 0x800>;
 
-&peri_rst {
-	compatible = "socionext,uniphier-ld4-peri-reset";
-};
+			mio_clk: clock {
+				compatible = "socionext,uniphier-ld4-mio-clock";
+				#clock-cells = <1>;
+			};
 
-&pinctrl {
-	compatible = "socionext,uniphier-ld4-pinctrl";
-};
+			mio_rst: reset {
+				compatible = "socionext,uniphier-ld4-mio-reset";
+				#reset-cells = <1>;
+			};
+		};
 
-&sys_clk {
-	compatible = "socionext,uniphier-ld4-clock";
-};
+		perictrl@59820000 {
+			compatible = "socionext,uniphier-ld4-perictrl",
+				     "simple-mfd", "syscon";
+			reg = <0x59820000 0x200>;
+
+			peri_clk: clock {
+				compatible = "socionext,uniphier-ld4-peri-clock";
+				#clock-cells = <1>;
+			};
+
+			peri_rst: reset {
+				compatible = "socionext,uniphier-ld4-peri-reset";
+				#reset-cells = <1>;
+			};
+		};
+
+		usb0: usb@5a800100 {
+			compatible = "socionext,uniphier-ehci", "generic-ehci";
+			status = "disabled";
+			reg = <0x5a800100 0x100>;
+			interrupts = <0 80 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_usb0>;
+			clocks = <&mio_clk 7>, <&mio_clk 8>, <&mio_clk 12>;
+			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 8>,
+				 <&mio_rst 12>;
+		};
+
+		usb1: usb@5a810100 {
+			compatible = "socionext,uniphier-ehci", "generic-ehci";
+			status = "disabled";
+			reg = <0x5a810100 0x100>;
+			interrupts = <0 81 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_usb1>;
+			clocks = <&mio_clk 7>, <&mio_clk 9>, <&mio_clk 13>;
+			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 9>,
+				 <&mio_rst 13>;
+		};
 
-&sys_rst {
-	compatible = "socionext,uniphier-ld4-reset";
+		usb2: usb@5a820100 {
+			compatible = "socionext,uniphier-ehci", "generic-ehci";
+			status = "disabled";
+			reg = <0x5a820100 0x100>;
+			interrupts = <0 82 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_usb2>;
+			clocks = <&mio_clk 7>, <&mio_clk 10>, <&mio_clk 14>;
+			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 10>,
+				 <&mio_rst 14>;
+		};
+
+		soc-glue@5f800000 {
+			compatible = "socionext,uniphier-ld4-soc-glue",
+				     "simple-mfd", "syscon";
+			reg = <0x5f800000 0x2000>;
+
+			pinctrl: pinctrl {
+				compatible = "socionext,uniphier-ld4-pinctrl";
+			};
+		};
+
+		timer@60000200 {
+			compatible = "arm,cortex-a9-global-timer";
+			reg = <0x60000200 0x20>;
+			interrupts = <1 11 0x104>;
+			clocks = <&arm_timer_clk>;
+		};
+
+		timer@60000600 {
+			compatible = "arm,cortex-a9-twd-timer";
+			reg = <0x60000600 0x20>;
+			interrupts = <1 13 0x104>;
+			clocks = <&arm_timer_clk>;
+		};
+
+		intc: interrupt-controller@60001000 {
+			compatible = "arm,cortex-a9-gic";
+			reg = <0x60001000 0x1000>,
+			      <0x60000100 0x100>;
+			#interrupt-cells = <3>;
+			interrupt-controller;
+		};
+
+		sysctrl@61840000 {
+			compatible = "socionext,uniphier-ld4-sysctrl",
+				     "simple-mfd", "syscon";
+			reg = <0x61840000 0x10000>;
+
+			sys_clk: clock {
+				compatible = "socionext,uniphier-ld4-clock";
+				#clock-cells = <1>;
+			};
+
+			sys_rst: reset {
+				compatible = "socionext,uniphier-ld4-reset";
+				#reset-cells = <1>;
+			};
+		};
+	};
 };
+
+/include/ "uniphier-pinctrl.dtsi"
diff --git a/arch/arm/boot/dts/uniphier-pro4.dtsi b/arch/arm/boot/dts/uniphier-pro4.dtsi
index ba700267ad66..e960b09ff01c 100644
--- a/arch/arm/boot/dts/uniphier-pro4.dtsi
+++ b/arch/arm/boot/dts/uniphier-pro4.dtsi
@@ -43,7 +43,7 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
-/include/ "uniphier-common32.dtsi"
+/include/ "skeleton.dtsi"
 
 / {
 	compatible = "socionext,uniphier-pro4";
@@ -69,155 +69,279 @@
 		};
 	};
 
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
 	clocks {
+		refclk: ref {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <25000000>;
+		};
+
 		arm_timer_clk: arm_timer_clk {
 			#clock-cells = <0>;
 			compatible = "fixed-clock";
 			clock-frequency = <50000000>;
 		};
 	};
-};
 
-&soc {
-	l2: l2-cache@500c0000 {
-		compatible = "socionext,uniphier-system-cache";
-		reg = <0x500c0000 0x2000>, <0x503c0100 0x4>, <0x506c0000 0x400>;
-		interrupts = <0 174 4>, <0 175 4>;
-		cache-unified;
-		cache-size = <(768 * 1024)>;
-		cache-sets = <256>;
-		cache-line-size = <128>;
-		cache-level = <2>;
-	};
-
-	i2c0: i2c@58780000 {
-		compatible = "socionext,uniphier-fi2c";
-		status = "disabled";
-		reg = <0x58780000 0x80>;
+	soc {
+		compatible = "simple-bus";
 		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 41 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c0>;
-		clocks = <&peri_clk 4>;
-		clock-frequency = <100000>;
-	};
+		#size-cells = <1>;
+		ranges;
+		interrupt-parent = <&intc>;
 
-	i2c1: i2c@58781000 {
-		compatible = "socionext,uniphier-fi2c";
-		status = "disabled";
-		reg = <0x58781000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 42 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c1>;
-		clocks = <&peri_clk 5>;
-		clock-frequency = <100000>;
-	};
+		l2: l2-cache@500c0000 {
+			compatible = "socionext,uniphier-system-cache";
+			reg = <0x500c0000 0x2000>, <0x503c0100 0x4>,
+			      <0x506c0000 0x400>;
+			interrupts = <0 174 4>, <0 175 4>;
+			cache-unified;
+			cache-size = <(768 * 1024)>;
+			cache-sets = <256>;
+			cache-line-size = <128>;
+			cache-level = <2>;
+		};
 
-	i2c2: i2c@58782000 {
-		compatible = "socionext,uniphier-fi2c";
-		status = "disabled";
-		reg = <0x58782000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 43 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c2>;
-		clocks = <&peri_clk 6>;
-		clock-frequency = <100000>;
-	};
+		serial0: serial@54006800 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006800 0x40>;
+			interrupts = <0 33 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart0>;
+			clocks = <&peri_clk 0>;
+		};
 
-	i2c3: i2c@58783000 {
-		compatible = "socionext,uniphier-fi2c";
-		status = "disabled";
-		reg = <0x58783000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 44 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c3>;
-		clocks = <&peri_clk 7>;
-		clock-frequency = <100000>;
-	};
+		serial1: serial@54006900 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006900 0x40>;
+			interrupts = <0 35 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart1>;
+			clocks = <&peri_clk 1>;
+		};
 
-	/* i2c4 does not exist */
+		serial2: serial@54006a00 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006a00 0x40>;
+			interrupts = <0 37 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart2>;
+			clocks = <&peri_clk 2>;
+		};
 
-	/* chip-internal connection for DMD */
-	i2c5: i2c@58785000 {
-		compatible = "socionext,uniphier-fi2c";
-		reg = <0x58785000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 25 4>;
-		clocks = <&peri_clk 9>;
-		clock-frequency = <400000>;
-	};
+		serial3: serial@54006b00 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006b00 0x40>;
+			interrupts = <0 177 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart3>;
+			clocks = <&peri_clk 3>;
+		};
 
-	/* chip-internal connection for HDMI */
-	i2c6: i2c@58786000 {
-		compatible = "socionext,uniphier-fi2c";
-		reg = <0x58786000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 26 4>;
-		clocks = <&peri_clk 10>;
-		clock-frequency = <400000>;
-	};
+		i2c0: i2c@58780000 {
+			compatible = "socionext,uniphier-fi2c";
+			status = "disabled";
+			reg = <0x58780000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 41 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c0>;
+			clocks = <&peri_clk 4>;
+			clock-frequency = <100000>;
+		};
 
-	usb2: usb@5a800100 {
-		compatible = "socionext,uniphier-ehci", "generic-ehci";
-		status = "disabled";
-		reg = <0x5a800100 0x100>;
-		interrupts = <0 80 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_usb2>;
-		clocks = <&mio_clk 7>, <&mio_clk 8>, <&mio_clk 12>;
-		resets = <&mio_rst 7>, <&mio_rst 8>, <&mio_rst 12>, <&sys_rst 8>;
-	};
+		i2c1: i2c@58781000 {
+			compatible = "socionext,uniphier-fi2c";
+			status = "disabled";
+			reg = <0x58781000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 42 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c1>;
+			clocks = <&peri_clk 5>;
+			clock-frequency = <100000>;
+		};
 
-	usb3: usb@5a810100 {
-		compatible = "socionext,uniphier-ehci", "generic-ehci";
-		status = "disabled";
-		reg = <0x5a810100 0x100>;
-		interrupts = <0 81 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_usb3>;
-		clocks = <&mio_clk 7>, <&mio_clk 9>, <&mio_clk 13>;
-		resets = <&mio_rst 7>, <&mio_rst 9>, <&mio_rst 13>, <&sys_rst 8>;
-	};
-};
+		i2c2: i2c@58782000 {
+			compatible = "socionext,uniphier-fi2c";
+			status = "disabled";
+			reg = <0x58782000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 43 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c2>;
+			clocks = <&peri_clk 6>;
+			clock-frequency = <100000>;
+		};
 
-&refclk {
-	clock-frequency = <25000000>;
-};
+		i2c3: i2c@58783000 {
+			compatible = "socionext,uniphier-fi2c";
+			status = "disabled";
+			reg = <0x58783000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 44 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c3>;
+			clocks = <&peri_clk 7>;
+			clock-frequency = <100000>;
+		};
 
-&mio_clk {
-	compatible = "socionext,uniphier-pro4-mio-clock";
-};
+		/* i2c4 does not exist */
 
-&mio_rst {
-	compatible = "socionext,uniphier-pro4-mio-reset";
-	resets = <&sys_rst 7>;
-};
+		/* chip-internal connection for DMD */
+		i2c5: i2c@58785000 {
+			compatible = "socionext,uniphier-fi2c";
+			reg = <0x58785000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 25 4>;
+			clocks = <&peri_clk 9>;
+			clock-frequency = <400000>;
+		};
 
-&peri_clk {
-	compatible = "socionext,uniphier-pro4-peri-clock";
-};
+		/* chip-internal connection for HDMI */
+		i2c6: i2c@58786000 {
+			compatible = "socionext,uniphier-fi2c";
+			reg = <0x58786000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 26 4>;
+			clocks = <&peri_clk 10>;
+			clock-frequency = <400000>;
+		};
 
-&peri_rst {
-	compatible = "socionext,uniphier-pro4-peri-reset";
-};
+		system_bus: system-bus@58c00000 {
+			compatible = "socionext,uniphier-system-bus";
+			status = "disabled";
+			reg = <0x58c00000 0x400>;
+			#address-cells = <2>;
+			#size-cells = <1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_system_bus>;
+		};
 
-&pinctrl {
-	compatible = "socionext,uniphier-pro4-pinctrl";
-};
+		smpctrl@59800000 {
+			compatible = "socionext,uniphier-smpctrl";
+			reg = <0x59801000 0x400>;
+		};
 
-&sys_clk {
-	compatible = "socionext,uniphier-pro4-clock";
-};
+		mioctrl@59810000 {
+			compatible = "socionext,uniphier-pro4-mioctrl",
+				     "simple-mfd", "syscon";
+			reg = <0x59810000 0x800>;
+
+			mio_clk: clock {
+				compatible = "socionext,uniphier-pro4-mio-clock";
+				#clock-cells = <1>;
+			};
+
+			mio_rst: reset {
+				compatible = "socionext,uniphier-pro4-mio-reset";
+				#reset-cells = <1>;
+			};
+		};
+
+		perictrl@59820000 {
+			compatible = "socionext,uniphier-pro4-perictrl",
+				     "simple-mfd", "syscon";
+			reg = <0x59820000 0x200>;
+
+			peri_clk: clock {
+				compatible = "socionext,uniphier-pro4-peri-clock";
+				#clock-cells = <1>;
+			};
+
+			peri_rst: reset {
+				compatible = "socionext,uniphier-pro4-peri-reset";
+				#reset-cells = <1>;
+			};
+		};
 
-&sys_rst {
-	compatible = "socionext,uniphier-pro4-reset";
+		usb2: usb@5a800100 {
+			compatible = "socionext,uniphier-ehci", "generic-ehci";
+			status = "disabled";
+			reg = <0x5a800100 0x100>;
+			interrupts = <0 80 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_usb2>;
+			clocks = <&mio_clk 7>, <&mio_clk 8>, <&mio_clk 12>;
+			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 8>,
+				 <&mio_rst 12>;
+		};
+
+		usb3: usb@5a810100 {
+			compatible = "socionext,uniphier-ehci", "generic-ehci";
+			status = "disabled";
+			reg = <0x5a810100 0x100>;
+			interrupts = <0 81 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_usb3>;
+			clocks = <&mio_clk 7>, <&mio_clk 9>, <&mio_clk 13>;
+			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 9>,
+				 <&mio_rst 13>;
+		};
+
+		soc-glue@5f800000 {
+			compatible = "socionext,uniphier-pro4-soc-glue",
+				     "simple-mfd", "syscon";
+			reg = <0x5f800000 0x2000>;
+
+			pinctrl: pinctrl {
+				compatible = "socionext,uniphier-pro4-pinctrl";
+			};
+		};
+
+		timer@60000200 {
+			compatible = "arm,cortex-a9-global-timer";
+			reg = <0x60000200 0x20>;
+			interrupts = <1 11 0x304>;
+			clocks = <&arm_timer_clk>;
+		};
+
+		timer@60000600 {
+			compatible = "arm,cortex-a9-twd-timer";
+			reg = <0x60000600 0x20>;
+			interrupts = <1 13 0x304>;
+			clocks = <&arm_timer_clk>;
+		};
+
+		intc: interrupt-controller@60001000 {
+			compatible = "arm,cortex-a9-gic";
+			reg = <0x60001000 0x1000>,
+			      <0x60000100 0x100>;
+			#interrupt-cells = <3>;
+			interrupt-controller;
+		};
+
+		sysctrl@61840000 {
+			compatible = "socionext,uniphier-pro4-sysctrl",
+				     "simple-mfd", "syscon";
+			reg = <0x61840000 0x10000>;
+
+			sys_clk: clock {
+				compatible = "socionext,uniphier-pro4-clock";
+				#clock-cells = <1>;
+			};
+
+			sys_rst: reset {
+				compatible = "socionext,uniphier-pro4-reset";
+				#reset-cells = <1>;
+			};
+		};
+	};
 };
+
+/include/ "uniphier-pinctrl.dtsi"
diff --git a/arch/arm/boot/dts/uniphier-pro5.dtsi b/arch/arm/boot/dts/uniphier-pro5.dtsi
index 5357ea9c14b1..dbc5e5333163 100644
--- a/arch/arm/boot/dts/uniphier-pro5.dtsi
+++ b/arch/arm/boot/dts/uniphier-pro5.dtsi
@@ -43,7 +43,7 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
-/include/ "uniphier-common32.dtsi"
+/include/ "skeleton.dtsi"
 
 / {
 	compatible = "socionext,uniphier-pro5";
@@ -56,157 +56,355 @@
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <0>;
+			clocks = <&sys_clk 32>;
 			enable-method = "psci";
 			next-level-cache = <&l2>;
+			operating-points-v2 = <&cpu_opp>;
 		};
 
 		cpu@1 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <1>;
+			clocks = <&sys_clk 32>;
 			enable-method = "psci";
 			next-level-cache = <&l2>;
+			operating-points-v2 = <&cpu_opp>;
 		};
 	};
 
+	cpu_opp: opp_table {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp@100000000 {
+			opp-hz = /bits/ 64 <100000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@116667000 {
+			opp-hz = /bits/ 64 <116667000>;
+			clock-latency-ns = <300>;
+		};
+		opp@150000000 {
+			opp-hz = /bits/ 64 <150000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@175000000 {
+			opp-hz = /bits/ 64 <175000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@200000000 {
+			opp-hz = /bits/ 64 <200000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@233334000 {
+			opp-hz = /bits/ 64 <233334000>;
+			clock-latency-ns = <300>;
+		};
+		opp@300000000 {
+			opp-hz = /bits/ 64 <300000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@350000000 {
+			opp-hz = /bits/ 64 <350000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@400000000 {
+			opp-hz = /bits/ 64 <400000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@466667000 {
+			opp-hz = /bits/ 64 <466667000>;
+			clock-latency-ns = <300>;
+		};
+		opp@600000000 {
+			opp-hz = /bits/ 64 <600000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@700000000 {
+			opp-hz = /bits/ 64 <700000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@933334000 {
+			opp-hz = /bits/ 64 <933334000>;
+			clock-latency-ns = <300>;
+		};
+		opp@1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@1400000000 {
+			opp-hz = /bits/ 64 <1400000000>;
+			clock-latency-ns = <300>;
+		};
+	};
+
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
 	clocks {
+		refclk: ref {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <20000000>;
+		};
+
 		arm_timer_clk: arm_timer_clk {
 			#clock-cells = <0>;
 			compatible = "fixed-clock";
 			clock-frequency = <50000000>;
 		};
 	};
-};
 
-&soc {
-	l2: l2-cache@500c0000 {
-		compatible = "socionext,uniphier-system-cache";
-		reg = <0x500c0000 0x2000>, <0x503c0100 0x8>, <0x506c0000 0x400>;
-		interrupts = <0 190 4>, <0 191 4>;
-		cache-unified;
-		cache-size = <(2 * 1024 * 1024)>;
-		cache-sets = <512>;
-		cache-line-size = <128>;
-		cache-level = <2>;
-		next-level-cache = <&l3>;
-	};
+	soc {
+		compatible = "simple-bus";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+		interrupt-parent = <&intc>;
 
-	l3: l3-cache@500c8000 {
-		compatible = "socionext,uniphier-system-cache";
-		reg = <0x500c8000 0x2000>, <0x503c8100 0x8>, <0x506c8000 0x400>;
-		interrupts = <0 174 4>, <0 175 4>;
-		cache-unified;
-		cache-size = <(2 * 1024 * 1024)>;
-		cache-sets = <512>;
-		cache-line-size = <256>;
-		cache-level = <3>;
-	};
+		l2: l2-cache@500c0000 {
+			compatible = "socionext,uniphier-system-cache";
+			reg = <0x500c0000 0x2000>, <0x503c0100 0x8>,
+			      <0x506c0000 0x400>;
+			interrupts = <0 190 4>, <0 191 4>;
+			cache-unified;
+			cache-size = <(2 * 1024 * 1024)>;
+			cache-sets = <512>;
+			cache-line-size = <128>;
+			cache-level = <2>;
+			next-level-cache = <&l3>;
+		};
 
-	i2c0: i2c@58780000 {
-		compatible = "socionext,uniphier-fi2c";
-		status = "disabled";
-		reg = <0x58780000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 41 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c0>;
-		clocks = <&peri_clk 4>;
-		clock-frequency = <100000>;
-	};
+		l3: l3-cache@500c8000 {
+			compatible = "socionext,uniphier-system-cache";
+			reg = <0x500c8000 0x2000>, <0x503c8100 0x8>,
+			      <0x506c8000 0x400>;
+			interrupts = <0 174 4>, <0 175 4>;
+			cache-unified;
+			cache-size = <(2 * 1024 * 1024)>;
+			cache-sets = <512>;
+			cache-line-size = <256>;
+			cache-level = <3>;
+		};
 
-	i2c1: i2c@58781000 {
-		compatible = "socionext,uniphier-fi2c";
-		status = "disabled";
-		reg = <0x58781000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 42 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c1>;
-		clocks = <&peri_clk 5>;
-		clock-frequency = <100000>;
-	};
+		serial0: serial@54006800 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006800 0x40>;
+			interrupts = <0 33 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart0>;
+			clocks = <&peri_clk 0>;
+		};
 
-	i2c2: i2c@58782000 {
-		compatible = "socionext,uniphier-fi2c";
-		status = "disabled";
-		reg = <0x58782000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 43 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c2>;
-		clocks = <&peri_clk 6>;
-		clock-frequency = <100000>;
-	};
+		serial1: serial@54006900 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006900 0x40>;
+			interrupts = <0 35 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart1>;
+			clocks = <&peri_clk 1>;
+		};
 
-	i2c3: i2c@58783000 {
-		compatible = "socionext,uniphier-fi2c";
-		status = "disabled";
-		reg = <0x58783000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 44 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c3>;
-		clocks = <&peri_clk 7>;
-		clock-frequency = <100000>;
-	};
+		serial2: serial@54006a00 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006a00 0x40>;
+			interrupts = <0 37 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart2>;
+			clocks = <&peri_clk 2>;
+		};
 
-	/* i2c4 does not exist */
+		serial3: serial@54006b00 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006b00 0x40>;
+			interrupts = <0 177 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart3>;
+			clocks = <&peri_clk 3>;
+		};
 
-	/* chip-internal connection for DMD */
-	i2c5: i2c@58785000 {
-		compatible = "socionext,uniphier-fi2c";
-		reg = <0x58785000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 25 4>;
-		clocks = <&peri_clk 9>;
-		clock-frequency = <400000>;
-	};
+		i2c0: i2c@58780000 {
+			compatible = "socionext,uniphier-fi2c";
+			status = "disabled";
+			reg = <0x58780000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 41 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c0>;
+			clocks = <&peri_clk 4>;
+			clock-frequency = <100000>;
+		};
 
-	/* chip-internal connection for HDMI */
-	i2c6: i2c@58786000 {
-		compatible = "socionext,uniphier-fi2c";
-		reg = <0x58786000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 26 4>;
-		clocks = <&peri_clk 10>;
-		clock-frequency = <400000>;
-	};
-};
+		i2c1: i2c@58781000 {
+			compatible = "socionext,uniphier-fi2c";
+			status = "disabled";
+			reg = <0x58781000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 42 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c1>;
+			clocks = <&peri_clk 5>;
+			clock-frequency = <100000>;
+		};
 
-&refclk {
-	clock-frequency = <20000000>;
-};
+		i2c2: i2c@58782000 {
+			compatible = "socionext,uniphier-fi2c";
+			status = "disabled";
+			reg = <0x58782000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 43 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c2>;
+			clocks = <&peri_clk 6>;
+			clock-frequency = <100000>;
+		};
 
-&mio_clk {
-	compatible = "socionext,uniphier-pro5-sd-clock";
-};
+		i2c3: i2c@58783000 {
+			compatible = "socionext,uniphier-fi2c";
+			status = "disabled";
+			reg = <0x58783000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 44 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c3>;
+			clocks = <&peri_clk 7>;
+			clock-frequency = <100000>;
+		};
 
-&mio_rst {
-	compatible = "socionext,uniphier-pro5-sd-reset";
-};
+		/* i2c4 does not exist */
 
-&peri_clk {
-	compatible = "socionext,uniphier-pro5-peri-clock";
-};
+		/* chip-internal connection for DMD */
+		i2c5: i2c@58785000 {
+			compatible = "socionext,uniphier-fi2c";
+			reg = <0x58785000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 25 4>;
+			clocks = <&peri_clk 9>;
+			clock-frequency = <400000>;
+		};
 
-&peri_rst {
-	compatible = "socionext,uniphier-pro5-peri-reset";
-};
+		/* chip-internal connection for HDMI */
+		i2c6: i2c@58786000 {
+			compatible = "socionext,uniphier-fi2c";
+			reg = <0x58786000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 26 4>;
+			clocks = <&peri_clk 10>;
+			clock-frequency = <400000>;
+		};
 
-&pinctrl {
-	compatible = "socionext,uniphier-pro5-pinctrl";
-};
+		system_bus: system-bus@58c00000 {
+			compatible = "socionext,uniphier-system-bus";
+			status = "disabled";
+			reg = <0x58c00000 0x400>;
+			#address-cells = <2>;
+			#size-cells = <1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_system_bus>;
+		};
 
-&sys_clk {
-	compatible = "socionext,uniphier-pro5-clock";
-};
+		smpctrl@59800000 {
+			compatible = "socionext,uniphier-smpctrl";
+			reg = <0x59801000 0x400>;
+		};
 
-&sys_rst {
-	compatible = "socionext,uniphier-pro5-reset";
+		sdctrl@59810000 {
+			compatible = "socionext,uniphier-pro5-sdctrl",
+				     "simple-mfd", "syscon";
+			reg = <0x59810000 0x800>;
+
+			sd_clk: clock {
+				compatible = "socionext,uniphier-pro5-sd-clock";
+				#clock-cells = <1>;
+			};
+
+			sd_rst: reset {
+				compatible = "socionext,uniphier-pro5-sd-reset";
+				#reset-cells = <1>;
+			};
+		};
+
+		perictrl@59820000 {
+			compatible = "socionext,uniphier-pro5-perictrl",
+				     "simple-mfd", "syscon";
+			reg = <0x59820000 0x200>;
+
+			peri_clk: clock {
+				compatible = "socionext,uniphier-pro5-peri-clock";
+				#clock-cells = <1>;
+			};
+
+			peri_rst: reset {
+				compatible = "socionext,uniphier-pro5-peri-reset";
+				#reset-cells = <1>;
+			};
+		};
+
+		soc-glue@5f800000 {
+			compatible = "socionext,uniphier-pro5-soc-glue",
+				     "simple-mfd", "syscon";
+			reg = <0x5f800000 0x2000>;
+
+			pinctrl: pinctrl {
+				compatible = "socionext,uniphier-pro5-pinctrl";
+			};
+		};
+
+		timer@60000200 {
+			compatible = "arm,cortex-a9-global-timer";
+			reg = <0x60000200 0x20>;
+			interrupts = <1 11 0x304>;
+			clocks = <&arm_timer_clk>;
+		};
+
+		timer@60000600 {
+			compatible = "arm,cortex-a9-twd-timer";
+			reg = <0x60000600 0x20>;
+			interrupts = <1 13 0x304>;
+			clocks = <&arm_timer_clk>;
+		};
+
+		intc: interrupt-controller@60001000 {
+			compatible = "arm,cortex-a9-gic";
+			reg = <0x60001000 0x1000>,
+			      <0x60000100 0x100>;
+			#interrupt-cells = <3>;
+			interrupt-controller;
+		};
+
+		sysctrl@61840000 {
+			compatible = "socionext,uniphier-pro5-sysctrl",
+				     "simple-mfd", "syscon";
+			reg = <0x61840000 0x10000>;
+
+			sys_clk: clock {
+				compatible = "socionext,uniphier-pro5-clock";
+				#clock-cells = <1>;
+			};
+
+			sys_rst: reset {
+				compatible = "socionext,uniphier-pro5-reset";
+				#reset-cells = <1>;
+			};
+		};
+	};
 };
+
+/include/ "uniphier-pinctrl.dtsi"
diff --git a/arch/arm/boot/dts/uniphier-pxs2.dtsi b/arch/arm/boot/dts/uniphier-pxs2.dtsi
index 950f07ba0337..e9e031d63c1a 100644
--- a/arch/arm/boot/dts/uniphier-pxs2.dtsi
+++ b/arch/arm/boot/dts/uniphier-pxs2.dtsi
@@ -43,7 +43,7 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
-/include/ "uniphier-common32.dtsi"
+/include/ "skeleton.dtsi"
 
 / {
 	compatible = "socionext,uniphier-pxs2";
@@ -56,170 +56,339 @@
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <0>;
+			clocks = <&sys_clk 32>;
 			enable-method = "psci";
 			next-level-cache = <&l2>;
+			operating-points-v2 = <&cpu_opp>;
 		};
 
 		cpu@1 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <1>;
+			clocks = <&sys_clk 32>;
 			enable-method = "psci";
 			next-level-cache = <&l2>;
+			operating-points-v2 = <&cpu_opp>;
 		};
 
 		cpu@2 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <2>;
+			clocks = <&sys_clk 32>;
 			enable-method = "psci";
 			next-level-cache = <&l2>;
+			operating-points-v2 = <&cpu_opp>;
 		};
 
 		cpu@3 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <3>;
+			clocks = <&sys_clk 32>;
 			enable-method = "psci";
 			next-level-cache = <&l2>;
+			operating-points-v2 = <&cpu_opp>;
 		};
 	};
 
+	cpu_opp: opp_table {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp@100000000 {
+			opp-hz = /bits/ 64 <100000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@150000000 {
+			opp-hz = /bits/ 64 <150000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@200000000 {
+			opp-hz = /bits/ 64 <200000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@300000000 {
+			opp-hz = /bits/ 64 <300000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@400000000 {
+			opp-hz = /bits/ 64 <400000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@600000000 {
+			opp-hz = /bits/ 64 <600000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
+			clock-latency-ns = <300>;
+		};
+	};
+
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
 	clocks {
+		refclk: ref {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <25000000>;
+		};
+
 		arm_timer_clk: arm_timer_clk {
 			#clock-cells = <0>;
 			compatible = "fixed-clock";
 			clock-frequency = <50000000>;
 		};
 	};
-};
-
-&soc {
-	l2: l2-cache@500c0000 {
-		compatible = "socionext,uniphier-system-cache";
-		reg = <0x500c0000 0x2000>, <0x503c0100 0x4>, <0x506c0000 0x400>;
-		interrupts = <0 174 4>, <0 175 4>, <0 190 4>, <0 191 4>;
-		cache-unified;
-		cache-size = <(1280 * 1024)>;
-		cache-sets = <512>;
-		cache-line-size = <128>;
-		cache-level = <2>;
-	};
 
-	i2c0: i2c@58780000 {
-		compatible = "socionext,uniphier-fi2c";
-		status = "disabled";
-		reg = <0x58780000 0x80>;
+	soc {
+		compatible = "simple-bus";
 		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 41 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c0>;
-		clocks = <&peri_clk 4>;
-		clock-frequency = <100000>;
-	};
+		#size-cells = <1>;
+		ranges;
+		interrupt-parent = <&intc>;
 
-	i2c1: i2c@58781000 {
-		compatible = "socionext,uniphier-fi2c";
-		status = "disabled";
-		reg = <0x58781000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 42 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c1>;
-		clocks = <&peri_clk 5>;
-		clock-frequency = <100000>;
-	};
+		l2: l2-cache@500c0000 {
+			compatible = "socionext,uniphier-system-cache";
+			reg = <0x500c0000 0x2000>, <0x503c0100 0x8>,
+			      <0x506c0000 0x400>;
+			interrupts = <0 174 4>, <0 175 4>, <0 190 4>, <0 191 4>;
+			cache-unified;
+			cache-size = <(1280 * 1024)>;
+			cache-sets = <512>;
+			cache-line-size = <128>;
+			cache-level = <2>;
+		};
 
-	i2c2: i2c@58782000 {
-		compatible = "socionext,uniphier-fi2c";
-		status = "disabled";
-		reg = <0x58782000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c2>;
-		interrupts = <0 43 4>;
-		clocks = <&peri_clk 6>;
-		clock-frequency = <100000>;
-	};
+		serial0: serial@54006800 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006800 0x40>;
+			interrupts = <0 33 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart0>;
+			clocks = <&peri_clk 0>;
+		};
 
-	i2c3: i2c@58783000 {
-		compatible = "socionext,uniphier-fi2c";
-		status = "disabled";
-		reg = <0x58783000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 44 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c3>;
-		clocks = <&peri_clk 7>;
-		clock-frequency = <100000>;
-	};
+		serial1: serial@54006900 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006900 0x40>;
+			interrupts = <0 35 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart1>;
+			clocks = <&peri_clk 1>;
+		};
 
-	/* chip-internal connection for DMD */
-	i2c4: i2c@58784000 {
-		compatible = "socionext,uniphier-fi2c";
-		reg = <0x58784000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 45 4>;
-		clocks = <&peri_clk 8>;
-		clock-frequency = <400000>;
-	};
+		serial2: serial@54006a00 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006a00 0x40>;
+			interrupts = <0 37 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart2>;
+			clocks = <&peri_clk 2>;
+		};
 
-	/* chip-internal connection for STM */
-	i2c5: i2c@58785000 {
-		compatible = "socionext,uniphier-fi2c";
-		reg = <0x58785000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 25 4>;
-		clocks = <&peri_clk 9>;
-		clock-frequency = <400000>;
-	};
+		serial3: serial@54006b00 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006b00 0x40>;
+			interrupts = <0 177 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart3>;
+			clocks = <&peri_clk 3>;
+		};
 
-	/* chip-internal connection for HDMI */
-	i2c6: i2c@58786000 {
-		compatible = "socionext,uniphier-fi2c";
-		reg = <0x58786000 0x80>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 26 4>;
-		clocks = <&peri_clk 10>;
-		clock-frequency = <400000>;
-	};
-};
+		i2c0: i2c@58780000 {
+			compatible = "socionext,uniphier-fi2c";
+			status = "disabled";
+			reg = <0x58780000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 41 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c0>;
+			clocks = <&peri_clk 4>;
+			clock-frequency = <100000>;
+		};
 
-&refclk {
-	clock-frequency = <25000000>;
-};
+		i2c1: i2c@58781000 {
+			compatible = "socionext,uniphier-fi2c";
+			status = "disabled";
+			reg = <0x58781000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 42 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c1>;
+			clocks = <&peri_clk 5>;
+			clock-frequency = <100000>;
+		};
 
-&mio_clk {
-	compatible = "socionext,uniphier-pxs2-sd-clock";
-};
+		i2c2: i2c@58782000 {
+			compatible = "socionext,uniphier-fi2c";
+			status = "disabled";
+			reg = <0x58782000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 43 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c2>;
+			clocks = <&peri_clk 6>;
+			clock-frequency = <100000>;
+		};
 
-&mio_rst {
-	compatible = "socionext,uniphier-pxs2-sd-reset";
-};
+		i2c3: i2c@58783000 {
+			compatible = "socionext,uniphier-fi2c";
+			status = "disabled";
+			reg = <0x58783000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 44 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c3>;
+			clocks = <&peri_clk 7>;
+			clock-frequency = <100000>;
+		};
 
-&peri_clk {
-	compatible = "socionext,uniphier-pxs2-peri-clock";
-};
+		/* chip-internal connection for DMD */
+		i2c4: i2c@58784000 {
+			compatible = "socionext,uniphier-fi2c";
+			reg = <0x58784000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 45 4>;
+			clocks = <&peri_clk 8>;
+			clock-frequency = <400000>;
+		};
 
-&peri_rst {
-	compatible = "socionext,uniphier-pxs2-peri-reset";
-};
+		/* chip-internal connection for STM */
+		i2c5: i2c@58785000 {
+			compatible = "socionext,uniphier-fi2c";
+			reg = <0x58785000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 25 4>;
+			clocks = <&peri_clk 9>;
+			clock-frequency = <400000>;
+		};
 
-&pinctrl {
-	compatible = "socionext,uniphier-pxs2-pinctrl";
-};
+		/* chip-internal connection for HDMI */
+		i2c6: i2c@58786000 {
+			compatible = "socionext,uniphier-fi2c";
+			reg = <0x58786000 0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 26 4>;
+			clocks = <&peri_clk 10>;
+			clock-frequency = <400000>;
+		};
 
-&sys_clk {
-	compatible = "socionext,uniphier-pxs2-clock";
-};
+		system_bus: system-bus@58c00000 {
+			compatible = "socionext,uniphier-system-bus";
+			status = "disabled";
+			reg = <0x58c00000 0x400>;
+			#address-cells = <2>;
+			#size-cells = <1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_system_bus>;
+		};
+
+		smpctrl@59800000 {
+			compatible = "socionext,uniphier-smpctrl";
+			reg = <0x59801000 0x400>;
+		};
+
+		sdctrl@59810000 {
+			compatible = "socionext,uniphier-pxs2-sdctrl",
+				     "simple-mfd", "syscon";
+			reg = <0x59810000 0x800>;
 
-&sys_rst {
-	compatible = "socionext,uniphier-pxs2-reset";
+			sd_clk: clock {
+				compatible = "socionext,uniphier-pxs2-sd-clock";
+				#clock-cells = <1>;
+			};
+
+			sd_rst: reset {
+				compatible = "socionext,uniphier-pxs2-sd-reset";
+				#reset-cells = <1>;
+			};
+		};
+
+		perictrl@59820000 {
+			compatible = "socionext,uniphier-pxs2-perictrl",
+				     "simple-mfd", "syscon";
+			reg = <0x59820000 0x200>;
+
+			peri_clk: clock {
+				compatible = "socionext,uniphier-pxs2-peri-clock";
+				#clock-cells = <1>;
+			};
+
+			peri_rst: reset {
+				compatible = "socionext,uniphier-pxs2-peri-reset";
+				#reset-cells = <1>;
+			};
+		};
+
+		soc-glue@5f800000 {
+			compatible = "socionext,uniphier-pxs2-soc-glue",
+				     "simple-mfd", "syscon";
+			reg = <0x5f800000 0x2000>;
+
+			pinctrl: pinctrl {
+				compatible = "socionext,uniphier-pxs2-pinctrl";
+			};
+		};
+
+		timer@60000200 {
+			compatible = "arm,cortex-a9-global-timer";
+			reg = <0x60000200 0x20>;
+			interrupts = <1 11 0xf04>;
+			clocks = <&arm_timer_clk>;
+		};
+
+		timer@60000600 {
+			compatible = "arm,cortex-a9-twd-timer";
+			reg = <0x60000600 0x20>;
+			interrupts = <1 13 0xf04>;
+			clocks = <&arm_timer_clk>;
+		};
+
+		intc: interrupt-controller@60001000 {
+			compatible = "arm,cortex-a9-gic";
+			reg = <0x60001000 0x1000>,
+			      <0x60000100 0x100>;
+			#interrupt-cells = <3>;
+			interrupt-controller;
+		};
+
+		sysctrl@61840000 {
+			compatible = "socionext,uniphier-pxs2-sysctrl",
+				     "simple-mfd", "syscon";
+			reg = <0x61840000 0x10000>;
+
+			sys_clk: clock {
+				compatible = "socionext,uniphier-pxs2-clock";
+				#clock-cells = <1>;
+			};
+
+			sys_rst: reset {
+				compatible = "socionext,uniphier-pxs2-reset";
+				#reset-cells = <1>;
+			};
+		};
+	};
 };
+
+/include/ "uniphier-pinctrl.dtsi"
diff --git a/arch/arm/boot/dts/uniphier-sld3.dtsi b/arch/arm/boot/dts/uniphier-sld3.dtsi
index 5fa96c939b5c..9fad6bd2db8a 100644
--- a/arch/arm/boot/dts/uniphier-sld3.dtsi
+++ b/arch/arm/boot/dts/uniphier-sld3.dtsi
@@ -135,7 +135,6 @@
 			reg = <0x54006800 0x40>;
 			interrupts = <0 33 4>;
 			clocks = <&sys_clk 0>;
-			fifo-size = <64>;
 		};
 
 		serial1: serial@54006900 {
@@ -144,7 +143,6 @@
 			reg = <0x54006900 0x40>;
 			interrupts = <0 35 4>;
 			clocks = <&sys_clk 0>;
-			fifo-size = <64>;
 		};
 
 		serial2: serial@54006a00 {
@@ -153,7 +151,6 @@
 			reg = <0x54006a00 0x40>;
 			interrupts = <0 37 4>;
 			clocks = <&sys_clk 0>;
-			fifo-size = <64>;
 		};
 
 		i2c0: i2c@58400000 {
@@ -225,7 +222,7 @@
 		};
 
 		mioctrl@59810000 {
-			compatible = "socionext,uniphier-mioctrl",
+			compatible = "socionext,uniphier-sld3-mioctrl",
 				     "simple-mfd", "syscon";
 			reg = <0x59810000 0x800>;
 
@@ -245,6 +242,9 @@
 			status = "disabled";
 			reg = <0x5a800100 0x100>;
 			interrupts = <0 80 4>;
+			clocks = <&mio_clk 7>, <&mio_clk 8>, <&mio_clk 12>;
+			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 8>,
+				 <&mio_rst 12>;
 		};
 
 		usb1: usb@5a810100 {
@@ -252,6 +252,9 @@
 			status = "disabled";
 			reg = <0x5a810100 0x100>;
 			interrupts = <0 81 4>;
+			clocks = <&mio_clk 7>, <&mio_clk 9>, <&mio_clk 13>;
+			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 9>,
+				 <&mio_rst 13>;
 		};
 
 		usb2: usb@5a820100 {
@@ -259,6 +262,9 @@
 			status = "disabled";
 			reg = <0x5a820100 0x100>;
 			interrupts = <0 82 4>;
+			clocks = <&mio_clk 7>, <&mio_clk 10>, <&mio_clk 14>;
+			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 10>,
+				 <&mio_rst 14>;
 		};
 
 		usb3: usb@5a830100 {
@@ -266,12 +272,15 @@
 			status = "disabled";
 			reg = <0x5a830100 0x100>;
 			interrupts = <0 83 4>;
+			clocks = <&mio_clk 7>, <&mio_clk 11>, <&mio_clk 15>;
+			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 11>,
+				 <&mio_rst 15>;
 		};
 
 		sysctrl@f1840000 {
-			compatible = "socionext,uniphier-sysctrl",
+			compatible = "socionext,uniphier-sld3-sysctrl",
 				     "simple-mfd", "syscon";
-			reg = <0xf1840000 0x4000>;
+			reg = <0xf1840000 0x10000>;
 
 			sys_clk: clock {
 				compatible = "socionext,uniphier-sld3-clock";
diff --git a/arch/arm/boot/dts/uniphier-sld8.dtsi b/arch/arm/boot/dts/uniphier-sld8.dtsi
index d8cf0e7e11ea..b2c980ead7f0 100644
--- a/arch/arm/boot/dts/uniphier-sld8.dtsi
+++ b/arch/arm/boot/dts/uniphier-sld8.dtsi
@@ -43,7 +43,7 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
-/include/ "uniphier-common32.dtsi"
+/include/ "skeleton.dtsi"
 
 / {
 	compatible = "socionext,uniphier-sld8";
@@ -61,146 +61,267 @@
 		};
 	};
 
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
 	clocks {
+		refclk: ref {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <25000000>;
+		};
+
 		arm_timer_clk: arm_timer_clk {
 			#clock-cells = <0>;
 			compatible = "fixed-clock";
 			clock-frequency = <50000000>;
 		};
 	};
-};
-
-&soc {
-	l2: l2-cache@500c0000 {
-		compatible = "socionext,uniphier-system-cache";
-		reg = <0x500c0000 0x2000>, <0x503c0100 0x4>, <0x506c0000 0x400>;
-		interrupts = <0 174 4>, <0 175 4>;
-		cache-unified;
-		cache-size = <(256 * 1024)>;
-		cache-sets = <256>;
-		cache-line-size = <128>;
-		cache-level = <2>;
-	};
 
-	i2c0: i2c@58400000 {
-		compatible = "socionext,uniphier-i2c";
-		status = "disabled";
-		reg = <0x58400000 0x40>;
+	soc {
+		compatible = "simple-bus";
 		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 41 1>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c0>;
-		clocks = <&peri_clk 4>;
-		clock-frequency = <100000>;
-	};
+		#size-cells = <1>;
+		ranges;
+		interrupt-parent = <&intc>;
 
-	i2c1: i2c@58480000 {
-		compatible = "socionext,uniphier-i2c";
-		status = "disabled";
-		reg = <0x58480000 0x40>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 42 1>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c1>;
-		clocks = <&peri_clk 5>;
-		clock-frequency = <100000>;
-	};
+		l2: l2-cache@500c0000 {
+			compatible = "socionext,uniphier-system-cache";
+			reg = <0x500c0000 0x2000>, <0x503c0100 0x4>,
+			      <0x506c0000 0x400>;
+			interrupts = <0 174 4>, <0 175 4>;
+			cache-unified;
+			cache-size = <(256 * 1024)>;
+			cache-sets = <256>;
+			cache-line-size = <128>;
+			cache-level = <2>;
+		};
 
-	/* chip-internal connection for DMD */
-	i2c2: i2c@58500000 {
-		compatible = "socionext,uniphier-i2c";
-		reg = <0x58500000 0x40>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 43 1>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c2>;
-		clocks = <&peri_clk 6>;
-		clock-frequency = <400000>;
-	};
+		serial0: serial@54006800 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006800 0x40>;
+			interrupts = <0 33 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart0>;
+			clocks = <&peri_clk 0>;
+		};
 
-	i2c3: i2c@58580000 {
-		compatible = "socionext,uniphier-i2c";
-		status = "disabled";
-		reg = <0x58580000 0x40>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		interrupts = <0 44 1>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c3>;
-		clocks = <&peri_clk 7>;
-		clock-frequency = <100000>;
-	};
+		serial1: serial@54006900 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006900 0x40>;
+			interrupts = <0 35 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart1>;
+			clocks = <&peri_clk 1>;
+		};
 
-	usb0: usb@5a800100 {
-		compatible = "socionext,uniphier-ehci", "generic-ehci";
-		status = "disabled";
-		reg = <0x5a800100 0x100>;
-		interrupts = <0 80 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_usb0>;
-		clocks = <&mio_clk 7>, <&mio_clk 8>, <&mio_clk 12>;
-		resets = <&mio_rst 7>, <&mio_rst 8>, <&mio_rst 12>, <&sys_rst 8>;
-	};
+		serial2: serial@54006a00 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006a00 0x40>;
+			interrupts = <0 37 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart2>;
+			clocks = <&peri_clk 2>;
+		};
 
-	usb1: usb@5a810100 {
-		compatible = "socionext,uniphier-ehci", "generic-ehci";
-		status = "disabled";
-		reg = <0x5a810100 0x100>;
-		interrupts = <0 81 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_usb1>;
-		clocks = <&mio_clk 7>, <&mio_clk 9>, <&mio_clk 13>;
-		resets = <&mio_rst 7>, <&mio_rst 9>, <&mio_rst 13>, <&sys_rst 8>;
-	};
+		serial3: serial@54006b00 {
+			compatible = "socionext,uniphier-uart";
+			status = "disabled";
+			reg = <0x54006b00 0x40>;
+			interrupts = <0 29 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_uart3>;
+			clocks = <&peri_clk 3>;
+		};
 
-	usb2: usb@5a820100 {
-		compatible = "socionext,uniphier-ehci", "generic-ehci";
-		status = "disabled";
-		reg = <0x5a820100 0x100>;
-		interrupts = <0 82 4>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_usb2>;
-		clocks = <&mio_clk 7>, <&mio_clk 10>, <&mio_clk 14>;
-		resets = <&mio_rst 7>, <&mio_rst 10>, <&mio_rst 14>, <&sys_rst 8>;
-	};
-};
+		i2c0: i2c@58400000 {
+			compatible = "socionext,uniphier-i2c";
+			status = "disabled";
+			reg = <0x58400000 0x40>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 41 1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c0>;
+			clocks = <&peri_clk 4>;
+			clock-frequency = <100000>;
+		};
 
-&refclk {
-	clock-frequency = <25000000>;
-};
+		i2c1: i2c@58480000 {
+			compatible = "socionext,uniphier-i2c";
+			status = "disabled";
+			reg = <0x58480000 0x40>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 42 1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c1>;
+			clocks = <&peri_clk 5>;
+			clock-frequency = <100000>;
+		};
 
-&serial3 {
-	interrupts = <0 29 4>;
-};
+		/* chip-internal connection for DMD */
+		i2c2: i2c@58500000 {
+			compatible = "socionext,uniphier-i2c";
+			reg = <0x58500000 0x40>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 43 1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c2>;
+			clocks = <&peri_clk 6>;
+			clock-frequency = <400000>;
+		};
 
-&mio_clk {
-	compatible = "socionext,uniphier-sld8-mio-clock";
-};
+		i2c3: i2c@58580000 {
+			compatible = "socionext,uniphier-i2c";
+			status = "disabled";
+			reg = <0x58580000 0x40>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <0 44 1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_i2c3>;
+			clocks = <&peri_clk 7>;
+			clock-frequency = <100000>;
+		};
 
-&mio_rst {
-	compatible = "socionext,uniphier-sld8-mio-reset";
-	resets = <&sys_rst 7>;
-};
+		system_bus: system-bus@58c00000 {
+			compatible = "socionext,uniphier-system-bus";
+			status = "disabled";
+			reg = <0x58c00000 0x400>;
+			#address-cells = <2>;
+			#size-cells = <1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_system_bus>;
+		};
 
-&peri_clk {
-	compatible = "socionext,uniphier-sld8-peri-clock";
-};
+		smpctrl@59800000 {
+			compatible = "socionext,uniphier-smpctrl";
+			reg = <0x59801000 0x400>;
+		};
 
-&peri_rst {
-	compatible = "socionext,uniphier-sld8-peri-reset";
-};
+		mioctrl@59810000 {
+			compatible = "socionext,uniphier-sld8-mioctrl",
+				     "simple-mfd", "syscon";
+			reg = <0x59810000 0x800>;
 
-&pinctrl {
-	compatible = "socionext,uniphier-sld8-pinctrl";
-};
+			mio_clk: clock {
+				compatible = "socionext,uniphier-sld8-mio-clock";
+				#clock-cells = <1>;
+			};
 
-&sys_clk {
-	compatible = "socionext,uniphier-sld8-clock";
-};
+			mio_rst: reset {
+				compatible = "socionext,uniphier-sld8-mio-reset";
+				#reset-cells = <1>;
+			};
+		};
+
+		perictrl@59820000 {
+			compatible = "socionext,uniphier-sld8-perictrl",
+				     "simple-mfd", "syscon";
+			reg = <0x59820000 0x200>;
+
+			peri_clk: clock {
+				compatible = "socionext,uniphier-sld8-peri-clock";
+				#clock-cells = <1>;
+			};
+
+			peri_rst: reset {
+				compatible = "socionext,uniphier-sld8-peri-reset";
+				#reset-cells = <1>;
+			};
+		};
+
+		usb0: usb@5a800100 {
+			compatible = "socionext,uniphier-ehci", "generic-ehci";
+			status = "disabled";
+			reg = <0x5a800100 0x100>;
+			interrupts = <0 80 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_usb0>;
+			clocks = <&mio_clk 7>, <&mio_clk 8>, <&mio_clk 12>;
+			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 8>,
+				 <&mio_rst 12>;
+		};
+
+		usb1: usb@5a810100 {
+			compatible = "socionext,uniphier-ehci", "generic-ehci";
+			status = "disabled";
+			reg = <0x5a810100 0x100>;
+			interrupts = <0 81 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_usb1>;
+			clocks = <&mio_clk 7>, <&mio_clk 9>, <&mio_clk 13>;
+			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 9>,
+				 <&mio_rst 13>;
+		};
+
+		usb2: usb@5a820100 {
+			compatible = "socionext,uniphier-ehci", "generic-ehci";
+			status = "disabled";
+			reg = <0x5a820100 0x100>;
+			interrupts = <0 82 4>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_usb2>;
+			clocks = <&mio_clk 7>, <&mio_clk 10>, <&mio_clk 14>;
+			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 10>,
+				 <&mio_rst 14>;
+		};
 
-&sys_rst {
-	compatible = "socionext,uniphier-sld8-reset";
+		soc-glue@5f800000 {
+			compatible = "socionext,uniphier-sld8-soc-glue",
+				     "simple-mfd", "syscon";
+			reg = <0x5f800000 0x2000>;
+
+			pinctrl: pinctrl {
+				compatible = "socionext,uniphier-sld8-pinctrl";
+			};
+		};
+
+		timer@60000200 {
+			compatible = "arm,cortex-a9-global-timer";
+			reg = <0x60000200 0x20>;
+			interrupts = <1 11 0x104>;
+			clocks = <&arm_timer_clk>;
+		};
+
+		timer@60000600 {
+			compatible = "arm,cortex-a9-twd-timer";
+			reg = <0x60000600 0x20>;
+			interrupts = <1 13 0x104>;
+			clocks = <&arm_timer_clk>;
+		};
+
+		intc: interrupt-controller@60001000 {
+			compatible = "arm,cortex-a9-gic";
+			reg = <0x60001000 0x1000>,
+			      <0x60000100 0x100>;
+			#interrupt-cells = <3>;
+			interrupt-controller;
+		};
+
+		sysctrl@61840000 {
+			compatible = "socionext,uniphier-sld8-sysctrl",
+				     "simple-mfd", "syscon";
+			reg = <0x61840000 0x10000>;
+
+			sys_clk: clock {
+				compatible = "socionext,uniphier-sld8-clock";
+				#clock-cells = <1>;
+			};
+
+			sys_rst: reset {
+				compatible = "socionext,uniphier-sld8-reset";
+				#reset-cells = <1>;
+			};
+		};
+	};
 };
+
+/include/ "uniphier-pinctrl.dtsi"
diff --git a/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts b/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts
index 0205c97efdef..45d08cc37b01 100644
--- a/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts
+++ b/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts
@@ -39,6 +39,7 @@
 			reg = <0>;
 			cci-control-port = <&cci_control1>;
 			cpu-idle-states = <&CLUSTER_SLEEP_BIG>;
+			capacity-dmips-mhz = <1024>;
 		};
 
 		cpu1: cpu@1 {
@@ -47,6 +48,7 @@
 			reg = <1>;
 			cci-control-port = <&cci_control1>;
 			cpu-idle-states = <&CLUSTER_SLEEP_BIG>;
+			capacity-dmips-mhz = <1024>;
 		};
 
 		cpu2: cpu@2 {
@@ -55,6 +57,7 @@
 			reg = <0x100>;
 			cci-control-port = <&cci_control2>;
 			cpu-idle-states = <&CLUSTER_SLEEP_LITTLE>;
+			capacity-dmips-mhz = <516>;
 		};
 
 		cpu3: cpu@3 {
@@ -63,6 +66,7 @@
 			reg = <0x101>;
 			cci-control-port = <&cci_control2>;
 			cpu-idle-states = <&CLUSTER_SLEEP_LITTLE>;
+			capacity-dmips-mhz = <516>;
 		};
 
 		cpu4: cpu@4 {
@@ -71,6 +75,7 @@
 			reg = <0x102>;
 			cci-control-port = <&cci_control2>;
 			cpu-idle-states = <&CLUSTER_SLEEP_LITTLE>;
+			capacity-dmips-mhz = <516>;
 		};
 
 		idle-states {
diff --git a/arch/arm/boot/dts/vf-colibri.dtsi b/arch/arm/boot/dts/vf-colibri.dtsi
index b7417094dc11..21bfef957b68 100644
--- a/arch/arm/boot/dts/vf-colibri.dtsi
+++ b/arch/arm/boot/dts/vf-colibri.dtsi
@@ -108,6 +108,10 @@
 	status = "okay";
 };
 
+&edma1 {
+	status = "okay";
+};
+
 &esdhc1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_esdhc1>;
diff --git a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
index 1552db00cc59..7ea617e47fe4 100644
--- a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
+++ b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
@@ -538,13 +538,6 @@
 	};
 };
 
-&i2c3 {
-	clock-frequency = <100000>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_i2c3>;
-	status = "okay";
-};
-
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart0>;
@@ -714,13 +707,6 @@
 		>;
 	};
 
-	pinctrl_i2c3: i2c3grp {
-		fsl,pins = <
-			VF610_PAD_PTA30__I2C3_SCL	0x37ff
-			VF610_PAD_PTA31__I2C3_SDA	0x37ff
-		>;
-	};
-
 	pinctrl_leds_debug: pinctrl-leds-debug {
 		fsl,pins = <
 			 VF610_PAD_PTD20__GPIO_74	0x31c2
diff --git a/arch/arm/boot/dts/vfxxx.dtsi b/arch/arm/boot/dts/vfxxx.dtsi
index 2c13ec696ac5..e9d28474c26a 100644
--- a/arch/arm/boot/dts/vfxxx.dtsi
+++ b/arch/arm/boot/dts/vfxxx.dtsi
@@ -194,6 +194,9 @@
 				clocks = <&clks VF610_CLK_DSPI0>;
 				clock-names = "dspi";
 				spi-num-chipselects = <6>;
+				dmas = <&edma1 1 12>,
+					<&edma1 1 13>;
+				dma-names = "rx", "tx";
 				status = "disabled";
 			};
 
@@ -206,6 +209,9 @@
 				clocks = <&clks VF610_CLK_DSPI1>;
 				clock-names = "dspi";
 				spi-num-chipselects = <4>;
+				dmas = <&edma1 1 14>,
+					<&edma1 1 15>;
+				dma-names = "rx", "tx";
 				status = "disabled";
 			};
 
@@ -520,6 +526,12 @@
 				status = "disabled";
 			};
 
+			ocotp: ocotp@400a5000 {
+				compatible = "fsl,vf610-ocotp";
+				reg = <0x400a5000 0x1000>;
+				clocks = <&clks VF610_CLK_OCOTP>;
+			};
+
 			snvs0: snvs@400a7000 {
 			    compatible = "fsl,sec-v4.0-mon", "syscon", "simple-mfd";
 				reg = <0x400a7000 0x2000>;
@@ -561,6 +573,9 @@
 				clocks = <&clks VF610_CLK_DSPI2>;
 				clock-names = "dspi";
 				spi-num-chipselects = <2>;
+				dmas = <&edma1 0 10>,
+					<&edma1 0 11>;
+				dma-names = "rx", "tx";
 				status = "disabled";
 			};
 
@@ -573,6 +588,9 @@
 				clocks = <&clks VF610_CLK_DSPI3>;
 				clock-names = "dspi";
 				spi-num-chipselects = <2>;
+				dmas = <&edma1 0 12>,
+					<&edma1 0 13>;
+				dma-names = "rx", "tx";
 				status = "disabled";
 			};
 
diff --git a/arch/arm/boot/dts/zynq-7000.dtsi b/arch/arm/boot/dts/zynq-7000.dtsi
index f283ff08381c..f3ac9bfe580e 100644
--- a/arch/arm/boot/dts/zynq-7000.dtsi
+++ b/arch/arm/boot/dts/zynq-7000.dtsi
@@ -10,9 +10,10 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  */
-/include/ "skeleton.dtsi"
 
 / {
+	#address-cells = <1>;
+	#size-cells = <1>;
 	compatible = "xlnx,zynq-7000";
 
 	cpus {
@@ -41,14 +42,15 @@
 		};
 	};
 
-	pmu {
+	pmu@f8891000 {
 		compatible = "arm,cortex-a9-pmu";
 		interrupts = <0 5 4>, <0 6 4>;
 		interrupt-parent = <&intc>;
-		reg = < 0xf8891000 0x1000 0xf8893000 0x1000 >;
+		reg = <0xf8891000 0x1000>,
+		      <0xf8893000 0x1000>;
 	};
 
-	regulator_vccpint: fixedregulator@0 {
+	regulator_vccpint: fixedregulator {
 		compatible = "regulator-fixed";
 		regulator-name = "VCCPINT";
 		regulator-min-microvolt = <1000000>;
diff --git a/arch/arm/boot/dts/zynq-microzed.dts b/arch/arm/boot/dts/zynq-microzed.dts
new file mode 100644
index 000000000000..b9376a4904b4
--- /dev/null
+++ b/arch/arm/boot/dts/zynq-microzed.dts
@@ -0,0 +1,96 @@
+/*
+ * Copyright (C) 2011 - 2014 Xilinx
+ * Copyright (C) 2016 Jagan Teki <jteki@openedev.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+/dts-v1/;
+/include/ "zynq-7000.dtsi"
+
+/ {
+	model = "Zynq MicroZED Development Board";
+	compatible = "xlnx,zynq-microzed", "xlnx,zynq-7000";
+
+	aliases {
+		ethernet0 = &gem0;
+		serial0 = &uart1;
+	};
+
+	memory {
+		device_type = "memory";
+		reg = <0x0 0x40000000>;
+	};
+
+	chosen {
+		bootargs = "earlycon";
+		stdout-path = "serial0:115200n8";
+	};
+
+	usb_phy0: phy0 {
+		compatible = "usb-nop-xceiv";
+		#phy-cells = <0>;
+	};
+};
+
+&clkc {
+	ps-clk-frequency = <33333333>;
+};
+
+&gem0 {
+	status = "okay";
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethernet_phy>;
+
+	ethernet_phy: ethernet-phy@0 {
+		reg = <0>;
+	};
+};
+
+&sdhci0 {
+	status = "okay";
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&usb0 {
+	status = "okay";
+	dr_mode = "host";
+	usb-phy = <&usb_phy0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usb0_default>;
+};
+
+&pinctrl0 {
+	pinctrl_usb0_default: usb0-default {
+		mux {
+			groups = "usb0_0_grp";
+			function = "usb0";
+		};
+
+		conf {
+			groups = "usb0_0_grp";
+			slew-rate = <0>;
+			io-standard = <1>;
+		};
+
+		conf-rx {
+			pins = "MIO29", "MIO31", "MIO36";
+			bias-high-impedance;
+		};
+
+		conf-tx {
+			pins = "MIO28", "MIO30", "MIO32", "MIO33", "MIO34",
+			       "MIO35", "MIO37", "MIO38", "MIO39";
+			bias-disable;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/zynq-parallella.dts b/arch/arm/boot/dts/zynq-parallella.dts
index 307ed201d658..64a6390fc501 100644
--- a/arch/arm/boot/dts/zynq-parallella.dts
+++ b/arch/arm/boot/dts/zynq-parallella.dts
@@ -28,7 +28,7 @@
 		serial0 = &uart1;
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x0 0x40000000>;
 	};
diff --git a/arch/arm/boot/dts/zynq-zc702.dts b/arch/arm/boot/dts/zynq-zc702.dts
index e96959b2e67a..0cdad2cc8b78 100644
--- a/arch/arm/boot/dts/zynq-zc702.dts
+++ b/arch/arm/boot/dts/zynq-zc702.dts
@@ -24,7 +24,7 @@
 		serial0 = &uart1;
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x0 0x40000000>;
 	};
diff --git a/arch/arm/boot/dts/zynq-zc706.dts b/arch/arm/boot/dts/zynq-zc706.dts
index be6a986bbbd8..ad4bb06dba25 100644
--- a/arch/arm/boot/dts/zynq-zc706.dts
+++ b/arch/arm/boot/dts/zynq-zc706.dts
@@ -24,7 +24,7 @@
 		serial0 = &uart1;
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x0 0x40000000>;
 	};
diff --git a/arch/arm/boot/dts/zynq-zed.dts b/arch/arm/boot/dts/zynq-zed.dts
index 7250c1eac7f9..325379f7983c 100644
--- a/arch/arm/boot/dts/zynq-zed.dts
+++ b/arch/arm/boot/dts/zynq-zed.dts
@@ -23,7 +23,7 @@
 		serial0 = &uart1;
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x0 0x20000000>;
 	};
diff --git a/arch/arm/boot/dts/zynq-zybo.dts b/arch/arm/boot/dts/zynq-zybo.dts
index d9e0f3e70671..590ec24b8749 100644
--- a/arch/arm/boot/dts/zynq-zybo.dts
+++ b/arch/arm/boot/dts/zynq-zybo.dts
@@ -23,7 +23,7 @@
 		serial0 = &uart1;
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x0 0x20000000>;
 	};
diff --git a/arch/arm/common/dmabounce.c b/arch/arm/common/dmabounce.c
index 301281645d08..75055df1cda3 100644
--- a/arch/arm/common/dmabounce.c
+++ b/arch/arm/common/dmabounce.c
@@ -243,7 +243,8 @@ static int needs_bounce(struct device *dev, dma_addr_t dma_addr, size_t size)
 }
 
 static inline dma_addr_t map_single(struct device *dev, void *ptr, size_t size,
-		enum dma_data_direction dir)
+				    enum dma_data_direction dir,
+				    unsigned long attrs)
 {
 	struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
 	struct safe_buffer *buf;
@@ -262,7 +263,8 @@ static inline dma_addr_t map_single(struct device *dev, void *ptr, size_t size,
 		__func__, buf->ptr, virt_to_dma(dev, buf->ptr),
 		buf->safe, buf->safe_dma_addr);
 
-	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) {
+	if ((dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) &&
+	    !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
 		dev_dbg(dev, "%s: copy unsafe %p to safe %p, size %d\n",
 			__func__, ptr, buf->safe, size);
 		memcpy(buf->safe, ptr, size);
@@ -272,7 +274,8 @@ static inline dma_addr_t map_single(struct device *dev, void *ptr, size_t size,
 }
 
 static inline void unmap_single(struct device *dev, struct safe_buffer *buf,
-		size_t size, enum dma_data_direction dir)
+				size_t size, enum dma_data_direction dir,
+				unsigned long attrs)
 {
 	BUG_ON(buf->size != size);
 	BUG_ON(buf->direction != dir);
@@ -283,7 +286,8 @@ static inline void unmap_single(struct device *dev, struct safe_buffer *buf,
 
 	DO_STATS(dev->archdata.dmabounce->bounce_count++);
 
-	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) {
+	if ((dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) &&
+	    !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
 		void *ptr = buf->ptr;
 
 		dev_dbg(dev, "%s: copy back safe %p to unsafe %p size %d\n",
@@ -334,7 +338,7 @@ static dma_addr_t dmabounce_map_page(struct device *dev, struct page *page,
 		return DMA_ERROR_CODE;
 	}
 
-	return map_single(dev, page_address(page) + offset, size, dir);
+	return map_single(dev, page_address(page) + offset, size, dir, attrs);
 }
 
 /*
@@ -357,7 +361,7 @@ static void dmabounce_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t
 		return;
 	}
 
-	unmap_single(dev, buf, size, dir);
+	unmap_single(dev, buf, size, dir, attrs);
 }
 
 static int __dmabounce_sync_for_cpu(struct device *dev, dma_addr_t addr,
diff --git a/arch/arm/configs/am200epdkit_defconfig b/arch/arm/configs/am200epdkit_defconfig
index f0dea52e49c4..113a5d815060 100644
--- a/arch/arm/configs/am200epdkit_defconfig
+++ b/arch/arm/configs/am200epdkit_defconfig
@@ -55,8 +55,9 @@ CONFIG_MTD_CFI_INTELEXT=y
 CONFIG_MTD_COMPLEX_MAPPINGS=y
 CONFIG_MTD_PXA2XX=y
 CONFIG_BLK_DEV_LOOP=m
-CONFIG_IDE=m
-CONFIG_BLK_DEV_IDECS=m
+CONFIG_BLK_DEV_SD=m
+CONFIG_ATA=m
+CONFIG_PATA_PCMCIA=m
 CONFIG_NETDEVICES=y
 CONFIG_NET_ETHERNET=y
 CONFIG_SMC91X=m
diff --git a/arch/arm/configs/assabet_defconfig b/arch/arm/configs/assabet_defconfig
index 558ecd8f66ff..ab19ff1a0b71 100644
--- a/arch/arm/configs/assabet_defconfig
+++ b/arch/arm/configs/assabet_defconfig
@@ -34,7 +34,6 @@ CONFIG_MTD_CFI_INTELEXT=y
 CONFIG_MTD_SA1100=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_RAM=y
-CONFIG_IDE=y
 CONFIG_NETDEVICES=y
 CONFIG_NET_ETHERNET=y
 CONFIG_NET_PCMCIA=y
diff --git a/arch/arm/configs/badge4_defconfig b/arch/arm/configs/badge4_defconfig
index d59009878312..2a604aa3195b 100644
--- a/arch/arm/configs/badge4_defconfig
+++ b/arch/arm/configs/badge4_defconfig
@@ -42,8 +42,6 @@ CONFIG_MTD_SA1100=y
 CONFIG_PARPORT=m
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_NBD=m
-CONFIG_IDE=m
-CONFIG_BLK_DEV_IDECD=m
 CONFIG_SCSI=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=m
diff --git a/arch/arm/configs/bcm2835_defconfig b/arch/arm/configs/bcm2835_defconfig
index 79de828e49ad..4b89f4e6e849 100644
--- a/arch/arm/configs/bcm2835_defconfig
+++ b/arch/arm/configs/bcm2835_defconfig
@@ -73,6 +73,8 @@ CONFIG_SPI_BCM2835=y
 CONFIG_SPI_BCM2835AUX=y
 CONFIG_GPIO_SYSFS=y
 # CONFIG_HWMON is not set
+CONFIG_THERMAL=y
+CONFIG_BCM2835_THERMAL=y
 CONFIG_WATCHDOG=y
 CONFIG_BCM2835_WDT=y
 CONFIG_DRM=y
diff --git a/arch/arm/configs/cerfcube_defconfig b/arch/arm/configs/cerfcube_defconfig
index dce912d146b4..57a2a18690b1 100644
--- a/arch/arm/configs/cerfcube_defconfig
+++ b/arch/arm/configs/cerfcube_defconfig
@@ -39,7 +39,6 @@ CONFIG_MTD_CFI_INTELEXT=y
 CONFIG_MTD_SA1100=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_RAM=m
-CONFIG_IDE=y
 CONFIG_NETDEVICES=y
 CONFIG_NET_ETHERNET=y
 CONFIG_NET_PCI=y
diff --git a/arch/arm/configs/collie_defconfig b/arch/arm/configs/collie_defconfig
index 52dbad5619e2..a8f3c596c39c 100644
--- a/arch/arm/configs/collie_defconfig
+++ b/arch/arm/configs/collie_defconfig
@@ -43,8 +43,9 @@ CONFIG_MTD_SA1100=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=1024
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECS=y
+CONFIG_BLK_DEV_SD=y
+CONFIG_ATA=y
+CONFIG_PATA_PCMCIA=y
 # CONFIG_INPUT_MOUSEDEV is not set
 CONFIG_INPUT_EVDEV=y
 # CONFIG_KEYBOARD_ATKBD is not set
diff --git a/arch/arm/configs/corgi_defconfig b/arch/arm/configs/corgi_defconfig
index c1470a00f55a..462533bd84c6 100644
--- a/arch/arm/configs/corgi_defconfig
+++ b/arch/arm/configs/corgi_defconfig
@@ -99,15 +99,14 @@ CONFIG_MTD_COMPLEX_MAPPINGS=y
 CONFIG_MTD_NAND=y
 CONFIG_MTD_NAND_SHARPSL=y
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECS=y
-CONFIG_SCSI=m
-CONFIG_BLK_DEV_SD=m
+CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=m
 CONFIG_CHR_DEV_OSST=m
 CONFIG_BLK_DEV_SR=m
 CONFIG_CHR_DEV_SG=m
 CONFIG_SCSI_MULTI_LUN=y
+CONFIG_ATA=y
+CONFIG_PATA_PCMCIA=y
 CONFIG_NETDEVICES=y
 CONFIG_NET_ETHERNET=y
 CONFIG_USB_CATC=m
diff --git a/arch/arm/configs/davinci_all_defconfig b/arch/arm/configs/davinci_all_defconfig
index 5e5dd6bc5ed9..8806754f7175 100644
--- a/arch/arm/configs/davinci_all_defconfig
+++ b/arch/arm/configs/davinci_all_defconfig
@@ -7,13 +7,13 @@ CONFIG_IKCONFIG=y
 CONFIG_IKCONFIG_PROC=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_CGROUPS=y
+CONFIG_CHECKPOINT_RESTORE=y
 CONFIG_BLK_DEV_INITRD=y
 CONFIG_EXPERT=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_MODULE_FORCE_UNLOAD=y
 CONFIG_MODVERSIONS=y
-# CONFIG_BLK_DEV_BSG is not set
 CONFIG_PARTITION_ADVANCED=y
 # CONFIG_IOSCHED_DEADLINE is not set
 # CONFIG_IOSCHED_CFQ is not set
@@ -34,6 +34,7 @@ CONFIG_DAVINCI_MUX_WARNINGS=y
 CONFIG_DAVINCI_RESET_CLOCKS=y
 CONFIG_PREEMPT=y
 CONFIG_AEABI=y
+CONFIG_SECCOMP=y
 CONFIG_ZBOOT_ROM_TEXT=0x0
 CONFIG_ZBOOT_ROM_BSS=0x0
 CONFIG_ARM_APPENDED_DTB=y
@@ -52,10 +53,10 @@ CONFIG_INET=y
 CONFIG_IP_PNP=y
 CONFIG_IP_PNP_DHCP=y
 CONFIG_NETFILTER=y
-CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 # CONFIG_FW_LOADER is not set
+CONFIG_DA8XX_MSTPRI=y
 CONFIG_MTD=m
 CONFIG_MTD_BLOCK=m
 CONFIG_MTD_CFI=m
@@ -116,6 +117,8 @@ CONFIG_SPI_DAVINCI=m
 CONFIG_PINCTRL_SINGLE=y
 CONFIG_GPIO_SYSFS=y
 CONFIG_GPIO_PCA953X=y
+CONFIG_POWER_RESET=y
+CONFIG_POWER_RESET_GPIO=y
 CONFIG_WATCHDOG=y
 CONFIG_DAVINCI_WATCHDOG=m
 CONFIG_MFD_DM355EVM_MSP=y
@@ -123,6 +126,8 @@ CONFIG_TPS6507X=y
 CONFIG_REGULATOR=y
 CONFIG_REGULATOR_FIXED_VOLTAGE=y
 CONFIG_REGULATOR_TPS6507X=y
+CONFIG_DRM=m
+CONFIG_DRM_TILCDC=m
 CONFIG_FB=y
 CONFIG_FIRMWARE_EDID=y
 CONFIG_FB_DA8XX=y
@@ -153,10 +158,13 @@ CONFIG_HID_SONY=m
 CONFIG_HID_SUNPLUS=m
 CONFIG_USB=m
 CONFIG_USB_MON=m
+CONFIG_USB_OHCI_HCD=m
 CONFIG_USB_STORAGE=m
 CONFIG_USB_MUSB_HDRC=m
+CONFIG_USB_MUSB_DA8XX=m
 CONFIG_MUSB_PIO_ONLY=y
 CONFIG_USB_TEST=m
+CONFIG_NOP_USB_XCEIV=m
 CONFIG_USB_GADGET=m
 CONFIG_USB_GADGET_DEBUG_FILES=y
 CONFIG_USB_GADGET_DEBUG_FS=y
@@ -167,28 +175,32 @@ CONFIG_USB_MASS_STORAGE=m
 CONFIG_USB_G_SERIAL=m
 CONFIG_USB_G_PRINTER=m
 CONFIG_USB_CDC_COMPOSITE=m
-CONFIG_MMC=m
+CONFIG_MMC=y
 # CONFIG_MMC_BLOCK_BOUNCE is not set
-CONFIG_MMC_DAVINCI=m
+CONFIG_MMC_DAVINCI=y
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=m
 CONFIG_LEDS_GPIO=m
 CONFIG_LEDS_TRIGGERS=y
 CONFIG_LEDS_TRIGGER_TIMER=m
 CONFIG_LEDS_TRIGGER_HEARTBEAT=m
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=m
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_OMAP=m
 CONFIG_DMADEVICES=y
 CONFIG_TI_EDMA=y
 CONFIG_MEMORY=y
 CONFIG_TI_AEMIF=m
+CONFIG_DA8XX_DDRCTL=y
 CONFIG_EXT2_FS=y
 CONFIG_EXT3_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
 CONFIG_XFS_FS=m
 CONFIG_AUTOFS4_FS=m
 CONFIG_MSDOS_FS=y
 CONFIG_VFAT_FS=y
 CONFIG_TMPFS=y
+CONFIG_TMPFS_XATTR=y
 CONFIG_JFFS2_FS=m
 CONFIG_UBIFS_FS=m
 CONFIG_CRAMFS=y
diff --git a/arch/arm/configs/dram_0xc0000000.config b/arch/arm/configs/dram_0xc0000000.config
new file mode 100644
index 000000000000..343d5333d973
--- /dev/null
+++ b/arch/arm/configs/dram_0xc0000000.config
@@ -0,0 +1 @@
+CONFIG_DRAM_BASE=0xc0000000
diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig
index c58f6841f8aa..79c415c33f69 100644
--- a/arch/arm/configs/exynos_defconfig
+++ b/arch/arm/configs/exynos_defconfig
@@ -155,6 +155,7 @@ CONFIG_VIDEO_EXYNOS4_FIMC_IS=m
 CONFIG_V4L_MEM2MEM_DRIVERS=y
 CONFIG_VIDEO_SAMSUNG_S5P_JPEG=m
 CONFIG_VIDEO_SAMSUNG_S5P_MFC=m
+CONFIG_VIDEO_SAMSUNG_EXYNOS_GSC=m
 CONFIG_V4L_TEST_DRIVERS=y
 CONFIG_DRM=y
 CONFIG_DRM_EXYNOS=y
diff --git a/arch/arm/configs/h3600_defconfig b/arch/arm/configs/h3600_defconfig
index 0142ec37e0be..ebeca11faa48 100644
--- a/arch/arm/configs/h3600_defconfig
+++ b/arch/arm/configs/h3600_defconfig
@@ -39,8 +39,9 @@ CONFIG_MTD_SA1100=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=8192
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECS=y
+CONFIG_BLK_DEV_SD=y
+CONFIG_ATA=y
+CONFIG_PATA_PCMCIA=y
 CONFIG_NETDEVICES=y
 CONFIG_PCMCIA_PCNET=y
 CONFIG_PPP=m
diff --git a/arch/arm/configs/imx_v6_v7_defconfig b/arch/arm/configs/imx_v6_v7_defconfig
index 8ec4dbbb50b0..cbe7faf55245 100644
--- a/arch/arm/configs/imx_v6_v7_defconfig
+++ b/arch/arm/configs/imx_v6_v7_defconfig
@@ -86,6 +86,7 @@ CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 # CONFIG_STANDALONE is not set
 CONFIG_DMA_CMA=y
+CONFIG_CMA_SIZE_MBYTES=64
 CONFIG_IMX_WEIM=y
 CONFIG_CONNECTOR=y
 CONFIG_MTD=y
@@ -256,6 +257,7 @@ CONFIG_SND_IMX_SOC=y
 CONFIG_SND_SOC_PHYCORE_AC97=y
 CONFIG_SND_SOC_EUKREA_TLV320=y
 CONFIG_SND_SOC_IMX_WM8962=y
+CONFIG_SND_SOC_IMX_ES8328=y
 CONFIG_SND_SOC_IMX_SGTL5000=y
 CONFIG_SND_SOC_IMX_SPDIF=y
 CONFIG_SND_SOC_IMX_MC13783=y
diff --git a/arch/arm/configs/integrator_defconfig b/arch/arm/configs/integrator_defconfig
index 869faae67201..69cb8f1efcea 100644
--- a/arch/arm/configs/integrator_defconfig
+++ b/arch/arm/configs/integrator_defconfig
@@ -26,6 +26,7 @@ CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_GOV_POWERSAVE=y
 CONFIG_CPU_FREQ_GOV_USERSPACE=y
 CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPUFREQ_DT=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
diff --git a/arch/arm/configs/ixp4xx_defconfig b/arch/arm/configs/ixp4xx_defconfig
index cf4918a2c51f..bb910d9df6c1 100644
--- a/arch/arm/configs/ixp4xx_defconfig
+++ b/arch/arm/configs/ixp4xx_defconfig
@@ -127,16 +127,17 @@ CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=8192
 CONFIG_EEPROM_LEGACY=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_CMD64X=y
-CONFIG_BLK_DEV_HPT366=y
-CONFIG_BLK_DEV_PDC202XX_NEW=y
 # CONFIG_SCSI_PROC_FS is not set
 CONFIG_BLK_DEV_SD=y
 # CONFIG_SCSI_LOWLEVEL is not set
 CONFIG_ATA=y
 CONFIG_SATA_VIA=y
 CONFIG_PATA_ARTOP=y
+CONFIG_PATA_CMD64X=y
+CONFIG_PATA_HPT366=y
+CONFIG_PATA_HPT37X=y
+CONFIG_PATA_HPT3X2N=y
+CONFIG_PATA_PDC2027X=y
 CONFIG_PATA_IXP4XX_CF=y
 CONFIG_NETDEVICES=y
 CONFIG_DUMMY=y
diff --git a/arch/arm/configs/jornada720_defconfig b/arch/arm/configs/jornada720_defconfig
index ea80e7e867c2..9056284139be 100644
--- a/arch/arm/configs/jornada720_defconfig
+++ b/arch/arm/configs/jornada720_defconfig
@@ -29,8 +29,9 @@ CONFIG_SA1100_FIR=m
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_NBD=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECS=y
+CONFIG_BLK_DEV_SD=y
+CONFIG_ATA=y
+CONFIG_PATA_PCMCIA=y
 CONFIG_NETDEVICES=y
 CONFIG_DUMMY=y
 CONFIG_NET_ETHERNET=y
diff --git a/arch/arm/configs/lart_defconfig b/arch/arm/configs/lart_defconfig
index faa2865658ac..8fc6fd09eb6d 100644
--- a/arch/arm/configs/lart_defconfig
+++ b/arch/arm/configs/lart_defconfig
@@ -36,8 +36,6 @@ CONFIG_MTD_CHAR=y
 CONFIG_MTD_BLOCK=y
 CONFIG_MTD_LART=y
 CONFIG_BLK_DEV_RAM=y
-CONFIG_IDE=m
-CONFIG_BLK_DEV_IDECD=m
 CONFIG_NETDEVICES=y
 CONFIG_DUMMY=m
 CONFIG_NET_ETHERNET=y
diff --git a/arch/arm/configs/mainstone_defconfig b/arch/arm/configs/mainstone_defconfig
index 04efa1b3ef25..e8d26b805be6 100644
--- a/arch/arm/configs/mainstone_defconfig
+++ b/arch/arm/configs/mainstone_defconfig
@@ -27,7 +27,6 @@ CONFIG_MTD_CFI_ADV_OPTIONS=y
 CONFIG_MTD_CFI_GEOMETRY=y
 # CONFIG_MTD_CFI_I1 is not set
 CONFIG_MTD_CFI_INTELEXT=y
-CONFIG_IDE=y
 CONFIG_NETDEVICES=y
 CONFIG_NET_ETHERNET=y
 CONFIG_SMC91X=y
diff --git a/arch/arm/configs/multi_v5_defconfig b/arch/arm/configs/multi_v5_defconfig
index 2658b80fa263..361686a362f1 100644
--- a/arch/arm/configs/multi_v5_defconfig
+++ b/arch/arm/configs/multi_v5_defconfig
@@ -150,7 +150,6 @@ CONFIG_SPI=y
 CONFIG_SPI_ATMEL=y
 CONFIG_SPI_IMX=y
 CONFIG_SPI_ORION=y
-CONFIG_GPIO_SYSFS=y
 CONFIG_POWER_RESET=y
 CONFIG_POWER_RESET_GPIO=y
 CONFIG_POWER_RESET_QNAP=y
diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig
index 11f37ed1dbff..b01a43851294 100644
--- a/arch/arm/configs/multi_v7_defconfig
+++ b/arch/arm/configs/multi_v7_defconfig
@@ -173,6 +173,11 @@ CONFIG_CAN_RCAR=m
 CONFIG_CAN_XILINXCAN=y
 CONFIG_CAN_MCP251X=y
 CONFIG_NET_DSA_BCM_SF2=m
+CONFIG_B53=m
+CONFIG_B53_SPI_DRIVER=m
+CONFIG_B53_MDIO_DRIVER=m
+CONFIG_B53_MMAP_DRIVER=m
+CONFIG_B53_SRAB_DRIVER=m
 CONFIG_CAN_SUN4I=y
 CONFIG_BT=m
 CONFIG_BT_MRVL=m
@@ -235,6 +240,7 @@ CONFIG_HIX5HD2_GMAC=y
 CONFIG_SUN4I_EMAC=y
 CONFIG_MACB=y
 CONFIG_BCMGENET=m
+CONFIG_BGMAC_BCMA=y
 CONFIG_SYSTEMPORT=m
 CONFIG_NET_CALXEDA_XGMAC=y
 CONFIG_GIANFAR=y
@@ -404,7 +410,6 @@ CONFIG_PINCTRL_MSM8X74=y
 CONFIG_PINCTRL_MSM8916=y
 CONFIG_PINCTRL_QCOM_SPMI_PMIC=y
 CONFIG_PINCTRL_QCOM_SSBI_PMIC=y
-CONFIG_GPIO_SYSFS=y
 CONFIG_GPIO_GENERIC_PLATFORM=y
 CONFIG_GPIO_DAVINCI=y
 CONFIG_GPIO_DWAPB=y
@@ -450,7 +455,6 @@ CONFIG_RCAR_THERMAL=y
 CONFIG_ARMADA_THERMAL=y
 CONFIG_DAVINCI_WATCHDOG=m
 CONFIG_EXYNOS_THERMAL=m
-CONFIG_ST_THERMAL_SYSCFG=y
 CONFIG_ST_THERMAL_MEMMAP=y
 CONFIG_WATCHDOG=y
 CONFIG_DA9063_WATCHDOG=m
@@ -467,6 +471,7 @@ CONFIG_MESON_WATCHDOG=y
 CONFIG_DW_WATCHDOG=y
 CONFIG_DIGICOLOR_WATCHDOG=y
 CONFIG_BCM2835_WDT=y
+CONFIG_BCM47XX_WATCHDOG=y
 CONFIG_BCM7038_WDT=m
 CONFIG_BCM_KONA_WDT=y
 CONFIG_MFD_ACT8945A=y
@@ -489,7 +494,7 @@ CONFIG_MFD_MAX8907=y
 CONFIG_MFD_MAX8997=y
 CONFIG_MFD_MAX8998=y
 CONFIG_MFD_RK808=y
-CONFIG_MFD_PM8921_CORE=y
+CONFIG_MFD_PM8XXX=y
 CONFIG_MFD_QCOM_RPM=y
 CONFIG_MFD_SPMI_PMIC=y
 CONFIG_MFD_SEC_CORE=y
@@ -561,7 +566,9 @@ CONFIG_VIDEO_EXYNOS4_FIMC_IS=m
 CONFIG_V4L_MEM2MEM_DRIVERS=y
 CONFIG_VIDEO_SAMSUNG_S5P_JPEG=m
 CONFIG_VIDEO_SAMSUNG_S5P_MFC=m
+CONFIG_VIDEO_SAMSUNG_EXYNOS_GSC=m
 CONFIG_VIDEO_STI_BDISP=m
+CONFIG_VIDEO_STI_HVA=m
 CONFIG_VIDEO_RENESAS_JPU=m
 CONFIG_VIDEO_RENESAS_VSP1=m
 CONFIG_V4L_TEST_DRIVERS=y
@@ -572,6 +579,7 @@ CONFIG_DRM=y
 CONFIG_DRM_I2C_ADV7511=m
 # CONFIG_DRM_I2C_CH7006 is not set
 # CONFIG_DRM_I2C_SIL164 is not set
+CONFIG_DRM_DUMB_VGA_DAC=m
 CONFIG_DRM_NXP_PTN3460=m
 CONFIG_DRM_PARADE_PS8622=m
 CONFIG_DRM_NOUVEAU=m
@@ -649,6 +657,9 @@ CONFIG_SND_SOC_AK4642=m
 CONFIG_SND_SOC_SGTL5000=m
 CONFIG_SND_SOC_SPDIF=m
 CONFIG_SND_SOC_WM8978=m
+CONFIG_SND_SOC_STI=m
+CONFIG_SND_SOC_STI_SAS=m
+CONFIG_SND_SIMPLE_CARD=m
 CONFIG_USB=y
 CONFIG_USB_XHCI_HCD=y
 CONFIG_USB_XHCI_MVEBU=y
@@ -790,6 +801,7 @@ CONFIG_DMA_OMAP=y
 CONFIG_QCOM_BAM_DMA=y
 CONFIG_XILINX_DMA=y
 CONFIG_DMA_SUN6I=y
+CONFIG_ST_FDMA=m
 CONFIG_STAGING=y
 CONFIG_SENSORS_ISL29018=y
 CONFIG_SENSORS_ISL29028=y
@@ -798,6 +810,10 @@ CONFIG_KEYBOARD_NVEC=y
 CONFIG_SERIO_NVEC_PS2=y
 CONFIG_NVEC_POWER=y
 CONFIG_NVEC_PAZ00=y
+CONFIG_BCMA=y
+CONFIG_BCMA_HOST_SOC=y
+CONFIG_BCMA_DRIVER_GMAC_CMN=y
+CONFIG_BCMA_DRIVER_GPIO=y
 CONFIG_QCOM_GSBI=y
 CONFIG_QCOM_PM=y
 CONFIG_QCOM_SMEM=y
@@ -823,6 +839,8 @@ CONFIG_HWSPINLOCK_QCOM=y
 CONFIG_ROCKCHIP_IOMMU=y
 CONFIG_TEGRA_IOMMU_GART=y
 CONFIG_TEGRA_IOMMU_SMMU=y
+CONFIG_REMOTEPROC=m
+CONFIG_ST_REMOTEPROC=m
 CONFIG_PM_DEVFREQ=y
 CONFIG_ARM_TEGRA_DEVFREQ=m
 CONFIG_MEMORY=y
@@ -862,9 +880,7 @@ CONFIG_PHY_ROCKCHIP_DP=m
 CONFIG_PHY_ROCKCHIP_USB=m
 CONFIG_PHY_QCOM_APQ8064_SATA=m
 CONFIG_PHY_MIPHY28LP=y
-CONFIG_PHY_MIPHY365X=y
 CONFIG_PHY_RCAR_GEN2=m
-CONFIG_PHY_STIH41X_USB=y
 CONFIG_PHY_STIH407_USB=y
 CONFIG_PHY_SUN4I_USB=y
 CONFIG_PHY_SUN9I_USB=y
@@ -877,6 +893,8 @@ CONFIG_BCM2835_MBOX=y
 CONFIG_RASPBERRYPI_FIRMWARE=y
 CONFIG_EFI_VARS=m
 CONFIG_EFI_CAPSULE_LOADER=m
+CONFIG_CONFIG_BCM47XX_NVRAM=y
+CONFIG_BCM47XX_SPROM=y
 CONFIG_EXT4_FS=y
 CONFIG_AUTOFS4_FS=y
 CONFIG_MSDOS_FS=y
diff --git a/arch/arm/configs/netwinder_defconfig b/arch/arm/configs/netwinder_defconfig
index 4f3dfb21772b..f1395bbd436c 100644
--- a/arch/arm/configs/netwinder_defconfig
+++ b/arch/arm/configs/netwinder_defconfig
@@ -8,7 +8,7 @@ CONFIG_LEDS_CPU=y
 CONFIG_DEPRECATED_PARAM_STRUCT=y
 CONFIG_ZBOOT_ROM_TEXT=0x0
 CONFIG_ZBOOT_ROM_BSS=0x0
-CONFIG_CMDLINE="root=0x301"
+CONFIG_CMDLINE="root=0x801"
 CONFIG_FPE_NWFPE=y
 CONFIG_BINFMT_AOUT=y
 CONFIG_NET=y
@@ -27,8 +27,9 @@ CONFIG_PARPORT=y
 CONFIG_PARPORT_PC=y
 CONFIG_PARPORT_PC_SUPERIO=y
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_SL82C105=y
+CONFIG_BLK_DEV_SD=y
+CONFIG_ATA=y
+CONFIG_PATA_WINBOND=y
 CONFIG_NETDEVICES=y
 CONFIG_NET_ETHERNET=y
 CONFIG_MII=y
diff --git a/arch/arm/configs/omap1_defconfig b/arch/arm/configs/omap1_defconfig
index 0c8a78734536..6ffc9844542d 100644
--- a/arch/arm/configs/omap1_defconfig
+++ b/arch/arm/configs/omap1_defconfig
@@ -96,14 +96,14 @@ CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_COUNT=2
 CONFIG_BLK_DEV_RAM_SIZE=8192
-CONFIG_IDE=m
-CONFIG_BLK_DEV_IDECS=m
 CONFIG_SCSI=y
 # CONFIG_SCSI_PROC_FS is not set
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=y
 CONFIG_BLK_DEV_SR=y
 CONFIG_CHR_DEV_SG=y
+CONFIG_ATA=m
+CONFIG_PATA_PCMCIA=m
 CONFIG_NETDEVICES=y
 CONFIG_TUN=y
 CONFIG_PHYLIB=y
diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig
index 53e1a884a1ea..195c98b85568 100644
--- a/arch/arm/configs/omap2plus_defconfig
+++ b/arch/arm/configs/omap2plus_defconfig
@@ -1,7 +1,6 @@
 CONFIG_KERNEL_LZMA=y
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_FHANDLE=y
 CONFIG_AUDIT=y
 CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
@@ -40,7 +39,6 @@ CONFIG_ARCH_MULTI_V6=y
 CONFIG_POWER_AVS_OMAP=y
 CONFIG_POWER_AVS_OMAP_CLASS3=y
 CONFIG_OMAP_RESET_CLOCKS=y
-CONFIG_OMAP_MUX_DEBUG=y
 CONFIG_ARCH_OMAP2=y
 CONFIG_ARCH_OMAP3=y
 CONFIG_ARCH_OMAP4=y
@@ -50,7 +48,6 @@ CONFIG_SOC_AM43XX=y
 CONFIG_SOC_DRA7XX=y
 CONFIG_ARM_THUMBEE=y
 CONFIG_ARM_ERRATA_411920=y
-CONFIG_ARM_ERRATA_430973=y
 CONFIG_SMP=y
 CONFIG_NR_CPUS=2
 CONFIG_CMA=y
@@ -62,7 +59,6 @@ CONFIG_ARM_ATAG_DTB_COMPAT=y
 CONFIG_CMDLINE="root=/dev/mmcblk0p2 rootwait console=ttyO2,115200"
 CONFIG_KEXEC=y
 CONFIG_CPU_FREQ=y
-CONFIG_CPU_FREQ_STAT_DETAILS=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
 CONFIG_CPU_FREQ_GOV_POWERSAVE=y
 CONFIG_CPU_FREQ_GOV_USERSPACE=y
@@ -205,6 +201,7 @@ CONFIG_TOUCHSCREEN_ADS7846=m
 CONFIG_TOUCHSCREEN_EDT_FT5X06=m
 CONFIG_TOUCHSCREEN_TI_AM335X_TSC=m
 CONFIG_TOUCHSCREEN_PIXCIR=m
+CONFIG_TOUCHSCREEN_TSC2004=m
 CONFIG_TOUCHSCREEN_TSC2005=m
 CONFIG_TOUCHSCREEN_TSC2007=m
 CONFIG_INPUT_MISC=y
@@ -240,14 +237,14 @@ CONFIG_GPIO_PALMAS=y
 CONFIG_GPIO_TWL4030=y
 CONFIG_W1=m
 CONFIG_HDQ_MASTER_OMAP=m
+CONFIG_POWER_AVS=y
+CONFIG_POWER_RESET=y
 CONFIG_BATTERY_BQ27XXX=m
 CONFIG_CHARGER_ISP1704=m
 CONFIG_CHARGER_TWL4030=m
 CONFIG_CHARGER_BQ2415X=m
 CONFIG_CHARGER_BQ24190=m
 CONFIG_CHARGER_BQ24735=m
-CONFIG_POWER_RESET=y
-CONFIG_POWER_AVS=y
 CONFIG_HWMON=m
 CONFIG_SENSORS_GPIO_FAN=m
 CONFIG_SENSORS_LM75=m
@@ -267,10 +264,13 @@ CONFIG_TWL4030_WATCHDOG=m
 CONFIG_MFD_TI_AM335X_TSCADC=m
 CONFIG_MFD_PALMAS=y
 CONFIG_MFD_TPS65217=y
+CONFIG_MFD_TI_LP873X=y
 CONFIG_MFD_TPS65218=y
 CONFIG_MFD_TPS65910=y
 CONFIG_TWL6040_CORE=y
+CONFIG_REGULATOR_GPIO=y
 CONFIG_REGULATOR_LP872X=y
+CONFIG_REGULATOR_LP873X=y
 CONFIG_REGULATOR_PALMAS=y
 CONFIG_REGULATOR_PBIAS=y
 CONFIG_REGULATOR_TI_ABB=y
diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig
index a016ecc0084b..e4314b1227a3 100644
--- a/arch/arm/configs/pxa_defconfig
+++ b/arch/arm/configs/pxa_defconfig
@@ -411,7 +411,6 @@ CONFIG_MFD_MAX77693=y
 CONFIG_MFD_MAX8907=m
 CONFIG_EZX_PCAP=y
 CONFIG_UCB1400_CORE=m
-CONFIG_MFD_PM8921_CORE=m
 CONFIG_MFD_SEC_CORE=y
 CONFIG_MFD_PALMAS=y
 CONFIG_MFD_TPS65090=y
diff --git a/arch/arm/configs/qcom_defconfig b/arch/arm/configs/qcom_defconfig
index c2dff4fd5fc4..8c3a0108a231 100644
--- a/arch/arm/configs/qcom_defconfig
+++ b/arch/arm/configs/qcom_defconfig
@@ -119,7 +119,6 @@ CONFIG_POWER_RESET=y
 CONFIG_POWER_RESET_MSM=y
 CONFIG_THERMAL=y
 CONFIG_MFD_PM8XXX=y
-CONFIG_MFD_PM8921_CORE=y
 CONFIG_MFD_QCOM_RPM=y
 CONFIG_MFD_SPMI_PMIC=y
 CONFIG_REGULATOR=y
@@ -162,8 +161,8 @@ CONFIG_APQ_MMCC_8084=y
 CONFIG_IPQ_LCC_806X=y
 CONFIG_MSM_GCC_8660=y
 CONFIG_MSM_LCC_8960=y
-CONFIG_MSM_GCC_9615=y
-CONFIG_MSM_LCC_9615=y
+CONFIG_MDM_GCC_9615=y
+CONFIG_MDM_LCC_9615=y
 CONFIG_MSM_MMCC_8960=y
 CONFIG_MSM_MMCC_8974=y
 CONFIG_HWSPINLOCK_QCOM=y
diff --git a/arch/arm/configs/s3c2410_defconfig b/arch/arm/configs/s3c2410_defconfig
index bc4bfe02e611..4364040ed696 100644
--- a/arch/arm/configs/s3c2410_defconfig
+++ b/arch/arm/configs/s3c2410_defconfig
@@ -219,20 +219,16 @@ CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_UB=m
 CONFIG_BLK_DEV_RAM=y
 CONFIG_ATA_OVER_ETH=m
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_BLK_DEV_IDETAPE=m
-CONFIG_BLK_DEV_PLATFORM=y
-CONFIG_SCSI=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=m
-CONFIG_BLK_DEV_SR=m
+CONFIG_BLK_DEV_SR=y
 CONFIG_BLK_DEV_SR_VENDOR=y
 CONFIG_CHR_DEV_SG=y
 CONFIG_CHR_DEV_SCH=m
-CONFIG_SCSI_MULTI_LUN=y
 CONFIG_SCSI_CONSTANTS=y
 CONFIG_SCSI_SCAN_ASYNC=y
+CONFIG_ATA=y
+CONFIG_PATA_PLATFORM=y
 CONFIG_NETDEVICES=y
 CONFIG_DM9000=y
 CONFIG_INPUT_EVDEV=y
diff --git a/arch/arm/configs/shannon_defconfig b/arch/arm/configs/shannon_defconfig
index b0b96942f4bd..e52395629810 100644
--- a/arch/arm/configs/shannon_defconfig
+++ b/arch/arm/configs/shannon_defconfig
@@ -25,7 +25,6 @@ CONFIG_MTD_CFI_AMDSTD=y
 CONFIG_MTD_SA1100=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=8192
-CONFIG_IDE=m
 CONFIG_NETDEVICES=y
 CONFIG_NET_ETHERNET=y
 CONFIG_NET_PCMCIA=y
diff --git a/arch/arm/configs/shmobile_defconfig b/arch/arm/configs/shmobile_defconfig
index baa07a46a88b..1b0f8ae36fb3 100644
--- a/arch/arm/configs/shmobile_defconfig
+++ b/arch/arm/configs/shmobile_defconfig
@@ -2,6 +2,7 @@ CONFIG_SYSVIPC=y
 CONFIG_NO_HZ=y
 CONFIG_IKCONFIG=y
 CONFIG_IKCONFIG_PROC=y
+CONFIG_CGROUPS=y
 CONFIG_BLK_DEV_INITRD=y
 CONFIG_CC_OPTIMIZE_FOR_SIZE=y
 CONFIG_SYSCTL_SYSCALL=y
diff --git a/arch/arm/configs/socfpga_defconfig b/arch/arm/configs/socfpga_defconfig
index 9f84be5b3ac5..2e1d254e06a2 100644
--- a/arch/arm/configs/socfpga_defconfig
+++ b/arch/arm/configs/socfpga_defconfig
@@ -25,6 +25,7 @@ CONFIG_PCIE_ALTERA_MSI=y
 CONFIG_SMP=y
 CONFIG_NR_CPUS=2
 CONFIG_AEABI=y
+CONFIG_HIGHMEM=y
 CONFIG_ZBOOT_ROM_TEXT=0x0
 CONFIG_ZBOOT_ROM_BSS=0x0
 CONFIG_VFP=y
@@ -50,6 +51,10 @@ CONFIG_CAN_DEBUG_DEVICES=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_MTD=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_SPI_CADENCE_QUADSPI=y
+CONFIG_OF_OVERLAY=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_COUNT=2
 CONFIG_BLK_DEV_RAM_SIZE=8192
@@ -101,18 +106,28 @@ CONFIG_DMADEVICES=y
 CONFIG_PL330_DMA=y
 CONFIG_DMATEST=m
 CONFIG_FPGA=y
+CONFIG_FPGA_REGION=y
 CONFIG_FPGA_MGR_SOCFPGA=y
+CONFIG_FPGA_MGR_SOCFPGA_A10=y
+CONFIG_FPGA_BRIDGE=y
+CONFIG_SOCFPGA_FPGA_BRIDGE=y
+CONFIG_ALTERA_FREEZE_BRIDGE=y
 CONFIG_EXT2_FS=y
 CONFIG_EXT2_FS_XATTR=y
 CONFIG_EXT2_FS_POSIX_ACL=y
 CONFIG_EXT3_FS=y
+CONFIG_AUTOFS4_FS=y
 CONFIG_VFAT_FS=y
 CONFIG_NTFS_FS=y
 CONFIG_NTFS_RW=y
 CONFIG_TMPFS=y
 CONFIG_CONFIGFS_FS=y
 CONFIG_NFS_FS=y
+CONFIG_NFS_V3_ACL=y
 CONFIG_ROOT_NFS=y
+CONFIG_NFSD=y
+CONFIG_NFSD_V3_ACL=y
+CONFIG_NFSD_V4=y
 CONFIG_NLS_CODEPAGE_437=y
 CONFIG_NLS_ISO8859_1=y
 CONFIG_PRINTK_TIME=y
diff --git a/arch/arm/configs/spitz_defconfig b/arch/arm/configs/spitz_defconfig
index a1ede1966baf..d8c529332fb4 100644
--- a/arch/arm/configs/spitz_defconfig
+++ b/arch/arm/configs/spitz_defconfig
@@ -96,15 +96,13 @@ CONFIG_MTD_COMPLEX_MAPPINGS=y
 CONFIG_MTD_NAND=y
 CONFIG_MTD_NAND_SHARPSL=y
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECS=y
-CONFIG_SCSI=m
-CONFIG_BLK_DEV_SD=m
+CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=m
 CONFIG_CHR_DEV_OSST=m
 CONFIG_BLK_DEV_SR=m
 CONFIG_CHR_DEV_SG=m
-CONFIG_SCSI_MULTI_LUN=y
+CONFIG_ATA=y
+CONFIG_PATA_PCMCIA=y
 CONFIG_NETDEVICES=y
 CONFIG_NET_ETHERNET=y
 CONFIG_USB_CATC=m
diff --git a/arch/arm/configs/stm32_defconfig b/arch/arm/configs/stm32_defconfig
index 1e5ec2a0e4cf..5a72d694662f 100644
--- a/arch/arm/configs/stm32_defconfig
+++ b/arch/arm/configs/stm32_defconfig
@@ -38,8 +38,7 @@ CONFIG_DEVTMPFS_MOUNT=y
 # CONFIG_FW_LOADER is not set
 # CONFIG_BLK_DEV is not set
 CONFIG_EEPROM_93CX6=y
-# CONFIG_INPUT is not set
-# CONFIG_SERIO is not set
+CONFIG_KEYBOARD_GPIO=y
 # CONFIG_VT is not set
 # CONFIG_UNIX98_PTYS is not set
 # CONFIG_LEGACY_PTYS is not set
diff --git a/arch/arm/configs/sunxi_defconfig b/arch/arm/configs/sunxi_defconfig
index 714da336ec86..dfeee5c51b40 100644
--- a/arch/arm/configs/sunxi_defconfig
+++ b/arch/arm/configs/sunxi_defconfig
@@ -98,6 +98,7 @@ CONFIG_MEDIA_RC_SUPPORT=y
 CONFIG_RC_DEVICES=y
 CONFIG_IR_SUNXI=y
 CONFIG_DRM=y
+CONFIG_DRM_DUMB_VGA_DAC=y
 CONFIG_DRM_SUN4I=y
 CONFIG_FB=y
 CONFIG_FB_SIMPLE=y
diff --git a/arch/arm/configs/tegra_defconfig b/arch/arm/configs/tegra_defconfig
index 6012a1ec779f..844eeef5a509 100644
--- a/arch/arm/configs/tegra_defconfig
+++ b/arch/arm/configs/tegra_defconfig
@@ -1,16 +1,15 @@
 CONFIG_SYSVIPC=y
-CONFIG_FHANDLE=y
 CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_IKCONFIG=y
 CONFIG_IKCONFIG_PROC=y
 CONFIG_CGROUPS=y
-CONFIG_CGROUP_DEBUG=y
-CONFIG_CGROUP_FREEZER=y
-CONFIG_CGROUP_CPUACCT=y
 CONFIG_CGROUP_SCHED=y
 CONFIG_RT_GROUP_SCHED=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_DEBUG=y
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_ELF_CORE is not set
 CONFIG_EMBEDDED=y
@@ -24,14 +23,10 @@ CONFIG_PARTITION_ADVANCED=y
 # CONFIG_IOSCHED_DEADLINE is not set
 # CONFIG_IOSCHED_CFQ is not set
 CONFIG_ARCH_TEGRA=y
-CONFIG_ARCH_TEGRA_2x_SOC=y
-CONFIG_ARCH_TEGRA_3x_SOC=y
-CONFIG_ARCH_TEGRA_114_SOC=y
-CONFIG_ARCH_TEGRA_124_SOC=y
 CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
 CONFIG_PCI_MSI=y
 CONFIG_PCI_TEGRA=y
-CONFIG_PCIEPORTBUS=y
 CONFIG_SMP=y
 CONFIG_PREEMPT=y
 CONFIG_AEABI=y
@@ -41,7 +36,6 @@ CONFIG_ZBOOT_ROM_TEXT=0x0
 CONFIG_ZBOOT_ROM_BSS=0x0
 CONFIG_KEXEC=y
 CONFIG_CPU_FREQ=y
-CONFIG_CPU_FREQ_STAT_DETAILS=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
 CONFIG_CPUFREQ_DT=y
 CONFIG_CPU_IDLE=y
@@ -59,7 +53,6 @@ CONFIG_IP_PNP_RARP=y
 CONFIG_INET_ESP=y
 # CONFIG_INET_XFRM_MODE_TUNNEL is not set
 # CONFIG_INET_XFRM_MODE_BEET is not set
-# CONFIG_INET_LRO is not set
 # CONFIG_INET_DIAG is not set
 CONFIG_IPV6_ROUTER_PREF=y
 CONFIG_IPV6_OPTIMISTIC_DAD=y
@@ -86,6 +79,7 @@ CONFIG_DEVTMPFS_MOUNT=y
 # CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_DMA_CMA=y
 CONFIG_CMA_SIZE_MBYTES=64
+CONFIG_TEGRA_GMI=y
 CONFIG_MTD=y
 CONFIG_MTD_M25P80=y
 CONFIG_MTD_SPI_NOR=y
@@ -131,8 +125,8 @@ CONFIG_INPUT_MPU3050=y
 # CONFIG_DEVKMEM is not set
 CONFIG_SERIAL_8250=y
 CONFIG_SERIAL_8250_CONSOLE=y
-CONFIG_SERIAL_TEGRA=y
 CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_TEGRA=y
 # CONFIG_HW_RANDOM is not set
 # CONFIG_I2C_COMPAT is not set
 CONFIG_I2C_CHARDEV=y
@@ -151,11 +145,11 @@ CONFIG_GPIO_PCA953X_IRQ=y
 CONFIG_GPIO_PALMAS=y
 CONFIG_GPIO_TPS6586X=y
 CONFIG_GPIO_TPS65910=y
-CONFIG_BATTERY_SBS=y
-CONFIG_CHARGER_TPS65090=y
 CONFIG_POWER_RESET=y
 CONFIG_POWER_RESET_AS3722=y
 CONFIG_POWER_RESET_GPIO=y
+CONFIG_BATTERY_SBS=y
+CONFIG_CHARGER_TPS65090=y
 CONFIG_SENSORS_LM90=y
 CONFIG_SENSORS_LM95245=y
 CONFIG_WATCHDOG=y
@@ -216,6 +210,7 @@ CONFIG_SND_SOC_TEGRA_WM9712=y
 CONFIG_SND_SOC_TEGRA_TRIMSLICE=y
 CONFIG_SND_SOC_TEGRA_ALC5632=y
 CONFIG_SND_SOC_TEGRA_MAX98090=y
+CONFIG_SND_SOC_TEGRA_SGTL5000=y
 CONFIG_USB=y
 CONFIG_USB_XHCI_HCD=y
 CONFIG_USB_XHCI_TEGRA=y
@@ -262,6 +257,10 @@ CONFIG_NVEC_POWER=y
 CONFIG_NVEC_PAZ00=y
 CONFIG_TEGRA_IOMMU_GART=y
 CONFIG_TEGRA_IOMMU_SMMU=y
+CONFIG_ARCH_TEGRA_2x_SOC=y
+CONFIG_ARCH_TEGRA_3x_SOC=y
+CONFIG_ARCH_TEGRA_114_SOC=y
+CONFIG_ARCH_TEGRA_124_SOC=y
 CONFIG_MEMORY=y
 CONFIG_IIO=y
 CONFIG_AK8975=y
diff --git a/arch/arm/configs/u8500_defconfig b/arch/arm/configs/u8500_defconfig
index b7b09189f1c5..e2151a7aaf49 100644
--- a/arch/arm/configs/u8500_defconfig
+++ b/arch/arm/configs/u8500_defconfig
@@ -4,7 +4,6 @@ CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_BLK_DEV_INITRD=y
 CONFIG_KALLSYMS_ALL=y
-CONFIG_PERF_EVENTS=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 # CONFIG_BLK_DEV_BSG is not set
@@ -26,7 +25,6 @@ CONFIG_CPU_IDLE=y
 CONFIG_ARM_U8500_CPUIDLE=y
 CONFIG_VFP=y
 CONFIG_NEON=y
-CONFIG_PM=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -108,18 +106,19 @@ CONFIG_DMADEVICES=y
 CONFIG_STE_DMA40=y
 CONFIG_HSEM_U8500=y
 CONFIG_IIO=y
-CONFIG_IIO_BUFFER=y
+CONFIG_IIO_SW_TRIGGER=y
 CONFIG_IIO_ST_ACCEL_3AXIS=y
 CONFIG_IIO_ST_GYRO_3AXIS=y
 CONFIG_BH1780=y
+CONFIG_AK8974=y
 CONFIG_IIO_ST_MAGN_3AXIS=y
+CONFIG_IIO_HRTIMER_TRIGGER=y
 CONFIG_IIO_ST_PRESS=y
 CONFIG_EXT2_FS=y
 CONFIG_EXT2_FS_XATTR=y
 CONFIG_EXT2_FS_POSIX_ACL=y
 CONFIG_EXT2_FS_SECURITY=y
 CONFIG_EXT3_FS=y
-CONFIG_EXT4_FS=y
 CONFIG_VFAT_FS=y
 CONFIG_TMPFS=y
 CONFIG_TMPFS_POSIX_ACL=y
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
index 27ed1b1cd1d7..13f1b4c289d4 100644
--- a/arch/arm/crypto/Kconfig
+++ b/arch/arm/crypto/Kconfig
@@ -88,9 +88,9 @@ config CRYPTO_AES_ARM
 config CRYPTO_AES_ARM_BS
 	tristate "Bit sliced AES using NEON instructions"
 	depends on KERNEL_MODE_NEON
-	select CRYPTO_ALGAPI
 	select CRYPTO_AES_ARM
-	select CRYPTO_ABLK_HELPER
+	select CRYPTO_BLKCIPHER
+	select CRYPTO_SIMD
 	help
 	  Use a faster and more secure NEON based implementation of AES in CBC,
 	  CTR and XTS modes
@@ -104,8 +104,8 @@ config CRYPTO_AES_ARM_BS
 config CRYPTO_AES_ARM_CE
 	tristate "Accelerated AES using ARMv8 Crypto Extensions"
 	depends on KERNEL_MODE_NEON
-	select CRYPTO_ALGAPI
-	select CRYPTO_ABLK_HELPER
+	select CRYPTO_BLKCIPHER
+	select CRYPTO_SIMD
 	help
 	  Use an implementation of AES in CBC, CTR and XTS modes that uses
 	  ARMv8 Crypto Extensions
@@ -120,4 +120,14 @@ config CRYPTO_GHASH_ARM_CE
 	  that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64)
 	  that is part of the ARMv8 Crypto Extensions
 
+config CRYPTO_CRCT10DIF_ARM_CE
+	tristate "CRCT10DIF digest algorithm using PMULL instructions"
+	depends on KERNEL_MODE_NEON && CRC_T10DIF
+	select CRYPTO_HASH
+
+config CRYPTO_CRC32_ARM_CE
+	tristate "CRC32(C) digest algorithm using CRC and/or PMULL instructions"
+	depends on KERNEL_MODE_NEON && CRC32
+	select CRYPTO_HASH
+
 endif
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index fc5150702b64..b578a1820ab1 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -13,6 +13,8 @@ ce-obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o
 ce-obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o
 ce-obj-$(CONFIG_CRYPTO_SHA2_ARM_CE) += sha2-arm-ce.o
 ce-obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o
+ce-obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM_CE) += crct10dif-arm-ce.o
+ce-obj-$(CONFIG_CRYPTO_CRC32_ARM_CE) += crc32-arm-ce.o
 
 ifneq ($(ce-obj-y)$(ce-obj-m),)
 ifeq ($(call as-instr,.fpu crypto-neon-fp-armv8,y,n),y)
@@ -36,6 +38,8 @@ sha1-arm-ce-y	:= sha1-ce-core.o sha1-ce-glue.o
 sha2-arm-ce-y	:= sha2-ce-core.o sha2-ce-glue.o
 aes-arm-ce-y	:= aes-ce-core.o aes-ce-glue.o
 ghash-arm-ce-y	:= ghash-ce-core.o ghash-ce-glue.o
+crct10dif-arm-ce-y	:= crct10dif-ce-core.o crct10dif-ce-glue.o
+crc32-arm-ce-y:= crc32-ce-core.o crc32-ce-glue.o
 
 quiet_cmd_perl = PERL    $@
       cmd_perl = $(PERL) $(<) > $(@)
diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c
index aef022a87c53..8857531915bf 100644
--- a/arch/arm/crypto/aes-ce-glue.c
+++ b/arch/arm/crypto/aes-ce-glue.c
@@ -12,8 +12,8 @@
 #include <asm/neon.h>
 #include <asm/hwcap.h>
 #include <crypto/aes.h>
-#include <crypto/ablk_helper.h>
-#include <crypto/algapi.h>
+#include <crypto/internal/simd.h>
+#include <crypto/internal/skcipher.h>
 #include <linux/module.h>
 #include <crypto/xts.h>
 
@@ -88,8 +88,13 @@ static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 		u32 *rki = ctx->key_enc + (i * kwords);
 		u32 *rko = rki + kwords;
 
+#ifndef CONFIG_CPU_BIG_ENDIAN
 		rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8);
 		rko[0] = rko[0] ^ rki[0] ^ rcon[i];
+#else
+		rko[0] = rol32(ce_aes_sub(rki[kwords - 1]), 8);
+		rko[0] = rko[0] ^ rki[0] ^ (rcon[i] << 24);
+#endif
 		rko[1] = rko[0] ^ rki[1];
 		rko[2] = rko[1] ^ rki[2];
 		rko[3] = rko[2] ^ rki[3];
@@ -128,17 +133,17 @@ static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 	return 0;
 }
 
-static int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
+static int ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 			 unsigned int key_len)
 {
-	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int ret;
 
 	ret = ce_aes_expandkey(ctx, in_key, key_len);
 	if (!ret)
 		return 0;
 
-	tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+	crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 	return -EINVAL;
 }
 
@@ -147,13 +152,13 @@ struct crypto_aes_xts_ctx {
 	struct crypto_aes_ctx __aligned(8) key2;
 };
 
-static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
+static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
 		       unsigned int key_len)
 {
-	struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int ret;
 
-	ret = xts_check_key(tfm, in_key, key_len);
+	ret = xts_verify_key(tfm, in_key, key_len);
 	if (ret)
 		return ret;
 
@@ -164,130 +169,113 @@ static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 	if (!ret)
 		return 0;
 
-	tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+	crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 	return -EINVAL;
 }
 
-static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int ecb_encrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_walk walk;
 	unsigned int blocks;
 	int err;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 		ce_aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				   (u8 *)ctx->key_enc, num_rounds(ctx), blocks);
-		err = blkcipher_walk_done(desc, &walk,
-					  walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	kernel_neon_end();
 	return err;
 }
 
-static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int ecb_decrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_walk walk;
 	unsigned int blocks;
 	int err;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 		ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				   (u8 *)ctx->key_dec, num_rounds(ctx), blocks);
-		err = blkcipher_walk_done(desc, &walk,
-					  walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	kernel_neon_end();
 	return err;
 }
 
-static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int cbc_encrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_walk walk;
 	unsigned int blocks;
 	int err;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 		ce_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				   (u8 *)ctx->key_enc, num_rounds(ctx), blocks,
 				   walk.iv);
-		err = blkcipher_walk_done(desc, &walk,
-					  walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	kernel_neon_end();
 	return err;
 }
 
-static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int cbc_decrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_walk walk;
 	unsigned int blocks;
 	int err;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 		ce_aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				   (u8 *)ctx->key_dec, num_rounds(ctx), blocks,
 				   walk.iv);
-		err = blkcipher_walk_done(desc, &walk,
-					  walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	kernel_neon_end();
 	return err;
 }
 
-static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int ctr_encrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_walk walk;
 	int err, blocks;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 		ce_aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				   (u8 *)ctx->key_enc, num_rounds(ctx), blocks,
 				   walk.iv);
-		nbytes -= blocks * AES_BLOCK_SIZE;
-		if (nbytes && nbytes == walk.nbytes % AES_BLOCK_SIZE)
-			break;
-		err = blkcipher_walk_done(desc, &walk,
-					  walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
-	if (walk.nbytes % AES_BLOCK_SIZE) {
-		u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
-		u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
+	if (walk.nbytes) {
 		u8 __aligned(8) tail[AES_BLOCK_SIZE];
+		unsigned int nbytes = walk.nbytes;
+		u8 *tdst = walk.dst.virt.addr;
+		u8 *tsrc = walk.src.virt.addr;
 
 		/*
 		 * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need
@@ -298,231 +286,172 @@ static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 		ce_aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc,
 				   num_rounds(ctx), blocks, walk.iv);
 		memcpy(tdst, tail, nbytes);
-		err = blkcipher_walk_done(desc, &walk, 0);
+		err = skcipher_walk_done(&walk, 0);
 	}
 	kernel_neon_end();
 
 	return err;
 }
 
-static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int xts_encrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int err, first, rounds = num_rounds(&ctx->key1);
-	struct blkcipher_walk walk;
+	struct skcipher_walk walk;
 	unsigned int blocks;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 		ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				   (u8 *)ctx->key1.key_enc, rounds, blocks,
 				   walk.iv, (u8 *)ctx->key2.key_enc, first);
-		err = blkcipher_walk_done(desc, &walk,
-					  walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	kernel_neon_end();
 
 	return err;
 }
 
-static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int xts_decrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int err, first, rounds = num_rounds(&ctx->key1);
-	struct blkcipher_walk walk;
+	struct skcipher_walk walk;
 	unsigned int blocks;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 		ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				   (u8 *)ctx->key1.key_dec, rounds, blocks,
 				   walk.iv, (u8 *)ctx->key2.key_enc, first);
-		err = blkcipher_walk_done(desc, &walk,
-					  walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	kernel_neon_end();
 
 	return err;
 }
 
-static struct crypto_alg aes_algs[] = { {
-	.cra_name		= "__ecb-aes-ce",
-	.cra_driver_name	= "__driver-ecb-aes-ce",
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_blkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= 0,
-		.setkey		= ce_aes_setkey,
-		.encrypt	= ecb_encrypt,
-		.decrypt	= ecb_decrypt,
+static struct skcipher_alg aes_algs[] = { {
+	.base = {
+		.cra_name		= "__ecb(aes)",
+		.cra_driver_name	= "__ecb-aes-ce",
+		.cra_priority		= 300,
+		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_blocksize		= AES_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
+		.cra_alignmask		= 7,
+		.cra_module		= THIS_MODULE,
 	},
+	.min_keysize	= AES_MIN_KEY_SIZE,
+	.max_keysize	= AES_MAX_KEY_SIZE,
+	.setkey		= ce_aes_setkey,
+	.encrypt	= ecb_encrypt,
+	.decrypt	= ecb_decrypt,
 }, {
-	.cra_name		= "__cbc-aes-ce",
-	.cra_driver_name	= "__driver-cbc-aes-ce",
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_blkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= ce_aes_setkey,
-		.encrypt	= cbc_encrypt,
-		.decrypt	= cbc_decrypt,
+	.base = {
+		.cra_name		= "__cbc(aes)",
+		.cra_driver_name	= "__cbc-aes-ce",
+		.cra_priority		= 300,
+		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_blocksize		= AES_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
+		.cra_alignmask		= 7,
+		.cra_module		= THIS_MODULE,
 	},
+	.min_keysize	= AES_MIN_KEY_SIZE,
+	.max_keysize	= AES_MAX_KEY_SIZE,
+	.ivsize		= AES_BLOCK_SIZE,
+	.setkey		= ce_aes_setkey,
+	.encrypt	= cbc_encrypt,
+	.decrypt	= cbc_decrypt,
 }, {
-	.cra_name		= "__ctr-aes-ce",
-	.cra_driver_name	= "__driver-ctr-aes-ce",
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= 1,
-	.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_blkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= ce_aes_setkey,
-		.encrypt	= ctr_encrypt,
-		.decrypt	= ctr_encrypt,
+	.base = {
+		.cra_name		= "__ctr(aes)",
+		.cra_driver_name	= "__ctr-aes-ce",
+		.cra_priority		= 300,
+		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_blocksize		= 1,
+		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
+		.cra_alignmask		= 7,
+		.cra_module		= THIS_MODULE,
 	},
+	.min_keysize	= AES_MIN_KEY_SIZE,
+	.max_keysize	= AES_MAX_KEY_SIZE,
+	.ivsize		= AES_BLOCK_SIZE,
+	.chunksize	= AES_BLOCK_SIZE,
+	.setkey		= ce_aes_setkey,
+	.encrypt	= ctr_encrypt,
+	.decrypt	= ctr_encrypt,
 }, {
-	.cra_name		= "__xts-aes-ce",
-	.cra_driver_name	= "__driver-xts-aes-ce",
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct crypto_aes_xts_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_blkcipher = {
-		.min_keysize	= 2 * AES_MIN_KEY_SIZE,
-		.max_keysize	= 2 * AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= xts_set_key,
-		.encrypt	= xts_encrypt,
-		.decrypt	= xts_decrypt,
+	.base = {
+		.cra_name		= "__xts(aes)",
+		.cra_driver_name	= "__xts-aes-ce",
+		.cra_priority		= 300,
+		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_blocksize		= AES_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct crypto_aes_xts_ctx),
+		.cra_alignmask		= 7,
+		.cra_module		= THIS_MODULE,
 	},
-}, {
-	.cra_name		= "ecb(aes)",
-	.cra_driver_name	= "ecb-aes-ce",
-	.cra_priority		= 300,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_ablkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= 0,
-		.setkey		= ablk_set_key,
-		.encrypt	= ablk_encrypt,
-		.decrypt	= ablk_decrypt,
-	}
-}, {
-	.cra_name		= "cbc(aes)",
-	.cra_driver_name	= "cbc-aes-ce",
-	.cra_priority		= 300,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_ablkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= ablk_set_key,
-		.encrypt	= ablk_encrypt,
-		.decrypt	= ablk_decrypt,
-	}
-}, {
-	.cra_name		= "ctr(aes)",
-	.cra_driver_name	= "ctr-aes-ce",
-	.cra_priority		= 300,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= 1,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_ablkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= ablk_set_key,
-		.encrypt	= ablk_encrypt,
-		.decrypt	= ablk_decrypt,
-	}
-}, {
-	.cra_name		= "xts(aes)",
-	.cra_driver_name	= "xts-aes-ce",
-	.cra_priority		= 300,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_ablkcipher = {
-		.min_keysize	= 2 * AES_MIN_KEY_SIZE,
-		.max_keysize	= 2 * AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= ablk_set_key,
-		.encrypt	= ablk_encrypt,
-		.decrypt	= ablk_decrypt,
-	}
+	.min_keysize	= 2 * AES_MIN_KEY_SIZE,
+	.max_keysize	= 2 * AES_MAX_KEY_SIZE,
+	.ivsize		= AES_BLOCK_SIZE,
+	.setkey		= xts_set_key,
+	.encrypt	= xts_encrypt,
+	.decrypt	= xts_decrypt,
 } };
 
+static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
+
+static void aes_exit(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++)
+		simd_skcipher_free(aes_simd_algs[i]);
+
+	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
+}
+
 static int __init aes_init(void)
 {
+	struct simd_skcipher_alg *simd;
+	const char *basename;
+	const char *algname;
+	const char *drvname;
+	int err;
+	int i;
+
 	if (!(elf_hwcap2 & HWCAP2_AES))
 		return -ENODEV;
-	return crypto_register_algs(aes_algs, ARRAY_SIZE(aes_algs));
-}
 
-static void __exit aes_exit(void)
-{
-	crypto_unregister_algs(aes_algs, ARRAY_SIZE(aes_algs));
+	err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
+	if (err)
+		return err;
+
+	for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
+		algname = aes_algs[i].base.cra_name + 2;
+		drvname = aes_algs[i].base.cra_driver_name + 2;
+		basename = aes_algs[i].base.cra_driver_name;
+		simd = simd_skcipher_create_compat(algname, drvname, basename);
+		err = PTR_ERR(simd);
+		if (IS_ERR(simd))
+			goto unregister_simds;
+
+		aes_simd_algs[i] = simd;
+	}
+
+	return 0;
+
+unregister_simds:
+	aes_exit();
+	return err;
 }
 
 module_init(aes_init);
diff --git a/arch/arm/crypto/aesbs-glue.c b/arch/arm/crypto/aesbs-glue.c
index 0511a6cafe24..d8e06de72ef3 100644
--- a/arch/arm/crypto/aesbs-glue.c
+++ b/arch/arm/crypto/aesbs-glue.c
@@ -10,8 +10,9 @@
 
 #include <asm/neon.h>
 #include <crypto/aes.h>
-#include <crypto/ablk_helper.h>
-#include <crypto/algapi.h>
+#include <crypto/cbc.h>
+#include <crypto/internal/simd.h>
+#include <crypto/internal/skcipher.h>
 #include <linux/module.h>
 #include <crypto/xts.h>
 
@@ -55,14 +56,14 @@ struct aesbs_xts_ctx {
 	struct AES_KEY	twkey;
 };
 
-static int aesbs_cbc_set_key(struct crypto_tfm *tfm, const u8 *in_key,
+static int aesbs_cbc_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
 			     unsigned int key_len)
 {
-	struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int bits = key_len * 8;
 
 	if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc)) {
-		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 		return -EINVAL;
 	}
 	ctx->dec.rk = ctx->enc;
@@ -71,33 +72,33 @@ static int aesbs_cbc_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 	return 0;
 }
 
-static int aesbs_ctr_set_key(struct crypto_tfm *tfm, const u8 *in_key,
+static int aesbs_ctr_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
 			     unsigned int key_len)
 {
-	struct aesbs_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int bits = key_len * 8;
 
 	if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) {
-		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 		return -EINVAL;
 	}
 	ctx->enc.converted = 0;
 	return 0;
 }
 
-static int aesbs_xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
+static int aesbs_xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
 			     unsigned int key_len)
 {
-	struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int bits = key_len * 4;
 	int err;
 
-	err = xts_check_key(tfm, in_key, key_len);
+	err = xts_verify_key(tfm, in_key, key_len);
 	if (err)
 		return err;
 
 	if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) {
-		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 		return -EINVAL;
 	}
 	ctx->dec.rk = ctx->enc.rk;
@@ -107,88 +108,52 @@ static int aesbs_xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 	return 0;
 }
 
-static int aesbs_cbc_encrypt(struct blkcipher_desc *desc,
-			     struct scatterlist *dst,
-			     struct scatterlist *src, unsigned int nbytes)
+static inline void aesbs_encrypt_one(struct crypto_skcipher *tfm,
+				     const u8 *src, u8 *dst)
 {
-	struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk walk;
-	int err;
+	struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	AES_encrypt(src, dst, &ctx->enc);
+}
 
-	while (walk.nbytes) {
-		u32 blocks = walk.nbytes / AES_BLOCK_SIZE;
-		u8 *src = walk.src.virt.addr;
+static int aesbs_cbc_encrypt(struct skcipher_request *req)
+{
+	return crypto_cbc_encrypt_walk(req, aesbs_encrypt_one);
+}
 
-		if (walk.dst.virt.addr == walk.src.virt.addr) {
-			u8 *iv = walk.iv;
-
-			do {
-				crypto_xor(src, iv, AES_BLOCK_SIZE);
-				AES_encrypt(src, src, &ctx->enc);
-				iv = src;
-				src += AES_BLOCK_SIZE;
-			} while (--blocks);
-			memcpy(walk.iv, iv, AES_BLOCK_SIZE);
-		} else {
-			u8 *dst = walk.dst.virt.addr;
-
-			do {
-				crypto_xor(walk.iv, src, AES_BLOCK_SIZE);
-				AES_encrypt(walk.iv, dst, &ctx->enc);
-				memcpy(walk.iv, dst, AES_BLOCK_SIZE);
-				src += AES_BLOCK_SIZE;
-				dst += AES_BLOCK_SIZE;
-			} while (--blocks);
-		}
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
-	}
-	return err;
+static inline void aesbs_decrypt_one(struct crypto_skcipher *tfm,
+				     const u8 *src, u8 *dst)
+{
+	struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+	AES_decrypt(src, dst, &ctx->dec.rk);
 }
 
-static int aesbs_cbc_decrypt(struct blkcipher_desc *desc,
-			     struct scatterlist *dst,
-			     struct scatterlist *src, unsigned int nbytes)
+static int aesbs_cbc_decrypt(struct skcipher_request *req)
 {
-	struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_walk walk;
+	unsigned int nbytes;
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
-
-	while ((walk.nbytes / AES_BLOCK_SIZE) >= 8) {
-		kernel_neon_begin();
-		bsaes_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
-				  walk.nbytes, &ctx->dec, walk.iv);
-		kernel_neon_end();
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
-	}
-	while (walk.nbytes) {
-		u32 blocks = walk.nbytes / AES_BLOCK_SIZE;
+	for (err = skcipher_walk_virt(&walk, req, false);
+	     (nbytes = walk.nbytes); err = skcipher_walk_done(&walk, nbytes)) {
+		u32 blocks = nbytes / AES_BLOCK_SIZE;
 		u8 *dst = walk.dst.virt.addr;
 		u8 *src = walk.src.virt.addr;
-		u8 bk[2][AES_BLOCK_SIZE];
 		u8 *iv = walk.iv;
 
-		do {
-			if (walk.dst.virt.addr == walk.src.virt.addr)
-				memcpy(bk[blocks & 1], src, AES_BLOCK_SIZE);
-
-			AES_decrypt(src, dst, &ctx->dec.rk);
-			crypto_xor(dst, iv, AES_BLOCK_SIZE);
-
-			if (walk.dst.virt.addr == walk.src.virt.addr)
-				iv = bk[blocks & 1];
-			else
-				iv = src;
+		if (blocks >= 8) {
+			kernel_neon_begin();
+			bsaes_cbc_encrypt(src, dst, nbytes, &ctx->dec, iv);
+			kernel_neon_end();
+			nbytes %= AES_BLOCK_SIZE;
+			continue;
+		}
 
-			dst += AES_BLOCK_SIZE;
-			src += AES_BLOCK_SIZE;
-		} while (--blocks);
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
+		nbytes = crypto_cbc_decrypt_blocks(&walk, tfm,
+						   aesbs_decrypt_one);
 	}
 	return err;
 }
@@ -206,17 +171,15 @@ static void inc_be128_ctr(__be32 ctr[], u32 addend)
 	}
 }
 
-static int aesbs_ctr_encrypt(struct blkcipher_desc *desc,
-			     struct scatterlist *dst, struct scatterlist *src,
-			     unsigned int nbytes)
+static int aesbs_ctr_encrypt(struct skcipher_request *req)
 {
-	struct aesbs_ctr_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_walk walk;
 	u32 blocks;
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
+	err = skcipher_walk_virt(&walk, req, false);
 
 	while ((blocks = walk.nbytes / AES_BLOCK_SIZE)) {
 		u32 tail = walk.nbytes % AES_BLOCK_SIZE;
@@ -235,11 +198,7 @@ static int aesbs_ctr_encrypt(struct blkcipher_desc *desc,
 		kernel_neon_end();
 		inc_be128_ctr(ctr, blocks);
 
-		nbytes -= blocks * AES_BLOCK_SIZE;
-		if (nbytes && nbytes == tail && nbytes <= AES_BLOCK_SIZE)
-			break;
-
-		err = blkcipher_walk_done(desc, &walk, tail);
+		err = skcipher_walk_done(&walk, tail);
 	}
 	if (walk.nbytes) {
 		u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
@@ -248,23 +207,21 @@ static int aesbs_ctr_encrypt(struct blkcipher_desc *desc,
 
 		AES_encrypt(walk.iv, ks, &ctx->enc.rk);
 		if (tdst != tsrc)
-			memcpy(tdst, tsrc, nbytes);
-		crypto_xor(tdst, ks, nbytes);
-		err = blkcipher_walk_done(desc, &walk, 0);
+			memcpy(tdst, tsrc, walk.nbytes);
+		crypto_xor(tdst, ks, walk.nbytes);
+		err = skcipher_walk_done(&walk, 0);
 	}
 	return err;
 }
 
-static int aesbs_xts_encrypt(struct blkcipher_desc *desc,
-			     struct scatterlist *dst,
-			     struct scatterlist *src, unsigned int nbytes)
+static int aesbs_xts_encrypt(struct skcipher_request *req)
 {
-	struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_walk walk;
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
+	err = skcipher_walk_virt(&walk, req, false);
 
 	/* generate the initial tweak */
 	AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
@@ -274,21 +231,19 @@ static int aesbs_xts_encrypt(struct blkcipher_desc *desc,
 		bsaes_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
 				  walk.nbytes, &ctx->enc, walk.iv);
 		kernel_neon_end();
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	return err;
 }
 
-static int aesbs_xts_decrypt(struct blkcipher_desc *desc,
-			     struct scatterlist *dst,
-			     struct scatterlist *src, unsigned int nbytes)
+static int aesbs_xts_decrypt(struct skcipher_request *req)
 {
-	struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_walk walk;
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
+	err = skcipher_walk_virt(&walk, req, false);
 
 	/* generate the initial tweak */
 	AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
@@ -298,141 +253,110 @@ static int aesbs_xts_decrypt(struct blkcipher_desc *desc,
 		bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr,
 				  walk.nbytes, &ctx->dec, walk.iv);
 		kernel_neon_end();
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	return err;
 }
 
-static struct crypto_alg aesbs_algs[] = { {
-	.cra_name		= "__cbc-aes-neonbs",
-	.cra_driver_name	= "__driver-cbc-aes-neonbs",
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct aesbs_cbc_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_blkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= aesbs_cbc_set_key,
-		.encrypt	= aesbs_cbc_encrypt,
-		.decrypt	= aesbs_cbc_decrypt,
+static struct skcipher_alg aesbs_algs[] = { {
+	.base = {
+		.cra_name		= "__cbc(aes)",
+		.cra_driver_name	= "__cbc-aes-neonbs",
+		.cra_priority		= 300,
+		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_blocksize		= AES_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct aesbs_cbc_ctx),
+		.cra_alignmask		= 7,
+		.cra_module		= THIS_MODULE,
 	},
+	.min_keysize	= AES_MIN_KEY_SIZE,
+	.max_keysize	= AES_MAX_KEY_SIZE,
+	.ivsize		= AES_BLOCK_SIZE,
+	.setkey		= aesbs_cbc_set_key,
+	.encrypt	= aesbs_cbc_encrypt,
+	.decrypt	= aesbs_cbc_decrypt,
 }, {
-	.cra_name		= "__ctr-aes-neonbs",
-	.cra_driver_name	= "__driver-ctr-aes-neonbs",
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= 1,
-	.cra_ctxsize		= sizeof(struct aesbs_ctr_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_blkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= aesbs_ctr_set_key,
-		.encrypt	= aesbs_ctr_encrypt,
-		.decrypt	= aesbs_ctr_encrypt,
+	.base = {
+		.cra_name		= "__ctr(aes)",
+		.cra_driver_name	= "__ctr-aes-neonbs",
+		.cra_priority		= 300,
+		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_blocksize		= 1,
+		.cra_ctxsize		= sizeof(struct aesbs_ctr_ctx),
+		.cra_alignmask		= 7,
+		.cra_module		= THIS_MODULE,
 	},
+	.min_keysize	= AES_MIN_KEY_SIZE,
+	.max_keysize	= AES_MAX_KEY_SIZE,
+	.ivsize		= AES_BLOCK_SIZE,
+	.chunksize	= AES_BLOCK_SIZE,
+	.setkey		= aesbs_ctr_set_key,
+	.encrypt	= aesbs_ctr_encrypt,
+	.decrypt	= aesbs_ctr_encrypt,
 }, {
-	.cra_name		= "__xts-aes-neonbs",
-	.cra_driver_name	= "__driver-xts-aes-neonbs",
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct aesbs_xts_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_blkcipher = {
-		.min_keysize	= 2 * AES_MIN_KEY_SIZE,
-		.max_keysize	= 2 * AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= aesbs_xts_set_key,
-		.encrypt	= aesbs_xts_encrypt,
-		.decrypt	= aesbs_xts_decrypt,
+	.base = {
+		.cra_name		= "__xts(aes)",
+		.cra_driver_name	= "__xts-aes-neonbs",
+		.cra_priority		= 300,
+		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_blocksize		= AES_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct aesbs_xts_ctx),
+		.cra_alignmask		= 7,
+		.cra_module		= THIS_MODULE,
 	},
-}, {
-	.cra_name		= "cbc(aes)",
-	.cra_driver_name	= "cbc-aes-neonbs",
-	.cra_priority		= 300,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_ablkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= ablk_set_key,
-		.encrypt	= __ablk_encrypt,
-		.decrypt	= ablk_decrypt,
-	}
-}, {
-	.cra_name		= "ctr(aes)",
-	.cra_driver_name	= "ctr-aes-neonbs",
-	.cra_priority		= 300,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= 1,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_ablkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= ablk_set_key,
-		.encrypt	= ablk_encrypt,
-		.decrypt	= ablk_decrypt,
-	}
-}, {
-	.cra_name		= "xts(aes)",
-	.cra_driver_name	= "xts-aes-neonbs",
-	.cra_priority		= 300,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_ablkcipher = {
-		.min_keysize	= 2 * AES_MIN_KEY_SIZE,
-		.max_keysize	= 2 * AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= ablk_set_key,
-		.encrypt	= ablk_encrypt,
-		.decrypt	= ablk_decrypt,
-	}
+	.min_keysize	= 2 * AES_MIN_KEY_SIZE,
+	.max_keysize	= 2 * AES_MAX_KEY_SIZE,
+	.ivsize		= AES_BLOCK_SIZE,
+	.setkey		= aesbs_xts_set_key,
+	.encrypt	= aesbs_xts_encrypt,
+	.decrypt	= aesbs_xts_decrypt,
 } };
 
+struct simd_skcipher_alg *aesbs_simd_algs[ARRAY_SIZE(aesbs_algs)];
+
+static void aesbs_mod_exit(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(aesbs_simd_algs) && aesbs_simd_algs[i]; i++)
+		simd_skcipher_free(aesbs_simd_algs[i]);
+
+	crypto_unregister_skciphers(aesbs_algs, ARRAY_SIZE(aesbs_algs));
+}
+
 static int __init aesbs_mod_init(void)
 {
+	struct simd_skcipher_alg *simd;
+	const char *basename;
+	const char *algname;
+	const char *drvname;
+	int err;
+	int i;
+
 	if (!cpu_has_neon())
 		return -ENODEV;
 
-	return crypto_register_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs));
-}
+	err = crypto_register_skciphers(aesbs_algs, ARRAY_SIZE(aesbs_algs));
+	if (err)
+		return err;
 
-static void __exit aesbs_mod_exit(void)
-{
-	crypto_unregister_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs));
+	for (i = 0; i < ARRAY_SIZE(aesbs_algs); i++) {
+		algname = aesbs_algs[i].base.cra_name + 2;
+		drvname = aesbs_algs[i].base.cra_driver_name + 2;
+		basename = aesbs_algs[i].base.cra_driver_name;
+		simd = simd_skcipher_create_compat(algname, drvname, basename);
+		err = PTR_ERR(simd);
+		if (IS_ERR(simd))
+			goto unregister_simds;
+
+		aesbs_simd_algs[i] = simd;
+	}
+
+	return 0;
+
+unregister_simds:
+	aesbs_mod_exit();
+	return err;
 }
 
 module_init(aesbs_mod_init);
diff --git a/arch/arm/crypto/crc32-ce-core.S b/arch/arm/crypto/crc32-ce-core.S
new file mode 100644
index 000000000000..e63d400dc5c1
--- /dev/null
+++ b/arch/arm/crypto/crc32-ce-core.S
@@ -0,0 +1,306 @@
+/*
+ * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/* GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please  visit http://www.xyratex.com/contact if you need additional
+ * information or have any questions.
+ *
+ * GPL HEADER END
+ */
+
+/*
+ * Copyright 2012 Xyratex Technology Limited
+ *
+ * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
+ * calculation.
+ * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
+ * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
+ * at:
+ * http://www.intel.com/products/processor/manuals/
+ * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
+ * Volume 2B: Instruction Set Reference, N-Z
+ *
+ * Authors:   Gregory Prestas <Gregory_Prestas@us.xyratex.com>
+ *	      Alexander Boyko <Alexander_Boyko@xyratex.com>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.text
+	.align		6
+	.arch		armv8-a
+	.arch_extension	crc
+	.fpu		crypto-neon-fp-armv8
+
+.Lcrc32_constants:
+	/*
+	 * [x4*128+32 mod P(x) << 32)]'  << 1   = 0x154442bd4
+	 * #define CONSTANT_R1  0x154442bd4LL
+	 *
+	 * [(x4*128-32 mod P(x) << 32)]' << 1   = 0x1c6e41596
+	 * #define CONSTANT_R2  0x1c6e41596LL
+	 */
+	.quad		0x0000000154442bd4
+	.quad		0x00000001c6e41596
+
+	/*
+	 * [(x128+32 mod P(x) << 32)]'   << 1   = 0x1751997d0
+	 * #define CONSTANT_R3  0x1751997d0LL
+	 *
+	 * [(x128-32 mod P(x) << 32)]'   << 1   = 0x0ccaa009e
+	 * #define CONSTANT_R4  0x0ccaa009eLL
+	 */
+	.quad		0x00000001751997d0
+	.quad		0x00000000ccaa009e
+
+	/*
+	 * [(x64 mod P(x) << 32)]'       << 1   = 0x163cd6124
+	 * #define CONSTANT_R5  0x163cd6124LL
+	 */
+	.quad		0x0000000163cd6124
+	.quad		0x00000000FFFFFFFF
+
+	/*
+	 * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
+	 *
+	 * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))`
+	 *                                                      = 0x1F7011641LL
+	 * #define CONSTANT_RU  0x1F7011641LL
+	 */
+	.quad		0x00000001DB710641
+	.quad		0x00000001F7011641
+
+.Lcrc32c_constants:
+	.quad		0x00000000740eef02
+	.quad		0x000000009e4addf8
+	.quad		0x00000000f20c0dfe
+	.quad		0x000000014cd00bd6
+	.quad		0x00000000dd45aab8
+	.quad		0x00000000FFFFFFFF
+	.quad		0x0000000105ec76f0
+	.quad		0x00000000dea713f1
+
+	dCONSTANTl	.req	d0
+	dCONSTANTh	.req	d1
+	qCONSTANT	.req	q0
+
+	BUF		.req	r0
+	LEN		.req	r1
+	CRC		.req	r2
+
+	qzr		.req	q9
+
+	/**
+	 * Calculate crc32
+	 * BUF - buffer
+	 * LEN - sizeof buffer (multiple of 16 bytes), LEN should be > 63
+	 * CRC - initial crc32
+	 * return %eax crc32
+	 * uint crc32_pmull_le(unsigned char const *buffer,
+	 *                     size_t len, uint crc32)
+	 */
+ENTRY(crc32_pmull_le)
+	adr		r3, .Lcrc32_constants
+	b		0f
+
+ENTRY(crc32c_pmull_le)
+	adr		r3, .Lcrc32c_constants
+
+0:	bic		LEN, LEN, #15
+	vld1.8		{q1-q2}, [BUF, :128]!
+	vld1.8		{q3-q4}, [BUF, :128]!
+	vmov.i8		qzr, #0
+	vmov.i8		qCONSTANT, #0
+	vmov		dCONSTANTl[0], CRC
+	veor.8		d2, d2, dCONSTANTl
+	sub		LEN, LEN, #0x40
+	cmp		LEN, #0x40
+	blt		less_64
+
+	vld1.64		{qCONSTANT}, [r3]
+
+loop_64:		/* 64 bytes Full cache line folding */
+	sub		LEN, LEN, #0x40
+
+	vmull.p64	q5, d3, dCONSTANTh
+	vmull.p64	q6, d5, dCONSTANTh
+	vmull.p64	q7, d7, dCONSTANTh
+	vmull.p64	q8, d9, dCONSTANTh
+
+	vmull.p64	q1, d2, dCONSTANTl
+	vmull.p64	q2, d4, dCONSTANTl
+	vmull.p64	q3, d6, dCONSTANTl
+	vmull.p64	q4, d8, dCONSTANTl
+
+	veor.8		q1, q1, q5
+	vld1.8		{q5}, [BUF, :128]!
+	veor.8		q2, q2, q6
+	vld1.8		{q6}, [BUF, :128]!
+	veor.8		q3, q3, q7
+	vld1.8		{q7}, [BUF, :128]!
+	veor.8		q4, q4, q8
+	vld1.8		{q8}, [BUF, :128]!
+
+	veor.8		q1, q1, q5
+	veor.8		q2, q2, q6
+	veor.8		q3, q3, q7
+	veor.8		q4, q4, q8
+
+	cmp		LEN, #0x40
+	bge		loop_64
+
+less_64:		/* Folding cache line into 128bit */
+	vldr		dCONSTANTl, [r3, #16]
+	vldr		dCONSTANTh, [r3, #24]
+
+	vmull.p64	q5, d3, dCONSTANTh
+	vmull.p64	q1, d2, dCONSTANTl
+	veor.8		q1, q1, q5
+	veor.8		q1, q1, q2
+
+	vmull.p64	q5, d3, dCONSTANTh
+	vmull.p64	q1, d2, dCONSTANTl
+	veor.8		q1, q1, q5
+	veor.8		q1, q1, q3
+
+	vmull.p64	q5, d3, dCONSTANTh
+	vmull.p64	q1, d2, dCONSTANTl
+	veor.8		q1, q1, q5
+	veor.8		q1, q1, q4
+
+	teq		LEN, #0
+	beq		fold_64
+
+loop_16:		/* Folding rest buffer into 128bit */
+	subs		LEN, LEN, #0x10
+
+	vld1.8		{q2}, [BUF, :128]!
+	vmull.p64	q5, d3, dCONSTANTh
+	vmull.p64	q1, d2, dCONSTANTl
+	veor.8		q1, q1, q5
+	veor.8		q1, q1, q2
+
+	bne		loop_16
+
+fold_64:
+	/* perform the last 64 bit fold, also adds 32 zeroes
+	 * to the input stream */
+	vmull.p64	q2, d2, dCONSTANTh
+	vext.8		q1, q1, qzr, #8
+	veor.8		q1, q1, q2
+
+	/* final 32-bit fold */
+	vldr		dCONSTANTl, [r3, #32]
+	vldr		d6, [r3, #40]
+	vmov.i8		d7, #0
+
+	vext.8		q2, q1, qzr, #4
+	vand.8		d2, d2, d6
+	vmull.p64	q1, d2, dCONSTANTl
+	veor.8		q1, q1, q2
+
+	/* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
+	vldr		dCONSTANTl, [r3, #48]
+	vldr		dCONSTANTh, [r3, #56]
+
+	vand.8		q2, q1, q3
+	vext.8		q2, qzr, q2, #8
+	vmull.p64	q2, d5, dCONSTANTh
+	vand.8		q2, q2, q3
+	vmull.p64	q2, d4, dCONSTANTl
+	veor.8		q1, q1, q2
+	vmov		r0, s5
+
+	bx		lr
+ENDPROC(crc32_pmull_le)
+ENDPROC(crc32c_pmull_le)
+
+	.macro		__crc32, c
+	subs		ip, r2, #8
+	bmi		.Ltail\c
+
+	tst		r1, #3
+	bne		.Lunaligned\c
+
+	teq		ip, #0
+.Laligned8\c:
+	ldrd		r2, r3, [r1], #8
+ARM_BE8(rev		r2, r2		)
+ARM_BE8(rev		r3, r3		)
+	crc32\c\()w	r0, r0, r2
+	crc32\c\()w	r0, r0, r3
+	bxeq		lr
+	subs		ip, ip, #8
+	bpl		.Laligned8\c
+
+.Ltail\c:
+	tst		ip, #4
+	beq		2f
+	ldr		r3, [r1], #4
+ARM_BE8(rev		r3, r3		)
+	crc32\c\()w	r0, r0, r3
+
+2:	tst		ip, #2
+	beq		1f
+	ldrh		r3, [r1], #2
+ARM_BE8(rev16		r3, r3		)
+	crc32\c\()h	r0, r0, r3
+
+1:	tst		ip, #1
+	bxeq		lr
+	ldrb		r3, [r1]
+	crc32\c\()b	r0, r0, r3
+	bx		lr
+
+.Lunaligned\c:
+	tst		r1, #1
+	beq		2f
+	ldrb		r3, [r1], #1
+	subs		r2, r2, #1
+	crc32\c\()b	r0, r0, r3
+
+	tst		r1, #2
+	beq		0f
+2:	ldrh		r3, [r1], #2
+	subs		r2, r2, #2
+ARM_BE8(rev16		r3, r3		)
+	crc32\c\()h	r0, r0, r3
+
+0:	subs		ip, r2, #8
+	bpl		.Laligned8\c
+	b		.Ltail\c
+	.endm
+
+	.align		5
+ENTRY(crc32_armv8_le)
+	__crc32
+ENDPROC(crc32_armv8_le)
+
+	.align		5
+ENTRY(crc32c_armv8_le)
+	__crc32		c
+ENDPROC(crc32c_armv8_le)
diff --git a/arch/arm/crypto/crc32-ce-glue.c b/arch/arm/crypto/crc32-ce-glue.c
new file mode 100644
index 000000000000..e1566bec1016
--- /dev/null
+++ b/arch/arm/crypto/crc32-ce-glue.c
@@ -0,0 +1,242 @@
+/*
+ * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/crc32.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h>
+
+#include <crypto/internal/hash.h>
+
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+#include <asm/simd.h>
+#include <asm/unaligned.h>
+
+#define PMULL_MIN_LEN		64L	/* minimum size of buffer
+					 * for crc32_pmull_le_16 */
+#define SCALE_F			16L	/* size of NEON register */
+
+asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc);
+asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len);
+
+asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc);
+asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len);
+
+static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], u32 len);
+static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], u32 len);
+
+static int crc32_cra_init(struct crypto_tfm *tfm)
+{
+	u32 *key = crypto_tfm_ctx(tfm);
+
+	*key = 0;
+	return 0;
+}
+
+static int crc32c_cra_init(struct crypto_tfm *tfm)
+{
+	u32 *key = crypto_tfm_ctx(tfm);
+
+	*key = ~0;
+	return 0;
+}
+
+static int crc32_setkey(struct crypto_shash *hash, const u8 *key,
+			unsigned int keylen)
+{
+	u32 *mctx = crypto_shash_ctx(hash);
+
+	if (keylen != sizeof(u32)) {
+		crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+	*mctx = le32_to_cpup((__le32 *)key);
+	return 0;
+}
+
+static int crc32_init(struct shash_desc *desc)
+{
+	u32 *mctx = crypto_shash_ctx(desc->tfm);
+	u32 *crc = shash_desc_ctx(desc);
+
+	*crc = *mctx;
+	return 0;
+}
+
+static int crc32_update(struct shash_desc *desc, const u8 *data,
+			unsigned int length)
+{
+	u32 *crc = shash_desc_ctx(desc);
+
+	*crc = crc32_armv8_le(*crc, data, length);
+	return 0;
+}
+
+static int crc32c_update(struct shash_desc *desc, const u8 *data,
+			 unsigned int length)
+{
+	u32 *crc = shash_desc_ctx(desc);
+
+	*crc = crc32c_armv8_le(*crc, data, length);
+	return 0;
+}
+
+static int crc32_final(struct shash_desc *desc, u8 *out)
+{
+	u32 *crc = shash_desc_ctx(desc);
+
+	put_unaligned_le32(*crc, out);
+	return 0;
+}
+
+static int crc32c_final(struct shash_desc *desc, u8 *out)
+{
+	u32 *crc = shash_desc_ctx(desc);
+
+	put_unaligned_le32(~*crc, out);
+	return 0;
+}
+
+static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
+			      unsigned int length)
+{
+	u32 *crc = shash_desc_ctx(desc);
+	unsigned int l;
+
+	if (may_use_simd()) {
+		if ((u32)data % SCALE_F) {
+			l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
+
+			*crc = fallback_crc32(*crc, data, l);
+
+			data += l;
+			length -= l;
+		}
+
+		if (length >= PMULL_MIN_LEN) {
+			l = round_down(length, SCALE_F);
+
+			kernel_neon_begin();
+			*crc = crc32_pmull_le(data, l, *crc);
+			kernel_neon_end();
+
+			data += l;
+			length -= l;
+		}
+	}
+
+	if (length > 0)
+		*crc = fallback_crc32(*crc, data, length);
+
+	return 0;
+}
+
+static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
+			       unsigned int length)
+{
+	u32 *crc = shash_desc_ctx(desc);
+	unsigned int l;
+
+	if (may_use_simd()) {
+		if ((u32)data % SCALE_F) {
+			l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F));
+
+			*crc = fallback_crc32c(*crc, data, l);
+
+			data += l;
+			length -= l;
+		}
+
+		if (length >= PMULL_MIN_LEN) {
+			l = round_down(length, SCALE_F);
+
+			kernel_neon_begin();
+			*crc = crc32c_pmull_le(data, l, *crc);
+			kernel_neon_end();
+
+			data += l;
+			length -= l;
+		}
+	}
+
+	if (length > 0)
+		*crc = fallback_crc32c(*crc, data, length);
+
+	return 0;
+}
+
+static struct shash_alg crc32_pmull_algs[] = { {
+	.setkey			= crc32_setkey,
+	.init			= crc32_init,
+	.update			= crc32_update,
+	.final			= crc32_final,
+	.descsize		= sizeof(u32),
+	.digestsize		= sizeof(u32),
+
+	.base.cra_ctxsize	= sizeof(u32),
+	.base.cra_init		= crc32_cra_init,
+	.base.cra_name		= "crc32",
+	.base.cra_driver_name	= "crc32-arm-ce",
+	.base.cra_priority	= 200,
+	.base.cra_blocksize	= 1,
+	.base.cra_module	= THIS_MODULE,
+}, {
+	.setkey			= crc32_setkey,
+	.init			= crc32_init,
+	.update			= crc32c_update,
+	.final			= crc32c_final,
+	.descsize		= sizeof(u32),
+	.digestsize		= sizeof(u32),
+
+	.base.cra_ctxsize	= sizeof(u32),
+	.base.cra_init		= crc32c_cra_init,
+	.base.cra_name		= "crc32c",
+	.base.cra_driver_name	= "crc32c-arm-ce",
+	.base.cra_priority	= 200,
+	.base.cra_blocksize	= 1,
+	.base.cra_module	= THIS_MODULE,
+} };
+
+static int __init crc32_pmull_mod_init(void)
+{
+	if (elf_hwcap2 & HWCAP2_PMULL) {
+		crc32_pmull_algs[0].update = crc32_pmull_update;
+		crc32_pmull_algs[1].update = crc32c_pmull_update;
+
+		if (elf_hwcap2 & HWCAP2_CRC32) {
+			fallback_crc32 = crc32_armv8_le;
+			fallback_crc32c = crc32c_armv8_le;
+		} else {
+			fallback_crc32 = crc32_le;
+			fallback_crc32c = __crc32c_le;
+		}
+	} else if (!(elf_hwcap2 & HWCAP2_CRC32)) {
+		return -ENODEV;
+	}
+
+	return crypto_register_shashes(crc32_pmull_algs,
+				       ARRAY_SIZE(crc32_pmull_algs));
+}
+
+static void __exit crc32_pmull_mod_exit(void)
+{
+	crypto_unregister_shashes(crc32_pmull_algs,
+				  ARRAY_SIZE(crc32_pmull_algs));
+}
+
+module_init(crc32_pmull_mod_init);
+module_exit(crc32_pmull_mod_exit);
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("crc32");
+MODULE_ALIAS_CRYPTO("crc32c");
diff --git a/arch/arm/crypto/crct10dif-ce-core.S b/arch/arm/crypto/crct10dif-ce-core.S
new file mode 100644
index 000000000000..ce45ba0c0687
--- /dev/null
+++ b/arch/arm/crypto/crct10dif-ce-core.S
@@ -0,0 +1,427 @@
+//
+// Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions
+//
+// Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License version 2 as
+// published by the Free Software Foundation.
+//
+
+//
+// Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions
+//
+// Copyright (c) 2013, Intel Corporation
+//
+// Authors:
+//     Erdinc Ozturk <erdinc.ozturk@intel.com>
+//     Vinodh Gopal <vinodh.gopal@intel.com>
+//     James Guilford <james.guilford@intel.com>
+//     Tim Chen <tim.c.chen@linux.intel.com>
+//
+// This software is available to you under a choice of one of two
+// licenses.  You may choose to be licensed under the terms of the GNU
+// General Public License (GPL) Version 2, available from the file
+// COPYING in the main directory of this source tree, or the
+// OpenIB.org BSD license below:
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+//   notice, this list of conditions and the following disclaimer.
+//
+// * Redistributions in binary form must reproduce the above copyright
+//   notice, this list of conditions and the following disclaimer in the
+//   documentation and/or other materials provided with the
+//   distribution.
+//
+// * Neither the name of the Intel Corporation nor the names of its
+//   contributors may be used to endorse or promote products derived from
+//   this software without specific prior written permission.
+//
+//
+// THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+//       Function API:
+//       UINT16 crc_t10dif_pcl(
+//               UINT16 init_crc, //initial CRC value, 16 bits
+//               const unsigned char *buf, //buffer pointer to calculate CRC on
+//               UINT64 len //buffer length in bytes (64-bit data)
+//       );
+//
+//       Reference paper titled "Fast CRC Computation for Generic
+//	Polynomials Using PCLMULQDQ Instruction"
+//       URL: http://www.intel.com/content/dam/www/public/us/en/documents
+//  /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
+//
+//
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+#ifdef CONFIG_CPU_ENDIAN_BE8
+#define CPU_LE(code...)
+#else
+#define CPU_LE(code...)		code
+#endif
+
+	.text
+	.fpu		crypto-neon-fp-armv8
+
+	arg1_low32	.req	r0
+	arg2		.req	r1
+	arg3		.req	r2
+
+	qzr		.req	q13
+
+	q0l		.req	d0
+	q0h		.req	d1
+	q1l		.req	d2
+	q1h		.req	d3
+	q2l		.req	d4
+	q2h		.req	d5
+	q3l		.req	d6
+	q3h		.req	d7
+	q4l		.req	d8
+	q4h		.req	d9
+	q5l		.req	d10
+	q5h		.req	d11
+	q6l		.req	d12
+	q6h		.req	d13
+	q7l		.req	d14
+	q7h		.req	d15
+
+ENTRY(crc_t10dif_pmull)
+	vmov.i8		qzr, #0			// init zero register
+
+	// adjust the 16-bit initial_crc value, scale it to 32 bits
+	lsl		arg1_low32, arg1_low32, #16
+
+	// check if smaller than 256
+	cmp		arg3, #256
+
+	// for sizes less than 128, we can't fold 64B at a time...
+	blt		_less_than_128
+
+	// load the initial crc value
+	// crc value does not need to be byte-reflected, but it needs
+	// to be moved to the high part of the register.
+	// because data will be byte-reflected and will align with
+	// initial crc at correct place.
+	vmov		s0, arg1_low32		// initial crc
+	vext.8		q10, qzr, q0, #4
+
+	// receive the initial 64B data, xor the initial crc value
+	vld1.64		{q0-q1}, [arg2, :128]!
+	vld1.64		{q2-q3}, [arg2, :128]!
+	vld1.64		{q4-q5}, [arg2, :128]!
+	vld1.64		{q6-q7}, [arg2, :128]!
+CPU_LE(	vrev64.8	q0, q0			)
+CPU_LE(	vrev64.8	q1, q1			)
+CPU_LE(	vrev64.8	q2, q2			)
+CPU_LE(	vrev64.8	q3, q3			)
+CPU_LE(	vrev64.8	q4, q4			)
+CPU_LE(	vrev64.8	q5, q5			)
+CPU_LE(	vrev64.8	q6, q6			)
+CPU_LE(	vrev64.8	q7, q7			)
+
+	vswp		d0, d1
+	vswp		d2, d3
+	vswp		d4, d5
+	vswp		d6, d7
+	vswp		d8, d9
+	vswp		d10, d11
+	vswp		d12, d13
+	vswp		d14, d15
+
+	// XOR the initial_crc value
+	veor.8		q0, q0, q10
+
+	adr		ip, rk3
+	vld1.64		{q10}, [ip, :128]	// xmm10 has rk3 and rk4
+
+	//
+	// we subtract 256 instead of 128 to save one instruction from the loop
+	//
+	sub		arg3, arg3, #256
+
+	// at this section of the code, there is 64*x+y (0<=y<64) bytes of
+	// buffer. The _fold_64_B_loop will fold 64B at a time
+	// until we have 64+y Bytes of buffer
+
+
+	// fold 64B at a time. This section of the code folds 4 vector
+	// registers in parallel
+_fold_64_B_loop:
+
+	.macro		fold64, reg1, reg2
+	vld1.64		{q11-q12}, [arg2, :128]!
+
+	vmull.p64	q8, \reg1\()h, d21
+	vmull.p64	\reg1, \reg1\()l, d20
+	vmull.p64	q9, \reg2\()h, d21
+	vmull.p64	\reg2, \reg2\()l, d20
+
+CPU_LE(	vrev64.8	q11, q11		)
+CPU_LE(	vrev64.8	q12, q12		)
+	vswp		d22, d23
+	vswp		d24, d25
+
+	veor.8		\reg1, \reg1, q8
+	veor.8		\reg2, \reg2, q9
+	veor.8		\reg1, \reg1, q11
+	veor.8		\reg2, \reg2, q12
+	.endm
+
+	fold64		q0, q1
+	fold64		q2, q3
+	fold64		q4, q5
+	fold64		q6, q7
+
+	subs		arg3, arg3, #128
+
+	// check if there is another 64B in the buffer to be able to fold
+	bge		_fold_64_B_loop
+
+	// at this point, the buffer pointer is pointing at the last y Bytes
+	// of the buffer the 64B of folded data is in 4 of the vector
+	// registers: v0, v1, v2, v3
+
+	// fold the 8 vector registers to 1 vector register with different
+	// constants
+
+	adr		ip, rk9
+	vld1.64		{q10}, [ip, :128]!
+
+	.macro		fold16, reg, rk
+	vmull.p64	q8, \reg\()l, d20
+	vmull.p64	\reg, \reg\()h, d21
+	.ifnb		\rk
+	vld1.64		{q10}, [ip, :128]!
+	.endif
+	veor.8		q7, q7, q8
+	veor.8		q7, q7, \reg
+	.endm
+
+	fold16		q0, rk11
+	fold16		q1, rk13
+	fold16		q2, rk15
+	fold16		q3, rk17
+	fold16		q4, rk19
+	fold16		q5, rk1
+	fold16		q6
+
+	// instead of 64, we add 48 to the loop counter to save 1 instruction
+	// from the loop instead of a cmp instruction, we use the negative
+	// flag with the jl instruction
+	adds		arg3, arg3, #(128-16)
+	blt		_final_reduction_for_128
+
+	// now we have 16+y bytes left to reduce. 16 Bytes is in register v7
+	// and the rest is in memory. We can fold 16 bytes at a time if y>=16
+	// continue folding 16B at a time
+
+_16B_reduction_loop:
+	vmull.p64	q8, d14, d20
+	vmull.p64	q7, d15, d21
+	veor.8		q7, q7, q8
+
+	vld1.64		{q0}, [arg2, :128]!
+CPU_LE(	vrev64.8	q0, q0		)
+	vswp		d0, d1
+	veor.8		q7, q7, q0
+	subs		arg3, arg3, #16
+
+	// instead of a cmp instruction, we utilize the flags with the
+	// jge instruction equivalent of: cmp arg3, 16-16
+	// check if there is any more 16B in the buffer to be able to fold
+	bge		_16B_reduction_loop
+
+	// now we have 16+z bytes left to reduce, where 0<= z < 16.
+	// first, we reduce the data in the xmm7 register
+
+_final_reduction_for_128:
+	// check if any more data to fold. If not, compute the CRC of
+	// the final 128 bits
+	adds		arg3, arg3, #16
+	beq		_128_done
+
+	// here we are getting data that is less than 16 bytes.
+	// since we know that there was data before the pointer, we can
+	// offset the input pointer before the actual point, to receive
+	// exactly 16 bytes. after that the registers need to be adjusted.
+_get_last_two_regs:
+	add		arg2, arg2, arg3
+	sub		arg2, arg2, #16
+	vld1.64		{q1}, [arg2]
+CPU_LE(	vrev64.8	q1, q1			)
+	vswp		d2, d3
+
+	// get rid of the extra data that was loaded before
+	// load the shift constant
+	adr		ip, tbl_shf_table + 16
+	sub		ip, ip, arg3
+	vld1.8		{q0}, [ip]
+
+	// shift v2 to the left by arg3 bytes
+	vtbl.8		d4, {d14-d15}, d0
+	vtbl.8		d5, {d14-d15}, d1
+
+	// shift v7 to the right by 16-arg3 bytes
+	vmov.i8		q9, #0x80
+	veor.8		q0, q0, q9
+	vtbl.8		d18, {d14-d15}, d0
+	vtbl.8		d19, {d14-d15}, d1
+
+	// blend
+	vshr.s8		q0, q0, #7		// convert to 8-bit mask
+	vbsl.8		q0, q2, q1
+
+	// fold 16 Bytes
+	vmull.p64	q8, d18, d20
+	vmull.p64	q7, d19, d21
+	veor.8		q7, q7, q8
+	veor.8		q7, q7, q0
+
+_128_done:
+	// compute crc of a 128-bit value
+	vldr		d20, rk5
+	vldr		d21, rk6		// rk5 and rk6 in xmm10
+
+	// 64b fold
+	vext.8		q0, qzr, q7, #8
+	vmull.p64	q7, d15, d20
+	veor.8		q7, q7, q0
+
+	// 32b fold
+	vext.8		q0, q7, qzr, #12
+	vmov		s31, s3
+	vmull.p64	q0, d0, d21
+	veor.8		q7, q0, q7
+
+	// barrett reduction
+_barrett:
+	vldr		d20, rk7
+	vldr		d21, rk8
+
+	vmull.p64	q0, d15, d20
+	vext.8		q0, qzr, q0, #12
+	vmull.p64	q0, d1, d21
+	vext.8		q0, qzr, q0, #12
+	veor.8		q7, q7, q0
+	vmov		r0, s29
+
+_cleanup:
+	// scale the result back to 16 bits
+	lsr		r0, r0, #16
+	bx		lr
+
+_less_than_128:
+	teq		arg3, #0
+	beq		_cleanup
+
+	vmov.i8		q0, #0
+	vmov		s3, arg1_low32		// get the initial crc value
+
+	vld1.64		{q7}, [arg2, :128]!
+CPU_LE(	vrev64.8	q7, q7		)
+	vswp		d14, d15
+	veor.8		q7, q7, q0
+
+	cmp		arg3, #16
+	beq		_128_done		// exactly 16 left
+	blt		_less_than_16_left
+
+	// now if there is, load the constants
+	vldr		d20, rk1
+	vldr		d21, rk2		// rk1 and rk2 in xmm10
+
+	// check if there is enough buffer to be able to fold 16B at a time
+	subs		arg3, arg3, #32
+	addlt		arg3, arg3, #16
+	blt		_get_last_two_regs
+	b		_16B_reduction_loop
+
+_less_than_16_left:
+	// shl r9, 4
+	adr		ip, tbl_shf_table + 16
+	sub		ip, ip, arg3
+	vld1.8		{q0}, [ip]
+	vmov.i8		q9, #0x80
+	veor.8		q0, q0, q9
+	vtbl.8		d18, {d14-d15}, d0
+	vtbl.8		d15, {d14-d15}, d1
+	vmov		d14, d18
+	b		_128_done
+ENDPROC(crc_t10dif_pmull)
+
+// precomputed constants
+// these constants are precomputed from the poly:
+// 0x8bb70000 (0x8bb7 scaled to 32 bits)
+	.align		4
+// Q = 0x18BB70000
+// rk1 = 2^(32*3) mod Q << 32
+// rk2 = 2^(32*5) mod Q << 32
+// rk3 = 2^(32*15) mod Q << 32
+// rk4 = 2^(32*17) mod Q << 32
+// rk5 = 2^(32*3) mod Q << 32
+// rk6 = 2^(32*2) mod Q << 32
+// rk7 = floor(2^64/Q)
+// rk8 = Q
+
+rk3:	.quad		0x9d9d000000000000
+rk4:	.quad		0x7cf5000000000000
+rk5:	.quad		0x2d56000000000000
+rk6:	.quad		0x1368000000000000
+rk7:	.quad		0x00000001f65a57f8
+rk8:	.quad		0x000000018bb70000
+rk9:	.quad		0xceae000000000000
+rk10:	.quad		0xbfd6000000000000
+rk11:	.quad		0x1e16000000000000
+rk12:	.quad		0x713c000000000000
+rk13:	.quad		0xf7f9000000000000
+rk14:	.quad		0x80a6000000000000
+rk15:	.quad		0x044c000000000000
+rk16:	.quad		0xe658000000000000
+rk17:	.quad		0xad18000000000000
+rk18:	.quad		0xa497000000000000
+rk19:	.quad		0x6ee3000000000000
+rk20:	.quad		0xe7b5000000000000
+rk1:	.quad		0x2d56000000000000
+rk2:	.quad		0x06df000000000000
+
+tbl_shf_table:
+// use these values for shift constants for the tbl/tbx instruction
+// different alignments result in values as shown:
+//	DDQ 0x008f8e8d8c8b8a898887868584838281 # shl 15 (16-1) / shr1
+//	DDQ 0x01008f8e8d8c8b8a8988878685848382 # shl 14 (16-3) / shr2
+//	DDQ 0x0201008f8e8d8c8b8a89888786858483 # shl 13 (16-4) / shr3
+//	DDQ 0x030201008f8e8d8c8b8a898887868584 # shl 12 (16-4) / shr4
+//	DDQ 0x04030201008f8e8d8c8b8a8988878685 # shl 11 (16-5) / shr5
+//	DDQ 0x0504030201008f8e8d8c8b8a89888786 # shl 10 (16-6) / shr6
+//	DDQ 0x060504030201008f8e8d8c8b8a898887 # shl 9  (16-7) / shr7
+//	DDQ 0x07060504030201008f8e8d8c8b8a8988 # shl 8  (16-8) / shr8
+//	DDQ 0x0807060504030201008f8e8d8c8b8a89 # shl 7  (16-9) / shr9
+//	DDQ 0x090807060504030201008f8e8d8c8b8a # shl 6  (16-10) / shr10
+//	DDQ 0x0a090807060504030201008f8e8d8c8b # shl 5  (16-11) / shr11
+//	DDQ 0x0b0a090807060504030201008f8e8d8c # shl 4  (16-12) / shr12
+//	DDQ 0x0c0b0a090807060504030201008f8e8d # shl 3  (16-13) / shr13
+//	DDQ 0x0d0c0b0a090807060504030201008f8e # shl 2  (16-14) / shr14
+//	DDQ 0x0e0d0c0b0a090807060504030201008f # shl 1  (16-15) / shr15
+
+	.byte		 0x0, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87
+	.byte		0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f
+	.byte		 0x0,  0x1,  0x2,  0x3,  0x4,  0x5,  0x6,  0x7
+	.byte		 0x8,  0x9,  0xa,  0xb,  0xc,  0xd,  0xe , 0x0
diff --git a/arch/arm/crypto/crct10dif-ce-glue.c b/arch/arm/crypto/crct10dif-ce-glue.c
new file mode 100644
index 000000000000..d428355cf38d
--- /dev/null
+++ b/arch/arm/crypto/crct10dif-ce-glue.c
@@ -0,0 +1,101 @@
+/*
+ * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/crc-t10dif.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h>
+
+#include <crypto/internal/hash.h>
+
+#include <asm/neon.h>
+#include <asm/simd.h>
+
+#define CRC_T10DIF_PMULL_CHUNK_SIZE	16U
+
+asmlinkage u16 crc_t10dif_pmull(u16 init_crc, const u8 buf[], u32 len);
+
+static int crct10dif_init(struct shash_desc *desc)
+{
+	u16 *crc = shash_desc_ctx(desc);
+
+	*crc = 0;
+	return 0;
+}
+
+static int crct10dif_update(struct shash_desc *desc, const u8 *data,
+			    unsigned int length)
+{
+	u16 *crc = shash_desc_ctx(desc);
+	unsigned int l;
+
+	if (!may_use_simd()) {
+		*crc = crc_t10dif_generic(*crc, data, length);
+	} else {
+		if (unlikely((u32)data % CRC_T10DIF_PMULL_CHUNK_SIZE)) {
+			l = min_t(u32, length, CRC_T10DIF_PMULL_CHUNK_SIZE -
+				  ((u32)data % CRC_T10DIF_PMULL_CHUNK_SIZE));
+
+			*crc = crc_t10dif_generic(*crc, data, l);
+
+			length -= l;
+			data += l;
+		}
+		if (length > 0) {
+			kernel_neon_begin();
+			*crc = crc_t10dif_pmull(*crc, data, length);
+			kernel_neon_end();
+		}
+	}
+	return 0;
+}
+
+static int crct10dif_final(struct shash_desc *desc, u8 *out)
+{
+	u16 *crc = shash_desc_ctx(desc);
+
+	*(u16 *)out = *crc;
+	return 0;
+}
+
+static struct shash_alg crc_t10dif_alg = {
+	.digestsize		= CRC_T10DIF_DIGEST_SIZE,
+	.init			= crct10dif_init,
+	.update			= crct10dif_update,
+	.final			= crct10dif_final,
+	.descsize		= CRC_T10DIF_DIGEST_SIZE,
+
+	.base.cra_name		= "crct10dif",
+	.base.cra_driver_name	= "crct10dif-arm-ce",
+	.base.cra_priority	= 200,
+	.base.cra_blocksize	= CRC_T10DIF_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+};
+
+static int __init crc_t10dif_mod_init(void)
+{
+	if (!(elf_hwcap2 & HWCAP2_PMULL))
+		return -ENODEV;
+
+	return crypto_register_shash(&crc_t10dif_alg);
+}
+
+static void __exit crc_t10dif_mod_exit(void)
+{
+	crypto_unregister_shash(&crc_t10dif_alg);
+}
+
+module_init(crc_t10dif_mod_init);
+module_exit(crc_t10dif_mod_exit);
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("crct10dif");
diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild
index 55e0e3ea9cb6..efb21757d41f 100644
--- a/arch/arm/include/asm/Kbuild
+++ b/arch/arm/include/asm/Kbuild
@@ -38,3 +38,6 @@ generic-y += termios.h
 generic-y += timex.h
 generic-y += trace_clock.h
 generic-y += unaligned.h
+
+generated-y += mach-types.h
+generated-y += unistd-nr.h
diff --git a/arch/arm/include/asm/delay.h b/arch/arm/include/asm/delay.h
index b1ce037e4380..e986b7f717c4 100644
--- a/arch/arm/include/asm/delay.h
+++ b/arch/arm/include/asm/delay.h
@@ -9,6 +9,33 @@
 #include <asm/memory.h>
 #include <asm/param.h>	/* HZ */
 
+/*
+ * Loop (or tick) based delay:
+ *
+ * loops = loops_per_jiffy * jiffies_per_sec * delay_us / us_per_sec
+ *
+ * where:
+ *
+ * jiffies_per_sec = HZ
+ * us_per_sec = 1000000
+ *
+ * Therefore the constant part is HZ / 1000000 which is a small
+ * fractional number. To make this usable with integer math, we
+ * scale up this constant by 2^31, perform the actual multiplication,
+ * and scale the result back down by 2^31 with a simple shift:
+ *
+ * loops = (loops_per_jiffy * delay_us * UDELAY_MULT) >> 31
+ *
+ * where:
+ *
+ * UDELAY_MULT = 2^31 * HZ / 1000000
+ *             = (2^31 / 1000000) * HZ
+ *             = 2147.483648 * HZ
+ *             = 2147 * HZ + 483648 * HZ / 1000000
+ *
+ * 31 is the biggest scale shift value that won't overflow 32 bits for
+ * delay_us * UDELAY_MULT assuming HZ <= 1000 and delay_us <= 2000.
+ */
 #define MAX_UDELAY_MS	2
 #define UDELAY_MULT	UL(2147 * HZ + 483648 * HZ / 1000000)
 #define UDELAY_SHIFT	31
diff --git a/arch/arm/include/asm/io.h b/arch/arm/include/asm/io.h
index 021692c64de3..42871fb8340e 100644
--- a/arch/arm/include/asm/io.h
+++ b/arch/arm/include/asm/io.h
@@ -25,7 +25,6 @@
 
 #include <linux/string.h>
 #include <linux/types.h>
-#include <linux/blk_types.h>
 #include <asm/byteorder.h>
 #include <asm/memory.h>
 #include <asm-generic/pci_iomap.h>
diff --git a/arch/arm/include/asm/mach-types.h b/arch/arm/include/asm/mach-types.h
deleted file mode 100644
index 948178cc6ba8..000000000000
--- a/arch/arm/include/asm/mach-types.h
+++ /dev/null
@@ -1 +0,0 @@
-#include <generated/mach-types.h>
diff --git a/arch/arm/include/asm/unistd.h b/arch/arm/include/asm/unistd.h
index ada0d29a660f..076090d2dbf5 100644
--- a/arch/arm/include/asm/unistd.h
+++ b/arch/arm/include/asm/unistd.h
@@ -14,12 +14,7 @@
 #define __ASM_ARM_UNISTD_H
 
 #include <uapi/asm/unistd.h>
-
-/*
- * This may need to be greater than __NR_last_syscall+1 in order to
- * account for the padding in the syscall table
- */
-#define __NR_syscalls  (400)
+#include <asm/unistd-nr.h>
 
 #define __ARCH_WANT_STAT64
 #define __ARCH_WANT_SYS_GETHOSTNAME
@@ -52,4 +47,23 @@
 #define __IGNORE_fadvise64_64
 #define __IGNORE_migrate_pages
 
+#ifdef __ARM_EABI__
+/*
+ * The following syscalls are obsolete and no longer available for EABI:
+ *  __NR_time
+ *  __NR_umount
+ *  __NR_stime
+ *  __NR_alarm
+ *  __NR_utime
+ *  __NR_getrlimit
+ *  __NR_select
+ *  __NR_readdir
+ *  __NR_mmap
+ *  __NR_socketcall
+ *  __NR_syscall
+ *  __NR_ipc
+ */
+#define __IGNORE_getrlimit
+#endif
+
 #endif /* __ASM_ARM_UNISTD_H */
diff --git a/arch/arm/include/asm/xen/hypercall.h b/arch/arm/include/asm/xen/hypercall.h
index 9d874db13c0e..3522cbaed316 100644
--- a/arch/arm/include/asm/xen/hypercall.h
+++ b/arch/arm/include/asm/xen/hypercall.h
@@ -1,87 +1 @@
-/******************************************************************************
- * hypercall.h
- *
- * Linux-specific hypervisor handling.
- *
- * Stefano Stabellini <stefano.stabellini@eu.citrix.com>, Citrix, 2012
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version 2
- * as published by the Free Software Foundation; or, when distributed
- * separately from the Linux kernel or incorporated into other
- * software packages, subject to the following license:
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this source file (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use, copy, modify,
- * merge, publish, distribute, sublicense, and/or sell copies of the Software,
- * and to permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-#ifndef _ASM_ARM_XEN_HYPERCALL_H
-#define _ASM_ARM_XEN_HYPERCALL_H
-
-#include <linux/bug.h>
-
-#include <xen/interface/xen.h>
-#include <xen/interface/sched.h>
-#include <xen/interface/platform.h>
-
-long privcmd_call(unsigned call, unsigned long a1,
-		unsigned long a2, unsigned long a3,
-		unsigned long a4, unsigned long a5);
-int HYPERVISOR_xen_version(int cmd, void *arg);
-int HYPERVISOR_console_io(int cmd, int count, char *str);
-int HYPERVISOR_grant_table_op(unsigned int cmd, void *uop, unsigned int count);
-int HYPERVISOR_sched_op(int cmd, void *arg);
-int HYPERVISOR_event_channel_op(int cmd, void *arg);
-unsigned long HYPERVISOR_hvm_op(int op, void *arg);
-int HYPERVISOR_memory_op(unsigned int cmd, void *arg);
-int HYPERVISOR_physdev_op(int cmd, void *arg);
-int HYPERVISOR_vcpu_op(int cmd, int vcpuid, void *extra_args);
-int HYPERVISOR_tmem_op(void *arg);
-int HYPERVISOR_vm_assist(unsigned int cmd, unsigned int type);
-int HYPERVISOR_platform_op_raw(void *arg);
-static inline int HYPERVISOR_platform_op(struct xen_platform_op *op)
-{
-	op->interface_version = XENPF_INTERFACE_VERSION;
-	return HYPERVISOR_platform_op_raw(op);
-}
-int HYPERVISOR_multicall(struct multicall_entry *calls, uint32_t nr);
-
-static inline int
-HYPERVISOR_suspend(unsigned long start_info_mfn)
-{
-	struct sched_shutdown r = { .reason = SHUTDOWN_suspend };
-
-	/* start_info_mfn is unused on ARM */
-	return HYPERVISOR_sched_op(SCHEDOP_shutdown, &r);
-}
-
-static inline void
-MULTI_update_va_mapping(struct multicall_entry *mcl, unsigned long va,
-			unsigned int new_val, unsigned long flags)
-{
-	BUG();
-}
-
-static inline void
-MULTI_mmu_update(struct multicall_entry *mcl, struct mmu_update *req,
-		 int count, int *success_count, domid_t domid)
-{
-	BUG();
-}
-
-#endif /* _ASM_ARM_XEN_HYPERCALL_H */
+#include <xen/arm/hypercall.h>
diff --git a/arch/arm/include/asm/xen/hypervisor.h b/arch/arm/include/asm/xen/hypervisor.h
index 95251512e2c4..d6e7709d0688 100644
--- a/arch/arm/include/asm/xen/hypervisor.h
+++ b/arch/arm/include/asm/xen/hypervisor.h
@@ -1,39 +1 @@
-#ifndef _ASM_ARM_XEN_HYPERVISOR_H
-#define _ASM_ARM_XEN_HYPERVISOR_H
-
-#include <linux/init.h>
-
-extern struct shared_info *HYPERVISOR_shared_info;
-extern struct start_info *xen_start_info;
-
-/* Lazy mode for batching updates / context switch */
-enum paravirt_lazy_mode {
-	PARAVIRT_LAZY_NONE,
-	PARAVIRT_LAZY_MMU,
-	PARAVIRT_LAZY_CPU,
-};
-
-static inline enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
-{
-	return PARAVIRT_LAZY_NONE;
-}
-
-extern struct dma_map_ops *xen_dma_ops;
-
-#ifdef CONFIG_XEN
-void __init xen_early_init(void);
-#else
-static inline void xen_early_init(void) { return; }
-#endif
-
-#ifdef CONFIG_HOTPLUG_CPU
-static inline void xen_arch_register_cpu(int num)
-{
-}
-
-static inline void xen_arch_unregister_cpu(int num)
-{
-}
-#endif
-
-#endif /* _ASM_ARM_XEN_HYPERVISOR_H */
+#include <xen/arm/hypervisor.h>
diff --git a/arch/arm/include/asm/xen/interface.h b/arch/arm/include/asm/xen/interface.h
index 75d596862892..88c0d75da190 100644
--- a/arch/arm/include/asm/xen/interface.h
+++ b/arch/arm/include/asm/xen/interface.h
@@ -1,85 +1 @@
-/******************************************************************************
- * Guest OS interface to ARM Xen.
- *
- * Stefano Stabellini <stefano.stabellini@eu.citrix.com>, Citrix, 2012
- */
-
-#ifndef _ASM_ARM_XEN_INTERFACE_H
-#define _ASM_ARM_XEN_INTERFACE_H
-
-#include <linux/types.h>
-
-#define uint64_aligned_t uint64_t __attribute__((aligned(8)))
-
-#define __DEFINE_GUEST_HANDLE(name, type) \
-	typedef struct { union { type *p; uint64_aligned_t q; }; }  \
-        __guest_handle_ ## name
-
-#define DEFINE_GUEST_HANDLE_STRUCT(name) \
-	__DEFINE_GUEST_HANDLE(name, struct name)
-#define DEFINE_GUEST_HANDLE(name) __DEFINE_GUEST_HANDLE(name, name)
-#define GUEST_HANDLE(name)        __guest_handle_ ## name
-
-#define set_xen_guest_handle(hnd, val)			\
-	do {						\
-		if (sizeof(hnd) == 8)			\
-			*(uint64_t *)&(hnd) = 0;	\
-		(hnd).p = val;				\
-	} while (0)
-
-#define __HYPERVISOR_platform_op_raw __HYPERVISOR_platform_op
-
-#ifndef __ASSEMBLY__
-/* Explicitly size integers that represent pfns in the interface with
- * Xen so that we can have one ABI that works for 32 and 64 bit guests.
- * Note that this means that the xen_pfn_t type may be capable of
- * representing pfn's which the guest cannot represent in its own pfn
- * type. However since pfn space is controlled by the guest this is
- * fine since it simply wouldn't be able to create any sure pfns in
- * the first place.
- */
-typedef uint64_t xen_pfn_t;
-#define PRI_xen_pfn "llx"
-typedef uint64_t xen_ulong_t;
-#define PRI_xen_ulong "llx"
-typedef int64_t xen_long_t;
-#define PRI_xen_long "llx"
-/* Guest handles for primitive C types. */
-__DEFINE_GUEST_HANDLE(uchar, unsigned char);
-__DEFINE_GUEST_HANDLE(uint,  unsigned int);
-DEFINE_GUEST_HANDLE(char);
-DEFINE_GUEST_HANDLE(int);
-DEFINE_GUEST_HANDLE(void);
-DEFINE_GUEST_HANDLE(uint64_t);
-DEFINE_GUEST_HANDLE(uint32_t);
-DEFINE_GUEST_HANDLE(xen_pfn_t);
-DEFINE_GUEST_HANDLE(xen_ulong_t);
-
-/* Maximum number of virtual CPUs in multi-processor guests. */
-#define MAX_VIRT_CPUS 1
-
-struct arch_vcpu_info { };
-struct arch_shared_info { };
-
-/* TODO: Move pvclock definitions some place arch independent */
-struct pvclock_vcpu_time_info {
-	u32   version;
-	u32   pad0;
-	u64   tsc_timestamp;
-	u64   system_time;
-	u32   tsc_to_system_mul;
-	s8    tsc_shift;
-	u8    flags;
-	u8    pad[2];
-} __attribute__((__packed__)); /* 32 bytes */
-
-/* It is OK to have a 12 bytes struct with no padding because it is packed */
-struct pvclock_wall_clock {
-	u32   version;
-	u32   sec;
-	u32   nsec;
-	u32   sec_hi;
-} __attribute__((__packed__));
-#endif
-
-#endif /* _ASM_ARM_XEN_INTERFACE_H */
+#include <xen/arm/interface.h>
diff --git a/arch/arm/include/asm/xen/page-coherent.h b/arch/arm/include/asm/xen/page-coherent.h
index 95ce6ac3a971..b3ef061d8b74 100644
--- a/arch/arm/include/asm/xen/page-coherent.h
+++ b/arch/arm/include/asm/xen/page-coherent.h
@@ -1,98 +1 @@
-#ifndef _ASM_ARM_XEN_PAGE_COHERENT_H
-#define _ASM_ARM_XEN_PAGE_COHERENT_H
-
-#include <asm/page.h>
-#include <linux/dma-mapping.h>
-
-void __xen_dma_map_page(struct device *hwdev, struct page *page,
-	     dma_addr_t dev_addr, unsigned long offset, size_t size,
-	     enum dma_data_direction dir, unsigned long attrs);
-void __xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle,
-		size_t size, enum dma_data_direction dir,
-		unsigned long attrs);
-void __xen_dma_sync_single_for_cpu(struct device *hwdev,
-		dma_addr_t handle, size_t size, enum dma_data_direction dir);
-
-void __xen_dma_sync_single_for_device(struct device *hwdev,
-		dma_addr_t handle, size_t size, enum dma_data_direction dir);
-
-static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size,
-		dma_addr_t *dma_handle, gfp_t flags, unsigned long attrs)
-{
-	return __generic_dma_ops(hwdev)->alloc(hwdev, size, dma_handle, flags, attrs);
-}
-
-static inline void xen_free_coherent_pages(struct device *hwdev, size_t size,
-		void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs)
-{
-	__generic_dma_ops(hwdev)->free(hwdev, size, cpu_addr, dma_handle, attrs);
-}
-
-static inline void xen_dma_map_page(struct device *hwdev, struct page *page,
-	     dma_addr_t dev_addr, unsigned long offset, size_t size,
-	     enum dma_data_direction dir, unsigned long attrs)
-{
-	unsigned long page_pfn = page_to_xen_pfn(page);
-	unsigned long dev_pfn = XEN_PFN_DOWN(dev_addr);
-	unsigned long compound_pages =
-		(1<<compound_order(page)) * XEN_PFN_PER_PAGE;
-	bool local = (page_pfn <= dev_pfn) &&
-		(dev_pfn - page_pfn < compound_pages);
-
-	/*
-	 * Dom0 is mapped 1:1, while the Linux page can span across
-	 * multiple Xen pages, it's not possible for it to contain a
-	 * mix of local and foreign Xen pages. So if the first xen_pfn
-	 * == mfn the page is local otherwise it's a foreign page
-	 * grant-mapped in dom0. If the page is local we can safely
-	 * call the native dma_ops function, otherwise we call the xen
-	 * specific function.
-	 */
-	if (local)
-		__generic_dma_ops(hwdev)->map_page(hwdev, page, offset, size, dir, attrs);
-	else
-		__xen_dma_map_page(hwdev, page, dev_addr, offset, size, dir, attrs);
-}
-
-static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle,
-		size_t size, enum dma_data_direction dir, unsigned long attrs)
-{
-	unsigned long pfn = PFN_DOWN(handle);
-	/*
-	 * Dom0 is mapped 1:1, while the Linux page can be spanned accross
-	 * multiple Xen page, it's not possible to have a mix of local and
-	 * foreign Xen page. Dom0 is mapped 1:1, so calling pfn_valid on a
-	 * foreign mfn will always return false. If the page is local we can
-	 * safely call the native dma_ops function, otherwise we call the xen
-	 * specific function.
-	 */
-	if (pfn_valid(pfn)) {
-		if (__generic_dma_ops(hwdev)->unmap_page)
-			__generic_dma_ops(hwdev)->unmap_page(hwdev, handle, size, dir, attrs);
-	} else
-		__xen_dma_unmap_page(hwdev, handle, size, dir, attrs);
-}
-
-static inline void xen_dma_sync_single_for_cpu(struct device *hwdev,
-		dma_addr_t handle, size_t size, enum dma_data_direction dir)
-{
-	unsigned long pfn = PFN_DOWN(handle);
-	if (pfn_valid(pfn)) {
-		if (__generic_dma_ops(hwdev)->sync_single_for_cpu)
-			__generic_dma_ops(hwdev)->sync_single_for_cpu(hwdev, handle, size, dir);
-	} else
-		__xen_dma_sync_single_for_cpu(hwdev, handle, size, dir);
-}
-
-static inline void xen_dma_sync_single_for_device(struct device *hwdev,
-		dma_addr_t handle, size_t size, enum dma_data_direction dir)
-{
-	unsigned long pfn = PFN_DOWN(handle);
-	if (pfn_valid(pfn)) {
-		if (__generic_dma_ops(hwdev)->sync_single_for_device)
-			__generic_dma_ops(hwdev)->sync_single_for_device(hwdev, handle, size, dir);
-	} else
-		__xen_dma_sync_single_for_device(hwdev, handle, size, dir);
-}
-
-#endif /* _ASM_ARM_XEN_PAGE_COHERENT_H */
+#include <xen/arm/page-coherent.h>
diff --git a/arch/arm/include/asm/xen/page.h b/arch/arm/include/asm/xen/page.h
index 415dbc6e43fd..31bbc803cecb 100644
--- a/arch/arm/include/asm/xen/page.h
+++ b/arch/arm/include/asm/xen/page.h
@@ -1,122 +1 @@
-#ifndef _ASM_ARM_XEN_PAGE_H
-#define _ASM_ARM_XEN_PAGE_H
-
-#include <asm/page.h>
-#include <asm/pgtable.h>
-
-#include <linux/pfn.h>
-#include <linux/types.h>
-#include <linux/dma-mapping.h>
-
-#include <xen/xen.h>
-#include <xen/interface/grant_table.h>
-
-#define phys_to_machine_mapping_valid(pfn) (1)
-
-/* Xen machine address */
-typedef struct xmaddr {
-	phys_addr_t maddr;
-} xmaddr_t;
-
-/* Xen pseudo-physical address */
-typedef struct xpaddr {
-	phys_addr_t paddr;
-} xpaddr_t;
-
-#define XMADDR(x)	((xmaddr_t) { .maddr = (x) })
-#define XPADDR(x)	((xpaddr_t) { .paddr = (x) })
-
-#define INVALID_P2M_ENTRY      (~0UL)
-
-/*
- * The pseudo-physical frame (pfn) used in all the helpers is always based
- * on Xen page granularity (i.e 4KB).
- *
- * A Linux page may be split across multiple non-contiguous Xen page so we
- * have to keep track with frame based on 4KB page granularity.
- *
- * PV drivers should never make a direct usage of those helpers (particularly
- * pfn_to_gfn and gfn_to_pfn).
- */
-
-unsigned long __pfn_to_mfn(unsigned long pfn);
-extern struct rb_root phys_to_mach;
-
-/* Pseudo-physical <-> Guest conversion */
-static inline unsigned long pfn_to_gfn(unsigned long pfn)
-{
-	return pfn;
-}
-
-static inline unsigned long gfn_to_pfn(unsigned long gfn)
-{
-	return gfn;
-}
-
-/* Pseudo-physical <-> BUS conversion */
-static inline unsigned long pfn_to_bfn(unsigned long pfn)
-{
-	unsigned long mfn;
-
-	if (phys_to_mach.rb_node != NULL) {
-		mfn = __pfn_to_mfn(pfn);
-		if (mfn != INVALID_P2M_ENTRY)
-			return mfn;
-	}
-
-	return pfn;
-}
-
-static inline unsigned long bfn_to_pfn(unsigned long bfn)
-{
-	return bfn;
-}
-
-#define bfn_to_local_pfn(bfn)	bfn_to_pfn(bfn)
-
-/* VIRT <-> GUEST conversion */
-#define virt_to_gfn(v)		(pfn_to_gfn(virt_to_phys(v) >> XEN_PAGE_SHIFT))
-#define gfn_to_virt(m)		(__va(gfn_to_pfn(m) << XEN_PAGE_SHIFT))
-
-/* Only used in PV code. But ARM guests are always HVM. */
-static inline xmaddr_t arbitrary_virt_to_machine(void *vaddr)
-{
-	BUG();
-}
-
-/* TODO: this shouldn't be here but it is because the frontend drivers
- * are using it (its rolled in headers) even though we won't hit the code path.
- * So for right now just punt with this.
- */
-static inline pte_t *lookup_address(unsigned long address, unsigned int *level)
-{
-	BUG();
-	return NULL;
-}
-
-extern int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops,
-				   struct gnttab_map_grant_ref *kmap_ops,
-				   struct page **pages, unsigned int count);
-
-extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
-				     struct gnttab_unmap_grant_ref *kunmap_ops,
-				     struct page **pages, unsigned int count);
-
-bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn);
-bool __set_phys_to_machine_multi(unsigned long pfn, unsigned long mfn,
-		unsigned long nr_pages);
-
-static inline bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)
-{
-	return __set_phys_to_machine(pfn, mfn);
-}
-
-#define xen_remap(cookie, size) ioremap_cache((cookie), (size))
-#define xen_unmap(cookie) iounmap((cookie))
-
-bool xen_arch_need_swiotlb(struct device *dev,
-			   phys_addr_t phys,
-			   dma_addr_t dev_addr);
-unsigned long xen_get_swiotlb_free_pages(unsigned int order);
-
-#endif /* _ASM_ARM_XEN_PAGE_H */
+#include <xen/arm/page.h>
diff --git a/arch/arm/include/uapi/asm/Kbuild b/arch/arm/include/uapi/asm/Kbuild
index a1c05f93d920..46a76cd6acb6 100644
--- a/arch/arm/include/uapi/asm/Kbuild
+++ b/arch/arm/include/uapi/asm/Kbuild
@@ -18,3 +18,6 @@ header-y += stat.h
 header-y += statfs.h
 header-y += swab.h
 header-y += unistd.h
+genhdr-y += unistd-common.h
+genhdr-y += unistd-oabi.h
+genhdr-y += unistd-eabi.h
diff --git a/arch/arm/include/uapi/asm/kvm.h b/arch/arm/include/uapi/asm/kvm.h
index b38c10c73579..af05f8e0903e 100644
--- a/arch/arm/include/uapi/asm/kvm.h
+++ b/arch/arm/include/uapi/asm/kvm.h
@@ -87,9 +87,11 @@ struct kvm_regs {
 /* Supported VGICv3 address types  */
 #define KVM_VGIC_V3_ADDR_TYPE_DIST	2
 #define KVM_VGIC_V3_ADDR_TYPE_REDIST	3
+#define KVM_VGIC_ITS_ADDR_TYPE		4
 
 #define KVM_VGIC_V3_DIST_SIZE		SZ_64K
 #define KVM_VGIC_V3_REDIST_SIZE		(2 * SZ_64K)
+#define KVM_VGIC_V3_ITS_SIZE		(2 * SZ_64K)
 
 #define KVM_ARM_VCPU_POWER_OFF		0 /* CPU is started in OFF state */
 #define KVM_ARM_VCPU_PSCI_0_2		1 /* CPU uses PSCI v0.2 */
diff --git a/arch/arm/include/uapi/asm/unistd.h b/arch/arm/include/uapi/asm/unistd.h
index 314100a06ccb..28bd456494a3 100644
--- a/arch/arm/include/uapi/asm/unistd.h
+++ b/arch/arm/include/uapi/asm/unistd.h
@@ -17,412 +17,14 @@
 
 #if defined(__thumb__) || defined(__ARM_EABI__)
 #define __NR_SYSCALL_BASE	0
+#include <asm/unistd-eabi.h>
 #else
 #define __NR_SYSCALL_BASE	__NR_OABI_SYSCALL_BASE
+#include <asm/unistd-oabi.h>
 #endif
 
-/*
- * This file contains the system call numbers.
- */
-
-#define __NR_restart_syscall		(__NR_SYSCALL_BASE+  0)
-#define __NR_exit			(__NR_SYSCALL_BASE+  1)
-#define __NR_fork			(__NR_SYSCALL_BASE+  2)
-#define __NR_read			(__NR_SYSCALL_BASE+  3)
-#define __NR_write			(__NR_SYSCALL_BASE+  4)
-#define __NR_open			(__NR_SYSCALL_BASE+  5)
-#define __NR_close			(__NR_SYSCALL_BASE+  6)
-					/* 7 was sys_waitpid */
-#define __NR_creat			(__NR_SYSCALL_BASE+  8)
-#define __NR_link			(__NR_SYSCALL_BASE+  9)
-#define __NR_unlink			(__NR_SYSCALL_BASE+ 10)
-#define __NR_execve			(__NR_SYSCALL_BASE+ 11)
-#define __NR_chdir			(__NR_SYSCALL_BASE+ 12)
-#define __NR_time			(__NR_SYSCALL_BASE+ 13)
-#define __NR_mknod			(__NR_SYSCALL_BASE+ 14)
-#define __NR_chmod			(__NR_SYSCALL_BASE+ 15)
-#define __NR_lchown			(__NR_SYSCALL_BASE+ 16)
-					/* 17 was sys_break */
-					/* 18 was sys_stat */
-#define __NR_lseek			(__NR_SYSCALL_BASE+ 19)
-#define __NR_getpid			(__NR_SYSCALL_BASE+ 20)
-#define __NR_mount			(__NR_SYSCALL_BASE+ 21)
-#define __NR_umount			(__NR_SYSCALL_BASE+ 22)
-#define __NR_setuid			(__NR_SYSCALL_BASE+ 23)
-#define __NR_getuid			(__NR_SYSCALL_BASE+ 24)
-#define __NR_stime			(__NR_SYSCALL_BASE+ 25)
-#define __NR_ptrace			(__NR_SYSCALL_BASE+ 26)
-#define __NR_alarm			(__NR_SYSCALL_BASE+ 27)
-					/* 28 was sys_fstat */
-#define __NR_pause			(__NR_SYSCALL_BASE+ 29)
-#define __NR_utime			(__NR_SYSCALL_BASE+ 30)
-					/* 31 was sys_stty */
-					/* 32 was sys_gtty */
-#define __NR_access			(__NR_SYSCALL_BASE+ 33)
-#define __NR_nice			(__NR_SYSCALL_BASE+ 34)
-					/* 35 was sys_ftime */
-#define __NR_sync			(__NR_SYSCALL_BASE+ 36)
-#define __NR_kill			(__NR_SYSCALL_BASE+ 37)
-#define __NR_rename			(__NR_SYSCALL_BASE+ 38)
-#define __NR_mkdir			(__NR_SYSCALL_BASE+ 39)
-#define __NR_rmdir			(__NR_SYSCALL_BASE+ 40)
-#define __NR_dup			(__NR_SYSCALL_BASE+ 41)
-#define __NR_pipe			(__NR_SYSCALL_BASE+ 42)
-#define __NR_times			(__NR_SYSCALL_BASE+ 43)
-					/* 44 was sys_prof */
-#define __NR_brk			(__NR_SYSCALL_BASE+ 45)
-#define __NR_setgid			(__NR_SYSCALL_BASE+ 46)
-#define __NR_getgid			(__NR_SYSCALL_BASE+ 47)
-					/* 48 was sys_signal */
-#define __NR_geteuid			(__NR_SYSCALL_BASE+ 49)
-#define __NR_getegid			(__NR_SYSCALL_BASE+ 50)
-#define __NR_acct			(__NR_SYSCALL_BASE+ 51)
-#define __NR_umount2			(__NR_SYSCALL_BASE+ 52)
-					/* 53 was sys_lock */
-#define __NR_ioctl			(__NR_SYSCALL_BASE+ 54)
-#define __NR_fcntl			(__NR_SYSCALL_BASE+ 55)
-					/* 56 was sys_mpx */
-#define __NR_setpgid			(__NR_SYSCALL_BASE+ 57)
-					/* 58 was sys_ulimit */
-					/* 59 was sys_olduname */
-#define __NR_umask			(__NR_SYSCALL_BASE+ 60)
-#define __NR_chroot			(__NR_SYSCALL_BASE+ 61)
-#define __NR_ustat			(__NR_SYSCALL_BASE+ 62)
-#define __NR_dup2			(__NR_SYSCALL_BASE+ 63)
-#define __NR_getppid			(__NR_SYSCALL_BASE+ 64)
-#define __NR_getpgrp			(__NR_SYSCALL_BASE+ 65)
-#define __NR_setsid			(__NR_SYSCALL_BASE+ 66)
-#define __NR_sigaction			(__NR_SYSCALL_BASE+ 67)
-					/* 68 was sys_sgetmask */
-					/* 69 was sys_ssetmask */
-#define __NR_setreuid			(__NR_SYSCALL_BASE+ 70)
-#define __NR_setregid			(__NR_SYSCALL_BASE+ 71)
-#define __NR_sigsuspend			(__NR_SYSCALL_BASE+ 72)
-#define __NR_sigpending			(__NR_SYSCALL_BASE+ 73)
-#define __NR_sethostname		(__NR_SYSCALL_BASE+ 74)
-#define __NR_setrlimit			(__NR_SYSCALL_BASE+ 75)
-#define __NR_getrlimit			(__NR_SYSCALL_BASE+ 76)	/* Back compat 2GB limited rlimit */
-#define __NR_getrusage			(__NR_SYSCALL_BASE+ 77)
-#define __NR_gettimeofday		(__NR_SYSCALL_BASE+ 78)
-#define __NR_settimeofday		(__NR_SYSCALL_BASE+ 79)
-#define __NR_getgroups			(__NR_SYSCALL_BASE+ 80)
-#define __NR_setgroups			(__NR_SYSCALL_BASE+ 81)
-#define __NR_select			(__NR_SYSCALL_BASE+ 82)
-#define __NR_symlink			(__NR_SYSCALL_BASE+ 83)
-					/* 84 was sys_lstat */
-#define __NR_readlink			(__NR_SYSCALL_BASE+ 85)
-#define __NR_uselib			(__NR_SYSCALL_BASE+ 86)
-#define __NR_swapon			(__NR_SYSCALL_BASE+ 87)
-#define __NR_reboot			(__NR_SYSCALL_BASE+ 88)
-#define __NR_readdir			(__NR_SYSCALL_BASE+ 89)
-#define __NR_mmap			(__NR_SYSCALL_BASE+ 90)
-#define __NR_munmap			(__NR_SYSCALL_BASE+ 91)
-#define __NR_truncate			(__NR_SYSCALL_BASE+ 92)
-#define __NR_ftruncate			(__NR_SYSCALL_BASE+ 93)
-#define __NR_fchmod			(__NR_SYSCALL_BASE+ 94)
-#define __NR_fchown			(__NR_SYSCALL_BASE+ 95)
-#define __NR_getpriority		(__NR_SYSCALL_BASE+ 96)
-#define __NR_setpriority		(__NR_SYSCALL_BASE+ 97)
-					/* 98 was sys_profil */
-#define __NR_statfs			(__NR_SYSCALL_BASE+ 99)
-#define __NR_fstatfs			(__NR_SYSCALL_BASE+100)
-					/* 101 was sys_ioperm */
-#define __NR_socketcall			(__NR_SYSCALL_BASE+102)
-#define __NR_syslog			(__NR_SYSCALL_BASE+103)
-#define __NR_setitimer			(__NR_SYSCALL_BASE+104)
-#define __NR_getitimer			(__NR_SYSCALL_BASE+105)
-#define __NR_stat			(__NR_SYSCALL_BASE+106)
-#define __NR_lstat			(__NR_SYSCALL_BASE+107)
-#define __NR_fstat			(__NR_SYSCALL_BASE+108)
-					/* 109 was sys_uname */
-					/* 110 was sys_iopl */
-#define __NR_vhangup			(__NR_SYSCALL_BASE+111)
-					/* 112 was sys_idle */
-#define __NR_syscall			(__NR_SYSCALL_BASE+113) /* syscall to call a syscall! */
-#define __NR_wait4			(__NR_SYSCALL_BASE+114)
-#define __NR_swapoff			(__NR_SYSCALL_BASE+115)
-#define __NR_sysinfo			(__NR_SYSCALL_BASE+116)
-#define __NR_ipc			(__NR_SYSCALL_BASE+117)
-#define __NR_fsync			(__NR_SYSCALL_BASE+118)
-#define __NR_sigreturn			(__NR_SYSCALL_BASE+119)
-#define __NR_clone			(__NR_SYSCALL_BASE+120)
-#define __NR_setdomainname		(__NR_SYSCALL_BASE+121)
-#define __NR_uname			(__NR_SYSCALL_BASE+122)
-					/* 123 was sys_modify_ldt */
-#define __NR_adjtimex			(__NR_SYSCALL_BASE+124)
-#define __NR_mprotect			(__NR_SYSCALL_BASE+125)
-#define __NR_sigprocmask		(__NR_SYSCALL_BASE+126)
-					/* 127 was sys_create_module */
-#define __NR_init_module		(__NR_SYSCALL_BASE+128)
-#define __NR_delete_module		(__NR_SYSCALL_BASE+129)
-					/* 130 was sys_get_kernel_syms */
-#define __NR_quotactl			(__NR_SYSCALL_BASE+131)
-#define __NR_getpgid			(__NR_SYSCALL_BASE+132)
-#define __NR_fchdir			(__NR_SYSCALL_BASE+133)
-#define __NR_bdflush			(__NR_SYSCALL_BASE+134)
-#define __NR_sysfs			(__NR_SYSCALL_BASE+135)
-#define __NR_personality		(__NR_SYSCALL_BASE+136)
-					/* 137 was sys_afs_syscall */
-#define __NR_setfsuid			(__NR_SYSCALL_BASE+138)
-#define __NR_setfsgid			(__NR_SYSCALL_BASE+139)
-#define __NR__llseek			(__NR_SYSCALL_BASE+140)
-#define __NR_getdents			(__NR_SYSCALL_BASE+141)
-#define __NR__newselect			(__NR_SYSCALL_BASE+142)
-#define __NR_flock			(__NR_SYSCALL_BASE+143)
-#define __NR_msync			(__NR_SYSCALL_BASE+144)
-#define __NR_readv			(__NR_SYSCALL_BASE+145)
-#define __NR_writev			(__NR_SYSCALL_BASE+146)
-#define __NR_getsid			(__NR_SYSCALL_BASE+147)
-#define __NR_fdatasync			(__NR_SYSCALL_BASE+148)
-#define __NR__sysctl			(__NR_SYSCALL_BASE+149)
-#define __NR_mlock			(__NR_SYSCALL_BASE+150)
-#define __NR_munlock			(__NR_SYSCALL_BASE+151)
-#define __NR_mlockall			(__NR_SYSCALL_BASE+152)
-#define __NR_munlockall			(__NR_SYSCALL_BASE+153)
-#define __NR_sched_setparam		(__NR_SYSCALL_BASE+154)
-#define __NR_sched_getparam		(__NR_SYSCALL_BASE+155)
-#define __NR_sched_setscheduler		(__NR_SYSCALL_BASE+156)
-#define __NR_sched_getscheduler		(__NR_SYSCALL_BASE+157)
-#define __NR_sched_yield		(__NR_SYSCALL_BASE+158)
-#define __NR_sched_get_priority_max	(__NR_SYSCALL_BASE+159)
-#define __NR_sched_get_priority_min	(__NR_SYSCALL_BASE+160)
-#define __NR_sched_rr_get_interval	(__NR_SYSCALL_BASE+161)
-#define __NR_nanosleep			(__NR_SYSCALL_BASE+162)
-#define __NR_mremap			(__NR_SYSCALL_BASE+163)
-#define __NR_setresuid			(__NR_SYSCALL_BASE+164)
-#define __NR_getresuid			(__NR_SYSCALL_BASE+165)
-					/* 166 was sys_vm86 */
-					/* 167 was sys_query_module */
-#define __NR_poll			(__NR_SYSCALL_BASE+168)
-#define __NR_nfsservctl			(__NR_SYSCALL_BASE+169)
-#define __NR_setresgid			(__NR_SYSCALL_BASE+170)
-#define __NR_getresgid			(__NR_SYSCALL_BASE+171)
-#define __NR_prctl			(__NR_SYSCALL_BASE+172)
-#define __NR_rt_sigreturn		(__NR_SYSCALL_BASE+173)
-#define __NR_rt_sigaction		(__NR_SYSCALL_BASE+174)
-#define __NR_rt_sigprocmask		(__NR_SYSCALL_BASE+175)
-#define __NR_rt_sigpending		(__NR_SYSCALL_BASE+176)
-#define __NR_rt_sigtimedwait		(__NR_SYSCALL_BASE+177)
-#define __NR_rt_sigqueueinfo		(__NR_SYSCALL_BASE+178)
-#define __NR_rt_sigsuspend		(__NR_SYSCALL_BASE+179)
-#define __NR_pread64			(__NR_SYSCALL_BASE+180)
-#define __NR_pwrite64			(__NR_SYSCALL_BASE+181)
-#define __NR_chown			(__NR_SYSCALL_BASE+182)
-#define __NR_getcwd			(__NR_SYSCALL_BASE+183)
-#define __NR_capget			(__NR_SYSCALL_BASE+184)
-#define __NR_capset			(__NR_SYSCALL_BASE+185)
-#define __NR_sigaltstack		(__NR_SYSCALL_BASE+186)
-#define __NR_sendfile			(__NR_SYSCALL_BASE+187)
-					/* 188 reserved */
-					/* 189 reserved */
-#define __NR_vfork			(__NR_SYSCALL_BASE+190)
-#define __NR_ugetrlimit			(__NR_SYSCALL_BASE+191)	/* SuS compliant getrlimit */
-#define __NR_mmap2			(__NR_SYSCALL_BASE+192)
-#define __NR_truncate64			(__NR_SYSCALL_BASE+193)
-#define __NR_ftruncate64		(__NR_SYSCALL_BASE+194)
-#define __NR_stat64			(__NR_SYSCALL_BASE+195)
-#define __NR_lstat64			(__NR_SYSCALL_BASE+196)
-#define __NR_fstat64			(__NR_SYSCALL_BASE+197)
-#define __NR_lchown32			(__NR_SYSCALL_BASE+198)
-#define __NR_getuid32			(__NR_SYSCALL_BASE+199)
-#define __NR_getgid32			(__NR_SYSCALL_BASE+200)
-#define __NR_geteuid32			(__NR_SYSCALL_BASE+201)
-#define __NR_getegid32			(__NR_SYSCALL_BASE+202)
-#define __NR_setreuid32			(__NR_SYSCALL_BASE+203)
-#define __NR_setregid32			(__NR_SYSCALL_BASE+204)
-#define __NR_getgroups32		(__NR_SYSCALL_BASE+205)
-#define __NR_setgroups32		(__NR_SYSCALL_BASE+206)
-#define __NR_fchown32			(__NR_SYSCALL_BASE+207)
-#define __NR_setresuid32		(__NR_SYSCALL_BASE+208)
-#define __NR_getresuid32		(__NR_SYSCALL_BASE+209)
-#define __NR_setresgid32		(__NR_SYSCALL_BASE+210)
-#define __NR_getresgid32		(__NR_SYSCALL_BASE+211)
-#define __NR_chown32			(__NR_SYSCALL_BASE+212)
-#define __NR_setuid32			(__NR_SYSCALL_BASE+213)
-#define __NR_setgid32			(__NR_SYSCALL_BASE+214)
-#define __NR_setfsuid32			(__NR_SYSCALL_BASE+215)
-#define __NR_setfsgid32			(__NR_SYSCALL_BASE+216)
-#define __NR_getdents64			(__NR_SYSCALL_BASE+217)
-#define __NR_pivot_root			(__NR_SYSCALL_BASE+218)
-#define __NR_mincore			(__NR_SYSCALL_BASE+219)
-#define __NR_madvise			(__NR_SYSCALL_BASE+220)
-#define __NR_fcntl64			(__NR_SYSCALL_BASE+221)
-					/* 222 for tux */
-					/* 223 is unused */
-#define __NR_gettid			(__NR_SYSCALL_BASE+224)
-#define __NR_readahead			(__NR_SYSCALL_BASE+225)
-#define __NR_setxattr			(__NR_SYSCALL_BASE+226)
-#define __NR_lsetxattr			(__NR_SYSCALL_BASE+227)
-#define __NR_fsetxattr			(__NR_SYSCALL_BASE+228)
-#define __NR_getxattr			(__NR_SYSCALL_BASE+229)
-#define __NR_lgetxattr			(__NR_SYSCALL_BASE+230)
-#define __NR_fgetxattr			(__NR_SYSCALL_BASE+231)
-#define __NR_listxattr			(__NR_SYSCALL_BASE+232)
-#define __NR_llistxattr			(__NR_SYSCALL_BASE+233)
-#define __NR_flistxattr			(__NR_SYSCALL_BASE+234)
-#define __NR_removexattr		(__NR_SYSCALL_BASE+235)
-#define __NR_lremovexattr		(__NR_SYSCALL_BASE+236)
-#define __NR_fremovexattr		(__NR_SYSCALL_BASE+237)
-#define __NR_tkill			(__NR_SYSCALL_BASE+238)
-#define __NR_sendfile64			(__NR_SYSCALL_BASE+239)
-#define __NR_futex			(__NR_SYSCALL_BASE+240)
-#define __NR_sched_setaffinity		(__NR_SYSCALL_BASE+241)
-#define __NR_sched_getaffinity		(__NR_SYSCALL_BASE+242)
-#define __NR_io_setup			(__NR_SYSCALL_BASE+243)
-#define __NR_io_destroy			(__NR_SYSCALL_BASE+244)
-#define __NR_io_getevents		(__NR_SYSCALL_BASE+245)
-#define __NR_io_submit			(__NR_SYSCALL_BASE+246)
-#define __NR_io_cancel			(__NR_SYSCALL_BASE+247)
-#define __NR_exit_group			(__NR_SYSCALL_BASE+248)
-#define __NR_lookup_dcookie		(__NR_SYSCALL_BASE+249)
-#define __NR_epoll_create		(__NR_SYSCALL_BASE+250)
-#define __NR_epoll_ctl			(__NR_SYSCALL_BASE+251)
-#define __NR_epoll_wait			(__NR_SYSCALL_BASE+252)
-#define __NR_remap_file_pages		(__NR_SYSCALL_BASE+253)
-					/* 254 for set_thread_area */
-					/* 255 for get_thread_area */
-#define __NR_set_tid_address		(__NR_SYSCALL_BASE+256)
-#define __NR_timer_create		(__NR_SYSCALL_BASE+257)
-#define __NR_timer_settime		(__NR_SYSCALL_BASE+258)
-#define __NR_timer_gettime		(__NR_SYSCALL_BASE+259)
-#define __NR_timer_getoverrun		(__NR_SYSCALL_BASE+260)
-#define __NR_timer_delete		(__NR_SYSCALL_BASE+261)
-#define __NR_clock_settime		(__NR_SYSCALL_BASE+262)
-#define __NR_clock_gettime		(__NR_SYSCALL_BASE+263)
-#define __NR_clock_getres		(__NR_SYSCALL_BASE+264)
-#define __NR_clock_nanosleep		(__NR_SYSCALL_BASE+265)
-#define __NR_statfs64			(__NR_SYSCALL_BASE+266)
-#define __NR_fstatfs64			(__NR_SYSCALL_BASE+267)
-#define __NR_tgkill			(__NR_SYSCALL_BASE+268)
-#define __NR_utimes			(__NR_SYSCALL_BASE+269)
-#define __NR_arm_fadvise64_64		(__NR_SYSCALL_BASE+270)
-#define __NR_pciconfig_iobase		(__NR_SYSCALL_BASE+271)
-#define __NR_pciconfig_read		(__NR_SYSCALL_BASE+272)
-#define __NR_pciconfig_write		(__NR_SYSCALL_BASE+273)
-#define __NR_mq_open			(__NR_SYSCALL_BASE+274)
-#define __NR_mq_unlink			(__NR_SYSCALL_BASE+275)
-#define __NR_mq_timedsend		(__NR_SYSCALL_BASE+276)
-#define __NR_mq_timedreceive		(__NR_SYSCALL_BASE+277)
-#define __NR_mq_notify			(__NR_SYSCALL_BASE+278)
-#define __NR_mq_getsetattr		(__NR_SYSCALL_BASE+279)
-#define __NR_waitid			(__NR_SYSCALL_BASE+280)
-#define __NR_socket			(__NR_SYSCALL_BASE+281)
-#define __NR_bind			(__NR_SYSCALL_BASE+282)
-#define __NR_connect			(__NR_SYSCALL_BASE+283)
-#define __NR_listen			(__NR_SYSCALL_BASE+284)
-#define __NR_accept			(__NR_SYSCALL_BASE+285)
-#define __NR_getsockname		(__NR_SYSCALL_BASE+286)
-#define __NR_getpeername		(__NR_SYSCALL_BASE+287)
-#define __NR_socketpair			(__NR_SYSCALL_BASE+288)
-#define __NR_send			(__NR_SYSCALL_BASE+289)
-#define __NR_sendto			(__NR_SYSCALL_BASE+290)
-#define __NR_recv			(__NR_SYSCALL_BASE+291)
-#define __NR_recvfrom			(__NR_SYSCALL_BASE+292)
-#define __NR_shutdown			(__NR_SYSCALL_BASE+293)
-#define __NR_setsockopt			(__NR_SYSCALL_BASE+294)
-#define __NR_getsockopt			(__NR_SYSCALL_BASE+295)
-#define __NR_sendmsg			(__NR_SYSCALL_BASE+296)
-#define __NR_recvmsg			(__NR_SYSCALL_BASE+297)
-#define __NR_semop			(__NR_SYSCALL_BASE+298)
-#define __NR_semget			(__NR_SYSCALL_BASE+299)
-#define __NR_semctl			(__NR_SYSCALL_BASE+300)
-#define __NR_msgsnd			(__NR_SYSCALL_BASE+301)
-#define __NR_msgrcv			(__NR_SYSCALL_BASE+302)
-#define __NR_msgget			(__NR_SYSCALL_BASE+303)
-#define __NR_msgctl			(__NR_SYSCALL_BASE+304)
-#define __NR_shmat			(__NR_SYSCALL_BASE+305)
-#define __NR_shmdt			(__NR_SYSCALL_BASE+306)
-#define __NR_shmget			(__NR_SYSCALL_BASE+307)
-#define __NR_shmctl			(__NR_SYSCALL_BASE+308)
-#define __NR_add_key			(__NR_SYSCALL_BASE+309)
-#define __NR_request_key		(__NR_SYSCALL_BASE+310)
-#define __NR_keyctl			(__NR_SYSCALL_BASE+311)
-#define __NR_semtimedop			(__NR_SYSCALL_BASE+312)
-#define __NR_vserver			(__NR_SYSCALL_BASE+313)
-#define __NR_ioprio_set			(__NR_SYSCALL_BASE+314)
-#define __NR_ioprio_get			(__NR_SYSCALL_BASE+315)
-#define __NR_inotify_init		(__NR_SYSCALL_BASE+316)
-#define __NR_inotify_add_watch		(__NR_SYSCALL_BASE+317)
-#define __NR_inotify_rm_watch		(__NR_SYSCALL_BASE+318)
-#define __NR_mbind			(__NR_SYSCALL_BASE+319)
-#define __NR_get_mempolicy		(__NR_SYSCALL_BASE+320)
-#define __NR_set_mempolicy		(__NR_SYSCALL_BASE+321)
-#define __NR_openat			(__NR_SYSCALL_BASE+322)
-#define __NR_mkdirat			(__NR_SYSCALL_BASE+323)
-#define __NR_mknodat			(__NR_SYSCALL_BASE+324)
-#define __NR_fchownat			(__NR_SYSCALL_BASE+325)
-#define __NR_futimesat			(__NR_SYSCALL_BASE+326)
-#define __NR_fstatat64			(__NR_SYSCALL_BASE+327)
-#define __NR_unlinkat			(__NR_SYSCALL_BASE+328)
-#define __NR_renameat			(__NR_SYSCALL_BASE+329)
-#define __NR_linkat			(__NR_SYSCALL_BASE+330)
-#define __NR_symlinkat			(__NR_SYSCALL_BASE+331)
-#define __NR_readlinkat			(__NR_SYSCALL_BASE+332)
-#define __NR_fchmodat			(__NR_SYSCALL_BASE+333)
-#define __NR_faccessat			(__NR_SYSCALL_BASE+334)
-#define __NR_pselect6			(__NR_SYSCALL_BASE+335)
-#define __NR_ppoll			(__NR_SYSCALL_BASE+336)
-#define __NR_unshare			(__NR_SYSCALL_BASE+337)
-#define __NR_set_robust_list		(__NR_SYSCALL_BASE+338)
-#define __NR_get_robust_list		(__NR_SYSCALL_BASE+339)
-#define __NR_splice			(__NR_SYSCALL_BASE+340)
-#define __NR_arm_sync_file_range	(__NR_SYSCALL_BASE+341)
+#include <asm/unistd-common.h>
 #define __NR_sync_file_range2		__NR_arm_sync_file_range
-#define __NR_tee			(__NR_SYSCALL_BASE+342)
-#define __NR_vmsplice			(__NR_SYSCALL_BASE+343)
-#define __NR_move_pages			(__NR_SYSCALL_BASE+344)
-#define __NR_getcpu			(__NR_SYSCALL_BASE+345)
-#define __NR_epoll_pwait		(__NR_SYSCALL_BASE+346)
-#define __NR_kexec_load			(__NR_SYSCALL_BASE+347)
-#define __NR_utimensat			(__NR_SYSCALL_BASE+348)
-#define __NR_signalfd			(__NR_SYSCALL_BASE+349)
-#define __NR_timerfd_create		(__NR_SYSCALL_BASE+350)
-#define __NR_eventfd			(__NR_SYSCALL_BASE+351)
-#define __NR_fallocate			(__NR_SYSCALL_BASE+352)
-#define __NR_timerfd_settime		(__NR_SYSCALL_BASE+353)
-#define __NR_timerfd_gettime		(__NR_SYSCALL_BASE+354)
-#define __NR_signalfd4			(__NR_SYSCALL_BASE+355)
-#define __NR_eventfd2			(__NR_SYSCALL_BASE+356)
-#define __NR_epoll_create1		(__NR_SYSCALL_BASE+357)
-#define __NR_dup3			(__NR_SYSCALL_BASE+358)
-#define __NR_pipe2			(__NR_SYSCALL_BASE+359)
-#define __NR_inotify_init1		(__NR_SYSCALL_BASE+360)
-#define __NR_preadv			(__NR_SYSCALL_BASE+361)
-#define __NR_pwritev			(__NR_SYSCALL_BASE+362)
-#define __NR_rt_tgsigqueueinfo		(__NR_SYSCALL_BASE+363)
-#define __NR_perf_event_open		(__NR_SYSCALL_BASE+364)
-#define __NR_recvmmsg			(__NR_SYSCALL_BASE+365)
-#define __NR_accept4			(__NR_SYSCALL_BASE+366)
-#define __NR_fanotify_init		(__NR_SYSCALL_BASE+367)
-#define __NR_fanotify_mark		(__NR_SYSCALL_BASE+368)
-#define __NR_prlimit64			(__NR_SYSCALL_BASE+369)
-#define __NR_name_to_handle_at		(__NR_SYSCALL_BASE+370)
-#define __NR_open_by_handle_at		(__NR_SYSCALL_BASE+371)
-#define __NR_clock_adjtime		(__NR_SYSCALL_BASE+372)
-#define __NR_syncfs			(__NR_SYSCALL_BASE+373)
-#define __NR_sendmmsg			(__NR_SYSCALL_BASE+374)
-#define __NR_setns			(__NR_SYSCALL_BASE+375)
-#define __NR_process_vm_readv		(__NR_SYSCALL_BASE+376)
-#define __NR_process_vm_writev		(__NR_SYSCALL_BASE+377)
-#define __NR_kcmp			(__NR_SYSCALL_BASE+378)
-#define __NR_finit_module		(__NR_SYSCALL_BASE+379)
-#define __NR_sched_setattr		(__NR_SYSCALL_BASE+380)
-#define __NR_sched_getattr		(__NR_SYSCALL_BASE+381)
-#define __NR_renameat2			(__NR_SYSCALL_BASE+382)
-#define __NR_seccomp			(__NR_SYSCALL_BASE+383)
-#define __NR_getrandom			(__NR_SYSCALL_BASE+384)
-#define __NR_memfd_create		(__NR_SYSCALL_BASE+385)
-#define __NR_bpf			(__NR_SYSCALL_BASE+386)
-#define __NR_execveat			(__NR_SYSCALL_BASE+387)
-#define __NR_userfaultfd		(__NR_SYSCALL_BASE+388)
-#define __NR_membarrier			(__NR_SYSCALL_BASE+389)
-#define __NR_mlock2			(__NR_SYSCALL_BASE+390)
-#define __NR_copy_file_range		(__NR_SYSCALL_BASE+391)
-#define __NR_preadv2			(__NR_SYSCALL_BASE+392)
-#define __NR_pwritev2			(__NR_SYSCALL_BASE+393)
-#define __NR_pkey_mprotect		(__NR_SYSCALL_BASE+394)
-#define __NR_pkey_alloc			(__NR_SYSCALL_BASE+395)
-#define __NR_pkey_free			(__NR_SYSCALL_BASE+396)
 
 /*
  * The following SWIs are ARM private.
@@ -434,24 +36,4 @@
 #define __ARM_NR_usr32			(__ARM_NR_BASE+4)
 #define __ARM_NR_set_tls		(__ARM_NR_BASE+5)
 
-/*
- * The following syscalls are obsolete and no longer available for EABI.
- */
-#if !defined(__KERNEL__)
-#if defined(__ARM_EABI__)
-#undef __NR_time
-#undef __NR_umount
-#undef __NR_stime
-#undef __NR_alarm
-#undef __NR_utime
-#undef __NR_getrlimit
-#undef __NR_select
-#undef __NR_readdir
-#undef __NR_mmap
-#undef __NR_socketcall
-#undef __NR_syscall
-#undef __NR_ipc
-#endif
-#endif
-
 #endif /* _UAPI__ASM_ARM_UNISTD_H */
diff --git a/arch/arm/kernel/calls.S b/arch/arm/kernel/calls.S
deleted file mode 100644
index 08030b18f10a..000000000000
--- a/arch/arm/kernel/calls.S
+++ /dev/null
@@ -1,415 +0,0 @@
-/*
- *  linux/arch/arm/kernel/calls.S
- *
- *  Copyright (C) 1995-2005 Russell King
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- *  This file is included thrice in entry-common.S
- */
-/* 0 */		CALL(sys_restart_syscall)
-		CALL(sys_exit)
-		CALL(sys_fork)
-		CALL(sys_read)
-		CALL(sys_write)
-/* 5 */		CALL(sys_open)
-		CALL(sys_close)
-		CALL(sys_ni_syscall)		/* was sys_waitpid */
-		CALL(sys_creat)
-		CALL(sys_link)
-/* 10 */	CALL(sys_unlink)
-		CALL(sys_execve)
-		CALL(sys_chdir)
-		CALL(OBSOLETE(sys_time))	/* used by libc4 */
-		CALL(sys_mknod)
-/* 15 */	CALL(sys_chmod)
-		CALL(sys_lchown16)
-		CALL(sys_ni_syscall)		/* was sys_break */
-		CALL(sys_ni_syscall)		/* was sys_stat */
-		CALL(sys_lseek)
-/* 20 */	CALL(sys_getpid)
-		CALL(sys_mount)
-		CALL(OBSOLETE(sys_oldumount))	/* used by libc4 */
-		CALL(sys_setuid16)
-		CALL(sys_getuid16)
-/* 25 */	CALL(OBSOLETE(sys_stime))
-		CALL(sys_ptrace)
-		CALL(OBSOLETE(sys_alarm))	/* used by libc4 */
-		CALL(sys_ni_syscall)		/* was sys_fstat */
-		CALL(sys_pause)
-/* 30 */	CALL(OBSOLETE(sys_utime))	/* used by libc4 */
-		CALL(sys_ni_syscall)		/* was sys_stty */
-		CALL(sys_ni_syscall)		/* was sys_getty */
-		CALL(sys_access)
-		CALL(sys_nice)
-/* 35 */	CALL(sys_ni_syscall)		/* was sys_ftime */
-		CALL(sys_sync)
-		CALL(sys_kill)
-		CALL(sys_rename)
-		CALL(sys_mkdir)
-/* 40 */	CALL(sys_rmdir)
-		CALL(sys_dup)
-		CALL(sys_pipe)
-		CALL(sys_times)
-		CALL(sys_ni_syscall)		/* was sys_prof */
-/* 45 */	CALL(sys_brk)
-		CALL(sys_setgid16)
-		CALL(sys_getgid16)
-		CALL(sys_ni_syscall)		/* was sys_signal */
-		CALL(sys_geteuid16)
-/* 50 */	CALL(sys_getegid16)
-		CALL(sys_acct)
-		CALL(sys_umount)
-		CALL(sys_ni_syscall)		/* was sys_lock */
-		CALL(sys_ioctl)
-/* 55 */	CALL(sys_fcntl)
-		CALL(sys_ni_syscall)		/* was sys_mpx */
-		CALL(sys_setpgid)
-		CALL(sys_ni_syscall)		/* was sys_ulimit */
-		CALL(sys_ni_syscall)		/* was sys_olduname */
-/* 60 */	CALL(sys_umask)
-		CALL(sys_chroot)
-		CALL(sys_ustat)
-		CALL(sys_dup2)
-		CALL(sys_getppid)
-/* 65 */	CALL(sys_getpgrp)
-		CALL(sys_setsid)
-		CALL(sys_sigaction)
-		CALL(sys_ni_syscall)		/* was sys_sgetmask */
-		CALL(sys_ni_syscall)		/* was sys_ssetmask */
-/* 70 */	CALL(sys_setreuid16)
-		CALL(sys_setregid16)
-		CALL(sys_sigsuspend)
-		CALL(sys_sigpending)
-		CALL(sys_sethostname)
-/* 75 */	CALL(sys_setrlimit)
-		CALL(OBSOLETE(sys_old_getrlimit)) /* used by libc4 */
-		CALL(sys_getrusage)
-		CALL(sys_gettimeofday)
-		CALL(sys_settimeofday)
-/* 80 */	CALL(sys_getgroups16)
-		CALL(sys_setgroups16)
-		CALL(OBSOLETE(sys_old_select))	/* used by libc4 */
-		CALL(sys_symlink)
-		CALL(sys_ni_syscall)		/* was sys_lstat */
-/* 85 */	CALL(sys_readlink)
-		CALL(sys_uselib)
-		CALL(sys_swapon)
-		CALL(sys_reboot)
-		CALL(OBSOLETE(sys_old_readdir))	/* used by libc4 */
-/* 90 */	CALL(OBSOLETE(sys_old_mmap))	/* used by libc4 */
-		CALL(sys_munmap)
-		CALL(sys_truncate)
-		CALL(sys_ftruncate)
-		CALL(sys_fchmod)
-/* 95 */	CALL(sys_fchown16)
-		CALL(sys_getpriority)
-		CALL(sys_setpriority)
-		CALL(sys_ni_syscall)		/* was sys_profil */
-		CALL(sys_statfs)
-/* 100 */	CALL(sys_fstatfs)
-		CALL(sys_ni_syscall)		/* sys_ioperm */
-		CALL(OBSOLETE(ABI(sys_socketcall, sys_oabi_socketcall)))
-		CALL(sys_syslog)
-		CALL(sys_setitimer)
-/* 105 */	CALL(sys_getitimer)
-		CALL(sys_newstat)
-		CALL(sys_newlstat)
-		CALL(sys_newfstat)
-		CALL(sys_ni_syscall)		/* was sys_uname */
-/* 110 */	CALL(sys_ni_syscall)		/* was sys_iopl */
-		CALL(sys_vhangup)
-		CALL(sys_ni_syscall)
-		CALL(OBSOLETE(sys_syscall))	/* call a syscall */
-		CALL(sys_wait4)
-/* 115 */	CALL(sys_swapoff)
-		CALL(sys_sysinfo)
-		CALL(OBSOLETE(ABI(sys_ipc, sys_oabi_ipc)))
-		CALL(sys_fsync)
-		CALL(sys_sigreturn_wrapper)
-/* 120 */	CALL(sys_clone)
-		CALL(sys_setdomainname)
-		CALL(sys_newuname)
-		CALL(sys_ni_syscall)		/* modify_ldt */
-		CALL(sys_adjtimex)
-/* 125 */	CALL(sys_mprotect)
-		CALL(sys_sigprocmask)
-		CALL(sys_ni_syscall)		/* was sys_create_module */
-		CALL(sys_init_module)
-		CALL(sys_delete_module)
-/* 130 */	CALL(sys_ni_syscall)		/* was sys_get_kernel_syms */
-		CALL(sys_quotactl)
-		CALL(sys_getpgid)
-		CALL(sys_fchdir)
-		CALL(sys_bdflush)
-/* 135 */	CALL(sys_sysfs)
-		CALL(sys_personality)
-		CALL(sys_ni_syscall)		/* reserved for afs_syscall */
-		CALL(sys_setfsuid16)
-		CALL(sys_setfsgid16)
-/* 140 */	CALL(sys_llseek)
-		CALL(sys_getdents)
-		CALL(sys_select)
-		CALL(sys_flock)
-		CALL(sys_msync)
-/* 145 */	CALL(sys_readv)
-		CALL(sys_writev)
-		CALL(sys_getsid)
-		CALL(sys_fdatasync)
-		CALL(sys_sysctl)
-/* 150 */	CALL(sys_mlock)
-		CALL(sys_munlock)
-		CALL(sys_mlockall)
-		CALL(sys_munlockall)
-		CALL(sys_sched_setparam)
-/* 155 */	CALL(sys_sched_getparam)
-		CALL(sys_sched_setscheduler)
-		CALL(sys_sched_getscheduler)
-		CALL(sys_sched_yield)
-		CALL(sys_sched_get_priority_max)
-/* 160 */	CALL(sys_sched_get_priority_min)
-		CALL(sys_sched_rr_get_interval)
-		CALL(sys_nanosleep)
-		CALL(sys_mremap)
-		CALL(sys_setresuid16)
-/* 165 */	CALL(sys_getresuid16)
-		CALL(sys_ni_syscall)		/* vm86 */
-		CALL(sys_ni_syscall)		/* was sys_query_module */
-		CALL(sys_poll)
-		CALL(sys_ni_syscall)		/* was nfsservctl */
-/* 170 */	CALL(sys_setresgid16)
-		CALL(sys_getresgid16)
-		CALL(sys_prctl)
-		CALL(sys_rt_sigreturn_wrapper)
-		CALL(sys_rt_sigaction)
-/* 175 */	CALL(sys_rt_sigprocmask)
-		CALL(sys_rt_sigpending)
-		CALL(sys_rt_sigtimedwait)
-		CALL(sys_rt_sigqueueinfo)
-		CALL(sys_rt_sigsuspend)
-/* 180 */	CALL(ABI(sys_pread64, sys_oabi_pread64))
-		CALL(ABI(sys_pwrite64, sys_oabi_pwrite64))
-		CALL(sys_chown16)
-		CALL(sys_getcwd)
-		CALL(sys_capget)
-/* 185 */	CALL(sys_capset)
-		CALL(sys_sigaltstack)
-		CALL(sys_sendfile)
-		CALL(sys_ni_syscall)		/* getpmsg */
-		CALL(sys_ni_syscall)		/* putpmsg */
-/* 190 */	CALL(sys_vfork)
-		CALL(sys_getrlimit)
-		CALL(sys_mmap2)
-		CALL(ABI(sys_truncate64, sys_oabi_truncate64))
-		CALL(ABI(sys_ftruncate64, sys_oabi_ftruncate64))
-/* 195 */	CALL(ABI(sys_stat64, sys_oabi_stat64))
-		CALL(ABI(sys_lstat64, sys_oabi_lstat64))
-		CALL(ABI(sys_fstat64, sys_oabi_fstat64))
-		CALL(sys_lchown)
-		CALL(sys_getuid)
-/* 200 */	CALL(sys_getgid)
-		CALL(sys_geteuid)
-		CALL(sys_getegid)
-		CALL(sys_setreuid)
-		CALL(sys_setregid)
-/* 205 */	CALL(sys_getgroups)
-		CALL(sys_setgroups)
-		CALL(sys_fchown)
-		CALL(sys_setresuid)
-		CALL(sys_getresuid)
-/* 210 */	CALL(sys_setresgid)
-		CALL(sys_getresgid)
-		CALL(sys_chown)
-		CALL(sys_setuid)
-		CALL(sys_setgid)
-/* 215 */	CALL(sys_setfsuid)
-		CALL(sys_setfsgid)
-		CALL(sys_getdents64)
-		CALL(sys_pivot_root)
-		CALL(sys_mincore)
-/* 220 */	CALL(sys_madvise)
-		CALL(ABI(sys_fcntl64, sys_oabi_fcntl64))
-		CALL(sys_ni_syscall) /* TUX */
-		CALL(sys_ni_syscall)
-		CALL(sys_gettid)
-/* 225 */	CALL(ABI(sys_readahead, sys_oabi_readahead))
-		CALL(sys_setxattr)
-		CALL(sys_lsetxattr)
-		CALL(sys_fsetxattr)
-		CALL(sys_getxattr)
-/* 230 */	CALL(sys_lgetxattr)
-		CALL(sys_fgetxattr)
-		CALL(sys_listxattr)
-		CALL(sys_llistxattr)
-		CALL(sys_flistxattr)
-/* 235 */	CALL(sys_removexattr)
-		CALL(sys_lremovexattr)
-		CALL(sys_fremovexattr)
-		CALL(sys_tkill)
-		CALL(sys_sendfile64)
-/* 240 */	CALL(sys_futex)
-		CALL(sys_sched_setaffinity)
-		CALL(sys_sched_getaffinity)
-		CALL(sys_io_setup)
-		CALL(sys_io_destroy)
-/* 245 */	CALL(sys_io_getevents)
-		CALL(sys_io_submit)
-		CALL(sys_io_cancel)
-		CALL(sys_exit_group)
-		CALL(sys_lookup_dcookie)
-/* 250 */	CALL(sys_epoll_create)
-		CALL(ABI(sys_epoll_ctl, sys_oabi_epoll_ctl))
-		CALL(ABI(sys_epoll_wait, sys_oabi_epoll_wait))
-		CALL(sys_remap_file_pages)
-		CALL(sys_ni_syscall)	/* sys_set_thread_area */
-/* 255 */	CALL(sys_ni_syscall)	/* sys_get_thread_area */
-		CALL(sys_set_tid_address)
-		CALL(sys_timer_create)
-		CALL(sys_timer_settime)
-		CALL(sys_timer_gettime)
-/* 260 */	CALL(sys_timer_getoverrun)
-		CALL(sys_timer_delete)
-		CALL(sys_clock_settime)
-		CALL(sys_clock_gettime)
-		CALL(sys_clock_getres)
-/* 265 */	CALL(sys_clock_nanosleep)
-		CALL(sys_statfs64_wrapper)
-		CALL(sys_fstatfs64_wrapper)
-		CALL(sys_tgkill)
-		CALL(sys_utimes)
-/* 270 */	CALL(sys_arm_fadvise64_64)
-		CALL(sys_pciconfig_iobase)
-		CALL(sys_pciconfig_read)
-		CALL(sys_pciconfig_write)
-		CALL(sys_mq_open)
-/* 275 */	CALL(sys_mq_unlink)
-		CALL(sys_mq_timedsend)
-		CALL(sys_mq_timedreceive)
-		CALL(sys_mq_notify)
-		CALL(sys_mq_getsetattr)
-/* 280 */	CALL(sys_waitid)
-		CALL(sys_socket)
-		CALL(ABI(sys_bind, sys_oabi_bind))
-		CALL(ABI(sys_connect, sys_oabi_connect))
-		CALL(sys_listen)
-/* 285 */	CALL(sys_accept)
-		CALL(sys_getsockname)
-		CALL(sys_getpeername)
-		CALL(sys_socketpair)
-		CALL(sys_send)
-/* 290 */	CALL(ABI(sys_sendto, sys_oabi_sendto))
-		CALL(sys_recv)
-		CALL(sys_recvfrom)
-		CALL(sys_shutdown)
-		CALL(sys_setsockopt)
-/* 295 */	CALL(sys_getsockopt)
-		CALL(ABI(sys_sendmsg, sys_oabi_sendmsg))
-		CALL(sys_recvmsg)
-		CALL(ABI(sys_semop, sys_oabi_semop))
-		CALL(sys_semget)
-/* 300 */	CALL(sys_semctl)
-		CALL(sys_msgsnd)
-		CALL(sys_msgrcv)
-		CALL(sys_msgget)
-		CALL(sys_msgctl)
-/* 305 */	CALL(sys_shmat)
-		CALL(sys_shmdt)
-		CALL(sys_shmget)
-		CALL(sys_shmctl)
-		CALL(sys_add_key)
-/* 310 */	CALL(sys_request_key)
-		CALL(sys_keyctl)
-		CALL(ABI(sys_semtimedop, sys_oabi_semtimedop))
-/* vserver */	CALL(sys_ni_syscall)
-		CALL(sys_ioprio_set)
-/* 315 */	CALL(sys_ioprio_get)
-		CALL(sys_inotify_init)
-		CALL(sys_inotify_add_watch)
-		CALL(sys_inotify_rm_watch)
-		CALL(sys_mbind)
-/* 320 */	CALL(sys_get_mempolicy)
-		CALL(sys_set_mempolicy)
-		CALL(sys_openat)
-		CALL(sys_mkdirat)
-		CALL(sys_mknodat)
-/* 325 */	CALL(sys_fchownat)
-		CALL(sys_futimesat)
-		CALL(ABI(sys_fstatat64,  sys_oabi_fstatat64))
-		CALL(sys_unlinkat)
-		CALL(sys_renameat)
-/* 330 */	CALL(sys_linkat)
-		CALL(sys_symlinkat)
-		CALL(sys_readlinkat)
-		CALL(sys_fchmodat)
-		CALL(sys_faccessat)
-/* 335 */	CALL(sys_pselect6)
-		CALL(sys_ppoll)
-		CALL(sys_unshare)
-		CALL(sys_set_robust_list)
-		CALL(sys_get_robust_list)
-/* 340 */	CALL(sys_splice)
-		CALL(sys_sync_file_range2)
-		CALL(sys_tee)
-		CALL(sys_vmsplice)
-		CALL(sys_move_pages)
-/* 345 */	CALL(sys_getcpu)
-		CALL(sys_epoll_pwait)
-		CALL(sys_kexec_load)
-		CALL(sys_utimensat)
-		CALL(sys_signalfd)
-/* 350 */	CALL(sys_timerfd_create)
-		CALL(sys_eventfd)
-		CALL(sys_fallocate)
-		CALL(sys_timerfd_settime)
-		CALL(sys_timerfd_gettime)
-/* 355 */	CALL(sys_signalfd4)
-		CALL(sys_eventfd2)
-		CALL(sys_epoll_create1)
-		CALL(sys_dup3)
-		CALL(sys_pipe2)
-/* 360 */	CALL(sys_inotify_init1)
-		CALL(sys_preadv)
-		CALL(sys_pwritev)
-		CALL(sys_rt_tgsigqueueinfo)
-		CALL(sys_perf_event_open)
-/* 365 */	CALL(sys_recvmmsg)
-		CALL(sys_accept4)
-		CALL(sys_fanotify_init)
-		CALL(sys_fanotify_mark)
-		CALL(sys_prlimit64)
-/* 370 */	CALL(sys_name_to_handle_at)
-		CALL(sys_open_by_handle_at)
-		CALL(sys_clock_adjtime)
-		CALL(sys_syncfs)
-		CALL(sys_sendmmsg)
-/* 375 */	CALL(sys_setns)
-		CALL(sys_process_vm_readv)
-		CALL(sys_process_vm_writev)
-		CALL(sys_kcmp)
-		CALL(sys_finit_module)
-/* 380 */	CALL(sys_sched_setattr)
-		CALL(sys_sched_getattr)
-		CALL(sys_renameat2)
-		CALL(sys_seccomp)
-		CALL(sys_getrandom)
-/* 385 */	CALL(sys_memfd_create)
-		CALL(sys_bpf)
-		CALL(sys_execveat)
-		CALL(sys_userfaultfd)
-		CALL(sys_membarrier)
-/* 390 */	CALL(sys_mlock2)
-		CALL(sys_copy_file_range)
-		CALL(sys_preadv2)
-		CALL(sys_pwritev2)
-		CALL(sys_pkey_mprotect)
-/* 395 */	CALL(sys_pkey_alloc)
-		CALL(sys_pkey_free)
-#ifndef syscalls_counted
-.equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls
-#define syscalls_counted
-#endif
-.rept syscalls_padding
-		CALL(sys_ni_syscall)
-.endr
diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S
index 10c3283d6c19..eb5cd77bf1d8 100644
--- a/arch/arm/kernel/entry-common.S
+++ b/arch/arm/kernel/entry-common.S
@@ -12,6 +12,11 @@
 #include <asm/unistd.h>
 #include <asm/ftrace.h>
 #include <asm/unwind.h>
+#ifdef CONFIG_AEABI
+#include <asm/unistd-oabi.h>
+#endif
+
+	.equ	NR_syscalls, __NR_syscalls
 
 #ifdef CONFIG_NEED_RET_TO_USER
 #include <mach/entry-macro.S>
@@ -120,21 +125,6 @@ ENTRY(ret_from_fork)
 	b	ret_slow_syscall
 ENDPROC(ret_from_fork)
 
-	.equ NR_syscalls,0
-#define CALL(x) .equ NR_syscalls,NR_syscalls+1
-#include "calls.S"
-
-/*
- * Ensure that the system call table is equal to __NR_syscalls,
- * which is the value the rest of the system sees
- */
-.ifne NR_syscalls - __NR_syscalls
-.error "__NR_syscalls is not equal to the size of the syscall table"
-.endif
-
-#undef CALL
-#define CALL(x) .long x
-
 /*=============================================================================
  * SWI handler
  *-----------------------------------------------------------------------------
@@ -291,22 +281,48 @@ __cr_alignment:
 #endif
 	.ltorg
 
+	.macro	syscall_table_start, sym
+	.equ	__sys_nr, 0
+	.type	\sym, #object
+ENTRY(\sym)
+	.endm
+
+	.macro	syscall, nr, func
+	.ifgt	__sys_nr - \nr
+	.error	"Duplicated/unorded system call entry"
+	.endif
+	.rept	\nr - __sys_nr
+	.long	sys_ni_syscall
+	.endr
+	.long	\func
+	.equ	__sys_nr, \nr + 1
+	.endm
+
+	.macro	syscall_table_end, sym
+	.ifgt	__sys_nr - __NR_syscalls
+	.error	"System call table too big"
+	.endif
+	.rept	__NR_syscalls - __sys_nr
+	.long	sys_ni_syscall
+	.endr
+	.size	\sym, . - \sym
+	.endm
+
+#define NATIVE(nr, func) syscall nr, func
+
 /*
  * This is the syscall table declaration for native ABI syscalls.
  * With EABI a couple syscalls are obsolete and defined as sys_ni_syscall.
  */
-#define ABI(native, compat) native
+	syscall_table_start sys_call_table
+#define COMPAT(nr, native, compat) syscall nr, native
 #ifdef CONFIG_AEABI
-#define OBSOLETE(syscall) sys_ni_syscall
+#include <calls-eabi.S>
 #else
-#define OBSOLETE(syscall) syscall
+#include <calls-oabi.S>
 #endif
-
-	.type	sys_call_table, #object
-ENTRY(sys_call_table)
-#include "calls.S"
-#undef ABI
-#undef OBSOLETE
+#undef COMPAT
+	syscall_table_end sys_call_table
 
 /*============================================================================
  * Special system call wrappers
@@ -407,14 +423,10 @@ ENDPROC(sys_oabi_readahead)
  * Let's declare a second syscall table for old ABI binaries
  * using the compatibility syscall entries.
  */
-#define ABI(native, compat) compat
-#define OBSOLETE(syscall) syscall
-
-	.type	sys_oabi_call_table, #object
-ENTRY(sys_oabi_call_table)
-#include "calls.S"
-#undef ABI
-#undef OBSOLETE
+	syscall_table_start sys_oabi_call_table
+#define COMPAT(nr, native, compat) syscall nr, compat
+#include <calls-oabi.S>
+	syscall_table_end sys_oabi_call_table
 
 #endif
 
diff --git a/arch/arm/kernel/topology.c b/arch/arm/kernel/topology.c
index ec279d161b32..ebf47d91b804 100644
--- a/arch/arm/kernel/topology.c
+++ b/arch/arm/kernel/topology.c
@@ -12,6 +12,7 @@
  */
 
 #include <linux/cpu.h>
+#include <linux/cpufreq.h>
 #include <linux/cpumask.h>
 #include <linux/export.h>
 #include <linux/init.h>
@@ -21,7 +22,9 @@
 #include <linux/of.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
+#include <linux/string.h>
 
+#include <asm/cpu.h>
 #include <asm/cputype.h>
 #include <asm/topology.h>
 
@@ -41,6 +44,7 @@
  * updated during this sequence.
  */
 static DEFINE_PER_CPU(unsigned long, cpu_scale) = SCHED_CAPACITY_SCALE;
+static DEFINE_MUTEX(cpu_scale_mutex);
 
 unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu)
 {
@@ -52,6 +56,65 @@ static void set_capacity_scale(unsigned int cpu, unsigned long capacity)
 	per_cpu(cpu_scale, cpu) = capacity;
 }
 
+#ifdef CONFIG_PROC_SYSCTL
+static ssize_t cpu_capacity_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct cpu *cpu = container_of(dev, struct cpu, dev);
+
+	return sprintf(buf, "%lu\n",
+			arch_scale_cpu_capacity(NULL, cpu->dev.id));
+}
+
+static ssize_t cpu_capacity_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf,
+				  size_t count)
+{
+	struct cpu *cpu = container_of(dev, struct cpu, dev);
+	int this_cpu = cpu->dev.id, i;
+	unsigned long new_capacity;
+	ssize_t ret;
+
+	if (count) {
+		ret = kstrtoul(buf, 0, &new_capacity);
+		if (ret)
+			return ret;
+		if (new_capacity > SCHED_CAPACITY_SCALE)
+			return -EINVAL;
+
+		mutex_lock(&cpu_scale_mutex);
+		for_each_cpu(i, &cpu_topology[this_cpu].core_sibling)
+			set_capacity_scale(i, new_capacity);
+		mutex_unlock(&cpu_scale_mutex);
+	}
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(cpu_capacity);
+
+static int register_cpu_capacity_sysctl(void)
+{
+	int i;
+	struct device *cpu;
+
+	for_each_possible_cpu(i) {
+		cpu = get_cpu_device(i);
+		if (!cpu) {
+			pr_err("%s: too early to get CPU%d device!\n",
+			       __func__, i);
+			continue;
+		}
+		device_create_file(cpu, &dev_attr_cpu_capacity);
+	}
+
+	return 0;
+}
+subsys_initcall(register_cpu_capacity_sysctl);
+#endif
+
 #ifdef CONFIG_OF
 struct cpu_efficiency {
 	const char *compatible;
@@ -78,6 +141,146 @@ static unsigned long *__cpu_capacity;
 #define cpu_capacity(cpu)	__cpu_capacity[cpu]
 
 static unsigned long middle_capacity = 1;
+static bool cap_from_dt = true;
+static u32 *raw_capacity;
+static bool cap_parsing_failed;
+static u32 capacity_scale;
+
+static int __init parse_cpu_capacity(struct device_node *cpu_node, int cpu)
+{
+	int ret = 1;
+	u32 cpu_capacity;
+
+	if (cap_parsing_failed)
+		return !ret;
+
+	ret = of_property_read_u32(cpu_node,
+				   "capacity-dmips-mhz",
+				   &cpu_capacity);
+	if (!ret) {
+		if (!raw_capacity) {
+			raw_capacity = kcalloc(num_possible_cpus(),
+					       sizeof(*raw_capacity),
+					       GFP_KERNEL);
+			if (!raw_capacity) {
+				pr_err("cpu_capacity: failed to allocate memory for raw capacities\n");
+				cap_parsing_failed = true;
+				return !ret;
+			}
+		}
+		capacity_scale = max(cpu_capacity, capacity_scale);
+		raw_capacity[cpu] = cpu_capacity;
+		pr_debug("cpu_capacity: %s cpu_capacity=%u (raw)\n",
+			cpu_node->full_name, raw_capacity[cpu]);
+	} else {
+		if (raw_capacity) {
+			pr_err("cpu_capacity: missing %s raw capacity\n",
+				cpu_node->full_name);
+			pr_err("cpu_capacity: partial information: fallback to 1024 for all CPUs\n");
+		}
+		cap_parsing_failed = true;
+		kfree(raw_capacity);
+	}
+
+	return !ret;
+}
+
+static void normalize_cpu_capacity(void)
+{
+	u64 capacity;
+	int cpu;
+
+	if (!raw_capacity || cap_parsing_failed)
+		return;
+
+	pr_debug("cpu_capacity: capacity_scale=%u\n", capacity_scale);
+	mutex_lock(&cpu_scale_mutex);
+	for_each_possible_cpu(cpu) {
+		capacity = (raw_capacity[cpu] << SCHED_CAPACITY_SHIFT)
+			/ capacity_scale;
+		set_capacity_scale(cpu, capacity);
+		pr_debug("cpu_capacity: CPU%d cpu_capacity=%lu\n",
+			cpu, arch_scale_cpu_capacity(NULL, cpu));
+	}
+	mutex_unlock(&cpu_scale_mutex);
+}
+
+#ifdef CONFIG_CPU_FREQ
+static cpumask_var_t cpus_to_visit;
+static bool cap_parsing_done;
+static void parsing_done_workfn(struct work_struct *work);
+static DECLARE_WORK(parsing_done_work, parsing_done_workfn);
+
+static int
+init_cpu_capacity_callback(struct notifier_block *nb,
+			   unsigned long val,
+			   void *data)
+{
+	struct cpufreq_policy *policy = data;
+	int cpu;
+
+	if (cap_parsing_failed || cap_parsing_done)
+		return 0;
+
+	switch (val) {
+	case CPUFREQ_NOTIFY:
+		pr_debug("cpu_capacity: init cpu capacity for CPUs [%*pbl] (to_visit=%*pbl)\n",
+				cpumask_pr_args(policy->related_cpus),
+				cpumask_pr_args(cpus_to_visit));
+		cpumask_andnot(cpus_to_visit,
+			       cpus_to_visit,
+			       policy->related_cpus);
+		for_each_cpu(cpu, policy->related_cpus) {
+			raw_capacity[cpu] = arch_scale_cpu_capacity(NULL, cpu) *
+					    policy->cpuinfo.max_freq / 1000UL;
+			capacity_scale = max(raw_capacity[cpu], capacity_scale);
+		}
+		if (cpumask_empty(cpus_to_visit)) {
+			normalize_cpu_capacity();
+			kfree(raw_capacity);
+			pr_debug("cpu_capacity: parsing done\n");
+			cap_parsing_done = true;
+			schedule_work(&parsing_done_work);
+		}
+	}
+	return 0;
+}
+
+static struct notifier_block init_cpu_capacity_notifier = {
+	.notifier_call = init_cpu_capacity_callback,
+};
+
+static int __init register_cpufreq_notifier(void)
+{
+	if (cap_parsing_failed)
+		return -EINVAL;
+
+	if (!alloc_cpumask_var(&cpus_to_visit, GFP_KERNEL)) {
+		pr_err("cpu_capacity: failed to allocate memory for cpus_to_visit\n");
+		return -ENOMEM;
+	}
+	cpumask_copy(cpus_to_visit, cpu_possible_mask);
+
+	return cpufreq_register_notifier(&init_cpu_capacity_notifier,
+					 CPUFREQ_POLICY_NOTIFIER);
+}
+core_initcall(register_cpufreq_notifier);
+
+static void parsing_done_workfn(struct work_struct *work)
+{
+	cpufreq_unregister_notifier(&init_cpu_capacity_notifier,
+					 CPUFREQ_POLICY_NOTIFIER);
+}
+
+#else
+static int __init free_raw_capacity(void)
+{
+	kfree(raw_capacity);
+
+	return 0;
+}
+core_initcall(free_raw_capacity);
+#endif
 
 /*
  * Iterate all CPUs' descriptor in DT and compute the efficiency
@@ -99,6 +302,12 @@ static void __init parse_dt_topology(void)
 	__cpu_capacity = kcalloc(nr_cpu_ids, sizeof(*__cpu_capacity),
 				 GFP_NOWAIT);
 
+	cn = of_find_node_by_path("/cpus");
+	if (!cn) {
+		pr_err("No CPU information found in DT\n");
+		return;
+	}
+
 	for_each_possible_cpu(cpu) {
 		const u32 *rate;
 		int len;
@@ -110,6 +319,13 @@ static void __init parse_dt_topology(void)
 			continue;
 		}
 
+		if (parse_cpu_capacity(cn, cpu)) {
+			of_node_put(cn);
+			continue;
+		}
+
+		cap_from_dt = false;
+
 		for (cpu_eff = table_efficiency; cpu_eff->compatible; cpu_eff++)
 			if (of_device_is_compatible(cn, cpu_eff->compatible))
 				break;
@@ -151,6 +367,8 @@ static void __init parse_dt_topology(void)
 		middle_capacity = ((max_capacity / 3)
 				>> (SCHED_CAPACITY_SHIFT-1)) + 1;
 
+	if (cap_from_dt && !cap_parsing_failed)
+		normalize_cpu_capacity();
 }
 
 /*
@@ -160,7 +378,7 @@ static void __init parse_dt_topology(void)
  */
 static void update_cpu_capacity(unsigned int cpu)
 {
-	if (!cpu_capacity(cpu))
+	if (!cpu_capacity(cpu) || cap_from_dt)
 		return;
 
 	set_capacity_scale(cpu, cpu_capacity(cpu) / middle_capacity);
diff --git a/arch/arm/kvm/Kconfig b/arch/arm/kvm/Kconfig
index 3e1cd0452d67..90d0176fb30d 100644
--- a/arch/arm/kvm/Kconfig
+++ b/arch/arm/kvm/Kconfig
@@ -34,6 +34,7 @@ config KVM
 	select HAVE_KVM_IRQFD
 	select HAVE_KVM_IRQCHIP
 	select HAVE_KVM_IRQ_ROUTING
+	select HAVE_KVM_MSI
 	depends on ARM_VIRT_EXT && ARM_LPAE && ARM_ARCH_TIMER
 	---help---
 	  Support hosting virtualized guest machines.
diff --git a/arch/arm/kvm/Makefile b/arch/arm/kvm/Makefile
index f19842ea5418..d571243ab4d1 100644
--- a/arch/arm/kvm/Makefile
+++ b/arch/arm/kvm/Makefile
@@ -32,5 +32,6 @@ obj-y += $(KVM)/arm/vgic/vgic-mmio.o
 obj-y += $(KVM)/arm/vgic/vgic-mmio-v2.o
 obj-y += $(KVM)/arm/vgic/vgic-mmio-v3.o
 obj-y += $(KVM)/arm/vgic/vgic-kvm-device.o
+obj-y += $(KVM)/arm/vgic/vgic-its.o
 obj-y += $(KVM)/irqchip.o
 obj-y += $(KVM)/arm/arch_timer.o
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 19b5f5c1c0ff..8f92efa8460e 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -221,6 +221,12 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_MAX_VCPUS:
 		r = KVM_MAX_VCPUS;
 		break;
+	case KVM_CAP_MSI_DEVID:
+		if (!kvm)
+			r = -EINVAL;
+		else
+			r = kvm->arch.vgic.msis_require_devid;
+		break;
 	default:
 		r = kvm_arch_dev_ioctl_check_extension(kvm, ext);
 		break;
diff --git a/arch/arm/lib/delay-loop.S b/arch/arm/lib/delay-loop.S
index 792c59d885bc..c766694e929c 100644
--- a/arch/arm/lib/delay-loop.S
+++ b/arch/arm/lib/delay-loop.S
@@ -17,24 +17,23 @@
 .LC1:		.word	UDELAY_MULT
 
 /*
+ * loops = r0 * HZ * loops_per_jiffy / 1000000
+ *
  * r0  <= 2000
  * HZ  <= 1000
  */
 
 ENTRY(__loop_udelay)
 		ldr	r2, .LC1
-		mul	r0, r2, r0
-ENTRY(__loop_const_udelay)			@ 0 <= r0 <= 0x7fffff06
+		mul	r0, r2, r0		@ r0 = delay_us * UDELAY_MULT
+ENTRY(__loop_const_udelay)			@ 0 <= r0 <= 0xfffffaf0
 		ldr	r2, .LC0
 		ldr	r2, [r2]
-		umull	r1, r0, r2, r0
-		adds	r1, r1, #0xffffffff
-		adcs	r0, r0, r0
+		umull	r1, r0, r2, r0		@ r0-r1 = r0 * loops_per_jiffy
+		adds	r1, r1, #0xffffffff	@ rounding up ...
+		adcs	r0, r0, r0		@ and right shift by 31
 		reteq	lr
 
-/*
- * loops = r0 * HZ * loops_per_jiffy / 1000000
- */
 		.align 3
 
 @ Delay routine
diff --git a/arch/arm/mach-artpec/Kconfig b/arch/arm/mach-artpec/Kconfig
index 6cbe5a2eabab..85a962abb77f 100644
--- a/arch/arm/mach-artpec/Kconfig
+++ b/arch/arm/mach-artpec/Kconfig
@@ -14,6 +14,7 @@ config MACH_ARTPEC6
 	select HAVE_ARM_ARCH_TIMER
 	select HAVE_ARM_SCU
 	select HAVE_ARM_TWD if SMP
+	select MFD_SYSCON
 	help
 	  Support for Axis ARTPEC-6 ARM Cortex A9 Platform
 
diff --git a/arch/arm/mach-bcm/bcm_5301x.c b/arch/arm/mach-bcm/bcm_5301x.c
index c8830a2b0d60..fe067f6cebb6 100644
--- a/arch/arm/mach-bcm/bcm_5301x.c
+++ b/arch/arm/mach-bcm/bcm_5301x.c
@@ -9,14 +9,42 @@
 #include <asm/hardware/cache-l2x0.h>
 
 #include <asm/mach/arch.h>
+#include <asm/siginfo.h>
+#include <asm/signal.h>
+
+#define FSR_EXTERNAL		(1 << 12)
+#define FSR_READ		(0 << 10)
+#define FSR_IMPRECISE		0x0406
 
 static const char *const bcm5301x_dt_compat[] __initconst = {
 	"brcm,bcm4708",
 	NULL,
 };
 
+static int bcm5301x_abort_handler(unsigned long addr, unsigned int fsr,
+				  struct pt_regs *regs)
+{
+	/*
+	 * We want to ignore aborts forwarded from the PCIe bus that are
+	 * expected and shouldn't really be passed by the PCIe controller.
+	 * The biggest disadvantage is the same FSR code may be reported when
+	 * reading non-existing APB register and we shouldn't ignore that.
+	 */
+	if (fsr == (FSR_EXTERNAL | FSR_READ | FSR_IMPRECISE))
+		return 0;
+
+	return 1;
+}
+
+static void __init bcm5301x_init_early(void)
+{
+	hook_fault_code(16 + 6, bcm5301x_abort_handler, SIGBUS, BUS_OBJERR,
+			"imprecise external abort");
+}
+
 DT_MACHINE_START(BCM5301X, "BCM5301X")
 	.l2c_aux_val	= 0,
 	.l2c_aux_mask	= ~0,
 	.dt_compat	= bcm5301x_dt_compat,
+	.init_early	= bcm5301x_init_early,
 MACHINE_END
diff --git a/arch/arm/mach-davinci/Makefile b/arch/arm/mach-davinci/Makefile
index da4c336b4637..0a2e6da45f28 100644
--- a/arch/arm/mach-davinci/Makefile
+++ b/arch/arm/mach-davinci/Makefile
@@ -36,5 +36,7 @@ obj-$(CONFIG_MACH_OMAPL138_HAWKBOARD)	+= board-omapl138-hawk.o
 
 # Power Management
 obj-$(CONFIG_CPU_IDLE)			+= cpuidle.o
-obj-$(CONFIG_SUSPEND)			+= pm.o sleep.o
 obj-$(CONFIG_HAVE_CLK)			+= pm_domain.o
+ifeq ($(CONFIG_SUSPEND),y)
+obj-$(CONFIG_ARCH_DAVINCI_DA850)	+= pm.o sleep.o
+endif
diff --git a/arch/arm/mach-davinci/board-da830-evm.c b/arch/arm/mach-davinci/board-da830-evm.c
index 3d8cf8cbd98a..58075627c6df 100644
--- a/arch/arm/mach-davinci/board-da830-evm.c
+++ b/arch/arm/mach-davinci/board-da830-evm.c
@@ -14,6 +14,7 @@
 #include <linux/console.h>
 #include <linux/interrupt.h>
 #include <linux/gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/platform_device.h>
 #include <linux/i2c.h>
 #include <linux/i2c/pcf857x.h>
@@ -27,6 +28,7 @@
 #include <linux/platform_data/mtd-davinci-aemif.h>
 #include <linux/platform_data/spi-davinci.h>
 #include <linux/platform_data/usb-davinci.h>
+#include <linux/regulator/machine.h>
 
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
@@ -106,43 +108,24 @@ static irqreturn_t da830_evm_usb_ocic_irq(int irq, void *dev_id)
 
 static __init void da830_evm_usb_init(void)
 {
-	u32 cfgchip2;
 	int ret;
 
-	/*
-	 * Set up USB clock/mode in the CFGCHIP2 register.
-	 * FYI:  CFGCHIP2 is 0x0000ef00 initially.
-	 */
-	cfgchip2 = __raw_readl(DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
-
-	/* USB2.0 PHY reference clock is 24 MHz */
-	cfgchip2 &= ~CFGCHIP2_REFFREQ;
-	cfgchip2 |=  CFGCHIP2_REFFREQ_24MHZ;
-
-	/*
-	 * Select internal reference clock for USB 2.0 PHY
-	 * and use it as a clock source for USB 1.1 PHY
-	 * (this is the default setting anyway).
-	 */
-	cfgchip2 &= ~CFGCHIP2_USB1PHYCLKMUX;
-	cfgchip2 |=  CFGCHIP2_USB2PHYCLKMUX;
-
-	/*
-	 * We have to override VBUS/ID signals when MUSB is configured into the
-	 * host-only mode -- ID pin will float if no cable is connected, so the
-	 * controller won't be able to drive VBUS thinking that it's a B-device.
-	 * Otherwise, we want to use the OTG mode and enable VBUS comparators.
-	 */
-	cfgchip2 &= ~CFGCHIP2_OTGMODE;
-#ifdef	CONFIG_USB_MUSB_HOST
-	cfgchip2 |=  CFGCHIP2_FORCE_HOST;
-#else
-	cfgchip2 |=  CFGCHIP2_SESENDEN | CFGCHIP2_VBDTCTEN;
-#endif
+	/* USB_REFCLKIN is not used. */
+	ret = da8xx_register_usb20_phy_clk(false);
+	if (ret)
+		pr_warn("%s: USB 2.0 PHY CLK registration failed: %d\n",
+			__func__, ret);
 
-	__raw_writel(cfgchip2, DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
+	ret = da8xx_register_usb11_phy_clk(false);
+	if (ret)
+		pr_warn("%s: USB 1.1 PHY CLK registration failed: %d\n",
+			__func__, ret);
+
+	ret = da8xx_register_usb_phy();
+	if (ret)
+		pr_warn("%s: USB PHY registration failed: %d\n",
+			__func__, ret);
 
-	/* USB_REFCLKIN is not used. */
 	ret = davinci_cfg_reg(DA830_USB0_DRVVBUS);
 	if (ret)
 		pr_warn("%s: USB 2.0 PinMux setup failed: %d\n", __func__, ret);
@@ -222,22 +205,16 @@ static const short da830_evm_mmc_sd_pins[] = {
 	-1
 };
 
-#define DA830_MMCSD_WP_PIN		GPIO_TO_PIN(2, 1)
-#define DA830_MMCSD_CD_PIN		GPIO_TO_PIN(2, 2)
-
-static int da830_evm_mmc_get_ro(int index)
-{
-	return gpio_get_value(DA830_MMCSD_WP_PIN);
-}
-
-static int da830_evm_mmc_get_cd(int index)
-{
-	return !gpio_get_value(DA830_MMCSD_CD_PIN);
-}
+static struct gpiod_lookup_table mmc_gpios_table = {
+	.dev_id = "da830-mmc.0",
+	.table = {
+		/* gpio chip 1 contains gpio range 32-63 */
+		GPIO_LOOKUP("davinci_gpio.1", 2, "cd", GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP("davinci_gpio.1", 1, "wp", GPIO_ACTIVE_LOW),
+	},
+};
 
 static struct davinci_mmc_config da830_evm_mmc_config = {
-	.get_ro			= da830_evm_mmc_get_ro,
-	.get_cd			= da830_evm_mmc_get_cd,
 	.wires			= 8,
 	.max_freq		= 50000000,
 	.caps			= MMC_CAP_MMC_HIGHSPEED | MMC_CAP_SD_HIGHSPEED,
@@ -253,26 +230,12 @@ static inline void da830_evm_init_mmc(void)
 		return;
 	}
 
-	ret = gpio_request(DA830_MMCSD_WP_PIN, "MMC WP");
-	if (ret) {
-		pr_warn("%s: can not open GPIO %d\n",
-			__func__, DA830_MMCSD_WP_PIN);
-		return;
-	}
-	gpio_direction_input(DA830_MMCSD_WP_PIN);
-
-	ret = gpio_request(DA830_MMCSD_CD_PIN, "MMC CD\n");
-	if (ret) {
-		pr_warn("%s: can not open GPIO %d\n",
-			__func__, DA830_MMCSD_CD_PIN);
-		return;
-	}
-	gpio_direction_input(DA830_MMCSD_CD_PIN);
+	gpiod_add_lookup_table(&mmc_gpios_table);
 
 	ret = da8xx_register_mmcsd0(&da830_evm_mmc_config);
 	if (ret) {
 		pr_warn("%s: mmc/sd registration failed: %d\n", __func__, ret);
-		gpio_free(DA830_MMCSD_WP_PIN);
+		gpiod_remove_lookup_table(&mmc_gpios_table);
 	}
 }
 
@@ -588,6 +551,10 @@ static __init void da830_evm_init(void)
 	struct davinci_soc_info *soc_info = &davinci_soc_info;
 	int ret;
 
+	ret = da8xx_register_cfgchip();
+	if (ret)
+		pr_warn("%s: CFGCHIP registration failed: %d\n", __func__, ret);
+
 	ret = da830_register_gpio();
 	if (ret)
 		pr_warn("%s: GPIO init failed: %d\n", __func__, ret);
@@ -647,6 +614,8 @@ static __init void da830_evm_init(void)
 	ret = da8xx_register_spi_bus(0, ARRAY_SIZE(da830evm_spi_info));
 	if (ret)
 		pr_warn("%s: spi 0 registration failed: %d\n", __func__, ret);
+
+	regulator_has_full_constraints();
 }
 
 #ifdef CONFIG_SERIAL_8250_CONSOLE
diff --git a/arch/arm/mach-davinci/board-da850-evm.c b/arch/arm/mach-davinci/board-da850-evm.c
index 8e4539f69fdc..aac3ab1a044f 100644
--- a/arch/arm/mach-davinci/board-da850-evm.c
+++ b/arch/arm/mach-davinci/board-da850-evm.c
@@ -15,6 +15,7 @@
 #include <linux/delay.h>
 #include <linux/gpio.h>
 #include <linux/gpio_keys.h>
+#include <linux/gpio/machine.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/i2c.h>
@@ -56,9 +57,6 @@
 #define DA850_LCD_PWR_PIN		GPIO_TO_PIN(2, 8)
 #define DA850_LCD_BL_PIN		GPIO_TO_PIN(2, 15)
 
-#define DA850_MMCSD_CD_PIN		GPIO_TO_PIN(4, 0)
-#define DA850_MMCSD_WP_PIN		GPIO_TO_PIN(4, 1)
-
 #define DA850_MII_MDIO_CLKEN_PIN	GPIO_TO_PIN(2, 6)
 
 static struct mtd_partition da850evm_spiflash_part[] = {
@@ -196,18 +194,6 @@ static struct platform_device da850_evm_norflash_device = {
 	.resource	= da850_evm_norflash_resource,
 };
 
-static struct davinci_pm_config da850_pm_pdata = {
-	.sleepcount = 128,
-};
-
-static struct platform_device da850_pm_device = {
-	.name           = "pm-davinci",
-	.dev = {
-		.platform_data	= &da850_pm_pdata,
-	},
-	.id             = -1,
-};
-
 /* DA850/OMAP-L138 EVM includes a 512 MByte large-page NAND flash
  * (128K blocks). It may be used instead of the (default) SPI flash
  * to boot, using TI's tools to install the secondary boot loader
@@ -776,19 +762,16 @@ static const short da850_evm_mcasp_pins[] __initconst = {
 	-1
 };
 
-static int da850_evm_mmc_get_ro(int index)
-{
-	return gpio_get_value(DA850_MMCSD_WP_PIN);
-}
-
-static int da850_evm_mmc_get_cd(int index)
-{
-	return !gpio_get_value(DA850_MMCSD_CD_PIN);
-}
+static struct gpiod_lookup_table mmc_gpios_table = {
+	.dev_id = "da830-mmc.0",
+	.table = {
+		/* gpio chip 2 contains gpio range 64-95 */
+		GPIO_LOOKUP("davinci_gpio.2", 0, "cd", GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP("davinci_gpio.2", 1, "wp", GPIO_ACTIVE_LOW),
+	},
+};
 
 static struct davinci_mmc_config da850_mmc_config = {
-	.get_ro		= da850_evm_mmc_get_ro,
-	.get_cd		= da850_evm_mmc_get_cd,
 	.wires		= 4,
 	.max_freq	= 50000000,
 	.caps		= MMC_CAP_MMC_HIGHSPEED | MMC_CAP_SD_HIGHSPEED,
@@ -1345,6 +1328,10 @@ static __init void da850_evm_init(void)
 {
 	int ret;
 
+	ret = da8xx_register_cfgchip();
+	if (ret)
+		pr_warn("%s: CFGCHIP registration failed: %d\n", __func__, ret);
+
 	ret = da850_register_gpio();
 	if (ret)
 		pr_warn("%s: GPIO init failed: %d\n", __func__, ret);
@@ -1379,17 +1366,7 @@ static __init void da850_evm_init(void)
 			pr_warn("%s: MMCSD0 mux setup failed: %d\n",
 				__func__, ret);
 
-		ret = gpio_request(DA850_MMCSD_CD_PIN, "MMC CD\n");
-		if (ret)
-			pr_warn("%s: can not open GPIO %d\n",
-				__func__, DA850_MMCSD_CD_PIN);
-		gpio_direction_input(DA850_MMCSD_CD_PIN);
-
-		ret = gpio_request(DA850_MMCSD_WP_PIN, "MMC WP\n");
-		if (ret)
-			pr_warn("%s: can not open GPIO %d\n",
-				__func__, DA850_MMCSD_WP_PIN);
-		gpio_direction_input(DA850_MMCSD_WP_PIN);
+		gpiod_add_lookup_table(&mmc_gpios_table);
 
 		ret = da8xx_register_mmcsd0(&da850_mmc_config);
 		if (ret)
@@ -1453,10 +1430,7 @@ static __init void da850_evm_init(void)
 	if (ret)
 		pr_warn("%s: cpuidle registration failed: %d\n", __func__, ret);
 
-	ret = da850_register_pm(&da850_pm_device);
-	if (ret)
-		pr_warn("%s: suspend registration failed: %d\n", __func__, ret);
-
+	davinci_pm_init();
 	da850_vpif_init();
 
 	ret = spi_register_board_info(da850evm_spi_info,
diff --git a/arch/arm/mach-davinci/board-mityomapl138.c b/arch/arm/mach-davinci/board-mityomapl138.c
index bc4e63fa9808..b73ce7bae81f 100644
--- a/arch/arm/mach-davinci/board-mityomapl138.c
+++ b/arch/arm/mach-davinci/board-mityomapl138.c
@@ -498,22 +498,14 @@ static void __init mityomapl138_config_emac(void)
 		pr_warn("emac registration failed: %d\n", ret);
 }
 
-static struct davinci_pm_config da850_pm_pdata = {
-	.sleepcount = 128,
-};
-
-static struct platform_device da850_pm_device = {
-	.name	= "pm-davinci",
-	.dev = {
-		.platform_data  = &da850_pm_pdata,
-	},
-	.id	= -1,
-};
-
 static void __init mityomapl138_init(void)
 {
 	int ret;
 
+	ret = da8xx_register_cfgchip();
+	if (ret)
+		pr_warn("%s: CFGCHIP registration failed: %d\n", __func__, ret);
+
 	/* for now, no special EDMA channels are reserved */
 	ret = da850_register_edma(NULL);
 	if (ret)
@@ -555,9 +547,7 @@ static void __init mityomapl138_init(void)
 	if (ret)
 		pr_warn("cpuidle registration failed: %d\n", ret);
 
-	ret = da850_register_pm(&da850_pm_device);
-	if (ret)
-		pr_warn("suspend registration failed: %d\n", ret);
+	davinci_pm_init();
 }
 
 #ifdef CONFIG_SERIAL_8250_CONSOLE
diff --git a/arch/arm/mach-davinci/board-omapl138-hawk.c b/arch/arm/mach-davinci/board-omapl138-hawk.c
index ee624861ca66..41d5500996b2 100644
--- a/arch/arm/mach-davinci/board-omapl138-hawk.c
+++ b/arch/arm/mach-davinci/board-omapl138-hawk.c
@@ -13,7 +13,9 @@
 #include <linux/init.h>
 #include <linux/console.h>
 #include <linux/gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/platform_data/gpio-davinci.h>
+#include <linux/regulator/machine.h>
 
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
@@ -24,8 +26,6 @@
 #include <mach/mux.h>
 
 #define HAWKBOARD_PHY_ID		"davinci_mdio-0:07"
-#define DA850_HAWK_MMCSD_CD_PIN		GPIO_TO_PIN(3, 12)
-#define DA850_HAWK_MMCSD_WP_PIN		GPIO_TO_PIN(3, 13)
 
 #define DA850_USB1_VBUS_PIN		GPIO_TO_PIN(2, 4)
 #define DA850_USB1_OC_PIN		GPIO_TO_PIN(6, 13)
@@ -122,19 +122,16 @@ static const short hawk_mmcsd0_pins[] = {
 	-1
 };
 
-static int da850_hawk_mmc_get_ro(int index)
-{
-	return gpio_get_value(DA850_HAWK_MMCSD_WP_PIN);
-}
-
-static int da850_hawk_mmc_get_cd(int index)
-{
-	return !gpio_get_value(DA850_HAWK_MMCSD_CD_PIN);
-}
+static struct gpiod_lookup_table mmc_gpios_table = {
+	.dev_id = "da830-mmc.0",
+	.table = {
+		/* CD: gpio3_12: gpio60: chip 1 contains gpio range 32-63*/
+		GPIO_LOOKUP("davinci_gpio.1", 28, "cd", GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP("davinci_gpio.1", 29, "wp", GPIO_ACTIVE_LOW),
+	},
+};
 
 static struct davinci_mmc_config da850_mmc_config = {
-	.get_ro		= da850_hawk_mmc_get_ro,
-	.get_cd		= da850_hawk_mmc_get_cd,
 	.wires		= 4,
 	.max_freq	= 50000000,
 	.caps		= MMC_CAP_MMC_HIGHSPEED | MMC_CAP_SD_HIGHSPEED,
@@ -150,21 +147,7 @@ static __init void omapl138_hawk_mmc_init(void)
 		return;
 	}
 
-	ret = gpio_request_one(DA850_HAWK_MMCSD_CD_PIN,
-			GPIOF_DIR_IN, "MMC CD");
-	if (ret < 0) {
-		pr_warn("%s: can not open GPIO %d\n",
-			__func__, DA850_HAWK_MMCSD_CD_PIN);
-		return;
-	}
-
-	ret = gpio_request_one(DA850_HAWK_MMCSD_WP_PIN,
-			GPIOF_DIR_IN, "MMC WP");
-	if (ret < 0) {
-		pr_warn("%s: can not open GPIO %d\n",
-			__func__, DA850_HAWK_MMCSD_WP_PIN);
-		goto mmc_setup_wp_fail;
-	}
+	gpiod_add_lookup_table(&mmc_gpios_table);
 
 	ret = da8xx_register_mmcsd0(&da850_mmc_config);
 	if (ret) {
@@ -175,9 +158,7 @@ static __init void omapl138_hawk_mmc_init(void)
 	return;
 
 mmc_setup_mmcsd_fail:
-	gpio_free(DA850_HAWK_MMCSD_WP_PIN);
-mmc_setup_wp_fail:
-	gpio_free(DA850_HAWK_MMCSD_CD_PIN);
+	gpiod_remove_lookup_table(&mmc_gpios_table);
 }
 
 static irqreturn_t omapl138_hawk_usb_ocic_irq(int irq, void *dev_id);
@@ -243,7 +224,6 @@ static irqreturn_t omapl138_hawk_usb_ocic_irq(int irq, void *dev_id)
 static __init void omapl138_hawk_usb_init(void)
 {
 	int ret;
-	u32 cfgchip2;
 
 	ret = davinci_cfg_reg_list(da850_hawk_usb11_pins);
 	if (ret) {
@@ -251,12 +231,20 @@ static __init void omapl138_hawk_usb_init(void)
 		return;
 	}
 
-	/* Setup the Ref. clock frequency for the HAWK at 24 MHz. */
+	ret = da8xx_register_usb20_phy_clk(false);
+	if (ret)
+		pr_warn("%s: USB 2.0 PHY CLK registration failed: %d\n",
+			__func__, ret);
 
-	cfgchip2 = __raw_readl(DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
-	cfgchip2 &= ~CFGCHIP2_REFFREQ;
-	cfgchip2 |=  CFGCHIP2_REFFREQ_24MHZ;
-	__raw_writel(cfgchip2, DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
+	ret = da8xx_register_usb11_phy_clk(false);
+	if (ret)
+		pr_warn("%s: USB 1.1 PHY CLK registration failed: %d\n",
+			__func__, ret);
+
+	ret = da8xx_register_usb_phy();
+	if (ret)
+		pr_warn("%s: USB PHY registration failed: %d\n",
+			__func__, ret);
 
 	ret = gpio_request_one(DA850_USB1_VBUS_PIN,
 			GPIOF_DIR_OUT, "USB1 VBUS");
@@ -292,6 +280,10 @@ static __init void omapl138_hawk_init(void)
 {
 	int ret;
 
+	ret = da8xx_register_cfgchip();
+	if (ret)
+		pr_warn("%s: CFGCHIP registration failed: %d\n", __func__, ret);
+
 	ret = da850_register_gpio();
 	if (ret)
 		pr_warn("%s: GPIO init failed: %d\n", __func__, ret);
@@ -317,6 +309,8 @@ static __init void omapl138_hawk_init(void)
 	if (ret)
 		pr_warn("%s: dsp/rproc registration failed: %d\n",
 			__func__, ret);
+
+	regulator_has_full_constraints();
 }
 
 #ifdef CONFIG_SERIAL_8250_CONSOLE
diff --git a/arch/arm/mach-davinci/common.c b/arch/arm/mach-davinci/common.c
index 049025f6d531..9f9fbfa6da0d 100644
--- a/arch/arm/mach-davinci/common.c
+++ b/arch/arm/mach-davinci/common.c
@@ -118,6 +118,5 @@ err:
 void __init davinci_init_late(void)
 {
 	davinci_cpufreq_init();
-	davinci_pm_init();
 	davinci_clk_disable_unused();
 }
diff --git a/arch/arm/mach-davinci/da830.c b/arch/arm/mach-davinci/da830.c
index 426fd7477357..073c458d0c67 100644
--- a/arch/arm/mach-davinci/da830.c
+++ b/arch/arm/mach-davinci/da830.c
@@ -412,7 +412,7 @@ static struct clk_lookup da830_clks[] = {
 	CLK("davinci-mcasp.0",	NULL,		&mcasp0_clk),
 	CLK("davinci-mcasp.1",	NULL,		&mcasp1_clk),
 	CLK("davinci-mcasp.2",	NULL,		&mcasp2_clk),
-	CLK(NULL,		"usb20",	&usb20_clk),
+	CLK("musb-da8xx",	"usb20",	&usb20_clk),
 	CLK(NULL,		"aemif",	&aemif_clk),
 	CLK(NULL,		"aintc",	&aintc_clk),
 	CLK(NULL,		"secu_mgr",	&secu_mgr_clk),
@@ -420,7 +420,7 @@ static struct clk_lookup da830_clks[] = {
 	CLK("davinci_mdio.0",   "fck",          &emac_clk),
 	CLK(NULL,		"gpio",		&gpio_clk),
 	CLK("i2c_davinci.2",	NULL,		&i2c1_clk),
-	CLK(NULL,		"usb11",	&usb11_clk),
+	CLK("ohci-da8xx",	"usb11",	&usb11_clk),
 	CLK(NULL,		"emif3",	&emif3_clk),
 	CLK(NULL,		"arm",		&arm_clk),
 	CLK(NULL,		"rmii",		&rmii_clk),
diff --git a/arch/arm/mach-davinci/da850.c b/arch/arm/mach-davinci/da850.c
index ed3d0e9f72ac..e770c97ea45c 100644
--- a/arch/arm/mach-davinci/da850.c
+++ b/arch/arm/mach-davinci/da850.c
@@ -424,6 +424,16 @@ static struct clk ehrpwm_clk = {
 	.gpsc		= 1,
 };
 
+static struct clk ehrpwm0_clk = {
+	.name		= "ehrpwm0",
+	.parent		= &ehrpwm_clk,
+};
+
+static struct clk ehrpwm1_clk = {
+	.name		= "ehrpwm1",
+	.parent		= &ehrpwm_clk,
+};
+
 #define DA8XX_EHRPWM_TBCLKSYNC	BIT(12)
 
 static void ehrpwm_tblck_enable(struct clk *clk)
@@ -451,6 +461,16 @@ static struct clk ehrpwm_tbclk = {
 	.clk_disable	= ehrpwm_tblck_disable,
 };
 
+static struct clk ehrpwm0_tbclk = {
+	.name		= "ehrpwm0_tbclk",
+	.parent		= &ehrpwm_tbclk,
+};
+
+static struct clk ehrpwm1_tbclk = {
+	.name		= "ehrpwm1_tbclk",
+	.parent		= &ehrpwm_tbclk,
+};
+
 static struct clk ecap_clk = {
 	.name		= "ecap",
 	.parent		= &async3_clk,
@@ -458,6 +478,21 @@ static struct clk ecap_clk = {
 	.gpsc		= 1,
 };
 
+static struct clk ecap0_clk = {
+	.name		= "ecap0_clk",
+	.parent		= &ecap_clk,
+};
+
+static struct clk ecap1_clk = {
+	.name		= "ecap1_clk",
+	.parent		= &ecap_clk,
+};
+
+static struct clk ecap2_clk = {
+	.name		= "ecap2_clk",
+	.parent		= &ecap_clk,
+};
+
 static struct clk_lookup da850_clks[] = {
 	CLK(NULL,		"ref",		&ref_clk),
 	CLK(NULL,		"pll0",		&pll0_clk),
@@ -503,16 +538,23 @@ static struct clk_lookup da850_clks[] = {
 	CLK("da830-mmc.1",	NULL,		&mmcsd1_clk),
 	CLK("ti-aemif",		NULL,		&aemif_clk),
 	CLK(NULL,		"aemif",	&aemif_clk),
-	CLK(NULL,		"usb11",	&usb11_clk),
-	CLK(NULL,		"usb20",	&usb20_clk),
+	CLK("ohci-da8xx",	"usb11",	&usb11_clk),
+	CLK("musb-da8xx",	"usb20",	&usb20_clk),
 	CLK("spi_davinci.0",	NULL,		&spi0_clk),
 	CLK("spi_davinci.1",	NULL,		&spi1_clk),
 	CLK("vpif",		NULL,		&vpif_clk),
 	CLK("ahci_da850",		NULL,		&sata_clk),
 	CLK("davinci-rproc.0",	NULL,		&dsp_clk),
-	CLK("ehrpwm",		"fck",		&ehrpwm_clk),
-	CLK("ehrpwm",		"tbclk",	&ehrpwm_tbclk),
-	CLK("ecap",		"fck",		&ecap_clk),
+	CLK(NULL,		NULL,		&ehrpwm_clk),
+	CLK("ehrpwm.0",		"fck",		&ehrpwm0_clk),
+	CLK("ehrpwm.1",		"fck",		&ehrpwm1_clk),
+	CLK(NULL,		NULL,		&ehrpwm_tbclk),
+	CLK("ehrpwm.0",		"tbclk",	&ehrpwm0_tbclk),
+	CLK("ehrpwm.1",		"tbclk",	&ehrpwm1_tbclk),
+	CLK(NULL,		NULL,		&ecap_clk),
+	CLK("ecap.0",		"fck",		&ecap0_clk),
+	CLK("ecap.1",		"fck",		&ecap1_clk),
+	CLK("ecap.2",		"fck",		&ecap2_clk),
 	CLK(NULL,		NULL,		NULL),
 };
 
@@ -1172,44 +1214,6 @@ static int da850_round_armrate(struct clk *clk, unsigned long rate)
 }
 #endif
 
-int __init da850_register_pm(struct platform_device *pdev)
-{
-	int ret;
-	struct davinci_pm_config *pdata = pdev->dev.platform_data;
-
-	ret = davinci_cfg_reg(DA850_RTC_ALARM);
-	if (ret)
-		return ret;
-
-	pdata->ddr2_ctlr_base = da8xx_get_mem_ctlr();
-	pdata->deepsleep_reg = DA8XX_SYSCFG1_VIRT(DA8XX_DEEPSLEEP_REG);
-	pdata->ddrpsc_num = DA8XX_LPSC1_EMIF3C;
-
-	pdata->cpupll_reg_base = ioremap(DA8XX_PLL0_BASE, SZ_4K);
-	if (!pdata->cpupll_reg_base)
-		return -ENOMEM;
-
-	pdata->ddrpll_reg_base = ioremap(DA850_PLL1_BASE, SZ_4K);
-	if (!pdata->ddrpll_reg_base) {
-		ret = -ENOMEM;
-		goto no_ddrpll_mem;
-	}
-
-	pdata->ddrpsc_reg_base = ioremap(DA8XX_PSC1_BASE, SZ_4K);
-	if (!pdata->ddrpsc_reg_base) {
-		ret = -ENOMEM;
-		goto no_ddrpsc_mem;
-	}
-
-	return platform_device_register(pdev);
-
-no_ddrpsc_mem:
-	iounmap(pdata->ddrpll_reg_base);
-no_ddrpll_mem:
-	iounmap(pdata->cpupll_reg_base);
-	return ret;
-}
-
 /* VPIF resource, platform data */
 static u64 da850_vpif_dma_mask = DMA_BIT_MASK(32);
 
diff --git a/arch/arm/mach-davinci/da8xx-dt.c b/arch/arm/mach-davinci/da8xx-dt.c
index c9f7e9274aa8..9ee44da6eb7b 100644
--- a/arch/arm/mach-davinci/da8xx-dt.c
+++ b/arch/arm/mach-davinci/da8xx-dt.c
@@ -23,11 +23,11 @@ static struct of_dev_auxdata da850_auxdata_lookup[] __initdata = {
 	OF_DEV_AUXDATA("ti,davinci-i2c", 0x01e28000, "i2c_davinci.2", NULL),
 	OF_DEV_AUXDATA("ti,davinci-wdt", 0x01c21000, "davinci-wdt", NULL),
 	OF_DEV_AUXDATA("ti,da830-mmc", 0x01c40000, "da830-mmc.0", NULL),
-	OF_DEV_AUXDATA("ti,da850-ehrpwm", 0x01f00000, "ehrpwm", NULL),
-	OF_DEV_AUXDATA("ti,da850-ehrpwm", 0x01f02000, "ehrpwm", NULL),
-	OF_DEV_AUXDATA("ti,da850-ecap", 0x01f06000, "ecap", NULL),
-	OF_DEV_AUXDATA("ti,da850-ecap", 0x01f07000, "ecap", NULL),
-	OF_DEV_AUXDATA("ti,da850-ecap", 0x01f08000, "ecap", NULL),
+	OF_DEV_AUXDATA("ti,da850-ehrpwm", 0x01f00000, "ehrpwm.0", NULL),
+	OF_DEV_AUXDATA("ti,da850-ehrpwm", 0x01f02000, "ehrpwm.1", NULL),
+	OF_DEV_AUXDATA("ti,da850-ecap", 0x01f06000, "ecap.0", NULL),
+	OF_DEV_AUXDATA("ti,da850-ecap", 0x01f07000, "ecap.1", NULL),
+	OF_DEV_AUXDATA("ti,da850-ecap", 0x01f08000, "ecap.2", NULL),
 	OF_DEV_AUXDATA("ti,da830-spi", 0x01c41000, "spi_davinci.0", NULL),
 	OF_DEV_AUXDATA("ti,da830-spi", 0x01f0e000, "spi_davinci.1", NULL),
 	OF_DEV_AUXDATA("ns16550a", 0x01c42000, "serial8250.0", NULL),
@@ -38,6 +38,10 @@ static struct of_dev_auxdata da850_auxdata_lookup[] __initdata = {
 		       NULL),
 	OF_DEV_AUXDATA("ti,da830-mcasp-audio", 0x01d00000, "davinci-mcasp.0", NULL),
 	OF_DEV_AUXDATA("ti,da850-aemif", 0x68000000, "ti-aemif", NULL),
+	OF_DEV_AUXDATA("ti,da850-tilcdc", 0x01e13000, "da8xx_lcdc.0", NULL),
+	OF_DEV_AUXDATA("ti,da830-ohci", 0x01e25000, "ohci-da8xx", NULL),
+	OF_DEV_AUXDATA("ti,da830-musb", 0x01e00000, "musb-da8xx", NULL),
+	OF_DEV_AUXDATA("ti,da830-usb-phy", 0x01c1417c, "da8xx-usb-phy", NULL),
 	{}
 };
 
@@ -45,7 +49,19 @@ static struct of_dev_auxdata da850_auxdata_lookup[] __initdata = {
 
 static void __init da850_init_machine(void)
 {
+	int ret;
+
+	ret = da8xx_register_usb20_phy_clk(false);
+	if (ret)
+		pr_warn("%s: registering USB 2.0 PHY clock failed: %d",
+			__func__, ret);
+	ret = da8xx_register_usb11_phy_clk(false);
+	if (ret)
+		pr_warn("%s: registering USB 1.1 PHY clock failed: %d",
+			__func__, ret);
+
 	of_platform_default_populate(NULL, da850_auxdata_lookup, NULL);
+	davinci_pm_init();
 }
 
 static const char *const da850_boards_compat[] __initconst = {
diff --git a/arch/arm/mach-davinci/devices-da8xx.c b/arch/arm/mach-davinci/devices-da8xx.c
index add3771d38f6..c2457b3fdb5f 100644
--- a/arch/arm/mach-davinci/devices-da8xx.c
+++ b/arch/arm/mach-davinci/devices-da8xx.c
@@ -11,6 +11,7 @@
  * (at your option) any later version.
  */
 #include <linux/init.h>
+#include <linux/platform_data/syscon.h>
 #include <linux/platform_device.h>
 #include <linux/dma-contiguous.h>
 #include <linux/serial_8250.h>
@@ -57,15 +58,6 @@
 #define DA8XX_EMAC_RAM_OFFSET		0x0000
 #define DA8XX_EMAC_CTRL_RAM_SIZE	SZ_8K
 
-#define DA8XX_DMA_SPI0_RX	EDMA_CTLR_CHAN(0, 14)
-#define DA8XX_DMA_SPI0_TX	EDMA_CTLR_CHAN(0, 15)
-#define DA8XX_DMA_MMCSD0_RX	EDMA_CTLR_CHAN(0, 16)
-#define DA8XX_DMA_MMCSD0_TX	EDMA_CTLR_CHAN(0, 17)
-#define DA8XX_DMA_SPI1_RX	EDMA_CTLR_CHAN(0, 18)
-#define DA8XX_DMA_SPI1_TX	EDMA_CTLR_CHAN(0, 19)
-#define DA850_DMA_MMCSD1_RX	EDMA_CTLR_CHAN(1, 28)
-#define DA850_DMA_MMCSD1_TX	EDMA_CTLR_CHAN(1, 29)
-
 void __iomem *da8xx_syscfg0_base;
 void __iomem *da8xx_syscfg1_base;
 
@@ -964,16 +956,6 @@ static struct resource da8xx_spi0_resources[] = {
 		.end	= IRQ_DA8XX_SPINT0,
 		.flags	= IORESOURCE_IRQ,
 	},
-	[2] = {
-		.start	= DA8XX_DMA_SPI0_RX,
-		.end	= DA8XX_DMA_SPI0_RX,
-		.flags	= IORESOURCE_DMA,
-	},
-	[3] = {
-		.start	= DA8XX_DMA_SPI0_TX,
-		.end	= DA8XX_DMA_SPI0_TX,
-		.flags	= IORESOURCE_DMA,
-	},
 };
 
 static struct resource da8xx_spi1_resources[] = {
@@ -987,16 +969,6 @@ static struct resource da8xx_spi1_resources[] = {
 		.end	= IRQ_DA8XX_SPINT1,
 		.flags	= IORESOURCE_IRQ,
 	},
-	[2] = {
-		.start	= DA8XX_DMA_SPI1_RX,
-		.end	= DA8XX_DMA_SPI1_RX,
-		.flags	= IORESOURCE_DMA,
-	},
-	[3] = {
-		.start	= DA8XX_DMA_SPI1_TX,
-		.end	= DA8XX_DMA_SPI1_TX,
-		.flags	= IORESOURCE_DMA,
-	},
 };
 
 static struct davinci_spi_platform_data da8xx_spi_pdata[] = {
@@ -1089,3 +1061,30 @@ int __init da850_register_sata(unsigned long refclkpn)
 	return platform_device_register(&da850_sata_device);
 }
 #endif
+
+static struct syscon_platform_data da8xx_cfgchip_platform_data = {
+	.label	= "cfgchip",
+};
+
+static struct resource da8xx_cfgchip_resources[] = {
+	{
+		.start	= DA8XX_SYSCFG0_BASE + DA8XX_CFGCHIP0_REG,
+		.end	= DA8XX_SYSCFG0_BASE + DA8XX_CFGCHIP4_REG + 3,
+		.flags	= IORESOURCE_MEM,
+	},
+};
+
+static struct platform_device da8xx_cfgchip_device = {
+	.name	= "syscon",
+	.id	= -1,
+	.dev	= {
+		.platform_data	= &da8xx_cfgchip_platform_data,
+	},
+	.num_resources	= ARRAY_SIZE(da8xx_cfgchip_resources),
+	.resource	= da8xx_cfgchip_resources,
+};
+
+int __init da8xx_register_cfgchip(void)
+{
+	return platform_device_register(&da8xx_cfgchip_device);
+}
diff --git a/arch/arm/mach-davinci/devices.c b/arch/arm/mach-davinci/devices.c
index 67d26c5bda0b..3ae70f2909b0 100644
--- a/arch/arm/mach-davinci/devices.c
+++ b/arch/arm/mach-davinci/devices.c
@@ -36,9 +36,6 @@
 #define DM365_MMCSD0_BASE	     0x01D11000
 #define DM365_MMCSD1_BASE	     0x01D00000
 
-#define DAVINCI_DMA_MMCRXEVT	26
-#define DAVINCI_DMA_MMCTXEVT	27
-
 void __iomem  *davinci_sysmod_base;
 
 void davinci_map_sysmod(void)
diff --git a/arch/arm/mach-davinci/dm355.c b/arch/arm/mach-davinci/dm355.c
index d33322ddedab..bd50367f654e 100644
--- a/arch/arm/mach-davinci/dm355.c
+++ b/arch/arm/mach-davinci/dm355.c
@@ -397,14 +397,6 @@ static struct resource dm355_spi0_resources[] = {
 		.start = IRQ_DM355_SPINT0_0,
 		.flags = IORESOURCE_IRQ,
 	},
-	{
-		.start = 17,
-		.flags = IORESOURCE_DMA,
-	},
-	{
-		.start = 16,
-		.flags = IORESOURCE_DMA,
-	},
 };
 
 static struct davinci_spi_platform_data dm355_spi0_pdata = {
diff --git a/arch/arm/mach-davinci/dm365.c b/arch/arm/mach-davinci/dm365.c
index ef3add999263..8be04ec95adf 100644
--- a/arch/arm/mach-davinci/dm365.c
+++ b/arch/arm/mach-davinci/dm365.c
@@ -660,14 +660,6 @@ static struct resource dm365_spi0_resources[] = {
 		.start = IRQ_DM365_SPIINT0_0,
 		.flags = IORESOURCE_IRQ,
 	},
-	{
-		.start = 17,
-		.flags = IORESOURCE_DMA,
-	},
-	{
-		.start = 16,
-		.flags = IORESOURCE_DMA,
-	},
 };
 
 static struct platform_device dm365_spi0_device = {
diff --git a/arch/arm/mach-davinci/include/mach/da8xx.h b/arch/arm/mach-davinci/include/mach/da8xx.h
index f9f9713aacdd..85ff2183b6db 100644
--- a/arch/arm/mach-davinci/include/mach/da8xx.h
+++ b/arch/arm/mach-davinci/include/mach/da8xx.h
@@ -61,6 +61,7 @@ extern unsigned int da850_max_speed;
 #define DA8XX_CFGCHIP1_REG	0x180
 #define DA8XX_CFGCHIP2_REG	0x184
 #define DA8XX_CFGCHIP3_REG	0x188
+#define DA8XX_CFGCHIP4_REG	0x18c
 
 #define DA8XX_SYSCFG1_BASE	(IO_PHYS + 0x22C000)
 #define DA8XX_SYSCFG1_VIRT(x)	(da8xx_syscfg1_base + (x))
@@ -88,8 +89,12 @@ int da850_register_edma(struct edma_rsv_info *rsv[2]);
 int da8xx_register_i2c(int instance, struct davinci_i2c_platform_data *pdata);
 int da8xx_register_spi_bus(int instance, unsigned num_chipselect);
 int da8xx_register_watchdog(void);
+int da8xx_register_usb_phy(void);
 int da8xx_register_usb20(unsigned mA, unsigned potpgt);
 int da8xx_register_usb11(struct da8xx_ohci_root_hub *pdata);
+int da8xx_register_usb_refclkin(int rate);
+int da8xx_register_usb20_phy_clk(bool use_usb_refclkin);
+int da8xx_register_usb11_phy_clk(bool use_usb_refclkin);
 int da8xx_register_emac(void);
 int da8xx_register_uio_pruss(void);
 int da8xx_register_lcdc(struct da8xx_lcdc_platform_data *pdata);
@@ -101,7 +106,6 @@ int da8xx_register_gpio(void *pdata);
 int da850_register_cpufreq(char *async_clk);
 int da8xx_register_cpuidle(void);
 void __iomem *da8xx_get_mem_ctlr(void);
-int da850_register_pm(struct platform_device *pdev);
 int da850_register_sata(unsigned long refclkpn);
 int da850_register_vpif(void);
 int da850_register_vpif_display
@@ -113,6 +117,7 @@ void da8xx_rproc_reserve_cma(void);
 int da8xx_register_rproc(void);
 int da850_register_gpio(void);
 int da830_register_gpio(void);
+int da8xx_register_cfgchip(void);
 
 extern struct platform_device da8xx_serial_device[];
 extern struct emac_platform_data da8xx_emac_pdata;
diff --git a/arch/arm/mach-davinci/pm.c b/arch/arm/mach-davinci/pm.c
index 8929569b1f8a..0afd201ab980 100644
--- a/arch/arm/mach-davinci/pm.c
+++ b/arch/arm/mach-davinci/pm.c
@@ -21,15 +21,22 @@
 
 #include <mach/common.h>
 #include <mach/da8xx.h>
-#include "sram.h"
+#include <mach/mux.h>
 #include <mach/pm.h>
 
 #include "clock.h"
+#include "psc.h"
+#include "sram.h"
 
+#define DA850_PLL1_BASE		0x01e1a000
 #define DEEPSLEEP_SLEEPCOUNT_MASK	0xFFFF
+#define DEEPSLEEP_SLEEPCOUNT		128
 
 static void (*davinci_sram_suspend) (struct davinci_pm_config *);
-static struct davinci_pm_config *pdata;
+static struct davinci_pm_config pm_config = {
+	.sleepcount = DEEPSLEEP_SLEEPCOUNT,
+	.ddrpsc_num = DA8XX_LPSC1_EMIF3C,
+};
 
 static void davinci_sram_push(void *dest, void *src, unsigned int size)
 {
@@ -41,58 +48,58 @@ static void davinci_pm_suspend(void)
 {
 	unsigned val;
 
-	if (pdata->cpupll_reg_base != pdata->ddrpll_reg_base) {
+	if (pm_config.cpupll_reg_base != pm_config.ddrpll_reg_base) {
 
 		/* Switch CPU PLL to bypass mode */
-		val = __raw_readl(pdata->cpupll_reg_base + PLLCTL);
+		val = __raw_readl(pm_config.cpupll_reg_base + PLLCTL);
 		val &= ~(PLLCTL_PLLENSRC | PLLCTL_PLLEN);
-		__raw_writel(val, pdata->cpupll_reg_base + PLLCTL);
+		__raw_writel(val, pm_config.cpupll_reg_base + PLLCTL);
 
 		udelay(PLL_BYPASS_TIME);
 
 		/* Powerdown CPU PLL */
-		val = __raw_readl(pdata->cpupll_reg_base + PLLCTL);
+		val = __raw_readl(pm_config.cpupll_reg_base + PLLCTL);
 		val |= PLLCTL_PLLPWRDN;
-		__raw_writel(val, pdata->cpupll_reg_base + PLLCTL);
+		__raw_writel(val, pm_config.cpupll_reg_base + PLLCTL);
 	}
 
 	/* Configure sleep count in deep sleep register */
-	val = __raw_readl(pdata->deepsleep_reg);
+	val = __raw_readl(pm_config.deepsleep_reg);
 	val &= ~DEEPSLEEP_SLEEPCOUNT_MASK,
-	val |= pdata->sleepcount;
-	__raw_writel(val, pdata->deepsleep_reg);
+	val |= pm_config.sleepcount;
+	__raw_writel(val, pm_config.deepsleep_reg);
 
 	/* System goes to sleep in this call */
-	davinci_sram_suspend(pdata);
+	davinci_sram_suspend(&pm_config);
 
-	if (pdata->cpupll_reg_base != pdata->ddrpll_reg_base) {
+	if (pm_config.cpupll_reg_base != pm_config.ddrpll_reg_base) {
 
 		/* put CPU PLL in reset */
-		val = __raw_readl(pdata->cpupll_reg_base + PLLCTL);
+		val = __raw_readl(pm_config.cpupll_reg_base + PLLCTL);
 		val &= ~PLLCTL_PLLRST;
-		__raw_writel(val, pdata->cpupll_reg_base + PLLCTL);
+		__raw_writel(val, pm_config.cpupll_reg_base + PLLCTL);
 
 		/* put CPU PLL in power down */
-		val = __raw_readl(pdata->cpupll_reg_base + PLLCTL);
+		val = __raw_readl(pm_config.cpupll_reg_base + PLLCTL);
 		val &= ~PLLCTL_PLLPWRDN;
-		__raw_writel(val, pdata->cpupll_reg_base + PLLCTL);
+		__raw_writel(val, pm_config.cpupll_reg_base + PLLCTL);
 
 		/* wait for CPU PLL reset */
 		udelay(PLL_RESET_TIME);
 
 		/* bring CPU PLL out of reset */
-		val = __raw_readl(pdata->cpupll_reg_base + PLLCTL);
+		val = __raw_readl(pm_config.cpupll_reg_base + PLLCTL);
 		val |= PLLCTL_PLLRST;
-		__raw_writel(val, pdata->cpupll_reg_base + PLLCTL);
+		__raw_writel(val, pm_config.cpupll_reg_base + PLLCTL);
 
 		/* Wait for CPU PLL to lock */
 		udelay(PLL_LOCK_TIME);
 
 		/* Remove CPU PLL from bypass mode */
-		val = __raw_readl(pdata->cpupll_reg_base + PLLCTL);
+		val = __raw_readl(pm_config.cpupll_reg_base + PLLCTL);
 		val &= ~PLLCTL_PLLENSRC;
 		val |= PLLCTL_PLLEN;
-		__raw_writel(val, pdata->cpupll_reg_base + PLLCTL);
+		__raw_writel(val, pm_config.cpupll_reg_base + PLLCTL);
 	}
 }
 
@@ -117,17 +124,36 @@ static const struct platform_suspend_ops davinci_pm_ops = {
 	.valid		= suspend_valid_only_mem,
 };
 
-static int __init davinci_pm_probe(struct platform_device *pdev)
+int __init davinci_pm_init(void)
 {
-	pdata = pdev->dev.platform_data;
-	if (!pdata) {
-		dev_err(&pdev->dev, "cannot get platform data\n");
-		return -ENOENT;
+	int ret;
+
+	ret = davinci_cfg_reg(DA850_RTC_ALARM);
+	if (ret)
+		return ret;
+
+	pm_config.ddr2_ctlr_base = da8xx_get_mem_ctlr();
+	pm_config.deepsleep_reg = DA8XX_SYSCFG1_VIRT(DA8XX_DEEPSLEEP_REG);
+
+	pm_config.cpupll_reg_base = ioremap(DA8XX_PLL0_BASE, SZ_4K);
+	if (!pm_config.cpupll_reg_base)
+		return -ENOMEM;
+
+	pm_config.ddrpll_reg_base = ioremap(DA850_PLL1_BASE, SZ_4K);
+	if (!pm_config.ddrpll_reg_base) {
+		ret = -ENOMEM;
+		goto no_ddrpll_mem;
+	}
+
+	pm_config.ddrpsc_reg_base = ioremap(DA8XX_PSC1_BASE, SZ_4K);
+	if (!pm_config.ddrpsc_reg_base) {
+		ret = -ENOMEM;
+		goto no_ddrpsc_mem;
 	}
 
 	davinci_sram_suspend = sram_alloc(davinci_cpu_suspend_sz, NULL);
 	if (!davinci_sram_suspend) {
-		dev_err(&pdev->dev, "cannot allocate SRAM memory\n");
+		pr_err("PM: cannot allocate SRAM memory\n");
 		return -ENOMEM;
 	}
 
@@ -136,23 +162,9 @@ static int __init davinci_pm_probe(struct platform_device *pdev)
 
 	suspend_set_ops(&davinci_pm_ops);
 
-	return 0;
-}
-
-static int __exit davinci_pm_remove(struct platform_device *pdev)
-{
-	sram_free(davinci_sram_suspend, davinci_cpu_suspend_sz);
-	return 0;
-}
-
-static struct platform_driver davinci_pm_driver = {
-	.driver = {
-		.name	 = "pm-davinci",
-	},
-	.remove = __exit_p(davinci_pm_remove),
-};
-
-int __init davinci_pm_init(void)
-{
-	return platform_driver_probe(&davinci_pm_driver, davinci_pm_probe);
+no_ddrpsc_mem:
+	iounmap(pm_config.ddrpll_reg_base);
+no_ddrpll_mem:
+	iounmap(pm_config.cpupll_reg_base);
+	return ret;
 }
diff --git a/arch/arm/mach-davinci/usb-da8xx.c b/arch/arm/mach-davinci/usb-da8xx.c
index f141f5171906..c6feecf7ae24 100644
--- a/arch/arm/mach-davinci/usb-da8xx.c
+++ b/arch/arm/mach-davinci/usb-da8xx.c
@@ -1,21 +1,44 @@
 /*
  * DA8xx USB
  */
+#include <linux/clk.h>
+#include <linux/delay.h>
 #include <linux/dma-mapping.h>
 #include <linux/init.h>
+#include <linux/mfd/da8xx-cfgchip.h>
+#include <linux/phy/phy.h>
 #include <linux/platform_data/usb-davinci.h>
 #include <linux/platform_device.h>
 #include <linux/usb/musb.h>
 
+#include <mach/clock.h>
 #include <mach/common.h>
 #include <mach/cputype.h>
 #include <mach/da8xx.h>
 #include <mach/irqs.h>
 
+#include "clock.h"
+
 #define DA8XX_USB0_BASE		0x01e00000
 #define DA8XX_USB1_BASE		0x01e25000
 
-#if IS_ENABLED(CONFIG_USB_MUSB_HDRC)
+static struct platform_device da8xx_usb_phy = {
+	.name		= "da8xx-usb-phy",
+	.id		= -1,
+	.dev		= {
+		/*
+		 * Setting init_name so that clock lookup will work in
+		 * da8xx_register_usb11_phy_clk() even if this device is not
+		 * registered yet.
+		 */
+		.init_name	= "da8xx-usb-phy",
+	},
+};
+
+int __init da8xx_register_usb_phy(void)
+{
+	return platform_device_register(&da8xx_usb_phy);
+}
 
 static struct musb_hdrc_config musb_config = {
 	.multipoint	= true,
@@ -45,10 +68,15 @@ static struct resource da8xx_usb20_resources[] = {
 
 static u64 usb_dmamask = DMA_BIT_MASK(32);
 
-static struct platform_device usb_dev = {
+static struct platform_device da8xx_usb20_dev = {
 	.name		= "musb-da8xx",
 	.id             = -1,
 	.dev = {
+		/*
+		 * Setting init_name so that clock lookup will work in
+		 * usb20_phy_clk_enable() even if this device is not registered.
+		 */
+		.init_name		= "musb-da8xx",
 		.platform_data		= &usb_data,
 		.dma_mask		= &usb_dmamask,
 		.coherent_dma_mask      = DMA_BIT_MASK(32),
@@ -62,18 +90,9 @@ int __init da8xx_register_usb20(unsigned int mA, unsigned int potpgt)
 	usb_data.power	= mA > 510 ? 255 : mA / 2;
 	usb_data.potpgt = (potpgt + 1) / 2;
 
-	return platform_device_register(&usb_dev);
-}
-
-#else
-
-int __init da8xx_register_usb20(unsigned int mA, unsigned int potpgt)
-{
-	return 0;
+	return platform_device_register(&da8xx_usb20_dev);
 }
 
-#endif  /* CONFIG_USB_MUSB_HDRC */
-
 static struct resource da8xx_usb11_resources[] = {
 	[0] = {
 		.start	= DA8XX_USB1_BASE,
@@ -90,8 +109,8 @@ static struct resource da8xx_usb11_resources[] = {
 static u64 da8xx_usb11_dma_mask = DMA_BIT_MASK(32);
 
 static struct platform_device da8xx_usb11_device = {
-	.name		= "ohci",
-	.id		= 0,
+	.name		= "ohci-da8xx",
+	.id		= -1,
 	.dev = {
 		.dma_mask		= &da8xx_usb11_dma_mask,
 		.coherent_dma_mask	= DMA_BIT_MASK(32),
@@ -105,3 +124,236 @@ int __init da8xx_register_usb11(struct da8xx_ohci_root_hub *pdata)
 	da8xx_usb11_device.dev.platform_data = pdata;
 	return platform_device_register(&da8xx_usb11_device);
 }
+
+static struct clk usb_refclkin = {
+	.name		= "usb_refclkin",
+	.set_rate	= davinci_simple_set_rate,
+};
+
+static struct clk_lookup usb_refclkin_lookup =
+	CLK(NULL, "usb_refclkin", &usb_refclkin);
+
+/**
+ * da8xx_register_usb_refclkin - register USB_REFCLKIN clock
+ *
+ * @rate: The clock rate in Hz
+ *
+ * This clock is only needed if the board provides an external USB_REFCLKIN
+ * signal, in which case it will be used as the parent of usb20_phy_clk and/or
+ * usb11_phy_clk.
+ */
+int __init da8xx_register_usb_refclkin(int rate)
+{
+	int ret;
+
+	usb_refclkin.rate = rate;
+	ret = clk_register(&usb_refclkin);
+	if (ret)
+		return ret;
+
+	clkdev_add(&usb_refclkin_lookup);
+
+	return 0;
+}
+
+static void usb20_phy_clk_enable(struct clk *clk)
+{
+	struct clk *usb20_clk;
+	int err;
+	u32 val;
+	u32 timeout = 500000; /* 500 msec */
+
+	val = readl(DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
+
+	usb20_clk = clk_get(&da8xx_usb20_dev.dev, "usb20");
+	if (IS_ERR(usb20_clk)) {
+		pr_err("could not get usb20 clk: %ld\n", PTR_ERR(usb20_clk));
+		return;
+	}
+
+	/* The USB 2.O PLL requires that the USB 2.O PSC is enabled as well. */
+	err = clk_prepare_enable(usb20_clk);
+	if (err) {
+		pr_err("failed to enable usb20 clk: %d\n", err);
+		clk_put(usb20_clk);
+		return;
+	}
+
+	/*
+	 * Turn on the USB 2.0 PHY, but just the PLL, and not OTG. The USB 1.1
+	 * host may use the PLL clock without USB 2.0 OTG being used.
+	 */
+	val &= ~(CFGCHIP2_RESET | CFGCHIP2_PHYPWRDN);
+	val |= CFGCHIP2_PHY_PLLON;
+
+	writel(val, DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
+
+	while (--timeout) {
+		val = readl(DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
+		if (val & CFGCHIP2_PHYCLKGD)
+			goto done;
+		udelay(1);
+	}
+
+	pr_err("Timeout waiting for USB 2.0 PHY clock good\n");
+done:
+	clk_disable_unprepare(usb20_clk);
+	clk_put(usb20_clk);
+}
+
+static void usb20_phy_clk_disable(struct clk *clk)
+{
+	u32 val;
+
+	val = readl(DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
+	val |= CFGCHIP2_PHYPWRDN;
+	writel(val, DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
+}
+
+static int usb20_phy_clk_set_parent(struct clk *clk, struct clk *parent)
+{
+	u32 val;
+
+	val = readl(DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
+
+	/* Set the mux depending on the parent clock. */
+	if (parent == &usb_refclkin) {
+		val &= ~CFGCHIP2_USB2PHYCLKMUX;
+	} else if (strcmp(parent->name, "pll0_aux_clk") == 0) {
+		val |= CFGCHIP2_USB2PHYCLKMUX;
+	} else {
+		pr_err("Bad parent on USB 2.0 PHY clock\n");
+		return -EINVAL;
+	}
+
+	/* reference frequency also comes from parent clock */
+	val &= ~CFGCHIP2_REFFREQ_MASK;
+	switch (clk_get_rate(parent)) {
+	case 12000000:
+		val |= CFGCHIP2_REFFREQ_12MHZ;
+		break;
+	case 13000000:
+		val |= CFGCHIP2_REFFREQ_13MHZ;
+		break;
+	case 19200000:
+		val |= CFGCHIP2_REFFREQ_19_2MHZ;
+		break;
+	case 20000000:
+		val |= CFGCHIP2_REFFREQ_20MHZ;
+		break;
+	case 24000000:
+		val |= CFGCHIP2_REFFREQ_24MHZ;
+		break;
+	case 26000000:
+		val |= CFGCHIP2_REFFREQ_26MHZ;
+		break;
+	case 38400000:
+		val |= CFGCHIP2_REFFREQ_38_4MHZ;
+		break;
+	case 40000000:
+		val |= CFGCHIP2_REFFREQ_40MHZ;
+		break;
+	case 48000000:
+		val |= CFGCHIP2_REFFREQ_48MHZ;
+		break;
+	default:
+		pr_err("Bad parent clock rate on USB 2.0 PHY clock\n");
+		return -EINVAL;
+	}
+
+	writel(val, DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
+
+	return 0;
+}
+
+static struct clk usb20_phy_clk = {
+	.name		= "usb20_phy",
+	.clk_enable	= usb20_phy_clk_enable,
+	.clk_disable	= usb20_phy_clk_disable,
+	.set_parent	= usb20_phy_clk_set_parent,
+};
+
+static struct clk_lookup usb20_phy_clk_lookup =
+	CLK("da8xx-usb-phy", "usb20_phy", &usb20_phy_clk);
+
+/**
+ * da8xx_register_usb20_phy_clk - register USB0PHYCLKMUX clock
+ *
+ * @use_usb_refclkin: Selects the parent clock - either "usb_refclkin" if true
+ *	or "pll0_aux" if false.
+ */
+int __init da8xx_register_usb20_phy_clk(bool use_usb_refclkin)
+{
+	struct clk *parent;
+	int ret = 0;
+
+	parent = clk_get(NULL, use_usb_refclkin ? "usb_refclkin" : "pll0_aux");
+	if (IS_ERR(parent))
+		return PTR_ERR(parent);
+
+	usb20_phy_clk.parent = parent;
+	ret = clk_register(&usb20_phy_clk);
+	if (!ret)
+		clkdev_add(&usb20_phy_clk_lookup);
+
+	clk_put(parent);
+
+	return ret;
+}
+
+static int usb11_phy_clk_set_parent(struct clk *clk, struct clk *parent)
+{
+	u32 val;
+
+	val = readl(DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
+
+	/* Set the USB 1.1 PHY clock mux based on the parent clock. */
+	if (parent == &usb20_phy_clk) {
+		val &= ~CFGCHIP2_USB1PHYCLKMUX;
+	} else if (parent == &usb_refclkin) {
+		val |= CFGCHIP2_USB1PHYCLKMUX;
+	} else {
+		pr_err("Bad parent on USB 1.1 PHY clock\n");
+		return -EINVAL;
+	}
+
+	writel(val, DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG));
+
+	return 0;
+}
+
+static struct clk usb11_phy_clk = {
+	.name		= "usb11_phy",
+	.set_parent	= usb11_phy_clk_set_parent,
+};
+
+static struct clk_lookup usb11_phy_clk_lookup =
+	CLK("da8xx-usb-phy", "usb11_phy", &usb11_phy_clk);
+
+/**
+ * da8xx_register_usb11_phy_clk - register USB1PHYCLKMUX clock
+ *
+ * @use_usb_refclkin: Selects the parent clock - either "usb_refclkin" if true
+ *	or "usb20_phy" if false.
+ */
+int __init da8xx_register_usb11_phy_clk(bool use_usb_refclkin)
+{
+	struct clk *parent;
+	int ret = 0;
+
+	if (use_usb_refclkin)
+		parent = clk_get(NULL, "usb_refclkin");
+	else
+		parent = clk_get(&da8xx_usb_phy.dev, "usb20_phy");
+	if (IS_ERR(parent))
+		return PTR_ERR(parent);
+
+	usb11_phy_clk.parent = parent;
+	ret = clk_register(&usb11_phy_clk);
+	if (!ret)
+		clkdev_add(&usb11_phy_clk_lookup);
+
+	clk_put(parent);
+
+	return ret;
+}
diff --git a/arch/arm/mach-imx/Kconfig b/arch/arm/mach-imx/Kconfig
index 9155b639c9aa..936c59d0e18b 100644
--- a/arch/arm/mach-imx/Kconfig
+++ b/arch/arm/mach-imx/Kconfig
@@ -557,7 +557,6 @@ config SOC_VF610
 	bool "Vybrid Family VF610 support"
 	select ARM_GIC if ARCH_MULTI_V7
 	select PINCTRL_VF610
-	select PL310_ERRATA_769419 if CACHE_L2X0
 
 	help
 	  This enables support for Freescale Vybrid VF610 processor.
diff --git a/arch/arm/mach-imx/common.h b/arch/arm/mach-imx/common.h
index c4436d9c52ff..b09a2ec19267 100644
--- a/arch/arm/mach-imx/common.h
+++ b/arch/arm/mach-imx/common.h
@@ -43,7 +43,6 @@ int mx21_clocks_init(unsigned long lref, unsigned long fref);
 int mx27_clocks_init(unsigned long fref);
 int mx31_clocks_init(unsigned long fref);
 int mx35_clocks_init(void);
-int mx31_clocks_init_dt(void);
 struct platform_device *mxc_register_gpio(char *name, int id,
 	resource_size_t iobase, resource_size_t iosize, int irq, int irq_high);
 void mxc_set_cpu_type(unsigned int type);
diff --git a/arch/arm/mach-imx/gpc.c b/arch/arm/mach-imx/gpc.c
index b54db47f6f32..1dc2a34b9dbd 100644
--- a/arch/arm/mach-imx/gpc.c
+++ b/arch/arm/mach-imx/gpc.c
@@ -380,13 +380,6 @@ static struct pu_domain imx6q_pu_domain = {
 		.name = "PU",
 		.power_off = imx6q_pm_pu_power_off,
 		.power_on = imx6q_pm_pu_power_on,
-		.states = {
-			[0] = {
-				.power_off_latency_ns = 25000,
-				.power_on_latency_ns = 2000000,
-			},
-		},
-		.state_count = 1,
 	},
 };
 
@@ -430,6 +423,16 @@ static int imx_gpc_genpd_init(struct device *dev, struct regulator *pu_reg)
 	if (!IS_ENABLED(CONFIG_PM_GENERIC_DOMAINS))
 		return 0;
 
+	imx6q_pu_domain.base.states = devm_kzalloc(dev,
+					sizeof(*imx6q_pu_domain.base.states),
+					GFP_KERNEL);
+	if (!imx6q_pu_domain.base.states)
+		return -ENOMEM;
+
+	imx6q_pu_domain.base.states[0].power_off_latency_ns = 25000;
+	imx6q_pu_domain.base.states[0].power_on_latency_ns = 2000000;
+	imx6q_pu_domain.base.state_count = 1;
+
 	for (i = 0; i < ARRAY_SIZE(imx_gpc_domains); i++)
 		pm_genpd_init(imx_gpc_domains[i], NULL, false);
 
diff --git a/arch/arm/mach-imx/imx31-dt.c b/arch/arm/mach-imx/imx31-dt.c
index 62e6b4fb5370..668d74b72511 100644
--- a/arch/arm/mach-imx/imx31-dt.c
+++ b/arch/arm/mach-imx/imx31-dt.c
@@ -23,11 +23,6 @@ static const char * const imx31_dt_board_compat[] __initconst = {
 	NULL
 };
 
-static void __init imx31_dt_timer_init(void)
-{
-	mx31_clocks_init_dt();
-}
-
 /* FIXME: replace with DT binding */
 static const struct resource imx31_rnga_res[] __initconst = {
 	DEFINE_RES_MEM(MX31_RNGA_BASE_ADDR, SZ_16K),
@@ -43,7 +38,6 @@ DT_MACHINE_START(IMX31_DT, "Freescale i.MX31 (Device Tree Support)")
 	.map_io		= mx31_map_io,
 	.init_early	= imx31_init_early,
 	.init_irq	= mx31_init_irq,
-	.init_time	= imx31_dt_timer_init,
 	.init_machine	= imx31_dt_mach_init,
 	.dt_compat	= imx31_dt_board_compat,
 MACHINE_END
diff --git a/arch/arm/mach-imx/mach-imx6ul.c b/arch/arm/mach-imx/mach-imx6ul.c
index 58a2b88233e6..6cb8a22b617d 100644
--- a/arch/arm/mach-imx/mach-imx6ul.c
+++ b/arch/arm/mach-imx/mach-imx6ul.c
@@ -89,6 +89,7 @@ static void __init imx6ul_init_late(void)
 
 static const char * const imx6ul_dt_compat[] __initconst = {
 	"fsl,imx6ul",
+	"fsl,imx6ull",
 	NULL,
 };
 
diff --git a/arch/arm/mach-imx/mmdc.c b/arch/arm/mach-imx/mmdc.c
index db9621c718ec..ba96bf979625 100644
--- a/arch/arm/mach-imx/mmdc.c
+++ b/arch/arm/mach-imx/mmdc.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2011 Freescale Semiconductor, Inc.
+ * Copyright 2011,2016 Freescale Semiconductor, Inc.
  * Copyright 2011 Linaro Ltd.
  *
  * The code contained herein is licensed under the GNU General Public
@@ -10,12 +10,16 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
+#include <linux/hrtimer.h>
 #include <linux/init.h>
+#include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/of_device.h>
+#include <linux/perf_event.h>
+#include <linux/slab.h>
 
 #include "common.h"
 
@@ -27,8 +31,489 @@
 #define BM_MMDC_MDMISC_DDR_TYPE	0x18
 #define BP_MMDC_MDMISC_DDR_TYPE	0x3
 
+#define TOTAL_CYCLES		0x0
+#define BUSY_CYCLES		0x1
+#define READ_ACCESSES		0x2
+#define WRITE_ACCESSES		0x3
+#define READ_BYTES		0x4
+#define WRITE_BYTES		0x5
+
+/* Enables, resets, freezes, overflow profiling*/
+#define DBG_DIS			0x0
+#define DBG_EN			0x1
+#define DBG_RST			0x2
+#define PRF_FRZ			0x4
+#define CYC_OVF			0x8
+#define PROFILE_SEL		0x10
+
+#define MMDC_MADPCR0	0x410
+#define MMDC_MADPSR0	0x418
+#define MMDC_MADPSR1	0x41C
+#define MMDC_MADPSR2	0x420
+#define MMDC_MADPSR3	0x424
+#define MMDC_MADPSR4	0x428
+#define MMDC_MADPSR5	0x42C
+
+#define MMDC_NUM_COUNTERS	6
+
+#define MMDC_FLAG_PROFILE_SEL	0x1
+
+#define to_mmdc_pmu(p) container_of(p, struct mmdc_pmu, pmu)
+
 static int ddr_type;
 
+struct fsl_mmdc_devtype_data {
+	unsigned int flags;
+};
+
+static const struct fsl_mmdc_devtype_data imx6q_data = {
+};
+
+static const struct fsl_mmdc_devtype_data imx6qp_data = {
+	.flags = MMDC_FLAG_PROFILE_SEL,
+};
+
+static const struct of_device_id imx_mmdc_dt_ids[] = {
+	{ .compatible = "fsl,imx6q-mmdc", .data = (void *)&imx6q_data},
+	{ .compatible = "fsl,imx6qp-mmdc", .data = (void *)&imx6qp_data},
+	{ /* sentinel */ }
+};
+
+#ifdef CONFIG_PERF_EVENTS
+
+static DEFINE_IDA(mmdc_ida);
+
+PMU_EVENT_ATTR_STRING(total-cycles, mmdc_pmu_total_cycles, "event=0x00")
+PMU_EVENT_ATTR_STRING(busy-cycles, mmdc_pmu_busy_cycles, "event=0x01")
+PMU_EVENT_ATTR_STRING(read-accesses, mmdc_pmu_read_accesses, "event=0x02")
+PMU_EVENT_ATTR_STRING(write-accesses, mmdc_pmu_write_accesses, "config=0x03")
+PMU_EVENT_ATTR_STRING(read-bytes, mmdc_pmu_read_bytes, "event=0x04")
+PMU_EVENT_ATTR_STRING(read-bytes.unit, mmdc_pmu_read_bytes_unit, "MB");
+PMU_EVENT_ATTR_STRING(read-bytes.scale, mmdc_pmu_read_bytes_scale, "0.000001");
+PMU_EVENT_ATTR_STRING(write-bytes, mmdc_pmu_write_bytes, "event=0x05")
+PMU_EVENT_ATTR_STRING(write-bytes.unit, mmdc_pmu_write_bytes_unit, "MB");
+PMU_EVENT_ATTR_STRING(write-bytes.scale, mmdc_pmu_write_bytes_scale, "0.000001");
+
+struct mmdc_pmu {
+	struct pmu pmu;
+	void __iomem *mmdc_base;
+	cpumask_t cpu;
+	struct hrtimer hrtimer;
+	unsigned int active_events;
+	struct device *dev;
+	struct perf_event *mmdc_events[MMDC_NUM_COUNTERS];
+	struct hlist_node node;
+	struct fsl_mmdc_devtype_data *devtype_data;
+};
+
+/*
+ * Polling period is set to one second, overflow of total-cycles (the fastest
+ * increasing counter) takes ten seconds so one second is safe
+ */
+static unsigned int mmdc_pmu_poll_period_us = 1000000;
+
+module_param_named(pmu_pmu_poll_period_us, mmdc_pmu_poll_period_us, uint,
+		S_IRUGO | S_IWUSR);
+
+static ktime_t mmdc_pmu_timer_period(void)
+{
+	return ns_to_ktime((u64)mmdc_pmu_poll_period_us * 1000);
+}
+
+static ssize_t mmdc_pmu_cpumask_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct mmdc_pmu *pmu_mmdc = dev_get_drvdata(dev);
+
+	return cpumap_print_to_pagebuf(true, buf, &pmu_mmdc->cpu);
+}
+
+static struct device_attribute mmdc_pmu_cpumask_attr =
+	__ATTR(cpumask, S_IRUGO, mmdc_pmu_cpumask_show, NULL);
+
+static struct attribute *mmdc_pmu_cpumask_attrs[] = {
+	&mmdc_pmu_cpumask_attr.attr,
+	NULL,
+};
+
+static struct attribute_group mmdc_pmu_cpumask_attr_group = {
+	.attrs = mmdc_pmu_cpumask_attrs,
+};
+
+static struct attribute *mmdc_pmu_events_attrs[] = {
+	&mmdc_pmu_total_cycles.attr.attr,
+	&mmdc_pmu_busy_cycles.attr.attr,
+	&mmdc_pmu_read_accesses.attr.attr,
+	&mmdc_pmu_write_accesses.attr.attr,
+	&mmdc_pmu_read_bytes.attr.attr,
+	&mmdc_pmu_read_bytes_unit.attr.attr,
+	&mmdc_pmu_read_bytes_scale.attr.attr,
+	&mmdc_pmu_write_bytes.attr.attr,
+	&mmdc_pmu_write_bytes_unit.attr.attr,
+	&mmdc_pmu_write_bytes_scale.attr.attr,
+	NULL,
+};
+
+static struct attribute_group mmdc_pmu_events_attr_group = {
+	.name = "events",
+	.attrs = mmdc_pmu_events_attrs,
+};
+
+PMU_FORMAT_ATTR(event, "config:0-63");
+static struct attribute *mmdc_pmu_format_attrs[] = {
+	&format_attr_event.attr,
+	NULL,
+};
+
+static struct attribute_group mmdc_pmu_format_attr_group = {
+	.name = "format",
+	.attrs = mmdc_pmu_format_attrs,
+};
+
+static const struct attribute_group *attr_groups[] = {
+	&mmdc_pmu_events_attr_group,
+	&mmdc_pmu_format_attr_group,
+	&mmdc_pmu_cpumask_attr_group,
+	NULL,
+};
+
+static u32 mmdc_pmu_read_counter(struct mmdc_pmu *pmu_mmdc, int cfg)
+{
+	void __iomem *mmdc_base, *reg;
+
+	mmdc_base = pmu_mmdc->mmdc_base;
+
+	switch (cfg) {
+	case TOTAL_CYCLES:
+		reg = mmdc_base + MMDC_MADPSR0;
+		break;
+	case BUSY_CYCLES:
+		reg = mmdc_base + MMDC_MADPSR1;
+		break;
+	case READ_ACCESSES:
+		reg = mmdc_base + MMDC_MADPSR2;
+		break;
+	case WRITE_ACCESSES:
+		reg = mmdc_base + MMDC_MADPSR3;
+		break;
+	case READ_BYTES:
+		reg = mmdc_base + MMDC_MADPSR4;
+		break;
+	case WRITE_BYTES:
+		reg = mmdc_base + MMDC_MADPSR5;
+		break;
+	default:
+		return WARN_ONCE(1,
+			"invalid configuration %d for mmdc counter", cfg);
+	}
+	return readl(reg);
+}
+
+static int mmdc_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
+{
+	struct mmdc_pmu *pmu_mmdc = hlist_entry_safe(node, struct mmdc_pmu, node);
+	int target;
+
+	if (!cpumask_test_and_clear_cpu(cpu, &pmu_mmdc->cpu))
+		return 0;
+
+	target = cpumask_any_but(cpu_online_mask, cpu);
+	if (target >= nr_cpu_ids)
+		return 0;
+
+	perf_pmu_migrate_context(&pmu_mmdc->pmu, cpu, target);
+	cpumask_set_cpu(target, &pmu_mmdc->cpu);
+
+	return 0;
+}
+
+static bool mmdc_pmu_group_event_is_valid(struct perf_event *event,
+					  struct pmu *pmu,
+					  unsigned long *used_counters)
+{
+	int cfg = event->attr.config;
+
+	if (is_software_event(event))
+		return true;
+
+	if (event->pmu != pmu)
+		return false;
+
+	return !test_and_set_bit(cfg, used_counters);
+}
+
+/*
+ * Each event has a single fixed-purpose counter, so we can only have a
+ * single active event for each at any point in time. Here we just check
+ * for duplicates, and rely on mmdc_pmu_event_init to verify that the HW
+ * event numbers are valid.
+ */
+static bool mmdc_pmu_group_is_valid(struct perf_event *event)
+{
+	struct pmu *pmu = event->pmu;
+	struct perf_event *leader = event->group_leader;
+	struct perf_event *sibling;
+	unsigned long counter_mask = 0;
+
+	set_bit(leader->attr.config, &counter_mask);
+
+	if (event != leader) {
+		if (!mmdc_pmu_group_event_is_valid(event, pmu, &counter_mask))
+			return false;
+	}
+
+	list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
+		if (!mmdc_pmu_group_event_is_valid(sibling, pmu, &counter_mask))
+			return false;
+	}
+
+	return true;
+}
+
+static int mmdc_pmu_event_init(struct perf_event *event)
+{
+	struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
+	int cfg = event->attr.config;
+
+	if (event->attr.type != event->pmu->type)
+		return -ENOENT;
+
+	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
+		return -EOPNOTSUPP;
+
+	if (event->cpu < 0) {
+		dev_warn(pmu_mmdc->dev, "Can't provide per-task data!\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (event->attr.exclude_user		||
+			event->attr.exclude_kernel	||
+			event->attr.exclude_hv		||
+			event->attr.exclude_idle	||
+			event->attr.exclude_host	||
+			event->attr.exclude_guest	||
+			event->attr.sample_period)
+		return -EINVAL;
+
+	if (cfg < 0 || cfg >= MMDC_NUM_COUNTERS)
+		return -EINVAL;
+
+	if (!mmdc_pmu_group_is_valid(event))
+		return -EINVAL;
+
+	event->cpu = cpumask_first(&pmu_mmdc->cpu);
+	return 0;
+}
+
+static void mmdc_pmu_event_update(struct perf_event *event)
+{
+	struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	u64 delta, prev_raw_count, new_raw_count;
+
+	do {
+		prev_raw_count = local64_read(&hwc->prev_count);
+		new_raw_count = mmdc_pmu_read_counter(pmu_mmdc,
+						      event->attr.config);
+	} while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
+		new_raw_count) != prev_raw_count);
+
+	delta = (new_raw_count - prev_raw_count) & 0xFFFFFFFF;
+
+	local64_add(delta, &event->count);
+}
+
+static void mmdc_pmu_event_start(struct perf_event *event, int flags)
+{
+	struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	void __iomem *mmdc_base, *reg;
+	u32 val;
+
+	mmdc_base = pmu_mmdc->mmdc_base;
+	reg = mmdc_base + MMDC_MADPCR0;
+
+	/*
+	 * hrtimer is required because mmdc does not provide an interrupt so
+	 * polling is necessary
+	 */
+	hrtimer_start(&pmu_mmdc->hrtimer, mmdc_pmu_timer_period(),
+			HRTIMER_MODE_REL_PINNED);
+
+	local64_set(&hwc->prev_count, 0);
+
+	writel(DBG_RST, reg);
+
+	val = DBG_EN;
+	if (pmu_mmdc->devtype_data->flags & MMDC_FLAG_PROFILE_SEL)
+		val |= PROFILE_SEL;
+
+	writel(val, reg);
+}
+
+static int mmdc_pmu_event_add(struct perf_event *event, int flags)
+{
+	struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+
+	int cfg = event->attr.config;
+
+	if (flags & PERF_EF_START)
+		mmdc_pmu_event_start(event, flags);
+
+	if (pmu_mmdc->mmdc_events[cfg] != NULL)
+		return -EAGAIN;
+
+	pmu_mmdc->mmdc_events[cfg] = event;
+	pmu_mmdc->active_events++;
+
+	local64_set(&hwc->prev_count, mmdc_pmu_read_counter(pmu_mmdc, cfg));
+
+	return 0;
+}
+
+static void mmdc_pmu_event_stop(struct perf_event *event, int flags)
+{
+	struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
+	void __iomem *mmdc_base, *reg;
+
+	mmdc_base = pmu_mmdc->mmdc_base;
+	reg = mmdc_base + MMDC_MADPCR0;
+
+	writel(PRF_FRZ, reg);
+	mmdc_pmu_event_update(event);
+}
+
+static void mmdc_pmu_event_del(struct perf_event *event, int flags)
+{
+	struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
+	int cfg = event->attr.config;
+
+	pmu_mmdc->mmdc_events[cfg] = NULL;
+	pmu_mmdc->active_events--;
+
+	if (pmu_mmdc->active_events == 0)
+		hrtimer_cancel(&pmu_mmdc->hrtimer);
+
+	mmdc_pmu_event_stop(event, PERF_EF_UPDATE);
+}
+
+static void mmdc_pmu_overflow_handler(struct mmdc_pmu *pmu_mmdc)
+{
+	int i;
+
+	for (i = 0; i < MMDC_NUM_COUNTERS; i++) {
+		struct perf_event *event = pmu_mmdc->mmdc_events[i];
+
+		if (event)
+			mmdc_pmu_event_update(event);
+	}
+}
+
+static enum hrtimer_restart mmdc_pmu_timer_handler(struct hrtimer *hrtimer)
+{
+	struct mmdc_pmu *pmu_mmdc = container_of(hrtimer, struct mmdc_pmu,
+			hrtimer);
+
+	mmdc_pmu_overflow_handler(pmu_mmdc);
+	hrtimer_forward_now(hrtimer, mmdc_pmu_timer_period());
+
+	return HRTIMER_RESTART;
+}
+
+static int mmdc_pmu_init(struct mmdc_pmu *pmu_mmdc,
+		void __iomem *mmdc_base, struct device *dev)
+{
+	int mmdc_num;
+
+	*pmu_mmdc = (struct mmdc_pmu) {
+		.pmu = (struct pmu) {
+			.task_ctx_nr    = perf_invalid_context,
+			.attr_groups    = attr_groups,
+			.event_init     = mmdc_pmu_event_init,
+			.add            = mmdc_pmu_event_add,
+			.del            = mmdc_pmu_event_del,
+			.start          = mmdc_pmu_event_start,
+			.stop           = mmdc_pmu_event_stop,
+			.read           = mmdc_pmu_event_update,
+		},
+		.mmdc_base = mmdc_base,
+		.dev = dev,
+		.active_events = 0,
+	};
+
+	mmdc_num = ida_simple_get(&mmdc_ida, 0, 0, GFP_KERNEL);
+
+	return mmdc_num;
+}
+
+static int imx_mmdc_remove(struct platform_device *pdev)
+{
+	struct mmdc_pmu *pmu_mmdc = platform_get_drvdata(pdev);
+
+	perf_pmu_unregister(&pmu_mmdc->pmu);
+	cpuhp_remove_state_nocalls(CPUHP_ONLINE);
+	kfree(pmu_mmdc);
+	return 0;
+}
+
+static int imx_mmdc_perf_init(struct platform_device *pdev, void __iomem *mmdc_base)
+{
+	struct mmdc_pmu *pmu_mmdc;
+	char *name;
+	int mmdc_num;
+	int ret;
+	const struct of_device_id *of_id =
+		of_match_device(imx_mmdc_dt_ids, &pdev->dev);
+
+	pmu_mmdc = kzalloc(sizeof(*pmu_mmdc), GFP_KERNEL);
+	if (!pmu_mmdc) {
+		pr_err("failed to allocate PMU device!\n");
+		return -ENOMEM;
+	}
+
+	mmdc_num = mmdc_pmu_init(pmu_mmdc, mmdc_base, &pdev->dev);
+	if (mmdc_num == 0)
+		name = "mmdc";
+	else
+		name = devm_kasprintf(&pdev->dev,
+				GFP_KERNEL, "mmdc%d", mmdc_num);
+
+	pmu_mmdc->devtype_data = (struct fsl_mmdc_devtype_data *)of_id->data;
+
+	hrtimer_init(&pmu_mmdc->hrtimer, CLOCK_MONOTONIC,
+			HRTIMER_MODE_REL);
+	pmu_mmdc->hrtimer.function = mmdc_pmu_timer_handler;
+
+	cpuhp_state_add_instance_nocalls(CPUHP_ONLINE,
+					 &pmu_mmdc->node);
+	cpumask_set_cpu(smp_processor_id(), &pmu_mmdc->cpu);
+	ret = cpuhp_setup_state_multi(CPUHP_AP_NOTIFY_ONLINE,
+				      "MMDC_ONLINE", NULL,
+				      mmdc_pmu_offline_cpu);
+	if (ret) {
+		pr_err("cpuhp_setup_state_multi failure\n");
+		goto pmu_register_err;
+	}
+
+	ret = perf_pmu_register(&(pmu_mmdc->pmu), name, -1);
+	platform_set_drvdata(pdev, pmu_mmdc);
+	if (ret)
+		goto pmu_register_err;
+	return 0;
+
+pmu_register_err:
+	pr_warn("MMDC Perf PMU failed (%d), disabled\n", ret);
+	hrtimer_cancel(&pmu_mmdc->hrtimer);
+	kfree(pmu_mmdc);
+	return ret;
+}
+
+#else
+#define imx_mmdc_remove NULL
+#define imx_mmdc_perf_init(pdev, mmdc_base) 0
+#endif
+
 static int imx_mmdc_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
@@ -62,7 +547,7 @@ static int imx_mmdc_probe(struct platform_device *pdev)
 		return -EBUSY;
 	}
 
-	return 0;
+	return imx_mmdc_perf_init(pdev, mmdc_base);
 }
 
 int imx_mmdc_get_ddr_type(void)
@@ -70,17 +555,13 @@ int imx_mmdc_get_ddr_type(void)
 	return ddr_type;
 }
 
-static const struct of_device_id imx_mmdc_dt_ids[] = {
-	{ .compatible = "fsl,imx6q-mmdc", },
-	{ /* sentinel */ }
-};
-
 static struct platform_driver imx_mmdc_driver = {
 	.driver		= {
 		.name	= "imx-mmdc",
 		.of_match_table = imx_mmdc_dt_ids,
 	},
 	.probe		= imx_mmdc_probe,
+	.remove		= imx_mmdc_remove,
 };
 
 static int __init imx_mmdc_init(void)
diff --git a/arch/arm/mach-integrator/impd1.c b/arch/arm/mach-integrator/impd1.c
index ed9a01484030..a109f6482413 100644
--- a/arch/arm/mach-integrator/impd1.c
+++ b/arch/arm/mach-integrator/impd1.c
@@ -21,7 +21,6 @@
 #include <linux/amba/bus.h>
 #include <linux/amba/clcd.h>
 #include <linux/amba/mmci.h>
-#include <linux/amba/pl061.h>
 #include <linux/io.h>
 #include <linux/platform_data/clk-integrator.h>
 #include <linux/slab.h>
diff --git a/arch/arm/mach-integrator/integrator_ap.c b/arch/arm/mach-integrator/integrator_ap.c
index 23b98fd414bf..a1af634f8709 100644
--- a/arch/arm/mach-integrator/integrator_ap.c
+++ b/arch/arm/mach-integrator/integrator_ap.c
@@ -27,6 +27,8 @@
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
 #include <linux/termios.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
 
 #include <asm/mach/arch.h>
 #include <asm/mach/map.h>
@@ -37,11 +39,8 @@
 #include "pci_v3.h"
 #include "lm.h"
 
-/* Base address to the AP system controller */
-void __iomem *ap_syscon_base;
-/* Base address to the external bus interface */
-static void __iomem *ebi_base;
-
+/* Regmap to the AP system controller */
+static struct regmap *ap_syscon_map;
 
 /*
  * All IO addresses are mapped onto VA 0xFFFx.xxxx, where x.xxxx
@@ -125,6 +124,7 @@ static void integrator_uart_set_mctrl(struct amba_device *dev,
 {
 	unsigned int ctrls = 0, ctrlc = 0, rts_mask, dtr_mask;
 	u32 phybase = dev->res.start;
+	int ret;
 
 	if (phybase == INTEGRATOR_UART0_BASE) {
 		/* UART0 */
@@ -146,8 +146,17 @@ static void integrator_uart_set_mctrl(struct amba_device *dev,
 	else
 		ctrls |= dtr_mask;
 
-	__raw_writel(ctrls, ap_syscon_base + INTEGRATOR_SC_CTRLS_OFFSET);
-	__raw_writel(ctrlc, ap_syscon_base + INTEGRATOR_SC_CTRLC_OFFSET);
+	ret = regmap_write(ap_syscon_map,
+			   INTEGRATOR_SC_CTRLS_OFFSET,
+			   ctrls);
+	if (ret)
+		pr_err("MODEM: unable to write PL010 UART CTRLS\n");
+
+	ret = regmap_write(ap_syscon_map,
+			   INTEGRATOR_SC_CTRLC_OFFSET,
+			   ctrlc);
+	if (ret)
+		pr_err("MODEM: unable to write PL010 UART CRTLC\n");
 }
 
 struct amba_pl010_data ap_uart_data = {
@@ -178,35 +187,32 @@ static const struct of_device_id ap_syscon_match[] = {
 	{ },
 };
 
-static const struct of_device_id ebi_match[] = {
-	{ .compatible = "arm,external-bus-interface"},
-	{ },
-};
-
 static void __init ap_init_of(void)
 {
-	unsigned long sc_dec;
+	u32 sc_dec;
 	struct device_node *syscon;
-	struct device_node *ebi;
+	int ret;
 	int i;
 
+	of_platform_default_populate(NULL, ap_auxdata_lookup, NULL);
+
 	syscon = of_find_matching_node(NULL, ap_syscon_match);
 	if (!syscon)
 		return;
-	ebi = of_find_matching_node(NULL, ebi_match);
-	if (!ebi)
+	ap_syscon_map = syscon_node_to_regmap(syscon);
+	if (IS_ERR(ap_syscon_map)) {
+		pr_crit("could not find Integrator/AP system controller\n");
 		return;
+	}
 
-	ap_syscon_base = of_iomap(syscon, 0);
-	if (!ap_syscon_base)
-		return;
-	ebi_base = of_iomap(ebi, 0);
-	if (!ebi_base)
+	ret = regmap_read(ap_syscon_map,
+			  INTEGRATOR_SC_DEC_OFFSET,
+			  &sc_dec);
+	if (ret) {
+		pr_crit("could not read from Integrator/AP syscon\n");
 		return;
+	}
 
-	of_platform_default_populate(NULL, ap_auxdata_lookup, NULL);
-
-	sc_dec = readl(ap_syscon_base + INTEGRATOR_SC_DEC_OFFSET);
 	for (i = 0; i < 4; i++) {
 		struct lm_device *lmdev;
 
diff --git a/arch/arm/mach-ixp4xx/dsmg600-setup.c b/arch/arm/mach-ixp4xx/dsmg600-setup.c
index 43ee06d3abe5..b3bd0e137f6d 100644
--- a/arch/arm/mach-ixp4xx/dsmg600-setup.c
+++ b/arch/arm/mach-ixp4xx/dsmg600-setup.c
@@ -26,7 +26,6 @@
 #include <linux/reboot.h>
 #include <linux/i2c.h>
 #include <linux/i2c-gpio.h>
-#include <linux/gpio.h>
 
 #include <mach/hardware.h>
 
diff --git a/arch/arm/mach-lpc32xx/clock.h b/arch/arm/mach-lpc32xx/clock.h
deleted file mode 100644
index c0a8434307f7..000000000000
--- a/arch/arm/mach-lpc32xx/clock.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * arch/arm/mach-lpc32xx/clock.h
- *
- * Author: Kevin Wells <kevin.wells@nxp.com>
- *
- * Copyright (C) 2010 NXP Semiconductors
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#ifndef __LPC32XX_CLOCK_H
-#define __LPC32XX_CLOCK_H
-
-struct clk {
-	struct list_head node;
-	struct clk *parent;
-	u32 rate;
-	u32 usecount;
-
-	int (*set_rate) (struct clk *, unsigned long);
-	unsigned long (*round_rate) (struct clk *, unsigned long);
-	unsigned long (*get_rate) (struct clk *clk);
-	int (*enable) (struct clk *, int);
-
-	/* Register address and bit mask for simple clocks */
-	void __iomem *enable_reg;
-	u32 enable_mask;
-};
-
-#endif
diff --git a/arch/arm/mach-lpc32xx/common.h b/arch/arm/mach-lpc32xx/common.h
index 30c9e64fc65b..02575c2444e4 100644
--- a/arch/arm/mach-lpc32xx/common.h
+++ b/arch/arm/mach-lpc32xx/common.h
@@ -24,7 +24,6 @@
 /*
  * Other arch specific structures and functions
  */
-extern void __init lpc32xx_init_irq(void);
 extern void __init lpc32xx_map_io(void);
 extern void __init lpc32xx_serial_init(void);
 
diff --git a/arch/arm/mach-lpc32xx/include/mach/irqs.h b/arch/arm/mach-lpc32xx/include/mach/irqs.h
deleted file mode 100644
index 00190535df90..000000000000
--- a/arch/arm/mach-lpc32xx/include/mach/irqs.h
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * arch/arm/mach-lpc32xx/include/mach/irqs.h
- *
- * Author: Kevin Wells <kevin.wells@nxp.com>
- *
- * Copyright (C) 2010 NXP Semiconductors
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#ifndef __ASM_ARM_ARCH_IRQS_H
-#define __ASM_ARM_ARCH_IRQS_H
-
-#define LPC32XX_SIC1_IRQ(n)		(32 + (n))
-#define LPC32XX_SIC2_IRQ(n)		(64 + (n))
-
-/*
- * MIC interrupts
- */
-#define IRQ_LPC32XX_SUB1IRQ		0
-#define IRQ_LPC32XX_SUB2IRQ		1
-#define IRQ_LPC32XX_PWM3		3
-#define IRQ_LPC32XX_PWM4		4
-#define IRQ_LPC32XX_HSTIMER		5
-#define IRQ_LPC32XX_WATCH		6
-#define IRQ_LPC32XX_UART_IIR3		7
-#define IRQ_LPC32XX_UART_IIR4		8
-#define IRQ_LPC32XX_UART_IIR5		9
-#define IRQ_LPC32XX_UART_IIR6		10
-#define IRQ_LPC32XX_FLASH		11
-#define IRQ_LPC32XX_SD1			13
-#define IRQ_LPC32XX_LCD			14
-#define IRQ_LPC32XX_SD0			15
-#define IRQ_LPC32XX_TIMER0		16
-#define IRQ_LPC32XX_TIMER1		17
-#define IRQ_LPC32XX_TIMER2		18
-#define IRQ_LPC32XX_TIMER3		19
-#define IRQ_LPC32XX_SSP0		20
-#define IRQ_LPC32XX_SSP1		21
-#define IRQ_LPC32XX_I2S0		22
-#define IRQ_LPC32XX_I2S1		23
-#define IRQ_LPC32XX_UART_IIR7		24
-#define IRQ_LPC32XX_UART_IIR2		25
-#define IRQ_LPC32XX_UART_IIR1		26
-#define IRQ_LPC32XX_MSTIMER		27
-#define IRQ_LPC32XX_DMA			28
-#define IRQ_LPC32XX_ETHERNET		29
-#define IRQ_LPC32XX_SUB1FIQ		30
-#define IRQ_LPC32XX_SUB2FIQ		31
-
-/*
- * SIC1 interrupts start at offset 32
- */
-#define IRQ_LPC32XX_JTAG_COMM_TX	LPC32XX_SIC1_IRQ(1)
-#define IRQ_LPC32XX_JTAG_COMM_RX	LPC32XX_SIC1_IRQ(2)
-#define IRQ_LPC32XX_GPI_28		LPC32XX_SIC1_IRQ(4)
-#define IRQ_LPC32XX_TS_P		LPC32XX_SIC1_IRQ(6)
-#define IRQ_LPC32XX_TS_IRQ		LPC32XX_SIC1_IRQ(7)
-#define IRQ_LPC32XX_TS_AUX		LPC32XX_SIC1_IRQ(8)
-#define IRQ_LPC32XX_SPI2		LPC32XX_SIC1_IRQ(12)
-#define IRQ_LPC32XX_PLLUSB		LPC32XX_SIC1_IRQ(13)
-#define IRQ_LPC32XX_PLLHCLK		LPC32XX_SIC1_IRQ(14)
-#define IRQ_LPC32XX_PLL397		LPC32XX_SIC1_IRQ(17)
-#define IRQ_LPC32XX_I2C_2		LPC32XX_SIC1_IRQ(18)
-#define IRQ_LPC32XX_I2C_1		LPC32XX_SIC1_IRQ(19)
-#define IRQ_LPC32XX_RTC			LPC32XX_SIC1_IRQ(20)
-#define IRQ_LPC32XX_KEY			LPC32XX_SIC1_IRQ(22)
-#define IRQ_LPC32XX_SPI1		LPC32XX_SIC1_IRQ(23)
-#define IRQ_LPC32XX_SW			LPC32XX_SIC1_IRQ(24)
-#define IRQ_LPC32XX_USB_OTG_TIMER	LPC32XX_SIC1_IRQ(25)
-#define IRQ_LPC32XX_USB_OTG_ATX		LPC32XX_SIC1_IRQ(26)
-#define IRQ_LPC32XX_USB_HOST		LPC32XX_SIC1_IRQ(27)
-#define IRQ_LPC32XX_USB_DEV_DMA		LPC32XX_SIC1_IRQ(28)
-#define IRQ_LPC32XX_USB_DEV_LP		LPC32XX_SIC1_IRQ(29)
-#define IRQ_LPC32XX_USB_DEV_HP		LPC32XX_SIC1_IRQ(30)
-#define IRQ_LPC32XX_USB_I2C		LPC32XX_SIC1_IRQ(31)
-
-/*
- * SIC2 interrupts start at offset 64
- */
-#define IRQ_LPC32XX_GPIO_00		LPC32XX_SIC2_IRQ(0)
-#define IRQ_LPC32XX_GPIO_01		LPC32XX_SIC2_IRQ(1)
-#define IRQ_LPC32XX_GPIO_02		LPC32XX_SIC2_IRQ(2)
-#define IRQ_LPC32XX_GPIO_03		LPC32XX_SIC2_IRQ(3)
-#define IRQ_LPC32XX_GPIO_04		LPC32XX_SIC2_IRQ(4)
-#define IRQ_LPC32XX_GPIO_05		LPC32XX_SIC2_IRQ(5)
-#define IRQ_LPC32XX_SPI2_DATAIN		LPC32XX_SIC2_IRQ(6)
-#define IRQ_LPC32XX_U2_HCTS		LPC32XX_SIC2_IRQ(7)
-#define IRQ_LPC32XX_P0_P1_IRQ		LPC32XX_SIC2_IRQ(8)
-#define IRQ_LPC32XX_GPI_08		LPC32XX_SIC2_IRQ(9)
-#define IRQ_LPC32XX_GPI_09		LPC32XX_SIC2_IRQ(10)
-#define IRQ_LPC32XX_GPI_19		LPC32XX_SIC2_IRQ(11)
-#define IRQ_LPC32XX_U7_HCTS		LPC32XX_SIC2_IRQ(12)
-#define IRQ_LPC32XX_GPI_07		LPC32XX_SIC2_IRQ(15)
-#define IRQ_LPC32XX_SDIO		LPC32XX_SIC2_IRQ(18)
-#define IRQ_LPC32XX_U5_RX		LPC32XX_SIC2_IRQ(19)
-#define IRQ_LPC32XX_SPI1_DATAIN		LPC32XX_SIC2_IRQ(20)
-#define IRQ_LPC32XX_GPI_00		LPC32XX_SIC2_IRQ(22)
-#define IRQ_LPC32XX_GPI_01		LPC32XX_SIC2_IRQ(23)
-#define IRQ_LPC32XX_GPI_02		LPC32XX_SIC2_IRQ(24)
-#define IRQ_LPC32XX_GPI_03		LPC32XX_SIC2_IRQ(25)
-#define IRQ_LPC32XX_GPI_04		LPC32XX_SIC2_IRQ(26)
-#define IRQ_LPC32XX_GPI_05		LPC32XX_SIC2_IRQ(27)
-#define IRQ_LPC32XX_GPI_06		LPC32XX_SIC2_IRQ(28)
-#define IRQ_LPC32XX_SYSCLK		LPC32XX_SIC2_IRQ(31)
-
-#define LPC32XX_NR_IRQS			96
-
-#endif
diff --git a/arch/arm/mach-lpc32xx/phy3250.c b/arch/arm/mach-lpc32xx/phy3250.c
index 0e4cbbe980eb..6c52bd32610e 100644
--- a/arch/arm/mach-lpc32xx/phy3250.c
+++ b/arch/arm/mach-lpc32xx/phy3250.c
@@ -23,7 +23,6 @@
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/dma-mapping.h>
-#include <linux/device.h>
 #include <linux/gpio.h>
 #include <linux/amba/bus.h>
 #include <linux/amba/clcd.h>
diff --git a/arch/arm/mach-lpc32xx/pm.c b/arch/arm/mach-lpc32xx/pm.c
index 207e81275ff0..62471570d586 100644
--- a/arch/arm/mach-lpc32xx/pm.c
+++ b/arch/arm/mach-lpc32xx/pm.c
@@ -73,7 +73,6 @@
 #include <mach/hardware.h>
 #include <mach/platform.h>
 #include "common.h"
-#include "clock.h"
 
 #define TEMP_IRAM_AREA  IO_ADDRESS(LPC32XX_IRAM_BASE)
 
diff --git a/arch/arm/mach-mediatek/Makefile b/arch/arm/mach-mediatek/Makefile
index 21164605b83f..dadae67d79b7 100644
--- a/arch/arm/mach-mediatek/Makefile
+++ b/arch/arm/mach-mediatek/Makefile
@@ -1,4 +1,2 @@
-ifeq ($(CONFIG_SMP),y)
-obj-$(CONFIG_ARCH_MEDIATEK) += platsmp.o
-endif
-obj-$(CONFIG_ARCH_MEDIATEK) += mediatek.o
+obj-$(CONFIG_SMP)	+= platsmp.o
+obj-y			+= mediatek.o
diff --git a/arch/arm/mach-omap1/Kconfig b/arch/arm/mach-omap1/Kconfig
index afb809509140..45c6b733c881 100644
--- a/arch/arm/mach-omap1/Kconfig
+++ b/arch/arm/mach-omap1/Kconfig
@@ -31,6 +31,32 @@ config ARCH_OMAP16XX
 	select ARCH_OMAP_OTG
 	select CPU_ARM926T
 
+config OMAP_MUX
+	bool "OMAP multiplexing support"
+	depends on ARCH_OMAP
+	default y
+	help
+	  Pin multiplexing support for OMAP boards. If your bootloader
+	  sets the multiplexing correctly, say N. Otherwise, or if unsure,
+	  say Y.
+
+config OMAP_MUX_DEBUG
+	bool "Multiplexing debug output"
+	depends on OMAP_MUX
+	help
+	  Makes the multiplexing functions print out a lot of debug info.
+	  This is useful if you want to find out the correct values of the
+	  multiplexing registers.
+
+config OMAP_MUX_WARNINGS
+	bool "Warn about pins the bootloader didn't set up"
+	depends on OMAP_MUX
+	default y
+	help
+	  Choose Y here to warn whenever driver initialization logic needs
+	  to change the pin multiplexing setup.	 When there are no warnings
+	  printed, it's safe to deselect OMAP_MUX for your product.
+
 comment "OMAP Board Type"
 	depends on ARCH_OMAP1
 
diff --git a/arch/arm/mach-omap1/i2c.c b/arch/arm/mach-omap1/i2c.c
index 82887d645a6a..32f6c53367bf 100644
--- a/arch/arm/mach-omap1/i2c.c
+++ b/arch/arm/mach-omap1/i2c.c
@@ -19,6 +19,7 @@
  *
  */
 
+#include <linux/i2c.h>
 #include <linux/i2c-omap.h>
 #include <mach/mux.h>
 #include "soc.h"
@@ -91,6 +92,88 @@ int __init omap_i2c_add_bus(struct omap_i2c_bus_platform_data *pdata,
 	return platform_device_register(pdev);
 }
 
+#define OMAP_I2C_MAX_CONTROLLERS 4
+static struct omap_i2c_bus_platform_data i2c_pdata[OMAP_I2C_MAX_CONTROLLERS];
+
+#define OMAP_I2C_CMDLINE_SETUP	(BIT(31))
+
+/**
+ * omap_i2c_bus_setup - Process command line options for the I2C bus speed
+ * @str: String of options
+ *
+ * This function allow to override the default I2C bus speed for given I2C
+ * bus with a command line option.
+ *
+ * Format: i2c_bus=bus_id,clkrate (in kHz)
+ *
+ * Returns 1 on success, 0 otherwise.
+ */
+static int __init omap_i2c_bus_setup(char *str)
+{
+	int ints[3];
+
+	get_options(str, 3, ints);
+	if (ints[0] < 2 || ints[1] < 1 ||
+			ints[1] > OMAP_I2C_MAX_CONTROLLERS)
+		return 0;
+	i2c_pdata[ints[1] - 1].clkrate = ints[2];
+	i2c_pdata[ints[1] - 1].clkrate |= OMAP_I2C_CMDLINE_SETUP;
+
+	return 1;
+}
+__setup("i2c_bus=", omap_i2c_bus_setup);
+
+/*
+ * Register busses defined in command line but that are not registered with
+ * omap_register_i2c_bus from board initialization code.
+ */
+int __init omap_register_i2c_bus_cmdline(void)
+{
+	int i, err = 0;
+
+	for (i = 0; i < ARRAY_SIZE(i2c_pdata); i++)
+		if (i2c_pdata[i].clkrate & OMAP_I2C_CMDLINE_SETUP) {
+			i2c_pdata[i].clkrate &= ~OMAP_I2C_CMDLINE_SETUP;
+			err = omap_i2c_add_bus(&i2c_pdata[i], i + 1);
+			if (err)
+				goto out;
+		}
+
+out:
+	return err;
+}
+
+/**
+ * omap_register_i2c_bus - register I2C bus with device descriptors
+ * @bus_id: bus id counting from number 1
+ * @clkrate: clock rate of the bus in kHz
+ * @info: pointer into I2C device descriptor table or NULL
+ * @len: number of descriptors in the table
+ *
+ * Returns 0 on success or an error code.
+ */
+int __init omap_register_i2c_bus(int bus_id, u32 clkrate,
+			  struct i2c_board_info const *info,
+			  unsigned len)
+{
+	int err;
+
+	BUG_ON(bus_id < 1 || bus_id > OMAP_I2C_MAX_CONTROLLERS);
+
+	if (info) {
+		err = i2c_register_board_info(bus_id, info, len);
+		if (err)
+			return err;
+	}
+
+	if (!i2c_pdata[bus_id - 1].clkrate)
+		i2c_pdata[bus_id - 1].clkrate = clkrate;
+
+	i2c_pdata[bus_id - 1].clkrate &= ~OMAP_I2C_CMDLINE_SETUP;
+
+	return omap_i2c_add_bus(&i2c_pdata[bus_id - 1], bus_id);
+}
+
 static  int __init omap_i2c_cmdline(void)
 {
 	return omap_register_i2c_bus_cmdline();
diff --git a/arch/arm/mach-omap2/Makefile b/arch/arm/mach-omap2/Makefile
index 5b37ec29996e..469894082fea 100644
--- a/arch/arm/mach-omap2/Makefile
+++ b/arch/arm/mach-omap2/Makefile
@@ -6,7 +6,7 @@ ccflags-y := -I$(srctree)/$(src)/include \
 	-I$(srctree)/arch/arm/plat-omap/include
 
 # Common support
-obj-y := id.o io.o control.o mux.o devices.o fb.o serial.o timer.o pm.o \
+obj-y := id.o io.o control.o devices.o fb.o timer.o pm.o \
 	 common.o gpio.o dma.o wd_timer.o display.o i2c.o hdq1w.o omap_hwmod.o \
 	 omap_device.o omap-headsmp.o sram.o drm.o
 
@@ -63,9 +63,6 @@ obj-$(CONFIG_ARCH_OMAP4)		+= omap4-restart.o
 obj-$(CONFIG_SOC_OMAP5)			+= omap4-restart.o
 obj-$(CONFIG_SOC_DRA7XX)		+= omap4-restart.o
 
-# Pin multiplexing
-obj-$(CONFIG_ARCH_OMAP3)		+= mux34xx.o
-
 # SMS/SDRC
 obj-$(CONFIG_ARCH_OMAP2)		+= sdrc2xxx.o
 # obj-$(CONFIG_ARCH_OMAP3)		+= sdrc3xxx.o
@@ -80,7 +77,7 @@ endif
 # Power Management
 omap-4-5-pm-common			= omap-mpuss-lowpower.o
 obj-$(CONFIG_ARCH_OMAP4)		+= $(omap-4-5-pm-common)
-obj-$(CONFIG_ARCH_OMAP5)		+= $(omap-4-5-pm-common)
+obj-$(CONFIG_SOC_OMAP5)			+= $(omap-4-5-pm-common)
 obj-$(CONFIG_OMAP_PM_NOOP)		+= omap-pm-noop.o
 
 ifeq ($(CONFIG_PM),y)
@@ -235,26 +232,15 @@ obj-$(CONFIG_MACH_NOKIA_N8X0)		+= board-n8x0.o
 
 # Platform specific device init code
 
-omap-flash-$(CONFIG_MTD_NAND_OMAP2)	:= board-flash.o
-omap-flash-$(CONFIG_MTD_ONENAND_OMAP2)	:= board-flash.o
-obj-y					+= $(omap-flash-y) $(omap-flash-m)
-
 omap-hsmmc-$(CONFIG_MMC_OMAP_HS)	:= hsmmc.o
 obj-y					+= $(omap-hsmmc-m) $(omap-hsmmc-y)
 
-obj-y					+= usb-musb.o
 obj-y					+= omap_phy_internal.o
 
 obj-$(CONFIG_MACH_OMAP2_TUSB6010)	+= usb-tusb6010.o
-obj-y					+= usb-host.o
 
 onenand-$(CONFIG_MTD_ONENAND_OMAP2)	:= gpmc-onenand.o
 obj-y					+= $(onenand-m) $(onenand-y)
 
 nand-$(CONFIG_MTD_NAND_OMAP2)		:= gpmc-nand.o
 obj-y					+= $(nand-m) $(nand-y)
-
-smsc911x-$(CONFIG_SMSC911X)		:= gpmc-smsc911x.o
-obj-y					+= $(smsc911x-m) $(smsc911x-y)
-
-obj-y					+= common-board-devices.o twl-common.o dss-common.o
diff --git a/arch/arm/mach-omap2/board-flash.c b/arch/arm/mach-omap2/board-flash.c
deleted file mode 100644
index 2188dc30e232..000000000000
--- a/arch/arm/mach-omap2/board-flash.c
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * board-flash.c
- * Modified from mach-omap2/board-3430sdp-flash.c
- *
- * Copyright (C) 2009 Nokia Corporation
- * Copyright (C) 2009 Texas Instruments
- *
- * Vimal Singh <vimalsingh@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/kernel.h>
-#include <linux/omap-gpmc.h>
-#include <linux/platform_device.h>
-#include <linux/mtd/physmap.h>
-#include <linux/io.h>
-
-#include <linux/platform_data/mtd-nand-omap2.h>
-#include <linux/platform_data/mtd-onenand-omap2.h>
-
-#include "soc.h"
-#include "common.h"
-#include "board-flash.h"
-
-#define REG_FPGA_REV			0x10
-#define REG_FPGA_DIP_SWITCH_INPUT2	0x60
-#define MAX_SUPPORTED_GPMC_CONFIG	3
-
-#define DEBUG_BASE		0x08000000 /* debug board */
-
-/* various memory sizes */
-#define FLASH_SIZE_SDPV1	SZ_64M	/* NOR flash (64 Meg aligned) */
-#define FLASH_SIZE_SDPV2	SZ_128M	/* NOR flash (256 Meg aligned) */
-
-static struct physmap_flash_data board_nor_data = {
-	.width		= 2,
-};
-
-static struct resource board_nor_resource = {
-	.flags		= IORESOURCE_MEM,
-};
-
-static struct platform_device board_nor_device = {
-	.name		= "physmap-flash",
-	.id		= 0,
-	.dev		= {
-			.platform_data = &board_nor_data,
-	},
-	.num_resources	= 1,
-	.resource	= &board_nor_resource,
-};
-
-static void
-__init board_nor_init(struct mtd_partition *nor_parts, u8 nr_parts, u8 cs)
-{
-	int err;
-
-	board_nor_data.parts	= nor_parts;
-	board_nor_data.nr_parts	= nr_parts;
-
-	/* Configure start address and size of NOR device */
-	if (omap_rev() >= OMAP3430_REV_ES1_0) {
-		err = gpmc_cs_request(cs, FLASH_SIZE_SDPV2 - 1,
-				(unsigned long *)&board_nor_resource.start);
-		board_nor_resource.end = board_nor_resource.start
-					+ FLASH_SIZE_SDPV2 - 1;
-	} else {
-		err = gpmc_cs_request(cs, FLASH_SIZE_SDPV1 - 1,
-				(unsigned long *)&board_nor_resource.start);
-		board_nor_resource.end = board_nor_resource.start
-					+ FLASH_SIZE_SDPV1 - 1;
-	}
-	if (err < 0) {
-		pr_err("NOR: Can't request GPMC CS\n");
-		return;
-	}
-	if (platform_device_register(&board_nor_device) < 0)
-		pr_err("Unable to register NOR device\n");
-}
-
-#if IS_ENABLED(CONFIG_MTD_ONENAND_OMAP2)
-static struct omap_onenand_platform_data board_onenand_data = {
-	.dma_channel	= -1,   /* disable DMA in OMAP OneNAND driver */
-};
-
-void
-__init board_onenand_init(struct mtd_partition *onenand_parts,
-				u8 nr_parts, u8 cs)
-{
-	board_onenand_data.cs		= cs;
-	board_onenand_data.parts	= onenand_parts;
-	board_onenand_data.nr_parts	= nr_parts;
-
-	gpmc_onenand_init(&board_onenand_data);
-}
-#endif /* IS_ENABLED(CONFIG_MTD_ONENAND_OMAP2) */
-
-#if IS_ENABLED(CONFIG_MTD_NAND_OMAP2)
-
-/* Note that all values in this struct are in nanoseconds */
-struct gpmc_timings nand_default_timings[1] = {
-	{
-		.sync_clk = 0,
-
-		.cs_on = 0,
-		.cs_rd_off = 36,
-		.cs_wr_off = 36,
-
-		.we_on = 6,
-		.oe_on = 6,
-
-		.adv_on = 6,
-		.adv_rd_off = 24,
-		.adv_wr_off = 36,
-
-		.we_off = 30,
-		.oe_off = 48,
-
-		.access = 54,
-		.rd_cycle = 72,
-		.wr_cycle = 72,
-
-		.wr_access = 30,
-		.wr_data_mux_bus = 0,
-	},
-};
-
-static struct omap_nand_platform_data board_nand_data;
-
-void
-__init board_nand_init(struct mtd_partition *nand_parts, u8 nr_parts, u8 cs,
-				int nand_type, struct gpmc_timings *gpmc_t)
-{
-	board_nand_data.cs		= cs;
-	board_nand_data.parts		= nand_parts;
-	board_nand_data.nr_parts	= nr_parts;
-	board_nand_data.devsize		= nand_type;
-
-	board_nand_data.ecc_opt = OMAP_ECC_HAM1_CODE_SW;
-	gpmc_nand_init(&board_nand_data, gpmc_t);
-}
-#endif /* IS_ENABLED(CONFIG_MTD_NAND_OMAP2) */
-
-/**
- * get_gpmc0_type - Reads the FPGA DIP_SWITCH_INPUT_REGISTER2 to get
- * the various cs values.
- */
-static u8 get_gpmc0_type(void)
-{
-	u8 cs = 0;
-	void __iomem *fpga_map_addr;
-
-	fpga_map_addr = ioremap(DEBUG_BASE, 4096);
-	if (!fpga_map_addr)
-		return -ENOMEM;
-
-	if (!(readw_relaxed(fpga_map_addr + REG_FPGA_REV)))
-		/* we dont have an DEBUG FPGA??? */
-		/* Depend on #defines!! default to strata boot return param */
-		goto unmap;
-
-	/* S8-DIP-OFF = 1, S8-DIP-ON = 0 */
-	cs = readw_relaxed(fpga_map_addr + REG_FPGA_DIP_SWITCH_INPUT2) & 0xf;
-
-	/* ES2.0 SDP's onwards 4 dip switches are provided for CS */
-	if (omap_rev() >= OMAP3430_REV_ES1_0)
-		/* change (S8-1:4=DS-2:0) to (S8-4:1=DS-2:0) */
-		cs = ((cs & 8) >> 3) | ((cs & 4) >> 1) |
-			((cs & 2) << 1) | ((cs & 1) << 3);
-	else
-		/* change (S8-1:3=DS-2:0) to (S8-3:1=DS-2:0) */
-		cs = ((cs & 4) >> 2) | (cs & 2) | ((cs & 1) << 2);
-unmap:
-	iounmap(fpga_map_addr);
-	return cs;
-}
-
-/**
- * board_flash_init - Identify devices connected to GPMC and register.
- *
- * @return - void.
- */
-void __init board_flash_init(struct flash_partitions partition_info[],
-			char chip_sel_board[][GPMC_CS_NUM], int nand_type)
-{
-	u8		cs = 0;
-	u8		norcs = GPMC_CS_NUM + 1;
-	u8		nandcs = GPMC_CS_NUM + 1;
-	u8		onenandcs = GPMC_CS_NUM + 1;
-	u8		idx;
-	unsigned char	*config_sel = NULL;
-
-	/* REVISIT: Is this return correct idx for 2430 SDP?
-	 * for which cs configuration matches for 2430 SDP?
-	 */
-	idx = get_gpmc0_type();
-	if (idx >= MAX_SUPPORTED_GPMC_CONFIG) {
-		pr_err("%s: Invalid chip select: %d\n", __func__, cs);
-		return;
-	}
-	config_sel = (unsigned char *)(chip_sel_board[idx]);
-
-	while (cs < GPMC_CS_NUM) {
-		switch (config_sel[cs]) {
-		case PDC_NOR:
-			if (norcs > GPMC_CS_NUM)
-				norcs = cs;
-			break;
-		case PDC_NAND:
-			if (nandcs > GPMC_CS_NUM)
-				nandcs = cs;
-			break;
-		case PDC_ONENAND:
-			if (onenandcs > GPMC_CS_NUM)
-				onenandcs = cs;
-			break;
-		}
-		cs++;
-	}
-
-	if (norcs > GPMC_CS_NUM)
-		pr_err("NOR: Unable to find configuration in GPMC\n");
-	else
-		board_nor_init(partition_info[0].parts,
-				partition_info[0].nr_parts, norcs);
-
-	if (onenandcs > GPMC_CS_NUM)
-		pr_err("OneNAND: Unable to find configuration in GPMC\n");
-	else
-		board_onenand_init(partition_info[1].parts,
-					partition_info[1].nr_parts, onenandcs);
-
-	if (nandcs > GPMC_CS_NUM)
-		pr_err("NAND: Unable to find configuration in GPMC\n");
-	else
-		board_nand_init(partition_info[2].parts,
-			partition_info[2].nr_parts, nandcs,
-			nand_type, nand_default_timings);
-}
diff --git a/arch/arm/mach-omap2/board-flash.h b/arch/arm/mach-omap2/board-flash.h
deleted file mode 100644
index 8b39eec07318..000000000000
--- a/arch/arm/mach-omap2/board-flash.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- *  board-sdp.h
- *
- *  Information structures for SDP-specific board config data
- *
- *  Copyright (C) 2009 Nokia Corporation
- *  Copyright (C) 2009 Texas Instruments
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#include <linux/mtd/mtd.h>
-#include <linux/mtd/partitions.h>
-
-#define PDC_NOR		1
-#define PDC_NAND	2
-#define PDC_ONENAND	3
-#define DBG_MPDB	4
-
-struct flash_partitions {
-	struct mtd_partition *parts;
-	int nr_parts;
-};
-
-#if IS_ENABLED(CONFIG_MTD_NAND_OMAP2) || IS_ENABLED(CONFIG_MTD_ONENAND_OMAP2)
-extern void board_flash_init(struct flash_partitions [],
-				char chip_sel[][GPMC_CS_NUM], int nand_type);
-#else
-static inline void board_flash_init(struct flash_partitions part[],
-				char chip_sel[][GPMC_CS_NUM], int nand_type)
-{
-}
-#endif
-
-#if IS_ENABLED(CONFIG_MTD_NAND_OMAP2)
-extern void board_nand_init(struct mtd_partition *nand_parts,
-		u8 nr_parts, u8 cs, int nand_type, struct gpmc_timings *gpmc_t);
-extern struct gpmc_timings nand_default_timings[];
-#else
-static inline void board_nand_init(struct mtd_partition *nand_parts,
-		u8 nr_parts, u8 cs, int nand_type, struct gpmc_timings *gpmc_t)
-{
-}
-#define	nand_default_timings	NULL
-#endif
-
-#if IS_ENABLED(CONFIG_MTD_ONENAND_OMAP2)
-extern void board_onenand_init(struct mtd_partition *nand_parts,
-					u8 nr_parts, u8 cs);
-#else
-static inline void board_onenand_init(struct mtd_partition *nand_parts,
-					u8 nr_parts, u8 cs)
-{
-}
-#endif
diff --git a/arch/arm/mach-omap2/board-generic.c b/arch/arm/mach-omap2/board-generic.c
index bab814d2f37d..36d9943205ca 100644
--- a/arch/arm/mach-omap2/board-generic.c
+++ b/arch/arm/mach-omap2/board-generic.c
@@ -30,8 +30,6 @@ static const struct of_device_id omap_dt_match_table[] __initconst = {
 
 static void __init omap_generic_init(void)
 {
-	omapdss_early_init_of();
-
 	pdata_quirks_init(omap_dt_match_table);
 
 	omapdss_init_of();
@@ -341,6 +339,7 @@ static const char *const dra72x_boards_compat[] __initconst = {
 	"ti,am5718",
 	"ti,am5716",
 	"ti,dra722",
+	"ti,dra718",
 	NULL,
 };
 
diff --git a/arch/arm/mach-omap2/clockdomains7xx_data.c b/arch/arm/mach-omap2/clockdomains7xx_data.c
index ef9ed36e8a61..6c679659cda5 100644
--- a/arch/arm/mach-omap2/clockdomains7xx_data.c
+++ b/arch/arm/mach-omap2/clockdomains7xx_data.c
@@ -409,7 +409,7 @@ static struct clockdomain l4sec_7xx_clkdm = {
 	.dep_bit	  = DRA7XX_L4SEC_STATDEP_SHIFT,
 	.wkdep_srcs	  = l4sec_wkup_sleep_deps,
 	.sleepdep_srcs	  = l4sec_wkup_sleep_deps,
-	.flags		  = CLKDM_CAN_HWSUP_SWSUP,
+	.flags		  = CLKDM_CAN_SWSUP,
 };
 
 static struct clockdomain l3main1_7xx_clkdm = {
diff --git a/arch/arm/mach-omap2/common-board-devices.c b/arch/arm/mach-omap2/common-board-devices.c
deleted file mode 100644
index 5388fcd3de72..000000000000
--- a/arch/arm/mach-omap2/common-board-devices.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * common-board-devices.c
- *
- * Copyright (C) 2011 CompuLab, Ltd.
- * Author: Mike Rapoport <mike@compulab.co.il>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/gpio.h>
-#include <linux/spi/spi.h>
-#include <linux/spi/ads7846.h>
-
-#include <linux/platform_data/spi-omap2-mcspi.h>
-
-#include "common.h"
-#include "common-board-devices.h"
-
-#if IS_ENABLED(CONFIG_TOUCHSCREEN_ADS7846)
-static struct omap2_mcspi_device_config ads7846_mcspi_config = {
-	.turbo_mode	= 0,
-};
-
-static struct ads7846_platform_data ads7846_config = {
-	.x_max			= 0x0fff,
-	.y_max			= 0x0fff,
-	.x_plate_ohms		= 180,
-	.pressure_max		= 255,
-	.debounce_max		= 10,
-	.debounce_tol		= 3,
-	.debounce_rep		= 1,
-	.gpio_pendown		= -EINVAL,
-	.keep_vref_on		= 1,
-};
-
-static struct spi_board_info ads7846_spi_board_info __initdata = {
-	.modalias		= "ads7846",
-	.bus_num		= -EINVAL,
-	.chip_select		= 0,
-	.max_speed_hz		= 1500000,
-	.controller_data	= &ads7846_mcspi_config,
-	.irq			= -EINVAL,
-	.platform_data		= &ads7846_config,
-};
-
-void __init omap_ads7846_init(int bus_num, int gpio_pendown, int gpio_debounce,
-			      struct ads7846_platform_data *board_pdata)
-{
-	struct spi_board_info *spi_bi = &ads7846_spi_board_info;
-	int err;
-
-	/*
-	 * If a board defines get_pendown_state() function, request the pendown
-	 * GPIO and set the GPIO debounce time.
-	 * If a board does not define the get_pendown_state() function, then
-	 * the ads7846 driver will setup the pendown GPIO itself.
-	 */
-	if (board_pdata && board_pdata->get_pendown_state) {
-		err = gpio_request_one(gpio_pendown, GPIOF_IN, "TSPenDown");
-		if (err) {
-			pr_err("Couldn't obtain gpio for TSPenDown: %d\n", err);
-			return;
-		}
-
-		if (gpio_debounce)
-			gpio_set_debounce(gpio_pendown, gpio_debounce);
-
-		gpio_export(gpio_pendown, 0);
-	}
-
-	spi_bi->bus_num	= bus_num;
-	spi_bi->irq	= gpio_to_irq(gpio_pendown);
-
-	ads7846_config.gpio_pendown = gpio_pendown;
-
-	if (board_pdata) {
-		board_pdata->gpio_pendown = gpio_pendown;
-		board_pdata->gpio_pendown_debounce = gpio_debounce;
-		spi_bi->platform_data = board_pdata;
-	}
-
-	spi_register_board_info(&ads7846_spi_board_info, 1);
-}
-#else
-void __init omap_ads7846_init(int bus_num, int gpio_pendown, int gpio_debounce,
-			      struct ads7846_platform_data *board_pdata)
-{
-}
-#endif
diff --git a/arch/arm/mach-omap2/common-board-devices.h b/arch/arm/mach-omap2/common-board-devices.h
index 07c88ae083fb..335c7822fea1 100644
--- a/arch/arm/mach-omap2/common-board-devices.h
+++ b/arch/arm/mach-omap2/common-board-devices.h
@@ -3,15 +3,7 @@
 
 #include <sound/tlv320aic3x.h>
 #include <linux/mfd/menelaus.h>
-#include "twl-common.h"
 
-#define NAND_BLOCK_SIZE	SZ_128K
-
-struct mtd_partition;
-struct ads7846_platform_data;
-
-void omap_ads7846_init(int bus_num, int gpio_pendown, int gpio_debounce,
-		       struct ads7846_platform_data *board_pdata);
 void *n8x0_legacy_init(void);
 
 extern struct menelaus_platform_data n8x0_menelaus_platform_data;
diff --git a/arch/arm/mach-omap2/common.h b/arch/arm/mach-omap2/common.h
index deed42e1dd9c..c4f2ace91ea2 100644
--- a/arch/arm/mach-omap2/common.h
+++ b/arch/arm/mach-omap2/common.h
@@ -77,15 +77,6 @@ static inline int omap4_pm_init_early(void)
 }
 #endif
 
-#ifdef CONFIG_OMAP_MUX
-int omap_mux_late_init(void);
-#else
-static inline int omap_mux_late_init(void)
-{
-	return 0;
-}
-#endif
-
 extern void omap2_init_common_infrastructure(void);
 
 extern void omap_init_time(void);
@@ -262,8 +253,6 @@ extern void __iomem *omap4_get_sar_ram_base(void);
 extern void omap4_mpuss_early_init(void);
 extern void omap_do_wfi(void);
 
-extern void omap4_secondary_startup(void);
-extern void omap4460_secondary_startup(void);
 
 #ifdef CONFIG_SMP
 /* Needed for secondary core boot */
@@ -275,16 +264,11 @@ extern void omap4_cpu_die(unsigned int cpu);
 extern int omap4_cpu_kill(unsigned int cpu);
 
 extern const struct smp_operations omap4_smp_ops;
-
-extern void omap5_secondary_startup(void);
-extern void omap5_secondary_hyp_startup(void);
 #endif
 
 #if defined(CONFIG_SMP) && defined(CONFIG_PM)
 extern int omap4_mpuss_init(void);
 extern int omap4_enter_lowpower(unsigned int cpu, unsigned int power_state);
-extern int omap4_finish_suspend(unsigned long cpu_state);
-extern void omap4_cpu_resume(void);
 extern int omap4_hotplug_cpu(unsigned int cpu, unsigned int power_state);
 #else
 static inline int omap4_enter_lowpower(unsigned int cpu,
@@ -305,14 +289,41 @@ static inline int omap4_mpuss_init(void)
 	return 0;
 }
 
+#endif
+
+#ifdef CONFIG_ARCH_OMAP4
+void omap4_secondary_startup(void);
+void omap4460_secondary_startup(void);
+int omap4_finish_suspend(unsigned long cpu_state);
+void omap4_cpu_resume(void);
+#else
+static inline void omap4_secondary_startup(void)
+{
+}
+
+static inline void omap4460_secondary_startup(void)
+{
+}
 static inline int omap4_finish_suspend(unsigned long cpu_state)
 {
 	return 0;
 }
-
 static inline void omap4_cpu_resume(void)
-{}
+{
+}
+#endif
 
+#if defined(CONFIG_SOC_OMAP5) || defined(CONFIG_SOC_DRA7XX)
+void omap5_secondary_startup(void);
+void omap5_secondary_hyp_startup(void);
+#else
+static inline void omap5_secondary_startup(void)
+{
+}
+
+static inline void omap5_secondary_hyp_startup(void)
+{
+}
 #endif
 
 void pdata_quirks_init(const struct of_device_id *);
@@ -332,7 +343,6 @@ extern int omap_dss_reset(struct omap_hwmod *);
 int omap_clk_init(void);
 
 int __init omapdss_init_of(void);
-void __init omapdss_early_init_of(void);
 
 #endif /* __ASSEMBLER__ */
 #endif /* __ARCH_ARM_MACH_OMAP2PLUS_COMMON_H */
diff --git a/arch/arm/mach-omap2/cpuidle44xx.c b/arch/arm/mach-omap2/cpuidle44xx.c
index fa138d4032b6..a8b291f00109 100644
--- a/arch/arm/mach-omap2/cpuidle44xx.c
+++ b/arch/arm/mach-omap2/cpuidle44xx.c
@@ -21,6 +21,7 @@
 #include "common.h"
 #include "pm.h"
 #include "prm.h"
+#include "soc.h"
 #include "clockdomain.h"
 
 #define MAX_CPUS	2
@@ -30,6 +31,7 @@ struct idle_statedata {
 	u32 cpu_state;
 	u32 mpu_logic_state;
 	u32 mpu_state;
+	u32 mpu_state_vote;
 };
 
 static struct idle_statedata omap4_idle_data[] = {
@@ -50,12 +52,26 @@ static struct idle_statedata omap4_idle_data[] = {
 	},
 };
 
+static struct idle_statedata omap5_idle_data[] = {
+	{
+		.cpu_state = PWRDM_POWER_ON,
+		.mpu_state = PWRDM_POWER_ON,
+		.mpu_logic_state = PWRDM_POWER_ON,
+	},
+	{
+		.cpu_state = PWRDM_POWER_RET,
+		.mpu_state = PWRDM_POWER_RET,
+		.mpu_logic_state = PWRDM_POWER_RET,
+	},
+};
+
 static struct powerdomain *mpu_pd, *cpu_pd[MAX_CPUS];
 static struct clockdomain *cpu_clkdm[MAX_CPUS];
 
 static atomic_t abort_barrier;
 static bool cpu_done[MAX_CPUS];
 static struct idle_statedata *state_ptr = &omap4_idle_data[0];
+static DEFINE_RAW_SPINLOCK(mpu_lock);
 
 /* Private functions */
 
@@ -77,6 +93,32 @@ static int omap_enter_idle_simple(struct cpuidle_device *dev,
 	return index;
 }
 
+static int omap_enter_idle_smp(struct cpuidle_device *dev,
+			       struct cpuidle_driver *drv,
+			       int index)
+{
+	struct idle_statedata *cx = state_ptr + index;
+	unsigned long flag;
+
+	raw_spin_lock_irqsave(&mpu_lock, flag);
+	cx->mpu_state_vote++;
+	if (cx->mpu_state_vote == num_online_cpus()) {
+		pwrdm_set_logic_retst(mpu_pd, cx->mpu_logic_state);
+		omap_set_pwrdm_state(mpu_pd, cx->mpu_state);
+	}
+	raw_spin_unlock_irqrestore(&mpu_lock, flag);
+
+	omap4_enter_lowpower(dev->cpu, cx->cpu_state);
+
+	raw_spin_lock_irqsave(&mpu_lock, flag);
+	if (cx->mpu_state_vote == num_online_cpus())
+		omap_set_pwrdm_state(mpu_pd, PWRDM_POWER_ON);
+	cx->mpu_state_vote--;
+	raw_spin_unlock_irqrestore(&mpu_lock, flag);
+
+	return index;
+}
+
 static int omap_enter_idle_coupled(struct cpuidle_device *dev,
 			struct cpuidle_driver *drv,
 			int index)
@@ -220,6 +262,32 @@ static struct cpuidle_driver omap4_idle_driver = {
 	.safe_state_index = 0,
 };
 
+static struct cpuidle_driver omap5_idle_driver = {
+	.name				= "omap5_idle",
+	.owner				= THIS_MODULE,
+	.states = {
+		{
+			/* C1 - CPU0 ON + CPU1 ON + MPU ON */
+			.exit_latency = 2 + 2,
+			.target_residency = 5,
+			.enter = omap_enter_idle_simple,
+			.name = "C1",
+			.desc = "CPUx WFI, MPUSS ON"
+		},
+		{
+			/* C2 - CPU0 RET + CPU1 RET + MPU CSWR */
+			.exit_latency = 48 + 60,
+			.target_residency = 100,
+			.flags = CPUIDLE_FLAG_TIMER_STOP,
+			.enter = omap_enter_idle_smp,
+			.name = "C2",
+			.desc = "CPUx CSWR, MPUSS CSWR",
+		},
+	},
+	.state_count = ARRAY_SIZE(omap5_idle_data),
+	.safe_state_index = 0,
+};
+
 /* Public functions */
 
 /**
@@ -230,6 +298,16 @@ static struct cpuidle_driver omap4_idle_driver = {
  */
 int __init omap4_idle_init(void)
 {
+	struct cpuidle_driver *idle_driver;
+
+	if (soc_is_omap54xx()) {
+		state_ptr = &omap5_idle_data[0];
+		idle_driver = &omap5_idle_driver;
+	} else {
+		state_ptr = &omap4_idle_data[0];
+		idle_driver = &omap4_idle_driver;
+	}
+
 	mpu_pd = pwrdm_lookup("mpu_pwrdm");
 	cpu_pd[0] = pwrdm_lookup("cpu0_pwrdm");
 	cpu_pd[1] = pwrdm_lookup("cpu1_pwrdm");
@@ -244,5 +322,5 @@ int __init omap4_idle_init(void)
 	/* Configure the broadcast timer on each cpu */
 	on_each_cpu(omap_setup_broadcast_timer, NULL, 1);
 
-	return cpuidle_register(&omap4_idle_driver, cpu_online_mask);
+	return cpuidle_register(idle_driver, cpu_online_mask);
 }
diff --git a/arch/arm/mach-omap2/devices.c b/arch/arm/mach-omap2/devices.c
index 60a20f3b44de..3fdb94599184 100644
--- a/arch/arm/mach-omap2/devices.c
+++ b/arch/arm/mach-omap2/devices.c
@@ -30,7 +30,6 @@
 
 #include "soc.h"
 #include "common.h"
-#include "mux.h"
 #include "control.h"
 #include "display.h"
 
diff --git a/arch/arm/mach-omap2/display.c b/arch/arm/mach-omap2/display.c
index 70b3eaf085e4..e71cca0950e9 100644
--- a/arch/arm/mach-omap2/display.c
+++ b/arch/arm/mach-omap2/display.c
@@ -565,11 +565,6 @@ int omap_dss_reset(struct omap_hwmod *oh)
 	return r;
 }
 
-void __init omapdss_early_init_of(void)
-{
-
-}
-
 static const char * const omapdss_compat_names[] __initconst = {
 	"ti,omap2-dss",
 	"ti,omap3-dss",
diff --git a/arch/arm/mach-omap2/dss-common.c b/arch/arm/mach-omap2/dss-common.c
deleted file mode 100644
index 1d583bc0b1a9..000000000000
--- a/arch/arm/mach-omap2/dss-common.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2012 Texas Instruments, Inc..
- * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-/*
- * NOTE: this is a transitional file to help with DT adaptation.
- * This file will be removed when DSS supports DT.
- */
-
-#include <linux/kernel.h>
-#include <linux/gpio.h>
-#include <linux/platform_device.h>
-
-#include <linux/platform_data/omapdss.h>
-#include <video/omap-panel-data.h>
-
-#include "soc.h"
-#include "dss-common.h"
-#include "mux.h"
-#include "display.h"
-
diff --git a/arch/arm/mach-omap2/dss-common.h b/arch/arm/mach-omap2/dss-common.h
deleted file mode 100644
index a9becf0d5be8..000000000000
--- a/arch/arm/mach-omap2/dss-common.h
+++ /dev/null
@@ -1,13 +0,0 @@
-#ifndef __OMAP_DSS_COMMON__
-#define __OMAP_DSS_COMMON__
-
-/*
- * NOTE: this is a transitional file to help with DT adaptation.
- * This file will be removed when DSS supports DT.
- */
-
-void __init omap4_panda_display_init_of(void);
-void __init omap_4430sdp_display_init_of(void);
-void __init omap3_igep2_display_init_of(void);
-
-#endif
diff --git a/arch/arm/mach-omap2/gpmc-smsc911x.c b/arch/arm/mach-omap2/gpmc-smsc911x.c
deleted file mode 100644
index 2757504a13c4..000000000000
--- a/arch/arm/mach-omap2/gpmc-smsc911x.c
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * linux/arch/arm/mach-omap2/gpmc-smsc911x.c
- *
- * Copyright (C) 2009 Li-Pro.Net
- * Stephan Linz <linz@li-pro.net>
- *
- * Modified from linux/arch/arm/mach-omap2/gpmc-smc91x.c
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#define pr_fmt(fmt) "%s: " fmt, __func__
-
-#include <linux/kernel.h>
-#include <linux/platform_device.h>
-#include <linux/gpio.h>
-#include <linux/delay.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/smsc911x.h>
-
-#include "gpmc.h"
-#include "gpmc-smsc911x.h"
-
-static struct resource gpmc_smsc911x_resources[] = {
-	[0] = {
-		.flags		= IORESOURCE_MEM,
-	},
-	[1] = {
-		.flags		= IORESOURCE_IRQ | IORESOURCE_IRQ_LOWLEVEL,
-	},
-};
-
-static struct smsc911x_platform_config gpmc_smsc911x_config = {
-	.phy_interface	= PHY_INTERFACE_MODE_MII,
-	.irq_polarity	= SMSC911X_IRQ_POLARITY_ACTIVE_LOW,
-	.irq_type	= SMSC911X_IRQ_TYPE_OPEN_DRAIN,
-};
-
-/*
- * Initialize smsc911x device connected to the GPMC. Note that we
- * assume that pin multiplexing is done in the board-*.c file,
- * or in the bootloader.
- */
-void __init gpmc_smsc911x_init(struct omap_smsc911x_platform_data *gpmc_cfg)
-{
-	struct platform_device *pdev;
-	unsigned long cs_mem_base;
-	int ret;
-
-	if (gpmc_cs_request(gpmc_cfg->cs, SZ_16M, &cs_mem_base) < 0) {
-		pr_err("Failed to request GPMC mem region\n");
-		return;
-	}
-
-	gpmc_smsc911x_resources[0].start = cs_mem_base + 0x0;
-	gpmc_smsc911x_resources[0].end = cs_mem_base + 0xff;
-
-	if (gpio_request_one(gpmc_cfg->gpio_irq, GPIOF_IN, "smsc911x irq")) {
-		pr_err("Failed to request IRQ GPIO%d\n", gpmc_cfg->gpio_irq);
-		goto free1;
-	}
-
-	gpmc_smsc911x_resources[1].start = gpio_to_irq(gpmc_cfg->gpio_irq);
-
-	if (gpio_is_valid(gpmc_cfg->gpio_reset)) {
-		ret = gpio_request_one(gpmc_cfg->gpio_reset,
-				       GPIOF_OUT_INIT_HIGH, "smsc911x reset");
-		if (ret) {
-			pr_err("Failed to request reset GPIO%d\n",
-			       gpmc_cfg->gpio_reset);
-			goto free2;
-		}
-
-		gpio_set_value(gpmc_cfg->gpio_reset, 0);
-		msleep(100);
-		gpio_set_value(gpmc_cfg->gpio_reset, 1);
-	}
-
-	gpmc_smsc911x_config.flags = gpmc_cfg->flags ? : SMSC911X_USE_16BIT;
-
-	pdev = platform_device_register_resndata(NULL, "smsc911x", gpmc_cfg->id,
-		 gpmc_smsc911x_resources, ARRAY_SIZE(gpmc_smsc911x_resources),
-		 &gpmc_smsc911x_config, sizeof(gpmc_smsc911x_config));
-	if (IS_ERR(pdev)) {
-		pr_err("Unable to register platform device\n");
-		gpio_free(gpmc_cfg->gpio_reset);
-		goto free2;
-	}
-
-	return;
-
-free2:
-	gpio_free(gpmc_cfg->gpio_irq);
-free1:
-	gpmc_cs_free(gpmc_cfg->cs);
-
-	pr_err("Could not initialize smsc911x device\n");
-}
diff --git a/arch/arm/mach-omap2/gpmc-smsc911x.h b/arch/arm/mach-omap2/gpmc-smsc911x.h
deleted file mode 100644
index 99a05b8412fa..000000000000
--- a/arch/arm/mach-omap2/gpmc-smsc911x.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * arch/arm/plat-omap/include/plat/gpmc-smsc911x.h
- *
- * Copyright (C) 2009 Li-Pro.Net
- * Stephan Linz <linz@li-pro.net>
- *
- * Modified from arch/arm/plat-omap/include/plat/gpmc-smc91x.h
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifndef __ASM_ARCH_OMAP_GPMC_SMSC911X_H__
-
-struct omap_smsc911x_platform_data {
-	int	id;
-	int	cs;
-	int	gpio_irq;
-	int	gpio_reset;
-	u32	flags;
-};
-
-#if IS_ENABLED(CONFIG_SMSC911X)
-
-extern void gpmc_smsc911x_init(struct omap_smsc911x_platform_data *d);
-
-#else
-
-static inline void gpmc_smsc911x_init(struct omap_smsc911x_platform_data *d)
-{
-}
-
-#endif
-#endif
diff --git a/arch/arm/mach-omap2/hsmmc.c b/arch/arm/mach-omap2/hsmmc.c
index 478097741bce..cb754c46747e 100644
--- a/arch/arm/mach-omap2/hsmmc.c
+++ b/arch/arm/mach-omap2/hsmmc.c
@@ -22,7 +22,6 @@
 #include "omap_device.h"
 #include "omap-pm.h"
 
-#include "mux.h"
 #include "hsmmc.h"
 #include "control.h"
 
@@ -147,91 +146,6 @@ static int nop_mmc_set_power(struct device *dev, int power_on, int vdd)
 	return 0;
 }
 
-static inline void omap_hsmmc_mux(struct omap_hsmmc_platform_data
-				  *mmc_controller, int controller_nr)
-{
-	if (gpio_is_valid(mmc_controller->gpio_cd) &&
-	    (mmc_controller->gpio_cd < OMAP_MAX_GPIO_LINES))
-		omap_mux_init_gpio(mmc_controller->gpio_cd,
-				   OMAP_PIN_INPUT_PULLUP);
-	if (gpio_is_valid(mmc_controller->gpio_cod) &&
-	    (mmc_controller->gpio_cod < OMAP_MAX_GPIO_LINES))
-		omap_mux_init_gpio(mmc_controller->gpio_cod,
-				   OMAP_PIN_INPUT_PULLUP);
-	if (gpio_is_valid(mmc_controller->gpio_wp) &&
-	    (mmc_controller->gpio_wp < OMAP_MAX_GPIO_LINES))
-		omap_mux_init_gpio(mmc_controller->gpio_wp,
-				   OMAP_PIN_INPUT_PULLUP);
-	if (cpu_is_omap34xx()) {
-		if (controller_nr == 0) {
-			omap_mux_init_signal("sdmmc1_clk",
-				OMAP_PIN_INPUT_PULLUP);
-			omap_mux_init_signal("sdmmc1_cmd",
-				OMAP_PIN_INPUT_PULLUP);
-			omap_mux_init_signal("sdmmc1_dat0",
-				OMAP_PIN_INPUT_PULLUP);
-			if (mmc_controller->caps &
-				(MMC_CAP_4_BIT_DATA | MMC_CAP_8_BIT_DATA)) {
-				omap_mux_init_signal("sdmmc1_dat1",
-					OMAP_PIN_INPUT_PULLUP);
-				omap_mux_init_signal("sdmmc1_dat2",
-					OMAP_PIN_INPUT_PULLUP);
-				omap_mux_init_signal("sdmmc1_dat3",
-					OMAP_PIN_INPUT_PULLUP);
-			}
-			if (mmc_controller->caps &
-						MMC_CAP_8_BIT_DATA) {
-				omap_mux_init_signal("sdmmc1_dat4",
-					OMAP_PIN_INPUT_PULLUP);
-				omap_mux_init_signal("sdmmc1_dat5",
-					OMAP_PIN_INPUT_PULLUP);
-				omap_mux_init_signal("sdmmc1_dat6",
-					OMAP_PIN_INPUT_PULLUP);
-				omap_mux_init_signal("sdmmc1_dat7",
-					OMAP_PIN_INPUT_PULLUP);
-			}
-		}
-		if (controller_nr == 1) {
-			/* MMC2 */
-			omap_mux_init_signal("sdmmc2_clk",
-				OMAP_PIN_INPUT_PULLUP);
-			omap_mux_init_signal("sdmmc2_cmd",
-				OMAP_PIN_INPUT_PULLUP);
-			omap_mux_init_signal("sdmmc2_dat0",
-				OMAP_PIN_INPUT_PULLUP);
-
-			/*
-			 * For 8 wire configurations, Lines DAT4, 5, 6 and 7
-			 * need to be muxed in the board-*.c files
-			 */
-			if (mmc_controller->caps &
-				(MMC_CAP_4_BIT_DATA | MMC_CAP_8_BIT_DATA)) {
-				omap_mux_init_signal("sdmmc2_dat1",
-					OMAP_PIN_INPUT_PULLUP);
-				omap_mux_init_signal("sdmmc2_dat2",
-					OMAP_PIN_INPUT_PULLUP);
-				omap_mux_init_signal("sdmmc2_dat3",
-					OMAP_PIN_INPUT_PULLUP);
-			}
-			if (mmc_controller->caps &
-							MMC_CAP_8_BIT_DATA) {
-				omap_mux_init_signal("sdmmc2_dat4.sdmmc2_dat4",
-					OMAP_PIN_INPUT_PULLUP);
-				omap_mux_init_signal("sdmmc2_dat5.sdmmc2_dat5",
-					OMAP_PIN_INPUT_PULLUP);
-				omap_mux_init_signal("sdmmc2_dat6.sdmmc2_dat6",
-					OMAP_PIN_INPUT_PULLUP);
-				omap_mux_init_signal("sdmmc2_dat7.sdmmc2_dat7",
-					OMAP_PIN_INPUT_PULLUP);
-			}
-		}
-
-		/*
-		 * For MMC3 the pins need to be muxed in the board-*.c files
-		 */
-	}
-}
-
 static int __init omap_hsmmc_pdata_init(struct omap2_hsmmc_info *c,
 					struct omap_hsmmc_platform_data *mmc)
 {
@@ -410,8 +324,6 @@ static void __init omap_hsmmc_init_one(struct omap2_hsmmc_info *hsmmcinfo,
 	if (res < 0)
 		goto free_mmc;
 
-	omap_hsmmc_mux(mmc_data, (ctrl_nr - 1));
-
 	name = "omap_hsmmc";
 	res = snprintf(oh_name, MAX_OMAP_MMC_HWMOD_NAME_LEN,
 		     "mmc%d", ctrl_nr);
diff --git a/arch/arm/mach-omap2/i2c.c b/arch/arm/mach-omap2/i2c.c
index b9d8e47ffe8e..91a21c3923b2 100644
--- a/arch/arm/mach-omap2/i2c.c
+++ b/arch/arm/mach-omap2/i2c.c
@@ -26,7 +26,6 @@
 
 #include "prm.h"
 #include "common.h"
-#include "mux.h"
 #include "i2c.h"
 
 /* In register I2C_CON, Bit 15 is the I2C enable bit */
@@ -36,20 +35,6 @@
 
 #define MAX_OMAP_I2C_HWMOD_NAME_LEN	16
 
-static void __init omap2_i2c_mux_pins(int bus_id)
-{
-	char mux_name[sizeof("i2c2_scl.i2c2_scl")];
-
-	/* First I2C bus is not muxable */
-	if (bus_id == 1)
-		return;
-
-	sprintf(mux_name, "i2c%i_scl.i2c%i_scl", bus_id, bus_id);
-	omap_mux_init_signal(mux_name, OMAP_PIN_INPUT);
-	sprintf(mux_name, "i2c%i_sda.i2c%i_sda", bus_id, bus_id);
-	omap_mux_init_signal(mux_name, OMAP_PIN_INPUT);
-}
-
 /**
  * omap_i2c_reset - reset the omap i2c module.
  * @oh: struct omap_hwmod *
@@ -107,85 +92,3 @@ int omap_i2c_reset(struct omap_hwmod *oh)
 
 	return 0;
 }
-
-static int __init omap_i2c_nr_ports(void)
-{
-	int ports = 0;
-
-	if (cpu_is_omap24xx())
-		ports = 2;
-	else if (cpu_is_omap34xx())
-		ports = 3;
-	else if (cpu_is_omap44xx())
-		ports = 4;
-	return ports;
-}
-
-/*
- * XXX This function is a temporary compatibility wrapper - only
- * needed until the I2C driver can be converted to call
- * omap_pm_set_max_dev_wakeup_lat() and handle a return code.
- */
-static void omap_pm_set_max_mpu_wakeup_lat_compat(struct device *dev, long t)
-{
-	omap_pm_set_max_mpu_wakeup_lat(dev, t);
-}
-
-static const char name[] = "omap_i2c";
-
-int __init omap_i2c_add_bus(struct omap_i2c_bus_platform_data *i2c_pdata,
-				int bus_id)
-{
-	int l;
-	struct omap_hwmod *oh;
-	struct platform_device *pdev;
-	char oh_name[MAX_OMAP_I2C_HWMOD_NAME_LEN];
-	struct omap_i2c_bus_platform_data *pdata;
-	struct omap_i2c_dev_attr *dev_attr;
-
-	if (bus_id > omap_i2c_nr_ports())
-		return -EINVAL;
-
-	omap2_i2c_mux_pins(bus_id);
-
-	l = snprintf(oh_name, MAX_OMAP_I2C_HWMOD_NAME_LEN, "i2c%d", bus_id);
-	WARN(l >= MAX_OMAP_I2C_HWMOD_NAME_LEN,
-		"String buffer overflow in I2C%d device setup\n", bus_id);
-	oh = omap_hwmod_lookup(oh_name);
-	if (!oh) {
-			pr_err("Could not look up %s\n", oh_name);
-			return -EEXIST;
-	}
-
-	pdata = i2c_pdata;
-	/*
-	 * pass the hwmod class's CPU-specific knowledge of I2C IP revision in
-	 * use, and functionality implementation flags, up to the OMAP I2C
-	 * driver via platform data
-	 */
-	pdata->rev = oh->class->rev;
-
-	dev_attr = (struct omap_i2c_dev_attr *)oh->dev_attr;
-	pdata->flags = dev_attr->flags;
-
-	/*
-	 * When waiting for completion of a i2c transfer, we need to
-	 * set a wake up latency constraint for the MPU. This is to
-	 * ensure quick enough wakeup from idle, when transfer
-	 * completes.
-	 * Only omap3 has support for constraints
-	 */
-	if (cpu_is_omap34xx())
-		pdata->set_mpu_wkup_lat = omap_pm_set_max_mpu_wakeup_lat_compat;
-	pdev = omap_device_build(name, bus_id, oh, pdata,
-				 sizeof(struct omap_i2c_bus_platform_data));
-	WARN(IS_ERR(pdev), "Could not build omap_device for %s\n", name);
-
-	return PTR_ERR_OR_ZERO(pdev);
-}
-
-static  int __init omap_i2c_cmdline(void)
-{
-	return omap_register_i2c_bus_cmdline();
-}
-omap_subsys_initcall(omap_i2c_cmdline);
diff --git a/arch/arm/mach-omap2/io.c b/arch/arm/mach-omap2/io.c
index 0e9acdd95d70..5aafb8449c40 100644
--- a/arch/arm/mach-omap2/io.c
+++ b/arch/arm/mach-omap2/io.c
@@ -427,7 +427,6 @@ static void __init omap_hwmod_init_postsetup(void)
 
 static void __init __maybe_unused omap_common_late_init(void)
 {
-	omap_mux_late_init();
 	omap2_common_pm_late_init();
 	omap_soc_device_init();
 }
@@ -717,10 +716,11 @@ void __init omap5_init_early(void)
 			      OMAP2_L4_IO_ADDRESS(OMAP54XX_SCM_BASE));
 	omap2_set_globals_prcm_mpu(OMAP2_L4_IO_ADDRESS(OMAP54XX_PRCM_MPU_BASE));
 	omap2_control_base_init();
-	omap4_pm_init_early();
 	omap2_prcm_base_init();
 	omap5xxx_check_revision();
 	omap4_sar_ram_init();
+	omap4_mpuss_early_init();
+	omap4_pm_init_early();
 	omap54xx_voltagedomains_init();
 	omap54xx_powerdomains_init();
 	omap54xx_clockdomains_init();
diff --git a/arch/arm/mach-omap2/msdi.c b/arch/arm/mach-omap2/msdi.c
index 8bdf182422bd..5a3bc3de58d0 100644
--- a/arch/arm/mach-omap2/msdi.c
+++ b/arch/arm/mach-omap2/msdi.c
@@ -30,7 +30,6 @@
 #include "control.h"
 #include "omap_hwmod.h"
 #include "omap_device.h"
-#include "mux.h"
 #include "mmc.h"
 
 /*
diff --git a/arch/arm/mach-omap2/mux.c b/arch/arm/mach-omap2/mux.c
deleted file mode 100644
index 176eef6ef338..000000000000
--- a/arch/arm/mach-omap2/mux.c
+++ /dev/null
@@ -1,1153 +0,0 @@
-/*
- * linux/arch/arm/mach-omap2/mux.c
- *
- * OMAP2, OMAP3 and OMAP4 pin multiplexing configurations
- *
- * Copyright (C) 2004 - 2010 Texas Instruments Inc.
- * Copyright (C) 2003 - 2008 Nokia Corporation
- *
- * Written by Tony Lindgren
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- *
- */
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/io.h>
-#include <linux/list.h>
-#include <linux/slab.h>
-#include <linux/ctype.h>
-#include <linux/debugfs.h>
-#include <linux/seq_file.h>
-#include <linux/uaccess.h>
-#include <linux/irq.h>
-#include <linux/interrupt.h>
-
-
-#include "omap_hwmod.h"
-
-#include "soc.h"
-#include "control.h"
-#include "mux.h"
-#include "prm.h"
-#include "common.h"
-
-#define OMAP_MUX_BASE_OFFSET		0x30	/* Offset from CTRL_BASE */
-#define OMAP_MUX_BASE_SZ		0x5ca
-
-struct omap_mux_entry {
-	struct omap_mux		mux;
-	struct list_head	node;
-};
-
-static LIST_HEAD(mux_partitions);
-static DEFINE_MUTEX(muxmode_mutex);
-
-struct omap_mux_partition *omap_mux_get(const char *name)
-{
-	struct omap_mux_partition *partition;
-
-	list_for_each_entry(partition, &mux_partitions, node) {
-		if (!strcmp(name, partition->name))
-			return partition;
-	}
-
-	return NULL;
-}
-
-u16 omap_mux_read(struct omap_mux_partition *partition, u16 reg)
-{
-	if (partition->flags & OMAP_MUX_REG_8BIT)
-		return readb_relaxed(partition->base + reg);
-	else
-		return readw_relaxed(partition->base + reg);
-}
-
-void omap_mux_write(struct omap_mux_partition *partition, u16 val,
-			   u16 reg)
-{
-	if (partition->flags & OMAP_MUX_REG_8BIT)
-		writeb_relaxed(val, partition->base + reg);
-	else
-		writew_relaxed(val, partition->base + reg);
-}
-
-void omap_mux_write_array(struct omap_mux_partition *partition,
-				 struct omap_board_mux *board_mux)
-{
-	if (!board_mux)
-		return;
-
-	while (board_mux->reg_offset != OMAP_MUX_TERMINATOR) {
-		omap_mux_write(partition, board_mux->value,
-			       board_mux->reg_offset);
-		board_mux++;
-	}
-}
-
-#ifdef CONFIG_OMAP_MUX
-
-static char *omap_mux_options;
-
-static int __init _omap_mux_init_gpio(struct omap_mux_partition *partition,
-				      int gpio, int val)
-{
-	struct omap_mux_entry *e;
-	struct omap_mux *gpio_mux = NULL;
-	u16 old_mode;
-	u16 mux_mode;
-	int found = 0;
-	struct list_head *muxmodes = &partition->muxmodes;
-
-	if (!gpio)
-		return -EINVAL;
-
-	list_for_each_entry(e, muxmodes, node) {
-		struct omap_mux *m = &e->mux;
-		if (gpio == m->gpio) {
-			gpio_mux = m;
-			found++;
-		}
-	}
-
-	if (found == 0) {
-		pr_err("%s: Could not set gpio%i\n", __func__, gpio);
-		return -ENODEV;
-	}
-
-	if (found > 1) {
-		pr_info("%s: Multiple gpio paths (%d) for gpio%i\n", __func__,
-			found, gpio);
-		return -EINVAL;
-	}
-
-	old_mode = omap_mux_read(partition, gpio_mux->reg_offset);
-	mux_mode = val & ~(OMAP_MUX_NR_MODES - 1);
-	mux_mode |= partition->gpio;
-	pr_debug("%s: Setting signal %s.gpio%i 0x%04x -> 0x%04x\n", __func__,
-		 gpio_mux->muxnames[0], gpio, old_mode, mux_mode);
-	omap_mux_write(partition, mux_mode, gpio_mux->reg_offset);
-
-	return 0;
-}
-
-int __init omap_mux_init_gpio(int gpio, int val)
-{
-	struct omap_mux_partition *partition;
-	int ret;
-
-	list_for_each_entry(partition, &mux_partitions, node) {
-		ret = _omap_mux_init_gpio(partition, gpio, val);
-		if (!ret)
-			return ret;
-	}
-
-	return -ENODEV;
-}
-
-static int __init _omap_mux_get_by_name(struct omap_mux_partition *partition,
-					const char *muxname,
-					struct omap_mux **found_mux)
-{
-	struct omap_mux *mux = NULL;
-	struct omap_mux_entry *e;
-	const char *mode_name;
-	int found = 0, found_mode = 0, mode0_len = 0;
-	struct list_head *muxmodes = &partition->muxmodes;
-
-	mode_name = strchr(muxname, '.');
-	if (mode_name) {
-		mode0_len = strlen(muxname) - strlen(mode_name);
-		mode_name++;
-	} else {
-		mode_name = muxname;
-	}
-
-	list_for_each_entry(e, muxmodes, node) {
-		char *m0_entry;
-		int i;
-
-		mux = &e->mux;
-		m0_entry = mux->muxnames[0];
-
-		/* First check for full name in mode0.muxmode format */
-		if (mode0_len)
-			if (strncmp(muxname, m0_entry, mode0_len) ||
-			    (strlen(m0_entry) != mode0_len))
-				continue;
-
-		/* Then check for muxmode only */
-		for (i = 0; i < OMAP_MUX_NR_MODES; i++) {
-			char *mode_cur = mux->muxnames[i];
-
-			if (!mode_cur)
-				continue;
-
-			if (!strcmp(mode_name, mode_cur)) {
-				*found_mux = mux;
-				found++;
-				found_mode = i;
-			}
-		}
-	}
-
-	if (found == 1) {
-		return found_mode;
-	}
-
-	if (found > 1) {
-		pr_err("%s: Multiple signal paths (%i) for %s\n", __func__,
-		       found, muxname);
-		return -EINVAL;
-	}
-
-	return -ENODEV;
-}
-
-int __init omap_mux_get_by_name(const char *muxname,
-			struct omap_mux_partition **found_partition,
-			struct omap_mux **found_mux)
-{
-	struct omap_mux_partition *partition;
-
-	list_for_each_entry(partition, &mux_partitions, node) {
-		struct omap_mux *mux = NULL;
-		int mux_mode = _omap_mux_get_by_name(partition, muxname, &mux);
-		if (mux_mode < 0)
-			continue;
-
-		*found_partition = partition;
-		*found_mux = mux;
-
-		return mux_mode;
-	}
-
-	pr_err("%s: Could not find signal %s\n", __func__, muxname);
-
-	return -ENODEV;
-}
-
-int __init omap_mux_init_signal(const char *muxname, int val)
-{
-	struct omap_mux_partition *partition = NULL;
-	struct omap_mux *mux = NULL;
-	u16 old_mode;
-	int mux_mode;
-
-	mux_mode = omap_mux_get_by_name(muxname, &partition, &mux);
-	if (mux_mode < 0 || !mux)
-		return mux_mode;
-
-	old_mode = omap_mux_read(partition, mux->reg_offset);
-	mux_mode |= val;
-	pr_debug("%s: Setting signal %s 0x%04x -> 0x%04x\n",
-			 __func__, muxname, old_mode, mux_mode);
-	omap_mux_write(partition, mux_mode, mux->reg_offset);
-
-	return 0;
-}
-
-struct omap_hwmod_mux_info * __init
-omap_hwmod_mux_init(struct omap_device_pad *bpads, int nr_pads)
-{
-	struct omap_hwmod_mux_info *hmux;
-	int i, nr_pads_dynamic = 0;
-
-	if (!bpads || nr_pads < 1)
-		return NULL;
-
-	hmux = kzalloc(sizeof(struct omap_hwmod_mux_info), GFP_KERNEL);
-	if (!hmux)
-		goto err1;
-
-	hmux->nr_pads = nr_pads;
-
-	hmux->pads = kzalloc(sizeof(struct omap_device_pad) *
-				nr_pads, GFP_KERNEL);
-	if (!hmux->pads)
-		goto err2;
-
-	for (i = 0; i < hmux->nr_pads; i++) {
-		struct omap_mux_partition *partition;
-		struct omap_device_pad *bpad = &bpads[i], *pad = &hmux->pads[i];
-		struct omap_mux *mux;
-		int mux_mode;
-
-		mux_mode = omap_mux_get_by_name(bpad->name, &partition, &mux);
-		if (mux_mode < 0)
-			goto err3;
-		if (!pad->partition)
-			pad->partition = partition;
-		if (!pad->mux)
-			pad->mux = mux;
-
-		pad->name = kzalloc(strlen(bpad->name) + 1, GFP_KERNEL);
-		if (!pad->name) {
-			int j;
-
-			for (j = i - 1; j >= 0; j--)
-				kfree(hmux->pads[j].name);
-			goto err3;
-		}
-		strcpy(pad->name, bpad->name);
-
-		pad->flags = bpad->flags;
-		pad->enable = bpad->enable;
-		pad->idle = bpad->idle;
-		pad->off = bpad->off;
-
-		if (pad->flags &
-		    (OMAP_DEVICE_PAD_REMUX | OMAP_DEVICE_PAD_WAKEUP))
-			nr_pads_dynamic++;
-
-		pr_debug("%s: Initialized %s\n", __func__, pad->name);
-	}
-
-	if (!nr_pads_dynamic)
-		return hmux;
-
-	/*
-	 * Add pads that need dynamic muxing into a separate list
-	 */
-
-	hmux->nr_pads_dynamic = nr_pads_dynamic;
-	hmux->pads_dynamic = kzalloc(sizeof(struct omap_device_pad *) *
-					nr_pads_dynamic, GFP_KERNEL);
-	if (!hmux->pads_dynamic) {
-		pr_err("%s: Could not allocate dynamic pads\n", __func__);
-		return hmux;
-	}
-
-	nr_pads_dynamic = 0;
-	for (i = 0; i < hmux->nr_pads; i++) {
-		struct omap_device_pad *pad = &hmux->pads[i];
-
-		if (pad->flags &
-		    (OMAP_DEVICE_PAD_REMUX | OMAP_DEVICE_PAD_WAKEUP)) {
-			pr_debug("%s: pad %s tagged dynamic\n",
-					__func__, pad->name);
-			hmux->pads_dynamic[nr_pads_dynamic] = pad;
-			nr_pads_dynamic++;
-		}
-	}
-
-	return hmux;
-
-err3:
-	kfree(hmux->pads);
-err2:
-	kfree(hmux);
-err1:
-	pr_err("%s: Could not allocate device mux entry\n", __func__);
-
-	return NULL;
-}
-
-/**
- * omap_hwmod_mux_scan_wakeups - omap hwmod scan wakeup pads
- * @hmux: Pads for a hwmod
- * @mpu_irqs: MPU irq array for a hwmod
- *
- * Scans the wakeup status of pads for a single hwmod.  If an irq
- * array is defined for this mux, the parser will call the registered
- * ISRs for corresponding pads, otherwise the parser will stop at the
- * first wakeup active pad and return.  Returns true if there is a
- * pending and non-served wakeup event for the mux, otherwise false.
- */
-static bool omap_hwmod_mux_scan_wakeups(struct omap_hwmod_mux_info *hmux,
-		struct omap_hwmod_irq_info *mpu_irqs)
-{
-	int i, irq;
-	unsigned int val;
-	u32 handled_irqs = 0;
-
-	for (i = 0; i < hmux->nr_pads_dynamic; i++) {
-		struct omap_device_pad *pad = hmux->pads_dynamic[i];
-
-		if (!(pad->flags & OMAP_DEVICE_PAD_WAKEUP) ||
-		    !(pad->idle & OMAP_WAKEUP_EN))
-			continue;
-
-		val = omap_mux_read(pad->partition, pad->mux->reg_offset);
-		if (!(val & OMAP_WAKEUP_EVENT))
-			continue;
-
-		if (!hmux->irqs)
-			return true;
-
-		irq = hmux->irqs[i];
-		/* make sure we only handle each irq once */
-		if (handled_irqs & 1 << irq)
-			continue;
-
-		handled_irqs |= 1 << irq;
-
-		generic_handle_irq(mpu_irqs[irq].irq);
-	}
-
-	return false;
-}
-
-/**
- * _omap_hwmod_mux_handle_irq - Process wakeup events for a single hwmod
- *
- * Checks a single hwmod for every wakeup capable pad to see if there is an
- * active wakeup event. If this is the case, call the corresponding ISR.
- */
-static int _omap_hwmod_mux_handle_irq(struct omap_hwmod *oh, void *data)
-{
-	if (!oh->mux || !oh->mux->enabled)
-		return 0;
-	if (omap_hwmod_mux_scan_wakeups(oh->mux, oh->mpu_irqs))
-		generic_handle_irq(oh->mpu_irqs[0].irq);
-	return 0;
-}
-
-/**
- * omap_hwmod_mux_handle_irq - Process pad wakeup irqs.
- *
- * Calls a function for each registered omap_hwmod to check
- * pad wakeup statuses.
- */
-static irqreturn_t omap_hwmod_mux_handle_irq(int irq, void *unused)
-{
-	omap_hwmod_for_each(_omap_hwmod_mux_handle_irq, NULL);
-	return IRQ_HANDLED;
-}
-
-/* Assumes the calling function takes care of locking */
-void omap_hwmod_mux(struct omap_hwmod_mux_info *hmux, u8 state)
-{
-	int i;
-
-	/* Runtime idling of dynamic pads */
-	if (state == _HWMOD_STATE_IDLE && hmux->enabled) {
-		for (i = 0; i < hmux->nr_pads_dynamic; i++) {
-			struct omap_device_pad *pad = hmux->pads_dynamic[i];
-			int val = -EINVAL;
-
-			val = pad->idle;
-			omap_mux_write(pad->partition, val,
-					pad->mux->reg_offset);
-		}
-
-		return;
-	}
-
-	/* Runtime enabling of dynamic pads */
-	if ((state == _HWMOD_STATE_ENABLED) && hmux->pads_dynamic
-					&& hmux->enabled) {
-		for (i = 0; i < hmux->nr_pads_dynamic; i++) {
-			struct omap_device_pad *pad = hmux->pads_dynamic[i];
-			int val = -EINVAL;
-
-			val = pad->enable;
-			omap_mux_write(pad->partition, val,
-					pad->mux->reg_offset);
-		}
-
-		return;
-	}
-
-	/* Enabling or disabling of all pads */
-	for (i = 0; i < hmux->nr_pads; i++) {
-		struct omap_device_pad *pad = &hmux->pads[i];
-		int flags, val = -EINVAL;
-
-		flags = pad->flags;
-
-		switch (state) {
-		case _HWMOD_STATE_ENABLED:
-			val = pad->enable;
-			pr_debug("%s: Enabling %s %x\n", __func__,
-					pad->name, val);
-			break;
-		case _HWMOD_STATE_DISABLED:
-			/* Use safe mode unless OMAP_DEVICE_PAD_REMUX */
-			if (flags & OMAP_DEVICE_PAD_REMUX)
-				val = pad->off;
-			else
-				val = OMAP_MUX_MODE7;
-			pr_debug("%s: Disabling %s %x\n", __func__,
-					pad->name, val);
-			break;
-		default:
-			/* Nothing to be done */
-			break;
-		}
-
-		if (val >= 0) {
-			omap_mux_write(pad->partition, val,
-					pad->mux->reg_offset);
-			pad->flags = flags;
-		}
-	}
-
-	if (state == _HWMOD_STATE_ENABLED)
-		hmux->enabled = true;
-	else
-		hmux->enabled = false;
-}
-
-#ifdef CONFIG_DEBUG_FS
-
-#define OMAP_MUX_MAX_NR_FLAGS	10
-#define OMAP_MUX_TEST_FLAG(val, mask)				\
-	if (((val) & (mask)) == (mask)) {			\
-		i++;						\
-		flags[i] =  #mask;				\
-	}
-
-/* REVISIT: Add checking for non-optimal mux settings */
-static inline void omap_mux_decode(struct seq_file *s, u16 val)
-{
-	char *flags[OMAP_MUX_MAX_NR_FLAGS];
-	char mode[sizeof("OMAP_MUX_MODE") + 1];
-	int i = -1;
-
-	sprintf(mode, "OMAP_MUX_MODE%d", val & 0x7);
-	i++;
-	flags[i] = mode;
-
-	OMAP_MUX_TEST_FLAG(val, OMAP_PIN_OFF_WAKEUPENABLE);
-	if (val & OMAP_OFF_EN) {
-		if (!(val & OMAP_OFFOUT_EN)) {
-			if (!(val & OMAP_OFF_PULL_UP)) {
-				OMAP_MUX_TEST_FLAG(val,
-					OMAP_PIN_OFF_INPUT_PULLDOWN);
-			} else {
-				OMAP_MUX_TEST_FLAG(val,
-					OMAP_PIN_OFF_INPUT_PULLUP);
-			}
-		} else {
-			if (!(val & OMAP_OFFOUT_VAL)) {
-				OMAP_MUX_TEST_FLAG(val,
-					OMAP_PIN_OFF_OUTPUT_LOW);
-			} else {
-				OMAP_MUX_TEST_FLAG(val,
-					OMAP_PIN_OFF_OUTPUT_HIGH);
-			}
-		}
-	}
-
-	if (val & OMAP_INPUT_EN) {
-		if (val & OMAP_PULL_ENA) {
-			if (!(val & OMAP_PULL_UP)) {
-				OMAP_MUX_TEST_FLAG(val,
-					OMAP_PIN_INPUT_PULLDOWN);
-			} else {
-				OMAP_MUX_TEST_FLAG(val, OMAP_PIN_INPUT_PULLUP);
-			}
-		} else {
-			OMAP_MUX_TEST_FLAG(val, OMAP_PIN_INPUT);
-		}
-	} else {
-		i++;
-		flags[i] = "OMAP_PIN_OUTPUT";
-	}
-
-	do {
-		seq_printf(s, "%s", flags[i]);
-		if (i > 0)
-			seq_printf(s, " | ");
-	} while (i-- > 0);
-}
-
-#define OMAP_MUX_DEFNAME_LEN	32
-
-static int omap_mux_dbg_board_show(struct seq_file *s, void *unused)
-{
-	struct omap_mux_partition *partition = s->private;
-	struct omap_mux_entry *e;
-	u8 omap_gen = omap_rev() >> 28;
-
-	list_for_each_entry(e, &partition->muxmodes, node) {
-		struct omap_mux *m = &e->mux;
-		char m0_def[OMAP_MUX_DEFNAME_LEN];
-		char *m0_name = m->muxnames[0];
-		u16 val;
-		int i, mode;
-
-		if (!m0_name)
-			continue;
-
-		/* REVISIT: Needs to be updated if mode0 names get longer */
-		for (i = 0; i < OMAP_MUX_DEFNAME_LEN; i++) {
-			if (m0_name[i] == '\0') {
-				m0_def[i] = m0_name[i];
-				break;
-			}
-			m0_def[i] = toupper(m0_name[i]);
-		}
-		val = omap_mux_read(partition, m->reg_offset);
-		mode = val & OMAP_MUX_MODE7;
-		if (mode != 0)
-			seq_printf(s, "/* %s */\n", m->muxnames[mode]);
-
-		/*
-		 * XXX: Might be revisited to support differences across
-		 * same OMAP generation.
-		 */
-		seq_printf(s, "OMAP%d_MUX(%s, ", omap_gen, m0_def);
-		omap_mux_decode(s, val);
-		seq_printf(s, "),\n");
-	}
-
-	return 0;
-}
-
-static int omap_mux_dbg_board_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, omap_mux_dbg_board_show, inode->i_private);
-}
-
-static const struct file_operations omap_mux_dbg_board_fops = {
-	.open		= omap_mux_dbg_board_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-static struct omap_mux_partition *omap_mux_get_partition(struct omap_mux *mux)
-{
-	struct omap_mux_partition *partition;
-
-	list_for_each_entry(partition, &mux_partitions, node) {
-		struct list_head *muxmodes = &partition->muxmodes;
-		struct omap_mux_entry *e;
-
-		list_for_each_entry(e, muxmodes, node) {
-			struct omap_mux *m = &e->mux;
-
-			if (m == mux)
-				return partition;
-		}
-	}
-
-	return NULL;
-}
-
-static int omap_mux_dbg_signal_show(struct seq_file *s, void *unused)
-{
-	struct omap_mux *m = s->private;
-	struct omap_mux_partition *partition;
-	const char *none = "NA";
-	u16 val;
-	int mode;
-
-	partition = omap_mux_get_partition(m);
-	if (!partition)
-		return 0;
-
-	val = omap_mux_read(partition, m->reg_offset);
-	mode = val & OMAP_MUX_MODE7;
-
-	seq_printf(s, "name: %s.%s (0x%08x/0x%03x = 0x%04x), b %s, t %s\n",
-			m->muxnames[0], m->muxnames[mode],
-			partition->phys + m->reg_offset, m->reg_offset, val,
-			m->balls[0] ? m->balls[0] : none,
-			m->balls[1] ? m->balls[1] : none);
-	seq_printf(s, "mode: ");
-	omap_mux_decode(s, val);
-	seq_printf(s, "\n");
-	seq_printf(s, "signals: %s | %s | %s | %s | %s | %s | %s | %s\n",
-			m->muxnames[0] ? m->muxnames[0] : none,
-			m->muxnames[1] ? m->muxnames[1] : none,
-			m->muxnames[2] ? m->muxnames[2] : none,
-			m->muxnames[3] ? m->muxnames[3] : none,
-			m->muxnames[4] ? m->muxnames[4] : none,
-			m->muxnames[5] ? m->muxnames[5] : none,
-			m->muxnames[6] ? m->muxnames[6] : none,
-			m->muxnames[7] ? m->muxnames[7] : none);
-
-	return 0;
-}
-
-#define OMAP_MUX_MAX_ARG_CHAR  7
-
-static ssize_t omap_mux_dbg_signal_write(struct file *file,
-					 const char __user *user_buf,
-					 size_t count, loff_t *ppos)
-{
-	struct seq_file *seqf;
-	struct omap_mux *m;
-	u16 val;
-	int ret;
-	struct omap_mux_partition *partition;
-
-	if (count > OMAP_MUX_MAX_ARG_CHAR)
-		return -EINVAL;
-
-	ret = kstrtou16_from_user(user_buf, count, 0x10, &val);
-	if (ret < 0)
-		return ret;
-
-	seqf = file->private_data;
-	m = seqf->private;
-
-	partition = omap_mux_get_partition(m);
-	if (!partition)
-		return -ENODEV;
-
-	omap_mux_write(partition, val, m->reg_offset);
-	*ppos += count;
-
-	return count;
-}
-
-static int omap_mux_dbg_signal_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, omap_mux_dbg_signal_show, inode->i_private);
-}
-
-static const struct file_operations omap_mux_dbg_signal_fops = {
-	.open		= omap_mux_dbg_signal_open,
-	.read		= seq_read,
-	.write		= omap_mux_dbg_signal_write,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-static struct dentry *mux_dbg_dir;
-
-static void __init omap_mux_dbg_create_entry(
-				struct omap_mux_partition *partition,
-				struct dentry *mux_dbg_dir)
-{
-	struct omap_mux_entry *e;
-
-	list_for_each_entry(e, &partition->muxmodes, node) {
-		struct omap_mux *m = &e->mux;
-
-		(void)debugfs_create_file(m->muxnames[0], S_IWUSR | S_IRUGO,
-					  mux_dbg_dir, m,
-					  &omap_mux_dbg_signal_fops);
-	}
-}
-
-static void __init omap_mux_dbg_init(void)
-{
-	struct omap_mux_partition *partition;
-	static struct dentry *mux_dbg_board_dir;
-
-	mux_dbg_dir = debugfs_create_dir("omap_mux", NULL);
-	if (!mux_dbg_dir)
-		return;
-
-	mux_dbg_board_dir = debugfs_create_dir("board", mux_dbg_dir);
-	if (!mux_dbg_board_dir)
-		return;
-
-	list_for_each_entry(partition, &mux_partitions, node) {
-		omap_mux_dbg_create_entry(partition, mux_dbg_dir);
-		(void)debugfs_create_file(partition->name, S_IRUGO,
-					  mux_dbg_board_dir, partition,
-					  &omap_mux_dbg_board_fops);
-	}
-}
-
-#else
-static inline void omap_mux_dbg_init(void)
-{
-}
-#endif	/* CONFIG_DEBUG_FS */
-
-static void __init omap_mux_free_names(struct omap_mux *m)
-{
-	int i;
-
-	for (i = 0; i < OMAP_MUX_NR_MODES; i++)
-		kfree(m->muxnames[i]);
-
-#ifdef CONFIG_DEBUG_FS
-	for (i = 0; i < OMAP_MUX_NR_SIDES; i++)
-		kfree(m->balls[i]);
-#endif
-
-}
-
-/* Free all data except for GPIO pins unless CONFIG_DEBUG_FS is set */
-int __init omap_mux_late_init(void)
-{
-	struct omap_mux_partition *partition;
-	int ret;
-
-	list_for_each_entry(partition, &mux_partitions, node) {
-		struct omap_mux_entry *e, *tmp;
-		list_for_each_entry_safe(e, tmp, &partition->muxmodes, node) {
-			struct omap_mux *m = &e->mux;
-			u16 mode = omap_mux_read(partition, m->reg_offset);
-
-			if (OMAP_MODE_GPIO(partition, mode))
-				continue;
-
-#ifndef CONFIG_DEBUG_FS
-			mutex_lock(&muxmode_mutex);
-			list_del(&e->node);
-			mutex_unlock(&muxmode_mutex);
-			omap_mux_free_names(m);
-			kfree(m);
-#endif
-		}
-	}
-
-	omap_mux_dbg_init();
-
-	/* see pinctrl-single-omap for the wake-up interrupt handling */
-	if (of_have_populated_dt())
-		return 0;
-
-	ret = request_irq(omap_prcm_event_to_irq("io"),
-		omap_hwmod_mux_handle_irq, IRQF_SHARED | IRQF_NO_SUSPEND,
-			"hwmod_io", omap_mux_late_init);
-
-	if (ret)
-		pr_warn("mux: Failed to setup hwmod io irq %d\n", ret);
-
-	return 0;
-}
-
-static void __init omap_mux_package_fixup(struct omap_mux *p,
-					struct omap_mux *superset)
-{
-	while (p->reg_offset !=  OMAP_MUX_TERMINATOR) {
-		struct omap_mux *s = superset;
-		int found = 0;
-
-		while (s->reg_offset != OMAP_MUX_TERMINATOR) {
-			if (s->reg_offset == p->reg_offset) {
-				*s = *p;
-				found++;
-				break;
-			}
-			s++;
-		}
-		if (!found)
-			pr_err("%s: Unknown entry offset 0x%x\n", __func__,
-			       p->reg_offset);
-		p++;
-	}
-}
-
-#ifdef CONFIG_DEBUG_FS
-
-static void __init omap_mux_package_init_balls(struct omap_ball *b,
-				struct omap_mux *superset)
-{
-	while (b->reg_offset != OMAP_MUX_TERMINATOR) {
-		struct omap_mux *s = superset;
-		int found = 0;
-
-		while (s->reg_offset != OMAP_MUX_TERMINATOR) {
-			if (s->reg_offset == b->reg_offset) {
-				s->balls[0] = b->balls[0];
-				s->balls[1] = b->balls[1];
-				found++;
-				break;
-			}
-			s++;
-		}
-		if (!found)
-			pr_err("%s: Unknown ball offset 0x%x\n", __func__,
-			       b->reg_offset);
-		b++;
-	}
-}
-
-#else	/* CONFIG_DEBUG_FS */
-
-static inline void omap_mux_package_init_balls(struct omap_ball *b,
-					struct omap_mux *superset)
-{
-}
-
-#endif	/* CONFIG_DEBUG_FS */
-
-static int __init omap_mux_setup(char *options)
-{
-	if (!options)
-		return 0;
-
-	omap_mux_options = options;
-
-	return 1;
-}
-__setup("omap_mux=", omap_mux_setup);
-
-/*
- * Note that the omap_mux=some.signal1=0x1234,some.signal2=0x1234
- * cmdline options only override the bootloader values.
- * During development, please enable CONFIG_DEBUG_FS, and use the
- * signal specific entries under debugfs.
- */
-static void __init omap_mux_set_cmdline_signals(void)
-{
-	char *options, *next_opt, *token;
-
-	if (!omap_mux_options)
-		return;
-
-	options = kstrdup(omap_mux_options, GFP_KERNEL);
-	if (!options)
-		return;
-
-	next_opt = options;
-
-	while ((token = strsep(&next_opt, ",")) != NULL) {
-		char *keyval, *name;
-		u16 val;
-
-		keyval = token;
-		name = strsep(&keyval, "=");
-		if (name) {
-			int res;
-
-			res = kstrtou16(keyval, 0x10, &val);
-			if (res < 0)
-				continue;
-
-			omap_mux_init_signal(name, (u16)val);
-		}
-	}
-
-	kfree(options);
-}
-
-static int __init omap_mux_copy_names(struct omap_mux *src,
-				      struct omap_mux *dst)
-{
-	int i;
-
-	for (i = 0; i < OMAP_MUX_NR_MODES; i++) {
-		if (src->muxnames[i]) {
-			dst->muxnames[i] = kstrdup(src->muxnames[i],
-						   GFP_KERNEL);
-			if (!dst->muxnames[i])
-				goto free;
-		}
-	}
-
-#ifdef CONFIG_DEBUG_FS
-	for (i = 0; i < OMAP_MUX_NR_SIDES; i++) {
-		if (src->balls[i]) {
-			dst->balls[i] = kstrdup(src->balls[i], GFP_KERNEL);
-			if (!dst->balls[i])
-				goto free;
-		}
-	}
-#endif
-
-	return 0;
-
-free:
-	omap_mux_free_names(dst);
-	return -ENOMEM;
-
-}
-
-#endif	/* CONFIG_OMAP_MUX */
-
-static struct omap_mux *omap_mux_get_by_gpio(
-				struct omap_mux_partition *partition,
-				int gpio)
-{
-	struct omap_mux_entry *e;
-	struct omap_mux *ret = NULL;
-
-	list_for_each_entry(e, &partition->muxmodes, node) {
-		struct omap_mux *m = &e->mux;
-		if (m->gpio == gpio) {
-			ret = m;
-			break;
-		}
-	}
-
-	return ret;
-}
-
-/* Needed for dynamic muxing of GPIO pins for off-idle */
-u16 omap_mux_get_gpio(int gpio)
-{
-	struct omap_mux_partition *partition;
-	struct omap_mux *m = NULL;
-
-	list_for_each_entry(partition, &mux_partitions, node) {
-		m = omap_mux_get_by_gpio(partition, gpio);
-		if (m)
-			return omap_mux_read(partition, m->reg_offset);
-	}
-
-	if (!m || m->reg_offset == OMAP_MUX_TERMINATOR)
-		pr_err("%s: Could not get gpio%i\n", __func__, gpio);
-
-	return OMAP_MUX_TERMINATOR;
-}
-
-/* Needed for dynamic muxing of GPIO pins for off-idle */
-void omap_mux_set_gpio(u16 val, int gpio)
-{
-	struct omap_mux_partition *partition;
-	struct omap_mux *m = NULL;
-
-	list_for_each_entry(partition, &mux_partitions, node) {
-		m = omap_mux_get_by_gpio(partition, gpio);
-		if (m) {
-			omap_mux_write(partition, val, m->reg_offset);
-			return;
-		}
-	}
-
-	if (!m || m->reg_offset == OMAP_MUX_TERMINATOR)
-		pr_err("%s: Could not set gpio%i\n", __func__, gpio);
-}
-
-static struct omap_mux * __init omap_mux_list_add(
-					struct omap_mux_partition *partition,
-					struct omap_mux *src)
-{
-	struct omap_mux_entry *entry;
-	struct omap_mux *m;
-
-	entry = kzalloc(sizeof(struct omap_mux_entry), GFP_KERNEL);
-	if (!entry)
-		return NULL;
-
-	m = &entry->mux;
-	entry->mux = *src;
-
-#ifdef CONFIG_OMAP_MUX
-	if (omap_mux_copy_names(src, m)) {
-		kfree(entry);
-		return NULL;
-	}
-#endif
-
-	mutex_lock(&muxmode_mutex);
-	list_add_tail(&entry->node, &partition->muxmodes);
-	mutex_unlock(&muxmode_mutex);
-
-	return m;
-}
-
-/*
- * Note if CONFIG_OMAP_MUX is not selected, we will only initialize
- * the GPIO to mux offset mapping that is needed for dynamic muxing
- * of GPIO pins for off-idle.
- */
-static void __init omap_mux_init_list(struct omap_mux_partition *partition,
-				      struct omap_mux *superset)
-{
-	while (superset->reg_offset !=  OMAP_MUX_TERMINATOR) {
-		struct omap_mux *entry;
-
-#ifdef CONFIG_OMAP_MUX
-		if (!superset->muxnames[0]) {
-			superset++;
-			continue;
-		}
-#else
-		/* Skip pins that are not muxed as GPIO by bootloader */
-		if (!OMAP_MODE_GPIO(partition, omap_mux_read(partition,
-				    superset->reg_offset))) {
-			superset++;
-			continue;
-		}
-#endif
-
-		entry = omap_mux_list_add(partition, superset);
-		if (!entry) {
-			pr_err("%s: Could not add entry\n", __func__);
-			return;
-		}
-		superset++;
-	}
-}
-
-#ifdef CONFIG_OMAP_MUX
-
-static void omap_mux_init_package(struct omap_mux *superset,
-				  struct omap_mux *package_subset,
-				  struct omap_ball *package_balls)
-{
-	if (package_subset)
-		omap_mux_package_fixup(package_subset, superset);
-	if (package_balls)
-		omap_mux_package_init_balls(package_balls, superset);
-}
-
-static void __init omap_mux_init_signals(struct omap_mux_partition *partition,
-					 struct omap_board_mux *board_mux)
-{
-	omap_mux_set_cmdline_signals();
-	omap_mux_write_array(partition, board_mux);
-}
-
-#else
-
-static void omap_mux_init_package(struct omap_mux *superset,
-				  struct omap_mux *package_subset,
-				  struct omap_ball *package_balls)
-{
-}
-
-static void __init omap_mux_init_signals(struct omap_mux_partition *partition,
-					 struct omap_board_mux *board_mux)
-{
-}
-
-#endif
-
-static u32 mux_partitions_cnt;
-
-int __init omap_mux_init(const char *name, u32 flags,
-			 u32 mux_pbase, u32 mux_size,
-			 struct omap_mux *superset,
-			 struct omap_mux *package_subset,
-			 struct omap_board_mux *board_mux,
-			 struct omap_ball *package_balls)
-{
-	struct omap_mux_partition *partition;
-
-	partition = kzalloc(sizeof(struct omap_mux_partition), GFP_KERNEL);
-	if (!partition)
-		return -ENOMEM;
-
-	partition->name = name;
-	partition->flags = flags;
-	partition->gpio = flags & OMAP_MUX_MODE7;
-	partition->size = mux_size;
-	partition->phys = mux_pbase;
-	partition->base = ioremap(mux_pbase, mux_size);
-	if (!partition->base) {
-		pr_err("%s: Could not ioremap mux partition at 0x%08x\n",
-			__func__, partition->phys);
-		kfree(partition);
-		return -ENODEV;
-	}
-
-	INIT_LIST_HEAD(&partition->muxmodes);
-
-	list_add_tail(&partition->node, &mux_partitions);
-	mux_partitions_cnt++;
-	pr_info("%s: Add partition: #%d: %s, flags: %x\n", __func__,
-		mux_partitions_cnt, partition->name, partition->flags);
-
-	omap_mux_init_package(superset, package_subset, package_balls);
-	omap_mux_init_list(partition, superset);
-	omap_mux_init_signals(partition, board_mux);
-
-	return 0;
-}
-
diff --git a/arch/arm/mach-omap2/mux.h b/arch/arm/mach-omap2/mux.h
deleted file mode 100644
index d121fb6df4e6..000000000000
--- a/arch/arm/mach-omap2/mux.h
+++ /dev/null
@@ -1,352 +0,0 @@
-/*
- * Copyright (C) 2009 Nokia
- * Copyright (C) 2009-2010 Texas Instruments
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include "mux34xx.h"
-
-#define OMAP_MUX_TERMINATOR	0xffff
-
-/* 34xx mux mode options for each pin. See TRM for options */
-#define OMAP_MUX_MODE0      0
-#define OMAP_MUX_MODE1      1
-#define OMAP_MUX_MODE2      2
-#define OMAP_MUX_MODE3      3
-#define OMAP_MUX_MODE4      4
-#define OMAP_MUX_MODE5      5
-#define OMAP_MUX_MODE6      6
-#define OMAP_MUX_MODE7      7
-
-/* 24xx/34xx mux bit defines */
-#define OMAP_PULL_ENA			(1 << 3)
-#define OMAP_PULL_UP			(1 << 4)
-#define OMAP_ALTELECTRICALSEL		(1 << 5)
-
-/* omap3/4/5 specific mux bit defines */
-#define OMAP_INPUT_EN			(1 << 8)
-#define OMAP_OFF_EN			(1 << 9)
-#define OMAP_OFFOUT_EN			(1 << 10)
-#define OMAP_OFFOUT_VAL			(1 << 11)
-#define OMAP_OFF_PULL_EN		(1 << 12)
-#define OMAP_OFF_PULL_UP		(1 << 13)
-#define OMAP_WAKEUP_EN			(1 << 14)
-#define OMAP_WAKEUP_EVENT		(1 << 15)
-
-/* Active pin states */
-#define OMAP_PIN_OUTPUT			0
-#define OMAP_PIN_INPUT			OMAP_INPUT_EN
-#define OMAP_PIN_INPUT_PULLUP		(OMAP_PULL_ENA | OMAP_INPUT_EN \
-						| OMAP_PULL_UP)
-#define OMAP_PIN_INPUT_PULLDOWN		(OMAP_PULL_ENA | OMAP_INPUT_EN)
-
-/* Off mode states */
-#define OMAP_PIN_OFF_NONE		0
-#define OMAP_PIN_OFF_OUTPUT_HIGH	(OMAP_OFF_EN | OMAP_OFFOUT_EN \
-						| OMAP_OFFOUT_VAL)
-#define OMAP_PIN_OFF_OUTPUT_LOW		(OMAP_OFF_EN | OMAP_OFFOUT_EN)
-#define OMAP_PIN_OFF_INPUT_PULLUP	(OMAP_OFF_EN | OMAP_OFF_PULL_EN \
-						| OMAP_OFF_PULL_UP)
-#define OMAP_PIN_OFF_INPUT_PULLDOWN	(OMAP_OFF_EN | OMAP_OFF_PULL_EN)
-#define OMAP_PIN_OFF_WAKEUPENABLE	OMAP_WAKEUP_EN
-
-#define OMAP_MODE_GPIO(partition, x)	(((x) & OMAP_MUX_MODE7) == \
-					  partition->gpio)
-#define OMAP_MODE_UART(x)	(((x) & OMAP_MUX_MODE7) == OMAP_MUX_MODE0)
-
-/* Flags for omapX_mux_init */
-#define OMAP_PACKAGE_MASK		0xffff
-#define OMAP_PACKAGE_CBP		6		/* 515-pin 0.40 0.50 */
-#define OMAP_PACKAGE_CUS		5		/* 423-pin 0.65 */
-#define OMAP_PACKAGE_CBB		4		/* 515-pin 0.40 0.50 */
-#define OMAP_PACKAGE_CBC		3		/* 515-pin 0.50 0.65 */
-
-#define OMAP_MUX_NR_MODES		8		/* Available modes */
-#define OMAP_MUX_NR_SIDES		2		/* Bottom & top */
-
-/*
- * omap_mux_init flags definition:
- *
- * OMAP_GPIO_MUX_MODE, bits 0-2: gpio muxing mode, same like pad control
- *      register which includes values from 0-7.
- * OMAP_MUX_REG_8BIT: Ensure that access to padconf is done in 8 bits.
- * The default value is 16 bits.
- */
-#define OMAP_MUX_GPIO_IN_MODE0		OMAP_MUX_MODE0
-#define OMAP_MUX_GPIO_IN_MODE1		OMAP_MUX_MODE1
-#define OMAP_MUX_GPIO_IN_MODE2		OMAP_MUX_MODE2
-#define OMAP_MUX_GPIO_IN_MODE3		OMAP_MUX_MODE3
-#define OMAP_MUX_GPIO_IN_MODE4		OMAP_MUX_MODE4
-#define OMAP_MUX_GPIO_IN_MODE5		OMAP_MUX_MODE5
-#define OMAP_MUX_GPIO_IN_MODE6		OMAP_MUX_MODE6
-#define OMAP_MUX_GPIO_IN_MODE7		OMAP_MUX_MODE7
-#define OMAP_MUX_REG_8BIT		(1 << 3)
-
-/**
- * struct omap_board_data - board specific device data
- * @id: instance id
- * @flags: additional flags for platform init code
- * @pads: array of device specific pads
- * @pads_cnt: ARRAY_SIZE() of pads
- */
-struct omap_board_data {
-	int			id;
-	u32			flags;
-	struct omap_device_pad	*pads;
-	int			pads_cnt;
-};
-
-/**
- * struct mux_partition - contain partition related information
- * @name: name of the current partition
- * @flags: flags specific to this partition
- * @gpio: gpio mux mode
- * @phys: physical address
- * @size: partition size
- * @base: virtual address after ioremap
- * @muxmodes: list of nodes that belong to a partition
- * @node: list node for the partitions linked list
- */
-struct omap_mux_partition {
-	const char		*name;
-	u32			flags;
-	u32			gpio;
-	u32			phys;
-	u32			size;
-	void __iomem		*base;
-	struct list_head	muxmodes;
-	struct list_head	node;
-};
-
-/**
- * struct omap_mux - data for omap mux register offset and it's value
- * @reg_offset:	mux register offset from the mux base
- * @gpio:	GPIO number
- * @muxnames:	available signal modes for a ball
- * @balls:	available balls on the package
- */
-struct omap_mux {
-	u16	reg_offset;
-	u16	gpio;
-#ifdef CONFIG_OMAP_MUX
-	char	*muxnames[OMAP_MUX_NR_MODES];
-#ifdef CONFIG_DEBUG_FS
-	char	*balls[OMAP_MUX_NR_SIDES];
-#endif
-#endif
-};
-
-/**
- * struct omap_ball - data for balls on omap package
- * @reg_offset:	mux register offset from the mux base
- * @balls:	available balls on the package
- */
-struct omap_ball {
-	u16	reg_offset;
-	char	*balls[OMAP_MUX_NR_SIDES];
-};
-
-/**
- * struct omap_board_mux - data for initializing mux registers
- * @reg_offset:	mux register offset from the mux base
- * @mux_value:	desired mux value to set
- */
-struct omap_board_mux {
-	u16	reg_offset;
-	u16	value;
-};
-
-#define OMAP_DEVICE_PAD_REMUX		BIT(1)	/* Dynamically remux a pad,
-						   needs enable, idle and off
-						   values */
-#define OMAP_DEVICE_PAD_WAKEUP		BIT(0)	/* Pad is wake-up capable */
-
-/**
- * struct omap_device_pad - device specific pad configuration
- * @name:		signal name
- * @flags:		pad specific runtime flags
- * @enable:		runtime value for a pad
- * @idle:		idle value for a pad
- * @off:		off value for a pad, defaults to safe mode
- * @partition:		mux partition
- * @mux:		mux register
- */
-struct omap_device_pad {
-	char				*name;
-	u8				flags;
-	u16				enable;
-	u16				idle;
-	u16				off;
-	struct omap_mux_partition	*partition;
-	struct omap_mux			*mux;
-};
-
-struct omap_hwmod_mux_info;
-
-#define OMAP_MUX_STATIC(signal, mode)					\
-{									\
-	.name	= (signal),						\
-	.enable	= (mode),						\
-}
-
-#if defined(CONFIG_OMAP_MUX)
-
-/**
- * omap_mux_init_gpio - initialize a signal based on the GPIO number
- * @gpio:		GPIO number
- * @val:		Options for the mux register value
- */
-int omap_mux_init_gpio(int gpio, int val);
-
-/**
- * omap_mux_init_signal - initialize a signal based on the signal name
- * @muxname:		Mux name in mode0_name.signal_name format
- * @val:		Options for the mux register value
- */
-int omap_mux_init_signal(const char *muxname, int val);
-
-/**
- * omap_hwmod_mux_init - initialize hwmod specific mux data
- * @bpads:		Board specific device signal names
- * @nr_pads:		Number of signal names for the device
- */
-extern struct omap_hwmod_mux_info *
-omap_hwmod_mux_init(struct omap_device_pad *bpads, int nr_pads);
-
-/**
- * omap_hwmod_mux - omap hwmod specific pin muxing
- * @hmux:		Pads for a hwmod
- * @state:		Desired _HWMOD_STATE
- *
- * Called only from omap_hwmod.c, do not use.
- */
-void omap_hwmod_mux(struct omap_hwmod_mux_info *hmux, u8 state);
-
-int omap_mux_get_by_name(const char *muxname,
-		struct omap_mux_partition **found_partition,
-		struct omap_mux **found_mux);
-#else
-
-static inline int omap_mux_get_by_name(const char *muxname,
-		struct omap_mux_partition **found_partition,
-		struct omap_mux **found_mux)
-{
-	return 0;
-}
-
-static inline int omap_mux_init_gpio(int gpio, int val)
-{
-	return 0;
-}
-static inline int omap_mux_init_signal(char *muxname, int val)
-{
-	return 0;
-}
-
-static inline struct omap_hwmod_mux_info *
-omap_hwmod_mux_init(struct omap_device_pad *bpads, int nr_pads)
-{
-	return NULL;
-}
-
-static inline void omap_hwmod_mux(struct omap_hwmod_mux_info *hmux, u8 state)
-{
-}
-
-static struct omap_board_mux *board_mux __maybe_unused;
-
-#endif
-
-/**
- * omap_mux_get_gpio() - get mux register value based on GPIO number
- * @gpio:		GPIO number
- *
- */
-u16 omap_mux_get_gpio(int gpio);
-
-/**
- * omap_mux_set_gpio() - set mux register value based on GPIO number
- * @val:		New mux register value
- * @gpio:		GPIO number
- *
- */
-void omap_mux_set_gpio(u16 val, int gpio);
-
-/**
- * omap_mux_get() - get a mux partition by name
- * @name:		Name of the mux partition
- *
- */
-struct omap_mux_partition *omap_mux_get(const char *name);
-
-/**
- * omap_mux_read() - read mux register
- * @partition:		Mux partition
- * @mux_offset:		Offset of the mux register
- *
- */
-u16 omap_mux_read(struct omap_mux_partition *p, u16 mux_offset);
-
-/**
- * omap_mux_write() - write mux register
- * @partition:		Mux partition
- * @val:		New mux register value
- * @mux_offset:		Offset of the mux register
- *
- * This should be only needed for dynamic remuxing of non-gpio signals.
- */
-void omap_mux_write(struct omap_mux_partition *p, u16 val, u16 mux_offset);
-
-/**
- * omap_mux_write_array() - write an array of mux registers
- * @partition:		Mux partition
- * @board_mux:		Array of mux registers terminated by MAP_MUX_TERMINATOR
- *
- * This should be only needed for dynamic remuxing of non-gpio signals.
- */
-void omap_mux_write_array(struct omap_mux_partition *p,
-			  struct omap_board_mux *board_mux);
-
-/**
- * omap2420_mux_init() - initialize mux system with board specific set
- * @board_mux:		Board specific mux table
- * @flags:		OMAP package type used for the board
- */
-int omap2420_mux_init(struct omap_board_mux *board_mux, int flags);
-
-/**
- * omap2430_mux_init() - initialize mux system with board specific set
- * @board_mux:		Board specific mux table
- * @flags:		OMAP package type used for the board
- */
-int omap2430_mux_init(struct omap_board_mux *board_mux, int flags);
-
-/**
- * omap3_mux_init() - initialize mux system with board specific set
- * @board_mux:		Board specific mux table
- * @flags:		OMAP package type used for the board
- */
-int omap3_mux_init(struct omap_board_mux *board_mux, int flags);
-
-/**
- * omap4_mux_init() - initialize mux system with board specific set
- * @board_subset:	Board specific mux table
- * @board_wkup_subset:	Board specific mux table for wakeup instance
- * @flags:		OMAP package type used for the board
- */
-int omap4_mux_init(struct omap_board_mux *board_subset,
-	struct omap_board_mux *board_wkup_subset, int flags);
-
-/**
- * omap_mux_init - private mux init function, do not call
- */
-int omap_mux_init(const char *name, u32 flags,
-		  u32 mux_pbase, u32 mux_size,
-		  struct omap_mux *superset,
-		  struct omap_mux *package_subset,
-		  struct omap_board_mux *board_mux,
-		  struct omap_ball *package_balls);
-
diff --git a/arch/arm/mach-omap2/mux34xx.c b/arch/arm/mach-omap2/mux34xx.c
deleted file mode 100644
index 393e687f99e2..000000000000
--- a/arch/arm/mach-omap2/mux34xx.c
+++ /dev/null
@@ -1,2061 +0,0 @@
-/*
- * Copyright (C) 2009 Nokia
- * Copyright (C) 2009 Texas Instruments
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/module.h>
-#include <linux/init.h>
-
-#include "mux.h"
-
-#ifdef CONFIG_OMAP_MUX
-
-#define _OMAP3_MUXENTRY(M0, g, m0, m1, m2, m3, m4, m5, m6, m7)		\
-{									\
-	.reg_offset	= (OMAP3_CONTROL_PADCONF_##M0##_OFFSET),	\
-	.gpio		= (g),						\
-	.muxnames	= { m0, m1, m2, m3, m4, m5, m6, m7 },		\
-}
-
-#else
-
-#define _OMAP3_MUXENTRY(M0, g, m0, m1, m2, m3, m4, m5, m6, m7)		\
-{									\
-	.reg_offset	= (OMAP3_CONTROL_PADCONF_##M0##_OFFSET),	\
-	.gpio		= (g),						\
-}
-
-#endif
-
-#define _OMAP3_BALLENTRY(M0, bb, bt)					\
-{									\
-	.reg_offset	= (OMAP3_CONTROL_PADCONF_##M0##_OFFSET),	\
-	.balls		= { bb, bt },					\
-}
-
-/*
- * Superset of all mux modes for omap3
- */
-static struct omap_mux __initdata omap3_muxmodes[] = {
-	_OMAP3_MUXENTRY(CAM_D0, 99,
-		"cam_d0", NULL, NULL, NULL,
-		"gpio_99", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D1, 100,
-		"cam_d1", NULL, NULL, NULL,
-		"gpio_100", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D10, 109,
-		"cam_d10", NULL, NULL, NULL,
-		"gpio_109", "hw_dbg8", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D11, 110,
-		"cam_d11", NULL, NULL, NULL,
-		"gpio_110", "hw_dbg9", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D2, 101,
-		"cam_d2", NULL, NULL, NULL,
-		"gpio_101", "hw_dbg4", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D3, 102,
-		"cam_d3", NULL, NULL, NULL,
-		"gpio_102", "hw_dbg5", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D4, 103,
-		"cam_d4", NULL, NULL, NULL,
-		"gpio_103", "hw_dbg6", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D5, 104,
-		"cam_d5", NULL, NULL, NULL,
-		"gpio_104", "hw_dbg7", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D6, 105,
-		"cam_d6", NULL, NULL, NULL,
-		"gpio_105", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D7, 106,
-		"cam_d7", NULL, NULL, NULL,
-		"gpio_106", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D8, 107,
-		"cam_d8", NULL, NULL, NULL,
-		"gpio_107", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D9, 108,
-		"cam_d9", NULL, NULL, NULL,
-		"gpio_108", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_FLD, 98,
-		"cam_fld", NULL, "cam_global_reset", NULL,
-		"gpio_98", "hw_dbg3", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_HS, 94,
-		"cam_hs", NULL, NULL, NULL,
-		"gpio_94", "hw_dbg0", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_PCLK, 97,
-		"cam_pclk", NULL, NULL, NULL,
-		"gpio_97", "hw_dbg2", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_STROBE, 126,
-		"cam_strobe", NULL, NULL, NULL,
-		"gpio_126", "hw_dbg11", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_VS, 95,
-		"cam_vs", NULL, NULL, NULL,
-		"gpio_95", "hw_dbg1", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_WEN, 167,
-		"cam_wen", NULL, "cam_shutter", NULL,
-		"gpio_167", "hw_dbg10", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_XCLKA, 96,
-		"cam_xclka", NULL, NULL, NULL,
-		"gpio_96", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_XCLKB, 111,
-		"cam_xclkb", NULL, NULL, NULL,
-		"gpio_111", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CSI2_DX0, 112,
-		"csi2_dx0", NULL, NULL, NULL,
-		"gpio_112", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CSI2_DX1, 114,
-		"csi2_dx1", NULL, NULL, NULL,
-		"gpio_114", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CSI2_DY0, 113,
-		"csi2_dy0", NULL, NULL, NULL,
-		"gpio_113", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CSI2_DY1, 115,
-		"csi2_dy1", NULL, NULL, NULL,
-		"gpio_115", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_ACBIAS, 69,
-		"dss_acbias", NULL, NULL, NULL,
-		"gpio_69", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA0, 70,
-		"dss_data0", NULL, "uart1_cts", NULL,
-		"gpio_70", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA1, 71,
-		"dss_data1", NULL, "uart1_rts", NULL,
-		"gpio_71", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA10, 80,
-		"dss_data10", NULL, NULL, NULL,
-		"gpio_80", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA11, 81,
-		"dss_data11", NULL, NULL, NULL,
-		"gpio_81", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA12, 82,
-		"dss_data12", NULL, NULL, NULL,
-		"gpio_82", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA13, 83,
-		"dss_data13", NULL, NULL, NULL,
-		"gpio_83", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA14, 84,
-		"dss_data14", NULL, NULL, NULL,
-		"gpio_84", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA15, 85,
-		"dss_data15", NULL, NULL, NULL,
-		"gpio_85", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA16, 86,
-		"dss_data16", NULL, NULL, NULL,
-		"gpio_86", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA17, 87,
-		"dss_data17", NULL, NULL, NULL,
-		"gpio_87", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA18, 88,
-		"dss_data18", NULL, "mcspi3_clk", "dss_data0",
-		"gpio_88", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA19, 89,
-		"dss_data19", NULL, "mcspi3_simo", "dss_data1",
-		"gpio_89", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA20, 90,
-		"dss_data20", NULL, "mcspi3_somi", "dss_data2",
-		"gpio_90", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA21, 91,
-		"dss_data21", NULL, "mcspi3_cs0", "dss_data3",
-		"gpio_91", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA22, 92,
-		"dss_data22", NULL, "mcspi3_cs1", "dss_data4",
-		"gpio_92", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA23, 93,
-		"dss_data23", NULL, NULL, "dss_data5",
-		"gpio_93", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA2, 72,
-		"dss_data2", NULL, NULL, NULL,
-		"gpio_72", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA3, 73,
-		"dss_data3", NULL, NULL, NULL,
-		"gpio_73", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA4, 74,
-		"dss_data4", NULL, "uart3_rx_irrx", NULL,
-		"gpio_74", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA5, 75,
-		"dss_data5", NULL, "uart3_tx_irtx", NULL,
-		"gpio_75", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA6, 76,
-		"dss_data6", NULL, "uart1_tx", NULL,
-		"gpio_76", "hw_dbg14", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA7, 77,
-		"dss_data7", NULL, "uart1_rx", NULL,
-		"gpio_77", "hw_dbg15", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA8, 78,
-		"dss_data8", NULL, NULL, NULL,
-		"gpio_78", "hw_dbg16", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA9, 79,
-		"dss_data9", NULL, NULL, NULL,
-		"gpio_79", "hw_dbg17", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_HSYNC, 67,
-		"dss_hsync", NULL, NULL, NULL,
-		"gpio_67", "hw_dbg13", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_PCLK, 66,
-		"dss_pclk", NULL, NULL, NULL,
-		"gpio_66", "hw_dbg12", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_VSYNC, 68,
-		"dss_vsync", NULL, NULL, NULL,
-		"gpio_68", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(ETK_CLK, 12,
-		"etk_clk", "mcbsp5_clkx", "sdmmc3_clk", "hsusb1_stp",
-		"gpio_12", "mm1_rxdp", "hsusb1_tll_stp", "hw_dbg0"),
-	_OMAP3_MUXENTRY(ETK_CTL, 13,
-		"etk_ctl", NULL, "sdmmc3_cmd", "hsusb1_clk",
-		"gpio_13", NULL, "hsusb1_tll_clk", "hw_dbg1"),
-	_OMAP3_MUXENTRY(ETK_D0, 14,
-		"etk_d0", "mcspi3_simo", "sdmmc3_dat4", "hsusb1_data0",
-		"gpio_14", "mm1_rxrcv", "hsusb1_tll_data0", "hw_dbg2"),
-	_OMAP3_MUXENTRY(ETK_D1, 15,
-		"etk_d1", "mcspi3_somi", NULL, "hsusb1_data1",
-		"gpio_15", "mm1_txse0", "hsusb1_tll_data1", "hw_dbg3"),
-	_OMAP3_MUXENTRY(ETK_D10, 24,
-		"etk_d10", NULL, "uart1_rx", "hsusb2_clk",
-		"gpio_24", NULL, "hsusb2_tll_clk", "hw_dbg12"),
-	_OMAP3_MUXENTRY(ETK_D11, 25,
-		"etk_d11", NULL, NULL, "hsusb2_stp",
-		"gpio_25", "mm2_rxdp", "hsusb2_tll_stp", "hw_dbg13"),
-	_OMAP3_MUXENTRY(ETK_D12, 26,
-		"etk_d12", NULL, NULL, "hsusb2_dir",
-		"gpio_26", NULL, "hsusb2_tll_dir", "hw_dbg14"),
-	_OMAP3_MUXENTRY(ETK_D13, 27,
-		"etk_d13", NULL, NULL, "hsusb2_nxt",
-		"gpio_27", "mm2_rxdm", "hsusb2_tll_nxt", "hw_dbg15"),
-	_OMAP3_MUXENTRY(ETK_D14, 28,
-		"etk_d14", NULL, NULL, "hsusb2_data0",
-		"gpio_28", "mm2_rxrcv", "hsusb2_tll_data0", "hw_dbg16"),
-	_OMAP3_MUXENTRY(ETK_D15, 29,
-		"etk_d15", NULL, NULL, "hsusb2_data1",
-		"gpio_29", "mm2_txse0", "hsusb2_tll_data1", "hw_dbg17"),
-	_OMAP3_MUXENTRY(ETK_D2, 16,
-		"etk_d2", "mcspi3_cs0", NULL, "hsusb1_data2",
-		"gpio_16", "mm1_txdat", "hsusb1_tll_data2", "hw_dbg4"),
-	_OMAP3_MUXENTRY(ETK_D3, 17,
-		"etk_d3", "mcspi3_clk", "sdmmc3_dat3", "hsusb1_data7",
-		"gpio_17", NULL, "hsusb1_tll_data7", "hw_dbg5"),
-	_OMAP3_MUXENTRY(ETK_D4, 18,
-		"etk_d4", "mcbsp5_dr", "sdmmc3_dat0", "hsusb1_data4",
-		"gpio_18", NULL, "hsusb1_tll_data4", "hw_dbg6"),
-	_OMAP3_MUXENTRY(ETK_D5, 19,
-		"etk_d5", "mcbsp5_fsx", "sdmmc3_dat1", "hsusb1_data5",
-		"gpio_19", NULL, "hsusb1_tll_data5", "hw_dbg7"),
-	_OMAP3_MUXENTRY(ETK_D6, 20,
-		"etk_d6", "mcbsp5_dx", "sdmmc3_dat2", "hsusb1_data6",
-		"gpio_20", NULL, "hsusb1_tll_data6", "hw_dbg8"),
-	_OMAP3_MUXENTRY(ETK_D7, 21,
-		"etk_d7", "mcspi3_cs1", "sdmmc3_dat7", "hsusb1_data3",
-		"gpio_21", "mm1_txen_n", "hsusb1_tll_data3", "hw_dbg9"),
-	_OMAP3_MUXENTRY(ETK_D8, 22,
-		"etk_d8", "sys_drm_msecure", "sdmmc3_dat6", "hsusb1_dir",
-		"gpio_22", NULL, "hsusb1_tll_dir", "hw_dbg10"),
-	_OMAP3_MUXENTRY(ETK_D9, 23,
-		"etk_d9", "sys_secure_indicator", "sdmmc3_dat5", "hsusb1_nxt",
-		"gpio_23", "mm1_rxdm", "hsusb1_tll_nxt", "hw_dbg11"),
-	_OMAP3_MUXENTRY(GPMC_A1, 34,
-		"gpmc_a1", NULL, NULL, NULL,
-		"gpio_34", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_A10, 43,
-		"gpmc_a10", "sys_ndmareq3", NULL, NULL,
-		"gpio_43", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_A2, 35,
-		"gpmc_a2", NULL, NULL, NULL,
-		"gpio_35", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_A3, 36,
-		"gpmc_a3", NULL, NULL, NULL,
-		"gpio_36", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_A4, 37,
-		"gpmc_a4", NULL, NULL, NULL,
-		"gpio_37", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_A5, 38,
-		"gpmc_a5", NULL, NULL, NULL,
-		"gpio_38", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_A6, 39,
-		"gpmc_a6", NULL, NULL, NULL,
-		"gpio_39", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_A7, 40,
-		"gpmc_a7", NULL, NULL, NULL,
-		"gpio_40", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_A8, 41,
-		"gpmc_a8", NULL, NULL, NULL,
-		"gpio_41", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_A9, 42,
-		"gpmc_a9", "sys_ndmareq2", NULL, NULL,
-		"gpio_42", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_CLK, 59,
-		"gpmc_clk", NULL, NULL, NULL,
-		"gpio_59", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_D10, 46,
-		"gpmc_d10", NULL, NULL, NULL,
-		"gpio_46", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_D11, 47,
-		"gpmc_d11", NULL, NULL, NULL,
-		"gpio_47", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_D12, 48,
-		"gpmc_d12", NULL, NULL, NULL,
-		"gpio_48", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_D13, 49,
-		"gpmc_d13", NULL, NULL, NULL,
-		"gpio_49", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_D14, 50,
-		"gpmc_d14", NULL, NULL, NULL,
-		"gpio_50", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_D15, 51,
-		"gpmc_d15", NULL, NULL, NULL,
-		"gpio_51", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_D8, 44,
-		"gpmc_d8", NULL, NULL, NULL,
-		"gpio_44", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_D9, 45,
-		"gpmc_d9", NULL, NULL, NULL,
-		"gpio_45", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_NBE0_CLE, 60,
-		"gpmc_nbe0_cle", NULL, NULL, NULL,
-		"gpio_60", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_NBE1, 61,
-		"gpmc_nbe1", NULL, NULL, NULL,
-		"gpio_61", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_NCS1, 52,
-		"gpmc_ncs1", NULL, NULL, NULL,
-		"gpio_52", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_NCS2, 53,
-		"gpmc_ncs2", NULL, NULL, NULL,
-		"gpio_53", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_NCS3, 54,
-		"gpmc_ncs3", "sys_ndmareq0", NULL, NULL,
-		"gpio_54", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_NCS4, 55,
-		"gpmc_ncs4", "sys_ndmareq1", "mcbsp4_clkx", "gpt9_pwm_evt",
-		"gpio_55", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_NCS5, 56,
-		"gpmc_ncs5", "sys_ndmareq2", "mcbsp4_dr", "gpt10_pwm_evt",
-		"gpio_56", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_NCS6, 57,
-		"gpmc_ncs6", "sys_ndmareq3", "mcbsp4_dx", "gpt11_pwm_evt",
-		"gpio_57", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_NCS7, 58,
-		"gpmc_ncs7", "gpmc_io_dir", "mcbsp4_fsx", "gpt8_pwm_evt",
-		"gpio_58", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_NWP, 62,
-		"gpmc_nwp", NULL, NULL, NULL,
-		"gpio_62", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_WAIT1, 63,
-		"gpmc_wait1", NULL, NULL, NULL,
-		"gpio_63", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_WAIT2, 64,
-		"gpmc_wait2", NULL, NULL, NULL,
-		"gpio_64", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_WAIT3, 65,
-		"gpmc_wait3", "sys_ndmareq1", NULL, NULL,
-		"gpio_65", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HDQ_SIO, 170,
-		"hdq_sio", "sys_altclk", "i2c2_sccbe", "i2c3_sccbe",
-		"gpio_170", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_CLK, 120,
-		"hsusb0_clk", NULL, NULL, NULL,
-		"gpio_120", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DATA0, 125,
-		"hsusb0_data0", NULL, "uart3_tx_irtx", NULL,
-		"gpio_125", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DATA1, 130,
-		"hsusb0_data1", NULL, "uart3_rx_irrx", NULL,
-		"gpio_130", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DATA2, 131,
-		"hsusb0_data2", NULL, "uart3_rts_sd", NULL,
-		"gpio_131", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DATA3, 169,
-		"hsusb0_data3", NULL, "uart3_cts_rctx", NULL,
-		"gpio_169", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DATA4, 188,
-		"hsusb0_data4", NULL, NULL, NULL,
-		"gpio_188", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DATA5, 189,
-		"hsusb0_data5", NULL, NULL, NULL,
-		"gpio_189", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DATA6, 190,
-		"hsusb0_data6", NULL, NULL, NULL,
-		"gpio_190", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DATA7, 191,
-		"hsusb0_data7", NULL, NULL, NULL,
-		"gpio_191", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DIR, 122,
-		"hsusb0_dir", NULL, NULL, NULL,
-		"gpio_122", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_NXT, 124,
-		"hsusb0_nxt", NULL, NULL, NULL,
-		"gpio_124", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_STP, 121,
-		"hsusb0_stp", NULL, NULL, NULL,
-		"gpio_121", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(I2C2_SCL, 168,
-		"i2c2_scl", NULL, NULL, NULL,
-		"gpio_168", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(I2C2_SDA, 183,
-		"i2c2_sda", NULL, NULL, NULL,
-		"gpio_183", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(I2C3_SCL, 184,
-		"i2c3_scl", NULL, NULL, NULL,
-		"gpio_184", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(I2C3_SDA, 185,
-		"i2c3_sda", NULL, NULL, NULL,
-		"gpio_185", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(I2C4_SCL, 0,
-		"i2c4_scl", "sys_nvmode1", NULL, NULL,
-		NULL, NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(I2C4_SDA, 0,
-		"i2c4_sda", "sys_nvmode2", NULL, NULL,
-		NULL, NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(JTAG_EMU0, 11,
-		"jtag_emu0", NULL, NULL, NULL,
-		"gpio_11", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(JTAG_EMU1, 31,
-		"jtag_emu1", NULL, NULL, NULL,
-		"gpio_31", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP1_CLKR, 156,
-		"mcbsp1_clkr", "mcspi4_clk", NULL, NULL,
-		"gpio_156", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP1_CLKX, 162,
-		"mcbsp1_clkx", NULL, "mcbsp3_clkx", NULL,
-		"gpio_162", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP1_DR, 159,
-		"mcbsp1_dr", "mcspi4_somi", "mcbsp3_dr", NULL,
-		"gpio_159", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP1_DX, 158,
-		"mcbsp1_dx", "mcspi4_simo", "mcbsp3_dx", NULL,
-		"gpio_158", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP1_FSR, 157,
-		"mcbsp1_fsr", NULL, "cam_global_reset", NULL,
-		"gpio_157", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP1_FSX, 161,
-		"mcbsp1_fsx", "mcspi4_cs0", "mcbsp3_fsx", NULL,
-		"gpio_161", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP2_CLKX, 117,
-		"mcbsp2_clkx", NULL, NULL, NULL,
-		"gpio_117", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP2_DR, 118,
-		"mcbsp2_dr", NULL, NULL, NULL,
-		"gpio_118", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP2_DX, 119,
-		"mcbsp2_dx", NULL, NULL, NULL,
-		"gpio_119", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP2_FSX, 116,
-		"mcbsp2_fsx", NULL, NULL, NULL,
-		"gpio_116", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP3_CLKX, 142,
-		"mcbsp3_clkx", "uart2_tx", NULL, NULL,
-		"gpio_142", "hsusb3_tll_data6", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP3_DR, 141,
-		"mcbsp3_dr", "uart2_rts", NULL, NULL,
-		"gpio_141", "hsusb3_tll_data5", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP3_DX, 140,
-		"mcbsp3_dx", "uart2_cts", NULL, NULL,
-		"gpio_140", "hsusb3_tll_data4", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP3_FSX, 143,
-		"mcbsp3_fsx", "uart2_rx", NULL, NULL,
-		"gpio_143", "hsusb3_tll_data7", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP4_CLKX, 152,
-		"mcbsp4_clkx", NULL, NULL, NULL,
-		"gpio_152", "hsusb3_tll_data1", "mm3_txse0", "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP4_DR, 153,
-		"mcbsp4_dr", NULL, NULL, NULL,
-		"gpio_153", "hsusb3_tll_data0", "mm3_rxrcv", "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP4_DX, 154,
-		"mcbsp4_dx", NULL, NULL, NULL,
-		"gpio_154", "hsusb3_tll_data2", "mm3_txdat", "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP4_FSX, 155,
-		"mcbsp4_fsx", NULL, NULL, NULL,
-		"gpio_155", "hsusb3_tll_data3", "mm3_txen_n", "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP_CLKS, 160,
-		"mcbsp_clks", NULL, "cam_shutter", NULL,
-		"gpio_160", "uart1_cts", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI1_CLK, 171,
-		"mcspi1_clk", "sdmmc2_dat4", NULL, NULL,
-		"gpio_171", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI1_CS0, 174,
-		"mcspi1_cs0", "sdmmc2_dat7", NULL, NULL,
-		"gpio_174", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI1_CS1, 175,
-		"mcspi1_cs1", NULL, NULL, "sdmmc3_cmd",
-		"gpio_175", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI1_CS2, 176,
-		"mcspi1_cs2", NULL, NULL, "sdmmc3_clk",
-		"gpio_176", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI1_CS3, 177,
-		"mcspi1_cs3", NULL, "hsusb2_tll_data2", "hsusb2_data2",
-		"gpio_177", "mm2_txdat", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI1_SIMO, 172,
-		"mcspi1_simo", "sdmmc2_dat5", NULL, NULL,
-		"gpio_172", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI1_SOMI, 173,
-		"mcspi1_somi", "sdmmc2_dat6", NULL, NULL,
-		"gpio_173", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI2_CLK, 178,
-		"mcspi2_clk", NULL, "hsusb2_tll_data7", "hsusb2_data7",
-		"gpio_178", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI2_CS0, 181,
-		"mcspi2_cs0", "gpt11_pwm_evt",
-		"hsusb2_tll_data6", "hsusb2_data6",
-		"gpio_181", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI2_CS1, 182,
-		"mcspi2_cs1", "gpt8_pwm_evt",
-		"hsusb2_tll_data3", "hsusb2_data3",
-		"gpio_182", "mm2_txen_n", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI2_SIMO, 179,
-		"mcspi2_simo", "gpt9_pwm_evt",
-		"hsusb2_tll_data4", "hsusb2_data4",
-		"gpio_179", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI2_SOMI, 180,
-		"mcspi2_somi", "gpt10_pwm_evt",
-		"hsusb2_tll_data5", "hsusb2_data5",
-		"gpio_180", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_CLK, 120,
-		"sdmmc1_clk", NULL, NULL, NULL,
-		"gpio_120", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_CMD, 121,
-		"sdmmc1_cmd", NULL, NULL, NULL,
-		"gpio_121", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_DAT0, 122,
-		"sdmmc1_dat0", NULL, NULL, NULL,
-		"gpio_122", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_DAT1, 123,
-		"sdmmc1_dat1", NULL, NULL, NULL,
-		"gpio_123", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_DAT2, 124,
-		"sdmmc1_dat2", NULL, NULL, NULL,
-		"gpio_124", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_DAT3, 125,
-		"sdmmc1_dat3", NULL, NULL, NULL,
-		"gpio_125", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_DAT4, 126,
-		"sdmmc1_dat4", NULL, "sim_io", NULL,
-		"gpio_126", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_DAT5, 127,
-		"sdmmc1_dat5", NULL, "sim_clk", NULL,
-		"gpio_127", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_DAT6, 128,
-		"sdmmc1_dat6", NULL, "sim_pwrctrl", NULL,
-		"gpio_128", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_DAT7, 129,
-		"sdmmc1_dat7", NULL, "sim_rst", NULL,
-		"gpio_129", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_CLK, 130,
-		"sdmmc2_clk", "mcspi3_clk", NULL, NULL,
-		"gpio_130", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_CMD, 131,
-		"sdmmc2_cmd", "mcspi3_simo", NULL, NULL,
-		"gpio_131", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_DAT0, 132,
-		"sdmmc2_dat0", "mcspi3_somi", NULL, NULL,
-		"gpio_132", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_DAT1, 133,
-		"sdmmc2_dat1", NULL, NULL, NULL,
-		"gpio_133", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_DAT2, 134,
-		"sdmmc2_dat2", "mcspi3_cs1", NULL, NULL,
-		"gpio_134", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_DAT3, 135,
-		"sdmmc2_dat3", "mcspi3_cs0", NULL, NULL,
-		"gpio_135", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_DAT4, 136,
-		"sdmmc2_dat4", "sdmmc2_dir_dat0", NULL, "sdmmc3_dat0",
-		"gpio_136", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_DAT5, 137,
-		"sdmmc2_dat5", "sdmmc2_dir_dat1",
-		"cam_global_reset", "sdmmc3_dat1",
-		"gpio_137", "hsusb3_tll_stp", "mm3_rxdp", "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_DAT6, 138,
-		"sdmmc2_dat6", "sdmmc2_dir_cmd", "cam_shutter", "sdmmc3_dat2",
-		"gpio_138", "hsusb3_tll_dir", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_DAT7, 139,
-		"sdmmc2_dat7", "sdmmc2_clkin", NULL, "sdmmc3_dat3",
-		"gpio_139", "hsusb3_tll_nxt", "mm3_rxdm", "safe_mode"),
-	_OMAP3_MUXENTRY(SDRC_CKE0, 0,
-		"sdrc_cke0", NULL, NULL, NULL,
-		NULL, NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDRC_CKE1, 0,
-		"sdrc_cke1", NULL, NULL, NULL,
-		NULL, NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT0, 2,
-		"sys_boot0", NULL, NULL, NULL,
-		"gpio_2", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT1, 3,
-		"sys_boot1", NULL, NULL, NULL,
-		"gpio_3", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT2, 4,
-		"sys_boot2", NULL, NULL, NULL,
-		"gpio_4", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT3, 5,
-		"sys_boot3", NULL, NULL, NULL,
-		"gpio_5", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT4, 6,
-		"sys_boot4", "sdmmc2_dir_dat2", NULL, NULL,
-		"gpio_6", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT5, 7,
-		"sys_boot5", "sdmmc2_dir_dat3", NULL, NULL,
-		"gpio_7", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT6, 8,
-		"sys_boot6", NULL, NULL, NULL,
-		"gpio_8", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_CLKOUT1, 10,
-		"sys_clkout1", NULL, NULL, NULL,
-		"gpio_10", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_CLKOUT2, 186,
-		"sys_clkout2", NULL, NULL, NULL,
-		"gpio_186", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_CLKREQ, 1,
-		"sys_clkreq", NULL, NULL, NULL,
-		"gpio_1", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_NIRQ, 0,
-		"sys_nirq", NULL, NULL, NULL,
-		"gpio_0", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_NRESWARM, 30,
-		"sys_nreswarm", NULL, NULL, NULL,
-		"gpio_30", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_OFF_MODE, 9,
-		"sys_off_mode", NULL, NULL, NULL,
-		"gpio_9", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART1_CTS, 150,
-		"uart1_cts", "ssi1_rdy_tx", NULL, NULL,
-		"gpio_150", "hsusb3_tll_clk", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART1_RTS, 149,
-		"uart1_rts", "ssi1_flag_tx", NULL, NULL,
-		"gpio_149", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART1_RX, 151,
-		"uart1_rx", "ssi1_wake_tx", "mcbsp1_clkr", "mcspi4_clk",
-		"gpio_151", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART1_TX, 148,
-		"uart1_tx", "ssi1_dat_tx", NULL, NULL,
-		"gpio_148", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART2_CTS, 144,
-		"uart2_cts", "mcbsp3_dx", "gpt9_pwm_evt", NULL,
-		"gpio_144", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART2_RTS, 145,
-		"uart2_rts", "mcbsp3_dr", "gpt10_pwm_evt", NULL,
-		"gpio_145", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART2_RX, 147,
-		"uart2_rx", "mcbsp3_fsx", "gpt8_pwm_evt", NULL,
-		"gpio_147", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART2_TX, 146,
-		"uart2_tx", "mcbsp3_clkx", "gpt11_pwm_evt", NULL,
-		"gpio_146", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART3_CTS_RCTX, 163,
-		"uart3_cts_rctx", NULL, NULL, NULL,
-		"gpio_163", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART3_RTS_SD, 164,
-		"uart3_rts_sd", NULL, NULL, NULL,
-		"gpio_164", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART3_RX_IRRX, 165,
-		"uart3_rx_irrx", NULL, NULL, NULL,
-		"gpio_165", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART3_TX_IRTX, 166,
-		"uart3_tx_irtx", NULL, NULL, NULL,
-		"gpio_166", NULL, NULL, "safe_mode"),
-
-	/* Only on 3630, see omap36xx_cbp_subset for the signals */
-	_OMAP3_MUXENTRY(GPMC_A11, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MBUSFLAG, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MREAD, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MWRITE, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_SBUSFLAG, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_SREAD, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_SWRITE, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(GPMC_A11, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD28, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD29, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD32, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD33, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD34, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD35, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD36, 0,
-		NULL, NULL, NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	{ .reg_offset = OMAP_MUX_TERMINATOR },
-};
-
-/*
- * Signals different on CBC package compared to the superset
- */
-#if defined(CONFIG_OMAP_MUX) && defined(CONFIG_OMAP_PACKAGE_CBC)
-static struct omap_mux __initdata omap3_cbc_subset[] = {
-	{ .reg_offset = OMAP_MUX_TERMINATOR },
-};
-#else
-#define omap3_cbc_subset	NULL
-#endif
-
-/*
- * Balls for CBC package
- * 515-pin s-PBGA Package, 0.65mm Ball Pitch (Top), 0.50mm Ball Pitch (Bottom)
- *
- * FIXME: What's up with the outdated TI documentation? See:
- *
- * http://wiki.davincidsp.com/index.php/Datasheet_Errata_for_OMAP35x_CBC_Package
- * http://community.ti.com/forums/t/10982.aspx
- */
-#if defined(CONFIG_OMAP_MUX) && defined(CONFIG_DEBUG_FS)	\
-		&& defined(CONFIG_OMAP_PACKAGE_CBC)
-static struct omap_ball __initdata omap3_cbc_ball[] = {
-	_OMAP3_BALLENTRY(CAM_D0, "ae16", NULL),
-	_OMAP3_BALLENTRY(CAM_D1, "ae15", NULL),
-	_OMAP3_BALLENTRY(CAM_D10, "d25", NULL),
-	_OMAP3_BALLENTRY(CAM_D11, "e26", NULL),
-	_OMAP3_BALLENTRY(CAM_D2, "a24", NULL),
-	_OMAP3_BALLENTRY(CAM_D3, "b24", NULL),
-	_OMAP3_BALLENTRY(CAM_D4, "d24", NULL),
-	_OMAP3_BALLENTRY(CAM_D5, "c24", NULL),
-	_OMAP3_BALLENTRY(CAM_D6, "p25", NULL),
-	_OMAP3_BALLENTRY(CAM_D7, "p26", NULL),
-	_OMAP3_BALLENTRY(CAM_D8, "n25", NULL),
-	_OMAP3_BALLENTRY(CAM_D9, "n26", NULL),
-	_OMAP3_BALLENTRY(CAM_FLD, "b23", NULL),
-	_OMAP3_BALLENTRY(CAM_HS, "c23", NULL),
-	_OMAP3_BALLENTRY(CAM_PCLK, "c26", NULL),
-	_OMAP3_BALLENTRY(CAM_STROBE, "d26", NULL),
-	_OMAP3_BALLENTRY(CAM_VS, "d23", NULL),
-	_OMAP3_BALLENTRY(CAM_WEN, "a23", NULL),
-	_OMAP3_BALLENTRY(CAM_XCLKA, "c25", NULL),
-	_OMAP3_BALLENTRY(CAM_XCLKB, "e25", NULL),
-	_OMAP3_BALLENTRY(CSI2_DX0, "ad17", NULL),
-	_OMAP3_BALLENTRY(CSI2_DX1, "ae18", NULL),
-	_OMAP3_BALLENTRY(CSI2_DY0, "ad16", NULL),
-	_OMAP3_BALLENTRY(CSI2_DY1, "ae17", NULL),
-	_OMAP3_BALLENTRY(DSS_ACBIAS, "f26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA0, "ae21", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA1, "ae22", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA10, "ac26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA11, "ad26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA12, "aa25", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA13, "y25", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA14, "aa26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA15, "ab26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA16, "l25", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA17, "l26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA18, "m24", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA19, "m26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA2, "ae23", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA20, "f25", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA21, "n24", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA22, "ac25", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA23, "ab25", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA3, "ae24", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA4, "ad23", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA5, "ad24", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA6, "g26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA7, "h25", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA8, "h26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA9, "j26", NULL),
-	_OMAP3_BALLENTRY(DSS_HSYNC, "k24", NULL),
-	_OMAP3_BALLENTRY(DSS_PCLK, "g25", NULL),
-	_OMAP3_BALLENTRY(DSS_VSYNC, "m25", NULL),
-	_OMAP3_BALLENTRY(ETK_CLK, "ab2", NULL),
-	_OMAP3_BALLENTRY(ETK_CTL, "ab3", NULL),
-	_OMAP3_BALLENTRY(ETK_D0, "ac3", NULL),
-	_OMAP3_BALLENTRY(ETK_D1, "ad4", NULL),
-	_OMAP3_BALLENTRY(ETK_D10, "ae4", NULL),
-	_OMAP3_BALLENTRY(ETK_D11, "af6", NULL),
-	_OMAP3_BALLENTRY(ETK_D12, "ae6", NULL),
-	_OMAP3_BALLENTRY(ETK_D13, "af7", NULL),
-	_OMAP3_BALLENTRY(ETK_D14, "af9", NULL),
-	_OMAP3_BALLENTRY(ETK_D15, "ae9", NULL),
-	_OMAP3_BALLENTRY(ETK_D2, "ad3", NULL),
-	_OMAP3_BALLENTRY(ETK_D3, "aa3", NULL),
-	_OMAP3_BALLENTRY(ETK_D4, "y3", NULL),
-	_OMAP3_BALLENTRY(ETK_D5, "ab1", NULL),
-	_OMAP3_BALLENTRY(ETK_D6, "ae3", NULL),
-	_OMAP3_BALLENTRY(ETK_D7, "ad2", NULL),
-	_OMAP3_BALLENTRY(ETK_D8, "aa4", NULL),
-	_OMAP3_BALLENTRY(ETK_D9, "v2", NULL),
-	_OMAP3_BALLENTRY(GPMC_A1, "j2", NULL),
-	_OMAP3_BALLENTRY(GPMC_A10, "d2", NULL),
-	_OMAP3_BALLENTRY(GPMC_A2, "h1", NULL),
-	_OMAP3_BALLENTRY(GPMC_A3, "h2", NULL),
-	_OMAP3_BALLENTRY(GPMC_A4, "g2", NULL),
-	_OMAP3_BALLENTRY(GPMC_A5, "f1", NULL),
-	_OMAP3_BALLENTRY(GPMC_A6, "f2", NULL),
-	_OMAP3_BALLENTRY(GPMC_A7, "e1", NULL),
-	_OMAP3_BALLENTRY(GPMC_A8, "e2", NULL),
-	_OMAP3_BALLENTRY(GPMC_A9, "d1", NULL),
-	_OMAP3_BALLENTRY(GPMC_CLK, "n1", "l1"),
-	_OMAP3_BALLENTRY(GPMC_D10, "t1", "n1"),
-	_OMAP3_BALLENTRY(GPMC_D11, "u2", "p2"),
-	_OMAP3_BALLENTRY(GPMC_D12, "u1", "p1"),
-	_OMAP3_BALLENTRY(GPMC_D13, "p1", "m1"),
-	_OMAP3_BALLENTRY(GPMC_D14, "l2", "j2"),
-	_OMAP3_BALLENTRY(GPMC_D15, "m2", "k2"),
-	_OMAP3_BALLENTRY(GPMC_D8, "v1", "r1"),
-	_OMAP3_BALLENTRY(GPMC_D9, "y1", "t1"),
-	_OMAP3_BALLENTRY(GPMC_NBE0_CLE, "k2", NULL),
-	_OMAP3_BALLENTRY(GPMC_NBE1, "j1", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS1, "ad1", "w1"),
-	_OMAP3_BALLENTRY(GPMC_NCS2, "a3", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS3, "b6", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS4, "b4", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS5, "c4", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS6, "b5", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS7, "c5", NULL),
-	_OMAP3_BALLENTRY(GPMC_NWP, "ac6", "y5"),
-	_OMAP3_BALLENTRY(GPMC_WAIT1, "ac8", "y8"),
-	_OMAP3_BALLENTRY(GPMC_WAIT2, "b3", NULL),
-	_OMAP3_BALLENTRY(GPMC_WAIT3, "c6", NULL),
-	_OMAP3_BALLENTRY(HDQ_SIO, "j23", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_CLK, "w19", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA0, "v20", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA1, "y20", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA2, "v18", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA3, "w20", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA4, "w17", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA5, "y18", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA6, "y19", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA7, "y17", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DIR, "v19", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_NXT, "w18", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_STP, "u20", NULL),
-	_OMAP3_BALLENTRY(I2C2_SCL, "c2", NULL),
-	_OMAP3_BALLENTRY(I2C2_SDA, "c1", NULL),
-	_OMAP3_BALLENTRY(I2C3_SCL, "ab4", NULL),
-	_OMAP3_BALLENTRY(I2C3_SDA, "ac4", NULL),
-	_OMAP3_BALLENTRY(I2C4_SCL, "ad15", NULL),
-	_OMAP3_BALLENTRY(I2C4_SDA, "w16", NULL),
-	_OMAP3_BALLENTRY(JTAG_EMU0, "y15", NULL),
-	_OMAP3_BALLENTRY(JTAG_EMU1, "y14", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_CLKR, "u19", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_CLKX, "t17", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_DR, "t20", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_DX, "u17", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_FSR, "v17", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_FSX, "p20", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_CLKX, "r18", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_DR, "t18", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_DX, "r19", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_FSX, "u18", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_CLKX, "u3", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_DR, "n3", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_DX, "p3", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_FSX, "w3", NULL),
-	_OMAP3_BALLENTRY(MCBSP4_CLKX, "v3", NULL),
-	_OMAP3_BALLENTRY(MCBSP4_DR, "u4", NULL),
-	_OMAP3_BALLENTRY(MCBSP4_DX, "r3", NULL),
-	_OMAP3_BALLENTRY(MCBSP4_FSX, "t3", NULL),
-	_OMAP3_BALLENTRY(MCBSP_CLKS, "t19", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CLK, "p9", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS0, "r7", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS1, "r8", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS2, "r9", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS3, "t8", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_SIMO, "p8", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_SOMI, "p7", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_CLK, "w7", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_CS0, "v8", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_CS1, "v9", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_SIMO, "w8", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_SOMI, "u8", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_CLK, "n19", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_CMD, "l18", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT0, "m19", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT1, "m18", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT2, "k18", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT3, "n20", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT4, "m20", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT5, "p17", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT6, "p18", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT7, "p19", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_CLK, "w10", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_CMD, "r10", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT0, "t10", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT1, "t9", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT2, "u10", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT3, "u9", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT4, "v10", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT5, "m3", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT6, "l3", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT7, "k3", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT0, "f3", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT1, "d3", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT2, "c3", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT3, "e3", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT4, "e4", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT5, "g3", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT6, "d4", NULL),
-	_OMAP3_BALLENTRY(SYS_CLKOUT1, "ae14", NULL),
-	_OMAP3_BALLENTRY(SYS_CLKOUT2, "w11", NULL),
-	_OMAP3_BALLENTRY(SYS_CLKREQ, "w15", NULL),
-	_OMAP3_BALLENTRY(SYS_NIRQ, "v16", NULL),
-	_OMAP3_BALLENTRY(SYS_NRESWARM, "ad7", "aa5"),
-	_OMAP3_BALLENTRY(SYS_OFF_MODE, "v12", NULL),
-	_OMAP3_BALLENTRY(UART1_CTS, "w2", NULL),
-	_OMAP3_BALLENTRY(UART1_RTS, "r2", NULL),
-	_OMAP3_BALLENTRY(UART1_RX, "h3", NULL),
-	_OMAP3_BALLENTRY(UART1_TX, "l4", NULL),
-	_OMAP3_BALLENTRY(UART2_CTS, "y24", NULL),
-	_OMAP3_BALLENTRY(UART2_RTS, "aa24", NULL),
-	_OMAP3_BALLENTRY(UART2_RX, "ad21", NULL),
-	_OMAP3_BALLENTRY(UART2_TX, "ad22", NULL),
-	_OMAP3_BALLENTRY(UART3_CTS_RCTX, "f23", NULL),
-	_OMAP3_BALLENTRY(UART3_RTS_SD, "f24", NULL),
-	_OMAP3_BALLENTRY(UART3_RX_IRRX, "h24", NULL),
-	_OMAP3_BALLENTRY(UART3_TX_IRTX, "g24", NULL),
-	{ .reg_offset = OMAP_MUX_TERMINATOR },
-};
-#else
-#define omap3_cbc_ball	 NULL
-#endif
-
-/*
- * Signals different on CUS package compared to superset
- */
-#if defined(CONFIG_OMAP_MUX) && defined(CONFIG_OMAP_PACKAGE_CUS)
-static struct omap_mux __initdata omap3_cus_subset[] = {
-	_OMAP3_MUXENTRY(CAM_D10, 109,
-		"cam_d10", NULL, NULL, NULL,
-		"gpio_109", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D11, 110,
-		"cam_d11", NULL, NULL, NULL,
-		"gpio_110", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D2, 101,
-		"cam_d2", NULL, NULL, NULL,
-		"gpio_101", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D3, 102,
-		"cam_d3", NULL, NULL, NULL,
-		"gpio_102", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D4, 103,
-		"cam_d4", NULL, NULL, NULL,
-		"gpio_103", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D5, 104,
-		"cam_d5", NULL, NULL, NULL,
-		"gpio_104", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_FLD, 98,
-		"cam_fld", NULL, "cam_global_reset", NULL,
-		"gpio_98", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_HS, 94,
-		"cam_hs", NULL, NULL, NULL,
-		"gpio_94", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_PCLK, 97,
-		"cam_pclk", NULL, NULL, NULL,
-		"gpio_97", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_STROBE, 126,
-		"cam_strobe", NULL, NULL, NULL,
-		"gpio_126", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_VS, 95,
-		"cam_vs", NULL, NULL, NULL,
-		"gpio_95", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_WEN, 167,
-		"cam_wen", NULL, "cam_shutter", NULL,
-		"gpio_167", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA6, 76,
-		"dss_data6", NULL, "uart1_tx", NULL,
-		"gpio_76", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA7, 77,
-		"dss_data7", NULL, "uart1_rx", NULL,
-		"gpio_77", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA8, 78,
-		"dss_data8", NULL, NULL, NULL,
-		"gpio_78", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA9, 79,
-		"dss_data9", NULL, NULL, NULL,
-		"gpio_79", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_HSYNC, 67,
-		"dss_hsync", NULL, NULL, NULL,
-		"gpio_67", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_PCLK, 66,
-		"dss_pclk", NULL, NULL, NULL,
-		"gpio_66", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(ETK_CLK, 12,
-		"etk_clk", "mcbsp5_clkx", "sdmmc3_clk", "hsusb1_stp",
-		"gpio_12", "mm1_rxdp", "hsusb1_tll_stp", NULL),
-	_OMAP3_MUXENTRY(ETK_CTL, 13,
-		"etk_ctl", NULL, "sdmmc3_cmd", "hsusb1_clk",
-		"gpio_13", NULL, "hsusb1_tll_clk", NULL),
-	_OMAP3_MUXENTRY(ETK_D0, 14,
-		"etk_d0", "mcspi3_simo", "sdmmc3_dat4", "hsusb1_data0",
-		"gpio_14", "mm1_rxrcv", "hsusb1_tll_data0", NULL),
-	_OMAP3_MUXENTRY(ETK_D1, 15,
-		"etk_d1", "mcspi3_somi", NULL, "hsusb1_data1",
-		"gpio_15", "mm1_txse0", "hsusb1_tll_data1", NULL),
-	_OMAP3_MUXENTRY(ETK_D10, 24,
-		"etk_d10", NULL, "uart1_rx", "hsusb2_clk",
-		"gpio_24", NULL, "hsusb2_tll_clk", NULL),
-	_OMAP3_MUXENTRY(ETK_D11, 25,
-		"etk_d11", NULL, NULL, "hsusb2_stp",
-		"gpio_25", "mm2_rxdp", "hsusb2_tll_stp", NULL),
-	_OMAP3_MUXENTRY(ETK_D12, 26,
-		"etk_d12", NULL, NULL, "hsusb2_dir",
-		"gpio_26", NULL, "hsusb2_tll_dir", NULL),
-	_OMAP3_MUXENTRY(ETK_D13, 27,
-		"etk_d13", NULL, NULL, "hsusb2_nxt",
-		"gpio_27", "mm2_rxdm", "hsusb2_tll_nxt", NULL),
-	_OMAP3_MUXENTRY(ETK_D14, 28,
-		"etk_d14", NULL, NULL, "hsusb2_data0",
-		"gpio_28", "mm2_rxrcv", "hsusb2_tll_data0", NULL),
-	_OMAP3_MUXENTRY(ETK_D15, 29,
-		"etk_d15", NULL, NULL, "hsusb2_data1",
-		"gpio_29", "mm2_txse0", "hsusb2_tll_data1", NULL),
-	_OMAP3_MUXENTRY(ETK_D2, 16,
-		"etk_d2", "mcspi3_cs0", NULL, "hsusb1_data2",
-		"gpio_16", "mm1_txdat", "hsusb1_tll_data2", NULL),
-	_OMAP3_MUXENTRY(ETK_D3, 17,
-		"etk_d3", "mcspi3_clk", "sdmmc3_dat3", "hsusb1_data7",
-		"gpio_17", NULL, "hsusb1_tll_data7", NULL),
-	_OMAP3_MUXENTRY(ETK_D4, 18,
-		"etk_d4", "mcbsp5_dr", "sdmmc3_dat0", "hsusb1_data4",
-		"gpio_18", NULL, "hsusb1_tll_data4", NULL),
-	_OMAP3_MUXENTRY(ETK_D5, 19,
-		"etk_d5", "mcbsp5_fsx", "sdmmc3_dat1", "hsusb1_data5",
-		"gpio_19", NULL, "hsusb1_tll_data5", NULL),
-	_OMAP3_MUXENTRY(ETK_D6, 20,
-		"etk_d6", "mcbsp5_dx", "sdmmc3_dat2", "hsusb1_data6",
-		"gpio_20", NULL, "hsusb1_tll_data6", NULL),
-	_OMAP3_MUXENTRY(ETK_D7, 21,
-		"etk_d7", "mcspi3_cs1", "sdmmc3_dat7", "hsusb1_data3",
-		"gpio_21", "mm1_txen_n", "hsusb1_tll_data3", NULL),
-	_OMAP3_MUXENTRY(ETK_D8, 22,
-		"etk_d8", "sys_drm_msecure", "sdmmc3_dat6", "hsusb1_dir",
-		"gpio_22", NULL, "hsusb1_tll_dir", NULL),
-	_OMAP3_MUXENTRY(ETK_D9, 23,
-		"etk_d9", "sys_secure_indicator", "sdmmc3_dat5", "hsusb1_nxt",
-		"gpio_23", "mm1_rxdm", "hsusb1_tll_nxt", NULL),
-	_OMAP3_MUXENTRY(MCBSP3_CLKX, 142,
-		"mcbsp3_clkx", "uart2_tx", NULL, NULL,
-		"gpio_142", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP3_DR, 141,
-		"mcbsp3_dr", "uart2_rts", NULL, NULL,
-		"gpio_141", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP3_DX, 140,
-		"mcbsp3_dx", "uart2_cts", NULL, NULL,
-		"gpio_140", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP3_FSX, 143,
-		"mcbsp3_fsx", "uart2_rx", NULL, NULL,
-		"gpio_143", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_DAT5, 137,
-		"sdmmc2_dat5", "sdmmc2_dir_dat1",
-		"cam_global_reset", "sdmmc3_dat1",
-		"gpio_137", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_DAT6, 138,
-		"sdmmc2_dat6", "sdmmc2_dir_cmd", "cam_shutter", "sdmmc3_dat2",
-		"gpio_138", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC2_DAT7, 139,
-		"sdmmc2_dat7", "sdmmc2_clkin", NULL, "sdmmc3_dat3",
-		"gpio_139", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART1_CTS, 150,
-		"uart1_cts", NULL, NULL, NULL,
-		"gpio_150", NULL, NULL, "safe_mode"),
-	{ .reg_offset = OMAP_MUX_TERMINATOR },
-};
-#else
-#define omap3_cus_subset	NULL
-#endif
-
-/*
- * Balls for CUS package
- * 423-pin s-PBGA Package, 0.65mm Ball Pitch (Bottom)
- */
-#if defined(CONFIG_OMAP_MUX) && defined(CONFIG_DEBUG_FS)		\
-		&& defined(CONFIG_OMAP_PACKAGE_CUS)
-static struct omap_ball __initdata omap3_cus_ball[] = {
-	_OMAP3_BALLENTRY(CAM_D0, "ab18", NULL),
-	_OMAP3_BALLENTRY(CAM_D1, "ac18", NULL),
-	_OMAP3_BALLENTRY(CAM_D10, "f21", NULL),
-	_OMAP3_BALLENTRY(CAM_D11, "g21", NULL),
-	_OMAP3_BALLENTRY(CAM_D2, "g19", NULL),
-	_OMAP3_BALLENTRY(CAM_D3, "f19", NULL),
-	_OMAP3_BALLENTRY(CAM_D4, "g20", NULL),
-	_OMAP3_BALLENTRY(CAM_D5, "b21", NULL),
-	_OMAP3_BALLENTRY(CAM_D6, "l24", NULL),
-	_OMAP3_BALLENTRY(CAM_D7, "k24", NULL),
-	_OMAP3_BALLENTRY(CAM_D8, "j23", NULL),
-	_OMAP3_BALLENTRY(CAM_D9, "k23", NULL),
-	_OMAP3_BALLENTRY(CAM_FLD, "h24", NULL),
-	_OMAP3_BALLENTRY(CAM_HS, "a22", NULL),
-	_OMAP3_BALLENTRY(CAM_PCLK, "j19", NULL),
-	_OMAP3_BALLENTRY(CAM_STROBE, "j20", NULL),
-	_OMAP3_BALLENTRY(CAM_VS, "e18", NULL),
-	_OMAP3_BALLENTRY(CAM_WEN, "f18", NULL),
-	_OMAP3_BALLENTRY(CAM_XCLKA, "b22", NULL),
-	_OMAP3_BALLENTRY(CAM_XCLKB, "c22", NULL),
-	_OMAP3_BALLENTRY(DSS_ACBIAS, "j21", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA0, "ac19", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA1, "ab19", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA10, "ac22", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA11, "ac23", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA12, "ab22", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA13, "y22", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA14, "w22", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA15, "v22", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA16, "j22", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA17, "g23", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA18, "g24", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA19, "h23", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA2, "ad20", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA20, "d23", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA21, "k22", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA22, "v21", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA23, "w21", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA3, "ac20", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA4, "ad21", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA5, "ac21", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA6, "d24", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA7, "e23", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA8, "e24", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA9, "f23", NULL),
-	_OMAP3_BALLENTRY(DSS_HSYNC, "e22", NULL),
-	_OMAP3_BALLENTRY(DSS_PCLK, "g22", NULL),
-	_OMAP3_BALLENTRY(DSS_VSYNC, "f22", NULL),
-	_OMAP3_BALLENTRY(ETK_CLK, "ac1", NULL),
-	_OMAP3_BALLENTRY(ETK_CTL, "ad3", NULL),
-	_OMAP3_BALLENTRY(ETK_D0, "ad6", NULL),
-	_OMAP3_BALLENTRY(ETK_D1, "ac6", NULL),
-	_OMAP3_BALLENTRY(ETK_D10, "ac3", NULL),
-	_OMAP3_BALLENTRY(ETK_D11, "ac9", NULL),
-	_OMAP3_BALLENTRY(ETK_D12, "ac10", NULL),
-	_OMAP3_BALLENTRY(ETK_D13, "ad11", NULL),
-	_OMAP3_BALLENTRY(ETK_D14, "ac11", NULL),
-	_OMAP3_BALLENTRY(ETK_D15, "ad12", NULL),
-	_OMAP3_BALLENTRY(ETK_D2, "ac7", NULL),
-	_OMAP3_BALLENTRY(ETK_D3, "ad8", NULL),
-	_OMAP3_BALLENTRY(ETK_D4, "ac5", NULL),
-	_OMAP3_BALLENTRY(ETK_D5, "ad2", NULL),
-	_OMAP3_BALLENTRY(ETK_D6, "ac8", NULL),
-	_OMAP3_BALLENTRY(ETK_D7, "ad9", NULL),
-	_OMAP3_BALLENTRY(ETK_D8, "ac4", NULL),
-	_OMAP3_BALLENTRY(ETK_D9, "ad5", NULL),
-	_OMAP3_BALLENTRY(GPMC_A1, "k4", NULL),
-	_OMAP3_BALLENTRY(GPMC_A10, "g2", NULL),
-	_OMAP3_BALLENTRY(GPMC_A2, "k3", NULL),
-	_OMAP3_BALLENTRY(GPMC_A3, "k2", NULL),
-	_OMAP3_BALLENTRY(GPMC_A4, "j4", NULL),
-	_OMAP3_BALLENTRY(GPMC_A5, "j3", NULL),
-	_OMAP3_BALLENTRY(GPMC_A6, "j2", NULL),
-	_OMAP3_BALLENTRY(GPMC_A7, "j1", NULL),
-	_OMAP3_BALLENTRY(GPMC_A8, "h1", NULL),
-	_OMAP3_BALLENTRY(GPMC_A9, "h2", NULL),
-	_OMAP3_BALLENTRY(GPMC_CLK, "w2", NULL),
-	_OMAP3_BALLENTRY(GPMC_D10, "u1", NULL),
-	_OMAP3_BALLENTRY(GPMC_D11, "r3", NULL),
-	_OMAP3_BALLENTRY(GPMC_D12, "t3", NULL),
-	_OMAP3_BALLENTRY(GPMC_D13, "u2", NULL),
-	_OMAP3_BALLENTRY(GPMC_D14, "v1", NULL),
-	_OMAP3_BALLENTRY(GPMC_D15, "v2", NULL),
-	_OMAP3_BALLENTRY(GPMC_D8, "r2", NULL),
-	_OMAP3_BALLENTRY(GPMC_D9, "t2", NULL),
-	_OMAP3_BALLENTRY(GPMC_NBE0_CLE, "k5", NULL),
-	_OMAP3_BALLENTRY(GPMC_NBE1, "l1", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS3, "d2", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS4, "f4", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS5, "g5", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS6, "f3", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS7, "g4", NULL),
-	_OMAP3_BALLENTRY(GPMC_NWP, "e1", NULL),
-	_OMAP3_BALLENTRY(GPMC_WAIT3, "c2", NULL),
-	_OMAP3_BALLENTRY(HDQ_SIO, "a24", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_CLK, "r21", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA0, "t24", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA1, "t23", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA2, "u24", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA3, "u23", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA4, "w24", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA5, "v23", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA6, "w23", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA7, "t22", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DIR, "p23", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_NXT, "r22", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_STP, "r23", NULL),
-	_OMAP3_BALLENTRY(I2C2_SCL, "ac15", NULL),
-	_OMAP3_BALLENTRY(I2C2_SDA, "ac14", NULL),
-	_OMAP3_BALLENTRY(I2C3_SCL, "ac13", NULL),
-	_OMAP3_BALLENTRY(I2C3_SDA, "ac12", NULL),
-	_OMAP3_BALLENTRY(I2C4_SCL, "y16", NULL),
-	_OMAP3_BALLENTRY(I2C4_SDA, "y15", NULL),
-	_OMAP3_BALLENTRY(JTAG_EMU0, "ac24", NULL),
-	_OMAP3_BALLENTRY(JTAG_EMU1, "ad24", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_CLKR, "w19", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_CLKX, "v18", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_DR, "y18", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_DX, "w18", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_FSR, "ab20", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_FSX, "aa19", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_CLKX, "t21", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_DR, "v19", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_DX, "r20", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_FSX, "v20", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_CLKX, "w4", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_DR, "v5", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_DX, "v6", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_FSX, "v4", NULL),
-	_OMAP3_BALLENTRY(MCBSP_CLKS, "aa18", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CLK, "t5", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS0, "t6", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS3, "r5", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_SIMO, "r4", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_SOMI, "t4", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_CLK, "n5", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_CS0, "m5", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_CS1, "m4", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_SIMO, "n4", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_SOMI, "n3", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_CLK, "m23", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_CMD, "l23", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT0, "m22", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT1, "m21", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT2, "m20", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT3, "n23", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT4, "n22", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT5, "n21", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT6, "n20", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT7, "p24", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_CLK, "y1", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_CMD, "ab5", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT0, "ab3", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT1, "y3", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT2, "w3", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT3, "v3", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT4, "ab2", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT5, "aa2", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT6, "y2", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT7, "aa1", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT0, "ab12", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT1, "ac16", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT2, "ad17", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT3, "ad18", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT4, "ac17", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT5, "ab16", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT6, "aa15", NULL),
-	_OMAP3_BALLENTRY(SYS_CLKOUT1, "y7", NULL),
-	_OMAP3_BALLENTRY(SYS_CLKOUT2, "aa6", NULL),
-	_OMAP3_BALLENTRY(SYS_CLKREQ, "y13", NULL),
-	_OMAP3_BALLENTRY(SYS_NIRQ, "w16", NULL),
-	_OMAP3_BALLENTRY(SYS_NRESWARM, "y10", NULL),
-	_OMAP3_BALLENTRY(SYS_OFF_MODE, "ad23", NULL),
-	_OMAP3_BALLENTRY(UART1_CTS, "ac2", NULL),
-	_OMAP3_BALLENTRY(UART1_RTS, "w6", NULL),
-	_OMAP3_BALLENTRY(UART1_RX, "v7", NULL),
-	_OMAP3_BALLENTRY(UART1_TX, "w7", NULL),
-	_OMAP3_BALLENTRY(UART3_CTS_RCTX, "a23", NULL),
-	_OMAP3_BALLENTRY(UART3_RTS_SD, "b23", NULL),
-	_OMAP3_BALLENTRY(UART3_RX_IRRX, "b24", NULL),
-	_OMAP3_BALLENTRY(UART3_TX_IRTX, "c23", NULL),
-	{ .reg_offset = OMAP_MUX_TERMINATOR },
-};
-#else
-#define omap3_cus_ball	 NULL
-#endif
-
-/*
- * Signals different on CBB package compared to superset
- */
-#if defined(CONFIG_OMAP_MUX) && defined(CONFIG_OMAP_PACKAGE_CBB)
-static struct omap_mux __initdata omap3_cbb_subset[] = {
-	_OMAP3_MUXENTRY(CAM_D10, 109,
-		"cam_d10", NULL, NULL, NULL,
-		"gpio_109", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D11, 110,
-		"cam_d11", NULL, NULL, NULL,
-		"gpio_110", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D2, 101,
-		"cam_d2", NULL, NULL, NULL,
-		"gpio_101", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D3, 102,
-		"cam_d3", NULL, NULL, NULL,
-		"gpio_102", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D4, 103,
-		"cam_d4", NULL, NULL, NULL,
-		"gpio_103", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D5, 104,
-		"cam_d5", NULL, NULL, NULL,
-		"gpio_104", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_FLD, 98,
-		"cam_fld", NULL, "cam_global_reset", NULL,
-		"gpio_98", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_HS, 94,
-		"cam_hs", NULL, NULL, NULL,
-		"gpio_94", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_PCLK, 97,
-		"cam_pclk", NULL, NULL, NULL,
-		"gpio_97", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_STROBE, 126,
-		"cam_strobe", NULL, NULL, NULL,
-		"gpio_126", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_VS, 95,
-		"cam_vs", NULL, NULL, NULL,
-		"gpio_95", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_WEN, 167,
-		"cam_wen", NULL, "cam_shutter", NULL,
-		"gpio_167", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA6, 76,
-		"dss_data6", NULL, "uart1_tx", NULL,
-		"gpio_76", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA7, 77,
-		"dss_data7", NULL, "uart1_rx", NULL,
-		"gpio_77", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA8, 78,
-		"dss_data8", NULL, NULL, NULL,
-		"gpio_78", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA9, 79,
-		"dss_data9", NULL, NULL, NULL,
-		"gpio_79", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_HSYNC, 67,
-		"dss_hsync", NULL, NULL, NULL,
-		"gpio_67", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_PCLK, 66,
-		"dss_pclk", NULL, NULL, NULL,
-		"gpio_66", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(ETK_CLK, 12,
-		"etk_clk", "mcbsp5_clkx", "sdmmc3_clk", "hsusb1_stp",
-		"gpio_12", "mm1_rxdp", "hsusb1_tll_stp", NULL),
-	_OMAP3_MUXENTRY(ETK_CTL, 13,
-		"etk_ctl", NULL, "sdmmc3_cmd", "hsusb1_clk",
-		"gpio_13", NULL, "hsusb1_tll_clk", NULL),
-	_OMAP3_MUXENTRY(ETK_D0, 14,
-		"etk_d0", "mcspi3_simo", "sdmmc3_dat4", "hsusb1_data0",
-		"gpio_14", "mm1_rxrcv", "hsusb1_tll_data0", NULL),
-	_OMAP3_MUXENTRY(ETK_D1, 15,
-		"etk_d1", "mcspi3_somi", NULL, "hsusb1_data1",
-		"gpio_15", "mm1_txse0", "hsusb1_tll_data1", NULL),
-	_OMAP3_MUXENTRY(ETK_D10, 24,
-		"etk_d10", NULL, "uart1_rx", "hsusb2_clk",
-		"gpio_24", NULL, "hsusb2_tll_clk", NULL),
-	_OMAP3_MUXENTRY(ETK_D11, 25,
-		"etk_d11", NULL, NULL, "hsusb2_stp",
-		"gpio_25", "mm2_rxdp", "hsusb2_tll_stp", NULL),
-	_OMAP3_MUXENTRY(ETK_D12, 26,
-		"etk_d12", NULL, NULL, "hsusb2_dir",
-		"gpio_26", NULL, "hsusb2_tll_dir", NULL),
-	_OMAP3_MUXENTRY(ETK_D13, 27,
-		"etk_d13", NULL, NULL, "hsusb2_nxt",
-		"gpio_27", "mm2_rxdm", "hsusb2_tll_nxt", NULL),
-	_OMAP3_MUXENTRY(ETK_D14, 28,
-		"etk_d14", NULL, NULL, "hsusb2_data0",
-		"gpio_28", "mm2_rxrcv", "hsusb2_tll_data0", NULL),
-	_OMAP3_MUXENTRY(ETK_D15, 29,
-		"etk_d15", NULL, NULL, "hsusb2_data1",
-		"gpio_29", "mm2_txse0", "hsusb2_tll_data1", NULL),
-	_OMAP3_MUXENTRY(ETK_D2, 16,
-		"etk_d2", "mcspi3_cs0", NULL, "hsusb1_data2",
-		"gpio_16", "mm1_txdat", "hsusb1_tll_data2", NULL),
-	_OMAP3_MUXENTRY(ETK_D3, 17,
-		"etk_d3", "mcspi3_clk", "sdmmc3_dat3", "hsusb1_data7",
-		"gpio_17", NULL, "hsusb1_tll_data7", NULL),
-	_OMAP3_MUXENTRY(ETK_D4, 18,
-		"etk_d4", "mcbsp5_dr", "sdmmc3_dat0", "hsusb1_data4",
-		"gpio_18", NULL, "hsusb1_tll_data4", NULL),
-	_OMAP3_MUXENTRY(ETK_D5, 19,
-		"etk_d5", "mcbsp5_fsx", "sdmmc3_dat1", "hsusb1_data5",
-		"gpio_19", NULL, "hsusb1_tll_data5", NULL),
-	_OMAP3_MUXENTRY(ETK_D6, 20,
-		"etk_d6", "mcbsp5_dx", "sdmmc3_dat2", "hsusb1_data6",
-		"gpio_20", NULL, "hsusb1_tll_data6", NULL),
-	_OMAP3_MUXENTRY(ETK_D7, 21,
-		"etk_d7", "mcspi3_cs1", "sdmmc3_dat7", "hsusb1_data3",
-		"gpio_21", "mm1_txen_n", "hsusb1_tll_data3", NULL),
-	_OMAP3_MUXENTRY(ETK_D8, 22,
-		"etk_d8", "sys_drm_msecure", "sdmmc3_dat6", "hsusb1_dir",
-		"gpio_22", NULL, "hsusb1_tll_dir", NULL),
-	_OMAP3_MUXENTRY(ETK_D9, 23,
-		"etk_d9", "sys_secure_indicator", "sdmmc3_dat5", "hsusb1_nxt",
-		"gpio_23", "mm1_rxdm", "hsusb1_tll_nxt", NULL),
-	{ .reg_offset = OMAP_MUX_TERMINATOR },
-};
-#else
-#define omap3_cbb_subset	NULL
-#endif
-
-/*
- * Balls for CBB package
- * 515-pin s-PBGA Package, 0.50mm Ball Pitch (Top), 0.40mm Ball Pitch (Bottom)
- */
-#if defined(CONFIG_OMAP_MUX) && defined(CONFIG_DEBUG_FS)		\
-		&& defined(CONFIG_OMAP_PACKAGE_CBB)
-static struct omap_ball __initdata omap3_cbb_ball[] = {
-	_OMAP3_BALLENTRY(CAM_D0, "ag17", NULL),
-	_OMAP3_BALLENTRY(CAM_D1, "ah17", NULL),
-	_OMAP3_BALLENTRY(CAM_D10, "b25", NULL),
-	_OMAP3_BALLENTRY(CAM_D11, "c26", NULL),
-	_OMAP3_BALLENTRY(CAM_D2, "b24", NULL),
-	_OMAP3_BALLENTRY(CAM_D3, "c24", NULL),
-	_OMAP3_BALLENTRY(CAM_D4, "d24", NULL),
-	_OMAP3_BALLENTRY(CAM_D5, "a25", NULL),
-	_OMAP3_BALLENTRY(CAM_D6, "k28", NULL),
-	_OMAP3_BALLENTRY(CAM_D7, "l28", NULL),
-	_OMAP3_BALLENTRY(CAM_D8, "k27", NULL),
-	_OMAP3_BALLENTRY(CAM_D9, "l27", NULL),
-	_OMAP3_BALLENTRY(CAM_FLD, "c23", NULL),
-	_OMAP3_BALLENTRY(CAM_HS, "a24", NULL),
-	_OMAP3_BALLENTRY(CAM_PCLK, "c27", NULL),
-	_OMAP3_BALLENTRY(CAM_STROBE, "d25", NULL),
-	_OMAP3_BALLENTRY(CAM_VS, "a23", NULL),
-	_OMAP3_BALLENTRY(CAM_WEN, "b23", NULL),
-	_OMAP3_BALLENTRY(CAM_XCLKA, "c25", NULL),
-	_OMAP3_BALLENTRY(CAM_XCLKB, "b26", NULL),
-	_OMAP3_BALLENTRY(CSI2_DX0, "ag19", NULL),
-	_OMAP3_BALLENTRY(CSI2_DX1, "ag18", NULL),
-	_OMAP3_BALLENTRY(CSI2_DY0, "ah19", NULL),
-	_OMAP3_BALLENTRY(CSI2_DY1, "ah18", NULL),
-	_OMAP3_BALLENTRY(DSS_ACBIAS, "e27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA0, "ag22", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA1, "ah22", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA10, "ad28", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA11, "ad27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA12, "ab28", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA13, "ab27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA14, "aa28", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA15, "aa27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA16, "g25", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA17, "h27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA18, "h26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA19, "h25", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA2, "ag23", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA20, "e28", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA21, "j26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA22, "ac27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA23, "ac28", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA3, "ah23", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA4, "ag24", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA5, "ah24", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA6, "e26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA7, "f28", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA8, "f27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA9, "g26", NULL),
-	_OMAP3_BALLENTRY(DSS_HSYNC, "d26", NULL),
-	_OMAP3_BALLENTRY(DSS_PCLK, "d28", NULL),
-	_OMAP3_BALLENTRY(DSS_VSYNC, "d27", NULL),
-	_OMAP3_BALLENTRY(ETK_CLK, "af10", NULL),
-	_OMAP3_BALLENTRY(ETK_CTL, "ae10", NULL),
-	_OMAP3_BALLENTRY(ETK_D0, "af11", NULL),
-	_OMAP3_BALLENTRY(ETK_D1, "ag12", NULL),
-	_OMAP3_BALLENTRY(ETK_D10, "ae7", NULL),
-	_OMAP3_BALLENTRY(ETK_D11, "af7", NULL),
-	_OMAP3_BALLENTRY(ETK_D12, "ag7", NULL),
-	_OMAP3_BALLENTRY(ETK_D13, "ah7", NULL),
-	_OMAP3_BALLENTRY(ETK_D14, "ag8", NULL),
-	_OMAP3_BALLENTRY(ETK_D15, "ah8", NULL),
-	_OMAP3_BALLENTRY(ETK_D2, "ah12", NULL),
-	_OMAP3_BALLENTRY(ETK_D3, "ae13", NULL),
-	_OMAP3_BALLENTRY(ETK_D4, "ae11", NULL),
-	_OMAP3_BALLENTRY(ETK_D5, "ah9", NULL),
-	_OMAP3_BALLENTRY(ETK_D6, "af13", NULL),
-	_OMAP3_BALLENTRY(ETK_D7, "ah14", NULL),
-	_OMAP3_BALLENTRY(ETK_D8, "af9", NULL),
-	_OMAP3_BALLENTRY(ETK_D9, "ag9", NULL),
-	_OMAP3_BALLENTRY(GPMC_A1, "n4", "ac15"),
-	_OMAP3_BALLENTRY(GPMC_A10, "k3", "ab19"),
-	_OMAP3_BALLENTRY(GPMC_A2, "m4", "ab15"),
-	_OMAP3_BALLENTRY(GPMC_A3, "l4", "ac16"),
-	_OMAP3_BALLENTRY(GPMC_A4, "k4", "ab16"),
-	_OMAP3_BALLENTRY(GPMC_A5, "t3", "ac17"),
-	_OMAP3_BALLENTRY(GPMC_A6, "r3", "ab17"),
-	_OMAP3_BALLENTRY(GPMC_A7, "n3", "ac18"),
-	_OMAP3_BALLENTRY(GPMC_A8, "m3", "ab18"),
-	_OMAP3_BALLENTRY(GPMC_A9, "l3", "ac19"),
-	_OMAP3_BALLENTRY(GPMC_CLK, "t4", "w2"),
-	_OMAP3_BALLENTRY(GPMC_D10, "p1", "ab4"),
-	_OMAP3_BALLENTRY(GPMC_D11, "r1", "ac4"),
-	_OMAP3_BALLENTRY(GPMC_D12, "r2", "ab6"),
-	_OMAP3_BALLENTRY(GPMC_D13, "t2", "ac6"),
-	_OMAP3_BALLENTRY(GPMC_D14, "w1", "ab7"),
-	_OMAP3_BALLENTRY(GPMC_D15, "y1", "ac7"),
-	_OMAP3_BALLENTRY(GPMC_D8, "h2", "ab3"),
-	_OMAP3_BALLENTRY(GPMC_D9, "k2", "ac3"),
-	_OMAP3_BALLENTRY(GPMC_NBE0_CLE, "g3", "ac12"),
-	_OMAP3_BALLENTRY(GPMC_NBE1, "u3", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS1, "h3", "y1"),
-	_OMAP3_BALLENTRY(GPMC_NCS2, "v8", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS3, "u8", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS4, "t8", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS5, "r8", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS6, "p8", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS7, "n8", NULL),
-	_OMAP3_BALLENTRY(GPMC_NWP, "h1", "ab10"),
-	_OMAP3_BALLENTRY(GPMC_WAIT1, "l8", "ac10"),
-	_OMAP3_BALLENTRY(GPMC_WAIT2, "k8", NULL),
-	_OMAP3_BALLENTRY(GPMC_WAIT3, "j8", NULL),
-	_OMAP3_BALLENTRY(HDQ_SIO, "j25", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_CLK, "t28", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA0, "t27", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA1, "u28", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA2, "u27", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA3, "u26", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA4, "u25", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA5, "v28", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA6, "v27", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA7, "v26", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DIR, "r28", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_NXT, "t26", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_STP, "t25", NULL),
-	_OMAP3_BALLENTRY(I2C2_SCL, "af15", NULL),
-	_OMAP3_BALLENTRY(I2C2_SDA, "ae15", NULL),
-	_OMAP3_BALLENTRY(I2C3_SCL, "af14", NULL),
-	_OMAP3_BALLENTRY(I2C3_SDA, "ag14", NULL),
-	_OMAP3_BALLENTRY(I2C4_SCL, "ad26", NULL),
-	_OMAP3_BALLENTRY(I2C4_SDA, "ae26", NULL),
-	_OMAP3_BALLENTRY(JTAG_EMU0, "aa11", NULL),
-	_OMAP3_BALLENTRY(JTAG_EMU1, "aa10", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_CLKR, "y21", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_CLKX, "w21", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_DR, "u21", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_DX, "v21", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_FSR, "aa21", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_FSX, "k26", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_CLKX, "n21", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_DR, "r21", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_DX, "m21", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_FSX, "p21", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_CLKX, "af5", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_DR, "ae6", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_DX, "af6", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_FSX, "ae5", NULL),
-	_OMAP3_BALLENTRY(MCBSP4_CLKX, "ae1", NULL),
-	_OMAP3_BALLENTRY(MCBSP4_DR, "ad1", NULL),
-	_OMAP3_BALLENTRY(MCBSP4_DX, "ad2", NULL),
-	_OMAP3_BALLENTRY(MCBSP4_FSX, "ac1", NULL),
-	_OMAP3_BALLENTRY(MCBSP_CLKS, "t21", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CLK, "ab3", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS0, "ac2", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS1, "ac3", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS2, "ab1", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS3, "ab2", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_SIMO, "ab4", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_SOMI, "aa4", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_CLK, "aa3", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_CS0, "y4", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_CS1, "v3", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_SIMO, "y2", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_SOMI, "y3", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_CLK, "n28", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_CMD, "m27", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT0, "n27", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT1, "n26", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT2, "n25", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT3, "p28", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT4, "p27", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT5, "p26", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT6, "r27", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT7, "r25", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_CLK, "ae2", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_CMD, "ag5", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT0, "ah5", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT1, "ah4", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT2, "ag4", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT3, "af4", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT4, "ae4", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT5, "ah3", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT6, "af3", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT7, "ae3", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT0, "ah26", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT1, "ag26", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT2, "ae14", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT3, "af18", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT4, "af19", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT5, "ae21", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT6, "af21", NULL),
-	_OMAP3_BALLENTRY(SYS_CLKOUT1, "ag25", NULL),
-	_OMAP3_BALLENTRY(SYS_CLKOUT2, "ae22", NULL),
-	_OMAP3_BALLENTRY(SYS_CLKREQ, "af25", NULL),
-	_OMAP3_BALLENTRY(SYS_NIRQ, "af26", NULL),
-	_OMAP3_BALLENTRY(SYS_NRESWARM, "af24", NULL),
-	_OMAP3_BALLENTRY(SYS_OFF_MODE, "af22", NULL),
-	_OMAP3_BALLENTRY(UART1_CTS, "w8", NULL),
-	_OMAP3_BALLENTRY(UART1_RTS, "aa9", NULL),
-	_OMAP3_BALLENTRY(UART1_RX, "y8", NULL),
-	_OMAP3_BALLENTRY(UART1_TX, "aa8", NULL),
-	_OMAP3_BALLENTRY(UART2_CTS, "ab26", NULL),
-	_OMAP3_BALLENTRY(UART2_RTS, "ab25", NULL),
-	_OMAP3_BALLENTRY(UART2_RX, "ad25", NULL),
-	_OMAP3_BALLENTRY(UART2_TX, "aa25", NULL),
-	_OMAP3_BALLENTRY(UART3_CTS_RCTX, "h18", NULL),
-	_OMAP3_BALLENTRY(UART3_RTS_SD, "h19", NULL),
-	_OMAP3_BALLENTRY(UART3_RX_IRRX, "h20", NULL),
-	_OMAP3_BALLENTRY(UART3_TX_IRTX, "h21", NULL),
-	{ .reg_offset = OMAP_MUX_TERMINATOR },
-};
-#else
-#define omap3_cbb_ball	 NULL
-#endif
-
-/*
- * Signals different on 36XX CBP package compared to 34XX CBC package
- */
-#if defined(CONFIG_OMAP_MUX) && defined(CONFIG_OMAP_PACKAGE_CBP)
-static struct omap_mux __initdata omap36xx_cbp_subset[] = {
-	_OMAP3_MUXENTRY(CAM_D0, 99,
-		"cam_d0", NULL, "csi2_dx2", NULL,
-		"gpio_99", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D1, 100,
-		"cam_d1", NULL, "csi2_dy2", NULL,
-		"gpio_100", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D10, 109,
-		"cam_d10", "ssi2_wake", NULL, NULL,
-		"gpio_109", "hw_dbg8", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D2, 101,
-		"cam_d2", "ssi2_rdy_tx", NULL, NULL,
-		"gpio_101", "hw_dbg4", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D3, 102,
-		"cam_d3", "ssi2_dat_rx", NULL, NULL,
-		"gpio_102", "hw_dbg5", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D4, 103,
-		"cam_d4", "ssi2_flag_rx", NULL, NULL,
-		"gpio_103", "hw_dbg6", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_D5, 104,
-		"cam_d5", "ssi2_rdy_rx", NULL, NULL,
-		"gpio_104", "hw_dbg7", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_HS, 94,
-		"cam_hs", "ssi2_dat_tx", NULL, NULL,
-		"gpio_94", "hw_dbg0", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(CAM_VS, 95,
-		"cam_vs", "ssi2_flag_tx", NULL, NULL,
-		"gpio_95", "hw_dbg1", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA0, 70,
-		"dss_data0", "dsi_dx0", "uart1_cts", NULL,
-		"gpio_70", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA1, 71,
-		"dss_data1", "dsi_dy0", "uart1_rts", NULL,
-		"gpio_71", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA2, 72,
-		"dss_data2", "dsi_dx1", NULL, NULL,
-		"gpio_72", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA3, 73,
-		"dss_data3", "dsi_dy1", NULL, NULL,
-		"gpio_73", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA4, 74,
-		"dss_data4", "dsi_dx2", "uart3_rx_irrx", NULL,
-		"gpio_74", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA5, 75,
-		"dss_data5", "dsi_dy2", "uart3_tx_irtx", NULL,
-		"gpio_75", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA6, 76,
-		"dss_data6", NULL, "uart1_tx", "dssvenc656_data6",
-		"gpio_76", "hw_dbg14", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA7, 77,
-		"dss_data7", NULL, "uart1_rx", "dssvenc656_data7",
-		"gpio_77", "hw_dbg15", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA8, 78,
-		"dss_data8", NULL, "uart3_rx_irrx", NULL,
-		"gpio_78", "hw_dbg16", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(DSS_DATA9, 79,
-		"dss_data9", NULL, "uart3_tx_irtx", NULL,
-		"gpio_79", "hw_dbg17", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(ETK_D12, 26,
-		"etk_d12", "sys_drm_msecure", NULL, "hsusb2_dir",
-		"gpio_26", NULL, "hsusb2_tll_dir", "hw_dbg14"),
-	_OMAP3_MUXENTRY(GPMC_A11, 0,
-		"gpmc_a11", NULL, NULL, NULL,
-		NULL, NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_WAIT2, 64,
-		"gpmc_wait2", NULL, "uart4_tx", NULL,
-		"gpio_64", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(GPMC_WAIT3, 65,
-		"gpmc_wait3", "sys_ndmareq1", "uart4_rx", NULL,
-		"gpio_65", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DATA0, 125,
-		"hsusb0_data0", NULL, "uart3_tx_irtx", NULL,
-		"gpio_125", "uart2_tx", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DATA1, 130,
-		"hsusb0_data1", NULL, "uart3_rx_irrx", NULL,
-		"gpio_130", "uart2_rx", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DATA2, 131,
-		"hsusb0_data2", NULL, "uart3_rts_sd", NULL,
-		"gpio_131", "uart2_rts", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(HSUSB0_DATA3, 169,
-		"hsusb0_data3", NULL, "uart3_cts_rctx", NULL,
-		"gpio_169", "uart2_cts", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP1_CLKR, 156,
-		"mcbsp1_clkr", "mcspi4_clk", "sim_cd", NULL,
-		"gpio_156", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP1_FSR, 157,
-		"mcbsp1_fsr", "adpllv2d_dithering_en1",
-		"cam_global_reset", NULL,
-		"gpio_157", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP4_CLKX, 152,
-		"mcbsp4_clkx", "ssi1_dat_rx", NULL, NULL,
-		"gpio_152", "hsusb3_tll_data1", "mm3_txse0", "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP4_DR, 153,
-		"mcbsp4_dr", "ssi1_flag_rx", NULL, NULL,
-		"gpio_153", "hsusb3_tll_data0", "mm3_rxrcv", "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP4_DX, 154,
-		"mcbsp4_dx", "ssi1_rdy_rx", NULL, NULL,
-		"gpio_154", "hsusb3_tll_data2", "mm3_txdat", "safe_mode"),
-	_OMAP3_MUXENTRY(MCBSP4_FSX, 155,
-		"mcbsp4_fsx", "ssi1_wake", NULL, NULL,
-		"gpio_155", "hsusb3_tll_data3", "mm3_txen_n", "safe_mode"),
-	_OMAP3_MUXENTRY(MCSPI1_CS1, 175,
-		"mcspi1_cs1", "adpllv2d_dithering_en2", NULL, "sdmmc3_cmd",
-		"gpio_175", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SAD2D_MBUSFLAG, 0,
-		"sad2d_mbusflag", "mad2d_sbusflag", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD28, 0,
-		"sad2d_mcad28", "mad2d_mcad28", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD29, 0,
-		"sad2d_mcad29", "mad2d_mcad29", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD32, 0,
-		"sad2d_mcad32", "mad2d_mcad32", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD33, 0,
-		"sad2d_mcad33", "mad2d_mcad33", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD34, 0,
-		"sad2d_mcad34", "mad2d_mcad34", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD35, 0,
-		"sad2d_mcad35", "mad2d_mcad35", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MCAD36, 0,
-		"sad2d_mcad36", "mad2d_mcad36", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MREAD, 0,
-		"sad2d_mread", "mad2d_sread", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_MWRITE, 0,
-		"sad2d_mwrite", "mad2d_swrite", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_SBUSFLAG, 0,
-		"sad2d_sbusflag", "mad2d_mbusflag", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_SREAD, 0,
-		"sad2d_sread", "mad2d_mread", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SAD2D_SWRITE, 0,
-		"sad2d_swrite", "mad2d_mwrite", NULL, NULL,
-		NULL, NULL, NULL, NULL),
-	_OMAP3_MUXENTRY(SDMMC1_CLK, 120,
-		"sdmmc1_clk", "ms_clk", NULL, NULL,
-		"gpio_120", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_CMD, 121,
-		"sdmmc1_cmd", "ms_bs", NULL, NULL,
-		"gpio_121", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_DAT0, 122,
-		"sdmmc1_dat0", "ms_dat0", NULL, NULL,
-		"gpio_122", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_DAT1, 123,
-		"sdmmc1_dat1", "ms_dat1", NULL, NULL,
-		"gpio_123", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_DAT2, 124,
-		"sdmmc1_dat2", "ms_dat2", NULL, NULL,
-		"gpio_124", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDMMC1_DAT3, 125,
-		"sdmmc1_dat3", "ms_dat3", NULL, NULL,
-		"gpio_125", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SDRC_CKE0, 0,
-		"sdrc_cke0", NULL, NULL, NULL,
-		NULL, NULL, NULL, "safe_mode_out1"),
-	_OMAP3_MUXENTRY(SDRC_CKE1, 0,
-		"sdrc_cke1", NULL, NULL, NULL,
-		NULL, NULL, NULL, "safe_mode_out1"),
-	_OMAP3_MUXENTRY(SIM_IO, 126,
-		"sim_io", "sim_io_low_impedance", NULL, NULL,
-		"gpio_126", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SIM_CLK, 127,
-		"sim_clk", NULL, NULL, NULL,
-		"gpio_127", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SIM_PWRCTRL, 128,
-		"sim_pwrctrl", NULL, NULL, NULL,
-		"gpio_128", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SIM_RST, 129,
-		"sim_rst", NULL, NULL, NULL,
-		"gpio_129", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT0, 2,
-		"sys_boot0", NULL, NULL, "dss_data18",
-		"gpio_2", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT1, 3,
-		"sys_boot1", NULL, NULL, "dss_data19",
-		"gpio_3", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT3, 5,
-		"sys_boot3", NULL, NULL, "dss_data20",
-		"gpio_5", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT4, 6,
-		"sys_boot4", "sdmmc2_dir_dat2", NULL, "dss_data21",
-		"gpio_6", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT5, 7,
-		"sys_boot5", "sdmmc2_dir_dat3", NULL, "dss_data22",
-		"gpio_7", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(SYS_BOOT6, 8,
-		"sys_boot6", NULL, NULL, "dss_data23",
-		"gpio_8", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART1_CTS, 150,
-		"uart1_cts", "ssi1_rdy_tx", NULL, NULL,
-		"gpio_150", "hsusb3_tll_clk", NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART1_RTS, 149,
-		"uart1_rts", "ssi1_flag_tx", NULL, NULL,
-		"gpio_149", NULL, NULL, "safe_mode"),
-	_OMAP3_MUXENTRY(UART1_TX, 148,
-		"uart1_tx", "ssi1_dat_tx", NULL, NULL,
-		"gpio_148", NULL, NULL, "safe_mode"),
-	{ .reg_offset = OMAP_MUX_TERMINATOR },
-};
-#else
-#define omap36xx_cbp_subset	NULL
-#endif
-
-/*
- * Balls for 36XX CBP package
- * 515-pin s-PBGA Package, 0.50mm Ball Pitch (Top), 0.40mm Ball Pitch (Bottom)
- */
-#if defined(CONFIG_OMAP_MUX) && defined(CONFIG_DEBUG_FS)		\
-		&& defined (CONFIG_OMAP_PACKAGE_CBP)
-static struct omap_ball __initdata omap36xx_cbp_ball[] = {
-	_OMAP3_BALLENTRY(CAM_D0, "ag17", NULL),
-	_OMAP3_BALLENTRY(CAM_D1, "ah17", NULL),
-	_OMAP3_BALLENTRY(CAM_D10, "b25", NULL),
-	_OMAP3_BALLENTRY(CAM_D11, "c26", NULL),
-	_OMAP3_BALLENTRY(CAM_D2, "b24", NULL),
-	_OMAP3_BALLENTRY(CAM_D3, "c24", NULL),
-	_OMAP3_BALLENTRY(CAM_D4, "d24", NULL),
-	_OMAP3_BALLENTRY(CAM_D5, "a25", NULL),
-	_OMAP3_BALLENTRY(CAM_D6, "k28", NULL),
-	_OMAP3_BALLENTRY(CAM_D7, "l28", NULL),
-	_OMAP3_BALLENTRY(CAM_D8, "k27", NULL),
-	_OMAP3_BALLENTRY(CAM_D9, "l27", NULL),
-	_OMAP3_BALLENTRY(CAM_FLD, "c23", NULL),
-	_OMAP3_BALLENTRY(CAM_HS, "a24", NULL),
-	_OMAP3_BALLENTRY(CAM_PCLK, "c27", NULL),
-	_OMAP3_BALLENTRY(CAM_STROBE, "d25", NULL),
-	_OMAP3_BALLENTRY(CAM_VS, "a23", NULL),
-	_OMAP3_BALLENTRY(CAM_WEN, "b23", NULL),
-	_OMAP3_BALLENTRY(CAM_XCLKA, "c25", NULL),
-	_OMAP3_BALLENTRY(CAM_XCLKB, "b26", NULL),
-	_OMAP3_BALLENTRY(CSI2_DX0, "ag19", NULL),
-	_OMAP3_BALLENTRY(CSI2_DX1, "ag18", NULL),
-	_OMAP3_BALLENTRY(CSI2_DY0, "ah19", NULL),
-	_OMAP3_BALLENTRY(CSI2_DY1, "ah18", NULL),
-	_OMAP3_BALLENTRY(DSS_ACBIAS, "e27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA0, "ag22", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA1, "ah22", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA10, "ad28", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA11, "ad27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA12, "ab28", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA13, "ab27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA14, "aa28", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA15, "aa27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA16, "g25", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA17, "h27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA18, "h26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA19, "h25", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA2, "ag23", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA20, "e28", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA21, "j26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA22, "ac27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA23, "ac28", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA3, "ah23", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA4, "ag24", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA5, "ah24", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA6, "e26", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA7, "f28", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA8, "f27", NULL),
-	_OMAP3_BALLENTRY(DSS_DATA9, "g26", NULL),
-	_OMAP3_BALLENTRY(DSS_HSYNC, "d26", NULL),
-	_OMAP3_BALLENTRY(DSS_PCLK, "d28", NULL),
-	_OMAP3_BALLENTRY(DSS_VSYNC, "d27", NULL),
-	_OMAP3_BALLENTRY(ETK_CLK, "af10", NULL),
-	_OMAP3_BALLENTRY(ETK_CTL, "ae10", NULL),
-	_OMAP3_BALLENTRY(ETK_D0, "af11", NULL),
-	_OMAP3_BALLENTRY(ETK_D1, "ag12", NULL),
-	_OMAP3_BALLENTRY(ETK_D10, "ae7", NULL),
-	_OMAP3_BALLENTRY(ETK_D11, "af7", NULL),
-	_OMAP3_BALLENTRY(ETK_D12, "ag7", NULL),
-	_OMAP3_BALLENTRY(ETK_D13, "ah7", NULL),
-	_OMAP3_BALLENTRY(ETK_D14, "ag8", NULL),
-	_OMAP3_BALLENTRY(ETK_D15, "ah8", NULL),
-	_OMAP3_BALLENTRY(ETK_D2, "ah12", NULL),
-	_OMAP3_BALLENTRY(ETK_D3, "ae13", NULL),
-	_OMAP3_BALLENTRY(ETK_D4, "ae11", NULL),
-	_OMAP3_BALLENTRY(ETK_D5, "ah9", NULL),
-	_OMAP3_BALLENTRY(ETK_D6, "af13", NULL),
-	_OMAP3_BALLENTRY(ETK_D7, "ah14", NULL),
-	_OMAP3_BALLENTRY(ETK_D8, "af9", NULL),
-	_OMAP3_BALLENTRY(ETK_D9, "ag9", NULL),
-	_OMAP3_BALLENTRY(GPMC_A1, "n4", "ac15"),
-	_OMAP3_BALLENTRY(GPMC_A10, "k3", "ab19"),
-	_OMAP3_BALLENTRY(GPMC_A11, NULL, "ac20"),
-	_OMAP3_BALLENTRY(GPMC_A2, "m4", "ab15"),
-	_OMAP3_BALLENTRY(GPMC_A3, "l4", "ac16"),
-	_OMAP3_BALLENTRY(GPMC_A4, "k4", "ab16"),
-	_OMAP3_BALLENTRY(GPMC_A5, "t3", "ac17"),
-	_OMAP3_BALLENTRY(GPMC_A6, "r3", "ab17"),
-	_OMAP3_BALLENTRY(GPMC_A7, "n3", "ac18"),
-	_OMAP3_BALLENTRY(GPMC_A8, "m3", "ab18"),
-	_OMAP3_BALLENTRY(GPMC_A9, "l3", "ac19"),
-	_OMAP3_BALLENTRY(GPMC_CLK, "t4", "w2"),
-	_OMAP3_BALLENTRY(GPMC_D10, "p1", "ab4"),
-	_OMAP3_BALLENTRY(GPMC_D11, "r1", "ac4"),
-	_OMAP3_BALLENTRY(GPMC_D12, "r2", "ab6"),
-	_OMAP3_BALLENTRY(GPMC_D13, "t2", "ac6"),
-	_OMAP3_BALLENTRY(GPMC_D14, "w1", "ab7"),
-	_OMAP3_BALLENTRY(GPMC_D15, "y1", "ac7"),
-	_OMAP3_BALLENTRY(GPMC_D9, "k2", "ac3"),
-	_OMAP3_BALLENTRY(GPMC_NBE0_CLE, "g3", "ac12"),
-	_OMAP3_BALLENTRY(GPMC_NBE1, "u3", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS1, "h3", "y1"),
-	_OMAP3_BALLENTRY(GPMC_NCS2, "v8", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS3, "u8", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS4, "t8", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS5, "r8", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS6, "p8", NULL),
-	_OMAP3_BALLENTRY(GPMC_NCS7, "n8", NULL),
-	_OMAP3_BALLENTRY(GPMC_NWP, "h1", "ab10"),
-	_OMAP3_BALLENTRY(GPMC_WAIT1, "l8", "ac10"),
-	_OMAP3_BALLENTRY(GPMC_WAIT2, "k8", NULL),
-	_OMAP3_BALLENTRY(GPMC_WAIT3, "j8", NULL),
-	_OMAP3_BALLENTRY(HDQ_SIO, "j25", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_CLK, "t28", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA0, "t27", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA1, "u28", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA2, "u27", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA3, "u26", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA4, "u25", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA5, "v28", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA6, "v27", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DATA7, "v26", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_DIR, "r28", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_NXT, "t26", NULL),
-	_OMAP3_BALLENTRY(HSUSB0_STP, "t25", NULL),
-	_OMAP3_BALLENTRY(I2C2_SCL, "af15", NULL),
-	_OMAP3_BALLENTRY(I2C2_SDA, "ae15", NULL),
-	_OMAP3_BALLENTRY(I2C3_SCL, "af14", NULL),
-	_OMAP3_BALLENTRY(I2C3_SDA, "ag14", NULL),
-	_OMAP3_BALLENTRY(I2C4_SCL, "ad26", NULL),
-	_OMAP3_BALLENTRY(I2C4_SDA, "ae26", NULL),
-	_OMAP3_BALLENTRY(JTAG_EMU0, "aa11", NULL),
-	_OMAP3_BALLENTRY(JTAG_EMU1, "aa10", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_CLKR, "y21", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_CLKX, "w21", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_DR, "u21", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_DX, "v21", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_FSR, "aa21", NULL),
-	_OMAP3_BALLENTRY(MCBSP1_FSX, "k26", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_CLKX, "n21", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_DR, "r21", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_DX, "m21", NULL),
-	_OMAP3_BALLENTRY(MCBSP2_FSX, "p21", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_CLKX, "af5", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_DR, "ae6", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_DX, "af6", NULL),
-	_OMAP3_BALLENTRY(MCBSP3_FSX, "ae5", NULL),
-	_OMAP3_BALLENTRY(MCBSP4_CLKX, "ae1", NULL),
-	_OMAP3_BALLENTRY(MCBSP4_DR, "ad1", NULL),
-	_OMAP3_BALLENTRY(MCBSP4_DX, "ad2", NULL),
-	_OMAP3_BALLENTRY(MCBSP4_FSX, "ac1", NULL),
-	_OMAP3_BALLENTRY(MCBSP_CLKS, "t21", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CLK, "ab3", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS0, "ac2", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS1, "ac3", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS2, "ab1", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_CS3, "ab2", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_SIMO, "ab4", NULL),
-	_OMAP3_BALLENTRY(MCSPI1_SOMI, "aa4", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_CLK, "aa3", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_CS0, "y4", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_CS1, "v3", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_SIMO, "y2", NULL),
-	_OMAP3_BALLENTRY(MCSPI2_SOMI, "y3", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_CLK, "n28", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_CMD, "m27", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT0, "n27", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT1, "n26", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT2, "n25", NULL),
-	_OMAP3_BALLENTRY(SDMMC1_DAT3, "p28", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_CLK, "ae2", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_CMD, "ag5", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT0, "ah5", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT1, "ah4", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT2, "ag4", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT3, "af4", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT4, "ae4", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT5, "ah3", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT6, "af3", NULL),
-	_OMAP3_BALLENTRY(SDMMC2_DAT7, "ae3", NULL),
-	_OMAP3_BALLENTRY(SDRC_CKE0, "h16", "j22"),
-	_OMAP3_BALLENTRY(SDRC_CKE1, "h17", "j23"),
-	_OMAP3_BALLENTRY(SIM_CLK, "p26", NULL),
-	_OMAP3_BALLENTRY(SIM_IO, "p27", NULL),
-	_OMAP3_BALLENTRY(SIM_PWRCTRL, "r27", NULL),
-	_OMAP3_BALLENTRY(SIM_RST, "r25", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT0, "ah26", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT1, "ag26", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT2, "ae14", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT3, "af18", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT4, "af19", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT5, "ae21", NULL),
-	_OMAP3_BALLENTRY(SYS_BOOT6, "af21", NULL),
-	_OMAP3_BALLENTRY(SYS_CLKOUT1, "ag25", NULL),
-	_OMAP3_BALLENTRY(SYS_CLKOUT2, "ae22", NULL),
-	_OMAP3_BALLENTRY(SYS_CLKREQ, "af25", NULL),
-	_OMAP3_BALLENTRY(SYS_NIRQ, "af26", NULL),
-	_OMAP3_BALLENTRY(SYS_NRESWARM, "af24", NULL),
-	_OMAP3_BALLENTRY(SYS_OFF_MODE, "af22", NULL),
-	_OMAP3_BALLENTRY(UART1_CTS, "w8", NULL),
-	_OMAP3_BALLENTRY(UART1_RTS, "aa9", NULL),
-	_OMAP3_BALLENTRY(UART1_RX, "y8", NULL),
-	_OMAP3_BALLENTRY(UART1_TX, "aa8", NULL),
-	_OMAP3_BALLENTRY(UART2_CTS, "ab26", NULL),
-	_OMAP3_BALLENTRY(UART2_RTS, "ab25", NULL),
-	_OMAP3_BALLENTRY(UART2_RX, "ad25", NULL),
-	_OMAP3_BALLENTRY(UART2_TX, "aa25", NULL),
-	_OMAP3_BALLENTRY(UART3_CTS_RCTX, "h18", NULL),
-	_OMAP3_BALLENTRY(UART3_RTS_SD, "h19", NULL),
-	_OMAP3_BALLENTRY(UART3_RX_IRRX, "h20", NULL),
-	_OMAP3_BALLENTRY(UART3_TX_IRTX, "h21", NULL),
-	{ .reg_offset = OMAP_MUX_TERMINATOR },
-};
-#else
-#define omap36xx_cbp_ball	 NULL
-#endif
-
-int __init omap3_mux_init(struct omap_board_mux *board_subset, int flags)
-{
-	struct omap_mux *package_subset;
-	struct omap_ball *package_balls;
-
-	switch (flags & OMAP_PACKAGE_MASK) {
-	case OMAP_PACKAGE_CBC:
-		package_subset = omap3_cbc_subset;
-		package_balls = omap3_cbc_ball;
-		break;
-	case OMAP_PACKAGE_CBB:
-		package_subset = omap3_cbb_subset;
-		package_balls = omap3_cbb_ball;
-		break;
-	case OMAP_PACKAGE_CUS:
-		package_subset = omap3_cus_subset;
-		package_balls = omap3_cus_ball;
-		break;
-	case OMAP_PACKAGE_CBP:
-		package_subset = omap36xx_cbp_subset;
-		package_balls = omap36xx_cbp_ball;
-		break;
-	default:
-		pr_err("%s Unknown omap package, mux disabled\n", __func__);
-		return -EINVAL;
-	}
-
-	return omap_mux_init("core", OMAP_MUX_GPIO_IN_MODE4,
-			     OMAP3_CONTROL_PADCONF_MUX_PBASE,
-			     OMAP3_CONTROL_PADCONF_MUX_SIZE,
-			     omap3_muxmodes, package_subset, board_subset,
-			     package_balls);
-}
diff --git a/arch/arm/mach-omap2/mux34xx.h b/arch/arm/mach-omap2/mux34xx.h
deleted file mode 100644
index 3f26d297c082..000000000000
--- a/arch/arm/mach-omap2/mux34xx.h
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Copyright (C) 2009 Nokia
- * Copyright (C) 2009 Texas Instruments
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#define OMAP3_CONTROL_PADCONF_MUX_PBASE				0x48002030LU
-
-#define OMAP3_MUX(mode0, mux_value)					\
-{									\
-	.reg_offset	= (OMAP3_CONTROL_PADCONF_##mode0##_OFFSET),	\
-	.value		= (mux_value),					\
-}
-
-/*
- * OMAP3 CONTROL_PADCONF* register offsets for pin-muxing
- *
- * Extracted from the TRM.  Add 0x48002030 to these values to get the
- * absolute addresses.  The name in the macro is the mode-0 name of
- * the pin.  NOTE: These registers are 16-bits wide.
- *
- * Note that 34XX TRM uses MMC instead of SDMMC and SAD2D instead
- * of CHASSIS for some registers. For the defines, we follow the
- * 36XX naming, and use SDMMC and CHASSIS.
- */
-#define OMAP3_CONTROL_PADCONF_SDRC_D0_OFFSET			0x000
-#define OMAP3_CONTROL_PADCONF_SDRC_D1_OFFSET			0x002
-#define OMAP3_CONTROL_PADCONF_SDRC_D2_OFFSET			0x004
-#define OMAP3_CONTROL_PADCONF_SDRC_D3_OFFSET			0x006
-#define OMAP3_CONTROL_PADCONF_SDRC_D4_OFFSET			0x008
-#define OMAP3_CONTROL_PADCONF_SDRC_D5_OFFSET			0x00a
-#define OMAP3_CONTROL_PADCONF_SDRC_D6_OFFSET			0x00c
-#define OMAP3_CONTROL_PADCONF_SDRC_D7_OFFSET			0x00e
-#define OMAP3_CONTROL_PADCONF_SDRC_D8_OFFSET			0x010
-#define OMAP3_CONTROL_PADCONF_SDRC_D9_OFFSET			0x012
-#define OMAP3_CONTROL_PADCONF_SDRC_D10_OFFSET			0x014
-#define OMAP3_CONTROL_PADCONF_SDRC_D11_OFFSET			0x016
-#define OMAP3_CONTROL_PADCONF_SDRC_D12_OFFSET			0x018
-#define OMAP3_CONTROL_PADCONF_SDRC_D13_OFFSET			0x01a
-#define OMAP3_CONTROL_PADCONF_SDRC_D14_OFFSET			0x01c
-#define OMAP3_CONTROL_PADCONF_SDRC_D15_OFFSET			0x01e
-#define OMAP3_CONTROL_PADCONF_SDRC_D16_OFFSET			0x020
-#define OMAP3_CONTROL_PADCONF_SDRC_D17_OFFSET			0x022
-#define OMAP3_CONTROL_PADCONF_SDRC_D18_OFFSET			0x024
-#define OMAP3_CONTROL_PADCONF_SDRC_D19_OFFSET			0x026
-#define OMAP3_CONTROL_PADCONF_SDRC_D20_OFFSET			0x028
-#define OMAP3_CONTROL_PADCONF_SDRC_D21_OFFSET			0x02a
-#define OMAP3_CONTROL_PADCONF_SDRC_D22_OFFSET			0x02c
-#define OMAP3_CONTROL_PADCONF_SDRC_D23_OFFSET			0x02e
-#define OMAP3_CONTROL_PADCONF_SDRC_D24_OFFSET			0x030
-#define OMAP3_CONTROL_PADCONF_SDRC_D25_OFFSET			0x032
-#define OMAP3_CONTROL_PADCONF_SDRC_D26_OFFSET			0x034
-#define OMAP3_CONTROL_PADCONF_SDRC_D27_OFFSET			0x036
-#define OMAP3_CONTROL_PADCONF_SDRC_D28_OFFSET			0x038
-#define OMAP3_CONTROL_PADCONF_SDRC_D29_OFFSET			0x03a
-#define OMAP3_CONTROL_PADCONF_SDRC_D30_OFFSET			0x03c
-#define OMAP3_CONTROL_PADCONF_SDRC_D31_OFFSET			0x03e
-#define OMAP3_CONTROL_PADCONF_SDRC_CLK_OFFSET			0x040
-#define OMAP3_CONTROL_PADCONF_SDRC_DQS0_OFFSET			0x042
-#define OMAP3_CONTROL_PADCONF_SDRC_DQS1_OFFSET			0x044
-#define OMAP3_CONTROL_PADCONF_SDRC_DQS2_OFFSET			0x046
-#define OMAP3_CONTROL_PADCONF_SDRC_DQS3_OFFSET			0x048
-#define OMAP3_CONTROL_PADCONF_GPMC_A1_OFFSET			0x04a
-#define OMAP3_CONTROL_PADCONF_GPMC_A2_OFFSET			0x04c
-#define OMAP3_CONTROL_PADCONF_GPMC_A3_OFFSET			0x04e
-#define OMAP3_CONTROL_PADCONF_GPMC_A4_OFFSET			0x050
-#define OMAP3_CONTROL_PADCONF_GPMC_A5_OFFSET			0x052
-#define OMAP3_CONTROL_PADCONF_GPMC_A6_OFFSET			0x054
-#define OMAP3_CONTROL_PADCONF_GPMC_A7_OFFSET			0x056
-#define OMAP3_CONTROL_PADCONF_GPMC_A8_OFFSET			0x058
-#define OMAP3_CONTROL_PADCONF_GPMC_A9_OFFSET			0x05a
-#define OMAP3_CONTROL_PADCONF_GPMC_A10_OFFSET			0x05c
-#define OMAP3_CONTROL_PADCONF_GPMC_D0_OFFSET			0x05e
-#define OMAP3_CONTROL_PADCONF_GPMC_D1_OFFSET			0x060
-#define OMAP3_CONTROL_PADCONF_GPMC_D2_OFFSET			0x062
-#define OMAP3_CONTROL_PADCONF_GPMC_D3_OFFSET			0x064
-#define OMAP3_CONTROL_PADCONF_GPMC_D4_OFFSET			0x066
-#define OMAP3_CONTROL_PADCONF_GPMC_D5_OFFSET			0x068
-#define OMAP3_CONTROL_PADCONF_GPMC_D6_OFFSET			0x06a
-#define OMAP3_CONTROL_PADCONF_GPMC_D7_OFFSET			0x06c
-#define OMAP3_CONTROL_PADCONF_GPMC_D8_OFFSET			0x06e
-#define OMAP3_CONTROL_PADCONF_GPMC_D9_OFFSET			0x070
-#define OMAP3_CONTROL_PADCONF_GPMC_D10_OFFSET			0x072
-#define OMAP3_CONTROL_PADCONF_GPMC_D11_OFFSET			0x074
-#define OMAP3_CONTROL_PADCONF_GPMC_D12_OFFSET			0x076
-#define OMAP3_CONTROL_PADCONF_GPMC_D13_OFFSET			0x078
-#define OMAP3_CONTROL_PADCONF_GPMC_D14_OFFSET			0x07a
-#define OMAP3_CONTROL_PADCONF_GPMC_D15_OFFSET			0x07c
-#define OMAP3_CONTROL_PADCONF_GPMC_NCS0_OFFSET			0x07e
-#define OMAP3_CONTROL_PADCONF_GPMC_NCS1_OFFSET			0x080
-#define OMAP3_CONTROL_PADCONF_GPMC_NCS2_OFFSET			0x082
-#define OMAP3_CONTROL_PADCONF_GPMC_NCS3_OFFSET			0x084
-#define OMAP3_CONTROL_PADCONF_GPMC_NCS4_OFFSET			0x086
-#define OMAP3_CONTROL_PADCONF_GPMC_NCS5_OFFSET			0x088
-#define OMAP3_CONTROL_PADCONF_GPMC_NCS6_OFFSET			0x08a
-#define OMAP3_CONTROL_PADCONF_GPMC_NCS7_OFFSET			0x08c
-#define OMAP3_CONTROL_PADCONF_GPMC_CLK_OFFSET			0x08e
-#define OMAP3_CONTROL_PADCONF_GPMC_NADV_ALE_OFFSET		0x090
-#define OMAP3_CONTROL_PADCONF_GPMC_NOE_OFFSET			0x092
-#define OMAP3_CONTROL_PADCONF_GPMC_NWE_OFFSET			0x094
-#define OMAP3_CONTROL_PADCONF_GPMC_NBE0_CLE_OFFSET		0x096
-#define OMAP3_CONTROL_PADCONF_GPMC_NBE1_OFFSET			0x098
-#define OMAP3_CONTROL_PADCONF_GPMC_NWP_OFFSET			0x09a
-#define OMAP3_CONTROL_PADCONF_GPMC_WAIT0_OFFSET			0x09c
-#define OMAP3_CONTROL_PADCONF_GPMC_WAIT1_OFFSET			0x09e
-#define OMAP3_CONTROL_PADCONF_GPMC_WAIT2_OFFSET			0x0a0
-#define OMAP3_CONTROL_PADCONF_GPMC_WAIT3_OFFSET			0x0a2
-#define OMAP3_CONTROL_PADCONF_DSS_PCLK_OFFSET			0x0a4
-#define OMAP3_CONTROL_PADCONF_DSS_HSYNC_OFFSET			0x0a6
-#define OMAP3_CONTROL_PADCONF_DSS_VSYNC_OFFSET			0x0a8
-#define OMAP3_CONTROL_PADCONF_DSS_ACBIAS_OFFSET			0x0aa
-#define OMAP3_CONTROL_PADCONF_DSS_DATA0_OFFSET			0x0ac
-#define OMAP3_CONTROL_PADCONF_DSS_DATA1_OFFSET			0x0ae
-#define OMAP3_CONTROL_PADCONF_DSS_DATA2_OFFSET			0x0b0
-#define OMAP3_CONTROL_PADCONF_DSS_DATA3_OFFSET			0x0b2
-#define OMAP3_CONTROL_PADCONF_DSS_DATA4_OFFSET			0x0b4
-#define OMAP3_CONTROL_PADCONF_DSS_DATA5_OFFSET			0x0b6
-#define OMAP3_CONTROL_PADCONF_DSS_DATA6_OFFSET			0x0b8
-#define OMAP3_CONTROL_PADCONF_DSS_DATA7_OFFSET			0x0ba
-#define OMAP3_CONTROL_PADCONF_DSS_DATA8_OFFSET			0x0bc
-#define OMAP3_CONTROL_PADCONF_DSS_DATA9_OFFSET			0x0be
-#define OMAP3_CONTROL_PADCONF_DSS_DATA10_OFFSET			0x0c0
-#define OMAP3_CONTROL_PADCONF_DSS_DATA11_OFFSET			0x0c2
-#define OMAP3_CONTROL_PADCONF_DSS_DATA12_OFFSET			0x0c4
-#define OMAP3_CONTROL_PADCONF_DSS_DATA13_OFFSET			0x0c6
-#define OMAP3_CONTROL_PADCONF_DSS_DATA14_OFFSET			0x0c8
-#define OMAP3_CONTROL_PADCONF_DSS_DATA15_OFFSET			0x0ca
-#define OMAP3_CONTROL_PADCONF_DSS_DATA16_OFFSET			0x0cc
-#define OMAP3_CONTROL_PADCONF_DSS_DATA17_OFFSET			0x0ce
-#define OMAP3_CONTROL_PADCONF_DSS_DATA18_OFFSET			0x0d0
-#define OMAP3_CONTROL_PADCONF_DSS_DATA19_OFFSET			0x0d2
-#define OMAP3_CONTROL_PADCONF_DSS_DATA20_OFFSET			0x0d4
-#define OMAP3_CONTROL_PADCONF_DSS_DATA21_OFFSET			0x0d6
-#define OMAP3_CONTROL_PADCONF_DSS_DATA22_OFFSET			0x0d8
-#define OMAP3_CONTROL_PADCONF_DSS_DATA23_OFFSET			0x0da
-#define OMAP3_CONTROL_PADCONF_CAM_HS_OFFSET			0x0dc
-#define OMAP3_CONTROL_PADCONF_CAM_VS_OFFSET			0x0de
-#define OMAP3_CONTROL_PADCONF_CAM_XCLKA_OFFSET			0x0e0
-#define OMAP3_CONTROL_PADCONF_CAM_PCLK_OFFSET			0x0e2
-#define OMAP3_CONTROL_PADCONF_CAM_FLD_OFFSET			0x0e4
-#define OMAP3_CONTROL_PADCONF_CAM_D0_OFFSET			0x0e6
-#define OMAP3_CONTROL_PADCONF_CAM_D1_OFFSET			0x0e8
-#define OMAP3_CONTROL_PADCONF_CAM_D2_OFFSET			0x0ea
-#define OMAP3_CONTROL_PADCONF_CAM_D3_OFFSET			0x0ec
-#define OMAP3_CONTROL_PADCONF_CAM_D4_OFFSET			0x0ee
-#define OMAP3_CONTROL_PADCONF_CAM_D5_OFFSET			0x0f0
-#define OMAP3_CONTROL_PADCONF_CAM_D6_OFFSET			0x0f2
-#define OMAP3_CONTROL_PADCONF_CAM_D7_OFFSET			0x0f4
-#define OMAP3_CONTROL_PADCONF_CAM_D8_OFFSET			0x0f6
-#define OMAP3_CONTROL_PADCONF_CAM_D9_OFFSET			0x0f8
-#define OMAP3_CONTROL_PADCONF_CAM_D10_OFFSET			0x0fa
-#define OMAP3_CONTROL_PADCONF_CAM_D11_OFFSET			0x0fc
-#define OMAP3_CONTROL_PADCONF_CAM_XCLKB_OFFSET			0x0fe
-#define OMAP3_CONTROL_PADCONF_CAM_WEN_OFFSET			0x100
-#define OMAP3_CONTROL_PADCONF_CAM_STROBE_OFFSET			0x102
-#define OMAP3_CONTROL_PADCONF_CSI2_DX0_OFFSET			0x104
-#define OMAP3_CONTROL_PADCONF_CSI2_DY0_OFFSET			0x106
-#define OMAP3_CONTROL_PADCONF_CSI2_DX1_OFFSET			0x108
-#define OMAP3_CONTROL_PADCONF_CSI2_DY1_OFFSET			0x10a
-#define OMAP3_CONTROL_PADCONF_MCBSP2_FSX_OFFSET			0x10c
-#define OMAP3_CONTROL_PADCONF_MCBSP2_CLKX_OFFSET		0x10e
-#define OMAP3_CONTROL_PADCONF_MCBSP2_DR_OFFSET			0x110
-#define OMAP3_CONTROL_PADCONF_MCBSP2_DX_OFFSET			0x112
-#define OMAP3_CONTROL_PADCONF_SDMMC1_CLK_OFFSET			0x114
-#define OMAP3_CONTROL_PADCONF_SDMMC1_CMD_OFFSET			0x116
-#define OMAP3_CONTROL_PADCONF_SDMMC1_DAT0_OFFSET		0x118
-#define OMAP3_CONTROL_PADCONF_SDMMC1_DAT1_OFFSET		0x11a
-#define OMAP3_CONTROL_PADCONF_SDMMC1_DAT2_OFFSET		0x11c
-#define OMAP3_CONTROL_PADCONF_SDMMC1_DAT3_OFFSET		0x11e
-
-/* SDMMC1_DAT4 - DAT7 are SIM_IO SIM_CLK SIM_PWRCTRL and SIM_RST on 36xx */
-#define OMAP3_CONTROL_PADCONF_SDMMC1_DAT4_OFFSET		0x120
-#define OMAP3_CONTROL_PADCONF_SDMMC1_DAT5_OFFSET		0x122
-#define OMAP3_CONTROL_PADCONF_SDMMC1_DAT6_OFFSET		0x124
-#define OMAP3_CONTROL_PADCONF_SDMMC1_DAT7_OFFSET		0x126
-
-#define OMAP3_CONTROL_PADCONF_SDMMC2_CLK_OFFSET			0x128
-#define OMAP3_CONTROL_PADCONF_SDMMC2_CMD_OFFSET			0x12a
-#define OMAP3_CONTROL_PADCONF_SDMMC2_DAT0_OFFSET		0x12c
-#define OMAP3_CONTROL_PADCONF_SDMMC2_DAT1_OFFSET		0x12e
-#define OMAP3_CONTROL_PADCONF_SDMMC2_DAT2_OFFSET		0x130
-#define OMAP3_CONTROL_PADCONF_SDMMC2_DAT3_OFFSET		0x132
-#define OMAP3_CONTROL_PADCONF_SDMMC2_DAT4_OFFSET		0x134
-#define OMAP3_CONTROL_PADCONF_SDMMC2_DAT5_OFFSET		0x136
-#define OMAP3_CONTROL_PADCONF_SDMMC2_DAT6_OFFSET		0x138
-#define OMAP3_CONTROL_PADCONF_SDMMC2_DAT7_OFFSET		0x13a
-#define OMAP3_CONTROL_PADCONF_MCBSP3_DX_OFFSET			0x13c
-#define OMAP3_CONTROL_PADCONF_MCBSP3_DR_OFFSET			0x13e
-#define OMAP3_CONTROL_PADCONF_MCBSP3_CLKX_OFFSET		0x140
-#define OMAP3_CONTROL_PADCONF_MCBSP3_FSX_OFFSET			0x142
-#define OMAP3_CONTROL_PADCONF_UART2_CTS_OFFSET			0x144
-#define OMAP3_CONTROL_PADCONF_UART2_RTS_OFFSET			0x146
-#define OMAP3_CONTROL_PADCONF_UART2_TX_OFFSET			0x148
-#define OMAP3_CONTROL_PADCONF_UART2_RX_OFFSET			0x14a
-#define OMAP3_CONTROL_PADCONF_UART1_TX_OFFSET			0x14c
-#define OMAP3_CONTROL_PADCONF_UART1_RTS_OFFSET			0x14e
-#define OMAP3_CONTROL_PADCONF_UART1_CTS_OFFSET			0x150
-#define OMAP3_CONTROL_PADCONF_UART1_RX_OFFSET			0x152
-#define OMAP3_CONTROL_PADCONF_MCBSP4_CLKX_OFFSET		0x154
-#define OMAP3_CONTROL_PADCONF_MCBSP4_DR_OFFSET			0x156
-#define OMAP3_CONTROL_PADCONF_MCBSP4_DX_OFFSET			0x158
-#define OMAP3_CONTROL_PADCONF_MCBSP4_FSX_OFFSET			0x15a
-#define OMAP3_CONTROL_PADCONF_MCBSP1_CLKR_OFFSET		0x15c
-#define OMAP3_CONTROL_PADCONF_MCBSP1_FSR_OFFSET			0x15e
-#define OMAP3_CONTROL_PADCONF_MCBSP1_DX_OFFSET			0x160
-#define OMAP3_CONTROL_PADCONF_MCBSP1_DR_OFFSET			0x162
-#define OMAP3_CONTROL_PADCONF_MCBSP_CLKS_OFFSET			0x164
-#define OMAP3_CONTROL_PADCONF_MCBSP1_FSX_OFFSET			0x166
-#define OMAP3_CONTROL_PADCONF_MCBSP1_CLKX_OFFSET		0x168
-#define OMAP3_CONTROL_PADCONF_UART3_CTS_RCTX_OFFSET		0x16a
-#define OMAP3_CONTROL_PADCONF_UART3_RTS_SD_OFFSET		0x16c
-#define OMAP3_CONTROL_PADCONF_UART3_RX_IRRX_OFFSET		0x16e
-#define OMAP3_CONTROL_PADCONF_UART3_TX_IRTX_OFFSET		0x170
-#define OMAP3_CONTROL_PADCONF_HSUSB0_CLK_OFFSET			0x172
-#define OMAP3_CONTROL_PADCONF_HSUSB0_STP_OFFSET			0x174
-#define OMAP3_CONTROL_PADCONF_HSUSB0_DIR_OFFSET			0x176
-#define OMAP3_CONTROL_PADCONF_HSUSB0_NXT_OFFSET			0x178
-#define OMAP3_CONTROL_PADCONF_HSUSB0_DATA0_OFFSET		0x17a
-#define OMAP3_CONTROL_PADCONF_HSUSB0_DATA1_OFFSET		0x17c
-#define OMAP3_CONTROL_PADCONF_HSUSB0_DATA2_OFFSET		0x17e
-#define OMAP3_CONTROL_PADCONF_HSUSB0_DATA3_OFFSET		0x180
-#define OMAP3_CONTROL_PADCONF_HSUSB0_DATA4_OFFSET		0x182
-#define OMAP3_CONTROL_PADCONF_HSUSB0_DATA5_OFFSET		0x184
-#define OMAP3_CONTROL_PADCONF_HSUSB0_DATA6_OFFSET		0x186
-#define OMAP3_CONTROL_PADCONF_HSUSB0_DATA7_OFFSET		0x188
-#define OMAP3_CONTROL_PADCONF_I2C1_SCL_OFFSET			0x18a
-#define OMAP3_CONTROL_PADCONF_I2C1_SDA_OFFSET			0x18c
-#define OMAP3_CONTROL_PADCONF_I2C2_SCL_OFFSET			0x18e
-#define OMAP3_CONTROL_PADCONF_I2C2_SDA_OFFSET			0x190
-#define OMAP3_CONTROL_PADCONF_I2C3_SCL_OFFSET			0x192
-#define OMAP3_CONTROL_PADCONF_I2C3_SDA_OFFSET			0x194
-#define OMAP3_CONTROL_PADCONF_HDQ_SIO_OFFSET			0x196
-#define OMAP3_CONTROL_PADCONF_MCSPI1_CLK_OFFSET			0x198
-#define OMAP3_CONTROL_PADCONF_MCSPI1_SIMO_OFFSET		0x19a
-#define OMAP3_CONTROL_PADCONF_MCSPI1_SOMI_OFFSET		0x19c
-#define OMAP3_CONTROL_PADCONF_MCSPI1_CS0_OFFSET			0x19e
-#define OMAP3_CONTROL_PADCONF_MCSPI1_CS1_OFFSET			0x1a0
-#define OMAP3_CONTROL_PADCONF_MCSPI1_CS2_OFFSET			0x1a2
-#define OMAP3_CONTROL_PADCONF_MCSPI1_CS3_OFFSET			0x1a4
-#define OMAP3_CONTROL_PADCONF_MCSPI2_CLK_OFFSET			0x1a6
-#define OMAP3_CONTROL_PADCONF_MCSPI2_SIMO_OFFSET		0x1a8
-#define OMAP3_CONTROL_PADCONF_MCSPI2_SOMI_OFFSET		0x1aa
-#define OMAP3_CONTROL_PADCONF_MCSPI2_CS0_OFFSET			0x1ac
-#define OMAP3_CONTROL_PADCONF_MCSPI2_CS1_OFFSET			0x1ae
-#define OMAP3_CONTROL_PADCONF_SYS_NIRQ_OFFSET			0x1b0
-#define OMAP3_CONTROL_PADCONF_SYS_CLKOUT2_OFFSET		0x1b2
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD0_OFFSET		0x1b4
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD1_OFFSET		0x1b6
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD2_OFFSET		0x1b8
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD3_OFFSET		0x1ba
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD4_OFFSET		0x1bc
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD5_OFFSET		0x1be
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD6_OFFSET		0x1c0
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD7_OFFSET		0x1c2
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD8_OFFSET		0x1c4
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD9_OFFSET		0x1c6
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD10_OFFSET		0x1c8
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD11_OFFSET		0x1ca
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD12_OFFSET		0x1cc
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD13_OFFSET		0x1ce
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD14_OFFSET		0x1d0
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD15_OFFSET		0x1d2
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD16_OFFSET		0x1d4
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD17_OFFSET		0x1d6
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD18_OFFSET		0x1d8
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD19_OFFSET		0x1da
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD20_OFFSET		0x1dc
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD21_OFFSET		0x1de
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD22_OFFSET		0x1e0
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD23_OFFSET		0x1e2
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD24_OFFSET		0x1e4
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD25_OFFSET		0x1e6
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD26_OFFSET		0x1e8
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD27_OFFSET		0x1ea
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD28_OFFSET		0x1ec
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD29_OFFSET		0x1ee
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD30_OFFSET		0x1f0
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD31_OFFSET		0x1f2
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD32_OFFSET		0x1f4
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD33_OFFSET		0x1f6
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD34_OFFSET		0x1f8
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD35_OFFSET		0x1fa
-#define OMAP3_CONTROL_PADCONF_SAD2D_MCAD36_OFFSET		0x1fc
-
-/* Note that 34xx TRM has SAD2D instead of CHASSIS for these */
-#define OMAP3_CONTROL_PADCONF_CHASSIS_CLK26MI_OFFSET		0x1fe
-#define OMAP3_CONTROL_PADCONF_CHASSIS_NRESPWRON_OFFSET		0x200
-#define OMAP3_CONTROL_PADCONF_CHASSIS_NRESWARW_OFFSET		0x202
-#define OMAP3_CONTROL_PADCONF_CHASSIS_NIRQ_OFFSET		0x204
-#define OMAP3_CONTROL_PADCONF_CHASSIS_FIQ_OFFSET		0x206
-#define OMAP3_CONTROL_PADCONF_CHASSIS_ARMIRQ_OFFSET		0x208
-#define OMAP3_CONTROL_PADCONF_CHASSIS_IVAIRQ_OFFSET		0x20a
-#define OMAP3_CONTROL_PADCONF_CHASSIS_DMAREQ0_OFFSET		0x20c
-#define OMAP3_CONTROL_PADCONF_CHASSIS_DMAREQ1_OFFSET		0x20e
-#define OMAP3_CONTROL_PADCONF_CHASSIS_DMAREQ2_OFFSET		0x210
-#define OMAP3_CONTROL_PADCONF_CHASSIS_DMAREQ3_OFFSET		0x212
-#define OMAP3_CONTROL_PADCONF_CHASSIS_NTRST_OFFSET		0x214
-#define OMAP3_CONTROL_PADCONF_CHASSIS_TDI_OFFSET		0x216
-#define OMAP3_CONTROL_PADCONF_CHASSIS_TDO_OFFSET		0x218
-#define OMAP3_CONTROL_PADCONF_CHASSIS_TMS_OFFSET		0x21a
-#define OMAP3_CONTROL_PADCONF_CHASSIS_TCK_OFFSET		0x21c
-#define OMAP3_CONTROL_PADCONF_CHASSIS_RTCK_OFFSET		0x21e
-#define OMAP3_CONTROL_PADCONF_CHASSIS_MSTDBY_OFFSET		0x220
-#define OMAP3_CONTROL_PADCONF_CHASSIS_IDLEREQ_OFFSET		0x222
-#define OMAP3_CONTROL_PADCONF_CHASSIS_IDLEACK_OFFSET		0x224
-
-#define OMAP3_CONTROL_PADCONF_SAD2D_MWRITE_OFFSET		0x226
-#define OMAP3_CONTROL_PADCONF_SAD2D_SWRITE_OFFSET		0x228
-#define OMAP3_CONTROL_PADCONF_SAD2D_MREAD_OFFSET		0x22a
-#define OMAP3_CONTROL_PADCONF_SAD2D_SREAD_OFFSET		0x22c
-#define OMAP3_CONTROL_PADCONF_SAD2D_MBUSFLAG_OFFSET		0x22e
-#define OMAP3_CONTROL_PADCONF_SAD2D_SBUSFLAG_OFFSET		0x230
-#define OMAP3_CONTROL_PADCONF_SDRC_CKE0_OFFSET			0x232
-#define OMAP3_CONTROL_PADCONF_SDRC_CKE1_OFFSET			0x234
-
-/* 36xx only */
-#define OMAP3_CONTROL_PADCONF_GPMC_A11_OFFSET			0x236
-#define OMAP3_CONTROL_PADCONF_SDRC_BA0_OFFSET			0x570
-#define OMAP3_CONTROL_PADCONF_SDRC_BA1_OFFSET			0x572
-#define OMAP3_CONTROL_PADCONF_SDRC_A0_OFFSET			0x574
-#define OMAP3_CONTROL_PADCONF_SDRC_A1_OFFSET			0x576
-#define OMAP3_CONTROL_PADCONF_SDRC_A2_OFFSET			0x578
-#define OMAP3_CONTROL_PADCONF_SDRC_A3_OFFSET			0x57a
-#define OMAP3_CONTROL_PADCONF_SDRC_A4_OFFSET			0x57c
-#define OMAP3_CONTROL_PADCONF_SDRC_A5_OFFSET			0x57e
-#define OMAP3_CONTROL_PADCONF_SDRC_A6_OFFSET			0x580
-#define OMAP3_CONTROL_PADCONF_SDRC_A7_OFFSET			0x582
-#define OMAP3_CONTROL_PADCONF_SDRC_A8_OFFSET			0x584
-#define OMAP3_CONTROL_PADCONF_SDRC_A9_OFFSET			0x586
-#define OMAP3_CONTROL_PADCONF_SDRC_A10_OFFSET			0x588
-#define OMAP3_CONTROL_PADCONF_SDRC_A11_OFFSET			0x58a
-#define OMAP3_CONTROL_PADCONF_SDRC_A12_OFFSET			0x58c
-#define OMAP3_CONTROL_PADCONF_SDRC_A13_OFFSET			0x58e
-#define OMAP3_CONTROL_PADCONF_SDRC_A14_OFFSET			0x590
-#define OMAP3_CONTROL_PADCONF_SDRC_NCS0_OFFSET			0x592
-#define OMAP3_CONTROL_PADCONF_SDRC_NCS1_OFFSET			0x594
-#define OMAP3_CONTROL_PADCONF_SDRC_NCLK_OFFSET			0x596
-#define OMAP3_CONTROL_PADCONF_SDRC_NRAS_OFFSET			0x598
-#define OMAP3_CONTROL_PADCONF_SDRC_NCAS_OFFSET			0x59a
-#define OMAP3_CONTROL_PADCONF_SDRC_NWE_OFFSET			0x59c
-#define OMAP3_CONTROL_PADCONF_SDRC_DM0_OFFSET			0x59e
-#define OMAP3_CONTROL_PADCONF_SDRC_DM1_OFFSET			0x5a0
-#define OMAP3_CONTROL_PADCONF_SDRC_DM2_OFFSET			0x5a2
-#define OMAP3_CONTROL_PADCONF_SDRC_DM3_OFFSET			0x5a4
-
-/* 36xx only, these are SDMMC1_DAT4 - DAT7 on 34xx */
-#define OMAP3_CONTROL_PADCONF_SIM_IO_OFFSET			0x120
-#define OMAP3_CONTROL_PADCONF_SIM_CLK_OFFSET			0x122
-#define OMAP3_CONTROL_PADCONF_SIM_PWRCTRL_OFFSET		0x124
-#define OMAP3_CONTROL_PADCONF_SIM_RST_OFFSET			0x126
-
-#define OMAP3_CONTROL_PADCONF_ETK_CLK_OFFSET			0x5a8
-#define OMAP3_CONTROL_PADCONF_ETK_CTL_OFFSET			0x5aa
-#define OMAP3_CONTROL_PADCONF_ETK_D0_OFFSET			0x5ac
-#define OMAP3_CONTROL_PADCONF_ETK_D1_OFFSET			0x5ae
-#define OMAP3_CONTROL_PADCONF_ETK_D2_OFFSET			0x5b0
-#define OMAP3_CONTROL_PADCONF_ETK_D3_OFFSET			0x5b2
-#define OMAP3_CONTROL_PADCONF_ETK_D4_OFFSET			0x5b4
-#define OMAP3_CONTROL_PADCONF_ETK_D5_OFFSET			0x5b6
-#define OMAP3_CONTROL_PADCONF_ETK_D6_OFFSET			0x5b8
-#define OMAP3_CONTROL_PADCONF_ETK_D7_OFFSET			0x5ba
-#define OMAP3_CONTROL_PADCONF_ETK_D8_OFFSET			0x5bc
-#define OMAP3_CONTROL_PADCONF_ETK_D9_OFFSET			0x5be
-#define OMAP3_CONTROL_PADCONF_ETK_D10_OFFSET			0x5c0
-#define OMAP3_CONTROL_PADCONF_ETK_D11_OFFSET			0x5c2
-#define OMAP3_CONTROL_PADCONF_ETK_D12_OFFSET			0x5c4
-#define OMAP3_CONTROL_PADCONF_ETK_D13_OFFSET			0x5c6
-#define OMAP3_CONTROL_PADCONF_ETK_D14_OFFSET			0x5c8
-#define OMAP3_CONTROL_PADCONF_ETK_D15_OFFSET			0x5ca
-#define OMAP3_CONTROL_PADCONF_I2C4_SCL_OFFSET			0x9d0
-#define OMAP3_CONTROL_PADCONF_I2C4_SDA_OFFSET			0x9d2
-#define OMAP3_CONTROL_PADCONF_SYS_32K_OFFSET			0x9d4
-#define OMAP3_CONTROL_PADCONF_SYS_CLKREQ_OFFSET			0x9d6
-#define OMAP3_CONTROL_PADCONF_SYS_NRESWARM_OFFSET		0x9d8
-#define OMAP3_CONTROL_PADCONF_SYS_BOOT0_OFFSET			0x9da
-#define OMAP3_CONTROL_PADCONF_SYS_BOOT1_OFFSET			0x9dc
-#define OMAP3_CONTROL_PADCONF_SYS_BOOT2_OFFSET			0x9de
-#define OMAP3_CONTROL_PADCONF_SYS_BOOT3_OFFSET			0x9e0
-#define OMAP3_CONTROL_PADCONF_SYS_BOOT4_OFFSET			0x9e2
-#define OMAP3_CONTROL_PADCONF_SYS_BOOT5_OFFSET			0x9e4
-#define OMAP3_CONTROL_PADCONF_SYS_BOOT6_OFFSET			0x9e6
-#define OMAP3_CONTROL_PADCONF_SYS_OFF_MODE_OFFSET		0x9e8
-#define OMAP3_CONTROL_PADCONF_SYS_CLKOUT1_OFFSET		0x9ea
-#define OMAP3_CONTROL_PADCONF_JTAG_NTRST_OFFSET			0x9ec
-#define OMAP3_CONTROL_PADCONF_JTAG_TCK_OFFSET			0x9ee
-#define OMAP3_CONTROL_PADCONF_JTAG_TMS_TMSC_OFFSET		0x9f0
-#define OMAP3_CONTROL_PADCONF_JTAG_TDI_OFFSET			0x9f2
-#define OMAP3_CONTROL_PADCONF_JTAG_EMU0_OFFSET			0x9f4
-#define OMAP3_CONTROL_PADCONF_JTAG_EMU1_OFFSET			0x9f6
-#define OMAP3_CONTROL_PADCONF_SAD2D_SWAKEUP_OFFSET		0xa1c
-#define OMAP3_CONTROL_PADCONF_JTAG_RTCK_OFFSET			0xa1e
-#define OMAP3_CONTROL_PADCONF_JTAG_TDO_OFFSET			0xa20
-#define OMAP3_CONTROL_PADCONF_GPIO_127				0xa24
-#define OMAP3_CONTROL_PADCONF_GPIO_126				0xa26
-#define OMAP3_CONTROL_PADCONF_GPIO_128				0xa28
-#define OMAP3_CONTROL_PADCONF_GPIO_129				0xa2a
-
-#define OMAP3_CONTROL_PADCONF_MUX_SIZE				\
-		(OMAP3_CONTROL_PADCONF_GPIO_129 + 0x2)
diff --git a/arch/arm/mach-omap2/omap-mpuss-lowpower.c b/arch/arm/mach-omap2/omap-mpuss-lowpower.c
index ad982465efd0..7d62ad48c7c9 100644
--- a/arch/arm/mach-omap2/omap-mpuss-lowpower.c
+++ b/arch/arm/mach-omap2/omap-mpuss-lowpower.c
@@ -48,6 +48,7 @@
 #include <asm/smp_scu.h>
 #include <asm/pgalloc.h>
 #include <asm/suspend.h>
+#include <asm/virt.h>
 #include <asm/hardware/cache-l2x0.h>
 
 #include "soc.h"
@@ -244,10 +245,9 @@ int omap4_enter_lowpower(unsigned int cpu, unsigned int power_state)
 		save_state = 1;
 		break;
 	case PWRDM_POWER_RET:
-		if (IS_PM44XX_ERRATUM(PM_OMAP4_CPU_OSWR_DISABLE)) {
+		if (IS_PM44XX_ERRATUM(PM_OMAP4_CPU_OSWR_DISABLE))
 			save_state = 0;
-			break;
-		}
+		break;
 	default:
 		/*
 		 * CPUx CSWR is invalid hardware state. Also CPUx OSWR
@@ -371,8 +371,12 @@ int __init omap4_mpuss_init(void)
 	pm_info = &per_cpu(omap4_pm_info, 0x0);
 	if (sar_base) {
 		pm_info->scu_sar_addr = sar_base + SCU_OFFSET0;
-		pm_info->wkup_sar_addr = sar_base +
-					CPU0_WAKEUP_NS_PA_ADDR_OFFSET;
+		if (cpu_is_omap44xx())
+			pm_info->wkup_sar_addr = sar_base +
+				CPU0_WAKEUP_NS_PA_ADDR_OFFSET;
+		else
+			pm_info->wkup_sar_addr = sar_base +
+				OMAP5_CPU0_WAKEUP_NS_PA_ADDR_OFFSET;
 		pm_info->l2x0_sar_addr = sar_base + L2X0_SAVE_OFFSET0;
 	}
 	pm_info->pwrdm = pwrdm_lookup("cpu0_pwrdm");
@@ -391,8 +395,12 @@ int __init omap4_mpuss_init(void)
 	pm_info = &per_cpu(omap4_pm_info, 0x1);
 	if (sar_base) {
 		pm_info->scu_sar_addr = sar_base + SCU_OFFSET1;
-		pm_info->wkup_sar_addr = sar_base +
-					CPU1_WAKEUP_NS_PA_ADDR_OFFSET;
+		if (cpu_is_omap44xx())
+			pm_info->wkup_sar_addr = sar_base +
+				CPU1_WAKEUP_NS_PA_ADDR_OFFSET;
+		else
+			pm_info->wkup_sar_addr = sar_base +
+				OMAP5_CPU1_WAKEUP_NS_PA_ADDR_OFFSET;
 		pm_info->l2x0_sar_addr = sar_base + L2X0_SAVE_OFFSET1;
 	}
 
@@ -453,15 +461,24 @@ void __init omap4_mpuss_early_init(void)
 {
 	unsigned long startup_pa;
 
-	if (!cpu_is_omap44xx())
+	if (!(cpu_is_omap44xx() || soc_is_omap54xx()))
 		return;
 
 	sar_base = omap4_get_sar_ram_base();
 
 	if (cpu_is_omap443x())
 		startup_pa = virt_to_phys(omap4_secondary_startup);
-	else
+	else if (cpu_is_omap446x())
 		startup_pa = virt_to_phys(omap4460_secondary_startup);
+	else if ((__boot_cpu_mode & MODE_MASK) == HYP_MODE)
+		startup_pa = virt_to_phys(omap5_secondary_hyp_startup);
+	else
+		startup_pa = virt_to_phys(omap5_secondary_startup);
 
-	writel_relaxed(startup_pa, sar_base + CPU1_WAKEUP_NS_PA_ADDR_OFFSET);
+	if (cpu_is_omap44xx())
+		writel_relaxed(startup_pa, sar_base +
+			       CPU1_WAKEUP_NS_PA_ADDR_OFFSET);
+	else
+		writel_relaxed(startup_pa, sar_base +
+			       OMAP5_CPU1_WAKEUP_NS_PA_ADDR_OFFSET);
 }
diff --git a/arch/arm/mach-omap2/omap4-sar-layout.h b/arch/arm/mach-omap2/omap4-sar-layout.h
index 792b1069f724..5b2966a0f733 100644
--- a/arch/arm/mach-omap2/omap4-sar-layout.h
+++ b/arch/arm/mach-omap2/omap4-sar-layout.h
@@ -31,6 +31,8 @@
 /* CPUx Wakeup Non-Secure Physical Address offsets in SAR_BANK3 */
 #define CPU0_WAKEUP_NS_PA_ADDR_OFFSET		0xa04
 #define CPU1_WAKEUP_NS_PA_ADDR_OFFSET		0xa08
+#define OMAP5_CPU0_WAKEUP_NS_PA_ADDR_OFFSET	0xe00
+#define OMAP5_CPU1_WAKEUP_NS_PA_ADDR_OFFSET	0xe04
 
 #define SAR_BACKUP_STATUS_OFFSET		(SAR_BANK3_OFFSET + 0x500)
 #define SAR_SECURE_RAM_SIZE_OFFSET		(SAR_BANK3_OFFSET + 0x504)
diff --git a/arch/arm/mach-omap2/omap_hwmod.c b/arch/arm/mach-omap2/omap_hwmod.c
index 1052b29697b8..759e1d45ba25 100644
--- a/arch/arm/mach-omap2/omap_hwmod.c
+++ b/arch/arm/mach-omap2/omap_hwmod.c
@@ -160,7 +160,6 @@
 #include "prm44xx.h"
 #include "prm33xx.h"
 #include "prminst44xx.h"
-#include "mux.h"
 #include "pm.h"
 
 /* Name of the OMAP hwmod for the MPU */
@@ -217,9 +216,6 @@ static LIST_HEAD(omap_hwmod_list);
 /* mpu_oh: used to add/remove MPU initiator from sleepdep list */
 static struct omap_hwmod *mpu_oh;
 
-/* io_chain_lock: used to serialize reconfigurations of the I/O chain */
-static DEFINE_SPINLOCK(io_chain_lock);
-
 /*
  * linkspace: ptr to a buffer that struct omap_hwmod_link records are
  * allocated from - used to reduce the number of small memory
@@ -594,51 +590,6 @@ static int _set_module_autoidle(struct omap_hwmod *oh, u8 autoidle,
 }
 
 /**
- * _set_idle_ioring_wakeup - enable/disable IO pad wakeup on hwmod idle for mux
- * @oh: struct omap_hwmod *
- * @set_wake: bool value indicating to set (true) or clear (false) wakeup enable
- *
- * Set or clear the I/O pad wakeup flag in the mux entries for the
- * hwmod @oh.  This function changes the @oh->mux->pads_dynamic array
- * in memory.  If the hwmod is currently idled, and the new idle
- * values don't match the previous ones, this function will also
- * update the SCM PADCTRL registers.  Otherwise, if the hwmod is not
- * currently idled, this function won't touch the hardware: the new
- * mux settings are written to the SCM PADCTRL registers when the
- * hwmod is idled.  No return value.
- */
-static void _set_idle_ioring_wakeup(struct omap_hwmod *oh, bool set_wake)
-{
-	struct omap_device_pad *pad;
-	bool change = false;
-	u16 prev_idle;
-	int j;
-
-	if (!oh->mux || !oh->mux->enabled)
-		return;
-
-	for (j = 0; j < oh->mux->nr_pads_dynamic; j++) {
-		pad = oh->mux->pads_dynamic[j];
-
-		if (!(pad->flags & OMAP_DEVICE_PAD_WAKEUP))
-			continue;
-
-		prev_idle = pad->idle;
-
-		if (set_wake)
-			pad->idle |= OMAP_WAKEUP_EN;
-		else
-			pad->idle &= ~OMAP_WAKEUP_EN;
-
-		if (prev_idle != pad->idle)
-			change = true;
-	}
-
-	if (change && oh->_state == _HWMOD_STATE_IDLE)
-		omap_hwmod_mux(oh->mux, _HWMOD_STATE_IDLE);
-}
-
-/**
  * _enable_wakeup: set OCP_SYSCONFIG.ENAWAKEUP bit in the hardware
  * @oh: struct omap_hwmod *
  *
@@ -2018,29 +1969,6 @@ static int _reset(struct omap_hwmod *oh)
 }
 
 /**
- * _reconfigure_io_chain - clear any I/O chain wakeups and reconfigure chain
- *
- * Call the appropriate PRM function to clear any logged I/O chain
- * wakeups and to reconfigure the chain.  This apparently needs to be
- * done upon every mux change.  Since hwmods can be concurrently
- * enabled and idled, hold a spinlock around the I/O chain
- * reconfiguration sequence.  No return value.
- *
- * XXX When the PRM code is moved to drivers, this function can be removed,
- * as the PRM infrastructure should abstract this.
- */
-static void _reconfigure_io_chain(void)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&io_chain_lock, flags);
-
-	omap_prm_reconfigure_io_chain();
-
-	spin_unlock_irqrestore(&io_chain_lock, flags);
-}
-
-/**
  * _omap4_update_context_lost - increment hwmod context loss counter if
  * hwmod context was lost, and clear hardware context loss reg
  * @oh: hwmod to check for context loss
@@ -2109,18 +2037,9 @@ static int _enable(struct omap_hwmod *oh)
 
 	/*
 	 * hwmods with HWMOD_INIT_NO_IDLE flag set are left in enabled
-	 * state at init.  Now that someone is really trying to enable
-	 * them, just ensure that the hwmod mux is set.
+	 * state at init.
 	 */
 	if (oh->_int_flags & _HWMOD_SKIP_ENABLE) {
-		/*
-		 * If the caller has mux data populated, do the mux'ing
-		 * which wouldn't have been done as part of the _enable()
-		 * done during setup.
-		 */
-		if (oh->mux)
-			omap_hwmod_mux(oh->mux, _HWMOD_STATE_ENABLED);
-
 		oh->_int_flags &= ~_HWMOD_SKIP_ENABLE;
 		return 0;
 	}
@@ -2145,16 +2064,6 @@ static int _enable(struct omap_hwmod *oh)
 	if (_are_all_hardreset_lines_asserted(oh))
 		return 0;
 
-	/* Mux pins for device runtime if populated */
-	if (oh->mux && (!oh->mux->enabled ||
-			((oh->_state == _HWMOD_STATE_IDLE) &&
-			 oh->mux->pads_dynamic))) {
-		omap_hwmod_mux(oh->mux, _HWMOD_STATE_ENABLED);
-		_reconfigure_io_chain();
-	} else if (oh->flags & HWMOD_RECONFIG_IO_CHAIN) {
-		_reconfigure_io_chain();
-	}
-
 	_add_initiator_dep(oh, mpu_oh);
 
 	if (oh->clkdm) {
@@ -2260,14 +2169,6 @@ static int _idle(struct omap_hwmod *oh)
 		clkdm_hwmod_disable(oh->clkdm, oh);
 	}
 
-	/* Mux pins for device idle if populated */
-	if (oh->mux && oh->mux->pads_dynamic) {
-		omap_hwmod_mux(oh->mux, _HWMOD_STATE_IDLE);
-		_reconfigure_io_chain();
-	} else if (oh->flags & HWMOD_RECONFIG_IO_CHAIN) {
-		_reconfigure_io_chain();
-	}
-
 	oh->_state = _HWMOD_STATE_IDLE;
 
 	return 0;
@@ -2334,10 +2235,6 @@ static int _shutdown(struct omap_hwmod *oh)
 	for (i = 0; i < oh->rst_lines_cnt; i++)
 		_assert_hardreset(oh, oh->rst_lines[i].name);
 
-	/* Mux pins to safe mode or use populated off mode values */
-	if (oh->mux)
-		omap_hwmod_mux(oh->mux, _HWMOD_STATE_DISABLED);
-
 	oh->_state = _HWMOD_STATE_DISABLED;
 
 	return 0;
@@ -3729,7 +3626,6 @@ int omap_hwmod_enable_wakeup(struct omap_hwmod *oh)
 		_write_sysconfig(v, oh);
 	}
 
-	_set_idle_ioring_wakeup(oh, true);
 	spin_unlock_irqrestore(&oh->_lock, flags);
 
 	return 0;
@@ -3762,7 +3658,6 @@ int omap_hwmod_disable_wakeup(struct omap_hwmod *oh)
 		_write_sysconfig(v, oh);
 	}
 
-	_set_idle_ioring_wakeup(oh, false);
 	spin_unlock_irqrestore(&oh->_lock, flags);
 
 	return 0;
diff --git a/arch/arm/mach-omap2/omap_hwmod_2xxx_3xxx_interconnect_data.c b/arch/arm/mach-omap2/omap_hwmod_2xxx_3xxx_interconnect_data.c
index c1e98d589100..6d2e32462df9 100644
--- a/arch/arm/mach-omap2/omap_hwmod_2xxx_3xxx_interconnect_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_2xxx_3xxx_interconnect_data.c
@@ -17,156 +17,11 @@
 
 #include "omap_hwmod_common_data.h"
 
-struct omap_hwmod_addr_space omap2430_mmc1_addr_space[] = {
-	{
-		.pa_start	= 0x4809c000,
-		.pa_end		= 0x4809c1ff,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2430_mmc2_addr_space[] = {
-	{
-		.pa_start	= 0x480b4000,
-		.pa_end		= 0x480b41ff,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2_i2c1_addr_space[] = {
-	{
-		.pa_start	= 0x48070000,
-		.pa_end		= 0x48070000 + SZ_128 - 1,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2_i2c2_addr_space[] = {
-	{
-		.pa_start	= 0x48072000,
-		.pa_end		= 0x48072000 + SZ_128 - 1,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2_dss_addrs[] = {
-	{
-		.pa_start	= 0x48050000,
-		.pa_end		= 0x48050000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2_dss_dispc_addrs[] = {
-	{
-		.pa_start	= 0x48050400,
-		.pa_end		= 0x48050400 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2_dss_rfbi_addrs[] = {
-	{
-		.pa_start	= 0x48050800,
-		.pa_end		= 0x48050800 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2_dss_venc_addrs[] = {
-	{
-		.pa_start	= 0x48050C00,
-		.pa_end		= 0x48050C00 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2_timer10_addrs[] = {
-	{
-		.pa_start	= 0x48086000,
-		.pa_end		= 0x48086000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2_timer11_addrs[] = {
-	{
-		.pa_start	= 0x48088000,
-		.pa_end		= 0x48088000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2xxx_timer12_addrs[] = {
-	{
-		.pa_start	= 0x4808a000,
-		.pa_end		= 0x4808a000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2_mcspi1_addr_space[] = {
-	{
-		.pa_start	= 0x48098000,
-		.pa_end		= 0x48098000 + SZ_256 - 1,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2_mcspi2_addr_space[] = {
-	{
-		.pa_start	= 0x4809a000,
-		.pa_end		= 0x4809a000 + SZ_256 - 1,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2430_mcspi3_addr_space[] = {
-	{
-		.pa_start	= 0x480b8000,
-		.pa_end		= 0x480b8000 + SZ_256 - 1,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
-
 struct omap_hwmod_addr_space omap2_dma_system_addrs[] = {
 	{
 		.pa_start	= 0x48056000,
 		.pa_end		= 0x48056000 + SZ_4K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2_mcbsp1_addrs[] = {
-	{
-		.name		= "mpu",
-		.pa_start	= 0x48074000,
-		.pa_end		= 0x480740ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
-
-struct omap_hwmod_addr_space omap2_hdq1w_addr_space[] = {
-	{
-		.pa_start       = 0x480b2000,
-		.pa_end         = 0x480b2fff,
-		.flags          = ADDR_TYPE_RT,
+		.flags		= ADDR_TYPE_RT,
 	},
-	{ }
+	{ },
 };
diff --git a/arch/arm/mach-omap2/omap_hwmod_2xxx_3xxx_ipblock_data.c b/arch/arm/mach-omap2/omap_hwmod_2xxx_3xxx_ipblock_data.c
index c6c6384de867..cfaeb0f78cc8 100644
--- a/arch/arm/mach-omap2/omap_hwmod_2xxx_3xxx_ipblock_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_2xxx_3xxx_ipblock_data.c
@@ -45,204 +45,31 @@ struct omap_hwmod_class omap2_venc_hwmod_class = {
 	.name = "venc",
 };
 
-
-/* Common DMA request line data */
-struct omap_hwmod_dma_info omap2_uart1_sdma_reqs[] = {
-	{ .name = "rx", .dma_req = 50, },
-	{ .name = "tx", .dma_req = 49, },
-	{ .dma_req = -1 }
-};
-
-struct omap_hwmod_dma_info omap2_uart2_sdma_reqs[] = {
-	{ .name = "rx", .dma_req = 52, },
-	{ .name = "tx", .dma_req = 51, },
-	{ .dma_req = -1 }
-};
-
-struct omap_hwmod_dma_info omap2_uart3_sdma_reqs[] = {
-	{ .name = "rx", .dma_req = 54, },
-	{ .name = "tx", .dma_req = 53, },
-	{ .dma_req = -1 }
-};
-
-struct omap_hwmod_dma_info omap2_i2c1_sdma_reqs[] = {
-	{ .name = "tx", .dma_req = 27 },
-	{ .name = "rx", .dma_req = 28 },
-	{ .dma_req = -1 }
-};
-
-struct omap_hwmod_dma_info omap2_i2c2_sdma_reqs[] = {
-	{ .name = "tx", .dma_req = 29 },
-	{ .name = "rx", .dma_req = 30 },
-	{ .dma_req = -1 }
-};
-
-struct omap_hwmod_dma_info omap2_mcspi1_sdma_reqs[] = {
-	{ .name = "tx0", .dma_req = 35 }, /* DMA_SPI1_TX0 */
-	{ .name = "rx0", .dma_req = 36 }, /* DMA_SPI1_RX0 */
-	{ .name = "tx1", .dma_req = 37 }, /* DMA_SPI1_TX1 */
-	{ .name = "rx1", .dma_req = 38 }, /* DMA_SPI1_RX1 */
-	{ .name = "tx2", .dma_req = 39 }, /* DMA_SPI1_TX2 */
-	{ .name = "rx2", .dma_req = 40 }, /* DMA_SPI1_RX2 */
-	{ .name = "tx3", .dma_req = 41 }, /* DMA_SPI1_TX3 */
-	{ .name = "rx3", .dma_req = 42 }, /* DMA_SPI1_RX3 */
-	{ .dma_req = -1 }
-};
-
-struct omap_hwmod_dma_info omap2_mcspi2_sdma_reqs[] = {
-	{ .name = "tx0", .dma_req = 43 }, /* DMA_SPI2_TX0 */
-	{ .name = "rx0", .dma_req = 44 }, /* DMA_SPI2_RX0 */
-	{ .name = "tx1", .dma_req = 45 }, /* DMA_SPI2_TX1 */
-	{ .name = "rx1", .dma_req = 46 }, /* DMA_SPI2_RX1 */
-	{ .dma_req = -1 }
-};
-
-struct omap_hwmod_dma_info omap2_mcbsp1_sdma_reqs[] = {
-	{ .name = "rx", .dma_req = 32 },
-	{ .name = "tx", .dma_req = 31 },
-	{ .dma_req = -1 }
-};
-
-struct omap_hwmod_dma_info omap2_mcbsp2_sdma_reqs[] = {
-	{ .name = "rx", .dma_req = 34 },
-	{ .name = "tx", .dma_req = 33 },
-	{ .dma_req = -1 }
-};
-
-struct omap_hwmod_dma_info omap2_mcbsp3_sdma_reqs[] = {
-	{ .name = "rx", .dma_req = 18 },
-	{ .name = "tx", .dma_req = 17 },
-	{ .dma_req = -1 }
-};
-
-/* Other IP block data */
-
-
 /*
  * omap_hwmod class data
  */
 
 struct omap_hwmod_class l3_hwmod_class = {
-	.name = "l3"
+	.name = "l3",
 };
 
 struct omap_hwmod_class l4_hwmod_class = {
-	.name = "l4"
+	.name = "l4",
 };
 
 struct omap_hwmod_class mpu_hwmod_class = {
-	.name = "mpu"
+	.name = "mpu",
 };
 
 struct omap_hwmod_class iva_hwmod_class = {
-	.name = "iva"
+	.name = "iva",
 };
 
 /* Common MPU IRQ line data */
 
-struct omap_hwmod_irq_info omap2_timer1_mpu_irqs[] = {
-	{ .irq = 37 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_timer2_mpu_irqs[] = {
-	{ .irq = 38 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_timer3_mpu_irqs[] = {
-	{ .irq = 39 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_timer4_mpu_irqs[] = {
-	{ .irq = 40 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_timer5_mpu_irqs[] = {
-	{ .irq = 41 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_timer6_mpu_irqs[] = {
-	{ .irq = 42 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_timer7_mpu_irqs[] = {
-	{ .irq = 43 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_timer8_mpu_irqs[] = {
-	{ .irq = 44 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_timer9_mpu_irqs[] = {
-	{ .irq = 45 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_timer10_mpu_irqs[] = {
-	{ .irq = 46 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_timer11_mpu_irqs[] = {
-	{ .irq = 47 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_uart1_mpu_irqs[] = {
-	{ .irq = 72 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_uart2_mpu_irqs[] = {
-	{ .irq = 73 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_uart3_mpu_irqs[] = {
-	{ .irq = 74 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
 struct omap_hwmod_irq_info omap2_dispc_irqs[] = {
 	{ .irq = 25 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_i2c1_mpu_irqs[] = {
-	{ .irq = 56 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_i2c2_mpu_irqs[] = {
-	{ .irq = 57 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_gpio1_irqs[] = {
-	{ .irq = 29 + OMAP_INTC_START, }, /* INT_24XX_GPIO_BANK1 */
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_gpio2_irqs[] = {
-	{ .irq = 30 + OMAP_INTC_START, }, /* INT_24XX_GPIO_BANK2 */
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_gpio3_irqs[] = {
-	{ .irq = 31 + OMAP_INTC_START, }, /* INT_24XX_GPIO_BANK3 */
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_gpio4_irqs[] = {
-	{ .irq = 32 + OMAP_INTC_START, }, /* INT_24XX_GPIO_BANK4 */
-	{ .irq = -1 },
+	{ .irq = -1, },
 };
 
 struct omap_hwmod_irq_info omap2_dma_system_irqs[] = {
@@ -250,17 +77,7 @@ struct omap_hwmod_irq_info omap2_dma_system_irqs[] = {
 	{ .name = "1", .irq = 13 + OMAP_INTC_START, }, /* INT_24XX_SDMA_IRQ1 */
 	{ .name = "2", .irq = 14 + OMAP_INTC_START, }, /* INT_24XX_SDMA_IRQ2 */
 	{ .name = "3", .irq = 15 + OMAP_INTC_START, }, /* INT_24XX_SDMA_IRQ3 */
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_mcspi1_mpu_irqs[] = {
-	{ .irq = 65 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
-struct omap_hwmod_irq_info omap2_mcspi2_mpu_irqs[] = {
-	{ .irq = 66 + OMAP_INTC_START, },
-	{ .irq = -1 },
+	{ .irq = -1, },
 };
 
 struct omap_hwmod_class_sysconfig omap2_hdq1w_sysc = {
@@ -277,9 +94,3 @@ struct omap_hwmod_class omap2_hdq1w_class = {
 	.sysc	= &omap2_hdq1w_sysc,
 	.reset	= &omap_hdq1w_reset,
 };
-
-struct omap_hwmod_irq_info omap2_hdq1w_mpu_irqs[] = {
-	{ .irq = 58 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
diff --git a/arch/arm/mach-omap2/omap_hwmod_2xxx_interconnect_data.c b/arch/arm/mach-omap2/omap_hwmod_2xxx_interconnect_data.c
index 656861c29d5c..9b30b6b471ae 100644
--- a/arch/arm/mach-omap2/omap_hwmod_2xxx_interconnect_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_2xxx_interconnect_data.c
@@ -191,7 +191,6 @@ struct omap_hwmod_ocp_if omap2xxx_l4_core__dss = {
 	.master		= &omap2xxx_l4_core_hwmod,
 	.slave		= &omap2xxx_dss_core_hwmod,
 	.clk		= "dss_ick",
-	.addr		= omap2_dss_addrs,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP2420_L4_CORE_FW_DSS_CORE_REGION,
@@ -206,7 +205,6 @@ struct omap_hwmod_ocp_if omap2xxx_l4_core__dss_dispc = {
 	.master		= &omap2xxx_l4_core_hwmod,
 	.slave		= &omap2xxx_dss_dispc_hwmod,
 	.clk		= "dss_ick",
-	.addr		= omap2_dss_dispc_addrs,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP2420_L4_CORE_FW_DSS_DISPC_REGION,
@@ -221,7 +219,6 @@ struct omap_hwmod_ocp_if omap2xxx_l4_core__dss_rfbi = {
 	.master		= &omap2xxx_l4_core_hwmod,
 	.slave		= &omap2xxx_dss_rfbi_hwmod,
 	.clk		= "dss_ick",
-	.addr		= omap2_dss_rfbi_addrs,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP2420_L4_CORE_FW_DSS_CORE_REGION,
@@ -236,7 +233,6 @@ struct omap_hwmod_ocp_if omap2xxx_l4_core__dss_venc = {
 	.master		= &omap2xxx_l4_core_hwmod,
 	.slave		= &omap2xxx_dss_venc_hwmod,
 	.clk		= "dss_ick",
-	.addr		= omap2_dss_venc_addrs,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP2420_L4_CORE_FW_DSS_VENC_REGION,
diff --git a/arch/arm/mach-omap2/omap_hwmod_2xxx_ipblock_data.c b/arch/arm/mach-omap2/omap_hwmod_2xxx_ipblock_data.c
index 36bcd2e75422..e047033caa3e 100644
--- a/arch/arm/mach-omap2/omap_hwmod_2xxx_ipblock_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_2xxx_ipblock_data.c
@@ -569,7 +569,6 @@ struct omap_hwmod omap2xxx_dss_core_hwmod = {
 struct omap_hwmod omap2xxx_dss_dispc_hwmod = {
 	.name		= "dss_dispc",
 	.class		= &omap2_dispc_hwmod_class,
-	.mpu_irqs	= omap2_dispc_irqs,
 	.main_clk	= "dss1_fck",
 	.prcm		= {
 		.omap2 = {
diff --git a/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_common_data.h b/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_common_data.h
index d3e61d1a02d7..434bd1a77229 100644
--- a/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_common_data.h
+++ b/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_common_data.h
@@ -68,6 +68,7 @@ extern struct omap_hwmod_ocp_if am33xx_l4_ls__uart6;
 extern struct omap_hwmod_ocp_if am33xx_l3_main__ocmc;
 extern struct omap_hwmod_ocp_if am33xx_l3_main__sha0;
 extern struct omap_hwmod_ocp_if am33xx_l3_main__aes0;
+extern struct omap_hwmod_ocp_if am33xx_l4_per__rng;
 
 extern struct omap_hwmod am33xx_l3_main_hwmod;
 extern struct omap_hwmod am33xx_l3_s_hwmod;
@@ -80,6 +81,7 @@ extern struct omap_hwmod am33xx_gfx_hwmod;
 extern struct omap_hwmod am33xx_prcm_hwmod;
 extern struct omap_hwmod am33xx_aes0_hwmod;
 extern struct omap_hwmod am33xx_sha0_hwmod;
+extern struct omap_hwmod am33xx_rng_hwmod;
 extern struct omap_hwmod am33xx_ocmcram_hwmod;
 extern struct omap_hwmod am33xx_smartreflex0_hwmod;
 extern struct omap_hwmod am33xx_smartreflex1_hwmod;
diff --git a/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_interconnect_data.c b/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_interconnect_data.c
index 10dff2f0086a..8236e5c49ec3 100644
--- a/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_interconnect_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_interconnect_data.c
@@ -547,3 +547,11 @@ struct omap_hwmod_ocp_if am33xx_l3_main__aes0 = {
 	.addr		= am33xx_aes0_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
+
+/* l4 per -> rng */
+struct omap_hwmod_ocp_if am33xx_l4_per__rng = {
+	.master		= &am33xx_l4_ls_hwmod,
+	.slave		= &am33xx_rng_hwmod,
+	.clk		= "rng_fck",
+	.user		= OCP_USER_MPU,
+};
diff --git a/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c b/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c
index e2d84aa7f595..de06a1d5ffab 100644
--- a/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_33xx_43xx_ipblock_data.c
@@ -268,6 +268,33 @@ struct omap_hwmod am33xx_sha0_hwmod = {
 	},
 };
 
+/* rng */
+static struct omap_hwmod_class_sysconfig am33xx_rng_sysc = {
+	.rev_offs	= 0x1fe0,
+	.sysc_offs	= 0x1fe4,
+	.sysc_flags	= SYSC_HAS_AUTOIDLE | SYSC_HAS_SIDLEMODE,
+	.idlemodes	= SIDLE_FORCE | SIDLE_NO,
+	.sysc_fields	= &omap_hwmod_sysc_type1,
+};
+
+static struct omap_hwmod_class am33xx_rng_hwmod_class = {
+	.name		= "rng",
+	.sysc		= &am33xx_rng_sysc,
+};
+
+struct omap_hwmod am33xx_rng_hwmod = {
+	.name		= "rng",
+	.class		= &am33xx_rng_hwmod_class,
+	.clkdm_name	= "l4ls_clkdm",
+	.flags		= HWMOD_SWSUP_SIDLE,
+	.main_clk	= "rng_fck",
+	.prcm		= {
+		.omap4	= {
+			.modulemode	= MODULEMODE_SWCTRL,
+		},
+	},
+};
+
 /* ocmcram */
 static struct omap_hwmod_class am33xx_ocmcram_hwmod_class = {
 	.name = "ocmcram",
@@ -1315,6 +1342,7 @@ static void omap_hwmod_am33xx_clkctrl(void)
 	CLKCTRL(am33xx_ocmcram_hwmod , AM33XX_CM_PER_OCMCRAM_CLKCTRL_OFFSET);
 	CLKCTRL(am33xx_sha0_hwmod , AM33XX_CM_PER_SHA0_CLKCTRL_OFFSET);
 	CLKCTRL(am33xx_aes0_hwmod , AM33XX_CM_PER_AES0_CLKCTRL_OFFSET);
+	CLKCTRL(am33xx_rng_hwmod, AM33XX_CM_PER_RNG_CLKCTRL_OFFSET);
 }
 
 static void omap_hwmod_am33xx_rst(void)
@@ -1388,6 +1416,7 @@ static void omap_hwmod_am43xx_clkctrl(void)
 	CLKCTRL(am33xx_ocmcram_hwmod , AM43XX_CM_PER_OCMCRAM_CLKCTRL_OFFSET);
 	CLKCTRL(am33xx_sha0_hwmod , AM43XX_CM_PER_SHA0_CLKCTRL_OFFSET);
 	CLKCTRL(am33xx_aes0_hwmod , AM43XX_CM_PER_AES0_CLKCTRL_OFFSET);
+	CLKCTRL(am33xx_rng_hwmod, AM43XX_CM_PER_RNG_CLKCTRL_OFFSET);
 }
 
 static void omap_hwmod_am43xx_rst(void)
diff --git a/arch/arm/mach-omap2/omap_hwmod_33xx_data.c b/arch/arm/mach-omap2/omap_hwmod_33xx_data.c
index e1c2025d6d3e..6dc51a774a26 100644
--- a/arch/arm/mach-omap2/omap_hwmod_33xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_33xx_data.c
@@ -503,41 +503,6 @@ static struct omap_hwmod_ocp_if am33xx_l3_s__usbss = {
 	.flags		= OCPIF_SWSUP_IDLE,
 };
 
-/* rng */
-static struct omap_hwmod_class_sysconfig am33xx_rng_sysc = {
-	.rev_offs	= 0x1fe0,
-	.sysc_offs	= 0x1fe4,
-	.sysc_flags	= SYSC_HAS_AUTOIDLE | SYSC_HAS_SIDLEMODE,
-	.idlemodes	= SIDLE_FORCE | SIDLE_NO,
-	.sysc_fields	= &omap_hwmod_sysc_type1,
-};
-
-static struct omap_hwmod_class am33xx_rng_hwmod_class = {
-	.name		= "rng",
-	.sysc		= &am33xx_rng_sysc,
-};
-
-static struct omap_hwmod am33xx_rng_hwmod = {
-	.name		= "rng",
-	.class		= &am33xx_rng_hwmod_class,
-	.clkdm_name	= "l4ls_clkdm",
-	.flags		= HWMOD_SWSUP_SIDLE,
-	.main_clk	= "rng_fck",
-	.prcm		= {
-		.omap4	= {
-			.clkctrl_offs	= AM33XX_CM_PER_RNG_CLKCTRL_OFFSET,
-			.modulemode	= MODULEMODE_SWCTRL,
-		},
-	},
-};
-
-static struct omap_hwmod_ocp_if am33xx_l4_per__rng = {
-	.master		= &am33xx_l4_ls_hwmod,
-	.slave		= &am33xx_rng_hwmod,
-	.clk		= "rng_fck",
-	.user		= OCP_USER_MPU,
-};
-
 static struct omap_hwmod_ocp_if *am33xx_hwmod_ocp_ifs[] __initdata = {
 	&am33xx_l3_main__emif,
 	&am33xx_mpu__l3_main,
diff --git a/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c b/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c
index 1cc4a6f3954e..56f917ec8621 100644
--- a/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c
@@ -53,16 +53,10 @@
  */
 
 /* L3 */
-static struct omap_hwmod_irq_info omap3xxx_l3_main_irqs[] = {
-	{ .irq = 9 + OMAP_INTC_START, },
-	{ .irq = 10 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod omap3xxx_l3_main_hwmod = {
 	.name		= "l3_main",
 	.class		= &l3_hwmod_class,
-	.mpu_irqs	= omap3xxx_l3_main_irqs,
 	.flags		= HWMOD_NO_IDLEST,
 };
 
@@ -95,14 +89,9 @@ static struct omap_hwmod omap3xxx_l4_sec_hwmod = {
 };
 
 /* MPU */
-static struct omap_hwmod_irq_info omap3xxx_mpu_irqs[] = {
-	{ .name = "pmu", .irq = 3 + OMAP_INTC_START },
-	{ .irq = -1 }
-};
 
 static struct omap_hwmod omap3xxx_mpu_hwmod = {
 	.name		= "mpu",
-	.mpu_irqs	= omap3xxx_mpu_irqs,
 	.class		= &mpu_hwmod_class,
 	.main_clk	= "arm_fck",
 };
@@ -128,7 +117,7 @@ static struct omap_hwmod omap3xxx_iva_hwmod = {
 			.module_bit = OMAP3430_CM_FCLKEN_IVA2_EN_IVA2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_IVA2_SHIFT,
-		}
+		},
 	},
 };
 
@@ -197,7 +186,6 @@ static struct omap_timer_capability_dev_attr capability_dsp_pwm_dev_attr = {
 /* timer1 */
 static struct omap_hwmod omap3xxx_timer1_hwmod = {
 	.name		= "timer1",
-	.mpu_irqs	= omap2_timer1_mpu_irqs,
 	.main_clk	= "gpt1_fck",
 	.prcm		= {
 		.omap2 = {
@@ -216,7 +204,6 @@ static struct omap_hwmod omap3xxx_timer1_hwmod = {
 /* timer2 */
 static struct omap_hwmod omap3xxx_timer2_hwmod = {
 	.name		= "timer2",
-	.mpu_irqs	= omap2_timer2_mpu_irqs,
 	.main_clk	= "gpt2_fck",
 	.prcm		= {
 		.omap2 = {
@@ -234,7 +221,6 @@ static struct omap_hwmod omap3xxx_timer2_hwmod = {
 /* timer3 */
 static struct omap_hwmod omap3xxx_timer3_hwmod = {
 	.name		= "timer3",
-	.mpu_irqs	= omap2_timer3_mpu_irqs,
 	.main_clk	= "gpt3_fck",
 	.prcm		= {
 		.omap2 = {
@@ -252,7 +238,6 @@ static struct omap_hwmod omap3xxx_timer3_hwmod = {
 /* timer4 */
 static struct omap_hwmod omap3xxx_timer4_hwmod = {
 	.name		= "timer4",
-	.mpu_irqs	= omap2_timer4_mpu_irqs,
 	.main_clk	= "gpt4_fck",
 	.prcm		= {
 		.omap2 = {
@@ -270,7 +255,6 @@ static struct omap_hwmod omap3xxx_timer4_hwmod = {
 /* timer5 */
 static struct omap_hwmod omap3xxx_timer5_hwmod = {
 	.name		= "timer5",
-	.mpu_irqs	= omap2_timer5_mpu_irqs,
 	.main_clk	= "gpt5_fck",
 	.prcm		= {
 		.omap2 = {
@@ -289,7 +273,6 @@ static struct omap_hwmod omap3xxx_timer5_hwmod = {
 /* timer6 */
 static struct omap_hwmod omap3xxx_timer6_hwmod = {
 	.name		= "timer6",
-	.mpu_irqs	= omap2_timer6_mpu_irqs,
 	.main_clk	= "gpt6_fck",
 	.prcm		= {
 		.omap2 = {
@@ -308,7 +291,6 @@ static struct omap_hwmod omap3xxx_timer6_hwmod = {
 /* timer7 */
 static struct omap_hwmod omap3xxx_timer7_hwmod = {
 	.name		= "timer7",
-	.mpu_irqs	= omap2_timer7_mpu_irqs,
 	.main_clk	= "gpt7_fck",
 	.prcm		= {
 		.omap2 = {
@@ -327,7 +309,6 @@ static struct omap_hwmod omap3xxx_timer7_hwmod = {
 /* timer8 */
 static struct omap_hwmod omap3xxx_timer8_hwmod = {
 	.name		= "timer8",
-	.mpu_irqs	= omap2_timer8_mpu_irqs,
 	.main_clk	= "gpt8_fck",
 	.prcm		= {
 		.omap2 = {
@@ -346,7 +327,6 @@ static struct omap_hwmod omap3xxx_timer8_hwmod = {
 /* timer9 */
 static struct omap_hwmod omap3xxx_timer9_hwmod = {
 	.name		= "timer9",
-	.mpu_irqs	= omap2_timer9_mpu_irqs,
 	.main_clk	= "gpt9_fck",
 	.prcm		= {
 		.omap2 = {
@@ -365,7 +345,6 @@ static struct omap_hwmod omap3xxx_timer9_hwmod = {
 /* timer10 */
 static struct omap_hwmod omap3xxx_timer10_hwmod = {
 	.name		= "timer10",
-	.mpu_irqs	= omap2_timer10_mpu_irqs,
 	.main_clk	= "gpt10_fck",
 	.prcm		= {
 		.omap2 = {
@@ -384,7 +363,6 @@ static struct omap_hwmod omap3xxx_timer10_hwmod = {
 /* timer11 */
 static struct omap_hwmod omap3xxx_timer11_hwmod = {
 	.name		= "timer11",
-	.mpu_irqs	= omap2_timer11_mpu_irqs,
 	.main_clk	= "gpt11_fck",
 	.prcm		= {
 		.omap2 = {
@@ -401,14 +379,9 @@ static struct omap_hwmod omap3xxx_timer11_hwmod = {
 };
 
 /* timer12 */
-static struct omap_hwmod_irq_info omap3xxx_timer12_mpu_irqs[] = {
-	{ .irq = 95 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod omap3xxx_timer12_hwmod = {
 	.name		= "timer12",
-	.mpu_irqs	= omap3xxx_timer12_mpu_irqs,
 	.main_clk	= "gpt12_fck",
 	.prcm		= {
 		.omap2 = {
@@ -485,8 +458,6 @@ static struct omap_hwmod omap3xxx_wd_timer2_hwmod = {
 /* UART1 */
 static struct omap_hwmod omap3xxx_uart1_hwmod = {
 	.name		= "uart1",
-	.mpu_irqs	= omap2_uart1_mpu_irqs,
-	.sdma_reqs	= omap2_uart1_sdma_reqs,
 	.main_clk	= "uart1_fck",
 	.flags		= DEBUG_TI81XXUART1_FLAGS | HWMOD_SWSUP_SIDLE,
 	.prcm		= {
@@ -504,8 +475,6 @@ static struct omap_hwmod omap3xxx_uart1_hwmod = {
 /* UART2 */
 static struct omap_hwmod omap3xxx_uart2_hwmod = {
 	.name		= "uart2",
-	.mpu_irqs	= omap2_uart2_mpu_irqs,
-	.sdma_reqs	= omap2_uart2_sdma_reqs,
 	.main_clk	= "uart2_fck",
 	.flags		= DEBUG_TI81XXUART2_FLAGS | HWMOD_SWSUP_SIDLE,
 	.prcm		= {
@@ -523,8 +492,6 @@ static struct omap_hwmod omap3xxx_uart2_hwmod = {
 /* UART3 */
 static struct omap_hwmod omap3xxx_uart3_hwmod = {
 	.name		= "uart3",
-	.mpu_irqs	= omap2_uart3_mpu_irqs,
-	.sdma_reqs	= omap2_uart3_sdma_reqs,
 	.main_clk	= "uart3_fck",
 	.flags		= DEBUG_OMAP3UART3_FLAGS | DEBUG_TI81XXUART3_FLAGS |
 				HWMOD_SWSUP_SIDLE,
@@ -541,21 +508,10 @@ static struct omap_hwmod omap3xxx_uart3_hwmod = {
 };
 
 /* UART4 */
-static struct omap_hwmod_irq_info uart4_mpu_irqs[] = {
-	{ .irq = 80 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
-static struct omap_hwmod_dma_info uart4_sdma_reqs[] = {
-	{ .name = "rx",	.dma_req = 82, },
-	{ .name = "tx",	.dma_req = 81, },
-	{ .dma_req = -1 }
-};
 
 static struct omap_hwmod omap36xx_uart4_hwmod = {
 	.name		= "uart4",
-	.mpu_irqs	= uart4_mpu_irqs,
-	.sdma_reqs	= uart4_sdma_reqs,
 	.main_clk	= "uart4_fck",
 	.flags		= DEBUG_OMAP3UART4_FLAGS | HWMOD_SWSUP_SIDLE,
 	.prcm		= {
@@ -570,16 +526,7 @@ static struct omap_hwmod omap36xx_uart4_hwmod = {
 	.class		= &omap2_uart_class,
 };
 
-static struct omap_hwmod_irq_info am35xx_uart4_mpu_irqs[] = {
-	{ .irq = 84 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
-static struct omap_hwmod_dma_info am35xx_uart4_sdma_reqs[] = {
-	{ .name = "rx", .dma_req = 55, },
-	{ .name = "tx", .dma_req = 54, },
-	{ .dma_req = -1 }
-};
 
 /*
  * XXX AM35xx UART4 cannot complete its softreset without uart1_fck or
@@ -597,8 +544,6 @@ static struct omap_hwmod_opt_clk am35xx_uart4_opt_clks[] = {
 
 static struct omap_hwmod am35xx_uart4_hwmod = {
 	.name		= "uart4",
-	.mpu_irqs	= am35xx_uart4_mpu_irqs,
-	.sdma_reqs	= am35xx_uart4_sdma_reqs,
 	.main_clk	= "uart4_fck",
 	.prcm		= {
 		.omap2 = {
@@ -625,7 +570,7 @@ static struct omap_hwmod_class i2c_class = {
 static struct omap_hwmod_dma_info omap3xxx_dss_sdma_chs[] = {
 	{ .name = "dispc", .dma_req = 5 },
 	{ .name = "dsi1", .dma_req = 74 },
-	{ .dma_req = -1 }
+	{ .dma_req = -1, },
 };
 
 /* dss */
@@ -714,7 +659,7 @@ static struct omap_hwmod omap3xxx_dss_dispc_hwmod = {
 		},
 	},
 	.flags		= HWMOD_NO_IDLEST,
-	.dev_attr	= &omap2_3_dss_dispc_dev_attr
+	.dev_attr	= &omap2_3_dss_dispc_dev_attr,
 };
 
 /*
@@ -738,11 +683,6 @@ static struct omap_hwmod_class omap3xxx_dsi_hwmod_class = {
 	.sysc	= &omap3xxx_dsi_sysc,
 };
 
-static struct omap_hwmod_irq_info omap3xxx_dsi1_irqs[] = {
-	{ .irq = 25 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
-
 /* dss_dsi1 */
 static struct omap_hwmod_opt_clk dss_dsi1_opt_clks[] = {
 	{ .role = "sys_clk", .clk = "dss2_alwon_fck" },
@@ -751,7 +691,6 @@ static struct omap_hwmod_opt_clk dss_dsi1_opt_clks[] = {
 static struct omap_hwmod omap3xxx_dss_dsi1_hwmod = {
 	.name		= "dss_dsi1",
 	.class		= &omap3xxx_dsi_hwmod_class,
-	.mpu_irqs	= omap3xxx_dsi1_irqs,
 	.main_clk	= "dss1_alwon_fck",
 	.prcm		= {
 		.omap2 = {
@@ -815,8 +754,6 @@ static struct omap_i2c_dev_attr i2c1_dev_attr = {
 static struct omap_hwmod omap3xxx_i2c1_hwmod = {
 	.name		= "i2c1",
 	.flags		= HWMOD_16BIT_REG | HWMOD_SET_DEFAULT_CLOCKACT,
-	.mpu_irqs	= omap2_i2c1_mpu_irqs,
-	.sdma_reqs	= omap2_i2c1_sdma_reqs,
 	.main_clk	= "i2c1_fck",
 	.prcm		= {
 		.omap2 = {
@@ -840,8 +777,6 @@ static struct omap_i2c_dev_attr i2c2_dev_attr = {
 static struct omap_hwmod omap3xxx_i2c2_hwmod = {
 	.name		= "i2c2",
 	.flags		= HWMOD_16BIT_REG | HWMOD_SET_DEFAULT_CLOCKACT,
-	.mpu_irqs	= omap2_i2c2_mpu_irqs,
-	.sdma_reqs	= omap2_i2c2_sdma_reqs,
 	.main_clk	= "i2c2_fck",
 	.prcm		= {
 		.omap2 = {
@@ -862,22 +797,11 @@ static struct omap_i2c_dev_attr i2c3_dev_attr = {
 	.flags = OMAP_I2C_FLAG_BUS_SHIFT_2,
 };
 
-static struct omap_hwmod_irq_info i2c3_mpu_irqs[] = {
-	{ .irq = 61 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
-static struct omap_hwmod_dma_info i2c3_sdma_reqs[] = {
-	{ .name = "tx", .dma_req = 25 },
-	{ .name = "rx", .dma_req = 26 },
-	{ .dma_req = -1 }
-};
 
 static struct omap_hwmod omap3xxx_i2c3_hwmod = {
 	.name		= "i2c3",
 	.flags		= HWMOD_16BIT_REG | HWMOD_SET_DEFAULT_CLOCKACT,
-	.mpu_irqs	= i2c3_mpu_irqs,
-	.sdma_reqs	= i2c3_sdma_reqs,
 	.main_clk	= "i2c3_fck",
 	.prcm		= {
 		.omap2 = {
@@ -928,7 +852,6 @@ static struct omap_hwmod_opt_clk gpio1_opt_clks[] = {
 static struct omap_hwmod omap3xxx_gpio1_hwmod = {
 	.name		= "gpio1",
 	.flags		= HWMOD_CONTROL_OPT_CLKS_IN_RESET,
-	.mpu_irqs	= omap2_gpio1_irqs,
 	.main_clk	= "gpio1_ick",
 	.opt_clks	= gpio1_opt_clks,
 	.opt_clks_cnt	= ARRAY_SIZE(gpio1_opt_clks),
@@ -953,7 +876,6 @@ static struct omap_hwmod_opt_clk gpio2_opt_clks[] = {
 static struct omap_hwmod omap3xxx_gpio2_hwmod = {
 	.name		= "gpio2",
 	.flags		= HWMOD_CONTROL_OPT_CLKS_IN_RESET,
-	.mpu_irqs	= omap2_gpio2_irqs,
 	.main_clk	= "gpio2_ick",
 	.opt_clks	= gpio2_opt_clks,
 	.opt_clks_cnt	= ARRAY_SIZE(gpio2_opt_clks),
@@ -978,7 +900,6 @@ static struct omap_hwmod_opt_clk gpio3_opt_clks[] = {
 static struct omap_hwmod omap3xxx_gpio3_hwmod = {
 	.name		= "gpio3",
 	.flags		= HWMOD_CONTROL_OPT_CLKS_IN_RESET,
-	.mpu_irqs	= omap2_gpio3_irqs,
 	.main_clk	= "gpio3_ick",
 	.opt_clks	= gpio3_opt_clks,
 	.opt_clks_cnt	= ARRAY_SIZE(gpio3_opt_clks),
@@ -1003,7 +924,6 @@ static struct omap_hwmod_opt_clk gpio4_opt_clks[] = {
 static struct omap_hwmod omap3xxx_gpio4_hwmod = {
 	.name		= "gpio4",
 	.flags		= HWMOD_CONTROL_OPT_CLKS_IN_RESET,
-	.mpu_irqs	= omap2_gpio4_irqs,
 	.main_clk	= "gpio4_ick",
 	.opt_clks	= gpio4_opt_clks,
 	.opt_clks_cnt	= ARRAY_SIZE(gpio4_opt_clks),
@@ -1021,10 +941,6 @@ static struct omap_hwmod omap3xxx_gpio4_hwmod = {
 };
 
 /* gpio5 */
-static struct omap_hwmod_irq_info omap3xxx_gpio5_irqs[] = {
-	{ .irq = 33 + OMAP_INTC_START, }, /* INT_34XX_GPIO_BANK5 */
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod_opt_clk gpio5_opt_clks[] = {
 	{ .role = "dbclk", .clk = "gpio5_dbck", },
@@ -1033,7 +949,6 @@ static struct omap_hwmod_opt_clk gpio5_opt_clks[] = {
 static struct omap_hwmod omap3xxx_gpio5_hwmod = {
 	.name		= "gpio5",
 	.flags		= HWMOD_CONTROL_OPT_CLKS_IN_RESET,
-	.mpu_irqs	= omap3xxx_gpio5_irqs,
 	.main_clk	= "gpio5_ick",
 	.opt_clks	= gpio5_opt_clks,
 	.opt_clks_cnt	= ARRAY_SIZE(gpio5_opt_clks),
@@ -1051,10 +966,6 @@ static struct omap_hwmod omap3xxx_gpio5_hwmod = {
 };
 
 /* gpio6 */
-static struct omap_hwmod_irq_info omap3xxx_gpio6_irqs[] = {
-	{ .irq = 34 + OMAP_INTC_START, }, /* INT_34XX_GPIO_BANK6 */
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod_opt_clk gpio6_opt_clks[] = {
 	{ .role = "dbclk", .clk = "gpio6_dbck", },
@@ -1063,7 +974,6 @@ static struct omap_hwmod_opt_clk gpio6_opt_clks[] = {
 static struct omap_hwmod omap3xxx_gpio6_hwmod = {
 	.name		= "gpio6",
 	.flags		= HWMOD_CONTROL_OPT_CLKS_IN_RESET,
-	.mpu_irqs	= omap3xxx_gpio6_irqs,
 	.main_clk	= "gpio6_ick",
 	.opt_clks	= gpio6_opt_clks,
 	.opt_clks_cnt	= ARRAY_SIZE(gpio6_opt_clks),
@@ -1156,18 +1066,10 @@ static struct omap_hwmod_opt_clk mcbsp234_opt_clks[] = {
 };
 
 /* mcbsp1 */
-static struct omap_hwmod_irq_info omap3xxx_mcbsp1_irqs[] = {
-	{ .name = "common", .irq = 16 + OMAP_INTC_START, },
-	{ .name = "tx", .irq = 59 + OMAP_INTC_START, },
-	{ .name = "rx", .irq = 60 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod omap3xxx_mcbsp1_hwmod = {
 	.name		= "mcbsp1",
 	.class		= &omap3xxx_mcbsp_hwmod_class,
-	.mpu_irqs	= omap3xxx_mcbsp1_irqs,
-	.sdma_reqs	= omap2_mcbsp1_sdma_reqs,
 	.main_clk	= "mcbsp1_fck",
 	.prcm		= {
 		.omap2 = {
@@ -1183,12 +1085,6 @@ static struct omap_hwmod omap3xxx_mcbsp1_hwmod = {
 };
 
 /* mcbsp2 */
-static struct omap_hwmod_irq_info omap3xxx_mcbsp2_irqs[] = {
-	{ .name = "common", .irq = 17 + OMAP_INTC_START, },
-	{ .name = "tx", .irq = 62 + OMAP_INTC_START, },
-	{ .name = "rx", .irq = 63 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_mcbsp_dev_attr omap34xx_mcbsp2_dev_attr = {
 	.sidetone	= "mcbsp2_sidetone",
@@ -1197,8 +1093,6 @@ static struct omap_mcbsp_dev_attr omap34xx_mcbsp2_dev_attr = {
 static struct omap_hwmod omap3xxx_mcbsp2_hwmod = {
 	.name		= "mcbsp2",
 	.class		= &omap3xxx_mcbsp_hwmod_class,
-	.mpu_irqs	= omap3xxx_mcbsp2_irqs,
-	.sdma_reqs	= omap2_mcbsp2_sdma_reqs,
 	.main_clk	= "mcbsp2_fck",
 	.prcm		= {
 		.omap2 = {
@@ -1215,12 +1109,6 @@ static struct omap_hwmod omap3xxx_mcbsp2_hwmod = {
 };
 
 /* mcbsp3 */
-static struct omap_hwmod_irq_info omap3xxx_mcbsp3_irqs[] = {
-	{ .name = "common", .irq = 22 + OMAP_INTC_START, },
-	{ .name = "tx", .irq = 89 + OMAP_INTC_START, },
-	{ .name = "rx", .irq = 90 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_mcbsp_dev_attr omap34xx_mcbsp3_dev_attr = {
 	.sidetone	= "mcbsp3_sidetone",
@@ -1229,8 +1117,6 @@ static struct omap_mcbsp_dev_attr omap34xx_mcbsp3_dev_attr = {
 static struct omap_hwmod omap3xxx_mcbsp3_hwmod = {
 	.name		= "mcbsp3",
 	.class		= &omap3xxx_mcbsp_hwmod_class,
-	.mpu_irqs	= omap3xxx_mcbsp3_irqs,
-	.sdma_reqs	= omap2_mcbsp3_sdma_reqs,
 	.main_clk	= "mcbsp3_fck",
 	.prcm		= {
 		.omap2 = {
@@ -1247,24 +1133,11 @@ static struct omap_hwmod omap3xxx_mcbsp3_hwmod = {
 };
 
 /* mcbsp4 */
-static struct omap_hwmod_irq_info omap3xxx_mcbsp4_irqs[] = {
-	{ .name = "common", .irq = 23 + OMAP_INTC_START, },
-	{ .name = "tx", .irq = 54 + OMAP_INTC_START, },
-	{ .name = "rx", .irq = 55 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
-static struct omap_hwmod_dma_info omap3xxx_mcbsp4_sdma_chs[] = {
-	{ .name = "rx", .dma_req = 20 },
-	{ .name = "tx", .dma_req = 19 },
-	{ .dma_req = -1 }
-};
 
 static struct omap_hwmod omap3xxx_mcbsp4_hwmod = {
 	.name		= "mcbsp4",
 	.class		= &omap3xxx_mcbsp_hwmod_class,
-	.mpu_irqs	= omap3xxx_mcbsp4_irqs,
-	.sdma_reqs	= omap3xxx_mcbsp4_sdma_chs,
 	.main_clk	= "mcbsp4_fck",
 	.prcm		= {
 		.omap2 = {
@@ -1280,24 +1153,11 @@ static struct omap_hwmod omap3xxx_mcbsp4_hwmod = {
 };
 
 /* mcbsp5 */
-static struct omap_hwmod_irq_info omap3xxx_mcbsp5_irqs[] = {
-	{ .name = "common", .irq = 27 + OMAP_INTC_START, },
-	{ .name = "tx", .irq = 81 + OMAP_INTC_START, },
-	{ .name = "rx", .irq = 82 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
-static struct omap_hwmod_dma_info omap3xxx_mcbsp5_sdma_chs[] = {
-	{ .name = "rx", .dma_req = 22 },
-	{ .name = "tx", .dma_req = 21 },
-	{ .dma_req = -1 }
-};
 
 static struct omap_hwmod omap3xxx_mcbsp5_hwmod = {
 	.name		= "mcbsp5",
 	.class		= &omap3xxx_mcbsp_hwmod_class,
-	.mpu_irqs	= omap3xxx_mcbsp5_irqs,
-	.sdma_reqs	= omap3xxx_mcbsp5_sdma_chs,
 	.main_clk	= "mcbsp5_fck",
 	.prcm		= {
 		.omap2 = {
@@ -1325,29 +1185,19 @@ static struct omap_hwmod_class omap3xxx_mcbsp_sidetone_hwmod_class = {
 };
 
 /* mcbsp2_sidetone */
-static struct omap_hwmod_irq_info omap3xxx_mcbsp2_sidetone_irqs[] = {
-	{ .name = "irq", .irq = 4 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod omap3xxx_mcbsp2_sidetone_hwmod = {
 	.name		= "mcbsp2_sidetone",
 	.class		= &omap3xxx_mcbsp_sidetone_hwmod_class,
-	.mpu_irqs	= omap3xxx_mcbsp2_sidetone_irqs,
 	.main_clk	= "mcbsp2_ick",
 	.flags		= HWMOD_NO_IDLEST,
 };
 
 /* mcbsp3_sidetone */
-static struct omap_hwmod_irq_info omap3xxx_mcbsp3_sidetone_irqs[] = {
-	{ .name = "irq", .irq = 5 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod omap3xxx_mcbsp3_sidetone_hwmod = {
 	.name		= "mcbsp3_sidetone",
 	.class		= &omap3xxx_mcbsp_sidetone_hwmod_class,
-	.mpu_irqs	= omap3xxx_mcbsp3_sidetone_irqs,
 	.main_clk	= "mcbsp3_ick",
 	.flags		= HWMOD_NO_IDLEST,
 };
@@ -1394,10 +1244,6 @@ static struct omap_smartreflex_dev_attr sr1_dev_attr = {
 	.sensor_voltdm_name   = "mpu_iva",
 };
 
-static struct omap_hwmod_irq_info omap3_smartreflex_mpu_irqs[] = {
-	{ .irq = 18 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod omap34xx_sr1_hwmod = {
 	.name		= "smartreflex_mpu_iva",
@@ -1413,7 +1259,6 @@ static struct omap_hwmod omap34xx_sr1_hwmod = {
 		},
 	},
 	.dev_attr	= &sr1_dev_attr,
-	.mpu_irqs	= omap3_smartreflex_mpu_irqs,
 	.flags		= HWMOD_SET_DEFAULT_CLOCKACT,
 };
 
@@ -1431,7 +1276,6 @@ static struct omap_hwmod omap36xx_sr1_hwmod = {
 		},
 	},
 	.dev_attr	= &sr1_dev_attr,
-	.mpu_irqs	= omap3_smartreflex_mpu_irqs,
 };
 
 /* SR2 */
@@ -1439,10 +1283,6 @@ static struct omap_smartreflex_dev_attr sr2_dev_attr = {
 	.sensor_voltdm_name	= "core",
 };
 
-static struct omap_hwmod_irq_info omap3_smartreflex_core_irqs[] = {
-	{ .irq = 19 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod omap34xx_sr2_hwmod = {
 	.name		= "smartreflex_core",
@@ -1458,7 +1298,6 @@ static struct omap_hwmod omap34xx_sr2_hwmod = {
 		},
 	},
 	.dev_attr	= &sr2_dev_attr,
-	.mpu_irqs	= omap3_smartreflex_core_irqs,
 	.flags		= HWMOD_SET_DEFAULT_CLOCKACT,
 };
 
@@ -1476,7 +1315,6 @@ static struct omap_hwmod omap36xx_sr2_hwmod = {
 		},
 	},
 	.dev_attr	= &sr2_dev_attr,
-	.mpu_irqs	= omap3_smartreflex_core_irqs,
 };
 
 /*
@@ -1545,8 +1383,6 @@ static struct omap2_mcspi_dev_attr omap_mcspi1_dev_attr = {
 
 static struct omap_hwmod omap34xx_mcspi1 = {
 	.name		= "mcspi1",
-	.mpu_irqs	= omap2_mcspi1_mpu_irqs,
-	.sdma_reqs	= omap2_mcspi1_sdma_reqs,
 	.main_clk	= "mcspi1_fck",
 	.prcm		= {
 		.omap2 = {
@@ -1568,8 +1404,6 @@ static struct omap2_mcspi_dev_attr omap_mcspi2_dev_attr = {
 
 static struct omap_hwmod omap34xx_mcspi2 = {
 	.name		= "mcspi2",
-	.mpu_irqs	= omap2_mcspi2_mpu_irqs,
-	.sdma_reqs	= omap2_mcspi2_sdma_reqs,
 	.main_clk	= "mcspi2_fck",
 	.prcm		= {
 		.omap2 = {
@@ -1585,18 +1419,7 @@ static struct omap_hwmod omap34xx_mcspi2 = {
 };
 
 /* mcspi3 */
-static struct omap_hwmod_irq_info omap34xx_mcspi3_mpu_irqs[] = {
-	{ .name = "irq", .irq = 91 + OMAP_INTC_START, }, /* 91 */
-	{ .irq = -1 },
-};
 
-static struct omap_hwmod_dma_info omap34xx_mcspi3_sdma_reqs[] = {
-	{ .name = "tx0", .dma_req = 15 },
-	{ .name = "rx0", .dma_req = 16 },
-	{ .name = "tx1", .dma_req = 23 },
-	{ .name = "rx1", .dma_req = 24 },
-	{ .dma_req = -1 }
-};
 
 static struct omap2_mcspi_dev_attr omap_mcspi3_dev_attr = {
 	.num_chipselect = 2,
@@ -1604,8 +1427,6 @@ static struct omap2_mcspi_dev_attr omap_mcspi3_dev_attr = {
 
 static struct omap_hwmod omap34xx_mcspi3 = {
 	.name		= "mcspi3",
-	.mpu_irqs	= omap34xx_mcspi3_mpu_irqs,
-	.sdma_reqs	= omap34xx_mcspi3_sdma_reqs,
 	.main_clk	= "mcspi3_fck",
 	.prcm		= {
 		.omap2 = {
@@ -1621,16 +1442,7 @@ static struct omap_hwmod omap34xx_mcspi3 = {
 };
 
 /* mcspi4 */
-static struct omap_hwmod_irq_info omap34xx_mcspi4_mpu_irqs[] = {
-	{ .name = "irq", .irq = 48 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
-static struct omap_hwmod_dma_info omap34xx_mcspi4_sdma_reqs[] = {
-	{ .name = "tx0", .dma_req = 70 }, /* DMA_SPI4_TX0 */
-	{ .name = "rx0", .dma_req = 71 }, /* DMA_SPI4_RX0 */
-	{ .dma_req = -1 }
-};
 
 static struct omap2_mcspi_dev_attr omap_mcspi4_dev_attr = {
 	.num_chipselect = 1,
@@ -1638,8 +1450,6 @@ static struct omap2_mcspi_dev_attr omap_mcspi4_dev_attr = {
 
 static struct omap_hwmod omap34xx_mcspi4 = {
 	.name		= "mcspi4",
-	.mpu_irqs	= omap34xx_mcspi4_mpu_irqs,
-	.sdma_reqs	= omap34xx_mcspi4_sdma_reqs,
 	.main_clk	= "mcspi4_fck",
 	.prcm		= {
 		.omap2 = {
@@ -1673,16 +1483,9 @@ static struct omap_hwmod_class usbotg_class = {
 };
 
 /* usb_otg_hs */
-static struct omap_hwmod_irq_info omap3xxx_usbhsotg_mpu_irqs[] = {
-
-	{ .name = "mc", .irq = 92 + OMAP_INTC_START, },
-	{ .name = "dma", .irq = 93 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod omap3xxx_usbhsotg_hwmod = {
 	.name		= "usb_otg_hs",
-	.mpu_irqs	= omap3xxx_usbhsotg_mpu_irqs,
 	.main_clk	= "hsotgusb_ick",
 	.prcm		= {
 		.omap2 = {
@@ -1691,7 +1494,7 @@ static struct omap_hwmod omap3xxx_usbhsotg_hwmod = {
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430ES2_ST_HSOTGUSB_IDLE_SHIFT,
-			.idlest_stdby_bit = OMAP3430ES2_ST_HSOTGUSB_STDBY_SHIFT
+			.idlest_stdby_bit = OMAP3430ES2_ST_HSOTGUSB_STDBY_SHIFT,
 		},
 	},
 	.class		= &usbotg_class,
@@ -1711,10 +1514,6 @@ static struct omap_hwmod omap3xxx_usbhsotg_hwmod = {
 };
 
 /* usb_otg_hs */
-static struct omap_hwmod_irq_info am35xx_usbhsotg_mpu_irqs[] = {
-	{ .name = "mc", .irq = 71 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod_class am35xx_usbotg_class = {
 	.name = "am35xx_usbotg",
@@ -1722,7 +1521,6 @@ static struct omap_hwmod_class am35xx_usbotg_class = {
 
 static struct omap_hwmod am35xx_usbhsotg_hwmod = {
 	.name		= "am35x_otg_hs",
-	.mpu_irqs	= am35xx_usbhsotg_mpu_irqs,
 	.main_clk	= "hsotgusb_fck",
 	.class		= &am35xx_usbotg_class,
 	.flags		= HWMOD_NO_IDLEST,
@@ -1747,16 +1545,7 @@ static struct omap_hwmod_class omap34xx_mmc_class = {
 
 /* MMC/SD/SDIO1 */
 
-static struct omap_hwmod_irq_info omap34xx_mmc1_mpu_irqs[] = {
-	{ .irq = 83 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
-static struct omap_hwmod_dma_info omap34xx_mmc1_sdma_reqs[] = {
-	{ .name = "tx",	.dma_req = 61, },
-	{ .name = "rx",	.dma_req = 62, },
-	{ .dma_req = -1 }
-};
 
 static struct omap_hwmod_opt_clk omap34xx_mmc1_opt_clks[] = {
 	{ .role = "dbck", .clk = "omap_32k_fck", },
@@ -1774,8 +1563,6 @@ static struct omap_hsmmc_dev_attr mmc1_pre_es3_dev_attr = {
 
 static struct omap_hwmod omap3xxx_pre_es3_mmc1_hwmod = {
 	.name		= "mmc1",
-	.mpu_irqs	= omap34xx_mmc1_mpu_irqs,
-	.sdma_reqs	= omap34xx_mmc1_sdma_reqs,
 	.opt_clks	= omap34xx_mmc1_opt_clks,
 	.opt_clks_cnt	= ARRAY_SIZE(omap34xx_mmc1_opt_clks),
 	.main_clk	= "mmchs1_fck",
@@ -1794,8 +1581,6 @@ static struct omap_hwmod omap3xxx_pre_es3_mmc1_hwmod = {
 
 static struct omap_hwmod omap3xxx_es3plus_mmc1_hwmod = {
 	.name		= "mmc1",
-	.mpu_irqs	= omap34xx_mmc1_mpu_irqs,
-	.sdma_reqs	= omap34xx_mmc1_sdma_reqs,
 	.opt_clks	= omap34xx_mmc1_opt_clks,
 	.opt_clks_cnt	= ARRAY_SIZE(omap34xx_mmc1_opt_clks),
 	.main_clk	= "mmchs1_fck",
@@ -1814,16 +1599,7 @@ static struct omap_hwmod omap3xxx_es3plus_mmc1_hwmod = {
 
 /* MMC/SD/SDIO2 */
 
-static struct omap_hwmod_irq_info omap34xx_mmc2_mpu_irqs[] = {
-	{ .irq = 86 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
-static struct omap_hwmod_dma_info omap34xx_mmc2_sdma_reqs[] = {
-	{ .name = "tx",	.dma_req = 47, },
-	{ .name = "rx",	.dma_req = 48, },
-	{ .dma_req = -1 }
-};
 
 static struct omap_hwmod_opt_clk omap34xx_mmc2_opt_clks[] = {
 	{ .role = "dbck", .clk = "omap_32k_fck", },
@@ -1836,8 +1612,6 @@ static struct omap_hsmmc_dev_attr mmc2_pre_es3_dev_attr = {
 
 static struct omap_hwmod omap3xxx_pre_es3_mmc2_hwmod = {
 	.name		= "mmc2",
-	.mpu_irqs	= omap34xx_mmc2_mpu_irqs,
-	.sdma_reqs	= omap34xx_mmc2_sdma_reqs,
 	.opt_clks	= omap34xx_mmc2_opt_clks,
 	.opt_clks_cnt	= ARRAY_SIZE(omap34xx_mmc2_opt_clks),
 	.main_clk	= "mmchs2_fck",
@@ -1856,8 +1630,6 @@ static struct omap_hwmod omap3xxx_pre_es3_mmc2_hwmod = {
 
 static struct omap_hwmod omap3xxx_es3plus_mmc2_hwmod = {
 	.name		= "mmc2",
-	.mpu_irqs	= omap34xx_mmc2_mpu_irqs,
-	.sdma_reqs	= omap34xx_mmc2_sdma_reqs,
 	.opt_clks	= omap34xx_mmc2_opt_clks,
 	.opt_clks_cnt	= ARRAY_SIZE(omap34xx_mmc2_opt_clks),
 	.main_clk	= "mmchs2_fck",
@@ -1875,16 +1647,7 @@ static struct omap_hwmod omap3xxx_es3plus_mmc2_hwmod = {
 
 /* MMC/SD/SDIO3 */
 
-static struct omap_hwmod_irq_info omap34xx_mmc3_mpu_irqs[] = {
-	{ .irq = 94 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
-static struct omap_hwmod_dma_info omap34xx_mmc3_sdma_reqs[] = {
-	{ .name = "tx",	.dma_req = 77, },
-	{ .name = "rx",	.dma_req = 78, },
-	{ .dma_req = -1 }
-};
 
 static struct omap_hwmod_opt_clk omap34xx_mmc3_opt_clks[] = {
 	{ .role = "dbck", .clk = "omap_32k_fck", },
@@ -1892,8 +1655,6 @@ static struct omap_hwmod_opt_clk omap34xx_mmc3_opt_clks[] = {
 
 static struct omap_hwmod omap3xxx_mmc3_hwmod = {
 	.name		= "mmc3",
-	.mpu_irqs	= omap34xx_mmc3_mpu_irqs,
-	.sdma_reqs	= omap34xx_mmc3_sdma_reqs,
 	.opt_clks	= omap34xx_mmc3_opt_clks,
 	.opt_clks_cnt	= ARRAY_SIZE(omap34xx_mmc3_opt_clks),
 	.main_clk	= "mmchs3_fck",
@@ -1931,17 +1692,11 @@ static struct omap_hwmod_class omap3xxx_usb_host_hs_hwmod_class = {
 	.sysc = &omap3xxx_usb_host_hs_sysc,
 };
 
-static struct omap_hwmod_irq_info omap3xxx_usb_host_hs_irqs[] = {
-	{ .name = "ohci-irq", .irq = 76 + OMAP_INTC_START, },
-	{ .name = "ehci-irq", .irq = 77 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod omap3xxx_usb_host_hs_hwmod = {
 	.name		= "usb_host_hs",
 	.class		= &omap3xxx_usb_host_hs_hwmod_class,
 	.clkdm_name	= "usbhost_clkdm",
-	.mpu_irqs	= omap3xxx_usb_host_hs_irqs,
 	.main_clk	= "usbhost_48m_fck",
 	.prcm = {
 		.omap2 = {
@@ -2015,16 +1770,11 @@ static struct omap_hwmod_class omap3xxx_usb_tll_hs_hwmod_class = {
 	.sysc = &omap3xxx_usb_tll_hs_sysc,
 };
 
-static struct omap_hwmod_irq_info omap3xxx_usb_tll_hs_irqs[] = {
-	{ .name = "tll-irq", .irq = 78 + OMAP_INTC_START, },
-	{ .irq = -1 },
-};
 
 static struct omap_hwmod omap3xxx_usb_tll_hs_hwmod = {
 	.name		= "usb_tll_hs",
 	.class		= &omap3xxx_usb_tll_hs_hwmod_class,
 	.clkdm_name	= "core_l4_clkdm",
-	.mpu_irqs	= omap3xxx_usb_tll_hs_irqs,
 	.main_clk	= "usbtll_fck",
 	.prcm = {
 		.omap2 = {
@@ -2039,7 +1789,6 @@ static struct omap_hwmod omap3xxx_usb_tll_hs_hwmod = {
 
 static struct omap_hwmod omap3xxx_hdq1w_hwmod = {
 	.name		= "hdq1w",
-	.mpu_irqs	= omap2_hdq1w_mpu_irqs,
 	.main_clk	= "hdq_fck",
 	.prcm		= {
 		.omap2 = {
@@ -2134,16 +1883,10 @@ static struct omap_hwmod_class omap3xxx_gpmc_hwmod_class = {
 	.sysc	= &omap3xxx_gpmc_sysc,
 };
 
-static struct omap_hwmod_irq_info omap3xxx_gpmc_irqs[] = {
-	{ .irq = 20 + OMAP_INTC_START, },
-	{ .irq = -1 }
-};
-
 static struct omap_hwmod omap3xxx_gpmc_hwmod = {
 	.name		= "gpmc",
 	.class		= &omap3xxx_gpmc_hwmod_class,
 	.clkdm_name	= "core_l3_clkdm",
-	.mpu_irqs	= omap3xxx_gpmc_irqs,
 	.main_clk	= "gpmc_fck",
 	/* Skip reset for CONFIG_OMAP_GPMC_DEBUG for bootloader timings */
 	.flags		= HWMOD_NO_IDLEST | DEBUG_OMAP_GPMC_HWMOD_FLAGS,
@@ -2167,37 +1910,19 @@ static struct omap_hwmod_ocp_if omap3xxx_l3_main__l4_per = {
 	.user	= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_l3_main_addrs[] = {
-	{
-		.pa_start	= 0x68000000,
-		.pa_end		= 0x6800ffff,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
 
 /* MPU -> L3 interface */
 static struct omap_hwmod_ocp_if omap3xxx_mpu__l3_main = {
 	.master   = &omap3xxx_mpu_hwmod,
 	.slave    = &omap3xxx_l3_main_hwmod,
-	.addr     = omap3xxx_l3_main_addrs,
 	.user	= OCP_USER_MPU,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_l4_emu_addrs[] = {
-	{
-		.pa_start	= 0x54000000,
-		.pa_end		= 0x547fffff,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
 
 /* l3 -> debugss */
 static struct omap_hwmod_ocp_if omap3xxx_l3_main__l4_debugss = {
 	.master		= &omap3xxx_l3_main_hwmod,
 	.slave		= &omap3xxx_debugss_hwmod,
-	.addr		= omap3xxx_l4_emu_addrs,
 	.user		= OCP_USER_MPU,
 };
 
@@ -2215,7 +1940,7 @@ static struct omap_hwmod_ocp_if omap3xxx_dss__l3 = {
 		.omap2 = {
 			.l3_perm_bit  = OMAP3_L3_CORE_FW_INIT_ID_DSS,
 			.flags	= OMAP_FIREWALL_L3,
-		}
+		},
 	},
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
@@ -2256,18 +1981,16 @@ static struct omap_hwmod_ocp_if omap3xxx_l4_core__pre_es3_mmc1 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_pre_es3_mmc1_hwmod,
 	.clk		= "mmchs1_ick",
-	.addr		= omap2430_mmc1_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
-	.flags		= OMAP_FIREWALL_L4
+	.flags		= OMAP_FIREWALL_L4,
 };
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_core__es3plus_mmc1 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_es3plus_mmc1_hwmod,
 	.clk		= "mmchs1_ick",
-	.addr		= omap2430_mmc1_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
-	.flags		= OMAP_FIREWALL_L4
+	.flags		= OMAP_FIREWALL_L4,
 };
 
 /* L4 CORE -> MMC2 interface */
@@ -2275,126 +1998,70 @@ static struct omap_hwmod_ocp_if omap3xxx_l4_core__pre_es3_mmc2 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_pre_es3_mmc2_hwmod,
 	.clk		= "mmchs2_ick",
-	.addr		= omap2430_mmc2_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
-	.flags		= OMAP_FIREWALL_L4
+	.flags		= OMAP_FIREWALL_L4,
 };
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_core__es3plus_mmc2 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_es3plus_mmc2_hwmod,
 	.clk		= "mmchs2_ick",
-	.addr		= omap2430_mmc2_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
-	.flags		= OMAP_FIREWALL_L4
+	.flags		= OMAP_FIREWALL_L4,
 };
 
 /* L4 CORE -> MMC3 interface */
-static struct omap_hwmod_addr_space omap3xxx_mmc3_addr_space[] = {
-	{
-		.pa_start	= 0x480ad000,
-		.pa_end		= 0x480ad1ff,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_core__mmc3 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_mmc3_hwmod,
 	.clk		= "mmchs3_ick",
-	.addr		= omap3xxx_mmc3_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
-	.flags		= OMAP_FIREWALL_L4
+	.flags		= OMAP_FIREWALL_L4,
 };
 
 /* L4 CORE -> UART1 interface */
-static struct omap_hwmod_addr_space omap3xxx_uart1_addr_space[] = {
-	{
-		.pa_start	= OMAP3_UART1_BASE,
-		.pa_end		= OMAP3_UART1_BASE + SZ_8K - 1,
-		.flags		= ADDR_MAP_ON_INIT | ADDR_TYPE_RT,
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3_l4_core__uart1 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_uart1_hwmod,
 	.clk		= "uart1_ick",
-	.addr		= omap3xxx_uart1_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* L4 CORE -> UART2 interface */
-static struct omap_hwmod_addr_space omap3xxx_uart2_addr_space[] = {
-	{
-		.pa_start	= OMAP3_UART2_BASE,
-		.pa_end		= OMAP3_UART2_BASE + SZ_1K - 1,
-		.flags		= ADDR_MAP_ON_INIT | ADDR_TYPE_RT,
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3_l4_core__uart2 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_uart2_hwmod,
 	.clk		= "uart2_ick",
-	.addr		= omap3xxx_uart2_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* L4 PER -> UART3 interface */
-static struct omap_hwmod_addr_space omap3xxx_uart3_addr_space[] = {
-	{
-		.pa_start	= OMAP3_UART3_BASE,
-		.pa_end		= OMAP3_UART3_BASE + SZ_1K - 1,
-		.flags		= ADDR_MAP_ON_INIT | ADDR_TYPE_RT,
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3_l4_per__uart3 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_uart3_hwmod,
 	.clk		= "uart3_ick",
-	.addr		= omap3xxx_uart3_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* L4 PER -> UART4 interface */
-static struct omap_hwmod_addr_space omap36xx_uart4_addr_space[] = {
-	{
-		.pa_start	= OMAP3_UART4_BASE,
-		.pa_end		= OMAP3_UART4_BASE + SZ_1K - 1,
-		.flags		= ADDR_MAP_ON_INIT | ADDR_TYPE_RT,
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap36xx_l4_per__uart4 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap36xx_uart4_hwmod,
 	.clk		= "uart4_ick",
-	.addr		= omap36xx_uart4_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* AM35xx: L4 CORE -> UART4 interface */
-static struct omap_hwmod_addr_space am35xx_uart4_addr_space[] = {
-	{
-		.pa_start	= OMAP3_UART4_AM35XX_BASE,
-		.pa_end		= OMAP3_UART4_AM35XX_BASE + SZ_1K - 1,
-		.flags		= ADDR_MAP_ON_INIT | ADDR_TYPE_RT,
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if am35xx_l4_core__uart4 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &am35xx_uart4_hwmod,
 	.clk		= "uart4_ick",
-	.addr		= am35xx_uart4_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -2403,13 +2070,12 @@ static struct omap_hwmod_ocp_if omap3_l4_core__i2c1 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_i2c1_hwmod,
 	.clk		= "i2c1_ick",
-	.addr		= omap2_i2c1_addr_space,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP3_L4_CORE_FW_I2C1_REGION,
 			.l4_prot_group = 7,
 			.flags	= OMAP_FIREWALL_L4,
-		}
+		},
 	},
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
@@ -2419,57 +2085,38 @@ static struct omap_hwmod_ocp_if omap3_l4_core__i2c2 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_i2c2_hwmod,
 	.clk		= "i2c2_ick",
-	.addr		= omap2_i2c2_addr_space,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP3_L4_CORE_FW_I2C2_REGION,
 			.l4_prot_group = 7,
 			.flags = OMAP_FIREWALL_L4,
-		}
+		},
 	},
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* L4 CORE -> I2C3 interface */
-static struct omap_hwmod_addr_space omap3xxx_i2c3_addr_space[] = {
-	{
-		.pa_start	= 0x48060000,
-		.pa_end		= 0x48060000 + SZ_128 - 1,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3_l4_core__i2c3 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_i2c3_hwmod,
 	.clk		= "i2c3_ick",
-	.addr		= omap3xxx_i2c3_addr_space,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP3_L4_CORE_FW_I2C3_REGION,
 			.l4_prot_group = 7,
 			.flags = OMAP_FIREWALL_L4,
-		}
+		},
 	},
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* L4 CORE -> SR1 interface */
-static struct omap_hwmod_addr_space omap3_sr1_addr_space[] = {
-	{
-		.pa_start	= OMAP34XX_SR1_BASE,
-		.pa_end		= OMAP34XX_SR1_BASE + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap34xx_l4_core__sr1 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap34xx_sr1_hwmod,
 	.clk		= "sr_l4_ick",
-	.addr		= omap3_sr1_addr_space,
 	.user		= OCP_USER_MPU,
 };
 
@@ -2477,25 +2124,15 @@ static struct omap_hwmod_ocp_if omap36xx_l4_core__sr1 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap36xx_sr1_hwmod,
 	.clk		= "sr_l4_ick",
-	.addr		= omap3_sr1_addr_space,
 	.user		= OCP_USER_MPU,
 };
 
 /* L4 CORE -> SR1 interface */
-static struct omap_hwmod_addr_space omap3_sr2_addr_space[] = {
-	{
-		.pa_start	= OMAP34XX_SR2_BASE,
-		.pa_end		= OMAP34XX_SR2_BASE + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap34xx_l4_core__sr2 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap34xx_sr2_hwmod,
 	.clk		= "sr_l4_ick",
-	.addr		= omap3_sr2_addr_space,
 	.user		= OCP_USER_MPU,
 };
 
@@ -2503,43 +2140,24 @@ static struct omap_hwmod_ocp_if omap36xx_l4_core__sr2 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap36xx_sr2_hwmod,
 	.clk		= "sr_l4_ick",
-	.addr		= omap3_sr2_addr_space,
 	.user		= OCP_USER_MPU,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_usbhsotg_addrs[] = {
-	{
-		.pa_start	= OMAP34XX_HSUSB_OTG_BASE,
-		.pa_end		= OMAP34XX_HSUSB_OTG_BASE + SZ_4K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_core -> usbhsotg  */
 static struct omap_hwmod_ocp_if omap3xxx_l4_core__usbhsotg = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_usbhsotg_hwmod,
 	.clk		= "l4_ick",
-	.addr		= omap3xxx_usbhsotg_addrs,
 	.user		= OCP_USER_MPU,
 };
 
-static struct omap_hwmod_addr_space am35xx_usbhsotg_addrs[] = {
-	{
-		.pa_start	= AM35XX_IPSS_USBOTGSS_BASE,
-		.pa_end		= AM35XX_IPSS_USBOTGSS_BASE + SZ_4K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_core -> usbhsotg  */
 static struct omap_hwmod_ocp_if am35xx_l4_core__usbhsotg = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &am35xx_usbhsotg_hwmod,
 	.clk		= "hsotgusb_ick",
-	.addr		= am35xx_usbhsotg_addrs,
 	.user		= OCP_USER_MPU,
 };
 
@@ -2558,165 +2176,84 @@ static struct omap_hwmod_ocp_if omap3xxx_l3__iva = {
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_timer1_addrs[] = {
-	{
-		.pa_start	= 0x48318000,
-		.pa_end		= 0x48318000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_wkup -> timer1 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_wkup__timer1 = {
 	.master		= &omap3xxx_l4_wkup_hwmod,
 	.slave		= &omap3xxx_timer1_hwmod,
 	.clk		= "gpt1_ick",
-	.addr		= omap3xxx_timer1_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_timer2_addrs[] = {
-	{
-		.pa_start	= 0x49032000,
-		.pa_end		= 0x49032000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> timer2 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__timer2 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_timer2_hwmod,
 	.clk		= "gpt2_ick",
-	.addr		= omap3xxx_timer2_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_timer3_addrs[] = {
-	{
-		.pa_start	= 0x49034000,
-		.pa_end		= 0x49034000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> timer3 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__timer3 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_timer3_hwmod,
 	.clk		= "gpt3_ick",
-	.addr		= omap3xxx_timer3_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_timer4_addrs[] = {
-	{
-		.pa_start	= 0x49036000,
-		.pa_end		= 0x49036000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> timer4 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__timer4 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_timer4_hwmod,
 	.clk		= "gpt4_ick",
-	.addr		= omap3xxx_timer4_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_timer5_addrs[] = {
-	{
-		.pa_start	= 0x49038000,
-		.pa_end		= 0x49038000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> timer5 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__timer5 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_timer5_hwmod,
 	.clk		= "gpt5_ick",
-	.addr		= omap3xxx_timer5_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_timer6_addrs[] = {
-	{
-		.pa_start	= 0x4903A000,
-		.pa_end		= 0x4903A000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> timer6 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__timer6 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_timer6_hwmod,
 	.clk		= "gpt6_ick",
-	.addr		= omap3xxx_timer6_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_timer7_addrs[] = {
-	{
-		.pa_start	= 0x4903C000,
-		.pa_end		= 0x4903C000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> timer7 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__timer7 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_timer7_hwmod,
 	.clk		= "gpt7_ick",
-	.addr		= omap3xxx_timer7_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_timer8_addrs[] = {
-	{
-		.pa_start	= 0x4903E000,
-		.pa_end		= 0x4903E000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> timer8 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__timer8 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_timer8_hwmod,
 	.clk		= "gpt8_ick",
-	.addr		= omap3xxx_timer8_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_timer9_addrs[] = {
-	{
-		.pa_start	= 0x49040000,
-		.pa_end		= 0x49040000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> timer9 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__timer9 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_timer9_hwmod,
 	.clk		= "gpt9_ick",
-	.addr		= omap3xxx_timer9_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -2725,7 +2262,6 @@ static struct omap_hwmod_ocp_if omap3xxx_l4_core__timer10 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_timer10_hwmod,
 	.clk		= "gpt10_ick",
-	.addr		= omap2_timer10_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -2734,43 +2270,24 @@ static struct omap_hwmod_ocp_if omap3xxx_l4_core__timer11 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_timer11_hwmod,
 	.clk		= "gpt11_ick",
-	.addr		= omap2_timer11_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_timer12_addrs[] = {
-	{
-		.pa_start	= 0x48304000,
-		.pa_end		= 0x48304000 + SZ_1K - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_core -> timer12 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_sec__timer12 = {
 	.master		= &omap3xxx_l4_sec_hwmod,
 	.slave		= &omap3xxx_timer12_hwmod,
 	.clk		= "gpt12_ick",
-	.addr		= omap3xxx_timer12_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* l4_wkup -> wd_timer2 */
-static struct omap_hwmod_addr_space omap3xxx_wd_timer2_addrs[] = {
-	{
-		.pa_start	= 0x48314000,
-		.pa_end		= 0x4831407f,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_wkup__wd_timer2 = {
 	.master		= &omap3xxx_l4_wkup_hwmod,
 	.slave		= &omap3xxx_wd_timer2_hwmod,
 	.clk		= "wdt2_ick",
-	.addr		= omap3xxx_wd_timer2_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -2779,13 +2296,12 @@ static struct omap_hwmod_ocp_if omap3430es1_l4_core__dss = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3430es1_dss_core_hwmod,
 	.clk		= "dss_ick",
-	.addr		= omap2_dss_addrs,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP3ES1_L4_CORE_FW_DSS_CORE_REGION,
 			.l4_prot_group = OMAP3_L4_CORE_FW_DSS_PROT_GROUP,
 			.flags	= OMAP_FIREWALL_L4,
-		}
+		},
 	},
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
@@ -2794,13 +2310,12 @@ static struct omap_hwmod_ocp_if omap3xxx_l4_core__dss = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_dss_core_hwmod,
 	.clk		= "dss_ick",
-	.addr		= omap2_dss_addrs,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP3_L4_CORE_FW_DSS_CORE_REGION,
 			.l4_prot_group = OMAP3_L4_CORE_FW_DSS_PROT_GROUP,
 			.flags	= OMAP_FIREWALL_L4,
-		}
+		},
 	},
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
@@ -2810,38 +2325,27 @@ static struct omap_hwmod_ocp_if omap3xxx_l4_core__dss_dispc = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_dss_dispc_hwmod,
 	.clk		= "dss_ick",
-	.addr		= omap2_dss_dispc_addrs,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP3_L4_CORE_FW_DSS_DISPC_REGION,
 			.l4_prot_group = OMAP3_L4_CORE_FW_DSS_PROT_GROUP,
 			.flags	= OMAP_FIREWALL_L4,
-		}
+		},
 	},
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_dss_dsi1_addrs[] = {
-	{
-		.pa_start	= 0x4804FC00,
-		.pa_end		= 0x4804FFFF,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
-
 /* l4_core -> dss_dsi1 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_core__dss_dsi1 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_dss_dsi1_hwmod,
 	.clk		= "dss_ick",
-	.addr		= omap3xxx_dss_dsi1_addrs,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP3_L4_CORE_FW_DSS_DSI_REGION,
 			.l4_prot_group = OMAP3_L4_CORE_FW_DSS_PROT_GROUP,
 			.flags	= OMAP_FIREWALL_L4,
-		}
+		},
 	},
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
@@ -2851,13 +2355,12 @@ static struct omap_hwmod_ocp_if omap3xxx_l4_core__dss_rfbi = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_dss_rfbi_hwmod,
 	.clk		= "dss_ick",
-	.addr		= omap2_dss_rfbi_addrs,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP3_L4_CORE_FW_DSS_RFBI_REGION,
 			.l4_prot_group = OMAP3_L4_CORE_FW_DSS_PROT_GROUP ,
 			.flags	= OMAP_FIREWALL_L4,
-		}
+		},
 	},
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
@@ -2867,66 +2370,38 @@ static struct omap_hwmod_ocp_if omap3xxx_l4_core__dss_venc = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_dss_venc_hwmod,
 	.clk		= "dss_ick",
-	.addr		= omap2_dss_venc_addrs,
 	.fw = {
 		.omap2 = {
 			.l4_fw_region  = OMAP3_L4_CORE_FW_DSS_VENC_REGION,
 			.l4_prot_group = OMAP3_L4_CORE_FW_DSS_PROT_GROUP,
 			.flags	= OMAP_FIREWALL_L4,
-		}
+		},
 	},
 	.flags		= OCPIF_SWSUP_IDLE,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* l4_wkup -> gpio1 */
-static struct omap_hwmod_addr_space omap3xxx_gpio1_addrs[] = {
-	{
-		.pa_start	= 0x48310000,
-		.pa_end		= 0x483101ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_wkup__gpio1 = {
 	.master		= &omap3xxx_l4_wkup_hwmod,
 	.slave		= &omap3xxx_gpio1_hwmod,
-	.addr		= omap3xxx_gpio1_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* l4_per -> gpio2 */
-static struct omap_hwmod_addr_space omap3xxx_gpio2_addrs[] = {
-	{
-		.pa_start	= 0x49050000,
-		.pa_end		= 0x490501ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__gpio2 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_gpio2_hwmod,
-	.addr		= omap3xxx_gpio2_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* l4_per -> gpio3 */
-static struct omap_hwmod_addr_space omap3xxx_gpio3_addrs[] = {
-	{
-		.pa_start	= 0x49052000,
-		.pa_end		= 0x490521ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__gpio3 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_gpio3_hwmod,
-	.addr		= omap3xxx_gpio3_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -3002,53 +2477,26 @@ static struct omap_hwmod omap3xxx_mmu_iva_hwmod = {
 };
 
 /* l4_per -> gpio4 */
-static struct omap_hwmod_addr_space omap3xxx_gpio4_addrs[] = {
-	{
-		.pa_start	= 0x49054000,
-		.pa_end		= 0x490541ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__gpio4 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_gpio4_hwmod,
-	.addr		= omap3xxx_gpio4_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* l4_per -> gpio5 */
-static struct omap_hwmod_addr_space omap3xxx_gpio5_addrs[] = {
-	{
-		.pa_start	= 0x49056000,
-		.pa_end		= 0x490561ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__gpio5 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_gpio5_hwmod,
-	.addr		= omap3xxx_gpio5_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* l4_per -> gpio6 */
-static struct omap_hwmod_addr_space omap3xxx_gpio6_addrs[] = {
-	{
-		.pa_start	= 0x49058000,
-		.pa_end		= 0x490581ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__gpio6 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_gpio6_hwmod,
-	.addr		= omap3xxx_gpio6_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -3064,9 +2512,9 @@ static struct omap_hwmod_addr_space omap3xxx_dma_system_addrs[] = {
 	{
 		.pa_start	= 0x48056000,
 		.pa_end		= 0x48056fff,
-		.flags		= ADDR_TYPE_RT
+		.flags		= ADDR_TYPE_RT,
 	},
-	{ }
+	{ },
 };
 
 /* l4_cfg -> dma_system */
@@ -3078,136 +2526,66 @@ static struct omap_hwmod_ocp_if omap3xxx_l4_core__dma_system = {
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_mcbsp1_addrs[] = {
-	{
-		.name		= "mpu",
-		.pa_start	= 0x48074000,
-		.pa_end		= 0x480740ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_core -> mcbsp1 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_core__mcbsp1 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_mcbsp1_hwmod,
 	.clk		= "mcbsp1_ick",
-	.addr		= omap3xxx_mcbsp1_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_mcbsp2_addrs[] = {
-	{
-		.name		= "mpu",
-		.pa_start	= 0x49022000,
-		.pa_end		= 0x490220ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> mcbsp2 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__mcbsp2 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_mcbsp2_hwmod,
 	.clk		= "mcbsp2_ick",
-	.addr		= omap3xxx_mcbsp2_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_mcbsp3_addrs[] = {
-	{
-		.name		= "mpu",
-		.pa_start	= 0x49024000,
-		.pa_end		= 0x490240ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> mcbsp3 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__mcbsp3 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_mcbsp3_hwmod,
 	.clk		= "mcbsp3_ick",
-	.addr		= omap3xxx_mcbsp3_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_mcbsp4_addrs[] = {
-	{
-		.name		= "mpu",
-		.pa_start	= 0x49026000,
-		.pa_end		= 0x490260ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> mcbsp4 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__mcbsp4 = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_mcbsp4_hwmod,
 	.clk		= "mcbsp4_ick",
-	.addr		= omap3xxx_mcbsp4_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_mcbsp5_addrs[] = {
-	{
-		.name		= "mpu",
-		.pa_start	= 0x48096000,
-		.pa_end		= 0x480960ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_core -> mcbsp5 */
 static struct omap_hwmod_ocp_if omap3xxx_l4_core__mcbsp5 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_mcbsp5_hwmod,
 	.clk		= "mcbsp5_ick",
-	.addr		= omap3xxx_mcbsp5_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_mcbsp2_sidetone_addrs[] = {
-	{
-		.name		= "sidetone",
-		.pa_start	= 0x49028000,
-		.pa_end		= 0x490280ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> mcbsp2_sidetone */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__mcbsp2_sidetone = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_mcbsp2_sidetone_hwmod,
 	.clk		= "mcbsp2_ick",
-	.addr		= omap3xxx_mcbsp2_sidetone_addrs,
 	.user		= OCP_USER_MPU,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_mcbsp3_sidetone_addrs[] = {
-	{
-		.name		= "sidetone",
-		.pa_start	= 0x4902A000,
-		.pa_end		= 0x4902A0ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 /* l4_per -> mcbsp3_sidetone */
 static struct omap_hwmod_ocp_if omap3xxx_l4_per__mcbsp3_sidetone = {
 	.master		= &omap3xxx_l4_per_hwmod,
 	.slave		= &omap3xxx_mcbsp3_sidetone_hwmod,
 	.clk		= "mcbsp3_ick",
-	.addr		= omap3xxx_mcbsp3_sidetone_addrs,
 	.user		= OCP_USER_MPU,
 };
 
@@ -3223,7 +2601,6 @@ static struct omap_hwmod_ocp_if omap34xx_l4_core__mcspi1 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap34xx_mcspi1,
 	.clk		= "mcspi1_ick",
-	.addr		= omap2_mcspi1_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -3232,7 +2609,6 @@ static struct omap_hwmod_ocp_if omap34xx_l4_core__mcspi2 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap34xx_mcspi2,
 	.clk		= "mcspi2_ick",
-	.addr		= omap2_mcspi2_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -3241,25 +2617,15 @@ static struct omap_hwmod_ocp_if omap34xx_l4_core__mcspi3 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap34xx_mcspi3,
 	.clk		= "mcspi3_ick",
-	.addr		= omap2430_mcspi3_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
 /* l4 core -> mcspi4 interface */
-static struct omap_hwmod_addr_space omap34xx_mcspi4_addr_space[] = {
-	{
-		.pa_start	= 0x480ba000,
-		.pa_end		= 0x480ba0ff,
-		.flags		= ADDR_TYPE_RT,
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap34xx_l4_core__mcspi4 = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap34xx_mcspi4,
 	.clk		= "mcspi4_ick",
-	.addr		= omap34xx_mcspi4_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -3270,49 +2636,19 @@ static struct omap_hwmod_ocp_if omap3xxx_usb_host_hs__l3_main_2 = {
 	.user		= OCP_USER_MPU,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_usb_host_hs_addrs[] = {
-	{
-		.name		= "uhh",
-		.pa_start	= 0x48064000,
-		.pa_end		= 0x480643ff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{
-		.name		= "ohci",
-		.pa_start	= 0x48064400,
-		.pa_end		= 0x480647ff,
-	},
-	{
-		.name		= "ehci",
-		.pa_start	= 0x48064800,
-		.pa_end		= 0x48064cff,
-	},
-	{}
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_core__usb_host_hs = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_usb_host_hs_hwmod,
 	.clk		= "usbhost_ick",
-	.addr		= omap3xxx_usb_host_hs_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_usb_tll_hs_addrs[] = {
-	{
-		.name		= "tll",
-		.pa_start	= 0x48062000,
-		.pa_end		= 0x48062fff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{}
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_core__usb_tll_hs = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_usb_tll_hs_hwmod,
 	.clk		= "usbtll_ick",
-	.addr		= omap3xxx_usb_tll_hs_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -3321,35 +2657,17 @@ static struct omap_hwmod_ocp_if omap3xxx_l4_core__hdq1w = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_hdq1w_hwmod,
 	.clk		= "hdq_ick",
-	.addr		= omap2_hdq1w_addr_space,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 	.flags		= OMAP_FIREWALL_L4 | OCPIF_SWSUP_IDLE,
 };
 
 /* l4_wkup -> 32ksync_counter */
-static struct omap_hwmod_addr_space omap3xxx_counter_32k_addrs[] = {
-	{
-		.pa_start	= 0x48320000,
-		.pa_end		= 0x4832001f,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
-static struct omap_hwmod_addr_space omap3xxx_gpmc_addrs[] = {
-	{
-		.pa_start	= 0x6e000000,
-		.pa_end		= 0x6e000fff,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_wkup__counter_32k = {
 	.master		= &omap3xxx_l4_wkup_hwmod,
 	.slave		= &omap3xxx_counter_32k_hwmod,
 	.clk		= "omap_32ksync_ick",
-	.addr		= omap3xxx_counter_32k_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -3434,7 +2752,6 @@ static struct omap_hwmod_ocp_if omap3xxx_l3_main__gpmc = {
 	.master		= &omap3xxx_l3_main_hwmod,
 	.slave		= &omap3xxx_gpmc_hwmod,
 	.clk		= "core_l3_ick",
-	.addr		= omap3xxx_gpmc_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -3459,20 +2776,10 @@ static struct omap_hwmod_class omap3xxx_sham_class = {
 	.sysc	= &omap3_sham_sysc,
 };
 
-static struct omap_hwmod_irq_info omap3_sham_mpu_irqs[] = {
-	{ .irq = 49 + OMAP_INTC_START, },
-	{ .irq = -1 }
-};
 
-static struct omap_hwmod_dma_info omap3_sham_sdma_reqs[] = {
-	{ .name = "rx", .dma_req = 69, },
-	{ .dma_req = -1 }
-};
 
 static struct omap_hwmod omap3xxx_sham_hwmod = {
 	.name		= "sham",
-	.mpu_irqs	= omap3_sham_mpu_irqs,
-	.sdma_reqs	= omap3_sham_sdma_reqs,
 	.main_clk	= "sha12_ick",
 	.prcm		= {
 		.omap2 = {
@@ -3486,20 +2793,11 @@ static struct omap_hwmod omap3xxx_sham_hwmod = {
 	.class		= &omap3xxx_sham_class,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_sham_addrs[] = {
-	{
-		.pa_start	= 0x480c3000,
-		.pa_end		= 0x480c3000 + 0x64 - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_core__sham = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_sham_hwmod,
 	.clk		= "sha12_ick",
-	.addr		= omap3xxx_sham_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -3525,15 +2823,9 @@ static struct omap_hwmod_class omap3xxx_aes_class = {
 	.sysc	= &omap3_aes_sysc,
 };
 
-static struct omap_hwmod_dma_info omap3_aes_sdma_reqs[] = {
-	{ .name = "tx", .dma_req = 65, },
-	{ .name = "rx", .dma_req = 66, },
-	{ .dma_req = -1 }
-};
 
 static struct omap_hwmod omap3xxx_aes_hwmod = {
 	.name		= "aes",
-	.sdma_reqs	= omap3_aes_sdma_reqs,
 	.main_clk	= "aes2_ick",
 	.prcm		= {
 		.omap2 = {
@@ -3547,20 +2839,11 @@ static struct omap_hwmod omap3xxx_aes_hwmod = {
 	.class		= &omap3xxx_aes_class,
 };
 
-static struct omap_hwmod_addr_space omap3xxx_aes_addrs[] = {
-	{
-		.pa_start	= 0x480c5000,
-		.pa_end		= 0x480c5000 + 0x50 - 1,
-		.flags		= ADDR_TYPE_RT
-	},
-	{ }
-};
 
 static struct omap_hwmod_ocp_if omap3xxx_l4_core__aes = {
 	.master		= &omap3xxx_l4_core_hwmod,
 	.slave		= &omap3xxx_aes_hwmod,
 	.clk		= "aes2_ick",
-	.addr		= omap3xxx_aes_addrs,
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
@@ -3661,28 +2944,28 @@ static struct omap_hwmod_ocp_if *omap3xxx_hwmod_ocp_ifs[] __initdata = {
 /* GP-only hwmod links */
 static struct omap_hwmod_ocp_if *omap34xx_gp_hwmod_ocp_ifs[] __initdata = {
 	&omap3xxx_l4_sec__timer12,
-	NULL
+	NULL,
 };
 
 static struct omap_hwmod_ocp_if *omap36xx_gp_hwmod_ocp_ifs[] __initdata = {
 	&omap3xxx_l4_sec__timer12,
-	NULL
+	NULL,
 };
 
 static struct omap_hwmod_ocp_if *am35xx_gp_hwmod_ocp_ifs[] __initdata = {
 	&omap3xxx_l4_sec__timer12,
-	NULL
+	NULL,
 };
 
 /* crypto hwmod links */
 static struct omap_hwmod_ocp_if *omap34xx_sham_hwmod_ocp_ifs[] __initdata = {
 	&omap3xxx_l4_core__sham,
-	NULL
+	NULL,
 };
 
 static struct omap_hwmod_ocp_if *omap34xx_aes_hwmod_ocp_ifs[] __initdata = {
 	&omap3xxx_l4_core__aes,
-	NULL
+	NULL,
 };
 
 static struct omap_hwmod_ocp_if *omap36xx_sham_hwmod_ocp_ifs[] __initdata = {
@@ -3710,14 +2993,14 @@ static struct omap_hwmod_ocp_if *am35xx_sham_hwmod_ocp_ifs[] __initdata = {
 
 static struct omap_hwmod_ocp_if *am35xx_aes_hwmod_ocp_ifs[] __initdata = {
 	/* &omap3xxx_l4_core__aes, */
-	NULL
+	NULL,
 };
 
 /* 3430ES1-only hwmod links */
 static struct omap_hwmod_ocp_if *omap3430es1_hwmod_ocp_ifs[] __initdata = {
 	&omap3430es1_dss__l3,
 	&omap3430es1_l4_core__dss,
-	NULL
+	NULL,
 };
 
 /* 3430ES2+-only hwmod links */
@@ -3729,21 +3012,21 @@ static struct omap_hwmod_ocp_if *omap3430es2plus_hwmod_ocp_ifs[] __initdata = {
 	&omap3xxx_usb_host_hs__l3_main_2,
 	&omap3xxx_l4_core__usb_host_hs,
 	&omap3xxx_l4_core__usb_tll_hs,
-	NULL
+	NULL,
 };
 
 /* <= 3430ES3-only hwmod links */
 static struct omap_hwmod_ocp_if *omap3430_pre_es3_hwmod_ocp_ifs[] __initdata = {
 	&omap3xxx_l4_core__pre_es3_mmc1,
 	&omap3xxx_l4_core__pre_es3_mmc2,
-	NULL
+	NULL,
 };
 
 /* 3430ES3+-only hwmod links */
 static struct omap_hwmod_ocp_if *omap3430_es3plus_hwmod_ocp_ifs[] __initdata = {
 	&omap3xxx_l4_core__es3plus_mmc1,
 	&omap3xxx_l4_core__es3plus_mmc2,
-	NULL
+	NULL,
 };
 
 /* 34xx-only hwmod links (all ES revisions) */
@@ -3757,7 +3040,7 @@ static struct omap_hwmod_ocp_if *omap34xx_hwmod_ocp_ifs[] __initdata = {
 	&omap3xxx_l4_core__mmu_isp,
 	&omap3xxx_l3_main__mmu_iva,
 	&omap3xxx_l4_core__ssi,
-	NULL
+	NULL,
 };
 
 /* 36xx-only hwmod links (all ES revisions) */
@@ -3781,7 +3064,7 @@ static struct omap_hwmod_ocp_if *omap36xx_hwmod_ocp_ifs[] __initdata = {
 	&omap3xxx_l4_core__mmu_isp,
 	&omap3xxx_l3_main__mmu_iva,
 	&omap3xxx_l4_core__ssi,
-	NULL
+	NULL,
 };
 
 static struct omap_hwmod_ocp_if *am35xx_hwmod_ocp_ifs[] __initdata = {
@@ -3800,7 +3083,7 @@ static struct omap_hwmod_ocp_if *am35xx_hwmod_ocp_ifs[] __initdata = {
 	&am35xx_l4_core__mdio,
 	&am35xx_emac__l3,
 	&am35xx_l4_core__emac,
-	NULL
+	NULL,
 };
 
 static struct omap_hwmod_ocp_if *omap3xxx_dss_hwmod_ocp_ifs[] __initdata = {
@@ -3808,7 +3091,7 @@ static struct omap_hwmod_ocp_if *omap3xxx_dss_hwmod_ocp_ifs[] __initdata = {
 	&omap3xxx_l4_core__dss_dsi1,
 	&omap3xxx_l4_core__dss_rfbi,
 	&omap3xxx_l4_core__dss_venc,
-	NULL
+	NULL,
 };
 
 /**
diff --git a/arch/arm/mach-omap2/omap_hwmod_43xx_data.c b/arch/arm/mach-omap2/omap_hwmod_43xx_data.c
index 61f2f301d739..afbce1f6f641 100644
--- a/arch/arm/mach-omap2/omap_hwmod_43xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_43xx_data.c
@@ -442,6 +442,31 @@ static struct omap_hwmod am43xx_adc_tsc_hwmod = {
 	},
 };
 
+static struct omap_hwmod_class_sysconfig am43xx_des_sysc = {
+	.rev_offs	= 0x30,
+	.sysc_offs	= 0x34,
+	.syss_offs	= 0x38,
+	.sysc_flags	= SYSS_HAS_RESET_STATUS,
+};
+
+static struct omap_hwmod_class am43xx_des_hwmod_class = {
+	.name		= "des",
+	.sysc		= &am43xx_des_sysc,
+};
+
+static struct omap_hwmod am43xx_des_hwmod = {
+	.name		= "des",
+	.class		= &am43xx_des_hwmod_class,
+	.clkdm_name	= "l3_clkdm",
+	.main_clk	= "l3_gclk",
+	.prcm		= {
+		.omap4	= {
+			.clkctrl_offs	= AM43XX_CM_PER_DES_CLKCTRL_OFFSET,
+			.modulemode	= MODULEMODE_SWCTRL,
+		},
+	},
+};
+
 /* dss */
 
 static struct omap_hwmod am43xx_dss_core_hwmod = {
@@ -870,6 +895,13 @@ static struct omap_hwmod_ocp_if am43xx_l4_ls__vpfe1 = {
 	.user           = OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+static struct omap_hwmod_ocp_if am43xx_l3_main__des = {
+	.master		= &am33xx_l3_main_hwmod,
+	.slave		= &am43xx_des_hwmod,
+	.clk		= "l3_gclk",
+	.user		= OCP_USER_MPU,
+};
+
 static struct omap_hwmod_ocp_if *am43xx_hwmod_ocp_ifs[] __initdata = {
 	&am33xx_l4_wkup__synctimer,
 	&am43xx_l4_ls__timer8,
@@ -917,6 +949,7 @@ static struct omap_hwmod_ocp_if *am43xx_hwmod_ocp_ifs[] __initdata = {
 	&am33xx_l4_per__i2c2,
 	&am33xx_l4_per__i2c3,
 	&am33xx_l4_per__mailbox,
+	&am33xx_l4_per__rng,
 	&am33xx_l4_ls__mcasp0,
 	&am33xx_l4_ls__mcasp1,
 	&am33xx_l4_ls__mmc0,
@@ -950,6 +983,7 @@ static struct omap_hwmod_ocp_if *am43xx_hwmod_ocp_ifs[] __initdata = {
 	&am33xx_cpgmac0__mdio,
 	&am33xx_l3_main__sha0,
 	&am33xx_l3_main__aes0,
+	&am43xx_l3_main__des,
 	&am43xx_l4_ls__ocp2scp0,
 	&am43xx_l4_ls__ocp2scp1,
 	&am43xx_l3_s__usbotgss0,
diff --git a/arch/arm/mach-omap2/omap_hwmod_7xx_data.c b/arch/arm/mach-omap2/omap_hwmod_7xx_data.c
index 1ab7096af8e2..d0585293a381 100644
--- a/arch/arm/mach-omap2/omap_hwmod_7xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_7xx_data.c
@@ -690,6 +690,78 @@ static struct omap_hwmod dra7xx_dss_hdmi_hwmod = {
 	.parent_hwmod	= &dra7xx_dss_hwmod,
 };
 
+/* AES (the 'P' (public) device) */
+static struct omap_hwmod_class_sysconfig dra7xx_aes_sysc = {
+	.rev_offs	= 0x0080,
+	.sysc_offs	= 0x0084,
+	.syss_offs	= 0x0088,
+	.sysc_flags	= SYSS_HAS_RESET_STATUS,
+};
+
+static struct omap_hwmod_class dra7xx_aes_hwmod_class = {
+	.name	= "aes",
+	.sysc	= &dra7xx_aes_sysc,
+	.rev	= 2,
+};
+
+/* AES1 */
+static struct omap_hwmod dra7xx_aes1_hwmod = {
+	.name		= "aes1",
+	.class		= &dra7xx_aes_hwmod_class,
+	.clkdm_name	= "l4sec_clkdm",
+	.main_clk	= "l3_iclk_div",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_L4SEC_AES1_CLKCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_L4SEC_AES1_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+/* AES2 */
+static struct omap_hwmod dra7xx_aes2_hwmod = {
+	.name		= "aes2",
+	.class		= &dra7xx_aes_hwmod_class,
+	.clkdm_name	= "l4sec_clkdm",
+	.main_clk	= "l3_iclk_div",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_L4SEC_AES2_CLKCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_L4SEC_AES2_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+/* sha0 HIB2 (the 'P' (public) device) */
+static struct omap_hwmod_class_sysconfig dra7xx_sha0_sysc = {
+	.rev_offs	= 0x100,
+	.sysc_offs	= 0x110,
+	.syss_offs	= 0x114,
+	.sysc_flags	= SYSS_HAS_RESET_STATUS,
+};
+
+static struct omap_hwmod_class dra7xx_sha0_hwmod_class = {
+	.name		= "sham",
+	.sysc		= &dra7xx_sha0_sysc,
+	.rev		= 2,
+};
+
+struct omap_hwmod dra7xx_sha0_hwmod = {
+	.name		= "sham",
+	.class		= &dra7xx_sha0_hwmod_class,
+	.clkdm_name	= "l4sec_clkdm",
+	.main_clk	= "l3_iclk_div",
+	.prcm		= {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_L4SEC_SHA2MD51_CLKCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_L4SEC_SHA2MD51_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
 /*
  * 'elm' class
  *
@@ -2541,6 +2613,62 @@ static struct omap_hwmod dra7xx_uart10_hwmod = {
 	},
 };
 
+/* DES (the 'P' (public) device) */
+static struct omap_hwmod_class_sysconfig dra7xx_des_sysc = {
+	.rev_offs	= 0x0030,
+	.sysc_offs	= 0x0034,
+	.syss_offs	= 0x0038,
+	.sysc_flags	= SYSS_HAS_RESET_STATUS,
+};
+
+static struct omap_hwmod_class dra7xx_des_hwmod_class = {
+	.name	= "des",
+	.sysc	= &dra7xx_des_sysc,
+};
+
+/* DES */
+static struct omap_hwmod dra7xx_des_hwmod = {
+	.name		= "des",
+	.class		= &dra7xx_des_hwmod_class,
+	.clkdm_name	= "l4sec_clkdm",
+	.main_clk	= "l3_iclk_div",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_L4SEC_DES3DES_CLKCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_L4SEC_DES3DES_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
+/* rng */
+static struct omap_hwmod_class_sysconfig dra7xx_rng_sysc = {
+	.rev_offs       = 0x1fe0,
+	.sysc_offs      = 0x1fe4,
+	.sysc_flags     = SYSC_HAS_AUTOIDLE | SYSC_HAS_SIDLEMODE,
+	.idlemodes      = SIDLE_FORCE | SIDLE_NO,
+	.sysc_fields    = &omap_hwmod_sysc_type1,
+};
+
+static struct omap_hwmod_class dra7xx_rng_hwmod_class = {
+	.name           = "rng",
+	.sysc           = &dra7xx_rng_sysc,
+};
+
+static struct omap_hwmod dra7xx_rng_hwmod = {
+	.name           = "rng",
+	.class          = &dra7xx_rng_hwmod_class,
+	.flags		= HWMOD_SWSUP_SIDLE,
+	.clkdm_name     = "l4sec_clkdm",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = DRA7XX_CM_L4SEC_RNG_CLKCTRL_OFFSET,
+			.context_offs = DRA7XX_RM_L4SEC_RNG_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_HWCTRL,
+		},
+	},
+};
+
 /*
  * 'usb_otg_ss' class
  *
@@ -2929,6 +3057,30 @@ static struct omap_hwmod_ocp_if dra7xx_l3_main_1__hdmi = {
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+/* l3_main_1 -> aes1 */
+static struct omap_hwmod_ocp_if dra7xx_l3_main_1__aes1 = {
+	.master		= &dra7xx_l3_main_1_hwmod,
+	.slave		= &dra7xx_aes1_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l3_main_1 -> aes2 */
+static struct omap_hwmod_ocp_if dra7xx_l3_main_1__aes2 = {
+	.master		= &dra7xx_l3_main_1_hwmod,
+	.slave		= &dra7xx_aes2_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* l3_main_1 -> sha0 */
+static struct omap_hwmod_ocp_if dra7xx_l3_main_1__sha0 = {
+	.master		= &dra7xx_l3_main_1_hwmod,
+	.slave		= &dra7xx_sha0_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
 /* l4_per2 -> mcasp1 */
 static struct omap_hwmod_ocp_if dra7xx_l4_per2__mcasp1 = {
 	.master		= &dra7xx_l4_per2_hwmod,
@@ -3642,6 +3794,14 @@ static struct omap_hwmod_ocp_if dra7xx_l4_per2__uart7 = {
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+/* l4_per1 -> des */
+static struct omap_hwmod_ocp_if dra7xx_l4_per1__des = {
+	.master		= &dra7xx_l4_per1_hwmod,
+	.slave		= &dra7xx_des_hwmod,
+	.clk		= "l3_iclk_div",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
 /* l4_per2 -> uart8 */
 static struct omap_hwmod_ocp_if dra7xx_l4_per2__uart8 = {
 	.master		= &dra7xx_l4_per2_hwmod,
@@ -3666,6 +3826,13 @@ static struct omap_hwmod_ocp_if dra7xx_l4_wkup__uart10 = {
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+/* l4_per1 -> rng */
+static struct omap_hwmod_ocp_if dra7xx_l4_per1__rng = {
+	.master         = &dra7xx_l4_per1_hwmod,
+	.slave          = &dra7xx_rng_hwmod,
+	.user           = OCP_USER_MPU,
+};
+
 /* l4_per3 -> usb_otg_ss1 */
 static struct omap_hwmod_ocp_if dra7xx_l4_per3__usb_otg_ss1 = {
 	.master		= &dra7xx_l4_per3_hwmod,
@@ -3800,6 +3967,9 @@ static struct omap_hwmod_ocp_if *dra7xx_hwmod_ocp_ifs[] __initdata = {
 	&dra7xx_l3_main_1__dss,
 	&dra7xx_l3_main_1__dispc,
 	&dra7xx_l3_main_1__hdmi,
+	&dra7xx_l3_main_1__aes1,
+	&dra7xx_l3_main_1__aes2,
+	&dra7xx_l3_main_1__sha0,
 	&dra7xx_l4_per1__elm,
 	&dra7xx_l4_wkup__gpio1,
 	&dra7xx_l4_per1__gpio2,
@@ -3845,7 +4015,6 @@ static struct omap_hwmod_ocp_if *dra7xx_hwmod_ocp_ifs[] __initdata = {
 	&dra7xx_l3_main_1__pciess2,
 	&dra7xx_l4_cfg__pciess2,
 	&dra7xx_l3_main_1__qspi,
-	&dra7xx_l4_per3__rtcss,
 	&dra7xx_l4_cfg__sata,
 	&dra7xx_l4_cfg__smartreflex_core,
 	&dra7xx_l4_cfg__smartreflex_mpu,
@@ -3875,6 +4044,7 @@ static struct omap_hwmod_ocp_if *dra7xx_hwmod_ocp_ifs[] __initdata = {
 	&dra7xx_l4_per2__uart8,
 	&dra7xx_l4_per2__uart9,
 	&dra7xx_l4_wkup__uart10,
+	&dra7xx_l4_per1__des,
 	&dra7xx_l4_per3__usb_otg_ss1,
 	&dra7xx_l4_per3__usb_otg_ss2,
 	&dra7xx_l4_per3__usb_otg_ss3,
@@ -3892,6 +4062,7 @@ static struct omap_hwmod_ocp_if *dra7xx_hwmod_ocp_ifs[] __initdata = {
 /* GP-only hwmod links */
 static struct omap_hwmod_ocp_if *dra7xx_gp_hwmod_ocp_ifs[] __initdata = {
 	&dra7xx_l4_wkup__timer12,
+	&dra7xx_l4_per1__rng,
 	NULL,
 };
 
@@ -3905,6 +4076,11 @@ static struct omap_hwmod_ocp_if *dra72x_hwmod_ocp_ifs[] __initdata = {
 	NULL,
 };
 
+static struct omap_hwmod_ocp_if *dra74x_dra72x_hwmod_ocp_ifs[] __initdata = {
+	&dra7xx_l4_per3__rtcss,
+	NULL,
+};
+
 int __init dra7xx_hwmod_init(void)
 {
 	int ret;
@@ -3920,5 +4096,9 @@ int __init dra7xx_hwmod_init(void)
 	if (!ret && omap_type() == OMAP2_DEVICE_TYPE_GP)
 		ret = omap_hwmod_register_links(dra7xx_gp_hwmod_ocp_ifs);
 
+	/* now for the IPs *NOT* in dra71 */
+	if (!ret && !of_machine_is_compatible("ti,dra718"))
+		ret = omap_hwmod_register_links(dra74x_dra72x_hwmod_ocp_ifs);
+
 	return ret;
 }
diff --git a/arch/arm/mach-omap2/omap_hwmod_common_data.h b/arch/arm/mach-omap2/omap_hwmod_common_data.h
index 11ed5a17dd77..cdfbb44ceb0c 100644
--- a/arch/arm/mach-omap2/omap_hwmod_common_data.h
+++ b/arch/arm/mach-omap2/omap_hwmod_common_data.h
@@ -19,22 +19,7 @@
 #include "display.h"
 
 /* Common address space across OMAP2xxx/3xxx */
-extern struct omap_hwmod_addr_space omap2_i2c1_addr_space[];
-extern struct omap_hwmod_addr_space omap2_i2c2_addr_space[];
-extern struct omap_hwmod_addr_space omap2_dss_addrs[];
-extern struct omap_hwmod_addr_space omap2_dss_dispc_addrs[];
-extern struct omap_hwmod_addr_space omap2_dss_rfbi_addrs[];
-extern struct omap_hwmod_addr_space omap2_dss_venc_addrs[];
-extern struct omap_hwmod_addr_space omap2_timer10_addrs[];
-extern struct omap_hwmod_addr_space omap2_timer11_addrs[];
-extern struct omap_hwmod_addr_space omap2430_mmc1_addr_space[];
-extern struct omap_hwmod_addr_space omap2430_mmc2_addr_space[];
-extern struct omap_hwmod_addr_space omap2_mcspi1_addr_space[];
-extern struct omap_hwmod_addr_space omap2_mcspi2_addr_space[];
-extern struct omap_hwmod_addr_space omap2430_mcspi3_addr_space[];
 extern struct omap_hwmod_addr_space omap2_dma_system_addrs[];
-extern struct omap_hwmod_addr_space omap2_mcbsp1_addrs[];
-extern struct omap_hwmod_addr_space omap2_hdq1w_addr_space[];
 
 /* Common IP block data across OMAP2xxx */
 extern struct omap_gpio_dev_attr omap2xxx_gpio_dev_attr;
diff --git a/arch/arm/mach-omap2/pdata-quirks.c b/arch/arm/mach-omap2/pdata-quirks.c
index 05e20aaf68dd..477910a48448 100644
--- a/arch/arm/mach-omap2/pdata-quirks.c
+++ b/arch/arm/mach-omap2/pdata-quirks.c
@@ -31,7 +31,6 @@
 
 #include "common.h"
 #include "common-board-devices.h"
-#include "dss-common.h"
 #include "control.h"
 #include "omap_device.h"
 #include "omap-pm.h"
diff --git a/arch/arm/mach-omap2/pm-debug.c b/arch/arm/mach-omap2/pm-debug.c
index 0b339861d751..003a6cb248be 100644
--- a/arch/arm/mach-omap2/pm-debug.c
+++ b/arch/arm/mach-omap2/pm-debug.c
@@ -114,8 +114,7 @@ static int pwrdm_dbg_show_counter(struct powerdomain *pwrdm, void *user)
 		seq_printf(s, ",RET-MEMBANK%d-OFF:%d", i + 1,
 				pwrdm->ret_mem_off_counter[i]);
 
-	seq_printf(s, "\n");
-
+	seq_putc(s, '\n');
 	return 0;
 }
 
@@ -138,7 +137,7 @@ static int pwrdm_dbg_show_timer(struct powerdomain *pwrdm, void *user)
 		seq_printf(s, ",%s:%lld", pwrdm_state_names[i],
 			pwrdm->state_timer[i]);
 
-	seq_printf(s, "\n");
+	seq_putc(s, '\n');
 	return 0;
 }
 
diff --git a/arch/arm/mach-omap2/pm.c b/arch/arm/mach-omap2/pm.c
index 678d2a31dcb8..76b0454ddc49 100644
--- a/arch/arm/mach-omap2/pm.c
+++ b/arch/arm/mach-omap2/pm.c
@@ -30,7 +30,6 @@
 #include "powerdomain.h"
 #include "clockdomain.h"
 #include "pm.h"
-#include "twl-common.h"
 
 #ifdef CONFIG_SUSPEND
 /*
@@ -72,42 +71,6 @@ void omap_pm_get_oscillator(u32 *tstart, u32 *tshut)
 }
 #endif
 
-static int __init _init_omap_device(char *name)
-{
-	struct omap_hwmod *oh;
-	struct platform_device *pdev;
-
-	oh = omap_hwmod_lookup(name);
-	if (WARN(!oh, "%s: could not find omap_hwmod for %s\n",
-		 __func__, name))
-		return -ENODEV;
-
-	pdev = omap_device_build(oh->name, 0, oh, NULL, 0);
-	if (WARN(IS_ERR(pdev), "%s: could not build omap_device for %s\n",
-		 __func__, name))
-		return -ENODEV;
-
-	return 0;
-}
-
-/*
- * Build omap_devices for processors and bus.
- */
-static void __init omap2_init_processor_devices(void)
-{
-	_init_omap_device("mpu");
-	if (omap3_has_iva())
-		_init_omap_device("iva");
-
-	if (cpu_is_omap44xx()) {
-		_init_omap_device("l3_main_1");
-		_init_omap_device("dsp");
-		_init_omap_device("iva");
-	} else {
-		_init_omap_device("l3_main");
-	}
-}
-
 int __init omap_pm_clkdms_setup(struct clockdomain *clkdm, void *unused)
 {
 	clkdm_allow_idle(clkdm);
@@ -215,7 +178,7 @@ static int omap_pm_enter(suspend_state_t suspend_state)
 static int omap_pm_begin(suspend_state_t state)
 {
 	cpu_idle_poll_ctrl(true);
-	if (cpu_is_omap34xx())
+	if (soc_is_omap34xx())
 		omap_prcm_irq_prepare();
 	return 0;
 }
@@ -227,7 +190,7 @@ static void omap_pm_end(void)
 
 static void omap_pm_finish(void)
 {
-	if (cpu_is_omap34xx())
+	if (soc_is_omap34xx())
 		omap_prcm_irq_complete();
 }
 
@@ -252,7 +215,7 @@ void omap_common_suspend_init(void *pm_suspend)
 
 static void __init omap3_init_voltages(void)
 {
-	if (!cpu_is_omap34xx())
+	if (!soc_is_omap34xx())
 		return;
 
 	omap2_set_init_voltage("mpu_iva", "dpll1_ck", "mpu");
@@ -261,7 +224,7 @@ static void __init omap3_init_voltages(void)
 
 static void __init omap4_init_voltages(void)
 {
-	if (!cpu_is_omap44xx())
+	if (!soc_is_omap44xx())
 		return;
 
 	omap2_set_init_voltage("mpu", "dpll_mpu_ck", "mpu");
@@ -269,18 +232,8 @@ static void __init omap4_init_voltages(void)
 	omap2_set_init_voltage("iva", "dpll_iva_m5x2_ck", "iva");
 }
 
-static inline void omap_init_cpufreq(void)
-{
-	struct platform_device_info devinfo = { .name = "omap-cpufreq" };
-
-	if (!of_have_populated_dt())
-		platform_device_register_full(&devinfo);
-}
-
 static int __init omap2_common_pm_init(void)
 {
-	if (!of_have_populated_dt())
-		omap2_init_processor_devices();
 	omap_pm_if_init();
 
 	return 0;
@@ -289,13 +242,9 @@ omap_postcore_initcall(omap2_common_pm_init);
 
 int __init omap2_common_pm_late_init(void)
 {
-	if (of_have_populated_dt()) {
-		omap3_twl_init();
-		omap4_twl_init();
-	}
-
 	/* Init the voltage layer */
-	omap_pmic_late_init();
+	omap3_twl_init();
+	omap4_twl_init();
 	omap_voltage_late_init();
 
 	/* Initialize the voltages */
@@ -305,8 +254,5 @@ int __init omap2_common_pm_late_init(void)
 	/* Smartreflex device init */
 	omap_devinit_smartreflex();
 
-	/* cpufreq dummy device instantiation */
-	omap_init_cpufreq();
-
 	return 0;
 }
diff --git a/arch/arm/mach-omap2/pm44xx.c b/arch/arm/mach-omap2/pm44xx.c
index 178e22c146b7..b3870220612e 100644
--- a/arch/arm/mach-omap2/pm44xx.c
+++ b/arch/arm/mach-omap2/pm44xx.c
@@ -287,7 +287,7 @@ int __init omap4_pm_init(void)
 	/* Overwrite the default cpu_do_idle() */
 	arm_pm_idle = omap_default_idle;
 
-	if (cpu_is_omap44xx())
+	if (cpu_is_omap44xx() || soc_is_omap54xx())
 		omap4_idle_init();
 
 err2:
diff --git a/arch/arm/mach-omap2/prcm43xx.h b/arch/arm/mach-omap2/prcm43xx.h
index babb5db5a3a4..e2ad14e77064 100644
--- a/arch/arm/mach-omap2/prcm43xx.h
+++ b/arch/arm/mach-omap2/prcm43xx.h
@@ -92,6 +92,7 @@
 #define AM43XX_CM_PER_MAILBOX0_CLKCTRL_OFFSET		0x04b8
 #define AM43XX_CM_PER_MMC0_CLKCTRL_OFFSET		0x04c0
 #define AM43XX_CM_PER_MMC1_CLKCTRL_OFFSET		0x04c8
+#define AM43XX_CM_PER_RNG_CLKCTRL_OFFSET		0x04e0
 #define AM43XX_CM_PER_SPI0_CLKCTRL_OFFSET		0x0500
 #define AM43XX_CM_PER_SPI1_CLKCTRL_OFFSET		0x0508
 #define AM43XX_CM_PER_SPINLOCK_CLKCTRL_OFFSET		0x0528
@@ -133,6 +134,7 @@
 #define AM43XX_CM_PER_OCMCRAM_CLKCTRL_OFFSET		0x0050
 #define AM43XX_CM_PER_SHA0_CLKCTRL_OFFSET		0x0058
 #define AM43XX_CM_PER_AES0_CLKCTRL_OFFSET		0x0028
+#define AM43XX_CM_PER_DES_CLKCTRL_OFFSET		0x0030
 #define AM43XX_CM_PER_TIMER8_CLKCTRL_OFFSET		0x0560
 #define AM43XX_CM_PER_TIMER9_CLKCTRL_OFFSET		0x0568
 #define AM43XX_CM_PER_TIMER10_CLKCTRL_OFFSET		0x0570
diff --git a/arch/arm/mach-omap2/sdram-hynix-h8mbx00u0mer-0em.h b/arch/arm/mach-omap2/sdram-hynix-h8mbx00u0mer-0em.h
deleted file mode 100644
index 1ee58c281a31..000000000000
--- a/arch/arm/mach-omap2/sdram-hynix-h8mbx00u0mer-0em.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * SDRC register values for the Hynix H8MBX00U0MER-0EM
- *
- * Copyright (C) 2009 Texas Instruments, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifndef __ARCH_ARM_MACH_OMAP2_SDRAM_HYNIX_H8MBX00U0MER0EM
-#define __ARCH_ARM_MACH_OMAP2_SDRAM_HYNIX_H8MBX00U0MER0EM
-
-#include "sdrc.h"
-
-/* Hynix H8MBX00U0MER-0EM */
-static struct omap_sdrc_params h8mbx00u0mer0em_sdrc_params[] = {
-	[0] = {
-		.rate        = 200000000,
-		.actim_ctrla = 0xa2e1b4c6,
-		.actim_ctrlb = 0x0002131c,
-		.rfr_ctrl    = 0x0005e601,
-		.mr          = 0x00000032,
-	},
-	[1] = {
-		.rate        = 166000000,
-		.actim_ctrla = 0x629db4c6,
-		.actim_ctrlb = 0x00012214,
-		.rfr_ctrl    = 0x0004dc01,
-		.mr          = 0x00000032,
-	},
-	[2] = {
-		.rate        = 100000000,
-		.actim_ctrla = 0x51912284,
-		.actim_ctrlb = 0x0002120e,
-		.rfr_ctrl    = 0x0002d101,
-		.mr          = 0x00000022,
-	},
-	[3] = {
-		.rate        = 83000000,
-		.actim_ctrla = 0x31512283,
-		.actim_ctrlb = 0x0001220a,
-		.rfr_ctrl    = 0x00025501,
-		.mr          = 0x00000022,
-	},
-	[4] = {
-		.rate        = 0
-	},
-};
-
-#endif
diff --git a/arch/arm/mach-omap2/sdram-micron-mt46h32m32lf-6.h b/arch/arm/mach-omap2/sdram-micron-mt46h32m32lf-6.h
deleted file mode 100644
index 85cccc004c06..000000000000
--- a/arch/arm/mach-omap2/sdram-micron-mt46h32m32lf-6.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * SDRC register values for the Micron MT46H32M32LF-6
- *
- * Copyright (C) 2008 Texas Instruments, Inc.
- * Copyright (C) 2008-2009 Nokia Corporation
- *
- * Paul Walmsley
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifndef ARCH_ARM_MACH_OMAP2_SDRAM_MICRON_MT46H32M32LF
-#define ARCH_ARM_MACH_OMAP2_SDRAM_MICRON_MT46H32M32LF
-
-#include "sdrc.h"
-
-/* Micron MT46H32M32LF-6 */
-/* XXX Using ARE = 0x1 (no autorefresh burst) -- can this be changed? */
-static struct omap_sdrc_params mt46h32m32lf6_sdrc_params[] = {
-	[0] = {
-		.rate	     = 166000000,
-		.actim_ctrla = 0x9a9db4c6,
-		.actim_ctrlb = 0x00011217,
-		.rfr_ctrl    = 0x0004dc01,
-		.mr	     = 0x00000032,
-	},
-	[1] = {
-		.rate	     = 165941176,
-		.actim_ctrla = 0x9a9db4c6,
-		.actim_ctrlb = 0x00011217,
-		.rfr_ctrl    = 0x0004dc01,
-		.mr	     = 0x00000032,
-	},
-	[2] = {
-		.rate	     = 83000000,
-		.actim_ctrla = 0x51512283,
-		.actim_ctrlb = 0x0001120c,
-		.rfr_ctrl    = 0x00025501,
-		.mr	     = 0x00000032,
-	},
-	[3] = {
-		.rate	     = 82970588,
-		.actim_ctrla = 0x51512283,
-		.actim_ctrlb = 0x0001120c,
-		.rfr_ctrl    = 0x00025501,
-		.mr	     = 0x00000032,
-	},
-	[4] = {
-		.rate	     = 0
-	},
-};
-
-#endif
diff --git a/arch/arm/mach-omap2/sdram-nokia.c b/arch/arm/mach-omap2/sdram-nokia.c
deleted file mode 100644
index 0fa7ffa9b5ed..000000000000
--- a/arch/arm/mach-omap2/sdram-nokia.c
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * SDRC register values for Nokia boards
- *
- * Copyright (C) 2008, 2010-2011 Nokia Corporation
- *
- * Lauri Leukkunen <lauri.leukkunen@nokia.com>
- *
- * Original code by Juha Yrjola <juha.yrjola@solidboot.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/kernel.h>
-#include <linux/clk.h>
-#include <linux/err.h>
-#include <linux/io.h>
-
-#include "common.h"
-#include "sdram-nokia.h"
-#include "sdrc.h"
-
-/* In picoseconds, except for tREF (ns), tXP, tCKE, tWTR (clks) */
-struct sdram_timings {
-	u32 casl;
-	u32 tDAL;
-	u32 tDPL;
-	u32 tRRD;
-	u32 tRCD;
-	u32 tRP;
-	u32 tRAS;
-	u32 tRC;
-	u32 tRFC;
-	u32 tXSR;
-
-	u32 tREF; /* in ns */
-
-	u32 tXP;
-	u32 tCKE;
-	u32 tWTR;
-};
-
-static const struct sdram_timings nokia_97dot6mhz_timings[] = {
-	{
-		.casl = 3,
-		.tDAL = 30725,
-		.tDPL = 15362,
-		.tRRD = 10241,
-		.tRCD = 20483,
-		.tRP = 15362,
-		.tRAS = 40967,
-		.tRC = 56330,
-		.tRFC = 138266,
-		.tXSR = 204839,
-
-		.tREF = 7798,
-
-		.tXP = 2,
-		.tCKE = 4,
-		.tWTR = 2,
-	},
-};
-
-static const struct sdram_timings nokia_166mhz_timings[] = {
-	{
-		.casl = 3,
-		.tDAL = 33000,
-		.tDPL = 15000,
-		.tRRD = 12000,
-		.tRCD = 22500,
-		.tRP = 18000,
-		.tRAS = 42000,
-		.tRC = 66000,
-		.tRFC = 138000,
-		.tXSR = 200000,
-
-		.tREF = 7800,
-
-		.tXP = 2,
-		.tCKE = 2,
-		.tWTR = 2
-	},
-};
-
-static const struct sdram_timings nokia_195dot2mhz_timings[] = {
-	{
-		.casl = 3,
-		.tDAL = 30725,
-		.tDPL = 15362,
-		.tRRD = 10241,
-		.tRCD = 20483,
-		.tRP = 15362,
-		.tRAS = 40967,
-		.tRC = 56330,
-		.tRFC = 138266,
-		.tXSR = 204839,
-
-		.tREF = 7752,
-
-		.tXP = 2,
-		.tCKE = 4,
-		.tWTR = 2,
-	},
-};
-
-static const struct sdram_timings nokia_200mhz_timings[] = {
-	{
-		.casl = 3,
-		.tDAL = 30000,
-		.tDPL = 15000,
-		.tRRD = 10000,
-		.tRCD = 20000,
-		.tRP = 15000,
-		.tRAS = 40000,
-		.tRC = 55000,
-		.tRFC = 140000,
-		.tXSR = 200000,
-
-		.tREF = 7800,
-
-		.tXP = 2,
-		.tCKE = 4,
-		.tWTR = 2
-	},
-};
-
-static const struct {
-	long rate;
-	struct sdram_timings const *data;
-} nokia_timings[] = {
-	{ 83000000, nokia_166mhz_timings },
-	{ 97600000, nokia_97dot6mhz_timings },
-	{ 100000000, nokia_200mhz_timings },
-	{ 166000000, nokia_166mhz_timings },
-	{ 195200000, nokia_195dot2mhz_timings },
-	{ 200000000, nokia_200mhz_timings },
-};
-static struct omap_sdrc_params nokia_sdrc_params[ARRAY_SIZE(nokia_timings) + 1];
-
-static unsigned long sdrc_get_fclk_period(long rate)
-{
-	/* In picoseconds */
-	return 1000000000 / rate;
-}
-
-static unsigned int sdrc_ps_to_ticks(unsigned int time_ps, long rate)
-{
-	unsigned long tick_ps;
-
-	/* Calculate in picosecs to yield more exact results */
-	tick_ps = sdrc_get_fclk_period(rate);
-
-	return (time_ps + tick_ps - 1) / tick_ps;
-}
-#undef DEBUG
-#ifdef DEBUG
-static int set_sdrc_timing_regval(u32 *regval, int st_bit, int end_bit,
-				int ticks, long rate, const char *name)
-#else
-static int set_sdrc_timing_regval(u32 *regval, int st_bit, int end_bit,
-			       int ticks)
-#endif
-{
-	int mask, nr_bits;
-
-	nr_bits = end_bit - st_bit + 1;
-	if (ticks >= 1 << nr_bits)
-		return -1;
-	mask = (1 << nr_bits) - 1;
-	*regval &= ~(mask << st_bit);
-	*regval |= ticks << st_bit;
-#ifdef DEBUG
-	printk(KERN_INFO "SDRC %s: %i ticks %i ns\n", name, ticks,
-			(unsigned int)sdrc_get_fclk_period(rate) * ticks /
-			1000);
-#endif
-
-	return 0;
-}
-
-#ifdef DEBUG
-#define SDRC_SET_ONE(reg, st, end, field, rate) \
-	if (set_sdrc_timing_regval((reg), (st), (end), \
-			memory_timings->field, (rate), #field) < 0) \
-		err = -1;
-#else
-#define SDRC_SET_ONE(reg, st, end, field, rate) \
-	if (set_sdrc_timing_regval((reg), (st), (end), \
-			memory_timings->field) < 0) \
-		err = -1;
-#endif
-
-#ifdef DEBUG
-static int set_sdrc_timing_regval_ps(u32 *regval, int st_bit, int end_bit,
-				int time, long rate, const char *name)
-#else
-static int set_sdrc_timing_regval_ps(u32 *regval, int st_bit, int end_bit,
-				int time, long rate)
-#endif
-{
-	int ticks, ret;
-	ret = 0;
-
-	if (time == 0)
-		ticks = 0;
-	else
-		ticks = sdrc_ps_to_ticks(time, rate);
-
-#ifdef DEBUG
-	ret = set_sdrc_timing_regval(regval, st_bit, end_bit, ticks,
-				     rate, name);
-#else
-	ret = set_sdrc_timing_regval(regval, st_bit, end_bit, ticks);
-#endif
-
-	return ret;
-}
-
-#ifdef DEBUG
-#define SDRC_SET_ONE_PS(reg, st, end, field, rate) \
-	if (set_sdrc_timing_regval_ps((reg), (st), (end), \
-			memory_timings->field, \
-			(rate), #field) < 0) \
-		err = -1;
-
-#else
-#define SDRC_SET_ONE_PS(reg, st, end, field, rate) \
-	if (set_sdrc_timing_regval_ps((reg), (st), (end), \
-			memory_timings->field, (rate)) < 0) \
-		err = -1;
-#endif
-
-static int sdrc_timings(int id, long rate,
-			const struct sdram_timings *memory_timings)
-{
-	u32 ticks_per_ms;
-	u32 rfr, l;
-	u32 actim_ctrla = 0, actim_ctrlb = 0;
-	u32 rfr_ctrl;
-	int err = 0;
-	long l3_rate = rate / 1000;
-
-	SDRC_SET_ONE_PS(&actim_ctrla,  0,  4, tDAL, l3_rate);
-	SDRC_SET_ONE_PS(&actim_ctrla,  6,  8, tDPL, l3_rate);
-	SDRC_SET_ONE_PS(&actim_ctrla,  9, 11, tRRD, l3_rate);
-	SDRC_SET_ONE_PS(&actim_ctrla, 12, 14, tRCD, l3_rate);
-	SDRC_SET_ONE_PS(&actim_ctrla, 15, 17, tRP, l3_rate);
-	SDRC_SET_ONE_PS(&actim_ctrla, 18, 21, tRAS, l3_rate);
-	SDRC_SET_ONE_PS(&actim_ctrla, 22, 26, tRC, l3_rate);
-	SDRC_SET_ONE_PS(&actim_ctrla, 27, 31, tRFC, l3_rate);
-
-	SDRC_SET_ONE_PS(&actim_ctrlb,  0,  7, tXSR, l3_rate);
-
-	SDRC_SET_ONE(&actim_ctrlb,  8, 10, tXP, l3_rate);
-	SDRC_SET_ONE(&actim_ctrlb, 12, 14, tCKE, l3_rate);
-	SDRC_SET_ONE(&actim_ctrlb, 16, 17, tWTR, l3_rate);
-
-	ticks_per_ms = l3_rate;
-	rfr = memory_timings[0].tREF * ticks_per_ms / 1000000;
-	if (rfr > 65535 + 50)
-		rfr = 65535;
-	else
-		rfr -= 50;
-
-#ifdef DEBUG
-	printk(KERN_INFO "SDRC tREF: %i ticks\n", rfr);
-#endif
-
-	l = rfr << 8;
-	rfr_ctrl = l | 0x1; /* autorefresh, reload counter with 1xARCV */
-
-	nokia_sdrc_params[id].rate = rate;
-	nokia_sdrc_params[id].actim_ctrla = actim_ctrla;
-	nokia_sdrc_params[id].actim_ctrlb = actim_ctrlb;
-	nokia_sdrc_params[id].rfr_ctrl = rfr_ctrl;
-	nokia_sdrc_params[id].mr = 0x32;
-
-	nokia_sdrc_params[id + 1].rate = 0;
-
-	return err;
-}
-
-struct omap_sdrc_params *nokia_get_sdram_timings(void)
-{
-	int err = 0;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(nokia_timings); i++) {
-		err |= sdrc_timings(i, nokia_timings[i].rate,
-				       nokia_timings[i].data);
-		if (err)
-			pr_err("%s: error with rate %ld: %d\n", __func__,
-			       nokia_timings[i].rate, err);
-	}
-
-	return err ? NULL : nokia_sdrc_params;
-}
-
diff --git a/arch/arm/mach-omap2/sdram-nokia.h b/arch/arm/mach-omap2/sdram-nokia.h
deleted file mode 100644
index ee63da5f8df0..000000000000
--- a/arch/arm/mach-omap2/sdram-nokia.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * SDRC register values for Nokia boards
- *
- * Copyright (C) 2010 Nokia
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-struct omap_sdrc_params *nokia_get_sdram_timings(void);
-
diff --git a/arch/arm/mach-omap2/sdram-numonyx-m65kxxxxam.h b/arch/arm/mach-omap2/sdram-numonyx-m65kxxxxam.h
deleted file mode 100644
index 003f7bf4e2e3..000000000000
--- a/arch/arm/mach-omap2/sdram-numonyx-m65kxxxxam.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * SDRC register values for the Numonyx M65KXXXXAM
- *
- * Copyright (C) 2009 Integration Software and Electronic Engineering.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifndef __ARCH_ARM_MACH_OMAP2_SDRAM_NUMONYX_M65KXXXXAM
-#define __ARCH_ARM_MACH_OMAP2_SDRAM_NUMONYX_M65KXXXXAM
-
-#include "sdrc.h"
-
-/* Numonyx  M65KXXXXAM */
-static struct omap_sdrc_params m65kxxxxam_sdrc_params[] = {
-	[0] = {
-		.rate		= 200000000,
-		.actim_ctrla	= 0xe321d4c6,
-		.actim_ctrlb	= 0x00022328,
-		.rfr_ctrl	= 0x0005e601,
-		.mr		= 0x00000032,
-	},
-	[1] = {
-		.rate		= 166000000,
-		.actim_ctrla	= 0xba9dc485,
-		.actim_ctrlb	= 0x00022321,
-		.rfr_ctrl	= 0x0004dc01,
-		.mr		= 0x00000032,
-	},
-	[2] = {
-		.rate		= 133000000,
-		.actim_ctrla	= 0x9a19b485,
-		.actim_ctrlb	= 0x0002231b,
-		.rfr_ctrl	= 0x0003de01,
-		.mr		= 0x00000032,
-	},
-	[3] = {
-		.rate		= 83000000,
-		.actim_ctrla	= 0x594ca242,
-		.actim_ctrlb	= 0x00022310,
-		.rfr_ctrl	= 0x00025501,
-		.mr		= 0x00000032,
-	},
-	[4] = {
-		.rate			= 0
-	},
-};
-
-#endif
diff --git a/arch/arm/mach-omap2/sdram-qimonda-hyb18m512160af-6.h b/arch/arm/mach-omap2/sdram-qimonda-hyb18m512160af-6.h
deleted file mode 100644
index 8dc3de5ebb5b..000000000000
--- a/arch/arm/mach-omap2/sdram-qimonda-hyb18m512160af-6.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * SDRC register values for the Qimonda HYB18M512160AF-6
- *
- * Copyright (C) 2008-2009 Texas Instruments, Inc.
- * Copyright (C) 2008-2009 Nokia Corporation
- *
- * Paul Walmsley
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifndef ARCH_ARM_MACH_OMAP2_SDRAM_QIMONDA_HYB18M512160AF6
-#define ARCH_ARM_MACH_OMAP2_SDRAM_QIMONDA_HYB18M512160AF6
-
-#include "sdrc.h"
-
-/* Qimonda HYB18M512160AF-6 */
-static struct omap_sdrc_params hyb18m512160af6_sdrc_params[] = {
-	[0] = {
-		.rate	     = 166000000,
-		.actim_ctrla = 0x629db4c6,
-		.actim_ctrlb = 0x00012214,
-		.rfr_ctrl    = 0x0004dc01,
-		.mr	     = 0x00000032,
-	},
-	[1] = {
-		.rate	     = 165941176,
-		.actim_ctrla = 0x629db4c6,
-		.actim_ctrlb = 0x00012214,
-		.rfr_ctrl    = 0x0004dc01,
-		.mr	     = 0x00000032,
-	},
-	[2] = {
-		.rate	     = 83000000,
-		.actim_ctrla = 0x31512283,
-		.actim_ctrlb = 0x0001220a,
-		.rfr_ctrl    = 0x00025501,
-		.mr	     = 0x00000022,
-	},
-	[3] = {
-		.rate	     = 82970588,
-		.actim_ctrla = 0x31512283,
-		.actim_ctrlb = 0x0001220a,
-		.rfr_ctrl    = 0x00025501,
-		.mr	     = 0x00000022,
-	},
-	[4] = {
-		.rate	     = 0
-	},
-};
-
-#endif
diff --git a/arch/arm/mach-omap2/serial.c b/arch/arm/mach-omap2/serial.c
deleted file mode 100644
index 8e072de89fed..000000000000
--- a/arch/arm/mach-omap2/serial.c
+++ /dev/null
@@ -1,332 +0,0 @@
-/*
- * arch/arm/mach-omap2/serial.c
- *
- * OMAP2 serial support.
- *
- * Copyright (C) 2005-2008 Nokia Corporation
- * Author: Paul Mundt <paul.mundt@nokia.com>
- *
- * Major rework for PM support by Kevin Hilman
- *
- * Based off of arch/arm/mach-omap/omap1/serial.c
- *
- * Copyright (C) 2009 Texas Instruments
- * Added OMAP4 support - Santosh Shilimkar <santosh.shilimkar@ti.com
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
- */
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/clk.h>
-#include <linux/io.h>
-#include <linux/delay.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-#include <linux/pm_runtime.h>
-#include <linux/console.h>
-#include <linux/omap-dma.h>
-#include <linux/platform_data/serial-omap.h>
-
-#include "common.h"
-#include "omap_hwmod.h"
-#include "omap_device.h"
-#include "omap-pm.h"
-#include "soc.h"
-#include "prm2xxx_3xxx.h"
-#include "pm.h"
-#include "cm2xxx_3xxx.h"
-#include "prm-regbits-34xx.h"
-#include "control.h"
-#include "mux.h"
-#include "serial.h"
-
-/*
- * NOTE: By default the serial auto_suspend timeout is disabled as it causes
- * lost characters over the serial ports. This means that the UART clocks will
- * stay on until power/autosuspend_delay is set for the uart from sysfs.
- * This also causes that any deeper omap sleep states are blocked.
- */
-#define DEFAULT_AUTOSUSPEND_DELAY	-1
-
-#define MAX_UART_HWMOD_NAME_LEN		16
-
-struct omap_uart_state {
-	int num;
-
-	struct list_head node;
-	struct omap_hwmod *oh;
-	struct omap_device_pad default_omap_uart_pads[2];
-};
-
-static LIST_HEAD(uart_list);
-static u8 num_uarts;
-static u8 console_uart_id = -1;
-static u8 uart_debug;
-
-#define DEFAULT_RXDMA_POLLRATE		1	/* RX DMA polling rate (us) */
-#define DEFAULT_RXDMA_BUFSIZE		4096	/* RX DMA buffer size */
-#define DEFAULT_RXDMA_TIMEOUT		(3 * HZ)/* RX DMA timeout (jiffies) */
-
-static struct omap_uart_port_info omap_serial_default_info[] __initdata = {
-	{
-		.dma_enabled	= false,
-		.dma_rx_buf_size = DEFAULT_RXDMA_BUFSIZE,
-		.dma_rx_poll_rate = DEFAULT_RXDMA_POLLRATE,
-		.dma_rx_timeout = DEFAULT_RXDMA_TIMEOUT,
-		.autosuspend_timeout = DEFAULT_AUTOSUSPEND_DELAY,
-	},
-};
-
-#ifdef CONFIG_PM
-static void omap_uart_enable_wakeup(struct device *dev, bool enable)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct omap_device *od = to_omap_device(pdev);
-
-	if (!od)
-		return;
-
-	if (enable)
-		omap_hwmod_enable_wakeup(od->hwmods[0]);
-	else
-		omap_hwmod_disable_wakeup(od->hwmods[0]);
-}
-
-#else
-static void omap_uart_enable_wakeup(struct device *dev, bool enable)
-{}
-#endif /* CONFIG_PM */
-
-#ifdef CONFIG_OMAP_MUX
-
-#define OMAP_UART_DEFAULT_PAD_NAME_LEN	28
-static char rx_pad_name[OMAP_UART_DEFAULT_PAD_NAME_LEN],
-		tx_pad_name[OMAP_UART_DEFAULT_PAD_NAME_LEN] __initdata;
-
-static void  __init
-omap_serial_fill_uart_tx_rx_pads(struct omap_board_data *bdata,
-				struct omap_uart_state *uart)
-{
-	uart->default_omap_uart_pads[0].name = rx_pad_name;
-	uart->default_omap_uart_pads[0].flags = OMAP_DEVICE_PAD_REMUX |
-							OMAP_DEVICE_PAD_WAKEUP;
-	uart->default_omap_uart_pads[0].enable = OMAP_PIN_INPUT |
-							OMAP_MUX_MODE0;
-	uart->default_omap_uart_pads[0].idle = OMAP_PIN_INPUT | OMAP_MUX_MODE0;
-	uart->default_omap_uart_pads[1].name = tx_pad_name;
-	uart->default_omap_uart_pads[1].enable = OMAP_PIN_OUTPUT |
-							OMAP_MUX_MODE0;
-	bdata->pads = uart->default_omap_uart_pads;
-	bdata->pads_cnt = ARRAY_SIZE(uart->default_omap_uart_pads);
-}
-
-static void  __init omap_serial_check_wakeup(struct omap_board_data *bdata,
-						struct omap_uart_state *uart)
-{
-	struct omap_mux_partition *tx_partition = NULL, *rx_partition = NULL;
-	struct omap_mux *rx_mux = NULL, *tx_mux = NULL;
-	char *rx_fmt, *tx_fmt;
-	int uart_nr = bdata->id + 1;
-
-	if (bdata->id != 2) {
-		rx_fmt = "uart%d_rx.uart%d_rx";
-		tx_fmt = "uart%d_tx.uart%d_tx";
-	} else {
-		rx_fmt = "uart%d_rx_irrx.uart%d_rx_irrx";
-		tx_fmt = "uart%d_tx_irtx.uart%d_tx_irtx";
-	}
-
-	snprintf(rx_pad_name, OMAP_UART_DEFAULT_PAD_NAME_LEN, rx_fmt,
-			uart_nr, uart_nr);
-	snprintf(tx_pad_name, OMAP_UART_DEFAULT_PAD_NAME_LEN, tx_fmt,
-			uart_nr, uart_nr);
-
-	if (omap_mux_get_by_name(rx_pad_name, &rx_partition, &rx_mux) >= 0 &&
-			omap_mux_get_by_name
-				(tx_pad_name, &tx_partition, &tx_mux) >= 0) {
-		u16 tx_mode, rx_mode;
-
-		tx_mode = omap_mux_read(tx_partition, tx_mux->reg_offset);
-		rx_mode = omap_mux_read(rx_partition, rx_mux->reg_offset);
-
-		/*
-		 * Check if uart is used in default tx/rx mode i.e. in mux mode0
-		 * if yes then configure rx pin for wake up capability
-		 */
-		if (OMAP_MODE_UART(rx_mode) && OMAP_MODE_UART(tx_mode))
-			omap_serial_fill_uart_tx_rx_pads(bdata, uart);
-	}
-}
-#else
-static void __init omap_serial_check_wakeup(struct omap_board_data *bdata,
-		struct omap_uart_state *uart)
-{
-}
-#endif
-
-static char *cmdline_find_option(char *str)
-{
-	extern char *saved_command_line;
-
-	return strstr(saved_command_line, str);
-}
-
-static int __init omap_serial_early_init(void)
-{
-	if (of_have_populated_dt())
-		return -ENODEV;
-
-	do {
-		char oh_name[MAX_UART_HWMOD_NAME_LEN];
-		struct omap_hwmod *oh;
-		struct omap_uart_state *uart;
-		char uart_name[MAX_UART_HWMOD_NAME_LEN];
-
-		snprintf(oh_name, MAX_UART_HWMOD_NAME_LEN,
-			 "uart%d", num_uarts + 1);
-		oh = omap_hwmod_lookup(oh_name);
-		if (!oh)
-			break;
-
-		uart = kzalloc(sizeof(struct omap_uart_state), GFP_KERNEL);
-		if (WARN_ON(!uart))
-			return -ENODEV;
-
-		uart->oh = oh;
-		uart->num = num_uarts++;
-		list_add_tail(&uart->node, &uart_list);
-		snprintf(uart_name, MAX_UART_HWMOD_NAME_LEN,
-				"%s%d", OMAP_SERIAL_NAME, uart->num);
-
-		if (cmdline_find_option(uart_name)) {
-			console_uart_id = uart->num;
-
-			if (console_loglevel >= CONSOLE_LOGLEVEL_DEBUG) {
-				uart_debug = true;
-				pr_info("%s used as console in debug mode: uart%d clocks will not be gated",
-					uart_name, uart->num);
-			}
-		}
-	} while (1);
-
-	return 0;
-}
-omap_postcore_initcall(omap_serial_early_init);
-
-/**
- * omap_serial_init_port() - initialize single serial port
- * @bdata: port specific board data pointer
- * @info: platform specific data pointer
- *
- * This function initialies serial driver for given port only.
- * Platforms can call this function instead of omap_serial_init()
- * if they don't plan to use all available UARTs as serial ports.
- *
- * Don't mix calls to omap_serial_init_port() and omap_serial_init(),
- * use only one of the two.
- */
-void __init omap_serial_init_port(struct omap_board_data *bdata,
-			struct omap_uart_port_info *info)
-{
-	struct omap_uart_state *uart;
-	struct omap_hwmod *oh;
-	struct platform_device *pdev;
-	void *pdata = NULL;
-	u32 pdata_size = 0;
-	char *name;
-	struct omap_uart_port_info omap_up;
-
-	if (WARN_ON(!bdata))
-		return;
-	if (WARN_ON(bdata->id < 0))
-		return;
-	if (WARN_ON(bdata->id >= num_uarts))
-		return;
-
-	list_for_each_entry(uart, &uart_list, node)
-		if (bdata->id == uart->num)
-			break;
-	if (!info)
-		info = omap_serial_default_info;
-
-	oh = uart->oh;
-	name = OMAP_SERIAL_DRIVER_NAME;
-
-	omap_up.dma_enabled = info->dma_enabled;
-	omap_up.uartclk = OMAP24XX_BASE_BAUD * 16;
-	omap_up.flags = UPF_BOOT_AUTOCONF;
-	omap_up.get_context_loss_count = omap_pm_get_dev_context_loss_count;
-	omap_up.enable_wakeup = omap_uart_enable_wakeup;
-	omap_up.dma_rx_buf_size = info->dma_rx_buf_size;
-	omap_up.dma_rx_timeout = info->dma_rx_timeout;
-	omap_up.dma_rx_poll_rate = info->dma_rx_poll_rate;
-	omap_up.autosuspend_timeout = info->autosuspend_timeout;
-
-	pdata = &omap_up;
-	pdata_size = sizeof(struct omap_uart_port_info);
-
-	if (WARN_ON(!oh))
-		return;
-
-	pdev = omap_device_build(name, uart->num, oh, pdata, pdata_size);
-	if (IS_ERR(pdev)) {
-		WARN(1, "Could not build omap_device for %s: %s.\n", name,
-		     oh->name);
-		return;
-	}
-
-	oh->mux = omap_hwmod_mux_init(bdata->pads, bdata->pads_cnt);
-
-	if (console_uart_id == bdata->id) {
-		omap_device_enable(pdev);
-		pm_runtime_set_active(&pdev->dev);
-	}
-
-	oh->dev_attr = uart;
-
-	if (((cpu_is_omap34xx() || cpu_is_omap44xx()) && bdata->pads)
-			&& !uart_debug)
-		device_init_wakeup(&pdev->dev, true);
-}
-
-/**
- * omap_serial_board_init() - initialize all supported serial ports
- * @info: platform specific data pointer
- *
- * Initializes all available UARTs as serial ports. Platforms
- * can call this function when they want to have default behaviour
- * for serial ports (e.g initialize them all as serial ports).
- */
-void __init omap_serial_board_init(struct omap_uart_port_info *info)
-{
-	struct omap_uart_state *uart;
-	struct omap_board_data bdata;
-
-	list_for_each_entry(uart, &uart_list, node) {
-		bdata.id = uart->num;
-		bdata.flags = 0;
-		bdata.pads = NULL;
-		bdata.pads_cnt = 0;
-
-		omap_serial_check_wakeup(&bdata, uart);
-
-		if (!info)
-			omap_serial_init_port(&bdata, NULL);
-		else
-			omap_serial_init_port(&bdata, &info[uart->num]);
-	}
-}
-
-/**
- * omap_serial_init() - initialize all supported serial ports
- *
- * Initializes all available UARTs.
- * Platforms can call this function when they want to have default behaviour
- * for serial ports (e.g initialize them all as serial ports).
- */
-void __init omap_serial_init(void)
-{
-	omap_serial_board_init(NULL);
-}
diff --git a/arch/arm/mach-omap2/twl-common.c b/arch/arm/mach-omap2/twl-common.c
deleted file mode 100644
index a72738eab009..000000000000
--- a/arch/arm/mach-omap2/twl-common.c
+++ /dev/null
@@ -1,564 +0,0 @@
-/*
- * twl-common.c
- *
- * Copyright (C) 2011 Texas Instruments, Inc..
- * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/i2c.h>
-#include <linux/i2c/twl.h>
-#include <linux/gpio.h>
-#include <linux/string.h>
-#include <linux/phy/phy.h>
-#include <linux/regulator/machine.h>
-#include <linux/regulator/fixed.h>
-
-#include "soc.h"
-#include "twl-common.h"
-#include "pm.h"
-#include "voltage.h"
-#include "mux.h"
-
-static struct i2c_board_info __initdata pmic_i2c_board_info = {
-	.addr		= 0x48,
-	.flags		= I2C_CLIENT_WAKE,
-};
-
-#if defined(CONFIG_ARCH_OMAP3) || defined(CONFIG_ARCH_OMAP4)
-static int twl_set_voltage(void *data, int target_uV)
-{
-	struct voltagedomain *voltdm = (struct voltagedomain *)data;
-	return voltdm_scale(voltdm, target_uV);
-}
-
-static int twl_get_voltage(void *data)
-{
-	struct voltagedomain *voltdm = (struct voltagedomain *)data;
-	return voltdm_get_voltage(voltdm);
-}
-#endif
-
-void __init omap_pmic_init(int bus, u32 clkrate,
-			   const char *pmic_type, int pmic_irq,
-			   struct twl4030_platform_data *pmic_data)
-{
-	omap_mux_init_signal("sys_nirq", OMAP_PIN_INPUT_PULLUP | OMAP_PIN_OFF_WAKEUPENABLE);
-	strlcpy(pmic_i2c_board_info.type, pmic_type,
-		sizeof(pmic_i2c_board_info.type));
-	pmic_i2c_board_info.irq = pmic_irq;
-	pmic_i2c_board_info.platform_data = pmic_data;
-
-	omap_register_i2c_bus(bus, clkrate, &pmic_i2c_board_info, 1);
-}
-
-#ifdef CONFIG_ARCH_OMAP4
-void __init omap4_pmic_init(const char *pmic_type,
-		    struct twl4030_platform_data *pmic_data,
-		    struct i2c_board_info *devices, int nr_devices)
-{
-	/* PMIC part*/
-	unsigned int irq;
-
-	omap_mux_init_signal("sys_nirq1", OMAP_PIN_INPUT_PULLUP | OMAP_PIN_OFF_WAKEUPENABLE);
-	omap_mux_init_signal("fref_clk0_out.sys_drm_msecure", OMAP_PIN_OUTPUT);
-	irq = omap4_xlate_irq(7 + OMAP44XX_IRQ_GIC_START);
-	omap_pmic_init(1, 400, pmic_type, irq, pmic_data);
-
-	/* Register additional devices on i2c1 bus if needed */
-	if (devices)
-		i2c_register_board_info(1, devices, nr_devices);
-}
-#endif
-
-void __init omap_pmic_late_init(void)
-{
-	/* Init the OMAP TWL parameters (if PMIC has been registerd) */
-	if (!pmic_i2c_board_info.irq)
-		return;
-
-	omap3_twl_init();
-	omap4_twl_init();
-}
-
-#if defined(CONFIG_ARCH_OMAP3)
-static struct twl4030_usb_data omap3_usb_pdata = {
-	.usb_mode = T2_USB_MODE_ULPI,
-};
-
-static int omap3_batt_table[] = {
-/* 0 C */
-30800, 29500, 28300, 27100,
-26000, 24900, 23900, 22900, 22000, 21100, 20300, 19400, 18700, 17900,
-17200, 16500, 15900, 15300, 14700, 14100, 13600, 13100, 12600, 12100,
-11600, 11200, 10800, 10400, 10000, 9630,  9280,  8950,  8620,  8310,
-8020,  7730,  7460,  7200,  6950,  6710,  6470,  6250,  6040,  5830,
-5640,  5450,  5260,  5090,  4920,  4760,  4600,  4450,  4310,  4170,
-4040,  3910,  3790,  3670,  3550
-};
-
-static struct twl4030_bci_platform_data omap3_bci_pdata = {
-	.battery_tmp_tbl	= omap3_batt_table,
-	.tblsize		= ARRAY_SIZE(omap3_batt_table),
-};
-
-static struct twl4030_madc_platform_data omap3_madc_pdata = {
-	.irq_line	= 1,
-};
-
-static struct twl4030_codec_data omap3_codec;
-
-static struct twl4030_audio_data omap3_audio_pdata = {
-	.audio_mclk = 26000000,
-	.codec = &omap3_codec,
-};
-
-static struct regulator_consumer_supply omap3_vdda_dac_supplies[] = {
-	REGULATOR_SUPPLY("vdda_dac", "omapdss_venc"),
-};
-
-static struct regulator_init_data omap3_vdac_idata = {
-	.constraints = {
-		.min_uV			= 1800000,
-		.max_uV			= 1800000,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL
-					| REGULATOR_MODE_STANDBY,
-		.valid_ops_mask		= REGULATOR_CHANGE_MODE
-					| REGULATOR_CHANGE_STATUS,
-	},
-	.num_consumer_supplies	= ARRAY_SIZE(omap3_vdda_dac_supplies),
-	.consumer_supplies	= omap3_vdda_dac_supplies,
-};
-
-static struct regulator_consumer_supply omap3_vpll2_supplies[] = {
-	REGULATOR_SUPPLY("vdds_dsi", "omapdss"),
-	REGULATOR_SUPPLY("vdds_dsi", "omapdss_dpi.0"),
-	REGULATOR_SUPPLY("vdds_dsi", "omapdss_dsi.0"),
-};
-
-static struct regulator_init_data omap3_vpll2_idata = {
-	.constraints = {
-		.min_uV                 = 1800000,
-		.max_uV                 = 1800000,
-		.valid_modes_mask       = REGULATOR_MODE_NORMAL
-					| REGULATOR_MODE_STANDBY,
-		.valid_ops_mask         = REGULATOR_CHANGE_MODE
-					| REGULATOR_CHANGE_STATUS,
-	},
-	.num_consumer_supplies		= ARRAY_SIZE(omap3_vpll2_supplies),
-	.consumer_supplies		= omap3_vpll2_supplies,
-};
-
-static struct regulator_consumer_supply omap3_vdd1_supply[] = {
-	REGULATOR_SUPPLY("vcc", "cpu0"),
-};
-
-static struct regulator_consumer_supply omap3_vdd2_supply[] = {
-	REGULATOR_SUPPLY("vcc", "l3_main.0"),
-};
-
-static struct regulator_init_data omap3_vdd1 = {
-	.constraints = {
-		.name			= "vdd_mpu_iva",
-		.min_uV			= 600000,
-		.max_uV			= 1450000,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL,
-		.valid_ops_mask		= REGULATOR_CHANGE_VOLTAGE,
-	},
-	.num_consumer_supplies		= ARRAY_SIZE(omap3_vdd1_supply),
-	.consumer_supplies		= omap3_vdd1_supply,
-};
-
-static struct regulator_init_data omap3_vdd2 = {
-	.constraints = {
-		.name			= "vdd_core",
-		.min_uV			= 600000,
-		.max_uV			= 1450000,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL,
-		.valid_ops_mask		= REGULATOR_CHANGE_VOLTAGE,
-	},
-	.num_consumer_supplies		= ARRAY_SIZE(omap3_vdd2_supply),
-	.consumer_supplies		= omap3_vdd2_supply,
-};
-
-static struct twl_regulator_driver_data omap3_vdd1_drvdata = {
-	.get_voltage = twl_get_voltage,
-	.set_voltage = twl_set_voltage,
-};
-
-static struct twl_regulator_driver_data omap3_vdd2_drvdata = {
-	.get_voltage = twl_get_voltage,
-	.set_voltage = twl_set_voltage,
-};
-
-void __init omap3_pmic_get_config(struct twl4030_platform_data *pmic_data,
-				  u32 pdata_flags, u32 regulators_flags)
-{
-	if (!pmic_data->vdd1) {
-		omap3_vdd1.driver_data = &omap3_vdd1_drvdata;
-		omap3_vdd1_drvdata.data = voltdm_lookup("mpu_iva");
-		pmic_data->vdd1 = &omap3_vdd1;
-	}
-	if (!pmic_data->vdd2) {
-		omap3_vdd2.driver_data = &omap3_vdd2_drvdata;
-		omap3_vdd2_drvdata.data = voltdm_lookup("core");
-		pmic_data->vdd2 = &omap3_vdd2;
-	}
-
-	/* Common platform data configurations */
-	if (pdata_flags & TWL_COMMON_PDATA_USB && !pmic_data->usb)
-		pmic_data->usb = &omap3_usb_pdata;
-
-	if (pdata_flags & TWL_COMMON_PDATA_BCI && !pmic_data->bci)
-		pmic_data->bci = &omap3_bci_pdata;
-
-	if (pdata_flags & TWL_COMMON_PDATA_MADC && !pmic_data->madc)
-		pmic_data->madc = &omap3_madc_pdata;
-
-	if (pdata_flags & TWL_COMMON_PDATA_AUDIO && !pmic_data->audio)
-		pmic_data->audio = &omap3_audio_pdata;
-
-	/* Common regulator configurations */
-	if (regulators_flags & TWL_COMMON_REGULATOR_VDAC && !pmic_data->vdac)
-		pmic_data->vdac = &omap3_vdac_idata;
-
-	if (regulators_flags & TWL_COMMON_REGULATOR_VPLL2 && !pmic_data->vpll2)
-		pmic_data->vpll2 = &omap3_vpll2_idata;
-}
-#endif /* CONFIG_ARCH_OMAP3 */
-
-#if defined(CONFIG_ARCH_OMAP4)
-static struct twl4030_usb_data omap4_usb_pdata = {
-};
-
-static struct regulator_consumer_supply omap4_vdda_hdmi_dac_supplies[] = {
-	REGULATOR_SUPPLY("vdda_hdmi_dac", "omapdss_hdmi"),
-};
-
-static struct regulator_init_data omap4_vdac_idata = {
-	.constraints = {
-		.min_uV			= 1800000,
-		.max_uV			= 1800000,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL
-					| REGULATOR_MODE_STANDBY,
-		.valid_ops_mask		= REGULATOR_CHANGE_MODE
-					| REGULATOR_CHANGE_STATUS,
-	},
-	.num_consumer_supplies	= ARRAY_SIZE(omap4_vdda_hdmi_dac_supplies),
-	.consumer_supplies	= omap4_vdda_hdmi_dac_supplies,
-	.supply_regulator	= "V2V1",
-};
-
-static struct regulator_init_data omap4_vaux2_idata = {
-	.constraints = {
-		.min_uV			= 1200000,
-		.max_uV			= 2800000,
-		.apply_uV		= true,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL
-					| REGULATOR_MODE_STANDBY,
-		.valid_ops_mask		= REGULATOR_CHANGE_VOLTAGE
-					| REGULATOR_CHANGE_MODE
-					| REGULATOR_CHANGE_STATUS,
-	},
-};
-
-static struct regulator_init_data omap4_vaux3_idata = {
-	.constraints = {
-		.min_uV			= 1000000,
-		.max_uV			= 3000000,
-		.apply_uV		= true,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL
-					| REGULATOR_MODE_STANDBY,
-		.valid_ops_mask		= REGULATOR_CHANGE_VOLTAGE
-					| REGULATOR_CHANGE_MODE
-					| REGULATOR_CHANGE_STATUS,
-	},
-};
-
-static struct regulator_consumer_supply omap4_vmmc_supply[] = {
-	REGULATOR_SUPPLY("vmmc", "omap_hsmmc.0"),
-};
-
-/* VMMC1 for MMC1 card */
-static struct regulator_init_data omap4_vmmc_idata = {
-	.constraints = {
-		.min_uV			= 1200000,
-		.max_uV			= 3000000,
-		.apply_uV		= true,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL
-					| REGULATOR_MODE_STANDBY,
-		.valid_ops_mask		= REGULATOR_CHANGE_VOLTAGE
-					| REGULATOR_CHANGE_MODE
-					| REGULATOR_CHANGE_STATUS,
-	},
-	.num_consumer_supplies  = ARRAY_SIZE(omap4_vmmc_supply),
-	.consumer_supplies      = omap4_vmmc_supply,
-};
-
-static struct regulator_init_data omap4_vpp_idata = {
-	.constraints = {
-		.min_uV			= 1800000,
-		.max_uV			= 2500000,
-		.apply_uV		= true,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL
-					| REGULATOR_MODE_STANDBY,
-		.valid_ops_mask		= REGULATOR_CHANGE_VOLTAGE
-					| REGULATOR_CHANGE_MODE
-					| REGULATOR_CHANGE_STATUS,
-	},
-};
-
-static struct regulator_init_data omap4_vana_idata = {
-	.constraints = {
-		.min_uV			= 2100000,
-		.max_uV			= 2100000,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL
-					| REGULATOR_MODE_STANDBY,
-		.valid_ops_mask		= REGULATOR_CHANGE_MODE
-					| REGULATOR_CHANGE_STATUS,
-	},
-};
-
-static struct regulator_consumer_supply omap4_vcxio_supply[] = {
-	REGULATOR_SUPPLY("vdds_dsi", "omapdss_dss"),
-	REGULATOR_SUPPLY("vdds_dsi", "omapdss_dsi.0"),
-	REGULATOR_SUPPLY("vdds_dsi", "omapdss_dsi.1"),
-};
-
-static struct regulator_init_data omap4_vcxio_idata = {
-	.constraints = {
-		.min_uV			= 1800000,
-		.max_uV			= 1800000,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL
-					| REGULATOR_MODE_STANDBY,
-		.valid_ops_mask		= REGULATOR_CHANGE_MODE
-					| REGULATOR_CHANGE_STATUS,
-		.always_on		= true,
-	},
-	.num_consumer_supplies	= ARRAY_SIZE(omap4_vcxio_supply),
-	.consumer_supplies	= omap4_vcxio_supply,
-	.supply_regulator	= "V2V1",
-};
-
-static struct regulator_init_data omap4_vusb_idata = {
-	.constraints = {
-		.min_uV			= 3300000,
-		.max_uV			= 3300000,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL
-					| REGULATOR_MODE_STANDBY,
-		.valid_ops_mask		= REGULATOR_CHANGE_MODE
-					| REGULATOR_CHANGE_STATUS,
-	},
-};
-
-static struct regulator_init_data omap4_clk32kg_idata = {
-	.constraints = {
-		.valid_ops_mask		= REGULATOR_CHANGE_STATUS,
-	},
-};
-
-static struct regulator_consumer_supply omap4_vdd1_supply[] = {
-	REGULATOR_SUPPLY("vcc", "cpu0"),
-};
-
-static struct regulator_consumer_supply omap4_vdd2_supply[] = {
-	REGULATOR_SUPPLY("vcc", "iva.0"),
-};
-
-static struct regulator_consumer_supply omap4_vdd3_supply[] = {
-	REGULATOR_SUPPLY("vcc", "l3_main.0"),
-};
-
-static struct regulator_init_data omap4_vdd1 = {
-	.constraints = {
-		.name			= "vdd_mpu",
-		.min_uV			= 500000,
-		.max_uV			= 1500000,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL,
-		.valid_ops_mask		= REGULATOR_CHANGE_VOLTAGE,
-	},
-	.num_consumer_supplies		= ARRAY_SIZE(omap4_vdd1_supply),
-	.consumer_supplies		= omap4_vdd1_supply,
-};
-
-static struct regulator_init_data omap4_vdd2 = {
-	.constraints = {
-		.name			= "vdd_iva",
-		.min_uV			= 500000,
-		.max_uV			= 1500000,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL,
-		.valid_ops_mask		= REGULATOR_CHANGE_VOLTAGE,
-	},
-	.num_consumer_supplies		= ARRAY_SIZE(omap4_vdd2_supply),
-	.consumer_supplies		= omap4_vdd2_supply,
-};
-
-static struct regulator_init_data omap4_vdd3 = {
-	.constraints = {
-		.name			= "vdd_core",
-		.min_uV			= 500000,
-		.max_uV			= 1500000,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL,
-		.valid_ops_mask		= REGULATOR_CHANGE_VOLTAGE,
-	},
-	.num_consumer_supplies		= ARRAY_SIZE(omap4_vdd3_supply),
-	.consumer_supplies		= omap4_vdd3_supply,
-};
-
-
-static struct twl_regulator_driver_data omap4_vdd1_drvdata = {
-	.get_voltage = twl_get_voltage,
-	.set_voltage = twl_set_voltage,
-};
-
-static struct twl_regulator_driver_data omap4_vdd2_drvdata = {
-	.get_voltage = twl_get_voltage,
-	.set_voltage = twl_set_voltage,
-};
-
-static struct twl_regulator_driver_data omap4_vdd3_drvdata = {
-	.get_voltage = twl_get_voltage,
-	.set_voltage = twl_set_voltage,
-};
-
-static struct regulator_consumer_supply omap4_v1v8_supply[] = {
-	REGULATOR_SUPPLY("vio", "1-004b"),
-};
-
-static struct regulator_init_data omap4_v1v8_idata = {
-	.constraints = {
-		.min_uV			= 1800000,
-		.max_uV			= 1800000,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL
-					| REGULATOR_MODE_STANDBY,
-		.valid_ops_mask		= REGULATOR_CHANGE_MODE
-					| REGULATOR_CHANGE_STATUS,
-		.always_on		= true,
-	},
-	.num_consumer_supplies	= ARRAY_SIZE(omap4_v1v8_supply),
-	.consumer_supplies	= omap4_v1v8_supply,
-};
-
-static struct regulator_consumer_supply omap4_v2v1_supply[] = {
-	REGULATOR_SUPPLY("v2v1", "1-004b"),
-};
-
-static struct regulator_init_data omap4_v2v1_idata = {
-	.constraints = {
-		.min_uV			= 2100000,
-		.max_uV			= 2100000,
-		.valid_modes_mask	= REGULATOR_MODE_NORMAL
-					| REGULATOR_MODE_STANDBY,
-		.valid_ops_mask		= REGULATOR_CHANGE_MODE
-					| REGULATOR_CHANGE_STATUS,
-	},
-	.num_consumer_supplies	= ARRAY_SIZE(omap4_v2v1_supply),
-	.consumer_supplies	= omap4_v2v1_supply,
-};
-
-void __init omap4_pmic_get_config(struct twl4030_platform_data *pmic_data,
-				  u32 pdata_flags, u32 regulators_flags)
-{
-	if (!pmic_data->vdd1) {
-		omap4_vdd1.driver_data = &omap4_vdd1_drvdata;
-		omap4_vdd1_drvdata.data = voltdm_lookup("mpu");
-		pmic_data->vdd1 = &omap4_vdd1;
-	}
-
-	if (!pmic_data->vdd2) {
-		omap4_vdd2.driver_data = &omap4_vdd2_drvdata;
-		omap4_vdd2_drvdata.data = voltdm_lookup("iva");
-		pmic_data->vdd2 = &omap4_vdd2;
-	}
-
-	if (!pmic_data->vdd3) {
-		omap4_vdd3.driver_data = &omap4_vdd3_drvdata;
-		omap4_vdd3_drvdata.data = voltdm_lookup("core");
-		pmic_data->vdd3 = &omap4_vdd3;
-	}
-
-	/* Common platform data configurations */
-	if (pdata_flags & TWL_COMMON_PDATA_USB && !pmic_data->usb)
-		pmic_data->usb = &omap4_usb_pdata;
-
-	/* Common regulator configurations */
-	if (regulators_flags & TWL_COMMON_REGULATOR_VDAC && !pmic_data->vdac)
-		pmic_data->vdac = &omap4_vdac_idata;
-
-	if (regulators_flags & TWL_COMMON_REGULATOR_VAUX2 && !pmic_data->vaux2)
-		pmic_data->vaux2 = &omap4_vaux2_idata;
-
-	if (regulators_flags & TWL_COMMON_REGULATOR_VAUX3 && !pmic_data->vaux3)
-		pmic_data->vaux3 = &omap4_vaux3_idata;
-
-	if (regulators_flags & TWL_COMMON_REGULATOR_VMMC && !pmic_data->vmmc)
-		pmic_data->vmmc = &omap4_vmmc_idata;
-
-	if (regulators_flags & TWL_COMMON_REGULATOR_VPP && !pmic_data->vpp)
-		pmic_data->vpp = &omap4_vpp_idata;
-
-	if (regulators_flags & TWL_COMMON_REGULATOR_VANA && !pmic_data->vana)
-		pmic_data->vana = &omap4_vana_idata;
-
-	if (regulators_flags & TWL_COMMON_REGULATOR_VCXIO && !pmic_data->vcxio)
-		pmic_data->vcxio = &omap4_vcxio_idata;
-
-	if (regulators_flags & TWL_COMMON_REGULATOR_VUSB && !pmic_data->vusb)
-		pmic_data->vusb = &omap4_vusb_idata;
-
-	if (regulators_flags & TWL_COMMON_REGULATOR_CLK32KG &&
-	    !pmic_data->clk32kg)
-		pmic_data->clk32kg = &omap4_clk32kg_idata;
-
-	if (regulators_flags & TWL_COMMON_REGULATOR_V1V8 && !pmic_data->v1v8)
-		pmic_data->v1v8 = &omap4_v1v8_idata;
-
-	if (regulators_flags & TWL_COMMON_REGULATOR_V2V1 && !pmic_data->v2v1)
-		pmic_data->v2v1 = &omap4_v2v1_idata;
-}
-#endif /* CONFIG_ARCH_OMAP4 */
-
-#if IS_ENABLED(CONFIG_SND_OMAP_SOC_OMAP_TWL4030)
-#include <linux/platform_data/omap-twl4030.h>
-
-/* Commonly used configuration */
-static struct omap_tw4030_pdata omap_twl4030_audio_data;
-
-static struct platform_device audio_device = {
-	.name		= "omap-twl4030",
-	.id		= -1,
-};
-
-void omap_twl4030_audio_init(char *card_name,
-				    struct omap_tw4030_pdata *pdata)
-{
-	if (!pdata)
-		pdata = &omap_twl4030_audio_data;
-
-	pdata->card_name = card_name;
-
-	audio_device.dev.platform_data = pdata;
-	platform_device_register(&audio_device);
-}
-
-#else /* SOC_OMAP_TWL4030 */
-void omap_twl4030_audio_init(char *card_name,
-				    struct omap_tw4030_pdata *pdata)
-{
-	return;
-}
-#endif /* SOC_OMAP_TWL4030 */
diff --git a/arch/arm/mach-omap2/twl-common.h b/arch/arm/mach-omap2/twl-common.h
deleted file mode 100644
index 24b65d081b69..000000000000
--- a/arch/arm/mach-omap2/twl-common.h
+++ /dev/null
@@ -1,66 +0,0 @@
-#ifndef __OMAP_PMIC_COMMON__
-#define __OMAP_PMIC_COMMON__
-
-#include "common.h"
-
-#define TWL_COMMON_PDATA_USB		(1 << 0)
-#define TWL_COMMON_PDATA_BCI		(1 << 1)
-#define TWL_COMMON_PDATA_MADC		(1 << 2)
-#define TWL_COMMON_PDATA_AUDIO		(1 << 3)
-
-/* Common LDO regulators for TWL4030/TWL6030 */
-#define TWL_COMMON_REGULATOR_VDAC	(1 << 0)
-#define TWL_COMMON_REGULATOR_VAUX1	(1 << 1)
-#define TWL_COMMON_REGULATOR_VAUX2	(1 << 2)
-#define TWL_COMMON_REGULATOR_VAUX3	(1 << 3)
-
-/* TWL6030 LDO regulators */
-#define TWL_COMMON_REGULATOR_VMMC	(1 << 4)
-#define TWL_COMMON_REGULATOR_VPP	(1 << 5)
-#define TWL_COMMON_REGULATOR_VUSIM	(1 << 6)
-#define TWL_COMMON_REGULATOR_VANA	(1 << 7)
-#define TWL_COMMON_REGULATOR_VCXIO	(1 << 8)
-#define TWL_COMMON_REGULATOR_VUSB	(1 << 9)
-#define TWL_COMMON_REGULATOR_CLK32KG	(1 << 10)
-#define TWL_COMMON_REGULATOR_V1V8	(1 << 11)
-#define TWL_COMMON_REGULATOR_V2V1	(1 << 12)
-
-/* TWL4030 LDO regulators */
-#define TWL_COMMON_REGULATOR_VPLL1	(1 << 4)
-#define TWL_COMMON_REGULATOR_VPLL2	(1 << 5)
-
-
-struct twl4030_platform_data;
-struct twl6040_platform_data;
-struct omap_tw4030_pdata;
-struct i2c_board_info;
-
-void omap_pmic_init(int bus, u32 clkrate, const char *pmic_type, int pmic_irq,
-		    struct twl4030_platform_data *pmic_data);
-void omap_pmic_late_init(void);
-
-static inline void omap2_pmic_init(const char *pmic_type,
-				   struct twl4030_platform_data *pmic_data)
-{
-	omap_pmic_init(2, 2600, pmic_type, 7 + OMAP_INTC_START, pmic_data);
-}
-
-static inline void omap3_pmic_init(const char *pmic_type,
-				   struct twl4030_platform_data *pmic_data)
-{
-	omap_pmic_init(1, 2600, pmic_type, 7 + OMAP_INTC_START, pmic_data);
-}
-
-void omap4_pmic_init(const char *pmic_type,
-		    struct twl4030_platform_data *pmic_data,
-		    struct i2c_board_info *devices, int nr_devices);
-
-void omap3_pmic_get_config(struct twl4030_platform_data *pmic_data,
-			   u32 pdata_flags, u32 regulators_flags);
-
-void omap4_pmic_get_config(struct twl4030_platform_data *pmic_data,
-			   u32 pdata_flags, u32 regulators_flags);
-
-void omap_twl4030_audio_init(char *card_name, struct omap_tw4030_pdata *pdata);
-
-#endif /* __OMAP_PMIC_COMMON__ */
diff --git a/arch/arm/mach-omap2/usb-host.c b/arch/arm/mach-omap2/usb-host.c
deleted file mode 100644
index 745367c0c2bb..000000000000
--- a/arch/arm/mach-omap2/usb-host.c
+++ /dev/null
@@ -1,496 +0,0 @@
-/*
- * usb-host.c - OMAP USB Host
- *
- * This file will contain the board specific details for the
- * Synopsys EHCI/OHCI host controller on OMAP3430 and onwards
- *
- * Copyright (C) 2007-2011 Texas Instruments
- * Author: Vikram Pandita <vikram.pandita@ti.com>
- * Author: Keshava Munegowda <keshava_mgowda@ti.com>
- *
- * Generalization by:
- * Felipe Balbi <balbi@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/types.h>
-#include <linux/errno.h>
-#include <linux/delay.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-#include <linux/dma-mapping.h>
-#include <linux/regulator/machine.h>
-#include <linux/regulator/fixed.h>
-#include <linux/string.h>
-#include <linux/io.h>
-#include <linux/gpio.h>
-#include <linux/usb/phy.h>
-#include <linux/usb/usb_phy_generic.h>
-
-#include "soc.h"
-#include "omap_device.h"
-#include "mux.h"
-#include "usb.h"
-
-#ifdef CONFIG_MFD_OMAP_USB_HOST
-
-#define OMAP_USBHS_DEVICE	"usbhs_omap"
-#define OMAP_USBTLL_DEVICE	"usbhs_tll"
-#define	USBHS_UHH_HWMODNAME	"usb_host_hs"
-#define USBHS_TLL_HWMODNAME	"usb_tll_hs"
-
-/* MUX settings for EHCI pins */
-/*
- * setup_ehci_io_mux - initialize IO pad mux for USBHOST
- */
-static void __init setup_ehci_io_mux(const enum usbhs_omap_port_mode *port_mode)
-{
-	switch (port_mode[0]) {
-	case OMAP_EHCI_PORT_MODE_PHY:
-		omap_mux_init_signal("hsusb1_stp", OMAP_PIN_OUTPUT);
-		omap_mux_init_signal("hsusb1_clk", OMAP_PIN_OUTPUT);
-		omap_mux_init_signal("hsusb1_dir", OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_nxt", OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_data0", OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_data1", OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_data2", OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_data3", OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_data4", OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_data5", OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_data6", OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_data7", OMAP_PIN_INPUT_PULLDOWN);
-		break;
-	case OMAP_EHCI_PORT_MODE_TLL:
-		omap_mux_init_signal("hsusb1_tll_stp",
-			OMAP_PIN_INPUT_PULLUP);
-		omap_mux_init_signal("hsusb1_tll_clk",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_tll_dir",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_tll_nxt",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_tll_data0",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_tll_data1",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_tll_data2",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_tll_data3",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_tll_data4",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_tll_data5",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_tll_data6",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb1_tll_data7",
-			OMAP_PIN_INPUT_PULLDOWN);
-		break;
-	case OMAP_USBHS_PORT_MODE_UNUSED:
-		/* FALLTHROUGH */
-	default:
-		break;
-	}
-
-	switch (port_mode[1]) {
-	case OMAP_EHCI_PORT_MODE_PHY:
-		omap_mux_init_signal("hsusb2_stp", OMAP_PIN_OUTPUT);
-		omap_mux_init_signal("hsusb2_clk", OMAP_PIN_OUTPUT);
-		omap_mux_init_signal("hsusb2_dir", OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_nxt", OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_data0",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_data1",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_data2",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_data3",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_data4",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_data5",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_data6",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_data7",
-			OMAP_PIN_INPUT_PULLDOWN);
-		break;
-	case OMAP_EHCI_PORT_MODE_TLL:
-		omap_mux_init_signal("hsusb2_tll_stp",
-			OMAP_PIN_INPUT_PULLUP);
-		omap_mux_init_signal("hsusb2_tll_clk",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_tll_dir",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_tll_nxt",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_tll_data0",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_tll_data1",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_tll_data2",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_tll_data3",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_tll_data4",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_tll_data5",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_tll_data6",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb2_tll_data7",
-			OMAP_PIN_INPUT_PULLDOWN);
-		break;
-	case OMAP_USBHS_PORT_MODE_UNUSED:
-		/* FALLTHROUGH */
-	default:
-		break;
-	}
-
-	switch (port_mode[2]) {
-	case OMAP_EHCI_PORT_MODE_PHY:
-		printk(KERN_WARNING "Port3 can't be used in PHY mode\n");
-		break;
-	case OMAP_EHCI_PORT_MODE_TLL:
-		omap_mux_init_signal("hsusb3_tll_stp",
-			OMAP_PIN_INPUT_PULLUP);
-		omap_mux_init_signal("hsusb3_tll_clk",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb3_tll_dir",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb3_tll_nxt",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb3_tll_data0",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb3_tll_data1",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb3_tll_data2",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb3_tll_data3",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb3_tll_data4",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb3_tll_data5",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb3_tll_data6",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("hsusb3_tll_data7",
-			OMAP_PIN_INPUT_PULLDOWN);
-		break;
-	case OMAP_USBHS_PORT_MODE_UNUSED:
-		/* FALLTHROUGH */
-	default:
-		break;
-	}
-
-	return;
-}
-
-static void __init setup_ohci_io_mux(const enum usbhs_omap_port_mode *port_mode)
-{
-	switch (port_mode[0]) {
-	case OMAP_OHCI_PORT_MODE_PHY_6PIN_DATSE0:
-	case OMAP_OHCI_PORT_MODE_PHY_6PIN_DPDM:
-	case OMAP_OHCI_PORT_MODE_TLL_6PIN_DATSE0:
-	case OMAP_OHCI_PORT_MODE_TLL_6PIN_DPDM:
-		omap_mux_init_signal("mm1_rxdp",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("mm1_rxdm",
-			OMAP_PIN_INPUT_PULLDOWN);
-		/* FALLTHROUGH */
-	case OMAP_OHCI_PORT_MODE_PHY_4PIN_DPDM:
-	case OMAP_OHCI_PORT_MODE_TLL_4PIN_DPDM:
-		omap_mux_init_signal("mm1_rxrcv",
-			OMAP_PIN_INPUT_PULLDOWN);
-		/* FALLTHROUGH */
-	case OMAP_OHCI_PORT_MODE_PHY_3PIN_DATSE0:
-	case OMAP_OHCI_PORT_MODE_TLL_3PIN_DATSE0:
-		omap_mux_init_signal("mm1_txen_n", OMAP_PIN_OUTPUT);
-		/* FALLTHROUGH */
-	case OMAP_OHCI_PORT_MODE_TLL_2PIN_DATSE0:
-	case OMAP_OHCI_PORT_MODE_TLL_2PIN_DPDM:
-		omap_mux_init_signal("mm1_txse0",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("mm1_txdat",
-			OMAP_PIN_INPUT_PULLDOWN);
-		break;
-	case OMAP_USBHS_PORT_MODE_UNUSED:
-		/* FALLTHROUGH */
-	default:
-		break;
-	}
-	switch (port_mode[1]) {
-	case OMAP_OHCI_PORT_MODE_PHY_6PIN_DATSE0:
-	case OMAP_OHCI_PORT_MODE_PHY_6PIN_DPDM:
-	case OMAP_OHCI_PORT_MODE_TLL_6PIN_DATSE0:
-	case OMAP_OHCI_PORT_MODE_TLL_6PIN_DPDM:
-		omap_mux_init_signal("mm2_rxdp",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("mm2_rxdm",
-			OMAP_PIN_INPUT_PULLDOWN);
-		/* FALLTHROUGH */
-	case OMAP_OHCI_PORT_MODE_PHY_4PIN_DPDM:
-	case OMAP_OHCI_PORT_MODE_TLL_4PIN_DPDM:
-		omap_mux_init_signal("mm2_rxrcv",
-			OMAP_PIN_INPUT_PULLDOWN);
-		/* FALLTHROUGH */
-	case OMAP_OHCI_PORT_MODE_PHY_3PIN_DATSE0:
-	case OMAP_OHCI_PORT_MODE_TLL_3PIN_DATSE0:
-		omap_mux_init_signal("mm2_txen_n", OMAP_PIN_OUTPUT);
-		/* FALLTHROUGH */
-	case OMAP_OHCI_PORT_MODE_TLL_2PIN_DATSE0:
-	case OMAP_OHCI_PORT_MODE_TLL_2PIN_DPDM:
-		omap_mux_init_signal("mm2_txse0",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("mm2_txdat",
-			OMAP_PIN_INPUT_PULLDOWN);
-		break;
-	case OMAP_USBHS_PORT_MODE_UNUSED:
-		/* FALLTHROUGH */
-	default:
-		break;
-	}
-	switch (port_mode[2]) {
-	case OMAP_OHCI_PORT_MODE_PHY_6PIN_DATSE0:
-	case OMAP_OHCI_PORT_MODE_PHY_6PIN_DPDM:
-	case OMAP_OHCI_PORT_MODE_TLL_6PIN_DATSE0:
-	case OMAP_OHCI_PORT_MODE_TLL_6PIN_DPDM:
-		omap_mux_init_signal("mm3_rxdp",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("mm3_rxdm",
-			OMAP_PIN_INPUT_PULLDOWN);
-		/* FALLTHROUGH */
-	case OMAP_OHCI_PORT_MODE_PHY_4PIN_DPDM:
-	case OMAP_OHCI_PORT_MODE_TLL_4PIN_DPDM:
-		omap_mux_init_signal("mm3_rxrcv",
-			OMAP_PIN_INPUT_PULLDOWN);
-		/* FALLTHROUGH */
-	case OMAP_OHCI_PORT_MODE_PHY_3PIN_DATSE0:
-	case OMAP_OHCI_PORT_MODE_TLL_3PIN_DATSE0:
-		omap_mux_init_signal("mm3_txen_n", OMAP_PIN_OUTPUT);
-		/* FALLTHROUGH */
-	case OMAP_OHCI_PORT_MODE_TLL_2PIN_DATSE0:
-	case OMAP_OHCI_PORT_MODE_TLL_2PIN_DPDM:
-		omap_mux_init_signal("mm3_txse0",
-			OMAP_PIN_INPUT_PULLDOWN);
-		omap_mux_init_signal("mm3_txdat",
-			OMAP_PIN_INPUT_PULLDOWN);
-		break;
-	case OMAP_USBHS_PORT_MODE_UNUSED:
-		/* FALLTHROUGH */
-	default:
-		break;
-	}
-}
-
-void __init usbhs_init(struct usbhs_omap_platform_data *pdata)
-{
-	struct omap_hwmod	*uhh_hwm, *tll_hwm;
-	struct platform_device	*pdev;
-	int			bus_id = -1;
-
-	if (cpu_is_omap34xx()) {
-		setup_ehci_io_mux(pdata->port_mode);
-		setup_ohci_io_mux(pdata->port_mode);
-
-		if (omap_rev() <= OMAP3430_REV_ES2_1)
-			pdata->single_ulpi_bypass = true;
-
-	}
-
-	uhh_hwm = omap_hwmod_lookup(USBHS_UHH_HWMODNAME);
-	if (!uhh_hwm) {
-		pr_err("Could not look up %s\n", USBHS_UHH_HWMODNAME);
-		return;
-	}
-
-	tll_hwm = omap_hwmod_lookup(USBHS_TLL_HWMODNAME);
-	if (!tll_hwm) {
-		pr_err("Could not look up %s\n", USBHS_TLL_HWMODNAME);
-		return;
-	}
-
-	pdev = omap_device_build(OMAP_USBTLL_DEVICE, bus_id, tll_hwm,
-				pdata, sizeof(*pdata));
-	if (IS_ERR(pdev)) {
-		pr_err("Could not build hwmod device %s\n",
-		       USBHS_TLL_HWMODNAME);
-		return;
-	}
-
-	pdev = omap_device_build(OMAP_USBHS_DEVICE, bus_id, uhh_hwm,
-				pdata, sizeof(*pdata));
-	if (IS_ERR(pdev)) {
-		pr_err("Could not build hwmod devices %s\n",
-		       USBHS_UHH_HWMODNAME);
-		return;
-	}
-}
-
-#else
-
-void __init usbhs_init(struct usbhs_omap_platform_data *pdata)
-{
-}
-
-#endif
-
-/* Template for PHY regulators */
-static struct fixed_voltage_config hsusb_reg_config = {
-	/* .supply_name filled later */
-	.microvolts = 3300000,
-	.gpio = -1,		/* updated later */
-	.startup_delay = 70000, /* 70msec */
-	.enable_high = 1,	/* updated later */
-	.enabled_at_boot = 0,	/* keep in RESET */
-	/* .init_data filled later */
-};
-
-static const char *nop_name = "usb_phy_generic"; /* NOP PHY driver */
-static const char *reg_name = "reg-fixed-voltage"; /* Regulator driver */
-
-/**
- * usbhs_add_regulator - Add a gpio based fixed voltage regulator device
- * @name: name for the regulator
- * @dev_id: device id of the device this regulator supplies power to
- * @dev_supply: supply name that the device expects
- * @gpio: GPIO number
- * @polarity: 1 - Active high, 0 - Active low
- */
-static int usbhs_add_regulator(char *name, char *dev_id, char *dev_supply,
-						int gpio, int polarity)
-{
-	struct regulator_consumer_supply *supplies;
-	struct regulator_init_data *reg_data;
-	struct fixed_voltage_config *config;
-	struct platform_device *pdev;
-	struct platform_device_info pdevinfo;
-	int ret = -ENOMEM;
-
-	supplies = kzalloc(sizeof(*supplies), GFP_KERNEL);
-	if (!supplies)
-		return -ENOMEM;
-
-	supplies->supply = dev_supply;
-	supplies->dev_name = dev_id;
-
-	reg_data = kzalloc(sizeof(*reg_data), GFP_KERNEL);
-	if (!reg_data)
-		goto err_data;
-
-	reg_data->constraints.valid_ops_mask = REGULATOR_CHANGE_STATUS;
-	reg_data->consumer_supplies = supplies;
-	reg_data->num_consumer_supplies = 1;
-
-	config = kmemdup(&hsusb_reg_config, sizeof(hsusb_reg_config),
-			GFP_KERNEL);
-	if (!config)
-		goto err_config;
-
-	config->supply_name = kstrdup(name, GFP_KERNEL);
-	if (!config->supply_name)
-		goto err_supplyname;
-
-	config->gpio = gpio;
-	config->enable_high = polarity;
-	config->init_data = reg_data;
-
-	/* create a regulator device */
-	memset(&pdevinfo, 0, sizeof(pdevinfo));
-	pdevinfo.name = reg_name;
-	pdevinfo.id = PLATFORM_DEVID_AUTO;
-	pdevinfo.data = config;
-	pdevinfo.size_data = sizeof(*config);
-
-	pdev = platform_device_register_full(&pdevinfo);
-	if (IS_ERR(pdev)) {
-		ret = PTR_ERR(pdev);
-		pr_err("%s: Failed registering regulator %s for %s : %d\n",
-				__func__, name, dev_id, ret);
-		goto err_register;
-	}
-
-	return 0;
-
-err_register:
-	kfree(config->supply_name);
-err_supplyname:
-	kfree(config);
-err_config:
-	kfree(reg_data);
-err_data:
-	kfree(supplies);
-	return ret;
-}
-
-#define MAX_STR 20
-
-int usbhs_init_phys(struct usbhs_phy_data *phy, int num_phys)
-{
-	char rail_name[MAX_STR];
-	int i;
-	struct platform_device *pdev;
-	char *phy_id;
-	struct platform_device_info pdevinfo;
-	struct usb_phy_generic_platform_data nop_pdata;
-
-	for (i = 0; i < num_phys; i++) {
-
-		if (!phy->port) {
-			pr_err("%s: Invalid port 0. Must start from 1\n",
-						__func__);
-			continue;
-		}
-
-		/* do we need a NOP PHY device ? */
-		if (!gpio_is_valid(phy->reset_gpio) &&
-			!gpio_is_valid(phy->vcc_gpio))
-			continue;
-
-		phy_id = kmalloc(MAX_STR, GFP_KERNEL);
-		if (!phy_id) {
-			pr_err("%s: kmalloc() failed\n", __func__);
-			return -ENOMEM;
-		}
-
-		/* set platform data */
-		memset(&nop_pdata, 0, sizeof(nop_pdata));
-		if (gpio_is_valid(phy->vcc_gpio))
-			nop_pdata.needs_vcc = true;
-		nop_pdata.gpio_reset = phy->reset_gpio;
-		nop_pdata.type = USB_PHY_TYPE_USB2;
-
-		/* create a NOP PHY device */
-		memset(&pdevinfo, 0, sizeof(pdevinfo));
-		pdevinfo.name = nop_name;
-		pdevinfo.id = phy->port;
-		pdevinfo.data = &nop_pdata;
-		pdevinfo.size_data =
-			sizeof(struct usb_phy_generic_platform_data);
-		scnprintf(phy_id, MAX_STR, "usb_phy_generic.%d",
-					phy->port);
-		pdev = platform_device_register_full(&pdevinfo);
-		if (IS_ERR(pdev)) {
-			pr_err("%s: Failed to register device %s : %ld\n",
-				__func__,  phy_id, PTR_ERR(pdev));
-			kfree(phy_id);
-			continue;
-		}
-
-		usb_bind_phy("ehci-omap.0", phy->port - 1, phy_id);
-
-		/* Do we need VCC regulator ? */
-		if (gpio_is_valid(phy->vcc_gpio)) {
-			scnprintf(rail_name, MAX_STR, "hsusb%d_vcc", phy->port);
-			usbhs_add_regulator(rail_name, phy_id, "vcc",
-					phy->vcc_gpio, phy->vcc_polarity);
-		}
-
-		phy++;
-	}
-
-	return 0;
-}
diff --git a/arch/arm/mach-omap2/usb-musb.c b/arch/arm/mach-omap2/usb-musb.c
deleted file mode 100644
index e4562b2b973b..000000000000
--- a/arch/arm/mach-omap2/usb-musb.c
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * linux/arch/arm/mach-omap2/usb-musb.c
- *
- * This file will contain the board specific details for the
- * MENTOR USB OTG controller on OMAP3430
- *
- * Copyright (C) 2007-2008 Texas Instruments
- * Copyright (C) 2008 Nokia Corporation
- * Author: Vikram Pandita
- *
- * Generalization by:
- * Felipe Balbi <felipe.balbi@nokia.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/types.h>
-#include <linux/errno.h>
-#include <linux/delay.h>
-#include <linux/platform_device.h>
-#include <linux/clk.h>
-#include <linux/dma-mapping.h>
-#include <linux/io.h>
-#include <linux/usb/musb.h>
-
-#include "omap_device.h"
-#include "soc.h"
-#include "mux.h"
-#include "usb.h"
-
-static struct musb_hdrc_config musb_config = {
-	.multipoint	= 1,
-	.dyn_fifo	= 1,
-	.num_eps	= 16,
-	.ram_bits	= 12,
-};
-
-static struct musb_hdrc_platform_data musb_plat = {
-	.mode		= MUSB_OTG,
-
-	/* .clock is set dynamically */
-	.config		= &musb_config,
-
-	/* REVISIT charge pump on TWL4030 can supply up to
-	 * 100 mA ... but this value is board-specific, like
-	 * "mode", and should be passed to usb_musb_init().
-	 */
-	.power		= 50,			/* up to 100 mA */
-};
-
-static u64 musb_dmamask = DMA_BIT_MASK(32);
-
-static struct omap_musb_board_data musb_default_board_data = {
-	.interface_type		= MUSB_INTERFACE_ULPI,
-	.mode			= MUSB_OTG,
-	.power			= 100,
-};
-
-void __init usb_musb_init(struct omap_musb_board_data *musb_board_data)
-{
-	struct omap_hwmod		*oh;
-	struct platform_device		*pdev;
-	struct device			*dev;
-	int				bus_id = -1;
-	const char			*oh_name, *name;
-	struct omap_musb_board_data	*board_data;
-
-	if (musb_board_data)
-		board_data = musb_board_data;
-	else
-		board_data = &musb_default_board_data;
-
-	/*
-	 * REVISIT: This line can be removed once all the platforms using
-	 * musb_core.c have been converted to use use clkdev.
-	 */
-	musb_plat.clock = "ick";
-	musb_plat.board_data = board_data;
-	musb_plat.power = board_data->power >> 1;
-	musb_plat.mode = board_data->mode;
-	musb_plat.extvbus = board_data->extvbus;
-
-	oh_name = "usb_otg_hs";
-	name = "musb-omap2430";
-
-        oh = omap_hwmod_lookup(oh_name);
-        if (WARN(!oh, "%s: could not find omap_hwmod for %s\n",
-                 __func__, oh_name))
-                return;
-
-	pdev = omap_device_build(name, bus_id, oh, &musb_plat,
-				 sizeof(musb_plat));
-	if (IS_ERR(pdev)) {
-		pr_err("Could not build omap_device for %s %s\n",
-						name, oh_name);
-		return;
-	}
-
-	dev = &pdev->dev;
-	get_device(dev);
-	dev->dma_mask = &musb_dmamask;
-	dev->coherent_dma_mask = musb_dmamask;
-	put_device(dev);
-}
diff --git a/arch/arm/mach-omap2/usb-tusb6010.c b/arch/arm/mach-omap2/usb-tusb6010.c
index e554d9e66a1c..c2a6fbd7f8a9 100644
--- a/arch/arm/mach-omap2/usb-tusb6010.c
+++ b/arch/arm/mach-omap2/usb-tusb6010.c
@@ -22,8 +22,6 @@
 
 #include "gpmc.h"
 
-#include "mux.h"
-
 static u8		async_cs, sync_cs;
 static unsigned		refclk_psec;
 
@@ -226,25 +224,6 @@ tusb6010_setup_interface(struct musb_hdrc_platform_data *data,
 	}
 	tusb_device.dev.platform_data = data;
 
-	/* REVISIT let the driver know what DMA channels work */
-	if (!dmachan)
-		tusb_device.dev.dma_mask = NULL;
-	else {
-		/* assume OMAP 2420 ES2.0 and later */
-		if (dmachan & (1 << 0))
-			omap_mux_init_signal("sys_ndmareq0", 0);
-		if (dmachan & (1 << 1))
-			omap_mux_init_signal("sys_ndmareq1", 0);
-		if (dmachan & (1 << 2))
-			omap_mux_init_signal("sys_ndmareq2", 0);
-		if (dmachan & (1 << 3))
-			omap_mux_init_signal("sys_ndmareq3", 0);
-		if (dmachan & (1 << 4))
-			omap_mux_init_signal("sys_ndmareq4", 0);
-		if (dmachan & (1 << 5))
-			omap_mux_init_signal("sys_ndmareq5", 0);
-	}
-
 	/* so far so good ... register the device */
 	status = platform_device_register(&tusb_device);
 	if (status < 0) {
diff --git a/arch/arm/mach-orion5x/Kconfig b/arch/arm/mach-orion5x/Kconfig
index 89bb0fc796bd..633442ad4e4c 100644
--- a/arch/arm/mach-orion5x/Kconfig
+++ b/arch/arm/mach-orion5x/Kconfig
@@ -84,13 +84,6 @@ config MACH_LINKSTATION_PRO
 	  Buffalo Linkstation Pro/Live platform. Both v1 and
 	  v2 devices are supported.
 
-config MACH_LINKSTATION_LSCHL
-	bool "Buffalo Linkstation Live v3 (LS-CHL)"
-	select I2C_BOARDINFO if I2C
-	help
-	  Say 'Y' here if you want your kernel to support the
-	  Buffalo Linkstation Live v3 (LS-CHL) platform.
-
 config MACH_LINKSTATION_MINI
 	bool "Buffalo Linkstation Mini (Flattened Device Tree)"
 	select ARCH_ORION5X_DT
diff --git a/arch/arm/mach-orion5x/Makefile b/arch/arm/mach-orion5x/Makefile
index 4b2502b4ca0d..ae91872eeee4 100644
--- a/arch/arm/mach-orion5x/Makefile
+++ b/arch/arm/mach-orion5x/Makefile
@@ -18,7 +18,6 @@ obj-$(CONFIG_MACH_WNR854T)	+= wnr854t-setup.o
 obj-$(CONFIG_MACH_RD88F5181L_GE)	+= rd88f5181l-ge-setup.o
 obj-$(CONFIG_MACH_RD88F5181L_FXO)	+= rd88f5181l-fxo-setup.o
 obj-$(CONFIG_MACH_RD88F6183AP_GE)	+= rd88f6183ap-ge-setup.o
-obj-$(CONFIG_MACH_LINKSTATION_LSCHL)	+= ls-chl-setup.o
 
 obj-$(CONFIG_ARCH_ORION5X_DT)		+= board-dt.o
 obj-$(CONFIG_MACH_D2NET_DT)	+= board-d2net.o
diff --git a/arch/arm/mach-orion5x/ls-chl-setup.c b/arch/arm/mach-orion5x/ls-chl-setup.c
deleted file mode 100644
index dfdaa8a498a4..000000000000
--- a/arch/arm/mach-orion5x/ls-chl-setup.c
+++ /dev/null
@@ -1,331 +0,0 @@
-/*
- * arch/arm/mach-orion5x/ls-chl-setup.c
- *
- * Maintainer: Ash Hughes <ashley.hughes@blueyonder.co.uk>
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2.  This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
- */
-
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/platform_device.h>
-#include <linux/mtd/physmap.h>
-#include <linux/mv643xx_eth.h>
-#include <linux/leds.h>
-#include <linux/gpio_keys.h>
-#include <linux/gpio-fan.h>
-#include <linux/input.h>
-#include <linux/i2c.h>
-#include <linux/ata_platform.h>
-#include <linux/gpio.h>
-#include <asm/mach-types.h>
-#include <asm/mach/arch.h>
-#include "common.h"
-#include "mpp.h"
-#include "orion5x.h"
-
-/*****************************************************************************
- * Linkstation LS-CHL Info
- ****************************************************************************/
-
-/*
- * 256K NOR flash Device bus boot chip select
- */
-
-#define LSCHL_NOR_BOOT_BASE	0xf4000000
-#define LSCHL_NOR_BOOT_SIZE	SZ_256K
-
-/*****************************************************************************
- * 256KB NOR Flash on BOOT Device
- ****************************************************************************/
-
-static struct physmap_flash_data lschl_nor_flash_data = {
-	.width = 1,
-};
-
-static struct resource lschl_nor_flash_resource = {
-	.flags	= IORESOURCE_MEM,
-	.start	= LSCHL_NOR_BOOT_BASE,
-	.end	= LSCHL_NOR_BOOT_BASE + LSCHL_NOR_BOOT_SIZE - 1,
-};
-
-static struct platform_device lschl_nor_flash = {
-	.name = "physmap-flash",
-	.id = 0,
-	.dev = {
-		.platform_data	= &lschl_nor_flash_data,
-	},
-	.num_resources = 1,
-	.resource = &lschl_nor_flash_resource,
-};
-
-/*****************************************************************************
- * Ethernet
- ****************************************************************************/
-
-static struct mv643xx_eth_platform_data lschl_eth_data = {
-	.phy_addr = MV643XX_ETH_PHY_ADDR(8),
-};
-
-/*****************************************************************************
- * RTC 5C372a on I2C bus
- ****************************************************************************/
-
-static struct i2c_board_info __initdata lschl_i2c_rtc = {
-	I2C_BOARD_INFO("rs5c372a", 0x32),
-};
-
-/*****************************************************************************
- * LEDs attached to GPIO
- ****************************************************************************/
-
-#define LSCHL_GPIO_LED_ALARM	2
-#define LSCHL_GPIO_LED_INFO	3
-#define LSCHL_GPIO_LED_FUNC	17
-#define LSCHL_GPIO_LED_PWR	0
-
-static struct gpio_led lschl_led_pins[] = {
-	{
-		.name = "alarm:red",
-		.gpio = LSCHL_GPIO_LED_ALARM,
-		.active_low = 1,
-	}, {
-		.name = "info:amber",
-		.gpio = LSCHL_GPIO_LED_INFO,
-		.active_low = 1,
-	}, {
-		.name = "func:blue:top",
-		.gpio = LSCHL_GPIO_LED_FUNC,
-		.active_low = 1,
-	}, {
-		.name = "power:blue:bottom",
-		.gpio = LSCHL_GPIO_LED_PWR,
-	},
-};
-
-static struct gpio_led_platform_data lschl_led_data = {
-	.leds = lschl_led_pins,
-	.num_leds = ARRAY_SIZE(lschl_led_pins),
-};
-
-static struct platform_device lschl_leds = {
-	.name = "leds-gpio",
-	.id = -1,
-	.dev = {
-		.platform_data = &lschl_led_data,
-	},
-};
-
-/*****************************************************************************
- * SATA
- ****************************************************************************/
-static struct mv_sata_platform_data lschl_sata_data = {
-	.n_ports = 2,
-};
-
-/*****************************************************************************
- * LS-CHL specific power off method: reboot
- ****************************************************************************/
-/*
- * On the LS-CHL, the shutdown process is following:
- * - Userland monitors key events until the power switch goes to off position
- * - The board reboots
- * - U-boot starts and goes into an idle mode waiting for the user
- *   to move the switch to ON position
- *
- */
-
-static void lschl_power_off(void)
-{
-	orion5x_restart(REBOOT_HARD, NULL);
-}
-
-/*****************************************************************************
- * General Setup
- ****************************************************************************/
-#define LSCHL_GPIO_USB_POWER	9
-#define LSCHL_GPIO_AUTO_POWER	17
-#define LSCHL_GPIO_POWER	18
-
-/****************************************************************************
- * GPIO Attached Keys
- ****************************************************************************/
-#define LSCHL_GPIO_KEY_FUNC		15
-#define LSCHL_GPIO_KEY_POWER		8
-#define LSCHL_GPIO_KEY_AUTOPOWER	10
-#define LSCHL_SW_POWER		0x00
-#define LSCHL_SW_AUTOPOWER	0x01
-#define LSCHL_SW_FUNC		0x02
-
-static struct gpio_keys_button lschl_buttons[] = {
-	{
-		.type = EV_SW,
-		.code = LSCHL_SW_POWER,
-		.gpio = LSCHL_GPIO_KEY_POWER,
-		.desc = "Power-on Switch",
-		.active_low = 1,
-	}, {
-		.type = EV_SW,
-		.code = LSCHL_SW_AUTOPOWER,
-		.gpio = LSCHL_GPIO_KEY_AUTOPOWER,
-		.desc = "Power-auto Switch",
-		.active_low = 1,
-	}, {
-		.type = EV_SW,
-		.code = LSCHL_SW_FUNC,
-		.gpio = LSCHL_GPIO_KEY_FUNC,
-		.desc = "Function Switch",
-		.active_low = 1,
-	},
-};
-
-static struct gpio_keys_platform_data lschl_button_data = {
-	.buttons = lschl_buttons,
-	.nbuttons = ARRAY_SIZE(lschl_buttons),
-};
-
-static struct platform_device lschl_button_device = {
-	.name = "gpio-keys",
-	.id = -1,
-	.num_resources = 0,
-	.dev = {
-		.platform_data = &lschl_button_data,
-	},
-};
-
-#define LSCHL_GPIO_HDD_POWER	1
-
-/****************************************************************************
- * GPIO Fan
- ****************************************************************************/
-
-#define LSCHL_GPIO_FAN_LOW	16
-#define LSCHL_GPIO_FAN_HIGH	14
-#define LSCHL_GPIO_FAN_LOCK	6
-
-static struct gpio_fan_alarm lschl_alarm = {
-	.gpio = LSCHL_GPIO_FAN_LOCK,
-};
-
-static struct gpio_fan_speed lschl_speeds[] = {
-	{
-		.rpm = 0,
-		.ctrl_val = 3,
-	}, {
-		.rpm = 1500,
-		.ctrl_val = 2,
-	}, {
-		.rpm = 3250,
-		.ctrl_val = 1,
-	}, {
-		.rpm = 5000,
-		.ctrl_val = 0,
-	},
-};
-
-static int lschl_gpio_list[] = {
-	LSCHL_GPIO_FAN_HIGH, LSCHL_GPIO_FAN_LOW,
-};
-
-static struct gpio_fan_platform_data lschl_fan_data = {
-	.num_ctrl = ARRAY_SIZE(lschl_gpio_list),
-	.ctrl = lschl_gpio_list,
-	.alarm = &lschl_alarm,
-	.num_speed = ARRAY_SIZE(lschl_speeds),
-	.speed = lschl_speeds,
-};
-
-static struct platform_device lschl_fan_device = {
-	.name = "gpio-fan",
-	.id = -1,
-	.num_resources = 0,
-	.dev = {
-		.platform_data = &lschl_fan_data,
-	},
-};
-
-/****************************************************************************
- * GPIO Data
- ****************************************************************************/
-
-static unsigned int lschl_mpp_modes[] __initdata = {
-	MPP0_GPIO, /* LED POWER */
-	MPP1_GPIO, /* HDD POWER */
-	MPP2_GPIO, /* LED ALARM */
-	MPP3_GPIO, /* LED INFO */
-	MPP4_UNUSED,
-	MPP5_UNUSED,
-	MPP6_GPIO, /* FAN LOCK */
-	MPP7_GPIO, /* SW INIT */
-	MPP8_GPIO, /* SW POWER */
-	MPP9_GPIO, /* USB POWER */
-	MPP10_GPIO, /* SW AUTO POWER */
-	MPP11_UNUSED,
-	MPP12_UNUSED,
-	MPP13_UNUSED,
-	MPP14_GPIO, /* FAN HIGH */
-	MPP15_GPIO, /* SW FUNC */
-	MPP16_GPIO, /* FAN LOW */
-	MPP17_GPIO, /* LED FUNC */
-	MPP18_UNUSED,
-	MPP19_UNUSED,
-	0,
-};
-
-static void __init lschl_init(void)
-{
-	/*
-	 * Setup basic Orion functions. Needs to be called early.
-	 */
-	orion5x_init();
-
-	orion5x_mpp_conf(lschl_mpp_modes);
-
-	/*
-	 * Configure peripherals.
-	 */
-	orion5x_ehci0_init();
-	orion5x_ehci1_init();
-	orion5x_eth_init(&lschl_eth_data);
-	orion5x_i2c_init();
-	orion5x_sata_init(&lschl_sata_data);
-	orion5x_uart0_init();
-	orion5x_xor_init();
-
-	mvebu_mbus_add_window_by_id(ORION_MBUS_DEVBUS_BOOT_TARGET,
-				    ORION_MBUS_DEVBUS_BOOT_ATTR,
-				    LSCHL_NOR_BOOT_BASE,
-				    LSCHL_NOR_BOOT_SIZE);
-	platform_device_register(&lschl_nor_flash);
-
-	platform_device_register(&lschl_leds);
-
-	platform_device_register(&lschl_button_device);
-
-	platform_device_register(&lschl_fan_device);
-
-	i2c_register_board_info(0, &lschl_i2c_rtc, 1);
-
-	/* usb power on */
-	gpio_set_value(LSCHL_GPIO_USB_POWER, 1);
-
-	/* register power-off method */
-	pm_power_off = lschl_power_off;
-
-	pr_info("%s: finished\n", __func__);
-}
-
-MACHINE_START(LINKSTATION_LSCHL, "Buffalo Linkstation LiveV3 (LS-CHL)")
-	/* Maintainer: Ash Hughes <ashley.hughes@blueyonder.co.uk> */
-	.atag_offset	= 0x100,
-	.nr_irqs	= ORION5X_NR_IRQS,
-	.init_machine	= lschl_init,
-	.map_io		= orion5x_map_io,
-	.init_early	= orion5x_init_early,
-	.init_irq	= orion5x_init_irq,
-	.init_time	= orion5x_timer_init,
-	.fixup		= tag_fixup_mem32,
-	.restart	= orion5x_restart,
-MACHINE_END
diff --git a/arch/arm/mach-oxnas/Kconfig b/arch/arm/mach-oxnas/Kconfig
index 29100beb2e7f..8fa4557e27a9 100644
--- a/arch/arm/mach-oxnas/Kconfig
+++ b/arch/arm/mach-oxnas/Kconfig
@@ -1,9 +1,16 @@
 menuconfig ARCH_OXNAS
 	bool "Oxford Semiconductor OXNAS Family SoCs"
 	select ARCH_HAS_RESET_CONTROLLER
+	select COMMON_CLK_OXNAS
 	select GPIOLIB
+	select MFD_SYSCON
+	select OXNAS_RPS_TIMER
+	select PINCTRL_OXNAS
+	select RESET_CONTROLLER
+	select RESET_OXNAS
+	select VERSATILE_FPGA_IRQ
 	select PINCTRL
-	depends on ARCH_MULTI_V5
+	depends on ARCH_MULTI_V5 || ARCH_MULTI_V6
 	help
 	  Support for OxNas SoC family developed by Oxford Semiconductor.
 
@@ -11,16 +18,21 @@ if ARCH_OXNAS
 
 config MACH_OX810SE
 	bool "Support OX810SE Based Products"
-	select ARCH_HAS_RESET_CONTROLLER
-	select COMMON_CLK_OXNAS
+	depends on ARCH_MULTI_V5
 	select CPU_ARM926T
-	select MFD_SYSCON
-	select OXNAS_RPS_TIMER
-	select PINCTRL_OXNAS
-	select RESET_CONTROLLER
-	select RESET_OXNAS
-	select VERSATILE_FPGA_IRQ
 	help
 	  Include Support for the Oxford Semiconductor OX810SE SoC Based Products.
 
+config MACH_OX820
+	bool "Support OX820 Based Products"
+	depends on ARCH_MULTI_V6
+	select ARM_GIC
+	select DMA_CACHE_RWFO if SMP
+	select CPU_V6K
+	select HAVE_SMP
+	select HAVE_ARM_SCU if SMP
+	select HAVE_ARM_TWD if SMP
+	help
+	  Include Support for the Oxford Semiconductor OX820 SoC Based Products.
+
 endif
diff --git a/arch/arm/mach-oxnas/Makefile b/arch/arm/mach-oxnas/Makefile
new file mode 100644
index 000000000000..b625906a9970
--- /dev/null
+++ b/arch/arm/mach-oxnas/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_SMP)		+= platsmp.o headsmp.o
+obj-$(CONFIG_HOTPLUG_CPU) 	+= hotplug.o
diff --git a/arch/arm/mach-oxnas/headsmp.S b/arch/arm/mach-oxnas/headsmp.S
new file mode 100644
index 000000000000..25fd4f82ab3a
--- /dev/null
+++ b/arch/arm/mach-oxnas/headsmp.S
@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2013 Ma Haijun <mahaijuns@gmail.com>
+ * Copyright (c) 2003 ARM Limited
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/linkage.h>
+#include <linux/init.h>
+
+	__INIT
+
+/*
+ * OX820 specific entry point for secondary CPUs.
+ */
+ENTRY(ox820_secondary_startup)
+	mov r4, #0
+	/* invalidate both caches and branch target cache */
+	mcr p15, 0, r4, c7, c7, 0
+	/*
+	 * we've been released from the holding pen: secondary_stack
+	 * should now contain the SVC stack for this core
+	 */
+	b	secondary_startup
diff --git a/arch/arm/mach-oxnas/hotplug.c b/arch/arm/mach-oxnas/hotplug.c
new file mode 100644
index 000000000000..854f29b8cba6
--- /dev/null
+++ b/arch/arm/mach-oxnas/hotplug.c
@@ -0,0 +1,109 @@
+/*
+ *  Copyright (C) 2002 ARM Ltd.
+ *  All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/smp.h>
+
+#include <asm/cp15.h>
+#include <asm/smp_plat.h>
+
+static inline void cpu_enter_lowpower(void)
+{
+	unsigned int v;
+
+	asm volatile(
+	"	mcr	p15, 0, %1, c7, c5, 0\n"
+	"	mcr	p15, 0, %1, c7, c10, 4\n"
+	/*
+	 * Turn off coherency
+	 */
+	"	mrc	p15, 0, %0, c1, c0, 1\n"
+	"	bic	%0, %0, #0x20\n"
+	"	mcr	p15, 0, %0, c1, c0, 1\n"
+	"	mrc	p15, 0, %0, c1, c0, 0\n"
+	"	bic	%0, %0, %2\n"
+	"	mcr	p15, 0, %0, c1, c0, 0\n"
+	  : "=&r" (v)
+	  : "r" (0), "Ir" (CR_C)
+	  : "cc");
+}
+
+static inline void cpu_leave_lowpower(void)
+{
+	unsigned int v;
+
+	asm volatile(	"mrc	p15, 0, %0, c1, c0, 0\n"
+	"	orr	%0, %0, %1\n"
+	"	mcr	p15, 0, %0, c1, c0, 0\n"
+	"	mrc	p15, 0, %0, c1, c0, 1\n"
+	"	orr	%0, %0, #0x20\n"
+	"	mcr	p15, 0, %0, c1, c0, 1\n"
+	  : "=&r" (v)
+	  : "Ir" (CR_C)
+	  : "cc");
+}
+
+static inline void platform_do_lowpower(unsigned int cpu, int *spurious)
+{
+	/*
+	 * there is no power-control hardware on this platform, so all
+	 * we can do is put the core into WFI; this is safe as the calling
+	 * code will have already disabled interrupts
+	 */
+	for (;;) {
+		/*
+		 * here's the WFI
+		 */
+		asm(".word	0xe320f003\n"
+		    :
+		    :
+		    : "memory", "cc");
+
+		if (pen_release == cpu_logical_map(cpu)) {
+			/*
+			 * OK, proper wakeup, we're done
+			 */
+			break;
+		}
+
+		/*
+		 * Getting here, means that we have come out of WFI without
+		 * having been woken up - this shouldn't happen
+		 *
+		 * Just note it happening - when we're woken, we can report
+		 * its occurrence.
+		 */
+		(*spurious)++;
+	}
+}
+
+/*
+ * platform-specific code to shutdown a CPU
+ *
+ * Called with IRQs disabled
+ */
+void ox820_cpu_die(unsigned int cpu)
+{
+	int spurious = 0;
+
+	/*
+	 * we're ready for shutdown now, so do it
+	 */
+	cpu_enter_lowpower();
+	platform_do_lowpower(cpu, &spurious);
+
+	/*
+	 * bring this CPU back into the world of cache
+	 * coherency, and then restore interrupts
+	 */
+	cpu_leave_lowpower();
+
+	if (spurious)
+		pr_warn("CPU%u: %u spurious wakeup calls\n", cpu, spurious);
+}
diff --git a/arch/arm/mach-oxnas/platsmp.c b/arch/arm/mach-oxnas/platsmp.c
new file mode 100644
index 000000000000..442cc8a2f7dc
--- /dev/null
+++ b/arch/arm/mach-oxnas/platsmp.c
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2016 Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2013 Ma Haijun <mahaijuns@gmail.com>
+ * Copyright (C) 2002 ARM Ltd.
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+
+#include <asm/cacheflush.h>
+#include <asm/cp15.h>
+#include <asm/smp_plat.h>
+#include <asm/smp_scu.h>
+
+extern void ox820_secondary_startup(void);
+extern void ox820_cpu_die(unsigned int cpu);
+
+static void __iomem *cpu_ctrl;
+static void __iomem *gic_cpu_ctrl;
+
+#define HOLDINGPEN_CPU_OFFSET		0xc8
+#define HOLDINGPEN_LOCATION_OFFSET	0xc4
+
+#define GIC_NCPU_OFFSET(cpu)		(0x100 + (cpu)*0x100)
+#define GIC_CPU_CTRL			0x00
+#define GIC_CPU_CTRL_ENABLE		1
+
+int __init ox820_boot_secondary(unsigned int cpu, struct task_struct *idle)
+{
+	/*
+	 * Write the address of secondary startup into the
+	 * system-wide flags register. The BootMonitor waits
+	 * until it receives a soft interrupt, and then the
+	 * secondary CPU branches to this address.
+	 */
+	writel(virt_to_phys(ox820_secondary_startup),
+			cpu_ctrl + HOLDINGPEN_LOCATION_OFFSET);
+
+	writel(cpu, cpu_ctrl + HOLDINGPEN_CPU_OFFSET);
+
+	/*
+	 * Enable GIC cpu interface in CPU Interface Control Register
+	 */
+	writel(GIC_CPU_CTRL_ENABLE,
+		gic_cpu_ctrl + GIC_NCPU_OFFSET(cpu) + GIC_CPU_CTRL);
+
+	/*
+	 * Send the secondary CPU a soft interrupt, thereby causing
+	 * the boot monitor to read the system wide flags register,
+	 * and branch to the address found there.
+	 */
+	arch_send_wakeup_ipi_mask(cpumask_of(cpu));
+
+	return 0;
+}
+
+static void __init ox820_smp_prepare_cpus(unsigned int max_cpus)
+{
+	struct device_node *np;
+	void __iomem *scu_base;
+
+	np = of_find_compatible_node(NULL, NULL, "arm,arm11mp-scu");
+	scu_base = of_iomap(np, 0);
+	of_node_put(np);
+	if (!scu_base)
+		return;
+
+	/* Remap CPU Interrupt Interface Registers */
+	np = of_find_compatible_node(NULL, NULL, "arm,arm11mp-gic");
+	gic_cpu_ctrl = of_iomap(np, 1);
+	of_node_put(np);
+	if (!gic_cpu_ctrl)
+		goto unmap_scu;
+
+	np = of_find_compatible_node(NULL, NULL, "oxsemi,ox820-sys-ctrl");
+	cpu_ctrl = of_iomap(np, 0);
+	of_node_put(np);
+	if (!cpu_ctrl)
+		goto unmap_scu;
+
+	scu_enable(scu_base);
+	flush_cache_all();
+
+unmap_scu:
+	iounmap(scu_base);
+}
+
+static const struct smp_operations ox820_smp_ops __initconst = {
+	.smp_prepare_cpus	= ox820_smp_prepare_cpus,
+	.smp_boot_secondary	= ox820_boot_secondary,
+#ifdef CONFIG_HOTPLUG_CPU
+	.cpu_die		= ox820_cpu_die,
+#endif
+};
+
+CPU_METHOD_OF_DECLARE(ox820_smp, "oxsemi,ox820-smp", &ox820_smp_ops);
diff --git a/arch/arm/mach-pxa/corgi.c b/arch/arm/mach-pxa/corgi.c
index 10bfdb169366..183cd3446f25 100644
--- a/arch/arm/mach-pxa/corgi.c
+++ b/arch/arm/mach-pxa/corgi.c
@@ -35,7 +35,6 @@
 #include <linux/mtd/sharpsl.h>
 #include <linux/input/matrix_keypad.h>
 #include <linux/gpio_keys.h>
-#include <linux/module.h>
 #include <linux/memblock.h>
 #include <video/w100fb.h>
 
diff --git a/arch/arm/mach-pxa/em-x270.c b/arch/arm/mach-pxa/em-x270.c
index 03354c21e1f2..811a7317f3ea 100644
--- a/arch/arm/mach-pxa/em-x270.c
+++ b/arch/arm/mach-pxa/em-x270.c
@@ -23,6 +23,7 @@
 #include <linux/gpio.h>
 #include <linux/mfd/da903x.h>
 #include <linux/regulator/machine.h>
+#include <linux/regulator/fixed.h>
 #include <linux/spi/spi.h>
 #include <linux/spi/tdo24m.h>
 #include <linux/spi/libertas_spi.h>
@@ -34,8 +35,6 @@
 #include <linux/i2c/pxa-i2c.h>
 #include <linux/regulator/userspace-consumer.h>
 
-#include <media/soc_camera.h>
-
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
 
@@ -958,8 +957,6 @@ static inline void em_x270_init_gpio_keys(void) {}
 
 /* Quick Capture Interface and sensor setup */
 #if defined(CONFIG_VIDEO_PXA27x) || defined(CONFIG_VIDEO_PXA27x_MODULE)
-static struct regulator *em_x270_camera_ldo;
-
 static int em_x270_sensor_init(void)
 {
 	int ret;
@@ -969,81 +966,53 @@ static int em_x270_sensor_init(void)
 		return ret;
 
 	gpio_direction_output(cam_reset, 0);
-
-	em_x270_camera_ldo = regulator_get(NULL, "vcc cam");
-	if (em_x270_camera_ldo == NULL) {
-		gpio_free(cam_reset);
-		return -ENODEV;
-	}
-
-	ret = regulator_enable(em_x270_camera_ldo);
-	if (ret) {
-		regulator_put(em_x270_camera_ldo);
-		gpio_free(cam_reset);
-		return ret;
-	}
-
 	gpio_set_value(cam_reset, 1);
 
 	return 0;
 }
 
-struct pxacamera_platform_data em_x270_camera_platform_data = {
-	.flags  = PXA_CAMERA_MASTER | PXA_CAMERA_DATAWIDTH_8 |
-		PXA_CAMERA_PCLK_EN | PXA_CAMERA_MCLK_EN,
-	.mclk_10khz = 2600,
+static struct regulator_consumer_supply camera_dummy_supplies[] = {
+	REGULATOR_SUPPLY("vdd", "0-005d"),
 };
 
-static int em_x270_sensor_power(struct device *dev, int on)
-{
-	int ret;
-	int is_on = regulator_is_enabled(em_x270_camera_ldo);
-
-	if (on == is_on)
-		return 0;
-
-	gpio_set_value(cam_reset, !on);
-
-	if (on)
-		ret = regulator_enable(em_x270_camera_ldo);
-	else
-		ret = regulator_disable(em_x270_camera_ldo);
-
-	if (ret)
-		return ret;
-
-	gpio_set_value(cam_reset, on);
-
-	return 0;
-}
-
-static struct i2c_board_info em_x270_i2c_cam_info[] = {
-	{
-		I2C_BOARD_INFO("mt9m111", 0x48),
+static struct regulator_init_data camera_dummy_initdata = {
+	.consumer_supplies = camera_dummy_supplies,
+	.num_consumer_supplies = ARRAY_SIZE(camera_dummy_supplies),
+	.constraints = {
+		.valid_ops_mask = REGULATOR_CHANGE_STATUS,
 	},
 };
 
-static struct soc_camera_link iclink = {
-	.bus_id		= 0,
-	.power		= em_x270_sensor_power,
-	.board_info	= &em_x270_i2c_cam_info[0],
-	.i2c_adapter_id	= 0,
+static struct fixed_voltage_config camera_dummy_config = {
+	.supply_name		= "camera_vdd",
+	.input_supply		= "vcc cam",
+	.microvolts		= 2800000,
+	.gpio			= -1,
+	.enable_high		= 0,
+	.init_data		= &camera_dummy_initdata,
 };
 
-static struct platform_device em_x270_camera = {
-	.name	= "soc-camera-pdrv",
-	.id	= -1,
+static struct platform_device camera_supply_dummy_device = {
+	.name	= "reg-fixed-voltage",
+	.id	= 1,
 	.dev	= {
-		.platform_data = &iclink,
+		.platform_data = &camera_dummy_config,
 	},
 };
 
+struct pxacamera_platform_data em_x270_camera_platform_data = {
+	.flags  = PXA_CAMERA_MASTER | PXA_CAMERA_DATAWIDTH_8 |
+		PXA_CAMERA_PCLK_EN | PXA_CAMERA_MCLK_EN,
+	.mclk_10khz = 2600,
+	.sensor_i2c_adapter_id = 0,
+	.sensor_i2c_address = 0x5d,
+};
+
 static void  __init em_x270_init_camera(void)
 {
-	if (em_x270_sensor_init() == 0) {
+	if (em_x270_sensor_init() == 0)
 		pxa_set_camera_info(&em_x270_camera_platform_data);
-		platform_device_register(&em_x270_camera);
-	}
+	platform_device_register(&camera_supply_dummy_device);
 }
 #else
 static inline void em_x270_init_camera(void) {}
diff --git a/arch/arm/mach-pxa/ezx.c b/arch/arm/mach-pxa/ezx.c
index 34ad0a89d4a9..0b8300e6fca3 100644
--- a/arch/arm/mach-pxa/ezx.c
+++ b/arch/arm/mach-pxa/ezx.c
@@ -17,14 +17,14 @@
 #include <linux/delay.h>
 #include <linux/pwm.h>
 #include <linux/pwm_backlight.h>
+#include <linux/regulator/machine.h>
+#include <linux/regulator/fixed.h>
 #include <linux/input.h>
 #include <linux/gpio.h>
 #include <linux/gpio_keys.h>
 #include <linux/leds-lp3944.h>
 #include <linux/i2c/pxa-i2c.h>
 
-#include <media/soc_camera.h>
-
 #include <asm/setup.h>
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
@@ -723,6 +723,42 @@ static struct platform_device a780_gpio_keys = {
 };
 
 /* camera */
+static struct regulator_consumer_supply camera_dummy_supplies[] = {
+	REGULATOR_SUPPLY("vdd", "0-005d"),
+};
+
+static struct regulator_init_data camera_dummy_initdata = {
+	.consumer_supplies = camera_dummy_supplies,
+	.num_consumer_supplies = ARRAY_SIZE(camera_dummy_supplies),
+	.constraints = {
+		.valid_ops_mask = REGULATOR_CHANGE_STATUS,
+	},
+};
+
+static struct fixed_voltage_config camera_dummy_config = {
+	.supply_name		= "camera_vdd",
+	.microvolts		= 2800000,
+	.gpio			= GPIO50_nCAM_EN,
+	.enable_high		= 0,
+	.init_data		= &camera_dummy_initdata,
+};
+
+static struct platform_device camera_supply_dummy_device = {
+	.name	= "reg-fixed-voltage",
+	.id	= 1,
+	.dev	= {
+		.platform_data = &camera_dummy_config,
+	},
+};
+static int a780_camera_reset(struct device *dev)
+{
+	gpio_set_value(GPIO19_GEN1_CAM_RST, 0);
+	msleep(10);
+	gpio_set_value(GPIO19_GEN1_CAM_RST, 1);
+
+	return 0;
+}
+
 static int a780_camera_init(void)
 {
 	int err;
@@ -731,73 +767,36 @@ static int a780_camera_init(void)
 	 * GPIO50_nCAM_EN is active low
 	 * GPIO19_GEN1_CAM_RST is active on rising edge
 	 */
-	err = gpio_request(GPIO50_nCAM_EN, "nCAM_EN");
-	if (err) {
-		pr_err("%s: Failed to request nCAM_EN\n", __func__);
-		goto fail;
-	}
-
 	err = gpio_request(GPIO19_GEN1_CAM_RST, "CAM_RST");
 	if (err) {
 		pr_err("%s: Failed to request CAM_RST\n", __func__);
-		goto fail_gpio_cam_rst;
+		return err;
 	}
 
-	gpio_direction_output(GPIO50_nCAM_EN, 1);
 	gpio_direction_output(GPIO19_GEN1_CAM_RST, 0);
-
-	return 0;
-
-fail_gpio_cam_rst:
-	gpio_free(GPIO50_nCAM_EN);
-fail:
-	return err;
-}
-
-static int a780_camera_power(struct device *dev, int on)
-{
-	gpio_set_value(GPIO50_nCAM_EN, !on);
-	return 0;
-}
-
-static int a780_camera_reset(struct device *dev)
-{
-	gpio_set_value(GPIO19_GEN1_CAM_RST, 0);
-	msleep(10);
-	gpio_set_value(GPIO19_GEN1_CAM_RST, 1);
+	a780_camera_reset(NULL);
 
 	return 0;
 }
 
 struct pxacamera_platform_data a780_pxacamera_platform_data = {
 	.flags  = PXA_CAMERA_MASTER | PXA_CAMERA_DATAWIDTH_8 |
-		PXA_CAMERA_PCLK_EN | PXA_CAMERA_MCLK_EN,
+		PXA_CAMERA_PCLK_EN | PXA_CAMERA_MCLK_EN |
+		PXA_CAMERA_PCP,
 	.mclk_10khz = 5000,
+	.sensor_i2c_adapter_id = 0,
+	.sensor_i2c_address = 0x5d,
 };
 
-static struct i2c_board_info a780_camera_i2c_board_info = {
-	I2C_BOARD_INFO("mt9m111", 0x5d),
-};
-
-static struct soc_camera_link a780_iclink = {
-	.bus_id         = 0,
-	.flags          = SOCAM_SENSOR_INVERT_PCLK,
-	.i2c_adapter_id = 0,
-	.board_info     = &a780_camera_i2c_board_info,
-	.power          = a780_camera_power,
-	.reset          = a780_camera_reset,
-};
-
-static struct platform_device a780_camera = {
-	.name   = "soc-camera-pdrv",
-	.id     = 0,
-	.dev    = {
-		.platform_data = &a780_iclink,
+static struct i2c_board_info a780_i2c_board_info[] = {
+	{
+		I2C_BOARD_INFO("mt9m111", 0x5d),
 	},
 };
 
 static struct platform_device *a780_devices[] __initdata = {
 	&a780_gpio_keys,
+	&camera_supply_dummy_device,
 };
 
 static void __init a780_init(void)
@@ -811,19 +810,19 @@ static void __init a780_init(void)
 	pxa_set_stuart_info(NULL);
 
 	pxa_set_i2c_info(NULL);
+	i2c_register_board_info(0, ARRAY_AND_SIZE(a780_i2c_board_info));
 
 	pxa_set_fb_info(NULL, &ezx_fb_info_1);
 
 	pxa_set_keypad_info(&a780_keypad_platform_data);
 
-	if (a780_camera_init() == 0) {
+	if (a780_camera_init() == 0)
 		pxa_set_camera_info(&a780_pxacamera_platform_data);
-		platform_device_register(&a780_camera);
-	}
 
 	pwm_add_table(ezx_pwm_lookup, ARRAY_SIZE(ezx_pwm_lookup));
 	platform_add_devices(ARRAY_AND_SIZE(ezx_devices));
 	platform_add_devices(ARRAY_AND_SIZE(a780_devices));
+	regulator_has_full_constraints();
 }
 
 MACHINE_START(EZX_A780, "Motorola EZX A780")
@@ -1001,6 +1000,15 @@ static struct platform_device a910_gpio_keys = {
 };
 
 /* camera */
+static int a910_camera_reset(struct device *dev)
+{
+	gpio_set_value(GPIO28_GEN2_CAM_RST, 0);
+	msleep(10);
+	gpio_set_value(GPIO28_GEN2_CAM_RST, 1);
+
+	return 0;
+}
+
 static int a910_camera_init(void)
 {
 	int err;
@@ -1009,68 +1017,25 @@ static int a910_camera_init(void)
 	 * GPIO50_nCAM_EN is active low
 	 * GPIO28_GEN2_CAM_RST is active on rising edge
 	 */
-	err = gpio_request(GPIO50_nCAM_EN, "nCAM_EN");
-	if (err) {
-		pr_err("%s: Failed to request nCAM_EN\n", __func__);
-		goto fail;
-	}
-
 	err = gpio_request(GPIO28_GEN2_CAM_RST, "CAM_RST");
 	if (err) {
 		pr_err("%s: Failed to request CAM_RST\n", __func__);
-		goto fail_gpio_cam_rst;
+		return err;
 	}
 
-	gpio_direction_output(GPIO50_nCAM_EN, 1);
 	gpio_direction_output(GPIO28_GEN2_CAM_RST, 0);
-
-	return 0;
-
-fail_gpio_cam_rst:
-	gpio_free(GPIO50_nCAM_EN);
-fail:
-	return err;
-}
-
-static int a910_camera_power(struct device *dev, int on)
-{
-	gpio_set_value(GPIO50_nCAM_EN, !on);
-	return 0;
-}
-
-static int a910_camera_reset(struct device *dev)
-{
-	gpio_set_value(GPIO28_GEN2_CAM_RST, 0);
-	msleep(10);
-	gpio_set_value(GPIO28_GEN2_CAM_RST, 1);
+	a910_camera_reset(NULL);
 
 	return 0;
 }
 
 struct pxacamera_platform_data a910_pxacamera_platform_data = {
 	.flags  = PXA_CAMERA_MASTER | PXA_CAMERA_DATAWIDTH_8 |
-		PXA_CAMERA_PCLK_EN | PXA_CAMERA_MCLK_EN,
+		PXA_CAMERA_PCLK_EN | PXA_CAMERA_MCLK_EN |
+		PXA_CAMERA_PCP,
 	.mclk_10khz = 5000,
-};
-
-static struct i2c_board_info a910_camera_i2c_board_info = {
-	I2C_BOARD_INFO("mt9m111", 0x5d),
-};
-
-static struct soc_camera_link a910_iclink = {
-	.bus_id         = 0,
-	.i2c_adapter_id = 0,
-	.board_info     = &a910_camera_i2c_board_info,
-	.power          = a910_camera_power,
-	.reset          = a910_camera_reset,
-};
-
-static struct platform_device a910_camera = {
-	.name   = "soc-camera-pdrv",
-	.id     = 0,
-	.dev    = {
-		.platform_data = &a910_iclink,
-	},
+	.sensor_i2c_adapter_id = 0,
+	.sensor_i2c_address = 0x5d,
 };
 
 /* leds-lp3944 */
@@ -1122,10 +1087,14 @@ static struct i2c_board_info __initdata a910_i2c_board_info[] = {
 		I2C_BOARD_INFO("lp3944", 0x60),
 		.platform_data = &a910_lp3944_leds,
 	},
+	{
+		I2C_BOARD_INFO("mt9m111", 0x5d),
+	},
 };
 
 static struct platform_device *a910_devices[] __initdata = {
 	&a910_gpio_keys,
+	&camera_supply_dummy_device,
 };
 
 static void __init a910_init(void)
@@ -1145,14 +1114,13 @@ static void __init a910_init(void)
 
 	pxa_set_keypad_info(&a910_keypad_platform_data);
 
-	if (a910_camera_init() == 0) {
+	if (a910_camera_init() == 0)
 		pxa_set_camera_info(&a910_pxacamera_platform_data);
-		platform_device_register(&a910_camera);
-	}
 
 	pwm_add_table(ezx_pwm_lookup, ARRAY_SIZE(ezx_pwm_lookup));
 	platform_add_devices(ARRAY_AND_SIZE(ezx_devices));
 	platform_add_devices(ARRAY_AND_SIZE(a910_devices));
+	regulator_has_full_constraints();
 }
 
 MACHINE_START(EZX_A910, "Motorola EZX A910")
diff --git a/arch/arm/mach-pxa/generic.c b/arch/arm/mach-pxa/generic.c
index ec510ecf8370..cb73a9723d0e 100644
--- a/arch/arm/mach-pxa/generic.c
+++ b/arch/arm/mach-pxa/generic.c
@@ -43,21 +43,6 @@ void clear_reset_status(unsigned int mask)
 	}
 }
 
-unsigned long get_clock_tick_rate(void)
-{
-	unsigned long clock_tick_rate;
-
-	if (cpu_is_pxa25x())
-		clock_tick_rate = 3686400;
-	else if (machine_is_mainstone())
-		clock_tick_rate = 3249600;
-	else
-		clock_tick_rate = 3250000;
-
-	return clock_tick_rate;
-}
-EXPORT_SYMBOL(get_clock_tick_rate);
-
 /*
  * For non device-tree builds, keep legacy timer init
  */
@@ -69,8 +54,7 @@ void __init pxa_timer_init(void)
 		pxa27x_clocks_init();
 	if (cpu_is_pxa3xx())
 		pxa3xx_clocks_init();
-	pxa_timer_nodt_init(IRQ_OST0, io_p2v(0x40a00000),
-			    get_clock_tick_rate());
+	pxa_timer_nodt_init(IRQ_OST0, io_p2v(0x40a00000));
 }
 
 /*
diff --git a/arch/arm/mach-pxa/include/mach/hardware.h b/arch/arm/mach-pxa/include/mach/hardware.h
index 8d63c211b22f..55064124ca4e 100644
--- a/arch/arm/mach-pxa/include/mach/hardware.h
+++ b/arch/arm/mach-pxa/include/mach/hardware.h
@@ -303,8 +303,6 @@
  */
 extern unsigned int get_memclk_frequency_10khz(void);
 
-/* return the clock tick rate of the OS timer */
-extern unsigned long get_clock_tick_rate(void);
 #endif
 
 #endif  /* _ASM_ARCH_HARDWARE_H */
diff --git a/arch/arm/mach-pxa/mioa701.c b/arch/arm/mach-pxa/mioa701.c
index 38a96a193dc4..8a5d0491e73c 100644
--- a/arch/arm/mach-pxa/mioa701.c
+++ b/arch/arm/mach-pxa/mioa701.c
@@ -57,7 +57,6 @@
 #include <linux/platform_data/media/camera-pxa.h>
 #include <mach/audio.h>
 #include <mach/smemc.h>
-#include <media/soc_camera.h>
 
 #include "mioa701.h"
 
@@ -627,6 +626,8 @@ struct pxacamera_platform_data mioa701_pxacamera_platform_data = {
 	.flags  = PXA_CAMERA_MASTER | PXA_CAMERA_DATAWIDTH_8 |
 		PXA_CAMERA_PCLK_EN | PXA_CAMERA_MCLK_EN,
 	.mclk_10khz = 5000,
+	.sensor_i2c_adapter_id = 0,
+	.sensor_i2c_address = 0x5d,
 };
 
 static struct i2c_board_info __initdata mioa701_pi2c_devices[] = {
@@ -643,12 +644,6 @@ static struct i2c_board_info mioa701_i2c_devices[] = {
 	},
 };
 
-static struct soc_camera_link iclink = {
-	.bus_id		= 0, /* Match id in pxa27x_device_camera in device.c */
-	.board_info	= &mioa701_i2c_devices[0],
-	.i2c_adapter_id	= 0,
-};
-
 struct i2c_pxa_platform_data i2c_pdata = {
 	.fast_mode = 1,
 };
@@ -684,7 +679,6 @@ MIO_SIMPLE_DEV(mioa701_sound,	  "mioa701-wm9713", NULL)
 MIO_SIMPLE_DEV(mioa701_board,	  "mioa701-board",  NULL)
 MIO_SIMPLE_DEV(wm9713_acodec,	  "wm9713-codec",   NULL);
 MIO_SIMPLE_DEV(gpio_vbus,	  "gpio-vbus",      &gpio_vbus_data);
-MIO_SIMPLE_DEV(mioa701_camera,	  "soc-camera-pdrv",&iclink);
 
 static struct platform_device *devices[] __initdata = {
 	&mioa701_gpio_keys,
@@ -696,7 +690,6 @@ static struct platform_device *devices[] __initdata = {
 	&power_dev,
 	&docg3,
 	&gpio_vbus,
-	&mioa701_camera,
 	&mioa701_board,
 };
 
@@ -761,6 +754,7 @@ static void __init mioa701_machine_init(void)
 	platform_add_devices(devices, ARRAY_SIZE(devices));
 	gsm_init();
 
+	i2c_register_board_info(0, ARRAY_AND_SIZE(mioa701_i2c_devices));
 	i2c_register_board_info(1, ARRAY_AND_SIZE(mioa701_pi2c_devices));
 	pxa_set_i2c_info(&i2c_pdata);
 	pxa27x_set_i2c_power_info(NULL);
@@ -769,6 +763,7 @@ static void __init mioa701_machine_init(void)
 	regulator_register_always_on(0, "fixed-5.0V", fixed_5v0_consumers,
 				     ARRAY_SIZE(fixed_5v0_consumers),
 				     5000000);
+	regulator_has_full_constraints();
 }
 
 static void mioa701_machine_exit(void)
diff --git a/arch/arm/mach-pxa/pxa25x.c b/arch/arm/mach-pxa/pxa25x.c
index 12b94357fbc1..c725baf119e1 100644
--- a/arch/arm/mach-pxa/pxa25x.c
+++ b/arch/arm/mach-pxa/pxa25x.c
@@ -156,7 +156,7 @@ static int __init __init
 pxa25x_dt_init_irq(struct device_node *node, struct device_node *parent)
 {
 	pxa_dt_irq_init(pxa25x_set_wake);
-	set_handle_irq(ichp_handle_irq);
+	set_handle_irq(icip_handle_irq);
 
 	return 0;
 }
diff --git a/arch/arm/mach-pxa/pxa_cplds_irqs.c b/arch/arm/mach-pxa/pxa_cplds_irqs.c
index e362f865fcd2..941508585e34 100644
--- a/arch/arm/mach-pxa/pxa_cplds_irqs.c
+++ b/arch/arm/mach-pxa/pxa_cplds_irqs.c
@@ -120,13 +120,9 @@ static int cplds_probe(struct platform_device *pdev)
 	if (!fpga)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-	if (res) {
-		fpga->irq = (unsigned int)res->start;
-		irqflags = res->flags;
-	}
-	if (!fpga->irq)
-		return -ENODEV;
+	fpga->irq = platform_get_irq(pdev, 0);
+	if (fpga->irq <= 0)
+		return fpga->irq;
 
 	base_irq = platform_get_irq(pdev, 1);
 	if (base_irq < 0)
@@ -142,6 +138,7 @@ static int cplds_probe(struct platform_device *pdev)
 	writel(fpga->irq_mask, fpga->base + FPGA_IRQ_MASK_EN);
 	writel(0, fpga->base + FPGA_IRQ_SET_CLR);
 
+	irqflags = irq_get_trigger_type(fpga->irq);
 	ret = devm_request_irq(&pdev->dev, fpga->irq, cplds_irq_handler,
 			       irqflags, dev_name(&pdev->dev), fpga);
 	if (ret == -ENOSYS)
diff --git a/arch/arm/mach-pxa/spitz.c b/arch/arm/mach-pxa/spitz.c
index 2c150bfc0cd5..67d66c702574 100644
--- a/arch/arm/mach-pxa/spitz.c
+++ b/arch/arm/mach-pxa/spitz.c
@@ -31,7 +31,6 @@
 #include <linux/input/matrix_keypad.h>
 #include <linux/regulator/machine.h>
 #include <linux/io.h>
-#include <linux/module.h>
 #include <linux/reboot.h>
 #include <linux/memblock.h>
 
diff --git a/arch/arm/mach-s3c64xx/pl080.c b/arch/arm/mach-s3c64xx/pl080.c
index 89c5a62830a7..261820a855ec 100644
--- a/arch/arm/mach-s3c64xx/pl080.c
+++ b/arch/arm/mach-s3c64xx/pl080.c
@@ -117,6 +117,25 @@ static struct pl08x_channel_data s3c64xx_dma0_info[] = {
 	}
 };
 
+static const struct dma_slave_map s3c64xx_dma0_slave_map[] = {
+	{ "s3c6400-uart.0", "tx", &s3c64xx_dma0_info[0] },
+	{ "s3c6400-uart.0", "rx", &s3c64xx_dma0_info[1] },
+	{ "s3c6400-uart.1", "tx", &s3c64xx_dma0_info[2] },
+	{ "s3c6400-uart.1", "rx", &s3c64xx_dma0_info[3] },
+	{ "s3c6400-uart.2", "tx", &s3c64xx_dma0_info[4] },
+	{ "s3c6400-uart.2", "rx", &s3c64xx_dma0_info[5] },
+	{ "s3c6400-uart.3", "tx", &s3c64xx_dma0_info[6] },
+	{ "s3c6400-uart.3", "rx", &s3c64xx_dma0_info[7] },
+	{ "samsung-pcm.0", "tx", &s3c64xx_dma0_info[8] },
+	{ "samsung-pcm.0", "rx", &s3c64xx_dma0_info[9] },
+	{ "samsung-i2s.0", "tx", &s3c64xx_dma0_info[10] },
+	{ "samsung-i2s.0", "rx", &s3c64xx_dma0_info[11] },
+	{ "s3c6410-spi.0", "tx", &s3c64xx_dma0_info[12] },
+	{ "s3c6410-spi.0", "rx", &s3c64xx_dma0_info[13] },
+	{ "samsung-i2s.2", "tx", &s3c64xx_dma0_info[14] },
+	{ "samsung-i2s.2", "rx", &s3c64xx_dma0_info[15] },
+};
+
 struct pl08x_platform_data s3c64xx_dma0_plat_data = {
 	.memcpy_channel = {
 		.bus_id = "memcpy",
@@ -134,6 +153,8 @@ struct pl08x_platform_data s3c64xx_dma0_plat_data = {
 	.put_xfer_signal = pl08x_put_xfer_signal,
 	.slave_channels = s3c64xx_dma0_info,
 	.num_slave_channels = ARRAY_SIZE(s3c64xx_dma0_info),
+	.slave_map = s3c64xx_dma0_slave_map,
+	.slave_map_len = ARRAY_SIZE(s3c64xx_dma0_slave_map),
 };
 
 static AMBA_AHB_DEVICE(s3c64xx_dma0, "dma-pl080s.0", 0,
@@ -207,6 +228,15 @@ static struct pl08x_channel_data s3c64xx_dma1_info[] = {
 	},
 };
 
+static const struct dma_slave_map s3c64xx_dma1_slave_map[] = {
+	{ "samsung-pcm.1", "tx", &s3c64xx_dma1_info[0] },
+	{ "samsung-pcm.1", "rx", &s3c64xx_dma1_info[1] },
+	{ "samsung-i2s.1", "tx", &s3c64xx_dma1_info[2] },
+	{ "samsung-i2s.1", "rx", &s3c64xx_dma1_info[3] },
+	{ "s3c6410-spi.1", "tx", &s3c64xx_dma1_info[4] },
+	{ "s3c6410-spi.1", "rx", &s3c64xx_dma1_info[5] },
+};
+
 struct pl08x_platform_data s3c64xx_dma1_plat_data = {
 	.memcpy_channel = {
 		.bus_id = "memcpy",
@@ -224,6 +254,8 @@ struct pl08x_platform_data s3c64xx_dma1_plat_data = {
 	.put_xfer_signal = pl08x_put_xfer_signal,
 	.slave_channels = s3c64xx_dma1_info,
 	.num_slave_channels = ARRAY_SIZE(s3c64xx_dma1_info),
+	.slave_map = s3c64xx_dma1_slave_map,
+	.slave_map_len = ARRAY_SIZE(s3c64xx_dma1_slave_map),
 };
 
 static AMBA_AHB_DEVICE(s3c64xx_dma1, "dma-pl080s.1", 0,
diff --git a/arch/arm/mach-sa1100/generic.c b/arch/arm/mach-sa1100/generic.c
index 3e09beddb6e8..2eb00691b07d 100644
--- a/arch/arm/mach-sa1100/generic.c
+++ b/arch/arm/mach-sa1100/generic.c
@@ -378,7 +378,7 @@ void __init sa1100_map_io(void)
 
 void __init sa1100_timer_init(void)
 {
-	pxa_timer_nodt_init(IRQ_OST0, io_p2v(0x90000000), 3686400);
+	pxa_timer_nodt_init(IRQ_OST0, io_p2v(0x90000000));
 }
 
 static struct resource irq_resource =
diff --git a/arch/arm/mach-sa1100/include/mach/SA-1101.h b/arch/arm/mach-sa1100/include/mach/SA-1101.h
deleted file mode 100644
index 5d2ad7db991c..000000000000
--- a/arch/arm/mach-sa1100/include/mach/SA-1101.h
+++ /dev/null
@@ -1,925 +0,0 @@
-/*
- * SA-1101.h
- *
- * Copyright (c) Peter Danielsson 1999
- *
- * Definition of constants related to the sa1101
- * support chip for the sa1100
- *
- */
-
-
-/* Be sure that virtual mapping is defined right */
-#ifndef __ASM_ARCH_HARDWARE_H
-#error You must include hardware.h not SA-1101.h
-#endif
-
-#ifndef SA1101_BASE
-#error You must define SA-1101 physical base address
-#endif
-
-#ifndef LANGUAGE
-# ifdef __ASSEMBLY__
-#  define LANGUAGE Assembly
-# else
-#  define LANGUAGE C
-# endif
-#endif
-
-/*
- * We have mapped the sa1101 depending on the value of SA1101_BASE.
- * It then appears from 0xf4000000.
- */
-
-#define SA1101_p2v( x )         ((x) - SA1101_BASE + 0xf4000000)
-#define SA1101_v2p( x )         ((x) - 0xf4000000  + SA1101_BASE)
-
-#ifndef SA1101_p2v
-#define SA1101_p2v(PhAdd)  (PhAdd)
-#endif
-
-#include <mach/bitfield.h>
-
-#define C               0
-#define Assembly        1
-
-
-/*
- * Memory map
- */
-
-#define __SHMEM_CONTROL0	0x00000000
-#define __SYSTEM_CONTROL1	0x00000400
-#define __ARBITER		0x00020000
-#define __SYSTEM_CONTROL2	0x00040000
-#define __SYSTEM_CONTROL3	0x00060000
-#define __PARALLEL_PORT		0x00080000
-#define __VIDMEM_CONTROL	0x00100000
-#define __UPDATE_FIFO		0x00120000
-#define __SHMEM_CONTROL1	0x00140000
-#define __INTERRUPT_CONTROL	0x00160000
-#define __USB_CONTROL		0x00180000
-#define __TRACK_INTERFACE	0x001a0000
-#define __MOUSE_INTERFACE	0x001b0000
-#define __KEYPAD_INTERFACE	0x001c0000
-#define __PCMCIA_INTERFACE	0x001e0000
-#define	__VGA_CONTROL		0x00200000
-#define __GPIO_INTERFACE	0x00300000
-
-/*
- * Macro that calculates real address for registers in the SA-1101
- */
-
-#define _SA1101( x )    ((x) + SA1101_BASE)
-
-/*
- * Interface and shared memory controller registers
- *
- * Registers
- *	SKCR		SA-1101 control register (read/write)
- *	SMCR		Shared Memory Controller Register
- *	SNPR		Snoop Register
- */
-
-#define _SKCR		_SA1101( 0x00000000 ) /* SA-1101 Control Reg. */
-#define _SMCR		_SA1101( 0x00140000 ) /* Shared Mem. Control Reg. */
-#define _SNPR		_SA1101( 0x00140400 ) /* Snoop Reg. */
-
-#if LANGUAGE == C
-#define SKCR		(*((volatile Word *) SA1101_p2v (_SKCR)))
-#define SMCR		(*((volatile Word *) SA1101_p2v (_SMCR)))
-#define SNPR		(*((volatile Word *) SA1101_p2v (_SNPR)))
-
-#define SKCR_PLLEn	  0x0001	  /* Enable On-Chip PLL */
-#define SKCR_BCLKEn	  0x0002	  /* Enables BCLK */
-#define SKCR_Sleep	  0x0004	  /* Sleep Mode */
-#define SKCR_IRefEn	  0x0008	  /* DAC Iref input enable */
-#define SKCR_VCOON	  0x0010	  /* VCO bias */
-#define SKCR_ScanTestEn	  0x0020	  /* Enables scan test */
-#define SKCR_ClockTestEn  0x0040	  /* Enables clock test */
-
-#define SMCR_DCAC	  Fld(2,0)	  /* Number of column address bits */
-#define SMCR_DRAC	  Fld(2,2)	  /* Number of row address bits */
-#define SMCR_ArbiterBias  0x0008	  /* favor video or USB */
-#define SMCR_TopVidMem	  Fld(4,5)	  /* Top 4 bits of vidmem addr. */
-
-#define SMCR_ColAdrBits( x )		  /* col. addr bits 8..11 */ \
-	(( (x) - 8 ) << FShft (SMCR_DCAC))
-#define SMCR_RowAdrBits( x )		  /* row addr bits 9..12 */\
-	(( (x) - 9 ) << FShft (SMCR_DRAC))
-
-#define SNPR_VFBstart	  Fld(12,0)	/* Video frame buffer addr */
-#define SNPR_VFBsize	  Fld(11,12)	/* Video frame buffer size */
-#define SNPR_WholeBank	  (1 << 23)	/* Whole bank bit */
-#define SNPR_BankSelect	  Fld(2,27)	/* Bank select */
-#define SNPR_SnoopEn	  (1 << 31)	/* Enable snoop operation */
-
-#define SNPR_Set_VFBsize( x )   /* set frame buffer size (in kb) */ \
-	( (x) << FShft (SNPR_VFBsize))
-#define SNPR_Select_Bank(x)     /* select bank 0 or 1 */  \
-	(( (x) + 1 ) << FShft (SNPR_BankSelect ))
-
-#endif /* LANGUAGE == C */
-
-/*
- * Video Memory Controller
- *
- * Registers
- *    VMCCR	Configuration register
- *    VMCAR	VMC address register
- *    VMCDR	VMC data register
- *
- */
-
-#define _VMCCR		_SA1101( 0x00100000 )	/* Configuration register */
-#define _VMCAR		_SA1101( 0x00101000 )	/* VMC address register */
-#define _VMCDR		_SA1101( 0x00101400 )	/* VMC data register */
-
-#if LANGUAGE == C
-#define VMCCR		(*((volatile Word *) SA1101_p2v (_VMCCR)))
-#define VMCAR		(*((volatile Word *) SA1101_p2v (_VMCAR)))
-#define VMCDR		(*((volatile Word *) SA1101_p2v (_VMCDR)))
-
-#define VMCCR_RefreshEn	    0x0000	  /* Enable memory refresh */
-#define VMCCR_Config	    0x0001	  /* DRAM size */
-#define VMCCR_RefPeriod	    Fld(2,3)	  /* Refresh period */
-#define VMCCR_StaleDataWait Fld(4,5)	  /* Stale FIFO data timeout counter */
-#define VMCCR_SleepState    (1<<9)	  /* State of interface pins in sleep*/
-#define VMCCR_RefTest	    (1<<10)	  /* refresh test */
-#define VMCCR_RefLow	    Fld(6,11)	  /* refresh low counter */
-#define VMCCR_RefHigh	    Fld(7,17)	  /* refresh high counter */
-#define VMCCR_SDTCTest	    Fld(7,24)	  /* stale data timeout counter */
-#define VMCCR_ForceSelfRef  (1<<31)	  /* Force self refresh */
-
-#endif LANGUAGE == C
-
-
-/* Update FIFO
- *
- * Registers
- *    UFCR	Update FIFO Control Register
- *    UFSR	Update FIFO Status Register
- *    UFLVLR	update FIFO level register
- *    UFDR	update FIFO data register
- */
-
-#define _UFCR	_SA1101(0x00120000)   /* Update FIFO Control Reg. */
-#define _UFSR	_SA1101(0x00120400)   /* Update FIFO Status Reg. */	
-#define _UFLVLR	_SA1101(0x00120800)   /* Update FIFO level reg. */
-#define _UFDR	_SA1101(0x00120c00)   /* Update FIFO data reg. */
-
-#if LANGUAGE == C
-
-#define UFCR 	(*((volatile Word *) SA1101_p2v (_UFCR)))
-#define UFSR	(*((volatile Word *) SA1101_p2v (_UFSR)))
-#define UFLVLR	(*((volatile Word *) SA1101_p2v (_UFLVLR))) 
-#define UFDR	(*((volatile Word *) SA1101_p2v (_UFDR)))
-
-
-#define UFCR_FifoThreshhold	Fld(7,0)	/* Level for FifoGTn flag */
-
-#define UFSR_FifoGTnFlag	0x01		/* FifoGTn flag */#define UFSR_FifoEmpty		0x80		/* FIFO is empty */
-
-#endif /* LANGUAGE == C */
-
-/* System Controller
- *
- * Registers
- *    SKPCR	Power Control Register
- *    SKCDR	Clock Divider Register
- *    DACDR1	DAC1 Data register
- *    DACDR2	DAC2 Data register
- */
-
-#define _SKPCR		_SA1101(0x00000400)
-#define _SKCDR		_SA1101(0x00040000)
-#define _DACDR1		_SA1101(0x00060000)
-#define _DACDR2		_SA1101(0x00060400)
-
-#if LANGUAGE == C
-#define SKPCR 	(*((volatile Word *) SA1101_p2v (_SKPCR)))
-#define SKCDR	(*((volatile Word *) SA1101_p2v (_SKCDR)))
-#define DACDR1	(*((volatile Word *) SA1101_p2v (_DACDR1)))
-#define DACDR2	(*((volatile Word *) SA1101_p2v (_DACDR2)))
-
-#define SKPCR_UCLKEn	     0x01    /* USB Enable */
-#define SKPCR_PCLKEn	     0x02    /* PS/2 Enable */
-#define SKPCR_ICLKEn	     0x04    /* Interrupt Controller Enable */
-#define SKPCR_VCLKEn	     0x08    /* Video Controller Enable */
-#define SKPCR_PICLKEn	     0x10    /* parallel port Enable */
-#define SKPCR_DCLKEn	     0x20    /* DACs Enable */
-#define SKPCR_nKPADEn	     0x40    /* Multiplexer */
-
-#define SKCDR_PLLMul	     Fld(7,0)	/* PLL Multiplier */
-#define SKCDR_VCLKEn	     Fld(2,7)	/* Video controller clock divider */
-#define SKDCR_BCLKEn	     (1<<9)	/* BCLK Divider */
-#define SKDCR_UTESTCLKEn     (1<<10)	/* Route USB clock during test mode */
-#define SKDCR_DivRValue	     Fld(6,11)	/* Input clock divider for PLL */
-#define SKDCR_DivNValue	     Fld(5,17)	/* Output clock divider for PLL */
-#define SKDCR_PLLRSH	     Fld(3,22)	/* PLL bandwidth control */
-#define SKDCR_ChargePump     (1<<25)	/* Charge pump control */
-#define SKDCR_ClkTestMode    (1<<26)	/* Clock output test mode */
-#define SKDCR_ClkTestEn	     (1<<27)	/* Test clock generator */
-#define SKDCR_ClkJitterCntl  Fld(3,28)	/* video clock jitter compensation */
-
-#define DACDR_DACCount	     Fld(8,0)	/* Count value */
-#define DACDR1_DACCount	     DACDR_DACCount
-#define DACDR2_DACCount	     DACDR_DACCount
-
-#endif /* LANGUAGE == C */
-
-/*
- * Parallel Port Interface
- *
- * Registers
- *    IEEE_Config	IEEE mode selection and programmable attributes
- *    IEEE_Control	Controls the states of IEEE port control outputs
- *    IEEE_Data		Forward transfer data register
- *    IEEE_Addr		Forward transfer address register
- *    IEEE_Status	Port IO signal status register
- *    IEEE_IntStatus	Port interrupts status register
- *    IEEE_FifoLevels   Rx and Tx FIFO interrupt generation levels
- *    IEEE_InitTime	Forward timeout counter initial value
- *    IEEE_TimerStatus	Forward timeout counter current value
- *    IEEE_FifoReset	Reset forward transfer FIFO
- *    IEEE_ReloadValue	Counter reload value
- *    IEEE_TestControl	Control testmode
- *    IEEE_TestDataIn	Test data register
- *    IEEE_TestDataInEn	Enable test data
- *    IEEE_TestCtrlIn	Test control signals
- *    IEEE_TestCtrlInEn	Enable test control signals
- *    IEEE_TestDataStat	Current data bus value
- *
- */
-
-/*
- * The control registers are defined as offsets from a base address 
- */
- 
-#define _IEEE( x ) _SA1101( (x) + __PARALLEL_PORT )
-
-#define _IEEE_Config	    _IEEE( 0x0000 )
-#define _IEEE_Control	    _IEEE( 0x0400 )
-#define _IEEE_Data	    _IEEE( 0x4000 )
-#define _IEEE_Addr	    _IEEE( 0x0800 )
-#define _IEEE_Status	    _IEEE( 0x0c00 )
-#define _IEEE_IntStatus	    _IEEE( 0x1000 )
-#define _IEEE_FifoLevels    _IEEE( 0x1400 )
-#define _IEEE_InitTime	    _IEEE( 0x1800 )
-#define _IEEE_TimerStatus   _IEEE( 0x1c00 )
-#define _IEEE_FifoReset	    _IEEE( 0x2000 )
-#define _IEEE_ReloadValue   _IEEE( 0x3c00 )
-#define _IEEE_TestControl   _IEEE( 0x2400 )
-#define _IEEE_TestDataIn    _IEEE( 0x2800 )
-#define _IEEE_TestDataInEn  _IEEE( 0x2c00 )
-#define _IEEE_TestCtrlIn    _IEEE( 0x3000 )
-#define _IEEE_TestCtrlInEn  _IEEE( 0x3400 )
-#define _IEEE_TestDataStat  _IEEE( 0x3800 )
- 
-
-#if LANGUAGE == C
-#define IEEE_Config	    (*((volatile Word *) SA1101_p2v (_IEEE_Config)))
-#define IEEE_Control	    (*((volatile Word *) SA1101_p2v (_IEEE_Control)))
-#define IEEE_Data	    (*((volatile Word *) SA1101_p2v (_IEEE_Data)))
-#define IEEE_Addr	    (*((volatile Word *) SA1101_p2v (_IEEE_Addr)))
-#define IEEE_Status	    (*((volatile Word *) SA1101_p2v (_IEEE_Status)))
-#define IEEE_IntStatus	    (*((volatile Word *) SA1101_p2v (_IEEE_IntStatus)))
-#define IEEE_FifoLevels	    (*((volatile Word *) SA1101_p2v (_IEEE_FifoLevels)))
-#define IEEE_InitTime	    (*((volatile Word *) SA1101_p2v (_IEEE_InitTime)))
-#define IEEE_TimerStatus    (*((volatile Word *) SA1101_p2v (_IEEE_TimerStatus)))
-#define IEEE_FifoReset	    (*((volatile Word *) SA1101_p2v (_IEEE_FifoReset)))
-#define IEEE_ReloadValue    (*((volatile Word *) SA1101_p2v (_IEEE_ReloadValue)))
-#define IEEE_TestControl    (*((volatile Word *) SA1101_p2v (_IEEE_TestControl)))
-#define IEEE_TestDataIn     (*((volatile Word *) SA1101_p2v (_IEEE_TestDataIn)))
-#define IEEE_TestDataInEn   (*((volatile Word *) SA1101_p2v (_IEEE_TestDataInEn)))
-#define IEEE_TestCtrlIn     (*((volatile Word *) SA1101_p2v (_IEEE_TestCtrlIn)))
-#define IEEE_TestCtrlInEn   (*((volatile Word *) SA1101_p2v (_IEEE_TestCtrlInEn)))
-#define IEEE_TestDataStat   (*((volatile Word *) SA1101_p2v (_IEEE_TestDataStat)))
-
-
-#define IEEE_Config_M	    Fld(3,0)	 /* Mode select */
-#define IEEE_Config_D	    0x04	 /* FIFO access enable */
-#define IEEE_Config_B	    0x08	 /* 9-bit word enable */
-#define IEEE_Config_T	    0x10	 /* Data transfer enable */
-#define IEEE_Config_A	    0x20	 /* Data transfer direction */
-#define IEEE_Config_E	    0x40	 /* Timer enable */
-#define IEEE_Control_A	    0x08	 /* AutoFd output */
-#define IEEE_Control_E	    0x04	 /* Selectin output */
-#define IEEE_Control_T	    0x02	 /* Strobe output */
-#define IEEE_Control_I	    0x01	 /* Port init output */
-#define IEEE_Data_C	    (1<<31)	 /* Byte count */
-#define IEEE_Data_Db	    Fld(9,16)	 /* Data byte 2 */
-#define IEEE_Data_Da	    Fld(9,0)	 /* Data byte 1 */
-#define IEEE_Addr_A	    Fld(8,0)	 /* forward address transfer byte */
-#define IEEE_Status_A	    0x0100	 /* nAutoFd port output status */
-#define IEEE_Status_E	    0x0080	 /* nSelectIn port output status */
-#define IEEE_Status_T	    0x0040	 /* nStrobe port output status */
-#define IEEE_Status_I	    0x0020	 /* nInit port output status */
-#define IEEE_Status_B	    0x0010	 /* Busy port inout status */
-#define IEEE_Status_S	    0x0008	 /* Select port input status */
-#define IEEE_Status_K	    0x0004	 /* nAck port input status */
-#define IEEE_Status_F	    0x0002	 /* nFault port input status */
-#define IEEE_Status_R	    0x0001	 /* pError port input status */
-
-#define IEEE_IntStatus_IntReqDat	 0x0100
-#define IEEE_IntStatus_IntReqEmp	 0x0080
-#define IEEE_IntStatus_IntReqInt	 0x0040
-#define IEEE_IntStatus_IntReqRav	 0x0020
-#define IEEE_IntStatus_IntReqTim	 0x0010
-#define IEEE_IntStatus_RevAddrComp	 0x0008
-#define IEEE_IntStatus_RevDataComp	 0x0004
-#define IEEE_IntStatus_FwdAddrComp	 0x0002
-#define IEEE_IntStatus_FwdDataComp	 0x0001
-#define IEEE_FifoLevels_RevFifoLevel	 2
-#define IEEE_FifoLevels_FwdFifoLevel	 1
-#define IEEE_InitTime_TimValInit	 Fld(22,0)
-#define IEEE_TimerStatus_TimValStat	 Fld(22,0)
-#define IEEE_ReloadValue_Reload		 Fld(4,0)
-
-#define IEEE_TestControl_RegClk		 0x04
-#define IEEE_TestControl_ClockSelect	 Fld(2,1)
-#define IEEE_TestControl_TimerTestModeEn 0x01
-#define IEEE_TestCtrlIn_PError		 0x10
-#define IEEE_TestCtrlIn_nFault		 0x08
-#define IEEE_TestCtrlIn_nAck		 0x04
-#define IEEE_TestCtrlIn_PSel		 0x02
-#define IEEE_TestCtrlIn_Busy		 0x01
-
-#endif /* LANGUAGE == C */
-
-/*
- * VGA Controller
- *
- * Registers
- *    VideoControl	Video Control Register
- *    VgaTiming0	VGA Timing Register 0
- *    VgaTiming1	VGA Timing Register 1
- *    VgaTiming2	VGA Timing Register 2
- *    VgaTiming3	VGA Timing Register 3
- *    VgaBorder		VGA Border Color Register
- *    VgaDBAR		VGADMA Base Address Register
- *    VgaDCAR		VGADMA Channel Current Address Register
- *    VgaStatus		VGA Status Register
- *    VgaInterruptMask	VGA Interrupt Mask Register
- *    VgaPalette	VGA Palette Registers
- *    DacControl	DAC Control Register
- *    VgaTest		VGA Controller Test Register
- */
-
-#define _VGA( x )	_SA1101( ( x ) + __VGA_CONTROL )
-
-#define _VideoControl	    _VGA( 0x0000 )
-#define _VgaTiming0	    _VGA( 0x0400 )
-#define _VgaTiming1	    _VGA( 0x0800 )
-#define _VgaTiming2	    _VGA( 0x0c00 )
-#define _VgaTiming3	    _VGA( 0x1000 )
-#define _VgaBorder	    _VGA( 0x1400 )
-#define _VgaDBAR	    _VGA( 0x1800 )
-#define _VgaDCAR	    _VGA( 0x1c00 )
-#define _VgaStatus	    _VGA( 0x2000 )
-#define _VgaInterruptMask   _VGA( 0x2400 )
-#define _VgaPalette	    _VGA( 0x40000 )
-#define _DacControl	    _VGA( 0x3000 )
-#define _VgaTest	    _VGA( 0x2c00 )
-
-#if (LANGUAGE == C)
-#define VideoControl   (*((volatile Word *) SA1101_p2v (_VideoControl)))
-#define VgaTiming0     (*((volatile Word *) SA1101_p2v (_VgaTiming0)))
-#define VgaTiming1     (*((volatile Word *) SA1101_p2v (_VgaTiming1)))
-#define VgaTiming2     (*((volatile Word *) SA1101_p2v (_VgaTiming2)))
-#define VgaTiming3     (*((volatile Word *) SA1101_p2v (_VgaTiming3)))
-#define VgaBorder      (*((volatile Word *) SA1101_p2v (_VgaBorder)))
-#define VgaDBAR	       (*((volatile Word *) SA1101_p2v (_VgaDBAR)))
-#define VgaDCAR	       (*((volatile Word *) SA1101_p2v (_VgaDCAR)))
-#define VgaStatus      (*((volatile Word *) SA1101_p2v (_VgaStatus)))
-#define VgaInterruptMask (*((volatile Word *) SA1101_p2v (_VgaInterruptMask)))
-#define VgaPalette     (*((volatile Word *) SA1101_p2v (_VgaPalette)))
-#define DacControl     (*((volatile Word *) SA1101_p2v (_DacControl)))
-#define VgaTest        (*((volatile Word *) SA1101_p2v (_VgaTest)))
-
-#define VideoControl_VgaEn    0x00000000
-#define VideoControl_BGR      0x00000001
-#define VideoControl_VCompVal Fld(2,2)
-#define VideoControl_VgaReq   Fld(4,4)
-#define VideoControl_VBurstL  Fld(4,8)
-#define VideoControl_VMode    (1<<12)
-#define VideoControl_PalRead  (1<<13)
-
-#define VgaTiming0_PPL	      Fld(6,2)
-#define VgaTiming0_HSW	      Fld(8,8)
-#define VgaTiming0_HFP	      Fld(8,16)
-#define VgaTiming0_HBP	      Fld(8,24)
-
-#define VgaTiming1_LPS	      Fld(10,0)
-#define VgaTiming1_VSW	      Fld(6,10)
-#define VgaTiming1_VFP	      Fld(8,16)
-#define VgaTiming1_VBP	      Fld(8,24)
-
-#define VgaTiming2_IVS	      0x01
-#define VgaTiming2_IHS	      0x02
-#define VgaTiming2_CVS	      0x04
-#define VgaTiming2_CHS	      0x08
-
-#define VgaTiming3_HBS	      Fld(8,0)
-#define VgaTiming3_HBE	      Fld(8,8)
-#define VgaTiming3_VBS	      Fld(8,16)
-#define VgaTiming3_VBE	      Fld(8,24)
-
-#define VgaBorder_BCOL	      Fld(24,0)
-
-#define VgaStatus_VFUF	      0x01
-#define VgaStatus_VNext	      0x02
-#define VgaStatus_VComp	      0x04
-
-#define VgaInterruptMask_VFUFMask   0x00
-#define VgaInterruptMask_VNextMask  0x01
-#define VgaInterruptMask_VCompMask  0x02
-
-#define VgaPalette_R	      Fld(8,0)
-#define VgaPalette_G	      Fld(8,8)
-#define VgaPalette_B	      Fld(8,16)
-
-#define DacControl_DACON      0x0001
-#define DacControl_COMPON     0x0002
-#define DacControl_PEDON      0x0004
-#define DacControl_RTrim      Fld(5,4)
-#define DacControl_GTrim      Fld(5,9)
-#define DacControl_BTrim      Fld(5,14)
-
-#define VgaTest_TDAC	      0x00
-#define VgaTest_Datatest      Fld(4,1)
-#define VgaTest_DACTESTDAC    0x10
-#define VgaTest_DACTESTOUT    Fld(3,5)
-
-#endif /* LANGUAGE == C */
-
-/*
- * USB Host Interface Controller
- *
- * Registers
- *    Revision
- *    Control
- *    CommandStatus
- *    InterruptStatus
- *    InterruptEnable
- *    HCCA
- *    PeriodCurrentED
- *    ControlHeadED
- *    BulkHeadED
- *    BulkCurrentED
- *    DoneHead
- *    FmInterval
- *    FmRemaining
- *    FmNumber
- *    PeriodicStart
- *    LSThreshold
- *    RhDescriptorA
- *    RhDescriptorB
- *    RhStatus
- *    RhPortStatus
- *    USBStatus
- *    USBReset
- *    USTAR
- *    USWER
- *    USRFR
- *    USNFR
- *    USTCSR
- *    USSR
- *    
- */
-
-#define _USB( x )	_SA1101( ( x ) + __USB_CONTROL )
-
-
-#define _Revision	  _USB( 0x0000 )
-#define _Control	  _USB( 0x0888 )
-#define _CommandStatus	  _USB( 0x0c00 )
-#define _InterruptStatus  _USB( 0x1000 )
-#define _InterruptEnable  _USB( 0x1400 )
-#define _HCCA		  _USB( 0x1800 )
-#define _PeriodCurrentED  _USB( 0x1c00 )
-#define _ControlHeadED	  _USB( 0x2000 )
-#define _BulkHeadED	  _USB( 0x2800 )
-#define _BulkCurrentED	  _USB( 0x2c00 )
-#define _DoneHead	  _USB( 0x3000 )
-#define _FmInterval	  _USB( 0x3400 )
-#define _FmRemaining	  _USB( 0x3800 )
-#define _FmNumber	  _USB( 0x3c00 )
-#define _PeriodicStart	  _USB( 0x4000 )
-#define _LSThreshold	  _USB( 0x4400 )
-#define _RhDescriptorA	  _USB( 0x4800 )
-#define _RhDescriptorB	  _USB( 0x4c00 )
-#define _RhStatus	  _USB( 0x5000 )
-#define _RhPortStatus	  _USB( 0x5400 )
-#define _USBStatus	  _USB( 0x11800 )
-#define _USBReset	  _USB( 0x11c00 )
-
-#define _USTAR		  _USB( 0x10400 )
-#define _USWER		  _USB( 0x10800 )
-#define _USRFR		  _USB( 0x10c00 )
-#define _USNFR		  _USB( 0x11000 )
-#define _USTCSR		  _USB( 0x11400 )
-#define _USSR		  _USB( 0x11800 )
-
-
-#if (LANGUAGE == C)
-
-#define Revision	(*((volatile Word *) SA1101_p2v (_Revision)))
-#define Control		(*((volatile Word *) SA1101_p2v (_Control)))
-#define CommandStatus	(*((volatile Word *) SA1101_p2v (_CommandStatus)))
-#define InterruptStatus	(*((volatile Word *) SA1101_p2v (_InterruptStatus)))
-#define InterruptEnable	(*((volatile Word *) SA1101_p2v (_InterruptEnable)))
-#define HCCA		(*((volatile Word *) SA1101_p2v (_HCCA)))
-#define PeriodCurrentED	(*((volatile Word *) SA1101_p2v (_PeriodCurrentED)))
-#define ControlHeadED	(*((volatile Word *) SA1101_p2v (_ControlHeadED)))
-#define BulkHeadED	(*((volatile Word *) SA1101_p2v (_BulkHeadED)))
-#define BulkCurrentED	(*((volatile Word *) SA1101_p2v (_BulkCurrentED)))
-#define DoneHead	(*((volatile Word *) SA1101_p2v (_DoneHead)))
-#define FmInterval	(*((volatile Word *) SA1101_p2v (_FmInterval)))
-#define FmRemaining	(*((volatile Word *) SA1101_p2v (_FmRemaining)))
-#define FmNumber	(*((volatile Word *) SA1101_p2v (_FmNumber)))
-#define PeriodicStart	(*((volatile Word *) SA1101_p2v (_PeriodicStart)))
-#define LSThreshold	(*((volatile Word *) SA1101_p2v (_LSThreshold)))
-#define RhDescriptorA	(*((volatile Word *) SA1101_p2v (_RhDescriptorA)))
-#define RhDescriptorB	(*((volatile Word *) SA1101_p2v (_RhDescriptorB)))
-#define RhStatus	(*((volatile Word *) SA1101_p2v (_RhStatus)))
-#define RhPortStatus	(*((volatile Word *) SA1101_p2v (_RhPortStatus)))
-#define USBStatus	(*((volatile Word *) SA1101_p2v (_USBStatus)))
-#define USBReset	(*((volatile Word *) SA1101_p2v (_USBReset)))
-#define USTAR		(*((volatile Word *) SA1101_p2v (_USTAR)))
-#define USWER		(*((volatile Word *) SA1101_p2v (_USWER)))
-#define USRFR		(*((volatile Word *) SA1101_p2v (_USRFR)))
-#define USNFR		(*((volatile Word *) SA1101_p2v (_USNFR)))
-#define USTCSR		(*((volatile Word *) SA1101_p2v (_USTCSR)))
-#define USSR		(*((volatile Word *) SA1101_p2v (_USSR)))
-
-
-#define USBStatus_IrqHciRmtWkp	     (1<<7)
-#define USBStatus_IrqHciBuffAcc	     (1<<8)
-#define USBStatus_nIrqHciM	     (1<<9)
-#define USBStatus_nHciMFClr	     (1<<10)
-
-#define USBReset_ForceIfReset	     0x01
-#define USBReset_ForceHcReset	     0x02
-#define USBReset_ClkGenReset	     0x04
-
-#define USTCR_RdBstCntrl	     Fld(3,0)
-#define USTCR_ByteEnable	     Fld(4,3)
-#define USTCR_WriteEn		     (1<<7)
-#define USTCR_FifoCir		     (1<<8)
-#define USTCR_TestXferSel	     (1<<9)
-#define USTCR_FifoCirAtEnd	     (1<<10)
-#define USTCR_nSimScaleDownClk	     (1<<11)
-
-#define USSR_nAppMDEmpty	     0x01
-#define USSR_nAppMDFirst	     0x02
-#define USSR_nAppMDLast		     0x04
-#define USSR_nAppMDFull		     0x08
-#define USSR_nAppMAFull		     0x10
-#define USSR_XferReq		     0x20
-#define USSR_XferEnd		     0x40
-
-#endif /* LANGUAGE == C */
-
-
-/*
- * Interrupt Controller
- *
- * Registers
- *    INTTEST0		Test register 0
- *    INTTEST1		Test register 1
- *    INTENABLE0	Interrupt Enable register 0
- *    INTENABLE1	Interrupt Enable register 1
- *    INTPOL0		Interrupt Polarity selection 0
- *    INTPOL1		Interrupt Polarity selection 1
- *    INTTSTSEL		Interrupt source selection
- *    INTSTATCLR0	Interrupt Status 0
- *    INTSTATCLR1	Interrupt Status 1
- *    INTSET0		Interrupt Set 0
- *    INTSET1		Interrupt Set 1
- */
-
-#define _INT( x )	_SA1101( ( x ) + __INTERRUPT_CONTROL)
-
-#define _INTTEST0	_INT( 0x1000 )
-#define _INTTEST1	_INT( 0x1400 )
-#define _INTENABLE0	_INT( 0x2000 )
-#define _INTENABLE1	_INT( 0x2400 )
-#define _INTPOL0	_INT( 0x3000 )
-#define _INTPOL1	_INT( 0x3400 )
-#define _INTTSTSEL     	_INT( 0x5000 )
-#define _INTSTATCLR0	_INT( 0x6000 )
-#define _INTSTATCLR1	_INT( 0x6400 )
-#define _INTSET0	_INT( 0x7000 )
-#define _INTSET1	_INT( 0x7400 )
-
-#if ( LANGUAGE == C )
-#define INTTEST0	(*((volatile Word *) SA1101_p2v (_INTTEST0)))
-#define INTTEST1	(*((volatile Word *) SA1101_p2v (_INTTEST1)))
-#define INTENABLE0	(*((volatile Word *) SA1101_p2v (_INTENABLE0)))
-#define INTENABLE1	(*((volatile Word *) SA1101_p2v (_INTENABLE1)))
-#define INTPOL0		(*((volatile Word *) SA1101_p2v (_INTPOL0)))
-#define INTPOL1		(*((volatile Word *) SA1101_p2v (_INTPOL1)))
-#define INTTSTSEL	(*((volatile Word *) SA1101_p2v (_INTTSTSEL)))
-#define INTSTATCLR0	(*((volatile Word *) SA1101_p2v (_INTSTATCLR0)))
-#define INTSTATCLR1	(*((volatile Word *) SA1101_p2v (_INTSTATCLR1)))
-#define INTSET0		(*((volatile Word *) SA1101_p2v (_INTSET0)))
-#define INTSET1		(*((volatile Word *) SA1101_p2v (_INTSET1)))
-
-#endif /* LANGUAGE == C */
-
-/*
- * PS/2 Trackpad and Mouse Interfaces
- *
- * Registers   (prefix kbd applies to trackpad interface, mse to mouse)
- *    KBDCR		Control Register
- *    KBDSTAT		Status Register
- *    KBDDATA		Transmit/Receive Data register
- *    KBDCLKDIV		Clock Division Register
- *    KBDPRECNT		Clock Precount Register
- *    KBDTEST1		Test register 1
- *    KBDTEST2		Test register 2
- *    KBDTEST3		Test register 3
- *    KBDTEST4		Test register 4
- *    MSECR	
- *    MSESTAT
- *    MSEDATA
- *    MSECLKDIV
- *    MSEPRECNT
- *    MSETEST1
- *    MSETEST2
- *    MSETEST3
- *    MSETEST4
- *     
- */
-
-#define _KBD( x )	_SA1101( ( x ) + __TRACK_INTERFACE )
-#define _MSE( x )	_SA1101( ( x ) + __MOUSE_INTERFACE )
-
-#define _KBDCR		_KBD( 0x0000 )
-#define _KBDSTAT	_KBD( 0x0400 )
-#define _KBDDATA	_KBD( 0x0800 )
-#define _KBDCLKDIV	_KBD( 0x0c00 )
-#define _KBDPRECNT	_KBD( 0x1000 )
-#define	_KBDTEST1	_KBD( 0x2000 )
-#define _KBDTEST2	_KBD( 0x2400 )
-#define _KBDTEST3	_KBD( 0x2800 )
-#define _KBDTEST4	_KBD( 0x2c00 )
-#define _MSECR		_MSE( 0x0000 )
-#define _MSESTAT	_MSE( 0x0400 )
-#define _MSEDATA	_MSE( 0x0800 )
-#define _MSECLKDIV	_MSE( 0x0c00 )
-#define _MSEPRECNT	_MSE( 0x1000 )
-#define	_MSETEST1	_MSE( 0x2000 )
-#define _MSETEST2	_MSE( 0x2400 )
-#define _MSETEST3	_MSE( 0x2800 )
-#define _MSETEST4	_MSE( 0x2c00 )
-
-#if ( LANGUAGE == C )
-
-#define KBDCR	    (*((volatile Word *) SA1101_p2v (_KBDCR)))
-#define KBDSTAT	    (*((volatile Word *) SA1101_p2v (_KBDSTAT)))
-#define KBDDATA	    (*((volatile Word *) SA1101_p2v (_KBDDATA)))
-#define KBDCLKDIV   (*((volatile Word *) SA1101_p2v (_KBDCLKDIV)))
-#define KBDPRECNT   (*((volatile Word *) SA1101_p2v (_KBDPRECNT)))
-#define KBDTEST1    (*((volatile Word *) SA1101_p2v (_KBDTEST1)))
-#define KBDTEST2    (*((volatile Word *) SA1101_p2v (_KBDTEST2)))
-#define KBDTEST3    (*((volatile Word *) SA1101_p2v (_KBDTEST3)))
-#define KBDTEST4    (*((volatile Word *) SA1101_p2v (_KBDTEST4)))
-#define MSECR	    (*((volatile Word *) SA1101_p2v (_MSECR)))
-#define MSESTAT	    (*((volatile Word *) SA1101_p2v (_MSESTAT)))
-#define MSEDATA	    (*((volatile Word *) SA1101_p2v (_MSEDATA)))
-#define MSECLKDIV   (*((volatile Word *) SA1101_p2v (_MSECLKDIV)))
-#define MSEPRECNT   (*((volatile Word *) SA1101_p2v (_MSEPRECNT)))
-#define MSETEST1    (*((volatile Word *) SA1101_p2v (_MSETEST1)))
-#define MSETEST2    (*((volatile Word *) SA1101_p2v (_MSETEST2)))
-#define MSETEST3    (*((volatile Word *) SA1101_p2v (_MSETEST3)))
-#define MSETEST4    (*((volatile Word *) SA1101_p2v (_MSETEST4)))
-
-
-#define KBDCR_ENA		 0x08
-#define KBDCR_FKD		 0x02
-#define KBDCR_FKC		 0x01
-
-#define KBDSTAT_TXE		 0x80
-#define KBDSTAT_TXB		 0x40
-#define KBDSTAT_RXF		 0x20
-#define KBDSTAT_RXB		 0x10
-#define KBDSTAT_ENA		 0x08
-#define KBDSTAT_RXP		 0x04
-#define KBDSTAT_KBD		 0x02
-#define KBDSTAT_KBC		 0x01
-
-#define KBDCLKDIV_DivVal	 Fld(4,0)
-
-#define MSECR_ENA		 0x08
-#define MSECR_FKD		 0x02
-#define MSECR_FKC		 0x01
-
-#define MSESTAT_TXE		 0x80
-#define MSESTAT_TXB		 0x40
-#define MSESTAT_RXF		 0x20
-#define MSESTAT_RXB		 0x10
-#define MSESTAT_ENA		 0x08
-#define MSESTAT_RXP		 0x04	
-#define MSESTAT_MSD		 0x02
-#define MSESTAT_MSC		 0x01
-
-#define MSECLKDIV_DivVal	 Fld(4,0)
-
-#define KBDTEST1_CD		 0x80
-#define KBDTEST1_RC1		 0x40
-#define KBDTEST1_MC		 0x20
-#define KBDTEST1_C		 Fld(2,3)
-#define KBDTEST1_T2		 0x40
-#define KBDTEST1_T1		 0x20
-#define KBDTEST1_T0		 0x10
-#define KBDTEST2_TICBnRES	 0x08
-#define KBDTEST2_RKC		 0x04
-#define KBDTEST2_RKD		 0x02
-#define KBDTEST2_SEL		 0x01
-#define KBDTEST3_ms_16		 0x80
-#define KBDTEST3_us_64		 0x40
-#define KBDTEST3_us_16		 0x20
-#define KBDTEST3_DIV8		 0x10
-#define KBDTEST3_DIn		 0x08
-#define KBDTEST3_CIn		 0x04
-#define KBDTEST3_KD		 0x02
-#define KBDTEST3_KC		 0x01
-#define KBDTEST4_BC12		 0x80
-#define KBDTEST4_BC11		 0x40
-#define KBDTEST4_TRES		 0x20
-#define KBDTEST4_CLKOE		 0x10
-#define KBDTEST4_CRES		 0x08
-#define KBDTEST4_RXB		 0x04
-#define KBDTEST4_TXB		 0x02
-#define KBDTEST4_SRX		 0x01
-
-#define MSETEST1_CD		 0x80
-#define MSETEST1_RC1		 0x40
-#define MSETEST1_MC		 0x20
-#define MSETEST1_C		 Fld(2,3)
-#define MSETEST1_T2		 0x40
-#define MSETEST1_T1		 0x20
-#define MSETEST1_T0		 0x10
-#define MSETEST2_TICBnRES	 0x08
-#define MSETEST2_RKC		 0x04
-#define MSETEST2_RKD		 0x02
-#define MSETEST2_SEL		 0x01
-#define MSETEST3_ms_16		 0x80
-#define MSETEST3_us_64		 0x40
-#define MSETEST3_us_16		 0x20
-#define MSETEST3_DIV8		 0x10
-#define MSETEST3_DIn		 0x08
-#define MSETEST3_CIn		 0x04
-#define MSETEST3_KD		 0x02
-#define MSETEST3_KC		 0x01
-#define MSETEST4_BC12		 0x80
-#define MSETEST4_BC11		 0x40
-#define MSETEST4_TRES		 0x20
-#define MSETEST4_CLKOE		 0x10
-#define MSETEST4_CRES		 0x08
-#define MSETEST4_RXB		 0x04
-#define MSETEST4_TXB		 0x02
-#define MSETEST4_SRX		 0x01
-
-#endif  /* LANGUAGE == C */
-
-
-/*
- * General-Purpose I/O Interface
- *
- * Registers
- *    PADWR	Port A Data Write Register
- *    PBDWR	Port B Data Write Register
- *    PADRR	Port A Data Read Register
- *    PBDRR	Port B Data Read Register
- *    PADDR	Port A Data Direction Register
- *    PBDDR	Port B Data Direction Register
- *    PASSR	Port A Sleep State Register
- *    PBSSR	Port B Sleep State Register
- *
- */
-
-#define _PIO( x )      _SA1101( ( x ) + __GPIO_INTERFACE )
-
-#define _PADWR	       _PIO( 0x0000 )
-#define _PBDWR	       _PIO( 0x0400 )
-#define _PADRR	       _PIO( 0x0000 )
-#define _PBDRR	       _PIO( 0x0400 )
-#define _PADDR	       _PIO( 0x0800 )
-#define _PBDDR	       _PIO( 0x0c00 )
-#define _PASSR	       _PIO( 0x1000 )
-#define _PBSSR	       _PIO( 0x1400 )
-
-
-#if ( LANGUAGE == C )
-
-
-#define PADWR	    (*((volatile Word *) SA1101_p2v (_PADWR)))
-#define PBDWR	    (*((volatile Word *) SA1101_p2v (_PBDWR)))
-#define PADRR	    (*((volatile Word *) SA1101_p2v (_PADRR)))
-#define PBDRR	    (*((volatile Word *) SA1101_p2v (_PBDRR)))
-#define PADDR	    (*((volatile Word *) SA1101_p2v (_PADDR)))
-#define PBDDR	    (*((volatile Word *) SA1101_p2v (_PBDDR)))
-#define PASSR	    (*((volatile Word *) SA1101_p2v (_PASSR)))
-#define PBSSR	    (*((volatile Word *) SA1101_p2v (_PBSSR)))
-
-#endif
-
-
-
-/*
- * Keypad Interface
- *
- * Registers
- *    PXDWR
- *    PXDRR
- *    PYDWR
- *    PYDRR
- *
- */
-
-#define _KEYPAD( x )	_SA1101( ( x ) + __KEYPAD_INTERFACE ) 
-
-#define _PXDWR	   _KEYPAD( 0x0000 )
-#define _PXDRR	   _KEYPAD( 0x0000 )
-#define _PYDWR	   _KEYPAD( 0x0400 )
-#define _PYDRR	   _KEYPAD( 0x0400 )
-
-#if ( LANGUAGE == C )
-
-
-#define PXDWR	    (*((volatile Word *) SA1101_p2v (_PXDWR)))
-#define PXDRR	    (*((volatile Word *) SA1101_p2v (_PXDRR)))
-#define PYDWR	    (*((volatile Word *) SA1101_p2v (_PYDWR)))
-#define PYDRR	    (*((volatile Word *) SA1101_p2v (_PYDRR)))
-
-#endif
-
-
-
-/*
- * PCMCIA Interface
- *
- * Registers
- *    PCSR	Status Register
- *    PCCR	Control Register
- *    PCSSR	Sleep State Register
- *
- */
-
-#define _CARD( x )	_SA1101( ( x ) + __PCMCIA_INTERFACE )
-
-#define _PCSR	   _CARD( 0x0000 )
-#define _PCCR	   _CARD( 0x0400 )
-#define _PCSSR	   _CARD( 0x0800 )
-
-#if ( LANGUAGE == C )
-#define PCSR    (*((volatile Word *) SA1101_p2v (_PCSR)))
-#define PCCR	(*((volatile Word *) SA1101_p2v (_PCCR)))
-#define PCSSR	(*((volatile Word *) SA1101_p2v (_PCSSR)))
-
-#define PCSR_S0_ready		0x0001
-#define PCSR_S1_ready		0x0002
-#define PCSR_S0_detected	0x0004
-#define PCSR_S1_detected	0x0008
-#define PCSR_S0_VS1		0x0010
-#define PCSR_S0_VS2		0x0020
-#define PCSR_S1_VS1		0x0040
-#define PCSR_S1_VS2		0x0080
-#define PCSR_S0_WP		0x0100
-#define PCSR_S1_WP		0x0200
-#define PCSR_S0_BVD1_nSTSCHG	0x0400
-#define PCSR_S0_BVD2_nSPKR	0x0800
-#define PCSR_S1_BVD1_nSTSCHG	0x1000
-#define PCSR_S1_BVD2_nSPKR	0x2000
-
-#define PCCR_S0_VPP0		0x0001
-#define PCCR_S0_VPP1		0x0002
-#define PCCR_S0_VCC0		0x0004
-#define PCCR_S0_VCC1		0x0008
-#define PCCR_S1_VPP0		0x0010
-#define PCCR_S1_VPP1		0x0020
-#define PCCR_S1_VCC0		0x0040
-#define PCCR_S1_VCC1		0x0080
-#define PCCR_S0_reset		0x0100
-#define PCCR_S1_reset		0x0200
-#define PCCR_S0_float		0x0400
-#define PCCR_S1_float		0x0800
-
-#define PCSSR_S0_VCC0		0x0001
-#define PCSSR_S0_VCC1		0x0002
-#define PCSSR_S0_VPP0		0x0004
-#define PCSSR_S0_VPP1		0x0008
-#define PCSSR_S0_control	0x0010
-#define PCSSR_S1_VCC0		0x0020
-#define PCSSR_S1_VCC1		0x0040
-#define PCSSR_S1_VPP0		0x0080
-#define PCSSR_S1_VPP1		0x0100
-#define PCSSR_S1_control	0x0200
-
-#endif
-
-#undef C
-#undef Assembly
diff --git a/arch/arm/mach-sa1100/include/mach/hardware.h b/arch/arm/mach-sa1100/include/mach/hardware.h
index d944fd7e464f..cc43f95f33cc 100644
--- a/arch/arm/mach-sa1100/include/mach/hardware.h
+++ b/arch/arm/mach-sa1100/include/mach/hardware.h
@@ -43,10 +43,6 @@
 # define __REG(x)	(*((volatile unsigned long __iomem *)io_p2v(x)))
 # define __PREG(x)	(io_v2p((unsigned long)&(x)))
 
-static inline unsigned long get_clock_tick_rate(void)
-{
-	return 3686400;
-}
 #else
 
 # define __REG(x)	io_p2v(x)
@@ -56,8 +52,4 @@ static inline unsigned long get_clock_tick_rate(void)
 
 #include "SA-1100.h"
 
-#ifdef CONFIG_SA1101
-#include "SA-1101.h"
-#endif
-
 #endif  /* _ASM_ARCH_HARDWARE_H */
diff --git a/arch/arm/mach-shmobile/Kconfig b/arch/arm/mach-shmobile/Kconfig
index 09817bae4558..2bb4b09f079e 100644
--- a/arch/arm/mach-shmobile/Kconfig
+++ b/arch/arm/mach-shmobile/Kconfig
@@ -32,15 +32,16 @@ config ARCH_RMOBILE
 menuconfig ARCH_RENESAS
 	bool "Renesas ARM SoCs"
 	depends on ARCH_MULTI_V7 && MMU
+	select ARCH_DMA_ADDR_T_64BIT if ARM_LPAE
 	select ARCH_SHMOBILE
 	select ARCH_SHMOBILE_MULTI
+	select ARM_GIC
+	select GPIOLIB
 	select HAVE_ARM_SCU if SMP
 	select HAVE_ARM_TWD if SMP
-	select ARM_GIC
-	select ARCH_DMA_ADDR_T_64BIT if ARM_LPAE
 	select NO_IOPORT_MAP
 	select PINCTRL
-	select GPIOLIB
+	select SOC_BUS
 	select ZONE_DMA if ARM_LPAE
 
 if ARCH_RENESAS
@@ -60,6 +61,7 @@ config ARCH_R7S72100
 config ARCH_R8A73A4
 	bool "R-Mobile APE6 (R8A73A40)"
 	select ARCH_RMOBILE
+	select ARM_ERRATA_798181 if SMP
 	select RENESAS_IRQC
 
 config ARCH_R8A7740
@@ -67,6 +69,15 @@ config ARCH_R8A7740
 	select ARCH_RMOBILE
 	select RENESAS_INTC_IRQPIN
 
+config ARCH_R8A7743
+	bool "RZ/G1M (R8A77430)"
+	select ARCH_RCAR_GEN2
+	select ARM_ERRATA_798181 if SMP
+
+config ARCH_R8A7745
+	bool "RZ/G1E (R8A77450)"
+	select ARCH_RCAR_GEN2
+
 config ARCH_R8A7778
 	bool "R-Car M1A (R8A77781)"
 	select ARCH_RCAR_GEN1
@@ -78,20 +89,24 @@ config ARCH_R8A7779
 config ARCH_R8A7790
 	bool "R-Car H2 (R8A77900)"
 	select ARCH_RCAR_GEN2
+	select ARM_ERRATA_798181 if SMP
 	select I2C
 
 config ARCH_R8A7791
 	bool "R-Car M2-W (R8A77910)"
 	select ARCH_RCAR_GEN2
+	select ARM_ERRATA_798181 if SMP
 	select I2C
 
 config ARCH_R8A7792
 	bool "R-Car V2H (R8A77920)"
 	select ARCH_RCAR_GEN2
+	select ARM_ERRATA_798181 if SMP
 
 config ARCH_R8A7793
 	bool "R-Car M2-N (R8A7793)"
 	select ARCH_RCAR_GEN2
+	select ARM_ERRATA_798181 if SMP
 	select I2C
 
 config ARCH_R8A7794
diff --git a/arch/arm/mach-shmobile/Makefile b/arch/arm/mach-shmobile/Makefile
index 3fc48b02eb4f..64611a1b4276 100644
--- a/arch/arm/mach-shmobile/Makefile
+++ b/arch/arm/mach-shmobile/Makefile
@@ -13,9 +13,6 @@ obj-$(CONFIG_ARCH_R8A7778)	+= setup-r8a7778.o
 obj-$(CONFIG_ARCH_R8A7779)	+= setup-r8a7779.o pm-r8a7779.o
 obj-$(CONFIG_ARCH_R8A7790)	+= setup-r8a7790.o
 obj-$(CONFIG_ARCH_R8A7791)	+= setup-r8a7791.o
-obj-$(CONFIG_ARCH_R8A7792)	+= setup-r8a7792.o
-obj-$(CONFIG_ARCH_R8A7793)	+= setup-r8a7793.o
-obj-$(CONFIG_ARCH_R8A7794)	+= setup-r8a7794.o
 obj-$(CONFIG_ARCH_EMEV2)	+= setup-emev2.o
 obj-$(CONFIG_ARCH_R7S72100)	+= setup-r7s72100.o
 
diff --git a/arch/arm/mach-shmobile/setup-r8a7778.c b/arch/arm/mach-shmobile/setup-r8a7778.c
index cf236db686a9..7fa4a0b5f654 100644
--- a/arch/arm/mach-shmobile/setup-r8a7778.c
+++ b/arch/arm/mach-shmobile/setup-r8a7778.c
@@ -15,7 +15,6 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/clk/renesas.h>
 #include <linux/io.h>
 #include <linux/irqchip.h>
 
@@ -23,19 +22,6 @@
 
 #include "common.h"
 
-#define MODEMR 0xffcc0020
-
-static void __init r8a7778_timer_init(void)
-{
-	u32 mode;
-	void __iomem *modemr = ioremap_nocache(MODEMR, 4);
-
-	BUG_ON(!modemr);
-	mode = ioread32(modemr);
-	iounmap(modemr);
-	r8a7778_clocks_init(mode);
-}
-
 #define INT2SMSKCR0	0x82288 /* 0xfe782288 */
 #define INT2SMSKCR1	0x8228c /* 0xfe78228c */
 
@@ -70,6 +56,5 @@ DT_MACHINE_START(R8A7778_DT, "Generic R8A7778 (Flattened Device Tree)")
 	.init_early	= shmobile_init_delay,
 	.init_irq	= r8a7778_init_irq_dt,
 	.init_late	= shmobile_init_late,
-	.init_time	= r8a7778_timer_init,
 	.dt_compat	= r8a7778_compat_dt,
 MACHINE_END
diff --git a/arch/arm/mach-shmobile/setup-r8a7779.c b/arch/arm/mach-shmobile/setup-r8a7779.c
index 0007ff51d180..0686112f2435 100644
--- a/arch/arm/mach-shmobile/setup-r8a7779.c
+++ b/arch/arm/mach-shmobile/setup-r8a7779.c
@@ -14,8 +14,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  */
-#include <linux/clk/renesas.h>
-#include <linux/clocksource.h>
 #include <linux/init.h>
 #include <linux/irq.h>
 #include <linux/irqchip.h>
@@ -76,30 +74,6 @@ static void __init r8a7779_init_irq_dt(void)
 	__raw_writel(0x003fee3f, INT2SMSKCR4);
 }
 
-#define MODEMR		0xffcc0020
-
-static u32 __init r8a7779_read_mode_pins(void)
-{
-	static u32 mode;
-	static bool mode_valid;
-
-	if (!mode_valid) {
-		void __iomem *modemr = ioremap_nocache(MODEMR, PAGE_SIZE);
-		BUG_ON(!modemr);
-		mode = ioread32(modemr);
-		iounmap(modemr);
-		mode_valid = true;
-	}
-
-	return mode;
-}
-
-static void __init r8a7779_init_time(void)
-{
-	r8a7779_clocks_init(r8a7779_read_mode_pins());
-	clocksource_probe();
-}
-
 static const char *const r8a7779_compat_dt[] __initconst = {
 	"renesas,r8a7779",
 	NULL,
@@ -109,7 +83,6 @@ DT_MACHINE_START(R8A7779_DT, "Generic R8A7779 (Flattened Device Tree)")
 	.smp		= smp_ops(r8a7779_smp_ops),
 	.map_io		= r8a7779_map_io,
 	.init_early	= shmobile_init_delay,
-	.init_time	= r8a7779_init_time,
 	.init_irq	= r8a7779_init_irq_dt,
 	.init_late	= shmobile_init_late,
 	.dt_compat	= r8a7779_compat_dt,
diff --git a/arch/arm/mach-shmobile/setup-r8a7792.c b/arch/arm/mach-shmobile/setup-r8a7792.c
deleted file mode 100644
index a0910395da09..000000000000
--- a/arch/arm/mach-shmobile/setup-r8a7792.c
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * r8a7792 processor support
- *
- * Copyright (C) 2014 Renesas Electronics Corporation
- * Copyright (C) 2016 Cogent  Embedded, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/of_platform.h>
-
-#include <asm/mach/arch.h>
-
-#include "common.h"
-#include "rcar-gen2.h"
-
-static const char * const r8a7792_boards_compat_dt[] __initconst = {
-	"renesas,r8a7792",
-	NULL,
-};
-
-DT_MACHINE_START(R8A7792_DT, "Generic R8A7792 (Flattened Device Tree)")
-	.init_early	= shmobile_init_delay,
-	.init_late	= shmobile_init_late,
-	.init_time	= rcar_gen2_timer_init,
-	.reserve	= rcar_gen2_reserve,
-	.dt_compat	= r8a7792_boards_compat_dt,
-MACHINE_END
diff --git a/arch/arm/mach-shmobile/setup-r8a7793.c b/arch/arm/mach-shmobile/setup-r8a7793.c
deleted file mode 100644
index 5fce87f7f254..000000000000
--- a/arch/arm/mach-shmobile/setup-r8a7793.c
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * r8a7793 processor support
- *
- * Copyright (C) 2015  Ulrich Hecht
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/init.h>
-#include <asm/mach/arch.h>
-
-#include "common.h"
-#include "rcar-gen2.h"
-
-static const char * const r8a7793_boards_compat_dt[] __initconst = {
-	"renesas,r8a7793",
-	NULL,
-};
-
-DT_MACHINE_START(R8A7793_DT, "Generic R8A7793 (Flattened Device Tree)")
-	.init_early	= shmobile_init_delay,
-	.init_time	= rcar_gen2_timer_init,
-	.init_late	= shmobile_init_late,
-	.reserve	= rcar_gen2_reserve,
-	.dt_compat	= r8a7793_boards_compat_dt,
-MACHINE_END
diff --git a/arch/arm/mach-shmobile/setup-r8a7794.c b/arch/arm/mach-shmobile/setup-r8a7794.c
deleted file mode 100644
index d2b093033132..000000000000
--- a/arch/arm/mach-shmobile/setup-r8a7794.c
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * r8a7794 processor support
- *
- * Copyright (C) 2014  Renesas Electronics Corporation
- * Copyright (C) 2014  Ulrich Hecht
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/of_platform.h>
-#include "common.h"
-#include "rcar-gen2.h"
-#include <asm/mach/arch.h>
-
-static const char * const r8a7794_boards_compat_dt[] __initconst = {
-	"renesas,r8a7794",
-	NULL,
-};
-
-DT_MACHINE_START(R8A7794_DT, "Generic R8A7794 (Flattened Device Tree)")
-	.init_early	= shmobile_init_delay,
-	.init_late	= shmobile_init_late,
-	.init_time	= rcar_gen2_timer_init,
-	.reserve	= rcar_gen2_reserve,
-	.dt_compat	= r8a7794_boards_compat_dt,
-MACHINE_END
diff --git a/arch/arm/mach-shmobile/setup-rcar-gen2.c b/arch/arm/mach-shmobile/setup-rcar-gen2.c
index afb9fdcd3d90..ac63fa407b64 100644
--- a/arch/arm/mach-shmobile/setup-rcar-gen2.c
+++ b/arch/arm/mach-shmobile/setup-rcar-gen2.c
@@ -15,7 +15,7 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/clk/renesas.h>
+#include <linux/clk-provider.h>
 #include <linux/clocksource.h>
 #include <linux/device.h>
 #include <linux/dma-contiguous.h>
@@ -24,6 +24,7 @@
 #include <linux/memblock.h>
 #include <linux/of.h>
 #include <linux/of_fdt.h>
+#include <linux/of_platform.h>
 #include <asm/mach/arch.h>
 #include "common.h"
 #include "rcar-gen2.h"
@@ -71,7 +72,6 @@ static unsigned int __init get_extal_freq(void)
 
 void __init rcar_gen2_timer_init(void)
 {
-	u32 mode = rcar_gen2_read_mode_pins();
 #ifdef CONFIG_ARM_ARCH_TIMER
 	void __iomem *base;
 	u32 freq;
@@ -130,7 +130,7 @@ void __init rcar_gen2_timer_init(void)
 	iounmap(base);
 #endif /* CONFIG_ARM_ARCH_TIMER */
 
-	rcar_gen2_clocks_init(mode);
+	of_clk_init(NULL);
 	clocksource_probe();
 }
 
@@ -203,3 +203,36 @@ void __init rcar_gen2_reserve(void)
 	}
 #endif
 }
+
+static const char * const rcar_gen2_boards_compat_dt[] __initconst = {
+	/*
+	 * R8A7790 and R8A7791 can't be handled here as long as they need SMP
+	 * initialization fallback.
+	 */
+	"renesas,r8a7792",
+	"renesas,r8a7793",
+	"renesas,r8a7794",
+	NULL,
+};
+
+DT_MACHINE_START(RCAR_GEN2_DT, "Generic R-Car Gen2 (Flattened Device Tree)")
+	.init_early	= shmobile_init_delay,
+	.init_late	= shmobile_init_late,
+	.init_time	= rcar_gen2_timer_init,
+	.reserve	= rcar_gen2_reserve,
+	.dt_compat	= rcar_gen2_boards_compat_dt,
+MACHINE_END
+
+static const char * const rz_g1_boards_compat_dt[] __initconst = {
+	"renesas,r8a7743",
+	"renesas,r8a7745",
+	NULL,
+};
+
+DT_MACHINE_START(RZ_G1_DT, "Generic RZ/G1 (Flattened Device Tree)")
+	.init_early	= shmobile_init_delay,
+	.init_late	= shmobile_init_late,
+	.init_time	= rcar_gen2_timer_init,
+	.reserve	= rcar_gen2_reserve,
+	.dt_compat	= rz_g1_boards_compat_dt,
+MACHINE_END
diff --git a/arch/arm/mach-socfpga/l2_cache.c b/arch/arm/mach-socfpga/l2_cache.c
index 4267c95f2158..bb359d727b34 100644
--- a/arch/arm/mach-socfpga/l2_cache.c
+++ b/arch/arm/mach-socfpga/l2_cache.c
@@ -74,7 +74,7 @@ void socfpga_init_arria10_l2_ecc(void)
 	}
 
 	if (!sys_manager_base_addr) {
-		pr_err("System Mananger not mapped for L2 ECC\n");
+		pr_err("System Manager not mapped for L2 ECC\n");
 		goto exit;
 	}
 	/* Clear any pending IRQs */
diff --git a/arch/arm/mach-spear/time.c b/arch/arm/mach-spear/time.c
index 9ccffc1d0f28..4878ba90026d 100644
--- a/arch/arm/mach-spear/time.c
+++ b/arch/arm/mach-spear/time.c
@@ -233,7 +233,7 @@ void __init spear_setup_of_timer(void)
 	}
 
 	gpt_clk = clk_get_sys("gpt0", NULL);
-	if (!gpt_clk) {
+	if (IS_ERR(gpt_clk)) {
 		pr_err("%s:couldn't get clk for gpt\n", __func__);
 		goto err_iomap;
 	}
diff --git a/arch/arm/mach-sti/Kconfig b/arch/arm/mach-sti/Kconfig
index 119e1108b1f8..f8eeeffddaff 100644
--- a/arch/arm/mach-sti/Kconfig
+++ b/arch/arm/mach-sti/Kconfig
@@ -28,7 +28,6 @@ if ARCH_STI
 config SOC_STIH415
 	bool "STiH415 STMicroelectronics Consumer Electronics family"
 	default y
-	select STIH415_RESET
 	help
 	  This enables support for STMicroelectronics Digital Consumer
 	  Electronics family StiH415 parts, primarily targeted at set-top-box
@@ -38,7 +37,6 @@ config SOC_STIH415
 config SOC_STIH416
 	bool "STiH416 STMicroelectronics Consumer Electronics family"
 	default y
-	select STIH416_RESET
 	help
 	  This enables support for STMicroelectronics Digital Consumer
 	  Electronics family StiH416 parts, primarily targeted at set-top-box
diff --git a/arch/arm/mach-stm32/board-dt.c b/arch/arm/mach-stm32/board-dt.c
index ceee47735eec..c354222a4158 100644
--- a/arch/arm/mach-stm32/board-dt.c
+++ b/arch/arm/mach-stm32/board-dt.c
@@ -11,6 +11,7 @@
 static const char *const stm32_compat[] __initconst = {
 	"st,stm32f429",
 	"st,stm32f469",
+	"st,stm32f746",
 	NULL
 };
 
diff --git a/arch/arm/mach-vexpress/platsmp.c b/arch/arm/mach-vexpress/platsmp.c
index 8b8d0724f6c6..98e29dee91e8 100644
--- a/arch/arm/mach-vexpress/platsmp.c
+++ b/arch/arm/mach-vexpress/platsmp.c
@@ -26,19 +26,37 @@
 bool __init vexpress_smp_init_ops(void)
 {
 #ifdef CONFIG_MCPM
+	int cpu;
+	struct device_node *cpu_node, *cci_node;
+
 	/*
-	 * The best way to detect a multi-cluster configuration at the moment
-	 * is to look for the presence of a CCI in the system.
+	 * The best way to detect a multi-cluster configuration
+	 * is to detect if the kernel can take over CCI ports
+	 * control. Loop over possible CPUs and check if CCI
+	 * port control is available.
 	 * Override the default vexpress_smp_ops if so.
 	 */
-	struct device_node *node;
-	node = of_find_compatible_node(NULL, NULL, "arm,cci-400");
-	if (node && of_device_is_available(node)) {
-		mcpm_smp_set_ops();
-		return true;
+	for_each_possible_cpu(cpu) {
+		bool available;
+
+		cpu_node = of_get_cpu_node(cpu, NULL);
+		if (WARN(!cpu_node, "Missing cpu device node!"))
+			return false;
+
+		cci_node = of_parse_phandle(cpu_node, "cci-control-port", 0);
+		available = cci_node && of_device_is_available(cci_node);
+		of_node_put(cci_node);
+		of_node_put(cpu_node);
+
+		if (!available)
+			return false;
 	}
-#endif
+
+	mcpm_smp_set_ops();
+	return true;
+#else
 	return false;
+#endif
 }
 
 static const struct of_device_id vexpress_smp_dt_scu_match[] __initconst = {
diff --git a/arch/arm/mach-zx/zx296702-pm-domain.c b/arch/arm/mach-zx/zx296702-pm-domain.c
index e08574d4e2ca..79dcf2549267 100644
--- a/arch/arm/mach-zx/zx296702-pm-domain.c
+++ b/arch/arm/mach-zx/zx296702-pm-domain.c
@@ -169,7 +169,7 @@ static int zx296702_pd_probe(struct platform_device *pdev)
 	}
 
 	pcubase = devm_ioremap_resource(&pdev->dev, res);
-	if (!pcubase) {
+	if (IS_ERR(pcubase)) {
 		dev_err(&pdev->dev, "ioremap fail.\n");
 		return -EIO;
 	}
diff --git a/arch/arm/mach-zynq/common.c b/arch/arm/mach-zynq/common.c
index d12002cd63bc..ed118648313f 100644
--- a/arch/arm/mach-zynq/common.c
+++ b/arch/arm/mach-zynq/common.c
@@ -59,7 +59,7 @@ void __iomem *zynq_scu_base;
 static void __init zynq_memory_init(void)
 {
 	if (!__pa(PAGE_OFFSET))
-		memblock_reserve(__pa(PAGE_OFFSET), __pa(swapper_pg_dir));
+		memblock_reserve(__pa(PAGE_OFFSET), 0x80000);
 }
 
 static struct platform_device zynq_cpuidle_device = {
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index c1799dd1d0d9..f68e8ec29447 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -991,7 +991,7 @@ config CACHE_TAUROS2
 config CACHE_UNIPHIER
 	bool "Enable the UniPhier outer cache controller"
 	depends on ARCH_UNIPHIER
-	default y
+	select ARM_L1_CACHE_SHIFT_7
 	select OUTER_CACHE
 	select OUTER_CACHE_SYNC
 	help
@@ -1012,8 +1012,14 @@ config ARM_L1_CACHE_SHIFT_6
 	help
 	  Setting ARM L1 cache line size to 64 Bytes.
 
+config ARM_L1_CACHE_SHIFT_7
+	bool
+	help
+	  Setting ARM L1 cache line size to 128 Bytes.
+
 config ARM_L1_CACHE_SHIFT
 	int
+	default 7 if ARM_L1_CACHE_SHIFT_7
 	default 6 if ARM_L1_CACHE_SHIFT_6
 	default 5
 
diff --git a/arch/arm/mm/pageattr.c b/arch/arm/mm/pageattr.c
index d19b1ad29b07..3b69f2642513 100644
--- a/arch/arm/mm/pageattr.c
+++ b/arch/arm/mm/pageattr.c
@@ -34,28 +34,29 @@ static int change_page_range(pte_t *ptep, pgtable_t token, unsigned long addr,
 	return 0;
 }
 
+static bool in_range(unsigned long start, unsigned long size,
+	unsigned long range_start, unsigned long range_end)
+{
+	return start >= range_start && start < range_end &&
+		size <= range_end - start;
+}
+
 static int change_memory_common(unsigned long addr, int numpages,
 				pgprot_t set_mask, pgprot_t clear_mask)
 {
-	unsigned long start = addr;
-	unsigned long size = PAGE_SIZE*numpages;
-	unsigned long end = start + size;
+	unsigned long start = addr & PAGE_MASK;
+	unsigned long end = PAGE_ALIGN(addr) + numpages * PAGE_SIZE;
+	unsigned long size = end - start;
 	int ret;
 	struct page_change_data data;
 
-	if (!IS_ALIGNED(addr, PAGE_SIZE)) {
-		start &= PAGE_MASK;
-		end = start + size;
-		WARN_ON_ONCE(1);
-	}
+	WARN_ON_ONCE(start != addr);
 
-	if (!numpages)
+	if (!size)
 		return 0;
 
-	if (start < MODULES_VADDR || start >= MODULES_END)
-		return -EINVAL;
-
-	if (end < MODULES_VADDR || start >= MODULES_END)
+	if (!in_range(start, size, MODULES_VADDR, MODULES_END) &&
+	    !in_range(start, size, VMALLOC_START, VMALLOC_END))
 		return -EINVAL;
 
 	data.set_mask = set_mask;
diff --git a/arch/arm/plat-omap/Kconfig b/arch/arm/plat-omap/Kconfig
index d055db32ffcb..3e27bffb352d 100644
--- a/arch/arm/plat-omap/Kconfig
+++ b/arch/arm/plat-omap/Kconfig
@@ -63,32 +63,6 @@ config OMAP_RESET_CLOCKS
 	  probably do not want this option enabled until your
 	  device drivers work properly.
 
-config OMAP_MUX
-	bool "OMAP multiplexing support"
-	depends on ARCH_OMAP
-	default y
-	help
-	  Pin multiplexing support for OMAP boards. If your bootloader
-	  sets the multiplexing correctly, say N. Otherwise, or if unsure,
-	  say Y.
-
-config OMAP_MUX_DEBUG
-	bool "Multiplexing debug output"
-	depends on OMAP_MUX
-	help
-	  Makes the multiplexing functions print out a lot of debug info.
-	  This is useful if you want to find out the correct values of the
-	  multiplexing registers.
-
-config OMAP_MUX_WARNINGS
-	bool "Warn about pins the bootloader didn't set up"
-	depends on OMAP_MUX
-	default y
-	help
-	  Choose Y here to warn whenever driver initialization logic needs
-	  to change the pin multiplexing setup.	 When there are no warnings
-	  printed, it's safe to deselect OMAP_MUX for your product.
-
 config OMAP_MPU_TIMER
 	bool "Use mpu timer"
 	depends on ARCH_OMAP1
diff --git a/arch/arm/plat-omap/Makefile b/arch/arm/plat-omap/Makefile
index 97a50e8883f9..47e186729d44 100644
--- a/arch/arm/plat-omap/Makefile
+++ b/arch/arm/plat-omap/Makefile
@@ -11,6 +11,3 @@ obj-y := sram.o dma.o counter_32k.o
 
 obj-$(CONFIG_OMAP_DM_TIMER) += dmtimer.o
 obj-$(CONFIG_OMAP_DEBUG_LEDS) += debug-leds.o
-i2c-omap-$(CONFIG_I2C_OMAP) := i2c.o
-obj-y += $(i2c-omap-m) $(i2c-omap-y)
-
diff --git a/arch/arm/plat-omap/i2c.c b/arch/arm/plat-omap/i2c.c
deleted file mode 100644
index 58213d9714cd..000000000000
--- a/arch/arm/plat-omap/i2c.c
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * linux/arch/arm/plat-omap/i2c.c
- *
- * Helper module for board specific I2C bus registration
- *
- * Copyright (C) 2007 Nokia Corporation.
- *
- * Contact: Jarkko Nikula <jhnikula@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA
- *
- */
-
-#include <linux/kernel.h>
-#include <linux/platform_device.h>
-#include <linux/i2c.h>
-#include <linux/i2c-omap.h>
-#include <linux/slab.h>
-#include <linux/err.h>
-#include <linux/clk.h>
-
-#include <plat/i2c.h>
-
-#define OMAP_I2C_MAX_CONTROLLERS 4
-static struct omap_i2c_bus_platform_data i2c_pdata[OMAP_I2C_MAX_CONTROLLERS];
-
-#define OMAP_I2C_CMDLINE_SETUP	(BIT(31))
-
-/**
- * omap_i2c_bus_setup - Process command line options for the I2C bus speed
- * @str: String of options
- *
- * This function allow to override the default I2C bus speed for given I2C
- * bus with a command line option.
- *
- * Format: i2c_bus=bus_id,clkrate (in kHz)
- *
- * Returns 1 on success, 0 otherwise.
- */
-static int __init omap_i2c_bus_setup(char *str)
-{
-	int ints[3];
-
-	get_options(str, 3, ints);
-	if (ints[0] < 2 || ints[1] < 1 ||
-			ints[1] > OMAP_I2C_MAX_CONTROLLERS)
-		return 0;
-	i2c_pdata[ints[1] - 1].clkrate = ints[2];
-	i2c_pdata[ints[1] - 1].clkrate |= OMAP_I2C_CMDLINE_SETUP;
-
-	return 1;
-}
-__setup("i2c_bus=", omap_i2c_bus_setup);
-
-/*
- * Register busses defined in command line but that are not registered with
- * omap_register_i2c_bus from board initialization code.
- */
-int __init omap_register_i2c_bus_cmdline(void)
-{
-	int i, err = 0;
-
-	for (i = 0; i < ARRAY_SIZE(i2c_pdata); i++)
-		if (i2c_pdata[i].clkrate & OMAP_I2C_CMDLINE_SETUP) {
-			i2c_pdata[i].clkrate &= ~OMAP_I2C_CMDLINE_SETUP;
-			err = omap_i2c_add_bus(&i2c_pdata[i], i + 1);
-			if (err)
-				goto out;
-		}
-
-out:
-	return err;
-}
-
-/**
- * omap_register_i2c_bus - register I2C bus with device descriptors
- * @bus_id: bus id counting from number 1
- * @clkrate: clock rate of the bus in kHz
- * @info: pointer into I2C device descriptor table or NULL
- * @len: number of descriptors in the table
- *
- * Returns 0 on success or an error code.
- */
-int __init omap_register_i2c_bus(int bus_id, u32 clkrate,
-			  struct i2c_board_info const *info,
-			  unsigned len)
-{
-	int err;
-
-	BUG_ON(bus_id < 1 || bus_id > OMAP_I2C_MAX_CONTROLLERS);
-
-	if (info) {
-		err = i2c_register_board_info(bus_id, info, len);
-		if (err)
-			return err;
-	}
-
-	if (!i2c_pdata[bus_id - 1].clkrate)
-		i2c_pdata[bus_id - 1].clkrate = clkrate;
-
-	i2c_pdata[bus_id - 1].clkrate &= ~OMAP_I2C_CMDLINE_SETUP;
-
-	return omap_i2c_add_bus(&i2c_pdata[bus_id - 1], bus_id);
-}
diff --git a/arch/arm/plat-orion/gpio.c b/arch/arm/plat-orion/gpio.c
index f74069386c13..26a531ebb6e9 100644
--- a/arch/arm/plat-orion/gpio.c
+++ b/arch/arm/plat-orion/gpio.c
@@ -478,13 +478,13 @@ static void orion_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 			   (data_in ^ in_pol) & msk  ? "hi" : "lo",
 			   in_pol & msk ? "lo" : "hi");
 		if (!((edg_msk | lvl_msk) & msk)) {
-			seq_printf(s, " disabled\n");
+			seq_puts(s, " disabled\n");
 			continue;
 		}
 		if (edg_msk & msk)
-			seq_printf(s, " edge ");
+			seq_puts(s, " edge ");
 		if (lvl_msk & msk)
-			seq_printf(s, " level");
+			seq_puts(s, " level");
 		seq_printf(s, " (%s)\n", cause & msk ? "pending" : "clear  ");
 	}
 }
diff --git a/arch/arm/plat-samsung/devs.c b/arch/arm/plat-samsung/devs.c
index e93aa6734147..cf7b95fddbb3 100644
--- a/arch/arm/plat-samsung/devs.c
+++ b/arch/arm/plat-samsung/devs.c
@@ -1124,15 +1124,6 @@ void __init s3c64xx_spi0_set_platdata(int (*cfg_gpio)(void), int src_clk_nr,
 	pd.num_cs = num_cs;
 	pd.src_clk_nr = src_clk_nr;
 	pd.cfg_gpio = (cfg_gpio) ? cfg_gpio : s3c64xx_spi0_cfg_gpio;
-	pd.dma_tx = (void *)DMACH_SPI0_TX;
-	pd.dma_rx = (void *)DMACH_SPI0_RX;
-#if defined(CONFIG_PL330_DMA)
-	pd.filter = pl330_filter;
-#elif defined(CONFIG_S3C64XX_PL080)
-	pd.filter = pl08x_filter_id;
-#elif defined(CONFIG_S3C24XX_DMAC)
-	pd.filter = s3c24xx_dma_filter;
-#endif
 
 	s3c_set_platdata(&pd, sizeof(pd), &s3c64xx_device_spi0);
 }
@@ -1169,14 +1160,6 @@ void __init s3c64xx_spi1_set_platdata(int (*cfg_gpio)(void), int src_clk_nr,
 	pd.num_cs = num_cs;
 	pd.src_clk_nr = src_clk_nr;
 	pd.cfg_gpio = (cfg_gpio) ? cfg_gpio : s3c64xx_spi1_cfg_gpio;
-	pd.dma_tx = (void *)DMACH_SPI1_TX;
-	pd.dma_rx = (void *)DMACH_SPI1_RX;
-#if defined(CONFIG_PL330_DMA)
-	pd.filter = pl330_filter;
-#elif defined(CONFIG_S3C64XX_PL080)
-	pd.filter = pl08x_filter_id;
-#endif
-
 
 	s3c_set_platdata(&pd, sizeof(pd), &s3c64xx_device_spi1);
 }
@@ -1213,13 +1196,6 @@ void __init s3c64xx_spi2_set_platdata(int (*cfg_gpio)(void), int src_clk_nr,
 	pd.num_cs = num_cs;
 	pd.src_clk_nr = src_clk_nr;
 	pd.cfg_gpio = (cfg_gpio) ? cfg_gpio : s3c64xx_spi2_cfg_gpio;
-	pd.dma_tx = (void *)DMACH_SPI2_TX;
-	pd.dma_rx = (void *)DMACH_SPI2_RX;
-#if defined(CONFIG_PL330_DMA)
-	pd.filter = pl330_filter;
-#elif defined(CONFIG_S3C64XX_PL080)
-	pd.filter = pl08x_filter_id;
-#endif
 
 	s3c_set_platdata(&pd, sizeof(pd), &s3c64xx_device_spi2);
 }
diff --git a/arch/arm/plat-samsung/include/plat/gpio-cfg.h b/arch/arm/plat-samsung/include/plat/gpio-cfg.h
index 21391faab068..e55d1f597db8 100644
--- a/arch/arm/plat-samsung/include/plat/gpio-cfg.h
+++ b/arch/arm/plat-samsung/include/plat/gpio-cfg.h
@@ -26,7 +26,7 @@
 
 #include <linux/types.h>
 
-typedef unsigned int __bitwise__ samsung_gpio_pull_t;
+typedef unsigned int __bitwise samsung_gpio_pull_t;
 
 /* forward declaration if gpio-core.h hasn't been included */
 struct samsung_gpio_chip;
diff --git a/arch/arm/tools/Makefile b/arch/arm/tools/Makefile
index 6e4cd1867a9f..92eb5c3b486c 100644
--- a/arch/arm/tools/Makefile
+++ b/arch/arm/tools/Makefile
@@ -4,10 +4,76 @@
 # Copyright (C) 2001 Russell King
 #
 
+gen := arch/$(ARCH)/include/generated
+kapi := $(gen)/asm
+uapi := $(gen)/uapi/asm
+syshdr := $(srctree)/$(src)/syscallhdr.sh
+sysnr := $(srctree)/$(src)/syscallnr.sh
+systbl := $(srctree)/$(src)/syscalltbl.sh
+syscall := $(srctree)/$(src)/syscall.tbl
+
+gen-y := $(gen)/calls-oabi.S
+gen-y += $(gen)/calls-eabi.S
+kapi-hdrs-y := $(kapi)/unistd-nr.h
+kapi-hdrs-y += $(kapi)/mach-types.h
+uapi-hdrs-y := $(uapi)/unistd-common.h
+uapi-hdrs-y += $(uapi)/unistd-oabi.h
+uapi-hdrs-y += $(uapi)/unistd-eabi.h
+
+targets += $(addprefix ../../../,$(gen-y) $(kapi-hdrs-y) $(uapi-hdrs-y))
+
+PHONY += kapi uapi
+
+kapi:	$(kapi-hdrs-y) $(gen-y)
+
+uapi:	$(uapi-hdrs-y)
+
+# Create output directory if not already present
+_dummy := $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') \
+          $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)')
+
 quiet_cmd_gen_mach = GEN     $@
       cmd_gen_mach = mkdir -p $(dir $@) && \
 		     $(AWK) -f $(filter-out $(PHONY),$^) > $@ || \
 		     { rm -f $@; /bin/false; }
 
-include/generated/mach-types.h: $(src)/gen-mach-types $(src)/mach-types FORCE
+$(kapi)/mach-types.h: $(src)/gen-mach-types $(src)/mach-types FORCE
 	$(call if_changed,gen_mach)
+
+quiet_cmd_syshdr = SYSHDR  $@
+      cmd_syshdr = $(CONFIG_SHELL) '$(syshdr)' '$<' '$@' \
+		   '$(syshdr_abi_$(basetarget))' \
+		   '$(syshdr_pfx_$(basetarget))' \
+		   '__NR_SYSCALL_BASE'
+
+quiet_cmd_systbl = SYSTBL  $@
+      cmd_systbl = $(CONFIG_SHELL) '$(systbl)' '$<' '$@' \
+		   '$(systbl_abi_$(basetarget))'
+
+quiet_cmd_sysnr  = SYSNR   $@
+      cmd_sysnr  = $(CONFIG_SHELL) '$(sysnr)' '$<' '$@' \
+		   '$(syshdr_abi_$(basetarget))'
+
+syshdr_abi_unistd-common := common
+$(uapi)/unistd-common.h: $(syscall) $(syshdr) FORCE
+	$(call if_changed,syshdr)
+
+syshdr_abi_unistd-oabi := oabi
+$(uapi)/unistd-oabi.h: $(syscall) $(syshdr) FORCE
+	$(call if_changed,syshdr)
+
+syshdr_abi_unistd-eabi := eabi
+$(uapi)/unistd-eabi.h: $(syscall) $(syshdr) FORCE
+	$(call if_changed,syshdr)
+
+sysnr_abi_unistd-nr := common,oabi,eabi,compat
+$(kapi)/unistd-nr.h: $(syscall) $(sysnr) FORCE
+	$(call if_changed,sysnr)
+
+systbl_abi_calls-oabi := common,oabi
+$(gen)/calls-oabi.S: $(syscall) $(systbl) FORCE
+	$(call if_changed,systbl)
+
+systbl_abi_calls-eabi := common,eabi
+$(gen)/calls-eabi.S: $(syscall) $(systbl) FORCE
+	$(call if_changed,systbl)
diff --git a/arch/arm/tools/mach-types b/arch/arm/tools/mach-types
index 2ed1b8a922ed..a9313b66f770 100644
--- a/arch/arm/tools/mach-types
+++ b/arch/arm/tools/mach-types
@@ -16,7 +16,7 @@
 # are merged into mainline or have been edited in the machine database
 # within the last 12 months.  References to machine_is_NAME() do not count!
 #
-# Last update: Fri Mar 22 17:24:50 2013
+# Last update: Sun Oct 30 20:21:01 2016
 #
 # machine_is_xxx	CONFIG_xxxx		MACH_TYPE_xxx		number
 #
@@ -152,7 +152,6 @@ colibri			MACH_COLIBRI		COLIBRI			729
 gateway7001		MACH_GATEWAY7001	GATEWAY7001		731
 pcm027			MACH_PCM027		PCM027			732
 anubis			MACH_ANUBIS		ANUBIS			734
-xboardgp8		MACH_XBOARDGP8		XBOARDGP8		742
 akita			MACH_AKITA		AKITA			744
 e330			MACH_E330		E330			753
 nokia770		MACH_NOKIA770		NOKIA770		755
@@ -393,7 +392,6 @@ anw6410			MACH_ANW6410		ANW6410			2183
 imx27_visstrim_m10	MACH_IMX27_VISSTRIM_M10	IMX27_VISSTRIM_M10	2187
 portuxg20		MACH_PORTUXG20		PORTUXG20		2191
 smdkc110		MACH_SMDKC110		SMDKC110		2193
-cabespresso		MACH_CABESPRESSO	CABESPRESSO		2194
 omap3517evm		MACH_OMAP3517EVM	OMAP3517EVM		2200
 netspace_v2		MACH_NETSPACE_V2	NETSPACE_V2		2201
 netspace_max_v2		MACH_NETSPACE_MAX_V2	NETSPACE_MAX_V2		2202
@@ -412,7 +410,6 @@ bigdisk			MACH_BIGDISK		BIGDISK			2283
 at91sam9g20ek_2mmc	MACH_AT91SAM9G20EK_2MMC	AT91SAM9G20EK_2MMC	2288
 bcmring			MACH_BCMRING		BCMRING			2289
 mahimahi		MACH_MAHIMAHI		MAHIMAHI		2304
-cerebric		MACH_CEREBRIC		CEREBRIC		2311
 smdk6442		MACH_SMDK6442		SMDK6442		2324
 openrd_base		MACH_OPENRD_BASE	OPENRD_BASE		2325
 devkit8000		MACH_DEVKIT8000		DEVKIT8000		2330
@@ -435,9 +432,7 @@ tnetv107x		MACH_TNETV107X		TNETV107X		2418
 smdkv210		MACH_SMDKV210		SMDKV210		2456
 omap_zoom3		MACH_OMAP_ZOOM3		OMAP_ZOOM3		2464
 omap_3630sdp		MACH_OMAP_3630SDP	OMAP_3630SDP		2465
-cybook2440		MACH_CYBOOK2440		CYBOOK2440		2466
 smartq7			MACH_SMARTQ7		SMARTQ7			2479
-watson_efm_plugin	MACH_WATSON_EFM_PLUGIN	WATSON_EFM_PLUGIN	2491
 g4evm			MACH_G4EVM		G4EVM			2493
 omapl138_hawkboard	MACH_OMAPL138_HAWKBOARD	OMAPL138_HAWKBOARD	2495
 ts41x			MACH_TS41X		TS41X			2502
@@ -472,7 +467,6 @@ igep0030		MACH_IGEP0030		IGEP0030		2717
 sbc3530			MACH_SBC3530		SBC3530			2722
 saarb			MACH_SAARB		SAARB			2727
 harmony			MACH_HARMONY		HARMONY			2731
-cybook_orizon		MACH_CYBOOK_ORIZON	CYBOOK_ORIZON		2733
 msm7x30_fluid		MACH_MSM7X30_FLUID	MSM7X30_FLUID		2741
 cm_t3517		MACH_CM_T3517		CM_T3517		2750
 wbd222			MACH_WBD222		WBD222			2753
@@ -490,6 +484,7 @@ eukrea_cpuimx51		MACH_EUKREA_CPUIMX51	EUKREA_CPUIMX51		2823
 smdkc210		MACH_SMDKC210		SMDKC210		2838
 t5325			MACH_T5325		T5325			2846
 income			MACH_INCOME		INCOME			2849
+meson			MACH_MESON		MESON			2853
 goni			MACH_GONI		GONI			2862
 bv07			MACH_BV07		BV07			2882
 openrd_ultimate		MACH_OPENRD_ULTIMATE	OPENRD_ULTIMATE		2884
@@ -523,9 +518,9 @@ prima2_evb		MACH_PRIMA2_EVB		PRIMA2_EVB		3103
 paz00			MACH_PAZ00		PAZ00			3128
 acmenetusfoxg20		MACH_ACMENETUSFOXG20	ACMENETUSFOXG20		3129
 ag5evm			MACH_AG5EVM		AG5EVM			3189
-ics_if_voip		MACH_ICS_IF_VOIP	ICS_IF_VOIP		3206
 wlf_cragg_6410		MACH_WLF_CRAGG_6410	WLF_CRAGG_6410		3207
 trimslice		MACH_TRIMSLICE		TRIMSLICE		3209
+mackerel		MACH_MACKEREL		MACKEREL		3211
 kaen			MACH_KAEN		KAEN			3217
 nokia_rm680		MACH_NOKIA_RM680	NOKIA_RM680		3220
 msm8960_sim		MACH_MSM8960_SIM	MSM8960_SIM		3230
@@ -540,469 +535,66 @@ snowball		MACH_SNOWBALL		SNOWBALL		3363
 xilinx_ep107		MACH_XILINX_EP107	XILINX_EP107		3378
 nuri			MACH_NURI		NURI			3379
 origen			MACH_ORIGEN		ORIGEN			3455
+xarina			MACH_XARINA		XARINA			3476
 nspire			MACH_NSPIRE		NSPIRE			3503
 nokia_rm696		MACH_NOKIA_RM696	NOKIA_RM696		3522
-mikrap_x168		MACH_MIKRAP_X168	MIKRAP_X168		3543
-deto_macarm9		MACH_DETO_MACARM9	DETO_MACARM9		3568
 m28evk			MACH_M28EVK		M28EVK			3613
 kota2			MACH_KOTA2		KOTA2			3616
 bonito			MACH_BONITO		BONITO			3623
-omap3_egf		MACH_OMAP3_EGF		OMAP3_EGF		3637
 smdk4212		MACH_SMDK4212		SMDK4212		3638
 apx4devkit		MACH_APX4DEVKIT		APX4DEVKIT		3712
 smdk4412		MACH_SMDK4412		SMDK4412		3765
 marzen			MACH_MARZEN		MARZEN			3790
-krome			MACH_KROME		KROME			3797
-armadillo800eva		MACH_ARMADILLO800EVA	ARMADILLO800EVA		3863
-mx53_umobo		MACH_MX53_UMOBO		MX53_UMOBO		3927
-mt4			MACH_MT4		MT4			3981
+empc_a500		MACH_EMPC_A500		EMPC_A500		3848
 u8520			MACH_U8520		U8520			3990
-chupacabra		MACH_CHUPACABRA		CHUPACABRA		4098
-scorpion		MACH_SCORPION		SCORPION		4099
-davinci_he_hmi10	MACH_DAVINCI_HE_HMI10	DAVINCI_HE_HMI10	4100
-topkick			MACH_TOPKICK		TOPKICK			4101
-m3_auguestrush		MACH_M3_AUGUESTRUSH	M3_AUGUESTRUSH		4102
-ipc335x			MACH_IPC335X		IPC335X			4103
-sun4i			MACH_SUN4I		SUN4I			4104
-imx233_olinuxino	MACH_IMX233_OLINUXINO	IMX233_OLINUXINO	4105
-k2_wl			MACH_K2_WL		K2_WL			4106
-k2_ul			MACH_K2_UL		K2_UL			4107
-k2_cl			MACH_K2_CL		K2_CL			4108
-minbari_w		MACH_MINBARI_W		MINBARI_W		4109
-minbari_m		MACH_MINBARI_M		MINBARI_M		4110
-k035			MACH_K035		K035			4111
-ariel			MACH_ARIEL		ARIEL			4112
-arielsaarc		MACH_ARIELSAARC		ARIELSAARC		4113
-arieldkb		MACH_ARIELDKB		ARIELDKB		4114
-armadillo810		MACH_ARMADILLO810	ARMADILLO810		4115
-tam335x			MACH_TAM335X		TAM335X			4116
-grouper			MACH_GROUPER		GROUPER			4117
-mpcsa21_9g20		MACH_MPCSA21_9G20	MPCSA21_9G20		4118
-m6u_cpu			MACH_M6U_CPU		M6U_CPU			4119
-ginkgo			MACH_GINKGO		GINKGO			4121
-cgt_qmx6		MACH_CGT_QMX6		CGT_QMX6		4122
-profpga			MACH_PROFPGA		PROFPGA			4123
-acfx100oc		MACH_ACFX100OC		ACFX100OC		4124
-acfx100nb		MACH_ACFX100NB		ACFX100NB		4125
-capricorn		MACH_CAPRICORN		CAPRICORN		4126
-pisces			MACH_PISCES		PISCES			4127
-aries			MACH_ARIES		ARIES			4128
-cancer			MACH_CANCER		CANCER			4129
-leo			MACH_LEO		LEO			4130
-virgo			MACH_VIRGO		VIRGO			4131
-sagittarius		MACH_SAGITTARIUS	SAGITTARIUS		4132
-devil			MACH_DEVIL		DEVIL			4133
-ballantines		MACH_BALLANTINES	BALLANTINES		4134
-omap3_procerusvpu	MACH_OMAP3_PROCERUSVPU	OMAP3_PROCERUSVPU	4135
-my27			MACH_MY27		MY27			4136
-sun6i			MACH_SUN6I		SUN6I			4137
-sun5i			MACH_SUN5I		SUN5I			4138
-mx512_mx		MACH_MX512_MX		MX512_MX		4139
-kzm9g			MACH_KZM9G		KZM9G			4140
-vdstbn			MACH_VDSTBN		VDSTBN			4141
-cfa10036		MACH_CFA10036		CFA10036		4142
-cfa10049		MACH_CFA10049		CFA10049		4143
-pcm051			MACH_PCM051		PCM051			4144
-vybrid_vf7xx		MACH_VYBRID_VF7XX	VYBRID_VF7XX		4145
-vybrid_vf6xx		MACH_VYBRID_VF6XX	VYBRID_VF6XX		4146
-vybrid_vf5xx		MACH_VYBRID_VF5XX	VYBRID_VF5XX		4147
-vybrid_vf4xx		MACH_VYBRID_VF4XX	VYBRID_VF4XX		4148
-aria_g25		MACH_ARIA_G25		ARIA_G25		4149
-bcm21553		MACH_BCM21553		BCM21553		4150
-smdk5410		MACH_SMDK5410		SMDK5410		4151
-lpc18xx			MACH_LPC18XX		LPC18XX			4152
-oratisparty		MACH_ORATISPARTY	ORATISPARTY		4153
-qseven			MACH_QSEVEN		QSEVEN			4154
-gmv_generic		MACH_GMV_GENERIC	GMV_GENERIC		4155
-th_link_eth		MACH_TH_LINK_ETH	TH_LINK_ETH		4156
-tn_muninn		MACH_TN_MUNINN		TN_MUNINN		4157
-rampage			MACH_RAMPAGE		RAMPAGE			4158
-visstrim_mv10		MACH_VISSTRIM_MV10	VISSTRIM_MV10		4159
-mx28_wilma		MACH_MX28_WILMA		MX28_WILMA		4164
-msm8625_ffa		MACH_MSM8625_FFA	MSM8625_FFA		4166
-vpu101			MACH_VPU101		VPU101			4167
-baileys			MACH_BAILEYS		BAILEYS			4169
-familybox		MACH_FAMILYBOX		FAMILYBOX		4170
-ensemble_mx35		MACH_ENSEMBLE_MX35	ENSEMBLE_MX35		4171
-sc_sps_1		MACH_SC_SPS_1		SC_SPS_1		4172
-ucsimply_sam9260	MACH_UCSIMPLY_SAM9260	UCSIMPLY_SAM9260	4173
-unicorn			MACH_UNICORN		UNICORN			4174
-m9g45a			MACH_M9G45A		M9G45A			4175
-mtwebif			MACH_MTWEBIF		MTWEBIF			4176
-playstone		MACH_PLAYSTONE		PLAYSTONE		4177
-chelsea			MACH_CHELSEA		CHELSEA			4178
-bayern			MACH_BAYERN		BAYERN			4179
-mitwo			MACH_MITWO		MITWO			4180
-mx25_noah		MACH_MX25_NOAH		MX25_NOAH		4181
-stm_b2020		MACH_STM_B2020		STM_B2020		4182
-annax_src		MACH_ANNAX_SRC		ANNAX_SRC		4183
-ionics_stratus		MACH_IONICS_STRATUS	IONICS_STRATUS		4184
-hugo			MACH_HUGO		HUGO			4185
-em300			MACH_EM300		EM300			4186
-mmp3_qseven		MACH_MMP3_QSEVEN	MMP3_QSEVEN		4187
-bosphorus2		MACH_BOSPHORUS2		BOSPHORUS2		4188
-tt2200			MACH_TT2200		TT2200			4189
-ocelot3			MACH_OCELOT3		OCELOT3			4190
-tek_cobra		MACH_TEK_COBRA		TEK_COBRA		4191
-protou			MACH_PROTOU		PROTOU			4192
-msm8625_evt		MACH_MSM8625_EVT	MSM8625_EVT		4193
-mx53_sellwood		MACH_MX53_SELLWOOD	MX53_SELLWOOD		4194
-somiq_am35		MACH_SOMIQ_AM35		SOMIQ_AM35		4195
-somiq_am37		MACH_SOMIQ_AM37		SOMIQ_AM37		4196
-k2_plc_cl		MACH_K2_PLC_CL		K2_PLC_CL		4197
-tc2			MACH_TC2		TC2			4198
-dulex_j			MACH_DULEX_J		DULEX_J			4199
-stm_b2044		MACH_STM_B2044		STM_B2044		4200
-deluxe_j		MACH_DELUXE_J		DELUXE_J		4201
-mango2443		MACH_MANGO2443		MANGO2443		4202
-cp2dcg			MACH_CP2DCG		CP2DCG			4203
-cp2dtg			MACH_CP2DTG		CP2DTG			4204
-cp2dug			MACH_CP2DUG		CP2DUG			4205
-var_som_am33		MACH_VAR_SOM_AM33	VAR_SOM_AM33		4206
-pepper			MACH_PEPPER		PEPPER			4207
-mango2450		MACH_MANGO2450		MANGO2450		4208
-valente_wx_c9		MACH_VALENTE_WX_C9	VALENTE_WX_C9		4209
-minitv			MACH_MINITV		MINITV			4210
-u8540			MACH_U8540		U8540			4211
-iv_atlas_i_z7e		MACH_IV_ATLAS_I_Z7E	IV_ATLAS_I_Z7E		4212
-mach_type_sky		MACH_MACH_TYPE_SKY	MACH_TYPE_SKY		4214
-bluesky			MACH_BLUESKY		BLUESKY			4215
-ngrouter		MACH_NGROUTER		NGROUTER		4216
-mx53_denetim		MACH_MX53_DENETIM	MX53_DENETIM		4217
-opal			MACH_OPAL		OPAL			4218
-gnet_us3gref		MACH_GNET_US3GREF	GNET_US3GREF		4219
-gnet_nc3g		MACH_GNET_NC3G		GNET_NC3G		4220
-gnet_ge3g		MACH_GNET_GE3G		GNET_GE3G		4221
-adp2			MACH_ADP2		ADP2			4222
-tqma28			MACH_TQMA28		TQMA28			4223
-kacom3			MACH_KACOM3		KACOM3			4224
-rrhdemo			MACH_RRHDEMO		RRHDEMO			4225
-protodug		MACH_PROTODUG		PROTODUG		4226
-lago			MACH_LAGO		LAGO			4227
-ktt30			MACH_KTT30		KTT30			4228
-ts43xx			MACH_TS43XX		TS43XX			4229
-mx6q_denso		MACH_MX6Q_DENSO		MX6Q_DENSO		4230
-comsat_gsmumts8		MACH_COMSAT_GSMUMTS8	COMSAT_GSMUMTS8		4231
-dreamx			MACH_DREAMX		DREAMX			4232
-thunderstonem		MACH_THUNDERSTONEM	THUNDERSTONEM		4233
-yoyopad			MACH_YOYOPAD		YOYOPAD			4234
-yoyopatient		MACH_YOYOPATIENT	YOYOPATIENT		4235
-a10l			MACH_A10L		A10L			4236
-mq60			MACH_MQ60		MQ60			4237
-linkstation_lsql	MACH_LINKSTATION_LSQL	LINKSTATION_LSQL	4238
-am3703gateway		MACH_AM3703GATEWAY	AM3703GATEWAY		4239
-accipiter		MACH_ACCIPITER		ACCIPITER		4240
-magnidug		MACH_MAGNIDUG		MAGNIDUG		4242
-hydra			MACH_HYDRA		HYDRA			4243
-sun3i			MACH_SUN3I		SUN3I			4244
-stm_b2078		MACH_STM_B2078		STM_B2078		4245
-at91sam9263deskv2	MACH_AT91SAM9263DESKV2	AT91SAM9263DESKV2	4246
-deluxe_r		MACH_DELUXE_R		DELUXE_R		4247
-p_98_v			MACH_P_98_V		P_98_V			4248
-p_98_c			MACH_P_98_C		P_98_C			4249
-davinci_am18xx_omn	MACH_DAVINCI_AM18XX_OMN	DAVINCI_AM18XX_OMN	4250
-socfpga_cyclone5	MACH_SOCFPGA_CYCLONE5	SOCFPGA_CYCLONE5	4251
-cabatuin		MACH_CABATUIN		CABATUIN		4252
-yoyopad_ft		MACH_YOYOPAD_FT		YOYOPAD_FT		4253
-dan2400evb		MACH_DAN2400EVB		DAN2400EVB		4254
-dan3400evb		MACH_DAN3400EVB		DAN3400EVB		4255
-edm_sf_imx6		MACH_EDM_SF_IMX6	EDM_SF_IMX6		4256
-edm_cf_imx6		MACH_EDM_CF_IMX6	EDM_CF_IMX6		4257
-vpos3xx			MACH_VPOS3XX		VPOS3XX			4258
-vulcano_9x5		MACH_VULCANO_9X5	VULCANO_9X5		4259
-spmp8000		MACH_SPMP8000		SPMP8000		4260
-catalina		MACH_CATALINA		CATALINA		4261
-rd88f5181l_fe		MACH_RD88F5181L_FE	RD88F5181L_FE		4262
-mx535_mx		MACH_MX535_MX		MX535_MX		4263
-armadillo840		MACH_ARMADILLO840	ARMADILLO840		4264
-spc9000baseboard	MACH_SPC9000BASEBOARD	SPC9000BASEBOARD	4265
-iris			MACH_IRIS		IRIS			4266
-protodcg		MACH_PROTODCG		PROTODCG		4267
-palmtree		MACH_PALMTREE		PALMTREE		4268
-novena			MACH_NOVENA		NOVENA			4269
-ma_um			MACH_MA_UM		MA_UM			4270
-ma_am			MACH_MA_AM		MA_AM			4271
-ems348			MACH_EMS348		EMS348			4272
-cm_fx6			MACH_CM_FX6		CM_FX6			4273
-arndale			MACH_ARNDALE		ARNDALE			4274
-q5xr5			MACH_Q5XR5		Q5XR5			4275
-willow			MACH_WILLOW		WILLOW			4276
-omap3621_odyv3		MACH_OMAP3621_ODYV3	OMAP3621_ODYV3		4277
-omapl138_presonus	MACH_OMAPL138_PRESONUS	OMAPL138_PRESONUS	4278
-dvf99			MACH_DVF99		DVF99			4279
-impression_j		MACH_IMPRESSION_J	IMPRESSION_J		4280
-qblissa9		MACH_QBLISSA9		QBLISSA9		4281
-robin_heliview10	MACH_ROBIN_HELIVIEW10	ROBIN_HELIVIEW10	4282
-sun7i			MACH_SUN7I		SUN7I			4283
-mx6q_hdmidongle		MACH_MX6Q_HDMIDONGLE	MX6Q_HDMIDONGLE		4284
-mx6_sid2		MACH_MX6_SID2		MX6_SID2		4285
-helios_v3		MACH_HELIOS_V3		HELIOS_V3		4286
-helios_v4		MACH_HELIOS_V4		HELIOS_V4		4287
-q7_imx6			MACH_Q7_IMX6		Q7_IMX6			4288
-odroidx			MACH_ODROIDX		ODROIDX			4289
-robpro			MACH_ROBPRO		ROBPRO			4290
-research59if_mk1	MACH_RESEARCH59IF_MK1	RESEARCH59IF_MK1	4291
-bobsleigh		MACH_BOBSLEIGH		BOBSLEIGH		4292
-dcshgwt3		MACH_DCSHGWT3		DCSHGWT3		4293
-gld1018			MACH_GLD1018		GLD1018			4294
-ev10			MACH_EV10		EV10			4295
-nitrogen6x		MACH_NITROGEN6X		NITROGEN6X		4296
-p_107_bb		MACH_P_107_BB		P_107_BB		4297
-evita_utl		MACH_EVITA_UTL		EVITA_UTL		4298
-falconwing		MACH_FALCONWING		FALCONWING		4299
-dct3			MACH_DCT3		DCT3			4300
-cpx2e_cell		MACH_CPX2E_CELL		CPX2E_CELL		4301
-amiro			MACH_AMIRO		AMIRO			4302
-mx6q_brassboard		MACH_MX6Q_BRASSBOARD	MX6Q_BRASSBOARD		4303
-dalmore			MACH_DALMORE		DALMORE			4304
-omap3_portal7cp		MACH_OMAP3_PORTAL7CP	OMAP3_PORTAL7CP		4305
-tegra_pluto		MACH_TEGRA_PLUTO	TEGRA_PLUTO		4306
-mx6sl_evk		MACH_MX6SL_EVK		MX6SL_EVK		4307
-m7			MACH_M7			M7			4308
-pxm2			MACH_PXM2		PXM2			4309
-haba_knx_lite		MACH_HABA_KNX_LITE	HABA_KNX_LITE		4310
-tai			MACH_TAI		TAI			4311
-prototd			MACH_PROTOTD		PROTOTD			4312
-dst_tonto		MACH_DST_TONTO		DST_TONTO		4313
-draco			MACH_DRACO		DRACO			4314
-dxr2			MACH_DXR2		DXR2			4315
-rut			MACH_RUT		RUT			4316
-am180x_wsc		MACH_AM180X_WSC		AM180X_WSC		4317
-deluxe_u		MACH_DELUXE_U		DELUXE_U		4318
-deluxe_ul		MACH_DELUXE_UL		DELUXE_UL		4319
-at91sam9260medths	MACH_AT91SAM9260MEDTHS	AT91SAM9260MEDTHS	4320
-matrix516		MACH_MATRIX516		MATRIX516		4321
-vid401x			MACH_VID401X		VID401X			4322
-helios_v5		MACH_HELIOS_V5		HELIOS_V5		4323
-playpaq2		MACH_PLAYPAQ2		PLAYPAQ2		4324
-igam			MACH_IGAM		IGAM			4325
-amico_i			MACH_AMICO_I		AMICO_I			4326
-amico_e			MACH_AMICO_E		AMICO_E			4327
-sentient_mm3_ck		MACH_SENTIENT_MM3_CK	SENTIENT_MM3_CK		4328
-smx6			MACH_SMX6		SMX6			4329
-pango			MACH_PANGO		PANGO			4330
-ns115_stick		MACH_NS115_STICK	NS115_STICK		4331
-bctrm3			MACH_BCTRM3		BCTRM3			4332
-doctorws		MACH_DOCTORWS		DOCTORWS		4333
-m2601			MACH_M2601		M2601			4334
-vgg1111			MACH_VGG1111		VGG1111			4337
-countach		MACH_COUNTACH		COUNTACH		4338
-visstrim_sm20		MACH_VISSTRIM_SM20	VISSTRIM_SM20		4339
-a639			MACH_A639		A639			4340
-spacemonkey		MACH_SPACEMONKEY	SPACEMONKEY		4341
-zpdu_stamp		MACH_ZPDU_STAMP		ZPDU_STAMP		4342
-htc_g7_clone		MACH_HTC_G7_CLONE	HTC_G7_CLONE		4343
-ft2080_corvus		MACH_FT2080_CORVUS	FT2080_CORVUS		4344
-fisland			MACH_FISLAND		FISLAND			4345
-zpdu			MACH_ZPDU		ZPDU			4346
 urt			MACH_URT		URT			4347
-conti_ovip		MACH_CONTI_OVIP		CONTI_OVIP		4348
-omapl138_nagra		MACH_OMAPL138_NAGRA	OMAPL138_NAGRA		4349
-da850_at3kp1		MACH_DA850_AT3KP1	DA850_AT3KP1		4350
-da850_at3kp2		MACH_DA850_AT3KP2	DA850_AT3KP2		4351
-surma			MACH_SURMA		SURMA			4352
-stm_b2092		MACH_STM_B2092		STM_B2092		4353
-mx535_ycr		MACH_MX535_YCR		MX535_YCR		4354
-m7_wl			MACH_M7_WL		M7_WL			4355
-m7_u			MACH_M7_U		M7_U			4356
-omap3_stndt_evm		MACH_OMAP3_STNDT_EVM	OMAP3_STNDT_EVM		4357
-m7_wlv			MACH_M7_WLV		M7_WLV			4358
-xam3517			MACH_XAM3517		XAM3517			4359
-a220			MACH_A220		A220			4360
-aclima_odie		MACH_ACLIMA_ODIE	ACLIMA_ODIE		4361
-vibble			MACH_VIBBLE		VIBBLE			4362
-k2_u			MACH_K2_U		K2_U			4363
-mx53_egf		MACH_MX53_EGF		MX53_EGF		4364
-novpek_imx53		MACH_NOVPEK_IMX53	NOVPEK_IMX53		4365
-novpek_imx6x		MACH_NOVPEK_IMX6X	NOVPEK_IMX6X		4366
-mx25_smartbox		MACH_MX25_SMARTBOX	MX25_SMARTBOX		4367
-eicg6410		MACH_EICG6410		EICG6410		4368
-picasso_e3		MACH_PICASSO_E3		PICASSO_E3		4369
-motonavigator		MACH_MOTONAVIGATOR	MOTONAVIGATOR		4370
-varioconnect2		MACH_VARIOCONNECT2	VARIOCONNECT2		4371
-deluxe_tw		MACH_DELUXE_TW		DELUXE_TW		4372
-kore3			MACH_KORE3		KORE3			4374
-mx6s_drs		MACH_MX6S_DRS		MX6S_DRS		4375
-cmimx6			MACH_CMIMX6		CMIMX6			4376
-roth			MACH_ROTH		ROTH			4377
-eq4ux			MACH_EQ4UX		EQ4UX			4378
-x1plus			MACH_X1PLUS		X1PLUS			4379
-modimx27		MACH_MODIMX27		MODIMX27		4380
-videon_hduac		MACH_VIDEON_HDUAC	VIDEON_HDUAC		4381
-blackbird		MACH_BLACKBIRD		BLACKBIRD		4382
-runmaster		MACH_RUNMASTER		RUNMASTER		4383
-ceres			MACH_CERES		CERES			4384
-nad435			MACH_NAD435		NAD435			4385
-ns115_proto_type	MACH_NS115_PROTO_TYPE	NS115_PROTO_TYPE	4386
-fs20_vcc		MACH_FS20_VCC		FS20_VCC		4387
-meson6tv_skt		MACH_MESON6TV_SKT	MESON6TV_SKT		4389
 keystone		MACH_KEYSTONE		KEYSTONE		4390
-pcm052			MACH_PCM052		PCM052			4391
-qrd_skud_prime		MACH_QRD_SKUD_PRIME	QRD_SKUD_PRIME		4393
-guf_santaro		MACH_GUF_SANTARO	GUF_SANTARO		4395
-sheepshead		MACH_SHEEPSHEAD		SHEEPSHEAD		4396
-mx6_iwg15m_mxm		MACH_MX6_IWG15M_MXM	MX6_IWG15M_MXM		4397
-mx6_iwg15m_q7		MACH_MX6_IWG15M_Q7	MX6_IWG15M_Q7		4398
-at91sam9263if8mic	MACH_AT91SAM9263IF8MIC	AT91SAM9263IF8MIC	4399
-marcopolo		MACH_MARCOPOLO		MARCOPOLO		4401
-mx535_sdcr		MACH_MX535_SDCR		MX535_SDCR		4402
-mx53_csb2733		MACH_MX53_CSB2733	MX53_CSB2733		4403
-diva			MACH_DIVA		DIVA			4404
-ncr_7744		MACH_NCR_7744		NCR_7744		4405
-macallan		MACH_MACALLAN		MACALLAN		4406
-wnr3500			MACH_WNR3500		WNR3500			4407
-pgavrf			MACH_PGAVRF		PGAVRF			4408
-helios_v6		MACH_HELIOS_V6		HELIOS_V6		4409
-lcct			MACH_LCCT		LCCT			4410
-csndug			MACH_CSNDUG		CSNDUG			4411
-wandboard_imx6		MACH_WANDBOARD_IMX6	WANDBOARD_IMX6		4412
-omap4_jet		MACH_OMAP4_JET		OMAP4_JET		4413
-tegra_roth		MACH_TEGRA_ROTH		TEGRA_ROTH		4414
-m7dcg			MACH_M7DCG		M7DCG			4415
-m7dug			MACH_M7DUG		M7DUG			4416
-m7dtg			MACH_M7DTG		M7DTG			4417
-ap42x			MACH_AP42X		AP42X			4418
-var_som_mx6		MACH_VAR_SOM_MX6	VAR_SOM_MX6		4419
-pdlu			MACH_PDLU		PDLU			4420
-hydrogen		MACH_HYDROGEN		HYDROGEN		4421
-npa211e			MACH_NPA211E		NPA211E			4422
-arcadia			MACH_ARCADIA		ARCADIA			4423
-arcadia_l		MACH_ARCADIA_L		ARCADIA_L		4424
-msm8930dt		MACH_MSM8930DT		MSM8930DT		4425
-ktam3874		MACH_KTAM3874		KTAM3874		4426
-cec4			MACH_CEC4		CEC4			4427
-ape6evm			MACH_APE6EVM		APE6EVM			4428
-tx6			MACH_TX6		TX6			4429
-cfa10037		MACH_CFA10037		CFA10037		4431
-ezp1000			MACH_EZP1000		EZP1000			4433
-wgr826v			MACH_WGR826V		WGR826V			4434
-exuma			MACH_EXUMA		EXUMA			4435
-fregate			MACH_FREGATE		FREGATE			4436
-osirisimx508		MACH_OSIRISIMX508	OSIRISIMX508		4437
-st_exigo		MACH_ST_EXIGO		ST_EXIGO		4438
-pismo			MACH_PISMO		PISMO			4439
-atc7			MACH_ATC7		ATC7			4440
-nspireclp		MACH_NSPIRECLP		NSPIRECLP		4441
-nspiretp		MACH_NSPIRETP		NSPIRETP		4442
-nspirecx		MACH_NSPIRECX		NSPIRECX		4443
-maya			MACH_MAYA		MAYA			4444
-wecct			MACH_WECCT		WECCT			4445
-m2s			MACH_M2S		M2S			4446
-msm8625q_evbd		MACH_MSM8625Q_EVBD	MSM8625Q_EVBD		4447
-tiny210			MACH_TINY210		TINY210			4448
-g3			MACH_G3			G3			4449
-hurricane		MACH_HURRICANE		HURRICANE		4450
-mx6_pod			MACH_MX6_POD		MX6_POD			4451
-elondcn			MACH_ELONDCN		ELONDCN			4452
-cwmx535			MACH_CWMX535		CWMX535			4453
-m7_wlj			MACH_M7_WLJ		M7_WLJ			4454
-qsp_arm			MACH_QSP_ARM		QSP_ARM			4455
-msm8625q_skud		MACH_MSM8625Q_SKUD	MSM8625Q_SKUD		4456
-htcmondrian		MACH_HTCMONDRIAN	HTCMONDRIAN		4457
-watson_ead		MACH_WATSON_EAD		WATSON_EAD		4458
-mitwoa			MACH_MITWOA		MITWOA			4459
-omap3_wolverine		MACH_OMAP3_WOLVERINE	OMAP3_WOLVERINE		4460
-mapletree		MACH_MAPLETREE		MAPLETREE		4461
-msm8625_fih_sae		MACH_MSM8625_FIH_SAE	MSM8625_FIH_SAE		4462
-epc35			MACH_EPC35		EPC35			4463
-smartrtu		MACH_SMARTRTU		SMARTRTU		4464
-rcm101			MACH_RCM101		RCM101			4465
-amx_imx53_mxx		MACH_AMX_IMX53_MXX	AMX_IMX53_MXX		4466
-acer_a12		MACH_ACER_A12		ACER_A12		4470
-sbc6x			MACH_SBC6X		SBC6X			4471
-u2			MACH_U2			U2			4472
-smdk4270		MACH_SMDK4270		SMDK4270		4473
-priscillag		MACH_PRISCILLAG		PRISCILLAG		4474
-priscillac		MACH_PRISCILLAC		PRISCILLAC		4475
-priscilla		MACH_PRISCILLA		PRISCILLA		4476
-innova_shpu_v2		MACH_INNOVA_SHPU_V2	INNOVA_SHPU_V2		4477
-mach_type_dep2410	MACH_MACH_TYPE_DEP2410	MACH_TYPE_DEP2410	4479
-bctre3			MACH_BCTRE3		BCTRE3			4480
-omap_m100		MACH_OMAP_M100		OMAP_M100		4481
-flo			MACH_FLO		FLO			4482
-nanobone		MACH_NANOBONE		NANOBONE		4483
-stm_b2105		MACH_STM_B2105		STM_B2105		4484
-omap4_bsc_bap_v3	MACH_OMAP4_BSC_BAP_V3	OMAP4_BSC_BAP_V3	4485
-ss1pam			MACH_SS1PAM		SS1PAM			4486
-primominiu		MACH_PRIMOMINIU		PRIMOMINIU		4488
-mrt_35hd_dualnas_e	MACH_MRT_35HD_DUALNAS_E	MRT_35HD_DUALNAS_E	4489
-kiwi			MACH_KIWI		KIWI			4490
-hw90496			MACH_HW90496		HW90496			4491
-mep2440			MACH_MEP2440		MEP2440			4492
-colibri_t30		MACH_COLIBRI_T30	COLIBRI_T30		4493
-cwv1			MACH_CWV1		CWV1			4494
-nsa325			MACH_NSA325		NSA325			4495
-dpxmtc			MACH_DPXMTC		DPXMTC			4497
-tt_stuttgart		MACH_TT_STUTTGART	TT_STUTTGART		4498
-miranda_apcii		MACH_MIRANDA_APCII	MIRANDA_APCII		4499
-mx6q_moderox		MACH_MX6Q_MODEROX	MX6Q_MODEROX		4500
-mudskipper		MACH_MUDSKIPPER		MUDSKIPPER		4501
-urania			MACH_URANIA		URANIA			4502
-stm_b2112		MACH_STM_B2112		STM_B2112		4503
-mx6q_ats_phoenix	MACH_MX6Q_ATS_PHOENIX	MX6Q_ATS_PHOENIX	4505
-stm_b2116		MACH_STM_B2116		STM_B2116		4506
-mythology		MACH_MYTHOLOGY		MYTHOLOGY		4507
-fc360v1			MACH_FC360V1		FC360V1			4508
-gps_sensor		MACH_GPS_SENSOR		GPS_SENSOR		4509
-gazelle			MACH_GAZELLE		GAZELLE			4510
-mpq8064_dma		MACH_MPQ8064_DMA	MPQ8064_DMA		4511
-wems_asd01		MACH_WEMS_ASD01		WEMS_ASD01		4512
-apalis_t30		MACH_APALIS_T30		APALIS_T30		4513
-armstonea9		MACH_ARMSTONEA9		ARMSTONEA9		4515
-omap_blazetablet	MACH_OMAP_BLAZETABLET	OMAP_BLAZETABLET	4516
-ar6mxq			MACH_AR6MXQ		AR6MXQ			4517
-ar6mxs			MACH_AR6MXS		AR6MXS			4518
-gwventana		MACH_GWVENTANA		GWVENTANA		4520
-igep0033		MACH_IGEP0033		IGEP0033		4521
-h52c1_concerto		MACH_H52C1_CONCERTO	H52C1_CONCERTO		4524
-fcmbrd			MACH_FCMBRD		FCMBRD			4525
-pcaaxs1			MACH_PCAAXS1		PCAAXS1			4526
-ls_orca			MACH_LS_ORCA		LS_ORCA			4527
-pcm051lb		MACH_PCM051LB		PCM051LB		4528
-mx6s_lp507_gvci		MACH_MX6S_LP507_GVCI	MX6S_LP507_GVCI		4529
-dido			MACH_DIDO		DIDO			4530
-swarco_itc3_9g20	MACH_SWARCO_ITC3_9G20	SWARCO_ITC3_9G20	4531
-robo_roady		MACH_ROBO_ROADY		ROBO_ROADY		4532
-rskrza1			MACH_RSKRZA1		RSKRZA1			4533
-swarco_sid		MACH_SWARCO_SID		SWARCO_SID		4534
-mx6_iwg15s_sbc		MACH_MX6_IWG15S_SBC	MX6_IWG15S_SBC		4535
-mx6q_camaro		MACH_MX6Q_CAMARO	MX6Q_CAMARO		4536
-hb6mxs			MACH_HB6MXS		HB6MXS			4537
-lager			MACH_LAGER		LAGER			4538
-lp8x4x			MACH_LP8X4X		LP8X4X			4539
-tegratab7		MACH_TEGRATAB7		TEGRATAB7		4540
-andromeda		MACH_ANDROMEDA		ANDROMEDA		4541
-bootes			MACH_BOOTES		BOOTES			4542
-nethmi			MACH_NETHMI		NETHMI			4543
-tegratab		MACH_TEGRATAB		TEGRATAB		4544
-som5_evb		MACH_SOM5_EVB		SOM5_EVB		4545
-venaticorum		MACH_VENATICORUM	VENATICORUM		4546
-stm_b2110		MACH_STM_B2110		STM_B2110		4547
-elux_hathor		MACH_ELUX_HATHOR	ELUX_HATHOR		4548
-helios_v7		MACH_HELIOS_V7		HELIOS_V7		4549
-xc10v1			MACH_XC10V1		XC10V1			4550
-cp2u			MACH_CP2U		CP2U			4551
-iap_f			MACH_IAP_F		IAP_F			4552
-iap_g			MACH_IAP_G		IAP_G			4553
-aae			MACH_AAE		AAE			4554
-pegasus			MACH_PEGASUS		PEGASUS			4555
-cygnus			MACH_CYGNUS		CYGNUS			4556
-centaurus		MACH_CENTAURUS		CENTAURUS		4557
-msm8930_qrd8930		MACH_MSM8930_QRD8930	MSM8930_QRD8930		4558
-quby_tim		MACH_QUBY_TIM		QUBY_TIM		4559
-zedi3250a		MACH_ZEDI3250A		ZEDI3250A		4560
-grus			MACH_GRUS		GRUS			4561
-apollo3			MACH_APOLLO3		APOLLO3			4562
-cowon_r7		MACH_COWON_R7		COWON_R7		4563
-tonga3			MACH_TONGA3		TONGA3			4564
-p535			MACH_P535		P535			4565
-sa3874i			MACH_SA3874I		SA3874I			4566
-mx6_navico_com		MACH_MX6_NAVICO_COM	MX6_NAVICO_COM		4567
-proxmobil2		MACH_PROXMOBIL2		PROXMOBIL2		4568
-ubinux1			MACH_UBINUX1		UBINUX1			4569
-istos			MACH_ISTOS		ISTOS			4570
-benvolio4		MACH_BENVOLIO4		BENVOLIO4		4571
-eco5_bx2		MACH_ECO5_BX2		ECO5_BX2		4572
-eukrea_cpuimx28sd	MACH_EUKREA_CPUIMX28SD	EUKREA_CPUIMX28SD	4573
-domotab			MACH_DOMOTAB		DOMOTAB			4574
-pfla03			MACH_PFLA03		PFLA03			4575
+ckb_rza1h		MACH_CKB_RZA1H		CKB_RZA1H		4780
+bcm2835			MACH_BCM2835		BCM2835			4828
+cm_3g			MACH_CM_3G		CM_3G			4943
+empc_aimx6		MACH_EMPC_AIMX6		EMPC_AIMX6		4958
+diyefis6410		MACH_DIYEFIS6410	DIYEFIS6410		5063
+mx53_turing		MACH_MX53_TURING	MX53_TURING		5064
+mx6dl_turing		MACH_MX6DL_TURING	MX6DL_TURING		5066
+mx53_indash		MACH_MX53_INDASH	MX53_INDASH		5067
+mx6q_indash		MACH_MX6Q_INDASH	MX6Q_INDASH		5068
+mx6dl_indash		MACH_MX6DL_INDASH	MX6DL_INDASH		5069
+rts_g6			MACH_RTS_G6		RTS_G6			5070
+ka_titan		MACH_KA_TITAN		KA_TITAN		5071
+cl_som_imx7		MACH_CL_SOM_IMX7	CL_SOM_IMX7		5072
+vvdn_mgsi_vsis		MACH_VVDN_MGSI_VSIS	VVDN_MGSI_VSIS		5073
+mx6q_nano		MACH_MX6Q_NANO		MX6Q_NANO		5074
+pdu001			MACH_PDU001		PDU001			5075
+cab_proyk		MACH_CAB_PROYK		CAB_PROYK		5076
+klin			MACH_KLIN		KLIN			5077
+enman_steuerbox		MACH_ENMAN_STEUERBOX	ENMAN_STEUERBOX		5078
+ls_stingray		MACH_LS_STINGRAY	LS_STINGRAY		5079
+ipdu			MACH_IPDU		IPDU			5080
+linda			MACH_LINDA		LINDA			5081
+mx6q_openrex		MACH_MX6Q_OPENREX	MX6Q_OPENREX		5082
+on100			MACH_ON100		ON100			5083
+eminds_rtu12		MACH_EMINDS_RTU12	EMINDS_RTU12		5084
+eminds_avl10		MACH_EMINDS_AVL10	EMINDS_AVL10		5085
+main_plc_lme		MACH_MAIN_PLC_LME	MAIN_PLC_LME		5086
+mspx			MACH_MSPX		MSPX			5087
+cgw_300			MACH_CGW_300		CGW_300			5088
+mx7d_cicada		MACH_MX7D_CICADA	MX7D_CICADA		5089
+virt2real_dm365		MACH_VIRT2REAL_DM365	VIRT2REAL_DM365		5090
+dm365_virt2real		MACH_DM365_VIRT2REAL	DM365_VIRT2REAL		5091
+h6073			MACH_H6073		H6073			5092
+gtgateway		MACH_GTGATEWAY		GTGATEWAY		5093
+xarina_standard		MACH_XARINA_STANDARD	XARINA_STANDARD		5094
+novasoms		MACH_NOVASOMS		NOVASOMS		5095
+novasomp		MACH_NOVASOMP		NOVASOMP		5096
+novasomu		MACH_NOVASOMU		NOVASOMU		5097
+mx6q_mpbd		MACH_MX6Q_MPBD		MX6Q_MPBD		5098
+ncr_1930		MACH_NCR_1930		NCR_1930		5099
+uap301			MACH_UAP301		UAP301			5100
+urt02			MACH_URT02		URT02			5101
+atc8			MACH_ATC8		ATC8			5102
+iot_gateway		MACH_IOT_GATEWAY	IOT_GATEWAY		5103
+hsm_phoenix		MACH_HSM_PHOENIX	HSM_PHOENIX		5104
+missouri		MACH_MISSOURI		MISSOURI		5105
+remarkable		MACH_REMARKABLE		REMARKABLE		5106
+fa0113			MACH_FA0113		FA0113			5107
+innova_statnettawm	MACH_INNOVA_STATNETTAWM	INNOVA_STATNETTAWM	5108
diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl
new file mode 100644
index 000000000000..3c2cb5d5adfa
--- /dev/null
+++ b/arch/arm/tools/syscall.tbl
@@ -0,0 +1,413 @@
+#
+# Linux system call numbers and entry vectors
+#
+# The format is:
+# <num>	<abi>	<name>			[<entry point>			[<oabi compat entry point>]]
+#
+# Where abi is:
+#  common - for system calls shared between oabi and eabi (may have compat)
+#  oabi   - for oabi-only system calls (may have compat)
+#  eabi   - for eabi-only system calls
+#
+# For each syscall number, "common" is mutually exclusive with oabi and eabi
+#
+0	common	restart_syscall		sys_restart_syscall
+1	common	exit			sys_exit
+2	common	fork			sys_fork
+3	common	read			sys_read
+4	common	write			sys_write
+5	common	open			sys_open
+6	common	close			sys_close
+# 7 was sys_waitpid
+8	common	creat			sys_creat
+9	common	link			sys_link
+10	common	unlink			sys_unlink
+11	common	execve			sys_execve
+12	common	chdir			sys_chdir
+13	oabi	time			sys_time
+14	common	mknod			sys_mknod
+15	common	chmod			sys_chmod
+16	common	lchown			sys_lchown16
+# 17 was sys_break
+# 18 was sys_stat
+19	common	lseek			sys_lseek
+20	common	getpid			sys_getpid
+21	common	mount			sys_mount
+22	oabi	umount			sys_oldumount
+23	common	setuid			sys_setuid16
+24	common	getuid			sys_getuid16
+25	oabi	stime			sys_stime
+26	common	ptrace			sys_ptrace
+27	oabi	alarm			sys_alarm
+# 28 was sys_fstat
+29	common	pause			sys_pause
+30	oabi	utime			sys_utime
+# 31 was sys_stty
+# 32 was sys_gtty
+33	common	access			sys_access
+34	common	nice			sys_nice
+# 35 was sys_ftime
+36	common	sync			sys_sync
+37	common	kill			sys_kill
+38	common	rename			sys_rename
+39	common	mkdir			sys_mkdir
+40	common	rmdir			sys_rmdir
+41	common	dup			sys_dup
+42	common	pipe			sys_pipe
+43	common	times			sys_times
+# 44 was sys_prof
+45	common	brk			sys_brk
+46	common	setgid			sys_setgid16
+47	common	getgid			sys_getgid16
+# 48 was sys_signal
+49	common	geteuid			sys_geteuid16
+50	common	getegid			sys_getegid16
+51	common	acct			sys_acct
+52	common	umount2			sys_umount
+# 53 was sys_lock
+54	common	ioctl			sys_ioctl
+55	common	fcntl			sys_fcntl
+# 56 was sys_mpx
+57	common	setpgid			sys_setpgid
+# 58 was sys_ulimit
+# 59 was sys_olduname
+60	common	umask			sys_umask
+61	common	chroot			sys_chroot
+62	common	ustat			sys_ustat
+63	common	dup2			sys_dup2
+64	common	getppid			sys_getppid
+65	common	getpgrp			sys_getpgrp
+66	common	setsid			sys_setsid
+67	common	sigaction		sys_sigaction
+# 68 was sys_sgetmask
+# 69 was sys_ssetmask
+70	common	setreuid		sys_setreuid16
+71	common	setregid		sys_setregid16
+72	common	sigsuspend		sys_sigsuspend
+73	common	sigpending		sys_sigpending
+74	common	sethostname		sys_sethostname
+75	common	setrlimit		sys_setrlimit
+# Back compat 2GB limited rlimit
+76	oabi	getrlimit		sys_old_getrlimit
+77	common	getrusage		sys_getrusage
+78	common	gettimeofday		sys_gettimeofday
+79	common	settimeofday		sys_settimeofday
+80	common	getgroups		sys_getgroups16
+81	common	setgroups		sys_setgroups16
+82	oabi	select			sys_old_select
+83	common	symlink			sys_symlink
+# 84 was sys_lstat
+85	common	readlink		sys_readlink
+86	common	uselib			sys_uselib
+87	common	swapon			sys_swapon
+88	common	reboot			sys_reboot
+89	oabi	readdir			sys_old_readdir
+90	oabi	mmap			sys_old_mmap
+91	common	munmap			sys_munmap
+92	common	truncate		sys_truncate
+93	common	ftruncate		sys_ftruncate
+94	common	fchmod			sys_fchmod
+95	common	fchown			sys_fchown16
+96	common	getpriority		sys_getpriority
+97	common	setpriority		sys_setpriority
+# 98 was sys_profil
+99	common	statfs			sys_statfs
+100	common	fstatfs			sys_fstatfs
+# 101 was sys_ioperm
+102	oabi	socketcall		sys_socketcall		sys_oabi_socketcall
+103	common	syslog			sys_syslog
+104	common	setitimer		sys_setitimer
+105	common	getitimer		sys_getitimer
+106	common	stat			sys_newstat
+107	common	lstat			sys_newlstat
+108	common	fstat			sys_newfstat
+# 109 was sys_uname
+# 110 was sys_iopl
+111	common	vhangup			sys_vhangup
+# 112 was sys_idle
+# syscall to call a syscall!
+113	oabi	syscall			sys_syscall
+114	common	wait4			sys_wait4
+115	common	swapoff			sys_swapoff
+116	common	sysinfo			sys_sysinfo
+117	oabi	ipc			sys_ipc			sys_oabi_ipc
+118	common	fsync			sys_fsync
+119	common	sigreturn		sys_sigreturn_wrapper
+120	common	clone			sys_clone
+121	common	setdomainname		sys_setdomainname
+122	common	uname			sys_newuname
+# 123 was sys_modify_ldt
+124	common	adjtimex		sys_adjtimex
+125	common	mprotect		sys_mprotect
+126	common	sigprocmask		sys_sigprocmask
+# 127 was sys_create_module
+128	common	init_module		sys_init_module
+129	common	delete_module		sys_delete_module
+# 130 was sys_get_kernel_syms
+131	common	quotactl		sys_quotactl
+132	common	getpgid			sys_getpgid
+133	common	fchdir			sys_fchdir
+134	common	bdflush			sys_bdflush
+135	common	sysfs			sys_sysfs
+136	common	personality		sys_personality
+# 137 was sys_afs_syscall
+138	common	setfsuid		sys_setfsuid16
+139	common	setfsgid		sys_setfsgid16
+140	common	_llseek			sys_llseek
+141	common	getdents		sys_getdents
+142	common	_newselect		sys_select
+143	common	flock			sys_flock
+144	common	msync			sys_msync
+145	common	readv			sys_readv
+146	common	writev			sys_writev
+147	common	getsid			sys_getsid
+148	common	fdatasync		sys_fdatasync
+149	common	_sysctl			sys_sysctl
+150	common	mlock			sys_mlock
+151	common	munlock			sys_munlock
+152	common	mlockall		sys_mlockall
+153	common	munlockall		sys_munlockall
+154	common	sched_setparam		sys_sched_setparam
+155	common	sched_getparam		sys_sched_getparam
+156	common	sched_setscheduler	sys_sched_setscheduler
+157	common	sched_getscheduler	sys_sched_getscheduler
+158	common	sched_yield		sys_sched_yield
+159	common	sched_get_priority_max	sys_sched_get_priority_max
+160	common	sched_get_priority_min	sys_sched_get_priority_min
+161	common	sched_rr_get_interval	sys_sched_rr_get_interval
+162	common	nanosleep		sys_nanosleep
+163	common	mremap			sys_mremap
+164	common	setresuid		sys_setresuid16
+165	common	getresuid		sys_getresuid16
+# 166 was sys_vm86
+# 167 was sys_query_module
+168	common	poll			sys_poll
+169	common	nfsservctl
+170	common	setresgid		sys_setresgid16
+171	common	getresgid		sys_getresgid16
+172	common	prctl			sys_prctl
+173	common	rt_sigreturn		sys_rt_sigreturn_wrapper
+174	common	rt_sigaction		sys_rt_sigaction
+175	common	rt_sigprocmask		sys_rt_sigprocmask
+176	common	rt_sigpending		sys_rt_sigpending
+177	common	rt_sigtimedwait		sys_rt_sigtimedwait
+178	common	rt_sigqueueinfo		sys_rt_sigqueueinfo
+179	common	rt_sigsuspend		sys_rt_sigsuspend
+180	common	pread64			sys_pread64		sys_oabi_pread64
+181	common	pwrite64		sys_pwrite64		sys_oabi_pwrite64
+182	common	chown			sys_chown16
+183	common	getcwd			sys_getcwd
+184	common	capget			sys_capget
+185	common	capset			sys_capset
+186	common	sigaltstack		sys_sigaltstack
+187	common	sendfile		sys_sendfile
+# 188 reserved
+# 189 reserved
+190	common	vfork			sys_vfork
+# SuS compliant getrlimit
+191	common	ugetrlimit		sys_getrlimit
+192	common	mmap2			sys_mmap2
+193	common	truncate64		sys_truncate64		sys_oabi_truncate64
+194	common	ftruncate64		sys_ftruncate64		sys_oabi_ftruncate64
+195	common	stat64			sys_stat64		sys_oabi_stat64
+196	common	lstat64			sys_lstat64		sys_oabi_lstat64
+197	common	fstat64			sys_fstat64		sys_oabi_fstat64
+198	common	lchown32		sys_lchown
+199	common	getuid32		sys_getuid
+200	common	getgid32		sys_getgid
+201	common	geteuid32		sys_geteuid
+202	common	getegid32		sys_getegid
+203	common	setreuid32		sys_setreuid
+204	common	setregid32		sys_setregid
+205	common	getgroups32		sys_getgroups
+206	common	setgroups32		sys_setgroups
+207	common	fchown32		sys_fchown
+208	common	setresuid32		sys_setresuid
+209	common	getresuid32		sys_getresuid
+210	common	setresgid32		sys_setresgid
+211	common	getresgid32		sys_getresgid
+212	common	chown32			sys_chown
+213	common	setuid32		sys_setuid
+214	common	setgid32		sys_setgid
+215	common	setfsuid32		sys_setfsuid
+216	common	setfsgid32		sys_setfsgid
+217	common	getdents64		sys_getdents64
+218	common	pivot_root		sys_pivot_root
+219	common	mincore			sys_mincore
+220	common	madvise			sys_madvise
+221	common	fcntl64			sys_fcntl64		sys_oabi_fcntl64
+# 222 for tux
+# 223 is unused
+224	common	gettid			sys_gettid
+225	common	readahead		sys_readahead		sys_oabi_readahead
+226	common	setxattr		sys_setxattr
+227	common	lsetxattr		sys_lsetxattr
+228	common	fsetxattr		sys_fsetxattr
+229	common	getxattr		sys_getxattr
+230	common	lgetxattr		sys_lgetxattr
+231	common	fgetxattr		sys_fgetxattr
+232	common	listxattr		sys_listxattr
+233	common	llistxattr		sys_llistxattr
+234	common	flistxattr		sys_flistxattr
+235	common	removexattr		sys_removexattr
+236	common	lremovexattr		sys_lremovexattr
+237	common	fremovexattr		sys_fremovexattr
+238	common	tkill			sys_tkill
+239	common	sendfile64		sys_sendfile64
+240	common	futex			sys_futex
+241	common	sched_setaffinity	sys_sched_setaffinity
+242	common	sched_getaffinity	sys_sched_getaffinity
+243	common	io_setup		sys_io_setup
+244	common	io_destroy		sys_io_destroy
+245	common	io_getevents		sys_io_getevents
+246	common	io_submit		sys_io_submit
+247	common	io_cancel		sys_io_cancel
+248	common	exit_group		sys_exit_group
+249	common	lookup_dcookie		sys_lookup_dcookie
+250	common	epoll_create		sys_epoll_create
+251	common	epoll_ctl		sys_epoll_ctl		sys_oabi_epoll_ctl
+252	common	epoll_wait		sys_epoll_wait		sys_oabi_epoll_wait
+253	common	remap_file_pages	sys_remap_file_pages
+# 254 for set_thread_area
+# 255 for get_thread_area
+256	common	set_tid_address		sys_set_tid_address
+257	common	timer_create		sys_timer_create
+258	common	timer_settime		sys_timer_settime
+259	common	timer_gettime		sys_timer_gettime
+260	common	timer_getoverrun	sys_timer_getoverrun
+261	common	timer_delete		sys_timer_delete
+262	common	clock_settime		sys_clock_settime
+263	common	clock_gettime		sys_clock_gettime
+264	common	clock_getres		sys_clock_getres
+265	common	clock_nanosleep		sys_clock_nanosleep
+266	common	statfs64		sys_statfs64_wrapper
+267	common	fstatfs64		sys_fstatfs64_wrapper
+268	common	tgkill			sys_tgkill
+269	common	utimes			sys_utimes
+270	common	arm_fadvise64_64	sys_arm_fadvise64_64
+271	common	pciconfig_iobase	sys_pciconfig_iobase
+272	common	pciconfig_read		sys_pciconfig_read
+273	common	pciconfig_write		sys_pciconfig_write
+274	common	mq_open			sys_mq_open
+275	common	mq_unlink		sys_mq_unlink
+276	common	mq_timedsend		sys_mq_timedsend
+277	common	mq_timedreceive		sys_mq_timedreceive
+278	common	mq_notify		sys_mq_notify
+279	common	mq_getsetattr		sys_mq_getsetattr
+280	common	waitid			sys_waitid
+281	common	socket			sys_socket
+282	common	bind			sys_bind		sys_oabi_bind
+283	common	connect			sys_connect		sys_oabi_connect
+284	common	listen			sys_listen
+285	common	accept			sys_accept
+286	common	getsockname		sys_getsockname
+287	common	getpeername		sys_getpeername
+288	common	socketpair		sys_socketpair
+289	common	send			sys_send
+290	common	sendto			sys_sendto		sys_oabi_sendto
+291	common	recv			sys_recv
+292	common	recvfrom		sys_recvfrom
+293	common	shutdown		sys_shutdown
+294	common	setsockopt		sys_setsockopt
+295	common	getsockopt		sys_getsockopt
+296	common	sendmsg			sys_sendmsg		sys_oabi_sendmsg
+297	common	recvmsg			sys_recvmsg
+298	common	semop			sys_semop		sys_oabi_semop
+299	common	semget			sys_semget
+300	common	semctl			sys_semctl
+301	common	msgsnd			sys_msgsnd
+302	common	msgrcv			sys_msgrcv
+303	common	msgget			sys_msgget
+304	common	msgctl			sys_msgctl
+305	common	shmat			sys_shmat
+306	common	shmdt			sys_shmdt
+307	common	shmget			sys_shmget
+308	common	shmctl			sys_shmctl
+309	common	add_key			sys_add_key
+310	common	request_key		sys_request_key
+311	common	keyctl			sys_keyctl
+312	common	semtimedop		sys_semtimedop		sys_oabi_semtimedop
+313	common	vserver
+314	common	ioprio_set		sys_ioprio_set
+315	common	ioprio_get		sys_ioprio_get
+316	common	inotify_init		sys_inotify_init
+317	common	inotify_add_watch	sys_inotify_add_watch
+318	common	inotify_rm_watch	sys_inotify_rm_watch
+319	common	mbind			sys_mbind
+320	common	get_mempolicy		sys_get_mempolicy
+321	common	set_mempolicy		sys_set_mempolicy
+322	common	openat			sys_openat
+323	common	mkdirat			sys_mkdirat
+324	common	mknodat			sys_mknodat
+325	common	fchownat		sys_fchownat
+326	common	futimesat		sys_futimesat
+327	common	fstatat64		sys_fstatat64		sys_oabi_fstatat64
+328	common	unlinkat		sys_unlinkat
+329	common	renameat		sys_renameat
+330	common	linkat			sys_linkat
+331	common	symlinkat		sys_symlinkat
+332	common	readlinkat		sys_readlinkat
+333	common	fchmodat		sys_fchmodat
+334	common	faccessat		sys_faccessat
+335	common	pselect6		sys_pselect6
+336	common	ppoll			sys_ppoll
+337	common	unshare			sys_unshare
+338	common	set_robust_list		sys_set_robust_list
+339	common	get_robust_list		sys_get_robust_list
+340	common	splice			sys_splice
+341	common	arm_sync_file_range	sys_sync_file_range2
+342	common	tee			sys_tee
+343	common	vmsplice		sys_vmsplice
+344	common	move_pages		sys_move_pages
+345	common	getcpu			sys_getcpu
+346	common	epoll_pwait		sys_epoll_pwait
+347	common	kexec_load		sys_kexec_load
+348	common	utimensat		sys_utimensat
+349	common	signalfd		sys_signalfd
+350	common	timerfd_create		sys_timerfd_create
+351	common	eventfd			sys_eventfd
+352	common	fallocate		sys_fallocate
+353	common	timerfd_settime		sys_timerfd_settime
+354	common	timerfd_gettime		sys_timerfd_gettime
+355	common	signalfd4		sys_signalfd4
+356	common	eventfd2		sys_eventfd2
+357	common	epoll_create1		sys_epoll_create1
+358	common	dup3			sys_dup3
+359	common	pipe2			sys_pipe2
+360	common	inotify_init1		sys_inotify_init1
+361	common	preadv			sys_preadv
+362	common	pwritev			sys_pwritev
+363	common	rt_tgsigqueueinfo	sys_rt_tgsigqueueinfo
+364	common	perf_event_open		sys_perf_event_open
+365	common	recvmmsg		sys_recvmmsg
+366	common	accept4			sys_accept4
+367	common	fanotify_init		sys_fanotify_init
+368	common	fanotify_mark		sys_fanotify_mark
+369	common	prlimit64		sys_prlimit64
+370	common	name_to_handle_at	sys_name_to_handle_at
+371	common	open_by_handle_at	sys_open_by_handle_at
+372	common	clock_adjtime		sys_clock_adjtime
+373	common	syncfs			sys_syncfs
+374	common	sendmmsg		sys_sendmmsg
+375	common	setns			sys_setns
+376	common	process_vm_readv	sys_process_vm_readv
+377	common	process_vm_writev	sys_process_vm_writev
+378	common	kcmp			sys_kcmp
+379	common	finit_module		sys_finit_module
+380	common	sched_setattr		sys_sched_setattr
+381	common	sched_getattr		sys_sched_getattr
+382	common	renameat2		sys_renameat2
+383	common	seccomp			sys_seccomp
+384	common	getrandom		sys_getrandom
+385	common	memfd_create		sys_memfd_create
+386	common	bpf			sys_bpf
+387	common	execveat		sys_execveat
+388	common	userfaultfd		sys_userfaultfd
+389	common	membarrier		sys_membarrier
+390	common	mlock2			sys_mlock2
+391	common	copy_file_range		sys_copy_file_range
+392	common	preadv2			sys_preadv2
+393	common	pwritev2		sys_pwritev2
+394	common	pkey_mprotect		sys_pkey_mprotect
+395	common	pkey_alloc		sys_pkey_alloc
+396	common	pkey_free		sys_pkey_free
diff --git a/arch/arm/tools/syscallhdr.sh b/arch/arm/tools/syscallhdr.sh
new file mode 100644
index 000000000000..72d4b2e3bdec
--- /dev/null
+++ b/arch/arm/tools/syscallhdr.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+in="$1"
+out="$2"
+my_abis=`echo "($3)" | tr ',' '|'`
+prefix="$4"
+offset="$5"
+
+fileguard=_ASM_ARM_`basename "$out" | sed \
+    -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/' \
+    -e 's/[^A-Z0-9_]/_/g' -e 's/__/_/g'`
+if echo $out | grep -q uapi; then
+    fileguard="_UAPI$fileguard"
+fi
+grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | (
+    echo "#ifndef ${fileguard}"
+    echo "#define ${fileguard} 1"
+    echo ""
+
+    while read nr abi name entry ; do
+	if [ -z "$offset" ]; then
+	    echo "#define __NR_${prefix}${name} $nr"
+	else
+	    echo "#define __NR_${prefix}${name} ($offset + $nr)"
+        fi
+    done
+
+    echo ""
+    echo "#endif /* ${fileguard} */"
+) > "$out"
diff --git a/arch/arm/tools/syscallnr.sh b/arch/arm/tools/syscallnr.sh
new file mode 100644
index 000000000000..d2971296469a
--- /dev/null
+++ b/arch/arm/tools/syscallnr.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+in="$1"
+out="$2"
+my_abis=`echo "($3)" | tr ',' '|'`
+align=1
+
+fileguard=_ASM_ARM_`basename "$out" | sed \
+    -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/' \
+    -e 's/[^A-Z0-9_]/_/g' -e 's/__/_/g'`
+
+grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | tail -n1 | (
+    echo "#ifndef ${fileguard}
+#define ${fileguard} 1
+
+/*
+ * This needs to be greater than __NR_last_syscall+1 in order to account
+ * for the padding in the syscall table.
+ */
+"
+
+    while read nr abi name entry; do
+        nr=$(($nr + 1))
+        while [ "$(($nr / (256 * $align) ))" -gt 0 ]; do
+            align=$(( $align * 4 ))
+        done
+        nr=$(( ($nr + $align - 1) & ~($align - 1) ))
+        echo "/* aligned to $align */"
+        echo "#define __NR_syscalls $nr"
+    done
+
+    echo ""
+    echo "#endif /* ${fileguard} */"
+) > "$out"
diff --git a/arch/arm/tools/syscalltbl.sh b/arch/arm/tools/syscalltbl.sh
new file mode 100644
index 000000000000..5ca834545ed3
--- /dev/null
+++ b/arch/arm/tools/syscalltbl.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+in="$1"
+out="$2"
+my_abis=`echo "($3)" | tr ',' '|'`
+
+grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | (
+    while read nr abi name entry compat; do
+        if [ "$abi" = "eabi" -a -n "$compat" ]; then
+            echo "$in: error: a compat entry for an EABI syscall ($name) makes no sense" >&2
+            exit 1
+        fi
+
+	if [ -n "$entry" ]; then
+            if [ -z "$compat" ]; then
+                echo "NATIVE($nr, $entry)"
+            else
+                echo "COMPAT($nr, $entry, $compat)"
+            fi
+        fi
+    done
+) > "$out"
diff --git a/arch/arm/vfp/vfp.h b/arch/arm/vfp/vfp.h
index c8c98dd44ad4..89773e5ddf35 100644
--- a/arch/arm/vfp/vfp.h
+++ b/arch/arm/vfp/vfp.h
@@ -155,8 +155,8 @@ struct vfp_single {
 	u32	significand;
 };
 
-extern s32 vfp_get_float(unsigned int reg);
-extern void vfp_put_float(s32 val, unsigned int reg);
+asmlinkage s32 vfp_get_float(unsigned int reg);
+asmlinkage void vfp_put_float(s32 val, unsigned int reg);
 
 /*
  * VFP_SINGLE_MANTISSA_BITS - number of bits in the mantissa
@@ -270,8 +270,8 @@ struct vfp_double {
 #else
 #define VFP_REG_ZERO	16
 #endif
-extern u64 vfp_get_double(unsigned int reg);
-extern void vfp_put_double(u64 val, unsigned int reg);
+asmlinkage u64 vfp_get_double(unsigned int reg);
+asmlinkage void vfp_put_double(u64 val, unsigned int reg);
 
 #define VFP_DOUBLE_MANTISSA_BITS	(52)
 #define VFP_DOUBLE_EXPONENT_BITS	(11)
@@ -377,4 +377,4 @@ struct op {
 	u32 flags;
 };
 
-extern void vfp_save_state(void *location, u32 fpexc);
+asmlinkage void vfp_save_state(void *location, u32 fpexc);
diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c
index da0b33deba6d..0351f5645fb1 100644
--- a/arch/arm/vfp/vfpmodule.c
+++ b/arch/arm/vfp/vfpmodule.c
@@ -34,11 +34,11 @@
 /*
  * Our undef handlers (in entry.S)
  */
-void vfp_testing_entry(void);
-void vfp_support_entry(void);
-void vfp_null_entry(void);
+asmlinkage void vfp_testing_entry(void);
+asmlinkage void vfp_support_entry(void);
+asmlinkage void vfp_null_entry(void);
 
-void (*vfp_vector)(void) = vfp_null_entry;
+asmlinkage void (*vfp_vector)(void) = vfp_null_entry;
 
 /*
  * Dual-use variable.
diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c
index f193414d0f6f..4986dc0c1dff 100644
--- a/arch/arm/xen/enlighten.c
+++ b/arch/arm/xen/enlighten.c
@@ -372,8 +372,7 @@ static int __init xen_guest_init(void)
 	 * for secondary CPUs as they are brought up.
 	 * For uniformity we use VCPUOP_register_vcpu_info even on cpu0.
 	 */
-	xen_vcpu_info = __alloc_percpu(sizeof(struct vcpu_info),
-			                       sizeof(struct vcpu_info));
+	xen_vcpu_info = alloc_percpu(struct vcpu_info);
 	if (xen_vcpu_info == NULL)
 		return -ENOMEM;
 
diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c
index d062f08f5020..bd62d94f8ac5 100644
--- a/arch/arm/xen/mm.c
+++ b/arch/arm/xen/mm.c
@@ -186,7 +186,6 @@ struct dma_map_ops *xen_dma_ops;
 EXPORT_SYMBOL(xen_dma_ops);
 
 static struct dma_map_ops xen_swiotlb_dma_ops = {
-	.mapping_error = xen_swiotlb_dma_mapping_error,
 	.alloc = xen_swiotlb_alloc_coherent,
 	.free = xen_swiotlb_free_coherent,
 	.sync_single_for_cpu = xen_swiotlb_sync_single_for_cpu,
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 969ef880d234..111742126897 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -52,6 +52,7 @@ config ARM64
 	select GENERIC_TIME_VSYSCALL
 	select HANDLE_DOMAIN_IRQ
 	select HARDIRQS_SW_RESEND
+	select HAVE_ACPI_APEI if (ACPI && EFI)
 	select HAVE_ALIGNED_STRUCT_PAGE if SLUB
 	select HAVE_ARCH_AUDITSYSCALL
 	select HAVE_ARCH_BITREVERSE
@@ -109,6 +110,7 @@ config ARM64
 	select POWER_SUPPLY
 	select SPARSE_IRQ
 	select SYSCTL_EXCEPTION_TRACE
+	select THREAD_INFO_IN_TASK
 	help
 	  ARM 64-bit (AArch64) Linux support.
 
@@ -238,6 +240,9 @@ config PGTABLE_LEVELS
 	default 3 if ARM64_16K_PAGES && ARM64_VA_BITS_47
 	default 4 if !ARM64_64K_PAGES && ARM64_VA_BITS_48
 
+config ARCH_SUPPORTS_UPROBES
+	def_bool y
+
 source "init/Kconfig"
 
 source "kernel/Kconfig.freezer"
@@ -790,6 +795,14 @@ config SETEND_EMULATION
 	  If unsure, say Y
 endif
 
+config ARM64_SW_TTBR0_PAN
+	bool "Emulate Privileged Access Never using TTBR0_EL1 switching"
+	help
+	  Enabling this option prevents the kernel from accessing
+	  user-space memory directly by pointing TTBR0_EL1 to a reserved
+	  zeroed area and reserved ASID. The user access routines
+	  restore the valid TTBR0_EL1 temporarily.
+
 menu "ARMv8.1 architectural features"
 
 config ARM64_HW_AFDBM
diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug
index b661fe742615..d1ebd46872fd 100644
--- a/arch/arm64/Kconfig.debug
+++ b/arch/arm64/Kconfig.debug
@@ -2,9 +2,13 @@ menu "Kernel hacking"
 
 source "lib/Kconfig.debug"
 
-config ARM64_PTDUMP
+config ARM64_PTDUMP_CORE
+	def_bool n
+
+config ARM64_PTDUMP_DEBUGFS
 	bool "Export kernel pagetable layout to userspace via debugfs"
 	depends on DEBUG_KERNEL
+	select ARM64_PTDUMP_CORE
 	select DEBUG_FS
         help
 	  Say Y here if you want to show the kernel pagetable layout in a
@@ -38,6 +42,35 @@ config ARM64_RANDOMIZE_TEXT_OFFSET
 	  of TEXT_OFFSET and platforms must not require a specific
 	  value.
 
+config DEBUG_WX
+	bool "Warn on W+X mappings at boot"
+	select ARM64_PTDUMP_CORE
+	---help---
+	  Generate a warning if any W+X mappings are found at boot.
+
+	  This is useful for discovering cases where the kernel is leaving
+	  W+X mappings after applying NX, as such mappings are a security risk.
+	  This check also includes UXN, which should be set on all kernel
+	  mappings.
+
+	  Look for a message in dmesg output like this:
+
+	    arm64/mm: Checked W+X mappings: passed, no W+X pages found.
+
+	  or like this, if the check failed:
+
+	    arm64/mm: Checked W+X mappings: FAILED, <N> W+X pages found.
+
+	  Note that even if the check fails, your kernel is possibly
+	  still fine, as W+X mappings are not a security hole in
+	  themselves, what they do is that they make the exploitation
+	  of other unfixed kernel bugs easier.
+
+	  There is no runtime or memory usage effect of this option
+	  once the kernel has booted up - it's a one time check.
+
+	  If in doubt, say "Y".
+
 config DEBUG_SET_MODULE_RONX
 	bool "Set loadable kernel module data as NX and text as RO"
 	depends on MODULES
diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms
index 101794f5ce10..715ef1256838 100644
--- a/arch/arm64/Kconfig.platforms
+++ b/arch/arm64/Kconfig.platforms
@@ -113,6 +113,7 @@ config ARCH_MVEBU
 
 config ARCH_QCOM
 	bool "Qualcomm Platforms"
+	select GPIOLIB
 	select PINCTRL
 	help
 	  This enables support for the ARMv8 based Qualcomm chipsets.
@@ -143,6 +144,7 @@ config ARCH_RENESAS
 	select PM
 	select PM_GENERIC_DOMAINS
 	select RENESAS_IRQC
+	select SOC_BUS
 	help
 	  This enables support for the ARMv8 based Renesas SoCs.
 
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index 3635b8662724..b9a4a934ca05 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -37,10 +37,16 @@ $(warning LSE atomics not supported by binutils)
   endif
 endif
 
-KBUILD_CFLAGS	+= -mgeneral-regs-only $(lseinstr)
+brokengasinst := $(call as-instr,1:\n.inst 0\n.rept . - 1b\n\nnop\n.endr\n,,-DCONFIG_BROKEN_GAS_INST=1)
+
+ifneq ($(brokengasinst),)
+$(warning Detected assembler with broken .inst; disassembly will be unreliable)
+endif
+
+KBUILD_CFLAGS	+= -mgeneral-regs-only $(lseinstr) $(brokengasinst)
 KBUILD_CFLAGS	+= -fno-asynchronous-unwind-tables
 KBUILD_CFLAGS	+= $(call cc-option, -mpc-relative-literal-loads)
-KBUILD_AFLAGS	+= $(lseinstr)
+KBUILD_AFLAGS	+= $(lseinstr) $(brokengasinst)
 
 ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
 KBUILD_CPPFLAGS	+= -mbig-endian
diff --git a/arch/arm64/boot/dts/Makefile b/arch/arm64/boot/dts/Makefile
index 6684f97c2722..080232b0270e 100644
--- a/arch/arm64/boot/dts/Makefile
+++ b/arch/arm64/boot/dts/Makefile
@@ -1,4 +1,5 @@
 dts-dirs += al
+dts-dirs += allwinner
 dts-dirs += altera
 dts-dirs += amd
 dts-dirs += amlogic
diff --git a/arch/arm64/boot/dts/allwinner/Makefile b/arch/arm64/boot/dts/allwinner/Makefile
new file mode 100644
index 000000000000..1e29a5ae8282
--- /dev/null
+++ b/arch/arm64/boot/dts/allwinner/Makefile
@@ -0,0 +1,5 @@
+dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a64-pine64-plus.dtb sun50i-a64-pine64.dtb
+
+always		:= $(dtb-y)
+subdir-y	:= $(dts-dirs)
+clean-files	:= *.dtb
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-pine64-plus.dts b/arch/arm64/boot/dts/allwinner/sun50i-a64-pine64-plus.dts
new file mode 100644
index 000000000000..790d14daaa6a
--- /dev/null
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-pine64-plus.dts
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2016 ARM Ltd.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "sun50i-a64-pine64.dts"
+
+/ {
+	model = "Pine64+";
+	compatible = "pine64,pine64-plus", "allwinner,sun50i-a64";
+
+	/* TODO: Camera, Ethernet PHY, touchscreen, etc. */
+};
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-pine64.dts b/arch/arm64/boot/dts/allwinner/sun50i-a64-pine64.dts
new file mode 100644
index 000000000000..47095909d9d6
--- /dev/null
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-pine64.dts
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2016 ARM Ltd.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "sun50i-a64.dtsi"
+
+/ {
+	model = "Pine64";
+	compatible = "pine64,pine64", "allwinner,sun50i-a64";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pins_a>;
+	status = "okay";
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins>;
+	status = "okay";
+};
+
+&i2c1_pins {
+	bias-pull-up;
+};
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a64.dtsi
new file mode 100644
index 000000000000..e0dcab8eb035
--- /dev/null
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a64.dtsi
@@ -0,0 +1,261 @@
+/*
+ * Copyright (C) 2016 ARM Ltd.
+ * based on the Allwinner H3 dtsi:
+ *    Copyright (C) 2015 Jens Kuske <jenskuske@gmail.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/pinctrl/sun4i-a10.h>
+
+/ {
+	interrupt-parent = <&gic>;
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			compatible = "arm,cortex-a53", "arm,armv8";
+			device_type = "cpu";
+			reg = <0>;
+			enable-method = "psci";
+		};
+
+		cpu1: cpu@1 {
+			compatible = "arm,cortex-a53", "arm,armv8";
+			device_type = "cpu";
+			reg = <1>;
+			enable-method = "psci";
+		};
+
+		cpu2: cpu@2 {
+			compatible = "arm,cortex-a53", "arm,armv8";
+			device_type = "cpu";
+			reg = <2>;
+			enable-method = "psci";
+		};
+
+		cpu3: cpu@3 {
+			compatible = "arm,cortex-a53", "arm,armv8";
+			device_type = "cpu";
+			reg = <3>;
+			enable-method = "psci";
+		};
+	};
+
+	osc24M: osc24M_clk {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <24000000>;
+		clock-output-names = "osc24M";
+	};
+
+	osc32k: osc32k_clk {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <32768>;
+		clock-output-names = "osc32k";
+	};
+
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13
+			(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>,
+			     <GIC_PPI 14
+			(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>,
+			     <GIC_PPI 11
+			(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>,
+			     <GIC_PPI 10
+			(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+	};
+
+	soc {
+		compatible = "simple-bus";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		ccu: clock@01c20000 {
+			compatible = "allwinner,sun50i-a64-ccu";
+			reg = <0x01c20000 0x400>;
+			clocks = <&osc24M>, <&osc32k>;
+			clock-names = "hosc", "losc";
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		pio: pinctrl@1c20800 {
+			compatible = "allwinner,sun50i-a64-pinctrl";
+			reg = <0x01c20800 0x400>;
+			interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu 58>;
+			gpio-controller;
+			#gpio-cells = <3>;
+			interrupt-controller;
+			#interrupt-cells = <3>;
+
+			i2c1_pins: i2c1_pins {
+				pins = "PH2", "PH3";
+				function = "i2c1";
+			};
+
+			uart0_pins_a: uart0@0 {
+				pins = "PB8", "PB9";
+				function = "uart0";
+			};
+		};
+
+		uart0: serial@1c28000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x01c28000 0x400>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			clocks = <&ccu 67>;
+			resets = <&ccu 46>;
+			status = "disabled";
+		};
+
+		uart1: serial@1c28400 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x01c28400 0x400>;
+			interrupts = <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			clocks = <&ccu 68>;
+			resets = <&ccu 47>;
+			status = "disabled";
+		};
+
+		uart2: serial@1c28800 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x01c28800 0x400>;
+			interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			clocks = <&ccu 69>;
+			resets = <&ccu 48>;
+			status = "disabled";
+		};
+
+		uart3: serial@1c28c00 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x01c28c00 0x400>;
+			interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			clocks = <&ccu 70>;
+			resets = <&ccu 49>;
+			status = "disabled";
+		};
+
+		uart4: serial@1c29000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x01c29000 0x400>;
+			interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			clocks = <&ccu 71>;
+			resets = <&ccu 50>;
+			status = "disabled";
+		};
+
+		i2c0: i2c@1c2ac00 {
+			compatible = "allwinner,sun6i-a31-i2c";
+			reg = <0x01c2ac00 0x400>;
+			interrupts = <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu 63>;
+			resets = <&ccu 42>;
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		i2c1: i2c@1c2b000 {
+			compatible = "allwinner,sun6i-a31-i2c";
+			reg = <0x01c2b000 0x400>;
+			interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu 64>;
+			resets = <&ccu 43>;
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		i2c2: i2c@1c2b400 {
+			compatible = "allwinner,sun6i-a31-i2c";
+			reg = <0x01c2b400 0x400>;
+			interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu 65>;
+			resets = <&ccu 44>;
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		gic: interrupt-controller@1c81000 {
+			compatible = "arm,gic-400";
+			reg = <0x01c81000 0x1000>,
+			      <0x01c82000 0x2000>,
+			      <0x01c84000 0x2000>,
+			      <0x01c86000 0x2000>;
+			interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+			interrupt-controller;
+			#interrupt-cells = <3>;
+		};
+
+		rtc: rtc@1f00000 {
+			compatible = "allwinner,sun6i-a31-rtc";
+			reg = <0x01f00000 0x54>;
+			interrupts = <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/amlogic/Makefile b/arch/arm64/boot/dts/amlogic/Makefile
index 47ec703cb230..0d7bfbf7d922 100644
--- a/arch/arm64/boot/dts/amlogic/Makefile
+++ b/arch/arm64/boot/dts/amlogic/Makefile
@@ -1,9 +1,17 @@
+dtb-$(CONFIG_ARCH_MESON) += meson-gxbb-nexbox-a95x.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-gxbb-odroidc2.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-gxbb-p200.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-gxbb-p201.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-gxbb-vega-s95-pro.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-gxbb-vega-s95-meta.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-gxbb-vega-s95-telos.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905x-p212.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905d-p230.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905d-p231.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxl-nexbox-a95x.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxm-s912-q200.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxm-s912-q201.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxm-nexbox-a1.dtb
 
 always		:= $(dtb-y)
 subdir-y	:= $(dts-dirs)
diff --git a/arch/arm64/boot/dts/amlogic/meson-gx-p23x-q20x.dtsi b/arch/arm64/boot/dts/amlogic/meson-gx-p23x-q20x.dtsi
new file mode 100644
index 000000000000..7a078bef04cd
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gx-p23x-q20x.dtsi
@@ -0,0 +1,190 @@
+/*
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/* Common DTSI for same Amlogic Q200/Q201 and P230/P231 boards using either
+ * the pin-compatible S912 (GXM) or S905D (GXL) SoCs.
+ */
+
+/ {
+	aliases {
+		serial0 = &uart_AO;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x80000000>;
+	};
+
+	vddio_boot: regulator-vddio_boot {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDIO_BOOT";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	vddao_3v3: regulator-vddao_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDAO_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	vcc_3v3: regulator-vcc_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	emmc_pwrseq: emmc-pwrseq {
+		compatible = "mmc-pwrseq-emmc";
+		reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
+	};
+
+	wifi32k: wifi32k {
+		compatible = "pwm-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		pwms = <&pwm_ef 0 30518 0>; /* PWM_E at 32.768KHz */
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&gpio GPIOX_6 GPIO_ACTIVE_LOW>;
+		clocks = <&wifi32k>;
+		clock-names = "ext_clock";
+	};
+};
+
+/* This UART is brought out to the DB9 connector */
+&uart_AO {
+	status = "okay";
+	pinctrl-0 = <&uart_ao_a_pins>;
+	pinctrl-names = "default";
+};
+
+&ir {
+	status = "okay";
+	pinctrl-0 = <&remote_input_ao_pins>;
+	pinctrl-names = "default";
+};
+
+/* Wireless SDIO Module */
+&sd_emmc_a {
+	status = "okay";
+	pinctrl-0 = <&sdio_pins>;
+	pinctrl-names = "default";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+
+	non-removable;
+	disable-wp;
+
+	mmc-pwrseq = <&sdio_pwrseq>;
+
+	vmmc-supply = <&vddao_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+
+	brcmf: bcrmf@1 {
+		reg = <1>;
+		compatible = "brcm,bcm4329-fmac";
+	};
+};
+
+/* SD card */
+&sd_emmc_b {
+	status = "okay";
+	pinctrl-0 = <&sdcard_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+	disable-wp;
+
+	cd-gpios = <&gpio CARD_6 GPIO_ACTIVE_HIGH>;
+	cd-inverted;
+
+	vmmc-supply = <&vddao_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+};
+
+/* eMMC */
+&sd_emmc_c {
+	status = "okay";
+	pinctrl-0 = <&emmc_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <8>;
+	cap-sd-highspeed;
+	cap-mmc-highspeed;
+	max-frequency = <200000000>;
+	non-removable;
+	disable-wp;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+
+	mmc-pwrseq = <&emmc_pwrseq>;
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+};
+
+&pwm_ef {
+	status = "okay";
+	pinctrl-0 = <&pwm_e_pins>;
+	pinctrl-names = "default";
+	clocks = <&clkc CLKID_FCLK_DIV4>;
+	clock-names = "clkin0";
+};
+
+&ethmac {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gx.dtsi b/arch/arm64/boot/dts/amlogic/meson-gx.dtsi
new file mode 100644
index 000000000000..fc033c0d2a0f
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gx.dtsi
@@ -0,0 +1,360 @@
+/*
+ * Copyright (c) 2016 Andreas Färber
+ *
+ * Copyright (c) 2016 BayLibre, SAS.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	interrupt-parent = <&gic>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <0x2>;
+		#size-cells = <0x0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x0>;
+			enable-method = "psci";
+			next-level-cache = <&l2>;
+		};
+
+		cpu1: cpu@1 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x1>;
+			enable-method = "psci";
+			next-level-cache = <&l2>;
+		};
+
+		cpu2: cpu@2 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x2>;
+			enable-method = "psci";
+			next-level-cache = <&l2>;
+		};
+
+		cpu3: cpu@3 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x3>;
+			enable-method = "psci";
+			next-level-cache = <&l2>;
+		};
+
+		l2: l2-cache0 {
+			compatible = "cache";
+		};
+	};
+
+	arm-pmu {
+		compatible = "arm,cortex-a53-pmu";
+		interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
+	};
+
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13
+			(GIC_CPU_MASK_RAW(0xff) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 14
+			(GIC_CPU_MASK_RAW(0xff) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 11
+			(GIC_CPU_MASK_RAW(0xff) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 10
+			(GIC_CPU_MASK_RAW(0xff) | IRQ_TYPE_LEVEL_LOW)>;
+	};
+
+	xtal: xtal-clk {
+		compatible = "fixed-clock";
+		clock-frequency = <24000000>;
+		clock-output-names = "xtal";
+		#clock-cells = <0>;
+	};
+
+	firmware {
+		sm: secure-monitor {
+			compatible = "amlogic,meson-gx-sm", "amlogic,meson-gxbb-sm";
+		};
+	};
+
+	efuse: efuse {
+		compatible = "amlogic,meson-gx-efuse", "amlogic,meson-gxbb-efuse";
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		sn: sn@14 {
+			reg = <0x14 0x10>;
+		};
+
+		eth_mac: eth_mac@34 {
+			reg = <0x34 0x10>;
+		};
+
+		bid: bid@46 {
+			reg = <0x46 0x30>;
+		};
+	};
+
+	soc {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		cbus: cbus@c1100000 {
+			compatible = "simple-bus";
+			reg = <0x0 0xc1100000 0x0 0x100000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x0 0x0 0x0 0xc1100000 0x0 0x100000>;
+
+			reset: reset-controller@4404 {
+				compatible = "amlogic,meson-gx-reset", "amlogic,meson-gxbb-reset";
+				reg = <0x0 0x04404 0x0 0x20>;
+				#reset-cells = <1>;
+			};
+
+			uart_A: serial@84c0 {
+				compatible = "amlogic,meson-uart";
+				reg = <0x0 0x84c0 0x0 0x14>;
+				interrupts = <GIC_SPI 26 IRQ_TYPE_EDGE_RISING>;
+				clocks = <&xtal>;
+				status = "disabled";
+			};
+
+			uart_B: serial@84dc {
+				compatible = "amlogic,meson-uart";
+				reg = <0x0 0x84dc 0x0 0x14>;
+				interrupts = <GIC_SPI 75 IRQ_TYPE_EDGE_RISING>;
+				clocks = <&xtal>;
+				status = "disabled";
+			};
+
+			i2c_A: i2c@8500 {
+				compatible = "amlogic,meson-gxbb-i2c";
+				reg = <0x0 0x08500 0x0 0x20>;
+				interrupts = <GIC_SPI 21 IRQ_TYPE_EDGE_RISING>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "disabled";
+			};
+
+			pwm_ab: pwm@8550 {
+				compatible = "amlogic,meson-gx-pwm", "amlogic,meson-gxbb-pwm";
+				reg = <0x0 0x08550 0x0 0x10>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			pwm_cd: pwm@8650 {
+				compatible = "amlogic,meson-gx-pwm", "amlogic,meson-gxbb-pwm";
+				reg = <0x0 0x08650 0x0 0x10>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			pwm_ef: pwm@86c0 {
+				compatible = "amlogic,meson-gx-pwm", "amlogic,meson-gxbb-pwm";
+				reg = <0x0 0x086c0 0x0 0x10>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			uart_C: serial@8700 {
+				compatible = "amlogic,meson-uart";
+				reg = <0x0 0x8700 0x0 0x14>;
+				interrupts = <GIC_SPI 93 IRQ_TYPE_EDGE_RISING>;
+				clocks = <&xtal>;
+				status = "disabled";
+			};
+
+			i2c_B: i2c@87c0 {
+				compatible = "amlogic,meson-gxbb-i2c";
+				reg = <0x0 0x087c0 0x0 0x20>;
+				interrupts = <GIC_SPI 214 IRQ_TYPE_EDGE_RISING>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "disabled";
+			};
+
+			i2c_C: i2c@87e0 {
+				compatible = "amlogic,meson-gxbb-i2c";
+				reg = <0x0 0x087e0 0x0 0x20>;
+				interrupts = <GIC_SPI 215 IRQ_TYPE_EDGE_RISING>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "disabled";
+			};
+
+			watchdog@98d0 {
+				compatible = "amlogic,meson-gx-wdt", "amlogic,meson-gxbb-wdt";
+				reg = <0x0 0x098d0 0x0 0x10>;
+				clocks = <&xtal>;
+			};
+		};
+
+		gic: interrupt-controller@c4301000 {
+			compatible = "arm,gic-400";
+			reg = <0x0 0xc4301000 0 0x1000>,
+			      <0x0 0xc4302000 0 0x2000>,
+			      <0x0 0xc4304000 0 0x2000>,
+			      <0x0 0xc4306000 0 0x2000>;
+			interrupt-controller;
+			interrupts = <GIC_PPI 9
+				(GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_HIGH)>;
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+		};
+
+		aobus: aobus@c8100000 {
+			compatible = "simple-bus";
+			reg = <0x0 0xc8100000 0x0 0x100000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x0 0x0 0x0 0xc8100000 0x0 0x100000>;
+
+			uart_AO: serial@4c0 {
+				compatible = "amlogic,meson-uart";
+				reg = <0x0 0x004c0 0x0 0x14>;
+				interrupts = <GIC_SPI 193 IRQ_TYPE_EDGE_RISING>;
+				clocks = <&xtal>;
+				status = "disabled";
+			};
+
+			ir: ir@580 {
+				compatible = "amlogic,meson-gxbb-ir";
+				reg = <0x0 0x00580 0x0 0x40>;
+				interrupts = <GIC_SPI 196 IRQ_TYPE_EDGE_RISING>;
+				status = "disabled";
+			};
+		};
+
+		periphs: periphs@c8834000 {
+			compatible = "simple-bus";
+			reg = <0x0 0xc8834000 0x0 0x2000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x0 0x0 0x0 0xc8834000 0x0 0x2000>;
+
+			rng {
+				compatible = "amlogic,meson-rng";
+				reg = <0x0 0x0 0x0 0x4>;
+			};
+		};
+
+
+		hiubus: hiubus@c883c000 {
+			compatible = "simple-bus";
+			reg = <0x0 0xc883c000 0x0 0x2000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x0 0x0 0x0 0xc883c000 0x0 0x2000>;
+
+			mailbox: mailbox@404 {
+				compatible = "amlogic,meson-gx-mhu", "amlogic,meson-gxbb-mhu";
+				reg = <0 0x404 0 0x4c>;
+				interrupts = <0 208 IRQ_TYPE_EDGE_RISING>,
+					     <0 209 IRQ_TYPE_EDGE_RISING>,
+					     <0 210 IRQ_TYPE_EDGE_RISING>;
+				#mbox-cells = <1>;
+			};
+		};
+
+		ethmac: ethernet@c9410000 {
+			compatible = "amlogic,meson-gx-dwmac", "amlogic,meson-gxbb-dwmac", "snps,dwmac";
+			reg = <0x0 0xc9410000 0x0 0x10000
+			       0x0 0xc8834540 0x0 0x4>;
+			interrupts = <0 8 1>;
+			interrupt-names = "macirq";
+			phy-mode = "rgmii";
+			status = "disabled";
+		};
+
+		apb: apb@d0000000 {
+			compatible = "simple-bus";
+			reg = <0x0 0xd0000000 0x0 0x200000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x0 0x0 0x0 0xd0000000 0x0 0x200000>;
+
+			sd_emmc_a: mmc@70000 {
+				compatible = "amlogic,meson-gx-mmc", "amlogic,meson-gxbb-mmc";
+				reg = <0x0 0x70000 0x0 0x2000>;
+				interrupts = <GIC_SPI 216 IRQ_TYPE_EDGE_RISING>;
+				status = "disabled";
+			};
+
+			sd_emmc_b: mmc@72000 {
+				compatible = "amlogic,meson-gx-mmc", "amlogic,meson-gxbb-mmc";
+				reg = <0x0 0x72000 0x0 0x2000>;
+				interrupts = <GIC_SPI 217 IRQ_TYPE_EDGE_RISING>;
+				status = "disabled";
+			};
+
+			sd_emmc_c: mmc@74000 {
+				compatible = "amlogic,meson-gx-mmc", "amlogic,meson-gxbb-mmc";
+				reg = <0x0 0x74000 0x0 0x2000>;
+				interrupts = <GIC_SPI 218 IRQ_TYPE_EDGE_RISING>;
+				status = "disabled";
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb-nexbox-a95x.dts b/arch/arm64/boot/dts/amlogic/meson-gxbb-nexbox-a95x.dts
new file mode 100644
index 000000000000..969682092e0f
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxbb-nexbox-a95x.dts
@@ -0,0 +1,231 @@
+/*
+ * Copyright (c) 2016 Andreas Färber
+ * Copyright (c) 2016 BayLibre, Inc.
+ * Author: Neil Armstrong <narmstrong@kernel.org>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxbb.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/ {
+	compatible = "nexbox,a95x", "amlogic,meson-gxbb";
+	model = "NEXBOX A95X";
+	
+	aliases {
+		serial0 = &uart_AO;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x40000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		blue {
+			label = "a95x:system-status";
+			gpios = <&gpio_ao GPIOAO_13 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "heartbeat";
+			default-state = "off";
+		};
+	};
+
+	gpio-keys-polled {
+		compatible = "gpio-keys-polled";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		poll-interval = <100>;
+
+		button@0 {
+			label = "reset";
+			linux,code = <KEY_RESTART>;
+			gpios = <&gpio_ao GPIOAO_3 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	vddio_card: gpio-regulator {
+		compatible = "regulator-gpio";
+
+		regulator-name = "VDDIO_CARD";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpios = <&gpio_ao GPIOAO_5 GPIO_ACTIVE_HIGH>;
+		gpios-states = <1>;
+
+		/* Based on P200 schematics, signal CARD_1.8V/3.3V_CTR */
+		states = <1800000 0
+			  3300000 1>;
+	};
+
+	vddio_boot: regulator-vddio_boot {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDIO_BOOT";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	vddao_3v3: regulator-vddao_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDAO_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	vcc_3v3: regulator-vcc_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	emmc_pwrseq: emmc-pwrseq {
+		compatible = "mmc-pwrseq-emmc";
+		reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
+	};
+
+	wifi32k: wifi32k {
+		compatible = "pwm-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		pwms = <&pwm_ef 0 30518 0>; /* PWM_E at 32.768KHz */
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&gpio GPIOX_6 GPIO_ACTIVE_LOW>;
+		clocks = <&wifi32k>;
+		clock-names = "ext_clock";
+	};
+};
+
+&uart_AO {
+	status = "okay";
+	pinctrl-0 = <&uart_ao_a_pins>;
+	pinctrl-names = "default";
+};
+
+&ethmac {
+	status = "okay";
+	pinctrl-0 = <&eth_rmii_pins>;
+	pinctrl-names = "default";
+	phy-mode = "rmii";
+};
+
+&ir {
+	status = "okay";
+	pinctrl-0 = <&remote_input_ao_pins>;
+	pinctrl-names = "default";
+};
+
+/* Wireless SDIO Module */
+&sd_emmc_a {
+	status = "okay";
+	pinctrl-0 = <&sdio_pins>;
+	pinctrl-names = "default";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+
+	non-removable;
+	disable-wp;
+
+	mmc-pwrseq = <&sdio_pwrseq>;
+
+	vmmc-supply = <&vddao_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+};
+
+/* SD card */
+&sd_emmc_b {
+	status = "okay";
+	pinctrl-0 = <&sdcard_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+	disable-wp;
+
+	cd-gpios = <&gpio CARD_6 GPIO_ACTIVE_HIGH>;
+	cd-inverted;
+
+	vmmc-supply = <&vddao_3v3>;
+	vqmmc-supply = <&vddio_card>;
+};
+
+/* eMMC */
+&sd_emmc_c {
+	status = "okay";
+	pinctrl-0 = <&emmc_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <8>;
+	cap-sd-highspeed;
+	cap-mmc-highspeed;
+	max-frequency = <200000000>;
+	non-removable;
+	disable-wp;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+
+	mmc-pwrseq = <&emmc_pwrseq>;
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+};
+
+&pwm_ef {
+	status = "okay";
+	pinctrl-0 = <&pwm_e_pins>;
+	pinctrl-names = "default";
+	clocks = <&clkc CLKID_FCLK_DIV4>;
+	clock-names = "clkin0";
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb-odroidc2.dts b/arch/arm64/boot/dts/amlogic/meson-gxbb-odroidc2.dts
index e6e3491d48a5..238fbeacd330 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxbb-odroidc2.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-gxbb-odroidc2.dts
@@ -64,6 +64,18 @@
 		reg = <0x0 0x0 0x0 0x80000000>;
 	};
 
+	usb_otg_pwr: regulator-usb-pwrs {
+		compatible = "regulator-fixed";
+
+		regulator-name = "USB_OTG_PWR";
+
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+
+		gpio = <&gpio_ao GPIOAO_5 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
 	leds {
 		compatible = "gpio-leds";
 		blue {
@@ -73,6 +85,56 @@
 			default-state = "off";
 		};
 	};
+
+	tflash_vdd: regulator-tflash_vdd {
+		/*
+		 * signal name from schematics: TFLASH_VDD_EN
+		 */
+		compatible = "regulator-fixed";
+
+		regulator-name = "TFLASH_VDD";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&gpio_ao GPIOAO_12 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	tf_io: gpio-regulator-tf_io {
+		compatible = "regulator-gpio";
+
+		regulator-name = "TF_IO";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+
+		/*
+		 * signal name from schematics: TF_3V3N_1V8_EN
+		 */
+		gpios = <&gpio_ao GPIOAO_3 GPIO_ACTIVE_HIGH>;
+		gpios-states = <0>;
+
+		states = <3300000 0
+			  1800000 1>;
+	};
+
+	vcc1v8: regulator-vcc1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC1V8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	vcc3v3: regulator-vcc3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	emmc_pwrseq: emmc-pwrseq {
+		compatible = "mmc-pwrseq-emmc";
+		reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
+	};
 };
 
 &uart_AO {
@@ -83,7 +145,7 @@
 
 &ethmac {
 	status = "okay";
-	pinctrl-0 = <&eth_pins>;
+	pinctrl-0 = <&eth_rgmii_pins>;
 	pinctrl-names = "default";
 };
 
@@ -98,3 +160,58 @@
 	pinctrl-0 = <&i2c_a_pins>;
 	pinctrl-names = "default";
 };
+
+&usb0_phy {
+	status = "okay";
+	phy-supply = <&usb_otg_pwr>;
+};
+
+&usb1_phy {
+	status = "okay";
+};
+
+&usb0 {
+	status = "okay";
+};
+
+&usb1 {
+	status = "okay";
+};
+
+/* SD */
+&sd_emmc_b {
+	status = "okay";
+	pinctrl-0 = <&sdcard_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+	disable-wp;
+
+	cd-gpios = <&gpio CARD_6 GPIO_ACTIVE_HIGH>;
+	cd-inverted;
+
+	vmmc-supply = <&tflash_vdd>;
+	vqmmc-supply = <&tf_io>;
+};
+
+/* eMMC */
+&sd_emmc_c {
+	status = "okay";
+	pinctrl-0 = <&emmc_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <8>;
+	cap-sd-highspeed;
+	max-frequency = <200000000>;
+	non-removable;
+	disable-wp;
+	cap-mmc-highspeed;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+
+	mmc-pwrseq = <&emmc_pwrseq>;
+	vmmc-supply = <&vcc3v3>;
+	vqmmc-supply = <&vcc1v8>;
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb-p20x.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb-p20x.dtsi
index 06a34dc6002f..203be28978d5 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxbb-p20x.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-gxbb-p20x.dtsi
@@ -70,6 +70,61 @@
 		gpio = <&gpio GPIODV_24 GPIO_ACTIVE_HIGH>;
 		enable-active-high;
 	};
+
+	vddio_card: gpio-regulator {
+		compatible = "regulator-gpio";
+
+		regulator-name = "VDDIO_CARD";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpios = <&gpio_ao GPIOAO_5 GPIO_ACTIVE_HIGH>;
+		gpios-states = <1>;
+
+		/* Based on P200 schematics, signal CARD_1.8V/3.3V_CTR */
+		states = <1800000 0
+			  3300000 1>;
+	};
+
+	vddio_boot: regulator-vddio_boot {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDIO_BOOT";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	vddao_3v3: regulator-vddao_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDAO_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	vcc_3v3: regulator-vcc_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	emmc_pwrseq: emmc-pwrseq {
+		compatible = "mmc-pwrseq-emmc";
+		reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
+	};
+
+	wifi32k: wifi32k {
+		compatible = "pwm-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		pwms = <&pwm_ef 0 30518 0>; /* PWM_E at 32.768KHz */
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&gpio GPIOX_6 GPIO_ACTIVE_LOW>;
+		clocks = <&wifi32k>;
+		clock-names = "ext_clock";
+	};
 };
 
 /* This UART is brought out to the DB9 connector */
@@ -81,7 +136,7 @@
 
 &ethmac {
 	status = "okay";
-	pinctrl-0 = <&eth_pins>;
+	pinctrl-0 = <&eth_rgmii_pins>;
 	pinctrl-names = "default";
 };
 
@@ -107,3 +162,75 @@
 &usb1 {
 	status = "okay";
 };
+
+/* Wireless SDIO Module */
+&sd_emmc_a {
+	status = "okay";
+	pinctrl-0 = <&sdio_pins>;
+	pinctrl-names = "default";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+
+	non-removable;
+	disable-wp;
+
+	mmc-pwrseq = <&sdio_pwrseq>;
+
+	vmmc-supply = <&vddao_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+
+	brcmf: bcrmf@1 {
+		reg = <1>;
+		compatible = "brcm,bcm4329-fmac";
+	};
+};
+
+/* SD card */
+&sd_emmc_b {
+	status = "okay";
+	pinctrl-0 = <&sdcard_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+	disable-wp;
+
+	cd-gpios = <&gpio CARD_6 GPIO_ACTIVE_HIGH>;
+	cd-inverted;
+
+	vmmc-supply = <&vddao_3v3>;
+	vqmmc-supply = <&vddio_card>;
+};
+
+/* eMMC */
+&sd_emmc_c {
+	status = "okay";
+	pinctrl-0 = <&emmc_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <8>;
+	cap-sd-highspeed;
+	cap-mmc-highspeed;
+	max-frequency = <200000000>;
+	non-removable;
+	disable-wp;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+
+	mmc-pwrseq = <&emmc_pwrseq>;
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+};
+
+&pwm_ef {
+	status = "okay";
+	pinctrl-0 = <&pwm_e_pins>;
+	pinctrl-names = "default";
+	clocks = <&clkc CLKID_FCLK_DIV4>;
+	clock-names = "clkin0";
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb-vega-s95.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb-vega-s95.dtsi
index 73f159370188..e59ad308192f 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxbb-vega-s95.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-gxbb-vega-s95.dtsi
@@ -65,6 +65,39 @@
 		enable-active-high;
 	};
 
+	vcc_3v3: regulator-vcc_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	vcc_1v8: regulator-vcc_1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_1V8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	emmc_pwrseq: emmc-pwrseq {
+		compatible = "mmc-pwrseq-emmc";
+		reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
+	};
+
+	wifi32k: wifi32k {
+		compatible = "pwm-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		pwms = <&pwm_ef 0 30518 0>; /* PWM_E at 32.768KHz */
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&gpio GPIOX_6 GPIO_ACTIVE_LOW>,
+				<&gpio GPIOX_20 GPIO_ACTIVE_LOW>;
+		clocks = <&wifi32k>;
+		clock-names = "ext_clock";
+	};
 };
 
 &uart_AO {
@@ -82,7 +115,7 @@
 
 &ethmac {
 	status = "okay";
-	pinctrl-0 = <&eth_pins>;
+	pinctrl-0 = <&eth_rgmii_pins>;
 	pinctrl-names = "default";
 };
 
@@ -102,3 +135,74 @@
 &usb1 {
 	status = "okay";
 };
+
+/* Wireless SDIO Module */
+&sd_emmc_a {
+	status = "okay";
+	pinctrl-0 = <&sdio_pins &sdio_irq_pins>;
+	pinctrl-names = "default";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+
+	non-removable;
+	disable-wp;
+
+	mmc-pwrseq = <&sdio_pwrseq>;
+
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vcc_1v8>;
+
+	brcmf: bcrmf@1 {
+		reg = <1>;
+		compatible = "brcm,bcm4329-fmac";
+	};
+};
+
+/* SD card */
+&sd_emmc_b {
+	status = "okay";
+	pinctrl-0 = <&sdcard_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+	disable-wp;
+
+	cd-gpios = <&gpio CARD_6 GPIO_ACTIVE_HIGH>;
+	cd-inverted;
+
+	vmmc-supply = <&vcc_3v3>;
+};
+
+/* eMMC */
+&sd_emmc_c {
+	status = "okay";
+	pinctrl-0 = <&emmc_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <8>;
+	cap-sd-highspeed;
+	cap-mmc-highspeed;
+	max-frequency = <200000000>;
+	non-removable;
+	disable-wp;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+
+	mmc-pwrseq = <&emmc_pwrseq>;
+	vmmc-supply = <&vcc_3v3>;
+	vmmcq-sumpply = <&vcc_1v8>;
+};
+
+&pwm_ef {
+	status = "okay";
+	pinctrl-0 = <&pwm_e_pins>;
+	pinctrl-names = "default";
+	clocks = <&clkc CLKID_FCLK_DIV4>;
+	clock-names = "clkin0";
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi
index 610e0e1c3cee..51edd5b5c460 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi
@@ -40,9 +40,7 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <dt-bindings/gpio/gpio.h>
-#include <dt-bindings/interrupt-controller/irq.h>
-#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include "meson-gx.dtsi"
 #include <dt-bindings/gpio/meson-gxbb-gpio.h>
 #include <dt-bindings/reset/amlogic,meson-gxbb-reset.h>
 #include <dt-bindings/clock/gxbb-clkc.h>
@@ -51,106 +49,30 @@
 
 / {
 	compatible = "amlogic,meson-gxbb";
-	interrupt-parent = <&gic>;
-	#address-cells = <2>;
-	#size-cells = <2>;
 
-	cpus {
-		#address-cells = <0x2>;
-		#size-cells = <0x0>;
+	scpi {
+		compatible = "amlogic,meson-gxbb-scpi", "arm,scpi-pre-1.0";
+		mboxes = <&mailbox 1 &mailbox 2>;
+		shmem = <&cpu_scp_lpri &cpu_scp_hpri>;
 
-		cpu0: cpu@0 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a53", "arm,armv8";
-			reg = <0x0 0x0>;
-			enable-method = "psci";
-		};
-
-		cpu1: cpu@1 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a53", "arm,armv8";
-			reg = <0x0 0x1>;
-			enable-method = "psci";
-		};
-
-		cpu2: cpu@2 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a53", "arm,armv8";
-			reg = <0x0 0x2>;
-			enable-method = "psci";
-		};
-
-		cpu3: cpu@3 {
-			device_type = "cpu";
-			compatible = "arm,cortex-a53", "arm,armv8";
-			reg = <0x0 0x3>;
-			enable-method = "psci";
-		};
-	};
-
-	arm-pmu {
-		compatible = "arm,cortex-a53-pmu";
-		interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
-	};
+		clocks {
+			compatible = "arm,scpi-clocks";
 
-	psci {
-		compatible = "arm,psci-0.2";
-		method = "smc";
-	};
-
-	firmware {
-		sm: secure-monitor {
-			compatible = "amlogic,meson-gxbb-sm";
-		};
-	};
-
-	efuse: efuse {
-		compatible = "amlogic,meson-gxbb-efuse";
-		#address-cells = <1>;
-		#size-cells = <1>;
-
-		sn: sn@14 {
-			reg = <0x14 0x10>;
-		};
-
-		eth_mac: eth_mac@34 {
-			reg = <0x34 0x10>;
+			scpi_dvfs: scpi_clocks@0 {
+				compatible = "arm,scpi-dvfs-clocks";
+				#clock-cells = <1>;
+				clock-indices = <0>;
+				clock-output-names = "vcpu";
+			};
 		};
 
-		bid: bid@46 {
-			reg = <0x46 0x30>;
+		scpi_sensors: sensors {
+			compatible = "arm,scpi-sensors";
+			#thermal-sensor-cells = <1>;
 		};
 	};
 
-	timer {
-		compatible = "arm,armv8-timer";
-		interrupts = <GIC_PPI 13
-			(GIC_CPU_MASK_RAW(0xff) | IRQ_TYPE_LEVEL_LOW)>,
-			     <GIC_PPI 14
-			(GIC_CPU_MASK_RAW(0xff) | IRQ_TYPE_LEVEL_LOW)>,
-			     <GIC_PPI 11
-			(GIC_CPU_MASK_RAW(0xff) | IRQ_TYPE_LEVEL_LOW)>,
-			     <GIC_PPI 10
-			(GIC_CPU_MASK_RAW(0xff) | IRQ_TYPE_LEVEL_LOW)>;
-	};
-
-	xtal: xtal-clk {
-		compatible = "fixed-clock";
-		clock-frequency = <24000000>;
-		clock-output-names = "xtal";
-		#clock-cells = <0>;
-	};
-
 	soc {
-		compatible = "simple-bus";
-		#address-cells = <2>;
-		#size-cells = <2>;
-		ranges;
-
 		usb0_phy: phy@c0000000 {
 			compatible = "amlogic,meson-gxbb-usb2-phy";
 			#phy-cells = <0>;
@@ -165,505 +87,422 @@
 			compatible = "amlogic,meson-gxbb-usb2-phy";
 			#phy-cells = <0>;
 			reg = <0x0 0xc0000020 0x0 0x20>;
+			resets = <&reset RESET_USB_OTG>;
 			clocks = <&clkc CLKID_USB>, <&clkc CLKID_USB1>;
 			clock-names = "usb_general", "usb";
 			status = "disabled";
 		};
 
-		cbus: cbus@c1100000 {
-			compatible = "simple-bus";
-			reg = <0x0 0xc1100000 0x0 0x100000>;
-			#address-cells = <2>;
-			#size-cells = <2>;
-			ranges = <0x0 0x0 0x0 0xc1100000 0x0 0x100000>;
+		sram: sram@c8000000 {
+			compatible = "amlogic,meson-gxbb-sram", "mmio-sram";
+			reg = <0x0 0xc8000000 0x0 0x14000>;
 
-			reset: reset-controller@4404 {
-				compatible = "amlogic,meson-gxbb-reset";
-				reg = <0x0 0x04404 0x0 0x20>;
-				#reset-cells = <1>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x0 0xc8000000 0x14000>;
+
+			cpu_scp_lpri: scp-shmem@0 {
+				compatible = "amlogic,meson-gxbb-scp-shmem";
+				reg = <0x13000 0x400>;
+			};
+
+			cpu_scp_hpri: scp-shmem@200 {
+				compatible = "amlogic,meson-gxbb-scp-shmem";
+				reg = <0x13400 0x400>;
 			};
+		};
+
+		usb0: usb@c9000000 {
+			compatible = "amlogic,meson-gxbb-usb", "snps,dwc2";
+			reg = <0x0 0xc9000000 0x0 0x40000>;
+			interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clkc CLKID_USB0_DDR_BRIDGE>;
+			clock-names = "otg";
+			phys = <&usb0_phy>;
+			phy-names = "usb2-phy";
+			dr_mode = "host";
+			status = "disabled";
+		};
+
+		usb1: usb@c9100000 {
+			compatible = "amlogic,meson-gxbb-usb", "snps,dwc2";
+			reg = <0x0 0xc9100000 0x0 0x40000>;
+			interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clkc CLKID_USB1_DDR_BRIDGE>;
+			clock-names = "otg";
+			phys = <&usb1_phy>;
+			phy-names = "usb2-phy";
+			dr_mode = "host";
+			status = "disabled";
+		};
+	};
+};
+
+&cpu0 {
+	clocks = <&scpi_dvfs 0>;
+};
+
+&cpu1 {
+	clocks = <&scpi_dvfs 0>;
+};
+
+&cpu2 {
+	clocks = <&scpi_dvfs 0>;
+};
+
+&cpu3 {
+	clocks = <&scpi_dvfs 0>;
+};
+
+&cbus {
+	spifc: spi@8c80 {
+		compatible = "amlogic,meson-gxbb-spifc";
+		reg = <0x0 0x08c80 0x0 0x80>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&clkc CLKID_SPI>;
+		status = "disabled";
+	};
+};
+
+&ethmac {
+	clocks = <&clkc CLKID_ETH>,
+		 <&clkc CLKID_FCLK_DIV2>,
+		 <&clkc CLKID_MPLL2>;
+	clock-names = "stmmaceth", "clkin0", "clkin1";
+};
+
+&aobus {
+	pinctrl_aobus: pinctrl@14 {
+		compatible = "amlogic,meson-gxbb-aobus-pinctrl";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		gpio_ao: bank@14 {
+			reg = <0x0 0x00014 0x0 0x8>,
+			      <0x0 0x0002c 0x0 0x4>,
+			      <0x0 0x00024 0x0 0x8>;
+			reg-names = "mux", "pull", "gpio";
+			gpio-controller;
+			#gpio-cells = <2>;
+		};
 
-			uart_A: serial@84c0 {
-				compatible = "amlogic,meson-uart";
-				reg = <0x0 0x84c0 0x0 0x14>;
-				interrupts = <GIC_SPI 26 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&xtal>;
-				status = "disabled";
+		uart_ao_a_pins: uart_ao_a {
+			mux {
+				groups = "uart_tx_ao_a", "uart_rx_ao_a";
+				function = "uart_ao";
 			};
+		};
 
-			uart_B: serial@84dc {
-				compatible = "amlogic,meson-uart";
-				reg = <0x0 0x84dc 0x0 0x14>;
-				interrupts = <GIC_SPI 75 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&xtal>;
-				status = "disabled";
+		remote_input_ao_pins: remote_input_ao {
+			mux {
+				groups = "remote_input_ao";
+				function = "remote_input_ao";
 			};
+		};
 
-			pwm_ab: pwm@8550 {
-				compatible = "amlogic,meson-gxbb-pwm";
-				reg = <0x0 0x08550 0x0 0x10>;
-				#pwm-cells = <3>;
-				status = "disabled";
+		i2c_ao_pins: i2c_ao {
+			mux {
+				groups = "i2c_sck_ao",
+				       "i2c_sda_ao";
+				function = "i2c_ao";
 			};
+		};
 
-			pwm_cd: pwm@8650 {
-				compatible = "amlogic,meson-gxbb-pwm";
-				reg = <0x0 0x08650 0x0 0x10>;
-				#pwm-cells = <3>;
-				status = "disabled";
+		pwm_ao_a_3_pins: pwm_ao_a_3 {
+			mux {
+				groups = "pwm_ao_a_3";
+				function = "pwm_ao_a_3";
 			};
+		};
 
-			pwm_ef: pwm@86c0 {
-				compatible = "amlogic,meson-gxbb-pwm";
-				reg = <0x0 0x086c0 0x0 0x10>;
-				#pwm-cells = <3>;
-				status = "disabled";
+		pwm_ao_a_6_pins: pwm_ao_a_6 {
+			mux {
+				groups = "pwm_ao_a_6";
+				function = "pwm_ao_a_6";
 			};
+		};
 
-			uart_C: serial@8700 {
-				compatible = "amlogic,meson-uart";
-				reg = <0x0 0x8700 0x0 0x14>;
-				interrupts = <GIC_SPI 93 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&xtal>;
-				status = "disabled";
+		pwm_ao_a_12_pins: pwm_ao_a_12 {
+			mux {
+				groups = "pwm_ao_a_12";
+				function = "pwm_ao_a_12";
 			};
+		};
 
-			watchdog@98d0 {
-				compatible = "amlogic,meson-gxbb-wdt";
-				reg = <0x0 0x098d0 0x0 0x10>;
-				clocks = <&xtal>;
+		pwm_ao_b_pins: pwm_ao_b {
+			mux {
+				groups = "pwm_ao_b";
+				function = "pwm_ao_b";
 			};
+		};
+	};
+
+	clkc_AO: clock-controller@040 {
+		compatible = "amlogic,gxbb-aoclkc";
+		reg = <0x0 0x00040 0x0 0x4>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+	};
+
+	pwm_ab_AO: pwm@550 {
+		compatible = "amlogic,meson-gxbb-pwm";
+		reg = <0x0 0x0550 0x0 0x10>;
+		#pwm-cells = <3>;
+		status = "disabled";
+	};
+
+	i2c_AO: i2c@500 {
+		compatible = "amlogic,meson-gxbb-i2c";
+		reg = <0x0 0x500 0x0 0x20>;
+		interrupts = <GIC_SPI 195 IRQ_TYPE_EDGE_RISING>;
+		clocks = <&clkc CLKID_AO_I2C>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+};
+
+&periphs {
+	pinctrl_periphs: pinctrl@4b0 {
+		compatible = "amlogic,meson-gxbb-periphs-pinctrl";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		gpio: bank@4b0 {
+			reg = <0x0 0x004b0 0x0 0x28>,
+			      <0x0 0x004e8 0x0 0x14>,
+			      <0x0 0x00120 0x0 0x14>,
+			      <0x0 0x00430 0x0 0x40>;
+			reg-names = "mux", "pull", "pull-enable", "gpio";
+			gpio-controller;
+			#gpio-cells = <2>;
+		};
 
-			spifc: spi@8c80 {
-				compatible = "amlogic,meson-gxbb-spifc";
-				reg = <0x0 0x08c80 0x0 0x80>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-				clocks = <&clkc CLKID_SPI>;
-				status = "disabled";
+		emmc_pins: emmc {
+			mux {
+				groups = "emmc_nand_d07",
+				       "emmc_cmd",
+				       "emmc_clk",
+				       "emmc_ds";
+				function = "emmc";
 			};
+		};
 
-			i2c_A: i2c@8500 {
-				compatible = "amlogic,meson-gxbb-i2c";
-				reg = <0x0 0x08500 0x0 0x20>;
-				interrupts = <GIC_SPI 21 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&clkc CLKID_I2C>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-				status = "disabled";
+		nor_pins: nor {
+			mux {
+				groups = "nor_d",
+				       "nor_q",
+				       "nor_c",
+				       "nor_cs";
+				function = "nor";
 			};
+		};
 
-			i2c_B: i2c@87c0 {
-				compatible = "amlogic,meson-gxbb-i2c";
-				reg = <0x0 0x087c0 0x0 0x20>;
-				interrupts = <GIC_SPI 214 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&clkc CLKID_I2C>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-				status = "disabled";
+		sdcard_pins: sdcard {
+			mux {
+				groups = "sdcard_d0",
+				       "sdcard_d1",
+				       "sdcard_d2",
+				       "sdcard_d3",
+				       "sdcard_cmd",
+				       "sdcard_clk";
+				function = "sdcard";
 			};
+		};
 
-			i2c_C: i2c@87e0 {
-				compatible = "amlogic,meson-gxbb-i2c";
-				reg = <0x0 0x087e0 0x0 0x20>;
-				interrupts = <GIC_SPI 215 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&clkc CLKID_I2C>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-				status = "disabled";
+		sdio_pins: sdio {
+			mux {
+				groups = "sdio_d0",
+				       "sdio_d1",
+				       "sdio_d2",
+				       "sdio_d3",
+				       "sdio_cmd",
+				       "sdio_clk";
+				function = "sdio";
 			};
 		};
 
-		gic: interrupt-controller@c4301000 {
-			compatible = "arm,gic-400";
-			reg = <0x0 0xc4301000 0 0x1000>,
-			      <0x0 0xc4302000 0 0x2000>,
-			      <0x0 0xc4304000 0 0x2000>,
-			      <0x0 0xc4306000 0 0x2000>;
-			interrupt-controller;
-			interrupts = <GIC_PPI 9
-				(GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_HIGH)>;
-			#interrupt-cells = <3>;
-			#address-cells = <0>;
-		};
-
-		aobus: aobus@c8100000 {
-			compatible = "simple-bus";
-			reg = <0x0 0xc8100000 0x0 0x100000>;
-			#address-cells = <2>;
-			#size-cells = <2>;
-			ranges = <0x0 0x0 0x0 0xc8100000 0x0 0x100000>;
-
-			pinctrl_aobus: pinctrl@14 {
-				compatible = "amlogic,meson-gxbb-aobus-pinctrl";
-				#address-cells = <2>;
-				#size-cells = <2>;
-				ranges;
-
-				gpio_ao: bank@14 {
-					reg = <0x0 0x00014 0x0 0x8>,
-					      <0x0 0x0002c 0x0 0x4>,
-					      <0x0 0x00024 0x0 0x8>;
-					reg-names = "mux", "pull", "gpio";
-					gpio-controller;
-					#gpio-cells = <2>;
-				};
-
-				uart_ao_a_pins: uart_ao_a {
-					mux {
-						groups = "uart_tx_ao_a", "uart_rx_ao_a";
-						function = "uart_ao";
-					};
-				};
-
-				remote_input_ao_pins: remote_input_ao {
-					mux {
-						groups = "remote_input_ao";
-						function = "remote_input_ao";
-					};
-				};
-
-				i2c_ao_pins: i2c_ao {
-					mux {
-						groups = "i2c_sck_ao",
-						       "i2c_sda_ao";
-						function = "i2c_ao";
-					};
-				};
-
-				pwm_ao_a_3_pins: pwm_ao_a_3 {
-					mux {
-						groups = "pwm_ao_a_3";
-						function = "pwm_ao_a_3";
-					};
-				};
-
-				pwm_ao_a_6_pins: pwm_ao_a_6 {
-					mux {
-						groups = "pwm_ao_a_6";
-						function = "pwm_ao_a_6";
-					};
-				};
-
-				pwm_ao_a_12_pins: pwm_ao_a_12 {
-					mux {
-						groups = "pwm_ao_a_12";
-						function = "pwm_ao_a_12";
-					};
-				};
-
-				pwm_ao_b_pins: pwm_ao_b {
-					mux {
-						groups = "pwm_ao_b";
-						function = "pwm_ao_b";
-					};
-				};
+		sdio_irq_pins: sdio_irq {
+			mux {
+				groups = "sdio_irq";
+				function = "sdio";
 			};
+		};
 
-			clkc_AO: clock-controller@040 {
-				compatible = "amlogic,gxbb-aoclkc";
-				reg = <0x0 0x00040 0x0 0x4>;
-				#clock-cells = <1>;
-				#reset-cells = <1>;
+		uart_a_pins: uart_a {
+			mux {
+				groups = "uart_tx_a",
+				       "uart_rx_a";
+				function = "uart_a";
 			};
+		};
 
-			uart_AO: serial@4c0 {
-				compatible = "amlogic,meson-uart";
-				reg = <0x0 0x004c0 0x0 0x14>;
-				interrupts = <GIC_SPI 193 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&xtal>;
-				status = "disabled";
+		uart_b_pins: uart_b {
+			mux {
+				groups = "uart_tx_b",
+				       "uart_rx_b";
+				function = "uart_b";
 			};
+		};
 
-			ir: ir@580 {
-				compatible = "amlogic,meson-gxbb-ir";
-				reg = <0x0 0x00580 0x0 0x40>;
-				interrupts = <GIC_SPI 196 IRQ_TYPE_EDGE_RISING>;
-				status = "disabled";
+		uart_c_pins: uart_c {
+			mux {
+				groups = "uart_tx_c",
+				       "uart_rx_c";
+				function = "uart_c";
 			};
+		};
 
-			pwm_ab_AO: pwm@550 {
-				compatible = "amlogic,meson-gxbb-pwm";
-				reg = <0x0 0x0550 0x0 0x10>;
-				#pwm-cells = <3>;
-				status = "disabled";
+		i2c_a_pins: i2c_a {
+			mux {
+				groups = "i2c_sck_a",
+				       "i2c_sda_a";
+				function = "i2c_a";
 			};
+		};
 
-			i2c_AO: i2c@500 {
-				compatible = "amlogic,meson-gxbb-i2c";
-				reg = <0x0 0x500 0x0 0x20>;
-				interrupts = <GIC_SPI 195 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&clkc CLKID_AO_I2C>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-				status = "disabled";
+		i2c_b_pins: i2c_b {
+			mux {
+				groups = "i2c_sck_b",
+				       "i2c_sda_b";
+				function = "i2c_b";
 			};
 		};
 
-		periphs: periphs@c8834000 {
-			compatible = "simple-bus";
-			reg = <0x0 0xc8834000 0x0 0x2000>;
-			#address-cells = <2>;
-			#size-cells = <2>;
-			ranges = <0x0 0x0 0x0 0xc8834000 0x0 0x2000>;
+		i2c_c_pins: i2c_c {
+			mux {
+				groups = "i2c_sck_c",
+				       "i2c_sda_c";
+				function = "i2c_c";
+			};
+		};
 
-			rng {
-				compatible = "amlogic,meson-rng";
-				reg = <0x0 0x0 0x0 0x4>;
+		eth_rgmii_pins: eth-rgmii {
+			mux {
+				groups = "eth_mdio",
+				       "eth_mdc",
+				       "eth_clk_rx_clk",
+				       "eth_rx_dv",
+				       "eth_rxd0",
+				       "eth_rxd1",
+				       "eth_rxd2",
+				       "eth_rxd3",
+				       "eth_rgmii_tx_clk",
+				       "eth_tx_en",
+				       "eth_txd0",
+				       "eth_txd1",
+				       "eth_txd2",
+				       "eth_txd3";
+				function = "eth";
 			};
+		};
 
-			pinctrl_periphs: pinctrl@4b0 {
-				compatible = "amlogic,meson-gxbb-periphs-pinctrl";
-				#address-cells = <2>;
-				#size-cells = <2>;
-				ranges;
-
-				gpio: bank@4b0 {
-					reg = <0x0 0x004b0 0x0 0x28>,
-					      <0x0 0x004e8 0x0 0x14>,
-					      <0x0 0x00120 0x0 0x14>,
-					      <0x0 0x00430 0x0 0x40>;
-					reg-names = "mux", "pull", "pull-enable", "gpio";
-					gpio-controller;
-					#gpio-cells = <2>;
-				};
-
-				emmc_pins: emmc {
-					mux {
-						groups = "emmc_nand_d07",
-						       "emmc_cmd",
-						       "emmc_clk";
-						function = "emmc";
-					};
-				};
-
-				nor_pins: nor {
-					mux {
-						groups = "nor_d",
-						       "nor_q",
-						       "nor_c",
-						       "nor_cs";
-						function = "nor";
-					};
-				};
-
-				sdcard_pins: sdcard {
-					mux {
-						groups = "sdcard_d0",
-						       "sdcard_d1",
-						       "sdcard_d2",
-						       "sdcard_d3",
-						       "sdcard_cmd",
-						       "sdcard_clk";
-						function = "sdcard";
-					};
-				};
-
-				sdio_pins: sdio {
-					mux {
-						groups = "sdio_d0",
-						       "sdio_d1",
-						       "sdio_d2",
-						       "sdio_d3",
-						       "sdio_cmd",
-						       "sdio_clk";
-						function = "sdio";
-					};
-				};
-
-				sdio_irq_pins: sdio_irq {
-					mux {
-						groups = "sdio_irq";
-						function = "sdio";
-					};
-				};
-
-				uart_a_pins: uart_a {
-					mux {
-						groups = "uart_tx_a",
-						       "uart_rx_a";
-						function = "uart_a";
-					};
-				};
-
-				uart_b_pins: uart_b {
-					mux {
-						groups = "uart_tx_b",
-						       "uart_rx_b";
-						function = "uart_b";
-					};
-				};
-
-				uart_c_pins: uart_c {
-					mux {
-						groups = "uart_tx_c",
-						       "uart_rx_c";
-						function = "uart_c";
-					};
-				};
-
-				i2c_a_pins: i2c_a {
-					mux {
-						groups = "i2c_sck_a",
-						       "i2c_sda_a";
-						function = "i2c_a";
-					};
-				};
-
-				i2c_b_pins: i2c_b {
-					mux {
-						groups = "i2c_sck_b",
-						       "i2c_sda_b";
-						function = "i2c_b";
-					};
-				};
-
-				i2c_c_pins: i2c_c {
-					mux {
-						groups = "i2c_sck_c",
-						       "i2c_sda_c";
-						function = "i2c_c";
-					};
-				};
-
-				eth_pins: eth_c {
-					mux {
-						groups = "eth_mdio",
-						       "eth_mdc",
-						       "eth_clk_rx_clk",
-						       "eth_rx_dv",
-						       "eth_rxd0",
-						       "eth_rxd1",
-						       "eth_rxd2",
-						       "eth_rxd3",
-						       "eth_rgmii_tx_clk",
-						       "eth_tx_en",
-						       "eth_txd0",
-						       "eth_txd1",
-						       "eth_txd2",
-						       "eth_txd3";
-						function = "eth";
-					};
-				};
-
-				pwm_a_x_pins: pwm_a_x {
-					mux {
-						groups = "pwm_a_x";
-						function = "pwm_a_x";
-					};
-				};
-
-				pwm_a_y_pins: pwm_a_y {
-					mux {
-						groups = "pwm_a_y";
-						function = "pwm_a_y";
-					};
-				};
-
-				pwm_b_pins: pwm_b {
-					mux {
-						groups = "pwm_b";
-						function = "pwm_b";
-					};
-				};
-
-				pwm_d_pins: pwm_d {
-					mux {
-						groups = "pwm_d";
-						function = "pwm_d";
-					};
-				};
-
-				pwm_e_pins: pwm_e {
-					mux {
-						groups = "pwm_e";
-						function = "pwm_e";
-					};
-				};
-
-				pwm_f_x_pins: pwm_f_x {
-					mux {
-						groups = "pwm_f_x";
-						function = "pwm_f_x";
-					};
-				};
-
-				pwm_f_y_pins: pwm_f_y {
-					mux {
-						groups = "pwm_f_y";
-						function = "pwm_f_y";
-					};
-				};
+		eth_rmii_pins: eth-rmii {
+			mux {
+				groups = "eth_mdio",
+				       "eth_mdc",
+				       "eth_clk_rx_clk",
+				       "eth_rx_dv",
+				       "eth_rxd0",
+				       "eth_rxd1",
+				       "eth_tx_en",
+				       "eth_txd0",
+				       "eth_txd1";
+				function = "eth";
 			};
 		};
 
-		hiubus: hiubus@c883c000 {
-			compatible = "simple-bus";
-			reg = <0x0 0xc883c000 0x0 0x2000>;
-			#address-cells = <2>;
-			#size-cells = <2>;
-			ranges = <0x0 0x0 0x0 0xc883c000 0x0 0x2000>;
+		pwm_a_x_pins: pwm_a_x {
+			mux {
+				groups = "pwm_a_x";
+				function = "pwm_a_x";
+			};
+		};
 
-			clkc: clock-controller@0 {
-				compatible = "amlogic,gxbb-clkc";
-				#clock-cells = <1>;
-				reg = <0x0 0x0 0x0 0x3db>;
+		pwm_a_y_pins: pwm_a_y {
+			mux {
+				groups = "pwm_a_y";
+				function = "pwm_a_y";
 			};
+		};
 
-			mailbox: mailbox@404 {
-				compatible = "amlogic,meson-gxbb-mhu";
-				reg = <0 0x404 0 0x4c>;
-				interrupts = <0 208 IRQ_TYPE_EDGE_RISING>,
-					     <0 209 IRQ_TYPE_EDGE_RISING>,
-					     <0 210 IRQ_TYPE_EDGE_RISING>;
-				#mbox-cells = <1>;
+		pwm_b_pins: pwm_b {
+			mux {
+				groups = "pwm_b";
+				function = "pwm_b";
 			};
 		};
 
-		apb: apb@d0000000 {
-			compatible = "simple-bus";
-			reg = <0x0 0xd0000000 0x0 0x200000>;
-			#address-cells = <2>;
-			#size-cells = <2>;
-			ranges = <0x0 0x0 0x0 0xd0000000 0x0 0x200000>;
+		pwm_d_pins: pwm_d {
+			mux {
+				groups = "pwm_d";
+				function = "pwm_d";
+			};
 		};
 
-		usb0: usb@c9000000 {
-			compatible = "amlogic,meson-gxbb-usb", "snps,dwc2";
-			reg = <0x0 0xc9000000 0x0 0x40000>;
-			interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clkc CLKID_USB0_DDR_BRIDGE>;
-			clock-names = "otg";
-			phys = <&usb0_phy>;
-			phy-names = "usb2-phy";
-			dr_mode = "host";
-			status = "disabled";
+		pwm_e_pins: pwm_e {
+			mux {
+				groups = "pwm_e";
+				function = "pwm_e";
+			};
 		};
 
-		usb1: usb@c9100000 {
-			compatible = "amlogic,meson-gxbb-usb", "snps,dwc2";
-			reg = <0x0 0xc9100000 0x0 0x40000>;
-			interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clkc CLKID_USB1_DDR_BRIDGE>;
-			clock-names = "otg";
-			phys = <&usb1_phy>;
-			phy-names = "usb2-phy";
-			dr_mode = "host";
-			status = "disabled";
+		pwm_f_x_pins: pwm_f_x {
+			mux {
+				groups = "pwm_f_x";
+				function = "pwm_f_x";
+			};
 		};
 
-		ethmac: ethernet@c9410000 {
-			compatible = "amlogic,meson-gxbb-dwmac", "snps,dwmac";
-			reg = <0x0 0xc9410000 0x0 0x10000
-			       0x0 0xc8834540 0x0 0x4>;
-			interrupts = <0 8 1>;
-			interrupt-names = "macirq";
-			clocks = <&clkc CLKID_ETH>,
-				 <&clkc CLKID_FCLK_DIV2>,
-				 <&clkc CLKID_MPLL2>;
-			clock-names = "stmmaceth", "clkin0", "clkin1";
-			phy-mode = "rgmii";
-			status = "disabled";
+		pwm_f_y_pins: pwm_f_y {
+			mux {
+				groups = "pwm_f_y";
+				function = "pwm_f_y";
+			};
 		};
 	};
 };
+
+&hiubus {
+	clkc: clock-controller@0 {
+		compatible = "amlogic,gxbb-clkc";
+		#clock-cells = <1>;
+		reg = <0x0 0x0 0x0 0x3db>;
+	};
+};
+
+&i2c_A {
+	clocks = <&clkc CLKID_I2C>;
+};
+
+&i2c_B {
+	clocks = <&clkc CLKID_I2C>;
+};
+
+&i2c_C {
+	clocks = <&clkc CLKID_I2C>;
+};
+
+&sd_emmc_a {
+	clocks = <&clkc CLKID_SD_EMMC_A>,
+		 <&xtal>,
+		 <&clkc CLKID_FCLK_DIV2>;
+	clock-names = "core", "clkin0", "clkin1";
+};
+
+&sd_emmc_b {
+	clocks = <&clkc CLKID_SD_EMMC_B>,
+		 <&xtal>,
+		 <&clkc CLKID_FCLK_DIV2>;
+	clock-names = "core", "clkin0", "clkin1";
+};
+
+&sd_emmc_c {
+	clocks = <&clkc CLKID_SD_EMMC_C>,
+		 <&xtal>,
+		 <&clkc CLKID_FCLK_DIV2>;
+	clock-names = "core", "clkin0", "clkin1";
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl-nexbox-a95x.dts b/arch/arm64/boot/dts/amlogic/meson-gxl-nexbox-a95x.dts
new file mode 100644
index 000000000000..e99101ae9664
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl-nexbox-a95x.dts
@@ -0,0 +1,205 @@
+/*
+ * Copyright (c) 2016 Andreas Färber
+ * Copyright (c) 2016 BayLibre, Inc.
+ * Author: Neil Armstrong <narmstrong@kernel.org>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxl-s905x.dtsi"
+
+/ {
+	compatible = "nexbox,a95x", "amlogic,s905x", "amlogic,meson-gxl";
+	model = "NEXBOX A95X (S905X)";
+
+	aliases {
+		serial0 = &uart_AO;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x80000000>;
+	};
+
+	vddio_card: gpio-regulator {
+		compatible = "regulator-gpio";
+
+		regulator-name = "VDDIO_CARD";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpios = <&gpio_ao GPIOAO_5 GPIO_ACTIVE_HIGH>;
+		gpios-states = <1>;
+
+		/* Based on P200 schematics, signal CARD_1.8V/3.3V_CTR */
+		states = <1800000 0
+			  3300000 1>;
+	};
+
+	vddio_boot: regulator-vddio_boot {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDIO_BOOT";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	vddao_3v3: regulator-vddao_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDAO_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	vcc_3v3: regulator-vcc_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	emmc_pwrseq: emmc-pwrseq {
+		compatible = "mmc-pwrseq-emmc";
+		reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
+	};
+
+	wifi32k: wifi32k {
+		compatible = "pwm-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		pwms = <&pwm_ef 0 30518 0>; /* PWM_E at 32.768KHz */
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&gpio GPIOX_6 GPIO_ACTIVE_LOW>;
+		clocks = <&wifi32k>;
+		clock-names = "ext_clock";
+	};
+};
+
+&uart_AO {
+	status = "okay";
+	pinctrl-0 = <&uart_ao_a_pins>;
+	pinctrl-names = "default";
+};
+
+&ethmac {
+	status = "okay";
+	phy-mode = "rmii";
+	phy-handle = <&internal_phy>;
+};
+
+&ir {
+	status = "okay";
+	pinctrl-0 = <&remote_input_ao_pins>;
+	pinctrl-names = "default";
+};
+
+/* Wireless SDIO Module */
+&sd_emmc_a {
+	status = "okay";
+	pinctrl-0 = <&sdio_pins>;
+	pinctrl-names = "default";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+
+	non-removable;
+	disable-wp;
+
+	mmc-pwrseq = <&sdio_pwrseq>;
+
+	vmmc-supply = <&vddao_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+};
+
+/* SD card */
+&sd_emmc_b {
+	status = "okay";
+	pinctrl-0 = <&sdcard_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+	disable-wp;
+
+	cd-gpios = <&gpio CARD_6 GPIO_ACTIVE_HIGH>;
+	cd-inverted;
+
+	vmmc-supply = <&vddao_3v3>;
+	vqmmc-supply = <&vddio_card>;
+};
+
+/* eMMC */
+&sd_emmc_c {
+	status = "okay";
+	pinctrl-0 = <&emmc_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <8>;
+	cap-sd-highspeed;
+	cap-mmc-highspeed;
+	max-frequency = <200000000>;
+	non-removable;
+	disable-wp;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+
+	mmc-pwrseq = <&emmc_pwrseq>;
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+};
+
+&pwm_ef {
+	status = "okay";
+	pinctrl-0 = <&pwm_e_pins>;
+	pinctrl-names = "default";
+	clocks = <&clkc CLKID_FCLK_DIV4>;
+	clock-names = "clkin0";
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-p230.dts b/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-p230.dts
new file mode 100644
index 000000000000..f66939cacd37
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-p230.dts
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxl-s905d.dtsi"
+#include "meson-gx-p23x-q20x.dtsi"
+
+/ {
+	compatible = "amlogic,p230", "amlogic,s905d", "amlogic,meson-gxl";
+	model = "Amlogic Meson GXL (S905D) P230 Development Board";
+};
+
+/* P230 has exclusive choice between internal or external PHY */
+&ethmac {
+	pinctrl-0 = <&eth_pins>;
+	pinctrl-names = "default";
+
+	/* Select external PHY by default */
+	phy-handle = <&external_phy>;
+
+	/* External PHY reset is shared with internal PHY Led signals */
+	snps,reset-gpio = <&gpio GPIOZ_14 0>;
+	snps,reset-delays-us = <0 10000 1000000>;
+	snps,reset-active-low;
+
+	/* External PHY is in RGMII */
+	phy-mode = "rgmii";
+};
+
+&external_mdio {
+	external_phy: ethernet-phy@0 {
+		compatible = "ethernet-phy-id001c.c916", "ethernet-phy-ieee802.3-c22";
+		reg = <0>;
+		max-speed = <1000>;
+	};
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-p231.dts b/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-p231.dts
new file mode 100644
index 000000000000..95992cf1fe61
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-p231.dts
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxl-s905d.dtsi"
+#include "meson-gx-p23x-q20x.dtsi"
+
+/ {
+	compatible = "amlogic,p231", "amlogic,s905d", "amlogic,meson-gxl";
+	model = "Amlogic Meson GXL (S905D) P231 Development Board";
+};
+
+/* P231 has only internal PHY port */
+&ethmac {
+	phy-mode = "rmii";
+	phy-handle = <&internal_phy>;
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl-s905d.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxl-s905d.dtsi
new file mode 100644
index 000000000000..615308e55576
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl-s905d.dtsi
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "meson-gxl.dtsi"
+
+/ {
+	compatible = "amlogic,s905d", "amlogic,meson-gxl";
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-p212.dts b/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-p212.dts
new file mode 100644
index 000000000000..9639f012b02b
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-p212.dts
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxl-s905x.dtsi"
+
+/ {
+	compatible = "amlogic,p212", "amlogic,s905x", "amlogic,meson-gxl";
+	model = "Amlogic Meson GXL (S905X) P212 Development Board";
+
+	aliases {
+		serial0 = &uart_AO;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x80000000>;
+	};
+};
+
+/* This UART is brought out to the DB9 connector */
+&uart_AO {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl-s905x.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxl-s905x.dtsi
new file mode 100644
index 000000000000..08237ee1e362
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl-s905x.dtsi
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "meson-gxl.dtsi"
+
+/ {
+	compatible = "amlogic,s905x", "amlogic,meson-gxl";
+};
+
+/* S905X Only has access to its internal PHY */
+&ethmac {
+	phy-mode = "rmii";
+	phy-handle = <&internal_phy>;
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi
new file mode 100644
index 000000000000..9f89b99c4806
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi
@@ -0,0 +1,301 @@
+/*
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "meson-gx.dtsi"
+#include <dt-bindings/clock/gxbb-clkc.h>
+#include <dt-bindings/gpio/meson-gxbb-gpio.h>
+
+/ {
+	compatible = "amlogic,meson-gxl";
+};
+
+&ethmac {
+	reg = <0x0 0xc9410000 0x0 0x10000
+	       0x0 0xc8834540 0x0 0x4>;
+
+	clocks = <&clkc CLKID_ETH>,
+		 <&clkc CLKID_FCLK_DIV2>,
+		 <&clkc CLKID_MPLL2>;
+	clock-names = "stmmaceth", "clkin0", "clkin1";
+
+	mdio0: mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+	};
+};
+
+&aobus {
+	pinctrl_aobus: pinctrl@14 {
+		compatible = "amlogic,meson-gxl-aobus-pinctrl";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		gpio_ao: bank@14 {
+			reg = <0x0 0x00014 0x0 0x8>,
+			      <0x0 0x0002c 0x0 0x4>,
+			      <0x0 0x00024 0x0 0x8>;
+			reg-names = "mux", "pull", "gpio";
+			gpio-controller;
+			#gpio-cells = <2>;
+		};
+
+		uart_ao_a_pins: uart_ao_a {
+			mux {
+				groups = "uart_tx_ao_a", "uart_rx_ao_a";
+				function = "uart_ao";
+			};
+		};
+
+		remote_input_ao_pins: remote_input_ao {
+			mux {
+				groups = "remote_input_ao";
+				function = "remote_input_ao";
+			};
+		};
+	};
+};
+
+&periphs {
+	pinctrl_periphs: pinctrl@4b0 {
+		compatible = "amlogic,meson-gxl-periphs-pinctrl";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		gpio: bank@4b0 {
+			reg = <0x0 0x004b0 0x0 0x28>,
+			      <0x0 0x004e8 0x0 0x14>,
+			      <0x0 0x00120 0x0 0x14>,
+			      <0x0 0x00430 0x0 0x40>;
+			reg-names = "mux", "pull", "pull-enable", "gpio";
+			gpio-controller;
+			#gpio-cells = <2>;
+		};
+
+		emmc_pins: emmc {
+			mux {
+				groups = "emmc_nand_d07",
+				       "emmc_cmd",
+				       "emmc_clk",
+				       "emmc_ds";
+				function = "emmc";
+			};
+		};
+
+		sdcard_pins: sdcard {
+			mux {
+				groups = "sdcard_d0",
+				       "sdcard_d1",
+				       "sdcard_d2",
+				       "sdcard_d3",
+				       "sdcard_cmd",
+				       "sdcard_clk";
+				function = "sdcard";
+			};
+		};
+
+		sdio_pins: sdio {
+			mux {
+				groups = "sdio_d0",
+				       "sdio_d1",
+				       "sdio_d2",
+				       "sdio_d3",
+				       "sdio_cmd",
+				       "sdio_clk";
+				function = "sdio";
+			};
+		};
+
+		sdio_irq_pins: sdio_irq {
+			mux {
+				groups = "sdio_irq";
+				function = "sdio";
+			};
+		};
+
+		uart_a_pins: uart_a {
+			mux {
+				groups = "uart_tx_a",
+				       "uart_rx_a";
+				function = "uart_a";
+			};
+		};
+
+		uart_b_pins: uart_b {
+			mux {
+				groups = "uart_tx_b",
+				       "uart_rx_b";
+				function = "uart_b";
+			};
+		};
+
+		uart_c_pins: uart_c {
+			mux {
+				groups = "uart_tx_c",
+				       "uart_rx_c";
+				function = "uart_c";
+			};
+		};
+
+		i2c_a_pins: i2c_a {
+			mux {
+				groups = "i2c_sck_a",
+				     "i2c_sda_a";
+				function = "i2c_a";
+			};
+		};
+
+		i2c_b_pins: i2c_b {
+			mux {
+				groups = "i2c_sck_b",
+				      "i2c_sda_b";
+				function = "i2c_b";
+			};
+		};
+
+		i2c_c_pins: i2c_c {
+			mux {
+				groups = "i2c_sck_c",
+				      "i2c_sda_c";
+				function = "i2c_c";
+			};
+		};
+
+		eth_pins: eth_c {
+			mux {
+				groups = "eth_mdio",
+				       "eth_mdc",
+				       "eth_clk_rx_clk",
+				       "eth_rx_dv",
+				       "eth_rxd0",
+				       "eth_rxd1",
+				       "eth_rxd2",
+				       "eth_rxd3",
+				       "eth_rgmii_tx_clk",
+				       "eth_tx_en",
+				       "eth_txd0",
+				       "eth_txd1",
+				       "eth_txd2",
+				       "eth_txd3";
+				function = "eth";
+			};
+		};
+
+		pwm_e_pins: pwm_e {
+			mux {
+				groups = "pwm_e";
+				function = "pwm_e";
+			};
+		};
+	};
+
+	eth-phy-mux {
+		compatible = "mdio-mux-mmioreg", "mdio-mux";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x0 0x55c 0x0 0x4>;
+		mux-mask = <0xffffffff>;
+		mdio-parent-bus = <&mdio0>;
+
+		internal_mdio: mdio@e40908ff {
+			reg = <0xe40908ff>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			internal_phy: ethernet-phy@8 {
+				compatible = "ethernet-phy-id0181.4400", "ethernet-phy-ieee802.3-c22";
+				reg = <8>;
+				max-speed = <100>;
+			};
+		};
+
+		external_mdio: mdio@2009087f {
+			reg = <0x2009087f>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+	};
+};
+
+&hiubus {
+	clkc: clock-controller@0 {
+		compatible = "amlogic,gxl-clkc", "amlogic,gxbb-clkc";
+		#clock-cells = <1>;
+		reg = <0x0 0x0 0x0 0x3db>;
+	};
+};
+
+&i2c_A {
+	clocks = <&clkc CLKID_I2C>;
+};
+
+&i2c_B {
+	clocks = <&clkc CLKID_I2C>;
+};
+
+&i2c_C {
+	clocks = <&clkc CLKID_I2C>;
+};
+
+&sd_emmc_a {
+	clocks = <&clkc CLKID_SD_EMMC_A>,
+		 <&xtal>,
+		 <&clkc CLKID_FCLK_DIV2>;
+	clock-names = "core", "clkin0", "clkin1";
+};
+
+&sd_emmc_b {
+	clocks = <&clkc CLKID_SD_EMMC_B>,
+		 <&xtal>,
+		 <&clkc CLKID_FCLK_DIV2>;
+       clock-names = "core", "clkin0", "clkin1";
+};
+
+&sd_emmc_c {
+	clocks = <&clkc CLKID_SD_EMMC_C>,
+		 <&xtal>,
+		 <&clkc CLKID_FCLK_DIV2>;
+	clock-names = "core", "clkin0", "clkin1";
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxm-nexbox-a1.dts b/arch/arm64/boot/dts/amlogic/meson-gxm-nexbox-a1.dts
new file mode 100644
index 000000000000..f859d75db8bd
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxm-nexbox-a1.dts
@@ -0,0 +1,169 @@
+/*
+ * Copyright (c) 2016 BayLibre, SAS.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxm.dtsi"
+
+/ {
+	compatible = "nexbox,a1", "amlogic,s912", "amlogic,meson-gxm";
+	model = "NEXBOX A1";
+
+	aliases {
+		serial0 = &uart_AO;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x80000000>;
+	};
+
+	vddio_boot: regulator-vddio-boot {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDIO_BOOT";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	vddao_3v3: regulator-vddao-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDAO_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	vcc_3v3: regulator-vcc-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	emmc_pwrseq: emmc-pwrseq {
+		compatible = "mmc-pwrseq-emmc";
+		reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
+	};
+};
+
+/* This UART is brought out to the DB9 connector */
+&uart_AO {
+	status = "okay";
+	pinctrl-0 = <&uart_ao_a_pins>;
+	pinctrl-names = "default";
+};
+
+&ir {
+	status = "okay";
+	pinctrl-0 = <&remote_input_ao_pins>;
+	pinctrl-names = "default";
+};
+
+/* SD card */
+&sd_emmc_b {
+	status = "okay";
+	pinctrl-0 = <&sdcard_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <100000000>;
+	disable-wp;
+
+	cd-gpios = <&gpio CARD_6 GPIO_ACTIVE_HIGH>;
+	cd-inverted;
+
+	vmmc-supply = <&vddao_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+};
+
+/* eMMC */
+&sd_emmc_c {
+	status = "okay";
+	pinctrl-0 = <&emmc_pins>;
+	pinctrl-names = "default";
+
+	bus-width = <8>;
+	cap-sd-highspeed;
+	cap-mmc-highspeed;
+	max-frequency = <200000000>;
+	non-removable;
+	disable-wp;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+
+	mmc-pwrseq = <&emmc_pwrseq>;
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+};
+
+&ethmac {
+	status = "okay";
+
+	pinctrl-0 = <&eth_pins>;
+	pinctrl-names = "default";
+
+	/* Select external PHY by default */
+	phy-handle = <&external_phy>;
+
+	snps,reset-gpio = <&gpio GPIOZ_14 0>;
+	snps,reset-delays-us = <0 10000 1000000>;
+	snps,reset-active-low;
+
+	/* External PHY is in RGMII */
+	phy-mode = "rgmii";
+};
+
+&external_mdio {
+	external_phy: ethernet-phy@0 {
+		compatible = "ethernet-phy-id001c.c916", "ethernet-phy-ieee802.3-c22";
+		reg = <0>;
+		max-speed = <1000>;
+	};
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxm-s912-q200.dts b/arch/arm64/boot/dts/amlogic/meson-gxm-s912-q200.dts
new file mode 100644
index 000000000000..5dbc66088355
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxm-s912-q200.dts
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxm.dtsi"
+#include "meson-gx-p23x-q20x.dtsi"
+
+/ {
+	compatible = "amlogic,q200", "amlogic,s912", "amlogic,meson-gxm";
+	model = "Amlogic Meson GXM (S912) Q200 Development Board";
+};
+
+/* Q200 has exclusive choice between internal or external PHY */
+&ethmac {
+	pinctrl-0 = <&eth_pins>;
+	pinctrl-names = "default";
+
+	/* Select external PHY by default */
+	phy-handle = <&external_phy>;
+
+	/* External PHY reset is shared with internal PHY Led signals */
+	snps,reset-gpio = <&gpio GPIOZ_14 0>;
+	snps,reset-delays-us = <0 10000 1000000>;
+	snps,reset-active-low;
+
+	/* External PHY is in RGMII */
+	phy-mode = "rgmii";
+};
+
+&external_mdio {
+	external_phy: ethernet-phy@0 {
+		compatible = "ethernet-phy-id001c.c916", "ethernet-phy-ieee802.3-c22";
+		reg = <0>;
+		max-speed = <1000>;
+	};
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxm-s912-q201.dts b/arch/arm64/boot/dts/amlogic/meson-gxm-s912-q201.dts
new file mode 100644
index 000000000000..95e11d7faab8
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxm-s912-q201.dts
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "meson-gxm.dtsi"
+#include "meson-gx-p23x-q20x.dtsi"
+
+/ {
+	compatible = "amlogic,q201", "amlogic,s912", "amlogic,meson-gxm";
+	model = "Amlogic Meson GXM (S912) Q201 Development Board";
+};
+
+/* Q201 has only internal PHY port */
+&ethmac {
+	phy-mode = "rmii";
+	phy-handle = <&internal_phy>;
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxm.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxm.dtsi
new file mode 100644
index 000000000000..c1974bbbddea
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxm.dtsi
@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2016 Endless Computers, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "meson-gxl.dtsi"
+
+/ {
+	compatible = "amlogic,meson-gxm";
+
+	cpus {
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+				core1 {
+					cpu = <&cpu1>;
+				};
+				core2 {
+					cpu = <&cpu2>;
+				};
+				core3 {
+					cpu = <&cpu3>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu4>;
+				};
+				core1 {
+					cpu = <&cpu5>;
+				};
+				core2 {
+					cpu = <&cpu6>;
+				};
+				core3 {
+					cpu = <&cpu7>;
+				};
+			};
+		};
+
+		cpu4: cpu@100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x100>;
+			enable-method = "psci";
+			next-level-cache = <&l2>;
+		};
+
+		cpu5: cpu@101 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x101>;
+			enable-method = "psci";
+			next-level-cache = <&l2>;
+		};
+
+		cpu6: cpu@102 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x102>;
+			enable-method = "psci";
+			next-level-cache = <&l2>;
+		};
+
+		cpu7: cpu@103 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x103>;
+			enable-method = "psci";
+			next-level-cache = <&l2>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/arm/juno-base.dtsi b/arch/arm64/boot/dts/arm/juno-base.dtsi
index 7d3a2acc6a55..7d832247d0db 100644
--- a/arch/arm64/boot/dts/arm/juno-base.dtsi
+++ b/arch/arm64/boot/dts/arm/juno-base.dtsi
@@ -29,6 +29,28 @@
 		clock-names = "apb_pclk";
 	};
 
+	smmu_pcie: iommu@2b500000 {
+		compatible = "arm,mmu-401", "arm,smmu-v1";
+		reg = <0x0 0x2b500000 0x0 0x10000>;
+		interrupts = <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>;
+		#iommu-cells = <1>;
+		#global-interrupts = <1>;
+		dma-coherent;
+		status = "disabled";
+	};
+
+	smmu_etr: iommu@2b600000 {
+		compatible = "arm,mmu-401", "arm,smmu-v1";
+		reg = <0x0 0x2b600000 0x0 0x10000>;
+		interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
+		#iommu-cells = <1>;
+		#global-interrupts = <1>;
+		dma-coherent;
+		status = "disabled";
+	};
+
 	gic: interrupt-controller@2c010000 {
 		compatible = "arm,gic-400", "arm,cortex-a15-gic";
 		reg = <0x0 0x2c010000 0 0x1000>,
@@ -146,6 +168,7 @@
 	etr@20070000 {
 		compatible = "arm,coresight-tmc", "arm,primecell";
 		reg = <0 0x20070000 0 0x1000>;
+		iommus = <&smmu_etr 0>;
 
 		clocks = <&soc_smc50mhz>;
 		clock-names = "apb_pclk";
@@ -404,6 +427,8 @@
 				<0 0 0 4 &gic 0 0 0 139 4>;
 		msi-parent = <&v2m_0>;
 		status = "disabled";
+		iommu-map-mask = <0x0>;	/* RC has no means to output PCI RID */
+		iommu-map = <0x0 &smmu_pcie 0x0 0x1>;
 	};
 
 	scpi {
@@ -484,6 +509,48 @@
 
 	/include/ "juno-clocks.dtsi"
 
+	smmu_dma: iommu@7fb00000 {
+		compatible = "arm,mmu-401", "arm,smmu-v1";
+		reg = <0x0 0x7fb00000 0x0 0x10000>;
+		interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
+		#iommu-cells = <1>;
+		#global-interrupts = <1>;
+		dma-coherent;
+		status = "disabled";
+	};
+
+	smmu_hdlcd1: iommu@7fb10000 {
+		compatible = "arm,mmu-401", "arm,smmu-v1";
+		reg = <0x0 0x7fb10000 0x0 0x10000>;
+		interrupts = <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>;
+		#iommu-cells = <1>;
+		#global-interrupts = <1>;
+		status = "disabled";
+	};
+
+	smmu_hdlcd0: iommu@7fb20000 {
+		compatible = "arm,mmu-401", "arm,smmu-v1";
+		reg = <0x0 0x7fb20000 0x0 0x10000>;
+		interrupts = <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>;
+		#iommu-cells = <1>;
+		#global-interrupts = <1>;
+		status = "disabled";
+	};
+
+	smmu_usb: iommu@7fb30000 {
+		compatible = "arm,mmu-401", "arm,smmu-v1";
+		reg = <0x0 0x7fb30000 0x0 0x10000>;
+		interrupts = <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>;
+		#iommu-cells = <1>;
+		#global-interrupts = <1>;
+		dma-coherent;
+		status = "disabled";
+	};
+
 	dma@7ff00000 {
 		compatible = "arm,pl330", "arm,primecell";
 		reg = <0x0 0x7ff00000 0 0x1000>;
@@ -499,6 +566,15 @@
 			     <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>;
+		iommus = <&smmu_dma 0>,
+			 <&smmu_dma 1>,
+			 <&smmu_dma 2>,
+			 <&smmu_dma 3>,
+			 <&smmu_dma 4>,
+			 <&smmu_dma 5>,
+			 <&smmu_dma 6>,
+			 <&smmu_dma 7>,
+			 <&smmu_dma 8>;
 		clocks = <&soc_faxiclk>;
 		clock-names = "apb_pclk";
 	};
@@ -507,6 +583,7 @@
 		compatible = "arm,hdlcd";
 		reg = <0 0x7ff50000 0 0x1000>;
 		interrupts = <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>;
+		iommus = <&smmu_hdlcd1 0>;
 		clocks = <&scpi_clk 3>;
 		clock-names = "pxlclk";
 
@@ -521,6 +598,7 @@
 		compatible = "arm,hdlcd";
 		reg = <0 0x7ff60000 0 0x1000>;
 		interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
+		iommus = <&smmu_hdlcd0 0>;
 		clocks = <&scpi_clk 3>;
 		clock-names = "pxlclk";
 
@@ -574,6 +652,7 @@
 		compatible = "generic-ohci";
 		reg = <0x0 0x7ffb0000 0x0 0x10000>;
 		interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>;
+		iommus = <&smmu_usb 0>;
 		clocks = <&soc_usb48mhz>;
 	};
 
@@ -581,6 +660,7 @@
 		compatible = "generic-ehci";
 		reg = <0x0 0x7ffc0000 0x0 0x10000>;
 		interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>;
+		iommus = <&smmu_usb 0>;
 		clocks = <&soc_usb48mhz>;
 	};
 
diff --git a/arch/arm64/boot/dts/arm/juno-r1.dts b/arch/arm64/boot/dts/arm/juno-r1.dts
index f0b857d6d73c..eec37feee8fc 100644
--- a/arch/arm64/boot/dts/arm/juno-r1.dts
+++ b/arch/arm64/boot/dts/arm/juno-r1.dts
@@ -90,6 +90,7 @@
 			next-level-cache = <&A57_L2>;
 			clocks = <&scpi_dvfs 0>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <1024>;
 		};
 
 		A57_1: cpu@1 {
@@ -100,6 +101,7 @@
 			next-level-cache = <&A57_L2>;
 			clocks = <&scpi_dvfs 0>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <1024>;
 		};
 
 		A53_0: cpu@100 {
@@ -110,6 +112,7 @@
 			next-level-cache = <&A53_L2>;
 			clocks = <&scpi_dvfs 1>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <578>;
 		};
 
 		A53_1: cpu@101 {
@@ -120,6 +123,7 @@
 			next-level-cache = <&A53_L2>;
 			clocks = <&scpi_dvfs 1>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <578>;
 		};
 
 		A53_2: cpu@102 {
@@ -130,6 +134,7 @@
 			next-level-cache = <&A53_L2>;
 			clocks = <&scpi_dvfs 1>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <578>;
 		};
 
 		A53_3: cpu@103 {
@@ -140,6 +145,7 @@
 			next-level-cache = <&A53_L2>;
 			clocks = <&scpi_dvfs 1>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <578>;
 		};
 
 		A57_L2: l2-cache0 {
diff --git a/arch/arm64/boot/dts/arm/juno-r2.dts b/arch/arm64/boot/dts/arm/juno-r2.dts
index 26aaa6a7670f..28f40ec44090 100644
--- a/arch/arm64/boot/dts/arm/juno-r2.dts
+++ b/arch/arm64/boot/dts/arm/juno-r2.dts
@@ -90,6 +90,7 @@
 			next-level-cache = <&A72_L2>;
 			clocks = <&scpi_dvfs 0>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <1024>;
 		};
 
 		A72_1: cpu@1 {
@@ -100,6 +101,7 @@
 			next-level-cache = <&A72_L2>;
 			clocks = <&scpi_dvfs 0>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <1024>;
 		};
 
 		A53_0: cpu@100 {
@@ -110,6 +112,7 @@
 			next-level-cache = <&A53_L2>;
 			clocks = <&scpi_dvfs 1>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <485>;
 		};
 
 		A53_1: cpu@101 {
@@ -120,6 +123,7 @@
 			next-level-cache = <&A53_L2>;
 			clocks = <&scpi_dvfs 1>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <485>;
 		};
 
 		A53_2: cpu@102 {
@@ -130,6 +134,7 @@
 			next-level-cache = <&A53_L2>;
 			clocks = <&scpi_dvfs 1>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <485>;
 		};
 
 		A53_3: cpu@103 {
@@ -140,6 +145,7 @@
 			next-level-cache = <&A53_L2>;
 			clocks = <&scpi_dvfs 1>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <485>;
 		};
 
 		A72_L2: l2-cache0 {
diff --git a/arch/arm64/boot/dts/arm/juno.dts b/arch/arm64/boot/dts/arm/juno.dts
index 6e154d948a80..ac5ceb73f45f 100644
--- a/arch/arm64/boot/dts/arm/juno.dts
+++ b/arch/arm64/boot/dts/arm/juno.dts
@@ -90,6 +90,7 @@
 			next-level-cache = <&A57_L2>;
 			clocks = <&scpi_dvfs 0>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <1024>;
 		};
 
 		A57_1: cpu@1 {
@@ -100,6 +101,7 @@
 			next-level-cache = <&A57_L2>;
 			clocks = <&scpi_dvfs 0>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <1024>;
 		};
 
 		A53_0: cpu@100 {
@@ -110,6 +112,7 @@
 			next-level-cache = <&A53_L2>;
 			clocks = <&scpi_dvfs 1>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <578>;
 		};
 
 		A53_1: cpu@101 {
@@ -120,6 +123,7 @@
 			next-level-cache = <&A53_L2>;
 			clocks = <&scpi_dvfs 1>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <578>;
 		};
 
 		A53_2: cpu@102 {
@@ -130,6 +134,7 @@
 			next-level-cache = <&A53_L2>;
 			clocks = <&scpi_dvfs 1>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <578>;
 		};
 
 		A53_3: cpu@103 {
@@ -140,6 +145,7 @@
 			next-level-cache = <&A53_L2>;
 			clocks = <&scpi_dvfs 1>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <578>;
 		};
 
 		A57_L2: l2-cache0 {
diff --git a/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b.dts b/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b.dts
index 7841b724e340..c309633a1e87 100644
--- a/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b.dts
+++ b/arch/arm64/boot/dts/broadcom/bcm2837-rpi-3-b.dts
@@ -2,6 +2,7 @@
 #include "bcm2837.dtsi"
 #include "bcm2835-rpi.dtsi"
 #include "bcm283x-rpi-smsc9514.dtsi"
+#include "bcm283x-rpi-usb-host.dtsi"
 
 / {
 	compatible = "raspberrypi,3-model-b", "brcm,bcm2837";
@@ -15,13 +16,6 @@
 		act {
 			gpios = <&gpio 47 0>;
 		};
-
-		pwr {
-			label = "PWR";
-			gpios = <&gpio 35 0>;
-			default-state = "keep";
-			linux,default-trigger = "default-on";
-		};
 	};
 };
 
diff --git a/arch/arm64/boot/dts/broadcom/bcm2837.dtsi b/arch/arm64/boot/dts/broadcom/bcm2837.dtsi
index 8216bbb29fe0..19f2fe620a21 100644
--- a/arch/arm64/boot/dts/broadcom/bcm2837.dtsi
+++ b/arch/arm64/boot/dts/broadcom/bcm2837.dtsi
@@ -1,7 +1,7 @@
 #include "bcm283x.dtsi"
 
 / {
-	compatible = "brcm,bcm2836";
+	compatible = "brcm,bcm2837";
 
 	soc {
 		ranges = <0x7e000000 0x3f000000 0x1000000>,
@@ -74,3 +74,9 @@
 	interrupt-parent = <&local_intc>;
 	interrupts = <8>;
 };
+
+/* enable thermal sensor with the correct compatible property set */
+&thermal {
+	compatible = "brcm,bcm2837-thermal";
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/broadcom/bcm283x-rpi-usb-host.dtsi b/arch/arm64/boot/dts/broadcom/bcm283x-rpi-usb-host.dtsi
new file mode 120000
index 000000000000..cbeebe312ff8
--- /dev/null
+++ b/arch/arm64/boot/dts/broadcom/bcm283x-rpi-usb-host.dtsi
@@ -0,0 +1 @@
+../../../../arm/boot/dts/bcm283x-rpi-usb-host.dtsi
\ No newline at end of file
diff --git a/arch/arm64/boot/dts/broadcom/ns2-svk.dts b/arch/arm64/boot/dts/broadcom/ns2-svk.dts
index c4d544244b19..de8d379f44e2 100644
--- a/arch/arm64/boot/dts/broadcom/ns2-svk.dts
+++ b/arch/arm64/boot/dts/broadcom/ns2-svk.dts
@@ -161,6 +161,10 @@
 	status = "ok";
 };
 
+&sdio1 {
+	status = "ok";
+};
+
 &nand {
 	nandcs@0 {
 		compatible = "brcm,nandcs";
@@ -192,3 +196,37 @@
 		groups = "nand_grp";
 	};
 };
+
+&qspi {
+	bspi-sel = <0>;
+	flash: m25p80@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "m25p80";
+		reg = <0x0>;
+		spi-max-frequency = <12500000>;
+		m25p,fast-read;
+		spi-cpol;
+		spi-cpha;
+
+		partition@0 {
+			label = "boot";
+			reg = <0x00000000 0x000a0000>;
+		};
+
+		partition@a0000 {
+			label = "env";
+			reg = <0x000a0000 0x00060000>;
+		};
+
+		partition@100000 {
+			label = "system";
+			reg = <0x00100000 0x00600000>;
+		};
+
+		partition@700000 {
+			label = "rootfs";
+			reg = <0x00700000 0x01900000>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/broadcom/ns2.dtsi b/arch/arm64/boot/dts/broadcom/ns2.dtsi
index 773ed593da4d..4fcdeca3a983 100644
--- a/arch/arm64/boot/dts/broadcom/ns2.dtsi
+++ b/arch/arm64/boot/dts/broadcom/ns2.dtsi
@@ -133,6 +133,9 @@
 
 		status = "disabled";
 
+		phys = <&pci_phy0>;
+		phy-names = "pcie-phy";
+
 		msi-parent = <&msi0>;
 		msi0: msi@20020000 {
 			compatible = "brcm,iproc-msi";
@@ -171,6 +174,9 @@
 
 		status = "disabled";
 
+		phys = <&pci_phy1>;
+		phy-names = "pcie-phy";
+
 		msi-parent = <&msi4>;
 		msi4: msi@50020000 {
 			compatible = "brcm,iproc-msi";
@@ -203,6 +209,42 @@
 			status = "disabled";
 		};
 
+		pdc0: iproc-pdc0@612c0000 {
+			compatible = "brcm,iproc-pdc-mbox";
+			reg = <0x612c0000 0x445>;  /* PDC FS0 regs */
+			interrupts = <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+			brcm,rx-status-len = <32>;
+			brcm,use-bcm-hdr;
+		};
+
+		pdc1: iproc-pdc1@612e0000 {
+			compatible = "brcm,iproc-pdc-mbox";
+			reg = <0x612e0000 0x445>;  /* PDC FS1 regs */
+			interrupts = <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+			brcm,rx-status-len = <32>;
+			brcm,use-bcm-hdr;
+		};
+
+		pdc2: iproc-pdc2@61300000 {
+			compatible = "brcm,iproc-pdc-mbox";
+			reg = <0x61300000 0x445>;  /* PDC FS2 regs */
+			interrupts = <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+			brcm,rx-status-len = <32>;
+			brcm,use-bcm-hdr;
+		};
+
+		pdc3: iproc-pdc3@61320000 {
+			compatible = "brcm,iproc-pdc-mbox";
+			reg = <0x61320000 0x445>;  /* PDC FS3 regs */
+			interrupts = <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+			brcm,rx-status-len = <32>;
+			brcm,use-bcm-hdr;
+		};
+
 		dma0: dma@61360000 {
 			compatible = "arm,pl330", "arm,primecell";
 			reg = <0x61360000 0x1000>;
@@ -260,7 +302,7 @@
 				     <GIC_SPI 260 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 261 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 262 IRQ_TYPE_LEVEL_HIGH>;
-			mmu-masters;
+			#iommu-cells = <1>;
 		};
 
 		pinctrl: pinctrl@6501d130 {
@@ -577,5 +619,23 @@
 
 			brcm,nand-has-wp;
 		};
+
+		qspi: spi@66470200 {
+			compatible = "brcm,spi-bcm-qspi", "brcm,spi-ns2-qspi";
+			reg = <0x66470200 0x184>,
+				<0x66470000 0x124>,
+				<0x67017408 0x004>,
+				<0x664703a0 0x01c>;
+			reg-names = "mspi", "bspi", "intr_regs",
+				"intr_status_reg";
+			interrupts = <GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "spi_l1_intr";
+			clocks = <&iprocmed>;
+			clock-names = "iprocmed";
+			num-cs = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
 	};
 };
diff --git a/arch/arm64/boot/dts/exynos/Makefile b/arch/arm64/boot/dts/exynos/Makefile
index 50c9b9383cfa..7ddea53769a7 100644
--- a/arch/arm64/boot/dts/exynos/Makefile
+++ b/arch/arm64/boot/dts/exynos/Makefile
@@ -1,4 +1,7 @@
-dtb-$(CONFIG_ARCH_EXYNOS) += exynos7-espresso.dtb
+dtb-$(CONFIG_ARCH_EXYNOS) += \
+	exynos5433-tm2.dtb	\
+	exynos5433-tm2e.dtb	\
+	exynos7-espresso.dtb
 
 always		:= $(dtb-y)
 subdir-y	:= $(dts-dirs)
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-pinctrl.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-pinctrl.dtsi
new file mode 100644
index 000000000000..ad71247b074f
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos5433-pinctrl.dtsi
@@ -0,0 +1,804 @@
+/*
+ * Samsung's Exynos5433 SoC pin-mux and pin-config device tree source
+ *
+ * Copyright (c) 2016 Samsung Electronics Co., Ltd.
+ * Chanwoo Choi <cw00.choi@samsung.com>
+ *
+ * Samsung's Exynos5433 SoC pin-mux and pin-config options are listed as device
+ * tree nodes are listed in this file.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#define PIN_PULL_NONE		0
+#define PIN_PULL_DOWN		1
+#define PIN_PULL_UP		3
+
+#define PIN_DRV_LV1		0
+#define PIN_DRV_LV2		2
+#define PIN_DRV_LV3		1
+#define PIN_DRV_LV4		3
+
+#define PIN_IN			0
+#define PIN_OUT			1
+#define PIN_FUNC1		2
+
+#define PIN(_func, _pin, _pull, _drv)			\
+	_pin {						\
+		samsung,pins = #_pin;			\
+		samsung,pin-function = <PIN_ ##_func>;	\
+		samsung,pin-pud = <PIN_PULL_ ##_pull>;	\
+		samsung,pin-drv = <PIN_DRV_ ##_drv>;	\
+	}
+
+&pinctrl_alive {
+	gpa0: gpa0 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
+		#interrupt-cells = <2>;
+	};
+
+	gpa1: gpa1 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
+		#interrupt-cells = <2>;
+	};
+
+	gpa2: gpa2 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpa3: gpa3 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpf1: gpf1 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpf2: gpf2 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpf3: gpf3 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpf4: gpf4 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpf5: gpf5 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+};
+
+&pinctrl_aud {
+	gpz0: gpz0 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpz1: gpz1 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	i2s0_bus: i2s0-bus {
+		samsung,pins = "gpz0-0", "gpz0-1", "gpz0-2", "gpz0-3",
+				"gpz0-4", "gpz0-5", "gpz0-6";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <1>;
+		samsung,pin-drv = <0>;
+	};
+
+	pcm0_bus: pcm0-bus {
+		samsung,pins = "gpz1-0", "gpz1-1", "gpz1-2", "gpz1-3";
+		samsung,pin-function = <3>;
+		samsung,pin-pud = <1>;
+		samsung,pin-drv = <0>;
+	};
+
+	uart_aud_bus: uart-aud-bus {
+		samsung,pins = "gpz1-3", "gpz1-2", "gpz1-1", "gpz1-0";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+};
+
+&pinctrl_cpif {
+	gpv6: gpv6 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+};
+
+&pinctrl_ese {
+	gpj2: gpj2 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+};
+
+&pinctrl_finger {
+	gpd5: gpd5 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	spi2_bus: spi2-bus {
+		samsung,pins = "gpd5-0", "gpd5-2", "gpd5-3";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	hs_i2c6_bus: hs-i2c6-bus {
+		samsung,pins = "gpd5-3", "gpd5-2";
+		samsung,pin-function = <4>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+};
+
+&pinctrl_fsys {
+	gph1: gph1 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpr4: gpr4 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpr0: gpr0 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpr1: gpr1 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpr2: gpr2 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpr3: gpr3 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	sd0_clk: sd0-clk {
+		samsung,pins = "gpr0-0";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd0_cmd: sd0-cmd {
+		samsung,pins = "gpr0-1";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd0_rdqs: sd0-rdqs {
+		samsung,pins = "gpr0-2";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <1>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd0_qrdy: sd0-qrdy {
+		samsung,pins = "gpr0-3";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <1>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd0_bus1: sd0-bus-width1 {
+		samsung,pins = "gpr1-0";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd0_bus4: sd0-bus-width4 {
+		samsung,pins = "gpr1-1", "gpr1-2", "gpr1-3";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd0_bus8: sd0-bus-width8 {
+		samsung,pins = "gpr1-4", "gpr1-5", "gpr1-6", "gpr1-7";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd1_clk: sd1-clk {
+		samsung,pins = "gpr2-0";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd1_cmd: sd1-cmd {
+		samsung,pins = "gpr2-1";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd1_bus1: sd1-bus-width1 {
+		samsung,pins = "gpr3-0";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd1_bus4: sd1-bus-width4 {
+		samsung,pins = "gpr3-1", "gpr3-2", "gpr3-3";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd1_bus8: sd1-bus-width8 {
+		samsung,pins = "gpr3-4", "gpr3-5", "gpr3-6", "gpr3-7";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <3>;
+	};
+
+	pcie_bus: pcie_bus {
+		samsung,pins = "gpr3-4", "gpr3-5", "gpr3-6", "gpr3-7";
+		samsung,pin-function = <3>;
+		samsung,pin-pud = <3>;
+	};
+
+	sd2_clk: sd2-clk {
+		samsung,pins = "gpr4-0";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd2_cmd: sd2-cmd {
+		samsung,pins = "gpr4-1";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd2_cd: sd2-cd {
+		samsung,pins = "gpr4-2";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd2_bus1: sd2-bus-width1 {
+		samsung,pins = "gpr4-3";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd2_bus4: sd2-bus-width4 {
+		samsung,pins = "gpr4-4", "gpr4-5", "gpr4-6";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <3>;
+	};
+
+	sd2_clk_output: sd2-clk-output {
+		samsung,pins = "gpr4-0";
+		samsung,pin-function = <1>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <2>;
+	};
+
+	sd2_cmd_output: sd2-cmd-output {
+		samsung,pins = "gpr4-1";
+		samsung,pin-function = <1>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <2>;
+	};
+};
+
+&pinctrl_imem {
+	gpf0: gpf0 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+};
+
+&pinctrl_nfc {
+	gpj0: gpj0 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	hs_i2c4_bus: hs-i2c4-bus {
+		samsung,pins = "gpj0-1", "gpj0-0";
+		samsung,pin-function = <4>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+};
+
+&pinctrl_peric {
+	gpv7: gpv7 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpb0: gpb0 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpc0: gpc0 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpc1: gpc1 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpc2: gpc2 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpc3: gpc3 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpg0: gpg0 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpd0: gpd0 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpd1: gpd1 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpd2: gpd2 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpd4: gpd4 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpd8: gpd8 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpd6: gpd6 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpd7: gpd7 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpg1: gpg1 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpg2: gpg2 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpg3: gpg3 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	hs_i2c8_bus: hs-i2c8-bus {
+		samsung,pins = "gpb0-1", "gpb0-0";
+		samsung,pin-function = <4>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	hs_i2c9_bus: hs-i2c9-bus {
+		samsung,pins = "gpb0-3", "gpb0-2";
+		samsung,pin-function = <4>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	i2s1_bus: i2s1-bus {
+		samsung,pins = "gpd4-0", "gpd4-1", "gpd4-2",
+				"gpd4-3", "gpd4-4";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <1>;
+		samsung,pin-drv = <0>;
+	};
+
+	pcm1_bus: pcm1-bus {
+		samsung,pins = "gpd4-0", "gpd4-1", "gpd4-2",
+				"gpd4-3", "gpd4-4";
+		samsung,pin-function = <3>;
+		samsung,pin-pud = <1>;
+		samsung,pin-drv = <0>;
+	};
+
+	spdif_bus: spdif-bus {
+		samsung,pins = "gpd4-3", "gpd4-4";
+		samsung,pin-function = <4>;
+		samsung,pin-pud = <1>;
+		samsung,pin-drv = <0>;
+	};
+
+	fimc_is_spi_pin0: fimc-is-spi-pin0 {
+		samsung,pins = "gpc3-3", "gpc3-2", "gpc3-1", "gpc3-0";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
+	fimc_is_spi_pin1: fimc-is-spi-pin1 {
+		samsung,pins = "gpc3-7", "gpc3-6", "gpc3-5", "gpc3-4";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
+	uart0_bus: uart0-bus {
+		samsung,pins = "gpd0-3", "gpd0-2", "gpd0-1", "gpd0-0";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+	};
+
+	hs_i2c2_bus: hs-i2c2-bus {
+		samsung,pins = "gpd0-3", "gpd0-2";
+		samsung,pin-function = <3>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	uart2_bus: uart2-bus {
+		samsung,pins = "gpd1-5", "gpd1-4";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+	};
+
+	uart1_bus: uart1-bus {
+		samsung,pins = "gpd1-3", "gpd1-2", "gpd1-1", "gpd1-0";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+	};
+
+	hs_i2c3_bus: hs-i2c3-bus {
+		samsung,pins = "gpd1-3", "gpd1-2";
+		samsung,pin-function = <3>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	hs_i2c0_bus: hs-i2c0-bus {
+		samsung,pins = "gpd2-1", "gpd2-0";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	hs_i2c1_bus: hs-i2c1-bus {
+		samsung,pins = "gpd2-3", "gpd2-2";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	pwm0_out: pwm0-out {
+		samsung,pins = "gpd2-4";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
+	pwm1_out: pwm1-out {
+		samsung,pins = "gpd2-5";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
+	pwm2_out: pwm2-out {
+		samsung,pins = "gpd2-6";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
+	pwm3_out: pwm3-out {
+		samsung,pins = "gpd2-7";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
+	spi1_bus: spi1-bus {
+		samsung,pins = "gpd6-2", "gpd6-4", "gpd6-5";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	hs_i2c7_bus: hs-i2c7-bus {
+		samsung,pins = "gpd2-7", "gpd2-6";
+		samsung,pin-function = <4>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	spi0_bus: spi0-bus {
+		samsung,pins = "gpd8-0", "gpd6-0", "gpd6-1";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	hs_i2c10_bus: hs-i2c10-bus {
+		samsung,pins = "gpg3-1", "gpg3-0";
+		samsung,pin-function = <4>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	hs_i2c11_bus: hs-i2c11-bus {
+		samsung,pins = "gpg3-3", "gpg3-2";
+		samsung,pin-function = <4>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	spi3_bus: spi3-bus {
+		samsung,pins = "gpg3-4", "gpg3-6", "gpg3-7";
+		samsung,pin-function = <3>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	spi4_bus: spi4-bus {
+		samsung,pins = "gpv7-1", "gpv7-3", "gpv7-4";
+		samsung,pin-function = <3>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+
+	fimc_is_uart: fimc-is-uart {
+		samsung,pins = "gpc1-1", "gpc0-7";
+		samsung,pin-function = <3>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
+	fimc_is_ch0_i2c: fimc-is-ch0_i2c {
+		samsung,pins = "gpc2-1", "gpc2-0";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
+	fimc_is_ch0_mclk: fimc-is-ch0_mclk {
+		samsung,pins = "gpd7-0";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
+	fimc_is_ch1_i2c: fimc-is-ch1-i2c {
+		samsung,pins = "gpc2-3", "gpc2-2";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
+	fimc_is_ch1_mclk: fimc-is-ch1-mclk {
+		samsung,pins = "gpd7-1";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
+	fimc_is_ch2_i2c: fimc-is-ch2-i2c {
+		samsung,pins = "gpc2-5", "gpc2-4";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+
+	fimc_is_ch2_mclk: fimc-is-ch2-mclk {
+		samsung,pins = "gpd7-2";
+		samsung,pin-function = <2>;
+		samsung,pin-pud = <0>;
+		samsung,pin-drv = <0>;
+	};
+};
+
+&pinctrl_touch {
+	gpj1: gpj1 {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	hs_i2c5_bus: hs-i2c5-bus {
+		samsung,pins = "gpj1-1", "gpj1-0";
+		samsung,pin-function = <4>;
+		samsung,pin-pud = <3>;
+		samsung,pin-drv = <0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tm2.dts b/arch/arm64/boot/dts/exynos/exynos5433-tm2.dts
new file mode 100644
index 000000000000..f21bdc2ff834
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tm2.dts
@@ -0,0 +1,1049 @@
+/*
+ * SAMSUNG Exynos5433 TM2 board device tree source
+ *
+ * Copyright (c) 2016 Samsung Electronics Co., Ltd.
+ *
+ * Device tree source file for Samsung's TM2 board which is based on
+ * Samsung Exynos5433 SoC.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/dts-v1/;
+#include "exynos5433.dtsi"
+#include <dt-bindings/clock/samsung,s2mps11.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	model = "Samsung TM2 board";
+	compatible = "samsung,tm2", "samsung,exynos5433";
+
+	aliases {
+		gsc0 = &gsc_0;
+		gsc1 = &gsc_1;
+		gsc2 = &gsc_2;
+		pinctrl0 = &pinctrl_alive;
+		pinctrl1 = &pinctrl_aud;
+		pinctrl2 = &pinctrl_cpif;
+		pinctrl3 = &pinctrl_ese;
+		pinctrl4 = &pinctrl_finger;
+		pinctrl5 = &pinctrl_fsys;
+		pinctrl6 = &pinctrl_imem;
+		pinctrl7 = &pinctrl_nfc;
+		pinctrl8 = &pinctrl_peric;
+		pinctrl9 = &pinctrl_touch;
+		serial0 = &serial_0;
+		serial1 = &serial_1;
+		serial2 = &serial_2;
+		serial3 = &serial_3;
+		spi0 = &spi_0;
+		spi1 = &spi_1;
+		spi2 = &spi_2;
+		spi3 = &spi_3;
+		spi4 = &spi_4;
+		mshc0 = &mshc_0;
+		mshc2 = &mshc_2;
+	};
+
+	chosen {
+		stdout-path = &serial_1;
+	};
+
+	memory@20000000 {
+		device_type = "memory";
+		reg = <0x0 0x20000000 0x0 0xc0000000>;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		power-key {
+			gpios = <&gpa2 7 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_POWER>;
+			label = "power key";
+			debounce-interval = <10>;
+		};
+
+		volume-up-key {
+			gpios = <&gpa2 0 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+			label = "volume-up key";
+			debounce-interval = <10>;
+		};
+
+		volume-down-key {
+			gpios = <&gpa2 1 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEDOWN>;
+			label = "volume-down key";
+			debounce-interval = <10>;
+		};
+
+		homepage-key {
+			gpios = <&gpa0 3 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_MENU>;
+			label = "homepage key";
+			debounce-interval = <10>;
+		};
+	};
+
+	i2c_max98504: i2c-gpio-0 {
+		compatible = "i2c-gpio";
+		gpios = <&gpd0 1 GPIO_ACTIVE_HIGH /* SPK_AMP_SDA */
+			 &gpd0 0 GPIO_ACTIVE_HIGH /* SPK_AMP_SCL */ >;
+		i2c-gpio,delay-us = <2>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "okay";
+
+		max98504: max98504@31 {
+			compatible = "maxim,max98504";
+			reg = <0x31>;
+			maxim,rx-path = <1>;
+			maxim,tx-path = <1>;
+			maxim,tx-channel-mask = <3>;
+			maxim,tx-channel-source = <2>;
+		};
+	};
+
+	sound {
+		compatible = "samsung,tm2-audio";
+		audio-codec = <&wm5110>;
+		i2s-controller = <&i2s0>;
+		audio-amplifier = <&max98504>;
+		mic-bias-gpios = <&gpr3 2 GPIO_ACTIVE_HIGH>;
+		model = "wm5110";
+		samsung,audio-routing =
+			/* Headphone */
+			"HP", "HPOUT1L",
+			"HP", "HPOUT1R",
+
+			/* Speaker */
+			"SPK", "SPKOUT",
+			"SPKOUT", "HPOUT2L",
+			"SPKOUT", "HPOUT2R",
+
+			/* Receiver */
+			"RCV", "HPOUT3L",
+			"RCV", "HPOUT3R";
+		status = "okay";
+	};
+};
+
+&adc {
+	vdd-supply = <&ldo3_reg>;
+	status = "okay";
+
+	thermistor-ap {
+		compatible = "murata,ncp03wf104";
+		pullup-uv = <1800000>;
+		pullup-ohm = <100000>;
+		pulldown-ohm = <0>;
+		io-channels = <&adc 0>;
+	};
+
+	thermistor-battery {
+		compatible = "murata,ncp03wf104";
+		pullup-uv = <1800000>;
+		pullup-ohm = <100000>;
+		pulldown-ohm = <0>;
+		io-channels = <&adc 1>;
+		#thermal-sensor-cells = <0>;
+	};
+
+	thermistor-charger {
+		compatible = "murata,ncp03wf104";
+		pullup-uv = <1800000>;
+		pullup-ohm = <100000>;
+		pulldown-ohm = <0>;
+		io-channels = <&adc 2>;
+	};
+};
+
+&cmu_aud {
+	assigned-clocks = <&cmu_aud CLK_MOUT_AUD_PLL_USER>;
+	assigned-clock-parents = <&cmu_top CLK_FOUT_AUD_PLL>;
+};
+
+&cmu_fsys {
+	assigned-clocks = <&cmu_top CLK_MOUT_SCLK_USBDRD30>,
+		<&cmu_top CLK_MOUT_SCLK_USBHOST30>,
+		<&cmu_fsys CLK_MOUT_SCLK_USBDRD30_USER>,
+		<&cmu_fsys CLK_MOUT_SCLK_USBHOST30_USER>,
+		<&cmu_fsys CLK_MOUT_PHYCLK_USBDRD30_UDRD30_PIPE_PCLK_USER>,
+		<&cmu_fsys CLK_MOUT_PHYCLK_USBHOST30_UHOST30_PIPE_PCLK_USER>,
+		<&cmu_fsys CLK_MOUT_PHYCLK_USBDRD30_UDRD30_PHYCLOCK_USER>,
+		<&cmu_fsys CLK_MOUT_PHYCLK_USBHOST30_UHOST30_PHYCLOCK_USER>,
+		<&cmu_top CLK_DIV_SCLK_USBDRD30>,
+		<&cmu_top CLK_DIV_SCLK_USBHOST30>;
+	assigned-clock-parents = <&cmu_top CLK_MOUT_BUS_PLL_USER>,
+		<&cmu_top CLK_MOUT_BUS_PLL_USER>,
+		<&cmu_top CLK_SCLK_USBDRD30_FSYS>,
+		<&cmu_top CLK_SCLK_USBHOST30_FSYS>,
+		<&cmu_fsys CLK_PHYCLK_USBDRD30_UDRD30_PIPE_PCLK_PHY>,
+		<&cmu_fsys CLK_PHYCLK_USBHOST30_UHOST30_PIPE_PCLK_PHY>,
+		<&cmu_fsys CLK_PHYCLK_USBDRD30_UDRD30_PHYCLOCK_PHY>,
+		<&cmu_fsys CLK_PHYCLK_USBHOST30_UHOST30_PHYCLOCK_PHY>;
+	assigned-clock-rates = <0>, <0>, <0>, <0>, <0>, <0>, <0>, <0>,
+			       <66700000>, <66700000>;
+};
+
+&cmu_gscl {
+	assigned-clocks = <&cmu_gscl CLK_MOUT_ACLK_GSCL_111_USER>,
+			  <&cmu_gscl CLK_MOUT_ACLK_GSCL_333_USER>;
+	assigned-clock-parents = <&cmu_top CLK_ACLK_GSCL_111>,
+				 <&cmu_top CLK_ACLK_GSCL_333>;
+};
+
+&cmu_mfc {
+	assigned-clocks = <&cmu_mfc CLK_MOUT_ACLK_MFC_400_USER>;
+	assigned-clock-parents = <&cmu_top CLK_ACLK_MFC_400>;
+};
+
+&cmu_mscl {
+	assigned-clocks = <&cmu_mscl CLK_MOUT_ACLK_MSCL_400_USER>,
+			  <&cmu_mscl CLK_MOUT_SCLK_JPEG_USER>,
+			  <&cmu_mscl CLK_MOUT_SCLK_JPEG>,
+			  <&cmu_top CLK_MOUT_SCLK_JPEG_A>;
+	assigned-clock-parents = <&cmu_top CLK_ACLK_MSCL_400>,
+				 <&cmu_top CLK_SCLK_JPEG_MSCL>,
+				 <&cmu_mscl CLK_MOUT_SCLK_JPEG_USER>,
+				 <&cmu_top CLK_MOUT_BUS_PLL_USER>;
+};
+
+&cpu0 {
+	cpu-supply = <&buck3_reg>;
+};
+
+&cpu4 {
+	cpu-supply = <&buck2_reg>;
+};
+
+&decon {
+	status = "okay";
+
+	i80-if-timings {
+	};
+};
+
+&dsi {
+	status = "okay";
+	vddcore-supply = <&ldo6_reg>;
+	vddio-supply = <&ldo7_reg>;
+	samsung,pll-clock-frequency = <24000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&te_irq>;
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@1 {
+			reg = <1>;
+
+			dsi_out: endpoint {
+				samsung,burst-clock-frequency = <512000000>;
+				samsung,esc-clock-frequency = <16000000>;
+			};
+		};
+	};
+};
+
+&hsi2c_0 {
+	status = "okay";
+	clock-frequency = <2500000>;
+
+	s2mps13-pmic@66 {
+		compatible = "samsung,s2mps13-pmic";
+		interrupt-parent = <&gpa0>;
+		interrupts = <7 IRQ_TYPE_NONE>;
+		reg = <0x66>;
+		samsung,s2mps11-wrstbi-ground;
+
+		s2mps13_osc: clocks {
+			compatible = "samsung,s2mps13-clk";
+			#clock-cells = <1>;
+			clock-output-names = "s2mps13_ap", "s2mps13_cp",
+				"s2mps13_bt";
+		};
+
+		regulators {
+			ldo1_reg: LDO1 {
+				regulator-name = "VDD_ALIVE_0.9V_AP";
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+			};
+
+			ldo2_reg: LDO2 {
+				regulator-name = "VDDQ_MMC2_2.8V_AP";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			ldo3_reg: LDO3 {
+				regulator-name = "VDD1_E_1.8V_AP";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+			};
+
+			ldo4_reg: LDO4 {
+				regulator-name = "VDD10_MIF_PLL_1.0V_AP";
+				regulator-min-microvolt = <1300000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			ldo5_reg: LDO5 {
+				regulator-name = "VDD10_DPLL_1.0V_AP";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			ldo6_reg: LDO6 {
+				regulator-name = "VDD10_MIPI2L_1.0V_AP";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			ldo7_reg: LDO7 {
+				regulator-name = "VDD18_MIPI2L_1.8V_AP";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			ldo8_reg: LDO8 {
+				regulator-name = "VDD18_LLI_1.8V_AP";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			ldo9_reg: LDO9 {
+				regulator-name = "VDD18_ABB_ETC_1.8V_AP";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			ldo10_reg: LDO10 {
+				regulator-name = "VDD33_USB30_3.0V_AP";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			ldo11_reg: LDO11 {
+				regulator-name = "VDD_INT_M_1.0V_AP";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			ldo12_reg: LDO12 {
+				regulator-name = "VDD_KFC_M_1.1V_AP";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-always-on;
+			};
+
+			ldo13_reg: LDO13 {
+				regulator-name = "VDD_G3D_M_0.95V_AP";
+				regulator-min-microvolt = <950000>;
+				regulator-max-microvolt = <950000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			ldo14_reg: LDO14 {
+				regulator-name = "VDDQ_M1_LDO_1.2V_AP";
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			ldo15_reg: LDO15 {
+				regulator-name = "VDDQ_M2_LDO_1.2V_AP";
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			ldo16_reg: LDO16 {
+				regulator-name = "VDDQ_EFUSE";
+				regulator-min-microvolt = <1400000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-always-on;
+			};
+
+			ldo17_reg: LDO17 {
+				regulator-name = "V_TFLASH_2.8V_AP";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+			};
+
+			ldo18_reg: LDO18 {
+				regulator-name = "V_CODEC_1.8V_AP";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			ldo19_reg: LDO19 {
+				regulator-name = "VDDA_1.8V_COMP";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+			};
+
+			ldo20_reg: LDO20 {
+				regulator-name = "VCC_2.8V_AP";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+				regulator-always-on;
+			};
+
+			ldo21_reg: LDO21 {
+				regulator-name = "VT_CAM_1.8V";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			ldo22_reg: LDO22 {
+				regulator-name = "CAM_IO_1.8V_AP";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			ldo23_reg: LDO23 {
+				regulator-name = "CAM_SEN_CORE_1.2V_AP";
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1200000>;
+			};
+
+			ldo24_reg: LDO24 {
+				regulator-name = "VT_CAM_1.2V";
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+			};
+
+			ldo25_reg: LDO25 {
+				regulator-name = "CAM_SEN_A2.8V_AP";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+			};
+
+			ldo26_reg: LDO26 {
+				regulator-name = "CAM_AF_2.8V_AP";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+			};
+
+			ldo27_reg: LDO27 {
+				regulator-name = "VCC_3.0V_LCD_AP";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+			};
+
+			ldo28_reg: LDO28 {
+				regulator-name = "VCC_1.8V_LCD_AP";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			ldo29_reg: LDO29 {
+				regulator-name = "VT_CAM_2.8V";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+			};
+
+			ldo30_reg: LDO30 {
+				regulator-name = "TSP_AVDD_3.3V_AP";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			ldo31_reg: LDO31 {
+				regulator-name = "TSP_VDD_1.85V_AP";
+				regulator-min-microvolt = <1850000>;
+				regulator-max-microvolt = <1850000>;
+			};
+
+			ldo32_reg: LDO32 {
+				regulator-name = "VTOUCH_1.8V_AP";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			ldo33_reg: LDO33 {
+				regulator-name = "VTOUCH_LED_3.3V";
+				regulator-min-microvolt = <2500000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-ramp-delay = <12500>;
+			};
+
+			ldo34_reg: LDO34 {
+				regulator-name = "VCC_1.8V_MHL_AP";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <2100000>;
+			};
+
+			ldo35_reg: LDO35 {
+				regulator-name = "OIS_VM_2.8V";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <2800000>;
+			};
+
+			ldo36_reg: LDO36 {
+				regulator-name = "VSIL_1.0V";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+			};
+
+			ldo37_reg: LDO37 {
+				regulator-name = "VF_1.8V";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			ldo38_reg: LDO38 {
+				regulator-name = "VCC_3.0V_MOTOR_AP";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+			};
+
+			ldo39_reg: LDO39 {
+				regulator-name = "V_HRM_1.8V";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			ldo40_reg: LDO40 {
+				regulator-name = "V_HRM_3.3V";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			buck1_reg: BUCK1 {
+				regulator-name = "VDD_MIF_0.9V_AP";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			buck2_reg: BUCK2 {
+				regulator-name = "VDD_EGL_1.0V_AP";
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			buck3_reg: BUCK3 {
+				regulator-name = "VDD_KFC_1.0V_AP";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			buck4_reg: BUCK4 {
+				regulator-name = "VDD_INT_0.95V_AP";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			buck5_reg: BUCK5 {
+				regulator-name = "VDD_DISP_CAM0_0.9V_AP";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			buck6_reg: BUCK6 {
+				regulator-name = "VDD_G3D_0.9V_AP";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-always-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			buck7_reg: BUCK7 {
+				regulator-name = "VDD_MEM1_1.2V_AP";
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+			};
+
+			buck8_reg: BUCK8 {
+				regulator-name = "VDD_LLDO_1.35V_AP";
+				regulator-min-microvolt = <1350000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			buck9_reg: BUCK9 {
+				regulator-name = "VDD_MLDO_2.0V_AP";
+				regulator-min-microvolt = <1350000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			buck10_reg: BUCK10 {
+				regulator-name = "vdd_mem2";
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&hsi2c_8 {
+	status = "okay";
+
+	max77843@66 {
+		compatible = "maxim,max77843";
+		interrupt-parent = <&gpa1>;
+		interrupts = <5 IRQ_TYPE_EDGE_FALLING>;
+		reg = <0x66>;
+
+		muic: max77843-muic {
+			compatible = "maxim,max77843-muic";
+		};
+
+		regulators {
+			compatible = "maxim,max77843-regulator";
+			safeout1_reg: SAFEOUT1 {
+				regulator-name = "SAFEOUT1";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <4950000>;
+			};
+
+			safeout2_reg: SAFEOUT2 {
+				regulator-name = "SAFEOUT2";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <4950000>;
+			};
+
+			charger_reg: CHARGER {
+				regulator-name = "CHARGER";
+				regulator-min-microamp = <100000>;
+				regulator-max-microamp = <3150000>;
+			};
+		};
+
+		haptic: max77843-haptic {
+			compatible = "maxim,max77843-haptic";
+			haptic-supply = <&ldo38_reg>;
+			pwms = <&pwm 0 33670 0>;
+			pwm-names = "haptic";
+		};
+	};
+};
+
+&i2s0 {
+	status = "okay";
+};
+
+&mshc_0 {
+	status = "okay";
+	num-slots = <1>;
+	mmc-hs200-1_8v;
+	mmc-hs400-1_8v;
+	cap-mmc-highspeed;
+	non-removable;
+	card-detect-delay = <200>;
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 4>;
+	samsung,dw-mshc-ddr-timing = <0 2>;
+	samsung,dw-mshc-hs400-timing = <0 3>;
+	samsung,read-strobe-delay = <90>;
+	fifo-depth = <0x80>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_qrdy &sd0_bus1 &sd0_bus4
+			&sd0_bus8 &sd0_rdqs>;
+	bus-width = <8>;
+	assigned-clocks = <&cmu_top CLK_SCLK_MMC0_FSYS>;
+	assigned-clock-rates = <800000000>;
+};
+
+&mshc_2 {
+	status = "okay";
+	num-slots = <1>;
+	cap-sd-highspeed;
+	disable-wp;
+	cd-gpios = <&gpa2 4 GPIO_ACTIVE_HIGH>;
+	cd-inverted;
+	card-detect-delay = <200>;
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 4>;
+	samsung,dw-mshc-ddr-timing = <0 2>;
+	fifo-depth = <0x80>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sd2_clk &sd2_cmd &sd2_bus1 &sd2_bus4>;
+	bus-width = <4>;
+};
+
+&pinctrl_alive {
+	pinctrl-names = "default";
+	pinctrl-0 = <&initial_alive>;
+
+	initial_alive: initial-state {
+		PIN(IN, gpa0-0, DOWN, LV1);
+		PIN(IN, gpa0-1, NONE, LV1);
+		PIN(IN, gpa0-2, DOWN, LV1);
+		PIN(IN, gpa0-3, NONE, LV1);
+		PIN(IN, gpa0-4, NONE, LV1);
+		PIN(IN, gpa0-5, DOWN, LV1);
+		PIN(IN, gpa0-6, NONE, LV1);
+		PIN(IN, gpa0-7, NONE, LV1);
+
+		PIN(IN, gpa1-0, UP, LV1);
+		PIN(IN, gpa1-1, NONE, LV1);
+		PIN(IN, gpa1-2, NONE, LV1);
+		PIN(IN, gpa1-3, DOWN, LV1);
+		PIN(IN, gpa1-4, DOWN, LV1);
+		PIN(IN, gpa1-5, NONE, LV1);
+		PIN(IN, gpa1-6, NONE, LV1);
+		PIN(IN, gpa1-7, NONE, LV1);
+
+		PIN(IN, gpa2-0, NONE, LV1);
+		PIN(IN, gpa2-1, NONE, LV1);
+		PIN(IN, gpa2-2, NONE, LV1);
+		PIN(IN, gpa2-3, DOWN, LV1);
+		PIN(IN, gpa2-4, NONE, LV1);
+		PIN(IN, gpa2-5, DOWN, LV1);
+		PIN(IN, gpa2-6, DOWN, LV1);
+		PIN(IN, gpa2-7, NONE, LV1);
+
+		PIN(IN, gpa3-0, DOWN, LV1);
+		PIN(IN, gpa3-1, DOWN, LV1);
+		PIN(IN, gpa3-2, NONE, LV1);
+		PIN(IN, gpa3-3, DOWN, LV1);
+		PIN(IN, gpa3-4, NONE, LV1);
+		PIN(IN, gpa3-5, DOWN, LV1);
+		PIN(IN, gpa3-6, DOWN, LV1);
+		PIN(IN, gpa3-7, DOWN, LV1);
+
+		PIN(IN, gpf1-0, NONE, LV1);
+		PIN(IN, gpf1-1, NONE, LV1);
+		PIN(IN, gpf1-2, DOWN, LV1);
+		PIN(IN, gpf1-4, UP, LV1);
+		PIN(OUT, gpf1-5, NONE, LV1);
+		PIN(IN, gpf1-6, DOWN, LV1);
+		PIN(IN, gpf1-7, DOWN, LV1);
+
+		PIN(IN, gpf2-0, DOWN, LV1);
+		PIN(IN, gpf2-1, DOWN, LV1);
+		PIN(IN, gpf2-2, DOWN, LV1);
+		PIN(IN, gpf2-3, DOWN, LV1);
+
+		PIN(IN, gpf3-0, DOWN, LV1);
+		PIN(IN, gpf3-1, DOWN, LV1);
+		PIN(IN, gpf3-2, NONE, LV1);
+		PIN(IN, gpf3-3, DOWN, LV1);
+
+		PIN(IN, gpf4-0, DOWN, LV1);
+		PIN(IN, gpf4-1, DOWN, LV1);
+		PIN(IN, gpf4-2, DOWN, LV1);
+		PIN(IN, gpf4-3, DOWN, LV1);
+		PIN(IN, gpf4-4, DOWN, LV1);
+		PIN(IN, gpf4-5, DOWN, LV1);
+		PIN(IN, gpf4-6, DOWN, LV1);
+		PIN(IN, gpf4-7, DOWN, LV1);
+
+		PIN(IN, gpf5-0, DOWN, LV1);
+		PIN(IN, gpf5-1, DOWN, LV1);
+		PIN(IN, gpf5-2, DOWN, LV1);
+		PIN(IN, gpf5-3, DOWN, LV1);
+		PIN(OUT, gpf5-4, NONE, LV1);
+		PIN(IN, gpf5-5, DOWN, LV1);
+		PIN(IN, gpf5-6, DOWN, LV1);
+		PIN(IN, gpf5-7, DOWN, LV1);
+	};
+
+	te_irq: te_irq {
+		samsung,pins = "gpf1-3";
+		samsung,pin-function = <0xf>;
+	};
+};
+
+&pinctrl_cpif {
+	pinctrl-names = "default";
+	pinctrl-0 = <&initial_cpif>;
+
+	initial_cpif: initial-state {
+		PIN(IN, gpv6-0, DOWN, LV1);
+		PIN(IN, gpv6-1, DOWN, LV1);
+	};
+};
+
+&pinctrl_ese {
+	pinctrl-names = "default";
+	pinctrl-0 = <&initial_ese>;
+
+	initial_ese: initial-state {
+		PIN(IN, gpj2-0, DOWN, LV1);
+		PIN(IN, gpj2-1, DOWN, LV1);
+		PIN(IN, gpj2-2, DOWN, LV1);
+	};
+};
+
+&pinctrl_fsys {
+	pinctrl-names = "default";
+	pinctrl-0 = <&initial_fsys>;
+
+	initial_fsys: initial-state {
+		PIN(IN, gpr3-0, NONE, LV1);
+		PIN(IN, gpr3-1, DOWN, LV1);
+		PIN(IN, gpr3-2, DOWN, LV1);
+		PIN(IN, gpr3-3, DOWN, LV1);
+		PIN(IN, gpr3-7, NONE, LV1);
+	};
+};
+
+&pinctrl_imem {
+	pinctrl-names = "default";
+	pinctrl-0 = <&initial_imem>;
+
+	initial_imem: initial-state {
+		PIN(IN, gpf0-0, UP, LV1);
+		PIN(IN, gpf0-1, UP, LV1);
+		PIN(IN, gpf0-2, DOWN, LV1);
+		PIN(IN, gpf0-3, UP, LV1);
+		PIN(IN, gpf0-4, DOWN, LV1);
+		PIN(IN, gpf0-5, NONE, LV1);
+		PIN(IN, gpf0-6, DOWN, LV1);
+		PIN(IN, gpf0-7, UP, LV1);
+	};
+};
+
+&pinctrl_nfc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&initial_nfc>;
+
+	initial_nfc: initial-state {
+		PIN(IN, gpj0-2, DOWN, LV1);
+	};
+};
+
+&pinctrl_peric {
+	pinctrl-names = "default";
+	pinctrl-0 = <&initial_peric>;
+
+	initial_peric: initial-state {
+		PIN(IN, gpv7-0, DOWN, LV1);
+		PIN(IN, gpv7-1, DOWN, LV1);
+		PIN(IN, gpv7-2, NONE, LV1);
+		PIN(IN, gpv7-3, DOWN, LV1);
+		PIN(IN, gpv7-4, DOWN, LV1);
+		PIN(IN, gpv7-5, DOWN, LV1);
+
+		PIN(IN, gpb0-4, DOWN, LV1);
+
+		PIN(IN, gpc0-2, DOWN, LV1);
+		PIN(IN, gpc0-5, DOWN, LV1);
+		PIN(IN, gpc0-7, DOWN, LV1);
+
+		PIN(IN, gpc1-1, DOWN, LV1);
+
+		PIN(IN, gpc3-4, NONE, LV1);
+		PIN(IN, gpc3-5, NONE, LV1);
+		PIN(IN, gpc3-6, NONE, LV1);
+		PIN(IN, gpc3-7, NONE, LV1);
+
+		PIN(OUT, gpg0-0, NONE, LV1);
+		PIN(FUNC1, gpg0-1, DOWN, LV1);
+
+		PIN(IN, gpd2-5, DOWN, LV1);
+
+		PIN(IN, gpd4-0, NONE, LV1);
+		PIN(IN, gpd4-1, DOWN, LV1);
+		PIN(IN, gpd4-2, DOWN, LV1);
+		PIN(IN, gpd4-3, DOWN, LV1);
+		PIN(IN, gpd4-4, DOWN, LV1);
+
+		PIN(IN, gpd6-3, DOWN, LV1);
+
+		PIN(IN, gpd8-1, UP, LV1);
+
+		PIN(IN, gpg1-0, DOWN, LV1);
+		PIN(IN, gpg1-1, DOWN, LV1);
+		PIN(IN, gpg1-2, DOWN, LV1);
+		PIN(IN, gpg1-3, DOWN, LV1);
+		PIN(IN, gpg1-4, DOWN, LV1);
+
+		PIN(IN, gpg2-0, DOWN, LV1);
+		PIN(IN, gpg2-1, DOWN, LV1);
+
+		PIN(IN, gpg3-0, DOWN, LV1);
+		PIN(IN, gpg3-1, DOWN, LV1);
+		PIN(IN, gpg3-5, DOWN, LV1);
+		PIN(IN, gpg3-7, DOWN, LV1);
+	};
+};
+
+&pinctrl_touch {
+	pinctrl-names = "default";
+	pinctrl-0 = <&initial_touch>;
+
+	initial_touch: initial-state {
+		PIN(IN, gpj1-2, DOWN, LV1);
+	};
+};
+
+&pwm {
+	pinctrl-0 = <&pwm0_out>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&mic {
+	status = "okay";
+
+	i80-if-timings {
+	};
+};
+
+&pmu_system_controller {
+	assigned-clocks = <&pmu_system_controller 0>;
+	assigned-clock-parents = <&xxti>;
+};
+
+&serial_1 {
+	status = "okay";
+};
+
+&spi_1 {
+	cs-gpios = <&gpd6 3 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+
+	wm5110: wm5110-codec@0 {
+		compatible = "wlf,wm5110";
+		reg = <0x0>;
+		spi-max-frequency = <20000000>;
+		interrupt-parent = <&gpa0>;
+		interrupts = <4 IRQ_TYPE_NONE>;
+		clocks = <&pmu_system_controller 0>,
+			<&s2mps13_osc S2MPS11_CLK_BT>;
+		clock-names = "mclk1", "mclk2";
+
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		wlf,micd-detect-debounce = <300>;
+		wlf,micd-bias-start-time = <0x1>;
+		wlf,micd-rate = <0x7>;
+		wlf,micd-dbtime = <0x1>;
+		wlf,micd-force-micbias;
+		wlf,micd-configs = <0x0 1 0>;
+		wlf,hpdet-channel = <1>;
+		wlf,gpsw = <0x1>;
+		wlf,inmode = <2 0 2 0>;
+
+		wlf,reset = <&gpc0 7 GPIO_ACTIVE_HIGH>;
+		wlf,ldoena = <&gpf0 0 GPIO_ACTIVE_HIGH>;
+
+		/* core supplies */
+		AVDD-supply = <&ldo18_reg>;
+		DBVDD1-supply = <&ldo18_reg>;
+		CPVDD-supply = <&ldo18_reg>;
+		DBVDD2-supply = <&ldo18_reg>;
+		DBVDD3-supply = <&ldo18_reg>;
+
+		controller-data {
+			samsung,spi-feedback-delay = <0>;
+		};
+	};
+};
+
+&timer {
+	clock-frequency = <24000000>;
+};
+
+&tmu_atlas0 {
+	vtmu-supply = <&ldo3_reg>;
+	status = "okay";
+};
+
+&tmu_apollo {
+	vtmu-supply = <&ldo3_reg>;
+	status = "okay";
+};
+
+&tmu_g3d {
+	vtmu-supply = <&ldo3_reg>;
+	status = "okay";
+};
+
+&usbdrd30 {
+	vdd33-supply = <&ldo10_reg>;
+	vdd10-supply = <&ldo6_reg>;
+	status = "okay";
+};
+
+&usbdrd_dwc3_0 {
+	dr_mode = "otg";
+};
+
+&usbdrd30_phy {
+	vbus-supply = <&safeout1_reg>;
+	status = "okay";
+};
+
+&xxti {
+	clock-frequency = <24000000>;
+};
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tm2e.dts b/arch/arm64/boot/dts/exynos/exynos5433-tm2e.dts
new file mode 100644
index 000000000000..1db4e7f363a9
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tm2e.dts
@@ -0,0 +1,41 @@
+/*
+ * SAMSUNG Exynos5433 TM2E board device tree source
+ *
+ * Copyright (c) 2016 Samsung Electronics Co., Ltd.
+ *
+ * Device tree source file for Samsung's TM2E(TM2 EDGE) board which is based on
+ * Samsung Exynos5433 SoC.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include "exynos5433-tm2.dts"
+
+/ {
+	model = "Samsung TM2E board";
+	compatible = "samsung,tm2e", "samsung,exynos5433";
+};
+
+&ldo23_reg {
+	regulator-name = "CAM_SEN_CORE_1.025V_AP";
+	regulator-max-microvolt = <1050000>;
+};
+
+&ldo25_reg {
+	regulator-name = "UNUSED_LDO25";
+	regulator-always-off;
+};
+
+&ldo31_reg {
+	regulator-name = "TSP_VDD_1.8V_AP";
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+};
+
+&ldo38_reg {
+	regulator-name = "VCC_3.3V_MOTOR_AP";
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+};
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tmu-g3d-sensor-conf.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-tmu-g3d-sensor-conf.dtsi
new file mode 100644
index 000000000000..9be2978f1b9a
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tmu-g3d-sensor-conf.dtsi
@@ -0,0 +1,23 @@
+/*
+ * Device tree sources for Exynos5433 TMU sensor configuration
+ *
+ * Copyright (c) 2016 Jonghwa Lee <jonghwa3.lee@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <dt-bindings/thermal/thermal_exynos.h>
+
+#thermal-sensor-cells = <0>;
+samsung,tmu_gain = <8>;
+samsung,tmu_reference_voltage = <23>;
+samsung,tmu_noise_cancel_mode = <4>;
+samsung,tmu_efuse_value = <75>;
+samsung,tmu_min_efuse_value = <40>;
+samsung,tmu_max_efuse_value = <150>;
+samsung,tmu_first_point_trim = <25>;
+samsung,tmu_second_point_trim = <85>;
+samsung,tmu_default_temp_offset = <50>;
+samsung,tmu_mux_addr = <6>;
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tmu-sensor-conf.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-tmu-sensor-conf.dtsi
new file mode 100644
index 000000000000..125fe58d77ce
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tmu-sensor-conf.dtsi
@@ -0,0 +1,22 @@
+/*
+ * Device tree sources for Exynos5433 TMU sensor configuration
+ *
+ * Copyright (c) 2016 Chanwoo Choi <cw00.choi@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <dt-bindings/thermal/thermal_exynos.h>
+
+#thermal-sensor-cells = <0>;
+samsung,tmu_gain = <8>;
+samsung,tmu_reference_voltage = <16>;
+samsung,tmu_noise_cancel_mode = <4>;
+samsung,tmu_efuse_value = <75>;
+samsung,tmu_min_efuse_value = <40>;
+samsung,tmu_max_efuse_value = <150>;
+samsung,tmu_first_point_trim = <25>;
+samsung,tmu_second_point_trim = <85>;
+samsung,tmu_default_temp_offset = <50>;
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tmu.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-tmu.dtsi
new file mode 100644
index 000000000000..ceaa05145b8a
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tmu.dtsi
@@ -0,0 +1,296 @@
+/*
+ * Device tree sources for Exynos5433 thermal zone
+ *
+ * Copyright (c) 2016 Chanwoo Choi <cw00.choi@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <dt-bindings/thermal/thermal.h>
+
+/ {
+thermal-zones {
+	atlas0_thermal: atlas0-thermal {
+		thermal-sensors = <&tmu_atlas0>;
+		polling-delay-passive = <0>;
+		polling-delay = <0>;
+		trips {
+			atlas0_alert_0: atlas0-alert-0 {
+				temperature = <65000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			atlas0_alert_1: atlas0-alert-1 {
+				temperature = <70000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			atlas0_alert_2: atlas0-alert-2 {
+				temperature = <75000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			atlas0_alert_3: atlas0-alert-3 {
+				temperature = <80000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			atlas0_alert_4: atlas0-alert-4 {
+				temperature = <85000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			atlas0_alert_5: atlas0-alert-5 {
+				temperature = <90000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			atlas0_alert_6: atlas0-alert-6 {
+				temperature = <95000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				/* Set maximum frequency as 1800MHz  */
+				trip = <&atlas0_alert_0>;
+				cooling-device = <&cpu4 1 2>;
+			};
+			map1 {
+				/* Set maximum frequency as 1700MHz  */
+				trip = <&atlas0_alert_1>;
+				cooling-device = <&cpu4 2 3>;
+			};
+			map2 {
+				/* Set maximum frequency as 1600MHz  */
+				trip = <&atlas0_alert_2>;
+				cooling-device = <&cpu4 3 4>;
+			};
+			map3 {
+				/* Set maximum frequency as 1500MHz  */
+				trip = <&atlas0_alert_3>;
+				cooling-device = <&cpu4 4 5>;
+			};
+			map4 {
+				/* Set maximum frequency as 1400MHz  */
+				trip = <&atlas0_alert_4>;
+				cooling-device = <&cpu4 5 7>;
+			};
+			map5 {
+				/* Set maximum frequencyas 1200MHz  */
+				trip = <&atlas0_alert_5>;
+				cooling-device = <&cpu4 7 9>;
+			};
+			map6 {
+				/* Set maximum frequency as 1000MHz  */
+				trip = <&atlas0_alert_6>;
+				cooling-device = <&cpu4 9 14>;
+			};
+		};
+	};
+
+	atlas1_thermal: atlas1-thermal {
+		thermal-sensors = <&tmu_atlas1>;
+		polling-delay-passive = <0>;
+		polling-delay = <0>;
+		trips {
+			atlas1_alert_0: atlas1-alert-0 {
+				temperature = <65000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			atlas1_alert_1: atlas1-alert-1 {
+				temperature = <70000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			atlas1_alert_2: atlas1-alert-2 {
+				temperature = <75000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			atlas1_alert_3: atlas1-alert-3 {
+				temperature = <80000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			atlas1_alert_4: atlas1-alert-4 {
+				temperature = <85000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			atlas1_alert_5: atlas1-alert-5 {
+				temperature = <90000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			atlas1_alert_6: atlas1-alert-6 {
+				temperature = <95000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+		};
+	};
+
+	g3d_thermal: g3d-thermal {
+		thermal-sensors = <&tmu_g3d>;
+		polling-delay-passive = <0>;
+		polling-delay = <0>;
+		trips {
+			g3d_alert_0: g3d-alert-0 {
+				temperature = <70000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			g3d_alert_1: g3d-alert-1 {
+				temperature = <75000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			g3d_alert_2: g3d-alert-2 {
+				temperature = <80000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			g3d_alert_3: g3d-alert-3 {
+				temperature = <85000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			g3d_alert_4: g3d-alert-4 {
+				temperature = <90000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			g3d_alert_5: g3d-alert-5 {
+				temperature = <95000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			g3d_alert_6: g3d-alert-6 {
+				temperature = <100000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+		};
+	};
+
+	apollo_thermal: apollo-thermal {
+		thermal-sensors = <&tmu_apollo>;
+		polling-delay-passive = <0>;
+		polling-delay = <0>;
+		trips {
+			apollo_alert_0: apollo-alert-0 {
+				temperature = <65000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			apollo_alert_1: apollo-alert-1 {
+				temperature = <70000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			apollo_alert_2: apollo-alert-2 {
+				temperature = <75000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			apollo_alert_3: apollo-alert-3 {
+				temperature = <80000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			apollo_alert_4: apollo-alert-4 {
+				temperature = <85000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			apollo_alert_5: apollo-alert-5 {
+				temperature = <90000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			apollo_alert_6: apollo-alert-6 {
+				temperature = <95000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				/* Set maximum frequency as 1200MHz  */
+				trip = <&apollo_alert_2>;
+				cooling-device = <&cpu0 1 2>;
+			};
+			map1 {
+				/* Set maximum frequency as 1100MHz  */
+				trip = <&apollo_alert_3>;
+				cooling-device = <&cpu0 2 3>;
+			};
+			map2 {
+				/* Set maximum frequency as 1000MHz  */
+				trip = <&apollo_alert_4>;
+				cooling-device = <&cpu0 3 4>;
+			};
+			map3 {
+				/* Set maximum frequency as 900MHz  */
+				trip = <&apollo_alert_5>;
+				cooling-device = <&cpu0 4 5>;
+			};
+			map4 {
+				/* Set maximum frequency as 800MHz  */
+				trip = <&apollo_alert_6>;
+				cooling-device = <&cpu0 5 9>;
+			};
+		};
+	};
+
+	isp_thermal: isp-thermal {
+		thermal-sensors = <&tmu_isp>;
+		polling-delay-passive = <0>;
+		polling-delay = <0>;
+		trips {
+			isp_alert_0: isp-alert-0 {
+				temperature = <80000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			isp_alert_1: isp-alert-1 {
+				temperature = <85000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			isp_alert_2: isp-alert-2 {
+				temperature = <90000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			isp_alert_3: isp-alert-3 {
+				temperature = <95000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			isp_alert_4: isp-alert-4 {
+				temperature = <100000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			isp_alert_5: isp-alert-5 {
+				temperature = <105000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+			isp_alert_6: isp-alert-6 {
+				temperature = <110000>;	/* millicelsius */
+				hysteresis = <1000>;	/* millicelsius */
+				type = "active";
+			};
+		};
+	};
+};
+};
diff --git a/arch/arm64/boot/dts/exynos/exynos5433.dtsi b/arch/arm64/boot/dts/exynos/exynos5433.dtsi
new file mode 100644
index 000000000000..64226d5ae471
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos5433.dtsi
@@ -0,0 +1,1462 @@
+/*
+ * Samsung's Exynos5433 SoC device tree source
+ *
+ * Copyright (c) 2016 Samsung Electronics Co., Ltd.
+ *
+ * Samsung's Exynos5433 SoC device nodes are listed in this file.
+ * Exynos5433 based board files can include this file and provide
+ * values for board specific bindings.
+ *
+ * Note: This file does not include device nodes for all the controllers in
+ * Exynos5433 SoC. As device tree coverage for Exynos5433 increases,
+ * additional nodes can be added to this file.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <dt-bindings/clock/exynos5433.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	compatible = "samsung,exynos5433";
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	interrupt-parent = <&gic>;
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			enable-method = "psci";
+			reg = <0x100>;
+			clock-frequency = <1300000000>;
+			clocks = <&cmu_apollo CLK_SCLK_APOLLO>;
+			clock-names = "apolloclk";
+			operating-points-v2 = <&cluster_a53_opp_table>;
+			#cooling-cells = <2>;
+		};
+
+		cpu1: cpu@101 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			enable-method = "psci";
+			reg = <0x101>;
+			clock-frequency = <1300000000>;
+			operating-points-v2 = <&cluster_a53_opp_table>;
+			#cooling-cells = <2>;
+		};
+
+		cpu2: cpu@102 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			enable-method = "psci";
+			reg = <0x102>;
+			clock-frequency = <1300000000>;
+			operating-points-v2 = <&cluster_a53_opp_table>;
+			#cooling-cells = <2>;
+		};
+
+		cpu3: cpu@103 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			enable-method = "psci";
+			reg = <0x103>;
+			clock-frequency = <1300000000>;
+			operating-points-v2 = <&cluster_a53_opp_table>;
+			#cooling-cells = <2>;
+		};
+
+		cpu4: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a57", "arm,armv8";
+			enable-method = "psci";
+			reg = <0x0>;
+			clock-frequency = <1900000000>;
+			clocks = <&cmu_atlas CLK_SCLK_ATLAS>;
+			clock-names = "atlasclk";
+			operating-points-v2 = <&cluster_a57_opp_table>;
+			#cooling-cells = <2>;
+		};
+
+		cpu5: cpu@1 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a57", "arm,armv8";
+			enable-method = "psci";
+			reg = <0x1>;
+			clock-frequency = <1900000000>;
+			operating-points-v2 = <&cluster_a57_opp_table>;
+			#cooling-cells = <2>;
+		};
+
+		cpu6: cpu@2 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a57", "arm,armv8";
+			enable-method = "psci";
+			reg = <0x2>;
+			clock-frequency = <1900000000>;
+			operating-points-v2 = <&cluster_a57_opp_table>;
+			#cooling-cells = <2>;
+		};
+
+		cpu7: cpu@3 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a57", "arm,armv8";
+			enable-method = "psci";
+			reg = <0x3>;
+			clock-frequency = <1900000000>;
+			operating-points-v2 = <&cluster_a57_opp_table>;
+			#cooling-cells = <2>;
+		};
+	};
+
+	cluster_a53_opp_table: opp_table0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp@400000000 {
+			opp-hz = /bits/ 64 <400000000>;
+			opp-microvolt = <900000>;
+		};
+		opp@500000000 {
+			opp-hz = /bits/ 64 <500000000>;
+			opp-microvolt = <925000>;
+		};
+		opp@600000000 {
+			opp-hz = /bits/ 64 <600000000>;
+			opp-microvolt = <950000>;
+		};
+		opp@700000000 {
+			opp-hz = /bits/ 64 <700000000>;
+			opp-microvolt = <975000>;
+		};
+		opp@800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			opp-microvolt = <1000000>;
+		};
+		opp@900000000 {
+			opp-hz = /bits/ 64 <900000000>;
+			opp-microvolt = <1050000>;
+		};
+		opp@1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-microvolt = <1075000>;
+		};
+		opp@1100000000 {
+			opp-hz = /bits/ 64 <1100000000>;
+			opp-microvolt = <1112500>;
+		};
+		opp@1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
+			opp-microvolt = <1112500>;
+		};
+		opp@1300000000 {
+			opp-hz = /bits/ 64 <1300000000>;
+			opp-microvolt = <1150000>;
+		};
+	};
+
+	cluster_a57_opp_table: opp_table1 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp@500000000 {
+			opp-hz = /bits/ 64 <500000000>;
+			opp-microvolt = <900000>;
+		};
+		opp@600000000 {
+			opp-hz = /bits/ 64 <600000000>;
+			opp-microvolt = <900000>;
+		};
+		opp@700000000 {
+			opp-hz = /bits/ 64 <700000000>;
+			opp-microvolt = <912500>;
+		};
+		opp@800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			opp-microvolt = <912500>;
+		};
+		opp@900000000 {
+			opp-hz = /bits/ 64 <900000000>;
+			opp-microvolt = <937500>;
+		};
+		opp@1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-microvolt = <975000>;
+		};
+		opp@1100000000 {
+			opp-hz = /bits/ 64 <1100000000>;
+			opp-microvolt = <1012500>;
+		};
+		opp@1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
+			opp-microvolt = <1037500>;
+		};
+		opp@1300000000 {
+			opp-hz = /bits/ 64 <1300000000>;
+			opp-microvolt = <1062500>;
+		};
+		opp@1400000000 {
+			opp-hz = /bits/ 64 <1400000000>;
+			opp-microvolt = <1087500>;
+		};
+		opp@1500000000 {
+			opp-hz = /bits/ 64 <1500000000>;
+			opp-microvolt = <1125000>;
+		};
+		opp@1600000000 {
+			opp-hz = /bits/ 64 <1600000000>;
+			opp-microvolt = <1137500>;
+		};
+		opp@1700000000 {
+			opp-hz = /bits/ 64 <1700000000>;
+			opp-microvolt = <1175000>;
+		};
+		opp@1800000000 {
+			opp-hz = /bits/ 64 <1800000000>;
+			opp-microvolt = <1212500>;
+		};
+		opp@1900000000 {
+			opp-hz = /bits/ 64 <1900000000>;
+			opp-microvolt = <1262500>;
+		};
+	};
+
+	psci {
+		compatible = "arm,psci";
+		method = "smc";
+		cpu_off = <0x84000002>;
+		cpu_on = <0xC4000003>;
+	};
+
+	reboot: syscon-reboot {
+		compatible = "syscon-reboot";
+		regmap = <&pmu_system_controller>;
+		offset = <0x400>; /* SWRESET */
+		mask = <0x1>;
+	};
+
+	soc: soc {
+		compatible = "simple-bus";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0 0x0 0x0 0x18000000>;
+
+		chipid@10000000 {
+			compatible = "samsung,exynos4210-chipid";
+			reg = <0x10000000 0x100>;
+		};
+
+		xxti: xxti {
+			compatible = "fixed-clock";
+			clock-output-names = "oscclk";
+			#clock-cells = <0>;
+		};
+
+		cmu_top: clock-controller@10030000 {
+			compatible = "samsung,exynos5433-cmu-top";
+			reg = <0x10030000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk",
+				"sclk_mphy_pll",
+				"sclk_mfc_pll",
+				"sclk_bus_pll";
+			clocks = <&xxti>,
+				<&cmu_cpif CLK_SCLK_MPHY_PLL>,
+				<&cmu_mif CLK_SCLK_MFC_PLL>,
+				<&cmu_mif CLK_SCLK_BUS_PLL>;
+		};
+
+		cmu_cpif: clock-controller@10fc0000 {
+			compatible = "samsung,exynos5433-cmu-cpif";
+			reg = <0x10fc0000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk";
+			clocks = <&xxti>;
+		};
+
+		cmu_mif: clock-controller@105b0000 {
+			compatible = "samsung,exynos5433-cmu-mif";
+			reg = <0x105b0000 0x2000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk",
+				"sclk_mphy_pll";
+			clocks = <&xxti>,
+				<&cmu_cpif CLK_SCLK_MPHY_PLL>;
+		};
+
+		cmu_peric: clock-controller@14c80000 {
+			compatible = "samsung,exynos5433-cmu-peric";
+			reg = <0x14c80000 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		cmu_peris: clock-controller@0x10040000 {
+			compatible = "samsung,exynos5433-cmu-peris";
+			reg = <0x10040000 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		cmu_fsys: clock-controller@156e0000 {
+			compatible = "samsung,exynos5433-cmu-fsys";
+			reg = <0x156e0000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk",
+				"sclk_ufs_mphy",
+				"aclk_fsys_200",
+				"sclk_pcie_100_fsys",
+				"sclk_ufsunipro_fsys",
+				"sclk_mmc2_fsys",
+				"sclk_mmc1_fsys",
+				"sclk_mmc0_fsys",
+				"sclk_usbhost30_fsys",
+				"sclk_usbdrd30_fsys";
+			clocks = <&xxti>,
+				<&cmu_cpif CLK_SCLK_UFS_MPHY>,
+				<&cmu_top CLK_ACLK_FSYS_200>,
+				<&cmu_top CLK_SCLK_PCIE_100_FSYS>,
+				<&cmu_top CLK_SCLK_UFSUNIPRO_FSYS>,
+				<&cmu_top CLK_SCLK_MMC2_FSYS>,
+				<&cmu_top CLK_SCLK_MMC1_FSYS>,
+				<&cmu_top CLK_SCLK_MMC0_FSYS>,
+				<&cmu_top CLK_SCLK_USBHOST30_FSYS>,
+				<&cmu_top CLK_SCLK_USBDRD30_FSYS>;
+		};
+
+		cmu_g2d: clock-controller@12460000 {
+			compatible = "samsung,exynos5433-cmu-g2d";
+			reg = <0x12460000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk",
+				"aclk_g2d_266",
+				"aclk_g2d_400";
+			clocks = <&xxti>,
+				<&cmu_top CLK_ACLK_G2D_266>,
+				<&cmu_top CLK_ACLK_G2D_400>;
+		};
+
+		cmu_disp: clock-controller@13b90000 {
+			compatible = "samsung,exynos5433-cmu-disp";
+			reg = <0x13b90000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk",
+				"sclk_dsim1_disp",
+				"sclk_dsim0_disp",
+				"sclk_dsd_disp",
+				"sclk_decon_tv_eclk_disp",
+				"sclk_decon_vclk_disp",
+				"sclk_decon_eclk_disp",
+				"sclk_decon_tv_vclk_disp",
+				"aclk_disp_333";
+			clocks = <&xxti>,
+				<&cmu_mif CLK_SCLK_DSIM1_DISP>,
+				<&cmu_mif CLK_SCLK_DSIM0_DISP>,
+				<&cmu_mif CLK_SCLK_DSD_DISP>,
+				<&cmu_mif CLK_SCLK_DECON_TV_ECLK_DISP>,
+				<&cmu_mif CLK_SCLK_DECON_VCLK_DISP>,
+				<&cmu_mif CLK_SCLK_DECON_ECLK_DISP>,
+				<&cmu_mif CLK_SCLK_DECON_TV_VCLK_DISP>,
+				<&cmu_mif CLK_ACLK_DISP_333>;
+		};
+
+		cmu_aud: clock-controller@114c0000 {
+			compatible = "samsung,exynos5433-cmu-aud";
+			reg = <0x114c0000 0x1000>;
+			#clock-cells = <1>;
+			clock-names = "oscclk", "fout_aud_pll";
+			clocks = <&xxti>, <&cmu_top CLK_FOUT_AUD_PLL>;
+		};
+
+		cmu_bus0: clock-controller@13600000 {
+			compatible = "samsung,exynos5433-cmu-bus0";
+			reg = <0x13600000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "aclk_bus0_400";
+			clocks = <&cmu_top CLK_ACLK_BUS0_400>;
+		};
+
+		cmu_bus1: clock-controller@14800000 {
+			compatible = "samsung,exynos5433-cmu-bus1";
+			reg = <0x14800000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "aclk_bus1_400";
+			clocks = <&cmu_top CLK_ACLK_BUS1_400>;
+		};
+
+		cmu_bus2: clock-controller@13400000 {
+			compatible = "samsung,exynos5433-cmu-bus2";
+			reg = <0x13400000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk", "aclk_bus2_400";
+			clocks = <&xxti>, <&cmu_mif CLK_ACLK_BUS2_400>;
+		};
+
+		cmu_g3d: clock-controller@14aa0000 {
+			compatible = "samsung,exynos5433-cmu-g3d";
+			reg = <0x14aa0000 0x2000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk", "aclk_g3d_400";
+			clocks = <&xxti>, <&cmu_top CLK_ACLK_G3D_400>;
+		};
+
+		cmu_gscl: clock-controller@13cf0000 {
+			compatible = "samsung,exynos5433-cmu-gscl";
+			reg = <0x13cf0000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk",
+				"aclk_gscl_111",
+				"aclk_gscl_333";
+			clocks = <&xxti>,
+				<&cmu_top CLK_ACLK_GSCL_111>,
+				<&cmu_top CLK_ACLK_GSCL_333>;
+		};
+
+		cmu_apollo: clock-controller@11900000 {
+			compatible = "samsung,exynos5433-cmu-apollo";
+			reg = <0x11900000 0x2000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk", "sclk_bus_pll_apollo";
+			clocks = <&xxti>, <&cmu_mif CLK_SCLK_BUS_PLL_APOLLO>;
+		};
+
+		cmu_atlas: clock-controller@11800000 {
+			compatible = "samsung,exynos5433-cmu-atlas";
+			reg = <0x11800000 0x2000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk", "sclk_bus_pll_atlas";
+			clocks = <&xxti>, <&cmu_mif CLK_SCLK_BUS_PLL_ATLAS>;
+		};
+
+		cmu_mscl: clock-controller@105d0000 {
+			compatible = "samsung,exynos5433-cmu-mscl";
+			reg = <0x150d0000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk",
+				"sclk_jpeg_mscl",
+				"aclk_mscl_400";
+			clocks = <&xxti>,
+				<&cmu_top CLK_SCLK_JPEG_MSCL>,
+				<&cmu_top CLK_ACLK_MSCL_400>;
+		};
+
+		cmu_mfc: clock-controller@15280000 {
+			compatible = "samsung,exynos5433-cmu-mfc";
+			reg = <0x15280000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk", "aclk_mfc_400";
+			clocks = <&xxti>, <&cmu_top CLK_ACLK_MFC_400>;
+		};
+
+		cmu_hevc: clock-controller@14f80000 {
+			compatible = "samsung,exynos5433-cmu-hevc";
+			reg = <0x14f80000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk", "aclk_hevc_400";
+			clocks = <&xxti>, <&cmu_top CLK_ACLK_HEVC_400>;
+		};
+
+		cmu_isp: clock-controller@146d0000 {
+			compatible = "samsung,exynos5433-cmu-isp";
+			reg = <0x146d0000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk",
+				"aclk_isp_dis_400",
+				"aclk_isp_400";
+			clocks = <&xxti>,
+				<&cmu_top CLK_ACLK_ISP_DIS_400>,
+				<&cmu_top CLK_ACLK_ISP_400>;
+		};
+
+		cmu_cam0: clock-controller@120d0000 {
+			compatible = "samsung,exynos5433-cmu-cam0";
+			reg = <0x120d0000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk",
+				"aclk_cam0_333",
+				"aclk_cam0_400",
+				"aclk_cam0_552";
+			clocks = <&xxti>,
+				<&cmu_top CLK_ACLK_CAM0_333>,
+				<&cmu_top CLK_ACLK_CAM0_400>,
+				<&cmu_top CLK_ACLK_CAM0_552>;
+		};
+
+		cmu_cam1: clock-controller@145d0000 {
+			compatible = "samsung,exynos5433-cmu-cam1";
+			reg = <0x145d0000 0x1000>;
+			#clock-cells = <1>;
+
+			clock-names = "oscclk",
+				"sclk_isp_uart_cam1",
+				"sclk_isp_spi1_cam1",
+				"sclk_isp_spi0_cam1",
+				"aclk_cam1_333",
+				"aclk_cam1_400",
+				"aclk_cam1_552";
+			clocks = <&xxti>,
+				<&cmu_top CLK_SCLK_ISP_UART_CAM1>,
+				<&cmu_top CLK_SCLK_ISP_SPI1_CAM1>,
+				<&cmu_top CLK_SCLK_ISP_SPI0_CAM1>,
+				<&cmu_top CLK_ACLK_CAM1_333>,
+				<&cmu_top CLK_ACLK_CAM1_400>,
+				<&cmu_top CLK_ACLK_CAM1_552>;
+		};
+
+		tmu_atlas0: tmu@10060000 {
+			compatible = "samsung,exynos5433-tmu";
+			reg = <0x10060000 0x200>;
+			interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cmu_peris CLK_PCLK_TMU0_APBIF>,
+				<&cmu_peris CLK_SCLK_TMU0>;
+			clock-names = "tmu_apbif", "tmu_sclk";
+			#include "exynos5433-tmu-sensor-conf.dtsi"
+			status = "disabled";
+		};
+
+		tmu_atlas1: tmu@10068000 {
+			compatible = "samsung,exynos5433-tmu";
+			reg = <0x10068000 0x200>;
+			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cmu_peris CLK_PCLK_TMU0_APBIF>,
+				<&cmu_peris CLK_SCLK_TMU0>;
+			clock-names = "tmu_apbif", "tmu_sclk";
+			#include "exynos5433-tmu-sensor-conf.dtsi"
+			status = "disabled";
+		};
+
+		tmu_g3d: tmu@10070000 {
+			compatible = "samsung,exynos5433-tmu";
+			reg = <0x10070000 0x200>;
+			interrupts = <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cmu_peris CLK_PCLK_TMU1_APBIF>,
+				<&cmu_peris CLK_SCLK_TMU1>;
+			clock-names = "tmu_apbif", "tmu_sclk";
+			#include "exynos5433-tmu-g3d-sensor-conf.dtsi"
+			status = "disabled";
+		};
+
+		tmu_apollo: tmu@10078000 {
+			compatible = "samsung,exynos5433-tmu";
+			reg = <0x10078000 0x200>;
+			interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cmu_peris CLK_PCLK_TMU1_APBIF>,
+				<&cmu_peris CLK_SCLK_TMU1>;
+			clock-names = "tmu_apbif", "tmu_sclk";
+			#include "exynos5433-tmu-sensor-conf.dtsi"
+			status = "disabled";
+		};
+
+		tmu_isp: tmu@1007c000 {
+			compatible = "samsung,exynos5433-tmu";
+			reg = <0x1007c000 0x200>;
+			interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cmu_peris CLK_PCLK_TMU1_APBIF>,
+				<&cmu_peris CLK_SCLK_TMU1>;
+			clock-names = "tmu_apbif", "tmu_sclk";
+			#include "exynos5433-tmu-sensor-conf.dtsi"
+			status = "disabled";
+		};
+
+		mct@101c0000 {
+			compatible = "samsung,exynos4210-mct";
+			reg = <0x101c0000 0x800>;
+			interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>,
+				<GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&xxti>, <&cmu_peris CLK_PCLK_MCT>;
+			clock-names = "fin_pll", "mct";
+		};
+
+		pinctrl_alive: pinctrl@10580000 {
+			compatible = "samsung,exynos5433-pinctrl";
+			reg = <0x10580000 0x1a20>, <0x11090000 0x100>;
+
+			wakeup-interrupt-controller {
+				compatible = "samsung,exynos7-wakeup-eint";
+				interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
+			};
+		};
+
+		pinctrl_aud: pinctrl@114b0000 {
+			compatible = "samsung,exynos5433-pinctrl";
+			reg = <0x114b0000 0x1000>;
+			interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_cpif: pinctrl@10fe0000 {
+			compatible = "samsung,exynos5433-pinctrl";
+			reg = <0x10fe0000 0x1000>;
+			interrupts = <GIC_SPI 179 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_ese: pinctrl@14ca0000 {
+			compatible = "samsung,exynos5433-pinctrl";
+			reg = <0x14ca0000 0x1000>;
+			interrupts = <GIC_SPI 413 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_finger: pinctrl@14cb0000 {
+			compatible = "samsung,exynos5433-pinctrl";
+			reg = <0x14cb0000 0x1000>;
+			interrupts = <GIC_SPI 414 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_fsys: pinctrl@15690000 {
+			compatible = "samsung,exynos5433-pinctrl";
+			reg = <0x15690000 0x1000>;
+			interrupts = <GIC_SPI 229 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_imem: pinctrl@11090000 {
+			compatible = "samsung,exynos5433-pinctrl";
+			reg = <0x11090000 0x1000>;
+			interrupts = <GIC_SPI 325 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_nfc: pinctrl@14cd0000 {
+			compatible = "samsung,exynos5433-pinctrl";
+			reg = <0x14cd0000 0x1000>;
+			interrupts = <GIC_SPI 441 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_peric: pinctrl@14cc0000 {
+			compatible = "samsung,exynos5433-pinctrl";
+			reg = <0x14cc0000 0x1100>;
+			interrupts = <GIC_SPI 440 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_touch: pinctrl@14ce0000 {
+			compatible = "samsung,exynos5433-pinctrl";
+			reg = <0x14ce0000 0x1100>;
+			interrupts = <GIC_SPI 442 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pmu_system_controller: system-controller@105c0000 {
+			compatible = "samsung,exynos5433-pmu", "syscon";
+			reg = <0x105c0000 0x5008>;
+			#clock-cells = <1>;
+			clock-names = "clkout16";
+			clocks = <&xxti>;
+		};
+
+		gic: interrupt-controller@11001000 {
+			compatible = "arm,gic-400";
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			reg = <0x11001000 0x1000>,
+				<0x11002000 0x2000>,
+				<0x11004000 0x2000>,
+				<0x11006000 0x2000>;
+			interrupts = <GIC_PPI 9 0xf04>;
+		};
+
+		mipi_phy: video-phy@105c0710 {
+			compatible = "samsung,exynos5433-mipi-video-phy";
+			#phy-cells = <1>;
+			samsung,pmu-syscon = <&pmu_system_controller>;
+			samsung,cam0-sysreg = <&syscon_cam0>;
+			samsung,cam1-sysreg = <&syscon_cam1>;
+			samsung,disp-sysreg = <&syscon_disp>;
+		};
+
+		decon: decon@13800000 {
+			compatible = "samsung,exynos5433-decon";
+			reg = <0x13800000 0x2104>;
+			clocks = <&cmu_disp CLK_PCLK_DECON>,
+				<&cmu_disp CLK_ACLK_DECON>,
+				<&cmu_disp CLK_ACLK_SMMU_DECON0X>,
+				<&cmu_disp CLK_ACLK_XIU_DECON0X>,
+				<&cmu_disp CLK_PCLK_SMMU_DECON0X>,
+				<&cmu_disp CLK_SCLK_DECON_VCLK>,
+				<&cmu_disp CLK_SCLK_DECON_ECLK>;
+			clock-names = "pclk", "aclk_decon", "aclk_smmu_decon0x",
+				"aclk_xiu_decon0x", "pclk_smmu_decon0x",
+				"sclk_decon_vclk", "sclk_decon_eclk";
+			interrupt-names = "fifo", "vsync", "lcd_sys";
+			interrupts = <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH>;
+			samsung,disp-sysreg = <&syscon_disp>;
+			status = "disabled";
+			iommus = <&sysmmu_decon0x>, <&sysmmu_decon1x>;
+			iommu-names = "m0", "m1";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					decon_to_mic: endpoint {
+						remote-endpoint =
+							<&mic_to_decon>;
+					};
+				};
+			};
+		};
+
+		dsi: dsi@13900000 {
+			compatible = "samsung,exynos5433-mipi-dsi";
+			reg = <0x13900000 0xC0>;
+			interrupts = <GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH>;
+			phys = <&mipi_phy 1>;
+			phy-names = "dsim";
+			clocks = <&cmu_disp CLK_PCLK_DSIM0>,
+				<&cmu_disp CLK_PHYCLK_MIPIDPHY0_BITCLKDIV8>,
+				<&cmu_disp CLK_PHYCLK_MIPIDPHY0_RXCLKESC0>,
+				<&cmu_disp CLK_SCLK_RGB_VCLK_TO_DSIM0>,
+				<&cmu_disp CLK_SCLK_DSIM0>;
+			clock-names = "bus_clk",
+					"phyclk_mipidphy0_bitclkdiv8",
+					"phyclk_mipidphy0_rxclkesc0",
+					"sclk_rgb_vclk_to_dsim0",
+					"sclk_mipi";
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					dsi_to_mic: endpoint {
+						remote-endpoint = <&mic_to_dsi>;
+					};
+				};
+			};
+		};
+
+		mic: mic@13930000 {
+			compatible = "samsung,exynos5433-mic";
+			reg = <0x13930000 0x48>;
+			clocks = <&cmu_disp CLK_PCLK_MIC0>,
+				<&cmu_disp CLK_SCLK_RGB_VCLK_TO_MIC0>;
+			clock-names = "pclk_mic0", "sclk_rgb_vclk_to_mic0";
+			samsung,disp-syscon = <&syscon_disp>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					mic_to_decon: endpoint {
+						remote-endpoint =
+							<&decon_to_mic>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					mic_to_dsi: endpoint {
+						remote-endpoint = <&dsi_to_mic>;
+					};
+				};
+			};
+		};
+
+		syscon_disp: syscon@13b80000 {
+			compatible = "syscon";
+			reg = <0x13b80000 0x1010>;
+		};
+
+		syscon_cam0: syscon@120f0000 {
+			compatible = "syscon";
+			reg = <0x120f0000 0x1020>;
+		};
+
+		syscon_cam1: syscon@145f0000 {
+			compatible = "syscon";
+			reg = <0x145f0000 0x1038>;
+		};
+
+		gsc_0: video-scaler@13C00000 {
+			compatible = "samsung,exynos5433-gsc";
+			reg = <0x13c00000 0x1000>;
+			interrupts = <GIC_SPI 297 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk", "aclk_xiu",
+				      "aclk_gsclbend";
+			clocks = <&cmu_gscl CLK_PCLK_GSCL0>,
+				 <&cmu_gscl CLK_ACLK_GSCL0>,
+				 <&cmu_gscl CLK_ACLK_XIU_GSCLX>,
+				 <&cmu_gscl CLK_ACLK_GSCLBEND_333>;
+			iommus = <&sysmmu_gscl0>;
+		};
+
+		gsc_1: video-scaler@13C10000 {
+			compatible = "samsung,exynos5433-gsc";
+			reg = <0x13c10000 0x1000>;
+			interrupts = <GIC_SPI 298 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk", "aclk_xiu",
+				      "aclk_gsclbend";
+			clocks = <&cmu_gscl CLK_PCLK_GSCL1>,
+				 <&cmu_gscl CLK_ACLK_GSCL1>,
+				 <&cmu_gscl CLK_ACLK_XIU_GSCLX>,
+				 <&cmu_gscl CLK_ACLK_GSCLBEND_333>;
+			iommus = <&sysmmu_gscl1>;
+		};
+
+		gsc_2: video-scaler@13C20000 {
+			compatible = "samsung,exynos5433-gsc";
+			reg = <0x13c20000 0x1000>;
+			interrupts = <GIC_SPI 299 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk", "aclk_xiu",
+				      "aclk_gsclbend";
+			clocks = <&cmu_gscl CLK_PCLK_GSCL2>,
+				 <&cmu_gscl CLK_ACLK_GSCL2>,
+				 <&cmu_gscl CLK_ACLK_XIU_GSCLX>,
+				 <&cmu_gscl CLK_ACLK_GSCLBEND_333>;
+			iommus = <&sysmmu_gscl2>;
+		};
+
+		jpeg: codec@15020000 {
+			compatible = "samsung,exynos5433-jpeg";
+			reg = <0x15020000 0x10000>;
+			interrupts = <GIC_SPI 411 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk", "aclk_xiu", "sclk";
+			clocks = <&cmu_mscl CLK_PCLK_JPEG>,
+				 <&cmu_mscl CLK_ACLK_JPEG>,
+				 <&cmu_mscl CLK_ACLK_XIU_MSCLX>,
+				 <&cmu_mscl CLK_SCLK_JPEG>;
+			iommus = <&sysmmu_jpeg>;
+		};
+
+		mfc: codec@152E0000 {
+			compatible = "samsung,exynos5433-mfc";
+			reg = <0x152E0000 0x10000>;
+			interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk", "aclk_xiu";
+			clocks = <&cmu_mfc CLK_PCLK_MFC>,
+				 <&cmu_mfc CLK_ACLK_MFC>,
+				 <&cmu_mfc CLK_ACLK_XIU_MFCX>;
+			iommus = <&sysmmu_mfc_0>, <&sysmmu_mfc_1>;
+			iommu-names = "left", "right";
+		};
+
+		sysmmu_decon0x: sysmmu@0x13a00000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x13a00000 0x1000>;
+			interrupts = <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk";
+			clocks = <&cmu_disp CLK_PCLK_SMMU_DECON0X>,
+				<&cmu_disp CLK_ACLK_SMMU_DECON0X>;
+			#iommu-cells = <0>;
+		};
+
+		sysmmu_decon1x: sysmmu@0x13a10000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x13a10000 0x1000>;
+			interrupts = <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk";
+			clocks = <&cmu_disp CLK_PCLK_SMMU_DECON1X>,
+				<&cmu_disp CLK_ACLK_SMMU_DECON1X>;
+			#iommu-cells = <0>;
+		};
+
+		sysmmu_gscl0: sysmmu@0x13C80000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x13C80000 0x1000>;
+			interrupts = <GIC_SPI 288 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "aclk", "pclk";
+			clocks = <&cmu_gscl CLK_ACLK_SMMU_GSCL0>,
+				 <&cmu_gscl CLK_PCLK_SMMU_GSCL0>;
+			#iommu-cells = <0>;
+		};
+
+		sysmmu_gscl1: sysmmu@0x13C90000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x13C90000 0x1000>;
+			interrupts = <GIC_SPI 290 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "aclk", "pclk";
+			clocks = <&cmu_gscl CLK_ACLK_SMMU_GSCL1>,
+				 <&cmu_gscl CLK_PCLK_SMMU_GSCL1>;
+			#iommu-cells = <0>;
+		};
+
+		sysmmu_gscl2: sysmmu@0x13CA0000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x13CA0000 0x1000>;
+			interrupts = <GIC_SPI 292 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "aclk", "pclk";
+			clocks = <&cmu_gscl CLK_ACLK_SMMU_GSCL2>,
+				 <&cmu_gscl CLK_PCLK_SMMU_GSCL2>;
+			#iommu-cells = <0>;
+		};
+
+		sysmmu_jpeg: sysmmu@0x15060000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x15060000 0x1000>;
+			interrupts = <GIC_SPI 408 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk";
+			clocks = <&cmu_mscl CLK_PCLK_SMMU_JPEG>,
+				 <&cmu_mscl CLK_ACLK_SMMU_JPEG>;
+			#iommu-cells = <0>;
+		};
+
+		sysmmu_mfc_0: sysmmu@0x15200000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x15200000 0x1000>;
+			interrupts = <GIC_SPI 352 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk";
+			clocks = <&cmu_mfc CLK_PCLK_SMMU_MFC_0>,
+				 <&cmu_mfc CLK_ACLK_SMMU_MFC_0>;
+			#iommu-cells = <0>;
+		};
+
+		sysmmu_mfc_1: sysmmu@0x15210000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x15210000 0x1000>;
+			interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "pclk", "aclk";
+			clocks = <&cmu_mfc CLK_PCLK_SMMU_MFC_1>,
+				 <&cmu_mfc CLK_ACLK_SMMU_MFC_1>;
+			#iommu-cells = <0>;
+		};
+
+		serial_0: serial@14c10000 {
+			compatible = "samsung,exynos5433-uart";
+			reg = <0x14c10000 0x100>;
+			interrupts = <GIC_SPI 421 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cmu_peric CLK_PCLK_UART0>,
+				<&cmu_peric CLK_SCLK_UART0>;
+			clock-names = "uart", "clk_uart_baud0";
+			pinctrl-names = "default";
+			pinctrl-0 = <&uart0_bus>;
+			status = "disabled";
+		};
+
+		serial_1: serial@14c20000 {
+			compatible = "samsung,exynos5433-uart";
+			reg = <0x14c20000 0x100>;
+			interrupts = <GIC_SPI 422 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cmu_peric CLK_PCLK_UART1>,
+				<&cmu_peric CLK_SCLK_UART1>;
+			clock-names = "uart", "clk_uart_baud0";
+			pinctrl-names = "default";
+			pinctrl-0 = <&uart1_bus>;
+			status = "disabled";
+		};
+
+		serial_2: serial@14c30000 {
+			compatible = "samsung,exynos5433-uart";
+			reg = <0x14c30000 0x100>;
+			interrupts = <GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cmu_peric CLK_PCLK_UART2>,
+				<&cmu_peric CLK_SCLK_UART2>;
+			clock-names = "uart", "clk_uart_baud0";
+			pinctrl-names = "default";
+			pinctrl-0 = <&uart2_bus>;
+			status = "disabled";
+		};
+
+		spi_0: spi@14d20000 {
+			compatible = "samsung,exynos5433-spi";
+			reg = <0x14d20000 0x100>;
+			interrupts = <GIC_SPI 432 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&pdma0 9>, <&pdma0 8>;
+			dma-names = "tx", "rx";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&cmu_peric CLK_PCLK_SPI0>,
+				<&cmu_peric CLK_SCLK_SPI0>,
+				<&cmu_peric CLK_SCLK_IOCLK_SPI0>;
+			clock-names = "spi", "spi_busclk0", "spi_ioclk";
+			samsung,spi-src-clk = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&spi0_bus>;
+			num-cs = <1>;
+			status = "disabled";
+		};
+
+		spi_1: spi@14d30000 {
+			compatible = "samsung,exynos5433-spi";
+			reg = <0x14d30000 0x100>;
+			interrupts = <GIC_SPI 433 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&pdma0 11>, <&pdma0 10>;
+			dma-names = "tx", "rx";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&cmu_peric CLK_PCLK_SPI1>,
+				<&cmu_peric CLK_SCLK_SPI1>,
+				<&cmu_peric CLK_SCLK_IOCLK_SPI1>;
+			clock-names = "spi", "spi_busclk0", "spi_ioclk";
+			samsung,spi-src-clk = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&spi1_bus>;
+			num-cs = <1>;
+			status = "disabled";
+		};
+
+		spi_2: spi@14d40000 {
+			compatible = "samsung,exynos5433-spi";
+			reg = <0x14d40000 0x100>;
+			interrupts = <GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&pdma0 13>, <&pdma0 12>;
+			dma-names = "tx", "rx";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&cmu_peric CLK_PCLK_SPI2>,
+				<&cmu_peric CLK_SCLK_SPI2>,
+				<&cmu_peric CLK_SCLK_IOCLK_SPI2>;
+			clock-names = "spi", "spi_busclk0", "spi_ioclk";
+			samsung,spi-src-clk = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&spi2_bus>;
+			num-cs = <1>;
+			status = "disabled";
+		};
+
+		spi_3: spi@14d50000 {
+			compatible = "samsung,exynos5433-spi";
+			reg = <0x14d50000 0x100>;
+			interrupts = <GIC_SPI 447 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&pdma0 23>, <&pdma0 22>;
+			dma-names = "tx", "rx";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&cmu_peric CLK_PCLK_SPI3>,
+				<&cmu_peric CLK_SCLK_SPI3>,
+				<&cmu_peric CLK_SCLK_IOCLK_SPI3>;
+			clock-names = "spi", "spi_busclk0", "spi_ioclk";
+			samsung,spi-src-clk = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&spi3_bus>;
+			num-cs = <1>;
+			status = "disabled";
+		};
+
+		spi_4: spi@14d00000 {
+			compatible = "samsung,exynos5433-spi";
+			reg = <0x14d00000 0x100>;
+			interrupts = <GIC_SPI 412 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&pdma0 25>, <&pdma0 24>;
+			dma-names = "tx", "rx";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&cmu_peric CLK_PCLK_SPI4>,
+				<&cmu_peric CLK_SCLK_SPI4>,
+				<&cmu_peric CLK_SCLK_IOCLK_SPI4>;
+			clock-names = "spi", "spi_busclk0", "spi_ioclk";
+			samsung,spi-src-clk = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&spi4_bus>;
+			num-cs = <1>;
+			status = "disabled";
+		};
+
+		adc: adc@14d10000 {
+			compatible = "samsung,exynos7-adc";
+			reg = <0x14d10000 0x100>;
+			interrupts = <GIC_SPI 438 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "adc";
+			clocks = <&cmu_peric CLK_PCLK_ADCIF>;
+			#io-channel-cells = <1>;
+			io-channel-ranges;
+			status = "disabled";
+		};
+
+		pwm: pwm@14dd0000 {
+			compatible = "samsung,exynos4210-pwm";
+			reg = <0x14dd0000 0x100>;
+			interrupts = <GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 417 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 418 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 420 IRQ_TYPE_LEVEL_HIGH>;
+			samsung,pwm-outputs = <0>, <1>, <2>, <3>;
+			clocks = <&cmu_peric CLK_PCLK_PWM>;
+			clock-names = "timers";
+			#pwm-cells = <3>;
+			status = "disabled";
+		};
+
+		hsi2c_0: hsi2c@14e40000 {
+			compatible = "samsung,exynos7-hsi2c";
+			reg = <0x14e40000 0x1000>;
+			interrupts = <GIC_SPI 428 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&hs_i2c0_bus>;
+			clocks = <&cmu_peric CLK_PCLK_HSI2C0>;
+			clock-names = "hsi2c";
+			status = "disabled";
+		};
+
+		hsi2c_1: hsi2c@14e50000 {
+			compatible = "samsung,exynos7-hsi2c";
+			reg = <0x14e50000 0x1000>;
+			interrupts = <GIC_SPI 429 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&hs_i2c1_bus>;
+			clocks = <&cmu_peric CLK_PCLK_HSI2C1>;
+			clock-names = "hsi2c";
+			status = "disabled";
+		};
+
+		hsi2c_2: hsi2c@14e60000 {
+			compatible = "samsung,exynos7-hsi2c";
+			reg = <0x14e60000 0x1000>;
+			interrupts = <GIC_SPI 430 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&hs_i2c2_bus>;
+			clocks = <&cmu_peric CLK_PCLK_HSI2C2>;
+			clock-names = "hsi2c";
+			status = "disabled";
+		};
+
+		hsi2c_3: hsi2c@14e70000 {
+			compatible = "samsung,exynos7-hsi2c";
+			reg = <0x14e70000 0x1000>;
+			interrupts = <GIC_SPI 431 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&hs_i2c3_bus>;
+			clocks = <&cmu_peric CLK_PCLK_HSI2C3>;
+			clock-names = "hsi2c";
+			status = "disabled";
+		};
+
+		hsi2c_4: hsi2c@14ec0000 {
+			compatible = "samsung,exynos7-hsi2c";
+			reg = <0x14ec0000 0x1000>;
+			interrupts = <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&hs_i2c4_bus>;
+			clocks = <&cmu_peric CLK_PCLK_HSI2C4>;
+			clock-names = "hsi2c";
+			status = "disabled";
+		};
+
+		hsi2c_5: hsi2c@14ed0000 {
+			compatible = "samsung,exynos7-hsi2c";
+			reg = <0x14ed0000 0x1000>;
+			interrupts = <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&hs_i2c5_bus>;
+			clocks = <&cmu_peric CLK_PCLK_HSI2C5>;
+			clock-names = "hsi2c";
+			status = "disabled";
+		};
+
+		hsi2c_6: hsi2c@14ee0000 {
+			compatible = "samsung,exynos7-hsi2c";
+			reg = <0x14ee0000 0x1000>;
+			interrupts = <GIC_SPI 426 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&hs_i2c6_bus>;
+			clocks = <&cmu_peric CLK_PCLK_HSI2C6>;
+			clock-names = "hsi2c";
+			status = "disabled";
+		};
+
+		hsi2c_7: hsi2c@14ef0000 {
+			compatible = "samsung,exynos7-hsi2c";
+			reg = <0x14ef0000 0x1000>;
+			interrupts = <GIC_SPI 427 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&hs_i2c7_bus>;
+			clocks = <&cmu_peric CLK_PCLK_HSI2C7>;
+			clock-names = "hsi2c";
+			status = "disabled";
+		};
+
+		hsi2c_8: hsi2c@14d90000 {
+			compatible = "samsung,exynos7-hsi2c";
+			reg = <0x14d90000 0x1000>;
+			interrupts = <GIC_SPI 443 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&hs_i2c8_bus>;
+			clocks = <&cmu_peric CLK_PCLK_HSI2C8>;
+			clock-names = "hsi2c";
+			status = "disabled";
+		};
+
+		hsi2c_9: hsi2c@14da0000 {
+			compatible = "samsung,exynos7-hsi2c";
+			reg = <0x14da0000 0x1000>;
+			interrupts = <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&hs_i2c9_bus>;
+			clocks = <&cmu_peric CLK_PCLK_HSI2C9>;
+			clock-names = "hsi2c";
+			status = "disabled";
+		};
+
+		hsi2c_10: hsi2c@14de0000 {
+			compatible = "samsung,exynos7-hsi2c";
+			reg = <0x14de0000 0x1000>;
+			interrupts = <GIC_SPI 445 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&hs_i2c10_bus>;
+			clocks = <&cmu_peric CLK_PCLK_HSI2C10>;
+			clock-names = "hsi2c";
+			status = "disabled";
+		};
+
+		hsi2c_11: hsi2c@14df0000 {
+			compatible = "samsung,exynos7-hsi2c";
+			reg = <0x14df0000 0x1000>;
+			interrupts = <GIC_SPI 446 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&hs_i2c11_bus>;
+			clocks = <&cmu_peric CLK_PCLK_HSI2C11>;
+			clock-names = "hsi2c";
+			status = "disabled";
+		};
+
+		usbdrd30: usb@15400000  {
+			compatible = "samsung,exynos5250-dwusb3";
+			clocks = <&cmu_fsys CLK_ACLK_USBDRD30>,
+				<&cmu_fsys CLK_SCLK_USBDRD30>;
+			clock-names = "usbdrd30", "usbdrd30_susp_clk";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			status = "disabled";
+
+			dwc3@15400000 {
+				compatible = "snps,dwc3";
+				reg = <0x15400000 0x10000>;
+				interrupts = <GIC_SPI 231 IRQ_TYPE_LEVEL_HIGH>;
+				phys = <&usbdrd30_phy 0>, <&usbdrd30_phy 1>;
+				phy-names = "usb2-phy", "usb3-phy";
+			};
+		};
+
+		usbdrd30_phy: phy@15500000 {
+			compatible = "samsung,exynos5433-usbdrd-phy";
+			reg = <0x15500000 0x100>;
+			clocks = <&cmu_fsys CLK_ACLK_USBDRD30>, <&xxti>,
+				<&cmu_fsys CLK_PHYCLK_USBDRD30_UDRD30_PHYCLOCK>,
+				<&cmu_fsys CLK_PHYCLK_USBDRD30_UDRD30_PIPE_PCLK>,
+				<&cmu_fsys CLK_SCLK_USBDRD30>;
+			clock-names = "phy", "ref", "phy_utmi", "phy_pipe",
+					"itp";
+			#phy-cells = <1>;
+			samsung,pmu-syscon = <&pmu_system_controller>;
+			status = "disabled";
+		};
+
+		usbhost30_phy: phy@15580000 {
+			compatible = "samsung,exynos5433-usbdrd-phy";
+			reg = <0x15580000 0x100>;
+			clocks = <&cmu_fsys CLK_ACLK_USBHOST30>, <&xxti>,
+				<&cmu_fsys CLK_PHYCLK_USBHOST30_UHOST30_PHYCLOCK>,
+				<&cmu_fsys CLK_PHYCLK_USBHOST30_UHOST30_PIPE_PCLK>,
+				<&cmu_fsys CLK_SCLK_USBHOST30>;
+			clock-names = "phy", "ref", "phy_utmi", "phy_pipe",
+					"itp";
+			#phy-cells = <1>;
+			samsung,pmu-syscon = <&pmu_system_controller>;
+			status = "disabled";
+		};
+
+		usbhost30: usb@15a00000 {
+			compatible = "samsung,exynos5250-dwusb3";
+			clocks = <&cmu_fsys CLK_ACLK_USBHOST30>,
+				<&cmu_fsys CLK_SCLK_USBHOST30>;
+			clock-names = "usbdrd30", "usbdrd30_susp_clk";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			status = "disabled";
+
+			usbdrd_dwc3_0: dwc3@15a00000 {
+				compatible = "snps,dwc3";
+				reg = <0x15a00000 0x10000>;
+				interrupts = <GIC_SPI 244 IRQ_TYPE_LEVEL_HIGH>;
+				phys = <&usbhost30_phy 0>, <&usbhost30_phy 1>;
+				phy-names = "usb2-phy", "usb3-phy";
+			};
+		};
+
+		mshc_0: mshc@15540000 {
+			compatible = "samsung,exynos7-dw-mshc-smu";
+			interrupts = <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x15540000 0x2000>;
+			clocks = <&cmu_fsys CLK_ACLK_MMC0>,
+				<&cmu_fsys CLK_SCLK_MMC0>;
+			clock-names = "biu", "ciu";
+			fifo-depth = <0x40>;
+			status = "disabled";
+		};
+
+		mshc_1: mshc@15550000 {
+			compatible = "samsung,exynos7-dw-mshc-smu";
+			interrupts = <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x15550000 0x2000>;
+			clocks = <&cmu_fsys CLK_ACLK_MMC1>,
+				<&cmu_fsys CLK_SCLK_MMC1>;
+			clock-names = "biu", "ciu";
+			fifo-depth = <0x40>;
+			status = "disabled";
+		};
+
+		mshc_2: mshc@15560000 {
+			compatible = "samsung,exynos7-dw-mshc-smu";
+			interrupts = <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x15560000 0x2000>;
+			clocks = <&cmu_fsys CLK_ACLK_MMC2>,
+				<&cmu_fsys CLK_SCLK_MMC2>;
+			clock-names = "biu", "ciu";
+			fifo-depth = <0x40>;
+			status = "disabled";
+		};
+
+		amba {
+			compatible = "arm,amba-bus";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			pdma0: pdma@15610000 {
+				compatible = "arm,pl330", "arm,primecell";
+				reg = <0x15610000 0x1000>;
+				interrupts = <GIC_SPI 228 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&cmu_fsys CLK_PDMA0>;
+				clock-names = "apb_pclk";
+				#dma-cells = <1>;
+				#dma-channels = <8>;
+				#dma-requests = <32>;
+			};
+
+			pdma1: pdma@15600000 {
+				compatible = "arm,pl330", "arm,primecell";
+				reg = <0x15600000 0x1000>;
+				interrupts = <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&cmu_fsys CLK_PDMA1>;
+				clock-names = "apb_pclk";
+				#dma-cells = <1>;
+				#dma-channels = <8>;
+				#dma-requests = <32>;
+			};
+		};
+
+		audio-subsystem@11400000 {
+			compatible = "samsung,exynos5433-lpass";
+			reg = <0x11400000 0x100>, <0x11500000 0x08>;
+			samsung,pmu-syscon = <&pmu_system_controller>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			adma: adma@11420000 {
+				compatible = "arm,pl330", "arm,primecell";
+				reg = <0x11420000 0x1000>;
+				interrupts = <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&cmu_aud CLK_ACLK_DMAC>;
+				clock-names = "apb_pclk";
+				#dma-cells = <1>;
+				#dma-channels = <8>;
+				#dma-requests = <32>;
+			};
+
+			i2s0: i2s0@11440000 {
+				compatible = "samsung,exynos7-i2s";
+				reg = <0x11440000 0x100>;
+				dmas = <&adma 0 &adma 2>;
+				dma-names = "tx", "rx";
+				interrupts = <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				clocks = <&cmu_aud CLK_PCLK_AUD_I2S>,
+					<&cmu_aud CLK_SCLK_AUD_I2S>,
+					<&cmu_aud CLK_SCLK_I2S_BCLK>;
+				clock-names = "iis", "i2s_opclk0", "i2s_opclk1";
+				pinctrl-names = "default";
+				pinctrl-0 = <&i2s0_bus>;
+				status = "disabled";
+			};
+
+			serial_3: serial@11460000 {
+				compatible = "samsung,exynos5433-uart";
+				reg = <0x11460000 0x100>;
+				interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&cmu_aud CLK_PCLK_AUD_UART>,
+					<&cmu_aud CLK_SCLK_AUD_UART>;
+				clock-names = "uart", "clk_uart_baud0";
+				pinctrl-names = "default";
+				pinctrl-0 = <&uart_aud_bus>;
+				status = "disabled";
+			};
+		};
+	};
+
+	timer: timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13
+				(GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_HIGH)>,
+			<GIC_PPI 14
+				(GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_HIGH)>,
+			<GIC_PPI 11
+				(GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_HIGH)>,
+			<GIC_PPI 10
+				(GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_HIGH)>;
+	};
+};
+
+#include "exynos5433-pinctrl.dtsi"
+#include "exynos5433-tmu.dtsi"
diff --git a/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi b/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi
index f77ddaf21d04..82321984e1fb 100644
--- a/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi
@@ -20,8 +20,14 @@
 		interrupt-controller;
 		interrupt-parent = <&gic>;
 		#interrupt-cells = <2>;
-		interrupts = <0 0 0>, <0 1 0>, <0 2 0>, <0 3 0>,
-			     <0 4 0>, <0 5 0>, <0 6 0>, <0 7 0>;
+		interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
 	gpa1: gpa1 {
@@ -31,8 +37,14 @@
 		interrupt-controller;
 		interrupt-parent = <&gic>;
 		#interrupt-cells = <2>;
-		interrupts = <0 8 0>, <0 9 0>, <0 10 0>, <0 11 0>,
-			     <0 12 0>, <0 13 0>, <0 14 0>, <0 15 0>;
+		interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
 	gpa2: gpa2 {
diff --git a/arch/arm64/boot/dts/exynos/exynos7.dtsi b/arch/arm64/boot/dts/exynos/exynos7.dtsi
index 6328a66ed97e..80aa60e38237 100644
--- a/arch/arm64/boot/dts/exynos/exynos7.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7.dtsi
@@ -35,28 +35,28 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		cpu@0 {
+		cpu_atlas0: cpu@0 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57", "arm,armv8";
 			reg = <0x0>;
 			enable-method = "psci";
 		};
 
-		cpu@1 {
+		cpu_atlas1: cpu@1 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57", "arm,armv8";
 			reg = <0x1>;
 			enable-method = "psci";
 		};
 
-		cpu@2 {
+		cpu_atlas2: cpu@2 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57", "arm,armv8";
 			reg = <0x2>;
 			enable-method = "psci";
 		};
 
-		cpu@3 {
+		cpu_atlas3: cpu@3 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57", "arm,armv8";
 			reg = <0x3>;
@@ -106,7 +106,7 @@
 			pdma0: pdma@10E10000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x10E10000 0x1000>;
-				interrupts = <0 225 0>;
+				interrupts = <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock_fsys0 ACLK_PDMA0>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -117,7 +117,7 @@
 			pdma1: pdma@10EB0000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x10EB0000 0x1000>;
-				interrupts = <0 226 0>;
+				interrupts = <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock_fsys0 ACLK_PDMA1>;
 				clock-names = "apb_pclk";
 				#dma-cells = <1>;
@@ -220,7 +220,7 @@
 		serial_0: serial@13630000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x13630000 0x100>;
-			interrupts = <0 440 0>;
+			interrupts = <GIC_SPI 440 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peric0 PCLK_UART0>,
 				 <&clock_peric0 SCLK_UART0>;
 			clock-names = "uart", "clk_uart_baud0";
@@ -230,7 +230,7 @@
 		serial_1: serial@14c20000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x14c20000 0x100>;
-			interrupts = <0 456 0>;
+			interrupts = <GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peric1 PCLK_UART1>,
 				 <&clock_peric1 SCLK_UART1>;
 			clock-names = "uart", "clk_uart_baud0";
@@ -240,7 +240,7 @@
 		serial_2: serial@14c30000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x14c30000 0x100>;
-			interrupts = <0 457 0>;
+			interrupts = <GIC_SPI 457 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peric1 PCLK_UART2>,
 				 <&clock_peric1 SCLK_UART2>;
 			clock-names = "uart", "clk_uart_baud0";
@@ -250,7 +250,7 @@
 		serial_3: serial@14c40000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x14c40000 0x100>;
-			interrupts = <0 458 0>;
+			interrupts = <GIC_SPI 458 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peric1 PCLK_UART3>,
 				 <&clock_peric1 SCLK_UART3>;
 			clock-names = "uart", "clk_uart_baud0";
@@ -264,62 +264,62 @@
 			wakeup-interrupt-controller {
 				compatible = "samsung,exynos7-wakeup-eint";
 				interrupt-parent = <&gic>;
-				interrupts = <0 16 0>;
+				interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
 			};
 		};
 
 		pinctrl_bus0: pinctrl@13470000 {
 			compatible = "samsung,exynos7-pinctrl";
 			reg = <0x13470000 0x1000>;
-			interrupts = <0 383 0>;
+			interrupts = <GIC_SPI 383 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_nfc: pinctrl@14cd0000 {
 			compatible = "samsung,exynos7-pinctrl";
 			reg = <0x14cd0000 0x1000>;
-			interrupts = <0 473 0>;
+			interrupts = <GIC_SPI 473 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_touch: pinctrl@14ce0000 {
 			compatible = "samsung,exynos7-pinctrl";
 			reg = <0x14ce0000 0x1000>;
-			interrupts = <0 474 0>;
+			interrupts = <GIC_SPI 474 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_ff: pinctrl@14c90000 {
 			compatible = "samsung,exynos7-pinctrl";
 			reg = <0x14c90000 0x1000>;
-			interrupts = <0 475 0>;
+			interrupts = <GIC_SPI 475 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_ese: pinctrl@14ca0000 {
 			compatible = "samsung,exynos7-pinctrl";
 			reg = <0x14ca0000 0x1000>;
-			interrupts = <0 476 0>;
+			interrupts = <GIC_SPI 476 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_fsys0: pinctrl@10e60000 {
 			compatible = "samsung,exynos7-pinctrl";
 			reg = <0x10e60000 0x1000>;
-			interrupts = <0 221 0>;
+			interrupts = <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_fsys1: pinctrl@15690000 {
 			compatible = "samsung,exynos7-pinctrl";
 			reg = <0x15690000 0x1000>;
-			interrupts = <0 203 0>;
+			interrupts = <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		pinctrl_bus1: pinctrl@14870000 {
 			compatible = "samsung,exynos7-pinctrl";
 			reg = <0x14870000 0x1000>;
-			interrupts = <0 384 0>;
+			interrupts = <GIC_SPI 384 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		hsi2c_0: hsi2c@13640000 {
 			compatible = "samsung,exynos7-hsi2c";
 			reg = <0x13640000 0x1000>;
-			interrupts = <0 441 0>;
+			interrupts = <GIC_SPI 441 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -332,7 +332,7 @@
 		hsi2c_1: hsi2c@13650000 {
 			compatible = "samsung,exynos7-hsi2c";
 			reg = <0x13650000 0x1000>;
-			interrupts = <0 442 0>;
+			interrupts = <GIC_SPI 442 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -345,7 +345,7 @@
 		hsi2c_2: hsi2c@14e60000 {
 			compatible = "samsung,exynos7-hsi2c";
 			reg = <0x14e60000 0x1000>;
-			interrupts = <0 459 0>;
+			interrupts = <GIC_SPI 459 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -358,7 +358,7 @@
 		hsi2c_3: hsi2c@14e70000 {
 			compatible = "samsung,exynos7-hsi2c";
 			reg = <0x14e70000 0x1000>;
-			interrupts = <0 460 0>;
+			interrupts = <GIC_SPI 460 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -371,7 +371,7 @@
 		hsi2c_4: hsi2c@13660000 {
 			compatible = "samsung,exynos7-hsi2c";
 			reg = <0x13660000 0x1000>;
-			interrupts = <0 443 0>;
+			interrupts = <GIC_SPI 443 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -384,7 +384,7 @@
 		hsi2c_5: hsi2c@13670000 {
 			compatible = "samsung,exynos7-hsi2c";
 			reg = <0x13670000 0x1000>;
-			interrupts = <0 444 0>;
+			interrupts = <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -397,7 +397,7 @@
 		hsi2c_6: hsi2c@14e00000 {
 			compatible = "samsung,exynos7-hsi2c";
 			reg = <0x14e00000 0x1000>;
-			interrupts = <0 461 0>;
+			interrupts = <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -410,7 +410,7 @@
 		hsi2c_7: hsi2c@13e10000 {
 			compatible = "samsung,exynos7-hsi2c";
 			reg = <0x13e10000 0x1000>;
-			interrupts = <0 462 0>;
+			interrupts = <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -423,7 +423,7 @@
 		hsi2c_8: hsi2c@14e20000 {
 			compatible = "samsung,exynos7-hsi2c";
 			reg = <0x14e20000 0x1000>;
-			interrupts = <0 463 0>;
+			interrupts = <GIC_SPI 463 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -436,7 +436,7 @@
 		hsi2c_9: hsi2c@13680000 {
 			compatible = "samsung,exynos7-hsi2c";
 			reg = <0x13680000 0x1000>;
-			interrupts = <0 445 0>;
+			interrupts = <GIC_SPI 445 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -449,7 +449,7 @@
 		hsi2c_10: hsi2c@13690000 {
 			compatible = "samsung,exynos7-hsi2c";
 			reg = <0x13690000 0x1000>;
-			interrupts = <0 446 0>;
+			interrupts = <GIC_SPI 446 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -462,7 +462,7 @@
 		hsi2c_11: hsi2c@136a0000 {
 			compatible = "samsung,exynos7-hsi2c";
 			reg = <0x136a0000 0x1000>;
-			interrupts = <0 447 0>;
+			interrupts = <GIC_SPI 447 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			pinctrl-names = "default";
@@ -472,6 +472,16 @@
 			status = "disabled";
 		};
 
+		arm-pmu {
+			compatible = "arm,cortex-a57-pmu", "arm,armv8-pmuv3";
+			interrupts = <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-affinity = <&cpu_atlas0>, <&cpu_atlas1>,
+					     <&cpu_atlas2>, <&cpu_atlas3>;
+		};
+
 		timer {
 			compatible = "arm,armv8-timer";
 			interrupts = <GIC_PPI 13
@@ -499,7 +509,8 @@
 		rtc: rtc@10590000 {
 			compatible = "samsung,s3c6410-rtc";
 			reg = <0x10590000 0x100>;
-			interrupts = <0 355 0>, <0 356 0>;
+			interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_ccore PCLK_RTC>;
 			clock-names = "rtc";
 			status = "disabled";
@@ -508,7 +519,7 @@
 		watchdog: watchdog@101d0000 {
 			compatible = "samsung,exynos7-wdt";
 			reg = <0x101d0000 0x100>;
-			interrupts = <0 110 0>;
+			interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peris PCLK_WDT>;
 			clock-names = "watchdog";
 			samsung,syscon-phandle = <&pmu_system_controller>;
@@ -517,7 +528,7 @@
 
 		mmc_0: mmc@15740000 {
 			compatible = "samsung,exynos7-dw-mshc-smu";
-			interrupts = <0 201 0>;
+			interrupts = <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x15740000 0x2000>;
@@ -530,7 +541,7 @@
 
 		mmc_1: mmc@15750000 {
 			compatible = "samsung,exynos7-dw-mshc";
-			interrupts = <0 202 0>;
+			interrupts = <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x15750000 0x2000>;
@@ -543,7 +554,7 @@
 
 		mmc_2: mmc@15560000 {
 			compatible = "samsung,exynos7-dw-mshc-smu";
-			interrupts = <0 216 0>;
+			interrupts = <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x15560000 0x2000>;
@@ -557,7 +568,7 @@
 		adc: adc@13620000 {
 			compatible = "samsung,exynos7-adc";
 			reg = <0x13620000 0x100>;
-			interrupts = <0 448 0>;
+			interrupts = <GIC_SPI 448 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peric0 PCLK_ADCIF>;
 			clock-names = "adc";
 			#io-channel-cells = <1>;
@@ -577,7 +588,7 @@
 		tmuctrl_0: tmu@10060000 {
 			compatible = "samsung,exynos7-tmu";
 			reg = <0x10060000 0x200>;
-			interrupts = <0 108 0>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peris PCLK_TMU>,
 				 <&clock_peris SCLK_TMU>;
 			clock-names = "tmu_apbif", "tmu_sclk";
diff --git a/arch/arm64/boot/dts/freescale/Makefile b/arch/arm64/boot/dts/freescale/Makefile
index 1b7783db7de4..66027181fba4 100644
--- a/arch/arm64/boot/dts/freescale/Makefile
+++ b/arch/arm64/boot/dts/freescale/Makefile
@@ -1,5 +1,7 @@
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1043a-qds.dtb
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1043a-rdb.dtb
+dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1046a-qds.dtb
+dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1046a-rdb.dtb
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-qds.dtb
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-rdb.dtb
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-simu.dtb
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a-qds.dts b/arch/arm64/boot/dts/freescale/fsl-ls1043a-qds.dts
index dd9e91941df4..0989d635b558 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1043a-qds.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a-qds.dts
@@ -45,7 +45,7 @@
  */
 
 /dts-v1/;
-/include/ "fsl-ls1043a.dtsi"
+#include "fsl-ls1043a.dtsi"
 
 / {
 	model = "LS1043A QDS Board";
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts
index d2313e05fd22..c37110bc1506 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts
@@ -45,7 +45,7 @@
  */
 
 /dts-v1/;
-/include/ "fsl-ls1043a.dtsi"
+#include "fsl-ls1043a.dtsi"
 
 / {
 	model = "LS1043A RDB Board";
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi
index 97d331ec2500..ec13a6ecb754 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi
@@ -44,6 +44,8 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include <dt-bindings/thermal/thermal.h>
+
 / {
 	compatible = "fsl,ls1043a";
 	interrupt-parent = <&gic>;
@@ -66,6 +68,7 @@
 			reg = <0x0>;
 			clocks = <&clockgen 1 0>;
 			next-level-cache = <&l2>;
+			#cooling-cells = <2>;
 		};
 
 		cpu1: cpu@1 {
@@ -255,6 +258,81 @@
 			big-endian;
 		};
 
+		tmu: tmu@1f00000 {
+			compatible = "fsl,qoriq-tmu";
+			reg = <0x0 0x1f00000 0x0 0x10000>;
+			interrupts = <0 33 0x4>;
+			fsl,tmu-range = <0xb0000 0x9002a 0x6004c 0x30062>;
+			fsl,tmu-calibration = <0x00000000 0x00000026
+					       0x00000001 0x0000002d
+					       0x00000002 0x00000032
+					       0x00000003 0x00000039
+					       0x00000004 0x0000003f
+					       0x00000005 0x00000046
+					       0x00000006 0x0000004d
+					       0x00000007 0x00000054
+					       0x00000008 0x0000005a
+					       0x00000009 0x00000061
+					       0x0000000a 0x0000006a
+					       0x0000000b 0x00000071
+
+					       0x00010000 0x00000025
+					       0x00010001 0x0000002c
+					       0x00010002 0x00000035
+					       0x00010003 0x0000003d
+					       0x00010004 0x00000045
+					       0x00010005 0x0000004e
+					       0x00010006 0x00000057
+					       0x00010007 0x00000061
+					       0x00010008 0x0000006b
+					       0x00010009 0x00000076
+
+					       0x00020000 0x00000029
+					       0x00020001 0x00000033
+					       0x00020002 0x0000003d
+					       0x00020003 0x00000049
+					       0x00020004 0x00000056
+					       0x00020005 0x00000061
+					       0x00020006 0x0000006d
+
+					       0x00030000 0x00000021
+					       0x00030001 0x0000002a
+					       0x00030002 0x0000003c
+					       0x00030003 0x0000004e>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		thermal-zones {
+			cpu_thermal: cpu-thermal {
+				polling-delay-passive = <1000>;
+				polling-delay = <5000>;
+
+				thermal-sensors = <&tmu 3>;
+
+				trips {
+					cpu_alert: cpu-alert {
+						temperature = <85000>;
+						hysteresis = <2000>;
+						type = "passive";
+					};
+					cpu_crit: cpu-crit {
+						temperature = <95000>;
+						hysteresis = <2000>;
+						type = "critical";
+					};
+				};
+
+				cooling-maps {
+					map0 {
+						trip = <&cpu_alert>;
+						cooling-device =
+							<&cpu0 THERMAL_NO_LIMIT
+							THERMAL_NO_LIMIT>;
+					};
+				};
+			};
+		};
+
 		dspi0: dspi@2100000 {
 			compatible = "fsl,ls1043a-dspi", "fsl,ls1021a-v1.0-dspi";
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1046a-qds.dts b/arch/arm64/boot/dts/freescale/fsl-ls1046a-qds.dts
new file mode 100644
index 000000000000..290e5b014414
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1046a-qds.dts
@@ -0,0 +1,212 @@
+/*
+ * Device Tree Include file for Freescale Layerscape-1046A family SoC.
+ *
+ * Copyright 2016, Freescale Semiconductor, Inc.
+ *
+ * Shaohui Xie <Shaohui.Xie@nxp.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPLv2 or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "fsl-ls1046a.dtsi"
+
+/ {
+	model = "LS1046A QDS Board";
+	compatible = "fsl,ls1046a-qds", "fsl,ls1046a";
+
+	aliases {
+		gpio0 = &gpio0;
+		gpio1 = &gpio1;
+		gpio2 = &gpio2;
+		gpio3 = &gpio3;
+		serial0 = &duart0;
+		serial1 = &duart1;
+		serial2 = &duart2;
+		serial3 = &duart3;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+};
+
+&dspi {
+	bus-num = <0>;
+	status = "okay";
+
+	flash@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "n25q128a11", "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <10000000>;
+	};
+
+	flash@1 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "sst25wf040b", "jedec,spi-nor";
+		spi-cpol;
+		spi-cpha;
+		reg = <1>;
+		spi-max-frequency = <10000000>;
+	};
+
+	flash@2 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "en25s64", "jedec,spi-nor";
+		spi-cpol;
+		spi-cpha;
+		reg = <2>;
+		spi-max-frequency = <10000000>;
+	};
+};
+
+&duart0 {
+	status = "okay";
+};
+
+&duart1 {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+
+	pca9547@77 {
+		compatible = "nxp,pca9547";
+		reg = <0x77>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x2>;
+
+			ina220@40 {
+				compatible = "ti,ina220";
+				reg = <0x40>;
+				shunt-resistor = <1000>;
+			};
+
+			ina220@41 {
+				compatible = "ti,ina220";
+				reg = <0x41>;
+				shunt-resistor = <1000>;
+			};
+		};
+
+		i2c@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x3>;
+
+			rtc@51 {
+				compatible = "nxp,pcf2129";
+				reg = <0x51>;
+				/* IRQ10_B */
+				interrupts = <0 150 0x4>;
+			};
+
+			eeprom@56 {
+				compatible = "atmel,24c512";
+				reg = <0x56>;
+			};
+
+			eeprom@57 {
+				compatible = "atmel,24c512";
+				reg = <0x57>;
+			};
+
+			temp-sensor@4c {
+				compatible = "adi,adt7461a";
+				reg = <0x4c>;
+			};
+		};
+	};
+};
+
+&ifc {
+	#address-cells = <2>;
+	#size-cells = <1>;
+	/* NOR, NAND Flashes and FPGA on board */
+	ranges = <0x0 0x0 0x0 0x60000000 0x08000000
+		  0x1 0x0 0x0 0x7e800000 0x00010000
+		  0x2 0x0 0x0 0x7fb00000 0x00000100>;
+	status = "okay";
+
+	nor@0,0 {
+		compatible = "cfi-flash";
+		reg = <0x0 0x0 0x8000000>;
+		bank-width = <2>;
+		device-width = <1>;
+	};
+
+	nand@1,0 {
+		compatible = "fsl,ifc-nand";
+		reg = <0x1 0x0 0x10000>;
+	};
+
+	fpga: board-control@2,0 {
+		compatible = "fsl,ls1046aqds-fpga", "fsl,fpga-qixis";
+		reg = <0x2 0x0 0x0000100>;
+	};
+};
+
+&lpuart0 {
+	status = "okay";
+};
+
+&qspi {
+	num-cs = <2>;
+	bus-num = <0>;
+	status = "okay";
+
+	qflash0: s25fl128s@0 {
+		compatible = "spansion,m25p80";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <20000000>;
+		reg = <0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1046a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1046a-rdb.dts
new file mode 100644
index 000000000000..d1ccc000d05a
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1046a-rdb.dts
@@ -0,0 +1,150 @@
+/*
+ * Device Tree Include file for Freescale Layerscape-1046A family SoC.
+ *
+ * Copyright 2016, Freescale Semiconductor, Inc.
+ *
+ * Mingkai Hu <mingkai.hu@nxp.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPLv2 or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "fsl-ls1046a.dtsi"
+
+/ {
+	model = "LS1046A RDB Board";
+	compatible = "fsl,ls1046a-rdb", "fsl,ls1046a";
+
+	aliases {
+		serial0 = &duart0;
+		serial1 = &duart1;
+		serial2 = &duart2;
+		serial3 = &duart3;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+};
+
+&duart0 {
+	status = "okay";
+};
+
+&duart1 {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+
+	ina220@40 {
+		compatible = "ti,ina220";
+		reg = <0x40>;
+		shunt-resistor = <1000>;
+	};
+
+	temp-sensor@4c {
+		compatible = "adi,adt7461";
+		reg = <0x4c>;
+	};
+
+	eeprom@56 {
+		compatible = "atmel,24c512";
+		reg = <0x52>;
+	};
+
+	eeprom@57 {
+		compatible = "atmel,24c512";
+		reg = <0x53>;
+	};
+};
+
+&i2c3 {
+	status = "okay";
+
+	rtc@51 {
+		compatible = "nxp,pcf2129";
+		reg = <0x51>;
+	};
+};
+
+&ifc {
+	#address-cells = <2>;
+	#size-cells = <1>;
+	/* NAND Flashe and CPLD on board */
+	ranges = <0x0 0x0 0x0 0x7e800000 0x00010000
+		  0x2 0x0 0x0 0x7fb00000 0x00000100>;
+	status = "okay";
+
+	nand@0,0 {
+		compatible = "fsl,ifc-nand";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		reg = <0x0 0x0 0x10000>;
+	};
+
+	cpld: board-control@2,0 {
+		compatible = "fsl,ls1046ardb-cpld";
+		reg = <0x2 0x0 0x0000100>;
+	};
+};
+
+&qspi {
+	num-cs = <2>;
+	bus-num = <0>;
+	status = "okay";
+
+	qflash0: s25fs512s@0 {
+		compatible = "spansion,m25p80";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <20000000>;
+		reg = <0>;
+	};
+
+	qflash1: s25fs512s@1 {
+		compatible = "spansion,m25p80";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <20000000>;
+		reg = <1>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi
new file mode 100644
index 000000000000..38806ca53829
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi
@@ -0,0 +1,515 @@
+/*
+ * Device Tree Include file for Freescale Layerscape-1046A family SoC.
+ *
+ * Copyright 2016, Freescale Semiconductor, Inc.
+ *
+ * Mingkai Hu <mingkai.hu@nxp.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPLv2 or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	compatible = "fsl,ls1046a";
+	interrupt-parent = <&gic>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	aliases {
+		crypto = &crypto;
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0>;
+			clocks = <&clockgen 1 0>;
+			next-level-cache = <&l2>;
+			cpu-idle-states = <&CPU_PH20>;
+		};
+
+		cpu1: cpu@1 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x1>;
+			clocks = <&clockgen 1 0>;
+			next-level-cache = <&l2>;
+			cpu-idle-states = <&CPU_PH20>;
+		};
+
+		cpu2: cpu@2 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x2>;
+			clocks = <&clockgen 1 0>;
+			next-level-cache = <&l2>;
+			cpu-idle-states = <&CPU_PH20>;
+		};
+
+		cpu3: cpu@3 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x3>;
+			clocks = <&clockgen 1 0>;
+			next-level-cache = <&l2>;
+			cpu-idle-states = <&CPU_PH20>;
+		};
+
+		l2: l2-cache {
+			compatible = "cache";
+		};
+	};
+
+	idle-states {
+		/*
+		 * PSCI node is not added default, U-boot will add missing
+		 * parts if it determines to use PSCI.
+		 */
+		entry-method = "arm,psci";
+
+		CPU_PH20: cpu-ph20 {
+			compatible = "arm,idle-state";
+			idle-state-name = "PH20";
+			arm,psci-suspend-param = <0x00010000>;
+			entry-latency-us = <1000>;
+			exit-latency-us = <1000>;
+			min-residency-us = <3000>;
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+	};
+
+	sysclk: sysclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <100000000>;
+		clock-output-names = "sysclk";
+	};
+
+	reboot {
+		compatible ="syscon-reboot";
+		regmap = <&dcfg>;
+		offset = <0xb0>;
+		mask = <0x02>;
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 (GIC_CPU_MASK_RAW(0xf) |
+					  IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 14 (GIC_CPU_MASK_RAW(0xf) |
+					  IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 11 (GIC_CPU_MASK_RAW(0xf) |
+					  IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 10 (GIC_CPU_MASK_RAW(0xf) |
+					  IRQ_TYPE_LEVEL_LOW)>;
+	};
+
+	pmu {
+		compatible = "arm,cortex-a72-pmu";
+		interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>,
+				     <&cpu1>,
+				     <&cpu2>,
+				     <&cpu3>;
+	};
+
+	gic: interrupt-controller@1400000 {
+		compatible = "arm,gic-400";
+		#interrupt-cells = <3>;
+		interrupt-controller;
+		reg = <0x0 0x1410000 0 0x10000>, /* GICD */
+		      <0x0 0x1420000 0 0x20000>, /* GICC */
+		      <0x0 0x1440000 0 0x20000>, /* GICH */
+		      <0x0 0x1460000 0 0x20000>; /* GICV */
+		interrupts = <GIC_PPI 9 (GIC_CPU_MASK_RAW(0xf) |
+					 IRQ_TYPE_LEVEL_LOW)>;
+	};
+
+	soc {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ddr: memory-controller@1080000 {
+			compatible = "fsl,qoriq-memory-controller";
+			reg = <0x0 0x1080000 0x0 0x1000>;
+			interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
+			big-endian;
+		};
+
+		ifc: ifc@1530000 {
+			compatible = "fsl,ifc", "simple-bus";
+			reg = <0x0 0x1530000 0x0 0x10000>;
+			interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		qspi: quadspi@1550000 {
+			compatible = "fsl,ls1021a-qspi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x1550000 0x0 0x10000>,
+				<0x0 0x40000000 0x0 0x10000000>;
+			reg-names = "QuadSPI", "QuadSPI-memory";
+			interrupts = <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "qspi_en", "qspi";
+			clocks = <&clockgen 4 1>, <&clockgen 4 1>;
+			big-endian;
+			fsl,qspi-has-second-chip;
+			status = "disabled";
+		};
+
+		esdhc: esdhc@1560000 {
+			compatible = "fsl,esdhc";
+			reg = <0x0 0x1560000 0x0 0x10000>;
+			interrupts = <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>;
+			clock-frequency = <0>;
+			voltage-ranges = <1800 1800 3300 3300>;
+			sdhci,auto-cmd12;
+			big-endian;
+			bus-width = <4>;
+		};
+
+		scfg: scfg@1570000 {
+			compatible = "fsl,ls1046a-scfg", "syscon";
+			reg = <0x0 0x1570000 0x0 0x10000>;
+			big-endian;
+		};
+
+		crypto: crypto@1700000 {
+			compatible = "fsl,sec-v5.4", "fsl,sec-v5.0",
+				     "fsl,sec-v4.0";
+			fsl,sec-era = <8>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x0 0x00 0x1700000 0x100000>;
+			reg = <0x00 0x1700000 0x0 0x100000>;
+			interrupts = <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
+
+			sec_jr0: jr@10000 {
+				compatible = "fsl,sec-v5.4-job-ring",
+					     "fsl,sec-v5.0-job-ring",
+					     "fsl,sec-v4.0-job-ring";
+				reg	   = <0x10000 0x10000>;
+				interrupts = <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
+			};
+
+			sec_jr1: jr@20000 {
+				compatible = "fsl,sec-v5.4-job-ring",
+					     "fsl,sec-v5.0-job-ring",
+					     "fsl,sec-v4.0-job-ring";
+				reg	   = <0x20000 0x10000>;
+				interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
+			};
+
+			sec_jr2: jr@30000 {
+				compatible = "fsl,sec-v5.4-job-ring",
+					     "fsl,sec-v5.0-job-ring",
+					     "fsl,sec-v4.0-job-ring";
+				reg	   = <0x30000 0x10000>;
+				interrupts = <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+			};
+
+			sec_jr3: jr@40000 {
+				compatible = "fsl,sec-v5.4-job-ring",
+					     "fsl,sec-v5.0-job-ring",
+					     "fsl,sec-v4.0-job-ring";
+				reg	   = <0x40000 0x10000>;
+				interrupts = <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>;
+			};
+		};
+
+		dcfg: dcfg@1ee0000 {
+			compatible = "fsl,ls1046a-dcfg", "syscon";
+			reg = <0x0 0x1ee0000 0x0 0x10000>;
+			big-endian;
+		};
+
+		clockgen: clocking@1ee1000 {
+			compatible = "fsl,ls1046a-clockgen";
+			reg = <0x0 0x1ee1000 0x0 0x1000>;
+			#clock-cells = <2>;
+			clocks = <&sysclk>;
+		};
+
+		dspi: dspi@2100000 {
+			compatible = "fsl,ls1021a-v1.0-dspi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x2100000 0x0 0x10000>;
+			interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "dspi";
+			clocks = <&clockgen 4 1>;
+			spi-num-chipselects = <5>;
+			big-endian;
+			status = "disabled";
+		};
+
+		i2c0: i2c@2180000 {
+			compatible = "fsl,vf610-i2c";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x2180000 0x0 0x10000>;
+			interrupts = <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+			dmas = <&edma0 1 39>,
+			       <&edma0 1 38>;
+			dma-names = "tx", "rx";
+			status = "disabled";
+		};
+
+		i2c1: i2c@2190000 {
+			compatible = "fsl,vf610-i2c";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x2190000 0x0 0x10000>;
+			interrupts = <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+			status = "disabled";
+		};
+
+		i2c2: i2c@21a0000 {
+			compatible = "fsl,vf610-i2c";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x21a0000 0x0 0x10000>;
+			interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+			status = "disabled";
+		};
+
+		i2c3: i2c@21b0000 {
+			compatible = "fsl,vf610-i2c";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x21b0000 0x0 0x10000>;
+			interrupts = <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+			status = "disabled";
+		};
+
+		duart0: serial@21c0500 {
+			compatible = "fsl,ns16550", "ns16550a";
+			reg = <0x00 0x21c0500 0x0 0x100>;
+			interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+		};
+
+		duart1: serial@21c0600 {
+			compatible = "fsl,ns16550", "ns16550a";
+			reg = <0x00 0x21c0600 0x0 0x100>;
+			interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+		};
+
+		duart2: serial@21d0500 {
+			compatible = "fsl,ns16550", "ns16550a";
+			reg = <0x0 0x21d0500 0x0 0x100>;
+			interrupts = <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+		};
+
+		duart3: serial@21d0600 {
+			compatible = "fsl,ns16550", "ns16550a";
+			reg = <0x0 0x21d0600 0x0 0x100>;
+			interrupts = <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+		};
+
+		gpio0: gpio@2300000 {
+			compatible = "fsl,qoriq-gpio";
+			reg = <0x0 0x2300000 0x0 0x10000>;
+			interrupts = <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio1: gpio@2310000 {
+			compatible = "fsl,qoriq-gpio";
+			reg = <0x0 0x2310000 0x0 0x10000>;
+			interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio2: gpio@2320000 {
+			compatible = "fsl,qoriq-gpio";
+			reg = <0x0 0x2320000 0x0 0x10000>;
+			interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio3: gpio@2330000 {
+			compatible = "fsl,qoriq-gpio";
+			reg = <0x0 0x2330000 0x0 0x10000>;
+			interrupts = <GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		lpuart0: serial@2950000 {
+			compatible = "fsl,ls1021a-lpuart";
+			reg = <0x0 0x2950000 0x0 0x1000>;
+			interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 0>;
+			clock-names = "ipg";
+			status = "disabled";
+		};
+
+		lpuart1: serial@2960000 {
+			compatible = "fsl,ls1021a-lpuart";
+			reg = <0x0 0x2960000 0x0 0x1000>;
+			interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+			clock-names = "ipg";
+			status = "disabled";
+		};
+
+		lpuart2: serial@2970000 {
+			compatible = "fsl,ls1021a-lpuart";
+			reg = <0x0 0x2970000 0x0 0x1000>;
+			interrupts = <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+			clock-names = "ipg";
+			status = "disabled";
+		};
+
+		lpuart3: serial@2980000 {
+			compatible = "fsl,ls1021a-lpuart";
+			reg = <0x0 0x2980000 0x0 0x1000>;
+			interrupts = <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+			clock-names = "ipg";
+			status = "disabled";
+		};
+
+		lpuart4: serial@2990000 {
+			compatible = "fsl,ls1021a-lpuart";
+			reg = <0x0 0x2990000 0x0 0x1000>;
+			interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+			clock-names = "ipg";
+			status = "disabled";
+		};
+
+		lpuart5: serial@29a0000 {
+			compatible = "fsl,ls1021a-lpuart";
+			reg = <0x0 0x29a0000 0x0 0x1000>;
+			interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+			clock-names = "ipg";
+			status = "disabled";
+		};
+
+		wdog0: watchdog@2ad0000 {
+			compatible = "fsl,imx21-wdt";
+			reg = <0x0 0x2ad0000 0x0 0x10000>;
+			interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+			big-endian;
+		};
+
+		edma0: edma@2c00000 {
+			#dma-cells = <2>;
+			compatible = "fsl,vf610-edma";
+			reg = <0x0 0x2c00000 0x0 0x10000>,
+			      <0x0 0x2c10000 0x0 0x10000>,
+			      <0x0 0x2c20000 0x0 0x10000>;
+			interrupts = <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "edma-tx", "edma-err";
+			dma-channels = <32>;
+			big-endian;
+			clock-names = "dmamux0", "dmamux1";
+			clocks = <&clockgen 4 1>,
+				 <&clockgen 4 1>;
+		};
+
+		usb0: usb@2f00000 {
+			compatible = "snps,dwc3";
+			reg = <0x0 0x2f00000 0x0 0x10000>;
+			interrupts = <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>;
+			dr_mode = "host";
+			snps,quirk-frame-length-adjustment = <0x20>;
+		};
+
+		usb1: usb@3000000 {
+			compatible = "snps,dwc3";
+			reg = <0x0 0x3000000 0x0 0x10000>;
+			interrupts = <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
+			dr_mode = "host";
+			snps,quirk-frame-length-adjustment = <0x20>;
+		};
+
+		usb2: usb@3100000 {
+			compatible = "snps,dwc3";
+			reg = <0x0 0x3100000 0x0 0x10000>;
+			interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
+			dr_mode = "host";
+			snps,quirk-frame-length-adjustment = <0x20>;
+		};
+
+		sata: sata@3200000 {
+			compatible = "fsl,ls1046a-ahci";
+			reg = <0x0 0x3200000 0x0 0x10000>;
+			interrupts = <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2080a-qds.dts b/arch/arm64/boot/dts/freescale/fsl-ls2080a-qds.dts
index b0dd010979e7..8bc1f8f6fcfc 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls2080a-qds.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a-qds.dts
@@ -46,7 +46,7 @@
 
 /dts-v1/;
 
-/include/ "fsl-ls2080a.dtsi"
+#include "fsl-ls2080a.dtsi"
 
 / {
 	model = "Freescale Layerscape 2080a QDS Board";
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts
index ad0ebb8a1949..265e0a8b107b 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts
@@ -46,7 +46,7 @@
 
 /dts-v1/;
 
-/include/ "fsl-ls2080a.dtsi"
+#include "fsl-ls2080a.dtsi"
 
 / {
 	model = "Freescale Layerscape 2080a RDB Board";
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2080a-simu.dts b/arch/arm64/boot/dts/freescale/fsl-ls2080a-simu.dts
index 505d038078a3..290604b0a603 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls2080a-simu.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a-simu.dts
@@ -46,7 +46,7 @@
 
 /dts-v1/;
 
-/include/ "fsl-ls2080a.dtsi"
+#include "fsl-ls2080a.dtsi"
 
 / {
 	model = "Freescale Layerscape 2080a software Simulator model";
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi
index 7f0dc13b4087..e5935f28848c 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi
@@ -44,6 +44,8 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include <dt-bindings/thermal/thermal.h>
+
 / {
 	compatible = "fsl,ls2080a";
 	interrupt-parent = <&gic>;
@@ -62,15 +64,16 @@
 		 */
 
 		/* We have 4 clusters having 2 Cortex-A57 cores each */
-		cpu@0 {
+		cpu0: cpu@0 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57";
 			reg = <0x0>;
 			clocks = <&clockgen 1 0>;
 			next-level-cache = <&cluster0_l2>;
+			#cooling-cells = <2>;
 		};
 
-		cpu@1 {
+		cpu1: cpu@1 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57";
 			reg = <0x1>;
@@ -78,15 +81,16 @@
 			next-level-cache = <&cluster0_l2>;
 		};
 
-		cpu@100 {
+		cpu2: cpu@100 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57";
 			reg = <0x100>;
 			clocks = <&clockgen 1 1>;
 			next-level-cache = <&cluster1_l2>;
+			#cooling-cells = <2>;
 		};
 
-		cpu@101 {
+		cpu3: cpu@101 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57";
 			reg = <0x101>;
@@ -94,15 +98,16 @@
 			next-level-cache = <&cluster1_l2>;
 		};
 
-		cpu@200 {
+		cpu4: cpu@200 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57";
 			reg = <0x200>;
 			clocks = <&clockgen 1 2>;
 			next-level-cache = <&cluster2_l2>;
+			#cooling-cells = <2>;
 		};
 
-		cpu@201 {
+		cpu5: cpu@201 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57";
 			reg = <0x201>;
@@ -110,15 +115,16 @@
 			next-level-cache = <&cluster2_l2>;
 		};
 
-		cpu@300 {
+		cpu6: cpu@300 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57";
 			reg = <0x300>;
 			clocks = <&clockgen 1 3>;
 			next-level-cache = <&cluster3_l2>;
+			#cooling-cells = <2>;
 		};
 
-		cpu@301 {
+		cpu7: cpu@301 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57";
 			reg = <0x301>;
@@ -216,6 +222,106 @@
 			clocks = <&sysclk>;
 		};
 
+		dcfg: dcfg@1e00000 {
+			compatible = "fsl,ls2080a-dcfg", "syscon";
+			reg = <0x0 0x1e00000 0x0 0x10000>;
+			little-endian;
+		};
+
+		tmu: tmu@1f80000 {
+			compatible = "fsl,qoriq-tmu";
+			reg = <0x0 0x1f80000 0x0 0x10000>;
+			interrupts = <0 23 0x4>;
+			fsl,tmu-range = <0xb0000 0x9002a 0x6004c 0x30062>;
+			fsl,tmu-calibration = <0x00000000 0x00000026
+					       0x00000001 0x0000002d
+					       0x00000002 0x00000032
+					       0x00000003 0x00000039
+					       0x00000004 0x0000003f
+					       0x00000005 0x00000046
+					       0x00000006 0x0000004d
+					       0x00000007 0x00000054
+					       0x00000008 0x0000005a
+					       0x00000009 0x00000061
+					       0x0000000a 0x0000006a
+					       0x0000000b 0x00000071
+
+					       0x00010000 0x00000025
+					       0x00010001 0x0000002c
+					       0x00010002 0x00000035
+					       0x00010003 0x0000003d
+					       0x00010004 0x00000045
+					       0x00010005 0x0000004e
+					       0x00010006 0x00000057
+					       0x00010007 0x00000061
+					       0x00010008 0x0000006b
+					       0x00010009 0x00000076
+
+					       0x00020000 0x00000029
+					       0x00020001 0x00000033
+					       0x00020002 0x0000003d
+					       0x00020003 0x00000049
+					       0x00020004 0x00000056
+					       0x00020005 0x00000061
+					       0x00020006 0x0000006d
+
+					       0x00030000 0x00000021
+					       0x00030001 0x0000002a
+					       0x00030002 0x0000003c
+					       0x00030003 0x0000004e>;
+			little-endian;
+			#thermal-sensor-cells = <1>;
+		};
+
+		thermal-zones {
+			cpu_thermal: cpu-thermal {
+				polling-delay-passive = <1000>;
+				polling-delay = <5000>;
+
+				thermal-sensors = <&tmu 4>;
+
+				trips {
+					cpu_alert: cpu-alert {
+						temperature = <75000>;
+						hysteresis = <2000>;
+						type = "passive";
+					};
+					cpu_crit: cpu-crit {
+						temperature = <85000>;
+						hysteresis = <2000>;
+						type = "critical";
+					};
+				};
+
+				cooling-maps {
+					map0 {
+						trip = <&cpu_alert>;
+						cooling-device =
+							<&cpu0 THERMAL_NO_LIMIT
+							THERMAL_NO_LIMIT>;
+					};
+					map1 {
+						trip = <&cpu_alert>;
+						cooling-device =
+							<&cpu2 THERMAL_NO_LIMIT
+							THERMAL_NO_LIMIT>;
+					};
+					map2 {
+						trip = <&cpu_alert>;
+						cooling-device =
+							<&cpu4 THERMAL_NO_LIMIT
+							THERMAL_NO_LIMIT>;
+					};
+					map3 {
+						trip = <&cpu_alert>;
+						cooling-device =
+							<&cpu6 THERMAL_NO_LIMIT
+							THERMAL_NO_LIMIT>;
+					};
+				};
+			};
+		};
+
 		serial0: serial@21c0500 {
 			compatible = "fsl,ns16550", "ns16550a";
 			reg = <0x0 0x21c0500 0x0 0x100>;
diff --git a/arch/arm64/boot/dts/hisilicon/Makefile b/arch/arm64/boot/dts/hisilicon/Makefile
index d5f43a06b1c1..c8b8f803cf90 100644
--- a/arch/arm64/boot/dts/hisilicon/Makefile
+++ b/arch/arm64/boot/dts/hisilicon/Makefile
@@ -1,6 +1,7 @@
 dtb-$(CONFIG_ARCH_HISI) += hi6220-hikey.dtb
 dtb-$(CONFIG_ARCH_HISI) += hip05-d02.dtb
 dtb-$(CONFIG_ARCH_HISI) += hip06-d03.dtb
+dtb-$(CONFIG_ARCH_HISI) += hip07-d05.dtb
 
 always		:= $(dtb-y)
 subdir-y	:= $(dts-dirs)
diff --git a/arch/arm64/boot/dts/hisilicon/hi6220.dtsi b/arch/arm64/boot/dts/hisilicon/hi6220.dtsi
index 17839db585d5..470461ddd427 100644
--- a/arch/arm64/boot/dts/hisilicon/hi6220.dtsi
+++ b/arch/arm64/boot/dts/hisilicon/hi6220.dtsi
@@ -364,6 +364,7 @@
 			reg = <0x0 0xf7010000  0x0 0x27c>;
 			#address-cells = <1>;
 			#size-cells = <1>;
+			#pinctrl-cells = <1>;
 			#gpio-range-cells = <3>;
 			pinctrl-single,register-width = <32>;
 			pinctrl-single,function-mask = <7>;
@@ -402,6 +403,7 @@
 			reg = <0x0 0xf7010800 0x0 0x28c>;
 			#address-cells = <1>;
 			#size-cells = <1>;
+			#pinctrl-cells = <1>;
 			pinctrl-single,register-width = <32>;
 		};
 
@@ -410,6 +412,7 @@
 			reg = <0x0 0xf8001800 0x0 0x78>;
 			#address-cells = <1>;
 			#size-cells = <1>;
+			#pinctrl-cells = <1>;
 			pinctrl-single,register-width = <32>;
 		};
 
@@ -747,7 +750,6 @@
 			clocks = <&sys_ctrl HI6220_USBOTG_HCLK>;
 			clock-names = "otg";
 			dr_mode = "otg";
-			g-use-dma;
 			g-rx-fifo-size = <512>;
 			g-np-tx-fifo-size = <128>;
 			g-tx-fifo-size = <128 128 128 128 128 128>;
diff --git a/arch/arm64/boot/dts/hisilicon/hip06-d03.dts b/arch/arm64/boot/dts/hisilicon/hip06-d03.dts
index f54b28359607..7c4114a67753 100644
--- a/arch/arm64/boot/dts/hisilicon/hip06-d03.dts
+++ b/arch/arm64/boot/dts/hisilicon/hip06-d03.dts
@@ -41,18 +41,10 @@
 	status = "ok";
 };
 
-&sas0 {
-	status = "ok";
-};
-
 &sas1 {
 	status = "ok";
 };
 
-&sas2 {
-	status = "ok";
-};
-
 &usb_ohci {
 	status = "ok";
 };
diff --git a/arch/arm64/boot/dts/hisilicon/hip06.dtsi b/arch/arm64/boot/dts/hisilicon/hip06.dtsi
index b548763366dd..a049b64f2101 100644
--- a/arch/arm64/boot/dts/hisilicon/hip06.dtsi
+++ b/arch/arm64/boot/dts/hisilicon/hip06.dtsi
@@ -318,11 +318,17 @@
 		#size-cells = <2>;
 		ranges;
 
+		refclk: refclk {
+			compatible = "fixed-clock";
+			clock-frequency = <50000000>;
+			#clock-cells = <0>;
+		};
+
 		usb_ohci: ohci@a7030000 {
 			compatible = "generic-ohci";
 			reg = <0x0 0xa7030000 0x0 0x10000>;
 			interrupt-parent = <&mbigen_usb>;
-			interrupts = <64 4>;
+			interrupts = <640 4>;
 			dma-coherent;
 			status = "disabled";
 		};
@@ -331,7 +337,7 @@
 			compatible = "generic-ehci";
 			reg = <0x0 0xa7020000 0x0 0x10000>;
 			interrupt-parent = <&mbigen_usb>;
-			interrupts = <65 4>;
+			interrupts = <641 4>;
 			dma-coherent;
 			status = "disabled";
 		};
@@ -508,7 +514,7 @@
 			};
 		};
 
-		eth0: ethernet@4{
+		eth0: ethernet-4{
 			compatible = "hisilicon,hns-nic-v2";
 			ae-handle = <&dsaf0>;
 			port-idx-in-ae = <4>;
@@ -517,7 +523,7 @@
 			dma-coherent;
 		};
 
-		eth1: ethernet@5{
+		eth1: ethernet-5{
 			compatible = "hisilicon,hns-nic-v2";
 			ae-handle = <&dsaf0>;
 			port-idx-in-ae = <5>;
@@ -526,7 +532,7 @@
 			dma-coherent;
 		};
 
-		eth2: ethernet@0{
+		eth2: ethernet-0{
 			compatible = "hisilicon,hns-nic-v2";
 			ae-handle = <&dsaf0>;
 			port-idx-in-ae = <0>;
@@ -535,7 +541,7 @@
 			dma-coherent;
 		};
 
-		eth3: ethernet@1{
+		eth3: ethernet-1{
 			compatible = "hisilicon,hns-nic-v2";
 			ae-handle = <&dsaf0>;
 			port-idx-in-ae = <1>;
@@ -552,6 +558,7 @@
 			ctrl-reset-reg = <0xa60>;
 			ctrl-reset-sts-reg = <0x5a30>;
 			ctrl-clock-ena-reg = <0x338>;
+			clocks = <&refclk 0>;
 			queue-count = <16>;
 			phy-count = <8>;
 			dma-coherent;
@@ -590,10 +597,11 @@
 			reg = <0 0xa2000000 0 0x10000>;
 			sas-addr = [50 01 88 20 16 00 00 00];
 			hisilicon,sas-syscon = <&pcie_subctl>;
-			am-max-trans;
+			hip06-sas-v2-quirk-amt;
 			ctrl-reset-reg = <0xa18>;
 			ctrl-reset-sts-reg = <0x5a0c>;
 			ctrl-clock-ena-reg = <0x318>;
+			clocks = <&refclk 0>;
 			queue-count = <16>;
 			phy-count = <8>;
 			dma-coherent;
@@ -635,6 +643,7 @@
 			ctrl-reset-reg = <0xae0>;
 			ctrl-reset-sts-reg = <0x5a70>;
 			ctrl-clock-ena-reg = <0x3a8>;
+			clocks = <&refclk 0>;
 			queue-count = <16>;
 			phy-count = <9>;
 			dma-coherent;
diff --git a/arch/arm64/boot/dts/hisilicon/hip07-d05.dts b/arch/arm64/boot/dts/hisilicon/hip07-d05.dts
new file mode 100644
index 000000000000..e05844230583
--- /dev/null
+++ b/arch/arm64/boot/dts/hisilicon/hip07-d05.dts
@@ -0,0 +1,66 @@
+/**
+ * dts file for Hisilicon D05 Development Board
+ *
+ * Copyright (C) 2016 Hisilicon Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * publishhed by the Free Software Foundation.
+ *
+ */
+
+/dts-v1/;
+
+#include "hip07.dtsi"
+
+/ {
+	model = "Hisilicon Hip07 D05 Development Board";
+	compatible = "hisilicon,hip07-d05";
+
+	/* the mem node will be updated by UEFI. */
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x00000000 0x0 0x40000000>;
+		numa-node-id = <0>;
+	};
+
+	distance-map {
+		compatible = "numa-distance-map-v1";
+		distance-matrix = <0 0 10>,
+				  <0 1 15>,
+				  <0 2 20>,
+				  <0 3 25>,
+				  <1 0 15>,
+				  <1 1 10>,
+				  <1 2 25>,
+				  <1 3 30>,
+				  <2 0 20>,
+				  <2 1 25>,
+				  <2 2 10>,
+				  <2 3 15>,
+				  <3 0 25>,
+				  <3 1 30>,
+				  <3 2 15>,
+				  <3 3 10>;
+	};
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+};
+
+&uart0 {
+	status = "ok";
+};
+
+&usb_ohci {
+	status = "ok";
+};
+
+&usb_ehci {
+	status = "ok";
+};
diff --git a/arch/arm64/boot/dts/hisilicon/hip07.dtsi b/arch/arm64/boot/dts/hisilicon/hip07.dtsi
new file mode 100644
index 000000000000..5144eb1c179d
--- /dev/null
+++ b/arch/arm64/boot/dts/hisilicon/hip07.dtsi
@@ -0,0 +1,1059 @@
+/**
+ * dts file for Hisilicon D05 Development Board
+ *
+ * Copyright (C) 2016 Hisilicon Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * publishhed by the Free Software Foundation.
+ *
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	compatible = "hisilicon,hip07-d05";
+	interrupt-parent = <&gic>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+				core1 {
+					cpu = <&cpu1>;
+				};
+				core2 {
+					cpu = <&cpu2>;
+				};
+				core3 {
+					cpu = <&cpu3>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu4>;
+				};
+				core1 {
+					cpu = <&cpu5>;
+				};
+				core2 {
+					cpu = <&cpu6>;
+				};
+				core3 {
+					cpu = <&cpu7>;
+				};
+			};
+
+			cluster2 {
+				core0 {
+					cpu = <&cpu8>;
+				};
+				core1 {
+					cpu = <&cpu9>;
+				};
+				core2 {
+					cpu = <&cpu10>;
+				};
+				core3 {
+					cpu = <&cpu11>;
+				};
+			};
+
+			cluster3 {
+				core0 {
+					cpu = <&cpu12>;
+				};
+				core1 {
+					cpu = <&cpu13>;
+				};
+				core2 {
+					cpu = <&cpu14>;
+				};
+				core3 {
+					cpu = <&cpu15>;
+				};
+			};
+
+			cluster4 {
+				core0 {
+					cpu = <&cpu16>;
+				};
+				core1 {
+					cpu = <&cpu17>;
+				};
+				core2 {
+					cpu = <&cpu18>;
+				};
+				core3 {
+					cpu = <&cpu19>;
+				};
+			};
+
+			cluster5 {
+				core0 {
+					cpu = <&cpu20>;
+				};
+				core1 {
+					cpu = <&cpu21>;
+				};
+				core2 {
+					cpu = <&cpu22>;
+				};
+				core3 {
+					cpu = <&cpu23>;
+				};
+			};
+
+			cluster6 {
+				core0 {
+					cpu = <&cpu24>;
+				};
+				core1 {
+					cpu = <&cpu25>;
+				};
+				core2 {
+					cpu = <&cpu26>;
+				};
+				core3 {
+					cpu = <&cpu27>;
+				};
+			};
+
+			cluster7 {
+				core0 {
+					cpu = <&cpu28>;
+				};
+				core1 {
+					cpu = <&cpu29>;
+				};
+				core2 {
+					cpu = <&cpu30>;
+				};
+				core3 {
+					cpu = <&cpu31>;
+				};
+			};
+
+			cluster8 {
+				core0 {
+					cpu = <&cpu32>;
+				};
+				core1 {
+					cpu = <&cpu33>;
+				};
+				core2 {
+					cpu = <&cpu34>;
+				};
+				core3 {
+					cpu = <&cpu35>;
+				};
+			};
+
+			cluster9 {
+				core0 {
+					cpu = <&cpu36>;
+				};
+				core1 {
+					cpu = <&cpu37>;
+				};
+				core2 {
+					cpu = <&cpu38>;
+				};
+				core3 {
+					cpu = <&cpu39>;
+				};
+			};
+
+			cluster10 {
+				core0 {
+					cpu = <&cpu40>;
+				};
+				core1 {
+					cpu = <&cpu41>;
+				};
+				core2 {
+					cpu = <&cpu42>;
+				};
+				core3 {
+					cpu = <&cpu43>;
+				};
+			};
+
+			cluster11 {
+				core0 {
+					cpu = <&cpu44>;
+				};
+				core1 {
+					cpu = <&cpu45>;
+				};
+				core2 {
+					cpu = <&cpu46>;
+				};
+				core3 {
+					cpu = <&cpu47>;
+				};
+			};
+
+			cluster12 {
+				core0 {
+					cpu = <&cpu48>;
+				};
+				core1 {
+					cpu = <&cpu49>;
+				};
+				core2 {
+					cpu = <&cpu50>;
+				};
+				core3 {
+					cpu = <&cpu51>;
+				};
+			};
+
+			cluster13 {
+				core0 {
+					cpu = <&cpu52>;
+				};
+				core1 {
+					cpu = <&cpu53>;
+				};
+				core2 {
+					cpu = <&cpu54>;
+				};
+				core3 {
+					cpu = <&cpu55>;
+				};
+			};
+
+			cluster14 {
+				core0 {
+					cpu = <&cpu56>;
+				};
+				core1 {
+					cpu = <&cpu57>;
+				};
+				core2 {
+					cpu = <&cpu58>;
+				};
+				core3 {
+					cpu = <&cpu59>;
+				};
+			};
+
+			cluster15 {
+				core0 {
+					cpu = <&cpu60>;
+				};
+				core1 {
+					cpu = <&cpu61>;
+				};
+				core2 {
+					cpu = <&cpu62>;
+				};
+				core3 {
+					cpu = <&cpu63>;
+				};
+			};
+		};
+
+		cpu0: cpu@10000 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10000>;
+			enable-method = "psci";
+			next-level-cache = <&cluster0_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu1: cpu@10001 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10001>;
+			enable-method = "psci";
+			next-level-cache = <&cluster0_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu2: cpu@10002 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10002>;
+			enable-method = "psci";
+			next-level-cache = <&cluster0_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu3: cpu@10003 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10003>;
+			enable-method = "psci";
+			next-level-cache = <&cluster0_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu4: cpu@10100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10100>;
+			enable-method = "psci";
+			next-level-cache = <&cluster1_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu5: cpu@10101 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10101>;
+			enable-method = "psci";
+			next-level-cache = <&cluster1_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu6: cpu@10102 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10102>;
+			enable-method = "psci";
+			next-level-cache = <&cluster1_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu7: cpu@10103 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10103>;
+			enable-method = "psci";
+			next-level-cache = <&cluster1_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu8: cpu@10200 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10200>;
+			enable-method = "psci";
+			next-level-cache = <&cluster2_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu9: cpu@10201 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10201>;
+			enable-method = "psci";
+			next-level-cache = <&cluster2_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu10: cpu@10202 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10202>;
+			enable-method = "psci";
+			next-level-cache = <&cluster2_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu11: cpu@10203 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10203>;
+			enable-method = "psci";
+			next-level-cache = <&cluster2_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu12: cpu@10300 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10300>;
+			enable-method = "psci";
+			next-level-cache = <&cluster3_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu13: cpu@10301 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10301>;
+			enable-method = "psci";
+			next-level-cache = <&cluster3_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu14: cpu@10302 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10302>;
+			enable-method = "psci";
+			next-level-cache = <&cluster3_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu15: cpu@10303 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x10303>;
+			enable-method = "psci";
+			next-level-cache = <&cluster3_l2>;
+			numa-node-id = <0>;
+		};
+
+		cpu16: cpu@30000 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30000>;
+			enable-method = "psci";
+			next-level-cache = <&cluster4_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu17: cpu@30001 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30001>;
+			enable-method = "psci";
+			next-level-cache = <&cluster4_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu18: cpu@30002 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30002>;
+			enable-method = "psci";
+			next-level-cache = <&cluster4_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu19: cpu@30003 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30003>;
+			enable-method = "psci";
+			next-level-cache = <&cluster4_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu20: cpu@30100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30100>;
+			enable-method = "psci";
+			next-level-cache = <&cluster5_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu21: cpu@30101 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30101>;
+			enable-method = "psci";
+			next-level-cache = <&cluster5_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu22: cpu@30102 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30102>;
+			enable-method = "psci";
+			next-level-cache = <&cluster5_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu23: cpu@30103 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30103>;
+			enable-method = "psci";
+			next-level-cache = <&cluster5_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu24: cpu@30200 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30200>;
+			enable-method = "psci";
+			next-level-cache = <&cluster6_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu25: cpu@30201 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30201>;
+			enable-method = "psci";
+			next-level-cache = <&cluster6_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu26: cpu@30202 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30202>;
+			enable-method = "psci";
+			next-level-cache = <&cluster6_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu27: cpu@30203 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30203>;
+			enable-method = "psci";
+			next-level-cache = <&cluster6_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu28: cpu@30300 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30300>;
+			enable-method = "psci";
+			next-level-cache = <&cluster7_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu29: cpu@30301 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30301>;
+			enable-method = "psci";
+			next-level-cache = <&cluster7_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu30: cpu@30302 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30302>;
+			enable-method = "psci";
+			next-level-cache = <&cluster7_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu31: cpu@30303 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x30303>;
+			enable-method = "psci";
+			next-level-cache = <&cluster7_l2>;
+			numa-node-id = <1>;
+		};
+
+		cpu32: cpu@50000 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50000>;
+			enable-method = "psci";
+			next-level-cache = <&cluster8_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu33: cpu@50001 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50001>;
+			enable-method = "psci";
+			next-level-cache = <&cluster8_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu34: cpu@50002 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50002>;
+			enable-method = "psci";
+			next-level-cache = <&cluster8_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu35: cpu@50003 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50003>;
+			enable-method = "psci";
+			next-level-cache = <&cluster8_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu36: cpu@50100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50100>;
+			enable-method = "psci";
+			next-level-cache = <&cluster9_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu37: cpu@50101 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50101>;
+			enable-method = "psci";
+			next-level-cache = <&cluster9_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu38: cpu@50102 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50102>;
+			enable-method = "psci";
+			next-level-cache = <&cluster9_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu39: cpu@50103 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50103>;
+			enable-method = "psci";
+			next-level-cache = <&cluster9_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu40: cpu@50200 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50200>;
+			enable-method = "psci";
+			next-level-cache = <&cluster10_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu41: cpu@50201 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50201>;
+			enable-method = "psci";
+			next-level-cache = <&cluster10_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu42: cpu@50202 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50202>;
+			enable-method = "psci";
+			next-level-cache = <&cluster10_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu43: cpu@50203 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50203>;
+			enable-method = "psci";
+			next-level-cache = <&cluster10_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu44: cpu@50300 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50300>;
+			enable-method = "psci";
+			next-level-cache = <&cluster11_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu45: cpu@50301 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50301>;
+			enable-method = "psci";
+			next-level-cache = <&cluster11_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu46: cpu@50302 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50302>;
+			enable-method = "psci";
+			next-level-cache = <&cluster11_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu47: cpu@50303 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x50303>;
+			enable-method = "psci";
+			next-level-cache = <&cluster11_l2>;
+			numa-node-id = <2>;
+		};
+
+		cpu48: cpu@70000 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70000>;
+			enable-method = "psci";
+			next-level-cache = <&cluster12_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu49: cpu@70001 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70001>;
+			enable-method = "psci";
+			next-level-cache = <&cluster12_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu50: cpu@70002 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70002>;
+			enable-method = "psci";
+			next-level-cache = <&cluster12_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu51: cpu@70003 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70003>;
+			enable-method = "psci";
+			next-level-cache = <&cluster12_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu52: cpu@70100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70100>;
+			enable-method = "psci";
+			next-level-cache = <&cluster13_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu53: cpu@70101 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70101>;
+			enable-method = "psci";
+			next-level-cache = <&cluster13_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu54: cpu@70102 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70102>;
+			enable-method = "psci";
+			next-level-cache = <&cluster13_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu55: cpu@70103 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70103>;
+			enable-method = "psci";
+			next-level-cache = <&cluster13_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu56: cpu@70200 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70200>;
+			enable-method = "psci";
+			next-level-cache = <&cluster14_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu57: cpu@70201 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70201>;
+			enable-method = "psci";
+			next-level-cache = <&cluster14_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu58: cpu@70202 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70202>;
+			enable-method = "psci";
+			next-level-cache = <&cluster14_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu59: cpu@70203 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70203>;
+			enable-method = "psci";
+			next-level-cache = <&cluster14_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu60: cpu@70300 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70300>;
+			enable-method = "psci";
+			next-level-cache = <&cluster15_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu61: cpu@70301 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70301>;
+			enable-method = "psci";
+			next-level-cache = <&cluster15_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu62: cpu@70302 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70302>;
+			enable-method = "psci";
+			next-level-cache = <&cluster15_l2>;
+			numa-node-id = <3>;
+		};
+
+		cpu63: cpu@70303 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a72", "arm,armv8";
+			reg = <0x70303>;
+			enable-method = "psci";
+			next-level-cache = <&cluster15_l2>;
+			numa-node-id = <3>;
+		};
+
+		cluster0_l2: l2-cache0 {
+			compatible = "cache";
+		};
+
+		cluster1_l2: l2-cache1 {
+			compatible = "cache";
+		};
+
+		cluster2_l2: l2-cache2 {
+			compatible = "cache";
+		};
+
+		cluster3_l2: l2-cache3 {
+			compatible = "cache";
+		};
+
+		cluster4_l2: l2-cache4 {
+			compatible = "cache";
+		};
+
+		cluster5_l2: l2-cache5 {
+			compatible = "cache";
+		};
+
+		cluster6_l2: l2-cache6 {
+			compatible = "cache";
+		};
+
+		cluster7_l2: l2-cache7 {
+			compatible = "cache";
+		};
+
+		cluster8_l2: l2-cache8 {
+			compatible = "cache";
+		};
+
+		cluster9_l2: l2-cache9 {
+			compatible = "cache";
+		};
+
+		cluster10_l2: l2-cache10 {
+			compatible = "cache";
+		};
+
+		cluster11_l2: l2-cache11 {
+			compatible = "cache";
+		};
+
+		cluster12_l2: l2-cache12 {
+			compatible = "cache";
+		};
+
+		cluster13_l2: l2-cache13 {
+			compatible = "cache";
+		};
+
+		cluster14_l2: l2-cache14 {
+			compatible = "cache";
+		};
+
+		cluster15_l2: l2-cache15 {
+			compatible = "cache";
+		};
+	};
+
+	gic: interrupt-controller@4d000000 {
+		compatible = "arm,gic-v3";
+		#interrupt-cells = <3>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		interrupt-controller;
+		#redistributor-regions = <4>;
+		redistributor-stride = <0x0 0x40000>;
+		reg = <0x0 0x4d000000 0x0 0x10000>,	/* GICD */
+		      <0x0 0x4d100000 0x0 0x400000>,	/* p0 GICR node 0 */
+		      <0x0 0x6d100000 0x0 0x400000>,	/* p0 GICR node 1 */
+		      <0x400 0x4d100000 0x0 0x400000>,	/* p1 GICR node 2 */
+		      <0x400 0x6d100000 0x0 0x400000>,	/* p1 GICR node 3 */
+		      <0x0 0xfe000000 0x0 0x10000>,	/* GICC */
+		      <0x0 0xfe010000 0x0 0x10000>,	/* GICH */
+		      <0x0 0xfe020000 0x0 0x10000>;	/* GICV */
+		interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+
+		p0_its_peri_a: interrupt-controller@4c000000 {
+			compatible = "arm,gic-v3-its";
+			msi-controller;
+			#msi-cells = <1>;
+			reg = <0x0 0x4c000000 0x0 0x40000>;
+		};
+
+		p0_its_peri_b: interrupt-controller@6c000000 {
+			compatible = "arm,gic-v3-its";
+			msi-controller;
+			#msi-cells = <1>;
+			reg = <0x0 0x6c000000 0x0 0x40000>;
+		};
+
+		p0_its_dsa_a: interrupt-controller@c6000000 {
+			compatible = "arm,gic-v3-its";
+			msi-controller;
+			#msi-cells = <1>;
+			reg = <0x0 0xc6000000 0x0 0x40000>;
+		};
+
+		p0_its_dsa_b: interrupt-controller@8,c6000000 {
+			compatible = "arm,gic-v3-its";
+			msi-controller;
+			#msi-cells = <1>;
+			reg = <0x8 0xc6000000 0x0 0x40000>;
+		};
+
+		p1_its_peri_a: interrupt-controller@400,4c000000 {
+			compatible = "arm,gic-v3-its";
+			msi-controller;
+			#msi-cells = <1>;
+			reg = <0x400 0x4c000000 0x0 0x40000>;
+		};
+
+		p1_its_peri_b: interrupt-controller@400,6c000000 {
+			compatible = "arm,gic-v3-its";
+			msi-controller;
+			#msi-cells = <1>;
+			reg = <0x400 0x6c000000 0x0 0x40000>;
+		};
+
+		p1_its_dsa_a: interrupt-controller@400,c6000000 {
+			compatible = "arm,gic-v3-its";
+			msi-controller;
+			#msi-cells = <1>;
+			reg = <0x400 0xc6000000 0x0 0x40000>;
+		};
+
+		p1_its_dsa_b: interrupt-controller@408,c6000000 {
+			compatible = "arm,gic-v3-its";
+			msi-controller;
+			#msi-cells = <1>;
+			reg = <0x408 0xc6000000 0x0 0x40000>;
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	pmu {
+		compatible = "arm,cortex-a72-pmu";
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	p0_mbigen_peri_b: interrupt-controller@60080000 {
+		compatible = "hisilicon,mbigen-v2";
+		reg = <0x0 0x60080000 0x0 0x10000>;
+
+		mbigen_uart: uart_intc {
+			msi-parent = <&p0_its_peri_b 0x120c7>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			num-pins = <1>;
+		};
+	};
+
+	p0_mbigen_pcie_a: interrupt-controller@a0080000 {
+		compatible = "hisilicon,mbigen-v2";
+		reg = <0x0 0xa0080000 0x0 0x10000>;
+
+		mbigen_usb: intc_usb {
+			msi-parent = <&p0_its_dsa_a 0x40080>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			num-pins = <2>;
+		};
+	};
+
+	soc {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		uart0: uart@602b0000 {
+			compatible = "arm,sbsa-uart";
+			reg = <0x0 0x602b0000 0x0 0x1000>;
+			interrupt-parent = <&mbigen_uart>;
+			interrupts = <807 4>;
+			current-speed = <115200>;
+			reg-io-width = <4>;
+			status = "disabled";
+		};
+
+		usb_ohci: ohci@a7030000 {
+			compatible = "generic-ohci";
+			reg = <0x0 0xa7030000 0x0 0x10000>;
+			interrupt-parent = <&mbigen_usb>;
+			interrupts = <640 4>;
+			dma-coherent;
+			status = "disabled";
+		};
+
+		usb_ehci: ehci@a7020000 {
+			compatible = "generic-ehci";
+			reg = <0x0 0xa7020000 0x0 0x10000>;
+			interrupt-parent = <&mbigen_usb>;
+			interrupts = <641 4>;
+			dma-coherent;
+			status = "disabled";
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/marvell/Makefile b/arch/arm64/boot/dts/marvell/Makefile
index cf3953124cef..1690883b931a 100644
--- a/arch/arm64/boot/dts/marvell/Makefile
+++ b/arch/arm64/boot/dts/marvell/Makefile
@@ -4,6 +4,7 @@ dtb-$(CONFIG_ARCH_BERLIN) += berlin4ct-stb.dtb
 
 # Mvebu SoC Family
 dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-db.dtb
+dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-espressobin.dtb
 dtb-$(CONFIG_ARCH_MVEBU) += armada-7040-db.dtb
 dtb-$(CONFIG_ARCH_MVEBU) += armada-8040-db.dtb
 
diff --git a/arch/arm64/boot/dts/marvell/armada-3720-db.dts b/arch/arm64/boot/dts/marvell/armada-3720-db.dts
index a59d36cd6caf..89de0a751093 100644
--- a/arch/arm64/boot/dts/marvell/armada-3720-db.dts
+++ b/arch/arm64/boot/dts/marvell/armada-3720-db.dts
@@ -56,7 +56,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@0 {
 		device_type = "memory";
 		reg = <0x00000000 0x00000000 0x00000000 0x20000000>;
 	};
diff --git a/arch/arm64/boot/dts/marvell/armada-3720-espressobin.dts b/arch/arm64/boot/dts/marvell/armada-3720-espressobin.dts
new file mode 100644
index 000000000000..83178d909fc2
--- /dev/null
+++ b/arch/arm64/boot/dts/marvell/armada-3720-espressobin.dts
@@ -0,0 +1,82 @@
+/*
+ * Device Tree file for Globalscale Marvell ESPRESSOBin Board
+ * Copyright (C) 2016 Marvell
+ *
+ * Romain Perier <romain.perier@free-electrons.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+#include "armada-372x.dtsi"
+
+/ {
+	model = "Globalscale Marvell ESPRESSOBin Board";
+	compatible = "globalscale,espressobin", "marvell,armada3720", "marvell,armada3710";
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@0 {
+		device_type = "memory";
+		reg = <0x00000000 0x00000000 0x00000000 0x20000000>;
+	};
+};
+
+/* J9 */
+&pcie0 {
+	status = "okay";
+};
+
+/* J6 */
+&sata {
+	status = "okay";
+};
+
+/* Exported on the micro USB connector J5 through an FTDI */
+&uart0 {
+	status = "okay";
+};
+
+/* J7 */
+&usb3 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
index 3b8eb45bdc76..bab5c6ff5745 100644
--- a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
@@ -91,7 +91,7 @@
 		#size-cells = <2>;
 		ranges;
 
-		internal-regs {
+		internal-regs@d0000000 {
 			#address-cells = <1>;
 			#size-cells = <1>;
 			compatible = "simple-bus";
diff --git a/arch/arm64/boot/dts/marvell/armada-ap806.dtsi b/arch/arm64/boot/dts/marvell/armada-ap806.dtsi
index 7b6136182ad0..a749ba2edec4 100644
--- a/arch/arm64/boot/dts/marvell/armada-ap806.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-ap806.dtsi
@@ -71,7 +71,7 @@
 		interrupt-parent = <&gic>;
 		ranges;
 
-		config-space {
+		config-space@f0000000 {
 			#address-cells = <1>;
 			#size-cells = <1>;
 			compatible = "simple-bus";
diff --git a/arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi b/arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi
index 602e2c2e9a4d..05222f749a45 100644
--- a/arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi
@@ -52,7 +52,7 @@
 		interrupt-parent = <&gic>;
 		ranges;
 
-		config-space {
+		config-space@f2000000 {
 			#address-cells = <1>;
 			#size-cells = <1>;
 			compatible = "simple-bus";
@@ -164,6 +164,14 @@
 				clocks = <&cpm_syscon0 1 21>;
 				status = "disabled";
 			};
+
+			cpm_trng: trng@760000 {
+				compatible = "marvell,armada-8k-rng", "inside-secure,safexcel-eip76";
+				reg = <0x760000 0x7d>;
+				interrupts = <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&cpm_syscon0 1 25>;
+				status = "okay";
+			};
 		};
 
 		cpm_pcie0: pcie@f2600000 {
diff --git a/arch/arm64/boot/dts/marvell/armada-cp110-slave.dtsi b/arch/arm64/boot/dts/marvell/armada-cp110-slave.dtsi
index 6bf9e241179b..638820ce977d 100644
--- a/arch/arm64/boot/dts/marvell/armada-cp110-slave.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-cp110-slave.dtsi
@@ -52,7 +52,7 @@
 		interrupt-parent = <&gic>;
 		ranges;
 
-		config-space {
+		config-space@f4000000 {
 			#address-cells = <1>;
 			#size-cells = <1>;
 			compatible = "simple-bus";
@@ -164,6 +164,14 @@
 				clocks = <&cps_syscon0 1 21>;
 				status = "disabled";
 			};
+
+			cps_trng: trng@760000 {
+				compatible = "marvell,armada-8k-rng", "inside-secure,safexcel-eip76";
+				reg = <0x760000 0x7d>;
+				interrupts = <GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&cps_syscon0 1 25>;
+				status = "okay";
+			};
 		};
 
 		cps_pcie0: pcie@f4600000 {
diff --git a/arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts b/arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts
index 0d70d39fa8d2..fae6c6924705 100644
--- a/arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts
+++ b/arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts
@@ -54,7 +54,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@1000000 {
 		device_type = "memory";
 		/* the first 16MB is for firmwares' usage */
 		reg = <0 0x01000000 0 0x7f000000>;
diff --git a/arch/arm64/boot/dts/marvell/berlin4ct-stb.dts b/arch/arm64/boot/dts/marvell/berlin4ct-stb.dts
index 348c37ecf069..d47edad13e68 100644
--- a/arch/arm64/boot/dts/marvell/berlin4ct-stb.dts
+++ b/arch/arm64/boot/dts/marvell/berlin4ct-stb.dts
@@ -54,7 +54,7 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	memory {
+	memory@1000000 {
 		device_type = "memory";
 		/* the first 16MB is for firmwares' usage */
 		reg = <0 0x01000000 0 0x7f000000>;
diff --git a/arch/arm64/boot/dts/marvell/berlin4ct.dtsi b/arch/arm64/boot/dts/marvell/berlin4ct.dtsi
index 85c23facb9fe..d6b800fd26d0 100644
--- a/arch/arm64/boot/dts/marvell/berlin4ct.dtsi
+++ b/arch/arm64/boot/dts/marvell/berlin4ct.dtsi
@@ -142,7 +142,7 @@
 			     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
 	};
 
-	soc {
+	soc@f7000000 {
 		compatible = "simple-bus";
 		#address-cells = <1>;
 		#size-cells = <1>;
diff --git a/arch/arm64/boot/dts/mediatek/mt8173-evb.dts b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts
index 2a7f731c7759..0ecaad4333a7 100644
--- a/arch/arm64/boot/dts/mediatek/mt8173-evb.dts
+++ b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts
@@ -34,15 +34,6 @@
 
 	chosen { };
 
-	usb_p1_vbus: regulator@0 {
-		compatible = "regulator-fixed";
-		regulator-name = "usb_vbus";
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		gpio = <&pio 130 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-	};
-
 	connector {
 		compatible = "hdmi-connector";
 		label = "hdmi";
@@ -54,6 +45,29 @@
 			};
 		};
 	};
+
+	extcon_usb: extcon_iddig {
+		compatible = "linux,extcon-usb-gpio";
+		id-gpio = <&pio 16 GPIO_ACTIVE_HIGH>;
+	};
+
+	usb_p1_vbus: regulator@0 {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&pio 130 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	usb_p0_vbus: regulator@1 {
+		compatible = "regulator-fixed";
+		regulator-name = "vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&pio 9 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
 };
 
 &cec {
@@ -243,6 +257,20 @@
 			bias-pull-down = <MTK_PUPD_SET_R1R0_10>;
 		};
 	};
+
+	usb_id_pins_float: usb_iddig_pull_up {
+		pins_iddig {
+			pinmux = <MT8173_PIN_16_IDDIG__FUNC_IDDIG>;
+			bias-pull-up;
+		};
+	};
+
+	usb_id_pins_ground: usb_iddig_pull_down {
+		pins_iddig {
+			pinmux = <MT8173_PIN_16_IDDIG__FUNC_IDDIG>;
+			bias-pull-down;
+		};
+	};
 };
 
 &pwm0 {
@@ -469,12 +497,25 @@
 	status = "okay";
 };
 
+&ssusb {
+	vusb33-supply = <&mt6397_vusb_reg>;
+	vbus-supply = <&usb_p0_vbus>;
+	extcon = <&extcon_usb>;
+	dr_mode = "otg";
+	mediatek,enable-wakeup;
+	pinctrl-names = "default", "id_float", "id_ground";
+	pinctrl-0 = <&usb_id_pins_float>;
+	pinctrl-1 = <&usb_id_pins_float>;
+	pinctrl-2 = <&usb_id_pins_ground>;
+	status = "okay";
+};
+
 &uart0 {
 	status = "okay";
 };
 
-&usb30 {
+&usb_host {
 	vusb33-supply = <&mt6397_vusb_reg>;
 	vbus-supply = <&usb_p1_vbus>;
-	mediatek,wakeup-src = <1>;
+	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi
index 6689abf99367..12e702771f5c 100644
--- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi
@@ -458,6 +458,9 @@
 		auxadc: auxadc@11001000 {
 			compatible = "mediatek,mt8173-auxadc";
 			reg = <0 0x11001000 0 0x1000>;
+			clocks = <&pericfg CLK_PERI_AUXADC>;
+			clock-names = "main";
+			#io-channel-cells = <1>;
 		};
 
 		uart0: serial@11002000 {
@@ -715,11 +718,14 @@
 			status = "disabled";
 		};
 
-		usb30: usb@11270000 {
-			compatible = "mediatek,mt8173-xhci";
-			reg = <0 0x11270000 0 0x1000>,
+		ssusb: usb@11271000 {
+			compatible = "mediatek,mt8173-mtu3";
+			reg = <0 0x11271000 0 0x3000>,
 			      <0 0x11280700 0 0x0100>;
-			interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_LOW>;
+			reg-names = "mac", "ippc";
+			interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_LOW>;
+			phys = <&phy_port0 PHY_TYPE_USB3>,
+			       <&phy_port1 PHY_TYPE_USB2>;
 			power-domains = <&scpsys MT8173_POWER_DOMAIN_USB>;
 			clocks = <&topckgen CLK_TOP_USB30_SEL>,
 				 <&pericfg CLK_PERI_USB0>,
@@ -727,10 +733,22 @@
 			clock-names = "sys_ck",
 				      "wakeup_deb_p0",
 				      "wakeup_deb_p1";
-			phys = <&phy_port0 PHY_TYPE_USB3>,
-			       <&phy_port1 PHY_TYPE_USB2>;
 			mediatek,syscon-wakeup = <&pericfg>;
-			status = "okay";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+			status = "disabled";
+
+			usb_host: xhci@11270000 {
+				compatible = "mediatek,mt8173-xhci";
+				reg = <0 0x11270000 0 0x1000>;
+				reg-names = "mac";
+				interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_LOW>;
+				power-domains = <&scpsys MT8173_POWER_DOMAIN_USB>;
+				clocks = <&topckgen CLK_TOP_USB30_SEL>;
+				clock-names = "sys_ck";
+				status = "disabled";
+			};
 		};
 
 		u3phy: usb-phy@11290000 {
diff --git a/arch/arm64/boot/dts/nvidia/Makefile b/arch/arm64/boot/dts/nvidia/Makefile
index 0f7cdf3e05c1..18941458cb4d 100644
--- a/arch/arm64/boot/dts/nvidia/Makefile
+++ b/arch/arm64/boot/dts/nvidia/Makefile
@@ -3,6 +3,7 @@ dtb-$(CONFIG_ARCH_TEGRA_210_SOC) += tegra210-p2371-0000.dtb
 dtb-$(CONFIG_ARCH_TEGRA_210_SOC) += tegra210-p2371-2180.dtb
 dtb-$(CONFIG_ARCH_TEGRA_210_SOC) += tegra210-p2571.dtb
 dtb-$(CONFIG_ARCH_TEGRA_210_SOC) += tegra210-smaug.dtb
+dtb-$(CONFIG_ARCH_TEGRA_186_SOC) += tegra186-p2771-0000.dtb
 
 always		:= $(dtb-y)
 clean-files	:= *.dtb
diff --git a/arch/arm64/boot/dts/nvidia/tegra186-p2771-0000.dts b/arch/arm64/boot/dts/nvidia/tegra186-p2771-0000.dts
new file mode 100644
index 000000000000..0d3c0996d832
--- /dev/null
+++ b/arch/arm64/boot/dts/nvidia/tegra186-p2771-0000.dts
@@ -0,0 +1,8 @@
+/dts-v1/;
+
+#include "tegra186-p3310.dtsi"
+
+/ {
+	model = "NVIDIA Tegra186 P2771-0000 Development Board";
+	compatible = "nvidia,p2771-0000", "nvidia,tegra186";
+};
diff --git a/arch/arm64/boot/dts/nvidia/tegra186-p3310.dtsi b/arch/arm64/boot/dts/nvidia/tegra186-p3310.dtsi
new file mode 100644
index 000000000000..1abe2eceb3d1
--- /dev/null
+++ b/arch/arm64/boot/dts/nvidia/tegra186-p3310.dtsi
@@ -0,0 +1,64 @@
+#include "tegra186.dtsi"
+
+/ {
+	model = "NVIDIA Tegra186 P3310 Processor Module";
+	compatible = "nvidia,p3310", "nvidia,tegra186";
+
+	aliases {
+		serial0 = &uarta;
+	};
+
+	chosen {
+		bootargs = "earlycon console=ttyS0,115200n8";
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x2 0x00000000>;
+	};
+
+	serial@3100000 {
+		status = "okay";
+	};
+
+	hsp@3c00000 {
+		status = "okay";
+	};
+
+	cpus {
+		cpu@0 {
+			enable-method = "psci";
+		};
+
+		cpu@1 {
+			enable-method = "psci";
+		};
+
+		cpu@2 {
+			enable-method = "psci";
+		};
+
+		cpu@3 {
+			enable-method = "psci";
+		};
+
+		cpu@4 {
+			enable-method = "psci";
+		};
+
+		cpu@5 {
+			enable-method = "psci";
+		};
+	};
+
+	bpmp {
+		status = "okay";
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		status = "okay";
+		method = "smc";
+	};
+};
diff --git a/arch/arm64/boot/dts/nvidia/tegra186.dtsi b/arch/arm64/boot/dts/nvidia/tegra186.dtsi
new file mode 100644
index 000000000000..a918e10240fd
--- /dev/null
+++ b/arch/arm64/boot/dts/nvidia/tegra186.dtsi
@@ -0,0 +1,398 @@
+#include <dt-bindings/gpio/tegra186-gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	compatible = "nvidia,tegra186";
+	interrupt-parent = <&gic>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	gpio: gpio@2200000 {
+		compatible = "nvidia,tegra186-gpio";
+		reg-names = "security", "gpio";
+		reg = <0x0 0x2200000 0x0 0x10000>,
+		      <0x0 0x2210000 0x0 0x10000>;
+		interrupts = <GIC_SPI  47 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI  50 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI  53 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI  56 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI  59 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		#gpio-cells = <2>;
+		gpio-controller;
+	};
+
+	uarta: serial@3100000 {
+		compatible = "nvidia,tegra186-uart", "nvidia,tegra20-uart";
+		reg = <0x0 0x03100000 0x0 0x40>;
+		reg-shift = <2>;
+		interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&bpmp 55>;
+		clock-names = "serial";
+		resets = <&bpmp 47>;
+		reset-names = "serial";
+		status = "disabled";
+	};
+
+	uartb: serial@3110000 {
+		compatible = "nvidia,tegra186-uart", "nvidia,tegra20-uart";
+		reg = <0x0 0x03110000 0x0 0x40>;
+		reg-shift = <2>;
+		interrupts = <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&bpmp 56>;
+		clock-names = "serial";
+		resets = <&bpmp 48>;
+		reset-names = "serial";
+		status = "disabled";
+	};
+
+	uartd: serial@3130000 {
+		compatible = "nvidia,tegra186-uart", "nvidia,tegra20-uart";
+		reg = <0x0 0x03130000 0x0 0x40>;
+		reg-shift = <2>;
+		interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&bpmp 77>;
+		clock-names = "serial";
+		resets = <&bpmp 50>;
+		reset-names = "serial";
+		status = "disabled";
+	};
+
+	uarte: serial@3140000 {
+		compatible = "nvidia,tegra186-uart", "nvidia,tegra20-uart";
+		reg = <0x0 0x03140000 0x0 0x40>;
+		reg-shift = <2>;
+		interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&bpmp 194>;
+		clock-names = "serial";
+		resets = <&bpmp 132>;
+		reset-names = "serial";
+		status = "disabled";
+	};
+
+	uartf: serial@3150000 {
+		compatible = "nvidia,tegra186-uart", "nvidia,tegra20-uart";
+		reg = <0x0 0x03150000 0x0 0x40>;
+		reg-shift = <2>;
+		interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&bpmp 195>;
+		clock-names = "serial";
+		resets = <&bpmp 111>;
+		reset-names = "serial";
+		status = "disabled";
+	};
+
+	gen1_i2c: i2c@3160000 {
+		compatible = "nvidia,tegra186-i2c", "nvidia,tegra114-i2c";
+		reg = <0x0 0x03160000 0x0 0x10000>;
+		interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&bpmp 47>;
+		clock-names = "div-clk";
+		resets = <&bpmp 19>;
+		reset-names = "i2c";
+		status = "disabled";
+	};
+
+	cam_i2c: i2c@3180000 {
+		compatible = "nvidia,tegra186-i2c", "nvidia,tegra114-i2c";
+		reg = <0x0 0x03180000 0x0 0x10000>;
+		interrupts = <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&bpmp 75>;
+		clock-names = "div-clk";
+		resets = <&bpmp 21>;
+		reset-names = "i2c";
+		status = "disabled";
+	};
+
+	/* shares pads with dpaux1 */
+	dp_aux_ch1_i2c: i2c@3190000 {
+		compatible = "nvidia,tegra186-i2c", "nvidia,tegra114-i2c";
+		reg = <0x0 0x03190000 0x0 0x10000>;
+		interrupts = <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&bpmp 86>;
+		clock-names = "div-clk";
+		resets = <&bpmp 22>;
+		reset-names = "i2c";
+		status = "disabled";
+	};
+
+	/* controlled by BPMP, should not be enabled */
+	pwr_i2c: i2c@31a0000 {
+		compatible = "nvidia,tegra186-i2c", "nvidia,tegra114-i2c";
+		reg = <0x0 0x031a0000 0x0 0x10000>;
+		interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&bpmp 48>;
+		clock-names = "div-clk";
+		resets = <&bpmp 23>;
+		reset-names = "i2c";
+		status = "disabled";
+	};
+
+	/* shares pads with dpaux0 */
+	dp_aux_ch0_i2c: i2c@31b0000 {
+		compatible = "nvidia,tegra186-i2c", "nvidia,tegra114-i2c";
+		reg = <0x0 0x031b0000 0x0 0x10000>;
+		interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&bpmp 125>;
+		clock-names = "div-clk";
+		resets = <&bpmp 24>;
+		reset-names = "i2c";
+		status = "disabled";
+	};
+
+	gen7_i2c: i2c@31c0000 {
+		compatible = "nvidia,tegra186-i2c", "nvidia,tegra114-i2c";
+		reg = <0x0 0x031c0000 0x0 0x10000>;
+		interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&bpmp 182>;
+		clock-names = "div-clk";
+		resets = <&bpmp 81>;
+		reset-names = "i2c";
+		status = "disabled";
+	};
+
+	gen9_i2c: i2c@31e0000 {
+		compatible = "nvidia,tegra186-i2c", "nvidia,tegra114-i2c";
+		reg = <0x0 0x031e0000 0x0 0x10000>;
+		interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&bpmp 183>;
+		clock-names = "div-clk";
+		resets = <&bpmp 83>;
+		reset-names = "i2c";
+		status = "disabled";
+	};
+
+	sdmmc1: sdhci@3400000 {
+		compatible = "nvidia,tegra186-sdhci";
+		reg = <0x0 0x03400000 0x0 0x10000>;
+		interrupts = <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&bpmp 52>;
+		clock-names = "sdhci";
+		resets = <&bpmp 33>;
+		reset-names = "sdhci";
+		status = "disabled";
+	};
+
+	sdmmc2: sdhci@3420000 {
+		compatible = "nvidia,tegra186-sdhci";
+		reg = <0x0 0x03420000 0x0 0x10000>;
+		interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&bpmp 53>;
+		clock-names = "sdhci";
+		resets = <&bpmp 34>;
+		reset-names = "sdhci";
+		status = "disabled";
+	};
+
+	sdmmc3: sdhci@3440000 {
+		compatible = "nvidia,tegra186-sdhci";
+		reg = <0x0 0x03440000 0x0 0x10000>;
+		interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&bpmp 76>;
+		clock-names = "sdhci";
+		resets = <&bpmp 35>;
+		reset-names = "sdhci";
+		status = "disabled";
+	};
+
+	sdmmc4: sdhci@3460000 {
+		compatible = "nvidia,tegra186-sdhci";
+		reg = <0x0 0x03460000 0x0 0x10000>;
+		interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&bpmp 54>;
+		clock-names = "sdhci";
+		resets = <&bpmp 36>;
+		reset-names = "sdhci";
+		status = "disabled";
+	};
+
+	gic: interrupt-controller@3881000 {
+		compatible = "arm,gic-400";
+		#interrupt-cells = <3>;
+		interrupt-controller;
+		reg = <0x0 0x03881000 0x0 0x1000>,
+		      <0x0 0x03882000 0x0 0x2000>;
+		interrupts = <GIC_PPI 9
+			(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+		interrupt-parent = <&gic>;
+	};
+
+	hsp_top0: hsp@3c00000 {
+		compatible = "nvidia,tegra186-hsp";
+		reg = <0x0 0x03c00000 0x0 0xa0000>;
+		interrupts = <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "doorbell";
+		#mbox-cells = <2>;
+		status = "disabled";
+	};
+
+	gen2_i2c: i2c@c240000 {
+		compatible = "nvidia,tegra186-i2c", "nvidia,tegra114-i2c";
+		reg = <0x0 0x0c240000 0x0 0x10000>;
+		interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&bpmp 218>;
+		clock-names = "div-clk";
+		resets = <&bpmp 20>;
+		reset-names = "i2c";
+		status = "disabled";
+	};
+
+	gen8_i2c: i2c@c250000 {
+		compatible = "nvidia,tegra186-i2c", "nvidia,tegra114-i2c";
+		reg = <0x0 0x0c250000 0x0 0x10000>;
+		interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&bpmp 219>;
+		clock-names = "div-clk";
+		resets = <&bpmp 82>;
+		reset-names = "i2c";
+		status = "disabled";
+	};
+
+	uartc: serial@c280000 {
+		compatible = "nvidia,tegra186-uart", "nvidia,tegra20-uart";
+		reg = <0x0 0x0c280000 0x0 0x40>;
+		reg-shift = <2>;
+		interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&bpmp 215>;
+		clock-names = "serial";
+		resets = <&bpmp 49>;
+		reset-names = "serial";
+		status = "disabled";
+	};
+
+	uartg: serial@c290000 {
+		compatible = "nvidia,tegra186-uart", "nvidia,tegra20-uart";
+		reg = <0x0 0x0c290000 0x0 0x40>;
+		reg-shift = <2>;
+		interrupts = <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&bpmp 216>;
+		clock-names = "serial";
+		resets = <&bpmp 112>;
+		reset-names = "serial";
+		status = "disabled";
+	};
+
+	gpio_aon: gpio@c2f0000 {
+		compatible = "nvidia,tegra186-gpio-aon";
+		reg-names = "security", "gpio";
+		reg = <0x0 0xc2f0000 0x0 0x1000>,
+		      <0x0 0xc2f1000 0x0 0x1000>;
+		interrupts = <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	sysram@30000000 {
+		compatible = "nvidia,tegra186-sysram", "mmio-sram";
+		reg = <0x0 0x30000000 0x0 0x50000>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges = <0 0x0 0x0 0x30000000 0x0 0x50000>;
+
+		cpu_bpmp_tx: shmem@4e000 {
+			compatible = "nvidia,tegra186-bpmp-shmem";
+			reg = <0x0 0x4e000 0x0 0x1000>;
+			label = "cpu-bpmp-tx";
+			pool;
+		};
+
+		cpu_bpmp_rx: shmem@4f000 {
+			compatible = "nvidia,tegra186-bpmp-shmem";
+			reg = <0x0 0x4f000 0x0 0x1000>;
+			label = "cpu-bpmp-rx";
+			pool;
+		};
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu@0 {
+			compatible = "nvidia,tegra186-denver", "arm,armv8";
+			device_type = "cpu";
+			reg = <0x000>;
+		};
+
+		cpu@1 {
+			compatible = "nvidia,tegra186-denver", "arm,armv8";
+			device_type = "cpu";
+			reg = <0x001>;
+		};
+
+		cpu@2 {
+			compatible = "arm,cortex-a57", "arm,armv8";
+			device_type = "cpu";
+			reg = <0x100>;
+		};
+
+		cpu@3 {
+			compatible = "arm,cortex-a57", "arm,armv8";
+			device_type = "cpu";
+			reg = <0x101>;
+		};
+
+		cpu@4 {
+			compatible = "arm,cortex-a57", "arm,armv8";
+			device_type = "cpu";
+			reg = <0x102>;
+		};
+
+		cpu@5 {
+			compatible = "arm,cortex-a57", "arm,armv8";
+			device_type = "cpu";
+			reg = <0x103>;
+		};
+	};
+
+	bpmp: bpmp {
+		compatible = "nvidia,tegra186-bpmp";
+		mboxes = <&hsp_top0 0 19>;
+		shmem = <&cpu_bpmp_tx &cpu_bpmp_rx>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+
+		bpmp_i2c: i2c {
+			compatible = "nvidia,tegra186-bpmp-i2c";
+			nvidia,bpmp-bus-id = <5>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13
+				(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 14
+				(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 11
+				(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 10
+				(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
+		interrupt-parent = <&gic>;
+	};
+};
diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi b/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi
index 5fda583351d7..906fb836d241 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi
@@ -21,6 +21,10 @@
 		reg = <0x0 0x80000000 0x1 0x0>;
 	};
 
+	gpu@57000000 {
+		vdd-supply = <&vdd_gpu>;
+	};
+
 	/* debug port */
 	serial@70006000 {
 		status = "okay";
@@ -291,4 +295,18 @@
 			clock-frequency = <32768>;
 		};
 	};
+
+	regulators {
+		vdd_gpu: regulator@100 {
+			compatible = "pwm-regulator";
+			reg = <100>;
+			pwms = <&pwm 1 4880>;
+			regulator-name = "VDD_GPU";
+			regulator-min-microvolt = <710000>;
+			regulator-max-microvolt = <1320000>;
+			enable-gpios = <&pmic 6 GPIO_ACTIVE_HIGH>;
+			regulator-ramp-delay = <80>;
+			regulator-enable-ramp-delay = <1000>;
+		};
+	};
 };
diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
index 983775e637a4..4c1ea7a08d43 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
+++ b/arch/arm64/boot/dts/nvidia/tegra210-p2371-2180.dts
@@ -7,6 +7,32 @@
 	model = "NVIDIA Jetson TX1 Developer Kit";
 	compatible = "nvidia,p2371-2180", "nvidia,tegra210";
 
+	pcie-controller@01003000 {
+		status = "okay";
+
+		avdd-pll-uerefe-supply = <&avdd_1v05_pll>;
+		hvddio-pex-supply = <&vdd_1v8>;
+		dvddio-pex-supply = <&vdd_pex_1v05>;
+		dvdd-pex-pll-supply = <&vdd_pex_1v05>;
+		hvdd-pex-pll-e-supply = <&vdd_1v8>;
+		vddio-pex-ctl-supply = <&vdd_1v8>;
+
+		pci@1,0 {
+			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-0}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-1}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-2}>,
+			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-3}>;
+			phy-names = "pcie-0", "pcie-1", "pcie-2", "pcie-3";
+			status = "okay";
+		};
+
+		pci@2,0 {
+			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-4}>;
+			phy-names = "pcie-0";
+			status = "okay";
+		};
+	};
+
 	host1x@50000000 {
 		dsi@54300000 {
 			status = "okay";
diff --git a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts
index c2becb603e11..7703227f5d1a 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts
+++ b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts
@@ -11,7 +11,8 @@
 	compatible = "google,smaug-rev8", "google,smaug-rev7",
 		     "google,smaug-rev6", "google,smaug-rev5",
 		     "google,smaug-rev4", "google,smaug-rev3",
-		     "google,smaug-rev1", "google,smaug", "nvidia,tegra210";
+		     "google,smaug-rev2", "google,smaug-rev1",
+		     "google,smaug", "nvidia,tegra210";
 
 	aliases {
 		serial0 = &uarta;
diff --git a/arch/arm64/boot/dts/nvidia/tegra210.dtsi b/arch/arm64/boot/dts/nvidia/tegra210.dtsi
index 46045fe719da..2f832df29da8 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra210.dtsi
@@ -11,6 +11,69 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	pcie-controller@01003000 {
+		compatible = "nvidia,tegra210-pcie";
+		device_type = "pci";
+		reg = <0x0 0x01003000 0x0 0x00000800   /* PADS registers */
+		       0x0 0x01003800 0x0 0x00000800   /* AFI registers */
+		       0x0 0x02000000 0x0 0x10000000>; /* configuration space */
+		reg-names = "pads", "afi", "cs";
+		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
+
+		bus-range = <0x00 0xff>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		ranges = <0x82000000 0 0x01000000 0x0 0x01000000 0 0x00001000   /* port 0 configuration space */
+			  0x82000000 0 0x01001000 0x0 0x01001000 0 0x00001000   /* port 1 configuration space */
+			  0x81000000 0 0x0        0x0 0x12000000 0 0x00010000   /* downstream I/O (64 KiB) */
+			  0x82000000 0 0x13000000 0x0 0x13000000 0 0x0d000000   /* non-prefetchable memory (208 MiB) */
+			  0xc2000000 0 0x20000000 0x0 0x20000000 0 0x20000000>; /* prefetchable memory (512 MiB) */
+
+		clocks = <&tegra_car TEGRA210_CLK_PCIE>,
+			 <&tegra_car TEGRA210_CLK_AFI>,
+			 <&tegra_car TEGRA210_CLK_PLL_E>,
+			 <&tegra_car TEGRA210_CLK_CML0>;
+		clock-names = "pex", "afi", "pll_e", "cml";
+		resets = <&tegra_car 70>,
+			 <&tegra_car 72>,
+			 <&tegra_car 74>;
+		reset-names = "pex", "afi", "pcie_x";
+		status = "disabled";
+
+		pci@1,0 {
+			device_type = "pci";
+			assigned-addresses = <0x82000800 0 0x01000000 0 0x1000>;
+			reg = <0x000800 0 0 0 0>;
+			status = "disabled";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			nvidia,num-lanes = <4>;
+		};
+
+		pci@2,0 {
+			device_type = "pci";
+			assigned-addresses = <0x82001000 0 0x01001000 0 0x1000>;
+			reg = <0x001000 0 0 0 0>;
+			status = "disabled";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			nvidia,num-lanes = <1>;
+		};
+	};
+
 	host1x@50000000 {
 		compatible = "nvidia,tegra210-host1x", "simple-bus";
 		reg = <0x0 0x50000000 0x0 0x00034000>;
diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile
index 5dd05de5619b..cc0f02d9dd02 100644
--- a/arch/arm64/boot/dts/qcom/Makefile
+++ b/arch/arm64/boot/dts/qcom/Makefile
@@ -1,6 +1,9 @@
-dtb-$(CONFIG_ARCH_QCOM)	+= apq8016-sbc.dtb msm8916-mtp.dtb
-dtb-$(CONFIG_ARCH_QCOM)	+= msm8996-mtp.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= apq8016-sbc.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= apq8096-db820c.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-mtp.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= msm8992-bullhead-rev-101.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= msm8994-angler-rev-101.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= msm8996-mtp.dtb
 
 always		:= $(dtb-y)
 subdir-y	:= $(dts-dirs)
diff --git a/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi b/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi
index bb062b547110..08bd5ebafb4e 100644
--- a/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi
+++ b/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi
@@ -15,6 +15,7 @@
 #include "pm8916.dtsi"
 #include "apq8016-sbc-soc-pins.dtsi"
 #include "apq8016-sbc-pmic-pins.dtsi"
+#include <dt-bindings/sound/apq8016-lpass.h>
 
 / {
 	aliases {
@@ -251,6 +252,60 @@
 				vddio-supply = <&pm8916_l6>;
 			};
 		};
+
+		lpass_codec: codec{
+			status = "okay";
+		};
+
+		/*
+		Internal Codec
+			playback - Primary MI2S
+			capture - Ter MI2S
+
+		External Primary:
+			playback - secondary MI2S
+			capture - Quat MI2S
+
+		External Secondary:
+			playback - Quat MI2S
+			capture - Quat MI2S
+
+		*/
+
+                sound: sound {
+                        compatible = "qcom,apq8016-sbc-sndcard";
+                        reg = <0x07702000 0x4>, <0x07702004 0x4>;
+                        reg-names = "mic-iomux", "spkr-iomux";
+
+                        status = "okay";
+                        pinctrl-0 = <&cdc_pdm_lines_act &ext_sec_tlmm_lines_act &ext_mclk_tlmm_lines_act>;
+                        pinctrl-1 = <&cdc_pdm_lines_sus &ext_sec_tlmm_lines_sus &ext_mclk_tlmm_lines_sus>;
+                        pinctrl-names = "default", "sleep";
+                        qcom,model = "DB410c";
+                        qcom,audio-routing =
+                                "AMIC2", "MIC BIAS Internal2",
+                                "AMIC3", "MIC BIAS External1";
+
+                        internal-codec-playback-dai-link@0 {            /* I2S - Internal codec */
+                                link-name = "WCD";
+                                cpu { /* PRIMARY */
+                                        sound-dai = <&lpass MI2S_PRIMARY>;
+                                };
+                                codec {
+                                        sound-dai = <&lpass_codec 0>, <&wcd_codec 0>;
+                                };
+                        };
+
+                        internal-codec-capture-dai-link@0 {             /* I2S - Internal codec */
+                                link-name = "WCD-Capture";
+                                cpu { /* PRIMARY */
+                                        sound-dai = <&lpass MI2S_TERTIARY>;
+                                };
+                                codec {
+                                        sound-dai = <&lpass_codec 1>, <&wcd_codec 1>;
+                                };
+                        };
+                };
 	};
 
 	usb2513 {
@@ -278,6 +333,12 @@
 	};
 };
 
+&wcd_codec {
+        status = "okay";
+        clocks = <&gcc GCC_CODEC_DIGCODEC_CLK>;
+        clock-names = "mclk";
+};
+
 &smd_rpm_regulators {
 	vdd_l1_l2_l3-supply = <&pm8916_s3>;
 	vdd_l5-supply = <&pm8916_s3>;
@@ -308,8 +369,8 @@
 	};
 
 	l2 {
-		regulator-min-microvolt = <375000>;
-		regulator-max-microvolt = <1525000>;
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
 	};
 
 	l3 {
@@ -328,8 +389,8 @@
 	};
 
 	l6 {
-		regulator-min-microvolt = <1750000>;
-		regulator-max-microvolt = <3337000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
 	};
 
 	l7 {
@@ -388,8 +449,8 @@
 	};
 
 	l17 {
-		regulator-min-microvolt = <1750000>;
-		regulator-max-microvolt = <3337000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
 	};
 
 	l18 {
diff --git a/arch/arm64/boot/dts/qcom/apq8096-db820c-pmic-pins.dtsi b/arch/arm64/boot/dts/qcom/apq8096-db820c-pmic-pins.dtsi
new file mode 100644
index 000000000000..0de95171d6d0
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/apq8096-db820c-pmic-pins.dtsi
@@ -0,0 +1,15 @@
+
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+&pm8994_gpios {
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&ls_exp_gpio_f>;
+
+	ls_exp_gpio_f: pm8916_mpp4 {
+		pinconf {
+			pins = "gpio5";
+			output-low;
+			power-source = <2>; // PM8994_GPIO_S4, 1.8V
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi b/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
index afb218cffc60..422959b87d12 100644
--- a/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
+++ b/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
@@ -12,7 +12,9 @@
  */
 
 #include "msm8996.dtsi"
+#include "pm8994.dtsi"
 #include "apq8096-db820c-pins.dtsi"
+#include "apq8096-db820c-pmic-pins.dtsi"
 
 / {
 	aliases {
diff --git a/arch/arm64/boot/dts/qcom/msm8916.dtsi b/arch/arm64/boot/dts/qcom/msm8916.dtsi
index 466ca5705c99..f8ff327667c5 100644
--- a/arch/arm64/boot/dts/qcom/msm8916.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916.dtsi
@@ -77,7 +77,7 @@
 			no-map;
 		};
 
-		mpss@86800000 {
+		mpss_mem: mpss@86800000 {
 			reg = <0x0 0x86800000 0x0 0x2b00000>;
 			no-map;
 		};
@@ -504,6 +504,15 @@
 			reg-names = "lpass-lpaif";
 		};
 
+                lpass_codec: codec{
+			compatible = "qcom,msm8916-wcd-digital-codec";
+			reg = <0x0771c000 0x400>;
+			clocks = <&gcc GCC_ULTAUDIO_AHBFABRIC_IXFABRIC_CLK>,
+				 <&gcc GCC_CODEC_DIGCODEC_CLK>;
+			clock-names = "ahbix-clk", "mclk";
+			#sound-dai-cells = <1>;
+                };
+
 		sdhc_1: sdhci@07824000 {
 			compatible = "qcom,sdhci-msm-v4";
 			reg = <0x07824900 0x11c>, <0x07824000 0x800>;
@@ -512,8 +521,10 @@
 			interrupts = <0 123 0>, <0 138 0>;
 			interrupt-names = "hc_irq", "pwr_irq";
 			clocks = <&gcc GCC_SDCC1_APPS_CLK>,
-				 <&gcc GCC_SDCC1_AHB_CLK>;
-			clock-names = "core", "iface";
+				 <&gcc GCC_SDCC1_AHB_CLK>,
+				 <&xo_board>;
+			clock-names = "core", "iface", "xo";
+			mmc-ddr-1_8v;
 			bus-width = <8>;
 			non-removable;
 			status = "disabled";
@@ -527,8 +538,9 @@
 			interrupts = <0 125 0>, <0 221 0>;
 			interrupt-names = "hc_irq", "pwr_irq";
 			clocks = <&gcc GCC_SDCC2_APPS_CLK>,
-				 <&gcc GCC_SDCC2_AHB_CLK>;
-			clock-names = "core", "iface";
+				 <&gcc GCC_SDCC2_AHB_CLK>,
+				 <&xo_board>;
+			clock-names = "core", "iface", "xo";
 			bus-width = <4>;
 			status = "disabled";
 		};
@@ -801,6 +813,49 @@
 				clock-names = "iface_clk";
 			};
 		};
+
+
+		hexagon@4080000 {
+			compatible = "qcom,q6v5-pil";
+			reg = <0x04080000 0x100>,
+			      <0x04020000 0x040>;
+
+			reg-names = "qdsp6", "rmb";
+
+			interrupts-extended = <&intc 0 24 1>,
+					      <&hexagon_smp2p_in 0 0>,
+					      <&hexagon_smp2p_in 1 0>,
+					      <&hexagon_smp2p_in 2 0>,
+					      <&hexagon_smp2p_in 3 0>;
+			interrupt-names = "wdog", "fatal", "ready",
+					  "handover", "stop-ack";
+
+			clocks = <&gcc GCC_MSS_CFG_AHB_CLK>,
+				 <&gcc GCC_MSS_Q6_BIMC_AXI_CLK>,
+				 <&gcc GCC_BOOT_ROM_AHB_CLK>;
+			clock-names = "iface", "bus", "mem";
+
+			qcom,smem-states = <&hexagon_smp2p_out 0>;
+			qcom,smem-state-names = "stop";
+
+			resets = <&scm 0>;
+			reset-names = "mss_restart";
+
+			mx-supply = <&pm8916_l3>;
+			pll-supply = <&pm8916_l7>;
+
+			qcom,halt-regs = <&tcsr 0x18000 0x19000 0x1a000>;
+
+			status = "disabled";
+
+			mba {
+				memory-region = <&mba_mem>;
+			};
+
+			mpss {
+				memory-region = <&mpss_mem>;
+			};
+		};
 	};
 
 	smd {
@@ -848,6 +903,14 @@
 				};
 			};
 		};
+
+		hexagon {
+			interrupts = <0 25 IRQ_TYPE_EDGE_RISING>;
+
+			qcom,smd-edge = <0>;
+			qcom,ipc = <&apcs 8 12>;
+			qcom,remote-pid = <1>;
+		};
 	};
 
 	hexagon-smp2p {
diff --git a/arch/arm64/boot/dts/qcom/msm8992-bullhead-rev-101.dts b/arch/arm64/boot/dts/qcom/msm8992-bullhead-rev-101.dts
new file mode 100644
index 000000000000..454213391671
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8992-bullhead-rev-101.dts
@@ -0,0 +1,41 @@
+/* Copyright (c) 2015, LGE Inc. All rights reserved.
+ * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+/dts-v1/;
+
+#include "msm8992.dtsi"
+
+/ {
+	model = "LG Nexus 5X";
+	compatible = "lg,bullhead", "qcom,msm8992";
+	/* required for bootloader to select correct board */
+	qcom,board-id = <0xb64 0>;
+	qcom,pmic-id = <0x10009 0x1000A 0x0 0x0>;
+
+	aliases {
+		serial0 = &blsp1_uart2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	soc {
+		serial@f991e000 {
+			status = "okay";
+			pinctrl-names = "default", "sleep";
+			pinctrl-0 = <&blsp1_uart2_default>;
+			pinctrl-1 = <&blsp1_uart2_sleep>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/msm8992-pins.dtsi b/arch/arm64/boot/dts/qcom/msm8992-pins.dtsi
new file mode 100644
index 000000000000..d2a26f0f8d73
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8992-pins.dtsi
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+&msmgpio {
+	blsp1_uart2_default: blsp1_uart2_default {
+		pinmux {
+			function = "blsp_uart2";
+			pins = "gpio4", "gpio5";
+		};
+		pinconf {
+			pins = "gpio4", "gpio5";
+			drive-strength = <16>;
+			bias-disable;
+		};
+	};
+
+	blsp1_uart2_sleep: blsp1_uart2_sleep {
+		pinmux {
+			function = "gpio";
+			pins = "gpio4", "gpio5";
+		};
+		pinconf {
+			pins = "gpio4", "gpio5";
+			drive-strength = <2>;
+			bias-pull-down;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/msm8992.dtsi b/arch/arm64/boot/dts/qcom/msm8992.dtsi
new file mode 100644
index 000000000000..44b2d37d8c4b
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8992.dtsi
@@ -0,0 +1,184 @@
+/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/qcom,gcc-msm8994.h>
+
+/ {
+	model = "Qualcomm Technologies, Inc. MSM 8992";
+	compatible = "qcom,msm8992";
+	// msm-id needed by bootloader for selecting correct blob
+	qcom,msm-id = <251 0>, <252 0>;
+	interrupt-parent = <&intc>;
+
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	chosen { };
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&CPU0>;
+				};
+			};
+		};
+
+		CPU0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x0>;
+			next-level-cache = <&L2_0>;
+			L2_0: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+			};
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 2 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 3 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 4 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 1 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
+	};
+
+	xo_board: xo_board {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <19200000>;
+	};
+
+	sleep_clk: sleep_clk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+	};
+
+	soc {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0 0 0xffffffff>;
+		compatible = "simple-bus";
+
+		intc: interrupt-controller@f9000000 {
+			compatible = "qcom,msm-qgic2";
+			interrupt-controller;
+			#interrupt-cells = <3>;
+			reg = <0xf9000000 0x1000>,
+				<0xf9002000 0x1000>;
+		};
+
+		timer@f9020000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			compatible = "arm,armv7-timer-mem";
+			reg = <0xf9020000 0x1000>;
+
+			frame@f9021000 {
+				frame-number = <0>;
+				interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
+						<GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9021000 0x1000>,
+					<0xf9022000 0x1000>;
+			};
+
+			frame@f9023000 {
+				frame-number = <1>;
+				interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9023000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@f9024000 {
+				frame-number = <2>;
+				interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9024000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@f9025000 {
+				frame-number = <3>;
+				interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9025000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@f9026000 {
+				frame-number = <4>;
+				interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9026000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@f9027000 {
+				frame-number = <5>;
+				interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9027000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@f9028000 {
+				frame-number = <6>;
+				interrupts = <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9028000 0x1000>;
+				status = "disabled";
+			};
+		};
+
+		restart@fc4ab000 {
+			compatible = "qcom,pshold";
+			reg = <0xfc4ab000 0x4>;
+		};
+
+		msmgpio: pinctrl@fd510000 {
+			compatible = "qcom,msm8994-pinctrl";
+			reg = <0xfd510000 0x4000>;
+			interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		blsp1_uart2: serial@f991e000 {
+			compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm";
+			reg = <0xf991e000 0x1000>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_LOW>;
+			status = "disabled";
+			clock-names = "core", "iface";
+			clocks = <&clock_gcc GCC_BLSP1_UART2_APPS_CLK>,
+				<&clock_gcc GCC_BLSP1_AHB_CLK>;
+		};
+
+		clock_gcc: clock-controller@fc400000 {
+			compatible = "qcom,gcc-msm8994";
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+			reg = <0xfc400000 0x2000>;
+		};
+	};
+
+	memory {
+		device_type = "memory";
+		reg = <0 0 0 0>; // bootloader will update
+	};
+};
+
+
+#include "msm8992-pins.dtsi"
diff --git a/arch/arm64/boot/dts/qcom/msm8994-angler-rev-101.dts b/arch/arm64/boot/dts/qcom/msm8994-angler-rev-101.dts
new file mode 100644
index 000000000000..dfa08f513dc4
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8994-angler-rev-101.dts
@@ -0,0 +1,40 @@
+/* Copyright (c) 2015, Huawei Inc. All rights reserved.
+ * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+/dts-v1/;
+
+#include "msm8994.dtsi"
+
+/ {
+	model = "Huawei Nexus 6P";
+	compatible = "huawei,angler", "qcom,msm8994";
+	/* required for bootloader to select correct board */
+	qcom,board-id = <8026 0>;
+
+	aliases {
+		serial0 = &blsp1_uart2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	soc {
+		serial@f991e000 {
+			status = "okay";
+			pinctrl-names = "default", "sleep";
+			pinctrl-0 = <&blsp1_uart2_default>;
+			pinctrl-1 = <&blsp1_uart2_sleep>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/msm8994-pins.dtsi b/arch/arm64/boot/dts/qcom/msm8994-pins.dtsi
new file mode 100644
index 000000000000..0e4eea0df25d
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8994-pins.dtsi
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+&msmgpio {
+	blsp1_uart2_default: blsp1_uart2_default {
+		pinmux {
+			function = "blsp_uart2";
+			pins = "gpio4", "gpio5";
+		};
+		pinconf {
+			pins = "gpio4", "gpio5";
+			drive-strength = <16>;
+			bias-disable;
+		};
+	};
+
+	blsp1_uart2_sleep: blsp1_uart2_sleep {
+		pinmux {
+			function = "gpio";
+			pins = "gpio4", "gpio5";
+		};
+		pinconf {
+			pins = "gpio4", "gpio5";
+			drive-strength = <2>;
+			bias-pull-down;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/msm8994.dtsi b/arch/arm64/boot/dts/qcom/msm8994.dtsi
new file mode 100644
index 000000000000..f33c41d01c86
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8994.dtsi
@@ -0,0 +1,216 @@
+/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/qcom,gcc-msm8994.h>
+
+/ {
+	model = "Qualcomm Technologies, Inc. MSM 8994";
+	compatible = "qcom,msm8994";
+	// msm-id and pmic-id are required by bootloader for
+	// proper selection of dt blob
+	qcom,msm-id = <207 0x20000>;
+	qcom,pmic-id = <0x10009 0x1000A 0x0 0x0>;
+	interrupt-parent = <&intc>;
+
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	chosen { };
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&CPU0>;
+				};
+			};
+		};
+
+		CPU0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0>;
+			next-level-cache = <&L2_0>;
+			L2_0: l2-cache {
+			      compatible = "cache";
+			      cache-level = <2>;
+			};
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <1 2 0xff08>,
+			     <1 3 0xff08>,
+			     <1 4 0xff08>,
+			     <1 1 0xff08>;
+	};
+
+	soc: soc {
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0 0 0xffffffff>;
+		compatible = "simple-bus";
+
+		intc: interrupt-controller@f9000000 {
+			compatible = "qcom,msm-qgic2";
+			interrupt-controller;
+			#interrupt-cells = <3>;
+			reg = <0xf9000000 0x1000>,
+				  <0xf9002000 0x1000>;
+		};
+
+		timer@f9020000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+			compatible = "arm,armv7-timer-mem";
+			reg = <0xf9020000 0x1000>;
+
+			frame@f9021000 {
+				frame-number = <0>;
+				interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9021000 0x1000>,
+				      <0xf9022000 0x1000>;
+			};
+
+			frame@f9023000 {
+				frame-number = <1>;
+				interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9023000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@f9024000 {
+				frame-number = <2>;
+				interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9024000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@f9025000 {
+				frame-number = <3>;
+				interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9025000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@f9026000 {
+				frame-number = <4>;
+				interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9026000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@f9027000 {
+				frame-number = <5>;
+				interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9027000 0x1000>;
+				status = "disabled";
+			};
+
+			frame@f9028000 {
+				frame-number = <6>;
+				interrupts = <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
+				reg = <0xf9028000 0x1000>;
+				status = "disabled";
+			};
+		};
+
+		restart@fc4ab000 {
+			compatible = "qcom,pshold";
+			reg = <0xfc4ab000 0x4>;
+		};
+
+		msmgpio: pinctrl@fd510000 {
+			compatible = "qcom,msm8994-pinctrl";
+			reg = <0xfd510000 0x4000>;
+			interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		blsp1_uart2: serial@f991e000 {
+			compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm";
+			reg = <0xf991e000 0x1000>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+			clock-names = "core", "iface";
+			clocks = <&clock_gcc GCC_BLSP1_UART2_APPS_CLK>,
+				 <&clock_gcc GCC_BLSP1_AHB_CLK>;
+		};
+
+		tcsr_mutex_regs: syscon@fd484000 {
+			compatible = "syscon";
+			reg = <0xfd484000 0x2000>;
+		};
+
+		clock_gcc: clock-controller@fc400000 {
+			compatible = "qcom,gcc-msm8994";
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+			reg = <0xfc400000 0x2000>;
+		};
+	};
+
+	memory {
+		device_type = "memory";
+		// We expect the bootloader to fill in the reg
+		reg = <0 0 0 0>;
+	};
+
+	xo_board: xo_board {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <19200000>;
+	};
+
+	sleep_clk: sleep_clk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		smem_mem: smem_region@6a00000 {
+			reg = <0x0 0x6a00000 0x0 0x200000>;
+			no-map;
+		};
+	};
+
+	tcsr_mutex: hwlock {
+		compatible = "qcom,tcsr-mutex";
+		syscon = <&tcsr_mutex_regs 0 0x80>;
+		#hwlock-cells = <1>;
+	};
+
+	qcom,smem@6a00000 {
+		compatible = "qcom,smem";
+		memory-region = <&smem_mem>;
+		hwlocks = <&tcsr_mutex 3>;
+	};
+};
+
+
+#include "msm8994-pins.dtsi"
diff --git a/arch/arm64/boot/dts/qcom/msm8996.dtsi b/arch/arm64/boot/dts/qcom/msm8996.dtsi
index 338f82a7fdc7..9d1d7ad9b075 100644
--- a/arch/arm64/boot/dts/qcom/msm8996.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8996.dtsi
@@ -30,6 +30,42 @@
 		reg = <0 0 0 0>;
 	};
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		mba_region: mba@91500000 {
+			reg = <0x0 0x91500000 0x0 0x200000>;
+			no-map;
+		};
+
+		slpi_region: slpi@90b00000 {
+			reg = <0x0 0x90b00000 0x0 0xa00000>;
+			no-map;
+		};
+
+		venus_region: venus@90400000 {
+			reg = <0x0 0x90400000 0x0 0x700000>;
+			no-map;
+		};
+
+		adsp_region: adsp@8ea00000 {
+			reg = <0x0 0x8ea00000 0x0 0x1a00000>;
+			no-map;
+		};
+
+		mpss_region: mpss@88800000 {
+			reg = <0x0 0x88800000 0x0 0x6200000>;
+			no-map;
+		};
+
+		smem_mem: smem-mem@86000000 {
+			reg = <0x0 0x86000000 0x0 0x200000>;
+			no-map;
+		};
+	};
+
 	cpus {
 		#address-cells = <2>;
 		#size-cells = <0>;
@@ -192,14 +228,14 @@
 	};
 
 	clocks {
-		xo_board {
+		xo_board: xo_board {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <19200000>;
 			clock-output-names = "xo_board";
 		};
 
-		sleep_clk {
+		sleep_clk: sleep_clk {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <32764>;
@@ -212,12 +248,29 @@
 		method = "smc";
 	};
 
+	tcsr_mutex: hwlock {
+		compatible = "qcom,tcsr-mutex";
+		syscon = <&tcsr_mutex_regs 0 0x1000>;
+		#hwlock-cells = <1>;
+	};
+
+	smem {
+		compatible = "qcom,smem";
+		memory-region = <&smem_mem>;
+		hwlocks = <&tcsr_mutex 3>;
+	};
+
 	soc: soc {
 		#address-cells = <1>;
 		#size-cells = <1>;
 		ranges = <0 0 0 0xffffffff>;
 		compatible = "simple-bus";
 
+		tcsr_mutex_regs: syscon@740000 {
+			compatible = "syscon";
+			reg = <0x740000 0x20000>;
+		};
+
 		intc: interrupt-controller@9bc0000 {
 			compatible = "arm,gic-v3";
 			#interrupt-cells = <3>;
@@ -229,6 +282,11 @@
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
+		apcs: syscon@9820000 {
+			compatible = "syscon";
+			reg = <0x9820000 0x1000>;
+		};
+
 		gcc: clock-controller@300000 {
 			compatible = "qcom,gcc-msm8996";
 			#clock-cells = <1>;
@@ -347,9 +405,10 @@
 			 interrupts = <0 125 0>, <0 221 0>;
 			 interrupt-names = "hc_irq", "pwr_irq";
 
-			 clock-names = "iface", "core";
+			 clock-names = "iface", "core", "xo";
 			 clocks = <&gcc GCC_SDCC2_AHB_CLK>,
-			 <&gcc GCC_SDCC2_APPS_CLK>;
+			 <&gcc GCC_SDCC2_APPS_CLK>,
+			 <&xo_board>;
 			 bus-width = <4>;
 		 };
 
@@ -458,5 +517,29 @@
 					       <825000000>;
 		};
 	};
+
+	adsp-smp2p {
+		compatible = "qcom,smp2p";
+		qcom,smem = <443>, <429>;
+
+		interrupts = <0 158 IRQ_TYPE_EDGE_RISING>;
+
+		qcom,ipc = <&apcs 16 10>;
+
+		qcom,local-pid = <0>;
+		qcom,remote-pid = <2>;
+
+		adsp_smp2p_out: master-kernel {
+			qcom,entry-name = "master-kernel";
+			#qcom,state-cells = <1>;
+		};
+
+		adsp_smp2p_in: slave-kernel {
+			qcom,entry-name = "slave-kernel";
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
 };
 #include "msm8996-pins.dtsi"
diff --git a/arch/arm64/boot/dts/qcom/pm8916.dtsi b/arch/arm64/boot/dts/qcom/pm8916.dtsi
index f71679b15d54..53deebf9f515 100644
--- a/arch/arm64/boot/dts/qcom/pm8916.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm8916.dtsi
@@ -91,9 +91,52 @@
 	};
 
 	pm8916_1: pm8916@1 {
-		compatible = "qcom,spmi-pmic";
+		compatible = "qcom,pm8916", "qcom,spmi-pmic";
 		reg = <0x1 SPMI_USID>;
 		#address-cells = <1>;
 		#size-cells = <0>;
+
+                wcd_codec: codec@f000 {
+                       compatible = "qcom,pm8916-wcd-analog-codec";
+	               reg = <0xf000 0x200>;
+	               reg-names = "pmic-codec-core";
+	               clocks = <&gcc GCC_CODEC_DIGCODEC_CLK>;
+	               clock-names = "mclk";
+	               interrupt-parent = <&spmi_bus>;
+	               interrupts = <0x1 0xf0 0x0 IRQ_TYPE_NONE>,
+	                            <0x1 0xf0 0x1 IRQ_TYPE_NONE>,
+	                            <0x1 0xf0 0x2 IRQ_TYPE_NONE>,
+	                            <0x1 0xf0 0x3 IRQ_TYPE_NONE>,
+	                            <0x1 0xf0 0x4 IRQ_TYPE_NONE>,
+	                            <0x1 0xf0 0x5 IRQ_TYPE_NONE>,
+	                            <0x1 0xf0 0x6 IRQ_TYPE_NONE>,
+	                            <0x1 0xf0 0x7 IRQ_TYPE_NONE>,
+	                            <0x1 0xf1 0x0 IRQ_TYPE_NONE>,
+	                            <0x1 0xf1 0x1 IRQ_TYPE_NONE>,
+	                            <0x1 0xf1 0x2 IRQ_TYPE_NONE>,
+	                            <0x1 0xf1 0x3 IRQ_TYPE_NONE>,
+	                            <0x1 0xf1 0x4 IRQ_TYPE_NONE>,
+	                            <0x1 0xf1 0x5 IRQ_TYPE_NONE>;
+	               interrupt-names = "cdc_spk_cnp_int",
+	                                 "cdc_spk_clip_int",
+	                                 "cdc_spk_ocp_int",
+	                                 "mbhc_ins_rem_det1",
+	                                 "mbhc_but_rel_det",
+	                                 "mbhc_but_press_det",
+	                                 "mbhc_ins_rem_det",
+	                                 "mbhc_switch_int",
+	                                 "cdc_ear_ocp_int",
+	                                 "cdc_hphr_ocp_int",
+	                                 "cdc_hphl_ocp_det",
+	                                 "cdc_ear_cnp_int",
+	                                 "cdc_hphr_cnp_int",
+	                                 "cdc_hphl_cnp_int";
+	               vdd-cdc-io-supply = <&pm8916_l5>;
+	               vdd-cdc-tx-rx-cx-supply = <&pm8916_l5>;
+	               vdd-micbias-supply = <&pm8916_l13>;
+	               #sound-dai-cells = <1>;
+
+                };
+
 	};
 };
diff --git a/arch/arm64/boot/dts/qcom/pm8994.dtsi b/arch/arm64/boot/dts/qcom/pm8994.dtsi
index 1222d2e904f6..0f1866024ae3 100644
--- a/arch/arm64/boot/dts/qcom/pm8994.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm8994.dtsi
@@ -29,6 +29,7 @@
 				     <0 0xcc 0 IRQ_TYPE_NONE>,
 				     <0 0xcd 0 IRQ_TYPE_NONE>,
 				     <0 0xce 0 IRQ_TYPE_NONE>,
+				     <0 0xcf 0 IRQ_TYPE_NONE>,
 				     <0 0xd0 0 IRQ_TYPE_NONE>,
 				     <0 0xd1 0 IRQ_TYPE_NONE>,
 				     <0 0xd2 0 IRQ_TYPE_NONE>,
diff --git a/arch/arm64/boot/dts/renesas/Makefile b/arch/arm64/boot/dts/renesas/Makefile
index eb72830ec9eb..1618e0a3c81d 100644
--- a/arch/arm64/boot/dts/renesas/Makefile
+++ b/arch/arm64/boot/dts/renesas/Makefile
@@ -1,5 +1,5 @@
 dtb-$(CONFIG_ARCH_R8A7795) += r8a7795-salvator-x.dtb r8a7795-h3ulcb.dtb
-dtb-$(CONFIG_ARCH_R8A7796) += r8a7796-salvator-x.dtb
+dtb-$(CONFIG_ARCH_R8A7796) += r8a7796-salvator-x.dtb r8a7796-m3ulcb.dtb
 
 always		:= $(dtb-y)
 clean-files	:= *.dtb
diff --git a/arch/arm64/boot/dts/renesas/r8a7795-h3ulcb.dts b/arch/arm64/boot/dts/renesas/r8a7795-h3ulcb.dts
index bcb11a868343..6ffb0517421a 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795-h3ulcb.dts
+++ b/arch/arm64/boot/dts/renesas/r8a7795-h3ulcb.dts
@@ -1,5 +1,5 @@
 /*
- * Device Tree Source for the H3ULCB board
+ * Device Tree Source for the H3ULCB (R-Car Starter Kit Premier) board
  *
  * Copyright (C) 2016 Renesas Electronics Corp.
  * Copyright (C) 2016 Cogent Embedded, Inc.
@@ -62,6 +62,24 @@
 		clock-frequency = <24576000>;
 	};
 
+	reg_1p8v: regulator0 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator1 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
 	vcc_sdhi0: regulator-vcc-sdhi0 {
 		compatible = "regulator-fixed";
 
@@ -145,18 +163,30 @@
 		function = "avb";
 	};
 
-	sdhi0_pins_3v3: sd0_3v3 {
+	sdhi0_pins: sd0 {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
 		power-source = <3300>;
 	};
 
-	sdhi0_pins_1v8: sd0_1v8 {
+	sdhi0_pins_uhs: sd0 {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
 		power-source = <1800>;
 	};
 
+	sdhi2_pins: sd2 {
+		groups = "sdhi2_data8", "sdhi2_ctrl";
+		function = "sdhi2";
+		power-source = <3300>;
+	};
+
+	sdhi2_pins_uhs: sd2_uhs {
+		groups = "sdhi2_data8", "sdhi2_ctrl";
+		function = "sdhi2";
+		power-source = <1800>;
+	};
+
 	sound_pins: sound {
 		groups = "ssi01239_ctrl", "ssi0_data", "ssi1_data_a";
 		function = "ssi";
@@ -261,8 +291,8 @@
 };
 
 &sdhi0 {
-	pinctrl-0 = <&sdhi0_pins_3v3>;
-	pinctrl-1 = <&sdhi0_pins_1v8>;
+	pinctrl-0 = <&sdhi0_pins>;
+	pinctrl-1 = <&sdhi0_pins_uhs>;
 	pinctrl-names = "default", "state_uhs";
 
 	vmmc-supply = <&vcc_sdhi0>;
@@ -273,6 +303,19 @@
 	status = "okay";
 };
 
+&sdhi2 {
+	/* used for on-board 8bit eMMC */
+	pinctrl-0 = <&sdhi2_pins>;
+	pinctrl-1 = <&sdhi2_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
+
+	vmmc-supply = <&reg_3p3v>;
+	vqmmc-supply = <&reg_1p8v>;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
 &ssi1 {
 	shared-pin;
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts b/arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts
index b1eab6876f8c..bcaf4008d32d 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts
+++ b/arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts
@@ -62,6 +62,24 @@
 		clock-frequency = <24576000>;
 	};
 
+	reg_1p8v: regulator0 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator1 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
 	vcc_sdhi0: regulator-vcc-sdhi0 {
 		compatible = "regulator-fixed";
 
@@ -191,6 +209,10 @@
 				remote-endpoint = <&adv7123_in>;
 			};
 		};
+		port@3 {
+			lvds_connector: endpoint {
+			};
+		};
 	};
 };
 
@@ -237,11 +259,37 @@
 	sdhi0_pins: sd0 {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
+		power-source = <3300>;
+	};
+
+	sdhi0_pins_uhs: sd0_uhs {
+		groups = "sdhi0_data4", "sdhi0_ctrl";
+		function = "sdhi0";
+		power-source = <1800>;
+	};
+
+	sdhi2_pins: sd2 {
+		groups = "sdhi2_data8", "sdhi2_ctrl";
+		function = "sdhi2";
+		power-source = <3300>;
+	};
+
+	sdhi2_pins_uhs: sd2_uhs {
+		groups = "sdhi2_data8", "sdhi2_ctrl";
+		function = "sdhi2";
+		power-source = <1800>;
 	};
 
 	sdhi3_pins: sd3 {
 		groups = "sdhi3_data4", "sdhi3_ctrl";
 		function = "sdhi3";
+		power-source = <3300>;
+	};
+
+	sdhi3_pins_uhs: sd3_uhs {
+		groups = "sdhi3_data4", "sdhi3_ctrl";
+		function = "sdhi3";
+		power-source = <1800>;
 	};
 
 	sound_pins: sound {
@@ -261,8 +309,20 @@
 	};
 
 	usb1_pins: usb1 {
-		groups = "usb1";
-		function = "usb1";
+		mux {
+			groups = "usb1";
+			function = "usb1";
+		};
+
+		ovc {
+			pins = "GP_6_27";
+			bias-pull-up;
+		};
+
+		pwen {
+			pins = "GP_6_26";
+			bias-pull-down;
+		};
 	};
 
 	usb2_pins: usb2 {
@@ -371,25 +431,42 @@
 
 &sdhi0 {
 	pinctrl-0 = <&sdhi0_pins>;
-	pinctrl-names = "default";
+	pinctrl-1 = <&sdhi0_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
 
 	vmmc-supply = <&vcc_sdhi0>;
 	vqmmc-supply = <&vccq_sdhi0>;
 	cd-gpios = <&gpio3 12 GPIO_ACTIVE_LOW>;
 	wp-gpios = <&gpio3 13 GPIO_ACTIVE_HIGH>;
 	bus-width = <4>;
+	sd-uhs-sdr50;
+	status = "okay";
+};
+
+&sdhi2 {
+	/* used for on-board 8bit eMMC */
+	pinctrl-0 = <&sdhi2_pins>;
+	pinctrl-1 = <&sdhi2_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
+
+	vmmc-supply = <&reg_3p3v>;
+	vqmmc-supply = <&reg_1p8v>;
+	bus-width = <8>;
+	non-removable;
 	status = "okay";
 };
 
 &sdhi3 {
 	pinctrl-0 = <&sdhi3_pins>;
-	pinctrl-names = "default";
+	pinctrl-1 = <&sdhi3_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
 
 	vmmc-supply = <&vcc_sdhi3>;
 	vqmmc-supply = <&vccq_sdhi3>;
 	cd-gpios = <&gpio4 15 GPIO_ACTIVE_LOW>;
 	wp-gpios = <&gpio4 16 GPIO_ACTIVE_HIGH>;
 	bus-width = <4>;
+	sd-uhs-sdr50;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/renesas/r8a7795.dtsi b/arch/arm64/boot/dts/renesas/r8a7795.dtsi
index 8c15040f2540..bbf594bce930 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a7795.dtsi
@@ -321,6 +321,16 @@
 			#power-domain-cells = <0>;
 		};
 
+		rst: reset-controller@e6160000 {
+			compatible = "renesas,r8a7795-rst";
+			reg = <0 0xe6160000 0 0x0200>;
+		};
+
+		prr: chipid@fff00044 {
+			compatible = "renesas,prr";
+			reg = <0 0xfff00044 0 4>;
+		};
+
 		sysc: system-controller@e6180000 {
 			compatible = "renesas,r8a7795-sysc";
 			reg = <0 0xe6180000 0 0x0400>;
@@ -1306,28 +1316,28 @@
 		};
 
 		fcpvb1: fcp@fe92f000 {
-			compatible = "renesas,r8a7795-fcpv", "renesas,fcpv";
+			compatible = "renesas,fcpv";
 			reg = <0 0xfe92f000 0 0x200>;
 			clocks = <&cpg CPG_MOD 606>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
 		};
 
 		fcpf0: fcp@fe950000 {
-			compatible = "renesas,r8a7795-fcpf", "renesas,fcpf";
+			compatible = "renesas,fcpf";
 			reg = <0 0xfe950000 0 0x200>;
 			clocks = <&cpg CPG_MOD 615>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
 		};
 
 		fcpf1: fcp@fe951000 {
-			compatible = "renesas,r8a7795-fcpf", "renesas,fcpf";
+			compatible = "renesas,fcpf";
 			reg = <0 0xfe951000 0 0x200>;
 			clocks = <&cpg CPG_MOD 614>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
 		};
 
 		fcpf2: fcp@fe952000 {
-			compatible = "renesas,r8a7795-fcpf", "renesas,fcpf";
+			compatible = "renesas,fcpf";
 			reg = <0 0xfe952000 0 0x200>;
 			clocks = <&cpg CPG_MOD 613>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
@@ -1344,7 +1354,7 @@
 		};
 
 		fcpvb0: fcp@fe96f000 {
-			compatible = "renesas,r8a7795-fcpv", "renesas,fcpv";
+			compatible = "renesas,fcpv";
 			reg = <0 0xfe96f000 0 0x200>;
 			clocks = <&cpg CPG_MOD 607>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
@@ -1361,7 +1371,7 @@
 		};
 
 		fcpvi0: fcp@fe9af000 {
-			compatible = "renesas,r8a7795-fcpv", "renesas,fcpv";
+			compatible = "renesas,fcpv";
 			reg = <0 0xfe9af000 0 0x200>;
 			clocks = <&cpg CPG_MOD 611>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
@@ -1378,7 +1388,7 @@
 		};
 
 		fcpvi1: fcp@fe9bf000 {
-			compatible = "renesas,r8a7795-fcpv", "renesas,fcpv";
+			compatible = "renesas,fcpv";
 			reg = <0 0xfe9bf000 0 0x200>;
 			clocks = <&cpg CPG_MOD 610>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
@@ -1395,7 +1405,7 @@
 		};
 
 		fcpvi2: fcp@fe9cf000 {
-			compatible = "renesas,r8a7795-fcpv", "renesas,fcpv";
+			compatible = "renesas,fcpv";
 			reg = <0 0xfe9cf000 0 0x200>;
 			clocks = <&cpg CPG_MOD 609>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
@@ -1412,7 +1422,7 @@
 		};
 
 		fcpvd0: fcp@fea27000 {
-			compatible = "renesas,r8a7795-fcpv", "renesas,fcpv";
+			compatible = "renesas,fcpv";
 			reg = <0 0xfea27000 0 0x200>;
 			clocks = <&cpg CPG_MOD 603>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
@@ -1429,7 +1439,7 @@
 		};
 
 		fcpvd1: fcp@fea2f000 {
-			compatible = "renesas,r8a7795-fcpv", "renesas,fcpv";
+			compatible = "renesas,fcpv";
 			reg = <0 0xfea2f000 0 0x200>;
 			clocks = <&cpg CPG_MOD 602>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
@@ -1446,7 +1456,7 @@
 		};
 
 		fcpvd2: fcp@fea37000 {
-			compatible = "renesas,r8a7795-fcpv", "renesas,fcpv";
+			compatible = "renesas,fcpv";
 			reg = <0 0xfea37000 0 0x200>;
 			clocks = <&cpg CPG_MOD 601>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
@@ -1463,7 +1473,7 @@
 		};
 
 		fcpvd3: fcp@fea3f000 {
-			compatible = "renesas,r8a7795-fcpv", "renesas,fcpv";
+			compatible = "renesas,fcpv";
 			reg = <0 0xfea3f000 0 0x200>;
 			clocks = <&cpg CPG_MOD 600>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
diff --git a/arch/arm64/boot/dts/renesas/r8a7796-m3ulcb.dts b/arch/arm64/boot/dts/renesas/r8a7796-m3ulcb.dts
new file mode 100644
index 000000000000..c3f064ac2cb4
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a7796-m3ulcb.dts
@@ -0,0 +1,189 @@
+/*
+ * Device Tree Source for the M3ULCB (R-Car Starter Kit Pro) board
+ *
+ * Copyright (C) 2016 Renesas Electronics Corp.
+ * Copyright (C) 2016 Cogent Embedded, Inc.
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2.  This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied.
+ */
+
+/dts-v1/;
+#include "r8a7796.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/ {
+	model = "Renesas M3ULCB board based on r8a7796";
+	compatible = "renesas,m3ulcb", "renesas,r8a7796";
+
+	aliases {
+		serial0 = &scif2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@48000000 {
+		device_type = "memory";
+		/* first 128MB is reserved for secure area. */
+		reg = <0x0 0x48000000 0x0 0x38000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led5 {
+			gpios = <&gpio6 12 GPIO_ACTIVE_HIGH>;
+		};
+		led6 {
+			gpios = <&gpio6 13 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	keyboard {
+		compatible = "gpio-keys";
+
+		key-1 {
+			linux,code = <KEY_1>;
+			label = "SW3";
+			wakeup-source;
+			debounce-interval = <20>;
+			gpios = <&gpio6 11 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	reg_1p8v: regulator0 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator1 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	vcc_sdhi0: regulator-vcc-sdhi0 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "SDHI0 Vcc";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&gpio5 2 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vccq_sdhi0: regulator-vccq-sdhi0 {
+		compatible = "regulator-gpio";
+
+		regulator-name = "SDHI0 VccQ";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpios = <&gpio5 1 GPIO_ACTIVE_HIGH>;
+		gpios-states = <1>;
+		states = <3300000 1
+			  1800000 0>;
+	};
+};
+
+&extal_clk {
+	clock-frequency = <16666666>;
+};
+
+&extalr_clk {
+	clock-frequency = <32768>;
+};
+
+&pfc {
+	pinctrl-0 = <&scif_clk_pins>;
+	pinctrl-names = "default";
+
+	scif2_pins: scif2 {
+		groups = "scif2_data_a";
+		function = "scif2";
+	};
+
+	scif_clk_pins: scif_clk {
+		groups = "scif_clk_a";
+		function = "scif_clk";
+	};
+
+	sdhi0_pins: sd0 {
+		groups = "sdhi0_data4", "sdhi0_ctrl";
+		function = "sdhi0";
+		power-source = <3300>;
+	};
+
+	sdhi0_pins_uhs: sd0_uhs {
+		groups = "sdhi0_data4", "sdhi0_ctrl";
+		function = "sdhi0";
+		power-source = <1800>;
+	};
+
+	sdhi2_pins: sd2 {
+		groups = "sdhi2_data8", "sdhi2_ctrl";
+		function = "sdhi2";
+		power-source = <3300>;
+	};
+
+	sdhi2_pins_uhs: sd2_uhs {
+		groups = "sdhi2_data8", "sdhi2_ctrl";
+		function = "sdhi2";
+		power-source = <1800>;
+	};
+};
+
+&sdhi0 {
+	pinctrl-0 = <&sdhi0_pins>;
+	pinctrl-1 = <&sdhi0_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
+
+	vmmc-supply = <&vcc_sdhi0>;
+	vqmmc-supply = <&vccq_sdhi0>;
+	cd-gpios = <&gpio3 12 GPIO_ACTIVE_LOW>;
+	bus-width = <4>;
+	sd-uhs-sdr50;
+	status = "okay";
+};
+
+&sdhi2 {
+	/* used for on-board 8bit eMMC */
+	pinctrl-0 = <&sdhi2_pins>;
+	pinctrl-1 = <&sdhi2_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
+
+	vmmc-supply = <&reg_3p3v>;
+	vqmmc-supply = <&reg_1p8v>;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+&scif2 {
+	pinctrl-0 = <&scif2_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&scif_clk {
+	clock-frequency = <14745600>;
+	status = "okay";
+};
+
+&wdt0 {
+	timeout-sec = <60>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts b/arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts
index 13db7d61c26c..f35e96ca7d60 100644
--- a/arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts
+++ b/arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts
@@ -10,6 +10,7 @@
 
 /dts-v1/;
 #include "r8a7796.dtsi"
+#include <dt-bindings/gpio/gpio.h>
 
 / {
 	model = "Renesas Salvator-X board based on r8a7796";
@@ -29,6 +30,72 @@
 		/* first 128MB is reserved for secure area. */
 		reg = <0x0 0x48000000 0x0 0x78000000>;
 	};
+
+	reg_1p8v: regulator0 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator1 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	vcc_sdhi0: regulator-vcc-sdhi0 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "SDHI0 Vcc";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&gpio5 2 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vccq_sdhi0: regulator-vccq-sdhi0 {
+		compatible = "regulator-gpio";
+
+		regulator-name = "SDHI0 VccQ";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpios = <&gpio5 1 GPIO_ACTIVE_HIGH>;
+		gpios-states = <1>;
+		states = <3300000 1
+			  1800000 0>;
+	};
+
+	vcc_sdhi3: regulator-vcc-sdhi3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "SDHI3 Vcc";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&gpio3 15 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vccq_sdhi3: regulator-vccq-sdhi3 {
+		compatible = "regulator-gpio";
+
+		regulator-name = "SDHI3 VccQ";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpios = <&gpio3 14 GPIO_ACTIVE_HIGH>;
+		gpios-states = <1>;
+		states = <3300000 1
+			  1800000 0>;
+	};
 };
 
 &pfc {
@@ -43,12 +110,98 @@
 		groups = "scif_clk_a";
 		function = "scif_clk";
 	};
+
+	i2c2_pins: i2c2 {
+		groups = "i2c2_a";
+		function = "i2c2";
+	};
+
+	sdhi0_pins: sd0 {
+		groups = "sdhi0_data4", "sdhi0_ctrl";
+		function = "sdhi0";
+		power-source = <3300>;
+	};
+
+	sdhi0_pins_uhs: sd0_uhs {
+		groups = "sdhi0_data4", "sdhi0_ctrl";
+		function = "sdhi0";
+		power-source = <1800>;
+	};
+
+	sdhi2_pins: sd2 {
+		groups = "sdhi2_data8", "sdhi2_ctrl";
+		function = "sdhi2";
+		power-source = <3300>;
+	};
+
+	sdhi2_pins_uhs: sd2_uhs {
+		groups = "sdhi2_data8", "sdhi2_ctrl";
+		function = "sdhi2";
+		power-source = <1800>;
+	};
+
+	sdhi3_pins: sd3 {
+		groups = "sdhi3_data4", "sdhi3_ctrl";
+		function = "sdhi3";
+		power-source = <3300>;
+	};
+
+	sdhi3_pins_uhs: sd3_uhs {
+		groups = "sdhi3_data4", "sdhi3_ctrl";
+		function = "sdhi3";
+		power-source = <1800>;
+	};
 };
 
 &extal_clk {
 	clock-frequency = <16666666>;
 };
 
+&extalr_clk {
+	clock-frequency = <32768>;
+};
+
+&sdhi0 {
+	pinctrl-0 = <&sdhi0_pins>;
+	pinctrl-1 = <&sdhi0_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
+
+	vmmc-supply = <&vcc_sdhi0>;
+	vqmmc-supply = <&vccq_sdhi0>;
+	cd-gpios = <&gpio3 12 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&gpio3 13 GPIO_ACTIVE_HIGH>;
+	bus-width = <4>;
+	sd-uhs-sdr50;
+	status = "okay";
+};
+
+&sdhi2 {
+	/* used for on-board 8bit eMMC */
+	pinctrl-0 = <&sdhi2_pins>;
+	pinctrl-1 = <&sdhi2_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
+
+	vmmc-supply = <&reg_3p3v>;
+	vqmmc-supply = <&reg_1p8v>;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+&sdhi3 {
+	pinctrl-0 = <&sdhi3_pins>;
+	pinctrl-1 = <&sdhi3_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
+
+	vmmc-supply = <&vcc_sdhi3>;
+	vqmmc-supply = <&vccq_sdhi3>;
+	cd-gpios = <&gpio4 15 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&gpio4 16 GPIO_ACTIVE_HIGH>;
+	bus-width = <4>;
+	sd-uhs-sdr50;
+	status = "okay";
+};
+
 &scif2 {
 	pinctrl-0 = <&scif2_pins>;
 	pinctrl-names = "default";
@@ -60,6 +213,13 @@
 	status = "okay";
 };
 
+&i2c2 {
+	pinctrl-0 = <&i2c2_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
 &wdt0 {
 	timeout-sec = <60>;
 	status = "okay";
diff --git a/arch/arm64/boot/dts/renesas/r8a7796.dtsi b/arch/arm64/boot/dts/renesas/r8a7796.dtsi
index 9217da983525..28ba59a00cd8 100644
--- a/arch/arm64/boot/dts/renesas/r8a7796.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a7796.dtsi
@@ -17,6 +17,16 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
+		i2c5 = &i2c5;
+		i2c6 = &i2c6;
+	};
+
 	psci {
 		compatible = "arm,psci-0.2";
 		method = "smc";
@@ -233,12 +243,123 @@
 			#power-domain-cells = <0>;
 		};
 
+		rst: reset-controller@e6160000 {
+			compatible = "renesas,r8a7796-rst";
+			reg = <0 0xe6160000 0 0x0200>;
+		};
+
+		prr: chipid@fff00044 {
+			compatible = "renesas,prr";
+			reg = <0 0xfff00044 0 4>;
+		};
+
 		sysc: system-controller@e6180000 {
 			compatible = "renesas,r8a7796-sysc";
 			reg = <0 0xe6180000 0 0x0400>;
 			#power-domain-cells = <1>;
 		};
 
+		i2c0: i2c@e6500000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7796";
+			reg = <0 0xe6500000 0 0x40>;
+			interrupts = <GIC_SPI 287 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 931>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			dmas = <&dmac1 0x91>, <&dmac1 0x90>,
+			       <&dmac2 0x91>, <&dmac2 0x90>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
+			status = "disabled";
+		};
+
+		i2c1: i2c@e6508000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7796";
+			reg = <0 0xe6508000 0 0x40>;
+			interrupts = <GIC_SPI 288 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 930>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			dmas = <&dmac1 0x93>, <&dmac1 0x92>,
+			       <&dmac2 0x93>, <&dmac2 0x92>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
+			status = "disabled";
+		};
+
+		i2c2: i2c@e6510000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7796";
+			reg = <0 0xe6510000 0 0x40>;
+			interrupts = <GIC_SPI 286 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 929>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			dmas = <&dmac1 0x95>, <&dmac1 0x94>,
+			       <&dmac2 0x95>, <&dmac2 0x94>;
+			dma-names = "tx", "rx", "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
+			status = "disabled";
+		};
+
+		i2c3: i2c@e66d0000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7796";
+			reg = <0 0xe66d0000 0 0x40>;
+			interrupts = <GIC_SPI 290 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 928>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			dmas = <&dmac0 0x97>, <&dmac0 0x96>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
+			status = "disabled";
+		};
+
+		i2c4: i2c@e66d8000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7796";
+			reg = <0 0xe66d8000 0 0x40>;
+			interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 927>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			dmas = <&dmac0 0x99>, <&dmac0 0x98>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
+			status = "disabled";
+		};
+
+		i2c5: i2c@e66e0000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7796";
+			reg = <0 0xe66e0000 0 0x40>;
+			interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 919>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			dmas = <&dmac0 0x9b>, <&dmac0 0x9a>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <110>;
+			status = "disabled";
+		};
+
+		i2c6: i2c@e66e8000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r8a7796";
+			reg = <0 0xe66e8000 0 0x40>;
+			interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 918>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			dmas = <&dmac0 0x9d>, <&dmac0 0x9c>;
+			dma-names = "tx", "rx";
+			i2c-scl-internal-delay-ns = <6>;
+			status = "disabled";
+		};
+
 		scif2: serial@e6e88000 {
 			compatible = "renesas,scif-r8a7796",
 				     "renesas,rcar-gen3-scif", "renesas,scif";
@@ -251,5 +372,144 @@
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
 			status = "disabled";
 		};
+
+		dmac0: dma-controller@e6700000 {
+			compatible = "renesas,dmac-r8a7796",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe6700000 0 0x10000>;
+			interrupts = <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 204 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 213 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 215 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 219>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#dma-cells = <1>;
+			dma-channels = <16>;
+		};
+
+		dmac1: dma-controller@e7300000 {
+			compatible = "renesas,dmac-r8a7796",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe7300000 0 0x10000>;
+			interrupts = <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 310 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 311 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 313 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 315 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 316 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 317 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 319 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 218>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#dma-cells = <1>;
+			dma-channels = <16>;
+		};
+
+		dmac2: dma-controller@e7310000 {
+			compatible = "renesas,dmac-r8a7796",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe7310000 0 0x10000>;
+			interrupts = <GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 417 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 418 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 420 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 421 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 422 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 426 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 427 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 428 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 429 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 430 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 431 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 397 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7",
+					"ch8", "ch9", "ch10", "ch11",
+					"ch12", "ch13", "ch14", "ch15";
+			clocks = <&cpg CPG_MOD 217>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#dma-cells = <1>;
+			dma-channels = <16>;
+		};
+
+		sdhi0: sd@ee100000 {
+			compatible = "renesas,sdhi-r8a7796";
+			reg = <0 0xee100000 0 0x2000>;
+			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 314>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		sdhi1: sd@ee120000 {
+			compatible = "renesas,sdhi-r8a7796";
+			reg = <0 0xee120000 0 0x2000>;
+			interrupts = <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 313>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		sdhi2: sd@ee140000 {
+			compatible = "renesas,sdhi-r8a7796";
+			reg = <0 0xee140000 0 0x2000>;
+			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 312>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
+
+		sdhi3: sd@ee160000 {
+			compatible = "renesas,sdhi-r8a7796";
+			reg = <0 0xee160000 0 0x2000>;
+			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 311>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			status = "disabled";
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile
index 87669f656454..3a862894ea44 100644
--- a/arch/arm64/boot/dts/rockchip/Makefile
+++ b/arch/arm64/boot/dts/rockchip/Makefile
@@ -1,6 +1,7 @@
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-evb-act8846.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-geekbox.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-orion-r68-meta.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-px5-evb.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-r88.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-evb.dtb
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3368-orion-r68-meta.dts b/arch/arm64/boot/dts/rockchip/rk3368-orion-r68-meta.dts
index ea0a8eceefd4..ff5a40399d02 100644
--- a/arch/arm64/boot/dts/rockchip/rk3368-orion-r68-meta.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3368-orion-r68-meta.dts
@@ -344,7 +344,7 @@
 &sdmmc {
 	bus-width = <4>;
 	clock-frequency = <50000000>;
-	clock-freq-min-max = <400000 50000000>;
+	max-frequency = <50000000>;
 	cap-sd-highspeed;
 	card-detect-delay = <200>;
 	num-slots = <1>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3368-px5-evb.dts b/arch/arm64/boot/dts/rockchip/rk3368-px5-evb.dts
new file mode 100644
index 000000000000..85f7a243d744
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3368-px5-evb.dts
@@ -0,0 +1,314 @@
+/*
+ * Copyright (c) 2016 Fuzhou Rockchip Electronics Co., Ltd
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+#include "rk3368.dtsi"
+#include <dt-bindings/input/input.h>
+
+/ {
+	model = "Rockchip PX5 EVB";
+	compatible = "rockchip,px5-evb", "rockchip,px5", "rockchip,rk3368";
+
+	chosen {
+		stdout-path = "serial4:115200n8";
+	};
+
+	memory@0 {
+		reg = <0x0 0x0 0x0 0x80000000>;
+		device_type = "memory";
+	};
+
+	keys: gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pwr_key>;
+
+		power {
+			gpios = <&gpio0 2 GPIO_ACTIVE_LOW>;
+			label = "GPIO Power";
+			linux,code = <KEY_POWER>;
+			wakeup-source;
+		};
+	};
+
+	vcc_sys: vcc-sys-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_sys";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&emmc {
+	status = "okay";
+	bus-width = <8>;
+	cap-mmc-highspeed;
+	clock-frequency = <150000000>;
+	disable-wp;
+	keep-power-in-suspend;
+	mmc-hs200-1_8v;
+	no-sdio;
+	no-sd;
+	non-removable;
+	num-slots = <1>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_clk>, <&emmc_cmd>, <&emmc_bus8>;
+	vmmc-supply = <&vcc_io>;
+	vqmmc-supply = <&vcc18_flash>;
+};
+
+&i2c0 {
+	status = "okay";
+
+	rk808: pmic@1b {
+		compatible = "rockchip,rk808";
+		reg = <0x1b>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <5 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_int>, <&pmic_sleep>;
+		rockchip,system-power-controller;
+		vcc1-supply = <&vcc_sys>;
+		vcc2-supply = <&vcc_sys>;
+		vcc3-supply = <&vcc_sys>;
+		vcc4-supply = <&vcc_sys>;
+		vcc6-supply = <&vcc_sys>;
+		vcc7-supply = <&vcc_sys>;
+		vcc8-supply = <&vcc_io>;
+		vcc9-supply = <&vcc_sys>;
+		vcc10-supply = <&vcc_sys>;
+		vcc11-supply = <&vcc_sys>;
+		vcc12-supply = <&vcc_io>;
+		clock-output-names = "xin32k", "rk808-clkout2";
+		#clock-cells = <1>;
+
+		regulators {
+			vdd_cpu: DCDC_REG1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-name = "vdd_cpu";
+			};
+
+			vdd_log: DCDC_REG2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-name = "vdd_log";
+			};
+
+			vcc_ddr: DCDC_REG3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vcc_ddr";
+			};
+
+			vcc_io: DCDC_REG4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc_io";
+			};
+
+			vcc18_flash: LDO_REG1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc18_flash";
+			};
+
+			vcca_33: LDO_REG2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcca_33";
+			};
+
+			vdd_10: LDO_REG3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-name = "vdd_10";
+			};
+
+			avdd_33: LDO_REG4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "avdd_33";
+			};
+
+			vccio_sd: LDO_REG5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vccio_sd";
+			};
+
+			vdd10_lcd: LDO_REG6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-name = "vdd10_lcd";
+			};
+
+			vcc_18: LDO_REG7 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc_18";
+			};
+
+			vcc18_lcd: LDO_REG8 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc18_lcd";
+			};
+
+			vcc_sd: SWITCH_REG1 {
+				regulator-name = "vcc_sd";
+			};
+
+			vcc33_lcd: SWITCH_REG2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vcc33_lcd";
+			};
+		};
+	};
+};
+
+&i2c1 {
+	status = "okay";
+
+	accelerometer@18 {
+		compatible = "bosch,bma250";
+		reg = <0x18>;
+		interrupt-parent = <&gpio2>;
+		interrupts = <17 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
+
+&i2c2 {
+	status = "okay";
+
+	gsl1680: touchscreen@40 {
+		compatible = "silead,gsl1680";
+		reg = <0x40>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <28 IRQ_TYPE_EDGE_FALLING>;
+		power-gpios = <&gpio3 15 GPIO_ACTIVE_HIGH>;
+		touchscreen-size-x = <800>;
+		touchscreen-size-y = <1280>;
+		silead,max-fingers = <5>;
+	};
+};
+
+&pinctrl {
+	keys {
+		pwr_key: pwr-key {
+			rockchip,pins = <0 2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	pmic {
+		pmic_sleep: pmic-sleep {
+			rockchip,pins = <0 0 RK_FUNC_2 &pcfg_pull_none>;
+		};
+
+		pmic_int: pmic-int {
+			rockchip,pins = <0 5 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+};
+
+&sdmmc {
+	status = "okay";
+	bus-width = <4>;
+	cap-mmc-highspeed;
+	cap-sd-highspeed;
+	card-detect-delay = <200>;
+	no-emmc;
+	no-sdio;
+	num-slots = <1>;
+	sd-uhs-sdr12;
+	sd-uhs-sdr25;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdmmc_clk>, <&sdmmc_cmd>, <&sdmmc_bus4>, <&sdmmc_cd>;
+	rockchip,default-sample-phase = <90>;
+	vmmc-supply = <&vcc_sd>;
+	vqmmc-supply = <&vccio_sd>;
+};
+
+&tsadc {
+	status = "okay";
+	rockchip,hw-tshut-mode = <0>; /* CRU */
+	rockchip,hw-tshut-polarity = <1>; /* high */
+};
+
+&uart4 {
+	status = "okay";
+};
+
+&usb_host0_ehci {
+	status = "okay";
+};
+
+&usb_otg {
+	status = "okay";
+};
+
+&wdt {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3368.dtsi b/arch/arm64/boot/dts/rockchip/rk3368.dtsi
index 0fcb2147c9f9..a635adc47e74 100644
--- a/arch/arm64/boot/dts/rockchip/rk3368.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3368.dtsi
@@ -231,7 +231,7 @@
 	sdmmc: dwmmc@ff0c0000 {
 		compatible = "rockchip,rk3368-dw-mshc", "rockchip,rk3288-dw-mshc";
 		reg = <0x0 0xff0c0000 0x0 0x4000>;
-		clock-freq-min-max = <400000 150000000>;
+		max-frequency = <150000000>;
 		clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>,
 			 <&cru SCLK_SDMMC_DRV>, <&cru SCLK_SDMMC_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
@@ -243,7 +243,7 @@
 	sdio0: dwmmc@ff0d0000 {
 		compatible = "rockchip,rk3368-dw-mshc", "rockchip,rk3288-dw-mshc";
 		reg = <0x0 0xff0d0000 0x0 0x4000>;
-		clock-freq-min-max = <400000 150000000>;
+		max-frequency = <150000000>;
 		clocks = <&cru HCLK_SDIO0>, <&cru SCLK_SDIO0>,
 			 <&cru SCLK_SDIO0_DRV>, <&cru SCLK_SDIO0_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
@@ -255,7 +255,7 @@
 	emmc: dwmmc@ff0f0000 {
 		compatible = "rockchip,rk3368-dw-mshc", "rockchip,rk3288-dw-mshc";
 		reg = <0x0 0xff0f0000 0x0 0x4000>;
-		clock-freq-min-max = <400000 150000000>;
+		max-frequency = <150000000>;
 		clocks = <&cru HCLK_EMMC>, <&cru SCLK_EMMC>,
 			 <&cru SCLK_EMMC_DRV>, <&cru SCLK_EMMC_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
@@ -315,16 +315,16 @@
 		status = "disabled";
 	};
 
-	i2c1: i2c@ff140000 {
+	i2c2: i2c@ff140000 {
 		compatible = "rockchip,rk3368-i2c", "rockchip,rk3288-i2c";
 		reg = <0x0 0xff140000 0x0 0x1000>;
 		interrupts = <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>;
 		#address-cells = <1>;
 		#size-cells = <0>;
 		clock-names = "i2c";
-		clocks = <&cru PCLK_I2C1>;
+		clocks = <&cru PCLK_I2C2>;
 		pinctrl-names = "default";
-		pinctrl-0 = <&i2c1_xfer>;
+		pinctrl-0 = <&i2c2_xfer>;
 		status = "disabled";
 	};
 
@@ -537,7 +537,6 @@
 		g-np-tx-fifo-size = <16>;
 		g-rx-fifo-size = <275>;
 		g-tx-fifo-size = <256 128 128 64 64 32>;
-		g-use-dma;
 		status = "disabled";
 	};
 
@@ -554,16 +553,16 @@
 		status = "disabled";
 	};
 
-	i2c2: i2c@ff660000 {
+	i2c1: i2c@ff660000 {
 		compatible = "rockchip,rk3368-i2c", "rockchip,rk3288-i2c";
 		reg = <0x0 0xff660000 0x0 0x1000>;
 		interrupts = <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
 		#address-cells = <1>;
 		#size-cells = <0>;
 		clock-names = "i2c";
-		clocks = <&cru PCLK_I2C2>;
+		clocks = <&cru PCLK_I2C1>;
 		pinctrl-names = "default";
-		pinctrl-0 = <&i2c2_xfer>;
+		pinctrl-0 = <&i2c1_xfer>;
 		status = "disabled";
 	};
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-evb.dts b/arch/arm64/boot/dts/rockchip/rk3399-evb.dts
index 8e82497925fe..3040a989d699 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-evb.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3399-evb.dts
@@ -49,6 +49,46 @@
 	compatible = "rockchip,rk3399-evb", "rockchip,rk3399",
 		     "google,rk3399evb-rev2";
 
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		brightness-levels = <
+			  0   1   2   3   4   5   6   7
+			  8   9  10  11  12  13  14  15
+			 16  17  18  19  20  21  22  23
+			 24  25  26  27  28  29  30  31
+			 32  33  34  35  36  37  38  39
+			 40  41  42  43  44  45  46  47
+			 48  49  50  51  52  53  54  55
+			 56  57  58  59  60  61  62  63
+			 64  65  66  67  68  69  70  71
+			 72  73  74  75  76  77  78  79
+			 80  81  82  83  84  85  86  87
+			 88  89  90  91  92  93  94  95
+			 96  97  98  99 100 101 102 103
+			104 105 106 107 108 109 110 111
+			112 113 114 115 116 117 118 119
+			120 121 122 123 124 125 126 127
+			128 129 130 131 132 133 134 135
+			136 137 138 139 140 141 142 143
+			144 145 146 147 148 149 150 151
+			152 153 154 155 156 157 158 159
+			160 161 162 163 164 165 166 167
+			168 169 170 171 172 173 174 175
+			176 177 178 179 180 181 182 183
+			184 185 186 187 188 189 190 191
+			192 193 194 195 196 197 198 199
+			200 201 202 203 204 205 206 207
+			208 209 210 211 212 213 214 215
+			216 217 218 219 220 221 222 223
+			224 225 226 227 228 229 230 231
+			232 233 234 235 236 237 238 239
+			240 241 242 243 244 245 246 247
+			248 249 250 251 252 253 254 255>;
+		default-brightness-level = <200>;
+		enable-gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>;
+		pwms = <&pwm0 0 25000 0>;
+	};
+
 	clkin_gmac: external-gmac-clock {
 		compatible = "fixed-clock";
 		clock-frequency = <125000000>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
index 1e24e455700b..c928015d39a2 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
@@ -236,7 +236,7 @@
 			     "rockchip,rk3288-dw-mshc";
 		reg = <0x0 0xfe310000 0x0 0x4000>;
 		interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH 0>;
-		clock-freq-min-max = <400000 150000000>;
+		max-frequency = <150000000>;
 		clocks = <&cru HCLK_SDIO>, <&cru SCLK_SDIO>,
 			 <&cru SCLK_SDIO_DRV>, <&cru SCLK_SDIO_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
@@ -249,11 +249,12 @@
 			     "rockchip,rk3288-dw-mshc";
 		reg = <0x0 0xfe320000 0x0 0x4000>;
 		interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH 0>;
-		clock-freq-min-max = <400000 150000000>;
+		max-frequency = <150000000>;
 		clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>,
 			 <&cru SCLK_SDMMC_DRV>, <&cru SCLK_SDMMC_SAMPLE>;
 		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
 		fifo-depth = <0x100>;
+		power-domains = <&power RK3399_PD_SD>;
 		status = "disabled";
 	};
 
@@ -270,6 +271,7 @@
 		#clock-cells = <0>;
 		phys = <&emmc_phy>;
 		phy-names = "phy_arasan";
+		power-domains = <&power RK3399_PD_EMMC>;
 		status = "disabled";
 	};
 
@@ -694,6 +696,16 @@
 		status = "disabled";
 	};
 
+	qos_sd: qos@ffa74000 {
+		compatible = "syscon";
+		reg = <0x0 0xffa74000 0x0 0x20>;
+	};
+
+	qos_emmc: qos@ffa58000 {
+		compatible = "syscon";
+		reg = <0x0 0xffa58000 0x0 0x20>;
+	};
+
 	qos_gmac: qos@ffa5c000 {
 		compatible = "syscon";
 		reg = <0x0 0xffa5c000 0x0 0x20>;
@@ -827,11 +839,23 @@
 			};
 
 			/* These power domains are grouped by VD_LOGIC */
+			pd_emmc@RK3399_PD_EMMC {
+				reg = <RK3399_PD_EMMC>;
+				clocks = <&cru ACLK_EMMC>;
+				pm_qos = <&qos_emmc>;
+			};
 			pd_gmac@RK3399_PD_GMAC {
 				reg = <RK3399_PD_GMAC>;
-				clocks = <&cru ACLK_GMAC>;
+				clocks = <&cru ACLK_GMAC>,
+					 <&cru PCLK_GMAC>;
 				pm_qos = <&qos_gmac>;
 			};
+			pd_sd@RK3399_PD_SD {
+				reg = <RK3399_PD_SD>;
+				clocks = <&cru HCLK_SDMMC>,
+					 <&cru SCLK_SDMMC>;
+				pm_qos = <&qos_sd>;
+			};
 			pd_vio@RK3399_PD_VIO {
 				reg = <RK3399_PD_VIO>;
 				#address-cells = <1>;
@@ -1027,6 +1051,9 @@
 		clock-names = "pclk_efuse";
 
 		/* Data cells */
+		cpu_id: cpu-id@7 {
+			reg = <0x07 0x10>;
+		};
 		cpub_leakage: cpu-leakage@17 {
 			reg = <0x17 0x1>;
 		};
@@ -1105,6 +1132,16 @@
 				interrupt-names = "linestate";
 				status = "disabled";
 			};
+
+			u2phy0_otg: otg-port {
+				#phy-cells = <0>;
+				interrupts = <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH 0>,
+					     <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH 0>,
+					     <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH 0>;
+				interrupt-names = "otg-bvalid", "otg-id",
+						  "linestate";
+				status = "disabled";
+			};
 		};
 
 		u2phy1: usb2-phy@e460 {
@@ -1122,6 +1159,16 @@
 				interrupt-names = "linestate";
 				status = "disabled";
 			};
+
+			u2phy1_otg: otg-port {
+				#phy-cells = <0>;
+				interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH 0>,
+					     <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH 0>,
+					     <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH 0>;
+				interrupt-names = "otg-bvalid", "otg-id",
+						  "linestate";
+				status = "disabled";
+			};
 		};
 
 		emmc_phy: phy@f780 {
@@ -1152,6 +1199,7 @@
 		clock-names = "tcpdcore", "tcpdphy-ref";
 		assigned-clocks = <&cru SCLK_UPHY0_TCPDCORE>;
 		assigned-clock-rates = <50000000>;
+		power-domains = <&power RK3399_PD_TCPD0>;
 		resets = <&cru SRST_UPHY0>,
 			 <&cru SRST_UPHY0_PIPE_L00>,
 			 <&cru SRST_P_UPHY0_TCPHY>;
@@ -1180,6 +1228,7 @@
 		clock-names = "tcpdcore", "tcpdphy-ref";
 		assigned-clocks = <&cru SCLK_UPHY1_TCPDCORE>;
 		assigned-clock-rates = <50000000>;
+		power-domains = <&power RK3399_PD_TCPD1>;
 		resets = <&cru SRST_UPHY1>,
 			 <&cru SRST_UPHY1_PIPE_L00>,
 			 <&cru SRST_P_UPHY1_TCPHY>;
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi b/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi
index 3eb4c42ce7b9..7c7511b9d231 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi
@@ -43,7 +43,7 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
-/memreserve/ 0x80000000 0x00000008;	/* cpu-release-addr */
+/memreserve/ 0x80000000 0x00080000;
 
 / {
 	compatible = "socionext,uniphier-ld11";
@@ -70,19 +70,60 @@
 			device_type = "cpu";
 			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0 0x000>;
-			enable-method = "spin-table";
-			cpu-release-addr = <0 0x80000000>;
+			clocks = <&sys_clk 33>;
+			enable-method = "psci";
+			operating-points-v2 = <&cluster0_opp>;
 		};
 
 		cpu1: cpu@1 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0 0x001>;
-			enable-method = "spin-table";
-			cpu-release-addr = <0 0x80000000>;
+			clocks = <&sys_clk 33>;
+			enable-method = "psci";
+			operating-points-v2 = <&cluster0_opp>;
 		};
 	};
 
+	cluster0_opp: opp_table {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp@245000000 {
+			opp-hz = /bits/ 64 <245000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@250000000 {
+			opp-hz = /bits/ 64 <250000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@490000000 {
+			opp-hz = /bits/ 64 <490000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@500000000 {
+			opp-hz = /bits/ 64 <500000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@653334000 {
+			opp-hz = /bits/ 64 <653334000>;
+			clock-latency-ns = <300>;
+		};
+		opp@666667000 {
+			opp-hz = /bits/ 64 <666667000>;
+			clock-latency-ns = <300>;
+		};
+		opp@980000000 {
+			opp-hz = /bits/ 64 <980000000>;
+			clock-latency-ns = <300>;
+		};
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+	};
+
 	clocks {
 		refclk: ref {
 			compatible = "fixed-clock";
@@ -233,7 +274,7 @@
 		};
 
 		perictrl@59820000 {
-			compatible = "socionext,uniphier-perictrl",
+			compatible = "socionext,uniphier-ld11-perictrl",
 				     "simple-mfd", "syscon";
 			reg = <0x59820000 0x200>;
 
@@ -282,7 +323,7 @@
 		};
 
 		mioctrl@5b3e0000 {
-			compatible = "socionext,uniphier-mioctrl",
+			compatible = "socionext,uniphier-ld11-mioctrl",
 				     "simple-mfd", "syscon";
 			reg = <0x5b3e0000 0x800>;
 
@@ -299,7 +340,7 @@
 		};
 
 		soc-glue@5f800000 {
-			compatible = "socionext,uniphier-soc-glue",
+			compatible = "socionext,uniphier-ld11-soc-glue",
 				     "simple-mfd", "syscon";
 			reg = <0x5f800000 0x2000>;
 
@@ -320,7 +361,7 @@
 		sysctrl@61840000 {
 			compatible = "socionext,uniphier-ld11-sysctrl",
 				     "simple-mfd", "syscon";
-			reg = <0x61840000 0x4000>;
+			reg = <0x61840000 0x10000>;
 
 			sys_clk: clock {
 				compatible = "socionext,uniphier-ld11-clock";
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi
index 56a1b2e92cf3..fcaecc6bdeac 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi
@@ -43,7 +43,7 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
-/memreserve/ 0x80000000 0x00000008;	/* cpu-release-addr */
+/memreserve/ 0x80000000 0x00080000;
 
 / {
 	compatible = "socionext,uniphier-ld20";
@@ -79,35 +79,120 @@
 			device_type = "cpu";
 			compatible = "arm,cortex-a72", "arm,armv8";
 			reg = <0 0x000>;
-			enable-method = "spin-table";
-			cpu-release-addr = <0 0x80000000>;
+			clocks = <&sys_clk 32>;
+			enable-method = "psci";
+			operating-points-v2 = <&cluster0_opp>;
 		};
 
 		cpu1: cpu@1 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a72", "arm,armv8";
 			reg = <0 0x001>;
-			enable-method = "spin-table";
-			cpu-release-addr = <0 0x80000000>;
+			clocks = <&sys_clk 32>;
+			enable-method = "psci";
+			operating-points-v2 = <&cluster0_opp>;
 		};
 
 		cpu2: cpu@100 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0 0x100>;
-			enable-method = "spin-table";
-			cpu-release-addr = <0 0x80000000>;
+			clocks = <&sys_clk 33>;
+			enable-method = "psci";
+			operating-points-v2 = <&cluster1_opp>;
 		};
 
 		cpu3: cpu@101 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0 0x101>;
-			enable-method = "spin-table";
-			cpu-release-addr = <0 0x80000000>;
+			clocks = <&sys_clk 33>;
+			enable-method = "psci";
+			operating-points-v2 = <&cluster1_opp>;
 		};
 	};
 
+	cluster0_opp: opp_table0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp@250000000 {
+			opp-hz = /bits/ 64 <250000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@275000000 {
+			opp-hz = /bits/ 64 <275000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@500000000 {
+			opp-hz = /bits/ 64 <500000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@550000000 {
+			opp-hz = /bits/ 64 <550000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@666667000 {
+			opp-hz = /bits/ 64 <666667000>;
+			clock-latency-ns = <300>;
+		};
+		opp@733334000 {
+			opp-hz = /bits/ 64 <733334000>;
+			clock-latency-ns = <300>;
+		};
+		opp@1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@1100000000 {
+			opp-hz = /bits/ 64 <1100000000>;
+			clock-latency-ns = <300>;
+		};
+	};
+
+	cluster1_opp: opp_table1 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp@250000000 {
+			opp-hz = /bits/ 64 <250000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@275000000 {
+			opp-hz = /bits/ 64 <275000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@500000000 {
+			opp-hz = /bits/ 64 <500000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@550000000 {
+			opp-hz = /bits/ 64 <550000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@666667000 {
+			opp-hz = /bits/ 64 <666667000>;
+			clock-latency-ns = <300>;
+		};
+		opp@733334000 {
+			opp-hz = /bits/ 64 <733334000>;
+			clock-latency-ns = <300>;
+		};
+		opp@1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			clock-latency-ns = <300>;
+		};
+		opp@1100000000 {
+			opp-hz = /bits/ 64 <1100000000>;
+			clock-latency-ns = <300>;
+		};
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+	};
+
 	clocks {
 		refclk: ref {
 			compatible = "fixed-clock";
@@ -274,7 +359,7 @@
 		};
 
 		perictrl@59820000 {
-			compatible = "socionext,uniphier-perictrl",
+			compatible = "socionext,uniphier-ld20-perictrl",
 				     "simple-mfd", "syscon";
 			reg = <0x59820000 0x200>;
 
@@ -290,7 +375,7 @@
 		};
 
 		soc-glue@5f800000 {
-			compatible = "socionext,uniphier-soc-glue",
+			compatible = "socionext,uniphier-ld20-soc-glue",
 				     "simple-mfd", "syscon";
 			reg = <0x5f800000 0x2000>;
 
@@ -309,9 +394,9 @@
 		};
 
 		sysctrl@61840000 {
-			compatible = "socionext,uniphier-sysctrl",
+			compatible = "socionext,uniphier-ld20-sysctrl",
 				     "simple-mfd", "syscon";
-			reg = <0x61840000 0x4000>;
+			reg = <0x61840000 0x10000>;
 
 			sys_clk: clock {
 				compatible = "socionext,uniphier-ld20-clock";
diff --git a/arch/arm64/boot/dts/zte/zx296718.dtsi b/arch/arm64/boot/dts/zte/zx296718.dtsi
index a223066f24ce..88ff70a06086 100644
--- a/arch/arm64/boot/dts/zte/zx296718.dtsi
+++ b/arch/arm64/boot/dts/zte/zx296718.dtsi
@@ -239,16 +239,9 @@
 		compatible = "arm,gic-v3";
 		#interrupt-cells = <3>;
 		#address-cells = <0>;
-		#redistributor-regions = <6>;
-		redistributor-stride = <0x0 0x40000>;
 		interrupt-controller;
 		reg = <0x02a00000 0x10000>,
-		      <0x02b00000 0x20000>,
-		      <0x02b20000 0x20000>,
-		      <0x02b40000 0x20000>,
-		      <0x02b60000 0x20000>,
-		      <0x02b80000 0x20000>,
-		      <0x02ba0000 0x20000>;
+		      <0x02b00000 0xc0000>;
 		interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
@@ -284,9 +277,33 @@
 			dma-requests = <32>;
 		};
 
+		lsp0crm: clock-controller@1420000 {
+			compatible = "zte,zx296718-lsp0crm";
+			reg = <0x01420000 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		lsp1crm: clock-controller@1430000 {
+			compatible = "zte,zx296718-lsp1crm";
+			reg = <0x01430000 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		topcrm: clock-controller@1461000 {
+			compatible = "zte,zx296718-topcrm";
+			reg = <0x01461000 0x1000>;
+			#clock-cells = <1>;
+		};
+
 		sysctrl: sysctrl@1463000 {
 			compatible = "zte,zx296718-sysctrl", "syscon";
 			reg = <0x1463000 0x1000>;
 		};
+
+		audiocrm: clock-controller@1480000 {
+			compatible = "zte,zx296718-audiocrm";
+			reg = <0x01480000 0x1000>;
+			#clock-cells = <1>;
+		};
 	};
 };
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 6be08113a96d..869dded0f09f 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -11,7 +11,6 @@ CONFIG_TASK_XACCT=y
 CONFIG_TASK_IO_ACCOUNTING=y
 CONFIG_IKCONFIG=y
 CONFIG_IKCONFIG_PROC=y
-CONFIG_LOG_BUF_SHIFT=14
 CONFIG_MEMCG=y
 CONFIG_MEMCG_SWAP=y
 CONFIG_BLK_CGROUP=y
@@ -34,6 +33,7 @@ CONFIG_MODULE_UNLOAD=y
 # CONFIG_IOSCHED_DEADLINE is not set
 CONFIG_ARCH_SUNXI=y
 CONFIG_ARCH_ALPINE=y
+CONFIG_ARCH_BCM2835=y
 CONFIG_ARCH_BCM_IPROC=y
 CONFIG_ARCH_BERLIN=y
 CONFIG_ARCH_EXYNOS=y
@@ -82,6 +82,7 @@ CONFIG_KEXEC=y
 # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
 CONFIG_COMPAT=y
 CONFIG_CPU_IDLE=y
+CONFIG_HIBERNATION=y
 CONFIG_ARM_CPUIDLE=y
 CONFIG_CPU_FREQ=y
 CONFIG_CPUFREQ_DT=y
@@ -147,6 +148,7 @@ CONFIG_MTD_SPI_NOR=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_NBD=m
 CONFIG_VIRTIO_BLK=y
+CONFIG_EEPROM_AT25=m
 CONFIG_SRAM=y
 # CONFIG_SCSI_PROC_FS is not set
 CONFIG_BLK_DEV_SD=y
@@ -183,7 +185,10 @@ CONFIG_SMC91X=y
 CONFIG_SMSC911X=y
 CONFIG_STMMAC_ETH=m
 CONFIG_REALTEK_PHY=m
+CONFIG_MESON_GXL_PHY=m
 CONFIG_MICREL_PHY=y
+CONFIG_MDIO_BUS_MUX=y
+CONFIG_MDIO_BUS_MUX_MMIOREG=y
 CONFIG_USB_PEGASUS=m
 CONFIG_USB_RTL8150=m
 CONFIG_USB_RTL8152=m
@@ -194,6 +199,7 @@ CONFIG_USB_NET_SMSC75XX=m
 CONFIG_USB_NET_SMSC95XX=m
 CONFIG_USB_NET_PLUSB=m
 CONFIG_USB_NET_MCS7830=m
+CONFIG_BRCMFMAC=m
 CONFIG_WL18XX=m
 CONFIG_WLCORE_SDIO=m
 CONFIG_INPUT_EVDEV=y
@@ -206,6 +212,9 @@ CONFIG_SERIO_AMBAKMI=y
 CONFIG_LEGACY_PTY_COUNT=16
 CONFIG_SERIAL_8250=y
 CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_EXTENDED=y
+CONFIG_SERIAL_8250_SHARE_IRQ=y
+CONFIG_SERIAL_8250_BCM2835AUX=y
 CONFIG_SERIAL_8250_DW=y
 CONFIG_SERIAL_8250_MT6577=y
 CONFIG_SERIAL_8250_UNIPHIER=y
@@ -229,17 +238,21 @@ CONFIG_VIRTIO_CONSOLE=y
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_MUX=y
 CONFIG_I2C_MUX_PCA954x=y
+CONFIG_I2C_BCM2835=m
 CONFIG_I2C_DESIGNWARE_PLATFORM=y
 CONFIG_I2C_IMX=y
 CONFIG_I2C_MESON=y
 CONFIG_I2C_MV64XXX=y
 CONFIG_I2C_QUP=y
+CONFIG_I2C_RK3X=y
 CONFIG_I2C_TEGRA=y
 CONFIG_I2C_UNIPHIER_F=y
 CONFIG_I2C_RCAR=y
 CONFIG_I2C_CROS_EC_TUNNEL=y
 CONFIG_SPI=y
 CONFIG_SPI_MESON_SPIFC=m
+CONFIG_SPI_BCM2835=m
+CONFIG_SPI_BCM2835AUX=m
 CONFIG_SPI_ORION=y
 CONFIG_SPI_PL022=y
 CONFIG_SPI_QUP=y
@@ -249,10 +262,10 @@ CONFIG_SPMI=y
 CONFIG_PINCTRL_SINGLE=y
 CONFIG_PINCTRL_MAX77620=y
 CONFIG_PINCTRL_MSM8916=y
+CONFIG_PINCTRL_MSM8994=y
 CONFIG_PINCTRL_MSM8996=y
 CONFIG_PINCTRL_QDF2XXX=y
 CONFIG_PINCTRL_QCOM_SPMI_PMIC=y
-CONFIG_GPIO_SYSFS=y
 CONFIG_GPIO_DWAPB=y
 CONFIG_GPIO_PL061=y
 CONFIG_GPIO_RCAR=y
@@ -272,13 +285,16 @@ CONFIG_THERMAL=y
 CONFIG_THERMAL_EMULATION=y
 CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
 CONFIG_CPU_THERMAL=y
+CONFIG_BCM2835_THERMAL=y
 CONFIG_EXYNOS_THERMAL=y
 CONFIG_WATCHDOG=y
+CONFIG_BCM2835_WDT=y
 CONFIG_RENESAS_WDT=y
 CONFIG_S3C2410_WATCHDOG=y
 CONFIG_MESON_GXBB_WATCHDOG=m
 CONFIG_MESON_WATCHDOG=m
 CONFIG_MFD_MAX77620=y
+CONFIG_MFD_RK808=y
 CONFIG_MFD_SPMI_PMIC=y
 CONFIG_MFD_SEC_CORE=y
 CONFIG_MFD_HI655X_PMIC=y
@@ -292,10 +308,26 @@ CONFIG_REGULATOR_MAX77620=y
 CONFIG_REGULATOR_PWM=y
 CONFIG_REGULATOR_QCOM_SMD_RPM=y
 CONFIG_REGULATOR_QCOM_SPMI=y
+CONFIG_REGULATOR_RK808=y
 CONFIG_REGULATOR_S2MPS11=y
+CONFIG_MEDIA_SUPPORT=m
+CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_ANALOG_TV_SUPPORT=y
+CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y
+CONFIG_MEDIA_CONTROLLER=y
+CONFIG_VIDEO_V4L2_SUBDEV_API=y
+# CONFIG_DVB_NET is not set
+CONFIG_V4L_MEM2MEM_DRIVERS=y
+CONFIG_VIDEO_RENESAS_FCP=m
+CONFIG_VIDEO_RENESAS_VSP1=m
 CONFIG_DRM=m
 CONFIG_DRM_NOUVEAU=m
+CONFIG_DRM_RCAR_DU=m
+CONFIG_DRM_RCAR_HDMI=y
+CONFIG_DRM_RCAR_LVDS=y
+CONFIG_DRM_RCAR_VSP=y
 CONFIG_DRM_TEGRA=m
+CONFIG_DRM_VC4=m
 CONFIG_DRM_PANEL_SIMPLE=m
 CONFIG_DRM_I2C_ADV7511=m
 CONFIG_DRM_HISI_KIRIN=m
@@ -310,6 +342,7 @@ CONFIG_LOGO=y
 CONFIG_SOUND=y
 CONFIG_SND=y
 CONFIG_SND_SOC=y
+CONFIG_SND_BCM2835_SOC_I2S=m
 CONFIG_SND_SOC_RCAR=y
 CONFIG_SND_SOC_SAMSUNG=y
 CONFIG_SND_SOC_AK4613=y
@@ -342,9 +375,11 @@ CONFIG_USB_RENESAS_USBHS_UDC=m
 CONFIG_MMC=y
 CONFIG_MMC_BLOCK_MINORS=32
 CONFIG_MMC_ARMMMCI=y
+CONFIG_MMC_MESON_GX=y
 CONFIG_MMC_SDHCI=y
 CONFIG_MMC_SDHCI_ACPI=y
 CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_ARASAN=y
 CONFIG_MMC_SDHCI_OF_ESDHC=y
 CONFIG_MMC_SDHCI_TEGRA=y
 CONFIG_MMC_SDHCI_MSM=y
@@ -353,6 +388,7 @@ CONFIG_MMC_SDHI=y
 CONFIG_MMC_DW=y
 CONFIG_MMC_DW_EXYNOS=y
 CONFIG_MMC_DW_K3=y
+CONFIG_MMC_DW_ROCKCHIP=y
 CONFIG_MMC_SUNXI=y
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=y
@@ -368,11 +404,13 @@ CONFIG_RTC_DRV_DS3232=y
 CONFIG_RTC_DRV_EFI=y
 CONFIG_RTC_DRV_PL031=y
 CONFIG_RTC_DRV_SUN6I=y
+CONFIG_RTC_DRV_RK808=m
 CONFIG_RTC_DRV_TEGRA=y
 CONFIG_RTC_DRV_XGENE=y
 CONFIG_RTC_DRV_S3C=y
 CONFIG_DMADEVICES=y
 CONFIG_PL330_DMA=y
+CONFIG_DMA_BCM2835=m
 CONFIG_TEGRA20_APB_DMA=y
 CONFIG_QCOM_BAM_DMA=y
 CONFIG_QCOM_HIDMA_MGMT=y
@@ -388,26 +426,39 @@ CONFIG_XEN_GRANT_DEV_ALLOC=y
 CONFIG_COMMON_CLK_SCPI=y
 CONFIG_COMMON_CLK_CS2000_CP=y
 CONFIG_COMMON_CLK_S2MPS11=y
+CONFIG_COMMON_CLK_PWM=y
+CONFIG_COMMON_CLK_RK808=y
 CONFIG_CLK_QORIQ=y
 CONFIG_COMMON_CLK_QCOM=y
 CONFIG_MSM_GCC_8916=y
+CONFIG_MSM_GCC_8994=y
 CONFIG_MSM_MMCC_8996=y
 CONFIG_HWSPINLOCK_QCOM=y
 CONFIG_MAILBOX=y
 CONFIG_ARM_MHU=y
+CONFIG_PLATFORM_MHU=y
+CONFIG_BCM2835_MBOX=y
 CONFIG_HI6220_MBOX=y
 CONFIG_ARM_SMMU=y
+CONFIG_RASPBERRYPI_POWER=y
 CONFIG_QCOM_SMEM=y
 CONFIG_QCOM_SMD=y
 CONFIG_QCOM_SMD_RPM=y
+CONFIG_ROCKCHIP_PM_DOMAINS=y
 CONFIG_ARCH_TEGRA_132_SOC=y
 CONFIG_ARCH_TEGRA_210_SOC=y
+CONFIG_ARCH_TEGRA_186_SOC=y
 CONFIG_EXTCON_USB_GPIO=y
 CONFIG_PWM=y
+CONFIG_PWM_BCM2835=m
+CONFIG_PWM_ROCKCHIP=y
 CONFIG_PWM_TEGRA=m
+CONFIG_PWM_MESON=m
 CONFIG_COMMON_RESET_HI6220=y
 CONFIG_PHY_RCAR_GEN3_USB2=y
 CONFIG_PHY_HI6220_USB=y
+CONFIG_PHY_ROCKCHIP_INNO_USB2=y
+CONFIG_PHY_ROCKCHIP_EMMC=y
 CONFIG_PHY_XGENE=y
 CONFIG_PHY_TEGRA_XUSB=y
 CONFIG_ARM_SCPI_PROTOCOL=y
@@ -415,6 +466,7 @@ CONFIG_ACPI=y
 CONFIG_IIO=y
 CONFIG_EXYNOS_ADC=y
 CONFIG_PWM_SAMSUNG=y
+CONFIG_RASPBERRYPI_FIRMWARE=y
 CONFIG_EXT2_FS=y
 CONFIG_EXT3_FS=y
 CONFIG_EXT4_FS_POSIX_ACL=y
diff --git a/arch/arm64/crypto/.gitignore b/arch/arm64/crypto/.gitignore
new file mode 100644
index 000000000000..879df8781ed5
--- /dev/null
+++ b/arch/arm64/crypto/.gitignore
@@ -0,0 +1,2 @@
+sha256-core.S
+sha512-core.S
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 2cf32e9887e1..450a85df041a 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -8,6 +8,14 @@ menuconfig ARM64_CRYPTO
 
 if ARM64_CRYPTO
 
+config CRYPTO_SHA256_ARM64
+	tristate "SHA-224/SHA-256 digest algorithm for arm64"
+	select CRYPTO_HASH
+
+config CRYPTO_SHA512_ARM64
+	tristate "SHA-384/SHA-512 digest algorithm for arm64"
+	select CRYPTO_HASH
+
 config CRYPTO_SHA1_ARM64_CE
 	tristate "SHA-1 digest algorithm (ARMv8 Crypto Extensions)"
 	depends on ARM64 && KERNEL_MODE_NEON
@@ -23,6 +31,16 @@ config CRYPTO_GHASH_ARM64_CE
 	depends on ARM64 && KERNEL_MODE_NEON
 	select CRYPTO_HASH
 
+config CRYPTO_CRCT10DIF_ARM64_CE
+	tristate "CRCT10DIF digest algorithm using PMULL instructions"
+	depends on KERNEL_MODE_NEON && CRC_T10DIF
+	select CRYPTO_HASH
+
+config CRYPTO_CRC32_ARM64_CE
+	tristate "CRC32 and CRC32C digest algorithms using PMULL instructions"
+	depends on KERNEL_MODE_NEON && CRC32
+	select CRYPTO_HASH
+
 config CRYPTO_AES_ARM64_CE
 	tristate "AES core cipher using ARMv8 Crypto Extensions"
 	depends on ARM64 && KERNEL_MODE_NEON
@@ -40,17 +58,18 @@ config CRYPTO_AES_ARM64_CE_BLK
 	depends on ARM64 && KERNEL_MODE_NEON
 	select CRYPTO_BLKCIPHER
 	select CRYPTO_AES_ARM64_CE
-	select CRYPTO_ABLK_HELPER
+	select CRYPTO_SIMD
 
 config CRYPTO_AES_ARM64_NEON_BLK
 	tristate "AES in ECB/CBC/CTR/XTS modes using NEON instructions"
 	depends on ARM64 && KERNEL_MODE_NEON
 	select CRYPTO_BLKCIPHER
 	select CRYPTO_AES
-	select CRYPTO_ABLK_HELPER
+	select CRYPTO_SIMD
 
 config CRYPTO_CRC32_ARM64
 	tristate "CRC32 and CRC32C using optional ARMv8 instructions"
 	depends on ARM64
 	select CRYPTO_HASH
+
 endif
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index abb79b3cfcfe..aa8888d7b744 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -17,6 +17,12 @@ sha2-ce-y := sha2-ce-glue.o sha2-ce-core.o
 obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o
 ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
 
+obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM64_CE) += crct10dif-ce.o
+crct10dif-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o
+
+obj-$(CONFIG_CRYPTO_CRC32_ARM64_CE) += crc32-ce.o
+crc32-ce-y:= crc32-ce-core.o crc32-ce-glue.o
+
 obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o
 CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto
 
@@ -29,6 +35,12 @@ aes-ce-blk-y := aes-glue-ce.o aes-ce.o
 obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o
 aes-neon-blk-y := aes-glue-neon.o aes-neon.o
 
+obj-$(CONFIG_CRYPTO_SHA256_ARM64) += sha256-arm64.o
+sha256-arm64-y := sha256-glue.o sha256-core.o
+
+obj-$(CONFIG_CRYPTO_SHA512_ARM64) += sha512-arm64.o
+sha512-arm64-y := sha512-glue.o sha512-core.o
+
 AFLAGS_aes-ce.o		:= -DINTERLEAVE=4
 AFLAGS_aes-neon.o	:= -DINTERLEAVE=4
 
@@ -40,3 +52,14 @@ CFLAGS_crc32-arm64.o	:= -mcpu=generic+crc
 
 $(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE
 	$(call if_changed_rule,cc_o_c)
+
+quiet_cmd_perlasm = PERLASM $@
+      cmd_perlasm = $(PERL) $(<) void $(@)
+
+$(src)/sha256-core.S_shipped: $(src)/sha512-armv8.pl
+	$(call cmd,perlasm)
+
+$(src)/sha512-core.S_shipped: $(src)/sha512-armv8.pl
+	$(call cmd,perlasm)
+
+.PRECIOUS: $(obj)/sha256-core.S $(obj)/sha512-core.S
diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S
index a2a7fbcacc14..3363560c79b7 100644
--- a/arch/arm64/crypto/aes-ce-ccm-core.S
+++ b/arch/arm64/crypto/aes-ce-ccm-core.S
@@ -9,6 +9,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/assembler.h>
 
 	.text
 	.arch	armv8-a+crypto
@@ -19,7 +20,7 @@
 	 */
 ENTRY(ce_aes_ccm_auth_data)
 	ldr	w8, [x3]			/* leftover from prev round? */
-	ld1	{v0.2d}, [x0]			/* load mac */
+	ld1	{v0.16b}, [x0]			/* load mac */
 	cbz	w8, 1f
 	sub	w8, w8, #16
 	eor	v1.16b, v1.16b, v1.16b
@@ -31,7 +32,7 @@ ENTRY(ce_aes_ccm_auth_data)
 	beq	8f				/* out of input? */
 	cbnz	w8, 0b
 	eor	v0.16b, v0.16b, v1.16b
-1:	ld1	{v3.2d}, [x4]			/* load first round key */
+1:	ld1	{v3.16b}, [x4]			/* load first round key */
 	prfm	pldl1strm, [x1]
 	cmp	w5, #12				/* which key size? */
 	add	x6, x4, #16
@@ -41,17 +42,17 @@ ENTRY(ce_aes_ccm_auth_data)
 	mov	v5.16b, v3.16b
 	b	4f
 2:	mov	v4.16b, v3.16b
-	ld1	{v5.2d}, [x6], #16		/* load 2nd round key */
+	ld1	{v5.16b}, [x6], #16		/* load 2nd round key */
 3:	aese	v0.16b, v4.16b
 	aesmc	v0.16b, v0.16b
-4:	ld1	{v3.2d}, [x6], #16		/* load next round key */
+4:	ld1	{v3.16b}, [x6], #16		/* load next round key */
 	aese	v0.16b, v5.16b
 	aesmc	v0.16b, v0.16b
-5:	ld1	{v4.2d}, [x6], #16		/* load next round key */
+5:	ld1	{v4.16b}, [x6], #16		/* load next round key */
 	subs	w7, w7, #3
 	aese	v0.16b, v3.16b
 	aesmc	v0.16b, v0.16b
-	ld1	{v5.2d}, [x6], #16		/* load next round key */
+	ld1	{v5.16b}, [x6], #16		/* load next round key */
 	bpl	3b
 	aese	v0.16b, v4.16b
 	subs	w2, w2, #16			/* last data? */
@@ -60,7 +61,7 @@ ENTRY(ce_aes_ccm_auth_data)
 	ld1	{v1.16b}, [x1], #16		/* load next input block */
 	eor	v0.16b, v0.16b, v1.16b		/* xor with mac */
 	bne	1b
-6:	st1	{v0.2d}, [x0]			/* store mac */
+6:	st1	{v0.16b}, [x0]			/* store mac */
 	beq	10f
 	adds	w2, w2, #16
 	beq	10f
@@ -79,7 +80,7 @@ ENTRY(ce_aes_ccm_auth_data)
 	adds	w7, w7, #1
 	bne	9b
 	eor	v0.16b, v0.16b, v1.16b
-	st1	{v0.2d}, [x0]
+	st1	{v0.16b}, [x0]
 10:	str	w8, [x3]
 	ret
 ENDPROC(ce_aes_ccm_auth_data)
@@ -89,27 +90,27 @@ ENDPROC(ce_aes_ccm_auth_data)
 	 * 			 u32 rounds);
 	 */
 ENTRY(ce_aes_ccm_final)
-	ld1	{v3.2d}, [x2], #16		/* load first round key */
-	ld1	{v0.2d}, [x0]			/* load mac */
+	ld1	{v3.16b}, [x2], #16		/* load first round key */
+	ld1	{v0.16b}, [x0]			/* load mac */
 	cmp	w3, #12				/* which key size? */
 	sub	w3, w3, #2			/* modified # of rounds */
-	ld1	{v1.2d}, [x1]			/* load 1st ctriv */
+	ld1	{v1.16b}, [x1]			/* load 1st ctriv */
 	bmi	0f
 	bne	3f
 	mov	v5.16b, v3.16b
 	b	2f
 0:	mov	v4.16b, v3.16b
-1:	ld1	{v5.2d}, [x2], #16		/* load next round key */
+1:	ld1	{v5.16b}, [x2], #16		/* load next round key */
 	aese	v0.16b, v4.16b
 	aesmc	v0.16b, v0.16b
 	aese	v1.16b, v4.16b
 	aesmc	v1.16b, v1.16b
-2:	ld1	{v3.2d}, [x2], #16		/* load next round key */
+2:	ld1	{v3.16b}, [x2], #16		/* load next round key */
 	aese	v0.16b, v5.16b
 	aesmc	v0.16b, v0.16b
 	aese	v1.16b, v5.16b
 	aesmc	v1.16b, v1.16b
-3:	ld1	{v4.2d}, [x2], #16		/* load next round key */
+3:	ld1	{v4.16b}, [x2], #16		/* load next round key */
 	subs	w3, w3, #3
 	aese	v0.16b, v3.16b
 	aesmc	v0.16b, v0.16b
@@ -120,47 +121,47 @@ ENTRY(ce_aes_ccm_final)
 	aese	v1.16b, v4.16b
 	/* final round key cancels out */
 	eor	v0.16b, v0.16b, v1.16b		/* en-/decrypt the mac */
-	st1	{v0.2d}, [x0]			/* store result */
+	st1	{v0.16b}, [x0]			/* store result */
 	ret
 ENDPROC(ce_aes_ccm_final)
 
 	.macro	aes_ccm_do_crypt,enc
 	ldr	x8, [x6, #8]			/* load lower ctr */
-	ld1	{v0.2d}, [x5]			/* load mac */
-	rev	x8, x8				/* keep swabbed ctr in reg */
+	ld1	{v0.16b}, [x5]			/* load mac */
+CPU_LE(	rev	x8, x8			)	/* keep swabbed ctr in reg */
 0:	/* outer loop */
-	ld1	{v1.1d}, [x6]			/* load upper ctr */
+	ld1	{v1.8b}, [x6]			/* load upper ctr */
 	prfm	pldl1strm, [x1]
 	add	x8, x8, #1
 	rev	x9, x8
 	cmp	w4, #12				/* which key size? */
 	sub	w7, w4, #2			/* get modified # of rounds */
 	ins	v1.d[1], x9			/* no carry in lower ctr */
-	ld1	{v3.2d}, [x3]			/* load first round key */
+	ld1	{v3.16b}, [x3]			/* load first round key */
 	add	x10, x3, #16
 	bmi	1f
 	bne	4f
 	mov	v5.16b, v3.16b
 	b	3f
 1:	mov	v4.16b, v3.16b
-	ld1	{v5.2d}, [x10], #16		/* load 2nd round key */
+	ld1	{v5.16b}, [x10], #16		/* load 2nd round key */
 2:	/* inner loop: 3 rounds, 2x interleaved */
 	aese	v0.16b, v4.16b
 	aesmc	v0.16b, v0.16b
 	aese	v1.16b, v4.16b
 	aesmc	v1.16b, v1.16b
-3:	ld1	{v3.2d}, [x10], #16		/* load next round key */
+3:	ld1	{v3.16b}, [x10], #16		/* load next round key */
 	aese	v0.16b, v5.16b
 	aesmc	v0.16b, v0.16b
 	aese	v1.16b, v5.16b
 	aesmc	v1.16b, v1.16b
-4:	ld1	{v4.2d}, [x10], #16		/* load next round key */
+4:	ld1	{v4.16b}, [x10], #16		/* load next round key */
 	subs	w7, w7, #3
 	aese	v0.16b, v3.16b
 	aesmc	v0.16b, v0.16b
 	aese	v1.16b, v3.16b
 	aesmc	v1.16b, v1.16b
-	ld1	{v5.2d}, [x10], #16		/* load next round key */
+	ld1	{v5.16b}, [x10], #16		/* load next round key */
 	bpl	2b
 	aese	v0.16b, v4.16b
 	aese	v1.16b, v4.16b
@@ -177,14 +178,14 @@ ENDPROC(ce_aes_ccm_final)
 	eor	v0.16b, v0.16b, v2.16b		/* xor mac with pt ^ rk[last] */
 	st1	{v1.16b}, [x0], #16		/* write output block */
 	bne	0b
-	rev	x8, x8
-	st1	{v0.2d}, [x5]			/* store mac */
+CPU_LE(	rev	x8, x8			)
+	st1	{v0.16b}, [x5]			/* store mac */
 	str	x8, [x6, #8]			/* store lsb end of ctr (BE) */
 5:	ret
 
 6:	eor	v0.16b, v0.16b, v5.16b		/* final round mac */
 	eor	v1.16b, v1.16b, v5.16b		/* final round enc */
-	st1	{v0.2d}, [x5]			/* store mac */
+	st1	{v0.16b}, [x5]			/* store mac */
 	add	w2, w2, #16			/* process partial tail block */
 7:	ldrb	w9, [x1], #1			/* get 1 byte of input */
 	umov	w6, v1.b[0]			/* get top crypted ctr byte */
diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c
index f4bf2f2a014c..cc5515dac74a 100644
--- a/arch/arm64/crypto/aes-ce-ccm-glue.c
+++ b/arch/arm64/crypto/aes-ce-ccm-glue.c
@@ -11,9 +11,9 @@
 #include <asm/neon.h>
 #include <asm/unaligned.h>
 #include <crypto/aes.h>
-#include <crypto/algapi.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
 #include <linux/module.h>
 
 #include "aes-ce-setkey.h"
@@ -149,12 +149,7 @@ static int ccm_encrypt(struct aead_request *req)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
-	struct blkcipher_desc desc = { .info = req->iv };
-	struct blkcipher_walk walk;
-	struct scatterlist srcbuf[2];
-	struct scatterlist dstbuf[2];
-	struct scatterlist *src;
-	struct scatterlist *dst;
+	struct skcipher_walk walk;
 	u8 __aligned(8) mac[AES_BLOCK_SIZE];
 	u8 buf[AES_BLOCK_SIZE];
 	u32 len = req->cryptlen;
@@ -172,27 +167,19 @@ static int ccm_encrypt(struct aead_request *req)
 	/* preserve the original iv for the final round */
 	memcpy(buf, req->iv, AES_BLOCK_SIZE);
 
-	src = scatterwalk_ffwd(srcbuf, req->src, req->assoclen);
-	dst = src;
-	if (req->src != req->dst)
-		dst = scatterwalk_ffwd(dstbuf, req->dst, req->assoclen);
-
-	blkcipher_walk_init(&walk, dst, src, len);
-	err = blkcipher_aead_walk_virt_block(&desc, &walk, aead,
-					     AES_BLOCK_SIZE);
+	err = skcipher_walk_aead_encrypt(&walk, req, true);
 
 	while (walk.nbytes) {
 		u32 tail = walk.nbytes % AES_BLOCK_SIZE;
 
-		if (walk.nbytes == len)
+		if (walk.nbytes == walk.total)
 			tail = 0;
 
 		ce_aes_ccm_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				   walk.nbytes - tail, ctx->key_enc,
 				   num_rounds(ctx), mac, walk.iv);
 
-		len -= walk.nbytes - tail;
-		err = blkcipher_walk_done(&desc, &walk, tail);
+		err = skcipher_walk_done(&walk, tail);
 	}
 	if (!err)
 		ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx));
@@ -203,7 +190,7 @@ static int ccm_encrypt(struct aead_request *req)
 		return err;
 
 	/* copy authtag to end of dst */
-	scatterwalk_map_and_copy(mac, dst, req->cryptlen,
+	scatterwalk_map_and_copy(mac, req->dst, req->assoclen + req->cryptlen,
 				 crypto_aead_authsize(aead), 1);
 
 	return 0;
@@ -214,12 +201,7 @@ static int ccm_decrypt(struct aead_request *req)
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
 	unsigned int authsize = crypto_aead_authsize(aead);
-	struct blkcipher_desc desc = { .info = req->iv };
-	struct blkcipher_walk walk;
-	struct scatterlist srcbuf[2];
-	struct scatterlist dstbuf[2];
-	struct scatterlist *src;
-	struct scatterlist *dst;
+	struct skcipher_walk walk;
 	u8 __aligned(8) mac[AES_BLOCK_SIZE];
 	u8 buf[AES_BLOCK_SIZE];
 	u32 len = req->cryptlen - authsize;
@@ -237,27 +219,19 @@ static int ccm_decrypt(struct aead_request *req)
 	/* preserve the original iv for the final round */
 	memcpy(buf, req->iv, AES_BLOCK_SIZE);
 
-	src = scatterwalk_ffwd(srcbuf, req->src, req->assoclen);
-	dst = src;
-	if (req->src != req->dst)
-		dst = scatterwalk_ffwd(dstbuf, req->dst, req->assoclen);
-
-	blkcipher_walk_init(&walk, dst, src, len);
-	err = blkcipher_aead_walk_virt_block(&desc, &walk, aead,
-					     AES_BLOCK_SIZE);
+	err = skcipher_walk_aead_decrypt(&walk, req, true);
 
 	while (walk.nbytes) {
 		u32 tail = walk.nbytes % AES_BLOCK_SIZE;
 
-		if (walk.nbytes == len)
+		if (walk.nbytes == walk.total)
 			tail = 0;
 
 		ce_aes_ccm_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				   walk.nbytes - tail, ctx->key_enc,
 				   num_rounds(ctx), mac, walk.iv);
 
-		len -= walk.nbytes - tail;
-		err = blkcipher_walk_done(&desc, &walk, tail);
+		err = skcipher_walk_done(&walk, tail);
 	}
 	if (!err)
 		ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx));
@@ -268,7 +242,8 @@ static int ccm_decrypt(struct aead_request *req)
 		return err;
 
 	/* compare calculated auth tag with the stored one */
-	scatterwalk_map_and_copy(buf, src, req->cryptlen - authsize,
+	scatterwalk_map_and_copy(buf, req->src,
+				 req->assoclen + req->cryptlen - authsize,
 				 authsize, 0);
 
 	if (crypto_memneq(mac, buf, authsize))
@@ -287,6 +262,7 @@ static struct aead_alg ccm_aes_alg = {
 		.cra_module		= THIS_MODULE,
 	},
 	.ivsize		= AES_BLOCK_SIZE,
+	.chunksize	= AES_BLOCK_SIZE,
 	.maxauthsize	= AES_BLOCK_SIZE,
 	.setkey		= ccm_setkey,
 	.setauthsize	= ccm_setauthsize,
diff --git a/arch/arm64/crypto/aes-ce-cipher.c b/arch/arm64/crypto/aes-ce-cipher.c
index f7bd9bf0bbb3..50d9fe11d0c8 100644
--- a/arch/arm64/crypto/aes-ce-cipher.c
+++ b/arch/arm64/crypto/aes-ce-cipher.c
@@ -47,24 +47,24 @@ static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
 	kernel_neon_begin_partial(4);
 
 	__asm__("	ld1	{v0.16b}, %[in]			;"
-		"	ld1	{v1.2d}, [%[key]], #16		;"
+		"	ld1	{v1.16b}, [%[key]], #16		;"
 		"	cmp	%w[rounds], #10			;"
 		"	bmi	0f				;"
 		"	bne	3f				;"
 		"	mov	v3.16b, v1.16b			;"
 		"	b	2f				;"
 		"0:	mov	v2.16b, v1.16b			;"
-		"	ld1	{v3.2d}, [%[key]], #16		;"
+		"	ld1	{v3.16b}, [%[key]], #16		;"
 		"1:	aese	v0.16b, v2.16b			;"
 		"	aesmc	v0.16b, v0.16b			;"
-		"2:	ld1	{v1.2d}, [%[key]], #16		;"
+		"2:	ld1	{v1.16b}, [%[key]], #16		;"
 		"	aese	v0.16b, v3.16b			;"
 		"	aesmc	v0.16b, v0.16b			;"
-		"3:	ld1	{v2.2d}, [%[key]], #16		;"
+		"3:	ld1	{v2.16b}, [%[key]], #16		;"
 		"	subs	%w[rounds], %w[rounds], #3	;"
 		"	aese	v0.16b, v1.16b			;"
 		"	aesmc	v0.16b, v0.16b			;"
-		"	ld1	{v3.2d}, [%[key]], #16		;"
+		"	ld1	{v3.16b}, [%[key]], #16		;"
 		"	bpl	1b				;"
 		"	aese	v0.16b, v2.16b			;"
 		"	eor	v0.16b, v0.16b, v3.16b		;"
@@ -92,24 +92,24 @@ static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
 	kernel_neon_begin_partial(4);
 
 	__asm__("	ld1	{v0.16b}, %[in]			;"
-		"	ld1	{v1.2d}, [%[key]], #16		;"
+		"	ld1	{v1.16b}, [%[key]], #16		;"
 		"	cmp	%w[rounds], #10			;"
 		"	bmi	0f				;"
 		"	bne	3f				;"
 		"	mov	v3.16b, v1.16b			;"
 		"	b	2f				;"
 		"0:	mov	v2.16b, v1.16b			;"
-		"	ld1	{v3.2d}, [%[key]], #16		;"
+		"	ld1	{v3.16b}, [%[key]], #16		;"
 		"1:	aesd	v0.16b, v2.16b			;"
 		"	aesimc	v0.16b, v0.16b			;"
-		"2:	ld1	{v1.2d}, [%[key]], #16		;"
+		"2:	ld1	{v1.16b}, [%[key]], #16		;"
 		"	aesd	v0.16b, v3.16b			;"
 		"	aesimc	v0.16b, v0.16b			;"
-		"3:	ld1	{v2.2d}, [%[key]], #16		;"
+		"3:	ld1	{v2.16b}, [%[key]], #16		;"
 		"	subs	%w[rounds], %w[rounds], #3	;"
 		"	aesd	v0.16b, v1.16b			;"
 		"	aesimc	v0.16b, v0.16b			;"
-		"	ld1	{v3.2d}, [%[key]], #16		;"
+		"	ld1	{v3.16b}, [%[key]], #16		;"
 		"	bpl	1b				;"
 		"	aesd	v0.16b, v2.16b			;"
 		"	eor	v0.16b, v0.16b, v3.16b		;"
@@ -173,7 +173,12 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 		u32 *rki = ctx->key_enc + (i * kwords);
 		u32 *rko = rki + kwords;
 
+#ifndef CONFIG_CPU_BIG_ENDIAN
 		rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0];
+#else
+		rko[0] = rol32(aes_sub(rki[kwords - 1]), 8) ^ (rcon[i] << 24) ^
+			 rki[0];
+#endif
 		rko[1] = rko[0] ^ rki[1];
 		rko[2] = rko[1] ^ rki[2];
 		rko[3] = rko[2] ^ rki[3];
diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S
index 78f3cfe92c08..b46093d567e5 100644
--- a/arch/arm64/crypto/aes-ce.S
+++ b/arch/arm64/crypto/aes-ce.S
@@ -10,6 +10,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/assembler.h>
 
 #define AES_ENTRY(func)		ENTRY(ce_ ## func)
 #define AES_ENDPROC(func)	ENDPROC(ce_ ## func)
diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index 6b2aa0fd6cd0..4e3f8adb1793 100644
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -11,8 +11,8 @@
 #include <asm/neon.h>
 #include <asm/hwcap.h>
 #include <crypto/aes.h>
-#include <crypto/ablk_helper.h>
-#include <crypto/algapi.h>
+#include <crypto/internal/simd.h>
+#include <crypto/internal/skcipher.h>
 #include <linux/module.h>
 #include <linux/cpufeature.h>
 #include <crypto/xts.h>
@@ -80,13 +80,19 @@ struct crypto_aes_xts_ctx {
 	struct crypto_aes_ctx __aligned(8) key2;
 };
 
-static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
+static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
+			       unsigned int key_len)
+{
+	return aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);
+}
+
+static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
 		       unsigned int key_len)
 {
-	struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int ret;
 
-	ret = xts_check_key(tfm, in_key, key_len);
+	ret = xts_verify_key(tfm, in_key, key_len);
 	if (ret)
 		return ret;
 
@@ -97,111 +103,101 @@ static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 	if (!ret)
 		return 0;
 
-	tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+	crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 	return -EINVAL;
 }
 
-static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int ecb_encrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int err, first, rounds = 6 + ctx->key_length / 4;
-	struct blkcipher_walk walk;
+	struct skcipher_walk walk;
 	unsigned int blocks;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 		aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				(u8 *)ctx->key_enc, rounds, blocks, first);
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	kernel_neon_end();
 	return err;
 }
 
-static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int ecb_decrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int err, first, rounds = 6 + ctx->key_length / 4;
-	struct blkcipher_walk walk;
+	struct skcipher_walk walk;
 	unsigned int blocks;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 		aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				(u8 *)ctx->key_dec, rounds, blocks, first);
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	kernel_neon_end();
 	return err;
 }
 
-static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int cbc_encrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int err, first, rounds = 6 + ctx->key_length / 4;
-	struct blkcipher_walk walk;
+	struct skcipher_walk walk;
 	unsigned int blocks;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 		aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				(u8 *)ctx->key_enc, rounds, blocks, walk.iv,
 				first);
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	kernel_neon_end();
 	return err;
 }
 
-static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int cbc_decrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int err, first, rounds = 6 + ctx->key_length / 4;
-	struct blkcipher_walk walk;
+	struct skcipher_walk walk;
 	unsigned int blocks;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 		aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				(u8 *)ctx->key_dec, rounds, blocks, walk.iv,
 				first);
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	kernel_neon_end();
 	return err;
 }
 
-static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int ctr_encrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int err, first, rounds = 6 + ctx->key_length / 4;
-	struct blkcipher_walk walk;
+	struct skcipher_walk walk;
 	int blocks;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	first = 1;
 	kernel_neon_begin();
@@ -209,17 +205,14 @@ static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 		aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				(u8 *)ctx->key_enc, rounds, blocks, walk.iv,
 				first);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 		first = 0;
-		nbytes -= blocks * AES_BLOCK_SIZE;
-		if (nbytes && nbytes == walk.nbytes % AES_BLOCK_SIZE)
-			break;
-		err = blkcipher_walk_done(desc, &walk,
-					  walk.nbytes % AES_BLOCK_SIZE);
 	}
-	if (walk.nbytes % AES_BLOCK_SIZE) {
-		u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
-		u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
+	if (walk.nbytes) {
 		u8 __aligned(8) tail[AES_BLOCK_SIZE];
+		unsigned int nbytes = walk.nbytes;
+		u8 *tdst = walk.dst.virt.addr;
+		u8 *tsrc = walk.src.virt.addr;
 
 		/*
 		 * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need
@@ -230,227 +223,169 @@ static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 		aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc, rounds,
 				blocks, walk.iv, first);
 		memcpy(tdst, tail, nbytes);
-		err = blkcipher_walk_done(desc, &walk, 0);
+		err = skcipher_walk_done(&walk, 0);
 	}
 	kernel_neon_end();
 
 	return err;
 }
 
-static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int xts_encrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int err, first, rounds = 6 + ctx->key1.key_length / 4;
-	struct blkcipher_walk walk;
+	struct skcipher_walk walk;
 	unsigned int blocks;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 		aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				(u8 *)ctx->key1.key_enc, rounds, blocks,
 				(u8 *)ctx->key2.key_enc, walk.iv, first);
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	kernel_neon_end();
 
 	return err;
 }
 
-static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int xts_decrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int err, first, rounds = 6 + ctx->key1.key_length / 4;
-	struct blkcipher_walk walk;
+	struct skcipher_walk walk;
 	unsigned int blocks;
 
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_neon_begin();
 	for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 		aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 				(u8 *)ctx->key1.key_dec, rounds, blocks,
 				(u8 *)ctx->key2.key_enc, walk.iv, first);
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
+		err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
 	}
 	kernel_neon_end();
 
 	return err;
 }
 
-static struct crypto_alg aes_algs[] = { {
-	.cra_name		= "__ecb-aes-" MODE,
-	.cra_driver_name	= "__driver-ecb-aes-" MODE,
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_blkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= 0,
-		.setkey		= aes_setkey,
-		.encrypt	= ecb_encrypt,
-		.decrypt	= ecb_decrypt,
+static struct skcipher_alg aes_algs[] = { {
+	.base = {
+		.cra_name		= "__ecb(aes)",
+		.cra_driver_name	= "__ecb-aes-" MODE,
+		.cra_priority		= PRIO,
+		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_blocksize		= AES_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
+		.cra_alignmask		= 7,
+		.cra_module		= THIS_MODULE,
 	},
+	.min_keysize	= AES_MIN_KEY_SIZE,
+	.max_keysize	= AES_MAX_KEY_SIZE,
+	.setkey		= skcipher_aes_setkey,
+	.encrypt	= ecb_encrypt,
+	.decrypt	= ecb_decrypt,
 }, {
-	.cra_name		= "__cbc-aes-" MODE,
-	.cra_driver_name	= "__driver-cbc-aes-" MODE,
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_blkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= aes_setkey,
-		.encrypt	= cbc_encrypt,
-		.decrypt	= cbc_decrypt,
+	.base = {
+		.cra_name		= "__cbc(aes)",
+		.cra_driver_name	= "__cbc-aes-" MODE,
+		.cra_priority		= PRIO,
+		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_blocksize		= AES_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
+		.cra_alignmask		= 7,
+		.cra_module		= THIS_MODULE,
 	},
+	.min_keysize	= AES_MIN_KEY_SIZE,
+	.max_keysize	= AES_MAX_KEY_SIZE,
+	.ivsize		= AES_BLOCK_SIZE,
+	.setkey		= skcipher_aes_setkey,
+	.encrypt	= cbc_encrypt,
+	.decrypt	= cbc_decrypt,
 }, {
-	.cra_name		= "__ctr-aes-" MODE,
-	.cra_driver_name	= "__driver-ctr-aes-" MODE,
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= 1,
-	.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_blkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= aes_setkey,
-		.encrypt	= ctr_encrypt,
-		.decrypt	= ctr_encrypt,
+	.base = {
+		.cra_name		= "__ctr(aes)",
+		.cra_driver_name	= "__ctr-aes-" MODE,
+		.cra_priority		= PRIO,
+		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_blocksize		= 1,
+		.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
+		.cra_alignmask		= 7,
+		.cra_module		= THIS_MODULE,
 	},
+	.min_keysize	= AES_MIN_KEY_SIZE,
+	.max_keysize	= AES_MAX_KEY_SIZE,
+	.ivsize		= AES_BLOCK_SIZE,
+	.chunksize	= AES_BLOCK_SIZE,
+	.setkey		= skcipher_aes_setkey,
+	.encrypt	= ctr_encrypt,
+	.decrypt	= ctr_encrypt,
 }, {
-	.cra_name		= "__xts-aes-" MODE,
-	.cra_driver_name	= "__driver-xts-aes-" MODE,
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct crypto_aes_xts_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_blkcipher = {
-		.min_keysize	= 2 * AES_MIN_KEY_SIZE,
-		.max_keysize	= 2 * AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= xts_set_key,
-		.encrypt	= xts_encrypt,
-		.decrypt	= xts_decrypt,
+	.base = {
+		.cra_name		= "__xts(aes)",
+		.cra_driver_name	= "__xts-aes-" MODE,
+		.cra_priority		= PRIO,
+		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_blocksize		= AES_BLOCK_SIZE,
+		.cra_ctxsize		= sizeof(struct crypto_aes_xts_ctx),
+		.cra_alignmask		= 7,
+		.cra_module		= THIS_MODULE,
 	},
-}, {
-	.cra_name		= "ecb(aes)",
-	.cra_driver_name	= "ecb-aes-" MODE,
-	.cra_priority		= PRIO,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_ablkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= 0,
-		.setkey		= ablk_set_key,
-		.encrypt	= ablk_encrypt,
-		.decrypt	= ablk_decrypt,
-	}
-}, {
-	.cra_name		= "cbc(aes)",
-	.cra_driver_name	= "cbc-aes-" MODE,
-	.cra_priority		= PRIO,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_ablkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= ablk_set_key,
-		.encrypt	= ablk_encrypt,
-		.decrypt	= ablk_decrypt,
-	}
-}, {
-	.cra_name		= "ctr(aes)",
-	.cra_driver_name	= "ctr-aes-" MODE,
-	.cra_priority		= PRIO,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= 1,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_ablkcipher = {
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= ablk_set_key,
-		.encrypt	= ablk_encrypt,
-		.decrypt	= ablk_decrypt,
-	}
-}, {
-	.cra_name		= "xts(aes)",
-	.cra_driver_name	= "xts-aes-" MODE,
-	.cra_priority		= PRIO,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 7,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_ablkcipher = {
-		.min_keysize	= 2 * AES_MIN_KEY_SIZE,
-		.max_keysize	= 2 * AES_MAX_KEY_SIZE,
-		.ivsize		= AES_BLOCK_SIZE,
-		.setkey		= ablk_set_key,
-		.encrypt	= ablk_encrypt,
-		.decrypt	= ablk_decrypt,
-	}
+	.min_keysize	= 2 * AES_MIN_KEY_SIZE,
+	.max_keysize	= 2 * AES_MAX_KEY_SIZE,
+	.ivsize		= AES_BLOCK_SIZE,
+	.setkey		= xts_set_key,
+	.encrypt	= xts_encrypt,
+	.decrypt	= xts_decrypt,
 } };
 
-static int __init aes_init(void)
+static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
+
+static void aes_exit(void)
 {
-	return crypto_register_algs(aes_algs, ARRAY_SIZE(aes_algs));
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++)
+		simd_skcipher_free(aes_simd_algs[i]);
+
+	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
 }
 
-static void __exit aes_exit(void)
+static int __init aes_init(void)
 {
-	crypto_unregister_algs(aes_algs, ARRAY_SIZE(aes_algs));
+	struct simd_skcipher_alg *simd;
+	const char *basename;
+	const char *algname;
+	const char *drvname;
+	int err;
+	int i;
+
+	err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
+	if (err)
+		return err;
+
+	for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
+		algname = aes_algs[i].base.cra_name + 2;
+		drvname = aes_algs[i].base.cra_driver_name + 2;
+		basename = aes_algs[i].base.cra_driver_name;
+		simd = simd_skcipher_create_compat(algname, drvname, basename);
+		err = PTR_ERR(simd);
+		if (IS_ERR(simd))
+			goto unregister_simds;
+
+		aes_simd_algs[i] = simd;
+	}
+
+	return 0;
+
+unregister_simds:
+	aes_exit();
+	return err;
 }
 
 #ifdef USE_V8_CRYPTO_EXTENSIONS
diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S
index f6e372c528eb..c53dbeae79f2 100644
--- a/arch/arm64/crypto/aes-modes.S
+++ b/arch/arm64/crypto/aes-modes.S
@@ -386,7 +386,8 @@ AES_ENDPROC(aes_ctr_encrypt)
 	.endm
 
 .Lxts_mul_x:
-	.word		1, 0, 0x87, 0
+CPU_LE(	.quad		1, 0x87		)
+CPU_BE(	.quad		0x87, 1		)
 
 AES_ENTRY(aes_xts_encrypt)
 	FRAME_PUSH
diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S
index b93170e1cc93..85f07ead7c5c 100644
--- a/arch/arm64/crypto/aes-neon.S
+++ b/arch/arm64/crypto/aes-neon.S
@@ -9,6 +9,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/assembler.h>
 
 #define AES_ENTRY(func)		ENTRY(neon_ ## func)
 #define AES_ENDPROC(func)	ENDPROC(neon_ ## func)
@@ -83,13 +84,13 @@
 	.endm
 
 	.macro		do_block, enc, in, rounds, rk, rkp, i
-	ld1		{v15.16b}, [\rk]
+	ld1		{v15.4s}, [\rk]
 	add		\rkp, \rk, #16
 	mov		\i, \rounds
 1111:	eor		\in\().16b, \in\().16b, v15.16b		/* ^round key */
 	tbl		\in\().16b, {\in\().16b}, v13.16b	/* ShiftRows */
 	sub_bytes	\in
-	ld1		{v15.16b}, [\rkp], #16
+	ld1		{v15.4s}, [\rkp], #16
 	subs		\i, \i, #1
 	beq		2222f
 	.if		\enc == 1
@@ -229,7 +230,7 @@
 	.endm
 
 	.macro		do_block_2x, enc, in0, in1 rounds, rk, rkp, i
-	ld1		{v15.16b}, [\rk]
+	ld1		{v15.4s}, [\rk]
 	add		\rkp, \rk, #16
 	mov		\i, \rounds
 1111:	eor		\in0\().16b, \in0\().16b, v15.16b	/* ^round key */
@@ -237,7 +238,7 @@
 	sub_bytes_2x	\in0, \in1
 	tbl		\in0\().16b, {\in0\().16b}, v13.16b	/* ShiftRows */
 	tbl		\in1\().16b, {\in1\().16b}, v13.16b	/* ShiftRows */
-	ld1		{v15.16b}, [\rkp], #16
+	ld1		{v15.4s}, [\rkp], #16
 	subs		\i, \i, #1
 	beq		2222f
 	.if		\enc == 1
@@ -254,7 +255,7 @@
 	.endm
 
 	.macro		do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i
-	ld1		{v15.16b}, [\rk]
+	ld1		{v15.4s}, [\rk]
 	add		\rkp, \rk, #16
 	mov		\i, \rounds
 1111:	eor		\in0\().16b, \in0\().16b, v15.16b	/* ^round key */
@@ -266,7 +267,7 @@
 	tbl		\in1\().16b, {\in1\().16b}, v13.16b	/* ShiftRows */
 	tbl		\in2\().16b, {\in2\().16b}, v13.16b	/* ShiftRows */
 	tbl		\in3\().16b, {\in3\().16b}, v13.16b	/* ShiftRows */
-	ld1		{v15.16b}, [\rkp], #16
+	ld1		{v15.4s}, [\rkp], #16
 	subs		\i, \i, #1
 	beq		2222f
 	.if		\enc == 1
@@ -306,12 +307,16 @@
 	.text
 	.align		4
 .LForward_ShiftRows:
-	.byte		0x0, 0x5, 0xa, 0xf, 0x4, 0x9, 0xe, 0x3
-	.byte		0x8, 0xd, 0x2, 0x7, 0xc, 0x1, 0x6, 0xb
+CPU_LE(	.byte		0x0, 0x5, 0xa, 0xf, 0x4, 0x9, 0xe, 0x3	)
+CPU_LE(	.byte		0x8, 0xd, 0x2, 0x7, 0xc, 0x1, 0x6, 0xb	)
+CPU_BE(	.byte		0xb, 0x6, 0x1, 0xc, 0x7, 0x2, 0xd, 0x8	)
+CPU_BE(	.byte		0x3, 0xe, 0x9, 0x4, 0xf, 0xa, 0x5, 0x0	)
 
 .LReverse_ShiftRows:
-	.byte		0x0, 0xd, 0xa, 0x7, 0x4, 0x1, 0xe, 0xb
-	.byte		0x8, 0x5, 0x2, 0xf, 0xc, 0x9, 0x6, 0x3
+CPU_LE(	.byte		0x0, 0xd, 0xa, 0x7, 0x4, 0x1, 0xe, 0xb	)
+CPU_LE(	.byte		0x8, 0x5, 0x2, 0xf, 0xc, 0x9, 0x6, 0x3	)
+CPU_BE(	.byte		0x3, 0x6, 0x9, 0xc, 0xf, 0x2, 0x5, 0x8	)
+CPU_BE(	.byte		0xb, 0xe, 0x1, 0x4, 0x7, 0xa, 0xd, 0x0	)
 
 .LForward_Sbox:
 	.byte		0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5
diff --git a/arch/arm64/crypto/crc32-ce-core.S b/arch/arm64/crypto/crc32-ce-core.S
new file mode 100644
index 000000000000..18f5a8442276
--- /dev/null
+++ b/arch/arm64/crypto/crc32-ce-core.S
@@ -0,0 +1,266 @@
+/*
+ * Accelerated CRC32(C) using arm64 CRC, NEON and Crypto Extensions instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/* GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * Please  visit http://www.xyratex.com/contact if you need additional
+ * information or have any questions.
+ *
+ * GPL HEADER END
+ */
+
+/*
+ * Copyright 2012 Xyratex Technology Limited
+ *
+ * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
+ * calculation.
+ * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
+ * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
+ * at:
+ * http://www.intel.com/products/processor/manuals/
+ * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
+ * Volume 2B: Instruction Set Reference, N-Z
+ *
+ * Authors:   Gregory Prestas <Gregory_Prestas@us.xyratex.com>
+ *	      Alexander Boyko <Alexander_Boyko@xyratex.com>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.text
+	.align		6
+	.cpu		generic+crypto+crc
+
+.Lcrc32_constants:
+	/*
+	 * [x4*128+32 mod P(x) << 32)]'  << 1   = 0x154442bd4
+	 * #define CONSTANT_R1  0x154442bd4LL
+	 *
+	 * [(x4*128-32 mod P(x) << 32)]' << 1   = 0x1c6e41596
+	 * #define CONSTANT_R2  0x1c6e41596LL
+	 */
+	.octa		0x00000001c6e415960000000154442bd4
+
+	/*
+	 * [(x128+32 mod P(x) << 32)]'   << 1   = 0x1751997d0
+	 * #define CONSTANT_R3  0x1751997d0LL
+	 *
+	 * [(x128-32 mod P(x) << 32)]'   << 1   = 0x0ccaa009e
+	 * #define CONSTANT_R4  0x0ccaa009eLL
+	 */
+	.octa		0x00000000ccaa009e00000001751997d0
+
+	/*
+	 * [(x64 mod P(x) << 32)]'       << 1   = 0x163cd6124
+	 * #define CONSTANT_R5  0x163cd6124LL
+	 */
+	.quad		0x0000000163cd6124
+	.quad		0x00000000FFFFFFFF
+
+	/*
+	 * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
+	 *
+	 * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))`
+	 *                                                      = 0x1F7011641LL
+	 * #define CONSTANT_RU  0x1F7011641LL
+	 */
+	.octa		0x00000001F701164100000001DB710641
+
+.Lcrc32c_constants:
+	.octa		0x000000009e4addf800000000740eef02
+	.octa		0x000000014cd00bd600000000f20c0dfe
+	.quad		0x00000000dd45aab8
+	.quad		0x00000000FFFFFFFF
+	.octa		0x00000000dea713f10000000105ec76f0
+
+	vCONSTANT	.req	v0
+	dCONSTANT	.req	d0
+	qCONSTANT	.req	q0
+
+	BUF		.req	x0
+	LEN		.req	x1
+	CRC		.req	x2
+
+	vzr		.req	v9
+
+	/**
+	 * Calculate crc32
+	 * BUF - buffer
+	 * LEN - sizeof buffer (multiple of 16 bytes), LEN should be > 63
+	 * CRC - initial crc32
+	 * return %eax crc32
+	 * uint crc32_pmull_le(unsigned char const *buffer,
+	 *                     size_t len, uint crc32)
+	 */
+ENTRY(crc32_pmull_le)
+	adr		x3, .Lcrc32_constants
+	b		0f
+
+ENTRY(crc32c_pmull_le)
+	adr		x3, .Lcrc32c_constants
+
+0:	bic		LEN, LEN, #15
+	ld1		{v1.16b-v4.16b}, [BUF], #0x40
+	movi		vzr.16b, #0
+	fmov		dCONSTANT, CRC
+	eor		v1.16b, v1.16b, vCONSTANT.16b
+	sub		LEN, LEN, #0x40
+	cmp		LEN, #0x40
+	b.lt		less_64
+
+	ldr		qCONSTANT, [x3]
+
+loop_64:		/* 64 bytes Full cache line folding */
+	sub		LEN, LEN, #0x40
+
+	pmull2		v5.1q, v1.2d, vCONSTANT.2d
+	pmull2		v6.1q, v2.2d, vCONSTANT.2d
+	pmull2		v7.1q, v3.2d, vCONSTANT.2d
+	pmull2		v8.1q, v4.2d, vCONSTANT.2d
+
+	pmull		v1.1q, v1.1d, vCONSTANT.1d
+	pmull		v2.1q, v2.1d, vCONSTANT.1d
+	pmull		v3.1q, v3.1d, vCONSTANT.1d
+	pmull		v4.1q, v4.1d, vCONSTANT.1d
+
+	eor		v1.16b, v1.16b, v5.16b
+	ld1		{v5.16b}, [BUF], #0x10
+	eor		v2.16b, v2.16b, v6.16b
+	ld1		{v6.16b}, [BUF], #0x10
+	eor		v3.16b, v3.16b, v7.16b
+	ld1		{v7.16b}, [BUF], #0x10
+	eor		v4.16b, v4.16b, v8.16b
+	ld1		{v8.16b}, [BUF], #0x10
+
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v2.16b, v2.16b, v6.16b
+	eor		v3.16b, v3.16b, v7.16b
+	eor		v4.16b, v4.16b, v8.16b
+
+	cmp		LEN, #0x40
+	b.ge		loop_64
+
+less_64:		/* Folding cache line into 128bit */
+	ldr		qCONSTANT, [x3, #16]
+
+	pmull2		v5.1q, v1.2d, vCONSTANT.2d
+	pmull		v1.1q, v1.1d, vCONSTANT.1d
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v1.16b, v1.16b, v2.16b
+
+	pmull2		v5.1q, v1.2d, vCONSTANT.2d
+	pmull		v1.1q, v1.1d, vCONSTANT.1d
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v1.16b, v1.16b, v3.16b
+
+	pmull2		v5.1q, v1.2d, vCONSTANT.2d
+	pmull		v1.1q, v1.1d, vCONSTANT.1d
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v1.16b, v1.16b, v4.16b
+
+	cbz		LEN, fold_64
+
+loop_16:		/* Folding rest buffer into 128bit */
+	subs		LEN, LEN, #0x10
+
+	ld1		{v2.16b}, [BUF], #0x10
+	pmull2		v5.1q, v1.2d, vCONSTANT.2d
+	pmull		v1.1q, v1.1d, vCONSTANT.1d
+	eor		v1.16b, v1.16b, v5.16b
+	eor		v1.16b, v1.16b, v2.16b
+
+	b.ne		loop_16
+
+fold_64:
+	/* perform the last 64 bit fold, also adds 32 zeroes
+	 * to the input stream */
+	ext		v2.16b, v1.16b, v1.16b, #8
+	pmull2		v2.1q, v2.2d, vCONSTANT.2d
+	ext		v1.16b, v1.16b, vzr.16b, #8
+	eor		v1.16b, v1.16b, v2.16b
+
+	/* final 32-bit fold */
+	ldr		dCONSTANT, [x3, #32]
+	ldr		d3, [x3, #40]
+
+	ext		v2.16b, v1.16b, vzr.16b, #4
+	and		v1.16b, v1.16b, v3.16b
+	pmull		v1.1q, v1.1d, vCONSTANT.1d
+	eor		v1.16b, v1.16b, v2.16b
+
+	/* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
+	ldr		qCONSTANT, [x3, #48]
+
+	and		v2.16b, v1.16b, v3.16b
+	ext		v2.16b, vzr.16b, v2.16b, #8
+	pmull2		v2.1q, v2.2d, vCONSTANT.2d
+	and		v2.16b, v2.16b, v3.16b
+	pmull		v2.1q, v2.1d, vCONSTANT.1d
+	eor		v1.16b, v1.16b, v2.16b
+	mov		w0, v1.s[1]
+
+	ret
+ENDPROC(crc32_pmull_le)
+ENDPROC(crc32c_pmull_le)
+
+	.macro		__crc32, c
+0:	subs		x2, x2, #16
+	b.mi		8f
+	ldp		x3, x4, [x1], #16
+CPU_BE(	rev		x3, x3		)
+CPU_BE(	rev		x4, x4		)
+	crc32\c\()x	w0, w0, x3
+	crc32\c\()x	w0, w0, x4
+	b.ne		0b
+	ret
+
+8:	tbz		x2, #3, 4f
+	ldr		x3, [x1], #8
+CPU_BE(	rev		x3, x3		)
+	crc32\c\()x	w0, w0, x3
+4:	tbz		x2, #2, 2f
+	ldr		w3, [x1], #4
+CPU_BE(	rev		w3, w3		)
+	crc32\c\()w	w0, w0, w3
+2:	tbz		x2, #1, 1f
+	ldrh		w3, [x1], #2
+CPU_BE(	rev16		w3, w3		)
+	crc32\c\()h	w0, w0, w3
+1:	tbz		x2, #0, 0f
+	ldrb		w3, [x1]
+	crc32\c\()b	w0, w0, w3
+0:	ret
+	.endm
+
+	.align		5
+ENTRY(crc32_armv8_le)
+	__crc32
+ENDPROC(crc32_armv8_le)
+
+	.align		5
+ENTRY(crc32c_armv8_le)
+	__crc32		c
+ENDPROC(crc32c_armv8_le)
diff --git a/arch/arm64/crypto/crc32-ce-glue.c b/arch/arm64/crypto/crc32-ce-glue.c
new file mode 100644
index 000000000000..8594127d5e01
--- /dev/null
+++ b/arch/arm64/crypto/crc32-ce-glue.c
@@ -0,0 +1,212 @@
+/*
+ * Accelerated CRC32(C) using arm64 NEON and Crypto Extensions instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/cpufeature.h>
+#include <linux/crc32.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h>
+
+#include <crypto/internal/hash.h>
+
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+#include <asm/unaligned.h>
+
+#define PMULL_MIN_LEN		64L	/* minimum size of buffer
+					 * for crc32_pmull_le_16 */
+#define SCALE_F			16L	/* size of NEON register */
+
+asmlinkage u32 crc32_pmull_le(const u8 buf[], u64 len, u32 init_crc);
+asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], size_t len);
+
+asmlinkage u32 crc32c_pmull_le(const u8 buf[], u64 len, u32 init_crc);
+asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], size_t len);
+
+static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], size_t len);
+static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], size_t len);
+
+static int crc32_pmull_cra_init(struct crypto_tfm *tfm)
+{
+	u32 *key = crypto_tfm_ctx(tfm);
+
+	*key = 0;
+	return 0;
+}
+
+static int crc32c_pmull_cra_init(struct crypto_tfm *tfm)
+{
+	u32 *key = crypto_tfm_ctx(tfm);
+
+	*key = ~0;
+	return 0;
+}
+
+static int crc32_pmull_setkey(struct crypto_shash *hash, const u8 *key,
+			      unsigned int keylen)
+{
+	u32 *mctx = crypto_shash_ctx(hash);
+
+	if (keylen != sizeof(u32)) {
+		crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+	*mctx = le32_to_cpup((__le32 *)key);
+	return 0;
+}
+
+static int crc32_pmull_init(struct shash_desc *desc)
+{
+	u32 *mctx = crypto_shash_ctx(desc->tfm);
+	u32 *crc = shash_desc_ctx(desc);
+
+	*crc = *mctx;
+	return 0;
+}
+
+static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
+			 unsigned int length)
+{
+	u32 *crc = shash_desc_ctx(desc);
+	unsigned int l;
+
+	if ((u64)data % SCALE_F) {
+		l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));
+
+		*crc = fallback_crc32(*crc, data, l);
+
+		data += l;
+		length -= l;
+	}
+
+	if (length >= PMULL_MIN_LEN) {
+		l = round_down(length, SCALE_F);
+
+		kernel_neon_begin_partial(10);
+		*crc = crc32_pmull_le(data, l, *crc);
+		kernel_neon_end();
+
+		data += l;
+		length -= l;
+	}
+
+	if (length > 0)
+		*crc = fallback_crc32(*crc, data, length);
+
+	return 0;
+}
+
+static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
+			 unsigned int length)
+{
+	u32 *crc = shash_desc_ctx(desc);
+	unsigned int l;
+
+	if ((u64)data % SCALE_F) {
+		l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));
+
+		*crc = fallback_crc32c(*crc, data, l);
+
+		data += l;
+		length -= l;
+	}
+
+	if (length >= PMULL_MIN_LEN) {
+		l = round_down(length, SCALE_F);
+
+		kernel_neon_begin_partial(10);
+		*crc = crc32c_pmull_le(data, l, *crc);
+		kernel_neon_end();
+
+		data += l;
+		length -= l;
+	}
+
+	if (length > 0) {
+		*crc = fallback_crc32c(*crc, data, length);
+	}
+
+	return 0;
+}
+
+static int crc32_pmull_final(struct shash_desc *desc, u8 *out)
+{
+	u32 *crc = shash_desc_ctx(desc);
+
+	put_unaligned_le32(*crc, out);
+	return 0;
+}
+
+static int crc32c_pmull_final(struct shash_desc *desc, u8 *out)
+{
+	u32 *crc = shash_desc_ctx(desc);
+
+	put_unaligned_le32(~*crc, out);
+	return 0;
+}
+
+static struct shash_alg crc32_pmull_algs[] = { {
+	.setkey			= crc32_pmull_setkey,
+	.init			= crc32_pmull_init,
+	.update			= crc32_pmull_update,
+	.final			= crc32_pmull_final,
+	.descsize		= sizeof(u32),
+	.digestsize		= sizeof(u32),
+
+	.base.cra_ctxsize	= sizeof(u32),
+	.base.cra_init		= crc32_pmull_cra_init,
+	.base.cra_name		= "crc32",
+	.base.cra_driver_name	= "crc32-arm64-ce",
+	.base.cra_priority	= 200,
+	.base.cra_blocksize	= 1,
+	.base.cra_module	= THIS_MODULE,
+}, {
+	.setkey			= crc32_pmull_setkey,
+	.init			= crc32_pmull_init,
+	.update			= crc32c_pmull_update,
+	.final			= crc32c_pmull_final,
+	.descsize		= sizeof(u32),
+	.digestsize		= sizeof(u32),
+
+	.base.cra_ctxsize	= sizeof(u32),
+	.base.cra_init		= crc32c_pmull_cra_init,
+	.base.cra_name		= "crc32c",
+	.base.cra_driver_name	= "crc32c-arm64-ce",
+	.base.cra_priority	= 200,
+	.base.cra_blocksize	= 1,
+	.base.cra_module	= THIS_MODULE,
+} };
+
+static int __init crc32_pmull_mod_init(void)
+{
+	if (elf_hwcap & HWCAP_CRC32) {
+		fallback_crc32 = crc32_armv8_le;
+		fallback_crc32c = crc32c_armv8_le;
+	} else {
+		fallback_crc32 = crc32_le;
+		fallback_crc32c = __crc32c_le;
+	}
+
+	return crypto_register_shashes(crc32_pmull_algs,
+				       ARRAY_SIZE(crc32_pmull_algs));
+}
+
+static void __exit crc32_pmull_mod_exit(void)
+{
+	crypto_unregister_shashes(crc32_pmull_algs,
+				  ARRAY_SIZE(crc32_pmull_algs));
+}
+
+module_cpu_feature_match(PMULL, crc32_pmull_mod_init);
+module_exit(crc32_pmull_mod_exit);
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/arch/arm64/crypto/crct10dif-ce-core.S b/arch/arm64/crypto/crct10dif-ce-core.S
new file mode 100644
index 000000000000..d5b5a8c038c8
--- /dev/null
+++ b/arch/arm64/crypto/crct10dif-ce-core.S
@@ -0,0 +1,392 @@
+//
+// Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
+//
+// Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License version 2 as
+// published by the Free Software Foundation.
+//
+
+//
+// Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions
+//
+// Copyright (c) 2013, Intel Corporation
+//
+// Authors:
+//     Erdinc Ozturk <erdinc.ozturk@intel.com>
+//     Vinodh Gopal <vinodh.gopal@intel.com>
+//     James Guilford <james.guilford@intel.com>
+//     Tim Chen <tim.c.chen@linux.intel.com>
+//
+// This software is available to you under a choice of one of two
+// licenses.  You may choose to be licensed under the terms of the GNU
+// General Public License (GPL) Version 2, available from the file
+// COPYING in the main directory of this source tree, or the
+// OpenIB.org BSD license below:
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+//   notice, this list of conditions and the following disclaimer.
+//
+// * Redistributions in binary form must reproduce the above copyright
+//   notice, this list of conditions and the following disclaimer in the
+//   documentation and/or other materials provided with the
+//   distribution.
+//
+// * Neither the name of the Intel Corporation nor the names of its
+//   contributors may be used to endorse or promote products derived from
+//   this software without specific prior written permission.
+//
+//
+// THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+//       Function API:
+//       UINT16 crc_t10dif_pcl(
+//               UINT16 init_crc, //initial CRC value, 16 bits
+//               const unsigned char *buf, //buffer pointer to calculate CRC on
+//               UINT64 len //buffer length in bytes (64-bit data)
+//       );
+//
+//       Reference paper titled "Fast CRC Computation for Generic
+//	Polynomials Using PCLMULQDQ Instruction"
+//       URL: http://www.intel.com/content/dam/www/public/us/en/documents
+//  /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
+//
+//
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.text
+	.cpu		generic+crypto
+
+	arg1_low32	.req	w0
+	arg2		.req	x1
+	arg3		.req	x2
+
+	vzr		.req	v13
+
+ENTRY(crc_t10dif_pmull)
+	movi		vzr.16b, #0		// init zero register
+
+	// adjust the 16-bit initial_crc value, scale it to 32 bits
+	lsl		arg1_low32, arg1_low32, #16
+
+	// check if smaller than 256
+	cmp		arg3, #256
+
+	// for sizes less than 128, we can't fold 64B at a time...
+	b.lt		_less_than_128
+
+	// load the initial crc value
+	// crc value does not need to be byte-reflected, but it needs
+	// to be moved to the high part of the register.
+	// because data will be byte-reflected and will align with
+	// initial crc at correct place.
+	movi		v10.16b, #0
+	mov		v10.s[3], arg1_low32		// initial crc
+
+	// receive the initial 64B data, xor the initial crc value
+	ldp		q0, q1, [arg2]
+	ldp		q2, q3, [arg2, #0x20]
+	ldp		q4, q5, [arg2, #0x40]
+	ldp		q6, q7, [arg2, #0x60]
+	add		arg2, arg2, #0x80
+
+CPU_LE(	rev64		v0.16b, v0.16b			)
+CPU_LE(	rev64		v1.16b, v1.16b			)
+CPU_LE(	rev64		v2.16b, v2.16b			)
+CPU_LE(	rev64		v3.16b, v3.16b			)
+CPU_LE(	rev64		v4.16b, v4.16b			)
+CPU_LE(	rev64		v5.16b, v5.16b			)
+CPU_LE(	rev64		v6.16b, v6.16b			)
+CPU_LE(	rev64		v7.16b, v7.16b			)
+
+CPU_LE(	ext		v0.16b, v0.16b, v0.16b, #8	)
+CPU_LE(	ext		v1.16b, v1.16b, v1.16b, #8	)
+CPU_LE(	ext		v2.16b, v2.16b, v2.16b, #8	)
+CPU_LE(	ext		v3.16b, v3.16b, v3.16b, #8	)
+CPU_LE(	ext		v4.16b, v4.16b, v4.16b, #8	)
+CPU_LE(	ext		v5.16b, v5.16b, v5.16b, #8	)
+CPU_LE(	ext		v6.16b, v6.16b, v6.16b, #8	)
+CPU_LE(	ext		v7.16b, v7.16b, v7.16b, #8	)
+
+	// XOR the initial_crc value
+	eor		v0.16b, v0.16b, v10.16b
+
+	ldr		q10, rk3	// xmm10 has rk3 and rk4
+					// type of pmull instruction
+					// will determine which constant to use
+
+	//
+	// we subtract 256 instead of 128 to save one instruction from the loop
+	//
+	sub		arg3, arg3, #256
+
+	// at this section of the code, there is 64*x+y (0<=y<64) bytes of
+	// buffer. The _fold_64_B_loop will fold 64B at a time
+	// until we have 64+y Bytes of buffer
+
+
+	// fold 64B at a time. This section of the code folds 4 vector
+	// registers in parallel
+_fold_64_B_loop:
+
+	.macro		fold64, reg1, reg2
+	ldp		q11, q12, [arg2], #0x20
+
+	pmull2		v8.1q, \reg1\().2d, v10.2d
+	pmull		\reg1\().1q, \reg1\().1d, v10.1d
+
+CPU_LE(	rev64		v11.16b, v11.16b		)
+CPU_LE(	rev64		v12.16b, v12.16b		)
+
+	pmull2		v9.1q, \reg2\().2d, v10.2d
+	pmull		\reg2\().1q, \reg2\().1d, v10.1d
+
+CPU_LE(	ext		v11.16b, v11.16b, v11.16b, #8	)
+CPU_LE(	ext		v12.16b, v12.16b, v12.16b, #8	)
+
+	eor		\reg1\().16b, \reg1\().16b, v8.16b
+	eor		\reg2\().16b, \reg2\().16b, v9.16b
+	eor		\reg1\().16b, \reg1\().16b, v11.16b
+	eor		\reg2\().16b, \reg2\().16b, v12.16b
+	.endm
+
+	fold64		v0, v1
+	fold64		v2, v3
+	fold64		v4, v5
+	fold64		v6, v7
+
+	subs		arg3, arg3, #128
+
+	// check if there is another 64B in the buffer to be able to fold
+	b.ge		_fold_64_B_loop
+
+	// at this point, the buffer pointer is pointing at the last y Bytes
+	// of the buffer the 64B of folded data is in 4 of the vector
+	// registers: v0, v1, v2, v3
+
+	// fold the 8 vector registers to 1 vector register with different
+	// constants
+
+	ldr		q10, rk9
+
+	.macro		fold16, reg, rk
+	pmull		v8.1q, \reg\().1d, v10.1d
+	pmull2		\reg\().1q, \reg\().2d, v10.2d
+	.ifnb		\rk
+	ldr		q10, \rk
+	.endif
+	eor		v7.16b, v7.16b, v8.16b
+	eor		v7.16b, v7.16b, \reg\().16b
+	.endm
+
+	fold16		v0, rk11
+	fold16		v1, rk13
+	fold16		v2, rk15
+	fold16		v3, rk17
+	fold16		v4, rk19
+	fold16		v5, rk1
+	fold16		v6
+
+	// instead of 64, we add 48 to the loop counter to save 1 instruction
+	// from the loop instead of a cmp instruction, we use the negative
+	// flag with the jl instruction
+	adds		arg3, arg3, #(128-16)
+	b.lt		_final_reduction_for_128
+
+	// now we have 16+y bytes left to reduce. 16 Bytes is in register v7
+	// and the rest is in memory. We can fold 16 bytes at a time if y>=16
+	// continue folding 16B at a time
+
+_16B_reduction_loop:
+	pmull		v8.1q, v7.1d, v10.1d
+	pmull2		v7.1q, v7.2d, v10.2d
+	eor		v7.16b, v7.16b, v8.16b
+
+	ldr		q0, [arg2], #16
+CPU_LE(	rev64		v0.16b, v0.16b			)
+CPU_LE(	ext		v0.16b, v0.16b, v0.16b, #8	)
+	eor		v7.16b, v7.16b, v0.16b
+	subs		arg3, arg3, #16
+
+	// instead of a cmp instruction, we utilize the flags with the
+	// jge instruction equivalent of: cmp arg3, 16-16
+	// check if there is any more 16B in the buffer to be able to fold
+	b.ge		_16B_reduction_loop
+
+	// now we have 16+z bytes left to reduce, where 0<= z < 16.
+	// first, we reduce the data in the xmm7 register
+
+_final_reduction_for_128:
+	// check if any more data to fold. If not, compute the CRC of
+	// the final 128 bits
+	adds		arg3, arg3, #16
+	b.eq		_128_done
+
+	// here we are getting data that is less than 16 bytes.
+	// since we know that there was data before the pointer, we can
+	// offset the input pointer before the actual point, to receive
+	// exactly 16 bytes. after that the registers need to be adjusted.
+_get_last_two_regs:
+	add		arg2, arg2, arg3
+	ldr		q1, [arg2, #-16]
+CPU_LE(	rev64		v1.16b, v1.16b			)
+CPU_LE(	ext		v1.16b, v1.16b, v1.16b, #8	)
+
+	// get rid of the extra data that was loaded before
+	// load the shift constant
+	adr		x4, tbl_shf_table + 16
+	sub		x4, x4, arg3
+	ld1		{v0.16b}, [x4]
+
+	// shift v2 to the left by arg3 bytes
+	tbl		v2.16b, {v7.16b}, v0.16b
+
+	// shift v7 to the right by 16-arg3 bytes
+	movi		v9.16b, #0x80
+	eor		v0.16b, v0.16b, v9.16b
+	tbl		v7.16b, {v7.16b}, v0.16b
+
+	// blend
+	sshr		v0.16b, v0.16b, #7	// convert to 8-bit mask
+	bsl		v0.16b, v2.16b, v1.16b
+
+	// fold 16 Bytes
+	pmull		v8.1q, v7.1d, v10.1d
+	pmull2		v7.1q, v7.2d, v10.2d
+	eor		v7.16b, v7.16b, v8.16b
+	eor		v7.16b, v7.16b, v0.16b
+
+_128_done:
+	// compute crc of a 128-bit value
+	ldr		q10, rk5		// rk5 and rk6 in xmm10
+
+	// 64b fold
+	ext		v0.16b, vzr.16b, v7.16b, #8
+	mov		v7.d[0], v7.d[1]
+	pmull		v7.1q, v7.1d, v10.1d
+	eor		v7.16b, v7.16b, v0.16b
+
+	// 32b fold
+	ext		v0.16b, v7.16b, vzr.16b, #4
+	mov		v7.s[3], vzr.s[0]
+	pmull2		v0.1q, v0.2d, v10.2d
+	eor		v7.16b, v7.16b, v0.16b
+
+	// barrett reduction
+_barrett:
+	ldr		q10, rk7
+	mov		v0.d[0], v7.d[1]
+
+	pmull		v0.1q, v0.1d, v10.1d
+	ext		v0.16b, vzr.16b, v0.16b, #12
+	pmull2		v0.1q, v0.2d, v10.2d
+	ext		v0.16b, vzr.16b, v0.16b, #12
+	eor		v7.16b, v7.16b, v0.16b
+	mov		w0, v7.s[1]
+
+_cleanup:
+	// scale the result back to 16 bits
+	lsr		x0, x0, #16
+	ret
+
+_less_than_128:
+	cbz		arg3, _cleanup
+
+	movi		v0.16b, #0
+	mov		v0.s[3], arg1_low32	// get the initial crc value
+
+	ldr		q7, [arg2], #0x10
+CPU_LE(	rev64		v7.16b, v7.16b			)
+CPU_LE(	ext		v7.16b, v7.16b, v7.16b, #8	)
+	eor		v7.16b, v7.16b, v0.16b	// xor the initial crc value
+
+	cmp		arg3, #16
+	b.eq		_128_done		// exactly 16 left
+	b.lt		_less_than_16_left
+
+	ldr		q10, rk1		// rk1 and rk2 in xmm10
+
+	// update the counter. subtract 32 instead of 16 to save one
+	// instruction from the loop
+	subs		arg3, arg3, #32
+	b.ge		_16B_reduction_loop
+
+	add		arg3, arg3, #16
+	b		_get_last_two_regs
+
+_less_than_16_left:
+	// shl r9, 4
+	adr		x0, tbl_shf_table + 16
+	sub		x0, x0, arg3
+	ld1		{v0.16b}, [x0]
+	movi		v9.16b, #0x80
+	eor		v0.16b, v0.16b, v9.16b
+	tbl		v7.16b, {v7.16b}, v0.16b
+	b		_128_done
+ENDPROC(crc_t10dif_pmull)
+
+// precomputed constants
+// these constants are precomputed from the poly:
+// 0x8bb70000 (0x8bb7 scaled to 32 bits)
+	.align		4
+// Q = 0x18BB70000
+// rk1 = 2^(32*3) mod Q << 32
+// rk2 = 2^(32*5) mod Q << 32
+// rk3 = 2^(32*15) mod Q << 32
+// rk4 = 2^(32*17) mod Q << 32
+// rk5 = 2^(32*3) mod Q << 32
+// rk6 = 2^(32*2) mod Q << 32
+// rk7 = floor(2^64/Q)
+// rk8 = Q
+
+rk1:	.octa		0x06df0000000000002d56000000000000
+rk3:	.octa		0x7cf50000000000009d9d000000000000
+rk5:	.octa		0x13680000000000002d56000000000000
+rk7:	.octa		0x000000018bb7000000000001f65a57f8
+rk9:	.octa		0xbfd6000000000000ceae000000000000
+rk11:	.octa		0x713c0000000000001e16000000000000
+rk13:	.octa		0x80a6000000000000f7f9000000000000
+rk15:	.octa		0xe658000000000000044c000000000000
+rk17:	.octa		0xa497000000000000ad18000000000000
+rk19:	.octa		0xe7b50000000000006ee3000000000000
+
+tbl_shf_table:
+// use these values for shift constants for the tbl/tbx instruction
+// different alignments result in values as shown:
+//	DDQ 0x008f8e8d8c8b8a898887868584838281 # shl 15 (16-1) / shr1
+//	DDQ 0x01008f8e8d8c8b8a8988878685848382 # shl 14 (16-3) / shr2
+//	DDQ 0x0201008f8e8d8c8b8a89888786858483 # shl 13 (16-4) / shr3
+//	DDQ 0x030201008f8e8d8c8b8a898887868584 # shl 12 (16-4) / shr4
+//	DDQ 0x04030201008f8e8d8c8b8a8988878685 # shl 11 (16-5) / shr5
+//	DDQ 0x0504030201008f8e8d8c8b8a89888786 # shl 10 (16-6) / shr6
+//	DDQ 0x060504030201008f8e8d8c8b8a898887 # shl 9  (16-7) / shr7
+//	DDQ 0x07060504030201008f8e8d8c8b8a8988 # shl 8  (16-8) / shr8
+//	DDQ 0x0807060504030201008f8e8d8c8b8a89 # shl 7  (16-9) / shr9
+//	DDQ 0x090807060504030201008f8e8d8c8b8a # shl 6  (16-10) / shr10
+//	DDQ 0x0a090807060504030201008f8e8d8c8b # shl 5  (16-11) / shr11
+//	DDQ 0x0b0a090807060504030201008f8e8d8c # shl 4  (16-12) / shr12
+//	DDQ 0x0c0b0a090807060504030201008f8e8d # shl 3  (16-13) / shr13
+//	DDQ 0x0d0c0b0a090807060504030201008f8e # shl 2  (16-14) / shr14
+//	DDQ 0x0e0d0c0b0a090807060504030201008f # shl 1  (16-15) / shr15
+
+	.byte		 0x0, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87
+	.byte		0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f
+	.byte		 0x0,  0x1,  0x2,  0x3,  0x4,  0x5,  0x6,  0x7
+	.byte		 0x8,  0x9,  0xa,  0xb,  0xc,  0xd,  0xe , 0x0
diff --git a/arch/arm64/crypto/crct10dif-ce-glue.c b/arch/arm64/crypto/crct10dif-ce-glue.c
new file mode 100644
index 000000000000..60cb590c2590
--- /dev/null
+++ b/arch/arm64/crypto/crct10dif-ce-glue.c
@@ -0,0 +1,95 @@
+/*
+ * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/cpufeature.h>
+#include <linux/crc-t10dif.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/string.h>
+
+#include <crypto/internal/hash.h>
+
+#include <asm/neon.h>
+
+#define CRC_T10DIF_PMULL_CHUNK_SIZE	16U
+
+asmlinkage u16 crc_t10dif_pmull(u16 init_crc, const u8 buf[], u64 len);
+
+static int crct10dif_init(struct shash_desc *desc)
+{
+	u16 *crc = shash_desc_ctx(desc);
+
+	*crc = 0;
+	return 0;
+}
+
+static int crct10dif_update(struct shash_desc *desc, const u8 *data,
+			    unsigned int length)
+{
+	u16 *crc = shash_desc_ctx(desc);
+	unsigned int l;
+
+	if (unlikely((u64)data % CRC_T10DIF_PMULL_CHUNK_SIZE)) {
+		l = min_t(u32, length, CRC_T10DIF_PMULL_CHUNK_SIZE -
+			  ((u64)data % CRC_T10DIF_PMULL_CHUNK_SIZE));
+
+		*crc = crc_t10dif_generic(*crc, data, l);
+
+		length -= l;
+		data += l;
+	}
+
+	if (length > 0) {
+		kernel_neon_begin_partial(14);
+		*crc = crc_t10dif_pmull(*crc, data, length);
+		kernel_neon_end();
+	}
+
+	return 0;
+}
+
+static int crct10dif_final(struct shash_desc *desc, u8 *out)
+{
+	u16 *crc = shash_desc_ctx(desc);
+
+	*(u16 *)out = *crc;
+	return 0;
+}
+
+static struct shash_alg crc_t10dif_alg = {
+	.digestsize		= CRC_T10DIF_DIGEST_SIZE,
+	.init			= crct10dif_init,
+	.update			= crct10dif_update,
+	.final			= crct10dif_final,
+	.descsize		= CRC_T10DIF_DIGEST_SIZE,
+
+	.base.cra_name		= "crct10dif",
+	.base.cra_driver_name	= "crct10dif-arm64-ce",
+	.base.cra_priority	= 200,
+	.base.cra_blocksize	= CRC_T10DIF_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+};
+
+static int __init crc_t10dif_mod_init(void)
+{
+	return crypto_register_shash(&crc_t10dif_alg);
+}
+
+static void __exit crc_t10dif_mod_exit(void)
+{
+	crypto_unregister_shash(&crc_t10dif_alg);
+}
+
+module_cpu_feature_match(PMULL, crc_t10dif_mod_init);
+module_exit(crc_t10dif_mod_exit);
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S
index dc457015884e..f0bb9f0b524f 100644
--- a/arch/arm64/crypto/ghash-ce-core.S
+++ b/arch/arm64/crypto/ghash-ce-core.S
@@ -29,8 +29,8 @@
 	 *			   struct ghash_key const *k, const char *head)
 	 */
 ENTRY(pmull_ghash_update)
-	ld1		{SHASH.16b}, [x3]
-	ld1		{XL.16b}, [x1]
+	ld1		{SHASH.2d}, [x3]
+	ld1		{XL.2d}, [x1]
 	movi		MASK.16b, #0xe1
 	ext		SHASH2.16b, SHASH.16b, SHASH.16b, #8
 	shl		MASK.2d, MASK.2d, #57
@@ -74,6 +74,6 @@ CPU_LE(	rev64		T1.16b, T1.16b	)
 
 	cbnz		w0, 0b
 
-	st1		{XL.16b}, [x1]
+	st1		{XL.2d}, [x1]
 	ret
 ENDPROC(pmull_ghash_update)
diff --git a/arch/arm64/crypto/sha1-ce-core.S b/arch/arm64/crypto/sha1-ce-core.S
index 033aae6d732a..c98e7e849f06 100644
--- a/arch/arm64/crypto/sha1-ce-core.S
+++ b/arch/arm64/crypto/sha1-ce-core.S
@@ -78,7 +78,7 @@ ENTRY(sha1_ce_transform)
 	ld1r		{k3.4s}, [x6]
 
 	/* load state */
-	ldr		dga, [x0]
+	ld1		{dgav.4s}, [x0]
 	ldr		dgb, [x0, #16]
 
 	/* load sha1_ce_state::finalize */
@@ -144,7 +144,7 @@ CPU_LE(	rev32		v11.16b, v11.16b	)
 	b		1b
 
 	/* store new state */
-3:	str		dga, [x0]
+3:	st1		{dgav.4s}, [x0]
 	str		dgb, [x0, #16]
 	ret
 ENDPROC(sha1_ce_transform)
diff --git a/arch/arm64/crypto/sha2-ce-core.S b/arch/arm64/crypto/sha2-ce-core.S
index 5df9d9d470ad..01cfee066837 100644
--- a/arch/arm64/crypto/sha2-ce-core.S
+++ b/arch/arm64/crypto/sha2-ce-core.S
@@ -85,7 +85,7 @@ ENTRY(sha2_ce_transform)
 	ld1		{v12.4s-v15.4s}, [x8]
 
 	/* load state */
-	ldp		dga, dgb, [x0]
+	ld1		{dgav.4s, dgbv.4s}, [x0]
 
 	/* load sha256_ce_state::finalize */
 	ldr		w4, [x0, #:lo12:sha256_ce_offsetof_finalize]
@@ -148,6 +148,6 @@ CPU_LE(	rev32		v19.16b, v19.16b	)
 	b		1b
 
 	/* store new state */
-3:	stp		dga, dgb, [x0]
+3:	st1		{dgav.4s, dgbv.4s}, [x0]
 	ret
 ENDPROC(sha2_ce_transform)
diff --git a/arch/arm64/crypto/sha256-core.S_shipped b/arch/arm64/crypto/sha256-core.S_shipped
new file mode 100644
index 000000000000..3ce82cc860bc
--- /dev/null
+++ b/arch/arm64/crypto/sha256-core.S_shipped
@@ -0,0 +1,2061 @@
+// Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
+//
+// Licensed under the OpenSSL license (the "License").  You may not use
+// this file except in compliance with the License.  You can obtain a copy
+// in the file LICENSE in the source distribution or at
+// https://www.openssl.org/source/license.html
+
+// ====================================================================
+// Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+// project. The module is, however, dual licensed under OpenSSL and
+// CRYPTOGAMS licenses depending on where you obtain it. For further
+// details see http://www.openssl.org/~appro/cryptogams/.
+//
+// Permission to use under GPLv2 terms is granted.
+// ====================================================================
+//
+// SHA256/512 for ARMv8.
+//
+// Performance in cycles per processed byte and improvement coefficient
+// over code generated with "default" compiler:
+//
+//		SHA256-hw	SHA256(*)	SHA512
+// Apple A7	1.97		10.5 (+33%)	6.73 (-1%(**))
+// Cortex-A53	2.38		15.5 (+115%)	10.0 (+150%(***))
+// Cortex-A57	2.31		11.6 (+86%)	7.51 (+260%(***))
+// Denver	2.01		10.5 (+26%)	6.70 (+8%)
+// X-Gene			20.0 (+100%)	12.8 (+300%(***))
+// Mongoose	2.36		13.0 (+50%)	8.36 (+33%)
+//
+// (*)	Software SHA256 results are of lesser relevance, presented
+//	mostly for informational purposes.
+// (**)	The result is a trade-off: it's possible to improve it by
+//	10% (or by 1 cycle per round), but at the cost of 20% loss
+//	on Cortex-A53 (or by 4 cycles per round).
+// (***)	Super-impressive coefficients over gcc-generated code are
+//	indication of some compiler "pathology", most notably code
+//	generated with -mgeneral-regs-only is significanty faster
+//	and the gap is only 40-90%.
+//
+// October 2016.
+//
+// Originally it was reckoned that it makes no sense to implement NEON
+// version of SHA256 for 64-bit processors. This is because performance
+// improvement on most wide-spread Cortex-A5x processors was observed
+// to be marginal, same on Cortex-A53 and ~10% on A57. But then it was
+// observed that 32-bit NEON SHA256 performs significantly better than
+// 64-bit scalar version on *some* of the more recent processors. As
+// result 64-bit NEON version of SHA256 was added to provide best
+// all-round performance. For example it executes ~30% faster on X-Gene
+// and Mongoose. [For reference, NEON version of SHA512 is bound to
+// deliver much less improvement, likely *negative* on Cortex-A5x.
+// Which is why NEON support is limited to SHA256.]
+
+#ifndef	__KERNEL__
+# include "arm_arch.h"
+#endif
+
+.text
+
+.extern	OPENSSL_armcap_P
+.globl	sha256_block_data_order
+.type	sha256_block_data_order,%function
+.align	6
+sha256_block_data_order:
+#ifndef	__KERNEL__
+# ifdef	__ILP32__
+	ldrsw	x16,.LOPENSSL_armcap_P
+# else
+	ldr	x16,.LOPENSSL_armcap_P
+# endif
+	adr	x17,.LOPENSSL_armcap_P
+	add	x16,x16,x17
+	ldr	w16,[x16]
+	tst	w16,#ARMV8_SHA256
+	b.ne	.Lv8_entry
+	tst	w16,#ARMV7_NEON
+	b.ne	.Lneon_entry
+#endif
+	stp	x29,x30,[sp,#-128]!
+	add	x29,sp,#0
+
+	stp	x19,x20,[sp,#16]
+	stp	x21,x22,[sp,#32]
+	stp	x23,x24,[sp,#48]
+	stp	x25,x26,[sp,#64]
+	stp	x27,x28,[sp,#80]
+	sub	sp,sp,#4*4
+
+	ldp	w20,w21,[x0]				// load context
+	ldp	w22,w23,[x0,#2*4]
+	ldp	w24,w25,[x0,#4*4]
+	add	x2,x1,x2,lsl#6	// end of input
+	ldp	w26,w27,[x0,#6*4]
+	adr	x30,.LK256
+	stp	x0,x2,[x29,#96]
+
+.Loop:
+	ldp	w3,w4,[x1],#2*4
+	ldr	w19,[x30],#4			// *K++
+	eor	w28,w21,w22				// magic seed
+	str	x1,[x29,#112]
+#ifndef	__AARCH64EB__
+	rev	w3,w3			// 0
+#endif
+	ror	w16,w24,#6
+	add	w27,w27,w19			// h+=K[i]
+	eor	w6,w24,w24,ror#14
+	and	w17,w25,w24
+	bic	w19,w26,w24
+	add	w27,w27,w3			// h+=X[i]
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w20,w21			// a^b, b^c in next round
+	eor	w16,w16,w6,ror#11	// Sigma1(e)
+	ror	w6,w20,#2
+	add	w27,w27,w17			// h+=Ch(e,f,g)
+	eor	w17,w20,w20,ror#9
+	add	w27,w27,w16			// h+=Sigma1(e)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	add	w23,w23,w27			// d+=h
+	eor	w28,w28,w21			// Maj(a,b,c)
+	eor	w17,w6,w17,ror#13	// Sigma0(a)
+	add	w27,w27,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	//add	w27,w27,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w4,w4			// 1
+#endif
+	ldp	w5,w6,[x1],#2*4
+	add	w27,w27,w17			// h+=Sigma0(a)
+	ror	w16,w23,#6
+	add	w26,w26,w28			// h+=K[i]
+	eor	w7,w23,w23,ror#14
+	and	w17,w24,w23
+	bic	w28,w25,w23
+	add	w26,w26,w4			// h+=X[i]
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w27,w20			// a^b, b^c in next round
+	eor	w16,w16,w7,ror#11	// Sigma1(e)
+	ror	w7,w27,#2
+	add	w26,w26,w17			// h+=Ch(e,f,g)
+	eor	w17,w27,w27,ror#9
+	add	w26,w26,w16			// h+=Sigma1(e)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	add	w22,w22,w26			// d+=h
+	eor	w19,w19,w20			// Maj(a,b,c)
+	eor	w17,w7,w17,ror#13	// Sigma0(a)
+	add	w26,w26,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	//add	w26,w26,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w5,w5			// 2
+#endif
+	add	w26,w26,w17			// h+=Sigma0(a)
+	ror	w16,w22,#6
+	add	w25,w25,w19			// h+=K[i]
+	eor	w8,w22,w22,ror#14
+	and	w17,w23,w22
+	bic	w19,w24,w22
+	add	w25,w25,w5			// h+=X[i]
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w26,w27			// a^b, b^c in next round
+	eor	w16,w16,w8,ror#11	// Sigma1(e)
+	ror	w8,w26,#2
+	add	w25,w25,w17			// h+=Ch(e,f,g)
+	eor	w17,w26,w26,ror#9
+	add	w25,w25,w16			// h+=Sigma1(e)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	add	w21,w21,w25			// d+=h
+	eor	w28,w28,w27			// Maj(a,b,c)
+	eor	w17,w8,w17,ror#13	// Sigma0(a)
+	add	w25,w25,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	//add	w25,w25,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w6,w6			// 3
+#endif
+	ldp	w7,w8,[x1],#2*4
+	add	w25,w25,w17			// h+=Sigma0(a)
+	ror	w16,w21,#6
+	add	w24,w24,w28			// h+=K[i]
+	eor	w9,w21,w21,ror#14
+	and	w17,w22,w21
+	bic	w28,w23,w21
+	add	w24,w24,w6			// h+=X[i]
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w25,w26			// a^b, b^c in next round
+	eor	w16,w16,w9,ror#11	// Sigma1(e)
+	ror	w9,w25,#2
+	add	w24,w24,w17			// h+=Ch(e,f,g)
+	eor	w17,w25,w25,ror#9
+	add	w24,w24,w16			// h+=Sigma1(e)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	add	w20,w20,w24			// d+=h
+	eor	w19,w19,w26			// Maj(a,b,c)
+	eor	w17,w9,w17,ror#13	// Sigma0(a)
+	add	w24,w24,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	//add	w24,w24,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w7,w7			// 4
+#endif
+	add	w24,w24,w17			// h+=Sigma0(a)
+	ror	w16,w20,#6
+	add	w23,w23,w19			// h+=K[i]
+	eor	w10,w20,w20,ror#14
+	and	w17,w21,w20
+	bic	w19,w22,w20
+	add	w23,w23,w7			// h+=X[i]
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w24,w25			// a^b, b^c in next round
+	eor	w16,w16,w10,ror#11	// Sigma1(e)
+	ror	w10,w24,#2
+	add	w23,w23,w17			// h+=Ch(e,f,g)
+	eor	w17,w24,w24,ror#9
+	add	w23,w23,w16			// h+=Sigma1(e)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	add	w27,w27,w23			// d+=h
+	eor	w28,w28,w25			// Maj(a,b,c)
+	eor	w17,w10,w17,ror#13	// Sigma0(a)
+	add	w23,w23,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	//add	w23,w23,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w8,w8			// 5
+#endif
+	ldp	w9,w10,[x1],#2*4
+	add	w23,w23,w17			// h+=Sigma0(a)
+	ror	w16,w27,#6
+	add	w22,w22,w28			// h+=K[i]
+	eor	w11,w27,w27,ror#14
+	and	w17,w20,w27
+	bic	w28,w21,w27
+	add	w22,w22,w8			// h+=X[i]
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w23,w24			// a^b, b^c in next round
+	eor	w16,w16,w11,ror#11	// Sigma1(e)
+	ror	w11,w23,#2
+	add	w22,w22,w17			// h+=Ch(e,f,g)
+	eor	w17,w23,w23,ror#9
+	add	w22,w22,w16			// h+=Sigma1(e)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	add	w26,w26,w22			// d+=h
+	eor	w19,w19,w24			// Maj(a,b,c)
+	eor	w17,w11,w17,ror#13	// Sigma0(a)
+	add	w22,w22,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	//add	w22,w22,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w9,w9			// 6
+#endif
+	add	w22,w22,w17			// h+=Sigma0(a)
+	ror	w16,w26,#6
+	add	w21,w21,w19			// h+=K[i]
+	eor	w12,w26,w26,ror#14
+	and	w17,w27,w26
+	bic	w19,w20,w26
+	add	w21,w21,w9			// h+=X[i]
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w22,w23			// a^b, b^c in next round
+	eor	w16,w16,w12,ror#11	// Sigma1(e)
+	ror	w12,w22,#2
+	add	w21,w21,w17			// h+=Ch(e,f,g)
+	eor	w17,w22,w22,ror#9
+	add	w21,w21,w16			// h+=Sigma1(e)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	add	w25,w25,w21			// d+=h
+	eor	w28,w28,w23			// Maj(a,b,c)
+	eor	w17,w12,w17,ror#13	// Sigma0(a)
+	add	w21,w21,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	//add	w21,w21,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w10,w10			// 7
+#endif
+	ldp	w11,w12,[x1],#2*4
+	add	w21,w21,w17			// h+=Sigma0(a)
+	ror	w16,w25,#6
+	add	w20,w20,w28			// h+=K[i]
+	eor	w13,w25,w25,ror#14
+	and	w17,w26,w25
+	bic	w28,w27,w25
+	add	w20,w20,w10			// h+=X[i]
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w21,w22			// a^b, b^c in next round
+	eor	w16,w16,w13,ror#11	// Sigma1(e)
+	ror	w13,w21,#2
+	add	w20,w20,w17			// h+=Ch(e,f,g)
+	eor	w17,w21,w21,ror#9
+	add	w20,w20,w16			// h+=Sigma1(e)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	add	w24,w24,w20			// d+=h
+	eor	w19,w19,w22			// Maj(a,b,c)
+	eor	w17,w13,w17,ror#13	// Sigma0(a)
+	add	w20,w20,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	//add	w20,w20,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w11,w11			// 8
+#endif
+	add	w20,w20,w17			// h+=Sigma0(a)
+	ror	w16,w24,#6
+	add	w27,w27,w19			// h+=K[i]
+	eor	w14,w24,w24,ror#14
+	and	w17,w25,w24
+	bic	w19,w26,w24
+	add	w27,w27,w11			// h+=X[i]
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w20,w21			// a^b, b^c in next round
+	eor	w16,w16,w14,ror#11	// Sigma1(e)
+	ror	w14,w20,#2
+	add	w27,w27,w17			// h+=Ch(e,f,g)
+	eor	w17,w20,w20,ror#9
+	add	w27,w27,w16			// h+=Sigma1(e)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	add	w23,w23,w27			// d+=h
+	eor	w28,w28,w21			// Maj(a,b,c)
+	eor	w17,w14,w17,ror#13	// Sigma0(a)
+	add	w27,w27,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	//add	w27,w27,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w12,w12			// 9
+#endif
+	ldp	w13,w14,[x1],#2*4
+	add	w27,w27,w17			// h+=Sigma0(a)
+	ror	w16,w23,#6
+	add	w26,w26,w28			// h+=K[i]
+	eor	w15,w23,w23,ror#14
+	and	w17,w24,w23
+	bic	w28,w25,w23
+	add	w26,w26,w12			// h+=X[i]
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w27,w20			// a^b, b^c in next round
+	eor	w16,w16,w15,ror#11	// Sigma1(e)
+	ror	w15,w27,#2
+	add	w26,w26,w17			// h+=Ch(e,f,g)
+	eor	w17,w27,w27,ror#9
+	add	w26,w26,w16			// h+=Sigma1(e)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	add	w22,w22,w26			// d+=h
+	eor	w19,w19,w20			// Maj(a,b,c)
+	eor	w17,w15,w17,ror#13	// Sigma0(a)
+	add	w26,w26,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	//add	w26,w26,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w13,w13			// 10
+#endif
+	add	w26,w26,w17			// h+=Sigma0(a)
+	ror	w16,w22,#6
+	add	w25,w25,w19			// h+=K[i]
+	eor	w0,w22,w22,ror#14
+	and	w17,w23,w22
+	bic	w19,w24,w22
+	add	w25,w25,w13			// h+=X[i]
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w26,w27			// a^b, b^c in next round
+	eor	w16,w16,w0,ror#11	// Sigma1(e)
+	ror	w0,w26,#2
+	add	w25,w25,w17			// h+=Ch(e,f,g)
+	eor	w17,w26,w26,ror#9
+	add	w25,w25,w16			// h+=Sigma1(e)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	add	w21,w21,w25			// d+=h
+	eor	w28,w28,w27			// Maj(a,b,c)
+	eor	w17,w0,w17,ror#13	// Sigma0(a)
+	add	w25,w25,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	//add	w25,w25,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w14,w14			// 11
+#endif
+	ldp	w15,w0,[x1],#2*4
+	add	w25,w25,w17			// h+=Sigma0(a)
+	str	w6,[sp,#12]
+	ror	w16,w21,#6
+	add	w24,w24,w28			// h+=K[i]
+	eor	w6,w21,w21,ror#14
+	and	w17,w22,w21
+	bic	w28,w23,w21
+	add	w24,w24,w14			// h+=X[i]
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w25,w26			// a^b, b^c in next round
+	eor	w16,w16,w6,ror#11	// Sigma1(e)
+	ror	w6,w25,#2
+	add	w24,w24,w17			// h+=Ch(e,f,g)
+	eor	w17,w25,w25,ror#9
+	add	w24,w24,w16			// h+=Sigma1(e)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	add	w20,w20,w24			// d+=h
+	eor	w19,w19,w26			// Maj(a,b,c)
+	eor	w17,w6,w17,ror#13	// Sigma0(a)
+	add	w24,w24,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	//add	w24,w24,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w15,w15			// 12
+#endif
+	add	w24,w24,w17			// h+=Sigma0(a)
+	str	w7,[sp,#0]
+	ror	w16,w20,#6
+	add	w23,w23,w19			// h+=K[i]
+	eor	w7,w20,w20,ror#14
+	and	w17,w21,w20
+	bic	w19,w22,w20
+	add	w23,w23,w15			// h+=X[i]
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w24,w25			// a^b, b^c in next round
+	eor	w16,w16,w7,ror#11	// Sigma1(e)
+	ror	w7,w24,#2
+	add	w23,w23,w17			// h+=Ch(e,f,g)
+	eor	w17,w24,w24,ror#9
+	add	w23,w23,w16			// h+=Sigma1(e)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	add	w27,w27,w23			// d+=h
+	eor	w28,w28,w25			// Maj(a,b,c)
+	eor	w17,w7,w17,ror#13	// Sigma0(a)
+	add	w23,w23,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	//add	w23,w23,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w0,w0			// 13
+#endif
+	ldp	w1,w2,[x1]
+	add	w23,w23,w17			// h+=Sigma0(a)
+	str	w8,[sp,#4]
+	ror	w16,w27,#6
+	add	w22,w22,w28			// h+=K[i]
+	eor	w8,w27,w27,ror#14
+	and	w17,w20,w27
+	bic	w28,w21,w27
+	add	w22,w22,w0			// h+=X[i]
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w23,w24			// a^b, b^c in next round
+	eor	w16,w16,w8,ror#11	// Sigma1(e)
+	ror	w8,w23,#2
+	add	w22,w22,w17			// h+=Ch(e,f,g)
+	eor	w17,w23,w23,ror#9
+	add	w22,w22,w16			// h+=Sigma1(e)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	add	w26,w26,w22			// d+=h
+	eor	w19,w19,w24			// Maj(a,b,c)
+	eor	w17,w8,w17,ror#13	// Sigma0(a)
+	add	w22,w22,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	//add	w22,w22,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w1,w1			// 14
+#endif
+	ldr	w6,[sp,#12]
+	add	w22,w22,w17			// h+=Sigma0(a)
+	str	w9,[sp,#8]
+	ror	w16,w26,#6
+	add	w21,w21,w19			// h+=K[i]
+	eor	w9,w26,w26,ror#14
+	and	w17,w27,w26
+	bic	w19,w20,w26
+	add	w21,w21,w1			// h+=X[i]
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w22,w23			// a^b, b^c in next round
+	eor	w16,w16,w9,ror#11	// Sigma1(e)
+	ror	w9,w22,#2
+	add	w21,w21,w17			// h+=Ch(e,f,g)
+	eor	w17,w22,w22,ror#9
+	add	w21,w21,w16			// h+=Sigma1(e)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	add	w25,w25,w21			// d+=h
+	eor	w28,w28,w23			// Maj(a,b,c)
+	eor	w17,w9,w17,ror#13	// Sigma0(a)
+	add	w21,w21,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	//add	w21,w21,w17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	w2,w2			// 15
+#endif
+	ldr	w7,[sp,#0]
+	add	w21,w21,w17			// h+=Sigma0(a)
+	str	w10,[sp,#12]
+	ror	w16,w25,#6
+	add	w20,w20,w28			// h+=K[i]
+	ror	w9,w4,#7
+	and	w17,w26,w25
+	ror	w8,w1,#17
+	bic	w28,w27,w25
+	ror	w10,w21,#2
+	add	w20,w20,w2			// h+=X[i]
+	eor	w16,w16,w25,ror#11
+	eor	w9,w9,w4,ror#18
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w21,w22			// a^b, b^c in next round
+	eor	w16,w16,w25,ror#25	// Sigma1(e)
+	eor	w10,w10,w21,ror#13
+	add	w20,w20,w17			// h+=Ch(e,f,g)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	eor	w8,w8,w1,ror#19
+	eor	w9,w9,w4,lsr#3	// sigma0(X[i+1])
+	add	w20,w20,w16			// h+=Sigma1(e)
+	eor	w19,w19,w22			// Maj(a,b,c)
+	eor	w17,w10,w21,ror#22	// Sigma0(a)
+	eor	w8,w8,w1,lsr#10	// sigma1(X[i+14])
+	add	w3,w3,w12
+	add	w24,w24,w20			// d+=h
+	add	w20,w20,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	add	w3,w3,w9
+	add	w20,w20,w17			// h+=Sigma0(a)
+	add	w3,w3,w8
+.Loop_16_xx:
+	ldr	w8,[sp,#4]
+	str	w11,[sp,#0]
+	ror	w16,w24,#6
+	add	w27,w27,w19			// h+=K[i]
+	ror	w10,w5,#7
+	and	w17,w25,w24
+	ror	w9,w2,#17
+	bic	w19,w26,w24
+	ror	w11,w20,#2
+	add	w27,w27,w3			// h+=X[i]
+	eor	w16,w16,w24,ror#11
+	eor	w10,w10,w5,ror#18
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w20,w21			// a^b, b^c in next round
+	eor	w16,w16,w24,ror#25	// Sigma1(e)
+	eor	w11,w11,w20,ror#13
+	add	w27,w27,w17			// h+=Ch(e,f,g)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	eor	w9,w9,w2,ror#19
+	eor	w10,w10,w5,lsr#3	// sigma0(X[i+1])
+	add	w27,w27,w16			// h+=Sigma1(e)
+	eor	w28,w28,w21			// Maj(a,b,c)
+	eor	w17,w11,w20,ror#22	// Sigma0(a)
+	eor	w9,w9,w2,lsr#10	// sigma1(X[i+14])
+	add	w4,w4,w13
+	add	w23,w23,w27			// d+=h
+	add	w27,w27,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	add	w4,w4,w10
+	add	w27,w27,w17			// h+=Sigma0(a)
+	add	w4,w4,w9
+	ldr	w9,[sp,#8]
+	str	w12,[sp,#4]
+	ror	w16,w23,#6
+	add	w26,w26,w28			// h+=K[i]
+	ror	w11,w6,#7
+	and	w17,w24,w23
+	ror	w10,w3,#17
+	bic	w28,w25,w23
+	ror	w12,w27,#2
+	add	w26,w26,w4			// h+=X[i]
+	eor	w16,w16,w23,ror#11
+	eor	w11,w11,w6,ror#18
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w27,w20			// a^b, b^c in next round
+	eor	w16,w16,w23,ror#25	// Sigma1(e)
+	eor	w12,w12,w27,ror#13
+	add	w26,w26,w17			// h+=Ch(e,f,g)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	eor	w10,w10,w3,ror#19
+	eor	w11,w11,w6,lsr#3	// sigma0(X[i+1])
+	add	w26,w26,w16			// h+=Sigma1(e)
+	eor	w19,w19,w20			// Maj(a,b,c)
+	eor	w17,w12,w27,ror#22	// Sigma0(a)
+	eor	w10,w10,w3,lsr#10	// sigma1(X[i+14])
+	add	w5,w5,w14
+	add	w22,w22,w26			// d+=h
+	add	w26,w26,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	add	w5,w5,w11
+	add	w26,w26,w17			// h+=Sigma0(a)
+	add	w5,w5,w10
+	ldr	w10,[sp,#12]
+	str	w13,[sp,#8]
+	ror	w16,w22,#6
+	add	w25,w25,w19			// h+=K[i]
+	ror	w12,w7,#7
+	and	w17,w23,w22
+	ror	w11,w4,#17
+	bic	w19,w24,w22
+	ror	w13,w26,#2
+	add	w25,w25,w5			// h+=X[i]
+	eor	w16,w16,w22,ror#11
+	eor	w12,w12,w7,ror#18
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w26,w27			// a^b, b^c in next round
+	eor	w16,w16,w22,ror#25	// Sigma1(e)
+	eor	w13,w13,w26,ror#13
+	add	w25,w25,w17			// h+=Ch(e,f,g)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	eor	w11,w11,w4,ror#19
+	eor	w12,w12,w7,lsr#3	// sigma0(X[i+1])
+	add	w25,w25,w16			// h+=Sigma1(e)
+	eor	w28,w28,w27			// Maj(a,b,c)
+	eor	w17,w13,w26,ror#22	// Sigma0(a)
+	eor	w11,w11,w4,lsr#10	// sigma1(X[i+14])
+	add	w6,w6,w15
+	add	w21,w21,w25			// d+=h
+	add	w25,w25,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	add	w6,w6,w12
+	add	w25,w25,w17			// h+=Sigma0(a)
+	add	w6,w6,w11
+	ldr	w11,[sp,#0]
+	str	w14,[sp,#12]
+	ror	w16,w21,#6
+	add	w24,w24,w28			// h+=K[i]
+	ror	w13,w8,#7
+	and	w17,w22,w21
+	ror	w12,w5,#17
+	bic	w28,w23,w21
+	ror	w14,w25,#2
+	add	w24,w24,w6			// h+=X[i]
+	eor	w16,w16,w21,ror#11
+	eor	w13,w13,w8,ror#18
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w25,w26			// a^b, b^c in next round
+	eor	w16,w16,w21,ror#25	// Sigma1(e)
+	eor	w14,w14,w25,ror#13
+	add	w24,w24,w17			// h+=Ch(e,f,g)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	eor	w12,w12,w5,ror#19
+	eor	w13,w13,w8,lsr#3	// sigma0(X[i+1])
+	add	w24,w24,w16			// h+=Sigma1(e)
+	eor	w19,w19,w26			// Maj(a,b,c)
+	eor	w17,w14,w25,ror#22	// Sigma0(a)
+	eor	w12,w12,w5,lsr#10	// sigma1(X[i+14])
+	add	w7,w7,w0
+	add	w20,w20,w24			// d+=h
+	add	w24,w24,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	add	w7,w7,w13
+	add	w24,w24,w17			// h+=Sigma0(a)
+	add	w7,w7,w12
+	ldr	w12,[sp,#4]
+	str	w15,[sp,#0]
+	ror	w16,w20,#6
+	add	w23,w23,w19			// h+=K[i]
+	ror	w14,w9,#7
+	and	w17,w21,w20
+	ror	w13,w6,#17
+	bic	w19,w22,w20
+	ror	w15,w24,#2
+	add	w23,w23,w7			// h+=X[i]
+	eor	w16,w16,w20,ror#11
+	eor	w14,w14,w9,ror#18
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w24,w25			// a^b, b^c in next round
+	eor	w16,w16,w20,ror#25	// Sigma1(e)
+	eor	w15,w15,w24,ror#13
+	add	w23,w23,w17			// h+=Ch(e,f,g)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	eor	w13,w13,w6,ror#19
+	eor	w14,w14,w9,lsr#3	// sigma0(X[i+1])
+	add	w23,w23,w16			// h+=Sigma1(e)
+	eor	w28,w28,w25			// Maj(a,b,c)
+	eor	w17,w15,w24,ror#22	// Sigma0(a)
+	eor	w13,w13,w6,lsr#10	// sigma1(X[i+14])
+	add	w8,w8,w1
+	add	w27,w27,w23			// d+=h
+	add	w23,w23,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	add	w8,w8,w14
+	add	w23,w23,w17			// h+=Sigma0(a)
+	add	w8,w8,w13
+	ldr	w13,[sp,#8]
+	str	w0,[sp,#4]
+	ror	w16,w27,#6
+	add	w22,w22,w28			// h+=K[i]
+	ror	w15,w10,#7
+	and	w17,w20,w27
+	ror	w14,w7,#17
+	bic	w28,w21,w27
+	ror	w0,w23,#2
+	add	w22,w22,w8			// h+=X[i]
+	eor	w16,w16,w27,ror#11
+	eor	w15,w15,w10,ror#18
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w23,w24			// a^b, b^c in next round
+	eor	w16,w16,w27,ror#25	// Sigma1(e)
+	eor	w0,w0,w23,ror#13
+	add	w22,w22,w17			// h+=Ch(e,f,g)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	eor	w14,w14,w7,ror#19
+	eor	w15,w15,w10,lsr#3	// sigma0(X[i+1])
+	add	w22,w22,w16			// h+=Sigma1(e)
+	eor	w19,w19,w24			// Maj(a,b,c)
+	eor	w17,w0,w23,ror#22	// Sigma0(a)
+	eor	w14,w14,w7,lsr#10	// sigma1(X[i+14])
+	add	w9,w9,w2
+	add	w26,w26,w22			// d+=h
+	add	w22,w22,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	add	w9,w9,w15
+	add	w22,w22,w17			// h+=Sigma0(a)
+	add	w9,w9,w14
+	ldr	w14,[sp,#12]
+	str	w1,[sp,#8]
+	ror	w16,w26,#6
+	add	w21,w21,w19			// h+=K[i]
+	ror	w0,w11,#7
+	and	w17,w27,w26
+	ror	w15,w8,#17
+	bic	w19,w20,w26
+	ror	w1,w22,#2
+	add	w21,w21,w9			// h+=X[i]
+	eor	w16,w16,w26,ror#11
+	eor	w0,w0,w11,ror#18
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w22,w23			// a^b, b^c in next round
+	eor	w16,w16,w26,ror#25	// Sigma1(e)
+	eor	w1,w1,w22,ror#13
+	add	w21,w21,w17			// h+=Ch(e,f,g)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	eor	w15,w15,w8,ror#19
+	eor	w0,w0,w11,lsr#3	// sigma0(X[i+1])
+	add	w21,w21,w16			// h+=Sigma1(e)
+	eor	w28,w28,w23			// Maj(a,b,c)
+	eor	w17,w1,w22,ror#22	// Sigma0(a)
+	eor	w15,w15,w8,lsr#10	// sigma1(X[i+14])
+	add	w10,w10,w3
+	add	w25,w25,w21			// d+=h
+	add	w21,w21,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	add	w10,w10,w0
+	add	w21,w21,w17			// h+=Sigma0(a)
+	add	w10,w10,w15
+	ldr	w15,[sp,#0]
+	str	w2,[sp,#12]
+	ror	w16,w25,#6
+	add	w20,w20,w28			// h+=K[i]
+	ror	w1,w12,#7
+	and	w17,w26,w25
+	ror	w0,w9,#17
+	bic	w28,w27,w25
+	ror	w2,w21,#2
+	add	w20,w20,w10			// h+=X[i]
+	eor	w16,w16,w25,ror#11
+	eor	w1,w1,w12,ror#18
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w21,w22			// a^b, b^c in next round
+	eor	w16,w16,w25,ror#25	// Sigma1(e)
+	eor	w2,w2,w21,ror#13
+	add	w20,w20,w17			// h+=Ch(e,f,g)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	eor	w0,w0,w9,ror#19
+	eor	w1,w1,w12,lsr#3	// sigma0(X[i+1])
+	add	w20,w20,w16			// h+=Sigma1(e)
+	eor	w19,w19,w22			// Maj(a,b,c)
+	eor	w17,w2,w21,ror#22	// Sigma0(a)
+	eor	w0,w0,w9,lsr#10	// sigma1(X[i+14])
+	add	w11,w11,w4
+	add	w24,w24,w20			// d+=h
+	add	w20,w20,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	add	w11,w11,w1
+	add	w20,w20,w17			// h+=Sigma0(a)
+	add	w11,w11,w0
+	ldr	w0,[sp,#4]
+	str	w3,[sp,#0]
+	ror	w16,w24,#6
+	add	w27,w27,w19			// h+=K[i]
+	ror	w2,w13,#7
+	and	w17,w25,w24
+	ror	w1,w10,#17
+	bic	w19,w26,w24
+	ror	w3,w20,#2
+	add	w27,w27,w11			// h+=X[i]
+	eor	w16,w16,w24,ror#11
+	eor	w2,w2,w13,ror#18
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w20,w21			// a^b, b^c in next round
+	eor	w16,w16,w24,ror#25	// Sigma1(e)
+	eor	w3,w3,w20,ror#13
+	add	w27,w27,w17			// h+=Ch(e,f,g)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	eor	w1,w1,w10,ror#19
+	eor	w2,w2,w13,lsr#3	// sigma0(X[i+1])
+	add	w27,w27,w16			// h+=Sigma1(e)
+	eor	w28,w28,w21			// Maj(a,b,c)
+	eor	w17,w3,w20,ror#22	// Sigma0(a)
+	eor	w1,w1,w10,lsr#10	// sigma1(X[i+14])
+	add	w12,w12,w5
+	add	w23,w23,w27			// d+=h
+	add	w27,w27,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	add	w12,w12,w2
+	add	w27,w27,w17			// h+=Sigma0(a)
+	add	w12,w12,w1
+	ldr	w1,[sp,#8]
+	str	w4,[sp,#4]
+	ror	w16,w23,#6
+	add	w26,w26,w28			// h+=K[i]
+	ror	w3,w14,#7
+	and	w17,w24,w23
+	ror	w2,w11,#17
+	bic	w28,w25,w23
+	ror	w4,w27,#2
+	add	w26,w26,w12			// h+=X[i]
+	eor	w16,w16,w23,ror#11
+	eor	w3,w3,w14,ror#18
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w27,w20			// a^b, b^c in next round
+	eor	w16,w16,w23,ror#25	// Sigma1(e)
+	eor	w4,w4,w27,ror#13
+	add	w26,w26,w17			// h+=Ch(e,f,g)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	eor	w2,w2,w11,ror#19
+	eor	w3,w3,w14,lsr#3	// sigma0(X[i+1])
+	add	w26,w26,w16			// h+=Sigma1(e)
+	eor	w19,w19,w20			// Maj(a,b,c)
+	eor	w17,w4,w27,ror#22	// Sigma0(a)
+	eor	w2,w2,w11,lsr#10	// sigma1(X[i+14])
+	add	w13,w13,w6
+	add	w22,w22,w26			// d+=h
+	add	w26,w26,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	add	w13,w13,w3
+	add	w26,w26,w17			// h+=Sigma0(a)
+	add	w13,w13,w2
+	ldr	w2,[sp,#12]
+	str	w5,[sp,#8]
+	ror	w16,w22,#6
+	add	w25,w25,w19			// h+=K[i]
+	ror	w4,w15,#7
+	and	w17,w23,w22
+	ror	w3,w12,#17
+	bic	w19,w24,w22
+	ror	w5,w26,#2
+	add	w25,w25,w13			// h+=X[i]
+	eor	w16,w16,w22,ror#11
+	eor	w4,w4,w15,ror#18
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w26,w27			// a^b, b^c in next round
+	eor	w16,w16,w22,ror#25	// Sigma1(e)
+	eor	w5,w5,w26,ror#13
+	add	w25,w25,w17			// h+=Ch(e,f,g)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	eor	w3,w3,w12,ror#19
+	eor	w4,w4,w15,lsr#3	// sigma0(X[i+1])
+	add	w25,w25,w16			// h+=Sigma1(e)
+	eor	w28,w28,w27			// Maj(a,b,c)
+	eor	w17,w5,w26,ror#22	// Sigma0(a)
+	eor	w3,w3,w12,lsr#10	// sigma1(X[i+14])
+	add	w14,w14,w7
+	add	w21,w21,w25			// d+=h
+	add	w25,w25,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	add	w14,w14,w4
+	add	w25,w25,w17			// h+=Sigma0(a)
+	add	w14,w14,w3
+	ldr	w3,[sp,#0]
+	str	w6,[sp,#12]
+	ror	w16,w21,#6
+	add	w24,w24,w28			// h+=K[i]
+	ror	w5,w0,#7
+	and	w17,w22,w21
+	ror	w4,w13,#17
+	bic	w28,w23,w21
+	ror	w6,w25,#2
+	add	w24,w24,w14			// h+=X[i]
+	eor	w16,w16,w21,ror#11
+	eor	w5,w5,w0,ror#18
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w25,w26			// a^b, b^c in next round
+	eor	w16,w16,w21,ror#25	// Sigma1(e)
+	eor	w6,w6,w25,ror#13
+	add	w24,w24,w17			// h+=Ch(e,f,g)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	eor	w4,w4,w13,ror#19
+	eor	w5,w5,w0,lsr#3	// sigma0(X[i+1])
+	add	w24,w24,w16			// h+=Sigma1(e)
+	eor	w19,w19,w26			// Maj(a,b,c)
+	eor	w17,w6,w25,ror#22	// Sigma0(a)
+	eor	w4,w4,w13,lsr#10	// sigma1(X[i+14])
+	add	w15,w15,w8
+	add	w20,w20,w24			// d+=h
+	add	w24,w24,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	add	w15,w15,w5
+	add	w24,w24,w17			// h+=Sigma0(a)
+	add	w15,w15,w4
+	ldr	w4,[sp,#4]
+	str	w7,[sp,#0]
+	ror	w16,w20,#6
+	add	w23,w23,w19			// h+=K[i]
+	ror	w6,w1,#7
+	and	w17,w21,w20
+	ror	w5,w14,#17
+	bic	w19,w22,w20
+	ror	w7,w24,#2
+	add	w23,w23,w15			// h+=X[i]
+	eor	w16,w16,w20,ror#11
+	eor	w6,w6,w1,ror#18
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w24,w25			// a^b, b^c in next round
+	eor	w16,w16,w20,ror#25	// Sigma1(e)
+	eor	w7,w7,w24,ror#13
+	add	w23,w23,w17			// h+=Ch(e,f,g)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	eor	w5,w5,w14,ror#19
+	eor	w6,w6,w1,lsr#3	// sigma0(X[i+1])
+	add	w23,w23,w16			// h+=Sigma1(e)
+	eor	w28,w28,w25			// Maj(a,b,c)
+	eor	w17,w7,w24,ror#22	// Sigma0(a)
+	eor	w5,w5,w14,lsr#10	// sigma1(X[i+14])
+	add	w0,w0,w9
+	add	w27,w27,w23			// d+=h
+	add	w23,w23,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	add	w0,w0,w6
+	add	w23,w23,w17			// h+=Sigma0(a)
+	add	w0,w0,w5
+	ldr	w5,[sp,#8]
+	str	w8,[sp,#4]
+	ror	w16,w27,#6
+	add	w22,w22,w28			// h+=K[i]
+	ror	w7,w2,#7
+	and	w17,w20,w27
+	ror	w6,w15,#17
+	bic	w28,w21,w27
+	ror	w8,w23,#2
+	add	w22,w22,w0			// h+=X[i]
+	eor	w16,w16,w27,ror#11
+	eor	w7,w7,w2,ror#18
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w23,w24			// a^b, b^c in next round
+	eor	w16,w16,w27,ror#25	// Sigma1(e)
+	eor	w8,w8,w23,ror#13
+	add	w22,w22,w17			// h+=Ch(e,f,g)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	eor	w6,w6,w15,ror#19
+	eor	w7,w7,w2,lsr#3	// sigma0(X[i+1])
+	add	w22,w22,w16			// h+=Sigma1(e)
+	eor	w19,w19,w24			// Maj(a,b,c)
+	eor	w17,w8,w23,ror#22	// Sigma0(a)
+	eor	w6,w6,w15,lsr#10	// sigma1(X[i+14])
+	add	w1,w1,w10
+	add	w26,w26,w22			// d+=h
+	add	w22,w22,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	add	w1,w1,w7
+	add	w22,w22,w17			// h+=Sigma0(a)
+	add	w1,w1,w6
+	ldr	w6,[sp,#12]
+	str	w9,[sp,#8]
+	ror	w16,w26,#6
+	add	w21,w21,w19			// h+=K[i]
+	ror	w8,w3,#7
+	and	w17,w27,w26
+	ror	w7,w0,#17
+	bic	w19,w20,w26
+	ror	w9,w22,#2
+	add	w21,w21,w1			// h+=X[i]
+	eor	w16,w16,w26,ror#11
+	eor	w8,w8,w3,ror#18
+	orr	w17,w17,w19			// Ch(e,f,g)
+	eor	w19,w22,w23			// a^b, b^c in next round
+	eor	w16,w16,w26,ror#25	// Sigma1(e)
+	eor	w9,w9,w22,ror#13
+	add	w21,w21,w17			// h+=Ch(e,f,g)
+	and	w28,w28,w19			// (b^c)&=(a^b)
+	eor	w7,w7,w0,ror#19
+	eor	w8,w8,w3,lsr#3	// sigma0(X[i+1])
+	add	w21,w21,w16			// h+=Sigma1(e)
+	eor	w28,w28,w23			// Maj(a,b,c)
+	eor	w17,w9,w22,ror#22	// Sigma0(a)
+	eor	w7,w7,w0,lsr#10	// sigma1(X[i+14])
+	add	w2,w2,w11
+	add	w25,w25,w21			// d+=h
+	add	w21,w21,w28			// h+=Maj(a,b,c)
+	ldr	w28,[x30],#4		// *K++, w19 in next round
+	add	w2,w2,w8
+	add	w21,w21,w17			// h+=Sigma0(a)
+	add	w2,w2,w7
+	ldr	w7,[sp,#0]
+	str	w10,[sp,#12]
+	ror	w16,w25,#6
+	add	w20,w20,w28			// h+=K[i]
+	ror	w9,w4,#7
+	and	w17,w26,w25
+	ror	w8,w1,#17
+	bic	w28,w27,w25
+	ror	w10,w21,#2
+	add	w20,w20,w2			// h+=X[i]
+	eor	w16,w16,w25,ror#11
+	eor	w9,w9,w4,ror#18
+	orr	w17,w17,w28			// Ch(e,f,g)
+	eor	w28,w21,w22			// a^b, b^c in next round
+	eor	w16,w16,w25,ror#25	// Sigma1(e)
+	eor	w10,w10,w21,ror#13
+	add	w20,w20,w17			// h+=Ch(e,f,g)
+	and	w19,w19,w28			// (b^c)&=(a^b)
+	eor	w8,w8,w1,ror#19
+	eor	w9,w9,w4,lsr#3	// sigma0(X[i+1])
+	add	w20,w20,w16			// h+=Sigma1(e)
+	eor	w19,w19,w22			// Maj(a,b,c)
+	eor	w17,w10,w21,ror#22	// Sigma0(a)
+	eor	w8,w8,w1,lsr#10	// sigma1(X[i+14])
+	add	w3,w3,w12
+	add	w24,w24,w20			// d+=h
+	add	w20,w20,w19			// h+=Maj(a,b,c)
+	ldr	w19,[x30],#4		// *K++, w28 in next round
+	add	w3,w3,w9
+	add	w20,w20,w17			// h+=Sigma0(a)
+	add	w3,w3,w8
+	cbnz	w19,.Loop_16_xx
+
+	ldp	x0,x2,[x29,#96]
+	ldr	x1,[x29,#112]
+	sub	x30,x30,#260		// rewind
+
+	ldp	w3,w4,[x0]
+	ldp	w5,w6,[x0,#2*4]
+	add	x1,x1,#14*4			// advance input pointer
+	ldp	w7,w8,[x0,#4*4]
+	add	w20,w20,w3
+	ldp	w9,w10,[x0,#6*4]
+	add	w21,w21,w4
+	add	w22,w22,w5
+	add	w23,w23,w6
+	stp	w20,w21,[x0]
+	add	w24,w24,w7
+	add	w25,w25,w8
+	stp	w22,w23,[x0,#2*4]
+	add	w26,w26,w9
+	add	w27,w27,w10
+	cmp	x1,x2
+	stp	w24,w25,[x0,#4*4]
+	stp	w26,w27,[x0,#6*4]
+	b.ne	.Loop
+
+	ldp	x19,x20,[x29,#16]
+	add	sp,sp,#4*4
+	ldp	x21,x22,[x29,#32]
+	ldp	x23,x24,[x29,#48]
+	ldp	x25,x26,[x29,#64]
+	ldp	x27,x28,[x29,#80]
+	ldp	x29,x30,[sp],#128
+	ret
+.size	sha256_block_data_order,.-sha256_block_data_order
+
+.align	6
+.type	.LK256,%object
+.LK256:
+	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
+	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
+	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
+	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
+	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
+	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
+	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
+	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
+	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
+	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
+	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
+	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
+	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
+	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
+	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
+	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
+	.long	0	//terminator
+.size	.LK256,.-.LK256
+#ifndef	__KERNEL__
+.align	3
+.LOPENSSL_armcap_P:
+# ifdef	__ILP32__
+	.long	OPENSSL_armcap_P-.
+# else
+	.quad	OPENSSL_armcap_P-.
+# endif
+#endif
+.asciz	"SHA256 block transform for ARMv8, CRYPTOGAMS by <appro@openssl.org>"
+.align	2
+#ifndef	__KERNEL__
+.type	sha256_block_armv8,%function
+.align	6
+sha256_block_armv8:
+.Lv8_entry:
+	stp		x29,x30,[sp,#-16]!
+	add		x29,sp,#0
+
+	ld1		{v0.4s,v1.4s},[x0]
+	adr		x3,.LK256
+
+.Loop_hw:
+	ld1		{v4.16b-v7.16b},[x1],#64
+	sub		x2,x2,#1
+	ld1		{v16.4s},[x3],#16
+	rev32		v4.16b,v4.16b
+	rev32		v5.16b,v5.16b
+	rev32		v6.16b,v6.16b
+	rev32		v7.16b,v7.16b
+	orr		v18.16b,v0.16b,v0.16b		// offload
+	orr		v19.16b,v1.16b,v1.16b
+	ld1		{v17.4s},[x3],#16
+	add		v16.4s,v16.4s,v4.4s
+	.inst	0x5e2828a4	//sha256su0 v4.16b,v5.16b
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e104020	//sha256h v0.16b,v1.16b,v16.4s
+	.inst	0x5e105041	//sha256h2 v1.16b,v2.16b,v16.4s
+	.inst	0x5e0760c4	//sha256su1 v4.16b,v6.16b,v7.16b
+	ld1		{v16.4s},[x3],#16
+	add		v17.4s,v17.4s,v5.4s
+	.inst	0x5e2828c5	//sha256su0 v5.16b,v6.16b
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e114020	//sha256h v0.16b,v1.16b,v17.4s
+	.inst	0x5e115041	//sha256h2 v1.16b,v2.16b,v17.4s
+	.inst	0x5e0460e5	//sha256su1 v5.16b,v7.16b,v4.16b
+	ld1		{v17.4s},[x3],#16
+	add		v16.4s,v16.4s,v6.4s
+	.inst	0x5e2828e6	//sha256su0 v6.16b,v7.16b
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e104020	//sha256h v0.16b,v1.16b,v16.4s
+	.inst	0x5e105041	//sha256h2 v1.16b,v2.16b,v16.4s
+	.inst	0x5e056086	//sha256su1 v6.16b,v4.16b,v5.16b
+	ld1		{v16.4s},[x3],#16
+	add		v17.4s,v17.4s,v7.4s
+	.inst	0x5e282887	//sha256su0 v7.16b,v4.16b
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e114020	//sha256h v0.16b,v1.16b,v17.4s
+	.inst	0x5e115041	//sha256h2 v1.16b,v2.16b,v17.4s
+	.inst	0x5e0660a7	//sha256su1 v7.16b,v5.16b,v6.16b
+	ld1		{v17.4s},[x3],#16
+	add		v16.4s,v16.4s,v4.4s
+	.inst	0x5e2828a4	//sha256su0 v4.16b,v5.16b
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e104020	//sha256h v0.16b,v1.16b,v16.4s
+	.inst	0x5e105041	//sha256h2 v1.16b,v2.16b,v16.4s
+	.inst	0x5e0760c4	//sha256su1 v4.16b,v6.16b,v7.16b
+	ld1		{v16.4s},[x3],#16
+	add		v17.4s,v17.4s,v5.4s
+	.inst	0x5e2828c5	//sha256su0 v5.16b,v6.16b
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e114020	//sha256h v0.16b,v1.16b,v17.4s
+	.inst	0x5e115041	//sha256h2 v1.16b,v2.16b,v17.4s
+	.inst	0x5e0460e5	//sha256su1 v5.16b,v7.16b,v4.16b
+	ld1		{v17.4s},[x3],#16
+	add		v16.4s,v16.4s,v6.4s
+	.inst	0x5e2828e6	//sha256su0 v6.16b,v7.16b
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e104020	//sha256h v0.16b,v1.16b,v16.4s
+	.inst	0x5e105041	//sha256h2 v1.16b,v2.16b,v16.4s
+	.inst	0x5e056086	//sha256su1 v6.16b,v4.16b,v5.16b
+	ld1		{v16.4s},[x3],#16
+	add		v17.4s,v17.4s,v7.4s
+	.inst	0x5e282887	//sha256su0 v7.16b,v4.16b
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e114020	//sha256h v0.16b,v1.16b,v17.4s
+	.inst	0x5e115041	//sha256h2 v1.16b,v2.16b,v17.4s
+	.inst	0x5e0660a7	//sha256su1 v7.16b,v5.16b,v6.16b
+	ld1		{v17.4s},[x3],#16
+	add		v16.4s,v16.4s,v4.4s
+	.inst	0x5e2828a4	//sha256su0 v4.16b,v5.16b
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e104020	//sha256h v0.16b,v1.16b,v16.4s
+	.inst	0x5e105041	//sha256h2 v1.16b,v2.16b,v16.4s
+	.inst	0x5e0760c4	//sha256su1 v4.16b,v6.16b,v7.16b
+	ld1		{v16.4s},[x3],#16
+	add		v17.4s,v17.4s,v5.4s
+	.inst	0x5e2828c5	//sha256su0 v5.16b,v6.16b
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e114020	//sha256h v0.16b,v1.16b,v17.4s
+	.inst	0x5e115041	//sha256h2 v1.16b,v2.16b,v17.4s
+	.inst	0x5e0460e5	//sha256su1 v5.16b,v7.16b,v4.16b
+	ld1		{v17.4s},[x3],#16
+	add		v16.4s,v16.4s,v6.4s
+	.inst	0x5e2828e6	//sha256su0 v6.16b,v7.16b
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e104020	//sha256h v0.16b,v1.16b,v16.4s
+	.inst	0x5e105041	//sha256h2 v1.16b,v2.16b,v16.4s
+	.inst	0x5e056086	//sha256su1 v6.16b,v4.16b,v5.16b
+	ld1		{v16.4s},[x3],#16
+	add		v17.4s,v17.4s,v7.4s
+	.inst	0x5e282887	//sha256su0 v7.16b,v4.16b
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e114020	//sha256h v0.16b,v1.16b,v17.4s
+	.inst	0x5e115041	//sha256h2 v1.16b,v2.16b,v17.4s
+	.inst	0x5e0660a7	//sha256su1 v7.16b,v5.16b,v6.16b
+	ld1		{v17.4s},[x3],#16
+	add		v16.4s,v16.4s,v4.4s
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e104020	//sha256h v0.16b,v1.16b,v16.4s
+	.inst	0x5e105041	//sha256h2 v1.16b,v2.16b,v16.4s
+
+	ld1		{v16.4s},[x3],#16
+	add		v17.4s,v17.4s,v5.4s
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e114020	//sha256h v0.16b,v1.16b,v17.4s
+	.inst	0x5e115041	//sha256h2 v1.16b,v2.16b,v17.4s
+
+	ld1		{v17.4s},[x3]
+	add		v16.4s,v16.4s,v6.4s
+	sub		x3,x3,#64*4-16	// rewind
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e104020	//sha256h v0.16b,v1.16b,v16.4s
+	.inst	0x5e105041	//sha256h2 v1.16b,v2.16b,v16.4s
+
+	add		v17.4s,v17.4s,v7.4s
+	orr		v2.16b,v0.16b,v0.16b
+	.inst	0x5e114020	//sha256h v0.16b,v1.16b,v17.4s
+	.inst	0x5e115041	//sha256h2 v1.16b,v2.16b,v17.4s
+
+	add		v0.4s,v0.4s,v18.4s
+	add		v1.4s,v1.4s,v19.4s
+
+	cbnz		x2,.Loop_hw
+
+	st1		{v0.4s,v1.4s},[x0]
+
+	ldr		x29,[sp],#16
+	ret
+.size	sha256_block_armv8,.-sha256_block_armv8
+#endif
+#ifdef	__KERNEL__
+.globl	sha256_block_neon
+#endif
+.type	sha256_block_neon,%function
+.align	4
+sha256_block_neon:
+.Lneon_entry:
+	stp	x29, x30, [sp, #-16]!
+	mov	x29, sp
+	sub	sp,sp,#16*4
+
+	adr	x16,.LK256
+	add	x2,x1,x2,lsl#6	// len to point at the end of inp
+
+	ld1	{v0.16b},[x1], #16
+	ld1	{v1.16b},[x1], #16
+	ld1	{v2.16b},[x1], #16
+	ld1	{v3.16b},[x1], #16
+	ld1	{v4.4s},[x16], #16
+	ld1	{v5.4s},[x16], #16
+	ld1	{v6.4s},[x16], #16
+	ld1	{v7.4s},[x16], #16
+	rev32	v0.16b,v0.16b		// yes, even on
+	rev32	v1.16b,v1.16b		// big-endian
+	rev32	v2.16b,v2.16b
+	rev32	v3.16b,v3.16b
+	mov	x17,sp
+	add	v4.4s,v4.4s,v0.4s
+	add	v5.4s,v5.4s,v1.4s
+	add	v6.4s,v6.4s,v2.4s
+	st1	{v4.4s-v5.4s},[x17], #32
+	add	v7.4s,v7.4s,v3.4s
+	st1	{v6.4s-v7.4s},[x17]
+	sub	x17,x17,#32
+
+	ldp	w3,w4,[x0]
+	ldp	w5,w6,[x0,#8]
+	ldp	w7,w8,[x0,#16]
+	ldp	w9,w10,[x0,#24]
+	ldr	w12,[sp,#0]
+	mov	w13,wzr
+	eor	w14,w4,w5
+	mov	w15,wzr
+	b	.L_00_48
+
+.align	4
+.L_00_48:
+	ext	v4.16b,v0.16b,v1.16b,#4
+	add	w10,w10,w12
+	add	w3,w3,w15
+	and	w12,w8,w7
+	bic	w15,w9,w7
+	ext	v7.16b,v2.16b,v3.16b,#4
+	eor	w11,w7,w7,ror#5
+	add	w3,w3,w13
+	mov	d19,v3.d[1]
+	orr	w12,w12,w15
+	eor	w11,w11,w7,ror#19
+	ushr	v6.4s,v4.4s,#7
+	eor	w15,w3,w3,ror#11
+	ushr	v5.4s,v4.4s,#3
+	add	w10,w10,w12
+	add	v0.4s,v0.4s,v7.4s
+	ror	w11,w11,#6
+	sli	v6.4s,v4.4s,#25
+	eor	w13,w3,w4
+	eor	w15,w15,w3,ror#20
+	ushr	v7.4s,v4.4s,#18
+	add	w10,w10,w11
+	ldr	w12,[sp,#4]
+	and	w14,w14,w13
+	eor	v5.16b,v5.16b,v6.16b
+	ror	w15,w15,#2
+	add	w6,w6,w10
+	sli	v7.4s,v4.4s,#14
+	eor	w14,w14,w4
+	ushr	v16.4s,v19.4s,#17
+	add	w9,w9,w12
+	add	w10,w10,w15
+	and	w12,w7,w6
+	eor	v5.16b,v5.16b,v7.16b
+	bic	w15,w8,w6
+	eor	w11,w6,w6,ror#5
+	sli	v16.4s,v19.4s,#15
+	add	w10,w10,w14
+	orr	w12,w12,w15
+	ushr	v17.4s,v19.4s,#10
+	eor	w11,w11,w6,ror#19
+	eor	w15,w10,w10,ror#11
+	ushr	v7.4s,v19.4s,#19
+	add	w9,w9,w12
+	ror	w11,w11,#6
+	add	v0.4s,v0.4s,v5.4s
+	eor	w14,w10,w3
+	eor	w15,w15,w10,ror#20
+	sli	v7.4s,v19.4s,#13
+	add	w9,w9,w11
+	ldr	w12,[sp,#8]
+	and	w13,w13,w14
+	eor	v17.16b,v17.16b,v16.16b
+	ror	w15,w15,#2
+	add	w5,w5,w9
+	eor	w13,w13,w3
+	eor	v17.16b,v17.16b,v7.16b
+	add	w8,w8,w12
+	add	w9,w9,w15
+	and	w12,w6,w5
+	add	v0.4s,v0.4s,v17.4s
+	bic	w15,w7,w5
+	eor	w11,w5,w5,ror#5
+	add	w9,w9,w13
+	ushr	v18.4s,v0.4s,#17
+	orr	w12,w12,w15
+	ushr	v19.4s,v0.4s,#10
+	eor	w11,w11,w5,ror#19
+	eor	w15,w9,w9,ror#11
+	sli	v18.4s,v0.4s,#15
+	add	w8,w8,w12
+	ushr	v17.4s,v0.4s,#19
+	ror	w11,w11,#6
+	eor	w13,w9,w10
+	eor	v19.16b,v19.16b,v18.16b
+	eor	w15,w15,w9,ror#20
+	add	w8,w8,w11
+	sli	v17.4s,v0.4s,#13
+	ldr	w12,[sp,#12]
+	and	w14,w14,w13
+	ror	w15,w15,#2
+	ld1	{v4.4s},[x16], #16
+	add	w4,w4,w8
+	eor	v19.16b,v19.16b,v17.16b
+	eor	w14,w14,w10
+	eor	v17.16b,v17.16b,v17.16b
+	add	w7,w7,w12
+	add	w8,w8,w15
+	and	w12,w5,w4
+	mov	v17.d[1],v19.d[0]
+	bic	w15,w6,w4
+	eor	w11,w4,w4,ror#5
+	add	w8,w8,w14
+	add	v0.4s,v0.4s,v17.4s
+	orr	w12,w12,w15
+	eor	w11,w11,w4,ror#19
+	eor	w15,w8,w8,ror#11
+	add	v4.4s,v4.4s,v0.4s
+	add	w7,w7,w12
+	ror	w11,w11,#6
+	eor	w14,w8,w9
+	eor	w15,w15,w8,ror#20
+	add	w7,w7,w11
+	ldr	w12,[sp,#16]
+	and	w13,w13,w14
+	ror	w15,w15,#2
+	add	w3,w3,w7
+	eor	w13,w13,w9
+	st1	{v4.4s},[x17], #16
+	ext	v4.16b,v1.16b,v2.16b,#4
+	add	w6,w6,w12
+	add	w7,w7,w15
+	and	w12,w4,w3
+	bic	w15,w5,w3
+	ext	v7.16b,v3.16b,v0.16b,#4
+	eor	w11,w3,w3,ror#5
+	add	w7,w7,w13
+	mov	d19,v0.d[1]
+	orr	w12,w12,w15
+	eor	w11,w11,w3,ror#19
+	ushr	v6.4s,v4.4s,#7
+	eor	w15,w7,w7,ror#11
+	ushr	v5.4s,v4.4s,#3
+	add	w6,w6,w12
+	add	v1.4s,v1.4s,v7.4s
+	ror	w11,w11,#6
+	sli	v6.4s,v4.4s,#25
+	eor	w13,w7,w8
+	eor	w15,w15,w7,ror#20
+	ushr	v7.4s,v4.4s,#18
+	add	w6,w6,w11
+	ldr	w12,[sp,#20]
+	and	w14,w14,w13
+	eor	v5.16b,v5.16b,v6.16b
+	ror	w15,w15,#2
+	add	w10,w10,w6
+	sli	v7.4s,v4.4s,#14
+	eor	w14,w14,w8
+	ushr	v16.4s,v19.4s,#17
+	add	w5,w5,w12
+	add	w6,w6,w15
+	and	w12,w3,w10
+	eor	v5.16b,v5.16b,v7.16b
+	bic	w15,w4,w10
+	eor	w11,w10,w10,ror#5
+	sli	v16.4s,v19.4s,#15
+	add	w6,w6,w14
+	orr	w12,w12,w15
+	ushr	v17.4s,v19.4s,#10
+	eor	w11,w11,w10,ror#19
+	eor	w15,w6,w6,ror#11
+	ushr	v7.4s,v19.4s,#19
+	add	w5,w5,w12
+	ror	w11,w11,#6
+	add	v1.4s,v1.4s,v5.4s
+	eor	w14,w6,w7
+	eor	w15,w15,w6,ror#20
+	sli	v7.4s,v19.4s,#13
+	add	w5,w5,w11
+	ldr	w12,[sp,#24]
+	and	w13,w13,w14
+	eor	v17.16b,v17.16b,v16.16b
+	ror	w15,w15,#2
+	add	w9,w9,w5
+	eor	w13,w13,w7
+	eor	v17.16b,v17.16b,v7.16b
+	add	w4,w4,w12
+	add	w5,w5,w15
+	and	w12,w10,w9
+	add	v1.4s,v1.4s,v17.4s
+	bic	w15,w3,w9
+	eor	w11,w9,w9,ror#5
+	add	w5,w5,w13
+	ushr	v18.4s,v1.4s,#17
+	orr	w12,w12,w15
+	ushr	v19.4s,v1.4s,#10
+	eor	w11,w11,w9,ror#19
+	eor	w15,w5,w5,ror#11
+	sli	v18.4s,v1.4s,#15
+	add	w4,w4,w12
+	ushr	v17.4s,v1.4s,#19
+	ror	w11,w11,#6
+	eor	w13,w5,w6
+	eor	v19.16b,v19.16b,v18.16b
+	eor	w15,w15,w5,ror#20
+	add	w4,w4,w11
+	sli	v17.4s,v1.4s,#13
+	ldr	w12,[sp,#28]
+	and	w14,w14,w13
+	ror	w15,w15,#2
+	ld1	{v4.4s},[x16], #16
+	add	w8,w8,w4
+	eor	v19.16b,v19.16b,v17.16b
+	eor	w14,w14,w6
+	eor	v17.16b,v17.16b,v17.16b
+	add	w3,w3,w12
+	add	w4,w4,w15
+	and	w12,w9,w8
+	mov	v17.d[1],v19.d[0]
+	bic	w15,w10,w8
+	eor	w11,w8,w8,ror#5
+	add	w4,w4,w14
+	add	v1.4s,v1.4s,v17.4s
+	orr	w12,w12,w15
+	eor	w11,w11,w8,ror#19
+	eor	w15,w4,w4,ror#11
+	add	v4.4s,v4.4s,v1.4s
+	add	w3,w3,w12
+	ror	w11,w11,#6
+	eor	w14,w4,w5
+	eor	w15,w15,w4,ror#20
+	add	w3,w3,w11
+	ldr	w12,[sp,#32]
+	and	w13,w13,w14
+	ror	w15,w15,#2
+	add	w7,w7,w3
+	eor	w13,w13,w5
+	st1	{v4.4s},[x17], #16
+	ext	v4.16b,v2.16b,v3.16b,#4
+	add	w10,w10,w12
+	add	w3,w3,w15
+	and	w12,w8,w7
+	bic	w15,w9,w7
+	ext	v7.16b,v0.16b,v1.16b,#4
+	eor	w11,w7,w7,ror#5
+	add	w3,w3,w13
+	mov	d19,v1.d[1]
+	orr	w12,w12,w15
+	eor	w11,w11,w7,ror#19
+	ushr	v6.4s,v4.4s,#7
+	eor	w15,w3,w3,ror#11
+	ushr	v5.4s,v4.4s,#3
+	add	w10,w10,w12
+	add	v2.4s,v2.4s,v7.4s
+	ror	w11,w11,#6
+	sli	v6.4s,v4.4s,#25
+	eor	w13,w3,w4
+	eor	w15,w15,w3,ror#20
+	ushr	v7.4s,v4.4s,#18
+	add	w10,w10,w11
+	ldr	w12,[sp,#36]
+	and	w14,w14,w13
+	eor	v5.16b,v5.16b,v6.16b
+	ror	w15,w15,#2
+	add	w6,w6,w10
+	sli	v7.4s,v4.4s,#14
+	eor	w14,w14,w4
+	ushr	v16.4s,v19.4s,#17
+	add	w9,w9,w12
+	add	w10,w10,w15
+	and	w12,w7,w6
+	eor	v5.16b,v5.16b,v7.16b
+	bic	w15,w8,w6
+	eor	w11,w6,w6,ror#5
+	sli	v16.4s,v19.4s,#15
+	add	w10,w10,w14
+	orr	w12,w12,w15
+	ushr	v17.4s,v19.4s,#10
+	eor	w11,w11,w6,ror#19
+	eor	w15,w10,w10,ror#11
+	ushr	v7.4s,v19.4s,#19
+	add	w9,w9,w12
+	ror	w11,w11,#6
+	add	v2.4s,v2.4s,v5.4s
+	eor	w14,w10,w3
+	eor	w15,w15,w10,ror#20
+	sli	v7.4s,v19.4s,#13
+	add	w9,w9,w11
+	ldr	w12,[sp,#40]
+	and	w13,w13,w14
+	eor	v17.16b,v17.16b,v16.16b
+	ror	w15,w15,#2
+	add	w5,w5,w9
+	eor	w13,w13,w3
+	eor	v17.16b,v17.16b,v7.16b
+	add	w8,w8,w12
+	add	w9,w9,w15
+	and	w12,w6,w5
+	add	v2.4s,v2.4s,v17.4s
+	bic	w15,w7,w5
+	eor	w11,w5,w5,ror#5
+	add	w9,w9,w13
+	ushr	v18.4s,v2.4s,#17
+	orr	w12,w12,w15
+	ushr	v19.4s,v2.4s,#10
+	eor	w11,w11,w5,ror#19
+	eor	w15,w9,w9,ror#11
+	sli	v18.4s,v2.4s,#15
+	add	w8,w8,w12
+	ushr	v17.4s,v2.4s,#19
+	ror	w11,w11,#6
+	eor	w13,w9,w10
+	eor	v19.16b,v19.16b,v18.16b
+	eor	w15,w15,w9,ror#20
+	add	w8,w8,w11
+	sli	v17.4s,v2.4s,#13
+	ldr	w12,[sp,#44]
+	and	w14,w14,w13
+	ror	w15,w15,#2
+	ld1	{v4.4s},[x16], #16
+	add	w4,w4,w8
+	eor	v19.16b,v19.16b,v17.16b
+	eor	w14,w14,w10
+	eor	v17.16b,v17.16b,v17.16b
+	add	w7,w7,w12
+	add	w8,w8,w15
+	and	w12,w5,w4
+	mov	v17.d[1],v19.d[0]
+	bic	w15,w6,w4
+	eor	w11,w4,w4,ror#5
+	add	w8,w8,w14
+	add	v2.4s,v2.4s,v17.4s
+	orr	w12,w12,w15
+	eor	w11,w11,w4,ror#19
+	eor	w15,w8,w8,ror#11
+	add	v4.4s,v4.4s,v2.4s
+	add	w7,w7,w12
+	ror	w11,w11,#6
+	eor	w14,w8,w9
+	eor	w15,w15,w8,ror#20
+	add	w7,w7,w11
+	ldr	w12,[sp,#48]
+	and	w13,w13,w14
+	ror	w15,w15,#2
+	add	w3,w3,w7
+	eor	w13,w13,w9
+	st1	{v4.4s},[x17], #16
+	ext	v4.16b,v3.16b,v0.16b,#4
+	add	w6,w6,w12
+	add	w7,w7,w15
+	and	w12,w4,w3
+	bic	w15,w5,w3
+	ext	v7.16b,v1.16b,v2.16b,#4
+	eor	w11,w3,w3,ror#5
+	add	w7,w7,w13
+	mov	d19,v2.d[1]
+	orr	w12,w12,w15
+	eor	w11,w11,w3,ror#19
+	ushr	v6.4s,v4.4s,#7
+	eor	w15,w7,w7,ror#11
+	ushr	v5.4s,v4.4s,#3
+	add	w6,w6,w12
+	add	v3.4s,v3.4s,v7.4s
+	ror	w11,w11,#6
+	sli	v6.4s,v4.4s,#25
+	eor	w13,w7,w8
+	eor	w15,w15,w7,ror#20
+	ushr	v7.4s,v4.4s,#18
+	add	w6,w6,w11
+	ldr	w12,[sp,#52]
+	and	w14,w14,w13
+	eor	v5.16b,v5.16b,v6.16b
+	ror	w15,w15,#2
+	add	w10,w10,w6
+	sli	v7.4s,v4.4s,#14
+	eor	w14,w14,w8
+	ushr	v16.4s,v19.4s,#17
+	add	w5,w5,w12
+	add	w6,w6,w15
+	and	w12,w3,w10
+	eor	v5.16b,v5.16b,v7.16b
+	bic	w15,w4,w10
+	eor	w11,w10,w10,ror#5
+	sli	v16.4s,v19.4s,#15
+	add	w6,w6,w14
+	orr	w12,w12,w15
+	ushr	v17.4s,v19.4s,#10
+	eor	w11,w11,w10,ror#19
+	eor	w15,w6,w6,ror#11
+	ushr	v7.4s,v19.4s,#19
+	add	w5,w5,w12
+	ror	w11,w11,#6
+	add	v3.4s,v3.4s,v5.4s
+	eor	w14,w6,w7
+	eor	w15,w15,w6,ror#20
+	sli	v7.4s,v19.4s,#13
+	add	w5,w5,w11
+	ldr	w12,[sp,#56]
+	and	w13,w13,w14
+	eor	v17.16b,v17.16b,v16.16b
+	ror	w15,w15,#2
+	add	w9,w9,w5
+	eor	w13,w13,w7
+	eor	v17.16b,v17.16b,v7.16b
+	add	w4,w4,w12
+	add	w5,w5,w15
+	and	w12,w10,w9
+	add	v3.4s,v3.4s,v17.4s
+	bic	w15,w3,w9
+	eor	w11,w9,w9,ror#5
+	add	w5,w5,w13
+	ushr	v18.4s,v3.4s,#17
+	orr	w12,w12,w15
+	ushr	v19.4s,v3.4s,#10
+	eor	w11,w11,w9,ror#19
+	eor	w15,w5,w5,ror#11
+	sli	v18.4s,v3.4s,#15
+	add	w4,w4,w12
+	ushr	v17.4s,v3.4s,#19
+	ror	w11,w11,#6
+	eor	w13,w5,w6
+	eor	v19.16b,v19.16b,v18.16b
+	eor	w15,w15,w5,ror#20
+	add	w4,w4,w11
+	sli	v17.4s,v3.4s,#13
+	ldr	w12,[sp,#60]
+	and	w14,w14,w13
+	ror	w15,w15,#2
+	ld1	{v4.4s},[x16], #16
+	add	w8,w8,w4
+	eor	v19.16b,v19.16b,v17.16b
+	eor	w14,w14,w6
+	eor	v17.16b,v17.16b,v17.16b
+	add	w3,w3,w12
+	add	w4,w4,w15
+	and	w12,w9,w8
+	mov	v17.d[1],v19.d[0]
+	bic	w15,w10,w8
+	eor	w11,w8,w8,ror#5
+	add	w4,w4,w14
+	add	v3.4s,v3.4s,v17.4s
+	orr	w12,w12,w15
+	eor	w11,w11,w8,ror#19
+	eor	w15,w4,w4,ror#11
+	add	v4.4s,v4.4s,v3.4s
+	add	w3,w3,w12
+	ror	w11,w11,#6
+	eor	w14,w4,w5
+	eor	w15,w15,w4,ror#20
+	add	w3,w3,w11
+	ldr	w12,[x16]
+	and	w13,w13,w14
+	ror	w15,w15,#2
+	add	w7,w7,w3
+	eor	w13,w13,w5
+	st1	{v4.4s},[x17], #16
+	cmp	w12,#0				// check for K256 terminator
+	ldr	w12,[sp,#0]
+	sub	x17,x17,#64
+	bne	.L_00_48
+
+	sub	x16,x16,#256		// rewind x16
+	cmp	x1,x2
+	mov	x17, #64
+	csel	x17, x17, xzr, eq
+	sub	x1,x1,x17			// avoid SEGV
+	mov	x17,sp
+	add	w10,w10,w12
+	add	w3,w3,w15
+	and	w12,w8,w7
+	ld1	{v0.16b},[x1],#16
+	bic	w15,w9,w7
+	eor	w11,w7,w7,ror#5
+	ld1	{v4.4s},[x16],#16
+	add	w3,w3,w13
+	orr	w12,w12,w15
+	eor	w11,w11,w7,ror#19
+	eor	w15,w3,w3,ror#11
+	rev32	v0.16b,v0.16b
+	add	w10,w10,w12
+	ror	w11,w11,#6
+	eor	w13,w3,w4
+	eor	w15,w15,w3,ror#20
+	add	v4.4s,v4.4s,v0.4s
+	add	w10,w10,w11
+	ldr	w12,[sp,#4]
+	and	w14,w14,w13
+	ror	w15,w15,#2
+	add	w6,w6,w10
+	eor	w14,w14,w4
+	add	w9,w9,w12
+	add	w10,w10,w15
+	and	w12,w7,w6
+	bic	w15,w8,w6
+	eor	w11,w6,w6,ror#5
+	add	w10,w10,w14
+	orr	w12,w12,w15
+	eor	w11,w11,w6,ror#19
+	eor	w15,w10,w10,ror#11
+	add	w9,w9,w12
+	ror	w11,w11,#6
+	eor	w14,w10,w3
+	eor	w15,w15,w10,ror#20
+	add	w9,w9,w11
+	ldr	w12,[sp,#8]
+	and	w13,w13,w14
+	ror	w15,w15,#2
+	add	w5,w5,w9
+	eor	w13,w13,w3
+	add	w8,w8,w12
+	add	w9,w9,w15
+	and	w12,w6,w5
+	bic	w15,w7,w5
+	eor	w11,w5,w5,ror#5
+	add	w9,w9,w13
+	orr	w12,w12,w15
+	eor	w11,w11,w5,ror#19
+	eor	w15,w9,w9,ror#11
+	add	w8,w8,w12
+	ror	w11,w11,#6
+	eor	w13,w9,w10
+	eor	w15,w15,w9,ror#20
+	add	w8,w8,w11
+	ldr	w12,[sp,#12]
+	and	w14,w14,w13
+	ror	w15,w15,#2
+	add	w4,w4,w8
+	eor	w14,w14,w10
+	add	w7,w7,w12
+	add	w8,w8,w15
+	and	w12,w5,w4
+	bic	w15,w6,w4
+	eor	w11,w4,w4,ror#5
+	add	w8,w8,w14
+	orr	w12,w12,w15
+	eor	w11,w11,w4,ror#19
+	eor	w15,w8,w8,ror#11
+	add	w7,w7,w12
+	ror	w11,w11,#6
+	eor	w14,w8,w9
+	eor	w15,w15,w8,ror#20
+	add	w7,w7,w11
+	ldr	w12,[sp,#16]
+	and	w13,w13,w14
+	ror	w15,w15,#2
+	add	w3,w3,w7
+	eor	w13,w13,w9
+	st1	{v4.4s},[x17], #16
+	add	w6,w6,w12
+	add	w7,w7,w15
+	and	w12,w4,w3
+	ld1	{v1.16b},[x1],#16
+	bic	w15,w5,w3
+	eor	w11,w3,w3,ror#5
+	ld1	{v4.4s},[x16],#16
+	add	w7,w7,w13
+	orr	w12,w12,w15
+	eor	w11,w11,w3,ror#19
+	eor	w15,w7,w7,ror#11
+	rev32	v1.16b,v1.16b
+	add	w6,w6,w12
+	ror	w11,w11,#6
+	eor	w13,w7,w8
+	eor	w15,w15,w7,ror#20
+	add	v4.4s,v4.4s,v1.4s
+	add	w6,w6,w11
+	ldr	w12,[sp,#20]
+	and	w14,w14,w13
+	ror	w15,w15,#2
+	add	w10,w10,w6
+	eor	w14,w14,w8
+	add	w5,w5,w12
+	add	w6,w6,w15
+	and	w12,w3,w10
+	bic	w15,w4,w10
+	eor	w11,w10,w10,ror#5
+	add	w6,w6,w14
+	orr	w12,w12,w15
+	eor	w11,w11,w10,ror#19
+	eor	w15,w6,w6,ror#11
+	add	w5,w5,w12
+	ror	w11,w11,#6
+	eor	w14,w6,w7
+	eor	w15,w15,w6,ror#20
+	add	w5,w5,w11
+	ldr	w12,[sp,#24]
+	and	w13,w13,w14
+	ror	w15,w15,#2
+	add	w9,w9,w5
+	eor	w13,w13,w7
+	add	w4,w4,w12
+	add	w5,w5,w15
+	and	w12,w10,w9
+	bic	w15,w3,w9
+	eor	w11,w9,w9,ror#5
+	add	w5,w5,w13
+	orr	w12,w12,w15
+	eor	w11,w11,w9,ror#19
+	eor	w15,w5,w5,ror#11
+	add	w4,w4,w12
+	ror	w11,w11,#6
+	eor	w13,w5,w6
+	eor	w15,w15,w5,ror#20
+	add	w4,w4,w11
+	ldr	w12,[sp,#28]
+	and	w14,w14,w13
+	ror	w15,w15,#2
+	add	w8,w8,w4
+	eor	w14,w14,w6
+	add	w3,w3,w12
+	add	w4,w4,w15
+	and	w12,w9,w8
+	bic	w15,w10,w8
+	eor	w11,w8,w8,ror#5
+	add	w4,w4,w14
+	orr	w12,w12,w15
+	eor	w11,w11,w8,ror#19
+	eor	w15,w4,w4,ror#11
+	add	w3,w3,w12
+	ror	w11,w11,#6
+	eor	w14,w4,w5
+	eor	w15,w15,w4,ror#20
+	add	w3,w3,w11
+	ldr	w12,[sp,#32]
+	and	w13,w13,w14
+	ror	w15,w15,#2
+	add	w7,w7,w3
+	eor	w13,w13,w5
+	st1	{v4.4s},[x17], #16
+	add	w10,w10,w12
+	add	w3,w3,w15
+	and	w12,w8,w7
+	ld1	{v2.16b},[x1],#16
+	bic	w15,w9,w7
+	eor	w11,w7,w7,ror#5
+	ld1	{v4.4s},[x16],#16
+	add	w3,w3,w13
+	orr	w12,w12,w15
+	eor	w11,w11,w7,ror#19
+	eor	w15,w3,w3,ror#11
+	rev32	v2.16b,v2.16b
+	add	w10,w10,w12
+	ror	w11,w11,#6
+	eor	w13,w3,w4
+	eor	w15,w15,w3,ror#20
+	add	v4.4s,v4.4s,v2.4s
+	add	w10,w10,w11
+	ldr	w12,[sp,#36]
+	and	w14,w14,w13
+	ror	w15,w15,#2
+	add	w6,w6,w10
+	eor	w14,w14,w4
+	add	w9,w9,w12
+	add	w10,w10,w15
+	and	w12,w7,w6
+	bic	w15,w8,w6
+	eor	w11,w6,w6,ror#5
+	add	w10,w10,w14
+	orr	w12,w12,w15
+	eor	w11,w11,w6,ror#19
+	eor	w15,w10,w10,ror#11
+	add	w9,w9,w12
+	ror	w11,w11,#6
+	eor	w14,w10,w3
+	eor	w15,w15,w10,ror#20
+	add	w9,w9,w11
+	ldr	w12,[sp,#40]
+	and	w13,w13,w14
+	ror	w15,w15,#2
+	add	w5,w5,w9
+	eor	w13,w13,w3
+	add	w8,w8,w12
+	add	w9,w9,w15
+	and	w12,w6,w5
+	bic	w15,w7,w5
+	eor	w11,w5,w5,ror#5
+	add	w9,w9,w13
+	orr	w12,w12,w15
+	eor	w11,w11,w5,ror#19
+	eor	w15,w9,w9,ror#11
+	add	w8,w8,w12
+	ror	w11,w11,#6
+	eor	w13,w9,w10
+	eor	w15,w15,w9,ror#20
+	add	w8,w8,w11
+	ldr	w12,[sp,#44]
+	and	w14,w14,w13
+	ror	w15,w15,#2
+	add	w4,w4,w8
+	eor	w14,w14,w10
+	add	w7,w7,w12
+	add	w8,w8,w15
+	and	w12,w5,w4
+	bic	w15,w6,w4
+	eor	w11,w4,w4,ror#5
+	add	w8,w8,w14
+	orr	w12,w12,w15
+	eor	w11,w11,w4,ror#19
+	eor	w15,w8,w8,ror#11
+	add	w7,w7,w12
+	ror	w11,w11,#6
+	eor	w14,w8,w9
+	eor	w15,w15,w8,ror#20
+	add	w7,w7,w11
+	ldr	w12,[sp,#48]
+	and	w13,w13,w14
+	ror	w15,w15,#2
+	add	w3,w3,w7
+	eor	w13,w13,w9
+	st1	{v4.4s},[x17], #16
+	add	w6,w6,w12
+	add	w7,w7,w15
+	and	w12,w4,w3
+	ld1	{v3.16b},[x1],#16
+	bic	w15,w5,w3
+	eor	w11,w3,w3,ror#5
+	ld1	{v4.4s},[x16],#16
+	add	w7,w7,w13
+	orr	w12,w12,w15
+	eor	w11,w11,w3,ror#19
+	eor	w15,w7,w7,ror#11
+	rev32	v3.16b,v3.16b
+	add	w6,w6,w12
+	ror	w11,w11,#6
+	eor	w13,w7,w8
+	eor	w15,w15,w7,ror#20
+	add	v4.4s,v4.4s,v3.4s
+	add	w6,w6,w11
+	ldr	w12,[sp,#52]
+	and	w14,w14,w13
+	ror	w15,w15,#2
+	add	w10,w10,w6
+	eor	w14,w14,w8
+	add	w5,w5,w12
+	add	w6,w6,w15
+	and	w12,w3,w10
+	bic	w15,w4,w10
+	eor	w11,w10,w10,ror#5
+	add	w6,w6,w14
+	orr	w12,w12,w15
+	eor	w11,w11,w10,ror#19
+	eor	w15,w6,w6,ror#11
+	add	w5,w5,w12
+	ror	w11,w11,#6
+	eor	w14,w6,w7
+	eor	w15,w15,w6,ror#20
+	add	w5,w5,w11
+	ldr	w12,[sp,#56]
+	and	w13,w13,w14
+	ror	w15,w15,#2
+	add	w9,w9,w5
+	eor	w13,w13,w7
+	add	w4,w4,w12
+	add	w5,w5,w15
+	and	w12,w10,w9
+	bic	w15,w3,w9
+	eor	w11,w9,w9,ror#5
+	add	w5,w5,w13
+	orr	w12,w12,w15
+	eor	w11,w11,w9,ror#19
+	eor	w15,w5,w5,ror#11
+	add	w4,w4,w12
+	ror	w11,w11,#6
+	eor	w13,w5,w6
+	eor	w15,w15,w5,ror#20
+	add	w4,w4,w11
+	ldr	w12,[sp,#60]
+	and	w14,w14,w13
+	ror	w15,w15,#2
+	add	w8,w8,w4
+	eor	w14,w14,w6
+	add	w3,w3,w12
+	add	w4,w4,w15
+	and	w12,w9,w8
+	bic	w15,w10,w8
+	eor	w11,w8,w8,ror#5
+	add	w4,w4,w14
+	orr	w12,w12,w15
+	eor	w11,w11,w8,ror#19
+	eor	w15,w4,w4,ror#11
+	add	w3,w3,w12
+	ror	w11,w11,#6
+	eor	w14,w4,w5
+	eor	w15,w15,w4,ror#20
+	add	w3,w3,w11
+	and	w13,w13,w14
+	ror	w15,w15,#2
+	add	w7,w7,w3
+	eor	w13,w13,w5
+	st1	{v4.4s},[x17], #16
+	add	w3,w3,w15			// h+=Sigma0(a) from the past
+	ldp	w11,w12,[x0,#0]
+	add	w3,w3,w13			// h+=Maj(a,b,c) from the past
+	ldp	w13,w14,[x0,#8]
+	add	w3,w3,w11			// accumulate
+	add	w4,w4,w12
+	ldp	w11,w12,[x0,#16]
+	add	w5,w5,w13
+	add	w6,w6,w14
+	ldp	w13,w14,[x0,#24]
+	add	w7,w7,w11
+	add	w8,w8,w12
+	 ldr	w12,[sp,#0]
+	stp	w3,w4,[x0,#0]
+	add	w9,w9,w13
+	 mov	w13,wzr
+	stp	w5,w6,[x0,#8]
+	add	w10,w10,w14
+	stp	w7,w8,[x0,#16]
+	 eor	w14,w4,w5
+	stp	w9,w10,[x0,#24]
+	 mov	w15,wzr
+	 mov	x17,sp
+	b.ne	.L_00_48
+
+	ldr	x29,[x29]
+	add	sp,sp,#16*4+16
+	ret
+.size	sha256_block_neon,.-sha256_block_neon
+#ifndef	__KERNEL__
+.comm	OPENSSL_armcap_P,4,4
+#endif
diff --git a/arch/arm64/crypto/sha256-glue.c b/arch/arm64/crypto/sha256-glue.c
new file mode 100644
index 000000000000..a2226f841960
--- /dev/null
+++ b/arch/arm64/crypto/sha256-glue.c
@@ -0,0 +1,185 @@
+/*
+ * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64
+ *
+ * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+#include <asm/simd.h>
+#include <crypto/internal/hash.h>
+#include <crypto/sha.h>
+#include <crypto/sha256_base.h>
+#include <linux/cryptohash.h>
+#include <linux/types.h>
+#include <linux/string.h>
+
+MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");
+MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("sha224");
+MODULE_ALIAS_CRYPTO("sha256");
+
+asmlinkage void sha256_block_data_order(u32 *digest, const void *data,
+					unsigned int num_blks);
+
+asmlinkage void sha256_block_neon(u32 *digest, const void *data,
+				  unsigned int num_blks);
+
+static int sha256_update(struct shash_desc *desc, const u8 *data,
+			 unsigned int len)
+{
+	return sha256_base_do_update(desc, data, len,
+				(sha256_block_fn *)sha256_block_data_order);
+}
+
+static int sha256_finup(struct shash_desc *desc, const u8 *data,
+			unsigned int len, u8 *out)
+{
+	if (len)
+		sha256_base_do_update(desc, data, len,
+				(sha256_block_fn *)sha256_block_data_order);
+	sha256_base_do_finalize(desc,
+				(sha256_block_fn *)sha256_block_data_order);
+
+	return sha256_base_finish(desc, out);
+}
+
+static int sha256_final(struct shash_desc *desc, u8 *out)
+{
+	return sha256_finup(desc, NULL, 0, out);
+}
+
+static struct shash_alg algs[] = { {
+	.digestsize		= SHA256_DIGEST_SIZE,
+	.init			= sha256_base_init,
+	.update			= sha256_update,
+	.final			= sha256_final,
+	.finup			= sha256_finup,
+	.descsize		= sizeof(struct sha256_state),
+	.base.cra_name		= "sha256",
+	.base.cra_driver_name	= "sha256-arm64",
+	.base.cra_priority	= 100,
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA256_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+}, {
+	.digestsize		= SHA224_DIGEST_SIZE,
+	.init			= sha224_base_init,
+	.update			= sha256_update,
+	.final			= sha256_final,
+	.finup			= sha256_finup,
+	.descsize		= sizeof(struct sha256_state),
+	.base.cra_name		= "sha224",
+	.base.cra_driver_name	= "sha224-arm64",
+	.base.cra_priority	= 100,
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA224_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+} };
+
+static int sha256_update_neon(struct shash_desc *desc, const u8 *data,
+			      unsigned int len)
+{
+	/*
+	 * Stacking and unstacking a substantial slice of the NEON register
+	 * file may significantly affect performance for small updates when
+	 * executing in interrupt context, so fall back to the scalar code
+	 * in that case.
+	 */
+	if (!may_use_simd())
+		return sha256_base_do_update(desc, data, len,
+				(sha256_block_fn *)sha256_block_data_order);
+
+	kernel_neon_begin();
+	sha256_base_do_update(desc, data, len,
+				(sha256_block_fn *)sha256_block_neon);
+	kernel_neon_end();
+
+	return 0;
+}
+
+static int sha256_finup_neon(struct shash_desc *desc, const u8 *data,
+			     unsigned int len, u8 *out)
+{
+	if (!may_use_simd()) {
+		if (len)
+			sha256_base_do_update(desc, data, len,
+				(sha256_block_fn *)sha256_block_data_order);
+		sha256_base_do_finalize(desc,
+				(sha256_block_fn *)sha256_block_data_order);
+	} else {
+		kernel_neon_begin();
+		if (len)
+			sha256_base_do_update(desc, data, len,
+				(sha256_block_fn *)sha256_block_neon);
+		sha256_base_do_finalize(desc,
+				(sha256_block_fn *)sha256_block_neon);
+		kernel_neon_end();
+	}
+	return sha256_base_finish(desc, out);
+}
+
+static int sha256_final_neon(struct shash_desc *desc, u8 *out)
+{
+	return sha256_finup_neon(desc, NULL, 0, out);
+}
+
+static struct shash_alg neon_algs[] = { {
+	.digestsize		= SHA256_DIGEST_SIZE,
+	.init			= sha256_base_init,
+	.update			= sha256_update_neon,
+	.final			= sha256_final_neon,
+	.finup			= sha256_finup_neon,
+	.descsize		= sizeof(struct sha256_state),
+	.base.cra_name		= "sha256",
+	.base.cra_driver_name	= "sha256-arm64-neon",
+	.base.cra_priority	= 150,
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA256_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+}, {
+	.digestsize		= SHA224_DIGEST_SIZE,
+	.init			= sha224_base_init,
+	.update			= sha256_update_neon,
+	.final			= sha256_final_neon,
+	.finup			= sha256_finup_neon,
+	.descsize		= sizeof(struct sha256_state),
+	.base.cra_name		= "sha224",
+	.base.cra_driver_name	= "sha224-arm64-neon",
+	.base.cra_priority	= 150,
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA224_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+} };
+
+static int __init sha256_mod_init(void)
+{
+	int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
+	if (ret)
+		return ret;
+
+	if (elf_hwcap & HWCAP_ASIMD) {
+		ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs));
+		if (ret)
+			crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
+	}
+	return ret;
+}
+
+static void __exit sha256_mod_fini(void)
+{
+	if (elf_hwcap & HWCAP_ASIMD)
+		crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs));
+	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
+}
+
+module_init(sha256_mod_init);
+module_exit(sha256_mod_fini);
diff --git a/arch/arm64/crypto/sha512-armv8.pl b/arch/arm64/crypto/sha512-armv8.pl
new file mode 100644
index 000000000000..c55efb308544
--- /dev/null
+++ b/arch/arm64/crypto/sha512-armv8.pl
@@ -0,0 +1,778 @@
+#! /usr/bin/env perl
+# Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the OpenSSL license (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+#
+# Permission to use under GPLv2 terms is granted.
+# ====================================================================
+#
+# SHA256/512 for ARMv8.
+#
+# Performance in cycles per processed byte and improvement coefficient
+# over code generated with "default" compiler:
+#
+#		SHA256-hw	SHA256(*)	SHA512
+# Apple A7	1.97		10.5 (+33%)	6.73 (-1%(**))
+# Cortex-A53	2.38		15.5 (+115%)	10.0 (+150%(***))
+# Cortex-A57	2.31		11.6 (+86%)	7.51 (+260%(***))
+# Denver	2.01		10.5 (+26%)	6.70 (+8%)
+# X-Gene			20.0 (+100%)	12.8 (+300%(***))
+# Mongoose	2.36		13.0 (+50%)	8.36 (+33%)
+#
+# (*)	Software SHA256 results are of lesser relevance, presented
+#	mostly for informational purposes.
+# (**)	The result is a trade-off: it's possible to improve it by
+#	10% (or by 1 cycle per round), but at the cost of 20% loss
+#	on Cortex-A53 (or by 4 cycles per round).
+# (***)	Super-impressive coefficients over gcc-generated code are
+#	indication of some compiler "pathology", most notably code
+#	generated with -mgeneral-regs-only is significanty faster
+#	and the gap is only 40-90%.
+#
+# October 2016.
+#
+# Originally it was reckoned that it makes no sense to implement NEON
+# version of SHA256 for 64-bit processors. This is because performance
+# improvement on most wide-spread Cortex-A5x processors was observed
+# to be marginal, same on Cortex-A53 and ~10% on A57. But then it was
+# observed that 32-bit NEON SHA256 performs significantly better than
+# 64-bit scalar version on *some* of the more recent processors. As
+# result 64-bit NEON version of SHA256 was added to provide best
+# all-round performance. For example it executes ~30% faster on X-Gene
+# and Mongoose. [For reference, NEON version of SHA512 is bound to
+# deliver much less improvement, likely *negative* on Cortex-A5x.
+# Which is why NEON support is limited to SHA256.]
+
+$output=pop;
+$flavour=pop;
+
+if ($flavour && $flavour ne "void") {
+    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
+    ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
+    die "can't locate arm-xlate.pl";
+
+    open OUT,"| \"$^X\" $xlate $flavour $output";
+    *STDOUT=*OUT;
+} else {
+    open STDOUT,">$output";
+}
+
+if ($output =~ /512/) {
+	$BITS=512;
+	$SZ=8;
+	@Sigma0=(28,34,39);
+	@Sigma1=(14,18,41);
+	@sigma0=(1,  8, 7);
+	@sigma1=(19,61, 6);
+	$rounds=80;
+	$reg_t="x";
+} else {
+	$BITS=256;
+	$SZ=4;
+	@Sigma0=( 2,13,22);
+	@Sigma1=( 6,11,25);
+	@sigma0=( 7,18, 3);
+	@sigma1=(17,19,10);
+	$rounds=64;
+	$reg_t="w";
+}
+
+$func="sha${BITS}_block_data_order";
+
+($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30));
+
+@X=map("$reg_t$_",(3..15,0..2));
+@V=($A,$B,$C,$D,$E,$F,$G,$H)=map("$reg_t$_",(20..27));
+($t0,$t1,$t2,$t3)=map("$reg_t$_",(16,17,19,28));
+
+sub BODY_00_xx {
+my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
+my $j=($i+1)&15;
+my ($T0,$T1,$T2)=(@X[($i-8)&15],@X[($i-9)&15],@X[($i-10)&15]);
+   $T0=@X[$i+3] if ($i<11);
+
+$code.=<<___	if ($i<16);
+#ifndef	__AARCH64EB__
+	rev	@X[$i],@X[$i]			// $i
+#endif
+___
+$code.=<<___	if ($i<13 && ($i&1));
+	ldp	@X[$i+1],@X[$i+2],[$inp],#2*$SZ
+___
+$code.=<<___	if ($i==13);
+	ldp	@X[14],@X[15],[$inp]
+___
+$code.=<<___	if ($i>=14);
+	ldr	@X[($i-11)&15],[sp,#`$SZ*(($i-11)%4)`]
+___
+$code.=<<___	if ($i>0 && $i<16);
+	add	$a,$a,$t1			// h+=Sigma0(a)
+___
+$code.=<<___	if ($i>=11);
+	str	@X[($i-8)&15],[sp,#`$SZ*(($i-8)%4)`]
+___
+# While ARMv8 specifies merged rotate-n-logical operation such as
+# 'eor x,y,z,ror#n', it was found to negatively affect performance
+# on Apple A7. The reason seems to be that it requires even 'y' to
+# be available earlier. This means that such merged instruction is
+# not necessarily best choice on critical path... On the other hand
+# Cortex-A5x handles merged instructions much better than disjoint
+# rotate and logical... See (**) footnote above.
+$code.=<<___	if ($i<15);
+	ror	$t0,$e,#$Sigma1[0]
+	add	$h,$h,$t2			// h+=K[i]
+	eor	$T0,$e,$e,ror#`$Sigma1[2]-$Sigma1[1]`
+	and	$t1,$f,$e
+	bic	$t2,$g,$e
+	add	$h,$h,@X[$i&15]			// h+=X[i]
+	orr	$t1,$t1,$t2			// Ch(e,f,g)
+	eor	$t2,$a,$b			// a^b, b^c in next round
+	eor	$t0,$t0,$T0,ror#$Sigma1[1]	// Sigma1(e)
+	ror	$T0,$a,#$Sigma0[0]
+	add	$h,$h,$t1			// h+=Ch(e,f,g)
+	eor	$t1,$a,$a,ror#`$Sigma0[2]-$Sigma0[1]`
+	add	$h,$h,$t0			// h+=Sigma1(e)
+	and	$t3,$t3,$t2			// (b^c)&=(a^b)
+	add	$d,$d,$h			// d+=h
+	eor	$t3,$t3,$b			// Maj(a,b,c)
+	eor	$t1,$T0,$t1,ror#$Sigma0[1]	// Sigma0(a)
+	add	$h,$h,$t3			// h+=Maj(a,b,c)
+	ldr	$t3,[$Ktbl],#$SZ		// *K++, $t2 in next round
+	//add	$h,$h,$t1			// h+=Sigma0(a)
+___
+$code.=<<___	if ($i>=15);
+	ror	$t0,$e,#$Sigma1[0]
+	add	$h,$h,$t2			// h+=K[i]
+	ror	$T1,@X[($j+1)&15],#$sigma0[0]
+	and	$t1,$f,$e
+	ror	$T2,@X[($j+14)&15],#$sigma1[0]
+	bic	$t2,$g,$e
+	ror	$T0,$a,#$Sigma0[0]
+	add	$h,$h,@X[$i&15]			// h+=X[i]
+	eor	$t0,$t0,$e,ror#$Sigma1[1]
+	eor	$T1,$T1,@X[($j+1)&15],ror#$sigma0[1]
+	orr	$t1,$t1,$t2			// Ch(e,f,g)
+	eor	$t2,$a,$b			// a^b, b^c in next round
+	eor	$t0,$t0,$e,ror#$Sigma1[2]	// Sigma1(e)
+	eor	$T0,$T0,$a,ror#$Sigma0[1]
+	add	$h,$h,$t1			// h+=Ch(e,f,g)
+	and	$t3,$t3,$t2			// (b^c)&=(a^b)
+	eor	$T2,$T2,@X[($j+14)&15],ror#$sigma1[1]
+	eor	$T1,$T1,@X[($j+1)&15],lsr#$sigma0[2]	// sigma0(X[i+1])
+	add	$h,$h,$t0			// h+=Sigma1(e)
+	eor	$t3,$t3,$b			// Maj(a,b,c)
+	eor	$t1,$T0,$a,ror#$Sigma0[2]	// Sigma0(a)
+	eor	$T2,$T2,@X[($j+14)&15],lsr#$sigma1[2]	// sigma1(X[i+14])
+	add	@X[$j],@X[$j],@X[($j+9)&15]
+	add	$d,$d,$h			// d+=h
+	add	$h,$h,$t3			// h+=Maj(a,b,c)
+	ldr	$t3,[$Ktbl],#$SZ		// *K++, $t2 in next round
+	add	@X[$j],@X[$j],$T1
+	add	$h,$h,$t1			// h+=Sigma0(a)
+	add	@X[$j],@X[$j],$T2
+___
+	($t2,$t3)=($t3,$t2);
+}
+
+$code.=<<___;
+#ifndef	__KERNEL__
+# include "arm_arch.h"
+#endif
+
+.text
+
+.extern	OPENSSL_armcap_P
+.globl	$func
+.type	$func,%function
+.align	6
+$func:
+___
+$code.=<<___	if ($SZ==4);
+#ifndef	__KERNEL__
+# ifdef	__ILP32__
+	ldrsw	x16,.LOPENSSL_armcap_P
+# else
+	ldr	x16,.LOPENSSL_armcap_P
+# endif
+	adr	x17,.LOPENSSL_armcap_P
+	add	x16,x16,x17
+	ldr	w16,[x16]
+	tst	w16,#ARMV8_SHA256
+	b.ne	.Lv8_entry
+	tst	w16,#ARMV7_NEON
+	b.ne	.Lneon_entry
+#endif
+___
+$code.=<<___;
+	stp	x29,x30,[sp,#-128]!
+	add	x29,sp,#0
+
+	stp	x19,x20,[sp,#16]
+	stp	x21,x22,[sp,#32]
+	stp	x23,x24,[sp,#48]
+	stp	x25,x26,[sp,#64]
+	stp	x27,x28,[sp,#80]
+	sub	sp,sp,#4*$SZ
+
+	ldp	$A,$B,[$ctx]				// load context
+	ldp	$C,$D,[$ctx,#2*$SZ]
+	ldp	$E,$F,[$ctx,#4*$SZ]
+	add	$num,$inp,$num,lsl#`log(16*$SZ)/log(2)`	// end of input
+	ldp	$G,$H,[$ctx,#6*$SZ]
+	adr	$Ktbl,.LK$BITS
+	stp	$ctx,$num,[x29,#96]
+
+.Loop:
+	ldp	@X[0],@X[1],[$inp],#2*$SZ
+	ldr	$t2,[$Ktbl],#$SZ			// *K++
+	eor	$t3,$B,$C				// magic seed
+	str	$inp,[x29,#112]
+___
+for ($i=0;$i<16;$i++)	{ &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
+$code.=".Loop_16_xx:\n";
+for (;$i<32;$i++)	{ &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
+$code.=<<___;
+	cbnz	$t2,.Loop_16_xx
+
+	ldp	$ctx,$num,[x29,#96]
+	ldr	$inp,[x29,#112]
+	sub	$Ktbl,$Ktbl,#`$SZ*($rounds+1)`		// rewind
+
+	ldp	@X[0],@X[1],[$ctx]
+	ldp	@X[2],@X[3],[$ctx,#2*$SZ]
+	add	$inp,$inp,#14*$SZ			// advance input pointer
+	ldp	@X[4],@X[5],[$ctx,#4*$SZ]
+	add	$A,$A,@X[0]
+	ldp	@X[6],@X[7],[$ctx,#6*$SZ]
+	add	$B,$B,@X[1]
+	add	$C,$C,@X[2]
+	add	$D,$D,@X[3]
+	stp	$A,$B,[$ctx]
+	add	$E,$E,@X[4]
+	add	$F,$F,@X[5]
+	stp	$C,$D,[$ctx,#2*$SZ]
+	add	$G,$G,@X[6]
+	add	$H,$H,@X[7]
+	cmp	$inp,$num
+	stp	$E,$F,[$ctx,#4*$SZ]
+	stp	$G,$H,[$ctx,#6*$SZ]
+	b.ne	.Loop
+
+	ldp	x19,x20,[x29,#16]
+	add	sp,sp,#4*$SZ
+	ldp	x21,x22,[x29,#32]
+	ldp	x23,x24,[x29,#48]
+	ldp	x25,x26,[x29,#64]
+	ldp	x27,x28,[x29,#80]
+	ldp	x29,x30,[sp],#128
+	ret
+.size	$func,.-$func
+
+.align	6
+.type	.LK$BITS,%object
+.LK$BITS:
+___
+$code.=<<___ if ($SZ==8);
+	.quad	0x428a2f98d728ae22,0x7137449123ef65cd
+	.quad	0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
+	.quad	0x3956c25bf348b538,0x59f111f1b605d019
+	.quad	0x923f82a4af194f9b,0xab1c5ed5da6d8118
+	.quad	0xd807aa98a3030242,0x12835b0145706fbe
+	.quad	0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
+	.quad	0x72be5d74f27b896f,0x80deb1fe3b1696b1
+	.quad	0x9bdc06a725c71235,0xc19bf174cf692694
+	.quad	0xe49b69c19ef14ad2,0xefbe4786384f25e3
+	.quad	0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
+	.quad	0x2de92c6f592b0275,0x4a7484aa6ea6e483
+	.quad	0x5cb0a9dcbd41fbd4,0x76f988da831153b5
+	.quad	0x983e5152ee66dfab,0xa831c66d2db43210
+	.quad	0xb00327c898fb213f,0xbf597fc7beef0ee4
+	.quad	0xc6e00bf33da88fc2,0xd5a79147930aa725
+	.quad	0x06ca6351e003826f,0x142929670a0e6e70
+	.quad	0x27b70a8546d22ffc,0x2e1b21385c26c926
+	.quad	0x4d2c6dfc5ac42aed,0x53380d139d95b3df
+	.quad	0x650a73548baf63de,0x766a0abb3c77b2a8
+	.quad	0x81c2c92e47edaee6,0x92722c851482353b
+	.quad	0xa2bfe8a14cf10364,0xa81a664bbc423001
+	.quad	0xc24b8b70d0f89791,0xc76c51a30654be30
+	.quad	0xd192e819d6ef5218,0xd69906245565a910
+	.quad	0xf40e35855771202a,0x106aa07032bbd1b8
+	.quad	0x19a4c116b8d2d0c8,0x1e376c085141ab53
+	.quad	0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
+	.quad	0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
+	.quad	0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
+	.quad	0x748f82ee5defb2fc,0x78a5636f43172f60
+	.quad	0x84c87814a1f0ab72,0x8cc702081a6439ec
+	.quad	0x90befffa23631e28,0xa4506cebde82bde9
+	.quad	0xbef9a3f7b2c67915,0xc67178f2e372532b
+	.quad	0xca273eceea26619c,0xd186b8c721c0c207
+	.quad	0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
+	.quad	0x06f067aa72176fba,0x0a637dc5a2c898a6
+	.quad	0x113f9804bef90dae,0x1b710b35131c471b
+	.quad	0x28db77f523047d84,0x32caab7b40c72493
+	.quad	0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
+	.quad	0x4cc5d4becb3e42b6,0x597f299cfc657e2a
+	.quad	0x5fcb6fab3ad6faec,0x6c44198c4a475817
+	.quad	0	// terminator
+___
+$code.=<<___ if ($SZ==4);
+	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
+	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
+	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
+	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
+	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
+	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
+	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
+	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
+	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
+	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
+	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
+	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
+	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
+	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
+	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
+	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
+	.long	0	//terminator
+___
+$code.=<<___;
+.size	.LK$BITS,.-.LK$BITS
+#ifndef	__KERNEL__
+.align	3
+.LOPENSSL_armcap_P:
+# ifdef	__ILP32__
+	.long	OPENSSL_armcap_P-.
+# else
+	.quad	OPENSSL_armcap_P-.
+# endif
+#endif
+.asciz	"SHA$BITS block transform for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
+.align	2
+___
+
+if ($SZ==4) {
+my $Ktbl="x3";
+
+my ($ABCD,$EFGH,$abcd)=map("v$_.16b",(0..2));
+my @MSG=map("v$_.16b",(4..7));
+my ($W0,$W1)=("v16.4s","v17.4s");
+my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b");
+
+$code.=<<___;
+#ifndef	__KERNEL__
+.type	sha256_block_armv8,%function
+.align	6
+sha256_block_armv8:
+.Lv8_entry:
+	stp		x29,x30,[sp,#-16]!
+	add		x29,sp,#0
+
+	ld1.32		{$ABCD,$EFGH},[$ctx]
+	adr		$Ktbl,.LK256
+
+.Loop_hw:
+	ld1		{@MSG[0]-@MSG[3]},[$inp],#64
+	sub		$num,$num,#1
+	ld1.32		{$W0},[$Ktbl],#16
+	rev32		@MSG[0],@MSG[0]
+	rev32		@MSG[1],@MSG[1]
+	rev32		@MSG[2],@MSG[2]
+	rev32		@MSG[3],@MSG[3]
+	orr		$ABCD_SAVE,$ABCD,$ABCD		// offload
+	orr		$EFGH_SAVE,$EFGH,$EFGH
+___
+for($i=0;$i<12;$i++) {
+$code.=<<___;
+	ld1.32		{$W1},[$Ktbl],#16
+	add.i32		$W0,$W0,@MSG[0]
+	sha256su0	@MSG[0],@MSG[1]
+	orr		$abcd,$ABCD,$ABCD
+	sha256h		$ABCD,$EFGH,$W0
+	sha256h2	$EFGH,$abcd,$W0
+	sha256su1	@MSG[0],@MSG[2],@MSG[3]
+___
+	($W0,$W1)=($W1,$W0);	push(@MSG,shift(@MSG));
+}
+$code.=<<___;
+	ld1.32		{$W1},[$Ktbl],#16
+	add.i32		$W0,$W0,@MSG[0]
+	orr		$abcd,$ABCD,$ABCD
+	sha256h		$ABCD,$EFGH,$W0
+	sha256h2	$EFGH,$abcd,$W0
+
+	ld1.32		{$W0},[$Ktbl],#16
+	add.i32		$W1,$W1,@MSG[1]
+	orr		$abcd,$ABCD,$ABCD
+	sha256h		$ABCD,$EFGH,$W1
+	sha256h2	$EFGH,$abcd,$W1
+
+	ld1.32		{$W1},[$Ktbl]
+	add.i32		$W0,$W0,@MSG[2]
+	sub		$Ktbl,$Ktbl,#$rounds*$SZ-16	// rewind
+	orr		$abcd,$ABCD,$ABCD
+	sha256h		$ABCD,$EFGH,$W0
+	sha256h2	$EFGH,$abcd,$W0
+
+	add.i32		$W1,$W1,@MSG[3]
+	orr		$abcd,$ABCD,$ABCD
+	sha256h		$ABCD,$EFGH,$W1
+	sha256h2	$EFGH,$abcd,$W1
+
+	add.i32		$ABCD,$ABCD,$ABCD_SAVE
+	add.i32		$EFGH,$EFGH,$EFGH_SAVE
+
+	cbnz		$num,.Loop_hw
+
+	st1.32		{$ABCD,$EFGH},[$ctx]
+
+	ldr		x29,[sp],#16
+	ret
+.size	sha256_block_armv8,.-sha256_block_armv8
+#endif
+___
+}
+
+if ($SZ==4) {	######################################### NEON stuff #
+# You'll surely note a lot of similarities with sha256-armv4 module,
+# and of course it's not a coincidence. sha256-armv4 was used as
+# initial template, but was adapted for ARMv8 instruction set and
+# extensively re-tuned for all-round performance.
+
+my @V = ($A,$B,$C,$D,$E,$F,$G,$H) = map("w$_",(3..10));
+my ($t0,$t1,$t2,$t3,$t4) = map("w$_",(11..15));
+my $Ktbl="x16";
+my $Xfer="x17";
+my @X = map("q$_",(0..3));
+my ($T0,$T1,$T2,$T3,$T4,$T5,$T6,$T7) = map("q$_",(4..7,16..19));
+my $j=0;
+
+sub AUTOLOAD()          # thunk [simplified] x86-style perlasm
+{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
+  my $arg = pop;
+    $arg = "#$arg" if ($arg*1 eq $arg);
+    $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
+}
+
+sub Dscalar { shift =~ m|[qv]([0-9]+)|?"d$1":""; }
+sub Dlo     { shift =~ m|[qv]([0-9]+)|?"v$1.d[0]":""; }
+sub Dhi     { shift =~ m|[qv]([0-9]+)|?"v$1.d[1]":""; }
+
+sub Xupdate()
+{ use integer;
+  my $body = shift;
+  my @insns = (&$body,&$body,&$body,&$body);
+  my ($a,$b,$c,$d,$e,$f,$g,$h);
+
+	&ext_8		($T0,@X[0],@X[1],4);	# X[1..4]
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&ext_8		($T3,@X[2],@X[3],4);	# X[9..12]
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&mov		(&Dscalar($T7),&Dhi(@X[3]));	# X[14..15]
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&ushr_32	($T2,$T0,$sigma0[0]);
+	 eval(shift(@insns));
+	&ushr_32	($T1,$T0,$sigma0[2]);
+	 eval(shift(@insns));
+	&add_32 	(@X[0],@X[0],$T3);	# X[0..3] += X[9..12]
+	 eval(shift(@insns));
+	&sli_32		($T2,$T0,32-$sigma0[0]);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&ushr_32	($T3,$T0,$sigma0[1]);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&eor_8		($T1,$T1,$T2);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&sli_32		($T3,$T0,32-$sigma0[1]);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	  &ushr_32	($T4,$T7,$sigma1[0]);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&eor_8		($T1,$T1,$T3);		# sigma0(X[1..4])
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	  &sli_32	($T4,$T7,32-$sigma1[0]);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	  &ushr_32	($T5,$T7,$sigma1[2]);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	  &ushr_32	($T3,$T7,$sigma1[1]);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&add_32		(@X[0],@X[0],$T1);	# X[0..3] += sigma0(X[1..4])
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	  &sli_u32	($T3,$T7,32-$sigma1[1]);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	  &eor_8	($T5,$T5,$T4);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	  &eor_8	($T5,$T5,$T3);		# sigma1(X[14..15])
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&add_32		(@X[0],@X[0],$T5);	# X[0..1] += sigma1(X[14..15])
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	  &ushr_32	($T6,@X[0],$sigma1[0]);
+	 eval(shift(@insns));
+	  &ushr_32	($T7,@X[0],$sigma1[2]);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	  &sli_32	($T6,@X[0],32-$sigma1[0]);
+	 eval(shift(@insns));
+	  &ushr_32	($T5,@X[0],$sigma1[1]);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	  &eor_8	($T7,$T7,$T6);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	  &sli_32	($T5,@X[0],32-$sigma1[1]);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&ld1_32		("{$T0}","[$Ktbl], #16");
+	 eval(shift(@insns));
+	  &eor_8	($T7,$T7,$T5);		# sigma1(X[16..17])
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&eor_8		($T5,$T5,$T5);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&mov		(&Dhi($T5), &Dlo($T7));
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&add_32		(@X[0],@X[0],$T5);	# X[2..3] += sigma1(X[16..17])
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&add_32		($T0,$T0,@X[0]);
+	 while($#insns>=1) { eval(shift(@insns)); }
+	&st1_32		("{$T0}","[$Xfer], #16");
+	 eval(shift(@insns));
+
+	push(@X,shift(@X));		# "rotate" X[]
+}
+
+sub Xpreload()
+{ use integer;
+  my $body = shift;
+  my @insns = (&$body,&$body,&$body,&$body);
+  my ($a,$b,$c,$d,$e,$f,$g,$h);
+
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&ld1_8		("{@X[0]}","[$inp],#16");
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&ld1_32		("{$T0}","[$Ktbl],#16");
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&rev32		(@X[0],@X[0]);
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	 eval(shift(@insns));
+	&add_32		($T0,$T0,@X[0]);
+	 foreach (@insns) { eval; }	# remaining instructions
+	&st1_32		("{$T0}","[$Xfer], #16");
+
+	push(@X,shift(@X));		# "rotate" X[]
+}
+
+sub body_00_15 () {
+	(
+	'($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
+	'&add	($h,$h,$t1)',			# h+=X[i]+K[i]
+	'&add	($a,$a,$t4);'.			# h+=Sigma0(a) from the past
+	'&and	($t1,$f,$e)',
+	'&bic	($t4,$g,$e)',
+	'&eor	($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
+	'&add	($a,$a,$t2)',			# h+=Maj(a,b,c) from the past
+	'&orr	($t1,$t1,$t4)',			# Ch(e,f,g)
+	'&eor	($t0,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))',	# Sigma1(e)
+	'&eor	($t4,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
+	'&add	($h,$h,$t1)',			# h+=Ch(e,f,g)
+	'&ror	($t0,$t0,"#$Sigma1[0]")',
+	'&eor	($t2,$a,$b)',			# a^b, b^c in next round
+	'&eor	($t4,$t4,$a,"ror#".($Sigma0[2]-$Sigma0[0]))',	# Sigma0(a)
+	'&add	($h,$h,$t0)',			# h+=Sigma1(e)
+	'&ldr	($t1,sprintf "[sp,#%d]",4*(($j+1)&15))	if (($j&15)!=15);'.
+	'&ldr	($t1,"[$Ktbl]")				if ($j==15);'.
+	'&and	($t3,$t3,$t2)',			# (b^c)&=(a^b)
+	'&ror	($t4,$t4,"#$Sigma0[0]")',
+	'&add	($d,$d,$h)',			# d+=h
+	'&eor	($t3,$t3,$b)',			# Maj(a,b,c)
+	'$j++;	unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
+	)
+}
+
+$code.=<<___;
+#ifdef	__KERNEL__
+.globl	sha256_block_neon
+#endif
+.type	sha256_block_neon,%function
+.align	4
+sha256_block_neon:
+.Lneon_entry:
+	stp	x29, x30, [sp, #-16]!
+	mov	x29, sp
+	sub	sp,sp,#16*4
+
+	adr	$Ktbl,.LK256
+	add	$num,$inp,$num,lsl#6	// len to point at the end of inp
+
+	ld1.8	{@X[0]},[$inp], #16
+	ld1.8	{@X[1]},[$inp], #16
+	ld1.8	{@X[2]},[$inp], #16
+	ld1.8	{@X[3]},[$inp], #16
+	ld1.32	{$T0},[$Ktbl], #16
+	ld1.32	{$T1},[$Ktbl], #16
+	ld1.32	{$T2},[$Ktbl], #16
+	ld1.32	{$T3},[$Ktbl], #16
+	rev32	@X[0],@X[0]		// yes, even on
+	rev32	@X[1],@X[1]		// big-endian
+	rev32	@X[2],@X[2]
+	rev32	@X[3],@X[3]
+	mov	$Xfer,sp
+	add.32	$T0,$T0,@X[0]
+	add.32	$T1,$T1,@X[1]
+	add.32	$T2,$T2,@X[2]
+	st1.32	{$T0-$T1},[$Xfer], #32
+	add.32	$T3,$T3,@X[3]
+	st1.32	{$T2-$T3},[$Xfer]
+	sub	$Xfer,$Xfer,#32
+
+	ldp	$A,$B,[$ctx]
+	ldp	$C,$D,[$ctx,#8]
+	ldp	$E,$F,[$ctx,#16]
+	ldp	$G,$H,[$ctx,#24]
+	ldr	$t1,[sp,#0]
+	mov	$t2,wzr
+	eor	$t3,$B,$C
+	mov	$t4,wzr
+	b	.L_00_48
+
+.align	4
+.L_00_48:
+___
+	&Xupdate(\&body_00_15);
+	&Xupdate(\&body_00_15);
+	&Xupdate(\&body_00_15);
+	&Xupdate(\&body_00_15);
+$code.=<<___;
+	cmp	$t1,#0				// check for K256 terminator
+	ldr	$t1,[sp,#0]
+	sub	$Xfer,$Xfer,#64
+	bne	.L_00_48
+
+	sub	$Ktbl,$Ktbl,#256		// rewind $Ktbl
+	cmp	$inp,$num
+	mov	$Xfer, #64
+	csel	$Xfer, $Xfer, xzr, eq
+	sub	$inp,$inp,$Xfer			// avoid SEGV
+	mov	$Xfer,sp
+___
+	&Xpreload(\&body_00_15);
+	&Xpreload(\&body_00_15);
+	&Xpreload(\&body_00_15);
+	&Xpreload(\&body_00_15);
+$code.=<<___;
+	add	$A,$A,$t4			// h+=Sigma0(a) from the past
+	ldp	$t0,$t1,[$ctx,#0]
+	add	$A,$A,$t2			// h+=Maj(a,b,c) from the past
+	ldp	$t2,$t3,[$ctx,#8]
+	add	$A,$A,$t0			// accumulate
+	add	$B,$B,$t1
+	ldp	$t0,$t1,[$ctx,#16]
+	add	$C,$C,$t2
+	add	$D,$D,$t3
+	ldp	$t2,$t3,[$ctx,#24]
+	add	$E,$E,$t0
+	add	$F,$F,$t1
+	 ldr	$t1,[sp,#0]
+	stp	$A,$B,[$ctx,#0]
+	add	$G,$G,$t2
+	 mov	$t2,wzr
+	stp	$C,$D,[$ctx,#8]
+	add	$H,$H,$t3
+	stp	$E,$F,[$ctx,#16]
+	 eor	$t3,$B,$C
+	stp	$G,$H,[$ctx,#24]
+	 mov	$t4,wzr
+	 mov	$Xfer,sp
+	b.ne	.L_00_48
+
+	ldr	x29,[x29]
+	add	sp,sp,#16*4+16
+	ret
+.size	sha256_block_neon,.-sha256_block_neon
+___
+}
+
+$code.=<<___;
+#ifndef	__KERNEL__
+.comm	OPENSSL_armcap_P,4,4
+#endif
+___
+
+{   my  %opcode = (
+	"sha256h"	=> 0x5e004000,	"sha256h2"	=> 0x5e005000,
+	"sha256su0"	=> 0x5e282800,	"sha256su1"	=> 0x5e006000	);
+
+    sub unsha256 {
+	my ($mnemonic,$arg)=@_;
+
+	$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
+	&&
+	sprintf ".inst\t0x%08x\t//%s %s",
+			$opcode{$mnemonic}|$1|($2<<5)|($3<<16),
+			$mnemonic,$arg;
+    }
+}
+
+open SELF,$0;
+while(<SELF>) {
+        next if (/^#!/);
+        last if (!s/^#/\/\// and !/^$/);
+        print;
+}
+close SELF;
+
+foreach(split("\n",$code)) {
+
+	s/\`([^\`]*)\`/eval($1)/ge;
+
+	s/\b(sha256\w+)\s+([qv].*)/unsha256($1,$2)/ge;
+
+	s/\bq([0-9]+)\b/v$1.16b/g;		# old->new registers
+
+	s/\.[ui]?8(\s)/$1/;
+	s/\.\w?32\b//		and s/\.16b/\.4s/g;
+	m/(ld|st)1[^\[]+\[0\]/	and s/\.4s/\.s/g;
+
+	print $_,"\n";
+}
+
+close STDOUT;
diff --git a/arch/arm64/crypto/sha512-core.S_shipped b/arch/arm64/crypto/sha512-core.S_shipped
new file mode 100644
index 000000000000..bd0f59f06c9d
--- /dev/null
+++ b/arch/arm64/crypto/sha512-core.S_shipped
@@ -0,0 +1,1085 @@
+// Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
+//
+// Licensed under the OpenSSL license (the "License").  You may not use
+// this file except in compliance with the License.  You can obtain a copy
+// in the file LICENSE in the source distribution or at
+// https://www.openssl.org/source/license.html
+
+// ====================================================================
+// Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+// project. The module is, however, dual licensed under OpenSSL and
+// CRYPTOGAMS licenses depending on where you obtain it. For further
+// details see http://www.openssl.org/~appro/cryptogams/.
+//
+// Permission to use under GPLv2 terms is granted.
+// ====================================================================
+//
+// SHA256/512 for ARMv8.
+//
+// Performance in cycles per processed byte and improvement coefficient
+// over code generated with "default" compiler:
+//
+//		SHA256-hw	SHA256(*)	SHA512
+// Apple A7	1.97		10.5 (+33%)	6.73 (-1%(**))
+// Cortex-A53	2.38		15.5 (+115%)	10.0 (+150%(***))
+// Cortex-A57	2.31		11.6 (+86%)	7.51 (+260%(***))
+// Denver	2.01		10.5 (+26%)	6.70 (+8%)
+// X-Gene			20.0 (+100%)	12.8 (+300%(***))
+// Mongoose	2.36		13.0 (+50%)	8.36 (+33%)
+//
+// (*)	Software SHA256 results are of lesser relevance, presented
+//	mostly for informational purposes.
+// (**)	The result is a trade-off: it's possible to improve it by
+//	10% (or by 1 cycle per round), but at the cost of 20% loss
+//	on Cortex-A53 (or by 4 cycles per round).
+// (***)	Super-impressive coefficients over gcc-generated code are
+//	indication of some compiler "pathology", most notably code
+//	generated with -mgeneral-regs-only is significanty faster
+//	and the gap is only 40-90%.
+//
+// October 2016.
+//
+// Originally it was reckoned that it makes no sense to implement NEON
+// version of SHA256 for 64-bit processors. This is because performance
+// improvement on most wide-spread Cortex-A5x processors was observed
+// to be marginal, same on Cortex-A53 and ~10% on A57. But then it was
+// observed that 32-bit NEON SHA256 performs significantly better than
+// 64-bit scalar version on *some* of the more recent processors. As
+// result 64-bit NEON version of SHA256 was added to provide best
+// all-round performance. For example it executes ~30% faster on X-Gene
+// and Mongoose. [For reference, NEON version of SHA512 is bound to
+// deliver much less improvement, likely *negative* on Cortex-A5x.
+// Which is why NEON support is limited to SHA256.]
+
+#ifndef	__KERNEL__
+# include "arm_arch.h"
+#endif
+
+.text
+
+.extern	OPENSSL_armcap_P
+.globl	sha512_block_data_order
+.type	sha512_block_data_order,%function
+.align	6
+sha512_block_data_order:
+	stp	x29,x30,[sp,#-128]!
+	add	x29,sp,#0
+
+	stp	x19,x20,[sp,#16]
+	stp	x21,x22,[sp,#32]
+	stp	x23,x24,[sp,#48]
+	stp	x25,x26,[sp,#64]
+	stp	x27,x28,[sp,#80]
+	sub	sp,sp,#4*8
+
+	ldp	x20,x21,[x0]				// load context
+	ldp	x22,x23,[x0,#2*8]
+	ldp	x24,x25,[x0,#4*8]
+	add	x2,x1,x2,lsl#7	// end of input
+	ldp	x26,x27,[x0,#6*8]
+	adr	x30,.LK512
+	stp	x0,x2,[x29,#96]
+
+.Loop:
+	ldp	x3,x4,[x1],#2*8
+	ldr	x19,[x30],#8			// *K++
+	eor	x28,x21,x22				// magic seed
+	str	x1,[x29,#112]
+#ifndef	__AARCH64EB__
+	rev	x3,x3			// 0
+#endif
+	ror	x16,x24,#14
+	add	x27,x27,x19			// h+=K[i]
+	eor	x6,x24,x24,ror#23
+	and	x17,x25,x24
+	bic	x19,x26,x24
+	add	x27,x27,x3			// h+=X[i]
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x20,x21			// a^b, b^c in next round
+	eor	x16,x16,x6,ror#18	// Sigma1(e)
+	ror	x6,x20,#28
+	add	x27,x27,x17			// h+=Ch(e,f,g)
+	eor	x17,x20,x20,ror#5
+	add	x27,x27,x16			// h+=Sigma1(e)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	add	x23,x23,x27			// d+=h
+	eor	x28,x28,x21			// Maj(a,b,c)
+	eor	x17,x6,x17,ror#34	// Sigma0(a)
+	add	x27,x27,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	//add	x27,x27,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x4,x4			// 1
+#endif
+	ldp	x5,x6,[x1],#2*8
+	add	x27,x27,x17			// h+=Sigma0(a)
+	ror	x16,x23,#14
+	add	x26,x26,x28			// h+=K[i]
+	eor	x7,x23,x23,ror#23
+	and	x17,x24,x23
+	bic	x28,x25,x23
+	add	x26,x26,x4			// h+=X[i]
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x27,x20			// a^b, b^c in next round
+	eor	x16,x16,x7,ror#18	// Sigma1(e)
+	ror	x7,x27,#28
+	add	x26,x26,x17			// h+=Ch(e,f,g)
+	eor	x17,x27,x27,ror#5
+	add	x26,x26,x16			// h+=Sigma1(e)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	add	x22,x22,x26			// d+=h
+	eor	x19,x19,x20			// Maj(a,b,c)
+	eor	x17,x7,x17,ror#34	// Sigma0(a)
+	add	x26,x26,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	//add	x26,x26,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x5,x5			// 2
+#endif
+	add	x26,x26,x17			// h+=Sigma0(a)
+	ror	x16,x22,#14
+	add	x25,x25,x19			// h+=K[i]
+	eor	x8,x22,x22,ror#23
+	and	x17,x23,x22
+	bic	x19,x24,x22
+	add	x25,x25,x5			// h+=X[i]
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x26,x27			// a^b, b^c in next round
+	eor	x16,x16,x8,ror#18	// Sigma1(e)
+	ror	x8,x26,#28
+	add	x25,x25,x17			// h+=Ch(e,f,g)
+	eor	x17,x26,x26,ror#5
+	add	x25,x25,x16			// h+=Sigma1(e)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	add	x21,x21,x25			// d+=h
+	eor	x28,x28,x27			// Maj(a,b,c)
+	eor	x17,x8,x17,ror#34	// Sigma0(a)
+	add	x25,x25,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	//add	x25,x25,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x6,x6			// 3
+#endif
+	ldp	x7,x8,[x1],#2*8
+	add	x25,x25,x17			// h+=Sigma0(a)
+	ror	x16,x21,#14
+	add	x24,x24,x28			// h+=K[i]
+	eor	x9,x21,x21,ror#23
+	and	x17,x22,x21
+	bic	x28,x23,x21
+	add	x24,x24,x6			// h+=X[i]
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x25,x26			// a^b, b^c in next round
+	eor	x16,x16,x9,ror#18	// Sigma1(e)
+	ror	x9,x25,#28
+	add	x24,x24,x17			// h+=Ch(e,f,g)
+	eor	x17,x25,x25,ror#5
+	add	x24,x24,x16			// h+=Sigma1(e)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	add	x20,x20,x24			// d+=h
+	eor	x19,x19,x26			// Maj(a,b,c)
+	eor	x17,x9,x17,ror#34	// Sigma0(a)
+	add	x24,x24,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	//add	x24,x24,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x7,x7			// 4
+#endif
+	add	x24,x24,x17			// h+=Sigma0(a)
+	ror	x16,x20,#14
+	add	x23,x23,x19			// h+=K[i]
+	eor	x10,x20,x20,ror#23
+	and	x17,x21,x20
+	bic	x19,x22,x20
+	add	x23,x23,x7			// h+=X[i]
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x24,x25			// a^b, b^c in next round
+	eor	x16,x16,x10,ror#18	// Sigma1(e)
+	ror	x10,x24,#28
+	add	x23,x23,x17			// h+=Ch(e,f,g)
+	eor	x17,x24,x24,ror#5
+	add	x23,x23,x16			// h+=Sigma1(e)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	add	x27,x27,x23			// d+=h
+	eor	x28,x28,x25			// Maj(a,b,c)
+	eor	x17,x10,x17,ror#34	// Sigma0(a)
+	add	x23,x23,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	//add	x23,x23,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x8,x8			// 5
+#endif
+	ldp	x9,x10,[x1],#2*8
+	add	x23,x23,x17			// h+=Sigma0(a)
+	ror	x16,x27,#14
+	add	x22,x22,x28			// h+=K[i]
+	eor	x11,x27,x27,ror#23
+	and	x17,x20,x27
+	bic	x28,x21,x27
+	add	x22,x22,x8			// h+=X[i]
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x23,x24			// a^b, b^c in next round
+	eor	x16,x16,x11,ror#18	// Sigma1(e)
+	ror	x11,x23,#28
+	add	x22,x22,x17			// h+=Ch(e,f,g)
+	eor	x17,x23,x23,ror#5
+	add	x22,x22,x16			// h+=Sigma1(e)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	add	x26,x26,x22			// d+=h
+	eor	x19,x19,x24			// Maj(a,b,c)
+	eor	x17,x11,x17,ror#34	// Sigma0(a)
+	add	x22,x22,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	//add	x22,x22,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x9,x9			// 6
+#endif
+	add	x22,x22,x17			// h+=Sigma0(a)
+	ror	x16,x26,#14
+	add	x21,x21,x19			// h+=K[i]
+	eor	x12,x26,x26,ror#23
+	and	x17,x27,x26
+	bic	x19,x20,x26
+	add	x21,x21,x9			// h+=X[i]
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x22,x23			// a^b, b^c in next round
+	eor	x16,x16,x12,ror#18	// Sigma1(e)
+	ror	x12,x22,#28
+	add	x21,x21,x17			// h+=Ch(e,f,g)
+	eor	x17,x22,x22,ror#5
+	add	x21,x21,x16			// h+=Sigma1(e)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	add	x25,x25,x21			// d+=h
+	eor	x28,x28,x23			// Maj(a,b,c)
+	eor	x17,x12,x17,ror#34	// Sigma0(a)
+	add	x21,x21,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	//add	x21,x21,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x10,x10			// 7
+#endif
+	ldp	x11,x12,[x1],#2*8
+	add	x21,x21,x17			// h+=Sigma0(a)
+	ror	x16,x25,#14
+	add	x20,x20,x28			// h+=K[i]
+	eor	x13,x25,x25,ror#23
+	and	x17,x26,x25
+	bic	x28,x27,x25
+	add	x20,x20,x10			// h+=X[i]
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x21,x22			// a^b, b^c in next round
+	eor	x16,x16,x13,ror#18	// Sigma1(e)
+	ror	x13,x21,#28
+	add	x20,x20,x17			// h+=Ch(e,f,g)
+	eor	x17,x21,x21,ror#5
+	add	x20,x20,x16			// h+=Sigma1(e)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	add	x24,x24,x20			// d+=h
+	eor	x19,x19,x22			// Maj(a,b,c)
+	eor	x17,x13,x17,ror#34	// Sigma0(a)
+	add	x20,x20,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	//add	x20,x20,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x11,x11			// 8
+#endif
+	add	x20,x20,x17			// h+=Sigma0(a)
+	ror	x16,x24,#14
+	add	x27,x27,x19			// h+=K[i]
+	eor	x14,x24,x24,ror#23
+	and	x17,x25,x24
+	bic	x19,x26,x24
+	add	x27,x27,x11			// h+=X[i]
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x20,x21			// a^b, b^c in next round
+	eor	x16,x16,x14,ror#18	// Sigma1(e)
+	ror	x14,x20,#28
+	add	x27,x27,x17			// h+=Ch(e,f,g)
+	eor	x17,x20,x20,ror#5
+	add	x27,x27,x16			// h+=Sigma1(e)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	add	x23,x23,x27			// d+=h
+	eor	x28,x28,x21			// Maj(a,b,c)
+	eor	x17,x14,x17,ror#34	// Sigma0(a)
+	add	x27,x27,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	//add	x27,x27,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x12,x12			// 9
+#endif
+	ldp	x13,x14,[x1],#2*8
+	add	x27,x27,x17			// h+=Sigma0(a)
+	ror	x16,x23,#14
+	add	x26,x26,x28			// h+=K[i]
+	eor	x15,x23,x23,ror#23
+	and	x17,x24,x23
+	bic	x28,x25,x23
+	add	x26,x26,x12			// h+=X[i]
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x27,x20			// a^b, b^c in next round
+	eor	x16,x16,x15,ror#18	// Sigma1(e)
+	ror	x15,x27,#28
+	add	x26,x26,x17			// h+=Ch(e,f,g)
+	eor	x17,x27,x27,ror#5
+	add	x26,x26,x16			// h+=Sigma1(e)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	add	x22,x22,x26			// d+=h
+	eor	x19,x19,x20			// Maj(a,b,c)
+	eor	x17,x15,x17,ror#34	// Sigma0(a)
+	add	x26,x26,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	//add	x26,x26,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x13,x13			// 10
+#endif
+	add	x26,x26,x17			// h+=Sigma0(a)
+	ror	x16,x22,#14
+	add	x25,x25,x19			// h+=K[i]
+	eor	x0,x22,x22,ror#23
+	and	x17,x23,x22
+	bic	x19,x24,x22
+	add	x25,x25,x13			// h+=X[i]
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x26,x27			// a^b, b^c in next round
+	eor	x16,x16,x0,ror#18	// Sigma1(e)
+	ror	x0,x26,#28
+	add	x25,x25,x17			// h+=Ch(e,f,g)
+	eor	x17,x26,x26,ror#5
+	add	x25,x25,x16			// h+=Sigma1(e)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	add	x21,x21,x25			// d+=h
+	eor	x28,x28,x27			// Maj(a,b,c)
+	eor	x17,x0,x17,ror#34	// Sigma0(a)
+	add	x25,x25,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	//add	x25,x25,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x14,x14			// 11
+#endif
+	ldp	x15,x0,[x1],#2*8
+	add	x25,x25,x17			// h+=Sigma0(a)
+	str	x6,[sp,#24]
+	ror	x16,x21,#14
+	add	x24,x24,x28			// h+=K[i]
+	eor	x6,x21,x21,ror#23
+	and	x17,x22,x21
+	bic	x28,x23,x21
+	add	x24,x24,x14			// h+=X[i]
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x25,x26			// a^b, b^c in next round
+	eor	x16,x16,x6,ror#18	// Sigma1(e)
+	ror	x6,x25,#28
+	add	x24,x24,x17			// h+=Ch(e,f,g)
+	eor	x17,x25,x25,ror#5
+	add	x24,x24,x16			// h+=Sigma1(e)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	add	x20,x20,x24			// d+=h
+	eor	x19,x19,x26			// Maj(a,b,c)
+	eor	x17,x6,x17,ror#34	// Sigma0(a)
+	add	x24,x24,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	//add	x24,x24,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x15,x15			// 12
+#endif
+	add	x24,x24,x17			// h+=Sigma0(a)
+	str	x7,[sp,#0]
+	ror	x16,x20,#14
+	add	x23,x23,x19			// h+=K[i]
+	eor	x7,x20,x20,ror#23
+	and	x17,x21,x20
+	bic	x19,x22,x20
+	add	x23,x23,x15			// h+=X[i]
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x24,x25			// a^b, b^c in next round
+	eor	x16,x16,x7,ror#18	// Sigma1(e)
+	ror	x7,x24,#28
+	add	x23,x23,x17			// h+=Ch(e,f,g)
+	eor	x17,x24,x24,ror#5
+	add	x23,x23,x16			// h+=Sigma1(e)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	add	x27,x27,x23			// d+=h
+	eor	x28,x28,x25			// Maj(a,b,c)
+	eor	x17,x7,x17,ror#34	// Sigma0(a)
+	add	x23,x23,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	//add	x23,x23,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x0,x0			// 13
+#endif
+	ldp	x1,x2,[x1]
+	add	x23,x23,x17			// h+=Sigma0(a)
+	str	x8,[sp,#8]
+	ror	x16,x27,#14
+	add	x22,x22,x28			// h+=K[i]
+	eor	x8,x27,x27,ror#23
+	and	x17,x20,x27
+	bic	x28,x21,x27
+	add	x22,x22,x0			// h+=X[i]
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x23,x24			// a^b, b^c in next round
+	eor	x16,x16,x8,ror#18	// Sigma1(e)
+	ror	x8,x23,#28
+	add	x22,x22,x17			// h+=Ch(e,f,g)
+	eor	x17,x23,x23,ror#5
+	add	x22,x22,x16			// h+=Sigma1(e)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	add	x26,x26,x22			// d+=h
+	eor	x19,x19,x24			// Maj(a,b,c)
+	eor	x17,x8,x17,ror#34	// Sigma0(a)
+	add	x22,x22,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	//add	x22,x22,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x1,x1			// 14
+#endif
+	ldr	x6,[sp,#24]
+	add	x22,x22,x17			// h+=Sigma0(a)
+	str	x9,[sp,#16]
+	ror	x16,x26,#14
+	add	x21,x21,x19			// h+=K[i]
+	eor	x9,x26,x26,ror#23
+	and	x17,x27,x26
+	bic	x19,x20,x26
+	add	x21,x21,x1			// h+=X[i]
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x22,x23			// a^b, b^c in next round
+	eor	x16,x16,x9,ror#18	// Sigma1(e)
+	ror	x9,x22,#28
+	add	x21,x21,x17			// h+=Ch(e,f,g)
+	eor	x17,x22,x22,ror#5
+	add	x21,x21,x16			// h+=Sigma1(e)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	add	x25,x25,x21			// d+=h
+	eor	x28,x28,x23			// Maj(a,b,c)
+	eor	x17,x9,x17,ror#34	// Sigma0(a)
+	add	x21,x21,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	//add	x21,x21,x17			// h+=Sigma0(a)
+#ifndef	__AARCH64EB__
+	rev	x2,x2			// 15
+#endif
+	ldr	x7,[sp,#0]
+	add	x21,x21,x17			// h+=Sigma0(a)
+	str	x10,[sp,#24]
+	ror	x16,x25,#14
+	add	x20,x20,x28			// h+=K[i]
+	ror	x9,x4,#1
+	and	x17,x26,x25
+	ror	x8,x1,#19
+	bic	x28,x27,x25
+	ror	x10,x21,#28
+	add	x20,x20,x2			// h+=X[i]
+	eor	x16,x16,x25,ror#18
+	eor	x9,x9,x4,ror#8
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x21,x22			// a^b, b^c in next round
+	eor	x16,x16,x25,ror#41	// Sigma1(e)
+	eor	x10,x10,x21,ror#34
+	add	x20,x20,x17			// h+=Ch(e,f,g)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	eor	x8,x8,x1,ror#61
+	eor	x9,x9,x4,lsr#7	// sigma0(X[i+1])
+	add	x20,x20,x16			// h+=Sigma1(e)
+	eor	x19,x19,x22			// Maj(a,b,c)
+	eor	x17,x10,x21,ror#39	// Sigma0(a)
+	eor	x8,x8,x1,lsr#6	// sigma1(X[i+14])
+	add	x3,x3,x12
+	add	x24,x24,x20			// d+=h
+	add	x20,x20,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	add	x3,x3,x9
+	add	x20,x20,x17			// h+=Sigma0(a)
+	add	x3,x3,x8
+.Loop_16_xx:
+	ldr	x8,[sp,#8]
+	str	x11,[sp,#0]
+	ror	x16,x24,#14
+	add	x27,x27,x19			// h+=K[i]
+	ror	x10,x5,#1
+	and	x17,x25,x24
+	ror	x9,x2,#19
+	bic	x19,x26,x24
+	ror	x11,x20,#28
+	add	x27,x27,x3			// h+=X[i]
+	eor	x16,x16,x24,ror#18
+	eor	x10,x10,x5,ror#8
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x20,x21			// a^b, b^c in next round
+	eor	x16,x16,x24,ror#41	// Sigma1(e)
+	eor	x11,x11,x20,ror#34
+	add	x27,x27,x17			// h+=Ch(e,f,g)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	eor	x9,x9,x2,ror#61
+	eor	x10,x10,x5,lsr#7	// sigma0(X[i+1])
+	add	x27,x27,x16			// h+=Sigma1(e)
+	eor	x28,x28,x21			// Maj(a,b,c)
+	eor	x17,x11,x20,ror#39	// Sigma0(a)
+	eor	x9,x9,x2,lsr#6	// sigma1(X[i+14])
+	add	x4,x4,x13
+	add	x23,x23,x27			// d+=h
+	add	x27,x27,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	add	x4,x4,x10
+	add	x27,x27,x17			// h+=Sigma0(a)
+	add	x4,x4,x9
+	ldr	x9,[sp,#16]
+	str	x12,[sp,#8]
+	ror	x16,x23,#14
+	add	x26,x26,x28			// h+=K[i]
+	ror	x11,x6,#1
+	and	x17,x24,x23
+	ror	x10,x3,#19
+	bic	x28,x25,x23
+	ror	x12,x27,#28
+	add	x26,x26,x4			// h+=X[i]
+	eor	x16,x16,x23,ror#18
+	eor	x11,x11,x6,ror#8
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x27,x20			// a^b, b^c in next round
+	eor	x16,x16,x23,ror#41	// Sigma1(e)
+	eor	x12,x12,x27,ror#34
+	add	x26,x26,x17			// h+=Ch(e,f,g)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	eor	x10,x10,x3,ror#61
+	eor	x11,x11,x6,lsr#7	// sigma0(X[i+1])
+	add	x26,x26,x16			// h+=Sigma1(e)
+	eor	x19,x19,x20			// Maj(a,b,c)
+	eor	x17,x12,x27,ror#39	// Sigma0(a)
+	eor	x10,x10,x3,lsr#6	// sigma1(X[i+14])
+	add	x5,x5,x14
+	add	x22,x22,x26			// d+=h
+	add	x26,x26,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	add	x5,x5,x11
+	add	x26,x26,x17			// h+=Sigma0(a)
+	add	x5,x5,x10
+	ldr	x10,[sp,#24]
+	str	x13,[sp,#16]
+	ror	x16,x22,#14
+	add	x25,x25,x19			// h+=K[i]
+	ror	x12,x7,#1
+	and	x17,x23,x22
+	ror	x11,x4,#19
+	bic	x19,x24,x22
+	ror	x13,x26,#28
+	add	x25,x25,x5			// h+=X[i]
+	eor	x16,x16,x22,ror#18
+	eor	x12,x12,x7,ror#8
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x26,x27			// a^b, b^c in next round
+	eor	x16,x16,x22,ror#41	// Sigma1(e)
+	eor	x13,x13,x26,ror#34
+	add	x25,x25,x17			// h+=Ch(e,f,g)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	eor	x11,x11,x4,ror#61
+	eor	x12,x12,x7,lsr#7	// sigma0(X[i+1])
+	add	x25,x25,x16			// h+=Sigma1(e)
+	eor	x28,x28,x27			// Maj(a,b,c)
+	eor	x17,x13,x26,ror#39	// Sigma0(a)
+	eor	x11,x11,x4,lsr#6	// sigma1(X[i+14])
+	add	x6,x6,x15
+	add	x21,x21,x25			// d+=h
+	add	x25,x25,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	add	x6,x6,x12
+	add	x25,x25,x17			// h+=Sigma0(a)
+	add	x6,x6,x11
+	ldr	x11,[sp,#0]
+	str	x14,[sp,#24]
+	ror	x16,x21,#14
+	add	x24,x24,x28			// h+=K[i]
+	ror	x13,x8,#1
+	and	x17,x22,x21
+	ror	x12,x5,#19
+	bic	x28,x23,x21
+	ror	x14,x25,#28
+	add	x24,x24,x6			// h+=X[i]
+	eor	x16,x16,x21,ror#18
+	eor	x13,x13,x8,ror#8
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x25,x26			// a^b, b^c in next round
+	eor	x16,x16,x21,ror#41	// Sigma1(e)
+	eor	x14,x14,x25,ror#34
+	add	x24,x24,x17			// h+=Ch(e,f,g)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	eor	x12,x12,x5,ror#61
+	eor	x13,x13,x8,lsr#7	// sigma0(X[i+1])
+	add	x24,x24,x16			// h+=Sigma1(e)
+	eor	x19,x19,x26			// Maj(a,b,c)
+	eor	x17,x14,x25,ror#39	// Sigma0(a)
+	eor	x12,x12,x5,lsr#6	// sigma1(X[i+14])
+	add	x7,x7,x0
+	add	x20,x20,x24			// d+=h
+	add	x24,x24,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	add	x7,x7,x13
+	add	x24,x24,x17			// h+=Sigma0(a)
+	add	x7,x7,x12
+	ldr	x12,[sp,#8]
+	str	x15,[sp,#0]
+	ror	x16,x20,#14
+	add	x23,x23,x19			// h+=K[i]
+	ror	x14,x9,#1
+	and	x17,x21,x20
+	ror	x13,x6,#19
+	bic	x19,x22,x20
+	ror	x15,x24,#28
+	add	x23,x23,x7			// h+=X[i]
+	eor	x16,x16,x20,ror#18
+	eor	x14,x14,x9,ror#8
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x24,x25			// a^b, b^c in next round
+	eor	x16,x16,x20,ror#41	// Sigma1(e)
+	eor	x15,x15,x24,ror#34
+	add	x23,x23,x17			// h+=Ch(e,f,g)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	eor	x13,x13,x6,ror#61
+	eor	x14,x14,x9,lsr#7	// sigma0(X[i+1])
+	add	x23,x23,x16			// h+=Sigma1(e)
+	eor	x28,x28,x25			// Maj(a,b,c)
+	eor	x17,x15,x24,ror#39	// Sigma0(a)
+	eor	x13,x13,x6,lsr#6	// sigma1(X[i+14])
+	add	x8,x8,x1
+	add	x27,x27,x23			// d+=h
+	add	x23,x23,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	add	x8,x8,x14
+	add	x23,x23,x17			// h+=Sigma0(a)
+	add	x8,x8,x13
+	ldr	x13,[sp,#16]
+	str	x0,[sp,#8]
+	ror	x16,x27,#14
+	add	x22,x22,x28			// h+=K[i]
+	ror	x15,x10,#1
+	and	x17,x20,x27
+	ror	x14,x7,#19
+	bic	x28,x21,x27
+	ror	x0,x23,#28
+	add	x22,x22,x8			// h+=X[i]
+	eor	x16,x16,x27,ror#18
+	eor	x15,x15,x10,ror#8
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x23,x24			// a^b, b^c in next round
+	eor	x16,x16,x27,ror#41	// Sigma1(e)
+	eor	x0,x0,x23,ror#34
+	add	x22,x22,x17			// h+=Ch(e,f,g)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	eor	x14,x14,x7,ror#61
+	eor	x15,x15,x10,lsr#7	// sigma0(X[i+1])
+	add	x22,x22,x16			// h+=Sigma1(e)
+	eor	x19,x19,x24			// Maj(a,b,c)
+	eor	x17,x0,x23,ror#39	// Sigma0(a)
+	eor	x14,x14,x7,lsr#6	// sigma1(X[i+14])
+	add	x9,x9,x2
+	add	x26,x26,x22			// d+=h
+	add	x22,x22,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	add	x9,x9,x15
+	add	x22,x22,x17			// h+=Sigma0(a)
+	add	x9,x9,x14
+	ldr	x14,[sp,#24]
+	str	x1,[sp,#16]
+	ror	x16,x26,#14
+	add	x21,x21,x19			// h+=K[i]
+	ror	x0,x11,#1
+	and	x17,x27,x26
+	ror	x15,x8,#19
+	bic	x19,x20,x26
+	ror	x1,x22,#28
+	add	x21,x21,x9			// h+=X[i]
+	eor	x16,x16,x26,ror#18
+	eor	x0,x0,x11,ror#8
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x22,x23			// a^b, b^c in next round
+	eor	x16,x16,x26,ror#41	// Sigma1(e)
+	eor	x1,x1,x22,ror#34
+	add	x21,x21,x17			// h+=Ch(e,f,g)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	eor	x15,x15,x8,ror#61
+	eor	x0,x0,x11,lsr#7	// sigma0(X[i+1])
+	add	x21,x21,x16			// h+=Sigma1(e)
+	eor	x28,x28,x23			// Maj(a,b,c)
+	eor	x17,x1,x22,ror#39	// Sigma0(a)
+	eor	x15,x15,x8,lsr#6	// sigma1(X[i+14])
+	add	x10,x10,x3
+	add	x25,x25,x21			// d+=h
+	add	x21,x21,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	add	x10,x10,x0
+	add	x21,x21,x17			// h+=Sigma0(a)
+	add	x10,x10,x15
+	ldr	x15,[sp,#0]
+	str	x2,[sp,#24]
+	ror	x16,x25,#14
+	add	x20,x20,x28			// h+=K[i]
+	ror	x1,x12,#1
+	and	x17,x26,x25
+	ror	x0,x9,#19
+	bic	x28,x27,x25
+	ror	x2,x21,#28
+	add	x20,x20,x10			// h+=X[i]
+	eor	x16,x16,x25,ror#18
+	eor	x1,x1,x12,ror#8
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x21,x22			// a^b, b^c in next round
+	eor	x16,x16,x25,ror#41	// Sigma1(e)
+	eor	x2,x2,x21,ror#34
+	add	x20,x20,x17			// h+=Ch(e,f,g)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	eor	x0,x0,x9,ror#61
+	eor	x1,x1,x12,lsr#7	// sigma0(X[i+1])
+	add	x20,x20,x16			// h+=Sigma1(e)
+	eor	x19,x19,x22			// Maj(a,b,c)
+	eor	x17,x2,x21,ror#39	// Sigma0(a)
+	eor	x0,x0,x9,lsr#6	// sigma1(X[i+14])
+	add	x11,x11,x4
+	add	x24,x24,x20			// d+=h
+	add	x20,x20,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	add	x11,x11,x1
+	add	x20,x20,x17			// h+=Sigma0(a)
+	add	x11,x11,x0
+	ldr	x0,[sp,#8]
+	str	x3,[sp,#0]
+	ror	x16,x24,#14
+	add	x27,x27,x19			// h+=K[i]
+	ror	x2,x13,#1
+	and	x17,x25,x24
+	ror	x1,x10,#19
+	bic	x19,x26,x24
+	ror	x3,x20,#28
+	add	x27,x27,x11			// h+=X[i]
+	eor	x16,x16,x24,ror#18
+	eor	x2,x2,x13,ror#8
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x20,x21			// a^b, b^c in next round
+	eor	x16,x16,x24,ror#41	// Sigma1(e)
+	eor	x3,x3,x20,ror#34
+	add	x27,x27,x17			// h+=Ch(e,f,g)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	eor	x1,x1,x10,ror#61
+	eor	x2,x2,x13,lsr#7	// sigma0(X[i+1])
+	add	x27,x27,x16			// h+=Sigma1(e)
+	eor	x28,x28,x21			// Maj(a,b,c)
+	eor	x17,x3,x20,ror#39	// Sigma0(a)
+	eor	x1,x1,x10,lsr#6	// sigma1(X[i+14])
+	add	x12,x12,x5
+	add	x23,x23,x27			// d+=h
+	add	x27,x27,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	add	x12,x12,x2
+	add	x27,x27,x17			// h+=Sigma0(a)
+	add	x12,x12,x1
+	ldr	x1,[sp,#16]
+	str	x4,[sp,#8]
+	ror	x16,x23,#14
+	add	x26,x26,x28			// h+=K[i]
+	ror	x3,x14,#1
+	and	x17,x24,x23
+	ror	x2,x11,#19
+	bic	x28,x25,x23
+	ror	x4,x27,#28
+	add	x26,x26,x12			// h+=X[i]
+	eor	x16,x16,x23,ror#18
+	eor	x3,x3,x14,ror#8
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x27,x20			// a^b, b^c in next round
+	eor	x16,x16,x23,ror#41	// Sigma1(e)
+	eor	x4,x4,x27,ror#34
+	add	x26,x26,x17			// h+=Ch(e,f,g)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	eor	x2,x2,x11,ror#61
+	eor	x3,x3,x14,lsr#7	// sigma0(X[i+1])
+	add	x26,x26,x16			// h+=Sigma1(e)
+	eor	x19,x19,x20			// Maj(a,b,c)
+	eor	x17,x4,x27,ror#39	// Sigma0(a)
+	eor	x2,x2,x11,lsr#6	// sigma1(X[i+14])
+	add	x13,x13,x6
+	add	x22,x22,x26			// d+=h
+	add	x26,x26,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	add	x13,x13,x3
+	add	x26,x26,x17			// h+=Sigma0(a)
+	add	x13,x13,x2
+	ldr	x2,[sp,#24]
+	str	x5,[sp,#16]
+	ror	x16,x22,#14
+	add	x25,x25,x19			// h+=K[i]
+	ror	x4,x15,#1
+	and	x17,x23,x22
+	ror	x3,x12,#19
+	bic	x19,x24,x22
+	ror	x5,x26,#28
+	add	x25,x25,x13			// h+=X[i]
+	eor	x16,x16,x22,ror#18
+	eor	x4,x4,x15,ror#8
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x26,x27			// a^b, b^c in next round
+	eor	x16,x16,x22,ror#41	// Sigma1(e)
+	eor	x5,x5,x26,ror#34
+	add	x25,x25,x17			// h+=Ch(e,f,g)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	eor	x3,x3,x12,ror#61
+	eor	x4,x4,x15,lsr#7	// sigma0(X[i+1])
+	add	x25,x25,x16			// h+=Sigma1(e)
+	eor	x28,x28,x27			// Maj(a,b,c)
+	eor	x17,x5,x26,ror#39	// Sigma0(a)
+	eor	x3,x3,x12,lsr#6	// sigma1(X[i+14])
+	add	x14,x14,x7
+	add	x21,x21,x25			// d+=h
+	add	x25,x25,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	add	x14,x14,x4
+	add	x25,x25,x17			// h+=Sigma0(a)
+	add	x14,x14,x3
+	ldr	x3,[sp,#0]
+	str	x6,[sp,#24]
+	ror	x16,x21,#14
+	add	x24,x24,x28			// h+=K[i]
+	ror	x5,x0,#1
+	and	x17,x22,x21
+	ror	x4,x13,#19
+	bic	x28,x23,x21
+	ror	x6,x25,#28
+	add	x24,x24,x14			// h+=X[i]
+	eor	x16,x16,x21,ror#18
+	eor	x5,x5,x0,ror#8
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x25,x26			// a^b, b^c in next round
+	eor	x16,x16,x21,ror#41	// Sigma1(e)
+	eor	x6,x6,x25,ror#34
+	add	x24,x24,x17			// h+=Ch(e,f,g)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	eor	x4,x4,x13,ror#61
+	eor	x5,x5,x0,lsr#7	// sigma0(X[i+1])
+	add	x24,x24,x16			// h+=Sigma1(e)
+	eor	x19,x19,x26			// Maj(a,b,c)
+	eor	x17,x6,x25,ror#39	// Sigma0(a)
+	eor	x4,x4,x13,lsr#6	// sigma1(X[i+14])
+	add	x15,x15,x8
+	add	x20,x20,x24			// d+=h
+	add	x24,x24,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	add	x15,x15,x5
+	add	x24,x24,x17			// h+=Sigma0(a)
+	add	x15,x15,x4
+	ldr	x4,[sp,#8]
+	str	x7,[sp,#0]
+	ror	x16,x20,#14
+	add	x23,x23,x19			// h+=K[i]
+	ror	x6,x1,#1
+	and	x17,x21,x20
+	ror	x5,x14,#19
+	bic	x19,x22,x20
+	ror	x7,x24,#28
+	add	x23,x23,x15			// h+=X[i]
+	eor	x16,x16,x20,ror#18
+	eor	x6,x6,x1,ror#8
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x24,x25			// a^b, b^c in next round
+	eor	x16,x16,x20,ror#41	// Sigma1(e)
+	eor	x7,x7,x24,ror#34
+	add	x23,x23,x17			// h+=Ch(e,f,g)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	eor	x5,x5,x14,ror#61
+	eor	x6,x6,x1,lsr#7	// sigma0(X[i+1])
+	add	x23,x23,x16			// h+=Sigma1(e)
+	eor	x28,x28,x25			// Maj(a,b,c)
+	eor	x17,x7,x24,ror#39	// Sigma0(a)
+	eor	x5,x5,x14,lsr#6	// sigma1(X[i+14])
+	add	x0,x0,x9
+	add	x27,x27,x23			// d+=h
+	add	x23,x23,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	add	x0,x0,x6
+	add	x23,x23,x17			// h+=Sigma0(a)
+	add	x0,x0,x5
+	ldr	x5,[sp,#16]
+	str	x8,[sp,#8]
+	ror	x16,x27,#14
+	add	x22,x22,x28			// h+=K[i]
+	ror	x7,x2,#1
+	and	x17,x20,x27
+	ror	x6,x15,#19
+	bic	x28,x21,x27
+	ror	x8,x23,#28
+	add	x22,x22,x0			// h+=X[i]
+	eor	x16,x16,x27,ror#18
+	eor	x7,x7,x2,ror#8
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x23,x24			// a^b, b^c in next round
+	eor	x16,x16,x27,ror#41	// Sigma1(e)
+	eor	x8,x8,x23,ror#34
+	add	x22,x22,x17			// h+=Ch(e,f,g)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	eor	x6,x6,x15,ror#61
+	eor	x7,x7,x2,lsr#7	// sigma0(X[i+1])
+	add	x22,x22,x16			// h+=Sigma1(e)
+	eor	x19,x19,x24			// Maj(a,b,c)
+	eor	x17,x8,x23,ror#39	// Sigma0(a)
+	eor	x6,x6,x15,lsr#6	// sigma1(X[i+14])
+	add	x1,x1,x10
+	add	x26,x26,x22			// d+=h
+	add	x22,x22,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	add	x1,x1,x7
+	add	x22,x22,x17			// h+=Sigma0(a)
+	add	x1,x1,x6
+	ldr	x6,[sp,#24]
+	str	x9,[sp,#16]
+	ror	x16,x26,#14
+	add	x21,x21,x19			// h+=K[i]
+	ror	x8,x3,#1
+	and	x17,x27,x26
+	ror	x7,x0,#19
+	bic	x19,x20,x26
+	ror	x9,x22,#28
+	add	x21,x21,x1			// h+=X[i]
+	eor	x16,x16,x26,ror#18
+	eor	x8,x8,x3,ror#8
+	orr	x17,x17,x19			// Ch(e,f,g)
+	eor	x19,x22,x23			// a^b, b^c in next round
+	eor	x16,x16,x26,ror#41	// Sigma1(e)
+	eor	x9,x9,x22,ror#34
+	add	x21,x21,x17			// h+=Ch(e,f,g)
+	and	x28,x28,x19			// (b^c)&=(a^b)
+	eor	x7,x7,x0,ror#61
+	eor	x8,x8,x3,lsr#7	// sigma0(X[i+1])
+	add	x21,x21,x16			// h+=Sigma1(e)
+	eor	x28,x28,x23			// Maj(a,b,c)
+	eor	x17,x9,x22,ror#39	// Sigma0(a)
+	eor	x7,x7,x0,lsr#6	// sigma1(X[i+14])
+	add	x2,x2,x11
+	add	x25,x25,x21			// d+=h
+	add	x21,x21,x28			// h+=Maj(a,b,c)
+	ldr	x28,[x30],#8		// *K++, x19 in next round
+	add	x2,x2,x8
+	add	x21,x21,x17			// h+=Sigma0(a)
+	add	x2,x2,x7
+	ldr	x7,[sp,#0]
+	str	x10,[sp,#24]
+	ror	x16,x25,#14
+	add	x20,x20,x28			// h+=K[i]
+	ror	x9,x4,#1
+	and	x17,x26,x25
+	ror	x8,x1,#19
+	bic	x28,x27,x25
+	ror	x10,x21,#28
+	add	x20,x20,x2			// h+=X[i]
+	eor	x16,x16,x25,ror#18
+	eor	x9,x9,x4,ror#8
+	orr	x17,x17,x28			// Ch(e,f,g)
+	eor	x28,x21,x22			// a^b, b^c in next round
+	eor	x16,x16,x25,ror#41	// Sigma1(e)
+	eor	x10,x10,x21,ror#34
+	add	x20,x20,x17			// h+=Ch(e,f,g)
+	and	x19,x19,x28			// (b^c)&=(a^b)
+	eor	x8,x8,x1,ror#61
+	eor	x9,x9,x4,lsr#7	// sigma0(X[i+1])
+	add	x20,x20,x16			// h+=Sigma1(e)
+	eor	x19,x19,x22			// Maj(a,b,c)
+	eor	x17,x10,x21,ror#39	// Sigma0(a)
+	eor	x8,x8,x1,lsr#6	// sigma1(X[i+14])
+	add	x3,x3,x12
+	add	x24,x24,x20			// d+=h
+	add	x20,x20,x19			// h+=Maj(a,b,c)
+	ldr	x19,[x30],#8		// *K++, x28 in next round
+	add	x3,x3,x9
+	add	x20,x20,x17			// h+=Sigma0(a)
+	add	x3,x3,x8
+	cbnz	x19,.Loop_16_xx
+
+	ldp	x0,x2,[x29,#96]
+	ldr	x1,[x29,#112]
+	sub	x30,x30,#648		// rewind
+
+	ldp	x3,x4,[x0]
+	ldp	x5,x6,[x0,#2*8]
+	add	x1,x1,#14*8			// advance input pointer
+	ldp	x7,x8,[x0,#4*8]
+	add	x20,x20,x3
+	ldp	x9,x10,[x0,#6*8]
+	add	x21,x21,x4
+	add	x22,x22,x5
+	add	x23,x23,x6
+	stp	x20,x21,[x0]
+	add	x24,x24,x7
+	add	x25,x25,x8
+	stp	x22,x23,[x0,#2*8]
+	add	x26,x26,x9
+	add	x27,x27,x10
+	cmp	x1,x2
+	stp	x24,x25,[x0,#4*8]
+	stp	x26,x27,[x0,#6*8]
+	b.ne	.Loop
+
+	ldp	x19,x20,[x29,#16]
+	add	sp,sp,#4*8
+	ldp	x21,x22,[x29,#32]
+	ldp	x23,x24,[x29,#48]
+	ldp	x25,x26,[x29,#64]
+	ldp	x27,x28,[x29,#80]
+	ldp	x29,x30,[sp],#128
+	ret
+.size	sha512_block_data_order,.-sha512_block_data_order
+
+.align	6
+.type	.LK512,%object
+.LK512:
+	.quad	0x428a2f98d728ae22,0x7137449123ef65cd
+	.quad	0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
+	.quad	0x3956c25bf348b538,0x59f111f1b605d019
+	.quad	0x923f82a4af194f9b,0xab1c5ed5da6d8118
+	.quad	0xd807aa98a3030242,0x12835b0145706fbe
+	.quad	0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
+	.quad	0x72be5d74f27b896f,0x80deb1fe3b1696b1
+	.quad	0x9bdc06a725c71235,0xc19bf174cf692694
+	.quad	0xe49b69c19ef14ad2,0xefbe4786384f25e3
+	.quad	0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
+	.quad	0x2de92c6f592b0275,0x4a7484aa6ea6e483
+	.quad	0x5cb0a9dcbd41fbd4,0x76f988da831153b5
+	.quad	0x983e5152ee66dfab,0xa831c66d2db43210
+	.quad	0xb00327c898fb213f,0xbf597fc7beef0ee4
+	.quad	0xc6e00bf33da88fc2,0xd5a79147930aa725
+	.quad	0x06ca6351e003826f,0x142929670a0e6e70
+	.quad	0x27b70a8546d22ffc,0x2e1b21385c26c926
+	.quad	0x4d2c6dfc5ac42aed,0x53380d139d95b3df
+	.quad	0x650a73548baf63de,0x766a0abb3c77b2a8
+	.quad	0x81c2c92e47edaee6,0x92722c851482353b
+	.quad	0xa2bfe8a14cf10364,0xa81a664bbc423001
+	.quad	0xc24b8b70d0f89791,0xc76c51a30654be30
+	.quad	0xd192e819d6ef5218,0xd69906245565a910
+	.quad	0xf40e35855771202a,0x106aa07032bbd1b8
+	.quad	0x19a4c116b8d2d0c8,0x1e376c085141ab53
+	.quad	0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
+	.quad	0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
+	.quad	0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
+	.quad	0x748f82ee5defb2fc,0x78a5636f43172f60
+	.quad	0x84c87814a1f0ab72,0x8cc702081a6439ec
+	.quad	0x90befffa23631e28,0xa4506cebde82bde9
+	.quad	0xbef9a3f7b2c67915,0xc67178f2e372532b
+	.quad	0xca273eceea26619c,0xd186b8c721c0c207
+	.quad	0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
+	.quad	0x06f067aa72176fba,0x0a637dc5a2c898a6
+	.quad	0x113f9804bef90dae,0x1b710b35131c471b
+	.quad	0x28db77f523047d84,0x32caab7b40c72493
+	.quad	0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
+	.quad	0x4cc5d4becb3e42b6,0x597f299cfc657e2a
+	.quad	0x5fcb6fab3ad6faec,0x6c44198c4a475817
+	.quad	0	// terminator
+.size	.LK512,.-.LK512
+#ifndef	__KERNEL__
+.align	3
+.LOPENSSL_armcap_P:
+# ifdef	__ILP32__
+	.long	OPENSSL_armcap_P-.
+# else
+	.quad	OPENSSL_armcap_P-.
+# endif
+#endif
+.asciz	"SHA512 block transform for ARMv8, CRYPTOGAMS by <appro@openssl.org>"
+.align	2
+#ifndef	__KERNEL__
+.comm	OPENSSL_armcap_P,4,4
+#endif
diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c
new file mode 100644
index 000000000000..aff35c9992a4
--- /dev/null
+++ b/arch/arm64/crypto/sha512-glue.c
@@ -0,0 +1,94 @@
+/*
+ * Linux/arm64 port of the OpenSSL SHA512 implementation for AArch64
+ *
+ * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#include <crypto/internal/hash.h>
+#include <linux/cryptohash.h>
+#include <linux/types.h>
+#include <linux/string.h>
+#include <crypto/sha.h>
+#include <crypto/sha512_base.h>
+#include <asm/neon.h>
+
+MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash for arm64");
+MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("sha384");
+MODULE_ALIAS_CRYPTO("sha512");
+
+asmlinkage void sha512_block_data_order(u32 *digest, const void *data,
+					unsigned int num_blks);
+
+static int sha512_update(struct shash_desc *desc, const u8 *data,
+			 unsigned int len)
+{
+	return sha512_base_do_update(desc, data, len,
+			(sha512_block_fn *)sha512_block_data_order);
+}
+
+static int sha512_finup(struct shash_desc *desc, const u8 *data,
+			unsigned int len, u8 *out)
+{
+	if (len)
+		sha512_base_do_update(desc, data, len,
+			(sha512_block_fn *)sha512_block_data_order);
+	sha512_base_do_finalize(desc,
+			(sha512_block_fn *)sha512_block_data_order);
+
+	return sha512_base_finish(desc, out);
+}
+
+static int sha512_final(struct shash_desc *desc, u8 *out)
+{
+	return sha512_finup(desc, NULL, 0, out);
+}
+
+static struct shash_alg algs[] = { {
+	.digestsize		= SHA512_DIGEST_SIZE,
+	.init			= sha512_base_init,
+	.update			= sha512_update,
+	.final			= sha512_final,
+	.finup			= sha512_finup,
+	.descsize		= sizeof(struct sha512_state),
+	.base.cra_name		= "sha512",
+	.base.cra_driver_name	= "sha512-arm64",
+	.base.cra_priority	= 150,
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA512_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+}, {
+	.digestsize		= SHA384_DIGEST_SIZE,
+	.init			= sha384_base_init,
+	.update			= sha512_update,
+	.final			= sha512_final,
+	.finup			= sha512_finup,
+	.descsize		= sizeof(struct sha512_state),
+	.base.cra_name		= "sha384",
+	.base.cra_driver_name	= "sha384-arm64",
+	.base.cra_priority	= 150,
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA384_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+} };
+
+static int __init sha512_mod_init(void)
+{
+	return crypto_register_shashes(algs, ARRAY_SIZE(algs));
+}
+
+static void __exit sha512_mod_fini(void)
+{
+	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
+}
+
+module_init(sha512_mod_init);
+module_exit(sha512_mod_fini);
diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild
index b4ab238a59ec..8365a84c2640 100644
--- a/arch/arm64/include/asm/Kbuild
+++ b/arch/arm64/include/asm/Kbuild
@@ -1,7 +1,6 @@
 generic-y += bugs.h
 generic-y += clkdev.h
 generic-y += cputime.h
-generic-y += current.h
 generic-y += delay.h
 generic-y += div64.h
 generic-y += dma.h
diff --git a/arch/arm64/include/asm/acpi.h b/arch/arm64/include/asm/acpi.h
index e517088d635f..d0de0e032bc2 100644
--- a/arch/arm64/include/asm/acpi.h
+++ b/arch/arm64/include/asm/acpi.h
@@ -17,6 +17,7 @@
 
 #include <asm/cputype.h>
 #include <asm/smp_plat.h>
+#include <asm/tlbflush.h>
 
 /* Macros for consistency checks of the GICC subtable of MADT */
 #define ACPI_MADT_GICC_LENGTH	\
@@ -114,8 +115,28 @@ static inline const char *acpi_get_enable_method(int cpu)
 }
 
 #ifdef	CONFIG_ACPI_APEI
+/*
+ * acpi_disable_cmcff is used in drivers/acpi/apei/hest.c for disabling
+ * IA-32 Architecture Corrected Machine Check (CMC) Firmware-First mode
+ * with a kernel command line parameter "acpi=nocmcoff". But we don't
+ * have this IA-32 specific feature on ARM64, this definition is only
+ * for compatibility.
+ */
+#define acpi_disable_cmcff 1
 pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr);
-#endif
+
+/*
+ * Despite its name, this function must still broadcast the TLB
+ * invalidation in order to ensure other CPUs don't end up with junk
+ * entries as a result of speculation. Unusually, its also called in
+ * IRQ context (ghes_iounmap_irq) so if we ever need to use IPIs for
+ * TLB broadcasting, then we're in trouble here.
+ */
+static inline void arch_apei_flush_tlb_one(unsigned long addr)
+{
+	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+}
+#endif /* CONFIG_ACPI_APEI */
 
 #ifdef CONFIG_ACPI_NUMA
 int arm64_acpi_numa_init(void);
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 28bfe6132eb6..446f6c46d4b1 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -41,6 +41,15 @@
 	msr	daifclr, #2
 	.endm
 
+	.macro	save_and_disable_irq, flags
+	mrs	\flags, daif
+	msr	daifset, #2
+	.endm
+
+	.macro	restore_irq, flags
+	msr	daif, \flags
+	.endm
+
 /*
  * Enable and disable debug exceptions.
  */
@@ -202,14 +211,25 @@ lr	.req	x30		// link register
 	.endm
 
 	/*
+	 * @dst: Result of per_cpu(sym, smp_processor_id())
 	 * @sym: The name of the per-cpu variable
-	 * @reg: Result of per_cpu(sym, smp_processor_id())
 	 * @tmp: scratch register
 	 */
-	.macro this_cpu_ptr, sym, reg, tmp
-	adr_l	\reg, \sym
+	.macro adr_this_cpu, dst, sym, tmp
+	adr_l	\dst, \sym
 	mrs	\tmp, tpidr_el1
-	add	\reg, \reg, \tmp
+	add	\dst, \dst, \tmp
+	.endm
+
+	/*
+	 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id()))
+	 * @sym: The name of the per-cpu variable
+	 * @tmp: scratch register
+	 */
+	.macro ldr_this_cpu dst, sym, tmp
+	adr_l	\dst, \sym
+	mrs	\tmp, tpidr_el1
+	ldr	\dst, [\dst, \tmp]
 	.endm
 
 /*
@@ -395,4 +415,24 @@ alternative_endif
 	movk	\reg, :abs_g0_nc:\val
 	.endm
 
+/*
+ * Return the current thread_info.
+ */
+	.macro	get_thread_info, rd
+	mrs	\rd, sp_el0
+	.endm
+
+/*
+ * Errata workaround post TTBR0_EL1 update.
+ */
+	.macro	post_ttbr0_update_workaround
+#ifdef CONFIG_CAVIUM_ERRATUM_27456
+alternative_if ARM64_WORKAROUND_CAVIUM_27456
+	ic	iallu
+	dsb	nsh
+	isb
+alternative_else_nop_endif
+#endif
+	.endm
+
 #endif	/* __ASM_ASSEMBLER_H */
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index 2e5fb976a572..5a2a6ee65f65 100644
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -65,12 +65,12 @@
  *		- kaddr  - page address
  *		- size   - region size
  */
-extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 extern void flush_icache_range(unsigned long start, unsigned long end);
 extern void __flush_dcache_area(void *addr, size_t len);
 extern void __clean_dcache_area_poc(void *addr, size_t len);
 extern void __clean_dcache_area_pou(void *addr, size_t len);
 extern long __flush_cache_user_range(unsigned long start, unsigned long end);
+extern void sync_icache_aliases(void *kaddr, unsigned long len);
 
 static inline void flush_cache_mm(struct mm_struct *mm)
 {
@@ -81,6 +81,11 @@ static inline void flush_cache_page(struct vm_area_struct *vma,
 {
 }
 
+static inline void flush_cache_range(struct vm_area_struct *vma,
+				     unsigned long start, unsigned long end)
+{
+}
+
 /*
  * Cache maintenance functions used by the DMA API. No to be used directly.
  */
diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
index 87b446535185..4174f09678c4 100644
--- a/arch/arm64/include/asm/cpucaps.h
+++ b/arch/arm64/include/asm/cpucaps.h
@@ -34,7 +34,8 @@
 #define ARM64_HAS_32BIT_EL0			13
 #define ARM64_HYP_OFFSET_LOW			14
 #define ARM64_MISMATCHED_CACHE_LINE_SIZE	15
+#define ARM64_HAS_NO_FPSIMD			16
 
-#define ARM64_NCAPS				16
+#define ARM64_NCAPS				17
 
 #endif /* __ASM_CPUCAPS_H */
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 0bc0b1de90c4..b4989df48670 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -9,8 +9,6 @@
 #ifndef __ASM_CPUFEATURE_H
 #define __ASM_CPUFEATURE_H
 
-#include <linux/jump_label.h>
-
 #include <asm/cpucaps.h>
 #include <asm/hwcap.h>
 #include <asm/sysreg.h>
@@ -27,6 +25,8 @@
 
 #ifndef __ASSEMBLY__
 
+#include <linux/bug.h>
+#include <linux/jump_label.h>
 #include <linux/kernel.h>
 
 /* CPU feature register tracking */
@@ -104,14 +104,19 @@ static inline bool cpu_have_feature(unsigned int num)
 	return elf_hwcap & (1UL << num);
 }
 
+/* System capability check for constant caps */
+static inline bool cpus_have_const_cap(int num)
+{
+	if (num >= ARM64_NCAPS)
+		return false;
+	return static_branch_unlikely(&cpu_hwcap_keys[num]);
+}
+
 static inline bool cpus_have_cap(unsigned int num)
 {
 	if (num >= ARM64_NCAPS)
 		return false;
-	if (__builtin_constant_p(num))
-		return static_branch_unlikely(&cpu_hwcap_keys[num]);
-	else
-		return test_bit(num, cpu_hwcaps);
+	return test_bit(num, cpu_hwcaps);
 }
 
 static inline void cpus_set_cap(unsigned int num)
@@ -200,7 +205,7 @@ static inline bool cpu_supports_mixed_endian_el0(void)
 
 static inline bool system_supports_32bit_el0(void)
 {
-	return cpus_have_cap(ARM64_HAS_32BIT_EL0);
+	return cpus_have_const_cap(ARM64_HAS_32BIT_EL0);
 }
 
 static inline bool system_supports_mixed_endian_el0(void)
@@ -208,6 +213,17 @@ static inline bool system_supports_mixed_endian_el0(void)
 	return id_aa64mmfr0_mixed_endian_el0(read_system_reg(SYS_ID_AA64MMFR0_EL1));
 }
 
+static inline bool system_supports_fpsimd(void)
+{
+	return !cpus_have_const_cap(ARM64_HAS_NO_FPSIMD);
+}
+
+static inline bool system_uses_ttbr0_pan(void)
+{
+	return IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN) &&
+		!cpus_have_cap(ARM64_HAS_PAN);
+}
+
 #endif /* __ASSEMBLY__ */
 
 #endif
diff --git a/arch/arm64/include/asm/current.h b/arch/arm64/include/asm/current.h
new file mode 100644
index 000000000000..f2bcbe2d9889
--- /dev/null
+++ b/arch/arm64/include/asm/current.h
@@ -0,0 +1,22 @@
+#ifndef __ASM_CURRENT_H
+#define __ASM_CURRENT_H
+
+#include <linux/compiler.h>
+
+#include <asm/sysreg.h>
+
+#ifndef __ASSEMBLY__
+
+struct task_struct;
+
+static __always_inline struct task_struct *get_current(void)
+{
+	return (struct task_struct *)read_sysreg(sp_el0);
+}
+
+#define current get_current()
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* __ASM_CURRENT_H */
+
diff --git a/arch/arm64/include/asm/debug-monitors.h b/arch/arm64/include/asm/debug-monitors.h
index b71420a12f26..a44cf5225429 100644
--- a/arch/arm64/include/asm/debug-monitors.h
+++ b/arch/arm64/include/asm/debug-monitors.h
@@ -68,6 +68,9 @@
 #define BRK64_ESR_MASK		0xFFFF
 #define BRK64_ESR_KPROBES	0x0004
 #define BRK64_OPCODE_KPROBES	(AARCH64_BREAK_MON | (BRK64_ESR_KPROBES << 5))
+/* uprobes BRK opcodes with ESR encoding  */
+#define BRK64_ESR_UPROBES	0x0005
+#define BRK64_OPCODE_UPROBES	(AARCH64_BREAK_MON | (BRK64_ESR_UPROBES << 5))
 
 /* AArch32 */
 #define DBG_ESR_EVT_BKPT	0x4
diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h
index 771b3f0bc757..0b6b1633017f 100644
--- a/arch/arm64/include/asm/efi.h
+++ b/arch/arm64/include/asm/efi.h
@@ -1,6 +1,7 @@
 #ifndef _ASM_EFI_H
 #define _ASM_EFI_H
 
+#include <asm/cpufeature.h>
 #include <asm/io.h>
 #include <asm/mmu_context.h>
 #include <asm/neon.h>
@@ -78,7 +79,30 @@ static inline void efifb_setup_from_dmi(struct screen_info *si, const char *opt)
 
 static inline void efi_set_pgd(struct mm_struct *mm)
 {
-	switch_mm(NULL, mm, NULL);
+	__switch_mm(mm);
+
+	if (system_uses_ttbr0_pan()) {
+		if (mm != current->active_mm) {
+			/*
+			 * Update the current thread's saved ttbr0 since it is
+			 * restored as part of a return from exception. Set
+			 * the hardware TTBR0_EL1 using cpu_switch_mm()
+			 * directly to enable potential errata workarounds.
+			 */
+			update_saved_ttbr0(current, mm);
+			cpu_switch_mm(mm->pgd, mm);
+		} else {
+			/*
+			 * Defer the switch to the current thread's TTBR0_EL1
+			 * until uaccess_enable(). Restore the current
+			 * thread's saved ttbr0 corresponding to its active_mm
+			 * (if different from init_mm).
+			 */
+			cpu_set_reserved_ttbr0();
+			if (current->active_mm != &init_mm)
+				update_saved_ttbr0(current, current->active_mm);
+		}
+	}
 }
 
 void efi_virtmap_load(void);
diff --git a/arch/arm64/include/asm/elf.h b/arch/arm64/include/asm/elf.h
index a55384f4a5d7..5d1700425efe 100644
--- a/arch/arm64/include/asm/elf.h
+++ b/arch/arm64/include/asm/elf.h
@@ -138,7 +138,11 @@ typedef struct user_fpsimd_state elf_fpregset_t;
  */
 #define ELF_PLAT_INIT(_r, load_addr)	(_r)->regs[0] = 0
 
-#define SET_PERSONALITY(ex)		clear_thread_flag(TIF_32BIT);
+#define SET_PERSONALITY(ex)						\
+({									\
+	clear_bit(TIF_32BIT, &current->mm->context.flags);		\
+	clear_thread_flag(TIF_32BIT);					\
+})
 
 /* update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT entries changes */
 #define ARCH_DLINFO							\
@@ -183,7 +187,11 @@ typedef compat_elf_greg_t		compat_elf_gregset_t[COMPAT_ELF_NGREG];
 					 ((x)->e_flags & EF_ARM_EABI_MASK))
 
 #define compat_start_thread		compat_start_thread
-#define COMPAT_SET_PERSONALITY(ex)	set_thread_flag(TIF_32BIT);
+#define COMPAT_SET_PERSONALITY(ex)					\
+({									\
+	set_bit(TIF_32BIT, &current->mm->context.flags);		\
+	set_thread_flag(TIF_32BIT);					\
+ })
 #define COMPAT_ARCH_DLINFO
 extern int aarch32_setup_vectors_page(struct linux_binprm *bprm,
 				      int uses_interp);
diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h
index f2585cdd32c2..85c4a8981d47 100644
--- a/arch/arm64/include/asm/futex.h
+++ b/arch/arm64/include/asm/futex.h
@@ -21,15 +21,12 @@
 #include <linux/futex.h>
 #include <linux/uaccess.h>
 
-#include <asm/alternative.h>
-#include <asm/cpufeature.h>
 #include <asm/errno.h>
-#include <asm/sysreg.h>
 
 #define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
+do {									\
+	uaccess_enable();						\
 	asm volatile(							\
-	ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,		\
-		    CONFIG_ARM64_PAN)					\
 "	prfm	pstl1strm, %2\n"					\
 "1:	ldxr	%w1, %2\n"						\
 	insn "\n"							\
@@ -44,11 +41,11 @@
 "	.popsection\n"							\
 	_ASM_EXTABLE(1b, 4b)						\
 	_ASM_EXTABLE(2b, 4b)						\
-	ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,		\
-		    CONFIG_ARM64_PAN)					\
 	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp)	\
 	: "r" (oparg), "Ir" (-EFAULT)					\
-	: "memory")
+	: "memory");							\
+	uaccess_disable();						\
+} while (0)
 
 static inline int
 futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr)
@@ -118,8 +115,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
 	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
 		return -EFAULT;
 
+	uaccess_enable();
 	asm volatile("// futex_atomic_cmpxchg_inatomic\n"
-ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN)
 "	prfm	pstl1strm, %2\n"
 "1:	ldxr	%w1, %2\n"
 "	sub	%w3, %w1, %w4\n"
@@ -134,10 +131,10 @@ ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN)
 "	.popsection\n"
 	_ASM_EXTABLE(1b, 4b)
 	_ASM_EXTABLE(2b, 4b)
-ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN)
 	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)
 	: "r" (oldval), "r" (newval), "Ir" (-EFAULT)
 	: "memory");
+	uaccess_disable();
 
 	*uval = val;
 	return ret;
diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h
index 9510ace570e2..b6b167ac082b 100644
--- a/arch/arm64/include/asm/hw_breakpoint.h
+++ b/arch/arm64/include/asm/hw_breakpoint.h
@@ -77,7 +77,11 @@ static inline void decode_ctrl_reg(u32 reg,
 /* Lengths */
 #define ARM_BREAKPOINT_LEN_1	0x1
 #define ARM_BREAKPOINT_LEN_2	0x3
+#define ARM_BREAKPOINT_LEN_3	0x7
 #define ARM_BREAKPOINT_LEN_4	0xf
+#define ARM_BREAKPOINT_LEN_5	0x1f
+#define ARM_BREAKPOINT_LEN_6	0x3f
+#define ARM_BREAKPOINT_LEN_7	0x7f
 #define ARM_BREAKPOINT_LEN_8	0xff
 
 /* Kernel stepping */
@@ -119,7 +123,7 @@ struct perf_event;
 struct pmu;
 
 extern int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
-				  int *gen_len, int *gen_type);
+				  int *gen_len, int *gen_type, int *offset);
 extern int arch_check_bp_in_kernelspace(struct perf_event *bp);
 extern int arch_validate_hwbkpt_settings(struct perf_event *bp);
 extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h
index 0bba427bb4c2..0c00c87bb9dd 100644
--- a/arch/arm64/include/asm/io.h
+++ b/arch/arm64/include/asm/io.h
@@ -22,7 +22,6 @@
 #ifdef __KERNEL__
 
 #include <linux/types.h>
-#include <linux/blk_types.h>
 
 #include <asm/byteorder.h>
 #include <asm/barrier.h>
diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h
index 7e51d1b57c0c..7803343e5881 100644
--- a/arch/arm64/include/asm/kernel-pgtable.h
+++ b/arch/arm64/include/asm/kernel-pgtable.h
@@ -19,6 +19,7 @@
 #ifndef __ASM_KERNEL_PGTABLE_H
 #define __ASM_KERNEL_PGTABLE_H
 
+#include <asm/pgtable.h>
 #include <asm/sparsemem.h>
 
 /*
@@ -54,6 +55,12 @@
 #define SWAPPER_DIR_SIZE	(SWAPPER_PGTABLE_LEVELS * PAGE_SIZE)
 #define IDMAP_DIR_SIZE		(IDMAP_PGTABLE_LEVELS * PAGE_SIZE)
 
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+#define RESERVED_TTBR0_SIZE	(PAGE_SIZE)
+#else
+#define RESERVED_TTBR0_SIZE	(0)
+#endif
+
 /* Initial memory map size */
 #if ARM64_SWAPPER_USES_SECTION_MAPS
 #define SWAPPER_BLOCK_SHIFT	SECTION_SHIFT
diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index 8d9fce037b2f..47619411f0ff 100644
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -19,6 +19,7 @@
 typedef struct {
 	atomic64_t	id;
 	void		*vdso;
+	unsigned long	flags;
 } mm_context_t;
 
 /*
@@ -34,7 +35,7 @@ extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
 extern void init_mem_pgprot(void);
 extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
 			       unsigned long virt, phys_addr_t size,
-			       pgprot_t prot, bool allow_block_mappings);
+			       pgprot_t prot, bool page_mappings_only);
 extern void *fixmap_remap_fdt(phys_addr_t dt_phys);
 
 #endif
diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index a50185375f09..0363fe80455c 100644
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -23,6 +23,7 @@
 #include <linux/sched.h>
 
 #include <asm/cacheflush.h>
+#include <asm/cpufeature.h>
 #include <asm/proc-fns.h>
 #include <asm-generic/mm_hooks.h>
 #include <asm/cputype.h>
@@ -103,7 +104,7 @@ static inline void cpu_uninstall_idmap(void)
 	local_flush_tlb_all();
 	cpu_set_default_tcr_t0sz();
 
-	if (mm != &init_mm)
+	if (mm != &init_mm && !system_uses_ttbr0_pan())
 		cpu_switch_mm(mm->pgd, mm);
 }
 
@@ -163,20 +164,26 @@ enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
 {
 }
 
-/*
- * This is the actual mm switch as far as the scheduler
- * is concerned.  No registers are touched.  We avoid
- * calling the CPU specific function when the mm hasn't
- * actually changed.
- */
-static inline void
-switch_mm(struct mm_struct *prev, struct mm_struct *next,
-	  struct task_struct *tsk)
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+static inline void update_saved_ttbr0(struct task_struct *tsk,
+				      struct mm_struct *mm)
 {
-	unsigned int cpu = smp_processor_id();
+	if (system_uses_ttbr0_pan()) {
+		BUG_ON(mm->pgd == swapper_pg_dir);
+		task_thread_info(tsk)->ttbr0 =
+			virt_to_phys(mm->pgd) | ASID(mm) << 48;
+	}
+}
+#else
+static inline void update_saved_ttbr0(struct task_struct *tsk,
+				      struct mm_struct *mm)
+{
+}
+#endif
 
-	if (prev == next)
-		return;
+static inline void __switch_mm(struct mm_struct *next)
+{
+	unsigned int cpu = smp_processor_id();
 
 	/*
 	 * init_mm.pgd does not contain any user mappings and it is always
@@ -190,8 +197,26 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,
 	check_and_switch_context(next, cpu);
 }
 
+static inline void
+switch_mm(struct mm_struct *prev, struct mm_struct *next,
+	  struct task_struct *tsk)
+{
+	if (prev != next)
+		__switch_mm(next);
+
+	/*
+	 * Update the saved TTBR0_EL1 of the scheduled-in task as the previous
+	 * value may have not been initialised yet (activate_mm caller) or the
+	 * ASID has changed since the last run (following the context switch
+	 * of another thread of the same process). Avoid setting the reserved
+	 * TTBR0_EL1 to swapper_pg_dir (init_mm; e.g. via idle_task_exit).
+	 */
+	if (next != &init_mm)
+		update_saved_ttbr0(tsk, next);
+}
+
 #define deactivate_mm(tsk,mm)	do { } while (0)
-#define activate_mm(prev,next)	switch_mm(prev, next, NULL)
+#define activate_mm(prev,next)	switch_mm(prev, next, current)
 
 void verify_cpu_asid_bits(void);
 
diff --git a/arch/arm64/include/asm/neon.h b/arch/arm64/include/asm/neon.h
index 13ce4cc18e26..ad4cdc966c0f 100644
--- a/arch/arm64/include/asm/neon.h
+++ b/arch/arm64/include/asm/neon.h
@@ -9,8 +9,9 @@
  */
 
 #include <linux/types.h>
+#include <asm/fpsimd.h>
 
-#define cpu_has_neon()		(1)
+#define cpu_has_neon()		system_supports_fpsimd()
 
 #define kernel_neon_begin()	kernel_neon_begin_partial(32)
 
diff --git a/arch/arm64/include/asm/opcodes.h b/arch/arm64/include/asm/opcodes.h
deleted file mode 100644
index 123f45d92cd1..000000000000
--- a/arch/arm64/include/asm/opcodes.h
+++ /dev/null
@@ -1,5 +0,0 @@
-#ifdef CONFIG_CPU_BIG_ENDIAN
-#define CONFIG_CPU_ENDIAN_BE8 CONFIG_CPU_BIG_ENDIAN
-#endif
-
-#include <../../arm/include/asm/opcodes.h>
diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
index 5394c8405e66..3bd498e4de4c 100644
--- a/arch/arm64/include/asm/percpu.h
+++ b/arch/arm64/include/asm/percpu.h
@@ -16,6 +16,8 @@
 #ifndef __ASM_PERCPU_H
 #define __ASM_PERCPU_H
 
+#include <asm/stack_pointer.h>
+
 static inline void set_my_cpu_offset(unsigned long off)
 {
 	asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory");
@@ -101,16 +103,16 @@ static inline unsigned long __percpu_read(void *ptr, int size)
 
 	switch (size) {
 	case 1:
-		ret = ACCESS_ONCE(*(u8 *)ptr);
+		ret = READ_ONCE(*(u8 *)ptr);
 		break;
 	case 2:
-		ret = ACCESS_ONCE(*(u16 *)ptr);
+		ret = READ_ONCE(*(u16 *)ptr);
 		break;
 	case 4:
-		ret = ACCESS_ONCE(*(u32 *)ptr);
+		ret = READ_ONCE(*(u32 *)ptr);
 		break;
 	case 8:
-		ret = ACCESS_ONCE(*(u64 *)ptr);
+		ret = READ_ONCE(*(u64 *)ptr);
 		break;
 	default:
 		BUILD_BUG();
@@ -123,16 +125,16 @@ static inline void __percpu_write(void *ptr, unsigned long val, int size)
 {
 	switch (size) {
 	case 1:
-		ACCESS_ONCE(*(u8 *)ptr) = (u8)val;
+		WRITE_ONCE(*(u8 *)ptr, (u8)val);
 		break;
 	case 2:
-		ACCESS_ONCE(*(u16 *)ptr) = (u16)val;
+		WRITE_ONCE(*(u16 *)ptr, (u16)val);
 		break;
 	case 4:
-		ACCESS_ONCE(*(u32 *)ptr) = (u32)val;
+		WRITE_ONCE(*(u32 *)ptr, (u32)val);
 		break;
 	case 8:
-		ACCESS_ONCE(*(u64 *)ptr) = (u64)val;
+		WRITE_ONCE(*(u64 *)ptr, (u64)val);
 		break;
 	default:
 		BUILD_BUG();
diff --git a/arch/arm64/include/asm/perf_event.h b/arch/arm64/include/asm/perf_event.h
index 38b6a2b49d68..8d5cbec17d80 100644
--- a/arch/arm64/include/asm/perf_event.h
+++ b/arch/arm64/include/asm/perf_event.h
@@ -17,6 +17,8 @@
 #ifndef __ASM_PERF_EVENT_H
 #define __ASM_PERF_EVENT_H
 
+#include <asm/stack_pointer.h>
+
 #define	ARMV8_PMU_MAX_COUNTERS	32
 #define	ARMV8_PMU_COUNTER_MASK	(ARMV8_PMU_MAX_COUNTERS - 1)
 
diff --git a/arch/arm64/include/asm/probes.h b/arch/arm64/include/asm/probes.h
index 5af574d632fa..6a5b28904c33 100644
--- a/arch/arm64/include/asm/probes.h
+++ b/arch/arm64/include/asm/probes.h
@@ -15,21 +15,22 @@
 #ifndef _ARM_PROBES_H
 #define _ARM_PROBES_H
 
-#include <asm/opcodes.h>
-
-struct kprobe;
-struct arch_specific_insn;
-
-typedef u32 kprobe_opcode_t;
-typedef void (kprobes_handler_t) (u32 opcode, long addr, struct pt_regs *);
+typedef u32 probe_opcode_t;
+typedef void (probes_handler_t) (u32 opcode, long addr, struct pt_regs *);
 
 /* architecture specific copy of original instruction */
-struct arch_specific_insn {
-	kprobe_opcode_t *insn;
+struct arch_probe_insn {
+	probe_opcode_t *insn;
 	pstate_check_t *pstate_cc;
-	kprobes_handler_t *handler;
+	probes_handler_t *handler;
 	/* restore address after step xol */
 	unsigned long restore;
 };
+#ifdef CONFIG_KPROBES
+typedef u32 kprobe_opcode_t;
+struct arch_specific_insn {
+	struct arch_probe_insn api;
+};
+#endif
 
 #endif
diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h
index 07b8ed037dee..6afd8476c60c 100644
--- a/arch/arm64/include/asm/ptdump.h
+++ b/arch/arm64/include/asm/ptdump.h
@@ -16,9 +16,10 @@
 #ifndef __ASM_PTDUMP_H
 #define __ASM_PTDUMP_H
 
-#ifdef CONFIG_ARM64_PTDUMP
+#ifdef CONFIG_ARM64_PTDUMP_CORE
 
 #include <linux/mm_types.h>
+#include <linux/seq_file.h>
 
 struct addr_marker {
 	unsigned long start_address;
@@ -29,16 +30,25 @@ struct ptdump_info {
 	struct mm_struct		*mm;
 	const struct addr_marker	*markers;
 	unsigned long			base_addr;
-	unsigned long			max_addr;
 };
 
-int ptdump_register(struct ptdump_info *info, const char *name);
-
+void ptdump_walk_pgd(struct seq_file *s, struct ptdump_info *info);
+#ifdef CONFIG_ARM64_PTDUMP_DEBUGFS
+int ptdump_debugfs_register(struct ptdump_info *info, const char *name);
 #else
-static inline int ptdump_register(struct ptdump_info *info, const char *name)
+static inline int ptdump_debugfs_register(struct ptdump_info *info,
+					const char *name)
 {
 	return 0;
 }
-#endif /* CONFIG_ARM64_PTDUMP */
+#endif
+void ptdump_check_wx(void);
+#endif /* CONFIG_ARM64_PTDUMP_CORE */
+
+#ifdef CONFIG_DEBUG_WX
+#define debug_checkwx()	ptdump_check_wx()
+#else
+#define debug_checkwx()	do { } while (0)
+#endif
 
 #endif /* __ASM_PTDUMP_H */
diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h
index ada08b5b036d..513daf050e84 100644
--- a/arch/arm64/include/asm/ptrace.h
+++ b/arch/arm64/include/asm/ptrace.h
@@ -217,6 +217,14 @@ int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task);
 
 #include <asm-generic/ptrace.h>
 
+#define procedure_link_pointer(regs)	((regs)->regs[30])
+
+static inline void procedure_link_pointer_set(struct pt_regs *regs,
+					   unsigned long val)
+{
+	procedure_link_pointer(regs) = val;
+}
+
 #undef profile_pc
 extern unsigned long profile_pc(struct pt_regs *regs);
 
diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h
index 022644704a93..d050d720a1b4 100644
--- a/arch/arm64/include/asm/smp.h
+++ b/arch/arm64/include/asm/smp.h
@@ -29,11 +29,22 @@
 
 #ifndef __ASSEMBLY__
 
+#include <asm/percpu.h>
+
 #include <linux/threads.h>
 #include <linux/cpumask.h>
 #include <linux/thread_info.h>
 
-#define raw_smp_processor_id() (current_thread_info()->cpu)
+DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);
+
+/*
+ * We don't use this_cpu_read(cpu_number) as that has implicit writes to
+ * preempt_count, and associated (compiler) barriers, that we'd like to avoid
+ * the expense of. If we're preemptible, the value can be stale at use anyway.
+ * And we can't use this_cpu_ptr() either, as that winds up recursing back
+ * here under CONFIG_DEBUG_PREEMPT=y.
+ */
+#define raw_smp_processor_id() (*raw_cpu_ptr(&cpu_number))
 
 struct seq_file;
 
@@ -73,6 +84,7 @@ asmlinkage void secondary_start_kernel(void);
  */
 struct secondary_data {
 	void *stack;
+	struct task_struct *task;
 	long status;
 };
 
diff --git a/arch/arm64/include/asm/stack_pointer.h b/arch/arm64/include/asm/stack_pointer.h
new file mode 100644
index 000000000000..ffcdf742cddf
--- /dev/null
+++ b/arch/arm64/include/asm/stack_pointer.h
@@ -0,0 +1,9 @@
+#ifndef __ASM_STACK_POINTER_H
+#define __ASM_STACK_POINTER_H
+
+/*
+ * how to get the current stack pointer from C
+ */
+register unsigned long current_stack_pointer asm ("sp");
+
+#endif /* __ASM_STACK_POINTER_H */
diff --git a/arch/arm64/include/asm/suspend.h b/arch/arm64/include/asm/suspend.h
index b8a313fd7a09..de5600f40adf 100644
--- a/arch/arm64/include/asm/suspend.h
+++ b/arch/arm64/include/asm/suspend.h
@@ -1,7 +1,7 @@
 #ifndef __ASM_SUSPEND_H
 #define __ASM_SUSPEND_H
 
-#define NR_CTX_REGS 10
+#define NR_CTX_REGS 12
 #define NR_CALLEE_SAVED_REGS 12
 
 /*
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 6c80b3699cb8..98ae03f8eedd 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -22,8 +22,6 @@
 
 #include <linux/stringify.h>
 
-#include <asm/opcodes.h>
-
 /*
  * ARMv8 ARM reserves the following encoding for system registers:
  * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview",
@@ -37,6 +35,33 @@
 #define sys_reg(op0, op1, crn, crm, op2) \
 	((((op0)&3)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5))
 
+#ifndef CONFIG_BROKEN_GAS_INST
+
+#ifdef __ASSEMBLY__
+#define __emit_inst(x)			.inst (x)
+#else
+#define __emit_inst(x)			".inst " __stringify((x)) "\n\t"
+#endif
+
+#else  /* CONFIG_BROKEN_GAS_INST */
+
+#ifndef CONFIG_CPU_BIG_ENDIAN
+#define __INSTR_BSWAP(x)		(x)
+#else  /* CONFIG_CPU_BIG_ENDIAN */
+#define __INSTR_BSWAP(x)		((((x) << 24) & 0xff000000)	| \
+					 (((x) <<  8) & 0x00ff0000)	| \
+					 (((x) >>  8) & 0x0000ff00)	| \
+					 (((x) >> 24) & 0x000000ff))
+#endif	/* CONFIG_CPU_BIG_ENDIAN */
+
+#ifdef __ASSEMBLY__
+#define __emit_inst(x)			.long __INSTR_BSWAP(x)
+#else  /* __ASSEMBLY__ */
+#define __emit_inst(x)			".long " __stringify(__INSTR_BSWAP(x)) "\n\t"
+#endif	/* __ASSEMBLY__ */
+
+#endif	/* CONFIG_BROKEN_GAS_INST */
+
 #define SYS_MIDR_EL1			sys_reg(3, 0, 0, 0, 0)
 #define SYS_MPIDR_EL1			sys_reg(3, 0, 0, 0, 5)
 #define SYS_REVIDR_EL1			sys_reg(3, 0, 0, 0, 6)
@@ -81,10 +106,10 @@
 #define REG_PSTATE_PAN_IMM		sys_reg(0, 0, 4, 0, 4)
 #define REG_PSTATE_UAO_IMM		sys_reg(0, 0, 4, 0, 3)
 
-#define SET_PSTATE_PAN(x) __inst_arm(0xd5000000 | REG_PSTATE_PAN_IMM |\
-				     (!!x)<<8 | 0x1f)
-#define SET_PSTATE_UAO(x) __inst_arm(0xd5000000 | REG_PSTATE_UAO_IMM |\
-				     (!!x)<<8 | 0x1f)
+#define SET_PSTATE_PAN(x) __emit_inst(0xd5000000 | REG_PSTATE_PAN_IMM |	\
+				      (!!x)<<8 | 0x1f)
+#define SET_PSTATE_UAO(x) __emit_inst(0xd5000000 | REG_PSTATE_UAO_IMM |	\
+				      (!!x)<<8 | 0x1f)
 
 /* Common SCTLR_ELx flags. */
 #define SCTLR_ELx_EE    (1 << 25)
@@ -228,11 +253,11 @@
 	.equ	.L__reg_num_xzr, 31
 
 	.macro	mrs_s, rt, sreg
-	.inst	0xd5200000|(\sreg)|(.L__reg_num_\rt)
+	 __emit_inst(0xd5200000|(\sreg)|(.L__reg_num_\rt))
 	.endm
 
 	.macro	msr_s, sreg, rt
-	.inst	0xd5000000|(\sreg)|(.L__reg_num_\rt)
+	__emit_inst(0xd5000000|(\sreg)|(.L__reg_num_\rt))
 	.endm
 
 #else
@@ -246,11 +271,11 @@ asm(
 "	.equ	.L__reg_num_xzr, 31\n"
 "\n"
 "	.macro	mrs_s, rt, sreg\n"
-"	.inst	0xd5200000|(\\sreg)|(.L__reg_num_\\rt)\n"
+	__emit_inst(0xd5200000|(\\sreg)|(.L__reg_num_\\rt))
 "	.endm\n"
 "\n"
 "	.macro	msr_s, sreg, rt\n"
-"	.inst	0xd5000000|(\\sreg)|(.L__reg_num_\\rt)\n"
+	__emit_inst(0xd5000000|(\\sreg)|(.L__reg_num_\\rt))
 "	.endm\n"
 );
 
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index e9ea5a6bd449..46c3b93cf865 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -36,58 +36,31 @@
 
 struct task_struct;
 
+#include <asm/stack_pointer.h>
 #include <asm/types.h>
 
 typedef unsigned long mm_segment_t;
 
 /*
  * low level task data that entry.S needs immediate access to.
- * __switch_to() assumes cpu_context follows immediately after cpu_domain.
  */
 struct thread_info {
 	unsigned long		flags;		/* low level flags */
 	mm_segment_t		addr_limit;	/* address limit */
-	struct task_struct	*task;		/* main task structure */
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+	u64			ttbr0;		/* saved TTBR0_EL1 */
+#endif
 	int			preempt_count;	/* 0 => preemptable, <0 => bug */
-	int			cpu;		/* cpu */
 };
 
 #define INIT_THREAD_INFO(tsk)						\
 {									\
-	.task		= &tsk,						\
-	.flags		= 0,						\
 	.preempt_count	= INIT_PREEMPT_COUNT,				\
 	.addr_limit	= KERNEL_DS,					\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
 #define init_stack		(init_thread_union.stack)
 
-/*
- * how to get the current stack pointer from C
- */
-register unsigned long current_stack_pointer asm ("sp");
-
-/*
- * how to get the thread information struct from C
- */
-static inline struct thread_info *current_thread_info(void) __attribute_const__;
-
-/*
- * struct thread_info can be accessed directly via sp_el0.
- *
- * We don't use read_sysreg() as we want the compiler to cache the value where
- * possible.
- */
-static inline struct thread_info *current_thread_info(void)
-{
-	unsigned long sp_el0;
-
-	asm ("mrs %0, sp_el0" : "=r" (sp_el0));
-
-	return (struct thread_info *)sp_el0;
-}
-
 #define thread_saved_pc(tsk)	\
 	((unsigned long)(tsk->thread.cpu_context.pc))
 #define thread_saved_sp(tsk)	\
@@ -112,6 +85,7 @@ static inline struct thread_info *current_thread_info(void)
 #define TIF_NEED_RESCHED	1
 #define TIF_NOTIFY_RESUME	2	/* callback before returning to user */
 #define TIF_FOREIGN_FPSTATE	3	/* CPU's FP state is not current's */
+#define TIF_UPROBE		4	/* uprobe breakpoint or singlestep */
 #define TIF_NOHZ		7
 #define TIF_SYSCALL_TRACE	8
 #define TIF_SYSCALL_AUDIT	9
@@ -132,10 +106,12 @@ static inline struct thread_info *current_thread_info(void)
 #define _TIF_SYSCALL_AUDIT	(1 << TIF_SYSCALL_AUDIT)
 #define _TIF_SYSCALL_TRACEPOINT	(1 << TIF_SYSCALL_TRACEPOINT)
 #define _TIF_SECCOMP		(1 << TIF_SECCOMP)
+#define _TIF_UPROBE		(1 << TIF_UPROBE)
 #define _TIF_32BIT		(1 << TIF_32BIT)
 
 #define _TIF_WORK_MASK		(_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
-				 _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE)
+				 _TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
+				 _TIF_UPROBE)
 
 #define _TIF_SYSCALL_WORK	(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
 				 _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 55d0adbf6509..d26750ca6e06 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -18,6 +18,12 @@
 #ifndef __ASM_UACCESS_H
 #define __ASM_UACCESS_H
 
+#include <asm/alternative.h>
+#include <asm/kernel-pgtable.h>
+#include <asm/sysreg.h>
+
+#ifndef __ASSEMBLY__
+
 /*
  * User space memory access functions
  */
@@ -26,10 +32,8 @@
 #include <linux/string.h>
 #include <linux/thread_info.h>
 
-#include <asm/alternative.h>
 #include <asm/cpufeature.h>
 #include <asm/ptrace.h>
-#include <asm/sysreg.h>
 #include <asm/errno.h>
 #include <asm/memory.h>
 #include <asm/compiler.h>
@@ -120,6 +124,99 @@ static inline void set_fs(mm_segment_t fs)
 	"	.popsection\n"
 
 /*
+ * User access enabling/disabling.
+ */
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+static inline void __uaccess_ttbr0_disable(void)
+{
+	unsigned long ttbr;
+
+	/* reserved_ttbr0 placed at the end of swapper_pg_dir */
+	ttbr = read_sysreg(ttbr1_el1) + SWAPPER_DIR_SIZE;
+	write_sysreg(ttbr, ttbr0_el1);
+	isb();
+}
+
+static inline void __uaccess_ttbr0_enable(void)
+{
+	unsigned long flags;
+
+	/*
+	 * Disable interrupts to avoid preemption between reading the 'ttbr0'
+	 * variable and the MSR. A context switch could trigger an ASID
+	 * roll-over and an update of 'ttbr0'.
+	 */
+	local_irq_save(flags);
+	write_sysreg(current_thread_info()->ttbr0, ttbr0_el1);
+	isb();
+	local_irq_restore(flags);
+}
+
+static inline bool uaccess_ttbr0_disable(void)
+{
+	if (!system_uses_ttbr0_pan())
+		return false;
+	__uaccess_ttbr0_disable();
+	return true;
+}
+
+static inline bool uaccess_ttbr0_enable(void)
+{
+	if (!system_uses_ttbr0_pan())
+		return false;
+	__uaccess_ttbr0_enable();
+	return true;
+}
+#else
+static inline bool uaccess_ttbr0_disable(void)
+{
+	return false;
+}
+
+static inline bool uaccess_ttbr0_enable(void)
+{
+	return false;
+}
+#endif
+
+#define __uaccess_disable(alt)						\
+do {									\
+	if (!uaccess_ttbr0_disable())					\
+		asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt,		\
+				CONFIG_ARM64_PAN));			\
+} while (0)
+
+#define __uaccess_enable(alt)						\
+do {									\
+	if (!uaccess_ttbr0_enable())					\
+		asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt,		\
+				CONFIG_ARM64_PAN));			\
+} while (0)
+
+static inline void uaccess_disable(void)
+{
+	__uaccess_disable(ARM64_HAS_PAN);
+}
+
+static inline void uaccess_enable(void)
+{
+	__uaccess_enable(ARM64_HAS_PAN);
+}
+
+/*
+ * These functions are no-ops when UAO is present.
+ */
+static inline void uaccess_disable_not_uao(void)
+{
+	__uaccess_disable(ARM64_ALT_PAN_NOT_UAO);
+}
+
+static inline void uaccess_enable_not_uao(void)
+{
+	__uaccess_enable(ARM64_ALT_PAN_NOT_UAO);
+}
+
+/*
  * The "__xxx" versions of the user access functions do not verify the address
  * space - it must have been done previously with a separate "access_ok()"
  * call.
@@ -146,8 +243,7 @@ static inline void set_fs(mm_segment_t fs)
 do {									\
 	unsigned long __gu_val;						\
 	__chk_user_ptr(ptr);						\
-	asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_ALT_PAN_NOT_UAO,\
-			CONFIG_ARM64_PAN));				\
+	uaccess_enable_not_uao();					\
 	switch (sizeof(*(ptr))) {					\
 	case 1:								\
 		__get_user_asm("ldrb", "ldtrb", "%w", __gu_val, (ptr),  \
@@ -168,9 +264,8 @@ do {									\
 	default:							\
 		BUILD_BUG();						\
 	}								\
+	uaccess_disable_not_uao();					\
 	(x) = (__force __typeof__(*(ptr)))__gu_val;			\
-	asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_ALT_PAN_NOT_UAO,\
-			CONFIG_ARM64_PAN));				\
 } while (0)
 
 #define __get_user(x, ptr)						\
@@ -215,8 +310,7 @@ do {									\
 do {									\
 	__typeof__(*(ptr)) __pu_val = (x);				\
 	__chk_user_ptr(ptr);						\
-	asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_ALT_PAN_NOT_UAO,\
-			CONFIG_ARM64_PAN));				\
+	uaccess_enable_not_uao();					\
 	switch (sizeof(*(ptr))) {					\
 	case 1:								\
 		__put_user_asm("strb", "sttrb", "%w", __pu_val, (ptr),	\
@@ -237,8 +331,7 @@ do {									\
 	default:							\
 		BUILD_BUG();						\
 	}								\
-	asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_ALT_PAN_NOT_UAO,\
-			CONFIG_ARM64_PAN));				\
+	uaccess_disable_not_uao();					\
 } while (0)
 
 #define __put_user(x, ptr)						\
@@ -331,4 +424,66 @@ extern long strncpy_from_user(char *dest, const char __user *src, long count);
 extern __must_check long strlen_user(const char __user *str);
 extern __must_check long strnlen_user(const char __user *str, long n);
 
+#else	/* __ASSEMBLY__ */
+
+#include <asm/assembler.h>
+
+/*
+ * User access enabling/disabling macros.
+ */
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+	.macro	__uaccess_ttbr0_disable, tmp1
+	mrs	\tmp1, ttbr1_el1		// swapper_pg_dir
+	add	\tmp1, \tmp1, #SWAPPER_DIR_SIZE	// reserved_ttbr0 at the end of swapper_pg_dir
+	msr	ttbr0_el1, \tmp1		// set reserved TTBR0_EL1
+	isb
+	.endm
+
+	.macro	__uaccess_ttbr0_enable, tmp1
+	get_thread_info \tmp1
+	ldr	\tmp1, [\tmp1, #TSK_TI_TTBR0]	// load saved TTBR0_EL1
+	msr	ttbr0_el1, \tmp1		// set the non-PAN TTBR0_EL1
+	isb
+	.endm
+
+	.macro	uaccess_ttbr0_disable, tmp1
+alternative_if_not ARM64_HAS_PAN
+	__uaccess_ttbr0_disable \tmp1
+alternative_else_nop_endif
+	.endm
+
+	.macro	uaccess_ttbr0_enable, tmp1, tmp2
+alternative_if_not ARM64_HAS_PAN
+	save_and_disable_irq \tmp2		// avoid preemption
+	__uaccess_ttbr0_enable \tmp1
+	restore_irq \tmp2
+alternative_else_nop_endif
+	.endm
+#else
+	.macro	uaccess_ttbr0_disable, tmp1
+	.endm
+
+	.macro	uaccess_ttbr0_enable, tmp1, tmp2
+	.endm
+#endif
+
+/*
+ * These macros are no-ops when UAO is present.
+ */
+	.macro	uaccess_disable_not_uao, tmp1
+	uaccess_ttbr0_disable \tmp1
+alternative_if ARM64_ALT_PAN_NOT_UAO
+	SET_PSTATE_PAN(1)
+alternative_else_nop_endif
+	.endm
+
+	.macro	uaccess_enable_not_uao, tmp1, tmp2
+	uaccess_ttbr0_enable \tmp1, \tmp2
+alternative_if ARM64_ALT_PAN_NOT_UAO
+	SET_PSTATE_PAN(0)
+alternative_else_nop_endif
+	.endm
+
+#endif	/* __ASSEMBLY__ */
+
 #endif /* __ASM_UACCESS_H */
diff --git a/arch/arm64/include/asm/uprobes.h b/arch/arm64/include/asm/uprobes.h
new file mode 100644
index 000000000000..8d004073d0e8
--- /dev/null
+++ b/arch/arm64/include/asm/uprobes.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2014-2016 Pratyush Anand <panand@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef _ASM_UPROBES_H
+#define _ASM_UPROBES_H
+
+#include <asm/debug-monitors.h>
+#include <asm/insn.h>
+#include <asm/probes.h>
+
+#define MAX_UINSN_BYTES		AARCH64_INSN_SIZE
+
+#define UPROBE_SWBP_INSN	BRK64_OPCODE_UPROBES
+#define UPROBE_SWBP_INSN_SIZE	AARCH64_INSN_SIZE
+#define UPROBE_XOL_SLOT_BYTES	MAX_UINSN_BYTES
+
+typedef u32 uprobe_opcode_t;
+
+struct arch_uprobe_task {
+};
+
+struct arch_uprobe {
+	union {
+		u8 insn[MAX_UINSN_BYTES];
+		u8 ixol[MAX_UINSN_BYTES];
+	};
+	struct arch_probe_insn api;
+	bool simulate;
+};
+
+#endif
diff --git a/arch/arm64/include/asm/xen/hypercall.h b/arch/arm64/include/asm/xen/hypercall.h
index 74b0c423ff5b..3522cbaed316 100644
--- a/arch/arm64/include/asm/xen/hypercall.h
+++ b/arch/arm64/include/asm/xen/hypercall.h
@@ -1 +1 @@
-#include <../../arm/include/asm/xen/hypercall.h>
+#include <xen/arm/hypercall.h>
diff --git a/arch/arm64/include/asm/xen/hypervisor.h b/arch/arm64/include/asm/xen/hypervisor.h
index f263da8e8769..d6e7709d0688 100644
--- a/arch/arm64/include/asm/xen/hypervisor.h
+++ b/arch/arm64/include/asm/xen/hypervisor.h
@@ -1 +1 @@
-#include <../../arm/include/asm/xen/hypervisor.h>
+#include <xen/arm/hypervisor.h>
diff --git a/arch/arm64/include/asm/xen/interface.h b/arch/arm64/include/asm/xen/interface.h
index 44457aebeed4..88c0d75da190 100644
--- a/arch/arm64/include/asm/xen/interface.h
+++ b/arch/arm64/include/asm/xen/interface.h
@@ -1 +1 @@
-#include <../../arm/include/asm/xen/interface.h>
+#include <xen/arm/interface.h>
diff --git a/arch/arm64/include/asm/xen/page-coherent.h b/arch/arm64/include/asm/xen/page-coherent.h
index 2052102b4e02..b3ef061d8b74 100644
--- a/arch/arm64/include/asm/xen/page-coherent.h
+++ b/arch/arm64/include/asm/xen/page-coherent.h
@@ -1 +1 @@
-#include <../../arm/include/asm/xen/page-coherent.h>
+#include <xen/arm/page-coherent.h>
diff --git a/arch/arm64/include/asm/xen/page.h b/arch/arm64/include/asm/xen/page.h
index bed87ec36780..31bbc803cecb 100644
--- a/arch/arm64/include/asm/xen/page.h
+++ b/arch/arm64/include/asm/xen/page.h
@@ -1 +1 @@
-#include <../../arm/include/asm/xen/page.h>
+#include <xen/arm/page.h>
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index b0988bb1bf64..04de188a36c9 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -14,10 +14,8 @@
 #include <linux/slab.h>
 #include <linux/sysctl.h>
 
-#include <asm/alternative.h>
 #include <asm/cpufeature.h>
 #include <asm/insn.h>
-#include <asm/opcodes.h>
 #include <asm/sysreg.h>
 #include <asm/system_misc.h>
 #include <asm/traps.h>
@@ -285,10 +283,10 @@ static void __init register_insn_emulation_sysctl(struct ctl_table *table)
 #define __SWP_LL_SC_LOOPS	4
 
 #define __user_swpX_asm(data, addr, res, temp, temp2, B)	\
+do {								\
+	uaccess_enable();					\
 	__asm__ __volatile__(					\
 	"	mov		%w3, %w7\n"			\
-	ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,	\
-		    CONFIG_ARM64_PAN)				\
 	"0:	ldxr"B"		%w2, [%4]\n"			\
 	"1:	stxr"B"		%w0, %w1, [%4]\n"		\
 	"	cbz		%w0, 2f\n"			\
@@ -306,12 +304,12 @@ static void __init register_insn_emulation_sysctl(struct ctl_table *table)
 	"	.popsection"					\
 	_ASM_EXTABLE(0b, 4b)					\
 	_ASM_EXTABLE(1b, 4b)					\
-	ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,	\
-		CONFIG_ARM64_PAN)				\
 	: "=&r" (res), "+r" (data), "=&r" (temp), "=&r" (temp2)	\
 	: "r" (addr), "i" (-EAGAIN), "i" (-EFAULT),		\
 	  "i" (__SWP_LL_SC_LOOPS)				\
-	: "memory")
+	: "memory");						\
+	uaccess_disable();					\
+} while (0)
 
 #define __user_swp_asm(data, addr, res, temp, temp2) \
 	__user_swpX_asm(data, addr, res, temp, temp2, "")
@@ -352,6 +350,10 @@ static int emulate_swpX(unsigned int address, unsigned int *data,
 	return res;
 }
 
+#define ARM_OPCODE_CONDTEST_FAIL   0
+#define ARM_OPCODE_CONDTEST_PASS   1
+#define ARM_OPCODE_CONDTEST_UNCOND 2
+
 #define	ARM_OPCODE_CONDITION_UNCOND	0xf
 
 static unsigned int __kprobes aarch32_check_condition(u32 opcode, u32 psr)
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 4a2f0f0fef32..bc049afc73a7 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -36,11 +36,13 @@ int main(void)
 {
   DEFINE(TSK_ACTIVE_MM,		offsetof(struct task_struct, active_mm));
   BLANK();
-  DEFINE(TI_FLAGS,		offsetof(struct thread_info, flags));
-  DEFINE(TI_PREEMPT,		offsetof(struct thread_info, preempt_count));
-  DEFINE(TI_ADDR_LIMIT,		offsetof(struct thread_info, addr_limit));
-  DEFINE(TI_TASK,		offsetof(struct thread_info, task));
-  DEFINE(TI_CPU,		offsetof(struct thread_info, cpu));
+  DEFINE(TSK_TI_FLAGS,		offsetof(struct task_struct, thread_info.flags));
+  DEFINE(TSK_TI_PREEMPT,	offsetof(struct task_struct, thread_info.preempt_count));
+  DEFINE(TSK_TI_ADDR_LIMIT,	offsetof(struct task_struct, thread_info.addr_limit));
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+  DEFINE(TSK_TI_TTBR0,		offsetof(struct task_struct, thread_info.ttbr0));
+#endif
+  DEFINE(TSK_STACK,		offsetof(struct task_struct, stack));
   BLANK();
   DEFINE(THREAD_CPU_CONTEXT,	offsetof(struct task_struct, thread.cpu_context));
   BLANK();
@@ -123,6 +125,7 @@ int main(void)
   DEFINE(TZ_DSTTIME,		offsetof(struct timezone, tz_dsttime));
   BLANK();
   DEFINE(CPU_BOOT_STACK,	offsetof(struct secondary_data, stack));
+  DEFINE(CPU_BOOT_TASK,		offsetof(struct secondary_data, task));
   BLANK();
 #ifdef CONFIG_KVM_ARM_HOST
   DEFINE(VCPU_CONTEXT,		offsetof(struct kvm_vcpu, arch.ctxt));
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index c02504ea304b..fdf8f045929f 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -47,6 +47,7 @@ unsigned int compat_elf_hwcap2 __read_mostly;
 #endif
 
 DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
+EXPORT_SYMBOL(cpu_hwcaps);
 
 DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS);
 EXPORT_SYMBOL(cpu_hwcap_keys);
@@ -746,6 +747,14 @@ static bool hyp_offset_low(const struct arm64_cpu_capabilities *entry,
 	return idmap_addr > GENMASK(VA_BITS - 2, 0) && !is_kernel_in_hyp_mode();
 }
 
+static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused)
+{
+	u64 pfr0 = read_system_reg(SYS_ID_AA64PFR0_EL1);
+
+	return cpuid_feature_extract_signed_field(pfr0,
+					ID_AA64PFR0_FP_SHIFT) < 0;
+}
+
 static const struct arm64_cpu_capabilities arm64_features[] = {
 	{
 		.desc = "GIC system register CPU interface",
@@ -829,6 +838,13 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.def_scope = SCOPE_SYSTEM,
 		.matches = hyp_offset_low,
 	},
+	{
+		/* FP/SIMD is not implemented */
+		.capability = ARM64_HAS_NO_FPSIMD,
+		.def_scope = SCOPE_SYSTEM,
+		.min_field_value = 0,
+		.matches = has_no_fpsimd,
+	},
 	{},
 };
 
@@ -1102,5 +1118,5 @@ void __init setup_cpu_features(void)
 static bool __maybe_unused
 cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused)
 {
-	return (cpus_have_cap(ARM64_HAS_PAN) && !cpus_have_cap(ARM64_HAS_UAO));
+	return (cpus_have_const_cap(ARM64_HAS_PAN) && !cpus_have_const_cap(ARM64_HAS_UAO));
 }
diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c
index 73ae90ef434c..605df76f0a06 100644
--- a/arch/arm64/kernel/debug-monitors.c
+++ b/arch/arm64/kernel/debug-monitors.c
@@ -226,6 +226,8 @@ static void send_user_sigtrap(int si_code)
 static int single_step_handler(unsigned long addr, unsigned int esr,
 			       struct pt_regs *regs)
 {
+	bool handler_found = false;
+
 	/*
 	 * If we are stepping a pending breakpoint, call the hw_breakpoint
 	 * handler first.
@@ -233,7 +235,14 @@ static int single_step_handler(unsigned long addr, unsigned int esr,
 	if (!reinstall_suspended_bps(regs))
 		return 0;
 
-	if (user_mode(regs)) {
+#ifdef	CONFIG_KPROBES
+	if (kprobe_single_step_handler(regs, esr) == DBG_HOOK_HANDLED)
+		handler_found = true;
+#endif
+	if (!handler_found && call_step_hook(regs, esr) == DBG_HOOK_HANDLED)
+		handler_found = true;
+
+	if (!handler_found && user_mode(regs)) {
 		send_user_sigtrap(TRAP_TRACE);
 
 		/*
@@ -243,15 +252,8 @@ static int single_step_handler(unsigned long addr, unsigned int esr,
 		 * to the active-not-pending state).
 		 */
 		user_rewind_single_step(current);
-	} else {
-#ifdef	CONFIG_KPROBES
-		if (kprobe_single_step_handler(regs, esr) == DBG_HOOK_HANDLED)
-			return 0;
-#endif
-		if (call_step_hook(regs, esr) == DBG_HOOK_HANDLED)
-			return 0;
-
-		pr_warning("Unexpected kernel single-step exception at EL1\n");
+	} else if (!handler_found) {
+		pr_warn("Unexpected kernel single-step exception at EL1\n");
 		/*
 		 * Re-enable stepping since we know that we will be
 		 * returning to regs.
@@ -304,16 +306,20 @@ NOKPROBE_SYMBOL(call_break_hook);
 static int brk_handler(unsigned long addr, unsigned int esr,
 		       struct pt_regs *regs)
 {
-	if (user_mode(regs)) {
-		send_user_sigtrap(TRAP_BRKPT);
-	}
+	bool handler_found = false;
+
 #ifdef	CONFIG_KPROBES
-	else if ((esr & BRK64_ESR_MASK) == BRK64_ESR_KPROBES) {
-		if (kprobe_breakpoint_handler(regs, esr) != DBG_HOOK_HANDLED)
-			return -EFAULT;
+	if ((esr & BRK64_ESR_MASK) == BRK64_ESR_KPROBES) {
+		if (kprobe_breakpoint_handler(regs, esr) == DBG_HOOK_HANDLED)
+			handler_found = true;
 	}
 #endif
-	else if (call_break_hook(regs, esr) != DBG_HOOK_HANDLED) {
+	if (!handler_found && call_break_hook(regs, esr) == DBG_HOOK_HANDLED)
+		handler_found = true;
+
+	if (!handler_found && user_mode(regs)) {
+		send_user_sigtrap(TRAP_BRKPT);
+	} else if (!handler_found) {
 		pr_warn("Unexpected kernel BRK exception at EL1\n");
 		return -EFAULT;
 	}
diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c
index ba9bee389fd5..5d17f377d905 100644
--- a/arch/arm64/kernel/efi.c
+++ b/arch/arm64/kernel/efi.c
@@ -62,8 +62,8 @@ struct screen_info screen_info __section(.data);
 int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md)
 {
 	pteval_t prot_val = create_mapping_protection(md);
-	bool allow_block_mappings = (md->type != EFI_RUNTIME_SERVICES_CODE &&
-				     md->type != EFI_RUNTIME_SERVICES_DATA);
+	bool page_mappings_only = (md->type == EFI_RUNTIME_SERVICES_CODE ||
+				   md->type == EFI_RUNTIME_SERVICES_DATA);
 
 	if (!PAGE_ALIGNED(md->phys_addr) ||
 	    !PAGE_ALIGNED(md->num_pages << EFI_PAGE_SHIFT)) {
@@ -76,12 +76,12 @@ int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md)
 		 * from the MMU routines. So avoid block mappings altogether in
 		 * that case.
 		 */
-		allow_block_mappings = false;
+		page_mappings_only = true;
 	}
 
 	create_pgd_mapping(mm, md->phys_addr, md->virt_addr,
 			   md->num_pages << EFI_PAGE_SHIFT,
-			   __pgprot(prot_val | PTE_NG), allow_block_mappings);
+			   __pgprot(prot_val | PTE_NG), page_mappings_only);
 	return 0;
 }
 
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 223d54a4d66b..4f0d76339414 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -29,7 +29,9 @@
 #include <asm/esr.h>
 #include <asm/irq.h>
 #include <asm/memory.h>
+#include <asm/ptrace.h>
 #include <asm/thread_info.h>
+#include <asm/uaccess.h>
 #include <asm/unistd.h>
 
 /*
@@ -90,9 +92,8 @@
 
 	.if	\el == 0
 	mrs	x21, sp_el0
-	mov	tsk, sp
-	and	tsk, tsk, #~(THREAD_SIZE - 1)	// Ensure MDSCR_EL1.SS is clear,
-	ldr	x19, [tsk, #TI_FLAGS]		// since we can unmask debug
+	ldr_this_cpu	tsk, __entry_task, x20	// Ensure MDSCR_EL1.SS is clear,
+	ldr	x19, [tsk, #TSK_TI_FLAGS]	// since we can unmask debug
 	disable_step_tsk x19, x20		// exceptions when scheduling.
 
 	mov	x29, xzr			// fp pointed to user-space
@@ -100,15 +101,41 @@
 	add	x21, sp, #S_FRAME_SIZE
 	get_thread_info tsk
 	/* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */
-	ldr	x20, [tsk, #TI_ADDR_LIMIT]
+	ldr	x20, [tsk, #TSK_TI_ADDR_LIMIT]
 	str	x20, [sp, #S_ORIG_ADDR_LIMIT]
 	mov	x20, #TASK_SIZE_64
-	str	x20, [tsk, #TI_ADDR_LIMIT]
+	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
 	/* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */
 	.endif /* \el == 0 */
 	mrs	x22, elr_el1
 	mrs	x23, spsr_el1
 	stp	lr, x21, [sp, #S_LR]
+
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+	/*
+	 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from
+	 * EL0, there is no need to check the state of TTBR0_EL1 since
+	 * accesses are always enabled.
+	 * Note that the meaning of this bit differs from the ARMv8.1 PAN
+	 * feature as all TTBR0_EL1 accesses are disabled, not just those to
+	 * user mappings.
+	 */
+alternative_if ARM64_HAS_PAN
+	b	1f				// skip TTBR0 PAN
+alternative_else_nop_endif
+
+	.if	\el != 0
+	mrs	x21, ttbr0_el1
+	tst	x21, #0xffff << 48		// Check for the reserved ASID
+	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR
+	b.eq	1f				// TTBR0 access already disabled
+	and	x23, x23, #~PSR_PAN_BIT		// Clear the emulated PAN in the saved SPSR
+	.endif
+
+	__uaccess_ttbr0_disable x21
+1:
+#endif
+
 	stp	x22, x23, [sp, #S_PC]
 
 	/*
@@ -139,7 +166,7 @@
 	.if	\el != 0
 	/* Restore the task's original addr_limit. */
 	ldr	x20, [sp, #S_ORIG_ADDR_LIMIT]
-	str	x20, [tsk, #TI_ADDR_LIMIT]
+	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
 
 	/* No need to restore UAO, it will be restored from SPSR_EL1 */
 	.endif
@@ -147,6 +174,40 @@
 	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR
 	.if	\el == 0
 	ct_user_enter
+	.endif
+
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+	/*
+	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR
+	 * PAN bit checking.
+	 */
+alternative_if ARM64_HAS_PAN
+	b	2f				// skip TTBR0 PAN
+alternative_else_nop_endif
+
+	.if	\el != 0
+	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
+	.endif
+
+	__uaccess_ttbr0_enable x0
+
+	.if	\el == 0
+	/*
+	 * Enable errata workarounds only if returning to user. The only
+	 * workaround currently required for TTBR0_EL1 changes are for the
+	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
+	 * corruption).
+	 */
+	post_ttbr0_update_workaround
+	.endif
+1:
+	.if	\el != 0
+	and	x22, x22, #~PSR_PAN_BIT		// ARMv8.0 CPUs do not understand this bit
+	.endif
+2:
+#endif
+
+	.if	\el == 0
 	ldr	x23, [sp, #S_SP]		// load return stack pointer
 	msr	sp_el0, x23
 #ifdef CONFIG_ARM64_ERRATUM_845719
@@ -162,6 +223,7 @@ alternative_if ARM64_WORKAROUND_845719
 alternative_else_nop_endif
 #endif
 	.endif
+
 	msr	elr_el1, x21			// set up the return data
 	msr	spsr_el1, x22
 	ldp	x0, x1, [sp, #16 * 0]
@@ -184,23 +246,20 @@ alternative_else_nop_endif
 	eret					// return to kernel
 	.endm
 
-	.macro	get_thread_info, rd
-	mrs	\rd, sp_el0
-	.endm
-
 	.macro	irq_stack_entry
 	mov	x19, sp			// preserve the original sp
 
 	/*
-	 * Compare sp with the current thread_info, if the top
-	 * ~(THREAD_SIZE - 1) bits match, we are on a task stack, and
-	 * should switch to the irq stack.
+	 * Compare sp with the base of the task stack.
+	 * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack,
+	 * and should switch to the irq stack.
 	 */
-	and	x25, x19, #~(THREAD_SIZE - 1)
-	cmp	x25, tsk
-	b.ne	9998f
+	ldr	x25, [tsk, TSK_STACK]
+	eor	x25, x25, x19
+	and	x25, x25, #~(THREAD_SIZE - 1)
+	cbnz	x25, 9998f
 
-	this_cpu_ptr irq_stack, x25, x26
+	adr_this_cpu x25, irq_stack, x26
 	mov	x26, #IRQ_STACK_START_SP
 	add	x26, x25, x26
 
@@ -427,9 +486,9 @@ el1_irq:
 	irq_handler
 
 #ifdef CONFIG_PREEMPT
-	ldr	w24, [tsk, #TI_PREEMPT]		// get preempt count
+	ldr	w24, [tsk, #TSK_TI_PREEMPT]	// get preempt count
 	cbnz	w24, 1f				// preempt count != 0
-	ldr	x0, [tsk, #TI_FLAGS]		// get flags
+	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get flags
 	tbz	x0, #TIF_NEED_RESCHED, 1f	// needs rescheduling?
 	bl	el1_preempt
 1:
@@ -444,7 +503,7 @@ ENDPROC(el1_irq)
 el1_preempt:
 	mov	x24, lr
 1:	bl	preempt_schedule_irq		// irq en/disable is done inside
-	ldr	x0, [tsk, #TI_FLAGS]		// get new tasks TI_FLAGS
+	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get new tasks TI_FLAGS
 	tbnz	x0, #TIF_NEED_RESCHED, 1b	// needs rescheduling?
 	ret	x24
 #endif
@@ -674,8 +733,7 @@ ENTRY(cpu_switch_to)
 	ldp	x29, x9, [x8], #16
 	ldr	lr, [x8]
 	mov	sp, x9
-	and	x9, x9, #~(THREAD_SIZE - 1)
-	msr	sp_el0, x9
+	msr	sp_el0, x1
 	ret
 ENDPROC(cpu_switch_to)
 
@@ -686,7 +744,7 @@ ENDPROC(cpu_switch_to)
 ret_fast_syscall:
 	disable_irq				// disable interrupts
 	str	x0, [sp, #S_X0]			// returned x0
-	ldr	x1, [tsk, #TI_FLAGS]		// re-check for syscall tracing
+	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for syscall tracing
 	and	x2, x1, #_TIF_SYSCALL_WORK
 	cbnz	x2, ret_fast_syscall_trace
 	and	x2, x1, #_TIF_WORK_MASK
@@ -706,14 +764,14 @@ work_pending:
 #ifdef CONFIG_TRACE_IRQFLAGS
 	bl	trace_hardirqs_on		// enabled while in userspace
 #endif
-	ldr	x1, [tsk, #TI_FLAGS]		// re-check for single-step
+	ldr	x1, [tsk, #TSK_TI_FLAGS]	// re-check for single-step
 	b	finish_ret_to_user
 /*
  * "slow" syscall return path.
  */
 ret_to_user:
 	disable_irq				// disable interrupts
-	ldr	x1, [tsk, #TI_FLAGS]
+	ldr	x1, [tsk, #TSK_TI_FLAGS]
 	and	x2, x1, #_TIF_WORK_MASK
 	cbnz	x2, work_pending
 finish_ret_to_user:
@@ -746,7 +804,7 @@ el0_svc_naked:					// compat entry point
 	enable_dbg_and_irq
 	ct_user_exit 1
 
-	ldr	x16, [tsk, #TI_FLAGS]		// check for syscall hooks
+	ldr	x16, [tsk, #TSK_TI_FLAGS]	// check for syscall hooks
 	tst	x16, #_TIF_SYSCALL_WORK
 	b.ne	__sys_trace
 	cmp     scno, sc_nr                     // check upper syscall limit
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 394c61db5566..b883f1f75216 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -127,6 +127,8 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
 
 void fpsimd_thread_switch(struct task_struct *next)
 {
+	if (!system_supports_fpsimd())
+		return;
 	/*
 	 * Save the current FPSIMD state to memory, but only if whatever is in
 	 * the registers is in fact the most recent userland FPSIMD state of
@@ -157,6 +159,8 @@ void fpsimd_thread_switch(struct task_struct *next)
 
 void fpsimd_flush_thread(void)
 {
+	if (!system_supports_fpsimd())
+		return;
 	memset(&current->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));
 	fpsimd_flush_task_state(current);
 	set_thread_flag(TIF_FOREIGN_FPSTATE);
@@ -168,6 +172,8 @@ void fpsimd_flush_thread(void)
  */
 void fpsimd_preserve_current_state(void)
 {
+	if (!system_supports_fpsimd())
+		return;
 	preempt_disable();
 	if (!test_thread_flag(TIF_FOREIGN_FPSTATE))
 		fpsimd_save_state(&current->thread.fpsimd_state);
@@ -181,6 +187,8 @@ void fpsimd_preserve_current_state(void)
  */
 void fpsimd_restore_current_state(void)
 {
+	if (!system_supports_fpsimd())
+		return;
 	preempt_disable();
 	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
 		struct fpsimd_state *st = &current->thread.fpsimd_state;
@@ -199,6 +207,8 @@ void fpsimd_restore_current_state(void)
  */
 void fpsimd_update_current_state(struct fpsimd_state *state)
 {
+	if (!system_supports_fpsimd())
+		return;
 	preempt_disable();
 	fpsimd_load_state(state);
 	if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
@@ -228,6 +238,8 @@ static DEFINE_PER_CPU(struct fpsimd_partial_state, softirq_fpsimdstate);
  */
 void kernel_neon_begin_partial(u32 num_regs)
 {
+	if (WARN_ON(!system_supports_fpsimd()))
+		return;
 	if (in_interrupt()) {
 		struct fpsimd_partial_state *s = this_cpu_ptr(
 			in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate);
@@ -252,6 +264,8 @@ EXPORT_SYMBOL(kernel_neon_begin_partial);
 
 void kernel_neon_end(void)
 {
+	if (!system_supports_fpsimd())
+		return;
 	if (in_interrupt()) {
 		struct fpsimd_partial_state *s = this_cpu_ptr(
 			in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate);
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 332e33193ccf..4b1abac3485a 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -326,14 +326,14 @@ __create_page_tables:
 	 * dirty cache lines being evicted.
 	 */
 	adrp	x0, idmap_pg_dir
-	adrp	x1, swapper_pg_dir + SWAPPER_DIR_SIZE
+	adrp	x1, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE
 	bl	__inval_cache_range
 
 	/*
 	 * Clear the idmap and swapper page tables.
 	 */
 	adrp	x0, idmap_pg_dir
-	adrp	x6, swapper_pg_dir + SWAPPER_DIR_SIZE
+	adrp	x6, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE
 1:	stp	xzr, xzr, [x0], #16
 	stp	xzr, xzr, [x0], #16
 	stp	xzr, xzr, [x0], #16
@@ -412,7 +412,7 @@ __create_page_tables:
 	 * tables again to remove any speculatively loaded cache lines.
 	 */
 	adrp	x0, idmap_pg_dir
-	adrp	x1, swapper_pg_dir + SWAPPER_DIR_SIZE
+	adrp	x1, swapper_pg_dir + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE
 	dmb	sy
 	bl	__inval_cache_range
 
@@ -428,7 +428,8 @@ ENDPROC(__create_page_tables)
 __primary_switched:
 	adrp	x4, init_thread_union
 	add	sp, x4, #THREAD_SIZE
-	msr	sp_el0, x4			// Save thread_info
+	adr_l	x5, init_task
+	msr	sp_el0, x5			// Save thread_info
 
 	adr_l	x8, vectors			// load VBAR_EL1 with virtual
 	msr	vbar_el1, x8			// vector table address
@@ -524,10 +525,21 @@ set_hcr:
 	msr	hcr_el2, x0
 	isb
 
-	/* Generic timers. */
+	/*
+	 * Allow Non-secure EL1 and EL0 to access physical timer and counter.
+	 * This is not necessary for VHE, since the host kernel runs in EL2,
+	 * and EL0 accesses are configured in the later stage of boot process.
+	 * Note that when HCR_EL2.E2H == 1, CNTHCTL_EL2 has the same bit layout
+	 * as CNTKCTL_EL1, and CNTKCTL_EL1 accessing instructions are redefined
+	 * to access CNTHCTL_EL2. This allows the kernel designed to run at EL1
+	 * to transparently mess with the EL0 bits via CNTKCTL_EL1 access in
+	 * EL2.
+	 */
+	cbnz	x2, 1f
 	mrs	x0, cnthctl_el2
 	orr	x0, x0, #3			// Enable EL1 physical timers
 	msr	cnthctl_el2, x0
+1:
 	msr	cntvoff_el2, xzr		// Clear virtual offset
 
 #ifdef CONFIG_ARM_GIC_V3
@@ -699,10 +711,10 @@ __secondary_switched:
 	isb
 
 	adr_l	x0, secondary_data
-	ldr	x0, [x0, #CPU_BOOT_STACK]	// get secondary_data.stack
-	mov	sp, x0
-	and	x0, x0, #~(THREAD_SIZE - 1)
-	msr	sp_el0, x0			// save thread_info
+	ldr	x1, [x0, #CPU_BOOT_STACK]	// get secondary_data.stack
+	mov	sp, x1
+	ldr	x2, [x0, #CPU_BOOT_TASK]
+	msr	sp_el0, x2
 	mov	x29, #0
 	b	secondary_start_kernel
 ENDPROC(__secondary_switched)
diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c
index d55a7b09959b..fe301cbcb442 100644
--- a/arch/arm64/kernel/hibernate.c
+++ b/arch/arm64/kernel/hibernate.c
@@ -136,7 +136,7 @@ int arch_hibernation_header_save(void *addr, unsigned int max_size)
 
 	/* Save the mpidr of the cpu we called cpu_suspend() on... */
 	if (sleep_cpu < 0) {
-		pr_err("Failing to hibernate on an unkown CPU.\n");
+		pr_err("Failing to hibernate on an unknown CPU.\n");
 		return -ENODEV;
 	}
 	hdr->sleep_cpu_mpidr = cpu_logical_map(sleep_cpu);
@@ -547,7 +547,7 @@ out:
 int hibernate_resume_nonboot_cpu_disable(void)
 {
 	if (sleep_cpu < 0) {
-		pr_err("Failing to resume from hibernate on an unkown CPU.\n");
+		pr_err("Failing to resume from hibernate on an unknown CPU.\n");
 		return -ENODEV;
 	}
 
diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c
index 948b73148d56..1b3c747fedda 100644
--- a/arch/arm64/kernel/hw_breakpoint.c
+++ b/arch/arm64/kernel/hw_breakpoint.c
@@ -317,9 +317,21 @@ static int get_hbp_len(u8 hbp_len)
 	case ARM_BREAKPOINT_LEN_2:
 		len_in_bytes = 2;
 		break;
+	case ARM_BREAKPOINT_LEN_3:
+		len_in_bytes = 3;
+		break;
 	case ARM_BREAKPOINT_LEN_4:
 		len_in_bytes = 4;
 		break;
+	case ARM_BREAKPOINT_LEN_5:
+		len_in_bytes = 5;
+		break;
+	case ARM_BREAKPOINT_LEN_6:
+		len_in_bytes = 6;
+		break;
+	case ARM_BREAKPOINT_LEN_7:
+		len_in_bytes = 7;
+		break;
 	case ARM_BREAKPOINT_LEN_8:
 		len_in_bytes = 8;
 		break;
@@ -349,7 +361,7 @@ int arch_check_bp_in_kernelspace(struct perf_event *bp)
  * to generic breakpoint descriptions.
  */
 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
-			   int *gen_len, int *gen_type)
+			   int *gen_len, int *gen_type, int *offset)
 {
 	/* Type */
 	switch (ctrl.type) {
@@ -369,17 +381,33 @@ int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
 		return -EINVAL;
 	}
 
+	if (!ctrl.len)
+		return -EINVAL;
+	*offset = __ffs(ctrl.len);
+
 	/* Len */
-	switch (ctrl.len) {
+	switch (ctrl.len >> *offset) {
 	case ARM_BREAKPOINT_LEN_1:
 		*gen_len = HW_BREAKPOINT_LEN_1;
 		break;
 	case ARM_BREAKPOINT_LEN_2:
 		*gen_len = HW_BREAKPOINT_LEN_2;
 		break;
+	case ARM_BREAKPOINT_LEN_3:
+		*gen_len = HW_BREAKPOINT_LEN_3;
+		break;
 	case ARM_BREAKPOINT_LEN_4:
 		*gen_len = HW_BREAKPOINT_LEN_4;
 		break;
+	case ARM_BREAKPOINT_LEN_5:
+		*gen_len = HW_BREAKPOINT_LEN_5;
+		break;
+	case ARM_BREAKPOINT_LEN_6:
+		*gen_len = HW_BREAKPOINT_LEN_6;
+		break;
+	case ARM_BREAKPOINT_LEN_7:
+		*gen_len = HW_BREAKPOINT_LEN_7;
+		break;
 	case ARM_BREAKPOINT_LEN_8:
 		*gen_len = HW_BREAKPOINT_LEN_8;
 		break;
@@ -423,9 +451,21 @@ static int arch_build_bp_info(struct perf_event *bp)
 	case HW_BREAKPOINT_LEN_2:
 		info->ctrl.len = ARM_BREAKPOINT_LEN_2;
 		break;
+	case HW_BREAKPOINT_LEN_3:
+		info->ctrl.len = ARM_BREAKPOINT_LEN_3;
+		break;
 	case HW_BREAKPOINT_LEN_4:
 		info->ctrl.len = ARM_BREAKPOINT_LEN_4;
 		break;
+	case HW_BREAKPOINT_LEN_5:
+		info->ctrl.len = ARM_BREAKPOINT_LEN_5;
+		break;
+	case HW_BREAKPOINT_LEN_6:
+		info->ctrl.len = ARM_BREAKPOINT_LEN_6;
+		break;
+	case HW_BREAKPOINT_LEN_7:
+		info->ctrl.len = ARM_BREAKPOINT_LEN_7;
+		break;
 	case HW_BREAKPOINT_LEN_8:
 		info->ctrl.len = ARM_BREAKPOINT_LEN_8;
 		break;
@@ -517,18 +557,17 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp)
 		default:
 			return -EINVAL;
 		}
-
-		info->address &= ~alignment_mask;
-		info->ctrl.len <<= offset;
 	} else {
 		if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE)
 			alignment_mask = 0x3;
 		else
 			alignment_mask = 0x7;
-		if (info->address & alignment_mask)
-			return -EINVAL;
+		offset = info->address & alignment_mask;
 	}
 
+	info->address &= ~alignment_mask;
+	info->ctrl.len <<= offset;
+
 	/*
 	 * Disallow per-task kernel breakpoints since these would
 	 * complicate the stepping code.
@@ -661,12 +700,47 @@ unlock:
 }
 NOKPROBE_SYMBOL(breakpoint_handler);
 
+/*
+ * Arm64 hardware does not always report a watchpoint hit address that matches
+ * one of the watchpoints set. It can also report an address "near" the
+ * watchpoint if a single instruction access both watched and unwatched
+ * addresses. There is no straight-forward way, short of disassembling the
+ * offending instruction, to map that address back to the watchpoint. This
+ * function computes the distance of the memory access from the watchpoint as a
+ * heuristic for the likelyhood that a given access triggered the watchpoint.
+ *
+ * See Section D2.10.5 "Determining the memory location that caused a Watchpoint
+ * exception" of ARMv8 Architecture Reference Manual for details.
+ *
+ * The function returns the distance of the address from the bytes watched by
+ * the watchpoint. In case of an exact match, it returns 0.
+ */
+static u64 get_distance_from_watchpoint(unsigned long addr, u64 val,
+					struct arch_hw_breakpoint_ctrl *ctrl)
+{
+	u64 wp_low, wp_high;
+	u32 lens, lene;
+
+	lens = __ffs(ctrl->len);
+	lene = __fls(ctrl->len);
+
+	wp_low = val + lens;
+	wp_high = val + lene;
+	if (addr < wp_low)
+		return wp_low - addr;
+	else if (addr > wp_high)
+		return addr - wp_high;
+	else
+		return 0;
+}
+
 static int watchpoint_handler(unsigned long addr, unsigned int esr,
 			      struct pt_regs *regs)
 {
-	int i, step = 0, *kernel_step, access;
+	int i, step = 0, *kernel_step, access, closest_match = 0;
+	u64 min_dist = -1, dist;
 	u32 ctrl_reg;
-	u64 val, alignment_mask;
+	u64 val;
 	struct perf_event *wp, **slots;
 	struct debug_info *debug_info;
 	struct arch_hw_breakpoint *info;
@@ -675,35 +749,15 @@ static int watchpoint_handler(unsigned long addr, unsigned int esr,
 	slots = this_cpu_ptr(wp_on_reg);
 	debug_info = &current->thread.debug;
 
+	/*
+	 * Find all watchpoints that match the reported address. If no exact
+	 * match is found. Attribute the hit to the closest watchpoint.
+	 */
+	rcu_read_lock();
 	for (i = 0; i < core_num_wrps; ++i) {
-		rcu_read_lock();
-
 		wp = slots[i];
-
 		if (wp == NULL)
-			goto unlock;
-
-		info = counter_arch_bp(wp);
-		/* AArch32 watchpoints are either 4 or 8 bytes aligned. */
-		if (is_compat_task()) {
-			if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
-				alignment_mask = 0x7;
-			else
-				alignment_mask = 0x3;
-		} else {
-			alignment_mask = 0x7;
-		}
-
-		/* Check if the watchpoint value matches. */
-		val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
-		if (val != (addr & ~alignment_mask))
-			goto unlock;
-
-		/* Possible match, check the byte address select to confirm. */
-		ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
-		decode_ctrl_reg(ctrl_reg, &ctrl);
-		if (!((1 << (addr & alignment_mask)) & ctrl.len))
-			goto unlock;
+			continue;
 
 		/*
 		 * Check that the access type matches.
@@ -712,18 +766,41 @@ static int watchpoint_handler(unsigned long addr, unsigned int esr,
 		access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W :
 			 HW_BREAKPOINT_R;
 		if (!(access & hw_breakpoint_type(wp)))
-			goto unlock;
+			continue;
 
+		/* Check if the watchpoint value and byte select match. */
+		val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
+		ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
+		decode_ctrl_reg(ctrl_reg, &ctrl);
+		dist = get_distance_from_watchpoint(addr, val, &ctrl);
+		if (dist < min_dist) {
+			min_dist = dist;
+			closest_match = i;
+		}
+		/* Is this an exact match? */
+		if (dist != 0)
+			continue;
+
+		info = counter_arch_bp(wp);
 		info->trigger = addr;
 		perf_bp_event(wp, regs);
 
 		/* Do we need to handle the stepping? */
 		if (is_default_overflow_handler(wp))
 			step = 1;
+	}
+	if (min_dist > 0 && min_dist != -1) {
+		/* No exact match found. */
+		wp = slots[closest_match];
+		info = counter_arch_bp(wp);
+		info->trigger = addr;
+		perf_bp_event(wp, regs);
 
-unlock:
-		rcu_read_unlock();
+		/* Do we need to handle the stepping? */
+		if (is_default_overflow_handler(wp))
+			step = 1;
 	}
+	rcu_read_unlock();
 
 	if (!step)
 		return 0;
diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c
index 6f2ac4fc66ca..94b62c1fa4df 100644
--- a/arch/arm64/kernel/insn.c
+++ b/arch/arm64/kernel/insn.c
@@ -30,7 +30,6 @@
 #include <asm/cacheflush.h>
 #include <asm/debug-monitors.h>
 #include <asm/fixmap.h>
-#include <asm/opcodes.h>
 #include <asm/insn.h>
 
 #define AARCH64_INSN_SF_BIT	BIT(31)
diff --git a/arch/arm64/kernel/kgdb.c b/arch/arm64/kernel/kgdb.c
index e017a9493b92..d217c9e95b06 100644
--- a/arch/arm64/kernel/kgdb.c
+++ b/arch/arm64/kernel/kgdb.c
@@ -247,6 +247,9 @@ NOKPROBE_SYMBOL(kgdb_compiled_brk_fn);
 
 static int kgdb_step_brk_fn(struct pt_regs *regs, unsigned int esr)
 {
+	if (!kgdb_single_step)
+		return DBG_HOOK_ERROR;
+
 	kgdb_handle_exception(1, SIGTRAP, 0, regs);
 	return 0;
 }
diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c
index acf38722457b..4f0e3ebfea4b 100644
--- a/arch/arm64/kernel/pci.c
+++ b/arch/arm64/kernel/pci.c
@@ -114,6 +114,19 @@ int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge)
 	return 0;
 }
 
+static int pci_acpi_root_prepare_resources(struct acpi_pci_root_info *ci)
+{
+	struct resource_entry *entry, *tmp;
+	int status;
+
+	status = acpi_pci_probe_root_resources(ci);
+	resource_list_for_each_entry_safe(entry, tmp, &ci->resources) {
+		if (!(entry->res->flags & IORESOURCE_WINDOW))
+			resource_list_destroy_entry(entry);
+	}
+	return status;
+}
+
 /*
  * Lookup the bus range for the domain in MCFG, and set up config space
  * mapping.
@@ -121,31 +134,33 @@ int pcibios_root_bridge_prepare(struct pci_host_bridge *bridge)
 static struct pci_config_window *
 pci_acpi_setup_ecam_mapping(struct acpi_pci_root *root)
 {
+	struct device *dev = &root->device->dev;
 	struct resource *bus_res = &root->secondary;
 	u16 seg = root->segment;
-	struct pci_config_window *cfg;
+	struct pci_ecam_ops *ecam_ops;
 	struct resource cfgres;
-	unsigned int bsz;
-
-	/* Use address from _CBA if present, otherwise lookup MCFG */
-	if (!root->mcfg_addr)
-		root->mcfg_addr = pci_mcfg_lookup(seg, bus_res);
+	struct acpi_device *adev;
+	struct pci_config_window *cfg;
+	int ret;
 
-	if (!root->mcfg_addr) {
-		dev_err(&root->device->dev, "%04x:%pR ECAM region not found\n",
-			seg, bus_res);
+	ret = pci_mcfg_lookup(root, &cfgres, &ecam_ops);
+	if (ret) {
+		dev_err(dev, "%04x:%pR ECAM region not found\n", seg, bus_res);
 		return NULL;
 	}
 
-	bsz = 1 << pci_generic_ecam_ops.bus_shift;
-	cfgres.start = root->mcfg_addr + bus_res->start * bsz;
-	cfgres.end = cfgres.start + resource_size(bus_res) * bsz - 1;
-	cfgres.flags = IORESOURCE_MEM;
-	cfg = pci_ecam_create(&root->device->dev, &cfgres, bus_res,
-			      &pci_generic_ecam_ops);
+	adev = acpi_resource_consumer(&cfgres);
+	if (adev)
+		dev_info(dev, "ECAM area %pR reserved by %s\n", &cfgres,
+			 dev_name(&adev->dev));
+	else
+		dev_warn(dev, FW_BUG "ECAM area %pR not reserved in ACPI namespace\n",
+			 &cfgres);
+
+	cfg = pci_ecam_create(dev, &cfgres, bus_res, ecam_ops);
 	if (IS_ERR(cfg)) {
-		dev_err(&root->device->dev, "%04x:%pR error %ld mapping ECAM\n",
-			seg, bus_res, PTR_ERR(cfg));
+		dev_err(dev, "%04x:%pR error %ld mapping ECAM\n", seg, bus_res,
+			PTR_ERR(cfg));
 		return NULL;
 	}
 
@@ -159,33 +174,37 @@ static void pci_acpi_generic_release_info(struct acpi_pci_root_info *ci)
 
 	ri = container_of(ci, struct acpi_pci_generic_root_info, common);
 	pci_ecam_free(ri->cfg);
+	kfree(ci->ops);
 	kfree(ri);
 }
 
-static struct acpi_pci_root_ops acpi_pci_root_ops = {
-	.release_info = pci_acpi_generic_release_info,
-};
-
 /* Interface called from ACPI code to setup PCI host controller */
 struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root)
 {
 	int node = acpi_get_node(root->device->handle);
 	struct acpi_pci_generic_root_info *ri;
 	struct pci_bus *bus, *child;
+	struct acpi_pci_root_ops *root_ops;
 
 	ri = kzalloc_node(sizeof(*ri), GFP_KERNEL, node);
 	if (!ri)
 		return NULL;
 
+	root_ops = kzalloc_node(sizeof(*root_ops), GFP_KERNEL, node);
+	if (!root_ops)
+		return NULL;
+
 	ri->cfg = pci_acpi_setup_ecam_mapping(root);
 	if (!ri->cfg) {
 		kfree(ri);
+		kfree(root_ops);
 		return NULL;
 	}
 
-	acpi_pci_root_ops.pci_ops = &ri->cfg->ops->pci_ops;
-	bus = acpi_pci_root_create(root, &acpi_pci_root_ops, &ri->common,
-				   ri->cfg);
+	root_ops->release_info = pci_acpi_generic_release_info;
+	root_ops->prepare_resources = pci_acpi_root_prepare_resources;
+	root_ops->pci_ops = &ri->cfg->ops->pci_ops;
+	bus = acpi_pci_root_create(root, root_ops, &ri->common, ri->cfg);
 	if (!bus)
 		return NULL;
 
diff --git a/arch/arm64/kernel/probes/Makefile b/arch/arm64/kernel/probes/Makefile
index ce06312e3d34..89b6df613dde 100644
--- a/arch/arm64/kernel/probes/Makefile
+++ b/arch/arm64/kernel/probes/Makefile
@@ -1,3 +1,5 @@
 obj-$(CONFIG_KPROBES)		+= kprobes.o decode-insn.o	\
 				   kprobes_trampoline.o		\
 				   simulate-insn.o
+obj-$(CONFIG_UPROBES)		+= uprobes.o decode-insn.o	\
+				   simulate-insn.o
diff --git a/arch/arm64/kernel/probes/decode-insn.c b/arch/arm64/kernel/probes/decode-insn.c
index d1731bf977ef..6bf6657a5a52 100644
--- a/arch/arm64/kernel/probes/decode-insn.c
+++ b/arch/arm64/kernel/probes/decode-insn.c
@@ -17,7 +17,6 @@
 #include <linux/kprobes.h>
 #include <linux/module.h>
 #include <linux/kallsyms.h>
-#include <asm/kprobes.h>
 #include <asm/insn.h>
 #include <asm/sections.h>
 
@@ -78,8 +77,8 @@ static bool __kprobes aarch64_insn_is_steppable(u32 insn)
  *   INSN_GOOD         If instruction is supported and uses instruction slot,
  *   INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot.
  */
-static enum kprobe_insn __kprobes
-arm_probe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi)
+enum probe_insn __kprobes
+arm_probe_decode_insn(probe_opcode_t insn, struct arch_probe_insn *api)
 {
 	/*
 	 * Instructions reading or modifying the PC won't work from the XOL
@@ -89,26 +88,26 @@ arm_probe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 		return INSN_GOOD;
 
 	if (aarch64_insn_is_bcond(insn)) {
-		asi->handler = simulate_b_cond;
+		api->handler = simulate_b_cond;
 	} else if (aarch64_insn_is_cbz(insn) ||
 	    aarch64_insn_is_cbnz(insn)) {
-		asi->handler = simulate_cbz_cbnz;
+		api->handler = simulate_cbz_cbnz;
 	} else if (aarch64_insn_is_tbz(insn) ||
 	    aarch64_insn_is_tbnz(insn)) {
-		asi->handler = simulate_tbz_tbnz;
+		api->handler = simulate_tbz_tbnz;
 	} else if (aarch64_insn_is_adr_adrp(insn)) {
-		asi->handler = simulate_adr_adrp;
+		api->handler = simulate_adr_adrp;
 	} else if (aarch64_insn_is_b(insn) ||
 	    aarch64_insn_is_bl(insn)) {
-		asi->handler = simulate_b_bl;
+		api->handler = simulate_b_bl;
 	} else if (aarch64_insn_is_br(insn) ||
 	    aarch64_insn_is_blr(insn) ||
 	    aarch64_insn_is_ret(insn)) {
-		asi->handler = simulate_br_blr_ret;
+		api->handler = simulate_br_blr_ret;
 	} else if (aarch64_insn_is_ldr_lit(insn)) {
-		asi->handler = simulate_ldr_literal;
+		api->handler = simulate_ldr_literal;
 	} else if (aarch64_insn_is_ldrsw_lit(insn)) {
-		asi->handler = simulate_ldrsw_literal;
+		api->handler = simulate_ldrsw_literal;
 	} else {
 		/*
 		 * Instruction cannot be stepped out-of-line and we don't
@@ -120,6 +119,7 @@ arm_probe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 	return INSN_GOOD_NO_SLOT;
 }
 
+#ifdef CONFIG_KPROBES
 static bool __kprobes
 is_probed_address_atomic(kprobe_opcode_t *scan_start, kprobe_opcode_t *scan_end)
 {
@@ -138,12 +138,12 @@ is_probed_address_atomic(kprobe_opcode_t *scan_start, kprobe_opcode_t *scan_end)
 	return false;
 }
 
-enum kprobe_insn __kprobes
+enum probe_insn __kprobes
 arm_kprobe_decode_insn(kprobe_opcode_t *addr, struct arch_specific_insn *asi)
 {
-	enum kprobe_insn decoded;
-	kprobe_opcode_t insn = le32_to_cpu(*addr);
-	kprobe_opcode_t *scan_end = NULL;
+	enum probe_insn decoded;
+	probe_opcode_t insn = le32_to_cpu(*addr);
+	probe_opcode_t *scan_end = NULL;
 	unsigned long size = 0, offset = 0;
 
 	/*
@@ -162,7 +162,7 @@ arm_kprobe_decode_insn(kprobe_opcode_t *addr, struct arch_specific_insn *asi)
 		else
 			scan_end = addr - MAX_ATOMIC_CONTEXT_SIZE;
 	}
-	decoded = arm_probe_decode_insn(insn, asi);
+	decoded = arm_probe_decode_insn(insn, &asi->api);
 
 	if (decoded != INSN_REJECTED && scan_end)
 		if (is_probed_address_atomic(addr - 1, scan_end))
@@ -170,3 +170,4 @@ arm_kprobe_decode_insn(kprobe_opcode_t *addr, struct arch_specific_insn *asi)
 
 	return decoded;
 }
+#endif
diff --git a/arch/arm64/kernel/probes/decode-insn.h b/arch/arm64/kernel/probes/decode-insn.h
index d438289646a6..76d3f315407f 100644
--- a/arch/arm64/kernel/probes/decode-insn.h
+++ b/arch/arm64/kernel/probes/decode-insn.h
@@ -23,13 +23,17 @@
  */
 #define MAX_ATOMIC_CONTEXT_SIZE	(128 / sizeof(kprobe_opcode_t))
 
-enum kprobe_insn {
+enum probe_insn {
 	INSN_REJECTED,
 	INSN_GOOD_NO_SLOT,
 	INSN_GOOD,
 };
 
-enum kprobe_insn __kprobes
+#ifdef CONFIG_KPROBES
+enum probe_insn __kprobes
 arm_kprobe_decode_insn(kprobe_opcode_t *addr, struct arch_specific_insn *asi);
+#endif
+enum probe_insn __kprobes
+arm_probe_decode_insn(probe_opcode_t insn, struct arch_probe_insn *asi);
 
 #endif /* _ARM_KERNEL_KPROBES_ARM64_H */
diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c
index f5077ea7af6d..1decd2b2c730 100644
--- a/arch/arm64/kernel/probes/kprobes.c
+++ b/arch/arm64/kernel/probes/kprobes.c
@@ -44,31 +44,31 @@ post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *);
 static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
 {
 	/* prepare insn slot */
-	p->ainsn.insn[0] = cpu_to_le32(p->opcode);
+	p->ainsn.api.insn[0] = cpu_to_le32(p->opcode);
 
-	flush_icache_range((uintptr_t) (p->ainsn.insn),
-			   (uintptr_t) (p->ainsn.insn) +
+	flush_icache_range((uintptr_t) (p->ainsn.api.insn),
+			   (uintptr_t) (p->ainsn.api.insn) +
 			   MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
 
 	/*
 	 * Needs restoring of return address after stepping xol.
 	 */
-	p->ainsn.restore = (unsigned long) p->addr +
+	p->ainsn.api.restore = (unsigned long) p->addr +
 	  sizeof(kprobe_opcode_t);
 }
 
 static void __kprobes arch_prepare_simulate(struct kprobe *p)
 {
 	/* This instructions is not executed xol. No need to adjust the PC */
-	p->ainsn.restore = 0;
+	p->ainsn.api.restore = 0;
 }
 
 static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs)
 {
 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 
-	if (p->ainsn.handler)
-		p->ainsn.handler((u32)p->opcode, (long)p->addr, regs);
+	if (p->ainsn.api.handler)
+		p->ainsn.api.handler((u32)p->opcode, (long)p->addr, regs);
 
 	/* single step simulated, now go for post processing */
 	post_kprobe_handler(kcb, regs);
@@ -98,18 +98,18 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p)
 		return -EINVAL;
 
 	case INSN_GOOD_NO_SLOT:	/* insn need simulation */
-		p->ainsn.insn = NULL;
+		p->ainsn.api.insn = NULL;
 		break;
 
 	case INSN_GOOD:	/* instruction uses slot */
-		p->ainsn.insn = get_insn_slot();
-		if (!p->ainsn.insn)
+		p->ainsn.api.insn = get_insn_slot();
+		if (!p->ainsn.api.insn)
 			return -ENOMEM;
 		break;
 	};
 
 	/* prepare the instruction */
-	if (p->ainsn.insn)
+	if (p->ainsn.api.insn)
 		arch_prepare_ss_slot(p);
 	else
 		arch_prepare_simulate(p);
@@ -142,9 +142,9 @@ void __kprobes arch_disarm_kprobe(struct kprobe *p)
 
 void __kprobes arch_remove_kprobe(struct kprobe *p)
 {
-	if (p->ainsn.insn) {
-		free_insn_slot(p->ainsn.insn, 0);
-		p->ainsn.insn = NULL;
+	if (p->ainsn.api.insn) {
+		free_insn_slot(p->ainsn.api.insn, 0);
+		p->ainsn.api.insn = NULL;
 	}
 }
 
@@ -244,9 +244,9 @@ static void __kprobes setup_singlestep(struct kprobe *p,
 	}
 
 
-	if (p->ainsn.insn) {
+	if (p->ainsn.api.insn) {
 		/* prepare for single stepping */
-		slot = (unsigned long)p->ainsn.insn;
+		slot = (unsigned long)p->ainsn.api.insn;
 
 		set_ss_context(kcb, slot);	/* mark pending ss */
 
@@ -295,8 +295,8 @@ post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs)
 		return;
 
 	/* return addr restore if non-branching insn */
-	if (cur->ainsn.restore != 0)
-		instruction_pointer_set(regs, cur->ainsn.restore);
+	if (cur->ainsn.api.restore != 0)
+		instruction_pointer_set(regs, cur->ainsn.api.restore);
 
 	/* restore back original saved kprobe variables and continue */
 	if (kcb->kprobe_status == KPROBE_REENTER) {
diff --git a/arch/arm64/kernel/probes/simulate-insn.c b/arch/arm64/kernel/probes/simulate-insn.c
index 8977ce9d009d..357d3efe1366 100644
--- a/arch/arm64/kernel/probes/simulate-insn.c
+++ b/arch/arm64/kernel/probes/simulate-insn.c
@@ -13,28 +13,26 @@
  * General Public License for more details.
  */
 
+#include <linux/bitops.h>
 #include <linux/kernel.h>
 #include <linux/kprobes.h>
 
 #include "simulate-insn.h"
 
-#define sign_extend(x, signbit)		\
-	((x) | (0 - ((x) & (1 << (signbit)))))
-
 #define bbl_displacement(insn)		\
-	sign_extend(((insn) & 0x3ffffff) << 2, 27)
+	sign_extend32(((insn) & 0x3ffffff) << 2, 27)
 
 #define bcond_displacement(insn)	\
-	sign_extend(((insn >> 5) & 0x7ffff) << 2, 20)
+	sign_extend32(((insn >> 5) & 0x7ffff) << 2, 20)
 
 #define cbz_displacement(insn)	\
-	sign_extend(((insn >> 5) & 0x7ffff) << 2, 20)
+	sign_extend32(((insn >> 5) & 0x7ffff) << 2, 20)
 
 #define tbz_displacement(insn)	\
-	sign_extend(((insn >> 5) & 0x3fff) << 2, 15)
+	sign_extend32(((insn >> 5) & 0x3fff) << 2, 15)
 
 #define ldr_displacement(insn)	\
-	sign_extend(((insn >> 5) & 0x7ffff) << 2, 20)
+	sign_extend32(((insn >> 5) & 0x7ffff) << 2, 20)
 
 static inline void set_x_reg(struct pt_regs *regs, int reg, u64 val)
 {
@@ -106,7 +104,7 @@ simulate_adr_adrp(u32 opcode, long addr, struct pt_regs *regs)
 
 	xn = opcode & 0x1f;
 	imm = ((opcode >> 3) & 0x1ffffc) | ((opcode >> 29) & 0x3);
-	imm = sign_extend(imm, 20);
+	imm = sign_extend64(imm, 20);
 	if (opcode & 0x80000000)
 		val = (imm<<12) + (addr & 0xfffffffffffff000);
 	else
diff --git a/arch/arm64/kernel/probes/uprobes.c b/arch/arm64/kernel/probes/uprobes.c
new file mode 100644
index 000000000000..26c998534dca
--- /dev/null
+++ b/arch/arm64/kernel/probes/uprobes.c
@@ -0,0 +1,216 @@
+/*
+ * Copyright (C) 2014-2016 Pratyush Anand <panand@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/highmem.h>
+#include <linux/ptrace.h>
+#include <linux/uprobes.h>
+#include <asm/cacheflush.h>
+
+#include "decode-insn.h"
+
+#define UPROBE_INV_FAULT_CODE	UINT_MAX
+
+void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr,
+		void *src, unsigned long len)
+{
+	void *xol_page_kaddr = kmap_atomic(page);
+	void *dst = xol_page_kaddr + (vaddr & ~PAGE_MASK);
+
+	/* Initialize the slot */
+	memcpy(dst, src, len);
+
+	/* flush caches (dcache/icache) */
+	sync_icache_aliases(dst, len);
+
+	kunmap_atomic(xol_page_kaddr);
+}
+
+unsigned long uprobe_get_swbp_addr(struct pt_regs *regs)
+{
+	return instruction_pointer(regs);
+}
+
+int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm,
+		unsigned long addr)
+{
+	probe_opcode_t insn;
+
+	/* TODO: Currently we do not support AARCH32 instruction probing */
+	if (test_bit(TIF_32BIT, &mm->context.flags))
+		return -ENOTSUPP;
+	else if (!IS_ALIGNED(addr, AARCH64_INSN_SIZE))
+		return -EINVAL;
+
+	insn = *(probe_opcode_t *)(&auprobe->insn[0]);
+
+	switch (arm_probe_decode_insn(insn, &auprobe->api)) {
+	case INSN_REJECTED:
+		return -EINVAL;
+
+	case INSN_GOOD_NO_SLOT:
+		auprobe->simulate = true;
+		break;
+
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
+{
+	struct uprobe_task *utask = current->utask;
+
+	/* Initialize with an invalid fault code to detect if ol insn trapped */
+	current->thread.fault_code = UPROBE_INV_FAULT_CODE;
+
+	/* Instruction points to execute ol */
+	instruction_pointer_set(regs, utask->xol_vaddr);
+
+	user_enable_single_step(current);
+
+	return 0;
+}
+
+int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
+{
+	struct uprobe_task *utask = current->utask;
+
+	WARN_ON_ONCE(current->thread.fault_code != UPROBE_INV_FAULT_CODE);
+
+	/* Instruction points to execute next to breakpoint address */
+	instruction_pointer_set(regs, utask->vaddr + 4);
+
+	user_disable_single_step(current);
+
+	return 0;
+}
+bool arch_uprobe_xol_was_trapped(struct task_struct *t)
+{
+	/*
+	 * Between arch_uprobe_pre_xol and arch_uprobe_post_xol, if an xol
+	 * insn itself is trapped, then detect the case with the help of
+	 * invalid fault code which is being set in arch_uprobe_pre_xol
+	 */
+	if (t->thread.fault_code != UPROBE_INV_FAULT_CODE)
+		return true;
+
+	return false;
+}
+
+bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
+{
+	probe_opcode_t insn;
+	unsigned long addr;
+
+	if (!auprobe->simulate)
+		return false;
+
+	insn = *(probe_opcode_t *)(&auprobe->insn[0]);
+	addr = instruction_pointer(regs);
+
+	if (auprobe->api.handler)
+		auprobe->api.handler(insn, addr, regs);
+
+	return true;
+}
+
+void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
+{
+	struct uprobe_task *utask = current->utask;
+
+	/*
+	 * Task has received a fatal signal, so reset back to probbed
+	 * address.
+	 */
+	instruction_pointer_set(regs, utask->vaddr);
+
+	user_disable_single_step(current);
+}
+
+bool arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
+		struct pt_regs *regs)
+{
+	/*
+	 * If a simple branch instruction (B) was called for retprobed
+	 * assembly label then return true even when regs->sp and ret->stack
+	 * are same. It will ensure that cleanup and reporting of return
+	 * instances corresponding to callee label is done when
+	 * handle_trampoline for called function is executed.
+	 */
+	if (ctx == RP_CHECK_CHAIN_CALL)
+		return regs->sp <= ret->stack;
+	else
+		return regs->sp < ret->stack;
+}
+
+unsigned long
+arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr,
+				  struct pt_regs *regs)
+{
+	unsigned long orig_ret_vaddr;
+
+	orig_ret_vaddr = procedure_link_pointer(regs);
+	/* Replace the return addr with trampoline addr */
+	procedure_link_pointer_set(regs, trampoline_vaddr);
+
+	return orig_ret_vaddr;
+}
+
+int arch_uprobe_exception_notify(struct notifier_block *self,
+				 unsigned long val, void *data)
+{
+	return NOTIFY_DONE;
+}
+
+static int uprobe_breakpoint_handler(struct pt_regs *regs,
+		unsigned int esr)
+{
+	if (user_mode(regs) && uprobe_pre_sstep_notifier(regs))
+		return DBG_HOOK_HANDLED;
+
+	return DBG_HOOK_ERROR;
+}
+
+static int uprobe_single_step_handler(struct pt_regs *regs,
+		unsigned int esr)
+{
+	struct uprobe_task *utask = current->utask;
+
+	if (user_mode(regs)) {
+		WARN_ON(utask &&
+			(instruction_pointer(regs) != utask->xol_vaddr + 4));
+
+		if (uprobe_post_sstep_notifier(regs))
+			return DBG_HOOK_HANDLED;
+	}
+
+	return DBG_HOOK_ERROR;
+}
+
+/* uprobe breakpoint handler hook */
+static struct break_hook uprobes_break_hook = {
+	.esr_mask = BRK64_ESR_MASK,
+	.esr_val = BRK64_ESR_UPROBES,
+	.fn = uprobe_breakpoint_handler,
+};
+
+/* uprobe single step handler hook */
+static struct step_hook uprobes_step_hook = {
+	.fn = uprobe_single_step_handler,
+};
+
+static int __init arch_init_uprobes(void)
+{
+	register_break_hook(&uprobes_break_hook);
+	register_step_hook(&uprobes_step_hook);
+
+	return 0;
+}
+
+device_initcall(arch_init_uprobes);
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 01753cd7d3f0..a3a2816ba73a 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -45,6 +45,7 @@
 #include <linux/personality.h>
 #include <linux/notifier.h>
 #include <trace/events/power.h>
+#include <linux/percpu.h>
 
 #include <asm/alternative.h>
 #include <asm/compat.h>
@@ -282,7 +283,7 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start,
 		memset(childregs, 0, sizeof(struct pt_regs));
 		childregs->pstate = PSR_MODE_EL1h;
 		if (IS_ENABLED(CONFIG_ARM64_UAO) &&
-		    cpus_have_cap(ARM64_HAS_UAO))
+		    cpus_have_const_cap(ARM64_HAS_UAO))
 			childregs->pstate |= PSR_UAO_BIT;
 		p->thread.cpu_context.x19 = stack_start;
 		p->thread.cpu_context.x20 = stk_sz;
@@ -322,6 +323,20 @@ void uao_thread_switch(struct task_struct *next)
 }
 
 /*
+ * We store our current task in sp_el0, which is clobbered by userspace. Keep a
+ * shadow copy so that we can restore this upon entry from userspace.
+ *
+ * This is *only* for exception entry from EL0, and is not valid until we
+ * __switch_to() a user task.
+ */
+DEFINE_PER_CPU(struct task_struct *, __entry_task);
+
+static void entry_task_switch(struct task_struct *next)
+{
+	__this_cpu_write(__entry_task, next);
+}
+
+/*
  * Thread switching.
  */
 struct task_struct *__switch_to(struct task_struct *prev,
@@ -333,6 +348,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
 	tls_thread_switch(next);
 	hw_breakpoint_thread_switch(next);
 	contextidr_thread_switch(next);
+	entry_task_switch(next);
 	uao_thread_switch(next);
 
 	/*
@@ -350,27 +366,35 @@ struct task_struct *__switch_to(struct task_struct *prev,
 unsigned long get_wchan(struct task_struct *p)
 {
 	struct stackframe frame;
-	unsigned long stack_page;
+	unsigned long stack_page, ret = 0;
 	int count = 0;
 	if (!p || p == current || p->state == TASK_RUNNING)
 		return 0;
 
+	stack_page = (unsigned long)try_get_task_stack(p);
+	if (!stack_page)
+		return 0;
+
 	frame.fp = thread_saved_fp(p);
 	frame.sp = thread_saved_sp(p);
 	frame.pc = thread_saved_pc(p);
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	frame.graph = p->curr_ret_stack;
 #endif
-	stack_page = (unsigned long)task_stack_page(p);
 	do {
 		if (frame.sp < stack_page ||
 		    frame.sp >= stack_page + THREAD_SIZE ||
 		    unwind_frame(p, &frame))
-			return 0;
-		if (!in_sched_functions(frame.pc))
-			return frame.pc;
+			goto out;
+		if (!in_sched_functions(frame.pc)) {
+			ret = frame.pc;
+			goto out;
+		}
 	} while (count ++ < 16);
-	return 0;
+
+out:
+	put_task_stack(p);
+	return ret;
 }
 
 unsigned long arch_align_stack(unsigned long sp)
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index e0c81da60f76..fc35e06ccaac 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -327,13 +327,13 @@ static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type,
 				     struct arch_hw_breakpoint_ctrl ctrl,
 				     struct perf_event_attr *attr)
 {
-	int err, len, type, disabled = !ctrl.enabled;
+	int err, len, type, offset, disabled = !ctrl.enabled;
 
 	attr->disabled = disabled;
 	if (disabled)
 		return 0;
 
-	err = arch_bp_generic_fields(ctrl, &len, &type);
+	err = arch_bp_generic_fields(ctrl, &len, &type, &offset);
 	if (err)
 		return err;
 
@@ -352,6 +352,7 @@ static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type,
 
 	attr->bp_len	= len;
 	attr->bp_type	= type;
+	attr->bp_addr	+= offset;
 
 	return 0;
 }
@@ -404,7 +405,7 @@ static int ptrace_hbp_get_addr(unsigned int note_type,
 	if (IS_ERR(bp))
 		return PTR_ERR(bp);
 
-	*addr = bp ? bp->attr.bp_addr : 0;
+	*addr = bp ? counter_arch_bp(bp)->address : 0;
 	return 0;
 }
 
diff --git a/arch/arm64/kernel/return_address.c b/arch/arm64/kernel/return_address.c
index 1718706fde83..12a87f2600f2 100644
--- a/arch/arm64/kernel/return_address.c
+++ b/arch/arm64/kernel/return_address.c
@@ -12,6 +12,7 @@
 #include <linux/export.h>
 #include <linux/ftrace.h>
 
+#include <asm/stack_pointer.h>
 #include <asm/stacktrace.h>
 
 struct return_address_data {
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index f534f492a268..a53f52ac81c6 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -291,6 +291,15 @@ void __init setup_arch(char **cmdline_p)
 	smp_init_cpus();
 	smp_build_mpidr_hash();
 
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+	/*
+	 * Make sure init_thread_info.ttbr0 always generates translation
+	 * faults in case uaccess_enable() is inadvertently called by the init
+	 * thread.
+	 */
+	init_task.thread_info.ttbr0 = virt_to_phys(empty_zero_page);
+#endif
+
 #ifdef CONFIG_VT
 #if defined(CONFIG_VGA_CONSOLE)
 	conswitchp = &vga_con;
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 404dd67080b9..c7b6de62f9d3 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -414,6 +414,9 @@ asmlinkage void do_notify_resume(struct pt_regs *regs,
 		} else {
 			local_irq_enable();
 
+			if (thread_flags & _TIF_UPROBE)
+				uprobe_notify_resume(regs);
+
 			if (thread_flags & _TIF_SIGPENDING)
 				do_signal(regs);
 
diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
index 1bec41b5fda3..df67652e46f0 100644
--- a/arch/arm64/kernel/sleep.S
+++ b/arch/arm64/kernel/sleep.S
@@ -125,9 +125,6 @@ ENTRY(_cpu_resume)
 	/* load sp from context */
 	ldr	x2, [x0, #CPU_CTX_SP]
 	mov	sp, x2
-	/* save thread_info */
-	and	x2, x2, #~(THREAD_SIZE - 1)
-	msr	sp_el0, x2
 	/*
 	 * cpu_do_resume expects x0 to contain context address pointer
 	 */
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 8507703dabe4..cb87234cfcf2 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -58,6 +58,9 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/ipi.h>
 
+DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number);
+EXPORT_PER_CPU_SYMBOL(cpu_number);
+
 /*
  * as from 2.5, kernels no longer have an init_tasks structure
  * so we need some other way of telling a new secondary core
@@ -146,6 +149,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
 	 * We need to tell the secondary core where to find its stack and the
 	 * page tables.
 	 */
+	secondary_data.task = idle;
 	secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
 	update_cpu_boot_status(CPU_MMU_OFF);
 	__flush_dcache_area(&secondary_data, sizeof(secondary_data));
@@ -170,6 +174,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
 		pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
 	}
 
+	secondary_data.task = NULL;
 	secondary_data.stack = NULL;
 	status = READ_ONCE(secondary_data.status);
 	if (ret && status) {
@@ -208,7 +213,10 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
 asmlinkage void secondary_start_kernel(void)
 {
 	struct mm_struct *mm = &init_mm;
-	unsigned int cpu = smp_processor_id();
+	unsigned int cpu;
+
+	cpu = task_cpu(current);
+	set_my_cpu_offset(per_cpu_offset(cpu));
 
 	/*
 	 * All kernel threads share the same mm context; grab a
@@ -217,8 +225,6 @@ asmlinkage void secondary_start_kernel(void)
 	atomic_inc(&mm->mm_count);
 	current->active_mm = mm;
 
-	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
-
 	/*
 	 * TTBR0 is only used for the identity mapping at this stage. Make it
 	 * point to zero page to avoid speculatively fetching new entries.
@@ -718,6 +724,8 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
 	 */
 	for_each_possible_cpu(cpu) {
 
+		per_cpu(cpu_number, cpu) = cpu;
+
 		if (cpu == smp_processor_id())
 			continue;
 
diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
index c2efddfca18c..8a552a33c6ef 100644
--- a/arch/arm64/kernel/stacktrace.c
+++ b/arch/arm64/kernel/stacktrace.c
@@ -22,6 +22,7 @@
 #include <linux/stacktrace.h>
 
 #include <asm/irq.h>
+#include <asm/stack_pointer.h>
 #include <asm/stacktrace.h>
 
 /*
@@ -128,7 +129,6 @@ void notrace walk_stackframe(struct task_struct *tsk, struct stackframe *frame,
 			break;
 	}
 }
-EXPORT_SYMBOL(walk_stackframe);
 
 #ifdef CONFIG_STACKTRACE
 struct stack_trace_data {
@@ -181,6 +181,9 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
 	struct stack_trace_data data;
 	struct stackframe frame;
 
+	if (!try_get_task_stack(tsk))
+		return;
+
 	data.trace = trace;
 	data.skip = trace->skip;
 
@@ -202,6 +205,8 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
 	walk_stackframe(tsk, &frame, save_trace, &data);
 	if (trace->nr_entries < trace->max_entries)
 		trace->entries[trace->nr_entries++] = ULONG_MAX;
+
+	put_task_stack(tsk);
 }
 
 void save_stack_trace(struct stack_trace *trace)
diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index bb0cd787a9d3..1e3be9064cfa 100644
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -47,12 +47,6 @@ void notrace __cpu_suspend_exit(void)
 	cpu_uninstall_idmap();
 
 	/*
-	 * Restore per-cpu offset before any kernel
-	 * subsystem relying on it has a chance to run.
-	 */
-	set_my_cpu_offset(per_cpu_offset(cpu));
-
-	/*
 	 * PSTATE was not saved over suspend/resume, re-enable any detected
 	 * features that might not have been set correctly.
 	 */
diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c
index 694f6deedbab..23e9e13bd2aa 100644
--- a/arch/arm64/kernel/topology.c
+++ b/arch/arm64/kernel/topology.c
@@ -19,10 +19,226 @@
 #include <linux/nodemask.h>
 #include <linux/of.h>
 #include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/cpufreq.h>
 
+#include <asm/cpu.h>
 #include <asm/cputype.h>
 #include <asm/topology.h>
 
+static DEFINE_PER_CPU(unsigned long, cpu_scale) = SCHED_CAPACITY_SCALE;
+static DEFINE_MUTEX(cpu_scale_mutex);
+
+unsigned long arch_scale_cpu_capacity(struct sched_domain *sd, int cpu)
+{
+	return per_cpu(cpu_scale, cpu);
+}
+
+static void set_capacity_scale(unsigned int cpu, unsigned long capacity)
+{
+	per_cpu(cpu_scale, cpu) = capacity;
+}
+
+#ifdef CONFIG_PROC_SYSCTL
+static ssize_t cpu_capacity_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct cpu *cpu = container_of(dev, struct cpu, dev);
+
+	return sprintf(buf, "%lu\n",
+			arch_scale_cpu_capacity(NULL, cpu->dev.id));
+}
+
+static ssize_t cpu_capacity_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf,
+				  size_t count)
+{
+	struct cpu *cpu = container_of(dev, struct cpu, dev);
+	int this_cpu = cpu->dev.id, i;
+	unsigned long new_capacity;
+	ssize_t ret;
+
+	if (count) {
+		ret = kstrtoul(buf, 0, &new_capacity);
+		if (ret)
+			return ret;
+		if (new_capacity > SCHED_CAPACITY_SCALE)
+			return -EINVAL;
+
+		mutex_lock(&cpu_scale_mutex);
+		for_each_cpu(i, &cpu_topology[this_cpu].core_sibling)
+			set_capacity_scale(i, new_capacity);
+		mutex_unlock(&cpu_scale_mutex);
+	}
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(cpu_capacity);
+
+static int register_cpu_capacity_sysctl(void)
+{
+	int i;
+	struct device *cpu;
+
+	for_each_possible_cpu(i) {
+		cpu = get_cpu_device(i);
+		if (!cpu) {
+			pr_err("%s: too early to get CPU%d device!\n",
+			       __func__, i);
+			continue;
+		}
+		device_create_file(cpu, &dev_attr_cpu_capacity);
+	}
+
+	return 0;
+}
+subsys_initcall(register_cpu_capacity_sysctl);
+#endif
+
+static u32 capacity_scale;
+static u32 *raw_capacity;
+static bool cap_parsing_failed;
+
+static void __init parse_cpu_capacity(struct device_node *cpu_node, int cpu)
+{
+	int ret;
+	u32 cpu_capacity;
+
+	if (cap_parsing_failed)
+		return;
+
+	ret = of_property_read_u32(cpu_node,
+				   "capacity-dmips-mhz",
+				   &cpu_capacity);
+	if (!ret) {
+		if (!raw_capacity) {
+			raw_capacity = kcalloc(num_possible_cpus(),
+					       sizeof(*raw_capacity),
+					       GFP_KERNEL);
+			if (!raw_capacity) {
+				pr_err("cpu_capacity: failed to allocate memory for raw capacities\n");
+				cap_parsing_failed = true;
+				return;
+			}
+		}
+		capacity_scale = max(cpu_capacity, capacity_scale);
+		raw_capacity[cpu] = cpu_capacity;
+		pr_debug("cpu_capacity: %s cpu_capacity=%u (raw)\n",
+			cpu_node->full_name, raw_capacity[cpu]);
+	} else {
+		if (raw_capacity) {
+			pr_err("cpu_capacity: missing %s raw capacity\n",
+				cpu_node->full_name);
+			pr_err("cpu_capacity: partial information: fallback to 1024 for all CPUs\n");
+		}
+		cap_parsing_failed = true;
+		kfree(raw_capacity);
+	}
+}
+
+static void normalize_cpu_capacity(void)
+{
+	u64 capacity;
+	int cpu;
+
+	if (!raw_capacity || cap_parsing_failed)
+		return;
+
+	pr_debug("cpu_capacity: capacity_scale=%u\n", capacity_scale);
+	mutex_lock(&cpu_scale_mutex);
+	for_each_possible_cpu(cpu) {
+		pr_debug("cpu_capacity: cpu=%d raw_capacity=%u\n",
+			 cpu, raw_capacity[cpu]);
+		capacity = (raw_capacity[cpu] << SCHED_CAPACITY_SHIFT)
+			/ capacity_scale;
+		set_capacity_scale(cpu, capacity);
+		pr_debug("cpu_capacity: CPU%d cpu_capacity=%lu\n",
+			cpu, arch_scale_cpu_capacity(NULL, cpu));
+	}
+	mutex_unlock(&cpu_scale_mutex);
+}
+
+#ifdef CONFIG_CPU_FREQ
+static cpumask_var_t cpus_to_visit;
+static bool cap_parsing_done;
+static void parsing_done_workfn(struct work_struct *work);
+static DECLARE_WORK(parsing_done_work, parsing_done_workfn);
+
+static int
+init_cpu_capacity_callback(struct notifier_block *nb,
+			   unsigned long val,
+			   void *data)
+{
+	struct cpufreq_policy *policy = data;
+	int cpu;
+
+	if (cap_parsing_failed || cap_parsing_done)
+		return 0;
+
+	switch (val) {
+	case CPUFREQ_NOTIFY:
+		pr_debug("cpu_capacity: init cpu capacity for CPUs [%*pbl] (to_visit=%*pbl)\n",
+				cpumask_pr_args(policy->related_cpus),
+				cpumask_pr_args(cpus_to_visit));
+		cpumask_andnot(cpus_to_visit,
+			       cpus_to_visit,
+			       policy->related_cpus);
+		for_each_cpu(cpu, policy->related_cpus) {
+			raw_capacity[cpu] = arch_scale_cpu_capacity(NULL, cpu) *
+					    policy->cpuinfo.max_freq / 1000UL;
+			capacity_scale = max(raw_capacity[cpu], capacity_scale);
+		}
+		if (cpumask_empty(cpus_to_visit)) {
+			normalize_cpu_capacity();
+			kfree(raw_capacity);
+			pr_debug("cpu_capacity: parsing done\n");
+			cap_parsing_done = true;
+			schedule_work(&parsing_done_work);
+		}
+	}
+	return 0;
+}
+
+static struct notifier_block init_cpu_capacity_notifier = {
+	.notifier_call = init_cpu_capacity_callback,
+};
+
+static int __init register_cpufreq_notifier(void)
+{
+	if (cap_parsing_failed)
+		return -EINVAL;
+
+	if (!alloc_cpumask_var(&cpus_to_visit, GFP_KERNEL)) {
+		pr_err("cpu_capacity: failed to allocate memory for cpus_to_visit\n");
+		return -ENOMEM;
+	}
+	cpumask_copy(cpus_to_visit, cpu_possible_mask);
+
+	return cpufreq_register_notifier(&init_cpu_capacity_notifier,
+					 CPUFREQ_POLICY_NOTIFIER);
+}
+core_initcall(register_cpufreq_notifier);
+
+static void parsing_done_workfn(struct work_struct *work)
+{
+	cpufreq_unregister_notifier(&init_cpu_capacity_notifier,
+					 CPUFREQ_POLICY_NOTIFIER);
+}
+
+#else
+static int __init free_raw_capacity(void)
+{
+	kfree(raw_capacity);
+
+	return 0;
+}
+core_initcall(free_raw_capacity);
+#endif
+
 static int __init get_cpu_for_node(struct device_node *node)
 {
 	struct device_node *cpu_node;
@@ -34,6 +250,7 @@ static int __init get_cpu_for_node(struct device_node *node)
 
 	for_each_possible_cpu(cpu) {
 		if (of_get_cpu_node(cpu, NULL) == cpu_node) {
+			parse_cpu_capacity(cpu_node, cpu);
 			of_node_put(cpu_node);
 			return cpu;
 		}
@@ -178,13 +395,17 @@ static int __init parse_dt_topology(void)
 	 * cluster with restricted subnodes.
 	 */
 	map = of_get_child_by_name(cn, "cpu-map");
-	if (!map)
+	if (!map) {
+		cap_parsing_failed = true;
 		goto out;
+	}
 
 	ret = parse_cluster(map, 0);
 	if (ret != 0)
 		goto out_map;
 
+	normalize_cpu_capacity();
+
 	/*
 	 * Check that all cores are in the topology; the SMP code will
 	 * only mark cores described in the DT as possible.
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index c9986b3e0a96..5b830be79c01 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -38,6 +38,7 @@
 #include <asm/esr.h>
 #include <asm/insn.h>
 #include <asm/traps.h>
+#include <asm/stack_pointer.h>
 #include <asm/stacktrace.h>
 #include <asm/exception.h>
 #include <asm/system_misc.h>
@@ -147,6 +148,9 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
 	if (!tsk)
 		tsk = current;
 
+	if (!try_get_task_stack(tsk))
+		return;
+
 	/*
 	 * Switching between stacks is valid when tracing current and in
 	 * non-preemptible context.
@@ -212,6 +216,8 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
 				 stack + sizeof(struct pt_regs));
 		}
 	}
+
+	put_task_stack(tsk);
 }
 
 void show_stack(struct task_struct *tsk, unsigned long *sp)
@@ -227,10 +233,9 @@ void show_stack(struct task_struct *tsk, unsigned long *sp)
 #endif
 #define S_SMP " SMP"
 
-static int __die(const char *str, int err, struct thread_info *thread,
-		 struct pt_regs *regs)
+static int __die(const char *str, int err, struct pt_regs *regs)
 {
-	struct task_struct *tsk = thread->task;
+	struct task_struct *tsk = current;
 	static int die_counter;
 	int ret;
 
@@ -245,7 +250,8 @@ static int __die(const char *str, int err, struct thread_info *thread,
 	print_modules();
 	__show_regs(regs);
 	pr_emerg("Process %.*s (pid: %d, stack limit = 0x%p)\n",
-		 TASK_COMM_LEN, tsk->comm, task_pid_nr(tsk), thread + 1);
+		 TASK_COMM_LEN, tsk->comm, task_pid_nr(tsk),
+		 end_of_stack(tsk));
 
 	if (!user_mode(regs)) {
 		dump_mem(KERN_EMERG, "Stack: ", regs->sp,
@@ -264,7 +270,6 @@ static DEFINE_RAW_SPINLOCK(die_lock);
  */
 void die(const char *str, struct pt_regs *regs, int err)
 {
-	struct thread_info *thread = current_thread_info();
 	int ret;
 
 	oops_enter();
@@ -272,9 +277,9 @@ void die(const char *str, struct pt_regs *regs, int err)
 	raw_spin_lock_irq(&die_lock);
 	console_verbose();
 	bust_spinlocks(1);
-	ret = __die(str, err, thread, regs);
+	ret = __die(str, err, regs);
 
-	if (regs && kexec_should_crash(thread->task))
+	if (regs && kexec_should_crash(current))
 		crash_kexec(regs);
 
 	bust_spinlocks(0);
@@ -435,9 +440,10 @@ int cpu_enable_cache_maint_trap(void *__unused)
 }
 
 #define __user_cache_maint(insn, address, res)			\
-	if (untagged_addr(address) >= user_addr_max())		\
+	if (untagged_addr(address) >= user_addr_max()) {	\
 		res = -EFAULT;					\
-	else							\
+	} else {						\
+		uaccess_ttbr0_enable();				\
 		asm volatile (					\
 			"1:	" insn ", %1\n"			\
 			"	mov	%w0, #0\n"		\
@@ -449,7 +455,9 @@ int cpu_enable_cache_maint_trap(void *__unused)
 			"	.popsection\n"			\
 			_ASM_EXTABLE(1b, 3b)			\
 			: "=r" (res)				\
-			: "r" (address), "i" (-EFAULT) )
+			: "r" (address), "i" (-EFAULT));	\
+		uaccess_ttbr0_disable();			\
+	}
 
 static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs)
 {
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 1105aab1e6d6..b8deffa9e1bf 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -216,6 +216,11 @@ SECTIONS
 	swapper_pg_dir = .;
 	. += SWAPPER_DIR_SIZE;
 
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+	reserved_ttbr0 = .;
+	. += RESERVED_TTBR0_SIZE;
+#endif
+
 	_end = .;
 
 	STABS_DEBUG
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index 6eaf12c1d627..52cb7ad9b2fd 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -16,9 +16,6 @@ menuconfig VIRTUALIZATION
 
 if VIRTUALIZATION
 
-config KVM_ARM_VGIC_V3_ITS
-	bool
-
 config KVM
 	bool "Kernel-based Virtual Machine (KVM) support"
 	depends on OF
@@ -34,7 +31,6 @@ config KVM
 	select KVM_VFIO
 	select HAVE_KVM_EVENTFD
 	select HAVE_KVM_IRQFD
-	select KVM_ARM_VGIC_V3_ITS
 	select KVM_ARM_PMU if HW_PERF_EVENTS
 	select HAVE_KVM_MSI
 	select HAVE_KVM_IRQCHIP
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index a204adf29f0a..1bfe30dfbfe7 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -57,6 +57,16 @@ static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 	return 1;
 }
 
+/*
+ * Guest access to FP/ASIMD registers are routed to this handler only
+ * when the system doesn't support FP/ASIMD.
+ */
+static int handle_no_fpsimd(struct kvm_vcpu *vcpu, struct kvm_run *run)
+{
+	kvm_inject_undefined(vcpu);
+	return 1;
+}
+
 /**
  * kvm_handle_wfx - handle a wait-for-interrupts or wait-for-event
  *		    instruction executed by a guest
@@ -144,6 +154,7 @@ static exit_handle_fn arm_exit_handlers[] = {
 	[ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug,
 	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
 	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
+	[ESR_ELx_EC_FP_ASIMD]	= handle_no_fpsimd,
 };
 
 static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index 4e92399f7105..5e9052f087f2 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -106,9 +106,16 @@ el1_trap:
 	 * x0: ESR_EC
 	 */
 
-	/* Guest accessed VFP/SIMD registers, save host, restore Guest */
+	/*
+	 * We trap the first access to the FP/SIMD to save the host context
+	 * and restore the guest context lazily.
+	 * If FP/SIMD is not implemented, handle the trap and inject an
+	 * undefined instruction exception to the guest.
+	 */
+alternative_if_not ARM64_HAS_NO_FPSIMD
 	cmp	x0, #ESR_ELx_EC_FP_ASIMD
 	b.eq	__fpsimd_guest_restore
+alternative_else_nop_endif
 
 	mrs	x1, tpidr_el2
 	mov	x0, #ARM_EXCEPTION_TRAP
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 83037cd62d01..75e83dd40d43 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -21,6 +21,7 @@
 #include <asm/kvm_asm.h>
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_hyp.h>
+#include <asm/fpsimd.h>
 
 static bool __hyp_text __fpsimd_enabled_nvhe(void)
 {
@@ -76,16 +77,24 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
 	 * traps are only taken to EL2 if the operation would not otherwise
 	 * trap to EL1.  Therefore, always make sure that for 32-bit guests,
 	 * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
+	 * If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to
+	 * it will cause an exception.
 	 */
 	val = vcpu->arch.hcr_el2;
-	if (!(val & HCR_RW)) {
+	if (!(val & HCR_RW) && system_supports_fpsimd()) {
 		write_sysreg(1 << 30, fpexc32_el2);
 		isb();
 	}
 	write_sysreg(val, hcr_el2);
 	/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
 	write_sysreg(1 << 15, hstr_el2);
-	/* Make sure we trap PMU access from EL0 to EL2 */
+	/*
+	 * Make sure we trap PMU access from EL0 to EL2. Also sanitize
+	 * PMSELR_EL0 to make sure it never contains the cycle
+	 * counter, which could make a PMXEVCNTR_EL0 access UNDEF at
+	 * EL1 instead of being trapped to EL2.
+	 */
+	write_sysreg(0, pmselr_el0);
 	write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
 	write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
 	__activate_traps_arch()();
diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
index 5bc460884639..e95d4f68bf54 100644
--- a/arch/arm64/kvm/reset.c
+++ b/arch/arm64/kvm/reset.c
@@ -86,12 +86,6 @@ int kvm_arch_dev_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_VCPU_ATTRIBUTES:
 		r = 1;
 		break;
-	case KVM_CAP_MSI_DEVID:
-		if (!kvm)
-			r = -EINVAL;
-		else
-			r = kvm->arch.vgic.msis_require_devid;
-		break;
 	default:
 		r = 0;
 	}
diff --git a/arch/arm64/lib/clear_user.S b/arch/arm64/lib/clear_user.S
index 5d1cad3ce6d6..d7150e30438a 100644
--- a/arch/arm64/lib/clear_user.S
+++ b/arch/arm64/lib/clear_user.S
@@ -17,10 +17,7 @@
  */
 #include <linux/linkage.h>
 
-#include <asm/alternative.h>
-#include <asm/assembler.h>
-#include <asm/cpufeature.h>
-#include <asm/sysreg.h>
+#include <asm/uaccess.h>
 
 	.text
 
@@ -33,8 +30,7 @@
  * Alignment fixed up by hardware.
  */
 ENTRY(__clear_user)
-ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \
-	    CONFIG_ARM64_PAN)
+	uaccess_enable_not_uao x2, x3
 	mov	x2, x1			// save the size for fixup return
 	subs	x1, x1, #8
 	b.mi	2f
@@ -54,8 +50,7 @@ uao_user_alternative 9f, strh, sttrh, wzr, x0, 2
 	b.mi	5f
 uao_user_alternative 9f, strb, sttrb, wzr, x0, 0
 5:	mov	x0, #0
-ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \
-	    CONFIG_ARM64_PAN)
+	uaccess_disable_not_uao x2
 	ret
 ENDPROC(__clear_user)
 
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 4fd67ea03bb0..cfe13396085b 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -16,11 +16,8 @@
 
 #include <linux/linkage.h>
 
-#include <asm/alternative.h>
-#include <asm/assembler.h>
 #include <asm/cache.h>
-#include <asm/cpufeature.h>
-#include <asm/sysreg.h>
+#include <asm/uaccess.h>
 
 /*
  * Copy from user space to a kernel buffer (alignment handled by the hardware)
@@ -67,12 +64,10 @@
 
 end	.req	x5
 ENTRY(__arch_copy_from_user)
-ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \
-	    CONFIG_ARM64_PAN)
+	uaccess_enable_not_uao x3, x4
 	add	end, x0, x2
 #include "copy_template.S"
-ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \
-	    CONFIG_ARM64_PAN)
+	uaccess_disable_not_uao x3
 	mov	x0, #0				// Nothing to copy
 	ret
 ENDPROC(__arch_copy_from_user)
diff --git a/arch/arm64/lib/copy_in_user.S b/arch/arm64/lib/copy_in_user.S
index f7292dd08c84..718b1c4e2f85 100644
--- a/arch/arm64/lib/copy_in_user.S
+++ b/arch/arm64/lib/copy_in_user.S
@@ -18,11 +18,8 @@
 
 #include <linux/linkage.h>
 
-#include <asm/alternative.h>
-#include <asm/assembler.h>
 #include <asm/cache.h>
-#include <asm/cpufeature.h>
-#include <asm/sysreg.h>
+#include <asm/uaccess.h>
 
 /*
  * Copy from user space to user space (alignment handled by the hardware)
@@ -68,12 +65,10 @@
 
 end	.req	x5
 ENTRY(__copy_in_user)
-ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \
-	    CONFIG_ARM64_PAN)
+	uaccess_enable_not_uao x3, x4
 	add	end, x0, x2
 #include "copy_template.S"
-ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \
-	    CONFIG_ARM64_PAN)
+	uaccess_disable_not_uao x3
 	mov	x0, #0
 	ret
 ENDPROC(__copy_in_user)
diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
index 7a7efe255034..e99e31c9acac 100644
--- a/arch/arm64/lib/copy_to_user.S
+++ b/arch/arm64/lib/copy_to_user.S
@@ -16,11 +16,8 @@
 
 #include <linux/linkage.h>
 
-#include <asm/alternative.h>
-#include <asm/assembler.h>
 #include <asm/cache.h>
-#include <asm/cpufeature.h>
-#include <asm/sysreg.h>
+#include <asm/uaccess.h>
 
 /*
  * Copy to user space from a kernel buffer (alignment handled by the hardware)
@@ -66,12 +63,10 @@
 
 end	.req	x5
 ENTRY(__arch_copy_to_user)
-ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_ALT_PAN_NOT_UAO, \
-	    CONFIG_ARM64_PAN)
+	uaccess_enable_not_uao x3, x4
 	add	end, x0, x2
 #include "copy_template.S"
-ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_ALT_PAN_NOT_UAO, \
-	    CONFIG_ARM64_PAN)
+	uaccess_disable_not_uao x3
 	mov	x0, #0
 	ret
 ENDPROC(__arch_copy_to_user)
diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
index 54bb209cae8e..e703fb9defad 100644
--- a/arch/arm64/mm/Makefile
+++ b/arch/arm64/mm/Makefile
@@ -3,7 +3,8 @@ obj-y				:= dma-mapping.o extable.o fault.o init.o \
 				   ioremap.o mmap.o pgd.o mmu.o \
 				   context.o proc.o pageattr.o
 obj-$(CONFIG_HUGETLB_PAGE)	+= hugetlbpage.o
-obj-$(CONFIG_ARM64_PTDUMP)	+= dump.o
+obj-$(CONFIG_ARM64_PTDUMP_CORE)	+= dump.o
+obj-$(CONFIG_ARM64_PTDUMP_DEBUGFS)	+= ptdump_debugfs.o
 obj-$(CONFIG_NUMA)		+= numa.o
 
 obj-$(CONFIG_KASAN)		+= kasan_init.o
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index 58b5a906ff78..da9576932322 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -23,6 +23,7 @@
 #include <asm/assembler.h>
 #include <asm/cpufeature.h>
 #include <asm/alternative.h>
+#include <asm/uaccess.h>
 
 /*
  *	flush_icache_range(start,end)
@@ -48,6 +49,7 @@ ENTRY(flush_icache_range)
  *	- end     - virtual end address of region
  */
 ENTRY(__flush_cache_user_range)
+	uaccess_ttbr0_enable x2, x3
 	dcache_line_size x2, x3
 	sub	x3, x2, #1
 	bic	x4, x0, x3
@@ -69,10 +71,12 @@ USER(9f, ic	ivau, x4	)		// invalidate I line PoU
 	dsb	ish
 	isb
 	mov	x0, #0
+1:
+	uaccess_ttbr0_disable x1
 	ret
 9:
 	mov	x0, #-EFAULT
-	ret
+	b	1b
 ENDPROC(flush_icache_range)
 ENDPROC(__flush_cache_user_range)
 
diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
index efcf1f7ef1e4..4c63cb154859 100644
--- a/arch/arm64/mm/context.c
+++ b/arch/arm64/mm/context.c
@@ -221,7 +221,12 @@ void check_and_switch_context(struct mm_struct *mm, unsigned int cpu)
 	raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
 
 switch_mm_fastpath:
-	cpu_switch_mm(mm->pgd, mm);
+	/*
+	 * Defer TTBR0_EL1 setting for user threads to uaccess_enable() when
+	 * emulating PAN.
+	 */
+	if (!system_uses_ttbr0_pan())
+		cpu_switch_mm(mm->pgd, mm);
 }
 
 static int asids_init(void)
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index 3f74d0d98de6..290a84f3351f 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -796,6 +796,8 @@ static struct dma_map_ops iommu_dma_ops = {
 	.sync_single_for_device = __iommu_sync_single_for_device,
 	.sync_sg_for_cpu = __iommu_sync_sg_for_cpu,
 	.sync_sg_for_device = __iommu_sync_sg_for_device,
+	.map_resource = iommu_dma_map_resource,
+	.unmap_resource = iommu_dma_unmap_resource,
 	.dma_supported = iommu_dma_supported,
 	.mapping_error = iommu_dma_mapping_error,
 };
@@ -938,11 +940,6 @@ static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
 
 void arch_teardown_dma_ops(struct device *dev)
 {
-	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
-
-	if (WARN_ON(domain))
-		iommu_detach_device(domain, dev);
-
 	dev->archdata.dma_ops = NULL;
 }
 
diff --git a/arch/arm64/mm/dump.c b/arch/arm64/mm/dump.c
index 9c3e75df2180..ca74a2aace42 100644
--- a/arch/arm64/mm/dump.c
+++ b/arch/arm64/mm/dump.c
@@ -50,6 +50,18 @@ static const struct addr_marker address_markers[] = {
 	{ -1,				NULL },
 };
 
+#define pt_dump_seq_printf(m, fmt, args...)	\
+({						\
+	if (m)					\
+		seq_printf(m, fmt, ##args);	\
+})
+
+#define pt_dump_seq_puts(m, fmt)	\
+({					\
+	if (m)				\
+		seq_printf(m, fmt);	\
+})
+
 /*
  * The page dumper groups page table entries of the same type into a single
  * description. It uses pg_state to track the range information while
@@ -62,6 +74,9 @@ struct pg_state {
 	unsigned long start_address;
 	unsigned level;
 	u64 current_prot;
+	bool check_wx;
+	unsigned long wx_pages;
+	unsigned long uxn_pages;
 };
 
 struct prot_bits {
@@ -186,10 +201,39 @@ static void dump_prot(struct pg_state *st, const struct prot_bits *bits,
 			s = bits->clear;
 
 		if (s)
-			seq_printf(st->seq, " %s", s);
+			pt_dump_seq_printf(st->seq, " %s", s);
 	}
 }
 
+static void note_prot_uxn(struct pg_state *st, unsigned long addr)
+{
+	if (!st->check_wx)
+		return;
+
+	if ((st->current_prot & PTE_UXN) == PTE_UXN)
+		return;
+
+	WARN_ONCE(1, "arm64/mm: Found non-UXN mapping at address %p/%pS\n",
+		  (void *)st->start_address, (void *)st->start_address);
+
+	st->uxn_pages += (addr - st->start_address) / PAGE_SIZE;
+}
+
+static void note_prot_wx(struct pg_state *st, unsigned long addr)
+{
+	if (!st->check_wx)
+		return;
+	if ((st->current_prot & PTE_RDONLY) == PTE_RDONLY)
+		return;
+	if ((st->current_prot & PTE_PXN) == PTE_PXN)
+		return;
+
+	WARN_ONCE(1, "arm64/mm: Found insecure W+X mapping at address %p/%pS\n",
+		  (void *)st->start_address, (void *)st->start_address);
+
+	st->wx_pages += (addr - st->start_address) / PAGE_SIZE;
+}
+
 static void note_page(struct pg_state *st, unsigned long addr, unsigned level,
 				u64 val)
 {
@@ -200,14 +244,16 @@ static void note_page(struct pg_state *st, unsigned long addr, unsigned level,
 		st->level = level;
 		st->current_prot = prot;
 		st->start_address = addr;
-		seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+		pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
 	} else if (prot != st->current_prot || level != st->level ||
 		   addr >= st->marker[1].start_address) {
 		const char *unit = units;
 		unsigned long delta;
 
 		if (st->current_prot) {
-			seq_printf(st->seq, "0x%016lx-0x%016lx   ",
+			note_prot_uxn(st, addr);
+			note_prot_wx(st, addr);
+			pt_dump_seq_printf(st->seq, "0x%016lx-0x%016lx   ",
 				   st->start_address, addr);
 
 			delta = (addr - st->start_address) >> 10;
@@ -215,17 +261,17 @@ static void note_page(struct pg_state *st, unsigned long addr, unsigned level,
 				delta >>= 10;
 				unit++;
 			}
-			seq_printf(st->seq, "%9lu%c %s", delta, *unit,
+			pt_dump_seq_printf(st->seq, "%9lu%c %s", delta, *unit,
 				   pg_level[st->level].name);
 			if (pg_level[st->level].bits)
 				dump_prot(st, pg_level[st->level].bits,
 					  pg_level[st->level].num);
-			seq_puts(st->seq, "\n");
+			pt_dump_seq_puts(st->seq, "\n");
 		}
 
 		if (addr >= st->marker[1].start_address) {
 			st->marker++;
-			seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+			pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
 		}
 
 		st->start_address = addr;
@@ -235,7 +281,7 @@ static void note_page(struct pg_state *st, unsigned long addr, unsigned level,
 
 	if (addr >= st->marker[1].start_address) {
 		st->marker++;
-		seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+		pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
 	}
 
 }
@@ -304,9 +350,8 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm,
 	}
 }
 
-static int ptdump_show(struct seq_file *m, void *v)
+void ptdump_walk_pgd(struct seq_file *m, struct ptdump_info *info)
 {
-	struct ptdump_info *info = m->private;
 	struct pg_state st = {
 		.seq = m,
 		.marker = info->markers,
@@ -315,33 +360,16 @@ static int ptdump_show(struct seq_file *m, void *v)
 	walk_pgd(&st, info->mm, info->base_addr);
 
 	note_page(&st, 0, 0, 0);
-	return 0;
 }
 
-static int ptdump_open(struct inode *inode, struct file *file)
+static void ptdump_initialize(void)
 {
-	return single_open(file, ptdump_show, inode->i_private);
-}
-
-static const struct file_operations ptdump_fops = {
-	.open		= ptdump_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-int ptdump_register(struct ptdump_info *info, const char *name)
-{
-	struct dentry *pe;
 	unsigned i, j;
 
 	for (i = 0; i < ARRAY_SIZE(pg_level); i++)
 		if (pg_level[i].bits)
 			for (j = 0; j < pg_level[i].num; j++)
 				pg_level[i].mask |= pg_level[i].bits[j].mask;
-
-	pe = debugfs_create_file(name, 0400, NULL, info, &ptdump_fops);
-	return pe ? 0 : -ENOMEM;
 }
 
 static struct ptdump_info kernel_ptdump_info = {
@@ -350,8 +378,30 @@ static struct ptdump_info kernel_ptdump_info = {
 	.base_addr	= VA_START,
 };
 
+void ptdump_check_wx(void)
+{
+	struct pg_state st = {
+		.seq = NULL,
+		.marker = (struct addr_marker[]) {
+			{ 0, NULL},
+			{ -1, NULL},
+		},
+		.check_wx = true,
+	};
+
+	walk_pgd(&st, &init_mm, 0);
+	note_page(&st, 0, 0, 0);
+	if (st.wx_pages || st.uxn_pages)
+		pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found, %lu non-UXN pages found\n",
+			st.wx_pages, st.uxn_pages);
+	else
+		pr_info("Checked W+X mappings: passed, no W+X pages found\n");
+}
+
 static int ptdump_init(void)
 {
-	return ptdump_register(&kernel_ptdump_info, "kernel_page_tables");
+	ptdump_initialize();
+	return ptdump_debugfs_register(&kernel_ptdump_info,
+					"kernel_page_tables");
 }
 device_initcall(ptdump_init);
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 0f8788374815..a78a5c401806 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -269,13 +269,19 @@ out:
 	return fault;
 }
 
-static inline bool is_permission_fault(unsigned int esr)
+static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs)
 {
 	unsigned int ec       = ESR_ELx_EC(esr);
 	unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;
 
-	return (ec == ESR_ELx_EC_DABT_CUR && fsc_type == ESR_ELx_FSC_PERM) ||
-	       (ec == ESR_ELx_EC_IABT_CUR && fsc_type == ESR_ELx_FSC_PERM);
+	if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR)
+		return false;
+
+	if (system_uses_ttbr0_pan())
+		return fsc_type == ESR_ELx_FSC_FAULT &&
+			(regs->pstate & PSR_PAN_BIT);
+	else
+		return fsc_type == ESR_ELx_FSC_PERM;
 }
 
 static bool is_el0_instruction_abort(unsigned int esr)
@@ -315,7 +321,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
 		mm_flags |= FAULT_FLAG_WRITE;
 	}
 
-	if (is_permission_fault(esr) && (addr < USER_DS)) {
+	if (addr < USER_DS && is_permission_fault(esr, regs)) {
 		/* regs->orig_addr_limit may be 0 if we entered from EL0 */
 		if (regs->orig_addr_limit == KERNEL_DS)
 			die("Accessing user space memory with fs=KERNEL_DS", regs, esr);
@@ -507,10 +513,10 @@ static const struct fault_info {
 	{ do_bad,		SIGBUS,  0,		"unknown 17"			},
 	{ do_bad,		SIGBUS,  0,		"unknown 18"			},
 	{ do_bad,		SIGBUS,  0,		"unknown 19"			},
-	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
-	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
-	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
-	{ do_bad,		SIGBUS,  0,		"synchronous abort (translation table walk)" },
+	{ do_bad,		SIGBUS,  0,		"synchronous external abort (translation table walk)" },
+	{ do_bad,		SIGBUS,  0,		"synchronous external abort (translation table walk)" },
+	{ do_bad,		SIGBUS,  0,		"synchronous external abort (translation table walk)" },
+	{ do_bad,		SIGBUS,  0,		"synchronous external abort (translation table walk)" },
 	{ do_bad,		SIGBUS,  0,		"synchronous parity error"	},
 	{ do_bad,		SIGBUS,  0,		"unknown 25"			},
 	{ do_bad,		SIGBUS,  0,		"unknown 26"			},
diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c
index 8377329d8c97..554a2558c12e 100644
--- a/arch/arm64/mm/flush.c
+++ b/arch/arm64/mm/flush.c
@@ -25,14 +25,7 @@
 #include <asm/cachetype.h>
 #include <asm/tlbflush.h>
 
-void flush_cache_range(struct vm_area_struct *vma, unsigned long start,
-		       unsigned long end)
-{
-	if (vma->vm_flags & VM_EXEC)
-		__flush_icache_all();
-}
-
-static void sync_icache_aliases(void *kaddr, unsigned long len)
+void sync_icache_aliases(void *kaddr, unsigned long len)
 {
 	unsigned long addr = (unsigned long)kaddr;
 
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index 2e49bd252fe7..964b7549af5c 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -51,20 +51,8 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr,
 	*pgsize = PAGE_SIZE;
 	if (!pte_cont(pte))
 		return 1;
-	if (!pgd_present(*pgd)) {
-		VM_BUG_ON(!pgd_present(*pgd));
-		return 1;
-	}
 	pud = pud_offset(pgd, addr);
-	if (!pud_present(*pud)) {
-		VM_BUG_ON(!pud_present(*pud));
-		return 1;
-	}
 	pmd = pmd_offset(pud, addr);
-	if (!pmd_present(*pmd)) {
-		VM_BUG_ON(!pmd_present(*pmd));
-		return 1;
-	}
 	if ((pte_t *)pmd == ptep) {
 		*pgsize = PMD_SIZE;
 		return CONT_PMDS;
@@ -212,7 +200,7 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
 		ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize);
 		/* save the 1st pte to return */
 		pte = ptep_get_and_clear(mm, addr, cpte);
-		for (i = 1; i < ncontig; ++i) {
+		for (i = 1, addr += pgsize; i < ncontig; ++i, addr += pgsize) {
 			/*
 			 * If HW_AFDBM is enabled, then the HW could
 			 * turn on the dirty bit for any of the page
@@ -250,7 +238,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,
 		pfn = pte_pfn(*cpte);
 		ncontig = find_num_contig(vma->vm_mm, addr, cpte,
 					  *cpte, &pgsize);
-		for (i = 0; i < ncontig; ++i, ++cpte) {
+		for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize) {
 			changed = ptep_set_access_flags(vma, addr, cpte,
 							pfn_pte(pfn,
 								hugeprot),
@@ -273,7 +261,7 @@ void huge_ptep_set_wrprotect(struct mm_struct *mm,
 
 		cpte = huge_pte_offset(mm, addr);
 		ncontig = find_num_contig(mm, addr, cpte, *cpte, &pgsize);
-		for (i = 0; i < ncontig; ++i, ++cpte)
+		for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize)
 			ptep_set_wrprotect(mm, addr, cpte);
 	} else {
 		ptep_set_wrprotect(mm, addr, ptep);
@@ -291,7 +279,7 @@ void huge_ptep_clear_flush(struct vm_area_struct *vma,
 		cpte = huge_pte_offset(vma->vm_mm, addr);
 		ncontig = find_num_contig(vma->vm_mm, addr, cpte,
 					  *cpte, &pgsize);
-		for (i = 0; i < ncontig; ++i, ++cpte)
+		for (i = 0; i < ncontig; ++i, ++cpte, addr += pgsize)
 			ptep_clear_flush(vma, addr, cpte);
 	} else {
 		ptep_clear_flush(vma, addr, ptep);
@@ -323,7 +311,7 @@ __setup("hugepagesz=", setup_hugepagesz);
 static __init int add_default_hugepagesz(void)
 {
 	if (size_to_hstate(CONT_PTES * PAGE_SIZE) == NULL)
-		hugetlb_add_hstate(CONT_PMD_SHIFT);
+		hugetlb_add_hstate(CONT_PTE_SHIFT);
 	return 0;
 }
 arch_initcall(add_default_hugepagesz);
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 05615a3fdc6f..17243e43184e 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -28,8 +28,6 @@
 #include <linux/memblock.h>
 #include <linux/fs.h>
 #include <linux/io.h>
-#include <linux/slab.h>
-#include <linux/stop_machine.h>
 
 #include <asm/barrier.h>
 #include <asm/cputype.h>
@@ -42,6 +40,7 @@
 #include <asm/tlb.h>
 #include <asm/memblock.h>
 #include <asm/mmu_context.h>
+#include <asm/ptdump.h>
 
 u64 idmap_t0sz = TCR_T0SZ(VA_BITS);
 
@@ -95,11 +94,24 @@ static phys_addr_t __init early_pgtable_alloc(void)
 	return phys;
 }
 
+static bool pgattr_change_is_safe(u64 old, u64 new)
+{
+	/*
+	 * The following mapping attributes may be updated in live
+	 * kernel mappings without the need for break-before-make.
+	 */
+	static const pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE;
+
+	return old  == 0 || new  == 0 || ((old ^ new) & ~mask) == 0;
+}
+
 static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
 				  unsigned long end, unsigned long pfn,
 				  pgprot_t prot,
-				  phys_addr_t (*pgtable_alloc)(void))
+				  phys_addr_t (*pgtable_alloc)(void),
+				  bool page_mappings_only)
 {
+	pgprot_t __prot = prot;
 	pte_t *pte;
 
 	BUG_ON(pmd_sect(*pmd));
@@ -115,8 +127,28 @@ static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
 
 	pte = pte_set_fixmap_offset(pmd, addr);
 	do {
-		set_pte(pte, pfn_pte(pfn, prot));
+		pte_t old_pte = *pte;
+
+		/*
+		 * Set the contiguous bit for the subsequent group of PTEs if
+		 * its size and alignment are appropriate.
+		 */
+		if (((addr | PFN_PHYS(pfn)) & ~CONT_PTE_MASK) == 0) {
+			if (end - addr >= CONT_PTE_SIZE && !page_mappings_only)
+				__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
+			else
+				__prot = prot;
+		}
+
+		set_pte(pte, pfn_pte(pfn, __prot));
 		pfn++;
+
+		/*
+		 * After the PTE entry has been populated once, we
+		 * only allow updates to the permission attributes.
+		 */
+		BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), pte_val(*pte)));
+
 	} while (pte++, addr += PAGE_SIZE, addr != end);
 
 	pte_clear_fixmap();
@@ -125,8 +157,9 @@ static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
 static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
 				  phys_addr_t phys, pgprot_t prot,
 				  phys_addr_t (*pgtable_alloc)(void),
-				  bool allow_block_mappings)
+				  bool page_mappings_only)
 {
+	pgprot_t __prot = prot;
 	pmd_t *pmd;
 	unsigned long next;
 
@@ -146,27 +179,39 @@ static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
 
 	pmd = pmd_set_fixmap_offset(pud, addr);
 	do {
+		pmd_t old_pmd = *pmd;
+
 		next = pmd_addr_end(addr, end);
+
 		/* try section mapping first */
 		if (((addr | next | phys) & ~SECTION_MASK) == 0 &&
-		      allow_block_mappings) {
-			pmd_t old_pmd =*pmd;
-			pmd_set_huge(pmd, phys, prot);
+		      !page_mappings_only) {
 			/*
-			 * Check for previous table entries created during
-			 * boot (__create_page_tables) and flush them.
+			 * Set the contiguous bit for the subsequent group of
+			 * PMDs if its size and alignment are appropriate.
 			 */
-			if (!pmd_none(old_pmd)) {
-				flush_tlb_all();
-				if (pmd_table(old_pmd)) {
-					phys_addr_t table = pmd_page_paddr(old_pmd);
-					if (!WARN_ON_ONCE(slab_is_available()))
-						memblock_free(table, PAGE_SIZE);
-				}
+			if (((addr | phys) & ~CONT_PMD_MASK) == 0) {
+				if (end - addr >= CONT_PMD_SIZE)
+					__prot = __pgprot(pgprot_val(prot) |
+							  PTE_CONT);
+				else
+					__prot = prot;
 			}
+			pmd_set_huge(pmd, phys, __prot);
+
+			/*
+			 * After the PMD entry has been populated once, we
+			 * only allow updates to the permission attributes.
+			 */
+			BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd),
+						      pmd_val(*pmd)));
 		} else {
 			alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
-				       prot, pgtable_alloc);
+				       prot, pgtable_alloc,
+				       page_mappings_only);
+
+			BUG_ON(pmd_val(old_pmd) != 0 &&
+			       pmd_val(old_pmd) != pmd_val(*pmd));
 		}
 		phys += next - addr;
 	} while (pmd++, addr = next, addr != end);
@@ -189,7 +234,7 @@ static inline bool use_1G_block(unsigned long addr, unsigned long next,
 static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
 				  phys_addr_t phys, pgprot_t prot,
 				  phys_addr_t (*pgtable_alloc)(void),
-				  bool allow_block_mappings)
+				  bool page_mappings_only)
 {
 	pud_t *pud;
 	unsigned long next;
@@ -204,33 +249,28 @@ static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
 
 	pud = pud_set_fixmap_offset(pgd, addr);
 	do {
+		pud_t old_pud = *pud;
+
 		next = pud_addr_end(addr, end);
 
 		/*
 		 * For 4K granule only, attempt to put down a 1GB block
 		 */
-		if (use_1G_block(addr, next, phys) && allow_block_mappings) {
-			pud_t old_pud = *pud;
+		if (use_1G_block(addr, next, phys) && !page_mappings_only) {
 			pud_set_huge(pud, phys, prot);
 
 			/*
-			 * If we have an old value for a pud, it will
-			 * be pointing to a pmd table that we no longer
-			 * need (from swapper_pg_dir).
-			 *
-			 * Look up the old pmd table and free it.
+			 * After the PUD entry has been populated once, we
+			 * only allow updates to the permission attributes.
 			 */
-			if (!pud_none(old_pud)) {
-				flush_tlb_all();
-				if (pud_table(old_pud)) {
-					phys_addr_t table = pud_page_paddr(old_pud);
-					if (!WARN_ON_ONCE(slab_is_available()))
-						memblock_free(table, PAGE_SIZE);
-				}
-			}
+			BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
+						      pud_val(*pud)));
 		} else {
 			alloc_init_pmd(pud, addr, next, phys, prot,
-				       pgtable_alloc, allow_block_mappings);
+				       pgtable_alloc, page_mappings_only);
+
+			BUG_ON(pud_val(old_pud) != 0 &&
+			       pud_val(old_pud) != pud_val(*pud));
 		}
 		phys += next - addr;
 	} while (pud++, addr = next, addr != end);
@@ -242,7 +282,7 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
 				 unsigned long virt, phys_addr_t size,
 				 pgprot_t prot,
 				 phys_addr_t (*pgtable_alloc)(void),
-				 bool allow_block_mappings)
+				 bool page_mappings_only)
 {
 	unsigned long addr, length, end, next;
 	pgd_t *pgd = pgd_offset_raw(pgdir, virt);
@@ -262,7 +302,7 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
 	do {
 		next = pgd_addr_end(addr, end);
 		alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc,
-			       allow_block_mappings);
+			       page_mappings_only);
 		phys += next - addr;
 	} while (pgd++, addr = next, addr != end);
 }
@@ -291,17 +331,17 @@ static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt,
 			&phys, virt);
 		return;
 	}
-	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, true);
+	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, false);
 }
 
 void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
 			       unsigned long virt, phys_addr_t size,
-			       pgprot_t prot, bool allow_block_mappings)
+			       pgprot_t prot, bool page_mappings_only)
 {
 	BUG_ON(mm == &init_mm);
 
 	__create_pgd_mapping(mm->pgd, phys, virt, size, prot,
-			     pgd_pgtable_alloc, allow_block_mappings);
+			     pgd_pgtable_alloc, page_mappings_only);
 }
 
 static void create_mapping_late(phys_addr_t phys, unsigned long virt,
@@ -314,7 +354,7 @@ static void create_mapping_late(phys_addr_t phys, unsigned long virt,
 	}
 
 	__create_pgd_mapping(init_mm.pgd, phys, virt, size, prot,
-			     NULL, !debug_pagealloc_enabled());
+			     NULL, debug_pagealloc_enabled());
 }
 
 static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end)
@@ -332,7 +372,7 @@ static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end
 		__create_pgd_mapping(pgd, start, __phys_to_virt(start),
 				     end - start, PAGE_KERNEL,
 				     early_pgtable_alloc,
-				     !debug_pagealloc_enabled());
+				     debug_pagealloc_enabled());
 		return;
 	}
 
@@ -345,13 +385,13 @@ static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end
 				     __phys_to_virt(start),
 				     kernel_start - start, PAGE_KERNEL,
 				     early_pgtable_alloc,
-				     !debug_pagealloc_enabled());
+				     debug_pagealloc_enabled());
 	if (kernel_end < end)
 		__create_pgd_mapping(pgd, kernel_end,
 				     __phys_to_virt(kernel_end),
 				     end - kernel_end, PAGE_KERNEL,
 				     early_pgtable_alloc,
-				     !debug_pagealloc_enabled());
+				     debug_pagealloc_enabled());
 
 	/*
 	 * Map the linear alias of the [_text, __init_begin) interval as
@@ -361,7 +401,7 @@ static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end
 	 */
 	__create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start),
 			     kernel_end - kernel_start, PAGE_KERNEL_RO,
-			     early_pgtable_alloc, !debug_pagealloc_enabled());
+			     early_pgtable_alloc, debug_pagealloc_enabled());
 }
 
 static void __init map_mem(pgd_t *pgd)
@@ -396,6 +436,11 @@ void mark_rodata_ro(void)
 	section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata;
 	create_mapping_late(__pa(__start_rodata), (unsigned long)__start_rodata,
 			    section_size, PAGE_KERNEL_RO);
+
+	/* flush the TLBs after updating live kernel mappings */
+	flush_tlb_all();
+
+	debug_checkwx();
 }
 
 static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
@@ -408,7 +453,7 @@ static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
 	BUG_ON(!PAGE_ALIGNED(size));
 
 	__create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot,
-			     early_pgtable_alloc, !debug_pagealloc_enabled());
+			     early_pgtable_alloc, debug_pagealloc_enabled());
 
 	vma->addr	= va_start;
 	vma->phys_addr	= pa_start;
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 352c73b6a59e..32682be978e0 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -70,11 +70,14 @@ ENTRY(cpu_do_suspend)
 	mrs	x8, mdscr_el1
 	mrs	x9, oslsr_el1
 	mrs	x10, sctlr_el1
+	mrs	x11, tpidr_el1
+	mrs	x12, sp_el0
 	stp	x2, x3, [x0]
 	stp	x4, xzr, [x0, #16]
 	stp	x5, x6, [x0, #32]
 	stp	x7, x8, [x0, #48]
 	stp	x9, x10, [x0, #64]
+	stp	x11, x12, [x0, #80]
 	ret
 ENDPROC(cpu_do_suspend)
 
@@ -90,6 +93,7 @@ ENTRY(cpu_do_resume)
 	ldp	x6, x8, [x0, #32]
 	ldp	x9, x10, [x0, #48]
 	ldp	x11, x12, [x0, #64]
+	ldp	x13, x14, [x0, #80]
 	msr	tpidr_el0, x2
 	msr	tpidrro_el0, x3
 	msr	contextidr_el1, x4
@@ -112,6 +116,8 @@ ENTRY(cpu_do_resume)
 	msr	mdscr_el1, x10
 
 	msr	sctlr_el1, x12
+	msr	tpidr_el1, x13
+	msr	sp_el0, x14
 	/*
 	 * Restore oslsr_el1 by writing oslar_el1
 	 */
@@ -136,11 +142,7 @@ ENTRY(cpu_do_switch_mm)
 	bfi	x0, x1, #48, #16		// set the ASID
 	msr	ttbr0_el1, x0			// set TTBR0
 	isb
-alternative_if ARM64_WORKAROUND_CAVIUM_27456
-	ic	iallu
-	dsb	nsh
-	isb
-alternative_else_nop_endif
+	post_ttbr0_update_workaround
 	ret
 ENDPROC(cpu_do_switch_mm)
 
diff --git a/arch/arm64/mm/ptdump_debugfs.c b/arch/arm64/mm/ptdump_debugfs.c
new file mode 100644
index 000000000000..eee4d864350c
--- /dev/null
+++ b/arch/arm64/mm/ptdump_debugfs.c
@@ -0,0 +1,31 @@
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+
+#include <asm/ptdump.h>
+
+static int ptdump_show(struct seq_file *m, void *v)
+{
+	struct ptdump_info *info = m->private;
+	ptdump_walk_pgd(m, info);
+	return 0;
+}
+
+static int ptdump_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, ptdump_show, inode->i_private);
+}
+
+static const struct file_operations ptdump_fops = {
+	.open		= ptdump_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+int ptdump_debugfs_register(struct ptdump_info *info, const char *name)
+{
+	struct dentry *pe;
+	pe = debugfs_create_file(name, 0400, NULL, info, &ptdump_fops);
+	return pe ? 0 : -ENOMEM;
+
+}
diff --git a/arch/arm64/xen/hypercall.S b/arch/arm64/xen/hypercall.S
index 329c8027b0a9..b41aff25426d 100644
--- a/arch/arm64/xen/hypercall.S
+++ b/arch/arm64/xen/hypercall.S
@@ -49,6 +49,7 @@
 
 #include <linux/linkage.h>
 #include <asm/assembler.h>
+#include <asm/uaccess.h>
 #include <xen/interface/xen.h>
 
 
@@ -91,6 +92,20 @@ ENTRY(privcmd_call)
 	mov x2, x3
 	mov x3, x4
 	mov x4, x5
+	/*
+	 * Privcmd calls are issued by the userspace. The kernel needs to
+	 * enable access to TTBR0_EL1 as the hypervisor would issue stage 1
+	 * translations to user memory via AT instructions. Since AT
+	 * instructions are not affected by the PAN bit (ARMv8.1), we only
+	 * need the explicit uaccess_enable/disable if the TTBR0 PAN emulation
+	 * is enabled (it implies that hardware UAO and PAN disabled).
+	 */
+	uaccess_ttbr0_enable x6, x7
 	hvc XEN_IMM
+
+	/*
+	 * Disable userspace access from kernel once the hyp call completed.
+	 */
+	uaccess_ttbr0_disable x6
 	ret
 ENDPROC(privcmd_call);
diff --git a/arch/avr32/mm/dma-coherent.c b/arch/avr32/mm/dma-coherent.c
index 58610d0df7ed..54534e5d0781 100644
--- a/arch/avr32/mm/dma-coherent.c
+++ b/arch/avr32/mm/dma-coherent.c
@@ -146,7 +146,8 @@ static dma_addr_t avr32_dma_map_page(struct device *dev, struct page *page,
 {
 	void *cpu_addr = page_address(page) + offset;
 
-	dma_cache_sync(dev, cpu_addr, size, direction);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		dma_cache_sync(dev, cpu_addr, size, direction);
 	return virt_to_bus(cpu_addr);
 }
 
@@ -162,6 +163,10 @@ static int avr32_dma_map_sg(struct device *dev, struct scatterlist *sglist,
 
 		sg->dma_address = page_to_bus(sg_page(sg)) + sg->offset;
 		virt = sg_virt(sg);
+
+		if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+			continue;
+
 		dma_cache_sync(dev, virt, sg->length, direction);
 	}
 
diff --git a/arch/blackfin/kernel/dma-mapping.c b/arch/blackfin/kernel/dma-mapping.c
index 53fbbb61aa86..a27a74a18fb0 100644
--- a/arch/blackfin/kernel/dma-mapping.c
+++ b/arch/blackfin/kernel/dma-mapping.c
@@ -118,6 +118,10 @@ static int bfin_dma_map_sg(struct device *dev, struct scatterlist *sg_list,
 
 	for_each_sg(sg_list, sg, nents, i) {
 		sg->dma_address = (dma_addr_t) sg_virt(sg);
+
+		if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+			continue;
+
 		__dma_sync(sg_dma_address(sg), sg_dma_len(sg), direction);
 	}
 
@@ -143,7 +147,9 @@ static dma_addr_t bfin_dma_map_page(struct device *dev, struct page *page,
 {
 	dma_addr_t handle = (dma_addr_t)(page_address(page) + offset);
 
-	_dma_sync(handle, size, dir);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		_dma_sync(handle, size, dir);
+
 	return handle;
 }
 
diff --git a/arch/blackfin/kernel/ptrace.c b/arch/blackfin/kernel/ptrace.c
index 8d79286ee4e8..360d99645163 100644
--- a/arch/blackfin/kernel/ptrace.c
+++ b/arch/blackfin/kernel/ptrace.c
@@ -270,7 +270,7 @@ long arch_ptrace(struct task_struct *child, long request,
 			switch (bfin_mem_access_type(addr, to_copy)) {
 			case BFIN_MEM_ACCESS_CORE:
 			case BFIN_MEM_ACCESS_CORE_ONLY:
-				copied = access_process_vm(child, addr, &tmp,
+				copied = ptrace_access_vm(child, addr, &tmp,
 							   to_copy, FOLL_FORCE);
 				if (copied)
 					break;
@@ -323,7 +323,7 @@ long arch_ptrace(struct task_struct *child, long request,
 			switch (bfin_mem_access_type(addr, to_copy)) {
 			case BFIN_MEM_ACCESS_CORE:
 			case BFIN_MEM_ACCESS_CORE_ONLY:
-				copied = access_process_vm(child, addr, &data,
+				copied = ptrace_access_vm(child, addr, &data,
 				                           to_copy,
 							   FOLL_FORCE | FOLL_WRITE);
 				break;
diff --git a/arch/blackfin/mach-bf561/coreb.c b/arch/blackfin/mach-bf561/coreb.c
index 8a2543c654b3..cf27554e76bf 100644
--- a/arch/blackfin/mach-bf561/coreb.c
+++ b/arch/blackfin/mach-bf561/coreb.c
@@ -1,5 +1,7 @@
 /* Load firmware into Core B on a BF561
  *
+ * Author: Bas Vermeulen <bvermeul@blackstar.xs4all.nl>
+ *
  * Copyright 2004-2009 Analog Devices Inc.
  * Licensed under the GPL-2 or later.
  */
@@ -14,9 +16,9 @@
 
 #include <linux/device.h>
 #include <linux/fs.h>
+#include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/miscdevice.h>
-#include <linux/module.h>
 
 #define CMD_COREB_START		_IO('b', 0)
 #define CMD_COREB_STOP		_IO('b', 1)
@@ -59,8 +61,4 @@ static struct miscdevice coreb_dev = {
 	.name  = "coreb",
 	.fops  = &coreb_fops,
 };
-module_misc_device(coreb_dev);
-
-MODULE_AUTHOR("Bas Vermeulen <bvermeul@blackstar.xs4all.nl>");
-MODULE_DESCRIPTION("BF561 Core B Support");
-MODULE_LICENSE("GPL");
+builtin_misc_device(coreb_dev);
diff --git a/arch/c6x/kernel/dma.c b/arch/c6x/kernel/dma.c
index db4a6a301f5e..6752df32ef06 100644
--- a/arch/c6x/kernel/dma.c
+++ b/arch/c6x/kernel/dma.c
@@ -42,14 +42,17 @@ static dma_addr_t c6x_dma_map_page(struct device *dev, struct page *page,
 {
 	dma_addr_t handle = virt_to_phys(page_address(page) + offset);
 
-	c6x_dma_sync(handle, size, dir);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		c6x_dma_sync(handle, size, dir);
+
 	return handle;
 }
 
 static void c6x_dma_unmap_page(struct device *dev, dma_addr_t handle,
 		size_t size, enum dma_data_direction dir, unsigned long attrs)
 {
-	c6x_dma_sync(handle, size, dir);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		c6x_dma_sync(handle, size, dir);
 }
 
 static int c6x_dma_map_sg(struct device *dev, struct scatterlist *sglist,
@@ -60,7 +63,8 @@ static int c6x_dma_map_sg(struct device *dev, struct scatterlist *sglist,
 
 	for_each_sg(sglist, sg, nents, i) {
 		sg->dma_address = sg_phys(sg);
-		c6x_dma_sync(sg->dma_address, sg->length, dir);
+		if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+			c6x_dma_sync(sg->dma_address, sg->length, dir);
 	}
 
 	return nents;
@@ -72,9 +76,11 @@ static void c6x_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
 	struct scatterlist *sg;
 	int i;
 
+	if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+		return;
+
 	for_each_sg(sglist, sg, nents, i)
 		c6x_dma_sync(sg_dma_address(sg), sg->length, dir);
-
 }
 
 static void c6x_dma_sync_single_for_cpu(struct device *dev, dma_addr_t handle,
diff --git a/arch/cris/arch-v32/kernel/ptrace.c b/arch/cris/arch-v32/kernel/ptrace.c
index f0df654ac6fc..fe1f9cf7b391 100644
--- a/arch/cris/arch-v32/kernel/ptrace.c
+++ b/arch/cris/arch-v32/kernel/ptrace.c
@@ -147,7 +147,7 @@ long arch_ptrace(struct task_struct *child, long request,
 				/* The trampoline page is globally mapped, no page table to traverse.*/
 				tmp = *(unsigned long*)addr;
 			} else {
-				copied = access_process_vm(child, addr, &tmp, sizeof(tmp), FOLL_FORCE);
+				copied = ptrace_access_vm(child, addr, &tmp, sizeof(tmp), FOLL_FORCE);
 
 				if (copied != sizeof(tmp))
 					break;
diff --git a/arch/frv/mb93090-mb00/pci-dma-nommu.c b/arch/frv/mb93090-mb00/pci-dma-nommu.c
index 90f2e4cb33d6..187688128c65 100644
--- a/arch/frv/mb93090-mb00/pci-dma-nommu.c
+++ b/arch/frv/mb93090-mb00/pci-dma-nommu.c
@@ -109,16 +109,19 @@ static int frv_dma_map_sg(struct device *dev, struct scatterlist *sglist,
 		int nents, enum dma_data_direction direction,
 		unsigned long attrs)
 {
-	int i;
 	struct scatterlist *sg;
+	int i;
+
+	BUG_ON(direction == DMA_NONE);
+
+	if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+		return nents;
 
 	for_each_sg(sglist, sg, nents, i) {
 		frv_cache_wback_inv(sg_dma_address(sg),
 				    sg_dma_address(sg) + sg_dma_len(sg));
 	}
 
-	BUG_ON(direction == DMA_NONE);
-
 	return nents;
 }
 
@@ -127,7 +130,10 @@ static dma_addr_t frv_dma_map_page(struct device *dev, struct page *page,
 		enum dma_data_direction direction, unsigned long attrs)
 {
 	BUG_ON(direction == DMA_NONE);
-	flush_dcache_page(page);
+
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		flush_dcache_page(page);
+
 	return (dma_addr_t) page_to_phys(page) + offset;
 }
 
diff --git a/arch/frv/mb93090-mb00/pci-dma.c b/arch/frv/mb93090-mb00/pci-dma.c
index f585745b1abc..dba7df918144 100644
--- a/arch/frv/mb93090-mb00/pci-dma.c
+++ b/arch/frv/mb93090-mb00/pci-dma.c
@@ -40,13 +40,16 @@ static int frv_dma_map_sg(struct device *dev, struct scatterlist *sglist,
 		int nents, enum dma_data_direction direction,
 		unsigned long attrs)
 {
+	struct scatterlist *sg;
 	unsigned long dampr2;
 	void *vaddr;
 	int i;
-	struct scatterlist *sg;
 
 	BUG_ON(direction == DMA_NONE);
 
+	if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+		return nents;
+
 	dampr2 = __get_DAMPR(2);
 
 	for_each_sg(sglist, sg, nents, i) {
@@ -70,7 +73,9 @@ static dma_addr_t frv_dma_map_page(struct device *dev, struct page *page,
 		unsigned long offset, size_t size,
 		enum dma_data_direction direction, unsigned long attrs)
 {
-	flush_dcache_page(page);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		flush_dcache_page(page);
+
 	return (dma_addr_t) page_to_phys(page) + offset;
 }
 
diff --git a/arch/hexagon/kernel/dma.c b/arch/hexagon/kernel/dma.c
index b9017785fb71..dbc4f1003da4 100644
--- a/arch/hexagon/kernel/dma.c
+++ b/arch/hexagon/kernel/dma.c
@@ -119,6 +119,9 @@ static int hexagon_map_sg(struct device *hwdev, struct scatterlist *sg,
 
 		s->dma_length = s->length;
 
+		if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+			continue;
+
 		flush_dcache_range(dma_addr_to_virt(s->dma_address),
 				   dma_addr_to_virt(s->dma_address + s->length));
 	}
@@ -180,7 +183,8 @@ static dma_addr_t hexagon_map_page(struct device *dev, struct page *page,
 	if (!check_addr("map_single", dev, bus, size))
 		return bad_dma_address;
 
-	dma_sync(dma_addr_to_virt(bus), size, dir);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		dma_sync(dma_addr_to_virt(bus), size, dir);
 
 	return bus;
 }
diff --git a/arch/ia64/kernel/ptrace.c b/arch/ia64/kernel/ptrace.c
index 31aa8c0f68e1..36f660da8124 100644
--- a/arch/ia64/kernel/ptrace.c
+++ b/arch/ia64/kernel/ptrace.c
@@ -1159,7 +1159,7 @@ arch_ptrace (struct task_struct *child, long request,
 	case PTRACE_PEEKTEXT:
 	case PTRACE_PEEKDATA:
 		/* read word at location addr */
-		if (access_process_vm(child, addr, &data, sizeof(data),
+		if (ptrace_access_vm(child, addr, &data, sizeof(data),
 				FOLL_FORCE)
 		    != sizeof(data))
 			return -EIO;
diff --git a/arch/m68k/kernel/dma.c b/arch/m68k/kernel/dma.c
index 8cf97cbadc91..07070065a425 100644
--- a/arch/m68k/kernel/dma.c
+++ b/arch/m68k/kernel/dma.c
@@ -134,7 +134,9 @@ static dma_addr_t m68k_dma_map_page(struct device *dev, struct page *page,
 {
 	dma_addr_t handle = page_to_phys(page) + offset;
 
-	dma_sync_single_for_device(dev, handle, size, dir);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		dma_sync_single_for_device(dev, handle, size, dir);
+
 	return handle;
 }
 
@@ -146,6 +148,10 @@ static int m68k_dma_map_sg(struct device *dev, struct scatterlist *sglist,
 
 	for_each_sg(sglist, sg, nents, i) {
 		sg->dma_address = sg_phys(sg);
+
+		if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+			continue;
+
 		dma_sync_single_for_device(dev, sg->dma_address, sg->length,
 					   dir);
 	}
diff --git a/arch/metag/kernel/dma.c b/arch/metag/kernel/dma.c
index 0db31e24c541..91968d92652b 100644
--- a/arch/metag/kernel/dma.c
+++ b/arch/metag/kernel/dma.c
@@ -484,8 +484,9 @@ static dma_addr_t metag_dma_map_page(struct device *dev, struct page *page,
 		unsigned long offset, size_t size,
 		enum dma_data_direction direction, unsigned long attrs)
 {
-	dma_sync_for_device((void *)(page_to_phys(page) + offset), size,
-			    direction);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		dma_sync_for_device((void *)(page_to_phys(page) + offset),
+				    size, direction);
 	return page_to_phys(page) + offset;
 }
 
@@ -493,7 +494,8 @@ static void metag_dma_unmap_page(struct device *dev, dma_addr_t dma_address,
 		size_t size, enum dma_data_direction direction,
 		unsigned long attrs)
 {
-	dma_sync_for_cpu(phys_to_virt(dma_address), size, direction);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		dma_sync_for_cpu(phys_to_virt(dma_address), size, direction);
 }
 
 static int metag_dma_map_sg(struct device *dev, struct scatterlist *sglist,
@@ -507,6 +509,10 @@ static int metag_dma_map_sg(struct device *dev, struct scatterlist *sglist,
 		BUG_ON(!sg_page(sg));
 
 		sg->dma_address = sg_phys(sg);
+
+		if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+			continue;
+
 		dma_sync_for_device(sg_virt(sg), sg->length, direction);
 	}
 
@@ -525,6 +531,10 @@ static void metag_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
 		BUG_ON(!sg_page(sg));
 
 		sg->dma_address = sg_phys(sg);
+
+		if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+			continue;
+
 		dma_sync_for_cpu(sg_virt(sg), sg->length, direction);
 	}
 }
diff --git a/arch/microblaze/kernel/dma.c b/arch/microblaze/kernel/dma.c
index ec04dc1e2527..818daf230eb4 100644
--- a/arch/microblaze/kernel/dma.c
+++ b/arch/microblaze/kernel/dma.c
@@ -61,6 +61,10 @@ static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl,
 	/* FIXME this part of code is untested */
 	for_each_sg(sgl, sg, nents, i) {
 		sg->dma_address = sg_phys(sg);
+
+		if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+			continue;
+
 		__dma_sync(page_to_phys(sg_page(sg)) + sg->offset,
 							sg->length, direction);
 	}
@@ -80,7 +84,8 @@ static inline dma_addr_t dma_direct_map_page(struct device *dev,
 					     enum dma_data_direction direction,
 					     unsigned long attrs)
 {
-	__dma_sync(page_to_phys(page) + offset, size, direction);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		__dma_sync(page_to_phys(page) + offset, size, direction);
 	return page_to_phys(page) + offset;
 }
 
@@ -95,7 +100,8 @@ static inline void dma_direct_unmap_page(struct device *dev,
  * phys_to_virt is here because in __dma_sync_page is __virt_to_phys and
  * dma_address is physical address
  */
-	__dma_sync(dma_address, size, direction);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		__dma_sync(dma_address, size, direction);
 }
 
 static inline void
diff --git a/arch/mips/kernel/asm-offsets.c b/arch/mips/kernel/asm-offsets.c
index fae2f9447792..6080582a26d1 100644
--- a/arch/mips/kernel/asm-offsets.c
+++ b/arch/mips/kernel/asm-offsets.c
@@ -341,7 +341,7 @@ void output_pm_defines(void)
 
 void output_kvm_defines(void)
 {
-	COMMENT(" KVM/MIPS Specfic offsets. ");
+	COMMENT(" KVM/MIPS Specific offsets. ");
 
 	OFFSET(VCPU_FPR0, kvm_vcpu_arch, fpu.fpr[0]);
 	OFFSET(VCPU_FPR1, kvm_vcpu_arch, fpu.fpr[1]);
diff --git a/arch/mips/kernel/ptrace32.c b/arch/mips/kernel/ptrace32.c
index 7e71a4e0281b..5fcbdcd7abd0 100644
--- a/arch/mips/kernel/ptrace32.c
+++ b/arch/mips/kernel/ptrace32.c
@@ -69,7 +69,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
 		if (get_user(addrOthers, (u32 __user * __user *) (unsigned long) addr) != 0)
 			break;
 
-		copied = access_process_vm(child, (u64)addrOthers, &tmp,
+		copied = ptrace_access_vm(child, (u64)addrOthers, &tmp,
 				sizeof(tmp), FOLL_FORCE);
 		if (copied != sizeof(tmp))
 			break;
@@ -178,7 +178,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
 		if (get_user(addrOthers, (u32 __user * __user *) (unsigned long) addr) != 0)
 			break;
 		ret = 0;
-		if (access_process_vm(child, (u64)addrOthers, &data,
+		if (ptrace_access_vm(child, (u64)addrOthers, &data,
 					sizeof(data),
 					FOLL_FORCE | FOLL_WRITE) == sizeof(data))
 			break;
diff --git a/arch/mips/loongson64/common/dma-swiotlb.c b/arch/mips/loongson64/common/dma-swiotlb.c
index 1a80b6f73ab2..aab4fd681e1f 100644
--- a/arch/mips/loongson64/common/dma-swiotlb.c
+++ b/arch/mips/loongson64/common/dma-swiotlb.c
@@ -61,7 +61,7 @@ static int loongson_dma_map_sg(struct device *dev, struct scatterlist *sg,
 				int nents, enum dma_data_direction dir,
 				unsigned long attrs)
 {
-	int r = swiotlb_map_sg_attrs(dev, sg, nents, dir, 0);
+	int r = swiotlb_map_sg_attrs(dev, sg, nents, dir, attrs);
 	mb();
 
 	return r;
diff --git a/arch/mips/mm/dma-default.c b/arch/mips/mm/dma-default.c
index 46d5696c4f27..a39c36af97ad 100644
--- a/arch/mips/mm/dma-default.c
+++ b/arch/mips/mm/dma-default.c
@@ -293,7 +293,7 @@ static inline void __dma_sync(struct page *page,
 static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
 	size_t size, enum dma_data_direction direction, unsigned long attrs)
 {
-	if (cpu_needs_post_dma_flush(dev))
+	if (cpu_needs_post_dma_flush(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 		__dma_sync(dma_addr_to_page(dev, dma_addr),
 			   dma_addr & ~PAGE_MASK, size, direction);
 	plat_post_dma_flush(dev);
@@ -307,7 +307,8 @@ static int mips_dma_map_sg(struct device *dev, struct scatterlist *sglist,
 	struct scatterlist *sg;
 
 	for_each_sg(sglist, sg, nents, i) {
-		if (!plat_device_is_coherent(dev))
+		if (!plat_device_is_coherent(dev) &&
+		    !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 			__dma_sync(sg_page(sg), sg->offset, sg->length,
 				   direction);
 #ifdef CONFIG_NEED_SG_DMA_LENGTH
@@ -324,7 +325,7 @@ static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
 	unsigned long offset, size_t size, enum dma_data_direction direction,
 	unsigned long attrs)
 {
-	if (!plat_device_is_coherent(dev))
+	if (!plat_device_is_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 		__dma_sync(page, offset, size, direction);
 
 	return plat_map_dma_mem_page(dev, page) + offset;
@@ -339,6 +340,7 @@ static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
 
 	for_each_sg(sglist, sg, nhwentries, i) {
 		if (!plat_device_is_coherent(dev) &&
+		    !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
 		    direction != DMA_TO_DEVICE)
 			__dma_sync(sg_page(sg), sg->offset, sg->length,
 				   direction);
diff --git a/arch/nios2/include/asm/page.h b/arch/nios2/include/asm/page.h
index c1683f51ad0f..f1fbdc47bdaf 100644
--- a/arch/nios2/include/asm/page.h
+++ b/arch/nios2/include/asm/page.h
@@ -76,8 +76,6 @@ extern unsigned long memory_size;
 
 extern struct page *mem_map;
 
-#endif /* !__ASSEMBLY__ */
-
 # define __pa(x)		\
 	((unsigned long)(x) - PAGE_OFFSET + PHYS_OFFSET)
 # define __va(x)		\
@@ -87,8 +85,15 @@ extern struct page *mem_map;
 	((void *)(((page) - mem_map) << PAGE_SHIFT) + PAGE_OFFSET)
 
 # define pfn_to_kaddr(pfn)	__va((pfn) << PAGE_SHIFT)
-# define pfn_valid(pfn)		((pfn) >= ARCH_PFN_OFFSET &&	\
-					(pfn) < max_mapnr)
+
+static inline bool pfn_valid(unsigned long pfn)
+{
+	/* avoid <linux/mm.h> include hell */
+	extern unsigned long max_mapnr;
+	unsigned long pfn_offset = ARCH_PFN_OFFSET;
+
+	return pfn >= pfn_offset && pfn < max_mapnr;
+}
 
 # define virt_to_page(vaddr)	pfn_to_page(PFN_DOWN(virt_to_phys(vaddr)))
 # define virt_addr_valid(vaddr)	pfn_valid(PFN_DOWN(virt_to_phys(vaddr)))
@@ -106,4 +111,6 @@ extern struct page *mem_map;
 
 #include <asm-generic/getorder.h>
 
+#endif /* !__ASSEMBLY__ */
+
 #endif /* _ASM_NIOS2_PAGE_H */
diff --git a/arch/nios2/kernel/setup.c b/arch/nios2/kernel/setup.c
index a4ff86d58d5c..a3fa80d1aacc 100644
--- a/arch/nios2/kernel/setup.c
+++ b/arch/nios2/kernel/setup.c
@@ -18,6 +18,7 @@
 #include <linux/bootmem.h>
 #include <linux/initrd.h>
 #include <linux/of_fdt.h>
+#include <linux/screen_info.h>
 
 #include <asm/mmu_context.h>
 #include <asm/sections.h>
@@ -36,6 +37,10 @@ static struct pt_regs fake_regs = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 					0, 0, 0, 0, 0, 0,
 					0};
 
+#ifdef CONFIG_VT
+struct screen_info screen_info;
+#endif
+
 /* Copy a short hook instruction sequence to the exception address */
 static inline void copy_exception_handler(unsigned int addr)
 {
diff --git a/arch/nios2/mm/dma-mapping.c b/arch/nios2/mm/dma-mapping.c
index d800fad87896..f6a5dcf9d682 100644
--- a/arch/nios2/mm/dma-mapping.c
+++ b/arch/nios2/mm/dma-mapping.c
@@ -98,13 +98,17 @@ static int nios2_dma_map_sg(struct device *dev, struct scatterlist *sg,
 	int i;
 
 	for_each_sg(sg, sg, nents, i) {
-		void *addr;
+		void *addr = sg_virt(sg);
 
-		addr = sg_virt(sg);
-		if (addr) {
-			__dma_sync_for_device(addr, sg->length, direction);
-			sg->dma_address = sg_phys(sg);
-		}
+		if (!addr)
+			continue;
+
+		sg->dma_address = sg_phys(sg);
+
+		if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+			continue;
+
+		__dma_sync_for_device(addr, sg->length, direction);
 	}
 
 	return nents;
@@ -117,7 +121,9 @@ static dma_addr_t nios2_dma_map_page(struct device *dev, struct page *page,
 {
 	void *addr = page_address(page) + offset;
 
-	__dma_sync_for_device(addr, size, direction);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		__dma_sync_for_device(addr, size, direction);
+
 	return page_to_phys(page) + offset;
 }
 
@@ -125,7 +131,8 @@ static void nios2_dma_unmap_page(struct device *dev, dma_addr_t dma_address,
 		size_t size, enum dma_data_direction direction,
 		unsigned long attrs)
 {
-	__dma_sync_for_cpu(phys_to_virt(dma_address), size, direction);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		__dma_sync_for_cpu(phys_to_virt(dma_address), size, direction);
 }
 
 static void nios2_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
@@ -138,6 +145,9 @@ static void nios2_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
 	if (direction == DMA_TO_DEVICE)
 		return;
 
+	if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+		return;
+
 	for_each_sg(sg, sg, nhwentries, i) {
 		addr = sg_virt(sg);
 		if (addr)
diff --git a/arch/openrisc/kernel/dma.c b/arch/openrisc/kernel/dma.c
index 140c99140649..906998bac957 100644
--- a/arch/openrisc/kernel/dma.c
+++ b/arch/openrisc/kernel/dma.c
@@ -141,6 +141,9 @@ or1k_map_page(struct device *dev, struct page *page,
 	unsigned long cl;
 	dma_addr_t addr = page_to_phys(page) + offset;
 
+	if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+		return addr;
+
 	switch (dir) {
 	case DMA_TO_DEVICE:
 		/* Flush the dcache for the requested range */
diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c
index 494ff6e8c88a..b6298a85e8ae 100644
--- a/arch/parisc/kernel/pci-dma.c
+++ b/arch/parisc/kernel/pci-dma.c
@@ -459,7 +459,9 @@ static dma_addr_t pa11_dma_map_page(struct device *dev, struct page *page,
 	void *addr = page_address(page) + offset;
 	BUG_ON(direction == DMA_NONE);
 
-	flush_kernel_dcache_range((unsigned long) addr, size);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		flush_kernel_dcache_range((unsigned long) addr, size);
+
 	return virt_to_phys(addr);
 }
 
@@ -469,8 +471,11 @@ static void pa11_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
 {
 	BUG_ON(direction == DMA_NONE);
 
+	if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+		return;
+
 	if (direction == DMA_TO_DEVICE)
-	    return;
+		return;
 
 	/*
 	 * For PCI_DMA_FROMDEVICE this flush is not necessary for the
@@ -479,7 +484,6 @@ static void pa11_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
 	 */
 
 	flush_kernel_dcache_range((unsigned long) phys_to_virt(dma_handle), size);
-	return;
 }
 
 static int pa11_dma_map_sg(struct device *dev, struct scatterlist *sglist,
@@ -496,6 +500,10 @@ static int pa11_dma_map_sg(struct device *dev, struct scatterlist *sglist,
 
 		sg_dma_address(sg) = (dma_addr_t) virt_to_phys(vaddr);
 		sg_dma_len(sg) = sg->length;
+
+		if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+			continue;
+
 		flush_kernel_dcache_range(vaddr, sg->length);
 	}
 	return nents;
@@ -510,14 +518,16 @@ static void pa11_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
 
 	BUG_ON(direction == DMA_NONE);
 
+	if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+		return;
+
 	if (direction == DMA_TO_DEVICE)
-	    return;
+		return;
 
 	/* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */
 
 	for_each_sg(sglist, sg, nents, i)
 		flush_kernel_vmap_range(sg_virt(sg), sg->length);
-	return;
 }
 
 static void pa11_dma_sync_single_for_cpu(struct device *dev,
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index c7f120aaa98f..3da87e198878 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -80,6 +80,7 @@ config ARCH_HAS_DMA_SET_COHERENT_MASK
 config PPC
 	bool
 	default y
+	select BUILDTIME_EXTABLE_SORT
 	select ARCH_MIGHT_HAVE_PC_PARPORT
 	select ARCH_MIGHT_HAVE_PC_SERIO
 	select BINFMT_ELF
@@ -163,6 +164,7 @@ config PPC
 	select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE
 	select HAVE_ARCH_HARDENED_USERCOPY
 	select HAVE_KERNEL_GZIP
+	select HAVE_CC_STACKPROTECTOR
 
 config GENERIC_CSUM
 	def_bool CPU_LITTLE_ENDIAN
@@ -396,6 +398,14 @@ config MPROFILE_KERNEL
 	depends on PPC64 && CPU_LITTLE_ENDIAN
 	def_bool !DISABLE_MPROFILE_KERNEL
 
+config USE_THIN_ARCHIVES
+	bool "Build the kernel using thin archives"
+	default n
+	select THIN_ARCHIVES
+	help
+	  Build the kernel using thin archives.
+	  If you're unsure say N.
+
 config IOMMU_HELPER
 	def_bool PPC64
 
@@ -456,6 +466,19 @@ config KEXEC
 	  interface is strongly in flux, so no good recommendation can be
 	  made.
 
+config KEXEC_FILE
+	bool "kexec file based system call"
+	select KEXEC_CORE
+	select BUILD_BIN2C
+	depends on PPC64
+	depends on CRYPTO=y
+	depends on CRYPTO_SHA256=y
+	help
+	  This is a new version of the kexec system call. This call is
+	  file based and takes in file descriptors as system call arguments
+	  for kernel and initramfs as opposed to a list of segments as is the
+	  case for the older kexec call.
+
 config RELOCATABLE
 	bool "Build a relocatable kernel"
 	depends on (PPC64 && !COMPILE_TEST) || (FLATMEM && (44x || FSL_BOOKE))
@@ -479,6 +502,15 @@ config RELOCATABLE
 	  setting can still be useful to bootwrappers that need to know the
 	  load address of the kernel (eg. u-boot/mkimage).
 
+config RELOCATABLE_TEST
+	bool "Test relocatable kernel"
+	depends on (PPC64 && RELOCATABLE)
+	default n
+	help
+	  This runs the relocatable kernel at the address it was initially
+	  loaded at, which tends to be non-zero and therefore test the
+	  relocation code.
+
 config CRASH_DUMP
 	bool "Build a kdump crash kernel"
 	depends on PPC64 || 6xx || FSL_BOOKE || (44x && !SMP)
@@ -490,7 +522,7 @@ config CRASH_DUMP
 
 config FA_DUMP
 	bool "Firmware-assisted dump"
-	depends on PPC64 && PPC_RTAS && CRASH_DUMP && KEXEC
+	depends on PPC64 && PPC_RTAS && CRASH_DUMP && KEXEC_CORE
 	help
 	  A robust mechanism to get reliable kernel crash dump with
 	  assistance from firmware. This approach does not use kexec,
@@ -549,6 +581,13 @@ config ARCH_SPARSEMEM_DEFAULT
 config SYS_SUPPORTS_HUGETLBFS
 	bool
 
+config ILLEGAL_POINTER_VALUE
+	hex
+	# This is roughly half way between the top of user space and the bottom
+	# of kernel space, which seems about as good as we can get.
+	default 0x5deadbeef0000000 if PPC64
+	default 0
+
 source "mm/Kconfig"
 
 config ARCH_MEMORY_PROBE
diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug
index 63292f64b25a..949258d412d0 100644
--- a/arch/powerpc/Kconfig.debug
+++ b/arch/powerpc/Kconfig.debug
@@ -354,4 +354,20 @@ config FAIL_IOMMU
 
 	  If you are unsure, say N.
 
+config PPC_PTDUMP
+        bool "Export kernel pagetable layout to userspace via debugfs"
+        depends on DEBUG_KERNEL
+        select DEBUG_FS
+        help
+	  This option exports the state of the kernel pagetables to a
+	  debugfs file. This is only useful for kernel developers who are
+	  working in architecture specific areas of the kernel - probably
+	  not a good idea to enable this feature in a production kernel.
+
+	  If you are unsure, say N.
+
+config PPC_HTDUMP
+	def_bool y
+	depends on PPC_PTDUMP && PPC_BOOK3S
+
 endmenu
diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile
index 617dece67924..31286fa7873c 100644
--- a/arch/powerpc/Makefile
+++ b/arch/powerpc/Makefile
@@ -23,7 +23,7 @@ CROSS32AR		:= $(CROSS32_COMPILE)ar
 ifeq ($(HAS_BIARCH),y)
 ifeq ($(CROSS32_COMPILE),)
 CROSS32CC	:= $(CC) -m32
-CROSS32AR	:= GNUTARGET=elf32-powerpc $(AR)
+KBUILD_ARFLAGS	+= --target=elf32-powerpc
 endif
 endif
 
@@ -85,7 +85,7 @@ ifeq ($(HAS_BIARCH),y)
 override AS	+= -a$(BITS)
 override LD	+= -m elf$(BITS)$(LDEMULATION)
 override CC	+= -m$(BITS)
-override AR	:= GNUTARGET=elf$(BITS)-$(GNUTARGET) $(AR)
+KBUILD_ARFLAGS	+= --target=elf$(BITS)-$(GNUTARGET)
 endif
 
 LDFLAGS_vmlinux-y := -Bstatic
@@ -121,6 +121,7 @@ CFLAGS-$(CONFIG_PPC32)	:= -ffixed-r2 $(MULTIPLEWORD)
 
 ifeq ($(CONFIG_PPC_BOOK3S_64),y)
 CFLAGS-$(CONFIG_GENERIC_CPU) += $(call cc-option,-mtune=power7,-mtune=power4)
+CFLAGS-$(CONFIG_GENERIC_CPU) += -mcpu=power4
 else
 CFLAGS-$(CONFIG_GENERIC_CPU) += -mcpu=powerpc64
 endif
@@ -249,6 +250,7 @@ core-y				+= arch/powerpc/kernel/ \
 core-$(CONFIG_XMON)		+= arch/powerpc/xmon/
 core-$(CONFIG_KVM) 		+= arch/powerpc/kvm/
 core-$(CONFIG_PERF_EVENTS)	+= arch/powerpc/perf/
+core-$(CONFIG_KEXEC_FILE)	+= arch/powerpc/purgatory/
 
 drivers-$(CONFIG_OPROFILE)	+= arch/powerpc/oprofile/
 
@@ -275,16 +277,16 @@ zImage: relocs_check
 endif
 
 $(BOOT_TARGETS1): vmlinux
-	$(Q)$(MAKE) ARCH=ppc64 $(build)=$(boot) $(patsubst %,$(boot)/%,$@)
+	$(Q)$(MAKE) $(build)=$(boot) $(patsubst %,$(boot)/%,$@)
 $(BOOT_TARGETS2): vmlinux
-	$(Q)$(MAKE) ARCH=ppc64 $(build)=$(boot) $(patsubst %,$(boot)/%,$@)
+	$(Q)$(MAKE) $(build)=$(boot) $(patsubst %,$(boot)/%,$@)
 
 
 bootwrapper_install:
-	$(Q)$(MAKE) ARCH=ppc64 $(build)=$(boot) $(patsubst %,$(boot)/%,$@)
+	$(Q)$(MAKE) $(build)=$(boot) $(patsubst %,$(boot)/%,$@)
 
 %.dtb: scripts
-	$(Q)$(MAKE) ARCH=ppc64 $(build)=$(boot) $(patsubst %,$(boot)/%,$@)
+	$(Q)$(MAKE) $(build)=$(boot) $(patsubst %,$(boot)/%,$@)
 
 # Used to create 'merged defconfigs'
 # To use it $(call) it with the first argument as the base defconfig
diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile
index 9d47f2efa830..e82f333cc84a 100644
--- a/arch/powerpc/boot/Makefile
+++ b/arch/powerpc/boot/Makefile
@@ -172,10 +172,6 @@ $(addprefix $(obj)/,$(libfdt) $(libfdtheader)): $(obj)/%: $(srctree)/scripts/dtc
 $(obj)/empty.c:
 	$(Q)touch $@
 
-$(obj)/zImage.lds: $(obj)/%: $(srctree)/$(src)/%.S
-	$(CROSS32CC) $(cpp_flags) -E -Wp,-MD,$(depfile) -P -Upowerpc \
-		-D__ASSEMBLY__ -DLINKER_SCRIPT -o $@ $<
-
 $(obj)/zImage.coff.lds $(obj)/zImage.ps3.lds : $(obj)/%: $(srctree)/$(src)/%.S
 	$(Q)cp $< $@
 
@@ -357,17 +353,17 @@ $(addprefix $(obj)/, $(initrd-y)): $(obj)/ramdisk.image.gz
 # Don't put the ramdisk on the pattern rule; when its missing make will try
 # the pattern rule with less dependencies that also matches (even with the
 # hard dependency listed).
-$(obj)/zImage.initrd.%: vmlinux $(wrapperbits)
+$(obj)/zImage.initrd.%: vmlinux $(wrapperbits) FORCE
 	$(call if_changed,wrap,$*,,,$(obj)/ramdisk.image.gz)
 
-$(addprefix $(obj)/, $(sort $(filter zImage.%, $(image-y)))): vmlinux $(wrapperbits)
+$(addprefix $(obj)/, $(sort $(filter zImage.%, $(image-y)))): vmlinux $(wrapperbits) FORCE
 	$(call if_changed,wrap,$(subst $(obj)/zImage.,,$@))
 
 # dtbImage% - a dtbImage is a zImage with an embedded device tree blob
-$(obj)/dtbImage.initrd.%: vmlinux $(wrapperbits) $(obj)/%.dtb
+$(obj)/dtbImage.initrd.%: vmlinux $(wrapperbits) $(obj)/%.dtb FORCE
 	$(call if_changed,wrap,$*,,$(obj)/$*.dtb,$(obj)/ramdisk.image.gz)
 
-$(obj)/dtbImage.%: vmlinux $(wrapperbits) $(obj)/%.dtb
+$(obj)/dtbImage.%: vmlinux $(wrapperbits) $(obj)/%.dtb FORCE
 	$(call if_changed,wrap,$*,,$(obj)/$*.dtb)
 
 # This cannot be in the root of $(src) as the zImage rule always adds a $(obj)
@@ -375,31 +371,31 @@ $(obj)/dtbImage.%: vmlinux $(wrapperbits) $(obj)/%.dtb
 $(obj)/vmlinux.strip: vmlinux
 	$(STRIP) -s -R .comment $< -o $@
 
-$(obj)/uImage: vmlinux $(wrapperbits)
+$(obj)/uImage: vmlinux $(wrapperbits) FORCE
 	$(call if_changed,wrap,uboot)
 
-$(obj)/uImage.initrd.%: vmlinux $(obj)/%.dtb $(wrapperbits)
+$(obj)/uImage.initrd.%: vmlinux $(obj)/%.dtb $(wrapperbits) FORCE
 	$(call if_changed,wrap,uboot-$*,,$(obj)/$*.dtb,$(obj)/ramdisk.image.gz)
 
-$(obj)/uImage.%: vmlinux $(obj)/%.dtb $(wrapperbits)
+$(obj)/uImage.%: vmlinux $(obj)/%.dtb $(wrapperbits) FORCE
 	$(call if_changed,wrap,uboot-$*,,$(obj)/$*.dtb)
 
-$(obj)/cuImage.initrd.%: vmlinux $(obj)/%.dtb $(wrapperbits)
+$(obj)/cuImage.initrd.%: vmlinux $(obj)/%.dtb $(wrapperbits) FORCE
 	$(call if_changed,wrap,cuboot-$*,,$(obj)/$*.dtb,$(obj)/ramdisk.image.gz)
 
-$(obj)/cuImage.%: vmlinux $(obj)/%.dtb $(wrapperbits)
+$(obj)/cuImage.%: vmlinux $(obj)/%.dtb $(wrapperbits) FORCE
 	$(call if_changed,wrap,cuboot-$*,,$(obj)/$*.dtb)
 
-$(obj)/simpleImage.initrd.%: vmlinux $(obj)/%.dtb $(wrapperbits)
+$(obj)/simpleImage.initrd.%: vmlinux $(obj)/%.dtb $(wrapperbits) FORCE
 	$(call if_changed,wrap,simpleboot-$*,,$(obj)/$*.dtb,$(obj)/ramdisk.image.gz)
 
-$(obj)/simpleImage.%: vmlinux $(obj)/%.dtb $(wrapperbits)
+$(obj)/simpleImage.%: vmlinux $(obj)/%.dtb $(wrapperbits) FORCE
 	$(call if_changed,wrap,simpleboot-$*,,$(obj)/$*.dtb)
 
-$(obj)/treeImage.initrd.%: vmlinux $(obj)/%.dtb $(wrapperbits)
+$(obj)/treeImage.initrd.%: vmlinux $(obj)/%.dtb $(wrapperbits) FORCE
 	$(call if_changed,wrap,treeboot-$*,,$(obj)/$*.dtb,$(obj)/ramdisk.image.gz)
 
-$(obj)/treeImage.%: vmlinux $(obj)/%.dtb $(wrapperbits)
+$(obj)/treeImage.%: vmlinux $(obj)/%.dtb $(wrapperbits) FORCE
 	$(call if_changed,wrap,treeboot-$*,,$(obj)/$*.dtb)
 
 # Rule to build device tree blobs
diff --git a/arch/powerpc/boot/dts/fsl/t1023rdb.dts b/arch/powerpc/boot/dts/fsl/t1023rdb.dts
index 29757623e5ba..5ba6fbfca274 100644
--- a/arch/powerpc/boot/dts/fsl/t1023rdb.dts
+++ b/arch/powerpc/boot/dts/fsl/t1023rdb.dts
@@ -41,6 +41,27 @@
 	#size-cells = <2>;
 	interrupt-parent = <&mpic>;
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		bman_fbpr: bman-fbpr {
+			size = <0 0x1000000>;
+			alignment = <0 0x1000000>;
+		};
+
+		qman_fqd: qman-fqd {
+			size = <0 0x400000>;
+			alignment = <0 0x400000>;
+		};
+
+		qman_pfdr: qman-pfdr {
+			size = <0 0x2000000>;
+			alignment = <0 0x2000000>;
+		};
+	};
+
 	ifc: localbus@ffe124000 {
 		reg = <0xf 0xfe124000 0 0x2000>;
 		ranges = <0 0 0xf 0xe8000000 0x08000000
@@ -72,6 +93,14 @@
 		ranges = <0x00000000 0xf 0x00000000 0x01072000>;
 	};
 
+	bportals: bman-portals@ff4000000 {
+		ranges = <0x0 0xf 0xf4000000 0x2000000>;
+	};
+
+	qportals: qman-portals@ff6000000 {
+		ranges = <0x0 0xf 0xf6000000 0x2000000>;
+	};
+
 	soc: soc@ffe000000 {
 		ranges = <0x00000000 0xf 0xfe000000 0x1000000>;
 		reg = <0xf 0xfe000000 0 0x00001000>;
diff --git a/arch/powerpc/boot/dts/fsl/t1023si-post.dtsi b/arch/powerpc/boot/dts/fsl/t1023si-post.dtsi
index 6e0b4892a740..da2894c59479 100644
--- a/arch/powerpc/boot/dts/fsl/t1023si-post.dtsi
+++ b/arch/powerpc/boot/dts/fsl/t1023si-post.dtsi
@@ -34,6 +34,21 @@
 
 #include <dt-bindings/thermal/thermal.h>
 
+&bman_fbpr {
+	compatible = "fsl,bman-fbpr";
+	alloc-ranges = <0 0 0x10000 0>;
+};
+
+&qman_fqd {
+	compatible = "fsl,qman-fqd";
+	alloc-ranges = <0 0 0x10000 0>;
+};
+
+&qman_pfdr {
+	compatible = "fsl,qman-pfdr";
+	alloc-ranges = <0 0 0x10000 0>;
+};
+
 &ifc {
 	#address-cells = <2>;
 	#size-cells = <1>;
@@ -180,6 +195,92 @@
 	};
 };
 
+&bportals {
+	#address-cells = <0x1>;
+	#size-cells = <0x1>;
+	compatible = "simple-bus";
+
+	bman-portal@0 {
+		cell-index = <0x0>;
+		compatible = "fsl,bman-portal";
+		reg = <0x0 0x4000>, <0x1000000 0x1000>;
+		interrupts = <105 2 0 0>;
+	};
+	bman-portal@4000 {
+		cell-index = <0x1>;
+		compatible = "fsl,bman-portal";
+		reg = <0x4000 0x4000>, <0x1001000 0x1000>;
+		interrupts = <107 2 0 0>;
+	};
+	bman-portal@8000 {
+		cell-index = <2>;
+		compatible = "fsl,bman-portal";
+		reg = <0x8000 0x4000>, <0x1002000 0x1000>;
+		interrupts = <109 2 0 0>;
+	};
+	bman-portal@c000 {
+		cell-index = <0x3>;
+		compatible = "fsl,bman-portal";
+		reg = <0xc000 0x4000>, <0x1003000 0x1000>;
+		interrupts = <111 2 0 0>;
+	};
+	bman-portal@10000 {
+		cell-index = <0x4>;
+		compatible = "fsl,bman-portal";
+		reg = <0x10000 0x4000>, <0x1004000 0x1000>;
+		interrupts = <113 2 0 0>;
+	};
+	bman-portal@14000 {
+		cell-index = <0x5>;
+		compatible = "fsl,bman-portal";
+		reg = <0x14000 0x4000>, <0x1005000 0x1000>;
+		interrupts = <115 2 0 0>;
+	};
+};
+
+&qportals {
+	#address-cells = <0x1>;
+	#size-cells = <0x1>;
+	compatible = "simple-bus";
+
+	qportal0: qman-portal@0 {
+		compatible = "fsl,qman-portal";
+		reg = <0x0 0x4000>, <0x1000000 0x1000>;
+		interrupts = <104 0x2 0 0>;
+		cell-index = <0x0>;
+	};
+	qportal1: qman-portal@4000 {
+		compatible = "fsl,qman-portal";
+		reg = <0x4000 0x4000>, <0x1001000 0x1000>;
+		interrupts = <106 0x2 0 0>;
+		cell-index = <0x1>;
+	};
+	qportal2: qman-portal@8000 {
+		compatible = "fsl,qman-portal";
+		reg = <0x8000 0x4000>, <0x1002000 0x1000>;
+		interrupts = <108 0x2 0 0>;
+		cell-index = <0x2>;
+	};
+	qportal3: qman-portal@c000 {
+		compatible = "fsl,qman-portal";
+		reg = <0xc000 0x4000>, <0x1003000 0x1000>;
+		interrupts = <110 0x2 0 0>;
+		cell-index = <0x3>;
+	};
+	qportal4: qman-portal@10000 {
+		compatible = "fsl,qman-portal";
+		reg = <0x10000 0x4000>, <0x1004000 0x1000>;
+		interrupts = <112 0x2 0 0>;
+		cell-index = <0x4>;
+	};
+	qportal5: qman-portal@14000 {
+		compatible = "fsl,qman-portal";
+		reg = <0x14000 0x4000>, <0x1005000 0x1000>;
+		interrupts = <114 0x2 0 0>;
+		cell-index = <0x5>;
+	};
+};
+
 &soc {
 	#address-cells = <1>;
 	#size-cells = <1>;
@@ -413,6 +514,8 @@
 	};
 
 /include/ "qoriq-sec5.0-0.dtsi"
+/include/ "qoriq-qman3.dtsi"
+/include/ "qoriq-bman1.dtsi"
 
 /include/ "qoriq-fman3l-0.dtsi"
 /include/ "qoriq-fman3-0-10g-0-best-effort.dtsi"
diff --git a/arch/powerpc/boot/dts/fsl/t1024qds.dts b/arch/powerpc/boot/dts/fsl/t1024qds.dts
index 772143da367f..d6858b7cd93f 100644
--- a/arch/powerpc/boot/dts/fsl/t1024qds.dts
+++ b/arch/powerpc/boot/dts/fsl/t1024qds.dts
@@ -41,6 +41,27 @@
 	#size-cells = <2>;
 	interrupt-parent = <&mpic>;
 
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		bman_fbpr: bman-fbpr {
+			size = <0 0x1000000>;
+			alignment = <0 0x1000000>;
+		};
+
+		qman_fqd: qman-fqd {
+			size = <0 0x400000>;
+			alignment = <0 0x400000>;
+		};
+
+		qman_pfdr: qman-pfdr {
+			size = <0 0x2000000>;
+			alignment = <0 0x2000000>;
+		};
+	};
+
 	ifc: localbus@ffe124000 {
 		reg = <0xf 0xfe124000 0 0x2000>;
 		ranges = <0 0 0xf 0xe8000000 0x08000000
@@ -80,6 +101,14 @@
 		ranges = <0x00000000 0xf 0x00000000 0x01072000>;
 	};
 
+	bportals: bman-portals@ff4000000 {
+		ranges = <0x0 0xf 0xf4000000 0x2000000>;
+	};
+
+	qportals: qman-portals@ff6000000 {
+		ranges = <0x0 0xf 0xf6000000 0x2000000>;
+	};
+
 	soc: soc@ffe000000 {
 		ranges = <0x00000000 0xf 0xfe000000 0x1000000>;
 		reg = <0xf 0xfe000000 0 0x00001000>;
diff --git a/arch/powerpc/boot/dts/fsl/t1024rdb.dts b/arch/powerpc/boot/dts/fsl/t1024rdb.dts
index 302cdd22b4bb..73a645324bc1 100644
--- a/arch/powerpc/boot/dts/fsl/t1024rdb.dts
+++ b/arch/powerpc/boot/dts/fsl/t1024rdb.dts
@@ -41,6 +41,31 @@
 	#size-cells = <2>;
 	interrupt-parent = <&mpic>;
 
+	aliases {
+		sg_2500_aqr105_phy4 = &sg_2500_aqr105_phy4;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		bman_fbpr: bman-fbpr {
+			size = <0 0x1000000>;
+			alignment = <0 0x1000000>;
+		};
+
+		qman_fqd: qman-fqd {
+			size = <0 0x400000>;
+			alignment = <0 0x400000>;
+		};
+
+		qman_pfdr: qman-pfdr {
+			size = <0 0x2000000>;
+			alignment = <0 0x2000000>;
+		};
+	};
+
 	ifc: localbus@ffe124000 {
 		reg = <0xf 0xfe124000 0 0x2000>;
 		ranges = <0 0 0xf 0xe8000000 0x08000000
@@ -82,6 +107,14 @@
 		ranges = <0x00000000 0xf 0x00000000 0x01072000>;
 	};
 
+	bportals: bman-portals@ff4000000 {
+		ranges = <0x0 0xf 0xf4000000 0x2000000>;
+	};
+
+	qportals: qman-portals@ff6000000 {
+		ranges = <0x0 0xf 0xf6000000 0x2000000>;
+	};
+
 	soc: soc@ffe000000 {
 		ranges = <0x00000000 0xf 0xfe000000 0x1000000>;
 		reg = <0xf 0xfe000000 0 0x00001000>;
diff --git a/arch/powerpc/boot/dts/fsl/t1042d4rdb.dts b/arch/powerpc/boot/dts/fsl/t1042d4rdb.dts
index 2a5a90dd272e..fcd2aeb5b8ac 100644
--- a/arch/powerpc/boot/dts/fsl/t1042d4rdb.dts
+++ b/arch/powerpc/boot/dts/fsl/t1042d4rdb.dts
@@ -48,6 +48,58 @@
 					"fsl,deepsleep-cpld";
 		};
 	};
+
+	soc: soc@ffe000000 {
+		fman0: fman@400000 {
+			ethernet@e0000 {
+				phy-handle = <&phy_sgmii_0>;
+				phy-connection-type = "sgmii";
+			};
+
+			ethernet@e2000 {
+				phy-handle = <&phy_sgmii_1>;
+				phy-connection-type = "sgmii";
+			};
+
+			ethernet@e4000 {
+				phy-handle = <&phy_sgmii_2>;
+				phy-connection-type = "sgmii";
+			};
+
+			ethernet@e6000 {
+				phy-handle = <&phy_rgmii_0>;
+				phy-connection-type = "rgmii";
+			};
+
+			ethernet@e8000 {
+				phy-handle = <&phy_rgmii_1>;
+				phy-connection-type = "rgmii";
+			};
+
+			mdio0: mdio@fc000 {
+				phy_sgmii_0: ethernet-phy@02 {
+					reg = <0x02>;
+				};
+
+				phy_sgmii_1: ethernet-phy@03 {
+					reg = <0x03>;
+				};
+
+				phy_sgmii_2: ethernet-phy@01 {
+					reg = <0x01>;
+				};
+
+				phy_rgmii_0: ethernet-phy@04 {
+					reg = <0x04>;
+				};
+
+				phy_rgmii_1: ethernet-phy@05 {
+					reg = <0x05>;
+				};
+			};
+		};
+	};
+
 };
 
 #include "t1042si-post.dtsi"
diff --git a/arch/powerpc/boot/dts/fsl/t4240rdb.dts b/arch/powerpc/boot/dts/fsl/t4240rdb.dts
index cc0a264b8acb..8166c660712a 100644
--- a/arch/powerpc/boot/dts/fsl/t4240rdb.dts
+++ b/arch/powerpc/boot/dts/fsl/t4240rdb.dts
@@ -125,6 +125,10 @@
 		};
 
 		i2c@118000 {
+			hwmon@2f {
+				compatible = "winbond,w83793";
+				reg = <0x2f>;
+			};
 			eeprom@52 {
 				compatible = "at24,24c256";
 				reg = <0x52>;
diff --git a/arch/powerpc/boot/ps3-head.S b/arch/powerpc/boot/ps3-head.S
index b6fcbaf5027b..3dc44b05fb97 100644
--- a/arch/powerpc/boot/ps3-head.S
+++ b/arch/powerpc/boot/ps3-head.S
@@ -57,11 +57,6 @@ __system_reset_overlay:
 	bctr
 
 1:
-	/* Save the value at addr zero for a null pointer write check later. */
-
-	li	r4, 0
-	lwz	r3, 0(r4)
-
 	/* Primary delays then goes to _zimage_start in wrapper. */
 
 	or	31, 31, 31 /* db16cyc */
diff --git a/arch/powerpc/boot/ps3.c b/arch/powerpc/boot/ps3.c
index 4ec2d86d3c50..a05558a7e51a 100644
--- a/arch/powerpc/boot/ps3.c
+++ b/arch/powerpc/boot/ps3.c
@@ -119,13 +119,12 @@ void ps3_copy_vectors(void)
 	flush_cache((void *)0x100, 512);
 }
 
-void platform_init(unsigned long null_check)
+void platform_init(void)
 {
 	const u32 heapsize = 0x1000000 - (u32)_end; /* 16MiB */
 	void *chosen;
 	unsigned long ft_addr;
 	u64 rm_size;
-	unsigned long val;
 
 	console_ops.write = ps3_console_write;
 	platform_ops.exit = ps3_exit;
@@ -153,11 +152,6 @@ void platform_init(unsigned long null_check)
 
 	printf(" flat tree at 0x%lx\n\r", ft_addr);
 
-	val = *(unsigned long *)0;
-
-	if (val != null_check)
-		printf("null check failed: %lx != %lx\n\r", val, null_check);
-
 	((kernel_entry_t)0)(ft_addr, 0, NULL);
 
 	ps3_exit();
diff --git a/arch/powerpc/boot/wrapper b/arch/powerpc/boot/wrapper
index 404b3aabdb4d..76fe3ccfd381 100755
--- a/arch/powerpc/boot/wrapper
+++ b/arch/powerpc/boot/wrapper
@@ -181,6 +181,28 @@ case "$elfformat" in
     elf32-powerpc)	format=elf32ppc	;;
 esac
 
+ld_version()
+{
+    # Poached from scripts/ld-version.sh, but we don't want to call that because
+    # this script (wrapper) is distributed separately from the kernel source.
+    # Extract linker version number from stdin and turn into single number.
+    awk '{
+	gsub(".*\\)", "");
+	gsub(".*version ", "");
+	gsub("-.*", "");
+	split($1,a, ".");
+	print a[1]*100000000 + a[2]*1000000 + a[3]*10000;
+	exit
+    }'
+}
+
+# Do not include PT_INTERP segment when linking pie. Non-pie linking
+# just ignores this option.
+LD_VERSION=$(${CROSS}ld --version | ld_version)
+LD_NO_DL_MIN_VERSION=$(echo 2.26 | ld_version)
+if [ "$LD_VERSION" -ge "$LD_NO_DL_MIN_VERSION" ] ; then
+	nodl="--no-dynamic-linker"
+fi
 
 platformo=$object/"$platform".o
 lds=$object/zImage.lds
@@ -446,7 +468,7 @@ if [ "$platform" != "miboot" ]; then
         text_start="-Ttext $link_address"
     fi
 #link everything
-    ${CROSS}ld -m $format -T $lds $text_start $pie -o "$ofile" \
+    ${CROSS}ld -m $format -T $lds $text_start $pie $nodl -o "$ofile" \
 	$platformo $tmp $object/wrapper.a
     rm $tmp
 fi
diff --git a/arch/powerpc/configs/amigaone_defconfig b/arch/powerpc/configs/amigaone_defconfig
index 8b83ce8a01e7..8d3e3c41258d 100644
--- a/arch/powerpc/configs/amigaone_defconfig
+++ b/arch/powerpc/configs/amigaone_defconfig
@@ -45,12 +45,6 @@ CONFIG_PARPORT_PC_FIFO=y
 CONFIG_BLK_DEV_FD=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-# CONFIG_IDEPCI_PCIBUS_ORDER is not set
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_SIIMAGE=y
-CONFIG_BLK_DEV_VIA82CXXX=y
 CONFIG_SCSI=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=y
@@ -61,6 +55,10 @@ CONFIG_SCSI_CONSTANTS=y
 CONFIG_SCSI_SYM53C8XX_2=y
 CONFIG_SCSI_SYM53C8XX_DMA_ADDRESSING_MODE=0
 # CONFIG_SCSI_SYM53C8XX_MMIO is not set
+CONFIG_ATA=y
+CONFIG_PATA_SIL680=y
+CONFIG_PATA_VIA=y
+CONFIG_ATA_GENERIC=y
 CONFIG_NETDEVICES=y
 CONFIG_VORTEX=y
 CONFIG_8139CP=y
diff --git a/arch/powerpc/configs/cell_defconfig b/arch/powerpc/configs/cell_defconfig
index 7b6f30dece34..2d7fcbe047ac 100644
--- a/arch/powerpc/configs/cell_defconfig
+++ b/arch/powerpc/configs/cell_defconfig
@@ -108,16 +108,15 @@ CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=131072
-CONFIG_IDE=y
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_AEC62XX=y
-CONFIG_BLK_DEV_SIIMAGE=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_BLK_DEV_SR=m
 CONFIG_CHR_DEV_SG=y
 CONFIG_ATA=y
 CONFIG_SATA_PROMISE=y
+CONFIG_PATA_ARTOP=y
 CONFIG_PATA_PDC2027X=m
+CONFIG_PATA_SIL680=y
+CONFIG_ATA_GENERIC=y
 CONFIG_MD=y
 CONFIG_BLK_DEV_MD=m
 CONFIG_MD_LINEAR=m
diff --git a/arch/powerpc/configs/chrp32_defconfig b/arch/powerpc/configs/chrp32_defconfig
index ac9a50da2dc6..1f6f90cd8aff 100644
--- a/arch/powerpc/configs/chrp32_defconfig
+++ b/arch/powerpc/configs/chrp32_defconfig
@@ -42,12 +42,6 @@ CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_BLK_DEV_FD=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_SL82C105=y
-CONFIG_BLK_DEV_VIA82CXXX=y
-CONFIG_SCSI=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=y
 CONFIG_BLK_DEV_SR=y
@@ -56,6 +50,10 @@ CONFIG_CHR_DEV_SG=y
 CONFIG_SCSI_CONSTANTS=y
 CONFIG_SCSI_SYM53C8XX_2=y
 CONFIG_SCSI_SYM53C8XX_DMA_ADDRESSING_MODE=0
+CONFIG_ATA=y
+CONFIG_PATA_VIA=y
+CONFIG_PATA_WINBOND=y
+CONFIG_ATA_GENERIC=y
 CONFIG_NETDEVICES=y
 CONFIG_PCNET32=y
 CONFIG_NET_TULIP=y
diff --git a/arch/powerpc/configs/fsl-emb-nonhw.config b/arch/powerpc/configs/fsl-emb-nonhw.config
index 1a61e81ab0cd..cc49c95494da 100644
--- a/arch/powerpc/configs/fsl-emb-nonhw.config
+++ b/arch/powerpc/configs/fsl-emb-nonhw.config
@@ -44,6 +44,7 @@ CONFIG_FORCE_MAX_ZONEORDER=13
 CONFIG_FRAMEBUFFER_CONSOLE=y
 CONFIG_FRAME_WARN=1024
 CONFIG_FTL=y
+CONFIG_GPIO_GENERIC_PLATFORM=y
 CONFIG_HFS_FS=m
 CONFIG_HFSPLUS_FS=m
 CONFIG_HIGH_RES_TIMERS=y
@@ -104,8 +105,13 @@ CONFIG_PACKET=y
 CONFIG_PARTITION_ADVANCED=y
 CONFIG_PERF_EVENTS=y
 CONFIG_POSIX_MQUEUE=y
+CONFIG_POWER_SUPPLY=y
+CONFIG_POWER_RESET=y
+CONFIG_POWER_RESET_GPIO=y
+CONFIG_POWER_RESET_GPIO_RESTART=y
 CONFIG_QNX4FS_FS=m
 CONFIG_RCU_TRACE=y
+CONFIG_RESET_CONTROLLER=y
 CONFIG_ROOT_NFS=y
 CONFIG_SYSV_FS=m
 CONFIG_SYSVIPC=y
diff --git a/arch/powerpc/configs/g5_defconfig b/arch/powerpc/configs/g5_defconfig
index 3b2511c090d8..e18f2e06553f 100644
--- a/arch/powerpc/configs/g5_defconfig
+++ b/arch/powerpc/configs/g5_defconfig
@@ -60,10 +60,6 @@ CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=65536
 CONFIG_CDROM_PKTCDVD=m
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_BLK_DEV_IDE_PMAC=y
-CONFIG_BLK_DEV_IDE_PMAC_ATA100FIRST=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=y
 CONFIG_BLK_DEV_SR=y
@@ -73,6 +69,7 @@ CONFIG_SCSI_CONSTANTS=y
 CONFIG_SCSI_SPI_ATTRS=y
 CONFIG_ATA=y
 CONFIG_SATA_SVW=y
+CONFIG_PATA_MACIO=y
 CONFIG_MD=y
 CONFIG_BLK_DEV_MD=y
 CONFIG_MD_LINEAR=y
diff --git a/arch/powerpc/configs/maple_defconfig b/arch/powerpc/configs/maple_defconfig
index 27abfab31219..c4018179e219 100644
--- a/arch/powerpc/configs/maple_defconfig
+++ b/arch/powerpc/configs/maple_defconfig
@@ -39,16 +39,15 @@ CONFIG_IP_PNP_DHCP=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=8192
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_IDE_TASK_IOCTL=y
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_AMD74XX=y
 # CONFIG_SCSI_PROC_FS is not set
 CONFIG_BLK_DEV_SD=y
+CONFIG_BLK_DEV_SR=y
+CONFIG_BLK_DEV_SR_VENDOR=y
 CONFIG_CHR_DEV_SG=y
 CONFIG_SCSI_IPR=y
 CONFIG_ATA=y
+CONFIG_PATA_AMD=y
+CONFIG_ATA_GENERIC=y
 CONFIG_NETDEVICES=y
 CONFIG_AMD8111_ETH=y
 CONFIG_TIGON3=y
diff --git a/arch/powerpc/configs/pasemi_defconfig b/arch/powerpc/configs/pasemi_defconfig
index 76f4edd441d3..5553c5ce4274 100644
--- a/arch/powerpc/configs/pasemi_defconfig
+++ b/arch/powerpc/configs/pasemi_defconfig
@@ -58,9 +58,6 @@ CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=16384
 CONFIG_EEPROM_LEGACY=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_IDE_TASK_IOCTL=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=y
 CONFIG_CHR_DEV_OSST=y
diff --git a/arch/powerpc/configs/pmac32_defconfig b/arch/powerpc/configs/pmac32_defconfig
index e5a674d4a716..fc1e7a7388b8 100644
--- a/arch/powerpc/configs/pmac32_defconfig
+++ b/arch/powerpc/configs/pmac32_defconfig
@@ -117,15 +117,6 @@ CONFIG_CONNECTOR=y
 CONFIG_MAC_FLOPPY=m
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECS=m
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_PDC202XX_NEW=y
-CONFIG_BLK_DEV_SL82C105=y
-CONFIG_BLK_DEV_IDE_PMAC=y
-CONFIG_BLK_DEV_IDE_PMAC_ATA100FIRST=y
-CONFIG_SCSI=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=y
 CONFIG_BLK_DEV_SR=y
@@ -140,6 +131,12 @@ CONFIG_SCSI_SYM53C8XX_2=y
 CONFIG_SCSI_SYM53C8XX_DMA_ADDRESSING_MODE=0
 CONFIG_SCSI_MESH=y
 CONFIG_SCSI_MAC53C94=y
+CONFIG_ATA=y
+CONFIG_PATA_MACIO=y
+CONFIG_PATA_PDC2027X=y
+CONFIG_PATA_WINBOND=y
+CONFIG_PATA_PCMCIA=m
+CONFIG_ATA_GENERIC=y
 CONFIG_MD=y
 CONFIG_BLK_DEV_MD=m
 CONFIG_MD_LINEAR=m
diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig
index d98b6eb3254f..e4d53fe5976a 100644
--- a/arch/powerpc/configs/powernv_defconfig
+++ b/arch/powerpc/configs/powernv_defconfig
@@ -49,6 +49,7 @@ CONFIG_BINFMT_MISC=m
 CONFIG_PPC_TRANSACTIONAL_MEM=y
 CONFIG_HOTPLUG_CPU=y
 CONFIG_KEXEC=y
+CONFIG_KEXEC_FILE=y
 CONFIG_IRQ_ALL_CPUS=y
 CONFIG_NUMA=y
 CONFIG_MEMORY_HOTPLUG=y
@@ -241,10 +242,6 @@ CONFIG_EXT2_FS_SECURITY=y
 CONFIG_EXT4_FS=y
 CONFIG_EXT4_FS_POSIX_ACL=y
 CONFIG_EXT4_FS_SECURITY=y
-CONFIG_REISERFS_FS=m
-CONFIG_REISERFS_FS_XATTR=y
-CONFIG_REISERFS_FS_POSIX_ACL=y
-CONFIG_REISERFS_FS_SECURITY=y
 CONFIG_JFS_FS=m
 CONFIG_JFS_POSIX_ACL=y
 CONFIG_JFS_SECURITY=y
@@ -300,7 +297,10 @@ CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_CCM=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_HMAC=y
+CONFIG_CRYPT_CRC32C_VPMSUM=m
+CONFIG_CRYPTO_MD5_PPC=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
+CONFIG_CRYPTO_SHA256=y
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
@@ -308,6 +308,7 @@ CONFIG_CRYPTO_BLOWFISH=m
 CONFIG_CRYPTO_CAST6=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
+CONFIG_CRYPTO_SHA1_PPC=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig
index 58a98d40086f..0396126ba6a8 100644
--- a/arch/powerpc/configs/ppc64_defconfig
+++ b/arch/powerpc/configs/ppc64_defconfig
@@ -46,6 +46,7 @@ CONFIG_HZ_100=y
 CONFIG_BINFMT_MISC=m
 CONFIG_PPC_TRANSACTIONAL_MEM=y
 CONFIG_KEXEC=y
+CONFIG_KEXEC_FILE=y
 CONFIG_CRASH_DUMP=y
 CONFIG_IRQ_ALL_CPUS=y
 CONFIG_MEMORY_HOTREMOVE=y
@@ -85,12 +86,6 @@ CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=65536
 CONFIG_VIRTIO_BLK=m
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_AMD74XX=y
-CONFIG_BLK_DEV_IDE_PMAC=y
-CONFIG_BLK_DEV_IDE_PMAC_ATA100FIRST=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=m
 CONFIG_BLK_DEV_SR=y
@@ -120,6 +115,9 @@ CONFIG_SATA_AHCI=y
 CONFIG_SATA_SIL24=y
 CONFIG_SATA_MV=y
 CONFIG_SATA_SVW=y
+CONFIG_PATA_AMD=y
+CONFIG_PATA_MACIO=y
+CONFIG_ATA_GENERIC=y
 CONFIG_MD=y
 CONFIG_BLK_DEV_MD=y
 CONFIG_MD_LINEAR=y
@@ -335,7 +333,10 @@ CONFIG_PPC_EARLY_DEBUG=y
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_HMAC=y
+CONFIG_CRYPT_CRC32C_VPMSUM=m
+CONFIG_CRYPTO_MD5_PPC=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
+CONFIG_CRYPTO_SHA256=y
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
@@ -343,6 +344,7 @@ CONFIG_CRYPTO_BLOWFISH=m
 CONFIG_CRYPTO_CAST6=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
+CONFIG_CRYPTO_SHA1_PPC=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
diff --git a/arch/powerpc/configs/ppc64e_defconfig b/arch/powerpc/configs/ppc64e_defconfig
index fd2edd650c20..11a3473f9e2e 100644
--- a/arch/powerpc/configs/ppc64e_defconfig
+++ b/arch/powerpc/configs/ppc64e_defconfig
@@ -59,10 +59,6 @@ CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=65536
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_AMD74XX=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=y
 CONFIG_BLK_DEV_SR=y
@@ -79,6 +75,8 @@ CONFIG_SCSI_DEBUG=m
 CONFIG_ATA=y
 CONFIG_SATA_SIL24=y
 CONFIG_SATA_SVW=y
+CONFIG_PATA_AMD=y
+CONFIG_ATA_GENERIC=y
 CONFIG_MD=y
 CONFIG_BLK_DEV_MD=y
 CONFIG_MD_LINEAR=y
diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig
index 8fbf49801233..3ce91a3df27f 100644
--- a/arch/powerpc/configs/ppc6xx_defconfig
+++ b/arch/powerpc/configs/ppc6xx_defconfig
@@ -378,13 +378,6 @@ CONFIG_EEPROM_AT24=m
 CONFIG_EEPROM_LEGACY=m
 CONFIG_EEPROM_MAX6875=m
 CONFIG_EEPROM_93CX6=m
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=m
-CONFIG_IDE_TASK_IOCTL=y
-# CONFIG_IDEPCI_PCIBUS_ORDER is not set
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_IDE_PMAC=y
-CONFIG_BLK_DEV_IDE_PMAC_ATA100FIRST=y
 CONFIG_RAID_ATTRS=m
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=m
@@ -411,13 +404,14 @@ CONFIG_ATA=y
 CONFIG_SATA_FSL=m
 CONFIG_PDC_ADMA=m
 CONFIG_ATA_PIIX=m
+CONFIG_PATA_MACIO=y
 CONFIG_PATA_MPC52xx=m
 CONFIG_PATA_OPTIDMA=m
 CONFIG_PATA_SCH=m
 CONFIG_PATA_VIA=m
 CONFIG_PATA_PLATFORM=m
 CONFIG_PATA_OF_PLATFORM=m
-CONFIG_ATA_GENERIC=m
+CONFIG_ATA_GENERIC=y
 CONFIG_MD=y
 CONFIG_BLK_DEV_MD=y
 CONFIG_MD_LINEAR=m
diff --git a/arch/powerpc/configs/pseries_defconfig b/arch/powerpc/configs/pseries_defconfig
index 8a3bc016b732..5a06bdde1674 100644
--- a/arch/powerpc/configs/pseries_defconfig
+++ b/arch/powerpc/configs/pseries_defconfig
@@ -52,6 +52,7 @@ CONFIG_HZ_100=y
 CONFIG_BINFMT_MISC=m
 CONFIG_PPC_TRANSACTIONAL_MEM=y
 CONFIG_KEXEC=y
+CONFIG_KEXEC_FILE=y
 CONFIG_IRQ_ALL_CPUS=y
 CONFIG_MEMORY_HOTPLUG=y
 CONFIG_MEMORY_HOTREMOVE=y
@@ -92,10 +93,6 @@ CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=65536
 CONFIG_VIRTIO_BLK=m
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_AMD74XX=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=m
 CONFIG_BLK_DEV_SR=y
@@ -122,7 +119,8 @@ CONFIG_SCSI_DH_RDAC=m
 CONFIG_SCSI_DH_ALUA=m
 CONFIG_ATA=y
 CONFIG_SATA_AHCI=y
-# CONFIG_ATA_SFF is not set
+CONFIG_PATA_AMD=y
+CONFIG_ATA_GENERIC=y
 CONFIG_MD=y
 CONFIG_BLK_DEV_MD=y
 CONFIG_MD_LINEAR=y
@@ -244,10 +242,6 @@ CONFIG_EXT2_FS_SECURITY=y
 CONFIG_EXT4_FS=y
 CONFIG_EXT4_FS_POSIX_ACL=y
 CONFIG_EXT4_FS_SECURITY=y
-CONFIG_REISERFS_FS=m
-CONFIG_REISERFS_FS_XATTR=y
-CONFIG_REISERFS_FS_POSIX_ACL=y
-CONFIG_REISERFS_FS_SECURITY=y
 CONFIG_JFS_FS=m
 CONFIG_JFS_POSIX_ACL=y
 CONFIG_JFS_SECURITY=y
@@ -302,7 +296,10 @@ CONFIG_XMON=y
 CONFIG_CRYPTO_TEST=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_HMAC=y
+CONFIG_CRYPT_CRC32C_VPMSUM=m
+CONFIG_CRYPTO_MD5_PPC=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
+CONFIG_CRYPTO_SHA256=y
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
@@ -310,6 +307,7 @@ CONFIG_CRYPTO_BLOWFISH=m
 CONFIG_CRYPTO_CAST6=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SALSA20=m
+CONFIG_CRYPTO_SHA1_PPC=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
diff --git a/arch/powerpc/configs/storcenter_defconfig b/arch/powerpc/configs/storcenter_defconfig
index e9122b15e5fd..74bca2eccd0f 100644
--- a/arch/powerpc/configs/storcenter_defconfig
+++ b/arch/powerpc/configs/storcenter_defconfig
@@ -36,12 +36,11 @@ CONFIG_NFTL_RW=y
 CONFIG_MTD_CFI=y
 CONFIG_MTD_CFI_AMDSTD=y
 CONFIG_MTD_PHYSMAP=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_VIA82CXXX=y
-CONFIG_SCSI=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_BLK_DEV_SR=y
 CONFIG_SCSI_SPI_ATTRS=y
+CONFIG_ATA=y
+CONFIG_PATA_VIA=y
 CONFIG_MD=y
 CONFIG_BLK_DEV_MD=y
 CONFIG_MD_LINEAR=y
diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile
index 7998c177f0a2..87f40454bad3 100644
--- a/arch/powerpc/crypto/Makefile
+++ b/arch/powerpc/crypto/Makefile
@@ -9,7 +9,7 @@ obj-$(CONFIG_CRYPTO_MD5_PPC) += md5-ppc.o
 obj-$(CONFIG_CRYPTO_SHA1_PPC) += sha1-powerpc.o
 obj-$(CONFIG_CRYPTO_SHA1_PPC_SPE) += sha1-ppc-spe.o
 obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o
-obj-$(CONFIG_CRYPT_CRC32C_VPMSUM) += crc32c-vpmsum.o
+obj-$(CONFIG_CRYPTO_CRC32C_VPMSUM) += crc32c-vpmsum.o
 
 aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o
 md5-ppc-y := md5-asm.o md5-glue.o
diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h
index e0baba1535e6..81592562e0f8 100644
--- a/arch/powerpc/include/asm/asm-prototypes.h
+++ b/arch/powerpc/include/asm/asm-prototypes.h
@@ -13,7 +13,6 @@
  */
 
 #include <linux/threads.h>
-#include <linux/kprobes.h>
 #include <asm/cacheflush.h>
 #include <asm/checksum.h>
 #include <asm/uaccess.h>
diff --git a/arch/powerpc/include/asm/book3s/32/pgalloc.h b/arch/powerpc/include/asm/book3s/32/pgalloc.h
index 8e21bb492dca..d310546e5d9d 100644
--- a/arch/powerpc/include/asm/book3s/32/pgalloc.h
+++ b/arch/powerpc/include/asm/book3s/32/pgalloc.h
@@ -2,14 +2,42 @@
 #define _ASM_POWERPC_BOOK3S_32_PGALLOC_H
 
 #include <linux/threads.h>
+#include <linux/slab.h>
 
-/* For 32-bit, all levels of page tables are just drawn from get_free_page() */
-#define MAX_PGTABLE_INDEX_SIZE	0
+/*
+ * Functions that deal with pagetables that could be at any level of
+ * the table need to be passed an "index_size" so they know how to
+ * handle allocation.  For PTE pages (which are linked to a struct
+ * page for now, and drawn from the main get_free_pages() pool), the
+ * allocation size will be (2^index_size * sizeof(pointer)) and
+ * allocations are drawn from the kmem_cache in PGT_CACHE(index_size).
+ *
+ * The maximum index size needs to be big enough to allow any
+ * pagetable sizes we need, but small enough to fit in the low bits of
+ * any page table pointer.  In other words all pagetables, even tiny
+ * ones, must be aligned to allow at least enough low 0 bits to
+ * contain this value.  This value is also used as a mask, so it must
+ * be one less than a power of two.
+ */
+#define MAX_PGTABLE_INDEX_SIZE	0xf
 
 extern void __bad_pte(pmd_t *pmd);
 
-extern pgd_t *pgd_alloc(struct mm_struct *mm);
-extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
+extern struct kmem_cache *pgtable_cache[];
+#define PGT_CACHE(shift) ({				\
+			BUG_ON(!(shift));		\
+			pgtable_cache[(shift) - 1];	\
+		})
+
+static inline pgd_t *pgd_alloc(struct mm_struct *mm)
+{
+	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), GFP_KERNEL);
+}
+
+static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
+{
+	kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd);
+}
 
 /*
  * We don't have any real pmd's, and this code never triggers because
@@ -68,8 +96,12 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
 
 static inline void pgtable_free(void *table, unsigned index_size)
 {
-	BUG_ON(index_size); /* 32-bit doesn't use this */
-	free_page((unsigned long)table);
+	if (!index_size) {
+		free_page((unsigned long)table);
+	} else {
+		BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE);
+		kmem_cache_free(PGT_CACHE(index_size), table);
+	}
 }
 
 #define check_pgt_cache()	do { } while (0)
diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h
index 6b8b2d57fdc8..012223638815 100644
--- a/arch/powerpc/include/asm/book3s/32/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/32/pgtable.h
@@ -8,6 +8,23 @@
 /* And here we include common definitions */
 #include <asm/pte-common.h>
 
+#define PTE_INDEX_SIZE	PTE_SHIFT
+#define PMD_INDEX_SIZE	0
+#define PUD_INDEX_SIZE	0
+#define PGD_INDEX_SIZE	(32 - PGDIR_SHIFT)
+
+#define PMD_CACHE_INDEX	PMD_INDEX_SIZE
+
+#ifndef __ASSEMBLY__
+#define PTE_TABLE_SIZE	(sizeof(pte_t) << PTE_INDEX_SIZE)
+#define PMD_TABLE_SIZE	0
+#define PUD_TABLE_SIZE	0
+#define PGD_TABLE_SIZE	(sizeof(pgd_t) << PGD_INDEX_SIZE)
+#endif	/* __ASSEMBLY__ */
+
+#define PTRS_PER_PTE	(1 << PTE_INDEX_SIZE)
+#define PTRS_PER_PGD	(1 << PGD_INDEX_SIZE)
+
 /*
  * The normal case is that PTEs are 32-bits and we have a 1-page
  * 1024-entry pgdir pointing to 1-page 1024-entry PTE pages.  -- paulus
@@ -19,14 +36,10 @@
  * -Matt
  */
 /* PGDIR_SHIFT determines what a top-level page table entry can map */
-#define PGDIR_SHIFT	(PAGE_SHIFT + PTE_SHIFT)
+#define PGDIR_SHIFT	(PAGE_SHIFT + PTE_INDEX_SIZE)
 #define PGDIR_SIZE	(1UL << PGDIR_SHIFT)
 #define PGDIR_MASK	(~(PGDIR_SIZE-1))
 
-#define PTRS_PER_PTE	(1 << PTE_SHIFT)
-#define PTRS_PER_PMD	1
-#define PTRS_PER_PGD	(1 << (32 - PGDIR_SHIFT))
-
 #define USER_PTRS_PER_PGD	(TASK_SIZE / PGDIR_SIZE)
 /*
  * This is the bottom of the PKMAP area with HIGHMEM or an arbitrary
@@ -82,12 +95,8 @@
 
 extern unsigned long ioremap_bot;
 
-/*
- * entries per page directory level: our page-table tree is two-level, so
- * we don't really have any PMD directory.
- */
-#define PTE_TABLE_SIZE	(sizeof(pte_t) << PTE_SHIFT)
-#define PGD_TABLE_SIZE	(sizeof(pgd_t) << (32 - PGDIR_SHIFT))
+/* Bits to mask out from a PGD to get to the PUD page */
+#define PGD_MASKED_BITS		0
 
 #define pte_ERROR(e) \
 	pr_err("%s:%d: bad pte %llx.\n", __FILE__, __LINE__, \
@@ -224,7 +233,8 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
 
 
 static inline void __ptep_set_access_flags(struct mm_struct *mm,
-					   pte_t *ptep, pte_t entry)
+					   pte_t *ptep, pte_t entry,
+					   unsigned long address)
 {
 	unsigned long set = pte_val(entry) &
 		(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC);
@@ -283,15 +293,6 @@ static inline void __ptep_set_access_flags(struct mm_struct *mm,
 #define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) >> 3 })
 #define __swp_entry_to_pte(x)		((pte_t) { (x).val << 3 })
 
-#ifndef CONFIG_PPC_4K_PAGES
-void pgtable_cache_init(void);
-#else
-/*
- * No page table caches to initialise
- */
-#define pgtable_cache_init()	do { } while (0)
-#endif
-
 extern int get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep,
 		      pmd_t **pmdp);
 
diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h
index 1af837c561ba..1c64bc6330bc 100644
--- a/arch/powerpc/include/asm/book3s/64/hash-4k.h
+++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h
@@ -16,9 +16,6 @@
 #define H_PUD_TABLE_SIZE	(sizeof(pud_t) << H_PUD_INDEX_SIZE)
 #define H_PGD_TABLE_SIZE	(sizeof(pgd_t) << H_PGD_INDEX_SIZE)
 
-/* With 4k base page size, hugepage PTEs go at the PMD level */
-#define MIN_HUGEPTE_SHIFT	PMD_SHIFT
-
 /* PTE flags to conserve for HPTE identification */
 #define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | \
 			 H_PAGE_F_SECOND | H_PAGE_F_GIX)
diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h
index 5aae4f530c21..f3dd21efa2ea 100644
--- a/arch/powerpc/include/asm/book3s/64/hash-64k.h
+++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h
@@ -6,9 +6,6 @@
 #define H_PUD_INDEX_SIZE  5
 #define H_PGD_INDEX_SIZE  12
 
-/* With 4k base page size, hugepage PTEs go at the PMD level */
-#define MIN_HUGEPTE_SHIFT	PAGE_SHIFT
-
 #define H_PAGE_COMBO	0x00001000 /* this is a combo 4k page */
 #define H_PAGE_4K_PFN	0x00002000 /* PFN is for a single 4k page */
 /*
diff --git a/arch/powerpc/include/asm/book3s/64/hugetlb-radix.h b/arch/powerpc/include/asm/book3s/64/hugetlb.h
index c45189aa7476..c62f14d0bec1 100644
--- a/arch/powerpc/include/asm/book3s/64/hugetlb-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/hugetlb.h
@@ -1,5 +1,5 @@
-#ifndef _ASM_POWERPC_BOOK3S_64_HUGETLB_RADIX_H
-#define _ASM_POWERPC_BOOK3S_64_HUGETLB_RADIX_H
+#ifndef _ASM_POWERPC_BOOK3S_64_HUGETLB_H
+#define _ASM_POWERPC_BOOK3S_64_HUGETLB_H
 /*
  * For radix we want generic code to handle hugetlb. But then if we want
  * both hash and radix to be enabled together we need to workaround the
@@ -21,9 +21,33 @@ static inline int hstate_get_psize(struct hstate *hstate)
 		return MMU_PAGE_2M;
 	else if (shift == mmu_psize_defs[MMU_PAGE_1G].shift)
 		return MMU_PAGE_1G;
+	else if (shift == mmu_psize_defs[MMU_PAGE_16M].shift)
+		return MMU_PAGE_16M;
+	else if (shift == mmu_psize_defs[MMU_PAGE_16G].shift)
+		return MMU_PAGE_16G;
 	else {
 		WARN(1, "Wrong huge page shift\n");
 		return mmu_virtual_psize;
 	}
 }
+
+#define arch_make_huge_pte arch_make_huge_pte
+static inline pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
+				       struct page *page, int writable)
+{
+	unsigned long page_shift;
+
+	if (!cpu_has_feature(CPU_FTR_POWER9_DD1))
+		return entry;
+
+	page_shift = huge_page_shift(hstate_vma(vma));
+	/*
+	 * We don't support 1G hugetlb pages yet.
+	 */
+	VM_WARN_ON(page_shift == mmu_psize_defs[MMU_PAGE_1G].shift);
+	if (page_shift == mmu_psize_defs[MMU_PAGE_2M].shift)
+		return __pte(pte_val(entry) | _PAGE_LARGE);
+	else
+		return entry;
+}
 #endif
diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
index e407af2b7333..2e6a823fa502 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
@@ -70,7 +70,9 @@
 
 #define HPTE_V_SSIZE_SHIFT	62
 #define HPTE_V_AVPN_SHIFT	7
+#define HPTE_V_COMMON_BITS	ASM_CONST(0x000fffffffffffff)
 #define HPTE_V_AVPN		ASM_CONST(0x3fffffffffffff80)
+#define HPTE_V_AVPN_3_0		ASM_CONST(0x000fffffffffff80)
 #define HPTE_V_AVPN_VAL(x)	(((x) & HPTE_V_AVPN) >> HPTE_V_AVPN_SHIFT)
 #define HPTE_V_COMPARE(x,y)	(!(((x) ^ (y)) & 0xffffffffffffff80UL))
 #define HPTE_V_BOLTED		ASM_CONST(0x0000000000000010)
@@ -80,14 +82,16 @@
 #define HPTE_V_VALID		ASM_CONST(0x0000000000000001)
 
 /*
- * ISA 3.0 have a different HPTE format.
+ * ISA 3.0 has a different HPTE format.
  */
 #define HPTE_R_3_0_SSIZE_SHIFT	58
+#define HPTE_R_3_0_SSIZE_MASK	(3ull << HPTE_R_3_0_SSIZE_SHIFT)
 #define HPTE_R_PP0		ASM_CONST(0x8000000000000000)
 #define HPTE_R_TS		ASM_CONST(0x4000000000000000)
 #define HPTE_R_KEY_HI		ASM_CONST(0x3000000000000000)
 #define HPTE_R_RPN_SHIFT	12
 #define HPTE_R_RPN		ASM_CONST(0x0ffffffffffff000)
+#define HPTE_R_RPN_3_0		ASM_CONST(0x01fffffffffff000)
 #define HPTE_R_PP		ASM_CONST(0x0000000000000003)
 #define HPTE_R_PPP		ASM_CONST(0x8000000000000003)
 #define HPTE_R_N		ASM_CONST(0x0000000000000004)
@@ -316,12 +320,43 @@ static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize,
 	 */
 	v = (vpn >> (23 - VPN_SHIFT)) & ~(mmu_psize_defs[psize].avpnm);
 	v <<= HPTE_V_AVPN_SHIFT;
-	if (!cpu_has_feature(CPU_FTR_ARCH_300))
-		v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT;
+	v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT;
 	return v;
 }
 
 /*
+ * ISA v3.0 defines a new HPTE format, which differs from the old
+ * format in having smaller AVPN and ARPN fields, and the B field
+ * in the second dword instead of the first.
+ */
+static inline unsigned long hpte_old_to_new_v(unsigned long v)
+{
+	/* trim AVPN, drop B */
+	return v & HPTE_V_COMMON_BITS;
+}
+
+static inline unsigned long hpte_old_to_new_r(unsigned long v, unsigned long r)
+{
+	/* move B field from 1st to 2nd dword, trim ARPN */
+	return (r & ~HPTE_R_3_0_SSIZE_MASK) |
+		(((v) >> HPTE_V_SSIZE_SHIFT) << HPTE_R_3_0_SSIZE_SHIFT);
+}
+
+static inline unsigned long hpte_new_to_old_v(unsigned long v, unsigned long r)
+{
+	/* insert B field */
+	return (v & HPTE_V_COMMON_BITS) |
+		((r & HPTE_R_3_0_SSIZE_MASK) <<
+		 (HPTE_V_SSIZE_SHIFT - HPTE_R_3_0_SSIZE_SHIFT));
+}
+
+static inline unsigned long hpte_new_to_old_r(unsigned long r)
+{
+	/* clear out B field */
+	return r & ~HPTE_R_3_0_SSIZE_MASK;
+}
+
+/*
  * This function sets the AVPN and L fields of the HPTE  appropriately
  * using the base page size and actual page size.
  */
@@ -341,12 +376,8 @@ static inline unsigned long hpte_encode_v(unsigned long vpn, int base_psize,
  * aligned for the requested page size
  */
 static inline unsigned long hpte_encode_r(unsigned long pa, int base_psize,
-					  int actual_psize, int ssize)
+					  int actual_psize)
 {
-
-	if (cpu_has_feature(CPU_FTR_ARCH_300))
-		pa |= ((unsigned long) ssize) << HPTE_R_3_0_SSIZE_SHIFT;
-
 	/* A 4K page needs no special encoding */
 	if (actual_psize == MMU_PAGE_4K)
 		return pa & HPTE_R_RPN;
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index 0ebfbc8f0449..5905f0ff57d1 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -26,6 +26,11 @@
 #define _RPAGE_SW1		0x00800
 #define _RPAGE_SW2		0x00400
 #define _RPAGE_SW3		0x00200
+#define _RPAGE_RSV1		0x1000000000000000UL
+#define _RPAGE_RSV2		0x0800000000000000UL
+#define _RPAGE_RSV3		0x0400000000000000UL
+#define _RPAGE_RSV4		0x0200000000000000UL
+
 #ifdef CONFIG_MEM_SOFT_DIRTY
 #define _PAGE_SOFT_DIRTY	_RPAGE_SW3 /* software: software dirty tracking */
 #else
@@ -33,6 +38,11 @@
 #endif
 #define _PAGE_SPECIAL		_RPAGE_SW2 /* software: special page */
 
+/*
+ * For P9 DD1 only, we need to track whether the pte's huge.
+ */
+#define _PAGE_LARGE	_RPAGE_RSV1
+
 
 #define _PAGE_PTE		(1ul << 62)	/* distinguishes PTEs from pointers */
 #define _PAGE_PRESENT		(1ul << 63)	/* pte contains a translation */
@@ -568,10 +578,11 @@ static inline bool check_pte_access(unsigned long access, unsigned long ptev)
  */
 
 static inline void __ptep_set_access_flags(struct mm_struct *mm,
-					   pte_t *ptep, pte_t entry)
+					   pte_t *ptep, pte_t entry,
+					   unsigned long address)
 {
 	if (radix_enabled())
-		return radix__ptep_set_access_flags(mm, ptep, entry);
+		return radix__ptep_set_access_flags(mm, ptep, entry, address);
 	return hash__ptep_set_access_flags(ptep, entry);
 }
 
@@ -789,9 +800,6 @@ extern struct page *pgd_page(pgd_t pgd);
 #define pgd_ERROR(e) \
 	pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
 
-void pgtable_cache_add(unsigned shift, void (*ctor)(void *));
-void pgtable_cache_init(void);
-
 static inline int map_kernel_page(unsigned long ea, unsigned long pa,
 				  unsigned long flags)
 {
diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h
index 2a46dea8e1b1..b4d1302387a3 100644
--- a/arch/powerpc/include/asm/book3s/64/radix.h
+++ b/arch/powerpc/include/asm/book3s/64/radix.h
@@ -140,19 +140,20 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,
 		unsigned long new_pte;
 
 		old_pte = __radix_pte_update(ptep, ~0, 0);
-		asm volatile("ptesync" : : : "memory");
 		/*
 		 * new value of pte
 		 */
 		new_pte = (old_pte | set) & ~clr;
-
 		/*
-		 * For now let's do heavy pid flush
-		 * radix__flush_tlb_page_psize(mm, addr, mmu_virtual_psize);
+		 * If we are trying to clear the pte, we can skip
+		 * the below sequence and batch the tlb flush. The
+		 * tlb flush batching is done by mmu gather code
 		 */
-		radix__flush_tlb_mm(mm);
-
-		__radix_pte_update(ptep, 0, new_pte);
+		if (new_pte) {
+			asm volatile("ptesync" : : : "memory");
+			radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr);
+			__radix_pte_update(ptep, 0, new_pte);
+		}
 	} else
 		old_pte = __radix_pte_update(ptep, clr, set);
 	asm volatile("ptesync" : : : "memory");
@@ -167,7 +168,8 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm,
  * function doesn't need to invalidate tlb.
  */
 static inline void radix__ptep_set_access_flags(struct mm_struct *mm,
-						pte_t *ptep, pte_t entry)
+						pte_t *ptep, pte_t entry,
+						unsigned long address)
 {
 
 	unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
@@ -183,13 +185,7 @@ static inline void radix__ptep_set_access_flags(struct mm_struct *mm,
 		 * new value of pte
 		 */
 		new_pte = old_pte | set;
-
-		/*
-		 * For now let's do heavy pid flush
-		 * radix__flush_tlb_page_psize(mm, addr, mmu_virtual_psize);
-		 */
-		radix__flush_tlb_mm(mm);
-
+		radix__flush_tlb_pte_p9_dd1(old_pte, mm, address);
 		__radix_pte_update(ptep, 0, new_pte);
 	} else
 		__radix_pte_update(ptep, 0, set);
@@ -243,6 +239,8 @@ static inline int radix__pmd_trans_huge(pmd_t pmd)
 
 static inline pmd_t radix__pmd_mkhuge(pmd_t pmd)
 {
+	if (cpu_has_feature(CPU_FTR_POWER9_DD1))
+		return __pmd(pmd_val(pmd) | _PAGE_PTE | _PAGE_LARGE);
 	return __pmd(pmd_val(pmd) | _PAGE_PTE);
 }
 static inline void radix__pmdp_huge_split_prepare(struct vm_area_struct *vma,
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index a9e19cb2f7c5..cc7fbde4f53c 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -42,4 +42,6 @@ extern void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
 				     unsigned long page_size);
 extern void radix__flush_tlb_lpid(unsigned long lpid);
 extern void radix__flush_tlb_all(void);
+extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
+					unsigned long address);
 #endif
diff --git a/arch/powerpc/include/asm/cache.h b/arch/powerpc/include/asm/cache.h
index ffbafbf76b19..7657aa897a38 100644
--- a/arch/powerpc/include/asm/cache.h
+++ b/arch/powerpc/include/asm/cache.h
@@ -20,12 +20,15 @@
 #endif
 #else /* CONFIG_PPC64 */
 #define L1_CACHE_SHIFT		7
+#define IFETCH_ALIGN_SHIFT	4 /* POWER8,9 */
 #endif
 
 #define	L1_CACHE_BYTES		(1 << L1_CACHE_SHIFT)
 
 #define	SMP_CACHE_BYTES		L1_CACHE_BYTES
 
+#define IFETCH_ALIGN_BYTES	(1 << IFETCH_ALIGN_SHIFT)
+
 #if defined(__powerpc64__) && !defined(__ASSEMBLY__)
 struct ppc64_caches {
 	u32	dsize;			/* L1 d-cache size */
diff --git a/arch/powerpc/include/asm/cmpxchg.h b/arch/powerpc/include/asm/cmpxchg.h
index 44efe739b6b9..fc46b664c49e 100644
--- a/arch/powerpc/include/asm/cmpxchg.h
+++ b/arch/powerpc/include/asm/cmpxchg.h
@@ -7,6 +7,71 @@
 #include <asm/asm-compat.h>
 #include <linux/bug.h>
 
+#ifdef __BIG_ENDIAN
+#define BITOFF_CAL(size, off)	((sizeof(u32) - size - off) * BITS_PER_BYTE)
+#else
+#define BITOFF_CAL(size, off)	(off * BITS_PER_BYTE)
+#endif
+
+#define XCHG_GEN(type, sfx, cl)				\
+static inline u32 __xchg_##type##sfx(volatile void *p, u32 val)	\
+{								\
+	unsigned int prev, prev_mask, tmp, bitoff, off;		\
+								\
+	off = (unsigned long)p % sizeof(u32);			\
+	bitoff = BITOFF_CAL(sizeof(type), off);			\
+	p -= off;						\
+	val <<= bitoff;						\
+	prev_mask = (u32)(type)-1 << bitoff;			\
+								\
+	__asm__ __volatile__(					\
+"1:	lwarx   %0,0,%3\n"					\
+"	andc	%1,%0,%5\n"					\
+"	or	%1,%1,%4\n"					\
+	PPC405_ERR77(0,%3)					\
+"	stwcx.	%1,0,%3\n"					\
+"	bne-	1b\n"						\
+	: "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p)		\
+	: "r" (p), "r" (val), "r" (prev_mask)			\
+	: "cc", cl);						\
+								\
+	return prev >> bitoff;					\
+}
+
+#define CMPXCHG_GEN(type, sfx, br, br2, cl)			\
+static inline							\
+u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new)	\
+{								\
+	unsigned int prev, prev_mask, tmp, bitoff, off;		\
+								\
+	off = (unsigned long)p % sizeof(u32);			\
+	bitoff = BITOFF_CAL(sizeof(type), off);			\
+	p -= off;						\
+	old <<= bitoff;						\
+	new <<= bitoff;						\
+	prev_mask = (u32)(type)-1 << bitoff;			\
+								\
+	__asm__ __volatile__(					\
+	br							\
+"1:	lwarx   %0,0,%3\n"					\
+"	and	%1,%0,%6\n"					\
+"	cmpw	0,%1,%4\n"					\
+"	bne-	2f\n"						\
+"	andc	%1,%0,%6\n"					\
+"	or	%1,%1,%5\n"					\
+	PPC405_ERR77(0,%3)					\
+"	stwcx.  %1,0,%3\n"					\
+"	bne-    1b\n"						\
+	br2							\
+	"\n"							\
+"2:"								\
+	: "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p)		\
+	: "r" (p), "r" (old), "r" (new), "r" (prev_mask)	\
+	: "cc", cl);						\
+								\
+	return prev >> bitoff;					\
+}
+
 /*
  * Atomic exchange
  *
@@ -14,6 +79,11 @@
  * the previous value stored there.
  */
 
+XCHG_GEN(u8, _local, "memory");
+XCHG_GEN(u8, _relaxed, "cc");
+XCHG_GEN(u16, _local, "memory");
+XCHG_GEN(u16, _relaxed, "cc");
+
 static __always_inline unsigned long
 __xchg_u32_local(volatile void *p, unsigned long val)
 {
@@ -85,9 +155,13 @@ __xchg_u64_relaxed(u64 *p, unsigned long val)
 #endif
 
 static __always_inline unsigned long
-__xchg_local(volatile void *ptr, unsigned long x, unsigned int size)
+__xchg_local(void *ptr, unsigned long x, unsigned int size)
 {
 	switch (size) {
+	case 1:
+		return __xchg_u8_local(ptr, x);
+	case 2:
+		return __xchg_u16_local(ptr, x);
 	case 4:
 		return __xchg_u32_local(ptr, x);
 #ifdef CONFIG_PPC64
@@ -103,6 +177,10 @@ static __always_inline unsigned long
 __xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
 {
 	switch (size) {
+	case 1:
+		return __xchg_u8_relaxed(ptr, x);
+	case 2:
+		return __xchg_u16_relaxed(ptr, x);
 	case 4:
 		return __xchg_u32_relaxed(ptr, x);
 #ifdef CONFIG_PPC64
@@ -131,6 +209,15 @@ __xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
  * and return the old value of *p.
  */
 
+CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
+CMPXCHG_GEN(u8, _local, , , "memory");
+CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
+CMPXCHG_GEN(u8, _relaxed, , , "cc");
+CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
+CMPXCHG_GEN(u16, _local, , , "memory");
+CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
+CMPXCHG_GEN(u16, _relaxed, , , "cc");
+
 static __always_inline unsigned long
 __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
 {
@@ -316,6 +403,10 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
 	  unsigned int size)
 {
 	switch (size) {
+	case 1:
+		return __cmpxchg_u8(ptr, old, new);
+	case 2:
+		return __cmpxchg_u16(ptr, old, new);
 	case 4:
 		return __cmpxchg_u32(ptr, old, new);
 #ifdef CONFIG_PPC64
@@ -328,10 +419,14 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
 }
 
 static __always_inline unsigned long
-__cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
+__cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
 	  unsigned int size)
 {
 	switch (size) {
+	case 1:
+		return __cmpxchg_u8_local(ptr, old, new);
+	case 2:
+		return __cmpxchg_u16_local(ptr, old, new);
 	case 4:
 		return __cmpxchg_u32_local(ptr, old, new);
 #ifdef CONFIG_PPC64
@@ -348,6 +443,10 @@ __cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
 		  unsigned int size)
 {
 	switch (size) {
+	case 1:
+		return __cmpxchg_u8_relaxed(ptr, old, new);
+	case 2:
+		return __cmpxchg_u16_relaxed(ptr, old, new);
 	case 4:
 		return __cmpxchg_u32_relaxed(ptr, old, new);
 #ifdef CONFIG_PPC64
@@ -364,6 +463,10 @@ __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
 		  unsigned int size)
 {
 	switch (size) {
+	case 1:
+		return __cmpxchg_u8_acquire(ptr, old, new);
+	case 2:
+		return __cmpxchg_u16_acquire(ptr, old, new);
 	case 4:
 		return __cmpxchg_u32_acquire(ptr, old, new);
 #ifdef CONFIG_PPC64
diff --git a/arch/powerpc/include/asm/debug.h b/arch/powerpc/include/asm/debug.h
index a954e4975049..86308f177f2d 100644
--- a/arch/powerpc/include/asm/debug.h
+++ b/arch/powerpc/include/asm/debug.h
@@ -10,7 +10,7 @@ struct pt_regs;
 
 extern struct dentry *powerpc_debugfs_root;
 
-#if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC)
+#if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC_CORE)
 
 extern int (*__debugger)(struct pt_regs *regs);
 extern int (*__debugger_ipi)(struct pt_regs *regs);
diff --git a/arch/powerpc/include/asm/futex.h b/arch/powerpc/include/asm/futex.h
index 2a9cf845473b..eaada6c92344 100644
--- a/arch/powerpc/include/asm/futex.h
+++ b/arch/powerpc/include/asm/futex.h
@@ -23,10 +23,8 @@
 "4:	li	%1,%3\n" \
 	"b	3b\n" \
 	".previous\n" \
-	".section __ex_table,\"a\"\n" \
-	".align 3\n" \
-	PPC_LONG "1b,4b,2b,4b\n" \
-	".previous" \
+	EX_TABLE(1b, 4b) \
+	EX_TABLE(2b, 4b) \
 	: "=&r" (oldval), "=&r" (ret) \
 	: "b" (uaddr), "i" (-EFAULT), "r" (oparg) \
 	: "cr0", "memory")
@@ -104,11 +102,9 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
 "3:	.section .fixup,\"ax\"\n\
 4:	li	%0,%6\n\
 	b	3b\n\
-	.previous\n\
-	.section __ex_table,\"a\"\n\
-	.align 3\n\
-	" PPC_LONG "1b,4b,2b,4b\n\
-	.previous" \
+	.previous\n"
+	EX_TABLE(1b, 4b)
+	EX_TABLE(2b, 4b)
         : "+r" (ret), "=&r" (prev), "+m" (*uaddr)
         : "r" (uaddr), "r" (oldval), "r" (newval), "i" (-EFAULT)
         : "cc", "memory");
diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h
index ab90c2fa1ea6..fca7033839a9 100644
--- a/arch/powerpc/include/asm/head-64.h
+++ b/arch/powerpc/include/asm/head-64.h
@@ -95,12 +95,12 @@ end_##sname:
 
 #define __FIXED_SECTION_ENTRY_BEGIN(sname, name, __align)	\
 	USE_FIXED_SECTION(sname);				\
-	.align __align;						\
+	.balign __align;					\
 	.global name;						\
 name:
 
 #define FIXED_SECTION_ENTRY_BEGIN(sname, name)			\
-	__FIXED_SECTION_ENTRY_BEGIN(sname, name, 0)
+	__FIXED_SECTION_ENTRY_BEGIN(sname, name, IFETCH_ALIGN_BYTES)
 
 #define FIXED_SECTION_ENTRY_BEGIN_LOCATION(sname, name, start)		\
 	USE_FIXED_SECTION(sname);				\
@@ -203,9 +203,9 @@ name:
 #define EXC_VIRT_END(name, start, end)			\
 	FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, end)
 
-#define EXC_COMMON_BEGIN(name)					\
+#define EXC_COMMON_BEGIN(name)						\
 	USE_TEXT_SECTION();						\
-	.align	7;							\
+	.balign IFETCH_ALIGN_BYTES;					\
 	.global name;							\
 	DEFINE_FIXED_SYMBOL(name);					\
 name:
diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h
index c5517f463ec7..ede215167d1a 100644
--- a/arch/powerpc/include/asm/hugetlb.h
+++ b/arch/powerpc/include/asm/hugetlb.h
@@ -9,7 +9,7 @@ extern struct kmem_cache *hugepte_cache;
 
 #ifdef CONFIG_PPC_BOOK3S_64
 
-#include <asm/book3s/64/hugetlb-radix.h>
+#include <asm/book3s/64/hugetlb.h>
 /*
  * This should work for other subarchs too. But right now we use the
  * new format only for 64bit book3s
@@ -51,12 +51,20 @@ static inline void __local_flush_hugetlb_page(struct vm_area_struct *vma,
 static inline pte_t *hugepd_page(hugepd_t hpd)
 {
 	BUG_ON(!hugepd_ok(hpd));
+#ifdef CONFIG_PPC_8xx
+	return (pte_t *)__va(hpd.pd & ~(_PMD_PAGE_MASK | _PMD_PRESENT_MASK));
+#else
 	return (pte_t *)((hpd.pd & ~HUGEPD_SHIFT_MASK) | PD_HUGE);
+#endif
 }
 
 static inline unsigned int hugepd_shift(hugepd_t hpd)
 {
+#ifdef CONFIG_PPC_8xx
+	return ((hpd.pd & _PMD_PAGE_MASK) >> 1) + 17;
+#else
 	return hpd.pd & HUGEPD_SHIFT_MASK;
+#endif
 }
 
 #endif /* CONFIG_PPC_BOOK3S_64 */
@@ -99,7 +107,15 @@ static inline int is_hugepage_only_range(struct mm_struct *mm,
 
 void book3e_hugetlb_preload(struct vm_area_struct *vma, unsigned long ea,
 			    pte_t pte);
+#ifdef CONFIG_PPC_8xx
+static inline void flush_hugetlb_page(struct vm_area_struct *vma,
+				      unsigned long vmaddr)
+{
+	flush_tlb_page(vma, vmaddr);
+}
+#else
 void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
+#endif
 
 void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
 			    unsigned long end, unsigned long floor,
@@ -205,7 +221,8 @@ static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr,
  * are reserved early in the boot process by memblock instead of via
  * the .dts as on IBM platforms.
  */
-#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_FSL_BOOK3E)
+#if defined(CONFIG_HUGETLB_PAGE) && (defined(CONFIG_PPC_FSL_BOOK3E) || \
+    defined(CONFIG_PPC_8xx))
 extern void __init reserve_hugetlb_gpages(void);
 #else
 static inline void reserve_hugetlb_gpages(void)
diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h
index 708edebcf147..77ff1ba99d1f 100644
--- a/arch/powerpc/include/asm/hvcall.h
+++ b/arch/powerpc/include/asm/hvcall.h
@@ -275,7 +275,9 @@
 #define H_COP			0x304
 #define H_GET_MPP_X		0x314
 #define H_SET_MODE		0x31C
-#define MAX_HCALL_OPCODE	H_SET_MODE
+#define H_CLEAR_HPT		0x358
+#define H_SIGNAL_SYS_RESET	0x380
+#define MAX_HCALL_OPCODE	H_SIGNAL_SYS_RESET
 
 /* H_VIOCTL functions */
 #define H_GET_VIOA_DUMP_SIZE	0x01
@@ -306,6 +308,11 @@
 #define H_SET_MODE_RESOURCE_ADDR_TRANS_MODE	3
 #define H_SET_MODE_RESOURCE_LE			4
 
+/* Values for argument to H_SIGNAL_SYS_RESET */
+#define H_SIGNAL_SYS_RESET_ALL			-1
+#define H_SIGNAL_SYS_RESET_ALL_OTHERS		-2
+/* >= 0 values are CPU number */
+
 #ifndef __ASSEMBLY__
 
 /**
@@ -412,27 +419,6 @@ static inline unsigned int get_longbusy_msecs(int longbusy_rc)
 	}
 }
 
-#ifdef CONFIG_PPC_PSERIES
-extern int CMO_PrPSP;
-extern int CMO_SecPSP;
-extern unsigned long CMO_PageSize;
-
-static inline int cmo_get_primary_psp(void)
-{
-	return CMO_PrPSP;
-}
-
-static inline int cmo_get_secondary_psp(void)
-{
-	return CMO_SecPSP;
-}
-
-static inline unsigned long cmo_get_page_size(void)
-{
-	return CMO_PageSize;
-}
-#endif /* CONFIG_PPC_PSERIES */
-
 #endif /* __ASSEMBLY__ */
 #endif /* __KERNEL__ */
 #endif /* _ASM_POWERPC_HVCALL_H */
diff --git a/arch/powerpc/include/asm/io.h b/arch/powerpc/include/asm/io.h
index f6fda8482f60..5ed292431b5b 100644
--- a/arch/powerpc/include/asm/io.h
+++ b/arch/powerpc/include/asm/io.h
@@ -33,6 +33,7 @@ extern struct pci_dev *isa_bridge_pcidev;
 #include <asm/synch.h>
 #include <asm/delay.h>
 #include <asm/mmu.h>
+#include <asm/ppc_asm.h>
 
 #include <asm-generic/iomap.h>
 
@@ -458,13 +459,10 @@ static inline unsigned int name(unsigned int port)	\
 		"5:	li	%0,-1\n"		\
 		"	b	4b\n"			\
 		".previous\n"				\
-		".section __ex_table,\"a\"\n"		\
-		"	.align	2\n"			\
-		"	.long	0b,5b\n"		\
-		"	.long	1b,5b\n"		\
-		"	.long	2b,5b\n"		\
-		"	.long	3b,5b\n"		\
-		".previous"				\
+		EX_TABLE(0b, 5b)			\
+		EX_TABLE(1b, 5b)			\
+		EX_TABLE(2b, 5b)			\
+		EX_TABLE(3b, 5b)			\
 		: "=&r" (x)				\
 		: "r" (port + _IO_BASE)			\
 		: "memory");  				\
@@ -479,11 +477,8 @@ static inline void name(unsigned int val, unsigned int port) \
 		"0:" op " %0,0,%1\n"			\
 		"1:	sync\n"				\
 		"2:\n"					\
-		".section __ex_table,\"a\"\n"		\
-		"	.align	2\n"			\
-		"	.long	0b,2b\n"		\
-		"	.long	1b,2b\n"		\
-		".previous"				\
+		EX_TABLE(0b, 2b)			\
+		EX_TABLE(1b, 2b)			\
 		: : "r" (val), "r" (port + _IO_BASE)	\
 		: "memory");   	   	   		\
 }
diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h
index a46f5f45570c..6c3b71502fbc 100644
--- a/arch/powerpc/include/asm/kexec.h
+++ b/arch/powerpc/include/asm/kexec.h
@@ -53,7 +53,7 @@
 
 typedef void (*crash_shutdown_t)(void);
 
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 
 /*
  * This function is responsible for capturing register states if coming
@@ -91,7 +91,17 @@ static inline bool kdump_in_progress(void)
 	return crashing_cpu >= 0;
 }
 
-#else /* !CONFIG_KEXEC */
+#ifdef CONFIG_KEXEC_FILE
+extern struct kexec_file_ops kexec_elf64_ops;
+
+int setup_purgatory(struct kimage *image, const void *slave_code,
+		    const void *fdt, unsigned long kernel_load_addr,
+		    unsigned long fdt_load_addr);
+int setup_new_fdt(void *fdt, unsigned long initrd_load_addr,
+		  unsigned long initrd_len, const char *cmdline);
+#endif /* CONFIG_KEXEC_FILE */
+
+#else /* !CONFIG_KEXEC_CORE */
 static inline void crash_kexec_secondary(struct pt_regs *regs) { }
 
 static inline int overlaps_crashkernel(unsigned long start, unsigned long size)
@@ -116,7 +126,7 @@ static inline bool kdump_in_progress(void)
 	return false;
 }
 
-#endif /* CONFIG_KEXEC */
+#endif /* CONFIG_KEXEC_CORE */
 #endif /* ! __ASSEMBLY__ */
 #endif /* __KERNEL__ */
 #endif /* _ASM_POWERPC_KEXEC_H */
diff --git a/arch/powerpc/include/asm/kprobes.h b/arch/powerpc/include/asm/kprobes.h
index 2c9759bdb63b..97b8c1f83453 100644
--- a/arch/powerpc/include/asm/kprobes.h
+++ b/arch/powerpc/include/asm/kprobes.h
@@ -32,6 +32,7 @@
 #include <asm/probes.h>
 #include <asm/code-patching.h>
 
+#ifdef CONFIG_KPROBES
 #define  __ARCH_WANT_KPROBES_INSN_SLOT
 
 struct pt_regs;
@@ -127,5 +128,11 @@ struct kprobe_ctlblk {
 extern int kprobe_exceptions_notify(struct notifier_block *self,
 					unsigned long val, void *data);
 extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr);
+extern int kprobe_handler(struct pt_regs *regs);
+extern int kprobe_post_handler(struct pt_regs *regs);
+#else
+static inline int kprobe_handler(struct pt_regs *regs) { return 0; }
+static inline int kprobe_post_handler(struct pt_regs *regs) { return 0; }
+#endif /* CONFIG_KPROBES */
 #endif /* __KERNEL__ */
 #endif	/* _ASM_POWERPC_KPROBES_H */
diff --git a/arch/powerpc/include/asm/kvm_asm.h b/arch/powerpc/include/asm/kvm_asm.h
index 05cabed3d1bd..09a802bb702f 100644
--- a/arch/powerpc/include/asm/kvm_asm.h
+++ b/arch/powerpc/include/asm/kvm_asm.h
@@ -99,6 +99,7 @@
 #define BOOK3S_INTERRUPT_H_EMUL_ASSIST	0xe40
 #define BOOK3S_INTERRUPT_HMI		0xe60
 #define BOOK3S_INTERRUPT_H_DOORBELL	0xe80
+#define BOOK3S_INTERRUPT_H_VIRT		0xea0
 #define BOOK3S_INTERRUPT_PERFMON	0xf00
 #define BOOK3S_INTERRUPT_ALTIVEC	0xf20
 #define BOOK3S_INTERRUPT_VSX		0xf40
diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index 28350a294b1e..e59b172666cd 100644
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -48,7 +48,7 @@
 #ifdef CONFIG_KVM_MMIO
 #define KVM_COALESCED_MMIO_PAGE_OFFSET 1
 #endif
-#define KVM_HALT_POLL_NS_DEFAULT 500000
+#define KVM_HALT_POLL_NS_DEFAULT 10000	/* 10 us */
 
 /* These values are internal and can be increased later */
 #define KVM_NR_IRQCHIPS          1
@@ -244,8 +244,10 @@ struct kvm_arch_memory_slot {
 struct kvm_arch {
 	unsigned int lpid;
 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+	unsigned int tlb_sets;
 	unsigned long hpt_virt;
 	struct revmap_entry *revmap;
+	atomic64_t mmio_update;
 	unsigned int host_lpid;
 	unsigned long host_lpcr;
 	unsigned long sdr1;
@@ -408,6 +410,24 @@ struct kvmppc_passthru_irqmap {
 #define KVMPPC_IRQ_MPIC		1
 #define KVMPPC_IRQ_XICS		2
 
+#define MMIO_HPTE_CACHE_SIZE	4
+
+struct mmio_hpte_cache_entry {
+	unsigned long hpte_v;
+	unsigned long hpte_r;
+	unsigned long rpte;
+	unsigned long pte_index;
+	unsigned long eaddr;
+	unsigned long slb_v;
+	long mmio_update;
+	unsigned int slb_base_pshift;
+};
+
+struct mmio_hpte_cache {
+	struct mmio_hpte_cache_entry entry[MMIO_HPTE_CACHE_SIZE];
+	unsigned int index;
+};
+
 struct openpic;
 
 struct kvm_vcpu_arch {
@@ -498,6 +518,8 @@ struct kvm_vcpu_arch {
 	ulong tcscr;
 	ulong acop;
 	ulong wort;
+	ulong tid;
+	ulong psscr;
 	ulong shadow_srr1;
 #endif
 	u32 vrsave; /* also USPRG0 */
@@ -546,6 +568,7 @@ struct kvm_vcpu_arch {
 	u64 tfiar;
 
 	u32 cr_tm;
+	u64 xer_tm;
 	u64 lr_tm;
 	u64 ctr_tm;
 	u64 amr_tm;
@@ -655,9 +678,11 @@ struct kvm_vcpu_arch {
 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 	struct kvm_vcpu_arch_shared shregs;
 
+	struct mmio_hpte_cache mmio_cache;
 	unsigned long pgfault_addr;
 	long pgfault_index;
 	unsigned long pgfault_hpte[2];
+	struct mmio_hpte_cache_entry *pgfault_cache;
 
 	struct task_struct *run_task;
 	struct kvm_run *kvm_run;
diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h
index f6e49640dbe1..2da67bf1f2ec 100644
--- a/arch/powerpc/include/asm/kvm_ppc.h
+++ b/arch/powerpc/include/asm/kvm_ppc.h
@@ -483,9 +483,10 @@ extern void kvmppc_xics_set_mapped(struct kvm *kvm, unsigned long guest_irq,
 				   unsigned long host_irq);
 extern void kvmppc_xics_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
 				   unsigned long host_irq);
-extern long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu, u32 xirr,
-				 struct kvmppc_irq_map *irq_map,
-				 struct kvmppc_passthru_irqmap *pimap);
+extern long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu, __be32 xirr,
+					struct kvmppc_irq_map *irq_map,
+					struct kvmppc_passthru_irqmap *pimap,
+					bool *again);
 extern int h_ipi_redirect;
 #else
 static inline struct kvmppc_passthru_irqmap *kvmppc_get_passthru_irqmap(
@@ -510,6 +511,48 @@ static inline int kvmppc_xics_hcall(struct kvm_vcpu *vcpu, u32 cmd)
 #endif
 
 /*
+ * Prototypes for functions called only from assembler code.
+ * Having prototypes reduces sparse errors.
+ */
+long kvmppc_rm_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
+			 unsigned long ioba, unsigned long tce);
+long kvmppc_rm_h_put_tce_indirect(struct kvm_vcpu *vcpu,
+				  unsigned long liobn, unsigned long ioba,
+				  unsigned long tce_list, unsigned long npages);
+long kvmppc_rm_h_stuff_tce(struct kvm_vcpu *vcpu,
+			   unsigned long liobn, unsigned long ioba,
+			   unsigned long tce_value, unsigned long npages);
+long int kvmppc_rm_h_confer(struct kvm_vcpu *vcpu, int target,
+                            unsigned int yield_count);
+long kvmppc_h_random(struct kvm_vcpu *vcpu);
+void kvmhv_commence_exit(int trap);
+long kvmppc_realmode_machine_check(struct kvm_vcpu *vcpu);
+void kvmppc_subcore_enter_guest(void);
+void kvmppc_subcore_exit_guest(void);
+long kvmppc_realmode_hmi_handler(void);
+long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags,
+                    long pte_index, unsigned long pteh, unsigned long ptel);
+long kvmppc_h_remove(struct kvm_vcpu *vcpu, unsigned long flags,
+                     unsigned long pte_index, unsigned long avpn);
+long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu);
+long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
+                      unsigned long pte_index, unsigned long avpn,
+                      unsigned long va);
+long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags,
+                   unsigned long pte_index);
+long kvmppc_h_clear_ref(struct kvm_vcpu *vcpu, unsigned long flags,
+                        unsigned long pte_index);
+long kvmppc_h_clear_mod(struct kvm_vcpu *vcpu, unsigned long flags,
+                        unsigned long pte_index);
+long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr,
+                          unsigned long slb_v, unsigned int status, bool data);
+unsigned long kvmppc_rm_h_xirr(struct kvm_vcpu *vcpu);
+int kvmppc_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server,
+                    unsigned long mfrr);
+int kvmppc_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr);
+int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr);
+
+/*
  * Host-side operations we want to set up while running in real
  * mode in the guest operating on the xics.
  * Currently only VCPU wakeup is supported.
diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h
index e02cbc6a6c70..5011b69107a7 100644
--- a/arch/powerpc/include/asm/machdep.h
+++ b/arch/powerpc/include/asm/machdep.h
@@ -183,7 +183,7 @@ struct machdep_calls {
 	 */
 	void (*machine_shutdown)(void);
 
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 	void (*kexec_cpu_down)(int crash_shutdown, int secondary);
 
 	/* Called to do what every setup is needed on image and the
@@ -198,7 +198,7 @@ struct machdep_calls {
 	 * no return.
 	 */
 	void (*machine_kexec)(struct kimage *image);
-#endif /* CONFIG_KEXEC */
+#endif /* CONFIG_KEXEC_CORE */
 
 #ifdef CONFIG_SUSPEND
 	/* These are called to disable and enable, respectively, IRQs when
diff --git a/arch/powerpc/include/asm/mmu-8xx.h b/arch/powerpc/include/asm/mmu-8xx.h
index 3e0e4927811c..798b5bf91427 100644
--- a/arch/powerpc/include/asm/mmu-8xx.h
+++ b/arch/powerpc/include/asm/mmu-8xx.h
@@ -172,6 +172,41 @@ typedef struct {
 
 #define PHYS_IMMR_BASE (mfspr(SPRN_IMMR) & 0xfff80000)
 #define VIRT_IMMR_BASE (__fix_to_virt(FIX_IMMR_BASE))
+
+/* Page size definitions, common between 32 and 64-bit
+ *
+ *    shift : is the "PAGE_SHIFT" value for that page size
+ *    penc  : is the pte encoding mask
+ *
+ */
+struct mmu_psize_def {
+	unsigned int	shift;	/* number of bits */
+	unsigned int	enc;	/* PTE encoding */
+	unsigned int    ind;    /* Corresponding indirect page size shift */
+	unsigned int	flags;
+#define MMU_PAGE_SIZE_DIRECT	0x1	/* Supported as a direct size */
+#define MMU_PAGE_SIZE_INDIRECT	0x2	/* Supported as an indirect size */
+};
+
+extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];
+
+static inline int shift_to_mmu_psize(unsigned int shift)
+{
+	int psize;
+
+	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize)
+		if (mmu_psize_defs[psize].shift == shift)
+			return psize;
+	return -1;
+}
+
+static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize)
+{
+	if (mmu_psize_defs[mmu_psize].shift)
+		return mmu_psize_defs[mmu_psize].shift;
+	BUG();
+}
+
 #endif /* !__ASSEMBLY__ */
 
 #if defined(CONFIG_PPC_4K_PAGES)
diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h
index e311c25751a4..a34c764ca8dd 100644
--- a/arch/powerpc/include/asm/mmu.h
+++ b/arch/powerpc/include/asm/mmu.h
@@ -214,6 +214,11 @@ extern u64 ppc64_rma_size;
 /* Cleanup function used by kexec */
 extern void mmu_cleanup_all(void);
 extern void radix__mmu_cleanup_all(void);
+
+/* Functions for creating and updating partition table on POWER9 */
+extern void mmu_partition_table_init(void);
+extern void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
+					  unsigned long dw1);
 #endif /* CONFIG_PPC64 */
 
 struct mm_struct;
@@ -270,19 +275,20 @@ static inline bool early_radix_enabled(void)
 #define MMU_PAGE_64K	2
 #define MMU_PAGE_64K_AP	3	/* "Admixed pages" (hash64 only) */
 #define MMU_PAGE_256K	4
-#define MMU_PAGE_1M	5
-#define MMU_PAGE_2M	6
-#define MMU_PAGE_4M	7
-#define MMU_PAGE_8M	8
-#define MMU_PAGE_16M	9
-#define MMU_PAGE_64M	10
-#define MMU_PAGE_256M	11
-#define MMU_PAGE_1G	12
-#define MMU_PAGE_16G	13
-#define MMU_PAGE_64G	14
+#define MMU_PAGE_512K	5
+#define MMU_PAGE_1M	6
+#define MMU_PAGE_2M	7
+#define MMU_PAGE_4M	8
+#define MMU_PAGE_8M	9
+#define MMU_PAGE_16M	10
+#define MMU_PAGE_64M	11
+#define MMU_PAGE_256M	12
+#define MMU_PAGE_1G	13
+#define MMU_PAGE_16G	14
+#define MMU_PAGE_64G	15
 
 /* N.B. we need to change the type of hpte_page_sizes if this gets to be > 16 */
-#define MMU_PAGE_COUNT	15
+#define MMU_PAGE_COUNT	16
 
 #ifdef CONFIG_PPC_BOOK3S_64
 #include <asm/book3s/64/mmu.h>
diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
index 5c451140660a..b9e3f0aca261 100644
--- a/arch/powerpc/include/asm/mmu_context.h
+++ b/arch/powerpc/include/asm/mmu_context.h
@@ -19,16 +19,18 @@ extern void destroy_context(struct mm_struct *mm);
 struct mm_iommu_table_group_mem_t;
 
 extern int isolate_lru_page(struct page *page);	/* from internal.h */
-extern bool mm_iommu_preregistered(void);
-extern long mm_iommu_get(unsigned long ua, unsigned long entries,
+extern bool mm_iommu_preregistered(struct mm_struct *mm);
+extern long mm_iommu_get(struct mm_struct *mm,
+		unsigned long ua, unsigned long entries,
 		struct mm_iommu_table_group_mem_t **pmem);
-extern long mm_iommu_put(struct mm_iommu_table_group_mem_t *mem);
-extern void mm_iommu_init(mm_context_t *ctx);
-extern void mm_iommu_cleanup(mm_context_t *ctx);
-extern struct mm_iommu_table_group_mem_t *mm_iommu_lookup(unsigned long ua,
-		unsigned long size);
-extern struct mm_iommu_table_group_mem_t *mm_iommu_find(unsigned long ua,
-		unsigned long entries);
+extern long mm_iommu_put(struct mm_struct *mm,
+		struct mm_iommu_table_group_mem_t *mem);
+extern void mm_iommu_init(struct mm_struct *mm);
+extern void mm_iommu_cleanup(struct mm_struct *mm);
+extern struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm,
+		unsigned long ua, unsigned long size);
+extern struct mm_iommu_table_group_mem_t *mm_iommu_find(struct mm_struct *mm,
+		unsigned long ua, unsigned long entries);
 extern long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem,
 		unsigned long ua, unsigned long *hpa);
 extern long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem);
diff --git a/arch/powerpc/include/asm/module.h b/arch/powerpc/include/asm/module.h
index cd4ffd86765f..cc12c61ef315 100644
--- a/arch/powerpc/include/asm/module.h
+++ b/arch/powerpc/include/asm/module.h
@@ -90,10 +90,6 @@ static inline int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sec
 }
 #endif
 
-struct exception_table_entry;
-void sort_ex_table(struct exception_table_entry *start,
-		   struct exception_table_entry *finish);
-
 #if defined(CONFIG_MODVERSIONS) && defined(CONFIG_PPC64)
 #define ARCH_RELOCATES_KCRCTAB
 #define reloc_start PHYSICAL_START
diff --git a/arch/powerpc/include/asm/nohash/32/pgalloc.h b/arch/powerpc/include/asm/nohash/32/pgalloc.h
index 76d6b9e0c8a9..633139291a48 100644
--- a/arch/powerpc/include/asm/nohash/32/pgalloc.h
+++ b/arch/powerpc/include/asm/nohash/32/pgalloc.h
@@ -2,14 +2,42 @@
 #define _ASM_POWERPC_PGALLOC_32_H
 
 #include <linux/threads.h>
+#include <linux/slab.h>
 
-/* For 32-bit, all levels of page tables are just drawn from get_free_page() */
-#define MAX_PGTABLE_INDEX_SIZE	0
+/*
+ * Functions that deal with pagetables that could be at any level of
+ * the table need to be passed an "index_size" so they know how to
+ * handle allocation.  For PTE pages (which are linked to a struct
+ * page for now, and drawn from the main get_free_pages() pool), the
+ * allocation size will be (2^index_size * sizeof(pointer)) and
+ * allocations are drawn from the kmem_cache in PGT_CACHE(index_size).
+ *
+ * The maximum index size needs to be big enough to allow any
+ * pagetable sizes we need, but small enough to fit in the low bits of
+ * any page table pointer.  In other words all pagetables, even tiny
+ * ones, must be aligned to allow at least enough low 0 bits to
+ * contain this value.  This value is also used as a mask, so it must
+ * be one less than a power of two.
+ */
+#define MAX_PGTABLE_INDEX_SIZE	0xf
 
 extern void __bad_pte(pmd_t *pmd);
 
-extern pgd_t *pgd_alloc(struct mm_struct *mm);
-extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
+extern struct kmem_cache *pgtable_cache[];
+#define PGT_CACHE(shift) ({				\
+			BUG_ON(!(shift));		\
+			pgtable_cache[(shift) - 1];	\
+		})
+
+static inline pgd_t *pgd_alloc(struct mm_struct *mm)
+{
+	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), GFP_KERNEL);
+}
+
+static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
+{
+	kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd);
+}
 
 /*
  * We don't have any real pmd's, and this code never triggers because
@@ -68,8 +96,12 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
 
 static inline void pgtable_free(void *table, unsigned index_size)
 {
-	BUG_ON(index_size); /* 32-bit doesn't use this */
-	free_page((unsigned long)table);
+	if (!index_size) {
+		free_page((unsigned long)table);
+	} else {
+		BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE);
+		kmem_cache_free(PGT_CACHE(index_size), table);
+	}
 }
 
 #define check_pgt_cache()	do { } while (0)
diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h
index c219ef7be53b..ba9921bf202e 100644
--- a/arch/powerpc/include/asm/nohash/32/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/32/pgtable.h
@@ -16,6 +16,23 @@ extern int icache_44x_need_flush;
 
 #endif /* __ASSEMBLY__ */
 
+#define PTE_INDEX_SIZE	PTE_SHIFT
+#define PMD_INDEX_SIZE	0
+#define PUD_INDEX_SIZE	0
+#define PGD_INDEX_SIZE	(32 - PGDIR_SHIFT)
+
+#define PMD_CACHE_INDEX	PMD_INDEX_SIZE
+
+#ifndef __ASSEMBLY__
+#define PTE_TABLE_SIZE	(sizeof(pte_t) << PTE_INDEX_SIZE)
+#define PMD_TABLE_SIZE	0
+#define PUD_TABLE_SIZE	0
+#define PGD_TABLE_SIZE	(sizeof(pgd_t) << PGD_INDEX_SIZE)
+#endif	/* __ASSEMBLY__ */
+
+#define PTRS_PER_PTE	(1 << PTE_INDEX_SIZE)
+#define PTRS_PER_PGD	(1 << PGD_INDEX_SIZE)
+
 /*
  * The normal case is that PTEs are 32-bits and we have a 1-page
  * 1024-entry pgdir pointing to 1-page 1024-entry PTE pages.  -- paulus
@@ -27,22 +44,12 @@ extern int icache_44x_need_flush;
  * -Matt
  */
 /* PGDIR_SHIFT determines what a top-level page table entry can map */
-#define PGDIR_SHIFT	(PAGE_SHIFT + PTE_SHIFT)
+#define PGDIR_SHIFT	(PAGE_SHIFT + PTE_INDEX_SIZE)
 #define PGDIR_SIZE	(1UL << PGDIR_SHIFT)
 #define PGDIR_MASK	(~(PGDIR_SIZE-1))
 
-/*
- * entries per page directory level: our page-table tree is two-level, so
- * we don't really have any PMD directory.
- */
-#ifndef __ASSEMBLY__
-#define PTE_TABLE_SIZE	(sizeof(pte_t) << PTE_SHIFT)
-#define PGD_TABLE_SIZE	(sizeof(pgd_t) << (32 - PGDIR_SHIFT))
-#endif	/* __ASSEMBLY__ */
-
-#define PTRS_PER_PTE	(1 << PTE_SHIFT)
-#define PTRS_PER_PMD	1
-#define PTRS_PER_PGD	(1 << (32 - PGDIR_SHIFT))
+/* Bits to mask out from a PGD to get to the PUD page */
+#define PGD_MASKED_BITS		0
 
 #define USER_PTRS_PER_PGD	(TASK_SIZE / PGDIR_SIZE)
 #define FIRST_USER_ADDRESS	0UL
@@ -268,7 +275,8 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
 
 
 static inline void __ptep_set_access_flags(struct mm_struct *mm,
-					   pte_t *ptep, pte_t entry)
+					   pte_t *ptep, pte_t entry,
+					   unsigned long address)
 {
 	unsigned long set = pte_val(entry) &
 		(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC);
@@ -328,15 +336,6 @@ static inline void __ptep_set_access_flags(struct mm_struct *mm,
 #define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) >> 3 })
 #define __swp_entry_to_pte(x)		((pte_t) { (x).val << 3 })
 
-#ifndef CONFIG_PPC_4K_PAGES
-void pgtable_cache_init(void);
-#else
-/*
- * No page table caches to initialise
- */
-#define pgtable_cache_init()	do { } while (0)
-#endif
-
 extern int get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep,
 		      pmd_t **pmdp);
 
diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h
index 3742b1919661..b4df2734c078 100644
--- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h
@@ -49,6 +49,7 @@
 #define _PMD_BAD	0x0ff0
 #define _PMD_PAGE_MASK	0x000c
 #define _PMD_PAGE_8M	0x000c
+#define _PMD_PAGE_512K	0x0004
 
 /* Until my rework is finished, 8xx still needs atomic PTE updates */
 #define PTE_ATOMIC_UPDATES	1
diff --git a/arch/powerpc/include/asm/nohash/64/pgtable-4k.h b/arch/powerpc/include/asm/nohash/64/pgtable-4k.h
index fc7d51753f81..d0db98793dd8 100644
--- a/arch/powerpc/include/asm/nohash/64/pgtable-4k.h
+++ b/arch/powerpc/include/asm/nohash/64/pgtable-4k.h
@@ -27,9 +27,6 @@
 #define PMD_SIZE	(1UL << PMD_SHIFT)
 #define PMD_MASK	(~(PMD_SIZE-1))
 
-/* With 4k base page size, hugepage PTEs go at the PMD level */
-#define MIN_HUGEPTE_SHIFT	PMD_SHIFT
-
 /* PUD_SHIFT determines what a third-level page table entry can map */
 #define PUD_SHIFT	(PMD_SHIFT + PMD_INDEX_SIZE)
 #define PUD_SIZE	(1UL << PUD_SHIFT)
diff --git a/arch/powerpc/include/asm/nohash/64/pgtable-64k.h b/arch/powerpc/include/asm/nohash/64/pgtable-64k.h
index 908324574f77..55b28ef3409a 100644
--- a/arch/powerpc/include/asm/nohash/64/pgtable-64k.h
+++ b/arch/powerpc/include/asm/nohash/64/pgtable-64k.h
@@ -31,9 +31,6 @@
 #define PTRS_PER_PMD	(1 << PMD_INDEX_SIZE)
 #define PTRS_PER_PGD	(1 << PGD_INDEX_SIZE)
 
-/* With 4k base page size, hugepage PTEs go at the PMD level */
-#define MIN_HUGEPTE_SHIFT	PAGE_SHIFT
-
 /* PMD_SHIFT determines what a second-level page table entry can map */
 #define PMD_SHIFT	(PAGE_SHIFT + PTE_INDEX_SIZE)
 #define PMD_SIZE	(1UL << PMD_SHIFT)
diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h
index 653a1838469d..c7f927e67d14 100644
--- a/arch/powerpc/include/asm/nohash/64/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/64/pgtable.h
@@ -26,15 +26,11 @@
 #else
 #define PMD_CACHE_INDEX	PMD_INDEX_SIZE
 #endif
+
 /*
  * Define the address range of the kernel non-linear virtual area
  */
-
-#ifdef CONFIG_PPC_BOOK3E
 #define KERN_VIRT_START ASM_CONST(0x8000000000000000)
-#else
-#define KERN_VIRT_START ASM_CONST(0xD000000000000000)
-#endif
 #define KERN_VIRT_SIZE	ASM_CONST(0x0000100000000000)
 
 /*
@@ -43,11 +39,7 @@
  * (we keep a quarter for the virtual memmap)
  */
 #define VMALLOC_START	KERN_VIRT_START
-#ifdef CONFIG_PPC_BOOK3E
 #define VMALLOC_SIZE	(KERN_VIRT_SIZE >> 2)
-#else
-#define VMALLOC_SIZE	(KERN_VIRT_SIZE >> 1)
-#endif
 #define VMALLOC_END	(VMALLOC_START + VMALLOC_SIZE)
 
 /*
@@ -85,12 +77,8 @@
  * Defines the address of the vmemap area, in its own region on
  * hash table CPUs and after the vmalloc space on Book3E
  */
-#ifdef CONFIG_PPC_BOOK3E
 #define VMEMMAP_BASE		VMALLOC_END
 #define VMEMMAP_END		KERN_IO_START
-#else
-#define VMEMMAP_BASE		(VMEMMAP_REGION_ID << REGION_SHIFT)
-#endif
 #define vmemmap			((struct page *)VMEMMAP_BASE)
 
 
@@ -301,7 +289,8 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
  * function doesn't need to flush the hash entry
  */
 static inline void __ptep_set_access_flags(struct mm_struct *mm,
-					   pte_t *ptep, pte_t entry)
+					   pte_t *ptep, pte_t entry,
+					   unsigned long address)
 {
 	unsigned long bits = pte_val(entry) &
 		(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC);
@@ -358,8 +347,6 @@ static inline void __ptep_set_access_flags(struct mm_struct *mm,
 #define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val((pte)) })
 #define __swp_entry_to_pte(x)		__pte((x).val)
 
-void pgtable_cache_add(unsigned shift, void (*ctor)(void *));
-void pgtable_cache_init(void);
 extern int map_kernel_page(unsigned long ea, unsigned long pa,
 			   unsigned long flags);
 extern int __meminit vmemmap_create_mapping(unsigned long start,
diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h
index 1263c22d60d8..172849727054 100644
--- a/arch/powerpc/include/asm/nohash/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/pgtable.h
@@ -226,7 +226,11 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 #ifdef CONFIG_HUGETLB_PAGE
 static inline int hugepd_ok(hugepd_t hpd)
 {
+#ifdef CONFIG_PPC_8xx
+	return ((hpd.pd & 0x4) != 0);
+#else
 	return (hpd.pd > 0);
+#endif
 }
 
 static inline int pmd_huge(pmd_t pmd)
diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h
index e958b7096f19..5c7db0f1a708 100644
--- a/arch/powerpc/include/asm/opal.h
+++ b/arch/powerpc/include/asm/opal.h
@@ -220,9 +220,12 @@ int64_t opal_pci_set_power_state(uint64_t async_token, uint64_t id,
 int64_t opal_pci_poll2(uint64_t id, uint64_t data);
 
 int64_t opal_int_get_xirr(uint32_t *out_xirr, bool just_poll);
+int64_t opal_rm_int_get_xirr(__be32 *out_xirr, bool just_poll);
 int64_t opal_int_set_cppr(uint8_t cppr);
 int64_t opal_int_eoi(uint32_t xirr);
+int64_t opal_rm_int_eoi(uint32_t xirr);
 int64_t opal_int_set_mfrr(uint32_t cpu, uint8_t mfrr);
+int64_t opal_rm_int_set_mfrr(uint32_t cpu, uint8_t mfrr);
 int64_t opal_pci_tce_kill(uint64_t phb_id, uint32_t kill_type,
 			  uint32_t pe_num, uint32_t tce_size,
 			  uint64_t dma_addr, uint32_t npages);
diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h
index 9bd87f269d6d..dd01212935ac 100644
--- a/arch/powerpc/include/asm/pgtable.h
+++ b/arch/powerpc/include/asm/pgtable.h
@@ -78,6 +78,8 @@ static inline pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea,
 
 unsigned long vmalloc_to_phys(void *vmalloc_addr);
 
+void pgtable_cache_add(unsigned shift, void (*ctor)(void *));
+void pgtable_cache_init(void);
 #endif /* __ASSEMBLY__ */
 
 #endif /* _ASM_POWERPC_PGTABLE_H */
diff --git a/arch/powerpc/include/asm/plpar_wrappers.h b/arch/powerpc/include/asm/plpar_wrappers.h
index 1b394247afc2..0bcc75e295e3 100644
--- a/arch/powerpc/include/asm/plpar_wrappers.h
+++ b/arch/powerpc/include/asm/plpar_wrappers.h
@@ -93,38 +93,6 @@ static inline long register_dtl(unsigned long cpu, unsigned long vpa)
 	return vpa_call(H_VPA_REG_DTL, cpu, vpa);
 }
 
-static inline long plpar_page_set_loaned(unsigned long vpa)
-{
-	unsigned long cmo_page_sz = cmo_get_page_size();
-	long rc = 0;
-	int i;
-
-	for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz)
-		rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED, vpa + i, 0);
-
-	for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz)
-		plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE,
-				   vpa + i - cmo_page_sz, 0);
-
-	return rc;
-}
-
-static inline long plpar_page_set_active(unsigned long vpa)
-{
-	unsigned long cmo_page_sz = cmo_get_page_size();
-	long rc = 0;
-	int i;
-
-	for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz)
-		rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE, vpa + i, 0);
-
-	for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz)
-		plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED,
-				   vpa + i - cmo_page_sz, 0);
-
-	return rc;
-}
-
 extern void vpa_init(int cpu);
 
 static inline long plpar_pte_enter(unsigned long flags,
@@ -340,4 +308,9 @@ static inline long plapr_set_watchpoint0(unsigned long dawr0, unsigned long dawr
 	return plpar_set_mode(0, H_SET_MODE_RESOURCE_SET_DAWR, dawr0, dawrx0);
 }
 
+static inline long plapr_signal_sys_reset(long cpu)
+{
+	return plpar_hcall_norets(H_SIGNAL_SYS_RESET, cpu);
+}
+
 #endif /* _ASM_POWERPC_PLPAR_WRAPPERS_H */
diff --git a/arch/powerpc/include/asm/ppc-pci.h b/arch/powerpc/include/asm/ppc-pci.h
index 0f73de069f19..726288048652 100644
--- a/arch/powerpc/include/asm/ppc-pci.h
+++ b/arch/powerpc/include/asm/ppc-pci.h
@@ -53,7 +53,7 @@ void eeh_addr_cache_rmv_dev(struct pci_dev *dev);
 struct eeh_dev *eeh_addr_cache_get_dev(unsigned long addr);
 void eeh_slot_error_detail(struct eeh_pe *pe, int severity);
 int eeh_pci_enable(struct eeh_pe *pe, int function);
-int eeh_reset_pe(struct eeh_pe *);
+int eeh_pe_reset_full(struct eeh_pe *pe);
 void eeh_save_bars(struct eeh_dev *edev);
 int rtas_write_config(struct pci_dn *, int where, int size, u32 val);
 int rtas_read_config(struct pci_dn *, int where, int size, u32 *val);
diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h
index c73750b0d9fa..025833b8df9f 100644
--- a/arch/powerpc/include/asm/ppc_asm.h
+++ b/arch/powerpc/include/asm/ppc_asm.h
@@ -10,9 +10,7 @@
 #include <asm/ppc-opcode.h>
 #include <asm/firmware.h>
 
-#ifndef __ASSEMBLY__
-#error __FILE__ should only be used in assembler files
-#else
+#ifdef __ASSEMBLY__
 
 #define SZL			(BITS_PER_LONG/8)
 
@@ -265,10 +263,14 @@ n:
  * latter is for those that incdentially must be excluded from probing
  * and allows them to be linked at more optimal location within text.
  */
+#ifdef CONFIG_KPROBES
 #define _ASM_NOKPROBE_SYMBOL(entry)			\
 	.pushsection "_kprobe_blacklist","aw";		\
 	PPC_LONG (entry) ;				\
 	.popsection
+#else
+#define _ASM_NOKPROBE_SYMBOL(entry)
+#endif
 
 #define FUNC_START(name)	_GLOBAL(name)
 #define FUNC_END(name)
@@ -779,5 +781,17 @@ END_FTR_SECTION_IFCLR(CPU_FTR_601)
 	.long 0xa6037b7d; /* mtsrr1 r11				*/ \
 	.long 0x2400004c  /* rfid				*/
 #endif /* !CONFIG_PPC_BOOK3E */
+
 #endif /*  __ASSEMBLY__ */
+
+/*
+ * Helper macro for exception table entries
+ */
+#define EX_TABLE(_fault, _target)		\
+	stringify_in_c(.section __ex_table,"a";)\
+	stringify_in_c(.balign 4;)		\
+	stringify_in_c(.long (_fault) - . ;)	\
+	stringify_in_c(.long (_target) - . ;)	\
+	stringify_in_c(.previous)
+
 #endif /* _ASM_POWERPC_PPC_ASM_H */
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index dac83fcb9445..1ba814436c73 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -312,8 +312,6 @@ struct thread_struct {
 	unsigned long	mmcr2;
 	unsigned 	mmcr0;
 	unsigned 	used_ebb;
-	unsigned long	lmrr;
-	unsigned long	lmser;
 #endif
 };
 
diff --git a/arch/powerpc/include/asm/prom.h b/arch/powerpc/include/asm/prom.h
index 7f436ba1b56f..5e57705b4759 100644
--- a/arch/powerpc/include/asm/prom.h
+++ b/arch/powerpc/include/asm/prom.h
@@ -159,11 +159,5 @@ struct of_drconf_cell {
 /* Option Vector 6: IBM PAPR hints */
 #define OV6_LINUX		0x02	/* Linux is our OS */
 
-/*
- * The architecture vector has an array of PVR mask/value pairs,
- * followed by # option vectors - 1, followed by the option vectors.
- */
-extern unsigned char ibm_architecture_vec[];
-
 #endif /* __KERNEL__ */
 #endif /* _POWERPC_PROM_H */
diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h
index 9e1499f98def..0d4531aa2052 100644
--- a/arch/powerpc/include/asm/reg.h
+++ b/arch/powerpc/include/asm/reg.h
@@ -153,6 +153,8 @@
 #define PSSCR_EC		0x00100000 /* Exit Criterion */
 #define PSSCR_ESL		0x00200000 /* Enable State Loss */
 #define PSSCR_SD		0x00400000 /* Status Disable */
+#define PSSCR_PLS	0xf000000000000000 /* Power-saving Level Status */
+#define PSSCR_GUEST_VIS	0xf0000000000003ff /* Guest-visible PSSCR fields */
 
 /* Floating Point Status and Control Register (FPSCR) Fields */
 #define FPSCR_FX	0x80000000	/* FPU exception summary */
@@ -236,6 +238,7 @@
 #define SPRN_TEXASRU	0x83	/* ''	   ''	   ''	 Upper 32  */
 #define   TEXASR_FS	__MASK(63-36) /* TEXASR Failure Summary */
 #define SPRN_TFHAR	0x80	/* Transaction Failure Handler Addr */
+#define SPRN_TIDR	144	/* Thread ID register */
 #define SPRN_CTRLF	0x088
 #define SPRN_CTRLT	0x098
 #define   CTRL_CT	0xc0000000	/* current thread */
@@ -292,8 +295,7 @@
 #define SPRN_HRMOR	0x139	/* Real mode offset register */
 #define SPRN_HSRR0	0x13A	/* Hypervisor Save/Restore 0 */
 #define SPRN_HSRR1	0x13B	/* Hypervisor Save/Restore 1 */
-#define SPRN_LMRR	0x32D	/* Load Monitor Region Register */
-#define SPRN_LMSER	0x32E	/* Load Monitor Section Enable Register */
+#define SPRN_ASDR	0x330	/* Access segment descriptor register */
 #define SPRN_IC		0x350	/* Virtual Instruction Count */
 #define SPRN_VTB	0x351	/* Virtual Time Base */
 #define SPRN_LDBAR	0x352	/* LD Base Address Register */
@@ -304,7 +306,7 @@
 #define SPRN_PMCR	0x374	/* Power Management Control Register */
 
 /* HFSCR and FSCR bit numbers are the same */
-#define FSCR_LM_LG	11	/* Enable Load Monitor Registers */
+#define FSCR_MSGP_LG	10	/* Enable MSGP */
 #define FSCR_TAR_LG	8	/* Enable Target Address Register */
 #define FSCR_EBB_LG	7	/* Enable Event Based Branching */
 #define FSCR_TM_LG	5	/* Enable Transactional Memory */
@@ -314,12 +316,11 @@
 #define FSCR_VECVSX_LG	1	/* Enable VMX/VSX  */
 #define FSCR_FP_LG	0	/* Enable Floating Point */
 #define SPRN_FSCR	0x099	/* Facility Status & Control Register */
-#define   FSCR_LM	__MASK(FSCR_LM_LG)
 #define   FSCR_TAR	__MASK(FSCR_TAR_LG)
 #define   FSCR_EBB	__MASK(FSCR_EBB_LG)
 #define   FSCR_DSCR	__MASK(FSCR_DSCR_LG)
 #define SPRN_HFSCR	0xbe	/* HV=1 Facility Status & Control Register */
-#define   HFSCR_LM	__MASK(FSCR_LM_LG)
+#define   HFSCR_MSGP	__MASK(FSCR_MSGP_LG)
 #define   HFSCR_TAR	__MASK(FSCR_TAR_LG)
 #define   HFSCR_EBB	__MASK(FSCR_EBB_LG)
 #define   HFSCR_TM	__MASK(FSCR_TM_LG)
@@ -358,6 +359,7 @@
 #define     LPCR_PECE_HVEE	ASM_CONST(0x0000400000000000)	/* P9 Wakeup on HV interrupts */
 #define   LPCR_MER		ASM_CONST(0x0000000000000800)	/* Mediated External Exception */
 #define   LPCR_MER_SH		11
+#define	  LPCR_GTSE		ASM_CONST(0x0000000000000400)  	/* Guest Translation Shootdown Enable */
 #define   LPCR_TC		ASM_CONST(0x0000000000000200)	/* Translation control */
 #define   LPCR_LPES		0x0000000c
 #define   LPCR_LPES0		ASM_CONST(0x0000000000000008)      /* LPAR Env selector 0 */
@@ -378,6 +380,12 @@
 #define   PCR_VEC_DIS	(1ul << (63-0))	/* Vec. disable (bit NA since POWER8) */
 #define   PCR_VSX_DIS	(1ul << (63-1))	/* VSX disable (bit NA since POWER8) */
 #define   PCR_TM_DIS	(1ul << (63-2))	/* Trans. memory disable (POWER8) */
+/*
+ * These bits are used in the function kvmppc_set_arch_compat() to specify and
+ * determine both the compatibility level which we want to emulate and the
+ * compatibility level which the host is capable of emulating.
+ */
+#define   PCR_ARCH_207	0x8		/* Architecture 2.07 */
 #define   PCR_ARCH_206	0x4		/* Architecture 2.06 */
 #define   PCR_ARCH_205	0x2		/* Architecture 2.05 */
 #define	SPRN_HEIR	0x153	/* Hypervisor Emulated Instruction Register */
@@ -1219,6 +1227,7 @@
 #define PVR_ARCH_206	0x0f000003
 #define PVR_ARCH_206p	0x0f100003
 #define PVR_ARCH_207	0x0f000004
+#define PVR_ARCH_300	0x0f000005
 
 /* Macros for setting and retrieving special purpose registers */
 #ifndef __ASSEMBLY__
diff --git a/arch/powerpc/include/asm/reg_8xx.h b/arch/powerpc/include/asm/reg_8xx.h
index 0197e12f7d48..1f1636124a04 100644
--- a/arch/powerpc/include/asm/reg_8xx.h
+++ b/arch/powerpc/include/asm/reg_8xx.h
@@ -4,7 +4,7 @@
 #ifndef _ASM_POWERPC_REG_8xx_H
 #define _ASM_POWERPC_REG_8xx_H
 
-#include <asm/mmu-8xx.h>
+#include <asm/mmu.h>
 
 /* Cache control on the MPC8xx is provided through some additional
  * special purpose registers.
diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h
index 0d02c11dc331..32db16d2e7ad 100644
--- a/arch/powerpc/include/asm/smp.h
+++ b/arch/powerpc/include/asm/smp.h
@@ -176,7 +176,7 @@ static inline void set_hard_smp_processor_id(int cpu, int phys)
 #endif /* !CONFIG_SMP */
 #endif /* !CONFIG_PPC64 */
 
-#if defined(CONFIG_PPC64) && (defined(CONFIG_SMP) || defined(CONFIG_KEXEC))
+#if defined(CONFIG_PPC64) && (defined(CONFIG_SMP) || defined(CONFIG_KEXEC_CORE))
 extern void smp_release_cpus(void);
 #else
 static inline void smp_release_cpus(void) { };
diff --git a/arch/powerpc/include/asm/stackprotector.h b/arch/powerpc/include/asm/stackprotector.h
new file mode 100644
index 000000000000..6720190eabec
--- /dev/null
+++ b/arch/powerpc/include/asm/stackprotector.h
@@ -0,0 +1,40 @@
+/*
+ * GCC stack protector support.
+ *
+ * Stack protector works by putting predefined pattern at the start of
+ * the stack frame and verifying that it hasn't been overwritten when
+ * returning from the function.  The pattern is called stack canary
+ * and gcc expects it to be defined by a global variable called
+ * "__stack_chk_guard" on PPC.  This unfortunately means that on SMP
+ * we cannot have a different canary value per task.
+ */
+
+#ifndef _ASM_STACKPROTECTOR_H
+#define _ASM_STACKPROTECTOR_H
+
+#include <linux/random.h>
+#include <linux/version.h>
+#include <asm/reg.h>
+
+extern unsigned long __stack_chk_guard;
+
+/*
+ * Initialize the stackprotector canary value.
+ *
+ * NOTE: this must only be called from functions that never return,
+ * and it must always be inlined.
+ */
+static __always_inline void boot_init_stack_canary(void)
+{
+	unsigned long canary;
+
+	/* Try to get a semi random initial value. */
+	get_random_bytes(&canary, sizeof(canary));
+	canary ^= mftb();
+	canary ^= LINUX_VERSION_CODE;
+
+	current->stack_canary = canary;
+	__stack_chk_guard = current->stack_canary;
+}
+
+#endif	/* _ASM_STACKPROTECTOR_H */
diff --git a/arch/powerpc/include/asm/systbl.h b/arch/powerpc/include/asm/systbl.h
index 2fc5d4db503c..4b369d83fe9c 100644
--- a/arch/powerpc/include/asm/systbl.h
+++ b/arch/powerpc/include/asm/systbl.h
@@ -386,3 +386,4 @@ SYSCALL(mlock2)
 SYSCALL(copy_file_range)
 COMPAT_SYS_SPU(preadv2)
 COMPAT_SYS_SPU(pwritev2)
+SYSCALL(kexec_file_load)
diff --git a/arch/powerpc/include/asm/trace.h b/arch/powerpc/include/asm/trace.h
index 32e36b16773f..c05cef6ee06c 100644
--- a/arch/powerpc/include/asm/trace.h
+++ b/arch/powerpc/include/asm/trace.h
@@ -54,7 +54,7 @@ DEFINE_EVENT(ppc64_interrupt_class, timer_interrupt_exit,
 );
 
 #ifdef CONFIG_PPC_PSERIES
-extern void hcall_tracepoint_regfunc(void);
+extern int hcall_tracepoint_regfunc(void);
 extern void hcall_tracepoint_unregfunc(void);
 
 TRACE_EVENT_FN_COND(hcall_entry,
@@ -104,7 +104,7 @@ TRACE_EVENT_FN_COND(hcall_exit,
 #endif
 
 #ifdef CONFIG_PPC_POWERNV
-extern void opal_tracepoint_regfunc(void);
+extern int opal_tracepoint_regfunc(void);
 extern void opal_tracepoint_unregfunc(void);
 
 TRACE_EVENT_FN(opal_entry,
diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h
index c266227fdd5b..a15d84d59356 100644
--- a/arch/powerpc/include/asm/uaccess.h
+++ b/arch/powerpc/include/asm/uaccess.h
@@ -7,6 +7,7 @@
 #include <linux/sched.h>
 #include <linux/errno.h>
 #include <asm/asm-compat.h>
+#include <asm/ppc_asm.h>
 #include <asm/processor.h>
 #include <asm/page.h>
 
@@ -63,23 +64,30 @@
 	 __access_ok((__force unsigned long)(addr), (size), get_fs()))
 
 /*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
+ * The exception table consists of pairs of relative addresses: the first is
+ * the address of an instruction that is allowed to fault, and the second is
  * the address at which the program should continue.  No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
+ * modified, so it is entirely up to the continuation code to figure out what
+ * to do.
  *
- * All the routines below use bits of fixup code that are out of line
- * with the main instruction path.  This means when everything is well,
- * we don't even have to jump over them.  Further, they do not intrude
- * on our cache or tlb entries.
+ * All the routines below use bits of fixup code that are out of line with the
+ * main instruction path.  This means when everything is well, we don't even
+ * have to jump over them.  Further, they do not intrude on our cache or tlb
+ * entries.
  */
 
+#define ARCH_HAS_RELATIVE_EXTABLE
+
 struct exception_table_entry {
-	unsigned long insn;
-	unsigned long fixup;
+	int insn;
+	int fixup;
 };
 
+static inline unsigned long extable_fixup(const struct exception_table_entry *x)
+{
+	return (unsigned long)&x->fixup + x->fixup;
+}
+
 /*
  * These are the main single-value transfer routines.  They automatically
  * use the right size if we just have the right pointer type.
@@ -132,10 +140,7 @@ extern long __put_user_bad(void);
 		"3:	li %0,%3\n"				\
 		"	b 2b\n"					\
 		".previous\n"					\
-		".section __ex_table,\"a\"\n"			\
-			PPC_LONG_ALIGN "\n"			\
-			PPC_LONG "1b,3b\n"			\
-		".previous"					\
+		EX_TABLE(1b, 3b)				\
 		: "=r" (err)					\
 		: "r" (x), "b" (addr), "i" (-EFAULT), "0" (err))
 
@@ -152,11 +157,8 @@ extern long __put_user_bad(void);
 		"4:	li %0,%3\n"				\
 		"	b 3b\n"					\
 		".previous\n"					\
-		".section __ex_table,\"a\"\n"			\
-			PPC_LONG_ALIGN "\n"			\
-			PPC_LONG "1b,4b\n"			\
-			PPC_LONG "2b,4b\n"			\
-		".previous"					\
+		EX_TABLE(1b, 4b)				\
+		EX_TABLE(2b, 4b)				\
 		: "=r" (err)					\
 		: "r" (x), "b" (addr), "i" (-EFAULT), "0" (err))
 #endif /* __powerpc64__ */
@@ -215,10 +217,7 @@ extern long __get_user_bad(void);
 		"	li %1,0\n"			\
 		"	b 2b\n"				\
 		".previous\n"				\
-		".section __ex_table,\"a\"\n"		\
-			PPC_LONG_ALIGN "\n"		\
-			PPC_LONG "1b,3b\n"		\
-		".previous"				\
+		EX_TABLE(1b, 3b)			\
 		: "=r" (err), "=r" (x)			\
 		: "b" (addr), "i" (-EFAULT), "0" (err))
 
@@ -237,11 +236,8 @@ extern long __get_user_bad(void);
 		"	li %1+1,0\n"			\
 		"	b 3b\n"				\
 		".previous\n"				\
-		".section __ex_table,\"a\"\n"		\
-			PPC_LONG_ALIGN "\n"		\
-			PPC_LONG "1b,4b\n"		\
-			PPC_LONG "2b,4b\n"		\
-		".previous"				\
+		EX_TABLE(1b, 4b)			\
+		EX_TABLE(2b, 4b)			\
 		: "=r" (err), "=&r" (x)			\
 		: "b" (addr), "i" (-EFAULT), "0" (err))
 #endif /* __powerpc64__ */
diff --git a/arch/powerpc/include/asm/unistd.h b/arch/powerpc/include/asm/unistd.h
index e8cdfec8d512..eb1acee91a20 100644
--- a/arch/powerpc/include/asm/unistd.h
+++ b/arch/powerpc/include/asm/unistd.h
@@ -12,7 +12,7 @@
 #include <uapi/asm/unistd.h>
 
 
-#define NR_syscalls		382
+#define NR_syscalls		383
 
 #define __NR__exit __NR_exit
 
diff --git a/arch/powerpc/include/asm/word-at-a-time.h b/arch/powerpc/include/asm/word-at-a-time.h
index 4afe66aa1400..f3f4710d4ff5 100644
--- a/arch/powerpc/include/asm/word-at-a-time.h
+++ b/arch/powerpc/include/asm/word-at-a-time.h
@@ -7,6 +7,7 @@
 
 #include <linux/kernel.h>
 #include <asm/asm-compat.h>
+#include <asm/ppc_asm.h>
 
 #ifdef __BIG_ENDIAN__
 
@@ -193,10 +194,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr)
 #endif
 	"b	2b\n"
 	".previous\n"
-	".section __ex_table,\"a\"\n\t"
-		PPC_LONG_ALIGN "\n\t"
-		PPC_LONG "1b,3b\n"
-	".previous"
+	EX_TABLE(1b, 3b)
 	: [tmp] "=&b" (tmp), [offset] "=&r" (offset), [ret] "=&r" (ret)
 	: [addr] "b" (addr), "m" (*(unsigned long *)addr));
 
diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h
index c93cf35ce379..3603b6f51b11 100644
--- a/arch/powerpc/include/uapi/asm/kvm.h
+++ b/arch/powerpc/include/uapi/asm/kvm.h
@@ -573,6 +573,10 @@ struct kvm_get_htab_header {
 #define KVM_REG_PPC_SPRG9	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xba)
 #define KVM_REG_PPC_DBSR	(KVM_REG_PPC | KVM_REG_SIZE_U32 | 0xbb)
 
+/* POWER9 registers */
+#define KVM_REG_PPC_TIDR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xbc)
+#define KVM_REG_PPC_PSSCR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xbd)
+
 /* Transactional Memory checkpointed state:
  * This is all GPRs, all VSX regs and a subset of SPRs
  */
@@ -596,6 +600,7 @@ struct kvm_get_htab_header {
 #define KVM_REG_PPC_TM_VSCR	(KVM_REG_PPC_TM | KVM_REG_SIZE_U32 | 0x67)
 #define KVM_REG_PPC_TM_DSCR	(KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x68)
 #define KVM_REG_PPC_TM_TAR	(KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x69)
+#define KVM_REG_PPC_TM_XER	(KVM_REG_PPC_TM | KVM_REG_SIZE_U64 | 0x6a)
 
 /* PPC64 eXternal Interrupt Controller Specification */
 #define KVM_DEV_XICS_GRP_SOURCES	1	/* 64-bit source attributes */
diff --git a/arch/powerpc/include/uapi/asm/unistd.h b/arch/powerpc/include/uapi/asm/unistd.h
index e9f5f41aa55a..2f26335a3c42 100644
--- a/arch/powerpc/include/uapi/asm/unistd.h
+++ b/arch/powerpc/include/uapi/asm/unistd.h
@@ -392,5 +392,6 @@
 #define __NR_copy_file_range	379
 #define __NR_preadv2		380
 #define __NR_pwritev2		381
+#define __NR_kexec_file_load	382
 
 #endif /* _UAPI_ASM_POWERPC_UNISTD_H_ */
diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
index 1925341dbb9c..a3a6047fd395 100644
--- a/arch/powerpc/kernel/Makefile
+++ b/arch/powerpc/kernel/Makefile
@@ -19,6 +19,10 @@ CFLAGS_init.o += $(DISABLE_LATENT_ENTROPY_PLUGIN)
 CFLAGS_btext.o += $(DISABLE_LATENT_ENTROPY_PLUGIN)
 CFLAGS_prom.o += $(DISABLE_LATENT_ENTROPY_PLUGIN)
 
+# -fstack-protector triggers protection checks in this code,
+# but it is being used too early to link to meaningful stack_chk logic.
+CFLAGS_prom_init.o += $(call cc-option, -fno-stack-protector)
+
 ifdef CONFIG_FUNCTION_TRACER
 # Do not trace early boot code
 CFLAGS_REMOVE_cputable.o = -mno-sched-epilog $(CC_FLAGS_FTRACE)
@@ -58,8 +62,6 @@ obj-$(CONFIG_PPC_RTAS)		+= rtas.o rtas-rtc.o $(rtaspci-y-y)
 obj-$(CONFIG_PPC_RTAS_DAEMON)	+= rtasd.o
 obj-$(CONFIG_RTAS_FLASH)	+= rtas_flash.o
 obj-$(CONFIG_RTAS_PROC)		+= rtas-proc.o
-obj-$(CONFIG_IBMVIO)		+= vio.o
-obj-$(CONFIG_IBMEBUS)           += ibmebus.o
 obj-$(CONFIG_EEH)              += eeh.o eeh_pe.o eeh_dev.o eeh_cache.o \
 				  eeh_driver.o eeh_event.o eeh_sysfs.o
 obj-$(CONFIG_GENERIC_TBSYNC)	+= smp-tbsync.o
@@ -107,8 +109,9 @@ pci64-$(CONFIG_PPC64)		+= pci_dn.o pci-hotplug.o isa-bridge.o
 obj-$(CONFIG_PCI)		+= pci_$(BITS).o $(pci64-y) \
 				   pci-common.o pci_of_scan.o
 obj-$(CONFIG_PCI_MSI)		+= msi.o
-obj-$(CONFIG_KEXEC)		+= machine_kexec.o crash.o \
+obj-$(CONFIG_KEXEC_CORE)	+= machine_kexec.o crash.o \
 				   machine_kexec_$(BITS).o
+obj-$(CONFIG_KEXEC_FILE)	+= machine_kexec_file_$(BITS).o kexec_elf_$(BITS).o
 obj-$(CONFIG_AUDIT)		+= audit.o
 obj64-$(CONFIG_AUDIT)		+= compat_audit.o
 
@@ -128,7 +131,7 @@ obj64-$(CONFIG_PPC_TRANSACTIONAL_MEM)	+= tm.o
 obj-$(CONFIG_PPC64)		+= $(obj64-y)
 obj-$(CONFIG_PPC32)		+= $(obj32-y)
 
-ifneq ($(CONFIG_XMON)$(CONFIG_KEXEC),)
+ifneq ($(CONFIG_XMON)$(CONFIG_KEXEC_CORE),)
 obj-y				+= ppc_save_regs.o
 endif
 
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index caec7bf3b99a..0601e6a7297c 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -91,6 +91,9 @@ int main(void)
 	DEFINE(TI_livepatch_sp, offsetof(struct thread_info, livepatch_sp));
 #endif
 
+#ifdef CONFIG_CC_STACKPROTECTOR
+	DEFINE(TSK_STACK_CANARY, offsetof(struct task_struct, stack_canary));
+#endif
 	DEFINE(KSP, offsetof(struct thread_struct, ksp));
 	DEFINE(PT_REGS, offsetof(struct thread_struct, regs));
 #ifdef CONFIG_BOOKE
@@ -487,6 +490,7 @@ int main(void)
 
 	/* book3s */
 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+	DEFINE(KVM_TLB_SETS, offsetof(struct kvm, arch.tlb_sets));
 	DEFINE(KVM_SDR1, offsetof(struct kvm, arch.sdr1));
 	DEFINE(KVM_HOST_LPID, offsetof(struct kvm, arch.host_lpid));
 	DEFINE(KVM_HOST_LPCR, offsetof(struct kvm, arch.host_lpcr));
@@ -548,6 +552,8 @@ int main(void)
 	DEFINE(VCPU_TCSCR, offsetof(struct kvm_vcpu, arch.tcscr));
 	DEFINE(VCPU_ACOP, offsetof(struct kvm_vcpu, arch.acop));
 	DEFINE(VCPU_WORT, offsetof(struct kvm_vcpu, arch.wort));
+	DEFINE(VCPU_TID, offsetof(struct kvm_vcpu, arch.tid));
+	DEFINE(VCPU_PSSCR, offsetof(struct kvm_vcpu, arch.psscr));
 	DEFINE(VCORE_ENTRY_EXIT, offsetof(struct kvmppc_vcore, entry_exit_map));
 	DEFINE(VCORE_IN_GUEST, offsetof(struct kvmppc_vcore, in_guest));
 	DEFINE(VCORE_NAPPING_THREADS, offsetof(struct kvmppc_vcore, napping_threads));
@@ -569,6 +575,7 @@ int main(void)
 	DEFINE(VCPU_VRS_TM, offsetof(struct kvm_vcpu, arch.vr_tm.vr));
 	DEFINE(VCPU_VRSAVE_TM, offsetof(struct kvm_vcpu, arch.vrsave_tm));
 	DEFINE(VCPU_CR_TM, offsetof(struct kvm_vcpu, arch.cr_tm));
+	DEFINE(VCPU_XER_TM, offsetof(struct kvm_vcpu, arch.xer_tm));
 	DEFINE(VCPU_LR_TM, offsetof(struct kvm_vcpu, arch.lr_tm));
 	DEFINE(VCPU_CTR_TM, offsetof(struct kvm_vcpu, arch.ctr_tm));
 	DEFINE(VCPU_AMR_TM, offsetof(struct kvm_vcpu, arch.amr_tm));
diff --git a/arch/powerpc/kernel/cpu_setup_power.S b/arch/powerpc/kernel/cpu_setup_power.S
index 37c027ca83b2..917188615bf5 100644
--- a/arch/powerpc/kernel/cpu_setup_power.S
+++ b/arch/powerpc/kernel/cpu_setup_power.S
@@ -96,6 +96,7 @@ _GLOBAL(__setup_cpu_power9)
 	mtlr	r11
 	beqlr
 	li	r0,0
+	mtspr	SPRN_PSSCR,r0
 	mtspr	SPRN_LPID,r0
 	mfspr	r3,SPRN_LPCR
 	LOAD_REG_IMMEDIATE(r4, LPCR_PECEDH | LPCR_PECE_HVEE | LPCR_HVICE)
@@ -116,6 +117,7 @@ _GLOBAL(__restore_cpu_power9)
 	mtlr	r11
 	beqlr
 	li	r0,0
+	mtspr	SPRN_PSSCR,r0
 	mtspr	SPRN_LPID,r0
 	mfspr   r3,SPRN_LPCR
 	LOAD_REG_IMMEDIATE(r4, LPCR_PECEDH | LPCR_PECE_HVEE | LPCR_HVICE)
@@ -174,7 +176,7 @@ __init_FSCR:
 __init_HFSCR:
 	mfspr	r3,SPRN_HFSCR
 	ori	r3,r3,HFSCR_TAR|HFSCR_TM|HFSCR_BHRB|HFSCR_PM|\
-		      HFSCR_DSCR|HFSCR_VECVSX|HFSCR_FP|HFSCR_EBB
+		      HFSCR_DSCR|HFSCR_VECVSX|HFSCR_FP|HFSCR_EBB|HFSCR_MSGP
 	mtspr	SPRN_HFSCR,r3
 	blr
 
diff --git a/arch/powerpc/kernel/dma.c b/arch/powerpc/kernel/dma.c
index e64a6016fba7..6877e3fa95bb 100644
--- a/arch/powerpc/kernel/dma.c
+++ b/arch/powerpc/kernel/dma.c
@@ -203,6 +203,10 @@ static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl,
 	for_each_sg(sgl, sg, nents, i) {
 		sg->dma_address = sg_phys(sg) + get_dma_offset(dev);
 		sg->dma_length = sg->length;
+
+		if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+			continue;
+
 		__dma_sync_page(sg_page(sg), sg->offset, sg->length, direction);
 	}
 
@@ -235,7 +239,10 @@ static inline dma_addr_t dma_direct_map_page(struct device *dev,
 					     unsigned long attrs)
 {
 	BUG_ON(dir == DMA_NONE);
-	__dma_sync_page(page, offset, size, dir);
+
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		__dma_sync_page(page, offset, size, dir);
+
 	return page_to_phys(page) + offset + get_dma_offset(dev);
 }
 
diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c
index f25731627d7f..8180bfd7ab93 100644
--- a/arch/powerpc/kernel/eeh.c
+++ b/arch/powerpc/kernel/eeh.c
@@ -372,7 +372,7 @@ static int eeh_phb_check_failure(struct eeh_pe *pe)
 	/* Find the PHB PE */
 	phb_pe = eeh_phb_pe_get(pe->phb);
 	if (!phb_pe) {
-		pr_warn("%s Can't find PE for PHB#%d\n",
+		pr_warn("%s Can't find PE for PHB#%x\n",
 			__func__, pe->phb->global_number);
 		return -EEXIST;
 	}
@@ -664,7 +664,7 @@ int eeh_pci_enable(struct eeh_pe *pe, int function)
 	rc = eeh_ops->set_option(pe, function);
 	if (rc)
 		pr_warn("%s: Unexpected state change %d on "
-			"PHB#%d-PE#%x, err=%d\n",
+			"PHB#%x-PE#%x, err=%d\n",
 			__func__, function, pe->phb->global_number,
 			pe->addr, rc);
 
@@ -808,76 +808,67 @@ static void *eeh_set_dev_freset(void *data, void *flag)
 }
 
 /**
- * eeh_reset_pe_once - Assert the pci #RST line for 1/4 second
+ * eeh_pe_reset_full - Complete a full reset process on the indicated PE
  * @pe: EEH PE
  *
- * Assert the PCI #RST line for 1/4 second.
+ * This function executes a full reset procedure on a PE, including setting
+ * the appropriate flags, performing a fundamental or hot reset, and then
+ * deactivating the reset status.  It is designed to be used within the EEH
+ * subsystem, as opposed to eeh_pe_reset which is exported to drivers and
+ * only performs a single operation at a time.
+ *
+ * This function will attempt to reset a PE three times before failing.
  */
-static void eeh_reset_pe_once(struct eeh_pe *pe)
+int eeh_pe_reset_full(struct eeh_pe *pe)
 {
+	int active_flags = (EEH_STATE_MMIO_ACTIVE | EEH_STATE_DMA_ACTIVE);
+	int reset_state = (EEH_PE_RESET | EEH_PE_CFG_BLOCKED);
+	int type = EEH_RESET_HOT;
 	unsigned int freset = 0;
+	int i, state, ret;
 
-	/* Determine type of EEH reset required for
-	 * Partitionable Endpoint, a hot-reset (1)
-	 * or a fundamental reset (3).
-	 * A fundamental reset required by any device under
-	 * Partitionable Endpoint trumps hot-reset.
+	/*
+	 * Determine the type of reset to perform - hot or fundamental.
+	 * Hot reset is the default operation, unless any device under the
+	 * PE requires a fundamental reset.
 	 */
 	eeh_pe_dev_traverse(pe, eeh_set_dev_freset, &freset);
 
 	if (freset)
-		eeh_ops->reset(pe, EEH_RESET_FUNDAMENTAL);
-	else
-		eeh_ops->reset(pe, EEH_RESET_HOT);
-
-	eeh_ops->reset(pe, EEH_RESET_DEACTIVATE);
-}
-
-/**
- * eeh_reset_pe - Reset the indicated PE
- * @pe: EEH PE
- *
- * This routine should be called to reset indicated device, including
- * PE. A PE might include multiple PCI devices and sometimes PCI bridges
- * might be involved as well.
- */
-int eeh_reset_pe(struct eeh_pe *pe)
-{
-	int flags = (EEH_STATE_MMIO_ACTIVE | EEH_STATE_DMA_ACTIVE);
-	int i, state, ret;
+		type = EEH_RESET_FUNDAMENTAL;
 
-	/* Mark as reset and block config space */
-	eeh_pe_state_mark(pe, EEH_PE_RESET | EEH_PE_CFG_BLOCKED);
+	/* Mark the PE as in reset state and block config space accesses */
+	eeh_pe_state_mark(pe, reset_state);
 
-	/* Take three shots at resetting the bus */
+	/* Make three attempts at resetting the bus */
 	for (i = 0; i < 3; i++) {
-		eeh_reset_pe_once(pe);
+		ret = eeh_pe_reset(pe, type);
+		if (ret)
+			break;
 
-		/*
-		 * EEH_PE_ISOLATED is expected to be removed after
-		 * BAR restore.
-		 */
+		ret = eeh_pe_reset(pe, EEH_RESET_DEACTIVATE);
+		if (ret)
+			break;
+
+		/* Wait until the PE is in a functioning state */
 		state = eeh_ops->wait_state(pe, PCI_BUS_RESET_WAIT_MSEC);
-		if ((state & flags) == flags) {
-			ret = 0;
-			goto out;
-		}
+		if ((state & active_flags) == active_flags)
+			break;
 
 		if (state < 0) {
-			pr_warn("%s: Unrecoverable slot failure on PHB#%d-PE#%x",
+			pr_warn("%s: Unrecoverable slot failure on PHB#%x-PE#%x",
 				__func__, pe->phb->global_number, pe->addr);
 			ret = -ENOTRECOVERABLE;
-			goto out;
+			break;
 		}
 
-		/* We might run out of credits */
+		/* Set error in case this is our last attempt */
 		ret = -EIO;
 		pr_warn("%s: Failure %d resetting PHB#%x-PE#%x\n (%d)\n",
 			__func__, state, pe->phb->global_number, pe->addr, (i + 1));
 	}
 
-out:
-	eeh_pe_state_clear(pe, EEH_PE_RESET | EEH_PE_CFG_BLOCKED);
+	eeh_pe_state_clear(pe, reset_state);
 	return ret;
 }
 
@@ -1601,6 +1592,7 @@ static int eeh_pe_reenable_devices(struct eeh_pe *pe)
 	return eeh_unfreeze_pe(pe, true);
 }
 
+
 /**
  * eeh_pe_reset - Issue PE reset according to specified type
  * @pe: EEH PE
diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c
index 5c31369435f2..d88573bdd090 100644
--- a/arch/powerpc/kernel/eeh_driver.c
+++ b/arch/powerpc/kernel/eeh_driver.c
@@ -588,7 +588,7 @@ int eeh_pe_reset_and_recover(struct eeh_pe *pe)
 	eeh_pe_dev_traverse(pe, eeh_dev_save_state, NULL);
 
 	/* Issue reset */
-	ret = eeh_reset_pe(pe);
+	ret = eeh_pe_reset_full(pe);
 	if (ret) {
 		eeh_pe_state_clear(pe, EEH_PE_RECOVERING);
 		return ret;
@@ -659,7 +659,7 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus,
 	 * config accesses. So we prefer to block them. However, controlled
 	 * PCI config accesses initiated from EEH itself are allowed.
 	 */
-	rc = eeh_reset_pe(pe);
+	rc = eeh_pe_reset_full(pe);
 	if (rc)
 		return rc;
 
@@ -734,7 +734,7 @@ static void eeh_handle_normal_event(struct eeh_pe *pe)
 
 	frozen_bus = eeh_pe_bus_get(pe);
 	if (!frozen_bus) {
-		pr_err("%s: Cannot find PCI bus for PHB#%d-PE#%x\n",
+		pr_err("%s: Cannot find PCI bus for PHB#%x-PE#%x\n",
 			__func__, pe->phb->global_number, pe->addr);
 		return;
 	}
@@ -878,7 +878,7 @@ excess_failures:
 	 * are due to poorly seated PCI cards. Only 10% or so are
 	 * due to actual, failed cards.
 	 */
-	pr_err("EEH: PHB#%d-PE#%x has failed %d times in the\n"
+	pr_err("EEH: PHB#%x-PE#%x has failed %d times in the\n"
 	       "last hour and has been permanently disabled.\n"
 	       "Please try reseating or replacing it.\n",
 		pe->phb->global_number, pe->addr,
@@ -886,7 +886,7 @@ excess_failures:
 	goto perm_error;
 
 hard_fail:
-	pr_err("EEH: Unable to recover from failure from PHB#%d-PE#%x.\n"
+	pr_err("EEH: Unable to recover from failure from PHB#%x-PE#%x.\n"
 	       "Please try reseating or replacing it\n",
 		pe->phb->global_number, pe->addr);
 
@@ -1000,7 +1000,7 @@ static void eeh_handle_special_event(void)
 				bus = eeh_pe_bus_get(phb_pe);
 				if (!bus) {
 					pr_err("%s: Cannot find PCI bus for "
-					       "PHB#%d-PE#%x\n",
+					       "PHB#%x-PE#%x\n",
 					       __func__,
 					       pe->phb->global_number,
 					       pe->addr);
diff --git a/arch/powerpc/kernel/eeh_event.c b/arch/powerpc/kernel/eeh_event.c
index 82e7327e3cd0..accbf8b5fd46 100644
--- a/arch/powerpc/kernel/eeh_event.c
+++ b/arch/powerpc/kernel/eeh_event.c
@@ -75,11 +75,11 @@ static int eeh_event_handler(void * dummy)
 		if (pe) {
 			eeh_pe_state_mark(pe, EEH_PE_RECOVERING);
 			if (pe->type & EEH_PE_PHB)
-				pr_info("EEH: Detected error on PHB#%d\n",
+				pr_info("EEH: Detected error on PHB#%x\n",
 					 pe->phb->global_number);
 			else
 				pr_info("EEH: Detected PCI bus error on "
-					"PHB#%d-PE#%x\n",
+					"PHB#%x-PE#%x\n",
 					pe->phb->global_number, pe->addr);
 			eeh_handle_event(pe);
 			eeh_pe_state_clear(pe, EEH_PE_RECOVERING);
diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c
index de7d091c4c31..cc4b206f77e4 100644
--- a/arch/powerpc/kernel/eeh_pe.c
+++ b/arch/powerpc/kernel/eeh_pe.c
@@ -104,7 +104,7 @@ int eeh_phb_pe_create(struct pci_controller *phb)
 	/* Put it into the list */
 	list_add_tail(&pe->child, &eeh_phb_pe);
 
-	pr_debug("EEH: Add PE for PHB#%d\n", phb->global_number);
+	pr_debug("EEH: Add PE for PHB#%x\n", phb->global_number);
 
 	return 0;
 }
@@ -333,7 +333,7 @@ int eeh_add_to_parent_pe(struct eeh_dev *edev)
 
 	/* Check if the PE number is valid */
 	if (!eeh_has_flag(EEH_VALID_PE_ZERO) && !edev->pe_config_addr) {
-		pr_err("%s: Invalid PE#0 for edev 0x%x on PHB#%d\n",
+		pr_err("%s: Invalid PE#0 for edev 0x%x on PHB#%x\n",
 		       __func__, edev->config_addr, edev->phb->global_number);
 		return -EINVAL;
 	}
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 3841d749a430..5742dbdbee46 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -674,7 +674,11 @@ BEGIN_FTR_SECTION
 	mtspr	SPRN_SPEFSCR,r0		/* restore SPEFSCR reg */
 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
 #endif /* CONFIG_SPE */
-
+#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP)
+	lwz	r0,TSK_STACK_CANARY(r2)
+	lis	r4,__stack_chk_guard@ha
+	stw	r0,__stack_chk_guard@l(r4)
+#endif
 	lwz	r0,_CCR(r1)
 	mtcrf	0xFF,r0
 	/* r3-r12 are destroyed -- Cort */
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index 38a1f96430e1..45b453e4d0c8 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -923,10 +923,10 @@ kernel_dbg_exc:
 			        PROLOG_ADDITION_NONE)
 	EXCEPTION_COMMON(0x340)
 	addi	r3,r1,STACK_FRAME_OVERHEAD
-	bl	.save_nvgprs
+	bl	save_nvgprs
 	INTS_RESTORE_HARD
-	bl	.unknown_exception
-	b	.ret_from_except
+	bl	unknown_exception
+	b	ret_from_except
 
 /*
  * An interrupt came in while soft-disabled; We mark paca->irq_happened
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 1ba82ea90230..d39d6118c6e9 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -1408,7 +1408,7 @@ USE_TEXT_SECTION()
 /*
  * Hash table stuff
  */
-	.align	7
+	.balign	IFETCH_ALIGN_BYTES
 do_hash_page:
 #ifdef CONFIG_PPC_STD_MMU_64
 	andis.	r0,r4,0xa410		/* weird error? */
diff --git a/arch/powerpc/kernel/ftrace.c b/arch/powerpc/kernel/ftrace.c
index a95639b8d4ac..5c9f50c1aa99 100644
--- a/arch/powerpc/kernel/ftrace.c
+++ b/arch/powerpc/kernel/ftrace.c
@@ -47,13 +47,11 @@ ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
 	unsigned int replaced;
 
 	/*
-	 * Note: Due to modules and __init, code can
-	 *  disappear and change, we need to protect against faulting
-	 *  as well as code changing. We do this by using the
-	 *  probe_kernel_* functions.
-	 *
-	 * No real locking needed, this code is run through
-	 * kstop_machine, or before SMP starts.
+	 * Note:
+	 * We are paranoid about modifying text, as if a bug was to happen, it
+	 * could cause us to read or write to someplace that could cause harm.
+	 * Carefully read and modify the code with probe_kernel_*(), and make
+	 * sure what we read is what we expected it to be before modifying it.
 	 */
 
 	/* read the text we want to modify */
diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S
index 04c546e20cc0..1dc5eae2ced3 100644
--- a/arch/powerpc/kernel/head_64.S
+++ b/arch/powerpc/kernel/head_64.S
@@ -107,12 +107,19 @@ __secondary_hold_acknowledge:
 	 * crash_kernel region.  The loader is responsible for
 	 * observing the alignment requirement.
 	 */
+
+#ifdef CONFIG_RELOCATABLE_TEST
+#define RUN_AT_LOAD_DEFAULT 1		/* Test relocation, do not copy to 0 */
+#else
+#define RUN_AT_LOAD_DEFAULT 0x72756e30  /* "run0" -- relocate to 0 by default */
+#endif
+
 	/* Do not move this variable as kexec-tools knows about it. */
 	. = 0x5c
 	.globl	__run_at_load
 __run_at_load:
 DEFINE_FIXED_SYMBOL(__run_at_load)
-	.long	0x72756e30	/* "run0" -- relocate to 0 by default */
+	.long	RUN_AT_LOAD_DEFAULT
 #endif
 
 	. = 0x60
@@ -153,7 +160,7 @@ __secondary_hold:
 	cmpdi	0,r12,0
 	beq	100b
 
-#if defined(CONFIG_SMP) || defined(CONFIG_KEXEC)
+#if defined(CONFIG_SMP) || defined(CONFIG_KEXEC_CORE)
 #ifdef CONFIG_PPC_BOOK3E
 	tovirt(r12,r12)
 #endif
@@ -214,9 +221,9 @@ booting_thread_hwid:
  */
 _GLOBAL(book3e_start_thread)
 	LOAD_REG_IMMEDIATE(r5, MSR_KERNEL)
-	cmpi	0, r3, 0
+	cmpwi	r3, 0
 	beq	10f
-	cmpi	0, r3, 1
+	cmpwi	r3, 1
 	beq	11f
 	/* If the thread id is invalid, just exit. */
 	b	13f
@@ -241,9 +248,9 @@ _GLOBAL(book3e_start_thread)
  * r3 = the thread physical id
  */
 _GLOBAL(book3e_stop_thread)
-	cmpi	0, r3, 0
+	cmpwi	r3, 0
 	beq	10f
-	cmpi	0, r3, 1
+	cmpwi	r3, 1
 	beq	10f
 	/* If the thread id is invalid, just exit. */
 	b	13f
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index fb133a163263..1a9c99d3e5d8 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -73,6 +73,9 @@
 #define RPN_PATTERN	0x00f0
 #endif
 
+#define PAGE_SHIFT_512K		19
+#define PAGE_SHIFT_8M		23
+
 	__HEAD
 _ENTRY(_stext);
 _ENTRY(_start);
@@ -322,7 +325,7 @@ SystemCall:
 #endif
 
 InstructionTLBMiss:
-#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
+#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
 	mtspr	SPRN_SPRG_SCRATCH2, r3
 #endif
 	EXCEPTION_PROLOG_0
@@ -332,10 +335,12 @@ InstructionTLBMiss:
 	 */
 	mfspr	r10, SPRN_SRR0	/* Get effective address of fault */
 	INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10)
-#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
 	/* Only modules will cause ITLB Misses as we always
 	 * pin the first 8MB of kernel memory */
+#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
 	mfcr	r3
+#endif
+#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
 	IS_KERNEL(r11, r10)
 #endif
 	mfspr	r11, SPRN_M_TW	/* Get level 1 table */
@@ -343,7 +348,6 @@ InstructionTLBMiss:
 	BRANCH_UNLESS_KERNEL(3f)
 	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha
 3:
-	mtcr	r3
 #endif
 	/* Insert level 1 index */
 	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
@@ -351,14 +355,25 @@ InstructionTLBMiss:
 
 	/* Extract level 2 index */
 	rlwinm	r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
+#ifdef CONFIG_HUGETLB_PAGE
+	mtcr	r11
+	bt-	28, 10f		/* bit 28 = Large page (8M) */
+	bt-	29, 20f		/* bit 29 = Large page (8M or 512k) */
+#endif
 	rlwimi	r10, r11, 0, 0, 32 - PAGE_SHIFT - 1	/* Add level 2 base */
 	lwz	r10, 0(r10)	/* Get the pte */
-
+4:
+#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
+	mtcr	r3
+#endif
 	/* Insert the APG into the TWC from the Linux PTE. */
 	rlwimi	r11, r10, 0, 25, 26
 	/* Load the MI_TWC with the attributes for this "segment." */
 	MTSPR_CPU6(SPRN_MI_TWC, r11, r3)	/* Set segment attributes */
 
+#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES)
+	rlwimi	r10, r11, 1, MI_SPS16K
+#endif
 #ifdef CONFIG_SWAP
 	rlwinm	r11, r10, 32-5, _PAGE_PRESENT
 	and	r11, r11, r10
@@ -371,16 +386,45 @@ InstructionTLBMiss:
 	 * set.  All other Linux PTE bits control the behavior
 	 * of the MMU.
 	 */
+#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES)
+	rlwimi	r10, r11, 0, 0x0ff0	/* Set 24-27, clear 20-23 */
+#else
 	rlwimi	r10, r11, 0, 0x0ff8	/* Set 24-27, clear 20-23,28 */
+#endif
 	MTSPR_CPU6(SPRN_MI_RPN, r10, r3)	/* Update TLB entry */
 
 	/* Restore registers */
-#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC)
+#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE)
 	mfspr	r3, SPRN_SPRG_SCRATCH2
 #endif
 	EXCEPTION_EPILOG_0
 	rfi
 
+#ifdef CONFIG_HUGETLB_PAGE
+10:	/* 8M pages */
+#ifdef CONFIG_PPC_16K_PAGES
+	/* Extract level 2 index */
+	rlwinm	r10, r10, 32 - (PAGE_SHIFT_8M - PAGE_SHIFT), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29
+	/* Add level 2 base */
+	rlwimi	r10, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1
+#else
+	/* Level 2 base */
+	rlwinm	r10, r11, 0, ~HUGEPD_SHIFT_MASK
+#endif
+	lwz	r10, 0(r10)	/* Get the pte */
+	rlwinm	r11, r11, 0, 0xf
+	b	4b
+
+20:	/* 512k pages */
+	/* Extract level 2 index */
+	rlwinm	r10, r10, 32 - (PAGE_SHIFT_512K - PAGE_SHIFT), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29
+	/* Add level 2 base */
+	rlwimi	r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
+	lwz	r10, 0(r10)	/* Get the pte */
+	rlwinm	r11, r11, 0, 0xf
+	b	4b
+#endif
+
 	. = 0x1200
 DataStoreTLBMiss:
 	mtspr	SPRN_SPRG_SCRATCH2, r3
@@ -407,7 +451,6 @@ _ENTRY(DTLBMiss_jmp)
 #endif
 	blt	cr7, DTLBMissLinear
 3:
-	mtcr	r3
 	mfspr	r10, SPRN_MD_EPN
 
 	/* Insert level 1 index */
@@ -418,8 +461,15 @@ _ENTRY(DTLBMiss_jmp)
 	 */
 	/* Extract level 2 index */
 	rlwinm	r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
+#ifdef CONFIG_HUGETLB_PAGE
+	mtcr	r11
+	bt-	28, 10f		/* bit 28 = Large page (8M) */
+	bt-	29, 20f		/* bit 29 = Large page (8M or 512k) */
+#endif
 	rlwimi	r10, r11, 0, 0, 32 - PAGE_SHIFT - 1	/* Add level 2 base */
 	lwz	r10, 0(r10)	/* Get the pte */
+4:
+	mtcr	r3
 
 	/* Insert the Guarded flag and APG into the TWC from the Linux PTE.
 	 * It is bit 26-27 of both the Linux PTE and the TWC (at least
@@ -434,6 +484,11 @@ _ENTRY(DTLBMiss_jmp)
 	rlwimi	r11, r10, 32-5, 30, 30
 	MTSPR_CPU6(SPRN_MD_TWC, r11, r3)
 
+	/* In 4k pages mode, SPS (bit 28) in RPN must match PS[1] (bit 29)
+	 * In 16k pages mode, SPS is always 1 */
+#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES)
+	rlwimi	r10, r11, 1, MD_SPS16K
+#endif
 	/* Both _PAGE_ACCESSED and _PAGE_PRESENT has to be set.
 	 * We also need to know if the insn is a load/store, so:
 	 * Clear _PAGE_PRESENT and load that which will
@@ -455,7 +510,11 @@ _ENTRY(DTLBMiss_jmp)
 	 * of the MMU.
 	 */
 	li	r11, RPN_PATTERN
+#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES)
+	rlwimi	r10, r11, 0, 24, 27	/* Set 24-27 */
+#else
 	rlwimi	r10, r11, 0, 24, 28	/* Set 24-27, clear 28 */
+#endif
 	rlwimi	r10, r11, 0, 20, 20	/* clear 20 */
 	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */
 
@@ -465,6 +524,30 @@ _ENTRY(DTLBMiss_jmp)
 	EXCEPTION_EPILOG_0
 	rfi
 
+#ifdef CONFIG_HUGETLB_PAGE
+10:	/* 8M pages */
+	/* Extract level 2 index */
+#ifdef CONFIG_PPC_16K_PAGES
+	rlwinm	r10, r10, 32 - (PAGE_SHIFT_8M - PAGE_SHIFT), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29
+	/* Add level 2 base */
+	rlwimi	r10, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1
+#else
+	/* Level 2 base */
+	rlwinm	r10, r11, 0, ~HUGEPD_SHIFT_MASK
+#endif
+	lwz	r10, 0(r10)	/* Get the pte */
+	rlwinm	r11, r11, 0, 0xf
+	b	4b
+
+20:	/* 512k pages */
+	/* Extract level 2 index */
+	rlwinm	r10, r10, 32 - (PAGE_SHIFT_512K - PAGE_SHIFT), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29
+	/* Add level 2 base */
+	rlwimi	r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
+	lwz	r10, 0(r10)	/* Get the pte */
+	rlwinm	r11, r11, 0, 0xf
+	b	4b
+#endif
 
 /* This is an instruction TLB error on the MPC8xx.  This could be due
  * to many reasons, such as executing guarded memory or illegal instruction
@@ -586,6 +669,9 @@ _ENTRY(FixupDAR_cmp)
 	/* Insert level 1 index */
 3:	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
 	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */
+	mtcr	r11
+	bt	28,200f		/* bit 28 = Large page (8M) */
+	bt	29,202f		/* bit 29 = Large page (8M or 512K) */
 	rlwinm	r11, r11,0,0,19	/* Extract page descriptor page address */
 	/* Insert level 2 index */
 	rlwimi	r11, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
@@ -611,6 +697,27 @@ _ENTRY(FixupDAR_cmp)
 141:	mfspr	r10,SPRN_SPRG_SCRATCH2
 	b	DARFixed	/* Nope, go back to normal TLB processing */
 
+	/* concat physical page address(r11) and page offset(r10) */
+200:
+#ifdef CONFIG_PPC_16K_PAGES
+	rlwinm	r11, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1
+	rlwimi	r11, r10, 32 - (PAGE_SHIFT_8M - 2), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29
+#else
+	rlwinm	r11, r10, 0, ~HUGEPD_SHIFT_MASK
+#endif
+	lwz	r11, 0(r11)	/* Get the pte */
+	/* concat physical page address(r11) and page offset(r10) */
+	rlwimi	r11, r10, 0, 32 - PAGE_SHIFT_8M, 31
+	b	201b
+
+202:
+	rlwinm	r11, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
+	rlwimi	r11, r10, 32 - (PAGE_SHIFT_512K - 2), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29
+	lwz	r11, 0(r11)	/* Get the pte */
+	/* concat physical page address(r11) and page offset(r10) */
+	rlwimi	r11, r10, 0, 32 - PAGE_SHIFT_512K, 31
+	b	201b
+
 144:	mfspr	r10, SPRN_DSISR
 	rlwinm	r10, r10,0,7,5	/* Clear store bit for buggy dcbst insn */
 	mtspr	SPRN_DSISR, r10
diff --git a/arch/powerpc/kernel/kexec_elf_64.c b/arch/powerpc/kernel/kexec_elf_64.c
new file mode 100644
index 000000000000..6acffd34a70f
--- /dev/null
+++ b/arch/powerpc/kernel/kexec_elf_64.c
@@ -0,0 +1,663 @@
+/*
+ * Load ELF vmlinux file for the kexec_file_load syscall.
+ *
+ * Copyright (C) 2004  Adam Litke (agl@us.ibm.com)
+ * Copyright (C) 2004  IBM Corp.
+ * Copyright (C) 2005  R Sharada (sharada@in.ibm.com)
+ * Copyright (C) 2006  Mohan Kumar M (mohan@in.ibm.com)
+ * Copyright (C) 2016  IBM Corporation
+ *
+ * Based on kexec-tools' kexec-elf-exec.c and kexec-elf-ppc64.c.
+ * Heavily modified for the kernel by
+ * Thiago Jung Bauermann <bauerman@linux.vnet.ibm.com>.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation (version 2 of the License).
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt)	"kexec_elf: " fmt
+
+#include <linux/elf.h>
+#include <linux/kexec.h>
+#include <linux/libfdt.h>
+#include <linux/module.h>
+#include <linux/of_fdt.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#define PURGATORY_STACK_SIZE	(16 * 1024)
+
+#define elf_addr_to_cpu	elf64_to_cpu
+
+#ifndef Elf_Rel
+#define Elf_Rel		Elf64_Rel
+#endif /* Elf_Rel */
+
+struct elf_info {
+	/*
+	 * Where the ELF binary contents are kept.
+	 * Memory managed by the user of the struct.
+	 */
+	const char *buffer;
+
+	const struct elfhdr *ehdr;
+	const struct elf_phdr *proghdrs;
+	struct elf_shdr *sechdrs;
+};
+
+static inline bool elf_is_elf_file(const struct elfhdr *ehdr)
+{
+       return memcmp(ehdr->e_ident, ELFMAG, SELFMAG) == 0;
+}
+
+static uint64_t elf64_to_cpu(const struct elfhdr *ehdr, uint64_t value)
+{
+	if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
+		value = le64_to_cpu(value);
+	else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
+		value = be64_to_cpu(value);
+
+	return value;
+}
+
+static uint16_t elf16_to_cpu(const struct elfhdr *ehdr, uint16_t value)
+{
+	if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
+		value = le16_to_cpu(value);
+	else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
+		value = be16_to_cpu(value);
+
+	return value;
+}
+
+static uint32_t elf32_to_cpu(const struct elfhdr *ehdr, uint32_t value)
+{
+	if (ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
+		value = le32_to_cpu(value);
+	else if (ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
+		value = be32_to_cpu(value);
+
+	return value;
+}
+
+/**
+ * elf_is_ehdr_sane - check that it is safe to use the ELF header
+ * @buf_len:	size of the buffer in which the ELF file is loaded.
+ */
+static bool elf_is_ehdr_sane(const struct elfhdr *ehdr, size_t buf_len)
+{
+	if (ehdr->e_phnum > 0 && ehdr->e_phentsize != sizeof(struct elf_phdr)) {
+		pr_debug("Bad program header size.\n");
+		return false;
+	} else if (ehdr->e_shnum > 0 &&
+		   ehdr->e_shentsize != sizeof(struct elf_shdr)) {
+		pr_debug("Bad section header size.\n");
+		return false;
+	} else if (ehdr->e_ident[EI_VERSION] != EV_CURRENT ||
+		   ehdr->e_version != EV_CURRENT) {
+		pr_debug("Unknown ELF version.\n");
+		return false;
+	}
+
+	if (ehdr->e_phoff > 0 && ehdr->e_phnum > 0) {
+		size_t phdr_size;
+
+		/*
+		 * e_phnum is at most 65535 so calculating the size of the
+		 * program header cannot overflow.
+		 */
+		phdr_size = sizeof(struct elf_phdr) * ehdr->e_phnum;
+
+		/* Sanity check the program header table location. */
+		if (ehdr->e_phoff + phdr_size < ehdr->e_phoff) {
+			pr_debug("Program headers at invalid location.\n");
+			return false;
+		} else if (ehdr->e_phoff + phdr_size > buf_len) {
+			pr_debug("Program headers truncated.\n");
+			return false;
+		}
+	}
+
+	if (ehdr->e_shoff > 0 && ehdr->e_shnum > 0) {
+		size_t shdr_size;
+
+		/*
+		 * e_shnum is at most 65536 so calculating
+		 * the size of the section header cannot overflow.
+		 */
+		shdr_size = sizeof(struct elf_shdr) * ehdr->e_shnum;
+
+		/* Sanity check the section header table location. */
+		if (ehdr->e_shoff + shdr_size < ehdr->e_shoff) {
+			pr_debug("Section headers at invalid location.\n");
+			return false;
+		} else if (ehdr->e_shoff + shdr_size > buf_len) {
+			pr_debug("Section headers truncated.\n");
+			return false;
+		}
+	}
+
+	return true;
+}
+
+static int elf_read_ehdr(const char *buf, size_t len, struct elfhdr *ehdr)
+{
+	struct elfhdr *buf_ehdr;
+
+	if (len < sizeof(*buf_ehdr)) {
+		pr_debug("Buffer is too small to hold ELF header.\n");
+		return -ENOEXEC;
+	}
+
+	memset(ehdr, 0, sizeof(*ehdr));
+	memcpy(ehdr->e_ident, buf, sizeof(ehdr->e_ident));
+	if (!elf_is_elf_file(ehdr)) {
+		pr_debug("No ELF header magic.\n");
+		return -ENOEXEC;
+	}
+
+	if (ehdr->e_ident[EI_CLASS] != ELF_CLASS) {
+		pr_debug("Not a supported ELF class.\n");
+		return -ENOEXEC;
+	} else  if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB &&
+		ehdr->e_ident[EI_DATA] != ELFDATA2MSB) {
+		pr_debug("Not a supported ELF data format.\n");
+		return -ENOEXEC;
+	}
+
+	buf_ehdr = (struct elfhdr *) buf;
+	if (elf16_to_cpu(ehdr, buf_ehdr->e_ehsize) != sizeof(*buf_ehdr)) {
+		pr_debug("Bad ELF header size.\n");
+		return -ENOEXEC;
+	}
+
+	ehdr->e_type      = elf16_to_cpu(ehdr, buf_ehdr->e_type);
+	ehdr->e_machine   = elf16_to_cpu(ehdr, buf_ehdr->e_machine);
+	ehdr->e_version   = elf32_to_cpu(ehdr, buf_ehdr->e_version);
+	ehdr->e_entry     = elf_addr_to_cpu(ehdr, buf_ehdr->e_entry);
+	ehdr->e_phoff     = elf_addr_to_cpu(ehdr, buf_ehdr->e_phoff);
+	ehdr->e_shoff     = elf_addr_to_cpu(ehdr, buf_ehdr->e_shoff);
+	ehdr->e_flags     = elf32_to_cpu(ehdr, buf_ehdr->e_flags);
+	ehdr->e_phentsize = elf16_to_cpu(ehdr, buf_ehdr->e_phentsize);
+	ehdr->e_phnum     = elf16_to_cpu(ehdr, buf_ehdr->e_phnum);
+	ehdr->e_shentsize = elf16_to_cpu(ehdr, buf_ehdr->e_shentsize);
+	ehdr->e_shnum     = elf16_to_cpu(ehdr, buf_ehdr->e_shnum);
+	ehdr->e_shstrndx  = elf16_to_cpu(ehdr, buf_ehdr->e_shstrndx);
+
+	return elf_is_ehdr_sane(ehdr, len) ? 0 : -ENOEXEC;
+}
+
+/**
+ * elf_is_phdr_sane - check that it is safe to use the program header
+ * @buf_len:	size of the buffer in which the ELF file is loaded.
+ */
+static bool elf_is_phdr_sane(const struct elf_phdr *phdr, size_t buf_len)
+{
+
+	if (phdr->p_offset + phdr->p_filesz < phdr->p_offset) {
+		pr_debug("ELF segment location wraps around.\n");
+		return false;
+	} else if (phdr->p_offset + phdr->p_filesz > buf_len) {
+		pr_debug("ELF segment not in file.\n");
+		return false;
+	} else if (phdr->p_paddr + phdr->p_memsz < phdr->p_paddr) {
+		pr_debug("ELF segment address wraps around.\n");
+		return false;
+	}
+
+	return true;
+}
+
+static int elf_read_phdr(const char *buf, size_t len, struct elf_info *elf_info,
+			 int idx)
+{
+	/* Override the const in proghdrs, we are the ones doing the loading. */
+	struct elf_phdr *phdr = (struct elf_phdr *) &elf_info->proghdrs[idx];
+	const char *pbuf;
+	struct elf_phdr *buf_phdr;
+
+	pbuf = buf + elf_info->ehdr->e_phoff + (idx * sizeof(*buf_phdr));
+	buf_phdr = (struct elf_phdr *) pbuf;
+
+	phdr->p_type   = elf32_to_cpu(elf_info->ehdr, buf_phdr->p_type);
+	phdr->p_offset = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_offset);
+	phdr->p_paddr  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_paddr);
+	phdr->p_vaddr  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_vaddr);
+	phdr->p_flags  = elf32_to_cpu(elf_info->ehdr, buf_phdr->p_flags);
+
+	/*
+	 * The following fields have a type equivalent to Elf_Addr
+	 * both in 32 bit and 64 bit ELF.
+	 */
+	phdr->p_filesz = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_filesz);
+	phdr->p_memsz  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_memsz);
+	phdr->p_align  = elf_addr_to_cpu(elf_info->ehdr, buf_phdr->p_align);
+
+	return elf_is_phdr_sane(phdr, len) ? 0 : -ENOEXEC;
+}
+
+/**
+ * elf_read_phdrs - read the program headers from the buffer
+ *
+ * This function assumes that the program header table was checked for sanity.
+ * Use elf_is_ehdr_sane() if it wasn't.
+ */
+static int elf_read_phdrs(const char *buf, size_t len,
+			  struct elf_info *elf_info)
+{
+	size_t phdr_size, i;
+	const struct elfhdr *ehdr = elf_info->ehdr;
+
+	/*
+	 * e_phnum is at most 65535 so calculating the size of the
+	 * program header cannot overflow.
+	 */
+	phdr_size = sizeof(struct elf_phdr) * ehdr->e_phnum;
+
+	elf_info->proghdrs = kzalloc(phdr_size, GFP_KERNEL);
+	if (!elf_info->proghdrs)
+		return -ENOMEM;
+
+	for (i = 0; i < ehdr->e_phnum; i++) {
+		int ret;
+
+		ret = elf_read_phdr(buf, len, elf_info, i);
+		if (ret) {
+			kfree(elf_info->proghdrs);
+			elf_info->proghdrs = NULL;
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * elf_is_shdr_sane - check that it is safe to use the section header
+ * @buf_len:	size of the buffer in which the ELF file is loaded.
+ */
+static bool elf_is_shdr_sane(const struct elf_shdr *shdr, size_t buf_len)
+{
+	bool size_ok;
+
+	/* SHT_NULL headers have undefined values, so we can't check them. */
+	if (shdr->sh_type == SHT_NULL)
+		return true;
+
+	/* Now verify sh_entsize */
+	switch (shdr->sh_type) {
+	case SHT_SYMTAB:
+		size_ok = shdr->sh_entsize == sizeof(Elf_Sym);
+		break;
+	case SHT_RELA:
+		size_ok = shdr->sh_entsize == sizeof(Elf_Rela);
+		break;
+	case SHT_DYNAMIC:
+		size_ok = shdr->sh_entsize == sizeof(Elf_Dyn);
+		break;
+	case SHT_REL:
+		size_ok = shdr->sh_entsize == sizeof(Elf_Rel);
+		break;
+	case SHT_NOTE:
+	case SHT_PROGBITS:
+	case SHT_HASH:
+	case SHT_NOBITS:
+	default:
+		/*
+		 * This is a section whose entsize requirements
+		 * I don't care about.  If I don't know about
+		 * the section I can't care about it's entsize
+		 * requirements.
+		 */
+		size_ok = true;
+		break;
+	}
+
+	if (!size_ok) {
+		pr_debug("ELF section with wrong entry size.\n");
+		return false;
+	} else if (shdr->sh_addr + shdr->sh_size < shdr->sh_addr) {
+		pr_debug("ELF section address wraps around.\n");
+		return false;
+	}
+
+	if (shdr->sh_type != SHT_NOBITS) {
+		if (shdr->sh_offset + shdr->sh_size < shdr->sh_offset) {
+			pr_debug("ELF section location wraps around.\n");
+			return false;
+		} else if (shdr->sh_offset + shdr->sh_size > buf_len) {
+			pr_debug("ELF section not in file.\n");
+			return false;
+		}
+	}
+
+	return true;
+}
+
+static int elf_read_shdr(const char *buf, size_t len, struct elf_info *elf_info,
+			 int idx)
+{
+	struct elf_shdr *shdr = &elf_info->sechdrs[idx];
+	const struct elfhdr *ehdr = elf_info->ehdr;
+	const char *sbuf;
+	struct elf_shdr *buf_shdr;
+
+	sbuf = buf + ehdr->e_shoff + idx * sizeof(*buf_shdr);
+	buf_shdr = (struct elf_shdr *) sbuf;
+
+	shdr->sh_name      = elf32_to_cpu(ehdr, buf_shdr->sh_name);
+	shdr->sh_type      = elf32_to_cpu(ehdr, buf_shdr->sh_type);
+	shdr->sh_addr      = elf_addr_to_cpu(ehdr, buf_shdr->sh_addr);
+	shdr->sh_offset    = elf_addr_to_cpu(ehdr, buf_shdr->sh_offset);
+	shdr->sh_link      = elf32_to_cpu(ehdr, buf_shdr->sh_link);
+	shdr->sh_info      = elf32_to_cpu(ehdr, buf_shdr->sh_info);
+
+	/*
+	 * The following fields have a type equivalent to Elf_Addr
+	 * both in 32 bit and 64 bit ELF.
+	 */
+	shdr->sh_flags     = elf_addr_to_cpu(ehdr, buf_shdr->sh_flags);
+	shdr->sh_size      = elf_addr_to_cpu(ehdr, buf_shdr->sh_size);
+	shdr->sh_addralign = elf_addr_to_cpu(ehdr, buf_shdr->sh_addralign);
+	shdr->sh_entsize   = elf_addr_to_cpu(ehdr, buf_shdr->sh_entsize);
+
+	return elf_is_shdr_sane(shdr, len) ? 0 : -ENOEXEC;
+}
+
+/**
+ * elf_read_shdrs - read the section headers from the buffer
+ *
+ * This function assumes that the section header table was checked for sanity.
+ * Use elf_is_ehdr_sane() if it wasn't.
+ */
+static int elf_read_shdrs(const char *buf, size_t len,
+			  struct elf_info *elf_info)
+{
+	size_t shdr_size, i;
+
+	/*
+	 * e_shnum is at most 65536 so calculating
+	 * the size of the section header cannot overflow.
+	 */
+	shdr_size = sizeof(struct elf_shdr) * elf_info->ehdr->e_shnum;
+
+	elf_info->sechdrs = kzalloc(shdr_size, GFP_KERNEL);
+	if (!elf_info->sechdrs)
+		return -ENOMEM;
+
+	for (i = 0; i < elf_info->ehdr->e_shnum; i++) {
+		int ret;
+
+		ret = elf_read_shdr(buf, len, elf_info, i);
+		if (ret) {
+			kfree(elf_info->sechdrs);
+			elf_info->sechdrs = NULL;
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * elf_read_from_buffer - read ELF file and sets up ELF header and ELF info
+ * @buf:	Buffer to read ELF file from.
+ * @len:	Size of @buf.
+ * @ehdr:	Pointer to existing struct which will be populated.
+ * @elf_info:	Pointer to existing struct which will be populated.
+ *
+ * This function allows reading ELF files with different byte order than
+ * the kernel, byte-swapping the fields as needed.
+ *
+ * Return:
+ * On success returns 0, and the caller should call elf_free_info(elf_info) to
+ * free the memory allocated for the section and program headers.
+ */
+int elf_read_from_buffer(const char *buf, size_t len, struct elfhdr *ehdr,
+			 struct elf_info *elf_info)
+{
+	int ret;
+
+	ret = elf_read_ehdr(buf, len, ehdr);
+	if (ret)
+		return ret;
+
+	elf_info->buffer = buf;
+	elf_info->ehdr = ehdr;
+	if (ehdr->e_phoff > 0 && ehdr->e_phnum > 0) {
+		ret = elf_read_phdrs(buf, len, elf_info);
+		if (ret)
+			return ret;
+	}
+	if (ehdr->e_shoff > 0 && ehdr->e_shnum > 0) {
+		ret = elf_read_shdrs(buf, len, elf_info);
+		if (ret) {
+			kfree(elf_info->proghdrs);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * elf_free_info - free memory allocated by elf_read_from_buffer
+ */
+void elf_free_info(struct elf_info *elf_info)
+{
+	kfree(elf_info->proghdrs);
+	kfree(elf_info->sechdrs);
+	memset(elf_info, 0, sizeof(*elf_info));
+}
+/**
+ * build_elf_exec_info - read ELF executable and check that we can use it
+ */
+static int build_elf_exec_info(const char *buf, size_t len, struct elfhdr *ehdr,
+			       struct elf_info *elf_info)
+{
+	int i;
+	int ret;
+
+	ret = elf_read_from_buffer(buf, len, ehdr, elf_info);
+	if (ret)
+		return ret;
+
+	/* Big endian vmlinux has type ET_DYN. */
+	if (ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN) {
+		pr_err("Not an ELF executable.\n");
+		goto error;
+	} else if (!elf_info->proghdrs) {
+		pr_err("No ELF program header.\n");
+		goto error;
+	}
+
+	for (i = 0; i < ehdr->e_phnum; i++) {
+		/*
+		 * Kexec does not support loading interpreters.
+		 * In addition this check keeps us from attempting
+		 * to kexec ordinay executables.
+		 */
+		if (elf_info->proghdrs[i].p_type == PT_INTERP) {
+			pr_err("Requires an ELF interpreter.\n");
+			goto error;
+		}
+	}
+
+	return 0;
+error:
+	elf_free_info(elf_info);
+	return -ENOEXEC;
+}
+
+static int elf64_probe(const char *buf, unsigned long len)
+{
+	struct elfhdr ehdr;
+	struct elf_info elf_info;
+	int ret;
+
+	ret = build_elf_exec_info(buf, len, &ehdr, &elf_info);
+	if (ret)
+		return ret;
+
+	elf_free_info(&elf_info);
+
+	return elf_check_arch(&ehdr) ? 0 : -ENOEXEC;
+}
+
+/**
+ * elf_exec_load - load ELF executable image
+ * @lowest_load_addr:	On return, will be the address where the first PT_LOAD
+ *			section will be loaded in memory.
+ *
+ * Return:
+ * 0 on success, negative value on failure.
+ */
+static int elf_exec_load(struct kimage *image, struct elfhdr *ehdr,
+			 struct elf_info *elf_info,
+			 unsigned long *lowest_load_addr)
+{
+	unsigned long base = 0, lowest_addr = UINT_MAX;
+	int ret;
+	size_t i;
+	struct kexec_buf kbuf = { .image = image, .buf_max = ppc64_rma_size,
+				  .top_down = false };
+
+	/* Read in the PT_LOAD segments. */
+	for (i = 0; i < ehdr->e_phnum; i++) {
+		unsigned long load_addr;
+		size_t size;
+		const struct elf_phdr *phdr;
+
+		phdr = &elf_info->proghdrs[i];
+		if (phdr->p_type != PT_LOAD)
+			continue;
+
+		size = phdr->p_filesz;
+		if (size > phdr->p_memsz)
+			size = phdr->p_memsz;
+
+		kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset;
+		kbuf.bufsz = size;
+		kbuf.memsz = phdr->p_memsz;
+		kbuf.buf_align = phdr->p_align;
+		kbuf.buf_min = phdr->p_paddr + base;
+		ret = kexec_add_buffer(&kbuf);
+		if (ret)
+			goto out;
+		load_addr = kbuf.mem;
+
+		if (load_addr < lowest_addr)
+			lowest_addr = load_addr;
+	}
+
+	/* Update entry point to reflect new load address. */
+	ehdr->e_entry += base;
+
+	*lowest_load_addr = lowest_addr;
+	ret = 0;
+ out:
+	return ret;
+}
+
+static void *elf64_load(struct kimage *image, char *kernel_buf,
+			unsigned long kernel_len, char *initrd,
+			unsigned long initrd_len, char *cmdline,
+			unsigned long cmdline_len)
+{
+	int ret;
+	unsigned int fdt_size;
+	unsigned long kernel_load_addr, purgatory_load_addr;
+	unsigned long initrd_load_addr = 0, fdt_load_addr;
+	void *fdt;
+	const void *slave_code;
+	struct elfhdr ehdr;
+	struct elf_info elf_info;
+	struct kexec_buf kbuf = { .image = image, .buf_min = 0,
+				  .buf_max = ppc64_rma_size };
+
+	ret = build_elf_exec_info(kernel_buf, kernel_len, &ehdr, &elf_info);
+	if (ret)
+		goto out;
+
+	ret = elf_exec_load(image, &ehdr, &elf_info, &kernel_load_addr);
+	if (ret)
+		goto out;
+
+	pr_debug("Loaded the kernel at 0x%lx\n", kernel_load_addr);
+
+	ret = kexec_load_purgatory(image, 0, ppc64_rma_size, true,
+				   &purgatory_load_addr);
+	if (ret) {
+		pr_err("Loading purgatory failed.\n");
+		goto out;
+	}
+
+	pr_debug("Loaded purgatory at 0x%lx\n", purgatory_load_addr);
+
+	if (initrd != NULL) {
+		kbuf.buffer = initrd;
+		kbuf.bufsz = kbuf.memsz = initrd_len;
+		kbuf.buf_align = PAGE_SIZE;
+		kbuf.top_down = false;
+		ret = kexec_add_buffer(&kbuf);
+		if (ret)
+			goto out;
+		initrd_load_addr = kbuf.mem;
+
+		pr_debug("Loaded initrd at 0x%lx\n", initrd_load_addr);
+	}
+
+	fdt_size = fdt_totalsize(initial_boot_params) * 2;
+	fdt = kmalloc(fdt_size, GFP_KERNEL);
+	if (!fdt) {
+		pr_err("Not enough memory for the device tree.\n");
+		ret = -ENOMEM;
+		goto out;
+	}
+	ret = fdt_open_into(initial_boot_params, fdt, fdt_size);
+	if (ret < 0) {
+		pr_err("Error setting up the new device tree.\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ret = setup_new_fdt(fdt, initrd_load_addr, initrd_len, cmdline);
+	if (ret)
+		goto out;
+
+	fdt_pack(fdt);
+
+	kbuf.buffer = fdt;
+	kbuf.bufsz = kbuf.memsz = fdt_size;
+	kbuf.buf_align = PAGE_SIZE;
+	kbuf.top_down = true;
+	ret = kexec_add_buffer(&kbuf);
+	if (ret)
+		goto out;
+	fdt_load_addr = kbuf.mem;
+
+	pr_debug("Loaded device tree at 0x%lx\n", fdt_load_addr);
+
+	slave_code = elf_info.buffer + elf_info.proghdrs[0].p_offset;
+	ret = setup_purgatory(image, slave_code, fdt, kernel_load_addr,
+			      fdt_load_addr);
+	if (ret)
+		pr_err("Error setting up the purgatory.\n");
+
+out:
+	elf_free_info(&elf_info);
+
+	/* Make kimage_file_post_load_cleanup free the fdt buffer for us. */
+	return ret ? ERR_PTR(ret) : fdt;
+}
+
+struct kexec_file_ops kexec_elf64_ops = {
+	.probe = elf64_probe,
+	.load = elf64_load,
+};
diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c
index e785cc9e1ecd..ad108b842669 100644
--- a/arch/powerpc/kernel/kprobes.c
+++ b/arch/powerpc/kernel/kprobes.c
@@ -140,13 +140,16 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
 	regs->link = (unsigned long)kretprobe_trampoline;
 }
 
-static int __kprobes kprobe_handler(struct pt_regs *regs)
+int __kprobes kprobe_handler(struct pt_regs *regs)
 {
 	struct kprobe *p;
 	int ret = 0;
 	unsigned int *addr = (unsigned int *)regs->nip;
 	struct kprobe_ctlblk *kcb;
 
+	if (user_mode(regs))
+		return 0;
+
 	/*
 	 * We don't want to be preempted for the entire
 	 * duration of kprobe processing
@@ -359,12 +362,12 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p,
  * single-stepped a copy of the instruction.  The address of this
  * copy is p->ainsn.insn.
  */
-static int __kprobes post_kprobe_handler(struct pt_regs *regs)
+int __kprobes kprobe_post_handler(struct pt_regs *regs)
 {
 	struct kprobe *cur = kprobe_running();
 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 
-	if (!cur)
+	if (!cur || user_mode(regs))
 		return 0;
 
 	/* make sure we got here for instruction we have a kprobe on */
@@ -449,7 +452,7 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
 		 * zero, try to fix up.
 		 */
 		if ((entry = search_exception_tables(regs->nip)) != NULL) {
-			regs->nip = entry->fixup;
+			regs->nip = extable_fixup(entry);
 			return 1;
 		}
 
@@ -470,25 +473,7 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
 int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
 				       unsigned long val, void *data)
 {
-	struct die_args *args = (struct die_args *)data;
-	int ret = NOTIFY_DONE;
-
-	if (args->regs && user_mode(args->regs))
-		return ret;
-
-	switch (val) {
-	case DIE_BPT:
-		if (kprobe_handler(args->regs))
-			ret = NOTIFY_STOP;
-		break;
-	case DIE_SSTEP:
-		if (post_kprobe_handler(args->regs))
-			ret = NOTIFY_STOP;
-		break;
-	default:
-		break;
-	}
-	return ret;
+	return NOTIFY_DONE;
 }
 
 unsigned long arch_deref_entry_point(void *entry)
diff --git a/arch/powerpc/kernel/machine_kexec_64.c b/arch/powerpc/kernel/machine_kexec_64.c
index a205fa3d9bf3..5c12e21d0d1a 100644
--- a/arch/powerpc/kernel/machine_kexec_64.c
+++ b/arch/powerpc/kernel/machine_kexec_64.c
@@ -310,7 +310,7 @@ void default_machine_kexec(struct kimage *image)
 	if (!kdump_in_progress())
 		kexec_prepare_cpus();
 
-	pr_debug("kexec: Starting switchover sequence.\n");
+	printk("kexec: Starting switchover sequence.\n");
 
 	/* switch to a staticly allocated stack.  Based on irq stack code.
 	 * We setup preempt_count to avoid using VMX in memcpy.
diff --git a/arch/powerpc/kernel/machine_kexec_file_64.c b/arch/powerpc/kernel/machine_kexec_file_64.c
new file mode 100644
index 000000000000..7abc8a75ee48
--- /dev/null
+++ b/arch/powerpc/kernel/machine_kexec_file_64.c
@@ -0,0 +1,338 @@
+/*
+ * ppc64 code to implement the kexec_file_load syscall
+ *
+ * Copyright (C) 2004  Adam Litke (agl@us.ibm.com)
+ * Copyright (C) 2004  IBM Corp.
+ * Copyright (C) 2004,2005  Milton D Miller II, IBM Corporation
+ * Copyright (C) 2005  R Sharada (sharada@in.ibm.com)
+ * Copyright (C) 2006  Mohan Kumar M (mohan@in.ibm.com)
+ * Copyright (C) 2016  IBM Corporation
+ *
+ * Based on kexec-tools' kexec-elf-ppc64.c, fs2dt.c.
+ * Heavily modified for the kernel by
+ * Thiago Jung Bauermann <bauerman@linux.vnet.ibm.com>.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation (version 2 of the License).
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/slab.h>
+#include <linux/kexec.h>
+#include <linux/memblock.h>
+#include <linux/of_fdt.h>
+#include <linux/libfdt.h>
+
+#define SLAVE_CODE_SIZE		256
+
+static struct kexec_file_ops *kexec_file_loaders[] = {
+	&kexec_elf64_ops,
+};
+
+int arch_kexec_kernel_image_probe(struct kimage *image, void *buf,
+				  unsigned long buf_len)
+{
+	int i, ret = -ENOEXEC;
+	struct kexec_file_ops *fops;
+
+	/* We don't support crash kernels yet. */
+	if (image->type == KEXEC_TYPE_CRASH)
+		return -ENOTSUPP;
+
+	for (i = 0; i < ARRAY_SIZE(kexec_file_loaders); i++) {
+		fops = kexec_file_loaders[i];
+		if (!fops || !fops->probe)
+			continue;
+
+		ret = fops->probe(buf, buf_len);
+		if (!ret) {
+			image->fops = fops;
+			return ret;
+		}
+	}
+
+	return ret;
+}
+
+void *arch_kexec_kernel_image_load(struct kimage *image)
+{
+	if (!image->fops || !image->fops->load)
+		return ERR_PTR(-ENOEXEC);
+
+	return image->fops->load(image, image->kernel_buf,
+				 image->kernel_buf_len, image->initrd_buf,
+				 image->initrd_buf_len, image->cmdline_buf,
+				 image->cmdline_buf_len);
+}
+
+int arch_kimage_file_post_load_cleanup(struct kimage *image)
+{
+	if (!image->fops || !image->fops->cleanup)
+		return 0;
+
+	return image->fops->cleanup(image->image_loader_data);
+}
+
+/**
+ * arch_kexec_walk_mem - call func(data) for each unreserved memory block
+ * @kbuf:	Context info for the search. Also passed to @func.
+ * @func:	Function to call for each memory block.
+ *
+ * This function is used by kexec_add_buffer and kexec_locate_mem_hole
+ * to find unreserved memory to load kexec segments into.
+ *
+ * Return: The memory walk will stop when func returns a non-zero value
+ * and that value will be returned. If all free regions are visited without
+ * func returning non-zero, then zero will be returned.
+ */
+int arch_kexec_walk_mem(struct kexec_buf *kbuf, int (*func)(u64, u64, void *))
+{
+	int ret = 0;
+	u64 i;
+	phys_addr_t mstart, mend;
+
+	if (kbuf->top_down) {
+		for_each_free_mem_range_reverse(i, NUMA_NO_NODE, 0,
+						&mstart, &mend, NULL) {
+			/*
+			 * In memblock, end points to the first byte after the
+			 * range while in kexec, end points to the last byte
+			 * in the range.
+			 */
+			ret = func(mstart, mend - 1, kbuf);
+			if (ret)
+				break;
+		}
+	} else {
+		for_each_free_mem_range(i, NUMA_NO_NODE, 0, &mstart, &mend,
+					NULL) {
+			/*
+			 * In memblock, end points to the first byte after the
+			 * range while in kexec, end points to the last byte
+			 * in the range.
+			 */
+			ret = func(mstart, mend - 1, kbuf);
+			if (ret)
+				break;
+		}
+	}
+
+	return ret;
+}
+
+/**
+ * setup_purgatory - initialize the purgatory's global variables
+ * @image:		kexec image.
+ * @slave_code:		Slave code for the purgatory.
+ * @fdt:		Flattened device tree for the next kernel.
+ * @kernel_load_addr:	Address where the kernel is loaded.
+ * @fdt_load_addr:	Address where the flattened device tree is loaded.
+ *
+ * Return: 0 on success, or negative errno on error.
+ */
+int setup_purgatory(struct kimage *image, const void *slave_code,
+		    const void *fdt, unsigned long kernel_load_addr,
+		    unsigned long fdt_load_addr)
+{
+	unsigned int *slave_code_buf, master_entry;
+	int ret;
+
+	slave_code_buf = kmalloc(SLAVE_CODE_SIZE, GFP_KERNEL);
+	if (!slave_code_buf)
+		return -ENOMEM;
+
+	/* Get the slave code from the new kernel and put it in purgatory. */
+	ret = kexec_purgatory_get_set_symbol(image, "purgatory_start",
+					     slave_code_buf, SLAVE_CODE_SIZE,
+					     true);
+	if (ret) {
+		kfree(slave_code_buf);
+		return ret;
+	}
+
+	master_entry = slave_code_buf[0];
+	memcpy(slave_code_buf, slave_code, SLAVE_CODE_SIZE);
+	slave_code_buf[0] = master_entry;
+	ret = kexec_purgatory_get_set_symbol(image, "purgatory_start",
+					     slave_code_buf, SLAVE_CODE_SIZE,
+					     false);
+	kfree(slave_code_buf);
+
+	ret = kexec_purgatory_get_set_symbol(image, "kernel", &kernel_load_addr,
+					     sizeof(kernel_load_addr), false);
+	if (ret)
+		return ret;
+	ret = kexec_purgatory_get_set_symbol(image, "dt_offset", &fdt_load_addr,
+					     sizeof(fdt_load_addr), false);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+/**
+ * delete_fdt_mem_rsv - delete memory reservation with given address and size
+ *
+ * Return: 0 on success, or negative errno on error.
+ */
+static int delete_fdt_mem_rsv(void *fdt, unsigned long start, unsigned long size)
+{
+	int i, ret, num_rsvs = fdt_num_mem_rsv(fdt);
+
+	for (i = 0; i < num_rsvs; i++) {
+		uint64_t rsv_start, rsv_size;
+
+		ret = fdt_get_mem_rsv(fdt, i, &rsv_start, &rsv_size);
+		if (ret) {
+			pr_err("Malformed device tree.\n");
+			return -EINVAL;
+		}
+
+		if (rsv_start == start && rsv_size == size) {
+			ret = fdt_del_mem_rsv(fdt, i);
+			if (ret) {
+				pr_err("Error deleting device tree reservation.\n");
+				return -EINVAL;
+			}
+
+			return 0;
+		}
+	}
+
+	return -ENOENT;
+}
+
+/*
+ * setup_new_fdt - modify /chosen and memory reservation for the next kernel
+ * @fdt:		Flattened device tree for the next kernel.
+ * @initrd_load_addr:	Address where the next initrd will be loaded.
+ * @initrd_len:		Size of the next initrd, or 0 if there will be none.
+ * @cmdline:		Command line for the next kernel, or NULL if there will
+ *			be none.
+ *
+ * Return: 0 on success, or negative errno on error.
+ */
+int setup_new_fdt(void *fdt, unsigned long initrd_load_addr,
+		  unsigned long initrd_len, const char *cmdline)
+{
+	int ret, chosen_node;
+	const void *prop;
+
+	/* Remove memory reservation for the current device tree. */
+	ret = delete_fdt_mem_rsv(fdt, __pa(initial_boot_params),
+				 fdt_totalsize(initial_boot_params));
+	if (ret == 0)
+		pr_debug("Removed old device tree reservation.\n");
+	else if (ret != -ENOENT)
+		return ret;
+
+	chosen_node = fdt_path_offset(fdt, "/chosen");
+	if (chosen_node == -FDT_ERR_NOTFOUND) {
+		chosen_node = fdt_add_subnode(fdt, fdt_path_offset(fdt, "/"),
+					      "chosen");
+		if (chosen_node < 0) {
+			pr_err("Error creating /chosen.\n");
+			return -EINVAL;
+		}
+	} else if (chosen_node < 0) {
+		pr_err("Malformed device tree: error reading /chosen.\n");
+		return -EINVAL;
+	}
+
+	/* Did we boot using an initrd? */
+	prop = fdt_getprop(fdt, chosen_node, "linux,initrd-start", NULL);
+	if (prop) {
+		uint64_t tmp_start, tmp_end, tmp_size;
+
+		tmp_start = fdt64_to_cpu(*((const fdt64_t *) prop));
+
+		prop = fdt_getprop(fdt, chosen_node, "linux,initrd-end", NULL);
+		if (!prop) {
+			pr_err("Malformed device tree.\n");
+			return -EINVAL;
+		}
+		tmp_end = fdt64_to_cpu(*((const fdt64_t *) prop));
+
+		/*
+		 * kexec reserves exact initrd size, while firmware may
+		 * reserve a multiple of PAGE_SIZE, so check for both.
+		 */
+		tmp_size = tmp_end - tmp_start;
+		ret = delete_fdt_mem_rsv(fdt, tmp_start, tmp_size);
+		if (ret == -ENOENT)
+			ret = delete_fdt_mem_rsv(fdt, tmp_start,
+						 round_up(tmp_size, PAGE_SIZE));
+		if (ret == 0)
+			pr_debug("Removed old initrd reservation.\n");
+		else if (ret != -ENOENT)
+			return ret;
+
+		/* If there's no new initrd, delete the old initrd's info. */
+		if (initrd_len == 0) {
+			ret = fdt_delprop(fdt, chosen_node,
+					  "linux,initrd-start");
+			if (ret) {
+				pr_err("Error deleting linux,initrd-start.\n");
+				return -EINVAL;
+			}
+
+			ret = fdt_delprop(fdt, chosen_node, "linux,initrd-end");
+			if (ret) {
+				pr_err("Error deleting linux,initrd-end.\n");
+				return -EINVAL;
+			}
+		}
+	}
+
+	if (initrd_len) {
+		ret = fdt_setprop_u64(fdt, chosen_node,
+				      "linux,initrd-start",
+				      initrd_load_addr);
+		if (ret < 0) {
+			pr_err("Error setting up the new device tree.\n");
+			return -EINVAL;
+		}
+
+		/* initrd-end is the first address after the initrd image. */
+		ret = fdt_setprop_u64(fdt, chosen_node, "linux,initrd-end",
+				      initrd_load_addr + initrd_len);
+		if (ret < 0) {
+			pr_err("Error setting up the new device tree.\n");
+			return -EINVAL;
+		}
+
+		ret = fdt_add_mem_rsv(fdt, initrd_load_addr, initrd_len);
+		if (ret) {
+			pr_err("Error reserving initrd memory: %s\n",
+			       fdt_strerror(ret));
+			return -EINVAL;
+		}
+	}
+
+	if (cmdline != NULL) {
+		ret = fdt_setprop_string(fdt, chosen_node, "bootargs", cmdline);
+		if (ret < 0) {
+			pr_err("Error setting up the new device tree.\n");
+			return -EINVAL;
+		}
+	} else {
+		ret = fdt_delprop(fdt, chosen_node, "bootargs");
+		if (ret && ret != -FDT_ERR_NOTFOUND) {
+			pr_err("Error deleting bootargs.\n");
+			return -EINVAL;
+		}
+	}
+
+	ret = fdt_setprop(fdt, chosen_node, "linux,booted-from-kexec", NULL, 0);
+	if (ret) {
+		pr_err("Error setting up the new device tree.\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/arch/powerpc/kernel/mce.c b/arch/powerpc/kernel/mce.c
index 5e7ece0fda9f..c6923ff45131 100644
--- a/arch/powerpc/kernel/mce.c
+++ b/arch/powerpc/kernel/mce.c
@@ -72,7 +72,6 @@ void save_mce_event(struct pt_regs *regs, long handled,
 		    struct mce_error_info *mce_err,
 		    uint64_t nip, uint64_t addr)
 {
-	uint64_t srr1;
 	int index = __this_cpu_inc_return(mce_nest_count) - 1;
 	struct machine_check_event *mce = this_cpu_ptr(&mce_event[index]);
 
@@ -99,8 +98,6 @@ void save_mce_event(struct pt_regs *regs, long handled,
 		mce->disposition = MCE_DISPOSITION_NOT_RECOVERED;
 	mce->severity = MCE_SEV_ERROR_SYNC;
 
-	srr1 = regs->msr;
-
 	/*
 	 * Populate the mce error_type and type-specific error_type.
 	 */
diff --git a/arch/powerpc/kernel/misc_32.S b/arch/powerpc/kernel/misc_32.S
index 93cf7a5846a6..1863324c6a3c 100644
--- a/arch/powerpc/kernel/misc_32.S
+++ b/arch/powerpc/kernel/misc_32.S
@@ -614,7 +614,7 @@ _GLOBAL(start_secondary_resume)
 _GLOBAL(__main)
 	blr
 
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 	/*
 	 * Must be relocatable PIC code callable as a C function.
 	 */
diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S
index 4f178671f230..32be2a844947 100644
--- a/arch/powerpc/kernel/misc_64.S
+++ b/arch/powerpc/kernel/misc_64.S
@@ -478,7 +478,7 @@ _GLOBAL(kexec_wait)
 	addi	r5,r5,kexec_flag-1b
 
 99:	HMT_LOW
-#ifdef CONFIG_KEXEC		/* use no memory without kexec */
+#ifdef CONFIG_KEXEC_CORE	/* use no memory without kexec */
 	lwz	r4,0(r5)
 	cmpwi	0,r4,0
 	beq	99b
@@ -503,7 +503,7 @@ kexec_flag:
 	.long	0
 
 
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 #ifdef CONFIG_PPC_BOOK3E
 /*
  * BOOK3E has no real MMU mode, so we have to setup the initial TLB
@@ -716,4 +716,4 @@ _GLOBAL(kexec_sequence)
 	mtlr	4
 	li	r5,0
 	blr	/* image->start(physid, image->start, 0); */
-#endif /* CONFIG_KEXEC */
+#endif /* CONFIG_KEXEC_CORE */
diff --git a/arch/powerpc/kernel/module_64.c b/arch/powerpc/kernel/module_64.c
index 183368e008cf..bb1807184bad 100644
--- a/arch/powerpc/kernel/module_64.c
+++ b/arch/powerpc/kernel/module_64.c
@@ -652,6 +652,11 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
 			*location = value - (unsigned long)location;
 			break;
 
+		case R_PPC64_REL32:
+			/* 32 bits relative (used by relative exception tables) */
+			*(u32 *)location = value - (unsigned long)location;
+			break;
+
 		case R_PPC64_TOCSAVE:
 			/*
 			 * Marker reloc indicates we don't have to save r2.
diff --git a/arch/powerpc/kernel/of_platform.c b/arch/powerpc/kernel/of_platform.c
index b60a67d92ebd..34aeac54f120 100644
--- a/arch/powerpc/kernel/of_platform.c
+++ b/arch/powerpc/kernel/of_platform.c
@@ -114,11 +114,6 @@ static struct platform_driver of_pci_phb_driver = {
 	},
 };
 
-static __init int of_pci_phb_init(void)
-{
-	return platform_driver_register(&of_pci_phb_driver);
-}
-
-device_initcall(of_pci_phb_init);
+builtin_platform_driver(of_pci_phb_driver);
 
 #endif /* CONFIG_PPC_OF_PLATFORM_PCI */
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 49a680d5ae37..04885cec24df 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -64,6 +64,12 @@
 #include <linux/kprobes.h>
 #include <linux/kdebug.h>
 
+#ifdef CONFIG_CC_STACKPROTECTOR
+#include <linux/stackprotector.h>
+unsigned long __stack_chk_guard __read_mostly;
+EXPORT_SYMBOL(__stack_chk_guard);
+#endif
+
 /* Transactional Memory debug */
 #ifdef TM_DEBUG_SW
 #define TM_DEBUG(x...) printk(KERN_INFO x)
@@ -1051,14 +1057,6 @@ static inline void save_sprs(struct thread_struct *t)
 		 */
 		t->tar = mfspr(SPRN_TAR);
 	}
-
-	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
-		/* Conditionally save Load Monitor registers, if enabled */
-		if (t->fscr & FSCR_LM) {
-			t->lmrr = mfspr(SPRN_LMRR);
-			t->lmser = mfspr(SPRN_LMSER);
-		}
-	}
 #endif
 }
 
@@ -1094,16 +1092,6 @@ static inline void restore_sprs(struct thread_struct *old_thread,
 		if (old_thread->tar != new_thread->tar)
 			mtspr(SPRN_TAR, new_thread->tar);
 	}
-
-	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
-		/* Conditionally restore Load Monitor registers, if enabled */
-		if (new_thread->fscr & FSCR_LM) {
-			if (old_thread->lmrr != new_thread->lmrr)
-				mtspr(SPRN_LMRR, new_thread->lmrr);
-			if (old_thread->lmser != new_thread->lmser)
-				mtspr(SPRN_LMSER, new_thread->lmser);
-		}
-	}
 #endif
 }
 
diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c
index b0245bed6f54..f5d399e46193 100644
--- a/arch/powerpc/kernel/prom.c
+++ b/arch/powerpc/kernel/prom.c
@@ -156,21 +156,22 @@ static struct ibm_pa_feature {
 	unsigned char	pabit;		/* bit number (big-endian) */
 	unsigned char	invert;		/* if 1, pa bit set => clear feature */
 } ibm_pa_features[] __initdata = {
-	{0, 0, PPC_FEATURE_HAS_MMU, 0,		0, 0, 0},
-	{0, 0, PPC_FEATURE_HAS_FPU, 0,		0, 1, 0},
-	{CPU_FTR_CTRL, 0, 0, 0,			0, 3, 0},
-	{CPU_FTR_NOEXECUTE, 0, 0, 0,		0, 6, 0},
-	{CPU_FTR_NODSISRALIGN, 0, 0, 0,		1, 1, 1},
-	{0, MMU_FTR_CI_LARGE_PAGE, 0, 0,		1, 2, 0},
-	{CPU_FTR_REAL_LE, 0, PPC_FEATURE_TRUE_LE, 0, 5, 0, 0},
+	{ .pabyte = 0,  .pabit = 0, .cpu_user_ftrs = PPC_FEATURE_HAS_MMU },
+	{ .pabyte = 0,  .pabit = 1, .cpu_user_ftrs = PPC_FEATURE_HAS_FPU },
+	{ .pabyte = 0,  .pabit = 3, .cpu_features  = CPU_FTR_CTRL },
+	{ .pabyte = 0,  .pabit = 6, .cpu_features  = CPU_FTR_NOEXECUTE },
+	{ .pabyte = 1,  .pabit = 2, .mmu_features  = MMU_FTR_CI_LARGE_PAGE },
+	{ .pabyte = 40, .pabit = 0, .mmu_features  = MMU_FTR_TYPE_RADIX },
+	{ .pabyte = 1,  .pabit = 1, .invert = 1, .cpu_features = CPU_FTR_NODSISRALIGN },
+	{ .pabyte = 5,  .pabit = 0, .cpu_features  = CPU_FTR_REAL_LE,
+				    .cpu_user_ftrs = PPC_FEATURE_TRUE_LE },
 	/*
 	 * If the kernel doesn't support TM (ie CONFIG_PPC_TRANSACTIONAL_MEM=n),
 	 * we don't want to turn on TM here, so we use the *_COMP versions
 	 * which are 0 if the kernel doesn't support TM.
 	 */
-	{CPU_FTR_TM_COMP, 0, 0,
-	 PPC_FEATURE2_HTM_COMP|PPC_FEATURE2_HTM_NOSC_COMP, 22, 0, 0},
-	{0, MMU_FTR_TYPE_RADIX, 0, 0,		40, 0, 0},
+	{ .pabyte = 22, .pabit = 0, .cpu_features = CPU_FTR_TM_COMP,
+	  .cpu_user_ftrs2 = PPC_FEATURE2_HTM_COMP | PPC_FEATURE2_HTM_NOSC_COMP },
 };
 
 static void __init scan_features(unsigned long node, const unsigned char *ftrs,
@@ -427,7 +428,7 @@ static int __init early_init_dt_scan_chosen_ppc(unsigned long node,
 		tce_alloc_end = *lprop;
 #endif
 
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 	lprop = of_get_flat_dt_prop(node, "linux,crashkernel-base", NULL);
 	if (lprop)
 		crashk_res.start = *lprop;
diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c
index 88ac964f4858..ec47a939cbdd 100644
--- a/arch/powerpc/kernel/prom_init.c
+++ b/arch/powerpc/kernel/prom_init.c
@@ -461,14 +461,14 @@ static int __init prom_next_node(phandle *nodep)
 	}
 }
 
-static int inline prom_getprop(phandle node, const char *pname,
+static inline int prom_getprop(phandle node, const char *pname,
 			       void *value, size_t valuelen)
 {
 	return call_prom("getprop", 4, 1, node, ADDR(pname),
 			 (u32)(unsigned long) value, (u32) valuelen);
 }
 
-static int inline prom_getproplen(phandle node, const char *pname)
+static inline int prom_getproplen(phandle node, const char *pname)
 {
 	return call_prom("getproplen", 2, 1, node, ADDR(pname));
 }
@@ -635,13 +635,7 @@ static void __init early_cmdline_parse(void)
  *
  * See prom.h for the definition of the bits specified in the
  * architecture vector.
- *
- * Because the description vector contains a mix of byte and word
- * values, we declare it as an unsigned char array, and use this
- * macro to put word values in.
  */
-#define W(x)	((x) >> 24) & 0xff, ((x) >> 16) & 0xff, \
-		((x) >> 8) & 0xff, (x) & 0xff
 
 /* Firmware expects the value to be n - 1, where n is the # of vectors */
 #define NUM_VECTORS(n)		((n) - 1)
@@ -652,92 +646,205 @@ static void __init early_cmdline_parse(void)
  */
 #define VECTOR_LENGTH(n)	(1 + (n) - 2)
 
-unsigned char ibm_architecture_vec[] = {
-	W(0xfffe0000), W(0x003a0000),	/* POWER5/POWER5+ */
-	W(0xffff0000), W(0x003e0000),	/* POWER6 */
-	W(0xffff0000), W(0x003f0000),	/* POWER7 */
-	W(0xffff0000), W(0x004b0000),	/* POWER8E */
-	W(0xffff0000), W(0x004c0000),   /* POWER8NVL */
-	W(0xffff0000), W(0x004d0000),	/* POWER8 */
-	W(0xffffffff), W(0x0f000004),	/* all 2.07-compliant */
-	W(0xffffffff), W(0x0f000003),	/* all 2.06-compliant */
-	W(0xffffffff), W(0x0f000002),	/* all 2.05-compliant */
-	W(0xfffffffe), W(0x0f000001),	/* all 2.04-compliant and earlier */
-	NUM_VECTORS(6),			/* 6 option vectors */
-
-	/* option vector 1: processor architectures supported */
-	VECTOR_LENGTH(2),		/* length */
-	0,				/* don't ignore, don't halt */
-	OV1_PPC_2_00 | OV1_PPC_2_01 | OV1_PPC_2_02 | OV1_PPC_2_03 |
-	OV1_PPC_2_04 | OV1_PPC_2_05 | OV1_PPC_2_06 | OV1_PPC_2_07,
+struct option_vector1 {
+	u8 byte1;
+	u8 arch_versions;
+} __packed;
+
+struct option_vector2 {
+	u8 byte1;
+	__be16 reserved;
+	__be32 real_base;
+	__be32 real_size;
+	__be32 virt_base;
+	__be32 virt_size;
+	__be32 load_base;
+	__be32 min_rma;
+	__be32 min_load;
+	u8 min_rma_percent;
+	u8 max_pft_size;
+} __packed;
+
+struct option_vector3 {
+	u8 byte1;
+	u8 byte2;
+} __packed;
+
+struct option_vector4 {
+	u8 byte1;
+	u8 min_vp_cap;
+} __packed;
+
+struct option_vector5 {
+	u8 byte1;
+	u8 byte2;
+	u8 byte3;
+	u8 cmo;
+	u8 associativity;
+	u8 bin_opts;
+	u8 micro_checkpoint;
+	u8 reserved0;
+	__be32 max_cpus;
+	__be16 papr_level;
+	__be16 reserved1;
+	u8 platform_facilities;
+	u8 reserved2;
+	__be16 reserved3;
+	u8 subprocessors;
+} __packed;
+
+struct option_vector6 {
+	u8 reserved;
+	u8 secondary_pteg;
+	u8 os_name;
+} __packed;
+
+struct ibm_arch_vec {
+	struct { u32 mask, val; } pvrs[10];
+
+	u8 num_vectors;
+
+	u8 vec1_len;
+	struct option_vector1 vec1;
+
+	u8 vec2_len;
+	struct option_vector2 vec2;
+
+	u8 vec3_len;
+	struct option_vector3 vec3;
+
+	u8 vec4_len;
+	struct option_vector4 vec4;
+
+	u8 vec5_len;
+	struct option_vector5 vec5;
+
+	u8 vec6_len;
+	struct option_vector6 vec6;
+} __packed;
+
+struct ibm_arch_vec __cacheline_aligned ibm_architecture_vec = {
+	.pvrs = {
+		{
+			.mask = cpu_to_be32(0xfffe0000), /* POWER5/POWER5+ */
+			.val  = cpu_to_be32(0x003a0000),
+		},
+		{
+			.mask = cpu_to_be32(0xffff0000), /* POWER6 */
+			.val  = cpu_to_be32(0x003e0000),
+		},
+		{
+			.mask = cpu_to_be32(0xffff0000), /* POWER7 */
+			.val  = cpu_to_be32(0x003f0000),
+		},
+		{
+			.mask = cpu_to_be32(0xffff0000), /* POWER8E */
+			.val  = cpu_to_be32(0x004b0000),
+		},
+		{
+			.mask = cpu_to_be32(0xffff0000), /* POWER8NVL */
+			.val  = cpu_to_be32(0x004c0000),
+		},
+		{
+			.mask = cpu_to_be32(0xffff0000), /* POWER8 */
+			.val  = cpu_to_be32(0x004d0000),
+		},
+		{
+			.mask = cpu_to_be32(0xffffffff), /* all 2.07-compliant */
+			.val  = cpu_to_be32(0x0f000004),
+		},
+		{
+			.mask = cpu_to_be32(0xffffffff), /* all 2.06-compliant */
+			.val  = cpu_to_be32(0x0f000003),
+		},
+		{
+			.mask = cpu_to_be32(0xffffffff), /* all 2.05-compliant */
+			.val  = cpu_to_be32(0x0f000002),
+		},
+		{
+			.mask = cpu_to_be32(0xfffffffe), /* all 2.04-compliant and earlier */
+			.val  = cpu_to_be32(0x0f000001),
+		},
+	},
+
+	.num_vectors = NUM_VECTORS(6),
 
+	.vec1_len = VECTOR_LENGTH(sizeof(struct option_vector1)),
+	.vec1 = {
+		.byte1 = 0,
+		.arch_versions = OV1_PPC_2_00 | OV1_PPC_2_01 | OV1_PPC_2_02 | OV1_PPC_2_03 |
+				 OV1_PPC_2_04 | OV1_PPC_2_05 | OV1_PPC_2_06 | OV1_PPC_2_07,
+	},
+
+	.vec2_len = VECTOR_LENGTH(sizeof(struct option_vector2)),
 	/* option vector 2: Open Firmware options supported */
-	VECTOR_LENGTH(33),		/* length */
-	OV2_REAL_MODE,
-	0, 0,
-	W(0xffffffff),			/* real_base */
-	W(0xffffffff),			/* real_size */
-	W(0xffffffff),			/* virt_base */
-	W(0xffffffff),			/* virt_size */
-	W(0xffffffff),			/* load_base */
-	W(256),				/* 256MB min RMA */
-	W(0xffffffff),			/* full client load */
-	0,				/* min RMA percentage of total RAM */
-	48,				/* max log_2(hash table size) */
+	.vec2 = {
+		.byte1 = OV2_REAL_MODE,
+		.reserved = 0,
+		.real_base = cpu_to_be32(0xffffffff),
+		.real_size = cpu_to_be32(0xffffffff),
+		.virt_base = cpu_to_be32(0xffffffff),
+		.virt_size = cpu_to_be32(0xffffffff),
+		.load_base = cpu_to_be32(0xffffffff),
+		.min_rma = cpu_to_be32(256),		/* 256MB min RMA */
+		.min_load = cpu_to_be32(0xffffffff),	/* full client load */
+		.min_rma_percent = 0,	/* min RMA percentage of total RAM */
+		.max_pft_size = 48,	/* max log_2(hash table size) */
+	},
 
+	.vec3_len = VECTOR_LENGTH(sizeof(struct option_vector3)),
 	/* option vector 3: processor options supported */
-	VECTOR_LENGTH(2),		/* length */
-	0,				/* don't ignore, don't halt */
-	OV3_FP | OV3_VMX | OV3_DFP,
+	.vec3 = {
+		.byte1 = 0,			/* don't ignore, don't halt */
+		.byte2 = OV3_FP | OV3_VMX | OV3_DFP,
+	},
 
+	.vec4_len = VECTOR_LENGTH(sizeof(struct option_vector4)),
 	/* option vector 4: IBM PAPR implementation */
-	VECTOR_LENGTH(2),		/* length */
-	0,				/* don't halt */
-	OV4_MIN_ENT_CAP,		/* minimum VP entitled capacity */
+	.vec4 = {
+		.byte1 = 0,			/* don't halt */
+		.min_vp_cap = OV4_MIN_ENT_CAP,	/* minimum VP entitled capacity */
+	},
 
+	.vec5_len = VECTOR_LENGTH(sizeof(struct option_vector5)),
 	/* option vector 5: PAPR/OF options */
-	VECTOR_LENGTH(21),		/* length */
-	0,				/* don't ignore, don't halt */
-	OV5_FEAT(OV5_LPAR) | OV5_FEAT(OV5_SPLPAR) | OV5_FEAT(OV5_LARGE_PAGES) |
-	OV5_FEAT(OV5_DRCONF_MEMORY) | OV5_FEAT(OV5_DONATE_DEDICATE_CPU) |
+	.vec5 = {
+		.byte1 = 0,				/* don't ignore, don't halt */
+		.byte2 = OV5_FEAT(OV5_LPAR) | OV5_FEAT(OV5_SPLPAR) | OV5_FEAT(OV5_LARGE_PAGES) |
+		OV5_FEAT(OV5_DRCONF_MEMORY) | OV5_FEAT(OV5_DONATE_DEDICATE_CPU) |
 #ifdef CONFIG_PCI_MSI
-	/* PCIe/MSI support.  Without MSI full PCIe is not supported */
-	OV5_FEAT(OV5_MSI),
+		/* PCIe/MSI support.  Without MSI full PCIe is not supported */
+		OV5_FEAT(OV5_MSI),
 #else
-	0,
+		0,
 #endif
-	0,
+		.byte3 = 0,
+		.cmo =
 #ifdef CONFIG_PPC_SMLPAR
-	OV5_FEAT(OV5_CMO) | OV5_FEAT(OV5_XCMO),
+		OV5_FEAT(OV5_CMO) | OV5_FEAT(OV5_XCMO),
 #else
-	0,
+		0,
 #endif
-	OV5_FEAT(OV5_TYPE1_AFFINITY) | OV5_FEAT(OV5_PRRN),
-	0,
-	0,
-	0,
-	/* WARNING: The offset of the "number of cores" field below
-	 * must match by the macro below. Update the definition if
-	 * the structure layout changes.
-	 */
-#define IBM_ARCH_VEC_NRCORES_OFFSET	133
-	W(NR_CPUS),			/* number of cores supported */
-	0,
-	0,
-	0,
-	0,
-	OV5_FEAT(OV5_PFO_HW_RNG) | OV5_FEAT(OV5_PFO_HW_ENCR) |
-	OV5_FEAT(OV5_PFO_HW_842),				/* Byte 17 */
-	0,							/* Byte 18 */
-	0,							/* Byte 19 */
-	0,							/* Byte 20 */
-	OV5_FEAT(OV5_SUB_PROCESSORS),				/* Byte 21 */
+		.associativity = OV5_FEAT(OV5_TYPE1_AFFINITY) | OV5_FEAT(OV5_PRRN),
+		.bin_opts = 0,
+		.micro_checkpoint = 0,
+		.reserved0 = 0,
+		.max_cpus = cpu_to_be32(NR_CPUS),	/* number of cores supported */
+		.papr_level = 0,
+		.reserved1 = 0,
+		.platform_facilities = OV5_FEAT(OV5_PFO_HW_RNG) | OV5_FEAT(OV5_PFO_HW_ENCR) | OV5_FEAT(OV5_PFO_HW_842),
+		.reserved2 = 0,
+		.reserved3 = 0,
+		.subprocessors = 1,
+	},
 
 	/* option vector 6: IBM PAPR hints */
-	VECTOR_LENGTH(3),		/* length */
-	0,
-	0,
-	OV6_LINUX,
+	.vec6_len = VECTOR_LENGTH(sizeof(struct option_vector6)),
+	.vec6 = {
+		.reserved = 0,
+		.secondary_pteg = 0,
+		.os_name = OV6_LINUX,
+	},
 };
 
 /* Old method - ELF header with PT_NOTE sections only works on BE */
@@ -873,7 +980,6 @@ static void __init prom_send_capabilities(void)
 	ihandle root;
 	prom_arg_t ret;
 	u32 cores;
-	unsigned char *ptcores;
 
 	root = call_prom("open", 1, 1, ADDR("/"));
 	if (root != 0) {
@@ -884,37 +990,18 @@ static void __init prom_send_capabilities(void)
 		 * divide NR_CPUS.
 		 */
 
-		/* The core value may start at an odd address. If such a word
-		 * access is made at a cache line boundary, this leads to an
-		 * exception which may not be handled at this time.
-		 * Forcing a per byte access to avoid exception.
-		 */
-		ptcores = &ibm_architecture_vec[IBM_ARCH_VEC_NRCORES_OFFSET];
-		cores = 0;
-		cores |= ptcores[0] << 24;
-		cores |= ptcores[1] << 16;
-		cores |= ptcores[2] << 8;
-		cores |= ptcores[3];
-		if (cores != NR_CPUS) {
-			prom_printf("WARNING ! "
-				    "ibm_architecture_vec structure inconsistent: %lu!\n",
-				    cores);
-		} else {
-			cores = DIV_ROUND_UP(NR_CPUS, prom_count_smt_threads());
-			prom_printf("Max number of cores passed to firmware: %lu (NR_CPUS = %lu)\n",
-				    cores, NR_CPUS);
-			ptcores[0] = (cores >> 24) & 0xff;
-			ptcores[1] = (cores >> 16) & 0xff;
-			ptcores[2] = (cores >> 8) & 0xff;
-			ptcores[3] = cores & 0xff;
-		}
+		cores = DIV_ROUND_UP(NR_CPUS, prom_count_smt_threads());
+		prom_printf("Max number of cores passed to firmware: %lu (NR_CPUS = %lu)\n",
+			    cores, NR_CPUS);
+
+		ibm_architecture_vec.vec5.max_cpus = cpu_to_be32(cores);
 
 		/* try calling the ibm,client-architecture-support method */
 		prom_printf("Calling ibm,client-architecture-support...");
 		if (call_prom_ret("call-method", 3, 2, &ret,
 				  ADDR("ibm,client-architecture-support"),
 				  root,
-				  ADDR(ibm_architecture_vec)) == 0) {
+				  ADDR(&ibm_architecture_vec)) == 0) {
 			/* the call exists... */
 			if (ret)
 				prom_printf("\nWARNING: ibm,client-architecture"
diff --git a/arch/powerpc/kernel/ptrace32.c b/arch/powerpc/kernel/ptrace32.c
index 010b7b310237..1e887f3a61a6 100644
--- a/arch/powerpc/kernel/ptrace32.c
+++ b/arch/powerpc/kernel/ptrace32.c
@@ -73,7 +73,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
 		if (get_user(addrOthers, (u32 __user * __user *)addr) != 0)
 			break;
 
-		copied = access_process_vm(child, (u64)addrOthers, &tmp,
+		copied = ptrace_access_vm(child, (u64)addrOthers, &tmp,
 				sizeof(tmp), FOLL_FORCE);
 		if (copied != sizeof(tmp))
 			break;
@@ -178,7 +178,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
 		if (get_user(addrOthers, (u32 __user * __user *)addr) != 0)
 			break;
 		ret = 0;
-		if (access_process_vm(child, (u64)addrOthers, &tmp,
+		if (ptrace_access_vm(child, (u64)addrOthers, &tmp,
 					sizeof(tmp),
 					FOLL_FORCE | FOLL_WRITE) == sizeof(tmp))
 			break;
diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c
index 270ee30abdcf..f516ac508ae3 100644
--- a/arch/powerpc/kernel/setup-common.c
+++ b/arch/powerpc/kernel/setup-common.c
@@ -915,7 +915,7 @@ void __init setup_arch(char **cmdline_p)
 	init_mm.context.pte_frag = NULL;
 #endif
 #ifdef CONFIG_SPAPR_TCE_IOMMU
-	mm_iommu_init(&init_mm.context);
+	mm_iommu_init(&init_mm);
 #endif
 	irqstack_early_init();
 	exc_lvl_early_init();
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index 8d586cff8a41..6824157e4d2e 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -354,7 +354,7 @@ void early_setup_secondary(void)
 
 #endif /* CONFIG_SMP */
 
-#if defined(CONFIG_SMP) || defined(CONFIG_KEXEC)
+#if defined(CONFIG_SMP) || defined(CONFIG_KEXEC_CORE)
 static bool use_spinloop(void)
 {
 	if (!IS_ENABLED(CONFIG_PPC_BOOK3E))
@@ -399,7 +399,7 @@ void smp_release_cpus(void)
 
 	DBG(" <- smp_release_cpus()\n");
 }
-#endif /* CONFIG_SMP || CONFIG_KEXEC */
+#endif /* CONFIG_SMP || CONFIG_KEXEC_CORE */
 
 /*
  * Initialize some remaining members of the ppc64_caches and systemcfg
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 9c6f3fd58059..893bd7f79be6 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -193,7 +193,7 @@ int smp_request_message_ipi(int virq, int msg)
 	if (msg < 0 || msg > PPC_MSG_DEBUGGER_BREAK) {
 		return -EINVAL;
 	}
-#if !defined(CONFIG_DEBUGGER) && !defined(CONFIG_KEXEC)
+#if !defined(CONFIG_DEBUGGER) && !defined(CONFIG_KEXEC_CORE)
 	if (msg == PPC_MSG_DEBUGGER_BREAK) {
 		return 1;
 	}
@@ -325,7 +325,7 @@ void tick_broadcast(const struct cpumask *mask)
 }
 #endif
 
-#if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC)
+#if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC_CORE)
 void smp_send_debugger_break(void)
 {
 	int cpu;
@@ -340,7 +340,7 @@ void smp_send_debugger_break(void)
 }
 #endif
 
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
 {
 	crash_ipi_function_ptr = crash_ipi_callback;
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 023a462725b5..4239aaf74886 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -64,8 +64,9 @@
 #include <asm/asm-prototypes.h>
 #include <asm/hmi.h>
 #include <sysdev/fsl_pci.h>
+#include <asm/kprobes.h>
 
-#if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC)
+#if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC_CORE)
 int (*__debugger)(struct pt_regs *regs) __read_mostly;
 int (*__debugger_ipi)(struct pt_regs *regs) __read_mostly;
 int (*__debugger_bpt)(struct pt_regs *regs) __read_mostly;
@@ -122,9 +123,6 @@ static unsigned long oops_begin(struct pt_regs *regs)
 	int cpu;
 	unsigned long flags;
 
-	if (debugger(regs))
-		return 1;
-
 	oops_enter();
 
 	/* racy, but better than risking deadlock. */
@@ -150,14 +148,15 @@ static void oops_end(unsigned long flags, struct pt_regs *regs,
 			       int signr)
 {
 	bust_spinlocks(0);
-	die_owner = -1;
 	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
 	die_nest_count--;
 	oops_exit();
 	printk("\n");
-	if (!die_nest_count)
+	if (!die_nest_count) {
 		/* Nest count reaches zero, release the lock. */
+		die_owner = -1;
 		arch_spin_unlock(&die_lock);
+	}
 	raw_local_irq_restore(flags);
 
 	crash_fadump(regs, "die oops");
@@ -227,8 +226,12 @@ NOKPROBE_SYMBOL(__die);
 
 void die(const char *str, struct pt_regs *regs, long err)
 {
-	unsigned long flags = oops_begin(regs);
+	unsigned long flags;
 
+	if (debugger(regs))
+		return;
+
+	flags = oops_begin(regs);
 	if (__die(str, regs, err))
 		err = 0;
 	oops_end(flags, regs, err);
@@ -365,7 +368,7 @@ static inline int check_io_access(struct pt_regs *regs)
 			       (*nip & 0x100)? "OUT to": "IN from",
 			       regs->gpr[rb] - _IO_BASE, nip);
 			regs->msr |= MSR_RI;
-			regs->nip = entry->fixup;
+			regs->nip = extable_fixup(entry);
 			return 1;
 		}
 	}
@@ -824,6 +827,9 @@ void single_step_exception(struct pt_regs *regs)
 
 	clear_single_step(regs);
 
+	if (kprobe_post_handler(regs))
+		return;
+
 	if (notify_die(DIE_SSTEP, "single_step", regs, 5,
 					5, SIGTRAP) == NOTIFY_STOP)
 		goto bail;
@@ -1177,6 +1183,9 @@ void program_check_exception(struct pt_regs *regs)
 		if (debugger_bpt(regs))
 			goto bail;
 
+		if (kprobe_handler(regs))
+			goto bail;
+
 		/* trap exception */
 		if (notify_die(DIE_BPT, "breakpoint", regs, 5, 5, SIGTRAP)
 				== NOTIFY_STOP)
@@ -1430,7 +1439,6 @@ void facility_unavailable_exception(struct pt_regs *regs)
 		[FSCR_TM_LG] = "TM",
 		[FSCR_EBB_LG] = "EBB",
 		[FSCR_TAR_LG] = "TAR",
-		[FSCR_LM_LG] = "LM",
 	};
 	char *facility = "unknown";
 	u64 value;
@@ -1488,14 +1496,6 @@ void facility_unavailable_exception(struct pt_regs *regs)
 			emulate_single_step(regs);
 		}
 		return;
-	} else if ((status == FSCR_LM_LG) && cpu_has_feature(CPU_FTR_ARCH_300)) {
-		/*
-		 * This process has touched LM, so turn it on forever
-		 * for this process
-		 */
-		current->thread.fscr |= FSCR_LM;
-		mtspr(SPRN_FSCR, current->thread.fscr);
-		return;
 	}
 
 	if (status == FSCR_TM_LG) {
@@ -1519,7 +1519,8 @@ void facility_unavailable_exception(struct pt_regs *regs)
 		return;
 	}
 
-	if ((status < ARRAY_SIZE(facility_strings)) &&
+	if ((hv || status >= 2) &&
+	    (status < ARRAY_SIZE(facility_strings)) &&
 	    facility_strings[status])
 		facility = facility_strings[status];
 
@@ -1527,9 +1528,8 @@ void facility_unavailable_exception(struct pt_regs *regs)
 	if (!arch_irq_disabled_regs(regs))
 		local_irq_enable();
 
-	pr_err_ratelimited(
-		"%sFacility '%s' unavailable, exception at 0x%lx, MSR=%lx\n",
-		hv ? "Hypervisor " : "", facility, regs->nip, regs->msr);
+	pr_err_ratelimited("%sFacility '%s' unavailable (%d), exception at 0x%lx, MSR=%lx\n",
+		hv ? "Hypervisor " : "", facility, status, regs->nip, regs->msr);
 
 out:
 	if (user_mode(regs)) {
@@ -1754,6 +1754,9 @@ void DebugException(struct pt_regs *regs, unsigned long debug_status)
 			return;
 		}
 
+		if (kprobe_post_handler(regs))
+			return;
+
 		if (notify_die(DIE_SSTEP, "block_step", regs, 5,
 			       5, SIGTRAP) == NOTIFY_STOP) {
 			return;
@@ -1768,6 +1771,9 @@ void DebugException(struct pt_regs *regs, unsigned long debug_status)
 		/* Clear the instruction completion event */
 		mtspr(SPRN_DBSR, DBSR_IC);
 
+		if (kprobe_post_handler(regs))
+			return;
+
 		if (notify_die(DIE_SSTEP, "single_step", regs, 5,
 			       5, SIGTRAP) == NOTIFY_STOP) {
 			return;
diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c
index 05f09ae82587..b795dd1ac2ef 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_hv.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c
@@ -88,6 +88,8 @@ long kvmppc_alloc_hpt(struct kvm *kvm, u32 *htab_orderp)
 	/* 128 (2**7) bytes in each HPTEG */
 	kvm->arch.hpt_mask = (1ul << (order - 7)) - 1;
 
+	atomic64_set(&kvm->arch.mmio_update, 0);
+
 	/* Allocate reverse map array */
 	rev = vmalloc(sizeof(struct revmap_entry) * kvm->arch.hpt_npte);
 	if (!rev) {
@@ -255,7 +257,7 @@ static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu *vcpu)
 	kvmppc_set_msr(vcpu, msr);
 }
 
-long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
+static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
 				long pte_index, unsigned long pteh,
 				unsigned long ptel, unsigned long *pte_idx_ret)
 {
@@ -312,7 +314,7 @@ static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
 	struct kvmppc_slb *slbe;
 	unsigned long slb_v;
 	unsigned long pp, key;
-	unsigned long v, gr;
+	unsigned long v, orig_v, gr;
 	__be64 *hptep;
 	int index;
 	int virtmode = vcpu->arch.shregs.msr & (data ? MSR_DR : MSR_IR);
@@ -337,10 +339,12 @@ static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
 		return -ENOENT;
 	}
 	hptep = (__be64 *)(kvm->arch.hpt_virt + (index << 4));
-	v = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
+	v = orig_v = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
+	if (cpu_has_feature(CPU_FTR_ARCH_300))
+		v = hpte_new_to_old_v(v, be64_to_cpu(hptep[1]));
 	gr = kvm->arch.revmap[index].guest_rpte;
 
-	unlock_hpte(hptep, v);
+	unlock_hpte(hptep, orig_v);
 	preempt_enable();
 
 	gpte->eaddr = eaddr;
@@ -438,6 +442,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 {
 	struct kvm *kvm = vcpu->kvm;
 	unsigned long hpte[3], r;
+	unsigned long hnow_v, hnow_r;
 	__be64 *hptep;
 	unsigned long mmu_seq, psize, pte_size;
 	unsigned long gpa_base, gfn_base;
@@ -451,6 +456,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 	unsigned int writing, write_ok;
 	struct vm_area_struct *vma;
 	unsigned long rcbits;
+	long mmio_update;
 
 	/*
 	 * Real-mode code has already searched the HPT and found the
@@ -460,6 +466,19 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 	 */
 	if (ea != vcpu->arch.pgfault_addr)
 		return RESUME_GUEST;
+
+	if (vcpu->arch.pgfault_cache) {
+		mmio_update = atomic64_read(&kvm->arch.mmio_update);
+		if (mmio_update == vcpu->arch.pgfault_cache->mmio_update) {
+			r = vcpu->arch.pgfault_cache->rpte;
+			psize = hpte_page_size(vcpu->arch.pgfault_hpte[0], r);
+			gpa_base = r & HPTE_R_RPN & ~(psize - 1);
+			gfn_base = gpa_base >> PAGE_SHIFT;
+			gpa = gpa_base | (ea & (psize - 1));
+			return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea,
+						dsisr & DSISR_ISSTORE);
+		}
+	}
 	index = vcpu->arch.pgfault_index;
 	hptep = (__be64 *)(kvm->arch.hpt_virt + (index << 4));
 	rev = &kvm->arch.revmap[index];
@@ -472,6 +491,10 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 	unlock_hpte(hptep, hpte[0]);
 	preempt_enable();
 
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		hpte[0] = hpte_new_to_old_v(hpte[0], hpte[1]);
+		hpte[1] = hpte_new_to_old_r(hpte[1]);
+	}
 	if (hpte[0] != vcpu->arch.pgfault_hpte[0] ||
 	    hpte[1] != vcpu->arch.pgfault_hpte[1])
 		return RESUME_GUEST;
@@ -575,16 +598,22 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 	 */
 	if (psize < PAGE_SIZE)
 		psize = PAGE_SIZE;
-	r = (r & ~(HPTE_R_PP0 - psize)) | ((pfn << PAGE_SHIFT) & ~(psize - 1));
+	r = (r & HPTE_R_KEY_HI) | (r & ~(HPTE_R_PP0 - psize)) |
+					((pfn << PAGE_SHIFT) & ~(psize - 1));
 	if (hpte_is_writable(r) && !write_ok)
 		r = hpte_make_readonly(r);
 	ret = RESUME_GUEST;
 	preempt_disable();
 	while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
 		cpu_relax();
-	if ((be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK) != hpte[0] ||
-		be64_to_cpu(hptep[1]) != hpte[1] ||
-		rev->guest_rpte != hpte[2])
+	hnow_v = be64_to_cpu(hptep[0]);
+	hnow_r = be64_to_cpu(hptep[1]);
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		hnow_v = hpte_new_to_old_v(hnow_v, hnow_r);
+		hnow_r = hpte_new_to_old_r(hnow_r);
+	}
+	if ((hnow_v & ~HPTE_V_HVLOCK) != hpte[0] || hnow_r != hpte[1] ||
+	    rev->guest_rpte != hpte[2])
 		/* HPTE has been changed under us; let the guest retry */
 		goto out_unlock;
 	hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID;
@@ -615,6 +644,10 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 		kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0);
 	}
 
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		r = hpte_old_to_new_r(hpte[0], r);
+		hpte[0] = hpte_old_to_new_v(hpte[0]);
+	}
 	hptep[1] = cpu_to_be64(r);
 	eieio();
 	__unlock_hpte(hptep, hpte[0]);
@@ -758,6 +791,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
 		    hpte_rpn(ptel, psize) == gfn) {
 			hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
 			kvmppc_invalidate_hpte(kvm, hptep, i);
+			hptep[1] &= ~cpu_to_be64(HPTE_R_KEY_HI | HPTE_R_KEY_LO);
 			/* Harvest R and C */
 			rcbits = be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
 			*rmapp |= rcbits << KVMPPC_RMAP_RC_SHIFT;
@@ -1165,7 +1199,7 @@ static long record_hpte(unsigned long flags, __be64 *hptp,
 			unsigned long *hpte, struct revmap_entry *revp,
 			int want_valid, int first_pass)
 {
-	unsigned long v, r;
+	unsigned long v, r, hr;
 	unsigned long rcbits_unset;
 	int ok = 1;
 	int valid, dirty;
@@ -1192,6 +1226,11 @@ static long record_hpte(unsigned long flags, __be64 *hptp,
 		while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
 			cpu_relax();
 		v = be64_to_cpu(hptp[0]);
+		hr = be64_to_cpu(hptp[1]);
+		if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+			v = hpte_new_to_old_v(v, hr);
+			hr = hpte_new_to_old_r(hr);
+		}
 
 		/* re-evaluate valid and dirty from synchronized HPTE value */
 		valid = !!(v & HPTE_V_VALID);
@@ -1199,8 +1238,8 @@ static long record_hpte(unsigned long flags, __be64 *hptp,
 
 		/* Harvest R and C into guest view if necessary */
 		rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
-		if (valid && (rcbits_unset & be64_to_cpu(hptp[1]))) {
-			revp->guest_rpte |= (be64_to_cpu(hptp[1]) &
+		if (valid && (rcbits_unset & hr)) {
+			revp->guest_rpte |= (hr &
 				(HPTE_R_R | HPTE_R_C)) | HPTE_GR_MODIFIED;
 			dirty = 1;
 		}
@@ -1608,7 +1647,7 @@ static ssize_t debugfs_htab_read(struct file *file, char __user *buf,
 	return ret;
 }
 
-ssize_t debugfs_htab_write(struct file *file, const char __user *buf,
+static ssize_t debugfs_htab_write(struct file *file, const char __user *buf,
 			   size_t len, loff_t *ppos)
 {
 	return -EACCES;
diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c
index d461c440889a..e4c4ea973e57 100644
--- a/arch/powerpc/kvm/book3s_64_vio_hv.c
+++ b/arch/powerpc/kvm/book3s_64_vio_hv.c
@@ -39,7 +39,6 @@
 #include <asm/udbg.h>
 #include <asm/iommu.h>
 #include <asm/tce.h>
-#include <asm/iommu.h>
 
 #define TCES_PER_PAGE	(PAGE_SIZE / sizeof(u64))
 
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index 39ef1f4a7b02..8dcbe37a4dac 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -54,6 +54,9 @@
 #include <asm/dbell.h>
 #include <asm/hmi.h>
 #include <asm/pnv-pci.h>
+#include <asm/mmu.h>
+#include <asm/opal.h>
+#include <asm/xics.h>
 #include <linux/gfp.h>
 #include <linux/vmalloc.h>
 #include <linux/highmem.h>
@@ -62,6 +65,7 @@
 #include <linux/irqbypass.h>
 #include <linux/module.h>
 #include <linux/compiler.h>
+#include <linux/of.h>
 
 #include "book3s.h"
 
@@ -104,23 +108,6 @@ module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect,
 MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core");
 #endif
 
-/* Maximum halt poll interval defaults to KVM_HALT_POLL_NS_DEFAULT */
-static unsigned int halt_poll_max_ns = KVM_HALT_POLL_NS_DEFAULT;
-module_param(halt_poll_max_ns, uint, S_IRUGO | S_IWUSR);
-MODULE_PARM_DESC(halt_poll_max_ns, "Maximum halt poll time in ns");
-
-/* Factor by which the vcore halt poll interval is grown, default is to double
- */
-static unsigned int halt_poll_ns_grow = 2;
-module_param(halt_poll_ns_grow, int, S_IRUGO);
-MODULE_PARM_DESC(halt_poll_ns_grow, "Factor halt poll time is grown by");
-
-/* Factor by which the vcore halt poll interval is shrunk, default is to reset
- */
-static unsigned int halt_poll_ns_shrink;
-module_param(halt_poll_ns_shrink, int, S_IRUGO);
-MODULE_PARM_DESC(halt_poll_ns_shrink, "Factor halt poll time is shrunk by");
-
 static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
 
@@ -146,12 +133,21 @@ static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc,
 
 static bool kvmppc_ipi_thread(int cpu)
 {
+	unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
+
+	/* On POWER9 we can use msgsnd to IPI any cpu */
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		msg |= get_hard_smp_processor_id(cpu);
+		smp_mb();
+		__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
+		return true;
+	}
+
 	/* On POWER8 for IPIs to threads in the same core, use msgsnd */
 	if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
 		preempt_disable();
 		if (cpu_first_thread_sibling(cpu) ==
 		    cpu_first_thread_sibling(smp_processor_id())) {
-			unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
 			msg |= cpu_thread_in_core(cpu);
 			smp_mb();
 			__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
@@ -162,8 +158,12 @@ static bool kvmppc_ipi_thread(int cpu)
 	}
 
 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP)
-	if (cpu >= 0 && cpu < nr_cpu_ids && paca[cpu].kvm_hstate.xics_phys) {
-		xics_wake_cpu(cpu);
+	if (cpu >= 0 && cpu < nr_cpu_ids) {
+		if (paca[cpu].kvm_hstate.xics_phys) {
+			xics_wake_cpu(cpu);
+			return true;
+		}
+		opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY);
 		return true;
 	}
 #endif
@@ -299,41 +299,54 @@ static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr)
 	vcpu->arch.pvr = pvr;
 }
 
+/* Dummy value used in computing PCR value below */
+#define PCR_ARCH_300	(PCR_ARCH_207 << 1)
+
 static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat)
 {
-	unsigned long pcr = 0;
+	unsigned long host_pcr_bit = 0, guest_pcr_bit = 0;
 	struct kvmppc_vcore *vc = vcpu->arch.vcore;
 
+	/* We can (emulate) our own architecture version and anything older */
+	if (cpu_has_feature(CPU_FTR_ARCH_300))
+		host_pcr_bit = PCR_ARCH_300;
+	else if (cpu_has_feature(CPU_FTR_ARCH_207S))
+		host_pcr_bit = PCR_ARCH_207;
+	else if (cpu_has_feature(CPU_FTR_ARCH_206))
+		host_pcr_bit = PCR_ARCH_206;
+	else
+		host_pcr_bit = PCR_ARCH_205;
+
+	/* Determine lowest PCR bit needed to run guest in given PVR level */
+	guest_pcr_bit = host_pcr_bit;
 	if (arch_compat) {
 		switch (arch_compat) {
 		case PVR_ARCH_205:
-			/*
-			 * If an arch bit is set in PCR, all the defined
-			 * higher-order arch bits also have to be set.
-			 */
-			pcr = PCR_ARCH_206 | PCR_ARCH_205;
+			guest_pcr_bit = PCR_ARCH_205;
 			break;
 		case PVR_ARCH_206:
 		case PVR_ARCH_206p:
-			pcr = PCR_ARCH_206;
+			guest_pcr_bit = PCR_ARCH_206;
 			break;
 		case PVR_ARCH_207:
+			guest_pcr_bit = PCR_ARCH_207;
+			break;
+		case PVR_ARCH_300:
+			guest_pcr_bit = PCR_ARCH_300;
 			break;
 		default:
 			return -EINVAL;
 		}
-
-		if (!cpu_has_feature(CPU_FTR_ARCH_207S)) {
-			/* POWER7 can't emulate POWER8 */
-			if (!(pcr & PCR_ARCH_206))
-				return -EINVAL;
-			pcr &= ~PCR_ARCH_206;
-		}
 	}
 
+	/* Check requested PCR bits don't exceed our capabilities */
+	if (guest_pcr_bit > host_pcr_bit)
+		return -EINVAL;
+
 	spin_lock(&vc->lock);
 	vc->arch_compat = arch_compat;
-	vc->pcr = pcr;
+	/* Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit */
+	vc->pcr = host_pcr_bit - guest_pcr_bit;
 	spin_unlock(&vc->lock);
 
 	return 0;
@@ -945,6 +958,7 @@ static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
 		break;
 	case BOOK3S_INTERRUPT_EXTERNAL:
 	case BOOK3S_INTERRUPT_H_DOORBELL:
+	case BOOK3S_INTERRUPT_H_VIRT:
 		vcpu->stat.ext_intr_exits++;
 		r = RESUME_GUEST;
 		break;
@@ -1229,6 +1243,12 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
 	case KVM_REG_PPC_WORT:
 		*val = get_reg_val(id, vcpu->arch.wort);
 		break;
+	case KVM_REG_PPC_TIDR:
+		*val = get_reg_val(id, vcpu->arch.tid);
+		break;
+	case KVM_REG_PPC_PSSCR:
+		*val = get_reg_val(id, vcpu->arch.psscr);
+		break;
 	case KVM_REG_PPC_VPA_ADDR:
 		spin_lock(&vcpu->arch.vpa_update_lock);
 		*val = get_reg_val(id, vcpu->arch.vpa.next_gpa);
@@ -1288,6 +1308,9 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
 	case KVM_REG_PPC_TM_CR:
 		*val = get_reg_val(id, vcpu->arch.cr_tm);
 		break;
+	case KVM_REG_PPC_TM_XER:
+		*val = get_reg_val(id, vcpu->arch.xer_tm);
+		break;
 	case KVM_REG_PPC_TM_LR:
 		*val = get_reg_val(id, vcpu->arch.lr_tm);
 		break;
@@ -1427,6 +1450,12 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
 	case KVM_REG_PPC_WORT:
 		vcpu->arch.wort = set_reg_val(id, *val);
 		break;
+	case KVM_REG_PPC_TIDR:
+		vcpu->arch.tid = set_reg_val(id, *val);
+		break;
+	case KVM_REG_PPC_PSSCR:
+		vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS;
+		break;
 	case KVM_REG_PPC_VPA_ADDR:
 		addr = set_reg_val(id, *val);
 		r = -EINVAL;
@@ -1498,6 +1527,9 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
 	case KVM_REG_PPC_TM_CR:
 		vcpu->arch.cr_tm = set_reg_val(id, *val);
 		break;
+	case KVM_REG_PPC_TM_XER:
+		vcpu->arch.xer_tm = set_reg_val(id, *val);
+		break;
 	case KVM_REG_PPC_TM_LR:
 		vcpu->arch.lr_tm = set_reg_val(id, *val);
 		break;
@@ -1540,6 +1572,20 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
 	return r;
 }
 
+/*
+ * On POWER9, threads are independent and can be in different partitions.
+ * Therefore we consider each thread to be a subcore.
+ * There is a restriction that all threads have to be in the same
+ * MMU mode (radix or HPT), unfortunately, but since we only support
+ * HPT guests on a HPT host so far, that isn't an impediment yet.
+ */
+static int threads_per_vcore(void)
+{
+	if (cpu_has_feature(CPU_FTR_ARCH_300))
+		return 1;
+	return threads_per_subcore;
+}
+
 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core)
 {
 	struct kvmppc_vcore *vcore;
@@ -1554,7 +1600,7 @@ static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core)
 	init_swait_queue_head(&vcore->wq);
 	vcore->preempt_tb = TB_NIL;
 	vcore->lpcr = kvm->arch.lpcr;
-	vcore->first_vcpuid = core * threads_per_subcore;
+	vcore->first_vcpuid = core * threads_per_vcore();
 	vcore->kvm = kvm;
 	INIT_LIST_HEAD(&vcore->preempt_list);
 
@@ -1717,7 +1763,7 @@ static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm,
 	int core;
 	struct kvmppc_vcore *vcore;
 
-	core = id / threads_per_subcore;
+	core = id / threads_per_vcore();
 	if (core >= KVM_MAX_VCORES)
 		goto out;
 
@@ -1935,7 +1981,10 @@ static void kvmppc_wait_for_nap(void)
 {
 	int cpu = smp_processor_id();
 	int i, loops;
+	int n_threads = threads_per_vcore();
 
+	if (n_threads <= 1)
+		return;
 	for (loops = 0; loops < 1000000; ++loops) {
 		/*
 		 * Check if all threads are finished.
@@ -1943,17 +1992,17 @@ static void kvmppc_wait_for_nap(void)
 		 * and the thread clears it when finished, so we look
 		 * for any threads that still have a non-NULL vcore ptr.
 		 */
-		for (i = 1; i < threads_per_subcore; ++i)
+		for (i = 1; i < n_threads; ++i)
 			if (paca[cpu + i].kvm_hstate.kvm_vcore)
 				break;
-		if (i == threads_per_subcore) {
+		if (i == n_threads) {
 			HMT_medium();
 			return;
 		}
 		HMT_low();
 	}
 	HMT_medium();
-	for (i = 1; i < threads_per_subcore; ++i)
+	for (i = 1; i < n_threads; ++i)
 		if (paca[cpu + i].kvm_hstate.kvm_vcore)
 			pr_err("KVM: CPU %d seems to be stuck\n", cpu + i);
 }
@@ -2019,7 +2068,7 @@ static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc)
 
 	vc->vcore_state = VCORE_PREEMPT;
 	vc->pcpu = smp_processor_id();
-	if (vc->num_threads < threads_per_subcore) {
+	if (vc->num_threads < threads_per_vcore()) {
 		spin_lock(&lp->lock);
 		list_add_tail(&vc->preempt_list, &lp->list);
 		spin_unlock(&lp->lock);
@@ -2123,8 +2172,7 @@ static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip)
 	cip->subcore_threads[sub] = vc->num_threads;
 	cip->subcore_vm[sub] = vc->kvm;
 	init_master_vcore(vc);
-	list_del(&vc->preempt_list);
-	list_add_tail(&vc->preempt_list, &cip->vcs[sub]);
+	list_move_tail(&vc->preempt_list, &cip->vcs[sub]);
 
 	return true;
 }
@@ -2309,6 +2357,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 	unsigned long cmd_bit, stat_bit;
 	int pcpu, thr;
 	int target_threads;
+	int controlled_threads;
 
 	/*
 	 * Remove from the list any threads that have a signal pending
@@ -2327,11 +2376,18 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 	vc->preempt_tb = TB_NIL;
 
 	/*
+	 * Number of threads that we will be controlling: the same as
+	 * the number of threads per subcore, except on POWER9,
+	 * where it's 1 because the threads are (mostly) independent.
+	 */
+	controlled_threads = threads_per_vcore();
+
+	/*
 	 * Make sure we are running on primary threads, and that secondary
 	 * threads are offline.  Also check if the number of threads in this
 	 * guest are greater than the current system threads per guest.
 	 */
-	if ((threads_per_core > 1) &&
+	if ((controlled_threads > 1) &&
 	    ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) {
 		for_each_runnable_thread(i, vcpu, vc) {
 			vcpu->arch.ret = -EBUSY;
@@ -2347,7 +2403,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 	 */
 	init_core_info(&core_info, vc);
 	pcpu = smp_processor_id();
-	target_threads = threads_per_subcore;
+	target_threads = controlled_threads;
 	if (target_smt_mode && target_smt_mode < target_threads)
 		target_threads = target_smt_mode;
 	if (vc->num_threads < target_threads)
@@ -2383,7 +2439,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 		smp_wmb();
 	}
 	pcpu = smp_processor_id();
-	for (thr = 0; thr < threads_per_subcore; ++thr)
+	for (thr = 0; thr < controlled_threads; ++thr)
 		paca[pcpu + thr].kvm_hstate.kvm_split_mode = sip;
 
 	/* Initiate micro-threading (split-core) if required */
@@ -2493,7 +2549,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 	}
 
 	/* Let secondaries go back to the offline loop */
-	for (i = 0; i < threads_per_subcore; ++i) {
+	for (i = 0; i < controlled_threads; ++i) {
 		kvmppc_release_hwthread(pcpu + i);
 		if (sip && sip->napped[i])
 			kvmppc_ipi_thread(pcpu + i);
@@ -2545,9 +2601,6 @@ static void grow_halt_poll_ns(struct kvmppc_vcore *vc)
 		vc->halt_poll_ns = 10000;
 	else
 		vc->halt_poll_ns *= halt_poll_ns_grow;
-
-	if (vc->halt_poll_ns > halt_poll_max_ns)
-		vc->halt_poll_ns = halt_poll_max_ns;
 }
 
 static void shrink_halt_poll_ns(struct kvmppc_vcore *vc)
@@ -2558,7 +2611,8 @@ static void shrink_halt_poll_ns(struct kvmppc_vcore *vc)
 		vc->halt_poll_ns /= halt_poll_ns_shrink;
 }
 
-/* Check to see if any of the runnable vcpus on the vcore have pending
+/*
+ * Check to see if any of the runnable vcpus on the vcore have pending
  * exceptions or are no longer ceded
  */
 static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc)
@@ -2657,16 +2711,18 @@ out:
 	}
 
 	/* Adjust poll time */
-	if (halt_poll_max_ns) {
+	if (halt_poll_ns) {
 		if (block_ns <= vc->halt_poll_ns)
 			;
 		/* We slept and blocked for longer than the max halt time */
-		else if (vc->halt_poll_ns && block_ns > halt_poll_max_ns)
+		else if (vc->halt_poll_ns && block_ns > halt_poll_ns)
 			shrink_halt_poll_ns(vc);
 		/* We slept and our poll time is too small */
-		else if (vc->halt_poll_ns < halt_poll_max_ns &&
-				block_ns < halt_poll_max_ns)
+		else if (vc->halt_poll_ns < halt_poll_ns &&
+				block_ns < halt_poll_ns)
 			grow_halt_poll_ns(vc);
+		if (vc->halt_poll_ns > halt_poll_ns)
+			vc->halt_poll_ns = halt_poll_ns;
 	} else
 		vc->halt_poll_ns = 0;
 
@@ -2973,6 +3029,15 @@ static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
 	struct kvm_memslots *slots;
 	struct kvm_memory_slot *memslot;
 
+	/*
+	 * If we are making a new memslot, it might make
+	 * some address that was previously cached as emulated
+	 * MMIO be no longer emulated MMIO, so invalidate
+	 * all the caches of emulated MMIO translations.
+	 */
+	if (npages)
+		atomic64_inc(&kvm->arch.mmio_update);
+
 	if (npages && old->npages) {
 		/*
 		 * If modifying a memslot, reset all the rmap dirty bits.
@@ -3017,6 +3082,22 @@ static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu)
 	return;
 }
 
+static void kvmppc_setup_partition_table(struct kvm *kvm)
+{
+	unsigned long dw0, dw1;
+
+	/* PS field - page size for VRMA */
+	dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) |
+		((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1);
+	/* HTABSIZE and HTABORG fields */
+	dw0 |= kvm->arch.sdr1;
+
+	/* Second dword has GR=0; other fields are unused since UPRT=0 */
+	dw1 = 0;
+
+	mmu_partition_table_set_entry(kvm->arch.lpid, dw0, dw1);
+}
+
 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
 {
 	int err = 0;
@@ -3068,17 +3149,20 @@ static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
 	      psize == 0x1000000))
 		goto out_srcu;
 
-	/* Update VRMASD field in the LPCR */
 	senc = slb_pgsize_encoding(psize);
 	kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
 		(VRMA_VSID << SLB_VSID_SHIFT_1T);
-	/* the -4 is to account for senc values starting at 0x10 */
-	lpcr = senc << (LPCR_VRMASD_SH - 4);
-
 	/* Create HPTEs in the hash page table for the VRMA */
 	kvmppc_map_vrma(vcpu, memslot, porder);
 
-	kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
+	/* Update VRMASD field in the LPCR */
+	if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
+		/* the -4 is to account for senc values starting at 0x10 */
+		lpcr = senc << (LPCR_VRMASD_SH - 4);
+		kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD);
+	} else {
+		kvmppc_setup_partition_table(kvm);
+	}
 
 	/* Order updates to kvm->arch.lpcr etc. vs. hpte_setup_done */
 	smp_wmb();
@@ -3193,14 +3277,18 @@ static int kvmppc_core_init_vm_hv(struct kvm *kvm)
 	 * Since we don't flush the TLB when tearing down a VM,
 	 * and this lpid might have previously been used,
 	 * make sure we flush on each core before running the new VM.
+	 * On POWER9, the tlbie in mmu_partition_table_set_entry()
+	 * does this flush for us.
 	 */
-	cpumask_setall(&kvm->arch.need_tlb_flush);
+	if (!cpu_has_feature(CPU_FTR_ARCH_300))
+		cpumask_setall(&kvm->arch.need_tlb_flush);
 
 	/* Start out with the default set of hcalls enabled */
 	memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls,
 	       sizeof(kvm->arch.enabled_hcalls));
 
-	kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
+	if (!cpu_has_feature(CPU_FTR_ARCH_300))
+		kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
 
 	/* Init LPCR for virtual RMA mode */
 	kvm->arch.host_lpid = mfspr(SPRN_LPID);
@@ -3213,9 +3301,29 @@ static int kvmppc_core_init_vm_hv(struct kvm *kvm)
 	/* On POWER8 turn on online bit to enable PURR/SPURR */
 	if (cpu_has_feature(CPU_FTR_ARCH_207S))
 		lpcr |= LPCR_ONL;
+	/*
+	 * On POWER9, VPM0 bit is reserved (VPM0=1 behaviour is assumed)
+	 * Set HVICE bit to enable hypervisor virtualization interrupts.
+	 */
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		lpcr &= ~LPCR_VPM0;
+		lpcr |= LPCR_HVICE;
+	}
+
 	kvm->arch.lpcr = lpcr;
 
 	/*
+	 * Work out how many sets the TLB has, for the use of
+	 * the TLB invalidation loop in book3s_hv_rmhandlers.S.
+	 */
+	if (cpu_has_feature(CPU_FTR_ARCH_300))
+		kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH;	/* 256 */
+	else if (cpu_has_feature(CPU_FTR_ARCH_207S))
+		kvm->arch.tlb_sets = POWER8_TLB_SETS;		/* 512 */
+	else
+		kvm->arch.tlb_sets = POWER7_TLB_SETS;		/* 128 */
+
+	/*
 	 * Track that we now have a HV mode VM active. This blocks secondary
 	 * CPU threads from coming online.
 	 */
@@ -3279,9 +3387,9 @@ static int kvmppc_core_check_processor_compat_hv(void)
 	    !cpu_has_feature(CPU_FTR_ARCH_206))
 		return -EIO;
 	/*
-	 * Disable KVM for Power9, untill the required bits merged.
+	 * Disable KVM for Power9 in radix mode.
 	 */
-	if (cpu_has_feature(CPU_FTR_ARCH_300))
+	if (cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled())
 		return -EIO;
 
 	return 0;
@@ -3635,6 +3743,23 @@ static int kvmppc_book3s_init_hv(void)
 	if (r)
 		return r;
 
+	/*
+	 * We need a way of accessing the XICS interrupt controller,
+	 * either directly, via paca[cpu].kvm_hstate.xics_phys, or
+	 * indirectly, via OPAL.
+	 */
+#ifdef CONFIG_SMP
+	if (!get_paca()->kvm_hstate.xics_phys) {
+		struct device_node *np;
+
+		np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc");
+		if (!np) {
+			pr_err("KVM-HV: Cannot determine method for accessing XICS\n");
+			return -ENODEV;
+		}
+	}
+#endif
+
 	kvm_ops_hv.owner = THIS_MODULE;
 	kvmppc_hv_ops = &kvm_ops_hv;
 
@@ -3657,3 +3782,4 @@ module_exit(kvmppc_book3s_exit_hv);
 MODULE_LICENSE("GPL");
 MODULE_ALIAS_MISCDEV(KVM_MINOR);
 MODULE_ALIAS("devname:kvm");
+
diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c
index 0c84d6bc8356..5bb24be0b346 100644
--- a/arch/powerpc/kvm/book3s_hv_builtin.c
+++ b/arch/powerpc/kvm/book3s_hv_builtin.c
@@ -26,6 +26,8 @@
 #include <asm/dbell.h>
 #include <asm/cputhreads.h>
 #include <asm/io.h>
+#include <asm/opal.h>
+#include <asm/smp.h>
 
 #define KVM_CMA_CHUNK_ORDER	18
 
@@ -205,12 +207,18 @@ static inline void rm_writeb(unsigned long paddr, u8 val)
 void kvmhv_rm_send_ipi(int cpu)
 {
 	unsigned long xics_phys;
+	unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
 
-	/* On POWER8 for IPIs to threads in the same core, use msgsnd */
+	/* On POWER9 we can use msgsnd for any destination cpu. */
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		msg |= get_hard_smp_processor_id(cpu);
+		__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
+		return;
+	}
+	/* On POWER8 for IPIs to threads in the same core, use msgsnd. */
 	if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
 	    cpu_first_thread_sibling(cpu) ==
 	    cpu_first_thread_sibling(raw_smp_processor_id())) {
-		unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
 		msg |= cpu_thread_in_core(cpu);
 		__asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
 		return;
@@ -218,7 +226,11 @@ void kvmhv_rm_send_ipi(int cpu)
 
 	/* Else poke the target with an IPI */
 	xics_phys = paca[cpu].kvm_hstate.xics_phys;
-	rm_writeb(xics_phys + XICS_MFRR, IPI_PRIORITY);
+	if (xics_phys)
+		rm_writeb(xics_phys + XICS_MFRR, IPI_PRIORITY);
+	else
+		opal_rm_int_set_mfrr(get_hard_smp_processor_id(cpu),
+				     IPI_PRIORITY);
 }
 
 /*
@@ -329,7 +341,7 @@ static struct kvmppc_irq_map *get_irqmap(struct kvmppc_passthru_irqmap *pimap,
  * saved a copy of the XIRR in the PACA, it will be picked up by
  * the host ICP driver.
  */
-static int kvmppc_check_passthru(u32 xisr, __be32 xirr)
+static int kvmppc_check_passthru(u32 xisr, __be32 xirr, bool *again)
 {
 	struct kvmppc_passthru_irqmap *pimap;
 	struct kvmppc_irq_map *irq_map;
@@ -348,11 +360,11 @@ static int kvmppc_check_passthru(u32 xisr, __be32 xirr)
 	/* We're handling this interrupt, generic code doesn't need to */
 	local_paca->kvm_hstate.saved_xirr = 0;
 
-	return kvmppc_deliver_irq_passthru(vcpu, xirr, irq_map, pimap);
+	return kvmppc_deliver_irq_passthru(vcpu, xirr, irq_map, pimap, again);
 }
 
 #else
-static inline int kvmppc_check_passthru(u32 xisr, __be32 xirr)
+static inline int kvmppc_check_passthru(u32 xisr, __be32 xirr, bool *again)
 {
 	return 1;
 }
@@ -367,14 +379,31 @@ static inline int kvmppc_check_passthru(u32 xisr, __be32 xirr)
  *	-1 if there was a guest wakeup IPI (which has now been cleared)
  *	-2 if there is PCI passthrough external interrupt that was handled
  */
+static long kvmppc_read_one_intr(bool *again);
 
 long kvmppc_read_intr(void)
 {
+	long ret = 0;
+	long rc;
+	bool again;
+
+	do {
+		again = false;
+		rc = kvmppc_read_one_intr(&again);
+		if (rc && (ret == 0 || rc > ret))
+			ret = rc;
+	} while (again);
+	return ret;
+}
+
+static long kvmppc_read_one_intr(bool *again)
+{
 	unsigned long xics_phys;
 	u32 h_xirr;
 	__be32 xirr;
 	u32 xisr;
 	u8 host_ipi;
+	int64_t rc;
 
 	/* see if a host IPI is pending */
 	host_ipi = local_paca->kvm_hstate.host_ipi;
@@ -383,8 +412,14 @@ long kvmppc_read_intr(void)
 
 	/* Now read the interrupt from the ICP */
 	xics_phys = local_paca->kvm_hstate.xics_phys;
-	if (unlikely(!xics_phys))
-		return 1;
+	if (!xics_phys) {
+		/* Use OPAL to read the XIRR */
+		rc = opal_rm_int_get_xirr(&xirr, false);
+		if (rc < 0)
+			return 1;
+	} else {
+		xirr = _lwzcix(xics_phys + XICS_XIRR);
+	}
 
 	/*
 	 * Save XIRR for later. Since we get control in reverse endian
@@ -392,7 +427,6 @@ long kvmppc_read_intr(void)
 	 * host endian. Note that xirr is the value read from the
 	 * XIRR register, while h_xirr is the host endian version.
 	 */
-	xirr = _lwzcix(xics_phys + XICS_XIRR);
 	h_xirr = be32_to_cpu(xirr);
 	local_paca->kvm_hstate.saved_xirr = h_xirr;
 	xisr = h_xirr & 0xffffff;
@@ -411,8 +445,16 @@ long kvmppc_read_intr(void)
 	 * If it is an IPI, clear the MFRR and EOI it.
 	 */
 	if (xisr == XICS_IPI) {
-		_stbcix(xics_phys + XICS_MFRR, 0xff);
-		_stwcix(xics_phys + XICS_XIRR, xirr);
+		if (xics_phys) {
+			_stbcix(xics_phys + XICS_MFRR, 0xff);
+			_stwcix(xics_phys + XICS_XIRR, xirr);
+		} else {
+			opal_rm_int_set_mfrr(hard_smp_processor_id(), 0xff);
+			rc = opal_rm_int_eoi(h_xirr);
+			/* If rc > 0, there is another interrupt pending */
+			*again = rc > 0;
+		}
+
 		/*
 		 * Need to ensure side effects of above stores
 		 * complete before proceeding.
@@ -429,7 +471,11 @@ long kvmppc_read_intr(void)
 			/* We raced with the host,
 			 * we need to resend that IPI, bummer
 			 */
-			_stbcix(xics_phys + XICS_MFRR, IPI_PRIORITY);
+			if (xics_phys)
+				_stbcix(xics_phys + XICS_MFRR, IPI_PRIORITY);
+			else
+				opal_rm_int_set_mfrr(hard_smp_processor_id(),
+						     IPI_PRIORITY);
 			/* Let side effects complete */
 			smp_mb();
 			return 1;
@@ -440,5 +486,5 @@ long kvmppc_read_intr(void)
 		return -1;
 	}
 
-	return kvmppc_check_passthru(xisr, xirr);
+	return kvmppc_check_passthru(xisr, xirr, again);
 }
diff --git a/arch/powerpc/kvm/book3s_hv_ras.c b/arch/powerpc/kvm/book3s_hv_ras.c
index 0fa70a9618d7..7ef0993214f3 100644
--- a/arch/powerpc/kvm/book3s_hv_ras.c
+++ b/arch/powerpc/kvm/book3s_hv_ras.c
@@ -16,6 +16,7 @@
 #include <asm/machdep.h>
 #include <asm/cputhreads.h>
 #include <asm/hmi.h>
+#include <asm/kvm_ppc.h>
 
 /* SRR1 bits for machine check on POWER7 */
 #define SRR1_MC_LDSTERR		(1ul << (63-42))
diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
index 99b4e9d5dd23..9ef3c4be952f 100644
--- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
@@ -264,8 +264,10 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
 
 	if (pa)
 		pteh |= HPTE_V_VALID;
-	else
+	else {
 		pteh |= HPTE_V_ABSENT;
+		ptel &= ~(HPTE_R_KEY_HI | HPTE_R_KEY_LO);
+	}
 
 	/*If we had host pte mapping then  Check WIMG */
 	if (ptep && !hpte_cache_flags_ok(ptel, is_ci)) {
@@ -351,6 +353,7 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
 			/* inval in progress, write a non-present HPTE */
 			pteh |= HPTE_V_ABSENT;
 			pteh &= ~HPTE_V_VALID;
+			ptel &= ~(HPTE_R_KEY_HI | HPTE_R_KEY_LO);
 			unlock_rmap(rmap);
 		} else {
 			kvmppc_add_revmap_chain(kvm, rev, rmap, pte_index,
@@ -361,6 +364,11 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
 		}
 	}
 
+	/* Convert to new format on P9 */
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		ptel = hpte_old_to_new_r(pteh, ptel);
+		pteh = hpte_old_to_new_v(pteh);
+	}
 	hpte[1] = cpu_to_be64(ptel);
 
 	/* Write the first HPTE dword, unlocking the HPTE and making it valid */
@@ -386,6 +394,13 @@ long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags,
 #define LOCK_TOKEN	(*(u32 *)(&get_paca()->paca_index))
 #endif
 
+static inline int is_mmio_hpte(unsigned long v, unsigned long r)
+{
+	return ((v & HPTE_V_ABSENT) &&
+		(r & (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) ==
+		(HPTE_R_KEY_HI | HPTE_R_KEY_LO));
+}
+
 static inline int try_lock_tlbie(unsigned int *lock)
 {
 	unsigned int tmp, old;
@@ -409,13 +424,18 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,
 {
 	long i;
 
+	/*
+	 * We use the POWER9 5-operand versions of tlbie and tlbiel here.
+	 * Since we are using RIC=0 PRS=0 R=0, and P7/P8 tlbiel ignores
+	 * the RS field, this is backwards-compatible with P7 and P8.
+	 */
 	if (global) {
 		while (!try_lock_tlbie(&kvm->arch.tlbie_lock))
 			cpu_relax();
 		if (need_sync)
 			asm volatile("ptesync" : : : "memory");
 		for (i = 0; i < npages; ++i)
-			asm volatile(PPC_TLBIE(%1,%0) : :
+			asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : :
 				     "r" (rbvalues[i]), "r" (kvm->arch.lpid));
 		asm volatile("eieio; tlbsync; ptesync" : : : "memory");
 		kvm->arch.tlbie_lock = 0;
@@ -423,7 +443,8 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,
 		if (need_sync)
 			asm volatile("ptesync" : : : "memory");
 		for (i = 0; i < npages; ++i)
-			asm volatile("tlbiel %0" : : "r" (rbvalues[i]));
+			asm volatile(PPC_TLBIEL(%0,%1,0,0,0) : :
+				     "r" (rbvalues[i]), "r" (0));
 		asm volatile("ptesync" : : : "memory");
 	}
 }
@@ -435,18 +456,23 @@ long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags,
 	__be64 *hpte;
 	unsigned long v, r, rb;
 	struct revmap_entry *rev;
-	u64 pte;
+	u64 pte, orig_pte, pte_r;
 
 	if (pte_index >= kvm->arch.hpt_npte)
 		return H_PARAMETER;
 	hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));
 	while (!try_lock_hpte(hpte, HPTE_V_HVLOCK))
 		cpu_relax();
-	pte = be64_to_cpu(hpte[0]);
+	pte = orig_pte = be64_to_cpu(hpte[0]);
+	pte_r = be64_to_cpu(hpte[1]);
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		pte = hpte_new_to_old_v(pte, pte_r);
+		pte_r = hpte_new_to_old_r(pte_r);
+	}
 	if ((pte & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 ||
 	    ((flags & H_AVPN) && (pte & ~0x7fUL) != avpn) ||
 	    ((flags & H_ANDCOND) && (pte & avpn) != 0)) {
-		__unlock_hpte(hpte, pte);
+		__unlock_hpte(hpte, orig_pte);
 		return H_NOT_FOUND;
 	}
 
@@ -454,7 +480,7 @@ long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags,
 	v = pte & ~HPTE_V_HVLOCK;
 	if (v & HPTE_V_VALID) {
 		hpte[0] &= ~cpu_to_be64(HPTE_V_VALID);
-		rb = compute_tlbie_rb(v, be64_to_cpu(hpte[1]), pte_index);
+		rb = compute_tlbie_rb(v, pte_r, pte_index);
 		do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true);
 		/*
 		 * The reference (R) and change (C) bits in a HPT
@@ -472,6 +498,9 @@ long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags,
 	note_hpte_modification(kvm, rev);
 	unlock_hpte(hpte, 0);
 
+	if (is_mmio_hpte(v, pte_r))
+		atomic64_inc(&kvm->arch.mmio_update);
+
 	if (v & HPTE_V_ABSENT)
 		v = (v & ~HPTE_V_ABSENT) | HPTE_V_VALID;
 	hpret[0] = v;
@@ -498,7 +527,7 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
 	int global;
 	long int ret = H_SUCCESS;
 	struct revmap_entry *rev, *revs[4];
-	u64 hp0;
+	u64 hp0, hp1;
 
 	global = global_invalidates(kvm, 0);
 	for (i = 0; i < 4 && ret == H_SUCCESS; ) {
@@ -531,6 +560,11 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
 			}
 			found = 0;
 			hp0 = be64_to_cpu(hp[0]);
+			hp1 = be64_to_cpu(hp[1]);
+			if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+				hp0 = hpte_new_to_old_v(hp0, hp1);
+				hp1 = hpte_new_to_old_r(hp1);
+			}
 			if (hp0 & (HPTE_V_ABSENT | HPTE_V_VALID)) {
 				switch (flags & 3) {
 				case 0:		/* absolute */
@@ -561,13 +595,14 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
 				rcbits = rev->guest_rpte & (HPTE_R_R|HPTE_R_C);
 				args[j] |= rcbits << (56 - 5);
 				hp[0] = 0;
+				if (is_mmio_hpte(hp0, hp1))
+					atomic64_inc(&kvm->arch.mmio_update);
 				continue;
 			}
 
 			/* leave it locked */
 			hp[0] &= ~cpu_to_be64(HPTE_V_VALID);
-			tlbrb[n] = compute_tlbie_rb(be64_to_cpu(hp[0]),
-				be64_to_cpu(hp[1]), pte_index);
+			tlbrb[n] = compute_tlbie_rb(hp0, hp1, pte_index);
 			indexes[n] = j;
 			hptes[n] = hp;
 			revs[n] = rev;
@@ -605,7 +640,7 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
 	__be64 *hpte;
 	struct revmap_entry *rev;
 	unsigned long v, r, rb, mask, bits;
-	u64 pte;
+	u64 pte_v, pte_r;
 
 	if (pte_index >= kvm->arch.hpt_npte)
 		return H_PARAMETER;
@@ -613,14 +648,16 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
 	hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));
 	while (!try_lock_hpte(hpte, HPTE_V_HVLOCK))
 		cpu_relax();
-	pte = be64_to_cpu(hpte[0]);
-	if ((pte & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 ||
-	    ((flags & H_AVPN) && (pte & ~0x7fUL) != avpn)) {
-		__unlock_hpte(hpte, pte);
+	v = pte_v = be64_to_cpu(hpte[0]);
+	if (cpu_has_feature(CPU_FTR_ARCH_300))
+		v = hpte_new_to_old_v(v, be64_to_cpu(hpte[1]));
+	if ((v & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 ||
+	    ((flags & H_AVPN) && (v & ~0x7fUL) != avpn)) {
+		__unlock_hpte(hpte, pte_v);
 		return H_NOT_FOUND;
 	}
 
-	v = pte;
+	pte_r = be64_to_cpu(hpte[1]);
 	bits = (flags << 55) & HPTE_R_PP0;
 	bits |= (flags << 48) & HPTE_R_KEY_HI;
 	bits |= flags & (HPTE_R_PP | HPTE_R_N | HPTE_R_KEY_LO);
@@ -642,22 +679,26 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
 		 * readonly to writable.  If it should be writable, we'll
 		 * take a trap and let the page fault code sort it out.
 		 */
-		pte = be64_to_cpu(hpte[1]);
-		r = (pte & ~mask) | bits;
-		if (hpte_is_writable(r) && !hpte_is_writable(pte))
+		r = (pte_r & ~mask) | bits;
+		if (hpte_is_writable(r) && !hpte_is_writable(pte_r))
 			r = hpte_make_readonly(r);
 		/* If the PTE is changing, invalidate it first */
-		if (r != pte) {
+		if (r != pte_r) {
 			rb = compute_tlbie_rb(v, r, pte_index);
-			hpte[0] = cpu_to_be64((v & ~HPTE_V_VALID) |
+			hpte[0] = cpu_to_be64((pte_v & ~HPTE_V_VALID) |
 					      HPTE_V_ABSENT);
 			do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags),
 				  true);
+			/* Don't lose R/C bit updates done by hardware */
+			r |= be64_to_cpu(hpte[1]) & (HPTE_R_R | HPTE_R_C);
 			hpte[1] = cpu_to_be64(r);
 		}
 	}
-	unlock_hpte(hpte, v & ~HPTE_V_HVLOCK);
+	unlock_hpte(hpte, pte_v & ~HPTE_V_HVLOCK);
 	asm volatile("ptesync" : : : "memory");
+	if (is_mmio_hpte(v, pte_r))
+		atomic64_inc(&kvm->arch.mmio_update);
+
 	return H_SUCCESS;
 }
 
@@ -681,6 +722,10 @@ long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags,
 		hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4));
 		v = be64_to_cpu(hpte[0]) & ~HPTE_V_HVLOCK;
 		r = be64_to_cpu(hpte[1]);
+		if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+			v = hpte_new_to_old_v(v, r);
+			r = hpte_new_to_old_r(r);
+		}
 		if (v & HPTE_V_ABSENT) {
 			v &= ~HPTE_V_ABSENT;
 			v |= HPTE_V_VALID;
@@ -798,10 +843,16 @@ void kvmppc_invalidate_hpte(struct kvm *kvm, __be64 *hptep,
 			unsigned long pte_index)
 {
 	unsigned long rb;
+	u64 hp0, hp1;
 
 	hptep[0] &= ~cpu_to_be64(HPTE_V_VALID);
-	rb = compute_tlbie_rb(be64_to_cpu(hptep[0]), be64_to_cpu(hptep[1]),
-			      pte_index);
+	hp0 = be64_to_cpu(hptep[0]);
+	hp1 = be64_to_cpu(hptep[1]);
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		hp0 = hpte_new_to_old_v(hp0, hp1);
+		hp1 = hpte_new_to_old_r(hp1);
+	}
+	rb = compute_tlbie_rb(hp0, hp1, pte_index);
 	do_tlbies(kvm, &rb, 1, 1, true);
 }
 EXPORT_SYMBOL_GPL(kvmppc_invalidate_hpte);
@@ -811,9 +862,15 @@ void kvmppc_clear_ref_hpte(struct kvm *kvm, __be64 *hptep,
 {
 	unsigned long rb;
 	unsigned char rbyte;
+	u64 hp0, hp1;
 
-	rb = compute_tlbie_rb(be64_to_cpu(hptep[0]), be64_to_cpu(hptep[1]),
-			      pte_index);
+	hp0 = be64_to_cpu(hptep[0]);
+	hp1 = be64_to_cpu(hptep[1]);
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		hp0 = hpte_new_to_old_v(hp0, hp1);
+		hp1 = hpte_new_to_old_r(hp1);
+	}
+	rb = compute_tlbie_rb(hp0, hp1, pte_index);
 	rbyte = (be64_to_cpu(hptep[1]) & ~HPTE_R_R) >> 8;
 	/* modify only the second-last byte, which contains the ref bit */
 	*((char *)hptep + 14) = rbyte;
@@ -828,6 +885,37 @@ static int slb_base_page_shift[4] = {
 	20,	/* 1M, unsupported */
 };
 
+static struct mmio_hpte_cache_entry *mmio_cache_search(struct kvm_vcpu *vcpu,
+		unsigned long eaddr, unsigned long slb_v, long mmio_update)
+{
+	struct mmio_hpte_cache_entry *entry = NULL;
+	unsigned int pshift;
+	unsigned int i;
+
+	for (i = 0; i < MMIO_HPTE_CACHE_SIZE; i++) {
+		entry = &vcpu->arch.mmio_cache.entry[i];
+		if (entry->mmio_update == mmio_update) {
+			pshift = entry->slb_base_pshift;
+			if ((entry->eaddr >> pshift) == (eaddr >> pshift) &&
+			    entry->slb_v == slb_v)
+				return entry;
+		}
+	}
+	return NULL;
+}
+
+static struct mmio_hpte_cache_entry *
+			next_mmio_cache_entry(struct kvm_vcpu *vcpu)
+{
+	unsigned int index = vcpu->arch.mmio_cache.index;
+
+	vcpu->arch.mmio_cache.index++;
+	if (vcpu->arch.mmio_cache.index == MMIO_HPTE_CACHE_SIZE)
+		vcpu->arch.mmio_cache.index = 0;
+
+	return &vcpu->arch.mmio_cache.entry[index];
+}
+
 /* When called from virtmode, this func should be protected by
  * preempt_disable(), otherwise, the holding of HPTE_V_HVLOCK
  * can trigger deadlock issue.
@@ -842,7 +930,7 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,
 	unsigned long avpn;
 	__be64 *hpte;
 	unsigned long mask, val;
-	unsigned long v, r;
+	unsigned long v, r, orig_v;
 
 	/* Get page shift, work out hash and AVPN etc. */
 	mask = SLB_VSID_B | HPTE_V_AVPN | HPTE_V_SECONDARY;
@@ -877,6 +965,8 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,
 		for (i = 0; i < 16; i += 2) {
 			/* Read the PTE racily */
 			v = be64_to_cpu(hpte[i]) & ~HPTE_V_HVLOCK;
+			if (cpu_has_feature(CPU_FTR_ARCH_300))
+				v = hpte_new_to_old_v(v, be64_to_cpu(hpte[i+1]));
 
 			/* Check valid/absent, hash, segment size and AVPN */
 			if (!(v & valid) || (v & mask) != val)
@@ -885,8 +975,12 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,
 			/* Lock the PTE and read it under the lock */
 			while (!try_lock_hpte(&hpte[i], HPTE_V_HVLOCK))
 				cpu_relax();
-			v = be64_to_cpu(hpte[i]) & ~HPTE_V_HVLOCK;
+			v = orig_v = be64_to_cpu(hpte[i]) & ~HPTE_V_HVLOCK;
 			r = be64_to_cpu(hpte[i+1]);
+			if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+				v = hpte_new_to_old_v(v, r);
+				r = hpte_new_to_old_r(r);
+			}
 
 			/*
 			 * Check the HPTE again, including base page size
@@ -896,7 +990,7 @@ long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,
 				/* Return with the HPTE still locked */
 				return (hash << 3) + (i >> 1);
 
-			__unlock_hpte(&hpte[i], v);
+			__unlock_hpte(&hpte[i], orig_v);
 		}
 
 		if (val & HPTE_V_SECONDARY)
@@ -924,30 +1018,45 @@ long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr,
 {
 	struct kvm *kvm = vcpu->kvm;
 	long int index;
-	unsigned long v, r, gr;
+	unsigned long v, r, gr, orig_v;
 	__be64 *hpte;
 	unsigned long valid;
 	struct revmap_entry *rev;
 	unsigned long pp, key;
+	struct mmio_hpte_cache_entry *cache_entry = NULL;
+	long mmio_update = 0;
 
 	/* For protection fault, expect to find a valid HPTE */
 	valid = HPTE_V_VALID;
-	if (status & DSISR_NOHPTE)
+	if (status & DSISR_NOHPTE) {
 		valid |= HPTE_V_ABSENT;
-
-	index = kvmppc_hv_find_lock_hpte(kvm, addr, slb_v, valid);
-	if (index < 0) {
-		if (status & DSISR_NOHPTE)
-			return status;	/* there really was no HPTE */
-		return 0;		/* for prot fault, HPTE disappeared */
+		mmio_update = atomic64_read(&kvm->arch.mmio_update);
+		cache_entry = mmio_cache_search(vcpu, addr, slb_v, mmio_update);
 	}
-	hpte = (__be64 *)(kvm->arch.hpt_virt + (index << 4));
-	v = be64_to_cpu(hpte[0]) & ~HPTE_V_HVLOCK;
-	r = be64_to_cpu(hpte[1]);
-	rev = real_vmalloc_addr(&kvm->arch.revmap[index]);
-	gr = rev->guest_rpte;
+	if (cache_entry) {
+		index = cache_entry->pte_index;
+		v = cache_entry->hpte_v;
+		r = cache_entry->hpte_r;
+		gr = cache_entry->rpte;
+	} else {
+		index = kvmppc_hv_find_lock_hpte(kvm, addr, slb_v, valid);
+		if (index < 0) {
+			if (status & DSISR_NOHPTE)
+				return status;	/* there really was no HPTE */
+			return 0;	/* for prot fault, HPTE disappeared */
+		}
+		hpte = (__be64 *)(kvm->arch.hpt_virt + (index << 4));
+		v = orig_v = be64_to_cpu(hpte[0]) & ~HPTE_V_HVLOCK;
+		r = be64_to_cpu(hpte[1]);
+		if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+			v = hpte_new_to_old_v(v, r);
+			r = hpte_new_to_old_r(r);
+		}
+		rev = real_vmalloc_addr(&kvm->arch.revmap[index]);
+		gr = rev->guest_rpte;
 
-	unlock_hpte(hpte, v);
+		unlock_hpte(hpte, orig_v);
+	}
 
 	/* For not found, if the HPTE is valid by now, retry the instruction */
 	if ((status & DSISR_NOHPTE) && (v & HPTE_V_VALID))
@@ -985,12 +1094,32 @@ long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr,
 	vcpu->arch.pgfault_index = index;
 	vcpu->arch.pgfault_hpte[0] = v;
 	vcpu->arch.pgfault_hpte[1] = r;
+	vcpu->arch.pgfault_cache = cache_entry;
 
 	/* Check the storage key to see if it is possibly emulated MMIO */
-	if (data && (vcpu->arch.shregs.msr & MSR_IR) &&
-	    (r & (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) ==
-	    (HPTE_R_KEY_HI | HPTE_R_KEY_LO))
-		return -2;	/* MMIO emulation - load instr word */
+	if ((r & (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) ==
+	    (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) {
+		if (!cache_entry) {
+			unsigned int pshift = 12;
+			unsigned int pshift_index;
+
+			if (slb_v & SLB_VSID_L) {
+				pshift_index = ((slb_v & SLB_VSID_LP) >> 4);
+				pshift = slb_base_page_shift[pshift_index];
+			}
+			cache_entry = next_mmio_cache_entry(vcpu);
+			cache_entry->eaddr = addr;
+			cache_entry->slb_base_pshift = pshift;
+			cache_entry->pte_index = index;
+			cache_entry->hpte_v = v;
+			cache_entry->hpte_r = r;
+			cache_entry->rpte = gr;
+			cache_entry->slb_v = slb_v;
+			cache_entry->mmio_update = mmio_update;
+		}
+		if (data && (vcpu->arch.shregs.msr & MSR_IR))
+			return -2;	/* MMIO emulation - load instr word */
+	}
 
 	return -1;		/* send fault up to host kernel mode */
 }
diff --git a/arch/powerpc/kvm/book3s_hv_rm_xics.c b/arch/powerpc/kvm/book3s_hv_rm_xics.c
index a0ea63ac2b52..06edc4366639 100644
--- a/arch/powerpc/kvm/book3s_hv_rm_xics.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_xics.c
@@ -70,7 +70,11 @@ static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu)
 	hcpu = hcore << threads_shift;
 	kvmppc_host_rm_ops_hv->rm_core[hcore].rm_data = vcpu;
 	smp_muxed_ipi_set_message(hcpu, PPC_MSG_RM_HOST_ACTION);
-	icp_native_cause_ipi_rm(hcpu);
+	if (paca[hcpu].kvm_hstate.xics_phys)
+		icp_native_cause_ipi_rm(hcpu);
+	else
+		opal_rm_int_set_mfrr(get_hard_smp_processor_id(hcpu),
+				     IPI_PRIORITY);
 }
 #else
 static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu) { }
@@ -737,7 +741,7 @@ int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr)
 
 unsigned long eoi_rc;
 
-static void icp_eoi(struct irq_chip *c, u32 hwirq, u32 xirr)
+static void icp_eoi(struct irq_chip *c, u32 hwirq, __be32 xirr, bool *again)
 {
 	unsigned long xics_phys;
 	int64_t rc;
@@ -751,7 +755,12 @@ static void icp_eoi(struct irq_chip *c, u32 hwirq, u32 xirr)
 
 	/* EOI it */
 	xics_phys = local_paca->kvm_hstate.xics_phys;
-	_stwcix(xics_phys + XICS_XIRR, xirr);
+	if (xics_phys) {
+		_stwcix(xics_phys + XICS_XIRR, xirr);
+	} else {
+		rc = opal_rm_int_eoi(be32_to_cpu(xirr));
+		*again = rc > 0;
+	}
 }
 
 static int xics_opal_rm_set_server(unsigned int hw_irq, int server_cpu)
@@ -809,9 +818,10 @@ static void kvmppc_rm_handle_irq_desc(struct irq_desc *desc)
 }
 
 long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu,
-				 u32 xirr,
+				 __be32 xirr,
 				 struct kvmppc_irq_map *irq_map,
-				 struct kvmppc_passthru_irqmap *pimap)
+				 struct kvmppc_passthru_irqmap *pimap,
+				 bool *again)
 {
 	struct kvmppc_xics *xics;
 	struct kvmppc_icp *icp;
@@ -825,7 +835,8 @@ long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu,
 	icp_rm_deliver_irq(xics, icp, irq);
 
 	/* EOI the interrupt */
-	icp_eoi(irq_desc_get_chip(irq_map->desc), irq_map->r_hwirq, xirr);
+	icp_eoi(irq_desc_get_chip(irq_map->desc), irq_map->r_hwirq, xirr,
+		again);
 
 	if (check_too_hard(xics, icp) == H_TOO_HARD)
 		return 2;
diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
index c3c1d1bcfc67..9338a818e05c 100644
--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -501,17 +501,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
 	cmpwi	r0, 0
 	beq	57f
 	li	r3, (LPCR_PECEDH | LPCR_PECE0) >> 4
-	mfspr	r4, SPRN_LPCR
-	rlwimi	r4, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
-	mtspr	SPRN_LPCR, r4
-	isync
-	std	r0, HSTATE_SCRATCH0(r13)
-	ptesync
-	ld	r0, HSTATE_SCRATCH0(r13)
-1:	cmpd	r0, r0
-	bne	1b
-	nap
-	b	.
+	mfspr	r5, SPRN_LPCR
+	rlwimi	r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
+	b	kvm_nap_sequence
 
 57:	li	r0, 0
 	stbx	r0, r3, r4
@@ -523,6 +515,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  *                                                                            *
  *****************************************************************************/
 
+/* Stack frame offsets */
+#define STACK_SLOT_TID		(112-16)
+#define STACK_SLOT_PSSCR	(112-24)
+
 .global kvmppc_hv_entry
 kvmppc_hv_entry:
 
@@ -581,12 +577,14 @@ kvmppc_hv_entry:
 	ld	r9,VCORE_KVM(r5)	/* pointer to struct kvm */
 	cmpwi	r6,0
 	bne	10f
-	ld	r6,KVM_SDR1(r9)
 	lwz	r7,KVM_LPID(r9)
+BEGIN_FTR_SECTION
+	ld	r6,KVM_SDR1(r9)
 	li	r0,LPID_RSVD		/* switch to reserved LPID */
 	mtspr	SPRN_LPID,r0
 	ptesync
 	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
+END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
 	mtspr	SPRN_LPID,r7
 	isync
 
@@ -607,12 +605,8 @@ kvmppc_hv_entry:
 	stdcx.	r7,0,r6
 	bne	23b
 	/* Flush the TLB of any entries for this LPID */
-	/* use arch 2.07S as a proxy for POWER8 */
-BEGIN_FTR_SECTION
-	li	r6,512			/* POWER8 has 512 sets */
-FTR_SECTION_ELSE
-	li	r6,128			/* POWER7 has 128 sets */
-ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
+	lwz	r6,KVM_TLB_SETS(r9)
+	li	r0,0			/* RS for P9 version of tlbiel */
 	mtctr	r6
 	li	r7,0x800		/* IS field = 0b10 */
 	ptesync
@@ -698,6 +692,14 @@ kvmppc_got_guest:
 	mtspr	SPRN_PURR,r7
 	mtspr	SPRN_SPURR,r8
 
+	/* Save host values of some registers */
+BEGIN_FTR_SECTION
+	mfspr	r5, SPRN_TIDR
+	mfspr	r6, SPRN_PSSCR
+	std	r5, STACK_SLOT_TID(r1)
+	std	r6, STACK_SLOT_PSSCR(r1)
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
+
 BEGIN_FTR_SECTION
 	/* Set partition DABR */
 	/* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
@@ -750,14 +752,16 @@ END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
 BEGIN_FTR_SECTION
 	ld	r5, VCPU_MMCR + 24(r4)
 	ld	r6, VCPU_SIER(r4)
+	mtspr	SPRN_MMCR2, r5
+	mtspr	SPRN_SIER, r6
+BEGIN_FTR_SECTION_NESTED(96)
 	lwz	r7, VCPU_PMC + 24(r4)
 	lwz	r8, VCPU_PMC + 28(r4)
 	ld	r9, VCPU_MMCR + 32(r4)
-	mtspr	SPRN_MMCR2, r5
-	mtspr	SPRN_SIER, r6
 	mtspr	SPRN_SPMC1, r7
 	mtspr	SPRN_SPMC2, r8
 	mtspr	SPRN_MMCRS, r9
+END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96)
 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
 	mtspr	SPRN_MMCR0, r3
 	isync
@@ -813,20 +817,30 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
 	mtspr	SPRN_EBBHR, r8
 	ld	r5, VCPU_EBBRR(r4)
 	ld	r6, VCPU_BESCR(r4)
-	ld	r7, VCPU_CSIGR(r4)
-	ld	r8, VCPU_TACR(r4)
+	lwz	r7, VCPU_GUEST_PID(r4)
+	ld	r8, VCPU_WORT(r4)
 	mtspr	SPRN_EBBRR, r5
 	mtspr	SPRN_BESCR, r6
-	mtspr	SPRN_CSIGR, r7
-	mtspr	SPRN_TACR, r8
+	mtspr	SPRN_PID, r7
+	mtspr	SPRN_WORT, r8
+BEGIN_FTR_SECTION
+	/* POWER8-only registers */
 	ld	r5, VCPU_TCSCR(r4)
 	ld	r6, VCPU_ACOP(r4)
-	lwz	r7, VCPU_GUEST_PID(r4)
-	ld	r8, VCPU_WORT(r4)
+	ld	r7, VCPU_CSIGR(r4)
+	ld	r8, VCPU_TACR(r4)
 	mtspr	SPRN_TCSCR, r5
 	mtspr	SPRN_ACOP, r6
-	mtspr	SPRN_PID, r7
-	mtspr	SPRN_WORT, r8
+	mtspr	SPRN_CSIGR, r7
+	mtspr	SPRN_TACR, r8
+FTR_SECTION_ELSE
+	/* POWER9-only registers */
+	ld	r5, VCPU_TID(r4)
+	ld	r6, VCPU_PSSCR(r4)
+	oris	r6, r6, PSSCR_EC@h	/* This makes stop trap to HV */
+	mtspr	SPRN_TIDR, r5
+	mtspr	SPRN_PSSCR, r6
+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
 8:
 
 	/*
@@ -1341,20 +1355,29 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
 	std	r8, VCPU_EBBHR(r9)
 	mfspr	r5, SPRN_EBBRR
 	mfspr	r6, SPRN_BESCR
-	mfspr	r7, SPRN_CSIGR
-	mfspr	r8, SPRN_TACR
+	mfspr	r7, SPRN_PID
+	mfspr	r8, SPRN_WORT
 	std	r5, VCPU_EBBRR(r9)
 	std	r6, VCPU_BESCR(r9)
-	std	r7, VCPU_CSIGR(r9)
-	std	r8, VCPU_TACR(r9)
+	stw	r7, VCPU_GUEST_PID(r9)
+	std	r8, VCPU_WORT(r9)
+BEGIN_FTR_SECTION
 	mfspr	r5, SPRN_TCSCR
 	mfspr	r6, SPRN_ACOP
-	mfspr	r7, SPRN_PID
-	mfspr	r8, SPRN_WORT
+	mfspr	r7, SPRN_CSIGR
+	mfspr	r8, SPRN_TACR
 	std	r5, VCPU_TCSCR(r9)
 	std	r6, VCPU_ACOP(r9)
-	stw	r7, VCPU_GUEST_PID(r9)
-	std	r8, VCPU_WORT(r9)
+	std	r7, VCPU_CSIGR(r9)
+	std	r8, VCPU_TACR(r9)
+FTR_SECTION_ELSE
+	mfspr	r5, SPRN_TIDR
+	mfspr	r6, SPRN_PSSCR
+	std	r5, VCPU_TID(r9)
+	rldicl	r6, r6, 4, 50		/* r6 &= PSSCR_GUEST_VIS */
+	rotldi	r6, r6, 60
+	std	r6, VCPU_PSSCR(r9)
+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
 	/*
 	 * Restore various registers to 0, where non-zero values
 	 * set by the guest could disrupt the host.
@@ -1363,12 +1386,14 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
 	mtspr	SPRN_IAMR, r0
 	mtspr	SPRN_CIABR, r0
 	mtspr	SPRN_DAWRX, r0
-	mtspr	SPRN_TCSCR, r0
 	mtspr	SPRN_WORT, r0
+BEGIN_FTR_SECTION
+	mtspr	SPRN_TCSCR, r0
 	/* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */
 	li	r0, 1
 	sldi	r0, r0, 31
 	mtspr	SPRN_MMCRS, r0
+END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
 8:
 
 	/* Save and reset AMR and UAMOR before turning on the MMU */
@@ -1502,15 +1527,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
 	stw	r8, VCPU_PMC + 20(r9)
 BEGIN_FTR_SECTION
 	mfspr	r5, SPRN_SIER
+	std	r5, VCPU_SIER(r9)
+BEGIN_FTR_SECTION_NESTED(96)
 	mfspr	r6, SPRN_SPMC1
 	mfspr	r7, SPRN_SPMC2
 	mfspr	r8, SPRN_MMCRS
-	std	r5, VCPU_SIER(r9)
 	stw	r6, VCPU_PMC + 24(r9)
 	stw	r7, VCPU_PMC + 28(r9)
 	std	r8, VCPU_MMCR + 32(r9)
 	lis	r4, 0x8000
 	mtspr	SPRN_MMCRS, r4
+END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96)
 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
 22:
 	/* Clear out SLB */
@@ -1519,6 +1546,14 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
 	slbia
 	ptesync
 
+	/* Restore host values of some registers */
+BEGIN_FTR_SECTION
+	ld	r5, STACK_SLOT_TID(r1)
+	ld	r6, STACK_SLOT_PSSCR(r1)
+	mtspr	SPRN_TIDR, r5
+	mtspr	SPRN_PSSCR, r6
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
+
 	/*
 	 * POWER7/POWER8 guest -> host partition switch code.
 	 * We don't have to lock against tlbies but we do
@@ -1552,12 +1587,14 @@ kvmhv_switch_to_host:
 	beq	19f
 
 	/* Primary thread switches back to host partition */
-	ld	r6,KVM_HOST_SDR1(r4)
 	lwz	r7,KVM_HOST_LPID(r4)
+BEGIN_FTR_SECTION
+	ld	r6,KVM_HOST_SDR1(r4)
 	li	r8,LPID_RSVD		/* switch to reserved LPID */
 	mtspr	SPRN_LPID,r8
 	ptesync
-	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
+	mtspr	SPRN_SDR1,r6		/* switch to host page table */
+END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
 	mtspr	SPRN_LPID,r7
 	isync
 
@@ -2211,6 +2248,21 @@ BEGIN_FTR_SECTION
 	ori	r5, r5, LPCR_PECEDH
 	rlwimi	r5, r3, 0, LPCR_PECEDP
 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
+
+kvm_nap_sequence:		/* desired LPCR value in r5 */
+BEGIN_FTR_SECTION
+	/*
+	 * PSSCR bits:	exit criterion = 1 (wakeup based on LPCR at sreset)
+	 *		enable state loss = 1 (allow SMT mode switch)
+	 *		requested level = 0 (just stop dispatching)
+	 */
+	lis	r3, (PSSCR_EC | PSSCR_ESL)@h
+	mtspr	SPRN_PSSCR, r3
+	/* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */
+	li	r4, LPCR_PECE_HVEE@higher
+	sldi	r4, r4, 32
+	or	r5, r5, r4
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 	mtspr	SPRN_LPCR,r5
 	isync
 	li	r0, 0
@@ -2219,7 +2271,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
 	ld	r0, HSTATE_SCRATCH0(r13)
 1:	cmpd	r0, r0
 	bne	1b
+BEGIN_FTR_SECTION
 	nap
+FTR_SECTION_ELSE
+	PPC_STOP
+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
 	b	.
 
 33:	mr	r4, r3
@@ -2600,11 +2656,13 @@ kvmppc_save_tm:
 	mfctr	r7
 	mfspr	r8, SPRN_AMR
 	mfspr	r10, SPRN_TAR
+	mfxer	r11
 	std	r5, VCPU_LR_TM(r9)
 	stw	r6, VCPU_CR_TM(r9)
 	std	r7, VCPU_CTR_TM(r9)
 	std	r8, VCPU_AMR_TM(r9)
 	std	r10, VCPU_TAR_TM(r9)
+	std	r11, VCPU_XER_TM(r9)
 
 	/* Restore r12 as trap number. */
 	lwz	r12, VCPU_TRAP(r9)
@@ -2697,11 +2755,13 @@ kvmppc_restore_tm:
 	ld	r7, VCPU_CTR_TM(r4)
 	ld	r8, VCPU_AMR_TM(r4)
 	ld	r9, VCPU_TAR_TM(r4)
+	ld	r10, VCPU_XER_TM(r4)
 	mtlr	r5
 	mtcr	r6
 	mtctr	r7
 	mtspr	SPRN_AMR, r8
 	mtspr	SPRN_TAR, r9
+	mtxer	r10
 
 	/*
 	 * Load up PPR and DSCR values but don't put them in the actual SPRs
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 70963c845e96..efd1183a6b16 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -536,7 +536,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 #ifdef CONFIG_PPC_BOOK3S_64
 	case KVM_CAP_SPAPR_TCE:
 	case KVM_CAP_SPAPR_TCE_64:
-	case KVM_CAP_PPC_ALLOC_HTAB:
 	case KVM_CAP_PPC_RTAS:
 	case KVM_CAP_PPC_FIXUP_HCALL:
 	case KVM_CAP_PPC_ENABLE_HCALL:
@@ -545,13 +544,20 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 #endif
 		r = 1;
 		break;
+
+	case KVM_CAP_PPC_ALLOC_HTAB:
+		r = hv_enabled;
+		break;
 #endif /* CONFIG_PPC_BOOK3S_64 */
 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 	case KVM_CAP_PPC_SMT:
-		if (hv_enabled)
-			r = threads_per_subcore;
-		else
-			r = 0;
+		r = 0;
+		if (hv_enabled) {
+			if (cpu_has_feature(CPU_FTR_ARCH_300))
+				r = 1;
+			else
+				r = threads_per_subcore;
+		}
 		break;
 	case KVM_CAP_PPC_RMA:
 		r = 0;
diff --git a/arch/powerpc/kvm/trace_hv.h b/arch/powerpc/kvm/trace_hv.h
index fb21990c0fb4..ebc6dd449556 100644
--- a/arch/powerpc/kvm/trace_hv.h
+++ b/arch/powerpc/kvm/trace_hv.h
@@ -449,7 +449,7 @@ TRACE_EVENT(kvmppc_vcore_wakeup,
 		__entry->tgid   = current->tgid;
 	),
 
-	TP_printk("%s time %lld ns, tgid=%d",
+	TP_printk("%s time %llu ns, tgid=%d",
 		__entry->waited ? "wait" : "poll",
 		__entry->ns, __entry->tgid)
 );
diff --git a/arch/powerpc/lib/checksum_32.S b/arch/powerpc/lib/checksum_32.S
index ea29a5d67743..9a671c774b22 100644
--- a/arch/powerpc/lib/checksum_32.S
+++ b/arch/powerpc/lib/checksum_32.S
@@ -103,17 +103,14 @@ EXPORT_SYMBOL(__csum_partial)
 	adde	r12,r12,r10
 
 #define CSUM_COPY_16_BYTES_EXCODE(n)		\
-.section __ex_table,"a";		\
-	.align	2;			\
-	.long	8 ## n ## 0b,src_error;	\
-	.long	8 ## n ## 1b,src_error;	\
-	.long	8 ## n ## 2b,src_error;	\
-	.long	8 ## n ## 3b,src_error;	\
-	.long	8 ## n ## 4b,dst_error;	\
-	.long	8 ## n ## 5b,dst_error;	\
-	.long	8 ## n ## 6b,dst_error;	\
-	.long	8 ## n ## 7b,dst_error;	\
-	.text
+	EX_TABLE(8 ## n ## 0b, src_error);	\
+	EX_TABLE(8 ## n ## 1b, src_error);	\
+	EX_TABLE(8 ## n ## 2b, src_error);	\
+	EX_TABLE(8 ## n ## 3b, src_error);	\
+	EX_TABLE(8 ## n ## 4b, dst_error);	\
+	EX_TABLE(8 ## n ## 5b, dst_error);	\
+	EX_TABLE(8 ## n ## 6b, dst_error);	\
+	EX_TABLE(8 ## n ## 7b, dst_error);
 
 	.text
 	.stabs	"arch/powerpc/lib/",N_SO,0,0,0f
@@ -263,14 +260,11 @@ dst_error:
 	stw	r0,0(r8)
 	blr
 
-	.section __ex_table,"a"
-	.align	2
-	.long	70b,src_error
-	.long	71b,dst_error
-	.long	72b,src_error
-	.long	73b,dst_error
-	.long	54b,dst_error
-	.text
+	EX_TABLE(70b, src_error);
+	EX_TABLE(71b, dst_error);
+	EX_TABLE(72b, src_error);
+	EX_TABLE(73b, dst_error);
+	EX_TABLE(54b, dst_error);
 
 /*
  * this stuff handles faults in the cacheline loop and branches to either
@@ -291,12 +285,11 @@ dst_error:
 #endif
 #endif
 
-	.section __ex_table,"a"
-	.align	2
-	.long	30b,src_error
-	.long	31b,dst_error
-	.long	40b,src_error
-	.long	41b,dst_error
-	.long	50b,src_error
-	.long	51b,dst_error
+	EX_TABLE(30b, src_error);
+	EX_TABLE(31b, dst_error);
+	EX_TABLE(40b, src_error);
+	EX_TABLE(41b, dst_error);
+	EX_TABLE(50b, src_error);
+	EX_TABLE(51b, dst_error);
+
 EXPORT_SYMBOL(csum_partial_copy_generic)
diff --git a/arch/powerpc/lib/checksum_64.S b/arch/powerpc/lib/checksum_64.S
index fd9176671f9f..d0d311e108ff 100644
--- a/arch/powerpc/lib/checksum_64.S
+++ b/arch/powerpc/lib/checksum_64.S
@@ -182,34 +182,22 @@ EXPORT_SYMBOL(__csum_partial)
 
 	.macro srcnr
 100:
-	.section __ex_table,"a"
-	.align 3
-	.llong 100b,.Lsrc_error_nr
-	.previous
+	EX_TABLE(100b,.Lsrc_error_nr)
 	.endm
 
 	.macro source
 150:
-	.section __ex_table,"a"
-	.align 3
-	.llong 150b,.Lsrc_error
-	.previous
+	EX_TABLE(150b,.Lsrc_error)
 	.endm
 
 	.macro dstnr
 200:
-	.section __ex_table,"a"
-	.align 3
-	.llong 200b,.Ldest_error_nr
-	.previous
+	EX_TABLE(200b,.Ldest_error_nr)
 	.endm
 
 	.macro dest
 250:
-	.section __ex_table,"a"
-	.align 3
-	.llong 250b,.Ldest_error
-	.previous
+	EX_TABLE(250b,.Ldest_error)
 	.endm
 
 /*
diff --git a/arch/powerpc/lib/copy_32.S b/arch/powerpc/lib/copy_32.S
index 40cce33b08d6..ff0d894d7ff9 100644
--- a/arch/powerpc/lib/copy_32.S
+++ b/arch/powerpc/lib/copy_32.S
@@ -49,17 +49,14 @@
 9 ## n ## 1:					\
 	addi	r5,r5,-(16 * n);		\
 	b	105f;				\
-.section __ex_table,"a";			\
-	.align	2;				\
-	.long	8 ## n ## 0b,9 ## n ## 0b;	\
-	.long	8 ## n ## 1b,9 ## n ## 0b;	\
-	.long	8 ## n ## 2b,9 ## n ## 0b;	\
-	.long	8 ## n ## 3b,9 ## n ## 0b;	\
-	.long	8 ## n ## 4b,9 ## n ## 1b;	\
-	.long	8 ## n ## 5b,9 ## n ## 1b;	\
-	.long	8 ## n ## 6b,9 ## n ## 1b;	\
-	.long	8 ## n ## 7b,9 ## n ## 1b;	\
-	.text
+	EX_TABLE(8 ## n ## 0b,9 ## n ## 0b);	\
+	EX_TABLE(8 ## n ## 1b,9 ## n ## 0b);	\
+	EX_TABLE(8 ## n ## 2b,9 ## n ## 0b);	\
+	EX_TABLE(8 ## n ## 3b,9 ## n ## 0b);	\
+	EX_TABLE(8 ## n ## 4b,9 ## n ## 1b);	\
+	EX_TABLE(8 ## n ## 5b,9 ## n ## 1b);	\
+	EX_TABLE(8 ## n ## 6b,9 ## n ## 1b);	\
+	EX_TABLE(8 ## n ## 7b,9 ## n ## 1b)
 
 	.text
 	.stabs	"arch/powerpc/lib/",N_SO,0,0,0f
@@ -323,13 +320,10 @@ _GLOBAL(__copy_tofrom_user)
 73:	stwu	r9,4(r6)
 	bdnz	72b
 
-	.section __ex_table,"a"
-	.align	2
-	.long	70b,100f
-	.long	71b,101f
-	.long	72b,102f
-	.long	73b,103f
-	.text
+	EX_TABLE(70b,100f)
+	EX_TABLE(71b,101f)
+	EX_TABLE(72b,102f)
+	EX_TABLE(73b,103f)
 
 58:	srwi.	r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
 	clrlwi	r5,r5,32-LG_CACHELINE_BYTES
@@ -364,10 +358,7 @@ _GLOBAL(__copy_tofrom_user)
 
 53:	dcbt	r3,r4
 54:	dcbz	r11,r6
-	.section __ex_table,"a"
-	.align	2
-	.long	54b,105f
-	.text
+	EX_TABLE(54b,105f)
 /* the main body of the cacheline loop */
 	COPY_16_BYTES_WITHEX(0)
 #if L1_CACHE_BYTES >= 32
@@ -500,15 +491,13 @@ _GLOBAL(__copy_tofrom_user)
 	bdnz	114b
 120:	blr
 
-	.section __ex_table,"a"
-	.align	2
-	.long	30b,108b
-	.long	31b,109b
-	.long	40b,110b
-	.long	41b,111b
-	.long	130b,132b
-	.long	131b,120b
-	.long	112b,120b
-	.long	114b,120b
-	.text
+	EX_TABLE(30b,108b)
+	EX_TABLE(31b,109b)
+	EX_TABLE(40b,110b)
+	EX_TABLE(41b,111b)
+	EX_TABLE(130b,132b)
+	EX_TABLE(131b,120b)
+	EX_TABLE(112b,120b)
+	EX_TABLE(114b,120b)
+
 EXPORT_SYMBOL(__copy_tofrom_user)
diff --git a/arch/powerpc/lib/copyuser_64.S b/arch/powerpc/lib/copyuser_64.S
index 60386b2c99bb..aee6e24e81ab 100644
--- a/arch/powerpc/lib/copyuser_64.S
+++ b/arch/powerpc/lib/copyuser_64.S
@@ -394,70 +394,66 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
 192:
 	blr			/* #bytes not copied in r3 */
 
-	.section __ex_table,"a"
-	.align	3
-	.llong	20b,120b
-	.llong	220b,320b
-	.llong	21b,121b
-	.llong	221b,321b
-	.llong	70b,170b
-	.llong	270b,370b
-	.llong	22b,122b
-	.llong	222b,322b
-	.llong	71b,171b
-	.llong	271b,371b
-	.llong	72b,172b
-	.llong	272b,372b
-	.llong	244b,344b
-	.llong	245b,345b
-	.llong	23b,123b
-	.llong	73b,173b
-	.llong	44b,144b
-	.llong	74b,174b
-	.llong	45b,145b
-	.llong	75b,175b
-	.llong	24b,124b
-	.llong	25b,125b
-	.llong	26b,126b
-	.llong	27b,127b
-	.llong	28b,128b
-	.llong	29b,129b
-	.llong	30b,130b
-	.llong	31b,131b
-	.llong	32b,132b
-	.llong	76b,176b
-	.llong	33b,133b
-	.llong	77b,177b
-	.llong	78b,178b
-	.llong	79b,179b
-	.llong	80b,180b
-	.llong	34b,134b
-	.llong	94b,194b
-	.llong	95b,195b
-	.llong	96b,196b
-	.llong	35b,135b
-	.llong	81b,181b
-	.llong	36b,136b
-	.llong	82b,182b
-	.llong	37b,137b
-	.llong	83b,183b
-	.llong	38b,138b
-	.llong	39b,139b
-	.llong	84b,184b
-	.llong	85b,185b
-	.llong	40b,140b
-	.llong	86b,186b
-	.llong	41b,141b
-	.llong	87b,187b
-	.llong	42b,142b
-	.llong	88b,188b
-	.llong	43b,143b
-	.llong	89b,189b
-	.llong	90b,190b
-	.llong	91b,191b
-	.llong	92b,192b
-	
-	.text
+	EX_TABLE(20b,120b)
+	EX_TABLE(220b,320b)
+	EX_TABLE(21b,121b)
+	EX_TABLE(221b,321b)
+	EX_TABLE(70b,170b)
+	EX_TABLE(270b,370b)
+	EX_TABLE(22b,122b)
+	EX_TABLE(222b,322b)
+	EX_TABLE(71b,171b)
+	EX_TABLE(271b,371b)
+	EX_TABLE(72b,172b)
+	EX_TABLE(272b,372b)
+	EX_TABLE(244b,344b)
+	EX_TABLE(245b,345b)
+	EX_TABLE(23b,123b)
+	EX_TABLE(73b,173b)
+	EX_TABLE(44b,144b)
+	EX_TABLE(74b,174b)
+	EX_TABLE(45b,145b)
+	EX_TABLE(75b,175b)
+	EX_TABLE(24b,124b)
+	EX_TABLE(25b,125b)
+	EX_TABLE(26b,126b)
+	EX_TABLE(27b,127b)
+	EX_TABLE(28b,128b)
+	EX_TABLE(29b,129b)
+	EX_TABLE(30b,130b)
+	EX_TABLE(31b,131b)
+	EX_TABLE(32b,132b)
+	EX_TABLE(76b,176b)
+	EX_TABLE(33b,133b)
+	EX_TABLE(77b,177b)
+	EX_TABLE(78b,178b)
+	EX_TABLE(79b,179b)
+	EX_TABLE(80b,180b)
+	EX_TABLE(34b,134b)
+	EX_TABLE(94b,194b)
+	EX_TABLE(95b,195b)
+	EX_TABLE(96b,196b)
+	EX_TABLE(35b,135b)
+	EX_TABLE(81b,181b)
+	EX_TABLE(36b,136b)
+	EX_TABLE(82b,182b)
+	EX_TABLE(37b,137b)
+	EX_TABLE(83b,183b)
+	EX_TABLE(38b,138b)
+	EX_TABLE(39b,139b)
+	EX_TABLE(84b,184b)
+	EX_TABLE(85b,185b)
+	EX_TABLE(40b,140b)
+	EX_TABLE(86b,186b)
+	EX_TABLE(41b,141b)
+	EX_TABLE(87b,187b)
+	EX_TABLE(42b,142b)
+	EX_TABLE(88b,188b)
+	EX_TABLE(43b,143b)
+	EX_TABLE(89b,189b)
+	EX_TABLE(90b,190b)
+	EX_TABLE(91b,191b)
+	EX_TABLE(92b,192b)
 
 /*
  * Routine to copy a whole page of data, optimized for POWER4.
@@ -598,78 +594,77 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
 	li	r5,4096
 	b	.Ldst_aligned
 
-	.section __ex_table,"a"
-	.align	3
-	.llong	20b,100b
-	.llong	21b,100b
-	.llong	22b,100b
-	.llong	23b,100b
-	.llong	24b,100b
-	.llong	25b,100b
-	.llong	26b,100b
-	.llong	27b,100b
-	.llong	28b,100b
-	.llong	29b,100b
-	.llong	30b,100b
-	.llong	31b,100b
-	.llong	32b,100b
-	.llong	33b,100b
-	.llong	34b,100b
-	.llong	35b,100b
-	.llong	36b,100b
-	.llong	37b,100b
-	.llong	38b,100b
-	.llong	39b,100b
-	.llong	40b,100b
-	.llong	41b,100b
-	.llong	42b,100b
-	.llong	43b,100b
-	.llong	44b,100b
-	.llong	45b,100b
-	.llong	46b,100b
-	.llong	47b,100b
-	.llong	48b,100b
-	.llong	49b,100b
-	.llong	50b,100b
-	.llong	51b,100b
-	.llong	52b,100b
-	.llong	53b,100b
-	.llong	54b,100b
-	.llong	55b,100b
-	.llong	56b,100b
-	.llong	57b,100b
-	.llong	58b,100b
-	.llong	59b,100b
-	.llong	60b,100b
-	.llong	61b,100b
-	.llong	62b,100b
-	.llong	63b,100b
-	.llong	64b,100b
-	.llong	65b,100b
-	.llong	66b,100b
-	.llong	67b,100b
-	.llong	68b,100b
-	.llong	69b,100b
-	.llong	70b,100b
-	.llong	71b,100b
-	.llong	72b,100b
-	.llong	73b,100b
-	.llong	74b,100b
-	.llong	75b,100b
-	.llong	76b,100b
-	.llong	77b,100b
-	.llong	78b,100b
-	.llong	79b,100b
-	.llong	80b,100b
-	.llong	81b,100b
-	.llong	82b,100b
-	.llong	83b,100b
-	.llong	84b,100b
-	.llong	85b,100b
-	.llong	86b,100b
-	.llong	87b,100b
-	.llong	88b,100b
-	.llong	89b,100b
-	.llong	90b,100b
-	.llong	91b,100b
+	EX_TABLE(20b,100b)
+	EX_TABLE(21b,100b)
+	EX_TABLE(22b,100b)
+	EX_TABLE(23b,100b)
+	EX_TABLE(24b,100b)
+	EX_TABLE(25b,100b)
+	EX_TABLE(26b,100b)
+	EX_TABLE(27b,100b)
+	EX_TABLE(28b,100b)
+	EX_TABLE(29b,100b)
+	EX_TABLE(30b,100b)
+	EX_TABLE(31b,100b)
+	EX_TABLE(32b,100b)
+	EX_TABLE(33b,100b)
+	EX_TABLE(34b,100b)
+	EX_TABLE(35b,100b)
+	EX_TABLE(36b,100b)
+	EX_TABLE(37b,100b)
+	EX_TABLE(38b,100b)
+	EX_TABLE(39b,100b)
+	EX_TABLE(40b,100b)
+	EX_TABLE(41b,100b)
+	EX_TABLE(42b,100b)
+	EX_TABLE(43b,100b)
+	EX_TABLE(44b,100b)
+	EX_TABLE(45b,100b)
+	EX_TABLE(46b,100b)
+	EX_TABLE(47b,100b)
+	EX_TABLE(48b,100b)
+	EX_TABLE(49b,100b)
+	EX_TABLE(50b,100b)
+	EX_TABLE(51b,100b)
+	EX_TABLE(52b,100b)
+	EX_TABLE(53b,100b)
+	EX_TABLE(54b,100b)
+	EX_TABLE(55b,100b)
+	EX_TABLE(56b,100b)
+	EX_TABLE(57b,100b)
+	EX_TABLE(58b,100b)
+	EX_TABLE(59b,100b)
+	EX_TABLE(60b,100b)
+	EX_TABLE(61b,100b)
+	EX_TABLE(62b,100b)
+	EX_TABLE(63b,100b)
+	EX_TABLE(64b,100b)
+	EX_TABLE(65b,100b)
+	EX_TABLE(66b,100b)
+	EX_TABLE(67b,100b)
+	EX_TABLE(68b,100b)
+	EX_TABLE(69b,100b)
+	EX_TABLE(70b,100b)
+	EX_TABLE(71b,100b)
+	EX_TABLE(72b,100b)
+	EX_TABLE(73b,100b)
+	EX_TABLE(74b,100b)
+	EX_TABLE(75b,100b)
+	EX_TABLE(76b,100b)
+	EX_TABLE(77b,100b)
+	EX_TABLE(78b,100b)
+	EX_TABLE(79b,100b)
+	EX_TABLE(80b,100b)
+	EX_TABLE(81b,100b)
+	EX_TABLE(82b,100b)
+	EX_TABLE(83b,100b)
+	EX_TABLE(84b,100b)
+	EX_TABLE(85b,100b)
+	EX_TABLE(86b,100b)
+	EX_TABLE(87b,100b)
+	EX_TABLE(88b,100b)
+	EX_TABLE(89b,100b)
+	EX_TABLE(90b,100b)
+	EX_TABLE(91b,100b)
+
 EXPORT_SYMBOL(__copy_tofrom_user)
diff --git a/arch/powerpc/lib/copyuser_power7.S b/arch/powerpc/lib/copyuser_power7.S
index da0c568d18c4..a24b4039352c 100644
--- a/arch/powerpc/lib/copyuser_power7.S
+++ b/arch/powerpc/lib/copyuser_power7.S
@@ -29,35 +29,23 @@
 
 	.macro err1
 100:
-	.section __ex_table,"a"
-	.align 3
-	.llong 100b,.Ldo_err1
-	.previous
+	EX_TABLE(100b,.Ldo_err1)
 	.endm
 
 	.macro err2
 200:
-	.section __ex_table,"a"
-	.align 3
-	.llong 200b,.Ldo_err2
-	.previous
+	EX_TABLE(200b,.Ldo_err2)
 	.endm
 
 #ifdef CONFIG_ALTIVEC
 	.macro err3
 300:
-	.section __ex_table,"a"
-	.align 3
-	.llong 300b,.Ldo_err3
-	.previous
+	EX_TABLE(300b,.Ldo_err3)
 	.endm
 
 	.macro err4
 400:
-	.section __ex_table,"a"
-	.align 3
-	.llong 400b,.Ldo_err4
-	.previous
+	EX_TABLE(400b,.Ldo_err4)
 	.endm
 
 
diff --git a/arch/powerpc/lib/ldstfp.S b/arch/powerpc/lib/ldstfp.S
index 5d0cdbfbe3f2..a58777c1b2cb 100644
--- a/arch/powerpc/lib/ldstfp.S
+++ b/arch/powerpc/lib/ldstfp.S
@@ -21,18 +21,12 @@
 
 #define STKFRM	(PPC_MIN_STKFRM + 16)
 
-	.macro	extab	instr,handler
-	.section __ex_table,"a"
-	PPC_LONG \instr,\handler
-	.previous
-	.endm
-
 	.macro	inst32	op
 reg = 0
 	.rept	32
 20:	\op	reg,0,r4
 	b	3f
-	extab	20b,99f
+	EX_TABLE(20b,99f)
 reg = reg + 1
 	.endr
 	.endm
@@ -100,7 +94,7 @@ _GLOBAL(do_lfs)
 	mr	r3,r9
 	addi	r1,r1,STKFRM
 	blr
-	extab	2b,3b
+	EX_TABLE(2b,3b)
 
 /* Load FP reg N from double at *p.  N is in r3, p in r4. */
 _GLOBAL(do_lfd)
@@ -127,7 +121,7 @@ _GLOBAL(do_lfd)
 	mr	r3,r9
 	addi	r1,r1,STKFRM
 	blr
-	extab	2b,3b
+	EX_TABLE(2b,3b)
 
 /* Store FP reg N to float at *p.  N is in r3, p in r4. */
 _GLOBAL(do_stfs)
@@ -154,7 +148,7 @@ _GLOBAL(do_stfs)
 	mr	r3,r9
 	addi	r1,r1,STKFRM
 	blr
-	extab	2b,3b
+	EX_TABLE(2b,3b)
 
 /* Store FP reg N to double at *p.  N is in r3, p in r4. */
 _GLOBAL(do_stfd)
@@ -181,7 +175,7 @@ _GLOBAL(do_stfd)
 	mr	r3,r9
 	addi	r1,r1,STKFRM
 	blr
-	extab	2b,3b
+	EX_TABLE(2b,3b)
 
 #ifdef CONFIG_ALTIVEC
 /* Get the contents of vrN into v0; N is in r3. */
@@ -248,7 +242,7 @@ _GLOBAL(do_lvx)
 	mr	r3,r9
 	addi	r1,r1,STKFRM
 	blr
-	extab	2b,3b
+	EX_TABLE(2b,3b)
 
 /* Store vector reg N to *p.  N is in r3, p in r4. */
 _GLOBAL(do_stvx)
@@ -276,7 +270,7 @@ _GLOBAL(do_stvx)
 	mr	r3,r9
 	addi	r1,r1,STKFRM
 	blr
-	extab	2b,3b
+	EX_TABLE(2b,3b)
 #endif /* CONFIG_ALTIVEC */
 
 #ifdef CONFIG_VSX
@@ -344,7 +338,7 @@ _GLOBAL(do_lxvd2x)
 	mr	r3,r9
 	addi	r1,r1,STKFRM
 	blr
-	extab	2b,3b
+	EX_TABLE(2b,3b)
 
 /* Store VSX reg N to vector doubleword *p.  N is in r3, p in r4. */
 _GLOBAL(do_stxvd2x)
@@ -372,7 +366,7 @@ _GLOBAL(do_stxvd2x)
 	mr	r3,r9
 	addi	r1,r1,STKFRM
 	blr
-	extab	2b,3b
+	EX_TABLE(2b,3b)
 
 #endif /* CONFIG_VSX */
 
diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
index 3362299b1859..9c78a9c102c3 100644
--- a/arch/powerpc/lib/sstep.c
+++ b/arch/powerpc/lib/sstep.c
@@ -15,6 +15,7 @@
 #include <asm/sstep.h>
 #include <asm/processor.h>
 #include <asm/uaccess.h>
+#include <asm/cpu_has_feature.h>
 #include <asm/cputable.h>
 
 extern char system_call_common[];
@@ -493,10 +494,7 @@ static int __kprobes do_vsx_store(int rn, int (*func)(int, unsigned long),
 		"3:	li	%0,%4\n"		\
 		"	b	2b\n"			\
 		".previous\n"				\
-		".section __ex_table,\"a\"\n"		\
-			PPC_LONG_ALIGN "\n"		\
-			PPC_LONG "1b,3b\n"		\
-		".previous"				\
+		EX_TABLE(1b, 3b)			\
 		: "=r" (err), "=r" (cr)			\
 		: "r" (x), "r" (addr), "i" (-EFAULT), "0" (err))
 
@@ -508,10 +506,7 @@ static int __kprobes do_vsx_store(int rn, int (*func)(int, unsigned long),
 		"3:	li	%0,%3\n"		\
 		"	b	2b\n"			\
 		".previous\n"				\
-		".section __ex_table,\"a\"\n"		\
-			PPC_LONG_ALIGN "\n"		\
-			PPC_LONG "1b,3b\n"		\
-		".previous"				\
+		EX_TABLE(1b, 3b)			\
 		: "=r" (err), "=r" (x)			\
 		: "r" (addr), "i" (-EFAULT), "0" (err))
 
@@ -523,10 +518,7 @@ static int __kprobes do_vsx_store(int rn, int (*func)(int, unsigned long),
 		"3:	li	%0,%3\n"		\
 		"	b	2b\n"			\
 		".previous\n"				\
-		".section __ex_table,\"a\"\n"		\
-			PPC_LONG_ALIGN "\n"		\
-			PPC_LONG "1b,3b\n"		\
-		".previous"				\
+		EX_TABLE(1b, 3b)			\
 		: "=r" (err)				\
 		: "r" (addr), "i" (-EFAULT), "0" (err))
 
diff --git a/arch/powerpc/lib/string.S b/arch/powerpc/lib/string.S
index d13e07603519..a787776822d8 100644
--- a/arch/powerpc/lib/string.S
+++ b/arch/powerpc/lib/string.S
@@ -13,8 +13,6 @@
 #include <asm/ppc_asm.h>
 #include <asm/export.h>
 
-	.section __ex_table,"a"
-	PPC_LONG_ALIGN
 	.text
 	
 /* This clears out any unused part of the destination buffer,
@@ -125,10 +123,9 @@ _GLOBAL(__clear_user)
 92:	mfctr	r3
 	blr
 
-	.section __ex_table,"a"
-	PPC_LONG	11b,90b
-	PPC_LONG	1b,91b
-	PPC_LONG	8b,92b
-	.text
+	EX_TABLE(11b, 90b)
+	EX_TABLE(1b, 91b)
+	EX_TABLE(8b, 92b)
+
 EXPORT_SYMBOL(__clear_user)
 #endif
diff --git a/arch/powerpc/lib/string_64.S b/arch/powerpc/lib/string_64.S
index 57ace356c949..c100f4d5d5d0 100644
--- a/arch/powerpc/lib/string_64.S
+++ b/arch/powerpc/lib/string_64.S
@@ -19,6 +19,7 @@
  */
 
 #include <asm/ppc_asm.h>
+#include <asm/linkage.h>
 #include <asm/asm-offsets.h>
 #include <asm/export.h>
 
@@ -41,26 +42,17 @@ PPC64_CACHES:
 
 	.macro err1
 100:
-	.section __ex_table,"a"
-	.align 3
-	.llong 100b,.Ldo_err1
-	.previous
+	EX_TABLE(100b,.Ldo_err1)
 	.endm
 
 	.macro err2
 200:
-	.section __ex_table,"a"
-	.align 3
-	.llong 200b,.Ldo_err2
-	.previous
+	EX_TABLE(200b,.Ldo_err2)
 	.endm
 
 	.macro err3
 300:
-	.section __ex_table,"a"
-	.align 3
-	.llong 300b,.Ldo_err3
-	.previous
+	EX_TABLE(300b,.Ldo_err3)
 	.endm
 
 .Ldo_err1:
diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile
index 1a4e570f7894..7414034df1c3 100644
--- a/arch/powerpc/mm/Makefile
+++ b/arch/powerpc/mm/Makefile
@@ -7,7 +7,8 @@ subdir-ccflags-$(CONFIG_PPC_WERROR) := -Werror
 ccflags-$(CONFIG_PPC64)	:= $(NO_MINIMAL_TOC)
 
 obj-y				:= fault.o mem.o pgtable.o mmap.o \
-				   init_$(BITS).o pgtable_$(BITS).o
+				   init_$(BITS).o pgtable_$(BITS).o \
+				   init-common.o
 obj-$(CONFIG_PPC_MMU_NOHASH)	+= mmu_context_nohash.o tlb_nohash.o \
 				   tlb_nohash_low.o
 obj-$(CONFIG_PPC_BOOK3E)	+= tlb_low_$(BITS)e.o
@@ -42,3 +43,5 @@ obj-$(CONFIG_NOT_COHERENT_CACHE) += dma-noncoherent.o
 obj-$(CONFIG_HIGHMEM)		+= highmem.o
 obj-$(CONFIG_PPC_COPRO_BASE)	+= copro_fault.o
 obj-$(CONFIG_SPAPR_TCE_IOMMU)	+= mmu_context_iommu.o
+obj-$(CONFIG_PPC_PTDUMP)	+= dump_linuxpagetables.o
+obj-$(CONFIG_PPC_HTDUMP)	+= dump_hashpagetable.o
diff --git a/arch/powerpc/mm/copro_fault.c b/arch/powerpc/mm/copro_fault.c
index 362954f98029..aaa7ec6788b9 100644
--- a/arch/powerpc/mm/copro_fault.c
+++ b/arch/powerpc/mm/copro_fault.c
@@ -134,6 +134,9 @@ int copro_calculate_slb(struct mm_struct *mm, u64 ea, struct copro_slb *slb)
 		pr_debug("%s: invalid region access at %016llx\n", __func__, ea);
 		return 1;
 	}
+	/* Bad address */
+	if (!vsid)
+		return 1;
 
 	vsid = (vsid << slb_vsid_shift(ssize)) | vsidkey;
 
diff --git a/arch/powerpc/mm/dump_hashpagetable.c b/arch/powerpc/mm/dump_hashpagetable.c
new file mode 100644
index 000000000000..d979709a0239
--- /dev/null
+++ b/arch/powerpc/mm/dump_hashpagetable.c
@@ -0,0 +1,551 @@
+/*
+ * Copyright 2016, Rashmica Gupta, IBM Corp.
+ *
+ * This traverses the kernel virtual memory and dumps the pages that are in
+ * the hash pagetable, along with their flags to
+ * /sys/kernel/debug/kernel_hash_pagetable.
+ *
+ * If radix is enabled then there is no hash page table and so no debugfs file
+ * is generated.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/sched.h>
+#include <linux/seq_file.h>
+#include <asm/fixmap.h>
+#include <asm/pgtable.h>
+#include <linux/const.h>
+#include <asm/page.h>
+#include <asm/pgalloc.h>
+#include <asm/plpar_wrappers.h>
+#include <linux/memblock.h>
+#include <asm/firmware.h>
+
+struct pg_state {
+	struct seq_file *seq;
+	const struct addr_marker *marker;
+	unsigned long start_address;
+	unsigned int level;
+	u64 current_flags;
+};
+
+struct addr_marker {
+	unsigned long start_address;
+	const char *name;
+};
+
+static struct addr_marker address_markers[] = {
+	{ 0,	"Start of kernel VM" },
+	{ 0,	"vmalloc() Area" },
+	{ 0,	"vmalloc() End" },
+	{ 0,	"isa I/O start" },
+	{ 0,	"isa I/O end" },
+	{ 0,	"phb I/O start" },
+	{ 0,	"phb I/O end" },
+	{ 0,	"I/O remap start" },
+	{ 0,	"I/O remap end" },
+	{ 0,	"vmemmap start" },
+	{ -1,	NULL },
+};
+
+struct flag_info {
+	u64		mask;
+	u64		val;
+	const char	*set;
+	const char	*clear;
+	bool		is_val;
+	int		shift;
+};
+
+static const struct flag_info v_flag_array[] = {
+	{
+		.mask   = SLB_VSID_B,
+		.val    = SLB_VSID_B_256M,
+		.set    = "ssize: 256M",
+		.clear  = "ssize: 1T  ",
+	}, {
+		.mask	= HPTE_V_SECONDARY,
+		.val	= HPTE_V_SECONDARY,
+		.set	= "secondary",
+		.clear	= "primary  ",
+	}, {
+		.mask	= HPTE_V_VALID,
+		.val	= HPTE_V_VALID,
+		.set	= "valid  ",
+		.clear	= "invalid",
+	}, {
+		.mask	= HPTE_V_BOLTED,
+		.val	= HPTE_V_BOLTED,
+		.set	= "bolted",
+		.clear	= "",
+	}
+};
+
+static const struct flag_info r_flag_array[] = {
+	{
+		.mask	= HPTE_R_PP0 | HPTE_R_PP,
+		.val	= PP_RWXX,
+		.set	= "prot:RW--",
+	}, {
+		.mask	= HPTE_R_PP0 | HPTE_R_PP,
+		.val	= PP_RWRX,
+		.set	= "prot:RWR-",
+	}, {
+		.mask	= HPTE_R_PP0 | HPTE_R_PP,
+		.val	= PP_RWRW,
+		.set	= "prot:RWRW",
+	}, {
+		.mask	= HPTE_R_PP0 | HPTE_R_PP,
+		.val	= PP_RXRX,
+		.set	= "prot:R-R-",
+	}, {
+		.mask	= HPTE_R_PP0 | HPTE_R_PP,
+		.val	= PP_RXXX,
+		.set	= "prot:R---",
+	}, {
+		.mask	= HPTE_R_KEY_HI | HPTE_R_KEY_LO,
+		.val	= HPTE_R_KEY_HI | HPTE_R_KEY_LO,
+		.set	= "key",
+		.clear	= "",
+		.is_val = true,
+	}, {
+		.mask	= HPTE_R_R,
+		.val	= HPTE_R_R,
+		.set	= "ref",
+		.clear	= "   ",
+	}, {
+		.mask	= HPTE_R_C,
+		.val	= HPTE_R_C,
+		.set	= "changed",
+		.clear	= "       ",
+	}, {
+		.mask	= HPTE_R_N,
+		.val	= HPTE_R_N,
+		.set	= "no execute",
+	}, {
+		.mask	= HPTE_R_WIMG,
+		.val	= HPTE_R_W,
+		.set	= "writethru",
+	}, {
+		.mask	= HPTE_R_WIMG,
+		.val	= HPTE_R_I,
+		.set	= "no cache",
+	}, {
+		.mask	= HPTE_R_WIMG,
+		.val	= HPTE_R_G,
+		.set	= "guarded",
+	}
+};
+
+static int calculate_pagesize(struct pg_state *st, int ps, char s[])
+{
+	static const char units[] = "BKMGTPE";
+	const char *unit = units;
+
+	while (ps > 9 && unit[1]) {
+		ps -= 10;
+		unit++;
+	}
+	seq_printf(st->seq, "  %s_ps: %i%c\t", s, 1<<ps, *unit);
+	return ps;
+}
+
+static void dump_flag_info(struct pg_state *st, const struct flag_info
+		*flag, u64 pte, int num)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++, flag++) {
+		const char *s = NULL;
+		u64 val;
+
+		/* flag not defined so don't check it */
+		if (flag->mask == 0)
+			continue;
+		/* Some 'flags' are actually values */
+		if (flag->is_val) {
+			val = pte & flag->val;
+			if (flag->shift)
+				val = val >> flag->shift;
+			seq_printf(st->seq, "  %s:%llx", flag->set, val);
+		} else {
+			if ((pte & flag->mask) == flag->val)
+				s = flag->set;
+			else
+				s = flag->clear;
+			if (s)
+				seq_printf(st->seq, "  %s", s);
+		}
+	}
+}
+
+static void dump_hpte_info(struct pg_state *st, unsigned long ea, u64 v, u64 r,
+		unsigned long rpn, int bps, int aps, unsigned long lp)
+{
+	int aps_index;
+
+	while (ea >= st->marker[1].start_address) {
+		st->marker++;
+		seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+	}
+	seq_printf(st->seq, "0x%lx:\t", ea);
+	seq_printf(st->seq, "AVPN:%llx\t", HPTE_V_AVPN_VAL(v));
+	dump_flag_info(st, v_flag_array, v, ARRAY_SIZE(v_flag_array));
+	seq_printf(st->seq, "  rpn: %lx\t", rpn);
+	dump_flag_info(st, r_flag_array, r, ARRAY_SIZE(r_flag_array));
+
+	calculate_pagesize(st, bps, "base");
+	aps_index = calculate_pagesize(st, aps, "actual");
+	if (aps_index != 2)
+		seq_printf(st->seq, "LP enc: %lx", lp);
+	seq_puts(st->seq, "\n");
+}
+
+
+static int native_find(unsigned long ea, int psize, bool primary, u64 *v, u64
+		*r)
+{
+	struct hash_pte *hptep;
+	unsigned long hash, vsid, vpn, hpte_group, want_v, hpte_v;
+	int i, ssize = mmu_kernel_ssize;
+	unsigned long shift = mmu_psize_defs[psize].shift;
+
+	/* calculate hash */
+	vsid = get_kernel_vsid(ea, ssize);
+	vpn  = hpt_vpn(ea, vsid, ssize);
+	hash = hpt_hash(vpn, shift, ssize);
+	want_v = hpte_encode_avpn(vpn, psize, ssize);
+
+	/* to check in the secondary hash table, we invert the hash */
+	if (!primary)
+		hash = ~hash;
+	hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP;
+	for (i = 0; i < HPTES_PER_GROUP; i++) {
+		hptep = htab_address + hpte_group;
+		hpte_v = be64_to_cpu(hptep->v);
+
+		if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
+			/* HPTE matches */
+			*v = be64_to_cpu(hptep->v);
+			*r = be64_to_cpu(hptep->r);
+			return 0;
+		}
+		++hpte_group;
+	}
+	return -1;
+}
+
+#ifdef CONFIG_PPC_PSERIES
+static int pseries_find(unsigned long ea, int psize, bool primary, u64 *v, u64 *r)
+{
+	struct hash_pte ptes[4];
+	unsigned long vsid, vpn, hash, hpte_group, want_v;
+	int i, j, ssize = mmu_kernel_ssize;
+	long lpar_rc = 0;
+	unsigned long shift = mmu_psize_defs[psize].shift;
+
+	/* calculate hash */
+	vsid = get_kernel_vsid(ea, ssize);
+	vpn  = hpt_vpn(ea, vsid, ssize);
+	hash = hpt_hash(vpn, shift, ssize);
+	want_v = hpte_encode_avpn(vpn, psize, ssize);
+
+	/* to check in the secondary hash table, we invert the hash */
+	if (!primary)
+		hash = ~hash;
+	hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
+	/* see if we can find an entry in the hpte with this hash */
+	for (i = 0; i < HPTES_PER_GROUP; i += 4, hpte_group += 4) {
+		lpar_rc = plpar_pte_read_4(0, hpte_group, (void *)ptes);
+
+		if (lpar_rc != H_SUCCESS)
+			continue;
+		for (j = 0; j < 4; j++) {
+			if (HPTE_V_COMPARE(ptes[j].v, want_v) &&
+					(ptes[j].v & HPTE_V_VALID)) {
+				/* HPTE matches */
+				*v = ptes[j].v;
+				*r = ptes[j].r;
+				return 0;
+			}
+		}
+	}
+	return -1;
+}
+#endif
+
+static void decode_r(int bps, unsigned long r, unsigned long *rpn, int *aps,
+		unsigned long *lp_bits)
+{
+	struct mmu_psize_def entry;
+	unsigned long arpn, mask, lp;
+	int penc = -2, idx = 0, shift;
+
+	/*.
+	 * The LP field has 8 bits. Depending on the actual page size, some of
+	 * these bits are concatenated with the APRN to get the RPN. The rest
+	 * of the bits in the LP field is the LP value and is an encoding for
+	 * the base page size and the actual page size.
+	 *
+	 *  -	find the mmu entry for our base page size
+	 *  -	go through all page encodings and use the associated mask to
+	 *	find an encoding that matches our encoding in the LP field.
+	 */
+	arpn = (r & HPTE_R_RPN) >> HPTE_R_RPN_SHIFT;
+	lp = arpn & 0xff;
+
+	entry = mmu_psize_defs[bps];
+	while (idx < MMU_PAGE_COUNT) {
+		penc = entry.penc[idx];
+		if ((penc != -1) && (mmu_psize_defs[idx].shift)) {
+			shift = mmu_psize_defs[idx].shift -  HPTE_R_RPN_SHIFT;
+			mask = (0x1 << (shift)) - 1;
+			if ((lp & mask) == penc) {
+				*aps = mmu_psize_to_shift(idx);
+				*lp_bits = lp & mask;
+				*rpn = arpn >> shift;
+				return;
+			}
+		}
+		idx++;
+	}
+}
+
+static int base_hpte_find(unsigned long ea, int psize, bool primary, u64 *v,
+			  u64 *r)
+{
+#ifdef CONFIG_PPC_PSERIES
+	if (firmware_has_feature(FW_FEATURE_LPAR))
+		return pseries_find(ea, psize, primary, v, r);
+#endif
+	return native_find(ea, psize, primary, v, r);
+}
+
+static unsigned long hpte_find(struct pg_state *st, unsigned long ea, int psize)
+{
+	unsigned long slot;
+	u64 v  = 0, r = 0;
+	unsigned long rpn, lp_bits;
+	int base_psize = 0, actual_psize = 0;
+
+	if (ea <= PAGE_OFFSET)
+		return -1;
+
+	/* Look in primary table */
+	slot = base_hpte_find(ea, psize, true, &v, &r);
+
+	/* Look in secondary table */
+	if (slot == -1)
+		slot = base_hpte_find(ea, psize, true, &v, &r);
+
+	/* No entry found */
+	if (slot == -1)
+		return -1;
+
+	/*
+	 * We found an entry in the hash page table:
+	 *  - check that this has the same base page
+	 *  - find the actual page size
+	 *  - find the RPN
+	 */
+	base_psize = mmu_psize_to_shift(psize);
+
+	if ((v & HPTE_V_LARGE) == HPTE_V_LARGE) {
+		decode_r(psize, r, &rpn, &actual_psize, &lp_bits);
+	} else {
+		/* 4K actual page size */
+		actual_psize = 12;
+		rpn = (r & HPTE_R_RPN) >> HPTE_R_RPN_SHIFT;
+		/* In this case there are no LP bits */
+		lp_bits = -1;
+	}
+	/*
+	 * We didn't find a matching encoding, so the PTE we found isn't for
+	 * this address.
+	 */
+	if (actual_psize == -1)
+		return -1;
+
+	dump_hpte_info(st, ea, v, r, rpn, base_psize, actual_psize, lp_bits);
+	return 0;
+}
+
+static void walk_pte(struct pg_state *st, pmd_t *pmd, unsigned long start)
+{
+	pte_t *pte = pte_offset_kernel(pmd, 0);
+	unsigned long addr, pteval, psize;
+	int i, status;
+
+	for (i = 0; i < PTRS_PER_PTE; i++, pte++) {
+		addr = start + i * PAGE_SIZE;
+		pteval = pte_val(*pte);
+
+		if (addr < VMALLOC_END)
+			psize = mmu_vmalloc_psize;
+		else
+			psize = mmu_io_psize;
+#ifdef CONFIG_PPC_64K_PAGES
+		/* check for secret 4K mappings */
+		if (((pteval & H_PAGE_COMBO) == H_PAGE_COMBO) ||
+			((pteval & H_PAGE_4K_PFN) == H_PAGE_4K_PFN))
+			psize = mmu_io_psize;
+#endif
+		/* check for hashpte */
+		status = hpte_find(st, addr, psize);
+
+		if (((pteval & H_PAGE_HASHPTE) != H_PAGE_HASHPTE)
+				&& (status != -1)) {
+		/* found a hpte that is not in the linux page tables */
+			seq_printf(st->seq, "page probably bolted before linux"
+				" pagetables were set: addr:%lx, pteval:%lx\n",
+				addr, pteval);
+		}
+	}
+}
+
+static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
+{
+	pmd_t *pmd = pmd_offset(pud, 0);
+	unsigned long addr;
+	unsigned int i;
+
+	for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
+		addr = start + i * PMD_SIZE;
+		if (!pmd_none(*pmd))
+			/* pmd exists */
+			walk_pte(st, pmd, addr);
+	}
+}
+
+static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
+{
+	pud_t *pud = pud_offset(pgd, 0);
+	unsigned long addr;
+	unsigned int i;
+
+	for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
+		addr = start + i * PUD_SIZE;
+		if (!pud_none(*pud))
+			/* pud exists */
+			walk_pmd(st, pud, addr);
+	}
+}
+
+static void walk_pagetables(struct pg_state *st)
+{
+	pgd_t *pgd = pgd_offset_k(0UL);
+	unsigned int i;
+	unsigned long addr;
+
+	/*
+	 * Traverse the linux pagetable structure and dump pages that are in
+	 * the hash pagetable.
+	 */
+	for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
+		addr = KERN_VIRT_START + i * PGDIR_SIZE;
+		if (!pgd_none(*pgd))
+			/* pgd exists */
+			walk_pud(st, pgd, addr);
+	}
+}
+
+
+static void walk_linearmapping(struct pg_state *st)
+{
+	unsigned long addr;
+
+	/*
+	 * Traverse the linear mapping section of virtual memory and dump pages
+	 * that are in the hash pagetable.
+	 */
+	unsigned long psize = 1 << mmu_psize_defs[mmu_linear_psize].shift;
+
+	for (addr = PAGE_OFFSET; addr < PAGE_OFFSET +
+			memblock_phys_mem_size(); addr += psize)
+		hpte_find(st, addr, mmu_linear_psize);
+}
+
+static void walk_vmemmap(struct pg_state *st)
+{
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
+	struct vmemmap_backing *ptr = vmemmap_list;
+
+	/*
+	 * Traverse the vmemmaped memory and dump pages that are in the hash
+	 * pagetable.
+	 */
+	while (ptr->list) {
+		hpte_find(st, ptr->virt_addr, mmu_vmemmap_psize);
+		ptr = ptr->list;
+	}
+	seq_puts(st->seq, "---[ vmemmap end ]---\n");
+#endif
+}
+
+static void populate_markers(void)
+{
+	address_markers[0].start_address = PAGE_OFFSET;
+	address_markers[1].start_address = VMALLOC_START;
+	address_markers[2].start_address = VMALLOC_END;
+	address_markers[3].start_address = ISA_IO_BASE;
+	address_markers[4].start_address = ISA_IO_END;
+	address_markers[5].start_address = PHB_IO_BASE;
+	address_markers[6].start_address = PHB_IO_END;
+	address_markers[7].start_address = IOREMAP_BASE;
+	address_markers[8].start_address = IOREMAP_END;
+#ifdef CONFIG_PPC_STD_MMU_64
+	address_markers[9].start_address =  H_VMEMMAP_BASE;
+#else
+	address_markers[9].start_address =  VMEMMAP_BASE;
+#endif
+}
+
+static int ptdump_show(struct seq_file *m, void *v)
+{
+	struct pg_state st = {
+		.seq = m,
+		.start_address = PAGE_OFFSET,
+		.marker = address_markers,
+	};
+	/*
+	 * Traverse the 0xc, 0xd and 0xf areas of the kernel virtual memory and
+	 * dump pages that are in the hash pagetable.
+	 */
+	walk_linearmapping(&st);
+	walk_pagetables(&st);
+	walk_vmemmap(&st);
+	return 0;
+}
+
+static int ptdump_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, ptdump_show, NULL);
+}
+
+static const struct file_operations ptdump_fops = {
+	.open		= ptdump_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int ptdump_init(void)
+{
+	struct dentry *debugfs_file;
+
+	if (!radix_enabled()) {
+		populate_markers();
+		debugfs_file = debugfs_create_file("kernel_hash_pagetable",
+				0400, NULL, NULL, &ptdump_fops);
+		return debugfs_file ? 0 : -ENOMEM;
+	}
+	return 0;
+}
+device_initcall(ptdump_init);
diff --git a/arch/powerpc/mm/dump_linuxpagetables.c b/arch/powerpc/mm/dump_linuxpagetables.c
new file mode 100644
index 000000000000..49abaf4dc8e3
--- /dev/null
+++ b/arch/powerpc/mm/dump_linuxpagetables.c
@@ -0,0 +1,442 @@
+/*
+ * Copyright 2016, Rashmica Gupta, IBM Corp.
+ *
+ * This traverses the kernel pagetables and dumps the
+ * information about the used sections of memory to
+ * /sys/kernel/debug/kernel_pagetables.
+ *
+ * Derived from the arm64 implementation:
+ * Copyright (c) 2014, The Linux Foundation, Laura Abbott.
+ * (C) Copyright 2008 Intel Corporation, Arjan van de Ven.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ */
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/sched.h>
+#include <linux/seq_file.h>
+#include <asm/fixmap.h>
+#include <asm/pgtable.h>
+#include <linux/const.h>
+#include <asm/page.h>
+#include <asm/pgalloc.h>
+
+/*
+ * To visualise what is happening,
+ *
+ *  - PTRS_PER_P** = how many entries there are in the corresponding P**
+ *  - P**_SHIFT = how many bits of the address we use to index into the
+ * corresponding P**
+ *  - P**_SIZE is how much memory we can access through the table - not the
+ * size of the table itself.
+ * P**={PGD, PUD, PMD, PTE}
+ *
+ *
+ * Each entry of the PGD points to a PUD. Each entry of a PUD points to a
+ * PMD. Each entry of a PMD points to a PTE. And every PTE entry points to
+ * a page.
+ *
+ * In the case where there are only 3 levels, the PUD is folded into the
+ * PGD: every PUD has only one entry which points to the PMD.
+ *
+ * The page dumper groups page table entries of the same type into a single
+ * description. It uses pg_state to track the range information while
+ * iterating over the PTE entries. When the continuity is broken it then
+ * dumps out a description of the range - ie PTEs that are virtually contiguous
+ * with the same PTE flags are chunked together. This is to make it clear how
+ * different areas of the kernel virtual memory are used.
+ *
+ */
+struct pg_state {
+	struct seq_file *seq;
+	const struct addr_marker *marker;
+	unsigned long start_address;
+	unsigned int level;
+	u64 current_flags;
+};
+
+struct addr_marker {
+	unsigned long start_address;
+	const char *name;
+};
+
+static struct addr_marker address_markers[] = {
+	{ 0,	"Start of kernel VM" },
+	{ 0,	"vmalloc() Area" },
+	{ 0,	"vmalloc() End" },
+	{ 0,	"isa I/O start" },
+	{ 0,	"isa I/O end" },
+	{ 0,	"phb I/O start" },
+	{ 0,	"phb I/O end" },
+	{ 0,	"I/O remap start" },
+	{ 0,	"I/O remap end" },
+	{ 0,	"vmemmap start" },
+	{ -1,	NULL },
+};
+
+struct flag_info {
+	u64		mask;
+	u64		val;
+	const char	*set;
+	const char	*clear;
+	bool		is_val;
+	int		shift;
+};
+
+static const struct flag_info flag_array[] = {
+	{
+#ifdef CONFIG_PPC_STD_MMU_64
+		.mask	= _PAGE_PRIVILEGED,
+		.val	= 0,
+#else
+		.mask	= _PAGE_USER,
+		.val	= _PAGE_USER,
+#endif
+		.set	= "user",
+		.clear	= "    ",
+	}, {
+		.mask	= _PAGE_RW,
+		.val	= _PAGE_RW,
+		.set	= "rw",
+		.clear	= "ro",
+	}, {
+		.mask	= _PAGE_EXEC,
+		.val	= _PAGE_EXEC,
+		.set	= " X ",
+		.clear	= "   ",
+	}, {
+		.mask	= _PAGE_PTE,
+		.val	= _PAGE_PTE,
+		.set	= "pte",
+		.clear	= "   ",
+	}, {
+		.mask	= _PAGE_PRESENT,
+		.val	= _PAGE_PRESENT,
+		.set	= "present",
+		.clear	= "       ",
+	}, {
+#ifdef CONFIG_PPC_STD_MMU_64
+		.mask	= H_PAGE_HASHPTE,
+		.val	= H_PAGE_HASHPTE,
+#else
+		.mask	= _PAGE_HASHPTE,
+		.val	= _PAGE_HASHPTE,
+#endif
+		.set	= "hpte",
+		.clear	= "    ",
+	}, {
+#ifndef CONFIG_PPC_STD_MMU_64
+		.mask	= _PAGE_GUARDED,
+		.val	= _PAGE_GUARDED,
+		.set	= "guarded",
+		.clear	= "       ",
+	}, {
+#endif
+		.mask	= _PAGE_DIRTY,
+		.val	= _PAGE_DIRTY,
+		.set	= "dirty",
+		.clear	= "     ",
+	}, {
+		.mask	= _PAGE_ACCESSED,
+		.val	= _PAGE_ACCESSED,
+		.set	= "accessed",
+		.clear	= "        ",
+	}, {
+#ifndef CONFIG_PPC_STD_MMU_64
+		.mask	= _PAGE_WRITETHRU,
+		.val	= _PAGE_WRITETHRU,
+		.set	= "write through",
+		.clear	= "             ",
+	}, {
+#endif
+		.mask	= _PAGE_NO_CACHE,
+		.val	= _PAGE_NO_CACHE,
+		.set	= "no cache",
+		.clear	= "        ",
+	}, {
+#ifdef CONFIG_PPC_BOOK3S_64
+		.mask	= H_PAGE_BUSY,
+		.val	= H_PAGE_BUSY,
+		.set	= "busy",
+	}, {
+#ifdef CONFIG_PPC_64K_PAGES
+		.mask	= H_PAGE_COMBO,
+		.val	= H_PAGE_COMBO,
+		.set	= "combo",
+	}, {
+		.mask	= H_PAGE_4K_PFN,
+		.val	= H_PAGE_4K_PFN,
+		.set	= "4K_pfn",
+	}, {
+#endif
+		.mask	= H_PAGE_F_GIX,
+		.val	= H_PAGE_F_GIX,
+		.set	= "f_gix",
+		.is_val	= true,
+		.shift	= H_PAGE_F_GIX_SHIFT,
+	}, {
+		.mask	= H_PAGE_F_SECOND,
+		.val	= H_PAGE_F_SECOND,
+		.set	= "f_second",
+	}, {
+#endif
+		.mask	= _PAGE_SPECIAL,
+		.val	= _PAGE_SPECIAL,
+		.set	= "special",
+	}
+};
+
+struct pgtable_level {
+	const struct flag_info *flag;
+	size_t num;
+	u64 mask;
+};
+
+static struct pgtable_level pg_level[] = {
+	{
+	}, { /* pgd */
+		.flag	= flag_array,
+		.num	= ARRAY_SIZE(flag_array),
+	}, { /* pud */
+		.flag	= flag_array,
+		.num	= ARRAY_SIZE(flag_array),
+	}, { /* pmd */
+		.flag	= flag_array,
+		.num	= ARRAY_SIZE(flag_array),
+	}, { /* pte */
+		.flag	= flag_array,
+		.num	= ARRAY_SIZE(flag_array),
+	},
+};
+
+static void dump_flag_info(struct pg_state *st, const struct flag_info
+		*flag, u64 pte, int num)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++, flag++) {
+		const char *s = NULL;
+		u64 val;
+
+		/* flag not defined so don't check it */
+		if (flag->mask == 0)
+			continue;
+		/* Some 'flags' are actually values */
+		if (flag->is_val) {
+			val = pte & flag->val;
+			if (flag->shift)
+				val = val >> flag->shift;
+			seq_printf(st->seq, "  %s:%llx", flag->set, val);
+		} else {
+			if ((pte & flag->mask) == flag->val)
+				s = flag->set;
+			else
+				s = flag->clear;
+			if (s)
+				seq_printf(st->seq, "  %s", s);
+		}
+		st->current_flags &= ~flag->mask;
+	}
+	if (st->current_flags != 0)
+		seq_printf(st->seq, "  unknown flags:%llx", st->current_flags);
+}
+
+static void dump_addr(struct pg_state *st, unsigned long addr)
+{
+	static const char units[] = "KMGTPE";
+	const char *unit = units;
+	unsigned long delta;
+
+	seq_printf(st->seq, "0x%016lx-0x%016lx   ", st->start_address, addr-1);
+	delta = (addr - st->start_address) >> 10;
+	/* Work out what appropriate unit to use */
+	while (!(delta & 1023) && unit[1]) {
+		delta >>= 10;
+		unit++;
+	}
+	seq_printf(st->seq, "%9lu%c", delta, *unit);
+
+}
+
+static void note_page(struct pg_state *st, unsigned long addr,
+	       unsigned int level, u64 val)
+{
+	u64 flag = val & pg_level[level].mask;
+	/* At first no level is set */
+	if (!st->level) {
+		st->level = level;
+		st->current_flags = flag;
+		st->start_address = addr;
+		seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+	/*
+	 * Dump the section of virtual memory when:
+	 *   - the PTE flags from one entry to the next differs.
+	 *   - we change levels in the tree.
+	 *   - the address is in a different section of memory and is thus
+	 *   used for a different purpose, regardless of the flags.
+	 */
+	} else if (flag != st->current_flags || level != st->level ||
+		   addr >= st->marker[1].start_address) {
+
+		/* Check the PTE flags */
+		if (st->current_flags) {
+			dump_addr(st, addr);
+
+			/* Dump all the flags */
+			if (pg_level[st->level].flag)
+				dump_flag_info(st, pg_level[st->level].flag,
+					  st->current_flags,
+					  pg_level[st->level].num);
+
+			seq_puts(st->seq, "\n");
+		}
+
+		/*
+		 * Address indicates we have passed the end of the
+		 * current section of virtual memory
+		 */
+		while (addr >= st->marker[1].start_address) {
+			st->marker++;
+			seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+		}
+		st->start_address = addr;
+		st->current_flags = flag;
+		st->level = level;
+	}
+}
+
+static void walk_pte(struct pg_state *st, pmd_t *pmd, unsigned long start)
+{
+	pte_t *pte = pte_offset_kernel(pmd, 0);
+	unsigned long addr;
+	unsigned int i;
+
+	for (i = 0; i < PTRS_PER_PTE; i++, pte++) {
+		addr = start + i * PAGE_SIZE;
+		note_page(st, addr, 4, pte_val(*pte));
+
+	}
+}
+
+static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
+{
+	pmd_t *pmd = pmd_offset(pud, 0);
+	unsigned long addr;
+	unsigned int i;
+
+	for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
+		addr = start + i * PMD_SIZE;
+		if (!pmd_none(*pmd))
+			/* pmd exists */
+			walk_pte(st, pmd, addr);
+		else
+			note_page(st, addr, 3, pmd_val(*pmd));
+	}
+}
+
+static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
+{
+	pud_t *pud = pud_offset(pgd, 0);
+	unsigned long addr;
+	unsigned int i;
+
+	for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
+		addr = start + i * PUD_SIZE;
+		if (!pud_none(*pud))
+			/* pud exists */
+			walk_pmd(st, pud, addr);
+		else
+			note_page(st, addr, 2, pud_val(*pud));
+	}
+}
+
+static void walk_pagetables(struct pg_state *st)
+{
+	pgd_t *pgd = pgd_offset_k(0UL);
+	unsigned int i;
+	unsigned long addr;
+
+	/*
+	 * Traverse the linux pagetable structure and dump pages that are in
+	 * the hash pagetable.
+	 */
+	for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
+		addr = KERN_VIRT_START + i * PGDIR_SIZE;
+		if (!pgd_none(*pgd))
+			/* pgd exists */
+			walk_pud(st, pgd, addr);
+		else
+			note_page(st, addr, 1, pgd_val(*pgd));
+	}
+}
+
+static void populate_markers(void)
+{
+	address_markers[0].start_address = PAGE_OFFSET;
+	address_markers[1].start_address = VMALLOC_START;
+	address_markers[2].start_address = VMALLOC_END;
+	address_markers[3].start_address = ISA_IO_BASE;
+	address_markers[4].start_address = ISA_IO_END;
+	address_markers[5].start_address = PHB_IO_BASE;
+	address_markers[6].start_address = PHB_IO_END;
+	address_markers[7].start_address = IOREMAP_BASE;
+	address_markers[8].start_address = IOREMAP_END;
+#ifdef CONFIG_PPC_STD_MMU_64
+	address_markers[9].start_address =  H_VMEMMAP_BASE;
+#else
+	address_markers[9].start_address =  VMEMMAP_BASE;
+#endif
+}
+
+static int ptdump_show(struct seq_file *m, void *v)
+{
+	struct pg_state st = {
+		.seq = m,
+		.start_address = KERN_VIRT_START,
+		.marker = address_markers,
+	};
+	/* Traverse kernel page tables */
+	walk_pagetables(&st);
+	note_page(&st, 0, 0, 0);
+	return 0;
+}
+
+
+static int ptdump_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, ptdump_show, NULL);
+}
+
+static const struct file_operations ptdump_fops = {
+	.open		= ptdump_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static void build_pgtable_complete_mask(void)
+{
+	unsigned int i, j;
+
+	for (i = 0; i < ARRAY_SIZE(pg_level); i++)
+		if (pg_level[i].flag)
+			for (j = 0; j < pg_level[i].num; j++)
+				pg_level[i].mask |= pg_level[i].flag[j].mask;
+}
+
+static int ptdump_init(void)
+{
+	struct dentry *debugfs_file;
+
+	populate_markers();
+	build_pgtable_complete_mask();
+	debugfs_file = debugfs_create_file("kernel_pagetables", 0400, NULL,
+			NULL, &ptdump_fops);
+	return debugfs_file ? 0 : -ENOMEM;
+}
+device_initcall(ptdump_init);
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index d0b137d96df1..6fd30ac7d14a 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -391,6 +391,20 @@ good_area:
 
 	if (is_exec) {
 		/*
+		 * An execution fault + no execute ?
+		 *
+		 * On CPUs that don't have CPU_FTR_COHERENT_ICACHE we
+		 * deliberately create NX mappings, and use the fault to do the
+		 * cache flush. This is usually handled in hash_page_do_lazy_icache()
+		 * but we could end up here if that races with a concurrent PTE
+		 * update. In that case we need to fall through here to the VMA
+		 * check below.
+		 */
+		if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE) &&
+			(regs->msr & SRR1_ISI_N_OR_G))
+			goto bad_area;
+
+		/*
 		 * Allow execution from readable areas if the MMU does not
 		 * provide separate controls over reading and executing.
 		 *
@@ -404,6 +418,7 @@ good_area:
 		    (cpu_has_feature(CPU_FTR_NOEXECUTE) ||
 		     !(vma->vm_flags & (VM_READ | VM_WRITE))))
 			goto bad_area;
+
 #ifdef CONFIG_PPC_STD_MMU
 		/*
 		 * protfault should only happen due to us
@@ -512,7 +527,7 @@ void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)
 
 	/* Are we prepared to handle this fault?  */
 	if ((entry = search_exception_tables(regs->nip)) != NULL) {
-		regs->nip = entry->fixup;
+		regs->nip = extable_fixup(entry);
 		return;
 	}
 
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index 83ddc0e171b0..cc332608e656 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -123,8 +123,9 @@ static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)
 		va |= ssize << 8;
 		sllp = get_sllp_encoding(apsize);
 		va |= sllp << 5;
-		asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)"
-			     : : "r"(va) : "memory");
+		asm volatile(ASM_FTR_IFSET("tlbiel %0", "tlbiel %0,0", %1)
+			     : : "r" (va), "i" (CPU_FTR_ARCH_206)
+			     : "memory");
 		break;
 	default:
 		/* We need 14 to 14 + i bits of va */
@@ -141,8 +142,9 @@ static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)
 		 */
 		va |= (vpn & 0xfe);
 		va |= 1; /* L */
-		asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)"
-			     : : "r"(va) : "memory");
+		asm volatile(ASM_FTR_IFSET("tlbiel %0", "tlbiel %0,1", %1)
+			     : : "r" (va), "i" (CPU_FTR_ARCH_206)
+			     : "memory");
 		break;
 	}
 
@@ -221,13 +223,18 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
 		return -1;
 
 	hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
-	hpte_r = hpte_encode_r(pa, psize, apsize, ssize) | rflags;
+	hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
 
 	if (!(vflags & HPTE_V_BOLTED)) {
 		DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
 			i, hpte_v, hpte_r);
 	}
 
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		hpte_r = hpte_old_to_new_r(hpte_v, hpte_r);
+		hpte_v = hpte_old_to_new_v(hpte_v);
+	}
+
 	hptep->r = cpu_to_be64(hpte_r);
 	/* Guarantee the second dword is visible before the valid bit */
 	eieio();
@@ -295,6 +302,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
 		vpn, want_v & HPTE_V_AVPN, slot, newpp);
 
 	hpte_v = be64_to_cpu(hptep->v);
+	if (cpu_has_feature(CPU_FTR_ARCH_300))
+		hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r));
 	/*
 	 * We need to invalidate the TLB always because hpte_remove doesn't do
 	 * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
@@ -309,6 +318,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
 		native_lock_hpte(hptep);
 		/* recheck with locks held */
 		hpte_v = be64_to_cpu(hptep->v);
+		if (cpu_has_feature(CPU_FTR_ARCH_300))
+			hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r));
 		if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) ||
 			     !(hpte_v & HPTE_V_VALID))) {
 			ret = -1;
@@ -350,6 +361,8 @@ static long native_hpte_find(unsigned long vpn, int psize, int ssize)
 	for (i = 0; i < HPTES_PER_GROUP; i++) {
 		hptep = htab_address + slot;
 		hpte_v = be64_to_cpu(hptep->v);
+		if (cpu_has_feature(CPU_FTR_ARCH_300))
+			hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r));
 
 		if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
 			/* HPTE matches */
@@ -409,6 +422,8 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
 	want_v = hpte_encode_avpn(vpn, bpsize, ssize);
 	native_lock_hpte(hptep);
 	hpte_v = be64_to_cpu(hptep->v);
+	if (cpu_has_feature(CPU_FTR_ARCH_300))
+		hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r));
 
 	/*
 	 * We need to invalidate the TLB always because hpte_remove doesn't do
@@ -467,6 +482,8 @@ static void native_hugepage_invalidate(unsigned long vsid,
 		want_v = hpte_encode_avpn(vpn, psize, ssize);
 		native_lock_hpte(hptep);
 		hpte_v = be64_to_cpu(hptep->v);
+		if (cpu_has_feature(CPU_FTR_ARCH_300))
+			hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r));
 
 		/* Even if we miss, we need to invalidate the TLB */
 		if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
@@ -504,6 +521,10 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
 	/* Look at the 8 bit LP value */
 	unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1);
 
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		hpte_v = hpte_new_to_old_v(hpte_v, hpte_r);
+		hpte_r = hpte_new_to_old_r(hpte_r);
+	}
 	if (!(hpte_v & HPTE_V_LARGE)) {
 		size   = MMU_PAGE_4K;
 		a_size = MMU_PAGE_4K;
@@ -512,11 +533,7 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
 		a_size = hpte_page_sizes[lp] >> 4;
 	}
 	/* This works for all page sizes, and for 256M and 1T segments */
-	if (cpu_has_feature(CPU_FTR_ARCH_300))
-		*ssize = hpte_r >> HPTE_R_3_0_SSIZE_SHIFT;
-	else
-		*ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
-
+	*ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
 	shift = mmu_psize_defs[size].shift;
 
 	avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm);
@@ -639,6 +656,9 @@ static void native_flush_hash_range(unsigned long number, int local)
 			want_v = hpte_encode_avpn(vpn, psize, ssize);
 			native_lock_hpte(hptep);
 			hpte_v = be64_to_cpu(hptep->v);
+			if (cpu_has_feature(CPU_FTR_ARCH_300))
+				hpte_v = hpte_new_to_old_v(hpte_v,
+						be64_to_cpu(hptep->r));
 			if (!HPTE_V_COMPARE(hpte_v, want_v) ||
 			    !(hpte_v & HPTE_V_VALID))
 				native_unlock_hpte(hptep);
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 78dabf065ba9..8410b4bb36ed 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -796,37 +796,17 @@ static void update_hid_for_hash(void)
 static void __init hash_init_partition_table(phys_addr_t hash_table,
 					     unsigned long htab_size)
 {
-	unsigned long ps_field;
-	unsigned long patb_size = 1UL << PATB_SIZE_SHIFT;
+	mmu_partition_table_init();
 
 	/*
-	 * slb llp encoding for the page size used in VPM real mode.
-	 * We can ignore that for lpid 0
+	 * PS field (VRMA page size) is not used for LPID 0, hence set to 0.
+	 * For now, UPRT is 0 and we have no segment table.
 	 */
-	ps_field = 0;
 	htab_size =  __ilog2(htab_size) - 18;
-
-	BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 24), "Partition table size too large.");
-	partition_tb = __va(memblock_alloc_base(patb_size, patb_size,
-						MEMBLOCK_ALLOC_ANYWHERE));
-
-	/* Initialize the Partition Table with no entries */
-	memset((void *)partition_tb, 0, patb_size);
-	partition_tb->patb0 = cpu_to_be64(ps_field | hash_table | htab_size);
-	/*
-	 * FIXME!! This should be done via update_partition table
-	 * For now UPRT is 0 for us.
-	 */
-	partition_tb->patb1 = 0;
+	mmu_partition_table_set_entry(0, hash_table | htab_size, 0);
 	pr_info("Partition table %p\n", partition_tb);
 	if (cpu_has_feature(CPU_FTR_POWER9_DD1))
 		update_hid_for_hash();
-	/*
-	 * update partition table control register,
-	 * 64 K size.
-	 */
-	mtspr(SPRN_PTCR, __pa(partition_tb) | (PATB_SIZE_SHIFT - 12));
-
 }
 
 static void __init htab_initialize(void)
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index a5d3ecdabc44..289df38fb7e0 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -26,6 +26,8 @@
 #ifdef CONFIG_HUGETLB_PAGE
 
 #define PAGE_SHIFT_64K	16
+#define PAGE_SHIFT_512K	19
+#define PAGE_SHIFT_8M	23
 #define PAGE_SHIFT_16M	24
 #define PAGE_SHIFT_16G	34
 
@@ -38,7 +40,7 @@ unsigned int HPAGE_SHIFT;
  * implementations may have more than one gpage size, so we need multiple
  * arrays
  */
-#ifdef CONFIG_PPC_FSL_BOOK3E
+#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
 #define MAX_NUMBER_GPAGES	128
 struct psize_gpages {
 	u64 gpage_list[MAX_NUMBER_GPAGES];
@@ -64,14 +66,16 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
 {
 	struct kmem_cache *cachep;
 	pte_t *new;
-
-#ifdef CONFIG_PPC_FSL_BOOK3E
 	int i;
-	int num_hugepd = 1 << (pshift - pdshift);
-	cachep = hugepte_cache;
-#else
-	cachep = PGT_CACHE(pdshift - pshift);
-#endif
+	int num_hugepd;
+
+	if (pshift >= pdshift) {
+		cachep = hugepte_cache;
+		num_hugepd = 1 << (pshift - pdshift);
+	} else {
+		cachep = PGT_CACHE(pdshift - pshift);
+		num_hugepd = 1;
+	}
 
 	new = kmem_cache_zalloc(cachep, GFP_KERNEL);
 
@@ -89,7 +93,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
 	smp_wmb();
 
 	spin_lock(&mm->page_table_lock);
-#ifdef CONFIG_PPC_FSL_BOOK3E
+
 	/*
 	 * We have multiple higher-level entries that point to the same
 	 * actual pte location.  Fill in each as we go and backtrack on error.
@@ -100,8 +104,18 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
 		if (unlikely(!hugepd_none(*hpdp)))
 			break;
 		else
+#ifdef CONFIG_PPC_BOOK3S_64
+			hpdp->pd = __pa(new) |
+				   (shift_to_mmu_psize(pshift) << 2);
+#elif defined(CONFIG_PPC_8xx)
+			hpdp->pd = __pa(new) |
+				   (pshift == PAGE_SHIFT_8M ? _PMD_PAGE_8M :
+							      _PMD_PAGE_512K) |
+				   _PMD_PRESENT;
+#else
 			/* We use the old format for PPC_FSL_BOOK3E */
 			hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift;
+#endif
 	}
 	/* If we bailed from the for loop early, an error occurred, clean up */
 	if (i < num_hugepd) {
@@ -109,17 +123,6 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
 			hpdp->pd = 0;
 		kmem_cache_free(cachep, new);
 	}
-#else
-	if (!hugepd_none(*hpdp))
-		kmem_cache_free(cachep, new);
-	else {
-#ifdef CONFIG_PPC_BOOK3S_64
-		hpdp->pd = __pa(new) | (shift_to_mmu_psize(pshift) << 2);
-#else
-		hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift;
-#endif
-	}
-#endif
 	spin_unlock(&mm->page_table_lock);
 	return 0;
 }
@@ -128,7 +131,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
  * These macros define how to determine which level of the page table holds
  * the hpdp.
  */
-#ifdef CONFIG_PPC_FSL_BOOK3E
+#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
 #define HUGEPD_PGD_SHIFT PGDIR_SHIFT
 #define HUGEPD_PUD_SHIFT PUD_SHIFT
 #else
@@ -136,7 +139,6 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
 #define HUGEPD_PUD_SHIFT PMD_SHIFT
 #endif
 
-#ifdef CONFIG_PPC_BOOK3S_64
 /*
  * At this point we do the placement change only for BOOK3S 64. This would
  * possibly work on other subarchs.
@@ -153,6 +155,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz
 	addr &= ~(sz-1);
 	pg = pgd_offset(mm, addr);
 
+#ifdef CONFIG_PPC_BOOK3S_64
 	if (pshift == PGDIR_SHIFT)
 		/* 16GB huge page */
 		return (pte_t *) pg;
@@ -178,32 +181,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz
 				hpdp = (hugepd_t *)pm;
 		}
 	}
-	if (!hpdp)
-		return NULL;
-
-	BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp));
-
-	if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, pdshift, pshift))
-		return NULL;
-
-	return hugepte_offset(*hpdp, addr, pdshift);
-}
-
 #else
-
-pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
-{
-	pgd_t *pg;
-	pud_t *pu;
-	pmd_t *pm;
-	hugepd_t *hpdp = NULL;
-	unsigned pshift = __ffs(sz);
-	unsigned pdshift = PGDIR_SHIFT;
-
-	addr &= ~(sz-1);
-
-	pg = pgd_offset(mm, addr);
-
 	if (pshift >= HUGEPD_PGD_SHIFT) {
 		hpdp = (hugepd_t *)pg;
 	} else {
@@ -217,7 +195,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz
 			hpdp = (hugepd_t *)pm;
 		}
 	}
-
+#endif
 	if (!hpdp)
 		return NULL;
 
@@ -228,9 +206,8 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz
 
 	return hugepte_offset(*hpdp, addr, pdshift);
 }
-#endif
 
-#ifdef CONFIG_PPC_FSL_BOOK3E
+#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
 /* Build list of addresses of gigantic pages.  This function is used in early
  * boot before the buddy allocator is setup.
  */
@@ -310,7 +287,11 @@ static int __init do_gpage_early_setup(char *param, char *val,
 				npages = 0;
 			if (npages > MAX_NUMBER_GPAGES) {
 				pr_warn("MMU: %lu pages requested for page "
+#ifdef CONFIG_PHYS_ADDR_T_64BIT
 					"size %llu KB, limiting to "
+#else
+					"size %u KB, limiting to "
+#endif
 					__stringify(MAX_NUMBER_GPAGES) "\n",
 					npages, size / 1024);
 				npages = MAX_NUMBER_GPAGES;
@@ -392,7 +373,7 @@ int alloc_bootmem_huge_page(struct hstate *hstate)
 }
 #endif
 
-#ifdef CONFIG_PPC_FSL_BOOK3E
+#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
 #define HUGEPD_FREELIST_SIZE \
 	((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t))
 
@@ -442,6 +423,8 @@ static void hugepd_free(struct mmu_gather *tlb, void *hugepte)
 	}
 	put_cpu_var(hugepd_freelist_cur);
 }
+#else
+static inline void hugepd_free(struct mmu_gather *tlb, void *hugepte) {}
 #endif
 
 static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshift,
@@ -453,13 +436,11 @@ static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshif
 
 	unsigned long pdmask = ~((1UL << pdshift) - 1);
 	unsigned int num_hugepd = 1;
+	unsigned int shift = hugepd_shift(*hpdp);
 
-#ifdef CONFIG_PPC_FSL_BOOK3E
 	/* Note: On fsl the hpdp may be the first of several */
-	num_hugepd = (1 << (hugepd_shift(*hpdp) - pdshift));
-#else
-	unsigned int shift = hugepd_shift(*hpdp);
-#endif
+	if (shift > pdshift)
+		num_hugepd = 1 << (shift - pdshift);
 
 	start &= pdmask;
 	if (start < floor)
@@ -475,11 +456,10 @@ static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshif
 	for (i = 0; i < num_hugepd; i++, hpdp++)
 		hpdp->pd = 0;
 
-#ifdef CONFIG_PPC_FSL_BOOK3E
-	hugepd_free(tlb, hugepte);
-#else
-	pgtable_free_tlb(tlb, hugepte, pdshift - shift);
-#endif
+	if (shift >= pdshift)
+		hugepd_free(tlb, hugepte);
+	else
+		pgtable_free_tlb(tlb, hugepte, pdshift - shift);
 }
 
 static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
@@ -492,6 +472,8 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
 
 	start = addr;
 	do {
+		unsigned long more;
+
 		pmd = pmd_offset(pud, addr);
 		next = pmd_addr_end(addr, end);
 		if (!is_hugepd(__hugepd(pmd_val(*pmd)))) {
@@ -502,15 +484,16 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
 			WARN_ON(!pmd_none_or_clear_bad(pmd));
 			continue;
 		}
-#ifdef CONFIG_PPC_FSL_BOOK3E
 		/*
 		 * Increment next by the size of the huge mapping since
 		 * there may be more than one entry at this level for a
 		 * single hugepage, but all of them point to
 		 * the same kmem cache that holds the hugepte.
 		 */
-		next = addr + (1 << hugepd_shift(*(hugepd_t *)pmd));
-#endif
+		more = addr + (1 << hugepd_shift(*(hugepd_t *)pmd));
+		if (more > next)
+			next = more;
+
 		free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT,
 				  addr, next, floor, ceiling);
 	} while (addr = next, addr != end);
@@ -550,15 +533,17 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
 			hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
 					       ceiling);
 		} else {
-#ifdef CONFIG_PPC_FSL_BOOK3E
+			unsigned long more;
 			/*
 			 * Increment next by the size of the huge mapping since
 			 * there may be more than one entry at this level for a
 			 * single hugepage, but all of them point to
 			 * the same kmem cache that holds the hugepte.
 			 */
-			next = addr + (1 << hugepd_shift(*(hugepd_t *)pud));
-#endif
+			more = addr + (1 << hugepd_shift(*(hugepd_t *)pud));
+			if (more > next)
+				next = more;
+
 			free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT,
 					  addr, next, floor, ceiling);
 		}
@@ -615,15 +600,17 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,
 				continue;
 			hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
 		} else {
-#ifdef CONFIG_PPC_FSL_BOOK3E
+			unsigned long more;
 			/*
 			 * Increment next by the size of the huge mapping since
 			 * there may be more than one entry at the pgd level
 			 * for a single hugepage, but all of them point to the
 			 * same kmem cache that holds the hugepte.
 			 */
-			next = addr + (1 << hugepd_shift(*(hugepd_t *)pgd));
-#endif
+			more = addr + (1 << hugepd_shift(*(hugepd_t *)pgd));
+			if (more > next)
+				next = more;
+
 			free_hugepd_range(tlb, (hugepd_t *)pgd, PGDIR_SHIFT,
 					  addr, next, floor, ceiling);
 		}
@@ -753,12 +740,13 @@ static int __init add_huge_page_size(unsigned long long size)
 
 	/* Check that it is a page size supported by the hardware and
 	 * that it fits within pagetable and slice limits. */
-#ifdef CONFIG_PPC_FSL_BOOK3E
-	if ((size < PAGE_SIZE) || !is_power_of_4(size))
+	if (size <= PAGE_SIZE)
 		return -EINVAL;
-#else
-	if (!is_power_of_2(size)
-	    || (shift > SLICE_HIGH_SHIFT) || (shift <= PAGE_SHIFT))
+#if defined(CONFIG_PPC_FSL_BOOK3E)
+	if (!is_power_of_4(size))
+		return -EINVAL;
+#elif !defined(CONFIG_PPC_8xx)
+	if (!is_power_of_2(size) || (shift > SLICE_HIGH_SHIFT))
 		return -EINVAL;
 #endif
 
@@ -791,53 +779,15 @@ static int __init hugepage_setup_sz(char *str)
 }
 __setup("hugepagesz=", hugepage_setup_sz);
 
-#ifdef CONFIG_PPC_FSL_BOOK3E
 struct kmem_cache *hugepte_cache;
 static int __init hugetlbpage_init(void)
 {
 	int psize;
 
-	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
-		unsigned shift;
-
-		if (!mmu_psize_defs[psize].shift)
-			continue;
-
-		shift = mmu_psize_to_shift(psize);
-
-		/* Don't treat normal page sizes as huge... */
-		if (shift != PAGE_SHIFT)
-			if (add_huge_page_size(1ULL << shift) < 0)
-				continue;
-	}
-
-	/*
-	 * Create a kmem cache for hugeptes.  The bottom bits in the pte have
-	 * size information encoded in them, so align them to allow this
-	 */
-	hugepte_cache =  kmem_cache_create("hugepte-cache", sizeof(pte_t),
-					   HUGEPD_SHIFT_MASK + 1, 0, NULL);
-	if (hugepte_cache == NULL)
-		panic("%s: Unable to create kmem cache for hugeptes\n",
-		      __func__);
-
-	/* Default hpage size = 4M */
-	if (mmu_psize_defs[MMU_PAGE_4M].shift)
-		HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_4M].shift;
-	else
-		panic("%s: Unable to set default huge page size\n", __func__);
-
-
-	return 0;
-}
-#else
-static int __init hugetlbpage_init(void)
-{
-	int psize;
-
+#if !defined(CONFIG_PPC_FSL_BOOK3E) && !defined(CONFIG_PPC_8xx)
 	if (!radix_enabled() && !mmu_has_feature(MMU_FTR_16M_PAGE))
 		return -ENODEV;
-
+#endif
 	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
 		unsigned shift;
 		unsigned pdshift;
@@ -850,9 +800,9 @@ static int __init hugetlbpage_init(void)
 		if (add_huge_page_size(1ULL << shift) < 0)
 			continue;
 
-		if (shift < PMD_SHIFT)
+		if (shift < HUGEPD_PUD_SHIFT)
 			pdshift = PMD_SHIFT;
-		else if (shift < PUD_SHIFT)
+		else if (shift < HUGEPD_PGD_SHIFT)
 			pdshift = PUD_SHIFT;
 		else
 			pdshift = PGDIR_SHIFT;
@@ -860,14 +810,38 @@ static int __init hugetlbpage_init(void)
 		 * if we have pdshift and shift value same, we don't
 		 * use pgt cache for hugepd.
 		 */
-		if (pdshift != shift) {
+		if (pdshift > shift) {
 			pgtable_cache_add(pdshift - shift, NULL);
 			if (!PGT_CACHE(pdshift - shift))
 				panic("hugetlbpage_init(): could not create "
 				      "pgtable cache for %d bit pagesize\n", shift);
 		}
+#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
+		else if (!hugepte_cache) {
+			/*
+			 * Create a kmem cache for hugeptes.  The bottom bits in
+			 * the pte have size information encoded in them, so
+			 * align them to allow this
+			 */
+			hugepte_cache = kmem_cache_create("hugepte-cache",
+							  sizeof(pte_t),
+							  HUGEPD_SHIFT_MASK + 1,
+							  0, NULL);
+			if (hugepte_cache == NULL)
+				panic("%s: Unable to create kmem cache "
+				      "for hugeptes\n", __func__);
+
+		}
+#endif
 	}
 
+#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
+	/* Default hpage size = 4M on FSL_BOOK3E and 512k on 8xx */
+	if (mmu_psize_defs[MMU_PAGE_4M].shift)
+		HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_4M].shift;
+	else if (mmu_psize_defs[MMU_PAGE_512K].shift)
+		HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_512K].shift;
+#else
 	/* Set default large page size. Currently, we pick 16M or 1M
 	 * depending on what is available
 	 */
@@ -877,11 +851,13 @@ static int __init hugetlbpage_init(void)
 		HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_1M].shift;
 	else if (mmu_psize_defs[MMU_PAGE_2M].shift)
 		HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_2M].shift;
-
+#endif
+	else
+		panic("%s: Unable to set default huge page size\n", __func__);
 
 	return 0;
 }
-#endif
+
 arch_initcall(hugetlbpage_init);
 
 void flush_dcache_icache_hugepage(struct page *page)
diff --git a/arch/powerpc/mm/init-common.c b/arch/powerpc/mm/init-common.c
new file mode 100644
index 000000000000..a175cd82ae8c
--- /dev/null
+++ b/arch/powerpc/mm/init-common.c
@@ -0,0 +1,107 @@
+/*
+ *  PowerPC version
+ *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
+ *
+ *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
+ *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
+ *    Copyright (C) 1996 Paul Mackerras
+ *
+ *  Derived from "arch/i386/mm/init.c"
+ *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
+ *
+ *  Dave Engebretsen <engebret@us.ibm.com>
+ *      Rework for PPC64 port.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version
+ *  2 of the License, or (at your option) any later version.
+ *
+ */
+
+#undef DEBUG
+
+#include <linux/string.h>
+#include <asm/pgalloc.h>
+#include <asm/pgtable.h>
+
+static void pgd_ctor(void *addr)
+{
+	memset(addr, 0, PGD_TABLE_SIZE);
+}
+
+static void pud_ctor(void *addr)
+{
+	memset(addr, 0, PUD_TABLE_SIZE);
+}
+
+static void pmd_ctor(void *addr)
+{
+	memset(addr, 0, PMD_TABLE_SIZE);
+}
+
+struct kmem_cache *pgtable_cache[MAX_PGTABLE_INDEX_SIZE];
+
+/*
+ * Create a kmem_cache() for pagetables.  This is not used for PTE
+ * pages - they're linked to struct page, come from the normal free
+ * pages pool and have a different entry size (see real_pte_t) to
+ * everything else.  Caches created by this function are used for all
+ * the higher level pagetables, and for hugepage pagetables.
+ */
+void pgtable_cache_add(unsigned shift, void (*ctor)(void *))
+{
+	char *name;
+	unsigned long table_size = sizeof(void *) << shift;
+	unsigned long align = table_size;
+
+	/* When batching pgtable pointers for RCU freeing, we store
+	 * the index size in the low bits.  Table alignment must be
+	 * big enough to fit it.
+	 *
+	 * Likewise, hugeapge pagetable pointers contain a (different)
+	 * shift value in the low bits.  All tables must be aligned so
+	 * as to leave enough 0 bits in the address to contain it. */
+	unsigned long minalign = max(MAX_PGTABLE_INDEX_SIZE + 1,
+				     HUGEPD_SHIFT_MASK + 1);
+	struct kmem_cache *new;
+
+	/* It would be nice if this was a BUILD_BUG_ON(), but at the
+	 * moment, gcc doesn't seem to recognize is_power_of_2 as a
+	 * constant expression, so so much for that. */
+	BUG_ON(!is_power_of_2(minalign));
+	BUG_ON((shift < 1) || (shift > MAX_PGTABLE_INDEX_SIZE));
+
+	if (PGT_CACHE(shift))
+		return; /* Already have a cache of this size */
+
+	align = max_t(unsigned long, align, minalign);
+	name = kasprintf(GFP_KERNEL, "pgtable-2^%d", shift);
+	new = kmem_cache_create(name, table_size, align, 0, ctor);
+	kfree(name);
+	pgtable_cache[shift - 1] = new;
+	pr_debug("Allocated pgtable cache for order %d\n", shift);
+}
+
+
+void pgtable_cache_init(void)
+{
+	pgtable_cache_add(PGD_INDEX_SIZE, pgd_ctor);
+
+	if (PMD_INDEX_SIZE && !PGT_CACHE(PMD_INDEX_SIZE))
+		pgtable_cache_add(PMD_CACHE_INDEX, pmd_ctor);
+	/*
+	 * In all current configs, when the PUD index exists it's the
+	 * same size as either the pgd or pmd index except with THP enabled
+	 * on book3s 64
+	 */
+	if (PUD_INDEX_SIZE && !PGT_CACHE(PUD_INDEX_SIZE))
+		pgtable_cache_add(PUD_INDEX_SIZE, pud_ctor);
+
+	if (!PGT_CACHE(PGD_INDEX_SIZE))
+		panic("Couldn't allocate pgd cache");
+	if (PMD_INDEX_SIZE && !PGT_CACHE(PMD_INDEX_SIZE))
+		panic("Couldn't allocate pmd pgtable caches");
+	if (PUD_INDEX_SIZE && !PGT_CACHE(PUD_INDEX_SIZE))
+		panic("Couldn't allocate pud pgtable caches");
+}
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index 16ada1eb7e26..a000c3585390 100644
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -80,83 +80,6 @@ EXPORT_SYMBOL_GPL(memstart_addr);
 phys_addr_t kernstart_addr;
 EXPORT_SYMBOL_GPL(kernstart_addr);
 
-static void pgd_ctor(void *addr)
-{
-	memset(addr, 0, PGD_TABLE_SIZE);
-}
-
-static void pud_ctor(void *addr)
-{
-	memset(addr, 0, PUD_TABLE_SIZE);
-}
-
-static void pmd_ctor(void *addr)
-{
-	memset(addr, 0, PMD_TABLE_SIZE);
-}
-
-struct kmem_cache *pgtable_cache[MAX_PGTABLE_INDEX_SIZE];
-
-/*
- * Create a kmem_cache() for pagetables.  This is not used for PTE
- * pages - they're linked to struct page, come from the normal free
- * pages pool and have a different entry size (see real_pte_t) to
- * everything else.  Caches created by this function are used for all
- * the higher level pagetables, and for hugepage pagetables.
- */
-void pgtable_cache_add(unsigned shift, void (*ctor)(void *))
-{
-	char *name;
-	unsigned long table_size = sizeof(void *) << shift;
-	unsigned long align = table_size;
-
-	/* When batching pgtable pointers for RCU freeing, we store
-	 * the index size in the low bits.  Table alignment must be
-	 * big enough to fit it.
-	 *
-	 * Likewise, hugeapge pagetable pointers contain a (different)
-	 * shift value in the low bits.  All tables must be aligned so
-	 * as to leave enough 0 bits in the address to contain it. */
-	unsigned long minalign = max(MAX_PGTABLE_INDEX_SIZE + 1,
-				     HUGEPD_SHIFT_MASK + 1);
-	struct kmem_cache *new;
-
-	/* It would be nice if this was a BUILD_BUG_ON(), but at the
-	 * moment, gcc doesn't seem to recognize is_power_of_2 as a
-	 * constant expression, so so much for that. */
-	BUG_ON(!is_power_of_2(minalign));
-	BUG_ON((shift < 1) || (shift > MAX_PGTABLE_INDEX_SIZE));
-
-	if (PGT_CACHE(shift))
-		return; /* Already have a cache of this size */
-
-	align = max_t(unsigned long, align, minalign);
-	name = kasprintf(GFP_KERNEL, "pgtable-2^%d", shift);
-	new = kmem_cache_create(name, table_size, align, 0, ctor);
-	kfree(name);
-	pgtable_cache[shift - 1] = new;
-	pr_debug("Allocated pgtable cache for order %d\n", shift);
-}
-
-
-void pgtable_cache_init(void)
-{
-	pgtable_cache_add(PGD_INDEX_SIZE, pgd_ctor);
-	pgtable_cache_add(PMD_CACHE_INDEX, pmd_ctor);
-	/*
-	 * In all current configs, when the PUD index exists it's the
-	 * same size as either the pgd or pmd index except with THP enabled
-	 * on book3s 64
-	 */
-	if (PUD_INDEX_SIZE && !PGT_CACHE(PUD_INDEX_SIZE))
-		pgtable_cache_add(PUD_INDEX_SIZE, pud_ctor);
-
-	if (!PGT_CACHE(PGD_INDEX_SIZE) || !PGT_CACHE(PMD_CACHE_INDEX))
-		panic("Couldn't allocate pgtable caches");
-	if (PUD_INDEX_SIZE && !PGT_CACHE(PUD_INDEX_SIZE))
-		panic("Couldn't allocate pud pgtable caches");
-}
-
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 /*
  * Given an address within the vmemmap, determine the pfn of the page that
diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
index b114f8b93ec9..73bf6e14c3aa 100644
--- a/arch/powerpc/mm/mmu_context_book3s64.c
+++ b/arch/powerpc/mm/mmu_context_book3s64.c
@@ -115,7 +115,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 	mm->context.pte_frag = NULL;
 #endif
 #ifdef CONFIG_SPAPR_TCE_IOMMU
-	mm_iommu_init(&mm->context);
+	mm_iommu_init(mm);
 #endif
 	return 0;
 }
@@ -156,13 +156,11 @@ static inline void destroy_pagetable_page(struct mm_struct *mm)
 }
 #endif
 
-
 void destroy_context(struct mm_struct *mm)
 {
 #ifdef CONFIG_SPAPR_TCE_IOMMU
-	mm_iommu_cleanup(&mm->context);
+	WARN_ON_ONCE(!list_empty(&mm->context.iommu_group_mem_list));
 #endif
-
 #ifdef CONFIG_PPC_ICSWX
 	drop_cop(mm->context.acop, mm);
 	kfree(mm->context.cop_lockp);
diff --git a/arch/powerpc/mm/mmu_context_iommu.c b/arch/powerpc/mm/mmu_context_iommu.c
index e0f1c33601dd..104bad029ce9 100644
--- a/arch/powerpc/mm/mmu_context_iommu.c
+++ b/arch/powerpc/mm/mmu_context_iommu.c
@@ -56,7 +56,7 @@ static long mm_iommu_adjust_locked_vm(struct mm_struct *mm,
 	}
 
 	pr_debug("[%d] RLIMIT_MEMLOCK HASH64 %c%ld %ld/%ld\n",
-			current->pid,
+			current ? current->pid : 0,
 			incr ? '+' : '-',
 			npages << PAGE_SHIFT,
 			mm->locked_vm << PAGE_SHIFT,
@@ -66,12 +66,9 @@ static long mm_iommu_adjust_locked_vm(struct mm_struct *mm,
 	return ret;
 }
 
-bool mm_iommu_preregistered(void)
+bool mm_iommu_preregistered(struct mm_struct *mm)
 {
-	if (!current || !current->mm)
-		return false;
-
-	return !list_empty(&current->mm->context.iommu_group_mem_list);
+	return !list_empty(&mm->context.iommu_group_mem_list);
 }
 EXPORT_SYMBOL_GPL(mm_iommu_preregistered);
 
@@ -124,19 +121,16 @@ static int mm_iommu_move_page_from_cma(struct page *page)
 	return 0;
 }
 
-long mm_iommu_get(unsigned long ua, unsigned long entries,
+long mm_iommu_get(struct mm_struct *mm, unsigned long ua, unsigned long entries,
 		struct mm_iommu_table_group_mem_t **pmem)
 {
 	struct mm_iommu_table_group_mem_t *mem;
 	long i, j, ret = 0, locked_entries = 0;
 	struct page *page = NULL;
 
-	if (!current || !current->mm)
-		return -ESRCH; /* process exited */
-
 	mutex_lock(&mem_list_mutex);
 
-	list_for_each_entry_rcu(mem, &current->mm->context.iommu_group_mem_list,
+	list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list,
 			next) {
 		if ((mem->ua == ua) && (mem->entries == entries)) {
 			++mem->used;
@@ -154,7 +148,7 @@ long mm_iommu_get(unsigned long ua, unsigned long entries,
 
 	}
 
-	ret = mm_iommu_adjust_locked_vm(current->mm, entries, true);
+	ret = mm_iommu_adjust_locked_vm(mm, entries, true);
 	if (ret)
 		goto unlock_exit;
 
@@ -215,11 +209,11 @@ populate:
 	mem->entries = entries;
 	*pmem = mem;
 
-	list_add_rcu(&mem->next, &current->mm->context.iommu_group_mem_list);
+	list_add_rcu(&mem->next, &mm->context.iommu_group_mem_list);
 
 unlock_exit:
 	if (locked_entries && ret)
-		mm_iommu_adjust_locked_vm(current->mm, locked_entries, false);
+		mm_iommu_adjust_locked_vm(mm, locked_entries, false);
 
 	mutex_unlock(&mem_list_mutex);
 
@@ -264,17 +258,13 @@ static void mm_iommu_free(struct rcu_head *head)
 static void mm_iommu_release(struct mm_iommu_table_group_mem_t *mem)
 {
 	list_del_rcu(&mem->next);
-	mm_iommu_adjust_locked_vm(current->mm, mem->entries, false);
 	call_rcu(&mem->rcu, mm_iommu_free);
 }
 
-long mm_iommu_put(struct mm_iommu_table_group_mem_t *mem)
+long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem)
 {
 	long ret = 0;
 
-	if (!current || !current->mm)
-		return -ESRCH; /* process exited */
-
 	mutex_lock(&mem_list_mutex);
 
 	if (mem->used == 0) {
@@ -297,6 +287,8 @@ long mm_iommu_put(struct mm_iommu_table_group_mem_t *mem)
 	/* @mapped became 0 so now mappings are disabled, release the region */
 	mm_iommu_release(mem);
 
+	mm_iommu_adjust_locked_vm(mm, mem->entries, false);
+
 unlock_exit:
 	mutex_unlock(&mem_list_mutex);
 
@@ -304,14 +296,12 @@ unlock_exit:
 }
 EXPORT_SYMBOL_GPL(mm_iommu_put);
 
-struct mm_iommu_table_group_mem_t *mm_iommu_lookup(unsigned long ua,
-		unsigned long size)
+struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm,
+		unsigned long ua, unsigned long size)
 {
 	struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
 
-	list_for_each_entry_rcu(mem,
-			&current->mm->context.iommu_group_mem_list,
-			next) {
+	list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
 		if ((mem->ua <= ua) &&
 				(ua + size <= mem->ua +
 				 (mem->entries << PAGE_SHIFT))) {
@@ -324,14 +314,12 @@ struct mm_iommu_table_group_mem_t *mm_iommu_lookup(unsigned long ua,
 }
 EXPORT_SYMBOL_GPL(mm_iommu_lookup);
 
-struct mm_iommu_table_group_mem_t *mm_iommu_find(unsigned long ua,
-		unsigned long entries)
+struct mm_iommu_table_group_mem_t *mm_iommu_find(struct mm_struct *mm,
+		unsigned long ua, unsigned long entries)
 {
 	struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
 
-	list_for_each_entry_rcu(mem,
-			&current->mm->context.iommu_group_mem_list,
-			next) {
+	list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
 		if ((mem->ua == ua) && (mem->entries == entries)) {
 			ret = mem;
 			break;
@@ -373,17 +361,7 @@ void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem)
 }
 EXPORT_SYMBOL_GPL(mm_iommu_mapped_dec);
 
-void mm_iommu_init(mm_context_t *ctx)
+void mm_iommu_init(struct mm_struct *mm)
 {
-	INIT_LIST_HEAD_RCU(&ctx->iommu_group_mem_list);
-}
-
-void mm_iommu_cleanup(mm_context_t *ctx)
-{
-	struct mm_iommu_table_group_mem_t *mem, *tmp;
-
-	list_for_each_entry_safe(mem, tmp, &ctx->iommu_group_mem_list, next) {
-		list_del_rcu(&mem->next);
-		mm_iommu_do_free(mem);
-	}
+	INIT_LIST_HEAD_RCU(&mm->context.iommu_group_mem_list);
 }
diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c
index f4f437cbabf1..ebf9782bacf9 100644
--- a/arch/powerpc/mm/pgtable-book3s64.c
+++ b/arch/powerpc/mm/pgtable-book3s64.c
@@ -35,7 +35,8 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
 #endif
 	changed = !pmd_same(*(pmdp), entry);
 	if (changed) {
-		__ptep_set_access_flags(vma->vm_mm, pmdp_ptep(pmdp), pmd_pte(entry));
+		__ptep_set_access_flags(vma->vm_mm, pmdp_ptep(pmdp),
+					pmd_pte(entry), address);
 		flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
 	}
 	return changed;
diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
index 688b54517655..cfa53ccc8baf 100644
--- a/arch/powerpc/mm/pgtable-radix.c
+++ b/arch/powerpc/mm/pgtable-radix.c
@@ -159,7 +159,7 @@ redo:
 	 * Allocate Partition table and process table for the
 	 * host.
 	 */
-	BUILD_BUG_ON_MSG((PRTB_SIZE_SHIFT > 23), "Process table size too large.");
+	BUILD_BUG_ON_MSG((PRTB_SIZE_SHIFT > 36), "Process table size too large.");
 	process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT);
 	/*
 	 * Fill in the process table.
@@ -177,23 +177,15 @@ redo:
 
 static void __init radix_init_partition_table(void)
 {
-	unsigned long rts_field;
+	unsigned long rts_field, dw0;
 
+	mmu_partition_table_init();
 	rts_field = radix__get_tree_size();
+	dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR;
+	mmu_partition_table_set_entry(0, dw0, 0);
 
-	BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 24), "Partition table size too large.");
-	partition_tb = early_alloc_pgtable(1UL << PATB_SIZE_SHIFT);
-	partition_tb->patb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) |
-					  RADIX_PGD_INDEX_SIZE | PATB_HR);
 	pr_info("Initializing Radix MMU\n");
 	pr_info("Partition table %p\n", partition_tb);
-
-	memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
-	/*
-	 * update partition table control register,
-	 * 64 K size.
-	 */
-	mtspr(SPRN_PTCR, __pa(partition_tb) | (PATB_SIZE_SHIFT - 12));
 }
 
 void __init radix_init_native(void)
@@ -248,7 +240,7 @@ static int __init radix_dt_scan_page_sizes(unsigned long node,
 		/* top 3 bit is AP encoding */
 		shift = be32_to_cpu(prop[0]) & ~(0xe << 28);
 		ap = be32_to_cpu(prop[0]) >> 29;
-		pr_info("Page size sift = %d AP=0x%x\n", shift, ap);
+		pr_info("Page size shift = %d AP=0x%x\n", shift, ap);
 
 		idx = get_idx_from_shift(shift);
 		if (idx < 0)
@@ -320,6 +312,38 @@ static void update_hid_for_radix(void)
 		cpu_relax();
 }
 
+static void radix_init_amor(void)
+{
+	/*
+	* In HV mode, we init AMOR (Authority Mask Override Register) so that
+	* the hypervisor and guest can setup IAMR (Instruction Authority Mask
+	* Register), enable key 0 and set it to 1.
+	*
+	* AMOR = 0b1100 .... 0000 (Mask for key 0 is 11)
+	*/
+	mtspr(SPRN_AMOR, (3ul << 62));
+}
+
+static void radix_init_iamr(void)
+{
+	unsigned long iamr;
+
+	/*
+	 * The IAMR should set to 0 on DD1.
+	 */
+	if (cpu_has_feature(CPU_FTR_POWER9_DD1))
+		iamr = 0;
+	else
+		iamr = (1ul << 62);
+
+	/*
+	 * Radix always uses key0 of the IAMR to determine if an access is
+	 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction
+	 * fetch.
+	 */
+	mtspr(SPRN_IAMR, iamr);
+}
+
 void __init radix__early_init_mmu(void)
 {
 	unsigned long lpcr;
@@ -376,8 +400,12 @@ void __init radix__early_init_mmu(void)
 		lpcr = mfspr(SPRN_LPCR);
 		mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
 		radix_init_partition_table();
+		radix_init_amor();
 	}
 
+	memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
+
+	radix_init_iamr();
 	radix_init_pgtable();
 }
 
@@ -397,7 +425,9 @@ void radix__early_init_mmu_secondary(void)
 
 		mtspr(SPRN_PTCR,
 		      __pa(partition_tb) | (PATB_SIZE_SHIFT - 12));
+		radix_init_amor();
 	}
+	radix_init_iamr();
 }
 
 void radix__mmu_cleanup_all(void)
diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c
index 911fdfb63ec1..cb39c8bd2436 100644
--- a/arch/powerpc/mm/pgtable.c
+++ b/arch/powerpc/mm/pgtable.c
@@ -224,7 +224,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address,
 	if (changed) {
 		if (!is_vm_hugetlb_page(vma))
 			assert_pte_locked(vma->vm_mm, address);
-		__ptep_set_access_flags(vma->vm_mm, ptep, entry);
+		__ptep_set_access_flags(vma->vm_mm, ptep, entry, address);
 		flush_tlb_page(vma, address);
 	}
 	return changed;
diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
index 0ae0572bc239..a65c0b4c0669 100644
--- a/arch/powerpc/mm/pgtable_32.c
+++ b/arch/powerpc/mm/pgtable_32.c
@@ -42,43 +42,6 @@ EXPORT_SYMBOL(ioremap_bot);	/* aka VMALLOC_END */
 
 extern char etext[], _stext[], _sinittext[], _einittext[];
 
-#define PGDIR_ORDER	(32 + PGD_T_LOG2 - PGDIR_SHIFT)
-
-#ifndef CONFIG_PPC_4K_PAGES
-static struct kmem_cache *pgtable_cache;
-
-void pgtable_cache_init(void)
-{
-	pgtable_cache = kmem_cache_create("PGDIR cache", 1 << PGDIR_ORDER,
-					  1 << PGDIR_ORDER, 0, NULL);
-	if (pgtable_cache == NULL)
-		panic("Couldn't allocate pgtable caches");
-}
-#endif
-
-pgd_t *pgd_alloc(struct mm_struct *mm)
-{
-	pgd_t *ret;
-
-	/* pgdir take page or two with 4K pages and a page fraction otherwise */
-#ifndef CONFIG_PPC_4K_PAGES
-	ret = kmem_cache_alloc(pgtable_cache, GFP_KERNEL | __GFP_ZERO);
-#else
-	ret = (pgd_t *)__get_free_pages(GFP_KERNEL|__GFP_ZERO,
-			PGDIR_ORDER - PAGE_SHIFT);
-#endif
-	return ret;
-}
-
-void pgd_free(struct mm_struct *mm, pgd_t *pgd)
-{
-#ifndef CONFIG_PPC_4K_PAGES
-	kmem_cache_free(pgtable_cache, (void *)pgd);
-#else
-	free_pages((unsigned long)pgd, PGDIR_ORDER - PAGE_SHIFT);
-#endif
-}
-
 __ref pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
 {
 	pte_t *pte;
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index f5e8d4edb808..8bca7f58afc4 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -431,3 +431,37 @@ void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
 	}
 }
 #endif
+
+#ifdef CONFIG_PPC_BOOK3S_64
+void __init mmu_partition_table_init(void)
+{
+	unsigned long patb_size = 1UL << PATB_SIZE_SHIFT;
+
+	BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 36), "Partition table size too large.");
+	partition_tb = __va(memblock_alloc_base(patb_size, patb_size,
+						MEMBLOCK_ALLOC_ANYWHERE));
+
+	/* Initialize the Partition Table with no entries */
+	memset((void *)partition_tb, 0, patb_size);
+
+	/*
+	 * update partition table control register,
+	 * 64 K size.
+	 */
+	mtspr(SPRN_PTCR, __pa(partition_tb) | (PATB_SIZE_SHIFT - 12));
+}
+
+void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
+				   unsigned long dw1)
+{
+	partition_tb[lpid].patb0 = cpu_to_be64(dw0);
+	partition_tb[lpid].patb1 = cpu_to_be64(dw1);
+
+	/* Global flush of TLBs and partition table caches for this lpid */
+	asm volatile("ptesync" : : : "memory");
+	asm volatile(PPC_TLBIE_5(%0,%1,2,0,0) : :
+		     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
+	asm volatile("eieio; tlbsync; ptesync" : : : "memory");
+}
+EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry);
+#endif /* CONFIG_PPC_BOOK3S_64 */
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 3493cf4e0452..61b79119065f 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -428,3 +428,21 @@ void radix__flush_tlb_all(void)
 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory");
 	asm volatile("eieio; tlbsync; ptesync": : :"memory");
 }
+
+void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
+				 unsigned long address)
+{
+	/*
+	 * We track page size in pte only for DD1, So we can
+	 * call this only on DD1.
+	 */
+	if (!cpu_has_feature(CPU_FTR_POWER9_DD1)) {
+		VM_WARN_ON(1);
+		return;
+	}
+
+	if (old_pte & _PAGE_LARGE)
+		radix__flush_tlb_page_psize(mm, address, MMU_PAGE_2M);
+	else
+		radix__flush_tlb_page_psize(mm, address, mmu_virtual_psize);
+}
diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c
index 050badc0ebd3..ba28fcb98597 100644
--- a/arch/powerpc/mm/tlb_nohash.c
+++ b/arch/powerpc/mm/tlb_nohash.c
@@ -53,7 +53,7 @@
  * other sizes not listed here.   The .ind field is only used on MMUs that have
  * indirect page table entries.
  */
-#ifdef CONFIG_PPC_BOOK3E_MMU
+#if defined(CONFIG_PPC_BOOK3E_MMU) || defined(CONFIG_PPC_8xx)
 #ifdef CONFIG_PPC_FSL_BOOK3E
 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
 	[MMU_PAGE_4K] = {
@@ -85,6 +85,25 @@ struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
 		.enc	= BOOK3E_PAGESZ_1GB,
 	},
 };
+#elif defined(CONFIG_PPC_8xx)
+struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
+	/* we only manage 4k and 16k pages as normal pages */
+#ifdef CONFIG_PPC_4K_PAGES
+	[MMU_PAGE_4K] = {
+		.shift	= 12,
+	},
+#else
+	[MMU_PAGE_16K] = {
+		.shift	= 14,
+	},
+#endif
+	[MMU_PAGE_512K] = {
+		.shift	= 19,
+	},
+	[MMU_PAGE_8M] = {
+		.shift	= 23,
+	},
+};
 #else
 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
 	[MMU_PAGE_4K] = {
diff --git a/arch/powerpc/perf/isa207-common.c b/arch/powerpc/perf/isa207-common.c
index 6143c99f3ec5..50e598cf644b 100644
--- a/arch/powerpc/perf/isa207-common.c
+++ b/arch/powerpc/perf/isa207-common.c
@@ -12,6 +12,40 @@
  */
 #include "isa207-common.h"
 
+PMU_FORMAT_ATTR(event,		"config:0-49");
+PMU_FORMAT_ATTR(pmcxsel,	"config:0-7");
+PMU_FORMAT_ATTR(mark,		"config:8");
+PMU_FORMAT_ATTR(combine,	"config:11");
+PMU_FORMAT_ATTR(unit,		"config:12-15");
+PMU_FORMAT_ATTR(pmc,		"config:16-19");
+PMU_FORMAT_ATTR(cache_sel,	"config:20-23");
+PMU_FORMAT_ATTR(sample_mode,	"config:24-28");
+PMU_FORMAT_ATTR(thresh_sel,	"config:29-31");
+PMU_FORMAT_ATTR(thresh_stop,	"config:32-35");
+PMU_FORMAT_ATTR(thresh_start,	"config:36-39");
+PMU_FORMAT_ATTR(thresh_cmp,	"config:40-49");
+
+struct attribute *isa207_pmu_format_attr[] = {
+	&format_attr_event.attr,
+	&format_attr_pmcxsel.attr,
+	&format_attr_mark.attr,
+	&format_attr_combine.attr,
+	&format_attr_unit.attr,
+	&format_attr_pmc.attr,
+	&format_attr_cache_sel.attr,
+	&format_attr_sample_mode.attr,
+	&format_attr_thresh_sel.attr,
+	&format_attr_thresh_stop.attr,
+	&format_attr_thresh_start.attr,
+	&format_attr_thresh_cmp.attr,
+	NULL,
+};
+
+struct attribute_group isa207_pmu_format_group = {
+	.name = "format",
+	.attrs = isa207_pmu_format_attr,
+};
+
 static inline bool event_is_fab_match(u64 event)
 {
 	/* Only check pmc, unit and pmcxsel, ignore the edge bit (0) */
@@ -21,6 +55,48 @@ static inline bool event_is_fab_match(u64 event)
 	return (event == 0x30056 || event == 0x4f052);
 }
 
+static bool is_event_valid(u64 event)
+{
+	u64 valid_mask = EVENT_VALID_MASK;
+
+	if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1))
+		valid_mask = p9_EVENT_VALID_MASK;
+
+	return !(event & ~valid_mask);
+}
+
+static u64 mmcra_sdar_mode(u64 event)
+{
+	if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1))
+		return p9_SDAR_MODE(event) << MMCRA_SDAR_MODE_SHIFT;
+
+	return MMCRA_SDAR_MODE_TLB;
+}
+
+static u64 thresh_cmp_val(u64 value)
+{
+	if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1))
+		return value << p9_MMCRA_THR_CMP_SHIFT;
+
+	return value << MMCRA_THR_CMP_SHIFT;
+}
+
+static unsigned long combine_from_event(u64 event)
+{
+	if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1))
+		return p9_EVENT_COMBINE(event);
+
+	return EVENT_COMBINE(event);
+}
+
+static unsigned long combine_shift(unsigned long pmc)
+{
+	if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1))
+		return p9_MMCR1_COMBINE_SHIFT(pmc);
+
+	return MMCR1_COMBINE_SHIFT(pmc);
+}
+
 int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp)
 {
 	unsigned int unit, pmc, cache, ebb;
@@ -28,7 +104,7 @@ int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp)
 
 	mask = value = 0;
 
-	if (event & ~EVENT_VALID_MASK)
+	if (!is_event_valid(event))
 		return -1;
 
 	pmc   = (event >> EVENT_PMC_SHIFT)        & EVENT_PMC_MASK;
@@ -155,15 +231,13 @@ int isa207_compute_mmcr(u64 event[], int n_ev,
 			pmc_inuse |= 1 << pmc;
 	}
 
-	/* In continuous sampling mode, update SDAR on TLB miss */
-	mmcra = MMCRA_SDAR_MODE_TLB;
-	mmcr1 = mmcr2 = 0;
+	mmcra = mmcr1 = mmcr2 = 0;
 
 	/* Second pass: assign PMCs, set all MMCR1 fields */
 	for (i = 0; i < n_ev; ++i) {
 		pmc     = (event[i] >> EVENT_PMC_SHIFT) & EVENT_PMC_MASK;
 		unit    = (event[i] >> EVENT_UNIT_SHIFT) & EVENT_UNIT_MASK;
-		combine = (event[i] >> EVENT_COMBINE_SHIFT) & EVENT_COMBINE_MASK;
+		combine = combine_from_event(event[i]);
 		psel    =  event[i] & EVENT_PSEL_MASK;
 
 		if (!pmc) {
@@ -177,10 +251,13 @@ int isa207_compute_mmcr(u64 event[], int n_ev,
 
 		if (pmc <= 4) {
 			mmcr1 |= unit << MMCR1_UNIT_SHIFT(pmc);
-			mmcr1 |= combine << MMCR1_COMBINE_SHIFT(pmc);
+			mmcr1 |= combine << combine_shift(pmc);
 			mmcr1 |= psel << MMCR1_PMCSEL_SHIFT(pmc);
 		}
 
+		/* In continuous sampling mode, update SDAR on TLB miss */
+		mmcra |= mmcra_sdar_mode(event[i]);
+
 		if (event[i] & EVENT_IS_L1) {
 			cache = event[i] >> EVENT_CACHE_SEL_SHIFT;
 			mmcr1 |= (cache & 1) << MMCR1_IC_QUAL_SHIFT;
@@ -211,7 +288,7 @@ int isa207_compute_mmcr(u64 event[], int n_ev,
 			val = (event[i] >> EVENT_THR_SEL_SHIFT) & EVENT_THR_SEL_MASK;
 			mmcra |= val << MMCRA_THR_SEL_SHIFT;
 			val = (event[i] >> EVENT_THR_CMP_SHIFT) & EVENT_THR_CMP_MASK;
-			mmcra |= val << MMCRA_THR_CMP_SHIFT;
+			mmcra |= thresh_cmp_val(val);
 		}
 
 		if (event[i] & EVENT_WANTS_BHRB) {
diff --git a/arch/powerpc/perf/isa207-common.h b/arch/powerpc/perf/isa207-common.h
index 4d0a4e5017c2..90495f1580c7 100644
--- a/arch/powerpc/perf/isa207-common.h
+++ b/arch/powerpc/perf/isa207-common.h
@@ -107,6 +107,7 @@
 #define EVENT_UNIT_MASK		0xf
 #define EVENT_COMBINE_SHIFT	11	/* Combine bit */
 #define EVENT_COMBINE_MASK	0x1
+#define EVENT_COMBINE(v)	(((v) >> EVENT_COMBINE_SHIFT) & EVENT_COMBINE_MASK)
 #define EVENT_MARKED_SHIFT	8	/* Marked bit */
 #define EVENT_MARKED_MASK	0x1
 #define EVENT_IS_MARKED		(EVENT_MARKED_MASK << EVENT_MARKED_SHIFT)
@@ -134,6 +135,26 @@
 	 PERF_SAMPLE_BRANCH_KERNEL      |\
 	 PERF_SAMPLE_BRANCH_HV)
 
+/* Contants to support power9 raw encoding format */
+#define p9_EVENT_COMBINE_SHIFT	10	/* Combine bit */
+#define p9_EVENT_COMBINE_MASK	0x3ull
+#define p9_EVENT_COMBINE(v)	(((v) >> p9_EVENT_COMBINE_SHIFT) & p9_EVENT_COMBINE_MASK)
+#define p9_SDAR_MODE_SHIFT	50
+#define p9_SDAR_MODE_MASK	0x3ull
+#define p9_SDAR_MODE(v)		(((v) >> p9_SDAR_MODE_SHIFT) & p9_SDAR_MODE_MASK)
+
+#define p9_EVENT_VALID_MASK		\
+	((p9_SDAR_MODE_MASK   << p9_SDAR_MODE_SHIFT		|	\
+	(EVENT_THRESH_MASK    << EVENT_THRESH_SHIFT)		|	\
+	(EVENT_SAMPLE_MASK    << EVENT_SAMPLE_SHIFT)		|	\
+	(EVENT_CACHE_SEL_MASK << EVENT_CACHE_SEL_SHIFT)		|	\
+	(EVENT_PMC_MASK       << EVENT_PMC_SHIFT)		|	\
+	(EVENT_UNIT_MASK      << EVENT_UNIT_SHIFT)		|	\
+	(p9_EVENT_COMBINE_MASK << p9_EVENT_COMBINE_SHIFT)	|	\
+	(EVENT_MARKED_MASK    << EVENT_MARKED_SHIFT)		|	\
+	 EVENT_LINUX_MASK					|	\
+	 EVENT_PSEL_MASK))
+
 /*
  * Layout of constraint bits:
  *
@@ -210,15 +231,22 @@
 #define MMCR1_DC_QUAL_SHIFT		47
 #define MMCR1_IC_QUAL_SHIFT		46
 
+/* MMCR1 Combine bits macro for power9 */
+#define p9_MMCR1_COMBINE_SHIFT(pmc)	(38 - ((pmc - 1) * 2))
+
 /* Bits in MMCRA for PowerISA v2.07 */
 #define MMCRA_SAMP_MODE_SHIFT		1
 #define MMCRA_SAMP_ELIG_SHIFT		4
 #define MMCRA_THR_CTL_SHIFT		8
 #define MMCRA_THR_SEL_SHIFT		16
 #define MMCRA_THR_CMP_SHIFT		32
-#define MMCRA_SDAR_MODE_TLB		(1ull << 42)
+#define MMCRA_SDAR_MODE_SHIFT		42
+#define MMCRA_SDAR_MODE_TLB		(1ull << MMCRA_SDAR_MODE_SHIFT)
 #define MMCRA_IFM_SHIFT			30
 
+/* MMCR1 Threshold Compare bit constant for power9 */
+#define p9_MMCRA_THR_CMP_SHIFT	45
+
 /* Bits in MMCR2 for PowerISA v2.07 */
 #define MMCR2_FCS(pmc)			(1ull << (63 - (((pmc) - 1) * 9)))
 #define MMCR2_FCP(pmc)			(1ull << (62 - (((pmc) - 1) * 9)))
diff --git a/arch/powerpc/perf/power8-pmu.c b/arch/powerpc/perf/power8-pmu.c
index ab830d106ec5..d07186382f3a 100644
--- a/arch/powerpc/perf/power8-pmu.c
+++ b/arch/powerpc/perf/power8-pmu.c
@@ -30,6 +30,9 @@ enum {
 #define	POWER8_MMCRA_IFM2		0x0000000080000000UL
 #define	POWER8_MMCRA_IFM3		0x00000000C0000000UL
 
+/* PowerISA v2.07 format attribute structure*/
+extern struct attribute_group isa207_pmu_format_group;
+
 /* Table of alternatives, sorted by column 0 */
 static const unsigned int event_alternatives[][MAX_ALT] = {
 	{ PM_MRK_ST_CMPL,		PM_MRK_ST_CMPL_ALT },
@@ -175,42 +178,8 @@ static struct attribute_group power8_pmu_events_group = {
 	.attrs = power8_events_attr,
 };
 
-PMU_FORMAT_ATTR(event,		"config:0-49");
-PMU_FORMAT_ATTR(pmcxsel,	"config:0-7");
-PMU_FORMAT_ATTR(mark,		"config:8");
-PMU_FORMAT_ATTR(combine,	"config:11");
-PMU_FORMAT_ATTR(unit,		"config:12-15");
-PMU_FORMAT_ATTR(pmc,		"config:16-19");
-PMU_FORMAT_ATTR(cache_sel,	"config:20-23");
-PMU_FORMAT_ATTR(sample_mode,	"config:24-28");
-PMU_FORMAT_ATTR(thresh_sel,	"config:29-31");
-PMU_FORMAT_ATTR(thresh_stop,	"config:32-35");
-PMU_FORMAT_ATTR(thresh_start,	"config:36-39");
-PMU_FORMAT_ATTR(thresh_cmp,	"config:40-49");
-
-static struct attribute *power8_pmu_format_attr[] = {
-	&format_attr_event.attr,
-	&format_attr_pmcxsel.attr,
-	&format_attr_mark.attr,
-	&format_attr_combine.attr,
-	&format_attr_unit.attr,
-	&format_attr_pmc.attr,
-	&format_attr_cache_sel.attr,
-	&format_attr_sample_mode.attr,
-	&format_attr_thresh_sel.attr,
-	&format_attr_thresh_stop.attr,
-	&format_attr_thresh_start.attr,
-	&format_attr_thresh_cmp.attr,
-	NULL,
-};
-
-static struct attribute_group power8_pmu_format_group = {
-	.name = "format",
-	.attrs = power8_pmu_format_attr,
-};
-
 static const struct attribute_group *power8_pmu_attr_groups[] = {
-	&power8_pmu_format_group,
+	&isa207_pmu_format_group,
 	&power8_pmu_events_group,
 	NULL,
 };
diff --git a/arch/powerpc/perf/power9-pmu.c b/arch/powerpc/perf/power9-pmu.c
index 8e9a81967ff8..346010e8d463 100644
--- a/arch/powerpc/perf/power9-pmu.c
+++ b/arch/powerpc/perf/power9-pmu.c
@@ -16,6 +16,78 @@
 #include "isa207-common.h"
 
 /*
+ * Raw event encoding for Power9:
+ *
+ *        60        56        52        48        44        40        36        32
+ * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - |
+ *   | | [ ]                       [ ] [      thresh_cmp     ]   [  thresh_ctl   ]
+ *   | |  |                         |                                     |
+ *   | |  *- IFM (Linux)            |    thresh start/stop OR FAB match -*
+ *   | *- BHRB (Linux)              *sm
+ *   *- EBB (Linux)
+ *
+ *        28        24        20        16        12         8         4         0
+ * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - |
+ *   [   ] [  sample ]   [cache]   [ pmc ]   [unit ]   []    m   [    pmcxsel    ]
+ *     |        |           |                          |     |
+ *     |        |           |                          |     *- mark
+ *     |        |           *- L1/L2/L3 cache_sel      |
+ *     |        |                                      |
+ *     |        *- sampling mode for marked events     *- combine
+ *     |
+ *     *- thresh_sel
+ *
+ * Below uses IBM bit numbering.
+ *
+ * MMCR1[x:y] = unit    (PMCxUNIT)
+ * MMCR1[24]   = pmc1combine[0]
+ * MMCR1[25]   = pmc1combine[1]
+ * MMCR1[26]   = pmc2combine[0]
+ * MMCR1[27]   = pmc2combine[1]
+ * MMCR1[28]   = pmc3combine[0]
+ * MMCR1[29]   = pmc3combine[1]
+ * MMCR1[30]   = pmc4combine[0]
+ * MMCR1[31]   = pmc4combine[1]
+ *
+ * if pmc == 3 and unit == 0 and pmcxsel[0:6] == 0b0101011
+ *	# PM_MRK_FAB_RSP_MATCH
+ *	MMCR1[20:27] = thresh_ctl   (FAB_CRESP_MATCH / FAB_TYPE_MATCH)
+ * else if pmc == 4 and unit == 0xf and pmcxsel[0:6] == 0b0101001
+ *	# PM_MRK_FAB_RSP_MATCH_CYC
+ *	MMCR1[20:27] = thresh_ctl   (FAB_CRESP_MATCH / FAB_TYPE_MATCH)
+ * else
+ *	MMCRA[48:55] = thresh_ctl   (THRESH START/END)
+ *
+ * if thresh_sel:
+ *	MMCRA[45:47] = thresh_sel
+ *
+ * if thresh_cmp:
+ *	MMCRA[9:11] = thresh_cmp[0:2]
+ *	MMCRA[12:18] = thresh_cmp[3:9]
+ *
+ * if unit == 6 or unit == 7
+ *	MMCRC[53:55] = cache_sel[1:3]      (L2EVENT_SEL)
+ * else if unit == 8 or unit == 9:
+ *	if cache_sel[0] == 0: # L3 bank
+ *		MMCRC[47:49] = cache_sel[1:3]  (L3EVENT_SEL0)
+ *	else if cache_sel[0] == 1:
+ *		MMCRC[50:51] = cache_sel[2:3]  (L3EVENT_SEL1)
+ * else if cache_sel[1]: # L1 event
+ *	MMCR1[16] = cache_sel[2]
+ *	MMCR1[17] = cache_sel[3]
+ *
+ * if mark:
+ *	MMCRA[63]    = 1		(SAMPLE_ENABLE)
+ *	MMCRA[57:59] = sample[0:2]	(RAND_SAMP_ELIG)
+ *	MMCRA[61:62] = sample[3:4]	(RAND_SAMP_MODE)
+ *
+ * if EBB and BHRB:
+ *	MMCRA[32:33] = IFM
+ *
+ * MMCRA[SDAR_MODE]  = sm
+ */
+
+/*
  * Some power9 event codes.
  */
 #define EVENT(_name, _code)	_name = _code,
@@ -31,6 +103,9 @@ enum {
 #define POWER9_MMCRA_IFM2		0x0000000080000000UL
 #define POWER9_MMCRA_IFM3		0x00000000C0000000UL
 
+/* PowerISA v2.07 format attribute structure*/
+extern struct attribute_group isa207_pmu_format_group;
+
 GENERIC_EVENT_ATTR(cpu-cycles,			PM_CYC);
 GENERIC_EVENT_ATTR(stalled-cycles-frontend,	PM_ICT_NOSLOT_CYC);
 GENERIC_EVENT_ATTR(stalled-cycles-backend,	PM_CMPLU_STALL);
@@ -90,10 +165,16 @@ static struct attribute_group power9_pmu_events_group = {
 	.attrs = power9_events_attr,
 };
 
-PMU_FORMAT_ATTR(event,		"config:0-49");
+static const struct attribute_group *power9_isa207_pmu_attr_groups[] = {
+	&isa207_pmu_format_group,
+	&power9_pmu_events_group,
+	NULL,
+};
+
+PMU_FORMAT_ATTR(event,		"config:0-51");
 PMU_FORMAT_ATTR(pmcxsel,	"config:0-7");
 PMU_FORMAT_ATTR(mark,		"config:8");
-PMU_FORMAT_ATTR(combine,	"config:11");
+PMU_FORMAT_ATTR(combine,	"config:10-11");
 PMU_FORMAT_ATTR(unit,		"config:12-15");
 PMU_FORMAT_ATTR(pmc,		"config:16-19");
 PMU_FORMAT_ATTR(cache_sel,	"config:20-23");
@@ -102,6 +183,7 @@ PMU_FORMAT_ATTR(thresh_sel,	"config:29-31");
 PMU_FORMAT_ATTR(thresh_stop,	"config:32-35");
 PMU_FORMAT_ATTR(thresh_start,	"config:36-39");
 PMU_FORMAT_ATTR(thresh_cmp,	"config:40-49");
+PMU_FORMAT_ATTR(sdar_mode,	"config:50-51");
 
 static struct attribute *power9_pmu_format_attr[] = {
 	&format_attr_event.attr,
@@ -116,6 +198,7 @@ static struct attribute *power9_pmu_format_attr[] = {
 	&format_attr_thresh_stop.attr,
 	&format_attr_thresh_start.attr,
 	&format_attr_thresh_cmp.attr,
+	&format_attr_sdar_mode.attr,
 	NULL,
 };
 
@@ -291,6 +374,24 @@ static int power9_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
 
 #undef C
 
+static struct power_pmu power9_isa207_pmu = {
+	.name			= "POWER9",
+	.n_counter		= MAX_PMU_COUNTERS,
+	.add_fields		= ISA207_ADD_FIELDS,
+	.test_adder		= ISA207_TEST_ADDER,
+	.compute_mmcr		= isa207_compute_mmcr,
+	.config_bhrb		= power9_config_bhrb,
+	.bhrb_filter_map	= power9_bhrb_filter_map,
+	.get_constraint		= isa207_get_constraint,
+	.disable_pmc		= isa207_disable_pmc,
+	.flags			= PPMU_HAS_SIER | PPMU_ARCH_207S,
+	.n_generic		= ARRAY_SIZE(power9_generic_events),
+	.generic_events		= power9_generic_events,
+	.cache_events		= &power9_cache_events,
+	.attr_groups		= power9_isa207_pmu_attr_groups,
+	.bhrb_nr		= 32,
+};
+
 static struct power_pmu power9_pmu = {
 	.name			= "POWER9",
 	.n_counter		= MAX_PMU_COUNTERS,
@@ -311,14 +412,19 @@ static struct power_pmu power9_pmu = {
 
 static int __init init_power9_pmu(void)
 {
-	int rc;
+	int rc = 0;
 
 	/* Comes from cpu_specs[] */
 	if (!cur_cpu_spec->oprofile_cpu_type ||
 	    strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/power9"))
 		return -ENODEV;
 
-	rc = register_power_pmu(&power9_pmu);
+	if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
+		rc = register_power_pmu(&power9_isa207_pmu);
+	} else {
+		rc = register_power_pmu(&power9_pmu);
+	}
+
 	if (rc)
 		return rc;
 
diff --git a/arch/powerpc/platforms/40x/Kconfig b/arch/powerpc/platforms/40x/Kconfig
index 1d7c1b142bf4..abc24501c4c0 100644
--- a/arch/powerpc/platforms/40x/Kconfig
+++ b/arch/powerpc/platforms/40x/Kconfig
@@ -103,18 +103,18 @@ config 405GP
 	bool
 	select IBM405_ERR77
 	select IBM405_ERR51
-	select IBM_EMAC_ZMII
+	select IBM_EMAC_ZMII if IBM_EMAC
 
 config 405EX
 	bool
-	select IBM_EMAC_EMAC4
-	select IBM_EMAC_RGMII
+	select IBM_EMAC_EMAC4 if IBM_EMAC
+	select IBM_EMAC_RGMII if IBM_EMAC
 
 config 405EZ
 	bool
-	select IBM_EMAC_NO_FLOW_CTRL
-	select IBM_EMAC_MAL_CLR_ICINTSTAT
-	select IBM_EMAC_MAL_COMMON_ERR
+	select IBM_EMAC_NO_FLOW_CTRL if IBM_EMAC
+	select IBM_EMAC_MAL_CLR_ICINTSTAT if IBM_EMAC
+	select IBM_EMAC_MAL_COMMON_ERR if IBM_EMAC
 
 config XILINX_VIRTEX
 	bool
diff --git a/arch/powerpc/platforms/44x/Kconfig b/arch/powerpc/platforms/44x/Kconfig
index 25b8d641ff9f..9b0afe935cc1 100644
--- a/arch/powerpc/platforms/44x/Kconfig
+++ b/arch/powerpc/platforms/44x/Kconfig
@@ -26,7 +26,7 @@ config BLUESTONE
 	select PCI_MSI
 	select PPC4xx_MSI
 	select PPC4xx_PCI_EXPRESS
-	select IBM_EMAC_RGMII
+	select IBM_EMAC_RGMII if IBM_EMAC
 	help
 	  This option enables support for the APM APM821xx Evaluation board.
 
@@ -125,8 +125,8 @@ config CANYONLANDS
 	select PPC4xx_PCI_EXPRESS
 	select PCI_MSI
 	select PPC4xx_MSI
-	select IBM_EMAC_RGMII
-	select IBM_EMAC_ZMII
+	select IBM_EMAC_RGMII if IBM_EMAC
+	select IBM_EMAC_ZMII if IBM_EMAC
 	help
 	  This option enables support for the AMCC PPC460EX evaluation board.
 
@@ -138,8 +138,8 @@ config GLACIER
 	select 460EX # Odd since it uses 460GT but the effects are the same
 	select PCI
 	select PPC4xx_PCI_EXPRESS
-	select IBM_EMAC_RGMII
-	select IBM_EMAC_ZMII
+	select IBM_EMAC_RGMII if IBM_EMAC
+	select IBM_EMAC_ZMII if IBM_EMAC
 	help
 	  This option enables support for the AMCC PPC460GT evaluation board.
 
@@ -164,7 +164,7 @@ config EIGER
 	select 460SX
 	select PCI
 	select PPC4xx_PCI_EXPRESS
-	select IBM_EMAC_RGMII
+	select IBM_EMAC_RGMII if IBM_EMAC
 	help
 	  This option enables support for the AMCC PPC460SX evaluation board.
 
@@ -213,7 +213,7 @@ config AKEBONO
 	select NETDEVICES
 	select ETHERNET
 	select NET_VENDOR_IBM
-	select IBM_EMAC_EMAC4
+	select IBM_EMAC_EMAC4 if IBM_EMAC
 	select USB if USB_SUPPORT
 	select USB_OHCI_HCD_PLATFORM if USB_OHCI_HCD
 	select USB_EHCI_HCD_PLATFORM if USB_EHCI_HCD
@@ -291,54 +291,54 @@ config 440EP
 	bool
 	select PPC_FPU
 	select IBM440EP_ERR42
-	select IBM_EMAC_ZMII
+	select IBM_EMAC_ZMII if IBM_EMAC
 
 config 440EPX
 	bool
 	select PPC_FPU
-	select IBM_EMAC_EMAC4
-	select IBM_EMAC_RGMII
-	select IBM_EMAC_ZMII
+	select IBM_EMAC_EMAC4 if IBM_EMAC
+	select IBM_EMAC_RGMII if IBM_EMAC
+	select IBM_EMAC_ZMII if IBM_EMAC
 	select USB_EHCI_BIG_ENDIAN_MMIO
 	select USB_EHCI_BIG_ENDIAN_DESC
 
 config 440GRX
 	bool
-	select IBM_EMAC_EMAC4
-	select IBM_EMAC_RGMII
-	select IBM_EMAC_ZMII
+	select IBM_EMAC_EMAC4 if IBM_EMAC
+	select IBM_EMAC_RGMII if IBM_EMAC
+	select IBM_EMAC_ZMII if IBM_EMAC
 
 config 440GP
 	bool
-	select IBM_EMAC_ZMII
+	select IBM_EMAC_ZMII if IBM_EMAC
 
 config 440GX
 	bool
-	select IBM_EMAC_EMAC4
-	select IBM_EMAC_RGMII
-	select IBM_EMAC_ZMII #test only
-	select IBM_EMAC_TAH  #test only
+	select IBM_EMAC_EMAC4 if IBM_EMAC
+	select IBM_EMAC_RGMII if IBM_EMAC
+	select IBM_EMAC_ZMII if IBM_EMAC #test only
+	select IBM_EMAC_TAH if IBM_EMAC  #test only
 
 config 440SP
 	bool
 
 config 440SPe
 	bool
-	select IBM_EMAC_EMAC4
+	select IBM_EMAC_EMAC4 if IBM_EMAC
 
 config 460EX
 	bool
 	select PPC_FPU
-	select IBM_EMAC_EMAC4
-	select IBM_EMAC_TAH
+	select IBM_EMAC_EMAC4 if IBM_EMAC
+	select IBM_EMAC_TAH if IBM_EMAC
 
 config 460SX
 	bool
 	select PPC_FPU
-	select IBM_EMAC_EMAC4
-	select IBM_EMAC_RGMII
-	select IBM_EMAC_ZMII
-	select IBM_EMAC_TAH
+	select IBM_EMAC_EMAC4 if IBM_EMAC
+	select IBM_EMAC_RGMII if IBM_EMAC
+	select IBM_EMAC_ZMII if IBM_EMAC
+	select IBM_EMAC_TAH if IBM_EMAC
 
 config 476FPE
 	bool
@@ -347,8 +347,8 @@ config 476FPE
 config APM821xx
 	bool
 	select PPC_FPU
-	select IBM_EMAC_EMAC4
-	select IBM_EMAC_TAH
+	select IBM_EMAC_EMAC4 if IBM_EMAC
+	select IBM_EMAC_TAH if IBM_EMAC
 
 config 476FPE_ERR46
 	depends on 476FPE
diff --git a/arch/powerpc/platforms/83xx/suspend.c b/arch/powerpc/platforms/83xx/suspend.c
index 24717d060008..08f92f6ed228 100644
--- a/arch/powerpc/platforms/83xx/suspend.c
+++ b/arch/powerpc/platforms/83xx/suspend.c
@@ -441,8 +441,4 @@ static struct platform_driver pmc_driver = {
 	.remove = pmc_remove
 };
 
-static int pmc_init(void)
-{
-	return platform_driver_register(&pmc_driver);
-}
-device_initcall(pmc_init);
+builtin_platform_driver(pmc_driver);
diff --git a/arch/powerpc/platforms/85xx/Kconfig b/arch/powerpc/platforms/85xx/Kconfig
index 9dc1d28975b9..47b389dc4938 100644
--- a/arch/powerpc/platforms/85xx/Kconfig
+++ b/arch/powerpc/platforms/85xx/Kconfig
@@ -253,6 +253,8 @@ endif # PPC32
 config PPC_QEMU_E500
 	bool "QEMU generic e500 platform"
 	select DEFAULT_UIMAGE
+	select E500
+	select PPC_E500MC if PPC64
 	help
 	  This option enables support for running as a QEMU guest using
 	  QEMU's generic e500 machine.  This is not required if you're
diff --git a/arch/powerpc/platforms/85xx/corenet_generic.c b/arch/powerpc/platforms/85xx/corenet_generic.c
index 1179115a4b5c..3803b0addf65 100644
--- a/arch/powerpc/platforms/85xx/corenet_generic.c
+++ b/arch/powerpc/platforms/85xx/corenet_generic.c
@@ -220,7 +220,7 @@ define_machine(corenet_generic) {
  *
  * Likewise, problems have been seen with kexec when coreint is enabled.
  */
-#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_KEXEC)
+#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_KEXEC_CORE)
 	.get_irq		= mpic_get_irq,
 #else
 	.get_irq		= mpic_get_coreint_irq,
diff --git a/arch/powerpc/platforms/85xx/smp.c b/arch/powerpc/platforms/85xx/smp.c
index fe9f19e5e935..a83a6d26090d 100644
--- a/arch/powerpc/platforms/85xx/smp.c
+++ b/arch/powerpc/platforms/85xx/smp.c
@@ -349,13 +349,13 @@ struct smp_ops_t smp_85xx_ops = {
 	.cpu_disable	= generic_cpu_disable,
 	.cpu_die	= generic_cpu_die,
 #endif
-#if defined(CONFIG_KEXEC) && !defined(CONFIG_PPC64)
+#if defined(CONFIG_KEXEC_CORE) && !defined(CONFIG_PPC64)
 	.give_timebase	= smp_generic_give_timebase,
 	.take_timebase	= smp_generic_take_timebase,
 #endif
 };
 
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 #ifdef CONFIG_PPC32
 atomic_t kexec_down_cpus = ATOMIC_INIT(0);
 
@@ -458,7 +458,7 @@ static void mpc85xx_smp_machine_kexec(struct kimage *image)
 
 	default_machine_kexec(image);
 }
-#endif /* CONFIG_KEXEC */
+#endif /* CONFIG_KEXEC_CORE */
 
 static void smp_85xx_basic_setup(int cpu_nr)
 {
@@ -512,7 +512,7 @@ void __init mpc85xx_smp_init(void)
 #endif
 	smp_ops = &smp_85xx_ops;
 
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 	ppc_md.kexec_cpu_down = mpc85xx_smp_kexec_cpu_down;
 	ppc_md.machine_kexec = mpc85xx_smp_machine_kexec;
 #endif
diff --git a/arch/powerpc/platforms/8xx/Kconfig b/arch/powerpc/platforms/8xx/Kconfig
index 564d99bb2a26..80cbcb0ad9b1 100644
--- a/arch/powerpc/platforms/8xx/Kconfig
+++ b/arch/powerpc/platforms/8xx/Kconfig
@@ -130,6 +130,7 @@ config 8xx_CPU6
 
 config 8xx_CPU15
 	bool "CPU15 Silicon Errata"
+	depends on !HUGETLB_PAGE
 	default y
 	help
 	  This enables a workaround for erratum CPU15 on MPC8xx chips.
diff --git a/arch/powerpc/platforms/Kconfig b/arch/powerpc/platforms/Kconfig
index fbdae8377b71..7e3a2ebba29b 100644
--- a/arch/powerpc/platforms/Kconfig
+++ b/arch/powerpc/platforms/Kconfig
@@ -168,17 +168,6 @@ config MPIC_BROKEN_REGREAD
 	  well, but enabling it uses about 8KB of memory to keep copies
 	  of the register contents in software.
 
-config IBMVIO
-	depends on PPC_PSERIES
-	bool
-	default y
-
-config IBMEBUS
-	depends on PPC_PSERIES
-	bool "Support for GX bus based adapters"
-	help
-	  Bus device driver for GX bus based adapters.
-
 config EEH
 	bool
 	depends on (PPC_POWERNV || PPC_PSERIES) && PCI
diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
index ca2da30ad2ab..6e89e5a8d4fb 100644
--- a/arch/powerpc/platforms/Kconfig.cputype
+++ b/arch/powerpc/platforms/Kconfig.cputype
@@ -34,6 +34,7 @@ config PPC_8xx
 	select FSL_SOC
 	select 8xx
 	select PPC_LIB_RHEAP
+	select SYS_SUPPORTS_HUGETLBFS
 
 config 40x
 	bool "AMCC 40x"
diff --git a/arch/powerpc/platforms/cell/Kconfig b/arch/powerpc/platforms/cell/Kconfig
index d9088f0b8fcc..a4522f09d65e 100644
--- a/arch/powerpc/platforms/cell/Kconfig
+++ b/arch/powerpc/platforms/cell/Kconfig
@@ -17,10 +17,10 @@ config PPC_CELL_NATIVE
 	select PPC_CELL_COMMON
 	select MPIC
 	select PPC_IO_WORKAROUNDS
-	select IBM_EMAC_EMAC4
-	select IBM_EMAC_RGMII
-	select IBM_EMAC_ZMII #test only
-	select IBM_EMAC_TAH  #test only
+	select IBM_EMAC_EMAC4 if IBM_EMAC
+	select IBM_EMAC_RGMII if IBM_EMAC
+	select IBM_EMAC_ZMII if IBM_EMAC #test only
+	select IBM_EMAC_TAH if IBM_EMAC  #test only
 	default n
 
 config PPC_IBM_CELL_BLADE
@@ -46,7 +46,6 @@ config SPU_FS
 	default m
 	depends on PPC_CELL
 	select SPU_BASE
-	select MEMORY_HOTPLUG
 	help
 	  The SPU file system is used to access Synergistic Processing
 	  Units on machines implementing the Broadband Processor
diff --git a/arch/powerpc/platforms/cell/spu_base.c b/arch/powerpc/platforms/cell/spu_base.c
index e84d8fbc2e21..96c2b8a40630 100644
--- a/arch/powerpc/platforms/cell/spu_base.c
+++ b/arch/powerpc/platforms/cell/spu_base.c
@@ -676,7 +676,7 @@ static ssize_t spu_stat_show(struct device *dev,
 
 static DEVICE_ATTR(stat, 0444, spu_stat_show, NULL);
 
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 
 struct crash_spu_info {
 	struct spu *spu;
diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c
index 06254467e4dd..3a147122bc98 100644
--- a/arch/powerpc/platforms/cell/spufs/file.c
+++ b/arch/powerpc/platforms/cell/spufs/file.c
@@ -236,7 +236,6 @@ static int
 spufs_mem_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct spu_context *ctx	= vma->vm_file->private_data;
-	unsigned long address = (unsigned long)vmf->virtual_address;
 	unsigned long pfn, offset;
 
 	offset = vmf->pgoff << PAGE_SHIFT;
@@ -244,7 +243,7 @@ spufs_mem_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 		return VM_FAULT_SIGBUS;
 
 	pr_debug("spufs_mem_mmap_fault address=0x%lx, offset=0x%lx\n",
-			address, offset);
+			vmf->address, offset);
 
 	if (spu_acquire(ctx))
 		return VM_FAULT_NOPAGE;
@@ -256,7 +255,7 @@ spufs_mem_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 		vma->vm_page_prot = pgprot_noncached_wc(vma->vm_page_prot);
 		pfn = (ctx->spu->local_store_phys + offset) >> PAGE_SHIFT;
 	}
-	vm_insert_pfn(vma, address, pfn);
+	vm_insert_pfn(vma, vmf->address, pfn);
 
 	spu_release(ctx);
 
@@ -355,8 +354,7 @@ static int spufs_ps_fault(struct vm_area_struct *vma,
 		down_read(&current->mm->mmap_sem);
 	} else {
 		area = ctx->spu->problem_phys + ps_offs;
-		vm_insert_pfn(vma, (unsigned long)vmf->virtual_address,
-					(area + offset) >> PAGE_SHIFT);
+		vm_insert_pfn(vma, vmf->address, (area + offset) >> PAGE_SHIFT);
 		spu_context_trace(spufs_ps_fault__insert, ctx, ctx->spu);
 	}
 
diff --git a/arch/powerpc/platforms/embedded6xx/holly.c b/arch/powerpc/platforms/embedded6xx/holly.c
index dfd310031549..0409714e8070 100644
--- a/arch/powerpc/platforms/embedded6xx/holly.c
+++ b/arch/powerpc/platforms/embedded6xx/holly.c
@@ -263,7 +263,7 @@ static int ppc750_machine_check_exception(struct pt_regs *regs)
 	if ((entry = search_exception_tables(regs->nip)) != NULL) {
 		tsi108_clear_pci_cfg_error();
 		regs->msr |= MSR_RI;
-		regs->nip = entry->fixup;
+		regs->nip = extable_fixup(entry);
 		return 1;
 	}
 	return 0;
diff --git a/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c b/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c
index f97bab8e37a2..9de100e22bf3 100644
--- a/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c
+++ b/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c
@@ -174,7 +174,7 @@ static int mpc7448_machine_check_exception(struct pt_regs *regs)
 	if ((entry = search_exception_tables(regs->nip)) != NULL) {
 		tsi108_clear_pci_cfg_error();
 		regs->msr |= MSR_RI;
-		regs->nip = entry->fixup;
+		regs->nip = extable_fixup(entry);
 		return 1;
 	}
 	return 0;
diff --git a/arch/powerpc/platforms/powermac/low_i2c.c b/arch/powerpc/platforms/powermac/low_i2c.c
index c8c217b7dd33..f627c9fd7b48 100644
--- a/arch/powerpc/platforms/powermac/low_i2c.c
+++ b/arch/powerpc/platforms/powermac/low_i2c.c
@@ -90,6 +90,7 @@ struct pmac_i2c_bus
 	int			opened;
 	int			polled;		/* open mode */
 	struct platform_device	*platform_dev;
+	struct lock_class_key   lock_key;
 
 	/* ops */
 	int (*open)(struct pmac_i2c_bus *bus);
@@ -587,6 +588,7 @@ static void __init kw_i2c_add(struct pmac_i2c_host_kw *host,
 	bus->close = kw_i2c_close;
 	bus->xfer = kw_i2c_xfer;
 	mutex_init(&bus->mutex);
+	lockdep_set_class(&bus->mutex, &bus->lock_key);
 	if (controller == busnode)
 		bus->flags = pmac_i2c_multibus;
 	list_add(&bus->link, &pmac_i2c_busses);
@@ -815,6 +817,7 @@ static void __init pmu_i2c_probe(void)
 		bus->hostdata = bus + 1;
 		bus->xfer = pmu_i2c_xfer;
 		mutex_init(&bus->mutex);
+		lockdep_set_class(&bus->mutex, &bus->lock_key);
 		bus->flags = pmac_i2c_multibus;
 		list_add(&bus->link, &pmac_i2c_busses);
 
@@ -938,6 +941,7 @@ static void __init smu_i2c_probe(void)
 		bus->hostdata = bus + 1;
 		bus->xfer = smu_i2c_xfer;
 		mutex_init(&bus->mutex);
+		lockdep_set_class(&bus->mutex, &bus->lock_key);
 		bus->flags = 0;
 		list_add(&bus->link, &pmac_i2c_busses);
 
diff --git a/arch/powerpc/platforms/powernv/eeh-powernv.c b/arch/powerpc/platforms/powernv/eeh-powernv.c
index 2354ea51e871..6fb5522acd70 100644
--- a/arch/powerpc/platforms/powernv/eeh-powernv.c
+++ b/arch/powerpc/platforms/powernv/eeh-powernv.c
@@ -393,7 +393,7 @@ static void *pnv_eeh_probe(struct pci_dn *pdn, void *data)
 	/* Create PE */
 	ret = eeh_add_to_parent_pe(edev);
 	if (ret) {
-		pr_warn("%s: Can't add PCI dev %04x:%02x:%02x.%01x to parent PE (%d)\n",
+		pr_warn("%s: Can't add PCI dev %04x:%02x:%02x.%01x to parent PE (%x)\n",
 			__func__, hose->global_number, pdn->busno,
 			PCI_SLOT(pdn->devfn), PCI_FUNC(pdn->devfn), ret);
 		return NULL;
@@ -1097,7 +1097,7 @@ static int pnv_eeh_reset(struct eeh_pe *pe, int option)
 
 	bus = eeh_pe_bus_get(pe);
 	if (!bus) {
-		pr_err("%s: Cannot find PCI bus for PHB#%d-PE#%x\n",
+		pr_err("%s: Cannot find PCI bus for PHB#%x-PE#%x\n",
 			__func__, pe->phb->global_number, pe->addr);
 		return -EIO;
 	}
diff --git a/arch/powerpc/platforms/powernv/npu-dma.c b/arch/powerpc/platforms/powernv/npu-dma.c
index aec85e778028..73b155fd4481 100644
--- a/arch/powerpc/platforms/powernv/npu-dma.c
+++ b/arch/powerpc/platforms/powernv/npu-dma.c
@@ -263,7 +263,7 @@ static int pnv_npu_dma_set_bypass(struct pnv_ioda_pe *npe)
 	/* Enable the bypass window */
 
 	top = roundup_pow_of_two(top);
-	dev_info(&npe->pdev->dev, "Enabling bypass for PE %d\n",
+	dev_info(&npe->pdev->dev, "Enabling bypass for PE %x\n",
 			npe->pe_number);
 	rc = opal_pci_map_pe_dma_window_real(phb->opal_id,
 			npe->pe_number, npe->pe_number,
diff --git a/arch/powerpc/platforms/powernv/opal-tracepoints.c b/arch/powerpc/platforms/powernv/opal-tracepoints.c
index 1e496b780efd..3c447002edff 100644
--- a/arch/powerpc/platforms/powernv/opal-tracepoints.c
+++ b/arch/powerpc/platforms/powernv/opal-tracepoints.c
@@ -6,9 +6,10 @@
 #ifdef HAVE_JUMP_LABEL
 struct static_key opal_tracepoint_key = STATIC_KEY_INIT;
 
-void opal_tracepoint_regfunc(void)
+int opal_tracepoint_regfunc(void)
 {
 	static_key_slow_inc(&opal_tracepoint_key);
+	return 0;
 }
 
 void opal_tracepoint_unregfunc(void)
@@ -25,9 +26,10 @@ void opal_tracepoint_unregfunc(void)
 /* NB: reg/unreg are called while guarded with the tracepoints_mutex */
 extern long opal_tracepoint_refcount;
 
-void opal_tracepoint_regfunc(void)
+int opal_tracepoint_regfunc(void)
 {
 	opal_tracepoint_refcount++;
+	return 0;
 }
 
 void opal_tracepoint_unregfunc(void)
diff --git a/arch/powerpc/platforms/powernv/opal-wrappers.S b/arch/powerpc/platforms/powernv/opal-wrappers.S
index 44d2d842cee7..3aa40f1b20f5 100644
--- a/arch/powerpc/platforms/powernv/opal-wrappers.S
+++ b/arch/powerpc/platforms/powernv/opal-wrappers.S
@@ -304,8 +304,11 @@ OPAL_CALL(opal_pci_get_presence_state,		OPAL_PCI_GET_PRESENCE_STATE);
 OPAL_CALL(opal_pci_get_power_state,		OPAL_PCI_GET_POWER_STATE);
 OPAL_CALL(opal_pci_set_power_state,		OPAL_PCI_SET_POWER_STATE);
 OPAL_CALL(opal_int_get_xirr,			OPAL_INT_GET_XIRR);
+OPAL_CALL_REAL(opal_rm_int_get_xirr,		OPAL_INT_GET_XIRR);
 OPAL_CALL(opal_int_set_cppr,			OPAL_INT_SET_CPPR);
 OPAL_CALL(opal_int_eoi,				OPAL_INT_EOI);
+OPAL_CALL_REAL(opal_rm_int_eoi,			OPAL_INT_EOI);
 OPAL_CALL(opal_int_set_mfrr,			OPAL_INT_SET_MFRR);
+OPAL_CALL_REAL(opal_rm_int_set_mfrr,		OPAL_INT_SET_MFRR);
 OPAL_CALL(opal_pci_tce_kill,			OPAL_PCI_TCE_KILL);
 OPAL_CALL_REAL(opal_rm_pci_tce_kill,		OPAL_PCI_TCE_KILL);
diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c
index 6c9a65b52e63..282293572dc8 100644
--- a/arch/powerpc/platforms/powernv/opal.c
+++ b/arch/powerpc/platforms/powernv/opal.c
@@ -632,21 +632,11 @@ static void __init opal_dump_region_init(void)
 			"rc = %d\n", rc);
 }
 
-static void opal_pdev_init(struct device_node *opal_node,
-		const char *compatible)
+static void opal_pdev_init(const char *compatible)
 {
 	struct device_node *np;
 
-	for_each_child_of_node(opal_node, np)
-		if (of_device_is_compatible(np, compatible))
-			of_platform_device_create(np, NULL, NULL);
-}
-
-static void opal_i2c_create_devs(void)
-{
-	struct device_node *np;
-
-	for_each_compatible_node(np, NULL, "ibm,opal-i2c")
+	for_each_compatible_node(np, NULL, compatible)
 		of_platform_device_create(np, NULL, NULL);
 }
 
@@ -718,7 +708,7 @@ static int __init opal_init(void)
 	opal_hmi_handler_init();
 
 	/* Create i2c platform devices */
-	opal_i2c_create_devs();
+	opal_pdev_init("ibm,opal-i2c");
 
 	/* Setup a heatbeat thread if requested by OPAL */
 	opal_init_heartbeat();
@@ -753,12 +743,12 @@ static int __init opal_init(void)
 	}
 
 	/* Initialize platform devices: IPMI backend, PRD & flash interface */
-	opal_pdev_init(opal_node, "ibm,opal-ipmi");
-	opal_pdev_init(opal_node, "ibm,opal-flash");
-	opal_pdev_init(opal_node, "ibm,opal-prd");
+	opal_pdev_init("ibm,opal-ipmi");
+	opal_pdev_init("ibm,opal-flash");
+	opal_pdev_init("ibm,opal-prd");
 
 	/* Initialise platform device: oppanel interface */
-	opal_pdev_init(opal_node, "ibm,opal-oppanel");
+	opal_pdev_init("ibm,opal-oppanel");
 
 	/* Initialise OPAL kmsg dumper for flushing console on panic */
 	opal_kmsg_init();
@@ -896,3 +886,5 @@ EXPORT_SYMBOL_GPL(opal_leds_get_ind);
 EXPORT_SYMBOL_GPL(opal_leds_set_ind);
 /* Export this symbol for PowerNV Operator Panel class driver */
 EXPORT_SYMBOL_GPL(opal_write_oppanel_async);
+/* Export this for KVM */
+EXPORT_SYMBOL_GPL(opal_int_set_mfrr);
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index d4b33dd2d9e7..b07680cd2518 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -83,7 +83,7 @@ void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
 			PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
 #endif /* CONFIG_PCI_IOV*/
 
-	printk("%spci %s: [PE# %.3d] %pV",
+	printk("%spci %s: [PE# %.2x] %pV",
 	       level, pfix, pe->pe_number, &vaf);
 
 	va_end(args);
@@ -145,8 +145,8 @@ static struct pnv_ioda_pe *pnv_ioda_init_pe(struct pnv_phb *phb, int pe_no)
 	 */
 	rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no,
 				       OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
-	if (rc != OPAL_SUCCESS)
-		pr_warn("%s: Error %lld unfreezing PHB#%d-PE#%d\n",
+	if (rc != OPAL_SUCCESS && rc != OPAL_UNSUPPORTED)
+		pr_warn("%s: Error %lld unfreezing PHB#%x-PE#%x\n",
 			__func__, rc, phb->hose->global_number, pe_no);
 
 	return &phb->ioda.pe_array[pe_no];
@@ -155,13 +155,13 @@ static struct pnv_ioda_pe *pnv_ioda_init_pe(struct pnv_phb *phb, int pe_no)
 static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
 {
 	if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe_num)) {
-		pr_warn("%s: Invalid PE %d on PHB#%x\n",
+		pr_warn("%s: Invalid PE %x on PHB#%x\n",
 			__func__, pe_no, phb->hose->global_number);
 		return;
 	}
 
 	if (test_and_set_bit(pe_no, phb->ioda.pe_alloc))
-		pr_debug("%s: PE %d was reserved on PHB#%x\n",
+		pr_debug("%s: PE %x was reserved on PHB#%x\n",
 			 __func__, pe_no, phb->hose->global_number);
 
 	pnv_ioda_init_pe(phb, pe_no);
@@ -229,7 +229,7 @@ static int pnv_ioda2_init_m64(struct pnv_phb *phb)
 	else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1))
 		r->end -= (2 * phb->ioda.m64_segsize);
 	else
-		pr_warn("  Cannot strip M64 segment for reserved PE#%d\n",
+		pr_warn("  Cannot strip M64 segment for reserved PE#%x\n",
 			phb->ioda.reserved_pe_idx);
 
 	return 0;
@@ -291,7 +291,7 @@ static int pnv_ioda1_init_m64(struct pnv_phb *phb)
 				OPAL_M64_WINDOW_TYPE, index, base, 0,
 				PNV_IODA1_M64_SEGS * segsz);
 		if (rc != OPAL_SUCCESS) {
-			pr_warn("  Error %lld setting M64 PHB#%d-BAR#%d\n",
+			pr_warn("  Error %lld setting M64 PHB#%x-BAR#%d\n",
 				rc, phb->hose->global_number, index);
 			goto fail;
 		}
@@ -300,7 +300,7 @@ static int pnv_ioda1_init_m64(struct pnv_phb *phb)
 				OPAL_M64_WINDOW_TYPE, index,
 				OPAL_ENABLE_M64_SPLIT);
 		if (rc != OPAL_SUCCESS) {
-			pr_warn("  Error %lld enabling M64 PHB#%d-BAR#%d\n",
+			pr_warn("  Error %lld enabling M64 PHB#%x-BAR#%d\n",
 				rc, phb->hose->global_number, index);
 			goto fail;
 		}
@@ -316,7 +316,7 @@ static int pnv_ioda1_init_m64(struct pnv_phb *phb)
 	else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1))
 		r->end -= (2 * phb->ioda.m64_segsize);
 	else
-		WARN(1, "Wrong reserved PE#%d on PHB#%d\n",
+		WARN(1, "Wrong reserved PE#%x on PHB#%x\n",
 		     phb->ioda.reserved_pe_idx, phb->hose->global_number);
 
 	return 0;
@@ -414,7 +414,7 @@ static struct pnv_ioda_pe *pnv_ioda_pick_m64_pe(struct pci_bus *bus, bool all)
 					pe->pe_number / PNV_IODA1_M64_SEGS,
 					pe->pe_number % PNV_IODA1_M64_SEGS);
 			if (rc != OPAL_SUCCESS)
-				pr_warn("%s: Error %lld mapping M64 for PHB#%d-PE#%d\n",
+				pr_warn("%s: Error %lld mapping M64 for PHB#%x-PE#%x\n",
 					__func__, rc, phb->hose->global_number,
 					pe->pe_number);
 		}
@@ -941,14 +941,14 @@ static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
 	pe->mve_number = pe->pe_number;
 	rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
 	if (rc != OPAL_SUCCESS) {
-		pe_err(pe, "OPAL error %ld setting up MVE %d\n",
+		pe_err(pe, "OPAL error %ld setting up MVE %x\n",
 		       rc, pe->mve_number);
 		pe->mve_number = -1;
 	} else {
 		rc = opal_pci_set_mve_enable(phb->opal_id,
 					     pe->mve_number, OPAL_ENABLE_MVE);
 		if (rc) {
-			pe_err(pe, "OPAL error %ld enabling MVE %d\n",
+			pe_err(pe, "OPAL error %ld enabling MVE %x\n",
 			       rc, pe->mve_number);
 			pe->mve_number = -1;
 		}
@@ -1159,10 +1159,10 @@ static struct pnv_ioda_pe *pnv_ioda_setup_bus_PE(struct pci_bus *bus, bool all)
 	pe->rid = bus->busn_res.start << 8;
 
 	if (all)
-		pe_info(pe, "Secondary bus %d..%d associated with PE#%d\n",
+		pe_info(pe, "Secondary bus %d..%d associated with PE#%x\n",
 			bus->busn_res.start, bus->busn_res.end, pe->pe_number);
 	else
-		pe_info(pe, "Secondary bus %d associated with PE#%d\n",
+		pe_info(pe, "Secondary bus %d associated with PE#%x\n",
 			bus->busn_res.start, pe->pe_number);
 
 	if (pnv_ioda_configure_pe(phb, pe)) {
@@ -1213,7 +1213,7 @@ static struct pnv_ioda_pe *pnv_ioda_setup_npu_PE(struct pci_dev *npu_pdev)
 			 * peer NPU.
 			 */
 			dev_info(&npu_pdev->dev,
-				"Associating to existing PE %d\n", pe_num);
+				"Associating to existing PE %x\n", pe_num);
 			pci_dev_get(npu_pdev);
 			npu_pdn = pci_get_pdn(npu_pdev);
 			rid = npu_pdev->bus->number << 8 | npu_pdn->devfn;
@@ -1539,7 +1539,7 @@ static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
 		pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) |
 			   pci_iov_virtfn_devfn(pdev, vf_index);
 
-		pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%d\n",
+		pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%x\n",
 			hose->global_number, pdev->bus->number,
 			PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
 			PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num);
@@ -2844,7 +2844,7 @@ static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
 	pnv_set_msi_irq_chip(phb, virq);
 
 	pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d),"
-		 " address=%x_%08x data=%x PE# %d\n",
+		 " address=%x_%08x data=%x PE# %x\n",
 		 pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num,
 		 msg->address_hi, msg->address_lo, data, pe->pe_number);
 
@@ -2993,7 +2993,7 @@ static void pnv_ioda_setup_pe_res(struct pnv_ioda_pe *pe,
 			rc = opal_pci_map_pe_mmio_window(phb->opal_id,
 				pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
 			if (rc != OPAL_SUCCESS) {
-				pr_err("%s: Error %lld mapping IO segment#%d to PE#%d\n",
+				pr_err("%s: Error %lld mapping IO segment#%d to PE#%x\n",
 				       __func__, rc, index, pe->pe_number);
 				break;
 			}
@@ -3017,7 +3017,7 @@ static void pnv_ioda_setup_pe_res(struct pnv_ioda_pe *pe,
 			rc = opal_pci_map_pe_mmio_window(phb->opal_id,
 				pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
 			if (rc != OPAL_SUCCESS) {
-				pr_err("%s: Error %lld mapping M32 segment#%d to PE#%d",
+				pr_err("%s: Error %lld mapping M32 segment#%d to PE#%x",
 				       __func__, rc, index, pe->pe_number);
 				break;
 			}
@@ -3281,7 +3281,7 @@ static void pnv_pci_setup_bridge(struct pci_bus *bus, unsigned long type)
 		pnv_pci_ioda2_setup_dma_pe(phb, pe);
 		break;
 	default:
-		pr_warn("%s: No DMA for PHB#%d (type %d)\n",
+		pr_warn("%s: No DMA for PHB#%x (type %d)\n",
 			__func__, phb->hose->global_number, phb->type);
 	}
 }
diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c
index db7b8020f68e..c6d554fe585c 100644
--- a/arch/powerpc/platforms/powernv/pci.c
+++ b/arch/powerpc/platforms/powernv/pci.c
@@ -234,7 +234,7 @@ static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose,
 	int i;
 
 	data = (struct OpalIoP7IOCPhbErrorData *)common;
-	pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n",
+	pr_info("P7IOC PHB#%x Diag-data (Version: %d)\n",
 		hose->global_number, be32_to_cpu(common->version));
 
 	if (data->brdgCtl)
@@ -326,7 +326,7 @@ static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
 	int i;
 
 	data = (struct OpalIoPhb3ErrorData*)common;
-	pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n",
+	pr_info("PHB3 PHB#%x Diag-data (Version: %d)\n",
 		hose->global_number, be32_to_cpu(common->version));
 	if (data->brdgCtl)
 		pr_info("brdgCtl:     %08x\n",
@@ -516,7 +516,7 @@ static void pnv_pci_config_check_eeh(struct pci_dn *pdn)
 		}
 	}
 
-	pr_devel(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n",
+	pr_devel(" -> EEH check, bdfn=%04x PE#%x fstate=%x\n",
 		 (pdn->busno << 8) | (pdn->devfn), pe_no, fstate);
 
 	/* Clear the frozen state if applicable */
diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c
index efe8b6bb168b..d50c7d99baaf 100644
--- a/arch/powerpc/platforms/powernv/setup.c
+++ b/arch/powerpc/platforms/powernv/setup.c
@@ -174,7 +174,7 @@ static void pnv_shutdown(void)
 	opal_shutdown();
 }
 
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 static void pnv_kexec_wait_secondaries_down(void)
 {
 	int my_cpu, i, notified = -1;
@@ -245,7 +245,7 @@ static void pnv_kexec_cpu_down(int crash_shutdown, int secondary)
 		opal_reinit_cpus(OPAL_REINIT_CPUS_HILE_BE);
 	}
 }
-#endif /* CONFIG_KEXEC */
+#endif /* CONFIG_KEXEC_CORE */
 
 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
 static unsigned long pnv_memory_block_size(void)
@@ -311,7 +311,7 @@ define_machine(powernv) {
 	.machine_shutdown	= pnv_shutdown,
 	.power_save             = NULL,
 	.calibrate_decr		= generic_calibrate_decr,
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 	.kexec_cpu_down		= pnv_kexec_cpu_down,
 #endif
 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
diff --git a/arch/powerpc/platforms/ps3/htab.c b/arch/powerpc/platforms/ps3/htab.c
index cb3c50328de8..cc2b281a3766 100644
--- a/arch/powerpc/platforms/ps3/htab.c
+++ b/arch/powerpc/platforms/ps3/htab.c
@@ -63,7 +63,7 @@ static long ps3_hpte_insert(unsigned long hpte_group, unsigned long vpn,
 	vflags &= ~HPTE_V_SECONDARY;
 
 	hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
-	hpte_r = hpte_encode_r(ps3_mm_phys_to_lpar(pa), psize, apsize, ssize) | rflags;
+	hpte_r = hpte_encode_r(ps3_mm_phys_to_lpar(pa), psize, apsize) | rflags;
 
 	spin_lock_irqsave(&ps3_htab_lock, flags);
 
diff --git a/arch/powerpc/platforms/ps3/setup.c b/arch/powerpc/platforms/ps3/setup.c
index 3a487e7f4a5e..6244bc849469 100644
--- a/arch/powerpc/platforms/ps3/setup.c
+++ b/arch/powerpc/platforms/ps3/setup.c
@@ -250,7 +250,7 @@ static int __init ps3_probe(void)
 	return 1;
 }
 
-#if defined(CONFIG_KEXEC)
+#if defined(CONFIG_KEXEC_CORE)
 static void ps3_kexec_cpu_down(int crash_shutdown, int secondary)
 {
 	int cpu = smp_processor_id();
@@ -276,7 +276,7 @@ define_machine(ps3) {
 	.progress			= ps3_progress,
 	.restart			= ps3_restart,
 	.halt				= ps3_halt,
-#if defined(CONFIG_KEXEC)
+#if defined(CONFIG_KEXEC_CORE)
 	.kexec_cpu_down			= ps3_kexec_cpu_down,
 #endif
 };
diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig
index bec90fb30425..e1c280a95d58 100644
--- a/arch/powerpc/platforms/pseries/Kconfig
+++ b/arch/powerpc/platforms/pseries/Kconfig
@@ -127,3 +127,14 @@ config HV_PERF_CTRS
 	  systems. 24x7 is available on Power 8 systems.
 
           If unsure, select Y.
+
+config IBMVIO
+	depends on PPC_PSERIES
+	bool
+	default y
+
+config IBMEBUS
+	depends on PPC_PSERIES && !CPU_LITTLE_ENDIAN
+	bool "Support for GX bus based adapters"
+	help
+	  Bus device driver for GX bus based adapters.
diff --git a/arch/powerpc/platforms/pseries/Makefile b/arch/powerpc/platforms/pseries/Makefile
index fedc2ccf029d..8f4ba089e802 100644
--- a/arch/powerpc/platforms/pseries/Makefile
+++ b/arch/powerpc/platforms/pseries/Makefile
@@ -8,7 +8,7 @@ obj-y			:= lpar.o hvCall.o nvram.o reconfig.o \
 			   pci.o pci_dlpar.o eeh_pseries.o msi.o
 obj-$(CONFIG_SMP)	+= smp.o
 obj-$(CONFIG_SCANLOG)	+= scanlog.o
-obj-$(CONFIG_KEXEC)	+= kexec.o
+obj-$(CONFIG_KEXEC_CORE)	+= kexec.o
 obj-$(CONFIG_PSERIES_ENERGY)	+= pseries_energy.o
 
 obj-$(CONFIG_HOTPLUG_CPU)	+= hotplug-cpu.o
@@ -21,6 +21,8 @@ obj-$(CONFIG_CMM)		+= cmm.o
 obj-$(CONFIG_DTL)		+= dtl.o
 obj-$(CONFIG_IO_EVENT_IRQ)	+= io_event_irq.o
 obj-$(CONFIG_LPARCFG)		+= lparcfg.o
+obj-$(CONFIG_IBMVIO)		+= vio.o
+obj-$(CONFIG_IBMEBUS)		+= ibmebus.o
 
 ifeq ($(CONFIG_PPC_PSERIES),y)
 obj-$(CONFIG_SUSPEND)		+= suspend.o
diff --git a/arch/powerpc/platforms/pseries/cmm.c b/arch/powerpc/platforms/pseries/cmm.c
index 66e7227469b8..972328829387 100644
--- a/arch/powerpc/platforms/pseries/cmm.c
+++ b/arch/powerpc/platforms/pseries/cmm.c
@@ -41,6 +41,8 @@
 #include <linux/memory.h>
 #include <asm/plpar_wrappers.h>
 
+#include "pseries.h"
+
 #define CMM_DRIVER_VERSION	"1.0.0"
 #define CMM_DEFAULT_DELAY	1
 #define CMM_HOTPLUG_DELAY	5
@@ -109,6 +111,38 @@ static int hotplug_occurred; /* protected by the hotplug mutex */
 
 static struct task_struct *cmm_thread_ptr;
 
+static long plpar_page_set_loaned(unsigned long vpa)
+{
+	unsigned long cmo_page_sz = cmo_get_page_size();
+	long rc = 0;
+	int i;
+
+	for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz)
+		rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED, vpa + i, 0);
+
+	for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz)
+		plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE,
+				   vpa + i - cmo_page_sz, 0);
+
+	return rc;
+}
+
+static long plpar_page_set_active(unsigned long vpa)
+{
+	unsigned long cmo_page_sz = cmo_get_page_size();
+	long rc = 0;
+	int i;
+
+	for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz)
+		rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE, vpa + i, 0);
+
+	for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz)
+		plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED,
+				   vpa + i - cmo_page_sz, 0);
+
+	return rc;
+}
+
 /**
  * cmm_alloc_pages - Allocate pages and mark them as loaned
  * @nr:	number of pages to allocate
diff --git a/arch/powerpc/platforms/pseries/dlpar.c b/arch/powerpc/platforms/pseries/dlpar.c
index 423e450efe07..76caa4a45ccd 100644
--- a/arch/powerpc/platforms/pseries/dlpar.c
+++ b/arch/powerpc/platforms/pseries/dlpar.c
@@ -418,84 +418,136 @@ void queue_hotplug_event(struct pseries_hp_errorlog *hp_errlog,
 	}
 }
 
-static ssize_t dlpar_store(struct class *class, struct class_attribute *attr,
-			   const char *buf, size_t count)
+static int dlpar_parse_resource(char **cmd, struct pseries_hp_errorlog *hp_elog)
 {
-	struct pseries_hp_errorlog *hp_elog;
-	struct completion hotplug_done;
-	const char *arg;
-	int rc;
+	char *arg;
 
-	hp_elog = kzalloc(sizeof(*hp_elog), GFP_KERNEL);
-	if (!hp_elog) {
-		rc = -ENOMEM;
-		goto dlpar_store_out;
-	}
+	arg = strsep(cmd, " ");
+	if (!arg)
+		return -EINVAL;
 
-	/* Parse out the request from the user, this will be in the form
-	 * <resource> <action> <id_type> <id>
-	 */
-	arg = buf;
-	if (!strncmp(arg, "memory", 6)) {
+	if (sysfs_streq(arg, "memory")) {
 		hp_elog->resource = PSERIES_HP_ELOG_RESOURCE_MEM;
-		arg += strlen("memory ");
-	} else if (!strncmp(arg, "cpu", 3)) {
+	} else if (sysfs_streq(arg, "cpu")) {
 		hp_elog->resource = PSERIES_HP_ELOG_RESOURCE_CPU;
-		arg += strlen("cpu ");
 	} else {
-		pr_err("Invalid resource specified: \"%s\"\n", buf);
-		rc = -EINVAL;
-		goto dlpar_store_out;
+		pr_err("Invalid resource specified.\n");
+		return -EINVAL;
 	}
 
-	if (!strncmp(arg, "add", 3)) {
+	return 0;
+}
+
+static int dlpar_parse_action(char **cmd, struct pseries_hp_errorlog *hp_elog)
+{
+	char *arg;
+
+	arg = strsep(cmd, " ");
+	if (!arg)
+		return -EINVAL;
+
+	if (sysfs_streq(arg, "add")) {
 		hp_elog->action = PSERIES_HP_ELOG_ACTION_ADD;
-		arg += strlen("add ");
-	} else if (!strncmp(arg, "remove", 6)) {
+	} else if (sysfs_streq(arg, "remove")) {
 		hp_elog->action = PSERIES_HP_ELOG_ACTION_REMOVE;
-		arg += strlen("remove ");
 	} else {
-		pr_err("Invalid action specified: \"%s\"\n", buf);
-		rc = -EINVAL;
-		goto dlpar_store_out;
+		pr_err("Invalid action specified.\n");
+		return -EINVAL;
 	}
 
-	if (!strncmp(arg, "index", 5)) {
-		u32 index;
+	return 0;
+}
 
+static int dlpar_parse_id_type(char **cmd, struct pseries_hp_errorlog *hp_elog)
+{
+	char *arg;
+	u32 count, index;
+
+	arg = strsep(cmd, " ");
+	if (!arg)
+		return -EINVAL;
+
+	if (sysfs_streq(arg, "index")) {
 		hp_elog->id_type = PSERIES_HP_ELOG_ID_DRC_INDEX;
-		arg += strlen("index ");
+		arg = strsep(cmd, " ");
+		if (!arg) {
+			pr_err("No DRC Index specified.\n");
+			return -EINVAL;
+		}
+
 		if (kstrtou32(arg, 0, &index)) {
-			rc = -EINVAL;
-			pr_err("Invalid drc_index specified: \"%s\"\n", buf);
-			goto dlpar_store_out;
+			pr_err("Invalid DRC Index specified.\n");
+			return -EINVAL;
 		}
 
 		hp_elog->_drc_u.drc_index = cpu_to_be32(index);
-	} else if (!strncmp(arg, "count", 5)) {
-		u32 count;
-
+	} else if (sysfs_streq(arg, "count")) {
 		hp_elog->id_type = PSERIES_HP_ELOG_ID_DRC_COUNT;
-		arg += strlen("count ");
+		arg = strsep(cmd, " ");
+		if (!arg) {
+			pr_err("No DRC count specified.\n");
+			return -EINVAL;
+		}
+
 		if (kstrtou32(arg, 0, &count)) {
-			rc = -EINVAL;
-			pr_err("Invalid count specified: \"%s\"\n", buf);
-			goto dlpar_store_out;
+			pr_err("Invalid DRC count specified.\n");
+			return -EINVAL;
 		}
 
 		hp_elog->_drc_u.drc_count = cpu_to_be32(count);
 	} else {
-		pr_err("Invalid id_type specified: \"%s\"\n", buf);
-		rc = -EINVAL;
-		goto dlpar_store_out;
+		pr_err("Invalid id_type specified.\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static ssize_t dlpar_store(struct class *class, struct class_attribute *attr,
+			   const char *buf, size_t count)
+{
+	struct pseries_hp_errorlog *hp_elog;
+	struct completion hotplug_done;
+	char *argbuf;
+	char *args;
+	int rc;
+
+	args = argbuf = kstrdup(buf, GFP_KERNEL);
+	hp_elog = kzalloc(sizeof(*hp_elog), GFP_KERNEL);
+	if (!hp_elog || !argbuf) {
+		pr_info("Could not allocate resources for DLPAR operation\n");
+		kfree(argbuf);
+		kfree(hp_elog);
+		return -ENOMEM;
 	}
 
+	/*
+	 * Parse out the request from the user, this will be in the form:
+	 * <resource> <action> <id_type> <id>
+	 */
+	rc = dlpar_parse_resource(&args, hp_elog);
+	if (rc)
+		goto dlpar_store_out;
+
+	rc = dlpar_parse_action(&args, hp_elog);
+	if (rc)
+		goto dlpar_store_out;
+
+	rc = dlpar_parse_id_type(&args, hp_elog);
+	if (rc)
+		goto dlpar_store_out;
+
 	init_completion(&hotplug_done);
 	queue_hotplug_event(hp_elog, &hotplug_done, &rc);
 	wait_for_completion(&hotplug_done);
 
 dlpar_store_out:
+	kfree(argbuf);
 	kfree(hp_elog);
+
+	if (rc)
+		pr_err("Could not handle DLPAR request \"%s\"\n", buf);
+
 	return rc ? rc : count;
 }
 
diff --git a/arch/powerpc/platforms/pseries/eeh_pseries.c b/arch/powerpc/platforms/pseries/eeh_pseries.c
index 1c428f06b14c..1eef46d9cf30 100644
--- a/arch/powerpc/platforms/pseries/eeh_pseries.c
+++ b/arch/powerpc/platforms/pseries/eeh_pseries.c
@@ -270,7 +270,7 @@ static void *pseries_eeh_probe(struct pci_dn *pdn, void *data)
 			eeh_add_flag(EEH_ENABLED);
 			eeh_add_to_parent_pe(edev);
 
-			pr_debug("%s: EEH enabled on %02x:%02x.%01x PHB#%d-PE#%x\n",
+			pr_debug("%s: EEH enabled on %02x:%02x.%01x PHB#%x-PE#%x\n",
 				__func__, pdn->busno, PCI_SLOT(pdn->devfn),
 				PCI_FUNC(pdn->devfn), pe.phb->global_number,
 				pe.addr);
@@ -371,7 +371,7 @@ static int pseries_eeh_get_pe_addr(struct eeh_pe *pe)
 				pe->config_addr, BUID_HI(pe->phb->buid),
 				BUID_LO(pe->phb->buid), 0);
 		if (ret) {
-			pr_warn("%s: Failed to get address for PHB#%d-PE#%x\n",
+			pr_warn("%s: Failed to get address for PHB#%x-PE#%x\n",
 				__func__, pe->phb->global_number, pe->config_addr);
 			return 0;
 		}
@@ -384,7 +384,7 @@ static int pseries_eeh_get_pe_addr(struct eeh_pe *pe)
 				pe->config_addr, BUID_HI(pe->phb->buid),
 				BUID_LO(pe->phb->buid), 0);
 		if (ret) {
-			pr_warn("%s: Failed to get address for PHB#%d-PE#%x\n",
+			pr_warn("%s: Failed to get address for PHB#%x-PE#%x\n",
 				__func__, pe->phb->global_number, pe->config_addr);
 			return 0;
 		}
@@ -653,7 +653,7 @@ static int pseries_eeh_configure_bridge(struct eeh_pe *pe)
 		rtas_busy_delay(ret);
 	}
 
-	pr_warn("%s: Unable to configure bridge PHB#%d-PE#%x (%d)\n",
+	pr_warn("%s: Unable to configure bridge PHB#%x-PE#%x (%d)\n",
 		__func__, pe->phb->global_number, pe->addr, ret);
 	return ret;
 }
diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c
index 76ec104e88be..2617f9f356bd 100644
--- a/arch/powerpc/platforms/pseries/hotplug-memory.c
+++ b/arch/powerpc/platforms/pseries/hotplug-memory.c
@@ -472,12 +472,15 @@ static int dlpar_memory_remove_by_count(u32 lmbs_to_remove,
 
 	/* Validate that there are enough LMBs to satisfy the request */
 	for (i = 0; i < num_lmbs; i++) {
-		if (lmbs[i].flags & DRCONF_MEM_ASSIGNED)
+		if (lmb_is_removable(&lmbs[i]))
 			lmbs_available++;
 	}
 
-	if (lmbs_available < lmbs_to_remove)
+	if (lmbs_available < lmbs_to_remove) {
+		pr_info("Not enough LMBs available (%d of %d) to satisfy request\n",
+			lmbs_available, lmbs_to_remove);
 		return -EINVAL;
+	}
 
 	for (i = 0; i < num_lmbs && lmbs_removed < lmbs_to_remove; i++) {
 		rc = dlpar_remove_lmb(&lmbs[i]);
diff --git a/arch/powerpc/kernel/ibmebus.c b/arch/powerpc/platforms/pseries/ibmebus.c
index 6ca9a2ffaac7..614c28537141 100644
--- a/arch/powerpc/kernel/ibmebus.c
+++ b/arch/powerpc/platforms/pseries/ibmebus.c
@@ -180,6 +180,7 @@ static int ibmebus_create_device(struct device_node *dn)
 static int ibmebus_create_devices(const struct of_device_id *matches)
 {
 	struct device_node *root, *child;
+	struct device *dev;
 	int ret = 0;
 
 	root = of_find_node_by_path("/");
@@ -188,9 +189,12 @@ static int ibmebus_create_devices(const struct of_device_id *matches)
 		if (!of_match_node(matches, child))
 			continue;
 
-		if (bus_find_device(&ibmebus_bus_type, NULL, child,
-				    ibmebus_match_node))
+		dev = bus_find_device(&ibmebus_bus_type, NULL, child,
+				      ibmebus_match_node);
+		if (dev) {
+			put_device(dev);
 			continue;
+		}
 
 		ret = ibmebus_create_device(child);
 		if (ret) {
@@ -262,6 +266,7 @@ static ssize_t ibmebus_store_probe(struct bus_type *bus,
 				   const char *buf, size_t count)
 {
 	struct device_node *dn = NULL;
+	struct device *dev;
 	char *path;
 	ssize_t rc = 0;
 
@@ -269,8 +274,10 @@ static ssize_t ibmebus_store_probe(struct bus_type *bus,
 	if (!path)
 		return -ENOMEM;
 
-	if (bus_find_device(&ibmebus_bus_type, NULL, path,
-			    ibmebus_match_path)) {
+	dev = bus_find_device(&ibmebus_bus_type, NULL, path,
+			      ibmebus_match_path);
+	if (dev) {
+		put_device(dev);
 		printk(KERN_WARNING "%s: %s has already been probed\n",
 		       __func__, path);
 		rc = -EEXIST;
@@ -307,6 +314,7 @@ static ssize_t ibmebus_store_remove(struct bus_type *bus,
 	if ((dev = bus_find_device(&ibmebus_bus_type, NULL, path,
 				   ibmebus_match_path))) {
 		of_device_unregister(to_platform_device(dev));
+		put_device(dev);
 
 		kfree(path);
 		return count;
@@ -415,303 +423,6 @@ static struct device_attribute ibmebus_bus_device_attrs[] = {
 	__ATTR_NULL
 };
 
-#ifdef CONFIG_PM_SLEEP
-static int ibmebus_bus_legacy_suspend(struct device *dev, pm_message_t mesg)
-{
-	struct platform_device *of_dev = to_platform_device(dev);
-	struct platform_driver *drv = to_platform_driver(dev->driver);
-	int ret = 0;
-
-	if (dev->driver && drv->suspend)
-		ret = drv->suspend(of_dev, mesg);
-	return ret;
-}
-
-static int ibmebus_bus_legacy_resume(struct device *dev)
-{
-	struct platform_device *of_dev = to_platform_device(dev);
-	struct platform_driver *drv = to_platform_driver(dev->driver);
-	int ret = 0;
-
-	if (dev->driver && drv->resume)
-		ret = drv->resume(of_dev);
-	return ret;
-}
-
-static int ibmebus_bus_pm_prepare(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (drv && drv->pm && drv->pm->prepare)
-		ret = drv->pm->prepare(dev);
-
-	return ret;
-}
-
-static void ibmebus_bus_pm_complete(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-
-	if (drv && drv->pm && drv->pm->complete)
-		drv->pm->complete(dev);
-}
-
-#ifdef CONFIG_SUSPEND
-
-static int ibmebus_bus_pm_suspend(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (!drv)
-		return 0;
-
-	if (drv->pm) {
-		if (drv->pm->suspend)
-			ret = drv->pm->suspend(dev);
-	} else {
-		ret = ibmebus_bus_legacy_suspend(dev, PMSG_SUSPEND);
-	}
-
-	return ret;
-}
-
-static int ibmebus_bus_pm_suspend_noirq(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (!drv)
-		return 0;
-
-	if (drv->pm) {
-		if (drv->pm->suspend_noirq)
-			ret = drv->pm->suspend_noirq(dev);
-	}
-
-	return ret;
-}
-
-static int ibmebus_bus_pm_resume(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (!drv)
-		return 0;
-
-	if (drv->pm) {
-		if (drv->pm->resume)
-			ret = drv->pm->resume(dev);
-	} else {
-		ret = ibmebus_bus_legacy_resume(dev);
-	}
-
-	return ret;
-}
-
-static int ibmebus_bus_pm_resume_noirq(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (!drv)
-		return 0;
-
-	if (drv->pm) {
-		if (drv->pm->resume_noirq)
-			ret = drv->pm->resume_noirq(dev);
-	}
-
-	return ret;
-}
-
-#else /* !CONFIG_SUSPEND */
-
-#define ibmebus_bus_pm_suspend		NULL
-#define ibmebus_bus_pm_resume		NULL
-#define ibmebus_bus_pm_suspend_noirq	NULL
-#define ibmebus_bus_pm_resume_noirq	NULL
-
-#endif /* !CONFIG_SUSPEND */
-
-#ifdef CONFIG_HIBERNATE_CALLBACKS
-
-static int ibmebus_bus_pm_freeze(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (!drv)
-		return 0;
-
-	if (drv->pm) {
-		if (drv->pm->freeze)
-			ret = drv->pm->freeze(dev);
-	} else {
-		ret = ibmebus_bus_legacy_suspend(dev, PMSG_FREEZE);
-	}
-
-	return ret;
-}
-
-static int ibmebus_bus_pm_freeze_noirq(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (!drv)
-		return 0;
-
-	if (drv->pm) {
-		if (drv->pm->freeze_noirq)
-			ret = drv->pm->freeze_noirq(dev);
-	}
-
-	return ret;
-}
-
-static int ibmebus_bus_pm_thaw(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (!drv)
-		return 0;
-
-	if (drv->pm) {
-		if (drv->pm->thaw)
-			ret = drv->pm->thaw(dev);
-	} else {
-		ret = ibmebus_bus_legacy_resume(dev);
-	}
-
-	return ret;
-}
-
-static int ibmebus_bus_pm_thaw_noirq(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (!drv)
-		return 0;
-
-	if (drv->pm) {
-		if (drv->pm->thaw_noirq)
-			ret = drv->pm->thaw_noirq(dev);
-	}
-
-	return ret;
-}
-
-static int ibmebus_bus_pm_poweroff(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (!drv)
-		return 0;
-
-	if (drv->pm) {
-		if (drv->pm->poweroff)
-			ret = drv->pm->poweroff(dev);
-	} else {
-		ret = ibmebus_bus_legacy_suspend(dev, PMSG_HIBERNATE);
-	}
-
-	return ret;
-}
-
-static int ibmebus_bus_pm_poweroff_noirq(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (!drv)
-		return 0;
-
-	if (drv->pm) {
-		if (drv->pm->poweroff_noirq)
-			ret = drv->pm->poweroff_noirq(dev);
-	}
-
-	return ret;
-}
-
-static int ibmebus_bus_pm_restore(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (!drv)
-		return 0;
-
-	if (drv->pm) {
-		if (drv->pm->restore)
-			ret = drv->pm->restore(dev);
-	} else {
-		ret = ibmebus_bus_legacy_resume(dev);
-	}
-
-	return ret;
-}
-
-static int ibmebus_bus_pm_restore_noirq(struct device *dev)
-{
-	struct device_driver *drv = dev->driver;
-	int ret = 0;
-
-	if (!drv)
-		return 0;
-
-	if (drv->pm) {
-		if (drv->pm->restore_noirq)
-			ret = drv->pm->restore_noirq(dev);
-	}
-
-	return ret;
-}
-
-#else /* !CONFIG_HIBERNATE_CALLBACKS */
-
-#define ibmebus_bus_pm_freeze		NULL
-#define ibmebus_bus_pm_thaw		NULL
-#define ibmebus_bus_pm_poweroff		NULL
-#define ibmebus_bus_pm_restore		NULL
-#define ibmebus_bus_pm_freeze_noirq	NULL
-#define ibmebus_bus_pm_thaw_noirq		NULL
-#define ibmebus_bus_pm_poweroff_noirq	NULL
-#define ibmebus_bus_pm_restore_noirq	NULL
-
-#endif /* !CONFIG_HIBERNATE_CALLBACKS */
-
-static struct dev_pm_ops ibmebus_bus_dev_pm_ops = {
-	.prepare = ibmebus_bus_pm_prepare,
-	.complete = ibmebus_bus_pm_complete,
-	.suspend = ibmebus_bus_pm_suspend,
-	.resume = ibmebus_bus_pm_resume,
-	.freeze = ibmebus_bus_pm_freeze,
-	.thaw = ibmebus_bus_pm_thaw,
-	.poweroff = ibmebus_bus_pm_poweroff,
-	.restore = ibmebus_bus_pm_restore,
-	.suspend_noirq = ibmebus_bus_pm_suspend_noirq,
-	.resume_noirq = ibmebus_bus_pm_resume_noirq,
-	.freeze_noirq = ibmebus_bus_pm_freeze_noirq,
-	.thaw_noirq = ibmebus_bus_pm_thaw_noirq,
-	.poweroff_noirq = ibmebus_bus_pm_poweroff_noirq,
-	.restore_noirq = ibmebus_bus_pm_restore_noirq,
-};
-
-#define IBMEBUS_BUS_PM_OPS_PTR	(&ibmebus_bus_dev_pm_ops)
-
-#else /* !CONFIG_PM_SLEEP */
-
-#define IBMEBUS_BUS_PM_OPS_PTR	NULL
-
-#endif /* !CONFIG_PM_SLEEP */
-
 struct bus_type ibmebus_bus_type = {
 	.name      = "ibmebus",
 	.uevent    = of_device_uevent_modalias,
@@ -721,7 +432,6 @@ struct bus_type ibmebus_bus_type = {
 	.remove    = ibmebus_bus_device_remove,
 	.shutdown  = ibmebus_bus_device_shutdown,
 	.dev_attrs = ibmebus_bus_device_attrs,
-	.pm        = IBMEBUS_BUS_PM_OPS_PTR,
 };
 EXPORT_SYMBOL(ibmebus_bus_type);
 
diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
index aa35245d8d6d..5dc1c3c6e716 100644
--- a/arch/powerpc/platforms/pseries/lpar.c
+++ b/arch/powerpc/platforms/pseries/lpar.c
@@ -145,7 +145,7 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group,
 			 hpte_group, vpn,  pa, rflags, vflags, psize);
 
 	hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
-	hpte_r = hpte_encode_r(pa, psize, apsize, ssize) | rflags;
+	hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
 
 	if (!(vflags & HPTE_V_BOLTED))
 		pr_devel(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r);
@@ -221,7 +221,7 @@ static long pSeries_lpar_hpte_remove(unsigned long hpte_group)
 	return -1;
 }
 
-static void pSeries_lpar_hptab_clear(void)
+static void manual_hpte_clear_all(void)
 {
 	unsigned long size_bytes = 1UL << ppc64_pft_size;
 	unsigned long hpte_count = size_bytes >> 4;
@@ -249,6 +249,26 @@ static void pSeries_lpar_hptab_clear(void)
 					&(ptes[j].pteh), &(ptes[j].ptel));
 		}
 	}
+}
+
+static int hcall_hpte_clear_all(void)
+{
+	int rc;
+
+	do {
+		rc = plpar_hcall_norets(H_CLEAR_HPT);
+	} while (rc == H_CONTINUE);
+
+	return rc;
+}
+
+static void pseries_hpte_clear_all(void)
+{
+	int rc;
+
+	rc = hcall_hpte_clear_all();
+	if (rc != H_SUCCESS)
+		manual_hpte_clear_all();
 
 #ifdef __LITTLE_ENDIAN__
 	/*
@@ -598,7 +618,7 @@ void __init hpte_init_pseries(void)
 	mmu_hash_ops.hpte_remove	 = pSeries_lpar_hpte_remove;
 	mmu_hash_ops.hpte_removebolted   = pSeries_lpar_hpte_removebolted;
 	mmu_hash_ops.flush_hash_range	 = pSeries_lpar_flush_hash_range;
-	mmu_hash_ops.hpte_clear_all      = pSeries_lpar_hptab_clear;
+	mmu_hash_ops.hpte_clear_all      = pseries_hpte_clear_all;
 	mmu_hash_ops.hugepage_invalidate = pSeries_lpar_hugepage_invalidate;
 }
 
@@ -661,9 +681,10 @@ EXPORT_SYMBOL(arch_free_page);
 #ifdef HAVE_JUMP_LABEL
 struct static_key hcall_tracepoint_key = STATIC_KEY_INIT;
 
-void hcall_tracepoint_regfunc(void)
+int hcall_tracepoint_regfunc(void)
 {
 	static_key_slow_inc(&hcall_tracepoint_key);
+	return 0;
 }
 
 void hcall_tracepoint_unregfunc(void)
@@ -680,9 +701,10 @@ void hcall_tracepoint_unregfunc(void)
 /* NB: reg/unreg are called while guarded with the tracepoints_mutex */
 extern long hcall_tracepoint_refcount;
 
-void hcall_tracepoint_regfunc(void)
+int hcall_tracepoint_regfunc(void)
 {
 	hcall_tracepoint_refcount++;
+	return 0;
 }
 
 void hcall_tracepoint_unregfunc(void)
diff --git a/arch/powerpc/platforms/pseries/lparcfg.c b/arch/powerpc/platforms/pseries/lparcfg.c
index afa05a2cb702..e6397976060e 100644
--- a/arch/powerpc/platforms/pseries/lparcfg.c
+++ b/arch/powerpc/platforms/pseries/lparcfg.c
@@ -37,6 +37,7 @@
 #include <asm/mmu.h>
 #include <asm/machdep.h>
 
+#include "pseries.h"
 
 /*
  * This isn't a module but we expose that to userspace
diff --git a/arch/powerpc/platforms/pseries/pseries.h b/arch/powerpc/platforms/pseries/pseries.h
index b1be7b713fe6..1361a9db534b 100644
--- a/arch/powerpc/platforms/pseries/pseries.h
+++ b/arch/powerpc/platforms/pseries/pseries.h
@@ -79,4 +79,23 @@ extern struct pci_controller_ops pseries_pci_controller_ops;
 
 unsigned long pseries_memory_block_size(void);
 
+extern int CMO_PrPSP;
+extern int CMO_SecPSP;
+extern unsigned long CMO_PageSize;
+
+static inline int cmo_get_primary_psp(void)
+{
+	return CMO_PrPSP;
+}
+
+static inline int cmo_get_secondary_psp(void)
+{
+	return CMO_SecPSP;
+}
+
+static inline unsigned long cmo_get_page_size(void)
+{
+	return CMO_PageSize;
+}
+
 #endif /* _PSERIES_PSERIES_H */
diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c
index 97aa3f332f24..7736352f7279 100644
--- a/arch/powerpc/platforms/pseries/setup.c
+++ b/arch/powerpc/platforms/pseries/setup.c
@@ -367,7 +367,7 @@ void pseries_disable_reloc_on_exc(void)
 }
 EXPORT_SYMBOL(pseries_disable_reloc_on_exc);
 
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 static void pSeries_machine_kexec(struct kimage *image)
 {
 	if (firmware_has_feature(FW_FEATURE_SET_MODE))
@@ -725,7 +725,7 @@ define_machine(pseries) {
 	.progress		= rtas_progress,
 	.system_reset_exception = pSeries_system_reset_exception,
 	.machine_check_exception = pSeries_machine_check_exception,
-#ifdef CONFIG_KEXEC
+#ifdef CONFIG_KEXEC_CORE
 	.machine_kexec          = pSeries_machine_kexec,
 	.kexec_cpu_down         = pseries_kexec_cpu_down,
 #endif
diff --git a/arch/powerpc/kernel/vio.c b/arch/powerpc/platforms/pseries/vio.c
index b3813ddb2fb4..2c8fb3ec989e 100644
--- a/arch/powerpc/kernel/vio.c
+++ b/arch/powerpc/platforms/pseries/vio.c
@@ -1648,6 +1648,9 @@ static struct vio_dev *vio_find_name(const char *name)
 /**
  * vio_find_node - find an already-registered vio_dev
  * @vnode: device_node of the virtual device we're looking for
+ *
+ * Takes a reference to the embedded struct device which needs to be dropped
+ * after use.
  */
 struct vio_dev *vio_find_node(struct device_node *vnode)
 {
diff --git a/arch/powerpc/purgatory/.gitignore b/arch/powerpc/purgatory/.gitignore
new file mode 100644
index 000000000000..e9e66f178a6d
--- /dev/null
+++ b/arch/powerpc/purgatory/.gitignore
@@ -0,0 +1,2 @@
+kexec-purgatory.c
+purgatory.ro
diff --git a/arch/powerpc/purgatory/Makefile b/arch/powerpc/purgatory/Makefile
new file mode 100644
index 000000000000..ac8793c13348
--- /dev/null
+++ b/arch/powerpc/purgatory/Makefile
@@ -0,0 +1,15 @@
+targets += trampoline.o purgatory.ro kexec-purgatory.c
+
+LDFLAGS_purgatory.ro := -e purgatory_start -r --no-undefined
+
+$(obj)/purgatory.ro: $(obj)/trampoline.o FORCE
+		$(call if_changed,ld)
+
+CMD_BIN2C = $(objtree)/scripts/basic/bin2c
+quiet_cmd_bin2c = BIN2C   $@
+      cmd_bin2c = $(CMD_BIN2C) kexec_purgatory < $< > $@
+
+$(obj)/kexec-purgatory.c: $(obj)/purgatory.ro FORCE
+	$(call if_changed,bin2c)
+
+obj-y	+= kexec-purgatory.o
diff --git a/arch/powerpc/purgatory/trampoline.S b/arch/powerpc/purgatory/trampoline.S
new file mode 100644
index 000000000000..f9760ccf4032
--- /dev/null
+++ b/arch/powerpc/purgatory/trampoline.S
@@ -0,0 +1,128 @@
+/*
+ * kexec trampoline
+ *
+ * Based on code taken from kexec-tools and kexec-lite.
+ *
+ * Copyright (C) 2004 - 2005, Milton D Miller II, IBM Corporation
+ * Copyright (C) 2006, Mohan Kumar M, IBM Corporation
+ * Copyright (C) 2013, Anton Blanchard, IBM Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation (version 2 of the License).
+ */
+
+#if defined(__LITTLE_ENDIAN__)
+#define STWX_BE	stwbrx
+#define LWZX_BE	lwbrx
+#elif defined(__BIG_ENDIAN__)
+#define STWX_BE	stwx
+#define LWZX_BE	lwzx
+#else
+#error no endianness defined!
+#endif
+
+	.machine ppc64
+	.balign 256
+	.globl purgatory_start
+purgatory_start:
+	b	master
+
+	/* ABI: possible run_at_load flag at 0x5c */
+	.org purgatory_start + 0x5c
+	.globl run_at_load
+run_at_load:
+	.long 0
+	.size run_at_load, . - run_at_load
+
+	/* ABI: slaves start at 60 with r3=phys */
+	.org purgatory_start + 0x60
+slave:
+	b .
+	/* ABI: end of copied region */
+	.org purgatory_start + 0x100
+	.size purgatory_start, . - purgatory_start
+
+/*
+ * The above 0x100 bytes at purgatory_start are replaced with the
+ * code from the kernel (or next stage) by setup_purgatory().
+ */
+
+master:
+	or	%r1,%r1,%r1	/* low priority to let other threads catchup */
+	isync
+	mr	%r17,%r3	/* save cpu id to r17 */
+	mr	%r15,%r4	/* save physical address in reg15 */
+
+	or	%r3,%r3,%r3	/* ok now to high priority, lets boot */
+	lis	%r6,0x1
+	mtctr	%r6		/* delay a bit for slaves to catch up */
+	bdnz	.		/* before we overwrite 0-100 again */
+
+	bl	0f		/* Work out where we're running */
+0:	mflr	%r18
+
+	/* load device-tree address */
+	ld	%r3, (dt_offset - 0b)(%r18)
+	mr	%r16,%r3	/* save dt address in reg16 */
+	li	%r4,20
+	LWZX_BE	%r6,%r3,%r4	/* fetch __be32 version number at byte 20 */
+	cmpwi	%r0,%r6,2	/* v2 or later? */
+	blt	1f
+	li	%r4,28
+	STWX_BE	%r17,%r3,%r4	/* Store my cpu as __be32 at byte 28 */
+1:
+	/* load the kernel address */
+	ld	%r4,(kernel - 0b)(%r18)
+
+	/* load the run_at_load flag */
+	/* possibly patched by kexec */
+	ld	%r6,(run_at_load - 0b)(%r18)
+	/* and patch it into the kernel */
+	stw	%r6,(0x5c)(%r4)
+
+	mr	%r3,%r16	/* restore dt address */
+
+	li	%r5,0		/* r5 will be 0 for kernel */
+
+	mfmsr	%r11
+	andi.	%r10,%r11,1	/* test MSR_LE */
+	bne	.Little_endian
+
+	mtctr	%r4		/* prepare branch to */
+	bctr			/* start kernel */
+
+.Little_endian:
+	mtsrr0	%r4		/* prepare branch to */
+
+	clrrdi	%r11,%r11,1	/* clear MSR_LE */
+	mtsrr1	%r11
+
+	rfid			/* update MSR and start kernel */
+
+
+	.balign 8
+	.globl kernel
+kernel:
+	.llong  0x0
+	.size kernel, . - kernel
+
+	.balign 8
+	.globl dt_offset
+dt_offset:
+	.llong  0x0
+	.size dt_offset, . - dt_offset
+
+
+	.data
+	.balign 8
+.globl sha256_digest
+sha256_digest:
+	.skip	32
+	.size sha256_digest, . - sha256_digest
+
+	.balign 8
+.globl sha_regions
+sha_regions:
+	.skip	8 * 2 * 16
+	.size sha_regions, . - sha_regions
diff --git a/arch/powerpc/sysdev/fsl_lbc.c b/arch/powerpc/sysdev/fsl_lbc.c
index 424b67fdb57f..5340a483cf55 100644
--- a/arch/powerpc/sysdev/fsl_lbc.c
+++ b/arch/powerpc/sysdev/fsl_lbc.c
@@ -31,7 +31,7 @@
 #include <asm/prom.h>
 #include <asm/fsl_lbc.h>
 
-static spinlock_t fsl_lbc_lock = __SPIN_LOCK_UNLOCKED(fsl_lbc_lock);
+static DEFINE_SPINLOCK(fsl_lbc_lock);
 struct fsl_lbc_ctrl *fsl_lbc_ctrl_dev;
 EXPORT_SYMBOL(fsl_lbc_ctrl_dev);
 
diff --git a/arch/powerpc/sysdev/fsl_pmc.c b/arch/powerpc/sysdev/fsl_pmc.c
index 1d6fd7c59fe9..232225e7f863 100644
--- a/arch/powerpc/sysdev/fsl_pmc.c
+++ b/arch/powerpc/sysdev/fsl_pmc.c
@@ -85,8 +85,4 @@ static struct platform_driver pmc_driver = {
 	.probe = pmc_probe,
 };
 
-static int __init pmc_init(void)
-{
-	return platform_driver_register(&pmc_driver);
-}
-device_initcall(pmc_init);
+builtin_platform_driver(pmc_driver);
diff --git a/arch/powerpc/sysdev/fsl_rio.c b/arch/powerpc/sysdev/fsl_rio.c
index 3cc7cace194a..1c41c51f22cb 100644
--- a/arch/powerpc/sysdev/fsl_rio.c
+++ b/arch/powerpc/sysdev/fsl_rio.c
@@ -80,10 +80,8 @@
 		"3:	li %1,-1\n"			\
 		"	li %0,%3\n"			\
 		"	b 2b\n"				\
-		".section __ex_table,\"a\"\n"		\
-			PPC_LONG_ALIGN "\n"		\
-			PPC_LONG "1b,3b\n"		\
-		".text"					\
+		".previous\n"				\
+		EX_TABLE(1b, 3b)			\
 		: "=r" (err), "=r" (x)			\
 		: "b" (addr), "i" (-EFAULT), "0" (err))
 
@@ -113,7 +111,7 @@ int fsl_rio_mcheck_exception(struct pt_regs *regs)
 			out_be32((u32 *)(rio_regs_win + RIO_LTLEDCSR),
 				 0);
 			regs->msr |= MSR_RI;
-			regs->nip = entry->fixup;
+			regs->nip = extable_fixup(entry);
 			return 1;
 		}
 	}
diff --git a/arch/powerpc/sysdev/fsl_soc.c b/arch/powerpc/sysdev/fsl_soc.c
index d93056eedcb0..19101f9cfcfc 100644
--- a/arch/powerpc/sysdev/fsl_soc.c
+++ b/arch/powerpc/sysdev/fsl_soc.c
@@ -77,13 +77,10 @@ phys_addr_t get_immrbase(void)
 
 EXPORT_SYMBOL(get_immrbase);
 
-static u32 sysfreq = -1;
-
 u32 fsl_get_sys_freq(void)
 {
+	static u32 sysfreq = -1;
 	struct device_node *soc;
-	const u32 *prop;
-	int size;
 
 	if (sysfreq != -1)
 		return sysfreq;
@@ -92,12 +89,9 @@ u32 fsl_get_sys_freq(void)
 	if (!soc)
 		return -1;
 
-	prop = of_get_property(soc, "clock-frequency", &size);
-	if (!prop || size != sizeof(*prop) || *prop == 0)
-		prop = of_get_property(soc, "bus-frequency", &size);
-
-	if (prop && size == sizeof(*prop))
-		sysfreq = *prop;
+	of_property_read_u32(soc, "clock-frequency", &sysfreq);
+	if (sysfreq == -1 || !sysfreq)
+		of_property_read_u32(soc, "bus-frequency", &sysfreq);
 
 	of_node_put(soc);
 	return sysfreq;
@@ -106,23 +100,17 @@ EXPORT_SYMBOL(fsl_get_sys_freq);
 
 #if defined(CONFIG_CPM2) || defined(CONFIG_QUICC_ENGINE) || defined(CONFIG_8xx)
 
-static u32 brgfreq = -1;
-
 u32 get_brgfreq(void)
 {
+	static u32 brgfreq = -1;
 	struct device_node *node;
-	const unsigned int *prop;
-	int size;
 
 	if (brgfreq != -1)
 		return brgfreq;
 
 	node = of_find_compatible_node(NULL, NULL, "fsl,cpm-brg");
 	if (node) {
-		prop = of_get_property(node, "clock-frequency", &size);
-		if (prop && size == 4)
-			brgfreq = *prop;
-
+		of_property_read_u32(node, "clock-frequency", &brgfreq);
 		of_node_put(node);
 		return brgfreq;
 	}
@@ -135,15 +123,11 @@ u32 get_brgfreq(void)
 		node = of_find_node_by_type(NULL, "qe");
 
 	if (node) {
-		prop = of_get_property(node, "brg-frequency", &size);
-		if (prop && size == 4)
-			brgfreq = *prop;
-
-		if (brgfreq == -1 || brgfreq == 0) {
-			prop = of_get_property(node, "bus-frequency", &size);
-			if (prop && size == 4)
-				brgfreq = *prop / 2;
-		}
+		of_property_read_u32(node, "brg-frequency", &brgfreq);
+		if (brgfreq == -1 || !brgfreq)
+			if (!of_property_read_u32(node, "bus-frequency",
+						  &brgfreq))
+				brgfreq /= 2;
 		of_node_put(node);
 	}
 
@@ -152,10 +136,9 @@ u32 get_brgfreq(void)
 
 EXPORT_SYMBOL(get_brgfreq);
 
-static u32 fs_baudrate = -1;
-
 u32 get_baudrate(void)
 {
+	static u32 fs_baudrate = -1;
 	struct device_node *node;
 
 	if (fs_baudrate != -1)
@@ -163,12 +146,7 @@ u32 get_baudrate(void)
 
 	node = of_find_node_by_type(NULL, "serial");
 	if (node) {
-		int size;
-		const unsigned int *prop = of_get_property(node,
-				"current-speed", &size);
-
-		if (prop)
-			fs_baudrate = *prop;
+		of_property_read_u32(node, "current-speed", &fs_baudrate);
 		of_node_put(node);
 	}
 
diff --git a/arch/powerpc/sysdev/tsi108_pci.c b/arch/powerpc/sysdev/tsi108_pci.c
index 57c971b7839c..53a16aa4d384 100644
--- a/arch/powerpc/sysdev/tsi108_pci.c
+++ b/arch/powerpc/sysdev/tsi108_pci.c
@@ -137,10 +137,8 @@ void tsi108_clear_pci_error(u32 pci_cfg_base)
 		".section .fixup,\"ax\"\n"		\
 		"3:	li %0,-1\n"			\
 		"	b 2b\n"				\
-		".section __ex_table,\"a\"\n"		\
-		"	.align 2\n"			\
-		"	.long 1b,3b\n"			\
-		".text"					\
+		".previous\n"				\
+		EX_TABLE(1b, 3b)			\
 		: "=r"(x) : "r"(addr))
 
 int
diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index 760545519a0b..9c0e17cf6886 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -10,6 +10,8 @@
  *      as published by the Free Software Foundation; either version
  *      2 of the License, or (at your option) any later version.
  */
+
+#include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/sched.h>
 #include <linux/smp.h>
@@ -225,6 +227,7 @@ Commands:\n\
 #endif
   "\
   dr	dump stream of raw bytes\n\
+  dt	dump the tracing buffers (uses printk)\n\
   e	print exception information\n\
   f	flush cache\n\
   la	lookup symbol+offset of specified address\n\
@@ -2364,6 +2367,9 @@ dump(void)
 		dump_log_buf();
 	} else if (c == 'o') {
 		dump_opal_msglog();
+	} else if (c == 't') {
+		ftrace_dump(DUMP_ALL);
+		tracing_on();
 	} else if (c == 'r') {
 		scanhex(&ndump);
 		if (ndump == 0)
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 028f97be5bae..c6722112527d 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -136,6 +136,7 @@ config S390
 	select HAVE_CMPXCHG_LOCAL
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DMA_API_DEBUG
+	select HAVE_DMA_CONTIGUOUS
 	select HAVE_DYNAMIC_FTRACE
 	select HAVE_DYNAMIC_FTRACE_WITH_REGS
 	select HAVE_EFFICIENT_UNALIGNED_ACCESS
@@ -169,6 +170,7 @@ config S390
 	select OLD_SIGSUSPEND3
 	select SPARSE_IRQ
 	select SYSCTL_EXCEPTION_TRACE
+	select THREAD_INFO_IN_TASK
 	select TTY
 	select VIRT_CPU_ACCOUNTING
 	select ARCH_HAS_SCALED_CPUTIME
diff --git a/arch/s390/boot/compressed/Makefile b/arch/s390/boot/compressed/Makefile
index 0daa070d6c9d..6bd2c9022be3 100644
--- a/arch/s390/boot/compressed/Makefile
+++ b/arch/s390/boot/compressed/Makefile
@@ -10,7 +10,7 @@ targets	:= vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2
 targets += vmlinux.bin.xz vmlinux.bin.lzma vmlinux.bin.lzo vmlinux.bin.lz4
 targets += misc.o piggy.o sizes.h head.o
 
-KBUILD_CFLAGS := -m64 -D__KERNEL__ $(LINUX_INCLUDE) -O2
+KBUILD_CFLAGS := -m64 -D__KERNEL__ -O2
 KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
 KBUILD_CFLAGS += $(cflags-y) -fno-delete-null-pointer-checks -msoft-float
 KBUILD_CFLAGS += $(call cc-option,-mpacked-stack)
diff --git a/arch/s390/boot/compressed/head.S b/arch/s390/boot/compressed/head.S
index 28c4f96a2d9c..11f6254c561e 100644
--- a/arch/s390/boot/compressed/head.S
+++ b/arch/s390/boot/compressed/head.S
@@ -46,7 +46,7 @@ mover_end:
 
 	.align	8
 .Lstack:
-	.quad	0x8000 + (1<<(PAGE_SHIFT+THREAD_ORDER))
+	.quad	0x8000 + (1<<(PAGE_SHIFT+THREAD_SIZE_ORDER))
 .Loffset:
 	.quad	0x11000
 .Lmvsize:
diff --git a/arch/s390/configs/default_defconfig b/arch/s390/configs/default_defconfig
index 45968686f918..e659daffe368 100644
--- a/arch/s390/configs/default_defconfig
+++ b/arch/s390/configs/default_defconfig
@@ -66,6 +66,8 @@ CONFIG_TRANSPARENT_HUGEPAGE=y
 CONFIG_CLEANCACHE=y
 CONFIG_FRONTSWAP=y
 CONFIG_CMA=y
+CONFIG_CMA_DEBUG=y
+CONFIG_CMA_DEBUGFS=y
 CONFIG_MEM_SOFT_DIRTY=y
 CONFIG_ZPOOL=m
 CONFIG_ZBUD=m
@@ -366,6 +368,8 @@ CONFIG_BPF_JIT=y
 CONFIG_NET_PKTGEN=m
 CONFIG_NET_TCPPROBE=m
 CONFIG_DEVTMPFS=y
+CONFIG_DMA_CMA=y
+CONFIG_CMA_SIZE_MBYTES=0
 CONFIG_CONNECTOR=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_CRYPTOLOOP=m
@@ -438,7 +442,6 @@ CONFIG_TUN=m
 CONFIG_VETH=m
 CONFIG_VIRTIO_NET=m
 CONFIG_NLMON=m
-CONFIG_VHOST_NET=m
 # CONFIG_NET_VENDOR_ARC is not set
 # CONFIG_NET_VENDOR_CHELSIO is not set
 # CONFIG_NET_VENDOR_INTEL is not set
@@ -693,3 +696,4 @@ CONFIG_CMM=m
 CONFIG_APPLDATA_BASE=y
 CONFIG_KVM=m
 CONFIG_KVM_S390_UCONTROL=y
+CONFIG_VHOST_NET=m
diff --git a/arch/s390/configs/gcov_defconfig b/arch/s390/configs/gcov_defconfig
index 1dd05e345c4d..95ceac50bc65 100644
--- a/arch/s390/configs/gcov_defconfig
+++ b/arch/s390/configs/gcov_defconfig
@@ -362,6 +362,8 @@ CONFIG_BPF_JIT=y
 CONFIG_NET_PKTGEN=m
 CONFIG_NET_TCPPROBE=m
 CONFIG_DEVTMPFS=y
+CONFIG_DMA_CMA=y
+CONFIG_CMA_SIZE_MBYTES=0
 CONFIG_CONNECTOR=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_CRYPTOLOOP=m
@@ -434,7 +436,6 @@ CONFIG_TUN=m
 CONFIG_VETH=m
 CONFIG_VIRTIO_NET=m
 CONFIG_NLMON=m
-CONFIG_VHOST_NET=m
 # CONFIG_NET_VENDOR_ARC is not set
 # CONFIG_NET_VENDOR_CHELSIO is not set
 # CONFIG_NET_VENDOR_INTEL is not set
@@ -633,3 +634,4 @@ CONFIG_CMM=m
 CONFIG_APPLDATA_BASE=y
 CONFIG_KVM=m
 CONFIG_KVM_S390_UCONTROL=y
+CONFIG_VHOST_NET=m
diff --git a/arch/s390/configs/performance_defconfig b/arch/s390/configs/performance_defconfig
index 29d1178666f0..bc7b176f5795 100644
--- a/arch/s390/configs/performance_defconfig
+++ b/arch/s390/configs/performance_defconfig
@@ -362,6 +362,8 @@ CONFIG_BPF_JIT=y
 CONFIG_NET_PKTGEN=m
 CONFIG_NET_TCPPROBE=m
 CONFIG_DEVTMPFS=y
+CONFIG_DMA_CMA=y
+CONFIG_CMA_SIZE_MBYTES=0
 CONFIG_CONNECTOR=y
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_BLK_DEV_CRYPTOLOOP=m
@@ -434,7 +436,6 @@ CONFIG_TUN=m
 CONFIG_VETH=m
 CONFIG_VIRTIO_NET=m
 CONFIG_NLMON=m
-CONFIG_VHOST_NET=m
 # CONFIG_NET_VENDOR_ARC is not set
 # CONFIG_NET_VENDOR_CHELSIO is not set
 # CONFIG_NET_VENDOR_INTEL is not set
@@ -632,3 +633,4 @@ CONFIG_CMM=m
 CONFIG_APPLDATA_BASE=y
 CONFIG_KVM=m
 CONFIG_KVM_S390_UCONTROL=y
+CONFIG_VHOST_NET=m
diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c
index 9cc050f9536c..1113389d0a39 100644
--- a/arch/s390/crypto/prng.c
+++ b/arch/s390/crypto/prng.c
@@ -507,8 +507,10 @@ static ssize_t prng_tdes_read(struct file *file, char __user *ubuf,
 		prng_data->prngws.byte_counter += n;
 		prng_data->prngws.reseed_counter += n;
 
-		if (copy_to_user(ubuf, prng_data->buf, chunk))
-			return -EFAULT;
+		if (copy_to_user(ubuf, prng_data->buf, chunk)) {
+			ret = -EFAULT;
+			break;
+		}
 
 		nbytes -= chunk;
 		ret += chunk;
diff --git a/arch/s390/hypfs/inode.c b/arch/s390/hypfs/inode.c
index 09bccb224d03..cf8a2d92467f 100644
--- a/arch/s390/hypfs/inode.c
+++ b/arch/s390/hypfs/inode.c
@@ -3,6 +3,7 @@
  *
  *    Copyright IBM Corp. 2006, 2008
  *    Author(s): Michael Holzheu <holzheu@de.ibm.com>
+ *    License: GPL
  */
 
 #define KMSG_COMPONENT "hypfs"
@@ -18,7 +19,8 @@
 #include <linux/time.h>
 #include <linux/parser.h>
 #include <linux/sysfs.h>
-#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kobject.h>
 #include <linux/seq_file.h>
 #include <linux/mount.h>
 #include <linux/uio.h>
@@ -443,7 +445,6 @@ static struct file_system_type hypfs_type = {
 	.mount		= hypfs_mount,
 	.kill_sb	= hypfs_kill_super
 };
-MODULE_ALIAS_FS("s390_hypfs");
 
 static const struct super_operations hypfs_s_ops = {
 	.statfs		= simple_statfs,
@@ -497,21 +498,4 @@ fail_dbfs_exit:
 	pr_err("Initialization of hypfs failed with rc=%i\n", rc);
 	return rc;
 }
-
-static void __exit hypfs_exit(void)
-{
-	unregister_filesystem(&hypfs_type);
-	sysfs_remove_mount_point(hypervisor_kobj, "s390");
-	hypfs_diag0c_exit();
-	hypfs_sprp_exit();
-	hypfs_vm_exit();
-	hypfs_diag_exit();
-	hypfs_dbfs_exit();
-}
-
-module_init(hypfs_init)
-module_exit(hypfs_exit)
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Michael Holzheu <holzheu@de.ibm.com>");
-MODULE_DESCRIPTION("s390 Hypervisor Filesystem");
+device_initcall(hypfs_init)
diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild
index 20f196b82a6e..8aea32fe8bd2 100644
--- a/arch/s390/include/asm/Kbuild
+++ b/arch/s390/include/asm/Kbuild
@@ -1,6 +1,6 @@
-
-
+generic-y += asm-offsets.h
 generic-y += clkdev.h
+generic-y += dma-contiguous.h
 generic-y += export.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
diff --git a/arch/s390/include/asm/asm-offsets.h b/arch/s390/include/asm/asm-offsets.h
deleted file mode 100644
index d370ee36a182..000000000000
--- a/arch/s390/include/asm/asm-offsets.h
+++ /dev/null
@@ -1 +0,0 @@
-#include <generated/asm-offsets.h>
diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h
index d28cc2f5b7b2..f7f69dfd2db2 100644
--- a/arch/s390/include/asm/atomic.h
+++ b/arch/s390/include/asm/atomic.h
@@ -1,13 +1,8 @@
 /*
- * Copyright IBM Corp. 1999, 2009
+ * Copyright IBM Corp. 1999, 2016
  * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
  *	      Denis Joseph Barrow,
- *	      Arnd Bergmann <arndb@de.ibm.com>,
- *
- * Atomic operations that C can't guarantee us.
- * Useful for resource counting etc.
- * s390 uses 'Compare And Swap' for atomicity in SMP environment.
- *
+ *	      Arnd Bergmann,
  */
 
 #ifndef __ARCH_S390_ATOMIC__
@@ -15,62 +10,12 @@
 
 #include <linux/compiler.h>
 #include <linux/types.h>
+#include <asm/atomic_ops.h>
 #include <asm/barrier.h>
 #include <asm/cmpxchg.h>
 
 #define ATOMIC_INIT(i)  { (i) }
 
-#define __ATOMIC_NO_BARRIER	"\n"
-
-#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
-
-#define __ATOMIC_OR	"lao"
-#define __ATOMIC_AND	"lan"
-#define __ATOMIC_ADD	"laa"
-#define __ATOMIC_XOR	"lax"
-#define __ATOMIC_BARRIER "bcr	14,0\n"
-
-#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier)		\
-({									\
-	int old_val;							\
-									\
-	typecheck(atomic_t *, ptr);					\
-	asm volatile(							\
-		op_string "	%0,%2,%1\n"				\
-		__barrier						\
-		: "=d" (old_val), "+Q" ((ptr)->counter)			\
-		: "d" (op_val)						\
-		: "cc", "memory");					\
-	old_val;							\
-})
-
-#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
-
-#define __ATOMIC_OR	"or"
-#define __ATOMIC_AND	"nr"
-#define __ATOMIC_ADD	"ar"
-#define __ATOMIC_XOR	"xr"
-#define __ATOMIC_BARRIER "\n"
-
-#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier)		\
-({									\
-	int old_val, new_val;						\
-									\
-	typecheck(atomic_t *, ptr);					\
-	asm volatile(							\
-		"	l	%0,%2\n"				\
-		"0:	lr	%1,%0\n"				\
-		op_string "	%1,%3\n"				\
-		"	cs	%0,%1,%2\n"				\
-		"	jl	0b"					\
-		: "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\
-		: "d" (op_val)						\
-		: "cc", "memory");					\
-	old_val;							\
-})
-
-#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
-
 static inline int atomic_read(const atomic_t *v)
 {
 	int c;
@@ -90,27 +35,23 @@ static inline void atomic_set(atomic_t *v, int i)
 
 static inline int atomic_add_return(int i, atomic_t *v)
 {
-	return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER) + i;
+	return __atomic_add_barrier(i, &v->counter) + i;
 }
 
 static inline int atomic_fetch_add(int i, atomic_t *v)
 {
-	return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER);
+	return __atomic_add_barrier(i, &v->counter);
 }
 
 static inline void atomic_add(int i, atomic_t *v)
 {
 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 	if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {
-		asm volatile(
-			"asi	%0,%1\n"
-			: "+Q" (v->counter)
-			: "i" (i)
-			: "cc", "memory");
+		__atomic_add_const(i, &v->counter);
 		return;
 	}
 #endif
-	__ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_NO_BARRIER);
+	__atomic_add(i, &v->counter);
 }
 
 #define atomic_add_negative(_i, _v)	(atomic_add_return(_i, _v) < 0)
@@ -125,19 +66,19 @@ static inline void atomic_add(int i, atomic_t *v)
 #define atomic_dec_return(_v)		atomic_sub_return(1, _v)
 #define atomic_dec_and_test(_v)		(atomic_sub_return(1, _v) == 0)
 
-#define ATOMIC_OPS(op, OP)						\
+#define ATOMIC_OPS(op)							\
 static inline void atomic_##op(int i, atomic_t *v)			\
 {									\
-	__ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_NO_BARRIER);	\
+	__atomic_##op(i, &v->counter);					\
 }									\
 static inline int atomic_fetch_##op(int i, atomic_t *v)			\
 {									\
-	return __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_BARRIER);	\
+	return __atomic_##op##_barrier(i, &v->counter);			\
 }
 
-ATOMIC_OPS(and, AND)
-ATOMIC_OPS(or, OR)
-ATOMIC_OPS(xor, XOR)
+ATOMIC_OPS(and)
+ATOMIC_OPS(or)
+ATOMIC_OPS(xor)
 
 #undef ATOMIC_OPS
 
@@ -145,12 +86,7 @@ ATOMIC_OPS(xor, XOR)
 
 static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
 {
-	asm volatile(
-		"	cs	%0,%2,%1"
-		: "+d" (old), "+Q" (v->counter)
-		: "d" (new)
-		: "cc", "memory");
-	return old;
+	return __atomic_cmpxchg(&v->counter, old, new);
 }
 
 static inline int __atomic_add_unless(atomic_t *v, int a, int u)
@@ -168,65 +104,11 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u)
 	return c;
 }
 
-
-#undef __ATOMIC_LOOP
-
 #define ATOMIC64_INIT(i)  { (i) }
 
-#define __ATOMIC64_NO_BARRIER	"\n"
-
-#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
-
-#define __ATOMIC64_OR	"laog"
-#define __ATOMIC64_AND	"lang"
-#define __ATOMIC64_ADD	"laag"
-#define __ATOMIC64_XOR	"laxg"
-#define __ATOMIC64_BARRIER "bcr	14,0\n"
-
-#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier)		\
-({									\
-	long long old_val;						\
-									\
-	typecheck(atomic64_t *, ptr);					\
-	asm volatile(							\
-		op_string "	%0,%2,%1\n"				\
-		__barrier						\
-		: "=d" (old_val), "+Q" ((ptr)->counter)			\
-		: "d" (op_val)						\
-		: "cc", "memory");					\
-	old_val;							\
-})
-
-#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
-
-#define __ATOMIC64_OR	"ogr"
-#define __ATOMIC64_AND	"ngr"
-#define __ATOMIC64_ADD	"agr"
-#define __ATOMIC64_XOR	"xgr"
-#define __ATOMIC64_BARRIER "\n"
-
-#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier)		\
-({									\
-	long long old_val, new_val;					\
-									\
-	typecheck(atomic64_t *, ptr);					\
-	asm volatile(							\
-		"	lg	%0,%2\n"				\
-		"0:	lgr	%1,%0\n"				\
-		op_string "	%1,%3\n"				\
-		"	csg	%0,%1,%2\n"				\
-		"	jl	0b"					\
-		: "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\
-		: "d" (op_val)						\
-		: "cc", "memory");					\
-	old_val;							\
-})
-
-#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
-
-static inline long long atomic64_read(const atomic64_t *v)
+static inline long atomic64_read(const atomic64_t *v)
 {
-	long long c;
+	long c;
 
 	asm volatile(
 		"	lg	%0,%1\n"
@@ -234,71 +116,60 @@ static inline long long atomic64_read(const atomic64_t *v)
 	return c;
 }
 
-static inline void atomic64_set(atomic64_t *v, long long i)
+static inline void atomic64_set(atomic64_t *v, long i)
 {
 	asm volatile(
 		"	stg	%1,%0\n"
 		: "=Q" (v->counter) : "d" (i));
 }
 
-static inline long long atomic64_add_return(long long i, atomic64_t *v)
+static inline long atomic64_add_return(long i, atomic64_t *v)
 {
-	return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER) + i;
+	return __atomic64_add_barrier(i, &v->counter) + i;
 }
 
-static inline long long atomic64_fetch_add(long long i, atomic64_t *v)
+static inline long atomic64_fetch_add(long i, atomic64_t *v)
 {
-	return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER);
+	return __atomic64_add_barrier(i, &v->counter);
 }
 
-static inline void atomic64_add(long long i, atomic64_t *v)
+static inline void atomic64_add(long i, atomic64_t *v)
 {
 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 	if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {
-		asm volatile(
-			"agsi	%0,%1\n"
-			: "+Q" (v->counter)
-			: "i" (i)
-			: "cc", "memory");
+		__atomic64_add_const(i, &v->counter);
 		return;
 	}
 #endif
-	__ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_NO_BARRIER);
+	__atomic64_add(i, &v->counter);
 }
 
 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
 
-static inline long long atomic64_cmpxchg(atomic64_t *v,
-					     long long old, long long new)
+static inline long atomic64_cmpxchg(atomic64_t *v, long old, long new)
 {
-	asm volatile(
-		"	csg	%0,%2,%1"
-		: "+d" (old), "+Q" (v->counter)
-		: "d" (new)
-		: "cc", "memory");
-	return old;
+	return __atomic64_cmpxchg(&v->counter, old, new);
 }
 
-#define ATOMIC64_OPS(op, OP)						\
+#define ATOMIC64_OPS(op)						\
 static inline void atomic64_##op(long i, atomic64_t *v)			\
 {									\
-	__ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_NO_BARRIER);	\
+	__atomic64_##op(i, &v->counter);				\
 }									\
 static inline long atomic64_fetch_##op(long i, atomic64_t *v)		\
 {									\
-	return __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_BARRIER); \
+	return __atomic64_##op##_barrier(i, &v->counter);		\
 }
 
-ATOMIC64_OPS(and, AND)
-ATOMIC64_OPS(or, OR)
-ATOMIC64_OPS(xor, XOR)
+ATOMIC64_OPS(and)
+ATOMIC64_OPS(or)
+ATOMIC64_OPS(xor)
 
 #undef ATOMIC64_OPS
-#undef __ATOMIC64_LOOP
 
-static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u)
+static inline int atomic64_add_unless(atomic64_t *v, long i, long u)
 {
-	long long c, old;
+	long c, old;
 
 	c = atomic64_read(v);
 	for (;;) {
@@ -312,9 +183,9 @@ static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u)
 	return c != u;
 }
 
-static inline long long atomic64_dec_if_positive(atomic64_t *v)
+static inline long atomic64_dec_if_positive(atomic64_t *v)
 {
-	long long c, old, dec;
+	long c, old, dec;
 
 	c = atomic64_read(v);
 	for (;;) {
@@ -333,9 +204,9 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v)
 #define atomic64_inc(_v)		atomic64_add(1, _v)
 #define atomic64_inc_return(_v)		atomic64_add_return(1, _v)
 #define atomic64_inc_and_test(_v)	(atomic64_add_return(1, _v) == 0)
-#define atomic64_sub_return(_i, _v)	atomic64_add_return(-(long long)(_i), _v)
-#define atomic64_fetch_sub(_i, _v)	atomic64_fetch_add(-(long long)(_i), _v)
-#define atomic64_sub(_i, _v)		atomic64_add(-(long long)(_i), _v)
+#define atomic64_sub_return(_i, _v)	atomic64_add_return(-(long)(_i), _v)
+#define atomic64_fetch_sub(_i, _v)	atomic64_fetch_add(-(long)(_i), _v)
+#define atomic64_sub(_i, _v)		atomic64_add(-(long)(_i), _v)
 #define atomic64_sub_and_test(_i, _v)	(atomic64_sub_return(_i, _v) == 0)
 #define atomic64_dec(_v)		atomic64_sub(1, _v)
 #define atomic64_dec_return(_v)		atomic64_sub_return(1, _v)
diff --git a/arch/s390/include/asm/atomic_ops.h b/arch/s390/include/asm/atomic_ops.h
new file mode 100644
index 000000000000..ac9e2b939d04
--- /dev/null
+++ b/arch/s390/include/asm/atomic_ops.h
@@ -0,0 +1,130 @@
+/*
+ * Low level function for atomic operations
+ *
+ * Copyright IBM Corp. 1999, 2016
+ */
+
+#ifndef __ARCH_S390_ATOMIC_OPS__
+#define __ARCH_S390_ATOMIC_OPS__
+
+#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
+
+#define __ATOMIC_OP(op_name, op_type, op_string, op_barrier)		\
+static inline op_type op_name(op_type val, op_type *ptr)		\
+{									\
+	op_type old;							\
+									\
+	asm volatile(							\
+		op_string "	%[old],%[val],%[ptr]\n"			\
+		op_barrier						\
+		: [old] "=d" (old), [ptr] "+Q" (*ptr)			\
+		: [val] "d" (val) : "cc", "memory");			\
+	return old;							\
+}									\
+
+#define __ATOMIC_OPS(op_name, op_type, op_string)			\
+	__ATOMIC_OP(op_name, op_type, op_string, "\n")			\
+	__ATOMIC_OP(op_name##_barrier, op_type, op_string, "bcr 14,0\n")
+
+__ATOMIC_OPS(__atomic_add, int, "laa")
+__ATOMIC_OPS(__atomic_and, int, "lan")
+__ATOMIC_OPS(__atomic_or,  int, "lao")
+__ATOMIC_OPS(__atomic_xor, int, "lax")
+
+__ATOMIC_OPS(__atomic64_add, long, "laag")
+__ATOMIC_OPS(__atomic64_and, long, "lang")
+__ATOMIC_OPS(__atomic64_or,  long, "laog")
+__ATOMIC_OPS(__atomic64_xor, long, "laxg")
+
+#undef __ATOMIC_OPS
+#undef __ATOMIC_OP
+
+static inline void __atomic_add_const(int val, int *ptr)
+{
+	asm volatile(
+		"	asi	%[ptr],%[val]\n"
+		: [ptr] "+Q" (*ptr) : [val] "i" (val) : "cc");
+}
+
+static inline void __atomic64_add_const(long val, long *ptr)
+{
+	asm volatile(
+		"	agsi	%[ptr],%[val]\n"
+		: [ptr] "+Q" (*ptr) : [val] "i" (val) : "cc");
+}
+
+#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
+
+#define __ATOMIC_OP(op_name, op_string)					\
+static inline int op_name(int val, int *ptr)				\
+{									\
+	int old, new;							\
+									\
+	asm volatile(							\
+		"0:	lr	%[new],%[old]\n"			\
+		op_string "	%[new],%[val]\n"			\
+		"	cs	%[old],%[new],%[ptr]\n"			\
+		"	jl	0b"					\
+		: [old] "=d" (old), [new] "=&d" (new), [ptr] "+Q" (*ptr)\
+		: [val] "d" (val), "0" (*ptr) : "cc", "memory");	\
+	return old;							\
+}
+
+#define __ATOMIC_OPS(op_name, op_string)				\
+	__ATOMIC_OP(op_name, op_string)					\
+	__ATOMIC_OP(op_name##_barrier, op_string)
+
+__ATOMIC_OPS(__atomic_add, "ar")
+__ATOMIC_OPS(__atomic_and, "nr")
+__ATOMIC_OPS(__atomic_or,  "or")
+__ATOMIC_OPS(__atomic_xor, "xr")
+
+#undef __ATOMIC_OPS
+
+#define __ATOMIC64_OP(op_name, op_string)				\
+static inline long op_name(long val, long *ptr)				\
+{									\
+	long old, new;							\
+									\
+	asm volatile(							\
+		"0:	lgr	%[new],%[old]\n"			\
+		op_string "	%[new],%[val]\n"			\
+		"	csg	%[old],%[new],%[ptr]\n"			\
+		"	jl	0b"					\
+		: [old] "=d" (old), [new] "=&d" (new), [ptr] "+Q" (*ptr)\
+		: [val] "d" (val), "0" (*ptr) : "cc", "memory");	\
+	return old;							\
+}
+
+#define __ATOMIC64_OPS(op_name, op_string)				\
+	__ATOMIC64_OP(op_name, op_string)				\
+	__ATOMIC64_OP(op_name##_barrier, op_string)
+
+__ATOMIC64_OPS(__atomic64_add, "agr")
+__ATOMIC64_OPS(__atomic64_and, "ngr")
+__ATOMIC64_OPS(__atomic64_or,  "ogr")
+__ATOMIC64_OPS(__atomic64_xor, "xgr")
+
+#undef __ATOMIC64_OPS
+
+#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
+
+static inline int __atomic_cmpxchg(int *ptr, int old, int new)
+{
+	asm volatile(
+		"	cs	%[old],%[new],%[ptr]"
+		: [old] "+d" (old), [ptr] "+Q" (*ptr)
+		: [new] "d" (new) : "cc", "memory");
+	return old;
+}
+
+static inline long __atomic64_cmpxchg(long *ptr, long old, long new)
+{
+	asm volatile(
+		"	csg	%[old],%[new],%[ptr]"
+		: [old] "+d" (old), [ptr] "+Q" (*ptr)
+		: [new] "d" (new) : "cc", "memory");
+	return old;
+}
+
+#endif /* __ARCH_S390_ATOMIC_OPS__  */
diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h
index 8043f10da6b5..d92047da5ccb 100644
--- a/arch/s390/include/asm/bitops.h
+++ b/arch/s390/include/asm/bitops.h
@@ -42,57 +42,9 @@
 
 #include <linux/typecheck.h>
 #include <linux/compiler.h>
+#include <asm/atomic_ops.h>
 #include <asm/barrier.h>
 
-#define __BITOPS_NO_BARRIER	"\n"
-
-#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
-
-#define __BITOPS_OR		"laog"
-#define __BITOPS_AND		"lang"
-#define __BITOPS_XOR		"laxg"
-#define __BITOPS_BARRIER	"bcr	14,0\n"
-
-#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)	\
-({								\
-	unsigned long __old;					\
-								\
-	typecheck(unsigned long *, (__addr));			\
-	asm volatile(						\
-		__op_string "	%0,%2,%1\n"			\
-		__barrier					\
-		: "=d" (__old),	"+Q" (*(__addr))		\
-		: "d" (__val)					\
-		: "cc", "memory");				\
-	__old;							\
-})
-
-#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
-
-#define __BITOPS_OR		"ogr"
-#define __BITOPS_AND		"ngr"
-#define __BITOPS_XOR		"xgr"
-#define __BITOPS_BARRIER	"\n"
-
-#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)	\
-({								\
-	unsigned long __old, __new;				\
-								\
-	typecheck(unsigned long *, (__addr));			\
-	asm volatile(						\
-		"	lg	%0,%2\n"			\
-		"0:	lgr	%1,%0\n"			\
-		__op_string "	%1,%3\n"			\
-		"	csg	%0,%1,%2\n"			\
-		"	jl	0b"				\
-		: "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\
-		: "d" (__val)					\
-		: "cc", "memory");				\
-	__old;							\
-})
-
-#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
-
 #define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)
 
 static inline unsigned long *
@@ -128,7 +80,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
 	}
 #endif
 	mask = 1UL << (nr & (BITS_PER_LONG - 1));
-	__BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_NO_BARRIER);
+	__atomic64_or(mask, addr);
 }
 
 static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
@@ -149,7 +101,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
 	}
 #endif
 	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
-	__BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_NO_BARRIER);
+	__atomic64_and(mask, addr);
 }
 
 static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
@@ -170,7 +122,7 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
 	}
 #endif
 	mask = 1UL << (nr & (BITS_PER_LONG - 1));
-	__BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_NO_BARRIER);
+	__atomic64_xor(mask, addr);
 }
 
 static inline int
@@ -180,7 +132,7 @@ test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
 	unsigned long old, mask;
 
 	mask = 1UL << (nr & (BITS_PER_LONG - 1));
-	old = __BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_BARRIER);
+	old = __atomic64_or_barrier(mask, addr);
 	return (old & mask) != 0;
 }
 
@@ -191,7 +143,7 @@ test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
 	unsigned long old, mask;
 
 	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
-	old = __BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_BARRIER);
+	old = __atomic64_and_barrier(mask, addr);
 	return (old & ~mask) != 0;
 }
 
@@ -202,7 +154,7 @@ test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
 	unsigned long old, mask;
 
 	mask = 1UL << (nr & (BITS_PER_LONG - 1));
-	old = __BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_BARRIER);
+	old = __atomic64_xor_barrier(mask, addr);
 	return (old & mask) != 0;
 }
 
diff --git a/arch/s390/include/asm/cpu_mf.h b/arch/s390/include/asm/cpu_mf.h
index 03516476127b..428c41239a49 100644
--- a/arch/s390/include/asm/cpu_mf.h
+++ b/arch/s390/include/asm/cpu_mf.h
@@ -104,7 +104,8 @@ struct hws_basic_entry {
 	unsigned int P:1;	    /* 28 PSW Problem state		 */
 	unsigned int AS:2;	    /* 29-30 PSW address-space control	 */
 	unsigned int I:1;	    /* 31 entry valid or invalid	 */
-	unsigned int:16;
+	unsigned int CL:2;	    /* 32-33 Configuration Level	 */
+	unsigned int:14;
 	unsigned int prim_asn:16;   /* primary ASN			 */
 	unsigned long long ia;	    /* Instruction Address		 */
 	unsigned long long gpp;     /* Guest Program Parameter		 */
@@ -212,18 +213,14 @@ static inline int stcctm5(u64 num, u64 *val)
 /* Query sampling information */
 static inline int qsi(struct hws_qsi_info_block *info)
 {
-	int cc;
-	cc = 1;
+	int cc = 1;
 
 	asm volatile(
-		"0:	.insn	s,0xb2860000,0(%1)\n"
+		"0:	.insn	s,0xb2860000,%1\n"
 		"1:	lhi	%0,0\n"
 		"2:\n"
 		EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
-		: "=d" (cc), "+a" (info)
-		: "m" (*info)
-		: "cc", "memory");
-
+		: "+d" (cc), "+Q" (*info));
 	return cc ? -EINVAL : 0;
 }
 
diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h
index 1736c7d3c94c..f4381e1fb19e 100644
--- a/arch/s390/include/asm/elf.h
+++ b/arch/s390/include/asm/elf.h
@@ -193,7 +193,7 @@ extern char elf_platform[];
 do {								\
 	set_personality(PER_LINUX |				\
 		(current->personality & (~PER_MASK)));		\
-	current_thread_info()->sys_call_table = 		\
+	current->thread.sys_call_table =			\
 		(unsigned long) &sys_call_table;		\
 } while (0)
 #else /* CONFIG_COMPAT */
@@ -204,11 +204,11 @@ do {								\
 			(current->personality & ~PER_MASK));	\
 	if ((ex).e_ident[EI_CLASS] == ELFCLASS32) {		\
 		set_thread_flag(TIF_31BIT);			\
-		current_thread_info()->sys_call_table =		\
+		current->thread.sys_call_table =		\
 			(unsigned long)	&sys_call_table_emu;	\
 	} else {						\
 		clear_thread_flag(TIF_31BIT);			\
-		current_thread_info()->sys_call_table =		\
+		current->thread.sys_call_table =		\
 			(unsigned long) &sys_call_table;	\
 	}							\
 } while (0)
diff --git a/arch/s390/include/asm/facilities_src.h b/arch/s390/include/asm/facilities_src.h
deleted file mode 100644
index 3b758f66e48b..000000000000
--- a/arch/s390/include/asm/facilities_src.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- *    Copyright IBM Corp. 2015
- */
-
-#ifndef S390_GEN_FACILITIES_C
-#error "This file can only be included by gen_facilities.c"
-#endif
-
-#include <linux/kconfig.h>
-
-struct facility_def {
-	char *name;
-	int *bits;
-};
-
-static struct facility_def facility_defs[] = {
-	{
-		/*
-		 * FACILITIES_ALS contains the list of facilities that are
-		 * required to run a kernel that is compiled e.g. with
-		 * -march=<machine>.
-		 */
-		.name = "FACILITIES_ALS",
-		.bits = (int[]){
-#ifdef CONFIG_HAVE_MARCH_Z900_FEATURES
-			0,  /* N3 instructions */
-			1,  /* z/Arch mode installed */
-#endif
-#ifdef CONFIG_HAVE_MARCH_Z990_FEATURES
-			18, /* long displacement facility */
-#endif
-#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
-			7,  /* stfle */
-			17, /* message security assist */
-			21, /* extended-immediate facility */
-			25, /* store clock fast */
-#endif
-#ifdef CONFIG_HAVE_MARCH_Z10_FEATURES
-			27, /* mvcos */
-			32, /* compare and swap and store */
-			33, /* compare and swap and store 2 */
-			34, /* general extension facility */
-			35, /* execute extensions */
-#endif
-#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
-			45, /* fast-BCR, etc. */
-#endif
-#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
-			49, /* misc-instruction-extensions */
-			52, /* interlocked facility 2 */
-#endif
-#ifdef CONFIG_HAVE_MARCH_Z13_FEATURES
-			53, /* load-and-zero-rightmost-byte, etc. */
-#endif
-			-1 /* END */
-		}
-	},
-	{
-		.name = "FACILITIES_KVM",
-		.bits = (int[]){
-			0,  /* N3 instructions */
-			1,  /* z/Arch mode installed */
-			2,  /* z/Arch mode active */
-			3,  /* DAT-enhancement */
-			4,  /* idte segment table */
-			5,  /* idte region table */
-			6,  /* ASN-and-LX reuse */
-			7,  /* stfle */
-			8,  /* enhanced-DAT 1 */
-			9,  /* sense-running-status */
-			10, /* conditional sske */
-			13, /* ipte-range */
-			14, /* nonquiescing key-setting */
-			73, /* transactional execution */
-			75, /* access-exception-fetch/store indication */
-			76, /* msa extension 3 */
-			77, /* msa extension 4 */
-			78, /* enhanced-DAT 2 */
-			-1  /* END */
-		}
-	},
-};
diff --git a/arch/s390/include/asm/ipl.h b/arch/s390/include/asm/ipl.h
index 4da22b2f0521..edb5161df7e2 100644
--- a/arch/s390/include/asm/ipl.h
+++ b/arch/s390/include/asm/ipl.h
@@ -97,7 +97,7 @@ void __init save_area_add_vxrs(struct save_area *, __vector128 *vxrs);
 extern void do_reipl(void);
 extern void do_halt(void);
 extern void do_poff(void);
-extern void ipl_save_parameters(void);
+extern void ipl_verify_parameters(void);
 extern void ipl_update_parameters(void);
 extern size_t append_ipl_vmparm(char *, size_t);
 extern size_t append_ipl_scpdata(char *, size_t);
diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h
index 7b93b78f423c..9bfad2ad6312 100644
--- a/arch/s390/include/asm/lowcore.h
+++ b/arch/s390/include/asm/lowcore.h
@@ -95,7 +95,7 @@ struct lowcore {
 
 	/* Current process. */
 	__u64	current_task;			/* 0x0310 */
-	__u64	thread_info;			/* 0x0318 */
+	__u8	pad_0x318[0x320-0x318];		/* 0x0318 */
 	__u64	kernel_stack;			/* 0x0320 */
 
 	/* Interrupt, panic and restart stack. */
@@ -126,7 +126,8 @@ struct lowcore {
 	__u64	percpu_offset;			/* 0x0378 */
 	__u64	vdso_per_cpu_data;		/* 0x0380 */
 	__u64	machine_flags;			/* 0x0388 */
-	__u8	pad_0x0390[0x0398-0x0390];	/* 0x0390 */
+	__u32	preempt_count;			/* 0x0390 */
+	__u8	pad_0x0394[0x0398-0x0394];	/* 0x0394 */
 	__u64	gmap;				/* 0x0398 */
 	__u32	spinlock_lockval;		/* 0x03a0 */
 	__u32	fpu_flags;			/* 0x03a4 */
diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
index 6611f798d2be..4e3186649578 100644
--- a/arch/s390/include/asm/pci.h
+++ b/arch/s390/include/asm/pci.h
@@ -133,6 +133,7 @@ struct zpci_dev {
 	/* Function measurement block */
 	struct zpci_fmb *fmb;
 	u16		fmb_update;	/* update interval */
+	u16		fmb_length;
 	/* software counters */
 	atomic64_t allocated_pages;
 	atomic64_t mapped_pages;
diff --git a/arch/s390/include/asm/pci_clp.h b/arch/s390/include/asm/pci_clp.h
index e75c64cbcf08..d6f1b1d94352 100644
--- a/arch/s390/include/asm/pci_clp.h
+++ b/arch/s390/include/asm/pci_clp.h
@@ -46,6 +46,8 @@ struct clp_fh_list_entry {
 #define CLP_UTIL_STR_LEN	64
 #define CLP_PFIP_NR_SEGMENTS	4
 
+extern bool zpci_unique_uid;
+
 /* List PCI functions request */
 struct clp_req_list_pci {
 	struct clp_req_hdr hdr;
@@ -59,7 +61,8 @@ struct clp_rsp_list_pci {
 	u64 resume_token;
 	u32 reserved2;
 	u16 max_fn;
-	u8 reserved3;
+	u8			: 7;
+	u8 uid_checking		: 1;
 	u8 entry_size;
 	struct clp_fh_list_entry fh_list[CLP_FH_LIST_NR_ENTRIES];
 } __packed;
@@ -84,7 +87,8 @@ struct clp_rsp_query_pci {
 	u16 pchid;
 	u32 bar[PCI_BAR_COUNT];
 	u8 pfip[CLP_PFIP_NR_SEGMENTS];	/* pci function internal path */
-	u32			: 24;
+	u32			: 16;
+	u8 fmb_len;
 	u8 pft;				/* pci function type */
 	u64 sdma;			/* start dma as */
 	u64 edma;			/* end dma as */
diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h
index f4eb9843eed4..166f703dad7c 100644
--- a/arch/s390/include/asm/pgalloc.h
+++ b/arch/s390/include/asm/pgalloc.h
@@ -27,17 +27,17 @@ extern int page_table_allocate_pgste;
 
 static inline void clear_table(unsigned long *s, unsigned long val, size_t n)
 {
-	typedef struct { char _[n]; } addrtype;
-
-	*s = val;
-	n = (n / 256) - 1;
-	asm volatile(
-		"	mvc	8(248,%0),0(%0)\n"
-		"0:	mvc	256(256,%0),0(%0)\n"
-		"	la	%0,256(%0)\n"
-		"	brct	%1,0b\n"
-		: "+a" (s), "+d" (n), "=m" (*(addrtype *) s)
-		: "m" (*(addrtype *) s));
+	struct addrtype { char _[256]; };
+	int i;
+
+	for (i = 0; i < n; i += 256) {
+		*s = val;
+		asm volatile(
+			"mvc	8(248,%[s]),0(%[s])\n"
+			: "+m" (*(struct addrtype *) s)
+			: [s] "a" (s));
+		s += 256 / sizeof(long);
+	}
 }
 
 static inline void crst_table_init(unsigned long *crst, unsigned long entry)
diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h
new file mode 100644
index 000000000000..b0776b2c8dcf
--- /dev/null
+++ b/arch/s390/include/asm/preempt.h
@@ -0,0 +1,137 @@
+#ifndef __ASM_PREEMPT_H
+#define __ASM_PREEMPT_H
+
+#include <asm/current.h>
+#include <linux/thread_info.h>
+#include <asm/atomic_ops.h>
+
+#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
+
+#define PREEMPT_ENABLED	(0 + PREEMPT_NEED_RESCHED)
+
+static inline int preempt_count(void)
+{
+	return READ_ONCE(S390_lowcore.preempt_count) & ~PREEMPT_NEED_RESCHED;
+}
+
+static inline void preempt_count_set(int pc)
+{
+	int old, new;
+
+	do {
+		old = READ_ONCE(S390_lowcore.preempt_count);
+		new = (old & PREEMPT_NEED_RESCHED) |
+			(pc & ~PREEMPT_NEED_RESCHED);
+	} while (__atomic_cmpxchg(&S390_lowcore.preempt_count,
+				  old, new) != old);
+}
+
+#define init_task_preempt_count(p)	do { } while (0)
+
+#define init_idle_preempt_count(p, cpu)	do { \
+	S390_lowcore.preempt_count = PREEMPT_ENABLED; \
+} while (0)
+
+static inline void set_preempt_need_resched(void)
+{
+	__atomic_and(~PREEMPT_NEED_RESCHED, &S390_lowcore.preempt_count);
+}
+
+static inline void clear_preempt_need_resched(void)
+{
+	__atomic_or(PREEMPT_NEED_RESCHED, &S390_lowcore.preempt_count);
+}
+
+static inline bool test_preempt_need_resched(void)
+{
+	return !(READ_ONCE(S390_lowcore.preempt_count) & PREEMPT_NEED_RESCHED);
+}
+
+static inline void __preempt_count_add(int val)
+{
+	if (__builtin_constant_p(val) && (val >= -128) && (val <= 127))
+		__atomic_add_const(val, &S390_lowcore.preempt_count);
+	else
+		__atomic_add(val, &S390_lowcore.preempt_count);
+}
+
+static inline void __preempt_count_sub(int val)
+{
+	__preempt_count_add(-val);
+}
+
+static inline bool __preempt_count_dec_and_test(void)
+{
+	return __atomic_add(-1, &S390_lowcore.preempt_count) == 1;
+}
+
+static inline bool should_resched(int preempt_offset)
+{
+	return unlikely(READ_ONCE(S390_lowcore.preempt_count) ==
+			preempt_offset);
+}
+
+#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
+
+#define PREEMPT_ENABLED	(0)
+
+static inline int preempt_count(void)
+{
+	return READ_ONCE(S390_lowcore.preempt_count);
+}
+
+static inline void preempt_count_set(int pc)
+{
+	S390_lowcore.preempt_count = pc;
+}
+
+#define init_task_preempt_count(p)	do { } while (0)
+
+#define init_idle_preempt_count(p, cpu)	do { \
+	S390_lowcore.preempt_count = PREEMPT_ENABLED; \
+} while (0)
+
+static inline void set_preempt_need_resched(void)
+{
+}
+
+static inline void clear_preempt_need_resched(void)
+{
+}
+
+static inline bool test_preempt_need_resched(void)
+{
+	return false;
+}
+
+static inline void __preempt_count_add(int val)
+{
+	S390_lowcore.preempt_count += val;
+}
+
+static inline void __preempt_count_sub(int val)
+{
+	S390_lowcore.preempt_count -= val;
+}
+
+static inline bool __preempt_count_dec_and_test(void)
+{
+	return !--S390_lowcore.preempt_count && tif_need_resched();
+}
+
+static inline bool should_resched(int preempt_offset)
+{
+	return unlikely(preempt_count() == preempt_offset &&
+			tif_need_resched());
+}
+
+#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
+
+#ifdef CONFIG_PREEMPT
+extern asmlinkage void preempt_schedule(void);
+#define __preempt_schedule() preempt_schedule()
+extern asmlinkage void preempt_schedule_notrace(void);
+#define __preempt_schedule_notrace() preempt_schedule_notrace()
+#endif /* CONFIG_PREEMPT */
+
+#endif /* __ASM_PREEMPT_H */
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index 9d3a21aedc97..6bca916a5ba0 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -110,14 +110,20 @@ typedef struct {
 struct thread_struct {
 	unsigned int  acrs[NUM_ACRS];
         unsigned long ksp;              /* kernel stack pointer             */
+	unsigned long user_timer;	/* task cputime in user space */
+	unsigned long system_timer;	/* task cputime in kernel space */
+	unsigned long sys_call_table;	/* system call table address */
 	mm_segment_t mm_segment;
 	unsigned long gmap_addr;	/* address of last gmap fault. */
 	unsigned int gmap_write_flag;	/* gmap fault write indication */
 	unsigned int gmap_int_code;	/* int code of last gmap fault */
 	unsigned int gmap_pfault;	/* signal of a pending guest pfault */
+	/* Per-thread information related to debugging */
 	struct per_regs per_user;	/* User specified PER registers */
 	struct per_event per_event;	/* Cause of the last PER trap */
 	unsigned long per_flags;	/* Flags to control debug behavior */
+	unsigned int system_call;	/* system call number in signal */
+	unsigned long last_break;	/* last breaking-event-address. */
         /* pfault_wait is used to block the process on a pfault event */
 	unsigned long pfault_wait;
 	struct list_head list;
diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h
index 2ad9c204b1a2..8db92a5b3bf1 100644
--- a/arch/s390/include/asm/sclp.h
+++ b/arch/s390/include/asm/sclp.h
@@ -101,7 +101,8 @@ struct zpci_report_error_header {
 	u8 data[0];	/* Subsequent Data passed verbatim to SCLP ET 24 */
 } __packed;
 
-int sclp_get_core_info(struct sclp_core_info *info);
+int _sclp_get_core_info_early(struct sclp_core_info *info);
+int _sclp_get_core_info(struct sclp_core_info *info);
 int sclp_core_configure(u8 core);
 int sclp_core_deconfigure(u8 core);
 int sclp_sdias_blk_count(void);
@@ -119,4 +120,11 @@ void sclp_early_detect(void);
 void _sclp_print_early(const char *);
 void sclp_ocf_cpc_name_copy(char *dst);
 
+static inline int sclp_get_core_info(struct sclp_core_info *info, int early)
+{
+	if (early)
+		return _sclp_get_core_info_early(info);
+	return _sclp_get_core_info(info);
+}
+
 #endif /* _ASM_S390_SCLP_H */
diff --git a/arch/s390/include/asm/scsw.h b/arch/s390/include/asm/scsw.h
index 4af99cdaddf5..17a7904f001a 100644
--- a/arch/s390/include/asm/scsw.h
+++ b/arch/s390/include/asm/scsw.h
@@ -96,7 +96,8 @@ struct tm_scsw {
 	u32 dstat:8;
 	u32 cstat:8;
 	u32 fcxs:8;
-	u32 schxs:8;
+	u32 ifob:1;
+	u32 sesq:7;
 } __attribute__ ((packed));
 
 /**
@@ -177,6 +178,9 @@ union scsw {
 #define SCHN_STAT_INTF_CTRL_CHK	 0x02
 #define SCHN_STAT_CHAIN_CHECK	 0x01
 
+#define SCSW_SESQ_DEV_NOFCX	 3
+#define SCSW_SESQ_PATH_NOFCX	 4
+
 /*
  * architectured values for first sense byte
  */
diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h
index 0cc383b9be7f..3deb134587b7 100644
--- a/arch/s390/include/asm/smp.h
+++ b/arch/s390/include/asm/smp.h
@@ -36,6 +36,7 @@ extern void smp_yield_cpu(int cpu);
 extern void smp_cpu_set_polarization(int cpu, int val);
 extern int smp_cpu_get_polarization(int cpu);
 extern void smp_fill_possible_mask(void);
+extern void smp_detect_cpus(void);
 
 #else /* CONFIG_SMP */
 
@@ -56,6 +57,7 @@ static inline int smp_store_status(int cpu) { return 0; }
 static inline int smp_vcpu_scheduled(int cpu) { return 1; }
 static inline void smp_yield_cpu(int cpu) { }
 static inline void smp_fill_possible_mask(void) { }
+static inline void smp_detect_cpus(void) { }
 
 #endif /* CONFIG_SMP */
 
@@ -69,6 +71,12 @@ static inline void smp_stop_cpu(void)
 	}
 }
 
+/* Return thread 0 CPU number as base CPU */
+static inline int smp_get_base_cpu(int cpu)
+{
+	return cpu - (cpu % (smp_cpu_mtid + 1));
+}
+
 #ifdef CONFIG_HOTPLUG_CPU
 extern int smp_rescan_cpus(void);
 extern void __noreturn cpu_die(void);
diff --git a/arch/s390/include/asm/string.h b/arch/s390/include/asm/string.h
index 8662f5c8e17f..e5f5c7074f2c 100644
--- a/arch/s390/include/asm/string.h
+++ b/arch/s390/include/asm/string.h
@@ -14,6 +14,7 @@
 #define __HAVE_ARCH_MEMCHR	/* inline & arch function */
 #define __HAVE_ARCH_MEMCMP	/* arch function */
 #define __HAVE_ARCH_MEMCPY	/* gcc builtin & arch function */
+#define __HAVE_ARCH_MEMMOVE	/* gcc builtin & arch function */
 #define __HAVE_ARCH_MEMSCAN	/* inline & arch function */
 #define __HAVE_ARCH_MEMSET	/* gcc builtin & arch function */
 #define __HAVE_ARCH_STRCAT	/* inline & arch function */
@@ -32,6 +33,7 @@
 extern int memcmp(const void *, const void *, size_t);
 extern void *memcpy(void *, const void *, size_t);
 extern void *memset(void *, int, size_t);
+extern void *memmove(void *, const void *, size_t);
 extern int strcmp(const char *,const char *);
 extern size_t strlcat(char *, const char *, size_t);
 extern size_t strlcpy(char *, const char *, size_t);
@@ -40,7 +42,6 @@ extern char *strncpy(char *, const char *, size_t);
 extern char *strrchr(const char *, int);
 extern char *strstr(const char *, const char *);
 
-#undef __HAVE_ARCH_MEMMOVE
 #undef __HAVE_ARCH_STRCHR
 #undef __HAVE_ARCH_STRNCHR
 #undef __HAVE_ARCH_STRNCMP
@@ -61,7 +62,7 @@ static inline void *memchr(const void * s, int c, size_t n)
 		"	jl	1f\n"
 		"	la	%0,0\n"
 		"1:"
-		: "+a" (ret), "+&a" (s) : "d" (r0) : "cc");
+		: "+a" (ret), "+&a" (s) : "d" (r0) : "cc", "memory");
 	return (void *) ret;
 }
 
@@ -73,7 +74,7 @@ static inline void *memscan(void *s, int c, size_t n)
 	asm volatile(
 		"0:	srst	%0,%1\n"
 		"	jo	0b\n"
-		: "+a" (ret), "+&a" (s) : "d" (r0) : "cc");
+		: "+a" (ret), "+&a" (s) : "d" (r0) : "cc", "memory");
 	return (void *) ret;
 }
 
@@ -114,7 +115,7 @@ static inline size_t strlen(const char *s)
 	asm volatile(
 		"0:	srst	%0,%1\n"
 		"	jo	0b"
-		: "+d" (r0), "+a" (tmp) :  : "cc");
+		: "+d" (r0), "+a" (tmp) :  : "cc", "memory");
 	return r0 - (unsigned long) s;
 }
 
@@ -127,7 +128,7 @@ static inline size_t strnlen(const char * s, size_t n)
 	asm volatile(
 		"0:	srst	%0,%1\n"
 		"	jo	0b"
-		: "+a" (end), "+a" (tmp) : "d" (r0)  : "cc");
+		: "+a" (end), "+a" (tmp) : "d" (r0)  : "cc", "memory");
 	return end - s;
 }
 #else /* IN_ARCH_STRING_C */
diff --git a/arch/s390/include/asm/sysinfo.h b/arch/s390/include/asm/sysinfo.h
index 2728114d5484..229326c942c7 100644
--- a/arch/s390/include/asm/sysinfo.h
+++ b/arch/s390/include/asm/sysinfo.h
@@ -107,6 +107,11 @@ struct sysinfo_2_2_2 {
 	char reserved_3[5];
 	unsigned short cpus_dedicated;
 	unsigned short cpus_shared;
+	char reserved_4[3];
+	unsigned char vsne;
+	uuid_be uuid;
+	char reserved_5[160];
+	char ext_name[256];
 };
 
 #define LPAR_CHAR_DEDICATED	(1 << 7)
@@ -127,7 +132,7 @@ struct sysinfo_3_2_2 {
 		unsigned int caf;
 		char cpi[16];
 		char reserved_1[3];
-		char ext_name_encoding;
+		unsigned char evmne;
 		unsigned int reserved_2;
 		uuid_be uuid;
 	} vm[8];
diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h
index f15c0398c363..a5b54a445eb8 100644
--- a/arch/s390/include/asm/thread_info.h
+++ b/arch/s390/include/asm/thread_info.h
@@ -12,10 +12,10 @@
 /*
  * Size of kernel stack for each process
  */
-#define THREAD_ORDER 2
+#define THREAD_SIZE_ORDER 2
 #define ASYNC_ORDER  2
 
-#define THREAD_SIZE (PAGE_SIZE << THREAD_ORDER)
+#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER)
 #define ASYNC_SIZE  (PAGE_SIZE << ASYNC_ORDER)
 
 #ifndef __ASSEMBLY__
@@ -30,15 +30,7 @@
  * - if the contents of this structure are changed, the assembly constants must also be changed
  */
 struct thread_info {
-	struct task_struct	*task;		/* main task structure */
 	unsigned long		flags;		/* low level flags */
-	unsigned long		sys_call_table;	/* System call table address */
-	unsigned int		cpu;		/* current CPU */
-	int			preempt_count;	/* 0 => preemptable, <0 => BUG */
-	unsigned int		system_call;
-	__u64			user_timer;
-	__u64			system_timer;
-	unsigned long		last_break;	/* last breaking-event-address. */
 };
 
 /*
@@ -46,26 +38,14 @@ struct thread_info {
  */
 #define INIT_THREAD_INFO(tsk)			\
 {						\
-	.task		= &tsk,			\
 	.flags		= 0,			\
-	.cpu		= 0,			\
-	.preempt_count	= INIT_PREEMPT_COUNT,	\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
 #define init_stack		(init_thread_union.stack)
 
-/* how to get the thread information struct from C */
-static inline struct thread_info *current_thread_info(void)
-{
-	return (struct thread_info *) S390_lowcore.thread_info;
-}
-
 void arch_release_task_struct(struct task_struct *tsk);
 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
 
-#define THREAD_SIZE_ORDER THREAD_ORDER
-
 #endif
 
 /*
diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h
index 0bb08f341c09..de8298800722 100644
--- a/arch/s390/include/asm/timex.h
+++ b/arch/s390/include/asm/timex.h
@@ -52,11 +52,9 @@ static inline void store_clock_comparator(__u64 *time)
 
 void clock_comparator_work(void);
 
-void __init ptff_init(void);
+void __init time_early_init(void);
 
 extern unsigned char ptff_function_mask[16];
-extern unsigned long lpar_offset;
-extern unsigned long initial_leap_seconds;
 
 /* Function codes for the ptff instruction. */
 #define PTFF_QAF	0x00	/* query available functions */
@@ -100,21 +98,28 @@ struct ptff_qui {
 	unsigned int pad_0x5c[41];
 } __packed;
 
-static inline int ptff(void *ptff_block, size_t len, unsigned int func)
-{
-	typedef struct { char _[len]; } addrtype;
-	register unsigned int reg0 asm("0") = func;
-	register unsigned long reg1 asm("1") = (unsigned long) ptff_block;
-	int rc;
-
-	asm volatile(
-		"	.word	0x0104\n"
-		"	ipm	%0\n"
-		"	srl	%0,28\n"
-		: "=d" (rc), "+m" (*(addrtype *) ptff_block)
-		: "d" (reg0), "d" (reg1) : "cc");
-	return rc;
-}
+/*
+ * ptff - Perform timing facility function
+ * @ptff_block: Pointer to ptff parameter block
+ * @len: Length of parameter block
+ * @func: Function code
+ * Returns: Condition code (0 on success)
+ */
+#define ptff(ptff_block, len, func)					\
+({									\
+	struct addrtype { char _[len]; };				\
+	register unsigned int reg0 asm("0") = func;			\
+	register unsigned long reg1 asm("1") = (unsigned long) (ptff_block);\
+	int rc;								\
+									\
+	asm volatile(							\
+		"	.word	0x0104\n"				\
+		"	ipm	%0\n"					\
+		"	srl	%0,28\n"				\
+		: "=d" (rc), "+m" (*(struct addrtype *) reg1)		\
+		: "d" (reg0), "d" (reg1) : "cc");			\
+	rc;								\
+})
 
 static inline unsigned long long local_tick_disable(void)
 {
diff --git a/arch/s390/include/asm/topology.h b/arch/s390/include/asm/topology.h
index f15f5571ca2b..fa1bfce10370 100644
--- a/arch/s390/include/asm/topology.h
+++ b/arch/s390/include/asm/topology.h
@@ -22,21 +22,22 @@ struct cpu_topology_s390 {
 	cpumask_t drawer_mask;
 };
 
-DECLARE_PER_CPU(struct cpu_topology_s390, cpu_topology);
-
-#define topology_physical_package_id(cpu) (per_cpu(cpu_topology, cpu).socket_id)
-#define topology_thread_id(cpu)		  (per_cpu(cpu_topology, cpu).thread_id)
-#define topology_sibling_cpumask(cpu) \
-		(&per_cpu(cpu_topology, cpu).thread_mask)
-#define topology_core_id(cpu)		  (per_cpu(cpu_topology, cpu).core_id)
-#define topology_core_cpumask(cpu)	  (&per_cpu(cpu_topology, cpu).core_mask)
-#define topology_book_id(cpu)		  (per_cpu(cpu_topology, cpu).book_id)
-#define topology_book_cpumask(cpu)	  (&per_cpu(cpu_topology, cpu).book_mask)
-#define topology_drawer_id(cpu)		  (per_cpu(cpu_topology, cpu).drawer_id)
-#define topology_drawer_cpumask(cpu)	  (&per_cpu(cpu_topology, cpu).drawer_mask)
+extern struct cpu_topology_s390 cpu_topology[NR_CPUS];
+extern cpumask_t cpus_with_topology;
+
+#define topology_physical_package_id(cpu) (cpu_topology[cpu].socket_id)
+#define topology_thread_id(cpu)		  (cpu_topology[cpu].thread_id)
+#define topology_sibling_cpumask(cpu)	  (&cpu_topology[cpu].thread_mask)
+#define topology_core_id(cpu)		  (cpu_topology[cpu].core_id)
+#define topology_core_cpumask(cpu)	  (&cpu_topology[cpu].core_mask)
+#define topology_book_id(cpu)		  (cpu_topology[cpu].book_id)
+#define topology_book_cpumask(cpu)	  (&cpu_topology[cpu].book_mask)
+#define topology_drawer_id(cpu)		  (cpu_topology[cpu].drawer_id)
+#define topology_drawer_cpumask(cpu)	  (&cpu_topology[cpu].drawer_mask)
 
 #define mc_capable() 1
 
+void topology_init_early(void);
 int topology_cpu_init(struct cpu *);
 int topology_set_cpu_management(int fc);
 void topology_schedule_update(void);
@@ -46,6 +47,7 @@ const struct cpumask *cpu_coregroup_mask(int cpu);
 
 #else /* CONFIG_SCHED_TOPOLOGY */
 
+static inline void topology_init_early(void) { }
 static inline void topology_schedule_update(void) { }
 static inline int topology_cpu_init(struct cpu *cpu) { return 0; }
 static inline void topology_expect_change(void) { }
@@ -65,7 +67,7 @@ static inline void topology_expect_change(void) { }
 #define cpu_to_node cpu_to_node
 static inline int cpu_to_node(int cpu)
 {
-	return per_cpu(cpu_topology, cpu).node_id;
+	return cpu_topology[cpu].node_id;
 }
 
 /* Returns a pointer to the cpumask of CPUs on node 'node'. */
diff --git a/arch/s390/include/asm/trace/zcrypt.h b/arch/s390/include/asm/trace/zcrypt.h
new file mode 100644
index 000000000000..adcb77fafa9d
--- /dev/null
+++ b/arch/s390/include/asm/trace/zcrypt.h
@@ -0,0 +1,122 @@
+/*
+ * Tracepoint definitions for the s390 zcrypt device driver
+ *
+ * Copyright IBM Corp. 2016
+ * Author(s): Harald Freudenberger <freude@de.ibm.com>
+ *
+ * Currently there are two tracepoint events defined here.
+ * An s390_zcrypt_req request event occurs as soon as the request is
+ * recognized by the zcrypt ioctl function. This event may act as some kind
+ * of request-processing-starts-now indication.
+ * As late as possible within the zcrypt ioctl function there occurs the
+ * s390_zcrypt_rep event which may act as the point in time where the
+ * request has been processed by the kernel and the result is about to be
+ * transferred back to userspace.
+ * The glue which binds together request and reply event is the ptr
+ * parameter, which is the local buffer address where the request from
+ * userspace has been stored by the ioctl function.
+ *
+ * The main purpose of this zcrypt tracepoint api is to get some data for
+ * performance measurements together with information about on which card
+ * and queue the request has been processed. It is not an ffdc interface as
+ * there is already code in the zcrypt device driver to serve the s390
+ * debug feature interface.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM s390
+
+#if !defined(_TRACE_S390_ZCRYPT_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_S390_ZCRYPT_H
+
+#include <linux/tracepoint.h>
+
+#define TP_ICARSAMODEXPO  0x0001
+#define TP_ICARSACRT	  0x0002
+#define TB_ZSECSENDCPRB   0x0003
+#define TP_ZSENDEP11CPRB  0x0004
+#define TP_HWRNGCPRB	  0x0005
+
+#define show_zcrypt_tp_type(type)				\
+	__print_symbolic(type,					\
+			 { TP_ICARSAMODEXPO, "ICARSAMODEXPO" }, \
+			 { TP_ICARSACRT, "ICARSACRT" },		\
+			 { TB_ZSECSENDCPRB, "ZSECSENDCPRB" },	\
+			 { TP_ZSENDEP11CPRB, "ZSENDEP11CPRB" }, \
+			 { TP_HWRNGCPRB, "HWRNGCPRB" })
+
+/**
+ * trace_s390_zcrypt_req - zcrypt request tracepoint function
+ * @ptr:  Address of the local buffer where the request from userspace
+ *	  is stored. Can be used as a unique id to relate together
+ *	  request and reply.
+ * @type: One of the TP_ defines above.
+ *
+ * Called when a request from userspace is recognised within the ioctl
+ * function of the zcrypt device driver and may act as an entry
+ * timestamp.
+ */
+TRACE_EVENT(s390_zcrypt_req,
+	    TP_PROTO(void *ptr, u32 type),
+	    TP_ARGS(ptr, type),
+	    TP_STRUCT__entry(
+		    __field(void *, ptr)
+		    __field(u32, type)),
+	    TP_fast_assign(
+		    __entry->ptr = ptr;
+		    __entry->type = type;),
+	    TP_printk("ptr=%p type=%s",
+		      __entry->ptr,
+		      show_zcrypt_tp_type(__entry->type))
+);
+
+/**
+ * trace_s390_zcrypt_rep - zcrypt reply tracepoint function
+ * @ptr:  Address of the local buffer where the request from userspace
+ *	  is stored. Can be used as a unique id to match together
+ *	  request and reply.
+ * @fc:   Function code.
+ * @rc:   The bare returncode as returned by the device driver ioctl
+ *	  function.
+ * @dev:  The adapter nr where this request was actually processed.
+ * @dom:  Domain id of the device where this request was processed.
+ *
+ * Called upon recognising the reply from the crypto adapter. This
+ * message may act as the exit timestamp for the request but also
+ * carries some info about on which adapter the request was processed
+ * and the returncode from the device driver.
+ */
+TRACE_EVENT(s390_zcrypt_rep,
+	    TP_PROTO(void *ptr, u32 fc, u32 rc, u16 dev, u16 dom),
+	    TP_ARGS(ptr, fc, rc, dev, dom),
+	    TP_STRUCT__entry(
+		    __field(void *, ptr)
+		    __field(u32, fc)
+		    __field(u32, rc)
+		    __field(u16, device)
+		    __field(u16, domain)),
+	    TP_fast_assign(
+		    __entry->ptr = ptr;
+		    __entry->fc = fc;
+		    __entry->rc = rc;
+		    __entry->device = dev;
+		    __entry->domain = dom;),
+	    TP_printk("ptr=%p fc=0x%04x rc=%d dev=0x%02hx domain=0x%04hx",
+		      __entry->ptr,
+		      (unsigned int) __entry->fc,
+		      (int) __entry->rc,
+		      (unsigned short) __entry->device,
+		      (unsigned short) __entry->domain)
+);
+
+#endif /* _TRACE_S390_ZCRYPT_H */
+
+/* This part must be outside protection */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+
+#define TRACE_INCLUDE_PATH asm/trace
+#define TRACE_INCLUDE_FILE zcrypt
+
+#include <trace/define_trace.h>
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index 52d7c8709279..f82b04e85a21 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -37,14 +37,14 @@
 #define get_ds()        (KERNEL_DS)
 #define get_fs()        (current->thread.mm_segment)
 
-#define set_fs(x) \
-({									\
+#define set_fs(x)							\
+{									\
 	unsigned long __pto;						\
 	current->thread.mm_segment = (x);				\
 	__pto = current->thread.mm_segment.ar4 ?			\
 		S390_lowcore.user_asce : S390_lowcore.kernel_asce;	\
 	__ctl_load(__pto, 7, 7);					\
-})
+}
 
 #define segment_eq(a,b) ((a).ar4 == (b).ar4)
 
diff --git a/arch/s390/include/asm/vdso.h b/arch/s390/include/asm/vdso.h
index d0a2dbf2433d..88bdc477a843 100644
--- a/arch/s390/include/asm/vdso.h
+++ b/arch/s390/include/asm/vdso.h
@@ -33,6 +33,8 @@ struct vdso_data {
 	__u32 ectg_available;		/* ECTG instruction present	0x58 */
 	__u32 tk_mult;			/* Mult. used for xtime_nsec	0x5c */
 	__u32 tk_shift;			/* Shift used for xtime_nsec	0x60 */
+	__u32 ts_dir;			/* TOD steering direction	0x64 */
+	__u64 ts_end;			/* TOD steering end		0x68 */
 };
 
 struct vdso_per_cpu_data {
diff --git a/arch/s390/include/uapi/asm/Kbuild b/arch/s390/include/uapi/asm/Kbuild
index cc44b09c25fc..bf736e764cb4 100644
--- a/arch/s390/include/uapi/asm/Kbuild
+++ b/arch/s390/include/uapi/asm/Kbuild
@@ -12,6 +12,7 @@ header-y += dasd.h
 header-y += debug.h
 header-y += errno.h
 header-y += fcntl.h
+header-y += hypfs.h
 header-y += ioctl.h
 header-y += ioctls.h
 header-y += ipcbuf.h
@@ -29,16 +30,16 @@ header-y += ptrace.h
 header-y += qeth.h
 header-y += resource.h
 header-y += schid.h
+header-y += sclp_ctl.h
 header-y += sembuf.h
 header-y += setup.h
 header-y += shmbuf.h
+header-y += sie.h
 header-y += sigcontext.h
 header-y += siginfo.h
 header-y += signal.h
 header-y += socket.h
 header-y += sockios.h
-header-y += sclp_ctl.h
-header-y += sie.h
 header-y += stat.h
 header-y += statfs.h
 header-y += swab.h
diff --git a/arch/s390/include/uapi/asm/zcrypt.h b/arch/s390/include/uapi/asm/zcrypt.h
index f2b18eacaca8..a777f87ef889 100644
--- a/arch/s390/include/uapi/asm/zcrypt.h
+++ b/arch/s390/include/uapi/asm/zcrypt.h
@@ -215,6 +215,42 @@ struct ep11_urb {
 	uint64_t		resp;
 } __attribute__((packed));
 
+/**
+ * struct zcrypt_device_status
+ * @hwtype:		raw hardware type
+ * @qid:		6 bit device index, 8 bit domain
+ * @functions:		AP device function bit field 'abcdef'
+ *			a, b, c = reserved
+ *			d = CCA coprocessor
+ *			e = Accelerator
+ *			f = EP11 coprocessor
+ * @online		online status
+ * @reserved		reserved
+ */
+struct zcrypt_device_status {
+	unsigned int hwtype:8;
+	unsigned int qid:14;
+	unsigned int online:1;
+	unsigned int functions:6;
+	unsigned int reserved:3;
+};
+
+#define MAX_ZDEV_CARDIDS 64
+#define MAX_ZDEV_DOMAINS 256
+
+/**
+ * Maximum number of zcrypt devices
+ */
+#define MAX_ZDEV_ENTRIES (MAX_ZDEV_CARDIDS * MAX_ZDEV_DOMAINS)
+
+/**
+ * zcrypt_device_matrix
+ * Device matrix of all zcrypt devices
+ */
+struct zcrypt_device_matrix {
+	struct zcrypt_device_status device[MAX_ZDEV_ENTRIES];
+};
+
 #define AUTOSELECT ((unsigned int)0xFFFFFFFF)
 
 #define ZCRYPT_IOCTL_MAGIC 'z'
@@ -321,6 +357,7 @@ struct ep11_urb {
 #define ICARSACRT	_IOC(_IOC_READ|_IOC_WRITE, ZCRYPT_IOCTL_MAGIC, 0x06, 0)
 #define ZSECSENDCPRB	_IOC(_IOC_READ|_IOC_WRITE, ZCRYPT_IOCTL_MAGIC, 0x81, 0)
 #define ZSENDEP11CPRB	_IOC(_IOC_READ|_IOC_WRITE, ZCRYPT_IOCTL_MAGIC, 0x04, 0)
+#define ZDEVICESTATUS	_IOC(_IOC_READ|_IOC_WRITE, ZCRYPT_IOCTL_MAGIC, 0x4f, 0)
 
 /* New status calls */
 #define Z90STAT_TOTALCOUNT	_IOR(ZCRYPT_IOCTL_MAGIC, 0x40, int)
diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile
index 1f0fe98f6db9..36b5101c8606 100644
--- a/arch/s390/kernel/Makefile
+++ b/arch/s390/kernel/Makefile
@@ -2,20 +2,47 @@
 # Makefile for the linux kernel.
 #
 
-KCOV_INSTRUMENT_early.o := n
-KCOV_INSTRUMENT_sclp.o := n
-KCOV_INSTRUMENT_als.o := n
-
 ifdef CONFIG_FUNCTION_TRACER
-# Don't trace early setup code and tracing code
-CFLAGS_REMOVE_early.o = $(CC_FLAGS_FTRACE)
-CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE)
+
+# Do not trace tracer code
+CFLAGS_REMOVE_ftrace.o	= $(CC_FLAGS_FTRACE)
+
+# Do not trace early setup code
+CFLAGS_REMOVE_als.o	= $(CC_FLAGS_FTRACE)
+CFLAGS_REMOVE_early.o	= $(CC_FLAGS_FTRACE)
+CFLAGS_REMOVE_sclp.o	= $(CC_FLAGS_FTRACE)
+
+endif
+
+GCOV_PROFILE_als.o	:= n
+GCOV_PROFILE_early.o	:= n
+GCOV_PROFILE_sclp.o	:= n
+
+KCOV_INSTRUMENT_als.o	:= n
+KCOV_INSTRUMENT_early.o	:= n
+KCOV_INSTRUMENT_sclp.o	:= n
+
+UBSAN_SANITIZE_als.o	:= n
+UBSAN_SANITIZE_early.o	:= n
+UBSAN_SANITIZE_sclp.o	:= n
+
+#
+# Use -march=z900 for sclp.c and als.c to be able to print an error
+# message if the kernel is started on a machine which is too old
+#
+ifneq ($(CC_FLAGS_MARCH),-march=z900)
+CFLAGS_REMOVE_als.o	+= $(CC_FLAGS_MARCH)
+CFLAGS_als.o		+= -march=z900
+CFLAGS_REMOVE_sclp.o	+= $(CC_FLAGS_MARCH)
+CFLAGS_sclp.o		+= -march=z900
+AFLAGS_REMOVE_head.o	+= $(CC_FLAGS_MARCH)
+AFLAGS_head.o		+= -march=z900
 endif
 
 #
 # Passing null pointers is ok for smp code, since we access the lowcore here.
 #
-CFLAGS_smp.o	:= -Wno-nonnull
+CFLAGS_smp.o		:= -Wno-nonnull
 
 #
 # Disable tailcall optimizations for stack / callchain walking functions
@@ -30,27 +57,7 @@ CFLAGS_dumpstack.o	+= -fno-optimize-sibling-calls
 #
 CFLAGS_ptrace.o		+= -DUTS_MACHINE='"$(UTS_MACHINE)"'
 
-CFLAGS_sysinfo.o += -w
-
-#
-# Use -march=z900 for sclp.c and als.c to be able to print an error
-# message if the kernel is started on a machine which is too old
-#
-CFLAGS_REMOVE_sclp.o = $(CC_FLAGS_FTRACE)
-CFLAGS_REMOVE_als.o = $(CC_FLAGS_FTRACE)
-ifneq ($(CC_FLAGS_MARCH),-march=z900)
-CFLAGS_REMOVE_sclp.o	+= $(CC_FLAGS_MARCH)
-CFLAGS_sclp.o		+= -march=z900
-CFLAGS_REMOVE_als.o	+= $(CC_FLAGS_MARCH)
-CFLAGS_als.o		+= -march=z900
-AFLAGS_REMOVE_head.o	+= $(CC_FLAGS_MARCH)
-AFLAGS_head.o		+= -march=z900
-endif
-GCOV_PROFILE_sclp.o := n
-GCOV_PROFILE_als.o := n
-UBSAN_SANITIZE_als.o := n
-UBSAN_SANITIZE_early.o := n
-UBSAN_SANITIZE_sclp.o := n
+CFLAGS_sysinfo.o	+= -w
 
 obj-y	:= traps.o time.o process.o base.o early.o setup.o idle.o vtime.o
 obj-y	+= processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index f3df9e0a5dec..c4b3570ded5b 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -25,12 +25,14 @@
 int main(void)
 {
 	/* task struct offsets */
-	OFFSET(__TASK_thread_info, task_struct, stack);
+	OFFSET(__TASK_stack, task_struct, stack);
 	OFFSET(__TASK_thread, task_struct, thread);
 	OFFSET(__TASK_pid, task_struct, pid);
 	BLANK();
 	/* thread struct offsets */
 	OFFSET(__THREAD_ksp, thread_struct, ksp);
+	OFFSET(__THREAD_sysc_table,  thread_struct, sys_call_table);
+	OFFSET(__THREAD_last_break, thread_struct, last_break);
 	OFFSET(__THREAD_FPU_fpc, thread_struct, fpu.fpc);
 	OFFSET(__THREAD_FPU_regs, thread_struct, fpu.regs);
 	OFFSET(__THREAD_per_cause, thread_struct, per_event.cause);
@@ -39,14 +41,7 @@ int main(void)
 	OFFSET(__THREAD_trap_tdb, thread_struct, trap_tdb);
 	BLANK();
 	/* thread info offsets */
-	OFFSET(__TI_task, thread_info, task);
-	OFFSET(__TI_flags, thread_info, flags);
-	OFFSET(__TI_sysc_table, thread_info, sys_call_table);
-	OFFSET(__TI_cpu, thread_info, cpu);
-	OFFSET(__TI_precount, thread_info, preempt_count);
-	OFFSET(__TI_user_timer, thread_info, user_timer);
-	OFFSET(__TI_system_timer, thread_info, system_timer);
-	OFFSET(__TI_last_break, thread_info, last_break);
+	OFFSET(__TI_flags, task_struct, thread_info.flags);
 	BLANK();
 	/* pt_regs offsets */
 	OFFSET(__PT_ARGS, pt_regs, args);
@@ -79,6 +74,8 @@ int main(void)
 	OFFSET(__VDSO_ECTG_OK, vdso_data, ectg_available);
 	OFFSET(__VDSO_TK_MULT, vdso_data, tk_mult);
 	OFFSET(__VDSO_TK_SHIFT, vdso_data, tk_shift);
+	OFFSET(__VDSO_TS_DIR, vdso_data, ts_dir);
+	OFFSET(__VDSO_TS_END, vdso_data, ts_end);
 	OFFSET(__VDSO_ECTG_BASE, vdso_per_cpu_data, ectg_timer_base);
 	OFFSET(__VDSO_ECTG_USER, vdso_per_cpu_data, ectg_user_time);
 	OFFSET(__VDSO_CPU_NR, vdso_per_cpu_data, cpu_nr);
@@ -159,7 +156,6 @@ int main(void)
 	OFFSET(__LC_INT_CLOCK, lowcore, int_clock);
 	OFFSET(__LC_MCCK_CLOCK, lowcore, mcck_clock);
 	OFFSET(__LC_CURRENT, lowcore, current_task);
-	OFFSET(__LC_THREAD_INFO, lowcore, thread_info);
 	OFFSET(__LC_KERNEL_STACK, lowcore, kernel_stack);
 	OFFSET(__LC_ASYNC_STACK, lowcore, async_stack);
 	OFFSET(__LC_PANIC_STACK, lowcore, panic_stack);
@@ -173,6 +169,7 @@ int main(void)
 	OFFSET(__LC_PERCPU_OFFSET, lowcore, percpu_offset);
 	OFFSET(__LC_VDSO_PER_CPU, lowcore, vdso_per_cpu_data);
 	OFFSET(__LC_MACHINE_FLAGS, lowcore, machine_flags);
+	OFFSET(__LC_PREEMPT_COUNT, lowcore, preempt_count);
 	OFFSET(__LC_GMAP, lowcore, gmap);
 	OFFSET(__LC_PASTE, lowcore, paste);
 	/* software defined ABI-relevant lowcore locations 0xe00 - 0xe20 */
diff --git a/arch/s390/kernel/compat_signal.c b/arch/s390/kernel/compat_signal.c
index 4af60374eba0..6f2a6ab13cb5 100644
--- a/arch/s390/kernel/compat_signal.c
+++ b/arch/s390/kernel/compat_signal.c
@@ -446,7 +446,7 @@ static int setup_frame32(struct ksignal *ksig, sigset_t *set,
 		/* set extra registers only for synchronous signals */
 		regs->gprs[4] = regs->int_code & 127;
 		regs->gprs[5] = regs->int_parm_long;
-		regs->gprs[6] = task_thread_info(current)->last_break;
+		regs->gprs[6] = current->thread.last_break;
 	}
 
 	return 0;
@@ -523,7 +523,7 @@ static int setup_rt_frame32(struct ksignal *ksig, sigset_t *set,
 	regs->gprs[2] = ksig->sig;
 	regs->gprs[3] = (__force __u64) &frame->info;
 	regs->gprs[4] = (__force __u64) &frame->uc;
-	regs->gprs[5] = task_thread_info(current)->last_break;
+	regs->gprs[5] = current->thread.last_break;
 	return 0;
 }
 
diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c
index 2374c5b46bbc..324f1c147a41 100644
--- a/arch/s390/kernel/early.c
+++ b/arch/s390/kernel/early.c
@@ -293,6 +293,7 @@ static noinline __init void setup_lowcore_early(void)
 	psw.addr = (unsigned long) s390_base_pgm_handler;
 	S390_lowcore.program_new_psw = psw;
 	s390_base_pgm_handler_fn = early_pgm_check_handler;
+	S390_lowcore.preempt_count = INIT_PREEMPT_COUNT;
 }
 
 static noinline __init void setup_facility_list(void)
@@ -391,7 +392,49 @@ static int __init cad_init(void)
 }
 early_initcall(cad_init);
 
-static __init void rescue_initrd(void)
+static __init void memmove_early(void *dst, const void *src, size_t n)
+{
+	unsigned long addr;
+	long incr;
+	psw_t old;
+
+	if (!n)
+		return;
+	incr = 1;
+	if (dst > src) {
+		incr = -incr;
+		dst += n - 1;
+		src += n - 1;
+	}
+	old = S390_lowcore.program_new_psw;
+	S390_lowcore.program_new_psw.mask = __extract_psw();
+	asm volatile(
+		"	larl	%[addr],1f\n"
+		"	stg	%[addr],%[psw_pgm_addr]\n"
+		"0:     mvc	0(1,%[dst]),0(%[src])\n"
+		"	agr	%[dst],%[incr]\n"
+		"	agr	%[src],%[incr]\n"
+		"	brctg	%[n],0b\n"
+		"1:\n"
+		: [addr] "=&d" (addr),
+		  [psw_pgm_addr] "=Q" (S390_lowcore.program_new_psw.addr),
+		  [dst] "+&a" (dst), [src] "+&a" (src),  [n] "+d" (n)
+		: [incr] "d" (incr)
+		: "cc", "memory");
+	S390_lowcore.program_new_psw = old;
+}
+
+static __init noinline void ipl_save_parameters(void)
+{
+	void *src, *dst;
+
+	src = (void *)(unsigned long) S390_lowcore.ipl_parmblock_ptr;
+	dst = (void *) IPL_PARMBLOCK_ORIGIN;
+	memmove_early(dst, src, PAGE_SIZE);
+	S390_lowcore.ipl_parmblock_ptr = IPL_PARMBLOCK_ORIGIN;
+}
+
+static __init noinline void rescue_initrd(void)
 {
 #ifdef CONFIG_BLK_DEV_INITRD
 	unsigned long min_initrd_addr = (unsigned long) _end + (4UL << 20);
@@ -405,7 +448,7 @@ static __init void rescue_initrd(void)
 		return;
 	if (INITRD_START >= min_initrd_addr)
 		return;
-	memmove((void *) min_initrd_addr, (void *) INITRD_START, INITRD_SIZE);
+	memmove_early((void *) min_initrd_addr, (void *) INITRD_START, INITRD_SIZE);
 	INITRD_START = min_initrd_addr;
 #endif
 }
@@ -467,7 +510,8 @@ void __init startup_init(void)
 	ipl_save_parameters();
 	rescue_initrd();
 	clear_bss_section();
-	ptff_init();
+	ipl_verify_parameters();
+	time_early_init();
 	init_kernel_storage_key();
 	lockdep_off();
 	setup_lowcore_early();
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 49a30737adde..97298c58b2be 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -42,7 +42,7 @@ __PT_R13     =	__PT_GPRS + 104
 __PT_R14     =	__PT_GPRS + 112
 __PT_R15     =	__PT_GPRS + 120
 
-STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
+STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER
 STACK_SIZE  = 1 << STACK_SHIFT
 STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
 
@@ -123,8 +123,14 @@ _PIF_WORK	= (_PIF_PER_TRAP)
 
 	.macro	LAST_BREAK scratch
 	srag	\scratch,%r10,23
+#ifdef CONFIG_HAVE_MARCH_Z990_FEATURES
 	jz	.+10
-	stg	%r10,__TI_last_break(%r12)
+	stg	%r10,__TASK_thread+__THREAD_last_break(%r12)
+#else
+	jz	.+14
+	lghi	\scratch,__TASK_thread
+	stg	%r10,__THREAD_last_break(\scratch,%r12)
+#endif
 	.endm
 
 	.macro REENABLE_IRQS
@@ -186,14 +192,13 @@ ENTRY(__switch_to)
 	stmg	%r6,%r15,__SF_GPRS(%r15)	# store gprs of prev task
 	lgr	%r1,%r2
 	aghi	%r1,__TASK_thread		# thread_struct of prev task
-	lg	%r5,__TASK_thread_info(%r3)	# get thread_info of next
+	lg	%r5,__TASK_stack(%r3)		# start of kernel stack of next
 	stg	%r15,__THREAD_ksp(%r1)		# store kernel stack of prev
 	lgr	%r1,%r3
 	aghi	%r1,__TASK_thread		# thread_struct of next task
 	lgr	%r15,%r5
 	aghi	%r15,STACK_INIT			# end of kernel stack of next
 	stg	%r3,__LC_CURRENT		# store task struct of next
-	stg	%r5,__LC_THREAD_INFO		# store thread info of next
 	stg	%r15,__LC_KERNEL_STACK		# store end of kernel stack
 	lg	%r15,__THREAD_ksp(%r1)		# load kernel stack of next
 	/* c4 is used in guest detection: arch/s390/kernel/perf_cpum_sf.c */
@@ -274,7 +279,7 @@ ENTRY(system_call)
 .Lsysc_stmg:
 	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
 	lg	%r10,__LC_LAST_BREAK
-	lg	%r12,__LC_THREAD_INFO
+	lg	%r12,__LC_CURRENT
 	lghi	%r14,_PIF_SYSCALL
 .Lsysc_per:
 	lg	%r15,__LC_KERNEL_STACK
@@ -288,7 +293,13 @@ ENTRY(system_call)
 	mvc	__PT_INT_CODE(4,%r11),__LC_SVC_ILC
 	stg	%r14,__PT_FLAGS(%r11)
 .Lsysc_do_svc:
-	lg	%r10,__TI_sysc_table(%r12)	# address of system call table
+	# load address of system call table
+#ifdef CONFIG_HAVE_MARCH_Z990_FEATURES
+	lg	%r10,__TASK_thread+__THREAD_sysc_table(%r12)
+#else
+	lghi	%r13,__TASK_thread
+	lg	%r10,__THREAD_sysc_table(%r13,%r12)
+#endif
 	llgh	%r8,__PT_INT_CODE+2(%r11)
 	slag	%r8,%r8,2			# shift and test for svc 0
 	jnz	.Lsysc_nr_ok
@@ -389,7 +400,6 @@ ENTRY(system_call)
 	TSTMSK	__PT_FLAGS(%r11),_PIF_SYSCALL
 	jno	.Lsysc_return
 	lmg	%r2,%r7,__PT_R2(%r11)	# load svc arguments
-	lg	%r10,__TI_sysc_table(%r12)	# address of system call table
 	lghi	%r8,0			# svc 0 returns -ENOSYS
 	llgh	%r1,__PT_INT_CODE+2(%r11)	# load new svc number
 	cghi	%r1,NR_syscalls
@@ -457,7 +467,7 @@ ENTRY(system_call)
 #
 ENTRY(ret_from_fork)
 	la	%r11,STACK_FRAME_OVERHEAD(%r15)
-	lg	%r12,__LC_THREAD_INFO
+	lg	%r12,__LC_CURRENT
 	brasl	%r14,schedule_tail
 	TRACE_IRQS_ON
 	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
@@ -478,7 +488,7 @@ ENTRY(pgm_check_handler)
 	stpt	__LC_SYNC_ENTER_TIMER
 	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
 	lg	%r10,__LC_LAST_BREAK
-	lg	%r12,__LC_THREAD_INFO
+	lg	%r12,__LC_CURRENT
 	larl	%r13,cleanup_critical
 	lmg	%r8,%r9,__LC_PGM_OLD_PSW
 	tmhh	%r8,0x0001		# test problem state bit
@@ -501,7 +511,7 @@ ENTRY(pgm_check_handler)
 2:	LAST_BREAK %r14
 	UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
 	lg	%r15,__LC_KERNEL_STACK
-	lg	%r14,__TI_task(%r12)
+	lgr	%r14,%r12
 	aghi	%r14,__TASK_thread	# pointer to thread_struct
 	lghi	%r13,__LC_PGM_TDB
 	tm	__LC_PGM_ILC+2,0x02	# check for transaction abort
@@ -567,7 +577,7 @@ ENTRY(io_int_handler)
 	stpt	__LC_ASYNC_ENTER_TIMER
 	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
 	lg	%r10,__LC_LAST_BREAK
-	lg	%r12,__LC_THREAD_INFO
+	lg	%r12,__LC_CURRENT
 	larl	%r13,cleanup_critical
 	lmg	%r8,%r9,__LC_IO_OLD_PSW
 	SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
@@ -626,7 +636,7 @@ ENTRY(io_int_handler)
 	jo	.Lio_work_user		# yes -> do resched & signal
 #ifdef CONFIG_PREEMPT
 	# check for preemptive scheduling
-	icm	%r0,15,__TI_precount(%r12)
+	icm	%r0,15,__LC_PREEMPT_COUNT
 	jnz	.Lio_restore		# preemption is disabled
 	TSTMSK	__TI_flags(%r12),_TIF_NEED_RESCHED
 	jno	.Lio_restore
@@ -741,7 +751,7 @@ ENTRY(ext_int_handler)
 	stpt	__LC_ASYNC_ENTER_TIMER
 	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
 	lg	%r10,__LC_LAST_BREAK
-	lg	%r12,__LC_THREAD_INFO
+	lg	%r12,__LC_CURRENT
 	larl	%r13,cleanup_critical
 	lmg	%r8,%r9,__LC_EXT_OLD_PSW
 	SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
@@ -798,13 +808,10 @@ ENTRY(save_fpu_regs)
 	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
 	bor	%r14
 	stfpc	__THREAD_FPU_fpc(%r2)
-.Lsave_fpu_regs_fpc_end:
 	lg	%r3,__THREAD_FPU_regs(%r2)
 	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_VX
 	jz	.Lsave_fpu_regs_fp	  # no -> store FP regs
-.Lsave_fpu_regs_vx_low:
 	VSTM	%v0,%v15,0,%r3		  # vstm 0,15,0(3)
-.Lsave_fpu_regs_vx_high:
 	VSTM	%v16,%v31,256,%r3	  # vstm 16,31,256(3)
 	j	.Lsave_fpu_regs_done	  # -> set CIF_FPU flag
 .Lsave_fpu_regs_fp:
@@ -851,9 +858,7 @@ load_fpu_regs:
 	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_VX
 	lg	%r4,__THREAD_FPU_regs(%r4)	# %r4 <- reg save area
 	jz	.Lload_fpu_regs_fp		# -> no VX, load FP regs
-.Lload_fpu_regs_vx:
 	VLM	%v0,%v15,0,%r4
-.Lload_fpu_regs_vx_high:
 	VLM	%v16,%v31,256,%r4
 	j	.Lload_fpu_regs_done
 .Lload_fpu_regs_fp:
@@ -889,7 +894,7 @@ ENTRY(mcck_int_handler)
 	spt	__LC_CPU_TIMER_SAVE_AREA-4095(%r1)	# revalidate cpu timer
 	lmg	%r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs
 	lg	%r10,__LC_LAST_BREAK
-	lg	%r12,__LC_THREAD_INFO
+	lg	%r12,__LC_CURRENT
 	larl	%r13,cleanup_critical
 	lmg	%r8,%r9,__LC_MCK_OLD_PSW
 	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
@@ -948,7 +953,7 @@ ENTRY(mcck_int_handler)
 
 .Lmcck_panic:
 	lg	%r15,__LC_PANIC_STACK
-	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
+	la	%r11,STACK_FRAME_OVERHEAD(%r15)
 	j	.Lmcck_skip
 
 #
@@ -1085,7 +1090,7 @@ cleanup_critical:
 	jhe	0f
 	# set up saved registers r10 and r12
 	stg	%r10,16(%r11)		# r10 last break
-	stg	%r12,32(%r11)		# r12 thread-info pointer
+	stg	%r12,32(%r11)		# r12 task struct pointer
 0:	# check if the user time update has been done
 	clg	%r9,BASED(.Lcleanup_system_call_insn+24)
 	jh	0f
@@ -1106,7 +1111,9 @@ cleanup_critical:
 	lg	%r9,16(%r11)
 	srag	%r9,%r9,23
 	jz	0f
-	mvc	__TI_last_break(8,%r12),16(%r11)
+	lgr	%r9,%r12
+	aghi	%r9,__TASK_thread
+	mvc	__THREAD_last_break(8,%r9),16(%r11)
 0:	# set up saved register r11
 	lg	%r15,__LC_KERNEL_STACK
 	la	%r9,STACK_FRAME_OVERHEAD(%r15)
diff --git a/arch/s390/kernel/head.S b/arch/s390/kernel/head.S
index 4431905f8cfa..0b5ebf8a3d30 100644
--- a/arch/s390/kernel/head.S
+++ b/arch/s390/kernel/head.S
@@ -315,7 +315,7 @@ ENTRY(startup_kdump)
 	jg	startup_continue
 
 .Lstack:
-	.long	0x8000 + (1<<(PAGE_SHIFT+THREAD_ORDER))
+	.long	0x8000 + (1<<(PAGE_SHIFT+THREAD_SIZE_ORDER))
 	.align	8
 6:	.long	0x7fffffff,0xffffffff
 
diff --git a/arch/s390/kernel/head64.S b/arch/s390/kernel/head64.S
index 03c2b469c472..482d3526e32b 100644
--- a/arch/s390/kernel/head64.S
+++ b/arch/s390/kernel/head64.S
@@ -32,11 +32,10 @@ ENTRY(startup_continue)
 #
 # Setup stack
 #
-	larl	%r15,init_thread_union
-	stg	%r15,__LC_THREAD_INFO	# cache thread info in lowcore
-	lg	%r14,__TI_task(%r15)	# cache current in lowcore
+	larl	%r14,init_task
 	stg	%r14,__LC_CURRENT
-	aghi	%r15,1<<(PAGE_SHIFT+THREAD_ORDER) # init_task_union + THREAD_SIZE
+	larl	%r15,init_thread_union
+	aghi	%r15,1<<(PAGE_SHIFT+THREAD_SIZE_ORDER) # init_task_union + THREAD_SIZE
 	stg	%r15,__LC_KERNEL_STACK	# set end of kernel stack
 	aghi	%r15,-160
 #
diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c
index 295bfb7124bc..ff3364a067ff 100644
--- a/arch/s390/kernel/ipl.c
+++ b/arch/s390/kernel/ipl.c
@@ -1991,10 +1991,9 @@ void __init ipl_update_parameters(void)
 		diag308_set_works = 1;
 }
 
-void __init ipl_save_parameters(void)
+void __init ipl_verify_parameters(void)
 {
 	struct cio_iplinfo iplinfo;
-	void *src, *dst;
 
 	if (cio_get_iplinfo(&iplinfo))
 		return;
@@ -2005,10 +2004,6 @@ void __init ipl_save_parameters(void)
 	if (!iplinfo.is_qdio)
 		return;
 	ipl_flags |= IPL_PARMBLOCK_VALID;
-	src = (void *)(unsigned long)S390_lowcore.ipl_parmblock_ptr;
-	dst = (void *)IPL_PARMBLOCK_ORIGIN;
-	memmove(dst, src, PAGE_SIZE);
-	S390_lowcore.ipl_parmblock_ptr = IPL_PARMBLOCK_ORIGIN;
 }
 
 static LIST_HEAD(rcall);
diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c
index 285d6561076d..ef60f4177331 100644
--- a/arch/s390/kernel/irq.c
+++ b/arch/s390/kernel/irq.c
@@ -168,7 +168,7 @@ void do_softirq_own_stack(void)
 	old = current_stack_pointer();
 	/* Check against async. stack address range. */
 	new = S390_lowcore.async_stack;
-	if (((new - old) >> (PAGE_SHIFT + THREAD_ORDER)) != 0) {
+	if (((new - old) >> (PAGE_SHIFT + THREAD_SIZE_ORDER)) != 0) {
 		/* Need to switch to the async. stack. */
 		new -= STACK_FRAME_OVERHEAD;
 		((struct stack_frame *) new)->back_chain = old;
diff --git a/arch/s390/kernel/lgr.c b/arch/s390/kernel/lgr.c
index 6ea6d69339b5..ae7dff110054 100644
--- a/arch/s390/kernel/lgr.c
+++ b/arch/s390/kernel/lgr.c
@@ -5,7 +5,8 @@
  * Author(s): Michael Holzheu <holzheu@linux.vnet.ibm.com>
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/export.h>
 #include <linux/timer.h>
 #include <linux/slab.h>
 #include <asm/facility.h>
@@ -183,4 +184,4 @@ static int __init lgr_init(void)
 	lgr_timer_set();
 	return 0;
 }
-module_init(lgr_init);
+device_initcall(lgr_init);
diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c
index 9a32f7419d78..9862196b4b89 100644
--- a/arch/s390/kernel/nmi.c
+++ b/arch/s390/kernel/nmi.c
@@ -102,7 +102,7 @@ static int notrace s390_validate_registers(union mci mci, int umode)
 {
 	int kill_task;
 	u64 zero;
-	void *fpt_save_area, *fpt_creg_save_area;
+	void *fpt_save_area;
 
 	kill_task = 0;
 	zero = 0;
@@ -130,7 +130,6 @@ static int notrace s390_validate_registers(union mci mci, int umode)
 			kill_task = 1;
 	}
 	fpt_save_area = &S390_lowcore.floating_pt_save_area;
-	fpt_creg_save_area = &S390_lowcore.fpt_creg_save_area;
 	if (!mci.fc) {
 		/*
 		 * Floating point control register can't be restored.
@@ -142,11 +141,13 @@ static int notrace s390_validate_registers(union mci mci, int umode)
 		 */
 		if (S390_lowcore.fpu_flags & KERNEL_FPC)
 			s390_handle_damage();
-		asm volatile("lfpc 0(%0)" : : "a" (&zero), "m" (zero));
+		asm volatile("lfpc %0" : : "Q" (zero));
 		if (!test_cpu_flag(CIF_FPU))
 			kill_task = 1;
-	} else
-		asm volatile("lfpc 0(%0)" : : "a" (fpt_creg_save_area));
+	} else {
+		asm volatile("lfpc %0"
+			     : : "Q" (S390_lowcore.fpt_creg_save_area));
+	}
 
 	if (!MACHINE_HAS_VX) {
 		/* Validate floating point registers */
@@ -167,7 +168,7 @@ static int notrace s390_validate_registers(union mci mci, int umode)
 			"	ld	13,104(%0)\n"
 			"	ld	14,112(%0)\n"
 			"	ld	15,120(%0)\n"
-			: : "a" (fpt_save_area));
+			: : "a" (fpt_save_area) : "memory");
 	} else {
 		/* Validate vector registers */
 		union ctlreg0 cr0;
@@ -217,7 +218,7 @@ static int notrace s390_validate_registers(union mci mci, int umode)
 	} else {
 		asm volatile(
 			"	lctlg	0,15,0(%0)"
-			: : "a" (&S390_lowcore.cregs_save_area));
+			: : "a" (&S390_lowcore.cregs_save_area) : "memory");
 	}
 	/*
 	 * We don't even try to validate the TOD register, since we simply
@@ -234,9 +235,9 @@ static int notrace s390_validate_registers(union mci mci, int umode)
 			: : : "0", "cc");
 	else
 		asm volatile(
-			"	l	0,0(%0)\n"
+			"	l	0,%0\n"
 			"	sckpf"
-			: : "a" (&S390_lowcore.tod_progreg_save_area)
+			: : "Q" (S390_lowcore.tod_progreg_save_area)
 			: "0", "cc");
 	/* Validate clock comparator register */
 	set_clock_comparator(S390_lowcore.clock_comparator);
diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c
index fcc634c1479a..763dec18edcd 100644
--- a/arch/s390/kernel/perf_cpum_sf.c
+++ b/arch/s390/kernel/perf_cpum_sf.c
@@ -995,39 +995,36 @@ static int perf_push_sample(struct perf_event *event, struct sf_raw_sample *sfr)
 	regs.int_parm = CPU_MF_INT_SF_PRA;
 	sde_regs = (struct perf_sf_sde_regs *) &regs.int_parm_long;
 
-	regs.psw.addr = sfr->basic.ia;
-	if (sfr->basic.T)
-		regs.psw.mask |= PSW_MASK_DAT;
-	if (sfr->basic.W)
-		regs.psw.mask |= PSW_MASK_WAIT;
-	if (sfr->basic.P)
-		regs.psw.mask |= PSW_MASK_PSTATE;
-	switch (sfr->basic.AS) {
-	case 0x0:
-		regs.psw.mask |= PSW_ASC_PRIMARY;
-		break;
-	case 0x1:
-		regs.psw.mask |= PSW_ASC_ACCREG;
-		break;
-	case 0x2:
-		regs.psw.mask |= PSW_ASC_SECONDARY;
-		break;
-	case 0x3:
-		regs.psw.mask |= PSW_ASC_HOME;
-		break;
-	}
+	psw_bits(regs.psw).ia = sfr->basic.ia;
+	psw_bits(regs.psw).t  = sfr->basic.T;
+	psw_bits(regs.psw).w  = sfr->basic.W;
+	psw_bits(regs.psw).p  = sfr->basic.P;
+	psw_bits(regs.psw).as = sfr->basic.AS;
 
 	/*
-	 * A non-zero guest program parameter indicates a guest
-	 * sample.
-	 * Note that some early samples or samples from guests without
+	 * Use the hardware provided configuration level to decide if the
+	 * sample belongs to a guest or host. If that is not available,
+	 * fall back to the following heuristics:
+	 * A non-zero guest program parameter always indicates a guest
+	 * sample. Some early samples or samples from guests without
 	 * lpp usage would be misaccounted to the host. We use the asn
-	 * value as a heuristic to detect most of these guest samples.
-	 * If the value differs from the host hpp value, we assume
-	 * it to be a KVM guest.
+	 * value as an addon heuristic to detect most of these guest samples.
+	 * If the value differs from the host hpp value, we assume to be a
+	 * KVM guest.
 	 */
-	if (sfr->basic.gpp || sfr->basic.prim_asn != (u16) sfr->basic.hpp)
+	switch (sfr->basic.CL) {
+	case 1: /* logical partition */
+		sde_regs->in_guest = 0;
+		break;
+	case 2: /* virtual machine */
 		sde_regs->in_guest = 1;
+		break;
+	default: /* old machine, use heuristics */
+		if (sfr->basic.gpp ||
+		    sfr->basic.prim_asn != (u16)sfr->basic.hpp)
+			sde_regs->in_guest = 1;
+		break;
+	}
 
 	overflow = 0;
 	if (perf_exclude_event(event, &regs, sde_regs))
diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c
index bba4fa74b321..400d14f0b9f5 100644
--- a/arch/s390/kernel/process.c
+++ b/arch/s390/kernel/process.c
@@ -103,7 +103,6 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
 int copy_thread(unsigned long clone_flags, unsigned long new_stackp,
 		unsigned long arg, struct task_struct *p)
 {
-	struct thread_info *ti;
 	struct fake_frame
 	{
 		struct stack_frame sf;
@@ -121,9 +120,8 @@ int copy_thread(unsigned long clone_flags, unsigned long new_stackp,
 	memset(&p->thread.per_event, 0, sizeof(p->thread.per_event));
 	clear_tsk_thread_flag(p, TIF_SINGLE_STEP);
 	/* Initialize per thread user and system timer values */
-	ti = task_thread_info(p);
-	ti->user_timer = 0;
-	ti->system_timer = 0;
+	p->thread.user_timer = 0;
+	p->thread.system_timer = 0;
 
 	frame->sf.back_chain = 0;
 	/* new return point is ret_from_fork */
diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c
index 9336e824e2db..b81ab8882e2e 100644
--- a/arch/s390/kernel/ptrace.c
+++ b/arch/s390/kernel/ptrace.c
@@ -461,7 +461,7 @@ long arch_ptrace(struct task_struct *child, long request,
 		}
 		return 0;
 	case PTRACE_GET_LAST_BREAK:
-		put_user(task_thread_info(child)->last_break,
+		put_user(child->thread.last_break,
 			 (unsigned long __user *) data);
 		return 0;
 	case PTRACE_ENABLE_TE:
@@ -811,7 +811,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
 		}
 		return 0;
 	case PTRACE_GET_LAST_BREAK:
-		put_user(task_thread_info(child)->last_break,
+		put_user(child->thread.last_break,
 			 (unsigned int __user *) data);
 		return 0;
 	}
@@ -997,10 +997,10 @@ static int s390_last_break_get(struct task_struct *target,
 	if (count > 0) {
 		if (kbuf) {
 			unsigned long *k = kbuf;
-			*k = task_thread_info(target)->last_break;
+			*k = target->thread.last_break;
 		} else {
 			unsigned long  __user *u = ubuf;
-			if (__put_user(task_thread_info(target)->last_break, u))
+			if (__put_user(target->thread.last_break, u))
 				return -EFAULT;
 		}
 	}
@@ -1113,7 +1113,7 @@ static int s390_system_call_get(struct task_struct *target,
 				unsigned int pos, unsigned int count,
 				void *kbuf, void __user *ubuf)
 {
-	unsigned int *data = &task_thread_info(target)->system_call;
+	unsigned int *data = &target->thread.system_call;
 	return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 				   data, 0, sizeof(unsigned int));
 }
@@ -1123,7 +1123,7 @@ static int s390_system_call_set(struct task_struct *target,
 				unsigned int pos, unsigned int count,
 				const void *kbuf, const void __user *ubuf)
 {
-	unsigned int *data = &task_thread_info(target)->system_call;
+	unsigned int *data = &target->thread.system_call;
 	return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 				  data, 0, sizeof(unsigned int));
 }
@@ -1327,7 +1327,7 @@ static int s390_compat_last_break_get(struct task_struct *target,
 	compat_ulong_t last_break;
 
 	if (count > 0) {
-		last_break = task_thread_info(target)->last_break;
+		last_break = target->thread.last_break;
 		if (kbuf) {
 			unsigned long *k = kbuf;
 			*k = last_break;
diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
index 7f7ba5f23f13..865a48871ca4 100644
--- a/arch/s390/kernel/setup.c
+++ b/arch/s390/kernel/setup.c
@@ -35,6 +35,7 @@
 #include <linux/root_dev.h>
 #include <linux/console.h>
 #include <linux/kernel_stat.h>
+#include <linux/dma-contiguous.h>
 #include <linux/device.h>
 #include <linux/notifier.h>
 #include <linux/pfn.h>
@@ -303,7 +304,7 @@ static void __init setup_lowcore(void)
 	 * Setup lowcore for boot cpu
 	 */
 	BUILD_BUG_ON(sizeof(struct lowcore) != LC_PAGES * 4096);
-	lc = __alloc_bootmem_low(LC_PAGES * PAGE_SIZE, LC_PAGES * PAGE_SIZE, 0);
+	lc = memblock_virt_alloc_low(sizeof(*lc), sizeof(*lc));
 	lc->restart_psw.mask = PSW_KERNEL_BITS;
 	lc->restart_psw.addr = (unsigned long) restart_int_handler;
 	lc->external_new_psw.mask = PSW_KERNEL_BITS |
@@ -324,15 +325,15 @@ static void __init setup_lowcore(void)
 	lc->kernel_stack = ((unsigned long) &init_thread_union)
 		+ THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
 	lc->async_stack = (unsigned long)
-		__alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0)
+		memblock_virt_alloc(ASYNC_SIZE, ASYNC_SIZE)
 		+ ASYNC_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
 	lc->panic_stack = (unsigned long)
-		__alloc_bootmem(PAGE_SIZE, PAGE_SIZE, 0)
+		memblock_virt_alloc(PAGE_SIZE, PAGE_SIZE)
 		+ PAGE_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
-	lc->current_task = (unsigned long) init_thread_union.thread_info.task;
-	lc->thread_info = (unsigned long) &init_thread_union;
+	lc->current_task = (unsigned long)&init_task;
 	lc->lpp = LPP_MAGIC;
 	lc->machine_flags = S390_lowcore.machine_flags;
+	lc->preempt_count = S390_lowcore.preempt_count;
 	lc->stfl_fac_list = S390_lowcore.stfl_fac_list;
 	memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list,
 	       MAX_FACILITY_BIT/8);
@@ -349,7 +350,7 @@ static void __init setup_lowcore(void)
 	lc->last_update_timer = S390_lowcore.last_update_timer;
 	lc->last_update_clock = S390_lowcore.last_update_clock;
 
-	restart_stack = __alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0);
+	restart_stack = memblock_virt_alloc(ASYNC_SIZE, ASYNC_SIZE);
 	restart_stack += ASYNC_SIZE;
 
 	/*
@@ -412,7 +413,7 @@ static void __init setup_resources(void)
 	bss_resource.end = (unsigned long) &__bss_stop - 1;
 
 	for_each_memblock(memory, reg) {
-		res = alloc_bootmem_low(sizeof(*res));
+		res = memblock_virt_alloc(sizeof(*res), 8);
 		res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM;
 
 		res->name = "System RAM";
@@ -426,7 +427,7 @@ static void __init setup_resources(void)
 			    std_res->start > res->end)
 				continue;
 			if (std_res->end > res->end) {
-				sub_res = alloc_bootmem_low(sizeof(*sub_res));
+				sub_res = memblock_virt_alloc(sizeof(*sub_res), 8);
 				*sub_res = *std_res;
 				sub_res->end = res->end;
 				std_res->start = res->end + 1;
@@ -445,7 +446,7 @@ static void __init setup_resources(void)
 	 * part of the System RAM resource.
 	 */
 	if (crashk_res.end) {
-		memblock_add(crashk_res.start, resource_size(&crashk_res));
+		memblock_add_node(crashk_res.start, resource_size(&crashk_res), 0);
 		memblock_reserve(crashk_res.start, resource_size(&crashk_res));
 		insert_resource(&iomem_resource, &crashk_res);
 	}
@@ -484,7 +485,7 @@ static void __init setup_memory_end(void)
 	max_pfn = max_low_pfn = PFN_DOWN(memory_end);
 	memblock_remove(memory_end, ULONG_MAX);
 
-	pr_notice("Max memory size: %luMB\n", memory_end >> 20);
+	pr_notice("The maximum memory size is %luMB\n", memory_end >> 20);
 }
 
 static void __init setup_vmcoreinfo(void)
@@ -649,7 +650,7 @@ static void __init check_initrd(void)
 #ifdef CONFIG_BLK_DEV_INITRD
 	if (INITRD_START && INITRD_SIZE &&
 	    !memblock_is_region_memory(INITRD_START, INITRD_SIZE)) {
-		pr_err("initrd does not fit memory.\n");
+		pr_err("The initial RAM disk does not fit into the memory\n");
 		memblock_free(INITRD_START, INITRD_SIZE);
 		initrd_start = initrd_end = 0;
 	}
@@ -903,6 +904,7 @@ void __init setup_arch(char **cmdline_p)
 
 	setup_memory_end();
 	setup_memory();
+	dma_contiguous_reserve(memory_end);
 
 	check_initrd();
 	reserve_crashkernel();
@@ -921,6 +923,8 @@ void __init setup_arch(char **cmdline_p)
 	cpu_detect_mhz_feature();
         cpu_init();
 	numa_setup();
+	smp_detect_cpus();
+	topology_init_early();
 
 	/*
 	 * Create kernel page tables and switch to virtual addressing.
diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c
index d82562cf0a0e..9f241d1efeda 100644
--- a/arch/s390/kernel/signal.c
+++ b/arch/s390/kernel/signal.c
@@ -359,7 +359,7 @@ static int setup_frame(int sig, struct k_sigaction *ka,
 		/* set extra registers only for synchronous signals */
 		regs->gprs[4] = regs->int_code & 127;
 		regs->gprs[5] = regs->int_parm_long;
-		regs->gprs[6] = task_thread_info(current)->last_break;
+		regs->gprs[6] = current->thread.last_break;
 	}
 	return 0;
 }
@@ -430,7 +430,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set,
 	regs->gprs[2] = ksig->sig;
 	regs->gprs[3] = (unsigned long) &frame->info;
 	regs->gprs[4] = (unsigned long) &frame->uc;
-	regs->gprs[5] = task_thread_info(current)->last_break;
+	regs->gprs[5] = current->thread.last_break;
 	return 0;
 }
 
@@ -467,13 +467,13 @@ void do_signal(struct pt_regs *regs)
 	 * the debugger may change all our registers, including the system
 	 * call information.
 	 */
-	current_thread_info()->system_call =
+	current->thread.system_call =
 		test_pt_regs_flag(regs, PIF_SYSCALL) ? regs->int_code : 0;
 
 	if (get_signal(&ksig)) {
 		/* Whee!  Actually deliver the signal.  */
-		if (current_thread_info()->system_call) {
-			regs->int_code = current_thread_info()->system_call;
+		if (current->thread.system_call) {
+			regs->int_code = current->thread.system_call;
 			/* Check for system call restarting. */
 			switch (regs->gprs[2]) {
 			case -ERESTART_RESTARTBLOCK:
@@ -506,8 +506,8 @@ void do_signal(struct pt_regs *regs)
 
 	/* No handlers present - check for system call restart */
 	clear_pt_regs_flag(regs, PIF_SYSCALL);
-	if (current_thread_info()->system_call) {
-		regs->int_code = current_thread_info()->system_call;
+	if (current->thread.system_call) {
+		regs->int_code = current->thread.system_call;
 		switch (regs->gprs[2]) {
 		case -ERESTART_RESTARTBLOCK:
 			/* Restart with sys_restart_syscall */
diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index df4a508ff35c..e49f61aadaf9 100644
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -19,6 +19,7 @@
 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 
 #include <linux/workqueue.h>
+#include <linux/bootmem.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/mm.h>
@@ -259,16 +260,14 @@ static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
 static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk)
 {
 	struct lowcore *lc = pcpu->lowcore;
-	struct thread_info *ti = task_thread_info(tsk);
 
 	lc->kernel_stack = (unsigned long) task_stack_page(tsk)
 		+ THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
-	lc->thread_info = (unsigned long) task_thread_info(tsk);
 	lc->current_task = (unsigned long) tsk;
 	lc->lpp = LPP_MAGIC;
 	lc->current_pid = tsk->pid;
-	lc->user_timer = ti->user_timer;
-	lc->system_timer = ti->system_timer;
+	lc->user_timer = tsk->thread.user_timer;
+	lc->system_timer = tsk->thread.system_timer;
 	lc->steal_timer = 0;
 }
 
@@ -662,14 +661,12 @@ int smp_cpu_get_polarization(int cpu)
 	return pcpu_devices[cpu].polarization;
 }
 
-static struct sclp_core_info *smp_get_core_info(void)
+static void __ref smp_get_core_info(struct sclp_core_info *info, int early)
 {
 	static int use_sigp_detection;
-	struct sclp_core_info *info;
 	int address;
 
-	info = kzalloc(sizeof(*info), GFP_KERNEL);
-	if (info && (use_sigp_detection || sclp_get_core_info(info))) {
+	if (use_sigp_detection || sclp_get_core_info(info, early)) {
 		use_sigp_detection = 1;
 		for (address = 0;
 		     address < (SCLP_MAX_CORES << smp_cpu_mt_shift);
@@ -683,7 +680,6 @@ static struct sclp_core_info *smp_get_core_info(void)
 		}
 		info->combined = info->configured;
 	}
-	return info;
 }
 
 static int smp_add_present_cpu(int cpu);
@@ -724,17 +720,15 @@ static int __smp_rescan_cpus(struct sclp_core_info *info, int sysfs_add)
 	return nr;
 }
 
-static void __init smp_detect_cpus(void)
+void __init smp_detect_cpus(void)
 {
 	unsigned int cpu, mtid, c_cpus, s_cpus;
 	struct sclp_core_info *info;
 	u16 address;
 
 	/* Get CPU information */
-	info = smp_get_core_info();
-	if (!info)
-		panic("smp_detect_cpus failed to allocate memory\n");
-
+	info = memblock_virt_alloc(sizeof(*info), 8);
+	smp_get_core_info(info, 1);
 	/* Find boot CPU type */
 	if (sclp.has_core_type) {
 		address = stap();
@@ -770,7 +764,7 @@ static void __init smp_detect_cpus(void)
 	get_online_cpus();
 	__smp_rescan_cpus(info, 0);
 	put_online_cpus();
-	kfree(info);
+	memblock_free_early((unsigned long)info, sizeof(*info));
 }
 
 /*
@@ -807,7 +801,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
 	pcpu = pcpu_devices + cpu;
 	if (pcpu->state != CPU_STATE_CONFIGURED)
 		return -EIO;
-	base = cpu - (cpu % (smp_cpu_mtid + 1));
+	base = smp_get_base_cpu(cpu);
 	for (i = 0; i <= smp_cpu_mtid; i++) {
 		if (base + i < nr_cpu_ids)
 			if (cpu_online(base + i))
@@ -907,7 +901,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
 	/* request the 0x1202 external call external interrupt */
 	if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt))
 		panic("Couldn't request external interrupt 0x1202");
-	smp_detect_cpus();
 }
 
 void __init smp_prepare_boot_cpu(void)
@@ -973,7 +966,7 @@ static ssize_t cpu_configure_store(struct device *dev,
 	rc = -EBUSY;
 	/* disallow configuration changes of online cpus and cpu 0 */
 	cpu = dev->id;
-	cpu -= cpu % (smp_cpu_mtid + 1);
+	cpu = smp_get_base_cpu(cpu);
 	if (cpu == 0)
 		goto out;
 	for (i = 0; i <= smp_cpu_mtid; i++)
@@ -1106,9 +1099,10 @@ int __ref smp_rescan_cpus(void)
 	struct sclp_core_info *info;
 	int nr;
 
-	info = smp_get_core_info();
+	info = kzalloc(sizeof(*info), GFP_KERNEL);
 	if (!info)
 		return -ENOMEM;
+	smp_get_core_info(info, 0);
 	get_online_cpus();
 	mutex_lock(&smp_cpu_state_mutex);
 	nr = __smp_rescan_cpus(info, 1);
diff --git a/arch/s390/kernel/swsusp.S b/arch/s390/kernel/swsusp.S
index 2d6b6e81f812..1ff21f05d7dd 100644
--- a/arch/s390/kernel/swsusp.S
+++ b/arch/s390/kernel/swsusp.S
@@ -194,7 +194,7 @@ pgm_check_entry:
 
 	/* Suspend CPU not available -> panic */
 	larl	%r15,init_thread_union
-	ahi	%r15,1<<(PAGE_SHIFT+THREAD_ORDER)
+	ahi	%r15,1<<(PAGE_SHIFT+THREAD_SIZE_ORDER)
 	larl	%r2,.Lpanic_string
 	larl	%r3,_sclp_print_early
 	lghi	%r1,0
diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c
index bfda6aa40280..24021c1e3ecb 100644
--- a/arch/s390/kernel/sysinfo.c
+++ b/arch/s390/kernel/sysinfo.c
@@ -56,6 +56,20 @@ int stsi(void *sysinfo, int fc, int sel1, int sel2)
 }
 EXPORT_SYMBOL(stsi);
 
+static bool convert_ext_name(unsigned char encoding, char *name, size_t len)
+{
+	switch (encoding) {
+	case 1: /* EBCDIC */
+		EBCASC(name, len);
+		break;
+	case 2:	/* UTF-8 */
+		break;
+	default:
+		return false;
+	}
+	return true;
+}
+
 static void stsi_1_1_1(struct seq_file *m, struct sysinfo_1_1_1 *info)
 {
 	int i;
@@ -207,24 +221,19 @@ static void stsi_2_2_2(struct seq_file *m, struct sysinfo_2_2_2 *info)
 		seq_printf(m, "LPAR CPUs S-MTID:     %d\n", info->mt_stid);
 		seq_printf(m, "LPAR CPUs PS-MTID:    %d\n", info->mt_psmtid);
 	}
+	if (convert_ext_name(info->vsne, info->ext_name, sizeof(info->ext_name))) {
+		seq_printf(m, "LPAR Extended Name:   %-.256s\n", info->ext_name);
+		seq_printf(m, "LPAR UUID:            %pUb\n", &info->uuid);
+	}
 }
 
 static void print_ext_name(struct seq_file *m, int lvl,
 			   struct sysinfo_3_2_2 *info)
 {
-	if (info->vm[lvl].ext_name_encoding == 0)
-		return;
-	if (info->ext_names[lvl][0] == 0)
-		return;
-	switch (info->vm[lvl].ext_name_encoding) {
-	case 1: /* EBCDIC */
-		EBCASC(info->ext_names[lvl], sizeof(info->ext_names[lvl]));
-		break;
-	case 2:	/* UTF-8 */
-		break;
-	default:
+	size_t len = sizeof(info->ext_names[lvl]);
+
+	if (!convert_ext_name(info->vm[lvl].evmne, info->ext_names[lvl], len))
 		return;
-	}
 	seq_printf(m, "VM%02d Extended Name:   %-.256s\n", lvl,
 		   info->ext_names[lvl]);
 }
diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c
index 0bfcc492987e..867d0a057046 100644
--- a/arch/s390/kernel/time.c
+++ b/arch/s390/kernel/time.c
@@ -59,19 +59,27 @@ ATOMIC_NOTIFIER_HEAD(s390_epoch_delta_notifier);
 EXPORT_SYMBOL(s390_epoch_delta_notifier);
 
 unsigned char ptff_function_mask[16];
-unsigned long lpar_offset;
-unsigned long initial_leap_seconds;
+
+static unsigned long long lpar_offset;
+static unsigned long long initial_leap_seconds;
+static unsigned long long tod_steering_end;
+static long long tod_steering_delta;
 
 /*
  * Get time offsets with PTFF
  */
-void __init ptff_init(void)
+void __init time_early_init(void)
 {
 	struct ptff_qto qto;
 	struct ptff_qui qui;
 
+	/* Initialize TOD steering parameters */
+	tod_steering_end = sched_clock_base_cc;
+	vdso_data->ts_end = tod_steering_end;
+
 	if (!test_facility(28))
 		return;
+
 	ptff(&ptff_function_mask, sizeof(ptff_function_mask), PTFF_QAF);
 
 	/* get LPAR offset */
@@ -80,7 +88,7 @@ void __init ptff_init(void)
 
 	/* get initial leap seconds */
 	if (ptff_query(PTFF_QUI) && ptff(&qui, sizeof(qui), PTFF_QUI) == 0)
-		initial_leap_seconds = (unsigned long)
+		initial_leap_seconds = (unsigned long long)
 			((long) qui.old_leap * 4096000000L);
 }
 
@@ -123,18 +131,6 @@ void clock_comparator_work(void)
 	cd->event_handler(cd);
 }
 
-/*
- * Fixup the clock comparator.
- */
-static void fixup_clock_comparator(unsigned long long delta)
-{
-	/* If nobody is waiting there's nothing to fix. */
-	if (S390_lowcore.clock_comparator == -1ULL)
-		return;
-	S390_lowcore.clock_comparator += delta;
-	set_clock_comparator(S390_lowcore.clock_comparator);
-}
-
 static int s390_next_event(unsigned long delta,
 			   struct clock_event_device *evt)
 {
@@ -215,7 +211,21 @@ void read_boot_clock64(struct timespec64 *ts)
 
 static cycle_t read_tod_clock(struct clocksource *cs)
 {
-	return get_tod_clock();
+	unsigned long long now, adj;
+
+	preempt_disable(); /* protect from changes to steering parameters */
+	now = get_tod_clock();
+	adj = tod_steering_end - now;
+	if (unlikely((s64) adj >= 0))
+		/*
+		 * manually steer by 1 cycle every 2^16 cycles. This
+		 * corresponds to shifting the tod delta by 15. 1s is
+		 * therefore steered in ~9h. The adjust will decrease
+		 * over time, until it finally reaches 0.
+		 */
+		now += (tod_steering_delta < 0) ? (adj >> 15) : -(adj >> 15);
+	preempt_enable();
+	return now;
 }
 
 static struct clocksource clocksource_tod = {
@@ -384,6 +394,55 @@ static inline int check_sync_clock(void)
 	return rc;
 }
 
+/*
+ * Apply clock delta to the global data structures.
+ * This is called once on the CPU that performed the clock sync.
+ */
+static void clock_sync_global(unsigned long long delta)
+{
+	unsigned long now, adj;
+	struct ptff_qto qto;
+
+	/* Fixup the monotonic sched clock. */
+	sched_clock_base_cc += delta;
+	/* Adjust TOD steering parameters. */
+	vdso_data->tb_update_count++;
+	now = get_tod_clock();
+	adj = tod_steering_end - now;
+	if (unlikely((s64) adj >= 0))
+		/* Calculate how much of the old adjustment is left. */
+		tod_steering_delta = (tod_steering_delta < 0) ?
+			-(adj >> 15) : (adj >> 15);
+	tod_steering_delta += delta;
+	if ((abs(tod_steering_delta) >> 48) != 0)
+		panic("TOD clock sync offset %lli is too large to drift\n",
+		      tod_steering_delta);
+	tod_steering_end = now + (abs(tod_steering_delta) << 15);
+	vdso_data->ts_dir = (tod_steering_delta < 0) ? 0 : 1;
+	vdso_data->ts_end = tod_steering_end;
+	vdso_data->tb_update_count++;
+	/* Update LPAR offset. */
+	if (ptff_query(PTFF_QTO) && ptff(&qto, sizeof(qto), PTFF_QTO) == 0)
+		lpar_offset = qto.tod_epoch_difference;
+	/* Call the TOD clock change notifier. */
+	atomic_notifier_call_chain(&s390_epoch_delta_notifier, 0, &delta);
+}
+
+/*
+ * Apply clock delta to the per-CPU data structures of this CPU.
+ * This is called for each online CPU after the call to clock_sync_global.
+ */
+static void clock_sync_local(unsigned long long delta)
+{
+	/* Add the delta to the clock comparator. */
+	if (S390_lowcore.clock_comparator != -1ULL) {
+		S390_lowcore.clock_comparator += delta;
+		set_clock_comparator(S390_lowcore.clock_comparator);
+	}
+	/* Adjust the last_update_clock time-stamp. */
+	S390_lowcore.last_update_clock += delta;
+}
+
 /* Single threaded workqueue used for stp sync events */
 static struct workqueue_struct *time_sync_wq;
 
@@ -397,31 +456,9 @@ static void __init time_init_wq(void)
 struct clock_sync_data {
 	atomic_t cpus;
 	int in_sync;
-	unsigned long long fixup_cc;
+	unsigned long long clock_delta;
 };
 
-static void clock_sync_cpu(struct clock_sync_data *sync)
-{
-	atomic_dec(&sync->cpus);
-	enable_sync_clock();
-	while (sync->in_sync == 0) {
-		__udelay(1);
-		/*
-		 * A different cpu changes *in_sync. Therefore use
-		 * barrier() to force memory access.
-		 */
-		barrier();
-	}
-	if (sync->in_sync != 1)
-		/* Didn't work. Clear per-cpu in sync bit again. */
-		disable_sync_clock(NULL);
-	/*
-	 * This round of TOD syncing is done. Set the clock comparator
-	 * to the next tick and let the processor continue.
-	 */
-	fixup_clock_comparator(sync->fixup_cc);
-}
-
 /*
  * Server Time Protocol (STP) code.
  */
@@ -523,54 +560,46 @@ void stp_queue_work(void)
 
 static int stp_sync_clock(void *data)
 {
-	static int first;
+	struct clock_sync_data *sync = data;
 	unsigned long long clock_delta;
-	struct clock_sync_data *stp_sync;
-	struct ptff_qto qto;
+	static int first;
 	int rc;
 
-	stp_sync = data;
-
-	if (xchg(&first, 1) == 1) {
-		/* Slave */
-		clock_sync_cpu(stp_sync);
-		return 0;
-	}
-
-	/* Wait until all other cpus entered the sync function. */
-	while (atomic_read(&stp_sync->cpus) != 0)
-		cpu_relax();
-
 	enable_sync_clock();
-
-	rc = 0;
-	if (stp_info.todoff[0] || stp_info.todoff[1] ||
-	    stp_info.todoff[2] || stp_info.todoff[3] ||
-	    stp_info.tmd != 2) {
-		rc = chsc_sstpc(stp_page, STP_OP_SYNC, 0, &clock_delta);
-		if (rc == 0) {
-			/* fixup the monotonic sched clock */
-			sched_clock_base_cc += clock_delta;
-			if (ptff_query(PTFF_QTO) &&
-			    ptff(&qto, sizeof(qto), PTFF_QTO) == 0)
-				/* Update LPAR offset */
-				lpar_offset = qto.tod_epoch_difference;
-			atomic_notifier_call_chain(&s390_epoch_delta_notifier,
-						   0, &clock_delta);
-			stp_sync->fixup_cc = clock_delta;
-			fixup_clock_comparator(clock_delta);
-			rc = chsc_sstpi(stp_page, &stp_info,
-					sizeof(struct stp_sstpi));
-			if (rc == 0 && stp_info.tmd != 2)
-				rc = -EAGAIN;
+	if (xchg(&first, 1) == 0) {
+		/* Wait until all other cpus entered the sync function. */
+		while (atomic_read(&sync->cpus) != 0)
+			cpu_relax();
+		rc = 0;
+		if (stp_info.todoff[0] || stp_info.todoff[1] ||
+		    stp_info.todoff[2] || stp_info.todoff[3] ||
+		    stp_info.tmd != 2) {
+			rc = chsc_sstpc(stp_page, STP_OP_SYNC, 0,
+					&clock_delta);
+			if (rc == 0) {
+				sync->clock_delta = clock_delta;
+				clock_sync_global(clock_delta);
+				rc = chsc_sstpi(stp_page, &stp_info,
+						sizeof(struct stp_sstpi));
+				if (rc == 0 && stp_info.tmd != 2)
+					rc = -EAGAIN;
+			}
 		}
+		sync->in_sync = rc ? -EAGAIN : 1;
+		xchg(&first, 0);
+	} else {
+		/* Slave */
+		atomic_dec(&sync->cpus);
+		/* Wait for in_sync to be set. */
+		while (READ_ONCE(sync->in_sync) == 0)
+			__udelay(1);
 	}
-	if (rc) {
+	if (sync->in_sync != 1)
+		/* Didn't work. Clear per-cpu in sync bit again. */
 		disable_sync_clock(NULL);
-		stp_sync->in_sync = -EAGAIN;
-	} else
-		stp_sync->in_sync = 1;
-	xchg(&first, 0);
+	/* Apply clock delta to per-CPU fields of this CPU. */
+	clock_sync_local(sync->clock_delta);
+
 	return 0;
 }
 
diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c
index e959c02e0cac..93dcbae1e98d 100644
--- a/arch/s390/kernel/topology.c
+++ b/arch/s390/kernel/topology.c
@@ -7,6 +7,7 @@
 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 
 #include <linux/workqueue.h>
+#include <linux/bootmem.h>
 #include <linux/cpuset.h>
 #include <linux/device.h>
 #include <linux/export.h>
@@ -41,15 +42,17 @@ static bool topology_enabled = true;
 static DECLARE_WORK(topology_work, topology_work_fn);
 
 /*
- * Socket/Book linked lists and per_cpu(cpu_topology) updates are
+ * Socket/Book linked lists and cpu_topology updates are
  * protected by "sched_domains_mutex".
  */
 static struct mask_info socket_info;
 static struct mask_info book_info;
 static struct mask_info drawer_info;
 
-DEFINE_PER_CPU(struct cpu_topology_s390, cpu_topology);
-EXPORT_PER_CPU_SYMBOL_GPL(cpu_topology);
+struct cpu_topology_s390 cpu_topology[NR_CPUS];
+EXPORT_SYMBOL_GPL(cpu_topology);
+
+cpumask_t cpus_with_topology;
 
 static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu)
 {
@@ -97,7 +100,7 @@ static void add_cpus_to_mask(struct topology_core *tl_core,
 		if (lcpu < 0)
 			continue;
 		for (i = 0; i <= smp_cpu_mtid; i++) {
-			topo = &per_cpu(cpu_topology, lcpu + i);
+			topo = &cpu_topology[lcpu + i];
 			topo->drawer_id = drawer->id;
 			topo->book_id = book->id;
 			topo->socket_id = socket->id;
@@ -106,6 +109,7 @@ static void add_cpus_to_mask(struct topology_core *tl_core,
 			cpumask_set_cpu(lcpu + i, &drawer->mask);
 			cpumask_set_cpu(lcpu + i, &book->mask);
 			cpumask_set_cpu(lcpu + i, &socket->mask);
+			cpumask_set_cpu(lcpu + i, &cpus_with_topology);
 			smp_cpu_set_polarization(lcpu + i, tl_core->pp);
 		}
 	}
@@ -220,7 +224,7 @@ static void update_cpu_masks(void)
 	int cpu;
 
 	for_each_possible_cpu(cpu) {
-		topo = &per_cpu(cpu_topology, cpu);
+		topo = &cpu_topology[cpu];
 		topo->thread_mask = cpu_thread_map(cpu);
 		topo->core_mask = cpu_group_map(&socket_info, cpu);
 		topo->book_mask = cpu_group_map(&book_info, cpu);
@@ -231,6 +235,8 @@ static void update_cpu_masks(void)
 			topo->socket_id = cpu;
 			topo->book_id = cpu;
 			topo->drawer_id = cpu;
+			if (cpu_present(cpu))
+				cpumask_set_cpu(cpu, &cpus_with_topology);
 		}
 	}
 	numa_update_cpu_topology();
@@ -241,12 +247,12 @@ void store_topology(struct sysinfo_15_1_x *info)
 	stsi(info, 15, 1, min(topology_max_mnest, 4));
 }
 
-int arch_update_cpu_topology(void)
+static int __arch_update_cpu_topology(void)
 {
 	struct sysinfo_15_1_x *info = tl_info;
-	struct device *dev;
-	int cpu, rc = 0;
+	int rc = 0;
 
+	cpumask_clear(&cpus_with_topology);
 	if (MACHINE_HAS_TOPOLOGY) {
 		rc = 1;
 		store_topology(info);
@@ -255,6 +261,15 @@ int arch_update_cpu_topology(void)
 	update_cpu_masks();
 	if (!MACHINE_HAS_TOPOLOGY)
 		topology_update_polarization_simple();
+	return rc;
+}
+
+int arch_update_cpu_topology(void)
+{
+	struct device *dev;
+	int cpu, rc;
+
+	rc = __arch_update_cpu_topology();
 	for_each_online_cpu(cpu) {
 		dev = get_cpu_device(cpu);
 		kobject_uevent(&dev->kobj, KOBJ_CHANGE);
@@ -394,23 +409,23 @@ int topology_cpu_init(struct cpu *cpu)
 
 static const struct cpumask *cpu_thread_mask(int cpu)
 {
-	return &per_cpu(cpu_topology, cpu).thread_mask;
+	return &cpu_topology[cpu].thread_mask;
 }
 
 
 const struct cpumask *cpu_coregroup_mask(int cpu)
 {
-	return &per_cpu(cpu_topology, cpu).core_mask;
+	return &cpu_topology[cpu].core_mask;
 }
 
 static const struct cpumask *cpu_book_mask(int cpu)
 {
-	return &per_cpu(cpu_topology, cpu).book_mask;
+	return &cpu_topology[cpu].book_mask;
 }
 
 static const struct cpumask *cpu_drawer_mask(int cpu)
 {
-	return &per_cpu(cpu_topology, cpu).drawer_mask;
+	return &cpu_topology[cpu].drawer_mask;
 }
 
 static int __init early_parse_topology(char *p)
@@ -438,19 +453,20 @@ static void __init alloc_masks(struct sysinfo_15_1_x *info,
 		nr_masks *= info->mag[TOPOLOGY_NR_MAG - offset - 1 - i];
 	nr_masks = max(nr_masks, 1);
 	for (i = 0; i < nr_masks; i++) {
-		mask->next = kzalloc(sizeof(*mask->next), GFP_KERNEL);
+		mask->next = memblock_virt_alloc(sizeof(*mask->next), 8);
 		mask = mask->next;
 	}
 }
 
-static int __init s390_topology_init(void)
+void __init topology_init_early(void)
 {
 	struct sysinfo_15_1_x *info;
 	int i;
 
+	set_sched_topology(s390_topology);
 	if (!MACHINE_HAS_TOPOLOGY)
-		return 0;
-	tl_info = (struct sysinfo_15_1_x *)__get_free_page(GFP_KERNEL);
+		goto out;
+	tl_info = memblock_virt_alloc(sizeof(*tl_info), PAGE_SIZE);
 	info = tl_info;
 	store_topology(info);
 	pr_info("The CPU configuration topology of the machine is:");
@@ -460,10 +476,9 @@ static int __init s390_topology_init(void)
 	alloc_masks(info, &socket_info, 1);
 	alloc_masks(info, &book_info, 2);
 	alloc_masks(info, &drawer_info, 3);
-	set_sched_topology(s390_topology);
-	return 0;
+out:
+	__arch_update_cpu_topology();
 }
-early_initcall(s390_topology_init);
 
 static int __init topology_init(void)
 {
diff --git a/arch/s390/kernel/vdso32/clock_gettime.S b/arch/s390/kernel/vdso32/clock_gettime.S
index 5eec9afbb5b5..a5769b83d90e 100644
--- a/arch/s390/kernel/vdso32/clock_gettime.S
+++ b/arch/s390/kernel/vdso32/clock_gettime.S
@@ -99,8 +99,27 @@ __kernel_clock_gettime:
 	tml	%r4,0x0001			/* pending update ? loop */
 	jnz	11b
 	stcke	0(%r15)				/* Store TOD clock */
-	lm	%r0,%r1,1(%r15)
-	s	%r0,__VDSO_XTIME_STAMP(%r5)	/* TOD - cycle_last */
+	lm	%r0,%r1,__VDSO_TS_END(%r5)	/* TOD steering end time */
+	s	%r0,1(%r15)			/* no - ts_steering_end */
+	sl	%r1,5(%r15)
+	brc	3,22f
+	ahi	%r0,-1
+22:	ltr	%r0,%r0				/* past end of steering? */
+	jm	24f
+	srdl	%r0,15				/* 1 per 2^16 */
+	tm	__VDSO_TS_DIR+3(%r5),0x01	/* steering direction? */
+	jz	23f
+	lcr	%r0,%r0				/* negative TOD offset */
+	lcr	%r1,%r1
+	je	23f
+	ahi	%r0,-1
+23:	a	%r0,1(%r15)			/* add TOD timestamp */
+	al	%r1,5(%r15)
+	brc	12,25f
+	ahi	%r0,1
+	j	25f
+24:	lm	%r0,%r1,1(%r15)			/* load TOD timestamp */
+25:	s	%r0,__VDSO_XTIME_STAMP(%r5)	/* TOD - cycle_last */
 	sl	%r1,__VDSO_XTIME_STAMP+4(%r5)
 	brc	3,12f
 	ahi	%r0,-1
diff --git a/arch/s390/kernel/vdso32/gettimeofday.S b/arch/s390/kernel/vdso32/gettimeofday.S
index 719de6186b20..63b86dceb0bf 100644
--- a/arch/s390/kernel/vdso32/gettimeofday.S
+++ b/arch/s390/kernel/vdso32/gettimeofday.S
@@ -31,8 +31,27 @@ __kernel_gettimeofday:
 	tml	%r4,0x0001			/* pending update ? loop */
 	jnz	1b
 	stcke	0(%r15)				/* Store TOD clock */
-	lm	%r0,%r1,1(%r15)
-	s	%r0,__VDSO_XTIME_STAMP(%r5)	/* TOD - cycle_last */
+	lm	%r0,%r1,__VDSO_TS_END(%r5)	/* TOD steering end time */
+	s	%r0,1(%r15)
+	sl	%r1,5(%r15)
+	brc	3,14f
+	ahi	%r0,-1
+14:	ltr	%r0,%r0				/* past end of steering? */
+	jm	16f
+	srdl	%r0,15				/* 1 per 2^16 */
+	tm	__VDSO_TS_DIR+3(%r5),0x01	/* steering direction? */
+	jz	15f
+	lcr	%r0,%r0				/* negative TOD offset */
+	lcr	%r1,%r1
+	je	15f
+	ahi	%r0,-1
+15:	a	%r0,1(%r15)			/* add TOD timestamp */
+	al	%r1,5(%r15)
+	brc	12,17f
+	ahi	%r0,1
+	j	17f
+16:	lm	%r0,%r1,1(%r15)			/* load TOD timestamp */
+17:	s	%r0,__VDSO_XTIME_STAMP(%r5)	/* TOD - cycle_last */
 	sl	%r1,__VDSO_XTIME_STAMP+4(%r5)
 	brc	3,3f
 	ahi	%r0,-1
diff --git a/arch/s390/kernel/vdso64/clock_gettime.S b/arch/s390/kernel/vdso64/clock_gettime.S
index 61541fb93dc6..9c3b12626dba 100644
--- a/arch/s390/kernel/vdso64/clock_gettime.S
+++ b/arch/s390/kernel/vdso64/clock_gettime.S
@@ -83,8 +83,17 @@ __kernel_clock_gettime:
 	tmll	%r4,0x0001			/* pending update ? loop */
 	jnz	5b
 	stcke	0(%r15)				/* Store TOD clock */
-	lgf	%r2,__VDSO_TK_SHIFT(%r5)	/* Timekeeper shift */
 	lg	%r1,1(%r15)
+	lg	%r0,__VDSO_TS_END(%r5)		/* TOD steering end time */
+	slgr	%r0,%r1				/* now - ts_steering_end */
+	ltgr	%r0,%r0				/* past end of steering ? */
+	jm	17f
+	srlg	%r0,%r0,15			/* 1 per 2^16 */
+	tm	__VDSO_TS_DIR+3(%r5),0x01	/* steering direction? */
+	jz	18f
+	lcgr	%r0,%r0				/* negative TOD offset */
+18:	algr	%r1,%r0				/* add steering offset */
+17:	lgf	%r2,__VDSO_TK_SHIFT(%r5)	/* Timekeeper shift */
 	sg	%r1,__VDSO_XTIME_STAMP(%r5)	/* TOD - cycle_last */
 	msgf	%r1,__VDSO_TK_MULT(%r5)		/*  * tk->mult */
 	alg	%r1,__VDSO_XTIME_NSEC(%r5)	/*  + tk->xtime_nsec */
diff --git a/arch/s390/kernel/vdso64/gettimeofday.S b/arch/s390/kernel/vdso64/gettimeofday.S
index 6ce46707663c..b02e62f3bc12 100644
--- a/arch/s390/kernel/vdso64/gettimeofday.S
+++ b/arch/s390/kernel/vdso64/gettimeofday.S
@@ -31,7 +31,16 @@ __kernel_gettimeofday:
 	jnz	0b
 	stcke	0(%r15)				/* Store TOD clock */
 	lg	%r1,1(%r15)
-	sg	%r1,__VDSO_XTIME_STAMP(%r5)	/* TOD - cycle_last */
+	lg	%r0,__VDSO_TS_END(%r5)		/* TOD steering end time */
+	slgr	%r0,%r1				/* now - ts_steering_end */
+	ltgr	%r0,%r0				/* past end of steering ? */
+	jm	6f
+	srlg	%r0,%r0,15			/* 1 per 2^16 */
+	tm	__VDSO_TS_DIR+3(%r5),0x01	/* steering direction? */
+	jz	7f
+	lcgr	%r0,%r0				/* negative TOD offset */
+7:	algr	%r1,%r0				/* add steering offset */
+6:	sg	%r1,__VDSO_XTIME_STAMP(%r5)	/* TOD - cycle_last */
 	msgf	%r1,__VDSO_TK_MULT(%r5)		/*  * tk->mult */
 	alg	%r1,__VDSO_XTIME_NSEC(%r5)	/*  + tk->xtime_nsec */
 	lg	%r0,__VDSO_XTIME_SEC(%r5)	/* tk->xtime_sec */
diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c
index 1bd5dde2d5a9..6b246aadf311 100644
--- a/arch/s390/kernel/vtime.c
+++ b/arch/s390/kernel/vtime.c
@@ -96,7 +96,6 @@ static void update_mt_scaling(void)
  */
 static int do_account_vtime(struct task_struct *tsk, int hardirq_offset)
 {
-	struct thread_info *ti = task_thread_info(tsk);
 	u64 timer, clock, user, system, steal;
 	u64 user_scaled, system_scaled;
 
@@ -119,13 +118,13 @@ static int do_account_vtime(struct task_struct *tsk, int hardirq_offset)
 	    time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
 		update_mt_scaling();
 
-	user = S390_lowcore.user_timer - ti->user_timer;
+	user = S390_lowcore.user_timer - tsk->thread.user_timer;
 	S390_lowcore.steal_timer -= user;
-	ti->user_timer = S390_lowcore.user_timer;
+	tsk->thread.user_timer = S390_lowcore.user_timer;
 
-	system = S390_lowcore.system_timer - ti->system_timer;
+	system = S390_lowcore.system_timer - tsk->thread.system_timer;
 	S390_lowcore.steal_timer -= system;
-	ti->system_timer = S390_lowcore.system_timer;
+	tsk->thread.system_timer = S390_lowcore.system_timer;
 
 	user_scaled = user;
 	system_scaled = system;
@@ -153,15 +152,11 @@ static int do_account_vtime(struct task_struct *tsk, int hardirq_offset)
 
 void vtime_task_switch(struct task_struct *prev)
 {
-	struct thread_info *ti;
-
 	do_account_vtime(prev, 0);
-	ti = task_thread_info(prev);
-	ti->user_timer = S390_lowcore.user_timer;
-	ti->system_timer = S390_lowcore.system_timer;
-	ti = task_thread_info(current);
-	S390_lowcore.user_timer = ti->user_timer;
-	S390_lowcore.system_timer = ti->system_timer;
+	prev->thread.user_timer = S390_lowcore.user_timer;
+	prev->thread.system_timer = S390_lowcore.system_timer;
+	S390_lowcore.user_timer = current->thread.user_timer;
+	S390_lowcore.system_timer = current->thread.system_timer;
 }
 
 /*
@@ -181,7 +176,6 @@ void vtime_account_user(struct task_struct *tsk)
  */
 void vtime_account_irq_enter(struct task_struct *tsk)
 {
-	struct thread_info *ti = task_thread_info(tsk);
 	u64 timer, system, system_scaled;
 
 	timer = S390_lowcore.last_update_timer;
@@ -193,9 +187,9 @@ void vtime_account_irq_enter(struct task_struct *tsk)
 	    time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
 		update_mt_scaling();
 
-	system = S390_lowcore.system_timer - ti->system_timer;
+	system = S390_lowcore.system_timer - tsk->thread.system_timer;
 	S390_lowcore.steal_timer -= system;
-	ti->system_timer = S390_lowcore.system_timer;
+	tsk->thread.system_timer = S390_lowcore.system_timer;
 	system_scaled = system;
 	/* Do MT utilization scaling */
 	if (smp_cpu_mtid) {
diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index be4db07f70d3..af13f1a135b6 100644
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -415,7 +415,7 @@ static int __write_machine_check(struct kvm_vcpu *vcpu,
 	int rc;
 
 	mci.val = mchk->mcic;
-	/* take care of lazy register loading via vcpu load/put */
+	/* take care of lazy register loading */
 	save_fpu_regs();
 	save_access_regs(vcpu->run->s.regs.acrs);
 
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 9c7a1ecfe6bd..bec71e902be3 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -1812,22 +1812,7 @@ __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
 
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
-	/* Save host register state */
-	save_fpu_regs();
-	vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
-	vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
-
-	if (MACHINE_HAS_VX)
-		current->thread.fpu.regs = vcpu->run->s.regs.vrs;
-	else
-		current->thread.fpu.regs = vcpu->run->s.regs.fprs;
-	current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
-	if (test_fp_ctl(current->thread.fpu.fpc))
-		/* User space provided an invalid FPC, let's clear it */
-		current->thread.fpu.fpc = 0;
 
-	save_access_regs(vcpu->arch.host_acrs);
-	restore_access_regs(vcpu->run->s.regs.acrs);
 	gmap_enable(vcpu->arch.enabled_gmap);
 	atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 	if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
@@ -1844,16 +1829,6 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 	vcpu->arch.enabled_gmap = gmap_get_enabled();
 	gmap_disable(vcpu->arch.enabled_gmap);
 
-	/* Save guest register state */
-	save_fpu_regs();
-	vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
-
-	/* Restore host register state */
-	current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
-	current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
-
-	save_access_regs(vcpu->run->s.regs.acrs);
-	restore_access_regs(vcpu->arch.host_acrs);
 }
 
 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
@@ -2243,7 +2218,6 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 {
 	memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
 	memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
-	restore_access_regs(vcpu->run->s.regs.acrs);
 	return 0;
 }
 
@@ -2257,11 +2231,9 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 
 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
-	/* make sure the new values will be lazily loaded */
-	save_fpu_regs();
 	if (test_fp_ctl(fpu->fpc))
 		return -EINVAL;
-	current->thread.fpu.fpc = fpu->fpc;
+	vcpu->run->s.regs.fpc = fpu->fpc;
 	if (MACHINE_HAS_VX)
 		convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
 				 (freg_t *) fpu->fprs);
@@ -2279,7 +2251,7 @@ int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 				 (__vector128 *) vcpu->run->s.regs.vrs);
 	else
 		memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
-	fpu->fpc = current->thread.fpu.fpc;
+	fpu->fpc = vcpu->run->s.regs.fpc;
 	return 0;
 }
 
@@ -2740,6 +2712,20 @@ static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 		if (riccb->valid)
 			vcpu->arch.sie_block->ecb3 |= 0x01;
 	}
+	save_access_regs(vcpu->arch.host_acrs);
+	restore_access_regs(vcpu->run->s.regs.acrs);
+	/* save host (userspace) fprs/vrs */
+	save_fpu_regs();
+	vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
+	vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
+	if (MACHINE_HAS_VX)
+		current->thread.fpu.regs = vcpu->run->s.regs.vrs;
+	else
+		current->thread.fpu.regs = vcpu->run->s.regs.fprs;
+	current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
+	if (test_fp_ctl(current->thread.fpu.fpc))
+		/* User space provided an invalid FPC, let's clear it */
+		current->thread.fpu.fpc = 0;
 
 	kvm_run->kvm_dirty_regs = 0;
 }
@@ -2758,6 +2744,15 @@ static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 	kvm_run->s.regs.pft = vcpu->arch.pfault_token;
 	kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
 	kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
+	save_access_regs(vcpu->run->s.regs.acrs);
+	restore_access_regs(vcpu->arch.host_acrs);
+	/* Save guest register state */
+	save_fpu_regs();
+	vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
+	/* Restore will be done lazily at return */
+	current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
+	current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
+
 }
 
 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
@@ -2874,7 +2869,7 @@ int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
 {
 	/*
 	 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
-	 * copying in vcpu load/put. Lets update our copies before we save
+	 * switch in the run ioctl. Let's update our copies before we save
 	 * it into the save area
 	 */
 	save_fpu_regs();
diff --git a/arch/s390/lib/mem.S b/arch/s390/lib/mem.S
index be9fa65bfac4..7422a706f310 100644
--- a/arch/s390/lib/mem.S
+++ b/arch/s390/lib/mem.S
@@ -8,6 +8,45 @@
 #include <asm/export.h>
 
 /*
+ * void *memmove(void *dest, const void *src, size_t n)
+ */
+ENTRY(memmove)
+	ltgr	%r4,%r4
+	lgr	%r1,%r2
+	bzr	%r14
+	clgr	%r2,%r3
+	jnh	.Lmemmove_forward
+	la	%r5,0(%r4,%r3)
+	clgr	%r2,%r5
+	jl	.Lmemmove_reverse
+.Lmemmove_forward:
+	aghi	%r4,-1
+	srlg	%r0,%r4,8
+	ltgr	%r0,%r0
+	jz	.Lmemmove_rest
+.Lmemmove_loop:
+	mvc	0(256,%r1),0(%r3)
+	la	%r1,256(%r1)
+	la	%r3,256(%r3)
+	brctg	%r0,.Lmemmove_loop
+.Lmemmove_rest:
+	larl	%r5,.Lmemmove_mvc
+	ex	%r4,0(%r5)
+	br	%r14
+.Lmemmove_reverse:
+	aghi	%r4,-1
+.Lmemmove_reverse_loop:
+	ic	%r0,0(%r4,%r3)
+	stc	%r0,0(%r4,%r1)
+	brctg	%r4,.Lmemmove_reverse_loop
+	ic	%r0,0(%r4,%r3)
+	stc	%r0,0(%r4,%r1)
+	br	%r14
+.Lmemmove_mvc:
+	mvc	0(1,%r1),0(%r3)
+EXPORT_SYMBOL(memmove)
+
+/*
  * memset implementation
  *
  * This code corresponds to the C construct below. We do distinguish
diff --git a/arch/s390/lib/string.c b/arch/s390/lib/string.c
index 48352bffbc92..f71d9f655970 100644
--- a/arch/s390/lib/string.c
+++ b/arch/s390/lib/string.c
@@ -20,7 +20,7 @@ static inline char *__strend(const char *s)
 
 	asm volatile ("0: srst  %0,%1\n"
 		      "   jo    0b"
-		      : "+d" (r0), "+a" (s) :  : "cc" );
+		      : "+d" (r0), "+a" (s) :  : "cc", "memory");
 	return (char *) r0;
 }
 
@@ -31,7 +31,7 @@ static inline char *__strnend(const char *s, size_t n)
 
 	asm volatile ("0: srst  %0,%1\n"
 		      "   jo    0b"
-		      : "+d" (p), "+a" (s) : "d" (r0) : "cc" );
+		      : "+d" (p), "+a" (s) : "d" (r0) : "cc", "memory");
 	return (char *) p;
 }
 
@@ -213,7 +213,7 @@ int strcmp(const char *cs, const char *ct)
 		      "   sr   %0,%1\n"
 		      "1:"
 		      : "+d" (ret), "+d" (r0), "+a" (cs), "+a" (ct)
-		      : : "cc" );
+		      : : "cc", "memory");
 	return ret;
 }
 EXPORT_SYMBOL(strcmp);
@@ -250,7 +250,7 @@ static inline int clcle(const char *s1, unsigned long l1,
 		      "   ipm   %0\n"
 		      "   srl   %0,28"
 		      : "=&d" (cc), "+a" (r2), "+a" (r3),
-			"+a" (r4), "+a" (r5) : : "cc");
+			"+a" (r4), "+a" (r5) : : "cc", "memory");
 	return cc;
 }
 
@@ -298,7 +298,7 @@ void *memchr(const void *s, int c, size_t n)
 		      "   jl	1f\n"
 		      "   la    %0,0\n"
 		      "1:"
-		      : "+a" (ret), "+&a" (s) : "d" (r0) : "cc" );
+		      : "+a" (ret), "+&a" (s) : "d" (r0) : "cc", "memory");
 	return (void *) ret;
 }
 EXPORT_SYMBOL(memchr);
@@ -336,7 +336,7 @@ void *memscan(void *s, int c, size_t n)
 
 	asm volatile ("0: srst  %0,%1\n"
 		      "   jo    0b\n"
-		      : "+a" (ret), "+&a" (s) : "d" (r0) : "cc" );
+		      : "+a" (ret), "+&a" (s) : "d" (r0) : "cc", "memory");
 	return (void *) ret;
 }
 EXPORT_SYMBOL(memscan);
diff --git a/arch/s390/mm/extmem.c b/arch/s390/mm/extmem.c
index 02042b6b66bf..362237203144 100644
--- a/arch/s390/mm/extmem.c
+++ b/arch/s390/mm/extmem.c
@@ -122,7 +122,7 @@ dcss_set_subcodes(void)
 		"1:	la	%2,3\n"
 		"2:\n"
 		EX_TABLE(0b, 1b)
-		: "+d" (rx), "+d" (ry), "=d" (rc) : : "cc");
+		: "+d" (rx), "+d" (ry), "=d" (rc) : : "cc", "memory");
 
 	kfree(name);
 	/* Diag x'64' new subcodes are supported, set to new subcodes */
diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index 661d9fe63c43..d1faae5cdd12 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -733,6 +733,7 @@ block:
 			 * return to userspace schedule() to block. */
 			__set_current_state(TASK_UNINTERRUPTIBLE);
 			set_tsk_need_resched(tsk);
+			set_preempt_need_resched();
 		}
 	}
 out:
diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index 1848292766ef..45becc8a44ec 100644
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -34,7 +34,7 @@ static void __ref *vmem_alloc_pages(unsigned int order)
 
 	if (slab_is_available())
 		return (void *)__get_free_pages(GFP_KERNEL, order);
-	return alloc_bootmem_align(size, size);
+	return (void *) memblock_alloc(size, size);
 }
 
 static inline pud_t *vmem_pud_alloc(void)
@@ -61,17 +61,16 @@ pmd_t *vmem_pmd_alloc(void)
 
 pte_t __ref *vmem_pte_alloc(void)
 {
+	unsigned long size = PTRS_PER_PTE * sizeof(pte_t);
 	pte_t *pte;
 
 	if (slab_is_available())
 		pte = (pte_t *) page_table_alloc(&init_mm);
 	else
-		pte = alloc_bootmem_align(PTRS_PER_PTE * sizeof(pte_t),
-					  PTRS_PER_PTE * sizeof(pte_t));
+		pte = (pte_t *) memblock_alloc(size, size);
 	if (!pte)
 		return NULL;
-	clear_table((unsigned long *) pte, _PAGE_INVALID,
-		    PTRS_PER_PTE * sizeof(pte_t));
+	clear_table((unsigned long *) pte, _PAGE_INVALID, size);
 	return pte;
 }
 
diff --git a/arch/s390/numa/mode_emu.c b/arch/s390/numa/mode_emu.c
index 37e0bb835516..cfd08384f0ab 100644
--- a/arch/s390/numa/mode_emu.c
+++ b/arch/s390/numa/mode_emu.c
@@ -21,6 +21,7 @@
 #include <linux/kernel.h>
 #include <linux/cpumask.h>
 #include <linux/memblock.h>
+#include <linux/bootmem.h>
 #include <linux/node.h>
 #include <linux/memory.h>
 #include <linux/slab.h>
@@ -307,13 +308,11 @@ fail:
 /*
  * Allocate and initialize core to node mapping
  */
-static void create_core_to_node_map(void)
+static void __ref create_core_to_node_map(void)
 {
 	int i;
 
-	emu_cores = kzalloc(sizeof(*emu_cores), GFP_KERNEL);
-	if (emu_cores == NULL)
-		panic("Could not allocate cores to node memory");
+	emu_cores = memblock_virt_alloc(sizeof(*emu_cores), 8);
 	for (i = 0; i < ARRAY_SIZE(emu_cores->to_node_id); i++)
 		emu_cores->to_node_id[i] = NODE_ID_FREE;
 }
@@ -354,13 +353,13 @@ static struct toptree *toptree_from_topology(void)
 
 	phys = toptree_new(TOPTREE_ID_PHYS, 1);
 
-	for_each_online_cpu(cpu) {
-		top = &per_cpu(cpu_topology, cpu);
+	for_each_cpu(cpu, &cpus_with_topology) {
+		top = &cpu_topology[cpu];
 		node = toptree_get_child(phys, 0);
 		drawer = toptree_get_child(node, top->drawer_id);
 		book = toptree_get_child(drawer, top->book_id);
 		mc = toptree_get_child(book, top->socket_id);
-		core = toptree_get_child(mc, top->core_id);
+		core = toptree_get_child(mc, smp_get_base_cpu(cpu));
 		if (!drawer || !book || !mc || !core)
 			panic("NUMA emulation could not allocate memory");
 		cpumask_set_cpu(cpu, &core->mask);
@@ -378,7 +377,7 @@ static void topology_add_core(struct toptree *core)
 	int cpu;
 
 	for_each_cpu(cpu, &core->mask) {
-		top = &per_cpu(cpu_topology, cpu);
+		top = &cpu_topology[cpu];
 		cpumask_copy(&top->thread_mask, &core->mask);
 		cpumask_copy(&top->core_mask, &core_mc(core)->mask);
 		cpumask_copy(&top->book_mask, &core_book(core)->mask);
@@ -425,6 +424,27 @@ static void print_node_to_core_map(void)
 	}
 }
 
+static void pin_all_possible_cpus(void)
+{
+	int core_id, node_id, cpu;
+	static int initialized;
+
+	if (initialized)
+		return;
+	print_node_to_core_map();
+	node_id = 0;
+	for_each_possible_cpu(cpu) {
+		core_id = smp_get_base_cpu(cpu);
+		if (emu_cores->to_node_id[core_id] != NODE_ID_FREE)
+			continue;
+		pin_core_to_node(core_id, node_id);
+		cpu_topology[cpu].node_id = node_id;
+		node_id = (node_id + 1) % emu_nodes;
+	}
+	print_node_to_core_map();
+	initialized = 1;
+}
+
 /*
  * Transfer physical topology into a NUMA topology and modify CPU masks
  * according to the NUMA topology.
@@ -442,7 +462,7 @@ static void emu_update_cpu_topology(void)
 	toptree_free(phys);
 	toptree_to_topology(numa);
 	toptree_free(numa);
-	print_node_to_core_map();
+	pin_all_possible_cpus();
 }
 
 /*
diff --git a/arch/s390/numa/toptree.c b/arch/s390/numa/toptree.c
index 902d350d859a..26f622b1cd11 100644
--- a/arch/s390/numa/toptree.c
+++ b/arch/s390/numa/toptree.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/kernel.h>
+#include <linux/bootmem.h>
 #include <linux/cpumask.h>
 #include <linux/list.h>
 #include <linux/list_sort.h>
@@ -25,10 +26,14 @@
  * RETURNS:
  * Pointer to the new tree node or NULL on error
  */
-struct toptree *toptree_alloc(int level, int id)
+struct toptree __ref *toptree_alloc(int level, int id)
 {
-	struct toptree *res = kzalloc(sizeof(struct toptree), GFP_KERNEL);
+	struct toptree *res;
 
+	if (slab_is_available())
+		res = kzalloc(sizeof(*res), GFP_KERNEL);
+	else
+		res = memblock_virt_alloc(sizeof(*res), 8);
 	if (!res)
 		return res;
 
@@ -65,7 +70,7 @@ static void toptree_remove(struct toptree *cand)
  * cleanly using toptree_remove. Possible children are freed
  * recursively. In the end @cand itself is freed.
  */
-void toptree_free(struct toptree *cand)
+void __ref toptree_free(struct toptree *cand)
 {
 	struct toptree *child, *tmp;
 
@@ -73,7 +78,10 @@ void toptree_free(struct toptree *cand)
 		toptree_remove(cand);
 	toptree_for_each_child_safe(child, tmp, cand)
 		toptree_free(child);
-	kfree(cand);
+	if (slab_is_available())
+		kfree(cand);
+	else
+		memblock_free_early((unsigned long)cand, sizeof(*cand));
 }
 
 /**
diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c
index 15ffc19c8c0c..38e17d4d9884 100644
--- a/arch/s390/pci/pci.c
+++ b/arch/s390/pci/pci.c
@@ -180,7 +180,7 @@ int zpci_fmb_enable_device(struct zpci_dev *zdev)
 {
 	struct mod_pci_args args = { 0, 0, 0, 0 };
 
-	if (zdev->fmb)
+	if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length)
 		return -EINVAL;
 
 	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);
@@ -722,6 +722,11 @@ struct dev_pm_ops pcibios_pm_ops = {
 
 static int zpci_alloc_domain(struct zpci_dev *zdev)
 {
+	if (zpci_unique_uid) {
+		zdev->domain = (u16) zdev->uid;
+		return 0;
+	}
+
 	spin_lock(&zpci_domain_lock);
 	zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
 	if (zdev->domain == ZPCI_NR_DEVICES) {
@@ -735,6 +740,9 @@ static int zpci_alloc_domain(struct zpci_dev *zdev)
 
 static void zpci_free_domain(struct zpci_dev *zdev)
 {
+	if (zpci_unique_uid)
+		return;
+
 	spin_lock(&zpci_domain_lock);
 	clear_bit(zdev->domain, zpci_domain);
 	spin_unlock(&zpci_domain_lock);
diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c
index 1a4512c8544a..1c3332ac1957 100644
--- a/arch/s390/pci/pci_clp.c
+++ b/arch/s390/pci/pci_clp.c
@@ -22,6 +22,8 @@
 #include <asm/clp.h>
 #include <uapi/asm/clp.h>
 
+bool zpci_unique_uid;
+
 static inline void zpci_err_clp(unsigned int rsp, int rc)
 {
 	struct {
@@ -146,6 +148,7 @@ static int clp_store_query_pci_fn(struct zpci_dev *zdev,
 	zdev->pft = response->pft;
 	zdev->vfn = response->vfn;
 	zdev->uid = response->uid;
+	zdev->fmb_length = sizeof(u32) * response->fmb_len;
 
 	memcpy(zdev->pfip, response->pfip, sizeof(zdev->pfip));
 	if (response->util_str_avail) {
@@ -315,6 +318,7 @@ static int clp_list_pci(struct clp_req_rsp_list_pci *rrb,
 			goto out;
 		}
 
+		zpci_unique_uid = rrb->response.uid_checking;
 		WARN_ON_ONCE(rrb->response.entry_size !=
 			sizeof(struct clp_fh_list_entry));
 
diff --git a/arch/s390/pci/pci_debug.c b/arch/s390/pci/pci_debug.c
index 38993b156924..c2f786f0ea06 100644
--- a/arch/s390/pci/pci_debug.c
+++ b/arch/s390/pci/pci_debug.c
@@ -69,7 +69,7 @@ static void pci_sw_counter_show(struct seq_file *m)
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(pci_sw_names); i++, counter++)
-		seq_printf(m, "%26s:\t%llu\n", pci_sw_names[i],
+		seq_printf(m, "%26s:\t%lu\n", pci_sw_names[i],
 			   atomic64_read(counter));
 }
 
diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c
index 6b2f72f523b9..1d7a9c71944a 100644
--- a/arch/s390/pci/pci_dma.c
+++ b/arch/s390/pci/pci_dma.c
@@ -181,14 +181,17 @@ static int __dma_purge_tlb(struct zpci_dev *zdev, dma_addr_t dma_addr,
 	/*
 	 * With zdev->tlb_refresh == 0, rpcit is not required to establish new
 	 * translations when previously invalid translation-table entries are
-	 * validated. With lazy unmap, it also is skipped for previously valid
+	 * validated. With lazy unmap, rpcit is skipped for previously valid
 	 * entries, but a global rpcit is then required before any address can
 	 * be re-used, i.e. after each iommu bitmap wrap-around.
 	 */
-	if (!zdev->tlb_refresh &&
-			(!s390_iommu_strict ||
-			((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID)))
-		return 0;
+	if ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID) {
+		if (!zdev->tlb_refresh)
+			return 0;
+	} else {
+		if (!s390_iommu_strict)
+			return 0;
+	}
 
 	return zpci_refresh_trans((u64) zdev->fh << 32, dma_addr,
 				  PAGE_ALIGN(size));
@@ -257,7 +260,7 @@ static dma_addr_t dma_alloc_address(struct device *dev, int size)
 	spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags);
 	offset = __dma_alloc_iommu(dev, zdev->next_bit, size);
 	if (offset == -1) {
-		if (!zdev->tlb_refresh && !s390_iommu_strict) {
+		if (!s390_iommu_strict) {
 			/* global flush before DMA addresses are reused */
 			if (zpci_refresh_global(zdev))
 				goto out_error;
@@ -292,7 +295,7 @@ static void dma_free_address(struct device *dev, dma_addr_t dma_addr, int size)
 	if (!zdev->iommu_bitmap)
 		goto out;
 
-	if (zdev->tlb_refresh || s390_iommu_strict)
+	if (s390_iommu_strict)
 		bitmap_clear(zdev->iommu_bitmap, offset, size);
 	else
 		bitmap_set(zdev->lazy_bitmap, offset, size);
@@ -388,8 +391,6 @@ static void *s390_dma_alloc(struct device *dev, size_t size,
 		return NULL;
 
 	pa = page_to_phys(page);
-	memset((void *) pa, 0, size);
-
 	map = s390_dma_map_pages(dev, page, 0, size, DMA_BIDIRECTIONAL, 0);
 	if (dma_mapping_error(dev, map)) {
 		free_pages(pa, get_order(size));
@@ -419,6 +420,7 @@ static int __s390_dma_map_sg(struct device *dev, struct scatterlist *sg,
 			     size_t size, dma_addr_t *handle,
 			     enum dma_data_direction dir)
 {
+	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
 	struct zpci_dev *zdev = to_zpci(to_pci_dev(dev));
 	dma_addr_t dma_addr_base, dma_addr;
 	int flags = ZPCI_PTE_VALID;
@@ -426,8 +428,7 @@ static int __s390_dma_map_sg(struct device *dev, struct scatterlist *sg,
 	unsigned long pa = 0;
 	int ret;
 
-	size = PAGE_ALIGN(size);
-	dma_addr_base = dma_alloc_address(dev, size >> PAGE_SHIFT);
+	dma_addr_base = dma_alloc_address(dev, nr_pages);
 	if (dma_addr_base == DMA_ERROR_CODE)
 		return -ENOMEM;
 
@@ -436,26 +437,27 @@ static int __s390_dma_map_sg(struct device *dev, struct scatterlist *sg,
 		flags |= ZPCI_TABLE_PROTECTED;
 
 	for (s = sg; dma_addr < dma_addr_base + size; s = sg_next(s)) {
-		pa = page_to_phys(sg_page(s)) + s->offset;
-		ret = __dma_update_trans(zdev, pa, dma_addr, s->length, flags);
+		pa = page_to_phys(sg_page(s));
+		ret = __dma_update_trans(zdev, pa, dma_addr,
+					 s->offset + s->length, flags);
 		if (ret)
 			goto unmap;
 
-		dma_addr += s->length;
+		dma_addr += s->offset + s->length;
 	}
 	ret = __dma_purge_tlb(zdev, dma_addr_base, size, flags);
 	if (ret)
 		goto unmap;
 
 	*handle = dma_addr_base;
-	atomic64_add(size >> PAGE_SHIFT, &zdev->mapped_pages);
+	atomic64_add(nr_pages, &zdev->mapped_pages);
 
 	return ret;
 
 unmap:
 	dma_update_trans(zdev, 0, dma_addr_base, dma_addr - dma_addr_base,
 			 ZPCI_PTE_INVALID);
-	dma_free_address(dev, dma_addr_base, size >> PAGE_SHIFT);
+	dma_free_address(dev, dma_addr_base, nr_pages);
 	zpci_err("map error:\n");
 	zpci_err_dma(ret, pa);
 	return ret;
@@ -564,7 +566,7 @@ int zpci_dma_init_device(struct zpci_dev *zdev)
 		rc = -ENOMEM;
 		goto free_dma_table;
 	}
-	if (!zdev->tlb_refresh && !s390_iommu_strict) {
+	if (!s390_iommu_strict) {
 		zdev->lazy_bitmap = vzalloc(zdev->iommu_pages / 8);
 		if (!zdev->lazy_bitmap) {
 			rc = -ENOMEM;
diff --git a/arch/s390/tools/Makefile b/arch/s390/tools/Makefile
index 6d9814c9df2b..4b5e1e499527 100644
--- a/arch/s390/tools/Makefile
+++ b/arch/s390/tools/Makefile
@@ -9,7 +9,5 @@ define filechk_facilities.h
 	$(obj)/gen_facilities
 endef
 
-$(obj)/gen_facilities.o: $(srctree)/arch/s390/tools/gen_facilities.c
-
 include/generated/facilities.h: $(obj)/gen_facilities FORCE
 	$(call filechk,facilities.h)
diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
index fe4e6c910dd7..8cc53b1e6d03 100644
--- a/arch/s390/tools/gen_facilities.c
+++ b/arch/s390/tools/gen_facilities.c
@@ -7,13 +7,83 @@
  *
  */
 
-#define S390_GEN_FACILITIES_C
-
 #include <strings.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
-#include <asm/facilities_src.h>
+
+struct facility_def {
+	char *name;
+	int *bits;
+};
+
+static struct facility_def facility_defs[] = {
+	{
+		/*
+		 * FACILITIES_ALS contains the list of facilities that are
+		 * required to run a kernel that is compiled e.g. with
+		 * -march=<machine>.
+		 */
+		.name = "FACILITIES_ALS",
+		.bits = (int[]){
+#ifdef CONFIG_HAVE_MARCH_Z900_FEATURES
+			0,  /* N3 instructions */
+			1,  /* z/Arch mode installed */
+#endif
+#ifdef CONFIG_HAVE_MARCH_Z990_FEATURES
+			18, /* long displacement facility */
+#endif
+#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
+			7,  /* stfle */
+			17, /* message security assist */
+			21, /* extended-immediate facility */
+			25, /* store clock fast */
+#endif
+#ifdef CONFIG_HAVE_MARCH_Z10_FEATURES
+			27, /* mvcos */
+			32, /* compare and swap and store */
+			33, /* compare and swap and store 2 */
+			34, /* general extension facility */
+			35, /* execute extensions */
+#endif
+#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
+			45, /* fast-BCR, etc. */
+#endif
+#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
+			49, /* misc-instruction-extensions */
+			52, /* interlocked facility 2 */
+#endif
+#ifdef CONFIG_HAVE_MARCH_Z13_FEATURES
+			53, /* load-and-zero-rightmost-byte, etc. */
+#endif
+			-1 /* END */
+		}
+	},
+	{
+		.name = "FACILITIES_KVM",
+		.bits = (int[]){
+			0,  /* N3 instructions */
+			1,  /* z/Arch mode installed */
+			2,  /* z/Arch mode active */
+			3,  /* DAT-enhancement */
+			4,  /* idte segment table */
+			5,  /* idte region table */
+			6,  /* ASN-and-LX reuse */
+			7,  /* stfle */
+			8,  /* enhanced-DAT 1 */
+			9,  /* sense-running-status */
+			10, /* conditional sske */
+			13, /* ipte-range */
+			14, /* nonquiescing key-setting */
+			73, /* transactional execution */
+			75, /* access-exception-fetch/store indication */
+			76, /* msa extension 3 */
+			77, /* msa extension 4 */
+			78, /* enhanced-DAT 2 */
+			-1  /* END */
+		}
+	},
+};
 
 static void print_facility_list(struct facility_def *def)
 {
diff --git a/arch/sh/kernel/cpu/Makefile b/arch/sh/kernel/cpu/Makefile
index accc7ca722e1..252e9fee687f 100644
--- a/arch/sh/kernel/cpu/Makefile
+++ b/arch/sh/kernel/cpu/Makefile
@@ -1,5 +1,5 @@
 #
-# Makefile for the Linux/SuperH CPU-specifc backends.
+# Makefile for the Linux/SuperH CPU-specific backends.
 #
 
 obj-$(CONFIG_CPU_SH2)		= sh2/
diff --git a/arch/sh/kernel/cpu/irq/Makefile b/arch/sh/kernel/cpu/irq/Makefile
index f0c7025a67d1..3f8e79402d7d 100644
--- a/arch/sh/kernel/cpu/irq/Makefile
+++ b/arch/sh/kernel/cpu/irq/Makefile
@@ -1,5 +1,5 @@
 #
-# Makefile for the Linux/SuperH CPU-specifc IRQ handlers.
+# Makefile for the Linux/SuperH CPU-specific IRQ handlers.
 #
 obj-$(CONFIG_SUPERH32)			+= imask.o
 obj-$(CONFIG_CPU_SH5)			+= intc-sh5.o
diff --git a/arch/sh/kernel/dma-nommu.c b/arch/sh/kernel/dma-nommu.c
index eadb669a7329..47fee3b6e29c 100644
--- a/arch/sh/kernel/dma-nommu.c
+++ b/arch/sh/kernel/dma-nommu.c
@@ -18,7 +18,9 @@ static dma_addr_t nommu_map_page(struct device *dev, struct page *page,
 	dma_addr_t addr = page_to_phys(page) + offset;
 
 	WARN_ON(size == 0);
-	dma_cache_sync(dev, page_address(page) + offset, size, dir);
+
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		dma_cache_sync(dev, page_address(page) + offset, size, dir);
 
 	return addr;
 }
@@ -35,7 +37,8 @@ static int nommu_map_sg(struct device *dev, struct scatterlist *sg,
 	for_each_sg(sg, s, nents, i) {
 		BUG_ON(!sg_page(s));
 
-		dma_cache_sync(dev, sg_virt(s), s->length, dir);
+		if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+			dma_cache_sync(dev, sg_virt(s), s->length, dir);
 
 		s->dma_address = sg_phys(s);
 		s->dma_length = s->length;
diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c
index 852a3291db96..9df997995f6b 100644
--- a/arch/sparc/kernel/iommu.c
+++ b/arch/sparc/kernel/iommu.c
@@ -415,7 +415,7 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr,
 		ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
 
 	/* Step 1: Kick data out of streaming buffers if necessary. */
-	if (strbuf->strbuf_enabled)
+	if (strbuf->strbuf_enabled && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 		strbuf_flush(strbuf, iommu, bus_addr, ctx,
 			     npages, direction);
 
@@ -640,7 +640,7 @@ static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist,
 		base = iommu->page_table + entry;
 
 		dma_handle &= IO_PAGE_MASK;
-		if (strbuf->strbuf_enabled)
+		if (strbuf->strbuf_enabled && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 			strbuf_flush(strbuf, iommu, dma_handle, ctx,
 				     npages, direction);
 
diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c
index 2344103414d1..6ffaec44931a 100644
--- a/arch/sparc/kernel/ioport.c
+++ b/arch/sparc/kernel/ioport.c
@@ -527,7 +527,7 @@ static dma_addr_t pci32_map_page(struct device *dev, struct page *page,
 static void pci32_unmap_page(struct device *dev, dma_addr_t ba, size_t size,
 			     enum dma_data_direction dir, unsigned long attrs)
 {
-	if (dir != PCI_DMA_TODEVICE)
+	if (dir != PCI_DMA_TODEVICE && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 		dma_make_coherent(ba, PAGE_ALIGN(size));
 }
 
@@ -572,7 +572,7 @@ static void pci32_unmap_sg(struct device *dev, struct scatterlist *sgl,
 	struct scatterlist *sg;
 	int n;
 
-	if (dir != PCI_DMA_TODEVICE) {
+	if (dir != PCI_DMA_TODEVICE && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
 		for_each_sg(sgl, sg, nents, n) {
 			dma_make_coherent(sg_phys(sg), PAGE_ALIGN(sg->length));
 		}
diff --git a/arch/sparc/kernel/nmi.c b/arch/sparc/kernel/nmi.c
index a9973bb4a1b2..95e73c63c99d 100644
--- a/arch/sparc/kernel/nmi.c
+++ b/arch/sparc/kernel/nmi.c
@@ -42,7 +42,7 @@ static int panic_on_timeout;
  */
 atomic_t nmi_active = ATOMIC_INIT(0);		/* oprofile uses this */
 EXPORT_SYMBOL(nmi_active);
-
+static int nmi_init_done;
 static unsigned int nmi_hz = HZ;
 static DEFINE_PER_CPU(short, wd_enabled);
 static int endflag __initdata;
@@ -153,6 +153,8 @@ static void report_broken_nmi(int cpu, int *prev_nmi_count)
 
 void stop_nmi_watchdog(void *unused)
 {
+	if (!__this_cpu_read(wd_enabled))
+		return;
 	pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable);
 	__this_cpu_write(wd_enabled, 0);
 	atomic_dec(&nmi_active);
@@ -207,6 +209,9 @@ error:
 
 void start_nmi_watchdog(void *unused)
 {
+	if (__this_cpu_read(wd_enabled))
+		return;
+
 	__this_cpu_write(wd_enabled, 1);
 	atomic_inc(&nmi_active);
 
@@ -259,6 +264,8 @@ int __init nmi_init(void)
 		}
 	}
 
+	nmi_init_done = 1;
+
 	return err;
 }
 
@@ -270,3 +277,38 @@ static int __init setup_nmi_watchdog(char *str)
 	return 0;
 }
 __setup("nmi_watchdog=", setup_nmi_watchdog);
+
+/*
+ * sparc specific NMI watchdog enable function.
+ * Enables watchdog if it is not enabled already.
+ */
+int watchdog_nmi_enable(unsigned int cpu)
+{
+	if (atomic_read(&nmi_active) == -1) {
+		pr_warn("NMI watchdog cannot be enabled or disabled\n");
+		return -1;
+	}
+
+	/*
+	 * watchdog thread could start even before nmi_init is called.
+	 * Just Return in that case. Let nmi_init finish the init
+	 * process first.
+	 */
+	if (!nmi_init_done)
+		return 0;
+
+	smp_call_function_single(cpu, start_nmi_watchdog, NULL, 1);
+
+	return 0;
+}
+/*
+ * sparc specific NMI watchdog disable function.
+ * Disables watchdog if it is not disabled already.
+ */
+void watchdog_nmi_disable(unsigned int cpu)
+{
+	if (atomic_read(&nmi_active) == -1)
+		pr_warn_once("NMI watchdog cannot be enabled or disabled\n");
+	else
+		smp_call_function_single(cpu, stop_nmi_watchdog, NULL, 1);
+}
diff --git a/arch/tile/kernel/pci-dma.c b/arch/tile/kernel/pci-dma.c
index 09bb774b39cd..24e0f8c21f2f 100644
--- a/arch/tile/kernel/pci-dma.c
+++ b/arch/tile/kernel/pci-dma.c
@@ -213,10 +213,12 @@ static int tile_dma_map_sg(struct device *dev, struct scatterlist *sglist,
 
 	for_each_sg(sglist, sg, nents, i) {
 		sg->dma_address = sg_phys(sg);
-		__dma_prep_pa_range(sg->dma_address, sg->length, direction);
 #ifdef CONFIG_NEED_SG_DMA_LENGTH
 		sg->dma_length = sg->length;
 #endif
+		if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+			continue;
+		__dma_prep_pa_range(sg->dma_address, sg->length, direction);
 	}
 
 	return nents;
@@ -232,6 +234,8 @@ static void tile_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
 	BUG_ON(!valid_dma_direction(direction));
 	for_each_sg(sglist, sg, nents, i) {
 		sg->dma_address = sg_phys(sg);
+		if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+			continue;
 		__dma_complete_pa_range(sg->dma_address, sg->length,
 					direction);
 	}
@@ -245,7 +249,8 @@ static dma_addr_t tile_dma_map_page(struct device *dev, struct page *page,
 	BUG_ON(!valid_dma_direction(direction));
 
 	BUG_ON(offset + size > PAGE_SIZE);
-	__dma_prep_page(page, offset, size, direction);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		__dma_prep_page(page, offset, size, direction);
 
 	return page_to_pa(page) + offset;
 }
@@ -256,6 +261,9 @@ static void tile_dma_unmap_page(struct device *dev, dma_addr_t dma_address,
 {
 	BUG_ON(!valid_dma_direction(direction));
 
+	if (attrs & DMA_ATTR_SKIP_CPU_SYNC)
+		return;
+
 	__dma_complete_page(pfn_to_page(PFN_DOWN(dma_address)),
 			    dma_address & (PAGE_SIZE - 1), size, direction);
 }
diff --git a/arch/um/drivers/ubd.h b/arch/um/drivers/ubd.h
index 3b48cd2081ee..cc1cc85f5afc 100644
--- a/arch/um/drivers/ubd.h
+++ b/arch/um/drivers/ubd.h
@@ -11,5 +11,10 @@ extern int start_io_thread(unsigned long sp, int *fds_out);
 extern int io_thread(void *arg);
 extern int kernel_fd;
 
+extern int ubd_read_poll(int timeout);
+extern int ubd_write_poll(int timeout);
+
+#define UBD_REQ_BUFFER_SIZE 64
+
 #endif
 
diff --git a/arch/um/drivers/ubd_kern.c b/arch/um/drivers/ubd_kern.c
index f3540270d096..85410279beab 100644
--- a/arch/um/drivers/ubd_kern.c
+++ b/arch/um/drivers/ubd_kern.c
@@ -1,4 +1,5 @@
 /*
+ * Copyright (C) 2015-2016 Anton Ivanov (aivanov@brocade.com)
  * Copyright (C) 2000 Jeff Dike (jdike@karaya.com)
  * Licensed under the GPL
  */
@@ -58,6 +59,17 @@ struct io_thread_req {
 	int error;
 };
 
+
+static struct io_thread_req * (*irq_req_buffer)[];
+static struct io_thread_req *irq_remainder;
+static int irq_remainder_size;
+
+static struct io_thread_req * (*io_req_buffer)[];
+static struct io_thread_req *io_remainder;
+static int io_remainder_size;
+
+
+
 static inline int ubd_test_bit(__u64 bit, unsigned char *data)
 {
 	__u64 n;
@@ -442,29 +454,91 @@ static void do_ubd_request(struct request_queue * q);
 static int thread_fd = -1;
 static LIST_HEAD(restart);
 
-/* XXX - move this inside ubd_intr. */
+/* Function to read several request pointers at a time
+* handling fractional reads if (and as) needed
+*/
+
+static int bulk_req_safe_read(
+	int fd,
+	struct io_thread_req * (*request_buffer)[],
+	struct io_thread_req **remainder,
+	int *remainder_size,
+	int max_recs
+	)
+{
+	int n = 0;
+	int res = 0;
+
+	if (*remainder_size > 0) {
+		memmove(
+			(char *) request_buffer,
+			(char *) remainder, *remainder_size
+		);
+		n = *remainder_size;
+	}
+
+	res = os_read_file(
+			fd,
+			((char *) request_buffer) + *remainder_size,
+			sizeof(struct io_thread_req *)*max_recs
+				- *remainder_size
+		);
+	if (res > 0) {
+		n += res;
+		if ((n % sizeof(struct io_thread_req *)) > 0) {
+			/*
+			* Read somehow returned not a multiple of dword
+			* theoretically possible, but never observed in the
+			* wild, so read routine must be able to handle it
+			*/
+			*remainder_size = n % sizeof(struct io_thread_req *);
+			WARN(*remainder_size > 0, "UBD IPC read returned a partial result");
+			memmove(
+				remainder,
+				((char *) request_buffer) +
+					(n/sizeof(struct io_thread_req *))*sizeof(struct io_thread_req *),
+				*remainder_size
+			);
+			n = n - *remainder_size;
+		}
+	} else {
+		n = res;
+	}
+	return n;
+}
+
 /* Called without dev->lock held, and only in interrupt context. */
 static void ubd_handler(void)
 {
-	struct io_thread_req *req;
 	struct ubd *ubd;
 	struct list_head *list, *next_ele;
 	unsigned long flags;
 	int n;
+	int count;
 
 	while(1){
-		n = os_read_file(thread_fd, &req,
-				 sizeof(struct io_thread_req *));
-		if(n != sizeof(req)){
+		n = bulk_req_safe_read(
+			thread_fd,
+			irq_req_buffer,
+			&irq_remainder,
+			&irq_remainder_size,
+			UBD_REQ_BUFFER_SIZE
+		);
+		if (n < 0) {
 			if(n == -EAGAIN)
 				break;
 			printk(KERN_ERR "spurious interrupt in ubd_handler, "
 			       "err = %d\n", -n);
 			return;
 		}
-
-		blk_end_request(req->req, 0, req->length);
-		kfree(req);
+		for (count = 0; count < n/sizeof(struct io_thread_req *); count++) {
+			blk_end_request(
+				(*irq_req_buffer)[count]->req,
+				0,
+				(*irq_req_buffer)[count]->length
+			);
+			kfree((*irq_req_buffer)[count]);
+		}
 	}
 	reactivate_fd(thread_fd, UBD_IRQ);
 
@@ -1064,6 +1138,28 @@ static int __init ubd_init(void)
 		if (register_blkdev(fake_major, "ubd"))
 			return -1;
 	}
+
+	irq_req_buffer = kmalloc(
+			sizeof(struct io_thread_req *) * UBD_REQ_BUFFER_SIZE,
+			GFP_KERNEL
+		);
+	irq_remainder = 0;
+
+	if (irq_req_buffer == NULL) {
+		printk(KERN_ERR "Failed to initialize ubd buffering\n");
+		return -1;
+	}
+	io_req_buffer = kmalloc(
+			sizeof(struct io_thread_req *) * UBD_REQ_BUFFER_SIZE,
+			GFP_KERNEL
+		);
+
+	io_remainder = 0;
+
+	if (io_req_buffer == NULL) {
+		printk(KERN_ERR "Failed to initialize ubd buffering\n");
+		return -1;
+	}
 	platform_driver_register(&ubd_driver);
 	mutex_lock(&ubd_lock);
 	for (i = 0; i < MAX_DEV; i++){
@@ -1458,31 +1554,51 @@ static int io_count = 0;
 
 int io_thread(void *arg)
 {
-	struct io_thread_req *req;
-	int n;
+	int n, count, written, res;
 
 	os_fix_helper_signals();
 
 	while(1){
-		n = os_read_file(kernel_fd, &req,
-				 sizeof(struct io_thread_req *));
-		if(n != sizeof(struct io_thread_req *)){
-			if(n < 0)
+		n = bulk_req_safe_read(
+			kernel_fd,
+			io_req_buffer,
+			&io_remainder,
+			&io_remainder_size,
+			UBD_REQ_BUFFER_SIZE
+		);
+		if (n < 0) {
+			if (n == -EAGAIN) {
+				ubd_read_poll(-1);
+				continue;
+			} else {
 				printk("io_thread - read failed, fd = %d, "
-				       "err = %d\n", kernel_fd, -n);
-			else {
-				printk("io_thread - short read, fd = %d, "
-				       "length = %d\n", kernel_fd, n);
+				       "err = %d,"
+				       "reminder = %d\n",
+				       kernel_fd, -n, io_remainder_size);
 			}
-			continue;
 		}
-		io_count++;
-		do_io(req);
-		n = os_write_file(kernel_fd, &req,
-				  sizeof(struct io_thread_req *));
-		if(n != sizeof(struct io_thread_req *))
-			printk("io_thread - write failed, fd = %d, err = %d\n",
-			       kernel_fd, -n);
+
+		for (count = 0; count < n/sizeof(struct io_thread_req *); count++) {
+			io_count++;
+			do_io((*io_req_buffer)[count]);
+		}
+
+		written = 0;
+
+		do {
+			res = os_write_file(kernel_fd, ((char *) io_req_buffer) + written, n);
+			if (res > 0) {
+				written += res;
+			} else {
+				if (res != -EAGAIN) {
+					printk("io_thread - read failed, fd = %d, "
+					       "err = %d\n", kernel_fd, -n);
+				}
+			}
+			if (written < n) {
+				ubd_write_poll(-1);
+			}
+		} while (written < n);
 	}
 
 	return 0;
diff --git a/arch/um/drivers/ubd_user.c b/arch/um/drivers/ubd_user.c
index e376f9b9c68d..6f744794d141 100644
--- a/arch/um/drivers/ubd_user.c
+++ b/arch/um/drivers/ubd_user.c
@@ -1,4 +1,5 @@
-/* 
+/*
+ * Copyright (C) 2016 Anton Ivanov (aivanov@brocade.com)
  * Copyright (C) 2000, 2001, 2002 Jeff Dike (jdike@karaya.com)
  * Copyright (C) 2001 Ridgerun,Inc (glonnon@ridgerun.com)
  * Licensed under the GPL
@@ -20,6 +21,9 @@
 
 #include "ubd.h"
 #include <os.h>
+#include <poll.h>
+
+struct pollfd kernel_pollfd;
 
 int start_io_thread(unsigned long sp, int *fd_out)
 {
@@ -32,9 +36,12 @@ int start_io_thread(unsigned long sp, int *fd_out)
 	}
 
 	kernel_fd = fds[0];
+	kernel_pollfd.fd = kernel_fd;
+	kernel_pollfd.events = POLLIN;
 	*fd_out = fds[1];
 
 	err = os_set_fd_block(*fd_out, 0);
+	err = os_set_fd_block(kernel_fd, 0);
 	if (err) {
 		printk("start_io_thread - failed to set nonblocking I/O.\n");
 		goto out_close;
@@ -57,3 +64,15 @@ int start_io_thread(unsigned long sp, int *fd_out)
  out:
 	return err;
 }
+
+int ubd_read_poll(int timeout)
+{
+	kernel_pollfd.events = POLLIN;
+	return poll(&kernel_pollfd, 1, timeout);
+}
+int ubd_write_poll(int timeout)
+{
+	kernel_pollfd.events = POLLOUT;
+	return poll(&kernel_pollfd, 1, timeout);
+}
+
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index 34d9e15857c3..44163e8c3868 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -25,7 +25,7 @@ KCOV_INSTRUMENT		:= n
 targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \
 	vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4
 
-KBUILD_CFLAGS := -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2
+KBUILD_CFLAGS := -m$(BITS) -D__KERNEL__ -O2
 KBUILD_CFLAGS += -fno-strict-aliasing $(call cc-option, -fPIE, -fPIC)
 KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
 cflags-$(CONFIG_X86_32) := -march=i386
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index aa8b0672f87a..31c34ee131f3 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -21,7 +21,6 @@
 
 #include <linux/hardirq.h>
 #include <linux/types.h>
-#include <linux/crypto.h>
 #include <linux/module.h>
 #include <linux/err.h>
 #include <crypto/algapi.h>
@@ -29,14 +28,14 @@
 #include <crypto/cryptd.h>
 #include <crypto/ctr.h>
 #include <crypto/b128ops.h>
-#include <crypto/lrw.h>
 #include <crypto/xts.h>
 #include <asm/cpu_device_id.h>
 #include <asm/fpu/api.h>
 #include <asm/crypto/aes.h>
-#include <crypto/ablk_helper.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/internal/aead.h>
+#include <crypto/internal/simd.h>
+#include <crypto/internal/skcipher.h>
 #include <linux/workqueue.h>
 #include <linux/spinlock.h>
 #ifdef CONFIG_X86_64
@@ -45,28 +44,26 @@
 
 
 #define AESNI_ALIGN	16
+#define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
 #define AES_BLOCK_MASK	(~(AES_BLOCK_SIZE - 1))
 #define RFC4106_HASH_SUBKEY_SIZE 16
+#define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
+#define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
+#define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
 
 /* This data is stored at the end of the crypto_tfm struct.
  * It's a type of per "session" data storage location.
  * This needs to be 16 byte aligned.
  */
 struct aesni_rfc4106_gcm_ctx {
-	u8 hash_subkey[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
-	struct crypto_aes_ctx aes_key_expanded
-		__attribute__ ((__aligned__(AESNI_ALIGN)));
+	u8 hash_subkey[16] AESNI_ALIGN_ATTR;
+	struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
 	u8 nonce[4];
 };
 
-struct aesni_lrw_ctx {
-	struct lrw_table_ctx lrw_table;
-	u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
-};
-
 struct aesni_xts_ctx {
-	u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
-	u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
+	u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
+	u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
 };
 
 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
@@ -360,96 +357,95 @@ static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 	aesni_dec(ctx, dst, src);
 }
 
-static int ecb_encrypt(struct blkcipher_desc *desc,
-		       struct scatterlist *dst, struct scatterlist *src,
-		       unsigned int nbytes)
+static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
+			         unsigned int len)
+{
+	return aes_set_key_common(crypto_skcipher_tfm(tfm),
+				  crypto_skcipher_ctx(tfm), key, len);
+}
+
+static int ecb_encrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
+	struct skcipher_walk walk;
+	unsigned int nbytes;
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_fpu_begin();
 	while ((nbytes = walk.nbytes)) {
 		aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 			      nbytes & AES_BLOCK_MASK);
 		nbytes &= AES_BLOCK_SIZE - 1;
-		err = blkcipher_walk_done(desc, &walk, nbytes);
+		err = skcipher_walk_done(&walk, nbytes);
 	}
 	kernel_fpu_end();
 
 	return err;
 }
 
-static int ecb_decrypt(struct blkcipher_desc *desc,
-		       struct scatterlist *dst, struct scatterlist *src,
-		       unsigned int nbytes)
+static int ecb_decrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
+	struct skcipher_walk walk;
+	unsigned int nbytes;
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_fpu_begin();
 	while ((nbytes = walk.nbytes)) {
 		aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 			      nbytes & AES_BLOCK_MASK);
 		nbytes &= AES_BLOCK_SIZE - 1;
-		err = blkcipher_walk_done(desc, &walk, nbytes);
+		err = skcipher_walk_done(&walk, nbytes);
 	}
 	kernel_fpu_end();
 
 	return err;
 }
 
-static int cbc_encrypt(struct blkcipher_desc *desc,
-		       struct scatterlist *dst, struct scatterlist *src,
-		       unsigned int nbytes)
+static int cbc_encrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
+	struct skcipher_walk walk;
+	unsigned int nbytes;
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_fpu_begin();
 	while ((nbytes = walk.nbytes)) {
 		aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 			      nbytes & AES_BLOCK_MASK, walk.iv);
 		nbytes &= AES_BLOCK_SIZE - 1;
-		err = blkcipher_walk_done(desc, &walk, nbytes);
+		err = skcipher_walk_done(&walk, nbytes);
 	}
 	kernel_fpu_end();
 
 	return err;
 }
 
-static int cbc_decrypt(struct blkcipher_desc *desc,
-		       struct scatterlist *dst, struct scatterlist *src,
-		       unsigned int nbytes)
+static int cbc_decrypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
+	struct skcipher_walk walk;
+	unsigned int nbytes;
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_fpu_begin();
 	while ((nbytes = walk.nbytes)) {
 		aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 			      nbytes & AES_BLOCK_MASK, walk.iv);
 		nbytes &= AES_BLOCK_SIZE - 1;
-		err = blkcipher_walk_done(desc, &walk, nbytes);
+		err = skcipher_walk_done(&walk, nbytes);
 	}
 	kernel_fpu_end();
 
@@ -458,7 +454,7 @@ static int cbc_decrypt(struct blkcipher_desc *desc,
 
 #ifdef CONFIG_X86_64
 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
-			    struct blkcipher_walk *walk)
+			    struct skcipher_walk *walk)
 {
 	u8 *ctrblk = walk->iv;
 	u8 keystream[AES_BLOCK_SIZE];
@@ -491,157 +487,53 @@ static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 }
 #endif
 
-static int ctr_crypt(struct blkcipher_desc *desc,
-		     struct scatterlist *dst, struct scatterlist *src,
-		     unsigned int nbytes)
+static int ctr_crypt(struct skcipher_request *req)
 {
-	struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
-	struct blkcipher_walk walk;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
+	struct skcipher_walk walk;
+	unsigned int nbytes;
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+	err = skcipher_walk_virt(&walk, req, true);
 
 	kernel_fpu_begin();
 	while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 		aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 			              nbytes & AES_BLOCK_MASK, walk.iv);
 		nbytes &= AES_BLOCK_SIZE - 1;
-		err = blkcipher_walk_done(desc, &walk, nbytes);
+		err = skcipher_walk_done(&walk, nbytes);
 	}
 	if (walk.nbytes) {
 		ctr_crypt_final(ctx, &walk);
-		err = blkcipher_walk_done(desc, &walk, 0);
+		err = skcipher_walk_done(&walk, 0);
 	}
 	kernel_fpu_end();
 
 	return err;
 }
-#endif
-
-static int ablk_ecb_init(struct crypto_tfm *tfm)
-{
-	return ablk_init_common(tfm, "__driver-ecb-aes-aesni");
-}
-
-static int ablk_cbc_init(struct crypto_tfm *tfm)
-{
-	return ablk_init_common(tfm, "__driver-cbc-aes-aesni");
-}
-
-#ifdef CONFIG_X86_64
-static int ablk_ctr_init(struct crypto_tfm *tfm)
-{
-	return ablk_init_common(tfm, "__driver-ctr-aes-aesni");
-}
-
-#endif
-
-#if IS_ENABLED(CONFIG_CRYPTO_PCBC)
-static int ablk_pcbc_init(struct crypto_tfm *tfm)
-{
-	return ablk_init_common(tfm, "fpu(pcbc(__driver-aes-aesni))");
-}
-#endif
-
-static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
-{
-	aesni_ecb_enc(ctx, blks, blks, nbytes);
-}
 
-static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
-{
-	aesni_ecb_dec(ctx, blks, blks, nbytes);
-}
-
-static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
+static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
 			    unsigned int keylen)
 {
-	struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int err;
 
-	err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key,
-				 keylen - AES_BLOCK_SIZE);
+	err = xts_verify_key(tfm, key, keylen);
 	if (err)
 		return err;
 
-	return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE);
-}
-
-static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm)
-{
-	struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	lrw_free_table(&ctx->lrw_table);
-}
-
-static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
-{
-	struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	be128 buf[8];
-	struct lrw_crypt_req req = {
-		.tbuf = buf,
-		.tbuflen = sizeof(buf),
-
-		.table_ctx = &ctx->lrw_table,
-		.crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
-		.crypt_fn = lrw_xts_encrypt_callback,
-	};
-	int ret;
-
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-
-	kernel_fpu_begin();
-	ret = lrw_crypt(desc, dst, src, nbytes, &req);
-	kernel_fpu_end();
-
-	return ret;
-}
-
-static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
-{
-	struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	be128 buf[8];
-	struct lrw_crypt_req req = {
-		.tbuf = buf,
-		.tbuflen = sizeof(buf),
-
-		.table_ctx = &ctx->lrw_table,
-		.crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
-		.crypt_fn = lrw_xts_decrypt_callback,
-	};
-	int ret;
-
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-
-	kernel_fpu_begin();
-	ret = lrw_crypt(desc, dst, src, nbytes, &req);
-	kernel_fpu_end();
-
-	return ret;
-}
-
-static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
-			    unsigned int keylen)
-{
-	struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm);
-	int err;
-
-	err = xts_check_key(tfm, key, keylen);
-	if (err)
-		return err;
+	keylen /= 2;
 
 	/* first half of xts-key is for crypt */
-	err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2);
+	err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
+				 key, keylen);
 	if (err)
 		return err;
 
 	/* second half of xts-key is for tweak */
-	return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2,
-				  keylen / 2);
+	return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
+				  key + keylen, keylen);
 }
 
 
@@ -650,8 +542,6 @@ static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
 	aesni_enc(ctx, out, in);
 }
 
-#ifdef CONFIG_X86_64
-
 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 {
 	glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
@@ -698,83 +588,28 @@ static const struct common_glue_ctx aesni_dec_xts = {
 	} }
 };
 
-static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int xts_encrypt(struct skcipher_request *req)
 {
-	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 
-	return glue_xts_crypt_128bit(&aesni_enc_xts, desc, dst, src, nbytes,
-				     XTS_TWEAK_CAST(aesni_xts_tweak),
-				     aes_ctx(ctx->raw_tweak_ctx),
-				     aes_ctx(ctx->raw_crypt_ctx));
+	return glue_xts_req_128bit(&aesni_enc_xts, req,
+				   XTS_TWEAK_CAST(aesni_xts_tweak),
+				   aes_ctx(ctx->raw_tweak_ctx),
+				   aes_ctx(ctx->raw_crypt_ctx));
 }
 
-static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
+static int xts_decrypt(struct skcipher_request *req)
 {
-	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-
-	return glue_xts_crypt_128bit(&aesni_dec_xts, desc, dst, src, nbytes,
-				     XTS_TWEAK_CAST(aesni_xts_tweak),
-				     aes_ctx(ctx->raw_tweak_ctx),
-				     aes_ctx(ctx->raw_crypt_ctx));
-}
-
-#else
-
-static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
-{
-	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	be128 buf[8];
-	struct xts_crypt_req req = {
-		.tbuf = buf,
-		.tbuflen = sizeof(buf),
-
-		.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
-		.tweak_fn = aesni_xts_tweak,
-		.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
-		.crypt_fn = lrw_xts_encrypt_callback,
-	};
-	int ret;
-
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-
-	kernel_fpu_begin();
-	ret = xts_crypt(desc, dst, src, nbytes, &req);
-	kernel_fpu_end();
-
-	return ret;
-}
-
-static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		       struct scatterlist *src, unsigned int nbytes)
-{
-	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
-	be128 buf[8];
-	struct xts_crypt_req req = {
-		.tbuf = buf,
-		.tbuflen = sizeof(buf),
-
-		.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
-		.tweak_fn = aesni_xts_tweak,
-		.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
-		.crypt_fn = lrw_xts_decrypt_callback,
-	};
-	int ret;
-
-	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-
-	kernel_fpu_begin();
-	ret = xts_crypt(desc, dst, src, nbytes, &req);
-	kernel_fpu_end();
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 
-	return ret;
+	return glue_xts_req_128bit(&aesni_dec_xts, req,
+				   XTS_TWEAK_CAST(aesni_xts_tweak),
+				   aes_ctx(ctx->raw_tweak_ctx),
+				   aes_ctx(ctx->raw_crypt_ctx));
 }
 
-#endif
-
-#ifdef CONFIG_X86_64
 static int rfc4106_init(struct crypto_aead *aead)
 {
 	struct cryptd_aead *cryptd_tfm;
@@ -1077,9 +912,7 @@ static struct crypto_alg aesni_algs[] = { {
 	.cra_priority		= 300,
 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct crypto_aes_ctx) +
-				  AESNI_ALIGN - 1,
-	.cra_alignmask		= 0,
+	.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
 	.cra_module		= THIS_MODULE,
 	.cra_u	= {
 		.cipher	= {
@@ -1091,14 +924,12 @@ static struct crypto_alg aesni_algs[] = { {
 		}
 	}
 }, {
-	.cra_name		= "__aes-aesni",
-	.cra_driver_name	= "__driver-aes-aesni",
-	.cra_priority		= 0,
+	.cra_name		= "__aes",
+	.cra_driver_name	= "__aes-aesni",
+	.cra_priority		= 300,
 	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
 	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct crypto_aes_ctx) +
-				  AESNI_ALIGN - 1,
-	.cra_alignmask		= 0,
+	.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
 	.cra_module		= THIS_MODULE,
 	.cra_u	= {
 		.cipher	= {
@@ -1109,250 +940,94 @@ static struct crypto_alg aesni_algs[] = { {
 			.cia_decrypt		= __aes_decrypt
 		}
 	}
-}, {
-	.cra_name		= "__ecb-aes-aesni",
-	.cra_driver_name	= "__driver-ecb-aes-aesni",
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct crypto_aes_ctx) +
-				  AESNI_ALIGN - 1,
-	.cra_alignmask		= 0,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_u = {
-		.blkcipher = {
-			.min_keysize	= AES_MIN_KEY_SIZE,
-			.max_keysize	= AES_MAX_KEY_SIZE,
-			.setkey		= aes_set_key,
-			.encrypt	= ecb_encrypt,
-			.decrypt	= ecb_decrypt,
-		},
-	},
-}, {
-	.cra_name		= "__cbc-aes-aesni",
-	.cra_driver_name	= "__driver-cbc-aes-aesni",
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct crypto_aes_ctx) +
-				  AESNI_ALIGN - 1,
-	.cra_alignmask		= 0,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_u = {
-		.blkcipher = {
-			.min_keysize	= AES_MIN_KEY_SIZE,
-			.max_keysize	= AES_MAX_KEY_SIZE,
-			.setkey		= aes_set_key,
-			.encrypt	= cbc_encrypt,
-			.decrypt	= cbc_decrypt,
-		},
-	},
-}, {
-	.cra_name		= "ecb(aes)",
-	.cra_driver_name	= "ecb-aes-aesni",
-	.cra_priority		= 400,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 0,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_ecb_init,
-	.cra_exit		= ablk_exit,
-	.cra_u = {
-		.ablkcipher = {
-			.min_keysize	= AES_MIN_KEY_SIZE,
-			.max_keysize	= AES_MAX_KEY_SIZE,
-			.setkey		= ablk_set_key,
-			.encrypt	= ablk_encrypt,
-			.decrypt	= ablk_decrypt,
+} };
+
+static struct skcipher_alg aesni_skciphers[] = {
+	{
+		.base = {
+			.cra_name		= "__ecb(aes)",
+			.cra_driver_name	= "__ecb-aes-aesni",
+			.cra_priority		= 400,
+			.cra_flags		= CRYPTO_ALG_INTERNAL,
+			.cra_blocksize		= AES_BLOCK_SIZE,
+			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
+			.cra_module		= THIS_MODULE,
 		},
-	},
-}, {
-	.cra_name		= "cbc(aes)",
-	.cra_driver_name	= "cbc-aes-aesni",
-	.cra_priority		= 400,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 0,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_cbc_init,
-	.cra_exit		= ablk_exit,
-	.cra_u = {
-		.ablkcipher = {
-			.min_keysize	= AES_MIN_KEY_SIZE,
-			.max_keysize	= AES_MAX_KEY_SIZE,
-			.ivsize		= AES_BLOCK_SIZE,
-			.setkey		= ablk_set_key,
-			.encrypt	= ablk_encrypt,
-			.decrypt	= ablk_decrypt,
+		.min_keysize	= AES_MIN_KEY_SIZE,
+		.max_keysize	= AES_MAX_KEY_SIZE,
+		.setkey		= aesni_skcipher_setkey,
+		.encrypt	= ecb_encrypt,
+		.decrypt	= ecb_decrypt,
+	}, {
+		.base = {
+			.cra_name		= "__cbc(aes)",
+			.cra_driver_name	= "__cbc-aes-aesni",
+			.cra_priority		= 400,
+			.cra_flags		= CRYPTO_ALG_INTERNAL,
+			.cra_blocksize		= AES_BLOCK_SIZE,
+			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
+			.cra_module		= THIS_MODULE,
 		},
-	},
+		.min_keysize	= AES_MIN_KEY_SIZE,
+		.max_keysize	= AES_MAX_KEY_SIZE,
+		.ivsize		= AES_BLOCK_SIZE,
+		.setkey		= aesni_skcipher_setkey,
+		.encrypt	= cbc_encrypt,
+		.decrypt	= cbc_decrypt,
 #ifdef CONFIG_X86_64
-}, {
-	.cra_name		= "__ctr-aes-aesni",
-	.cra_driver_name	= "__driver-ctr-aes-aesni",
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= 1,
-	.cra_ctxsize		= sizeof(struct crypto_aes_ctx) +
-				  AESNI_ALIGN - 1,
-	.cra_alignmask		= 0,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_u = {
-		.blkcipher = {
-			.min_keysize	= AES_MIN_KEY_SIZE,
-			.max_keysize	= AES_MAX_KEY_SIZE,
-			.ivsize		= AES_BLOCK_SIZE,
-			.setkey		= aes_set_key,
-			.encrypt	= ctr_crypt,
-			.decrypt	= ctr_crypt,
+	}, {
+		.base = {
+			.cra_name		= "__ctr(aes)",
+			.cra_driver_name	= "__ctr-aes-aesni",
+			.cra_priority		= 400,
+			.cra_flags		= CRYPTO_ALG_INTERNAL,
+			.cra_blocksize		= 1,
+			.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
+			.cra_module		= THIS_MODULE,
 		},
-	},
-}, {
-	.cra_name		= "ctr(aes)",
-	.cra_driver_name	= "ctr-aes-aesni",
-	.cra_priority		= 400,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= 1,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 0,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_ctr_init,
-	.cra_exit		= ablk_exit,
-	.cra_u = {
-		.ablkcipher = {
-			.min_keysize	= AES_MIN_KEY_SIZE,
-			.max_keysize	= AES_MAX_KEY_SIZE,
-			.ivsize		= AES_BLOCK_SIZE,
-			.setkey		= ablk_set_key,
-			.encrypt	= ablk_encrypt,
-			.decrypt	= ablk_encrypt,
-			.geniv		= "chainiv",
+		.min_keysize	= AES_MIN_KEY_SIZE,
+		.max_keysize	= AES_MAX_KEY_SIZE,
+		.ivsize		= AES_BLOCK_SIZE,
+		.chunksize	= AES_BLOCK_SIZE,
+		.setkey		= aesni_skcipher_setkey,
+		.encrypt	= ctr_crypt,
+		.decrypt	= ctr_crypt,
+	}, {
+		.base = {
+			.cra_name		= "__xts(aes)",
+			.cra_driver_name	= "__xts-aes-aesni",
+			.cra_priority		= 401,
+			.cra_flags		= CRYPTO_ALG_INTERNAL,
+			.cra_blocksize		= AES_BLOCK_SIZE,
+			.cra_ctxsize		= XTS_AES_CTX_SIZE,
+			.cra_module		= THIS_MODULE,
 		},
-	},
+		.min_keysize	= 2 * AES_MIN_KEY_SIZE,
+		.max_keysize	= 2 * AES_MAX_KEY_SIZE,
+		.ivsize		= AES_BLOCK_SIZE,
+		.setkey		= xts_aesni_setkey,
+		.encrypt	= xts_encrypt,
+		.decrypt	= xts_decrypt,
 #endif
+	}
+};
+
+struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
+
+struct {
+	const char *algname;
+	const char *drvname;
+	const char *basename;
+	struct simd_skcipher_alg *simd;
+} aesni_simd_skciphers2[] = {
 #if IS_ENABLED(CONFIG_CRYPTO_PCBC)
-}, {
-	.cra_name		= "pcbc(aes)",
-	.cra_driver_name	= "pcbc-aes-aesni",
-	.cra_priority		= 400,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 0,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_pcbc_init,
-	.cra_exit		= ablk_exit,
-	.cra_u = {
-		.ablkcipher = {
-			.min_keysize	= AES_MIN_KEY_SIZE,
-			.max_keysize	= AES_MAX_KEY_SIZE,
-			.ivsize		= AES_BLOCK_SIZE,
-			.setkey		= ablk_set_key,
-			.encrypt	= ablk_encrypt,
-			.decrypt	= ablk_decrypt,
-		},
+	{
+		.algname	= "pcbc(aes)",
+		.drvname	= "pcbc-aes-aesni",
+		.basename	= "fpu(pcbc(__aes-aesni))",
 	},
 #endif
-}, {
-	.cra_name		= "__lrw-aes-aesni",
-	.cra_driver_name	= "__driver-lrw-aes-aesni",
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct aesni_lrw_ctx),
-	.cra_alignmask		= 0,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_exit		= lrw_aesni_exit_tfm,
-	.cra_u = {
-		.blkcipher = {
-			.min_keysize	= AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
-			.max_keysize	= AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
-			.ivsize		= AES_BLOCK_SIZE,
-			.setkey		= lrw_aesni_setkey,
-			.encrypt	= lrw_encrypt,
-			.decrypt	= lrw_decrypt,
-		},
-	},
-}, {
-	.cra_name		= "__xts-aes-aesni",
-	.cra_driver_name	= "__driver-xts-aes-aesni",
-	.cra_priority		= 0,
-	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
-				  CRYPTO_ALG_INTERNAL,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct aesni_xts_ctx),
-	.cra_alignmask		= 0,
-	.cra_type		= &crypto_blkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_u = {
-		.blkcipher = {
-			.min_keysize	= 2 * AES_MIN_KEY_SIZE,
-			.max_keysize	= 2 * AES_MAX_KEY_SIZE,
-			.ivsize		= AES_BLOCK_SIZE,
-			.setkey		= xts_aesni_setkey,
-			.encrypt	= xts_encrypt,
-			.decrypt	= xts_decrypt,
-		},
-	},
-}, {
-	.cra_name		= "lrw(aes)",
-	.cra_driver_name	= "lrw-aes-aesni",
-	.cra_priority		= 400,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 0,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_u = {
-		.ablkcipher = {
-			.min_keysize	= AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
-			.max_keysize	= AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
-			.ivsize		= AES_BLOCK_SIZE,
-			.setkey		= ablk_set_key,
-			.encrypt	= ablk_encrypt,
-			.decrypt	= ablk_decrypt,
-		},
-	},
-}, {
-	.cra_name		= "xts(aes)",
-	.cra_driver_name	= "xts-aes-aesni",
-	.cra_priority		= 400,
-	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct async_helper_ctx),
-	.cra_alignmask		= 0,
-	.cra_type		= &crypto_ablkcipher_type,
-	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_init,
-	.cra_exit		= ablk_exit,
-	.cra_u = {
-		.ablkcipher = {
-			.min_keysize	= 2 * AES_MIN_KEY_SIZE,
-			.max_keysize	= 2 * AES_MAX_KEY_SIZE,
-			.ivsize		= AES_BLOCK_SIZE,
-			.setkey		= ablk_set_key,
-			.encrypt	= ablk_encrypt,
-			.decrypt	= ablk_decrypt,
-		},
-	},
-} };
+};
 
 #ifdef CONFIG_X86_64
 static struct aead_alg aesni_aead_algs[] = { {
@@ -1401,9 +1076,27 @@ static const struct x86_cpu_id aesni_cpu_id[] = {
 };
 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
 
+static void aesni_free_simds(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) &&
+		    aesni_simd_skciphers[i]; i++)
+		simd_skcipher_free(aesni_simd_skciphers[i]);
+
+	for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2) &&
+		    aesni_simd_skciphers2[i].simd; i++)
+		simd_skcipher_free(aesni_simd_skciphers2[i].simd);
+}
+
 static int __init aesni_init(void)
 {
+	struct simd_skcipher_alg *simd;
+	const char *basename;
+	const char *algname;
+	const char *drvname;
 	int err;
+	int i;
 
 	if (!x86_match_cpu(aesni_cpu_id))
 		return -ENODEV;
@@ -1445,13 +1138,48 @@ static int __init aesni_init(void)
 	if (err)
 		goto fpu_exit;
 
+	err = crypto_register_skciphers(aesni_skciphers,
+					ARRAY_SIZE(aesni_skciphers));
+	if (err)
+		goto unregister_algs;
+
 	err = crypto_register_aeads(aesni_aead_algs,
 				    ARRAY_SIZE(aesni_aead_algs));
 	if (err)
-		goto unregister_algs;
+		goto unregister_skciphers;
+
+	for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) {
+		algname = aesni_skciphers[i].base.cra_name + 2;
+		drvname = aesni_skciphers[i].base.cra_driver_name + 2;
+		basename = aesni_skciphers[i].base.cra_driver_name;
+		simd = simd_skcipher_create_compat(algname, drvname, basename);
+		err = PTR_ERR(simd);
+		if (IS_ERR(simd))
+			goto unregister_simds;
+
+		aesni_simd_skciphers[i] = simd;
+	}
 
-	return err;
+	for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++) {
+		algname = aesni_simd_skciphers2[i].algname;
+		drvname = aesni_simd_skciphers2[i].drvname;
+		basename = aesni_simd_skciphers2[i].basename;
+		simd = simd_skcipher_create_compat(algname, drvname, basename);
+		err = PTR_ERR(simd);
+		if (IS_ERR(simd))
+			goto unregister_simds;
 
+		aesni_simd_skciphers2[i].simd = simd;
+	}
+
+	return 0;
+
+unregister_simds:
+	aesni_free_simds();
+	crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
+unregister_skciphers:
+	crypto_unregister_skciphers(aesni_skciphers,
+				    ARRAY_SIZE(aesni_skciphers));
 unregister_algs:
 	crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
 fpu_exit:
@@ -1461,7 +1189,10 @@ fpu_exit:
 
 static void __exit aesni_exit(void)
 {
+	aesni_free_simds();
 	crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
+	crypto_unregister_skciphers(aesni_skciphers,
+				    ARRAY_SIZE(aesni_skciphers));
 	crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
 
 	crypto_fpu_exit();
diff --git a/arch/x86/crypto/fpu.c b/arch/x86/crypto/fpu.c
index e7d679e2a018..406680476c52 100644
--- a/arch/x86/crypto/fpu.c
+++ b/arch/x86/crypto/fpu.c
@@ -11,143 +11,186 @@
  *
  */
 
-#include <crypto/algapi.h>
+#include <crypto/internal/skcipher.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/slab.h>
-#include <linux/crypto.h>
 #include <asm/fpu/api.h>
 
 struct crypto_fpu_ctx {
-	struct crypto_blkcipher *child;
+	struct crypto_skcipher *child;
 };
 
-static int crypto_fpu_setkey(struct crypto_tfm *parent, const u8 *key,
+static int crypto_fpu_setkey(struct crypto_skcipher *parent, const u8 *key,
 			     unsigned int keylen)
 {
-	struct crypto_fpu_ctx *ctx = crypto_tfm_ctx(parent);
-	struct crypto_blkcipher *child = ctx->child;
+	struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(parent);
+	struct crypto_skcipher *child = ctx->child;
 	int err;
 
-	crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
-	crypto_blkcipher_set_flags(child, crypto_tfm_get_flags(parent) &
-				   CRYPTO_TFM_REQ_MASK);
-	err = crypto_blkcipher_setkey(child, key, keylen);
-	crypto_tfm_set_flags(parent, crypto_blkcipher_get_flags(child) &
-				     CRYPTO_TFM_RES_MASK);
+	crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
+					 CRYPTO_TFM_REQ_MASK);
+	err = crypto_skcipher_setkey(child, key, keylen);
+	crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) &
+					  CRYPTO_TFM_RES_MASK);
 	return err;
 }
 
-static int crypto_fpu_encrypt(struct blkcipher_desc *desc_in,
-			      struct scatterlist *dst, struct scatterlist *src,
-			      unsigned int nbytes)
+static int crypto_fpu_encrypt(struct skcipher_request *req)
 {
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct crypto_skcipher *child = ctx->child;
+	SKCIPHER_REQUEST_ON_STACK(subreq, child);
 	int err;
-	struct crypto_fpu_ctx *ctx = crypto_blkcipher_ctx(desc_in->tfm);
-	struct crypto_blkcipher *child = ctx->child;
-	struct blkcipher_desc desc = {
-		.tfm = child,
-		.info = desc_in->info,
-		.flags = desc_in->flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
-	};
+
+	skcipher_request_set_tfm(subreq, child);
+	skcipher_request_set_callback(subreq, 0, NULL, NULL);
+	skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
+				   req->iv);
 
 	kernel_fpu_begin();
-	err = crypto_blkcipher_crt(desc.tfm)->encrypt(&desc, dst, src, nbytes);
+	err = crypto_skcipher_encrypt(subreq);
 	kernel_fpu_end();
+
+	skcipher_request_zero(subreq);
 	return err;
 }
 
-static int crypto_fpu_decrypt(struct blkcipher_desc *desc_in,
-			      struct scatterlist *dst, struct scatterlist *src,
-			      unsigned int nbytes)
+static int crypto_fpu_decrypt(struct skcipher_request *req)
 {
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct crypto_skcipher *child = ctx->child;
+	SKCIPHER_REQUEST_ON_STACK(subreq, child);
 	int err;
-	struct crypto_fpu_ctx *ctx = crypto_blkcipher_ctx(desc_in->tfm);
-	struct crypto_blkcipher *child = ctx->child;
-	struct blkcipher_desc desc = {
-		.tfm = child,
-		.info = desc_in->info,
-		.flags = desc_in->flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
-	};
+
+	skcipher_request_set_tfm(subreq, child);
+	skcipher_request_set_callback(subreq, 0, NULL, NULL);
+	skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
+				   req->iv);
 
 	kernel_fpu_begin();
-	err = crypto_blkcipher_crt(desc.tfm)->decrypt(&desc, dst, src, nbytes);
+	err = crypto_skcipher_decrypt(subreq);
 	kernel_fpu_end();
+
+	skcipher_request_zero(subreq);
 	return err;
 }
 
-static int crypto_fpu_init_tfm(struct crypto_tfm *tfm)
+static int crypto_fpu_init_tfm(struct crypto_skcipher *tfm)
 {
-	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
-	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
-	struct crypto_fpu_ctx *ctx = crypto_tfm_ctx(tfm);
-	struct crypto_blkcipher *cipher;
+	struct skcipher_instance *inst = skcipher_alg_instance(tfm);
+	struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct crypto_skcipher_spawn *spawn;
+	struct crypto_skcipher *cipher;
 
-	cipher = crypto_spawn_blkcipher(spawn);
+	spawn = skcipher_instance_ctx(inst);
+	cipher = crypto_spawn_skcipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 
 	ctx->child = cipher;
+
 	return 0;
 }
 
-static void crypto_fpu_exit_tfm(struct crypto_tfm *tfm)
+static void crypto_fpu_exit_tfm(struct crypto_skcipher *tfm)
+{
+	struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+	crypto_free_skcipher(ctx->child);
+}
+
+static void crypto_fpu_free(struct skcipher_instance *inst)
 {
-	struct crypto_fpu_ctx *ctx = crypto_tfm_ctx(tfm);
-	crypto_free_blkcipher(ctx->child);
+	crypto_drop_skcipher(skcipher_instance_ctx(inst));
+	kfree(inst);
 }
 
-static struct crypto_instance *crypto_fpu_alloc(struct rtattr **tb)
+static int crypto_fpu_create(struct crypto_template *tmpl, struct rtattr **tb)
 {
-	struct crypto_instance *inst;
-	struct crypto_alg *alg;
+	struct crypto_skcipher_spawn *spawn;
+	struct skcipher_instance *inst;
+	struct crypto_attr_type *algt;
+	struct skcipher_alg *alg;
+	const char *cipher_name;
 	int err;
 
-	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
+	algt = crypto_get_attr_type(tb);
+	if (IS_ERR(algt))
+		return PTR_ERR(algt);
+
+	if ((algt->type ^ (CRYPTO_ALG_INTERNAL | CRYPTO_ALG_TYPE_SKCIPHER)) &
+	    algt->mask)
+		return -EINVAL;
+
+	if (!(algt->mask & CRYPTO_ALG_INTERNAL))
+		return -EINVAL;
+
+	cipher_name = crypto_attr_alg_name(tb[1]);
+	if (IS_ERR(cipher_name))
+		return PTR_ERR(cipher_name);
+
+	inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
+	if (!inst)
+		return -ENOMEM;
+
+	spawn = skcipher_instance_ctx(inst);
+
+	crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst));
+	err = crypto_grab_skcipher(spawn, cipher_name, CRYPTO_ALG_INTERNAL,
+				   CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC);
 	if (err)
-		return ERR_PTR(err);
-
-	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_BLKCIPHER,
-				  CRYPTO_ALG_TYPE_MASK);
-	if (IS_ERR(alg))
-		return ERR_CAST(alg);
-
-	inst = crypto_alloc_instance("fpu", alg);
-	if (IS_ERR(inst))
-		goto out_put_alg;
-
-	inst->alg.cra_flags = alg->cra_flags;
-	inst->alg.cra_priority = alg->cra_priority;
-	inst->alg.cra_blocksize = alg->cra_blocksize;
-	inst->alg.cra_alignmask = alg->cra_alignmask;
-	inst->alg.cra_type = alg->cra_type;
-	inst->alg.cra_blkcipher.ivsize = alg->cra_blkcipher.ivsize;
-	inst->alg.cra_blkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
-	inst->alg.cra_blkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
-	inst->alg.cra_ctxsize = sizeof(struct crypto_fpu_ctx);
-	inst->alg.cra_init = crypto_fpu_init_tfm;
-	inst->alg.cra_exit = crypto_fpu_exit_tfm;
-	inst->alg.cra_blkcipher.setkey = crypto_fpu_setkey;
-	inst->alg.cra_blkcipher.encrypt = crypto_fpu_encrypt;
-	inst->alg.cra_blkcipher.decrypt = crypto_fpu_decrypt;
-
-out_put_alg:
-	crypto_mod_put(alg);
-	return inst;
-}
+		goto out_free_inst;
 
-static void crypto_fpu_free(struct crypto_instance *inst)
-{
-	crypto_drop_spawn(crypto_instance_ctx(inst));
+	alg = crypto_skcipher_spawn_alg(spawn);
+
+	err = crypto_inst_setname(skcipher_crypto_instance(inst), "fpu",
+				  &alg->base);
+	if (err)
+		goto out_drop_skcipher;
+
+	inst->alg.base.cra_flags = CRYPTO_ALG_INTERNAL;
+	inst->alg.base.cra_priority = alg->base.cra_priority;
+	inst->alg.base.cra_blocksize = alg->base.cra_blocksize;
+	inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
+
+	inst->alg.ivsize = crypto_skcipher_alg_ivsize(alg);
+	inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg);
+	inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg);
+
+	inst->alg.base.cra_ctxsize = sizeof(struct crypto_fpu_ctx);
+
+	inst->alg.init = crypto_fpu_init_tfm;
+	inst->alg.exit = crypto_fpu_exit_tfm;
+
+	inst->alg.setkey = crypto_fpu_setkey;
+	inst->alg.encrypt = crypto_fpu_encrypt;
+	inst->alg.decrypt = crypto_fpu_decrypt;
+
+	inst->free = crypto_fpu_free;
+
+	err = skcipher_register_instance(tmpl, inst);
+	if (err)
+		goto out_drop_skcipher;
+
+out:
+	return err;
+
+out_drop_skcipher:
+	crypto_drop_skcipher(spawn);
+out_free_inst:
 	kfree(inst);
+	goto out;
 }
 
 static struct crypto_template crypto_fpu_tmpl = {
 	.name = "fpu",
-	.alloc = crypto_fpu_alloc,
-	.free = crypto_fpu_free,
+	.create = crypto_fpu_create,
 	.module = THIS_MODULE,
 };
 
diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c
index 6a85598931b5..260a060d7275 100644
--- a/arch/x86/crypto/glue_helper.c
+++ b/arch/x86/crypto/glue_helper.c
@@ -27,10 +27,10 @@
 
 #include <linux/module.h>
 #include <crypto/b128ops.h>
+#include <crypto/internal/skcipher.h>
 #include <crypto/lrw.h>
 #include <crypto/xts.h>
 #include <asm/crypto/glue_helper.h>
-#include <crypto/scatterwalk.h>
 
 static int __glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
 				   struct blkcipher_desc *desc,
@@ -339,6 +339,41 @@ done:
 	return nbytes;
 }
 
+static unsigned int __glue_xts_req_128bit(const struct common_glue_ctx *gctx,
+					  void *ctx,
+					  struct skcipher_walk *walk)
+{
+	const unsigned int bsize = 128 / 8;
+	unsigned int nbytes = walk->nbytes;
+	u128 *src = walk->src.virt.addr;
+	u128 *dst = walk->dst.virt.addr;
+	unsigned int num_blocks, func_bytes;
+	unsigned int i;
+
+	/* Process multi-block batch */
+	for (i = 0; i < gctx->num_funcs; i++) {
+		num_blocks = gctx->funcs[i].num_blocks;
+		func_bytes = bsize * num_blocks;
+
+		if (nbytes >= func_bytes) {
+			do {
+				gctx->funcs[i].fn_u.xts(ctx, dst, src,
+							walk->iv);
+
+				src += num_blocks;
+				dst += num_blocks;
+				nbytes -= func_bytes;
+			} while (nbytes >= func_bytes);
+
+			if (nbytes < bsize)
+				goto done;
+		}
+	}
+
+done:
+	return nbytes;
+}
+
 /* for implementations implementing faster XTS IV generator */
 int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
 			  struct blkcipher_desc *desc, struct scatterlist *dst,
@@ -379,6 +414,43 @@ int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
 }
 EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit);
 
+int glue_xts_req_128bit(const struct common_glue_ctx *gctx,
+			struct skcipher_request *req,
+			common_glue_func_t tweak_fn, void *tweak_ctx,
+			void *crypt_ctx)
+{
+	const unsigned int bsize = 128 / 8;
+	struct skcipher_walk walk;
+	bool fpu_enabled = false;
+	unsigned int nbytes;
+	int err;
+
+	err = skcipher_walk_virt(&walk, req, false);
+	nbytes = walk.nbytes;
+	if (!nbytes)
+		return err;
+
+	/* set minimum length to bsize, for tweak_fn */
+	fpu_enabled = glue_skwalk_fpu_begin(bsize, gctx->fpu_blocks_limit,
+					    &walk, fpu_enabled,
+					    nbytes < bsize ? bsize : nbytes);
+
+	/* calculate first value of T */
+	tweak_fn(tweak_ctx, walk.iv, walk.iv);
+
+	while (nbytes) {
+		nbytes = __glue_xts_req_128bit(gctx, crypt_ctx, &walk);
+
+		err = skcipher_walk_done(&walk, nbytes);
+		nbytes = walk.nbytes;
+	}
+
+	glue_fpu_end(fpu_enabled);
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(glue_xts_req_128bit);
+
 void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv,
 			       common_glue_func_t fn)
 {
diff --git a/arch/x86/crypto/sha1-mb/sha1_mb.c b/arch/x86/crypto/sha1-mb/sha1_mb.c
index 9e5b67127a09..acf9fdf01671 100644
--- a/arch/x86/crypto/sha1-mb/sha1_mb.c
+++ b/arch/x86/crypto/sha1-mb/sha1_mb.c
@@ -114,7 +114,7 @@ static inline void sha1_init_digest(uint32_t *digest)
 }
 
 static inline uint32_t sha1_pad(uint8_t padblock[SHA1_BLOCK_SIZE * 2],
-			 uint32_t total_len)
+			 uint64_t total_len)
 {
 	uint32_t i = total_len & (SHA1_BLOCK_SIZE - 1);
 
diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h b/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h
index 98a35bcc6f4a..13590ccf965c 100644
--- a/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h
+++ b/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h
@@ -125,7 +125,7 @@ struct sha1_hash_ctx {
 	/* error flag */
 	int error;
 
-	uint32_t	total_length;
+	uint64_t	total_length;
 	const void	*incoming_buffer;
 	uint32_t	incoming_buffer_length;
 	uint8_t		partial_block_buffer[SHA1_BLOCK_SIZE * 2];
diff --git a/arch/x86/crypto/sha256-mb/sha256_mb.c b/arch/x86/crypto/sha256-mb/sha256_mb.c
index 6f97fb33ae21..7926a226b120 100644
--- a/arch/x86/crypto/sha256-mb/sha256_mb.c
+++ b/arch/x86/crypto/sha256-mb/sha256_mb.c
@@ -115,7 +115,7 @@ inline void sha256_init_digest(uint32_t *digest)
 }
 
 inline uint32_t sha256_pad(uint8_t padblock[SHA256_BLOCK_SIZE * 2],
-			 uint32_t total_len)
+			 uint64_t total_len)
 {
 	uint32_t i = total_len & (SHA256_BLOCK_SIZE - 1);
 
diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h b/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h
index edd252b73206..aabb30320af0 100644
--- a/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h
+++ b/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h
@@ -125,7 +125,7 @@ struct sha256_hash_ctx {
 	/* error flag */
 	int error;
 
-	uint32_t	total_length;
+	uint64_t	total_length;
 	const void	*incoming_buffer;
 	uint32_t	incoming_buffer_length;
 	uint8_t		partial_block_buffer[SHA256_BLOCK_SIZE * 2];
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb.c b/arch/x86/crypto/sha512-mb/sha512_mb.c
index d210174a52b0..9c1bb6d58141 100644
--- a/arch/x86/crypto/sha512-mb/sha512_mb.c
+++ b/arch/x86/crypto/sha512-mb/sha512_mb.c
@@ -117,7 +117,7 @@ inline void sha512_init_digest(uint64_t *digest)
 }
 
 inline uint32_t sha512_pad(uint8_t padblock[SHA512_BLOCK_SIZE * 2],
-			 uint32_t total_len)
+			 uint64_t total_len)
 {
 	uint32_t i = total_len & (SHA512_BLOCK_SIZE - 1);
 
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h b/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h
index 9d4b2c8208d5..e4653f5eec3f 100644
--- a/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h
+++ b/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h
@@ -119,7 +119,7 @@ struct sha512_hash_ctx {
 	/* error flag */
 	int error;
 
-	uint32_t        total_length;
+	uint64_t        total_length;
 	const void      *incoming_buffer;
 	uint32_t        incoming_buffer_length;
 	uint8_t         partial_block_buffer[SHA512_BLOCK_SIZE * 2];
diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index acc0c6f36f3f..701d29f8e4d3 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -926,8 +926,8 @@ ftrace_graph_call:
 	jmp	ftrace_stub
 #endif
 
-.globl ftrace_stub
-ftrace_stub:
+/* This is weak to keep gas from relaxing the jumps */
+WEAK(ftrace_stub)
 	ret
 END(ftrace_caller)
 
diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c
index e739002427ed..40121d14d34d 100644
--- a/arch/x86/entry/vdso/vma.c
+++ b/arch/x86/entry/vdso/vma.c
@@ -109,7 +109,7 @@ static int vvar_fault(const struct vm_special_mapping *sm,
 		return VM_FAULT_SIGBUS;
 
 	if (sym_offset == image->sym_vvar_page) {
-		ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address,
+		ret = vm_insert_pfn(vma, vmf->address,
 				    __pa_symbol(&__vvar_page) >> PAGE_SHIFT);
 	} else if (sym_offset == image->sym_pvclock_page) {
 		struct pvclock_vsyscall_time_info *pvti =
@@ -117,7 +117,7 @@ static int vvar_fault(const struct vm_special_mapping *sm,
 		if (pvti && vclock_was_used(VCLOCK_PVCLOCK)) {
 			ret = vm_insert_pfn(
 				vma,
-				(unsigned long)vmf->virtual_address,
+				vmf->address,
 				__pa(pvti) >> PAGE_SHIFT);
 		}
 	}
diff --git a/arch/x86/include/asm/crypto/glue_helper.h b/arch/x86/include/asm/crypto/glue_helper.h
index 03bb1065c335..29e53ea7d764 100644
--- a/arch/x86/include/asm/crypto/glue_helper.h
+++ b/arch/x86/include/asm/crypto/glue_helper.h
@@ -5,8 +5,8 @@
 #ifndef _CRYPTO_GLUE_HELPER_H
 #define _CRYPTO_GLUE_HELPER_H
 
+#include <crypto/internal/skcipher.h>
 #include <linux/kernel.h>
-#include <linux/crypto.h>
 #include <asm/fpu/api.h>
 #include <crypto/b128ops.h>
 
@@ -69,6 +69,31 @@ static inline bool glue_fpu_begin(unsigned int bsize, int fpu_blocks_limit,
 	return true;
 }
 
+static inline bool glue_skwalk_fpu_begin(unsigned int bsize,
+					 int fpu_blocks_limit,
+					 struct skcipher_walk *walk,
+					 bool fpu_enabled, unsigned int nbytes)
+{
+	if (likely(fpu_blocks_limit < 0))
+		return false;
+
+	if (fpu_enabled)
+		return true;
+
+	/*
+	 * Vector-registers are only used when chunk to be processed is large
+	 * enough, so do not enable FPU until it is necessary.
+	 */
+	if (nbytes < bsize * (unsigned int)fpu_blocks_limit)
+		return false;
+
+	/* prevent sleeping if FPU is in use */
+	skcipher_walk_atomise(walk);
+
+	kernel_fpu_begin();
+	return true;
+}
+
 static inline void glue_fpu_end(bool fpu_enabled)
 {
 	if (fpu_enabled)
@@ -139,6 +164,18 @@ extern int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
 				 common_glue_func_t tweak_fn, void *tweak_ctx,
 				 void *crypt_ctx);
 
+extern int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
+				 struct blkcipher_desc *desc,
+				 struct scatterlist *dst,
+				 struct scatterlist *src, unsigned int nbytes,
+				 common_glue_func_t tweak_fn, void *tweak_ctx,
+				 void *crypt_ctx);
+
+extern int glue_xts_req_128bit(const struct common_glue_ctx *gctx,
+			       struct skcipher_request *req,
+			       common_glue_func_t tweak_fn, void *tweak_ctx,
+			       void *crypt_ctx);
+
 extern void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src,
 				      le128 *iv, common_glue_func_t fn);
 
diff --git a/arch/x86/include/asm/e820.h b/arch/x86/include/asm/e820.h
index 476b574de99e..ec23d8e1297c 100644
--- a/arch/x86/include/asm/e820.h
+++ b/arch/x86/include/asm/e820.h
@@ -1,13 +1,17 @@
 #ifndef _ASM_X86_E820_H
 #define _ASM_X86_E820_H
 
-#ifdef CONFIG_EFI
+/*
+ * E820_X_MAX is the maximum size of the extended E820 table.  The extended
+ * table may contain up to 3 extra E820 entries per possible NUMA node, so we
+ * make room for 3 * MAX_NUMNODES possible entries, beyond the standard 128.
+ * Also note that E820_X_MAX *must* be defined before we include uapi/asm/e820.h.
+ */
 #include <linux/numa.h>
 #define E820_X_MAX (E820MAX + 3 * MAX_NUMNODES)
-#else	/* ! CONFIG_EFI */
-#define E820_X_MAX E820MAX
-#endif
+
 #include <uapi/asm/e820.h>
+
 #ifndef __ASSEMBLY__
 /* see comment in arch/x86/kernel/e820.c */
 extern struct e820map *e820;
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index bdde80731f49..7892530cbacf 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -191,6 +191,8 @@ enum {
 #define PFERR_RSVD_BIT 3
 #define PFERR_FETCH_BIT 4
 #define PFERR_PK_BIT 5
+#define PFERR_GUEST_FINAL_BIT 32
+#define PFERR_GUEST_PAGE_BIT 33
 
 #define PFERR_PRESENT_MASK (1U << PFERR_PRESENT_BIT)
 #define PFERR_WRITE_MASK (1U << PFERR_WRITE_BIT)
@@ -198,6 +200,13 @@ enum {
 #define PFERR_RSVD_MASK (1U << PFERR_RSVD_BIT)
 #define PFERR_FETCH_MASK (1U << PFERR_FETCH_BIT)
 #define PFERR_PK_MASK (1U << PFERR_PK_BIT)
+#define PFERR_GUEST_FINAL_MASK (1ULL << PFERR_GUEST_FINAL_BIT)
+#define PFERR_GUEST_PAGE_MASK (1ULL << PFERR_GUEST_PAGE_BIT)
+
+#define PFERR_NESTED_GUEST_PAGE (PFERR_GUEST_PAGE_MASK |	\
+				 PFERR_USER_MASK |		\
+				 PFERR_WRITE_MASK |		\
+				 PFERR_PRESENT_MASK)
 
 /* apic attention bits */
 #define KVM_APIC_CHECK_VAPIC	0
@@ -1062,6 +1071,7 @@ unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm);
 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages);
 
 int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3);
+bool pdptrs_changed(struct kvm_vcpu *vcpu);
 
 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
 			  const void *val, int bytes);
@@ -1124,7 +1134,8 @@ int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr);
 struct x86_emulate_ctxt;
 
 int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port);
-void kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
+int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size, unsigned short port);
+int kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
 int kvm_emulate_halt(struct kvm_vcpu *vcpu);
 int kvm_vcpu_halt(struct kvm_vcpu *vcpu);
 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu);
@@ -1203,7 +1214,7 @@ void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu);
 
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
 
-int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code,
+int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u64 error_code,
 		       void *insn, int insn_len);
 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
 void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu);
@@ -1358,7 +1369,8 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu);
 extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
 
-void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
+int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu);
+int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
 
 int kvm_is_in_guest(void);
 
diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h
index c2b8d24a235c..d74747b031ec 100644
--- a/arch/x86/include/asm/kvm_page_track.h
+++ b/arch/x86/include/asm/kvm_page_track.h
@@ -29,9 +29,20 @@ struct kvm_page_track_notifier_node {
 	 * @gpa: the physical address written by guest.
 	 * @new: the data was written to the address.
 	 * @bytes: the written length.
+	 * @node: this node
 	 */
 	void (*track_write)(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new,
-			    int bytes);
+			    int bytes, struct kvm_page_track_notifier_node *node);
+	/*
+	 * It is called when memory slot is being moved or removed
+	 * users can drop write-protection for the pages in that memory slot
+	 *
+	 * @kvm: the kvm where memory slot being moved or removed
+	 * @slot: the memory slot being moved or removed
+	 * @node: this node
+	 */
+	void (*track_flush_slot)(struct kvm *kvm, struct kvm_memory_slot *slot,
+			    struct kvm_page_track_notifier_node *node);
 };
 
 void kvm_page_track_init(struct kvm *kvm);
@@ -58,4 +69,5 @@ kvm_page_track_unregister_notifier(struct kvm *kvm,
 				   struct kvm_page_track_notifier_node *n);
 void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new,
 			  int bytes);
+void kvm_page_track_flush_slot(struct kvm *kvm, struct kvm_memory_slot *slot);
 #endif
diff --git a/arch/x86/include/asm/trace/exceptions.h b/arch/x86/include/asm/trace/exceptions.h
index 2fbc66c7885b..2422b14c50a7 100644
--- a/arch/x86/include/asm/trace/exceptions.h
+++ b/arch/x86/include/asm/trace/exceptions.h
@@ -6,7 +6,7 @@
 
 #include <linux/tracepoint.h>
 
-extern void trace_irq_vector_regfunc(void);
+extern int trace_irq_vector_regfunc(void);
 extern void trace_irq_vector_unregfunc(void);
 
 DECLARE_EVENT_CLASS(x86_exceptions,
diff --git a/arch/x86/include/asm/trace/irq_vectors.h b/arch/x86/include/asm/trace/irq_vectors.h
index 38a09a13a9bc..32dd6a9e343c 100644
--- a/arch/x86/include/asm/trace/irq_vectors.h
+++ b/arch/x86/include/asm/trace/irq_vectors.h
@@ -6,7 +6,7 @@
 
 #include <linux/tracepoint.h>
 
-extern void trace_irq_vector_regfunc(void);
+extern int trace_irq_vector_regfunc(void);
 extern void trace_irq_vector_unregfunc(void);
 
 DECLARE_EVENT_CLASS(x86_irq_vector,
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index a002b07a7099..2b5b2d4b924e 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -25,6 +25,7 @@
 #define VMX_H
 
 
+#include <linux/bitops.h>
 #include <linux/types.h>
 #include <uapi/asm/vmx.h>
 
@@ -60,6 +61,7 @@
  */
 #define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001
 #define SECONDARY_EXEC_ENABLE_EPT               0x00000002
+#define SECONDARY_EXEC_DESC			0x00000004
 #define SECONDARY_EXEC_RDTSCP			0x00000008
 #define SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE   0x00000010
 #define SECONDARY_EXEC_ENABLE_VPID              0x00000020
@@ -110,6 +112,36 @@
 #define VMX_MISC_SAVE_EFER_LMA			0x00000020
 #define VMX_MISC_ACTIVITY_HLT			0x00000040
 
+static inline u32 vmx_basic_vmcs_revision_id(u64 vmx_basic)
+{
+	return vmx_basic & GENMASK_ULL(30, 0);
+}
+
+static inline u32 vmx_basic_vmcs_size(u64 vmx_basic)
+{
+	return (vmx_basic & GENMASK_ULL(44, 32)) >> 32;
+}
+
+static inline int vmx_misc_preemption_timer_rate(u64 vmx_misc)
+{
+	return vmx_misc & VMX_MISC_PREEMPTION_TIMER_RATE_MASK;
+}
+
+static inline int vmx_misc_cr3_count(u64 vmx_misc)
+{
+	return (vmx_misc & GENMASK_ULL(24, 16)) >> 16;
+}
+
+static inline int vmx_misc_max_msr(u64 vmx_misc)
+{
+	return (vmx_misc & GENMASK_ULL(27, 25)) >> 25;
+}
+
+static inline int vmx_misc_mseg_revid(u64 vmx_misc)
+{
+	return (vmx_misc & GENMASK_ULL(63, 32)) >> 32;
+}
+
 /* VMCS Encodings */
 enum vmcs_field {
 	VIRTUAL_PROCESSOR_ID            = 0x00000000,
@@ -399,10 +431,11 @@ enum vmcs_field {
 #define IDENTITY_PAGETABLE_PRIVATE_MEMSLOT	(KVM_USER_MEM_SLOTS + 2)
 
 #define VMX_NR_VPIDS				(1 << 16)
+#define VMX_VPID_EXTENT_INDIVIDUAL_ADDR		0
 #define VMX_VPID_EXTENT_SINGLE_CONTEXT		1
 #define VMX_VPID_EXTENT_ALL_CONTEXT		2
+#define VMX_VPID_EXTENT_SINGLE_NON_GLOBAL	3
 
-#define VMX_EPT_EXTENT_INDIVIDUAL_ADDR		0
 #define VMX_EPT_EXTENT_CONTEXT			1
 #define VMX_EPT_EXTENT_GLOBAL			2
 #define VMX_EPT_EXTENT_SHIFT			24
@@ -419,8 +452,10 @@ enum vmcs_field {
 #define VMX_EPT_EXTENT_GLOBAL_BIT		(1ull << 26)
 
 #define VMX_VPID_INVVPID_BIT                    (1ull << 0) /* (32 - 32) */
+#define VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT     (1ull << 8) /* (40 - 32) */
 #define VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT      (1ull << 9) /* (41 - 32) */
 #define VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT      (1ull << 10) /* (42 - 32) */
+#define VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT   (1ull << 11) /* (43 - 32) */
 
 #define VMX_EPT_DEFAULT_GAW			3
 #define VMX_EPT_MAX_GAW				0x4
diff --git a/arch/x86/include/uapi/asm/vmx.h b/arch/x86/include/uapi/asm/vmx.h
index 37fee272618f..14458658e988 100644
--- a/arch/x86/include/uapi/asm/vmx.h
+++ b/arch/x86/include/uapi/asm/vmx.h
@@ -65,6 +65,8 @@
 #define EXIT_REASON_TPR_BELOW_THRESHOLD 43
 #define EXIT_REASON_APIC_ACCESS         44
 #define EXIT_REASON_EOI_INDUCED         45
+#define EXIT_REASON_GDTR_IDTR           46
+#define EXIT_REASON_LDTR_TR             47
 #define EXIT_REASON_EPT_VIOLATION       48
 #define EXIT_REASON_EPT_MISCONFIG       49
 #define EXIT_REASON_INVEPT              50
@@ -113,6 +115,8 @@
 	{ EXIT_REASON_MCE_DURING_VMENTRY,    "MCE_DURING_VMENTRY" }, \
 	{ EXIT_REASON_TPR_BELOW_THRESHOLD,   "TPR_BELOW_THRESHOLD" }, \
 	{ EXIT_REASON_APIC_ACCESS,           "APIC_ACCESS" }, \
+	{ EXIT_REASON_GDTR_IDTR,	     "GDTR_IDTR" }, \
+	{ EXIT_REASON_LDTR_TR,		     "LDTR_TR" }, \
 	{ EXIT_REASON_EPT_VIOLATION,         "EPT_VIOLATION" }, \
 	{ EXIT_REASON_EPT_MISCONFIG,         "EPT_MISCONFIG" }, \
 	{ EXIT_REASON_INVEPT,                "INVEPT" }, \
@@ -129,6 +133,7 @@
 	{ EXIT_REASON_XRSTORS,               "XRSTORS" }
 
 #define VMX_ABORT_SAVE_GUEST_MSR_FAIL        1
+#define VMX_ABORT_LOAD_HOST_PDPTE_FAIL       2
 #define VMX_ABORT_LOAD_HOST_MSR_FAIL         4
 
 #endif /* _UAPIVMX_H */
diff --git a/arch/x86/kernel/acpi/apei.c b/arch/x86/kernel/acpi/apei.c
index c280df6b2aa2..ea3046e0b0cf 100644
--- a/arch/x86/kernel/acpi/apei.c
+++ b/arch/x86/kernel/acpi/apei.c
@@ -24,9 +24,6 @@ int arch_apei_enable_cmcff(struct acpi_hest_header *hest_hdr, void *data)
 	struct acpi_hest_ia_corrected *cmc;
 	struct acpi_hest_ia_error_bank *mc_bank;
 
-	if (hest_hdr->type != ACPI_HEST_TYPE_IA32_CORRECTED_CHECK)
-		return 0;
-
 	cmc = (struct acpi_hest_ia_corrected *)hest_hdr;
 	if (!cmc->enabled)
 		return 0;
diff --git a/arch/x86/kernel/acpi/wakeup_64.S b/arch/x86/kernel/acpi/wakeup_64.S
index 169963f471bb..50b8ed0317a3 100644
--- a/arch/x86/kernel/acpi/wakeup_64.S
+++ b/arch/x86/kernel/acpi/wakeup_64.S
@@ -109,6 +109,15 @@ ENTRY(do_suspend_lowlevel)
 	movq	pt_regs_r14(%rax), %r14
 	movq	pt_regs_r15(%rax), %r15
 
+#ifdef CONFIG_KASAN
+	/*
+	 * The suspend path may have poisoned some areas deeper in the stack,
+	 * which we now need to unpoison.
+	 */
+	movq	%rsp, %rdi
+	call	kasan_unpoison_task_stack_below
+#endif
+
 	xorl	%eax, %eax
 	addq	$8, %rsp
 	FRAME_END
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index de6626c18e42..be6337156502 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -934,6 +934,8 @@ static int __populate_cache_leaves(unsigned int cpu)
 		ci_leaf_init(this_leaf++, &id4_regs);
 		__cache_cpumap_setup(cpu, idx, &id4_regs);
 	}
+	this_cpu_ci->cpu_map_populated = true;
+
 	return 0;
 }
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 132e1ec67da0..00ef43233e03 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -516,7 +516,7 @@ int mce_available(struct cpuinfo_x86 *c)
 
 static void mce_schedule_work(void)
 {
-	if (!mce_gen_pool_empty() && keventd_up())
+	if (!mce_gen_pool_empty())
 		schedule_work(&mce_work);
 }
 
diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c
index 650830e39e3a..3741461c63a0 100644
--- a/arch/x86/kernel/crash.c
+++ b/arch/x86/kernel/crash.c
@@ -631,9 +631,9 @@ static int determine_backup_region(u64 start, u64 end, void *arg)
 
 int crash_load_segments(struct kimage *image)
 {
-	unsigned long src_start, src_sz, elf_sz;
-	void *elf_addr;
 	int ret;
+	struct kexec_buf kbuf = { .image = image, .buf_min = 0,
+				  .buf_max = ULONG_MAX, .top_down = false };
 
 	/*
 	 * Determine and load a segment for backup area. First 640K RAM
@@ -647,43 +647,44 @@ int crash_load_segments(struct kimage *image)
 	if (ret < 0)
 		return ret;
 
-	src_start = image->arch.backup_src_start;
-	src_sz = image->arch.backup_src_sz;
-
 	/* Add backup segment. */
-	if (src_sz) {
+	if (image->arch.backup_src_sz) {
+		kbuf.buffer = &crash_zero_bytes;
+		kbuf.bufsz = sizeof(crash_zero_bytes);
+		kbuf.memsz = image->arch.backup_src_sz;
+		kbuf.buf_align = PAGE_SIZE;
 		/*
 		 * Ideally there is no source for backup segment. This is
 		 * copied in purgatory after crash. Just add a zero filled
 		 * segment for now to make sure checksum logic works fine.
 		 */
-		ret = kexec_add_buffer(image, (char *)&crash_zero_bytes,
-				       sizeof(crash_zero_bytes), src_sz,
-				       PAGE_SIZE, 0, -1, 0,
-				       &image->arch.backup_load_addr);
+		ret = kexec_add_buffer(&kbuf);
 		if (ret)
 			return ret;
+		image->arch.backup_load_addr = kbuf.mem;
 		pr_debug("Loaded backup region at 0x%lx backup_start=0x%lx memsz=0x%lx\n",
-			 image->arch.backup_load_addr, src_start, src_sz);
+			 image->arch.backup_load_addr,
+			 image->arch.backup_src_start, kbuf.memsz);
 	}
 
 	/* Prepare elf headers and add a segment */
-	ret = prepare_elf_headers(image, &elf_addr, &elf_sz);
+	ret = prepare_elf_headers(image, &kbuf.buffer, &kbuf.bufsz);
 	if (ret)
 		return ret;
 
-	image->arch.elf_headers = elf_addr;
-	image->arch.elf_headers_sz = elf_sz;
+	image->arch.elf_headers = kbuf.buffer;
+	image->arch.elf_headers_sz = kbuf.bufsz;
 
-	ret = kexec_add_buffer(image, (char *)elf_addr, elf_sz, elf_sz,
-			ELF_CORE_HEADER_ALIGN, 0, -1, 0,
-			&image->arch.elf_load_addr);
+	kbuf.memsz = kbuf.bufsz;
+	kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
+	ret = kexec_add_buffer(&kbuf);
 	if (ret) {
 		vfree((void *)image->arch.elf_headers);
 		return ret;
 	}
+	image->arch.elf_load_addr = kbuf.mem;
 	pr_debug("Loaded ELF headers at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
-		 image->arch.elf_load_addr, elf_sz, elf_sz);
+		 image->arch.elf_load_addr, kbuf.bufsz, kbuf.bufsz);
 
 	return ret;
 }
diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c
index 3407b148c240..d0a814a9d96a 100644
--- a/arch/x86/kernel/kexec-bzimage64.c
+++ b/arch/x86/kernel/kexec-bzimage64.c
@@ -331,17 +331,17 @@ static void *bzImage64_load(struct kimage *image, char *kernel,
 
 	struct setup_header *header;
 	int setup_sects, kern16_size, ret = 0;
-	unsigned long setup_header_size, params_cmdline_sz, params_misc_sz;
+	unsigned long setup_header_size, params_cmdline_sz;
 	struct boot_params *params;
 	unsigned long bootparam_load_addr, kernel_load_addr, initrd_load_addr;
 	unsigned long purgatory_load_addr;
-	unsigned long kernel_bufsz, kernel_memsz, kernel_align;
-	char *kernel_buf;
 	struct bzimage64_data *ldata;
 	struct kexec_entry64_regs regs64;
 	void *stack;
 	unsigned int setup_hdr_offset = offsetof(struct boot_params, hdr);
 	unsigned int efi_map_offset, efi_map_sz, efi_setup_data_offset;
+	struct kexec_buf kbuf = { .image = image, .buf_max = ULONG_MAX,
+				  .top_down = true };
 
 	header = (struct setup_header *)(kernel + setup_hdr_offset);
 	setup_sects = header->setup_sects;
@@ -402,11 +402,11 @@ static void *bzImage64_load(struct kimage *image, char *kernel,
 	params_cmdline_sz = sizeof(struct boot_params) + cmdline_len +
 				MAX_ELFCOREHDR_STR_LEN;
 	params_cmdline_sz = ALIGN(params_cmdline_sz, 16);
-	params_misc_sz = params_cmdline_sz + efi_map_sz +
+	kbuf.bufsz = params_cmdline_sz + efi_map_sz +
 				sizeof(struct setup_data) +
 				sizeof(struct efi_setup_data);
 
-	params = kzalloc(params_misc_sz, GFP_KERNEL);
+	params = kzalloc(kbuf.bufsz, GFP_KERNEL);
 	if (!params)
 		return ERR_PTR(-ENOMEM);
 	efi_map_offset = params_cmdline_sz;
@@ -418,37 +418,41 @@ static void *bzImage64_load(struct kimage *image, char *kernel,
 	/* Is there a limit on setup header size? */
 	memcpy(&params->hdr, (kernel + setup_hdr_offset), setup_header_size);
 
-	ret = kexec_add_buffer(image, (char *)params, params_misc_sz,
-			       params_misc_sz, 16, MIN_BOOTPARAM_ADDR,
-			       ULONG_MAX, 1, &bootparam_load_addr);
+	kbuf.buffer = params;
+	kbuf.memsz = kbuf.bufsz;
+	kbuf.buf_align = 16;
+	kbuf.buf_min = MIN_BOOTPARAM_ADDR;
+	ret = kexec_add_buffer(&kbuf);
 	if (ret)
 		goto out_free_params;
+	bootparam_load_addr = kbuf.mem;
 	pr_debug("Loaded boot_param, command line and misc at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
-		 bootparam_load_addr, params_misc_sz, params_misc_sz);
+		 bootparam_load_addr, kbuf.bufsz, kbuf.bufsz);
 
 	/* Load kernel */
-	kernel_buf = kernel + kern16_size;
-	kernel_bufsz =  kernel_len - kern16_size;
-	kernel_memsz = PAGE_ALIGN(header->init_size);
-	kernel_align = header->kernel_alignment;
-
-	ret = kexec_add_buffer(image, kernel_buf,
-			       kernel_bufsz, kernel_memsz, kernel_align,
-			       MIN_KERNEL_LOAD_ADDR, ULONG_MAX, 1,
-			       &kernel_load_addr);
+	kbuf.buffer = kernel + kern16_size;
+	kbuf.bufsz =  kernel_len - kern16_size;
+	kbuf.memsz = PAGE_ALIGN(header->init_size);
+	kbuf.buf_align = header->kernel_alignment;
+	kbuf.buf_min = MIN_KERNEL_LOAD_ADDR;
+	ret = kexec_add_buffer(&kbuf);
 	if (ret)
 		goto out_free_params;
+	kernel_load_addr = kbuf.mem;
 
 	pr_debug("Loaded 64bit kernel at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
-		 kernel_load_addr, kernel_memsz, kernel_memsz);
+		 kernel_load_addr, kbuf.bufsz, kbuf.memsz);
 
 	/* Load initrd high */
 	if (initrd) {
-		ret = kexec_add_buffer(image, initrd, initrd_len, initrd_len,
-				       PAGE_SIZE, MIN_INITRD_LOAD_ADDR,
-				       ULONG_MAX, 1, &initrd_load_addr);
+		kbuf.buffer = initrd;
+		kbuf.bufsz = kbuf.memsz = initrd_len;
+		kbuf.buf_align = PAGE_SIZE;
+		kbuf.buf_min = MIN_INITRD_LOAD_ADDR;
+		ret = kexec_add_buffer(&kbuf);
 		if (ret)
 			goto out_free_params;
+		initrd_load_addr = kbuf.mem;
 
 		pr_debug("Loaded initrd at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
 				initrd_load_addr, initrd_len, initrd_len);
diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c
index 8c1f218926d7..307b1f4543de 100644
--- a/arch/x86/kernel/machine_kexec_64.c
+++ b/arch/x86/kernel/machine_kexec_64.c
@@ -328,7 +328,7 @@ void machine_kexec(struct kimage *image)
 
 void arch_crash_save_vmcoreinfo(void)
 {
-	VMCOREINFO_SYMBOL(phys_base);
+	VMCOREINFO_NUMBER(phys_base);
 	VMCOREINFO_SYMBOL(init_level4_pgt);
 
 #ifdef CONFIG_NUMA
@@ -337,9 +337,7 @@ void arch_crash_save_vmcoreinfo(void)
 #endif
 	vmcoreinfo_append_str("KERNELOFFSET=%lx\n",
 			      kaslr_offset());
-	VMCOREINFO_PAGE_OFFSET(PAGE_OFFSET);
-	VMCOREINFO_VMALLOC_START(VMALLOC_START);
-	VMCOREINFO_VMEMMAP_START(VMEMMAP_START);
+	VMCOREINFO_NUMBER(KERNEL_IMAGE_SIZE);
 }
 
 /* arch-dependent functionality related to kexec file-based syscall */
diff --git a/arch/x86/kernel/tracepoint.c b/arch/x86/kernel/tracepoint.c
index 1c113db9ed57..15515132bf0d 100644
--- a/arch/x86/kernel/tracepoint.c
+++ b/arch/x86/kernel/tracepoint.c
@@ -34,7 +34,7 @@ static void switch_idt(void *arg)
 	local_irq_restore(flags);
 }
 
-void trace_irq_vector_regfunc(void)
+int trace_irq_vector_regfunc(void)
 {
 	mutex_lock(&irq_vector_mutex);
 	if (!trace_irq_vector_refcount) {
@@ -44,6 +44,7 @@ void trace_irq_vector_regfunc(void)
 	}
 	trace_irq_vector_refcount++;
 	mutex_unlock(&irq_vector_mutex);
+	return 0;
 }
 
 void trace_irq_vector_unregfunc(void)
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 0aefb626fa8f..b2d3cf1ef54a 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -16,6 +16,7 @@
 #include <linux/export.h>
 #include <linux/vmalloc.h>
 #include <linux/uaccess.h>
+#include <asm/processor.h>
 #include <asm/user.h>
 #include <asm/fpu/xstate.h>
 #include "cpuid.h"
@@ -64,6 +65,11 @@ u64 kvm_supported_xcr0(void)
 
 #define F(x) bit(X86_FEATURE_##x)
 
+/* These are scattered features in cpufeatures.h. */
+#define KVM_CPUID_BIT_AVX512_4VNNIW     2
+#define KVM_CPUID_BIT_AVX512_4FMAPS     3
+#define KF(x) bit(KVM_CPUID_BIT_##x)
+
 int kvm_update_cpuid(struct kvm_vcpu *vcpu)
 {
 	struct kvm_cpuid_entry2 *best;
@@ -80,6 +86,10 @@ int kvm_update_cpuid(struct kvm_vcpu *vcpu)
 			best->ecx |= F(OSXSAVE);
 	}
 
+	best->edx &= ~F(APIC);
+	if (vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE)
+		best->edx |= F(APIC);
+
 	if (apic) {
 		if (best->ecx & F(TSC_DEADLINE_TIMER))
 			apic->lapic_timer.timer_mode_mask = 3 << 17;
@@ -374,6 +384,10 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 	/* cpuid 7.0.ecx*/
 	const u32 kvm_cpuid_7_0_ecx_x86_features = F(PKU) | 0 /*OSPKE*/;
 
+	/* cpuid 7.0.edx*/
+	const u32 kvm_cpuid_7_0_edx_x86_features =
+		KF(AVX512_4VNNIW) | KF(AVX512_4FMAPS);
+
 	/* all calls to cpuid_count() should be made on the same cpu */
 	get_cpu();
 
@@ -456,12 +470,14 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 			/* PKU is not yet implemented for shadow paging. */
 			if (!tdp_enabled)
 				entry->ecx &= ~F(PKU);
+			entry->edx &= kvm_cpuid_7_0_edx_x86_features;
+			entry->edx &= get_scattered_cpuid_leaf(7, 0, CPUID_EDX);
 		} else {
 			entry->ebx = 0;
 			entry->ecx = 0;
+			entry->edx = 0;
 		}
 		entry->eax = 0;
-		entry->edx = 0;
 		break;
 	}
 	case 9:
@@ -861,17 +877,17 @@ void kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
 }
 EXPORT_SYMBOL_GPL(kvm_cpuid);
 
-void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
+int kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
 {
-	u32 function, eax, ebx, ecx, edx;
+	u32 eax, ebx, ecx, edx;
 
-	function = eax = kvm_register_read(vcpu, VCPU_REGS_RAX);
+	eax = kvm_register_read(vcpu, VCPU_REGS_RAX);
 	ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
 	kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx);
 	kvm_register_write(vcpu, VCPU_REGS_RAX, eax);
 	kvm_register_write(vcpu, VCPU_REGS_RBX, ebx);
 	kvm_register_write(vcpu, VCPU_REGS_RCX, ecx);
 	kvm_register_write(vcpu, VCPU_REGS_RDX, edx);
-	kvm_x86_ops->skip_emulated_instruction(vcpu);
+	return kvm_skip_emulated_instruction(vcpu);
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index a3ce9d260d68..56628a44668b 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -158,9 +158,11 @@
 #define Src2GS      (OpGS << Src2Shift)
 #define Src2Mask    (OpMask << Src2Shift)
 #define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
+#define AlignMask   ((u64)7 << 41)
 #define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
-#define Unaligned   ((u64)1 << 42)  /* Explicitly unaligned (e.g. MOVDQU) */
-#define Avx         ((u64)1 << 43)  /* Advanced Vector Extensions */
+#define Unaligned   ((u64)2 << 41)  /* Explicitly unaligned (e.g. MOVDQU) */
+#define Avx         ((u64)3 << 41)  /* Advanced Vector Extensions */
+#define Aligned16   ((u64)4 << 41)  /* Aligned to 16 byte boundary (e.g. FXSAVE) */
 #define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
 #define NoWrite     ((u64)1 << 45)  /* No writeback */
 #define SrcWrite    ((u64)1 << 46)  /* Write back src operand */
@@ -446,6 +448,26 @@ FOP_END;
 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
 FOP_END;
 
+/*
+ * XXX: inoutclob user must know where the argument is being expanded.
+ *      Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
+ */
+#define asm_safe(insn, inoutclob...) \
+({ \
+	int _fault = 0; \
+ \
+	asm volatile("1:" insn "\n" \
+	             "2:\n" \
+	             ".pushsection .fixup, \"ax\"\n" \
+	             "3: movl $1, %[_fault]\n" \
+	             "   jmp  2b\n" \
+	             ".popsection\n" \
+	             _ASM_EXTABLE(1b, 3b) \
+	             : [_fault] "+qm"(_fault) inoutclob ); \
+ \
+	_fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
+})
+
 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
 				    enum x86_intercept intercept,
 				    enum x86_intercept_stage stage)
@@ -632,21 +654,26 @@ static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
  * depending on whether they're AVX encoded or not.
  *
  * Also included is CMPXCHG16B which is not a vector instruction, yet it is
- * subject to the same check.
+ * subject to the same check.  FXSAVE and FXRSTOR are checked here too as their
+ * 512 bytes of data must be aligned to a 16 byte boundary.
  */
-static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
+static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
 {
-	if (likely(size < 16))
-		return false;
+	u64 alignment = ctxt->d & AlignMask;
 
-	if (ctxt->d & Aligned)
-		return true;
-	else if (ctxt->d & Unaligned)
-		return false;
-	else if (ctxt->d & Avx)
-		return false;
-	else
-		return true;
+	if (likely(size < 16))
+		return 1;
+
+	switch (alignment) {
+	case Unaligned:
+	case Avx:
+		return 1;
+	case Aligned16:
+		return 16;
+	case Aligned:
+	default:
+		return size;
+	}
 }
 
 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
@@ -704,7 +731,7 @@ static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
 		}
 		break;
 	}
-	if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
+	if (la & (insn_alignment(ctxt, size) - 1))
 		return emulate_gp(ctxt, 0);
 	return X86EMUL_CONTINUE;
 bad:
@@ -3842,6 +3869,131 @@ static int em_movsxd(struct x86_emulate_ctxt *ctxt)
 	return X86EMUL_CONTINUE;
 }
 
+static int check_fxsr(struct x86_emulate_ctxt *ctxt)
+{
+	u32 eax = 1, ebx, ecx = 0, edx;
+
+	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
+	if (!(edx & FFL(FXSR)))
+		return emulate_ud(ctxt);
+
+	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
+		return emulate_nm(ctxt);
+
+	/*
+	 * Don't emulate a case that should never be hit, instead of working
+	 * around a lack of fxsave64/fxrstor64 on old compilers.
+	 */
+	if (ctxt->mode >= X86EMUL_MODE_PROT64)
+		return X86EMUL_UNHANDLEABLE;
+
+	return X86EMUL_CONTINUE;
+}
+
+/*
+ * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
+ *  1) 16 bit mode
+ *  2) 32 bit mode
+ *     - like (1), but FIP and FDP (foo) are only 16 bit.  At least Intel CPUs
+ *       preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
+ *       save and restore
+ *  3) 64-bit mode with REX.W prefix
+ *     - like (2), but XMM 8-15 are being saved and restored
+ *  4) 64-bit mode without REX.W prefix
+ *     - like (3), but FIP and FDP are 64 bit
+ *
+ * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
+ * desired result.  (4) is not emulated.
+ *
+ * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
+ * and FPU DS) should match.
+ */
+static int em_fxsave(struct x86_emulate_ctxt *ctxt)
+{
+	struct fxregs_state fx_state;
+	size_t size;
+	int rc;
+
+	rc = check_fxsr(ctxt);
+	if (rc != X86EMUL_CONTINUE)
+		return rc;
+
+	ctxt->ops->get_fpu(ctxt);
+
+	rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
+
+	ctxt->ops->put_fpu(ctxt);
+
+	if (rc != X86EMUL_CONTINUE)
+		return rc;
+
+	if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR)
+		size = offsetof(struct fxregs_state, xmm_space[8 * 16/4]);
+	else
+		size = offsetof(struct fxregs_state, xmm_space[0]);
+
+	return segmented_write(ctxt, ctxt->memop.addr.mem, &fx_state, size);
+}
+
+static int fxrstor_fixup(struct x86_emulate_ctxt *ctxt,
+		struct fxregs_state *new)
+{
+	int rc = X86EMUL_CONTINUE;
+	struct fxregs_state old;
+
+	rc = asm_safe("fxsave %[fx]", , [fx] "+m"(old));
+	if (rc != X86EMUL_CONTINUE)
+		return rc;
+
+	/*
+	 * 64 bit host will restore XMM 8-15, which is not correct on non-64
+	 * bit guests.  Load the current values in order to preserve 64 bit
+	 * XMMs after fxrstor.
+	 */
+#ifdef CONFIG_X86_64
+	/* XXX: accessing XMM 8-15 very awkwardly */
+	memcpy(&new->xmm_space[8 * 16/4], &old.xmm_space[8 * 16/4], 8 * 16);
+#endif
+
+	/*
+	 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but
+	 * does save and restore MXCSR.
+	 */
+	if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))
+		memcpy(new->xmm_space, old.xmm_space, 8 * 16);
+
+	return rc;
+}
+
+static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
+{
+	struct fxregs_state fx_state;
+	int rc;
+
+	rc = check_fxsr(ctxt);
+	if (rc != X86EMUL_CONTINUE)
+		return rc;
+
+	rc = segmented_read(ctxt, ctxt->memop.addr.mem, &fx_state, 512);
+	if (rc != X86EMUL_CONTINUE)
+		return rc;
+
+	if (fx_state.mxcsr >> 16)
+		return emulate_gp(ctxt, 0);
+
+	ctxt->ops->get_fpu(ctxt);
+
+	if (ctxt->mode < X86EMUL_MODE_PROT64)
+		rc = fxrstor_fixup(ctxt, &fx_state);
+
+	if (rc == X86EMUL_CONTINUE)
+		rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
+
+	ctxt->ops->put_fpu(ctxt);
+
+	return rc;
+}
+
 static bool valid_cr(int nr)
 {
 	switch (nr) {
@@ -4194,7 +4346,9 @@ static const struct gprefix pfx_0f_ae_7 = {
 };
 
 static const struct group_dual group15 = { {
-	N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
+	I(ModRM | Aligned16, em_fxsave),
+	I(ModRM | Aligned16, em_fxrstor),
+	N, N, N, N, N, GP(0, &pfx_0f_ae_7),
 }, {
 	N, N, N, N, N, N, N, N,
 } };
@@ -5066,21 +5220,13 @@ static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
 
 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
 {
-	bool fault = false;
+	int rc;
 
 	ctxt->ops->get_fpu(ctxt);
-	asm volatile("1: fwait \n\t"
-		     "2: \n\t"
-		     ".pushsection .fixup,\"ax\" \n\t"
-		     "3: \n\t"
-		     "movb $1, %[fault] \n\t"
-		     "jmp 2b \n\t"
-		     ".popsection \n\t"
-		     _ASM_EXTABLE(1b, 3b)
-		     : [fault]"+qm"(fault));
+	rc = asm_safe("fwait");
 	ctxt->ops->put_fpu(ctxt);
 
-	if (unlikely(fault))
+	if (unlikely(rc != X86EMUL_CONTINUE))
 		return emulate_exception(ctxt, MF_VECTOR, 0, false);
 
 	return X86EMUL_CONTINUE;
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index 42b1c83741c8..99cde5220e07 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -291,7 +291,7 @@ static int synic_get_msr(struct kvm_vcpu_hv_synic *synic, u32 msr, u64 *pdata)
 	return ret;
 }
 
-int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint)
+static int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint)
 {
 	struct kvm_vcpu *vcpu = synic_to_vcpu(synic);
 	struct kvm_lapic_irq irq;
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
index 16a7134eedac..a78b445ce411 100644
--- a/arch/x86/kvm/i8254.c
+++ b/arch/x86/kvm/i8254.c
@@ -212,7 +212,7 @@ static void kvm_pit_ack_irq(struct kvm_irq_ack_notifier *kian)
 	 */
 	smp_mb();
 	if (atomic_dec_if_positive(&ps->pending) > 0)
-		kthread_queue_work(&pit->worker, &pit->expired);
+		kthread_queue_work(pit->worker, &pit->expired);
 }
 
 void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu)
@@ -272,7 +272,7 @@ static enum hrtimer_restart pit_timer_fn(struct hrtimer *data)
 	if (atomic_read(&ps->reinject))
 		atomic_inc(&ps->pending);
 
-	kthread_queue_work(&pt->worker, &pt->expired);
+	kthread_queue_work(pt->worker, &pt->expired);
 
 	if (ps->is_periodic) {
 		hrtimer_add_expires_ns(&ps->timer, ps->period);
@@ -667,10 +667,8 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags)
 	pid_nr = pid_vnr(pid);
 	put_pid(pid);
 
-	kthread_init_worker(&pit->worker);
-	pit->worker_task = kthread_run(kthread_worker_fn, &pit->worker,
-				       "kvm-pit/%d", pid_nr);
-	if (IS_ERR(pit->worker_task))
+	pit->worker = kthread_create_worker(0, "kvm-pit/%d", pid_nr);
+	if (IS_ERR(pit->worker))
 		goto fail_kthread;
 
 	kthread_init_work(&pit->expired, pit_do_work);
@@ -713,7 +711,7 @@ fail_register_speaker:
 fail_register_pit:
 	mutex_unlock(&kvm->slots_lock);
 	kvm_pit_set_reinject(pit, false);
-	kthread_stop(pit->worker_task);
+	kthread_destroy_worker(pit->worker);
 fail_kthread:
 	kvm_free_irq_source_id(kvm, pit->irq_source_id);
 fail_request:
@@ -730,8 +728,7 @@ void kvm_free_pit(struct kvm *kvm)
 		kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS, &pit->speaker_dev);
 		kvm_pit_set_reinject(pit, false);
 		hrtimer_cancel(&pit->pit_state.timer);
-		kthread_flush_work(&pit->expired);
-		kthread_stop(pit->worker_task);
+		kthread_destroy_worker(pit->worker);
 		kvm_free_irq_source_id(kvm, pit->irq_source_id);
 		kfree(pit);
 	}
diff --git a/arch/x86/kvm/i8254.h b/arch/x86/kvm/i8254.h
index 2f5af0798326..600bee9dcbbd 100644
--- a/arch/x86/kvm/i8254.h
+++ b/arch/x86/kvm/i8254.h
@@ -44,8 +44,7 @@ struct kvm_pit {
 	struct kvm_kpit_state pit_state;
 	int irq_source_id;
 	struct kvm_irq_mask_notifier mask_notifier;
-	struct kthread_worker worker;
-	struct task_struct *worker_task;
+	struct kthread_worker *worker;
 	struct kthread_work expired;
 };
 
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 6f69340f9fa3..34a66b2d47e6 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -342,9 +342,11 @@ void __kvm_apic_update_irr(u32 *pir, void *regs)
 	u32 i, pir_val;
 
 	for (i = 0; i <= 7; i++) {
-		pir_val = xchg(&pir[i], 0);
-		if (pir_val)
+		pir_val = READ_ONCE(pir[i]);
+		if (pir_val) {
+			pir_val = xchg(&pir[i], 0);
 			*((u32 *)(regs + APIC_IRR + i * 0x10)) |= pir_val;
+		}
 	}
 }
 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
@@ -1090,7 +1092,7 @@ static void apic_send_ipi(struct kvm_lapic *apic)
 
 static u32 apic_get_tmcct(struct kvm_lapic *apic)
 {
-	ktime_t remaining;
+	ktime_t remaining, now;
 	s64 ns;
 	u32 tmcct;
 
@@ -1101,7 +1103,8 @@ static u32 apic_get_tmcct(struct kvm_lapic *apic)
 		apic->lapic_timer.period == 0)
 		return 0;
 
-	remaining = hrtimer_get_remaining(&apic->lapic_timer.timer);
+	now = ktime_get();
+	remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
 	if (ktime_to_ns(remaining) < 0)
 		remaining = ktime_set(0, 0);
 
@@ -1332,7 +1335,7 @@ static void start_sw_tscdeadline(struct kvm_lapic *apic)
 
 	local_irq_save(flags);
 
-	now = apic->lapic_timer.timer.base->get_time();
+	now = ktime_get();
 	guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
 	if (likely(tscdeadline > guest_tsc)) {
 		ns = (tscdeadline - guest_tsc) * 1000000ULL;
@@ -1347,6 +1350,79 @@ static void start_sw_tscdeadline(struct kvm_lapic *apic)
 	local_irq_restore(flags);
 }
 
+static void start_sw_period(struct kvm_lapic *apic)
+{
+	if (!apic->lapic_timer.period)
+		return;
+
+	if (apic_lvtt_oneshot(apic) &&
+	    ktime_after(ktime_get(),
+			apic->lapic_timer.target_expiration)) {
+		apic_timer_expired(apic);
+		return;
+	}
+
+	hrtimer_start(&apic->lapic_timer.timer,
+		apic->lapic_timer.target_expiration,
+		HRTIMER_MODE_ABS_PINNED);
+}
+
+static bool set_target_expiration(struct kvm_lapic *apic)
+{
+	ktime_t now;
+	u64 tscl = rdtsc();
+
+	now = ktime_get();
+	apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
+		* APIC_BUS_CYCLE_NS * apic->divide_count;
+
+	if (!apic->lapic_timer.period)
+		return false;
+
+	/*
+	 * Do not allow the guest to program periodic timers with small
+	 * interval, since the hrtimers are not throttled by the host
+	 * scheduler.
+	 */
+	if (apic_lvtt_period(apic)) {
+		s64 min_period = min_timer_period_us * 1000LL;
+
+		if (apic->lapic_timer.period < min_period) {
+			pr_info_ratelimited(
+			    "kvm: vcpu %i: requested %lld ns "
+			    "lapic timer period limited to %lld ns\n",
+			    apic->vcpu->vcpu_id,
+			    apic->lapic_timer.period, min_period);
+			apic->lapic_timer.period = min_period;
+		}
+	}
+
+	apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016"
+		   PRIx64 ", "
+		   "timer initial count 0x%x, period %lldns, "
+		   "expire @ 0x%016" PRIx64 ".\n", __func__,
+		   APIC_BUS_CYCLE_NS, ktime_to_ns(now),
+		   kvm_lapic_get_reg(apic, APIC_TMICT),
+		   apic->lapic_timer.period,
+		   ktime_to_ns(ktime_add_ns(now,
+				apic->lapic_timer.period)));
+
+	apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
+		nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
+	apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period);
+
+	return true;
+}
+
+static void advance_periodic_target_expiration(struct kvm_lapic *apic)
+{
+	apic->lapic_timer.tscdeadline +=
+		nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
+	apic->lapic_timer.target_expiration =
+		ktime_add_ns(apic->lapic_timer.target_expiration,
+				apic->lapic_timer.period);
+}
+
 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
 {
 	if (!lapic_in_kernel(vcpu))
@@ -1356,52 +1432,59 @@ bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
 }
 EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);
 
-static void cancel_hv_tscdeadline(struct kvm_lapic *apic)
+static void cancel_hv_timer(struct kvm_lapic *apic)
 {
 	kvm_x86_ops->cancel_hv_timer(apic->vcpu);
 	apic->lapic_timer.hv_timer_in_use = false;
 }
 
-void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
-{
-	struct kvm_lapic *apic = vcpu->arch.apic;
-
-	WARN_ON(!apic->lapic_timer.hv_timer_in_use);
-	WARN_ON(swait_active(&vcpu->wq));
-	cancel_hv_tscdeadline(apic);
-	apic_timer_expired(apic);
-}
-EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
-
-static bool start_hv_tscdeadline(struct kvm_lapic *apic)
+static bool start_hv_timer(struct kvm_lapic *apic)
 {
 	u64 tscdeadline = apic->lapic_timer.tscdeadline;
 
-	if (atomic_read(&apic->lapic_timer.pending) ||
+	if ((atomic_read(&apic->lapic_timer.pending) &&
+		!apic_lvtt_period(apic)) ||
 		kvm_x86_ops->set_hv_timer(apic->vcpu, tscdeadline)) {
 		if (apic->lapic_timer.hv_timer_in_use)
-			cancel_hv_tscdeadline(apic);
+			cancel_hv_timer(apic);
 	} else {
 		apic->lapic_timer.hv_timer_in_use = true;
 		hrtimer_cancel(&apic->lapic_timer.timer);
 
 		/* In case the sw timer triggered in the window */
-		if (atomic_read(&apic->lapic_timer.pending))
-			cancel_hv_tscdeadline(apic);
+		if (atomic_read(&apic->lapic_timer.pending) &&
+			!apic_lvtt_period(apic))
+			cancel_hv_timer(apic);
 	}
 	trace_kvm_hv_timer_state(apic->vcpu->vcpu_id,
 			apic->lapic_timer.hv_timer_in_use);
 	return apic->lapic_timer.hv_timer_in_use;
 }
 
+void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
+{
+	struct kvm_lapic *apic = vcpu->arch.apic;
+
+	WARN_ON(!apic->lapic_timer.hv_timer_in_use);
+	WARN_ON(swait_active(&vcpu->wq));
+	cancel_hv_timer(apic);
+	apic_timer_expired(apic);
+
+	if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
+		advance_periodic_target_expiration(apic);
+		if (!start_hv_timer(apic))
+			start_sw_period(apic);
+	}
+}
+EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
+
 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
 
 	WARN_ON(apic->lapic_timer.hv_timer_in_use);
 
-	if (apic_lvtt_tscdeadline(apic))
-		start_hv_tscdeadline(apic);
+	start_hv_timer(apic);
 }
 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);
 
@@ -1413,62 +1496,28 @@ void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
 	if (!apic->lapic_timer.hv_timer_in_use)
 		return;
 
-	cancel_hv_tscdeadline(apic);
+	cancel_hv_timer(apic);
 
 	if (atomic_read(&apic->lapic_timer.pending))
 		return;
 
-	start_sw_tscdeadline(apic);
+	if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
+		start_sw_period(apic);
+	else if (apic_lvtt_tscdeadline(apic))
+		start_sw_tscdeadline(apic);
 }
 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
 
 static void start_apic_timer(struct kvm_lapic *apic)
 {
-	ktime_t now;
-
 	atomic_set(&apic->lapic_timer.pending, 0);
 
 	if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) {
-		/* lapic timer in oneshot or periodic mode */
-		now = apic->lapic_timer.timer.base->get_time();
-		apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
-			    * APIC_BUS_CYCLE_NS * apic->divide_count;
-
-		if (!apic->lapic_timer.period)
-			return;
-		/*
-		 * Do not allow the guest to program periodic timers with small
-		 * interval, since the hrtimers are not throttled by the host
-		 * scheduler.
-		 */
-		if (apic_lvtt_period(apic)) {
-			s64 min_period = min_timer_period_us * 1000LL;
-
-			if (apic->lapic_timer.period < min_period) {
-				pr_info_ratelimited(
-				    "kvm: vcpu %i: requested %lld ns "
-				    "lapic timer period limited to %lld ns\n",
-				    apic->vcpu->vcpu_id,
-				    apic->lapic_timer.period, min_period);
-				apic->lapic_timer.period = min_period;
-			}
-		}
-
-		hrtimer_start(&apic->lapic_timer.timer,
-			      ktime_add_ns(now, apic->lapic_timer.period),
-			      HRTIMER_MODE_ABS_PINNED);
-
-		apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016"
-			   PRIx64 ", "
-			   "timer initial count 0x%x, period %lldns, "
-			   "expire @ 0x%016" PRIx64 ".\n", __func__,
-			   APIC_BUS_CYCLE_NS, ktime_to_ns(now),
-			   kvm_lapic_get_reg(apic, APIC_TMICT),
-			   apic->lapic_timer.period,
-			   ktime_to_ns(ktime_add_ns(now,
-					apic->lapic_timer.period)));
+		if (set_target_expiration(apic) &&
+			!(kvm_x86_ops->set_hv_timer && start_hv_timer(apic)))
+			start_sw_period(apic);
 	} else if (apic_lvtt_tscdeadline(apic)) {
-		if (!(kvm_x86_ops->set_hv_timer && start_hv_tscdeadline(apic)))
+		if (!(kvm_x86_ops->set_hv_timer && start_hv_timer(apic)))
 			start_sw_tscdeadline(apic);
 	}
 }
@@ -1701,13 +1750,22 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu)
  * LAPIC interface
  *----------------------------------------------------------------------
  */
+u64 kvm_get_lapic_target_expiration_tsc(struct kvm_vcpu *vcpu)
+{
+	struct kvm_lapic *apic = vcpu->arch.apic;
+
+	if (!lapic_in_kernel(vcpu))
+		return 0;
+
+	return apic->lapic_timer.tscdeadline;
+}
 
 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
 
-	if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) ||
-			apic_lvtt_period(apic))
+	if (!lapic_in_kernel(vcpu) ||
+		!apic_lvtt_tscdeadline(apic))
 		return 0;
 
 	return apic->lapic_timer.tscdeadline;
@@ -1748,14 +1806,17 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
 	u64 old_value = vcpu->arch.apic_base;
 	struct kvm_lapic *apic = vcpu->arch.apic;
 
-	if (!apic) {
+	if (!apic)
 		value |= MSR_IA32_APICBASE_BSP;
-		vcpu->arch.apic_base = value;
-		return;
-	}
 
 	vcpu->arch.apic_base = value;
 
+	if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
+		kvm_update_cpuid(vcpu);
+
+	if (!apic)
+		return;
+
 	/* update jump label if enable bit changes */
 	if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
 		if (value & MSR_IA32_APICBASE_ENABLE) {
@@ -1909,6 +1970,7 @@ static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
 	apic_timer_expired(apic);
 
 	if (lapic_is_periodic(apic)) {
+		advance_periodic_target_expiration(apic);
 		hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
 		return HRTIMER_RESTART;
 	} else
@@ -1993,6 +2055,10 @@ void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
 		kvm_apic_local_deliver(apic, APIC_LVTT);
 		if (apic_lvtt_tscdeadline(apic))
 			apic->lapic_timer.tscdeadline = 0;
+		if (apic_lvtt_oneshot(apic)) {
+			apic->lapic_timer.tscdeadline = 0;
+			apic->lapic_timer.target_expiration = ktime_set(0, 0);
+		}
 		atomic_set(&apic->lapic_timer.pending, 0);
 	}
 }
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index f60d01c29d51..e0c80233b3e1 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -15,6 +15,7 @@
 struct kvm_timer {
 	struct hrtimer timer;
 	s64 period; 				/* unit: ns */
+	ktime_t target_expiration;
 	u32 timer_mode;
 	u32 timer_mode_mask;
 	u64 tscdeadline;
@@ -85,6 +86,7 @@ int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s);
 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s);
 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu);
 
+u64 kvm_get_lapic_target_expiration_tsc(struct kvm_vcpu *vcpu);
 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu);
 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data);
 
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index d9c7e986b4e4..7012de4a1fed 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -1660,17 +1660,9 @@ int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
 	 * This has some overhead, but not as much as the cost of swapping
 	 * out actively used pages or breaking up actively used hugepages.
 	 */
-	if (!shadow_accessed_mask) {
-		/*
-		 * We are holding the kvm->mmu_lock, and we are blowing up
-		 * shadow PTEs. MMU notifier consumers need to be kept at bay.
-		 * This is correct as long as we don't decouple the mmu_lock
-		 * protected regions (like invalidate_range_start|end does).
-		 */
-		kvm->mmu_notifier_seq++;
+	if (!shadow_accessed_mask)
 		return kvm_handle_hva_range(kvm, start, end, 0,
 					    kvm_unmap_rmapp);
-	}
 
 	return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
 }
@@ -4405,7 +4397,8 @@ static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
 }
 
 static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
-			      const u8 *new, int bytes)
+			      const u8 *new, int bytes,
+			      struct kvm_page_track_notifier_node *node)
 {
 	gfn_t gfn = gpa >> PAGE_SHIFT;
 	struct kvm_mmu_page *sp;
@@ -4508,7 +4501,7 @@ static void make_mmu_pages_available(struct kvm_vcpu *vcpu)
 	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);
 }
 
-int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
+int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u64 error_code,
 		       void *insn, int insn_len)
 {
 	int r, emulation_type = EMULTYPE_RETRY;
@@ -4527,12 +4520,28 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code,
 			return r;
 	}
 
-	r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code, false);
+	r = vcpu->arch.mmu.page_fault(vcpu, cr2, lower_32_bits(error_code),
+				      false);
 	if (r < 0)
 		return r;
 	if (!r)
 		return 1;
 
+	/*
+	 * Before emulating the instruction, check if the error code
+	 * was due to a RO violation while translating the guest page.
+	 * This can occur when using nested virtualization with nested
+	 * paging in both guests. If true, we simply unprotect the page
+	 * and resume the guest.
+	 *
+	 * Note: AMD only (since it supports the PFERR_GUEST_PAGE_MASK used
+	 *       in PFERR_NEXT_GUEST_PAGE)
+	 */
+	if (error_code == PFERR_NESTED_GUEST_PAGE) {
+		kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2));
+		return 1;
+	}
+
 	if (mmio_info_in_cache(vcpu, cr2, direct))
 		emulation_type = 0;
 emulate:
@@ -4617,11 +4626,19 @@ void kvm_mmu_setup(struct kvm_vcpu *vcpu)
 	init_kvm_mmu(vcpu);
 }
 
+static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm,
+			struct kvm_memory_slot *slot,
+			struct kvm_page_track_notifier_node *node)
+{
+	kvm_mmu_invalidate_zap_all_pages(kvm);
+}
+
 void kvm_mmu_init_vm(struct kvm *kvm)
 {
 	struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
 
 	node->track_write = kvm_mmu_pte_write;
+	node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot;
 	kvm_page_track_register_notifier(kvm, node);
 }
 
@@ -4958,7 +4975,7 @@ void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, struct kvm_memslots *slots)
 	 * zap all shadow pages.
 	 */
 	if (unlikely((slots->generation & MMIO_GEN_MASK) == 0)) {
-		printk_ratelimited(KERN_DEBUG "kvm: zapping shadow pages for mmio generation wraparound\n");
+		kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n");
 		kvm_mmu_invalidate_zap_all_pages(kvm);
 	}
 }
diff --git a/arch/x86/kvm/page_track.c b/arch/x86/kvm/page_track.c
index b431539c3714..4a1c13eaa518 100644
--- a/arch/x86/kvm/page_track.c
+++ b/arch/x86/kvm/page_track.c
@@ -106,6 +106,7 @@ void kvm_slot_page_track_add_page(struct kvm *kvm,
 		if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn))
 			kvm_flush_remote_tlbs(kvm);
 }
+EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page);
 
 /*
  * remove the guest page from the tracking pool which stops the interception
@@ -135,6 +136,7 @@ void kvm_slot_page_track_remove_page(struct kvm *kvm,
 	 */
 	kvm_mmu_gfn_allow_lpage(slot, gfn);
 }
+EXPORT_SYMBOL_GPL(kvm_slot_page_track_remove_page);
 
 /*
  * check if the corresponding access on the specified guest page is tracked.
@@ -181,6 +183,7 @@ kvm_page_track_register_notifier(struct kvm *kvm,
 	hlist_add_head_rcu(&n->node, &head->track_notifier_list);
 	spin_unlock(&kvm->mmu_lock);
 }
+EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier);
 
 /*
  * stop receiving the event interception. It is the opposed operation of
@@ -199,6 +202,7 @@ kvm_page_track_unregister_notifier(struct kvm *kvm,
 	spin_unlock(&kvm->mmu_lock);
 	synchronize_srcu(&head->track_srcu);
 }
+EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier);
 
 /*
  * Notify the node that write access is intercepted and write emulation is
@@ -222,6 +226,31 @@ void kvm_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new,
 	idx = srcu_read_lock(&head->track_srcu);
 	hlist_for_each_entry_rcu(n, &head->track_notifier_list, node)
 		if (n->track_write)
-			n->track_write(vcpu, gpa, new, bytes);
+			n->track_write(vcpu, gpa, new, bytes, n);
+	srcu_read_unlock(&head->track_srcu, idx);
+}
+
+/*
+ * Notify the node that memory slot is being removed or moved so that it can
+ * drop write-protection for the pages in the memory slot.
+ *
+ * The node should figure out it has any write-protected pages in this slot
+ * by itself.
+ */
+void kvm_page_track_flush_slot(struct kvm *kvm, struct kvm_memory_slot *slot)
+{
+	struct kvm_page_track_notifier_head *head;
+	struct kvm_page_track_notifier_node *n;
+	int idx;
+
+	head = &kvm->arch.track_notifier_head;
+
+	if (hlist_empty(&head->track_notifier_list))
+		return;
+
+	idx = srcu_read_lock(&head->track_srcu);
+	hlist_for_each_entry_rcu(n, &head->track_notifier_list, node)
+		if (n->track_flush_slot)
+			n->track_flush_slot(kvm, slot, n);
 	srcu_read_unlock(&head->track_srcu, idx);
 }
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 8ca1eca5038d..08a4d3ab3455 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -2074,7 +2074,7 @@ static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
 static int pf_interception(struct vcpu_svm *svm)
 {
 	u64 fault_address = svm->vmcb->control.exit_info_2;
-	u32 error_code;
+	u64 error_code;
 	int r = 1;
 
 	switch (svm->apf_reason) {
@@ -2270,7 +2270,7 @@ static int io_interception(struct vcpu_svm *svm)
 	++svm->vcpu.stat.io_exits;
 	string = (io_info & SVM_IOIO_STR_MASK) != 0;
 	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
-	if (string || in)
+	if (string)
 		return emulate_instruction(vcpu, 0) == EMULATE_DONE;
 
 	port = io_info >> 16;
@@ -2278,7 +2278,8 @@ static int io_interception(struct vcpu_svm *svm)
 	svm->next_rip = svm->vmcb->control.exit_info_2;
 	skip_emulated_instruction(&svm->vcpu);
 
-	return kvm_fast_pio_out(vcpu, size, port);
+	return in ? kvm_fast_pio_in(vcpu, size, port)
+		  : kvm_fast_pio_out(vcpu, size, port);
 }
 
 static int nmi_interception(struct vcpu_svm *svm)
@@ -3150,8 +3151,7 @@ static int skinit_interception(struct vcpu_svm *svm)
 
 static int wbinvd_interception(struct vcpu_svm *svm)
 {
-	kvm_emulate_wbinvd(&svm->vcpu);
-	return 1;
+	return kvm_emulate_wbinvd(&svm->vcpu);
 }
 
 static int xsetbv_interception(struct vcpu_svm *svm)
@@ -3238,8 +3238,7 @@ static int task_switch_interception(struct vcpu_svm *svm)
 static int cpuid_interception(struct vcpu_svm *svm)
 {
 	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
-	kvm_emulate_cpuid(&svm->vcpu);
-	return 1;
+	return kvm_emulate_cpuid(&svm->vcpu);
 }
 
 static int iret_interception(struct vcpu_svm *svm)
@@ -3275,9 +3274,7 @@ static int rdpmc_interception(struct vcpu_svm *svm)
 		return emulate_on_interception(svm);
 
 	err = kvm_rdpmc(&svm->vcpu);
-	kvm_complete_insn_gp(&svm->vcpu, err);
-
-	return 1;
+	return kvm_complete_insn_gp(&svm->vcpu, err);
 }
 
 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
@@ -3374,9 +3371,7 @@ static int cr_interception(struct vcpu_svm *svm)
 		}
 		kvm_register_write(&svm->vcpu, reg, val);
 	}
-	kvm_complete_insn_gp(&svm->vcpu, err);
-
-	return 1;
+	return kvm_complete_insn_gp(&svm->vcpu, err);
 }
 
 static int dr_interception(struct vcpu_svm *svm)
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 3980da515fd0..aae43c6f2472 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -132,6 +132,22 @@ module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO);
 
 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5
 
+#define VMX_VPID_EXTENT_SUPPORTED_MASK		\
+	(VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT |	\
+	VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT |	\
+	VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT |	\
+	VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
+
+/*
+ * Hyper-V requires all of these, so mark them as supported even though
+ * they are just treated the same as all-context.
+ */
+#define VMX_VPID_EXTENT_SUPPORTED_MASK		\
+	(VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT |	\
+	VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT |	\
+	VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT |	\
+	VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
+
 /*
  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
  * ple_gap:    upper bound on the amount of time between two successive
@@ -446,23 +462,31 @@ struct nested_vmx {
 	u16 vpid02;
 	u16 last_vpid;
 
+	/*
+	 * We only store the "true" versions of the VMX capability MSRs. We
+	 * generate the "non-true" versions by setting the must-be-1 bits
+	 * according to the SDM.
+	 */
 	u32 nested_vmx_procbased_ctls_low;
 	u32 nested_vmx_procbased_ctls_high;
-	u32 nested_vmx_true_procbased_ctls_low;
 	u32 nested_vmx_secondary_ctls_low;
 	u32 nested_vmx_secondary_ctls_high;
 	u32 nested_vmx_pinbased_ctls_low;
 	u32 nested_vmx_pinbased_ctls_high;
 	u32 nested_vmx_exit_ctls_low;
 	u32 nested_vmx_exit_ctls_high;
-	u32 nested_vmx_true_exit_ctls_low;
 	u32 nested_vmx_entry_ctls_low;
 	u32 nested_vmx_entry_ctls_high;
-	u32 nested_vmx_true_entry_ctls_low;
 	u32 nested_vmx_misc_low;
 	u32 nested_vmx_misc_high;
 	u32 nested_vmx_ept_caps;
 	u32 nested_vmx_vpid_caps;
+	u64 nested_vmx_basic;
+	u64 nested_vmx_cr0_fixed0;
+	u64 nested_vmx_cr0_fixed1;
+	u64 nested_vmx_cr4_fixed0;
+	u64 nested_vmx_cr4_fixed1;
+	u64 nested_vmx_vmcs_enum;
 };
 
 #define POSTED_INTR_ON  0
@@ -520,6 +544,12 @@ static inline void pi_set_sn(struct pi_desc *pi_desc)
 			(unsigned long *)&pi_desc->control);
 }
 
+static inline void pi_clear_on(struct pi_desc *pi_desc)
+{
+	clear_bit(POSTED_INTR_ON,
+  		  (unsigned long *)&pi_desc->control);
+}
+
 static inline int pi_test_on(struct pi_desc *pi_desc)
 {
 	return test_bit(POSTED_INTR_ON,
@@ -920,16 +950,32 @@ static DEFINE_PER_CPU(struct desc_ptr, host_gdt);
 static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
 static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);
 
-static unsigned long *vmx_io_bitmap_a;
-static unsigned long *vmx_io_bitmap_b;
-static unsigned long *vmx_msr_bitmap_legacy;
-static unsigned long *vmx_msr_bitmap_longmode;
-static unsigned long *vmx_msr_bitmap_legacy_x2apic;
-static unsigned long *vmx_msr_bitmap_longmode_x2apic;
-static unsigned long *vmx_msr_bitmap_legacy_x2apic_apicv_inactive;
-static unsigned long *vmx_msr_bitmap_longmode_x2apic_apicv_inactive;
-static unsigned long *vmx_vmread_bitmap;
-static unsigned long *vmx_vmwrite_bitmap;
+enum {
+	VMX_IO_BITMAP_A,
+	VMX_IO_BITMAP_B,
+	VMX_MSR_BITMAP_LEGACY,
+	VMX_MSR_BITMAP_LONGMODE,
+	VMX_MSR_BITMAP_LEGACY_X2APIC_APICV,
+	VMX_MSR_BITMAP_LONGMODE_X2APIC_APICV,
+	VMX_MSR_BITMAP_LEGACY_X2APIC,
+	VMX_MSR_BITMAP_LONGMODE_X2APIC,
+	VMX_VMREAD_BITMAP,
+	VMX_VMWRITE_BITMAP,
+	VMX_BITMAP_NR
+};
+
+static unsigned long *vmx_bitmap[VMX_BITMAP_NR];
+
+#define vmx_io_bitmap_a                      (vmx_bitmap[VMX_IO_BITMAP_A])
+#define vmx_io_bitmap_b                      (vmx_bitmap[VMX_IO_BITMAP_B])
+#define vmx_msr_bitmap_legacy                (vmx_bitmap[VMX_MSR_BITMAP_LEGACY])
+#define vmx_msr_bitmap_longmode              (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE])
+#define vmx_msr_bitmap_legacy_x2apic_apicv   (vmx_bitmap[VMX_MSR_BITMAP_LEGACY_X2APIC_APICV])
+#define vmx_msr_bitmap_longmode_x2apic_apicv (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE_X2APIC_APICV])
+#define vmx_msr_bitmap_legacy_x2apic         (vmx_bitmap[VMX_MSR_BITMAP_LEGACY_X2APIC])
+#define vmx_msr_bitmap_longmode_x2apic       (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE_X2APIC])
+#define vmx_vmread_bitmap                    (vmx_bitmap[VMX_VMREAD_BITMAP])
+#define vmx_vmwrite_bitmap                   (vmx_bitmap[VMX_VMWRITE_BITMAP])
 
 static bool cpu_has_load_ia32_efer;
 static bool cpu_has_load_perf_global_ctrl;
@@ -2523,14 +2569,14 @@ static void vmx_set_msr_bitmap(struct kvm_vcpu *vcpu)
 		  SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) {
 		if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) {
 			if (is_long_mode(vcpu))
-				msr_bitmap = vmx_msr_bitmap_longmode_x2apic;
+				msr_bitmap = vmx_msr_bitmap_longmode_x2apic_apicv;
 			else
-				msr_bitmap = vmx_msr_bitmap_legacy_x2apic;
+				msr_bitmap = vmx_msr_bitmap_legacy_x2apic_apicv;
 		} else {
 			if (is_long_mode(vcpu))
-				msr_bitmap = vmx_msr_bitmap_longmode_x2apic_apicv_inactive;
+				msr_bitmap = vmx_msr_bitmap_longmode_x2apic;
 			else
-				msr_bitmap = vmx_msr_bitmap_legacy_x2apic_apicv_inactive;
+				msr_bitmap = vmx_msr_bitmap_legacy_x2apic;
 		}
 	} else {
 		if (is_long_mode(vcpu))
@@ -2706,9 +2752,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
 		vmx->nested.nested_vmx_exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;
 
 	/* We support free control of debug control saving. */
-	vmx->nested.nested_vmx_true_exit_ctls_low =
-		vmx->nested.nested_vmx_exit_ctls_low &
-		~VM_EXIT_SAVE_DEBUG_CONTROLS;
+	vmx->nested.nested_vmx_exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS;
 
 	/* entry controls */
 	rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
@@ -2727,9 +2771,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
 		vmx->nested.nested_vmx_entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;
 
 	/* We support free control of debug control loading. */
-	vmx->nested.nested_vmx_true_entry_ctls_low =
-		vmx->nested.nested_vmx_entry_ctls_low &
-		~VM_ENTRY_LOAD_DEBUG_CONTROLS;
+	vmx->nested.nested_vmx_entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS;
 
 	/* cpu-based controls */
 	rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
@@ -2762,8 +2804,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
 		CPU_BASED_USE_MSR_BITMAPS;
 
 	/* We support free control of CR3 access interception. */
-	vmx->nested.nested_vmx_true_procbased_ctls_low =
-		vmx->nested.nested_vmx_procbased_ctls_low &
+	vmx->nested.nested_vmx_procbased_ctls_low &=
 		~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);
 
 	/* secondary cpu-based controls */
@@ -2774,6 +2815,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
 	vmx->nested.nested_vmx_secondary_ctls_high &=
 		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
 		SECONDARY_EXEC_RDTSCP |
+		SECONDARY_EXEC_DESC |
 		SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
 		SECONDARY_EXEC_ENABLE_VPID |
 		SECONDARY_EXEC_APIC_REGISTER_VIRT |
@@ -2805,8 +2847,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
 	 */
 	if (enable_vpid)
 		vmx->nested.nested_vmx_vpid_caps = VMX_VPID_INVVPID_BIT |
-				VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT |
-				VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
+			VMX_VPID_EXTENT_SUPPORTED_MASK;
 	else
 		vmx->nested.nested_vmx_vpid_caps = 0;
 
@@ -2823,14 +2864,52 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
 		VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
 		VMX_MISC_ACTIVITY_HLT;
 	vmx->nested.nested_vmx_misc_high = 0;
+
+	/*
+	 * This MSR reports some information about VMX support. We
+	 * should return information about the VMX we emulate for the
+	 * guest, and the VMCS structure we give it - not about the
+	 * VMX support of the underlying hardware.
+	 */
+	vmx->nested.nested_vmx_basic =
+		VMCS12_REVISION |
+		VMX_BASIC_TRUE_CTLS |
+		((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
+		(VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
+
+	if (cpu_has_vmx_basic_inout())
+		vmx->nested.nested_vmx_basic |= VMX_BASIC_INOUT;
+
+	/*
+	 * These MSRs specify bits which the guest must keep fixed on
+	 * while L1 is in VMXON mode (in L1's root mode, or running an L2).
+	 * We picked the standard core2 setting.
+	 */
+#define VMXON_CR0_ALWAYSON     (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
+#define VMXON_CR4_ALWAYSON     X86_CR4_VMXE
+	vmx->nested.nested_vmx_cr0_fixed0 = VMXON_CR0_ALWAYSON;
+	vmx->nested.nested_vmx_cr4_fixed0 = VMXON_CR4_ALWAYSON;
+
+	/* These MSRs specify bits which the guest must keep fixed off. */
+	rdmsrl(MSR_IA32_VMX_CR0_FIXED1, vmx->nested.nested_vmx_cr0_fixed1);
+	rdmsrl(MSR_IA32_VMX_CR4_FIXED1, vmx->nested.nested_vmx_cr4_fixed1);
+
+	/* highest index: VMX_PREEMPTION_TIMER_VALUE */
+	vmx->nested.nested_vmx_vmcs_enum = 0x2e;
+}
+
+/*
+ * if fixed0[i] == 1: val[i] must be 1
+ * if fixed1[i] == 0: val[i] must be 0
+ */
+static inline bool fixed_bits_valid(u64 val, u64 fixed0, u64 fixed1)
+{
+	return ((val & fixed1) | fixed0) == val;
 }
 
 static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
 {
-	/*
-	 * Bits 0 in high must be 0, and bits 1 in low must be 1.
-	 */
-	return ((control & high) | low) == control;
+	return fixed_bits_valid(control, low, high);
 }
 
 static inline u64 vmx_control_msr(u32 low, u32 high)
@@ -2838,87 +2917,285 @@ static inline u64 vmx_control_msr(u32 low, u32 high)
 	return low | ((u64)high << 32);
 }
 
-/* Returns 0 on success, non-0 otherwise. */
-static int vmx_get_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
+static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask)
+{
+	superset &= mask;
+	subset &= mask;
+
+	return (superset | subset) == superset;
+}
+
+static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data)
+{
+	const u64 feature_and_reserved =
+		/* feature (except bit 48; see below) */
+		BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) |
+		/* reserved */
+		BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56);
+	u64 vmx_basic = vmx->nested.nested_vmx_basic;
+
+	if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved))
+		return -EINVAL;
+
+	/*
+	 * KVM does not emulate a version of VMX that constrains physical
+	 * addresses of VMX structures (e.g. VMCS) to 32-bits.
+	 */
+	if (data & BIT_ULL(48))
+		return -EINVAL;
+
+	if (vmx_basic_vmcs_revision_id(vmx_basic) !=
+	    vmx_basic_vmcs_revision_id(data))
+		return -EINVAL;
+
+	if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data))
+		return -EINVAL;
+
+	vmx->nested.nested_vmx_basic = data;
+	return 0;
+}
+
+static int
+vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
+{
+	u64 supported;
+	u32 *lowp, *highp;
+
+	switch (msr_index) {
+	case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
+		lowp = &vmx->nested.nested_vmx_pinbased_ctls_low;
+		highp = &vmx->nested.nested_vmx_pinbased_ctls_high;
+		break;
+	case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
+		lowp = &vmx->nested.nested_vmx_procbased_ctls_low;
+		highp = &vmx->nested.nested_vmx_procbased_ctls_high;
+		break;
+	case MSR_IA32_VMX_TRUE_EXIT_CTLS:
+		lowp = &vmx->nested.nested_vmx_exit_ctls_low;
+		highp = &vmx->nested.nested_vmx_exit_ctls_high;
+		break;
+	case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
+		lowp = &vmx->nested.nested_vmx_entry_ctls_low;
+		highp = &vmx->nested.nested_vmx_entry_ctls_high;
+		break;
+	case MSR_IA32_VMX_PROCBASED_CTLS2:
+		lowp = &vmx->nested.nested_vmx_secondary_ctls_low;
+		highp = &vmx->nested.nested_vmx_secondary_ctls_high;
+		break;
+	default:
+		BUG();
+	}
+
+	supported = vmx_control_msr(*lowp, *highp);
+
+	/* Check must-be-1 bits are still 1. */
+	if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0)))
+		return -EINVAL;
+
+	/* Check must-be-0 bits are still 0. */
+	if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32)))
+		return -EINVAL;
+
+	*lowp = data;
+	*highp = data >> 32;
+	return 0;
+}
+
+static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data)
+{
+	const u64 feature_and_reserved_bits =
+		/* feature */
+		BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) |
+		BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) |
+		/* reserved */
+		GENMASK_ULL(13, 9) | BIT_ULL(31);
+	u64 vmx_misc;
+
+	vmx_misc = vmx_control_msr(vmx->nested.nested_vmx_misc_low,
+				   vmx->nested.nested_vmx_misc_high);
+
+	if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits))
+		return -EINVAL;
+
+	if ((vmx->nested.nested_vmx_pinbased_ctls_high &
+	     PIN_BASED_VMX_PREEMPTION_TIMER) &&
+	    vmx_misc_preemption_timer_rate(data) !=
+	    vmx_misc_preemption_timer_rate(vmx_misc))
+		return -EINVAL;
+
+	if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc))
+		return -EINVAL;
+
+	if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc))
+		return -EINVAL;
+
+	if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc))
+		return -EINVAL;
+
+	vmx->nested.nested_vmx_misc_low = data;
+	vmx->nested.nested_vmx_misc_high = data >> 32;
+	return 0;
+}
+
+static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data)
+{
+	u64 vmx_ept_vpid_cap;
+
+	vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.nested_vmx_ept_caps,
+					   vmx->nested.nested_vmx_vpid_caps);
+
+	/* Every bit is either reserved or a feature bit. */
+	if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL))
+		return -EINVAL;
+
+	vmx->nested.nested_vmx_ept_caps = data;
+	vmx->nested.nested_vmx_vpid_caps = data >> 32;
+	return 0;
+}
+
+static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
+{
+	u64 *msr;
+
+	switch (msr_index) {
+	case MSR_IA32_VMX_CR0_FIXED0:
+		msr = &vmx->nested.nested_vmx_cr0_fixed0;
+		break;
+	case MSR_IA32_VMX_CR4_FIXED0:
+		msr = &vmx->nested.nested_vmx_cr4_fixed0;
+		break;
+	default:
+		BUG();
+	}
+
+	/*
+	 * 1 bits (which indicates bits which "must-be-1" during VMX operation)
+	 * must be 1 in the restored value.
+	 */
+	if (!is_bitwise_subset(data, *msr, -1ULL))
+		return -EINVAL;
+
+	*msr = data;
+	return 0;
+}
+
+/*
+ * Called when userspace is restoring VMX MSRs.
+ *
+ * Returns 0 on success, non-0 otherwise.
+ */
+static int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 
 	switch (msr_index) {
 	case MSR_IA32_VMX_BASIC:
+		return vmx_restore_vmx_basic(vmx, data);
+	case MSR_IA32_VMX_PINBASED_CTLS:
+	case MSR_IA32_VMX_PROCBASED_CTLS:
+	case MSR_IA32_VMX_EXIT_CTLS:
+	case MSR_IA32_VMX_ENTRY_CTLS:
+		/*
+		 * The "non-true" VMX capability MSRs are generated from the
+		 * "true" MSRs, so we do not support restoring them directly.
+		 *
+		 * If userspace wants to emulate VMX_BASIC[55]=0, userspace
+		 * should restore the "true" MSRs with the must-be-1 bits
+		 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND
+		 * DEFAULT SETTINGS".
+		 */
+		return -EINVAL;
+	case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
+	case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
+	case MSR_IA32_VMX_TRUE_EXIT_CTLS:
+	case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
+	case MSR_IA32_VMX_PROCBASED_CTLS2:
+		return vmx_restore_control_msr(vmx, msr_index, data);
+	case MSR_IA32_VMX_MISC:
+		return vmx_restore_vmx_misc(vmx, data);
+	case MSR_IA32_VMX_CR0_FIXED0:
+	case MSR_IA32_VMX_CR4_FIXED0:
+		return vmx_restore_fixed0_msr(vmx, msr_index, data);
+	case MSR_IA32_VMX_CR0_FIXED1:
+	case MSR_IA32_VMX_CR4_FIXED1:
+		/*
+		 * These MSRs are generated based on the vCPU's CPUID, so we
+		 * do not support restoring them directly.
+		 */
+		return -EINVAL;
+	case MSR_IA32_VMX_EPT_VPID_CAP:
+		return vmx_restore_vmx_ept_vpid_cap(vmx, data);
+	case MSR_IA32_VMX_VMCS_ENUM:
+		vmx->nested.nested_vmx_vmcs_enum = data;
+		return 0;
+	default:
 		/*
-		 * This MSR reports some information about VMX support. We
-		 * should return information about the VMX we emulate for the
-		 * guest, and the VMCS structure we give it - not about the
-		 * VMX support of the underlying hardware.
+		 * The rest of the VMX capability MSRs do not support restore.
 		 */
-		*pdata = VMCS12_REVISION | VMX_BASIC_TRUE_CTLS |
-			   ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
-			   (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
-		if (cpu_has_vmx_basic_inout())
-			*pdata |= VMX_BASIC_INOUT;
+		return -EINVAL;
+	}
+}
+
+/* Returns 0 on success, non-0 otherwise. */
+static int vmx_get_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
+{
+	struct vcpu_vmx *vmx = to_vmx(vcpu);
+
+	switch (msr_index) {
+	case MSR_IA32_VMX_BASIC:
+		*pdata = vmx->nested.nested_vmx_basic;
 		break;
 	case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
 	case MSR_IA32_VMX_PINBASED_CTLS:
 		*pdata = vmx_control_msr(
 			vmx->nested.nested_vmx_pinbased_ctls_low,
 			vmx->nested.nested_vmx_pinbased_ctls_high);
+		if (msr_index == MSR_IA32_VMX_PINBASED_CTLS)
+			*pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
 		break;
 	case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
-		*pdata = vmx_control_msr(
-			vmx->nested.nested_vmx_true_procbased_ctls_low,
-			vmx->nested.nested_vmx_procbased_ctls_high);
-		break;
 	case MSR_IA32_VMX_PROCBASED_CTLS:
 		*pdata = vmx_control_msr(
 			vmx->nested.nested_vmx_procbased_ctls_low,
 			vmx->nested.nested_vmx_procbased_ctls_high);
+		if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS)
+			*pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
 		break;
 	case MSR_IA32_VMX_TRUE_EXIT_CTLS:
-		*pdata = vmx_control_msr(
-			vmx->nested.nested_vmx_true_exit_ctls_low,
-			vmx->nested.nested_vmx_exit_ctls_high);
-		break;
 	case MSR_IA32_VMX_EXIT_CTLS:
 		*pdata = vmx_control_msr(
 			vmx->nested.nested_vmx_exit_ctls_low,
 			vmx->nested.nested_vmx_exit_ctls_high);
+		if (msr_index == MSR_IA32_VMX_EXIT_CTLS)
+			*pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
 		break;
 	case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
-		*pdata = vmx_control_msr(
-			vmx->nested.nested_vmx_true_entry_ctls_low,
-			vmx->nested.nested_vmx_entry_ctls_high);
-		break;
 	case MSR_IA32_VMX_ENTRY_CTLS:
 		*pdata = vmx_control_msr(
 			vmx->nested.nested_vmx_entry_ctls_low,
 			vmx->nested.nested_vmx_entry_ctls_high);
+		if (msr_index == MSR_IA32_VMX_ENTRY_CTLS)
+			*pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
 		break;
 	case MSR_IA32_VMX_MISC:
 		*pdata = vmx_control_msr(
 			vmx->nested.nested_vmx_misc_low,
 			vmx->nested.nested_vmx_misc_high);
 		break;
-	/*
-	 * These MSRs specify bits which the guest must keep fixed (on or off)
-	 * while L1 is in VMXON mode (in L1's root mode, or running an L2).
-	 * We picked the standard core2 setting.
-	 */
-#define VMXON_CR0_ALWAYSON	(X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
-#define VMXON_CR4_ALWAYSON	X86_CR4_VMXE
 	case MSR_IA32_VMX_CR0_FIXED0:
-		*pdata = VMXON_CR0_ALWAYSON;
+		*pdata = vmx->nested.nested_vmx_cr0_fixed0;
 		break;
 	case MSR_IA32_VMX_CR0_FIXED1:
-		*pdata = -1ULL;
+		*pdata = vmx->nested.nested_vmx_cr0_fixed1;
 		break;
 	case MSR_IA32_VMX_CR4_FIXED0:
-		*pdata = VMXON_CR4_ALWAYSON;
+		*pdata = vmx->nested.nested_vmx_cr4_fixed0;
 		break;
 	case MSR_IA32_VMX_CR4_FIXED1:
-		*pdata = -1ULL;
+		*pdata = vmx->nested.nested_vmx_cr4_fixed1;
 		break;
 	case MSR_IA32_VMX_VMCS_ENUM:
-		*pdata = 0x2e; /* highest index: VMX_PREEMPTION_TIMER_VALUE */
+		*pdata = vmx->nested.nested_vmx_vmcs_enum;
 		break;
 	case MSR_IA32_VMX_PROCBASED_CTLS2:
 		*pdata = vmx_control_msr(
@@ -3101,7 +3378,11 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 			vmx_leave_nested(vcpu);
 		break;
 	case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
-		return 1; /* they are read-only */
+		if (!msr_info->host_initiated)
+			return 1; /* they are read-only */
+		if (!nested_vmx_allowed(vcpu))
+			return 1;
+		return vmx_set_vmx_msr(vcpu, msr_index, data);
 	case MSR_IA32_XSS:
 		if (!vmx_xsaves_supported())
 			return 1;
@@ -3863,6 +4144,40 @@ static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
 		  (unsigned long *)&vcpu->arch.regs_dirty);
 }
 
+static bool nested_guest_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
+{
+	u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed0;
+	u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed1;
+	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
+
+	if (to_vmx(vcpu)->nested.nested_vmx_secondary_ctls_high &
+		SECONDARY_EXEC_UNRESTRICTED_GUEST &&
+	    nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST))
+		fixed0 &= ~(X86_CR0_PE | X86_CR0_PG);
+
+	return fixed_bits_valid(val, fixed0, fixed1);
+}
+
+static bool nested_host_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
+{
+	u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed0;
+	u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr0_fixed1;
+
+	return fixed_bits_valid(val, fixed0, fixed1);
+}
+
+static bool nested_cr4_valid(struct kvm_vcpu *vcpu, unsigned long val)
+{
+	u64 fixed0 = to_vmx(vcpu)->nested.nested_vmx_cr4_fixed0;
+	u64 fixed1 = to_vmx(vcpu)->nested.nested_vmx_cr4_fixed1;
+
+	return fixed_bits_valid(val, fixed0, fixed1);
+}
+
+/* No difference in the restrictions on guest and host CR4 in VMX operation. */
+#define nested_guest_cr4_valid	nested_cr4_valid
+#define nested_host_cr4_valid	nested_cr4_valid
+
 static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
 
 static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
@@ -3991,8 +4306,8 @@ static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 		if (!nested_vmx_allowed(vcpu))
 			return 1;
 	}
-	if (to_vmx(vcpu)->nested.vmxon &&
-	    ((cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON))
+
+	if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4))
 		return 1;
 
 	vcpu->arch.cr4 = cr4;
@@ -4569,41 +4884,6 @@ static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
 	}
 }
 
-static void __vmx_enable_intercept_for_msr(unsigned long *msr_bitmap,
-						u32 msr, int type)
-{
-	int f = sizeof(unsigned long);
-
-	if (!cpu_has_vmx_msr_bitmap())
-		return;
-
-	/*
-	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
-	 * have the write-low and read-high bitmap offsets the wrong way round.
-	 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
-	 */
-	if (msr <= 0x1fff) {
-		if (type & MSR_TYPE_R)
-			/* read-low */
-			__set_bit(msr, msr_bitmap + 0x000 / f);
-
-		if (type & MSR_TYPE_W)
-			/* write-low */
-			__set_bit(msr, msr_bitmap + 0x800 / f);
-
-	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
-		msr &= 0x1fff;
-		if (type & MSR_TYPE_R)
-			/* read-high */
-			__set_bit(msr, msr_bitmap + 0x400 / f);
-
-		if (type & MSR_TYPE_W)
-			/* write-high */
-			__set_bit(msr, msr_bitmap + 0xc00 / f);
-
-	}
-}
-
 /*
  * If a msr is allowed by L0, we should check whether it is allowed by L1.
  * The corresponding bit will be cleared unless both of L0 and L1 allow it.
@@ -4659,48 +4939,18 @@ static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
 						msr, MSR_TYPE_R | MSR_TYPE_W);
 }
 
-static void vmx_enable_intercept_msr_read_x2apic(u32 msr, bool apicv_active)
-{
-	if (apicv_active) {
-		__vmx_enable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
-				msr, MSR_TYPE_R);
-		__vmx_enable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
-				msr, MSR_TYPE_R);
-	} else {
-		__vmx_enable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic_apicv_inactive,
-				msr, MSR_TYPE_R);
-		__vmx_enable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic_apicv_inactive,
-				msr, MSR_TYPE_R);
-	}
-}
-
-static void vmx_disable_intercept_msr_read_x2apic(u32 msr, bool apicv_active)
+static void vmx_disable_intercept_msr_x2apic(u32 msr, int type, bool apicv_active)
 {
 	if (apicv_active) {
-		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
-				msr, MSR_TYPE_R);
-		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
-				msr, MSR_TYPE_R);
+		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic_apicv,
+				msr, type);
+		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic_apicv,
+				msr, type);
 	} else {
-		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic_apicv_inactive,
-				msr, MSR_TYPE_R);
-		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic_apicv_inactive,
-				msr, MSR_TYPE_R);
-	}
-}
-
-static void vmx_disable_intercept_msr_write_x2apic(u32 msr, bool apicv_active)
-{
-	if (apicv_active) {
 		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
-				msr, MSR_TYPE_W);
+				msr, type);
 		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
-				msr, MSR_TYPE_W);
-	} else {
-		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic_apicv_inactive,
-				msr, MSR_TYPE_W);
-		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic_apicv_inactive,
-				msr, MSR_TYPE_W);
+				msr, type);
 	}
 }
 
@@ -4822,9 +5072,15 @@ static void vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 
-	if (!pi_test_and_clear_on(&vmx->pi_desc))
+	if (!pi_test_on(&vmx->pi_desc))
 		return;
 
+	pi_clear_on(&vmx->pi_desc);
+	/*
+	 * IOMMU can write to PIR.ON, so the barrier matters even on UP.
+	 * But on x86 this is just a compiler barrier anyway.
+	 */
+	smp_mb__after_atomic();
 	kvm_apic_update_irr(vcpu, vmx->pi_desc.pir);
 }
 
@@ -5583,7 +5839,7 @@ static int handle_triple_fault(struct kvm_vcpu *vcpu)
 static int handle_io(struct kvm_vcpu *vcpu)
 {
 	unsigned long exit_qualification;
-	int size, in, string;
+	int size, in, string, ret;
 	unsigned port;
 
 	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
@@ -5597,9 +5853,14 @@ static int handle_io(struct kvm_vcpu *vcpu)
 
 	port = exit_qualification >> 16;
 	size = (exit_qualification & 7) + 1;
-	skip_emulated_instruction(vcpu);
 
-	return kvm_fast_pio_out(vcpu, size, port);
+	ret = kvm_skip_emulated_instruction(vcpu);
+
+	/*
+	 * TODO: we might be squashing a KVM_GUESTDBG_SINGLESTEP-triggered
+	 * KVM_EXIT_DEBUG here.
+	 */
+	return kvm_fast_pio_out(vcpu, size, port) && ret;
 }
 
 static void
@@ -5613,18 +5874,6 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
 	hypercall[2] = 0xc1;
 }
 
-static bool nested_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
-{
-	unsigned long always_on = VMXON_CR0_ALWAYSON;
-	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
-
-	if (to_vmx(vcpu)->nested.nested_vmx_secondary_ctls_high &
-		SECONDARY_EXEC_UNRESTRICTED_GUEST &&
-	    nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST))
-		always_on &= ~(X86_CR0_PE | X86_CR0_PG);
-	return (val & always_on) == always_on;
-}
-
 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */
 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
 {
@@ -5643,7 +5892,7 @@ static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
 		val = (val & ~vmcs12->cr0_guest_host_mask) |
 			(vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask);
 
-		if (!nested_cr0_valid(vcpu, val))
+		if (!nested_guest_cr0_valid(vcpu, val))
 			return 1;
 
 		if (kvm_set_cr0(vcpu, val))
@@ -5652,8 +5901,9 @@ static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
 		return 0;
 	} else {
 		if (to_vmx(vcpu)->nested.vmxon &&
-		    ((val & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON))
+		    !nested_host_cr0_valid(vcpu, val))
 			return 1;
+
 		return kvm_set_cr0(vcpu, val);
 	}
 }
@@ -5697,6 +5947,7 @@ static int handle_cr(struct kvm_vcpu *vcpu)
 	int cr;
 	int reg;
 	int err;
+	int ret;
 
 	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
 	cr = exit_qualification & 15;
@@ -5708,25 +5959,27 @@ static int handle_cr(struct kvm_vcpu *vcpu)
 		switch (cr) {
 		case 0:
 			err = handle_set_cr0(vcpu, val);
-			kvm_complete_insn_gp(vcpu, err);
-			return 1;
+			return kvm_complete_insn_gp(vcpu, err);
 		case 3:
 			err = kvm_set_cr3(vcpu, val);
-			kvm_complete_insn_gp(vcpu, err);
-			return 1;
+			return kvm_complete_insn_gp(vcpu, err);
 		case 4:
 			err = handle_set_cr4(vcpu, val);
-			kvm_complete_insn_gp(vcpu, err);
-			return 1;
+			return kvm_complete_insn_gp(vcpu, err);
 		case 8: {
 				u8 cr8_prev = kvm_get_cr8(vcpu);
 				u8 cr8 = (u8)val;
 				err = kvm_set_cr8(vcpu, cr8);
-				kvm_complete_insn_gp(vcpu, err);
+				ret = kvm_complete_insn_gp(vcpu, err);
 				if (lapic_in_kernel(vcpu))
-					return 1;
+					return ret;
 				if (cr8_prev <= cr8)
-					return 1;
+					return ret;
+				/*
+				 * TODO: we might be squashing a
+				 * KVM_GUESTDBG_SINGLESTEP-triggered
+				 * KVM_EXIT_DEBUG here.
+				 */
 				vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
 				return 0;
 			}
@@ -5735,23 +5988,20 @@ static int handle_cr(struct kvm_vcpu *vcpu)
 	case 2: /* clts */
 		handle_clts(vcpu);
 		trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
-		skip_emulated_instruction(vcpu);
 		vmx_fpu_activate(vcpu);
-		return 1;
+		return kvm_skip_emulated_instruction(vcpu);
 	case 1: /*mov from cr*/
 		switch (cr) {
 		case 3:
 			val = kvm_read_cr3(vcpu);
 			kvm_register_write(vcpu, reg, val);
 			trace_kvm_cr_read(cr, val);
-			skip_emulated_instruction(vcpu);
-			return 1;
+			return kvm_skip_emulated_instruction(vcpu);
 		case 8:
 			val = kvm_get_cr8(vcpu);
 			kvm_register_write(vcpu, reg, val);
 			trace_kvm_cr_read(cr, val);
-			skip_emulated_instruction(vcpu);
-			return 1;
+			return kvm_skip_emulated_instruction(vcpu);
 		}
 		break;
 	case 3: /* lmsw */
@@ -5759,8 +6009,7 @@ static int handle_cr(struct kvm_vcpu *vcpu)
 		trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
 		kvm_lmsw(vcpu, val);
 
-		skip_emulated_instruction(vcpu);
-		return 1;
+		return kvm_skip_emulated_instruction(vcpu);
 	default:
 		break;
 	}
@@ -5831,8 +6080,7 @@ static int handle_dr(struct kvm_vcpu *vcpu)
 		if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)))
 			return 1;
 
-	skip_emulated_instruction(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 static u64 vmx_get_dr6(struct kvm_vcpu *vcpu)
@@ -5864,8 +6112,7 @@ static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
 
 static int handle_cpuid(struct kvm_vcpu *vcpu)
 {
-	kvm_emulate_cpuid(vcpu);
-	return 1;
+	return kvm_emulate_cpuid(vcpu);
 }
 
 static int handle_rdmsr(struct kvm_vcpu *vcpu)
@@ -5886,8 +6133,7 @@ static int handle_rdmsr(struct kvm_vcpu *vcpu)
 	/* FIXME: handling of bits 32:63 of rax, rdx */
 	vcpu->arch.regs[VCPU_REGS_RAX] = msr_info.data & -1u;
 	vcpu->arch.regs[VCPU_REGS_RDX] = (msr_info.data >> 32) & -1u;
-	skip_emulated_instruction(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 static int handle_wrmsr(struct kvm_vcpu *vcpu)
@@ -5907,8 +6153,7 @@ static int handle_wrmsr(struct kvm_vcpu *vcpu)
 	}
 
 	trace_kvm_msr_write(ecx, data);
-	skip_emulated_instruction(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
@@ -5952,8 +6197,7 @@ static int handle_invlpg(struct kvm_vcpu *vcpu)
 	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
 
 	kvm_mmu_invlpg(vcpu, exit_qualification);
-	skip_emulated_instruction(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 static int handle_rdpmc(struct kvm_vcpu *vcpu)
@@ -5961,15 +6205,12 @@ static int handle_rdpmc(struct kvm_vcpu *vcpu)
 	int err;
 
 	err = kvm_rdpmc(vcpu);
-	kvm_complete_insn_gp(vcpu, err);
-
-	return 1;
+	return kvm_complete_insn_gp(vcpu, err);
 }
 
 static int handle_wbinvd(struct kvm_vcpu *vcpu)
 {
-	kvm_emulate_wbinvd(vcpu);
-	return 1;
+	return kvm_emulate_wbinvd(vcpu);
 }
 
 static int handle_xsetbv(struct kvm_vcpu *vcpu)
@@ -5978,20 +6219,20 @@ static int handle_xsetbv(struct kvm_vcpu *vcpu)
 	u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);
 
 	if (kvm_set_xcr(vcpu, index, new_bv) == 0)
-		skip_emulated_instruction(vcpu);
+		return kvm_skip_emulated_instruction(vcpu);
 	return 1;
 }
 
 static int handle_xsaves(struct kvm_vcpu *vcpu)
 {
-	skip_emulated_instruction(vcpu);
+	kvm_skip_emulated_instruction(vcpu);
 	WARN(1, "this should never happen\n");
 	return 1;
 }
 
 static int handle_xrstors(struct kvm_vcpu *vcpu)
 {
-	skip_emulated_instruction(vcpu);
+	kvm_skip_emulated_instruction(vcpu);
 	WARN(1, "this should never happen\n");
 	return 1;
 }
@@ -6012,8 +6253,7 @@ static int handle_apic_access(struct kvm_vcpu *vcpu)
 		if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
 		    (offset == APIC_EOI)) {
 			kvm_lapic_set_eoi(vcpu);
-			skip_emulated_instruction(vcpu);
-			return 1;
+			return kvm_skip_emulated_instruction(vcpu);
 		}
 	}
 	return emulate_instruction(vcpu, 0) == EMULATE_DONE;
@@ -6161,9 +6401,8 @@ static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
 
 	gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
 	if (!kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
-		skip_emulated_instruction(vcpu);
 		trace_kvm_fast_mmio(gpa);
-		return 1;
+		return kvm_skip_emulated_instruction(vcpu);
 	}
 
 	ret = handle_mmio_page_fault(vcpu, gpa, true);
@@ -6348,50 +6587,13 @@ static __init int hardware_setup(void)
 	for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i)
 		kvm_define_shared_msr(i, vmx_msr_index[i]);
 
-	vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL);
-	if (!vmx_io_bitmap_a)
-		return r;
+	for (i = 0; i < VMX_BITMAP_NR; i++) {
+		vmx_bitmap[i] = (unsigned long *)__get_free_page(GFP_KERNEL);
+		if (!vmx_bitmap[i])
+			goto out;
+	}
 
 	vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
-	if (!vmx_io_bitmap_b)
-		goto out;
-
-	vmx_msr_bitmap_legacy = (unsigned long *)__get_free_page(GFP_KERNEL);
-	if (!vmx_msr_bitmap_legacy)
-		goto out1;
-
-	vmx_msr_bitmap_legacy_x2apic =
-				(unsigned long *)__get_free_page(GFP_KERNEL);
-	if (!vmx_msr_bitmap_legacy_x2apic)
-		goto out2;
-
-	vmx_msr_bitmap_legacy_x2apic_apicv_inactive =
-				(unsigned long *)__get_free_page(GFP_KERNEL);
-	if (!vmx_msr_bitmap_legacy_x2apic_apicv_inactive)
-		goto out3;
-
-	vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL);
-	if (!vmx_msr_bitmap_longmode)
-		goto out4;
-
-	vmx_msr_bitmap_longmode_x2apic =
-				(unsigned long *)__get_free_page(GFP_KERNEL);
-	if (!vmx_msr_bitmap_longmode_x2apic)
-		goto out5;
-
-	vmx_msr_bitmap_longmode_x2apic_apicv_inactive =
-				(unsigned long *)__get_free_page(GFP_KERNEL);
-	if (!vmx_msr_bitmap_longmode_x2apic_apicv_inactive)
-		goto out6;
-
-	vmx_vmread_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL);
-	if (!vmx_vmread_bitmap)
-		goto out7;
-
-	vmx_vmwrite_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL);
-	if (!vmx_vmwrite_bitmap)
-		goto out8;
-
 	memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
 	memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);
 
@@ -6409,7 +6611,7 @@ static __init int hardware_setup(void)
 
 	if (setup_vmcs_config(&vmcs_config) < 0) {
 		r = -EIO;
-		goto out9;
+		goto out;
 	}
 
 	if (boot_cpu_has(X86_FEATURE_NX))
@@ -6472,39 +6674,34 @@ static __init int hardware_setup(void)
 	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
 	vmx_disable_intercept_for_msr(MSR_IA32_BNDCFGS, true);
 
-	memcpy(vmx_msr_bitmap_legacy_x2apic,
+	memcpy(vmx_msr_bitmap_legacy_x2apic_apicv,
 			vmx_msr_bitmap_legacy, PAGE_SIZE);
-	memcpy(vmx_msr_bitmap_longmode_x2apic,
+	memcpy(vmx_msr_bitmap_longmode_x2apic_apicv,
 			vmx_msr_bitmap_longmode, PAGE_SIZE);
-	memcpy(vmx_msr_bitmap_legacy_x2apic_apicv_inactive,
+	memcpy(vmx_msr_bitmap_legacy_x2apic,
 			vmx_msr_bitmap_legacy, PAGE_SIZE);
-	memcpy(vmx_msr_bitmap_longmode_x2apic_apicv_inactive,
+	memcpy(vmx_msr_bitmap_longmode_x2apic,
 			vmx_msr_bitmap_longmode, PAGE_SIZE);
 
 	set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
 
+	for (msr = 0x800; msr <= 0x8ff; msr++) {
+		if (msr == 0x839 /* TMCCT */)
+			continue;
+		vmx_disable_intercept_msr_x2apic(msr, MSR_TYPE_R, true);
+	}
+
 	/*
-	 * enable_apicv && kvm_vcpu_apicv_active()
+	 * TPR reads and writes can be virtualized even if virtual interrupt
+	 * delivery is not in use.
 	 */
-	for (msr = 0x800; msr <= 0x8ff; msr++)
-		vmx_disable_intercept_msr_read_x2apic(msr, true);
+	vmx_disable_intercept_msr_x2apic(0x808, MSR_TYPE_W, true);
+	vmx_disable_intercept_msr_x2apic(0x808, MSR_TYPE_R | MSR_TYPE_W, false);
 
-	/* TMCCT */
-	vmx_enable_intercept_msr_read_x2apic(0x839, true);
-	/* TPR */
-	vmx_disable_intercept_msr_write_x2apic(0x808, true);
 	/* EOI */
-	vmx_disable_intercept_msr_write_x2apic(0x80b, true);
+	vmx_disable_intercept_msr_x2apic(0x80b, MSR_TYPE_W, true);
 	/* SELF-IPI */
-	vmx_disable_intercept_msr_write_x2apic(0x83f, true);
-
-	/*
-	 * (enable_apicv && !kvm_vcpu_apicv_active()) ||
-	 * 	!enable_apicv
-	 */
-	/* TPR */
-	vmx_disable_intercept_msr_read_x2apic(0x808, false);
-	vmx_disable_intercept_msr_write_x2apic(0x808, false);
+	vmx_disable_intercept_msr_x2apic(0x83f, MSR_TYPE_W, true);
 
 	if (enable_ept) {
 		kvm_mmu_set_mask_ptes(VMX_EPT_READABLE_MASK,
@@ -6551,42 +6748,19 @@ static __init int hardware_setup(void)
 
 	return alloc_kvm_area();
 
-out9:
-	free_page((unsigned long)vmx_vmwrite_bitmap);
-out8:
-	free_page((unsigned long)vmx_vmread_bitmap);
-out7:
-	free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic_apicv_inactive);
-out6:
-	free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic);
-out5:
-	free_page((unsigned long)vmx_msr_bitmap_longmode);
-out4:
-	free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic_apicv_inactive);
-out3:
-	free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic);
-out2:
-	free_page((unsigned long)vmx_msr_bitmap_legacy);
-out1:
-	free_page((unsigned long)vmx_io_bitmap_b);
 out:
-	free_page((unsigned long)vmx_io_bitmap_a);
+	for (i = 0; i < VMX_BITMAP_NR; i++)
+		free_page((unsigned long)vmx_bitmap[i]);
 
     return r;
 }
 
 static __exit void hardware_unsetup(void)
 {
-	free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic);
-	free_page((unsigned long)vmx_msr_bitmap_legacy_x2apic_apicv_inactive);
-	free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic);
-	free_page((unsigned long)vmx_msr_bitmap_longmode_x2apic_apicv_inactive);
-	free_page((unsigned long)vmx_msr_bitmap_legacy);
-	free_page((unsigned long)vmx_msr_bitmap_longmode);
-	free_page((unsigned long)vmx_io_bitmap_b);
-	free_page((unsigned long)vmx_io_bitmap_a);
-	free_page((unsigned long)vmx_vmwrite_bitmap);
-	free_page((unsigned long)vmx_vmread_bitmap);
+	int i;
+
+	for (i = 0; i < VMX_BITMAP_NR; i++)
+		free_page((unsigned long)vmx_bitmap[i]);
 
 	free_kvm_area();
 }
@@ -6600,16 +6774,13 @@ static int handle_pause(struct kvm_vcpu *vcpu)
 	if (ple_gap)
 		grow_ple_window(vcpu);
 
-	skip_emulated_instruction(vcpu);
 	kvm_vcpu_on_spin(vcpu);
-
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 static int handle_nop(struct kvm_vcpu *vcpu)
 {
-	skip_emulated_instruction(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 static int handle_mwait(struct kvm_vcpu *vcpu)
@@ -6916,8 +7087,7 @@ static int nested_vmx_check_vmptr(struct kvm_vcpu *vcpu, int exit_reason,
 		 */
 		if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) {
 			nested_vmx_failInvalid(vcpu);
-			skip_emulated_instruction(vcpu);
-			return 1;
+			return kvm_skip_emulated_instruction(vcpu);
 		}
 
 		page = nested_get_page(vcpu, vmptr);
@@ -6925,8 +7095,7 @@ static int nested_vmx_check_vmptr(struct kvm_vcpu *vcpu, int exit_reason,
 		    *(u32 *)kmap(page) != VMCS12_REVISION) {
 			nested_vmx_failInvalid(vcpu);
 			kunmap(page);
-			skip_emulated_instruction(vcpu);
-			return 1;
+			return kvm_skip_emulated_instruction(vcpu);
 		}
 		kunmap(page);
 		vmx->nested.vmxon_ptr = vmptr;
@@ -6935,30 +7104,26 @@ static int nested_vmx_check_vmptr(struct kvm_vcpu *vcpu, int exit_reason,
 		if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) {
 			nested_vmx_failValid(vcpu,
 					     VMXERR_VMCLEAR_INVALID_ADDRESS);
-			skip_emulated_instruction(vcpu);
-			return 1;
+			return kvm_skip_emulated_instruction(vcpu);
 		}
 
 		if (vmptr == vmx->nested.vmxon_ptr) {
 			nested_vmx_failValid(vcpu,
 					     VMXERR_VMCLEAR_VMXON_POINTER);
-			skip_emulated_instruction(vcpu);
-			return 1;
+			return kvm_skip_emulated_instruction(vcpu);
 		}
 		break;
 	case EXIT_REASON_VMPTRLD:
 		if (!PAGE_ALIGNED(vmptr) || (vmptr >> maxphyaddr)) {
 			nested_vmx_failValid(vcpu,
 					     VMXERR_VMPTRLD_INVALID_ADDRESS);
-			skip_emulated_instruction(vcpu);
-			return 1;
+			return kvm_skip_emulated_instruction(vcpu);
 		}
 
 		if (vmptr == vmx->nested.vmxon_ptr) {
 			nested_vmx_failValid(vcpu,
 					     VMXERR_VMCLEAR_VMXON_POINTER);
-			skip_emulated_instruction(vcpu);
-			return 1;
+			return kvm_skip_emulated_instruction(vcpu);
 		}
 		break;
 	default:
@@ -7014,8 +7179,7 @@ static int handle_vmon(struct kvm_vcpu *vcpu)
 
 	if (vmx->nested.vmxon) {
 		nested_vmx_failValid(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
-		skip_emulated_instruction(vcpu);
-		return 1;
+		return kvm_skip_emulated_instruction(vcpu);
 	}
 
 	if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
@@ -7055,9 +7219,8 @@ static int handle_vmon(struct kvm_vcpu *vcpu)
 
 	vmx->nested.vmxon = true;
 
-	skip_emulated_instruction(vcpu);
 	nested_vmx_succeed(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 
 out_shadow_vmcs:
 	kfree(vmx->nested.cached_vmcs12);
@@ -7176,9 +7339,8 @@ static int handle_vmoff(struct kvm_vcpu *vcpu)
 	if (!nested_vmx_check_permission(vcpu))
 		return 1;
 	free_nested(to_vmx(vcpu));
-	skip_emulated_instruction(vcpu);
 	nested_vmx_succeed(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 /* Emulate the VMCLEAR instruction */
@@ -7217,9 +7379,8 @@ static int handle_vmclear(struct kvm_vcpu *vcpu)
 
 	nested_free_vmcs02(vmx, vmptr);
 
-	skip_emulated_instruction(vcpu);
 	nested_vmx_succeed(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch);
@@ -7417,7 +7578,6 @@ static int nested_vmx_check_vmcs12(struct kvm_vcpu *vcpu)
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	if (vmx->nested.current_vmptr == -1ull) {
 		nested_vmx_failInvalid(vcpu);
-		skip_emulated_instruction(vcpu);
 		return 0;
 	}
 	return 1;
@@ -7431,17 +7591,18 @@ static int handle_vmread(struct kvm_vcpu *vcpu)
 	u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
 	gva_t gva = 0;
 
-	if (!nested_vmx_check_permission(vcpu) ||
-	    !nested_vmx_check_vmcs12(vcpu))
+	if (!nested_vmx_check_permission(vcpu))
 		return 1;
 
+	if (!nested_vmx_check_vmcs12(vcpu))
+		return kvm_skip_emulated_instruction(vcpu);
+
 	/* Decode instruction info and find the field to read */
 	field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
 	/* Read the field, zero-extended to a u64 field_value */
 	if (vmcs12_read_any(vcpu, field, &field_value) < 0) {
 		nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
-		skip_emulated_instruction(vcpu);
-		return 1;
+		return kvm_skip_emulated_instruction(vcpu);
 	}
 	/*
 	 * Now copy part of this value to register or memory, as requested.
@@ -7461,8 +7622,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu)
 	}
 
 	nested_vmx_succeed(vcpu);
-	skip_emulated_instruction(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 
@@ -7481,10 +7641,12 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu)
 	u64 field_value = 0;
 	struct x86_exception e;
 
-	if (!nested_vmx_check_permission(vcpu) ||
-	    !nested_vmx_check_vmcs12(vcpu))
+	if (!nested_vmx_check_permission(vcpu))
 		return 1;
 
+	if (!nested_vmx_check_vmcs12(vcpu))
+		return kvm_skip_emulated_instruction(vcpu);
+
 	if (vmx_instruction_info & (1u << 10))
 		field_value = kvm_register_readl(vcpu,
 			(((vmx_instruction_info) >> 3) & 0xf));
@@ -7504,19 +7666,16 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu)
 	if (vmcs_field_readonly(field)) {
 		nested_vmx_failValid(vcpu,
 			VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
-		skip_emulated_instruction(vcpu);
-		return 1;
+		return kvm_skip_emulated_instruction(vcpu);
 	}
 
 	if (vmcs12_write_any(vcpu, field, field_value) < 0) {
 		nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
-		skip_emulated_instruction(vcpu);
-		return 1;
+		return kvm_skip_emulated_instruction(vcpu);
 	}
 
 	nested_vmx_succeed(vcpu);
-	skip_emulated_instruction(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 /* Emulate the VMPTRLD instruction */
@@ -7537,8 +7696,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu)
 		page = nested_get_page(vcpu, vmptr);
 		if (page == NULL) {
 			nested_vmx_failInvalid(vcpu);
-			skip_emulated_instruction(vcpu);
-			return 1;
+			return kvm_skip_emulated_instruction(vcpu);
 		}
 		new_vmcs12 = kmap(page);
 		if (new_vmcs12->revision_id != VMCS12_REVISION) {
@@ -7546,8 +7704,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu)
 			nested_release_page_clean(page);
 			nested_vmx_failValid(vcpu,
 				VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
-			skip_emulated_instruction(vcpu);
-			return 1;
+			return kvm_skip_emulated_instruction(vcpu);
 		}
 
 		nested_release_vmcs12(vmx);
@@ -7571,8 +7728,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu)
 	}
 
 	nested_vmx_succeed(vcpu);
-	skip_emulated_instruction(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 /* Emulate the VMPTRST instruction */
@@ -7597,8 +7753,7 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu)
 		return 1;
 	}
 	nested_vmx_succeed(vcpu);
-	skip_emulated_instruction(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 /* Emulate the INVEPT instruction */
@@ -7636,8 +7791,7 @@ static int handle_invept(struct kvm_vcpu *vcpu)
 	if (type >= 32 || !(types & (1 << type))) {
 		nested_vmx_failValid(vcpu,
 				VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
-		skip_emulated_instruction(vcpu);
-		return 1;
+		return kvm_skip_emulated_instruction(vcpu);
 	}
 
 	/* According to the Intel VMX instruction reference, the memory
@@ -7668,8 +7822,7 @@ static int handle_invept(struct kvm_vcpu *vcpu)
 		break;
 	}
 
-	skip_emulated_instruction(vcpu);
-	return 1;
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 static int handle_invvpid(struct kvm_vcpu *vcpu)
@@ -7694,13 +7847,13 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
 	vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
 	type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
 
-	types = (vmx->nested.nested_vmx_vpid_caps >> 8) & 0x7;
+	types = (vmx->nested.nested_vmx_vpid_caps &
+			VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8;
 
 	if (type >= 32 || !(types & (1 << type))) {
 		nested_vmx_failValid(vcpu,
 			VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
-		skip_emulated_instruction(vcpu);
-		return 1;
+		return kvm_skip_emulated_instruction(vcpu);
 	}
 
 	/* according to the intel vmx instruction reference, the memory
@@ -7716,23 +7869,26 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
 	}
 
 	switch (type) {
+	case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
 	case VMX_VPID_EXTENT_SINGLE_CONTEXT:
-		/*
-		 * Old versions of KVM use the single-context version so we
-		 * have to support it; just treat it the same as all-context.
-		 */
+	case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
+		if (!vpid) {
+			nested_vmx_failValid(vcpu,
+				VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
+			return kvm_skip_emulated_instruction(vcpu);
+		}
+		break;
 	case VMX_VPID_EXTENT_ALL_CONTEXT:
-		__vmx_flush_tlb(vcpu, to_vmx(vcpu)->nested.vpid02);
-		nested_vmx_succeed(vcpu);
 		break;
 	default:
-		/* Trap individual address invalidation invvpid calls */
-		BUG_ON(1);
-		break;
+		WARN_ON_ONCE(1);
+		return kvm_skip_emulated_instruction(vcpu);
 	}
 
-	skip_emulated_instruction(vcpu);
-	return 1;
+	__vmx_flush_tlb(vcpu, vmx->nested.vpid02);
+	nested_vmx_succeed(vcpu);
+
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 static int handle_pml_full(struct kvm_vcpu *vcpu)
@@ -8071,6 +8227,8 @@ static bool nested_vmx_exit_handled(struct kvm_vcpu *vcpu)
 		return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
 	case EXIT_REASON_IO_INSTRUCTION:
 		return nested_vmx_exit_handled_io(vcpu, vmcs12);
+	case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR:
+		return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC);
 	case EXIT_REASON_MSR_READ:
 	case EXIT_REASON_MSR_WRITE:
 		return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
@@ -8620,11 +8778,6 @@ static void vmx_handle_external_intr(struct kvm_vcpu *vcpu)
 	u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
 	register void *__sp asm(_ASM_SP);
 
-	/*
-	 * If external interrupt exists, IF bit is set in rflags/eflags on the
-	 * interrupt stack frame, and interrupt will be enabled on a return
-	 * from interrupt handler.
-	 */
 	if ((exit_intr_info & (INTR_INFO_VALID_MASK | INTR_INFO_INTR_TYPE_MASK))
 			== (INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR)) {
 		unsigned int vector;
@@ -8809,7 +8962,7 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
 					msrs[i].host);
 }
 
-void vmx_arm_hv_timer(struct kvm_vcpu *vcpu)
+static void vmx_arm_hv_timer(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	u64 tscl;
@@ -9279,6 +9432,50 @@ static void vmcs_set_secondary_exec_control(u32 new_ctl)
 		     (new_ctl & ~mask) | (cur_ctl & mask));
 }
 
+/*
+ * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits
+ * (indicating "allowed-1") if they are supported in the guest's CPUID.
+ */
+static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_vmx *vmx = to_vmx(vcpu);
+	struct kvm_cpuid_entry2 *entry;
+
+	vmx->nested.nested_vmx_cr0_fixed1 = 0xffffffff;
+	vmx->nested.nested_vmx_cr4_fixed1 = X86_CR4_PCE;
+
+#define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do {		\
+	if (entry && (entry->_reg & (_cpuid_mask)))			\
+		vmx->nested.nested_vmx_cr4_fixed1 |= (_cr4_mask);	\
+} while (0)
+
+	entry = kvm_find_cpuid_entry(vcpu, 0x1, 0);
+	cr4_fixed1_update(X86_CR4_VME,        edx, bit(X86_FEATURE_VME));
+	cr4_fixed1_update(X86_CR4_PVI,        edx, bit(X86_FEATURE_VME));
+	cr4_fixed1_update(X86_CR4_TSD,        edx, bit(X86_FEATURE_TSC));
+	cr4_fixed1_update(X86_CR4_DE,         edx, bit(X86_FEATURE_DE));
+	cr4_fixed1_update(X86_CR4_PSE,        edx, bit(X86_FEATURE_PSE));
+	cr4_fixed1_update(X86_CR4_PAE,        edx, bit(X86_FEATURE_PAE));
+	cr4_fixed1_update(X86_CR4_MCE,        edx, bit(X86_FEATURE_MCE));
+	cr4_fixed1_update(X86_CR4_PGE,        edx, bit(X86_FEATURE_PGE));
+	cr4_fixed1_update(X86_CR4_OSFXSR,     edx, bit(X86_FEATURE_FXSR));
+	cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, bit(X86_FEATURE_XMM));
+	cr4_fixed1_update(X86_CR4_VMXE,       ecx, bit(X86_FEATURE_VMX));
+	cr4_fixed1_update(X86_CR4_SMXE,       ecx, bit(X86_FEATURE_SMX));
+	cr4_fixed1_update(X86_CR4_PCIDE,      ecx, bit(X86_FEATURE_PCID));
+	cr4_fixed1_update(X86_CR4_OSXSAVE,    ecx, bit(X86_FEATURE_XSAVE));
+
+	entry = kvm_find_cpuid_entry(vcpu, 0x7, 0);
+	cr4_fixed1_update(X86_CR4_FSGSBASE,   ebx, bit(X86_FEATURE_FSGSBASE));
+	cr4_fixed1_update(X86_CR4_SMEP,       ebx, bit(X86_FEATURE_SMEP));
+	cr4_fixed1_update(X86_CR4_SMAP,       ebx, bit(X86_FEATURE_SMAP));
+	cr4_fixed1_update(X86_CR4_PKE,        ecx, bit(X86_FEATURE_PKU));
+	/* TODO: Use X86_CR4_UMIP and X86_FEATURE_UMIP macros */
+	cr4_fixed1_update(bit(11),            ecx, bit(2));
+
+#undef cr4_fixed1_update
+}
+
 static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
 {
 	struct kvm_cpuid_entry2 *best;
@@ -9320,6 +9517,9 @@ static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
 	else
 		to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
 			~FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
+
+	if (nested_vmx_allowed(vcpu))
+		nested_vmx_cr_fixed1_bits_update(vcpu);
 }
 
 static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
@@ -9774,6 +9974,49 @@ static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
 	return 0;
 }
 
+static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val)
+{
+	unsigned long invalid_mask;
+
+	invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
+	return (val & invalid_mask) == 0;
+}
+
+/*
+ * Load guest's/host's cr3 at nested entry/exit. nested_ept is true if we are
+ * emulating VM entry into a guest with EPT enabled.
+ * Returns 0 on success, 1 on failure. Invalid state exit qualification code
+ * is assigned to entry_failure_code on failure.
+ */
+static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept,
+			       unsigned long *entry_failure_code)
+{
+	if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) {
+		if (!nested_cr3_valid(vcpu, cr3)) {
+			*entry_failure_code = ENTRY_FAIL_DEFAULT;
+			return 1;
+		}
+
+		/*
+		 * If PAE paging and EPT are both on, CR3 is not used by the CPU and
+		 * must not be dereferenced.
+		 */
+		if (!is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu) &&
+		    !nested_ept) {
+			if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) {
+				*entry_failure_code = ENTRY_FAIL_PDPTE;
+				return 1;
+			}
+		}
+
+		vcpu->arch.cr3 = cr3;
+		__set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
+	}
+
+	kvm_mmu_reset_context(vcpu);
+	return 0;
+}
+
 /*
  * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
  * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
@@ -9782,11 +10025,15 @@ static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
  * needs. In addition to modifying the active vmcs (which is vmcs02), this
  * function also has additional necessary side-effects, like setting various
  * vcpu->arch fields.
+ * Returns 0 on success, 1 on failure. Invalid state exit qualification code
+ * is assigned to entry_failure_code on failure.
  */
-static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
+static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
+			  unsigned long *entry_failure_code)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	u32 exec_control;
+	bool nested_ept_enabled = false;
 
 	vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
 	vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
@@ -9951,6 +10198,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
 				vmcs12->guest_intr_status);
 		}
 
+		nested_ept_enabled = (exec_control & SECONDARY_EXEC_ENABLE_EPT) != 0;
 		vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
 	}
 
@@ -9964,6 +10212,15 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
 	vmx_set_constant_host_state(vmx);
 
 	/*
+	 * Set the MSR load/store lists to match L0's settings.
+	 */
+	vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
+	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
+	vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
+	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
+	vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
+
+	/*
 	 * HOST_RSP is normally set correctly in vmx_vcpu_run() just before
 	 * entry, but only if the current (host) sp changed from the value
 	 * we wrote last (vmx->host_rsp). This cache is no longer relevant
@@ -10069,15 +10326,6 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
 		nested_ept_init_mmu_context(vcpu);
 	}
 
-	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)
-		vcpu->arch.efer = vmcs12->guest_ia32_efer;
-	else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
-		vcpu->arch.efer |= (EFER_LMA | EFER_LME);
-	else
-		vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
-	/* Note: modifies VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
-	vmx_set_efer(vcpu, vcpu->arch.efer);
-
 	/*
 	 * This sets GUEST_CR0 to vmcs12->guest_cr0, with possibly a modified
 	 * TS bit (for lazy fpu) and bits which we consider mandatory enabled.
@@ -10092,8 +10340,20 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
 	vmx_set_cr4(vcpu, vmcs12->guest_cr4);
 	vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));
 
-	/* shadow page tables on either EPT or shadow page tables */
-	kvm_set_cr3(vcpu, vmcs12->guest_cr3);
+	if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)
+		vcpu->arch.efer = vmcs12->guest_ia32_efer;
+	else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
+		vcpu->arch.efer |= (EFER_LMA | EFER_LME);
+	else
+		vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
+	/* Note: modifies VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
+	vmx_set_efer(vcpu, vcpu->arch.efer);
+
+	/* Shadow page tables on either EPT or shadow page tables. */
+	if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_ept_enabled,
+				entry_failure_code))
+		return 1;
+
 	kvm_mmu_reset_context(vcpu);
 
 	if (!enable_ept)
@@ -10111,6 +10371,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
 
 	kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp);
 	kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip);
+	return 0;
 }
 
 /*
@@ -10125,12 +10386,14 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
 	struct loaded_vmcs *vmcs02;
 	bool ia32e;
 	u32 msr_entry_idx;
+	unsigned long exit_qualification;
 
-	if (!nested_vmx_check_permission(vcpu) ||
-	    !nested_vmx_check_vmcs12(vcpu))
+	if (!nested_vmx_check_permission(vcpu))
 		return 1;
 
-	skip_emulated_instruction(vcpu);
+	if (!nested_vmx_check_vmcs12(vcpu))
+		goto out;
+
 	vmcs12 = get_vmcs12(vcpu);
 
 	if (enable_shadow_vmcs)
@@ -10150,37 +10413,37 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
 		nested_vmx_failValid(vcpu,
 			launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
 			       : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
-		return 1;
+		goto out;
 	}
 
 	if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
 	    vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT) {
 		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
-		return 1;
+		goto out;
 	}
 
 	if (!nested_get_vmcs12_pages(vcpu, vmcs12)) {
 		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
-		return 1;
+		goto out;
 	}
 
 	if (nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12)) {
 		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
-		return 1;
+		goto out;
 	}
 
 	if (nested_vmx_check_apicv_controls(vcpu, vmcs12)) {
 		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
-		return 1;
+		goto out;
 	}
 
 	if (nested_vmx_check_msr_switch_controls(vcpu, vmcs12)) {
 		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
-		return 1;
+		goto out;
 	}
 
 	if (!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
-				vmx->nested.nested_vmx_true_procbased_ctls_low,
+				vmx->nested.nested_vmx_procbased_ctls_low,
 				vmx->nested.nested_vmx_procbased_ctls_high) ||
 	    !vmx_control_verify(vmcs12->secondary_vm_exec_control,
 				vmx->nested.nested_vmx_secondary_ctls_low,
@@ -10189,33 +10452,34 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
 				vmx->nested.nested_vmx_pinbased_ctls_low,
 				vmx->nested.nested_vmx_pinbased_ctls_high) ||
 	    !vmx_control_verify(vmcs12->vm_exit_controls,
-				vmx->nested.nested_vmx_true_exit_ctls_low,
+				vmx->nested.nested_vmx_exit_ctls_low,
 				vmx->nested.nested_vmx_exit_ctls_high) ||
 	    !vmx_control_verify(vmcs12->vm_entry_controls,
-				vmx->nested.nested_vmx_true_entry_ctls_low,
+				vmx->nested.nested_vmx_entry_ctls_low,
 				vmx->nested.nested_vmx_entry_ctls_high))
 	{
 		nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
-		return 1;
+		goto out;
 	}
 
-	if (((vmcs12->host_cr0 & VMXON_CR0_ALWAYSON) != VMXON_CR0_ALWAYSON) ||
-	    ((vmcs12->host_cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) {
+	if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) ||
+	    !nested_host_cr4_valid(vcpu, vmcs12->host_cr4) ||
+	    !nested_cr3_valid(vcpu, vmcs12->host_cr3)) {
 		nested_vmx_failValid(vcpu,
 			VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
-		return 1;
+		goto out;
 	}
 
-	if (!nested_cr0_valid(vcpu, vmcs12->guest_cr0) ||
-	    ((vmcs12->guest_cr4 & VMXON_CR4_ALWAYSON) != VMXON_CR4_ALWAYSON)) {
+	if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) ||
+	    !nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)) {
 		nested_vmx_entry_failure(vcpu, vmcs12,
 			EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
-		return 1;
+		goto out;
 	}
 	if (vmcs12->vmcs_link_pointer != -1ull) {
 		nested_vmx_entry_failure(vcpu, vmcs12,
 			EXIT_REASON_INVALID_STATE, ENTRY_FAIL_VMCS_LINK_PTR);
-		return 1;
+		goto out;
 	}
 
 	/*
@@ -10235,7 +10499,7 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
 		     ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))) {
 			nested_vmx_entry_failure(vcpu, vmcs12,
 				EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
-			return 1;
+			goto out;
 		}
 	}
 
@@ -10253,7 +10517,7 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
 		    ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)) {
 			nested_vmx_entry_failure(vcpu, vmcs12,
 				EXIT_REASON_INVALID_STATE, ENTRY_FAIL_DEFAULT);
-			return 1;
+			goto out;
 		}
 	}
 
@@ -10266,6 +10530,12 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
 	if (!vmcs02)
 		return -ENOMEM;
 
+	/*
+	 * After this point, the trap flag no longer triggers a singlestep trap
+	 * on the vm entry instructions. Don't call
+	 * kvm_skip_emulated_instruction.
+	 */
+	skip_emulated_instruction(vcpu);
 	enter_guest_mode(vcpu);
 
 	if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
@@ -10280,7 +10550,13 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
 
 	vmx_segment_cache_clear(vmx);
 
-	prepare_vmcs02(vcpu, vmcs12);
+	if (prepare_vmcs02(vcpu, vmcs12, &exit_qualification)) {
+		leave_guest_mode(vcpu);
+		vmx_load_vmcs01(vcpu);
+		nested_vmx_entry_failure(vcpu, vmcs12,
+				EXIT_REASON_INVALID_STATE, exit_qualification);
+		return 1;
+	}
 
 	msr_entry_idx = nested_vmx_load_msr(vcpu,
 					    vmcs12->vm_entry_msr_load_addr,
@@ -10307,6 +10583,9 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
 	 * the success flag) when L2 exits (see nested_vmx_vmexit()).
 	 */
 	return 1;
+
+out:
+	return kvm_skip_emulated_instruction(vcpu);
 }
 
 /*
@@ -10612,6 +10891,7 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
 				   struct vmcs12 *vmcs12)
 {
 	struct kvm_segment seg;
+	unsigned long entry_failure_code;
 
 	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
 		vcpu->arch.efer = vmcs12->host_ia32_efer;
@@ -10649,8 +10929,12 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
 
 	nested_ept_uninit_mmu_context(vcpu);
 
-	kvm_set_cr3(vcpu, vmcs12->host_cr3);
-	kvm_mmu_reset_context(vcpu);
+	/*
+	 * Only PDPTE load can fail as the value of cr3 was checked on entry and
+	 * couldn't have changed.
+	 */
+	if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code))
+		nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL);
 
 	if (!enable_ept)
 		vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
@@ -10751,6 +11035,7 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
+	u32 vm_inst_error = 0;
 
 	/* trying to cancel vmlaunch/vmresume is a bug */
 	WARN_ON_ONCE(vmx->nested.nested_run_pending);
@@ -10763,6 +11048,9 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
 				 vmcs12->vm_exit_msr_store_count))
 		nested_vmx_abort(vcpu, VMX_ABORT_SAVE_GUEST_MSR_FAIL);
 
+	if (unlikely(vmx->fail))
+		vm_inst_error = vmcs_read32(VM_INSTRUCTION_ERROR);
+
 	vmx_load_vmcs01(vcpu);
 
 	if ((exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT)
@@ -10791,6 +11079,8 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
 	load_vmcs12_host_state(vcpu, vmcs12);
 
 	/* Update any VMCS fields that might have changed while L2 ran */
+	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
+	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
 	vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
 	if (vmx->hv_deadline_tsc == -1)
 		vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL,
@@ -10839,7 +11129,7 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
 	 */
 	if (unlikely(vmx->fail)) {
 		vmx->fail = 0;
-		nested_vmx_failValid(vcpu, vmcs_read32(VM_INSTRUCTION_ERROR));
+		nested_vmx_failValid(vcpu, vm_inst_error);
 	} else
 		nested_vmx_succeed(vcpu);
 	if (enable_shadow_vmcs)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 6f5f465fdb6b..1f0d2383f5ee 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -434,12 +434,14 @@ void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
 }
 EXPORT_SYMBOL_GPL(kvm_requeue_exception);
 
-void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
+int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
 {
 	if (err)
 		kvm_inject_gp(vcpu, 0);
 	else
-		kvm_x86_ops->skip_emulated_instruction(vcpu);
+		return kvm_skip_emulated_instruction(vcpu);
+
+	return 1;
 }
 EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
 
@@ -573,7 +575,7 @@ out:
 }
 EXPORT_SYMBOL_GPL(load_pdptrs);
 
-static bool pdptrs_changed(struct kvm_vcpu *vcpu)
+bool pdptrs_changed(struct kvm_vcpu *vcpu)
 {
 	u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
 	bool changed = true;
@@ -599,6 +601,7 @@ out:
 
 	return changed;
 }
+EXPORT_SYMBOL_GPL(pdptrs_changed);
 
 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 {
@@ -2178,7 +2181,6 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		break;
 	case MSR_KVM_SYSTEM_TIME_NEW:
 	case MSR_KVM_SYSTEM_TIME: {
-		u64 gpa_offset;
 		struct kvm_arch *ka = &vcpu->kvm->arch;
 
 		kvmclock_reset(vcpu);
@@ -2200,8 +2202,6 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		if (!(data & 1))
 			break;
 
-		gpa_offset = data & ~(PAGE_MASK | 1);
-
 		if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
 		     &vcpu->arch.pv_time, data & ~1ULL,
 		     sizeof(struct pvclock_vcpu_time_info)))
@@ -2296,7 +2296,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		if (kvm_pmu_is_valid_msr(vcpu, msr))
 			return kvm_pmu_set_msr(vcpu, msr_info);
 		if (!ignore_msrs) {
-			vcpu_unimpl(vcpu, "unhandled wrmsr: 0x%x data 0x%llx\n",
+			vcpu_debug_ratelimited(vcpu, "unhandled wrmsr: 0x%x data 0x%llx\n",
 				    msr, data);
 			return 1;
 		} else {
@@ -2508,7 +2508,8 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
 			return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data);
 		if (!ignore_msrs) {
-			vcpu_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr_info->index);
+			vcpu_debug_ratelimited(vcpu, "unhandled rdmsr: 0x%x\n",
+					       msr_info->index);
 			return 1;
 		} else {
 			vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr_info->index);
@@ -2812,7 +2813,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 		}
 		if (kvm_lapic_hv_timer_in_use(vcpu) &&
 				kvm_x86_ops->set_hv_timer(vcpu,
-					kvm_get_lapic_tscdeadline_msr(vcpu)))
+					kvm_get_lapic_target_expiration_tsc(vcpu)))
 			kvm_lapic_switch_to_sw_timer(vcpu);
 		/*
 		 * On a host with synchronized TSC, there is no need to update
@@ -4832,7 +4833,7 @@ static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
 	kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
 }
 
-int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
+static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
 {
 	if (!need_emulate_wbinvd(vcpu))
 		return X86EMUL_CONTINUE;
@@ -4852,8 +4853,8 @@ int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
 
 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
 {
-	kvm_x86_ops->skip_emulated_instruction(vcpu);
-	return kvm_emulate_wbinvd_noskip(vcpu);
+	kvm_emulate_wbinvd_noskip(vcpu);
+	return kvm_skip_emulated_instruction(vcpu);
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
 
@@ -5451,7 +5452,6 @@ static void kvm_vcpu_check_singlestep(struct kvm_vcpu *vcpu, unsigned long rflag
 			kvm_run->exit_reason = KVM_EXIT_DEBUG;
 			*r = EMULATE_USER_EXIT;
 		} else {
-			vcpu->arch.emulate_ctxt.eflags &= ~X86_EFLAGS_TF;
 			/*
 			 * "Certain debug exceptions may clear bit 0-3.  The
 			 * remaining contents of the DR6 register are never
@@ -5464,6 +5464,17 @@ static void kvm_vcpu_check_singlestep(struct kvm_vcpu *vcpu, unsigned long rflag
 	}
 }
 
+int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
+{
+	unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
+	int r = EMULATE_DONE;
+
+	kvm_x86_ops->skip_emulated_instruction(vcpu);
+	kvm_vcpu_check_singlestep(vcpu, rflags, &r);
+	return r == EMULATE_DONE;
+}
+EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction);
+
 static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r)
 {
 	if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
@@ -5649,6 +5660,49 @@ int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
 }
 EXPORT_SYMBOL_GPL(kvm_fast_pio_out);
 
+static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
+{
+	unsigned long val;
+
+	/* We should only ever be called with arch.pio.count equal to 1 */
+	BUG_ON(vcpu->arch.pio.count != 1);
+
+	/* For size less than 4 we merge, else we zero extend */
+	val = (vcpu->arch.pio.size < 4) ? kvm_register_read(vcpu, VCPU_REGS_RAX)
+					: 0;
+
+	/*
+	 * Since vcpu->arch.pio.count == 1 let emulator_pio_in_emulated perform
+	 * the copy and tracing
+	 */
+	emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, vcpu->arch.pio.size,
+				 vcpu->arch.pio.port, &val, 1);
+	kvm_register_write(vcpu, VCPU_REGS_RAX, val);
+
+	return 1;
+}
+
+int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size, unsigned short port)
+{
+	unsigned long val;
+	int ret;
+
+	/* For size less than 4 we merge, else we zero extend */
+	val = (size < 4) ? kvm_register_read(vcpu, VCPU_REGS_RAX) : 0;
+
+	ret = emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, size, port,
+				       &val, 1);
+	if (ret) {
+		kvm_register_write(vcpu, VCPU_REGS_RAX, val);
+		return ret;
+	}
+
+	vcpu->arch.complete_userspace_io = complete_fast_pio_in;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(kvm_fast_pio_in);
+
 static int kvmclock_cpu_down_prep(unsigned int cpu)
 {
 	__this_cpu_write(cpu_tsc_khz, 0);
@@ -5998,8 +6052,12 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_halt);
 
 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
 {
-	kvm_x86_ops->skip_emulated_instruction(vcpu);
-	return kvm_vcpu_halt(vcpu);
+	int ret = kvm_skip_emulated_instruction(vcpu);
+	/*
+	 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered
+	 * KVM_EXIT_DEBUG here.
+	 */
+	return kvm_vcpu_halt(vcpu) && ret;
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
 
@@ -6030,9 +6088,9 @@ void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu)
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
 {
 	unsigned long nr, a0, a1, a2, a3, ret;
-	int op_64_bit, r = 1;
+	int op_64_bit, r;
 
-	kvm_x86_ops->skip_emulated_instruction(vcpu);
+	r = kvm_skip_emulated_instruction(vcpu);
 
 	if (kvm_hv_hypercall_enabled(vcpu->kvm))
 		return kvm_hv_hypercall(vcpu);
@@ -8175,7 +8233,7 @@ void kvm_arch_flush_shadow_all(struct kvm *kvm)
 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
 				   struct kvm_memory_slot *slot)
 {
-	kvm_mmu_invalidate_zap_all_pages(kvm);
+	kvm_page_track_flush_slot(kvm, slot);
 }
 
 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c
index bedfab98077a..e1fb269c87af 100644
--- a/arch/x86/pci/xen.c
+++ b/arch/x86/pci/xen.c
@@ -264,8 +264,8 @@ static int xen_hvm_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
 	return 0;
 
 error:
-	dev_err(&dev->dev,
-		"Xen PCI frontend has not registered MSI/MSI-X support!\n");
+	dev_err(&dev->dev, "Failed to create MSI%s! ret=%d!\n",
+		type == PCI_CAP_ID_MSI ? "" : "-X", irq);
 	return irq;
 }
 
diff --git a/arch/x86/platform/ce4100/ce4100.c b/arch/x86/platform/ce4100/ce4100.c
index b27bccd4390f..821cb41f00e6 100644
--- a/arch/x86/platform/ce4100/ce4100.c
+++ b/arch/x86/platform/ce4100/ce4100.c
@@ -89,7 +89,7 @@ static void ce4100_mem_serial_out(struct uart_port *p, int offset, int value)
 }
 
 static void ce4100_serial_fixup(int port, struct uart_port *up,
-	unsigned short *capabilites)
+	u32 *capabilites)
 {
 #ifdef CONFIG_EARLY_PRINTK
 	/*
diff --git a/arch/x86/power/hibernate_64.c b/arch/x86/power/hibernate_64.c
index 9634557a5444..ded2e8272382 100644
--- a/arch/x86/power/hibernate_64.c
+++ b/arch/x86/power/hibernate_64.c
@@ -11,6 +11,10 @@
 #include <linux/gfp.h>
 #include <linux/smp.h>
 #include <linux/suspend.h>
+#include <linux/scatterlist.h>
+#include <linux/kdebug.h>
+
+#include <crypto/hash.h>
 
 #include <asm/init.h>
 #include <asm/proto.h>
@@ -177,14 +181,86 @@ int pfn_is_nosave(unsigned long pfn)
 	return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn);
 }
 
+#define MD5_DIGEST_SIZE 16
+
 struct restore_data_record {
 	unsigned long jump_address;
 	unsigned long jump_address_phys;
 	unsigned long cr3;
 	unsigned long magic;
+	u8 e820_digest[MD5_DIGEST_SIZE];
 };
 
-#define RESTORE_MAGIC	0x123456789ABCDEF0UL
+#define RESTORE_MAGIC	0x23456789ABCDEF01UL
+
+#if IS_BUILTIN(CONFIG_CRYPTO_MD5)
+/**
+ * get_e820_md5 - calculate md5 according to given e820 map
+ *
+ * @map: the e820 map to be calculated
+ * @buf: the md5 result to be stored to
+ */
+static int get_e820_md5(struct e820map *map, void *buf)
+{
+	struct scatterlist sg;
+	struct crypto_ahash *tfm;
+	int size;
+	int ret = 0;
+
+	tfm = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC);
+	if (IS_ERR(tfm))
+		return -ENOMEM;
+
+	{
+		AHASH_REQUEST_ON_STACK(req, tfm);
+		size = offsetof(struct e820map, map)
+			+ sizeof(struct e820entry) * map->nr_map;
+		ahash_request_set_tfm(req, tfm);
+		sg_init_one(&sg, (u8 *)map, size);
+		ahash_request_set_callback(req, 0, NULL, NULL);
+		ahash_request_set_crypt(req, &sg, buf, size);
+
+		if (crypto_ahash_digest(req))
+			ret = -EINVAL;
+		ahash_request_zero(req);
+	}
+	crypto_free_ahash(tfm);
+
+	return ret;
+}
+
+static void hibernation_e820_save(void *buf)
+{
+	get_e820_md5(e820_saved, buf);
+}
+
+static bool hibernation_e820_mismatch(void *buf)
+{
+	int ret;
+	u8 result[MD5_DIGEST_SIZE];
+
+	memset(result, 0, MD5_DIGEST_SIZE);
+	/* If there is no digest in suspend kernel, let it go. */
+	if (!memcmp(result, buf, MD5_DIGEST_SIZE))
+		return false;
+
+	ret = get_e820_md5(e820_saved, result);
+	if (ret)
+		return true;
+
+	return memcmp(result, buf, MD5_DIGEST_SIZE) ? true : false;
+}
+#else
+static void hibernation_e820_save(void *buf)
+{
+}
+
+static bool hibernation_e820_mismatch(void *buf)
+{
+	/* If md5 is not builtin for restore kernel, let it go. */
+	return false;
+}
+#endif
 
 /**
  *	arch_hibernation_header_save - populate the architecture specific part
@@ -201,6 +277,9 @@ int arch_hibernation_header_save(void *addr, unsigned int max_size)
 	rdr->jump_address_phys = __pa_symbol(&restore_registers);
 	rdr->cr3 = restore_cr3;
 	rdr->magic = RESTORE_MAGIC;
+
+	hibernation_e820_save(rdr->e820_digest);
+
 	return 0;
 }
 
@@ -216,5 +295,16 @@ int arch_hibernation_header_restore(void *addr)
 	restore_jump_address = rdr->jump_address;
 	jump_address_phys = rdr->jump_address_phys;
 	restore_cr3 = rdr->cr3;
-	return (rdr->magic == RESTORE_MAGIC) ? 0 : -EINVAL;
+
+	if (rdr->magic != RESTORE_MAGIC) {
+		pr_crit("Unrecognized hibernate image header format!\n");
+		return -EINVAL;
+	}
+
+	if (hibernation_e820_mismatch(rdr->e820_digest)) {
+		pr_crit("Hibernate inconsistent memory map detected!\n");
+		return -ENODEV;
+	}
+
+	return 0;
 }
diff --git a/arch/x86/xen/pci-swiotlb-xen.c b/arch/x86/xen/pci-swiotlb-xen.c
index 0e98e5d241d0..a9fafb5c8738 100644
--- a/arch/x86/xen/pci-swiotlb-xen.c
+++ b/arch/x86/xen/pci-swiotlb-xen.c
@@ -19,7 +19,6 @@
 int xen_swiotlb __read_mostly;
 
 static struct dma_map_ops xen_swiotlb_dma_ops = {
-	.mapping_error = xen_swiotlb_dma_mapping_error,
 	.alloc = xen_swiotlb_alloc_coherent,
 	.free = xen_swiotlb_free_coherent,
 	.sync_single_for_cpu = xen_swiotlb_sync_single_for_cpu,
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index f8960fca0827..8c394e30e5fe 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -41,7 +41,7 @@ struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata;
 unsigned long xen_released_pages;
 
 /* E820 map used during setting up memory. */
-static struct e820entry xen_e820_map[E820MAX] __initdata;
+static struct e820entry xen_e820_map[E820_X_MAX] __initdata;
 static u32 xen_e820_map_entries __initdata;
 
 /*
@@ -750,7 +750,7 @@ char * __init xen_memory_setup(void)
 	max_pfn = min(max_pfn, xen_start_info->nr_pages);
 	mem_end = PFN_PHYS(max_pfn);
 
-	memmap.nr_entries = E820MAX;
+	memmap.nr_entries = ARRAY_SIZE(xen_e820_map);
 	set_xen_guest_handle(memmap.buffer, xen_e820_map);
 
 	op = xen_initial_domain() ?
@@ -923,7 +923,7 @@ char * __init xen_auto_xlated_memory_setup(void)
 	int i;
 	int rc;
 
-	memmap.nr_entries = E820MAX;
+	memmap.nr_entries = ARRAY_SIZE(xen_e820_map);
 	set_xen_guest_handle(memmap.buffer, xen_e820_map);
 
 	rc = HYPERVISOR_memory_op(XENMEM_memory_map, &memmap);
diff --git a/arch/xtensa/kernel/pci-dma.c b/arch/xtensa/kernel/pci-dma.c
index 1e68806d6695..6a16decf278f 100644
--- a/arch/xtensa/kernel/pci-dma.c
+++ b/arch/xtensa/kernel/pci-dma.c
@@ -189,7 +189,9 @@ static dma_addr_t xtensa_map_page(struct device *dev, struct page *page,
 {
 	dma_addr_t dma_handle = page_to_phys(page) + offset;
 
-	xtensa_sync_single_for_device(dev, dma_handle, size, dir);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		xtensa_sync_single_for_device(dev, dma_handle, size, dir);
+
 	return dma_handle;
 }
 
@@ -197,7 +199,8 @@ static void xtensa_unmap_page(struct device *dev, dma_addr_t dma_handle,
 			      size_t size, enum dma_data_direction dir,
 			      unsigned long attrs)
 {
-	xtensa_sync_single_for_cpu(dev, dma_handle, size, dir);
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		xtensa_sync_single_for_cpu(dev, dma_handle, size, dir);
 }
 
 static int xtensa_map_sg(struct device *dev, struct scatterlist *sg,
diff --git a/block/Kconfig b/block/Kconfig
index 1d4d624492fc..8bf114a3858a 100644
--- a/block/Kconfig
+++ b/block/Kconfig
@@ -5,6 +5,7 @@ menuconfig BLOCK
        bool "Enable the block layer" if EXPERT
        default y
        select SBITMAP
+       select SRCU
        help
 	 Provide block layer support for the kernel.
 
@@ -89,6 +90,14 @@ config BLK_DEV_INTEGRITY
 	T10/SCSI Data Integrity Field or the T13/ATA External Path
 	Protection.  If in doubt, say N.
 
+config BLK_DEV_ZONED
+	bool "Zoned block device support"
+	---help---
+	Block layer zoned block device support. This option enables
+	support for ZAC/ZBC host-managed and host-aware zoned block devices.
+
+	Say yes here if you have a ZAC or ZBC storage device.
+
 config BLK_DEV_THROTTLING
 	bool "Block layer bio throttling support"
 	depends on BLK_CGROUP=y
@@ -112,6 +121,32 @@ config BLK_CMDLINE_PARSER
 
 	See Documentation/block/cmdline-partition.txt for more information.
 
+config BLK_WBT
+	bool "Enable support for block device writeback throttling"
+	default n
+	---help---
+	Enabling this option enables the block layer to throttle buffered
+	background writeback from the VM, making it more smooth and having
+	less impact on foreground operations. The throttling is done
+	dynamically on an algorithm loosely based on CoDel, factoring in
+	the realtime performance of the disk.
+
+config BLK_WBT_SQ
+	bool "Single queue writeback throttling"
+	default n
+	depends on BLK_WBT
+	---help---
+	Enable writeback throttling by default on legacy single queue devices
+
+config BLK_WBT_MQ
+	bool "Multiqueue writeback throttling"
+	default y
+	depends on BLK_WBT
+	---help---
+	Enable writeback throttling by default on multiqueue devices.
+	Multiqueue currently doesn't have support for IO scheduling,
+	enabling this option is recommended.
+
 menu "Partition Types"
 
 source "block/partitions/Kconfig"
diff --git a/block/Makefile b/block/Makefile
index 36acdd7545be..a827f988c4e6 100644
--- a/block/Makefile
+++ b/block/Makefile
@@ -5,7 +5,7 @@
 obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \
 			blk-flush.o blk-settings.o blk-ioc.o blk-map.o \
 			blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \
-			blk-lib.o blk-mq.o blk-mq-tag.o \
+			blk-lib.o blk-mq.o blk-mq-tag.o blk-stat.o \
 			blk-mq-sysfs.o blk-mq-cpumap.o ioctl.o \
 			genhd.o scsi_ioctl.o partition-generic.o ioprio.o \
 			badblocks.o partitions/
@@ -23,3 +23,5 @@ obj-$(CONFIG_BLOCK_COMPAT)	+= compat_ioctl.o
 obj-$(CONFIG_BLK_CMDLINE_PARSER)	+= cmdline-parser.o
 obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o
 obj-$(CONFIG_BLK_MQ_PCI)	+= blk-mq-pci.o
+obj-$(CONFIG_BLK_DEV_ZONED)	+= blk-zoned.o
+obj-$(CONFIG_BLK_WBT)		+= blk-wbt.o
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index 63f72f00c72e..5384713d48bc 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -172,7 +172,7 @@ bool bio_integrity_enabled(struct bio *bio)
 {
 	struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
 
-	if (!bio_is_rw(bio))
+	if (bio_op(bio) != REQ_OP_READ && bio_op(bio) != REQ_OP_WRITE)
 		return false;
 
 	/* Already protected? */
diff --git a/block/bio.c b/block/bio.c
index db85c5753a76..2b375020fc49 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -270,11 +270,15 @@ static void bio_free(struct bio *bio)
 	}
 }
 
-void bio_init(struct bio *bio)
+void bio_init(struct bio *bio, struct bio_vec *table,
+	      unsigned short max_vecs)
 {
 	memset(bio, 0, sizeof(*bio));
 	atomic_set(&bio->__bi_remaining, 1);
 	atomic_set(&bio->__bi_cnt, 1);
+
+	bio->bi_io_vec = table;
+	bio->bi_max_vecs = max_vecs;
 }
 EXPORT_SYMBOL(bio_init);
 
@@ -480,7 +484,7 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
 		return NULL;
 
 	bio = p + front_pad;
-	bio_init(bio);
+	bio_init(bio, NULL, 0);
 
 	if (nr_iovecs > inline_vecs) {
 		unsigned long idx = 0;
@@ -670,6 +674,7 @@ struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask,
 	switch (bio_op(bio)) {
 	case REQ_OP_DISCARD:
 	case REQ_OP_SECURE_ERASE:
+	case REQ_OP_WRITE_ZEROES:
 		break;
 	case REQ_OP_WRITE_SAME:
 		bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0];
@@ -847,6 +852,55 @@ done:
 }
 EXPORT_SYMBOL(bio_add_page);
 
+/**
+ * bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
+ * @bio: bio to add pages to
+ * @iter: iov iterator describing the region to be mapped
+ *
+ * Pins as many pages from *iter and appends them to @bio's bvec array. The
+ * pages will have to be released using put_page() when done.
+ */
+int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
+{
+	unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
+	struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
+	struct page **pages = (struct page **)bv;
+	size_t offset, diff;
+	ssize_t size;
+
+	size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
+	if (unlikely(size <= 0))
+		return size ? size : -EFAULT;
+	nr_pages = (size + offset + PAGE_SIZE - 1) / PAGE_SIZE;
+
+	/*
+	 * Deep magic below:  We need to walk the pinned pages backwards
+	 * because we are abusing the space allocated for the bio_vecs
+	 * for the page array.  Because the bio_vecs are larger than the
+	 * page pointers by definition this will always work.  But it also
+	 * means we can't use bio_add_page, so any changes to it's semantics
+	 * need to be reflected here as well.
+	 */
+	bio->bi_iter.bi_size += size;
+	bio->bi_vcnt += nr_pages;
+
+	diff = (nr_pages * PAGE_SIZE - offset) - size;
+	while (nr_pages--) {
+		bv[nr_pages].bv_page = pages[nr_pages];
+		bv[nr_pages].bv_len = PAGE_SIZE;
+		bv[nr_pages].bv_offset = 0;
+	}
+
+	bv[0].bv_offset += offset;
+	bv[0].bv_len -= offset;
+	if (diff)
+		bv[bio->bi_vcnt - 1].bv_len -= diff;
+
+	iov_iter_advance(iter, size);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);
+
 struct submit_bio_ret {
 	struct completion event;
 	int error;
@@ -1786,15 +1840,7 @@ struct bio *bio_split(struct bio *bio, int sectors,
 	BUG_ON(sectors <= 0);
 	BUG_ON(sectors >= bio_sectors(bio));
 
-	/*
-	 * Discards need a mutable bio_vec to accommodate the payload
-	 * required by the DSM TRIM and UNMAP commands.
-	 */
-	if (bio_op(bio) == REQ_OP_DISCARD || bio_op(bio) == REQ_OP_SECURE_ERASE)
-		split = bio_clone_bioset(bio, gfp, bs);
-	else
-		split = bio_clone_fast(bio, gfp, bs);
-
+	split = bio_clone_fast(bio, gfp, bs);
 	if (!split)
 		return NULL;
 
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index b08ccbb9393a..8ba0af780e88 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -185,7 +185,8 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
 	}
 
 	wb_congested = wb_congested_get_create(&q->backing_dev_info,
-					       blkcg->css.id, GFP_NOWAIT);
+					       blkcg->css.id,
+					       GFP_NOWAIT | __GFP_NOWARN);
 	if (!wb_congested) {
 		ret = -ENOMEM;
 		goto err_put_css;
@@ -193,7 +194,7 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
 
 	/* allocate */
 	if (!new_blkg) {
-		new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT);
+		new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);
 		if (unlikely(!new_blkg)) {
 			ret = -ENOMEM;
 			goto err_put_congested;
@@ -1022,7 +1023,7 @@ blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
 	}
 
 	spin_lock_init(&blkcg->lock);
-	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT);
+	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
 	INIT_HLIST_HEAD(&blkcg->blkg_list);
 #ifdef CONFIG_CGROUP_WRITEBACK
 	INIT_LIST_HEAD(&blkcg->cgwb_list);
@@ -1240,7 +1241,7 @@ pd_prealloc:
 		if (blkg->pd[pol->plid])
 			continue;
 
-		pd = pol->pd_alloc_fn(GFP_NOWAIT, q->node);
+		pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q->node);
 		if (!pd)
 			swap(pd, pd_prealloc);
 		if (!pd) {
diff --git a/block/blk-core.c b/block/blk-core.c
index 14d7c0740dc0..61ba08c58b64 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -39,6 +39,7 @@
 
 #include "blk.h"
 #include "blk-mq.h"
+#include "blk-wbt.h"
 
 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
@@ -145,13 +146,13 @@ static void req_bio_endio(struct request *rq, struct bio *bio,
 	if (error)
 		bio->bi_error = error;
 
-	if (unlikely(rq->cmd_flags & REQ_QUIET))
+	if (unlikely(rq->rq_flags & RQF_QUIET))
 		bio_set_flag(bio, BIO_QUIET);
 
 	bio_advance(bio, nbytes);
 
 	/* don't actually finish bio if it's part of flush sequence */
-	if (bio->bi_iter.bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ))
+	if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
 		bio_endio(bio);
 }
 
@@ -882,6 +883,7 @@ blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
 
 fail:
 	blk_free_flush_queue(q->fq);
+	wbt_exit(q);
 	return NULL;
 }
 EXPORT_SYMBOL(blk_init_allocated_queue);
@@ -899,7 +901,7 @@ EXPORT_SYMBOL(blk_get_queue);
 
 static inline void blk_free_request(struct request_list *rl, struct request *rq)
 {
-	if (rq->cmd_flags & REQ_ELVPRIV) {
+	if (rq->rq_flags & RQF_ELVPRIV) {
 		elv_put_request(rl->q, rq);
 		if (rq->elv.icq)
 			put_io_context(rq->elv.icq->ioc);
@@ -961,14 +963,14 @@ static void __freed_request(struct request_list *rl, int sync)
  * A request has just been released.  Account for it, update the full and
  * congestion status, wake up any waiters.   Called under q->queue_lock.
  */
-static void freed_request(struct request_list *rl, int op, unsigned int flags)
+static void freed_request(struct request_list *rl, bool sync,
+		req_flags_t rq_flags)
 {
 	struct request_queue *q = rl->q;
-	int sync = rw_is_sync(op, flags);
 
 	q->nr_rqs[sync]--;
 	rl->count[sync]--;
-	if (flags & REQ_ELVPRIV)
+	if (rq_flags & RQF_ELVPRIV)
 		q->nr_rqs_elvpriv--;
 
 	__freed_request(rl, sync);
@@ -1056,8 +1058,7 @@ static struct io_context *rq_ioc(struct bio *bio)
 /**
  * __get_request - get a free request
  * @rl: request list to allocate from
- * @op: REQ_OP_READ/REQ_OP_WRITE
- * @op_flags: rq_flag_bits
+ * @op: operation and flags
  * @bio: bio to allocate request for (can be %NULL)
  * @gfp_mask: allocation mask
  *
@@ -1068,22 +1069,22 @@ static struct io_context *rq_ioc(struct bio *bio)
  * Returns ERR_PTR on failure, with @q->queue_lock held.
  * Returns request pointer on success, with @q->queue_lock *not held*.
  */
-static struct request *__get_request(struct request_list *rl, int op,
-				     int op_flags, struct bio *bio,
-				     gfp_t gfp_mask)
+static struct request *__get_request(struct request_list *rl, unsigned int op,
+		struct bio *bio, gfp_t gfp_mask)
 {
 	struct request_queue *q = rl->q;
 	struct request *rq;
 	struct elevator_type *et = q->elevator->type;
 	struct io_context *ioc = rq_ioc(bio);
 	struct io_cq *icq = NULL;
-	const bool is_sync = rw_is_sync(op, op_flags) != 0;
+	const bool is_sync = op_is_sync(op);
 	int may_queue;
+	req_flags_t rq_flags = RQF_ALLOCED;
 
 	if (unlikely(blk_queue_dying(q)))
 		return ERR_PTR(-ENODEV);
 
-	may_queue = elv_may_queue(q, op, op_flags);
+	may_queue = elv_may_queue(q, op);
 	if (may_queue == ELV_MQUEUE_NO)
 		goto rq_starved;
 
@@ -1127,7 +1128,7 @@ static struct request *__get_request(struct request_list *rl, int op,
 
 	/*
 	 * Decide whether the new request will be managed by elevator.  If
-	 * so, mark @op_flags and increment elvpriv.  Non-zero elvpriv will
+	 * so, mark @rq_flags and increment elvpriv.  Non-zero elvpriv will
 	 * prevent the current elevator from being destroyed until the new
 	 * request is freed.  This guarantees icq's won't be destroyed and
 	 * makes creating new ones safe.
@@ -1136,14 +1137,14 @@ static struct request *__get_request(struct request_list *rl, int op,
 	 * it will be created after releasing queue_lock.
 	 */
 	if (blk_rq_should_init_elevator(bio) && !blk_queue_bypass(q)) {
-		op_flags |= REQ_ELVPRIV;
+		rq_flags |= RQF_ELVPRIV;
 		q->nr_rqs_elvpriv++;
 		if (et->icq_cache && ioc)
 			icq = ioc_lookup_icq(ioc, q);
 	}
 
 	if (blk_queue_io_stat(q))
-		op_flags |= REQ_IO_STAT;
+		rq_flags |= RQF_IO_STAT;
 	spin_unlock_irq(q->queue_lock);
 
 	/* allocate and init request */
@@ -1153,10 +1154,12 @@ static struct request *__get_request(struct request_list *rl, int op,
 
 	blk_rq_init(q, rq);
 	blk_rq_set_rl(rq, rl);
-	req_set_op_attrs(rq, op, op_flags | REQ_ALLOCED);
+	blk_rq_set_prio(rq, ioc);
+	rq->cmd_flags = op;
+	rq->rq_flags = rq_flags;
 
 	/* init elvpriv */
-	if (op_flags & REQ_ELVPRIV) {
+	if (rq_flags & RQF_ELVPRIV) {
 		if (unlikely(et->icq_cache && !icq)) {
 			if (ioc)
 				icq = ioc_create_icq(ioc, q, gfp_mask);
@@ -1195,7 +1198,7 @@ fail_elvpriv:
 	printk_ratelimited(KERN_WARNING "%s: dev %s: request aux data allocation failed, iosched may be disturbed\n",
 			   __func__, dev_name(q->backing_dev_info.dev));
 
-	rq->cmd_flags &= ~REQ_ELVPRIV;
+	rq->rq_flags &= ~RQF_ELVPRIV;
 	rq->elv.icq = NULL;
 
 	spin_lock_irq(q->queue_lock);
@@ -1212,7 +1215,7 @@ fail_alloc:
 	 * queue, but this is pretty rare.
 	 */
 	spin_lock_irq(q->queue_lock);
-	freed_request(rl, op, op_flags);
+	freed_request(rl, is_sync, rq_flags);
 
 	/*
 	 * in the very unlikely event that allocation failed and no
@@ -1230,8 +1233,7 @@ rq_starved:
 /**
  * get_request - get a free request
  * @q: request_queue to allocate request from
- * @op: REQ_OP_READ/REQ_OP_WRITE
- * @op_flags: rq_flag_bits
+ * @op: operation and flags
  * @bio: bio to allocate request for (can be %NULL)
  * @gfp_mask: allocation mask
  *
@@ -1242,18 +1244,17 @@ rq_starved:
  * Returns ERR_PTR on failure, with @q->queue_lock held.
  * Returns request pointer on success, with @q->queue_lock *not held*.
  */
-static struct request *get_request(struct request_queue *q, int op,
-				   int op_flags, struct bio *bio,
-				   gfp_t gfp_mask)
+static struct request *get_request(struct request_queue *q, unsigned int op,
+		struct bio *bio, gfp_t gfp_mask)
 {
-	const bool is_sync = rw_is_sync(op, op_flags) != 0;
+	const bool is_sync = op_is_sync(op);
 	DEFINE_WAIT(wait);
 	struct request_list *rl;
 	struct request *rq;
 
 	rl = blk_get_rl(q, bio);	/* transferred to @rq on success */
 retry:
-	rq = __get_request(rl, op, op_flags, bio, gfp_mask);
+	rq = __get_request(rl, op, bio, gfp_mask);
 	if (!IS_ERR(rq))
 		return rq;
 
@@ -1295,7 +1296,7 @@ static struct request *blk_old_get_request(struct request_queue *q, int rw,
 	create_io_context(gfp_mask, q->node);
 
 	spin_lock_irq(q->queue_lock);
-	rq = get_request(q, rw, 0, NULL, gfp_mask);
+	rq = get_request(q, rw, NULL, gfp_mask);
 	if (IS_ERR(rq)) {
 		spin_unlock_irq(q->queue_lock);
 		return rq;
@@ -1346,8 +1347,9 @@ void blk_requeue_request(struct request_queue *q, struct request *rq)
 	blk_delete_timer(rq);
 	blk_clear_rq_complete(rq);
 	trace_block_rq_requeue(q, rq);
+	wbt_requeue(q->rq_wb, &rq->issue_stat);
 
-	if (rq->cmd_flags & REQ_QUEUED)
+	if (rq->rq_flags & RQF_QUEUED)
 		blk_queue_end_tag(q, rq);
 
 	BUG_ON(blk_queued_rq(rq));
@@ -1409,7 +1411,7 @@ EXPORT_SYMBOL_GPL(part_round_stats);
 #ifdef CONFIG_PM
 static void blk_pm_put_request(struct request *rq)
 {
-	if (rq->q->dev && !(rq->cmd_flags & REQ_PM) && !--rq->q->nr_pending)
+	if (rq->q->dev && !(rq->rq_flags & RQF_PM) && !--rq->q->nr_pending)
 		pm_runtime_mark_last_busy(rq->q->dev);
 }
 #else
@@ -1421,6 +1423,8 @@ static inline void blk_pm_put_request(struct request *rq) {}
  */
 void __blk_put_request(struct request_queue *q, struct request *req)
 {
+	req_flags_t rq_flags = req->rq_flags;
+
 	if (unlikely(!q))
 		return;
 
@@ -1436,20 +1440,21 @@ void __blk_put_request(struct request_queue *q, struct request *req)
 	/* this is a bio leak */
 	WARN_ON(req->bio != NULL);
 
+	wbt_done(q->rq_wb, &req->issue_stat);
+
 	/*
 	 * Request may not have originated from ll_rw_blk. if not,
 	 * it didn't come out of our reserved rq pools
 	 */
-	if (req->cmd_flags & REQ_ALLOCED) {
-		unsigned int flags = req->cmd_flags;
-		int op = req_op(req);
+	if (rq_flags & RQF_ALLOCED) {
 		struct request_list *rl = blk_rq_rl(req);
+		bool sync = op_is_sync(req->cmd_flags);
 
 		BUG_ON(!list_empty(&req->queuelist));
 		BUG_ON(ELV_ON_HASH(req));
 
 		blk_free_request(rl, req);
-		freed_request(rl, op, flags);
+		freed_request(rl, sync, rq_flags);
 		blk_put_rl(rl);
 	}
 }
@@ -1471,38 +1476,6 @@ void blk_put_request(struct request *req)
 }
 EXPORT_SYMBOL(blk_put_request);
 
-/**
- * blk_add_request_payload - add a payload to a request
- * @rq: request to update
- * @page: page backing the payload
- * @offset: offset in page
- * @len: length of the payload.
- *
- * This allows to later add a payload to an already submitted request by
- * a block driver.  The driver needs to take care of freeing the payload
- * itself.
- *
- * Note that this is a quite horrible hack and nothing but handling of
- * discard requests should ever use it.
- */
-void blk_add_request_payload(struct request *rq, struct page *page,
-		int offset, unsigned int len)
-{
-	struct bio *bio = rq->bio;
-
-	bio->bi_io_vec->bv_page = page;
-	bio->bi_io_vec->bv_offset = offset;
-	bio->bi_io_vec->bv_len = len;
-
-	bio->bi_iter.bi_size = len;
-	bio->bi_vcnt = 1;
-	bio->bi_phys_segments = 1;
-
-	rq->__data_len = rq->resid_len = len;
-	rq->nr_phys_segments = 1;
-}
-EXPORT_SYMBOL_GPL(blk_add_request_payload);
-
 bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
 			    struct bio *bio)
 {
@@ -1649,24 +1622,23 @@ out:
 void init_request_from_bio(struct request *req, struct bio *bio)
 {
 	req->cmd_type = REQ_TYPE_FS;
-
-	req->cmd_flags |= bio->bi_opf & REQ_COMMON_MASK;
 	if (bio->bi_opf & REQ_RAHEAD)
 		req->cmd_flags |= REQ_FAILFAST_MASK;
 
 	req->errors = 0;
 	req->__sector = bio->bi_iter.bi_sector;
-	req->ioprio = bio_prio(bio);
+	if (ioprio_valid(bio_prio(bio)))
+		req->ioprio = bio_prio(bio);
 	blk_rq_bio_prep(req->q, req, bio);
 }
 
 static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
 {
-	const bool sync = !!(bio->bi_opf & REQ_SYNC);
 	struct blk_plug *plug;
-	int el_ret, rw_flags = 0, where = ELEVATOR_INSERT_SORT;
+	int el_ret, where = ELEVATOR_INSERT_SORT;
 	struct request *req;
 	unsigned int request_count = 0;
+	unsigned int wb_acct;
 
 	/*
 	 * low level driver can indicate that it wants pages above a
@@ -1719,30 +1691,22 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
 	}
 
 get_rq:
-	/*
-	 * This sync check and mask will be re-done in init_request_from_bio(),
-	 * but we need to set it earlier to expose the sync flag to the
-	 * rq allocator and io schedulers.
-	 */
-	if (sync)
-		rw_flags |= REQ_SYNC;
-
-	/*
-	 * Add in META/PRIO flags, if set, before we get to the IO scheduler
-	 */
-	rw_flags |= (bio->bi_opf & (REQ_META | REQ_PRIO));
+	wb_acct = wbt_wait(q->rq_wb, bio, q->queue_lock);
 
 	/*
 	 * Grab a free request. This is might sleep but can not fail.
 	 * Returns with the queue unlocked.
 	 */
-	req = get_request(q, bio_data_dir(bio), rw_flags, bio, GFP_NOIO);
+	req = get_request(q, bio->bi_opf, bio, GFP_NOIO);
 	if (IS_ERR(req)) {
+		__wbt_done(q->rq_wb, wb_acct);
 		bio->bi_error = PTR_ERR(req);
 		bio_endio(bio);
 		goto out_unlock;
 	}
 
+	wbt_track(&req->issue_stat, wb_acct);
+
 	/*
 	 * After dropping the lock and possibly sleeping here, our request
 	 * may now be mergeable after it had proven unmergeable (above).
@@ -1759,11 +1723,16 @@ get_rq:
 		/*
 		 * If this is the first request added after a plug, fire
 		 * of a plug trace.
+		 *
+		 * @request_count may become stale because of schedule
+		 * out, so check plug list again.
 		 */
-		if (!request_count)
+		if (!request_count || list_empty(&plug->list))
 			trace_block_plug(q);
 		else {
-			if (request_count >= BLK_MAX_REQUEST_COUNT) {
+			struct request *last = list_entry_rq(plug->list.prev);
+			if (request_count >= BLK_MAX_REQUEST_COUNT ||
+			    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE) {
 				blk_flush_plug_list(plug, false);
 				trace_block_plug(q);
 			}
@@ -1788,7 +1757,12 @@ static inline void blk_partition_remap(struct bio *bio)
 {
 	struct block_device *bdev = bio->bi_bdev;
 
-	if (bio_sectors(bio) && bdev != bdev->bd_contains) {
+	/*
+	 * Zone reset does not include bi_size so bio_sectors() is always 0.
+	 * Include a test for the reset op code and perform the remap if needed.
+	 */
+	if (bdev != bdev->bd_contains &&
+	    (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET)) {
 		struct hd_struct *p = bdev->bd_part;
 
 		bio->bi_iter.bi_sector += p->start_sect;
@@ -1942,6 +1916,15 @@ generic_make_request_checks(struct bio *bio)
 		if (!bdev_write_same(bio->bi_bdev))
 			goto not_supported;
 		break;
+	case REQ_OP_ZONE_REPORT:
+	case REQ_OP_ZONE_RESET:
+		if (!bdev_is_zoned(bio->bi_bdev))
+			goto not_supported;
+		break;
+	case REQ_OP_WRITE_ZEROES:
+		if (!bdev_write_zeroes_sectors(bio->bi_bdev))
+			goto not_supported;
+		break;
 	default:
 		break;
 	}
@@ -2210,7 +2193,7 @@ unsigned int blk_rq_err_bytes(const struct request *rq)
 	unsigned int bytes = 0;
 	struct bio *bio;
 
-	if (!(rq->cmd_flags & REQ_MIXED_MERGE))
+	if (!(rq->rq_flags & RQF_MIXED_MERGE))
 		return blk_rq_bytes(rq);
 
 	/*
@@ -2253,7 +2236,7 @@ void blk_account_io_done(struct request *req)
 	 * normal IO on queueing nor completion.  Accounting the
 	 * containing request is enough.
 	 */
-	if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) {
+	if (blk_do_io_stat(req) && !(req->rq_flags & RQF_FLUSH_SEQ)) {
 		unsigned long duration = jiffies - req->start_time;
 		const int rw = rq_data_dir(req);
 		struct hd_struct *part;
@@ -2281,7 +2264,7 @@ static struct request *blk_pm_peek_request(struct request_queue *q,
 					   struct request *rq)
 {
 	if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
-	    (q->rpm_status != RPM_ACTIVE && !(rq->cmd_flags & REQ_PM))))
+	    (q->rpm_status != RPM_ACTIVE && !(rq->rq_flags & RQF_PM))))
 		return NULL;
 	else
 		return rq;
@@ -2357,13 +2340,13 @@ struct request *blk_peek_request(struct request_queue *q)
 		if (!rq)
 			break;
 
-		if (!(rq->cmd_flags & REQ_STARTED)) {
+		if (!(rq->rq_flags & RQF_STARTED)) {
 			/*
 			 * This is the first time the device driver
 			 * sees this request (possibly after
 			 * requeueing).  Notify IO scheduler.
 			 */
-			if (rq->cmd_flags & REQ_SORTED)
+			if (rq->rq_flags & RQF_SORTED)
 				elv_activate_rq(q, rq);
 
 			/*
@@ -2371,7 +2354,7 @@ struct request *blk_peek_request(struct request_queue *q)
 			 * it, a request that has been delayed should
 			 * not be passed by new incoming requests
 			 */
-			rq->cmd_flags |= REQ_STARTED;
+			rq->rq_flags |= RQF_STARTED;
 			trace_block_rq_issue(q, rq);
 		}
 
@@ -2380,7 +2363,7 @@ struct request *blk_peek_request(struct request_queue *q)
 			q->boundary_rq = NULL;
 		}
 
-		if (rq->cmd_flags & REQ_DONTPREP)
+		if (rq->rq_flags & RQF_DONTPREP)
 			break;
 
 		if (q->dma_drain_size && blk_rq_bytes(rq)) {
@@ -2403,11 +2386,11 @@ struct request *blk_peek_request(struct request_queue *q)
 			/*
 			 * the request may have been (partially) prepped.
 			 * we need to keep this request in the front to
-			 * avoid resource deadlock.  REQ_STARTED will
+			 * avoid resource deadlock.  RQF_STARTED will
 			 * prevent other fs requests from passing this one.
 			 */
 			if (q->dma_drain_size && blk_rq_bytes(rq) &&
-			    !(rq->cmd_flags & REQ_DONTPREP)) {
+			    !(rq->rq_flags & RQF_DONTPREP)) {
 				/*
 				 * remove the space for the drain we added
 				 * so that we don't add it again
@@ -2420,7 +2403,7 @@ struct request *blk_peek_request(struct request_queue *q)
 		} else if (ret == BLKPREP_KILL || ret == BLKPREP_INVALID) {
 			int err = (ret == BLKPREP_INVALID) ? -EREMOTEIO : -EIO;
 
-			rq->cmd_flags |= REQ_QUIET;
+			rq->rq_flags |= RQF_QUIET;
 			/*
 			 * Mark this request as started so we don't trigger
 			 * any debug logic in the end I/O path.
@@ -2475,6 +2458,12 @@ void blk_start_request(struct request *req)
 {
 	blk_dequeue_request(req);
 
+	if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {
+		blk_stat_set_issue_time(&req->issue_stat);
+		req->rq_flags |= RQF_STATS;
+		wbt_issue(req->q->rq_wb, &req->issue_stat);
+	}
+
 	/*
 	 * We are now handing the request to the hardware, initialize
 	 * resid_len to full count and add the timeout handler.
@@ -2557,7 +2546,7 @@ bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
 		req->errors = 0;
 
 	if (error && req->cmd_type == REQ_TYPE_FS &&
-	    !(req->cmd_flags & REQ_QUIET)) {
+	    !(req->rq_flags & RQF_QUIET)) {
 		char *error_type;
 
 		switch (error) {
@@ -2623,6 +2612,8 @@ bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
 		return false;
 	}
 
+	WARN_ON_ONCE(req->rq_flags & RQF_SPECIAL_PAYLOAD);
+
 	req->__data_len -= total_bytes;
 
 	/* update sector only for requests with clear definition of sector */
@@ -2630,7 +2621,7 @@ bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
 		req->__sector += total_bytes >> 9;
 
 	/* mixed attributes always follow the first bio */
-	if (req->cmd_flags & REQ_MIXED_MERGE) {
+	if (req->rq_flags & RQF_MIXED_MERGE) {
 		req->cmd_flags &= ~REQ_FAILFAST_MASK;
 		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
 	}
@@ -2683,7 +2674,7 @@ void blk_unprep_request(struct request *req)
 {
 	struct request_queue *q = req->q;
 
-	req->cmd_flags &= ~REQ_DONTPREP;
+	req->rq_flags &= ~RQF_DONTPREP;
 	if (q->unprep_rq_fn)
 		q->unprep_rq_fn(q, req);
 }
@@ -2694,8 +2685,13 @@ EXPORT_SYMBOL_GPL(blk_unprep_request);
  */
 void blk_finish_request(struct request *req, int error)
 {
-	if (req->cmd_flags & REQ_QUEUED)
-		blk_queue_end_tag(req->q, req);
+	struct request_queue *q = req->q;
+
+	if (req->rq_flags & RQF_STATS)
+		blk_stat_add(&q->rq_stats[rq_data_dir(req)], req);
+
+	if (req->rq_flags & RQF_QUEUED)
+		blk_queue_end_tag(q, req);
 
 	BUG_ON(blk_queued_rq(req));
 
@@ -2704,18 +2700,19 @@ void blk_finish_request(struct request *req, int error)
 
 	blk_delete_timer(req);
 
-	if (req->cmd_flags & REQ_DONTPREP)
+	if (req->rq_flags & RQF_DONTPREP)
 		blk_unprep_request(req);
 
 	blk_account_io_done(req);
 
-	if (req->end_io)
+	if (req->end_io) {
+		wbt_done(req->q->rq_wb, &req->issue_stat);
 		req->end_io(req, error);
-	else {
+	} else {
 		if (blk_bidi_rq(req))
 			__blk_put_request(req->next_rq->q, req->next_rq);
 
-		__blk_put_request(req->q, req);
+		__blk_put_request(q, req);
 	}
 }
 EXPORT_SYMBOL(blk_finish_request);
@@ -2939,8 +2936,6 @@ EXPORT_SYMBOL_GPL(__blk_end_request_err);
 void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
 		     struct bio *bio)
 {
-	req_set_op(rq, bio_op(bio));
-
 	if (bio_has_data(bio))
 		rq->nr_phys_segments = bio_phys_segments(q, bio);
 
@@ -3024,8 +3019,7 @@ EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
 static void __blk_rq_prep_clone(struct request *dst, struct request *src)
 {
 	dst->cpu = src->cpu;
-	req_set_op_attrs(dst, req_op(src),
-			 (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE);
+	dst->cmd_flags = src->cmd_flags | REQ_NOMERGE;
 	dst->cmd_type = src->cmd_type;
 	dst->__sector = blk_rq_pos(src);
 	dst->__data_len = blk_rq_bytes(src);
@@ -3303,52 +3297,6 @@ void blk_finish_plug(struct blk_plug *plug)
 }
 EXPORT_SYMBOL(blk_finish_plug);
 
-bool blk_poll(struct request_queue *q, blk_qc_t cookie)
-{
-	struct blk_plug *plug;
-	long state;
-	unsigned int queue_num;
-	struct blk_mq_hw_ctx *hctx;
-
-	if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
-	    !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
-		return false;
-
-	queue_num = blk_qc_t_to_queue_num(cookie);
-	hctx = q->queue_hw_ctx[queue_num];
-	hctx->poll_considered++;
-
-	plug = current->plug;
-	if (plug)
-		blk_flush_plug_list(plug, false);
-
-	state = current->state;
-	while (!need_resched()) {
-		int ret;
-
-		hctx->poll_invoked++;
-
-		ret = q->mq_ops->poll(hctx, blk_qc_t_to_tag(cookie));
-		if (ret > 0) {
-			hctx->poll_success++;
-			set_current_state(TASK_RUNNING);
-			return true;
-		}
-
-		if (signal_pending_state(state, current))
-			set_current_state(TASK_RUNNING);
-
-		if (current->state == TASK_RUNNING)
-			return true;
-		if (ret < 0)
-			break;
-		cpu_relax();
-	}
-
-	return false;
-}
-EXPORT_SYMBOL_GPL(blk_poll);
-
 #ifdef CONFIG_PM
 /**
  * blk_pm_runtime_init - Block layer runtime PM initialization routine
@@ -3530,8 +3478,11 @@ EXPORT_SYMBOL(blk_set_runtime_active);
 
 int __init blk_dev_init(void)
 {
-	BUILD_BUG_ON(__REQ_NR_BITS > 8 *
+	BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
+	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
 			FIELD_SIZEOF(struct request, cmd_flags));
+	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
+			FIELD_SIZEOF(struct bio, bi_opf));
 
 	/* used for unplugging and affects IO latency/throughput - HIGHPRI */
 	kblockd_workqueue = alloc_workqueue("kblockd",
diff --git a/block/blk-exec.c b/block/blk-exec.c
index 7ea04325d02f..3ecb00a6cf45 100644
--- a/block/blk-exec.c
+++ b/block/blk-exec.c
@@ -72,7 +72,7 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
 	spin_lock_irq(q->queue_lock);
 
 	if (unlikely(blk_queue_dying(q))) {
-		rq->cmd_flags |= REQ_QUIET; 
+		rq->rq_flags |= RQF_QUIET;
 		rq->errors = -ENXIO;
 		__blk_end_request_all(rq, rq->errors);
 		spin_unlock_irq(q->queue_lock);
diff --git a/block/blk-flush.c b/block/blk-flush.c
index 3c882cbc7541..20b7c7a02f1c 100644
--- a/block/blk-flush.c
+++ b/block/blk-flush.c
@@ -56,7 +56,7 @@
  * Once while executing DATA and again after the whole sequence is
  * complete.  The first completion updates the contained bio but doesn't
  * finish it so that the bio submitter is notified only after the whole
- * sequence is complete.  This is implemented by testing REQ_FLUSH_SEQ in
+ * sequence is complete.  This is implemented by testing RQF_FLUSH_SEQ in
  * req_bio_endio().
  *
  * The above peculiarity requires that each FLUSH/FUA request has only one
@@ -127,17 +127,14 @@ static void blk_flush_restore_request(struct request *rq)
 	rq->bio = rq->biotail;
 
 	/* make @rq a normal request */
-	rq->cmd_flags &= ~REQ_FLUSH_SEQ;
+	rq->rq_flags &= ~RQF_FLUSH_SEQ;
 	rq->end_io = rq->flush.saved_end_io;
 }
 
 static bool blk_flush_queue_rq(struct request *rq, bool add_front)
 {
 	if (rq->q->mq_ops) {
-		struct request_queue *q = rq->q;
-
-		blk_mq_add_to_requeue_list(rq, add_front);
-		blk_mq_kick_requeue_list(q);
+		blk_mq_add_to_requeue_list(rq, add_front, true);
 		return false;
 	} else {
 		if (add_front)
@@ -330,7 +327,8 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq)
 	}
 
 	flush_rq->cmd_type = REQ_TYPE_FS;
-	req_set_op_attrs(flush_rq, REQ_OP_FLUSH, WRITE_FLUSH | REQ_FLUSH_SEQ);
+	flush_rq->cmd_flags = REQ_OP_FLUSH | REQ_PREFLUSH;
+	flush_rq->rq_flags |= RQF_FLUSH_SEQ;
 	flush_rq->rq_disk = first_rq->rq_disk;
 	flush_rq->end_io = flush_end_io;
 
@@ -368,7 +366,7 @@ static void flush_data_end_io(struct request *rq, int error)
 	elv_completed_request(q, rq);
 
 	/* for avoiding double accounting */
-	rq->cmd_flags &= ~REQ_STARTED;
+	rq->rq_flags &= ~RQF_STARTED;
 
 	/*
 	 * After populating an empty queue, kick it to avoid stall.  Read
@@ -426,6 +424,13 @@ void blk_insert_flush(struct request *rq)
 		rq->cmd_flags &= ~REQ_FUA;
 
 	/*
+	 * REQ_PREFLUSH|REQ_FUA implies REQ_SYNC, so if we clear any
+	 * of those flags, we have to set REQ_SYNC to avoid skewing
+	 * the request accounting.
+	 */
+	rq->cmd_flags |= REQ_SYNC;
+
+	/*
 	 * An empty flush handed down from a stacking driver may
 	 * translate into nothing if the underlying device does not
 	 * advertise a write-back cache.  In this case, simply
@@ -449,7 +454,7 @@ void blk_insert_flush(struct request *rq)
 	if ((policy & REQ_FSEQ_DATA) &&
 	    !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) {
 		if (q->mq_ops) {
-			blk_mq_insert_request(rq, false, false, true);
+			blk_mq_insert_request(rq, false, true, false);
 		} else
 			list_add_tail(&rq->queuelist, &q->queue_head);
 		return;
@@ -461,7 +466,7 @@ void blk_insert_flush(struct request *rq)
 	 */
 	memset(&rq->flush, 0, sizeof(rq->flush));
 	INIT_LIST_HEAD(&rq->flush.list);
-	rq->cmd_flags |= REQ_FLUSH_SEQ;
+	rq->rq_flags |= RQF_FLUSH_SEQ;
 	rq->flush.saved_end_io = rq->end_io; /* Usually NULL */
 	if (q->mq_ops) {
 		rq->end_io = mq_flush_data_end_io;
@@ -513,7 +518,7 @@ int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
 
 	bio = bio_alloc(gfp_mask, 0);
 	bio->bi_bdev = bdev;
-	bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_FLUSH);
+	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 
 	ret = submit_bio_wait(bio);
 
diff --git a/block/blk-lib.c b/block/blk-lib.c
index 46fe9248410d..ed89c8f4b2a0 100644
--- a/block/blk-lib.c
+++ b/block/blk-lib.c
@@ -29,7 +29,7 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
 	struct request_queue *q = bdev_get_queue(bdev);
 	struct bio *bio = *biop;
 	unsigned int granularity;
-	enum req_op op;
+	unsigned int op;
 	int alignment;
 	sector_t bs_mask;
 
@@ -80,7 +80,7 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
 			req_sects = end_sect - sector;
 		}
 
-		bio = next_bio(bio, 1, gfp_mask);
+		bio = next_bio(bio, 0, gfp_mask);
 		bio->bi_iter.bi_sector = sector;
 		bio->bi_bdev = bdev;
 		bio_set_op_attrs(bio, op, 0);
@@ -137,24 +137,24 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
 EXPORT_SYMBOL(blkdev_issue_discard);
 
 /**
- * blkdev_issue_write_same - queue a write same operation
+ * __blkdev_issue_write_same - generate number of bios with same page
  * @bdev:	target blockdev
  * @sector:	start sector
  * @nr_sects:	number of sectors to write
  * @gfp_mask:	memory allocation flags (for bio_alloc)
  * @page:	page containing data to write
+ * @biop:	pointer to anchor bio
  *
  * Description:
- *    Issue a write same request for the sectors in question.
+ *  Generate and issue number of bios(REQ_OP_WRITE_SAME) with same page.
  */
-int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
-			    sector_t nr_sects, gfp_t gfp_mask,
-			    struct page *page)
+static int __blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
+		sector_t nr_sects, gfp_t gfp_mask, struct page *page,
+		struct bio **biop)
 {
 	struct request_queue *q = bdev_get_queue(bdev);
 	unsigned int max_write_same_sectors;
-	struct bio *bio = NULL;
-	int ret = 0;
+	struct bio *bio = *biop;
 	sector_t bs_mask;
 
 	if (!q)
@@ -164,6 +164,9 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
 	if ((sector | nr_sects) & bs_mask)
 		return -EINVAL;
 
+	if (!bdev_write_same(bdev))
+		return -EOPNOTSUPP;
+
 	/* Ensure that max_write_same_sectors doesn't overflow bi_size */
 	max_write_same_sectors = UINT_MAX >> 9;
 
@@ -185,32 +188,112 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
 			bio->bi_iter.bi_size = nr_sects << 9;
 			nr_sects = 0;
 		}
+		cond_resched();
 	}
 
-	if (bio) {
+	*biop = bio;
+	return 0;
+}
+
+/**
+ * blkdev_issue_write_same - queue a write same operation
+ * @bdev:	target blockdev
+ * @sector:	start sector
+ * @nr_sects:	number of sectors to write
+ * @gfp_mask:	memory allocation flags (for bio_alloc)
+ * @page:	page containing data
+ *
+ * Description:
+ *    Issue a write same request for the sectors in question.
+ */
+int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
+				sector_t nr_sects, gfp_t gfp_mask,
+				struct page *page)
+{
+	struct bio *bio = NULL;
+	struct blk_plug plug;
+	int ret;
+
+	blk_start_plug(&plug);
+	ret = __blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask, page,
+			&bio);
+	if (ret == 0 && bio) {
 		ret = submit_bio_wait(bio);
 		bio_put(bio);
 	}
+	blk_finish_plug(&plug);
 	return ret;
 }
 EXPORT_SYMBOL(blkdev_issue_write_same);
 
 /**
- * blkdev_issue_zeroout - generate number of zero filed write bios
+ * __blkdev_issue_write_zeroes - generate number of bios with WRITE ZEROES
  * @bdev:	blockdev to issue
  * @sector:	start sector
  * @nr_sects:	number of sectors to write
  * @gfp_mask:	memory allocation flags (for bio_alloc)
+ * @biop:	pointer to anchor bio
  *
  * Description:
- *  Generate and issue number of bios with zerofiled pages.
+ *  Generate and issue number of bios(REQ_OP_WRITE_ZEROES) with zerofiled pages.
  */
+static int __blkdev_issue_write_zeroes(struct block_device *bdev,
+		sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
+		struct bio **biop)
+{
+	struct bio *bio = *biop;
+	unsigned int max_write_zeroes_sectors;
+	struct request_queue *q = bdev_get_queue(bdev);
+
+	if (!q)
+		return -ENXIO;
+
+	/* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */
+	max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev);
+
+	if (max_write_zeroes_sectors == 0)
+		return -EOPNOTSUPP;
+
+	while (nr_sects) {
+		bio = next_bio(bio, 0, gfp_mask);
+		bio->bi_iter.bi_sector = sector;
+		bio->bi_bdev = bdev;
+		bio_set_op_attrs(bio, REQ_OP_WRITE_ZEROES, 0);
 
-static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
-				  sector_t nr_sects, gfp_t gfp_mask)
+		if (nr_sects > max_write_zeroes_sectors) {
+			bio->bi_iter.bi_size = max_write_zeroes_sectors << 9;
+			nr_sects -= max_write_zeroes_sectors;
+			sector += max_write_zeroes_sectors;
+		} else {
+			bio->bi_iter.bi_size = nr_sects << 9;
+			nr_sects = 0;
+		}
+		cond_resched();
+	}
+
+	*biop = bio;
+	return 0;
+}
+
+/**
+ * __blkdev_issue_zeroout - generate number of zero filed write bios
+ * @bdev:	blockdev to issue
+ * @sector:	start sector
+ * @nr_sects:	number of sectors to write
+ * @gfp_mask:	memory allocation flags (for bio_alloc)
+ * @biop:	pointer to anchor bio
+ * @discard:	discard flag
+ *
+ * Description:
+ *  Generate and issue number of bios with zerofiled pages.
+ */
+int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
+		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
+		bool discard)
 {
 	int ret;
-	struct bio *bio = NULL;
+	int bi_size = 0;
+	struct bio *bio = *biop;
 	unsigned int sz;
 	sector_t bs_mask;
 
@@ -218,6 +301,24 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
 	if ((sector | nr_sects) & bs_mask)
 		return -EINVAL;
 
+	if (discard) {
+		ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask,
+				BLKDEV_DISCARD_ZERO, biop);
+		if (ret == 0 || (ret && ret != -EOPNOTSUPP))
+			goto out;
+	}
+
+	ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask,
+			biop);
+	if (ret == 0 || (ret && ret != -EOPNOTSUPP))
+		goto out;
+
+	ret = __blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask,
+			ZERO_PAGE(0), biop);
+	if (ret == 0 || (ret && ret != -EOPNOTSUPP))
+		goto out;
+
+	ret = 0;
 	while (nr_sects != 0) {
 		bio = next_bio(bio, min(nr_sects, (sector_t)BIO_MAX_PAGES),
 				gfp_mask);
@@ -227,21 +328,20 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
 
 		while (nr_sects != 0) {
 			sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects);
-			ret = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0);
-			nr_sects -= ret >> 9;
-			sector += ret >> 9;
-			if (ret < (sz << 9))
+			bi_size = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0);
+			nr_sects -= bi_size >> 9;
+			sector += bi_size >> 9;
+			if (bi_size < (sz << 9))
 				break;
 		}
+		cond_resched();
 	}
 
-	if (bio) {
-		ret = submit_bio_wait(bio);
-		bio_put(bio);
-		return ret;
-	}
-	return 0;
+	*biop = bio;
+out:
+	return ret;
 }
+EXPORT_SYMBOL(__blkdev_issue_zeroout);
 
 /**
  * blkdev_issue_zeroout - zero-fill a block range
@@ -258,26 +358,27 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
  *  the discard request fail, if the discard flag is not set, or if
  *  discard_zeroes_data is not supported, this function will resort to
  *  zeroing the blocks manually, thus provisioning (allocating,
- *  anchoring) them. If the block device supports the WRITE SAME command
- *  blkdev_issue_zeroout() will use it to optimize the process of
+ *  anchoring) them. If the block device supports WRITE ZEROES or WRITE SAME
+ *  command(s), blkdev_issue_zeroout() will use it to optimize the process of
  *  clearing the block range. Otherwise the zeroing will be performed
  *  using regular WRITE calls.
  */
-
 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
 			 sector_t nr_sects, gfp_t gfp_mask, bool discard)
 {
-	if (discard) {
-		if (!blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask,
-				BLKDEV_DISCARD_ZERO))
-			return 0;
-	}
+	int ret;
+	struct bio *bio = NULL;
+	struct blk_plug plug;
 
-	if (bdev_write_same(bdev) &&
-	    blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask,
-				    ZERO_PAGE(0)) == 0)
-		return 0;
+	blk_start_plug(&plug);
+	ret = __blkdev_issue_zeroout(bdev, sector, nr_sects, gfp_mask,
+			&bio, discard);
+	if (ret == 0 && bio) {
+		ret = submit_bio_wait(bio);
+		bio_put(bio);
+	}
+	blk_finish_plug(&plug);
 
-	return __blkdev_issue_zeroout(bdev, sector, nr_sects, gfp_mask);
+	return ret;
 }
 EXPORT_SYMBOL(blkdev_issue_zeroout);
diff --git a/block/blk-map.c b/block/blk-map.c
index 27fd8d92892d..0acb6640ead7 100644
--- a/block/blk-map.c
+++ b/block/blk-map.c
@@ -16,6 +16,8 @@
 int blk_rq_append_bio(struct request *rq, struct bio *bio)
 {
 	if (!rq->bio) {
+		rq->cmd_flags &= REQ_OP_MASK;
+		rq->cmd_flags |= (bio->bi_opf & REQ_OP_MASK);
 		blk_rq_bio_prep(rq->q, rq, bio);
 	} else {
 		if (!ll_back_merge_fn(rq->q, rq, bio))
@@ -138,7 +140,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
 	} while (iov_iter_count(&i));
 
 	if (!bio_flagged(bio, BIO_USER_MAPPED))
-		rq->cmd_flags |= REQ_COPY_USER;
+		rq->rq_flags |= RQF_COPY_USER;
 	return 0;
 
 unmap_rq:
@@ -236,7 +238,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
 		bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
 
 	if (do_copy)
-		rq->cmd_flags |= REQ_COPY_USER;
+		rq->rq_flags |= RQF_COPY_USER;
 
 	ret = blk_rq_append_bio(rq, bio);
 	if (unlikely(ret)) {
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 2642e5fc8b69..182398cb1524 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -199,6 +199,10 @@ void blk_queue_split(struct request_queue *q, struct bio **bio,
 	case REQ_OP_SECURE_ERASE:
 		split = blk_bio_discard_split(q, *bio, bs, &nsegs);
 		break;
+	case REQ_OP_WRITE_ZEROES:
+		split = NULL;
+		nsegs = (*bio)->bi_phys_segments;
+		break;
 	case REQ_OP_WRITE_SAME:
 		split = blk_bio_write_same_split(q, *bio, bs, &nsegs);
 		break;
@@ -237,15 +241,14 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
 	if (!bio)
 		return 0;
 
-	/*
-	 * This should probably be returning 0, but blk_add_request_payload()
-	 * (Christoph!!!!)
-	 */
-	if (bio_op(bio) == REQ_OP_DISCARD || bio_op(bio) == REQ_OP_SECURE_ERASE)
-		return 1;
-
-	if (bio_op(bio) == REQ_OP_WRITE_SAME)
+	switch (bio_op(bio)) {
+	case REQ_OP_DISCARD:
+	case REQ_OP_SECURE_ERASE:
+	case REQ_OP_WRITE_ZEROES:
+		return 0;
+	case REQ_OP_WRITE_SAME:
 		return 1;
+	}
 
 	fbio = bio;
 	cluster = blk_queue_cluster(q);
@@ -402,38 +405,21 @@ new_segment:
 	*bvprv = *bvec;
 }
 
+static inline int __blk_bvec_map_sg(struct request_queue *q, struct bio_vec bv,
+		struct scatterlist *sglist, struct scatterlist **sg)
+{
+	*sg = sglist;
+	sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
+	return 1;
+}
+
 static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio,
 			     struct scatterlist *sglist,
 			     struct scatterlist **sg)
 {
 	struct bio_vec bvec, bvprv = { NULL };
 	struct bvec_iter iter;
-	int nsegs, cluster;
-
-	nsegs = 0;
-	cluster = blk_queue_cluster(q);
-
-	switch (bio_op(bio)) {
-	case REQ_OP_DISCARD:
-	case REQ_OP_SECURE_ERASE:
-		/*
-		 * This is a hack - drivers should be neither modifying the
-		 * biovec, nor relying on bi_vcnt - but because of
-		 * blk_add_request_payload(), a discard bio may or may not have
-		 * a payload we need to set up here (thank you Christoph) and
-		 * bi_vcnt is really the only way of telling if we need to.
-		 */
-		if (!bio->bi_vcnt)
-			return 0;
-		/* Fall through */
-	case REQ_OP_WRITE_SAME:
-		*sg = sglist;
-		bvec = bio_iovec(bio);
-		sg_set_page(*sg, bvec.bv_page, bvec.bv_len, bvec.bv_offset);
-		return 1;
-	default:
-		break;
-	}
+	int cluster = blk_queue_cluster(q), nsegs = 0;
 
 	for_each_bio(bio)
 		bio_for_each_segment(bvec, bio, iter)
@@ -453,10 +439,14 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq,
 	struct scatterlist *sg = NULL;
 	int nsegs = 0;
 
-	if (rq->bio)
+	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
+		nsegs = __blk_bvec_map_sg(q, rq->special_vec, sglist, &sg);
+	else if (rq->bio && bio_op(rq->bio) == REQ_OP_WRITE_SAME)
+		nsegs = __blk_bvec_map_sg(q, bio_iovec(rq->bio), sglist, &sg);
+	else if (rq->bio)
 		nsegs = __blk_bios_map_sg(q, rq->bio, sglist, &sg);
 
-	if (unlikely(rq->cmd_flags & REQ_COPY_USER) &&
+	if (unlikely(rq->rq_flags & RQF_COPY_USER) &&
 	    (blk_rq_bytes(rq) & q->dma_pad_mask)) {
 		unsigned int pad_len =
 			(q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1;
@@ -486,12 +476,19 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq,
 	 * Something must have been wrong if the figured number of
 	 * segment is bigger than number of req's physical segments
 	 */
-	WARN_ON(nsegs > rq->nr_phys_segments);
+	WARN_ON(nsegs > blk_rq_nr_phys_segments(rq));
 
 	return nsegs;
 }
 EXPORT_SYMBOL(blk_rq_map_sg);
 
+static void req_set_nomerge(struct request_queue *q, struct request *req)
+{
+	req->cmd_flags |= REQ_NOMERGE;
+	if (req == q->last_merge)
+		q->last_merge = NULL;
+}
+
 static inline int ll_new_hw_segment(struct request_queue *q,
 				    struct request *req,
 				    struct bio *bio)
@@ -512,9 +509,7 @@ static inline int ll_new_hw_segment(struct request_queue *q,
 	return 1;
 
 no_merge:
-	req->cmd_flags |= REQ_NOMERGE;
-	if (req == q->last_merge)
-		q->last_merge = NULL;
+	req_set_nomerge(q, req);
 	return 0;
 }
 
@@ -528,9 +523,7 @@ int ll_back_merge_fn(struct request_queue *q, struct request *req,
 		return 0;
 	if (blk_rq_sectors(req) + bio_sectors(bio) >
 	    blk_rq_get_max_sectors(req, blk_rq_pos(req))) {
-		req->cmd_flags |= REQ_NOMERGE;
-		if (req == q->last_merge)
-			q->last_merge = NULL;
+		req_set_nomerge(q, req);
 		return 0;
 	}
 	if (!bio_flagged(req->biotail, BIO_SEG_VALID))
@@ -552,9 +545,7 @@ int ll_front_merge_fn(struct request_queue *q, struct request *req,
 		return 0;
 	if (blk_rq_sectors(req) + bio_sectors(bio) >
 	    blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {
-		req->cmd_flags |= REQ_NOMERGE;
-		if (req == q->last_merge)
-			q->last_merge = NULL;
+		req_set_nomerge(q, req);
 		return 0;
 	}
 	if (!bio_flagged(bio, BIO_SEG_VALID))
@@ -634,7 +625,7 @@ void blk_rq_set_mixed_merge(struct request *rq)
 	unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
 	struct bio *bio;
 
-	if (rq->cmd_flags & REQ_MIXED_MERGE)
+	if (rq->rq_flags & RQF_MIXED_MERGE)
 		return;
 
 	/*
@@ -647,7 +638,7 @@ void blk_rq_set_mixed_merge(struct request *rq)
 			     (bio->bi_opf & REQ_FAILFAST_MASK) != ff);
 		bio->bi_opf |= ff;
 	}
-	rq->cmd_flags |= REQ_MIXED_MERGE;
+	rq->rq_flags |= RQF_MIXED_MERGE;
 }
 
 static void blk_account_io_merge(struct request *req)
@@ -709,7 +700,7 @@ static int attempt_merge(struct request_queue *q, struct request *req,
 	 * makes sure that all involved bios have mixable attributes
 	 * set properly.
 	 */
-	if ((req->cmd_flags | next->cmd_flags) & REQ_MIXED_MERGE ||
+	if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) ||
 	    (req->cmd_flags & REQ_FAILFAST_MASK) !=
 	    (next->cmd_flags & REQ_FAILFAST_MASK)) {
 		blk_rq_set_mixed_merge(req);
diff --git a/block/blk-mq-cpumap.c b/block/blk-mq-cpumap.c
index 19b1d9c5f07e..8e61e8640e17 100644
--- a/block/blk-mq-cpumap.c
+++ b/block/blk-mq-cpumap.c
@@ -87,6 +87,7 @@ int blk_mq_map_queues(struct blk_mq_tag_set *set)
 	free_cpumask_var(cpus);
 	return 0;
 }
+EXPORT_SYMBOL_GPL(blk_mq_map_queues);
 
 /*
  * We have no quick way of doing reverse lookups. This is only used at
diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c
index 01fb455d3377..eacd3af72099 100644
--- a/block/blk-mq-sysfs.c
+++ b/block/blk-mq-sysfs.c
@@ -259,6 +259,47 @@ static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
 	return ret;
 }
 
+static void blk_mq_stat_clear(struct blk_mq_hw_ctx *hctx)
+{
+	struct blk_mq_ctx *ctx;
+	unsigned int i;
+
+	hctx_for_each_ctx(hctx, ctx, i) {
+		blk_stat_init(&ctx->stat[BLK_STAT_READ]);
+		blk_stat_init(&ctx->stat[BLK_STAT_WRITE]);
+	}
+}
+
+static ssize_t blk_mq_hw_sysfs_stat_store(struct blk_mq_hw_ctx *hctx,
+					  const char *page, size_t count)
+{
+	blk_mq_stat_clear(hctx);
+	return count;
+}
+
+static ssize_t print_stat(char *page, struct blk_rq_stat *stat, const char *pre)
+{
+	return sprintf(page, "%s samples=%llu, mean=%lld, min=%lld, max=%lld\n",
+			pre, (long long) stat->nr_samples,
+			(long long) stat->mean, (long long) stat->min,
+			(long long) stat->max);
+}
+
+static ssize_t blk_mq_hw_sysfs_stat_show(struct blk_mq_hw_ctx *hctx, char *page)
+{
+	struct blk_rq_stat stat[2];
+	ssize_t ret;
+
+	blk_stat_init(&stat[BLK_STAT_READ]);
+	blk_stat_init(&stat[BLK_STAT_WRITE]);
+
+	blk_hctx_stat_get(hctx, stat);
+
+	ret = print_stat(page, &stat[BLK_STAT_READ], "read :");
+	ret += print_stat(page + ret, &stat[BLK_STAT_WRITE], "write:");
+	return ret;
+}
+
 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_dispatched = {
 	.attr = {.name = "dispatched", .mode = S_IRUGO },
 	.show = blk_mq_sysfs_dispatched_show,
@@ -317,6 +358,11 @@ static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_poll = {
 	.show = blk_mq_hw_sysfs_poll_show,
 	.store = blk_mq_hw_sysfs_poll_store,
 };
+static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_stat = {
+	.attr = {.name = "stats", .mode = S_IRUGO | S_IWUSR },
+	.show = blk_mq_hw_sysfs_stat_show,
+	.store = blk_mq_hw_sysfs_stat_store,
+};
 
 static struct attribute *default_hw_ctx_attrs[] = {
 	&blk_mq_hw_sysfs_queued.attr,
@@ -327,6 +373,7 @@ static struct attribute *default_hw_ctx_attrs[] = {
 	&blk_mq_hw_sysfs_cpus.attr,
 	&blk_mq_hw_sysfs_active.attr,
 	&blk_mq_hw_sysfs_poll.attr,
+	&blk_mq_hw_sysfs_stat.attr,
 	NULL,
 };
 
diff --git a/block/blk-mq.c b/block/blk-mq.c
index f3d27a6dee09..4bf850e8d6b5 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -30,6 +30,8 @@
 #include "blk.h"
 #include "blk-mq.h"
 #include "blk-mq-tag.h"
+#include "blk-stat.h"
+#include "blk-wbt.h"
 
 static DEFINE_MUTEX(all_q_mutex);
 static LIST_HEAD(all_q_list);
@@ -115,6 +117,33 @@ void blk_mq_unfreeze_queue(struct request_queue *q)
 }
 EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
 
+/**
+ * blk_mq_quiesce_queue() - wait until all ongoing queue_rq calls have finished
+ * @q: request queue.
+ *
+ * Note: this function does not prevent that the struct request end_io()
+ * callback function is invoked. Additionally, it is not prevented that
+ * new queue_rq() calls occur unless the queue has been stopped first.
+ */
+void blk_mq_quiesce_queue(struct request_queue *q)
+{
+	struct blk_mq_hw_ctx *hctx;
+	unsigned int i;
+	bool rcu = false;
+
+	blk_mq_stop_hw_queues(q);
+
+	queue_for_each_hw_ctx(q, hctx, i) {
+		if (hctx->flags & BLK_MQ_F_BLOCKING)
+			synchronize_srcu(&hctx->queue_rq_srcu);
+		else
+			rcu = true;
+	}
+	if (rcu)
+		synchronize_rcu();
+}
+EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
+
 void blk_mq_wake_waiters(struct request_queue *q)
 {
 	struct blk_mq_hw_ctx *hctx;
@@ -139,17 +168,15 @@ bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
 EXPORT_SYMBOL(blk_mq_can_queue);
 
 static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
-			       struct request *rq, int op,
-			       unsigned int op_flags)
+			       struct request *rq, unsigned int op)
 {
-	if (blk_queue_io_stat(q))
-		op_flags |= REQ_IO_STAT;
-
 	INIT_LIST_HEAD(&rq->queuelist);
 	/* csd/requeue_work/fifo_time is initialized before use */
 	rq->q = q;
 	rq->mq_ctx = ctx;
-	req_set_op_attrs(rq, op, op_flags);
+	rq->cmd_flags = op;
+	if (blk_queue_io_stat(q))
+		rq->rq_flags |= RQF_IO_STAT;
 	/* do not touch atomic flags, it needs atomic ops against the timer */
 	rq->cpu = -1;
 	INIT_HLIST_NODE(&rq->hash);
@@ -184,11 +211,11 @@ static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
 	rq->end_io_data = NULL;
 	rq->next_rq = NULL;
 
-	ctx->rq_dispatched[rw_is_sync(op, op_flags)]++;
+	ctx->rq_dispatched[op_is_sync(op)]++;
 }
 
 static struct request *
-__blk_mq_alloc_request(struct blk_mq_alloc_data *data, int op, int op_flags)
+__blk_mq_alloc_request(struct blk_mq_alloc_data *data, unsigned int op)
 {
 	struct request *rq;
 	unsigned int tag;
@@ -198,12 +225,12 @@ __blk_mq_alloc_request(struct blk_mq_alloc_data *data, int op, int op_flags)
 		rq = data->hctx->tags->rqs[tag];
 
 		if (blk_mq_tag_busy(data->hctx)) {
-			rq->cmd_flags = REQ_MQ_INFLIGHT;
+			rq->rq_flags = RQF_MQ_INFLIGHT;
 			atomic_inc(&data->hctx->nr_active);
 		}
 
 		rq->tag = tag;
-		blk_mq_rq_ctx_init(data->q, data->ctx, rq, op, op_flags);
+		blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
 		return rq;
 	}
 
@@ -226,7 +253,7 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
 	ctx = blk_mq_get_ctx(q);
 	hctx = blk_mq_map_queue(q, ctx->cpu);
 	blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx);
-	rq = __blk_mq_alloc_request(&alloc_data, rw, 0);
+	rq = __blk_mq_alloc_request(&alloc_data, rw);
 	blk_mq_put_ctx(ctx);
 
 	if (!rq) {
@@ -278,7 +305,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw,
 	ctx = __blk_mq_get_ctx(q, cpumask_first(hctx->cpumask));
 
 	blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx);
-	rq = __blk_mq_alloc_request(&alloc_data, rw, 0);
+	rq = __blk_mq_alloc_request(&alloc_data, rw);
 	if (!rq) {
 		ret = -EWOULDBLOCK;
 		goto out_queue_exit;
@@ -298,11 +325,14 @@ static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
 	const int tag = rq->tag;
 	struct request_queue *q = rq->q;
 
-	if (rq->cmd_flags & REQ_MQ_INFLIGHT)
+	if (rq->rq_flags & RQF_MQ_INFLIGHT)
 		atomic_dec(&hctx->nr_active);
-	rq->cmd_flags = 0;
+
+	wbt_done(q->rq_wb, &rq->issue_stat);
+	rq->rq_flags = 0;
 
 	clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
+	clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
 	blk_mq_put_tag(hctx, ctx, tag);
 	blk_queue_exit(q);
 }
@@ -328,6 +358,7 @@ inline void __blk_mq_end_request(struct request *rq, int error)
 	blk_account_io_done(rq);
 
 	if (rq->end_io) {
+		wbt_done(rq->q->rq_wb, &rq->issue_stat);
 		rq->end_io(rq, error);
 	} else {
 		if (unlikely(blk_bidi_rq(rq)))
@@ -378,10 +409,27 @@ static void blk_mq_ipi_complete_request(struct request *rq)
 	put_cpu();
 }
 
+static void blk_mq_stat_add(struct request *rq)
+{
+	if (rq->rq_flags & RQF_STATS) {
+		/*
+		 * We could rq->mq_ctx here, but there's less of a risk
+		 * of races if we have the completion event add the stats
+		 * to the local software queue.
+		 */
+		struct blk_mq_ctx *ctx;
+
+		ctx = __blk_mq_get_ctx(rq->q, raw_smp_processor_id());
+		blk_stat_add(&ctx->stat[rq_data_dir(rq)], rq);
+	}
+}
+
 static void __blk_mq_complete_request(struct request *rq)
 {
 	struct request_queue *q = rq->q;
 
+	blk_mq_stat_add(rq);
+
 	if (!q->softirq_done_fn)
 		blk_mq_end_request(rq, rq->errors);
 	else
@@ -425,6 +473,12 @@ void blk_mq_start_request(struct request *rq)
 	if (unlikely(blk_bidi_rq(rq)))
 		rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
 
+	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
+		blk_stat_set_issue_time(&rq->issue_stat);
+		rq->rq_flags |= RQF_STATS;
+		wbt_issue(q->rq_wb, &rq->issue_stat);
+	}
+
 	blk_add_timer(rq);
 
 	/*
@@ -460,6 +514,7 @@ static void __blk_mq_requeue_request(struct request *rq)
 	struct request_queue *q = rq->q;
 
 	trace_block_rq_requeue(q, rq);
+	wbt_requeue(q->rq_wb, &rq->issue_stat);
 
 	if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
 		if (q->dma_drain_size && blk_rq_bytes(rq))
@@ -467,12 +522,12 @@ static void __blk_mq_requeue_request(struct request *rq)
 	}
 }
 
-void blk_mq_requeue_request(struct request *rq)
+void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
 {
 	__blk_mq_requeue_request(rq);
 
 	BUG_ON(blk_queued_rq(rq));
-	blk_mq_add_to_requeue_list(rq, true);
+	blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
 }
 EXPORT_SYMBOL(blk_mq_requeue_request);
 
@@ -489,10 +544,10 @@ static void blk_mq_requeue_work(struct work_struct *work)
 	spin_unlock_irqrestore(&q->requeue_lock, flags);
 
 	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
-		if (!(rq->cmd_flags & REQ_SOFTBARRIER))
+		if (!(rq->rq_flags & RQF_SOFTBARRIER))
 			continue;
 
-		rq->cmd_flags &= ~REQ_SOFTBARRIER;
+		rq->rq_flags &= ~RQF_SOFTBARRIER;
 		list_del_init(&rq->queuelist);
 		blk_mq_insert_request(rq, true, false, false);
 	}
@@ -503,14 +558,11 @@ static void blk_mq_requeue_work(struct work_struct *work)
 		blk_mq_insert_request(rq, false, false, false);
 	}
 
-	/*
-	 * Use the start variant of queue running here, so that running
-	 * the requeue work will kick stopped queues.
-	 */
-	blk_mq_start_hw_queues(q);
+	blk_mq_run_hw_queues(q, false);
 }
 
-void blk_mq_add_to_requeue_list(struct request *rq, bool at_head)
+void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
+				bool kick_requeue_list)
 {
 	struct request_queue *q = rq->q;
 	unsigned long flags;
@@ -519,24 +571,21 @@ void blk_mq_add_to_requeue_list(struct request *rq, bool at_head)
 	 * We abuse this flag that is otherwise used by the I/O scheduler to
 	 * request head insertation from the workqueue.
 	 */
-	BUG_ON(rq->cmd_flags & REQ_SOFTBARRIER);
+	BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
 
 	spin_lock_irqsave(&q->requeue_lock, flags);
 	if (at_head) {
-		rq->cmd_flags |= REQ_SOFTBARRIER;
+		rq->rq_flags |= RQF_SOFTBARRIER;
 		list_add(&rq->queuelist, &q->requeue_list);
 	} else {
 		list_add_tail(&rq->queuelist, &q->requeue_list);
 	}
 	spin_unlock_irqrestore(&q->requeue_lock, flags);
-}
-EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
 
-void blk_mq_cancel_requeue_work(struct request_queue *q)
-{
-	cancel_delayed_work_sync(&q->requeue_work);
+	if (kick_requeue_list)
+		blk_mq_kick_requeue_list(q);
 }
-EXPORT_SYMBOL_GPL(blk_mq_cancel_requeue_work);
+EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
 
 void blk_mq_kick_requeue_list(struct request_queue *q)
 {
@@ -772,44 +821,13 @@ static inline unsigned int queued_to_index(unsigned int queued)
 	return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
 }
 
-/*
- * Run this hardware queue, pulling any software queues mapped to it in.
- * Note that this function currently has various problems around ordering
- * of IO. In particular, we'd like FIFO behaviour on handling existing
- * items on the hctx->dispatch list. Ignore that for now.
- */
-static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
+bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list)
 {
 	struct request_queue *q = hctx->queue;
 	struct request *rq;
-	LIST_HEAD(rq_list);
 	LIST_HEAD(driver_list);
 	struct list_head *dptr;
-	int queued;
-
-	if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
-		return;
-
-	WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
-		cpu_online(hctx->next_cpu));
-
-	hctx->run++;
-
-	/*
-	 * Touch any software queue that has pending entries.
-	 */
-	flush_busy_ctxs(hctx, &rq_list);
-
-	/*
-	 * If we have previous entries on our dispatch list, grab them
-	 * and stuff them at the front for more fair dispatch.
-	 */
-	if (!list_empty_careful(&hctx->dispatch)) {
-		spin_lock(&hctx->lock);
-		if (!list_empty(&hctx->dispatch))
-			list_splice_init(&hctx->dispatch, &rq_list);
-		spin_unlock(&hctx->lock);
-	}
+	int queued, ret = BLK_MQ_RQ_QUEUE_OK;
 
 	/*
 	 * Start off with dptr being NULL, so we start the first request
@@ -821,16 +839,15 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 	 * Now process all the entries, sending them to the driver.
 	 */
 	queued = 0;
-	while (!list_empty(&rq_list)) {
+	while (!list_empty(list)) {
 		struct blk_mq_queue_data bd;
-		int ret;
 
-		rq = list_first_entry(&rq_list, struct request, queuelist);
+		rq = list_first_entry(list, struct request, queuelist);
 		list_del_init(&rq->queuelist);
 
 		bd.rq = rq;
 		bd.list = dptr;
-		bd.last = list_empty(&rq_list);
+		bd.last = list_empty(list);
 
 		ret = q->mq_ops->queue_rq(hctx, &bd);
 		switch (ret) {
@@ -838,7 +855,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 			queued++;
 			break;
 		case BLK_MQ_RQ_QUEUE_BUSY:
-			list_add(&rq->queuelist, &rq_list);
+			list_add(&rq->queuelist, list);
 			__blk_mq_requeue_request(rq);
 			break;
 		default:
@@ -856,7 +873,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 		 * We've done the first request. If we have more than 1
 		 * left in the list, set dptr to defer issue.
 		 */
-		if (!dptr && rq_list.next != rq_list.prev)
+		if (!dptr && list->next != list->prev)
 			dptr = &driver_list;
 	}
 
@@ -866,10 +883,11 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 	 * Any items that need requeuing? Stuff them into hctx->dispatch,
 	 * that is where we will continue on next queue run.
 	 */
-	if (!list_empty(&rq_list)) {
+	if (!list_empty(list)) {
 		spin_lock(&hctx->lock);
-		list_splice(&rq_list, &hctx->dispatch);
+		list_splice(list, &hctx->dispatch);
 		spin_unlock(&hctx->lock);
+
 		/*
 		 * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but
 		 * it's possible the queue is stopped and restarted again
@@ -881,6 +899,61 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 		 **/
 		blk_mq_run_hw_queue(hctx, true);
 	}
+
+	return ret != BLK_MQ_RQ_QUEUE_BUSY;
+}
+
+/*
+ * Run this hardware queue, pulling any software queues mapped to it in.
+ * Note that this function currently has various problems around ordering
+ * of IO. In particular, we'd like FIFO behaviour on handling existing
+ * items on the hctx->dispatch list. Ignore that for now.
+ */
+static void blk_mq_process_rq_list(struct blk_mq_hw_ctx *hctx)
+{
+	LIST_HEAD(rq_list);
+	LIST_HEAD(driver_list);
+
+	if (unlikely(blk_mq_hctx_stopped(hctx)))
+		return;
+
+	hctx->run++;
+
+	/*
+	 * Touch any software queue that has pending entries.
+	 */
+	flush_busy_ctxs(hctx, &rq_list);
+
+	/*
+	 * If we have previous entries on our dispatch list, grab them
+	 * and stuff them at the front for more fair dispatch.
+	 */
+	if (!list_empty_careful(&hctx->dispatch)) {
+		spin_lock(&hctx->lock);
+		if (!list_empty(&hctx->dispatch))
+			list_splice_init(&hctx->dispatch, &rq_list);
+		spin_unlock(&hctx->lock);
+	}
+
+	blk_mq_dispatch_rq_list(hctx, &rq_list);
+}
+
+static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
+{
+	int srcu_idx;
+
+	WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
+		cpu_online(hctx->next_cpu));
+
+	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
+		rcu_read_lock();
+		blk_mq_process_rq_list(hctx);
+		rcu_read_unlock();
+	} else {
+		srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
+		blk_mq_process_rq_list(hctx);
+		srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
+	}
 }
 
 /*
@@ -895,7 +968,7 @@ static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
 		return WORK_CPU_UNBOUND;
 
 	if (--hctx->next_cpu_batch <= 0) {
-		int cpu = hctx->next_cpu, next_cpu;
+		int next_cpu;
 
 		next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
 		if (next_cpu >= nr_cpu_ids)
@@ -903,8 +976,6 @@ static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
 
 		hctx->next_cpu = next_cpu;
 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
-
-		return cpu;
 	}
 
 	return hctx->next_cpu;
@@ -912,8 +983,8 @@ static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
 
 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 {
-	if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state) ||
-	    !blk_mq_hw_queue_mapped(hctx)))
+	if (unlikely(blk_mq_hctx_stopped(hctx) ||
+		     !blk_mq_hw_queue_mapped(hctx)))
 		return;
 
 	if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
@@ -938,7 +1009,7 @@ void blk_mq_run_hw_queues(struct request_queue *q, bool async)
 	queue_for_each_hw_ctx(q, hctx, i) {
 		if ((!blk_mq_hctx_has_pending(hctx) &&
 		    list_empty_careful(&hctx->dispatch)) ||
-		    test_bit(BLK_MQ_S_STOPPED, &hctx->state))
+		    blk_mq_hctx_stopped(hctx))
 			continue;
 
 		blk_mq_run_hw_queue(hctx, async);
@@ -946,6 +1017,26 @@ void blk_mq_run_hw_queues(struct request_queue *q, bool async)
 }
 EXPORT_SYMBOL(blk_mq_run_hw_queues);
 
+/**
+ * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
+ * @q: request queue.
+ *
+ * The caller is responsible for serializing this function against
+ * blk_mq_{start,stop}_hw_queue().
+ */
+bool blk_mq_queue_stopped(struct request_queue *q)
+{
+	struct blk_mq_hw_ctx *hctx;
+	int i;
+
+	queue_for_each_hw_ctx(q, hctx, i)
+		if (blk_mq_hctx_stopped(hctx))
+			return true;
+
+	return false;
+}
+EXPORT_SYMBOL(blk_mq_queue_stopped);
+
 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
 {
 	cancel_work(&hctx->run_work);
@@ -982,18 +1073,23 @@ void blk_mq_start_hw_queues(struct request_queue *q)
 }
 EXPORT_SYMBOL(blk_mq_start_hw_queues);
 
+void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
+{
+	if (!blk_mq_hctx_stopped(hctx))
+		return;
+
+	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
+	blk_mq_run_hw_queue(hctx, async);
+}
+EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
+
 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
 {
 	struct blk_mq_hw_ctx *hctx;
 	int i;
 
-	queue_for_each_hw_ctx(q, hctx, i) {
-		if (!test_bit(BLK_MQ_S_STOPPED, &hctx->state))
-			continue;
-
-		clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
-		blk_mq_run_hw_queue(hctx, async);
-	}
+	queue_for_each_hw_ctx(q, hctx, i)
+		blk_mq_start_stopped_hw_queue(hctx, async);
 }
 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
 
@@ -1155,7 +1251,7 @@ static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
 {
 	init_request_from_bio(rq, bio);
 
-	blk_account_io_start(rq, 1);
+	blk_account_io_start(rq, true);
 }
 
 static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
@@ -1190,40 +1286,27 @@ insert_rq:
 	}
 }
 
-struct blk_map_ctx {
-	struct blk_mq_hw_ctx *hctx;
-	struct blk_mq_ctx *ctx;
-};
-
 static struct request *blk_mq_map_request(struct request_queue *q,
 					  struct bio *bio,
-					  struct blk_map_ctx *data)
+					  struct blk_mq_alloc_data *data)
 {
 	struct blk_mq_hw_ctx *hctx;
 	struct blk_mq_ctx *ctx;
 	struct request *rq;
-	int op = bio_data_dir(bio);
-	int op_flags = 0;
-	struct blk_mq_alloc_data alloc_data;
 
 	blk_queue_enter_live(q);
 	ctx = blk_mq_get_ctx(q);
 	hctx = blk_mq_map_queue(q, ctx->cpu);
 
-	if (rw_is_sync(bio_op(bio), bio->bi_opf))
-		op_flags |= REQ_SYNC;
-
-	trace_block_getrq(q, bio, op);
-	blk_mq_set_alloc_data(&alloc_data, q, 0, ctx, hctx);
-	rq = __blk_mq_alloc_request(&alloc_data, op, op_flags);
+	trace_block_getrq(q, bio, bio->bi_opf);
+	blk_mq_set_alloc_data(data, q, 0, ctx, hctx);
+	rq = __blk_mq_alloc_request(data, bio->bi_opf);
 
-	data->hctx = alloc_data.hctx;
-	data->ctx = alloc_data.ctx;
 	data->hctx->queued++;
 	return rq;
 }
 
-static int blk_mq_direct_issue_request(struct request *rq, blk_qc_t *cookie)
+static void blk_mq_try_issue_directly(struct request *rq, blk_qc_t *cookie)
 {
 	int ret;
 	struct request_queue *q = rq->q;
@@ -1235,6 +1318,9 @@ static int blk_mq_direct_issue_request(struct request *rq, blk_qc_t *cookie)
 	};
 	blk_qc_t new_cookie = blk_tag_to_qc_t(rq->tag, hctx->queue_num);
 
+	if (blk_mq_hctx_stopped(hctx))
+		goto insert;
+
 	/*
 	 * For OK queue, we are done. For error, kill it. Any other
 	 * error (busy), just add it to our list as we previously
@@ -1243,7 +1329,7 @@ static int blk_mq_direct_issue_request(struct request *rq, blk_qc_t *cookie)
 	ret = q->mq_ops->queue_rq(hctx, &bd);
 	if (ret == BLK_MQ_RQ_QUEUE_OK) {
 		*cookie = new_cookie;
-		return 0;
+		return;
 	}
 
 	__blk_mq_requeue_request(rq);
@@ -1252,10 +1338,11 @@ static int blk_mq_direct_issue_request(struct request *rq, blk_qc_t *cookie)
 		*cookie = BLK_QC_T_NONE;
 		rq->errors = -EIO;
 		blk_mq_end_request(rq, rq->errors);
-		return 0;
+		return;
 	}
 
-	return -1;
+insert:
+	blk_mq_insert_request(rq, false, true, true);
 }
 
 /*
@@ -1265,14 +1352,15 @@ static int blk_mq_direct_issue_request(struct request *rq, blk_qc_t *cookie)
  */
 static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
 {
-	const int is_sync = rw_is_sync(bio_op(bio), bio->bi_opf);
+	const int is_sync = op_is_sync(bio->bi_opf);
 	const int is_flush_fua = bio->bi_opf & (REQ_PREFLUSH | REQ_FUA);
-	struct blk_map_ctx data;
+	struct blk_mq_alloc_data data;
 	struct request *rq;
-	unsigned int request_count = 0;
+	unsigned int request_count = 0, srcu_idx;
 	struct blk_plug *plug;
 	struct request *same_queue_rq = NULL;
 	blk_qc_t cookie;
+	unsigned int wb_acct;
 
 	blk_queue_bounce(q, &bio);
 
@@ -1287,9 +1375,15 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
 	    blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
 		return BLK_QC_T_NONE;
 
+	wb_acct = wbt_wait(q->rq_wb, bio, NULL);
+
 	rq = blk_mq_map_request(q, bio, &data);
-	if (unlikely(!rq))
+	if (unlikely(!rq)) {
+		__wbt_done(q->rq_wb, wb_acct);
 		return BLK_QC_T_NONE;
+	}
+
+	wbt_track(&rq->issue_stat, wb_acct);
 
 	cookie = blk_tag_to_qc_t(rq->tag, data.hctx->queue_num);
 
@@ -1312,7 +1406,7 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
 		blk_mq_bio_to_request(rq, bio);
 
 		/*
-		 * We do limited pluging. If the bio can be merged, do that.
+		 * We do limited plugging. If the bio can be merged, do that.
 		 * Otherwise the existing request in the plug list will be
 		 * issued. So the plug list will have one request at most
 		 */
@@ -1332,9 +1426,16 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
 		blk_mq_put_ctx(data.ctx);
 		if (!old_rq)
 			goto done;
-		if (!blk_mq_direct_issue_request(old_rq, &cookie))
-			goto done;
-		blk_mq_insert_request(old_rq, false, true, true);
+
+		if (!(data.hctx->flags & BLK_MQ_F_BLOCKING)) {
+			rcu_read_lock();
+			blk_mq_try_issue_directly(old_rq, &cookie);
+			rcu_read_unlock();
+		} else {
+			srcu_idx = srcu_read_lock(&data.hctx->queue_rq_srcu);
+			blk_mq_try_issue_directly(old_rq, &cookie);
+			srcu_read_unlock(&data.hctx->queue_rq_srcu, srcu_idx);
+		}
 		goto done;
 	}
 
@@ -1359,13 +1460,14 @@ done:
  */
 static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio)
 {
-	const int is_sync = rw_is_sync(bio_op(bio), bio->bi_opf);
+	const int is_sync = op_is_sync(bio->bi_opf);
 	const int is_flush_fua = bio->bi_opf & (REQ_PREFLUSH | REQ_FUA);
 	struct blk_plug *plug;
 	unsigned int request_count = 0;
-	struct blk_map_ctx data;
+	struct blk_mq_alloc_data data;
 	struct request *rq;
 	blk_qc_t cookie;
+	unsigned int wb_acct;
 
 	blk_queue_bounce(q, &bio);
 
@@ -1382,9 +1484,15 @@ static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio)
 	} else
 		request_count = blk_plug_queued_count(q);
 
+	wb_acct = wbt_wait(q->rq_wb, bio, NULL);
+
 	rq = blk_mq_map_request(q, bio, &data);
-	if (unlikely(!rq))
+	if (unlikely(!rq)) {
+		__wbt_done(q->rq_wb, wb_acct);
 		return BLK_QC_T_NONE;
+	}
+
+	wbt_track(&rq->issue_stat, wb_acct);
 
 	cookie = blk_tag_to_qc_t(rq->tag, data.hctx->queue_num);
 
@@ -1401,13 +1509,25 @@ static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio)
 	 */
 	plug = current->plug;
 	if (plug) {
+		struct request *last = NULL;
+
 		blk_mq_bio_to_request(rq, bio);
+
+		/*
+		 * @request_count may become stale because of schedule
+		 * out, so check the list again.
+		 */
+		if (list_empty(&plug->mq_list))
+			request_count = 0;
 		if (!request_count)
 			trace_block_plug(q);
+		else
+			last = list_entry_rq(plug->mq_list.prev);
 
 		blk_mq_put_ctx(data.ctx);
 
-		if (request_count >= BLK_MAX_REQUEST_COUNT) {
+		if (request_count >= BLK_MAX_REQUEST_COUNT || (last &&
+		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
 			blk_flush_plug_list(plug, false);
 			trace_block_plug(q);
 		}
@@ -1485,7 +1605,7 @@ static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
 	INIT_LIST_HEAD(&tags->page_list);
 
 	tags->rqs = kzalloc_node(set->queue_depth * sizeof(struct request *),
-				 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY,
+				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
 				 set->numa_node);
 	if (!tags->rqs) {
 		blk_mq_free_tags(tags);
@@ -1511,7 +1631,7 @@ static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
 
 		do {
 			page = alloc_pages_node(set->numa_node,
-				GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
+				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
 				this_order);
 			if (page)
 				break;
@@ -1532,7 +1652,7 @@ static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
 		 * Allow kmemleak to scan these pages as they contain pointers
 		 * to additional allocations like via ops->init_request().
 		 */
-		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_KERNEL);
+		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
 		entries_per_page = order_to_size(this_order) / rq_size;
 		to_do = min(entries_per_page, set->queue_depth - i);
 		left -= to_do * rq_size;
@@ -1613,6 +1733,9 @@ static void blk_mq_exit_hctx(struct request_queue *q,
 	if (set->ops->exit_hctx)
 		set->ops->exit_hctx(hctx, hctx_idx);
 
+	if (hctx->flags & BLK_MQ_F_BLOCKING)
+		cleanup_srcu_struct(&hctx->queue_rq_srcu);
+
 	blk_mq_remove_cpuhp(hctx);
 	blk_free_flush_queue(hctx->fq);
 	sbitmap_free(&hctx->ctx_map);
@@ -1693,6 +1816,9 @@ static int blk_mq_init_hctx(struct request_queue *q,
 				   flush_start_tag + hctx_idx, node))
 		goto free_fq;
 
+	if (hctx->flags & BLK_MQ_F_BLOCKING)
+		init_srcu_struct(&hctx->queue_rq_srcu);
+
 	return 0;
 
  free_fq:
@@ -1723,6 +1849,8 @@ static void blk_mq_init_cpu_queues(struct request_queue *q,
 		spin_lock_init(&__ctx->lock);
 		INIT_LIST_HEAD(&__ctx->rq_list);
 		__ctx->queue = q;
+		blk_stat_init(&__ctx->stat[BLK_STAT_READ]);
+		blk_stat_init(&__ctx->stat[BLK_STAT_WRITE]);
 
 		/* If the cpu isn't online, the cpu is mapped to first hctx */
 		if (!cpu_online(i))
@@ -1742,7 +1870,7 @@ static void blk_mq_init_cpu_queues(struct request_queue *q,
 static void blk_mq_map_swqueue(struct request_queue *q,
 			       const struct cpumask *online_mask)
 {
-	unsigned int i;
+	unsigned int i, hctx_idx;
 	struct blk_mq_hw_ctx *hctx;
 	struct blk_mq_ctx *ctx;
 	struct blk_mq_tag_set *set = q->tag_set;
@@ -1765,6 +1893,21 @@ static void blk_mq_map_swqueue(struct request_queue *q,
 		if (!cpumask_test_cpu(i, online_mask))
 			continue;
 
+		hctx_idx = q->mq_map[i];
+		/* unmapped hw queue can be remapped after CPU topo changed */
+		if (!set->tags[hctx_idx]) {
+			set->tags[hctx_idx] = blk_mq_init_rq_map(set, hctx_idx);
+
+			/*
+			 * If tags initialization fail for some hctx,
+			 * that hctx won't be brought online.  In this
+			 * case, remap the current ctx to hctx[0] which
+			 * is guaranteed to always have tags allocated
+			 */
+			if (!set->tags[hctx_idx])
+				q->mq_map[i] = 0;
+		}
+
 		ctx = per_cpu_ptr(q->queue_ctx, i);
 		hctx = blk_mq_map_queue(q, i);
 
@@ -1781,7 +1924,11 @@ static void blk_mq_map_swqueue(struct request_queue *q,
 		 * disable it and free the request entries.
 		 */
 		if (!hctx->nr_ctx) {
-			if (set->tags[i]) {
+			/* Never unmap queue 0.  We need it as a
+			 * fallback in case of a new remap fails
+			 * allocation
+			 */
+			if (i && set->tags[i]) {
 				blk_mq_free_rq_map(set, set->tags[i], i);
 				set->tags[i] = NULL;
 			}
@@ -1789,9 +1936,6 @@ static void blk_mq_map_swqueue(struct request_queue *q,
 			continue;
 		}
 
-		/* unmapped hw queue can be remapped after CPU topo changed */
-		if (!set->tags[i])
-			set->tags[i] = blk_mq_init_rq_map(set, i);
 		hctx->tags = set->tags[i];
 		WARN_ON(!hctx->tags);
 
@@ -2018,6 +2162,11 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
 	 */
 	q->nr_requests = set->queue_depth;
 
+	/*
+	 * Default to classic polling
+	 */
+	q->poll_nsec = -1;
+
 	if (set->ops->complete)
 		blk_queue_softirq_done(q, set->ops->complete);
 
@@ -2053,6 +2202,8 @@ void blk_mq_free_queue(struct request_queue *q)
 	list_del_init(&q->all_q_node);
 	mutex_unlock(&all_q_mutex);
 
+	wbt_exit(q);
+
 	blk_mq_del_queue_tag_set(q);
 
 	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
@@ -2099,16 +2250,9 @@ static void blk_mq_queue_reinit_work(void)
 	 */
 	list_for_each_entry(q, &all_q_list, all_q_node)
 		blk_mq_freeze_queue_start(q);
-	list_for_each_entry(q, &all_q_list, all_q_node) {
+	list_for_each_entry(q, &all_q_list, all_q_node)
 		blk_mq_freeze_queue_wait(q);
 
-		/*
-		 * timeout handler can't touch hw queue during the
-		 * reinitialization
-		 */
-		del_timer_sync(&q->timeout);
-	}
-
 	list_for_each_entry(q, &all_q_list, all_q_node)
 		blk_mq_queue_reinit(q, &cpuhp_online_new);
 
@@ -2353,6 +2497,165 @@ void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
 }
 EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
 
+static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
+				       struct blk_mq_hw_ctx *hctx,
+				       struct request *rq)
+{
+	struct blk_rq_stat stat[2];
+	unsigned long ret = 0;
+
+	/*
+	 * If stats collection isn't on, don't sleep but turn it on for
+	 * future users
+	 */
+	if (!blk_stat_enable(q))
+		return 0;
+
+	/*
+	 * We don't have to do this once per IO, should optimize this
+	 * to just use the current window of stats until it changes
+	 */
+	memset(&stat, 0, sizeof(stat));
+	blk_hctx_stat_get(hctx, stat);
+
+	/*
+	 * As an optimistic guess, use half of the mean service time
+	 * for this type of request. We can (and should) make this smarter.
+	 * For instance, if the completion latencies are tight, we can
+	 * get closer than just half the mean. This is especially
+	 * important on devices where the completion latencies are longer
+	 * than ~10 usec.
+	 */
+	if (req_op(rq) == REQ_OP_READ && stat[BLK_STAT_READ].nr_samples)
+		ret = (stat[BLK_STAT_READ].mean + 1) / 2;
+	else if (req_op(rq) == REQ_OP_WRITE && stat[BLK_STAT_WRITE].nr_samples)
+		ret = (stat[BLK_STAT_WRITE].mean + 1) / 2;
+
+	return ret;
+}
+
+static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
+				     struct blk_mq_hw_ctx *hctx,
+				     struct request *rq)
+{
+	struct hrtimer_sleeper hs;
+	enum hrtimer_mode mode;
+	unsigned int nsecs;
+	ktime_t kt;
+
+	if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags))
+		return false;
+
+	/*
+	 * poll_nsec can be:
+	 *
+	 * -1:	don't ever hybrid sleep
+	 *  0:	use half of prev avg
+	 * >0:	use this specific value
+	 */
+	if (q->poll_nsec == -1)
+		return false;
+	else if (q->poll_nsec > 0)
+		nsecs = q->poll_nsec;
+	else
+		nsecs = blk_mq_poll_nsecs(q, hctx, rq);
+
+	if (!nsecs)
+		return false;
+
+	set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
+
+	/*
+	 * This will be replaced with the stats tracking code, using
+	 * 'avg_completion_time / 2' as the pre-sleep target.
+	 */
+	kt = ktime_set(0, nsecs);
+
+	mode = HRTIMER_MODE_REL;
+	hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
+	hrtimer_set_expires(&hs.timer, kt);
+
+	hrtimer_init_sleeper(&hs, current);
+	do {
+		if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
+			break;
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		hrtimer_start_expires(&hs.timer, mode);
+		if (hs.task)
+			io_schedule();
+		hrtimer_cancel(&hs.timer);
+		mode = HRTIMER_MODE_ABS;
+	} while (hs.task && !signal_pending(current));
+
+	__set_current_state(TASK_RUNNING);
+	destroy_hrtimer_on_stack(&hs.timer);
+	return true;
+}
+
+static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq)
+{
+	struct request_queue *q = hctx->queue;
+	long state;
+
+	/*
+	 * If we sleep, have the caller restart the poll loop to reset
+	 * the state. Like for the other success return cases, the
+	 * caller is responsible for checking if the IO completed. If
+	 * the IO isn't complete, we'll get called again and will go
+	 * straight to the busy poll loop.
+	 */
+	if (blk_mq_poll_hybrid_sleep(q, hctx, rq))
+		return true;
+
+	hctx->poll_considered++;
+
+	state = current->state;
+	while (!need_resched()) {
+		int ret;
+
+		hctx->poll_invoked++;
+
+		ret = q->mq_ops->poll(hctx, rq->tag);
+		if (ret > 0) {
+			hctx->poll_success++;
+			set_current_state(TASK_RUNNING);
+			return true;
+		}
+
+		if (signal_pending_state(state, current))
+			set_current_state(TASK_RUNNING);
+
+		if (current->state == TASK_RUNNING)
+			return true;
+		if (ret < 0)
+			break;
+		cpu_relax();
+	}
+
+	return false;
+}
+
+bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
+{
+	struct blk_mq_hw_ctx *hctx;
+	struct blk_plug *plug;
+	struct request *rq;
+
+	if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
+	    !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
+		return false;
+
+	plug = current->plug;
+	if (plug)
+		blk_flush_plug_list(plug, false);
+
+	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
+	rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
+
+	return __blk_mq_poll(hctx, rq);
+}
+EXPORT_SYMBOL_GPL(blk_mq_poll);
+
 void blk_mq_disable_hotplug(void)
 {
 	mutex_lock(&all_q_mutex);
diff --git a/block/blk-mq.h b/block/blk-mq.h
index e5d25249028c..63e9116cddbd 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -1,6 +1,8 @@
 #ifndef INT_BLK_MQ_H
 #define INT_BLK_MQ_H
 
+#include "blk-stat.h"
+
 struct blk_mq_tag_set;
 
 struct blk_mq_ctx {
@@ -18,6 +20,7 @@ struct blk_mq_ctx {
 
 	/* incremented at completion time */
 	unsigned long		____cacheline_aligned_in_smp rq_completed[2];
+	struct blk_rq_stat	stat[2];
 
 	struct request_queue	*queue;
 	struct kobject		kobj;
@@ -28,6 +31,7 @@ void blk_mq_freeze_queue(struct request_queue *q);
 void blk_mq_free_queue(struct request_queue *q);
 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
 void blk_mq_wake_waiters(struct request_queue *q);
+bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *, struct list_head *);
 
 /*
  * CPU hotplug helpers
@@ -38,7 +42,6 @@ void blk_mq_disable_hotplug(void);
 /*
  * CPU -> queue mappings
  */
-int blk_mq_map_queues(struct blk_mq_tag_set *set);
 extern int blk_mq_hw_queue_to_node(unsigned int *map, unsigned int);
 
 static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q,
@@ -100,6 +103,11 @@ static inline void blk_mq_set_alloc_data(struct blk_mq_alloc_data *data,
 	data->hctx = hctx;
 }
 
+static inline bool blk_mq_hctx_stopped(struct blk_mq_hw_ctx *hctx)
+{
+	return test_bit(BLK_MQ_S_STOPPED, &hctx->state);
+}
+
 static inline bool blk_mq_hw_queue_mapped(struct blk_mq_hw_ctx *hctx)
 {
 	return hctx->nr_ctx && hctx->tags;
diff --git a/block/blk-settings.c b/block/blk-settings.c
index 65f16cf4f850..529e55f52a03 100644
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -13,6 +13,7 @@
 #include <linux/gfp.h>
 
 #include "blk.h"
+#include "blk-wbt.h"
 
 unsigned long blk_max_low_pfn;
 EXPORT_SYMBOL(blk_max_low_pfn);
@@ -95,6 +96,7 @@ void blk_set_default_limits(struct queue_limits *lim)
 	lim->max_dev_sectors = 0;
 	lim->chunk_sectors = 0;
 	lim->max_write_same_sectors = 0;
+	lim->max_write_zeroes_sectors = 0;
 	lim->max_discard_sectors = 0;
 	lim->max_hw_discard_sectors = 0;
 	lim->discard_granularity = 0;
@@ -107,6 +109,7 @@ void blk_set_default_limits(struct queue_limits *lim)
 	lim->io_opt = 0;
 	lim->misaligned = 0;
 	lim->cluster = 1;
+	lim->zoned = BLK_ZONED_NONE;
 }
 EXPORT_SYMBOL(blk_set_default_limits);
 
@@ -130,6 +133,7 @@ void blk_set_stacking_limits(struct queue_limits *lim)
 	lim->max_sectors = UINT_MAX;
 	lim->max_dev_sectors = UINT_MAX;
 	lim->max_write_same_sectors = UINT_MAX;
+	lim->max_write_zeroes_sectors = UINT_MAX;
 }
 EXPORT_SYMBOL(blk_set_stacking_limits);
 
@@ -299,6 +303,19 @@ void blk_queue_max_write_same_sectors(struct request_queue *q,
 EXPORT_SYMBOL(blk_queue_max_write_same_sectors);
 
 /**
+ * blk_queue_max_write_zeroes_sectors - set max sectors for a single
+ *                                      write zeroes
+ * @q:  the request queue for the device
+ * @max_write_zeroes_sectors: maximum number of sectors to write per command
+ **/
+void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
+		unsigned int max_write_zeroes_sectors)
+{
+	q->limits.max_write_zeroes_sectors = max_write_zeroes_sectors;
+}
+EXPORT_SYMBOL(blk_queue_max_write_zeroes_sectors);
+
+/**
  * blk_queue_max_segments - set max hw segments for a request for this queue
  * @q:  the request queue for the device
  * @max_segments:  max number of segments
@@ -526,6 +543,8 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
 	t->max_dev_sectors = min_not_zero(t->max_dev_sectors, b->max_dev_sectors);
 	t->max_write_same_sectors = min(t->max_write_same_sectors,
 					b->max_write_same_sectors);
+	t->max_write_zeroes_sectors = min(t->max_write_zeroes_sectors,
+					b->max_write_zeroes_sectors);
 	t->bounce_pfn = min_not_zero(t->bounce_pfn, b->bounce_pfn);
 
 	t->seg_boundary_mask = min_not_zero(t->seg_boundary_mask,
@@ -631,6 +650,10 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
 			t->discard_granularity;
 	}
 
+	if (b->chunk_sectors)
+		t->chunk_sectors = min_not_zero(t->chunk_sectors,
+						b->chunk_sectors);
+
 	return ret;
 }
 EXPORT_SYMBOL(blk_stack_limits);
@@ -833,6 +856,19 @@ void blk_queue_flush_queueable(struct request_queue *q, bool queueable)
 EXPORT_SYMBOL_GPL(blk_queue_flush_queueable);
 
 /**
+ * blk_set_queue_depth - tell the block layer about the device queue depth
+ * @q:		the request queue for the device
+ * @depth:		queue depth
+ *
+ */
+void blk_set_queue_depth(struct request_queue *q, unsigned int depth)
+{
+	q->queue_depth = depth;
+	wbt_set_queue_depth(q->rq_wb, depth);
+}
+EXPORT_SYMBOL(blk_set_queue_depth);
+
+/**
  * blk_queue_write_cache - configure queue's write cache
  * @q:		the request queue for the device
  * @wc:		write back cache on or off
@@ -852,6 +888,8 @@ void blk_queue_write_cache(struct request_queue *q, bool wc, bool fua)
 	else
 		queue_flag_clear(QUEUE_FLAG_FUA, q);
 	spin_unlock_irq(q->queue_lock);
+
+	wbt_set_write_cache(q->rq_wb, test_bit(QUEUE_FLAG_WC, &q->queue_flags));
 }
 EXPORT_SYMBOL_GPL(blk_queue_write_cache);
 
diff --git a/block/blk-stat.c b/block/blk-stat.c
new file mode 100644
index 000000000000..9b43efb8933f
--- /dev/null
+++ b/block/blk-stat.c
@@ -0,0 +1,256 @@
+/*
+ * Block stat tracking code
+ *
+ * Copyright (C) 2016 Jens Axboe
+ */
+#include <linux/kernel.h>
+#include <linux/blk-mq.h>
+
+#include "blk-stat.h"
+#include "blk-mq.h"
+
+static void blk_stat_flush_batch(struct blk_rq_stat *stat)
+{
+	const s32 nr_batch = READ_ONCE(stat->nr_batch);
+	const s32 nr_samples = READ_ONCE(stat->nr_samples);
+
+	if (!nr_batch)
+		return;
+	if (!nr_samples)
+		stat->mean = div64_s64(stat->batch, nr_batch);
+	else {
+		stat->mean = div64_s64((stat->mean * nr_samples) +
+					stat->batch,
+					nr_batch + nr_samples);
+	}
+
+	stat->nr_samples += nr_batch;
+	stat->nr_batch = stat->batch = 0;
+}
+
+static void blk_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
+{
+	if (!src->nr_samples)
+		return;
+
+	blk_stat_flush_batch(src);
+
+	dst->min = min(dst->min, src->min);
+	dst->max = max(dst->max, src->max);
+
+	if (!dst->nr_samples)
+		dst->mean = src->mean;
+	else {
+		dst->mean = div64_s64((src->mean * src->nr_samples) +
+					(dst->mean * dst->nr_samples),
+					dst->nr_samples + src->nr_samples);
+	}
+	dst->nr_samples += src->nr_samples;
+}
+
+static void blk_mq_stat_get(struct request_queue *q, struct blk_rq_stat *dst)
+{
+	struct blk_mq_hw_ctx *hctx;
+	struct blk_mq_ctx *ctx;
+	uint64_t latest = 0;
+	int i, j, nr;
+
+	blk_stat_init(&dst[BLK_STAT_READ]);
+	blk_stat_init(&dst[BLK_STAT_WRITE]);
+
+	nr = 0;
+	do {
+		uint64_t newest = 0;
+
+		queue_for_each_hw_ctx(q, hctx, i) {
+			hctx_for_each_ctx(hctx, ctx, j) {
+				blk_stat_flush_batch(&ctx->stat[BLK_STAT_READ]);
+				blk_stat_flush_batch(&ctx->stat[BLK_STAT_WRITE]);
+
+				if (!ctx->stat[BLK_STAT_READ].nr_samples &&
+				    !ctx->stat[BLK_STAT_WRITE].nr_samples)
+					continue;
+				if (ctx->stat[BLK_STAT_READ].time > newest)
+					newest = ctx->stat[BLK_STAT_READ].time;
+				if (ctx->stat[BLK_STAT_WRITE].time > newest)
+					newest = ctx->stat[BLK_STAT_WRITE].time;
+			}
+		}
+
+		/*
+		 * No samples
+		 */
+		if (!newest)
+			break;
+
+		if (newest > latest)
+			latest = newest;
+
+		queue_for_each_hw_ctx(q, hctx, i) {
+			hctx_for_each_ctx(hctx, ctx, j) {
+				if (ctx->stat[BLK_STAT_READ].time == newest) {
+					blk_stat_sum(&dst[BLK_STAT_READ],
+						     &ctx->stat[BLK_STAT_READ]);
+					nr++;
+				}
+				if (ctx->stat[BLK_STAT_WRITE].time == newest) {
+					blk_stat_sum(&dst[BLK_STAT_WRITE],
+						     &ctx->stat[BLK_STAT_WRITE]);
+					nr++;
+				}
+			}
+		}
+		/*
+		 * If we race on finding an entry, just loop back again.
+		 * Should be very rare.
+		 */
+	} while (!nr);
+
+	dst[BLK_STAT_READ].time = dst[BLK_STAT_WRITE].time = latest;
+}
+
+void blk_queue_stat_get(struct request_queue *q, struct blk_rq_stat *dst)
+{
+	if (q->mq_ops)
+		blk_mq_stat_get(q, dst);
+	else {
+		blk_stat_flush_batch(&q->rq_stats[BLK_STAT_READ]);
+		blk_stat_flush_batch(&q->rq_stats[BLK_STAT_WRITE]);
+		memcpy(&dst[BLK_STAT_READ], &q->rq_stats[BLK_STAT_READ],
+				sizeof(struct blk_rq_stat));
+		memcpy(&dst[BLK_STAT_WRITE], &q->rq_stats[BLK_STAT_WRITE],
+				sizeof(struct blk_rq_stat));
+	}
+}
+
+void blk_hctx_stat_get(struct blk_mq_hw_ctx *hctx, struct blk_rq_stat *dst)
+{
+	struct blk_mq_ctx *ctx;
+	unsigned int i, nr;
+
+	nr = 0;
+	do {
+		uint64_t newest = 0;
+
+		hctx_for_each_ctx(hctx, ctx, i) {
+			blk_stat_flush_batch(&ctx->stat[BLK_STAT_READ]);
+			blk_stat_flush_batch(&ctx->stat[BLK_STAT_WRITE]);
+
+			if (!ctx->stat[BLK_STAT_READ].nr_samples &&
+			    !ctx->stat[BLK_STAT_WRITE].nr_samples)
+				continue;
+
+			if (ctx->stat[BLK_STAT_READ].time > newest)
+				newest = ctx->stat[BLK_STAT_READ].time;
+			if (ctx->stat[BLK_STAT_WRITE].time > newest)
+				newest = ctx->stat[BLK_STAT_WRITE].time;
+		}
+
+		if (!newest)
+			break;
+
+		hctx_for_each_ctx(hctx, ctx, i) {
+			if (ctx->stat[BLK_STAT_READ].time == newest) {
+				blk_stat_sum(&dst[BLK_STAT_READ],
+						&ctx->stat[BLK_STAT_READ]);
+				nr++;
+			}
+			if (ctx->stat[BLK_STAT_WRITE].time == newest) {
+				blk_stat_sum(&dst[BLK_STAT_WRITE],
+						&ctx->stat[BLK_STAT_WRITE]);
+				nr++;
+			}
+		}
+		/*
+		 * If we race on finding an entry, just loop back again.
+		 * Should be very rare, as the window is only updated
+		 * occasionally
+		 */
+	} while (!nr);
+}
+
+static void __blk_stat_init(struct blk_rq_stat *stat, s64 time_now)
+{
+	stat->min = -1ULL;
+	stat->max = stat->nr_samples = stat->mean = 0;
+	stat->batch = stat->nr_batch = 0;
+	stat->time = time_now & BLK_STAT_NSEC_MASK;
+}
+
+void blk_stat_init(struct blk_rq_stat *stat)
+{
+	__blk_stat_init(stat, ktime_to_ns(ktime_get()));
+}
+
+static bool __blk_stat_is_current(struct blk_rq_stat *stat, s64 now)
+{
+	return (now & BLK_STAT_NSEC_MASK) == (stat->time & BLK_STAT_NSEC_MASK);
+}
+
+bool blk_stat_is_current(struct blk_rq_stat *stat)
+{
+	return __blk_stat_is_current(stat, ktime_to_ns(ktime_get()));
+}
+
+void blk_stat_add(struct blk_rq_stat *stat, struct request *rq)
+{
+	s64 now, value;
+
+	now = __blk_stat_time(ktime_to_ns(ktime_get()));
+	if (now < blk_stat_time(&rq->issue_stat))
+		return;
+
+	if (!__blk_stat_is_current(stat, now))
+		__blk_stat_init(stat, now);
+
+	value = now - blk_stat_time(&rq->issue_stat);
+	if (value > stat->max)
+		stat->max = value;
+	if (value < stat->min)
+		stat->min = value;
+
+	if (stat->batch + value < stat->batch ||
+	    stat->nr_batch + 1 == BLK_RQ_STAT_BATCH)
+		blk_stat_flush_batch(stat);
+
+	stat->batch += value;
+	stat->nr_batch++;
+}
+
+void blk_stat_clear(struct request_queue *q)
+{
+	if (q->mq_ops) {
+		struct blk_mq_hw_ctx *hctx;
+		struct blk_mq_ctx *ctx;
+		int i, j;
+
+		queue_for_each_hw_ctx(q, hctx, i) {
+			hctx_for_each_ctx(hctx, ctx, j) {
+				blk_stat_init(&ctx->stat[BLK_STAT_READ]);
+				blk_stat_init(&ctx->stat[BLK_STAT_WRITE]);
+			}
+		}
+	} else {
+		blk_stat_init(&q->rq_stats[BLK_STAT_READ]);
+		blk_stat_init(&q->rq_stats[BLK_STAT_WRITE]);
+	}
+}
+
+void blk_stat_set_issue_time(struct blk_issue_stat *stat)
+{
+	stat->time = (stat->time & BLK_STAT_MASK) |
+			(ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK);
+}
+
+/*
+ * Enable stat tracking, return whether it was enabled
+ */
+bool blk_stat_enable(struct request_queue *q)
+{
+	if (!test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
+		set_bit(QUEUE_FLAG_STATS, &q->queue_flags);
+		return false;
+	}
+
+	return true;
+}
diff --git a/block/blk-stat.h b/block/blk-stat.h
new file mode 100644
index 000000000000..a2050a0a5314
--- /dev/null
+++ b/block/blk-stat.h
@@ -0,0 +1,42 @@
+#ifndef BLK_STAT_H
+#define BLK_STAT_H
+
+/*
+ * ~0.13s window as a power-of-2 (2^27 nsecs)
+ */
+#define BLK_STAT_NSEC		134217728ULL
+#define BLK_STAT_NSEC_MASK	~(BLK_STAT_NSEC - 1)
+
+/*
+ * Upper 3 bits can be used elsewhere
+ */
+#define BLK_STAT_RES_BITS	3
+#define BLK_STAT_SHIFT		(64 - BLK_STAT_RES_BITS)
+#define BLK_STAT_TIME_MASK	((1ULL << BLK_STAT_SHIFT) - 1)
+#define BLK_STAT_MASK		~BLK_STAT_TIME_MASK
+
+enum {
+	BLK_STAT_READ	= 0,
+	BLK_STAT_WRITE,
+};
+
+void blk_stat_add(struct blk_rq_stat *, struct request *);
+void blk_hctx_stat_get(struct blk_mq_hw_ctx *, struct blk_rq_stat *);
+void blk_queue_stat_get(struct request_queue *, struct blk_rq_stat *);
+void blk_stat_clear(struct request_queue *);
+void blk_stat_init(struct blk_rq_stat *);
+bool blk_stat_is_current(struct blk_rq_stat *);
+void blk_stat_set_issue_time(struct blk_issue_stat *);
+bool blk_stat_enable(struct request_queue *);
+
+static inline u64 __blk_stat_time(u64 time)
+{
+	return time & BLK_STAT_TIME_MASK;
+}
+
+static inline u64 blk_stat_time(struct blk_issue_stat *stat)
+{
+	return __blk_stat_time(stat->time);
+}
+
+#endif
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index ea374e820775..1dbce057592d 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -13,6 +13,7 @@
 
 #include "blk.h"
 #include "blk-mq.h"
+#include "blk-wbt.h"
 
 struct queue_sysfs_entry {
 	struct attribute attr;
@@ -41,6 +42,19 @@ queue_var_store(unsigned long *var, const char *page, size_t count)
 	return count;
 }
 
+static ssize_t queue_var_store64(s64 *var, const char *page)
+{
+	int err;
+	s64 v;
+
+	err = kstrtos64(page, 10, &v);
+	if (err < 0)
+		return err;
+
+	*var = v;
+	return 0;
+}
+
 static ssize_t queue_requests_show(struct request_queue *q, char *page)
 {
 	return queue_var_show(q->nr_requests, (page));
@@ -130,6 +144,11 @@ static ssize_t queue_physical_block_size_show(struct request_queue *q, char *pag
 	return queue_var_show(queue_physical_block_size(q), page);
 }
 
+static ssize_t queue_chunk_sectors_show(struct request_queue *q, char *page)
+{
+	return queue_var_show(q->limits.chunk_sectors, page);
+}
+
 static ssize_t queue_io_min_show(struct request_queue *q, char *page)
 {
 	return queue_var_show(queue_io_min(q), page);
@@ -192,6 +211,11 @@ static ssize_t queue_write_same_max_show(struct request_queue *q, char *page)
 		(unsigned long long)q->limits.max_write_same_sectors << 9);
 }
 
+static ssize_t queue_write_zeroes_max_show(struct request_queue *q, char *page)
+{
+	return sprintf(page, "%llu\n",
+		(unsigned long long)q->limits.max_write_zeroes_sectors << 9);
+}
 
 static ssize_t
 queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
@@ -258,6 +282,18 @@ QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0);
 QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0);
 #undef QUEUE_SYSFS_BIT_FNS
 
+static ssize_t queue_zoned_show(struct request_queue *q, char *page)
+{
+	switch (blk_queue_zoned_model(q)) {
+	case BLK_ZONED_HA:
+		return sprintf(page, "host-aware\n");
+	case BLK_ZONED_HM:
+		return sprintf(page, "host-managed\n");
+	default:
+		return sprintf(page, "none\n");
+	}
+}
+
 static ssize_t queue_nomerges_show(struct request_queue *q, char *page)
 {
 	return queue_var_show((blk_queue_nomerges(q) << 1) |
@@ -320,6 +356,38 @@ queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)
 	return ret;
 }
 
+static ssize_t queue_poll_delay_show(struct request_queue *q, char *page)
+{
+	int val;
+
+	if (q->poll_nsec == -1)
+		val = -1;
+	else
+		val = q->poll_nsec / 1000;
+
+	return sprintf(page, "%d\n", val);
+}
+
+static ssize_t queue_poll_delay_store(struct request_queue *q, const char *page,
+				size_t count)
+{
+	int err, val;
+
+	if (!q->mq_ops || !q->mq_ops->poll)
+		return -EINVAL;
+
+	err = kstrtoint(page, 10, &val);
+	if (err < 0)
+		return err;
+
+	if (val == -1)
+		q->poll_nsec = -1;
+	else
+		q->poll_nsec = val * 1000;
+
+	return count;
+}
+
 static ssize_t queue_poll_show(struct request_queue *q, char *page)
 {
 	return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);
@@ -348,6 +416,50 @@ static ssize_t queue_poll_store(struct request_queue *q, const char *page,
 	return ret;
 }
 
+static ssize_t queue_wb_lat_show(struct request_queue *q, char *page)
+{
+	if (!q->rq_wb)
+		return -EINVAL;
+
+	return sprintf(page, "%llu\n", div_u64(q->rq_wb->min_lat_nsec, 1000));
+}
+
+static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,
+				  size_t count)
+{
+	struct rq_wb *rwb;
+	ssize_t ret;
+	s64 val;
+
+	ret = queue_var_store64(&val, page);
+	if (ret < 0)
+		return ret;
+	if (val < -1)
+		return -EINVAL;
+
+	rwb = q->rq_wb;
+	if (!rwb) {
+		ret = wbt_init(q);
+		if (ret)
+			return ret;
+
+		rwb = q->rq_wb;
+		if (!rwb)
+			return -EINVAL;
+	}
+
+	if (val == -1)
+		rwb->min_lat_nsec = wbt_default_latency_nsec(q);
+	else if (val >= 0)
+		rwb->min_lat_nsec = val * 1000ULL;
+
+	if (rwb->enable_state == WBT_STATE_ON_DEFAULT)
+		rwb->enable_state = WBT_STATE_ON_MANUAL;
+
+	wbt_update_limits(rwb);
+	return count;
+}
+
 static ssize_t queue_wc_show(struct request_queue *q, char *page)
 {
 	if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
@@ -385,6 +497,26 @@ static ssize_t queue_dax_show(struct request_queue *q, char *page)
 	return queue_var_show(blk_queue_dax(q), page);
 }
 
+static ssize_t print_stat(char *page, struct blk_rq_stat *stat, const char *pre)
+{
+	return sprintf(page, "%s samples=%llu, mean=%lld, min=%lld, max=%lld\n",
+			pre, (long long) stat->nr_samples,
+			(long long) stat->mean, (long long) stat->min,
+			(long long) stat->max);
+}
+
+static ssize_t queue_stats_show(struct request_queue *q, char *page)
+{
+	struct blk_rq_stat stat[2];
+	ssize_t ret;
+
+	blk_queue_stat_get(q, stat);
+
+	ret = print_stat(page, &stat[BLK_STAT_READ], "read :");
+	ret += print_stat(page + ret, &stat[BLK_STAT_WRITE], "write:");
+	return ret;
+}
+
 static struct queue_sysfs_entry queue_requests_entry = {
 	.attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },
 	.show = queue_requests_show,
@@ -444,6 +576,11 @@ static struct queue_sysfs_entry queue_physical_block_size_entry = {
 	.show = queue_physical_block_size_show,
 };
 
+static struct queue_sysfs_entry queue_chunk_sectors_entry = {
+	.attr = {.name = "chunk_sectors", .mode = S_IRUGO },
+	.show = queue_chunk_sectors_show,
+};
+
 static struct queue_sysfs_entry queue_io_min_entry = {
 	.attr = {.name = "minimum_io_size", .mode = S_IRUGO },
 	.show = queue_io_min_show,
@@ -480,12 +617,22 @@ static struct queue_sysfs_entry queue_write_same_max_entry = {
 	.show = queue_write_same_max_show,
 };
 
+static struct queue_sysfs_entry queue_write_zeroes_max_entry = {
+	.attr = {.name = "write_zeroes_max_bytes", .mode = S_IRUGO },
+	.show = queue_write_zeroes_max_show,
+};
+
 static struct queue_sysfs_entry queue_nonrot_entry = {
 	.attr = {.name = "rotational", .mode = S_IRUGO | S_IWUSR },
 	.show = queue_show_nonrot,
 	.store = queue_store_nonrot,
 };
 
+static struct queue_sysfs_entry queue_zoned_entry = {
+	.attr = {.name = "zoned", .mode = S_IRUGO },
+	.show = queue_zoned_show,
+};
+
 static struct queue_sysfs_entry queue_nomerges_entry = {
 	.attr = {.name = "nomerges", .mode = S_IRUGO | S_IWUSR },
 	.show = queue_nomerges_show,
@@ -516,6 +663,12 @@ static struct queue_sysfs_entry queue_poll_entry = {
 	.store = queue_poll_store,
 };
 
+static struct queue_sysfs_entry queue_poll_delay_entry = {
+	.attr = {.name = "io_poll_delay", .mode = S_IRUGO | S_IWUSR },
+	.show = queue_poll_delay_show,
+	.store = queue_poll_delay_store,
+};
+
 static struct queue_sysfs_entry queue_wc_entry = {
 	.attr = {.name = "write_cache", .mode = S_IRUGO | S_IWUSR },
 	.show = queue_wc_show,
@@ -527,6 +680,17 @@ static struct queue_sysfs_entry queue_dax_entry = {
 	.show = queue_dax_show,
 };
 
+static struct queue_sysfs_entry queue_stats_entry = {
+	.attr = {.name = "stats", .mode = S_IRUGO },
+	.show = queue_stats_show,
+};
+
+static struct queue_sysfs_entry queue_wb_lat_entry = {
+	.attr = {.name = "wbt_lat_usec", .mode = S_IRUGO | S_IWUSR },
+	.show = queue_wb_lat_show,
+	.store = queue_wb_lat_store,
+};
+
 static struct attribute *default_attrs[] = {
 	&queue_requests_entry.attr,
 	&queue_ra_entry.attr,
@@ -539,6 +703,7 @@ static struct attribute *default_attrs[] = {
 	&queue_hw_sector_size_entry.attr,
 	&queue_logical_block_size_entry.attr,
 	&queue_physical_block_size_entry.attr,
+	&queue_chunk_sectors_entry.attr,
 	&queue_io_min_entry.attr,
 	&queue_io_opt_entry.attr,
 	&queue_discard_granularity_entry.attr,
@@ -546,7 +711,9 @@ static struct attribute *default_attrs[] = {
 	&queue_discard_max_hw_entry.attr,
 	&queue_discard_zeroes_data_entry.attr,
 	&queue_write_same_max_entry.attr,
+	&queue_write_zeroes_max_entry.attr,
 	&queue_nonrot_entry.attr,
+	&queue_zoned_entry.attr,
 	&queue_nomerges_entry.attr,
 	&queue_rq_affinity_entry.attr,
 	&queue_iostats_entry.attr,
@@ -554,6 +721,9 @@ static struct attribute *default_attrs[] = {
 	&queue_poll_entry.attr,
 	&queue_wc_entry.attr,
 	&queue_dax_entry.attr,
+	&queue_stats_entry.attr,
+	&queue_wb_lat_entry.attr,
+	&queue_poll_delay_entry.attr,
 	NULL,
 };
 
@@ -628,6 +798,7 @@ static void blk_release_queue(struct kobject *kobj)
 	struct request_queue *q =
 		container_of(kobj, struct request_queue, kobj);
 
+	wbt_exit(q);
 	bdi_exit(&q->backing_dev_info);
 	blkcg_exit_queue(q);
 
@@ -668,6 +839,23 @@ struct kobj_type blk_queue_ktype = {
 	.release	= blk_release_queue,
 };
 
+static void blk_wb_init(struct request_queue *q)
+{
+#ifndef CONFIG_BLK_WBT_MQ
+	if (q->mq_ops)
+		return;
+#endif
+#ifndef CONFIG_BLK_WBT_SQ
+	if (q->request_fn)
+		return;
+#endif
+
+	/*
+	 * If this fails, we don't get throttling
+	 */
+	wbt_init(q);
+}
+
 int blk_register_queue(struct gendisk *disk)
 {
 	int ret;
@@ -707,6 +895,8 @@ int blk_register_queue(struct gendisk *disk)
 	if (q->mq_ops)
 		blk_mq_register_dev(dev, q);
 
+	blk_wb_init(q);
+
 	if (!q->request_fn)
 		return 0;
 
diff --git a/block/blk-tag.c b/block/blk-tag.c
index f0344e6939d5..bae1decb6ec3 100644
--- a/block/blk-tag.c
+++ b/block/blk-tag.c
@@ -270,7 +270,7 @@ void blk_queue_end_tag(struct request_queue *q, struct request *rq)
 	BUG_ON(tag >= bqt->real_max_depth);
 
 	list_del_init(&rq->queuelist);
-	rq->cmd_flags &= ~REQ_QUEUED;
+	rq->rq_flags &= ~RQF_QUEUED;
 	rq->tag = -1;
 
 	if (unlikely(bqt->tag_index[tag] == NULL))
@@ -316,7 +316,7 @@ int blk_queue_start_tag(struct request_queue *q, struct request *rq)
 	unsigned max_depth;
 	int tag;
 
-	if (unlikely((rq->cmd_flags & REQ_QUEUED))) {
+	if (unlikely((rq->rq_flags & RQF_QUEUED))) {
 		printk(KERN_ERR
 		       "%s: request %p for device [%s] already tagged %d",
 		       __func__, rq,
@@ -371,7 +371,7 @@ int blk_queue_start_tag(struct request_queue *q, struct request *rq)
 	 */
 
 	bqt->next_tag = (tag + 1) % bqt->max_depth;
-	rq->cmd_flags |= REQ_QUEUED;
+	rq->rq_flags |= RQF_QUEUED;
 	rq->tag = tag;
 	bqt->tag_index[tag] = rq;
 	blk_start_request(rq);
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index a3ea8260c94c..a6bb4fe326c3 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -818,13 +818,13 @@ static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)
 	tg->io_disp[rw]++;
 
 	/*
-	 * REQ_THROTTLED is used to prevent the same bio to be throttled
+	 * BIO_THROTTLED is used to prevent the same bio to be throttled
 	 * more than once as a throttled bio will go through blk-throtl the
 	 * second time when it eventually gets issued.  Set it when a bio
 	 * is being charged to a tg.
 	 */
-	if (!(bio->bi_opf & REQ_THROTTLED))
-		bio->bi_opf |= REQ_THROTTLED;
+	if (!bio_flagged(bio, BIO_THROTTLED))
+		bio_set_flag(bio, BIO_THROTTLED);
 }
 
 /**
@@ -1401,7 +1401,7 @@ bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg,
 	WARN_ON_ONCE(!rcu_read_lock_held());
 
 	/* see throtl_charge_bio() */
-	if ((bio->bi_opf & REQ_THROTTLED) || !tg->has_rules[rw])
+	if (bio_flagged(bio, BIO_THROTTLED) || !tg->has_rules[rw])
 		goto out;
 
 	spin_lock_irq(q->queue_lock);
@@ -1480,7 +1480,7 @@ out:
 	 * being issued.
 	 */
 	if (!throttled)
-		bio->bi_opf &= ~REQ_THROTTLED;
+		bio_clear_flag(bio, BIO_THROTTLED);
 	return throttled;
 }
 
diff --git a/block/blk-wbt.c b/block/blk-wbt.c
new file mode 100644
index 000000000000..6e82769f4042
--- /dev/null
+++ b/block/blk-wbt.c
@@ -0,0 +1,750 @@
+/*
+ * buffered writeback throttling. loosely based on CoDel. We can't drop
+ * packets for IO scheduling, so the logic is something like this:
+ *
+ * - Monitor latencies in a defined window of time.
+ * - If the minimum latency in the above window exceeds some target, increment
+ *   scaling step and scale down queue depth by a factor of 2x. The monitoring
+ *   window is then shrunk to 100 / sqrt(scaling step + 1).
+ * - For any window where we don't have solid data on what the latencies
+ *   look like, retain status quo.
+ * - If latencies look good, decrement scaling step.
+ * - If we're only doing writes, allow the scaling step to go negative. This
+ *   will temporarily boost write performance, snapping back to a stable
+ *   scaling step of 0 if reads show up or the heavy writers finish. Unlike
+ *   positive scaling steps where we shrink the monitoring window, a negative
+ *   scaling step retains the default step==0 window size.
+ *
+ * Copyright (C) 2016 Jens Axboe
+ *
+ */
+#include <linux/kernel.h>
+#include <linux/blk_types.h>
+#include <linux/slab.h>
+#include <linux/backing-dev.h>
+#include <linux/swap.h>
+
+#include "blk-wbt.h"
+
+#define CREATE_TRACE_POINTS
+#include <trace/events/wbt.h>
+
+enum {
+	/*
+	 * Default setting, we'll scale up (to 75% of QD max) or down (min 1)
+	 * from here depending on device stats
+	 */
+	RWB_DEF_DEPTH	= 16,
+
+	/*
+	 * 100msec window
+	 */
+	RWB_WINDOW_NSEC		= 100 * 1000 * 1000ULL,
+
+	/*
+	 * Disregard stats, if we don't meet this minimum
+	 */
+	RWB_MIN_WRITE_SAMPLES	= 3,
+
+	/*
+	 * If we have this number of consecutive windows with not enough
+	 * information to scale up or down, scale up.
+	 */
+	RWB_UNKNOWN_BUMP	= 5,
+};
+
+static inline bool rwb_enabled(struct rq_wb *rwb)
+{
+	return rwb && rwb->wb_normal != 0;
+}
+
+/*
+ * Increment 'v', if 'v' is below 'below'. Returns true if we succeeded,
+ * false if 'v' + 1 would be bigger than 'below'.
+ */
+static bool atomic_inc_below(atomic_t *v, int below)
+{
+	int cur = atomic_read(v);
+
+	for (;;) {
+		int old;
+
+		if (cur >= below)
+			return false;
+		old = atomic_cmpxchg(v, cur, cur + 1);
+		if (old == cur)
+			break;
+		cur = old;
+	}
+
+	return true;
+}
+
+static void wb_timestamp(struct rq_wb *rwb, unsigned long *var)
+{
+	if (rwb_enabled(rwb)) {
+		const unsigned long cur = jiffies;
+
+		if (cur != *var)
+			*var = cur;
+	}
+}
+
+/*
+ * If a task was rate throttled in balance_dirty_pages() within the last
+ * second or so, use that to indicate a higher cleaning rate.
+ */
+static bool wb_recent_wait(struct rq_wb *rwb)
+{
+	struct bdi_writeback *wb = &rwb->queue->backing_dev_info.wb;
+
+	return time_before(jiffies, wb->dirty_sleep + HZ);
+}
+
+static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb, bool is_kswapd)
+{
+	return &rwb->rq_wait[is_kswapd];
+}
+
+static void rwb_wake_all(struct rq_wb *rwb)
+{
+	int i;
+
+	for (i = 0; i < WBT_NUM_RWQ; i++) {
+		struct rq_wait *rqw = &rwb->rq_wait[i];
+
+		if (waitqueue_active(&rqw->wait))
+			wake_up_all(&rqw->wait);
+	}
+}
+
+void __wbt_done(struct rq_wb *rwb, enum wbt_flags wb_acct)
+{
+	struct rq_wait *rqw;
+	int inflight, limit;
+
+	if (!(wb_acct & WBT_TRACKED))
+		return;
+
+	rqw = get_rq_wait(rwb, wb_acct & WBT_KSWAPD);
+	inflight = atomic_dec_return(&rqw->inflight);
+
+	/*
+	 * wbt got disabled with IO in flight. Wake up any potential
+	 * waiters, we don't have to do more than that.
+	 */
+	if (unlikely(!rwb_enabled(rwb))) {
+		rwb_wake_all(rwb);
+		return;
+	}
+
+	/*
+	 * If the device does write back caching, drop further down
+	 * before we wake people up.
+	 */
+	if (rwb->wc && !wb_recent_wait(rwb))
+		limit = 0;
+	else
+		limit = rwb->wb_normal;
+
+	/*
+	 * Don't wake anyone up if we are above the normal limit.
+	 */
+	if (inflight && inflight >= limit)
+		return;
+
+	if (waitqueue_active(&rqw->wait)) {
+		int diff = limit - inflight;
+
+		if (!inflight || diff >= rwb->wb_background / 2)
+			wake_up_all(&rqw->wait);
+	}
+}
+
+/*
+ * Called on completion of a request. Note that it's also called when
+ * a request is merged, when the request gets freed.
+ */
+void wbt_done(struct rq_wb *rwb, struct blk_issue_stat *stat)
+{
+	if (!rwb)
+		return;
+
+	if (!wbt_is_tracked(stat)) {
+		if (rwb->sync_cookie == stat) {
+			rwb->sync_issue = 0;
+			rwb->sync_cookie = NULL;
+		}
+
+		if (wbt_is_read(stat))
+			wb_timestamp(rwb, &rwb->last_comp);
+		wbt_clear_state(stat);
+	} else {
+		WARN_ON_ONCE(stat == rwb->sync_cookie);
+		__wbt_done(rwb, wbt_stat_to_mask(stat));
+		wbt_clear_state(stat);
+	}
+}
+
+/*
+ * Return true, if we can't increase the depth further by scaling
+ */
+static bool calc_wb_limits(struct rq_wb *rwb)
+{
+	unsigned int depth;
+	bool ret = false;
+
+	if (!rwb->min_lat_nsec) {
+		rwb->wb_max = rwb->wb_normal = rwb->wb_background = 0;
+		return false;
+	}
+
+	/*
+	 * For QD=1 devices, this is a special case. It's important for those
+	 * to have one request ready when one completes, so force a depth of
+	 * 2 for those devices. On the backend, it'll be a depth of 1 anyway,
+	 * since the device can't have more than that in flight. If we're
+	 * scaling down, then keep a setting of 1/1/1.
+	 */
+	if (rwb->queue_depth == 1) {
+		if (rwb->scale_step > 0)
+			rwb->wb_max = rwb->wb_normal = 1;
+		else {
+			rwb->wb_max = rwb->wb_normal = 2;
+			ret = true;
+		}
+		rwb->wb_background = 1;
+	} else {
+		/*
+		 * scale_step == 0 is our default state. If we have suffered
+		 * latency spikes, step will be > 0, and we shrink the
+		 * allowed write depths. If step is < 0, we're only doing
+		 * writes, and we allow a temporarily higher depth to
+		 * increase performance.
+		 */
+		depth = min_t(unsigned int, RWB_DEF_DEPTH, rwb->queue_depth);
+		if (rwb->scale_step > 0)
+			depth = 1 + ((depth - 1) >> min(31, rwb->scale_step));
+		else if (rwb->scale_step < 0) {
+			unsigned int maxd = 3 * rwb->queue_depth / 4;
+
+			depth = 1 + ((depth - 1) << -rwb->scale_step);
+			if (depth > maxd) {
+				depth = maxd;
+				ret = true;
+			}
+		}
+
+		/*
+		 * Set our max/normal/bg queue depths based on how far
+		 * we have scaled down (->scale_step).
+		 */
+		rwb->wb_max = depth;
+		rwb->wb_normal = (rwb->wb_max + 1) / 2;
+		rwb->wb_background = (rwb->wb_max + 3) / 4;
+	}
+
+	return ret;
+}
+
+static inline bool stat_sample_valid(struct blk_rq_stat *stat)
+{
+	/*
+	 * We need at least one read sample, and a minimum of
+	 * RWB_MIN_WRITE_SAMPLES. We require some write samples to know
+	 * that it's writes impacting us, and not just some sole read on
+	 * a device that is in a lower power state.
+	 */
+	return stat[BLK_STAT_READ].nr_samples >= 1 &&
+		stat[BLK_STAT_WRITE].nr_samples >= RWB_MIN_WRITE_SAMPLES;
+}
+
+static u64 rwb_sync_issue_lat(struct rq_wb *rwb)
+{
+	u64 now, issue = ACCESS_ONCE(rwb->sync_issue);
+
+	if (!issue || !rwb->sync_cookie)
+		return 0;
+
+	now = ktime_to_ns(ktime_get());
+	return now - issue;
+}
+
+enum {
+	LAT_OK = 1,
+	LAT_UNKNOWN,
+	LAT_UNKNOWN_WRITES,
+	LAT_EXCEEDED,
+};
+
+static int __latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat)
+{
+	struct backing_dev_info *bdi = &rwb->queue->backing_dev_info;
+	u64 thislat;
+
+	/*
+	 * If our stored sync issue exceeds the window size, or it
+	 * exceeds our min target AND we haven't logged any entries,
+	 * flag the latency as exceeded. wbt works off completion latencies,
+	 * but for a flooded device, a single sync IO can take a long time
+	 * to complete after being issued. If this time exceeds our
+	 * monitoring window AND we didn't see any other completions in that
+	 * window, then count that sync IO as a violation of the latency.
+	 */
+	thislat = rwb_sync_issue_lat(rwb);
+	if (thislat > rwb->cur_win_nsec ||
+	    (thislat > rwb->min_lat_nsec && !stat[BLK_STAT_READ].nr_samples)) {
+		trace_wbt_lat(bdi, thislat);
+		return LAT_EXCEEDED;
+	}
+
+	/*
+	 * No read/write mix, if stat isn't valid
+	 */
+	if (!stat_sample_valid(stat)) {
+		/*
+		 * If we had writes in this stat window and the window is
+		 * current, we're only doing writes. If a task recently
+		 * waited or still has writes in flights, consider us doing
+		 * just writes as well.
+		 */
+		if ((stat[BLK_STAT_WRITE].nr_samples && blk_stat_is_current(stat)) ||
+		    wb_recent_wait(rwb) || wbt_inflight(rwb))
+			return LAT_UNKNOWN_WRITES;
+		return LAT_UNKNOWN;
+	}
+
+	/*
+	 * If the 'min' latency exceeds our target, step down.
+	 */
+	if (stat[BLK_STAT_READ].min > rwb->min_lat_nsec) {
+		trace_wbt_lat(bdi, stat[BLK_STAT_READ].min);
+		trace_wbt_stat(bdi, stat);
+		return LAT_EXCEEDED;
+	}
+
+	if (rwb->scale_step)
+		trace_wbt_stat(bdi, stat);
+
+	return LAT_OK;
+}
+
+static int latency_exceeded(struct rq_wb *rwb)
+{
+	struct blk_rq_stat stat[2];
+
+	blk_queue_stat_get(rwb->queue, stat);
+	return __latency_exceeded(rwb, stat);
+}
+
+static void rwb_trace_step(struct rq_wb *rwb, const char *msg)
+{
+	struct backing_dev_info *bdi = &rwb->queue->backing_dev_info;
+
+	trace_wbt_step(bdi, msg, rwb->scale_step, rwb->cur_win_nsec,
+			rwb->wb_background, rwb->wb_normal, rwb->wb_max);
+}
+
+static void scale_up(struct rq_wb *rwb)
+{
+	/*
+	 * Hit max in previous round, stop here
+	 */
+	if (rwb->scaled_max)
+		return;
+
+	rwb->scale_step--;
+	rwb->unknown_cnt = 0;
+	blk_stat_clear(rwb->queue);
+
+	rwb->scaled_max = calc_wb_limits(rwb);
+
+	rwb_wake_all(rwb);
+
+	rwb_trace_step(rwb, "step up");
+}
+
+/*
+ * Scale rwb down. If 'hard_throttle' is set, do it quicker, since we
+ * had a latency violation.
+ */
+static void scale_down(struct rq_wb *rwb, bool hard_throttle)
+{
+	/*
+	 * Stop scaling down when we've hit the limit. This also prevents
+	 * ->scale_step from going to crazy values, if the device can't
+	 * keep up.
+	 */
+	if (rwb->wb_max == 1)
+		return;
+
+	if (rwb->scale_step < 0 && hard_throttle)
+		rwb->scale_step = 0;
+	else
+		rwb->scale_step++;
+
+	rwb->scaled_max = false;
+	rwb->unknown_cnt = 0;
+	blk_stat_clear(rwb->queue);
+	calc_wb_limits(rwb);
+	rwb_trace_step(rwb, "step down");
+}
+
+static void rwb_arm_timer(struct rq_wb *rwb)
+{
+	unsigned long expires;
+
+	if (rwb->scale_step > 0) {
+		/*
+		 * We should speed this up, using some variant of a fast
+		 * integer inverse square root calculation. Since we only do
+		 * this for every window expiration, it's not a huge deal,
+		 * though.
+		 */
+		rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4,
+					int_sqrt((rwb->scale_step + 1) << 8));
+	} else {
+		/*
+		 * For step < 0, we don't want to increase/decrease the
+		 * window size.
+		 */
+		rwb->cur_win_nsec = rwb->win_nsec;
+	}
+
+	expires = jiffies + nsecs_to_jiffies(rwb->cur_win_nsec);
+	mod_timer(&rwb->window_timer, expires);
+}
+
+static void wb_timer_fn(unsigned long data)
+{
+	struct rq_wb *rwb = (struct rq_wb *) data;
+	unsigned int inflight = wbt_inflight(rwb);
+	int status;
+
+	status = latency_exceeded(rwb);
+
+	trace_wbt_timer(&rwb->queue->backing_dev_info, status, rwb->scale_step,
+			inflight);
+
+	/*
+	 * If we exceeded the latency target, step down. If we did not,
+	 * step one level up. If we don't know enough to say either exceeded
+	 * or ok, then don't do anything.
+	 */
+	switch (status) {
+	case LAT_EXCEEDED:
+		scale_down(rwb, true);
+		break;
+	case LAT_OK:
+		scale_up(rwb);
+		break;
+	case LAT_UNKNOWN_WRITES:
+		/*
+		 * We started a the center step, but don't have a valid
+		 * read/write sample, but we do have writes going on.
+		 * Allow step to go negative, to increase write perf.
+		 */
+		scale_up(rwb);
+		break;
+	case LAT_UNKNOWN:
+		if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP)
+			break;
+		/*
+		 * We get here when previously scaled reduced depth, and we
+		 * currently don't have a valid read/write sample. For that
+		 * case, slowly return to center state (step == 0).
+		 */
+		if (rwb->scale_step > 0)
+			scale_up(rwb);
+		else if (rwb->scale_step < 0)
+			scale_down(rwb, false);
+		break;
+	default:
+		break;
+	}
+
+	/*
+	 * Re-arm timer, if we have IO in flight
+	 */
+	if (rwb->scale_step || inflight)
+		rwb_arm_timer(rwb);
+}
+
+void wbt_update_limits(struct rq_wb *rwb)
+{
+	rwb->scale_step = 0;
+	rwb->scaled_max = false;
+	calc_wb_limits(rwb);
+
+	rwb_wake_all(rwb);
+}
+
+static bool close_io(struct rq_wb *rwb)
+{
+	const unsigned long now = jiffies;
+
+	return time_before(now, rwb->last_issue + HZ / 10) ||
+		time_before(now, rwb->last_comp + HZ / 10);
+}
+
+#define REQ_HIPRIO	(REQ_SYNC | REQ_META | REQ_PRIO)
+
+static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw)
+{
+	unsigned int limit;
+
+	/*
+	 * At this point we know it's a buffered write. If this is
+	 * kswapd trying to free memory, or REQ_SYNC is set, set, then
+	 * it's WB_SYNC_ALL writeback, and we'll use the max limit for
+	 * that. If the write is marked as a background write, then use
+	 * the idle limit, or go to normal if we haven't had competing
+	 * IO for a bit.
+	 */
+	if ((rw & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd())
+		limit = rwb->wb_max;
+	else if ((rw & REQ_BACKGROUND) || close_io(rwb)) {
+		/*
+		 * If less than 100ms since we completed unrelated IO,
+		 * limit us to half the depth for background writeback.
+		 */
+		limit = rwb->wb_background;
+	} else
+		limit = rwb->wb_normal;
+
+	return limit;
+}
+
+static inline bool may_queue(struct rq_wb *rwb, struct rq_wait *rqw,
+			     wait_queue_t *wait, unsigned long rw)
+{
+	/*
+	 * inc it here even if disabled, since we'll dec it at completion.
+	 * this only happens if the task was sleeping in __wbt_wait(),
+	 * and someone turned it off at the same time.
+	 */
+	if (!rwb_enabled(rwb)) {
+		atomic_inc(&rqw->inflight);
+		return true;
+	}
+
+	/*
+	 * If the waitqueue is already active and we are not the next
+	 * in line to be woken up, wait for our turn.
+	 */
+	if (waitqueue_active(&rqw->wait) &&
+	    rqw->wait.task_list.next != &wait->task_list)
+		return false;
+
+	return atomic_inc_below(&rqw->inflight, get_limit(rwb, rw));
+}
+
+/*
+ * Block if we will exceed our limit, or if we are currently waiting for
+ * the timer to kick off queuing again.
+ */
+static void __wbt_wait(struct rq_wb *rwb, unsigned long rw, spinlock_t *lock)
+{
+	struct rq_wait *rqw = get_rq_wait(rwb, current_is_kswapd());
+	DEFINE_WAIT(wait);
+
+	if (may_queue(rwb, rqw, &wait, rw))
+		return;
+
+	do {
+		prepare_to_wait_exclusive(&rqw->wait, &wait,
+						TASK_UNINTERRUPTIBLE);
+
+		if (may_queue(rwb, rqw, &wait, rw))
+			break;
+
+		if (lock)
+			spin_unlock_irq(lock);
+
+		io_schedule();
+
+		if (lock)
+			spin_lock_irq(lock);
+	} while (1);
+
+	finish_wait(&rqw->wait, &wait);
+}
+
+static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio)
+{
+	const int op = bio_op(bio);
+
+	/*
+	 * If not a WRITE, do nothing
+	 */
+	if (op != REQ_OP_WRITE)
+		return false;
+
+	/*
+	 * Don't throttle WRITE_ODIRECT
+	 */
+	if ((bio->bi_opf & (REQ_SYNC | REQ_IDLE)) == (REQ_SYNC | REQ_IDLE))
+		return false;
+
+	return true;
+}
+
+/*
+ * Returns true if the IO request should be accounted, false if not.
+ * May sleep, if we have exceeded the writeback limits. Caller can pass
+ * in an irq held spinlock, if it holds one when calling this function.
+ * If we do sleep, we'll release and re-grab it.
+ */
+unsigned int wbt_wait(struct rq_wb *rwb, struct bio *bio, spinlock_t *lock)
+{
+	unsigned int ret = 0;
+
+	if (!rwb_enabled(rwb))
+		return 0;
+
+	if (bio_op(bio) == REQ_OP_READ)
+		ret = WBT_READ;
+
+	if (!wbt_should_throttle(rwb, bio)) {
+		if (ret & WBT_READ)
+			wb_timestamp(rwb, &rwb->last_issue);
+		return ret;
+	}
+
+	__wbt_wait(rwb, bio->bi_opf, lock);
+
+	if (!timer_pending(&rwb->window_timer))
+		rwb_arm_timer(rwb);
+
+	if (current_is_kswapd())
+		ret |= WBT_KSWAPD;
+
+	return ret | WBT_TRACKED;
+}
+
+void wbt_issue(struct rq_wb *rwb, struct blk_issue_stat *stat)
+{
+	if (!rwb_enabled(rwb))
+		return;
+
+	/*
+	 * Track sync issue, in case it takes a long time to complete. Allows
+	 * us to react quicker, if a sync IO takes a long time to complete.
+	 * Note that this is just a hint. 'stat' can go away when the
+	 * request completes, so it's important we never dereference it. We
+	 * only use the address to compare with, which is why we store the
+	 * sync_issue time locally.
+	 */
+	if (wbt_is_read(stat) && !rwb->sync_issue) {
+		rwb->sync_cookie = stat;
+		rwb->sync_issue = blk_stat_time(stat);
+	}
+}
+
+void wbt_requeue(struct rq_wb *rwb, struct blk_issue_stat *stat)
+{
+	if (!rwb_enabled(rwb))
+		return;
+	if (stat == rwb->sync_cookie) {
+		rwb->sync_issue = 0;
+		rwb->sync_cookie = NULL;
+	}
+}
+
+void wbt_set_queue_depth(struct rq_wb *rwb, unsigned int depth)
+{
+	if (rwb) {
+		rwb->queue_depth = depth;
+		wbt_update_limits(rwb);
+	}
+}
+
+void wbt_set_write_cache(struct rq_wb *rwb, bool write_cache_on)
+{
+	if (rwb)
+		rwb->wc = write_cache_on;
+}
+
+ /*
+ * Disable wbt, if enabled by default. Only called from CFQ, if we have
+ * cgroups enabled
+ */
+void wbt_disable_default(struct request_queue *q)
+{
+	struct rq_wb *rwb = q->rq_wb;
+
+	if (rwb && rwb->enable_state == WBT_STATE_ON_DEFAULT) {
+		del_timer_sync(&rwb->window_timer);
+		rwb->win_nsec = rwb->min_lat_nsec = 0;
+		wbt_update_limits(rwb);
+	}
+}
+EXPORT_SYMBOL_GPL(wbt_disable_default);
+
+u64 wbt_default_latency_nsec(struct request_queue *q)
+{
+	/*
+	 * We default to 2msec for non-rotational storage, and 75msec
+	 * for rotational storage.
+	 */
+	if (blk_queue_nonrot(q))
+		return 2000000ULL;
+	else
+		return 75000000ULL;
+}
+
+int wbt_init(struct request_queue *q)
+{
+	struct rq_wb *rwb;
+	int i;
+
+	/*
+	 * For now, we depend on the stats window being larger than
+	 * our monitoring window. Ensure that this isn't inadvertently
+	 * violated.
+	 */
+	BUILD_BUG_ON(RWB_WINDOW_NSEC > BLK_STAT_NSEC);
+	BUILD_BUG_ON(WBT_NR_BITS > BLK_STAT_RES_BITS);
+
+	rwb = kzalloc(sizeof(*rwb), GFP_KERNEL);
+	if (!rwb)
+		return -ENOMEM;
+
+	for (i = 0; i < WBT_NUM_RWQ; i++) {
+		atomic_set(&rwb->rq_wait[i].inflight, 0);
+		init_waitqueue_head(&rwb->rq_wait[i].wait);
+	}
+
+	setup_timer(&rwb->window_timer, wb_timer_fn, (unsigned long) rwb);
+	rwb->wc = 1;
+	rwb->queue_depth = RWB_DEF_DEPTH;
+	rwb->last_comp = rwb->last_issue = jiffies;
+	rwb->queue = q;
+	rwb->win_nsec = RWB_WINDOW_NSEC;
+	rwb->enable_state = WBT_STATE_ON_DEFAULT;
+	wbt_update_limits(rwb);
+
+	/*
+	 * Assign rwb, and turn on stats tracking for this queue
+	 */
+	q->rq_wb = rwb;
+	blk_stat_enable(q);
+
+	rwb->min_lat_nsec = wbt_default_latency_nsec(q);
+
+	wbt_set_queue_depth(rwb, blk_queue_depth(q));
+	wbt_set_write_cache(rwb, test_bit(QUEUE_FLAG_WC, &q->queue_flags));
+
+	return 0;
+}
+
+void wbt_exit(struct request_queue *q)
+{
+	struct rq_wb *rwb = q->rq_wb;
+
+	if (rwb) {
+		del_timer_sync(&rwb->window_timer);
+		q->rq_wb = NULL;
+		kfree(rwb);
+	}
+}
diff --git a/block/blk-wbt.h b/block/blk-wbt.h
new file mode 100644
index 000000000000..65f1de519f67
--- /dev/null
+++ b/block/blk-wbt.h
@@ -0,0 +1,171 @@
+#ifndef WB_THROTTLE_H
+#define WB_THROTTLE_H
+
+#include <linux/kernel.h>
+#include <linux/atomic.h>
+#include <linux/wait.h>
+#include <linux/timer.h>
+#include <linux/ktime.h>
+
+#include "blk-stat.h"
+
+enum wbt_flags {
+	WBT_TRACKED		= 1,	/* write, tracked for throttling */
+	WBT_READ		= 2,	/* read */
+	WBT_KSWAPD		= 4,	/* write, from kswapd */
+
+	WBT_NR_BITS		= 3,	/* number of bits */
+};
+
+enum {
+	WBT_NUM_RWQ		= 2,
+};
+
+/*
+ * Enable states. Either off, or on by default (done at init time),
+ * or on through manual setup in sysfs.
+ */
+enum {
+	WBT_STATE_ON_DEFAULT	= 1,
+	WBT_STATE_ON_MANUAL	= 2,
+};
+
+static inline void wbt_clear_state(struct blk_issue_stat *stat)
+{
+	stat->time &= BLK_STAT_TIME_MASK;
+}
+
+static inline enum wbt_flags wbt_stat_to_mask(struct blk_issue_stat *stat)
+{
+	return (stat->time & BLK_STAT_MASK) >> BLK_STAT_SHIFT;
+}
+
+static inline void wbt_track(struct blk_issue_stat *stat, enum wbt_flags wb_acct)
+{
+	stat->time |= ((u64) wb_acct) << BLK_STAT_SHIFT;
+}
+
+static inline bool wbt_is_tracked(struct blk_issue_stat *stat)
+{
+	return (stat->time >> BLK_STAT_SHIFT) & WBT_TRACKED;
+}
+
+static inline bool wbt_is_read(struct blk_issue_stat *stat)
+{
+	return (stat->time >> BLK_STAT_SHIFT) & WBT_READ;
+}
+
+struct rq_wait {
+	wait_queue_head_t wait;
+	atomic_t inflight;
+};
+
+struct rq_wb {
+	/*
+	 * Settings that govern how we throttle
+	 */
+	unsigned int wb_background;		/* background writeback */
+	unsigned int wb_normal;			/* normal writeback */
+	unsigned int wb_max;			/* max throughput writeback */
+	int scale_step;
+	bool scaled_max;
+
+	short enable_state;			/* WBT_STATE_* */
+
+	/*
+	 * Number of consecutive periods where we don't have enough
+	 * information to make a firm scale up/down decision.
+	 */
+	unsigned int unknown_cnt;
+
+	u64 win_nsec;				/* default window size */
+	u64 cur_win_nsec;			/* current window size */
+
+	struct timer_list window_timer;
+
+	s64 sync_issue;
+	void *sync_cookie;
+
+	unsigned int wc;
+	unsigned int queue_depth;
+
+	unsigned long last_issue;		/* last non-throttled issue */
+	unsigned long last_comp;		/* last non-throttled comp */
+	unsigned long min_lat_nsec;
+	struct request_queue *queue;
+	struct rq_wait rq_wait[WBT_NUM_RWQ];
+};
+
+static inline unsigned int wbt_inflight(struct rq_wb *rwb)
+{
+	unsigned int i, ret = 0;
+
+	for (i = 0; i < WBT_NUM_RWQ; i++)
+		ret += atomic_read(&rwb->rq_wait[i].inflight);
+
+	return ret;
+}
+
+#ifdef CONFIG_BLK_WBT
+
+void __wbt_done(struct rq_wb *, enum wbt_flags);
+void wbt_done(struct rq_wb *, struct blk_issue_stat *);
+enum wbt_flags wbt_wait(struct rq_wb *, struct bio *, spinlock_t *);
+int wbt_init(struct request_queue *);
+void wbt_exit(struct request_queue *);
+void wbt_update_limits(struct rq_wb *);
+void wbt_requeue(struct rq_wb *, struct blk_issue_stat *);
+void wbt_issue(struct rq_wb *, struct blk_issue_stat *);
+void wbt_disable_default(struct request_queue *);
+
+void wbt_set_queue_depth(struct rq_wb *, unsigned int);
+void wbt_set_write_cache(struct rq_wb *, bool);
+
+u64 wbt_default_latency_nsec(struct request_queue *);
+
+#else
+
+static inline void __wbt_done(struct rq_wb *rwb, enum wbt_flags flags)
+{
+}
+static inline void wbt_done(struct rq_wb *rwb, struct blk_issue_stat *stat)
+{
+}
+static inline enum wbt_flags wbt_wait(struct rq_wb *rwb, struct bio *bio,
+				      spinlock_t *lock)
+{
+	return 0;
+}
+static inline int wbt_init(struct request_queue *q)
+{
+	return -EINVAL;
+}
+static inline void wbt_exit(struct request_queue *q)
+{
+}
+static inline void wbt_update_limits(struct rq_wb *rwb)
+{
+}
+static inline void wbt_requeue(struct rq_wb *rwb, struct blk_issue_stat *stat)
+{
+}
+static inline void wbt_issue(struct rq_wb *rwb, struct blk_issue_stat *stat)
+{
+}
+static inline void wbt_disable_default(struct request_queue *q)
+{
+}
+static inline void wbt_set_queue_depth(struct rq_wb *rwb, unsigned int depth)
+{
+}
+static inline void wbt_set_write_cache(struct rq_wb *rwb, bool wc)
+{
+}
+static inline u64 wbt_default_latency_nsec(struct request_queue *q)
+{
+	return 0;
+}
+
+#endif /* CONFIG_BLK_WBT */
+
+#endif
diff --git a/block/blk-zoned.c b/block/blk-zoned.c
new file mode 100644
index 000000000000..472211fa183a
--- /dev/null
+++ b/block/blk-zoned.c
@@ -0,0 +1,348 @@
+/*
+ * Zoned block device handling
+ *
+ * Copyright (c) 2015, Hannes Reinecke
+ * Copyright (c) 2015, SUSE Linux GmbH
+ *
+ * Copyright (c) 2016, Damien Le Moal
+ * Copyright (c) 2016, Western Digital
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/rbtree.h>
+#include <linux/blkdev.h>
+
+static inline sector_t blk_zone_start(struct request_queue *q,
+				      sector_t sector)
+{
+	sector_t zone_mask = blk_queue_zone_size(q) - 1;
+
+	return sector & ~zone_mask;
+}
+
+/*
+ * Check that a zone report belongs to the partition.
+ * If yes, fix its start sector and write pointer, copy it in the
+ * zone information array and return true. Return false otherwise.
+ */
+static bool blkdev_report_zone(struct block_device *bdev,
+			       struct blk_zone *rep,
+			       struct blk_zone *zone)
+{
+	sector_t offset = get_start_sect(bdev);
+
+	if (rep->start < offset)
+		return false;
+
+	rep->start -= offset;
+	if (rep->start + rep->len > bdev->bd_part->nr_sects)
+		return false;
+
+	if (rep->type == BLK_ZONE_TYPE_CONVENTIONAL)
+		rep->wp = rep->start + rep->len;
+	else
+		rep->wp -= offset;
+	memcpy(zone, rep, sizeof(struct blk_zone));
+
+	return true;
+}
+
+/**
+ * blkdev_report_zones - Get zones information
+ * @bdev:	Target block device
+ * @sector:	Sector from which to report zones
+ * @zones:	Array of zone structures where to return the zones information
+ * @nr_zones:	Number of zone structures in the zone array
+ * @gfp_mask:	Memory allocation flags (for bio_alloc)
+ *
+ * Description:
+ *    Get zone information starting from the zone containing @sector.
+ *    The number of zone information reported may be less than the number
+ *    requested by @nr_zones. The number of zones actually reported is
+ *    returned in @nr_zones.
+ */
+int blkdev_report_zones(struct block_device *bdev,
+			sector_t sector,
+			struct blk_zone *zones,
+			unsigned int *nr_zones,
+			gfp_t gfp_mask)
+{
+	struct request_queue *q = bdev_get_queue(bdev);
+	struct blk_zone_report_hdr *hdr;
+	unsigned int nrz = *nr_zones;
+	struct page *page;
+	unsigned int nr_rep;
+	size_t rep_bytes;
+	unsigned int nr_pages;
+	struct bio *bio;
+	struct bio_vec *bv;
+	unsigned int i, n, nz;
+	unsigned int ofst;
+	void *addr;
+	int ret;
+
+	if (!q)
+		return -ENXIO;
+
+	if (!blk_queue_is_zoned(q))
+		return -EOPNOTSUPP;
+
+	if (!nrz)
+		return 0;
+
+	if (sector > bdev->bd_part->nr_sects) {
+		*nr_zones = 0;
+		return 0;
+	}
+
+	/*
+	 * The zone report has a header. So make room for it in the
+	 * payload. Also make sure that the report fits in a single BIO
+	 * that will not be split down the stack.
+	 */
+	rep_bytes = sizeof(struct blk_zone_report_hdr) +
+		sizeof(struct blk_zone) * nrz;
+	rep_bytes = (rep_bytes + PAGE_SIZE - 1) & PAGE_MASK;
+	if (rep_bytes > (queue_max_sectors(q) << 9))
+		rep_bytes = queue_max_sectors(q) << 9;
+
+	nr_pages = min_t(unsigned int, BIO_MAX_PAGES,
+			 rep_bytes >> PAGE_SHIFT);
+	nr_pages = min_t(unsigned int, nr_pages,
+			 queue_max_segments(q));
+
+	bio = bio_alloc(gfp_mask, nr_pages);
+	if (!bio)
+		return -ENOMEM;
+
+	bio->bi_bdev = bdev;
+	bio->bi_iter.bi_sector = blk_zone_start(q, sector);
+	bio_set_op_attrs(bio, REQ_OP_ZONE_REPORT, 0);
+
+	for (i = 0; i < nr_pages; i++) {
+		page = alloc_page(gfp_mask);
+		if (!page) {
+			ret = -ENOMEM;
+			goto out;
+		}
+		if (!bio_add_page(bio, page, PAGE_SIZE, 0)) {
+			__free_page(page);
+			break;
+		}
+	}
+
+	if (i == 0)
+		ret = -ENOMEM;
+	else
+		ret = submit_bio_wait(bio);
+	if (ret)
+		goto out;
+
+	/*
+	 * Process the report result: skip the header and go through the
+	 * reported zones to fixup and fixup the zone information for
+	 * partitions. At the same time, return the zone information into
+	 * the zone array.
+	 */
+	n = 0;
+	nz = 0;
+	nr_rep = 0;
+	bio_for_each_segment_all(bv, bio, i) {
+
+		if (!bv->bv_page)
+			break;
+
+		addr = kmap_atomic(bv->bv_page);
+
+		/* Get header in the first page */
+		ofst = 0;
+		if (!nr_rep) {
+			hdr = (struct blk_zone_report_hdr *) addr;
+			nr_rep = hdr->nr_zones;
+			ofst = sizeof(struct blk_zone_report_hdr);
+		}
+
+		/* Fixup and report zones */
+		while (ofst < bv->bv_len &&
+		       n < nr_rep && nz < nrz) {
+			if (blkdev_report_zone(bdev, addr + ofst, &zones[nz]))
+				nz++;
+			ofst += sizeof(struct blk_zone);
+			n++;
+		}
+
+		kunmap_atomic(addr);
+
+		if (n >= nr_rep || nz >= nrz)
+			break;
+
+	}
+
+	*nr_zones = nz;
+out:
+	bio_for_each_segment_all(bv, bio, i)
+		__free_page(bv->bv_page);
+	bio_put(bio);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(blkdev_report_zones);
+
+/**
+ * blkdev_reset_zones - Reset zones write pointer
+ * @bdev:	Target block device
+ * @sector:	Start sector of the first zone to reset
+ * @nr_sectors:	Number of sectors, at least the length of one zone
+ * @gfp_mask:	Memory allocation flags (for bio_alloc)
+ *
+ * Description:
+ *    Reset the write pointer of the zones contained in the range
+ *    @sector..@sector+@nr_sectors. Specifying the entire disk sector range
+ *    is valid, but the specified range should not contain conventional zones.
+ */
+int blkdev_reset_zones(struct block_device *bdev,
+		       sector_t sector, sector_t nr_sectors,
+		       gfp_t gfp_mask)
+{
+	struct request_queue *q = bdev_get_queue(bdev);
+	sector_t zone_sectors;
+	sector_t end_sector = sector + nr_sectors;
+	struct bio *bio;
+	int ret;
+
+	if (!q)
+		return -ENXIO;
+
+	if (!blk_queue_is_zoned(q))
+		return -EOPNOTSUPP;
+
+	if (end_sector > bdev->bd_part->nr_sects)
+		/* Out of range */
+		return -EINVAL;
+
+	/* Check alignment (handle eventual smaller last zone) */
+	zone_sectors = blk_queue_zone_size(q);
+	if (sector & (zone_sectors - 1))
+		return -EINVAL;
+
+	if ((nr_sectors & (zone_sectors - 1)) &&
+	    end_sector != bdev->bd_part->nr_sects)
+		return -EINVAL;
+
+	while (sector < end_sector) {
+
+		bio = bio_alloc(gfp_mask, 0);
+		bio->bi_iter.bi_sector = sector;
+		bio->bi_bdev = bdev;
+		bio_set_op_attrs(bio, REQ_OP_ZONE_RESET, 0);
+
+		ret = submit_bio_wait(bio);
+		bio_put(bio);
+
+		if (ret)
+			return ret;
+
+		sector += zone_sectors;
+
+		/* This may take a while, so be nice to others */
+		cond_resched();
+
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(blkdev_reset_zones);
+
+/**
+ * BLKREPORTZONE ioctl processing.
+ * Called from blkdev_ioctl.
+ */
+int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
+			      unsigned int cmd, unsigned long arg)
+{
+	void __user *argp = (void __user *)arg;
+	struct request_queue *q;
+	struct blk_zone_report rep;
+	struct blk_zone *zones;
+	int ret;
+
+	if (!argp)
+		return -EINVAL;
+
+	q = bdev_get_queue(bdev);
+	if (!q)
+		return -ENXIO;
+
+	if (!blk_queue_is_zoned(q))
+		return -ENOTTY;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EACCES;
+
+	if (copy_from_user(&rep, argp, sizeof(struct blk_zone_report)))
+		return -EFAULT;
+
+	if (!rep.nr_zones)
+		return -EINVAL;
+
+	zones = kcalloc(rep.nr_zones, sizeof(struct blk_zone), GFP_KERNEL);
+	if (!zones)
+		return -ENOMEM;
+
+	ret = blkdev_report_zones(bdev, rep.sector,
+				  zones, &rep.nr_zones,
+				  GFP_KERNEL);
+	if (ret)
+		goto out;
+
+	if (copy_to_user(argp, &rep, sizeof(struct blk_zone_report))) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	if (rep.nr_zones) {
+		if (copy_to_user(argp + sizeof(struct blk_zone_report), zones,
+				 sizeof(struct blk_zone) * rep.nr_zones))
+			ret = -EFAULT;
+	}
+
+ out:
+	kfree(zones);
+
+	return ret;
+}
+
+/**
+ * BLKRESETZONE ioctl processing.
+ * Called from blkdev_ioctl.
+ */
+int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode,
+			     unsigned int cmd, unsigned long arg)
+{
+	void __user *argp = (void __user *)arg;
+	struct request_queue *q;
+	struct blk_zone_range zrange;
+
+	if (!argp)
+		return -EINVAL;
+
+	q = bdev_get_queue(bdev);
+	if (!q)
+		return -ENXIO;
+
+	if (!blk_queue_is_zoned(q))
+		return -ENOTTY;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EACCES;
+
+	if (!(mode & FMODE_WRITE))
+		return -EBADF;
+
+	if (copy_from_user(&zrange, argp, sizeof(struct blk_zone_range)))
+		return -EFAULT;
+
+	return blkdev_reset_zones(bdev, zrange.sector, zrange.nr_sectors,
+				  GFP_KERNEL);
+}
diff --git a/block/blk.h b/block/blk.h
index 74444c49078f..041185e5f129 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -111,6 +111,7 @@ void blk_account_io_done(struct request *req);
 enum rq_atomic_flags {
 	REQ_ATOM_COMPLETE = 0,
 	REQ_ATOM_STARTED,
+	REQ_ATOM_POLL_SLEPT,
 };
 
 /*
@@ -130,7 +131,7 @@ static inline void blk_clear_rq_complete(struct request *rq)
 /*
  * Internal elevator interface
  */
-#define ELV_ON_HASH(rq) ((rq)->cmd_flags & REQ_HASHED)
+#define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED)
 
 void blk_insert_flush(struct request *rq);
 
@@ -247,7 +248,7 @@ extern int blk_update_nr_requests(struct request_queue *, unsigned int);
 static inline int blk_do_io_stat(struct request *rq)
 {
 	return rq->rq_disk &&
-	       (rq->cmd_flags & REQ_IO_STAT) &&
+	       (rq->rq_flags & RQF_IO_STAT) &&
 		(rq->cmd_type == REQ_TYPE_FS);
 }
 
diff --git a/block/bsg-lib.c b/block/bsg-lib.c
index 650f427d915b..9d652a992316 100644
--- a/block/bsg-lib.c
+++ b/block/bsg-lib.c
@@ -32,8 +32,13 @@
  * bsg_destroy_job - routine to teardown/delete a bsg job
  * @job: bsg_job that is to be torn down
  */
-static void bsg_destroy_job(struct bsg_job *job)
+static void bsg_destroy_job(struct kref *kref)
 {
+	struct bsg_job *job = container_of(kref, struct bsg_job, kref);
+	struct request *rq = job->req;
+
+	blk_end_request_all(rq, rq->errors);
+
 	put_device(job->dev);	/* release reference for the request */
 
 	kfree(job->request_payload.sg_list);
@@ -41,6 +46,18 @@ static void bsg_destroy_job(struct bsg_job *job)
 	kfree(job);
 }
 
+void bsg_job_put(struct bsg_job *job)
+{
+	kref_put(&job->kref, bsg_destroy_job);
+}
+EXPORT_SYMBOL_GPL(bsg_job_put);
+
+int bsg_job_get(struct bsg_job *job)
+{
+	return kref_get_unless_zero(&job->kref);
+}
+EXPORT_SYMBOL_GPL(bsg_job_get);
+
 /**
  * bsg_job_done - completion routine for bsg requests
  * @job: bsg_job that is complete
@@ -83,8 +100,7 @@ static void bsg_softirq_done(struct request *rq)
 {
 	struct bsg_job *job = rq->special;
 
-	blk_end_request_all(rq, rq->errors);
-	bsg_destroy_job(job);
+	bsg_job_put(job);
 }
 
 static int bsg_map_buffer(struct bsg_buffer *buf, struct request *req)
@@ -142,6 +158,7 @@ static int bsg_create_job(struct device *dev, struct request *req)
 	job->dev = dev;
 	/* take a reference for the request */
 	get_device(job->dev);
+	kref_init(&job->kref);
 	return 0;
 
 failjob_rls_rqst_payload:
@@ -161,6 +178,8 @@ failjob_rls_job:
  * Drivers/subsys should pass this to the queue init function.
  */
 void bsg_request_fn(struct request_queue *q)
+	__releases(q->queue_lock)
+	__acquires(q->queue_lock)
 {
 	struct device *dev = q->queuedata;
 	struct request *req;
diff --git a/block/bsg.c b/block/bsg.c
index d214e929ce18..8a05a404ae70 100644
--- a/block/bsg.c
+++ b/block/bsg.c
@@ -176,7 +176,7 @@ static int blk_fill_sgv4_hdr_rq(struct request_queue *q, struct request *rq,
  * Check if sg_io_v4 from user is allowed and valid
  */
 static int
-bsg_validate_sgv4_hdr(struct request_queue *q, struct sg_io_v4 *hdr, int *rw)
+bsg_validate_sgv4_hdr(struct sg_io_v4 *hdr, int *rw)
 {
 	int ret = 0;
 
@@ -226,7 +226,7 @@ bsg_map_hdr(struct bsg_device *bd, struct sg_io_v4 *hdr, fmode_t has_write_perm,
 		hdr->dout_xfer_len, (unsigned long long) hdr->din_xferp,
 		hdr->din_xfer_len);
 
-	ret = bsg_validate_sgv4_hdr(q, hdr, &rw);
+	ret = bsg_validate_sgv4_hdr(hdr, &rw);
 	if (ret)
 		return ERR_PTR(ret);
 
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 5e24d880306c..c73a6fcaeb9d 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -16,6 +16,7 @@
 #include <linux/blktrace_api.h>
 #include <linux/blk-cgroup.h>
 #include "blk.h"
+#include "blk-wbt.h"
 
 /*
  * tunables
@@ -667,10 +668,10 @@ static inline void cfqg_put(struct cfq_group *cfqg)
 } while (0)
 
 static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
-					    struct cfq_group *curr_cfqg, int op,
-					    int op_flags)
+					    struct cfq_group *curr_cfqg,
+					    unsigned int op)
 {
-	blkg_rwstat_add(&cfqg->stats.queued, op, op_flags, 1);
+	blkg_rwstat_add(&cfqg->stats.queued, op, 1);
 	cfqg_stats_end_empty_time(&cfqg->stats);
 	cfqg_stats_set_start_group_wait_time(cfqg, curr_cfqg);
 }
@@ -684,30 +685,29 @@ static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
 #endif
 }
 
-static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op,
-					       int op_flags)
+static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg,
+					       unsigned int op)
 {
-	blkg_rwstat_add(&cfqg->stats.queued, op, op_flags, -1);
+	blkg_rwstat_add(&cfqg->stats.queued, op, -1);
 }
 
-static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op,
-					       int op_flags)
+static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg,
+					       unsigned int op)
 {
-	blkg_rwstat_add(&cfqg->stats.merged, op, op_flags, 1);
+	blkg_rwstat_add(&cfqg->stats.merged, op, 1);
 }
 
 static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
-			uint64_t start_time, uint64_t io_start_time, int op,
-			int op_flags)
+			uint64_t start_time, uint64_t io_start_time,
+			unsigned int op)
 {
 	struct cfqg_stats *stats = &cfqg->stats;
 	unsigned long long now = sched_clock();
 
 	if (time_after64(now, io_start_time))
-		blkg_rwstat_add(&stats->service_time, op, op_flags,
-				now - io_start_time);
+		blkg_rwstat_add(&stats->service_time, op, now - io_start_time);
 	if (time_after64(io_start_time, start_time))
-		blkg_rwstat_add(&stats->wait_time, op, op_flags,
+		blkg_rwstat_add(&stats->wait_time, op,
 				io_start_time - start_time);
 }
 
@@ -786,16 +786,16 @@ static inline void cfqg_put(struct cfq_group *cfqg) { }
 #define cfq_log_cfqg(cfqd, cfqg, fmt, args...)		do {} while (0)
 
 static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
-			struct cfq_group *curr_cfqg, int op, int op_flags) { }
+			struct cfq_group *curr_cfqg, unsigned int op) { }
 static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
 			uint64_t time, unsigned long unaccounted_time) { }
-static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int op,
-			int op_flags) { }
-static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int op,
-			int op_flags) { }
+static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg,
+			unsigned int op) { }
+static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg,
+			unsigned int op) { }
 static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
-			uint64_t start_time, uint64_t io_start_time, int op,
-			int op_flags) { }
+			uint64_t start_time, uint64_t io_start_time,
+			unsigned int op) { }
 
 #endif	/* CONFIG_CFQ_GROUP_IOSCHED */
 
@@ -913,15 +913,6 @@ static inline struct cfq_data *cic_to_cfqd(struct cfq_io_cq *cic)
 }
 
 /*
- * We regard a request as SYNC, if it's either a read or has the SYNC bit
- * set (in which case it could also be direct WRITE).
- */
-static inline bool cfq_bio_sync(struct bio *bio)
-{
-	return bio_data_dir(bio) == READ || (bio->bi_opf & REQ_SYNC);
-}
-
-/*
  * scheduler run of queue, if there are requests pending and no one in the
  * driver that will restart queueing
  */
@@ -1596,7 +1587,7 @@ static struct blkcg_policy_data *cfq_cpd_alloc(gfp_t gfp)
 {
 	struct cfq_group_data *cgd;
 
-	cgd = kzalloc(sizeof(*cgd), GFP_KERNEL);
+	cgd = kzalloc(sizeof(*cgd), gfp);
 	if (!cgd)
 		return NULL;
 	return &cgd->cpd;
@@ -2474,10 +2465,10 @@ static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq)
 {
 	elv_rb_del(&cfqq->sort_list, rq);
 	cfqq->queued[rq_is_sync(rq)]--;
-	cfqg_stats_update_io_remove(RQ_CFQG(rq), req_op(rq), rq->cmd_flags);
+	cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags);
 	cfq_add_rq_rb(rq);
 	cfqg_stats_update_io_add(RQ_CFQG(rq), cfqq->cfqd->serving_group,
-				 req_op(rq), rq->cmd_flags);
+				 rq->cmd_flags);
 }
 
 static struct request *
@@ -2491,7 +2482,7 @@ cfq_find_rq_fmerge(struct cfq_data *cfqd, struct bio *bio)
 	if (!cic)
 		return NULL;
 
-	cfqq = cic_to_cfqq(cic, cfq_bio_sync(bio));
+	cfqq = cic_to_cfqq(cic, op_is_sync(bio->bi_opf));
 	if (cfqq)
 		return elv_rb_find(&cfqq->sort_list, bio_end_sector(bio));
 
@@ -2530,7 +2521,7 @@ static void cfq_remove_request(struct request *rq)
 	cfq_del_rq_rb(rq);
 
 	cfqq->cfqd->rq_queued--;
-	cfqg_stats_update_io_remove(RQ_CFQG(rq), req_op(rq), rq->cmd_flags);
+	cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags);
 	if (rq->cmd_flags & REQ_PRIO) {
 		WARN_ON(!cfqq->prio_pending);
 		cfqq->prio_pending--;
@@ -2565,7 +2556,7 @@ static void cfq_merged_request(struct request_queue *q, struct request *req,
 static void cfq_bio_merged(struct request_queue *q, struct request *req,
 				struct bio *bio)
 {
-	cfqg_stats_update_io_merged(RQ_CFQG(req), bio_op(bio), bio->bi_opf);
+	cfqg_stats_update_io_merged(RQ_CFQG(req), bio->bi_opf);
 }
 
 static void
@@ -2588,7 +2579,7 @@ cfq_merged_requests(struct request_queue *q, struct request *rq,
 	if (cfqq->next_rq == next)
 		cfqq->next_rq = rq;
 	cfq_remove_request(next);
-	cfqg_stats_update_io_merged(RQ_CFQG(rq), req_op(next), next->cmd_flags);
+	cfqg_stats_update_io_merged(RQ_CFQG(rq), next->cmd_flags);
 
 	cfqq = RQ_CFQQ(next);
 	/*
@@ -2605,13 +2596,14 @@ static int cfq_allow_bio_merge(struct request_queue *q, struct request *rq,
 			       struct bio *bio)
 {
 	struct cfq_data *cfqd = q->elevator->elevator_data;
+	bool is_sync = op_is_sync(bio->bi_opf);
 	struct cfq_io_cq *cic;
 	struct cfq_queue *cfqq;
 
 	/*
 	 * Disallow merge of a sync bio into an async request.
 	 */
-	if (cfq_bio_sync(bio) && !rq_is_sync(rq))
+	if (is_sync && !rq_is_sync(rq))
 		return false;
 
 	/*
@@ -2622,7 +2614,7 @@ static int cfq_allow_bio_merge(struct request_queue *q, struct request *rq,
 	if (!cic)
 		return false;
 
-	cfqq = cic_to_cfqq(cic, cfq_bio_sync(bio));
+	cfqq = cic_to_cfqq(cic, is_sync);
 	return cfqq == RQ_CFQQ(rq);
 }
 
@@ -3771,9 +3763,11 @@ static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio)
 	struct cfq_data *cfqd = cic_to_cfqd(cic);
 	struct cfq_queue *cfqq;
 	uint64_t serial_nr;
+	bool nonroot_cg;
 
 	rcu_read_lock();
 	serial_nr = bio_blkcg(bio)->css.serial_nr;
+	nonroot_cg = bio_blkcg(bio) != &blkcg_root;
 	rcu_read_unlock();
 
 	/*
@@ -3784,6 +3778,14 @@ static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio)
 		return;
 
 	/*
+	 * If we have a non-root cgroup, we can depend on that to
+	 * do proper throttling of writes. Turn off wbt for that
+	 * case, if it was enabled by default.
+	 */
+	if (nonroot_cg)
+		wbt_disable_default(cfqd->queue);
+
+	/*
 	 * Drop reference to queues.  New queues will be assigned in new
 	 * group upon arrival of fresh requests.
 	 */
@@ -3854,7 +3856,8 @@ cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic,
 			goto out;
 	}
 
-	cfqq = kmem_cache_alloc_node(cfq_pool, GFP_NOWAIT | __GFP_ZERO,
+	cfqq = kmem_cache_alloc_node(cfq_pool,
+				     GFP_NOWAIT | __GFP_ZERO | __GFP_NOWARN,
 				     cfqd->queue->node);
 	if (!cfqq) {
 		cfqq = &cfqd->oom_cfqq;
@@ -3923,6 +3926,12 @@ cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_queue *cfqq,
 		cfqq->seek_history |= (sdist > CFQQ_SEEK_THR);
 }
 
+static inline bool req_noidle(struct request *req)
+{
+	return req_op(req) == REQ_OP_WRITE &&
+		(req->cmd_flags & (REQ_SYNC | REQ_IDLE)) == REQ_SYNC;
+}
+
 /*
  * Disable idle window if the process thinks too long or seeks so much that
  * it doesn't matter
@@ -3944,7 +3953,7 @@ cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq,
 	if (cfqq->queued[0] + cfqq->queued[1] >= 4)
 		cfq_mark_cfqq_deep(cfqq);
 
-	if (cfqq->next_rq && (cfqq->next_rq->cmd_flags & REQ_NOIDLE))
+	if (cfqq->next_rq && req_noidle(cfqq->next_rq))
 		enable_idle = 0;
 	else if (!atomic_read(&cic->icq.ioc->active_ref) ||
 		 !cfqd->cfq_slice_idle ||
@@ -4142,7 +4151,7 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq)
 	rq->fifo_time = ktime_get_ns() + cfqd->cfq_fifo_expire[rq_is_sync(rq)];
 	list_add_tail(&rq->queuelist, &cfqq->fifo);
 	cfq_add_rq_rb(rq);
-	cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, req_op(rq),
+	cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group,
 				 rq->cmd_flags);
 	cfq_rq_enqueued(cfqd, cfqq, rq);
 }
@@ -4229,8 +4238,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
 	const int sync = rq_is_sync(rq);
 	u64 now = ktime_get_ns();
 
-	cfq_log_cfqq(cfqd, cfqq, "complete rqnoidle %d",
-		     !!(rq->cmd_flags & REQ_NOIDLE));
+	cfq_log_cfqq(cfqd, cfqq, "complete rqnoidle %d", req_noidle(rq));
 
 	cfq_update_hw_tag(cfqd);
 
@@ -4240,8 +4248,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
 	cfqq->dispatched--;
 	(RQ_CFQG(rq))->dispatched--;
 	cfqg_stats_update_completion(cfqq->cfqg, rq_start_time_ns(rq),
-				     rq_io_start_time_ns(rq), req_op(rq),
-				     rq->cmd_flags);
+				     rq_io_start_time_ns(rq), rq->cmd_flags);
 
 	cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--;
 
@@ -4319,14 +4326,14 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
 		cfq_schedule_dispatch(cfqd);
 }
 
-static void cfqq_boost_on_prio(struct cfq_queue *cfqq, int op_flags)
+static void cfqq_boost_on_prio(struct cfq_queue *cfqq, unsigned int op)
 {
 	/*
 	 * If REQ_PRIO is set, boost class and prio level, if it's below
 	 * BE/NORM. If prio is not set, restore the potentially boosted
 	 * class/prio level.
 	 */
-	if (!(op_flags & REQ_PRIO)) {
+	if (!(op & REQ_PRIO)) {
 		cfqq->ioprio_class = cfqq->org_ioprio_class;
 		cfqq->ioprio = cfqq->org_ioprio;
 	} else {
@@ -4347,7 +4354,7 @@ static inline int __cfq_may_queue(struct cfq_queue *cfqq)
 	return ELV_MQUEUE_MAY;
 }
 
-static int cfq_may_queue(struct request_queue *q, int op, int op_flags)
+static int cfq_may_queue(struct request_queue *q, unsigned int op)
 {
 	struct cfq_data *cfqd = q->elevator->elevator_data;
 	struct task_struct *tsk = current;
@@ -4364,10 +4371,10 @@ static int cfq_may_queue(struct request_queue *q, int op, int op_flags)
 	if (!cic)
 		return ELV_MQUEUE_MAY;
 
-	cfqq = cic_to_cfqq(cic, rw_is_sync(op, op_flags));
+	cfqq = cic_to_cfqq(cic, op_is_sync(op));
 	if (cfqq) {
 		cfq_init_prio_data(cfqq, cic);
-		cfqq_boost_on_prio(cfqq, op_flags);
+		cfqq_boost_on_prio(cfqq, op);
 
 		return __cfq_may_queue(cfqq);
 	}
diff --git a/block/elevator.c b/block/elevator.c
index f7d973a56fd7..40f0c04e5ad3 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -245,31 +245,31 @@ EXPORT_SYMBOL(elevator_exit);
 static inline void __elv_rqhash_del(struct request *rq)
 {
 	hash_del(&rq->hash);
-	rq->cmd_flags &= ~REQ_HASHED;
+	rq->rq_flags &= ~RQF_HASHED;
 }
 
-static void elv_rqhash_del(struct request_queue *q, struct request *rq)
+void elv_rqhash_del(struct request_queue *q, struct request *rq)
 {
 	if (ELV_ON_HASH(rq))
 		__elv_rqhash_del(rq);
 }
 
-static void elv_rqhash_add(struct request_queue *q, struct request *rq)
+void elv_rqhash_add(struct request_queue *q, struct request *rq)
 {
 	struct elevator_queue *e = q->elevator;
 
 	BUG_ON(ELV_ON_HASH(rq));
 	hash_add(e->hash, &rq->hash, rq_hash_key(rq));
-	rq->cmd_flags |= REQ_HASHED;
+	rq->rq_flags |= RQF_HASHED;
 }
 
-static void elv_rqhash_reposition(struct request_queue *q, struct request *rq)
+void elv_rqhash_reposition(struct request_queue *q, struct request *rq)
 {
 	__elv_rqhash_del(rq);
 	elv_rqhash_add(q, rq);
 }
 
-static struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)
+struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)
 {
 	struct elevator_queue *e = q->elevator;
 	struct hlist_node *next;
@@ -352,7 +352,6 @@ void elv_dispatch_sort(struct request_queue *q, struct request *rq)
 {
 	sector_t boundary;
 	struct list_head *entry;
-	int stop_flags;
 
 	if (q->last_merge == rq)
 		q->last_merge = NULL;
@@ -362,7 +361,6 @@ void elv_dispatch_sort(struct request_queue *q, struct request *rq)
 	q->nr_sorted--;
 
 	boundary = q->end_sector;
-	stop_flags = REQ_SOFTBARRIER | REQ_STARTED;
 	list_for_each_prev(entry, &q->queue_head) {
 		struct request *pos = list_entry_rq(entry);
 
@@ -370,7 +368,7 @@ void elv_dispatch_sort(struct request_queue *q, struct request *rq)
 			break;
 		if (rq_data_dir(rq) != rq_data_dir(pos))
 			break;
-		if (pos->cmd_flags & stop_flags)
+		if (pos->rq_flags & (RQF_STARTED | RQF_SOFTBARRIER))
 			break;
 		if (blk_rq_pos(rq) >= boundary) {
 			if (blk_rq_pos(pos) < boundary)
@@ -510,7 +508,7 @@ void elv_merge_requests(struct request_queue *q, struct request *rq,
 			     struct request *next)
 {
 	struct elevator_queue *e = q->elevator;
-	const int next_sorted = next->cmd_flags & REQ_SORTED;
+	const int next_sorted = next->rq_flags & RQF_SORTED;
 
 	if (next_sorted && e->type->ops.elevator_merge_req_fn)
 		e->type->ops.elevator_merge_req_fn(q, rq, next);
@@ -537,13 +535,13 @@ void elv_bio_merged(struct request_queue *q, struct request *rq,
 #ifdef CONFIG_PM
 static void blk_pm_requeue_request(struct request *rq)
 {
-	if (rq->q->dev && !(rq->cmd_flags & REQ_PM))
+	if (rq->q->dev && !(rq->rq_flags & RQF_PM))
 		rq->q->nr_pending--;
 }
 
 static void blk_pm_add_request(struct request_queue *q, struct request *rq)
 {
-	if (q->dev && !(rq->cmd_flags & REQ_PM) && q->nr_pending++ == 0 &&
+	if (q->dev && !(rq->rq_flags & RQF_PM) && q->nr_pending++ == 0 &&
 	    (q->rpm_status == RPM_SUSPENDED || q->rpm_status == RPM_SUSPENDING))
 		pm_request_resume(q->dev);
 }
@@ -563,11 +561,11 @@ void elv_requeue_request(struct request_queue *q, struct request *rq)
 	 */
 	if (blk_account_rq(rq)) {
 		q->in_flight[rq_is_sync(rq)]--;
-		if (rq->cmd_flags & REQ_SORTED)
+		if (rq->rq_flags & RQF_SORTED)
 			elv_deactivate_rq(q, rq);
 	}
 
-	rq->cmd_flags &= ~REQ_STARTED;
+	rq->rq_flags &= ~RQF_STARTED;
 
 	blk_pm_requeue_request(rq);
 
@@ -597,13 +595,13 @@ void __elv_add_request(struct request_queue *q, struct request *rq, int where)
 
 	rq->q = q;
 
-	if (rq->cmd_flags & REQ_SOFTBARRIER) {
+	if (rq->rq_flags & RQF_SOFTBARRIER) {
 		/* barriers are scheduling boundary, update end_sector */
 		if (rq->cmd_type == REQ_TYPE_FS) {
 			q->end_sector = rq_end_sector(rq);
 			q->boundary_rq = rq;
 		}
-	} else if (!(rq->cmd_flags & REQ_ELVPRIV) &&
+	} else if (!(rq->rq_flags & RQF_ELVPRIV) &&
 		    (where == ELEVATOR_INSERT_SORT ||
 		     where == ELEVATOR_INSERT_SORT_MERGE))
 		where = ELEVATOR_INSERT_BACK;
@@ -611,12 +609,12 @@ void __elv_add_request(struct request_queue *q, struct request *rq, int where)
 	switch (where) {
 	case ELEVATOR_INSERT_REQUEUE:
 	case ELEVATOR_INSERT_FRONT:
-		rq->cmd_flags |= REQ_SOFTBARRIER;
+		rq->rq_flags |= RQF_SOFTBARRIER;
 		list_add(&rq->queuelist, &q->queue_head);
 		break;
 
 	case ELEVATOR_INSERT_BACK:
-		rq->cmd_flags |= REQ_SOFTBARRIER;
+		rq->rq_flags |= RQF_SOFTBARRIER;
 		elv_drain_elevator(q);
 		list_add_tail(&rq->queuelist, &q->queue_head);
 		/*
@@ -642,7 +640,7 @@ void __elv_add_request(struct request_queue *q, struct request *rq, int where)
 			break;
 	case ELEVATOR_INSERT_SORT:
 		BUG_ON(rq->cmd_type != REQ_TYPE_FS);
-		rq->cmd_flags |= REQ_SORTED;
+		rq->rq_flags |= RQF_SORTED;
 		q->nr_sorted++;
 		if (rq_mergeable(rq)) {
 			elv_rqhash_add(q, rq);
@@ -659,7 +657,7 @@ void __elv_add_request(struct request_queue *q, struct request *rq, int where)
 		break;
 
 	case ELEVATOR_INSERT_FLUSH:
-		rq->cmd_flags |= REQ_SOFTBARRIER;
+		rq->rq_flags |= RQF_SOFTBARRIER;
 		blk_insert_flush(rq);
 		break;
 	default:
@@ -716,12 +714,12 @@ void elv_put_request(struct request_queue *q, struct request *rq)
 		e->type->ops.elevator_put_req_fn(rq);
 }
 
-int elv_may_queue(struct request_queue *q, int op, int op_flags)
+int elv_may_queue(struct request_queue *q, unsigned int op)
 {
 	struct elevator_queue *e = q->elevator;
 
 	if (e->type->ops.elevator_may_queue_fn)
-		return e->type->ops.elevator_may_queue_fn(q, op, op_flags);
+		return e->type->ops.elevator_may_queue_fn(q, op);
 
 	return ELV_MQUEUE_MAY;
 }
@@ -735,7 +733,7 @@ void elv_completed_request(struct request_queue *q, struct request *rq)
 	 */
 	if (blk_account_rq(rq)) {
 		q->in_flight[rq_is_sync(rq)]--;
-		if ((rq->cmd_flags & REQ_SORTED) &&
+		if ((rq->rq_flags & RQF_SORTED) &&
 		    e->type->ops.elevator_completed_req_fn)
 			e->type->ops.elevator_completed_req_fn(q, rq);
 	}
diff --git a/block/ioctl.c b/block/ioctl.c
index 755119c3c1b9..f856963204f4 100644
--- a/block/ioctl.c
+++ b/block/ioctl.c
@@ -519,6 +519,10 @@ int blkdev_ioctl(struct block_device *bdev, fmode_t mode, unsigned cmd,
 				BLKDEV_DISCARD_SECURE);
 	case BLKZEROOUT:
 		return blk_ioctl_zeroout(bdev, mode, arg);
+	case BLKREPORTZONE:
+		return blkdev_report_zones_ioctl(bdev, mode, cmd, arg);
+	case BLKRESETZONE:
+		return blkdev_reset_zones_ioctl(bdev, mode, cmd, arg);
 	case HDIO_GETGEO:
 		return blkdev_getgeo(bdev, argp);
 	case BLKRAGET:
diff --git a/block/partition-generic.c b/block/partition-generic.c
index 71d9ed9df8da..d7beb6bbbf66 100644
--- a/block/partition-generic.c
+++ b/block/partition-generic.c
@@ -430,6 +430,56 @@ static int drop_partitions(struct gendisk *disk, struct block_device *bdev)
 	return 0;
 }
 
+static bool part_zone_aligned(struct gendisk *disk,
+			      struct block_device *bdev,
+			      sector_t from, sector_t size)
+{
+	unsigned int zone_size = bdev_zone_size(bdev);
+
+	/*
+	 * If this function is called, then the disk is a zoned block device
+	 * (host-aware or host-managed). This can be detected even if the
+	 * zoned block device support is disabled (CONFIG_BLK_DEV_ZONED not
+	 * set). In this case, however, only host-aware devices will be seen
+	 * as a block device is not created for host-managed devices. Without
+	 * zoned block device support, host-aware drives can still be used as
+	 * regular block devices (no zone operation) and their zone size will
+	 * be reported as 0. Allow this case.
+	 */
+	if (!zone_size)
+		return true;
+
+	/*
+	 * Check partition start and size alignement. If the drive has a
+	 * smaller last runt zone, ignore it and allow the partition to
+	 * use it. Check the zone size too: it should be a power of 2 number
+	 * of sectors.
+	 */
+	if (WARN_ON_ONCE(!is_power_of_2(zone_size))) {
+		u32 rem;
+
+		div_u64_rem(from, zone_size, &rem);
+		if (rem)
+			return false;
+		if ((from + size) < get_capacity(disk)) {
+			div_u64_rem(size, zone_size, &rem);
+			if (rem)
+				return false;
+		}
+
+	} else {
+
+		if (from & (zone_size - 1))
+			return false;
+		if ((from + size) < get_capacity(disk) &&
+		    (size & (zone_size - 1)))
+			return false;
+
+	}
+
+	return true;
+}
+
 int rescan_partitions(struct gendisk *disk, struct block_device *bdev)
 {
 	struct parsed_partitions *state = NULL;
@@ -529,6 +579,21 @@ rescan:
 			}
 		}
 
+		/*
+		 * On a zoned block device, partitions should be aligned on the
+		 * device zone size (i.e. zone boundary crossing not allowed).
+		 * Otherwise, resetting the write pointer of the last zone of
+		 * one partition may impact the following partition.
+		 */
+		if (bdev_is_zoned(bdev) &&
+		    !part_zone_aligned(disk, bdev, from, size)) {
+			printk(KERN_WARNING
+			       "%s: p%d start %llu+%llu is not zone aligned\n",
+			       disk->disk_name, p, (unsigned long long) from,
+			       (unsigned long long) size);
+			continue;
+		}
+
 		part = add_partition(disk, p, from, size,
 				     state->parts[p].flags,
 				     &state->parts[p].info);
diff --git a/crypto/842.c b/crypto/842.c
index 98e387efb8c8..bc26dc942821 100644
--- a/crypto/842.c
+++ b/crypto/842.c
@@ -31,11 +31,46 @@
 #include <linux/module.h>
 #include <linux/crypto.h>
 #include <linux/sw842.h>
+#include <crypto/internal/scompress.h>
 
 struct crypto842_ctx {
-	char wmem[SW842_MEM_COMPRESS];	/* working memory for compress */
+	void *wmem;	/* working memory for compress */
 };
 
+static void *crypto842_alloc_ctx(struct crypto_scomp *tfm)
+{
+	void *ctx;
+
+	ctx = kmalloc(SW842_MEM_COMPRESS, GFP_KERNEL);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	return ctx;
+}
+
+static int crypto842_init(struct crypto_tfm *tfm)
+{
+	struct crypto842_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	ctx->wmem = crypto842_alloc_ctx(NULL);
+	if (IS_ERR(ctx->wmem))
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void crypto842_free_ctx(struct crypto_scomp *tfm, void *ctx)
+{
+	kfree(ctx);
+}
+
+static void crypto842_exit(struct crypto_tfm *tfm)
+{
+	struct crypto842_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	crypto842_free_ctx(NULL, ctx->wmem);
+}
+
 static int crypto842_compress(struct crypto_tfm *tfm,
 			      const u8 *src, unsigned int slen,
 			      u8 *dst, unsigned int *dlen)
@@ -45,6 +80,13 @@ static int crypto842_compress(struct crypto_tfm *tfm,
 	return sw842_compress(src, slen, dst, dlen, ctx->wmem);
 }
 
+static int crypto842_scompress(struct crypto_scomp *tfm,
+			       const u8 *src, unsigned int slen,
+			       u8 *dst, unsigned int *dlen, void *ctx)
+{
+	return sw842_compress(src, slen, dst, dlen, ctx);
+}
+
 static int crypto842_decompress(struct crypto_tfm *tfm,
 				const u8 *src, unsigned int slen,
 				u8 *dst, unsigned int *dlen)
@@ -52,6 +94,13 @@ static int crypto842_decompress(struct crypto_tfm *tfm,
 	return sw842_decompress(src, slen, dst, dlen);
 }
 
+static int crypto842_sdecompress(struct crypto_scomp *tfm,
+				 const u8 *src, unsigned int slen,
+				 u8 *dst, unsigned int *dlen, void *ctx)
+{
+	return sw842_decompress(src, slen, dst, dlen);
+}
+
 static struct crypto_alg alg = {
 	.cra_name		= "842",
 	.cra_driver_name	= "842-generic",
@@ -59,20 +108,48 @@ static struct crypto_alg alg = {
 	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
 	.cra_ctxsize		= sizeof(struct crypto842_ctx),
 	.cra_module		= THIS_MODULE,
+	.cra_init		= crypto842_init,
+	.cra_exit		= crypto842_exit,
 	.cra_u			= { .compress = {
 	.coa_compress		= crypto842_compress,
 	.coa_decompress		= crypto842_decompress } }
 };
 
+static struct scomp_alg scomp = {
+	.alloc_ctx		= crypto842_alloc_ctx,
+	.free_ctx		= crypto842_free_ctx,
+	.compress		= crypto842_scompress,
+	.decompress		= crypto842_sdecompress,
+	.base			= {
+		.cra_name	= "842",
+		.cra_driver_name = "842-scomp",
+		.cra_priority	 = 100,
+		.cra_module	 = THIS_MODULE,
+	}
+};
+
 static int __init crypto842_mod_init(void)
 {
-	return crypto_register_alg(&alg);
+	int ret;
+
+	ret = crypto_register_alg(&alg);
+	if (ret)
+		return ret;
+
+	ret = crypto_register_scomp(&scomp);
+	if (ret) {
+		crypto_unregister_alg(&alg);
+		return ret;
+	}
+
+	return ret;
 }
 module_init(crypto842_mod_init);
 
 static void __exit crypto842_mod_exit(void)
 {
 	crypto_unregister_alg(&alg);
+	crypto_unregister_scomp(&scomp);
 }
 module_exit(crypto842_mod_exit);
 
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 84d71482bf08..160f08e721cc 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -24,7 +24,7 @@ comment "Crypto core or helper"
 config CRYPTO_FIPS
 	bool "FIPS 200 compliance"
 	depends on (CRYPTO_ANSI_CPRNG || CRYPTO_DRBG) && !CRYPTO_MANAGER_DISABLE_TESTS
-	depends on MODULE_SIG
+	depends on (MODULE_SIG || !MODULES)
 	help
 	  This options enables the fips boot option which is
 	  required if you want to system to operate in a FIPS 200
@@ -102,6 +102,15 @@ config CRYPTO_KPP
 	select CRYPTO_ALGAPI
 	select CRYPTO_KPP2
 
+config CRYPTO_ACOMP2
+	tristate
+	select CRYPTO_ALGAPI2
+
+config CRYPTO_ACOMP
+	tristate
+	select CRYPTO_ALGAPI
+	select CRYPTO_ACOMP2
+
 config CRYPTO_RSA
 	tristate "RSA algorithm"
 	select CRYPTO_AKCIPHER
@@ -138,6 +147,7 @@ config CRYPTO_MANAGER2
 	select CRYPTO_BLKCIPHER2
 	select CRYPTO_AKCIPHER2
 	select CRYPTO_KPP2
+	select CRYPTO_ACOMP2
 
 config CRYPTO_USER
 	tristate "Userspace cryptographic algorithm configuration"
@@ -236,10 +246,14 @@ config CRYPTO_ABLK_HELPER
 	tristate
 	select CRYPTO_CRYPTD
 
+config CRYPTO_SIMD
+	tristate
+	select CRYPTO_CRYPTD
+
 config CRYPTO_GLUE_HELPER_X86
 	tristate
 	depends on X86
-	select CRYPTO_ALGAPI
+	select CRYPTO_BLKCIPHER
 
 config CRYPTO_ENGINE
 	tristate
@@ -437,7 +451,7 @@ config CRYPTO_CRC32C_INTEL
 	  gain performance compared with software implementation.
 	  Module will be crc32c-intel.
 
-config CRYPT_CRC32C_VPMSUM
+config CRYPTO_CRC32C_VPMSUM
 	tristate "CRC32c CRC algorithm (powerpc64)"
 	depends on PPC64 && ALTIVEC
 	select CRYPTO_HASH
@@ -928,14 +942,13 @@ config CRYPTO_AES_X86_64
 config CRYPTO_AES_NI_INTEL
 	tristate "AES cipher algorithms (AES-NI)"
 	depends on X86
+	select CRYPTO_AEAD
 	select CRYPTO_AES_X86_64 if 64BIT
 	select CRYPTO_AES_586 if !64BIT
-	select CRYPTO_CRYPTD
-	select CRYPTO_ABLK_HELPER
 	select CRYPTO_ALGAPI
+	select CRYPTO_BLKCIPHER
 	select CRYPTO_GLUE_HELPER_X86 if 64BIT
-	select CRYPTO_LRW
-	select CRYPTO_XTS
+	select CRYPTO_SIMD
 	help
 	  Use Intel AES-NI instructions for AES algorithm.
 
@@ -1568,6 +1581,7 @@ comment "Compression"
 config CRYPTO_DEFLATE
 	tristate "Deflate compression algorithm"
 	select CRYPTO_ALGAPI
+	select CRYPTO_ACOMP2
 	select ZLIB_INFLATE
 	select ZLIB_DEFLATE
 	help
@@ -1579,6 +1593,7 @@ config CRYPTO_DEFLATE
 config CRYPTO_LZO
 	tristate "LZO compression algorithm"
 	select CRYPTO_ALGAPI
+	select CRYPTO_ACOMP2
 	select LZO_COMPRESS
 	select LZO_DECOMPRESS
 	help
@@ -1587,6 +1602,7 @@ config CRYPTO_LZO
 config CRYPTO_842
 	tristate "842 compression algorithm"
 	select CRYPTO_ALGAPI
+	select CRYPTO_ACOMP2
 	select 842_COMPRESS
 	select 842_DECOMPRESS
 	help
@@ -1595,6 +1611,7 @@ config CRYPTO_842
 config CRYPTO_LZ4
 	tristate "LZ4 compression algorithm"
 	select CRYPTO_ALGAPI
+	select CRYPTO_ACOMP2
 	select LZ4_COMPRESS
 	select LZ4_DECOMPRESS
 	help
@@ -1603,6 +1620,7 @@ config CRYPTO_LZ4
 config CRYPTO_LZ4HC
 	tristate "LZ4HC compression algorithm"
 	select CRYPTO_ALGAPI
+	select CRYPTO_ACOMP2
 	select LZ4HC_COMPRESS
 	select LZ4_DECOMPRESS
 	help
diff --git a/crypto/Makefile b/crypto/Makefile
index bd6a029094e6..b8f0e3eb0791 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -51,6 +51,10 @@ rsa_generic-y += rsa_helper.o
 rsa_generic-y += rsa-pkcs1pad.o
 obj-$(CONFIG_CRYPTO_RSA) += rsa_generic.o
 
+crypto_acompress-y := acompress.o
+crypto_acompress-y += scompress.o
+obj-$(CONFIG_CRYPTO_ACOMP2) += crypto_acompress.o
+
 cryptomgr-y := algboss.o testmgr.o
 
 obj-$(CONFIG_CRYPTO_MANAGER2) += cryptomgr.o
@@ -139,3 +143,5 @@ obj-$(CONFIG_ASYNC_CORE) += async_tx/
 obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/
 obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o
 obj-$(CONFIG_CRYPTO_ABLK_HELPER) += ablk_helper.o
+crypto_simd-y := simd.o
+obj-$(CONFIG_CRYPTO_SIMD) += crypto_simd.o
diff --git a/crypto/acompress.c b/crypto/acompress.c
new file mode 100644
index 000000000000..887783d8e9a9
--- /dev/null
+++ b/crypto/acompress.c
@@ -0,0 +1,169 @@
+/*
+ * Asynchronous Compression operations
+ *
+ * Copyright (c) 2016, Intel Corporation
+ * Authors: Weigang Li <weigang.li@intel.com>
+ *          Giovanni Cabiddu <giovanni.cabiddu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/crypto.h>
+#include <crypto/algapi.h>
+#include <linux/cryptouser.h>
+#include <net/netlink.h>
+#include <crypto/internal/acompress.h>
+#include <crypto/internal/scompress.h>
+#include "internal.h"
+
+static const struct crypto_type crypto_acomp_type;
+
+#ifdef CONFIG_NET
+static int crypto_acomp_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+	struct crypto_report_acomp racomp;
+
+	strncpy(racomp.type, "acomp", sizeof(racomp.type));
+
+	if (nla_put(skb, CRYPTOCFGA_REPORT_ACOMP,
+		    sizeof(struct crypto_report_acomp), &racomp))
+		goto nla_put_failure;
+	return 0;
+
+nla_put_failure:
+	return -EMSGSIZE;
+}
+#else
+static int crypto_acomp_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+	return -ENOSYS;
+}
+#endif
+
+static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg)
+	__attribute__ ((unused));
+
+static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg)
+{
+	seq_puts(m, "type         : acomp\n");
+}
+
+static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm)
+{
+	struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
+	struct acomp_alg *alg = crypto_acomp_alg(acomp);
+
+	alg->exit(acomp);
+}
+
+static int crypto_acomp_init_tfm(struct crypto_tfm *tfm)
+{
+	struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
+	struct acomp_alg *alg = crypto_acomp_alg(acomp);
+
+	if (tfm->__crt_alg->cra_type != &crypto_acomp_type)
+		return crypto_init_scomp_ops_async(tfm);
+
+	acomp->compress = alg->compress;
+	acomp->decompress = alg->decompress;
+	acomp->dst_free = alg->dst_free;
+	acomp->reqsize = alg->reqsize;
+
+	if (alg->exit)
+		acomp->base.exit = crypto_acomp_exit_tfm;
+
+	if (alg->init)
+		return alg->init(acomp);
+
+	return 0;
+}
+
+static unsigned int crypto_acomp_extsize(struct crypto_alg *alg)
+{
+	int extsize = crypto_alg_extsize(alg);
+
+	if (alg->cra_type != &crypto_acomp_type)
+		extsize += sizeof(struct crypto_scomp *);
+
+	return extsize;
+}
+
+static const struct crypto_type crypto_acomp_type = {
+	.extsize = crypto_acomp_extsize,
+	.init_tfm = crypto_acomp_init_tfm,
+#ifdef CONFIG_PROC_FS
+	.show = crypto_acomp_show,
+#endif
+	.report = crypto_acomp_report,
+	.maskclear = ~CRYPTO_ALG_TYPE_MASK,
+	.maskset = CRYPTO_ALG_TYPE_ACOMPRESS_MASK,
+	.type = CRYPTO_ALG_TYPE_ACOMPRESS,
+	.tfmsize = offsetof(struct crypto_acomp, base),
+};
+
+struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type,
+					u32 mask)
+{
+	return crypto_alloc_tfm(alg_name, &crypto_acomp_type, type, mask);
+}
+EXPORT_SYMBOL_GPL(crypto_alloc_acomp);
+
+struct acomp_req *acomp_request_alloc(struct crypto_acomp *acomp)
+{
+	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp);
+	struct acomp_req *req;
+
+	req = __acomp_request_alloc(acomp);
+	if (req && (tfm->__crt_alg->cra_type != &crypto_acomp_type))
+		return crypto_acomp_scomp_alloc_ctx(req);
+
+	return req;
+}
+EXPORT_SYMBOL_GPL(acomp_request_alloc);
+
+void acomp_request_free(struct acomp_req *req)
+{
+	struct crypto_acomp *acomp = crypto_acomp_reqtfm(req);
+	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp);
+
+	if (tfm->__crt_alg->cra_type != &crypto_acomp_type)
+		crypto_acomp_scomp_free_ctx(req);
+
+	if (req->flags & CRYPTO_ACOMP_ALLOC_OUTPUT) {
+		acomp->dst_free(req->dst);
+		req->dst = NULL;
+	}
+
+	__acomp_request_free(req);
+}
+EXPORT_SYMBOL_GPL(acomp_request_free);
+
+int crypto_register_acomp(struct acomp_alg *alg)
+{
+	struct crypto_alg *base = &alg->base;
+
+	base->cra_type = &crypto_acomp_type;
+	base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
+	base->cra_flags |= CRYPTO_ALG_TYPE_ACOMPRESS;
+
+	return crypto_register_alg(base);
+}
+EXPORT_SYMBOL_GPL(crypto_register_acomp);
+
+int crypto_unregister_acomp(struct acomp_alg *alg)
+{
+	return crypto_unregister_alg(&alg->base);
+}
+EXPORT_SYMBOL_GPL(crypto_unregister_acomp);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Asynchronous compression type");
diff --git a/crypto/algboss.c b/crypto/algboss.c
index 6e39d9c05b98..ccb85e1798f2 100644
--- a/crypto/algboss.c
+++ b/crypto/algboss.c
@@ -247,12 +247,8 @@ static int cryptomgr_schedule_test(struct crypto_alg *alg)
 	memcpy(param->alg, alg->cra_name, sizeof(param->alg));
 	type = alg->cra_flags;
 
-	/* This piece of crap needs to disappear into per-type test hooks. */
-	if (!((type ^ CRYPTO_ALG_TYPE_BLKCIPHER) &
-	      CRYPTO_ALG_TYPE_BLKCIPHER_MASK) && !(type & CRYPTO_ALG_GENIV) &&
-	    ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
-	     CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize :
-					 alg->cra_ablkcipher.ivsize))
+	/* Do not test internal algorithms. */
+	if (type & CRYPTO_ALG_INTERNAL)
 		type |= CRYPTO_ALG_TESTED;
 
 	param->type = type;
diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c
index 235f54d4f8a9..668ef402c6eb 100644
--- a/crypto/algif_aead.c
+++ b/crypto/algif_aead.c
@@ -454,12 +454,13 @@ static int aead_recvmsg_async(struct socket *sock, struct msghdr *msg,
 	used -= ctx->aead_assoclen;
 
 	/* take over all tx sgls from ctx */
-	areq->tsgl = sock_kmalloc(sk, sizeof(*areq->tsgl) * sgl->cur,
+	areq->tsgl = sock_kmalloc(sk,
+				  sizeof(*areq->tsgl) * max_t(u32, sgl->cur, 1),
 				  GFP_KERNEL);
 	if (unlikely(!areq->tsgl))
 		goto free;
 
-	sg_init_table(areq->tsgl, sgl->cur);
+	sg_init_table(areq->tsgl, max_t(u32, sgl->cur, 1));
 	for (i = 0; i < sgl->cur; i++)
 		sg_set_page(&areq->tsgl[i], sg_page(&sgl->sg[i]),
 			    sgl->sg[i].length, sgl->sg[i].offset);
diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c
index 1e38aaa8303e..a9e79d8eff87 100644
--- a/crypto/algif_skcipher.c
+++ b/crypto/algif_skcipher.c
@@ -566,8 +566,10 @@ static int skcipher_recvmsg_async(struct socket *sock, struct msghdr *msg,
 			 * need to expand */
 			tmp = kcalloc(tx_nents * 2, sizeof(*tmp),
 				      GFP_KERNEL);
-			if (!tmp)
+			if (!tmp) {
+				err = -ENOMEM;
 				goto free;
+			}
 
 			sg_init_table(tmp, tx_nents * 2);
 			for (x = 0; x < tx_nents; x++)
diff --git a/crypto/api.c b/crypto/api.c
index bbc147cb5dec..b16ce1653284 100644
--- a/crypto/api.c
+++ b/crypto/api.c
@@ -211,8 +211,8 @@ struct crypto_alg *crypto_larval_lookup(const char *name, u32 type, u32 mask)
 	if (!name)
 		return ERR_PTR(-ENOENT);
 
+	type &= ~(CRYPTO_ALG_LARVAL | CRYPTO_ALG_DEAD);
 	mask &= ~(CRYPTO_ALG_LARVAL | CRYPTO_ALG_DEAD);
-	type &= mask;
 
 	alg = crypto_alg_lookup(name, type, mask);
 	if (!alg) {
@@ -310,24 +310,8 @@ static void crypto_exit_ops(struct crypto_tfm *tfm)
 {
 	const struct crypto_type *type = tfm->__crt_alg->cra_type;
 
-	if (type) {
-		if (tfm->exit)
-			tfm->exit(tfm);
-		return;
-	}
-
-	switch (crypto_tfm_alg_type(tfm)) {
-	case CRYPTO_ALG_TYPE_CIPHER:
-		crypto_exit_cipher_ops(tfm);
-		break;
-
-	case CRYPTO_ALG_TYPE_COMPRESS:
-		crypto_exit_compress_ops(tfm);
-		break;
-
-	default:
-		BUG();
-	}
+	if (type && tfm->exit)
+		tfm->exit(tfm);
 }
 
 static unsigned int crypto_ctxsize(struct crypto_alg *alg, u32 type, u32 mask)
diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c
index fd76b5fc3b3a..d3a989e718f5 100644
--- a/crypto/asymmetric_keys/public_key.c
+++ b/crypto/asymmetric_keys/public_key.c
@@ -121,6 +121,7 @@ int public_key_verify_signature(const struct public_key *pkey,
 	if (ret)
 		goto error_free_req;
 
+	ret = -ENOMEM;
 	outlen = crypto_akcipher_maxsize(tfm);
 	output = kmalloc(outlen, GFP_KERNEL);
 	if (!output)
diff --git a/crypto/authenc.c b/crypto/authenc.c
index a7e1ac786c5d..875470b0e026 100644
--- a/crypto/authenc.c
+++ b/crypto/authenc.c
@@ -324,7 +324,7 @@ static int crypto_authenc_init_tfm(struct crypto_aead *tfm)
 	if (IS_ERR(auth))
 		return PTR_ERR(auth);
 
-	enc = crypto_spawn_skcipher2(&ictx->enc);
+	enc = crypto_spawn_skcipher(&ictx->enc);
 	err = PTR_ERR(enc);
 	if (IS_ERR(enc))
 		goto err_free_ahash;
@@ -420,9 +420,9 @@ static int crypto_authenc_create(struct crypto_template *tmpl,
 		goto err_free_inst;
 
 	crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst));
-	err = crypto_grab_skcipher2(&ctx->enc, enc_name, 0,
-				    crypto_requires_sync(algt->type,
-							 algt->mask));
+	err = crypto_grab_skcipher(&ctx->enc, enc_name, 0,
+				   crypto_requires_sync(algt->type,
+							algt->mask));
 	if (err)
 		goto err_drop_auth;
 
diff --git a/crypto/authencesn.c b/crypto/authencesn.c
index 121010ac9962..6f8f6b86bfe2 100644
--- a/crypto/authencesn.c
+++ b/crypto/authencesn.c
@@ -342,7 +342,7 @@ static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm)
 	if (IS_ERR(auth))
 		return PTR_ERR(auth);
 
-	enc = crypto_spawn_skcipher2(&ictx->enc);
+	enc = crypto_spawn_skcipher(&ictx->enc);
 	err = PTR_ERR(enc);
 	if (IS_ERR(enc))
 		goto err_free_ahash;
@@ -441,9 +441,9 @@ static int crypto_authenc_esn_create(struct crypto_template *tmpl,
 		goto err_free_inst;
 
 	crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst));
-	err = crypto_grab_skcipher2(&ctx->enc, enc_name, 0,
-				    crypto_requires_sync(algt->type,
-							 algt->mask));
+	err = crypto_grab_skcipher(&ctx->enc, enc_name, 0,
+				   crypto_requires_sync(algt->type,
+							algt->mask));
 	if (err)
 		goto err_drop_auth;
 
diff --git a/crypto/cbc.c b/crypto/cbc.c
index 780ee27b2d43..68f751a41a84 100644
--- a/crypto/cbc.c
+++ b/crypto/cbc.c
@@ -1,7 +1,7 @@
 /*
  * CBC: Cipher Block Chaining mode
  *
- * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
+ * Copyright (c) 2006-2016 Herbert Xu <herbert@gondor.apana.org.au>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
@@ -10,191 +10,78 @@
  *
  */
 
-#include <crypto/algapi.h>
+#include <crypto/cbc.h>
+#include <crypto/internal/skcipher.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/log2.h>
 #include <linux/module.h>
-#include <linux/scatterlist.h>
 #include <linux/slab.h>
 
 struct crypto_cbc_ctx {
 	struct crypto_cipher *child;
 };
 
-static int crypto_cbc_setkey(struct crypto_tfm *parent, const u8 *key,
+static int crypto_cbc_setkey(struct crypto_skcipher *parent, const u8 *key,
 			     unsigned int keylen)
 {
-	struct crypto_cbc_ctx *ctx = crypto_tfm_ctx(parent);
+	struct crypto_cbc_ctx *ctx = crypto_skcipher_ctx(parent);
 	struct crypto_cipher *child = ctx->child;
 	int err;
 
 	crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
-	crypto_cipher_set_flags(child, crypto_tfm_get_flags(parent) &
+	crypto_cipher_set_flags(child, crypto_skcipher_get_flags(parent) &
 				       CRYPTO_TFM_REQ_MASK);
 	err = crypto_cipher_setkey(child, key, keylen);
-	crypto_tfm_set_flags(parent, crypto_cipher_get_flags(child) &
-				     CRYPTO_TFM_RES_MASK);
+	crypto_skcipher_set_flags(parent, crypto_cipher_get_flags(child) &
+					  CRYPTO_TFM_RES_MASK);
 	return err;
 }
 
-static int crypto_cbc_encrypt_segment(struct blkcipher_desc *desc,
-				      struct blkcipher_walk *walk,
-				      struct crypto_cipher *tfm)
+static inline void crypto_cbc_encrypt_one(struct crypto_skcipher *tfm,
+					  const u8 *src, u8 *dst)
 {
-	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
-		crypto_cipher_alg(tfm)->cia_encrypt;
-	int bsize = crypto_cipher_blocksize(tfm);
-	unsigned int nbytes = walk->nbytes;
-	u8 *src = walk->src.virt.addr;
-	u8 *dst = walk->dst.virt.addr;
-	u8 *iv = walk->iv;
-
-	do {
-		crypto_xor(iv, src, bsize);
-		fn(crypto_cipher_tfm(tfm), dst, iv);
-		memcpy(iv, dst, bsize);
-
-		src += bsize;
-		dst += bsize;
-	} while ((nbytes -= bsize) >= bsize);
-
-	return nbytes;
-}
-
-static int crypto_cbc_encrypt_inplace(struct blkcipher_desc *desc,
-				      struct blkcipher_walk *walk,
-				      struct crypto_cipher *tfm)
-{
-	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
-		crypto_cipher_alg(tfm)->cia_encrypt;
-	int bsize = crypto_cipher_blocksize(tfm);
-	unsigned int nbytes = walk->nbytes;
-	u8 *src = walk->src.virt.addr;
-	u8 *iv = walk->iv;
-
-	do {
-		crypto_xor(src, iv, bsize);
-		fn(crypto_cipher_tfm(tfm), src, src);
-		iv = src;
-
-		src += bsize;
-	} while ((nbytes -= bsize) >= bsize);
+	struct crypto_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 
-	memcpy(walk->iv, iv, bsize);
-
-	return nbytes;
+	crypto_cipher_encrypt_one(ctx->child, dst, src);
 }
 
-static int crypto_cbc_encrypt(struct blkcipher_desc *desc,
-			      struct scatterlist *dst, struct scatterlist *src,
-			      unsigned int nbytes)
+static int crypto_cbc_encrypt(struct skcipher_request *req)
 {
-	struct blkcipher_walk walk;
-	struct crypto_blkcipher *tfm = desc->tfm;
-	struct crypto_cbc_ctx *ctx = crypto_blkcipher_ctx(tfm);
-	struct crypto_cipher *child = ctx->child;
-	int err;
-
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
-
-	while ((nbytes = walk.nbytes)) {
-		if (walk.src.virt.addr == walk.dst.virt.addr)
-			nbytes = crypto_cbc_encrypt_inplace(desc, &walk, child);
-		else
-			nbytes = crypto_cbc_encrypt_segment(desc, &walk, child);
-		err = blkcipher_walk_done(desc, &walk, nbytes);
-	}
-
-	return err;
+	return crypto_cbc_encrypt_walk(req, crypto_cbc_encrypt_one);
 }
 
-static int crypto_cbc_decrypt_segment(struct blkcipher_desc *desc,
-				      struct blkcipher_walk *walk,
-				      struct crypto_cipher *tfm)
+static inline void crypto_cbc_decrypt_one(struct crypto_skcipher *tfm,
+					  const u8 *src, u8 *dst)
 {
-	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
-		crypto_cipher_alg(tfm)->cia_decrypt;
-	int bsize = crypto_cipher_blocksize(tfm);
-	unsigned int nbytes = walk->nbytes;
-	u8 *src = walk->src.virt.addr;
-	u8 *dst = walk->dst.virt.addr;
-	u8 *iv = walk->iv;
-
-	do {
-		fn(crypto_cipher_tfm(tfm), dst, src);
-		crypto_xor(dst, iv, bsize);
-		iv = src;
-
-		src += bsize;
-		dst += bsize;
-	} while ((nbytes -= bsize) >= bsize);
-
-	memcpy(walk->iv, iv, bsize);
-
-	return nbytes;
-}
+	struct crypto_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 
-static int crypto_cbc_decrypt_inplace(struct blkcipher_desc *desc,
-				      struct blkcipher_walk *walk,
-				      struct crypto_cipher *tfm)
-{
-	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
-		crypto_cipher_alg(tfm)->cia_decrypt;
-	int bsize = crypto_cipher_blocksize(tfm);
-	unsigned int nbytes = walk->nbytes;
-	u8 *src = walk->src.virt.addr;
-	u8 last_iv[bsize];
-
-	/* Start of the last block. */
-	src += nbytes - (nbytes & (bsize - 1)) - bsize;
-	memcpy(last_iv, src, bsize);
-
-	for (;;) {
-		fn(crypto_cipher_tfm(tfm), src, src);
-		if ((nbytes -= bsize) < bsize)
-			break;
-		crypto_xor(src, src - bsize, bsize);
-		src -= bsize;
-	}
-
-	crypto_xor(src, walk->iv, bsize);
-	memcpy(walk->iv, last_iv, bsize);
-
-	return nbytes;
+	crypto_cipher_decrypt_one(ctx->child, dst, src);
 }
 
-static int crypto_cbc_decrypt(struct blkcipher_desc *desc,
-			      struct scatterlist *dst, struct scatterlist *src,
-			      unsigned int nbytes)
+static int crypto_cbc_decrypt(struct skcipher_request *req)
 {
-	struct blkcipher_walk walk;
-	struct crypto_blkcipher *tfm = desc->tfm;
-	struct crypto_cbc_ctx *ctx = crypto_blkcipher_ctx(tfm);
-	struct crypto_cipher *child = ctx->child;
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct skcipher_walk walk;
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, false);
 
-	while ((nbytes = walk.nbytes)) {
-		if (walk.src.virt.addr == walk.dst.virt.addr)
-			nbytes = crypto_cbc_decrypt_inplace(desc, &walk, child);
-		else
-			nbytes = crypto_cbc_decrypt_segment(desc, &walk, child);
-		err = blkcipher_walk_done(desc, &walk, nbytes);
+	while (walk.nbytes) {
+		err = crypto_cbc_decrypt_blocks(&walk, tfm,
+						crypto_cbc_decrypt_one);
+		err = skcipher_walk_done(&walk, err);
 	}
 
 	return err;
 }
 
-static int crypto_cbc_init_tfm(struct crypto_tfm *tfm)
+static int crypto_cbc_init_tfm(struct crypto_skcipher *tfm)
 {
-	struct crypto_instance *inst = (void *)tfm->__crt_alg;
-	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
-	struct crypto_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct skcipher_instance *inst = skcipher_alg_instance(tfm);
+	struct crypto_spawn *spawn = skcipher_instance_ctx(inst);
+	struct crypto_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 	struct crypto_cipher *cipher;
 
 	cipher = crypto_spawn_cipher(spawn);
@@ -205,72 +92,94 @@ static int crypto_cbc_init_tfm(struct crypto_tfm *tfm)
 	return 0;
 }
 
-static void crypto_cbc_exit_tfm(struct crypto_tfm *tfm)
+static void crypto_cbc_exit_tfm(struct crypto_skcipher *tfm)
 {
-	struct crypto_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+
 	crypto_free_cipher(ctx->child);
 }
 
-static struct crypto_instance *crypto_cbc_alloc(struct rtattr **tb)
+static void crypto_cbc_free(struct skcipher_instance *inst)
+{
+	crypto_drop_skcipher(skcipher_instance_ctx(inst));
+	kfree(inst);
+}
+
+static int crypto_cbc_create(struct crypto_template *tmpl, struct rtattr **tb)
 {
-	struct crypto_instance *inst;
+	struct skcipher_instance *inst;
+	struct crypto_spawn *spawn;
 	struct crypto_alg *alg;
 	int err;
 
-	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
+	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SKCIPHER);
 	if (err)
-		return ERR_PTR(err);
+		return err;
+
+	inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
+	if (!inst)
+		return -ENOMEM;
 
 	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
 				  CRYPTO_ALG_TYPE_MASK);
+	err = PTR_ERR(alg);
 	if (IS_ERR(alg))
-		return ERR_CAST(alg);
+		goto err_free_inst;
 
-	inst = ERR_PTR(-EINVAL);
-	if (!is_power_of_2(alg->cra_blocksize))
-		goto out_put_alg;
+	spawn = skcipher_instance_ctx(inst);
+	err = crypto_init_spawn(spawn, alg, skcipher_crypto_instance(inst),
+				CRYPTO_ALG_TYPE_MASK);
+	crypto_mod_put(alg);
+	if (err)
+		goto err_free_inst;
 
-	inst = crypto_alloc_instance("cbc", alg);
-	if (IS_ERR(inst))
-		goto out_put_alg;
+	err = crypto_inst_setname(skcipher_crypto_instance(inst), "cbc", alg);
+	if (err)
+		goto err_drop_spawn;
 
-	inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
-	inst->alg.cra_priority = alg->cra_priority;
-	inst->alg.cra_blocksize = alg->cra_blocksize;
-	inst->alg.cra_alignmask = alg->cra_alignmask;
-	inst->alg.cra_type = &crypto_blkcipher_type;
+	err = -EINVAL;
+	if (!is_power_of_2(alg->cra_blocksize))
+		goto err_drop_spawn;
+
+	inst->alg.base.cra_priority = alg->cra_priority;
+	inst->alg.base.cra_blocksize = alg->cra_blocksize;
+	inst->alg.base.cra_alignmask = alg->cra_alignmask;
 
 	/* We access the data as u32s when xoring. */
-	inst->alg.cra_alignmask |= __alignof__(u32) - 1;
+	inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
 
-	inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
-	inst->alg.cra_blkcipher.min_keysize = alg->cra_cipher.cia_min_keysize;
-	inst->alg.cra_blkcipher.max_keysize = alg->cra_cipher.cia_max_keysize;
+	inst->alg.ivsize = alg->cra_blocksize;
+	inst->alg.min_keysize = alg->cra_cipher.cia_min_keysize;
+	inst->alg.max_keysize = alg->cra_cipher.cia_max_keysize;
 
-	inst->alg.cra_ctxsize = sizeof(struct crypto_cbc_ctx);
+	inst->alg.base.cra_ctxsize = sizeof(struct crypto_cbc_ctx);
 
-	inst->alg.cra_init = crypto_cbc_init_tfm;
-	inst->alg.cra_exit = crypto_cbc_exit_tfm;
+	inst->alg.init = crypto_cbc_init_tfm;
+	inst->alg.exit = crypto_cbc_exit_tfm;
 
-	inst->alg.cra_blkcipher.setkey = crypto_cbc_setkey;
-	inst->alg.cra_blkcipher.encrypt = crypto_cbc_encrypt;
-	inst->alg.cra_blkcipher.decrypt = crypto_cbc_decrypt;
+	inst->alg.setkey = crypto_cbc_setkey;
+	inst->alg.encrypt = crypto_cbc_encrypt;
+	inst->alg.decrypt = crypto_cbc_decrypt;
 
-out_put_alg:
-	crypto_mod_put(alg);
-	return inst;
-}
+	inst->free = crypto_cbc_free;
 
-static void crypto_cbc_free(struct crypto_instance *inst)
-{
-	crypto_drop_spawn(crypto_instance_ctx(inst));
+	err = skcipher_register_instance(tmpl, inst);
+	if (err)
+		goto err_drop_spawn;
+
+out:
+	return err;
+
+err_drop_spawn:
+	crypto_drop_spawn(spawn);
+err_free_inst:
 	kfree(inst);
+	goto out;
 }
 
 static struct crypto_template crypto_cbc_tmpl = {
 	.name = "cbc",
-	.alloc = crypto_cbc_alloc,
-	.free = crypto_cbc_free,
+	.create = crypto_cbc_create,
 	.module = THIS_MODULE,
 };
 
diff --git a/crypto/ccm.c b/crypto/ccm.c
index 006d8575ef5c..26b924d1e582 100644
--- a/crypto/ccm.c
+++ b/crypto/ccm.c
@@ -462,7 +462,7 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 
-	ctr = crypto_spawn_skcipher2(&ictx->ctr);
+	ctr = crypto_spawn_skcipher(&ictx->ctr);
 	err = PTR_ERR(ctr);
 	if (IS_ERR(ctr))
 		goto err_free_cipher;
@@ -544,9 +544,9 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
 		goto err_free_inst;
 
 	crypto_set_skcipher_spawn(&ictx->ctr, aead_crypto_instance(inst));
-	err = crypto_grab_skcipher2(&ictx->ctr, ctr_name, 0,
-				    crypto_requires_sync(algt->type,
-							 algt->mask));
+	err = crypto_grab_skcipher(&ictx->ctr, ctr_name, 0,
+				   crypto_requires_sync(algt->type,
+							algt->mask));
 	if (err)
 		goto err_drop_cipher;
 
diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c
index e899ef51dc8e..db1bc3147bc4 100644
--- a/crypto/chacha20poly1305.c
+++ b/crypto/chacha20poly1305.c
@@ -532,7 +532,7 @@ static int chachapoly_init(struct crypto_aead *tfm)
 	if (IS_ERR(poly))
 		return PTR_ERR(poly);
 
-	chacha = crypto_spawn_skcipher2(&ictx->chacha);
+	chacha = crypto_spawn_skcipher(&ictx->chacha);
 	if (IS_ERR(chacha)) {
 		crypto_free_ahash(poly);
 		return PTR_ERR(chacha);
@@ -625,9 +625,9 @@ static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb,
 		goto err_free_inst;
 
 	crypto_set_skcipher_spawn(&ctx->chacha, aead_crypto_instance(inst));
-	err = crypto_grab_skcipher2(&ctx->chacha, chacha_name, 0,
-				    crypto_requires_sync(algt->type,
-							 algt->mask));
+	err = crypto_grab_skcipher(&ctx->chacha, chacha_name, 0,
+				   crypto_requires_sync(algt->type,
+							algt->mask));
 	if (err)
 		goto err_drop_poly;
 
diff --git a/crypto/cipher.c b/crypto/cipher.c
index 39541e0e537d..94fa3551476b 100644
--- a/crypto/cipher.c
+++ b/crypto/cipher.c
@@ -116,7 +116,3 @@ int crypto_init_cipher_ops(struct crypto_tfm *tfm)
 
 	return 0;
 }
-
-void crypto_exit_cipher_ops(struct crypto_tfm *tfm)
-{
-}
diff --git a/crypto/cmac.c b/crypto/cmac.c
index 7a8bfbd548f6..04080dca8f0c 100644
--- a/crypto/cmac.c
+++ b/crypto/cmac.c
@@ -57,7 +57,8 @@ static int crypto_cmac_digest_setkey(struct crypto_shash *parent,
 	unsigned long alignmask = crypto_shash_alignmask(parent);
 	struct cmac_tfm_ctx *ctx = crypto_shash_ctx(parent);
 	unsigned int bs = crypto_shash_blocksize(parent);
-	__be64 *consts = PTR_ALIGN((void *)ctx->ctx, alignmask + 1);
+	__be64 *consts = PTR_ALIGN((void *)ctx->ctx,
+				   (alignmask | (__alignof__(__be64) - 1)) + 1);
 	u64 _const[2];
 	int i, err = 0;
 	u8 msb_mask, gfmask;
@@ -173,7 +174,8 @@ static int crypto_cmac_digest_final(struct shash_desc *pdesc, u8 *out)
 	struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
 	struct crypto_cipher *tfm = tctx->child;
 	int bs = crypto_shash_blocksize(parent);
-	u8 *consts = PTR_ALIGN((void *)tctx->ctx, alignmask + 1);
+	u8 *consts = PTR_ALIGN((void *)tctx->ctx,
+			       (alignmask | (__alignof__(__be64) - 1)) + 1);
 	u8 *odds = PTR_ALIGN((void *)ctx->ctx, alignmask + 1);
 	u8 *prev = odds + bs;
 	unsigned int offset = 0;
@@ -243,6 +245,7 @@ static int cmac_create(struct crypto_template *tmpl, struct rtattr **tb)
 	case 8:
 		break;
 	default:
+		err = -EINVAL;
 		goto out_put_alg;
 	}
 
@@ -257,7 +260,8 @@ static int cmac_create(struct crypto_template *tmpl, struct rtattr **tb)
 	if (err)
 		goto out_free_inst;
 
-	alignmask = alg->cra_alignmask | (sizeof(long) - 1);
+	/* We access the data as u32s when xoring. */
+	alignmask = alg->cra_alignmask | (__alignof__(u32) - 1);
 	inst->alg.base.cra_alignmask = alignmask;
 	inst->alg.base.cra_priority = alg->cra_priority;
 	inst->alg.base.cra_blocksize = alg->cra_blocksize;
@@ -269,7 +273,9 @@ static int cmac_create(struct crypto_template *tmpl, struct rtattr **tb)
 		+ alg->cra_blocksize * 2;
 
 	inst->alg.base.cra_ctxsize =
-		ALIGN(sizeof(struct cmac_tfm_ctx), alignmask + 1)
+		ALIGN(sizeof(struct cmac_tfm_ctx), crypto_tfm_ctx_alignment())
+		+ ((alignmask | (__alignof__(__be64) - 1)) &
+		   ~(crypto_tfm_ctx_alignment() - 1))
 		+ alg->cra_blocksize * 2;
 
 	inst->alg.base.cra_init = cmac_init_tfm;
diff --git a/crypto/compress.c b/crypto/compress.c
index c33f0763a956..f2d522924a07 100644
--- a/crypto/compress.c
+++ b/crypto/compress.c
@@ -42,7 +42,3 @@ int crypto_init_compress_ops(struct crypto_tfm *tfm)
 
 	return 0;
 }
-
-void crypto_exit_compress_ops(struct crypto_tfm *tfm)
-{
-}
diff --git a/crypto/cryptd.c b/crypto/cryptd.c
index 0c654e59f215..0508c48a45c4 100644
--- a/crypto/cryptd.c
+++ b/crypto/cryptd.c
@@ -17,9 +17,9 @@
  *
  */
 
-#include <crypto/algapi.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
 #include <crypto/cryptd.h>
 #include <crypto/crypto_wq.h>
 #include <linux/atomic.h>
@@ -48,6 +48,11 @@ struct cryptd_instance_ctx {
 	struct cryptd_queue *queue;
 };
 
+struct skcipherd_instance_ctx {
+	struct crypto_skcipher_spawn spawn;
+	struct cryptd_queue *queue;
+};
+
 struct hashd_instance_ctx {
 	struct crypto_shash_spawn spawn;
 	struct cryptd_queue *queue;
@@ -67,6 +72,15 @@ struct cryptd_blkcipher_request_ctx {
 	crypto_completion_t complete;
 };
 
+struct cryptd_skcipher_ctx {
+	atomic_t refcnt;
+	struct crypto_skcipher *child;
+};
+
+struct cryptd_skcipher_request_ctx {
+	crypto_completion_t complete;
+};
+
 struct cryptd_hash_ctx {
 	atomic_t refcnt;
 	struct crypto_shash *child;
@@ -122,7 +136,6 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue,
 {
 	int cpu, err;
 	struct cryptd_cpu_queue *cpu_queue;
-	struct crypto_tfm *tfm;
 	atomic_t *refcnt;
 	bool may_backlog;
 
@@ -141,7 +154,6 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue,
 	if (!atomic_read(refcnt))
 		goto out_put_cpu;
 
-	tfm = request->tfm;
 	atomic_inc(refcnt);
 
 out_put_cpu:
@@ -432,6 +444,216 @@ out_put_alg:
 	return err;
 }
 
+static int cryptd_skcipher_setkey(struct crypto_skcipher *parent,
+				  const u8 *key, unsigned int keylen)
+{
+	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(parent);
+	struct crypto_skcipher *child = ctx->child;
+	int err;
+
+	crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
+					 CRYPTO_TFM_REQ_MASK);
+	err = crypto_skcipher_setkey(child, key, keylen);
+	crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) &
+					  CRYPTO_TFM_RES_MASK);
+	return err;
+}
+
+static void cryptd_skcipher_complete(struct skcipher_request *req, int err)
+{
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
+	int refcnt = atomic_read(&ctx->refcnt);
+
+	local_bh_disable();
+	rctx->complete(&req->base, err);
+	local_bh_enable();
+
+	if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
+		crypto_free_skcipher(tfm);
+}
+
+static void cryptd_skcipher_encrypt(struct crypto_async_request *base,
+				    int err)
+{
+	struct skcipher_request *req = skcipher_request_cast(base);
+	struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct crypto_skcipher *child = ctx->child;
+	SKCIPHER_REQUEST_ON_STACK(subreq, child);
+
+	if (unlikely(err == -EINPROGRESS))
+		goto out;
+
+	skcipher_request_set_tfm(subreq, child);
+	skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP,
+				      NULL, NULL);
+	skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
+				   req->iv);
+
+	err = crypto_skcipher_encrypt(subreq);
+	skcipher_request_zero(subreq);
+
+	req->base.complete = rctx->complete;
+
+out:
+	cryptd_skcipher_complete(req, err);
+}
+
+static void cryptd_skcipher_decrypt(struct crypto_async_request *base,
+				    int err)
+{
+	struct skcipher_request *req = skcipher_request_cast(base);
+	struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct crypto_skcipher *child = ctx->child;
+	SKCIPHER_REQUEST_ON_STACK(subreq, child);
+
+	if (unlikely(err == -EINPROGRESS))
+		goto out;
+
+	skcipher_request_set_tfm(subreq, child);
+	skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP,
+				      NULL, NULL);
+	skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
+				   req->iv);
+
+	err = crypto_skcipher_decrypt(subreq);
+	skcipher_request_zero(subreq);
+
+	req->base.complete = rctx->complete;
+
+out:
+	cryptd_skcipher_complete(req, err);
+}
+
+static int cryptd_skcipher_enqueue(struct skcipher_request *req,
+				   crypto_completion_t compl)
+{
+	struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct cryptd_queue *queue;
+
+	queue = cryptd_get_queue(crypto_skcipher_tfm(tfm));
+	rctx->complete = req->base.complete;
+	req->base.complete = compl;
+
+	return cryptd_enqueue_request(queue, &req->base);
+}
+
+static int cryptd_skcipher_encrypt_enqueue(struct skcipher_request *req)
+{
+	return cryptd_skcipher_enqueue(req, cryptd_skcipher_encrypt);
+}
+
+static int cryptd_skcipher_decrypt_enqueue(struct skcipher_request *req)
+{
+	return cryptd_skcipher_enqueue(req, cryptd_skcipher_decrypt);
+}
+
+static int cryptd_skcipher_init_tfm(struct crypto_skcipher *tfm)
+{
+	struct skcipher_instance *inst = skcipher_alg_instance(tfm);
+	struct skcipherd_instance_ctx *ictx = skcipher_instance_ctx(inst);
+	struct crypto_skcipher_spawn *spawn = &ictx->spawn;
+	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct crypto_skcipher *cipher;
+
+	cipher = crypto_spawn_skcipher(spawn);
+	if (IS_ERR(cipher))
+		return PTR_ERR(cipher);
+
+	ctx->child = cipher;
+	crypto_skcipher_set_reqsize(
+		tfm, sizeof(struct cryptd_skcipher_request_ctx));
+	return 0;
+}
+
+static void cryptd_skcipher_exit_tfm(struct crypto_skcipher *tfm)
+{
+	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+	crypto_free_skcipher(ctx->child);
+}
+
+static void cryptd_skcipher_free(struct skcipher_instance *inst)
+{
+	struct skcipherd_instance_ctx *ctx = skcipher_instance_ctx(inst);
+
+	crypto_drop_skcipher(&ctx->spawn);
+}
+
+static int cryptd_create_skcipher(struct crypto_template *tmpl,
+				  struct rtattr **tb,
+				  struct cryptd_queue *queue)
+{
+	struct skcipherd_instance_ctx *ctx;
+	struct skcipher_instance *inst;
+	struct skcipher_alg *alg;
+	const char *name;
+	u32 type;
+	u32 mask;
+	int err;
+
+	type = 0;
+	mask = CRYPTO_ALG_ASYNC;
+
+	cryptd_check_internal(tb, &type, &mask);
+
+	name = crypto_attr_alg_name(tb[1]);
+	if (IS_ERR(name))
+		return PTR_ERR(name);
+
+	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
+	if (!inst)
+		return -ENOMEM;
+
+	ctx = skcipher_instance_ctx(inst);
+	ctx->queue = queue;
+
+	crypto_set_skcipher_spawn(&ctx->spawn, skcipher_crypto_instance(inst));
+	err = crypto_grab_skcipher(&ctx->spawn, name, type, mask);
+	if (err)
+		goto out_free_inst;
+
+	alg = crypto_spawn_skcipher_alg(&ctx->spawn);
+	err = cryptd_init_instance(skcipher_crypto_instance(inst), &alg->base);
+	if (err)
+		goto out_drop_skcipher;
+
+	inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC |
+				   (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
+
+	inst->alg.ivsize = crypto_skcipher_alg_ivsize(alg);
+	inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg);
+	inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg);
+	inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg);
+
+	inst->alg.base.cra_ctxsize = sizeof(struct cryptd_skcipher_ctx);
+
+	inst->alg.init = cryptd_skcipher_init_tfm;
+	inst->alg.exit = cryptd_skcipher_exit_tfm;
+
+	inst->alg.setkey = cryptd_skcipher_setkey;
+	inst->alg.encrypt = cryptd_skcipher_encrypt_enqueue;
+	inst->alg.decrypt = cryptd_skcipher_decrypt_enqueue;
+
+	inst->free = cryptd_skcipher_free;
+
+	err = skcipher_register_instance(tmpl, inst);
+	if (err) {
+out_drop_skcipher:
+		crypto_drop_skcipher(&ctx->spawn);
+out_free_inst:
+		kfree(inst);
+	}
+	return err;
+}
+
 static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
@@ -895,7 +1117,11 @@ static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
 
 	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
 	case CRYPTO_ALG_TYPE_BLKCIPHER:
-		return cryptd_create_blkcipher(tmpl, tb, &queue);
+		if ((algt->type & CRYPTO_ALG_TYPE_MASK) ==
+		    CRYPTO_ALG_TYPE_BLKCIPHER)
+			return cryptd_create_blkcipher(tmpl, tb, &queue);
+
+		return cryptd_create_skcipher(tmpl, tb, &queue);
 	case CRYPTO_ALG_TYPE_DIGEST:
 		return cryptd_create_hash(tmpl, tb, &queue);
 	case CRYPTO_ALG_TYPE_AEAD:
@@ -985,6 +1211,58 @@ void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm)
 }
 EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher);
 
+struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name,
+					      u32 type, u32 mask)
+{
+	char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
+	struct cryptd_skcipher_ctx *ctx;
+	struct crypto_skcipher *tfm;
+
+	if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
+		     "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
+		return ERR_PTR(-EINVAL);
+
+	tfm = crypto_alloc_skcipher(cryptd_alg_name, type, mask);
+	if (IS_ERR(tfm))
+		return ERR_CAST(tfm);
+
+	if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
+		crypto_free_skcipher(tfm);
+		return ERR_PTR(-EINVAL);
+	}
+
+	ctx = crypto_skcipher_ctx(tfm);
+	atomic_set(&ctx->refcnt, 1);
+
+	return container_of(tfm, struct cryptd_skcipher, base);
+}
+EXPORT_SYMBOL_GPL(cryptd_alloc_skcipher);
+
+struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm)
+{
+	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
+
+	return ctx->child;
+}
+EXPORT_SYMBOL_GPL(cryptd_skcipher_child);
+
+bool cryptd_skcipher_queued(struct cryptd_skcipher *tfm)
+{
+	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
+
+	return atomic_read(&ctx->refcnt) - 1;
+}
+EXPORT_SYMBOL_GPL(cryptd_skcipher_queued);
+
+void cryptd_free_skcipher(struct cryptd_skcipher *tfm)
+{
+	struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base);
+
+	if (atomic_dec_and_test(&ctx->refcnt))
+		crypto_free_skcipher(&tfm->base);
+}
+EXPORT_SYMBOL_GPL(cryptd_free_skcipher);
+
 struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
 					u32 type, u32 mask)
 {
diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c
index 6989ba0046df..f1bf3418d968 100644
--- a/crypto/crypto_engine.c
+++ b/crypto/crypto_engine.c
@@ -47,7 +47,7 @@ static void crypto_pump_requests(struct crypto_engine *engine,
 
 	/* If another context is idling then defer */
 	if (engine->idling) {
-		kthread_queue_work(&engine->kworker, &engine->pump_requests);
+		kthread_queue_work(engine->kworker, &engine->pump_requests);
 		goto out;
 	}
 
@@ -58,7 +58,7 @@ static void crypto_pump_requests(struct crypto_engine *engine,
 
 		/* Only do teardown in the thread */
 		if (!in_kthread) {
-			kthread_queue_work(&engine->kworker,
+			kthread_queue_work(engine->kworker,
 					   &engine->pump_requests);
 			goto out;
 		}
@@ -189,7 +189,7 @@ int crypto_transfer_cipher_request(struct crypto_engine *engine,
 	ret = ablkcipher_enqueue_request(&engine->queue, req);
 
 	if (!engine->busy && need_pump)
-		kthread_queue_work(&engine->kworker, &engine->pump_requests);
+		kthread_queue_work(engine->kworker, &engine->pump_requests);
 
 	spin_unlock_irqrestore(&engine->queue_lock, flags);
 	return ret;
@@ -231,7 +231,7 @@ int crypto_transfer_hash_request(struct crypto_engine *engine,
 	ret = ahash_enqueue_request(&engine->queue, req);
 
 	if (!engine->busy && need_pump)
-		kthread_queue_work(&engine->kworker, &engine->pump_requests);
+		kthread_queue_work(engine->kworker, &engine->pump_requests);
 
 	spin_unlock_irqrestore(&engine->queue_lock, flags);
 	return ret;
@@ -284,7 +284,7 @@ void crypto_finalize_cipher_request(struct crypto_engine *engine,
 
 	req->base.complete(&req->base, err);
 
-	kthread_queue_work(&engine->kworker, &engine->pump_requests);
+	kthread_queue_work(engine->kworker, &engine->pump_requests);
 }
 EXPORT_SYMBOL_GPL(crypto_finalize_cipher_request);
 
@@ -321,7 +321,7 @@ void crypto_finalize_hash_request(struct crypto_engine *engine,
 
 	req->base.complete(&req->base, err);
 
-	kthread_queue_work(&engine->kworker, &engine->pump_requests);
+	kthread_queue_work(engine->kworker, &engine->pump_requests);
 }
 EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);
 
@@ -345,7 +345,7 @@ int crypto_engine_start(struct crypto_engine *engine)
 	engine->running = true;
 	spin_unlock_irqrestore(&engine->queue_lock, flags);
 
-	kthread_queue_work(&engine->kworker, &engine->pump_requests);
+	kthread_queue_work(engine->kworker, &engine->pump_requests);
 
 	return 0;
 }
@@ -422,11 +422,8 @@ struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
 	crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN);
 	spin_lock_init(&engine->queue_lock);
 
-	kthread_init_worker(&engine->kworker);
-	engine->kworker_task = kthread_run(kthread_worker_fn,
-					   &engine->kworker, "%s",
-					   engine->name);
-	if (IS_ERR(engine->kworker_task)) {
+	engine->kworker = kthread_create_worker(0, "%s", engine->name);
+	if (IS_ERR(engine->kworker)) {
 		dev_err(dev, "failed to create crypto request pump task\n");
 		return NULL;
 	}
@@ -434,7 +431,7 @@ struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
 
 	if (engine->rt) {
 		dev_info(dev, "will run requests pump with realtime priority\n");
-		sched_setscheduler(engine->kworker_task, SCHED_FIFO, &param);
+		sched_setscheduler(engine->kworker->task, SCHED_FIFO, &param);
 	}
 
 	return engine;
@@ -455,8 +452,7 @@ int crypto_engine_exit(struct crypto_engine *engine)
 	if (ret)
 		return ret;
 
-	kthread_flush_worker(&engine->kworker);
-	kthread_stop(engine->kworker_task);
+	kthread_destroy_worker(engine->kworker);
 
 	return 0;
 }
diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c
index 1c5705481c69..a90404a0c5ff 100644
--- a/crypto/crypto_user.c
+++ b/crypto/crypto_user.c
@@ -112,6 +112,21 @@ nla_put_failure:
 	return -EMSGSIZE;
 }
 
+static int crypto_report_acomp(struct sk_buff *skb, struct crypto_alg *alg)
+{
+	struct crypto_report_acomp racomp;
+
+	strncpy(racomp.type, "acomp", sizeof(racomp.type));
+
+	if (nla_put(skb, CRYPTOCFGA_REPORT_ACOMP,
+		    sizeof(struct crypto_report_acomp), &racomp))
+		goto nla_put_failure;
+	return 0;
+
+nla_put_failure:
+	return -EMSGSIZE;
+}
+
 static int crypto_report_akcipher(struct sk_buff *skb, struct crypto_alg *alg)
 {
 	struct crypto_report_akcipher rakcipher;
@@ -186,7 +201,11 @@ static int crypto_report_one(struct crypto_alg *alg,
 			goto nla_put_failure;
 
 		break;
+	case CRYPTO_ALG_TYPE_ACOMPRESS:
+		if (crypto_report_acomp(skb, alg))
+			goto nla_put_failure;
 
+		break;
 	case CRYPTO_ALG_TYPE_AKCIPHER:
 		if (crypto_report_akcipher(skb, alg))
 			goto nla_put_failure;
diff --git a/crypto/ctr.c b/crypto/ctr.c
index ff4d21eddb83..a9a7a44f2783 100644
--- a/crypto/ctr.c
+++ b/crypto/ctr.c
@@ -312,7 +312,7 @@ static int crypto_rfc3686_init_tfm(struct crypto_skcipher *tfm)
 	unsigned long align;
 	unsigned int reqsize;
 
-	cipher = crypto_spawn_skcipher2(spawn);
+	cipher = crypto_spawn_skcipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 
@@ -370,9 +370,9 @@ static int crypto_rfc3686_create(struct crypto_template *tmpl,
 	spawn = skcipher_instance_ctx(inst);
 
 	crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst));
-	err = crypto_grab_skcipher2(spawn, cipher_name, 0,
-				    crypto_requires_sync(algt->type,
-							 algt->mask));
+	err = crypto_grab_skcipher(spawn, cipher_name, 0,
+				   crypto_requires_sync(algt->type,
+							algt->mask));
 	if (err)
 		goto err_free_inst;
 
diff --git a/crypto/cts.c b/crypto/cts.c
index 51976187b2bf..00254d76b21b 100644
--- a/crypto/cts.c
+++ b/crypto/cts.c
@@ -290,7 +290,7 @@ static int crypto_cts_init_tfm(struct crypto_skcipher *tfm)
 	unsigned bsize;
 	unsigned align;
 
-	cipher = crypto_spawn_skcipher2(spawn);
+	cipher = crypto_spawn_skcipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 
@@ -348,9 +348,9 @@ static int crypto_cts_create(struct crypto_template *tmpl, struct rtattr **tb)
 	spawn = skcipher_instance_ctx(inst);
 
 	crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst));
-	err = crypto_grab_skcipher2(spawn, cipher_name, 0,
-				    crypto_requires_sync(algt->type,
-							 algt->mask));
+	err = crypto_grab_skcipher(spawn, cipher_name, 0,
+				   crypto_requires_sync(algt->type,
+							algt->mask));
 	if (err)
 		goto err_free_inst;
 
diff --git a/crypto/deflate.c b/crypto/deflate.c
index 95d8d37c5021..f942cb391890 100644
--- a/crypto/deflate.c
+++ b/crypto/deflate.c
@@ -32,6 +32,7 @@
 #include <linux/interrupt.h>
 #include <linux/mm.h>
 #include <linux/net.h>
+#include <crypto/internal/scompress.h>
 
 #define DEFLATE_DEF_LEVEL		Z_DEFAULT_COMPRESSION
 #define DEFLATE_DEF_WINBITS		11
@@ -101,9 +102,8 @@ static void deflate_decomp_exit(struct deflate_ctx *ctx)
 	vfree(ctx->decomp_stream.workspace);
 }
 
-static int deflate_init(struct crypto_tfm *tfm)
+static int __deflate_init(void *ctx)
 {
-	struct deflate_ctx *ctx = crypto_tfm_ctx(tfm);
 	int ret;
 
 	ret = deflate_comp_init(ctx);
@@ -116,19 +116,55 @@ out:
 	return ret;
 }
 
-static void deflate_exit(struct crypto_tfm *tfm)
+static void *deflate_alloc_ctx(struct crypto_scomp *tfm)
+{
+	struct deflate_ctx *ctx;
+	int ret;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	ret = __deflate_init(ctx);
+	if (ret) {
+		kfree(ctx);
+		return ERR_PTR(ret);
+	}
+
+	return ctx;
+}
+
+static int deflate_init(struct crypto_tfm *tfm)
 {
 	struct deflate_ctx *ctx = crypto_tfm_ctx(tfm);
 
+	return __deflate_init(ctx);
+}
+
+static void __deflate_exit(void *ctx)
+{
 	deflate_comp_exit(ctx);
 	deflate_decomp_exit(ctx);
 }
 
-static int deflate_compress(struct crypto_tfm *tfm, const u8 *src,
-			    unsigned int slen, u8 *dst, unsigned int *dlen)
+static void deflate_free_ctx(struct crypto_scomp *tfm, void *ctx)
+{
+	__deflate_exit(ctx);
+	kzfree(ctx);
+}
+
+static void deflate_exit(struct crypto_tfm *tfm)
+{
+	struct deflate_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	__deflate_exit(ctx);
+}
+
+static int __deflate_compress(const u8 *src, unsigned int slen,
+			      u8 *dst, unsigned int *dlen, void *ctx)
 {
 	int ret = 0;
-	struct deflate_ctx *dctx = crypto_tfm_ctx(tfm);
+	struct deflate_ctx *dctx = ctx;
 	struct z_stream_s *stream = &dctx->comp_stream;
 
 	ret = zlib_deflateReset(stream);
@@ -153,12 +189,27 @@ out:
 	return ret;
 }
 
-static int deflate_decompress(struct crypto_tfm *tfm, const u8 *src,
-			      unsigned int slen, u8 *dst, unsigned int *dlen)
+static int deflate_compress(struct crypto_tfm *tfm, const u8 *src,
+			    unsigned int slen, u8 *dst, unsigned int *dlen)
+{
+	struct deflate_ctx *dctx = crypto_tfm_ctx(tfm);
+
+	return __deflate_compress(src, slen, dst, dlen, dctx);
+}
+
+static int deflate_scompress(struct crypto_scomp *tfm, const u8 *src,
+			     unsigned int slen, u8 *dst, unsigned int *dlen,
+			     void *ctx)
+{
+	return __deflate_compress(src, slen, dst, dlen, ctx);
+}
+
+static int __deflate_decompress(const u8 *src, unsigned int slen,
+				u8 *dst, unsigned int *dlen, void *ctx)
 {
 
 	int ret = 0;
-	struct deflate_ctx *dctx = crypto_tfm_ctx(tfm);
+	struct deflate_ctx *dctx = ctx;
 	struct z_stream_s *stream = &dctx->decomp_stream;
 
 	ret = zlib_inflateReset(stream);
@@ -194,6 +245,21 @@ out:
 	return ret;
 }
 
+static int deflate_decompress(struct crypto_tfm *tfm, const u8 *src,
+			      unsigned int slen, u8 *dst, unsigned int *dlen)
+{
+	struct deflate_ctx *dctx = crypto_tfm_ctx(tfm);
+
+	return __deflate_decompress(src, slen, dst, dlen, dctx);
+}
+
+static int deflate_sdecompress(struct crypto_scomp *tfm, const u8 *src,
+			       unsigned int slen, u8 *dst, unsigned int *dlen,
+			       void *ctx)
+{
+	return __deflate_decompress(src, slen, dst, dlen, ctx);
+}
+
 static struct crypto_alg alg = {
 	.cra_name		= "deflate",
 	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
@@ -206,14 +272,39 @@ static struct crypto_alg alg = {
 	.coa_decompress  	= deflate_decompress } }
 };
 
+static struct scomp_alg scomp = {
+	.alloc_ctx		= deflate_alloc_ctx,
+	.free_ctx		= deflate_free_ctx,
+	.compress		= deflate_scompress,
+	.decompress		= deflate_sdecompress,
+	.base			= {
+		.cra_name	= "deflate",
+		.cra_driver_name = "deflate-scomp",
+		.cra_module	 = THIS_MODULE,
+	}
+};
+
 static int __init deflate_mod_init(void)
 {
-	return crypto_register_alg(&alg);
+	int ret;
+
+	ret = crypto_register_alg(&alg);
+	if (ret)
+		return ret;
+
+	ret = crypto_register_scomp(&scomp);
+	if (ret) {
+		crypto_unregister_alg(&alg);
+		return ret;
+	}
+
+	return ret;
 }
 
 static void __exit deflate_mod_fini(void)
 {
 	crypto_unregister_alg(&alg);
+	crypto_unregister_scomp(&scomp);
 }
 
 module_init(deflate_mod_init);
diff --git a/crypto/dh.c b/crypto/dh.c
index 9d19360e7189..ddcb528ab2cc 100644
--- a/crypto/dh.c
+++ b/crypto/dh.c
@@ -118,7 +118,7 @@ static int dh_compute_value(struct kpp_request *req)
 	if (req->src) {
 		base = mpi_read_raw_from_sgl(req->src, req->src_len);
 		if (!base) {
-			ret = EINVAL;
+			ret = -EINVAL;
 			goto err_free_val;
 		}
 	} else {
diff --git a/crypto/drbg.c b/crypto/drbg.c
index 053035b5c8f8..8a4d98b4adba 100644
--- a/crypto/drbg.c
+++ b/crypto/drbg.c
@@ -1782,6 +1782,7 @@ static int drbg_kcapi_sym_ctr(struct drbg_state *drbg,
 		memcpy(outbuf, drbg->outscratchpad, cryptlen);
 
 		outlen -= cryptlen;
+		outbuf += cryptlen;
 	}
 	ret = 0;
 
diff --git a/crypto/gcm.c b/crypto/gcm.c
index f624ac98c94e..b7ad808be3d4 100644
--- a/crypto/gcm.c
+++ b/crypto/gcm.c
@@ -575,7 +575,7 @@ static int crypto_gcm_init_tfm(struct crypto_aead *tfm)
 	if (IS_ERR(ghash))
 		return PTR_ERR(ghash);
 
-	ctr = crypto_spawn_skcipher2(&ictx->ctr);
+	ctr = crypto_spawn_skcipher(&ictx->ctr);
 	err = PTR_ERR(ctr);
 	if (IS_ERR(ctr))
 		goto err_free_hash;
@@ -663,20 +663,20 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl,
 		goto err_drop_ghash;
 
 	crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst));
-	err = crypto_grab_skcipher2(&ctx->ctr, ctr_name, 0,
-				    crypto_requires_sync(algt->type,
-							 algt->mask));
+	err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
+				   crypto_requires_sync(algt->type,
+							algt->mask));
 	if (err)
 		goto err_drop_ghash;
 
 	ctr = crypto_spawn_skcipher_alg(&ctx->ctr);
 
 	/* We only support 16-byte blocks. */
+	err = -EINVAL;
 	if (crypto_skcipher_alg_ivsize(ctr) != 16)
 		goto out_put_ctr;
 
 	/* Not a stream cipher? */
-	err = -EINVAL;
 	if (ctr->base.cra_blocksize != 1)
 		goto out_put_ctr;
 
diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c
index 5276607c72d0..72015fee533d 100644
--- a/crypto/gf128mul.c
+++ b/crypto/gf128mul.c
@@ -263,48 +263,6 @@ EXPORT_SYMBOL(gf128mul_bbe);
  * t[1][BYTE] contains g*x^8*BYTE
  *  ..
  * t[15][BYTE] contains g*x^120*BYTE */
-struct gf128mul_64k *gf128mul_init_64k_lle(const be128 *g)
-{
-	struct gf128mul_64k *t;
-	int i, j, k;
-
-	t = kzalloc(sizeof(*t), GFP_KERNEL);
-	if (!t)
-		goto out;
-
-	for (i = 0; i < 16; i++) {
-		t->t[i] = kzalloc(sizeof(*t->t[i]), GFP_KERNEL);
-		if (!t->t[i]) {
-			gf128mul_free_64k(t);
-			t = NULL;
-			goto out;
-		}
-	}
-
-	t->t[0]->t[128] = *g;
-	for (j = 64; j > 0; j >>= 1)
-		gf128mul_x_lle(&t->t[0]->t[j], &t->t[0]->t[j + j]);
-
-	for (i = 0;;) {
-		for (j = 2; j < 256; j += j)
-			for (k = 1; k < j; ++k)
-				be128_xor(&t->t[i]->t[j + k],
-					  &t->t[i]->t[j], &t->t[i]->t[k]);
-
-		if (++i >= 16)
-			break;
-
-		for (j = 128; j > 0; j >>= 1) {
-			t->t[i]->t[j] = t->t[i - 1]->t[j];
-			gf128mul_x8_lle(&t->t[i]->t[j]);
-		}
-	}
-
-out:
-	return t;
-}
-EXPORT_SYMBOL(gf128mul_init_64k_lle);
-
 struct gf128mul_64k *gf128mul_init_64k_bbe(const be128 *g)
 {
 	struct gf128mul_64k *t;
@@ -352,24 +310,11 @@ void gf128mul_free_64k(struct gf128mul_64k *t)
 	int i;
 
 	for (i = 0; i < 16; i++)
-		kfree(t->t[i]);
-	kfree(t);
+		kzfree(t->t[i]);
+	kzfree(t);
 }
 EXPORT_SYMBOL(gf128mul_free_64k);
 
-void gf128mul_64k_lle(be128 *a, struct gf128mul_64k *t)
-{
-	u8 *ap = (u8 *)a;
-	be128 r[1];
-	int i;
-
-	*r = t->t[0]->t[ap[0]];
-	for (i = 1; i < 16; ++i)
-		be128_xor(r, r, &t->t[i]->t[ap[i]]);
-	*a = *r;
-}
-EXPORT_SYMBOL(gf128mul_64k_lle);
-
 void gf128mul_64k_bbe(be128 *a, struct gf128mul_64k *t)
 {
 	u8 *ap = (u8 *)a;
diff --git a/crypto/internal.h b/crypto/internal.h
index 7eefcdb00227..f07320423191 100644
--- a/crypto/internal.h
+++ b/crypto/internal.h
@@ -76,9 +76,6 @@ struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask);
 int crypto_init_cipher_ops(struct crypto_tfm *tfm);
 int crypto_init_compress_ops(struct crypto_tfm *tfm);
 
-void crypto_exit_cipher_ops(struct crypto_tfm *tfm);
-void crypto_exit_compress_ops(struct crypto_tfm *tfm);
-
 struct crypto_larval *crypto_larval_alloc(const char *name, u32 type, u32 mask);
 void crypto_larval_kill(struct crypto_alg *alg);
 struct crypto_alg *crypto_larval_lookup(const char *name, u32 type, u32 mask);
diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c
index c4938497eedb..787dccca3715 100644
--- a/crypto/jitterentropy-kcapi.c
+++ b/crypto/jitterentropy-kcapi.c
@@ -39,7 +39,6 @@
 
 #include <linux/module.h>
 #include <linux/slab.h>
-#include <linux/module.h>
 #include <linux/fips.h>
 #include <linux/time.h>
 #include <linux/crypto.h>
diff --git a/crypto/lrw.c b/crypto/lrw.c
index 6f9908a7ebcb..ecd8474018e3 100644
--- a/crypto/lrw.c
+++ b/crypto/lrw.c
@@ -17,7 +17,8 @@
  *
  * The test vectors are included in the testing module tcrypt.[ch] */
 
-#include <crypto/algapi.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
@@ -29,11 +30,30 @@
 #include <crypto/gf128mul.h>
 #include <crypto/lrw.h>
 
+#define LRW_BUFFER_SIZE 128u
+
 struct priv {
-	struct crypto_cipher *child;
+	struct crypto_skcipher *child;
 	struct lrw_table_ctx table;
 };
 
+struct rctx {
+	be128 buf[LRW_BUFFER_SIZE / sizeof(be128)];
+
+	be128 t;
+
+	be128 *ext;
+
+	struct scatterlist srcbuf[2];
+	struct scatterlist dstbuf[2];
+	struct scatterlist *src;
+	struct scatterlist *dst;
+
+	unsigned int left;
+
+	struct skcipher_request subreq;
+};
+
 static inline void setbit128_bbe(void *b, int bit)
 {
 	__set_bit(bit ^ (0x80 -
@@ -76,32 +96,26 @@ void lrw_free_table(struct lrw_table_ctx *ctx)
 }
 EXPORT_SYMBOL_GPL(lrw_free_table);
 
-static int setkey(struct crypto_tfm *parent, const u8 *key,
+static int setkey(struct crypto_skcipher *parent, const u8 *key,
 		  unsigned int keylen)
 {
-	struct priv *ctx = crypto_tfm_ctx(parent);
-	struct crypto_cipher *child = ctx->child;
+	struct priv *ctx = crypto_skcipher_ctx(parent);
+	struct crypto_skcipher *child = ctx->child;
 	int err, bsize = LRW_BLOCK_SIZE;
 	const u8 *tweak = key + keylen - bsize;
 
-	crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
-	crypto_cipher_set_flags(child, crypto_tfm_get_flags(parent) &
-				       CRYPTO_TFM_REQ_MASK);
-	err = crypto_cipher_setkey(child, key, keylen - bsize);
+	crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
+					 CRYPTO_TFM_REQ_MASK);
+	err = crypto_skcipher_setkey(child, key, keylen - bsize);
+	crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) &
+					  CRYPTO_TFM_RES_MASK);
 	if (err)
 		return err;
-	crypto_tfm_set_flags(parent, crypto_cipher_get_flags(child) &
-				     CRYPTO_TFM_RES_MASK);
 
 	return lrw_init_table(&ctx->table, tweak);
 }
 
-struct sinfo {
-	be128 t;
-	struct crypto_tfm *tfm;
-	void (*fn)(struct crypto_tfm *, u8 *, const u8 *);
-};
-
 static inline void inc(be128 *iv)
 {
 	be64_add_cpu(&iv->b, 1);
@@ -109,13 +123,6 @@ static inline void inc(be128 *iv)
 		be64_add_cpu(&iv->a, 1);
 }
 
-static inline void lrw_round(struct sinfo *s, void *dst, const void *src)
-{
-	be128_xor(dst, &s->t, src);		/* PP <- T xor P */
-	s->fn(s->tfm, dst, dst);		/* CC <- E(Key2,PP) */
-	be128_xor(dst, dst, &s->t);		/* C <- T xor CC */
-}
-
 /* this returns the number of consequative 1 bits starting
  * from the right, get_index128(00 00 00 00 00 00 ... 00 00 10 FB) = 2 */
 static inline int get_index128(be128 *block)
@@ -135,83 +142,263 @@ static inline int get_index128(be128 *block)
 	return x;
 }
 
-static int crypt(struct blkcipher_desc *d,
-		 struct blkcipher_walk *w, struct priv *ctx,
-		 void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
+static int post_crypt(struct skcipher_request *req)
 {
+	struct rctx *rctx = skcipher_request_ctx(req);
+	be128 *buf = rctx->ext ?: rctx->buf;
+	struct skcipher_request *subreq;
+	const int bs = LRW_BLOCK_SIZE;
+	struct skcipher_walk w;
+	struct scatterlist *sg;
+	unsigned offset;
 	int err;
-	unsigned int avail;
+
+	subreq = &rctx->subreq;
+	err = skcipher_walk_virt(&w, subreq, false);
+
+	while (w.nbytes) {
+		unsigned int avail = w.nbytes;
+		be128 *wdst;
+
+		wdst = w.dst.virt.addr;
+
+		do {
+			be128_xor(wdst, buf++, wdst);
+			wdst++;
+		} while ((avail -= bs) >= bs);
+
+		err = skcipher_walk_done(&w, avail);
+	}
+
+	rctx->left -= subreq->cryptlen;
+
+	if (err || !rctx->left)
+		goto out;
+
+	rctx->dst = rctx->dstbuf;
+
+	scatterwalk_done(&w.out, 0, 1);
+	sg = w.out.sg;
+	offset = w.out.offset;
+
+	if (rctx->dst != sg) {
+		rctx->dst[0] = *sg;
+		sg_unmark_end(rctx->dst);
+		scatterwalk_crypto_chain(rctx->dst, sg_next(sg), 0, 2);
+	}
+	rctx->dst[0].length -= offset - sg->offset;
+	rctx->dst[0].offset = offset;
+
+out:
+	return err;
+}
+
+static int pre_crypt(struct skcipher_request *req)
+{
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct rctx *rctx = skcipher_request_ctx(req);
+	struct priv *ctx = crypto_skcipher_ctx(tfm);
+	be128 *buf = rctx->ext ?: rctx->buf;
+	struct skcipher_request *subreq;
 	const int bs = LRW_BLOCK_SIZE;
-	struct sinfo s = {
-		.tfm = crypto_cipher_tfm(ctx->child),
-		.fn = fn
-	};
+	struct skcipher_walk w;
+	struct scatterlist *sg;
+	unsigned cryptlen;
+	unsigned offset;
 	be128 *iv;
-	u8 *wsrc;
-	u8 *wdst;
+	bool more;
+	int err;
 
-	err = blkcipher_walk_virt(d, w);
-	if (!(avail = w->nbytes))
-		return err;
+	subreq = &rctx->subreq;
+	skcipher_request_set_tfm(subreq, tfm);
 
-	wsrc = w->src.virt.addr;
-	wdst = w->dst.virt.addr;
+	cryptlen = subreq->cryptlen;
+	more = rctx->left > cryptlen;
+	if (!more)
+		cryptlen = rctx->left;
 
-	/* calculate first value of T */
-	iv = (be128 *)w->iv;
-	s.t = *iv;
+	skcipher_request_set_crypt(subreq, rctx->src, rctx->dst,
+				   cryptlen, req->iv);
 
-	/* T <- I*Key2 */
-	gf128mul_64k_bbe(&s.t, ctx->table.table);
+	err = skcipher_walk_virt(&w, subreq, false);
+	iv = w.iv;
 
-	goto first;
+	while (w.nbytes) {
+		unsigned int avail = w.nbytes;
+		be128 *wsrc;
+		be128 *wdst;
+
+		wsrc = w.src.virt.addr;
+		wdst = w.dst.virt.addr;
 
-	for (;;) {
 		do {
+			*buf++ = rctx->t;
+			be128_xor(wdst++, &rctx->t, wsrc++);
+
 			/* T <- I*Key2, using the optimization
 			 * discussed in the specification */
-			be128_xor(&s.t, &s.t,
+			be128_xor(&rctx->t, &rctx->t,
 				  &ctx->table.mulinc[get_index128(iv)]);
 			inc(iv);
+		} while ((avail -= bs) >= bs);
 
-first:
-			lrw_round(&s, wdst, wsrc);
+		err = skcipher_walk_done(&w, avail);
+	}
 
-			wsrc += bs;
-			wdst += bs;
-		} while ((avail -= bs) >= bs);
+	skcipher_request_set_tfm(subreq, ctx->child);
+	skcipher_request_set_crypt(subreq, rctx->dst, rctx->dst,
+				   cryptlen, NULL);
 
-		err = blkcipher_walk_done(d, w, avail);
-		if (!(avail = w->nbytes))
-			break;
+	if (err || !more)
+		goto out;
+
+	rctx->src = rctx->srcbuf;
+
+	scatterwalk_done(&w.in, 0, 1);
+	sg = w.in.sg;
+	offset = w.in.offset;
+
+	if (rctx->src != sg) {
+		rctx->src[0] = *sg;
+		sg_unmark_end(rctx->src);
+		scatterwalk_crypto_chain(rctx->src, sg_next(sg), 0, 2);
+	}
+	rctx->src[0].length -= offset - sg->offset;
+	rctx->src[0].offset = offset;
+
+out:
+	return err;
+}
+
+static int init_crypt(struct skcipher_request *req, crypto_completion_t done)
+{
+	struct priv *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+	struct rctx *rctx = skcipher_request_ctx(req);
+	struct skcipher_request *subreq;
+	gfp_t gfp;
+
+	subreq = &rctx->subreq;
+	skcipher_request_set_callback(subreq, req->base.flags, done, req);
+
+	gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
+							   GFP_ATOMIC;
+	rctx->ext = NULL;
+
+	subreq->cryptlen = LRW_BUFFER_SIZE;
+	if (req->cryptlen > LRW_BUFFER_SIZE) {
+		subreq->cryptlen = min(req->cryptlen, (unsigned)PAGE_SIZE);
+		rctx->ext = kmalloc(subreq->cryptlen, gfp);
+	}
+
+	rctx->src = req->src;
+	rctx->dst = req->dst;
+	rctx->left = req->cryptlen;
+
+	/* calculate first value of T */
+	memcpy(&rctx->t, req->iv, sizeof(rctx->t));
+
+	/* T <- I*Key2 */
+	gf128mul_64k_bbe(&rctx->t, ctx->table.table);
 
-		wsrc = w->src.virt.addr;
-		wdst = w->dst.virt.addr;
+	return 0;
+}
+
+static void exit_crypt(struct skcipher_request *req)
+{
+	struct rctx *rctx = skcipher_request_ctx(req);
+
+	rctx->left = 0;
+
+	if (rctx->ext)
+		kfree(rctx->ext);
+}
+
+static int do_encrypt(struct skcipher_request *req, int err)
+{
+	struct rctx *rctx = skcipher_request_ctx(req);
+	struct skcipher_request *subreq;
+
+	subreq = &rctx->subreq;
+
+	while (!err && rctx->left) {
+		err = pre_crypt(req) ?:
+		      crypto_skcipher_encrypt(subreq) ?:
+		      post_crypt(req);
+
+		if (err == -EINPROGRESS ||
+		    (err == -EBUSY &&
+		     req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+			return err;
 	}
 
+	exit_crypt(req);
 	return err;
 }
 
-static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		   struct scatterlist *src, unsigned int nbytes)
+static void encrypt_done(struct crypto_async_request *areq, int err)
+{
+	struct skcipher_request *req = areq->data;
+	struct skcipher_request *subreq;
+	struct rctx *rctx;
+
+	rctx = skcipher_request_ctx(req);
+	subreq = &rctx->subreq;
+	subreq->base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;
+
+	err = do_encrypt(req, err ?: post_crypt(req));
+	if (rctx->left)
+		return;
+
+	skcipher_request_complete(req, err);
+}
+
+static int encrypt(struct skcipher_request *req)
+{
+	return do_encrypt(req, init_crypt(req, encrypt_done));
+}
+
+static int do_decrypt(struct skcipher_request *req, int err)
 {
-	struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk w;
+	struct rctx *rctx = skcipher_request_ctx(req);
+	struct skcipher_request *subreq;
+
+	subreq = &rctx->subreq;
+
+	while (!err && rctx->left) {
+		err = pre_crypt(req) ?:
+		      crypto_skcipher_decrypt(subreq) ?:
+		      post_crypt(req);
+
+		if (err == -EINPROGRESS ||
+		    (err == -EBUSY &&
+		     req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+			return err;
+	}
 
-	blkcipher_walk_init(&w, dst, src, nbytes);
-	return crypt(desc, &w, ctx,
-		     crypto_cipher_alg(ctx->child)->cia_encrypt);
+	exit_crypt(req);
+	return err;
 }
 
-static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		   struct scatterlist *src, unsigned int nbytes)
+static void decrypt_done(struct crypto_async_request *areq, int err)
 {
-	struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk w;
+	struct skcipher_request *req = areq->data;
+	struct skcipher_request *subreq;
+	struct rctx *rctx;
+
+	rctx = skcipher_request_ctx(req);
+	subreq = &rctx->subreq;
+	subreq->base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;
+
+	err = do_decrypt(req, err ?: post_crypt(req));
+	if (rctx->left)
+		return;
 
-	blkcipher_walk_init(&w, dst, src, nbytes);
-	return crypt(desc, &w, ctx,
-		     crypto_cipher_alg(ctx->child)->cia_decrypt);
+	skcipher_request_complete(req, err);
+}
+
+static int decrypt(struct skcipher_request *req)
+{
+	return do_decrypt(req, init_crypt(req, decrypt_done));
 }
 
 int lrw_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
@@ -293,95 +480,161 @@ first:
 }
 EXPORT_SYMBOL_GPL(lrw_crypt);
 
-static int init_tfm(struct crypto_tfm *tfm)
+static int init_tfm(struct crypto_skcipher *tfm)
 {
-	struct crypto_cipher *cipher;
-	struct crypto_instance *inst = (void *)tfm->__crt_alg;
-	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
-	struct priv *ctx = crypto_tfm_ctx(tfm);
-	u32 *flags = &tfm->crt_flags;
+	struct skcipher_instance *inst = skcipher_alg_instance(tfm);
+	struct crypto_skcipher_spawn *spawn = skcipher_instance_ctx(inst);
+	struct priv *ctx = crypto_skcipher_ctx(tfm);
+	struct crypto_skcipher *cipher;
 
-	cipher = crypto_spawn_cipher(spawn);
+	cipher = crypto_spawn_skcipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 
-	if (crypto_cipher_blocksize(cipher) != LRW_BLOCK_SIZE) {
-		*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
-		crypto_free_cipher(cipher);
-		return -EINVAL;
-	}
-
 	ctx->child = cipher;
+
+	crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(cipher) +
+					 sizeof(struct rctx));
+
 	return 0;
 }
 
-static void exit_tfm(struct crypto_tfm *tfm)
+static void exit_tfm(struct crypto_skcipher *tfm)
 {
-	struct priv *ctx = crypto_tfm_ctx(tfm);
+	struct priv *ctx = crypto_skcipher_ctx(tfm);
 
 	lrw_free_table(&ctx->table);
-	crypto_free_cipher(ctx->child);
+	crypto_free_skcipher(ctx->child);
+}
+
+static void free(struct skcipher_instance *inst)
+{
+	crypto_drop_skcipher(skcipher_instance_ctx(inst));
+	kfree(inst);
 }
 
-static struct crypto_instance *alloc(struct rtattr **tb)
+static int create(struct crypto_template *tmpl, struct rtattr **tb)
 {
-	struct crypto_instance *inst;
-	struct crypto_alg *alg;
+	struct crypto_skcipher_spawn *spawn;
+	struct skcipher_instance *inst;
+	struct crypto_attr_type *algt;
+	struct skcipher_alg *alg;
+	const char *cipher_name;
+	char ecb_name[CRYPTO_MAX_ALG_NAME];
 	int err;
 
-	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
+	algt = crypto_get_attr_type(tb);
+	if (IS_ERR(algt))
+		return PTR_ERR(algt);
+
+	if ((algt->type ^ CRYPTO_ALG_TYPE_SKCIPHER) & algt->mask)
+		return -EINVAL;
+
+	cipher_name = crypto_attr_alg_name(tb[1]);
+	if (IS_ERR(cipher_name))
+		return PTR_ERR(cipher_name);
+
+	inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
+	if (!inst)
+		return -ENOMEM;
+
+	spawn = skcipher_instance_ctx(inst);
+
+	crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst));
+	err = crypto_grab_skcipher(spawn, cipher_name, 0,
+				   crypto_requires_sync(algt->type,
+							algt->mask));
+	if (err == -ENOENT) {
+		err = -ENAMETOOLONG;
+		if (snprintf(ecb_name, CRYPTO_MAX_ALG_NAME, "ecb(%s)",
+			     cipher_name) >= CRYPTO_MAX_ALG_NAME)
+			goto err_free_inst;
+
+		err = crypto_grab_skcipher(spawn, ecb_name, 0,
+					   crypto_requires_sync(algt->type,
+								algt->mask));
+	}
+
 	if (err)
-		return ERR_PTR(err);
+		goto err_free_inst;
 
-	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
-				  CRYPTO_ALG_TYPE_MASK);
-	if (IS_ERR(alg))
-		return ERR_CAST(alg);
+	alg = crypto_skcipher_spawn_alg(spawn);
 
-	inst = crypto_alloc_instance("lrw", alg);
-	if (IS_ERR(inst))
-		goto out_put_alg;
+	err = -EINVAL;
+	if (alg->base.cra_blocksize != LRW_BLOCK_SIZE)
+		goto err_drop_spawn;
 
-	inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
-	inst->alg.cra_priority = alg->cra_priority;
-	inst->alg.cra_blocksize = alg->cra_blocksize;
+	if (crypto_skcipher_alg_ivsize(alg))
+		goto err_drop_spawn;
 
-	if (alg->cra_alignmask < 7) inst->alg.cra_alignmask = 7;
-	else inst->alg.cra_alignmask = alg->cra_alignmask;
-	inst->alg.cra_type = &crypto_blkcipher_type;
+	err = crypto_inst_setname(skcipher_crypto_instance(inst), "lrw",
+				  &alg->base);
+	if (err)
+		goto err_drop_spawn;
 
-	if (!(alg->cra_blocksize % 4))
-		inst->alg.cra_alignmask |= 3;
-	inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
-	inst->alg.cra_blkcipher.min_keysize =
-		alg->cra_cipher.cia_min_keysize + alg->cra_blocksize;
-	inst->alg.cra_blkcipher.max_keysize =
-		alg->cra_cipher.cia_max_keysize + alg->cra_blocksize;
+	err = -EINVAL;
+	cipher_name = alg->base.cra_name;
 
-	inst->alg.cra_ctxsize = sizeof(struct priv);
+	/* Alas we screwed up the naming so we have to mangle the
+	 * cipher name.
+	 */
+	if (!strncmp(cipher_name, "ecb(", 4)) {
+		unsigned len;
 
-	inst->alg.cra_init = init_tfm;
-	inst->alg.cra_exit = exit_tfm;
+		len = strlcpy(ecb_name, cipher_name + 4, sizeof(ecb_name));
+		if (len < 2 || len >= sizeof(ecb_name))
+			goto err_drop_spawn;
 
-	inst->alg.cra_blkcipher.setkey = setkey;
-	inst->alg.cra_blkcipher.encrypt = encrypt;
-	inst->alg.cra_blkcipher.decrypt = decrypt;
+		if (ecb_name[len - 1] != ')')
+			goto err_drop_spawn;
 
-out_put_alg:
-	crypto_mod_put(alg);
-	return inst;
-}
+		ecb_name[len - 1] = 0;
 
-static void free(struct crypto_instance *inst)
-{
-	crypto_drop_spawn(crypto_instance_ctx(inst));
+		if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
+			     "lrw(%s)", ecb_name) >= CRYPTO_MAX_ALG_NAME)
+			return -ENAMETOOLONG;
+	}
+
+	inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
+	inst->alg.base.cra_priority = alg->base.cra_priority;
+	inst->alg.base.cra_blocksize = LRW_BLOCK_SIZE;
+	inst->alg.base.cra_alignmask = alg->base.cra_alignmask |
+				       (__alignof__(u64) - 1);
+
+	inst->alg.ivsize = LRW_BLOCK_SIZE;
+	inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg) +
+				LRW_BLOCK_SIZE;
+	inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg) +
+				LRW_BLOCK_SIZE;
+
+	inst->alg.base.cra_ctxsize = sizeof(struct priv);
+
+	inst->alg.init = init_tfm;
+	inst->alg.exit = exit_tfm;
+
+	inst->alg.setkey = setkey;
+	inst->alg.encrypt = encrypt;
+	inst->alg.decrypt = decrypt;
+
+	inst->free = free;
+
+	err = skcipher_register_instance(tmpl, inst);
+	if (err)
+		goto err_drop_spawn;
+
+out:
+	return err;
+
+err_drop_spawn:
+	crypto_drop_skcipher(spawn);
+err_free_inst:
 	kfree(inst);
+	goto out;
 }
 
 static struct crypto_template crypto_tmpl = {
 	.name = "lrw",
-	.alloc = alloc,
-	.free = free,
+	.create = create,
 	.module = THIS_MODULE,
 };
 
diff --git a/crypto/lz4.c b/crypto/lz4.c
index aefbceaf3104..99c1b2cc2976 100644
--- a/crypto/lz4.c
+++ b/crypto/lz4.c
@@ -23,36 +23,53 @@
 #include <linux/crypto.h>
 #include <linux/vmalloc.h>
 #include <linux/lz4.h>
+#include <crypto/internal/scompress.h>
 
 struct lz4_ctx {
 	void *lz4_comp_mem;
 };
 
+static void *lz4_alloc_ctx(struct crypto_scomp *tfm)
+{
+	void *ctx;
+
+	ctx = vmalloc(LZ4_MEM_COMPRESS);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	return ctx;
+}
+
 static int lz4_init(struct crypto_tfm *tfm)
 {
 	struct lz4_ctx *ctx = crypto_tfm_ctx(tfm);
 
-	ctx->lz4_comp_mem = vmalloc(LZ4_MEM_COMPRESS);
-	if (!ctx->lz4_comp_mem)
+	ctx->lz4_comp_mem = lz4_alloc_ctx(NULL);
+	if (IS_ERR(ctx->lz4_comp_mem))
 		return -ENOMEM;
 
 	return 0;
 }
 
+static void lz4_free_ctx(struct crypto_scomp *tfm, void *ctx)
+{
+	vfree(ctx);
+}
+
 static void lz4_exit(struct crypto_tfm *tfm)
 {
 	struct lz4_ctx *ctx = crypto_tfm_ctx(tfm);
-	vfree(ctx->lz4_comp_mem);
+
+	lz4_free_ctx(NULL, ctx->lz4_comp_mem);
 }
 
-static int lz4_compress_crypto(struct crypto_tfm *tfm, const u8 *src,
-			    unsigned int slen, u8 *dst, unsigned int *dlen)
+static int __lz4_compress_crypto(const u8 *src, unsigned int slen,
+				 u8 *dst, unsigned int *dlen, void *ctx)
 {
-	struct lz4_ctx *ctx = crypto_tfm_ctx(tfm);
 	size_t tmp_len = *dlen;
 	int err;
 
-	err = lz4_compress(src, slen, dst, &tmp_len, ctx->lz4_comp_mem);
+	err = lz4_compress(src, slen, dst, &tmp_len, ctx);
 
 	if (err < 0)
 		return -EINVAL;
@@ -61,8 +78,23 @@ static int lz4_compress_crypto(struct crypto_tfm *tfm, const u8 *src,
 	return 0;
 }
 
-static int lz4_decompress_crypto(struct crypto_tfm *tfm, const u8 *src,
-			      unsigned int slen, u8 *dst, unsigned int *dlen)
+static int lz4_scompress(struct crypto_scomp *tfm, const u8 *src,
+			 unsigned int slen, u8 *dst, unsigned int *dlen,
+			 void *ctx)
+{
+	return __lz4_compress_crypto(src, slen, dst, dlen, ctx);
+}
+
+static int lz4_compress_crypto(struct crypto_tfm *tfm, const u8 *src,
+			       unsigned int slen, u8 *dst, unsigned int *dlen)
+{
+	struct lz4_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	return __lz4_compress_crypto(src, slen, dst, dlen, ctx->lz4_comp_mem);
+}
+
+static int __lz4_decompress_crypto(const u8 *src, unsigned int slen,
+				   u8 *dst, unsigned int *dlen, void *ctx)
 {
 	int err;
 	size_t tmp_len = *dlen;
@@ -76,6 +108,20 @@ static int lz4_decompress_crypto(struct crypto_tfm *tfm, const u8 *src,
 	return err;
 }
 
+static int lz4_sdecompress(struct crypto_scomp *tfm, const u8 *src,
+			   unsigned int slen, u8 *dst, unsigned int *dlen,
+			   void *ctx)
+{
+	return __lz4_decompress_crypto(src, slen, dst, dlen, NULL);
+}
+
+static int lz4_decompress_crypto(struct crypto_tfm *tfm, const u8 *src,
+				 unsigned int slen, u8 *dst,
+				 unsigned int *dlen)
+{
+	return __lz4_decompress_crypto(src, slen, dst, dlen, NULL);
+}
+
 static struct crypto_alg alg_lz4 = {
 	.cra_name		= "lz4",
 	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
@@ -89,14 +135,39 @@ static struct crypto_alg alg_lz4 = {
 	.coa_decompress		= lz4_decompress_crypto } }
 };
 
+static struct scomp_alg scomp = {
+	.alloc_ctx		= lz4_alloc_ctx,
+	.free_ctx		= lz4_free_ctx,
+	.compress		= lz4_scompress,
+	.decompress		= lz4_sdecompress,
+	.base			= {
+		.cra_name	= "lz4",
+		.cra_driver_name = "lz4-scomp",
+		.cra_module	 = THIS_MODULE,
+	}
+};
+
 static int __init lz4_mod_init(void)
 {
-	return crypto_register_alg(&alg_lz4);
+	int ret;
+
+	ret = crypto_register_alg(&alg_lz4);
+	if (ret)
+		return ret;
+
+	ret = crypto_register_scomp(&scomp);
+	if (ret) {
+		crypto_unregister_alg(&alg_lz4);
+		return ret;
+	}
+
+	return ret;
 }
 
 static void __exit lz4_mod_fini(void)
 {
 	crypto_unregister_alg(&alg_lz4);
+	crypto_unregister_scomp(&scomp);
 }
 
 module_init(lz4_mod_init);
diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c
index a1d3b5bd3d85..75ffc4a3f786 100644
--- a/crypto/lz4hc.c
+++ b/crypto/lz4hc.c
@@ -22,37 +22,53 @@
 #include <linux/crypto.h>
 #include <linux/vmalloc.h>
 #include <linux/lz4.h>
+#include <crypto/internal/scompress.h>
 
 struct lz4hc_ctx {
 	void *lz4hc_comp_mem;
 };
 
+static void *lz4hc_alloc_ctx(struct crypto_scomp *tfm)
+{
+	void *ctx;
+
+	ctx = vmalloc(LZ4HC_MEM_COMPRESS);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	return ctx;
+}
+
 static int lz4hc_init(struct crypto_tfm *tfm)
 {
 	struct lz4hc_ctx *ctx = crypto_tfm_ctx(tfm);
 
-	ctx->lz4hc_comp_mem = vmalloc(LZ4HC_MEM_COMPRESS);
-	if (!ctx->lz4hc_comp_mem)
+	ctx->lz4hc_comp_mem = lz4hc_alloc_ctx(NULL);
+	if (IS_ERR(ctx->lz4hc_comp_mem))
 		return -ENOMEM;
 
 	return 0;
 }
 
+static void lz4hc_free_ctx(struct crypto_scomp *tfm, void *ctx)
+{
+	vfree(ctx);
+}
+
 static void lz4hc_exit(struct crypto_tfm *tfm)
 {
 	struct lz4hc_ctx *ctx = crypto_tfm_ctx(tfm);
 
-	vfree(ctx->lz4hc_comp_mem);
+	lz4hc_free_ctx(NULL, ctx->lz4hc_comp_mem);
 }
 
-static int lz4hc_compress_crypto(struct crypto_tfm *tfm, const u8 *src,
-			    unsigned int slen, u8 *dst, unsigned int *dlen)
+static int __lz4hc_compress_crypto(const u8 *src, unsigned int slen,
+				   u8 *dst, unsigned int *dlen, void *ctx)
 {
-	struct lz4hc_ctx *ctx = crypto_tfm_ctx(tfm);
 	size_t tmp_len = *dlen;
 	int err;
 
-	err = lz4hc_compress(src, slen, dst, &tmp_len, ctx->lz4hc_comp_mem);
+	err = lz4hc_compress(src, slen, dst, &tmp_len, ctx);
 
 	if (err < 0)
 		return -EINVAL;
@@ -61,8 +77,25 @@ static int lz4hc_compress_crypto(struct crypto_tfm *tfm, const u8 *src,
 	return 0;
 }
 
-static int lz4hc_decompress_crypto(struct crypto_tfm *tfm, const u8 *src,
-			      unsigned int slen, u8 *dst, unsigned int *dlen)
+static int lz4hc_scompress(struct crypto_scomp *tfm, const u8 *src,
+			   unsigned int slen, u8 *dst, unsigned int *dlen,
+			   void *ctx)
+{
+	return __lz4hc_compress_crypto(src, slen, dst, dlen, ctx);
+}
+
+static int lz4hc_compress_crypto(struct crypto_tfm *tfm, const u8 *src,
+				 unsigned int slen, u8 *dst,
+				 unsigned int *dlen)
+{
+	struct lz4hc_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	return __lz4hc_compress_crypto(src, slen, dst, dlen,
+					ctx->lz4hc_comp_mem);
+}
+
+static int __lz4hc_decompress_crypto(const u8 *src, unsigned int slen,
+				     u8 *dst, unsigned int *dlen, void *ctx)
 {
 	int err;
 	size_t tmp_len = *dlen;
@@ -76,6 +109,20 @@ static int lz4hc_decompress_crypto(struct crypto_tfm *tfm, const u8 *src,
 	return err;
 }
 
+static int lz4hc_sdecompress(struct crypto_scomp *tfm, const u8 *src,
+			     unsigned int slen, u8 *dst, unsigned int *dlen,
+			     void *ctx)
+{
+	return __lz4hc_decompress_crypto(src, slen, dst, dlen, NULL);
+}
+
+static int lz4hc_decompress_crypto(struct crypto_tfm *tfm, const u8 *src,
+				   unsigned int slen, u8 *dst,
+				   unsigned int *dlen)
+{
+	return __lz4hc_decompress_crypto(src, slen, dst, dlen, NULL);
+}
+
 static struct crypto_alg alg_lz4hc = {
 	.cra_name		= "lz4hc",
 	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
@@ -89,14 +136,39 @@ static struct crypto_alg alg_lz4hc = {
 	.coa_decompress		= lz4hc_decompress_crypto } }
 };
 
+static struct scomp_alg scomp = {
+	.alloc_ctx		= lz4hc_alloc_ctx,
+	.free_ctx		= lz4hc_free_ctx,
+	.compress		= lz4hc_scompress,
+	.decompress		= lz4hc_sdecompress,
+	.base			= {
+		.cra_name	= "lz4hc",
+		.cra_driver_name = "lz4hc-scomp",
+		.cra_module	 = THIS_MODULE,
+	}
+};
+
 static int __init lz4hc_mod_init(void)
 {
-	return crypto_register_alg(&alg_lz4hc);
+	int ret;
+
+	ret = crypto_register_alg(&alg_lz4hc);
+	if (ret)
+		return ret;
+
+	ret = crypto_register_scomp(&scomp);
+	if (ret) {
+		crypto_unregister_alg(&alg_lz4hc);
+		return ret;
+	}
+
+	return ret;
 }
 
 static void __exit lz4hc_mod_fini(void)
 {
 	crypto_unregister_alg(&alg_lz4hc);
+	crypto_unregister_scomp(&scomp);
 }
 
 module_init(lz4hc_mod_init);
diff --git a/crypto/lzo.c b/crypto/lzo.c
index c3f3dd9a28c5..168df784da84 100644
--- a/crypto/lzo.c
+++ b/crypto/lzo.c
@@ -22,40 +22,55 @@
 #include <linux/vmalloc.h>
 #include <linux/mm.h>
 #include <linux/lzo.h>
+#include <crypto/internal/scompress.h>
 
 struct lzo_ctx {
 	void *lzo_comp_mem;
 };
 
+static void *lzo_alloc_ctx(struct crypto_scomp *tfm)
+{
+	void *ctx;
+
+	ctx = kmalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL | __GFP_NOWARN);
+	if (!ctx)
+		ctx = vmalloc(LZO1X_MEM_COMPRESS);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	return ctx;
+}
+
 static int lzo_init(struct crypto_tfm *tfm)
 {
 	struct lzo_ctx *ctx = crypto_tfm_ctx(tfm);
 
-	ctx->lzo_comp_mem = kmalloc(LZO1X_MEM_COMPRESS,
-				    GFP_KERNEL | __GFP_NOWARN);
-	if (!ctx->lzo_comp_mem)
-		ctx->lzo_comp_mem = vmalloc(LZO1X_MEM_COMPRESS);
-	if (!ctx->lzo_comp_mem)
+	ctx->lzo_comp_mem = lzo_alloc_ctx(NULL);
+	if (IS_ERR(ctx->lzo_comp_mem))
 		return -ENOMEM;
 
 	return 0;
 }
 
+static void lzo_free_ctx(struct crypto_scomp *tfm, void *ctx)
+{
+	kvfree(ctx);
+}
+
 static void lzo_exit(struct crypto_tfm *tfm)
 {
 	struct lzo_ctx *ctx = crypto_tfm_ctx(tfm);
 
-	kvfree(ctx->lzo_comp_mem);
+	lzo_free_ctx(NULL, ctx->lzo_comp_mem);
 }
 
-static int lzo_compress(struct crypto_tfm *tfm, const u8 *src,
-			    unsigned int slen, u8 *dst, unsigned int *dlen)
+static int __lzo_compress(const u8 *src, unsigned int slen,
+			  u8 *dst, unsigned int *dlen, void *ctx)
 {
-	struct lzo_ctx *ctx = crypto_tfm_ctx(tfm);
 	size_t tmp_len = *dlen; /* size_t(ulong) <-> uint on 64 bit */
 	int err;
 
-	err = lzo1x_1_compress(src, slen, dst, &tmp_len, ctx->lzo_comp_mem);
+	err = lzo1x_1_compress(src, slen, dst, &tmp_len, ctx);
 
 	if (err != LZO_E_OK)
 		return -EINVAL;
@@ -64,8 +79,23 @@ static int lzo_compress(struct crypto_tfm *tfm, const u8 *src,
 	return 0;
 }
 
-static int lzo_decompress(struct crypto_tfm *tfm, const u8 *src,
-			      unsigned int slen, u8 *dst, unsigned int *dlen)
+static int lzo_compress(struct crypto_tfm *tfm, const u8 *src,
+			unsigned int slen, u8 *dst, unsigned int *dlen)
+{
+	struct lzo_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	return __lzo_compress(src, slen, dst, dlen, ctx->lzo_comp_mem);
+}
+
+static int lzo_scompress(struct crypto_scomp *tfm, const u8 *src,
+			 unsigned int slen, u8 *dst, unsigned int *dlen,
+			 void *ctx)
+{
+	return __lzo_compress(src, slen, dst, dlen, ctx);
+}
+
+static int __lzo_decompress(const u8 *src, unsigned int slen,
+			    u8 *dst, unsigned int *dlen)
 {
 	int err;
 	size_t tmp_len = *dlen; /* size_t(ulong) <-> uint on 64 bit */
@@ -77,7 +107,19 @@ static int lzo_decompress(struct crypto_tfm *tfm, const u8 *src,
 
 	*dlen = tmp_len;
 	return 0;
+}
 
+static int lzo_decompress(struct crypto_tfm *tfm, const u8 *src,
+			  unsigned int slen, u8 *dst, unsigned int *dlen)
+{
+	return __lzo_decompress(src, slen, dst, dlen);
+}
+
+static int lzo_sdecompress(struct crypto_scomp *tfm, const u8 *src,
+			   unsigned int slen, u8 *dst, unsigned int *dlen,
+			   void *ctx)
+{
+	return __lzo_decompress(src, slen, dst, dlen);
 }
 
 static struct crypto_alg alg = {
@@ -88,18 +130,43 @@ static struct crypto_alg alg = {
 	.cra_init		= lzo_init,
 	.cra_exit		= lzo_exit,
 	.cra_u			= { .compress = {
-	.coa_compress 		= lzo_compress,
-	.coa_decompress  	= lzo_decompress } }
+	.coa_compress		= lzo_compress,
+	.coa_decompress		= lzo_decompress } }
+};
+
+static struct scomp_alg scomp = {
+	.alloc_ctx		= lzo_alloc_ctx,
+	.free_ctx		= lzo_free_ctx,
+	.compress		= lzo_scompress,
+	.decompress		= lzo_sdecompress,
+	.base			= {
+		.cra_name	= "lzo",
+		.cra_driver_name = "lzo-scomp",
+		.cra_module	 = THIS_MODULE,
+	}
 };
 
 static int __init lzo_mod_init(void)
 {
-	return crypto_register_alg(&alg);
+	int ret;
+
+	ret = crypto_register_alg(&alg);
+	if (ret)
+		return ret;
+
+	ret = crypto_register_scomp(&scomp);
+	if (ret) {
+		crypto_unregister_alg(&alg);
+		return ret;
+	}
+
+	return ret;
 }
 
 static void __exit lzo_mod_fini(void)
 {
 	crypto_unregister_alg(&alg);
+	crypto_unregister_scomp(&scomp);
 }
 
 module_init(lzo_mod_init);
diff --git a/crypto/pcbc.c b/crypto/pcbc.c
index f654965f0933..e4538e07f7ca 100644
--- a/crypto/pcbc.c
+++ b/crypto/pcbc.c
@@ -14,40 +14,37 @@
  *
  */
 
-#include <crypto/algapi.h>
+#include <crypto/internal/skcipher.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/scatterlist.h>
 #include <linux/slab.h>
 
 struct crypto_pcbc_ctx {
 	struct crypto_cipher *child;
 };
 
-static int crypto_pcbc_setkey(struct crypto_tfm *parent, const u8 *key,
+static int crypto_pcbc_setkey(struct crypto_skcipher *parent, const u8 *key,
 			      unsigned int keylen)
 {
-	struct crypto_pcbc_ctx *ctx = crypto_tfm_ctx(parent);
+	struct crypto_pcbc_ctx *ctx = crypto_skcipher_ctx(parent);
 	struct crypto_cipher *child = ctx->child;
 	int err;
 
 	crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
-	crypto_cipher_set_flags(child, crypto_tfm_get_flags(parent) &
-				CRYPTO_TFM_REQ_MASK);
+	crypto_cipher_set_flags(child, crypto_skcipher_get_flags(parent) &
+				       CRYPTO_TFM_REQ_MASK);
 	err = crypto_cipher_setkey(child, key, keylen);
-	crypto_tfm_set_flags(parent, crypto_cipher_get_flags(child) &
-			     CRYPTO_TFM_RES_MASK);
+	crypto_skcipher_set_flags(parent, crypto_cipher_get_flags(child) &
+					  CRYPTO_TFM_RES_MASK);
 	return err;
 }
 
-static int crypto_pcbc_encrypt_segment(struct blkcipher_desc *desc,
-				       struct blkcipher_walk *walk,
+static int crypto_pcbc_encrypt_segment(struct skcipher_request *req,
+				       struct skcipher_walk *walk,
 				       struct crypto_cipher *tfm)
 {
-	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
-		crypto_cipher_alg(tfm)->cia_encrypt;
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
@@ -56,7 +53,7 @@ static int crypto_pcbc_encrypt_segment(struct blkcipher_desc *desc,
 
 	do {
 		crypto_xor(iv, src, bsize);
-		fn(crypto_cipher_tfm(tfm), dst, iv);
+		crypto_cipher_encrypt_one(tfm, dst, iv);
 		memcpy(iv, dst, bsize);
 		crypto_xor(iv, src, bsize);
 
@@ -67,12 +64,10 @@ static int crypto_pcbc_encrypt_segment(struct blkcipher_desc *desc,
 	return nbytes;
 }
 
-static int crypto_pcbc_encrypt_inplace(struct blkcipher_desc *desc,
-				       struct blkcipher_walk *walk,
+static int crypto_pcbc_encrypt_inplace(struct skcipher_request *req,
+				       struct skcipher_walk *walk,
 				       struct crypto_cipher *tfm)
 {
-	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
-		crypto_cipher_alg(tfm)->cia_encrypt;
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
@@ -82,7 +77,7 @@ static int crypto_pcbc_encrypt_inplace(struct blkcipher_desc *desc,
 	do {
 		memcpy(tmpbuf, src, bsize);
 		crypto_xor(iv, src, bsize);
-		fn(crypto_cipher_tfm(tfm), src, iv);
+		crypto_cipher_encrypt_one(tfm, src, iv);
 		memcpy(iv, tmpbuf, bsize);
 		crypto_xor(iv, src, bsize);
 
@@ -94,38 +89,34 @@ static int crypto_pcbc_encrypt_inplace(struct blkcipher_desc *desc,
 	return nbytes;
 }
 
-static int crypto_pcbc_encrypt(struct blkcipher_desc *desc,
-			       struct scatterlist *dst, struct scatterlist *src,
-			       unsigned int nbytes)
+static int crypto_pcbc_encrypt(struct skcipher_request *req)
 {
-	struct blkcipher_walk walk;
-	struct crypto_blkcipher *tfm = desc->tfm;
-	struct crypto_pcbc_ctx *ctx = crypto_blkcipher_ctx(tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_pcbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 	struct crypto_cipher *child = ctx->child;
+	struct skcipher_walk walk;
+	unsigned int nbytes;
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, false);
 
 	while ((nbytes = walk.nbytes)) {
 		if (walk.src.virt.addr == walk.dst.virt.addr)
-			nbytes = crypto_pcbc_encrypt_inplace(desc, &walk,
+			nbytes = crypto_pcbc_encrypt_inplace(req, &walk,
 							     child);
 		else
-			nbytes = crypto_pcbc_encrypt_segment(desc, &walk,
+			nbytes = crypto_pcbc_encrypt_segment(req, &walk,
 							     child);
-		err = blkcipher_walk_done(desc, &walk, nbytes);
+		err = skcipher_walk_done(&walk, nbytes);
 	}
 
 	return err;
 }
 
-static int crypto_pcbc_decrypt_segment(struct blkcipher_desc *desc,
-				       struct blkcipher_walk *walk,
+static int crypto_pcbc_decrypt_segment(struct skcipher_request *req,
+				       struct skcipher_walk *walk,
 				       struct crypto_cipher *tfm)
 {
-	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
-		crypto_cipher_alg(tfm)->cia_decrypt;
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
@@ -133,7 +124,7 @@ static int crypto_pcbc_decrypt_segment(struct blkcipher_desc *desc,
 	u8 *iv = walk->iv;
 
 	do {
-		fn(crypto_cipher_tfm(tfm), dst, src);
+		crypto_cipher_decrypt_one(tfm, dst, src);
 		crypto_xor(dst, iv, bsize);
 		memcpy(iv, src, bsize);
 		crypto_xor(iv, dst, bsize);
@@ -147,21 +138,19 @@ static int crypto_pcbc_decrypt_segment(struct blkcipher_desc *desc,
 	return nbytes;
 }
 
-static int crypto_pcbc_decrypt_inplace(struct blkcipher_desc *desc,
-				       struct blkcipher_walk *walk,
+static int crypto_pcbc_decrypt_inplace(struct skcipher_request *req,
+				       struct skcipher_walk *walk,
 				       struct crypto_cipher *tfm)
 {
-	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
-		crypto_cipher_alg(tfm)->cia_decrypt;
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 *iv = walk->iv;
-	u8 tmpbuf[bsize];
+	u8 tmpbuf[bsize] __attribute__ ((aligned(__alignof__(u32))));
 
 	do {
 		memcpy(tmpbuf, src, bsize);
-		fn(crypto_cipher_tfm(tfm), src, src);
+		crypto_cipher_decrypt_one(tfm, src, src);
 		crypto_xor(src, iv, bsize);
 		memcpy(iv, tmpbuf, bsize);
 		crypto_xor(iv, src, bsize);
@@ -174,37 +163,35 @@ static int crypto_pcbc_decrypt_inplace(struct blkcipher_desc *desc,
 	return nbytes;
 }
 
-static int crypto_pcbc_decrypt(struct blkcipher_desc *desc,
-			       struct scatterlist *dst, struct scatterlist *src,
-			       unsigned int nbytes)
+static int crypto_pcbc_decrypt(struct skcipher_request *req)
 {
-	struct blkcipher_walk walk;
-	struct crypto_blkcipher *tfm = desc->tfm;
-	struct crypto_pcbc_ctx *ctx = crypto_blkcipher_ctx(tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct crypto_pcbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 	struct crypto_cipher *child = ctx->child;
+	struct skcipher_walk walk;
+	unsigned int nbytes;
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt(desc, &walk);
+	err = skcipher_walk_virt(&walk, req, false);
 
 	while ((nbytes = walk.nbytes)) {
 		if (walk.src.virt.addr == walk.dst.virt.addr)
-			nbytes = crypto_pcbc_decrypt_inplace(desc, &walk,
+			nbytes = crypto_pcbc_decrypt_inplace(req, &walk,
 							     child);
 		else
-			nbytes = crypto_pcbc_decrypt_segment(desc, &walk,
+			nbytes = crypto_pcbc_decrypt_segment(req, &walk,
 							     child);
-		err = blkcipher_walk_done(desc, &walk, nbytes);
+		err = skcipher_walk_done(&walk, nbytes);
 	}
 
 	return err;
 }
 
-static int crypto_pcbc_init_tfm(struct crypto_tfm *tfm)
+static int crypto_pcbc_init_tfm(struct crypto_skcipher *tfm)
 {
-	struct crypto_instance *inst = (void *)tfm->__crt_alg;
-	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
-	struct crypto_pcbc_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct skcipher_instance *inst = skcipher_alg_instance(tfm);
+	struct crypto_spawn *spawn = skcipher_instance_ctx(inst);
+	struct crypto_pcbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 	struct crypto_cipher *cipher;
 
 	cipher = crypto_spawn_cipher(spawn);
@@ -215,68 +202,98 @@ static int crypto_pcbc_init_tfm(struct crypto_tfm *tfm)
 	return 0;
 }
 
-static void crypto_pcbc_exit_tfm(struct crypto_tfm *tfm)
+static void crypto_pcbc_exit_tfm(struct crypto_skcipher *tfm)
 {
-	struct crypto_pcbc_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto_pcbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+
 	crypto_free_cipher(ctx->child);
 }
 
-static struct crypto_instance *crypto_pcbc_alloc(struct rtattr **tb)
+static void crypto_pcbc_free(struct skcipher_instance *inst)
+{
+	crypto_drop_skcipher(skcipher_instance_ctx(inst));
+	kfree(inst);
+}
+
+static int crypto_pcbc_create(struct crypto_template *tmpl, struct rtattr **tb)
 {
-	struct crypto_instance *inst;
+	struct skcipher_instance *inst;
+	struct crypto_attr_type *algt;
+	struct crypto_spawn *spawn;
 	struct crypto_alg *alg;
 	int err;
 
-	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
-	if (err)
-		return ERR_PTR(err);
+	algt = crypto_get_attr_type(tb);
+	if (IS_ERR(algt))
+		return PTR_ERR(algt);
+
+	if (((algt->type ^ CRYPTO_ALG_TYPE_SKCIPHER) & algt->mask) &
+	    ~CRYPTO_ALG_INTERNAL)
+		return -EINVAL;
 
-	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
-				  CRYPTO_ALG_TYPE_MASK);
+	inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
+	if (!inst)
+		return -ENOMEM;
+
+	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER |
+				      (algt->type & CRYPTO_ALG_INTERNAL),
+				  CRYPTO_ALG_TYPE_MASK |
+				  (algt->mask & CRYPTO_ALG_INTERNAL));
+	err = PTR_ERR(alg);
 	if (IS_ERR(alg))
-		return ERR_CAST(alg);
+		goto err_free_inst;
+
+	spawn = skcipher_instance_ctx(inst);
+	err = crypto_init_spawn(spawn, alg, skcipher_crypto_instance(inst),
+				CRYPTO_ALG_TYPE_MASK);
+	crypto_mod_put(alg);
+	if (err)
+		goto err_free_inst;
 
-	inst = crypto_alloc_instance("pcbc", alg);
-	if (IS_ERR(inst))
-		goto out_put_alg;
+	err = crypto_inst_setname(skcipher_crypto_instance(inst), "pcbc", alg);
+	if (err)
+		goto err_drop_spawn;
 
-	inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
-	inst->alg.cra_priority = alg->cra_priority;
-	inst->alg.cra_blocksize = alg->cra_blocksize;
-	inst->alg.cra_alignmask = alg->cra_alignmask;
-	inst->alg.cra_type = &crypto_blkcipher_type;
+	inst->alg.base.cra_flags = alg->cra_flags & CRYPTO_ALG_INTERNAL;
+	inst->alg.base.cra_priority = alg->cra_priority;
+	inst->alg.base.cra_blocksize = alg->cra_blocksize;
+	inst->alg.base.cra_alignmask = alg->cra_alignmask;
 
 	/* We access the data as u32s when xoring. */
-	inst->alg.cra_alignmask |= __alignof__(u32) - 1;
+	inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
 
-	inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
-	inst->alg.cra_blkcipher.min_keysize = alg->cra_cipher.cia_min_keysize;
-	inst->alg.cra_blkcipher.max_keysize = alg->cra_cipher.cia_max_keysize;
+	inst->alg.ivsize = alg->cra_blocksize;
+	inst->alg.min_keysize = alg->cra_cipher.cia_min_keysize;
+	inst->alg.max_keysize = alg->cra_cipher.cia_max_keysize;
 
-	inst->alg.cra_ctxsize = sizeof(struct crypto_pcbc_ctx);
+	inst->alg.base.cra_ctxsize = sizeof(struct crypto_pcbc_ctx);
 
-	inst->alg.cra_init = crypto_pcbc_init_tfm;
-	inst->alg.cra_exit = crypto_pcbc_exit_tfm;
+	inst->alg.init = crypto_pcbc_init_tfm;
+	inst->alg.exit = crypto_pcbc_exit_tfm;
 
-	inst->alg.cra_blkcipher.setkey = crypto_pcbc_setkey;
-	inst->alg.cra_blkcipher.encrypt = crypto_pcbc_encrypt;
-	inst->alg.cra_blkcipher.decrypt = crypto_pcbc_decrypt;
+	inst->alg.setkey = crypto_pcbc_setkey;
+	inst->alg.encrypt = crypto_pcbc_encrypt;
+	inst->alg.decrypt = crypto_pcbc_decrypt;
 
-out_put_alg:
-	crypto_mod_put(alg);
-	return inst;
-}
+	inst->free = crypto_pcbc_free;
 
-static void crypto_pcbc_free(struct crypto_instance *inst)
-{
-	crypto_drop_spawn(crypto_instance_ctx(inst));
+	err = skcipher_register_instance(tmpl, inst);
+	if (err)
+		goto err_drop_spawn;
+
+out:
+	return err;
+
+err_drop_spawn:
+	crypto_drop_spawn(spawn);
+err_free_inst:
 	kfree(inst);
+	goto out;
 }
 
 static struct crypto_template crypto_pcbc_tmpl = {
 	.name = "pcbc",
-	.alloc = crypto_pcbc_alloc,
-	.free = crypto_pcbc_free,
+	.create = crypto_pcbc_create,
 	.module = THIS_MODULE,
 };
 
diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c
index 2df9835dfbc0..b1c2d57dc734 100644
--- a/crypto/poly1305_generic.c
+++ b/crypto/poly1305_generic.c
@@ -17,6 +17,7 @@
 #include <linux/crypto.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <asm/unaligned.h>
 
 static inline u64 mlt(u64 a, u64 b)
 {
@@ -33,11 +34,6 @@ static inline u32 and(u32 v, u32 mask)
 	return v & mask;
 }
 
-static inline u32 le32_to_cpuvp(const void *p)
-{
-	return le32_to_cpup(p);
-}
-
 int crypto_poly1305_init(struct shash_desc *desc)
 {
 	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
@@ -65,19 +61,19 @@ EXPORT_SYMBOL_GPL(crypto_poly1305_setkey);
 static void poly1305_setrkey(struct poly1305_desc_ctx *dctx, const u8 *key)
 {
 	/* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
-	dctx->r[0] = (le32_to_cpuvp(key +  0) >> 0) & 0x3ffffff;
-	dctx->r[1] = (le32_to_cpuvp(key +  3) >> 2) & 0x3ffff03;
-	dctx->r[2] = (le32_to_cpuvp(key +  6) >> 4) & 0x3ffc0ff;
-	dctx->r[3] = (le32_to_cpuvp(key +  9) >> 6) & 0x3f03fff;
-	dctx->r[4] = (le32_to_cpuvp(key + 12) >> 8) & 0x00fffff;
+	dctx->r[0] = (get_unaligned_le32(key +  0) >> 0) & 0x3ffffff;
+	dctx->r[1] = (get_unaligned_le32(key +  3) >> 2) & 0x3ffff03;
+	dctx->r[2] = (get_unaligned_le32(key +  6) >> 4) & 0x3ffc0ff;
+	dctx->r[3] = (get_unaligned_le32(key +  9) >> 6) & 0x3f03fff;
+	dctx->r[4] = (get_unaligned_le32(key + 12) >> 8) & 0x00fffff;
 }
 
 static void poly1305_setskey(struct poly1305_desc_ctx *dctx, const u8 *key)
 {
-	dctx->s[0] = le32_to_cpuvp(key +  0);
-	dctx->s[1] = le32_to_cpuvp(key +  4);
-	dctx->s[2] = le32_to_cpuvp(key +  8);
-	dctx->s[3] = le32_to_cpuvp(key + 12);
+	dctx->s[0] = get_unaligned_le32(key +  0);
+	dctx->s[1] = get_unaligned_le32(key +  4);
+	dctx->s[2] = get_unaligned_le32(key +  8);
+	dctx->s[3] = get_unaligned_le32(key + 12);
 }
 
 unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx,
@@ -137,11 +133,11 @@ static unsigned int poly1305_blocks(struct poly1305_desc_ctx *dctx,
 	while (likely(srclen >= POLY1305_BLOCK_SIZE)) {
 
 		/* h += m[i] */
-		h0 += (le32_to_cpuvp(src +  0) >> 0) & 0x3ffffff;
-		h1 += (le32_to_cpuvp(src +  3) >> 2) & 0x3ffffff;
-		h2 += (le32_to_cpuvp(src +  6) >> 4) & 0x3ffffff;
-		h3 += (le32_to_cpuvp(src +  9) >> 6) & 0x3ffffff;
-		h4 += (le32_to_cpuvp(src + 12) >> 8) | hibit;
+		h0 += (get_unaligned_le32(src +  0) >> 0) & 0x3ffffff;
+		h1 += (get_unaligned_le32(src +  3) >> 2) & 0x3ffffff;
+		h2 += (get_unaligned_le32(src +  6) >> 4) & 0x3ffffff;
+		h3 += (get_unaligned_le32(src +  9) >> 6) & 0x3ffffff;
+		h4 += (get_unaligned_le32(src + 12) >> 8) | hibit;
 
 		/* h *= r */
 		d0 = mlt(h0, r0) + mlt(h1, s4) + mlt(h2, s3) +
diff --git a/crypto/scompress.c b/crypto/scompress.c
new file mode 100644
index 000000000000..35e396d154b7
--- /dev/null
+++ b/crypto/scompress.c
@@ -0,0 +1,356 @@
+/*
+ * Synchronous Compression operations
+ *
+ * Copyright 2015 LG Electronics Inc.
+ * Copyright (c) 2016, Intel Corporation
+ * Author: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/crypto.h>
+#include <linux/vmalloc.h>
+#include <crypto/algapi.h>
+#include <linux/cryptouser.h>
+#include <net/netlink.h>
+#include <linux/scatterlist.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/internal/acompress.h>
+#include <crypto/internal/scompress.h>
+#include "internal.h"
+
+static const struct crypto_type crypto_scomp_type;
+static void * __percpu *scomp_src_scratches;
+static void * __percpu *scomp_dst_scratches;
+static int scomp_scratch_users;
+static DEFINE_MUTEX(scomp_lock);
+
+#ifdef CONFIG_NET
+static int crypto_scomp_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+	struct crypto_report_comp rscomp;
+
+	strncpy(rscomp.type, "scomp", sizeof(rscomp.type));
+
+	if (nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS,
+		    sizeof(struct crypto_report_comp), &rscomp))
+		goto nla_put_failure;
+	return 0;
+
+nla_put_failure:
+	return -EMSGSIZE;
+}
+#else
+static int crypto_scomp_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+	return -ENOSYS;
+}
+#endif
+
+static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
+	__attribute__ ((unused));
+
+static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
+{
+	seq_puts(m, "type         : scomp\n");
+}
+
+static int crypto_scomp_init_tfm(struct crypto_tfm *tfm)
+{
+	return 0;
+}
+
+static void crypto_scomp_free_scratches(void * __percpu *scratches)
+{
+	int i;
+
+	if (!scratches)
+		return;
+
+	for_each_possible_cpu(i)
+		vfree(*per_cpu_ptr(scratches, i));
+
+	free_percpu(scratches);
+}
+
+static void * __percpu *crypto_scomp_alloc_scratches(void)
+{
+	void * __percpu *scratches;
+	int i;
+
+	scratches = alloc_percpu(void *);
+	if (!scratches)
+		return NULL;
+
+	for_each_possible_cpu(i) {
+		void *scratch;
+
+		scratch = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i));
+		if (!scratch)
+			goto error;
+		*per_cpu_ptr(scratches, i) = scratch;
+	}
+
+	return scratches;
+
+error:
+	crypto_scomp_free_scratches(scratches);
+	return NULL;
+}
+
+static void crypto_scomp_free_all_scratches(void)
+{
+	if (!--scomp_scratch_users) {
+		crypto_scomp_free_scratches(scomp_src_scratches);
+		crypto_scomp_free_scratches(scomp_dst_scratches);
+		scomp_src_scratches = NULL;
+		scomp_dst_scratches = NULL;
+	}
+}
+
+static int crypto_scomp_alloc_all_scratches(void)
+{
+	if (!scomp_scratch_users++) {
+		scomp_src_scratches = crypto_scomp_alloc_scratches();
+		if (!scomp_src_scratches)
+			return -ENOMEM;
+		scomp_dst_scratches = crypto_scomp_alloc_scratches();
+		if (!scomp_dst_scratches)
+			return -ENOMEM;
+	}
+	return 0;
+}
+
+static void crypto_scomp_sg_free(struct scatterlist *sgl)
+{
+	int i, n;
+	struct page *page;
+
+	if (!sgl)
+		return;
+
+	n = sg_nents(sgl);
+	for_each_sg(sgl, sgl, n, i) {
+		page = sg_page(sgl);
+		if (page)
+			__free_page(page);
+	}
+
+	kfree(sgl);
+}
+
+static struct scatterlist *crypto_scomp_sg_alloc(size_t size, gfp_t gfp)
+{
+	struct scatterlist *sgl;
+	struct page *page;
+	int i, n;
+
+	n = ((size - 1) >> PAGE_SHIFT) + 1;
+
+	sgl = kmalloc_array(n, sizeof(struct scatterlist), gfp);
+	if (!sgl)
+		return NULL;
+
+	sg_init_table(sgl, n);
+
+	for (i = 0; i < n; i++) {
+		page = alloc_page(gfp);
+		if (!page)
+			goto err;
+		sg_set_page(sgl + i, page, PAGE_SIZE, 0);
+	}
+
+	return sgl;
+
+err:
+	sg_mark_end(sgl + i);
+	crypto_scomp_sg_free(sgl);
+	return NULL;
+}
+
+static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
+{
+	struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
+	void **tfm_ctx = acomp_tfm_ctx(tfm);
+	struct crypto_scomp *scomp = *tfm_ctx;
+	void **ctx = acomp_request_ctx(req);
+	const int cpu = get_cpu();
+	u8 *scratch_src = *per_cpu_ptr(scomp_src_scratches, cpu);
+	u8 *scratch_dst = *per_cpu_ptr(scomp_dst_scratches, cpu);
+	int ret;
+
+	if (!req->src || !req->slen || req->slen > SCOMP_SCRATCH_SIZE) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (req->dst && !req->dlen) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (!req->dlen || req->dlen > SCOMP_SCRATCH_SIZE)
+		req->dlen = SCOMP_SCRATCH_SIZE;
+
+	scatterwalk_map_and_copy(scratch_src, req->src, 0, req->slen, 0);
+	if (dir)
+		ret = crypto_scomp_compress(scomp, scratch_src, req->slen,
+					    scratch_dst, &req->dlen, *ctx);
+	else
+		ret = crypto_scomp_decompress(scomp, scratch_src, req->slen,
+					      scratch_dst, &req->dlen, *ctx);
+	if (!ret) {
+		if (!req->dst) {
+			req->dst = crypto_scomp_sg_alloc(req->dlen,
+				   req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+				   GFP_KERNEL : GFP_ATOMIC);
+			if (!req->dst)
+				goto out;
+		}
+		scatterwalk_map_and_copy(scratch_dst, req->dst, 0, req->dlen,
+					 1);
+	}
+out:
+	put_cpu();
+	return ret;
+}
+
+static int scomp_acomp_compress(struct acomp_req *req)
+{
+	return scomp_acomp_comp_decomp(req, 1);
+}
+
+static int scomp_acomp_decompress(struct acomp_req *req)
+{
+	return scomp_acomp_comp_decomp(req, 0);
+}
+
+static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm)
+{
+	struct crypto_scomp **ctx = crypto_tfm_ctx(tfm);
+
+	crypto_free_scomp(*ctx);
+}
+
+int crypto_init_scomp_ops_async(struct crypto_tfm *tfm)
+{
+	struct crypto_alg *calg = tfm->__crt_alg;
+	struct crypto_acomp *crt = __crypto_acomp_tfm(tfm);
+	struct crypto_scomp **ctx = crypto_tfm_ctx(tfm);
+	struct crypto_scomp *scomp;
+
+	if (!crypto_mod_get(calg))
+		return -EAGAIN;
+
+	scomp = crypto_create_tfm(calg, &crypto_scomp_type);
+	if (IS_ERR(scomp)) {
+		crypto_mod_put(calg);
+		return PTR_ERR(scomp);
+	}
+
+	*ctx = scomp;
+	tfm->exit = crypto_exit_scomp_ops_async;
+
+	crt->compress = scomp_acomp_compress;
+	crt->decompress = scomp_acomp_decompress;
+	crt->dst_free = crypto_scomp_sg_free;
+	crt->reqsize = sizeof(void *);
+
+	return 0;
+}
+
+struct acomp_req *crypto_acomp_scomp_alloc_ctx(struct acomp_req *req)
+{
+	struct crypto_acomp *acomp = crypto_acomp_reqtfm(req);
+	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp);
+	struct crypto_scomp **tfm_ctx = crypto_tfm_ctx(tfm);
+	struct crypto_scomp *scomp = *tfm_ctx;
+	void *ctx;
+
+	ctx = crypto_scomp_alloc_ctx(scomp);
+	if (IS_ERR(ctx)) {
+		kfree(req);
+		return NULL;
+	}
+
+	*req->__ctx = ctx;
+
+	return req;
+}
+
+void crypto_acomp_scomp_free_ctx(struct acomp_req *req)
+{
+	struct crypto_acomp *acomp = crypto_acomp_reqtfm(req);
+	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp);
+	struct crypto_scomp **tfm_ctx = crypto_tfm_ctx(tfm);
+	struct crypto_scomp *scomp = *tfm_ctx;
+	void *ctx = *req->__ctx;
+
+	if (ctx)
+		crypto_scomp_free_ctx(scomp, ctx);
+}
+
+static const struct crypto_type crypto_scomp_type = {
+	.extsize = crypto_alg_extsize,
+	.init_tfm = crypto_scomp_init_tfm,
+#ifdef CONFIG_PROC_FS
+	.show = crypto_scomp_show,
+#endif
+	.report = crypto_scomp_report,
+	.maskclear = ~CRYPTO_ALG_TYPE_MASK,
+	.maskset = CRYPTO_ALG_TYPE_MASK,
+	.type = CRYPTO_ALG_TYPE_SCOMPRESS,
+	.tfmsize = offsetof(struct crypto_scomp, base),
+};
+
+int crypto_register_scomp(struct scomp_alg *alg)
+{
+	struct crypto_alg *base = &alg->base;
+	int ret = -ENOMEM;
+
+	mutex_lock(&scomp_lock);
+	if (crypto_scomp_alloc_all_scratches())
+		goto error;
+
+	base->cra_type = &crypto_scomp_type;
+	base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
+	base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS;
+
+	ret = crypto_register_alg(base);
+	if (ret)
+		goto error;
+
+	mutex_unlock(&scomp_lock);
+	return ret;
+
+error:
+	crypto_scomp_free_all_scratches();
+	mutex_unlock(&scomp_lock);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_register_scomp);
+
+int crypto_unregister_scomp(struct scomp_alg *alg)
+{
+	int ret;
+
+	mutex_lock(&scomp_lock);
+	ret = crypto_unregister_alg(&alg->base);
+	crypto_scomp_free_all_scratches();
+	mutex_unlock(&scomp_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_unregister_scomp);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Synchronous compression type");
diff --git a/crypto/simd.c b/crypto/simd.c
new file mode 100644
index 000000000000..88203370a62f
--- /dev/null
+++ b/crypto/simd.c
@@ -0,0 +1,226 @@
+/*
+ * Shared crypto simd helpers
+ *
+ * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ * Copyright (c) 2016 Herbert Xu <herbert@gondor.apana.org.au>
+ *
+ * Based on aesni-intel_glue.c by:
+ *  Copyright (C) 2008, Intel Corp.
+ *    Author: Huang Ying <ying.huang@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+ * USA
+ *
+ */
+
+#include <crypto/cryptd.h>
+#include <crypto/internal/simd.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/preempt.h>
+#include <asm/simd.h>
+
+struct simd_skcipher_alg {
+	const char *ialg_name;
+	struct skcipher_alg alg;
+};
+
+struct simd_skcipher_ctx {
+	struct cryptd_skcipher *cryptd_tfm;
+};
+
+static int simd_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
+				unsigned int key_len)
+{
+	struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct crypto_skcipher *child = &ctx->cryptd_tfm->base;
+	int err;
+
+	crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(tfm) &
+					 CRYPTO_TFM_REQ_MASK);
+	err = crypto_skcipher_setkey(child, key, key_len);
+	crypto_skcipher_set_flags(tfm, crypto_skcipher_get_flags(child) &
+				       CRYPTO_TFM_RES_MASK);
+	return err;
+}
+
+static int simd_skcipher_encrypt(struct skcipher_request *req)
+{
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_request *subreq;
+	struct crypto_skcipher *child;
+
+	subreq = skcipher_request_ctx(req);
+	*subreq = *req;
+
+	if (!may_use_simd() ||
+	    (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm)))
+		child = &ctx->cryptd_tfm->base;
+	else
+		child = cryptd_skcipher_child(ctx->cryptd_tfm);
+
+	skcipher_request_set_tfm(subreq, child);
+
+	return crypto_skcipher_encrypt(subreq);
+}
+
+static int simd_skcipher_decrypt(struct skcipher_request *req)
+{
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_request *subreq;
+	struct crypto_skcipher *child;
+
+	subreq = skcipher_request_ctx(req);
+	*subreq = *req;
+
+	if (!may_use_simd() ||
+	    (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm)))
+		child = &ctx->cryptd_tfm->base;
+	else
+		child = cryptd_skcipher_child(ctx->cryptd_tfm);
+
+	skcipher_request_set_tfm(subreq, child);
+
+	return crypto_skcipher_decrypt(subreq);
+}
+
+static void simd_skcipher_exit(struct crypto_skcipher *tfm)
+{
+	struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+	cryptd_free_skcipher(ctx->cryptd_tfm);
+}
+
+static int simd_skcipher_init(struct crypto_skcipher *tfm)
+{
+	struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct cryptd_skcipher *cryptd_tfm;
+	struct simd_skcipher_alg *salg;
+	struct skcipher_alg *alg;
+	unsigned reqsize;
+
+	alg = crypto_skcipher_alg(tfm);
+	salg = container_of(alg, struct simd_skcipher_alg, alg);
+
+	cryptd_tfm = cryptd_alloc_skcipher(salg->ialg_name,
+					   CRYPTO_ALG_INTERNAL,
+					   CRYPTO_ALG_INTERNAL);
+	if (IS_ERR(cryptd_tfm))
+		return PTR_ERR(cryptd_tfm);
+
+	ctx->cryptd_tfm = cryptd_tfm;
+
+	reqsize = sizeof(struct skcipher_request);
+	reqsize += crypto_skcipher_reqsize(&cryptd_tfm->base);
+
+	crypto_skcipher_set_reqsize(tfm, reqsize);
+
+	return 0;
+}
+
+struct simd_skcipher_alg *simd_skcipher_create_compat(const char *algname,
+						      const char *drvname,
+						      const char *basename)
+{
+	struct simd_skcipher_alg *salg;
+	struct crypto_skcipher *tfm;
+	struct skcipher_alg *ialg;
+	struct skcipher_alg *alg;
+	int err;
+
+	tfm = crypto_alloc_skcipher(basename, CRYPTO_ALG_INTERNAL,
+				    CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC);
+	if (IS_ERR(tfm))
+		return ERR_CAST(tfm);
+
+	ialg = crypto_skcipher_alg(tfm);
+
+	salg = kzalloc(sizeof(*salg), GFP_KERNEL);
+	if (!salg) {
+		salg = ERR_PTR(-ENOMEM);
+		goto out_put_tfm;
+	}
+
+	salg->ialg_name = basename;
+	alg = &salg->alg;
+
+	err = -ENAMETOOLONG;
+	if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", algname) >=
+	    CRYPTO_MAX_ALG_NAME)
+		goto out_free_salg;
+
+	if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
+		     drvname) >= CRYPTO_MAX_ALG_NAME)
+		goto out_free_salg;
+
+	alg->base.cra_flags = CRYPTO_ALG_ASYNC;
+	alg->base.cra_priority = ialg->base.cra_priority;
+	alg->base.cra_blocksize = ialg->base.cra_blocksize;
+	alg->base.cra_alignmask = ialg->base.cra_alignmask;
+	alg->base.cra_module = ialg->base.cra_module;
+	alg->base.cra_ctxsize = sizeof(struct simd_skcipher_ctx);
+
+	alg->ivsize = ialg->ivsize;
+	alg->chunksize = ialg->chunksize;
+	alg->min_keysize = ialg->min_keysize;
+	alg->max_keysize = ialg->max_keysize;
+
+	alg->init = simd_skcipher_init;
+	alg->exit = simd_skcipher_exit;
+
+	alg->setkey = simd_skcipher_setkey;
+	alg->encrypt = simd_skcipher_encrypt;
+	alg->decrypt = simd_skcipher_decrypt;
+
+	err = crypto_register_skcipher(alg);
+	if (err)
+		goto out_free_salg;
+
+out_put_tfm:
+	crypto_free_skcipher(tfm);
+	return salg;
+
+out_free_salg:
+	kfree(salg);
+	salg = ERR_PTR(err);
+	goto out_put_tfm;
+}
+EXPORT_SYMBOL_GPL(simd_skcipher_create_compat);
+
+struct simd_skcipher_alg *simd_skcipher_create(const char *algname,
+					       const char *basename)
+{
+	char drvname[CRYPTO_MAX_ALG_NAME];
+
+	if (snprintf(drvname, CRYPTO_MAX_ALG_NAME, "simd-%s", basename) >=
+	    CRYPTO_MAX_ALG_NAME)
+		return ERR_PTR(-ENAMETOOLONG);
+
+	return simd_skcipher_create_compat(algname, drvname, basename);
+}
+EXPORT_SYMBOL_GPL(simd_skcipher_create);
+
+void simd_skcipher_free(struct simd_skcipher_alg *salg)
+{
+	crypto_unregister_skcipher(&salg->alg);
+	kfree(salg);
+}
+EXPORT_SYMBOL_GPL(simd_skcipher_free);
+
+MODULE_LICENSE("GPL");
diff --git a/crypto/skcipher.c b/crypto/skcipher.c
index f7d0018dcaee..0e1e6c35188e 100644
--- a/crypto/skcipher.c
+++ b/crypto/skcipher.c
@@ -14,9 +14,12 @@
  *
  */
 
+#include <crypto/internal/aead.h>
 #include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
 #include <linux/bug.h>
 #include <linux/cryptouser.h>
+#include <linux/list.h>
 #include <linux/module.h>
 #include <linux/rtnetlink.h>
 #include <linux/seq_file.h>
@@ -24,6 +27,545 @@
 
 #include "internal.h"
 
+enum {
+	SKCIPHER_WALK_PHYS = 1 << 0,
+	SKCIPHER_WALK_SLOW = 1 << 1,
+	SKCIPHER_WALK_COPY = 1 << 2,
+	SKCIPHER_WALK_DIFF = 1 << 3,
+	SKCIPHER_WALK_SLEEP = 1 << 4,
+};
+
+struct skcipher_walk_buffer {
+	struct list_head entry;
+	struct scatter_walk dst;
+	unsigned int len;
+	u8 *data;
+	u8 buffer[];
+};
+
+static int skcipher_walk_next(struct skcipher_walk *walk);
+
+static inline void skcipher_unmap(struct scatter_walk *walk, void *vaddr)
+{
+	if (PageHighMem(scatterwalk_page(walk)))
+		kunmap_atomic(vaddr);
+}
+
+static inline void *skcipher_map(struct scatter_walk *walk)
+{
+	struct page *page = scatterwalk_page(walk);
+
+	return (PageHighMem(page) ? kmap_atomic(page) : page_address(page)) +
+	       offset_in_page(walk->offset);
+}
+
+static inline void skcipher_map_src(struct skcipher_walk *walk)
+{
+	walk->src.virt.addr = skcipher_map(&walk->in);
+}
+
+static inline void skcipher_map_dst(struct skcipher_walk *walk)
+{
+	walk->dst.virt.addr = skcipher_map(&walk->out);
+}
+
+static inline void skcipher_unmap_src(struct skcipher_walk *walk)
+{
+	skcipher_unmap(&walk->in, walk->src.virt.addr);
+}
+
+static inline void skcipher_unmap_dst(struct skcipher_walk *walk)
+{
+	skcipher_unmap(&walk->out, walk->dst.virt.addr);
+}
+
+static inline gfp_t skcipher_walk_gfp(struct skcipher_walk *walk)
+{
+	return walk->flags & SKCIPHER_WALK_SLEEP ? GFP_KERNEL : GFP_ATOMIC;
+}
+
+/* Get a spot of the specified length that does not straddle a page.
+ * The caller needs to ensure that there is enough space for this operation.
+ */
+static inline u8 *skcipher_get_spot(u8 *start, unsigned int len)
+{
+	u8 *end_page = (u8 *)(((unsigned long)(start + len - 1)) & PAGE_MASK);
+
+	return max(start, end_page);
+}
+
+static int skcipher_done_slow(struct skcipher_walk *walk, unsigned int bsize)
+{
+	u8 *addr;
+
+	addr = (u8 *)ALIGN((unsigned long)walk->buffer, walk->alignmask + 1);
+	addr = skcipher_get_spot(addr, bsize);
+	scatterwalk_copychunks(addr, &walk->out, bsize,
+			       (walk->flags & SKCIPHER_WALK_PHYS) ? 2 : 1);
+	return 0;
+}
+
+int skcipher_walk_done(struct skcipher_walk *walk, int err)
+{
+	unsigned int n = walk->nbytes - err;
+	unsigned int nbytes;
+
+	nbytes = walk->total - n;
+
+	if (unlikely(err < 0)) {
+		nbytes = 0;
+		n = 0;
+	} else if (likely(!(walk->flags & (SKCIPHER_WALK_PHYS |
+					   SKCIPHER_WALK_SLOW |
+					   SKCIPHER_WALK_COPY |
+					   SKCIPHER_WALK_DIFF)))) {
+unmap_src:
+		skcipher_unmap_src(walk);
+	} else if (walk->flags & SKCIPHER_WALK_DIFF) {
+		skcipher_unmap_dst(walk);
+		goto unmap_src;
+	} else if (walk->flags & SKCIPHER_WALK_COPY) {
+		skcipher_map_dst(walk);
+		memcpy(walk->dst.virt.addr, walk->page, n);
+		skcipher_unmap_dst(walk);
+	} else if (unlikely(walk->flags & SKCIPHER_WALK_SLOW)) {
+		if (WARN_ON(err)) {
+			err = -EINVAL;
+			nbytes = 0;
+		} else
+			n = skcipher_done_slow(walk, n);
+	}
+
+	if (err > 0)
+		err = 0;
+
+	walk->total = nbytes;
+	walk->nbytes = nbytes;
+
+	scatterwalk_advance(&walk->in, n);
+	scatterwalk_advance(&walk->out, n);
+	scatterwalk_done(&walk->in, 0, nbytes);
+	scatterwalk_done(&walk->out, 1, nbytes);
+
+	if (nbytes) {
+		crypto_yield(walk->flags & SKCIPHER_WALK_SLEEP ?
+			     CRYPTO_TFM_REQ_MAY_SLEEP : 0);
+		return skcipher_walk_next(walk);
+	}
+
+	/* Short-circuit for the common/fast path. */
+	if (!((unsigned long)walk->buffer | (unsigned long)walk->page))
+		goto out;
+
+	if (walk->flags & SKCIPHER_WALK_PHYS)
+		goto out;
+
+	if (walk->iv != walk->oiv)
+		memcpy(walk->oiv, walk->iv, walk->ivsize);
+	if (walk->buffer != walk->page)
+		kfree(walk->buffer);
+	if (walk->page)
+		free_page((unsigned long)walk->page);
+
+out:
+	return err;
+}
+EXPORT_SYMBOL_GPL(skcipher_walk_done);
+
+void skcipher_walk_complete(struct skcipher_walk *walk, int err)
+{
+	struct skcipher_walk_buffer *p, *tmp;
+
+	list_for_each_entry_safe(p, tmp, &walk->buffers, entry) {
+		u8 *data;
+
+		if (err)
+			goto done;
+
+		data = p->data;
+		if (!data) {
+			data = PTR_ALIGN(&p->buffer[0], walk->alignmask + 1);
+			data = skcipher_get_spot(data, walk->chunksize);
+		}
+
+		scatterwalk_copychunks(data, &p->dst, p->len, 1);
+
+		if (offset_in_page(p->data) + p->len + walk->chunksize >
+		    PAGE_SIZE)
+			free_page((unsigned long)p->data);
+
+done:
+		list_del(&p->entry);
+		kfree(p);
+	}
+
+	if (!err && walk->iv != walk->oiv)
+		memcpy(walk->oiv, walk->iv, walk->ivsize);
+	if (walk->buffer != walk->page)
+		kfree(walk->buffer);
+	if (walk->page)
+		free_page((unsigned long)walk->page);
+}
+EXPORT_SYMBOL_GPL(skcipher_walk_complete);
+
+static void skcipher_queue_write(struct skcipher_walk *walk,
+				 struct skcipher_walk_buffer *p)
+{
+	p->dst = walk->out;
+	list_add_tail(&p->entry, &walk->buffers);
+}
+
+static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize)
+{
+	bool phys = walk->flags & SKCIPHER_WALK_PHYS;
+	unsigned alignmask = walk->alignmask;
+	struct skcipher_walk_buffer *p;
+	unsigned a;
+	unsigned n;
+	u8 *buffer;
+	void *v;
+
+	if (!phys) {
+		if (!walk->buffer)
+			walk->buffer = walk->page;
+		buffer = walk->buffer;
+		if (buffer)
+			goto ok;
+	}
+
+	/* Start with the minimum alignment of kmalloc. */
+	a = crypto_tfm_ctx_alignment() - 1;
+	n = bsize;
+
+	if (phys) {
+		/* Calculate the minimum alignment of p->buffer. */
+		a &= (sizeof(*p) ^ (sizeof(*p) - 1)) >> 1;
+		n += sizeof(*p);
+	}
+
+	/* Minimum size to align p->buffer by alignmask. */
+	n += alignmask & ~a;
+
+	/* Minimum size to ensure p->buffer does not straddle a page. */
+	n += (bsize - 1) & ~(alignmask | a);
+
+	v = kzalloc(n, skcipher_walk_gfp(walk));
+	if (!v)
+		return skcipher_walk_done(walk, -ENOMEM);
+
+	if (phys) {
+		p = v;
+		p->len = bsize;
+		skcipher_queue_write(walk, p);
+		buffer = p->buffer;
+	} else {
+		walk->buffer = v;
+		buffer = v;
+	}
+
+ok:
+	walk->dst.virt.addr = PTR_ALIGN(buffer, alignmask + 1);
+	walk->dst.virt.addr = skcipher_get_spot(walk->dst.virt.addr, bsize);
+	walk->src.virt.addr = walk->dst.virt.addr;
+
+	scatterwalk_copychunks(walk->src.virt.addr, &walk->in, bsize, 0);
+
+	walk->nbytes = bsize;
+	walk->flags |= SKCIPHER_WALK_SLOW;
+
+	return 0;
+}
+
+static int skcipher_next_copy(struct skcipher_walk *walk)
+{
+	struct skcipher_walk_buffer *p;
+	u8 *tmp = walk->page;
+
+	skcipher_map_src(walk);
+	memcpy(tmp, walk->src.virt.addr, walk->nbytes);
+	skcipher_unmap_src(walk);
+
+	walk->src.virt.addr = tmp;
+	walk->dst.virt.addr = tmp;
+
+	if (!(walk->flags & SKCIPHER_WALK_PHYS))
+		return 0;
+
+	p = kmalloc(sizeof(*p), skcipher_walk_gfp(walk));
+	if (!p)
+		return -ENOMEM;
+
+	p->data = walk->page;
+	p->len = walk->nbytes;
+	skcipher_queue_write(walk, p);
+
+	if (offset_in_page(walk->page) + walk->nbytes + walk->chunksize >
+	    PAGE_SIZE)
+		walk->page = NULL;
+	else
+		walk->page += walk->nbytes;
+
+	return 0;
+}
+
+static int skcipher_next_fast(struct skcipher_walk *walk)
+{
+	unsigned long diff;
+
+	walk->src.phys.page = scatterwalk_page(&walk->in);
+	walk->src.phys.offset = offset_in_page(walk->in.offset);
+	walk->dst.phys.page = scatterwalk_page(&walk->out);
+	walk->dst.phys.offset = offset_in_page(walk->out.offset);
+
+	if (walk->flags & SKCIPHER_WALK_PHYS)
+		return 0;
+
+	diff = walk->src.phys.offset - walk->dst.phys.offset;
+	diff |= walk->src.virt.page - walk->dst.virt.page;
+
+	skcipher_map_src(walk);
+	walk->dst.virt.addr = walk->src.virt.addr;
+
+	if (diff) {
+		walk->flags |= SKCIPHER_WALK_DIFF;
+		skcipher_map_dst(walk);
+	}
+
+	return 0;
+}
+
+static int skcipher_walk_next(struct skcipher_walk *walk)
+{
+	unsigned int bsize;
+	unsigned int n;
+	int err;
+
+	walk->flags &= ~(SKCIPHER_WALK_SLOW | SKCIPHER_WALK_COPY |
+			 SKCIPHER_WALK_DIFF);
+
+	n = walk->total;
+	bsize = min(walk->chunksize, max(n, walk->blocksize));
+	n = scatterwalk_clamp(&walk->in, n);
+	n = scatterwalk_clamp(&walk->out, n);
+
+	if (unlikely(n < bsize)) {
+		if (unlikely(walk->total < walk->blocksize))
+			return skcipher_walk_done(walk, -EINVAL);
+
+slow_path:
+		err = skcipher_next_slow(walk, bsize);
+		goto set_phys_lowmem;
+	}
+
+	if (unlikely((walk->in.offset | walk->out.offset) & walk->alignmask)) {
+		if (!walk->page) {
+			gfp_t gfp = skcipher_walk_gfp(walk);
+
+			walk->page = (void *)__get_free_page(gfp);
+			if (!walk->page)
+				goto slow_path;
+		}
+
+		walk->nbytes = min_t(unsigned, n,
+				     PAGE_SIZE - offset_in_page(walk->page));
+		walk->flags |= SKCIPHER_WALK_COPY;
+		err = skcipher_next_copy(walk);
+		goto set_phys_lowmem;
+	}
+
+	walk->nbytes = n;
+
+	return skcipher_next_fast(walk);
+
+set_phys_lowmem:
+	if (!err && (walk->flags & SKCIPHER_WALK_PHYS)) {
+		walk->src.phys.page = virt_to_page(walk->src.virt.addr);
+		walk->dst.phys.page = virt_to_page(walk->dst.virt.addr);
+		walk->src.phys.offset &= PAGE_SIZE - 1;
+		walk->dst.phys.offset &= PAGE_SIZE - 1;
+	}
+	return err;
+}
+EXPORT_SYMBOL_GPL(skcipher_walk_next);
+
+static int skcipher_copy_iv(struct skcipher_walk *walk)
+{
+	unsigned a = crypto_tfm_ctx_alignment() - 1;
+	unsigned alignmask = walk->alignmask;
+	unsigned ivsize = walk->ivsize;
+	unsigned bs = walk->chunksize;
+	unsigned aligned_bs;
+	unsigned size;
+	u8 *iv;
+
+	aligned_bs = ALIGN(bs, alignmask);
+
+	/* Minimum size to align buffer by alignmask. */
+	size = alignmask & ~a;
+
+	if (walk->flags & SKCIPHER_WALK_PHYS)
+		size += ivsize;
+	else {
+		size += aligned_bs + ivsize;
+
+		/* Minimum size to ensure buffer does not straddle a page. */
+		size += (bs - 1) & ~(alignmask | a);
+	}
+
+	walk->buffer = kmalloc(size, skcipher_walk_gfp(walk));
+	if (!walk->buffer)
+		return -ENOMEM;
+
+	iv = PTR_ALIGN(walk->buffer, alignmask + 1);
+	iv = skcipher_get_spot(iv, bs) + aligned_bs;
+
+	walk->iv = memcpy(iv, walk->iv, walk->ivsize);
+	return 0;
+}
+
+static int skcipher_walk_first(struct skcipher_walk *walk)
+{
+	walk->nbytes = 0;
+
+	if (WARN_ON_ONCE(in_irq()))
+		return -EDEADLK;
+
+	if (unlikely(!walk->total))
+		return 0;
+
+	walk->buffer = NULL;
+	if (unlikely(((unsigned long)walk->iv & walk->alignmask))) {
+		int err = skcipher_copy_iv(walk);
+		if (err)
+			return err;
+	}
+
+	walk->page = NULL;
+	walk->nbytes = walk->total;
+
+	return skcipher_walk_next(walk);
+}
+
+static int skcipher_walk_skcipher(struct skcipher_walk *walk,
+				  struct skcipher_request *req)
+{
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+
+	scatterwalk_start(&walk->in, req->src);
+	scatterwalk_start(&walk->out, req->dst);
+
+	walk->total = req->cryptlen;
+	walk->iv = req->iv;
+	walk->oiv = req->iv;
+
+	walk->flags &= ~SKCIPHER_WALK_SLEEP;
+	walk->flags |= req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+		       SKCIPHER_WALK_SLEEP : 0;
+
+	walk->blocksize = crypto_skcipher_blocksize(tfm);
+	walk->chunksize = crypto_skcipher_chunksize(tfm);
+	walk->ivsize = crypto_skcipher_ivsize(tfm);
+	walk->alignmask = crypto_skcipher_alignmask(tfm);
+
+	return skcipher_walk_first(walk);
+}
+
+int skcipher_walk_virt(struct skcipher_walk *walk,
+		       struct skcipher_request *req, bool atomic)
+{
+	int err;
+
+	walk->flags &= ~SKCIPHER_WALK_PHYS;
+
+	err = skcipher_walk_skcipher(walk, req);
+
+	walk->flags &= atomic ? ~SKCIPHER_WALK_SLEEP : ~0;
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(skcipher_walk_virt);
+
+void skcipher_walk_atomise(struct skcipher_walk *walk)
+{
+	walk->flags &= ~SKCIPHER_WALK_SLEEP;
+}
+EXPORT_SYMBOL_GPL(skcipher_walk_atomise);
+
+int skcipher_walk_async(struct skcipher_walk *walk,
+			struct skcipher_request *req)
+{
+	walk->flags |= SKCIPHER_WALK_PHYS;
+
+	INIT_LIST_HEAD(&walk->buffers);
+
+	return skcipher_walk_skcipher(walk, req);
+}
+EXPORT_SYMBOL_GPL(skcipher_walk_async);
+
+static int skcipher_walk_aead_common(struct skcipher_walk *walk,
+				     struct aead_request *req, bool atomic)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	int err;
+
+	walk->flags &= ~SKCIPHER_WALK_PHYS;
+
+	scatterwalk_start(&walk->in, req->src);
+	scatterwalk_start(&walk->out, req->dst);
+
+	scatterwalk_copychunks(NULL, &walk->in, req->assoclen, 2);
+	scatterwalk_copychunks(NULL, &walk->out, req->assoclen, 2);
+
+	walk->iv = req->iv;
+	walk->oiv = req->iv;
+
+	if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP)
+		walk->flags |= SKCIPHER_WALK_SLEEP;
+	else
+		walk->flags &= ~SKCIPHER_WALK_SLEEP;
+
+	walk->blocksize = crypto_aead_blocksize(tfm);
+	walk->chunksize = crypto_aead_chunksize(tfm);
+	walk->ivsize = crypto_aead_ivsize(tfm);
+	walk->alignmask = crypto_aead_alignmask(tfm);
+
+	err = skcipher_walk_first(walk);
+
+	if (atomic)
+		walk->flags &= ~SKCIPHER_WALK_SLEEP;
+
+	return err;
+}
+
+int skcipher_walk_aead(struct skcipher_walk *walk, struct aead_request *req,
+		       bool atomic)
+{
+	walk->total = req->cryptlen;
+
+	return skcipher_walk_aead_common(walk, req, atomic);
+}
+EXPORT_SYMBOL_GPL(skcipher_walk_aead);
+
+int skcipher_walk_aead_encrypt(struct skcipher_walk *walk,
+			       struct aead_request *req, bool atomic)
+{
+	walk->total = req->cryptlen;
+
+	return skcipher_walk_aead_common(walk, req, atomic);
+}
+EXPORT_SYMBOL_GPL(skcipher_walk_aead_encrypt);
+
+int skcipher_walk_aead_decrypt(struct skcipher_walk *walk,
+			       struct aead_request *req, bool atomic)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+
+	walk->total = req->cryptlen - crypto_aead_authsize(tfm);
+
+	return skcipher_walk_aead_common(walk, req, atomic);
+}
+EXPORT_SYMBOL_GPL(skcipher_walk_aead_decrypt);
+
 static unsigned int crypto_skcipher_extsize(struct crypto_alg *alg)
 {
 	if (alg->cra_type == &crypto_blkcipher_type)
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 62dffa0028ac..f616ad74cce7 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -33,6 +33,7 @@
 #include <crypto/drbg.h>
 #include <crypto/akcipher.h>
 #include <crypto/kpp.h>
+#include <crypto/acompress.h>
 
 #include "internal.h"
 
@@ -62,7 +63,7 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
  */
 #define IDX1		32
 #define IDX2		32400
-#define IDX3		1
+#define IDX3		1511
 #define IDX4		8193
 #define IDX5		22222
 #define IDX6		17101
@@ -1442,6 +1443,126 @@ out:
 	return ret;
 }
 
+static int test_acomp(struct crypto_acomp *tfm, struct comp_testvec *ctemplate,
+		      struct comp_testvec *dtemplate, int ctcount, int dtcount)
+{
+	const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm));
+	unsigned int i;
+	char *output;
+	int ret;
+	struct scatterlist src, dst;
+	struct acomp_req *req;
+	struct tcrypt_result result;
+
+	output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
+	if (!output)
+		return -ENOMEM;
+
+	for (i = 0; i < ctcount; i++) {
+		unsigned int dlen = COMP_BUF_SIZE;
+		int ilen = ctemplate[i].inlen;
+
+		memset(output, 0, dlen);
+		init_completion(&result.completion);
+		sg_init_one(&src, ctemplate[i].input, ilen);
+		sg_init_one(&dst, output, dlen);
+
+		req = acomp_request_alloc(tfm);
+		if (!req) {
+			pr_err("alg: acomp: request alloc failed for %s\n",
+			       algo);
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		acomp_request_set_params(req, &src, &dst, ilen, dlen);
+		acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+					   tcrypt_complete, &result);
+
+		ret = wait_async_op(&result, crypto_acomp_compress(req));
+		if (ret) {
+			pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
+			       i + 1, algo, -ret);
+			acomp_request_free(req);
+			goto out;
+		}
+
+		if (req->dlen != ctemplate[i].outlen) {
+			pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n",
+			       i + 1, algo, req->dlen);
+			ret = -EINVAL;
+			acomp_request_free(req);
+			goto out;
+		}
+
+		if (memcmp(output, ctemplate[i].output, req->dlen)) {
+			pr_err("alg: acomp: Compression test %d failed for %s\n",
+			       i + 1, algo);
+			hexdump(output, req->dlen);
+			ret = -EINVAL;
+			acomp_request_free(req);
+			goto out;
+		}
+
+		acomp_request_free(req);
+	}
+
+	for (i = 0; i < dtcount; i++) {
+		unsigned int dlen = COMP_BUF_SIZE;
+		int ilen = dtemplate[i].inlen;
+
+		memset(output, 0, dlen);
+		init_completion(&result.completion);
+		sg_init_one(&src, dtemplate[i].input, ilen);
+		sg_init_one(&dst, output, dlen);
+
+		req = acomp_request_alloc(tfm);
+		if (!req) {
+			pr_err("alg: acomp: request alloc failed for %s\n",
+			       algo);
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		acomp_request_set_params(req, &src, &dst, ilen, dlen);
+		acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+					   tcrypt_complete, &result);
+
+		ret = wait_async_op(&result, crypto_acomp_decompress(req));
+		if (ret) {
+			pr_err("alg: acomp: decompression failed on test %d for %s: ret=%d\n",
+			       i + 1, algo, -ret);
+			acomp_request_free(req);
+			goto out;
+		}
+
+		if (req->dlen != dtemplate[i].outlen) {
+			pr_err("alg: acomp: Decompression test %d failed for %s: output len = %d\n",
+			       i + 1, algo, req->dlen);
+			ret = -EINVAL;
+			acomp_request_free(req);
+			goto out;
+		}
+
+		if (memcmp(output, dtemplate[i].output, req->dlen)) {
+			pr_err("alg: acomp: Decompression test %d failed for %s\n",
+			       i + 1, algo);
+			hexdump(output, req->dlen);
+			ret = -EINVAL;
+			acomp_request_free(req);
+			goto out;
+		}
+
+		acomp_request_free(req);
+	}
+
+	ret = 0;
+
+out:
+	kfree(output);
+	return ret;
+}
+
 static int test_cprng(struct crypto_rng *tfm, struct cprng_testvec *template,
 		      unsigned int tcount)
 {
@@ -1509,7 +1630,7 @@ static int alg_test_aead(const struct alg_test_desc *desc, const char *driver,
 	struct crypto_aead *tfm;
 	int err = 0;
 
-	tfm = crypto_alloc_aead(driver, type | CRYPTO_ALG_INTERNAL, mask);
+	tfm = crypto_alloc_aead(driver, type, mask);
 	if (IS_ERR(tfm)) {
 		printk(KERN_ERR "alg: aead: Failed to load transform for %s: "
 		       "%ld\n", driver, PTR_ERR(tfm));
@@ -1538,7 +1659,7 @@ static int alg_test_cipher(const struct alg_test_desc *desc,
 	struct crypto_cipher *tfm;
 	int err = 0;
 
-	tfm = crypto_alloc_cipher(driver, type | CRYPTO_ALG_INTERNAL, mask);
+	tfm = crypto_alloc_cipher(driver, type, mask);
 	if (IS_ERR(tfm)) {
 		printk(KERN_ERR "alg: cipher: Failed to load transform for "
 		       "%s: %ld\n", driver, PTR_ERR(tfm));
@@ -1567,7 +1688,7 @@ static int alg_test_skcipher(const struct alg_test_desc *desc,
 	struct crypto_skcipher *tfm;
 	int err = 0;
 
-	tfm = crypto_alloc_skcipher(driver, type | CRYPTO_ALG_INTERNAL, mask);
+	tfm = crypto_alloc_skcipher(driver, type, mask);
 	if (IS_ERR(tfm)) {
 		printk(KERN_ERR "alg: skcipher: Failed to load transform for "
 		       "%s: %ld\n", driver, PTR_ERR(tfm));
@@ -1593,22 +1714,38 @@ out:
 static int alg_test_comp(const struct alg_test_desc *desc, const char *driver,
 			 u32 type, u32 mask)
 {
-	struct crypto_comp *tfm;
+	struct crypto_comp *comp;
+	struct crypto_acomp *acomp;
 	int err;
+	u32 algo_type = type & CRYPTO_ALG_TYPE_ACOMPRESS_MASK;
+
+	if (algo_type == CRYPTO_ALG_TYPE_ACOMPRESS) {
+		acomp = crypto_alloc_acomp(driver, type, mask);
+		if (IS_ERR(acomp)) {
+			pr_err("alg: acomp: Failed to load transform for %s: %ld\n",
+			       driver, PTR_ERR(acomp));
+			return PTR_ERR(acomp);
+		}
+		err = test_acomp(acomp, desc->suite.comp.comp.vecs,
+				 desc->suite.comp.decomp.vecs,
+				 desc->suite.comp.comp.count,
+				 desc->suite.comp.decomp.count);
+		crypto_free_acomp(acomp);
+	} else {
+		comp = crypto_alloc_comp(driver, type, mask);
+		if (IS_ERR(comp)) {
+			pr_err("alg: comp: Failed to load transform for %s: %ld\n",
+			       driver, PTR_ERR(comp));
+			return PTR_ERR(comp);
+		}
 
-	tfm = crypto_alloc_comp(driver, type, mask);
-	if (IS_ERR(tfm)) {
-		printk(KERN_ERR "alg: comp: Failed to load transform for %s: "
-		       "%ld\n", driver, PTR_ERR(tfm));
-		return PTR_ERR(tfm);
-	}
-
-	err = test_comp(tfm, desc->suite.comp.comp.vecs,
-			desc->suite.comp.decomp.vecs,
-			desc->suite.comp.comp.count,
-			desc->suite.comp.decomp.count);
+		err = test_comp(comp, desc->suite.comp.comp.vecs,
+				desc->suite.comp.decomp.vecs,
+				desc->suite.comp.comp.count,
+				desc->suite.comp.decomp.count);
 
-	crypto_free_comp(tfm);
+		crypto_free_comp(comp);
+	}
 	return err;
 }
 
@@ -1618,7 +1755,7 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,
 	struct crypto_ahash *tfm;
 	int err;
 
-	tfm = crypto_alloc_ahash(driver, type | CRYPTO_ALG_INTERNAL, mask);
+	tfm = crypto_alloc_ahash(driver, type, mask);
 	if (IS_ERR(tfm)) {
 		printk(KERN_ERR "alg: hash: Failed to load transform for %s: "
 		       "%ld\n", driver, PTR_ERR(tfm));
@@ -1646,7 +1783,7 @@ static int alg_test_crc32c(const struct alg_test_desc *desc,
 	if (err)
 		goto out;
 
-	tfm = crypto_alloc_shash(driver, type | CRYPTO_ALG_INTERNAL, mask);
+	tfm = crypto_alloc_shash(driver, type, mask);
 	if (IS_ERR(tfm)) {
 		printk(KERN_ERR "alg: crc32c: Failed to load transform for %s: "
 		       "%ld\n", driver, PTR_ERR(tfm));
@@ -1688,7 +1825,7 @@ static int alg_test_cprng(const struct alg_test_desc *desc, const char *driver,
 	struct crypto_rng *rng;
 	int err;
 
-	rng = crypto_alloc_rng(driver, type | CRYPTO_ALG_INTERNAL, mask);
+	rng = crypto_alloc_rng(driver, type, mask);
 	if (IS_ERR(rng)) {
 		printk(KERN_ERR "alg: cprng: Failed to load transform for %s: "
 		       "%ld\n", driver, PTR_ERR(rng));
@@ -1715,7 +1852,7 @@ static int drbg_cavs_test(struct drbg_testvec *test, int pr,
 	if (!buf)
 		return -ENOMEM;
 
-	drng = crypto_alloc_rng(driver, type | CRYPTO_ALG_INTERNAL, mask);
+	drng = crypto_alloc_rng(driver, type, mask);
 	if (IS_ERR(drng)) {
 		printk(KERN_ERR "alg: drbg: could not allocate DRNG handle for "
 		       "%s\n", driver);
@@ -1909,7 +2046,7 @@ static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver,
 	struct crypto_kpp *tfm;
 	int err = 0;
 
-	tfm = crypto_alloc_kpp(driver, type | CRYPTO_ALG_INTERNAL, mask);
+	tfm = crypto_alloc_kpp(driver, type, mask);
 	if (IS_ERR(tfm)) {
 		pr_err("alg: kpp: Failed to load tfm for %s: %ld\n",
 		       driver, PTR_ERR(tfm));
@@ -2068,7 +2205,7 @@ static int alg_test_akcipher(const struct alg_test_desc *desc,
 	struct crypto_akcipher *tfm;
 	int err = 0;
 
-	tfm = crypto_alloc_akcipher(driver, type | CRYPTO_ALG_INTERNAL, mask);
+	tfm = crypto_alloc_akcipher(driver, type, mask);
 	if (IS_ERR(tfm)) {
 		pr_err("alg: akcipher: Failed to load tfm for %s: %ld\n",
 		       driver, PTR_ERR(tfm));
@@ -2091,88 +2228,6 @@ static int alg_test_null(const struct alg_test_desc *desc,
 /* Please keep this list sorted by algorithm name. */
 static const struct alg_test_desc alg_test_descs[] = {
 	{
-		.alg = "__cbc-cast5-avx",
-		.test = alg_test_null,
-	}, {
-		.alg = "__cbc-cast6-avx",
-		.test = alg_test_null,
-	}, {
-		.alg = "__cbc-serpent-avx",
-		.test = alg_test_null,
-	}, {
-		.alg = "__cbc-serpent-avx2",
-		.test = alg_test_null,
-	}, {
-		.alg = "__cbc-serpent-sse2",
-		.test = alg_test_null,
-	}, {
-		.alg = "__cbc-twofish-avx",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-cbc-aes-aesni",
-		.test = alg_test_null,
-		.fips_allowed = 1,
-	}, {
-		.alg = "__driver-cbc-camellia-aesni",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-cbc-camellia-aesni-avx2",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-cbc-cast5-avx",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-cbc-cast6-avx",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-cbc-serpent-avx",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-cbc-serpent-avx2",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-cbc-serpent-sse2",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-cbc-twofish-avx",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-ecb-aes-aesni",
-		.test = alg_test_null,
-		.fips_allowed = 1,
-	}, {
-		.alg = "__driver-ecb-camellia-aesni",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-ecb-camellia-aesni-avx2",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-ecb-cast5-avx",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-ecb-cast6-avx",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-ecb-serpent-avx",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-ecb-serpent-avx2",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-ecb-serpent-sse2",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-ecb-twofish-avx",
-		.test = alg_test_null,
-	}, {
-		.alg = "__driver-gcm-aes-aesni",
-		.test = alg_test_null,
-		.fips_allowed = 1,
-	}, {
-		.alg = "__ghash-pclmulqdqni",
-		.test = alg_test_null,
-		.fips_allowed = 1,
-	}, {
 		.alg = "ansi_cprng",
 		.test = alg_test_cprng,
 		.suite = {
@@ -2659,55 +2714,6 @@ static const struct alg_test_desc alg_test_descs[] = {
 			}
 		}
 	}, {
-		.alg = "cryptd(__driver-cbc-aes-aesni)",
-		.test = alg_test_null,
-		.fips_allowed = 1,
-	}, {
-		.alg = "cryptd(__driver-cbc-camellia-aesni)",
-		.test = alg_test_null,
-	}, {
-		.alg = "cryptd(__driver-cbc-camellia-aesni-avx2)",
-		.test = alg_test_null,
-	}, {
-		.alg = "cryptd(__driver-cbc-serpent-avx2)",
-		.test = alg_test_null,
-	}, {
-		.alg = "cryptd(__driver-ecb-aes-aesni)",
-		.test = alg_test_null,
-		.fips_allowed = 1,
-	}, {
-		.alg = "cryptd(__driver-ecb-camellia-aesni)",
-		.test = alg_test_null,
-	}, {
-		.alg = "cryptd(__driver-ecb-camellia-aesni-avx2)",
-		.test = alg_test_null,
-	}, {
-		.alg = "cryptd(__driver-ecb-cast5-avx)",
-		.test = alg_test_null,
-	}, {
-		.alg = "cryptd(__driver-ecb-cast6-avx)",
-		.test = alg_test_null,
-	}, {
-		.alg = "cryptd(__driver-ecb-serpent-avx)",
-		.test = alg_test_null,
-	}, {
-		.alg = "cryptd(__driver-ecb-serpent-avx2)",
-		.test = alg_test_null,
-	}, {
-		.alg = "cryptd(__driver-ecb-serpent-sse2)",
-		.test = alg_test_null,
-	}, {
-		.alg = "cryptd(__driver-ecb-twofish-avx)",
-		.test = alg_test_null,
-	}, {
-		.alg = "cryptd(__driver-gcm-aes-aesni)",
-		.test = alg_test_null,
-		.fips_allowed = 1,
-	}, {
-		.alg = "cryptd(__ghash-pclmulqdqni)",
-		.test = alg_test_null,
-		.fips_allowed = 1,
-	}, {
 		.alg = "ctr(aes)",
 		.test = alg_test_skcipher,
 		.fips_allowed = 1,
@@ -3034,10 +3040,6 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.fips_allowed = 1,
 		.test = alg_test_null,
 	}, {
-		.alg = "ecb(__aes-aesni)",
-		.test = alg_test_null,
-		.fips_allowed = 1,
-	}, {
 		.alg = "ecb(aes)",
 		.test = alg_test_skcipher,
 		.fips_allowed = 1,
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index e64a4ef9d8ca..9b656be7f52f 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -1334,36 +1334,50 @@ static struct hash_testvec rmd320_tv_template[] = {
 	}
 };
 
-#define CRCT10DIF_TEST_VECTORS	3
+#define CRCT10DIF_TEST_VECTORS	ARRAY_SIZE(crct10dif_tv_template)
 static struct hash_testvec crct10dif_tv_template[] = {
 	{
-		.plaintext = "abc",
-		.psize  = 3,
-#ifdef __LITTLE_ENDIAN
-		.digest = "\x3b\x44",
-#else
-		.digest = "\x44\x3b",
-#endif
-	}, {
-		.plaintext = "1234567890123456789012345678901234567890"
-			     "123456789012345678901234567890123456789",
-		.psize	= 79,
-#ifdef __LITTLE_ENDIAN
-		.digest	= "\x70\x4b",
-#else
-		.digest	= "\x4b\x70",
-#endif
-	}, {
-		.plaintext =
-		"abcddddddddddddddddddddddddddddddddddddddddddddddddddddd",
-		.psize  = 56,
-#ifdef __LITTLE_ENDIAN
-		.digest = "\xe3\x9c",
-#else
-		.digest = "\x9c\xe3",
-#endif
-		.np     = 2,
-		.tap    = { 28, 28 }
+		.plaintext	= "abc",
+		.psize		= 3,
+		.digest		= (u8 *)(u16 []){ 0x443b },
+	}, {
+		.plaintext 	= "1234567890123456789012345678901234567890"
+				  "123456789012345678901234567890123456789",
+		.psize		= 79,
+		.digest 	= (u8 *)(u16 []){ 0x4b70 },
+		.np		= 2,
+		.tap		= { 63, 16 },
+	}, {
+		.plaintext	= "abcdddddddddddddddddddddddddddddddddddddddd"
+				  "ddddddddddddd",
+		.psize		= 56,
+		.digest		= (u8 *)(u16 []){ 0x9ce3 },
+		.np		= 8,
+		.tap		= { 1, 2, 28, 7, 6, 5, 4, 3 },
+	}, {
+		.plaintext 	= "1234567890123456789012345678901234567890"
+				  "1234567890123456789012345678901234567890"
+				  "1234567890123456789012345678901234567890"
+				  "1234567890123456789012345678901234567890"
+				  "1234567890123456789012345678901234567890"
+				  "1234567890123456789012345678901234567890"
+				  "1234567890123456789012345678901234567890"
+				  "123456789012345678901234567890123456789",
+		.psize		= 319,
+		.digest		= (u8 *)(u16 []){ 0x44c6 },
+	}, {
+		.plaintext 	= "1234567890123456789012345678901234567890"
+				  "1234567890123456789012345678901234567890"
+				  "1234567890123456789012345678901234567890"
+				  "1234567890123456789012345678901234567890"
+				  "1234567890123456789012345678901234567890"
+				  "1234567890123456789012345678901234567890"
+				  "1234567890123456789012345678901234567890"
+				  "123456789012345678901234567890123456789",
+		.psize		= 319,
+		.digest		= (u8 *)(u16 []){ 0x44c6 },
+		.np		= 4,
+		.tap		= { 1, 255, 57, 6 },
 	}
 };
 
diff --git a/crypto/xts.c b/crypto/xts.c
index 305343f22a02..410a2e299085 100644
--- a/crypto/xts.c
+++ b/crypto/xts.c
@@ -13,7 +13,8 @@
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  */
-#include <crypto/algapi.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
@@ -25,140 +26,320 @@
 #include <crypto/b128ops.h>
 #include <crypto/gf128mul.h>
 
+#define XTS_BUFFER_SIZE 128u
+
 struct priv {
-	struct crypto_cipher *child;
+	struct crypto_skcipher *child;
 	struct crypto_cipher *tweak;
 };
 
-static int setkey(struct crypto_tfm *parent, const u8 *key,
+struct xts_instance_ctx {
+	struct crypto_skcipher_spawn spawn;
+	char name[CRYPTO_MAX_ALG_NAME];
+};
+
+struct rctx {
+	be128 buf[XTS_BUFFER_SIZE / sizeof(be128)];
+
+	be128 t;
+
+	be128 *ext;
+
+	struct scatterlist srcbuf[2];
+	struct scatterlist dstbuf[2];
+	struct scatterlist *src;
+	struct scatterlist *dst;
+
+	unsigned int left;
+
+	struct skcipher_request subreq;
+};
+
+static int setkey(struct crypto_skcipher *parent, const u8 *key,
 		  unsigned int keylen)
 {
-	struct priv *ctx = crypto_tfm_ctx(parent);
-	struct crypto_cipher *child = ctx->tweak;
+	struct priv *ctx = crypto_skcipher_ctx(parent);
+	struct crypto_skcipher *child;
+	struct crypto_cipher *tweak;
 	int err;
 
-	err = xts_check_key(parent, key, keylen);
+	err = xts_verify_key(parent, key, keylen);
 	if (err)
 		return err;
 
+	keylen /= 2;
+
 	/* we need two cipher instances: one to compute the initial 'tweak'
 	 * by encrypting the IV (usually the 'plain' iv) and the other
 	 * one to encrypt and decrypt the data */
 
 	/* tweak cipher, uses Key2 i.e. the second half of *key */
-	crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
-	crypto_cipher_set_flags(child, crypto_tfm_get_flags(parent) &
+	tweak = ctx->tweak;
+	crypto_cipher_clear_flags(tweak, CRYPTO_TFM_REQ_MASK);
+	crypto_cipher_set_flags(tweak, crypto_skcipher_get_flags(parent) &
 				       CRYPTO_TFM_REQ_MASK);
-	err = crypto_cipher_setkey(child, key + keylen/2, keylen/2);
+	err = crypto_cipher_setkey(tweak, key + keylen, keylen);
+	crypto_skcipher_set_flags(parent, crypto_cipher_get_flags(tweak) &
+					  CRYPTO_TFM_RES_MASK);
 	if (err)
 		return err;
 
-	crypto_tfm_set_flags(parent, crypto_cipher_get_flags(child) &
-				     CRYPTO_TFM_RES_MASK);
-
+	/* data cipher, uses Key1 i.e. the first half of *key */
 	child = ctx->child;
+	crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
+	crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
+					 CRYPTO_TFM_REQ_MASK);
+	err = crypto_skcipher_setkey(child, key, keylen);
+	crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) &
+					  CRYPTO_TFM_RES_MASK);
 
-	/* data cipher, uses Key1 i.e. the first half of *key */
-	crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
-	crypto_cipher_set_flags(child, crypto_tfm_get_flags(parent) &
-				       CRYPTO_TFM_REQ_MASK);
-	err = crypto_cipher_setkey(child, key, keylen/2);
-	if (err)
-		return err;
+	return err;
+}
 
-	crypto_tfm_set_flags(parent, crypto_cipher_get_flags(child) &
-				     CRYPTO_TFM_RES_MASK);
+static int post_crypt(struct skcipher_request *req)
+{
+	struct rctx *rctx = skcipher_request_ctx(req);
+	be128 *buf = rctx->ext ?: rctx->buf;
+	struct skcipher_request *subreq;
+	const int bs = XTS_BLOCK_SIZE;
+	struct skcipher_walk w;
+	struct scatterlist *sg;
+	unsigned offset;
+	int err;
 
-	return 0;
-}
+	subreq = &rctx->subreq;
+	err = skcipher_walk_virt(&w, subreq, false);
 
-struct sinfo {
-	be128 *t;
-	struct crypto_tfm *tfm;
-	void (*fn)(struct crypto_tfm *, u8 *, const u8 *);
-};
+	while (w.nbytes) {
+		unsigned int avail = w.nbytes;
+		be128 *wdst;
 
-static inline void xts_round(struct sinfo *s, void *dst, const void *src)
-{
-	be128_xor(dst, s->t, src);		/* PP <- T xor P */
-	s->fn(s->tfm, dst, dst);		/* CC <- E(Key1,PP) */
-	be128_xor(dst, dst, s->t);		/* C <- T xor CC */
+		wdst = w.dst.virt.addr;
+
+		do {
+			be128_xor(wdst, buf++, wdst);
+			wdst++;
+		} while ((avail -= bs) >= bs);
+
+		err = skcipher_walk_done(&w, avail);
+	}
+
+	rctx->left -= subreq->cryptlen;
+
+	if (err || !rctx->left)
+		goto out;
+
+	rctx->dst = rctx->dstbuf;
+
+	scatterwalk_done(&w.out, 0, 1);
+	sg = w.out.sg;
+	offset = w.out.offset;
+
+	if (rctx->dst != sg) {
+		rctx->dst[0] = *sg;
+		sg_unmark_end(rctx->dst);
+		scatterwalk_crypto_chain(rctx->dst, sg_next(sg), 0, 2);
+	}
+	rctx->dst[0].length -= offset - sg->offset;
+	rctx->dst[0].offset = offset;
+
+out:
+	return err;
 }
 
-static int crypt(struct blkcipher_desc *d,
-		 struct blkcipher_walk *w, struct priv *ctx,
-		 void (*tw)(struct crypto_tfm *, u8 *, const u8 *),
-		 void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
+static int pre_crypt(struct skcipher_request *req)
 {
-	int err;
-	unsigned int avail;
+	struct rctx *rctx = skcipher_request_ctx(req);
+	be128 *buf = rctx->ext ?: rctx->buf;
+	struct skcipher_request *subreq;
 	const int bs = XTS_BLOCK_SIZE;
-	struct sinfo s = {
-		.tfm = crypto_cipher_tfm(ctx->child),
-		.fn = fn
-	};
-	u8 *wsrc;
-	u8 *wdst;
-
-	err = blkcipher_walk_virt(d, w);
-	if (!w->nbytes)
-		return err;
+	struct skcipher_walk w;
+	struct scatterlist *sg;
+	unsigned cryptlen;
+	unsigned offset;
+	bool more;
+	int err;
 
-	s.t = (be128 *)w->iv;
-	avail = w->nbytes;
+	subreq = &rctx->subreq;
+	cryptlen = subreq->cryptlen;
 
-	wsrc = w->src.virt.addr;
-	wdst = w->dst.virt.addr;
+	more = rctx->left > cryptlen;
+	if (!more)
+		cryptlen = rctx->left;
 
-	/* calculate first value of T */
-	tw(crypto_cipher_tfm(ctx->tweak), w->iv, w->iv);
+	skcipher_request_set_crypt(subreq, rctx->src, rctx->dst,
+				   cryptlen, NULL);
 
-	goto first;
+	err = skcipher_walk_virt(&w, subreq, false);
 
-	for (;;) {
-		do {
-			gf128mul_x_ble(s.t, s.t);
+	while (w.nbytes) {
+		unsigned int avail = w.nbytes;
+		be128 *wsrc;
+		be128 *wdst;
 
-first:
-			xts_round(&s, wdst, wsrc);
+		wsrc = w.src.virt.addr;
+		wdst = w.dst.virt.addr;
 
-			wsrc += bs;
-			wdst += bs;
+		do {
+			*buf++ = rctx->t;
+			be128_xor(wdst++, &rctx->t, wsrc++);
+			gf128mul_x_ble(&rctx->t, &rctx->t);
 		} while ((avail -= bs) >= bs);
 
-		err = blkcipher_walk_done(d, w, avail);
-		if (!w->nbytes)
-			break;
+		err = skcipher_walk_done(&w, avail);
+	}
+
+	skcipher_request_set_crypt(subreq, rctx->dst, rctx->dst,
+				   cryptlen, NULL);
 
-		avail = w->nbytes;
+	if (err || !more)
+		goto out;
 
-		wsrc = w->src.virt.addr;
-		wdst = w->dst.virt.addr;
+	rctx->src = rctx->srcbuf;
+
+	scatterwalk_done(&w.in, 0, 1);
+	sg = w.in.sg;
+	offset = w.in.offset;
+
+	if (rctx->src != sg) {
+		rctx->src[0] = *sg;
+		sg_unmark_end(rctx->src);
+		scatterwalk_crypto_chain(rctx->src, sg_next(sg), 0, 2);
 	}
+	rctx->src[0].length -= offset - sg->offset;
+	rctx->src[0].offset = offset;
 
+out:
 	return err;
 }
 
-static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		   struct scatterlist *src, unsigned int nbytes)
+static int init_crypt(struct skcipher_request *req, crypto_completion_t done)
 {
-	struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk w;
+	struct priv *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
+	struct rctx *rctx = skcipher_request_ctx(req);
+	struct skcipher_request *subreq;
+	gfp_t gfp;
+
+	subreq = &rctx->subreq;
+	skcipher_request_set_tfm(subreq, ctx->child);
+	skcipher_request_set_callback(subreq, req->base.flags, done, req);
+
+	gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
+							   GFP_ATOMIC;
+	rctx->ext = NULL;
+
+	subreq->cryptlen = XTS_BUFFER_SIZE;
+	if (req->cryptlen > XTS_BUFFER_SIZE) {
+		subreq->cryptlen = min(req->cryptlen, (unsigned)PAGE_SIZE);
+		rctx->ext = kmalloc(subreq->cryptlen, gfp);
+	}
+
+	rctx->src = req->src;
+	rctx->dst = req->dst;
+	rctx->left = req->cryptlen;
 
-	blkcipher_walk_init(&w, dst, src, nbytes);
-	return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt,
-		     crypto_cipher_alg(ctx->child)->cia_encrypt);
+	/* calculate first value of T */
+	crypto_cipher_encrypt_one(ctx->tweak, (u8 *)&rctx->t, req->iv);
+
+	return 0;
 }
 
-static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
-		   struct scatterlist *src, unsigned int nbytes)
+static void exit_crypt(struct skcipher_request *req)
 {
-	struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
-	struct blkcipher_walk w;
+	struct rctx *rctx = skcipher_request_ctx(req);
+
+	rctx->left = 0;
 
-	blkcipher_walk_init(&w, dst, src, nbytes);
-	return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt,
-		     crypto_cipher_alg(ctx->child)->cia_decrypt);
+	if (rctx->ext)
+		kzfree(rctx->ext);
+}
+
+static int do_encrypt(struct skcipher_request *req, int err)
+{
+	struct rctx *rctx = skcipher_request_ctx(req);
+	struct skcipher_request *subreq;
+
+	subreq = &rctx->subreq;
+
+	while (!err && rctx->left) {
+		err = pre_crypt(req) ?:
+		      crypto_skcipher_encrypt(subreq) ?:
+		      post_crypt(req);
+
+		if (err == -EINPROGRESS ||
+		    (err == -EBUSY &&
+		     req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+			return err;
+	}
+
+	exit_crypt(req);
+	return err;
+}
+
+static void encrypt_done(struct crypto_async_request *areq, int err)
+{
+	struct skcipher_request *req = areq->data;
+	struct skcipher_request *subreq;
+	struct rctx *rctx;
+
+	rctx = skcipher_request_ctx(req);
+	subreq = &rctx->subreq;
+	subreq->base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;
+
+	err = do_encrypt(req, err ?: post_crypt(req));
+	if (rctx->left)
+		return;
+
+	skcipher_request_complete(req, err);
+}
+
+static int encrypt(struct skcipher_request *req)
+{
+	return do_encrypt(req, init_crypt(req, encrypt_done));
+}
+
+static int do_decrypt(struct skcipher_request *req, int err)
+{
+	struct rctx *rctx = skcipher_request_ctx(req);
+	struct skcipher_request *subreq;
+
+	subreq = &rctx->subreq;
+
+	while (!err && rctx->left) {
+		err = pre_crypt(req) ?:
+		      crypto_skcipher_decrypt(subreq) ?:
+		      post_crypt(req);
+
+		if (err == -EINPROGRESS ||
+		    (err == -EBUSY &&
+		     req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+			return err;
+	}
+
+	exit_crypt(req);
+	return err;
+}
+
+static void decrypt_done(struct crypto_async_request *areq, int err)
+{
+	struct skcipher_request *req = areq->data;
+	struct skcipher_request *subreq;
+	struct rctx *rctx;
+
+	rctx = skcipher_request_ctx(req);
+	subreq = &rctx->subreq;
+	subreq->base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;
+
+	err = do_decrypt(req, err ?: post_crypt(req));
+	if (rctx->left)
+		return;
+
+	skcipher_request_complete(req, err);
+}
+
+static int decrypt(struct skcipher_request *req)
+{
+	return do_decrypt(req, init_crypt(req, decrypt_done));
 }
 
 int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
@@ -233,112 +414,168 @@ first:
 }
 EXPORT_SYMBOL_GPL(xts_crypt);
 
-static int init_tfm(struct crypto_tfm *tfm)
+static int init_tfm(struct crypto_skcipher *tfm)
 {
-	struct crypto_cipher *cipher;
-	struct crypto_instance *inst = (void *)tfm->__crt_alg;
-	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
-	struct priv *ctx = crypto_tfm_ctx(tfm);
-	u32 *flags = &tfm->crt_flags;
-
-	cipher = crypto_spawn_cipher(spawn);
-	if (IS_ERR(cipher))
-		return PTR_ERR(cipher);
-
-	if (crypto_cipher_blocksize(cipher) != XTS_BLOCK_SIZE) {
-		*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
-		crypto_free_cipher(cipher);
-		return -EINVAL;
-	}
+	struct skcipher_instance *inst = skcipher_alg_instance(tfm);
+	struct xts_instance_ctx *ictx = skcipher_instance_ctx(inst);
+	struct priv *ctx = crypto_skcipher_ctx(tfm);
+	struct crypto_skcipher *child;
+	struct crypto_cipher *tweak;
 
-	ctx->child = cipher;
+	child = crypto_spawn_skcipher(&ictx->spawn);
+	if (IS_ERR(child))
+		return PTR_ERR(child);
 
-	cipher = crypto_spawn_cipher(spawn);
-	if (IS_ERR(cipher)) {
-		crypto_free_cipher(ctx->child);
-		return PTR_ERR(cipher);
-	}
+	ctx->child = child;
 
-	/* this check isn't really needed, leave it here just in case */
-	if (crypto_cipher_blocksize(cipher) != XTS_BLOCK_SIZE) {
-		crypto_free_cipher(cipher);
-		crypto_free_cipher(ctx->child);
-		*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
-		return -EINVAL;
+	tweak = crypto_alloc_cipher(ictx->name, 0, 0);
+	if (IS_ERR(tweak)) {
+		crypto_free_skcipher(ctx->child);
+		return PTR_ERR(tweak);
 	}
 
-	ctx->tweak = cipher;
+	ctx->tweak = tweak;
+
+	crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) +
+					 sizeof(struct rctx));
 
 	return 0;
 }
 
-static void exit_tfm(struct crypto_tfm *tfm)
+static void exit_tfm(struct crypto_skcipher *tfm)
 {
-	struct priv *ctx = crypto_tfm_ctx(tfm);
-	crypto_free_cipher(ctx->child);
+	struct priv *ctx = crypto_skcipher_ctx(tfm);
+
+	crypto_free_skcipher(ctx->child);
 	crypto_free_cipher(ctx->tweak);
 }
 
-static struct crypto_instance *alloc(struct rtattr **tb)
+static void free(struct skcipher_instance *inst)
+{
+	crypto_drop_skcipher(skcipher_instance_ctx(inst));
+	kfree(inst);
+}
+
+static int create(struct crypto_template *tmpl, struct rtattr **tb)
 {
-	struct crypto_instance *inst;
-	struct crypto_alg *alg;
+	struct skcipher_instance *inst;
+	struct crypto_attr_type *algt;
+	struct xts_instance_ctx *ctx;
+	struct skcipher_alg *alg;
+	const char *cipher_name;
 	int err;
 
-	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
+	algt = crypto_get_attr_type(tb);
+	if (IS_ERR(algt))
+		return PTR_ERR(algt);
+
+	if ((algt->type ^ CRYPTO_ALG_TYPE_SKCIPHER) & algt->mask)
+		return -EINVAL;
+
+	cipher_name = crypto_attr_alg_name(tb[1]);
+	if (IS_ERR(cipher_name))
+		return PTR_ERR(cipher_name);
+
+	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
+	if (!inst)
+		return -ENOMEM;
+
+	ctx = skcipher_instance_ctx(inst);
+
+	crypto_set_skcipher_spawn(&ctx->spawn, skcipher_crypto_instance(inst));
+	err = crypto_grab_skcipher(&ctx->spawn, cipher_name, 0,
+				   crypto_requires_sync(algt->type,
+							algt->mask));
+	if (err == -ENOENT) {
+		err = -ENAMETOOLONG;
+		if (snprintf(ctx->name, CRYPTO_MAX_ALG_NAME, "ecb(%s)",
+			     cipher_name) >= CRYPTO_MAX_ALG_NAME)
+			goto err_free_inst;
+
+		err = crypto_grab_skcipher(&ctx->spawn, ctx->name, 0,
+					   crypto_requires_sync(algt->type,
+								algt->mask));
+	}
+
 	if (err)
-		return ERR_PTR(err);
+		goto err_free_inst;
 
-	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
-				  CRYPTO_ALG_TYPE_MASK);
-	if (IS_ERR(alg))
-		return ERR_CAST(alg);
+	alg = crypto_skcipher_spawn_alg(&ctx->spawn);
 
-	inst = crypto_alloc_instance("xts", alg);
-	if (IS_ERR(inst))
-		goto out_put_alg;
+	err = -EINVAL;
+	if (alg->base.cra_blocksize != XTS_BLOCK_SIZE)
+		goto err_drop_spawn;
 
-	inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
-	inst->alg.cra_priority = alg->cra_priority;
-	inst->alg.cra_blocksize = alg->cra_blocksize;
+	if (crypto_skcipher_alg_ivsize(alg))
+		goto err_drop_spawn;
 
-	if (alg->cra_alignmask < 7)
-		inst->alg.cra_alignmask = 7;
-	else
-		inst->alg.cra_alignmask = alg->cra_alignmask;
+	err = crypto_inst_setname(skcipher_crypto_instance(inst), "xts",
+				  &alg->base);
+	if (err)
+		goto err_drop_spawn;
 
-	inst->alg.cra_type = &crypto_blkcipher_type;
+	err = -EINVAL;
+	cipher_name = alg->base.cra_name;
 
-	inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
-	inst->alg.cra_blkcipher.min_keysize =
-		2 * alg->cra_cipher.cia_min_keysize;
-	inst->alg.cra_blkcipher.max_keysize =
-		2 * alg->cra_cipher.cia_max_keysize;
+	/* Alas we screwed up the naming so we have to mangle the
+	 * cipher name.
+	 */
+	if (!strncmp(cipher_name, "ecb(", 4)) {
+		unsigned len;
 
-	inst->alg.cra_ctxsize = sizeof(struct priv);
+		len = strlcpy(ctx->name, cipher_name + 4, sizeof(ctx->name));
+		if (len < 2 || len >= sizeof(ctx->name))
+			goto err_drop_spawn;
 
-	inst->alg.cra_init = init_tfm;
-	inst->alg.cra_exit = exit_tfm;
+		if (ctx->name[len - 1] != ')')
+			goto err_drop_spawn;
 
-	inst->alg.cra_blkcipher.setkey = setkey;
-	inst->alg.cra_blkcipher.encrypt = encrypt;
-	inst->alg.cra_blkcipher.decrypt = decrypt;
+		ctx->name[len - 1] = 0;
 
-out_put_alg:
-	crypto_mod_put(alg);
-	return inst;
-}
+		if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
+			     "xts(%s)", ctx->name) >= CRYPTO_MAX_ALG_NAME)
+			return -ENAMETOOLONG;
+	} else
+		goto err_drop_spawn;
 
-static void free(struct crypto_instance *inst)
-{
-	crypto_drop_spawn(crypto_instance_ctx(inst));
+	inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
+	inst->alg.base.cra_priority = alg->base.cra_priority;
+	inst->alg.base.cra_blocksize = XTS_BLOCK_SIZE;
+	inst->alg.base.cra_alignmask = alg->base.cra_alignmask |
+				       (__alignof__(u64) - 1);
+
+	inst->alg.ivsize = XTS_BLOCK_SIZE;
+	inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg) * 2;
+	inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg) * 2;
+
+	inst->alg.base.cra_ctxsize = sizeof(struct priv);
+
+	inst->alg.init = init_tfm;
+	inst->alg.exit = exit_tfm;
+
+	inst->alg.setkey = setkey;
+	inst->alg.encrypt = encrypt;
+	inst->alg.decrypt = decrypt;
+
+	inst->free = free;
+
+	err = skcipher_register_instance(tmpl, inst);
+	if (err)
+		goto err_drop_spawn;
+
+out:
+	return err;
+
+err_drop_spawn:
+	crypto_drop_skcipher(&ctx->spawn);
+err_free_inst:
 	kfree(inst);
+	goto out;
 }
 
 static struct crypto_template crypto_tmpl = {
 	.name = "xts",
-	.alloc = alloc,
-	.free = free,
+	.create = create,
 	.module = THIS_MODULE,
 };
 
diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig
index c5f9cbe0ae21..83e5f7e1a20d 100644
--- a/drivers/acpi/Kconfig
+++ b/drivers/acpi/Kconfig
@@ -104,7 +104,7 @@ config ACPI_PROCFS_POWER
 	  Say N to delete power /proc/acpi/ directories that have moved to /sys/
 
 config ACPI_REV_OVERRIDE_POSSIBLE
-	bool "Allow supported ACPI revision to be overriden"
+	bool "Allow supported ACPI revision to be overridden"
 	depends on X86
 	default y
 	help
diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c
index 7dd70927991e..26696b693e63 100644
--- a/drivers/acpi/acpi_apd.c
+++ b/drivers/acpi/acpi_apd.c
@@ -77,6 +77,11 @@ static const struct apd_device_desc cz_i2c_desc = {
 	.fixed_clk_rate = 133000000,
 };
 
+static const struct apd_device_desc wt_i2c_desc = {
+	.setup = acpi_apd_setup,
+	.fixed_clk_rate = 150000000,
+};
+
 static struct property_entry uart_properties[] = {
 	PROPERTY_ENTRY_U32("reg-io-width", 4),
 	PROPERTY_ENTRY_U32("reg-shift", 2),
@@ -156,7 +161,7 @@ static const struct acpi_device_id acpi_apd_device_ids[] = {
 	/* Generic apd devices */
 #ifdef CONFIG_X86_AMD_PLATFORM_DEVICE
 	{ "AMD0010", APD_ADDR(cz_i2c_desc) },
-	{ "AMDI0010", APD_ADDR(cz_i2c_desc) },
+	{ "AMDI0010", APD_ADDR(wt_i2c_desc) },
 	{ "AMD0020", APD_ADDR(cz_uart_desc) },
 	{ "AMDI0020", APD_ADDR(cz_uart_desc) },
 	{ "AMD0030", },
diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c
index 373657f7e35a..8ea836c046f8 100644
--- a/drivers/acpi/acpi_lpss.c
+++ b/drivers/acpi/acpi_lpss.c
@@ -718,13 +718,14 @@ static int acpi_lpss_resume_early(struct device *dev)
 #define LPSS_GPIODEF0_DMA1_D3		BIT(2)
 #define LPSS_GPIODEF0_DMA2_D3		BIT(3)
 #define LPSS_GPIODEF0_DMA_D3_MASK	GENMASK(3, 2)
+#define LPSS_GPIODEF0_DMA_LLP		BIT(13)
 
 static DEFINE_MUTEX(lpss_iosf_mutex);
 
 static void lpss_iosf_enter_d3_state(void)
 {
 	u32 value1 = 0;
-	u32 mask1 = LPSS_GPIODEF0_DMA_D3_MASK;
+	u32 mask1 = LPSS_GPIODEF0_DMA_D3_MASK | LPSS_GPIODEF0_DMA_LLP;
 	u32 value2 = LPSS_PMCSR_D3hot;
 	u32 mask2 = LPSS_PMCSR_Dx_MASK;
 	/*
@@ -768,8 +769,9 @@ exit:
 
 static void lpss_iosf_exit_d3_state(void)
 {
-	u32 value1 = LPSS_GPIODEF0_DMA1_D3 | LPSS_GPIODEF0_DMA2_D3;
-	u32 mask1 = LPSS_GPIODEF0_DMA_D3_MASK;
+	u32 value1 = LPSS_GPIODEF0_DMA1_D3 | LPSS_GPIODEF0_DMA2_D3 |
+		     LPSS_GPIODEF0_DMA_LLP;
+	u32 mask1 = LPSS_GPIODEF0_DMA_D3_MASK | LPSS_GPIODEF0_DMA_LLP;
 	u32 value2 = LPSS_PMCSR_D0;
 	u32 mask2 = LPSS_PMCSR_Dx_MASK;
 
diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c
index c5557d070954..201292e67ee8 100644
--- a/drivers/acpi/acpi_video.c
+++ b/drivers/acpi/acpi_video.c
@@ -43,17 +43,6 @@
 
 #define ACPI_VIDEO_BUS_NAME		"Video Bus"
 #define ACPI_VIDEO_DEVICE_NAME		"Video Device"
-#define ACPI_VIDEO_NOTIFY_SWITCH	0x80
-#define ACPI_VIDEO_NOTIFY_PROBE		0x81
-#define ACPI_VIDEO_NOTIFY_CYCLE		0x82
-#define ACPI_VIDEO_NOTIFY_NEXT_OUTPUT	0x83
-#define ACPI_VIDEO_NOTIFY_PREV_OUTPUT	0x84
-
-#define ACPI_VIDEO_NOTIFY_CYCLE_BRIGHTNESS	0x85
-#define	ACPI_VIDEO_NOTIFY_INC_BRIGHTNESS	0x86
-#define ACPI_VIDEO_NOTIFY_DEC_BRIGHTNESS	0x87
-#define ACPI_VIDEO_NOTIFY_ZERO_BRIGHTNESS	0x88
-#define ACPI_VIDEO_NOTIFY_DISPLAY_OFF		0x89
 
 #define MAX_NAME_LEN	20
 
diff --git a/drivers/acpi/acpica/acevents.h b/drivers/acpi/acpica/acevents.h
index 92fa47c6498c..8a0049d5cdf3 100644
--- a/drivers/acpi/acpica/acevents.h
+++ b/drivers/acpi/acpica/acevents.h
@@ -243,9 +243,7 @@ acpi_ev_default_region_setup(acpi_handle handle,
 			     u32 function,
 			     void *handler_context, void **region_context);
 
-acpi_status
-acpi_ev_initialize_region(union acpi_operand_object *region_obj,
-			  u8 acpi_ns_locked);
+acpi_status acpi_ev_initialize_region(union acpi_operand_object *region_obj);
 
 /*
  * evsci - SCI (System Control Interrupt) handling/dispatch
diff --git a/drivers/acpi/acpica/acglobal.h b/drivers/acpi/acpica/acglobal.h
index 750fa824d42c..edbb42e251a6 100644
--- a/drivers/acpi/acpica/acglobal.h
+++ b/drivers/acpi/acpica/acglobal.h
@@ -240,10 +240,6 @@ ACPI_INIT_GLOBAL(u32, acpi_gbl_nesting_level, 0);
 
 ACPI_GLOBAL(struct acpi_thread_state *, acpi_gbl_current_walk_list);
 
-/* Maximum number of While() loop iterations before forced abort */
-
-ACPI_GLOBAL(u16, acpi_gbl_max_loop_iterations);
-
 /* Control method single step flag */
 
 ACPI_GLOBAL(u8, acpi_gbl_cm_single_step);
@@ -318,6 +314,7 @@ ACPI_INIT_GLOBAL(u8, acpi_gbl_cstyle_disassembly, TRUE);
 ACPI_INIT_GLOBAL(u8, acpi_gbl_force_aml_disassembly, FALSE);
 ACPI_INIT_GLOBAL(u8, acpi_gbl_dm_opt_verbose, TRUE);
 ACPI_INIT_GLOBAL(u8, acpi_gbl_dm_emit_external_opcodes, FALSE);
+ACPI_INIT_GLOBAL(u8, acpi_gbl_do_disassembler_optimizations, TRUE);
 
 ACPI_GLOBAL(u8, acpi_gbl_dm_opt_disasm);
 ACPI_GLOBAL(u8, acpi_gbl_dm_opt_listing);
diff --git a/drivers/acpi/acpica/aclocal.h b/drivers/acpi/acpica/aclocal.h
index dff1207a6078..792660054992 100644
--- a/drivers/acpi/acpica/aclocal.h
+++ b/drivers/acpi/acpica/aclocal.h
@@ -765,7 +765,7 @@ union acpi_parse_value {
 	union acpi_parse_value          value;          /* Value or args associated with the opcode */\
 	u8                              arg_list_length; /* Number of elements in the arg list */\
 	ACPI_DISASM_ONLY_MEMBERS (\
-	u8                              disasm_flags;   /* Used during AML disassembly */\
+	u16                             disasm_flags;   /* Used during AML disassembly */\
 	u8                              disasm_opcode;  /* Subtype used for disassembly */\
 	char                            *operator_symbol;/* Used for C-style operator name strings */\
 	char                            aml_op_name[16])	/* Op name (debug only) */
@@ -868,14 +868,15 @@ struct acpi_parse_state {
 
 /* Parse object disasm_flags */
 
-#define ACPI_PARSEOP_IGNORE                 0x01
-#define ACPI_PARSEOP_PARAMETER_LIST         0x02
-#define ACPI_PARSEOP_EMPTY_TERMLIST         0x04
-#define ACPI_PARSEOP_PREDEFINED_CHECKED     0x08
-#define ACPI_PARSEOP_CLOSING_PAREN          0x10
-#define ACPI_PARSEOP_COMPOUND_ASSIGNMENT    0x20
-#define ACPI_PARSEOP_ASSIGNMENT             0x40
-#define ACPI_PARSEOP_ELSEIF                 0x80
+#define ACPI_PARSEOP_IGNORE                 0x0001
+#define ACPI_PARSEOP_PARAMETER_LIST         0x0002
+#define ACPI_PARSEOP_EMPTY_TERMLIST         0x0004
+#define ACPI_PARSEOP_PREDEFINED_CHECKED     0x0008
+#define ACPI_PARSEOP_CLOSING_PAREN          0x0010
+#define ACPI_PARSEOP_COMPOUND_ASSIGNMENT    0x0020
+#define ACPI_PARSEOP_ASSIGNMENT             0x0040
+#define ACPI_PARSEOP_ELSEIF                 0x0080
+#define ACPI_PARSEOP_LEGACY_ASL_ONLY        0x0100
 
 /*****************************************************************************
  *
diff --git a/drivers/acpi/acpica/acnamesp.h b/drivers/acpi/acpica/acnamesp.h
index bb7fca1c8ba3..7affdcdfcc81 100644
--- a/drivers/acpi/acpica/acnamesp.h
+++ b/drivers/acpi/acpica/acnamesp.h
@@ -292,6 +292,9 @@ char *acpi_ns_get_normalized_pathname(struct acpi_namespace_node *node,
 char *acpi_ns_name_of_current_scope(struct acpi_walk_state *walk_state);
 
 acpi_status
+acpi_ns_handle_to_name(acpi_handle target_handle, struct acpi_buffer *buffer);
+
+acpi_status
 acpi_ns_handle_to_pathname(acpi_handle target_handle,
 			   struct acpi_buffer *buffer, u8 no_trailing);
 
diff --git a/drivers/acpi/acpica/actables.h b/drivers/acpi/acpica/actables.h
index e85953b6fa0e..7dd527f8ca1d 100644
--- a/drivers/acpi/acpica/actables.h
+++ b/drivers/acpi/acpica/actables.h
@@ -127,10 +127,11 @@ acpi_status
 acpi_tb_load_table(u32 table_index, struct acpi_namespace_node *parent_node);
 
 acpi_status
-acpi_tb_install_and_load_table(struct acpi_table_header *table,
-			       acpi_physical_address address,
+acpi_tb_install_and_load_table(acpi_physical_address address,
 			       u8 flags, u8 override, u32 *table_index);
 
+acpi_status acpi_tb_unload_table(u32 table_index);
+
 void acpi_tb_terminate(void);
 
 acpi_status acpi_tb_delete_namespace_by_owner(u32 table_index);
diff --git a/drivers/acpi/acpica/acutils.h b/drivers/acpi/acpica/acutils.h
index 0a1b53c9ee0e..845afb180a7e 100644
--- a/drivers/acpi/acpica/acutils.h
+++ b/drivers/acpi/acpica/acutils.h
@@ -232,6 +232,8 @@ const char *acpi_ut_get_region_name(u8 space_id);
 
 const char *acpi_ut_get_event_name(u32 event_id);
 
+const char *acpi_ut_get_argument_type_name(u32 arg_type);
+
 char acpi_ut_hex_to_ascii_char(u64 integer, u32 position);
 
 acpi_status acpi_ut_ascii_to_hex_byte(char *two_ascii_chars, u8 *return_byte);
diff --git a/drivers/acpi/acpica/amlcode.h b/drivers/acpi/acpica/amlcode.h
index ceb4f7365f7f..6bd8d4bcff65 100644
--- a/drivers/acpi/acpica/amlcode.h
+++ b/drivers/acpi/acpica/amlcode.h
@@ -240,6 +240,7 @@
 #define ARGP_QWORDDATA              0x11
 #define ARGP_SIMPLENAME             0x12	/* name_string | local_term | arg_term */
 #define ARGP_NAME_OR_REF            0x13	/* For object_type only */
+#define ARGP_MAX                    0x13
 
 /*
  * Resolved argument types for the AML Interpreter
diff --git a/drivers/acpi/acpica/dsinit.c b/drivers/acpi/acpica/dsinit.c
index 54d48b90de2c..5de3f10cab03 100644
--- a/drivers/acpi/acpica/dsinit.c
+++ b/drivers/acpi/acpica/dsinit.c
@@ -221,8 +221,8 @@ acpi_ds_initialize_objects(u32 table_index,
 	 */
 	status =
 	    acpi_ns_walk_namespace(ACPI_TYPE_ANY, start_node, ACPI_UINT32_MAX,
-				   0, acpi_ds_init_one_object, NULL, &info,
-				   NULL);
+				   ACPI_NS_WALK_NO_UNLOCK,
+				   acpi_ds_init_one_object, NULL, &info, NULL);
 	if (ACPI_FAILURE(status)) {
 		ACPI_EXCEPTION((AE_INFO, status, "During WalkNamespace"));
 	}
diff --git a/drivers/acpi/acpica/dsopcode.c b/drivers/acpi/acpica/dsopcode.c
index 4cc9d989a114..77fd7c84ec39 100644
--- a/drivers/acpi/acpica/dsopcode.c
+++ b/drivers/acpi/acpica/dsopcode.c
@@ -84,7 +84,7 @@ acpi_status acpi_ds_initialize_region(acpi_handle obj_handle)
 
 	/* Namespace is NOT locked */
 
-	status = acpi_ev_initialize_region(obj_desc, FALSE);
+	status = acpi_ev_initialize_region(obj_desc);
 	return (status);
 }
 
diff --git a/drivers/acpi/acpica/dswload2.c b/drivers/acpi/acpica/dswload2.c
index e36218206bb0..651f35a66cc2 100644
--- a/drivers/acpi/acpica/dswload2.c
+++ b/drivers/acpi/acpica/dswload2.c
@@ -609,18 +609,7 @@ acpi_status acpi_ds_load2_end_op(struct acpi_walk_state *walk_state)
 
 			status =
 			    acpi_ev_initialize_region
-			    (acpi_ns_get_attached_object(node), FALSE);
-
-			if (ACPI_FAILURE(status)) {
-				/*
-				 *  If AE_NOT_EXIST is returned, it is not fatal
-				 *  because many regions get created before a handler
-				 *  is installed for said region.
-				 */
-				if (AE_NOT_EXIST == status) {
-					status = AE_OK;
-				}
-			}
+			    (acpi_ns_get_attached_object(node));
 			break;
 
 		case AML_NAME_OP:
diff --git a/drivers/acpi/acpica/evrgnini.c b/drivers/acpi/acpica/evrgnini.c
index 75ddd160a716..a9092251ce80 100644
--- a/drivers/acpi/acpica/evrgnini.c
+++ b/drivers/acpi/acpica/evrgnini.c
@@ -479,7 +479,6 @@ acpi_ev_default_region_setup(acpi_handle handle,
  * FUNCTION:    acpi_ev_initialize_region
  *
  * PARAMETERS:  region_obj      - Region we are initializing
- *              acpi_ns_locked  - Is namespace locked?
  *
  * RETURN:      Status
  *
@@ -497,19 +496,28 @@ acpi_ev_default_region_setup(acpi_handle handle,
  * MUTEX:       Interpreter should be unlocked, because we may run the _REG
  *              method for this region.
  *
+ * NOTE:        Possible incompliance:
+ *              There is a behavior conflict in automatic _REG execution:
+ *              1. When the interpreter is evaluating a method, we can only
+ *                 automatically run _REG for the following case:
+ *                   operation_region (OPR1, 0x80, 0x1000010, 0x4)
+ *              2. When the interpreter is loading a table, we can also
+ *                 automatically run _REG for the following case:
+ *                   operation_region (OPR1, 0x80, 0x1000010, 0x4)
+ *              Though this may not be compliant to the de-facto standard, the
+ *              logic is kept in order not to trigger regressions. And keeping
+ *              this logic should be taken care by the caller of this function.
+ *
  ******************************************************************************/
 
-acpi_status
-acpi_ev_initialize_region(union acpi_operand_object *region_obj,
-			  u8 acpi_ns_locked)
+acpi_status acpi_ev_initialize_region(union acpi_operand_object *region_obj)
 {
 	union acpi_operand_object *handler_obj;
 	union acpi_operand_object *obj_desc;
 	acpi_adr_space_type space_id;
 	struct acpi_namespace_node *node;
-	acpi_status status;
 
-	ACPI_FUNCTION_TRACE_U32(ev_initialize_region, acpi_ns_locked);
+	ACPI_FUNCTION_TRACE(ev_initialize_region);
 
 	if (!region_obj) {
 		return_ACPI_STATUS(AE_BAD_PARAMETER);
@@ -580,39 +588,17 @@ acpi_ev_initialize_region(union acpi_operand_object *region_obj,
 						  handler_obj, region_obj,
 						  obj_desc));
 
-				status =
-				    acpi_ev_attach_region(handler_obj,
-							  region_obj,
-							  acpi_ns_locked);
+				(void)acpi_ev_attach_region(handler_obj,
+							    region_obj, FALSE);
 
 				/*
 				 * Tell all users that this region is usable by
 				 * running the _REG method
 				 */
-				if (acpi_ns_locked) {
-					status =
-					    acpi_ut_release_mutex
-					    (ACPI_MTX_NAMESPACE);
-					if (ACPI_FAILURE(status)) {
-						return_ACPI_STATUS(status);
-					}
-				}
-
 				acpi_ex_exit_interpreter();
-				status =
-				    acpi_ev_execute_reg_method(region_obj,
-							       ACPI_REG_CONNECT);
+				(void)acpi_ev_execute_reg_method(region_obj,
+								 ACPI_REG_CONNECT);
 				acpi_ex_enter_interpreter();
-
-				if (acpi_ns_locked) {
-					status =
-					    acpi_ut_acquire_mutex
-					    (ACPI_MTX_NAMESPACE);
-					if (ACPI_FAILURE(status)) {
-						return_ACPI_STATUS(status);
-					}
-				}
-
 				return_ACPI_STATUS(AE_OK);
 			}
 		}
@@ -622,12 +608,15 @@ acpi_ev_initialize_region(union acpi_operand_object *region_obj,
 		node = node->parent;
 	}
 
-	/* If we get here, there is no handler for this region */
-
+	/*
+	 * If we get here, there is no handler for this region. This is not
+	 * fatal because many regions get created before a handler is installed
+	 * for said region.
+	 */
 	ACPI_DEBUG_PRINT((ACPI_DB_OPREGION,
 			  "No handler for RegionType %s(%X) (RegionObj %p)\n",
 			  acpi_ut_get_region_name(space_id), space_id,
 			  region_obj));
 
-	return_ACPI_STATUS(AE_NOT_EXIST);
+	return_ACPI_STATUS(AE_OK);
 }
diff --git a/drivers/acpi/acpica/exconfig.c b/drivers/acpi/acpica/exconfig.c
index 718428ba0b89..c32c7829878a 100644
--- a/drivers/acpi/acpica/exconfig.c
+++ b/drivers/acpi/acpica/exconfig.c
@@ -437,10 +437,9 @@ acpi_ex_load_op(union acpi_operand_object *obj_desc,
 
 	ACPI_INFO(("Dynamic OEM Table Load:"));
 	acpi_ex_exit_interpreter();
-	status =
-	    acpi_tb_install_and_load_table(table, ACPI_PTR_TO_PHYSADDR(table),
-					   ACPI_TABLE_ORIGIN_INTERNAL_VIRTUAL,
-					   TRUE, &table_index);
+	status = acpi_tb_install_and_load_table(ACPI_PTR_TO_PHYSADDR(table),
+						ACPI_TABLE_ORIGIN_INTERNAL_VIRTUAL,
+						TRUE, &table_index);
 	acpi_ex_enter_interpreter();
 	if (ACPI_FAILURE(status)) {
 
@@ -500,7 +499,6 @@ acpi_status acpi_ex_unload_table(union acpi_operand_object *ddb_handle)
 	acpi_status status = AE_OK;
 	union acpi_operand_object *table_desc = ddb_handle;
 	u32 table_index;
-	struct acpi_table_header *table;
 
 	ACPI_FUNCTION_TRACE(ex_unload_table);
 
@@ -537,39 +535,7 @@ acpi_status acpi_ex_unload_table(union acpi_operand_object *ddb_handle)
 	 * strict order requirement against it.
 	 */
 	acpi_ex_exit_interpreter();
-
-	/* Ensure the table is still loaded */
-
-	if (!acpi_tb_is_table_loaded(table_index)) {
-		status = AE_NOT_EXIST;
-		goto lock_and_exit;
-	}
-
-	/* Invoke table handler if present */
-
-	if (acpi_gbl_table_handler) {
-		status = acpi_get_table_by_index(table_index, &table);
-		if (ACPI_SUCCESS(status)) {
-			(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_UNLOAD,
-						     table,
-						     acpi_gbl_table_handler_context);
-		}
-	}
-
-	/* Delete the portion of the namespace owned by this table */
-
-	status = acpi_tb_delete_namespace_by_owner(table_index);
-	if (ACPI_FAILURE(status)) {
-		goto lock_and_exit;
-	}
-
-	(void)acpi_tb_release_owner_id(table_index);
-	acpi_tb_set_table_loaded_flag(table_index, FALSE);
-
-lock_and_exit:
-
-	/* Re-acquire the interpreter lock */
-
+	status = acpi_tb_unload_table(table_index);
 	acpi_ex_enter_interpreter();
 
 	/*
diff --git a/drivers/acpi/acpica/nsnames.c b/drivers/acpi/acpica/nsnames.c
index f03dd41e86d0..94d5d3339845 100644
--- a/drivers/acpi/acpica/nsnames.c
+++ b/drivers/acpi/acpica/nsnames.c
@@ -97,6 +97,51 @@ acpi_size acpi_ns_get_pathname_length(struct acpi_namespace_node *node)
 
 /*******************************************************************************
  *
+ * FUNCTION:    acpi_ns_handle_to_name
+ *
+ * PARAMETERS:  target_handle           - Handle of named object whose name is
+ *                                        to be found
+ *              buffer                  - Where the name is returned
+ *
+ * RETURN:      Status, Buffer is filled with name if status is AE_OK
+ *
+ * DESCRIPTION: Build and return a full namespace name
+ *
+ ******************************************************************************/
+
+acpi_status
+acpi_ns_handle_to_name(acpi_handle target_handle, struct acpi_buffer *buffer)
+{
+	acpi_status status;
+	struct acpi_namespace_node *node;
+	const char *node_name;
+
+	ACPI_FUNCTION_TRACE_PTR(ns_handle_to_name, target_handle);
+
+	node = acpi_ns_validate_handle(target_handle);
+	if (!node) {
+		return_ACPI_STATUS(AE_BAD_PARAMETER);
+	}
+
+	/* Validate/Allocate/Clear caller buffer */
+
+	status = acpi_ut_initialize_buffer(buffer, ACPI_PATH_SEGMENT_LENGTH);
+	if (ACPI_FAILURE(status)) {
+		return_ACPI_STATUS(status);
+	}
+
+	/* Just copy the ACPI name from the Node and zero terminate it */
+
+	node_name = acpi_ut_get_node_name(node);
+	ACPI_MOVE_NAME(buffer->pointer, node_name);
+	((char *)buffer->pointer)[ACPI_NAME_SIZE] = 0;
+
+	ACPI_DEBUG_PRINT((ACPI_DB_EXEC, "%4.4s\n", (char *)buffer->pointer));
+	return_ACPI_STATUS(AE_OK);
+}
+
+/*******************************************************************************
+ *
  * FUNCTION:    acpi_ns_handle_to_pathname
  *
  * PARAMETERS:  target_handle           - Handle of named object whose name is
diff --git a/drivers/acpi/acpica/nsxfname.c b/drivers/acpi/acpica/nsxfname.c
index 76a1bd4bb070..e525cbe7d83b 100644
--- a/drivers/acpi/acpica/nsxfname.c
+++ b/drivers/acpi/acpica/nsxfname.c
@@ -158,8 +158,6 @@ acpi_status
 acpi_get_name(acpi_handle handle, u32 name_type, struct acpi_buffer *buffer)
 {
 	acpi_status status;
-	struct acpi_namespace_node *node;
-	const char *node_name;
 
 	/* Parameter validation */
 
@@ -172,18 +170,6 @@ acpi_get_name(acpi_handle handle, u32 name_type, struct acpi_buffer *buffer)
 		return (status);
 	}
 
-	if (name_type == ACPI_FULL_PATHNAME ||
-	    name_type == ACPI_FULL_PATHNAME_NO_TRAILING) {
-
-		/* Get the full pathname (From the namespace root) */
-
-		status = acpi_ns_handle_to_pathname(handle, buffer,
-						    name_type ==
-						    ACPI_FULL_PATHNAME ? FALSE :
-						    TRUE);
-		return (status);
-	}
-
 	/*
 	 * Wants the single segment ACPI name.
 	 * Validate handle and convert to a namespace Node
@@ -193,27 +179,20 @@ acpi_get_name(acpi_handle handle, u32 name_type, struct acpi_buffer *buffer)
 		return (status);
 	}
 
-	node = acpi_ns_validate_handle(handle);
-	if (!node) {
-		status = AE_BAD_PARAMETER;
-		goto unlock_and_exit;
-	}
-
-	/* Validate/Allocate/Clear caller buffer */
-
-	status = acpi_ut_initialize_buffer(buffer, ACPI_PATH_SEGMENT_LENGTH);
-	if (ACPI_FAILURE(status)) {
-		goto unlock_and_exit;
-	}
+	if (name_type == ACPI_FULL_PATHNAME ||
+	    name_type == ACPI_FULL_PATHNAME_NO_TRAILING) {
 
-	/* Just copy the ACPI name from the Node and zero terminate it */
+		/* Get the full pathname (From the namespace root) */
 
-	node_name = acpi_ut_get_node_name(node);
-	ACPI_MOVE_NAME(buffer->pointer, node_name);
-	((char *)buffer->pointer)[ACPI_NAME_SIZE] = 0;
-	status = AE_OK;
+		status = acpi_ns_handle_to_pathname(handle, buffer,
+						    name_type ==
+						    ACPI_FULL_PATHNAME ? FALSE :
+						    TRUE);
+	} else {
+		/* Get the single name */
 
-unlock_and_exit:
+		status = acpi_ns_handle_to_name(handle, buffer);
+	}
 
 	(void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE);
 	return (status);
diff --git a/drivers/acpi/acpica/tbdata.c b/drivers/acpi/acpica/tbdata.c
index d9ca8c2aa2d3..82b0b5710979 100644
--- a/drivers/acpi/acpica/tbdata.c
+++ b/drivers/acpi/acpica/tbdata.c
@@ -832,9 +832,9 @@ acpi_tb_load_table(u32 table_index, struct acpi_namespace_node *parent_node)
  *
  * FUNCTION:    acpi_tb_install_and_load_table
  *
- * PARAMETERS:  table                   - Pointer to the table
- *              address                 - Physical address of the table
+ * PARAMETERS:  address                 - Physical address of the table
  *              flags                   - Allocation flags of the table
+ *              override                - Whether override should be performed
  *              table_index             - Where table index is returned
  *
  * RETURN:      Status
@@ -844,15 +844,13 @@ acpi_tb_load_table(u32 table_index, struct acpi_namespace_node *parent_node)
  ******************************************************************************/
 
 acpi_status
-acpi_tb_install_and_load_table(struct acpi_table_header *table,
-			       acpi_physical_address address,
+acpi_tb_install_and_load_table(acpi_physical_address address,
 			       u8 flags, u8 override, u32 *table_index)
 {
 	acpi_status status;
 	u32 i;
-	acpi_owner_id owner_id;
 
-	ACPI_FUNCTION_TRACE(acpi_load_table);
+	ACPI_FUNCTION_TRACE(tb_install_and_load_table);
 
 	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 
@@ -864,45 +862,60 @@ acpi_tb_install_and_load_table(struct acpi_table_header *table,
 		goto unlock_and_exit;
 	}
 
-	/*
-	 * Note: Now table is "INSTALLED", it must be validated before
-	 * using.
-	 */
-	status = acpi_tb_validate_table(&acpi_gbl_root_table_list.tables[i]);
-	if (ACPI_FAILURE(status)) {
-		goto unlock_and_exit;
-	}
+	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
+	status = acpi_tb_load_table(i, acpi_gbl_root_node);
+	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 
+unlock_and_exit:
+	*table_index = i;
 	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
-	status = acpi_ns_load_table(i, acpi_gbl_root_node);
+	return_ACPI_STATUS(status);
+}
 
-	/* Execute any module-level code that was found in the table */
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_tb_unload_table
+ *
+ * PARAMETERS:  table_index             - Table index
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Unload an ACPI table
+ *
+ ******************************************************************************/
 
-	if (!acpi_gbl_parse_table_as_term_list
-	    && acpi_gbl_group_module_level_code) {
-		acpi_ns_exec_module_code_list();
-	}
+acpi_status acpi_tb_unload_table(u32 table_index)
+{
+	acpi_status status = AE_OK;
+	struct acpi_table_header *table;
 
-	/*
-	 * Update GPEs for any new _Lxx/_Exx methods. Ignore errors. The host is
-	 * responsible for discovering any new wake GPEs by running _PRW methods
-	 * that may have been loaded by this table.
-	 */
-	status = acpi_tb_get_owner_id(i, &owner_id);
-	if (ACPI_SUCCESS(status)) {
-		acpi_ev_update_gpes(owner_id);
+	ACPI_FUNCTION_TRACE(tb_unload_table);
+
+	/* Ensure the table is still loaded */
+
+	if (!acpi_tb_is_table_loaded(table_index)) {
+		return_ACPI_STATUS(AE_NOT_EXIST);
 	}
 
 	/* Invoke table handler if present */
 
 	if (acpi_gbl_table_handler) {
-		(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_LOAD, table,
-					     acpi_gbl_table_handler_context);
+		status = acpi_get_table_by_index(table_index, &table);
+		if (ACPI_SUCCESS(status)) {
+			(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_UNLOAD,
+						     table,
+						     acpi_gbl_table_handler_context);
+		}
 	}
-	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 
-unlock_and_exit:
-	*table_index = i;
-	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
+	/* Delete the portion of the namespace owned by this table */
+
+	status = acpi_tb_delete_namespace_by_owner(table_index);
+	if (ACPI_FAILURE(status)) {
+		return_ACPI_STATUS(status);
+	}
+
+	(void)acpi_tb_release_owner_id(table_index);
+	acpi_tb_set_table_loaded_flag(table_index, FALSE);
 	return_ACPI_STATUS(status);
 }
diff --git a/drivers/acpi/acpica/tbxface.c b/drivers/acpi/acpica/tbxface.c
index 4ab6b9cd0aec..d5adb7ac4684 100644
--- a/drivers/acpi/acpica/tbxface.c
+++ b/drivers/acpi/acpica/tbxface.c
@@ -167,6 +167,7 @@ ACPI_EXPORT_SYMBOL_INIT(acpi_initialize_tables)
 acpi_status ACPI_INIT_FUNCTION acpi_reallocate_root_table(void)
 {
 	acpi_status status;
+	u32 i;
 
 	ACPI_FUNCTION_TRACE(acpi_reallocate_root_table);
 
@@ -178,6 +179,21 @@ acpi_status ACPI_INIT_FUNCTION acpi_reallocate_root_table(void)
 		return_ACPI_STATUS(AE_SUPPORT);
 	}
 
+	/*
+	 * Ensure OS early boot logic, which is required by some hosts. If the
+	 * table state is reported to be wrong, developers should fix the
+	 * issue by invoking acpi_put_table() for the reported table during the
+	 * early stage.
+	 */
+	for (i = 0; i < acpi_gbl_root_table_list.current_table_count; ++i) {
+		if (acpi_gbl_root_table_list.tables[i].pointer) {
+			ACPI_ERROR((AE_INFO,
+				    "Table [%4.4s] is not invalidated during early boot stage",
+				    acpi_gbl_root_table_list.tables[i].
+				    signature.ascii));
+		}
+	}
+
 	acpi_gbl_root_table_list.flags |= ACPI_ROOT_ALLOW_RESIZE;
 
 	status = acpi_tb_resize_root_table_list();
diff --git a/drivers/acpi/acpica/tbxfload.c b/drivers/acpi/acpica/tbxfload.c
index 5569f637f669..82019c01a0e5 100644
--- a/drivers/acpi/acpica/tbxfload.c
+++ b/drivers/acpi/acpica/tbxfload.c
@@ -239,7 +239,7 @@ acpi_status acpi_tb_load_namespace(void)
 	}
 
 	if (!tables_failed) {
-		ACPI_INFO(("%u ACPI AML tables successfully acquired and loaded\n", tables_loaded));
+		ACPI_INFO(("%u ACPI AML tables successfully acquired and loaded", tables_loaded));
 	} else {
 		ACPI_ERROR((AE_INFO,
 			    "%u table load failures, %u successful",
@@ -250,6 +250,10 @@ acpi_status acpi_tb_load_namespace(void)
 		status = AE_CTRL_TERMINATE;
 	}
 
+#ifdef ACPI_APPLICATION
+	ACPI_DEBUG_PRINT_RAW((ACPI_DB_INIT, "\n"));
+#endif
+
 unlock_and_exit:
 	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
 	return_ACPI_STATUS(status);
@@ -326,10 +330,9 @@ acpi_status acpi_load_table(struct acpi_table_header *table)
 	/* Install the table and load it into the namespace */
 
 	ACPI_INFO(("Host-directed Dynamic ACPI Table Load:"));
-	status =
-	    acpi_tb_install_and_load_table(table, ACPI_PTR_TO_PHYSADDR(table),
-					   ACPI_TABLE_ORIGIN_EXTERNAL_VIRTUAL,
-					   FALSE, &table_index);
+	status = acpi_tb_install_and_load_table(ACPI_PTR_TO_PHYSADDR(table),
+						ACPI_TABLE_ORIGIN_EXTERNAL_VIRTUAL,
+						FALSE, &table_index);
 	return_ACPI_STATUS(status);
 }
 
@@ -405,37 +408,8 @@ acpi_status acpi_unload_parent_table(acpi_handle object)
 			break;
 		}
 
-		/* Ensure the table is actually loaded */
-
 		(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
-		if (!acpi_tb_is_table_loaded(i)) {
-			status = AE_NOT_EXIST;
-			(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
-			break;
-		}
-
-		/* Invoke table handler if present */
-
-		if (acpi_gbl_table_handler) {
-			(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_UNLOAD,
-						     acpi_gbl_root_table_list.
-						     tables[i].pointer,
-						     acpi_gbl_table_handler_context);
-		}
-
-		/*
-		 * Delete all namespace objects owned by this table. Note that
-		 * these objects can appear anywhere in the namespace by virtue
-		 * of the AML "Scope" operator. Thus, we need to track ownership
-		 * by an ID, not simply a position within the hierarchy.
-		 */
-		status = acpi_tb_delete_namespace_by_owner(i);
-		if (ACPI_FAILURE(status)) {
-			break;
-		}
-
-		status = acpi_tb_release_owner_id(i);
-		acpi_tb_set_table_loaded_flag(i, FALSE);
+		status = acpi_tb_unload_table(i);
 		(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 		break;
 	}
diff --git a/drivers/acpi/acpica/utdecode.c b/drivers/acpi/acpica/utdecode.c
index 15728ad8356b..b3d8421cfb80 100644
--- a/drivers/acpi/acpica/utdecode.c
+++ b/drivers/acpi/acpica/utdecode.c
@@ -44,6 +44,7 @@
 #include <acpi/acpi.h>
 #include "accommon.h"
 #include "acnamesp.h"
+#include "amlcode.h"
 
 #define _COMPONENT          ACPI_UTILITIES
 ACPI_MODULE_NAME("utdecode")
@@ -532,6 +533,54 @@ const char *acpi_ut_get_notify_name(u32 notify_value, acpi_object_type type)
 
 	return ("Hardware-Specific");
 }
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ut_get_argument_type_name
+ *
+ * PARAMETERS:  arg_type            - an ARGP_* parser argument type
+ *
+ * RETURN:      Decoded ARGP_* type
+ *
+ * DESCRIPTION: Decode an ARGP_* parser type, as defined in the amlcode.h file,
+ *              and used in the acopcode.h file. For example, ARGP_TERMARG.
+ *              Used for debug only.
+ *
+ ******************************************************************************/
+
+static const char *acpi_gbl_argument_type[20] = {
+	/* 00 */ "Unknown ARGP",
+	/* 01 */ "ByteData",
+	/* 02 */ "ByteList",
+	/* 03 */ "CharList",
+	/* 04 */ "DataObject",
+	/* 05 */ "DataObjectList",
+	/* 06 */ "DWordData",
+	/* 07 */ "FieldList",
+	/* 08 */ "Name",
+	/* 09 */ "NameString",
+	/* 0A */ "ObjectList",
+	/* 0B */ "PackageLength",
+	/* 0C */ "SuperName",
+	/* 0D */ "Target",
+	/* 0E */ "TermArg",
+	/* 0F */ "TermList",
+	/* 10 */ "WordData",
+	/* 11 */ "QWordData",
+	/* 12 */ "SimpleName",
+	/* 13 */ "NameOrRef"
+};
+
+const char *acpi_ut_get_argument_type_name(u32 arg_type)
+{
+
+	if (arg_type > ARGP_MAX) {
+		return ("Unknown ARGP");
+	}
+
+	return (acpi_gbl_argument_type[arg_type]);
+}
+
 #endif
 
 /*******************************************************************************
diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
index 0d099a24f776..e53bef6cf53c 100644
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -852,6 +852,8 @@ static int ghes_notify_nmi(unsigned int cmd, struct pt_regs *regs)
 		if (ghes_read_estatus(ghes, 1)) {
 			ghes_clear_estatus(ghes);
 			continue;
+		} else {
+			ret = NMI_HANDLED;
 		}
 
 		sev = ghes_severity(ghes->estatus->error_severity);
@@ -863,12 +865,11 @@ static int ghes_notify_nmi(unsigned int cmd, struct pt_regs *regs)
 
 		__process_error(ghes);
 		ghes_clear_estatus(ghes);
-
-		ret = NMI_HANDLED;
 	}
 
 #ifdef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
-	irq_work_queue(&ghes_proc_irq_work);
+	if (ret == NMI_HANDLED)
+		irq_work_queue(&ghes_proc_irq_work);
 #endif
 	atomic_dec(&ghes_in_nmi);
 	return ret;
diff --git a/drivers/acpi/apei/hest.c b/drivers/acpi/apei/hest.c
index 20b3fcf4007c..8f2a98e23bba 100644
--- a/drivers/acpi/apei/hest.c
+++ b/drivers/acpi/apei/hest.c
@@ -123,7 +123,13 @@ EXPORT_SYMBOL_GPL(apei_hest_parse);
  */
 static int __init hest_parse_cmc(struct acpi_hest_header *hest_hdr, void *data)
 {
-	return arch_apei_enable_cmcff(hest_hdr, data);
+	if (hest_hdr->type != ACPI_HEST_TYPE_IA32_CORRECTED_CHECK)
+		return 0;
+
+	if (!acpi_disable_cmcff)
+		return !arch_apei_enable_cmcff(hest_hdr, data);
+
+	return 0;
 }
 
 struct ghes_arr {
@@ -232,8 +238,9 @@ void __init acpi_hest_init(void)
 		goto err;
 	}
 
-	if (!acpi_disable_cmcff)
-		apei_hest_parse(hest_parse_cmc, NULL);
+	rc = apei_hest_parse(hest_parse_cmc, NULL);
+	if (rc)
+		goto err;
 
 	if (!ghes_disable) {
 		rc = apei_hest_parse(hest_parse_ghes_count, &ghes_count);
diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c
index 6b81746cd13c..e0d2e6e6e40c 100644
--- a/drivers/acpi/arm64/iort.c
+++ b/drivers/acpi/arm64/iort.c
@@ -19,8 +19,17 @@
 #define pr_fmt(fmt)	"ACPI: IORT: " fmt
 
 #include <linux/acpi_iort.h>
+#include <linux/iommu.h>
 #include <linux/kernel.h>
+#include <linux/list.h>
 #include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#define IORT_TYPE_MASK(type)	(1 << (type))
+#define IORT_MSI_TYPE		(1 << ACPI_IORT_NODE_ITS_GROUP)
+#define IORT_IOMMU_TYPE		((1 << ACPI_IORT_NODE_SMMU) |	\
+				(1 << ACPI_IORT_NODE_SMMU_V3))
 
 struct iort_its_msi_chip {
 	struct list_head	list;
@@ -28,6 +37,90 @@ struct iort_its_msi_chip {
 	u32			translation_id;
 };
 
+struct iort_fwnode {
+	struct list_head list;
+	struct acpi_iort_node *iort_node;
+	struct fwnode_handle *fwnode;
+};
+static LIST_HEAD(iort_fwnode_list);
+static DEFINE_SPINLOCK(iort_fwnode_lock);
+
+/**
+ * iort_set_fwnode() - Create iort_fwnode and use it to register
+ *		       iommu data in the iort_fwnode_list
+ *
+ * @node: IORT table node associated with the IOMMU
+ * @fwnode: fwnode associated with the IORT node
+ *
+ * Returns: 0 on success
+ *          <0 on failure
+ */
+static inline int iort_set_fwnode(struct acpi_iort_node *iort_node,
+				  struct fwnode_handle *fwnode)
+{
+	struct iort_fwnode *np;
+
+	np = kzalloc(sizeof(struct iort_fwnode), GFP_ATOMIC);
+
+	if (WARN_ON(!np))
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&np->list);
+	np->iort_node = iort_node;
+	np->fwnode = fwnode;
+
+	spin_lock(&iort_fwnode_lock);
+	list_add_tail(&np->list, &iort_fwnode_list);
+	spin_unlock(&iort_fwnode_lock);
+
+	return 0;
+}
+
+/**
+ * iort_get_fwnode() - Retrieve fwnode associated with an IORT node
+ *
+ * @node: IORT table node to be looked-up
+ *
+ * Returns: fwnode_handle pointer on success, NULL on failure
+ */
+static inline
+struct fwnode_handle *iort_get_fwnode(struct acpi_iort_node *node)
+{
+	struct iort_fwnode *curr;
+	struct fwnode_handle *fwnode = NULL;
+
+	spin_lock(&iort_fwnode_lock);
+	list_for_each_entry(curr, &iort_fwnode_list, list) {
+		if (curr->iort_node == node) {
+			fwnode = curr->fwnode;
+			break;
+		}
+	}
+	spin_unlock(&iort_fwnode_lock);
+
+	return fwnode;
+}
+
+/**
+ * iort_delete_fwnode() - Delete fwnode associated with an IORT node
+ *
+ * @node: IORT table node associated with fwnode to delete
+ */
+static inline void iort_delete_fwnode(struct acpi_iort_node *node)
+{
+	struct iort_fwnode *curr, *tmp;
+
+	spin_lock(&iort_fwnode_lock);
+	list_for_each_entry_safe(curr, tmp, &iort_fwnode_list, list) {
+		if (curr->iort_node == node) {
+			list_del(&curr->list);
+			kfree(curr);
+			break;
+		}
+	}
+	spin_unlock(&iort_fwnode_lock);
+}
+
 typedef acpi_status (*iort_find_node_callback)
 	(struct acpi_iort_node *node, void *context);
 
@@ -141,6 +234,21 @@ static struct acpi_iort_node *iort_scan_node(enum acpi_iort_node_type type,
 	return NULL;
 }
 
+static acpi_status
+iort_match_type_callback(struct acpi_iort_node *node, void *context)
+{
+	return AE_OK;
+}
+
+bool iort_node_match(u8 type)
+{
+	struct acpi_iort_node *node;
+
+	node = iort_scan_node(type, iort_match_type_callback, NULL);
+
+	return node != NULL;
+}
+
 static acpi_status iort_match_node_callback(struct acpi_iort_node *node,
 					    void *context)
 {
@@ -212,9 +320,48 @@ static int iort_id_map(struct acpi_iort_id_mapping *map, u8 type, u32 rid_in,
 	return 0;
 }
 
+static
+struct acpi_iort_node *iort_node_get_id(struct acpi_iort_node *node,
+					u32 *id_out, u8 type_mask,
+					int index)
+{
+	struct acpi_iort_node *parent;
+	struct acpi_iort_id_mapping *map;
+
+	if (!node->mapping_offset || !node->mapping_count ||
+				     index >= node->mapping_count)
+		return NULL;
+
+	map = ACPI_ADD_PTR(struct acpi_iort_id_mapping, node,
+			   node->mapping_offset);
+
+	/* Firmware bug! */
+	if (!map->output_reference) {
+		pr_err(FW_BUG "[node %p type %d] ID map has NULL parent reference\n",
+		       node, node->type);
+		return NULL;
+	}
+
+	parent = ACPI_ADD_PTR(struct acpi_iort_node, iort_table,
+			       map->output_reference);
+
+	if (!(IORT_TYPE_MASK(parent->type) & type_mask))
+		return NULL;
+
+	if (map[index].flags & ACPI_IORT_ID_SINGLE_MAPPING) {
+		if (node->type == ACPI_IORT_NODE_NAMED_COMPONENT ||
+		    node->type == ACPI_IORT_NODE_PCI_ROOT_COMPLEX) {
+			*id_out = map[index].output_base;
+			return parent;
+		}
+	}
+
+	return NULL;
+}
+
 static struct acpi_iort_node *iort_node_map_rid(struct acpi_iort_node *node,
 						u32 rid_in, u32 *rid_out,
-						u8 type)
+						u8 type_mask)
 {
 	u32 rid = rid_in;
 
@@ -223,7 +370,7 @@ static struct acpi_iort_node *iort_node_map_rid(struct acpi_iort_node *node,
 		struct acpi_iort_id_mapping *map;
 		int i;
 
-		if (node->type == type) {
+		if (IORT_TYPE_MASK(node->type) & type_mask) {
 			if (rid_out)
 				*rid_out = rid;
 			return node;
@@ -296,7 +443,7 @@ u32 iort_msi_map_rid(struct device *dev, u32 req_id)
 	if (!node)
 		return req_id;
 
-	iort_node_map_rid(node, req_id, &dev_id, ACPI_IORT_NODE_ITS_GROUP);
+	iort_node_map_rid(node, req_id, &dev_id, IORT_MSI_TYPE);
 	return dev_id;
 }
 
@@ -318,7 +465,7 @@ static int iort_dev_find_its_id(struct device *dev, u32 req_id,
 	if (!node)
 		return -ENXIO;
 
-	node = iort_node_map_rid(node, req_id, NULL, ACPI_IORT_NODE_ITS_GROUP);
+	node = iort_node_map_rid(node, req_id, NULL, IORT_MSI_TYPE);
 	if (!node)
 		return -ENXIO;
 
@@ -356,13 +503,459 @@ struct irq_domain *iort_get_device_domain(struct device *dev, u32 req_id)
 	return irq_find_matching_fwnode(handle, DOMAIN_BUS_PCI_MSI);
 }
 
+static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data)
+{
+	u32 *rid = data;
+
+	*rid = alias;
+	return 0;
+}
+
+static int arm_smmu_iort_xlate(struct device *dev, u32 streamid,
+			       struct fwnode_handle *fwnode,
+			       const struct iommu_ops *ops)
+{
+	int ret = iommu_fwspec_init(dev, fwnode, ops);
+
+	if (!ret)
+		ret = iommu_fwspec_add_ids(dev, &streamid, 1);
+
+	return ret;
+}
+
+static const struct iommu_ops *iort_iommu_xlate(struct device *dev,
+					struct acpi_iort_node *node,
+					u32 streamid)
+{
+	const struct iommu_ops *ops = NULL;
+	int ret = -ENODEV;
+	struct fwnode_handle *iort_fwnode;
+
+	if (node) {
+		iort_fwnode = iort_get_fwnode(node);
+		if (!iort_fwnode)
+			return NULL;
+
+		ops = iommu_get_instance(iort_fwnode);
+		if (!ops)
+			return NULL;
+
+		ret = arm_smmu_iort_xlate(dev, streamid, iort_fwnode, ops);
+	}
+
+	return ret ? NULL : ops;
+}
+
+/**
+ * iort_set_dma_mask - Set-up dma mask for a device.
+ *
+ * @dev: device to configure
+ */
+void iort_set_dma_mask(struct device *dev)
+{
+	/*
+	 * Set default coherent_dma_mask to 32 bit.  Drivers are expected to
+	 * setup the correct supported mask.
+	 */
+	if (!dev->coherent_dma_mask)
+		dev->coherent_dma_mask = DMA_BIT_MASK(32);
+
+	/*
+	 * Set it to coherent_dma_mask by default if the architecture
+	 * code has not set it.
+	 */
+	if (!dev->dma_mask)
+		dev->dma_mask = &dev->coherent_dma_mask;
+}
+
+/**
+ * iort_iommu_configure - Set-up IOMMU configuration for a device.
+ *
+ * @dev: device to configure
+ *
+ * Returns: iommu_ops pointer on configuration success
+ *          NULL on configuration failure
+ */
+const struct iommu_ops *iort_iommu_configure(struct device *dev)
+{
+	struct acpi_iort_node *node, *parent;
+	const struct iommu_ops *ops = NULL;
+	u32 streamid = 0;
+
+	if (dev_is_pci(dev)) {
+		struct pci_bus *bus = to_pci_dev(dev)->bus;
+		u32 rid;
+
+		pci_for_each_dma_alias(to_pci_dev(dev), __get_pci_rid,
+				       &rid);
+
+		node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,
+				      iort_match_node_callback, &bus->dev);
+		if (!node)
+			return NULL;
+
+		parent = iort_node_map_rid(node, rid, &streamid,
+					   IORT_IOMMU_TYPE);
+
+		ops = iort_iommu_xlate(dev, parent, streamid);
+
+	} else {
+		int i = 0;
+
+		node = iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
+				      iort_match_node_callback, dev);
+		if (!node)
+			return NULL;
+
+		parent = iort_node_get_id(node, &streamid,
+					  IORT_IOMMU_TYPE, i++);
+
+		while (parent) {
+			ops = iort_iommu_xlate(dev, parent, streamid);
+
+			parent = iort_node_get_id(node, &streamid,
+						  IORT_IOMMU_TYPE, i++);
+		}
+	}
+
+	return ops;
+}
+
+static void __init acpi_iort_register_irq(int hwirq, const char *name,
+					  int trigger,
+					  struct resource *res)
+{
+	int irq = acpi_register_gsi(NULL, hwirq, trigger,
+				    ACPI_ACTIVE_HIGH);
+
+	if (irq <= 0) {
+		pr_err("could not register gsi hwirq %d name [%s]\n", hwirq,
+								      name);
+		return;
+	}
+
+	res->start = irq;
+	res->end = irq;
+	res->flags = IORESOURCE_IRQ;
+	res->name = name;
+}
+
+static int __init arm_smmu_v3_count_resources(struct acpi_iort_node *node)
+{
+	struct acpi_iort_smmu_v3 *smmu;
+	/* Always present mem resource */
+	int num_res = 1;
+
+	/* Retrieve SMMUv3 specific data */
+	smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
+
+	if (smmu->event_gsiv)
+		num_res++;
+
+	if (smmu->pri_gsiv)
+		num_res++;
+
+	if (smmu->gerr_gsiv)
+		num_res++;
+
+	if (smmu->sync_gsiv)
+		num_res++;
+
+	return num_res;
+}
+
+static void __init arm_smmu_v3_init_resources(struct resource *res,
+					      struct acpi_iort_node *node)
+{
+	struct acpi_iort_smmu_v3 *smmu;
+	int num_res = 0;
+
+	/* Retrieve SMMUv3 specific data */
+	smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
+
+	res[num_res].start = smmu->base_address;
+	res[num_res].end = smmu->base_address + SZ_128K - 1;
+	res[num_res].flags = IORESOURCE_MEM;
+
+	num_res++;
+
+	if (smmu->event_gsiv)
+		acpi_iort_register_irq(smmu->event_gsiv, "eventq",
+				       ACPI_EDGE_SENSITIVE,
+				       &res[num_res++]);
+
+	if (smmu->pri_gsiv)
+		acpi_iort_register_irq(smmu->pri_gsiv, "priq",
+				       ACPI_EDGE_SENSITIVE,
+				       &res[num_res++]);
+
+	if (smmu->gerr_gsiv)
+		acpi_iort_register_irq(smmu->gerr_gsiv, "gerror",
+				       ACPI_EDGE_SENSITIVE,
+				       &res[num_res++]);
+
+	if (smmu->sync_gsiv)
+		acpi_iort_register_irq(smmu->sync_gsiv, "cmdq-sync",
+				       ACPI_EDGE_SENSITIVE,
+				       &res[num_res++]);
+}
+
+static bool __init arm_smmu_v3_is_coherent(struct acpi_iort_node *node)
+{
+	struct acpi_iort_smmu_v3 *smmu;
+
+	/* Retrieve SMMUv3 specific data */
+	smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
+
+	return smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE;
+}
+
+static int __init arm_smmu_count_resources(struct acpi_iort_node *node)
+{
+	struct acpi_iort_smmu *smmu;
+
+	/* Retrieve SMMU specific data */
+	smmu = (struct acpi_iort_smmu *)node->node_data;
+
+	/*
+	 * Only consider the global fault interrupt and ignore the
+	 * configuration access interrupt.
+	 *
+	 * MMIO address and global fault interrupt resources are always
+	 * present so add them to the context interrupt count as a static
+	 * value.
+	 */
+	return smmu->context_interrupt_count + 2;
+}
+
+static void __init arm_smmu_init_resources(struct resource *res,
+					   struct acpi_iort_node *node)
+{
+	struct acpi_iort_smmu *smmu;
+	int i, hw_irq, trigger, num_res = 0;
+	u64 *ctx_irq, *glb_irq;
+
+	/* Retrieve SMMU specific data */
+	smmu = (struct acpi_iort_smmu *)node->node_data;
+
+	res[num_res].start = smmu->base_address;
+	res[num_res].end = smmu->base_address + smmu->span - 1;
+	res[num_res].flags = IORESOURCE_MEM;
+	num_res++;
+
+	glb_irq = ACPI_ADD_PTR(u64, node, smmu->global_interrupt_offset);
+	/* Global IRQs */
+	hw_irq = IORT_IRQ_MASK(glb_irq[0]);
+	trigger = IORT_IRQ_TRIGGER_MASK(glb_irq[0]);
+
+	acpi_iort_register_irq(hw_irq, "arm-smmu-global", trigger,
+				     &res[num_res++]);
+
+	/* Context IRQs */
+	ctx_irq = ACPI_ADD_PTR(u64, node, smmu->context_interrupt_offset);
+	for (i = 0; i < smmu->context_interrupt_count; i++) {
+		hw_irq = IORT_IRQ_MASK(ctx_irq[i]);
+		trigger = IORT_IRQ_TRIGGER_MASK(ctx_irq[i]);
+
+		acpi_iort_register_irq(hw_irq, "arm-smmu-context", trigger,
+				       &res[num_res++]);
+	}
+}
+
+static bool __init arm_smmu_is_coherent(struct acpi_iort_node *node)
+{
+	struct acpi_iort_smmu *smmu;
+
+	/* Retrieve SMMU specific data */
+	smmu = (struct acpi_iort_smmu *)node->node_data;
+
+	return smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK;
+}
+
+struct iort_iommu_config {
+	const char *name;
+	int (*iommu_init)(struct acpi_iort_node *node);
+	bool (*iommu_is_coherent)(struct acpi_iort_node *node);
+	int (*iommu_count_resources)(struct acpi_iort_node *node);
+	void (*iommu_init_resources)(struct resource *res,
+				     struct acpi_iort_node *node);
+};
+
+static const struct iort_iommu_config iort_arm_smmu_v3_cfg __initconst = {
+	.name = "arm-smmu-v3",
+	.iommu_is_coherent = arm_smmu_v3_is_coherent,
+	.iommu_count_resources = arm_smmu_v3_count_resources,
+	.iommu_init_resources = arm_smmu_v3_init_resources
+};
+
+static const struct iort_iommu_config iort_arm_smmu_cfg __initconst = {
+	.name = "arm-smmu",
+	.iommu_is_coherent = arm_smmu_is_coherent,
+	.iommu_count_resources = arm_smmu_count_resources,
+	.iommu_init_resources = arm_smmu_init_resources
+};
+
+static __init
+const struct iort_iommu_config *iort_get_iommu_cfg(struct acpi_iort_node *node)
+{
+	switch (node->type) {
+	case ACPI_IORT_NODE_SMMU_V3:
+		return &iort_arm_smmu_v3_cfg;
+	case ACPI_IORT_NODE_SMMU:
+		return &iort_arm_smmu_cfg;
+	default:
+		return NULL;
+	}
+}
+
+/**
+ * iort_add_smmu_platform_device() - Allocate a platform device for SMMU
+ * @node: Pointer to SMMU ACPI IORT node
+ *
+ * Returns: 0 on success, <0 failure
+ */
+static int __init iort_add_smmu_platform_device(struct acpi_iort_node *node)
+{
+	struct fwnode_handle *fwnode;
+	struct platform_device *pdev;
+	struct resource *r;
+	enum dev_dma_attr attr;
+	int ret, count;
+	const struct iort_iommu_config *ops = iort_get_iommu_cfg(node);
+
+	if (!ops)
+		return -ENODEV;
+
+	pdev = platform_device_alloc(ops->name, PLATFORM_DEVID_AUTO);
+	if (!pdev)
+		return PTR_ERR(pdev);
+
+	count = ops->iommu_count_resources(node);
+
+	r = kcalloc(count, sizeof(*r), GFP_KERNEL);
+	if (!r) {
+		ret = -ENOMEM;
+		goto dev_put;
+	}
+
+	ops->iommu_init_resources(r, node);
+
+	ret = platform_device_add_resources(pdev, r, count);
+	/*
+	 * Resources are duplicated in platform_device_add_resources,
+	 * free their allocated memory
+	 */
+	kfree(r);
+
+	if (ret)
+		goto dev_put;
+
+	/*
+	 * Add a copy of IORT node pointer to platform_data to
+	 * be used to retrieve IORT data information.
+	 */
+	ret = platform_device_add_data(pdev, &node, sizeof(node));
+	if (ret)
+		goto dev_put;
+
+	/*
+	 * We expect the dma masks to be equivalent for
+	 * all SMMUs set-ups
+	 */
+	pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
+
+	fwnode = iort_get_fwnode(node);
+
+	if (!fwnode) {
+		ret = -ENODEV;
+		goto dev_put;
+	}
+
+	pdev->dev.fwnode = fwnode;
+
+	attr = ops->iommu_is_coherent(node) ?
+			     DEV_DMA_COHERENT : DEV_DMA_NON_COHERENT;
+
+	/* Configure DMA for the page table walker */
+	acpi_dma_configure(&pdev->dev, attr);
+
+	ret = platform_device_add(pdev);
+	if (ret)
+		goto dma_deconfigure;
+
+	return 0;
+
+dma_deconfigure:
+	acpi_dma_deconfigure(&pdev->dev);
+dev_put:
+	platform_device_put(pdev);
+
+	return ret;
+}
+
+static void __init iort_init_platform_devices(void)
+{
+	struct acpi_iort_node *iort_node, *iort_end;
+	struct acpi_table_iort *iort;
+	struct fwnode_handle *fwnode;
+	int i, ret;
+
+	/*
+	 * iort_table and iort both point to the start of IORT table, but
+	 * have different struct types
+	 */
+	iort = (struct acpi_table_iort *)iort_table;
+
+	/* Get the first IORT node */
+	iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort,
+				 iort->node_offset);
+	iort_end = ACPI_ADD_PTR(struct acpi_iort_node, iort,
+				iort_table->length);
+
+	for (i = 0; i < iort->node_count; i++) {
+		if (iort_node >= iort_end) {
+			pr_err("iort node pointer overflows, bad table\n");
+			return;
+		}
+
+		if ((iort_node->type == ACPI_IORT_NODE_SMMU) ||
+			(iort_node->type == ACPI_IORT_NODE_SMMU_V3)) {
+
+			fwnode = acpi_alloc_fwnode_static();
+			if (!fwnode)
+				return;
+
+			iort_set_fwnode(iort_node, fwnode);
+
+			ret = iort_add_smmu_platform_device(iort_node);
+			if (ret) {
+				iort_delete_fwnode(iort_node);
+				acpi_free_fwnode_static(fwnode);
+				return;
+			}
+		}
+
+		iort_node = ACPI_ADD_PTR(struct acpi_iort_node, iort_node,
+					 iort_node->length);
+	}
+}
+
 void __init acpi_iort_init(void)
 {
 	acpi_status status;
 
 	status = acpi_get_table(ACPI_SIG_IORT, 0, &iort_table);
-	if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
-		const char *msg = acpi_format_exception(status);
-		pr_err("Failed to get table, %s\n", msg);
+	if (ACPI_FAILURE(status)) {
+		if (status != AE_NOT_FOUND) {
+			const char *msg = acpi_format_exception(status);
+
+			pr_err("Failed to get table, %s\n", msg);
+		}
+
+		return;
 	}
+
+	iort_init_platform_devices();
+
+	acpi_probe_device_table(iort);
 }
diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c
index 93ecae55fe6a..05fe9ebfb9b5 100644
--- a/drivers/acpi/battery.c
+++ b/drivers/acpi/battery.c
@@ -430,39 +430,24 @@ static int acpi_battery_get_status(struct acpi_battery *battery)
 	return 0;
 }
 
-static int acpi_battery_get_info(struct acpi_battery *battery)
+
+static int extract_battery_info(const int use_bix,
+			 struct acpi_battery *battery,
+			 const struct acpi_buffer *buffer)
 {
 	int result = -EFAULT;
-	acpi_status status = 0;
-	char *name = test_bit(ACPI_BATTERY_XINFO_PRESENT, &battery->flags) ?
-			"_BIX" : "_BIF";
-
-	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
 
-	if (!acpi_battery_present(battery))
-		return 0;
-	mutex_lock(&battery->lock);
-	status = acpi_evaluate_object(battery->device->handle, name,
-						NULL, &buffer);
-	mutex_unlock(&battery->lock);
-
-	if (ACPI_FAILURE(status)) {
-		ACPI_EXCEPTION((AE_INFO, status, "Evaluating %s", name));
-		return -ENODEV;
-	}
-
-	if (battery_bix_broken_package)
-		result = extract_package(battery, buffer.pointer,
+	if (use_bix && battery_bix_broken_package)
+		result = extract_package(battery, buffer->pointer,
 				extended_info_offsets + 1,
 				ARRAY_SIZE(extended_info_offsets) - 1);
-	else if (test_bit(ACPI_BATTERY_XINFO_PRESENT, &battery->flags))
-		result = extract_package(battery, buffer.pointer,
+	else if (use_bix)
+		result = extract_package(battery, buffer->pointer,
 				extended_info_offsets,
 				ARRAY_SIZE(extended_info_offsets));
 	else
-		result = extract_package(battery, buffer.pointer,
+		result = extract_package(battery, buffer->pointer,
 				info_offsets, ARRAY_SIZE(info_offsets));
-	kfree(buffer.pointer);
 	if (test_bit(ACPI_BATTERY_QUIRK_PERCENTAGE_CAPACITY, &battery->flags))
 		battery->full_charge_capacity = battery->design_capacity;
 	if (test_bit(ACPI_BATTERY_QUIRK_THINKPAD_MAH, &battery->flags) &&
@@ -483,6 +468,45 @@ static int acpi_battery_get_info(struct acpi_battery *battery)
 	return result;
 }
 
+static int acpi_battery_get_info(struct acpi_battery *battery)
+{
+	const int xinfo = test_bit(ACPI_BATTERY_XINFO_PRESENT, &battery->flags);
+	int use_bix;
+	int result = -ENODEV;
+
+	if (!acpi_battery_present(battery))
+		return 0;
+
+
+	for (use_bix = xinfo ? 1 : 0; use_bix >= 0; use_bix--) {
+		struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+		acpi_status status = AE_ERROR;
+
+		mutex_lock(&battery->lock);
+		status = acpi_evaluate_object(battery->device->handle,
+					      use_bix ? "_BIX":"_BIF",
+					      NULL, &buffer);
+		mutex_unlock(&battery->lock);
+
+		if (ACPI_FAILURE(status)) {
+			ACPI_EXCEPTION((AE_INFO, status, "Evaluating %s",
+					use_bix ? "_BIX":"_BIF"));
+		} else {
+			result = extract_battery_info(use_bix,
+						      battery,
+						      &buffer);
+
+			kfree(buffer.pointer);
+			break;
+		}
+	}
+
+	if (!result && !use_bix && xinfo)
+		pr_warn(FW_BUG "The _BIX method is broken, using _BIF.\n");
+
+	return result;
+}
+
 static int acpi_battery_get_state(struct acpi_battery *battery)
 {
 	int result = 0;
diff --git a/drivers/acpi/blacklist.c b/drivers/acpi/blacklist.c
index bdc67bad61a7..4421f7c9981c 100644
--- a/drivers/acpi/blacklist.c
+++ b/drivers/acpi/blacklist.c
@@ -160,6 +160,34 @@ static struct dmi_system_id acpi_rev_dmi_table[] __initdata = {
 		      DMI_MATCH(DMI_PRODUCT_NAME, "XPS 13 9343"),
 		},
 	},
+	{
+	 .callback = dmi_enable_rev_override,
+	 .ident = "DELL Precision 5520",
+	 .matches = {
+		      DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+		      DMI_MATCH(DMI_PRODUCT_NAME, "Precision 5520"),
+		},
+	},
+	{
+	 .callback = dmi_enable_rev_override,
+	 .ident = "DELL Precision 3520",
+	 .matches = {
+		      DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+		      DMI_MATCH(DMI_PRODUCT_NAME, "Precision 3520"),
+		},
+	},
+	/*
+	 * Resolves a quirk with the Dell Latitude 3350 that
+	 * causes the ethernet adapter to not function.
+	 */
+	{
+	 .callback = dmi_enable_rev_override,
+	 .ident = "DELL Latitude 3350",
+	 .matches = {
+		      DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+		      DMI_MATCH(DMI_PRODUCT_NAME, "Latitude 3350"),
+		},
+	},
 #endif
 	{}
 };
diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c
index d0d0504b7c89..3ca0729f7e0e 100644
--- a/drivers/acpi/cppc_acpi.c
+++ b/drivers/acpi/cppc_acpi.c
@@ -776,21 +776,25 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 		init_waitqueue_head(&pcc_data.pcc_write_wait_q);
 	}
 
-	/* Plug PSD data into this CPUs CPC descriptor. */
-	per_cpu(cpc_desc_ptr, pr->id) = cpc_ptr;
-
 	/* Everything looks okay */
 	pr_debug("Parsed CPC struct for CPU: %d\n", pr->id);
 
 	/* Add per logical CPU nodes for reading its feedback counters. */
 	cpu_dev = get_cpu_device(pr->id);
-	if (!cpu_dev)
+	if (!cpu_dev) {
+		ret = -EINVAL;
 		goto out_free;
+	}
+
+	/* Plug PSD data into this CPUs CPC descriptor. */
+	per_cpu(cpc_desc_ptr, pr->id) = cpc_ptr;
 
 	ret = kobject_init_and_add(&cpc_ptr->kobj, &cppc_ktype, &cpu_dev->kobj,
 			"acpi_cppc");
-	if (ret)
+	if (ret) {
+		per_cpu(cpc_desc_ptr, pr->id) = NULL;
 		goto out_free;
+	}
 
 	kfree(output.pointer);
 	return 0;
@@ -824,6 +828,8 @@ void acpi_cppc_processor_exit(struct acpi_processor *pr)
 	void __iomem *addr;
 
 	cpc_ptr = per_cpu(cpc_desc_ptr, pr->id);
+	if (!cpc_ptr)
+		return;
 
 	/* Free all the mapped sys mem areas for this CPU */
 	for (i = 2; i < cpc_ptr->num_entries; i++) {
diff --git a/drivers/acpi/device_sysfs.c b/drivers/acpi/device_sysfs.c
index 7b2c48fde4e2..24418932612e 100644
--- a/drivers/acpi/device_sysfs.c
+++ b/drivers/acpi/device_sysfs.c
@@ -52,7 +52,7 @@ struct acpi_data_node_attr {
 
 static ssize_t data_node_show_path(struct acpi_data_node *dn, char *buf)
 {
-	return acpi_object_path(dn->handle, buf);
+	return dn->handle ? acpi_object_path(dn->handle, buf) : 0;
 }
 
 DATA_NODE_ATTR(path);
@@ -105,10 +105,10 @@ static void acpi_expose_nondev_subnodes(struct kobject *kobj,
 		init_completion(&dn->kobj_done);
 		ret = kobject_init_and_add(&dn->kobj, &acpi_data_node_ktype,
 					   kobj, "%s", dn->name);
-		if (ret)
-			acpi_handle_err(dn->handle, "Failed to expose (%d)\n", ret);
-		else
+		if (!ret)
 			acpi_expose_nondev_subnodes(&dn->kobj, &dn->data);
+		else if (dn->handle)
+			acpi_handle_err(dn->handle, "Failed to expose (%d)\n", ret);
 	}
 }
 
diff --git a/drivers/acpi/glue.c b/drivers/acpi/glue.c
index 5ea5dc219f56..f8d65647ea79 100644
--- a/drivers/acpi/glue.c
+++ b/drivers/acpi/glue.c
@@ -227,8 +227,7 @@ int acpi_bind_one(struct device *dev, struct acpi_device *acpi_dev)
 
 	attr = acpi_get_dma_attr(acpi_dev);
 	if (attr != DEV_DMA_NOT_SUPPORTED)
-		arch_setup_dma_ops(dev, 0, 0, NULL,
-				   attr == DEV_DMA_COHERENT);
+		acpi_dma_configure(dev, attr);
 
 	acpi_physnode_link_name(physical_node_name, node_id);
 	retval = sysfs_create_link(&acpi_dev->dev.kobj, &dev->kobj,
@@ -251,6 +250,7 @@ int acpi_bind_one(struct device *dev, struct acpi_device *acpi_dev)
 	return 0;
 
  err:
+	acpi_dma_deconfigure(dev);
 	ACPI_COMPANION_SET(dev, NULL);
 	put_device(dev);
 	put_device(&acpi_dev->dev);
diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c
index 416953a42510..9a4c6abee63e 100644
--- a/drivers/acpi/osl.c
+++ b/drivers/acpi/osl.c
@@ -181,15 +181,15 @@ void acpi_os_vprintf(const char *fmt, va_list args)
 static unsigned long acpi_rsdp;
 static int __init setup_acpi_rsdp(char *arg)
 {
-	if (kstrtoul(arg, 16, &acpi_rsdp))
-		return -EINVAL;
-	return 0;
+	return kstrtoul(arg, 16, &acpi_rsdp);
 }
 early_param("acpi_rsdp", setup_acpi_rsdp);
 #endif
 
 acpi_physical_address __init acpi_os_get_root_pointer(void)
 {
+	acpi_physical_address pa = 0;
+
 #ifdef CONFIG_KEXEC
 	if (acpi_rsdp)
 		return acpi_rsdp;
@@ -198,21 +198,14 @@ acpi_physical_address __init acpi_os_get_root_pointer(void)
 	if (efi_enabled(EFI_CONFIG_TABLES)) {
 		if (efi.acpi20 != EFI_INVALID_TABLE_ADDR)
 			return efi.acpi20;
-		else if (efi.acpi != EFI_INVALID_TABLE_ADDR)
+		if (efi.acpi != EFI_INVALID_TABLE_ADDR)
 			return efi.acpi;
-		else {
-			printk(KERN_ERR PREFIX
-			       "System description tables not found\n");
-			return 0;
-		}
+		pr_err(PREFIX "System description tables not found\n");
 	} else if (IS_ENABLED(CONFIG_ACPI_LEGACY_TABLES_LOOKUP)) {
-		acpi_physical_address pa = 0;
-
 		acpi_find_root_pointer(&pa);
-		return pa;
 	}
 
-	return 0;
+	return pa;
 }
 
 /* Must be called with 'acpi_ioremap_lock' or RCU read lock held. */
diff --git a/drivers/acpi/pci_mcfg.c b/drivers/acpi/pci_mcfg.c
index b5b376e081f5..a6a4ceaa6cc3 100644
--- a/drivers/acpi/pci_mcfg.c
+++ b/drivers/acpi/pci_mcfg.c
@@ -22,6 +22,7 @@
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include <linux/pci-acpi.h>
+#include <linux/pci-ecam.h>
 
 /* Structure to hold entries from the MCFG table */
 struct mcfg_entry {
@@ -32,12 +33,166 @@ struct mcfg_entry {
 	u8			bus_end;
 };
 
+#ifdef CONFIG_PCI_QUIRKS
+struct mcfg_fixup {
+	char oem_id[ACPI_OEM_ID_SIZE + 1];
+	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
+	u32 oem_revision;
+	u16 segment;
+	struct resource bus_range;
+	struct pci_ecam_ops *ops;
+	struct resource cfgres;
+};
+
+#define MCFG_BUS_RANGE(start, end)	DEFINE_RES_NAMED((start),	\
+						((end) - (start) + 1),	\
+						NULL, IORESOURCE_BUS)
+#define MCFG_BUS_ANY			MCFG_BUS_RANGE(0x0, 0xff)
+
+static struct mcfg_fixup mcfg_quirks[] = {
+/*	{ OEM_ID, OEM_TABLE_ID, REV, SEGMENT, BUS_RANGE, ops, cfgres }, */
+
+#define QCOM_ECAM32(seg) \
+	{ "QCOM  ", "QDF2432 ", 1, seg, MCFG_BUS_ANY, &pci_32b_ops }
+	QCOM_ECAM32(0),
+	QCOM_ECAM32(1),
+	QCOM_ECAM32(2),
+	QCOM_ECAM32(3),
+	QCOM_ECAM32(4),
+	QCOM_ECAM32(5),
+	QCOM_ECAM32(6),
+	QCOM_ECAM32(7),
+
+#define HISI_QUAD_DOM(table_id, seg, ops) \
+	{ "HISI  ", table_id, 0, (seg) + 0, MCFG_BUS_ANY, ops }, \
+	{ "HISI  ", table_id, 0, (seg) + 1, MCFG_BUS_ANY, ops }, \
+	{ "HISI  ", table_id, 0, (seg) + 2, MCFG_BUS_ANY, ops }, \
+	{ "HISI  ", table_id, 0, (seg) + 3, MCFG_BUS_ANY, ops }
+	HISI_QUAD_DOM("HIP05   ",  0, &hisi_pcie_ops),
+	HISI_QUAD_DOM("HIP06   ",  0, &hisi_pcie_ops),
+	HISI_QUAD_DOM("HIP07   ",  0, &hisi_pcie_ops),
+	HISI_QUAD_DOM("HIP07   ",  4, &hisi_pcie_ops),
+	HISI_QUAD_DOM("HIP07   ",  8, &hisi_pcie_ops),
+	HISI_QUAD_DOM("HIP07   ", 12, &hisi_pcie_ops),
+
+#define THUNDER_PEM_RES(addr, node) \
+	DEFINE_RES_MEM((addr) + ((u64) (node) << 44), 0x39 * SZ_16M)
+#define THUNDER_PEM_QUIRK(rev, node) \
+	{ "CAVIUM", "THUNDERX", rev, 4 + (10 * (node)), MCFG_BUS_ANY,	    \
+	  &thunder_pem_ecam_ops, THUNDER_PEM_RES(0x88001f000000UL, node) },  \
+	{ "CAVIUM", "THUNDERX", rev, 5 + (10 * (node)), MCFG_BUS_ANY,	    \
+	  &thunder_pem_ecam_ops, THUNDER_PEM_RES(0x884057000000UL, node) },  \
+	{ "CAVIUM", "THUNDERX", rev, 6 + (10 * (node)), MCFG_BUS_ANY,	    \
+	  &thunder_pem_ecam_ops, THUNDER_PEM_RES(0x88808f000000UL, node) },  \
+	{ "CAVIUM", "THUNDERX", rev, 7 + (10 * (node)), MCFG_BUS_ANY,	    \
+	  &thunder_pem_ecam_ops, THUNDER_PEM_RES(0x89001f000000UL, node) },  \
+	{ "CAVIUM", "THUNDERX", rev, 8 + (10 * (node)), MCFG_BUS_ANY,	    \
+	  &thunder_pem_ecam_ops, THUNDER_PEM_RES(0x894057000000UL, node) },  \
+	{ "CAVIUM", "THUNDERX", rev, 9 + (10 * (node)), MCFG_BUS_ANY,	    \
+	  &thunder_pem_ecam_ops, THUNDER_PEM_RES(0x89808f000000UL, node) }
+	/* SoC pass2.x */
+	THUNDER_PEM_QUIRK(1, 0),
+	THUNDER_PEM_QUIRK(1, 1),
+
+#define THUNDER_ECAM_QUIRK(rev, seg)					\
+	{ "CAVIUM", "THUNDERX", rev, seg, MCFG_BUS_ANY,			\
+	&pci_thunder_ecam_ops }
+	/* SoC pass1.x */
+	THUNDER_PEM_QUIRK(2, 0),	/* off-chip devices */
+	THUNDER_PEM_QUIRK(2, 1),	/* off-chip devices */
+	THUNDER_ECAM_QUIRK(2,  0),
+	THUNDER_ECAM_QUIRK(2,  1),
+	THUNDER_ECAM_QUIRK(2,  2),
+	THUNDER_ECAM_QUIRK(2,  3),
+	THUNDER_ECAM_QUIRK(2, 10),
+	THUNDER_ECAM_QUIRK(2, 11),
+	THUNDER_ECAM_QUIRK(2, 12),
+	THUNDER_ECAM_QUIRK(2, 13),
+
+#define XGENE_V1_ECAM_MCFG(rev, seg) \
+	{"APM   ", "XGENE   ", rev, seg, MCFG_BUS_ANY, \
+		&xgene_v1_pcie_ecam_ops }
+#define XGENE_V2_ECAM_MCFG(rev, seg) \
+	{"APM   ", "XGENE   ", rev, seg, MCFG_BUS_ANY, \
+		&xgene_v2_pcie_ecam_ops }
+	/* X-Gene SoC with v1 PCIe controller */
+	XGENE_V1_ECAM_MCFG(1, 0),
+	XGENE_V1_ECAM_MCFG(1, 1),
+	XGENE_V1_ECAM_MCFG(1, 2),
+	XGENE_V1_ECAM_MCFG(1, 3),
+	XGENE_V1_ECAM_MCFG(1, 4),
+	XGENE_V1_ECAM_MCFG(2, 0),
+	XGENE_V1_ECAM_MCFG(2, 1),
+	XGENE_V1_ECAM_MCFG(2, 2),
+	XGENE_V1_ECAM_MCFG(2, 3),
+	XGENE_V1_ECAM_MCFG(2, 4),
+	/* X-Gene SoC with v2.1 PCIe controller */
+	XGENE_V2_ECAM_MCFG(3, 0),
+	XGENE_V2_ECAM_MCFG(3, 1),
+	/* X-Gene SoC with v2.2 PCIe controller */
+	XGENE_V2_ECAM_MCFG(4, 0),
+	XGENE_V2_ECAM_MCFG(4, 1),
+	XGENE_V2_ECAM_MCFG(4, 2),
+};
+
+static char mcfg_oem_id[ACPI_OEM_ID_SIZE];
+static char mcfg_oem_table_id[ACPI_OEM_TABLE_ID_SIZE];
+static u32 mcfg_oem_revision;
+
+static int pci_mcfg_quirk_matches(struct mcfg_fixup *f, u16 segment,
+				  struct resource *bus_range)
+{
+	if (!memcmp(f->oem_id, mcfg_oem_id, ACPI_OEM_ID_SIZE) &&
+	    !memcmp(f->oem_table_id, mcfg_oem_table_id,
+	            ACPI_OEM_TABLE_ID_SIZE) &&
+	    f->oem_revision == mcfg_oem_revision &&
+	    f->segment == segment &&
+	    resource_contains(&f->bus_range, bus_range))
+		return 1;
+
+	return 0;
+}
+#endif
+
+static void pci_mcfg_apply_quirks(struct acpi_pci_root *root,
+				  struct resource *cfgres,
+				  struct pci_ecam_ops **ecam_ops)
+{
+#ifdef CONFIG_PCI_QUIRKS
+	u16 segment = root->segment;
+	struct resource *bus_range = &root->secondary;
+	struct mcfg_fixup *f;
+	int i;
+
+	for (i = 0, f = mcfg_quirks; i < ARRAY_SIZE(mcfg_quirks); i++, f++) {
+		if (pci_mcfg_quirk_matches(f, segment, bus_range)) {
+			if (f->cfgres.start)
+				*cfgres = f->cfgres;
+			if (f->ops)
+				*ecam_ops =  f->ops;
+			dev_info(&root->device->dev, "MCFG quirk: ECAM at %pR for %pR with %ps\n",
+				 cfgres, bus_range, *ecam_ops);
+			return;
+		}
+	}
+#endif
+}
+
 /* List to save MCFG entries */
 static LIST_HEAD(pci_mcfg_list);
 
-phys_addr_t pci_mcfg_lookup(u16 seg, struct resource *bus_res)
+int pci_mcfg_lookup(struct acpi_pci_root *root, struct resource *cfgres,
+		    struct pci_ecam_ops **ecam_ops)
 {
+	struct pci_ecam_ops *ops = &pci_generic_ecam_ops;
+	struct resource *bus_res = &root->secondary;
+	u16 seg = root->segment;
 	struct mcfg_entry *e;
+	struct resource res;
+
+	/* Use address from _CBA if present, otherwise lookup MCFG */
+	if (root->mcfg_addr)
+		goto skip_lookup;
 
 	/*
 	 * We expect exact match, unless MCFG entry end bus covers more than
@@ -45,10 +200,32 @@ phys_addr_t pci_mcfg_lookup(u16 seg, struct resource *bus_res)
 	 */
 	list_for_each_entry(e, &pci_mcfg_list, list) {
 		if (e->segment == seg && e->bus_start == bus_res->start &&
-		    e->bus_end >= bus_res->end)
-			return e->addr;
+		    e->bus_end >= bus_res->end) {
+			root->mcfg_addr = e->addr;
+		}
+
+	}
+
+skip_lookup:
+	memset(&res, 0, sizeof(res));
+	if (root->mcfg_addr) {
+		res.start = root->mcfg_addr + (bus_res->start << 20);
+		res.end = res.start + (resource_size(bus_res) << 20) - 1;
+		res.flags = IORESOURCE_MEM;
 	}
 
+	/*
+	 * Allow quirks to override default ECAM ops and CFG resource
+	 * range.  This may even fabricate a CFG resource range in case
+	 * MCFG does not have it.  Invalid CFG start address means MCFG
+	 * firmware bug or we need another quirk in array.
+	 */
+	pci_mcfg_apply_quirks(root, &res, &ops);
+	if (!res.start)
+		return -ENXIO;
+
+	*cfgres = res;
+	*ecam_ops = ops;
 	return 0;
 }
 
@@ -79,6 +256,13 @@ static __init int pci_mcfg_parse(struct acpi_table_header *header)
 		list_add(&e->list, &pci_mcfg_list);
 	}
 
+#ifdef CONFIG_PCI_QUIRKS
+	/* Save MCFG IDs and revision for quirks matching */
+	memcpy(mcfg_oem_id, header->oem_id, ACPI_OEM_ID_SIZE);
+	memcpy(mcfg_oem_table_id, header->oem_table_id, ACPI_OEM_TABLE_ID_SIZE);
+	mcfg_oem_revision = header->oem_revision;
+#endif
+
 	pr_info("MCFG table detected, %d entries\n", n);
 	return 0;
 }
diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c
index bb01dea39fdc..f0b4a981b8d3 100644
--- a/drivers/acpi/processor_perflib.c
+++ b/drivers/acpi/processor_perflib.c
@@ -157,7 +157,7 @@ static void acpi_processor_ppc_ost(acpi_handle handle, int status)
 				  status, NULL);
 }
 
-int acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag)
+void acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag)
 {
 	int ret;
 
@@ -168,7 +168,7 @@ int acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag)
 		 */
 		if (event_flag)
 			acpi_processor_ppc_ost(pr->handle, 1);
-		return 0;
+		return;
 	}
 
 	ret = acpi_processor_get_platform_limit(pr);
@@ -182,10 +182,8 @@ int acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag)
 		else
 			acpi_processor_ppc_ost(pr->handle, 0);
 	}
-	if (ret < 0)
-		return (ret);
-	else
-		return cpufreq_update_policy(pr->id);
+	if (ret >= 0)
+		cpufreq_update_policy(pr->id);
 }
 
 int acpi_processor_get_bios_limit(int cpu, unsigned int *limit)
@@ -465,11 +463,33 @@ int acpi_processor_get_performance_info(struct acpi_processor *pr)
 	return result;
 }
 EXPORT_SYMBOL_GPL(acpi_processor_get_performance_info);
-int acpi_processor_notify_smm(struct module *calling_module)
+
+int acpi_processor_pstate_control(void)
 {
 	acpi_status status;
-	static int is_done = 0;
 
+	if (!acpi_gbl_FADT.smi_command || !acpi_gbl_FADT.pstate_control)
+		return 0;
+
+	ACPI_DEBUG_PRINT((ACPI_DB_INFO,
+			  "Writing pstate_control [0x%x] to smi_command [0x%x]\n",
+			  acpi_gbl_FADT.pstate_control, acpi_gbl_FADT.smi_command));
+
+	status = acpi_os_write_port(acpi_gbl_FADT.smi_command,
+				    (u32)acpi_gbl_FADT.pstate_control, 8);
+	if (ACPI_SUCCESS(status))
+		return 1;
+
+	ACPI_EXCEPTION((AE_INFO, status,
+			"Failed to write pstate_control [0x%x] to smi_command [0x%x]",
+			acpi_gbl_FADT.pstate_control, acpi_gbl_FADT.smi_command));
+	return -EIO;
+}
+
+int acpi_processor_notify_smm(struct module *calling_module)
+{
+	static int is_done = 0;
+	int result;
 
 	if (!(acpi_processor_ppc_status & PPC_REGISTERED))
 		return -EBUSY;
@@ -492,26 +512,15 @@ int acpi_processor_notify_smm(struct module *calling_module)
 
 	is_done = -EIO;
 
-	/* Can't write pstate_control to smi_command if either value is zero */
-	if ((!acpi_gbl_FADT.smi_command) || (!acpi_gbl_FADT.pstate_control)) {
+	result = acpi_processor_pstate_control();
+	if (!result) {
 		ACPI_DEBUG_PRINT((ACPI_DB_INFO, "No SMI port or pstate_control\n"));
 		module_put(calling_module);
 		return 0;
 	}
-
-	ACPI_DEBUG_PRINT((ACPI_DB_INFO,
-			  "Writing pstate_control [0x%x] to smi_command [0x%x]\n",
-			  acpi_gbl_FADT.pstate_control, acpi_gbl_FADT.smi_command));
-
-	status = acpi_os_write_port(acpi_gbl_FADT.smi_command,
-				    (u32) acpi_gbl_FADT.pstate_control, 8);
-	if (ACPI_FAILURE(status)) {
-		ACPI_EXCEPTION((AE_INFO, status,
-				"Failed to write pstate_control [0x%x] to "
-				"smi_command [0x%x]", acpi_gbl_FADT.pstate_control,
-				acpi_gbl_FADT.smi_command));
+	if (result < 0) {
 		module_put(calling_module);
-		return status;
+		return result;
 	}
 
 	/* Success. If there's no _PPC, we need to fear nothing, so
diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c
index 03f5ec11ab31..3afddcd834ef 100644
--- a/drivers/acpi/property.c
+++ b/drivers/acpi/property.c
@@ -41,14 +41,13 @@ static bool acpi_enumerate_nondev_subnodes(acpi_handle scope,
 static bool acpi_extract_properties(const union acpi_object *desc,
 				    struct acpi_device_data *data);
 
-static bool acpi_nondev_subnode_ok(acpi_handle scope,
-				   const union acpi_object *link,
-				   struct list_head *list)
+static bool acpi_nondev_subnode_extract(const union acpi_object *desc,
+					acpi_handle handle,
+					const union acpi_object *link,
+					struct list_head *list)
 {
-	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER };
 	struct acpi_data_node *dn;
-	acpi_handle handle;
-	acpi_status status;
+	bool result;
 
 	dn = kzalloc(sizeof(*dn), GFP_KERNEL);
 	if (!dn)
@@ -58,43 +57,75 @@ static bool acpi_nondev_subnode_ok(acpi_handle scope,
 	dn->fwnode.type = FWNODE_ACPI_DATA;
 	INIT_LIST_HEAD(&dn->data.subnodes);
 
-	status = acpi_get_handle(scope, link->package.elements[1].string.pointer,
-				 &handle);
-	if (ACPI_FAILURE(status))
-		goto fail;
+	result = acpi_extract_properties(desc, &dn->data);
 
-	status = acpi_evaluate_object_typed(handle, NULL, NULL, &buf,
-					    ACPI_TYPE_PACKAGE);
-	if (ACPI_FAILURE(status))
-		goto fail;
+	if (handle) {
+		acpi_handle scope;
+		acpi_status status;
 
-	if (acpi_extract_properties(buf.pointer, &dn->data))
-		dn->handle = handle;
+		/*
+		 * The scope for the subnode object lookup is the one of the
+		 * namespace node (device) containing the object that has
+		 * returned the package.  That is, it's the scope of that
+		 * object's parent.
+		 */
+		status = acpi_get_parent(handle, &scope);
+		if (ACPI_SUCCESS(status)
+		    && acpi_enumerate_nondev_subnodes(scope, desc, &dn->data))
+			result = true;
+	} else if (acpi_enumerate_nondev_subnodes(NULL, desc, &dn->data)) {
+		result = true;
+	}
 
-	/*
-	 * The scope for the subnode object lookup is the one of the namespace
-	 * node (device) containing the object that has returned the package.
-	 * That is, it's the scope of that object's parent.
-	 */
-	status = acpi_get_parent(handle, &scope);
-	if (ACPI_SUCCESS(status)
-	    && acpi_enumerate_nondev_subnodes(scope, buf.pointer, &dn->data))
+	if (result) {
 		dn->handle = handle;
-
-	if (dn->handle) {
-		dn->data.pointer = buf.pointer;
+		dn->data.pointer = desc;
 		list_add_tail(&dn->sibling, list);
 		return true;
 	}
 
+	kfree(dn);
 	acpi_handle_debug(handle, "Invalid properties/subnodes data, skipping\n");
+	return false;
+}
+
+static bool acpi_nondev_subnode_data_ok(acpi_handle handle,
+					const union acpi_object *link,
+					struct list_head *list)
+{
+	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER };
+	acpi_status status;
+
+	status = acpi_evaluate_object_typed(handle, NULL, NULL, &buf,
+					    ACPI_TYPE_PACKAGE);
+	if (ACPI_FAILURE(status))
+		return false;
+
+	if (acpi_nondev_subnode_extract(buf.pointer, handle, link, list))
+		return true;
 
- fail:
 	ACPI_FREE(buf.pointer);
-	kfree(dn);
 	return false;
 }
 
+static bool acpi_nondev_subnode_ok(acpi_handle scope,
+				   const union acpi_object *link,
+				   struct list_head *list)
+{
+	acpi_handle handle;
+	acpi_status status;
+
+	if (!scope)
+		return false;
+
+	status = acpi_get_handle(scope, link->package.elements[1].string.pointer,
+				 &handle);
+	if (ACPI_FAILURE(status))
+		return false;
+
+	return acpi_nondev_subnode_data_ok(handle, link, list);
+}
+
 static int acpi_add_nondev_subnodes(acpi_handle scope,
 				    const union acpi_object *links,
 				    struct list_head *list)
@@ -103,15 +134,37 @@ static int acpi_add_nondev_subnodes(acpi_handle scope,
 	int i;
 
 	for (i = 0; i < links->package.count; i++) {
-		const union acpi_object *link;
+		const union acpi_object *link, *desc;
+		acpi_handle handle;
+		bool result;
 
 		link = &links->package.elements[i];
-		/* Only two elements allowed, both must be strings. */
-		if (link->package.count == 2
-		    && link->package.elements[0].type == ACPI_TYPE_STRING
-		    && link->package.elements[1].type == ACPI_TYPE_STRING
-		    && acpi_nondev_subnode_ok(scope, link, list))
-			ret = true;
+		/* Only two elements allowed. */
+		if (link->package.count != 2)
+			continue;
+
+		/* The first one must be a string. */
+		if (link->package.elements[0].type != ACPI_TYPE_STRING)
+			continue;
+
+		/* The second one may be a string, a reference or a package. */
+		switch (link->package.elements[1].type) {
+		case ACPI_TYPE_STRING:
+			result = acpi_nondev_subnode_ok(scope, link, list);
+			break;
+		case ACPI_TYPE_LOCAL_REFERENCE:
+			handle = link->package.elements[1].reference.handle;
+			result = acpi_nondev_subnode_data_ok(handle, link, list);
+			break;
+		case ACPI_TYPE_PACKAGE:
+			desc = &link->package.elements[1];
+			result = acpi_nondev_subnode_extract(desc, NULL, link, list);
+			break;
+		default:
+			result = false;
+			break;
+		}
+		ret = ret || result;
 	}
 
 	return ret;
diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c
index 56241eb341f4..cb57962ef7c4 100644
--- a/drivers/acpi/resource.c
+++ b/drivers/acpi/resource.c
@@ -664,3 +664,60 @@ int acpi_dev_filter_resource_type(struct acpi_resource *ares,
 	return (type & types) ? 0 : 1;
 }
 EXPORT_SYMBOL_GPL(acpi_dev_filter_resource_type);
+
+static int acpi_dev_consumes_res(struct acpi_device *adev, struct resource *res)
+{
+	struct list_head resource_list;
+	struct resource_entry *rentry;
+	int ret, found = 0;
+
+	INIT_LIST_HEAD(&resource_list);
+	ret = acpi_dev_get_resources(adev, &resource_list, NULL, NULL);
+	if (ret < 0)
+		return 0;
+
+	list_for_each_entry(rentry, &resource_list, node) {
+		if (resource_contains(rentry->res, res)) {
+			found = 1;
+			break;
+		}
+
+	}
+
+	acpi_dev_free_resource_list(&resource_list);
+	return found;
+}
+
+static acpi_status acpi_res_consumer_cb(acpi_handle handle, u32 depth,
+					 void *context, void **ret)
+{
+	struct resource *res = context;
+	struct acpi_device **consumer = (struct acpi_device **) ret;
+	struct acpi_device *adev;
+
+	if (acpi_bus_get_device(handle, &adev))
+		return AE_OK;
+
+	if (acpi_dev_consumes_res(adev, res)) {
+		*consumer = adev;
+		return AE_CTRL_TERMINATE;
+	}
+
+	return AE_OK;
+}
+
+/**
+ * acpi_resource_consumer - Find the ACPI device that consumes @res.
+ * @res: Resource to search for.
+ *
+ * Search the current resource settings (_CRS) of every ACPI device node
+ * for @res.  If we find an ACPI device whose _CRS includes @res, return
+ * it.  Otherwise, return NULL.
+ */
+struct acpi_device *acpi_resource_consumer(struct resource *res)
+{
+	struct acpi_device *consumer = NULL;
+
+	acpi_get_devices(NULL, acpi_res_consumer_cb, res, (void **) &consumer);
+	return consumer;
+}
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index 3d1856f1f4d0..93b00cf4eb39 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -7,6 +7,7 @@
 #include <linux/slab.h>
 #include <linux/kernel.h>
 #include <linux/acpi.h>
+#include <linux/acpi_iort.h>
 #include <linux/signal.h>
 #include <linux/kthread.h>
 #include <linux/dmi.h>
@@ -1370,6 +1371,38 @@ enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev)
 		return DEV_DMA_NON_COHERENT;
 }
 
+/**
+ * acpi_dma_configure - Set-up DMA configuration for the device.
+ * @dev: The pointer to the device
+ * @attr: device dma attributes
+ */
+void acpi_dma_configure(struct device *dev, enum dev_dma_attr attr)
+{
+	const struct iommu_ops *iommu;
+
+	iort_set_dma_mask(dev);
+
+	iommu = iort_iommu_configure(dev);
+
+	/*
+	 * Assume dma valid range starts at 0 and covers the whole
+	 * coherent_dma_mask.
+	 */
+	arch_setup_dma_ops(dev, 0, dev->coherent_dma_mask + 1, iommu,
+			   attr == DEV_DMA_COHERENT);
+}
+EXPORT_SYMBOL_GPL(acpi_dma_configure);
+
+/**
+ * acpi_dma_deconfigure - Tear-down DMA configuration for the device.
+ * @dev: The pointer to the device
+ */
+void acpi_dma_deconfigure(struct device *dev)
+{
+	arch_teardown_dma_ops(dev);
+}
+EXPORT_SYMBOL_GPL(acpi_dma_deconfigure);
+
 static void acpi_init_coherency(struct acpi_device *adev)
 {
 	unsigned long long cca = 0;
diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c
index 54abb26b7366..9b6cebe227a0 100644
--- a/drivers/acpi/sleep.c
+++ b/drivers/acpi/sleep.c
@@ -674,6 +674,14 @@ static void acpi_sleep_suspend_setup(void)
 		if (acpi_sleep_state_supported(i))
 			sleep_states[i] = 1;
 
+	/*
+	 * Use suspend-to-idle by default if ACPI_FADT_LOW_POWER_S0 is set and
+	 * the default suspend mode was not selected from the command line.
+	 */
+	if (acpi_gbl_FADT.flags & ACPI_FADT_LOW_POWER_S0 &&
+	    mem_sleep_default > PM_SUSPEND_MEM)
+		mem_sleep_default = PM_SUSPEND_FREEZE;
+
 	suspend_set_ops(old_suspend_ordering ?
 		&acpi_suspend_ops_old : &acpi_suspend_ops);
 	freeze_set_ops(&acpi_freeze_ops);
diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c
index a6b36fc53aec..02ded25c82e4 100644
--- a/drivers/acpi/video_detect.c
+++ b/drivers/acpi/video_detect.c
@@ -296,6 +296,26 @@ static const struct dmi_system_id video_detect_dmi_table[] = {
 		DMI_MATCH(DMI_PRODUCT_NAME, "Vostro V131"),
 		},
 	},
+	{
+	 /* https://bugzilla.redhat.com/show_bug.cgi?id=1123661 */
+	 .callback = video_detect_force_native,
+	 .ident = "Dell XPS 17 L702X",
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "Dell System XPS L702X"),
+		},
+	},
+	{
+	/* https://bugzilla.redhat.com/show_bug.cgi?id=1204476 */
+	/* https://bugs.launchpad.net/ubuntu/+source/linux-lts-trusty/+bug/1416940 */
+	.callback = video_detect_force_native,
+	.ident = "HP Pavilion dv6",
+	.matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "HP Pavilion dv6 Notebook PC"),
+		},
+	},
+
 	{ },
 };
 
diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
index 74f4c662f776..2fc52407306c 100644
--- a/drivers/ata/ahci.c
+++ b/drivers/ata/ahci.c
@@ -46,6 +46,8 @@
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_cmnd.h>
 #include <linux/libata.h>
+#include <linux/ahci-remap.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
 #include "ahci.h"
 
 #define DRV_NAME	"ahci"
@@ -1400,6 +1402,40 @@ static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance)
 }
 #endif
 
+static void ahci_remap_check(struct pci_dev *pdev, int bar,
+		struct ahci_host_priv *hpriv)
+{
+	int i, count = 0;
+	u32 cap;
+
+	/*
+	 * Check if this device might have remapped nvme devices.
+	 */
+	if (pdev->vendor != PCI_VENDOR_ID_INTEL ||
+	    pci_resource_len(pdev, bar) < SZ_512K ||
+	    bar != AHCI_PCI_BAR_STANDARD ||
+	    !(readl(hpriv->mmio + AHCI_VSCAP) & 1))
+		return;
+
+	cap = readq(hpriv->mmio + AHCI_REMAP_CAP);
+	for (i = 0; i < AHCI_MAX_REMAP; i++) {
+		if ((cap & (1 << i)) == 0)
+			continue;
+		if (readl(hpriv->mmio + ahci_remap_dcc(i))
+				!= PCI_CLASS_STORAGE_EXPRESS)
+			continue;
+
+		/* We've found a remapped device */
+		count++;
+	}
+
+	if (!count)
+		return;
+
+	dev_warn(&pdev->dev, "Found %d remapped NVMe devices.\n", count);
+	dev_warn(&pdev->dev, "Switch your BIOS from RAID to AHCI mode to use them.\n");
+}
+
 static int ahci_get_irq_vector(struct ata_host *host, int port)
 {
 	return pci_irq_vector(to_pci_dev(host->dev), port);
@@ -1541,6 +1577,9 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	hpriv->mmio = pcim_iomap_table(pdev)[ahci_pci_bar];
 
+	/* detect remapped nvme devices */
+	ahci_remap_check(pdev, ahci_pci_bar, hpriv);
+
 	/* must set flag prior to save config in order to take effect */
 	if (ahci_broken_devslp(pdev))
 		hpriv->flags |= AHCI_HFLAG_NO_DEVSLP;
diff --git a/drivers/ata/ahci_qoriq.c b/drivers/ata/ahci_qoriq.c
index 1eba8dff875e..9884c8c6e934 100644
--- a/drivers/ata/ahci_qoriq.c
+++ b/drivers/ata/ahci_qoriq.c
@@ -46,11 +46,13 @@
 #define LS1021A_AXICC_ADDR	0xC0
 
 #define SATA_ECC_DISABLE	0x00020000
+#define LS1046A_SATA_ECC_DIS	0x80000000
 
 enum ahci_qoriq_type {
 	AHCI_LS1021A,
 	AHCI_LS1043A,
 	AHCI_LS2080A,
+	AHCI_LS1046A,
 };
 
 struct ahci_qoriq_priv {
@@ -63,6 +65,7 @@ static const struct of_device_id ahci_qoriq_of_match[] = {
 	{ .compatible = "fsl,ls1021a-ahci", .data = (void *)AHCI_LS1021A},
 	{ .compatible = "fsl,ls1043a-ahci", .data = (void *)AHCI_LS1043A},
 	{ .compatible = "fsl,ls2080a-ahci", .data = (void *)AHCI_LS2080A},
+	{ .compatible = "fsl,ls1046a-ahci", .data = (void *)AHCI_LS1046A},
 	{},
 };
 MODULE_DEVICE_TABLE(of, ahci_qoriq_of_match);
@@ -175,6 +178,13 @@ static int ahci_qoriq_phy_init(struct ahci_host_priv *hpriv)
 		writel(AHCI_PORT_TRANS_CFG, reg_base + PORT_TRANS);
 		writel(AHCI_PORT_AXICC_CFG, reg_base + PORT_AXICC);
 		break;
+
+	case AHCI_LS1046A:
+		writel(LS1046A_SATA_ECC_DIS, qpriv->ecc_addr);
+		writel(AHCI_PORT_PHY_1_CFG, reg_base + PORT_PHY1);
+		writel(AHCI_PORT_TRANS_CFG, reg_base + PORT_TRANS);
+		writel(AHCI_PORT_AXICC_CFG, reg_base + PORT_AXICC);
+		break;
 	}
 
 	return 0;
@@ -204,9 +214,9 @@ static int ahci_qoriq_probe(struct platform_device *pdev)
 
 	qoriq_priv->type = (enum ahci_qoriq_type)of_id->data;
 
-	if (qoriq_priv->type == AHCI_LS1021A) {
-		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
-				"sata-ecc");
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+			"sata-ecc");
+	if (res) {
 		qoriq_priv->ecc_addr = devm_ioremap_resource(dev, res);
 		if (IS_ERR(qoriq_priv->ecc_addr))
 			return PTR_ERR(qoriq_priv->ecc_addr);
diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c
index 0d028ead99e8..ee7db3119b18 100644
--- a/drivers/ata/libahci.c
+++ b/drivers/ata/libahci.c
@@ -140,6 +140,7 @@ EXPORT_SYMBOL_GPL(ahci_shost_attrs);
 struct device_attribute *ahci_sdev_attrs[] = {
 	&dev_attr_sw_activity,
 	&dev_attr_unload_heads,
+	&dev_attr_ncq_prio_enable,
 	NULL
 };
 EXPORT_SYMBOL_GPL(ahci_sdev_attrs);
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index 59ce0dd50701..9cd0a2d41816 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -739,6 +739,7 @@ u64 ata_tf_read_block(const struct ata_taskfile *tf, struct ata_device *dev)
  *	@n_block: Number of blocks
  *	@tf_flags: RW/FUA etc...
  *	@tag: tag
+ *	@class: IO priority class
  *
  *	LOCKING:
  *	None.
@@ -753,7 +754,7 @@ u64 ata_tf_read_block(const struct ata_taskfile *tf, struct ata_device *dev)
  */
 int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev,
 		    u64 block, u32 n_block, unsigned int tf_flags,
-		    unsigned int tag)
+		    unsigned int tag, int class)
 {
 	tf->flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_DEVICE;
 	tf->flags |= tf_flags;
@@ -785,6 +786,12 @@ int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev,
 		tf->device = ATA_LBA;
 		if (tf->flags & ATA_TFLAG_FUA)
 			tf->device |= 1 << 7;
+
+		if (dev->flags & ATA_DFLAG_NCQ_PRIO) {
+			if (class == IOPRIO_CLASS_RT)
+				tf->hob_nsect |= ATA_PRIO_HIGH <<
+						 ATA_SHIFT_PRIO;
+		}
 	} else if (dev->flags & ATA_DFLAG_LBA) {
 		tf->flags |= ATA_TFLAG_LBA;
 
@@ -2156,6 +2163,37 @@ static void ata_dev_config_ncq_non_data(struct ata_device *dev)
 	}
 }
 
+static void ata_dev_config_ncq_prio(struct ata_device *dev)
+{
+	struct ata_port *ap = dev->link->ap;
+	unsigned int err_mask;
+
+	if (!(dev->flags & ATA_DFLAG_NCQ_PRIO_ENABLE)) {
+		dev->flags &= ~ATA_DFLAG_NCQ_PRIO;
+		return;
+	}
+
+	err_mask = ata_read_log_page(dev,
+				     ATA_LOG_SATA_ID_DEV_DATA,
+				     ATA_LOG_SATA_SETTINGS,
+				     ap->sector_buf,
+				     1);
+	if (err_mask) {
+		ata_dev_dbg(dev,
+			    "failed to get Identify Device data, Emask 0x%x\n",
+			    err_mask);
+		return;
+	}
+
+	if (ap->sector_buf[ATA_LOG_NCQ_PRIO_OFFSET] & BIT(3)) {
+		dev->flags |= ATA_DFLAG_NCQ_PRIO;
+	} else {
+		dev->flags &= ~ATA_DFLAG_NCQ_PRIO;
+		ata_dev_dbg(dev, "SATA page does not support priority\n");
+	}
+
+}
+
 static int ata_dev_config_ncq(struct ata_device *dev,
 			       char *desc, size_t desc_sz)
 {
@@ -2205,6 +2243,8 @@ static int ata_dev_config_ncq(struct ata_device *dev,
 			ata_dev_config_ncq_send_recv(dev);
 		if (ata_id_has_ncq_non_data(dev->id))
 			ata_dev_config_ncq_non_data(dev);
+		if (ata_id_has_ncq_prio(dev->id))
+			ata_dev_config_ncq_prio(dev);
 	}
 
 	return 0;
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index 8e575fbdf31d..1f863e757ee4 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -50,6 +50,7 @@
 #include <linux/uaccess.h>
 #include <linux/suspend.h>
 #include <asm/unaligned.h>
+#include <linux/ioprio.h>
 
 #include "libata.h"
 #include "libata-transport.h"
@@ -270,6 +271,83 @@ DEVICE_ATTR(unload_heads, S_IRUGO | S_IWUSR,
 	    ata_scsi_park_show, ata_scsi_park_store);
 EXPORT_SYMBOL_GPL(dev_attr_unload_heads);
 
+static ssize_t ata_ncq_prio_enable_show(struct device *device,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct scsi_device *sdev = to_scsi_device(device);
+	struct ata_port *ap;
+	struct ata_device *dev;
+	bool ncq_prio_enable;
+	int rc = 0;
+
+	ap = ata_shost_to_port(sdev->host);
+
+	spin_lock_irq(ap->lock);
+	dev = ata_scsi_find_dev(ap, sdev);
+	if (!dev) {
+		rc = -ENODEV;
+		goto unlock;
+	}
+
+	ncq_prio_enable = dev->flags & ATA_DFLAG_NCQ_PRIO_ENABLE;
+
+unlock:
+	spin_unlock_irq(ap->lock);
+
+	return rc ? rc : snprintf(buf, 20, "%u\n", ncq_prio_enable);
+}
+
+static ssize_t ata_ncq_prio_enable_store(struct device *device,
+					 struct device_attribute *attr,
+					 const char *buf, size_t len)
+{
+	struct scsi_device *sdev = to_scsi_device(device);
+	struct ata_port *ap;
+	struct ata_device *dev;
+	long int input;
+	int rc;
+
+	rc = kstrtol(buf, 10, &input);
+	if (rc)
+		return rc;
+	if ((input < 0) || (input > 1))
+		return -EINVAL;
+
+	ap = ata_shost_to_port(sdev->host);
+	dev = ata_scsi_find_dev(ap, sdev);
+	if (unlikely(!dev))
+		return  -ENODEV;
+
+	spin_lock_irq(ap->lock);
+	if (input)
+		dev->flags |= ATA_DFLAG_NCQ_PRIO_ENABLE;
+	else
+		dev->flags &= ~ATA_DFLAG_NCQ_PRIO_ENABLE;
+
+	dev->link->eh_info.action |= ATA_EH_REVALIDATE;
+	dev->link->eh_info.flags |= ATA_EHI_QUIET;
+	ata_port_schedule_eh(ap);
+	spin_unlock_irq(ap->lock);
+
+	ata_port_wait_eh(ap);
+
+	if (input) {
+		spin_lock_irq(ap->lock);
+		if (!(dev->flags & ATA_DFLAG_NCQ_PRIO)) {
+			dev->flags &= ~ATA_DFLAG_NCQ_PRIO_ENABLE;
+			rc = -EIO;
+		}
+		spin_unlock_irq(ap->lock);
+	}
+
+	return rc ? rc : len;
+}
+
+DEVICE_ATTR(ncq_prio_enable, S_IRUGO | S_IWUSR,
+	    ata_ncq_prio_enable_show, ata_ncq_prio_enable_store);
+EXPORT_SYMBOL_GPL(dev_attr_ncq_prio_enable);
+
 void ata_scsi_set_sense(struct ata_device *dev, struct scsi_cmnd *cmd,
 			u8 sk, u8 asc, u8 ascq)
 {
@@ -401,6 +479,7 @@ EXPORT_SYMBOL_GPL(dev_attr_sw_activity);
 
 struct device_attribute *ata_common_sdev_attrs[] = {
 	&dev_attr_unload_heads,
+	&dev_attr_ncq_prio_enable,
 	NULL
 };
 EXPORT_SYMBOL_GPL(ata_common_sdev_attrs);
@@ -1756,6 +1835,8 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc)
 {
 	struct scsi_cmnd *scmd = qc->scsicmd;
 	const u8 *cdb = scmd->cmnd;
+	struct request *rq = scmd->request;
+	int class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
 	unsigned int tf_flags = 0;
 	u64 block;
 	u32 n_block;
@@ -1822,7 +1903,8 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc)
 	qc->nbytes = n_block * scmd->device->sector_size;
 
 	rc = ata_build_rw_tf(&qc->tf, qc->dev, block, n_block, tf_flags,
-			     qc->tag);
+			     qc->tag, class);
+
 	if (likely(rc == 0))
 		return 0;
 
diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h
index 3b301a48007c..8f3a5596dd67 100644
--- a/drivers/ata/libata.h
+++ b/drivers/ata/libata.h
@@ -66,7 +66,7 @@ extern u64 ata_tf_to_lba48(const struct ata_taskfile *tf);
 extern struct ata_queued_cmd *ata_qc_new_init(struct ata_device *dev, int tag);
 extern int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev,
 			   u64 block, u32 n_block, unsigned int tf_flags,
-			   unsigned int tag);
+			   unsigned int tag, int class);
 extern u64 ata_tf_read_block(const struct ata_taskfile *tf,
 			     struct ata_device *dev);
 extern unsigned ata_exec_internal(struct ata_device *dev,
diff --git a/drivers/ata/pata_imx.c b/drivers/ata/pata_imx.c
index 139d20778b29..d4caa23f5a88 100644
--- a/drivers/ata/pata_imx.c
+++ b/drivers/ata/pata_imx.c
@@ -11,19 +11,26 @@
  *
  * TODO:
  * - dmaengine support
- * - check if timing stuff needed
  */
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/blkdev.h>
-#include <scsi/scsi_host.h>
+
 #include <linux/ata.h>
+#include <linux/clk.h>
 #include <linux/libata.h>
+#include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/clk.h>
 
 #define DRV_NAME "pata_imx"
 
+#define PATA_IMX_ATA_TIME_OFF		0x00
+#define PATA_IMX_ATA_TIME_ON		0x01
+#define PATA_IMX_ATA_TIME_1		0x02
+#define PATA_IMX_ATA_TIME_2W		0x03
+#define PATA_IMX_ATA_TIME_2R		0x04
+#define PATA_IMX_ATA_TIME_AX		0x05
+#define PATA_IMX_ATA_TIME_PIO_RDX	0x06
+#define PATA_IMX_ATA_TIME_4		0x07
+#define PATA_IMX_ATA_TIME_9		0x08
+
 #define PATA_IMX_ATA_CONTROL		0x24
 #define PATA_IMX_ATA_CTRL_FIFO_RST_B	(1<<7)
 #define PATA_IMX_ATA_CTRL_ATA_RST_B	(1<<6)
@@ -33,6 +40,10 @@
 #define PATA_IMX_DRIVE_DATA		0xA0
 #define PATA_IMX_DRIVE_CONTROL		0xD8
 
+static u32 pio_t4[] = { 30,  20,  15,  10,  10 };
+static u32 pio_t9[] = { 20,  15,  10,  10,  10 };
+static u32 pio_tA[] = { 35,  35,  35,  35,  35 };
+
 struct pata_imx_priv {
 	struct clk *clk;
 	/* timings/interrupt/control regs */
@@ -40,28 +51,49 @@ struct pata_imx_priv {
 	u32 ata_ctl;
 };
 
-static int pata_imx_set_mode(struct ata_link *link, struct ata_device **unused)
+static void pata_imx_set_timing(struct ata_device *adev,
+				struct pata_imx_priv *priv)
+{
+	struct ata_timing timing;
+	unsigned long clkrate;
+	u32 T, mode;
+
+	clkrate = clk_get_rate(priv->clk);
+
+	if (adev->pio_mode < XFER_PIO_0 || adev->pio_mode > XFER_PIO_4 ||
+	    !clkrate)
+		return;
+
+	T = 1000000000 / clkrate;
+	ata_timing_compute(adev, adev->pio_mode, &timing, T * 1000, 0);
+
+	mode = adev->pio_mode - XFER_PIO_0;
+
+	writeb(3, priv->host_regs + PATA_IMX_ATA_TIME_OFF);
+	writeb(3, priv->host_regs + PATA_IMX_ATA_TIME_ON);
+	writeb(timing.setup, priv->host_regs + PATA_IMX_ATA_TIME_1);
+	writeb(timing.act8b, priv->host_regs + PATA_IMX_ATA_TIME_2W);
+	writeb(timing.act8b, priv->host_regs + PATA_IMX_ATA_TIME_2R);
+	writeb(1, priv->host_regs + PATA_IMX_ATA_TIME_PIO_RDX);
+
+	writeb(pio_t4[mode] / T + 1, priv->host_regs + PATA_IMX_ATA_TIME_4);
+	writeb(pio_t9[mode] / T + 1, priv->host_regs + PATA_IMX_ATA_TIME_9);
+	writeb(pio_tA[mode] / T + 1, priv->host_regs + PATA_IMX_ATA_TIME_AX);
+}
+
+static void pata_imx_set_piomode(struct ata_port *ap, struct ata_device *adev)
 {
-	struct ata_device *dev;
-	struct ata_port *ap = link->ap;
 	struct pata_imx_priv *priv = ap->host->private_data;
 	u32 val;
 
-	ata_for_each_dev(dev, link, ENABLED) {
-		dev->pio_mode = dev->xfer_mode = XFER_PIO_0;
-		dev->xfer_shift = ATA_SHIFT_PIO;
-		dev->flags |= ATA_DFLAG_PIO;
-
-		val = __raw_readl(priv->host_regs + PATA_IMX_ATA_CONTROL);
-		if (ata_pio_need_iordy(dev))
-			val |= PATA_IMX_ATA_CTRL_IORDY_EN;
-		else
-			val &= ~PATA_IMX_ATA_CTRL_IORDY_EN;
-		__raw_writel(val, priv->host_regs + PATA_IMX_ATA_CONTROL);
+	pata_imx_set_timing(adev, priv);
 
-		ata_dev_info(dev, "configured for PIO\n");
-	}
-	return 0;
+	val = __raw_readl(priv->host_regs + PATA_IMX_ATA_CONTROL);
+	if (ata_pio_need_iordy(adev))
+		val |= PATA_IMX_ATA_CTRL_IORDY_EN;
+	else
+		val &= ~PATA_IMX_ATA_CTRL_IORDY_EN;
+	__raw_writel(val, priv->host_regs + PATA_IMX_ATA_CONTROL);
 }
 
 static struct scsi_host_template pata_imx_sht = {
@@ -72,7 +104,7 @@ static struct ata_port_operations pata_imx_port_ops = {
 	.inherits		= &ata_sff_port_ops,
 	.sff_data_xfer		= ata_sff_data_xfer_noirq,
 	.cable_detect		= ata_cable_unknown,
-	.set_mode		= pata_imx_set_mode,
+	.set_piomode		= pata_imx_set_piomode,
 };
 
 static void pata_imx_setup_port(struct ata_ioports *ioaddr)
@@ -128,7 +160,7 @@ static int pata_imx_probe(struct platform_device *pdev)
 	ap = host->ports[0];
 
 	ap->ops = &pata_imx_port_ops;
-	ap->pio_mask = ATA_PIO0;
+	ap->pio_mask = ATA_PIO4;
 	ap->flags |= ATA_FLAG_SLAVE_POSS;
 
 	io_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
diff --git a/drivers/auxdisplay/Kconfig b/drivers/auxdisplay/Kconfig
index 10e1b9eee10e..4ef4c5caed4f 100644
--- a/drivers/auxdisplay/Kconfig
+++ b/drivers/auxdisplay/Kconfig
@@ -128,4 +128,17 @@ config IMG_ASCII_LCD
 	  development boards such as the MIPS Boston, MIPS Malta & MIPS SEAD3
 	  from Imagination Technologies.
 
+config HT16K33
+	tristate "Holtek Ht16K33 LED controller with keyscan"
+	depends on FB && OF && I2C && INPUT
+	select FB_SYS_FOPS
+	select FB_CFB_FILLRECT
+	select FB_CFB_COPYAREA
+	select FB_CFB_IMAGEBLIT
+	select INPUT_MATRIXKMAP
+	select FB_BACKLIGHT
+	help
+	  Say yes here to add support for Holtek HT16K33, RAM mapping 16*8
+	  LED controller driver with keyscan.
+
 endif # AUXDISPLAY
diff --git a/drivers/auxdisplay/Makefile b/drivers/auxdisplay/Makefile
index 3127175c89df..cb3dd847713b 100644
--- a/drivers/auxdisplay/Makefile
+++ b/drivers/auxdisplay/Makefile
@@ -5,3 +5,4 @@
 obj-$(CONFIG_KS0108)		+= ks0108.o
 obj-$(CONFIG_CFAG12864B)	+= cfag12864b.o cfag12864bfb.o
 obj-$(CONFIG_IMG_ASCII_LCD)	+= img-ascii-lcd.o
+obj-$(CONFIG_HT16K33)		+= ht16k33.o
diff --git a/drivers/auxdisplay/ht16k33.c b/drivers/auxdisplay/ht16k33.c
new file mode 100644
index 000000000000..eeb323f56c07
--- /dev/null
+++ b/drivers/auxdisplay/ht16k33.c
@@ -0,0 +1,563 @@
+/*
+ * HT16K33 driver
+ *
+ * Author: Robin van der Gracht <robin@protonic.nl>
+ *
+ * Copyright: (C) 2016 Protonic Holland.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/i2c.h>
+#include <linux/of.h>
+#include <linux/fb.h>
+#include <linux/slab.h>
+#include <linux/backlight.h>
+#include <linux/input.h>
+#include <linux/input/matrix_keypad.h>
+#include <linux/workqueue.h>
+#include <linux/mm.h>
+
+/* Registers */
+#define REG_SYSTEM_SETUP		0x20
+#define REG_SYSTEM_SETUP_OSC_ON		BIT(0)
+
+#define REG_DISPLAY_SETUP		0x80
+#define REG_DISPLAY_SETUP_ON		BIT(0)
+
+#define REG_ROWINT_SET			0xA0
+#define REG_ROWINT_SET_INT_EN		BIT(0)
+#define REG_ROWINT_SET_INT_ACT_HIGH	BIT(1)
+
+#define REG_BRIGHTNESS			0xE0
+
+/* Defines */
+#define DRIVER_NAME			"ht16k33"
+
+#define MIN_BRIGHTNESS			0x1
+#define MAX_BRIGHTNESS			0x10
+
+#define HT16K33_MATRIX_LED_MAX_COLS	8
+#define HT16K33_MATRIX_LED_MAX_ROWS	16
+#define HT16K33_MATRIX_KEYPAD_MAX_COLS	3
+#define HT16K33_MATRIX_KEYPAD_MAX_ROWS	12
+
+#define BYTES_PER_ROW		(HT16K33_MATRIX_LED_MAX_ROWS / 8)
+#define HT16K33_FB_SIZE		(HT16K33_MATRIX_LED_MAX_COLS * BYTES_PER_ROW)
+
+struct ht16k33_keypad {
+	struct input_dev *dev;
+	spinlock_t lock;
+	struct delayed_work work;
+	uint32_t cols;
+	uint32_t rows;
+	uint32_t row_shift;
+	uint32_t debounce_ms;
+	uint16_t last_key_state[HT16K33_MATRIX_KEYPAD_MAX_COLS];
+};
+
+struct ht16k33_fbdev {
+	struct fb_info *info;
+	uint32_t refresh_rate;
+	uint8_t *buffer;
+	uint8_t *cache;
+	struct delayed_work work;
+};
+
+struct ht16k33_priv {
+	struct i2c_client *client;
+	struct ht16k33_keypad keypad;
+	struct ht16k33_fbdev fbdev;
+	struct workqueue_struct *workqueue;
+};
+
+static struct fb_fix_screeninfo ht16k33_fb_fix = {
+	.id		= DRIVER_NAME,
+	.type		= FB_TYPE_PACKED_PIXELS,
+	.visual		= FB_VISUAL_MONO10,
+	.xpanstep	= 0,
+	.ypanstep	= 0,
+	.ywrapstep	= 0,
+	.line_length	= HT16K33_MATRIX_LED_MAX_ROWS,
+	.accel		= FB_ACCEL_NONE,
+};
+
+static struct fb_var_screeninfo ht16k33_fb_var = {
+	.xres = HT16K33_MATRIX_LED_MAX_ROWS,
+	.yres = HT16K33_MATRIX_LED_MAX_COLS,
+	.xres_virtual = HT16K33_MATRIX_LED_MAX_ROWS,
+	.yres_virtual = HT16K33_MATRIX_LED_MAX_COLS,
+	.bits_per_pixel = 1,
+	.red = { 0, 1, 0 },
+	.green = { 0, 1, 0 },
+	.blue = { 0, 1, 0 },
+	.left_margin = 0,
+	.right_margin = 0,
+	.upper_margin = 0,
+	.lower_margin = 0,
+	.vmode = FB_VMODE_NONINTERLACED,
+};
+
+static int ht16k33_display_on(struct ht16k33_priv *priv)
+{
+	uint8_t data = REG_DISPLAY_SETUP | REG_DISPLAY_SETUP_ON;
+
+	return i2c_smbus_write_byte(priv->client, data);
+}
+
+static int ht16k33_display_off(struct ht16k33_priv *priv)
+{
+	return i2c_smbus_write_byte(priv->client, REG_DISPLAY_SETUP);
+}
+
+static void ht16k33_fb_queue(struct ht16k33_priv *priv)
+{
+	struct ht16k33_fbdev *fbdev = &priv->fbdev;
+
+	queue_delayed_work(priv->workqueue, &fbdev->work,
+		msecs_to_jiffies(HZ / fbdev->refresh_rate));
+}
+
+static void ht16k33_keypad_queue(struct ht16k33_priv *priv)
+{
+	struct ht16k33_keypad *keypad = &priv->keypad;
+
+	queue_delayed_work(priv->workqueue, &keypad->work,
+		msecs_to_jiffies(keypad->debounce_ms));
+}
+
+/*
+ * This gets the fb data from cache and copies it to ht16k33 display RAM
+ */
+static void ht16k33_fb_update(struct work_struct *work)
+{
+	struct ht16k33_fbdev *fbdev =
+		container_of(work, struct ht16k33_fbdev, work.work);
+	struct ht16k33_priv *priv =
+		container_of(fbdev, struct ht16k33_priv, fbdev);
+
+	uint8_t *p1, *p2;
+	int len, pos = 0, first = -1;
+
+	p1 = fbdev->cache;
+	p2 = fbdev->buffer;
+
+	/* Search for the first byte with changes */
+	while (pos < HT16K33_FB_SIZE && first < 0) {
+		if (*(p1++) - *(p2++))
+			first = pos;
+		pos++;
+	}
+
+	/* No changes found */
+	if (first < 0)
+		goto requeue;
+
+	len = HT16K33_FB_SIZE - first;
+	p1 = fbdev->cache + HT16K33_FB_SIZE - 1;
+	p2 = fbdev->buffer + HT16K33_FB_SIZE - 1;
+
+	/* Determine i2c transfer length */
+	while (len > 1) {
+		if (*(p1--) - *(p2--))
+			break;
+		len--;
+	}
+
+	p1 = fbdev->cache + first;
+	p2 = fbdev->buffer + first;
+	if (!i2c_smbus_write_i2c_block_data(priv->client, first, len, p2))
+		memcpy(p1, p2, len);
+requeue:
+	ht16k33_fb_queue(priv);
+}
+
+static int ht16k33_keypad_start(struct input_dev *dev)
+{
+	struct ht16k33_priv *priv = input_get_drvdata(dev);
+	struct ht16k33_keypad *keypad = &priv->keypad;
+
+	/*
+	 * Schedule an immediate key scan to capture current key state;
+	 * columns will be activated and IRQs be enabled after the scan.
+	 */
+	queue_delayed_work(priv->workqueue, &keypad->work, 0);
+	return 0;
+}
+
+static void ht16k33_keypad_stop(struct input_dev *dev)
+{
+	struct ht16k33_priv *priv = input_get_drvdata(dev);
+	struct ht16k33_keypad *keypad = &priv->keypad;
+
+	cancel_delayed_work(&keypad->work);
+	/*
+	 * ht16k33_keypad_scan() will leave IRQs enabled;
+	 * we should disable them now.
+	 */
+	disable_irq_nosync(priv->client->irq);
+}
+
+static int ht16k33_initialize(struct ht16k33_priv *priv)
+{
+	uint8_t byte;
+	int err;
+	uint8_t data[HT16K33_MATRIX_LED_MAX_COLS * 2];
+
+	/* Clear RAM (8 * 16 bits) */
+	memset(data, 0, sizeof(data));
+	err = i2c_smbus_write_block_data(priv->client, 0, sizeof(data), data);
+	if (err)
+		return err;
+
+	/* Turn on internal oscillator */
+	byte = REG_SYSTEM_SETUP_OSC_ON | REG_SYSTEM_SETUP;
+	err = i2c_smbus_write_byte(priv->client, byte);
+	if (err)
+		return err;
+
+	/* Configure INT pin */
+	byte = REG_ROWINT_SET | REG_ROWINT_SET_INT_ACT_HIGH;
+	if (priv->client->irq > 0)
+		byte |= REG_ROWINT_SET_INT_EN;
+	return i2c_smbus_write_byte(priv->client, byte);
+}
+
+/*
+ * This gets the keys from keypad and reports it to input subsystem
+ */
+static void ht16k33_keypad_scan(struct work_struct *work)
+{
+	struct ht16k33_keypad *keypad =
+		container_of(work, struct ht16k33_keypad, work.work);
+	struct ht16k33_priv *priv =
+		container_of(keypad, struct ht16k33_priv, keypad);
+	const unsigned short *keycodes = keypad->dev->keycode;
+	uint16_t bits_changed, new_state[HT16K33_MATRIX_KEYPAD_MAX_COLS];
+	uint8_t data[HT16K33_MATRIX_KEYPAD_MAX_COLS * 2];
+	int row, col, code;
+	bool reschedule = false;
+
+	if (i2c_smbus_read_i2c_block_data(priv->client, 0x40, 6, data) != 6) {
+		dev_err(&priv->client->dev, "Failed to read key data\n");
+		goto end;
+	}
+
+	for (col = 0; col < keypad->cols; col++) {
+		new_state[col] = (data[col * 2 + 1] << 8) | data[col * 2];
+		if (new_state[col])
+			reschedule = true;
+		bits_changed = keypad->last_key_state[col] ^ new_state[col];
+
+		while (bits_changed) {
+			row = ffs(bits_changed) - 1;
+			code = MATRIX_SCAN_CODE(row, col, keypad->row_shift);
+			input_event(keypad->dev, EV_MSC, MSC_SCAN, code);
+			input_report_key(keypad->dev, keycodes[code],
+					 new_state[col] & BIT(row));
+			bits_changed &= ~BIT(row);
+		}
+	}
+	input_sync(keypad->dev);
+	memcpy(keypad->last_key_state, new_state, sizeof(new_state));
+
+end:
+	if (reschedule)
+		ht16k33_keypad_queue(priv);
+	else
+		enable_irq(priv->client->irq);
+}
+
+static irqreturn_t ht16k33_irq_thread(int irq, void *dev)
+{
+	struct ht16k33_priv *priv = dev;
+
+	disable_irq_nosync(priv->client->irq);
+	ht16k33_keypad_queue(priv);
+
+	return IRQ_HANDLED;
+}
+
+static int ht16k33_bl_update_status(struct backlight_device *bl)
+{
+	int brightness = bl->props.brightness;
+	struct ht16k33_priv *priv = bl_get_data(bl);
+
+	if (bl->props.power != FB_BLANK_UNBLANK ||
+	    bl->props.fb_blank != FB_BLANK_UNBLANK ||
+	    bl->props.state & BL_CORE_FBBLANK || brightness == 0) {
+		return ht16k33_display_off(priv);
+	}
+
+	ht16k33_display_on(priv);
+	return i2c_smbus_write_byte(priv->client,
+				    REG_BRIGHTNESS | (brightness - 1));
+}
+
+static int ht16k33_bl_check_fb(struct backlight_device *bl, struct fb_info *fi)
+{
+	struct ht16k33_priv *priv = bl_get_data(bl);
+
+	return (fi == NULL) || (fi->par == priv);
+}
+
+static const struct backlight_ops ht16k33_bl_ops = {
+	.update_status	= ht16k33_bl_update_status,
+	.check_fb	= ht16k33_bl_check_fb,
+};
+
+static int ht16k33_mmap(struct fb_info *info, struct vm_area_struct *vma)
+{
+	struct ht16k33_priv *priv = info->par;
+
+	return vm_insert_page(vma, vma->vm_start,
+			      virt_to_page(priv->fbdev.buffer));
+}
+
+static struct fb_ops ht16k33_fb_ops = {
+	.owner = THIS_MODULE,
+	.fb_read = fb_sys_read,
+	.fb_write = fb_sys_write,
+	.fb_fillrect = sys_fillrect,
+	.fb_copyarea = sys_copyarea,
+	.fb_imageblit = sys_imageblit,
+	.fb_mmap = ht16k33_mmap,
+};
+
+static int ht16k33_probe(struct i2c_client *client,
+				  const struct i2c_device_id *id)
+{
+	int err;
+	uint32_t rows, cols, dft_brightness;
+	struct backlight_device *bl;
+	struct backlight_properties bl_props;
+	struct ht16k33_priv *priv;
+	struct ht16k33_keypad *keypad;
+	struct ht16k33_fbdev *fbdev;
+	struct device_node *node = client->dev.of_node;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
+		dev_err(&client->dev, "i2c_check_functionality error\n");
+		return -EIO;
+	}
+
+	if (client->irq <= 0) {
+		dev_err(&client->dev, "No IRQ specified\n");
+		return -EINVAL;
+	}
+
+	priv = devm_kzalloc(&client->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->client = client;
+	i2c_set_clientdata(client, priv);
+	fbdev = &priv->fbdev;
+	keypad = &priv->keypad;
+
+	priv->workqueue = create_singlethread_workqueue(DRIVER_NAME "-wq");
+	if (priv->workqueue == NULL)
+		return -ENOMEM;
+
+	err = ht16k33_initialize(priv);
+	if (err)
+		goto err_destroy_wq;
+
+	/* Framebuffer (2 bytes per column) */
+	BUILD_BUG_ON(PAGE_SIZE < HT16K33_FB_SIZE);
+	fbdev->buffer = (unsigned char *) get_zeroed_page(GFP_KERNEL);
+	if (!fbdev->buffer) {
+		err = -ENOMEM;
+		goto err_free_fbdev;
+	}
+
+	fbdev->cache = devm_kmalloc(&client->dev, HT16K33_FB_SIZE, GFP_KERNEL);
+	if (!fbdev->cache) {
+		err = -ENOMEM;
+		goto err_fbdev_buffer;
+	}
+
+	fbdev->info = framebuffer_alloc(0, &client->dev);
+	if (!fbdev->info) {
+		err = -ENOMEM;
+		goto err_fbdev_buffer;
+	}
+
+	err = of_property_read_u32(node, "refresh-rate-hz",
+		&fbdev->refresh_rate);
+	if (err) {
+		dev_err(&client->dev, "refresh rate not specified\n");
+		goto err_fbdev_info;
+	}
+	fb_bl_default_curve(fbdev->info, 0, MIN_BRIGHTNESS, MAX_BRIGHTNESS);
+
+	INIT_DELAYED_WORK(&fbdev->work, ht16k33_fb_update);
+	fbdev->info->fbops = &ht16k33_fb_ops;
+	fbdev->info->screen_base = (char __iomem *) fbdev->buffer;
+	fbdev->info->screen_size = HT16K33_FB_SIZE;
+	fbdev->info->fix = ht16k33_fb_fix;
+	fbdev->info->var = ht16k33_fb_var;
+	fbdev->info->pseudo_palette = NULL;
+	fbdev->info->flags = FBINFO_FLAG_DEFAULT;
+	fbdev->info->par = priv;
+
+	err = register_framebuffer(fbdev->info);
+	if (err)
+		goto err_fbdev_info;
+
+	/* Keypad */
+	keypad->dev = devm_input_allocate_device(&client->dev);
+	if (!keypad->dev) {
+		err = -ENOMEM;
+		goto err_fbdev_unregister;
+	}
+
+	keypad->dev->name = DRIVER_NAME"-keypad";
+	keypad->dev->id.bustype = BUS_I2C;
+	keypad->dev->open = ht16k33_keypad_start;
+	keypad->dev->close = ht16k33_keypad_stop;
+
+	if (!of_get_property(node, "linux,no-autorepeat", NULL))
+		__set_bit(EV_REP, keypad->dev->evbit);
+
+	err = of_property_read_u32(node, "debounce-delay-ms",
+				   &keypad->debounce_ms);
+	if (err) {
+		dev_err(&client->dev, "key debounce delay not specified\n");
+		goto err_fbdev_unregister;
+	}
+
+	err = devm_request_threaded_irq(&client->dev, client->irq, NULL,
+					ht16k33_irq_thread,
+					IRQF_TRIGGER_RISING | IRQF_ONESHOT,
+					DRIVER_NAME, priv);
+	if (err) {
+		dev_err(&client->dev, "irq request failed %d, error %d\n",
+			client->irq, err);
+		goto err_fbdev_unregister;
+	}
+
+	disable_irq_nosync(client->irq);
+	rows = HT16K33_MATRIX_KEYPAD_MAX_ROWS;
+	cols = HT16K33_MATRIX_KEYPAD_MAX_COLS;
+	err = matrix_keypad_parse_of_params(&client->dev, &rows, &cols);
+	if (err)
+		goto err_fbdev_unregister;
+
+	err = matrix_keypad_build_keymap(NULL, NULL, rows, cols, NULL,
+					 keypad->dev);
+	if (err) {
+		dev_err(&client->dev, "failed to build keymap\n");
+		goto err_fbdev_unregister;
+	}
+
+	input_set_drvdata(keypad->dev, priv);
+	keypad->rows = rows;
+	keypad->cols = cols;
+	keypad->row_shift = get_count_order(cols);
+	INIT_DELAYED_WORK(&keypad->work, ht16k33_keypad_scan);
+
+	err = input_register_device(keypad->dev);
+	if (err)
+		goto err_fbdev_unregister;
+
+	/* Backlight */
+	memset(&bl_props, 0, sizeof(struct backlight_properties));
+	bl_props.type = BACKLIGHT_RAW;
+	bl_props.max_brightness = MAX_BRIGHTNESS;
+
+	bl = devm_backlight_device_register(&client->dev, DRIVER_NAME"-bl",
+					    &client->dev, priv,
+					    &ht16k33_bl_ops, &bl_props);
+	if (IS_ERR(bl)) {
+		dev_err(&client->dev, "failed to register backlight\n");
+		err = PTR_ERR(bl);
+		goto err_keypad_unregister;
+	}
+
+	err = of_property_read_u32(node, "default-brightness-level",
+				   &dft_brightness);
+	if (err) {
+		dft_brightness = MAX_BRIGHTNESS;
+	} else if (dft_brightness > MAX_BRIGHTNESS) {
+		dev_warn(&client->dev,
+			 "invalid default brightness level: %u, using %u\n",
+			 dft_brightness, MAX_BRIGHTNESS);
+		dft_brightness = MAX_BRIGHTNESS;
+	}
+
+	bl->props.brightness = dft_brightness;
+	ht16k33_bl_update_status(bl);
+
+	ht16k33_fb_queue(priv);
+	return 0;
+
+err_keypad_unregister:
+	input_unregister_device(keypad->dev);
+err_fbdev_unregister:
+	unregister_framebuffer(fbdev->info);
+err_fbdev_info:
+	framebuffer_release(fbdev->info);
+err_fbdev_buffer:
+	free_page((unsigned long) fbdev->buffer);
+err_free_fbdev:
+	kfree(fbdev);
+err_destroy_wq:
+	destroy_workqueue(priv->workqueue);
+
+	return err;
+}
+
+static int ht16k33_remove(struct i2c_client *client)
+{
+	struct ht16k33_priv *priv = i2c_get_clientdata(client);
+	struct ht16k33_keypad *keypad = &priv->keypad;
+	struct ht16k33_fbdev *fbdev = &priv->fbdev;
+
+	ht16k33_keypad_stop(keypad->dev);
+
+	cancel_delayed_work(&fbdev->work);
+	unregister_framebuffer(fbdev->info);
+	framebuffer_release(fbdev->info);
+	free_page((unsigned long) fbdev->buffer);
+
+	destroy_workqueue(priv->workqueue);
+	return 0;
+}
+
+static const struct i2c_device_id ht16k33_i2c_match[] = {
+	{ "ht16k33", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, ht16k33_i2c_match);
+
+static const struct of_device_id ht16k33_of_match[] = {
+	{ .compatible = "holtek,ht16k33", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ht16k33_of_match);
+
+static struct i2c_driver ht16k33_driver = {
+	.probe		= ht16k33_probe,
+	.remove		= ht16k33_remove,
+	.driver		= {
+		.name		= DRIVER_NAME,
+		.of_match_table	= of_match_ptr(ht16k33_of_match),
+	},
+	.id_table = ht16k33_i2c_match,
+};
+module_i2c_driver(ht16k33_driver);
+
+MODULE_DESCRIPTION("Holtek HT16K33 driver");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Robin van der Gracht <robin@protonic.nl>");
diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
index d02e7c0f5bfd..d718ae4b907a 100644
--- a/drivers/base/Kconfig
+++ b/drivers/base/Kconfig
@@ -224,6 +224,8 @@ config DEBUG_TEST_DRIVER_REMOVE
 	  unusable. You should say N here unless you are explicitly looking to
 	  test this functionality.
 
+source "drivers/base/test/Kconfig"
+
 config SYS_HYPERVISOR
 	bool
 	default n
@@ -237,6 +239,7 @@ config GENERIC_CPU_AUTOPROBE
 
 config SOC_BUS
 	bool
+	select GLOB
 
 source "drivers/base/regmap/Kconfig"
 
@@ -250,11 +253,11 @@ config DMA_SHARED_BUFFER
 	  APIs extension; the file's descriptor can then be passed on to other
 	  driver.
 
-config FENCE_TRACE
-	bool "Enable verbose FENCE_TRACE messages"
+config DMA_FENCE_TRACE
+	bool "Enable verbose DMA_FENCE_TRACE messages"
 	depends on DMA_SHARED_BUFFER
 	help
-	  Enable the FENCE_TRACE printks. This will add extra
+	  Enable the DMA_FENCE_TRACE printks. This will add extra
 	  spam to the console log, but will make it easier to diagnose
 	  lockup related problems for dma-buffers shared across multiple
 	  devices.
diff --git a/drivers/base/Makefile b/drivers/base/Makefile
index 2609ba20b396..f2816f6ff76a 100644
--- a/drivers/base/Makefile
+++ b/drivers/base/Makefile
@@ -24,5 +24,7 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o
 obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o
 obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o
 
+obj-y			+= test/
+
 ccflags-$(CONFIG_DEBUG_DRIVER) := -DDEBUG
 
diff --git a/drivers/base/base.h b/drivers/base/base.h
index e05db388bd1c..ada9dce34e6d 100644
--- a/drivers/base/base.h
+++ b/drivers/base/base.h
@@ -107,6 +107,9 @@ extern void bus_remove_device(struct device *dev);
 
 extern int bus_add_driver(struct device_driver *drv);
 extern void bus_remove_driver(struct device_driver *drv);
+extern void device_release_driver_internal(struct device *dev,
+					   struct device_driver *drv,
+					   struct device *parent);
 
 extern void driver_detach(struct device_driver *drv);
 extern int driver_probe_device(struct device_driver *drv, struct device *dev);
@@ -138,6 +141,8 @@ extern void device_unblock_probing(void);
 extern struct kset *devices_kset;
 extern void devices_kset_move_last(struct device *dev);
 
+extern struct device_attribute dev_attr_deferred_probe;
+
 #if defined(CONFIG_MODULES) && defined(CONFIG_SYSFS)
 extern void module_add_driver(struct module *mod, struct device_driver *drv);
 extern void module_remove_driver(struct device_driver *drv);
@@ -152,3 +157,13 @@ extern int devtmpfs_init(void);
 #else
 static inline int devtmpfs_init(void) { return 0; }
 #endif
+
+/* Device links support */
+extern int device_links_read_lock(void);
+extern void device_links_read_unlock(int idx);
+extern int device_links_check_suppliers(struct device *dev);
+extern void device_links_driver_bound(struct device *dev);
+extern void device_links_driver_cleanup(struct device *dev);
+extern void device_links_no_driver(struct device *dev);
+extern bool device_links_busy(struct device *dev);
+extern void device_links_unbind_consumers(struct device *dev);
diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c
index 47983a2ba621..1e3903d0d994 100644
--- a/drivers/base/cacheinfo.c
+++ b/drivers/base/cacheinfo.c
@@ -16,6 +16,9 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/acpi.h>
 #include <linux/bitops.h>
 #include <linux/cacheinfo.h>
 #include <linux/compiler.h>
@@ -85,7 +88,120 @@ static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf,
 {
 	return sib_leaf->of_node == this_leaf->of_node;
 }
+
+/* OF properties to query for a given cache type */
+struct cache_type_info {
+	const char *size_prop;
+	const char *line_size_props[2];
+	const char *nr_sets_prop;
+};
+
+static const struct cache_type_info cache_type_info[] = {
+	{
+		.size_prop       = "cache-size",
+		.line_size_props = { "cache-line-size",
+				     "cache-block-size", },
+		.nr_sets_prop    = "cache-sets",
+	}, {
+		.size_prop       = "i-cache-size",
+		.line_size_props = { "i-cache-line-size",
+				     "i-cache-block-size", },
+		.nr_sets_prop    = "i-cache-sets",
+	}, {
+		.size_prop       = "d-cache-size",
+		.line_size_props = { "d-cache-line-size",
+				     "d-cache-block-size", },
+		.nr_sets_prop    = "d-cache-sets",
+	},
+};
+
+static inline int get_cacheinfo_idx(enum cache_type type)
+{
+	if (type == CACHE_TYPE_UNIFIED)
+		return 0;
+	return type;
+}
+
+static void cache_size(struct cacheinfo *this_leaf)
+{
+	const char *propname;
+	const __be32 *cache_size;
+	int ct_idx;
+
+	ct_idx = get_cacheinfo_idx(this_leaf->type);
+	propname = cache_type_info[ct_idx].size_prop;
+
+	cache_size = of_get_property(this_leaf->of_node, propname, NULL);
+	if (cache_size)
+		this_leaf->size = of_read_number(cache_size, 1);
+}
+
+/* not cache_line_size() because that's a macro in include/linux/cache.h */
+static void cache_get_line_size(struct cacheinfo *this_leaf)
+{
+	const __be32 *line_size;
+	int i, lim, ct_idx;
+
+	ct_idx = get_cacheinfo_idx(this_leaf->type);
+	lim = ARRAY_SIZE(cache_type_info[ct_idx].line_size_props);
+
+	for (i = 0; i < lim; i++) {
+		const char *propname;
+
+		propname = cache_type_info[ct_idx].line_size_props[i];
+		line_size = of_get_property(this_leaf->of_node, propname, NULL);
+		if (line_size)
+			break;
+	}
+
+	if (line_size)
+		this_leaf->coherency_line_size = of_read_number(line_size, 1);
+}
+
+static void cache_nr_sets(struct cacheinfo *this_leaf)
+{
+	const char *propname;
+	const __be32 *nr_sets;
+	int ct_idx;
+
+	ct_idx = get_cacheinfo_idx(this_leaf->type);
+	propname = cache_type_info[ct_idx].nr_sets_prop;
+
+	nr_sets = of_get_property(this_leaf->of_node, propname, NULL);
+	if (nr_sets)
+		this_leaf->number_of_sets = of_read_number(nr_sets, 1);
+}
+
+static void cache_associativity(struct cacheinfo *this_leaf)
+{
+	unsigned int line_size = this_leaf->coherency_line_size;
+	unsigned int nr_sets = this_leaf->number_of_sets;
+	unsigned int size = this_leaf->size;
+
+	/*
+	 * If the cache is fully associative, there is no need to
+	 * check the other properties.
+	 */
+	if (!(nr_sets == 1) && (nr_sets > 0 && size > 0 && line_size > 0))
+		this_leaf->ways_of_associativity = (size / nr_sets) / line_size;
+}
+
+static void cache_of_override_properties(unsigned int cpu)
+{
+	int index;
+	struct cacheinfo *this_leaf;
+	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
+
+	for (index = 0; index < cache_leaves(cpu); index++) {
+		this_leaf = this_cpu_ci->info_list + index;
+		cache_size(this_leaf);
+		cache_get_line_size(this_leaf);
+		cache_nr_sets(this_leaf);
+		cache_associativity(this_leaf);
+	}
+}
 #else
+static void cache_of_override_properties(unsigned int cpu) { }
 static inline int cache_setup_of_node(unsigned int cpu) { return 0; }
 static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf,
 					   struct cacheinfo *sib_leaf)
@@ -104,9 +220,16 @@ static int cache_shared_cpu_map_setup(unsigned int cpu)
 	struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
 	struct cacheinfo *this_leaf, *sib_leaf;
 	unsigned int index;
-	int ret;
+	int ret = 0;
 
-	ret = cache_setup_of_node(cpu);
+	if (this_cpu_ci->cpu_map_populated)
+		return 0;
+
+	if (of_have_populated_dt())
+		ret = cache_setup_of_node(cpu);
+	else if (!acpi_disabled)
+		/* No cache property/hierarchy support yet in ACPI */
+		ret = -ENOTSUPP;
 	if (ret)
 		return ret;
 
@@ -161,6 +284,12 @@ static void cache_shared_cpu_map_remove(unsigned int cpu)
 	}
 }
 
+static void cache_override_properties(unsigned int cpu)
+{
+	if (of_have_populated_dt())
+		return cache_of_override_properties(cpu);
+}
+
 static void free_cache_attributes(unsigned int cpu)
 {
 	if (!per_cpu_cacheinfo(cpu))
@@ -203,10 +332,11 @@ static int detect_cache_attributes(unsigned int cpu)
 	 */
 	ret = cache_shared_cpu_map_setup(cpu);
 	if (ret) {
-		pr_warn("Unable to detect cache hierarchy from DT for CPU %d\n",
-			cpu);
+		pr_warn("Unable to detect cache hierarchy for CPU %d\n", cpu);
 		goto free_ci;
 	}
+
+	cache_override_properties(cpu);
 	return 0;
 
 free_ci:
diff --git a/drivers/base/class.c b/drivers/base/class.c
index 71059e32bebc..a2b2896693d6 100644
--- a/drivers/base/class.c
+++ b/drivers/base/class.c
@@ -163,6 +163,18 @@ static void klist_class_dev_put(struct klist_node *n)
 	put_device(dev);
 }
 
+static int class_add_groups(struct class *cls,
+			    const struct attribute_group **groups)
+{
+	return sysfs_create_groups(&cls->p->subsys.kobj, groups);
+}
+
+static void class_remove_groups(struct class *cls,
+				const struct attribute_group **groups)
+{
+	return sysfs_remove_groups(&cls->p->subsys.kobj, groups);
+}
+
 int __class_register(struct class *cls, struct lock_class_key *key)
 {
 	struct subsys_private *cp;
@@ -203,6 +215,8 @@ int __class_register(struct class *cls, struct lock_class_key *key)
 		kfree(cp);
 		return error;
 	}
+	error = class_add_groups(class_get(cls), cls->class_groups);
+	class_put(cls);
 	error = add_class_attrs(class_get(cls));
 	class_put(cls);
 	return error;
@@ -213,6 +227,7 @@ void class_unregister(struct class *cls)
 {
 	pr_debug("device class '%s': unregistering\n", cls->name);
 	remove_class_attrs(cls);
+	class_remove_groups(cls, cls->class_groups);
 	kset_unregister(&cls->p->subsys);
 }
 
diff --git a/drivers/base/core.c b/drivers/base/core.c
index ce057a568673..020ea7f05520 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -44,6 +44,572 @@ static int __init sysfs_deprecated_setup(char *arg)
 early_param("sysfs.deprecated", sysfs_deprecated_setup);
 #endif
 
+/* Device links support. */
+
+#ifdef CONFIG_SRCU
+static DEFINE_MUTEX(device_links_lock);
+DEFINE_STATIC_SRCU(device_links_srcu);
+
+static inline void device_links_write_lock(void)
+{
+	mutex_lock(&device_links_lock);
+}
+
+static inline void device_links_write_unlock(void)
+{
+	mutex_unlock(&device_links_lock);
+}
+
+int device_links_read_lock(void)
+{
+	return srcu_read_lock(&device_links_srcu);
+}
+
+void device_links_read_unlock(int idx)
+{
+	srcu_read_unlock(&device_links_srcu, idx);
+}
+#else /* !CONFIG_SRCU */
+static DECLARE_RWSEM(device_links_lock);
+
+static inline void device_links_write_lock(void)
+{
+	down_write(&device_links_lock);
+}
+
+static inline void device_links_write_unlock(void)
+{
+	up_write(&device_links_lock);
+}
+
+int device_links_read_lock(void)
+{
+	down_read(&device_links_lock);
+	return 0;
+}
+
+void device_links_read_unlock(int not_used)
+{
+	up_read(&device_links_lock);
+}
+#endif /* !CONFIG_SRCU */
+
+/**
+ * device_is_dependent - Check if one device depends on another one
+ * @dev: Device to check dependencies for.
+ * @target: Device to check against.
+ *
+ * Check if @target depends on @dev or any device dependent on it (its child or
+ * its consumer etc).  Return 1 if that is the case or 0 otherwise.
+ */
+static int device_is_dependent(struct device *dev, void *target)
+{
+	struct device_link *link;
+	int ret;
+
+	if (WARN_ON(dev == target))
+		return 1;
+
+	ret = device_for_each_child(dev, target, device_is_dependent);
+	if (ret)
+		return ret;
+
+	list_for_each_entry(link, &dev->links.consumers, s_node) {
+		if (WARN_ON(link->consumer == target))
+			return 1;
+
+		ret = device_is_dependent(link->consumer, target);
+		if (ret)
+			break;
+	}
+	return ret;
+}
+
+static int device_reorder_to_tail(struct device *dev, void *not_used)
+{
+	struct device_link *link;
+
+	/*
+	 * Devices that have not been registered yet will be put to the ends
+	 * of the lists during the registration, so skip them here.
+	 */
+	if (device_is_registered(dev))
+		devices_kset_move_last(dev);
+
+	if (device_pm_initialized(dev))
+		device_pm_move_last(dev);
+
+	device_for_each_child(dev, NULL, device_reorder_to_tail);
+	list_for_each_entry(link, &dev->links.consumers, s_node)
+		device_reorder_to_tail(link->consumer, NULL);
+
+	return 0;
+}
+
+/**
+ * device_link_add - Create a link between two devices.
+ * @consumer: Consumer end of the link.
+ * @supplier: Supplier end of the link.
+ * @flags: Link flags.
+ *
+ * The caller is responsible for the proper synchronization of the link creation
+ * with runtime PM.  First, setting the DL_FLAG_PM_RUNTIME flag will cause the
+ * runtime PM framework to take the link into account.  Second, if the
+ * DL_FLAG_RPM_ACTIVE flag is set in addition to it, the supplier devices will
+ * be forced into the active metastate and reference-counted upon the creation
+ * of the link.  If DL_FLAG_PM_RUNTIME is not set, DL_FLAG_RPM_ACTIVE will be
+ * ignored.
+ *
+ * If the DL_FLAG_AUTOREMOVE is set, the link will be removed automatically
+ * when the consumer device driver unbinds from it.  The combination of both
+ * DL_FLAG_AUTOREMOVE and DL_FLAG_STATELESS set is invalid and will cause NULL
+ * to be returned.
+ *
+ * A side effect of the link creation is re-ordering of dpm_list and the
+ * devices_kset list by moving the consumer device and all devices depending
+ * on it to the ends of these lists (that does not happen to devices that have
+ * not been registered when this function is called).
+ *
+ * The supplier device is required to be registered when this function is called
+ * and NULL will be returned if that is not the case.  The consumer device need
+ * not be registered, however.
+ */
+struct device_link *device_link_add(struct device *consumer,
+				    struct device *supplier, u32 flags)
+{
+	struct device_link *link;
+
+	if (!consumer || !supplier ||
+	    ((flags & DL_FLAG_STATELESS) && (flags & DL_FLAG_AUTOREMOVE)))
+		return NULL;
+
+	device_links_write_lock();
+	device_pm_lock();
+
+	/*
+	 * If the supplier has not been fully registered yet or there is a
+	 * reverse dependency between the consumer and the supplier already in
+	 * the graph, return NULL.
+	 */
+	if (!device_pm_initialized(supplier)
+	    || device_is_dependent(consumer, supplier)) {
+		link = NULL;
+		goto out;
+	}
+
+	list_for_each_entry(link, &supplier->links.consumers, s_node)
+		if (link->consumer == consumer)
+			goto out;
+
+	link = kzalloc(sizeof(*link), GFP_KERNEL);
+	if (!link)
+		goto out;
+
+	if (flags & DL_FLAG_PM_RUNTIME) {
+		if (flags & DL_FLAG_RPM_ACTIVE) {
+			if (pm_runtime_get_sync(supplier) < 0) {
+				pm_runtime_put_noidle(supplier);
+				kfree(link);
+				link = NULL;
+				goto out;
+			}
+			link->rpm_active = true;
+		}
+		pm_runtime_new_link(consumer);
+	}
+	get_device(supplier);
+	link->supplier = supplier;
+	INIT_LIST_HEAD(&link->s_node);
+	get_device(consumer);
+	link->consumer = consumer;
+	INIT_LIST_HEAD(&link->c_node);
+	link->flags = flags;
+
+	/* Determine the initial link state. */
+	if (flags & DL_FLAG_STATELESS) {
+		link->status = DL_STATE_NONE;
+	} else {
+		switch (supplier->links.status) {
+		case DL_DEV_DRIVER_BOUND:
+			switch (consumer->links.status) {
+			case DL_DEV_PROBING:
+				/*
+				 * Balance the decrementation of the supplier's
+				 * runtime PM usage counter after consumer probe
+				 * in driver_probe_device().
+				 */
+				if (flags & DL_FLAG_PM_RUNTIME)
+					pm_runtime_get_sync(supplier);
+
+				link->status = DL_STATE_CONSUMER_PROBE;
+				break;
+			case DL_DEV_DRIVER_BOUND:
+				link->status = DL_STATE_ACTIVE;
+				break;
+			default:
+				link->status = DL_STATE_AVAILABLE;
+				break;
+			}
+			break;
+		case DL_DEV_UNBINDING:
+			link->status = DL_STATE_SUPPLIER_UNBIND;
+			break;
+		default:
+			link->status = DL_STATE_DORMANT;
+			break;
+		}
+	}
+
+	/*
+	 * Move the consumer and all of the devices depending on it to the end
+	 * of dpm_list and the devices_kset list.
+	 *
+	 * It is necessary to hold dpm_list locked throughout all that or else
+	 * we may end up suspending with a wrong ordering of it.
+	 */
+	device_reorder_to_tail(consumer, NULL);
+
+	list_add_tail_rcu(&link->s_node, &supplier->links.consumers);
+	list_add_tail_rcu(&link->c_node, &consumer->links.suppliers);
+
+	dev_info(consumer, "Linked as a consumer to %s\n", dev_name(supplier));
+
+ out:
+	device_pm_unlock();
+	device_links_write_unlock();
+	return link;
+}
+EXPORT_SYMBOL_GPL(device_link_add);
+
+static void device_link_free(struct device_link *link)
+{
+	put_device(link->consumer);
+	put_device(link->supplier);
+	kfree(link);
+}
+
+#ifdef CONFIG_SRCU
+static void __device_link_free_srcu(struct rcu_head *rhead)
+{
+	device_link_free(container_of(rhead, struct device_link, rcu_head));
+}
+
+static void __device_link_del(struct device_link *link)
+{
+	dev_info(link->consumer, "Dropping the link to %s\n",
+		 dev_name(link->supplier));
+
+	if (link->flags & DL_FLAG_PM_RUNTIME)
+		pm_runtime_drop_link(link->consumer);
+
+	list_del_rcu(&link->s_node);
+	list_del_rcu(&link->c_node);
+	call_srcu(&device_links_srcu, &link->rcu_head, __device_link_free_srcu);
+}
+#else /* !CONFIG_SRCU */
+static void __device_link_del(struct device_link *link)
+{
+	dev_info(link->consumer, "Dropping the link to %s\n",
+		 dev_name(link->supplier));
+
+	list_del(&link->s_node);
+	list_del(&link->c_node);
+	device_link_free(link);
+}
+#endif /* !CONFIG_SRCU */
+
+/**
+ * device_link_del - Delete a link between two devices.
+ * @link: Device link to delete.
+ *
+ * The caller must ensure proper synchronization of this function with runtime
+ * PM.
+ */
+void device_link_del(struct device_link *link)
+{
+	device_links_write_lock();
+	device_pm_lock();
+	__device_link_del(link);
+	device_pm_unlock();
+	device_links_write_unlock();
+}
+EXPORT_SYMBOL_GPL(device_link_del);
+
+static void device_links_missing_supplier(struct device *dev)
+{
+	struct device_link *link;
+
+	list_for_each_entry(link, &dev->links.suppliers, c_node)
+		if (link->status == DL_STATE_CONSUMER_PROBE)
+			WRITE_ONCE(link->status, DL_STATE_AVAILABLE);
+}
+
+/**
+ * device_links_check_suppliers - Check presence of supplier drivers.
+ * @dev: Consumer device.
+ *
+ * Check links from this device to any suppliers.  Walk the list of the device's
+ * links to suppliers and see if all of them are available.  If not, simply
+ * return -EPROBE_DEFER.
+ *
+ * We need to guarantee that the supplier will not go away after the check has
+ * been positive here.  It only can go away in __device_release_driver() and
+ * that function  checks the device's links to consumers.  This means we need to
+ * mark the link as "consumer probe in progress" to make the supplier removal
+ * wait for us to complete (or bad things may happen).
+ *
+ * Links with the DL_FLAG_STATELESS flag set are ignored.
+ */
+int device_links_check_suppliers(struct device *dev)
+{
+	struct device_link *link;
+	int ret = 0;
+
+	device_links_write_lock();
+
+	list_for_each_entry(link, &dev->links.suppliers, c_node) {
+		if (link->flags & DL_FLAG_STATELESS)
+			continue;
+
+		if (link->status != DL_STATE_AVAILABLE) {
+			device_links_missing_supplier(dev);
+			ret = -EPROBE_DEFER;
+			break;
+		}
+		WRITE_ONCE(link->status, DL_STATE_CONSUMER_PROBE);
+	}
+	dev->links.status = DL_DEV_PROBING;
+
+	device_links_write_unlock();
+	return ret;
+}
+
+/**
+ * device_links_driver_bound - Update device links after probing its driver.
+ * @dev: Device to update the links for.
+ *
+ * The probe has been successful, so update links from this device to any
+ * consumers by changing their status to "available".
+ *
+ * Also change the status of @dev's links to suppliers to "active".
+ *
+ * Links with the DL_FLAG_STATELESS flag set are ignored.
+ */
+void device_links_driver_bound(struct device *dev)
+{
+	struct device_link *link;
+
+	device_links_write_lock();
+
+	list_for_each_entry(link, &dev->links.consumers, s_node) {
+		if (link->flags & DL_FLAG_STATELESS)
+			continue;
+
+		WARN_ON(link->status != DL_STATE_DORMANT);
+		WRITE_ONCE(link->status, DL_STATE_AVAILABLE);
+	}
+
+	list_for_each_entry(link, &dev->links.suppliers, c_node) {
+		if (link->flags & DL_FLAG_STATELESS)
+			continue;
+
+		WARN_ON(link->status != DL_STATE_CONSUMER_PROBE);
+		WRITE_ONCE(link->status, DL_STATE_ACTIVE);
+	}
+
+	dev->links.status = DL_DEV_DRIVER_BOUND;
+
+	device_links_write_unlock();
+}
+
+/**
+ * __device_links_no_driver - Update links of a device without a driver.
+ * @dev: Device without a drvier.
+ *
+ * Delete all non-persistent links from this device to any suppliers.
+ *
+ * Persistent links stay around, but their status is changed to "available",
+ * unless they already are in the "supplier unbind in progress" state in which
+ * case they need not be updated.
+ *
+ * Links with the DL_FLAG_STATELESS flag set are ignored.
+ */
+static void __device_links_no_driver(struct device *dev)
+{
+	struct device_link *link, *ln;
+
+	list_for_each_entry_safe_reverse(link, ln, &dev->links.suppliers, c_node) {
+		if (link->flags & DL_FLAG_STATELESS)
+			continue;
+
+		if (link->flags & DL_FLAG_AUTOREMOVE)
+			__device_link_del(link);
+		else if (link->status != DL_STATE_SUPPLIER_UNBIND)
+			WRITE_ONCE(link->status, DL_STATE_AVAILABLE);
+	}
+
+	dev->links.status = DL_DEV_NO_DRIVER;
+}
+
+void device_links_no_driver(struct device *dev)
+{
+	device_links_write_lock();
+	__device_links_no_driver(dev);
+	device_links_write_unlock();
+}
+
+/**
+ * device_links_driver_cleanup - Update links after driver removal.
+ * @dev: Device whose driver has just gone away.
+ *
+ * Update links to consumers for @dev by changing their status to "dormant" and
+ * invoke %__device_links_no_driver() to update links to suppliers for it as
+ * appropriate.
+ *
+ * Links with the DL_FLAG_STATELESS flag set are ignored.
+ */
+void device_links_driver_cleanup(struct device *dev)
+{
+	struct device_link *link;
+
+	device_links_write_lock();
+
+	list_for_each_entry(link, &dev->links.consumers, s_node) {
+		if (link->flags & DL_FLAG_STATELESS)
+			continue;
+
+		WARN_ON(link->flags & DL_FLAG_AUTOREMOVE);
+		WARN_ON(link->status != DL_STATE_SUPPLIER_UNBIND);
+		WRITE_ONCE(link->status, DL_STATE_DORMANT);
+	}
+
+	__device_links_no_driver(dev);
+
+	device_links_write_unlock();
+}
+
+/**
+ * device_links_busy - Check if there are any busy links to consumers.
+ * @dev: Device to check.
+ *
+ * Check each consumer of the device and return 'true' if its link's status
+ * is one of "consumer probe" or "active" (meaning that the given consumer is
+ * probing right now or its driver is present).  Otherwise, change the link
+ * state to "supplier unbind" to prevent the consumer from being probed
+ * successfully going forward.
+ *
+ * Return 'false' if there are no probing or active consumers.
+ *
+ * Links with the DL_FLAG_STATELESS flag set are ignored.
+ */
+bool device_links_busy(struct device *dev)
+{
+	struct device_link *link;
+	bool ret = false;
+
+	device_links_write_lock();
+
+	list_for_each_entry(link, &dev->links.consumers, s_node) {
+		if (link->flags & DL_FLAG_STATELESS)
+			continue;
+
+		if (link->status == DL_STATE_CONSUMER_PROBE
+		    || link->status == DL_STATE_ACTIVE) {
+			ret = true;
+			break;
+		}
+		WRITE_ONCE(link->status, DL_STATE_SUPPLIER_UNBIND);
+	}
+
+	dev->links.status = DL_DEV_UNBINDING;
+
+	device_links_write_unlock();
+	return ret;
+}
+
+/**
+ * device_links_unbind_consumers - Force unbind consumers of the given device.
+ * @dev: Device to unbind the consumers of.
+ *
+ * Walk the list of links to consumers for @dev and if any of them is in the
+ * "consumer probe" state, wait for all device probes in progress to complete
+ * and start over.
+ *
+ * If that's not the case, change the status of the link to "supplier unbind"
+ * and check if the link was in the "active" state.  If so, force the consumer
+ * driver to unbind and start over (the consumer will not re-probe as we have
+ * changed the state of the link already).
+ *
+ * Links with the DL_FLAG_STATELESS flag set are ignored.
+ */
+void device_links_unbind_consumers(struct device *dev)
+{
+	struct device_link *link;
+
+ start:
+	device_links_write_lock();
+
+	list_for_each_entry(link, &dev->links.consumers, s_node) {
+		enum device_link_state status;
+
+		if (link->flags & DL_FLAG_STATELESS)
+			continue;
+
+		status = link->status;
+		if (status == DL_STATE_CONSUMER_PROBE) {
+			device_links_write_unlock();
+
+			wait_for_device_probe();
+			goto start;
+		}
+		WRITE_ONCE(link->status, DL_STATE_SUPPLIER_UNBIND);
+		if (status == DL_STATE_ACTIVE) {
+			struct device *consumer = link->consumer;
+
+			get_device(consumer);
+
+			device_links_write_unlock();
+
+			device_release_driver_internal(consumer, NULL,
+						       consumer->parent);
+			put_device(consumer);
+			goto start;
+		}
+	}
+
+	device_links_write_unlock();
+}
+
+/**
+ * device_links_purge - Delete existing links to other devices.
+ * @dev: Target device.
+ */
+static void device_links_purge(struct device *dev)
+{
+	struct device_link *link, *ln;
+
+	/*
+	 * Delete all of the remaining links from this device to any other
+	 * devices (either consumers or suppliers).
+	 */
+	device_links_write_lock();
+
+	list_for_each_entry_safe_reverse(link, ln, &dev->links.suppliers, c_node) {
+		WARN_ON(link->status == DL_STATE_ACTIVE);
+		__device_link_del(link);
+	}
+
+	list_for_each_entry_safe_reverse(link, ln, &dev->links.consumers, s_node) {
+		WARN_ON(link->status != DL_STATE_DORMANT &&
+			link->status != DL_STATE_NONE);
+		__device_link_del(link);
+	}
+
+	device_links_write_unlock();
+}
+
+/* Device links support end. */
+
 int (*platform_notify)(struct device *dev) = NULL;
 int (*platform_notify_remove)(struct device *dev) = NULL;
 static struct kobject *dev_kobj;
@@ -494,8 +1060,14 @@ static int device_add_attrs(struct device *dev)
 			goto err_remove_dev_groups;
 	}
 
+	error = device_create_file(dev, &dev_attr_deferred_probe);
+	if (error)
+		goto err_remove_online;
+
 	return 0;
 
+ err_remove_online:
+	device_remove_file(dev, &dev_attr_online);
  err_remove_dev_groups:
 	device_remove_groups(dev, dev->groups);
  err_remove_type_groups:
@@ -513,6 +1085,7 @@ static void device_remove_attrs(struct device *dev)
 	struct class *class = dev->class;
 	const struct device_type *type = dev->type;
 
+	device_remove_file(dev, &dev_attr_deferred_probe);
 	device_remove_file(dev, &dev_attr_online);
 	device_remove_groups(dev, dev->groups);
 
@@ -711,6 +1284,9 @@ void device_initialize(struct device *dev)
 #ifdef CONFIG_GENERIC_MSI_IRQ
 	INIT_LIST_HEAD(&dev->msi_list);
 #endif
+	INIT_LIST_HEAD(&dev->links.consumers);
+	INIT_LIST_HEAD(&dev->links.suppliers);
+	dev->links.status = DL_DEV_NO_DRIVER;
 }
 EXPORT_SYMBOL_GPL(device_initialize);
 
@@ -1258,6 +1834,8 @@ void device_del(struct device *dev)
 	if (dev->bus)
 		blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
 					     BUS_NOTIFY_DEL_DEVICE, dev);
+
+	device_links_purge(dev);
 	dpm_sysfs_remove(dev);
 	if (parent)
 		klist_del(&dev->p->knode_parent);
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index d76cd97a98b6..a8b258e5407b 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -53,6 +53,19 @@ static LIST_HEAD(deferred_probe_pending_list);
 static LIST_HEAD(deferred_probe_active_list);
 static atomic_t deferred_trigger_count = ATOMIC_INIT(0);
 
+static ssize_t deferred_probe_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	bool value;
+
+	mutex_lock(&deferred_probe_mutex);
+	value = !list_empty(&dev->p->deferred_probe);
+	mutex_unlock(&deferred_probe_mutex);
+
+	return sprintf(buf, "%d\n", value);
+}
+DEVICE_ATTR_RO(deferred_probe);
+
 /*
  * In some cases, like suspend to RAM or hibernation, It might be reasonable
  * to prohibit probing of devices as it could be unsafe.
@@ -244,6 +257,7 @@ static void driver_bound(struct device *dev)
 		 __func__, dev_name(dev));
 
 	klist_add_tail(&dev->p->knode_driver, &dev->driver->p->klist_devices);
+	device_links_driver_bound(dev);
 
 	device_pm_check_callbacks(dev);
 
@@ -338,6 +352,10 @@ static int really_probe(struct device *dev, struct device_driver *drv)
 		return ret;
 	}
 
+	ret = device_links_check_suppliers(dev);
+	if (ret)
+		return ret;
+
 	atomic_inc(&probe_count);
 	pr_debug("bus: '%s': %s: probing driver %s with device %s\n",
 		 drv->bus->name, __func__, drv->name, dev_name(dev));
@@ -416,6 +434,7 @@ probe_failed:
 		blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
 					     BUS_NOTIFY_DRIVER_NOT_BOUND, dev);
 pinctrl_bind_failed:
+	device_links_no_driver(dev);
 	devres_release_all(dev);
 	driver_sysfs_remove(dev);
 	dev->driver = NULL;
@@ -508,6 +527,7 @@ int driver_probe_device(struct device_driver *drv, struct device *dev)
 	pr_debug("bus: '%s': %s: matched device %s with driver %s\n",
 		 drv->bus->name, __func__, dev_name(dev), drv->name);
 
+	pm_runtime_get_suppliers(dev);
 	if (dev->parent)
 		pm_runtime_get_sync(dev->parent);
 
@@ -518,6 +538,7 @@ int driver_probe_device(struct device_driver *drv, struct device *dev)
 	if (dev->parent)
 		pm_runtime_put(dev->parent);
 
+	pm_runtime_put_suppliers(dev);
 	return ret;
 }
 
@@ -772,7 +793,7 @@ EXPORT_SYMBOL_GPL(driver_attach);
  * __device_release_driver() must be called with @dev lock held.
  * When called for a USB interface, @dev->parent lock must be held as well.
  */
-static void __device_release_driver(struct device *dev)
+static void __device_release_driver(struct device *dev, struct device *parent)
 {
 	struct device_driver *drv;
 
@@ -781,7 +802,27 @@ static void __device_release_driver(struct device *dev)
 		if (driver_allows_async_probing(drv))
 			async_synchronize_full();
 
+		while (device_links_busy(dev)) {
+			device_unlock(dev);
+			if (parent)
+				device_unlock(parent);
+
+			device_links_unbind_consumers(dev);
+			if (parent)
+				device_lock(parent);
+
+			device_lock(dev);
+			/*
+			 * A concurrent invocation of the same function might
+			 * have released the driver successfully while this one
+			 * was waiting, so check for that.
+			 */
+			if (dev->driver != drv)
+				return;
+		}
+
 		pm_runtime_get_sync(dev);
+		pm_runtime_clean_up_links(dev);
 
 		driver_sysfs_remove(dev);
 
@@ -796,6 +837,8 @@ static void __device_release_driver(struct device *dev)
 			dev->bus->remove(dev);
 		else if (drv->remove)
 			drv->remove(dev);
+
+		device_links_driver_cleanup(dev);
 		devres_release_all(dev);
 		dev->driver = NULL;
 		dev_set_drvdata(dev, NULL);
@@ -812,12 +855,32 @@ static void __device_release_driver(struct device *dev)
 	}
 }
 
+void device_release_driver_internal(struct device *dev,
+				    struct device_driver *drv,
+				    struct device *parent)
+{
+	if (parent)
+		device_lock(parent);
+
+	device_lock(dev);
+	if (!drv || drv == dev->driver)
+		__device_release_driver(dev, parent);
+
+	device_unlock(dev);
+	if (parent)
+		device_unlock(parent);
+}
+
 /**
  * device_release_driver - manually detach device from driver.
  * @dev: device.
  *
  * Manually detach device from driver.
  * When called for a USB interface, @dev->parent lock must be held.
+ *
+ * If this function is to be called with @dev->parent lock held, ensure that
+ * the device's consumers are unbound in advance or that their locks can be
+ * acquired under the @dev->parent lock.
  */
 void device_release_driver(struct device *dev)
 {
@@ -826,9 +889,7 @@ void device_release_driver(struct device *dev)
 	 * within their ->remove callback for the same device, they
 	 * will deadlock right here.
 	 */
-	device_lock(dev);
-	__device_release_driver(dev);
-	device_unlock(dev);
+	device_release_driver_internal(dev, NULL, NULL);
 }
 EXPORT_SYMBOL_GPL(device_release_driver);
 
@@ -853,15 +914,7 @@ void driver_detach(struct device_driver *drv)
 		dev = dev_prv->device;
 		get_device(dev);
 		spin_unlock(&drv->p->klist_devices.k_lock);
-
-		if (dev->parent)	/* Needed for USB */
-			device_lock(dev->parent);
-		device_lock(dev);
-		if (dev->driver == drv)
-			__device_release_driver(dev);
-		device_unlock(dev);
-		if (dev->parent)
-			device_unlock(dev->parent);
+		device_release_driver_internal(dev, drv, dev->parent);
 		put_device(dev);
 	}
 }
diff --git a/drivers/base/devcoredump.c b/drivers/base/devcoredump.c
index 240374fd1838..7be310f7db73 100644
--- a/drivers/base/devcoredump.c
+++ b/drivers/base/devcoredump.c
@@ -160,18 +160,20 @@ static ssize_t disabled_store(struct class *class, struct class_attribute *attr,
 
 	return count;
 }
+static CLASS_ATTR_RW(disabled);
 
-static struct class_attribute devcd_class_attrs[] = {
-	__ATTR_RW(disabled),
-	__ATTR_NULL
+static struct attribute *devcd_class_attrs[] = {
+	&class_attr_disabled.attr,
+	NULL,
 };
+ATTRIBUTE_GROUPS(devcd_class);
 
 static struct class devcd_class = {
 	.name		= "devcoredump",
 	.owner		= THIS_MODULE,
 	.dev_release	= devcd_dev_release,
 	.dev_groups	= devcd_dev_groups,
-	.class_attrs	= devcd_class_attrs,
+	.class_groups	= devcd_class_groups,
 };
 
 static ssize_t devcd_readv(char *buffer, loff_t offset, size_t count,
diff --git a/drivers/base/dma-mapping.c b/drivers/base/dma-mapping.c
index 8f8b68c80986..efd71cf4fdea 100644
--- a/drivers/base/dma-mapping.c
+++ b/drivers/base/dma-mapping.c
@@ -108,13 +108,13 @@ void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
 EXPORT_SYMBOL(dmam_free_coherent);
 
 /**
- * dmam_alloc_non_coherent - Managed dma_alloc_non_coherent()
+ * dmam_alloc_non_coherent - Managed dma_alloc_noncoherent()
  * @dev: Device to allocate non_coherent memory for
  * @size: Size of allocation
  * @dma_handle: Out argument for allocated DMA handle
  * @gfp: Allocation flags
  *
- * Managed dma_alloc_non_coherent().  Memory allocated using this
+ * Managed dma_alloc_noncoherent().  Memory allocated using this
  * function will be automatically released on driver detach.
  *
  * RETURNS:
diff --git a/drivers/base/firmware_class.c b/drivers/base/firmware_class.c
index 22d1760a4278..4497d263209f 100644
--- a/drivers/base/firmware_class.c
+++ b/drivers/base/firmware_class.c
@@ -30,6 +30,7 @@
 #include <linux/syscore_ops.h>
 #include <linux/reboot.h>
 #include <linux/security.h>
+#include <linux/swait.h>
 
 #include <generated/utsrelease.h>
 
@@ -91,10 +92,11 @@ static inline bool fw_is_builtin_firmware(const struct firmware *fw)
 }
 #endif
 
-enum {
+enum fw_status {
+	FW_STATUS_UNKNOWN,
 	FW_STATUS_LOADING,
 	FW_STATUS_DONE,
-	FW_STATUS_ABORT,
+	FW_STATUS_ABORTED,
 };
 
 static int loading_timeout = 60;	/* In seconds */
@@ -104,6 +106,82 @@ static inline long firmware_loading_timeout(void)
 	return loading_timeout > 0 ? loading_timeout * HZ : MAX_JIFFY_OFFSET;
 }
 
+/*
+ * Concurrent request_firmware() for the same firmware need to be
+ * serialized.  struct fw_state is simple state machine which hold the
+ * state of the firmware loading.
+ */
+struct fw_state {
+	struct swait_queue_head wq;
+	enum fw_status status;
+};
+
+static void fw_state_init(struct fw_state *fw_st)
+{
+	init_swait_queue_head(&fw_st->wq);
+	fw_st->status = FW_STATUS_UNKNOWN;
+}
+
+static inline bool __fw_state_is_done(enum fw_status status)
+{
+	return status == FW_STATUS_DONE || status == FW_STATUS_ABORTED;
+}
+
+static int __fw_state_wait_common(struct fw_state *fw_st, long timeout)
+{
+	long ret;
+
+	ret = swait_event_interruptible_timeout(fw_st->wq,
+				__fw_state_is_done(READ_ONCE(fw_st->status)),
+				timeout);
+	if (ret != 0 && fw_st->status == FW_STATUS_ABORTED)
+		return -ENOENT;
+	if (!ret)
+		return -ETIMEDOUT;
+
+	return ret < 0 ? ret : 0;
+}
+
+static void __fw_state_set(struct fw_state *fw_st,
+			   enum fw_status status)
+{
+	WRITE_ONCE(fw_st->status, status);
+
+	if (status == FW_STATUS_DONE || status == FW_STATUS_ABORTED)
+		swake_up(&fw_st->wq);
+}
+
+#define fw_state_start(fw_st)					\
+	__fw_state_set(fw_st, FW_STATUS_LOADING)
+#define fw_state_done(fw_st)					\
+	__fw_state_set(fw_st, FW_STATUS_DONE)
+#define fw_state_wait(fw_st)					\
+	__fw_state_wait_common(fw_st, MAX_SCHEDULE_TIMEOUT)
+
+#ifndef CONFIG_FW_LOADER_USER_HELPER
+
+#define fw_state_is_aborted(fw_st)	false
+
+#else /* CONFIG_FW_LOADER_USER_HELPER */
+
+static int __fw_state_check(struct fw_state *fw_st, enum fw_status status)
+{
+	return fw_st->status == status;
+}
+
+#define fw_state_aborted(fw_st)					\
+	__fw_state_set(fw_st, FW_STATUS_ABORTED)
+#define fw_state_is_done(fw_st)					\
+	__fw_state_check(fw_st, FW_STATUS_DONE)
+#define fw_state_is_loading(fw_st)				\
+	__fw_state_check(fw_st, FW_STATUS_LOADING)
+#define fw_state_is_aborted(fw_st)				\
+	__fw_state_check(fw_st, FW_STATUS_ABORTED)
+#define fw_state_wait_timeout(fw_st, timeout)			\
+	__fw_state_wait_common(fw_st, timeout)
+
+#endif /* CONFIG_FW_LOADER_USER_HELPER */
+
 /* firmware behavior options */
 #define FW_OPT_UEVENT	(1U << 0)
 #define FW_OPT_NOWAIT	(1U << 1)
@@ -145,9 +223,8 @@ struct firmware_cache {
 struct firmware_buf {
 	struct kref ref;
 	struct list_head list;
-	struct completion completion;
 	struct firmware_cache *fwc;
-	unsigned long status;
+	struct fw_state fw_st;
 	void *data;
 	size_t size;
 	size_t allocated_size;
@@ -205,7 +282,7 @@ static struct firmware_buf *__allocate_fw_buf(const char *fw_name,
 	buf->fwc = fwc;
 	buf->data = dbuf;
 	buf->allocated_size = size;
-	init_completion(&buf->completion);
+	fw_state_init(&buf->fw_st);
 #ifdef CONFIG_FW_LOADER_USER_HELPER
 	INIT_LIST_HEAD(&buf->pending_list);
 #endif
@@ -305,15 +382,6 @@ static const char * const fw_path[] = {
 module_param_string(path, fw_path_para, sizeof(fw_path_para), 0644);
 MODULE_PARM_DESC(path, "customized firmware image search path with a higher priority than default path");
 
-static void fw_finish_direct_load(struct device *device,
-				  struct firmware_buf *buf)
-{
-	mutex_lock(&fw_lock);
-	set_bit(FW_STATUS_DONE, &buf->status);
-	complete_all(&buf->completion);
-	mutex_unlock(&fw_lock);
-}
-
 static int
 fw_get_filesystem_firmware(struct device *device, struct firmware_buf *buf)
 {
@@ -360,7 +428,7 @@ fw_get_filesystem_firmware(struct device *device, struct firmware_buf *buf)
 		}
 		dev_dbg(device, "direct-loading %s\n", buf->fw_id);
 		buf->size = size;
-		fw_finish_direct_load(device, buf);
+		fw_state_done(&buf->fw_st);
 		break;
 	}
 	__putname(path);
@@ -478,12 +546,11 @@ static void __fw_load_abort(struct firmware_buf *buf)
 	 * There is a small window in which user can write to 'loading'
 	 * between loading done and disappearance of 'loading'
 	 */
-	if (test_bit(FW_STATUS_DONE, &buf->status))
+	if (fw_state_is_done(&buf->fw_st))
 		return;
 
 	list_del_init(&buf->pending_list);
-	set_bit(FW_STATUS_ABORT, &buf->status);
-	complete_all(&buf->completion);
+	fw_state_aborted(&buf->fw_st);
 }
 
 static void fw_load_abort(struct firmware_priv *fw_priv)
@@ -496,9 +563,6 @@ static void fw_load_abort(struct firmware_priv *fw_priv)
 	fw_priv->buf = NULL;
 }
 
-#define is_fw_load_aborted(buf)	\
-	test_bit(FW_STATUS_ABORT, &(buf)->status)
-
 static LIST_HEAD(pending_fw_head);
 
 /* reboot notifier for avoid deadlock with usermode_lock */
@@ -546,11 +610,13 @@ static ssize_t timeout_store(struct class *class, struct class_attribute *attr,
 
 	return count;
 }
+static CLASS_ATTR_RW(timeout);
 
-static struct class_attribute firmware_class_attrs[] = {
-	__ATTR_RW(timeout),
-	__ATTR_NULL
+static struct attribute *firmware_class_attrs[] = {
+	&class_attr_timeout.attr,
+	NULL,
 };
+ATTRIBUTE_GROUPS(firmware_class);
 
 static void fw_dev_release(struct device *dev)
 {
@@ -585,7 +651,7 @@ static int firmware_uevent(struct device *dev, struct kobj_uevent_env *env)
 
 static struct class firmware_class = {
 	.name		= "firmware",
-	.class_attrs	= firmware_class_attrs,
+	.class_groups	= firmware_class_groups,
 	.dev_uevent	= firmware_uevent,
 	.dev_release	= fw_dev_release,
 };
@@ -598,7 +664,7 @@ static ssize_t firmware_loading_show(struct device *dev,
 
 	mutex_lock(&fw_lock);
 	if (fw_priv->buf)
-		loading = test_bit(FW_STATUS_LOADING, &fw_priv->buf->status);
+		loading = fw_state_is_loading(&fw_priv->buf->fw_st);
 	mutex_unlock(&fw_lock);
 
 	return sprintf(buf, "%d\n", loading);
@@ -653,23 +719,20 @@ static ssize_t firmware_loading_store(struct device *dev,
 	switch (loading) {
 	case 1:
 		/* discarding any previous partial load */
-		if (!test_bit(FW_STATUS_DONE, &fw_buf->status)) {
+		if (!fw_state_is_done(&fw_buf->fw_st)) {
 			for (i = 0; i < fw_buf->nr_pages; i++)
 				__free_page(fw_buf->pages[i]);
 			vfree(fw_buf->pages);
 			fw_buf->pages = NULL;
 			fw_buf->page_array_size = 0;
 			fw_buf->nr_pages = 0;
-			set_bit(FW_STATUS_LOADING, &fw_buf->status);
+			fw_state_start(&fw_buf->fw_st);
 		}
 		break;
 	case 0:
-		if (test_bit(FW_STATUS_LOADING, &fw_buf->status)) {
+		if (fw_state_is_loading(&fw_buf->fw_st)) {
 			int rc;
 
-			set_bit(FW_STATUS_DONE, &fw_buf->status);
-			clear_bit(FW_STATUS_LOADING, &fw_buf->status);
-
 			/*
 			 * Several loading requests may be pending on
 			 * one same firmware buf, so let all requests
@@ -691,10 +754,11 @@ static ssize_t firmware_loading_store(struct device *dev,
 			 */
 			list_del_init(&fw_buf->pending_list);
 			if (rc) {
-				set_bit(FW_STATUS_ABORT, &fw_buf->status);
+				fw_state_aborted(&fw_buf->fw_st);
 				written = rc;
+			} else {
+				fw_state_done(&fw_buf->fw_st);
 			}
-			complete_all(&fw_buf->completion);
 			break;
 		}
 		/* fallthrough */
@@ -755,7 +819,7 @@ static ssize_t firmware_data_read(struct file *filp, struct kobject *kobj,
 
 	mutex_lock(&fw_lock);
 	buf = fw_priv->buf;
-	if (!buf || test_bit(FW_STATUS_DONE, &buf->status)) {
+	if (!buf || fw_state_is_done(&buf->fw_st)) {
 		ret_count = -ENODEV;
 		goto out;
 	}
@@ -842,7 +906,7 @@ static ssize_t firmware_data_write(struct file *filp, struct kobject *kobj,
 
 	mutex_lock(&fw_lock);
 	buf = fw_priv->buf;
-	if (!buf || test_bit(FW_STATUS_DONE, &buf->status)) {
+	if (!buf || fw_state_is_done(&buf->fw_st)) {
 		retval = -ENODEV;
 		goto out;
 	}
@@ -955,17 +1019,14 @@ static int _request_firmware_load(struct firmware_priv *fw_priv,
 		timeout = MAX_JIFFY_OFFSET;
 	}
 
-	retval = wait_for_completion_interruptible_timeout(&buf->completion,
-			timeout);
-	if (retval == -ERESTARTSYS || !retval) {
+	retval = fw_state_wait_timeout(&buf->fw_st, timeout);
+	if (retval < 0) {
 		mutex_lock(&fw_lock);
 		fw_load_abort(fw_priv);
 		mutex_unlock(&fw_lock);
-	} else if (retval > 0) {
-		retval = 0;
 	}
 
-	if (is_fw_load_aborted(buf))
+	if (fw_state_is_aborted(&buf->fw_st))
 		retval = -EAGAIN;
 	else if (buf->is_paged_buf && !buf->data)
 		retval = -ENOMEM;
@@ -1015,35 +1076,12 @@ fw_load_from_user_helper(struct firmware *firmware, const char *name,
 	return -ENOENT;
 }
 
-/* No abort during direct loading */
-#define is_fw_load_aborted(buf) false
-
 #ifdef CONFIG_PM_SLEEP
 static inline void kill_requests_without_uevent(void) { }
 #endif
 
 #endif /* CONFIG_FW_LOADER_USER_HELPER */
 
-
-/* wait until the shared firmware_buf becomes ready (or error) */
-static int sync_cached_firmware_buf(struct firmware_buf *buf)
-{
-	int ret = 0;
-
-	mutex_lock(&fw_lock);
-	while (!test_bit(FW_STATUS_DONE, &buf->status)) {
-		if (is_fw_load_aborted(buf)) {
-			ret = -ENOENT;
-			break;
-		}
-		mutex_unlock(&fw_lock);
-		ret = wait_for_completion_interruptible(&buf->completion);
-		mutex_lock(&fw_lock);
-	}
-	mutex_unlock(&fw_lock);
-	return ret;
-}
-
 /* prepare firmware and firmware_buf structs;
  * return 0 if a firmware is already assigned, 1 if need to load one,
  * or a negative error code
@@ -1077,7 +1115,7 @@ _request_firmware_prepare(struct firmware **firmware_p, const char *name,
 	firmware->priv = buf;
 
 	if (ret > 0) {
-		ret = sync_cached_firmware_buf(buf);
+		ret = fw_state_wait(&buf->fw_st);
 		if (!ret) {
 			fw_set_page_data(buf, firmware);
 			return 0; /* assigned */
@@ -1095,7 +1133,7 @@ static int assign_firmware_buf(struct firmware *fw, struct device *device,
 	struct firmware_buf *buf = fw->priv;
 
 	mutex_lock(&fw_lock);
-	if (!buf->size || is_fw_load_aborted(buf)) {
+	if (!buf->size || fw_state_is_aborted(&buf->fw_st)) {
 		mutex_unlock(&fw_lock);
 		return -ENOENT;
 	}
@@ -1345,9 +1383,9 @@ static void request_firmware_work_func(struct work_struct *work)
  *
  *	Asynchronous variant of request_firmware() for user contexts:
  *		- sleep for as small periods as possible since it may
- *		increase kernel boot time of built-in device drivers
- *		requesting firmware in their ->probe() methods, if
- *		@gfp is GFP_KERNEL.
+ *		  increase kernel boot time of built-in device drivers
+ *		  requesting firmware in their ->probe() methods, if
+ *		  @gfp is GFP_KERNEL.
  *
  *		- can't sleep at all if @gfp is GFP_ATOMIC.
  **/
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index 62c63c0c5c22..bb69e58c29f3 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -226,11 +226,9 @@ memory_block_action(unsigned long phys_index, unsigned long action, int online_t
 {
 	unsigned long start_pfn;
 	unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
-	struct page *first_page;
 	int ret;
 
 	start_pfn = section_nr_to_pfn(phys_index);
-	first_page = pfn_to_page(start_pfn);
 
 	switch (action) {
 	case MEM_ONLINE:
diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c
index e023066e4215..5711708532db 100644
--- a/drivers/base/power/domain.c
+++ b/drivers/base/power/domain.c
@@ -39,6 +39,105 @@
 static LIST_HEAD(gpd_list);
 static DEFINE_MUTEX(gpd_list_lock);
 
+struct genpd_lock_ops {
+	void (*lock)(struct generic_pm_domain *genpd);
+	void (*lock_nested)(struct generic_pm_domain *genpd, int depth);
+	int (*lock_interruptible)(struct generic_pm_domain *genpd);
+	void (*unlock)(struct generic_pm_domain *genpd);
+};
+
+static void genpd_lock_mtx(struct generic_pm_domain *genpd)
+{
+	mutex_lock(&genpd->mlock);
+}
+
+static void genpd_lock_nested_mtx(struct generic_pm_domain *genpd,
+					int depth)
+{
+	mutex_lock_nested(&genpd->mlock, depth);
+}
+
+static int genpd_lock_interruptible_mtx(struct generic_pm_domain *genpd)
+{
+	return mutex_lock_interruptible(&genpd->mlock);
+}
+
+static void genpd_unlock_mtx(struct generic_pm_domain *genpd)
+{
+	return mutex_unlock(&genpd->mlock);
+}
+
+static const struct genpd_lock_ops genpd_mtx_ops = {
+	.lock = genpd_lock_mtx,
+	.lock_nested = genpd_lock_nested_mtx,
+	.lock_interruptible = genpd_lock_interruptible_mtx,
+	.unlock = genpd_unlock_mtx,
+};
+
+static void genpd_lock_spin(struct generic_pm_domain *genpd)
+	__acquires(&genpd->slock)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&genpd->slock, flags);
+	genpd->lock_flags = flags;
+}
+
+static void genpd_lock_nested_spin(struct generic_pm_domain *genpd,
+					int depth)
+	__acquires(&genpd->slock)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave_nested(&genpd->slock, flags, depth);
+	genpd->lock_flags = flags;
+}
+
+static int genpd_lock_interruptible_spin(struct generic_pm_domain *genpd)
+	__acquires(&genpd->slock)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&genpd->slock, flags);
+	genpd->lock_flags = flags;
+	return 0;
+}
+
+static void genpd_unlock_spin(struct generic_pm_domain *genpd)
+	__releases(&genpd->slock)
+{
+	spin_unlock_irqrestore(&genpd->slock, genpd->lock_flags);
+}
+
+static const struct genpd_lock_ops genpd_spin_ops = {
+	.lock = genpd_lock_spin,
+	.lock_nested = genpd_lock_nested_spin,
+	.lock_interruptible = genpd_lock_interruptible_spin,
+	.unlock = genpd_unlock_spin,
+};
+
+#define genpd_lock(p)			p->lock_ops->lock(p)
+#define genpd_lock_nested(p, d)		p->lock_ops->lock_nested(p, d)
+#define genpd_lock_interruptible(p)	p->lock_ops->lock_interruptible(p)
+#define genpd_unlock(p)			p->lock_ops->unlock(p)
+
+#define genpd_is_irq_safe(genpd)	(genpd->flags & GENPD_FLAG_IRQ_SAFE)
+
+static inline bool irq_safe_dev_in_no_sleep_domain(struct device *dev,
+		struct generic_pm_domain *genpd)
+{
+	bool ret;
+
+	ret = pm_runtime_is_irq_safe(dev) && !genpd_is_irq_safe(genpd);
+
+	/* Warn once for each IRQ safe dev in no sleep domain */
+	if (ret)
+		dev_warn_once(dev, "PM domain %s will not be powered off\n",
+				genpd->name);
+
+	return ret;
+}
+
 /*
  * Get the generic PM domain for a particular struct device.
  * This validates the struct device pointer, the PM domain pointer,
@@ -200,9 +299,9 @@ static int genpd_poweron(struct generic_pm_domain *genpd, unsigned int depth)
 
 		genpd_sd_counter_inc(master);
 
-		mutex_lock_nested(&master->lock, depth + 1);
+		genpd_lock_nested(master, depth + 1);
 		ret = genpd_poweron(master, depth + 1);
-		mutex_unlock(&master->lock);
+		genpd_unlock(master);
 
 		if (ret) {
 			genpd_sd_counter_dec(master);
@@ -255,9 +354,9 @@ static int genpd_dev_pm_qos_notifier(struct notifier_block *nb,
 		spin_unlock_irq(&dev->power.lock);
 
 		if (!IS_ERR(genpd)) {
-			mutex_lock(&genpd->lock);
+			genpd_lock(genpd);
 			genpd->max_off_time_changed = true;
-			mutex_unlock(&genpd->lock);
+			genpd_unlock(genpd);
 		}
 
 		dev = dev->parent;
@@ -303,7 +402,12 @@ static int genpd_poweroff(struct generic_pm_domain *genpd, bool is_async)
 		if (stat > PM_QOS_FLAGS_NONE)
 			return -EBUSY;
 
-		if (!pm_runtime_suspended(pdd->dev) || pdd->dev->power.irq_safe)
+		/*
+		 * Do not allow PM domain to be powered off, when an IRQ safe
+		 * device is part of a non-IRQ safe domain.
+		 */
+		if (!pm_runtime_suspended(pdd->dev) ||
+			irq_safe_dev_in_no_sleep_domain(pdd->dev, genpd))
 			not_suspended++;
 	}
 
@@ -354,9 +458,9 @@ static void genpd_power_off_work_fn(struct work_struct *work)
 
 	genpd = container_of(work, struct generic_pm_domain, power_off_work);
 
-	mutex_lock(&genpd->lock);
+	genpd_lock(genpd);
 	genpd_poweroff(genpd, true);
-	mutex_unlock(&genpd->lock);
+	genpd_unlock(genpd);
 }
 
 /**
@@ -466,15 +570,15 @@ static int genpd_runtime_suspend(struct device *dev)
 	}
 
 	/*
-	 * If power.irq_safe is set, this routine will be run with interrupts
-	 * off, so it can't use mutexes.
+	 * If power.irq_safe is set, this routine may be run with
+	 * IRQs disabled, so suspend only if the PM domain also is irq_safe.
 	 */
-	if (dev->power.irq_safe)
+	if (irq_safe_dev_in_no_sleep_domain(dev, genpd))
 		return 0;
 
-	mutex_lock(&genpd->lock);
+	genpd_lock(genpd);
 	genpd_poweroff(genpd, false);
-	mutex_unlock(&genpd->lock);
+	genpd_unlock(genpd);
 
 	return 0;
 }
@@ -503,15 +607,18 @@ static int genpd_runtime_resume(struct device *dev)
 	if (IS_ERR(genpd))
 		return -EINVAL;
 
-	/* If power.irq_safe, the PM domain is never powered off. */
-	if (dev->power.irq_safe) {
+	/*
+	 * As we don't power off a non IRQ safe domain, which holds
+	 * an IRQ safe device, we don't need to restore power to it.
+	 */
+	if (irq_safe_dev_in_no_sleep_domain(dev, genpd)) {
 		timed = false;
 		goto out;
 	}
 
-	mutex_lock(&genpd->lock);
+	genpd_lock(genpd);
 	ret = genpd_poweron(genpd, 0);
-	mutex_unlock(&genpd->lock);
+	genpd_unlock(genpd);
 
 	if (ret)
 		return ret;
@@ -546,10 +653,11 @@ static int genpd_runtime_resume(struct device *dev)
 err_stop:
 	genpd_stop_dev(genpd, dev);
 err_poweroff:
-	if (!dev->power.irq_safe) {
-		mutex_lock(&genpd->lock);
+	if (!pm_runtime_is_irq_safe(dev) ||
+		(pm_runtime_is_irq_safe(dev) && genpd_is_irq_safe(genpd))) {
+		genpd_lock(genpd);
 		genpd_poweroff(genpd, 0);
-		mutex_unlock(&genpd->lock);
+		genpd_unlock(genpd);
 	}
 
 	return ret;
@@ -732,20 +840,20 @@ static int pm_genpd_prepare(struct device *dev)
 	if (resume_needed(dev, genpd))
 		pm_runtime_resume(dev);
 
-	mutex_lock(&genpd->lock);
+	genpd_lock(genpd);
 
 	if (genpd->prepared_count++ == 0)
 		genpd->suspended_count = 0;
 
-	mutex_unlock(&genpd->lock);
+	genpd_unlock(genpd);
 
 	ret = pm_generic_prepare(dev);
 	if (ret) {
-		mutex_lock(&genpd->lock);
+		genpd_lock(genpd);
 
 		genpd->prepared_count--;
 
-		mutex_unlock(&genpd->lock);
+		genpd_unlock(genpd);
 	}
 
 	return ret;
@@ -936,13 +1044,13 @@ static void pm_genpd_complete(struct device *dev)
 
 	pm_generic_complete(dev);
 
-	mutex_lock(&genpd->lock);
+	genpd_lock(genpd);
 
 	genpd->prepared_count--;
 	if (!genpd->prepared_count)
 		genpd_queue_power_off_work(genpd);
 
-	mutex_unlock(&genpd->lock);
+	genpd_unlock(genpd);
 }
 
 /**
@@ -1071,7 +1179,7 @@ static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
 	if (IS_ERR(gpd_data))
 		return PTR_ERR(gpd_data);
 
-	mutex_lock(&genpd->lock);
+	genpd_lock(genpd);
 
 	if (genpd->prepared_count > 0) {
 		ret = -EAGAIN;
@@ -1088,7 +1196,7 @@ static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
 	list_add_tail(&gpd_data->base.list_node, &genpd->dev_list);
 
  out:
-	mutex_unlock(&genpd->lock);
+	genpd_unlock(genpd);
 
 	if (ret)
 		genpd_free_dev_data(dev, gpd_data);
@@ -1130,7 +1238,7 @@ static int genpd_remove_device(struct generic_pm_domain *genpd,
 	gpd_data = to_gpd_data(pdd);
 	dev_pm_qos_remove_notifier(dev, &gpd_data->nb);
 
-	mutex_lock(&genpd->lock);
+	genpd_lock(genpd);
 
 	if (genpd->prepared_count > 0) {
 		ret = -EAGAIN;
@@ -1145,14 +1253,14 @@ static int genpd_remove_device(struct generic_pm_domain *genpd,
 
 	list_del_init(&pdd->list_node);
 
-	mutex_unlock(&genpd->lock);
+	genpd_unlock(genpd);
 
 	genpd_free_dev_data(dev, gpd_data);
 
 	return 0;
 
  out:
-	mutex_unlock(&genpd->lock);
+	genpd_unlock(genpd);
 	dev_pm_qos_add_notifier(dev, &gpd_data->nb);
 
 	return ret;
@@ -1183,12 +1291,23 @@ static int genpd_add_subdomain(struct generic_pm_domain *genpd,
 	    || genpd == subdomain)
 		return -EINVAL;
 
+	/*
+	 * If the domain can be powered on/off in an IRQ safe
+	 * context, ensure that the subdomain can also be
+	 * powered on/off in that context.
+	 */
+	if (!genpd_is_irq_safe(genpd) && genpd_is_irq_safe(subdomain)) {
+		WARN(1, "Parent %s of subdomain %s must be IRQ safe\n",
+				genpd->name, subdomain->name);
+		return -EINVAL;
+	}
+
 	link = kzalloc(sizeof(*link), GFP_KERNEL);
 	if (!link)
 		return -ENOMEM;
 
-	mutex_lock(&subdomain->lock);
-	mutex_lock_nested(&genpd->lock, SINGLE_DEPTH_NESTING);
+	genpd_lock(subdomain);
+	genpd_lock_nested(genpd, SINGLE_DEPTH_NESTING);
 
 	if (genpd->status == GPD_STATE_POWER_OFF
 	    &&  subdomain->status != GPD_STATE_POWER_OFF) {
@@ -1211,8 +1330,8 @@ static int genpd_add_subdomain(struct generic_pm_domain *genpd,
 		genpd_sd_counter_inc(genpd);
 
  out:
-	mutex_unlock(&genpd->lock);
-	mutex_unlock(&subdomain->lock);
+	genpd_unlock(genpd);
+	genpd_unlock(subdomain);
 	if (ret)
 		kfree(link);
 	return ret;
@@ -1250,8 +1369,8 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd,
 	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain))
 		return -EINVAL;
 
-	mutex_lock(&subdomain->lock);
-	mutex_lock_nested(&genpd->lock, SINGLE_DEPTH_NESTING);
+	genpd_lock(subdomain);
+	genpd_lock_nested(genpd, SINGLE_DEPTH_NESTING);
 
 	if (!list_empty(&subdomain->master_links) || subdomain->device_count) {
 		pr_warn("%s: unable to remove subdomain %s\n", genpd->name,
@@ -1275,13 +1394,39 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd,
 	}
 
 out:
-	mutex_unlock(&genpd->lock);
-	mutex_unlock(&subdomain->lock);
+	genpd_unlock(genpd);
+	genpd_unlock(subdomain);
 
 	return ret;
 }
 EXPORT_SYMBOL_GPL(pm_genpd_remove_subdomain);
 
+static int genpd_set_default_power_state(struct generic_pm_domain *genpd)
+{
+	struct genpd_power_state *state;
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return -ENOMEM;
+
+	genpd->states = state;
+	genpd->state_count = 1;
+	genpd->free = state;
+
+	return 0;
+}
+
+static void genpd_lock_init(struct generic_pm_domain *genpd)
+{
+	if (genpd->flags & GENPD_FLAG_IRQ_SAFE) {
+		spin_lock_init(&genpd->slock);
+		genpd->lock_ops = &genpd_spin_ops;
+	} else {
+		mutex_init(&genpd->mlock);
+		genpd->lock_ops = &genpd_mtx_ops;
+	}
+}
+
 /**
  * pm_genpd_init - Initialize a generic I/O PM domain object.
  * @genpd: PM domain object to initialize.
@@ -1293,13 +1438,15 @@ EXPORT_SYMBOL_GPL(pm_genpd_remove_subdomain);
 int pm_genpd_init(struct generic_pm_domain *genpd,
 		  struct dev_power_governor *gov, bool is_off)
 {
+	int ret;
+
 	if (IS_ERR_OR_NULL(genpd))
 		return -EINVAL;
 
 	INIT_LIST_HEAD(&genpd->master_links);
 	INIT_LIST_HEAD(&genpd->slave_links);
 	INIT_LIST_HEAD(&genpd->dev_list);
-	mutex_init(&genpd->lock);
+	genpd_lock_init(genpd);
 	genpd->gov = gov;
 	INIT_WORK(&genpd->power_off_work, genpd_power_off_work_fn);
 	atomic_set(&genpd->sd_count, 0);
@@ -1325,19 +1472,12 @@ int pm_genpd_init(struct generic_pm_domain *genpd,
 		genpd->dev_ops.start = pm_clk_resume;
 	}
 
-	if (genpd->state_idx >= GENPD_MAX_NUM_STATES) {
-		pr_warn("Initial state index out of bounds.\n");
-		genpd->state_idx = GENPD_MAX_NUM_STATES - 1;
-	}
-
-	if (genpd->state_count > GENPD_MAX_NUM_STATES) {
-		pr_warn("Limiting states to  %d\n", GENPD_MAX_NUM_STATES);
-		genpd->state_count = GENPD_MAX_NUM_STATES;
-	}
-
 	/* Use only one "off" state if there were no states declared */
-	if (genpd->state_count == 0)
-		genpd->state_count = 1;
+	if (genpd->state_count == 0) {
+		ret = genpd_set_default_power_state(genpd);
+		if (ret)
+			return ret;
+	}
 
 	mutex_lock(&gpd_list_lock);
 	list_add(&genpd->gpd_list_node, &gpd_list);
@@ -1354,16 +1494,16 @@ static int genpd_remove(struct generic_pm_domain *genpd)
 	if (IS_ERR_OR_NULL(genpd))
 		return -EINVAL;
 
-	mutex_lock(&genpd->lock);
+	genpd_lock(genpd);
 
 	if (genpd->has_provider) {
-		mutex_unlock(&genpd->lock);
+		genpd_unlock(genpd);
 		pr_err("Provider present, unable to remove %s\n", genpd->name);
 		return -EBUSY;
 	}
 
 	if (!list_empty(&genpd->master_links) || genpd->device_count) {
-		mutex_unlock(&genpd->lock);
+		genpd_unlock(genpd);
 		pr_err("%s: unable to remove %s\n", __func__, genpd->name);
 		return -EBUSY;
 	}
@@ -1375,8 +1515,9 @@ static int genpd_remove(struct generic_pm_domain *genpd)
 	}
 
 	list_del(&genpd->gpd_list_node);
-	mutex_unlock(&genpd->lock);
+	genpd_unlock(genpd);
 	cancel_work_sync(&genpd->power_off_work);
+	kfree(genpd->free);
 	pr_debug("%s: removed %s\n", __func__, genpd->name);
 
 	return 0;
@@ -1890,21 +2031,117 @@ int genpd_dev_pm_attach(struct device *dev)
 	mutex_unlock(&gpd_list_lock);
 
 	if (ret < 0) {
-		dev_err(dev, "failed to add to PM domain %s: %d",
-			pd->name, ret);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "failed to add to PM domain %s: %d",
+				pd->name, ret);
 		goto out;
 	}
 
 	dev->pm_domain->detach = genpd_dev_pm_detach;
 	dev->pm_domain->sync = genpd_dev_pm_sync;
 
-	mutex_lock(&pd->lock);
+	genpd_lock(pd);
 	ret = genpd_poweron(pd, 0);
-	mutex_unlock(&pd->lock);
+	genpd_unlock(pd);
 out:
 	return ret ? -EPROBE_DEFER : 0;
 }
 EXPORT_SYMBOL_GPL(genpd_dev_pm_attach);
+
+static const struct of_device_id idle_state_match[] = {
+	{ .compatible = "domain-idle-state", },
+	{ }
+};
+
+static int genpd_parse_state(struct genpd_power_state *genpd_state,
+				    struct device_node *state_node)
+{
+	int err;
+	u32 residency;
+	u32 entry_latency, exit_latency;
+	const struct of_device_id *match_id;
+
+	match_id = of_match_node(idle_state_match, state_node);
+	if (!match_id)
+		return -EINVAL;
+
+	err = of_property_read_u32(state_node, "entry-latency-us",
+						&entry_latency);
+	if (err) {
+		pr_debug(" * %s missing entry-latency-us property\n",
+						state_node->full_name);
+		return -EINVAL;
+	}
+
+	err = of_property_read_u32(state_node, "exit-latency-us",
+						&exit_latency);
+	if (err) {
+		pr_debug(" * %s missing exit-latency-us property\n",
+						state_node->full_name);
+		return -EINVAL;
+	}
+
+	err = of_property_read_u32(state_node, "min-residency-us", &residency);
+	if (!err)
+		genpd_state->residency_ns = 1000 * residency;
+
+	genpd_state->power_on_latency_ns = 1000 * exit_latency;
+	genpd_state->power_off_latency_ns = 1000 * entry_latency;
+	genpd_state->fwnode = &state_node->fwnode;
+
+	return 0;
+}
+
+/**
+ * of_genpd_parse_idle_states: Return array of idle states for the genpd.
+ *
+ * @dn: The genpd device node
+ * @states: The pointer to which the state array will be saved.
+ * @n: The count of elements in the array returned from this function.
+ *
+ * Returns the device states parsed from the OF node. The memory for the states
+ * is allocated by this function and is the responsibility of the caller to
+ * free the memory after use.
+ */
+int of_genpd_parse_idle_states(struct device_node *dn,
+			struct genpd_power_state **states, int *n)
+{
+	struct genpd_power_state *st;
+	struct device_node *np;
+	int i = 0;
+	int err, ret;
+	int count;
+	struct of_phandle_iterator it;
+
+	count = of_count_phandle_with_args(dn, "domain-idle-states", NULL);
+	if (count <= 0)
+		return -EINVAL;
+
+	st = kcalloc(count, sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return -ENOMEM;
+
+	/* Loop over the phandles until all the requested entry is found */
+	of_for_each_phandle(&it, err, dn, "domain-idle-states", NULL, 0) {
+		np = it.node;
+		ret = genpd_parse_state(&st[i++], np);
+		if (ret) {
+			pr_err
+			("Parsing idle state node %s failed with err %d\n",
+							np->full_name, ret);
+			of_node_put(np);
+			kfree(st);
+			return ret;
+		}
+	}
+
+	*n = count;
+	*states = st;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(of_genpd_parse_idle_states);
+
 #endif /* CONFIG_PM_GENERIC_DOMAINS_OF */
 
 
@@ -1958,7 +2195,7 @@ static int pm_genpd_summary_one(struct seq_file *s,
 	char state[16];
 	int ret;
 
-	ret = mutex_lock_interruptible(&genpd->lock);
+	ret = genpd_lock_interruptible(genpd);
 	if (ret)
 		return -ERESTARTSYS;
 
@@ -1984,7 +2221,9 @@ static int pm_genpd_summary_one(struct seq_file *s,
 	}
 
 	list_for_each_entry(pm_data, &genpd->dev_list, list_node) {
-		kobj_path = kobject_get_path(&pm_data->dev->kobj, GFP_KERNEL);
+		kobj_path = kobject_get_path(&pm_data->dev->kobj,
+				genpd_is_irq_safe(genpd) ?
+				GFP_ATOMIC : GFP_KERNEL);
 		if (kobj_path == NULL)
 			continue;
 
@@ -1995,7 +2234,7 @@ static int pm_genpd_summary_one(struct seq_file *s,
 
 	seq_puts(s, "\n");
 exit:
-	mutex_unlock(&genpd->lock);
+	genpd_unlock(genpd);
 
 	return 0;
 }
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index 2932a5bd892f..48c6294e9c34 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -131,6 +131,7 @@ void device_pm_add(struct device *dev)
 		dev_warn(dev, "parent %s should not be sleeping\n",
 			dev_name(dev->parent));
 	list_add_tail(&dev->power.entry, &dpm_list);
+	dev->power.in_dpm_list = true;
 	mutex_unlock(&dpm_list_mtx);
 }
 
@@ -145,6 +146,7 @@ void device_pm_remove(struct device *dev)
 	complete_all(&dev->power.completion);
 	mutex_lock(&dpm_list_mtx);
 	list_del_init(&dev->power.entry);
+	dev->power.in_dpm_list = false;
 	mutex_unlock(&dpm_list_mtx);
 	device_wakeup_disable(dev);
 	pm_runtime_remove(dev);
@@ -244,6 +246,62 @@ static void dpm_wait_for_children(struct device *dev, bool async)
        device_for_each_child(dev, &async, dpm_wait_fn);
 }
 
+static void dpm_wait_for_suppliers(struct device *dev, bool async)
+{
+	struct device_link *link;
+	int idx;
+
+	idx = device_links_read_lock();
+
+	/*
+	 * If the supplier goes away right after we've checked the link to it,
+	 * we'll wait for its completion to change the state, but that's fine,
+	 * because the only things that will block as a result are the SRCU
+	 * callbacks freeing the link objects for the links in the list we're
+	 * walking.
+	 */
+	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
+		if (READ_ONCE(link->status) != DL_STATE_DORMANT)
+			dpm_wait(link->supplier, async);
+
+	device_links_read_unlock(idx);
+}
+
+static void dpm_wait_for_superior(struct device *dev, bool async)
+{
+	dpm_wait(dev->parent, async);
+	dpm_wait_for_suppliers(dev, async);
+}
+
+static void dpm_wait_for_consumers(struct device *dev, bool async)
+{
+	struct device_link *link;
+	int idx;
+
+	idx = device_links_read_lock();
+
+	/*
+	 * The status of a device link can only be changed from "dormant" by a
+	 * probe, but that cannot happen during system suspend/resume.  In
+	 * theory it can change to "dormant" at that time, but then it is
+	 * reasonable to wait for the target device anyway (eg. if it goes
+	 * away, it's better to wait for it to go away completely and then
+	 * continue instead of trying to continue in parallel with its
+	 * unregistration).
+	 */
+	list_for_each_entry_rcu(link, &dev->links.consumers, s_node)
+		if (READ_ONCE(link->status) != DL_STATE_DORMANT)
+			dpm_wait(link->consumer, async);
+
+	device_links_read_unlock(idx);
+}
+
+static void dpm_wait_for_subordinate(struct device *dev, bool async)
+{
+	dpm_wait_for_children(dev, async);
+	dpm_wait_for_consumers(dev, async);
+}
+
 /**
  * pm_op - Return the PM operation appropriate for given PM event.
  * @ops: PM operations to choose from.
@@ -488,7 +546,7 @@ static int device_resume_noirq(struct device *dev, pm_message_t state, bool asyn
 	if (!dev->power.is_noirq_suspended)
 		goto Out;
 
-	dpm_wait(dev->parent, async);
+	dpm_wait_for_superior(dev, async);
 
 	if (dev->pm_domain) {
 		info = "noirq power domain ";
@@ -618,7 +676,7 @@ static int device_resume_early(struct device *dev, pm_message_t state, bool asyn
 	if (!dev->power.is_late_suspended)
 		goto Out;
 
-	dpm_wait(dev->parent, async);
+	dpm_wait_for_superior(dev, async);
 
 	if (dev->pm_domain) {
 		info = "early power domain ";
@@ -750,7 +808,7 @@ static int device_resume(struct device *dev, pm_message_t state, bool async)
 		goto Complete;
 	}
 
-	dpm_wait(dev->parent, async);
+	dpm_wait_for_superior(dev, async);
 	dpm_watchdog_set(&wd, dev);
 	device_lock(dev);
 
@@ -1027,7 +1085,7 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a
 	TRACE_DEVICE(dev);
 	TRACE_SUSPEND(0);
 
-	dpm_wait_for_children(dev, async);
+	dpm_wait_for_subordinate(dev, async);
 
 	if (async_error)
 		goto Complete;
@@ -1174,7 +1232,7 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as
 
 	__pm_runtime_disable(dev, false);
 
-	dpm_wait_for_children(dev, async);
+	dpm_wait_for_subordinate(dev, async);
 
 	if (async_error)
 		goto Complete;
@@ -1342,6 +1400,22 @@ static int legacy_suspend(struct device *dev, pm_message_t state,
 	return error;
 }
 
+static void dpm_clear_suppliers_direct_complete(struct device *dev)
+{
+	struct device_link *link;
+	int idx;
+
+	idx = device_links_read_lock();
+
+	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) {
+		spin_lock_irq(&link->supplier->power.lock);
+		link->supplier->power.direct_complete = false;
+		spin_unlock_irq(&link->supplier->power.lock);
+	}
+
+	device_links_read_unlock(idx);
+}
+
 /**
  * device_suspend - Execute "suspend" callbacks for given device.
  * @dev: Device to handle.
@@ -1358,7 +1432,7 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
 	TRACE_DEVICE(dev);
 	TRACE_SUSPEND(0);
 
-	dpm_wait_for_children(dev, async);
+	dpm_wait_for_subordinate(dev, async);
 
 	if (async_error)
 		goto Complete;
@@ -1454,16 +1528,17 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
 
 			spin_unlock_irq(&parent->power.lock);
 		}
+		dpm_clear_suppliers_direct_complete(dev);
 	}
 
 	device_unlock(dev);
 	dpm_watchdog_clear(&wd);
 
  Complete:
-	complete_all(&dev->power.completion);
 	if (error)
 		async_error = error;
 
+	complete_all(&dev->power.completion);
 	TRACE_SUSPEND(error);
 	return error;
 }
diff --git a/drivers/base/power/opp/core.c b/drivers/base/power/opp/core.c
index 4c7c6da7a989..35ff06283738 100644
--- a/drivers/base/power/opp/core.c
+++ b/drivers/base/power/opp/core.c
@@ -93,6 +93,8 @@ struct opp_table *_find_opp_table(struct device *dev)
  * Return: voltage in micro volt corresponding to the opp, else
  * return 0
  *
+ * This is useful only for devices with single power supply.
+ *
  * Locking: This function must be called under rcu_read_lock(). opp is a rcu
  * protected pointer. This means that opp which could have been fetched by
  * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
@@ -112,7 +114,7 @@ unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp)
 	if (IS_ERR_OR_NULL(tmp_opp))
 		pr_err("%s: Invalid parameters\n", __func__);
 	else
-		v = tmp_opp->u_volt;
+		v = tmp_opp->supplies[0].u_volt;
 
 	return v;
 }
@@ -210,6 +212,24 @@ unsigned long dev_pm_opp_get_max_clock_latency(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_clock_latency);
 
+static int _get_regulator_count(struct device *dev)
+{
+	struct opp_table *opp_table;
+	int count;
+
+	rcu_read_lock();
+
+	opp_table = _find_opp_table(dev);
+	if (!IS_ERR(opp_table))
+		count = opp_table->regulator_count;
+	else
+		count = 0;
+
+	rcu_read_unlock();
+
+	return count;
+}
+
 /**
  * dev_pm_opp_get_max_volt_latency() - Get max voltage latency in nanoseconds
  * @dev: device for which we do this operation
@@ -222,34 +242,51 @@ unsigned long dev_pm_opp_get_max_volt_latency(struct device *dev)
 {
 	struct opp_table *opp_table;
 	struct dev_pm_opp *opp;
-	struct regulator *reg;
+	struct regulator *reg, **regulators;
 	unsigned long latency_ns = 0;
-	unsigned long min_uV = ~0, max_uV = 0;
-	int ret;
+	int ret, i, count;
+	struct {
+		unsigned long min;
+		unsigned long max;
+	} *uV;
+
+	count = _get_regulator_count(dev);
+
+	/* Regulator may not be required for the device */
+	if (!count)
+		return 0;
+
+	regulators = kmalloc_array(count, sizeof(*regulators), GFP_KERNEL);
+	if (!regulators)
+		return 0;
+
+	uV = kmalloc_array(count, sizeof(*uV), GFP_KERNEL);
+	if (!uV)
+		goto free_regulators;
 
 	rcu_read_lock();
 
 	opp_table = _find_opp_table(dev);
 	if (IS_ERR(opp_table)) {
 		rcu_read_unlock();
-		return 0;
+		goto free_uV;
 	}
 
-	reg = opp_table->regulator;
-	if (IS_ERR(reg)) {
-		/* Regulator may not be required for device */
-		rcu_read_unlock();
-		return 0;
-	}
+	memcpy(regulators, opp_table->regulators, count * sizeof(*regulators));
 
-	list_for_each_entry_rcu(opp, &opp_table->opp_list, node) {
-		if (!opp->available)
-			continue;
+	for (i = 0; i < count; i++) {
+		uV[i].min = ~0;
+		uV[i].max = 0;
+
+		list_for_each_entry_rcu(opp, &opp_table->opp_list, node) {
+			if (!opp->available)
+				continue;
 
-		if (opp->u_volt_min < min_uV)
-			min_uV = opp->u_volt_min;
-		if (opp->u_volt_max > max_uV)
-			max_uV = opp->u_volt_max;
+			if (opp->supplies[i].u_volt_min < uV[i].min)
+				uV[i].min = opp->supplies[i].u_volt_min;
+			if (opp->supplies[i].u_volt_max > uV[i].max)
+				uV[i].max = opp->supplies[i].u_volt_max;
+		}
 	}
 
 	rcu_read_unlock();
@@ -258,9 +295,16 @@ unsigned long dev_pm_opp_get_max_volt_latency(struct device *dev)
 	 * The caller needs to ensure that opp_table (and hence the regulator)
 	 * isn't freed, while we are executing this routine.
 	 */
-	ret = regulator_set_voltage_time(reg, min_uV, max_uV);
-	if (ret > 0)
-		latency_ns = ret * 1000;
+	for (i = 0; reg = regulators[i], i < count; i++) {
+		ret = regulator_set_voltage_time(reg, uV[i].min, uV[i].max);
+		if (ret > 0)
+			latency_ns += ret * 1000;
+	}
+
+free_uV:
+	kfree(uV);
+free_regulators:
+	kfree(regulators);
 
 	return latency_ns;
 }
@@ -542,8 +586,7 @@ unlock:
 }
 
 static int _set_opp_voltage(struct device *dev, struct regulator *reg,
-			    unsigned long u_volt, unsigned long u_volt_min,
-			    unsigned long u_volt_max)
+			    struct dev_pm_opp_supply *supply)
 {
 	int ret;
 
@@ -554,14 +597,78 @@ static int _set_opp_voltage(struct device *dev, struct regulator *reg,
 		return 0;
 	}
 
-	dev_dbg(dev, "%s: voltages (mV): %lu %lu %lu\n", __func__, u_volt_min,
-		u_volt, u_volt_max);
+	dev_dbg(dev, "%s: voltages (mV): %lu %lu %lu\n", __func__,
+		supply->u_volt_min, supply->u_volt, supply->u_volt_max);
 
-	ret = regulator_set_voltage_triplet(reg, u_volt_min, u_volt,
-					    u_volt_max);
+	ret = regulator_set_voltage_triplet(reg, supply->u_volt_min,
+					    supply->u_volt, supply->u_volt_max);
 	if (ret)
 		dev_err(dev, "%s: failed to set voltage (%lu %lu %lu mV): %d\n",
-			__func__, u_volt_min, u_volt, u_volt_max, ret);
+			__func__, supply->u_volt_min, supply->u_volt,
+			supply->u_volt_max, ret);
+
+	return ret;
+}
+
+static inline int
+_generic_set_opp_clk_only(struct device *dev, struct clk *clk,
+			  unsigned long old_freq, unsigned long freq)
+{
+	int ret;
+
+	ret = clk_set_rate(clk, freq);
+	if (ret) {
+		dev_err(dev, "%s: failed to set clock rate: %d\n", __func__,
+			ret);
+	}
+
+	return ret;
+}
+
+static int _generic_set_opp(struct dev_pm_set_opp_data *data)
+{
+	struct dev_pm_opp_supply *old_supply = data->old_opp.supplies;
+	struct dev_pm_opp_supply *new_supply = data->new_opp.supplies;
+	unsigned long old_freq = data->old_opp.rate, freq = data->new_opp.rate;
+	struct regulator *reg = data->regulators[0];
+	struct device *dev= data->dev;
+	int ret;
+
+	/* This function only supports single regulator per device */
+	if (WARN_ON(data->regulator_count > 1)) {
+		dev_err(dev, "multiple regulators are not supported\n");
+		return -EINVAL;
+	}
+
+	/* Scaling up? Scale voltage before frequency */
+	if (freq > old_freq) {
+		ret = _set_opp_voltage(dev, reg, new_supply);
+		if (ret)
+			goto restore_voltage;
+	}
+
+	/* Change frequency */
+	ret = _generic_set_opp_clk_only(dev, data->clk, old_freq, freq);
+	if (ret)
+		goto restore_voltage;
+
+	/* Scaling down? Scale voltage after frequency */
+	if (freq < old_freq) {
+		ret = _set_opp_voltage(dev, reg, new_supply);
+		if (ret)
+			goto restore_freq;
+	}
+
+	return 0;
+
+restore_freq:
+	if (_generic_set_opp_clk_only(dev, data->clk, freq, old_freq))
+		dev_err(dev, "%s: failed to restore old-freq (%lu Hz)\n",
+			__func__, old_freq);
+restore_voltage:
+	/* This shouldn't harm even if the voltages weren't updated earlier */
+	if (old_supply->u_volt)
+		_set_opp_voltage(dev, reg, old_supply);
 
 	return ret;
 }
@@ -579,12 +686,13 @@ static int _set_opp_voltage(struct device *dev, struct regulator *reg,
 int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq)
 {
 	struct opp_table *opp_table;
+	unsigned long freq, old_freq;
+	int (*set_opp)(struct dev_pm_set_opp_data *data);
 	struct dev_pm_opp *old_opp, *opp;
-	struct regulator *reg;
+	struct regulator **regulators;
+	struct dev_pm_set_opp_data *data;
 	struct clk *clk;
-	unsigned long freq, old_freq;
-	unsigned long u_volt, u_volt_min, u_volt_max;
-	int ret;
+	int ret, size;
 
 	if (unlikely(!target_freq)) {
 		dev_err(dev, "%s: Invalid target frequency %lu\n", __func__,
@@ -633,55 +741,41 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq)
 		return ret;
 	}
 
-	u_volt = opp->u_volt;
-	u_volt_min = opp->u_volt_min;
-	u_volt_max = opp->u_volt_max;
+	dev_dbg(dev, "%s: switching OPP: %lu Hz --> %lu Hz\n", __func__,
+		old_freq, freq);
 
-	reg = opp_table->regulator;
+	regulators = opp_table->regulators;
 
-	rcu_read_unlock();
-
-	/* Scaling up? Scale voltage before frequency */
-	if (freq > old_freq) {
-		ret = _set_opp_voltage(dev, reg, u_volt, u_volt_min,
-				       u_volt_max);
-		if (ret)
-			goto restore_voltage;
-	}
-
-	/* Change frequency */
-
-	dev_dbg(dev, "%s: switching OPP: %lu Hz --> %lu Hz\n",
-		__func__, old_freq, freq);
-
-	ret = clk_set_rate(clk, freq);
-	if (ret) {
-		dev_err(dev, "%s: failed to set clock rate: %d\n", __func__,
-			ret);
-		goto restore_voltage;
+	/* Only frequency scaling */
+	if (!regulators) {
+		rcu_read_unlock();
+		return _generic_set_opp_clk_only(dev, clk, old_freq, freq);
 	}
 
-	/* Scaling down? Scale voltage after frequency */
-	if (freq < old_freq) {
-		ret = _set_opp_voltage(dev, reg, u_volt, u_volt_min,
-				       u_volt_max);
-		if (ret)
-			goto restore_freq;
-	}
+	if (opp_table->set_opp)
+		set_opp = opp_table->set_opp;
+	else
+		set_opp = _generic_set_opp;
+
+	data = opp_table->set_opp_data;
+	data->regulators = regulators;
+	data->regulator_count = opp_table->regulator_count;
+	data->clk = clk;
+	data->dev = dev;
+
+	data->old_opp.rate = old_freq;
+	size = sizeof(*opp->supplies) * opp_table->regulator_count;
+	if (IS_ERR(old_opp))
+		memset(data->old_opp.supplies, 0, size);
+	else
+		memcpy(data->old_opp.supplies, old_opp->supplies, size);
 
-	return 0;
+	data->new_opp.rate = freq;
+	memcpy(data->new_opp.supplies, opp->supplies, size);
 
-restore_freq:
-	if (clk_set_rate(clk, old_freq))
-		dev_err(dev, "%s: failed to restore old-freq (%lu Hz)\n",
-			__func__, old_freq);
-restore_voltage:
-	/* This shouldn't harm even if the voltages weren't updated earlier */
-	if (!IS_ERR(old_opp))
-		_set_opp_voltage(dev, reg, old_opp->u_volt,
-				 old_opp->u_volt_min, old_opp->u_volt_max);
+	rcu_read_unlock();
 
-	return ret;
+	return set_opp(data);
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_set_rate);
 
@@ -764,9 +858,6 @@ static struct opp_table *_add_opp_table(struct device *dev)
 
 	_of_init_opp_table(opp_table, dev);
 
-	/* Set regulator to a non-NULL error value */
-	opp_table->regulator = ERR_PTR(-ENXIO);
-
 	/* Find clk for the device */
 	opp_table->clk = clk_get(dev, NULL);
 	if (IS_ERR(opp_table->clk)) {
@@ -815,7 +906,10 @@ static void _remove_opp_table(struct opp_table *opp_table)
 	if (opp_table->prop_name)
 		return;
 
-	if (!IS_ERR(opp_table->regulator))
+	if (opp_table->regulators)
+		return;
+
+	if (opp_table->set_opp)
 		return;
 
 	/* Release clk */
@@ -924,34 +1018,50 @@ struct dev_pm_opp *_allocate_opp(struct device *dev,
 				 struct opp_table **opp_table)
 {
 	struct dev_pm_opp *opp;
+	int count, supply_size;
+	struct opp_table *table;
 
-	/* allocate new OPP node */
-	opp = kzalloc(sizeof(*opp), GFP_KERNEL);
-	if (!opp)
+	table = _add_opp_table(dev);
+	if (!table)
 		return NULL;
 
-	INIT_LIST_HEAD(&opp->node);
+	/* Allocate space for at least one supply */
+	count = table->regulator_count ? table->regulator_count : 1;
+	supply_size = sizeof(*opp->supplies) * count;
 
-	*opp_table = _add_opp_table(dev);
-	if (!*opp_table) {
-		kfree(opp);
+	/* allocate new OPP node and supplies structures */
+	opp = kzalloc(sizeof(*opp) + supply_size, GFP_KERNEL);
+	if (!opp) {
+		kfree(table);
 		return NULL;
 	}
 
+	/* Put the supplies at the end of the OPP structure as an empty array */
+	opp->supplies = (struct dev_pm_opp_supply *)(opp + 1);
+	INIT_LIST_HEAD(&opp->node);
+
+	*opp_table = table;
+
 	return opp;
 }
 
 static bool _opp_supported_by_regulators(struct dev_pm_opp *opp,
 					 struct opp_table *opp_table)
 {
-	struct regulator *reg = opp_table->regulator;
-
-	if (!IS_ERR(reg) &&
-	    !regulator_is_supported_voltage(reg, opp->u_volt_min,
-					    opp->u_volt_max)) {
-		pr_warn("%s: OPP minuV: %lu maxuV: %lu, not supported by regulator\n",
-			__func__, opp->u_volt_min, opp->u_volt_max);
-		return false;
+	struct regulator *reg;
+	int i;
+
+	for (i = 0; i < opp_table->regulator_count; i++) {
+		reg = opp_table->regulators[i];
+
+		if (!regulator_is_supported_voltage(reg,
+					opp->supplies[i].u_volt_min,
+					opp->supplies[i].u_volt_max)) {
+			pr_warn("%s: OPP minuV: %lu maxuV: %lu, not supported by regulator\n",
+				__func__, opp->supplies[i].u_volt_min,
+				opp->supplies[i].u_volt_max);
+			return false;
+		}
 	}
 
 	return true;
@@ -983,11 +1093,13 @@ int _opp_add(struct device *dev, struct dev_pm_opp *new_opp,
 
 		/* Duplicate OPPs */
 		dev_warn(dev, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n",
-			 __func__, opp->rate, opp->u_volt, opp->available,
-			 new_opp->rate, new_opp->u_volt, new_opp->available);
+			 __func__, opp->rate, opp->supplies[0].u_volt,
+			 opp->available, new_opp->rate,
+			 new_opp->supplies[0].u_volt, new_opp->available);
 
-		return opp->available && new_opp->u_volt == opp->u_volt ?
-			0 : -EEXIST;
+		/* Should we compare voltages for all regulators here ? */
+		return opp->available &&
+		       new_opp->supplies[0].u_volt == opp->supplies[0].u_volt ? 0 : -EEXIST;
 	}
 
 	new_opp->opp_table = opp_table;
@@ -1054,9 +1166,9 @@ int _opp_add_v1(struct device *dev, unsigned long freq, long u_volt,
 	/* populate the opp table */
 	new_opp->rate = freq;
 	tol = u_volt * opp_table->voltage_tolerance_v1 / 100;
-	new_opp->u_volt = u_volt;
-	new_opp->u_volt_min = u_volt - tol;
-	new_opp->u_volt_max = u_volt + tol;
+	new_opp->supplies[0].u_volt = u_volt;
+	new_opp->supplies[0].u_volt_min = u_volt - tol;
+	new_opp->supplies[0].u_volt_max = u_volt + tol;
 	new_opp->available = true;
 	new_opp->dynamic = dynamic;
 
@@ -1300,13 +1412,47 @@ unlock:
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_put_prop_name);
 
+static int _allocate_set_opp_data(struct opp_table *opp_table)
+{
+	struct dev_pm_set_opp_data *data;
+	int len, count = opp_table->regulator_count;
+
+	if (WARN_ON(!count))
+		return -EINVAL;
+
+	/* space for set_opp_data */
+	len = sizeof(*data);
+
+	/* space for old_opp.supplies and new_opp.supplies */
+	len += 2 * sizeof(struct dev_pm_opp_supply) * count;
+
+	data = kzalloc(len, GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->old_opp.supplies = (void *)(data + 1);
+	data->new_opp.supplies = data->old_opp.supplies + count;
+
+	opp_table->set_opp_data = data;
+
+	return 0;
+}
+
+static void _free_set_opp_data(struct opp_table *opp_table)
+{
+	kfree(opp_table->set_opp_data);
+	opp_table->set_opp_data = NULL;
+}
+
 /**
- * dev_pm_opp_set_regulator() - Set regulator name for the device
+ * dev_pm_opp_set_regulators() - Set regulator names for the device
  * @dev: Device for which regulator name is being set.
- * @name: Name of the regulator.
+ * @names: Array of pointers to the names of the regulator.
+ * @count: Number of regulators.
  *
  * In order to support OPP switching, OPP layer needs to know the name of the
- * device's regulator, as the core would be required to switch voltages as well.
+ * device's regulators, as the core would be required to switch voltages as
+ * well.
  *
  * This must be called before any OPPs are initialized for the device.
  *
@@ -1316,11 +1462,13 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_put_prop_name);
  * that this function is *NOT* called under RCU protection or in contexts where
  * mutex cannot be locked.
  */
-int dev_pm_opp_set_regulator(struct device *dev, const char *name)
+struct opp_table *dev_pm_opp_set_regulators(struct device *dev,
+					    const char * const names[],
+					    unsigned int count)
 {
 	struct opp_table *opp_table;
 	struct regulator *reg;
-	int ret;
+	int ret, i;
 
 	mutex_lock(&opp_table_lock);
 
@@ -1336,22 +1484,146 @@ int dev_pm_opp_set_regulator(struct device *dev, const char *name)
 		goto err;
 	}
 
-	/* Already have a regulator set */
-	if (WARN_ON(!IS_ERR(opp_table->regulator))) {
+	/* Already have regulators set */
+	if (opp_table->regulators) {
 		ret = -EBUSY;
 		goto err;
 	}
-	/* Allocate the regulator */
-	reg = regulator_get_optional(dev, name);
-	if (IS_ERR(reg)) {
-		ret = PTR_ERR(reg);
-		if (ret != -EPROBE_DEFER)
-			dev_err(dev, "%s: no regulator (%s) found: %d\n",
-				__func__, name, ret);
+
+	opp_table->regulators = kmalloc_array(count,
+					      sizeof(*opp_table->regulators),
+					      GFP_KERNEL);
+	if (!opp_table->regulators) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	for (i = 0; i < count; i++) {
+		reg = regulator_get_optional(dev, names[i]);
+		if (IS_ERR(reg)) {
+			ret = PTR_ERR(reg);
+			if (ret != -EPROBE_DEFER)
+				dev_err(dev, "%s: no regulator (%s) found: %d\n",
+					__func__, names[i], ret);
+			goto free_regulators;
+		}
+
+		opp_table->regulators[i] = reg;
+	}
+
+	opp_table->regulator_count = count;
+
+	/* Allocate block only once to pass to set_opp() routines */
+	ret = _allocate_set_opp_data(opp_table);
+	if (ret)
+		goto free_regulators;
+
+	mutex_unlock(&opp_table_lock);
+	return opp_table;
+
+free_regulators:
+	while (i != 0)
+		regulator_put(opp_table->regulators[--i]);
+
+	kfree(opp_table->regulators);
+	opp_table->regulators = NULL;
+	opp_table->regulator_count = 0;
+err:
+	_remove_opp_table(opp_table);
+unlock:
+	mutex_unlock(&opp_table_lock);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(dev_pm_opp_set_regulators);
+
+/**
+ * dev_pm_opp_put_regulators() - Releases resources blocked for regulator
+ * @opp_table: OPP table returned from dev_pm_opp_set_regulators().
+ *
+ * Locking: The internal opp_table and opp structures are RCU protected.
+ * Hence this function internally uses RCU updater strategy with mutex locks
+ * to keep the integrity of the internal data structures. Callers should ensure
+ * that this function is *NOT* called under RCU protection or in contexts where
+ * mutex cannot be locked.
+ */
+void dev_pm_opp_put_regulators(struct opp_table *opp_table)
+{
+	int i;
+
+	mutex_lock(&opp_table_lock);
+
+	if (!opp_table->regulators) {
+		pr_err("%s: Doesn't have regulators set\n", __func__);
+		goto unlock;
+	}
+
+	/* Make sure there are no concurrent readers while updating opp_table */
+	WARN_ON(!list_empty(&opp_table->opp_list));
+
+	for (i = opp_table->regulator_count - 1; i >= 0; i--)
+		regulator_put(opp_table->regulators[i]);
+
+	_free_set_opp_data(opp_table);
+
+	kfree(opp_table->regulators);
+	opp_table->regulators = NULL;
+	opp_table->regulator_count = 0;
+
+	/* Try freeing opp_table if this was the last blocking resource */
+	_remove_opp_table(opp_table);
+
+unlock:
+	mutex_unlock(&opp_table_lock);
+}
+EXPORT_SYMBOL_GPL(dev_pm_opp_put_regulators);
+
+/**
+ * dev_pm_opp_register_set_opp_helper() - Register custom set OPP helper
+ * @dev: Device for which the helper is getting registered.
+ * @set_opp: Custom set OPP helper.
+ *
+ * This is useful to support complex platforms (like platforms with multiple
+ * regulators per device), instead of the generic OPP set rate helper.
+ *
+ * This must be called before any OPPs are initialized for the device.
+ *
+ * Locking: The internal opp_table and opp structures are RCU protected.
+ * Hence this function internally uses RCU updater strategy with mutex locks
+ * to keep the integrity of the internal data structures. Callers should ensure
+ * that this function is *NOT* called under RCU protection or in contexts where
+ * mutex cannot be locked.
+ */
+int dev_pm_opp_register_set_opp_helper(struct device *dev,
+			int (*set_opp)(struct dev_pm_set_opp_data *data))
+{
+	struct opp_table *opp_table;
+	int ret;
+
+	if (!set_opp)
+		return -EINVAL;
+
+	mutex_lock(&opp_table_lock);
+
+	opp_table = _add_opp_table(dev);
+	if (!opp_table) {
+		ret = -ENOMEM;
+		goto unlock;
+	}
+
+	/* This should be called before OPPs are initialized */
+	if (WARN_ON(!list_empty(&opp_table->opp_list))) {
+		ret = -EBUSY;
 		goto err;
 	}
 
-	opp_table->regulator = reg;
+	/* Already have custom set_opp helper */
+	if (WARN_ON(opp_table->set_opp)) {
+		ret = -EBUSY;
+		goto err;
+	}
+
+	opp_table->set_opp = set_opp;
 
 	mutex_unlock(&opp_table_lock);
 	return 0;
@@ -1363,11 +1635,12 @@ unlock:
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(dev_pm_opp_set_regulator);
+EXPORT_SYMBOL_GPL(dev_pm_opp_register_set_opp_helper);
 
 /**
- * dev_pm_opp_put_regulator() - Releases resources blocked for regulator
- * @dev: Device for which regulator was set.
+ * dev_pm_opp_register_put_opp_helper() - Releases resources blocked for
+ *					   set_opp helper
+ * @dev: Device for which custom set_opp helper has to be cleared.
  *
  * Locking: The internal opp_table and opp structures are RCU protected.
  * Hence this function internally uses RCU updater strategy with mutex locks
@@ -1375,7 +1648,7 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_set_regulator);
  * that this function is *NOT* called under RCU protection or in contexts where
  * mutex cannot be locked.
  */
-void dev_pm_opp_put_regulator(struct device *dev)
+void dev_pm_opp_register_put_opp_helper(struct device *dev)
 {
 	struct opp_table *opp_table;
 
@@ -1389,16 +1662,16 @@ void dev_pm_opp_put_regulator(struct device *dev)
 		goto unlock;
 	}
 
-	if (IS_ERR(opp_table->regulator)) {
-		dev_err(dev, "%s: Doesn't have regulator set\n", __func__);
+	if (!opp_table->set_opp) {
+		dev_err(dev, "%s: Doesn't have custom set_opp helper set\n",
+			__func__);
 		goto unlock;
 	}
 
 	/* Make sure there are no concurrent readers while updating opp_table */
 	WARN_ON(!list_empty(&opp_table->opp_list));
 
-	regulator_put(opp_table->regulator);
-	opp_table->regulator = ERR_PTR(-ENXIO);
+	opp_table->set_opp = NULL;
 
 	/* Try freeing opp_table if this was the last blocking resource */
 	_remove_opp_table(opp_table);
@@ -1406,7 +1679,7 @@ void dev_pm_opp_put_regulator(struct device *dev)
 unlock:
 	mutex_unlock(&opp_table_lock);
 }
-EXPORT_SYMBOL_GPL(dev_pm_opp_put_regulator);
+EXPORT_SYMBOL_GPL(dev_pm_opp_register_put_opp_helper);
 
 /**
  * dev_pm_opp_add()  - Add an OPP table from a table definitions
diff --git a/drivers/base/power/opp/debugfs.c b/drivers/base/power/opp/debugfs.c
index ef1ae6b52042..95f433db4ac7 100644
--- a/drivers/base/power/opp/debugfs.c
+++ b/drivers/base/power/opp/debugfs.c
@@ -15,6 +15,7 @@
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/limits.h>
+#include <linux/slab.h>
 
 #include "opp.h"
 
@@ -34,6 +35,46 @@ void opp_debug_remove_one(struct dev_pm_opp *opp)
 	debugfs_remove_recursive(opp->dentry);
 }
 
+static bool opp_debug_create_supplies(struct dev_pm_opp *opp,
+				      struct opp_table *opp_table,
+				      struct dentry *pdentry)
+{
+	struct dentry *d;
+	int i = 0;
+	char *name;
+
+	/* Always create at least supply-0 directory */
+	do {
+		name = kasprintf(GFP_KERNEL, "supply-%d", i);
+
+		/* Create per-opp directory */
+		d = debugfs_create_dir(name, pdentry);
+
+		kfree(name);
+
+		if (!d)
+			return false;
+
+		if (!debugfs_create_ulong("u_volt_target", S_IRUGO, d,
+					  &opp->supplies[i].u_volt))
+			return false;
+
+		if (!debugfs_create_ulong("u_volt_min", S_IRUGO, d,
+					  &opp->supplies[i].u_volt_min))
+			return false;
+
+		if (!debugfs_create_ulong("u_volt_max", S_IRUGO, d,
+					  &opp->supplies[i].u_volt_max))
+			return false;
+
+		if (!debugfs_create_ulong("u_amp", S_IRUGO, d,
+					  &opp->supplies[i].u_amp))
+			return false;
+	} while (++i < opp_table->regulator_count);
+
+	return true;
+}
+
 int opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table)
 {
 	struct dentry *pdentry = opp_table->dentry;
@@ -63,16 +104,7 @@ int opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table)
 	if (!debugfs_create_ulong("rate_hz", S_IRUGO, d, &opp->rate))
 		return -ENOMEM;
 
-	if (!debugfs_create_ulong("u_volt_target", S_IRUGO, d, &opp->u_volt))
-		return -ENOMEM;
-
-	if (!debugfs_create_ulong("u_volt_min", S_IRUGO, d, &opp->u_volt_min))
-		return -ENOMEM;
-
-	if (!debugfs_create_ulong("u_volt_max", S_IRUGO, d, &opp->u_volt_max))
-		return -ENOMEM;
-
-	if (!debugfs_create_ulong("u_amp", S_IRUGO, d, &opp->u_amp))
+	if (!opp_debug_create_supplies(opp, opp_table, d))
 		return -ENOMEM;
 
 	if (!debugfs_create_ulong("clock_latency_ns", S_IRUGO, d,
diff --git a/drivers/base/power/opp/of.c b/drivers/base/power/opp/of.c
index 5552211e6fcd..3f7d2591b173 100644
--- a/drivers/base/power/opp/of.c
+++ b/drivers/base/power/opp/of.c
@@ -17,6 +17,7 @@
 #include <linux/errno.h>
 #include <linux/device.h>
 #include <linux/of.h>
+#include <linux/slab.h>
 #include <linux/export.h>
 
 #include "opp.h"
@@ -101,16 +102,16 @@ static bool _opp_is_supported(struct device *dev, struct opp_table *opp_table,
 	return true;
 }
 
-/* TODO: Support multiple regulators */
 static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev,
 			      struct opp_table *opp_table)
 {
-	u32 microvolt[3] = {0};
-	u32 val;
-	int count, ret;
+	u32 *microvolt, *microamp = NULL;
+	int supplies, vcount, icount, ret, i, j;
 	struct property *prop = NULL;
 	char name[NAME_MAX];
 
+	supplies = opp_table->regulator_count ? opp_table->regulator_count : 1;
+
 	/* Search for "opp-microvolt-<name>" */
 	if (opp_table->prop_name) {
 		snprintf(name, sizeof(name), "opp-microvolt-%s",
@@ -128,34 +129,29 @@ static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev,
 			return 0;
 	}
 
-	count = of_property_count_u32_elems(opp->np, name);
-	if (count < 0) {
+	vcount = of_property_count_u32_elems(opp->np, name);
+	if (vcount < 0) {
 		dev_err(dev, "%s: Invalid %s property (%d)\n",
-			__func__, name, count);
-		return count;
+			__func__, name, vcount);
+		return vcount;
 	}
 
-	/* There can be one or three elements here */
-	if (count != 1 && count != 3) {
-		dev_err(dev, "%s: Invalid number of elements in %s property (%d)\n",
-			__func__, name, count);
+	/* There can be one or three elements per supply */
+	if (vcount != supplies && vcount != supplies * 3) {
+		dev_err(dev, "%s: Invalid number of elements in %s property (%d) with supplies (%d)\n",
+			__func__, name, vcount, supplies);
 		return -EINVAL;
 	}
 
-	ret = of_property_read_u32_array(opp->np, name, microvolt, count);
+	microvolt = kmalloc_array(vcount, sizeof(*microvolt), GFP_KERNEL);
+	if (!microvolt)
+		return -ENOMEM;
+
+	ret = of_property_read_u32_array(opp->np, name, microvolt, vcount);
 	if (ret) {
 		dev_err(dev, "%s: error parsing %s: %d\n", __func__, name, ret);
-		return -EINVAL;
-	}
-
-	opp->u_volt = microvolt[0];
-
-	if (count == 1) {
-		opp->u_volt_min = opp->u_volt;
-		opp->u_volt_max = opp->u_volt;
-	} else {
-		opp->u_volt_min = microvolt[1];
-		opp->u_volt_max = microvolt[2];
+		ret = -EINVAL;
+		goto free_microvolt;
 	}
 
 	/* Search for "opp-microamp-<name>" */
@@ -172,10 +168,59 @@ static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev,
 		prop = of_find_property(opp->np, name, NULL);
 	}
 
-	if (prop && !of_property_read_u32(opp->np, name, &val))
-		opp->u_amp = val;
+	if (prop) {
+		icount = of_property_count_u32_elems(opp->np, name);
+		if (icount < 0) {
+			dev_err(dev, "%s: Invalid %s property (%d)\n", __func__,
+				name, icount);
+			ret = icount;
+			goto free_microvolt;
+		}
 
-	return 0;
+		if (icount != supplies) {
+			dev_err(dev, "%s: Invalid number of elements in %s property (%d) with supplies (%d)\n",
+				__func__, name, icount, supplies);
+			ret = -EINVAL;
+			goto free_microvolt;
+		}
+
+		microamp = kmalloc_array(icount, sizeof(*microamp), GFP_KERNEL);
+		if (!microamp) {
+			ret = -EINVAL;
+			goto free_microvolt;
+		}
+
+		ret = of_property_read_u32_array(opp->np, name, microamp,
+						 icount);
+		if (ret) {
+			dev_err(dev, "%s: error parsing %s: %d\n", __func__,
+				name, ret);
+			ret = -EINVAL;
+			goto free_microamp;
+		}
+	}
+
+	for (i = 0, j = 0; i < supplies; i++) {
+		opp->supplies[i].u_volt = microvolt[j++];
+
+		if (vcount == supplies) {
+			opp->supplies[i].u_volt_min = opp->supplies[i].u_volt;
+			opp->supplies[i].u_volt_max = opp->supplies[i].u_volt;
+		} else {
+			opp->supplies[i].u_volt_min = microvolt[j++];
+			opp->supplies[i].u_volt_max = microvolt[j++];
+		}
+
+		if (microamp)
+			opp->supplies[i].u_amp = microamp[i];
+	}
+
+free_microamp:
+	kfree(microamp);
+free_microvolt:
+	kfree(microvolt);
+
+	return ret;
 }
 
 /**
@@ -198,7 +243,7 @@ void dev_pm_opp_of_remove_table(struct device *dev)
 EXPORT_SYMBOL_GPL(dev_pm_opp_of_remove_table);
 
 /* Returns opp descriptor node for a device, caller must do of_node_put() */
-struct device_node *_of_get_opp_desc_node(struct device *dev)
+static struct device_node *_of_get_opp_desc_node(struct device *dev)
 {
 	/*
 	 * TODO: Support for multiple OPP tables.
@@ -303,9 +348,9 @@ static int _opp_add_static_v2(struct device *dev, struct device_node *np)
 	mutex_unlock(&opp_table_lock);
 
 	pr_debug("%s: turbo:%d rate:%lu uv:%lu uvmin:%lu uvmax:%lu latency:%lu\n",
-		 __func__, new_opp->turbo, new_opp->rate, new_opp->u_volt,
-		 new_opp->u_volt_min, new_opp->u_volt_max,
-		 new_opp->clock_latency_ns);
+		 __func__, new_opp->turbo, new_opp->rate,
+		 new_opp->supplies[0].u_volt, new_opp->supplies[0].u_volt_min,
+		 new_opp->supplies[0].u_volt_max, new_opp->clock_latency_ns);
 
 	/*
 	 * Notify the changes in the availability of the operable
@@ -562,7 +607,7 @@ int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev,
 	/* Get OPP descriptor node */
 	np = _of_get_opp_desc_node(cpu_dev);
 	if (!np) {
-		dev_dbg(cpu_dev, "%s: Couldn't find cpu_dev node.\n", __func__);
+		dev_dbg(cpu_dev, "%s: Couldn't find opp node.\n", __func__);
 		return -ENOENT;
 	}
 
@@ -587,7 +632,7 @@ int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev,
 		/* Get OPP descriptor node */
 		tmp_np = _of_get_opp_desc_node(tcpu_dev);
 		if (!tmp_np) {
-			dev_err(tcpu_dev, "%s: Couldn't find tcpu_dev node.\n",
+			dev_err(tcpu_dev, "%s: Couldn't find opp node.\n",
 				__func__);
 			ret = -ENOENT;
 			goto put_cpu_node;
diff --git a/drivers/base/power/opp/opp.h b/drivers/base/power/opp/opp.h
index fabd5ca1a083..af9f2b849a66 100644
--- a/drivers/base/power/opp/opp.h
+++ b/drivers/base/power/opp/opp.h
@@ -61,10 +61,7 @@ extern struct list_head opp_tables;
  * @turbo:	true if turbo (boost) OPP
  * @suspend:	true if suspend OPP
  * @rate:	Frequency in hertz
- * @u_volt:	Target voltage in microvolts corresponding to this OPP
- * @u_volt_min:	Minimum voltage in microvolts corresponding to this OPP
- * @u_volt_max:	Maximum voltage in microvolts corresponding to this OPP
- * @u_amp:	Maximum current drawn by the device in microamperes
+ * @supplies:	Power supplies voltage/current values
  * @clock_latency_ns: Latency (in nanoseconds) of switching to this OPP's
  *		frequency from any other OPP's frequency.
  * @opp_table:	points back to the opp_table struct this opp belongs to
@@ -83,10 +80,8 @@ struct dev_pm_opp {
 	bool suspend;
 	unsigned long rate;
 
-	unsigned long u_volt;
-	unsigned long u_volt_min;
-	unsigned long u_volt_max;
-	unsigned long u_amp;
+	struct dev_pm_opp_supply *supplies;
+
 	unsigned long clock_latency_ns;
 
 	struct opp_table *opp_table;
@@ -144,7 +139,10 @@ enum opp_table_access {
  * @supported_hw_count: Number of elements in supported_hw array.
  * @prop_name: A name to postfix to many DT properties, while parsing them.
  * @clk: Device's clock handle
- * @regulator: Supply regulator
+ * @regulators: Supply regulators
+ * @regulator_count: Number of power supply regulators
+ * @set_opp: Platform specific set_opp callback
+ * @set_opp_data: Data to be passed to set_opp callback
  * @dentry:	debugfs dentry pointer of the real device directory (not links).
  * @dentry_name: Name of the real dentry.
  *
@@ -179,7 +177,11 @@ struct opp_table {
 	unsigned int supported_hw_count;
 	const char *prop_name;
 	struct clk *clk;
-	struct regulator *regulator;
+	struct regulator **regulators;
+	unsigned int regulator_count;
+
+	int (*set_opp)(struct dev_pm_set_opp_data *data);
+	struct dev_pm_set_opp_data *set_opp_data;
 
 #ifdef CONFIG_DEBUG_FS
 	struct dentry *dentry;
@@ -190,7 +192,6 @@ struct opp_table {
 /* Routines internal to opp core */
 struct opp_table *_find_opp_table(struct device *dev);
 struct opp_device *_add_opp_dev(const struct device *dev, struct opp_table *opp_table);
-struct device_node *_of_get_opp_desc_node(struct device *dev);
 void _dev_pm_opp_remove_table(struct device *dev, bool remove_all);
 struct dev_pm_opp *_allocate_opp(struct device *dev, struct opp_table **opp_table);
 int _opp_add(struct device *dev, struct dev_pm_opp *new_opp, struct opp_table *opp_table);
diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h
index 50e30e7b059d..a46e97e515c5 100644
--- a/drivers/base/power/power.h
+++ b/drivers/base/power/power.h
@@ -21,14 +21,22 @@ extern void pm_runtime_init(struct device *dev);
 extern void pm_runtime_reinit(struct device *dev);
 extern void pm_runtime_remove(struct device *dev);
 
+#define WAKE_IRQ_DEDICATED_ALLOCATED	BIT(0)
+#define WAKE_IRQ_DEDICATED_MANAGED	BIT(1)
+#define WAKE_IRQ_DEDICATED_MASK		(WAKE_IRQ_DEDICATED_ALLOCATED | \
+					 WAKE_IRQ_DEDICATED_MANAGED)
+
 struct wake_irq {
 	struct device *dev;
+	unsigned int status;
 	int irq;
-	bool dedicated_irq:1;
 };
 
 extern void dev_pm_arm_wake_irq(struct wake_irq *wirq);
 extern void dev_pm_disarm_wake_irq(struct wake_irq *wirq);
+extern void dev_pm_enable_wake_irq_check(struct device *dev,
+					 bool can_change_status);
+extern void dev_pm_disable_wake_irq_check(struct device *dev);
 
 #ifdef CONFIG_PM_SLEEP
 
@@ -104,6 +112,15 @@ static inline void dev_pm_disarm_wake_irq(struct wake_irq *wirq)
 {
 }
 
+static inline void dev_pm_enable_wake_irq_check(struct device *dev,
+						bool can_change_status)
+{
+}
+
+static inline void dev_pm_disable_wake_irq_check(struct device *dev)
+{
+}
+
 #endif
 
 #ifdef CONFIG_PM_SLEEP
@@ -127,6 +144,11 @@ extern void device_pm_move_after(struct device *, struct device *);
 extern void device_pm_move_last(struct device *);
 extern void device_pm_check_callbacks(struct device *dev);
 
+static inline bool device_pm_initialized(struct device *dev)
+{
+	return dev->power.in_dpm_list;
+}
+
 #else /* !CONFIG_PM_SLEEP */
 
 static inline void device_pm_sleep_init(struct device *dev) {}
@@ -146,6 +168,11 @@ static inline void device_pm_move_last(struct device *dev) {}
 
 static inline void device_pm_check_callbacks(struct device *dev) {}
 
+static inline bool device_pm_initialized(struct device *dev)
+{
+	return device_is_registered(dev);
+}
+
 #endif /* !CONFIG_PM_SLEEP */
 
 static inline void device_pm_init(struct device *dev)
diff --git a/drivers/base/power/qos.c b/drivers/base/power/qos.c
index 7f3646e459cb..58fcc758334e 100644
--- a/drivers/base/power/qos.c
+++ b/drivers/base/power/qos.c
@@ -856,7 +856,10 @@ int dev_pm_qos_update_user_latency_tolerance(struct device *dev, s32 val)
 		struct dev_pm_qos_request *req;
 
 		if (val < 0) {
-			ret = -EINVAL;
+			if (val == PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT)
+				ret = 0;
+			else
+				ret = -EINVAL;
 			goto out;
 		}
 		req = kzalloc(sizeof(*req), GFP_KERNEL);
@@ -883,6 +886,7 @@ int dev_pm_qos_update_user_latency_tolerance(struct device *dev, s32 val)
 	mutex_unlock(&dev_pm_qos_mtx);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(dev_pm_qos_update_user_latency_tolerance);
 
 /**
  * dev_pm_qos_expose_latency_tolerance - Expose latency tolerance to userspace
diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c
index 82a081ea4317..872eac4cb1df 100644
--- a/drivers/base/power/runtime.c
+++ b/drivers/base/power/runtime.c
@@ -12,6 +12,8 @@
 #include <linux/pm_runtime.h>
 #include <linux/pm_wakeirq.h>
 #include <trace/events/rpm.h>
+
+#include "../base.h"
 #include "power.h"
 
 typedef int (*pm_callback_t)(struct device *);
@@ -241,7 +243,8 @@ static int rpm_check_suspend_allowed(struct device *dev)
 		retval = -EACCES;
 	else if (atomic_read(&dev->power.usage_count) > 0)
 		retval = -EAGAIN;
-	else if (!pm_children_suspended(dev))
+	else if (!dev->power.ignore_children &&
+			atomic_read(&dev->power.child_count))
 		retval = -EBUSY;
 
 	/* Pending resume requests take precedence over suspends. */
@@ -258,6 +261,42 @@ static int rpm_check_suspend_allowed(struct device *dev)
 	return retval;
 }
 
+static int rpm_get_suppliers(struct device *dev)
+{
+	struct device_link *link;
+
+	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) {
+		int retval;
+
+		if (!(link->flags & DL_FLAG_PM_RUNTIME))
+			continue;
+
+		if (READ_ONCE(link->status) == DL_STATE_SUPPLIER_UNBIND ||
+		    link->rpm_active)
+			continue;
+
+		retval = pm_runtime_get_sync(link->supplier);
+		if (retval < 0) {
+			pm_runtime_put_noidle(link->supplier);
+			return retval;
+		}
+		link->rpm_active = true;
+	}
+	return 0;
+}
+
+static void rpm_put_suppliers(struct device *dev)
+{
+	struct device_link *link;
+
+	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
+		if (link->rpm_active &&
+		    READ_ONCE(link->status) != DL_STATE_SUPPLIER_UNBIND) {
+			pm_runtime_put(link->supplier);
+			link->rpm_active = false;
+		}
+}
+
 /**
  * __rpm_callback - Run a given runtime PM callback for a given device.
  * @cb: Runtime PM callback to run.
@@ -266,19 +305,57 @@ static int rpm_check_suspend_allowed(struct device *dev)
 static int __rpm_callback(int (*cb)(struct device *), struct device *dev)
 	__releases(&dev->power.lock) __acquires(&dev->power.lock)
 {
-	int retval;
+	int retval, idx;
+	bool use_links = dev->power.links_count > 0;
 
-	if (dev->power.irq_safe)
+	if (dev->power.irq_safe) {
 		spin_unlock(&dev->power.lock);
-	else
+	} else {
 		spin_unlock_irq(&dev->power.lock);
 
+		/*
+		 * Resume suppliers if necessary.
+		 *
+		 * The device's runtime PM status cannot change until this
+		 * routine returns, so it is safe to read the status outside of
+		 * the lock.
+		 */
+		if (use_links && dev->power.runtime_status == RPM_RESUMING) {
+			idx = device_links_read_lock();
+
+			retval = rpm_get_suppliers(dev);
+			if (retval)
+				goto fail;
+
+			device_links_read_unlock(idx);
+		}
+	}
+
 	retval = cb(dev);
 
-	if (dev->power.irq_safe)
+	if (dev->power.irq_safe) {
 		spin_lock(&dev->power.lock);
-	else
+	} else {
+		/*
+		 * If the device is suspending and the callback has returned
+		 * success, drop the usage counters of the suppliers that have
+		 * been reference counted on its resume.
+		 *
+		 * Do that if resume fails too.
+		 */
+		if (use_links
+		    && ((dev->power.runtime_status == RPM_SUSPENDING && !retval)
+		    || (dev->power.runtime_status == RPM_RESUMING && retval))) {
+			idx = device_links_read_lock();
+
+ fail:
+			rpm_put_suppliers(dev);
+
+			device_links_read_unlock(idx);
+		}
+
 		spin_lock_irq(&dev->power.lock);
+	}
 
 	return retval;
 }
@@ -515,7 +592,7 @@ static int rpm_suspend(struct device *dev, int rpmflags)
 
 	callback = RPM_GET_CALLBACK(dev, runtime_suspend);
 
-	dev_pm_enable_wake_irq(dev);
+	dev_pm_enable_wake_irq_check(dev, true);
 	retval = rpm_callback(callback, dev);
 	if (retval)
 		goto fail;
@@ -554,7 +631,7 @@ static int rpm_suspend(struct device *dev, int rpmflags)
 	return retval;
 
  fail:
-	dev_pm_disable_wake_irq(dev);
+	dev_pm_disable_wake_irq_check(dev);
 	__update_runtime_status(dev, RPM_ACTIVE);
 	dev->power.deferred_resume = false;
 	wake_up_all(&dev->power.wait_queue);
@@ -712,8 +789,8 @@ static int rpm_resume(struct device *dev, int rpmflags)
 
 		spin_lock(&parent->power.lock);
 		/*
-		 * We can resume if the parent's runtime PM is disabled or it
-		 * is set to ignore children.
+		 * Resume the parent if it has runtime PM enabled and not been
+		 * set to ignore its children.
 		 */
 		if (!parent->power.disable_depth
 		    && !parent->power.ignore_children) {
@@ -737,12 +814,12 @@ static int rpm_resume(struct device *dev, int rpmflags)
 
 	callback = RPM_GET_CALLBACK(dev, runtime_resume);
 
-	dev_pm_disable_wake_irq(dev);
+	dev_pm_disable_wake_irq_check(dev);
 	retval = rpm_callback(callback, dev);
 	if (retval) {
 		__update_runtime_status(dev, RPM_SUSPENDED);
 		pm_runtime_cancel_pending(dev);
-		dev_pm_enable_wake_irq(dev);
+		dev_pm_enable_wake_irq_check(dev, false);
 	} else {
  no_callback:
 		__update_runtime_status(dev, RPM_ACTIVE);
@@ -1027,7 +1104,17 @@ int __pm_runtime_set_status(struct device *dev, unsigned int status)
 		goto out_set;
 
 	if (status == RPM_SUSPENDED) {
-		/* It always is possible to set the status to 'suspended'. */
+		/*
+		 * It is invalid to suspend a device with an active child,
+		 * unless it has been set to ignore its children.
+		 */
+		if (!dev->power.ignore_children &&
+			atomic_read(&dev->power.child_count)) {
+			dev_err(dev, "runtime PM trying to suspend device but active child\n");
+			error = -EBUSY;
+			goto out;
+		}
+
 		if (parent) {
 			atomic_add_unless(&parent->power.child_count, -1, 0);
 			notify_parent = !parent->power.ignore_children;
@@ -1447,6 +1534,94 @@ void pm_runtime_remove(struct device *dev)
 }
 
 /**
+ * pm_runtime_clean_up_links - Prepare links to consumers for driver removal.
+ * @dev: Device whose driver is going to be removed.
+ *
+ * Check links from this device to any consumers and if any of them have active
+ * runtime PM references to the device, drop the usage counter of the device
+ * (once per link).
+ *
+ * Links with the DL_FLAG_STATELESS flag set are ignored.
+ *
+ * Since the device is guaranteed to be runtime-active at the point this is
+ * called, nothing else needs to be done here.
+ *
+ * Moreover, this is called after device_links_busy() has returned 'false', so
+ * the status of each link is guaranteed to be DL_STATE_SUPPLIER_UNBIND and
+ * therefore rpm_active can't be manipulated concurrently.
+ */
+void pm_runtime_clean_up_links(struct device *dev)
+{
+	struct device_link *link;
+	int idx;
+
+	idx = device_links_read_lock();
+
+	list_for_each_entry_rcu(link, &dev->links.consumers, s_node) {
+		if (link->flags & DL_FLAG_STATELESS)
+			continue;
+
+		if (link->rpm_active) {
+			pm_runtime_put_noidle(dev);
+			link->rpm_active = false;
+		}
+	}
+
+	device_links_read_unlock(idx);
+}
+
+/**
+ * pm_runtime_get_suppliers - Resume and reference-count supplier devices.
+ * @dev: Consumer device.
+ */
+void pm_runtime_get_suppliers(struct device *dev)
+{
+	struct device_link *link;
+	int idx;
+
+	idx = device_links_read_lock();
+
+	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
+		if (link->flags & DL_FLAG_PM_RUNTIME)
+			pm_runtime_get_sync(link->supplier);
+
+	device_links_read_unlock(idx);
+}
+
+/**
+ * pm_runtime_put_suppliers - Drop references to supplier devices.
+ * @dev: Consumer device.
+ */
+void pm_runtime_put_suppliers(struct device *dev)
+{
+	struct device_link *link;
+	int idx;
+
+	idx = device_links_read_lock();
+
+	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
+		if (link->flags & DL_FLAG_PM_RUNTIME)
+			pm_runtime_put(link->supplier);
+
+	device_links_read_unlock(idx);
+}
+
+void pm_runtime_new_link(struct device *dev)
+{
+	spin_lock_irq(&dev->power.lock);
+	dev->power.links_count++;
+	spin_unlock_irq(&dev->power.lock);
+}
+
+void pm_runtime_drop_link(struct device *dev)
+{
+	spin_lock_irq(&dev->power.lock);
+	WARN_ON(dev->power.links_count == 0);
+	dev->power.links_count--;
+	spin_unlock_irq(&dev->power.lock);
+}
+
+/**
  * pm_runtime_force_suspend - Force a device into suspend state if needed.
  * @dev: Device to suspend.
  *
@@ -1478,6 +1653,16 @@ int pm_runtime_force_suspend(struct device *dev)
 	if (ret)
 		goto err;
 
+	/*
+	 * Increase the runtime PM usage count for the device's parent, in case
+	 * when we find the device being used when system suspend was invoked.
+	 * This informs pm_runtime_force_resume() to resume the parent
+	 * immediately, which is needed to be able to resume its children,
+	 * when not deferring the resume to be managed via runtime PM.
+	 */
+	if (dev->parent && atomic_read(&dev->power.usage_count) > 1)
+		pm_runtime_get_noresume(dev->parent);
+
 	pm_runtime_set_suspended(dev);
 	return 0;
 err:
@@ -1487,16 +1672,20 @@ err:
 EXPORT_SYMBOL_GPL(pm_runtime_force_suspend);
 
 /**
- * pm_runtime_force_resume - Force a device into resume state.
+ * pm_runtime_force_resume - Force a device into resume state if needed.
  * @dev: Device to resume.
  *
  * Prior invoking this function we expect the user to have brought the device
  * into low power state by a call to pm_runtime_force_suspend(). Here we reverse
- * those actions and brings the device into full power. We update the runtime PM
- * status and re-enables runtime PM.
+ * those actions and brings the device into full power, if it is expected to be
+ * used on system resume. To distinguish that, we check whether the runtime PM
+ * usage count is greater than 1 (the PM core increases the usage count in the
+ * system PM prepare phase), as that indicates a real user (such as a subsystem,
+ * driver, userspace, etc.) is using it. If that is the case, the device is
+ * expected to be used on system resume as well, so then we resume it. In the
+ * other case, we defer the resume to be managed via runtime PM.
  *
- * Typically this function may be invoked from a system resume callback to make
- * sure the device is put into full power state.
+ * Typically this function may be invoked from a system resume callback.
  */
 int pm_runtime_force_resume(struct device *dev)
 {
@@ -1513,6 +1702,17 @@ int pm_runtime_force_resume(struct device *dev)
 	if (!pm_runtime_status_suspended(dev))
 		goto out;
 
+	/*
+	 * Decrease the parent's runtime PM usage count, if we increased it
+	 * during system suspend in pm_runtime_force_suspend().
+	*/
+	if (atomic_read(&dev->power.usage_count) > 1) {
+		if (dev->parent)
+			pm_runtime_put_noidle(dev->parent);
+	} else {
+		goto out;
+	}
+
 	ret = pm_runtime_set_active(dev);
 	if (ret)
 		goto out;
diff --git a/drivers/base/power/sysfs.c b/drivers/base/power/sysfs.c
index a7b46798c81d..33b4b902741a 100644
--- a/drivers/base/power/sysfs.c
+++ b/drivers/base/power/sysfs.c
@@ -263,7 +263,11 @@ static ssize_t pm_qos_latency_tolerance_store(struct device *dev,
 	s32 value;
 	int ret;
 
-	if (kstrtos32(buf, 0, &value)) {
+	if (kstrtos32(buf, 0, &value) == 0) {
+		/* Users can't write negative values directly */
+		if (value < 0)
+			return -EINVAL;
+	} else {
 		if (!strcmp(buf, "auto") || !strcmp(buf, "auto\n"))
 			value = PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT;
 		else if (!strcmp(buf, "any") || !strcmp(buf, "any\n"))
diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c
index 0d77cd6fd8d1..404d94c6c8bc 100644
--- a/drivers/base/power/wakeirq.c
+++ b/drivers/base/power/wakeirq.c
@@ -110,8 +110,10 @@ void dev_pm_clear_wake_irq(struct device *dev)
 	dev->power.wakeirq = NULL;
 	spin_unlock_irqrestore(&dev->power.lock, flags);
 
-	if (wirq->dedicated_irq)
+	if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED) {
 		free_irq(wirq->irq, wirq);
+		wirq->status &= ~WAKE_IRQ_DEDICATED_MASK;
+	}
 	kfree(wirq);
 }
 EXPORT_SYMBOL_GPL(dev_pm_clear_wake_irq);
@@ -179,7 +181,6 @@ int dev_pm_set_dedicated_wake_irq(struct device *dev, int irq)
 
 	wirq->dev = dev;
 	wirq->irq = irq;
-	wirq->dedicated_irq = true;
 	irq_set_status_flags(irq, IRQ_NOAUTOEN);
 
 	/*
@@ -195,6 +196,8 @@ int dev_pm_set_dedicated_wake_irq(struct device *dev, int irq)
 	if (err)
 		goto err_free_irq;
 
+	wirq->status = WAKE_IRQ_DEDICATED_ALLOCATED;
+
 	return err;
 
 err_free_irq:
@@ -210,9 +213,9 @@ EXPORT_SYMBOL_GPL(dev_pm_set_dedicated_wake_irq);
  * dev_pm_enable_wake_irq - Enable device wake-up interrupt
  * @dev: Device
  *
- * Called from the bus code or the device driver for
- * runtime_suspend() to enable the wake-up interrupt while
- * the device is running.
+ * Optionally called from the bus code or the device driver for
+ * runtime_resume() to override the PM runtime core managed wake-up
+ * interrupt handling to enable the wake-up interrupt.
  *
  * Note that for runtime_suspend()) the wake-up interrupts
  * should be unconditionally enabled unlike for suspend()
@@ -222,7 +225,7 @@ void dev_pm_enable_wake_irq(struct device *dev)
 {
 	struct wake_irq *wirq = dev->power.wakeirq;
 
-	if (wirq && wirq->dedicated_irq)
+	if (wirq && (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED))
 		enable_irq(wirq->irq);
 }
 EXPORT_SYMBOL_GPL(dev_pm_enable_wake_irq);
@@ -231,20 +234,73 @@ EXPORT_SYMBOL_GPL(dev_pm_enable_wake_irq);
  * dev_pm_disable_wake_irq - Disable device wake-up interrupt
  * @dev: Device
  *
- * Called from the bus code or the device driver for
- * runtime_resume() to disable the wake-up interrupt while
- * the device is running.
+ * Optionally called from the bus code or the device driver for
+ * runtime_suspend() to override the PM runtime core managed wake-up
+ * interrupt handling to disable the wake-up interrupt.
  */
 void dev_pm_disable_wake_irq(struct device *dev)
 {
 	struct wake_irq *wirq = dev->power.wakeirq;
 
-	if (wirq && wirq->dedicated_irq)
+	if (wirq && (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED))
 		disable_irq_nosync(wirq->irq);
 }
 EXPORT_SYMBOL_GPL(dev_pm_disable_wake_irq);
 
 /**
+ * dev_pm_enable_wake_irq_check - Checks and enables wake-up interrupt
+ * @dev: Device
+ * @can_change_status: Can change wake-up interrupt status
+ *
+ * Enables wakeirq conditionally. We need to enable wake-up interrupt
+ * lazily on the first rpm_suspend(). This is needed as the consumer device
+ * starts in RPM_SUSPENDED state, and the the first pm_runtime_get() would
+ * otherwise try to disable already disabled wakeirq. The wake-up interrupt
+ * starts disabled with IRQ_NOAUTOEN set.
+ *
+ * Should be only called from rpm_suspend() and rpm_resume() path.
+ * Caller must hold &dev->power.lock to change wirq->status
+ */
+void dev_pm_enable_wake_irq_check(struct device *dev,
+				  bool can_change_status)
+{
+	struct wake_irq *wirq = dev->power.wakeirq;
+
+	if (!wirq || !((wirq->status & WAKE_IRQ_DEDICATED_MASK)))
+		return;
+
+	if (likely(wirq->status & WAKE_IRQ_DEDICATED_MANAGED)) {
+		goto enable;
+	} else if (can_change_status) {
+		wirq->status |= WAKE_IRQ_DEDICATED_MANAGED;
+		goto enable;
+	}
+
+	return;
+
+enable:
+	enable_irq(wirq->irq);
+}
+
+/**
+ * dev_pm_disable_wake_irq_check - Checks and disables wake-up interrupt
+ * @dev: Device
+ *
+ * Disables wake-up interrupt conditionally based on status.
+ * Should be only called from rpm_suspend() and rpm_resume() path.
+ */
+void dev_pm_disable_wake_irq_check(struct device *dev)
+{
+	struct wake_irq *wirq = dev->power.wakeirq;
+
+	if (!wirq || !((wirq->status & WAKE_IRQ_DEDICATED_MASK)))
+		return;
+
+	if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED)
+		disable_irq_nosync(wirq->irq);
+}
+
+/**
  * dev_pm_arm_wake_irq - Arm device wake-up
  * @wirq: Device wake-up interrupt
  *
diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c
index 62e4de2aa8d1..bf9ba26981a5 100644
--- a/drivers/base/power/wakeup.c
+++ b/drivers/base/power/wakeup.c
@@ -811,7 +811,7 @@ void pm_print_active_wakeup_sources(void)
 	rcu_read_lock();
 	list_for_each_entry_rcu(ws, &wakeup_sources, entry) {
 		if (ws->active) {
-			pr_info("active wakeup source: %s\n", ws->name);
+			pr_debug("active wakeup source: %s\n", ws->name);
 			active = 1;
 		} else if (!active &&
 			   (!last_activity_ws ||
@@ -822,7 +822,7 @@ void pm_print_active_wakeup_sources(void)
 	}
 
 	if (!active && last_activity_ws)
-		pr_info("last active wakeup source: %s\n",
+		pr_debug("last active wakeup source: %s\n",
 			last_activity_ws->name);
 	rcu_read_unlock();
 }
@@ -905,7 +905,7 @@ bool pm_get_wakeup_count(unsigned int *count, bool block)
 			split_counters(&cnt, &inpr);
 			if (inpr == 0 || signal_pending(current))
 				break;
-
+			pm_print_active_wakeup_sources();
 			schedule();
 		}
 		finish_wait(&wakeup_count_wait_queue, &wait);
diff --git a/drivers/base/regmap/regcache-lzo.c b/drivers/base/regmap/regcache-lzo.c
index 6f77d7319fc6..4ff311374c4a 100644
--- a/drivers/base/regmap/regcache-lzo.c
+++ b/drivers/base/regmap/regcache-lzo.c
@@ -236,15 +236,13 @@ static int regcache_lzo_read(struct regmap *map,
 {
 	struct regcache_lzo_ctx *lzo_block, **lzo_blocks;
 	int ret, blkindex, blkpos;
-	size_t blksize, tmp_dst_len;
+	size_t tmp_dst_len;
 	void *tmp_dst;
 
 	/* index of the compressed lzo block */
 	blkindex = regcache_lzo_get_blkindex(map, reg);
 	/* register index within the decompressed block */
 	blkpos = regcache_lzo_get_blkpos(map, reg);
-	/* size of the compressed block */
-	blksize = regcache_lzo_get_blksize(map);
 	lzo_blocks = map->cache;
 	lzo_block = lzo_blocks[blkindex];
 
@@ -275,15 +273,13 @@ static int regcache_lzo_write(struct regmap *map,
 {
 	struct regcache_lzo_ctx *lzo_block, **lzo_blocks;
 	int ret, blkindex, blkpos;
-	size_t blksize, tmp_dst_len;
+	size_t tmp_dst_len;
 	void *tmp_dst;
 
 	/* index of the compressed lzo block */
 	blkindex = regcache_lzo_get_blkindex(map, reg);
 	/* register index within the decompressed block */
 	blkpos = regcache_lzo_get_blkpos(map, reg);
-	/* size of the compressed block */
-	blksize = regcache_lzo_get_blksize(map);
 	lzo_blocks = map->cache;
 	lzo_block = lzo_blocks[blkindex];
 
diff --git a/drivers/base/soc.c b/drivers/base/soc.c
index b63f23e6ad61..dc26e5949a32 100644
--- a/drivers/base/soc.c
+++ b/drivers/base/soc.c
@@ -13,6 +13,7 @@
 #include <linux/spinlock.h>
 #include <linux/sys_soc.h>
 #include <linux/err.h>
+#include <linux/glob.h>
 
 static DEFINE_IDA(soc_ida);
 
@@ -113,6 +114,12 @@ struct soc_device *soc_device_register(struct soc_device_attribute *soc_dev_attr
 	struct soc_device *soc_dev;
 	int ret;
 
+	if (!soc_bus_type.p) {
+		ret = bus_register(&soc_bus_type);
+		if (ret)
+			goto out1;
+	}
+
 	soc_dev = kzalloc(sizeof(*soc_dev), GFP_KERNEL);
 	if (!soc_dev) {
 		ret = -ENOMEM;
@@ -156,6 +163,78 @@ void soc_device_unregister(struct soc_device *soc_dev)
 
 static int __init soc_bus_register(void)
 {
+	if (soc_bus_type.p)
+		return 0;
+
 	return bus_register(&soc_bus_type);
 }
 core_initcall(soc_bus_register);
+
+static int soc_device_match_one(struct device *dev, void *arg)
+{
+	struct soc_device *soc_dev = container_of(dev, struct soc_device, dev);
+	const struct soc_device_attribute *match = arg;
+
+	if (match->machine &&
+	    (!soc_dev->attr->machine ||
+	     !glob_match(match->machine, soc_dev->attr->machine)))
+		return 0;
+
+	if (match->family &&
+	    (!soc_dev->attr->family ||
+	     !glob_match(match->family, soc_dev->attr->family)))
+		return 0;
+
+	if (match->revision &&
+	    (!soc_dev->attr->revision ||
+	     !glob_match(match->revision, soc_dev->attr->revision)))
+		return 0;
+
+	if (match->soc_id &&
+	    (!soc_dev->attr->soc_id ||
+	     !glob_match(match->soc_id, soc_dev->attr->soc_id)))
+		return 0;
+
+	return 1;
+}
+
+/*
+ * soc_device_match - identify the SoC in the machine
+ * @matches: zero-terminated array of possible matches
+ *
+ * returns the first matching entry of the argument array, or NULL
+ * if none of them match.
+ *
+ * This function is meant as a helper in place of of_match_node()
+ * in cases where either no device tree is available or the information
+ * in a device node is insufficient to identify a particular variant
+ * by its compatible strings or other properties. For new devices,
+ * the DT binding should always provide unique compatible strings
+ * that allow the use of of_match_node() instead.
+ *
+ * The calling function can use the .data entry of the
+ * soc_device_attribute to pass a structure or function pointer for
+ * each entry.
+ */
+const struct soc_device_attribute *soc_device_match(
+	const struct soc_device_attribute *matches)
+{
+	int ret = 0;
+
+	if (!matches)
+		return NULL;
+
+	while (!ret) {
+		if (!(matches->machine || matches->family ||
+		      matches->revision || matches->soc_id))
+			break;
+		ret = bus_for_each_dev(&soc_bus_type, NULL, (void *)matches,
+				       soc_device_match_one);
+		if (!ret)
+			matches++;
+		else
+			return matches;
+	}
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(soc_device_match);
diff --git a/drivers/base/test/Kconfig b/drivers/base/test/Kconfig
new file mode 100644
index 000000000000..9aa0d45a60db
--- /dev/null
+++ b/drivers/base/test/Kconfig
@@ -0,0 +1,9 @@
+config TEST_ASYNC_DRIVER_PROBE
+	tristate "Build kernel module to test asynchronous driver probing"
+	depends on m
+	help
+	  Enabling this option produces a kernel module that allows
+	  testing asynchronous driver probing by the device core.
+	  The module name will be test_async_driver_probe.ko
+
+	  If unsure say N.
diff --git a/drivers/base/test/Makefile b/drivers/base/test/Makefile
new file mode 100644
index 000000000000..90477c5fd9f9
--- /dev/null
+++ b/drivers/base/test/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_TEST_ASYNC_DRIVER_PROBE)	+= test_async_driver_probe.o
diff --git a/drivers/base/test/test_async_driver_probe.c b/drivers/base/test/test_async_driver_probe.c
new file mode 100644
index 000000000000..304d5c2bd5e9
--- /dev/null
+++ b/drivers/base/test/test_async_driver_probe.c
@@ -0,0 +1,169 @@
+/*
+ * Copyright (C) 2014 Google, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/hrtimer.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/time.h>
+
+#define TEST_PROBE_DELAY	(5 * 1000)	/* 5 sec */
+#define TEST_PROBE_THRESHOLD	(TEST_PROBE_DELAY / 2)
+
+static int test_probe(struct platform_device *pdev)
+{
+	dev_info(&pdev->dev, "sleeping for %d msecs in probe\n",
+		 TEST_PROBE_DELAY);
+	msleep(TEST_PROBE_DELAY);
+	dev_info(&pdev->dev, "done sleeping\n");
+
+	return 0;
+}
+
+static struct platform_driver async_driver = {
+	.driver = {
+		.name = "test_async_driver",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+	},
+	.probe = test_probe,
+};
+
+static struct platform_driver sync_driver = {
+	.driver = {
+		.name = "test_sync_driver",
+		.probe_type = PROBE_FORCE_SYNCHRONOUS,
+	},
+	.probe = test_probe,
+};
+
+static struct platform_device *async_dev_1, *async_dev_2;
+static struct platform_device *sync_dev_1;
+
+static int __init test_async_probe_init(void)
+{
+	ktime_t calltime, delta;
+	unsigned long long duration;
+	int error;
+
+	pr_info("registering first asynchronous device...\n");
+
+	async_dev_1 = platform_device_register_simple("test_async_driver", 1,
+						      NULL, 0);
+	if (IS_ERR(async_dev_1)) {
+		error = PTR_ERR(async_dev_1);
+		pr_err("failed to create async_dev_1: %d", error);
+		return error;
+	}
+
+	pr_info("registering asynchronous driver...\n");
+	calltime = ktime_get();
+	error = platform_driver_register(&async_driver);
+	if (error) {
+		pr_err("Failed to register async_driver: %d\n", error);
+		goto err_unregister_async_dev_1;
+	}
+
+	delta = ktime_sub(ktime_get(), calltime);
+	duration = (unsigned long long) ktime_to_ms(delta);
+	pr_info("registration took %lld msecs\n", duration);
+	if (duration > TEST_PROBE_THRESHOLD) {
+		pr_err("test failed: probe took too long\n");
+		error = -ETIMEDOUT;
+		goto err_unregister_async_driver;
+	}
+
+	pr_info("registering second asynchronous device...\n");
+	calltime = ktime_get();
+	async_dev_2 = platform_device_register_simple("test_async_driver", 2,
+						      NULL, 0);
+	if (IS_ERR(async_dev_2)) {
+		error = PTR_ERR(async_dev_2);
+		pr_err("failed to create async_dev_2: %d", error);
+		goto err_unregister_async_driver;
+	}
+
+	delta = ktime_sub(ktime_get(), calltime);
+	duration = (unsigned long long) ktime_to_ms(delta);
+	pr_info("registration took %lld msecs\n", duration);
+	if (duration > TEST_PROBE_THRESHOLD) {
+		pr_err("test failed: probe took too long\n");
+		error = -ETIMEDOUT;
+		goto err_unregister_async_dev_2;
+	}
+
+	pr_info("registering synchronous driver...\n");
+
+	error = platform_driver_register(&sync_driver);
+	if (error) {
+		pr_err("Failed to register async_driver: %d\n", error);
+		goto err_unregister_async_dev_2;
+	}
+
+	pr_info("registering synchronous device...\n");
+	calltime = ktime_get();
+	sync_dev_1 = platform_device_register_simple("test_sync_driver", 1,
+						     NULL, 0);
+	if (IS_ERR(sync_dev_1)) {
+		error = PTR_ERR(sync_dev_1);
+		pr_err("failed to create sync_dev_1: %d", error);
+		goto err_unregister_sync_driver;
+	}
+
+	delta = ktime_sub(ktime_get(), calltime);
+	duration = (unsigned long long) ktime_to_ms(delta);
+	pr_info("registration took %lld msecs\n", duration);
+	if (duration < TEST_PROBE_THRESHOLD) {
+		pr_err("test failed: probe was too quick\n");
+		error = -ETIMEDOUT;
+		goto err_unregister_sync_dev_1;
+	}
+
+	pr_info("completed successfully");
+
+	return 0;
+
+err_unregister_sync_dev_1:
+	platform_device_unregister(sync_dev_1);
+
+err_unregister_sync_driver:
+	platform_driver_unregister(&sync_driver);
+
+err_unregister_async_dev_2:
+	platform_device_unregister(async_dev_2);
+
+err_unregister_async_driver:
+	platform_driver_unregister(&async_driver);
+
+err_unregister_async_dev_1:
+	platform_device_unregister(async_dev_1);
+
+	return error;
+}
+module_init(test_async_probe_init);
+
+static void __exit test_async_probe_exit(void)
+{
+	platform_driver_unregister(&async_driver);
+	platform_driver_unregister(&sync_driver);
+	platform_device_unregister(async_dev_1);
+	platform_device_unregister(async_dev_2);
+	platform_device_unregister(sync_dev_1);
+}
+module_exit(test_async_probe_exit);
+
+MODULE_DESCRIPTION("Test module for asynchronous driver probing");
+MODULE_AUTHOR("Dmitry Torokhov <dtor@chromium.org>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig
index 39dd30b6ef86..223ff2fcae7e 100644
--- a/drivers/block/Kconfig
+++ b/drivers/block/Kconfig
@@ -384,9 +384,12 @@ config BLK_DEV_RAM_DAX
 	  allocated from highmem (only a problem for highmem systems).
 
 config CDROM_PKTCDVD
-	tristate "Packet writing on CD/DVD media"
+	tristate "Packet writing on CD/DVD media (DEPRECATED)"
 	depends on !UML
 	help
+	  Note: This driver is deprecated and will be removed from the
+	  kernel in the near future!
+
 	  If you have a CDROM/DVD drive that supports packet writing, say
 	  Y to include support. It should work with any MMC/Mt Fuji
 	  compliant ATAPI or SCSI drive, which is just about any newer
diff --git a/drivers/block/brd.c b/drivers/block/brd.c
index 0c76d4016eeb..ad793f35632c 100644
--- a/drivers/block/brd.c
+++ b/drivers/block/brd.c
@@ -395,44 +395,9 @@ static long brd_direct_access(struct block_device *bdev, sector_t sector,
 #define brd_direct_access NULL
 #endif
 
-static int brd_ioctl(struct block_device *bdev, fmode_t mode,
-			unsigned int cmd, unsigned long arg)
-{
-	int error;
-	struct brd_device *brd = bdev->bd_disk->private_data;
-
-	if (cmd != BLKFLSBUF)
-		return -ENOTTY;
-
-	/*
-	 * ram device BLKFLSBUF has special semantics, we want to actually
-	 * release and destroy the ramdisk data.
-	 */
-	mutex_lock(&brd_mutex);
-	mutex_lock(&bdev->bd_mutex);
-	error = -EBUSY;
-	if (bdev->bd_openers <= 1) {
-		/*
-		 * Kill the cache first, so it isn't written back to the
-		 * device.
-		 *
-		 * Another thread might instantiate more buffercache here,
-		 * but there is not much we can do to close that race.
-		 */
-		kill_bdev(bdev);
-		brd_free_pages(brd);
-		error = 0;
-	}
-	mutex_unlock(&bdev->bd_mutex);
-	mutex_unlock(&brd_mutex);
-
-	return error;
-}
-
 static const struct block_device_operations brd_fops = {
 	.owner =		THIS_MODULE,
 	.rw_page =		brd_rw_page,
-	.ioctl =		brd_ioctl,
 	.direct_access =	brd_direct_access,
 };
 
@@ -443,8 +408,8 @@ static int rd_nr = CONFIG_BLK_DEV_RAM_COUNT;
 module_param(rd_nr, int, S_IRUGO);
 MODULE_PARM_DESC(rd_nr, "Maximum number of brd devices");
 
-int rd_size = CONFIG_BLK_DEV_RAM_SIZE;
-module_param(rd_size, int, S_IRUGO);
+unsigned long rd_size = CONFIG_BLK_DEV_RAM_SIZE;
+module_param(rd_size, ulong, S_IRUGO);
 MODULE_PARM_DESC(rd_size, "Size of each RAM disk in kbytes.");
 
 static int max_part = 1;
diff --git a/drivers/block/cciss_scsi.c b/drivers/block/cciss_scsi.c
index 1537302e56e3..a18de9d727b0 100644
--- a/drivers/block/cciss_scsi.c
+++ b/drivers/block/cciss_scsi.c
@@ -260,43 +260,6 @@ scsi_cmd_stack_free(ctlr_info_t *h)
 }
 
 #if 0
-static int xmargin=8;
-static int amargin=60;
-
-static void
-print_bytes (unsigned char *c, int len, int hex, int ascii)
-{
-
-	int i;
-	unsigned char *x;
-
-	if (hex)
-	{
-		x = c;
-		for (i=0;i<len;i++)
-		{
-			if ((i % xmargin) == 0 && i>0) printk("\n");
-			if ((i % xmargin) == 0) printk("0x%04x:", i);
-			printk(" %02x", *x);
-			x++;
-		}
-		printk("\n");
-	}
-	if (ascii)
-	{
-		x = c;
-		for (i=0;i<len;i++)
-		{
-			if ((i % amargin) == 0 && i>0) printk("\n");
-			if ((i % amargin) == 0) printk("0x%04x:", i);
-			if (*x > 26 && *x < 128) printk("%c", *x);
-			else printk(".");
-			x++;
-		}
-		printk("\n");
-	}
-}
-
 static void
 print_cmd(CommandList_struct *cp)
 {
@@ -305,30 +268,13 @@ print_cmd(CommandList_struct *cp)
 	printk("sgtot:%d\n", cp->Header.SGTotal);
 	printk("Tag:0x%08x/0x%08x\n", cp->Header.Tag.upper, 
 			cp->Header.Tag.lower);
-	printk("LUN:0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
-		cp->Header.LUN.LunAddrBytes[0],
-		cp->Header.LUN.LunAddrBytes[1],
-		cp->Header.LUN.LunAddrBytes[2],
-		cp->Header.LUN.LunAddrBytes[3],
-		cp->Header.LUN.LunAddrBytes[4],
-		cp->Header.LUN.LunAddrBytes[5],
-		cp->Header.LUN.LunAddrBytes[6],
-		cp->Header.LUN.LunAddrBytes[7]);
+	printk("LUN:0x%8phN\n", cp->Header.LUN.LunAddrBytes);
 	printk("CDBLen:%d\n", cp->Request.CDBLen);
 	printk("Type:%d\n",cp->Request.Type.Type);
 	printk("Attr:%d\n",cp->Request.Type.Attribute);
 	printk(" Dir:%d\n",cp->Request.Type.Direction);
 	printk("Timeout:%d\n",cp->Request.Timeout);
-	printk( "CDB: %02x %02x %02x %02x %02x %02x %02x %02x"
-		" %02x %02x %02x %02x %02x %02x %02x %02x\n",
-		cp->Request.CDB[0], cp->Request.CDB[1],
-		cp->Request.CDB[2], cp->Request.CDB[3],
-		cp->Request.CDB[4], cp->Request.CDB[5],
-		cp->Request.CDB[6], cp->Request.CDB[7],
-		cp->Request.CDB[8], cp->Request.CDB[9],
-		cp->Request.CDB[10], cp->Request.CDB[11],
-		cp->Request.CDB[12], cp->Request.CDB[13],
-		cp->Request.CDB[14], cp->Request.CDB[15]),
+	printk("CDB: %16ph\n", cp->Request.CDB);
 	printk("edesc.Addr: 0x%08x/0%08x, Len  = %d\n", 
 		cp->ErrDesc.Addr.upper, cp->ErrDesc.Addr.lower, 
 			cp->ErrDesc.Len);
@@ -340,9 +286,7 @@ print_cmd(CommandList_struct *cp)
 	printk("offense size:%d\n", cp->err_info->MoreErrInfo.Invalid_Cmd.offense_size);
 	printk("offense byte:%d\n", cp->err_info->MoreErrInfo.Invalid_Cmd.offense_num);
 	printk("offense value:%d\n", cp->err_info->MoreErrInfo.Invalid_Cmd.offense_value);
-			
 }
-
 #endif
 
 static int 
@@ -782,8 +726,10 @@ static void complete_scsi_command(CommandList_struct *c, int timeout,
 					"reported\n", c);
 			break;
 			case CMD_INVALID: {
-				/* print_bytes(c, sizeof(*c), 1, 0);
-				print_cmd(c); */
+				/*
+				print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1, c, sizeof(*c), false);
+				print_cmd(c);
+				 */
      /* We get CMD_INVALID if you address a non-existent tape drive instead
 	of a selection timeout (no response).  You will see this if you yank 
 	out a tape drive, then try to access it. This is kind of a shame
@@ -985,8 +931,10 @@ cciss_scsi_interpret_error(ctlr_info_t *h, CommandList_struct *c)
 			dev_warn(&h->pdev->dev,
 				"%p is reported invalid (probably means "
 				"target device no longer present)\n", c);
-			/* print_bytes((unsigned char *) c, sizeof(*c), 1, 0);
-			print_cmd(c);  */
+			/*
+			print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1, c, sizeof(*c), false);
+			print_cmd(c);
+			 */
 			}
 		break;
 		case CMD_PROTOCOL_ERR:
diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c
index 2d3d50ab74bf..8d7bcfa49c12 100644
--- a/drivers/block/drbd/drbd_actlog.c
+++ b/drivers/block/drbd/drbd_actlog.c
@@ -148,7 +148,7 @@ static int _drbd_md_sync_page_io(struct drbd_device *device,
 
 	if ((op == REQ_OP_WRITE) && !test_bit(MD_NO_FUA, &device->flags))
 		op_flags |= REQ_FUA | REQ_PREFLUSH;
-	op_flags |= REQ_SYNC | REQ_NOIDLE;
+	op_flags |= REQ_SYNC;
 
 	bio = bio_alloc_drbd(GFP_NOIO);
 	bio->bi_bdev = bdev->md_bdev;
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index 942384f34e22..c7728dd77230 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -1266,7 +1266,7 @@ static void submit_one_flush(struct drbd_device *device, struct issue_flush_cont
 	bio->bi_bdev = device->ldev->backing_bdev;
 	bio->bi_private = octx;
 	bio->bi_end_io = one_flush_endio;
-	bio_set_op_attrs(bio, REQ_OP_FLUSH, WRITE_FLUSH);
+	bio->bi_opf = REQ_OP_FLUSH | REQ_PREFLUSH;
 
 	device->flush_jif = jiffies;
 	set_bit(FLUSH_PENDING, &device->flags);
@@ -1648,20 +1648,8 @@ next_bio:
 
 	page_chain_for_each(page) {
 		unsigned len = min_t(unsigned, data_size, PAGE_SIZE);
-		if (!bio_add_page(bio, page, len, 0)) {
-			/* A single page must always be possible!
-			 * But in case it fails anyways,
-			 * we deal with it, and complain (below). */
-			if (bio->bi_vcnt == 0) {
-				drbd_err(device,
-					"bio_add_page failed for len=%u, "
-					"bi_vcnt=0 (bi_sector=%llu)\n",
-					len, (uint64_t)bio->bi_iter.bi_sector);
-				err = -ENOSPC;
-				goto fail;
-			}
+		if (!bio_add_page(bio, page, len, 0))
 			goto next_bio;
-		}
 		data_size -= len;
 		sector += len >> 9;
 		--nr_pages;
diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c
index e3d8e4ced4a2..a391a3cfb3fe 100644
--- a/drivers/block/floppy.c
+++ b/drivers/block/floppy.c
@@ -3806,14 +3806,10 @@ static int __floppy_read_block_0(struct block_device *bdev, int drive)
 
 	cbdata.drive = drive;
 
-	bio_init(&bio);
-	bio.bi_io_vec = &bio_vec;
-	bio_vec.bv_page = page;
-	bio_vec.bv_len = size;
-	bio_vec.bv_offset = 0;
-	bio.bi_vcnt = 1;
-	bio.bi_iter.bi_size = size;
+	bio_init(&bio, &bio_vec, 1);
 	bio.bi_bdev = bdev;
+	bio_add_page(&bio, page, size, 0);
+
 	bio.bi_iter.bi_sector = 0;
 	bio.bi_flags |= (1 << BIO_QUIET);
 	bio.bi_private = &cbdata;
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index fa1b7a90ba11..4af818766797 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -1646,7 +1646,7 @@ static int loop_queue_rq(struct blk_mq_hw_ctx *hctx,
 	blk_mq_start_request(bd->rq);
 
 	if (lo->lo_state != Lo_bound)
-		return -EIO;
+		return BLK_MQ_RQ_QUEUE_ERROR;
 
 	switch (req_op(cmd->rq)) {
 	case REQ_OP_FLUSH:
diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c
index 3cfd879267b2..f96ab717534c 100644
--- a/drivers/block/mtip32xx/mtip32xx.c
+++ b/drivers/block/mtip32xx/mtip32xx.c
@@ -2035,18 +2035,14 @@ static int exec_drive_taskfile(struct driver_data *dd,
 	taskout = req_task->out_size;
 	taskin = req_task->in_size;
 	/* 130560 = 512 * 0xFF*/
-	if (taskin > 130560 || taskout > 130560) {
-		err = -EINVAL;
-		goto abort;
-	}
+	if (taskin > 130560 || taskout > 130560)
+		return -EINVAL;
 
 	if (taskout) {
 		outbuf = memdup_user(buf + outtotal, taskout);
-		if (IS_ERR(outbuf)) {
-			err = PTR_ERR(outbuf);
-			outbuf = NULL;
-			goto abort;
-		}
+		if (IS_ERR(outbuf))
+			return PTR_ERR(outbuf);
+
 		outbuf_dma = pci_map_single(dd->pdev,
 					 outbuf,
 					 taskout,
@@ -3937,8 +3933,10 @@ static int mtip_block_initialize(struct driver_data *dd)
 
 	/* Generate the disk name, implemented same as in sd.c */
 	do {
-		if (!ida_pre_get(&rssd_index_ida, GFP_KERNEL))
+		if (!ida_pre_get(&rssd_index_ida, GFP_KERNEL)) {
+			rv = -ENOMEM;
 			goto ida_get_error;
+		}
 
 		spin_lock(&rssd_index_lock);
 		rv = ida_get_new(&rssd_index_ida, &index);
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 7a1048755914..99c84468f154 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -41,26 +41,34 @@
 
 #include <linux/nbd.h>
 
+struct nbd_sock {
+	struct socket *sock;
+	struct mutex tx_lock;
+};
+
 #define NBD_TIMEDOUT			0
 #define NBD_DISCONNECT_REQUESTED	1
+#define NBD_DISCONNECTED		2
+#define NBD_RUNNING			3
 
 struct nbd_device {
 	u32 flags;
 	unsigned long runtime_flags;
-	struct socket * sock;	/* If == NULL, device is not ready, yet	*/
+	struct nbd_sock **socks;
 	int magic;
 
 	struct blk_mq_tag_set tag_set;
 
-	struct mutex tx_lock;
+	struct mutex config_lock;
 	struct gendisk *disk;
-	int blksize;
+	int num_connections;
+	atomic_t recv_threads;
+	wait_queue_head_t recv_wq;
+	loff_t blksize;
 	loff_t bytesize;
 
-	/* protects initialization and shutdown of the socket */
-	spinlock_t sock_lock;
 	struct task_struct *task_recv;
-	struct task_struct *task_send;
+	struct task_struct *task_setup;
 
 #if IS_ENABLED(CONFIG_DEBUG_FS)
 	struct dentry *dbg_dir;
@@ -69,7 +77,7 @@ struct nbd_device {
 
 struct nbd_cmd {
 	struct nbd_device *nbd;
-	struct list_head list;
+	struct completion send_complete;
 };
 
 #if IS_ENABLED(CONFIG_DEBUG_FS)
@@ -126,7 +134,7 @@ static void nbd_size_update(struct nbd_device *nbd, struct block_device *bdev)
 }
 
 static int nbd_size_set(struct nbd_device *nbd, struct block_device *bdev,
-			int blocksize, int nr_blocks)
+			loff_t blocksize, loff_t nr_blocks)
 {
 	int ret;
 
@@ -135,7 +143,7 @@ static int nbd_size_set(struct nbd_device *nbd, struct block_device *bdev,
 		return ret;
 
 	nbd->blksize = blocksize;
-	nbd->bytesize = (loff_t)blocksize * (loff_t)nr_blocks;
+	nbd->bytesize = blocksize * nr_blocks;
 
 	nbd_size_update(nbd, bdev);
 
@@ -159,22 +167,20 @@ static void nbd_end_request(struct nbd_cmd *cmd)
  */
 static void sock_shutdown(struct nbd_device *nbd)
 {
-	struct socket *sock;
-
-	spin_lock(&nbd->sock_lock);
+	int i;
 
-	if (!nbd->sock) {
-		spin_unlock(&nbd->sock_lock);
+	if (nbd->num_connections == 0)
+		return;
+	if (test_and_set_bit(NBD_DISCONNECTED, &nbd->runtime_flags))
 		return;
-	}
-
-	sock = nbd->sock;
-	dev_warn(disk_to_dev(nbd->disk), "shutting down socket\n");
-	nbd->sock = NULL;
-	spin_unlock(&nbd->sock_lock);
 
-	kernel_sock_shutdown(sock, SHUT_RDWR);
-	sockfd_put(sock);
+	for (i = 0; i < nbd->num_connections; i++) {
+		struct nbd_sock *nsock = nbd->socks[i];
+		mutex_lock(&nsock->tx_lock);
+		kernel_sock_shutdown(nsock->sock, SHUT_RDWR);
+		mutex_unlock(&nsock->tx_lock);
+	}
+	dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n");
 }
 
 static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req,
@@ -182,42 +188,38 @@ static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req,
 {
 	struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req);
 	struct nbd_device *nbd = cmd->nbd;
-	struct socket *sock = NULL;
-
-	spin_lock(&nbd->sock_lock);
 
+	dev_err(nbd_to_dev(nbd), "Connection timed out, shutting down connection\n");
 	set_bit(NBD_TIMEDOUT, &nbd->runtime_flags);
-
-	if (nbd->sock) {
-		sock = nbd->sock;
-		get_file(sock->file);
-	}
-
-	spin_unlock(&nbd->sock_lock);
-	if (sock) {
-		kernel_sock_shutdown(sock, SHUT_RDWR);
-		sockfd_put(sock);
-	}
-
 	req->errors++;
-	dev_err(nbd_to_dev(nbd), "Connection timed out, shutting down connection\n");
+
+	/*
+	 * If our disconnect packet times out then we're already holding the
+	 * config_lock and could deadlock here, so just set an error and return,
+	 * we'll handle shutting everything down later.
+	 */
+	if (req->cmd_type == REQ_TYPE_DRV_PRIV)
+		return BLK_EH_HANDLED;
+	mutex_lock(&nbd->config_lock);
+	sock_shutdown(nbd);
+	mutex_unlock(&nbd->config_lock);
 	return BLK_EH_HANDLED;
 }
 
 /*
  *  Send or receive packet.
  */
-static int sock_xmit(struct nbd_device *nbd, int send, void *buf, int size,
-		int msg_flags)
+static int sock_xmit(struct nbd_device *nbd, int index, int send, void *buf,
+		     int size, int msg_flags)
 {
-	struct socket *sock = nbd->sock;
+	struct socket *sock = nbd->socks[index]->sock;
 	int result;
 	struct msghdr msg;
 	struct kvec iov;
 	unsigned long pflags = current->flags;
 
 	if (unlikely(!sock)) {
-		dev_err(disk_to_dev(nbd->disk),
+		dev_err_ratelimited(disk_to_dev(nbd->disk),
 			"Attempted %s on closed socket in sock_xmit\n",
 			(send ? "send" : "recv"));
 		return -EINVAL;
@@ -254,29 +256,29 @@ static int sock_xmit(struct nbd_device *nbd, int send, void *buf, int size,
 	return result;
 }
 
-static inline int sock_send_bvec(struct nbd_device *nbd, struct bio_vec *bvec,
-		int flags)
+static inline int sock_send_bvec(struct nbd_device *nbd, int index,
+				 struct bio_vec *bvec, int flags)
 {
 	int result;
 	void *kaddr = kmap(bvec->bv_page);
-	result = sock_xmit(nbd, 1, kaddr + bvec->bv_offset,
+	result = sock_xmit(nbd, index, 1, kaddr + bvec->bv_offset,
 			   bvec->bv_len, flags);
 	kunmap(bvec->bv_page);
 	return result;
 }
 
 /* always call with the tx_lock held */
-static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd)
+static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd, int index)
 {
 	struct request *req = blk_mq_rq_from_pdu(cmd);
 	int result, flags;
 	struct nbd_request request;
 	unsigned long size = blk_rq_bytes(req);
+	struct bio *bio;
 	u32 type;
+	u32 tag = blk_mq_unique_tag(req);
 
-	if (req->cmd_type == REQ_TYPE_DRV_PRIV)
-		type = NBD_CMD_DISC;
-	else if (req_op(req) == REQ_OP_DISCARD)
+	if (req_op(req) == REQ_OP_DISCARD)
 		type = NBD_CMD_TRIM;
 	else if (req_op(req) == REQ_OP_FLUSH)
 		type = NBD_CMD_FLUSH;
@@ -288,73 +290,89 @@ static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd)
 	memset(&request, 0, sizeof(request));
 	request.magic = htonl(NBD_REQUEST_MAGIC);
 	request.type = htonl(type);
-	if (type != NBD_CMD_FLUSH && type != NBD_CMD_DISC) {
+	if (type != NBD_CMD_FLUSH) {
 		request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9);
 		request.len = htonl(size);
 	}
-	memcpy(request.handle, &req->tag, sizeof(req->tag));
+	memcpy(request.handle, &tag, sizeof(tag));
 
 	dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n",
 		cmd, nbdcmd_to_ascii(type),
 		(unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req));
-	result = sock_xmit(nbd, 1, &request, sizeof(request),
+	result = sock_xmit(nbd, index, 1, &request, sizeof(request),
 			(type == NBD_CMD_WRITE) ? MSG_MORE : 0);
 	if (result <= 0) {
-		dev_err(disk_to_dev(nbd->disk),
+		dev_err_ratelimited(disk_to_dev(nbd->disk),
 			"Send control failed (result %d)\n", result);
 		return -EIO;
 	}
 
-	if (type == NBD_CMD_WRITE) {
-		struct req_iterator iter;
+	if (type != NBD_CMD_WRITE)
+		return 0;
+
+	flags = 0;
+	bio = req->bio;
+	while (bio) {
+		struct bio *next = bio->bi_next;
+		struct bvec_iter iter;
 		struct bio_vec bvec;
-		/*
-		 * we are really probing at internals to determine
-		 * whether to set MSG_MORE or not...
-		 */
-		rq_for_each_segment(bvec, req, iter) {
-			flags = 0;
-			if (!rq_iter_last(bvec, iter))
+
+		bio_for_each_segment(bvec, bio, iter) {
+			bool is_last = !next && bio_iter_last(bvec, iter);
+
+			if (is_last)
 				flags = MSG_MORE;
 			dev_dbg(nbd_to_dev(nbd), "request %p: sending %d bytes data\n",
 				cmd, bvec.bv_len);
-			result = sock_send_bvec(nbd, &bvec, flags);
+			result = sock_send_bvec(nbd, index, &bvec, flags);
 			if (result <= 0) {
 				dev_err(disk_to_dev(nbd->disk),
 					"Send data failed (result %d)\n",
 					result);
 				return -EIO;
 			}
+			/*
+			 * The completion might already have come in,
+			 * so break for the last one instead of letting
+			 * the iterator do it. This prevents use-after-free
+			 * of the bio.
+			 */
+			if (is_last)
+				break;
 		}
+		bio = next;
 	}
 	return 0;
 }
 
-static inline int sock_recv_bvec(struct nbd_device *nbd, struct bio_vec *bvec)
+static inline int sock_recv_bvec(struct nbd_device *nbd, int index,
+				 struct bio_vec *bvec)
 {
 	int result;
 	void *kaddr = kmap(bvec->bv_page);
-	result = sock_xmit(nbd, 0, kaddr + bvec->bv_offset, bvec->bv_len,
-			MSG_WAITALL);
+	result = sock_xmit(nbd, index, 0, kaddr + bvec->bv_offset,
+			   bvec->bv_len, MSG_WAITALL);
 	kunmap(bvec->bv_page);
 	return result;
 }
 
 /* NULL returned = something went wrong, inform userspace */
-static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd)
+static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd, int index)
 {
 	int result;
 	struct nbd_reply reply;
 	struct nbd_cmd *cmd;
 	struct request *req = NULL;
 	u16 hwq;
-	int tag;
+	u32 tag;
 
 	reply.magic = 0;
-	result = sock_xmit(nbd, 0, &reply, sizeof(reply), MSG_WAITALL);
+	result = sock_xmit(nbd, index, 0, &reply, sizeof(reply), MSG_WAITALL);
 	if (result <= 0) {
-		dev_err(disk_to_dev(nbd->disk),
-			"Receive control failed (result %d)\n", result);
+		if (!test_bit(NBD_DISCONNECTED, &nbd->runtime_flags) &&
+		    !test_bit(NBD_DISCONNECT_REQUESTED, &nbd->runtime_flags))
+			dev_err(disk_to_dev(nbd->disk),
+				"Receive control failed (result %d)\n", result);
 		return ERR_PTR(result);
 	}
 
@@ -364,7 +382,7 @@ static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd)
 		return ERR_PTR(-EPROTO);
 	}
 
-	memcpy(&tag, reply.handle, sizeof(int));
+	memcpy(&tag, reply.handle, sizeof(u32));
 
 	hwq = blk_mq_unique_tag_to_hwq(tag);
 	if (hwq < nbd->tag_set.nr_hw_queues)
@@ -376,7 +394,6 @@ static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd)
 		return ERR_PTR(-ENOENT);
 	}
 	cmd = blk_mq_rq_to_pdu(req);
-
 	if (ntohl(reply.error)) {
 		dev_err(disk_to_dev(nbd->disk), "Other side returned error (%d)\n",
 			ntohl(reply.error));
@@ -390,7 +407,7 @@ static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd)
 		struct bio_vec bvec;
 
 		rq_for_each_segment(bvec, req, iter) {
-			result = sock_recv_bvec(nbd, &bvec);
+			result = sock_recv_bvec(nbd, index, &bvec);
 			if (result <= 0) {
 				dev_err(disk_to_dev(nbd->disk), "Receive data failed (result %d)\n",
 					result);
@@ -400,6 +417,9 @@ static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd)
 			dev_dbg(nbd_to_dev(nbd), "request %p: got %d bytes data\n",
 				cmd, bvec.bv_len);
 		}
+	} else {
+		/* See the comment in nbd_queue_rq. */
+		wait_for_completion(&cmd->send_complete);
 	}
 	return cmd;
 }
@@ -418,25 +438,24 @@ static struct device_attribute pid_attr = {
 	.show = pid_show,
 };
 
-static int nbd_thread_recv(struct nbd_device *nbd, struct block_device *bdev)
+struct recv_thread_args {
+	struct work_struct work;
+	struct nbd_device *nbd;
+	int index;
+};
+
+static void recv_work(struct work_struct *work)
 {
+	struct recv_thread_args *args = container_of(work,
+						     struct recv_thread_args,
+						     work);
+	struct nbd_device *nbd = args->nbd;
 	struct nbd_cmd *cmd;
-	int ret;
+	int ret = 0;
 
 	BUG_ON(nbd->magic != NBD_MAGIC);
-
-	sk_set_memalloc(nbd->sock->sk);
-
-	ret = device_create_file(disk_to_dev(nbd->disk), &pid_attr);
-	if (ret) {
-		dev_err(disk_to_dev(nbd->disk), "device_create_file failed!\n");
-		return ret;
-	}
-
-	nbd_size_update(nbd, bdev);
-
 	while (1) {
-		cmd = nbd_read_stat(nbd);
+		cmd = nbd_read_stat(nbd, args->index);
 		if (IS_ERR(cmd)) {
 			ret = PTR_ERR(cmd);
 			break;
@@ -445,10 +464,14 @@ static int nbd_thread_recv(struct nbd_device *nbd, struct block_device *bdev)
 		nbd_end_request(cmd);
 	}
 
-	nbd_size_clear(nbd, bdev);
-
-	device_remove_file(disk_to_dev(nbd->disk), &pid_attr);
-	return ret;
+	/*
+	 * We got an error, shut everybody down if this wasn't the result of a
+	 * disconnect request.
+	 */
+	if (ret && !test_bit(NBD_DISCONNECT_REQUESTED, &nbd->runtime_flags))
+		sock_shutdown(nbd);
+	atomic_dec(&nbd->recv_threads);
+	wake_up(&nbd->recv_wq);
 }
 
 static void nbd_clear_req(struct request *req, void *data, bool reserved)
@@ -466,51 +489,60 @@ static void nbd_clear_que(struct nbd_device *nbd)
 {
 	BUG_ON(nbd->magic != NBD_MAGIC);
 
-	/*
-	 * Because we have set nbd->sock to NULL under the tx_lock, all
-	 * modifications to the list must have completed by now.
-	 */
-	BUG_ON(nbd->sock);
-
 	blk_mq_tagset_busy_iter(&nbd->tag_set, nbd_clear_req, NULL);
 	dev_dbg(disk_to_dev(nbd->disk), "queue cleared\n");
 }
 
 
-static void nbd_handle_cmd(struct nbd_cmd *cmd)
+static void nbd_handle_cmd(struct nbd_cmd *cmd, int index)
 {
 	struct request *req = blk_mq_rq_from_pdu(cmd);
 	struct nbd_device *nbd = cmd->nbd;
+	struct nbd_sock *nsock;
+
+	if (index >= nbd->num_connections) {
+		dev_err_ratelimited(disk_to_dev(nbd->disk),
+				    "Attempted send on invalid socket\n");
+		goto error_out;
+	}
+
+	if (test_bit(NBD_DISCONNECTED, &nbd->runtime_flags)) {
+		dev_err_ratelimited(disk_to_dev(nbd->disk),
+				    "Attempted send on closed socket\n");
+		goto error_out;
+	}
 
-	if (req->cmd_type != REQ_TYPE_FS)
+	if (req->cmd_type != REQ_TYPE_FS &&
+	    req->cmd_type != REQ_TYPE_DRV_PRIV)
 		goto error_out;
 
-	if (rq_data_dir(req) == WRITE &&
+	if (req->cmd_type == REQ_TYPE_FS &&
+	    rq_data_dir(req) == WRITE &&
 	    (nbd->flags & NBD_FLAG_READ_ONLY)) {
-		dev_err(disk_to_dev(nbd->disk),
-			"Write on read-only\n");
+		dev_err_ratelimited(disk_to_dev(nbd->disk),
+				    "Write on read-only\n");
 		goto error_out;
 	}
 
 	req->errors = 0;
 
-	mutex_lock(&nbd->tx_lock);
-	nbd->task_send = current;
-	if (unlikely(!nbd->sock)) {
-		mutex_unlock(&nbd->tx_lock);
-		dev_err(disk_to_dev(nbd->disk),
-			"Attempted send on closed socket\n");
+	nsock = nbd->socks[index];
+	mutex_lock(&nsock->tx_lock);
+	if (unlikely(!nsock->sock)) {
+		mutex_unlock(&nsock->tx_lock);
+		dev_err_ratelimited(disk_to_dev(nbd->disk),
+				    "Attempted send on closed socket\n");
 		goto error_out;
 	}
 
-	if (nbd_send_cmd(nbd, cmd) != 0) {
-		dev_err(disk_to_dev(nbd->disk), "Request send failed\n");
+	if (nbd_send_cmd(nbd, cmd, index) != 0) {
+		dev_err_ratelimited(disk_to_dev(nbd->disk),
+				    "Request send failed\n");
 		req->errors++;
 		nbd_end_request(cmd);
 	}
 
-	nbd->task_send = NULL;
-	mutex_unlock(&nbd->tx_lock);
+	mutex_unlock(&nsock->tx_lock);
 
 	return;
 
@@ -524,39 +556,70 @@ static int nbd_queue_rq(struct blk_mq_hw_ctx *hctx,
 {
 	struct nbd_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
 
+	/*
+	 * Since we look at the bio's to send the request over the network we
+	 * need to make sure the completion work doesn't mark this request done
+	 * before we are done doing our send.  This keeps us from dereferencing
+	 * freed data if we have particularly fast completions (ie we get the
+	 * completion before we exit sock_xmit on the last bvec) or in the case
+	 * that the server is misbehaving (or there was an error) before we're
+	 * done sending everything over the wire.
+	 */
+	init_completion(&cmd->send_complete);
 	blk_mq_start_request(bd->rq);
-	nbd_handle_cmd(cmd);
+	nbd_handle_cmd(cmd, hctx->queue_num);
+	complete(&cmd->send_complete);
+
 	return BLK_MQ_RQ_QUEUE_OK;
 }
 
-static int nbd_set_socket(struct nbd_device *nbd, struct socket *sock)
+static int nbd_add_socket(struct nbd_device *nbd, struct socket *sock)
 {
-	int ret = 0;
-
-	spin_lock_irq(&nbd->sock_lock);
+	struct nbd_sock **socks;
+	struct nbd_sock *nsock;
 
-	if (nbd->sock) {
-		ret = -EBUSY;
-		goto out;
+	if (!nbd->task_setup)
+		nbd->task_setup = current;
+	if (nbd->task_setup != current) {
+		dev_err(disk_to_dev(nbd->disk),
+			"Device being setup by another task");
+		return -EINVAL;
 	}
 
-	nbd->sock = sock;
+	socks = krealloc(nbd->socks, (nbd->num_connections + 1) *
+			 sizeof(struct nbd_sock *), GFP_KERNEL);
+	if (!socks)
+		return -ENOMEM;
+	nsock = kzalloc(sizeof(struct nbd_sock), GFP_KERNEL);
+	if (!nsock)
+		return -ENOMEM;
 
-out:
-	spin_unlock_irq(&nbd->sock_lock);
+	nbd->socks = socks;
+
+	mutex_init(&nsock->tx_lock);
+	nsock->sock = sock;
+	socks[nbd->num_connections++] = nsock;
 
-	return ret;
+	return 0;
 }
 
 /* Reset all properties of an NBD device */
 static void nbd_reset(struct nbd_device *nbd)
 {
+	int i;
+
+	for (i = 0; i < nbd->num_connections; i++)
+		kfree(nbd->socks[i]);
+	kfree(nbd->socks);
+	nbd->socks = NULL;
 	nbd->runtime_flags = 0;
 	nbd->blksize = 1024;
 	nbd->bytesize = 0;
 	set_capacity(nbd->disk, 0);
 	nbd->flags = 0;
 	nbd->tag_set.timeout = 0;
+	nbd->num_connections = 0;
+	nbd->task_setup = NULL;
 	queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, nbd->disk->queue);
 }
 
@@ -582,48 +645,68 @@ static void nbd_parse_flags(struct nbd_device *nbd, struct block_device *bdev)
 		blk_queue_write_cache(nbd->disk->queue, false, false);
 }
 
+static void send_disconnects(struct nbd_device *nbd)
+{
+	struct nbd_request request = {};
+	int i, ret;
+
+	request.magic = htonl(NBD_REQUEST_MAGIC);
+	request.type = htonl(NBD_CMD_DISC);
+
+	for (i = 0; i < nbd->num_connections; i++) {
+		ret = sock_xmit(nbd, i, 1, &request, sizeof(request), 0);
+		if (ret <= 0)
+			dev_err(disk_to_dev(nbd->disk),
+				"Send disconnect failed %d\n", ret);
+	}
+}
+
 static int nbd_dev_dbg_init(struct nbd_device *nbd);
 static void nbd_dev_dbg_close(struct nbd_device *nbd);
 
-/* Must be called with tx_lock held */
-
+/* Must be called with config_lock held */
 static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 		       unsigned int cmd, unsigned long arg)
 {
 	switch (cmd) {
 	case NBD_DISCONNECT: {
-		struct request *sreq;
-
 		dev_info(disk_to_dev(nbd->disk), "NBD_DISCONNECT\n");
-		if (!nbd->sock)
+		if (!nbd->socks)
 			return -EINVAL;
 
-		sreq = blk_mq_alloc_request(bdev_get_queue(bdev), WRITE, 0);
-		if (IS_ERR(sreq))
-			return -ENOMEM;
-
-		mutex_unlock(&nbd->tx_lock);
+		mutex_unlock(&nbd->config_lock);
 		fsync_bdev(bdev);
-		mutex_lock(&nbd->tx_lock);
-		sreq->cmd_type = REQ_TYPE_DRV_PRIV;
+		mutex_lock(&nbd->config_lock);
 
 		/* Check again after getting mutex back.  */
-		if (!nbd->sock) {
-			blk_mq_free_request(sreq);
+		if (!nbd->socks)
 			return -EINVAL;
-		}
 
-		set_bit(NBD_DISCONNECT_REQUESTED, &nbd->runtime_flags);
-
-		nbd_send_cmd(nbd, blk_mq_rq_to_pdu(sreq));
-		blk_mq_free_request(sreq);
+		if (!test_and_set_bit(NBD_DISCONNECT_REQUESTED,
+				      &nbd->runtime_flags))
+			send_disconnects(nbd);
 		return 0;
 	}
- 
+
 	case NBD_CLEAR_SOCK:
 		sock_shutdown(nbd);
 		nbd_clear_que(nbd);
 		kill_bdev(bdev);
+		nbd_bdev_reset(bdev);
+		/*
+		 * We want to give the run thread a chance to wait for everybody
+		 * to clean up and then do it's own cleanup.
+		 */
+		if (!test_bit(NBD_RUNNING, &nbd->runtime_flags)) {
+			int i;
+
+			for (i = 0; i < nbd->num_connections; i++)
+				kfree(nbd->socks[i]);
+			kfree(nbd->socks);
+			nbd->socks = NULL;
+			nbd->num_connections = 0;
+			nbd->task_setup = NULL;
+		}
 		return 0;
 
 	case NBD_SET_SOCK: {
@@ -633,7 +716,7 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 		if (!sock)
 			return err;
 
-		err = nbd_set_socket(nbd, sock);
+		err = nbd_add_socket(nbd, sock);
 		if (!err && max_part)
 			bdev->bd_invalidated = 1;
 
@@ -648,7 +731,7 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 
 	case NBD_SET_SIZE:
 		return nbd_size_set(nbd, bdev, nbd->blksize,
-				    arg / nbd->blksize);
+					div_s64(arg, nbd->blksize));
 
 	case NBD_SET_SIZE_BLOCKS:
 		return nbd_size_set(nbd, bdev, nbd->blksize, arg);
@@ -662,26 +745,61 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 		return 0;
 
 	case NBD_DO_IT: {
-		int error;
+		struct recv_thread_args *args;
+		int num_connections = nbd->num_connections;
+		int error = 0, i;
 
 		if (nbd->task_recv)
 			return -EBUSY;
-		if (!nbd->sock)
+		if (!nbd->socks)
 			return -EINVAL;
+		if (num_connections > 1 &&
+		    !(nbd->flags & NBD_FLAG_CAN_MULTI_CONN)) {
+			dev_err(disk_to_dev(nbd->disk), "server does not support multiple connections per device.\n");
+			error = -EINVAL;
+			goto out_err;
+		}
 
-		/* We have to claim the device under the lock */
+		set_bit(NBD_RUNNING, &nbd->runtime_flags);
+		blk_mq_update_nr_hw_queues(&nbd->tag_set, nbd->num_connections);
+		args = kcalloc(num_connections, sizeof(*args), GFP_KERNEL);
+		if (!args) {
+			error = -ENOMEM;
+			goto out_err;
+		}
 		nbd->task_recv = current;
-		mutex_unlock(&nbd->tx_lock);
+		mutex_unlock(&nbd->config_lock);
 
 		nbd_parse_flags(nbd, bdev);
 
+		error = device_create_file(disk_to_dev(nbd->disk), &pid_attr);
+		if (error) {
+			dev_err(disk_to_dev(nbd->disk), "device_create_file failed!\n");
+			goto out_recv;
+		}
+
+		nbd_size_update(nbd, bdev);
+
 		nbd_dev_dbg_init(nbd);
-		error = nbd_thread_recv(nbd, bdev);
+		for (i = 0; i < num_connections; i++) {
+			sk_set_memalloc(nbd->socks[i]->sock->sk);
+			atomic_inc(&nbd->recv_threads);
+			INIT_WORK(&args[i].work, recv_work);
+			args[i].nbd = nbd;
+			args[i].index = i;
+			queue_work(system_long_wq, &args[i].work);
+		}
+		wait_event_interruptible(nbd->recv_wq,
+					 atomic_read(&nbd->recv_threads) == 0);
+		for (i = 0; i < num_connections; i++)
+			flush_work(&args[i].work);
 		nbd_dev_dbg_close(nbd);
-
-		mutex_lock(&nbd->tx_lock);
+		nbd_size_clear(nbd, bdev);
+		device_remove_file(disk_to_dev(nbd->disk), &pid_attr);
+out_recv:
+		mutex_lock(&nbd->config_lock);
 		nbd->task_recv = NULL;
-
+out_err:
 		sock_shutdown(nbd);
 		nbd_clear_que(nbd);
 		kill_bdev(bdev);
@@ -694,7 +812,6 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 			error = -ETIMEDOUT;
 
 		nbd_reset(nbd);
-
 		return error;
 	}
 
@@ -726,9 +843,9 @@ static int nbd_ioctl(struct block_device *bdev, fmode_t mode,
 
 	BUG_ON(nbd->magic != NBD_MAGIC);
 
-	mutex_lock(&nbd->tx_lock);
+	mutex_lock(&nbd->config_lock);
 	error = __nbd_ioctl(bdev, nbd, cmd, arg);
-	mutex_unlock(&nbd->tx_lock);
+	mutex_unlock(&nbd->config_lock);
 
 	return error;
 }
@@ -748,8 +865,6 @@ static int nbd_dbg_tasks_show(struct seq_file *s, void *unused)
 
 	if (nbd->task_recv)
 		seq_printf(s, "recv: %d\n", task_pid_nr(nbd->task_recv));
-	if (nbd->task_send)
-		seq_printf(s, "send: %d\n", task_pid_nr(nbd->task_send));
 
 	return 0;
 }
@@ -817,7 +932,7 @@ static int nbd_dev_dbg_init(struct nbd_device *nbd)
 	debugfs_create_file("tasks", 0444, dir, nbd, &nbd_dbg_tasks_ops);
 	debugfs_create_u64("size_bytes", 0444, dir, &nbd->bytesize);
 	debugfs_create_u32("timeout", 0444, dir, &nbd->tag_set.timeout);
-	debugfs_create_u32("blocksize", 0444, dir, &nbd->blksize);
+	debugfs_create_u64("blocksize", 0444, dir, &nbd->blksize);
 	debugfs_create_file("flags", 0444, dir, nbd, &nbd_dbg_flags_ops);
 
 	return 0;
@@ -873,9 +988,7 @@ static int nbd_init_request(void *data, struct request *rq,
 			    unsigned int numa_node)
 {
 	struct nbd_cmd *cmd = blk_mq_rq_to_pdu(rq);
-
 	cmd->nbd = data;
-	INIT_LIST_HEAD(&cmd->list);
 	return 0;
 }
 
@@ -985,13 +1098,13 @@ static int __init nbd_init(void)
 	for (i = 0; i < nbds_max; i++) {
 		struct gendisk *disk = nbd_dev[i].disk;
 		nbd_dev[i].magic = NBD_MAGIC;
-		spin_lock_init(&nbd_dev[i].sock_lock);
-		mutex_init(&nbd_dev[i].tx_lock);
+		mutex_init(&nbd_dev[i].config_lock);
 		disk->major = NBD_MAJOR;
 		disk->first_minor = i << part_shift;
 		disk->fops = &nbd_fops;
 		disk->private_data = &nbd_dev[i];
 		sprintf(disk->disk_name, "nbd%d", i);
+		init_waitqueue_head(&nbd_dev[i].recv_wq);
 		nbd_reset(&nbd_dev[i]);
 		add_disk(disk);
 	}
diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c
index ba6f4a2e73db..4943ee22716e 100644
--- a/drivers/block/null_blk.c
+++ b/drivers/block/null_blk.c
@@ -577,6 +577,7 @@ static void null_nvm_unregister(struct nullb *nullb)
 #else
 static int null_nvm_register(struct nullb *nullb)
 {
+	pr_err("null_blk: CONFIG_NVM needs to be enabled for LightNVM\n");
 	return -EINVAL;
 }
 static void null_nvm_unregister(struct nullb *nullb) {}
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index 90fa4ac149db..95c98de92971 100644
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -721,7 +721,7 @@ static int pkt_generic_packet(struct pktcdvd_device *pd, struct packet_command *
 
 	rq->timeout = 60*HZ;
 	if (cgc->quiet)
-		rq->cmd_flags |= REQ_QUIET;
+		rq->rq_flags |= RQF_QUIET;
 
 	blk_execute_rq(rq->q, pd->bdev->bd_disk, rq, 0);
 	if (rq->errors)
@@ -944,39 +944,6 @@ static int pkt_set_segment_merging(struct pktcdvd_device *pd, struct request_que
 	}
 }
 
-/*
- * Copy all data for this packet to pkt->pages[], so that
- * a) The number of required segments for the write bio is minimized, which
- *    is necessary for some scsi controllers.
- * b) The data can be used as cache to avoid read requests if we receive a
- *    new write request for the same zone.
- */
-static void pkt_make_local_copy(struct packet_data *pkt, struct bio_vec *bvec)
-{
-	int f, p, offs;
-
-	/* Copy all data to pkt->pages[] */
-	p = 0;
-	offs = 0;
-	for (f = 0; f < pkt->frames; f++) {
-		if (bvec[f].bv_page != pkt->pages[p]) {
-			void *vfrom = kmap_atomic(bvec[f].bv_page) + bvec[f].bv_offset;
-			void *vto = page_address(pkt->pages[p]) + offs;
-			memcpy(vto, vfrom, CD_FRAMESIZE);
-			kunmap_atomic(vfrom);
-			bvec[f].bv_page = pkt->pages[p];
-			bvec[f].bv_offset = offs;
-		} else {
-			BUG_ON(bvec[f].bv_offset != offs);
-		}
-		offs += CD_FRAMESIZE;
-		if (offs >= PAGE_SIZE) {
-			offs = 0;
-			p++;
-		}
-	}
-}
-
 static void pkt_end_io_read(struct bio *bio)
 {
 	struct packet_data *pkt = bio->bi_private;
@@ -1298,7 +1265,6 @@ try_next_bio:
 static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt)
 {
 	int f;
-	struct bio_vec *bvec = pkt->w_bio->bi_io_vec;
 
 	bio_reset(pkt->w_bio);
 	pkt->w_bio->bi_iter.bi_sector = pkt->sector;
@@ -1308,9 +1274,10 @@ static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt)
 
 	/* XXX: locking? */
 	for (f = 0; f < pkt->frames; f++) {
-		bvec[f].bv_page = pkt->pages[(f * CD_FRAMESIZE) / PAGE_SIZE];
-		bvec[f].bv_offset = (f * CD_FRAMESIZE) % PAGE_SIZE;
-		if (!bio_add_page(pkt->w_bio, bvec[f].bv_page, CD_FRAMESIZE, bvec[f].bv_offset))
+		struct page *page = pkt->pages[(f * CD_FRAMESIZE) / PAGE_SIZE];
+		unsigned offset = (f * CD_FRAMESIZE) % PAGE_SIZE;
+
+		if (!bio_add_page(pkt->w_bio, page, CD_FRAMESIZE, offset))
 			BUG();
 	}
 	pkt_dbg(2, pd, "vcnt=%d\n", pkt->w_bio->bi_vcnt);
@@ -1327,12 +1294,10 @@ static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt)
 	pkt_dbg(2, pd, "Writing %d frames for zone %llx\n",
 		pkt->write_size, (unsigned long long)pkt->sector);
 
-	if (test_bit(PACKET_MERGE_SEGS, &pd->flags) || (pkt->write_size < pkt->frames)) {
-		pkt_make_local_copy(pkt, bvec);
+	if (test_bit(PACKET_MERGE_SEGS, &pd->flags) || (pkt->write_size < pkt->frames))
 		pkt->cache_valid = 1;
-	} else {
+	else
 		pkt->cache_valid = 0;
-	}
 
 	/* Start the write request */
 	atomic_set(&pkt->io_wait, 1);
diff --git a/drivers/block/skd_main.c b/drivers/block/skd_main.c
index 3822eae102db..abf805e332e2 100644
--- a/drivers/block/skd_main.c
+++ b/drivers/block/skd_main.c
@@ -36,7 +36,6 @@
 #include <linux/scatterlist.h>
 #include <linux/version.h>
 #include <linux/err.h>
-#include <linux/scatterlist.h>
 #include <linux/aer.h>
 #include <linux/ctype.h>
 #include <linux/wait.h>
@@ -270,8 +269,6 @@ struct skd_device {
 	resource_size_t mem_phys[SKD_MAX_BARS];
 	u32 mem_size[SKD_MAX_BARS];
 
-	skd_irq_type_t irq_type;
-	u32 msix_count;
 	struct skd_msix_entry *msix_entries;
 
 	struct pci_dev *pdev;
@@ -2138,12 +2135,8 @@ static void skd_send_fitmsg(struct skd_device *skdev,
 		u8 *bp = (u8 *)skmsg->msg_buf;
 		int i;
 		for (i = 0; i < skmsg->length; i += 8) {
-			pr_debug("%s:%s:%d msg[%2d] %02x %02x %02x %02x "
-				 "%02x %02x %02x %02x\n",
-				 skdev->name, __func__, __LINE__,
-				 i, bp[i + 0], bp[i + 1], bp[i + 2],
-				 bp[i + 3], bp[i + 4], bp[i + 5],
-				 bp[i + 6], bp[i + 7]);
+			pr_debug("%s:%s:%d msg[%2d] %8ph\n",
+				 skdev->name, __func__, __LINE__, i, &bp[i]);
 			if (i == 0)
 				i = 64 - 8;
 		}
@@ -2164,7 +2157,6 @@ static void skd_send_fitmsg(struct skd_device *skdev,
 		qcmd |= FIT_QCMD_MSGSIZE_64;
 
 	SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
-
 }
 
 static void skd_send_special_fitmsg(struct skd_device *skdev,
@@ -2177,11 +2169,8 @@ static void skd_send_special_fitmsg(struct skd_device *skdev,
 		int i;
 
 		for (i = 0; i < SKD_N_SPECIAL_FITMSG_BYTES; i += 8) {
-			pr_debug("%s:%s:%d  spcl[%2d] %02x %02x %02x %02x  "
-				 "%02x %02x %02x %02x\n",
-				 skdev->name, __func__, __LINE__, i,
-				 bp[i + 0], bp[i + 1], bp[i + 2], bp[i + 3],
-				 bp[i + 4], bp[i + 5], bp[i + 6], bp[i + 7]);
+			pr_debug("%s:%s:%d  spcl[%2d] %8ph\n",
+				 skdev->name, __func__, __LINE__, i, &bp[i]);
 			if (i == 0)
 				i = 64 - 8;
 		}
@@ -2955,8 +2944,8 @@ static void skd_completion_worker(struct work_struct *work)
 
 static void skd_isr_msg_from_dev(struct skd_device *skdev);
 
-irqreturn_t
-static skd_isr(int irq, void *ptr)
+static irqreturn_t
+skd_isr(int irq, void *ptr)
 {
 	struct skd_device *skdev;
 	u32 intstat;
@@ -3821,10 +3810,6 @@ static irqreturn_t skd_qfull_isr(int irq, void *skd_host_data)
  */
 
 struct skd_msix_entry {
-	int have_irq;
-	u32 vector;
-	u32 entry;
-	struct skd_device *rsp;
 	char isr_name[30];
 };
 
@@ -3853,193 +3838,121 @@ static struct skd_init_msix_entry msix_entries[SKD_MAX_MSIX_COUNT] = {
 	{ "(Queue Full 3)", skd_qfull_isr    },
 };
 
-static void skd_release_msix(struct skd_device *skdev)
-{
-	struct skd_msix_entry *qentry;
-	int i;
-
-	if (skdev->msix_entries) {
-		for (i = 0; i < skdev->msix_count; i++) {
-			qentry = &skdev->msix_entries[i];
-			skdev = qentry->rsp;
-
-			if (qentry->have_irq)
-				devm_free_irq(&skdev->pdev->dev,
-					      qentry->vector, qentry->rsp);
-		}
-
-		kfree(skdev->msix_entries);
-	}
-
-	if (skdev->msix_count)
-		pci_disable_msix(skdev->pdev);
-
-	skdev->msix_count = 0;
-	skdev->msix_entries = NULL;
-}
-
 static int skd_acquire_msix(struct skd_device *skdev)
 {
 	int i, rc;
 	struct pci_dev *pdev = skdev->pdev;
-	struct msix_entry *entries;
-	struct skd_msix_entry *qentry;
-
-	entries = kzalloc(sizeof(struct msix_entry) * SKD_MAX_MSIX_COUNT,
-			  GFP_KERNEL);
-	if (!entries)
-		return -ENOMEM;
-
-	for (i = 0; i < SKD_MAX_MSIX_COUNT; i++)
-		entries[i].entry = i;
 
-	rc = pci_enable_msix_exact(pdev, entries, SKD_MAX_MSIX_COUNT);
-	if (rc) {
+	rc = pci_alloc_irq_vectors(pdev, SKD_MAX_MSIX_COUNT, SKD_MAX_MSIX_COUNT,
+			PCI_IRQ_MSIX);
+	if (rc < 0) {
 		pr_err("(%s): failed to enable MSI-X %d\n",
 		       skd_name(skdev), rc);
-		goto msix_out;
+		goto out;
 	}
 
-	skdev->msix_count = SKD_MAX_MSIX_COUNT;
-	skdev->msix_entries = kzalloc(sizeof(struct skd_msix_entry) *
-				      skdev->msix_count, GFP_KERNEL);
+	skdev->msix_entries = kcalloc(SKD_MAX_MSIX_COUNT,
+			sizeof(struct skd_msix_entry), GFP_KERNEL);
 	if (!skdev->msix_entries) {
 		rc = -ENOMEM;
 		pr_err("(%s): msix table allocation error\n",
 		       skd_name(skdev));
-		goto msix_out;
-	}
-
-	for (i = 0; i < skdev->msix_count; i++) {
-		qentry = &skdev->msix_entries[i];
-		qentry->vector = entries[i].vector;
-		qentry->entry = entries[i].entry;
-		qentry->rsp = NULL;
-		qentry->have_irq = 0;
-		pr_debug("%s:%s:%d %s: <%s> msix (%d) vec %d, entry %x\n",
-			 skdev->name, __func__, __LINE__,
-			 pci_name(pdev), skdev->name,
-			 i, qentry->vector, qentry->entry);
+		goto out;
 	}
 
 	/* Enable MSI-X vectors for the base queue */
-	for (i = 0; i < skdev->msix_count; i++) {
-		qentry = &skdev->msix_entries[i];
+	for (i = 0; i < SKD_MAX_MSIX_COUNT; i++) {
+		struct skd_msix_entry *qentry = &skdev->msix_entries[i];
+
 		snprintf(qentry->isr_name, sizeof(qentry->isr_name),
 			 "%s%d-msix %s", DRV_NAME, skdev->devno,
 			 msix_entries[i].name);
-		rc = devm_request_irq(&skdev->pdev->dev, qentry->vector,
-				      msix_entries[i].handler, 0,
-				      qentry->isr_name, skdev);
+
+		rc = devm_request_irq(&skdev->pdev->dev,
+				pci_irq_vector(skdev->pdev, i),
+				msix_entries[i].handler, 0,
+				qentry->isr_name, skdev);
 		if (rc) {
 			pr_err("(%s): Unable to register(%d) MSI-X "
 			       "handler %d: %s\n",
 			       skd_name(skdev), rc, i, qentry->isr_name);
 			goto msix_out;
-		} else {
-			qentry->have_irq = 1;
-			qentry->rsp = skdev;
 		}
 	}
+
 	pr_debug("%s:%s:%d %s: <%s> msix %d irq(s) enabled\n",
 		 skdev->name, __func__, __LINE__,
-		 pci_name(pdev), skdev->name, skdev->msix_count);
+		 pci_name(pdev), skdev->name, SKD_MAX_MSIX_COUNT);
 	return 0;
 
 msix_out:
-	if (entries)
-		kfree(entries);
-	skd_release_msix(skdev);
+	while (--i >= 0)
+		devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i), skdev);
+out:
+	kfree(skdev->msix_entries);
+	skdev->msix_entries = NULL;
 	return rc;
 }
 
 static int skd_acquire_irq(struct skd_device *skdev)
 {
+	struct pci_dev *pdev = skdev->pdev;
+	unsigned int irq_flag = PCI_IRQ_LEGACY;
 	int rc;
-	struct pci_dev *pdev;
-
-	pdev = skdev->pdev;
-	skdev->msix_count = 0;
 
-RETRY_IRQ_TYPE:
-	switch (skdev->irq_type) {
-	case SKD_IRQ_MSIX:
+	if (skd_isr_type == SKD_IRQ_MSIX) {
 		rc = skd_acquire_msix(skdev);
 		if (!rc)
-			pr_info("(%s): MSI-X %d irqs enabled\n",
-			       skd_name(skdev), skdev->msix_count);
-		else {
-			pr_err(
-			       "(%s): failed to enable MSI-X, re-trying with MSI %d\n",
-			       skd_name(skdev), rc);
-			skdev->irq_type = SKD_IRQ_MSI;
-			goto RETRY_IRQ_TYPE;
-		}
-		break;
-	case SKD_IRQ_MSI:
-		snprintf(skdev->isr_name, sizeof(skdev->isr_name), "%s%d-msi",
-			 DRV_NAME, skdev->devno);
-		rc = pci_enable_msi_range(pdev, 1, 1);
-		if (rc > 0) {
-			rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr, 0,
-					      skdev->isr_name, skdev);
-			if (rc) {
-				pci_disable_msi(pdev);
-				pr_err(
-				       "(%s): failed to allocate the MSI interrupt %d\n",
-				       skd_name(skdev), rc);
-				goto RETRY_IRQ_LEGACY;
-			}
-			pr_info("(%s): MSI irq %d enabled\n",
-			       skd_name(skdev), pdev->irq);
-		} else {
-RETRY_IRQ_LEGACY:
-			pr_err(
-			       "(%s): failed to enable MSI, re-trying with LEGACY %d\n",
-			       skd_name(skdev), rc);
-			skdev->irq_type = SKD_IRQ_LEGACY;
-			goto RETRY_IRQ_TYPE;
-		}
-		break;
-	case SKD_IRQ_LEGACY:
-		snprintf(skdev->isr_name, sizeof(skdev->isr_name),
-			 "%s%d-legacy", DRV_NAME, skdev->devno);
-		rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr,
-				      IRQF_SHARED, skdev->isr_name, skdev);
-		if (!rc)
-			pr_info("(%s): LEGACY irq %d enabled\n",
-			       skd_name(skdev), pdev->irq);
-		else
-			pr_err("(%s): request LEGACY irq error %d\n",
-			       skd_name(skdev), rc);
-		break;
-	default:
-		pr_info("(%s): irq_type %d invalid, re-set to %d\n",
-		       skd_name(skdev), skdev->irq_type, SKD_IRQ_DEFAULT);
-		skdev->irq_type = SKD_IRQ_LEGACY;
-		goto RETRY_IRQ_TYPE;
+			return 0;
+
+		pr_err("(%s): failed to enable MSI-X, re-trying with MSI %d\n",
+		       skd_name(skdev), rc);
 	}
-	return rc;
+
+	snprintf(skdev->isr_name, sizeof(skdev->isr_name), "%s%d", DRV_NAME,
+			skdev->devno);
+
+	if (skd_isr_type != SKD_IRQ_LEGACY)
+		irq_flag |= PCI_IRQ_MSI;
+	rc = pci_alloc_irq_vectors(pdev, 1, 1, irq_flag);
+	if (rc < 0) {
+		pr_err("(%s): failed to allocate the MSI interrupt %d\n",
+			skd_name(skdev), rc);
+		return rc;
+	}
+
+	rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr,
+			pdev->msi_enabled ? 0 : IRQF_SHARED,
+			skdev->isr_name, skdev);
+	if (rc) {
+		pci_free_irq_vectors(pdev);
+		pr_err("(%s): failed to allocate interrupt %d\n",
+			skd_name(skdev), rc);
+		return rc;
+	}
+
+	return 0;
 }
 
 static void skd_release_irq(struct skd_device *skdev)
 {
-	switch (skdev->irq_type) {
-	case SKD_IRQ_MSIX:
-		skd_release_msix(skdev);
-		break;
-	case SKD_IRQ_MSI:
-		devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
-		pci_disable_msi(skdev->pdev);
-		break;
-	case SKD_IRQ_LEGACY:
-		devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
-		break;
-	default:
-		pr_err("(%s): wrong irq type %d!",
-		       skd_name(skdev), skdev->irq_type);
-		break;
+	struct pci_dev *pdev = skdev->pdev;
+
+	if (skdev->msix_entries) {
+		int i;
+
+		for (i = 0; i < SKD_MAX_MSIX_COUNT; i++) {
+			devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i),
+					skdev);
+		}
+
+		kfree(skdev->msix_entries);
+		skdev->msix_entries = NULL;
+	} else {
+		devm_free_irq(&pdev->dev, pdev->irq, skdev);
 	}
+
+	pci_free_irq_vectors(pdev);
 }
 
 /*
@@ -4402,7 +4315,6 @@ static struct skd_device *skd_construct(struct pci_dev *pdev)
 	skdev->pdev = pdev;
 	skdev->devno = skd_next_devno++;
 	skdev->major = blk_major;
-	skdev->irq_type = skd_isr_type;
 	sprintf(skdev->name, DRV_NAME "%d", skdev->devno);
 	skdev->dev_max_queue_depth = 0;
 
diff --git a/drivers/block/umem.c b/drivers/block/umem.c
index be90e15854ed..46f4c719fed9 100644
--- a/drivers/block/umem.c
+++ b/drivers/block/umem.c
@@ -535,7 +535,7 @@ static blk_qc_t mm_make_request(struct request_queue *q, struct bio *bio)
 	*card->biotail = bio;
 	bio->bi_next = NULL;
 	card->biotail = &bio->bi_next;
-	if (bio->bi_opf & REQ_SYNC || !mm_check_plugged(card))
+	if (op_is_sync(bio->bi_opf) || !mm_check_plugged(card))
 		activate(card);
 	spin_unlock_irq(&card->lock);
 
diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c
index 4a80ee752597..726c32e35db9 100644
--- a/drivers/block/xen-blkback/blkback.c
+++ b/drivers/block/xen-blkback/blkback.c
@@ -1253,14 +1253,14 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
 	case BLKIF_OP_WRITE:
 		ring->st_wr_req++;
 		operation = REQ_OP_WRITE;
-		operation_flags = WRITE_ODIRECT;
+		operation_flags = REQ_SYNC | REQ_IDLE;
 		break;
 	case BLKIF_OP_WRITE_BARRIER:
 		drain = true;
 	case BLKIF_OP_FLUSH_DISKCACHE:
 		ring->st_f_req++;
 		operation = REQ_OP_WRITE;
-		operation_flags = WRITE_FLUSH;
+		operation_flags = REQ_PREFLUSH;
 		break;
 	default:
 		operation = 0; /* make gcc happy */
@@ -1272,7 +1272,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
 	nseg = req->operation == BLKIF_OP_INDIRECT ?
 	       req->u.indirect.nr_segments : req->u.rw.nr_segments;
 
-	if (unlikely(nseg == 0 && operation_flags != WRITE_FLUSH) ||
+	if (unlikely(nseg == 0 && operation_flags != REQ_PREFLUSH) ||
 	    unlikely((req->operation != BLKIF_OP_INDIRECT) &&
 		     (nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) ||
 	    unlikely((req->operation == BLKIF_OP_INDIRECT) &&
@@ -1334,7 +1334,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
 	}
 
 	/* Wait on all outstanding I/O's and once that has been completed
-	 * issue the WRITE_FLUSH.
+	 * issue the flush.
 	 */
 	if (drain)
 		xen_blk_drain_io(pending_req->ring);
@@ -1380,7 +1380,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
 
 	/* This will be hit if the operation was a flush or discard. */
 	if (!bio) {
-		BUG_ON(operation_flags != WRITE_FLUSH);
+		BUG_ON(operation_flags != REQ_PREFLUSH);
 
 		bio = bio_alloc(GFP_KERNEL, 0);
 		if (unlikely(bio == NULL))
diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c
index 3cc6d1d86f1e..415e79b69d34 100644
--- a/drivers/block/xen-blkback/xenbus.c
+++ b/drivers/block/xen-blkback/xenbus.c
@@ -533,13 +533,11 @@ static void xen_blkbk_discard(struct xenbus_transaction xbt, struct backend_info
 	struct xenbus_device *dev = be->dev;
 	struct xen_blkif *blkif = be->blkif;
 	int err;
-	int state = 0, discard_enable;
+	int state = 0;
 	struct block_device *bdev = be->blkif->vbd.bdev;
 	struct request_queue *q = bdev_get_queue(bdev);
 
-	err = xenbus_scanf(XBT_NIL, dev->nodename, "discard-enable", "%d",
-			   &discard_enable);
-	if (err == 1 && !discard_enable)
+	if (!xenbus_read_unsigned(dev->nodename, "discard-enable", 1))
 		return;
 
 	if (blk_queue_discard(q)) {
@@ -1039,30 +1037,24 @@ static int connect_ring(struct backend_info *be)
 		xenbus_dev_fatal(dev, err, "unknown fe protocol %s", protocol);
 		return -ENOSYS;
 	}
-	err = xenbus_scanf(XBT_NIL, dev->otherend,
-			   "feature-persistent", "%u", &pers_grants);
-	if (err <= 0)
-		pers_grants = 0;
-
+	pers_grants = xenbus_read_unsigned(dev->otherend, "feature-persistent",
+					   0);
 	be->blkif->vbd.feature_gnt_persistent = pers_grants;
 	be->blkif->vbd.overflow_max_grants = 0;
 
 	/*
 	 * Read the number of hardware queues from frontend.
 	 */
-	err = xenbus_scanf(XBT_NIL, dev->otherend, "multi-queue-num-queues",
-			   "%u", &requested_num_queues);
-	if (err < 0) {
-		requested_num_queues = 1;
-	} else {
-		if (requested_num_queues > xenblk_max_queues
-		    || requested_num_queues == 0) {
-			/* Buggy or malicious guest. */
-			xenbus_dev_fatal(dev, err,
-					"guest requested %u queues, exceeding the maximum of %u.",
-					requested_num_queues, xenblk_max_queues);
-			return -ENOSYS;
-		}
+	requested_num_queues = xenbus_read_unsigned(dev->otherend,
+						    "multi-queue-num-queues",
+						    1);
+	if (requested_num_queues > xenblk_max_queues
+	    || requested_num_queues == 0) {
+		/* Buggy or malicious guest. */
+		xenbus_dev_fatal(dev, err,
+				"guest requested %u queues, exceeding the maximum of %u.",
+				requested_num_queues, xenblk_max_queues);
+		return -ENOSYS;
 	}
 	be->blkif->nr_rings = requested_num_queues;
 	if (xen_blkif_alloc_rings(be->blkif))
diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c
index 9908597c5209..b2bdfa81f929 100644
--- a/drivers/block/xen-blkfront.c
+++ b/drivers/block/xen-blkfront.c
@@ -1758,17 +1758,13 @@ static int talk_to_blkback(struct xenbus_device *dev,
 	const char *message = NULL;
 	struct xenbus_transaction xbt;
 	int err;
-	unsigned int i, max_page_order = 0;
-	unsigned int ring_page_order = 0;
+	unsigned int i, max_page_order;
+	unsigned int ring_page_order;
 
-	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-			   "max-ring-page-order", "%u", &max_page_order);
-	if (err != 1)
-		info->nr_ring_pages = 1;
-	else {
-		ring_page_order = min(xen_blkif_max_ring_order, max_page_order);
-		info->nr_ring_pages = 1 << ring_page_order;
-	}
+	max_page_order = xenbus_read_unsigned(info->xbdev->otherend,
+					      "max-ring-page-order", 0);
+	ring_page_order = min(xen_blkif_max_ring_order, max_page_order);
+	info->nr_ring_pages = 1 << ring_page_order;
 
 	for (i = 0; i < info->nr_rings; i++) {
 		struct blkfront_ring_info *rinfo = &info->rinfo[i];
@@ -1877,18 +1873,14 @@ again:
 
 static int negotiate_mq(struct blkfront_info *info)
 {
-	unsigned int backend_max_queues = 0;
-	int err;
+	unsigned int backend_max_queues;
 	unsigned int i;
 
 	BUG_ON(info->nr_rings);
 
 	/* Check if backend supports multiple queues. */
-	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-			   "multi-queue-max-queues", "%u", &backend_max_queues);
-	if (err < 0)
-		backend_max_queues = 1;
-
+	backend_max_queues = xenbus_read_unsigned(info->xbdev->otherend,
+						  "multi-queue-max-queues", 1);
 	info->nr_rings = min(backend_max_queues, xen_blkif_max_queues);
 	/* We need at least one ring. */
 	if (!info->nr_rings)
@@ -2043,8 +2035,9 @@ static int blkif_recover(struct blkfront_info *info)
 		/* Requeue pending requests (flush or discard) */
 		list_del_init(&req->queuelist);
 		BUG_ON(req->nr_phys_segments > segs);
-		blk_mq_requeue_request(req);
+		blk_mq_requeue_request(req, false);
 	}
+	blk_mq_start_stopped_hw_queues(info->rq, true);
 	blk_mq_kick_requeue_list(info->rq);
 
 	while ((bio = bio_list_pop(&info->bio_list)) != NULL) {
@@ -2195,7 +2188,6 @@ static void blkfront_setup_discard(struct blkfront_info *info)
 	int err;
 	unsigned int discard_granularity;
 	unsigned int discard_alignment;
-	unsigned int discard_secure;
 
 	info->feature_discard = 1;
 	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
@@ -2206,10 +2198,9 @@ static void blkfront_setup_discard(struct blkfront_info *info)
 		info->discard_granularity = discard_granularity;
 		info->discard_alignment = discard_alignment;
 	}
-	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-			   "discard-secure", "%u", &discard_secure);
-	if (err > 0)
-		info->feature_secdiscard = !!discard_secure;
+	info->feature_secdiscard =
+		!!xenbus_read_unsigned(info->xbdev->otherend, "discard-secure",
+				       0);
 }
 
 static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo)
@@ -2301,16 +2292,11 @@ out_of_memory:
  */
 static void blkfront_gather_backend_features(struct blkfront_info *info)
 {
-	int err;
-	int barrier, flush, discard, persistent;
 	unsigned int indirect_segments;
 
 	info->feature_flush = 0;
 	info->feature_fua = 0;
 
-	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-			   "feature-barrier", "%d", &barrier);
-
 	/*
 	 * If there's no "feature-barrier" defined, then it means
 	 * we're dealing with a very old backend which writes
@@ -2318,7 +2304,7 @@ static void blkfront_gather_backend_features(struct blkfront_info *info)
 	 *
 	 * If there are barriers, then we use flush.
 	 */
-	if (err > 0 && barrier) {
+	if (xenbus_read_unsigned(info->xbdev->otherend, "feature-barrier", 0)) {
 		info->feature_flush = 1;
 		info->feature_fua = 1;
 	}
@@ -2327,35 +2313,23 @@ static void blkfront_gather_backend_features(struct blkfront_info *info)
 	 * And if there is "feature-flush-cache" use that above
 	 * barriers.
 	 */
-	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-			   "feature-flush-cache", "%d", &flush);
-
-	if (err > 0 && flush) {
+	if (xenbus_read_unsigned(info->xbdev->otherend, "feature-flush-cache",
+				 0)) {
 		info->feature_flush = 1;
 		info->feature_fua = 0;
 	}
 
-	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-			   "feature-discard", "%d", &discard);
-
-	if (err > 0 && discard)
+	if (xenbus_read_unsigned(info->xbdev->otherend, "feature-discard", 0))
 		blkfront_setup_discard(info);
 
-	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-			   "feature-persistent", "%d", &persistent);
-	if (err <= 0)
-		info->feature_persistent = 0;
-	else
-		info->feature_persistent = persistent;
+	info->feature_persistent =
+		xenbus_read_unsigned(info->xbdev->otherend,
+				     "feature-persistent", 0);
 
-	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-			   "feature-max-indirect-segments", "%u",
-			   &indirect_segments);
-	if (err <= 0)
-		info->max_indirect_segments = 0;
-	else
-		info->max_indirect_segments = min(indirect_segments,
-						  xen_blkif_max_segments);
+	indirect_segments = xenbus_read_unsigned(info->xbdev->otherend,
+					"feature-max-indirect-segments", 0);
+	info->max_indirect_segments = min(indirect_segments,
+					  xen_blkif_max_segments);
 }
 
 /*
@@ -2420,11 +2394,9 @@ static void blkfront_connect(struct blkfront_info *info)
 	 * provide this. Assume physical sector size to be the same as
 	 * sector_size in that case.
 	 */
-	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-			   "physical-sector-size", "%u", &physical_sector_size);
-	if (err != 1)
-		physical_sector_size = sector_size;
-
+	physical_sector_size = xenbus_read_unsigned(info->xbdev->otherend,
+						    "physical-sector-size",
+						    sector_size);
 	blkfront_gather_backend_features(info);
 	for (i = 0; i < info->nr_rings; i++) {
 		err = blkfront_setup_indirect(&info->rinfo[i]);
diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile
index b1fc29a697b7..80627187c8b6 100644
--- a/drivers/bluetooth/Makefile
+++ b/drivers/bluetooth/Makefile
@@ -40,5 +40,3 @@ hci_uart-$(CONFIG_BT_HCIUART_QCA)	+= hci_qca.o
 hci_uart-$(CONFIG_BT_HCIUART_AG6XX)	+= hci_ag6xx.o
 hci_uart-$(CONFIG_BT_HCIUART_MRVL)	+= hci_mrvl.o
 hci_uart-objs				:= $(hci_uart-y)
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index 78751057164a..b9e8cfc93c7e 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -150,6 +150,13 @@ config TEGRA_ACONNECT
 	  Driver for the Tegra ACONNECT bus which is used to interface with
 	  the devices inside the Audio Processing Engine (APE) for Tegra210.
 
+config TEGRA_GMI
+	tristate "Tegra Generic Memory Interface bus driver"
+	depends on ARCH_TEGRA
+	help
+	  Driver for the Tegra Generic Memory Interface bus which can be used
+	  to attach devices such as NOR, UART, FPGA and more.
+
 config UNIPHIER_SYSTEM_BUS
 	tristate "UniPhier System Bus driver"
 	depends on ARCH_UNIPHIER && OF
@@ -167,4 +174,13 @@ config VEXPRESS_CONFIG
 	help
 	  Platform configuration infrastructure for the ARM Ltd.
 	  Versatile Express.
+
+config DA8XX_MSTPRI
+	bool "TI da8xx master peripheral priority driver"
+	depends on ARCH_DAVINCI_DA8XX
+	help
+	  Driver for Texas Instruments da8xx master peripheral priority
+	  configuration. Allows to adjust the priorities of all master
+	  peripherals.
+
 endmenu
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
index c6cfa6b2606e..cc6364bec054 100644
--- a/drivers/bus/Makefile
+++ b/drivers/bus/Makefile
@@ -19,5 +19,8 @@ obj-$(CONFIG_QCOM_EBI2)		+= qcom-ebi2.o
 obj-$(CONFIG_SUNXI_RSB)		+= sunxi-rsb.o
 obj-$(CONFIG_SIMPLE_PM_BUS)	+= simple-pm-bus.o
 obj-$(CONFIG_TEGRA_ACONNECT)	+= tegra-aconnect.o
+obj-$(CONFIG_TEGRA_GMI)		+= tegra-gmi.o
 obj-$(CONFIG_UNIPHIER_SYSTEM_BUS)	+= uniphier-system-bus.o
 obj-$(CONFIG_VEXPRESS_CONFIG)	+= vexpress-config.o
+
+obj-$(CONFIG_DA8XX_MSTPRI)	+= da8xx-mstpri.o
diff --git a/drivers/bus/arm-cci.c b/drivers/bus/arm-cci.c
index 890082315054..231633328dfa 100644
--- a/drivers/bus/arm-cci.c
+++ b/drivers/bus/arm-cci.c
@@ -2190,6 +2190,9 @@ static int cci_probe_ports(struct device_node *np)
 		if (!of_match_node(arm_cci_ctrl_if_matches, cp))
 			continue;
 
+		if (!of_device_is_available(cp))
+			continue;
+
 		i = nb_ace + nb_ace_lite;
 
 		if (i >= nb_cci_ports)
@@ -2232,6 +2235,13 @@ static int cci_probe_ports(struct device_node *np)
 		ports[i].dn = cp;
 	}
 
+	/*
+	 * If there is no CCI port that is under kernel control
+	 * return early and report probe status.
+	 */
+	if (!nb_ace && !nb_ace_lite)
+		return -ENODEV;
+
 	 /* initialize a stashed array of ACE ports to speed-up look-up */
 	cci_ace_init_ports();
 
diff --git a/drivers/bus/da8xx-mstpri.c b/drivers/bus/da8xx-mstpri.c
new file mode 100644
index 000000000000..063397f2c0db
--- /dev/null
+++ b/drivers/bus/da8xx-mstpri.c
@@ -0,0 +1,267 @@
+/*
+ * TI da8xx master peripheral priority driver
+ *
+ * Copyright (C) 2016 BayLibre SAS
+ *
+ * Author:
+ *   Bartosz Golaszewski <bgolaszewski@baylibre.com.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+#include <linux/regmap.h>
+
+/*
+ * REVISIT: Linux doesn't have a good framework for the kind of performance
+ * knobs this driver controls. We can't use device tree properties as it deals
+ * with hardware configuration rather than description. We also don't want to
+ * commit to maintaining some random sysfs attributes.
+ *
+ * For now we just hardcode the register values for the boards that need
+ * some changes (as is the case for the LCD controller on da850-lcdk - the
+ * first board we support here). When linux gets an appropriate framework,
+ * we'll easily convert the driver to it.
+ */
+
+#define DA8XX_MSTPRI0_OFFSET		0
+#define DA8XX_MSTPRI1_OFFSET		4
+#define DA8XX_MSTPRI2_OFFSET		8
+
+enum {
+	DA8XX_MSTPRI_ARM_I = 0,
+	DA8XX_MSTPRI_ARM_D,
+	DA8XX_MSTPRI_UPP,
+	DA8XX_MSTPRI_SATA,
+	DA8XX_MSTPRI_PRU0,
+	DA8XX_MSTPRI_PRU1,
+	DA8XX_MSTPRI_EDMA30TC0,
+	DA8XX_MSTPRI_EDMA30TC1,
+	DA8XX_MSTPRI_EDMA31TC0,
+	DA8XX_MSTPRI_VPIF_DMA_0,
+	DA8XX_MSTPRI_VPIF_DMA_1,
+	DA8XX_MSTPRI_EMAC,
+	DA8XX_MSTPRI_USB0CFG,
+	DA8XX_MSTPRI_USB0CDMA,
+	DA8XX_MSTPRI_UHPI,
+	DA8XX_MSTPRI_USB1,
+	DA8XX_MSTPRI_LCDC,
+};
+
+struct da8xx_mstpri_descr {
+	int reg;
+	int shift;
+	int mask;
+};
+
+static const struct da8xx_mstpri_descr da8xx_mstpri_priority_list[] = {
+	[DA8XX_MSTPRI_ARM_I] = {
+		.reg = DA8XX_MSTPRI0_OFFSET,
+		.shift = 0,
+		.mask = 0x0000000f,
+	},
+	[DA8XX_MSTPRI_ARM_D] = {
+		.reg = DA8XX_MSTPRI0_OFFSET,
+		.shift = 4,
+		.mask = 0x000000f0,
+	},
+	[DA8XX_MSTPRI_UPP] = {
+		.reg = DA8XX_MSTPRI0_OFFSET,
+		.shift = 16,
+		.mask = 0x000f0000,
+	},
+	[DA8XX_MSTPRI_SATA] = {
+		.reg = DA8XX_MSTPRI0_OFFSET,
+		.shift = 20,
+		.mask = 0x00f00000,
+	},
+	[DA8XX_MSTPRI_PRU0] = {
+		.reg = DA8XX_MSTPRI1_OFFSET,
+		.shift = 0,
+		.mask = 0x0000000f,
+	},
+	[DA8XX_MSTPRI_PRU1] = {
+		.reg = DA8XX_MSTPRI1_OFFSET,
+		.shift = 4,
+		.mask = 0x000000f0,
+	},
+	[DA8XX_MSTPRI_EDMA30TC0] = {
+		.reg = DA8XX_MSTPRI1_OFFSET,
+		.shift = 8,
+		.mask = 0x00000f00,
+	},
+	[DA8XX_MSTPRI_EDMA30TC1] = {
+		.reg = DA8XX_MSTPRI1_OFFSET,
+		.shift = 12,
+		.mask = 0x0000f000,
+	},
+	[DA8XX_MSTPRI_EDMA31TC0] = {
+		.reg = DA8XX_MSTPRI1_OFFSET,
+		.shift = 16,
+		.mask = 0x000f0000,
+	},
+	[DA8XX_MSTPRI_VPIF_DMA_0] = {
+		.reg = DA8XX_MSTPRI1_OFFSET,
+		.shift = 24,
+		.mask = 0x0f000000,
+	},
+	[DA8XX_MSTPRI_VPIF_DMA_1] = {
+		.reg = DA8XX_MSTPRI1_OFFSET,
+		.shift = 28,
+		.mask = 0xf0000000,
+	},
+	[DA8XX_MSTPRI_EMAC] = {
+		.reg = DA8XX_MSTPRI2_OFFSET,
+		.shift = 0,
+		.mask = 0x0000000f,
+	},
+	[DA8XX_MSTPRI_USB0CFG] = {
+		.reg = DA8XX_MSTPRI2_OFFSET,
+		.shift = 8,
+		.mask = 0x00000f00,
+	},
+	[DA8XX_MSTPRI_USB0CDMA] = {
+		.reg = DA8XX_MSTPRI2_OFFSET,
+		.shift = 12,
+		.mask = 0x0000f000,
+	},
+	[DA8XX_MSTPRI_UHPI] = {
+		.reg = DA8XX_MSTPRI2_OFFSET,
+		.shift = 20,
+		.mask = 0x00f00000,
+	},
+	[DA8XX_MSTPRI_USB1] = {
+		.reg = DA8XX_MSTPRI2_OFFSET,
+		.shift = 24,
+		.mask = 0x0f000000,
+	},
+	[DA8XX_MSTPRI_LCDC] = {
+		.reg = DA8XX_MSTPRI2_OFFSET,
+		.shift = 28,
+		.mask = 0xf0000000,
+	},
+};
+
+struct da8xx_mstpri_priority {
+	int which;
+	u32 val;
+};
+
+struct da8xx_mstpri_board_priorities {
+	const char *board;
+	const struct da8xx_mstpri_priority *priorities;
+	size_t numprio;
+};
+
+/*
+ * Default memory settings of da850 do not meet the throughput/latency
+ * requirements of tilcdc. This results in the image displayed being
+ * incorrect and the following warning being displayed by the LCDC
+ * drm driver:
+ *
+ *   tilcdc da8xx_lcdc.0: tilcdc_crtc_irq(0x00000020): FIFO underfow
+ */
+static const struct da8xx_mstpri_priority da850_lcdk_priorities[] = {
+	{
+		.which = DA8XX_MSTPRI_LCDC,
+		.val = 0,
+	},
+	{
+		.which = DA8XX_MSTPRI_EDMA30TC1,
+		.val = 0,
+	},
+	{
+		.which = DA8XX_MSTPRI_EDMA30TC0,
+		.val = 1,
+	},
+};
+
+static const struct da8xx_mstpri_board_priorities da8xx_mstpri_board_confs[] = {
+	{
+		.board = "ti,da850-lcdk",
+		.priorities = da850_lcdk_priorities,
+		.numprio = ARRAY_SIZE(da850_lcdk_priorities),
+	},
+};
+
+static const struct da8xx_mstpri_board_priorities *
+da8xx_mstpri_get_board_prio(void)
+{
+	const struct da8xx_mstpri_board_priorities *board_prio;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(da8xx_mstpri_board_confs); i++) {
+		board_prio = &da8xx_mstpri_board_confs[i];
+
+		if (of_machine_is_compatible(board_prio->board))
+			return board_prio;
+	}
+
+	return NULL;
+}
+
+static int da8xx_mstpri_probe(struct platform_device *pdev)
+{
+	const struct da8xx_mstpri_board_priorities *prio_list;
+	const struct da8xx_mstpri_descr *prio_descr;
+	const struct da8xx_mstpri_priority *prio;
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	void __iomem *mstpri;
+	u32 reg;
+	int i;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	mstpri = devm_ioremap_resource(dev, res);
+	if (IS_ERR(mstpri)) {
+		dev_err(dev, "unable to map MSTPRI registers\n");
+		return PTR_ERR(mstpri);
+	}
+
+	prio_list = da8xx_mstpri_get_board_prio();
+	if (!prio_list) {
+		dev_err(dev, "no master priorities defined for this board\n");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < prio_list->numprio; i++) {
+		prio = &prio_list->priorities[i];
+		prio_descr = &da8xx_mstpri_priority_list[prio->which];
+
+		if (prio_descr->reg + sizeof(u32) > resource_size(res)) {
+			dev_warn(dev, "register offset out of range\n");
+			continue;
+		}
+
+		reg = readl(mstpri + prio_descr->reg);
+		reg &= ~prio_descr->mask;
+		reg |= prio->val << prio_descr->shift;
+
+		writel(reg, mstpri + prio_descr->reg);
+	}
+
+	return 0;
+}
+
+static const struct of_device_id da8xx_mstpri_of_match[] = {
+	{ .compatible = "ti,da850-mstpri", },
+	{ },
+};
+
+static struct platform_driver da8xx_mstpri_driver = {
+	.probe = da8xx_mstpri_probe,
+	.driver = {
+		.name = "da8xx-mstpri",
+		.of_match_table = da8xx_mstpri_of_match,
+	},
+};
+module_platform_driver(da8xx_mstpri_driver);
+
+MODULE_AUTHOR("Bartosz Golaszewski <bgolaszewski@baylibre.com>");
+MODULE_DESCRIPTION("TI da8xx master peripheral priority driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/bus/tegra-gmi.c b/drivers/bus/tegra-gmi.c
new file mode 100644
index 000000000000..a6570789f7af
--- /dev/null
+++ b/drivers/bus/tegra-gmi.c
@@ -0,0 +1,284 @@
+/*
+ * Driver for NVIDIA Generic Memory Interface
+ *
+ * Copyright (C) 2016 Host Mobility AB. All rights reserved.
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/reset.h>
+
+#define TEGRA_GMI_CONFIG		0x00
+#define TEGRA_GMI_CONFIG_GO		BIT(31)
+#define TEGRA_GMI_BUS_WIDTH_32BIT	BIT(30)
+#define TEGRA_GMI_MUX_MODE		BIT(28)
+#define TEGRA_GMI_RDY_BEFORE_DATA	BIT(24)
+#define TEGRA_GMI_RDY_ACTIVE_HIGH	BIT(23)
+#define TEGRA_GMI_ADV_ACTIVE_HIGH	BIT(22)
+#define TEGRA_GMI_OE_ACTIVE_HIGH	BIT(21)
+#define TEGRA_GMI_CS_ACTIVE_HIGH	BIT(20)
+#define TEGRA_GMI_CS_SELECT(x)		((x & 0x7) << 4)
+
+#define TEGRA_GMI_TIMING0		0x10
+#define TEGRA_GMI_MUXED_WIDTH(x)	((x & 0xf) << 12)
+#define TEGRA_GMI_HOLD_WIDTH(x)		((x & 0xf) << 8)
+#define TEGRA_GMI_ADV_WIDTH(x)		((x & 0xf) << 4)
+#define TEGRA_GMI_CE_WIDTH(x)		(x & 0xf)
+
+#define TEGRA_GMI_TIMING1		0x14
+#define TEGRA_GMI_WE_WIDTH(x)		((x & 0xff) << 16)
+#define TEGRA_GMI_OE_WIDTH(x)		((x & 0xff) << 8)
+#define TEGRA_GMI_WAIT_WIDTH(x)		(x & 0xff)
+
+#define TEGRA_GMI_MAX_CHIP_SELECT	8
+
+struct tegra_gmi {
+	struct device *dev;
+	void __iomem *base;
+	struct clk *clk;
+	struct reset_control *rst;
+
+	u32 snor_config;
+	u32 snor_timing0;
+	u32 snor_timing1;
+};
+
+static int tegra_gmi_enable(struct tegra_gmi *gmi)
+{
+	int err;
+
+	err = clk_prepare_enable(gmi->clk);
+	if (err < 0) {
+		dev_err(gmi->dev, "failed to enable clock: %d\n", err);
+		return err;
+	}
+
+	reset_control_assert(gmi->rst);
+	usleep_range(2000, 4000);
+	reset_control_deassert(gmi->rst);
+
+	writel(gmi->snor_timing0, gmi->base + TEGRA_GMI_TIMING0);
+	writel(gmi->snor_timing1, gmi->base + TEGRA_GMI_TIMING1);
+
+	gmi->snor_config |= TEGRA_GMI_CONFIG_GO;
+	writel(gmi->snor_config, gmi->base + TEGRA_GMI_CONFIG);
+
+	return 0;
+}
+
+static void tegra_gmi_disable(struct tegra_gmi *gmi)
+{
+	u32 config;
+
+	/* stop GMI operation */
+	config = readl(gmi->base + TEGRA_GMI_CONFIG);
+	config &= ~TEGRA_GMI_CONFIG_GO;
+	writel(config, gmi->base + TEGRA_GMI_CONFIG);
+
+	reset_control_assert(gmi->rst);
+	clk_disable_unprepare(gmi->clk);
+}
+
+static int tegra_gmi_parse_dt(struct tegra_gmi *gmi)
+{
+	struct device_node *child;
+	u32 property, ranges[4];
+	int err;
+
+	child = of_get_next_available_child(gmi->dev->of_node, NULL);
+	if (!child) {
+		dev_err(gmi->dev, "no child nodes found\n");
+		return -ENODEV;
+	}
+
+	/*
+	 * We currently only support one child device due to lack of
+	 * chip-select address decoding. Which means that we only have one
+	 * chip-select line from the GMI controller.
+	 */
+	if (of_get_child_count(gmi->dev->of_node) > 1)
+		dev_warn(gmi->dev, "only one child device is supported.");
+
+	if (of_property_read_bool(child, "nvidia,snor-data-width-32bit"))
+		gmi->snor_config |= TEGRA_GMI_BUS_WIDTH_32BIT;
+
+	if (of_property_read_bool(child, "nvidia,snor-mux-mode"))
+		gmi->snor_config |= TEGRA_GMI_MUX_MODE;
+
+	if (of_property_read_bool(child, "nvidia,snor-rdy-active-before-data"))
+		gmi->snor_config |= TEGRA_GMI_RDY_BEFORE_DATA;
+
+	if (of_property_read_bool(child, "nvidia,snor-rdy-active-high"))
+		gmi->snor_config |= TEGRA_GMI_RDY_ACTIVE_HIGH;
+
+	if (of_property_read_bool(child, "nvidia,snor-adv-active-high"))
+		gmi->snor_config |= TEGRA_GMI_ADV_ACTIVE_HIGH;
+
+	if (of_property_read_bool(child, "nvidia,snor-oe-active-high"))
+		gmi->snor_config |= TEGRA_GMI_OE_ACTIVE_HIGH;
+
+	if (of_property_read_bool(child, "nvidia,snor-cs-active-high"))
+		gmi->snor_config |= TEGRA_GMI_CS_ACTIVE_HIGH;
+
+	/* Decode the CS# */
+	err = of_property_read_u32_array(child, "ranges", ranges, 4);
+	if (err < 0) {
+		/* Invalid binding */
+		if (err == -EOVERFLOW) {
+			dev_err(gmi->dev,
+				"failed to decode CS: invalid ranges length\n");
+			goto error_cs;
+		}
+
+		/*
+		 * If we reach here it means that the child node has an empty
+		 * ranges or it does not exist at all. Attempt to decode the
+		 * CS# from the reg property instead.
+		 */
+		err = of_property_read_u32(child, "reg", &property);
+		if (err < 0) {
+			dev_err(gmi->dev,
+				"failed to decode CS: no reg property found\n");
+			goto error_cs;
+		}
+	} else {
+		property = ranges[1];
+	}
+
+	/* Valid chip selects are CS0-CS7 */
+	if (property >= TEGRA_GMI_MAX_CHIP_SELECT) {
+		dev_err(gmi->dev, "invalid chip select: %d", property);
+		err = -EINVAL;
+		goto error_cs;
+	}
+
+	gmi->snor_config |= TEGRA_GMI_CS_SELECT(property);
+
+	/* The default values that are provided below are reset values */
+	if (!of_property_read_u32(child, "nvidia,snor-muxed-width", &property))
+		gmi->snor_timing0 |= TEGRA_GMI_MUXED_WIDTH(property);
+	else
+		gmi->snor_timing0 |= TEGRA_GMI_MUXED_WIDTH(1);
+
+	if (!of_property_read_u32(child, "nvidia,snor-hold-width", &property))
+		gmi->snor_timing0 |= TEGRA_GMI_HOLD_WIDTH(property);
+	else
+		gmi->snor_timing0 |= TEGRA_GMI_HOLD_WIDTH(1);
+
+	if (!of_property_read_u32(child, "nvidia,snor-adv-width", &property))
+		gmi->snor_timing0 |= TEGRA_GMI_ADV_WIDTH(property);
+	else
+		gmi->snor_timing0 |= TEGRA_GMI_ADV_WIDTH(1);
+
+	if (!of_property_read_u32(child, "nvidia,snor-ce-width", &property))
+		gmi->snor_timing0 |= TEGRA_GMI_CE_WIDTH(property);
+	else
+		gmi->snor_timing0 |= TEGRA_GMI_CE_WIDTH(4);
+
+	if (!of_property_read_u32(child, "nvidia,snor-we-width", &property))
+		gmi->snor_timing1 |= TEGRA_GMI_WE_WIDTH(property);
+	else
+		gmi->snor_timing1 |= TEGRA_GMI_WE_WIDTH(1);
+
+	if (!of_property_read_u32(child, "nvidia,snor-oe-width", &property))
+		gmi->snor_timing1 |= TEGRA_GMI_OE_WIDTH(property);
+	else
+		gmi->snor_timing1 |= TEGRA_GMI_OE_WIDTH(1);
+
+	if (!of_property_read_u32(child, "nvidia,snor-wait-width", &property))
+		gmi->snor_timing1 |= TEGRA_GMI_WAIT_WIDTH(property);
+	else
+		gmi->snor_timing1 |= TEGRA_GMI_WAIT_WIDTH(3);
+
+error_cs:
+	of_node_put(child);
+	return err;
+}
+
+static int tegra_gmi_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct tegra_gmi *gmi;
+	struct resource *res;
+	int err;
+
+	gmi = devm_kzalloc(dev, sizeof(*gmi), GFP_KERNEL);
+	if (!gmi)
+		return -ENOMEM;
+
+	gmi->dev = dev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	gmi->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(gmi->base))
+		return PTR_ERR(gmi->base);
+
+	gmi->clk = devm_clk_get(dev, "gmi");
+	if (IS_ERR(gmi->clk)) {
+		dev_err(dev, "can not get clock\n");
+		return PTR_ERR(gmi->clk);
+	}
+
+	gmi->rst = devm_reset_control_get(dev, "gmi");
+	if (IS_ERR(gmi->rst)) {
+		dev_err(dev, "can not get reset\n");
+		return PTR_ERR(gmi->rst);
+	}
+
+	err = tegra_gmi_parse_dt(gmi);
+	if (err)
+		return err;
+
+	err = tegra_gmi_enable(gmi);
+	if (err < 0)
+		return err;
+
+	err = of_platform_default_populate(dev->of_node, NULL, dev);
+	if (err < 0) {
+		dev_err(dev, "fail to create devices.\n");
+		tegra_gmi_disable(gmi);
+		return err;
+	}
+
+	platform_set_drvdata(pdev, gmi);
+
+	return 0;
+}
+
+static int tegra_gmi_remove(struct platform_device *pdev)
+{
+	struct tegra_gmi *gmi = platform_get_drvdata(pdev);
+
+	of_platform_depopulate(gmi->dev);
+	tegra_gmi_disable(gmi);
+
+	return 0;
+}
+
+static const struct of_device_id tegra_gmi_id_table[] = {
+	{ .compatible = "nvidia,tegra20-gmi", },
+	{ .compatible = "nvidia,tegra30-gmi", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, tegra_gmi_id_table);
+
+static struct platform_driver tegra_gmi_driver = {
+	.probe = tegra_gmi_probe,
+	.remove = tegra_gmi_remove,
+	.driver = {
+		.name		= "tegra-gmi",
+		.of_match_table	= tegra_gmi_id_table,
+	},
+};
+module_platform_driver(tegra_gmi_driver);
+
+MODULE_AUTHOR("Mirza Krak <mirza.krak@gmail.com");
+MODULE_DESCRIPTION("NVIDIA Tegra GMI Bus Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/bus/vexpress-config.c b/drivers/bus/vexpress-config.c
index 9efdf1de4035..493e7b9fc813 100644
--- a/drivers/bus/vexpress-config.c
+++ b/drivers/bus/vexpress-config.c
@@ -171,6 +171,7 @@ static int vexpress_config_populate(struct device_node *node)
 {
 	struct device_node *bridge;
 	struct device *parent;
+	int ret;
 
 	bridge = of_parse_phandle(node, "arm,vexpress,config-bridge", 0);
 	if (!bridge)
@@ -182,7 +183,11 @@ static int vexpress_config_populate(struct device_node *node)
 	if (WARN_ON(!parent))
 		return -ENODEV;
 
-	return of_platform_populate(node, NULL, NULL, parent);
+	ret = of_platform_populate(node, NULL, NULL, parent);
+
+	put_device(parent);
+
+	return ret;
 }
 
 static int __init vexpress_config_init(void)
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
index 45ba878ae025..fde005ef9d36 100644
--- a/drivers/char/Kconfig
+++ b/drivers/char/Kconfig
@@ -17,7 +17,6 @@ config DEVMEM
 
 config DEVKMEM
 	bool "/dev/kmem virtual device support"
-	default y
 	help
 	  Say Y here if you want to support the /dev/kmem device. The
 	  /dev/kmem device is rarely used, but can be used for certain
@@ -579,7 +578,7 @@ config DEVPORT
 source "drivers/s390/char/Kconfig"
 
 config TILE_SROM
-	bool "Character-device access via hypervisor to the Tilera SPI ROM"
+	tristate "Character-device access via hypervisor to the Tilera SPI ROM"
 	depends on TILE
 	default y
 	---help---
diff --git a/drivers/char/agp/alpha-agp.c b/drivers/char/agp/alpha-agp.c
index 199b8e99f7d7..737187865269 100644
--- a/drivers/char/agp/alpha-agp.c
+++ b/drivers/char/agp/alpha-agp.c
@@ -19,8 +19,7 @@ static int alpha_core_agp_vm_fault(struct vm_area_struct *vma,
 	unsigned long pa;
 	struct page *page;
 
-	dma_addr = (unsigned long)vmf->virtual_address - vma->vm_start
-						+ agp->aperture.bus_base;
+	dma_addr = vmf->address - vma->vm_start + agp->aperture.bus_base;
 	pa = agp->ops->translate(agp, dma_addr);
 
 	if (pa == (unsigned long)-EINVAL)
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index 200dab5136a7..ceff2fc524b1 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -168,7 +168,7 @@ config HW_RANDOM_IXP4XX
 
 config HW_RANDOM_OMAP
 	tristate "OMAP Random Number Generator support"
-	depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS
+	depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS || ARCH_MVEBU
 	default HW_RANDOM
  	---help---
  	  This driver provides kernel-side support for the Random Number
diff --git a/drivers/char/hw_random/atmel-rng.c b/drivers/char/hw_random/atmel-rng.c
index 0fcc9e69a346..661c82cde0f2 100644
--- a/drivers/char/hw_random/atmel-rng.c
+++ b/drivers/char/hw_random/atmel-rng.c
@@ -48,6 +48,16 @@ static int atmel_trng_read(struct hwrng *rng, void *buf, size_t max,
 		return 0;
 }
 
+static void atmel_trng_enable(struct atmel_trng *trng)
+{
+	writel(TRNG_KEY | 1, trng->base + TRNG_CR);
+}
+
+static void atmel_trng_disable(struct atmel_trng *trng)
+{
+	writel(TRNG_KEY, trng->base + TRNG_CR);
+}
+
 static int atmel_trng_probe(struct platform_device *pdev)
 {
 	struct atmel_trng *trng;
@@ -71,7 +81,7 @@ static int atmel_trng_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	writel(TRNG_KEY | 1, trng->base + TRNG_CR);
+	atmel_trng_enable(trng);
 	trng->rng.name = pdev->name;
 	trng->rng.read = atmel_trng_read;
 
@@ -84,7 +94,7 @@ static int atmel_trng_probe(struct platform_device *pdev)
 	return 0;
 
 err_register:
-	clk_disable(trng->clk);
+	clk_disable_unprepare(trng->clk);
 	return ret;
 }
 
@@ -94,7 +104,7 @@ static int atmel_trng_remove(struct platform_device *pdev)
 
 	hwrng_unregister(&trng->rng);
 
-	writel(TRNG_KEY, trng->base + TRNG_CR);
+	atmel_trng_disable(trng);
 	clk_disable_unprepare(trng->clk);
 
 	return 0;
@@ -105,6 +115,7 @@ static int atmel_trng_suspend(struct device *dev)
 {
 	struct atmel_trng *trng = dev_get_drvdata(dev);
 
+	atmel_trng_disable(trng);
 	clk_disable_unprepare(trng->clk);
 
 	return 0;
@@ -113,8 +124,15 @@ static int atmel_trng_suspend(struct device *dev)
 static int atmel_trng_resume(struct device *dev)
 {
 	struct atmel_trng *trng = dev_get_drvdata(dev);
+	int ret;
 
-	return clk_prepare_enable(trng->clk);
+	ret = clk_prepare_enable(trng->clk);
+	if (ret)
+		return ret;
+
+	atmel_trng_enable(trng);
+
+	return 0;
 }
 
 static const struct dev_pm_ops atmel_trng_pm_ops = {
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index d2d2c89de5b4..f9766415ff10 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -92,6 +92,7 @@ static void add_early_randomness(struct hwrng *rng)
 	mutex_unlock(&reading_mutex);
 	if (bytes_read > 0)
 		add_device_randomness(rng_buffer, bytes_read);
+	memset(rng_buffer, 0, size);
 }
 
 static inline void cleanup_rng(struct kref *kref)
@@ -287,6 +288,7 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf,
 		}
 	}
 out:
+	memset(rng_buffer, 0, rng_buffer_size());
 	return ret ? : err;
 
 out_unlock_reading:
@@ -425,6 +427,7 @@ static int hwrng_fillfn(void *unused)
 		/* Outside lock, sure, but y'know: randomness. */
 		add_hwgenerator_randomness((void *)rng_fillbuf, rc,
 					   rc * current_quality * 8 >> 10);
+		memset(rng_fillbuf, 0, rng_buffer_size());
 	}
 	hwrng_fill = NULL;
 	return 0;
diff --git a/drivers/char/hw_random/meson-rng.c b/drivers/char/hw_random/meson-rng.c
index 58bef39f7286..119d698439ae 100644
--- a/drivers/char/hw_random/meson-rng.c
+++ b/drivers/char/hw_random/meson-rng.c
@@ -110,6 +110,7 @@ static const struct of_device_id meson_rng_of_match[] = {
 	{ .compatible = "amlogic,meson-rng", },
 	{},
 };
+MODULE_DEVICE_TABLE(of, meson_rng_of_match);
 
 static struct platform_driver meson_rng_driver = {
 	.probe	= meson_rng_probe,
@@ -121,7 +122,6 @@ static struct platform_driver meson_rng_driver = {
 
 module_platform_driver(meson_rng_driver);
 
-MODULE_ALIAS("platform:meson-rng");
 MODULE_DESCRIPTION("Meson H/W Random Number Generator driver");
 MODULE_AUTHOR("Lawrence Mok <lawrence.mok@amlogic.com>");
 MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
diff --git a/drivers/char/hw_random/msm-rng.c b/drivers/char/hw_random/msm-rng.c
index 96fb986402eb..841fee845ec9 100644
--- a/drivers/char/hw_random/msm-rng.c
+++ b/drivers/char/hw_random/msm-rng.c
@@ -90,10 +90,6 @@ static int msm_rng_read(struct hwrng *hwrng, void *data, size_t max, bool wait)
 	/* calculate max size bytes to transfer back to caller */
 	maxsize = min_t(size_t, MAX_HW_FIFO_SIZE, max);
 
-	/* no room for word data */
-	if (maxsize < WORD_SZ)
-		return 0;
-
 	ret = clk_prepare_enable(rng->clk);
 	if (ret)
 		return ret;
diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c
index f5c26a5f6875..3ad86fdf954e 100644
--- a/drivers/char/hw_random/omap-rng.c
+++ b/drivers/char/hw_random/omap-rng.c
@@ -28,6 +28,7 @@
 #include <linux/of_device.h>
 #include <linux/of_address.h>
 #include <linux/interrupt.h>
+#include <linux/clk.h>
 
 #include <asm/io.h>
 
@@ -63,10 +64,13 @@
 
 #define OMAP2_RNG_OUTPUT_SIZE			0x4
 #define OMAP4_RNG_OUTPUT_SIZE			0x8
+#define EIP76_RNG_OUTPUT_SIZE			0x10
 
 enum {
-	RNG_OUTPUT_L_REG = 0,
-	RNG_OUTPUT_H_REG,
+	RNG_OUTPUT_0_REG = 0,
+	RNG_OUTPUT_1_REG,
+	RNG_OUTPUT_2_REG,
+	RNG_OUTPUT_3_REG,
 	RNG_STATUS_REG,
 	RNG_INTMASK_REG,
 	RNG_INTACK_REG,
@@ -82,7 +86,7 @@ enum {
 };
 
 static const u16 reg_map_omap2[] = {
-	[RNG_OUTPUT_L_REG]	= 0x0,
+	[RNG_OUTPUT_0_REG]	= 0x0,
 	[RNG_STATUS_REG]	= 0x4,
 	[RNG_CONFIG_REG]	= 0x28,
 	[RNG_REV_REG]		= 0x3c,
@@ -90,8 +94,8 @@ static const u16 reg_map_omap2[] = {
 };
 
 static const u16 reg_map_omap4[] = {
-	[RNG_OUTPUT_L_REG]	= 0x0,
-	[RNG_OUTPUT_H_REG]	= 0x4,
+	[RNG_OUTPUT_0_REG]	= 0x0,
+	[RNG_OUTPUT_1_REG]	= 0x4,
 	[RNG_STATUS_REG]	= 0x8,
 	[RNG_INTMASK_REG]	= 0xc,
 	[RNG_INTACK_REG]	= 0x10,
@@ -106,6 +110,23 @@ static const u16 reg_map_omap4[] = {
 	[RNG_SYSCONFIG_REG]	= 0x1FE4,
 };
 
+static const u16 reg_map_eip76[] = {
+	[RNG_OUTPUT_0_REG]	= 0x0,
+	[RNG_OUTPUT_1_REG]	= 0x4,
+	[RNG_OUTPUT_2_REG]	= 0x8,
+	[RNG_OUTPUT_3_REG]	= 0xc,
+	[RNG_STATUS_REG]	= 0x10,
+	[RNG_INTACK_REG]	= 0x10,
+	[RNG_CONTROL_REG]	= 0x14,
+	[RNG_CONFIG_REG]	= 0x18,
+	[RNG_ALARMCNT_REG]	= 0x1c,
+	[RNG_FROENABLE_REG]	= 0x20,
+	[RNG_FRODETUNE_REG]	= 0x24,
+	[RNG_ALARMMASK_REG]	= 0x28,
+	[RNG_ALARMSTOP_REG]	= 0x2c,
+	[RNG_REV_REG]		= 0x7c,
+};
+
 struct omap_rng_dev;
 /**
  * struct omap_rng_pdata - RNG IP block-specific data
@@ -127,6 +148,8 @@ struct omap_rng_dev {
 	void __iomem			*base;
 	struct device			*dev;
 	const struct omap_rng_pdata	*pdata;
+	struct hwrng rng;
+	struct clk 			*clk;
 };
 
 static inline u32 omap_rng_read(struct omap_rng_dev *priv, u16 reg)
@@ -140,41 +163,35 @@ static inline void omap_rng_write(struct omap_rng_dev *priv, u16 reg,
 	__raw_writel(val, priv->base + priv->pdata->regs[reg]);
 }
 
-static int omap_rng_data_present(struct hwrng *rng, int wait)
+
+static int omap_rng_do_read(struct hwrng *rng, void *data, size_t max,
+			    bool wait)
 {
 	struct omap_rng_dev *priv;
-	int data, i;
+	int i, present;
 
 	priv = (struct omap_rng_dev *)rng->priv;
 
+	if (max < priv->pdata->data_size)
+		return 0;
+
 	for (i = 0; i < 20; i++) {
-		data = priv->pdata->data_present(priv);
-		if (data || !wait)
+		present = priv->pdata->data_present(priv);
+		if (present || !wait)
 			break;
-		/* RNG produces data fast enough (2+ MBit/sec, even
-		 * during "rngtest" loads, that these delays don't
-		 * seem to trigger.  We *could* use the RNG IRQ, but
-		 * that'd be higher overhead ... so why bother?
-		 */
+
 		udelay(10);
 	}
-	return data;
-}
-
-static int omap_rng_data_read(struct hwrng *rng, u32 *data)
-{
-	struct omap_rng_dev *priv;
-	u32 data_size, i;
-
-	priv = (struct omap_rng_dev *)rng->priv;
-	data_size = priv->pdata->data_size;
+	if (!present)
+		return 0;
 
-	for (i = 0; i < data_size / sizeof(u32); i++)
-		data[i] = omap_rng_read(priv, RNG_OUTPUT_L_REG + i);
+	memcpy_fromio(data, priv->base + priv->pdata->regs[RNG_OUTPUT_0_REG],
+		      priv->pdata->data_size);
 
 	if (priv->pdata->regs[RNG_INTACK_REG])
 		omap_rng_write(priv, RNG_INTACK_REG, RNG_REG_INTACK_RDY_MASK);
-	return data_size;
+
+	return priv->pdata->data_size;
 }
 
 static int omap_rng_init(struct hwrng *rng)
@@ -193,13 +210,6 @@ static void omap_rng_cleanup(struct hwrng *rng)
 	priv->pdata->cleanup(priv);
 }
 
-static struct hwrng omap_rng_ops = {
-	.name		= "omap",
-	.data_present	= omap_rng_data_present,
-	.data_read	= omap_rng_data_read,
-	.init		= omap_rng_init,
-	.cleanup	= omap_rng_cleanup,
-};
 
 static inline u32 omap2_rng_data_present(struct omap_rng_dev *priv)
 {
@@ -231,6 +241,38 @@ static inline u32 omap4_rng_data_present(struct omap_rng_dev *priv)
 	return omap_rng_read(priv, RNG_STATUS_REG) & RNG_REG_STATUS_RDY;
 }
 
+static int eip76_rng_init(struct omap_rng_dev *priv)
+{
+	u32 val;
+
+	/* Return if RNG is already running. */
+	if (omap_rng_read(priv, RNG_CONTROL_REG) & RNG_CONTROL_ENABLE_TRNG_MASK)
+		return 0;
+
+	/*  Number of 512 bit blocks of raw Noise Source output data that must
+	 *  be processed by either the Conditioning Function or the
+	 *  SP 800-90 DRBG ‘BC_DF’ functionality to yield a ‘full entropy’
+	 *  output value.
+	 */
+	val = 0x5 << RNG_CONFIG_MIN_REFIL_CYCLES_SHIFT;
+
+	/* Number of FRO samples that are XOR-ed together into one bit to be
+	 * shifted into the main shift register
+	 */
+	val |= RNG_CONFIG_MAX_REFIL_CYCLES << RNG_CONFIG_MAX_REFIL_CYCLES_SHIFT;
+	omap_rng_write(priv, RNG_CONFIG_REG, val);
+
+	/* Enable all available FROs */
+	omap_rng_write(priv, RNG_FRODETUNE_REG, 0x0);
+	omap_rng_write(priv, RNG_FROENABLE_REG, RNG_REG_FROENABLE_MASK);
+
+	/* Enable TRNG */
+	val = RNG_CONTROL_ENABLE_TRNG_MASK;
+	omap_rng_write(priv, RNG_CONTROL_REG, val);
+
+	return 0;
+}
+
 static int omap4_rng_init(struct omap_rng_dev *priv)
 {
 	u32 val;
@@ -300,6 +342,14 @@ static struct omap_rng_pdata omap4_rng_pdata = {
 	.cleanup	= omap4_rng_cleanup,
 };
 
+static struct omap_rng_pdata eip76_rng_pdata = {
+	.regs		= (u16 *)reg_map_eip76,
+	.data_size	= EIP76_RNG_OUTPUT_SIZE,
+	.data_present	= omap4_rng_data_present,
+	.init		= eip76_rng_init,
+	.cleanup	= omap4_rng_cleanup,
+};
+
 static const struct of_device_id omap_rng_of_match[] = {
 		{
 			.compatible	= "ti,omap2-rng",
@@ -309,6 +359,10 @@ static const struct of_device_id omap_rng_of_match[] = {
 			.compatible	= "ti,omap4-rng",
 			.data		= &omap4_rng_pdata,
 		},
+		{
+			.compatible	= "inside-secure,safexcel-eip76",
+			.data		= &eip76_rng_pdata,
+		},
 		{},
 };
 MODULE_DEVICE_TABLE(of, omap_rng_of_match);
@@ -327,7 +381,8 @@ static int of_get_omap_rng_device_details(struct omap_rng_dev *priv,
 	}
 	priv->pdata = match->data;
 
-	if (of_device_is_compatible(dev->of_node, "ti,omap4-rng")) {
+	if (of_device_is_compatible(dev->of_node, "ti,omap4-rng") ||
+	    of_device_is_compatible(dev->of_node, "inside-secure,safexcel-eip76")) {
 		irq = platform_get_irq(pdev, 0);
 		if (irq < 0) {
 			dev_err(dev, "%s: error getting IRQ resource - %d\n",
@@ -343,6 +398,16 @@ static int of_get_omap_rng_device_details(struct omap_rng_dev *priv,
 			return err;
 		}
 		omap_rng_write(priv, RNG_INTMASK_REG, RNG_SHUTDOWN_OFLO_MASK);
+
+		priv->clk = of_clk_get(pdev->dev.of_node, 0);
+		if (IS_ERR(priv->clk) && PTR_ERR(priv->clk) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		if (!IS_ERR(priv->clk)) {
+			err = clk_prepare_enable(priv->clk);
+			if (err)
+				dev_err(&pdev->dev, "unable to enable the clk, "
+						    "err = %d\n", err);
+		}
 	}
 	return 0;
 }
@@ -372,7 +437,11 @@ static int omap_rng_probe(struct platform_device *pdev)
 	if (!priv)
 		return -ENOMEM;
 
-	omap_rng_ops.priv = (unsigned long)priv;
+	priv->rng.read = omap_rng_do_read;
+	priv->rng.init = omap_rng_init;
+	priv->rng.cleanup = omap_rng_cleanup;
+
+	priv->rng.priv = (unsigned long)priv;
 	platform_set_drvdata(pdev, priv);
 	priv->dev = dev;
 
@@ -383,6 +452,12 @@ static int omap_rng_probe(struct platform_device *pdev)
 		goto err_ioremap;
 	}
 
+	priv->rng.name = devm_kstrdup(dev, dev_name(dev), GFP_KERNEL);
+	if (!priv->rng.name) {
+		ret = -ENOMEM;
+		goto err_ioremap;
+	}
+
 	pm_runtime_enable(&pdev->dev);
 	ret = pm_runtime_get_sync(&pdev->dev);
 	if (ret < 0) {
@@ -394,20 +469,24 @@ static int omap_rng_probe(struct platform_device *pdev)
 	ret = (dev->of_node) ? of_get_omap_rng_device_details(priv, pdev) :
 				get_omap_rng_device_details(priv);
 	if (ret)
-		goto err_ioremap;
+		goto err_register;
 
-	ret = hwrng_register(&omap_rng_ops);
+	ret = hwrng_register(&priv->rng);
 	if (ret)
 		goto err_register;
 
-	dev_info(&pdev->dev, "OMAP Random Number Generator ver. %02x\n",
+	dev_info(&pdev->dev, "Random Number Generator ver. %02x\n",
 		 omap_rng_read(priv, RNG_REV_REG));
 
 	return 0;
 
 err_register:
 	priv->base = NULL;
+	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
+
+	if (!IS_ERR(priv->clk))
+		clk_disable_unprepare(priv->clk);
 err_ioremap:
 	dev_err(dev, "initialization failed.\n");
 	return ret;
@@ -417,13 +496,16 @@ static int omap_rng_remove(struct platform_device *pdev)
 {
 	struct omap_rng_dev *priv = platform_get_drvdata(pdev);
 
-	hwrng_unregister(&omap_rng_ops);
+	hwrng_unregister(&priv->rng);
 
 	priv->pdata->cleanup(priv);
 
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
+	if (!IS_ERR(priv->clk))
+		clk_disable_unprepare(priv->clk);
+
 	return 0;
 }
 
diff --git a/drivers/char/hw_random/pic32-rng.c b/drivers/char/hw_random/pic32-rng.c
index 11dc9b7c09ce..9b5e68a71d01 100644
--- a/drivers/char/hw_random/pic32-rng.c
+++ b/drivers/char/hw_random/pic32-rng.c
@@ -62,9 +62,6 @@ static int pic32_rng_read(struct hwrng *rng, void *buf, size_t max,
 	u32 t;
 	unsigned int timeout = RNG_TIMEOUT;
 
-	if (max < 8)
-		return 0;
-
 	do {
 		t = readl(priv->base + RNGRCNT) & RCNT_MASK;
 		if (t == 64) {
diff --git a/drivers/char/hw_random/pseries-rng.c b/drivers/char/hw_random/pseries-rng.c
index 63ce51d09af1..d9f46b437cc2 100644
--- a/drivers/char/hw_random/pseries-rng.c
+++ b/drivers/char/hw_random/pseries-rng.c
@@ -28,7 +28,6 @@
 static int pseries_rng_read(struct hwrng *rng, void *data, size_t max, bool wait)
 {
 	u64 buffer[PLPAR_HCALL_BUFSIZE];
-	size_t size = max < 8 ? max : 8;
 	int rc;
 
 	rc = plpar_hcall(H_RANDOM, (unsigned long *)buffer);
@@ -36,10 +35,10 @@ static int pseries_rng_read(struct hwrng *rng, void *data, size_t max, bool wait
 		pr_err_ratelimited("H_RANDOM call failed %d\n", rc);
 		return -EIO;
 	}
-	memcpy(data, buffer, size);
+	memcpy(data, buffer, 8);
 
 	/* The hypervisor interface returns 64 bits */
-	return size;
+	return 8;
 }
 
 /**
diff --git a/drivers/char/ipmi/ipmi_devintf.c b/drivers/char/ipmi/ipmi_devintf.c
index 1786574536b2..a21407de46ae 100644
--- a/drivers/char/ipmi/ipmi_devintf.c
+++ b/drivers/char/ipmi/ipmi_devintf.c
@@ -989,4 +989,3 @@ module_exit(cleanup_ipmi);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Corey Minyard <minyard@mvista.com>");
 MODULE_DESCRIPTION("Linux device interface for the IPMI message handler.");
-MODULE_ALIAS("platform:ipmi_si");
diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
index fcdd886819f5..92e53acf2cd2 100644
--- a/drivers/char/ipmi/ipmi_msghandler.c
+++ b/drivers/char/ipmi/ipmi_msghandler.c
@@ -158,15 +158,16 @@ struct seq_table {
  * Store the information in a msgid (long) to allow us to find a
  * sequence table entry from the msgid.
  */
-#define STORE_SEQ_IN_MSGID(seq, seqid) (((seq&0xff)<<26) | (seqid&0x3ffffff))
+#define STORE_SEQ_IN_MSGID(seq, seqid) \
+	((((seq) & 0x3f) << 26) | ((seqid) & 0x3ffffff))
 
 #define GET_SEQ_FROM_MSGID(msgid, seq, seqid) \
 	do {								\
-		seq = ((msgid >> 26) & 0x3f);				\
-		seqid = (msgid & 0x3fffff);				\
+		seq = (((msgid) >> 26) & 0x3f);				\
+		seqid = ((msgid) & 0x3ffffff);				\
 	} while (0)
 
-#define NEXT_SEQID(seqid) (((seqid) + 1) & 0x3fffff)
+#define NEXT_SEQID(seqid) (((seqid) + 1) & 0x3ffffff)
 
 struct ipmi_channel {
 	unsigned char medium;
@@ -4645,3 +4646,4 @@ MODULE_AUTHOR("Corey Minyard <minyard@mvista.com>");
 MODULE_DESCRIPTION("Incoming and outgoing message routing for an IPMI"
 		   " interface.");
 MODULE_VERSION(IPMI_DRIVER_VERSION);
+MODULE_SOFTDEP("post: ipmi_devintf");
diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c
index a112c0146012..2a7c425ddfa7 100644
--- a/drivers/char/ipmi/ipmi_si_intf.c
+++ b/drivers/char/ipmi/ipmi_si_intf.c
@@ -789,7 +789,7 @@ static void handle_transaction_done(struct smi_info *smi_info)
 				smi_info->si_state = SI_NORMAL;
 				break;
 			}
-			start_getting_msg_queue(smi_info);
+			start_getting_events(smi_info);
 		} else {
 			smi_info->si_state = SI_NORMAL;
 		}
@@ -812,7 +812,7 @@ static void handle_transaction_done(struct smi_info *smi_info)
 				smi_info->si_state = SI_NORMAL;
 				break;
 			}
-			start_getting_msg_queue(smi_info);
+			start_getting_events(smi_info);
 		} else {
 			smi_info->si_state = SI_NORMAL;
 		}
@@ -1764,7 +1764,7 @@ static int parse_str(const struct hotmod_vals *v, int *val, char *name,
 
 	s = strchr(*curr, ',');
 	if (!s) {
-		printk(KERN_WARNING PFX "No hotmod %s given.\n", name);
+		pr_warn(PFX "No hotmod %s given.\n", name);
 		return -EINVAL;
 	}
 	*s = '\0';
@@ -1777,7 +1777,7 @@ static int parse_str(const struct hotmod_vals *v, int *val, char *name,
 		}
 	}
 
-	printk(KERN_WARNING PFX "Invalid hotmod %s '%s'\n", name, *curr);
+	pr_warn(PFX "Invalid hotmod %s '%s'\n", name, *curr);
 	return -EINVAL;
 }
 
@@ -1788,16 +1788,12 @@ static int check_hotmod_int_op(const char *curr, const char *option,
 
 	if (strcmp(curr, name) == 0) {
 		if (!option) {
-			printk(KERN_WARNING PFX
-			       "No option given for '%s'\n",
-			       curr);
+			pr_warn(PFX "No option given for '%s'\n", curr);
 			return -EINVAL;
 		}
 		*val = simple_strtoul(option, &n, 0);
 		if ((*n != '\0') || (*option == '\0')) {
-			printk(KERN_WARNING PFX
-			       "Bad option given for '%s'\n",
-			       curr);
+			pr_warn(PFX "Bad option given for '%s'\n", curr);
 			return -EINVAL;
 		}
 		return 1;
@@ -1877,8 +1873,7 @@ static int hotmod_handler(const char *val, struct kernel_param *kp)
 		}
 		addr = simple_strtoul(curr, &n, 0);
 		if ((*n != '\0') || (*curr == '\0')) {
-			printk(KERN_WARNING PFX "Invalid hotmod address"
-			       " '%s'\n", curr);
+			pr_warn(PFX "Invalid hotmod address '%s'\n", curr);
 			break;
 		}
 
@@ -1921,9 +1916,7 @@ static int hotmod_handler(const char *val, struct kernel_param *kp)
 				continue;
 
 			rv = -EINVAL;
-			printk(KERN_WARNING PFX
-			       "Invalid hotmod option '%s'\n",
-			       curr);
+			pr_warn(PFX "Invalid hotmod option '%s'\n", curr);
 			goto out;
 		}
 
@@ -2003,7 +1996,7 @@ static int hardcode_find_bmc(void)
 			return -ENOMEM;
 
 		info->addr_source = SI_HARDCODED;
-		printk(KERN_INFO PFX "probing via hardcoded address\n");
+		pr_info(PFX "probing via hardcoded address\n");
 
 		if (!si_type[i] || strcmp(si_type[i], "kcs") == 0) {
 			info->si_type = SI_KCS;
@@ -2012,9 +2005,8 @@ static int hardcode_find_bmc(void)
 		} else if (strcmp(si_type[i], "bt") == 0) {
 			info->si_type = SI_BT;
 		} else {
-			printk(KERN_WARNING PFX "Interface type specified "
-			       "for interface %d, was invalid: %s\n",
-			       i, si_type[i]);
+			pr_warn(PFX "Interface type specified for interface %d, was invalid: %s\n",
+				i, si_type[i]);
 			kfree(info);
 			continue;
 		}
@@ -2030,9 +2022,8 @@ static int hardcode_find_bmc(void)
 			info->io.addr_data = addrs[i];
 			info->io.addr_type = IPMI_MEM_ADDR_SPACE;
 		} else {
-			printk(KERN_WARNING PFX "Interface type specified "
-			       "for interface %d, but port and address were "
-			       "not set or set to zero.\n", i);
+			pr_warn(PFX "Interface type specified for interface %d, but port and address were not set or set to zero.\n",
+				i);
 			kfree(info);
 			continue;
 		}
@@ -2173,18 +2164,18 @@ static int try_init_spmi(struct SPMITable *spmi)
 	int rv;
 
 	if (spmi->IPMIlegacy != 1) {
-		printk(KERN_INFO PFX "Bad SPMI legacy %d\n", spmi->IPMIlegacy);
+		pr_info(PFX "Bad SPMI legacy %d\n", spmi->IPMIlegacy);
 		return -ENODEV;
 	}
 
 	info = smi_info_alloc();
 	if (!info) {
-		printk(KERN_ERR PFX "Could not allocate SI data (3)\n");
+		pr_err(PFX "Could not allocate SI data (3)\n");
 		return -ENOMEM;
 	}
 
 	info->addr_source = SI_SPMI;
-	printk(KERN_INFO PFX "probing via SPMI\n");
+	pr_info(PFX "probing via SPMI\n");
 
 	/* Figure out the interface type. */
 	switch (spmi->InterfaceType) {
@@ -2201,8 +2192,8 @@ static int try_init_spmi(struct SPMITable *spmi)
 		kfree(info);
 		return -EIO;
 	default:
-		printk(KERN_INFO PFX "Unknown ACPI/SPMI SI type %d\n",
-		       spmi->InterfaceType);
+		pr_info(PFX "Unknown ACPI/SPMI SI type %d\n",
+			spmi->InterfaceType);
 		kfree(info);
 		return -EIO;
 	}
@@ -2238,15 +2229,15 @@ static int try_init_spmi(struct SPMITable *spmi)
 		info->io.addr_type = IPMI_IO_ADDR_SPACE;
 	} else {
 		kfree(info);
-		printk(KERN_WARNING PFX "Unknown ACPI I/O Address type\n");
+		pr_warn(PFX "Unknown ACPI I/O Address type\n");
 		return -EIO;
 	}
 	info->io.addr_data = spmi->addr.address;
 
 	pr_info("ipmi_si: SPMI: %s %#lx regsize %d spacing %d irq %d\n",
-		 (info->io.addr_type == IPMI_IO_ADDR_SPACE) ? "io" : "mem",
-		 info->io.addr_data, info->io.regsize, info->io.regspacing,
-		 info->irq);
+		(info->io.addr_type == IPMI_IO_ADDR_SPACE) ? "io" : "mem",
+		info->io.addr_data, info->io.regsize, info->io.regspacing,
+		info->irq);
 
 	rv = add_smi(info);
 	if (rv)
@@ -2356,12 +2347,12 @@ static void try_init_dmi(struct dmi_ipmi_data *ipmi_data)
 
 	info = smi_info_alloc();
 	if (!info) {
-		printk(KERN_ERR PFX "Could not allocate SI data\n");
+		pr_err(PFX "Could not allocate SI data\n");
 		return;
 	}
 
 	info->addr_source = SI_SMBIOS;
-	printk(KERN_INFO PFX "probing via SMBIOS\n");
+	pr_info(PFX "probing via SMBIOS\n");
 
 	switch (ipmi_data->type) {
 	case 0x01: /* KCS */
@@ -2391,8 +2382,8 @@ static void try_init_dmi(struct dmi_ipmi_data *ipmi_data)
 
 	default:
 		kfree(info);
-		printk(KERN_WARNING PFX "Unknown SMBIOS I/O Address type: %d\n",
-		       ipmi_data->addr_space);
+		pr_warn(PFX "Unknown SMBIOS I/O Address type: %d\n",
+			ipmi_data->addr_space);
 		return;
 	}
 	info->io.addr_data = ipmi_data->base_addr;
@@ -2410,9 +2401,9 @@ static void try_init_dmi(struct dmi_ipmi_data *ipmi_data)
 		info->irq_setup = std_irq_setup;
 
 	pr_info("ipmi_si: SMBIOS: %s %#lx regsize %d spacing %d irq %d\n",
-		 (info->io.addr_type == IPMI_IO_ADDR_SPACE) ? "io" : "mem",
-		 info->io.addr_data, info->io.regsize, info->io.regspacing,
-		 info->irq);
+		(info->io.addr_type == IPMI_IO_ADDR_SPACE) ? "io" : "mem",
+		info->io.addr_data, info->io.regsize, info->io.regspacing,
+		info->irq);
 
 	if (add_smi(info))
 		kfree(info);
@@ -3141,9 +3132,7 @@ static int try_enable_event_buffer(struct smi_info *smi_info)
 
 	rv = wait_for_msg_done(smi_info);
 	if (rv) {
-		printk(KERN_WARNING PFX "Error getting response from get"
-		       " global enables command, the event buffer is not"
-		       " enabled.\n");
+		pr_warn(PFX "Error getting response from get global enables command, the event buffer is not enabled.\n");
 		goto out;
 	}
 
@@ -3154,8 +3143,7 @@ static int try_enable_event_buffer(struct smi_info *smi_info)
 			resp[0] != (IPMI_NETFN_APP_REQUEST | 1) << 2 ||
 			resp[1] != IPMI_GET_BMC_GLOBAL_ENABLES_CMD   ||
 			resp[2] != 0) {
-		printk(KERN_WARNING PFX "Invalid return from get global"
-		       " enables command, cannot enable the event buffer.\n");
+		pr_warn(PFX "Invalid return from get global enables command, cannot enable the event buffer.\n");
 		rv = -EINVAL;
 		goto out;
 	}
@@ -3173,9 +3161,7 @@ static int try_enable_event_buffer(struct smi_info *smi_info)
 
 	rv = wait_for_msg_done(smi_info);
 	if (rv) {
-		printk(KERN_WARNING PFX "Error getting response from set"
-		       " global, enables command, the event buffer is not"
-		       " enabled.\n");
+		pr_warn(PFX "Error getting response from set global, enables command, the event buffer is not enabled.\n");
 		goto out;
 	}
 
@@ -3185,8 +3171,7 @@ static int try_enable_event_buffer(struct smi_info *smi_info)
 	if (resp_len < 3 ||
 			resp[0] != (IPMI_NETFN_APP_REQUEST | 1) << 2 ||
 			resp[1] != IPMI_SET_BMC_GLOBAL_ENABLES_CMD) {
-		printk(KERN_WARNING PFX "Invalid return from get global,"
-		       "enables command, not enable the event buffer.\n");
+		pr_warn(PFX "Invalid return from get global, enables command, not enable the event buffer.\n");
 		rv = -EINVAL;
 		goto out;
 	}
@@ -3463,8 +3448,16 @@ static int is_new_interface(struct smi_info *info)
 	list_for_each_entry(e, &smi_infos, link) {
 		if (e->io.addr_type != info->io.addr_type)
 			continue;
-		if (e->io.addr_data == info->io.addr_data)
+		if (e->io.addr_data == info->io.addr_data) {
+			/*
+			 * This is a cheap hack, ACPI doesn't have a defined
+			 * slave address but SMBIOS does.  Pick it up from
+			 * any source that has it available.
+			 */
+			if (info->slave_addr && !e->slave_addr)
+				e->slave_addr = info->slave_addr;
 			return 0;
+		}
 	}
 
 	return 1;
@@ -3474,17 +3467,18 @@ static int add_smi(struct smi_info *new_smi)
 {
 	int rv = 0;
 
-	printk(KERN_INFO PFX "Adding %s-specified %s state machine",
-	       ipmi_addr_src_to_str(new_smi->addr_source),
-	       si_to_str[new_smi->si_type]);
 	mutex_lock(&smi_infos_lock);
 	if (!is_new_interface(new_smi)) {
-		printk(KERN_CONT " duplicate interface\n");
+		pr_info(PFX "%s-specified %s state machine: duplicate\n",
+			ipmi_addr_src_to_str(new_smi->addr_source),
+			si_to_str[new_smi->si_type]);
 		rv = -EBUSY;
 		goto out_err;
 	}
 
-	printk(KERN_CONT "\n");
+	pr_info(PFX "Adding %s-specified %s state machine\n",
+		ipmi_addr_src_to_str(new_smi->addr_source),
+		si_to_str[new_smi->si_type]);
 
 	/* So we know not to free it unless we have allocated one. */
 	new_smi->intf = NULL;
@@ -3502,15 +3496,14 @@ static int try_smi_init(struct smi_info *new_smi)
 {
 	int rv = 0;
 	int i;
+	char *init_name = NULL;
 
-	printk(KERN_INFO PFX "Trying %s-specified %s state"
-	       " machine at %s address 0x%lx, slave address 0x%x,"
-	       " irq %d\n",
-	       ipmi_addr_src_to_str(new_smi->addr_source),
-	       si_to_str[new_smi->si_type],
-	       addr_space_to_str[new_smi->io.addr_type],
-	       new_smi->io.addr_data,
-	       new_smi->slave_addr, new_smi->irq);
+	pr_info(PFX "Trying %s-specified %s state machine at %s address 0x%lx, slave address 0x%x, irq %d\n",
+		ipmi_addr_src_to_str(new_smi->addr_source),
+		si_to_str[new_smi->si_type],
+		addr_space_to_str[new_smi->io.addr_type],
+		new_smi->io.addr_data,
+		new_smi->slave_addr, new_smi->irq);
 
 	switch (new_smi->si_type) {
 	case SI_KCS:
@@ -3531,11 +3524,30 @@ static int try_smi_init(struct smi_info *new_smi)
 		goto out_err;
 	}
 
+	/* Do this early so it's available for logs. */
+	if (!new_smi->dev) {
+		init_name = kasprintf(GFP_KERNEL, "ipmi_si.%d", 0);
+
+		/*
+		 * If we don't already have a device from something
+		 * else (like PCI), then register a new one.
+		 */
+		new_smi->pdev = platform_device_alloc("ipmi_si",
+						      new_smi->intf_num);
+		if (!new_smi->pdev) {
+			pr_err(PFX "Unable to allocate platform device\n");
+			goto out_err;
+		}
+		new_smi->dev = &new_smi->pdev->dev;
+		new_smi->dev->driver = &ipmi_driver.driver;
+		/* Nulled by device_add() */
+		new_smi->dev->init_name = init_name;
+	}
+
 	/* Allocate the state machine's data and initialize it. */
 	new_smi->si_sm = kmalloc(new_smi->handlers->size(), GFP_KERNEL);
 	if (!new_smi->si_sm) {
-		printk(KERN_ERR PFX
-		       "Could not allocate state machine memory\n");
+		pr_err(PFX "Could not allocate state machine memory\n");
 		rv = -ENOMEM;
 		goto out_err;
 	}
@@ -3545,14 +3557,14 @@ static int try_smi_init(struct smi_info *new_smi)
 	/* Now that we know the I/O size, we can set up the I/O. */
 	rv = new_smi->io_setup(new_smi);
 	if (rv) {
-		printk(KERN_ERR PFX "Could not set up I/O space\n");
+		dev_err(new_smi->dev, "Could not set up I/O space\n");
 		goto out_err;
 	}
 
 	/* Do low-level detection first. */
 	if (new_smi->handlers->detect(new_smi->si_sm)) {
 		if (new_smi->addr_source)
-			printk(KERN_INFO PFX "Interface detection failed\n");
+			dev_err(new_smi->dev, "Interface detection failed\n");
 		rv = -ENODEV;
 		goto out_err;
 	}
@@ -3564,8 +3576,7 @@ static int try_smi_init(struct smi_info *new_smi)
 	rv = try_get_dev_id(new_smi);
 	if (rv) {
 		if (new_smi->addr_source)
-			printk(KERN_INFO PFX "There appears to be no BMC"
-			       " at this location\n");
+			dev_err(new_smi->dev, "There appears to be no BMC at this location\n");
 		goto out_err;
 	}
 
@@ -3604,27 +3615,12 @@ static int try_smi_init(struct smi_info *new_smi)
 		atomic_set(&new_smi->req_events, 1);
 	}
 
-	if (!new_smi->dev) {
-		/*
-		 * If we don't already have a device from something
-		 * else (like PCI), then register a new one.
-		 */
-		new_smi->pdev = platform_device_alloc("ipmi_si",
-						      new_smi->intf_num);
-		if (!new_smi->pdev) {
-			printk(KERN_ERR PFX
-			       "Unable to allocate platform device\n");
-			goto out_err;
-		}
-		new_smi->dev = &new_smi->pdev->dev;
-		new_smi->dev->driver = &ipmi_driver.driver;
-
+	if (new_smi->pdev) {
 		rv = platform_device_add(new_smi->pdev);
 		if (rv) {
-			printk(KERN_ERR PFX
-			       "Unable to register system interface device:"
-			       " %d\n",
-			       rv);
+			dev_err(new_smi->dev,
+				"Unable to register system interface device: %d\n",
+				rv);
 			goto out_err;
 		}
 		new_smi->dev_registered = true;
@@ -3668,6 +3664,9 @@ static int try_smi_init(struct smi_info *new_smi)
 	dev_info(new_smi->dev, "IPMI %s interface initialized\n",
 		 si_to_str[new_smi->si_type]);
 
+	WARN_ON(new_smi->dev->init_name != NULL);
+	kfree(init_name);
+
 	return 0;
 
 out_err_stop_timer:
@@ -3712,8 +3711,14 @@ out_err:
 	if (new_smi->dev_registered) {
 		platform_device_unregister(new_smi->pdev);
 		new_smi->dev_registered = false;
+		new_smi->pdev = NULL;
+	} else if (new_smi->pdev) {
+		platform_device_put(new_smi->pdev);
+		new_smi->pdev = NULL;
 	}
 
+	kfree(init_name);
+
 	return rv;
 }
 
@@ -3732,8 +3737,7 @@ static int init_ipmi_si(void)
 	if (si_tryplatform) {
 		rv = platform_driver_register(&ipmi_driver);
 		if (rv) {
-			printk(KERN_ERR PFX "Unable to register "
-			       "driver: %d\n", rv);
+			pr_err(PFX "Unable to register driver: %d\n", rv);
 			return rv;
 		}
 	}
@@ -3753,7 +3757,7 @@ static int init_ipmi_si(void)
 		}
 	}
 
-	printk(KERN_INFO "IPMI System Interface driver.\n");
+	pr_info("IPMI System Interface driver.\n");
 
 	/* If the user gave us a device, they presumably want us to use it */
 	if (!hardcode_find_bmc())
@@ -3763,8 +3767,7 @@ static int init_ipmi_si(void)
 	if (si_trypci) {
 		rv = pci_register_driver(&ipmi_pci_driver);
 		if (rv)
-			printk(KERN_ERR PFX "Unable to register "
-			       "PCI driver: %d\n", rv);
+			pr_err(PFX "Unable to register PCI driver: %d\n", rv);
 		else
 			pci_registered = true;
 	}
@@ -3826,8 +3829,7 @@ static int init_ipmi_si(void)
 	if (unload_when_empty && list_empty(&smi_infos)) {
 		mutex_unlock(&smi_infos_lock);
 		cleanup_ipmi_si();
-		printk(KERN_WARNING PFX
-		       "Unable to find any System Interface(s)\n");
+		pr_warn(PFX "Unable to find any System Interface(s)\n");
 		return -ENODEV;
 	} else {
 		mutex_unlock(&smi_infos_lock);
diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
index 5673ffff00be..cca6e5bc1cea 100644
--- a/drivers/char/ipmi/ipmi_ssif.c
+++ b/drivers/char/ipmi/ipmi_ssif.c
@@ -174,7 +174,6 @@ enum ssif_stat_indexes {
 };
 
 struct ssif_addr_info {
-	unsigned short addr;
 	struct i2c_board_info binfo;
 	char *adapter_name;
 	int debug;
@@ -1154,10 +1153,6 @@ static bool ssif_dbg_probe;
 module_param_named(dbg_probe, ssif_dbg_probe, bool, 0);
 MODULE_PARM_DESC(dbg_probe, "Enable debugging of probing of adapters.");
 
-static int use_thread;
-module_param(use_thread, int, 0);
-MODULE_PARM_DESC(use_thread, "Use the thread interface.");
-
 static bool ssif_tryacpi = true;
 module_param_named(tryacpi, ssif_tryacpi, bool, 0);
 MODULE_PARM_DESC(tryacpi, "Setting this to zero will disable the default scan of the interfaces identified via ACPI");
@@ -1405,6 +1400,34 @@ static bool check_acpi(struct ssif_info *ssif_info, struct device *dev)
 	return false;
 }
 
+static int find_slave_address(struct i2c_client *client, int slave_addr)
+{
+	struct ssif_addr_info *info;
+
+	if (slave_addr)
+		return slave_addr;
+
+	/*
+	 * Came in without a slave address, search around to see if
+	 * the other sources have a slave address.  This lets us pick
+	 * up an SMBIOS slave address when using ACPI.
+	 */
+	list_for_each_entry(info, &ssif_infos, link) {
+		if (info->binfo.addr != client->addr)
+			continue;
+		if (info->adapter_name && client->adapter->name &&
+		    strcmp_nospace(info->adapter_name,
+				   client->adapter->name))
+			continue;
+		if (info->slave_addr) {
+			slave_addr = info->slave_addr;
+			break;
+		}
+	}
+
+	return slave_addr;
+}
+
 /*
  * Global enables we care about.
  */
@@ -1447,6 +1470,8 @@ static int ssif_probe(struct i2c_client *client, const struct i2c_device_id *id)
 		}
 	}
 
+	slave_addr = find_slave_address(client, slave_addr);
+
 	pr_info(PFX "Trying %s-specified SSIF interface at i2c address 0x%x, adapter %s, slave address 0x%x\n",
 	       ipmi_addr_src_to_str(ssif_info->addr_source),
 	       client->addr, client->adapter->name, slave_addr);
@@ -1935,7 +1960,7 @@ static int decode_dmi(const struct dmi_device *dmi_dev)
 		slave_addr = data[6];
 	}
 
-	return new_ssif_client(myaddr, NULL, 0, 0, SI_SMBIOS);
+	return new_ssif_client(myaddr, NULL, 0, slave_addr, SI_SMBIOS);
 }
 
 static void dmi_iterator(void)
diff --git a/drivers/char/mspec.c b/drivers/char/mspec.c
index f3f92d5fcda0..a697ca0cab1e 100644
--- a/drivers/char/mspec.c
+++ b/drivers/char/mspec.c
@@ -227,7 +227,7 @@ mspec_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	 * be because another thread has installed the pte first, so it
 	 * is no problem.
 	 */
-	vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
+	vm_insert_pfn(vma, vmf->address, pfn);
 
 	return VM_FAULT_NOPAGE;
 }
diff --git a/drivers/char/pcmcia/Kconfig b/drivers/char/pcmcia/Kconfig
index 8d3dfb0c8a26..1d1e7da8ad27 100644
--- a/drivers/char/pcmcia/Kconfig
+++ b/drivers/char/pcmcia/Kconfig
@@ -43,6 +43,17 @@ config CARDMAN_4040
 	  (http://www.omnikey.com/), or a current development version of OpenCT
 	  (http://www.opensc-project.org/opensc).
 
+config SCR24X
+	tristate "SCR24x Chip Card Interface support"
+	depends on PCMCIA
+	help
+	  Enable support for the SCR24x PCMCIA Chip Card Interface.
+
+	  To compile this driver as a module, choose M here.
+	  The module will be called scr24x_cs..
+
+	  If unsure say N.
+
 config IPWIRELESS
 	tristate "IPWireless 3G UMTS PCMCIA card support"
 	depends on PCMCIA && NETDEVICES && TTY
diff --git a/drivers/char/pcmcia/Makefile b/drivers/char/pcmcia/Makefile
index 0aae20985d57..5b836bc21406 100644
--- a/drivers/char/pcmcia/Makefile
+++ b/drivers/char/pcmcia/Makefile
@@ -7,3 +7,4 @@
 obj-$(CONFIG_SYNCLINK_CS) += synclink_cs.o
 obj-$(CONFIG_CARDMAN_4000) += cm4000_cs.o
 obj-$(CONFIG_CARDMAN_4040) += cm4040_cs.o
+obj-$(CONFIG_SCR24X) += scr24x_cs.o
diff --git a/drivers/char/pcmcia/scr24x_cs.c b/drivers/char/pcmcia/scr24x_cs.c
new file mode 100644
index 000000000000..f6b43d9350f0
--- /dev/null
+++ b/drivers/char/pcmcia/scr24x_cs.c
@@ -0,0 +1,373 @@
+/*
+ * SCR24x PCMCIA Smart Card Reader Driver
+ *
+ * Copyright (C) 2005-2006 TL Sudheendran
+ * Copyright (C) 2016 Lubomir Rintel
+ *
+ * Derived from "scr24x_v4.2.6_Release.tar.gz" driver by TL Sudheendran.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.  If not, write to
+ * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/cdev.h>
+#include <linux/slab.h>
+#include <linux/fs.h>
+#include <linux/io.h>
+#include <linux/uaccess.h>
+
+#include <pcmcia/cistpl.h>
+#include <pcmcia/ds.h>
+
+#define CCID_HEADER_SIZE	10
+#define CCID_LENGTH_OFFSET	1
+#define CCID_MAX_LEN		271
+
+#define SCR24X_DATA(n)		(1 + n)
+#define SCR24X_CMD_STATUS	7
+#define CMD_START		0x40
+#define CMD_WRITE_BYTE		0x41
+#define CMD_READ_BYTE		0x42
+#define STATUS_BUSY		0x80
+
+struct scr24x_dev {
+	struct device *dev;
+	struct cdev c_dev;
+	unsigned char buf[CCID_MAX_LEN];
+	int devno;
+	struct mutex lock;
+	struct kref refcnt;
+	u8 __iomem *regs;
+};
+
+#define SCR24X_DEVS 8
+static DECLARE_BITMAP(scr24x_minors, SCR24X_DEVS);
+
+static struct class *scr24x_class;
+static dev_t scr24x_devt;
+
+static void scr24x_delete(struct kref *kref)
+{
+	struct scr24x_dev *dev = container_of(kref, struct scr24x_dev,
+								refcnt);
+
+	kfree(dev);
+}
+
+static int scr24x_wait_ready(struct scr24x_dev *dev)
+{
+	u_char status;
+	int timeout = 100;
+
+	do {
+		status = ioread8(dev->regs + SCR24X_CMD_STATUS);
+		if (!(status & STATUS_BUSY))
+			return 0;
+
+		msleep(20);
+	} while (--timeout);
+
+	return -EIO;
+}
+
+static int scr24x_open(struct inode *inode, struct file *filp)
+{
+	struct scr24x_dev *dev = container_of(inode->i_cdev,
+				struct scr24x_dev, c_dev);
+
+	kref_get(&dev->refcnt);
+	filp->private_data = dev;
+
+	return nonseekable_open(inode, filp);
+}
+
+static int scr24x_release(struct inode *inode, struct file *filp)
+{
+	struct scr24x_dev *dev = filp->private_data;
+
+	/* We must not take the dev->lock here as scr24x_delete()
+	 * might be called to remove the dev structure altogether.
+	 * We don't need the lock anyway, since after the reference
+	 * acquired in probe() is released in remove() the chrdev
+	 * is already unregistered and noone can possibly acquire
+	 * a reference via open() anymore. */
+	kref_put(&dev->refcnt, scr24x_delete);
+	return 0;
+}
+
+static int read_chunk(struct scr24x_dev *dev, size_t offset, size_t limit)
+{
+	size_t i, y;
+	int ret;
+
+	for (i = offset; i < limit; i += 5) {
+		iowrite8(CMD_READ_BYTE, dev->regs + SCR24X_CMD_STATUS);
+		ret = scr24x_wait_ready(dev);
+		if (ret < 0)
+			return ret;
+
+		for (y = 0; y < 5 && i + y < limit; y++)
+			dev->buf[i + y] = ioread8(dev->regs + SCR24X_DATA(y));
+	}
+
+	return 0;
+}
+
+static ssize_t scr24x_read(struct file *filp, char __user *buf, size_t count,
+								loff_t *ppos)
+{
+	struct scr24x_dev *dev = filp->private_data;
+	int ret;
+	int len;
+
+	if (count < CCID_HEADER_SIZE)
+		return -EINVAL;
+
+	if (mutex_lock_interruptible(&dev->lock))
+		return -ERESTARTSYS;
+
+	if (!dev->dev) {
+		ret = -ENODEV;
+		goto out;
+	}
+
+	ret = scr24x_wait_ready(dev);
+	if (ret < 0)
+		goto out;
+	len = CCID_HEADER_SIZE;
+	ret = read_chunk(dev, 0, len);
+	if (ret < 0)
+		goto out;
+
+	len += le32_to_cpu(*(__le32 *)(&dev->buf[CCID_LENGTH_OFFSET]));
+	if (len > sizeof(dev->buf)) {
+		ret = -EIO;
+		goto out;
+	}
+	ret = read_chunk(dev, CCID_HEADER_SIZE, len);
+	if (ret < 0)
+		goto out;
+
+	if (len < count)
+		count = len;
+
+	if (copy_to_user(buf, dev->buf, count)) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	ret = count;
+out:
+	mutex_unlock(&dev->lock);
+	return ret;
+}
+
+static ssize_t scr24x_write(struct file *filp, const char __user *buf,
+					size_t count, loff_t *ppos)
+{
+	struct scr24x_dev *dev = filp->private_data;
+	size_t i, y;
+	int ret;
+
+	if (mutex_lock_interruptible(&dev->lock))
+		return -ERESTARTSYS;
+
+	if (!dev->dev) {
+		ret = -ENODEV;
+		goto out;
+	}
+
+	if (count > sizeof(dev->buf)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (copy_from_user(dev->buf, buf, count)) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	ret = scr24x_wait_ready(dev);
+	if (ret < 0)
+		goto out;
+
+	iowrite8(CMD_START, dev->regs + SCR24X_CMD_STATUS);
+	ret = scr24x_wait_ready(dev);
+	if (ret < 0)
+		goto out;
+
+	for (i = 0; i < count; i += 5) {
+		for (y = 0; y < 5 && i + y < count; y++)
+			iowrite8(dev->buf[i + y], dev->regs + SCR24X_DATA(y));
+
+		iowrite8(CMD_WRITE_BYTE, dev->regs + SCR24X_CMD_STATUS);
+		ret = scr24x_wait_ready(dev);
+		if (ret < 0)
+			goto out;
+	}
+
+	ret = count;
+out:
+	mutex_unlock(&dev->lock);
+	return ret;
+}
+
+static const struct file_operations scr24x_fops = {
+	.owner		= THIS_MODULE,
+	.read		= scr24x_read,
+	.write		= scr24x_write,
+	.open		= scr24x_open,
+	.release	= scr24x_release,
+	.llseek		= no_llseek,
+};
+
+static int scr24x_config_check(struct pcmcia_device *link, void *priv_data)
+{
+	if (resource_size(link->resource[PCMCIA_IOPORT_0]) != 0x11)
+		return -ENODEV;
+	return pcmcia_request_io(link);
+}
+
+static int scr24x_probe(struct pcmcia_device *link)
+{
+	struct scr24x_dev *dev;
+	int ret;
+
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	dev->devno = find_first_zero_bit(scr24x_minors, SCR24X_DEVS);
+	if (dev->devno >= SCR24X_DEVS) {
+		ret = -EBUSY;
+		goto err;
+	}
+
+	mutex_init(&dev->lock);
+	kref_init(&dev->refcnt);
+
+	link->priv = dev;
+	link->config_flags |= CONF_ENABLE_IRQ | CONF_AUTO_SET_IO;
+
+	ret = pcmcia_loop_config(link, scr24x_config_check, NULL);
+	if (ret < 0)
+		goto err;
+
+	dev->dev = &link->dev;
+	dev->regs = devm_ioport_map(&link->dev,
+				link->resource[PCMCIA_IOPORT_0]->start,
+				resource_size(link->resource[PCMCIA_IOPORT_0]));
+	if (!dev->regs) {
+		ret = -EIO;
+		goto err;
+	}
+
+	cdev_init(&dev->c_dev, &scr24x_fops);
+	dev->c_dev.owner = THIS_MODULE;
+	dev->c_dev.ops = &scr24x_fops;
+	ret = cdev_add(&dev->c_dev, MKDEV(MAJOR(scr24x_devt), dev->devno), 1);
+	if (ret < 0)
+		goto err;
+
+	ret = pcmcia_enable_device(link);
+	if (ret < 0) {
+		pcmcia_disable_device(link);
+		goto err;
+	}
+
+	device_create(scr24x_class, NULL, MKDEV(MAJOR(scr24x_devt), dev->devno),
+		      NULL, "scr24x%d", dev->devno);
+
+	dev_info(&link->dev, "SCR24x Chip Card Interface\n");
+	return 0;
+
+err:
+	if (dev->devno < SCR24X_DEVS)
+		clear_bit(dev->devno, scr24x_minors);
+	kfree (dev);
+	return ret;
+}
+
+static void scr24x_remove(struct pcmcia_device *link)
+{
+	struct scr24x_dev *dev = (struct scr24x_dev *)link->priv;
+
+	device_destroy(scr24x_class, MKDEV(MAJOR(scr24x_devt), dev->devno));
+	mutex_lock(&dev->lock);
+	pcmcia_disable_device(link);
+	cdev_del(&dev->c_dev);
+	clear_bit(dev->devno, scr24x_minors);
+	dev->dev = NULL;
+	mutex_unlock(&dev->lock);
+
+	kref_put(&dev->refcnt, scr24x_delete);
+}
+
+static const struct pcmcia_device_id scr24x_ids[] = {
+	PCMCIA_DEVICE_PROD_ID12("HP", "PC Card Smart Card Reader",
+					0x53cb94f9, 0xbfdf89a5),
+	PCMCIA_DEVICE_PROD_ID1("SCR241 PCMCIA", 0x6271efa3),
+	PCMCIA_DEVICE_PROD_ID1("SCR243 PCMCIA", 0x2054e8de),
+	PCMCIA_DEVICE_PROD_ID1("SCR24x PCMCIA", 0x54a33665),
+	PCMCIA_DEVICE_NULL
+};
+MODULE_DEVICE_TABLE(pcmcia, scr24x_ids);
+
+static struct pcmcia_driver scr24x_driver = {
+	.owner		= THIS_MODULE,
+	.name		= "scr24x_cs",
+	.probe		= scr24x_probe,
+	.remove		= scr24x_remove,
+	.id_table	= scr24x_ids,
+};
+
+static int __init scr24x_init(void)
+{
+	int ret;
+
+	scr24x_class = class_create(THIS_MODULE, "scr24x");
+	if (IS_ERR(scr24x_class))
+		return PTR_ERR(scr24x_class);
+
+	ret = alloc_chrdev_region(&scr24x_devt, 0, SCR24X_DEVS, "scr24x");
+	if (ret < 0)  {
+		class_destroy(scr24x_class);
+		return ret;
+	}
+
+	ret = pcmcia_register_driver(&scr24x_driver);
+	if (ret < 0) {
+		unregister_chrdev_region(scr24x_devt, SCR24X_DEVS);
+		class_destroy(scr24x_class);
+	}
+
+	return ret;
+}
+
+static void __exit scr24x_exit(void)
+{
+	pcmcia_unregister_driver(&scr24x_driver);
+	unregister_chrdev_region(scr24x_devt, SCR24X_DEVS);
+	class_destroy(scr24x_class);
+}
+
+module_init(scr24x_init);
+module_exit(scr24x_exit);
+
+MODULE_AUTHOR("Lubomir Rintel");
+MODULE_DESCRIPTION("SCR24x PCMCIA Smart Card Reader Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/char/ppdev.c b/drivers/char/ppdev.c
index 6af1ce04b3da..02819e0703c8 100644
--- a/drivers/char/ppdev.c
+++ b/drivers/char/ppdev.c
@@ -86,6 +86,9 @@ struct pp_struct {
 	long default_inactivity;
 };
 
+/* should we use PARDEVICE_MAX here? */
+static struct device *devices[PARPORT_MAX];
+
 /* pp_struct.flags bitfields */
 #define PP_CLAIMED    (1<<0)
 #define PP_EXCL       (1<<1)
@@ -294,7 +297,7 @@ static int register_device(int minor, struct pp_struct *pp)
 
 	port = parport_find_number(minor);
 	if (!port) {
-		printk(KERN_WARNING "%s: no associated port!\n", name);
+		pr_warn("%s: no associated port!\n", name);
 		kfree(name);
 		return -ENXIO;
 	}
@@ -305,10 +308,10 @@ static int register_device(int minor, struct pp_struct *pp)
 	ppdev_cb.private = pp;
 	pdev = parport_register_dev_model(port, name, &ppdev_cb, minor);
 	parport_put_port(port);
+	kfree(name);
 
 	if (!pdev) {
-		printk(KERN_WARNING "%s: failed to register device!\n", name);
-		kfree(name);
+		pr_warn("%s: failed to register device!\n", name);
 		return -ENXIO;
 	}
 
@@ -789,13 +792,29 @@ static const struct file_operations pp_fops = {
 
 static void pp_attach(struct parport *port)
 {
-	device_create(ppdev_class, port->dev, MKDEV(PP_MAJOR, port->number),
-		      NULL, "parport%d", port->number);
+	struct device *ret;
+
+	if (devices[port->number])
+		return;
+
+	ret = device_create(ppdev_class, port->dev,
+			    MKDEV(PP_MAJOR, port->number), NULL,
+			    "parport%d", port->number);
+	if (IS_ERR(ret)) {
+		pr_err("Failed to create device parport%d\n",
+		       port->number);
+		return;
+	}
+	devices[port->number] = ret;
 }
 
 static void pp_detach(struct parport *port)
 {
+	if (!devices[port->number])
+		return;
+
 	device_destroy(ppdev_class, MKDEV(PP_MAJOR, port->number));
+	devices[port->number] = NULL;
 }
 
 static int pp_probe(struct pardevice *par_dev)
@@ -822,8 +841,7 @@ static int __init ppdev_init(void)
 	int err = 0;
 
 	if (register_chrdev(PP_MAJOR, CHRDEV, &pp_fops)) {
-		printk(KERN_WARNING CHRDEV ": unable to get major %d\n",
-		       PP_MAJOR);
+		pr_warn(CHRDEV ": unable to get major %d\n", PP_MAJOR);
 		return -EIO;
 	}
 	ppdev_class = class_create(THIS_MODULE, CHRDEV);
@@ -833,11 +851,11 @@ static int __init ppdev_init(void)
 	}
 	err = parport_register_driver(&pp_driver);
 	if (err < 0) {
-		printk(KERN_WARNING CHRDEV ": unable to register with parport\n");
+		pr_warn(CHRDEV ": unable to register with parport\n");
 		goto out_class;
 	}
 
-	printk(KERN_INFO PP_VERSION "\n");
+	pr_info(PP_VERSION "\n");
 	goto out;
 
 out_class:
diff --git a/drivers/char/snsc.c b/drivers/char/snsc.c
index 10e56323f390..ec07f0e99732 100644
--- a/drivers/char/snsc.c
+++ b/drivers/char/snsc.c
@@ -285,7 +285,7 @@ scdrv_write(struct file *file, const char __user *buf,
 		DECLARE_WAITQUEUE(wait, current);
 
 		if (file->f_flags & O_NONBLOCK) {
-			spin_unlock(&sd->sd_wlock);
+			spin_unlock_irqrestore(&sd->sd_wlock, flags);
 			up(&sd->sd_wbs);
 			return -EAGAIN;
 		}
diff --git a/drivers/char/tile-srom.c b/drivers/char/tile-srom.c
index 398800edb2cc..3d4cca64b2d4 100644
--- a/drivers/char/tile-srom.c
+++ b/drivers/char/tile-srom.c
@@ -312,7 +312,8 @@ ATTRIBUTE_GROUPS(srom_dev);
 
 static char *srom_devnode(struct device *dev, umode_t *mode)
 {
-	*mode = S_IRUGO | S_IWUSR;
+	if (mode)
+		*mode = 0644;
 	return kasprintf(GFP_KERNEL, "srom/%s", dev_name(dev));
 }
 
diff --git a/drivers/char/tpm/Kconfig b/drivers/char/tpm/Kconfig
index 9faa0b1e7766..277186d3b668 100644
--- a/drivers/char/tpm/Kconfig
+++ b/drivers/char/tpm/Kconfig
@@ -32,7 +32,7 @@ config TCG_TIS_CORE
 
 config TCG_TIS
 	tristate "TPM Interface Specification 1.2 Interface / TPM 2.0 FIFO Interface"
-	depends on X86
+	depends on X86 || OF
 	select TCG_TIS_CORE
 	---help---
 	  If you have a TPM security chip that is compliant with the
diff --git a/drivers/char/tpm/Makefile b/drivers/char/tpm/Makefile
index a385fb8c17de..a05b1ebd0b26 100644
--- a/drivers/char/tpm/Makefile
+++ b/drivers/char/tpm/Makefile
@@ -2,16 +2,10 @@
 # Makefile for the kernel tpm device drivers.
 #
 obj-$(CONFIG_TCG_TPM) += tpm.o
-tpm-y := tpm-interface.o tpm-dev.o tpm-sysfs.o tpm-chip.o tpm2-cmd.o
-tpm-$(CONFIG_ACPI) += tpm_ppi.o
-
-ifdef CONFIG_ACPI
-	tpm-y += tpm_eventlog.o tpm_acpi.o
-else
-ifdef CONFIG_TCG_IBMVTPM
-	tpm-y += tpm_eventlog.o tpm_of.o
-endif
-endif
+tpm-y := tpm-interface.o tpm-dev.o tpm-sysfs.o tpm-chip.o tpm2-cmd.o \
+		tpm_eventlog.o
+tpm-$(CONFIG_ACPI) += tpm_ppi.o tpm_acpi.o
+tpm-$(CONFIG_OF) += tpm_of.o
 obj-$(CONFIG_TCG_TIS_CORE) += tpm_tis_core.o
 obj-$(CONFIG_TCG_TIS) += tpm_tis.o
 obj-$(CONFIG_TCG_TIS_SPI) += tpm_tis_spi.o
diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c
index e5950131bd90..a77262d31911 100644
--- a/drivers/char/tpm/tpm-chip.c
+++ b/drivers/char/tpm/tpm-chip.c
@@ -84,7 +84,7 @@ EXPORT_SYMBOL_GPL(tpm_put_ops);
  *
  * The return'd chip has been tpm_try_get_ops'd and must be released via
  * tpm_put_ops
-  */
+ */
 struct tpm_chip *tpm_chip_find_get(int chip_num)
 {
 	struct tpm_chip *chip, *res = NULL;
@@ -103,7 +103,7 @@ struct tpm_chip *tpm_chip_find_get(int chip_num)
 			}
 		} while (chip_prev != chip_num);
 	} else {
-		chip = idr_find_slowpath(&dev_nums_idr, chip_num);
+		chip = idr_find(&dev_nums_idr, chip_num);
 		if (chip && !tpm_try_get_ops(chip))
 			res = chip;
 	}
@@ -127,6 +127,7 @@ static void tpm_dev_release(struct device *dev)
 	idr_remove(&dev_nums_idr, chip->dev_num);
 	mutex_unlock(&idr_lock);
 
+	kfree(chip->log.bios_event_log);
 	kfree(chip);
 }
 
@@ -276,27 +277,6 @@ static void tpm_del_char_device(struct tpm_chip *chip)
 	up_write(&chip->ops_sem);
 }
 
-static int tpm1_chip_register(struct tpm_chip *chip)
-{
-	if (chip->flags & TPM_CHIP_FLAG_TPM2)
-		return 0;
-
-	tpm_sysfs_add_device(chip);
-
-	chip->bios_dir = tpm_bios_log_setup(dev_name(&chip->dev));
-
-	return 0;
-}
-
-static void tpm1_chip_unregister(struct tpm_chip *chip)
-{
-	if (chip->flags & TPM_CHIP_FLAG_TPM2)
-		return;
-
-	if (chip->bios_dir)
-		tpm_bios_log_teardown(chip->bios_dir);
-}
-
 static void tpm_del_legacy_sysfs(struct tpm_chip *chip)
 {
 	struct attribute **i;
@@ -363,20 +343,20 @@ int tpm_chip_register(struct tpm_chip *chip)
 			return rc;
 	}
 
-	rc = tpm1_chip_register(chip);
-	if (rc)
+	tpm_sysfs_add_device(chip);
+
+	rc = tpm_bios_log_setup(chip);
+	if (rc != 0 && rc != -ENODEV)
 		return rc;
 
 	tpm_add_ppi(chip);
 
 	rc = tpm_add_char_device(chip);
 	if (rc) {
-		tpm1_chip_unregister(chip);
+		tpm_bios_log_teardown(chip);
 		return rc;
 	}
 
-	chip->flags |= TPM_CHIP_FLAG_REGISTERED;
-
 	rc = tpm_add_legacy_sysfs(chip);
 	if (rc) {
 		tpm_chip_unregister(chip);
@@ -402,12 +382,8 @@ EXPORT_SYMBOL_GPL(tpm_chip_register);
  */
 void tpm_chip_unregister(struct tpm_chip *chip)
 {
-	if (!(chip->flags & TPM_CHIP_FLAG_REGISTERED))
-		return;
-
 	tpm_del_legacy_sysfs(chip);
-
-	tpm1_chip_unregister(chip);
+	tpm_bios_log_teardown(chip);
 	tpm_del_char_device(chip);
 }
 EXPORT_SYMBOL_GPL(tpm_chip_unregister);
diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c
index 3a9149cf0110..a2688ac2b48f 100644
--- a/drivers/char/tpm/tpm-interface.c
+++ b/drivers/char/tpm/tpm-interface.c
@@ -29,6 +29,7 @@
 #include <linux/mutex.h>
 #include <linux/spinlock.h>
 #include <linux/freezer.h>
+#include <linux/pm_runtime.h>
 
 #include "tpm.h"
 #include "tpm_eventlog.h"
@@ -356,6 +357,9 @@ ssize_t tpm_transmit(struct tpm_chip *chip, const u8 *buf, size_t bufsiz,
 	if (!(flags & TPM_TRANSMIT_UNLOCKED))
 		mutex_lock(&chip->tpm_mutex);
 
+	if (chip->dev.parent)
+		pm_runtime_get_sync(chip->dev.parent);
+
 	rc = chip->ops->send(chip, (u8 *) buf, count);
 	if (rc < 0) {
 		dev_err(&chip->dev,
@@ -397,6 +401,9 @@ out_recv:
 		dev_err(&chip->dev,
 			"tpm_transmit: tpm_recv: error %zd\n", rc);
 out:
+	if (chip->dev.parent)
+		pm_runtime_put_sync(chip->dev.parent);
+
 	if (!(flags & TPM_TRANSMIT_UNLOCKED))
 		mutex_unlock(&chip->tpm_mutex);
 	return rc;
@@ -437,26 +444,29 @@ static const struct tpm_input_header tpm_getcap_header = {
 	.ordinal = TPM_ORD_GET_CAP
 };
 
-ssize_t tpm_getcap(struct tpm_chip *chip, __be32 subcap_id, cap_t *cap,
+ssize_t tpm_getcap(struct tpm_chip *chip, u32 subcap_id, cap_t *cap,
 		   const char *desc)
 {
 	struct tpm_cmd_t tpm_cmd;
 	int rc;
 
 	tpm_cmd.header.in = tpm_getcap_header;
-	if (subcap_id == CAP_VERSION_1_1 || subcap_id == CAP_VERSION_1_2) {
-		tpm_cmd.params.getcap_in.cap = subcap_id;
+	if (subcap_id == TPM_CAP_VERSION_1_1 ||
+	    subcap_id == TPM_CAP_VERSION_1_2) {
+		tpm_cmd.params.getcap_in.cap = cpu_to_be32(subcap_id);
 		/*subcap field not necessary */
 		tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(0);
 		tpm_cmd.header.in.length -= cpu_to_be32(sizeof(__be32));
 	} else {
 		if (subcap_id == TPM_CAP_FLAG_PERM ||
 		    subcap_id == TPM_CAP_FLAG_VOL)
-			tpm_cmd.params.getcap_in.cap = TPM_CAP_FLAG;
+			tpm_cmd.params.getcap_in.cap =
+				cpu_to_be32(TPM_CAP_FLAG);
 		else
-			tpm_cmd.params.getcap_in.cap = TPM_CAP_PROP;
+			tpm_cmd.params.getcap_in.cap =
+				cpu_to_be32(TPM_CAP_PROP);
 		tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4);
-		tpm_cmd.params.getcap_in.subcap = subcap_id;
+		tpm_cmd.params.getcap_in.subcap = cpu_to_be32(subcap_id);
 	}
 	rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0,
 			      desc);
@@ -488,12 +498,14 @@ static int tpm_startup(struct tpm_chip *chip, __be16 startup_type)
 
 int tpm_get_timeouts(struct tpm_chip *chip)
 {
-	struct tpm_cmd_t tpm_cmd;
+	cap_t cap;
 	unsigned long new_timeout[4];
 	unsigned long old_timeout[4];
-	struct duration_t *duration_cap;
 	ssize_t rc;
 
+	if (chip->flags & TPM_CHIP_FLAG_HAVE_TIMEOUTS)
+		return 0;
+
 	if (chip->flags & TPM_CHIP_FLAG_TPM2) {
 		/* Fixed timeouts for TPM2 */
 		chip->timeout_a = msecs_to_jiffies(TPM2_TIMEOUT_A);
@@ -506,46 +518,30 @@ int tpm_get_timeouts(struct tpm_chip *chip)
 		    msecs_to_jiffies(TPM2_DURATION_MEDIUM);
 		chip->duration[TPM_LONG] =
 		    msecs_to_jiffies(TPM2_DURATION_LONG);
+
+		chip->flags |= TPM_CHIP_FLAG_HAVE_TIMEOUTS;
 		return 0;
 	}
 
-	tpm_cmd.header.in = tpm_getcap_header;
-	tpm_cmd.params.getcap_in.cap = TPM_CAP_PROP;
-	tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4);
-	tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_TIMEOUT;
-	rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0,
-			      NULL);
-
+	rc = tpm_getcap(chip, TPM_CAP_PROP_TIS_TIMEOUT, &cap,
+			"attempting to determine the timeouts");
 	if (rc == TPM_ERR_INVALID_POSTINIT) {
 		/* The TPM is not started, we are the first to talk to it.
 		   Execute a startup command. */
-		dev_info(&chip->dev, "Issuing TPM_STARTUP");
+		dev_info(&chip->dev, "Issuing TPM_STARTUP\n");
 		if (tpm_startup(chip, TPM_ST_CLEAR))
 			return rc;
 
-		tpm_cmd.header.in = tpm_getcap_header;
-		tpm_cmd.params.getcap_in.cap = TPM_CAP_PROP;
-		tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4);
-		tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_TIMEOUT;
-		rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE,
-				      0, NULL);
+		rc = tpm_getcap(chip, TPM_CAP_PROP_TIS_TIMEOUT, &cap,
+				"attempting to determine the timeouts");
 	}
-	if (rc) {
-		dev_err(&chip->dev,
-			"A TPM error (%zd) occurred attempting to determine the timeouts\n",
-			rc);
-		goto duration;
-	}
-
-	if (be32_to_cpu(tpm_cmd.header.out.return_code) != 0 ||
-	    be32_to_cpu(tpm_cmd.header.out.length)
-	    != sizeof(tpm_cmd.header.out) + sizeof(u32) + 4 * sizeof(u32))
-		return -EINVAL;
+	if (rc)
+		return rc;
 
-	old_timeout[0] = be32_to_cpu(tpm_cmd.params.getcap_out.cap.timeout.a);
-	old_timeout[1] = be32_to_cpu(tpm_cmd.params.getcap_out.cap.timeout.b);
-	old_timeout[2] = be32_to_cpu(tpm_cmd.params.getcap_out.cap.timeout.c);
-	old_timeout[3] = be32_to_cpu(tpm_cmd.params.getcap_out.cap.timeout.d);
+	old_timeout[0] = be32_to_cpu(cap.timeout.a);
+	old_timeout[1] = be32_to_cpu(cap.timeout.b);
+	old_timeout[2] = be32_to_cpu(cap.timeout.c);
+	old_timeout[3] = be32_to_cpu(cap.timeout.d);
 	memcpy(new_timeout, old_timeout, sizeof(new_timeout));
 
 	/*
@@ -583,29 +579,17 @@ int tpm_get_timeouts(struct tpm_chip *chip)
 	chip->timeout_c = usecs_to_jiffies(new_timeout[2]);
 	chip->timeout_d = usecs_to_jiffies(new_timeout[3]);
 
-duration:
-	tpm_cmd.header.in = tpm_getcap_header;
-	tpm_cmd.params.getcap_in.cap = TPM_CAP_PROP;
-	tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4);
-	tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_DURATION;
-
-	rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0,
-			      "attempting to determine the durations");
+	rc = tpm_getcap(chip, TPM_CAP_PROP_TIS_DURATION, &cap,
+			"attempting to determine the durations");
 	if (rc)
 		return rc;
 
-	if (be32_to_cpu(tpm_cmd.header.out.return_code) != 0 ||
-	    be32_to_cpu(tpm_cmd.header.out.length)
-	    != sizeof(tpm_cmd.header.out) + sizeof(u32) + 3 * sizeof(u32))
-		return -EINVAL;
-
-	duration_cap = &tpm_cmd.params.getcap_out.cap.duration;
 	chip->duration[TPM_SHORT] =
-	    usecs_to_jiffies(be32_to_cpu(duration_cap->tpm_short));
+		usecs_to_jiffies(be32_to_cpu(cap.duration.tpm_short));
 	chip->duration[TPM_MEDIUM] =
-	    usecs_to_jiffies(be32_to_cpu(duration_cap->tpm_medium));
+		usecs_to_jiffies(be32_to_cpu(cap.duration.tpm_medium));
 	chip->duration[TPM_LONG] =
-	    usecs_to_jiffies(be32_to_cpu(duration_cap->tpm_long));
+		usecs_to_jiffies(be32_to_cpu(cap.duration.tpm_long));
 
 	/* The Broadcom BCM0102 chipset in a Dell Latitude D820 gets the above
 	 * value wrong and apparently reports msecs rather than usecs. So we
@@ -619,6 +603,8 @@ duration:
 		chip->duration_adjusted = true;
 		dev_info(&chip->dev, "Adjusting TPM timeout parameters.");
 	}
+
+	chip->flags |= TPM_CHIP_FLAG_HAVE_TIMEOUTS;
 	return 0;
 }
 EXPORT_SYMBOL_GPL(tpm_get_timeouts);
@@ -726,6 +712,14 @@ int tpm_pcr_read(u32 chip_num, int pcr_idx, u8 *res_buf)
 }
 EXPORT_SYMBOL_GPL(tpm_pcr_read);
 
+#define TPM_ORD_PCR_EXTEND cpu_to_be32(20)
+#define EXTEND_PCR_RESULT_SIZE 34
+static const struct tpm_input_header pcrextend_header = {
+	.tag = TPM_TAG_RQU_COMMAND,
+	.length = cpu_to_be32(34),
+	.ordinal = TPM_ORD_PCR_EXTEND
+};
+
 /**
  * tpm_pcr_extend - extend pcr value with hash
  * @chip_num:	tpm idx # or AN&
@@ -736,14 +730,6 @@ EXPORT_SYMBOL_GPL(tpm_pcr_read);
  * isn't, protect against the chip disappearing, by incrementing
  * the module usage count.
  */
-#define TPM_ORD_PCR_EXTEND cpu_to_be32(20)
-#define EXTEND_PCR_RESULT_SIZE 34
-static const struct tpm_input_header pcrextend_header = {
-	.tag = TPM_TAG_RQU_COMMAND,
-	.length = cpu_to_be32(34),
-	.ordinal = TPM_ORD_PCR_EXTEND
-};
-
 int tpm_pcr_extend(u32 chip_num, int pcr_idx, const u8 *hash)
 {
 	struct tpm_cmd_t cmd;
diff --git a/drivers/char/tpm/tpm-sysfs.c b/drivers/char/tpm/tpm-sysfs.c
index a76ab4af9fb2..848ad6580b46 100644
--- a/drivers/char/tpm/tpm-sysfs.c
+++ b/drivers/char/tpm/tpm-sysfs.c
@@ -193,7 +193,7 @@ static ssize_t caps_show(struct device *dev, struct device_attribute *attr,
 		       be32_to_cpu(cap.manufacturer_id));
 
 	/* Try to get a TPM version 1.2 TPM_CAP_VERSION_INFO */
-	rc = tpm_getcap(chip, CAP_VERSION_1_2, &cap,
+	rc = tpm_getcap(chip, TPM_CAP_VERSION_1_2, &cap,
 			"attempting to determine the 1.2 version");
 	if (!rc) {
 		str += sprintf(str,
@@ -204,7 +204,7 @@ static ssize_t caps_show(struct device *dev, struct device_attribute *attr,
 			       cap.tpm_version_1_2.revMinor);
 	} else {
 		/* Otherwise just use TPM_STRUCT_VER */
-		rc = tpm_getcap(chip, CAP_VERSION_1_1, &cap,
+		rc = tpm_getcap(chip, TPM_CAP_VERSION_1_1, &cap,
 				"attempting to determine the 1.1 version");
 		if (rc)
 			return 0;
@@ -284,6 +284,9 @@ static const struct attribute_group tpm_dev_group = {
 
 void tpm_sysfs_add_device(struct tpm_chip *chip)
 {
+	if (chip->flags & TPM_CHIP_FLAG_TPM2)
+		return;
+
 	/* The sysfs routines rely on an implicit tpm_try_get_ops, device_del
 	 * is called before ops is null'd and the sysfs core synchronizes this
 	 * removal so that no callbacks are running or can run again
diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h
index 4d183c97f6a6..1ae976894257 100644
--- a/drivers/char/tpm/tpm.h
+++ b/drivers/char/tpm/tpm.h
@@ -35,11 +35,14 @@
 #include <linux/cdev.h>
 #include <linux/highmem.h>
 
+#include "tpm_eventlog.h"
+
 enum tpm_const {
 	TPM_MINOR = 224,	/* officially assigned */
 	TPM_BUFSIZE = 4096,
 	TPM_NUM_DEVICES = 65536,
 	TPM_RETRY = 50,		/* 5 seconds */
+	TPM_NUM_EVENT_LOG_FILES = 3,
 };
 
 enum tpm_timeout {
@@ -139,10 +142,15 @@ enum tpm2_startup_types {
 #define TPM_PPI_VERSION_LEN		3
 
 enum tpm_chip_flags {
-	TPM_CHIP_FLAG_REGISTERED	= BIT(0),
 	TPM_CHIP_FLAG_TPM2		= BIT(1),
 	TPM_CHIP_FLAG_IRQ		= BIT(2),
 	TPM_CHIP_FLAG_VIRTUAL		= BIT(3),
+	TPM_CHIP_FLAG_HAVE_TIMEOUTS	= BIT(4),
+};
+
+struct tpm_chip_seqops {
+	struct tpm_chip *chip;
+	const struct seq_operations *seqops;
 };
 
 struct tpm_chip {
@@ -156,6 +164,10 @@ struct tpm_chip {
 	struct rw_semaphore ops_sem;
 	const struct tpm_class_ops *ops;
 
+	struct tpm_bios_log log;
+	struct tpm_chip_seqops bin_log_seqops;
+	struct tpm_chip_seqops ascii_log_seqops;
+
 	unsigned int flags;
 
 	int dev_num;		/* /dev/tpm# */
@@ -171,7 +183,7 @@ struct tpm_chip {
 	unsigned long duration[3]; /* jiffies */
 	bool duration_adjusted;
 
-	struct dentry **bios_dir;
+	struct dentry *bios_dir[TPM_NUM_EVENT_LOG_FILES];
 
 	const struct attribute_group *groups[3];
 	unsigned int groups_cnt;
@@ -282,21 +294,20 @@ typedef union {
 } cap_t;
 
 enum tpm_capabilities {
-	TPM_CAP_FLAG = cpu_to_be32(4),
-	TPM_CAP_PROP = cpu_to_be32(5),
-	CAP_VERSION_1_1 = cpu_to_be32(0x06),
-	CAP_VERSION_1_2 = cpu_to_be32(0x1A)
+	TPM_CAP_FLAG = 4,
+	TPM_CAP_PROP = 5,
+	TPM_CAP_VERSION_1_1 = 0x06,
+	TPM_CAP_VERSION_1_2 = 0x1A,
 };
 
 enum tpm_sub_capabilities {
-	TPM_CAP_PROP_PCR = cpu_to_be32(0x101),
-	TPM_CAP_PROP_MANUFACTURER = cpu_to_be32(0x103),
-	TPM_CAP_FLAG_PERM = cpu_to_be32(0x108),
-	TPM_CAP_FLAG_VOL = cpu_to_be32(0x109),
-	TPM_CAP_PROP_OWNER = cpu_to_be32(0x111),
-	TPM_CAP_PROP_TIS_TIMEOUT = cpu_to_be32(0x115),
-	TPM_CAP_PROP_TIS_DURATION = cpu_to_be32(0x120),
-
+	TPM_CAP_PROP_PCR = 0x101,
+	TPM_CAP_PROP_MANUFACTURER = 0x103,
+	TPM_CAP_FLAG_PERM = 0x108,
+	TPM_CAP_FLAG_VOL = 0x109,
+	TPM_CAP_PROP_OWNER = 0x111,
+	TPM_CAP_PROP_TIS_TIMEOUT = 0x115,
+	TPM_CAP_PROP_TIS_DURATION = 0x120,
 };
 
 struct	tpm_getcap_params_in {
@@ -484,7 +495,7 @@ ssize_t tpm_transmit(struct tpm_chip *chip, const u8 *buf, size_t bufsiz,
 		     unsigned int flags);
 ssize_t tpm_transmit_cmd(struct tpm_chip *chip, const void *cmd, int len,
 			 unsigned int flags, const char *desc);
-ssize_t tpm_getcap(struct tpm_chip *chip, __be32 subcap_id, cap_t *cap,
+ssize_t tpm_getcap(struct tpm_chip *chip, u32 subcap_id, cap_t *cap,
 		   const char *desc);
 int tpm_get_timeouts(struct tpm_chip *);
 int tpm1_auto_startup(struct tpm_chip *chip);
diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c
index 7df55d58c939..da5b782a9731 100644
--- a/drivers/char/tpm/tpm2-cmd.c
+++ b/drivers/char/tpm/tpm2-cmd.c
@@ -680,7 +680,7 @@ static int tpm2_unseal_cmd(struct tpm_chip *chip,
 }
 
 /**
- * tpm_unseal_trusted() - unseal the payload of a trusted key
+ * tpm2_unseal_trusted() - unseal the payload of a trusted key
  * @chip_num: TPM chip to use
  * @payload: the key data in clear and encrypted form
  * @options: authentication values and other options
diff --git a/drivers/char/tpm/tpm_acpi.c b/drivers/char/tpm/tpm_acpi.c
index 565a9478cb94..b7718c95fd0b 100644
--- a/drivers/char/tpm/tpm_acpi.c
+++ b/drivers/char/tpm/tpm_acpi.c
@@ -6,10 +6,11 @@
  *	Stefan Berger <stefanb@us.ibm.com>
  *	Reiner Sailer <sailer@watson.ibm.com>
  *	Kylene Hall <kjhall@us.ibm.com>
+ *	Nayna Jain <nayna@linux.vnet.ibm.com>
  *
  * Maintained by: <tpmdd-devel@lists.sourceforge.net>
  *
- * Access to the eventlog extended by the TCG BIOS of PC platform
+ * Access to the event log extended by the TCG BIOS of PC platform
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -45,29 +46,28 @@ struct acpi_tcpa {
 };
 
 /* read binary bios log */
-int read_log(struct tpm_bios_log *log)
+int tpm_read_log_acpi(struct tpm_chip *chip)
 {
 	struct acpi_tcpa *buff;
 	acpi_status status;
 	void __iomem *virt;
 	u64 len, start;
+	struct tpm_bios_log *log;
 
-	if (log->bios_event_log != NULL) {
-		printk(KERN_ERR
-		       "%s: ERROR - Eventlog already initialized\n",
-		       __func__);
-		return -EFAULT;
-	}
+	log = &chip->log;
+
+	/* Unfortuntely ACPI does not associate the event log with a specific
+	 * TPM, like PPI. Thus all ACPI TPMs will read the same log.
+	 */
+	if (!chip->acpi_dev_handle)
+		return -ENODEV;
 
 	/* Find TCPA entry in RSDT (ACPI_LOGICAL_ADDRESSING) */
 	status = acpi_get_table(ACPI_SIG_TCPA, 1,
 				(struct acpi_table_header **)&buff);
 
-	if (ACPI_FAILURE(status)) {
-		printk(KERN_ERR "%s: ERROR - Could not get TCPA table\n",
-		       __func__);
-		return -EIO;
-	}
+	if (ACPI_FAILURE(status))
+		return -ENODEV;
 
 	switch(buff->platform_class) {
 	case BIOS_SERVER:
@@ -81,29 +81,29 @@ int read_log(struct tpm_bios_log *log)
 		break;
 	}
 	if (!len) {
-		printk(KERN_ERR "%s: ERROR - TCPA log area empty\n", __func__);
+		dev_warn(&chip->dev, "%s: TCPA log area empty\n", __func__);
 		return -EIO;
 	}
 
 	/* malloc EventLog space */
 	log->bios_event_log = kmalloc(len, GFP_KERNEL);
-	if (!log->bios_event_log) {
-		printk("%s: ERROR - Not enough  Memory for BIOS measurements\n",
-			__func__);
+	if (!log->bios_event_log)
 		return -ENOMEM;
-	}
 
 	log->bios_event_log_end = log->bios_event_log + len;
 
 	virt = acpi_os_map_iomem(start, len);
-	if (!virt) {
-		kfree(log->bios_event_log);
-		printk("%s: ERROR - Unable to map memory\n", __func__);
-		return -EIO;
-	}
+	if (!virt)
+		goto err;
 
 	memcpy_fromio(log->bios_event_log, virt, len);
 
 	acpi_os_unmap_iomem(virt, len);
 	return 0;
+
+err:
+	kfree(log->bios_event_log);
+	log->bios_event_log = NULL;
+	return -EIO;
+
 }
diff --git a/drivers/char/tpm/tpm_crb.c b/drivers/char/tpm/tpm_crb.c
index a7c870af916c..717b6b47c042 100644
--- a/drivers/char/tpm/tpm_crb.c
+++ b/drivers/char/tpm/tpm_crb.c
@@ -19,6 +19,7 @@
 #include <linux/highmem.h>
 #include <linux/rculist.h>
 #include <linux/module.h>
+#include <linux/pm_runtime.h>
 #include "tpm.h"
 
 #define ACPI_SIG_TPM2 "TPM2"
@@ -83,7 +84,71 @@ struct crb_priv {
 	u32 cmd_size;
 };
 
-static SIMPLE_DEV_PM_OPS(crb_pm, tpm_pm_suspend, tpm_pm_resume);
+/**
+ * crb_go_idle - request tpm crb device to go the idle state
+ *
+ * @dev:  crb device
+ * @priv: crb private data
+ *
+ * Write CRB_CTRL_REQ_GO_IDLE to TPM_CRB_CTRL_REQ
+ * The device should respond within TIMEOUT_C by clearing the bit.
+ * Anyhow, we do not wait here as a consequent CMD_READY request
+ * will be handled correctly even if idle was not completed.
+ *
+ * The function does nothing for devices with ACPI-start method.
+ *
+ * Return: 0 always
+ */
+static int __maybe_unused crb_go_idle(struct device *dev, struct crb_priv *priv)
+{
+	if (priv->flags & CRB_FL_ACPI_START)
+		return 0;
+
+	iowrite32(CRB_CTRL_REQ_GO_IDLE, &priv->cca->req);
+	/* we don't really care when this settles */
+
+	return 0;
+}
+
+/**
+ * crb_cmd_ready - request tpm crb device to enter ready state
+ *
+ * @dev:  crb device
+ * @priv: crb private data
+ *
+ * Write CRB_CTRL_REQ_CMD_READY to TPM_CRB_CTRL_REQ
+ * and poll till the device acknowledge it by clearing the bit.
+ * The device should respond within TIMEOUT_C.
+ *
+ * The function does nothing for devices with ACPI-start method
+ *
+ * Return: 0 on success -ETIME on timeout;
+ */
+static int __maybe_unused crb_cmd_ready(struct device *dev,
+					struct crb_priv *priv)
+{
+	ktime_t stop, start;
+
+	if (priv->flags & CRB_FL_ACPI_START)
+		return 0;
+
+	iowrite32(CRB_CTRL_REQ_CMD_READY, &priv->cca->req);
+
+	start = ktime_get();
+	stop = ktime_add(start, ms_to_ktime(TPM2_TIMEOUT_C));
+	do {
+		if (!(ioread32(&priv->cca->req) & CRB_CTRL_REQ_CMD_READY))
+			return 0;
+		usleep_range(50, 100);
+	} while (ktime_before(ktime_get(), stop));
+
+	if (ioread32(&priv->cca->req) & CRB_CTRL_REQ_CMD_READY) {
+		dev_warn(dev, "cmdReady timed out\n");
+		return -ETIME;
+	}
+
+	return 0;
+}
 
 static u8 crb_status(struct tpm_chip *chip)
 {
@@ -196,21 +261,6 @@ static const struct tpm_class_ops tpm_crb = {
 	.req_complete_val = CRB_DRV_STS_COMPLETE,
 };
 
-static int crb_init(struct acpi_device *device, struct crb_priv *priv)
-{
-	struct tpm_chip *chip;
-
-	chip = tpmm_chip_alloc(&device->dev, &tpm_crb);
-	if (IS_ERR(chip))
-		return PTR_ERR(chip);
-
-	dev_set_drvdata(&chip->dev, priv);
-	chip->acpi_dev_handle = device->handle;
-	chip->flags = TPM_CHIP_FLAG_TPM2;
-
-	return tpm_chip_register(chip);
-}
-
 static int crb_check_resource(struct acpi_resource *ares, void *data)
 {
 	struct resource *io_res = data;
@@ -249,6 +299,7 @@ static int crb_map_io(struct acpi_device *device, struct crb_priv *priv,
 	struct list_head resources;
 	struct resource io_res;
 	struct device *dev = &device->dev;
+	u32 pa_high, pa_low;
 	u64 cmd_pa;
 	u32 cmd_size;
 	u64 rsp_pa;
@@ -276,12 +327,27 @@ static int crb_map_io(struct acpi_device *device, struct crb_priv *priv,
 	if (IS_ERR(priv->cca))
 		return PTR_ERR(priv->cca);
 
-	cmd_pa = ((u64) ioread32(&priv->cca->cmd_pa_high) << 32) |
-		  (u64) ioread32(&priv->cca->cmd_pa_low);
+	/*
+	 * PTT HW bug w/a: wake up the device to access
+	 * possibly not retained registers.
+	 */
+	ret = crb_cmd_ready(dev, priv);
+	if (ret)
+		return ret;
+
+	pa_high = ioread32(&priv->cca->cmd_pa_high);
+	pa_low  = ioread32(&priv->cca->cmd_pa_low);
+	cmd_pa = ((u64)pa_high << 32) | pa_low;
 	cmd_size = ioread32(&priv->cca->cmd_size);
+
+	dev_dbg(dev, "cmd_hi = %X cmd_low = %X cmd_size %X\n",
+		pa_high, pa_low, cmd_size);
+
 	priv->cmd = crb_map_res(dev, priv, &io_res, cmd_pa, cmd_size);
-	if (IS_ERR(priv->cmd))
-		return PTR_ERR(priv->cmd);
+	if (IS_ERR(priv->cmd)) {
+		ret = PTR_ERR(priv->cmd);
+		goto out;
+	}
 
 	memcpy_fromio(&rsp_pa, &priv->cca->rsp_pa, 8);
 	rsp_pa = le64_to_cpu(rsp_pa);
@@ -289,7 +355,8 @@ static int crb_map_io(struct acpi_device *device, struct crb_priv *priv,
 
 	if (cmd_pa != rsp_pa) {
 		priv->rsp = crb_map_res(dev, priv, &io_res, rsp_pa, rsp_size);
-		return PTR_ERR_OR_ZERO(priv->rsp);
+		ret = PTR_ERR_OR_ZERO(priv->rsp);
+		goto out;
 	}
 
 	/* According to the PTP specification, overlapping command and response
@@ -297,18 +364,25 @@ static int crb_map_io(struct acpi_device *device, struct crb_priv *priv,
 	 */
 	if (cmd_size != rsp_size) {
 		dev_err(dev, FW_BUG "overlapping command and response buffer sizes are not identical");
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
+
 	priv->cmd_size = cmd_size;
 
 	priv->rsp = priv->cmd;
-	return 0;
+
+out:
+	crb_go_idle(dev, priv);
+
+	return ret;
 }
 
 static int crb_acpi_add(struct acpi_device *device)
 {
 	struct acpi_table_tpm2 *buf;
 	struct crb_priv *priv;
+	struct tpm_chip *chip;
 	struct device *dev = &device->dev;
 	acpi_status status;
 	u32 sm;
@@ -346,7 +420,33 @@ static int crb_acpi_add(struct acpi_device *device)
 	if (rc)
 		return rc;
 
-	return crb_init(device, priv);
+	chip = tpmm_chip_alloc(dev, &tpm_crb);
+	if (IS_ERR(chip))
+		return PTR_ERR(chip);
+
+	dev_set_drvdata(&chip->dev, priv);
+	chip->acpi_dev_handle = device->handle;
+	chip->flags = TPM_CHIP_FLAG_TPM2;
+
+	rc  = crb_cmd_ready(dev, priv);
+	if (rc)
+		return rc;
+
+	pm_runtime_get_noresume(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+
+	rc = tpm_chip_register(chip);
+	if (rc) {
+		crb_go_idle(dev, priv);
+		pm_runtime_put_noidle(dev);
+		pm_runtime_disable(dev);
+		return rc;
+	}
+
+	pm_runtime_put(dev);
+
+	return 0;
 }
 
 static int crb_acpi_remove(struct acpi_device *device)
@@ -356,9 +456,34 @@ static int crb_acpi_remove(struct acpi_device *device)
 
 	tpm_chip_unregister(chip);
 
+	pm_runtime_disable(dev);
+
 	return 0;
 }
 
+#ifdef CONFIG_PM
+static int crb_pm_runtime_suspend(struct device *dev)
+{
+	struct tpm_chip *chip = dev_get_drvdata(dev);
+	struct crb_priv *priv = dev_get_drvdata(&chip->dev);
+
+	return crb_go_idle(dev, priv);
+}
+
+static int crb_pm_runtime_resume(struct device *dev)
+{
+	struct tpm_chip *chip = dev_get_drvdata(dev);
+	struct crb_priv *priv = dev_get_drvdata(&chip->dev);
+
+	return crb_cmd_ready(dev, priv);
+}
+#endif /* CONFIG_PM */
+
+static const struct dev_pm_ops crb_pm = {
+	SET_SYSTEM_SLEEP_PM_OPS(tpm_pm_suspend, tpm_pm_resume)
+	SET_RUNTIME_PM_OPS(crb_pm_runtime_suspend, crb_pm_runtime_resume, NULL)
+};
+
 static struct acpi_device_id crb_device_ids[] = {
 	{"MSFT0101", 0},
 	{"", 0},
diff --git a/drivers/char/tpm/tpm_eventlog.c b/drivers/char/tpm/tpm_eventlog.c
index e7228863290e..11bb1138a828 100644
--- a/drivers/char/tpm/tpm_eventlog.c
+++ b/drivers/char/tpm/tpm_eventlog.c
@@ -7,10 +7,11 @@
  *	Stefan Berger <stefanb@us.ibm.com>
  *	Reiner Sailer <sailer@watson.ibm.com>
  *	Kylene Hall <kjhall@us.ibm.com>
+ *	Nayna Jain <nayna@linux.vnet.ibm.com>
  *
  * Maintained by: <tpmdd-devel@lists.sourceforge.net>
  *
- * Access to the eventlog created by a system's firmware / BIOS
+ * Access to the event log created by a system's firmware / BIOS
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -72,7 +73,8 @@ static const char* tcpa_pc_event_id_strings[] = {
 static void *tpm_bios_measurements_start(struct seq_file *m, loff_t *pos)
 {
 	loff_t i;
-	struct tpm_bios_log *log = m->private;
+	struct tpm_chip *chip = m->private;
+	struct tpm_bios_log *log = &chip->log;
 	void *addr = log->bios_event_log;
 	void *limit = log->bios_event_log_end;
 	struct tcpa_event *event;
@@ -119,7 +121,8 @@ static void *tpm_bios_measurements_next(struct seq_file *m, void *v,
 					loff_t *pos)
 {
 	struct tcpa_event *event = v;
-	struct tpm_bios_log *log = m->private;
+	struct tpm_chip *chip = m->private;
+	struct tpm_bios_log *log = &chip->log;
 	void *limit = log->bios_event_log_end;
 	u32 converted_event_size;
 	u32 converted_event_type;
@@ -260,13 +263,10 @@ static int tpm_binary_bios_measurements_show(struct seq_file *m, void *v)
 static int tpm_bios_measurements_release(struct inode *inode,
 					 struct file *file)
 {
-	struct seq_file *seq = file->private_data;
-	struct tpm_bios_log *log = seq->private;
+	struct seq_file *seq = (struct seq_file *)file->private_data;
+	struct tpm_chip *chip = (struct tpm_chip *)seq->private;
 
-	if (log) {
-		kfree(log->bios_event_log);
-		kfree(log);
-	}
+	put_device(&chip->dev);
 
 	return seq_release(inode, file);
 }
@@ -304,151 +304,159 @@ static int tpm_ascii_bios_measurements_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static const struct seq_operations tpm_ascii_b_measurments_seqops = {
+static const struct seq_operations tpm_ascii_b_measurements_seqops = {
 	.start = tpm_bios_measurements_start,
 	.next = tpm_bios_measurements_next,
 	.stop = tpm_bios_measurements_stop,
 	.show = tpm_ascii_bios_measurements_show,
 };
 
-static const struct seq_operations tpm_binary_b_measurments_seqops = {
+static const struct seq_operations tpm_binary_b_measurements_seqops = {
 	.start = tpm_bios_measurements_start,
 	.next = tpm_bios_measurements_next,
 	.stop = tpm_bios_measurements_stop,
 	.show = tpm_binary_bios_measurements_show,
 };
 
-static int tpm_ascii_bios_measurements_open(struct inode *inode,
+static int tpm_bios_measurements_open(struct inode *inode,
 					    struct file *file)
 {
 	int err;
-	struct tpm_bios_log *log;
 	struct seq_file *seq;
-
-	log = kzalloc(sizeof(struct tpm_bios_log), GFP_KERNEL);
-	if (!log)
-		return -ENOMEM;
-
-	if ((err = read_log(log)))
-		goto out_free;
+	struct tpm_chip_seqops *chip_seqops;
+	const struct seq_operations *seqops;
+	struct tpm_chip *chip;
+
+	inode_lock(inode);
+	if (!inode->i_private) {
+		inode_unlock(inode);
+		return -ENODEV;
+	}
+	chip_seqops = (struct tpm_chip_seqops *)inode->i_private;
+	seqops = chip_seqops->seqops;
+	chip = chip_seqops->chip;
+	get_device(&chip->dev);
+	inode_unlock(inode);
 
 	/* now register seq file */
-	err = seq_open(file, &tpm_ascii_b_measurments_seqops);
+	err = seq_open(file, seqops);
 	if (!err) {
 		seq = file->private_data;
-		seq->private = log;
-	} else {
-		goto out_free;
+		seq->private = chip;
 	}
 
-out:
 	return err;
-out_free:
-	kfree(log->bios_event_log);
-	kfree(log);
-	goto out;
 }
 
-static const struct file_operations tpm_ascii_bios_measurements_ops = {
-	.open = tpm_ascii_bios_measurements_open,
+static const struct file_operations tpm_bios_measurements_ops = {
+	.owner = THIS_MODULE,
+	.open = tpm_bios_measurements_open,
 	.read = seq_read,
 	.llseek = seq_lseek,
 	.release = tpm_bios_measurements_release,
 };
 
-static int tpm_binary_bios_measurements_open(struct inode *inode,
-					     struct file *file)
+static int tpm_read_log(struct tpm_chip *chip)
 {
-	int err;
-	struct tpm_bios_log *log;
-	struct seq_file *seq;
-
-	log = kzalloc(sizeof(struct tpm_bios_log), GFP_KERNEL);
-	if (!log)
-		return -ENOMEM;
+	int rc;
 
-	if ((err = read_log(log)))
-		goto out_free;
-
-	/* now register seq file */
-	err = seq_open(file, &tpm_binary_b_measurments_seqops);
-	if (!err) {
-		seq = file->private_data;
-		seq->private = log;
-	} else {
-		goto out_free;
+	if (chip->log.bios_event_log != NULL) {
+		dev_dbg(&chip->dev,
+			"%s: ERROR - event log already initialized\n",
+			__func__);
+		return -EFAULT;
 	}
 
-out:
-	return err;
-out_free:
-	kfree(log->bios_event_log);
-	kfree(log);
-	goto out;
-}
-
-static const struct file_operations tpm_binary_bios_measurements_ops = {
-	.open = tpm_binary_bios_measurements_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = tpm_bios_measurements_release,
-};
+	rc = tpm_read_log_acpi(chip);
+	if (rc != -ENODEV)
+		return rc;
 
-static int is_bad(void *p)
-{
-	if (!p)
-		return 1;
-	if (IS_ERR(p) && (PTR_ERR(p) != -ENODEV))
-		return 1;
-	return 0;
+	return tpm_read_log_of(chip);
 }
 
-struct dentry **tpm_bios_log_setup(const char *name)
+/*
+ * tpm_bios_log_setup() - Read the event log from the firmware
+ * @chip: TPM chip to use.
+ *
+ * If an event log is found then the securityfs files are setup to
+ * export it to userspace, otherwise nothing is done.
+ *
+ * Returns -ENODEV if the firmware has no event log or securityfs is not
+ * supported.
+ */
+int tpm_bios_log_setup(struct tpm_chip *chip)
 {
-	struct dentry **ret = NULL, *tpm_dir, *bin_file, *ascii_file;
-
-	tpm_dir = securityfs_create_dir(name, NULL);
-	if (is_bad(tpm_dir))
-		goto out;
-
-	bin_file =
+	const char *name = dev_name(&chip->dev);
+	unsigned int cnt;
+	int rc = 0;
+
+	if (chip->flags & TPM_CHIP_FLAG_TPM2)
+		return 0;
+
+	rc = tpm_read_log(chip);
+	if (rc)
+		return rc;
+
+	cnt = 0;
+	chip->bios_dir[cnt] = securityfs_create_dir(name, NULL);
+	/* NOTE: securityfs_create_dir can return ENODEV if securityfs is
+	 * compiled out. The caller should ignore the ENODEV return code.
+	 */
+	if (IS_ERR(chip->bios_dir[cnt]))
+		goto err;
+	cnt++;
+
+	chip->bin_log_seqops.chip = chip;
+	chip->bin_log_seqops.seqops = &tpm_binary_b_measurements_seqops;
+
+	chip->bios_dir[cnt] =
 	    securityfs_create_file("binary_bios_measurements",
-				   S_IRUSR | S_IRGRP, tpm_dir, NULL,
-				   &tpm_binary_bios_measurements_ops);
-	if (is_bad(bin_file))
-		goto out_tpm;
+				   0440, chip->bios_dir[0],
+				   (void *)&chip->bin_log_seqops,
+				   &tpm_bios_measurements_ops);
+	if (IS_ERR(chip->bios_dir[cnt]))
+		goto err;
+	cnt++;
+
+	chip->ascii_log_seqops.chip = chip;
+	chip->ascii_log_seqops.seqops = &tpm_ascii_b_measurements_seqops;
 
-	ascii_file =
+	chip->bios_dir[cnt] =
 	    securityfs_create_file("ascii_bios_measurements",
-				   S_IRUSR | S_IRGRP, tpm_dir, NULL,
-				   &tpm_ascii_bios_measurements_ops);
-	if (is_bad(ascii_file))
-		goto out_bin;
-
-	ret = kmalloc(3 * sizeof(struct dentry *), GFP_KERNEL);
-	if (!ret)
-		goto out_ascii;
-
-	ret[0] = ascii_file;
-	ret[1] = bin_file;
-	ret[2] = tpm_dir;
-
-	return ret;
-
-out_ascii:
-	securityfs_remove(ascii_file);
-out_bin:
-	securityfs_remove(bin_file);
-out_tpm:
-	securityfs_remove(tpm_dir);
-out:
-	return NULL;
+				   0440, chip->bios_dir[0],
+				   (void *)&chip->ascii_log_seqops,
+				   &tpm_bios_measurements_ops);
+	if (IS_ERR(chip->bios_dir[cnt]))
+		goto err;
+	cnt++;
+
+	return 0;
+
+err:
+	rc = PTR_ERR(chip->bios_dir[cnt]);
+	chip->bios_dir[cnt] = NULL;
+	tpm_bios_log_teardown(chip);
+	return rc;
 }
 
-void tpm_bios_log_teardown(struct dentry **lst)
+void tpm_bios_log_teardown(struct tpm_chip *chip)
 {
 	int i;
-
-	for (i = 0; i < 3; i++)
-		securityfs_remove(lst[i]);
+	struct inode *inode;
+
+	/* securityfs_remove currently doesn't take care of handling sync
+	 * between removal and opening of pseudo files. To handle this, a
+	 * workaround is added by making i_private = NULL here during removal
+	 * and to check it during open(), both within inode_lock()/unlock().
+	 * This design ensures that open() either safely gets kref or fails.
+	 */
+	for (i = (TPM_NUM_EVENT_LOG_FILES - 1); i >= 0; i--) {
+		if (chip->bios_dir[i]) {
+			inode = d_inode(chip->bios_dir[i]);
+			inode_lock(inode);
+			inode->i_private = NULL;
+			inode_unlock(inode);
+			securityfs_remove(chip->bios_dir[i]);
+		}
+	}
 }
diff --git a/drivers/char/tpm/tpm_eventlog.h b/drivers/char/tpm/tpm_eventlog.h
index 8de62b09be51..1660d74ea79a 100644
--- a/drivers/char/tpm/tpm_eventlog.h
+++ b/drivers/char/tpm/tpm_eventlog.h
@@ -73,20 +73,24 @@ enum tcpa_pc_event_ids {
 	HOST_TABLE_OF_DEVICES,
 };
 
-int read_log(struct tpm_bios_log *log);
-
-#if defined(CONFIG_TCG_IBMVTPM) || defined(CONFIG_TCG_IBMVTPM_MODULE) || \
-	defined(CONFIG_ACPI)
-extern struct dentry **tpm_bios_log_setup(const char *);
-extern void tpm_bios_log_teardown(struct dentry **);
+#if defined(CONFIG_ACPI)
+int tpm_read_log_acpi(struct tpm_chip *chip);
 #else
-static inline struct dentry **tpm_bios_log_setup(const char *name)
+static inline int tpm_read_log_acpi(struct tpm_chip *chip)
 {
-	return NULL;
+	return -ENODEV;
 }
-static inline void tpm_bios_log_teardown(struct dentry **dir)
+#endif
+#if defined(CONFIG_OF)
+int tpm_read_log_of(struct tpm_chip *chip);
+#else
+static inline int tpm_read_log_of(struct tpm_chip *chip)
 {
+	return -ENODEV;
 }
 #endif
 
+int tpm_bios_log_setup(struct tpm_chip *chip);
+void tpm_bios_log_teardown(struct tpm_chip *chip);
+
 #endif
diff --git a/drivers/char/tpm/tpm_of.c b/drivers/char/tpm/tpm_of.c
index 570f30c5c5f4..7dee42d7b5e0 100644
--- a/drivers/char/tpm/tpm_of.c
+++ b/drivers/char/tpm/tpm_of.c
@@ -2,6 +2,7 @@
  * Copyright 2012 IBM Corporation
  *
  * Author: Ashley Lai <ashleydlai@gmail.com>
+ *         Nayna Jain <nayna@linux.vnet.ibm.com>
  *
  * Maintained by: <tpmdd-devel@lists.sourceforge.net>
  *
@@ -20,55 +21,38 @@
 #include "tpm.h"
 #include "tpm_eventlog.h"
 
-int read_log(struct tpm_bios_log *log)
+int tpm_read_log_of(struct tpm_chip *chip)
 {
 	struct device_node *np;
 	const u32 *sizep;
 	const u64 *basep;
+	struct tpm_bios_log *log;
 
-	if (log->bios_event_log != NULL) {
-		pr_err("%s: ERROR - Eventlog already initialized\n", __func__);
-		return -EFAULT;
-	}
-
-	np = of_find_node_by_name(NULL, "vtpm");
-	if (!np) {
-		pr_err("%s: ERROR - IBMVTPM not supported\n", __func__);
+	log = &chip->log;
+	if (chip->dev.parent && chip->dev.parent->of_node)
+		np = chip->dev.parent->of_node;
+	else
 		return -ENODEV;
-	}
 
 	sizep = of_get_property(np, "linux,sml-size", NULL);
-	if (sizep == NULL) {
-		pr_err("%s: ERROR - SML size not found\n", __func__);
-		goto cleanup_eio;
-	}
-	if (*sizep == 0) {
-		pr_err("%s: ERROR - event log area empty\n", __func__);
-		goto cleanup_eio;
-	}
-
 	basep = of_get_property(np, "linux,sml-base", NULL);
-	if (basep == NULL) {
-		pr_err("%s: ERROR - SML not found\n", __func__);
-		goto cleanup_eio;
+	if (sizep == NULL && basep == NULL)
+		return -ENODEV;
+	if (sizep == NULL || basep == NULL)
+		return -EIO;
+
+	if (*sizep == 0) {
+		dev_warn(&chip->dev, "%s: Event log area empty\n", __func__);
+		return -EIO;
 	}
 
 	log->bios_event_log = kmalloc(*sizep, GFP_KERNEL);
-	if (!log->bios_event_log) {
-		pr_err("%s: ERROR - Not enough memory for BIOS measurements\n",
-		       __func__);
-		of_node_put(np);
+	if (!log->bios_event_log)
 		return -ENOMEM;
-	}
 
 	log->bios_event_log_end = log->bios_event_log + *sizep;
 
 	memcpy(log->bios_event_log, __va(*basep), *sizep);
-	of_node_put(np);
 
 	return 0;
-
-cleanup_eio:
-	of_node_put(np);
-	return -EIO;
 }
diff --git a/drivers/char/tpm/tpm_tis.c b/drivers/char/tpm/tpm_tis.c
index eaf5730d79eb..0127af130cb1 100644
--- a/drivers/char/tpm/tpm_tis.c
+++ b/drivers/char/tpm/tpm_tis.c
@@ -28,6 +28,8 @@
 #include <linux/wait.h>
 #include <linux/acpi.h>
 #include <linux/freezer.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
 #include "tpm.h"
 #include "tpm_tis_core.h"
 
@@ -354,12 +356,21 @@ static int tpm_tis_plat_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_OF
+static const struct of_device_id tis_of_platform_match[] = {
+	{.compatible = "tcg,tpm-tis-mmio"},
+	{},
+};
+MODULE_DEVICE_TABLE(of, tis_of_platform_match);
+#endif
+
 static struct platform_driver tis_drv = {
 	.probe = tpm_tis_plat_probe,
 	.remove = tpm_tis_plat_remove,
 	.driver = {
 		.name		= "tpm_tis",
 		.pm		= &tpm_tis_pm,
+		.of_match_table = of_match_ptr(tis_of_platform_match),
 	},
 };
 
diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c
index e3bf31b37138..7993678954a2 100644
--- a/drivers/char/tpm/tpm_tis_core.c
+++ b/drivers/char/tpm/tpm_tis_core.c
@@ -180,12 +180,19 @@ static int recv_data(struct tpm_chip *chip, u8 *buf, size_t count)
 	struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
 	int size = 0, burstcnt, rc;
 
-	while (size < count &&
-	       wait_for_tpm_stat(chip,
+	while (size < count) {
+		rc = wait_for_tpm_stat(chip,
 				 TPM_STS_DATA_AVAIL | TPM_STS_VALID,
 				 chip->timeout_c,
-				 &priv->read_queue, true) == 0) {
-		burstcnt = min_t(int, get_burstcount(chip), count - size);
+				 &priv->read_queue, true);
+		if (rc < 0)
+			return rc;
+		burstcnt = get_burstcount(chip);
+		if (burstcnt < 0) {
+			dev_err(&chip->dev, "Unable to read burstcount\n");
+			return burstcnt;
+		}
+		burstcnt = min_t(int, burstcnt, count - size);
 
 		rc = tpm_tis_read_bytes(priv, TPM_DATA_FIFO(priv->locality),
 					burstcnt, buf + size);
@@ -229,8 +236,11 @@ static int tpm_tis_recv(struct tpm_chip *chip, u8 *buf, size_t count)
 		goto out;
 	}
 
-	wait_for_tpm_stat(chip, TPM_STS_VALID, chip->timeout_c,
-			  &priv->int_queue, false);
+	if (wait_for_tpm_stat(chip, TPM_STS_VALID, chip->timeout_c,
+				&priv->int_queue, false) < 0) {
+		size = -ETIME;
+		goto out;
+	}
 	status = tpm_tis_status(chip);
 	if (status & TPM_STS_DATA_AVAIL) {	/* retry? */
 		dev_err(&chip->dev, "Error left over data\n");
@@ -271,7 +281,13 @@ static int tpm_tis_send_data(struct tpm_chip *chip, u8 *buf, size_t len)
 	}
 
 	while (count < len - 1) {
-		burstcnt = min_t(int, get_burstcount(chip), len - count - 1);
+		burstcnt = get_burstcount(chip);
+		if (burstcnt < 0) {
+			dev_err(&chip->dev, "Unable to read burstcount\n");
+			rc = burstcnt;
+			goto out_err;
+		}
+		burstcnt = min_t(int, burstcnt, len - count - 1);
 		rc = tpm_tis_write_bytes(priv, TPM_DATA_FIFO(priv->locality),
 					 burstcnt, buf + count);
 		if (rc < 0)
@@ -279,8 +295,11 @@ static int tpm_tis_send_data(struct tpm_chip *chip, u8 *buf, size_t len)
 
 		count += burstcnt;
 
-		wait_for_tpm_stat(chip, TPM_STS_VALID, chip->timeout_c,
-				  &priv->int_queue, false);
+		if (wait_for_tpm_stat(chip, TPM_STS_VALID, chip->timeout_c,
+					&priv->int_queue, false) < 0) {
+			rc = -ETIME;
+			goto out_err;
+		}
 		status = tpm_tis_status(chip);
 		if (!itpm && (status & TPM_STS_DATA_EXPECT) == 0) {
 			rc = -EIO;
@@ -293,8 +312,11 @@ static int tpm_tis_send_data(struct tpm_chip *chip, u8 *buf, size_t len)
 	if (rc < 0)
 		goto out_err;
 
-	wait_for_tpm_stat(chip, TPM_STS_VALID, chip->timeout_c,
-			  &priv->int_queue, false);
+	if (wait_for_tpm_stat(chip, TPM_STS_VALID, chip->timeout_c,
+				&priv->int_queue, false) < 0) {
+		rc = -ETIME;
+		goto out_err;
+	}
 	status = tpm_tis_status(chip);
 	if (!itpm && (status & TPM_STS_DATA_EXPECT) != 0) {
 		rc = -EIO;
@@ -755,20 +777,20 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
 	if (intfcaps & TPM_INTF_DATA_AVAIL_INT)
 		dev_dbg(dev, "\tData Avail Int Support\n");
 
-	/* Very early on issue a command to the TPM in polling mode to make
-	 * sure it works. May as well use that command to set the proper
-	 *  timeouts for the driver.
-	 */
-	if (tpm_get_timeouts(chip)) {
-		dev_err(dev, "Could not get TPM timeouts and durations\n");
-		rc = -ENODEV;
-		goto out_err;
-	}
-
 	/* INTERRUPT Setup */
 	init_waitqueue_head(&priv->read_queue);
 	init_waitqueue_head(&priv->int_queue);
 	if (irq != -1) {
+		/* Before doing irq testing issue a command to the TPM in polling mode
+		 * to make sure it works. May as well use that command to set the
+		 * proper timeouts for the driver.
+		 */
+		if (tpm_get_timeouts(chip)) {
+			dev_err(dev, "Could not get TPM timeouts and durations\n");
+			rc = -ENODEV;
+			goto out_err;
+		}
+
 		if (irq) {
 			tpm_tis_probe_irq_single(chip, intmask, IRQF_SHARED,
 						 irq);
diff --git a/drivers/char/tpm/tpm_vtpm_proxy.c b/drivers/char/tpm/tpm_vtpm_proxy.c
index 9a940332c157..5463b58af26e 100644
--- a/drivers/char/tpm/tpm_vtpm_proxy.c
+++ b/drivers/char/tpm/tpm_vtpm_proxy.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (C) 2015, 2016 IBM Corporation
+ * Copyright (C) 2016 Intel Corporation
  *
  * Author: Stefan Berger <stefanb@us.ibm.com>
  *
@@ -41,6 +42,7 @@ struct proxy_dev {
 	long state;                  /* internal state */
 #define STATE_OPENED_FLAG        BIT(0)
 #define STATE_WAIT_RESPONSE_FLAG BIT(1)  /* waiting for emulator response */
+#define STATE_REGISTERED_FLAG	 BIT(2)
 
 	size_t req_len;              /* length of queued TPM request */
 	size_t resp_len;             /* length of queued TPM response */
@@ -369,12 +371,9 @@ static void vtpm_proxy_work(struct work_struct *work)
 
 	rc = tpm_chip_register(proxy_dev->chip);
 	if (rc)
-		goto err;
-
-	return;
-
-err:
-	vtpm_proxy_fops_undo_open(proxy_dev);
+		vtpm_proxy_fops_undo_open(proxy_dev);
+	else
+		proxy_dev->state |= STATE_REGISTERED_FLAG;
 }
 
 /*
@@ -515,7 +514,8 @@ static void vtpm_proxy_delete_device(struct proxy_dev *proxy_dev)
 	 */
 	vtpm_proxy_fops_undo_open(proxy_dev);
 
-	tpm_chip_unregister(proxy_dev->chip);
+	if (proxy_dev->state & STATE_REGISTERED_FLAG)
+		tpm_chip_unregister(proxy_dev->chip);
 
 	vtpm_proxy_delete_proxy_dev(proxy_dev);
 }
@@ -524,6 +524,50 @@ static void vtpm_proxy_delete_device(struct proxy_dev *proxy_dev)
  * Code related to the control device /dev/vtpmx
  */
 
+/**
+ * vtpmx_ioc_new_dev - handler for the %VTPM_PROXY_IOC_NEW_DEV ioctl
+ * @file:	/dev/vtpmx
+ * @ioctl:	the ioctl number
+ * @arg:	pointer to the struct vtpmx_proxy_new_dev
+ *
+ * Creates an anonymous file that is used by the process acting as a TPM to
+ * communicate with the client processes. The function will also add a new TPM
+ * device through which data is proxied to this TPM acting process. The caller
+ * will be provided with a file descriptor to communicate with the clients and
+ * major and minor numbers for the TPM device.
+ */
+static long vtpmx_ioc_new_dev(struct file *file, unsigned int ioctl,
+			      unsigned long arg)
+{
+	void __user *argp = (void __user *)arg;
+	struct vtpm_proxy_new_dev __user *vtpm_new_dev_p;
+	struct vtpm_proxy_new_dev vtpm_new_dev;
+	struct file *vtpm_file;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	vtpm_new_dev_p = argp;
+
+	if (copy_from_user(&vtpm_new_dev, vtpm_new_dev_p,
+			   sizeof(vtpm_new_dev)))
+		return -EFAULT;
+
+	vtpm_file = vtpm_proxy_create_device(&vtpm_new_dev);
+	if (IS_ERR(vtpm_file))
+		return PTR_ERR(vtpm_file);
+
+	if (copy_to_user(vtpm_new_dev_p, &vtpm_new_dev,
+			 sizeof(vtpm_new_dev))) {
+		put_unused_fd(vtpm_new_dev.fd);
+		fput(vtpm_file);
+		return -EFAULT;
+	}
+
+	fd_install(vtpm_new_dev.fd, vtpm_file);
+	return 0;
+}
+
 /*
  * vtpmx_fops_ioctl: ioctl on /dev/vtpmx
  *
@@ -531,34 +575,11 @@ static void vtpm_proxy_delete_device(struct proxy_dev *proxy_dev)
  *      Returns 0 on success, a negative error code otherwise.
  */
 static long vtpmx_fops_ioctl(struct file *f, unsigned int ioctl,
-				   unsigned long arg)
+			     unsigned long arg)
 {
-	void __user *argp = (void __user *)arg;
-	struct vtpm_proxy_new_dev __user *vtpm_new_dev_p;
-	struct vtpm_proxy_new_dev vtpm_new_dev;
-	struct file *file;
-
 	switch (ioctl) {
 	case VTPM_PROXY_IOC_NEW_DEV:
-		if (!capable(CAP_SYS_ADMIN))
-			return -EPERM;
-		vtpm_new_dev_p = argp;
-		if (copy_from_user(&vtpm_new_dev, vtpm_new_dev_p,
-				   sizeof(vtpm_new_dev)))
-			return -EFAULT;
-		file = vtpm_proxy_create_device(&vtpm_new_dev);
-		if (IS_ERR(file))
-			return PTR_ERR(file);
-		if (copy_to_user(vtpm_new_dev_p, &vtpm_new_dev,
-				 sizeof(vtpm_new_dev))) {
-			put_unused_fd(vtpm_new_dev.fd);
-			fput(file);
-			return -EFAULT;
-		}
-
-		fd_install(vtpm_new_dev.fd, file);
-		return 0;
-
+		return vtpmx_ioc_new_dev(f, ioctl, arg);
 	default:
 		return -ENOIOCTLCMD;
 	}
diff --git a/drivers/char/tpm/xen-tpmfront.c b/drivers/char/tpm/xen-tpmfront.c
index 62028f483bba..5aaa268f3a78 100644
--- a/drivers/char/tpm/xen-tpmfront.c
+++ b/drivers/char/tpm/xen-tpmfront.c
@@ -307,7 +307,6 @@ static int tpmfront_probe(struct xenbus_device *dev,
 	rv = setup_ring(dev, priv);
 	if (rv) {
 		chip = dev_get_drvdata(&dev->dev);
-		tpm_chip_unregister(chip);
 		ring_free(priv);
 		return rv;
 	}
@@ -337,18 +336,14 @@ static int tpmfront_resume(struct xenbus_device *dev)
 static void backend_changed(struct xenbus_device *dev,
 		enum xenbus_state backend_state)
 {
-	int val;
-
 	switch (backend_state) {
 	case XenbusStateInitialised:
 	case XenbusStateConnected:
 		if (dev->state == XenbusStateConnected)
 			break;
 
-		if (xenbus_scanf(XBT_NIL, dev->otherend,
-				"feature-protocol-v2", "%d", &val) < 0)
-			val = 0;
-		if (!val) {
+		if (!xenbus_read_unsigned(dev->otherend, "feature-protocol-v2",
+					  0)) {
 			xenbus_dev_fatal(dev, -EINVAL,
 					"vTPM protocol 2 required");
 			return;
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index 5649234b7316..8b00e79c2683 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -152,8 +152,8 @@ struct ports_device {
 	spinlock_t c_ivq_lock;
 	spinlock_t c_ovq_lock;
 
-	/* The current config space is stored here */
-	struct virtio_console_config config;
+	/* max. number of ports this device can hold */
+	u32 max_nr_ports;
 
 	/* The virtio device we're associated with */
 	struct virtio_device *vdev;
@@ -1649,11 +1649,11 @@ static void handle_control_message(struct virtio_device *vdev,
 			break;
 		}
 		if (virtio32_to_cpu(vdev, cpkt->id) >=
-		    portdev->config.max_nr_ports) {
+		    portdev->max_nr_ports) {
 			dev_warn(&portdev->vdev->dev,
 				"Request for adding port with "
 				"out-of-bound id %u, max. supported id: %u\n",
-				cpkt->id, portdev->config.max_nr_ports - 1);
+				cpkt->id, portdev->max_nr_ports - 1);
 			break;
 		}
 		add_port(portdev, virtio32_to_cpu(vdev, cpkt->id));
@@ -1894,7 +1894,7 @@ static int init_vqs(struct ports_device *portdev)
 	u32 i, j, nr_ports, nr_queues;
 	int err;
 
-	nr_ports = portdev->config.max_nr_ports;
+	nr_ports = portdev->max_nr_ports;
 	nr_queues = use_multiport(portdev) ? (nr_ports + 1) * 2 : 2;
 
 	vqs = kmalloc(nr_queues * sizeof(struct virtqueue *), GFP_KERNEL);
@@ -2047,13 +2047,13 @@ static int virtcons_probe(struct virtio_device *vdev)
 	}
 
 	multiport = false;
-	portdev->config.max_nr_ports = 1;
+	portdev->max_nr_ports = 1;
 
 	/* Don't test MULTIPORT at all if we're rproc: not a valid feature! */
 	if (!is_rproc_serial(vdev) &&
 	    virtio_cread_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT,
 				 struct virtio_console_config, max_nr_ports,
-				 &portdev->config.max_nr_ports) == 0) {
+				 &portdev->max_nr_ports) == 0) {
 		multiport = true;
 	}
 
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index 6a8ac04bedeb..56c1998ced3e 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -33,7 +33,7 @@ source "drivers/clk/versatile/Kconfig"
 
 config COMMON_CLK_MAX77686
 	tristate "Clock driver for Maxim 77620/77686/77802 MFD"
-	depends on MFD_MAX77686 || MFD_MAX77620
+	depends on MFD_MAX77686 || MFD_MAX77620 || COMPILE_TEST
 	---help---
 	  This driver supports Maxim 77620/77686/77802 crystal oscillator
 	  clock.
@@ -119,7 +119,7 @@ config COMMON_CLK_CS2000_CP
 
 config COMMON_CLK_S2MPS11
 	tristate "Clock driver for S2MPS1X/S5M8767 MFD"
-	depends on MFD_SEC_CORE
+	depends on MFD_SEC_CORE || COMPILE_TEST
 	---help---
 	  This driver supports S2MPS11/S2MPS14/S5M8767 crystal oscillator
 	  clock. These multi-function devices have two (S2MPS14) or three
diff --git a/drivers/clk/bcm/Kconfig b/drivers/clk/bcm/Kconfig
index e3eed5a78404..b5ae5311b0a2 100644
--- a/drivers/clk/bcm/Kconfig
+++ b/drivers/clk/bcm/Kconfig
@@ -1,7 +1,6 @@
 config CLK_BCM_63XX
 	bool "Broadcom BCM63xx clock support"
 	depends on ARCH_BCM_63XX || COMPILE_TEST
-	depends on COMMON_CLK
 	select COMMON_CLK_IPROC
 	default ARCH_BCM_63XX
 	help
@@ -11,27 +10,22 @@ config CLK_BCM_63XX
 config CLK_BCM_KONA
 	bool "Broadcom Kona CCU clock support"
 	depends on ARCH_BCM_MOBILE || COMPILE_TEST
-	depends on COMMON_CLK
-	default y
+	default ARCH_BCM_MOBILE
 	help
 	  Enable common clock framework support for Broadcom SoCs
 	  using "Kona" style clock control units, including those
 	  in the BCM281xx and BCM21664 families.
 
 config COMMON_CLK_IPROC
-	bool "Broadcom iProc clock support"
-	depends on ARCH_BCM_IPROC || ARCH_BCM_63XX || COMPILE_TEST
-	depends on COMMON_CLK
-	default ARCH_BCM_IPROC
+	bool
 	help
 	  Enable common clock framework support for Broadcom SoCs
 	  based on the iProc architecture
 
-if COMMON_CLK_IPROC
-
 config CLK_BCM_CYGNUS
 	bool "Broadcom Cygnus clock support"
 	depends on ARCH_BCM_CYGNUS || COMPILE_TEST
+	select COMMON_CLK_IPROC
 	default ARCH_BCM_CYGNUS
 	help
 	  Enable common clock framework support for the Broadcom Cygnus SoC
@@ -39,6 +33,7 @@ config CLK_BCM_CYGNUS
 config CLK_BCM_NSP
 	bool "Broadcom Northstar/Northstar Plus clock support"
 	depends on ARCH_BCM_5301X || ARCH_BCM_NSP || COMPILE_TEST
+	select COMMON_CLK_IPROC
 	default ARCH_BCM_5301X || ARCH_BCM_NSP
 	help
 	  Enable common clock framework support for the Broadcom Northstar and
@@ -47,8 +42,7 @@ config CLK_BCM_NSP
 config CLK_BCM_NS2
 	bool "Broadcom Northstar 2 clock support"
 	depends on ARCH_BCM_IPROC || COMPILE_TEST
+	select COMMON_CLK_IPROC
 	default ARCH_BCM_IPROC
 	help
 	  Enable common clock framework support for the Broadcom Northstar 2 SoC
-
-endif
diff --git a/drivers/clk/bcm/clk-bcm2835.c b/drivers/clk/bcm/clk-bcm2835.c
index 8c7763fd9efc..0d14409097e7 100644
--- a/drivers/clk/bcm/clk-bcm2835.c
+++ b/drivers/clk/bcm/clk-bcm2835.c
@@ -436,6 +436,9 @@ struct bcm2835_clock_data {
 	const char *const *parents;
 	int num_mux_parents;
 
+	/* Bitmap encoding which parents accept rate change propagation. */
+	unsigned int set_rate_parent;
+
 	u32 ctl_reg;
 	u32 div_reg;
 
@@ -751,7 +754,9 @@ static void bcm2835_pll_divider_off(struct clk_hw *hw)
 	cprman_write(cprman, data->cm_reg,
 		     (cprman_read(cprman, data->cm_reg) &
 		      ~data->load_mask) | data->hold_mask);
-	cprman_write(cprman, data->a2w_reg, A2W_PLL_CHANNEL_DISABLE);
+	cprman_write(cprman, data->a2w_reg,
+		     cprman_read(cprman, data->a2w_reg) |
+		     A2W_PLL_CHANNEL_DISABLE);
 	spin_unlock(&cprman->regs_lock);
 }
 
@@ -1015,10 +1020,60 @@ bcm2835_clk_is_pllc(struct clk_hw *hw)
 	return strncmp(clk_hw_get_name(hw), "pllc", 4) == 0;
 }
 
+static unsigned long bcm2835_clock_choose_div_and_prate(struct clk_hw *hw,
+							int parent_idx,
+							unsigned long rate,
+							u32 *div,
+							unsigned long *prate)
+{
+	struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
+	struct bcm2835_cprman *cprman = clock->cprman;
+	const struct bcm2835_clock_data *data = clock->data;
+	unsigned long best_rate = 0;
+	u32 curdiv, mindiv, maxdiv;
+	struct clk_hw *parent;
+
+	parent = clk_hw_get_parent_by_index(hw, parent_idx);
+
+	if (!(BIT(parent_idx) & data->set_rate_parent)) {
+		*prate = clk_hw_get_rate(parent);
+		*div = bcm2835_clock_choose_div(hw, rate, *prate, true);
+
+		return bcm2835_clock_rate_from_divisor(clock, *prate,
+						       *div);
+	}
+
+	if (data->frac_bits)
+		dev_warn(cprman->dev,
+			"frac bits are not used when propagating rate change");
+
+	/* clamp to min divider of 2 if we're dealing with a mash clock */
+	mindiv = data->is_mash_clock ? 2 : 1;
+	maxdiv = BIT(data->int_bits) - 1;
+
+	/* TODO: Be smart, and only test a subset of the available divisors. */
+	for (curdiv = mindiv; curdiv <= maxdiv; curdiv++) {
+		unsigned long tmp_rate;
+
+		tmp_rate = clk_hw_round_rate(parent, rate * curdiv);
+		tmp_rate /= curdiv;
+		if (curdiv == mindiv ||
+		    (tmp_rate > best_rate && tmp_rate <= rate))
+			best_rate = tmp_rate;
+
+		if (best_rate == rate)
+			break;
+	}
+
+	*div = curdiv << CM_DIV_FRAC_BITS;
+	*prate = curdiv * best_rate;
+
+	return best_rate;
+}
+
 static int bcm2835_clock_determine_rate(struct clk_hw *hw,
 					struct clk_rate_request *req)
 {
-	struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw);
 	struct clk_hw *parent, *best_parent = NULL;
 	bool current_parent_is_pllc;
 	unsigned long rate, best_rate = 0;
@@ -1046,9 +1101,8 @@ static int bcm2835_clock_determine_rate(struct clk_hw *hw,
 		if (bcm2835_clk_is_pllc(parent) && !current_parent_is_pllc)
 			continue;
 
-		prate = clk_hw_get_rate(parent);
-		div = bcm2835_clock_choose_div(hw, req->rate, prate, true);
-		rate = bcm2835_clock_rate_from_divisor(clock, prate, div);
+		rate = bcm2835_clock_choose_div_and_prate(hw, i, req->rate,
+							  &div, &prate);
 		if (rate > best_rate && rate <= req->rate) {
 			best_parent = parent;
 			best_prate = prate;
@@ -1260,6 +1314,13 @@ static struct clk_hw *bcm2835_register_clock(struct bcm2835_cprman *cprman,
 	init.name = data->name;
 	init.flags = data->flags | CLK_IGNORE_UNUSED;
 
+	/*
+	 * Pass the CLK_SET_RATE_PARENT flag if we are allowed to propagate
+	 * rate changes on at least of the parents.
+	 */
+	if (data->set_rate_parent)
+		init.flags |= CLK_SET_RATE_PARENT;
+
 	if (data->is_vpu_clock) {
 		init.ops = &bcm2835_vpu_clock_clk_ops;
 	} else {
@@ -1596,7 +1657,7 @@ static const struct bcm2835_clk_desc clk_desc_array[] = {
 		.a2w_reg = A2W_PLLH_AUX,
 		.load_mask = CM_PLLH_LOADAUX,
 		.hold_mask = 0,
-		.fixed_divider = 10),
+		.fixed_divider = 1),
 	[BCM2835_PLLH_PIX]	= REGISTER_PLL_DIV(
 		.name = "pllh_pix",
 		.source_pll = "pllh",
@@ -1800,7 +1861,12 @@ static const struct bcm2835_clk_desc clk_desc_array[] = {
 		.ctl_reg = CM_VECCTL,
 		.div_reg = CM_VECDIV,
 		.int_bits = 4,
-		.frac_bits = 0),
+		.frac_bits = 0,
+		/*
+		 * Allow rate change propagation only on PLLH_AUX which is
+		 * assigned index 7 in the parent array.
+		 */
+		.set_rate_parent = BIT(7)),
 
 	/* dsi clocks */
 	[BCM2835_CLOCK_DSI0E]	= REGISTER_PER_CLK(
diff --git a/drivers/clk/clk-cdce925.c b/drivers/clk/clk-cdce925.c
index b8459c14a1b7..f793b2d9238c 100644
--- a/drivers/clk/clk-cdce925.c
+++ b/drivers/clk/clk-cdce925.c
@@ -216,7 +216,7 @@ static int cdce925_pll_prepare(struct clk_hw *hw)
 		nn = n * BIT(p);
 		/* q = int(nn/m) */
 		q = nn / m;
-		if ((q < 16) || (1 > 64)) {
+		if ((q < 16) || (q > 63)) {
 			pr_debug("%s invalid q=%d\n", __func__, q);
 			return -EINVAL;
 		}
diff --git a/drivers/clk/clk-devres.c b/drivers/clk/clk-devres.c
index 8f571548870f..3a218c3a06ae 100644
--- a/drivers/clk/clk-devres.c
+++ b/drivers/clk/clk-devres.c
@@ -53,3 +53,24 @@ void devm_clk_put(struct device *dev, struct clk *clk)
 	WARN_ON(ret);
 }
 EXPORT_SYMBOL(devm_clk_put);
+
+struct clk *devm_get_clk_from_child(struct device *dev,
+				    struct device_node *np, const char *con_id)
+{
+	struct clk **ptr, *clk;
+
+	ptr = devres_alloc(devm_clk_release, sizeof(*ptr), GFP_KERNEL);
+	if (!ptr)
+		return ERR_PTR(-ENOMEM);
+
+	clk = of_clk_get_by_name(np, con_id);
+	if (!IS_ERR(clk)) {
+		*ptr = clk;
+		devres_add(dev, ptr);
+	} else {
+		devres_free(ptr);
+	}
+
+	return clk;
+}
+EXPORT_SYMBOL(devm_get_clk_from_child);
diff --git a/drivers/clk/clk-gate.c b/drivers/clk/clk-gate.c
index 4e691e35483a..4e0c054a787c 100644
--- a/drivers/clk/clk-gate.c
+++ b/drivers/clk/clk-gate.c
@@ -145,8 +145,8 @@ struct clk_hw *clk_hw_register_gate(struct device *dev, const char *name,
 	init.name = name;
 	init.ops = &clk_gate_ops;
 	init.flags = flags | CLK_IS_BASIC;
-	init.parent_names = (parent_name ? &parent_name: NULL);
-	init.num_parents = (parent_name ? 1 : 0);
+	init.parent_names = parent_name ? &parent_name : NULL;
+	init.num_parents = parent_name ? 1 : 0;
 
 	/* struct clk_gate assignments */
 	gate->reg = reg;
diff --git a/drivers/clk/clk-oxnas.c b/drivers/clk/clk-oxnas.c
index 47649ac5d399..e51e0023fc6e 100644
--- a/drivers/clk/clk-oxnas.c
+++ b/drivers/clk/clk-oxnas.c
@@ -20,31 +20,43 @@
 #include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/stringify.h>
 #include <linux/regmap.h>
 #include <linux/mfd/syscon.h>
 
+#include <dt-bindings/clock/oxsemi,ox810se.h>
+#include <dt-bindings/clock/oxsemi,ox820.h>
+
 /* Standard regmap gate clocks */
-struct clk_oxnas {
+struct clk_oxnas_gate {
 	struct clk_hw hw;
-	signed char bit;
+	unsigned int bit;
 	struct regmap *regmap;
 };
 
+struct oxnas_stdclk_data {
+	struct clk_hw_onecell_data *onecell_data;
+	struct clk_oxnas_gate **gates;
+	unsigned int ngates;
+	struct clk_oxnas_pll **plls;
+	unsigned int nplls;
+};
+
 /* Regmap offsets */
 #define CLK_STAT_REGOFFSET	0x24
 #define CLK_SET_REGOFFSET	0x2c
 #define CLK_CLR_REGOFFSET	0x30
 
-static inline struct clk_oxnas *to_clk_oxnas(struct clk_hw *hw)
+static inline struct clk_oxnas_gate *to_clk_oxnas_gate(struct clk_hw *hw)
 {
-	return container_of(hw, struct clk_oxnas, hw);
+	return container_of(hw, struct clk_oxnas_gate, hw);
 }
 
-static int oxnas_clk_is_enabled(struct clk_hw *hw)
+static int oxnas_clk_gate_is_enabled(struct clk_hw *hw)
 {
-	struct clk_oxnas *std = to_clk_oxnas(hw);
+	struct clk_oxnas_gate *std = to_clk_oxnas_gate(hw);
 	int ret;
 	unsigned int val;
 
@@ -55,29 +67,29 @@ static int oxnas_clk_is_enabled(struct clk_hw *hw)
 	return val & BIT(std->bit);
 }
 
-static int oxnas_clk_enable(struct clk_hw *hw)
+static int oxnas_clk_gate_enable(struct clk_hw *hw)
 {
-	struct clk_oxnas *std = to_clk_oxnas(hw);
+	struct clk_oxnas_gate *std = to_clk_oxnas_gate(hw);
 
 	regmap_write(std->regmap, CLK_SET_REGOFFSET, BIT(std->bit));
 
 	return 0;
 }
 
-static void oxnas_clk_disable(struct clk_hw *hw)
+static void oxnas_clk_gate_disable(struct clk_hw *hw)
 {
-	struct clk_oxnas *std = to_clk_oxnas(hw);
+	struct clk_oxnas_gate *std = to_clk_oxnas_gate(hw);
 
 	regmap_write(std->regmap, CLK_CLR_REGOFFSET, BIT(std->bit));
 }
 
-static const struct clk_ops oxnas_clk_ops = {
-	.enable = oxnas_clk_enable,
-	.disable = oxnas_clk_disable,
-	.is_enabled = oxnas_clk_is_enabled,
+static const struct clk_ops oxnas_clk_gate_ops = {
+	.enable = oxnas_clk_gate_enable,
+	.disable = oxnas_clk_gate_disable,
+	.is_enabled = oxnas_clk_gate_is_enabled,
 };
 
-static const char *const oxnas_clk_parents[] = {
+static const char *const osc_parents[] = {
 	"oscillator",
 };
 
@@ -85,63 +97,138 @@ static const char *const eth_parents[] = {
 	"gmacclk",
 };
 
-#define DECLARE_STD_CLKP(__clk, __parent)			\
-static const struct clk_init_data clk_##__clk##_init = {	\
-	.name = __stringify(__clk),				\
-	.ops = &oxnas_clk_ops,					\
-	.parent_names = __parent,				\
-	.num_parents = ARRAY_SIZE(__parent),			\
+#define OXNAS_GATE(_name, _bit, _parents)				\
+struct clk_oxnas_gate _name = {						\
+	.bit = (_bit),							\
+	.hw.init = &(struct clk_init_data) {				\
+		.name = #_name,						\
+		.ops = &oxnas_clk_gate_ops,				\
+		.parent_names = _parents,				\
+		.num_parents = ARRAY_SIZE(_parents),			\
+		.flags = (CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED),	\
+	},								\
 }
 
-#define DECLARE_STD_CLK(__clk) DECLARE_STD_CLKP(__clk, oxnas_clk_parents)
+static OXNAS_GATE(ox810se_leon, 0, osc_parents);
+static OXNAS_GATE(ox810se_dma_sgdma, 1, osc_parents);
+static OXNAS_GATE(ox810se_cipher, 2, osc_parents);
+static OXNAS_GATE(ox810se_sata, 4, osc_parents);
+static OXNAS_GATE(ox810se_audio, 5, osc_parents);
+static OXNAS_GATE(ox810se_usbmph, 6, osc_parents);
+static OXNAS_GATE(ox810se_etha, 7, eth_parents);
+static OXNAS_GATE(ox810se_pciea, 8, osc_parents);
+static OXNAS_GATE(ox810se_nand, 9, osc_parents);
+
+static struct clk_oxnas_gate *ox810se_gates[] = {
+	&ox810se_leon,
+	&ox810se_dma_sgdma,
+	&ox810se_cipher,
+	&ox810se_sata,
+	&ox810se_audio,
+	&ox810se_usbmph,
+	&ox810se_etha,
+	&ox810se_pciea,
+	&ox810se_nand,
+};
+
+static OXNAS_GATE(ox820_leon, 0, osc_parents);
+static OXNAS_GATE(ox820_dma_sgdma, 1, osc_parents);
+static OXNAS_GATE(ox820_cipher, 2, osc_parents);
+static OXNAS_GATE(ox820_sd, 3, osc_parents);
+static OXNAS_GATE(ox820_sata, 4, osc_parents);
+static OXNAS_GATE(ox820_audio, 5, osc_parents);
+static OXNAS_GATE(ox820_usbmph, 6, osc_parents);
+static OXNAS_GATE(ox820_etha, 7, eth_parents);
+static OXNAS_GATE(ox820_pciea, 8, osc_parents);
+static OXNAS_GATE(ox820_nand, 9, osc_parents);
+static OXNAS_GATE(ox820_ethb, 10, eth_parents);
+static OXNAS_GATE(ox820_pcieb, 11, osc_parents);
+static OXNAS_GATE(ox820_ref600, 12, osc_parents);
+static OXNAS_GATE(ox820_usbdev, 13, osc_parents);
+
+static struct clk_oxnas_gate *ox820_gates[] = {
+	&ox820_leon,
+	&ox820_dma_sgdma,
+	&ox820_cipher,
+	&ox820_sd,
+	&ox820_sata,
+	&ox820_audio,
+	&ox820_usbmph,
+	&ox820_etha,
+	&ox820_pciea,
+	&ox820_nand,
+	&ox820_etha,
+	&ox820_pciea,
+	&ox820_ref600,
+	&ox820_usbdev,
+};
+
+static struct clk_hw_onecell_data ox810se_hw_onecell_data = {
+	.hws = {
+		[CLK_810_LEON]	= &ox810se_leon.hw,
+		[CLK_810_DMA_SGDMA]	= &ox810se_dma_sgdma.hw,
+		[CLK_810_CIPHER]	= &ox810se_cipher.hw,
+		[CLK_810_SATA]	= &ox810se_sata.hw,
+		[CLK_810_AUDIO]	= &ox810se_audio.hw,
+		[CLK_810_USBMPH]	= &ox810se_usbmph.hw,
+		[CLK_810_ETHA]	= &ox810se_etha.hw,
+		[CLK_810_PCIEA]	= &ox810se_pciea.hw,
+		[CLK_810_NAND]	= &ox810se_nand.hw,
+	},
+	.num = ARRAY_SIZE(ox810se_gates),
+};
+
+static struct clk_hw_onecell_data ox820_hw_onecell_data = {
+	.hws = {
+		[CLK_820_LEON]	= &ox820_leon.hw,
+		[CLK_820_DMA_SGDMA]	= &ox820_dma_sgdma.hw,
+		[CLK_820_CIPHER]	= &ox820_cipher.hw,
+		[CLK_820_SD]	= &ox820_sd.hw,
+		[CLK_820_SATA]	= &ox820_sata.hw,
+		[CLK_820_AUDIO]	= &ox820_audio.hw,
+		[CLK_820_USBMPH]	= &ox820_usbmph.hw,
+		[CLK_820_ETHA]	= &ox820_etha.hw,
+		[CLK_820_PCIEA]	= &ox820_pciea.hw,
+		[CLK_820_NAND]	= &ox820_nand.hw,
+		[CLK_820_ETHB]	= &ox820_ethb.hw,
+		[CLK_820_PCIEB]	= &ox820_pcieb.hw,
+		[CLK_820_REF600]	= &ox820_ref600.hw,
+		[CLK_820_USBDEV]	= &ox820_usbdev.hw,
+	},
+	.num = ARRAY_SIZE(ox820_gates),
+};
 
-/* Hardware Bit - Clock association */
-struct clk_oxnas_init_data {
-	unsigned long bit;
-	const struct clk_init_data *clk_init;
+static struct oxnas_stdclk_data ox810se_stdclk_data = {
+	.onecell_data = &ox810se_hw_onecell_data,
+	.gates = ox810se_gates,
+	.ngates = ARRAY_SIZE(ox810se_gates),
 };
 
-/* Clk init data declaration */
-DECLARE_STD_CLK(leon);
-DECLARE_STD_CLK(dma_sgdma);
-DECLARE_STD_CLK(cipher);
-DECLARE_STD_CLK(sata);
-DECLARE_STD_CLK(audio);
-DECLARE_STD_CLK(usbmph);
-DECLARE_STD_CLKP(etha, eth_parents);
-DECLARE_STD_CLK(pciea);
-DECLARE_STD_CLK(nand);
-
-/* Table index is clock indice */
-static const struct clk_oxnas_init_data clk_oxnas_init[] = {
-	[0] = {0, &clk_leon_init},
-	[1] = {1, &clk_dma_sgdma_init},
-	[2] = {2, &clk_cipher_init},
-	/* Skip & Do not touch to DDR clock */
-	[3] = {4, &clk_sata_init},
-	[4] = {5, &clk_audio_init},
-	[5] = {6, &clk_usbmph_init},
-	[6] = {7, &clk_etha_init},
-	[7] = {8, &clk_pciea_init},
-	[8] = {9, &clk_nand_init},
+static struct oxnas_stdclk_data ox820_stdclk_data = {
+	.onecell_data = &ox820_hw_onecell_data,
+	.gates = ox820_gates,
+	.ngates = ARRAY_SIZE(ox820_gates),
 };
 
-struct clk_oxnas_data {
-	struct clk_oxnas clk_oxnas[ARRAY_SIZE(clk_oxnas_init)];
-	struct clk_onecell_data onecell_data[ARRAY_SIZE(clk_oxnas_init)];
-	struct clk *clks[ARRAY_SIZE(clk_oxnas_init)];
+static const struct of_device_id oxnas_stdclk_dt_ids[] = {
+	{ .compatible = "oxsemi,ox810se-stdclk", &ox810se_stdclk_data },
+	{ .compatible = "oxsemi,ox820-stdclk", &ox820_stdclk_data },
+	{ }
 };
 
 static int oxnas_stdclk_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
-	struct clk_oxnas_data *clk_oxnas;
+	const struct oxnas_stdclk_data *data;
+	const struct of_device_id *id;
 	struct regmap *regmap;
+	int ret;
 	int i;
 
-	clk_oxnas = devm_kzalloc(&pdev->dev, sizeof(*clk_oxnas), GFP_KERNEL);
-	if (!clk_oxnas)
-		return -ENOMEM;
+	id = of_match_device(oxnas_stdclk_dt_ids, &pdev->dev);
+	if (!id)
+		return -ENODEV;
+	data = id->data;
 
 	regmap = syscon_node_to_regmap(of_get_parent(np));
 	if (IS_ERR(regmap)) {
@@ -149,32 +236,23 @@ static int oxnas_stdclk_probe(struct platform_device *pdev)
 		return PTR_ERR(regmap);
 	}
 
-	for (i = 0; i < ARRAY_SIZE(clk_oxnas_init); i++) {
-		struct clk_oxnas *_clk;
+	for (i = 0 ; i < data->ngates ; ++i)
+		data->gates[i]->regmap = regmap;
 
-		_clk = &clk_oxnas->clk_oxnas[i];
-		_clk->bit = clk_oxnas_init[i].bit;
-		_clk->hw.init = clk_oxnas_init[i].clk_init;
-		_clk->regmap = regmap;
+	for (i = 0; i < data->onecell_data->num; i++) {
+		if (!data->onecell_data->hws[i])
+			continue;
 
-		clk_oxnas->clks[i] =
-			devm_clk_register(&pdev->dev, &_clk->hw);
-		if (WARN_ON(IS_ERR(clk_oxnas->clks[i])))
-			return PTR_ERR(clk_oxnas->clks[i]);
+		ret = devm_clk_hw_register(&pdev->dev,
+					   data->onecell_data->hws[i]);
+		if (ret)
+			return ret;
 	}
 
-	clk_oxnas->onecell_data->clks = clk_oxnas->clks;
-	clk_oxnas->onecell_data->clk_num = ARRAY_SIZE(clk_oxnas_init);
-
-	return of_clk_add_provider(np, of_clk_src_onecell_get,
-				   clk_oxnas->onecell_data);
+	return of_clk_add_hw_provider(np, of_clk_hw_onecell_get,
+				      data->onecell_data);
 }
 
-static const struct of_device_id oxnas_stdclk_dt_ids[] = {
-	{ .compatible = "oxsemi,ox810se-stdclk" },
-	{ }
-};
-
 static struct platform_driver oxnas_stdclk_driver = {
 	.probe = oxnas_stdclk_probe,
 	.driver	= {
diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c
index 80ae2a51452d..d0bf8b1c67de 100644
--- a/drivers/clk/clk-qoriq.c
+++ b/drivers/clk/clk-qoriq.c
@@ -266,6 +266,39 @@ static const struct clockgen_muxinfo ls1043a_hwa2 = {
 	},
 };
 
+static const struct clockgen_muxinfo ls1046a_hwa1 = {
+	{
+		{},
+		{},
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV2 },
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV3 },
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV4 },
+		{ CLKSEL_VALID, PLATFORM_PLL, PLL_DIV1 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV2 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV3 },
+	},
+};
+
+static const struct clockgen_muxinfo ls1046a_hwa2 = {
+	{
+		{},
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV1 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV2 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV3 },
+		{},
+		{},
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV2 },
+	},
+};
+
+static const struct clockgen_muxinfo ls1012a_cmux = {
+	{
+		[0] = { CLKSEL_VALID, CGA_PLL1, PLL_DIV1 },
+		{},
+		[2] = { CLKSEL_VALID, CGA_PLL1, PLL_DIV2 },
+	}
+};
+
 static const struct clockgen_muxinfo t1023_hwa1 = {
 	{
 		{},
@@ -489,6 +522,31 @@ static const struct clockgen_chipinfo chipinfo[] = {
 		.flags = CG_PLL_8BIT,
 	},
 	{
+		.compat = "fsl,ls1046a-clockgen",
+		.init_periph = t2080_init_periph,
+		.cmux_groups = {
+			&t1040_cmux
+		},
+		.hwaccel = {
+			&ls1046a_hwa1, &ls1046a_hwa2
+		},
+		.cmux_to_group = {
+			0, -1
+		},
+		.pll_mask = 0x07,
+		.flags = CG_PLL_8BIT,
+	},
+	{
+		.compat = "fsl,ls1012a-clockgen",
+		.cmux_groups = {
+			&ls1012a_cmux
+		},
+		.cmux_to_group = {
+			0, -1
+		},
+		.pll_mask = 0x03,
+	},
+	{
 		.compat = "fsl,ls2080a-clockgen",
 		.cmux_groups = {
 			&clockgen2_cmux_cga12, &clockgen2_cmux_cgb
@@ -1273,8 +1331,10 @@ err:
 
 CLK_OF_DECLARE(qoriq_clockgen_1, "fsl,qoriq-clockgen-1.0", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_2, "fsl,qoriq-clockgen-2.0", clockgen_init);
+CLK_OF_DECLARE(qoriq_clockgen_ls1012a, "fsl,ls1012a-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_ls1021a, "fsl,ls1021a-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_ls1043a, "fsl,ls1043a-clockgen", clockgen_init);
+CLK_OF_DECLARE(qoriq_clockgen_ls1046a, "fsl,ls1046a-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_ls2080a, "fsl,ls2080a-clockgen", clockgen_init);
 
 /* Legacy nodes */
diff --git a/drivers/clk/clk-stm32f4.c b/drivers/clk/clk-stm32f4.c
index 02d681008401..5eb05dbf59b8 100644
--- a/drivers/clk/clk-stm32f4.c
+++ b/drivers/clk/clk-stm32f4.c
@@ -19,10 +19,14 @@
 #include <linux/clk-provider.h>
 #include <linux/err.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/ioport.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
 
 #define STM32F4_RCC_PLLCFGR		0x04
 #define STM32F4_RCC_CFGR		0x08
@@ -31,6 +35,8 @@
 #define STM32F4_RCC_AHB3ENR		0x38
 #define STM32F4_RCC_APB1ENR		0x40
 #define STM32F4_RCC_APB2ENR		0x44
+#define STM32F4_RCC_BDCR		0x70
+#define STM32F4_RCC_CSR			0x74
 
 struct stm32f4_gate_data {
 	u8	offset;
@@ -40,7 +46,7 @@ struct stm32f4_gate_data {
 	unsigned long flags;
 };
 
-static const struct stm32f4_gate_data stm32f4_gates[] __initconst = {
+static const struct stm32f4_gate_data stm32f429_gates[] __initconst = {
 	{ STM32F4_RCC_AHB1ENR,  0,	"gpioa",	"ahb_div" },
 	{ STM32F4_RCC_AHB1ENR,  1,	"gpiob",	"ahb_div" },
 	{ STM32F4_RCC_AHB1ENR,  2,	"gpioc",	"ahb_div" },
@@ -120,26 +126,113 @@ static const struct stm32f4_gate_data stm32f4_gates[] __initconst = {
 	{ STM32F4_RCC_APB2ENR, 26,	"ltdc",		"apb2_div" },
 };
 
-/*
- * MAX_CLKS is the maximum value in the enumeration below plus the combined
- * hweight of stm32f42xx_gate_map (plus one).
- */
-#define MAX_CLKS 74
+static const struct stm32f4_gate_data stm32f469_gates[] __initconst = {
+	{ STM32F4_RCC_AHB1ENR,  0,	"gpioa",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  1,	"gpiob",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  2,	"gpioc",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  3,	"gpiod",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  4,	"gpioe",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  5,	"gpiof",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  6,	"gpiog",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  7,	"gpioh",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  8,	"gpioi",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  9,	"gpioj",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 10,	"gpiok",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 12,	"crc",		"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 18,	"bkpsra",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 20,	"ccmdatam",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 21,	"dma1",		"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 22,	"dma2",		"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 23,	"dma2d",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 25,	"ethmac",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 26,	"ethmactx",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 27,	"ethmacrx",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 28,	"ethmacptp",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 29,	"otghs",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 30,	"otghsulpi",	"ahb_div" },
+
+	{ STM32F4_RCC_AHB2ENR,  0,	"dcmi",		"ahb_div" },
+	{ STM32F4_RCC_AHB2ENR,  4,	"cryp",		"ahb_div" },
+	{ STM32F4_RCC_AHB2ENR,  5,	"hash",		"ahb_div" },
+	{ STM32F4_RCC_AHB2ENR,  6,	"rng",		"pll48" },
+	{ STM32F4_RCC_AHB2ENR,  7,	"otgfs",	"pll48" },
+
+	{ STM32F4_RCC_AHB3ENR,  0,	"fmc",		"ahb_div",
+		CLK_IGNORE_UNUSED },
+	{ STM32F4_RCC_AHB3ENR,  1,	"qspi",		"ahb_div",
+		CLK_IGNORE_UNUSED },
+
+	{ STM32F4_RCC_APB1ENR,  0,	"tim2",		"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  1,	"tim3",		"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  2,	"tim4",		"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  3,	"tim5",		"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  4,	"tim6",		"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  5,	"tim7",		"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  6,	"tim12",	"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  7,	"tim13",	"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  8,	"tim14",	"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR, 11,	"wwdg",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 14,	"spi2",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 15,	"spi3",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 17,	"uart2",	"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 18,	"uart3",	"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 19,	"uart4",	"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 20,	"uart5",	"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 21,	"i2c1",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 22,	"i2c2",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 23,	"i2c3",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 25,	"can1",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 26,	"can2",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 28,	"pwr",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 29,	"dac",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 30,	"uart7",	"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 31,	"uart8",	"apb1_div" },
+
+	{ STM32F4_RCC_APB2ENR,  0,	"tim1",		"apb2_mul" },
+	{ STM32F4_RCC_APB2ENR,  1,	"tim8",		"apb2_mul" },
+	{ STM32F4_RCC_APB2ENR,  4,	"usart1",	"apb2_div" },
+	{ STM32F4_RCC_APB2ENR,  5,	"usart6",	"apb2_div" },
+	{ STM32F4_RCC_APB2ENR,  8,	"adc1",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR,  9,	"adc2",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 10,	"adc3",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 11,	"sdio",		"pll48" },
+	{ STM32F4_RCC_APB2ENR, 12,	"spi1",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 13,	"spi4",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 14,	"syscfg",	"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 16,	"tim9",		"apb2_mul" },
+	{ STM32F4_RCC_APB2ENR, 17,	"tim10",	"apb2_mul" },
+	{ STM32F4_RCC_APB2ENR, 18,	"tim11",	"apb2_mul" },
+	{ STM32F4_RCC_APB2ENR, 20,	"spi5",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 21,	"spi6",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 22,	"sai1",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 26,	"ltdc",		"apb2_div" },
+};
 
-enum { SYSTICK, FCLK };
+enum { SYSTICK, FCLK, CLK_LSI, CLK_LSE, CLK_HSE_RTC, CLK_RTC, END_PRIMARY_CLK };
 
 /*
  * This bitmask tells us which bit offsets (0..192) on STM32F4[23]xxx
  * have gate bits associated with them. Its combined hweight is 71.
  */
-static const u64 stm32f42xx_gate_map[] = { 0x000000f17ef417ffull,
-					   0x0000000000000001ull,
-					   0x04777f33f6fec9ffull };
+#define MAX_GATE_MAP 3
+
+static const u64 stm32f42xx_gate_map[MAX_GATE_MAP] = { 0x000000f17ef417ffull,
+						       0x0000000000000001ull,
+						       0x04777f33f6fec9ffull };
+
+static const u64 stm32f46xx_gate_map[MAX_GATE_MAP] = { 0x000000f17ef417ffull,
+						       0x0000000000000003ull,
+						       0x0c777f33f6fec9ffull };
+
+static const u64 *stm32f4_gate_map;
+
+static struct clk_hw **clks;
 
-static struct clk_hw *clks[MAX_CLKS];
 static DEFINE_SPINLOCK(stm32f4_clk_lock);
 static void __iomem *base;
 
+static struct regmap *pdrm;
+
 /*
  * "Multiplier" device for APBx clocks.
  *
@@ -256,15 +349,15 @@ static void stm32f4_rcc_register_pll(const char *hse_clk, const char *hsi_clk)
  */
 static int stm32f4_rcc_lookup_clk_idx(u8 primary, u8 secondary)
 {
-	u64 table[ARRAY_SIZE(stm32f42xx_gate_map)];
+	u64 table[MAX_GATE_MAP];
 
 	if (primary == 1) {
-		if (WARN_ON(secondary > FCLK))
+		if (WARN_ON(secondary >= END_PRIMARY_CLK))
 			return -EINVAL;
 		return secondary;
 	}
 
-	memcpy(table, stm32f42xx_gate_map, sizeof(table));
+	memcpy(table, stm32f4_gate_map, sizeof(table));
 
 	/* only bits set in table can be used as indices */
 	if (WARN_ON(secondary >= BITS_PER_BYTE * sizeof(table) ||
@@ -276,7 +369,7 @@ static int stm32f4_rcc_lookup_clk_idx(u8 primary, u8 secondary)
 	table[BIT_ULL_WORD(secondary)] &=
 	    GENMASK_ULL(secondary % BITS_PER_LONG_LONG, 0);
 
-	return FCLK + hweight64(table[0]) +
+	return END_PRIMARY_CLK - 1 + hweight64(table[0]) +
 	       (BIT_ULL_WORD(secondary) >= 1 ? hweight64(table[1]) : 0) +
 	       (BIT_ULL_WORD(secondary) >= 2 ? hweight64(table[2]) : 0);
 }
@@ -292,6 +385,212 @@ stm32f4_rcc_lookup_clk(struct of_phandle_args *clkspec, void *data)
 	return clks[i];
 }
 
+#define to_rgclk(_rgate) container_of(_rgate, struct stm32_rgate, gate)
+
+static inline void disable_power_domain_write_protection(void)
+{
+	if (pdrm)
+		regmap_update_bits(pdrm, 0x00, (1 << 8), (1 << 8));
+}
+
+static inline void enable_power_domain_write_protection(void)
+{
+	if (pdrm)
+		regmap_update_bits(pdrm, 0x00, (1 << 8), (0 << 8));
+}
+
+static inline void sofware_reset_backup_domain(void)
+{
+	unsigned long val;
+
+	val = readl(base + STM32F4_RCC_BDCR);
+	writel(val | BIT(16), base + STM32F4_RCC_BDCR);
+	writel(val & ~BIT(16), base + STM32F4_RCC_BDCR);
+}
+
+struct stm32_rgate {
+	struct	clk_gate gate;
+	u8	bit_rdy_idx;
+};
+
+#define RTC_TIMEOUT 1000000
+
+static int rgclk_enable(struct clk_hw *hw)
+{
+	struct clk_gate *gate = to_clk_gate(hw);
+	struct stm32_rgate *rgate = to_rgclk(gate);
+	u32 reg;
+	int ret;
+
+	disable_power_domain_write_protection();
+
+	clk_gate_ops.enable(hw);
+
+	ret = readl_relaxed_poll_timeout_atomic(gate->reg, reg,
+			reg & rgate->bit_rdy_idx, 1000, RTC_TIMEOUT);
+
+	enable_power_domain_write_protection();
+	return ret;
+}
+
+static void rgclk_disable(struct clk_hw *hw)
+{
+	clk_gate_ops.disable(hw);
+}
+
+static int rgclk_is_enabled(struct clk_hw *hw)
+{
+	return clk_gate_ops.is_enabled(hw);
+}
+
+static const struct clk_ops rgclk_ops = {
+	.enable = rgclk_enable,
+	.disable = rgclk_disable,
+	.is_enabled = rgclk_is_enabled,
+};
+
+static struct clk_hw *clk_register_rgate(struct device *dev, const char *name,
+		const char *parent_name, unsigned long flags,
+		void __iomem *reg, u8 bit_idx, u8 bit_rdy_idx,
+		u8 clk_gate_flags, spinlock_t *lock)
+{
+	struct stm32_rgate *rgate;
+	struct clk_init_data init = { NULL };
+	struct clk_hw *hw;
+	int ret;
+
+	rgate = kzalloc(sizeof(*rgate), GFP_KERNEL);
+	if (!rgate)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &rgclk_ops;
+	init.flags = flags;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+
+	rgate->bit_rdy_idx = bit_rdy_idx;
+
+	rgate->gate.lock = lock;
+	rgate->gate.reg = reg;
+	rgate->gate.bit_idx = bit_idx;
+	rgate->gate.hw.init = &init;
+
+	hw = &rgate->gate.hw;
+	ret = clk_hw_register(dev, hw);
+	if (ret) {
+		kfree(rgate);
+		hw = ERR_PTR(ret);
+	}
+
+	return hw;
+}
+
+static int cclk_gate_enable(struct clk_hw *hw)
+{
+	int ret;
+
+	disable_power_domain_write_protection();
+
+	ret = clk_gate_ops.enable(hw);
+
+	enable_power_domain_write_protection();
+
+	return ret;
+}
+
+static void cclk_gate_disable(struct clk_hw *hw)
+{
+	disable_power_domain_write_protection();
+
+	clk_gate_ops.disable(hw);
+
+	enable_power_domain_write_protection();
+}
+
+static int cclk_gate_is_enabled(struct clk_hw *hw)
+{
+	return clk_gate_ops.is_enabled(hw);
+}
+
+static const struct clk_ops cclk_gate_ops = {
+	.enable		= cclk_gate_enable,
+	.disable	= cclk_gate_disable,
+	.is_enabled	= cclk_gate_is_enabled,
+};
+
+static u8 cclk_mux_get_parent(struct clk_hw *hw)
+{
+	return clk_mux_ops.get_parent(hw);
+}
+
+static int cclk_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+	int ret;
+
+	disable_power_domain_write_protection();
+
+	sofware_reset_backup_domain();
+
+	ret = clk_mux_ops.set_parent(hw, index);
+
+	enable_power_domain_write_protection();
+
+	return ret;
+}
+
+static const struct clk_ops cclk_mux_ops = {
+	.get_parent = cclk_mux_get_parent,
+	.set_parent = cclk_mux_set_parent,
+};
+
+static struct clk_hw *stm32_register_cclk(struct device *dev, const char *name,
+		const char * const *parent_names, int num_parents,
+		void __iomem *reg, u8 bit_idx, u8 shift, unsigned long flags,
+		spinlock_t *lock)
+{
+	struct clk_hw *hw;
+	struct clk_gate *gate;
+	struct clk_mux *mux;
+
+	gate = kzalloc(sizeof(*gate), GFP_KERNEL);
+	if (!gate) {
+		hw = ERR_PTR(-EINVAL);
+		goto fail;
+	}
+
+	mux = kzalloc(sizeof(*mux), GFP_KERNEL);
+	if (!mux) {
+		kfree(gate);
+		hw = ERR_PTR(-EINVAL);
+		goto fail;
+	}
+
+	gate->reg = reg;
+	gate->bit_idx = bit_idx;
+	gate->flags = 0;
+	gate->lock = lock;
+
+	mux->reg = reg;
+	mux->shift = shift;
+	mux->mask = 3;
+	mux->flags = 0;
+
+	hw = clk_hw_register_composite(dev, name, parent_names, num_parents,
+			&mux->hw, &cclk_mux_ops,
+			NULL, NULL,
+			&gate->hw, &cclk_gate_ops,
+			flags);
+
+	if (IS_ERR(hw)) {
+		kfree(gate);
+		kfree(mux);
+	}
+
+fail:
+	return hw;
+}
+
 static const char *sys_parents[] __initdata =   { "hsi", NULL, "pll" };
 
 static const struct clk_div_table ahb_div_table[] = {
@@ -308,10 +607,46 @@ static const struct clk_div_table apb_div_table[] = {
 	{ 0 },
 };
 
+static const char *rtc_parents[4] = {
+	"no-clock", "lse", "lsi", "hse-rtc"
+};
+
+struct stm32f4_clk_data {
+	const struct stm32f4_gate_data *gates_data;
+	const u64 *gates_map;
+	int gates_num;
+};
+
+static const struct stm32f4_clk_data stm32f429_clk_data = {
+	.gates_data	= stm32f429_gates,
+	.gates_map	= stm32f42xx_gate_map,
+	.gates_num	= ARRAY_SIZE(stm32f429_gates),
+};
+
+static const struct stm32f4_clk_data stm32f469_clk_data = {
+	.gates_data	= stm32f469_gates,
+	.gates_map	= stm32f46xx_gate_map,
+	.gates_num	= ARRAY_SIZE(stm32f469_gates),
+};
+
+static const struct of_device_id stm32f4_of_match[] = {
+	{
+		.compatible = "st,stm32f42xx-rcc",
+		.data = &stm32f429_clk_data
+	},
+	{
+		.compatible = "st,stm32f469-rcc",
+		.data = &stm32f469_clk_data
+	},
+	{}
+};
+
 static void __init stm32f4_rcc_init(struct device_node *np)
 {
 	const char *hse_clk;
 	int n;
+	const struct of_device_id *match;
+	const struct stm32f4_clk_data *data;
 
 	base = of_iomap(np, 0);
 	if (!base) {
@@ -319,6 +654,25 @@ static void __init stm32f4_rcc_init(struct device_node *np)
 		return;
 	}
 
+	pdrm = syscon_regmap_lookup_by_phandle(np, "st,syscfg");
+	if (IS_ERR(pdrm)) {
+		pdrm = NULL;
+		pr_warn("%s: Unable to get syscfg\n", __func__);
+	}
+
+	match = of_match_node(stm32f4_of_match, np);
+	if (WARN_ON(!match))
+		return;
+
+	data = match->data;
+
+	clks = kmalloc_array(data->gates_num + END_PRIMARY_CLK,
+			sizeof(*clks), GFP_KERNEL);
+	if (!clks)
+		goto fail;
+
+	stm32f4_gate_map = data->gates_map;
+
 	hse_clk = of_clk_get_parent_name(np, 0);
 
 	clk_register_fixed_rate_with_accuracy(NULL, "hsi", NULL, 0,
@@ -351,11 +705,15 @@ static void __init stm32f4_rcc_init(struct device_node *np)
 	clks[FCLK] = clk_hw_register_fixed_factor(NULL, "fclk", "ahb_div",
 					       0, 1, 1);
 
-	for (n = 0; n < ARRAY_SIZE(stm32f4_gates); n++) {
-		const struct stm32f4_gate_data *gd = &stm32f4_gates[n];
-		unsigned int secondary =
-		    8 * (gd->offset - STM32F4_RCC_AHB1ENR) + gd->bit_idx;
-		int idx = stm32f4_rcc_lookup_clk_idx(0, secondary);
+	for (n = 0; n < data->gates_num; n++) {
+		const struct stm32f4_gate_data *gd;
+		unsigned int secondary;
+		int idx;
+
+		gd = &data->gates_data[n];
+		secondary = 8 * (gd->offset - STM32F4_RCC_AHB1ENR) +
+			gd->bit_idx;
+		idx = stm32f4_rcc_lookup_clk_idx(0, secondary);
 
 		if (idx < 0)
 			goto fail;
@@ -371,9 +729,44 @@ static void __init stm32f4_rcc_init(struct device_node *np)
 		}
 	}
 
+	clks[CLK_LSI] = clk_register_rgate(NULL, "lsi", "clk-lsi", 0,
+			base + STM32F4_RCC_CSR, 0, 2, 0, &stm32f4_clk_lock);
+
+	if (IS_ERR(clks[CLK_LSI])) {
+		pr_err("Unable to register lsi clock\n");
+		goto fail;
+	}
+
+	clks[CLK_LSE] = clk_register_rgate(NULL, "lse", "clk-lse", 0,
+			base + STM32F4_RCC_BDCR, 0, 2, 0, &stm32f4_clk_lock);
+
+	if (IS_ERR(clks[CLK_LSE])) {
+		pr_err("Unable to register lse clock\n");
+		goto fail;
+	}
+
+	clks[CLK_HSE_RTC] = clk_hw_register_divider(NULL, "hse-rtc", "clk-hse",
+			0, base + STM32F4_RCC_CFGR, 16, 5, 0,
+			&stm32f4_clk_lock);
+
+	if (IS_ERR(clks[CLK_HSE_RTC])) {
+		pr_err("Unable to register hse-rtc clock\n");
+		goto fail;
+	}
+
+	clks[CLK_RTC] = stm32_register_cclk(NULL, "rtc", rtc_parents, 4,
+			base + STM32F4_RCC_BDCR, 15, 8, 0, &stm32f4_clk_lock);
+
+	if (IS_ERR(clks[CLK_RTC])) {
+		pr_err("Unable to register rtc clock\n");
+		goto fail;
+	}
+
 	of_clk_add_hw_provider(np, stm32f4_rcc_lookup_clk, NULL);
 	return;
 fail:
+	kfree(clks);
 	iounmap(base);
 }
-CLK_OF_DECLARE(stm32f4_rcc, "st,stm32f42xx-rcc", stm32f4_rcc_init);
+CLK_OF_DECLARE(stm32f42xx_rcc, "st,stm32f42xx-rcc", stm32f4_rcc_init);
+CLK_OF_DECLARE(stm32f46xx_rcc, "st,stm32f469-rcc", stm32f4_rcc_init);
diff --git a/drivers/clk/clk-wm831x.c b/drivers/clk/clk-wm831x.c
index f4fdac55727c..0621fbfb4beb 100644
--- a/drivers/clk/clk-wm831x.c
+++ b/drivers/clk/clk-wm831x.c
@@ -243,7 +243,7 @@ static int wm831x_clkout_is_prepared(struct clk_hw *hw)
 	if (ret < 0) {
 		dev_err(wm831x->dev, "Unable to read CLOCK_CONTROL_1: %d\n",
 			ret);
-		return true;
+		return false;
 	}
 
 	return (ret & WM831X_CLKOUT_ENA) != 0;
diff --git a/drivers/clk/clkdev.c b/drivers/clk/clkdev.c
index 97ae60fa1584..bb8a77a5985f 100644
--- a/drivers/clk/clkdev.c
+++ b/drivers/clk/clkdev.c
@@ -448,12 +448,20 @@ EXPORT_SYMBOL(clk_register_clkdev);
  *
  * con_id or dev_id may be NULL as a wildcard, just as in the rest of
  * clkdev.
+ *
+ * To make things easier for mass registration, we detect error clk_hws
+ * from a previous clk_hw_register_*() call, and return the error code for
+ * those.  This is to permit this function to be called immediately
+ * after clk_hw_register_*().
  */
 int clk_hw_register_clkdev(struct clk_hw *hw, const char *con_id,
 	const char *dev_id)
 {
 	struct clk_lookup *cl;
 
+	if (IS_ERR(hw))
+		return PTR_ERR(hw);
+
 	/*
 	 * Since dev_id can be NULL, and NULL is handled specially, we must
 	 * pass it as either a NULL format string, or with "%s".
diff --git a/drivers/clk/hisilicon/Kconfig b/drivers/clk/hisilicon/Kconfig
index 3f537a04c6a6..cbed6602172b 100644
--- a/drivers/clk/hisilicon/Kconfig
+++ b/drivers/clk/hisilicon/Kconfig
@@ -1,3 +1,11 @@
+config COMMON_CLK_HI3516CV300
+	tristate "HI3516CV300 Clock Driver"
+	depends on ARCH_HISI || COMPILE_TEST
+	select RESET_HISI
+	default ARCH_HISI
+	help
+	  Build the clock driver for hi3516cv300.
+
 config COMMON_CLK_HI3519
 	tristate "Hi3519 Clock Driver"
 	depends on ARCH_HISI || COMPILE_TEST
@@ -6,6 +14,14 @@ config COMMON_CLK_HI3519
 	help
 	  Build the clock driver for hi3519.
 
+config COMMON_CLK_HI3798CV200
+	tristate "Hi3798CV200 Clock Driver"
+	depends on ARCH_HISI || COMPILE_TEST
+	select RESET_HISI
+	default ARCH_HISI
+	help
+	  Build the clock driver for hi3798cv200.
+
 config COMMON_CLK_HI6220
 	bool "Hi6220 Clock Driver"
 	depends on ARCH_HISI || COMPILE_TEST
@@ -23,5 +39,6 @@ config RESET_HISI
 config STUB_CLK_HI6220
 	bool "Hi6220 Stub Clock Driver"
 	depends on COMMON_CLK_HI6220 && MAILBOX
+	default ARCH_HISI
 	help
 	  Build the Hisilicon Hi6220 stub clock driver.
diff --git a/drivers/clk/hisilicon/Makefile b/drivers/clk/hisilicon/Makefile
index e169ec7da023..4eec5e511e4c 100644
--- a/drivers/clk/hisilicon/Makefile
+++ b/drivers/clk/hisilicon/Makefile
@@ -7,7 +7,9 @@ obj-y	+= clk.o clkgate-separated.o clkdivider-hi6220.o
 obj-$(CONFIG_ARCH_HI3xxx)	+= clk-hi3620.o
 obj-$(CONFIG_ARCH_HIP04)	+= clk-hip04.o
 obj-$(CONFIG_ARCH_HIX5HD2)	+= clk-hix5hd2.o
+obj-$(CONFIG_COMMON_CLK_HI3516CV300)	+= crg-hi3516cv300.o
 obj-$(CONFIG_COMMON_CLK_HI3519)	+= clk-hi3519.o
+obj-$(CONFIG_COMMON_CLK_HI3798CV200)	+= crg-hi3798cv200.o
 obj-$(CONFIG_COMMON_CLK_HI6220)	+= clk-hi6220.o
 obj-$(CONFIG_RESET_HISI)	+= reset.o
 obj-$(CONFIG_STUB_CLK_HI6220)	+= clk-hi6220-stub.o
diff --git a/drivers/clk/hisilicon/crg-hi3516cv300.c b/drivers/clk/hisilicon/crg-hi3516cv300.c
new file mode 100644
index 000000000000..2007123832bb
--- /dev/null
+++ b/drivers/clk/hisilicon/crg-hi3516cv300.c
@@ -0,0 +1,330 @@
+/*
+ * Hi3516CV300 Clock and Reset Generator Driver
+ *
+ * Copyright (c) 2016 HiSilicon Technologies Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <dt-bindings/clock/hi3516cv300-clock.h>
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include "clk.h"
+#include "crg.h"
+#include "reset.h"
+
+/* hi3516CV300 core CRG */
+#define HI3516CV300_INNER_CLK_OFFSET	64
+#define HI3516CV300_FIXED_3M		65
+#define HI3516CV300_FIXED_6M		66
+#define HI3516CV300_FIXED_24M		67
+#define HI3516CV300_FIXED_49P5		68
+#define HI3516CV300_FIXED_50M		69
+#define HI3516CV300_FIXED_83P3M		70
+#define HI3516CV300_FIXED_99M		71
+#define HI3516CV300_FIXED_100M		72
+#define HI3516CV300_FIXED_148P5M	73
+#define HI3516CV300_FIXED_198M		74
+#define HI3516CV300_FIXED_297M		75
+#define HI3516CV300_UART_MUX		76
+#define HI3516CV300_FMC_MUX		77
+#define HI3516CV300_MMC0_MUX		78
+#define HI3516CV300_MMC1_MUX		79
+#define HI3516CV300_MMC2_MUX		80
+#define HI3516CV300_MMC3_MUX		81
+#define HI3516CV300_PWM_MUX		82
+#define HI3516CV300_CRG_NR_CLKS		128
+
+static const struct hisi_fixed_rate_clock hi3516cv300_fixed_rate_clks[] = {
+	{ HI3516CV300_FIXED_3M, "3m", NULL, 0, 3000000, },
+	{ HI3516CV300_FIXED_6M, "6m", NULL, 0, 6000000, },
+	{ HI3516CV300_FIXED_24M, "24m", NULL, 0, 24000000, },
+	{ HI3516CV300_FIXED_49P5, "49.5m", NULL, 0, 49500000, },
+	{ HI3516CV300_FIXED_50M, "50m", NULL, 0, 50000000, },
+	{ HI3516CV300_FIXED_83P3M, "83.3m", NULL, 0, 83300000, },
+	{ HI3516CV300_FIXED_99M, "99m", NULL, 0, 99000000, },
+	{ HI3516CV300_FIXED_100M, "100m", NULL, 0, 100000000, },
+	{ HI3516CV300_FIXED_148P5M, "148.5m", NULL, 0, 148500000, },
+	{ HI3516CV300_FIXED_198M, "198m", NULL, 0, 198000000, },
+	{ HI3516CV300_FIXED_297M, "297m", NULL, 0, 297000000, },
+	{ HI3516CV300_APB_CLK, "apb", NULL, 0, 50000000, },
+};
+
+static const char *const uart_mux_p[] = {"24m", "6m"};
+static const char *const fmc_mux_p[] = {
+	"24m", "83.3m", "148.5m", "198m", "297m"
+};
+static const char *const mmc_mux_p[] = {"49.5m"};
+static const char *const mmc2_mux_p[] = {"99m", "49.5m"};
+static const char *const pwm_mux_p[] = {"3m", "50m", "24m", "24m"};
+
+static u32 uart_mux_table[] = {0, 1};
+static u32 fmc_mux_table[] = {0, 1, 2, 3, 4};
+static u32 mmc_mux_table[] = {0};
+static u32 mmc2_mux_table[] = {0, 2};
+static u32 pwm_mux_table[] = {0, 1, 2, 3};
+
+static const struct hisi_mux_clock hi3516cv300_mux_clks[] = {
+	{ HI3516CV300_UART_MUX, "uart_mux", uart_mux_p, ARRAY_SIZE(uart_mux_p),
+		CLK_SET_RATE_PARENT, 0xe4, 19, 1, 0, uart_mux_table, },
+	{ HI3516CV300_FMC_MUX, "fmc_mux", fmc_mux_p, ARRAY_SIZE(fmc_mux_p),
+		CLK_SET_RATE_PARENT, 0xc0, 2, 3, 0, fmc_mux_table, },
+	{ HI3516CV300_MMC0_MUX, "mmc0_mux", mmc_mux_p, ARRAY_SIZE(mmc_mux_p),
+		CLK_SET_RATE_PARENT, 0xc4, 4, 2, 0, mmc_mux_table, },
+	{ HI3516CV300_MMC1_MUX, "mmc1_mux", mmc_mux_p, ARRAY_SIZE(mmc_mux_p),
+		CLK_SET_RATE_PARENT, 0xc4, 12, 2, 0, mmc_mux_table, },
+	{ HI3516CV300_MMC2_MUX, "mmc2_mux", mmc2_mux_p, ARRAY_SIZE(mmc2_mux_p),
+		CLK_SET_RATE_PARENT, 0xc4, 20, 2, 0, mmc2_mux_table, },
+	{ HI3516CV300_MMC3_MUX, "mmc3_mux", mmc_mux_p, ARRAY_SIZE(mmc_mux_p),
+		CLK_SET_RATE_PARENT, 0xc8, 4, 2, 0, mmc_mux_table, },
+	{ HI3516CV300_PWM_MUX, "pwm_mux", pwm_mux_p, ARRAY_SIZE(pwm_mux_p),
+		CLK_SET_RATE_PARENT, 0x38, 2, 2, 0, pwm_mux_table, },
+};
+
+static const struct hisi_gate_clock hi3516cv300_gate_clks[] = {
+
+	{ HI3516CV300_UART0_CLK, "clk_uart0", "uart_mux", CLK_SET_RATE_PARENT,
+		0xe4, 15, 0, },
+	{ HI3516CV300_UART1_CLK, "clk_uart1", "uart_mux", CLK_SET_RATE_PARENT,
+		0xe4, 16, 0, },
+	{ HI3516CV300_UART2_CLK, "clk_uart2", "uart_mux", CLK_SET_RATE_PARENT,
+		0xe4, 17, 0, },
+
+	{ HI3516CV300_SPI0_CLK, "clk_spi0", "100m", CLK_SET_RATE_PARENT,
+		0xe4, 13, 0, },
+	{ HI3516CV300_SPI1_CLK, "clk_spi1", "100m", CLK_SET_RATE_PARENT,
+		0xe4, 14, 0, },
+
+	{ HI3516CV300_FMC_CLK, "clk_fmc", "fmc_mux", CLK_SET_RATE_PARENT,
+		0xc0, 1, 0, },
+	{ HI3516CV300_MMC0_CLK, "clk_mmc0", "mmc0_mux", CLK_SET_RATE_PARENT,
+		0xc4, 1, 0, },
+	{ HI3516CV300_MMC1_CLK, "clk_mmc1", "mmc1_mux", CLK_SET_RATE_PARENT,
+		0xc4, 9, 0, },
+	{ HI3516CV300_MMC2_CLK, "clk_mmc2", "mmc2_mux", CLK_SET_RATE_PARENT,
+		0xc4, 17, 0, },
+	{ HI3516CV300_MMC3_CLK, "clk_mmc3", "mmc3_mux", CLK_SET_RATE_PARENT,
+		0xc8, 1, 0, },
+
+	{ HI3516CV300_ETH_CLK, "clk_eth", NULL, 0, 0xec, 1, 0, },
+
+	{ HI3516CV300_DMAC_CLK, "clk_dmac", NULL, 0, 0xd8, 5, 0, },
+	{ HI3516CV300_PWM_CLK, "clk_pwm", "pwm_mux", CLK_SET_RATE_PARENT,
+		0x38, 1, 0, },
+
+	{ HI3516CV300_USB2_BUS_CLK, "clk_usb2_bus", NULL, 0, 0xb8, 0, 0, },
+	{ HI3516CV300_USB2_OHCI48M_CLK, "clk_usb2_ohci48m", NULL, 0,
+		0xb8, 1, 0, },
+	{ HI3516CV300_USB2_OHCI12M_CLK, "clk_usb2_ohci12m", NULL, 0,
+		0xb8, 2, 0, },
+	{ HI3516CV300_USB2_OTG_UTMI_CLK, "clk_usb2_otg_utmi", NULL, 0,
+		0xb8, 3, 0, },
+	{ HI3516CV300_USB2_HST_PHY_CLK, "clk_usb2_hst_phy", NULL, 0,
+		0xb8, 4, 0, },
+	{ HI3516CV300_USB2_UTMI0_CLK, "clk_usb2_utmi0", NULL, 0, 0xb8, 5, 0, },
+	{ HI3516CV300_USB2_PHY_CLK, "clk_usb2_phy", NULL, 0, 0xb8, 7, 0, },
+};
+
+static struct hisi_clock_data *hi3516cv300_clk_register(
+		struct platform_device *pdev)
+{
+	struct hisi_clock_data *clk_data;
+	int ret;
+
+	clk_data = hisi_clk_alloc(pdev, HI3516CV300_CRG_NR_CLKS);
+	if (!clk_data)
+		return ERR_PTR(-ENOMEM);
+
+	ret = hisi_clk_register_fixed_rate(hi3516cv300_fixed_rate_clks,
+			ARRAY_SIZE(hi3516cv300_fixed_rate_clks), clk_data);
+	if (ret)
+		return ERR_PTR(ret);
+
+	ret = hisi_clk_register_mux(hi3516cv300_mux_clks,
+			ARRAY_SIZE(hi3516cv300_mux_clks), clk_data);
+	if (ret)
+		goto unregister_fixed_rate;
+
+	ret = hisi_clk_register_gate(hi3516cv300_gate_clks,
+			ARRAY_SIZE(hi3516cv300_gate_clks), clk_data);
+	if (ret)
+		goto unregister_mux;
+
+	ret = of_clk_add_provider(pdev->dev.of_node,
+			of_clk_src_onecell_get, &clk_data->clk_data);
+	if (ret)
+		goto unregister_gate;
+
+	return clk_data;
+
+unregister_gate:
+	hisi_clk_unregister_gate(hi3516cv300_gate_clks,
+				ARRAY_SIZE(hi3516cv300_gate_clks), clk_data);
+unregister_mux:
+	hisi_clk_unregister_mux(hi3516cv300_mux_clks,
+			ARRAY_SIZE(hi3516cv300_mux_clks), clk_data);
+unregister_fixed_rate:
+	hisi_clk_unregister_fixed_rate(hi3516cv300_fixed_rate_clks,
+			ARRAY_SIZE(hi3516cv300_fixed_rate_clks), clk_data);
+	return ERR_PTR(ret);
+}
+
+static void hi3516cv300_clk_unregister(struct platform_device *pdev)
+{
+	struct hisi_crg_dev *crg = platform_get_drvdata(pdev);
+
+	of_clk_del_provider(pdev->dev.of_node);
+
+	hisi_clk_unregister_gate(hi3516cv300_gate_clks,
+			ARRAY_SIZE(hi3516cv300_gate_clks), crg->clk_data);
+	hisi_clk_unregister_mux(hi3516cv300_mux_clks,
+			ARRAY_SIZE(hi3516cv300_mux_clks), crg->clk_data);
+	hisi_clk_unregister_fixed_rate(hi3516cv300_fixed_rate_clks,
+			ARRAY_SIZE(hi3516cv300_fixed_rate_clks), crg->clk_data);
+}
+
+static const struct hisi_crg_funcs hi3516cv300_crg_funcs = {
+	.register_clks = hi3516cv300_clk_register,
+	.unregister_clks = hi3516cv300_clk_unregister,
+};
+
+/* hi3516CV300 sysctrl CRG */
+#define HI3516CV300_SYSCTRL_NR_CLKS 16
+
+static const char *wdt_mux_p[] __initconst = { "3m", "apb" };
+static u32 wdt_mux_table[] = {0, 1};
+
+static const struct hisi_mux_clock hi3516cv300_sysctrl_mux_clks[] = {
+	{ HI3516CV300_WDT_CLK, "wdt", wdt_mux_p, ARRAY_SIZE(wdt_mux_p),
+		CLK_SET_RATE_PARENT, 0x0, 23, 1, 0, wdt_mux_table, },
+};
+
+static struct hisi_clock_data *hi3516cv300_sysctrl_clk_register(
+		struct platform_device *pdev)
+{
+	struct hisi_clock_data *clk_data;
+	int ret;
+
+	clk_data = hisi_clk_alloc(pdev, HI3516CV300_SYSCTRL_NR_CLKS);
+	if (!clk_data)
+		return ERR_PTR(-ENOMEM);
+
+	ret = hisi_clk_register_mux(hi3516cv300_sysctrl_mux_clks,
+			ARRAY_SIZE(hi3516cv300_sysctrl_mux_clks), clk_data);
+	if (ret)
+		return ERR_PTR(ret);
+
+
+	ret = of_clk_add_provider(pdev->dev.of_node,
+			of_clk_src_onecell_get, &clk_data->clk_data);
+	if (ret)
+		goto unregister_mux;
+
+	return clk_data;
+
+unregister_mux:
+	hisi_clk_unregister_mux(hi3516cv300_sysctrl_mux_clks,
+			ARRAY_SIZE(hi3516cv300_sysctrl_mux_clks), clk_data);
+	return ERR_PTR(ret);
+}
+
+static void hi3516cv300_sysctrl_clk_unregister(struct platform_device *pdev)
+{
+	struct hisi_crg_dev *crg = platform_get_drvdata(pdev);
+
+	of_clk_del_provider(pdev->dev.of_node);
+
+	hisi_clk_unregister_mux(hi3516cv300_sysctrl_mux_clks,
+			ARRAY_SIZE(hi3516cv300_sysctrl_mux_clks),
+			crg->clk_data);
+}
+
+static const struct hisi_crg_funcs hi3516cv300_sysctrl_funcs = {
+	.register_clks = hi3516cv300_sysctrl_clk_register,
+	.unregister_clks = hi3516cv300_sysctrl_clk_unregister,
+};
+
+static const struct of_device_id hi3516cv300_crg_match_table[] = {
+	{
+		.compatible = "hisilicon,hi3516cv300-crg",
+		.data = &hi3516cv300_crg_funcs
+	},
+	{
+		.compatible = "hisilicon,hi3516cv300-sysctrl",
+		.data = &hi3516cv300_sysctrl_funcs
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, hi3516cv300_crg_match_table);
+
+static int hi3516cv300_crg_probe(struct platform_device *pdev)
+{
+	struct hisi_crg_dev *crg;
+
+	crg = devm_kmalloc(&pdev->dev, sizeof(*crg), GFP_KERNEL);
+	if (!crg)
+		return -ENOMEM;
+
+	crg->funcs = of_device_get_match_data(&pdev->dev);
+	if (!crg->funcs)
+		return -ENOENT;
+
+	crg->rstc = hisi_reset_init(pdev);
+	if (!crg->rstc)
+		return -ENOMEM;
+
+	crg->clk_data = crg->funcs->register_clks(pdev);
+	if (IS_ERR(crg->clk_data)) {
+		hisi_reset_exit(crg->rstc);
+		return PTR_ERR(crg->clk_data);
+	}
+
+	platform_set_drvdata(pdev, crg);
+	return 0;
+}
+
+static int hi3516cv300_crg_remove(struct platform_device *pdev)
+{
+	struct hisi_crg_dev *crg = platform_get_drvdata(pdev);
+
+	hisi_reset_exit(crg->rstc);
+	crg->funcs->unregister_clks(pdev);
+	return 0;
+}
+
+static struct platform_driver hi3516cv300_crg_driver = {
+	.probe          = hi3516cv300_crg_probe,
+	.remove		= hi3516cv300_crg_remove,
+	.driver         = {
+		.name   = "hi3516cv300-crg",
+		.of_match_table = hi3516cv300_crg_match_table,
+	},
+};
+
+static int __init hi3516cv300_crg_init(void)
+{
+	return platform_driver_register(&hi3516cv300_crg_driver);
+}
+core_initcall(hi3516cv300_crg_init);
+
+static void __exit hi3516cv300_crg_exit(void)
+{
+	platform_driver_unregister(&hi3516cv300_crg_driver);
+}
+module_exit(hi3516cv300_crg_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("HiSilicon Hi3516CV300 CRG Driver");
diff --git a/drivers/clk/hisilicon/crg-hi3798cv200.c b/drivers/clk/hisilicon/crg-hi3798cv200.c
new file mode 100644
index 000000000000..fc8b5bc2d50d
--- /dev/null
+++ b/drivers/clk/hisilicon/crg-hi3798cv200.c
@@ -0,0 +1,337 @@
+/*
+ * Hi3798CV200 Clock and Reset Generator Driver
+ *
+ * Copyright (c) 2016 HiSilicon Technologies Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <dt-bindings/clock/histb-clock.h>
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include "clk.h"
+#include "crg.h"
+#include "reset.h"
+
+/* hi3798CV200 core CRG */
+#define HI3798CV200_INNER_CLK_OFFSET	64
+#define HI3798CV200_FIXED_24M	65
+#define HI3798CV200_FIXED_25M	66
+#define HI3798CV200_FIXED_50M	67
+#define HI3798CV200_FIXED_75M	68
+#define HI3798CV200_FIXED_100M	69
+#define HI3798CV200_FIXED_150M	70
+#define HI3798CV200_FIXED_200M	71
+#define HI3798CV200_FIXED_250M	72
+#define HI3798CV200_FIXED_300M	73
+#define HI3798CV200_FIXED_400M	74
+#define HI3798CV200_MMC_MUX	75
+#define HI3798CV200_ETH_PUB_CLK	76
+#define HI3798CV200_ETH_BUS_CLK	77
+#define HI3798CV200_ETH_BUS0_CLK	78
+#define HI3798CV200_ETH_BUS1_CLK	79
+#define HI3798CV200_COMBPHY1_MUX	80
+
+#define HI3798CV200_CRG_NR_CLKS		128
+
+static const struct hisi_fixed_rate_clock hi3798cv200_fixed_rate_clks[] = {
+	{ HISTB_OSC_CLK, "clk_osc", NULL, 0, 24000000, },
+	{ HISTB_APB_CLK, "clk_apb", NULL, 0, 100000000, },
+	{ HISTB_AHB_CLK, "clk_ahb", NULL, 0, 200000000, },
+	{ HI3798CV200_FIXED_24M, "24m", NULL, 0, 24000000, },
+	{ HI3798CV200_FIXED_25M, "25m", NULL, 0, 25000000, },
+	{ HI3798CV200_FIXED_50M, "50m", NULL, 0, 50000000, },
+	{ HI3798CV200_FIXED_75M, "75m", NULL, 0, 75000000, },
+	{ HI3798CV200_FIXED_100M, "100m", NULL, 0, 100000000, },
+	{ HI3798CV200_FIXED_150M, "150m", NULL, 0, 150000000, },
+	{ HI3798CV200_FIXED_200M, "200m", NULL, 0, 200000000, },
+	{ HI3798CV200_FIXED_250M, "250m", NULL, 0, 250000000, },
+};
+
+static const char *const mmc_mux_p[] = {
+		"100m", "50m", "25m", "200m", "150m" };
+static u32 mmc_mux_table[] = {0, 1, 2, 3, 6};
+
+static const char *const comphy1_mux_p[] = {
+		"100m", "25m"};
+static u32 comphy1_mux_table[] = {2, 3};
+
+static struct hisi_mux_clock hi3798cv200_mux_clks[] = {
+	{ HI3798CV200_MMC_MUX, "mmc_mux", mmc_mux_p, ARRAY_SIZE(mmc_mux_p),
+		CLK_SET_RATE_PARENT, 0xa0, 8, 3, 0, mmc_mux_table, },
+	{ HI3798CV200_COMBPHY1_MUX, "combphy1_mux",
+		comphy1_mux_p, ARRAY_SIZE(comphy1_mux_p),
+		CLK_SET_RATE_PARENT, 0x188, 10, 2, 0, comphy1_mux_table, },
+};
+
+static const struct hisi_gate_clock hi3798cv200_gate_clks[] = {
+	/* UART */
+	{ HISTB_UART2_CLK, "clk_uart2", "75m",
+		CLK_SET_RATE_PARENT, 0x68, 4, 0, },
+	/* I2C */
+	{ HISTB_I2C0_CLK, "clk_i2c0", "clk_apb",
+		CLK_SET_RATE_PARENT, 0x6C, 4, 0, },
+	{ HISTB_I2C1_CLK, "clk_i2c1", "clk_apb",
+		CLK_SET_RATE_PARENT, 0x6C, 8, 0, },
+	{ HISTB_I2C2_CLK, "clk_i2c2", "clk_apb",
+		CLK_SET_RATE_PARENT, 0x6C, 12, 0, },
+	{ HISTB_I2C3_CLK, "clk_i2c3", "clk_apb",
+		CLK_SET_RATE_PARENT, 0x6C, 16, 0, },
+	{ HISTB_I2C4_CLK, "clk_i2c4", "clk_apb",
+		CLK_SET_RATE_PARENT, 0x6C, 20, 0, },
+	/* SPI */
+	{ HISTB_SPI0_CLK, "clk_spi0", "clk_apb",
+		CLK_SET_RATE_PARENT, 0x70, 0, 0, },
+	/* SDIO */
+	{ HISTB_SDIO0_BIU_CLK, "clk_sdio0_biu", "200m",
+			CLK_SET_RATE_PARENT, 0x9c, 0, 0, },
+	{ HISTB_SDIO0_CIU_CLK, "clk_sdio0_ciu", "mmc_mux",
+		CLK_SET_RATE_PARENT, 0x9c, 1, 0, },
+	/* EMMC */
+	{ HISTB_MMC_BIU_CLK, "clk_mmc_biu", "200m",
+		CLK_SET_RATE_PARENT, 0xa0, 0, 0, },
+	{ HISTB_MMC_CIU_CLK, "clk_mmc_ciu", "mmc_mux",
+		CLK_SET_RATE_PARENT, 0xa0, 1, 0, },
+	/* PCIE*/
+	{ HISTB_PCIE_BUS_CLK, "clk_pcie_bus", "200m",
+		CLK_SET_RATE_PARENT, 0x18c, 0, 0, },
+	{ HISTB_PCIE_SYS_CLK, "clk_pcie_sys", "100m",
+		CLK_SET_RATE_PARENT, 0x18c, 1, 0, },
+	{ HISTB_PCIE_PIPE_CLK, "clk_pcie_pipe", "250m",
+		CLK_SET_RATE_PARENT, 0x18c, 2, 0, },
+	{ HISTB_PCIE_AUX_CLK, "clk_pcie_aux", "24m",
+		CLK_SET_RATE_PARENT, 0x18c, 3, 0, },
+	/* Ethernet */
+	{ HI3798CV200_ETH_PUB_CLK, "clk_pub", NULL,
+		CLK_SET_RATE_PARENT, 0xcc, 5, 0, },
+	{ HI3798CV200_ETH_BUS_CLK, "clk_bus", "clk_pub",
+		CLK_SET_RATE_PARENT, 0xcc, 0, 0, },
+	{ HI3798CV200_ETH_BUS0_CLK, "clk_bus_m0", "clk_bus",
+		CLK_SET_RATE_PARENT, 0xcc, 1, 0, },
+	{ HI3798CV200_ETH_BUS1_CLK, "clk_bus_m1", "clk_bus",
+		CLK_SET_RATE_PARENT, 0xcc, 2, 0, },
+	{ HISTB_ETH0_MAC_CLK, "clk_mac0", "clk_bus_m0",
+		CLK_SET_RATE_PARENT, 0xcc, 3, 0, },
+	{ HISTB_ETH0_MACIF_CLK, "clk_macif0", "clk_bus_m0",
+		CLK_SET_RATE_PARENT, 0xcc, 24, 0, },
+	{ HISTB_ETH1_MAC_CLK, "clk_mac1", "clk_bus_m1",
+		CLK_SET_RATE_PARENT, 0xcc, 4, 0, },
+	{ HISTB_ETH1_MACIF_CLK, "clk_macif1", "clk_bus_m1",
+		CLK_SET_RATE_PARENT, 0xcc, 25, 0, },
+	/* COMBPHY1 */
+	{ HISTB_COMBPHY1_CLK, "clk_combphy1", "combphy1_mux",
+		CLK_SET_RATE_PARENT, 0x188, 8, 0, },
+};
+
+static struct hisi_clock_data *hi3798cv200_clk_register(
+				struct platform_device *pdev)
+{
+	struct hisi_clock_data *clk_data;
+	int ret;
+
+	clk_data = hisi_clk_alloc(pdev, HI3798CV200_CRG_NR_CLKS);
+	if (!clk_data)
+		return ERR_PTR(-ENOMEM);
+
+	ret = hisi_clk_register_fixed_rate(hi3798cv200_fixed_rate_clks,
+				     ARRAY_SIZE(hi3798cv200_fixed_rate_clks),
+				     clk_data);
+	if (ret)
+		return ERR_PTR(ret);
+
+	ret = hisi_clk_register_mux(hi3798cv200_mux_clks,
+				ARRAY_SIZE(hi3798cv200_mux_clks),
+				clk_data);
+	if (ret)
+		goto unregister_fixed_rate;
+
+	ret = hisi_clk_register_gate(hi3798cv200_gate_clks,
+				ARRAY_SIZE(hi3798cv200_gate_clks),
+				clk_data);
+	if (ret)
+		goto unregister_mux;
+
+	ret = of_clk_add_provider(pdev->dev.of_node,
+			of_clk_src_onecell_get, &clk_data->clk_data);
+	if (ret)
+		goto unregister_gate;
+
+	return clk_data;
+
+unregister_fixed_rate:
+	hisi_clk_unregister_fixed_rate(hi3798cv200_fixed_rate_clks,
+				ARRAY_SIZE(hi3798cv200_fixed_rate_clks),
+				clk_data);
+
+unregister_mux:
+	hisi_clk_unregister_mux(hi3798cv200_mux_clks,
+				ARRAY_SIZE(hi3798cv200_mux_clks),
+				clk_data);
+unregister_gate:
+	hisi_clk_unregister_gate(hi3798cv200_gate_clks,
+				ARRAY_SIZE(hi3798cv200_gate_clks),
+				clk_data);
+	return ERR_PTR(ret);
+}
+
+static void hi3798cv200_clk_unregister(struct platform_device *pdev)
+{
+	struct hisi_crg_dev *crg = platform_get_drvdata(pdev);
+
+	of_clk_del_provider(pdev->dev.of_node);
+
+	hisi_clk_unregister_gate(hi3798cv200_gate_clks,
+				ARRAY_SIZE(hi3798cv200_gate_clks),
+				crg->clk_data);
+	hisi_clk_unregister_mux(hi3798cv200_mux_clks,
+				ARRAY_SIZE(hi3798cv200_mux_clks),
+				crg->clk_data);
+	hisi_clk_unregister_fixed_rate(hi3798cv200_fixed_rate_clks,
+				ARRAY_SIZE(hi3798cv200_fixed_rate_clks),
+				crg->clk_data);
+}
+
+static const struct hisi_crg_funcs hi3798cv200_crg_funcs = {
+	.register_clks = hi3798cv200_clk_register,
+	.unregister_clks = hi3798cv200_clk_unregister,
+};
+
+/* hi3798CV200 sysctrl CRG */
+
+#define HI3798CV200_SYSCTRL_NR_CLKS 16
+
+static const struct hisi_gate_clock hi3798cv200_sysctrl_gate_clks[] = {
+	{ HISTB_IR_CLK, "clk_ir", "100m",
+		CLK_SET_RATE_PARENT, 0x48, 4, 0, },
+	{ HISTB_TIMER01_CLK, "clk_timer01", "24m",
+		CLK_SET_RATE_PARENT, 0x48, 6, 0, },
+	{ HISTB_UART0_CLK, "clk_uart0", "75m",
+		CLK_SET_RATE_PARENT, 0x48, 10, 0, },
+};
+
+static struct hisi_clock_data *hi3798cv200_sysctrl_clk_register(
+					struct platform_device *pdev)
+{
+	struct hisi_clock_data *clk_data;
+	int ret;
+
+	clk_data = hisi_clk_alloc(pdev, HI3798CV200_SYSCTRL_NR_CLKS);
+	if (!clk_data)
+		return ERR_PTR(-ENOMEM);
+
+	ret = hisi_clk_register_gate(hi3798cv200_sysctrl_gate_clks,
+				ARRAY_SIZE(hi3798cv200_sysctrl_gate_clks),
+				clk_data);
+	if (ret)
+		return ERR_PTR(ret);
+
+	ret = of_clk_add_provider(pdev->dev.of_node,
+			of_clk_src_onecell_get, &clk_data->clk_data);
+	if (ret)
+		goto unregister_gate;
+
+	return clk_data;
+
+unregister_gate:
+	hisi_clk_unregister_gate(hi3798cv200_sysctrl_gate_clks,
+				ARRAY_SIZE(hi3798cv200_sysctrl_gate_clks),
+				clk_data);
+	return ERR_PTR(ret);
+}
+
+static void hi3798cv200_sysctrl_clk_unregister(struct platform_device *pdev)
+{
+	struct hisi_crg_dev *crg = platform_get_drvdata(pdev);
+
+	of_clk_del_provider(pdev->dev.of_node);
+
+	hisi_clk_unregister_gate(hi3798cv200_sysctrl_gate_clks,
+				ARRAY_SIZE(hi3798cv200_sysctrl_gate_clks),
+				crg->clk_data);
+}
+
+static const struct hisi_crg_funcs hi3798cv200_sysctrl_funcs = {
+	.register_clks = hi3798cv200_sysctrl_clk_register,
+	.unregister_clks = hi3798cv200_sysctrl_clk_unregister,
+};
+
+static const struct of_device_id hi3798cv200_crg_match_table[] = {
+	{ .compatible = "hisilicon,hi3798cv200-crg",
+		.data = &hi3798cv200_crg_funcs },
+	{ .compatible = "hisilicon,hi3798cv200-sysctrl",
+		.data = &hi3798cv200_sysctrl_funcs },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, hi3798cv200_crg_match_table);
+
+static int hi3798cv200_crg_probe(struct platform_device *pdev)
+{
+	struct hisi_crg_dev *crg;
+
+	crg = devm_kmalloc(&pdev->dev, sizeof(*crg), GFP_KERNEL);
+	if (!crg)
+		return -ENOMEM;
+
+	crg->funcs = of_device_get_match_data(&pdev->dev);
+	if (!crg->funcs)
+		return -ENOENT;
+
+	crg->rstc = hisi_reset_init(pdev);
+	if (!crg->rstc)
+		return -ENOMEM;
+
+	crg->clk_data = crg->funcs->register_clks(pdev);
+	if (IS_ERR(crg->clk_data)) {
+		hisi_reset_exit(crg->rstc);
+		return PTR_ERR(crg->clk_data);
+	}
+
+	platform_set_drvdata(pdev, crg);
+	return 0;
+}
+
+static int hi3798cv200_crg_remove(struct platform_device *pdev)
+{
+	struct hisi_crg_dev *crg = platform_get_drvdata(pdev);
+
+	hisi_reset_exit(crg->rstc);
+	crg->funcs->unregister_clks(pdev);
+	return 0;
+}
+
+static struct platform_driver hi3798cv200_crg_driver = {
+	.probe          = hi3798cv200_crg_probe,
+	.remove		= hi3798cv200_crg_remove,
+	.driver         = {
+		.name   = "hi3798cv200-crg",
+		.of_match_table = hi3798cv200_crg_match_table,
+	},
+};
+
+static int __init hi3798cv200_crg_init(void)
+{
+	return platform_driver_register(&hi3798cv200_crg_driver);
+}
+core_initcall(hi3798cv200_crg_init);
+
+static void __exit hi3798cv200_crg_exit(void)
+{
+	platform_driver_unregister(&hi3798cv200_crg_driver);
+}
+module_exit(hi3798cv200_crg_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("HiSilicon Hi3798CV200 CRG Driver");
diff --git a/drivers/clk/hisilicon/crg.h b/drivers/clk/hisilicon/crg.h
new file mode 100644
index 000000000000..e0739717de9a
--- /dev/null
+++ b/drivers/clk/hisilicon/crg.h
@@ -0,0 +1,34 @@
+/*
+ * HiSilicon Clock and Reset Driver Header
+ *
+ * Copyright (c) 2016 HiSilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __HISI_CRG_H
+#define __HISI_CRG_H
+
+struct hisi_clock_data;
+struct hisi_reset_controller;
+
+struct hisi_crg_funcs {
+	struct hisi_clock_data*	(*register_clks)(struct platform_device *pdev);
+	void (*unregister_clks)(struct platform_device *pdev);
+};
+
+struct hisi_crg_dev {
+	struct hisi_clock_data *clk_data;
+	struct hisi_reset_controller *rstc;
+	const struct hisi_crg_funcs *funcs;
+};
+
+#endif	/* __HISI_CRG_H */
diff --git a/drivers/clk/imx/clk-imx31.c b/drivers/clk/imx/clk-imx31.c
index 6a964144a5b5..cbce308aad04 100644
--- a/drivers/clk/imx/clk-imx31.c
+++ b/drivers/clk/imx/clk-imx31.c
@@ -21,6 +21,7 @@
 #include <linux/io.h>
 #include <linux/err.h>
 #include <linux/of.h>
+#include <linux/of_address.h>
 #include <soc/imx/revision.h>
 #include <soc/imx/timer.h>
 #include <asm/irq.h>
@@ -72,14 +73,8 @@ static struct clk ** const uart_clks[] __initconst = {
 	NULL
 };
 
-static void __init _mx31_clocks_init(unsigned long fref)
+static void __init _mx31_clocks_init(void __iomem *base, unsigned long fref)
 {
-	void __iomem *base;
-	struct device_node *np;
-
-	base = ioremap(MX31_CCM_BASE_ADDR, SZ_4K);
-	BUG_ON(!base);
-
 	clk[dummy] = imx_clk_fixed("dummy", 0);
 	clk[ckih] = imx_clk_fixed("ckih", fref);
 	clk[ckil] = imx_clk_fixed("ckil", 32768);
@@ -147,21 +142,17 @@ static void __init _mx31_clocks_init(unsigned long fref)
 	clk_prepare_enable(clk[iim_gate]);
 	mx31_revision();
 	clk_disable_unprepare(clk[iim_gate]);
-
-	np = of_find_compatible_node(NULL, NULL, "fsl,imx31-ccm");
-
-	if (np) {
-		clk_data.clks = clk;
-		clk_data.clk_num = ARRAY_SIZE(clk);
-		of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
-	}
 }
 
-int __init mx31_clocks_init(void)
+int __init mx31_clocks_init(unsigned long fref)
 {
-	u32 fref = 26000000; /* default */
+	void __iomem *base;
+
+	base = ioremap(MX31_CCM_BASE_ADDR, SZ_4K);
+	if (!base)
+		panic("%s: failed to map registers\n", __func__);
 
-	_mx31_clocks_init(fref);
+	_mx31_clocks_init(base, fref);
 
 	clk_register_clkdev(clk[gpt_gate], "per", "imx-gpt.0");
 	clk_register_clkdev(clk[ipg], "ipg", "imx-gpt.0");
@@ -224,22 +215,31 @@ int __init mx31_clocks_init(void)
 	return 0;
 }
 
-int __init mx31_clocks_init_dt(void)
+static void __init mx31_clocks_init_dt(struct device_node *np)
 {
-	struct device_node *np;
+	struct device_node *osc_np;
 	u32 fref = 26000000; /* default */
+	void __iomem *ccm;
 
-	for_each_compatible_node(np, NULL, "fixed-clock") {
-		if (!of_device_is_compatible(np, "fsl,imx-osc26m"))
+	for_each_compatible_node(osc_np, NULL, "fixed-clock") {
+		if (!of_device_is_compatible(osc_np, "fsl,imx-osc26m"))
 			continue;
 
-		if (!of_property_read_u32(np, "clock-frequency", &fref)) {
-			of_node_put(np);
+		if (!of_property_read_u32(osc_np, "clock-frequency", &fref)) {
+			of_node_put(osc_np);
 			break;
 		}
 	}
 
-	_mx31_clocks_init(fref);
+	ccm = of_iomap(np, 0);
+	if (!ccm)
+		panic("%s: failed to map registers\n", __func__);
 
-	return 0;
+	_mx31_clocks_init(ccm, fref);
+
+	clk_data.clks = clk;
+	clk_data.clk_num = ARRAY_SIZE(clk);
+	of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
 }
+
+CLK_OF_DECLARE(imx31_ccm, "fsl,imx31-ccm", mx31_clocks_init_dt);
diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c
index ce8ea10407e4..42ffc1c92bab 100644
--- a/drivers/clk/imx/clk-imx6q.c
+++ b/drivers/clk/imx/clk-imx6q.c
@@ -156,10 +156,267 @@ static struct clk ** const uart_clks[] __initconst = {
 	NULL
 };
 
+static int ldb_di_sel_by_clock_id(int clock_id)
+{
+	switch (clock_id) {
+	case IMX6QDL_CLK_PLL5_VIDEO_DIV:
+		if (clk_on_imx6q() &&
+		    imx_get_soc_revision() == IMX_CHIP_REVISION_1_0)
+			return -ENOENT;
+		return 0;
+	case IMX6QDL_CLK_PLL2_PFD0_352M:
+		return 1;
+	case IMX6QDL_CLK_PLL2_PFD2_396M:
+		return 2;
+	case IMX6QDL_CLK_MMDC_CH1_AXI:
+		return 3;
+	case IMX6QDL_CLK_PLL3_USB_OTG:
+		return 4;
+	default:
+		return -ENOENT;
+	}
+}
+
+static void of_assigned_ldb_sels(struct device_node *node,
+				 unsigned int *ldb_di0_sel,
+				 unsigned int *ldb_di1_sel)
+{
+	struct of_phandle_args clkspec;
+	int index, rc, num_parents;
+	int parent, child, sel;
+
+	num_parents = of_count_phandle_with_args(node, "assigned-clock-parents",
+						 "#clock-cells");
+	for (index = 0; index < num_parents; index++) {
+		rc = of_parse_phandle_with_args(node, "assigned-clock-parents",
+					"#clock-cells", index, &clkspec);
+		if (rc < 0) {
+			/* skip empty (null) phandles */
+			if (rc == -ENOENT)
+				continue;
+			else
+				return;
+		}
+		if (clkspec.np != node || clkspec.args[0] >= IMX6QDL_CLK_END) {
+			pr_err("ccm: parent clock %d not in ccm\n", index);
+			return;
+		}
+		parent = clkspec.args[0];
+
+		rc = of_parse_phandle_with_args(node, "assigned-clocks",
+				"#clock-cells", index, &clkspec);
+		if (rc < 0)
+			return;
+		if (clkspec.np != node || clkspec.args[0] >= IMX6QDL_CLK_END) {
+			pr_err("ccm: child clock %d not in ccm\n", index);
+			return;
+		}
+		child = clkspec.args[0];
+
+		if (child != IMX6QDL_CLK_LDB_DI0_SEL &&
+		    child != IMX6QDL_CLK_LDB_DI1_SEL)
+			continue;
+
+		sel = ldb_di_sel_by_clock_id(parent);
+		if (sel < 0) {
+			pr_err("ccm: invalid ldb_di%d parent clock: %d\n",
+			       child == IMX6QDL_CLK_LDB_DI1_SEL, parent);
+			continue;
+		}
+
+		if (child == IMX6QDL_CLK_LDB_DI0_SEL)
+			*ldb_di0_sel = sel;
+		if (child == IMX6QDL_CLK_LDB_DI1_SEL)
+			*ldb_di1_sel = sel;
+	}
+}
+
+#define CCM_CCDR		0x04
+#define CCM_CCSR		0x0c
+#define CCM_CS2CDR		0x2c
+
+#define CCDR_MMDC_CH1_MASK		BIT(16)
+#define CCSR_PLL3_SW_CLK_SEL		BIT(0)
+
+#define CS2CDR_LDB_DI0_CLK_SEL_SHIFT	9
+#define CS2CDR_LDB_DI1_CLK_SEL_SHIFT	12
+
+static void __init imx6q_mmdc_ch1_mask_handshake(void __iomem *ccm_base)
+{
+	unsigned int reg;
+
+	reg = readl_relaxed(ccm_base + CCM_CCDR);
+	reg |= CCDR_MMDC_CH1_MASK;
+	writel_relaxed(reg, ccm_base + CCM_CCDR);
+}
+
+/*
+ * The only way to disable the MMDC_CH1 clock is to move it to pll3_sw_clk
+ * via periph2_clk2_sel and then to disable pll3_sw_clk by selecting the
+ * bypass clock source, since there is no CG bit for mmdc_ch1.
+ */
+static void mmdc_ch1_disable(void __iomem *ccm_base)
+{
+	unsigned int reg;
+
+	clk_set_parent(clk[IMX6QDL_CLK_PERIPH2_CLK2_SEL],
+		       clk[IMX6QDL_CLK_PLL3_USB_OTG]);
+
+	/*
+	 * Handshake with mmdc_ch1 module must be masked when changing
+	 * periph2_clk_sel.
+	 */
+	clk_set_parent(clk[IMX6QDL_CLK_PERIPH2], clk[IMX6QDL_CLK_PERIPH2_CLK2]);
+
+	/* Disable pll3_sw_clk by selecting the bypass clock source */
+	reg = readl_relaxed(ccm_base + CCM_CCSR);
+	reg |= CCSR_PLL3_SW_CLK_SEL;
+	writel_relaxed(reg, ccm_base + CCM_CCSR);
+}
+
+static void mmdc_ch1_reenable(void __iomem *ccm_base)
+{
+	unsigned int reg;
+
+	/* Enable pll3_sw_clk by disabling the bypass */
+	reg = readl_relaxed(ccm_base + CCM_CCSR);
+	reg &= ~CCSR_PLL3_SW_CLK_SEL;
+	writel_relaxed(reg, ccm_base + CCM_CCSR);
+
+	clk_set_parent(clk[IMX6QDL_CLK_PERIPH2], clk[IMX6QDL_CLK_PERIPH2_PRE]);
+}
+
+/*
+ * We have to follow a strict procedure when changing the LDB clock source,
+ * otherwise we risk introducing a glitch that can lock up the LDB divider.
+ * Things to keep in mind:
+ *
+ * 1. The current and new parent clock inputs to the mux must be disabled.
+ * 2. The default clock input for ldb_di0/1_clk_sel is mmdc_ch1_axi, which
+ *    has no CG bit.
+ * 3. pll2_pfd2_396m can not be gated if it is used as memory clock.
+ * 4. In the RTL implementation of the LDB_DI_CLK_SEL muxes the top four
+ *    options are in one mux and the PLL3 option along with three unused
+ *    inputs is in a second mux. There is a third mux with two inputs used
+ *    to decide between the first and second 4-port mux:
+ *
+ *    pll5_video_div 0 --|\
+ *    pll2_pfd0_352m 1 --| |_
+ *    pll2_pfd2_396m 2 --| | `-|\
+ *    mmdc_ch1_axi   3 --|/    | |
+ *                             | |--
+ *    pll3_usb_otg   4 --|\    | |
+ *                   5 --| |_,-|/
+ *                   6 --| |
+ *                   7 --|/
+ *
+ * The ldb_di0/1_clk_sel[1:0] bits control both 4-port muxes at the same time.
+ * The ldb_di0/1_clk_sel[2] bit controls the 2-port mux. The code below
+ * switches the parent to the bottom mux first and then manipulates the top
+ * mux to ensure that no glitch will enter the divider.
+ */
+static void init_ldb_clks(struct device_node *np, void __iomem *ccm_base)
+{
+	unsigned int reg;
+	unsigned int sel[2][4];
+	int i;
+
+	reg = readl_relaxed(ccm_base + CCM_CS2CDR);
+	sel[0][0] = (reg >> CS2CDR_LDB_DI0_CLK_SEL_SHIFT) & 7;
+	sel[1][0] = (reg >> CS2CDR_LDB_DI1_CLK_SEL_SHIFT) & 7;
+
+	sel[0][3] = sel[0][2] = sel[0][1] = sel[0][0];
+	sel[1][3] = sel[1][2] = sel[1][1] = sel[1][0];
+
+	of_assigned_ldb_sels(np, &sel[0][3], &sel[1][3]);
+
+	for (i = 0; i < 2; i++) {
+		/* Warn if a glitch might have been introduced already */
+		if (sel[i][0] != 3) {
+			pr_warn("ccm: ldb_di%d_sel already changed from reset value: %d\n",
+				i, sel[i][0]);
+		}
+
+		if (sel[i][0] == sel[i][3])
+			continue;
+
+		/* Only switch to or from pll2_pfd2_396m if it is disabled */
+		if ((sel[i][0] == 2 || sel[i][3] == 2) &&
+		    (clk_get_parent(clk[IMX6QDL_CLK_PERIPH_PRE]) ==
+		     clk[IMX6QDL_CLK_PLL2_PFD2_396M])) {
+			pr_err("ccm: ldb_di%d_sel: couldn't disable pll2_pfd2_396m\n",
+			       i);
+			sel[i][3] = sel[i][2] = sel[i][1] = sel[i][0];
+			continue;
+		}
+
+		/* First switch to the bottom mux */
+		sel[i][1] = sel[i][0] | 4;
+
+		/* Then configure the top mux before switching back to it */
+		sel[i][2] = sel[i][3] | 4;
+
+		pr_debug("ccm: switching ldb_di%d_sel: %d->%d->%d->%d\n", i,
+			 sel[i][0], sel[i][1], sel[i][2], sel[i][3]);
+	}
+
+	if (sel[0][0] == sel[0][3] && sel[1][0] == sel[1][3])
+		return;
+
+	mmdc_ch1_disable(ccm_base);
+
+	for (i = 1; i < 4; i++) {
+		reg = readl_relaxed(ccm_base + CCM_CS2CDR);
+		reg &= ~((7 << CS2CDR_LDB_DI0_CLK_SEL_SHIFT) |
+			 (7 << CS2CDR_LDB_DI1_CLK_SEL_SHIFT));
+		reg |= ((sel[0][i] << CS2CDR_LDB_DI0_CLK_SEL_SHIFT) |
+			(sel[1][i] << CS2CDR_LDB_DI1_CLK_SEL_SHIFT));
+		writel_relaxed(reg, ccm_base + CCM_CS2CDR);
+	}
+
+	mmdc_ch1_reenable(ccm_base);
+}
+
+#define CCM_ANALOG_PLL_VIDEO	0xa0
+#define CCM_ANALOG_PFD_480	0xf0
+#define CCM_ANALOG_PFD_528	0x100
+
+#define PLL_ENABLE		BIT(13)
+
+#define PFD0_CLKGATE		BIT(7)
+#define PFD1_CLKGATE		BIT(15)
+#define PFD2_CLKGATE		BIT(23)
+#define PFD3_CLKGATE		BIT(31)
+
+static void disable_anatop_clocks(void __iomem *anatop_base)
+{
+	unsigned int reg;
+
+	/* Make sure PLL2 PFDs 0-2 are gated */
+	reg = readl_relaxed(anatop_base + CCM_ANALOG_PFD_528);
+	/* Cannot gate PFD2 if pll2_pfd2_396m is the parent of MMDC clock */
+	if (clk_get_parent(clk[IMX6QDL_CLK_PERIPH_PRE]) ==
+	    clk[IMX6QDL_CLK_PLL2_PFD2_396M])
+		reg |= PFD0_CLKGATE | PFD1_CLKGATE;
+	else
+		reg |= PFD0_CLKGATE | PFD1_CLKGATE | PFD2_CLKGATE;
+	writel_relaxed(reg, anatop_base + CCM_ANALOG_PFD_528);
+
+	/* Make sure PLL3 PFDs 0-3 are gated */
+	reg = readl_relaxed(anatop_base + CCM_ANALOG_PFD_480);
+	reg |= PFD0_CLKGATE | PFD1_CLKGATE | PFD2_CLKGATE | PFD3_CLKGATE;
+	writel_relaxed(reg, anatop_base + CCM_ANALOG_PFD_480);
+
+	/* Make sure PLL5 is disabled */
+	reg = readl_relaxed(anatop_base + CCM_ANALOG_PLL_VIDEO);
+	reg &= ~PLL_ENABLE;
+	writel_relaxed(reg, anatop_base + CCM_ANALOG_PLL_VIDEO);
+}
+
 static void __init imx6q_clocks_init(struct device_node *ccm_node)
 {
 	struct device_node *np;
-	void __iomem *base;
+	void __iomem *anatop_base, *base;
 	int i;
 	int ret;
 
@@ -172,7 +429,7 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
 	clk[IMX6QDL_CLK_ANACLK2] = imx_obtain_fixed_clock("anaclk2", 0);
 
 	np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-anatop");
-	base = of_iomap(np, 0);
+	anatop_base = base = of_iomap(np, 0);
 	WARN_ON(!base);
 
 	/* Audio/video PLL post dividers do not work on i.MX6q revision 1.0 */
@@ -330,8 +587,20 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
 		clk[IMX6QDL_CLK_GPU3D_SHADER_SEL] = imx_clk_mux("gpu3d_shader_sel", base + 0x18, 8,  2, gpu3d_shader_sels, ARRAY_SIZE(gpu3d_shader_sels));
 	clk[IMX6QDL_CLK_IPU1_SEL]         = imx_clk_mux("ipu1_sel",         base + 0x3c, 9,  2, ipu_sels,          ARRAY_SIZE(ipu_sels));
 	clk[IMX6QDL_CLK_IPU2_SEL]         = imx_clk_mux("ipu2_sel",         base + 0x3c, 14, 2, ipu_sels,          ARRAY_SIZE(ipu_sels));
-	clk[IMX6QDL_CLK_LDB_DI0_SEL]      = imx_clk_mux_flags("ldb_di0_sel", base + 0x2c, 9,  3, ldb_di_sels,      ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT);
-	clk[IMX6QDL_CLK_LDB_DI1_SEL]      = imx_clk_mux_flags("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di_sels,      ARRAY_SIZE(ldb_di_sels), CLK_SET_RATE_PARENT);
+
+	disable_anatop_clocks(anatop_base);
+
+	imx6q_mmdc_ch1_mask_handshake(base);
+
+	/*
+	 * The LDB_DI0/1_SEL muxes are registered read-only due to a hardware
+	 * bug. Set the muxes to the requested values before registering the
+	 * ldb_di_sel clocks.
+	 */
+	init_ldb_clks(np, base);
+
+	clk[IMX6QDL_CLK_LDB_DI0_SEL]      = imx_clk_mux_ldb("ldb_di0_sel", base + 0x2c, 9,  3, ldb_di_sels,      ARRAY_SIZE(ldb_di_sels));
+	clk[IMX6QDL_CLK_LDB_DI1_SEL]      = imx_clk_mux_ldb("ldb_di1_sel", base + 0x2c, 12, 3, ldb_di_sels,      ARRAY_SIZE(ldb_di_sels));
 	clk[IMX6QDL_CLK_IPU1_DI0_PRE_SEL] = imx_clk_mux_flags("ipu1_di0_pre_sel", base + 0x34, 6,  3, ipu_di_pre_sels,   ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT);
 	clk[IMX6QDL_CLK_IPU1_DI1_PRE_SEL] = imx_clk_mux_flags("ipu1_di1_pre_sel", base + 0x34, 15, 3, ipu_di_pre_sels,   ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT);
 	clk[IMX6QDL_CLK_IPU2_DI0_PRE_SEL] = imx_clk_mux_flags("ipu2_di0_pre_sel", base + 0x38, 6,  3, ipu_di_pre_sels,   ARRAY_SIZE(ipu_di_pre_sels), CLK_SET_RATE_PARENT);
@@ -582,12 +851,6 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node)
 
 	clk_register_clkdev(clk[IMX6QDL_CLK_ENET_REF], "enet_ref", NULL);
 
-	if ((imx_get_soc_revision() != IMX_CHIP_REVISION_1_0) ||
-	    clk_on_imx6dl()) {
-		clk_set_parent(clk[IMX6QDL_CLK_LDB_DI0_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]);
-		clk_set_parent(clk[IMX6QDL_CLK_LDB_DI1_SEL], clk[IMX6QDL_CLK_PLL5_VIDEO_DIV]);
-	}
-
 	clk_set_rate(clk[IMX6QDL_CLK_PLL3_PFD1_540M], 540000000);
 	if (clk_on_imx6dl())
 		clk_set_parent(clk[IMX6QDL_CLK_IPU1_SEL], clk[IMX6QDL_CLK_PLL3_PFD1_540M]);
diff --git a/drivers/clk/imx/clk-imx6ul.c b/drivers/clk/imx/clk-imx6ul.c
index d1d7787ce211..75c35fb12b60 100644
--- a/drivers/clk/imx/clk-imx6ul.c
+++ b/drivers/clk/imx/clk-imx6ul.c
@@ -64,6 +64,10 @@ static const char *perclk_sels[] = { "ipg", "osc", };
 static const char *lcdif_sels[] = { "lcdif_podf", "ipp_di0", "ipp_di1", "ldb_di0", "ldb_di1", };
 static const char *csi_sels[] = { "osc", "pll2_pfd2_396m", "pll3_120m", "pll3_pfd1_540m", };
 static const char *sim_sels[] = { "sim_podf", "ipp_di0", "ipp_di1", "ldb_di0", "ldb_di1", };
+/* epdc_pre_sels, epdc_sels, esai_sels only exists on i.MX6ULL */
+static const char *epdc_pre_sels[] = { "pll2_bus", "pll3_usb_otg", "pll5_video_div", "pll2_pfd0_352m", "pll2_pfd2_396m", "pll3_pfd2_508m", };
+static const char *esai_sels[] = { "pll4_audio_div", "pll3_pfd2_508m", "pll5_video_div", "pll3_usb_otg", };
+static const char *epdc_sels[] = { "epdc_podf", "ipp_di0", "ipp_di1", "ldb_di0", "ldb_di1", };
 
 static struct clk *clks[IMX6UL_CLK_END];
 static struct clk_onecell_data clk_data;
@@ -102,6 +106,17 @@ static u32 share_count_audio;
 static u32 share_count_sai1;
 static u32 share_count_sai2;
 static u32 share_count_sai3;
+static u32 share_count_esai;
+
+static inline int clk_on_imx6ul(void)
+{
+	return of_machine_is_compatible("fsl,imx6ul");
+}
+
+static inline int clk_on_imx6ull(void)
+{
+	return of_machine_is_compatible("fsl,imx6ull");
+}
 
 static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 {
@@ -238,12 +253,19 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 	clks[IMX6UL_CLK_QSPI1_SEL]	  = imx_clk_mux("qspi1_sel",    base + 0x1c, 7,  3, qspi1_sels, ARRAY_SIZE(qspi1_sels));
 	clks[IMX6UL_CLK_PERCLK_SEL]	  = imx_clk_mux("perclk_sel",	base + 0x1c, 6,  1, perclk_sels, ARRAY_SIZE(perclk_sels));
 	clks[IMX6UL_CLK_CAN_SEL]	  = imx_clk_mux("can_sel",	base + 0x20, 8,  2, can_sels, ARRAY_SIZE(can_sels));
+	if (clk_on_imx6ull())
+		clks[IMX6ULL_CLK_ESAI_SEL]	  = imx_clk_mux("esai_sel",	base + 0x20, 19, 2, esai_sels, ARRAY_SIZE(esai_sels));
 	clks[IMX6UL_CLK_UART_SEL]	  = imx_clk_mux("uart_sel",	base + 0x24, 6,  1, uart_sels, ARRAY_SIZE(uart_sels));
 	clks[IMX6UL_CLK_ENFC_SEL]	  = imx_clk_mux("enfc_sel",	base + 0x2c, 15, 3, enfc_sels, ARRAY_SIZE(enfc_sels));
 	clks[IMX6UL_CLK_LDB_DI0_SEL]	  = imx_clk_mux("ldb_di0_sel",	base + 0x2c, 9,  3, ldb_di0_sels, ARRAY_SIZE(ldb_di0_sels));
 	clks[IMX6UL_CLK_SPDIF_SEL]	  = imx_clk_mux("spdif_sel",	base + 0x30, 20, 2, spdif_sels, ARRAY_SIZE(spdif_sels));
-	clks[IMX6UL_CLK_SIM_PRE_SEL]	  = imx_clk_mux("sim_pre_sel",	base + 0x34, 15, 3, sim_pre_sels, ARRAY_SIZE(sim_pre_sels));
-	clks[IMX6UL_CLK_SIM_SEL]	  = imx_clk_mux("sim_sel",	base + 0x34, 9, 3, sim_sels, ARRAY_SIZE(sim_sels));
+	if (clk_on_imx6ul()) {
+		clks[IMX6UL_CLK_SIM_PRE_SEL] 	  = imx_clk_mux("sim_pre_sel",	base + 0x34, 15, 3, sim_pre_sels, ARRAY_SIZE(sim_pre_sels));
+		clks[IMX6UL_CLK_SIM_SEL]	  = imx_clk_mux("sim_sel", 	base + 0x34, 9, 3, sim_sels, ARRAY_SIZE(sim_sels));
+	} else if (clk_on_imx6ull()) {
+		clks[IMX6ULL_CLK_EPDC_PRE_SEL]	  = imx_clk_mux("epdc_pre_sel",	base + 0x34, 15, 3, epdc_pre_sels, ARRAY_SIZE(epdc_pre_sels));
+		clks[IMX6ULL_CLK_EPDC_SEL]	  = imx_clk_mux("epdc_sel",	base + 0x34, 9, 3, epdc_sels, ARRAY_SIZE(epdc_sels));
+	}
 	clks[IMX6UL_CLK_ECSPI_SEL]	  = imx_clk_mux("ecspi_sel",	base + 0x38, 18, 1, ecspi_sels, ARRAY_SIZE(ecspi_sels));
 	clks[IMX6UL_CLK_LCDIF_PRE_SEL]	  = imx_clk_mux("lcdif_pre_sel", base + 0x38, 15, 3, lcdif_pre_sels, ARRAY_SIZE(lcdif_pre_sels));
 	clks[IMX6UL_CLK_LCDIF_SEL]	  = imx_clk_mux("lcdif_sel",	base + 0x38, 9, 3, lcdif_sels, ARRAY_SIZE(lcdif_sels));
@@ -276,6 +298,10 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 	clks[IMX6UL_CLK_SAI3_PODF]	= imx_clk_divider("sai3_podf",	   "sai3_pred",		base + 0x28, 16, 6);
 	clks[IMX6UL_CLK_SAI1_PRED]	= imx_clk_divider("sai1_pred",	   "sai1_sel",		base + 0x28, 6,	 3);
 	clks[IMX6UL_CLK_SAI1_PODF]	= imx_clk_divider("sai1_podf",	   "sai1_pred",		base + 0x28, 0,	 6);
+	if (clk_on_imx6ull()) {
+		clks[IMX6ULL_CLK_ESAI_PRED]	= imx_clk_divider("esai_pred",     "esai_sel",		base + 0x28, 9,  3);
+		clks[IMX6ULL_CLK_ESAI_PODF]	= imx_clk_divider("esai_podf",     "esai_pred",		base + 0x28, 25, 3);
+	}
 	clks[IMX6UL_CLK_ENFC_PRED]	= imx_clk_divider("enfc_pred",	   "enfc_sel",		base + 0x2c, 18, 3);
 	clks[IMX6UL_CLK_ENFC_PODF]	= imx_clk_divider("enfc_podf",	   "enfc_pred",		base + 0x2c, 21, 6);
 	clks[IMX6UL_CLK_SAI2_PRED]	= imx_clk_divider("sai2_pred",	   "sai2_sel",		base + 0x2c, 6,	 3);
@@ -298,9 +324,15 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 	clks[IMX6UL_CLK_APBHDMA]	= imx_clk_gate2("apbh_dma",	"bch_podf",	base + 0x68,	4);
 	clks[IMX6UL_CLK_ASRC_IPG]	= imx_clk_gate2_shared("asrc_ipg",	"ahb",	base + 0x68,	6, &share_count_asrc);
 	clks[IMX6UL_CLK_ASRC_MEM]	= imx_clk_gate2_shared("asrc_mem",	"ahb",	base + 0x68,	6, &share_count_asrc);
-	clks[IMX6UL_CLK_CAAM_MEM]	= imx_clk_gate2("caam_mem",	"ahb",		base + 0x68,	8);
-	clks[IMX6UL_CLK_CAAM_ACLK]	= imx_clk_gate2("caam_aclk",	"ahb",		base + 0x68,	10);
-	clks[IMX6UL_CLK_CAAM_IPG]	= imx_clk_gate2("caam_ipg",	"ipg",		base + 0x68,	12);
+	if (clk_on_imx6ul()) {
+		clks[IMX6UL_CLK_CAAM_MEM]	= imx_clk_gate2("caam_mem",	"ahb",		base + 0x68,	8);
+		clks[IMX6UL_CLK_CAAM_ACLK]	= imx_clk_gate2("caam_aclk",	"ahb",		base + 0x68,	10);
+		clks[IMX6UL_CLK_CAAM_IPG]	= imx_clk_gate2("caam_ipg",	"ipg",		base + 0x68,	12);
+	} else if (clk_on_imx6ull()) {
+		clks[IMX6ULL_CLK_DCP_CLK]	= imx_clk_gate2("dcp",		"ahb",		base + 0x68,	10);
+		clks[IMX6UL_CLK_ENET]		= imx_clk_gate2("enet",		"ipg",		base + 0x68,	12);
+		clks[IMX6UL_CLK_ENET_AHB]	= imx_clk_gate2("enet_ahb",	"ahb",		base + 0x68,	12);
+	}
 	clks[IMX6UL_CLK_CAN1_IPG]	= imx_clk_gate2("can1_ipg",	"ipg",		base + 0x68,	14);
 	clks[IMX6UL_CLK_CAN1_SERIAL]	= imx_clk_gate2("can1_serial",	"can_podf",	base + 0x68,	16);
 	clks[IMX6UL_CLK_CAN2_IPG]	= imx_clk_gate2("can2_ipg",	"ipg",		base + 0x68,	18);
@@ -309,7 +341,10 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 	clks[IMX6UL_CLK_GPT2_SERIAL]	= imx_clk_gate2("gpt2_serial",	"perclk",	base + 0x68,	26);
 	clks[IMX6UL_CLK_UART2_IPG]	= imx_clk_gate2("uart2_ipg",	"ipg",		base + 0x68,	28);
 	clks[IMX6UL_CLK_UART2_SERIAL]	= imx_clk_gate2("uart2_serial",	"uart_podf",	base + 0x68,	28);
-	clks[IMX6UL_CLK_AIPSTZ3]	= imx_clk_gate2("aips_tz3",	"ahb",		base + 0x68,	30);
+	if (clk_on_imx6ul())
+		clks[IMX6UL_CLK_AIPSTZ3]	= imx_clk_gate2("aips_tz3",	"ahb",		base + 0x68,	30);
+	else if (clk_on_imx6ull())
+		clks[IMX6UL_CLK_AIPSTZ3]	= imx_clk_gate2("aips_tz3",	"ahb",		 base + 0x80,	18);
 
 	/* CCGR1 */
 	clks[IMX6UL_CLK_ECSPI1]		= imx_clk_gate2("ecspi1",	"ecspi_podf",	base + 0x6c,	0);
@@ -328,6 +363,11 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 	clks[IMX6UL_CLK_UART4_SERIAL]	= imx_clk_gate2("uart4_serail",	"uart_podf",	base + 0x6c,	24);
 
 	/* CCGR2 */
+	if (clk_on_imx6ull()) {
+		clks[IMX6ULL_CLK_ESAI_EXTAL]	= imx_clk_gate2_shared("esai_extal",	"esai_podf",	base + 0x70,	0, &share_count_esai);
+		clks[IMX6ULL_CLK_ESAI_IPG]	= imx_clk_gate2_shared("esai_ipg",	"ahb",		base + 0x70,	0, &share_count_esai);
+		clks[IMX6ULL_CLK_ESAI_MEM]	= imx_clk_gate2_shared("esai_mem",	"ahb",		base + 0x70,	0, &share_count_esai);
+	}
 	clks[IMX6UL_CLK_CSI]		= imx_clk_gate2("csi",		"csi_podf",		base + 0x70,	2);
 	clks[IMX6UL_CLK_I2C1]		= imx_clk_gate2("i2c1",		"perclk",	base + 0x70,	6);
 	clks[IMX6UL_CLK_I2C2]		= imx_clk_gate2("i2c2",		"perclk",	base + 0x70,	8);
@@ -340,8 +380,13 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 	/* CCGR3 */
 	clks[IMX6UL_CLK_UART5_IPG]	= imx_clk_gate2("uart5_ipg",	"ipg",		base + 0x74,	2);
 	clks[IMX6UL_CLK_UART5_SERIAL]	= imx_clk_gate2("uart5_serial",	"uart_podf",	base + 0x74,	2);
-	clks[IMX6UL_CLK_ENET]		= imx_clk_gate2("enet",		"ipg",		base + 0x74,	4);
-	clks[IMX6UL_CLK_ENET_AHB]	= imx_clk_gate2("enet_ahb",	"ahb",		base + 0x74,	4);
+	if (clk_on_imx6ul()) {
+		clks[IMX6UL_CLK_ENET]		= imx_clk_gate2("enet",		"ipg",		base + 0x74,	4);
+		clks[IMX6UL_CLK_ENET_AHB]	= imx_clk_gate2("enet_ahb",	"ahb",		base + 0x74,	4);
+	} else if (clk_on_imx6ull()) {
+		clks[IMX6ULL_CLK_EPDC_ACLK]	= imx_clk_gate2("epdc_aclk",	"axi",		base + 0x74,	4);
+		clks[IMX6ULL_CLK_EPDC_PIX]	= imx_clk_gate2("epdc_pix",	"epdc_podf",	base + 0x74,	4);
+	}
 	clks[IMX6UL_CLK_UART6_IPG]	= imx_clk_gate2("uart6_ipg",	"ipg",		base + 0x74,	6);
 	clks[IMX6UL_CLK_UART6_SERIAL]	= imx_clk_gate2("uart6_serial",	"uart_podf",	base + 0x74,	6);
 	clks[IMX6UL_CLK_LCDIF_PIX]	= imx_clk_gate2("lcdif_pix",	"lcdif_podf",	base + 0x74,	10);
@@ -385,8 +430,10 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 	clks[IMX6UL_CLK_USBOH3]		= imx_clk_gate2("usboh3",	"ipg",		 base + 0x80,	0);
 	clks[IMX6UL_CLK_USDHC1]		= imx_clk_gate2("usdhc1",	"usdhc1_podf",	 base + 0x80,	2);
 	clks[IMX6UL_CLK_USDHC2]		= imx_clk_gate2("usdhc2",	"usdhc2_podf",	 base + 0x80,	4);
-	clks[IMX6UL_CLK_SIM1]		= imx_clk_gate2("sim1",		"sim_sel",	 base + 0x80,	6);
-	clks[IMX6UL_CLK_SIM2]		= imx_clk_gate2("sim2",		"sim_sel",	 base + 0x80,	8);
+	if (clk_on_imx6ul()) {
+		clks[IMX6UL_CLK_SIM1]		= imx_clk_gate2("sim1",		"sim_sel",	 base + 0x80,	6);
+		clks[IMX6UL_CLK_SIM2]		= imx_clk_gate2("sim2",		"sim_sel",	 base + 0x80,	8);
+	}
 	clks[IMX6UL_CLK_EIM]		= imx_clk_gate2("eim",		"eim_slow_podf", base + 0x80,	10);
 	clks[IMX6UL_CLK_PWM8]		= imx_clk_gate2("pwm8",		"perclk",	 base + 0x80,	16);
 	clks[IMX6UL_CLK_UART8_IPG]	= imx_clk_gate2("uart8_ipg",	"ipg",		 base + 0x80,	14);
@@ -441,7 +488,10 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 	}
 
 	clk_set_parent(clks[IMX6UL_CLK_CAN_SEL], clks[IMX6UL_CLK_PLL3_60M]);
-	clk_set_parent(clks[IMX6UL_CLK_SIM_PRE_SEL], clks[IMX6UL_CLK_PLL3_USB_OTG]);
+	if (clk_on_imx6ul())
+		clk_set_parent(clks[IMX6UL_CLK_SIM_PRE_SEL], clks[IMX6UL_CLK_PLL3_USB_OTG]);
+	else if (clk_on_imx6ull())
+		clk_set_parent(clks[IMX6ULL_CLK_EPDC_PRE_SEL], clks[IMX6UL_CLK_PLL3_PFD2]);
 
 	clk_set_parent(clks[IMX6UL_CLK_ENFC_SEL], clks[IMX6UL_CLK_PLL2_PFD2]);
 }
diff --git a/drivers/clk/imx/clk-pllv3.c b/drivers/clk/imx/clk-pllv3.c
index 7a6acc3e4a92..ed3a2df536ea 100644
--- a/drivers/clk/imx/clk-pllv3.c
+++ b/drivers/clk/imx/clk-pllv3.c
@@ -234,6 +234,7 @@ static long clk_pllv3_av_round_rate(struct clk_hw *hw, unsigned long rate,
 	unsigned long max_rate = parent_rate * 54;
 	u32 div;
 	u32 mfn, mfd = 1000000;
+	u32 max_mfd = 0x3FFFFFFF;
 	u64 temp64;
 
 	if (rate > max_rate)
@@ -241,6 +242,9 @@ static long clk_pllv3_av_round_rate(struct clk_hw *hw, unsigned long rate,
 	else if (rate < min_rate)
 		rate = min_rate;
 
+	if (parent_rate <= max_mfd)
+		mfd = parent_rate;
+
 	div = rate / parent_rate;
 	temp64 = (u64) (rate - div * parent_rate);
 	temp64 *= mfd;
@@ -262,11 +266,15 @@ static int clk_pllv3_av_set_rate(struct clk_hw *hw, unsigned long rate,
 	unsigned long max_rate = parent_rate * 54;
 	u32 val, div;
 	u32 mfn, mfd = 1000000;
+	u32 max_mfd = 0x3FFFFFFF;
 	u64 temp64;
 
 	if (rate < min_rate || rate > max_rate)
 		return -EINVAL;
 
+	if (parent_rate <= max_mfd)
+		mfd = parent_rate;
+
 	div = rate / parent_rate;
 	temp64 = (u64) (rate - div * parent_rate);
 	temp64 *= mfd;
diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h
index 3799ff82a9b4..4afad3b96a61 100644
--- a/drivers/clk/imx/clk.h
+++ b/drivers/clk/imx/clk.h
@@ -75,6 +75,14 @@ static inline struct clk *imx_clk_fixed(const char *name, int rate)
 	return clk_register_fixed_rate(NULL, name, NULL, 0, rate);
 }
 
+static inline struct clk *imx_clk_mux_ldb(const char *name, void __iomem *reg,
+		u8 shift, u8 width, const char **parents, int num_parents)
+{
+	return clk_register_mux(NULL, name, parents, num_parents,
+			CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT, reg,
+			shift, width, CLK_MUX_READ_ONLY, &imx_ccm_lock);
+}
+
 static inline struct clk *imx_clk_fixed_factor(const char *name,
 		const char *parent, unsigned int mult, unsigned int div)
 {
diff --git a/drivers/clk/keystone/pll.c b/drivers/clk/keystone/pll.c
index a26ba2184454..e7e840fb74ea 100644
--- a/drivers/clk/keystone/pll.c
+++ b/drivers/clk/keystone/pll.c
@@ -154,7 +154,7 @@ out:
 }
 
 /**
- * _of_clk_init - PLL initialisation via DT
+ * _of_pll_clk_init - PLL initialisation via DT
  * @node: device tree node for this clock
  * @pllctrl: If true, lower 6 bits of multiplier is in pllm register of
  *		pll controller, else it is in the control register0(bit 11-6)
@@ -235,7 +235,7 @@ CLK_OF_DECLARE(keystone_pll_clock, "ti,keystone,pll-clock",
 					of_keystone_pll_clk_init);
 
 /**
- * of_keystone_pll_main_clk_init - Main PLL initialisation DT wrapper
+ * of_keystone_main_pll_clk_init - Main PLL initialisation DT wrapper
  * @node: device tree node for this clock
  */
 static void __init of_keystone_main_pll_clk_init(struct device_node *node)
@@ -267,25 +267,30 @@ static void __init of_pll_div_clk_init(struct device_node *node)
 	parent_name = of_clk_get_parent_name(node, 0);
 	if (!parent_name) {
 		pr_err("%s: missing parent clock\n", __func__);
+		iounmap(reg);
 		return;
 	}
 
 	if (of_property_read_u32(node, "bit-shift", &shift)) {
 		pr_err("%s: missing 'shift' property\n", __func__);
+		iounmap(reg);
 		return;
 	}
 
 	if (of_property_read_u32(node, "bit-mask", &mask)) {
 		pr_err("%s: missing 'bit-mask' property\n", __func__);
+		iounmap(reg);
 		return;
 	}
 
 	clk = clk_register_divider(NULL, clk_name, parent_name, 0, reg, shift,
 				 mask, 0, NULL);
-	if (clk)
+	if (clk) {
 		of_clk_add_provider(node, of_clk_src_simple_get, clk);
-	else
+	} else {
 		pr_err("%s: error registering divider %s\n", __func__, clk_name);
+		iounmap(reg);
+	}
 }
 CLK_OF_DECLARE(pll_divider_clock, "ti,keystone,pll-divider-clock", of_pll_div_clk_init);
 
diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig
index f042bd2a6a99..0bd631a41f6a 100644
--- a/drivers/clk/mediatek/Kconfig
+++ b/drivers/clk/mediatek/Kconfig
@@ -6,6 +6,49 @@ config COMMON_CLK_MEDIATEK
 	---help---
 	  Mediatek SoCs' clock support.
 
+config COMMON_CLK_MT2701
+	bool "Clock driver for Mediatek MT2701"
+	select COMMON_CLK_MEDIATEK
+	default ARCH_MEDIATEK
+	---help---
+	  This driver supports Mediatek MT2701 basic clocks.
+
+config COMMON_CLK_MT2701_MMSYS
+	bool "Clock driver for Mediatek MT2701 mmsys"
+	select COMMON_CLK_MT2701
+	---help---
+	  This driver supports Mediatek MT2701 mmsys clocks.
+
+config COMMON_CLK_MT2701_IMGSYS
+	bool "Clock driver for Mediatek MT2701 imgsys"
+	select COMMON_CLK_MT2701
+	---help---
+	  This driver supports Mediatek MT2701 imgsys clocks.
+
+config COMMON_CLK_MT2701_VDECSYS
+	bool "Clock driver for Mediatek MT2701 vdecsys"
+	select COMMON_CLK_MT2701
+	---help---
+	  This driver supports Mediatek MT2701 vdecsys clocks.
+
+config COMMON_CLK_MT2701_HIFSYS
+	bool "Clock driver for Mediatek MT2701 hifsys"
+	select COMMON_CLK_MT2701
+	---help---
+	  This driver supports Mediatek MT2701 hifsys clocks.
+
+config COMMON_CLK_MT2701_ETHSYS
+	bool "Clock driver for Mediatek MT2701 ethsys"
+	select COMMON_CLK_MT2701
+	---help---
+	  This driver supports Mediatek MT2701 ethsys clocks.
+
+config COMMON_CLK_MT2701_BDPSYS
+	bool "Clock driver for Mediatek MT2701 bdpsys"
+	select COMMON_CLK_MT2701
+	---help---
+	  This driver supports Mediatek MT2701 bdpsys clocks.
+
 config COMMON_CLK_MT8135
 	bool "Clock driver for Mediatek MT8135"
 	depends on ARCH_MEDIATEK || COMPILE_TEST
diff --git a/drivers/clk/mediatek/Makefile b/drivers/clk/mediatek/Makefile
index 32e7222e7305..19ae7ef79b57 100644
--- a/drivers/clk/mediatek/Makefile
+++ b/drivers/clk/mediatek/Makefile
@@ -1,4 +1,11 @@
 obj-$(CONFIG_COMMON_CLK_MEDIATEK) += clk-mtk.o clk-pll.o clk-gate.o clk-apmixed.o
 obj-$(CONFIG_RESET_CONTROLLER) += reset.o
+obj-$(CONFIG_COMMON_CLK_MT2701) += clk-mt2701.o
+obj-$(CONFIG_COMMON_CLK_MT2701_BDPSYS) += clk-mt2701-bdp.o
+obj-$(CONFIG_COMMON_CLK_MT2701_ETHSYS) += clk-mt2701-eth.o
+obj-$(CONFIG_COMMON_CLK_MT2701_HIFSYS) += clk-mt2701-hif.o
+obj-$(CONFIG_COMMON_CLK_MT2701_IMGSYS) += clk-mt2701-img.o
+obj-$(CONFIG_COMMON_CLK_MT2701_MMSYS) += clk-mt2701-mm.o
+obj-$(CONFIG_COMMON_CLK_MT2701_VDECSYS) += clk-mt2701-vdec.o
 obj-$(CONFIG_COMMON_CLK_MT8135) += clk-mt8135.o
 obj-$(CONFIG_COMMON_CLK_MT8173) += clk-mt8173.o
diff --git a/drivers/clk/mediatek/clk-gate.c b/drivers/clk/mediatek/clk-gate.c
index d8787bf444eb..934bf0e45e26 100644
--- a/drivers/clk/mediatek/clk-gate.c
+++ b/drivers/clk/mediatek/clk-gate.c
@@ -61,6 +61,22 @@ static void mtk_cg_clr_bit(struct clk_hw *hw)
 	regmap_write(cg->regmap, cg->clr_ofs, BIT(cg->bit));
 }
 
+static void mtk_cg_set_bit_no_setclr(struct clk_hw *hw)
+{
+	struct mtk_clk_gate *cg = to_mtk_clk_gate(hw);
+	u32 cgbit = BIT(cg->bit);
+
+	regmap_update_bits(cg->regmap, cg->sta_ofs, cgbit, cgbit);
+}
+
+static void mtk_cg_clr_bit_no_setclr(struct clk_hw *hw)
+{
+	struct mtk_clk_gate *cg = to_mtk_clk_gate(hw);
+	u32 cgbit = BIT(cg->bit);
+
+	regmap_update_bits(cg->regmap, cg->sta_ofs, cgbit, 0);
+}
+
 static int mtk_cg_enable(struct clk_hw *hw)
 {
 	mtk_cg_clr_bit(hw);
@@ -85,6 +101,30 @@ static void mtk_cg_disable_inv(struct clk_hw *hw)
 	mtk_cg_clr_bit(hw);
 }
 
+static int mtk_cg_enable_no_setclr(struct clk_hw *hw)
+{
+	mtk_cg_clr_bit_no_setclr(hw);
+
+	return 0;
+}
+
+static void mtk_cg_disable_no_setclr(struct clk_hw *hw)
+{
+	mtk_cg_set_bit_no_setclr(hw);
+}
+
+static int mtk_cg_enable_inv_no_setclr(struct clk_hw *hw)
+{
+	mtk_cg_set_bit_no_setclr(hw);
+
+	return 0;
+}
+
+static void mtk_cg_disable_inv_no_setclr(struct clk_hw *hw)
+{
+	mtk_cg_clr_bit_no_setclr(hw);
+}
+
 const struct clk_ops mtk_clk_gate_ops_setclr = {
 	.is_enabled	= mtk_cg_bit_is_cleared,
 	.enable		= mtk_cg_enable,
@@ -97,6 +137,18 @@ const struct clk_ops mtk_clk_gate_ops_setclr_inv = {
 	.disable	= mtk_cg_disable_inv,
 };
 
+const struct clk_ops mtk_clk_gate_ops_no_setclr = {
+	.is_enabled	= mtk_cg_bit_is_cleared,
+	.enable		= mtk_cg_enable_no_setclr,
+	.disable	= mtk_cg_disable_no_setclr,
+};
+
+const struct clk_ops mtk_clk_gate_ops_no_setclr_inv = {
+	.is_enabled	= mtk_cg_bit_is_set,
+	.enable		= mtk_cg_enable_inv_no_setclr,
+	.disable	= mtk_cg_disable_inv_no_setclr,
+};
+
 struct clk *mtk_clk_register_gate(
 		const char *name,
 		const char *parent_name,
diff --git a/drivers/clk/mediatek/clk-gate.h b/drivers/clk/mediatek/clk-gate.h
index b1821603b887..72ef89b3ad7b 100644
--- a/drivers/clk/mediatek/clk-gate.h
+++ b/drivers/clk/mediatek/clk-gate.h
@@ -36,6 +36,8 @@ static inline struct mtk_clk_gate *to_mtk_clk_gate(struct clk_hw *hw)
 
 extern const struct clk_ops mtk_clk_gate_ops_setclr;
 extern const struct clk_ops mtk_clk_gate_ops_setclr_inv;
+extern const struct clk_ops mtk_clk_gate_ops_no_setclr;
+extern const struct clk_ops mtk_clk_gate_ops_no_setclr_inv;
 
 struct clk *mtk_clk_register_gate(
 		const char *name,
diff --git a/drivers/clk/mediatek/clk-mt2701-bdp.c b/drivers/clk/mediatek/clk-mt2701-bdp.c
new file mode 100644
index 000000000000..fe4964d05b5f
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt2701-bdp.c
@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Shunli Wang <shunli.wang@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt2701-clk.h>
+
+static const struct mtk_gate_regs bdp0_cg_regs = {
+	.set_ofs = 0x0104,
+	.clr_ofs = 0x0108,
+	.sta_ofs = 0x0100,
+};
+
+static const struct mtk_gate_regs bdp1_cg_regs = {
+	.set_ofs = 0x0114,
+	.clr_ofs = 0x0118,
+	.sta_ofs = 0x0110,
+};
+
+#define GATE_BDP0(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &bdp0_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr_inv,	\
+	}
+
+#define GATE_BDP1(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &bdp1_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr_inv,	\
+	}
+
+static const struct mtk_gate bdp_clks[] = {
+	GATE_BDP0(CLK_BDP_BRG_BA, "brg_baclk", "mm_sel", 0),
+	GATE_BDP0(CLK_BDP_BRG_DRAM, "brg_dram", "mm_sel", 1),
+	GATE_BDP0(CLK_BDP_LARB_DRAM, "larb_dram", "mm_sel", 2),
+	GATE_BDP0(CLK_BDP_WR_VDI_PXL, "wr_vdi_pxl", "hdmi_0_deep340m", 3),
+	GATE_BDP0(CLK_BDP_WR_VDI_DRAM, "wr_vdi_dram", "mm_sel", 4),
+	GATE_BDP0(CLK_BDP_WR_B, "wr_bclk", "mm_sel", 5),
+	GATE_BDP0(CLK_BDP_DGI_IN, "dgi_in", "dpi1_sel", 6),
+	GATE_BDP0(CLK_BDP_DGI_OUT, "dgi_out", "dpi1_sel", 7),
+	GATE_BDP0(CLK_BDP_FMT_MAST_27, "fmt_mast_27", "dpi1_sel", 8),
+	GATE_BDP0(CLK_BDP_FMT_B, "fmt_bclk", "mm_sel", 9),
+	GATE_BDP0(CLK_BDP_OSD_B, "osd_bclk", "mm_sel", 10),
+	GATE_BDP0(CLK_BDP_OSD_DRAM, "osd_dram", "mm_sel", 11),
+	GATE_BDP0(CLK_BDP_OSD_AGENT, "osd_agent", "osd_sel", 12),
+	GATE_BDP0(CLK_BDP_OSD_PXL, "osd_pxl", "dpi1_sel", 13),
+	GATE_BDP0(CLK_BDP_RLE_B, "rle_bclk", "mm_sel", 14),
+	GATE_BDP0(CLK_BDP_RLE_AGENT, "rle_agent", "mm_sel", 15),
+	GATE_BDP0(CLK_BDP_RLE_DRAM, "rle_dram", "mm_sel", 16),
+	GATE_BDP0(CLK_BDP_F27M, "f27m", "di_sel", 17),
+	GATE_BDP0(CLK_BDP_F27M_VDOUT, "f27m_vdout", "di_sel", 18),
+	GATE_BDP0(CLK_BDP_F27_74_74, "f27_74_74", "di_sel", 19),
+	GATE_BDP0(CLK_BDP_F2FS, "f2fs", "di_sel", 20),
+	GATE_BDP0(CLK_BDP_F2FS74_148, "f2fs74_148", "di_sel", 21),
+	GATE_BDP0(CLK_BDP_FB, "fbclk", "mm_sel", 22),
+	GATE_BDP0(CLK_BDP_VDO_DRAM, "vdo_dram", "mm_sel", 23),
+	GATE_BDP0(CLK_BDP_VDO_2FS, "vdo_2fs", "di_sel", 24),
+	GATE_BDP0(CLK_BDP_VDO_B, "vdo_bclk", "mm_sel", 25),
+	GATE_BDP0(CLK_BDP_WR_DI_PXL, "wr_di_pxl", "di_sel", 26),
+	GATE_BDP0(CLK_BDP_WR_DI_DRAM, "wr_di_dram", "mm_sel", 27),
+	GATE_BDP0(CLK_BDP_WR_DI_B, "wr_di_bclk", "mm_sel", 28),
+	GATE_BDP0(CLK_BDP_NR_PXL, "nr_pxl", "nr_sel", 29),
+	GATE_BDP0(CLK_BDP_NR_DRAM, "nr_dram", "mm_sel", 30),
+	GATE_BDP0(CLK_BDP_NR_B, "nr_bclk", "mm_sel", 31),
+	GATE_BDP1(CLK_BDP_RX_F, "rx_fclk", "hadds2_fbclk", 0),
+	GATE_BDP1(CLK_BDP_RX_X, "rx_xclk", "clk26m", 1),
+	GATE_BDP1(CLK_BDP_RXPDT, "rxpdtclk", "hdmi_0_pix340m", 2),
+	GATE_BDP1(CLK_BDP_RX_CSCL_N, "rx_cscl_n", "clk26m", 3),
+	GATE_BDP1(CLK_BDP_RX_CSCL, "rx_cscl", "clk26m", 4),
+	GATE_BDP1(CLK_BDP_RX_DDCSCL_N, "rx_ddcscl_n", "hdmi_scl_rx", 5),
+	GATE_BDP1(CLK_BDP_RX_DDCSCL, "rx_ddcscl", "hdmi_scl_rx", 6),
+	GATE_BDP1(CLK_BDP_RX_VCO, "rx_vcoclk", "hadds2pll_294m", 7),
+	GATE_BDP1(CLK_BDP_RX_DP, "rx_dpclk", "hdmi_0_pll340m", 8),
+	GATE_BDP1(CLK_BDP_RX_P, "rx_pclk", "hdmi_0_pll340m", 9),
+	GATE_BDP1(CLK_BDP_RX_M, "rx_mclk", "hadds2pll_294m", 10),
+	GATE_BDP1(CLK_BDP_RX_PLL, "rx_pllclk", "hdmi_0_pix340m", 11),
+	GATE_BDP1(CLK_BDP_BRG_RT_B, "brg_rt_bclk", "mm_sel", 12),
+	GATE_BDP1(CLK_BDP_BRG_RT_DRAM, "brg_rt_dram", "mm_sel", 13),
+	GATE_BDP1(CLK_BDP_LARBRT_DRAM, "larbrt_dram", "mm_sel", 14),
+	GATE_BDP1(CLK_BDP_TMDS_SYN, "tmds_syn", "hdmi_0_pll340m", 15),
+	GATE_BDP1(CLK_BDP_HDMI_MON, "hdmi_mon", "hdmi_0_pll340m", 16),
+};
+
+static const struct of_device_id of_match_clk_mt2701_bdp[] = {
+	{ .compatible = "mediatek,mt2701-bdpsys", },
+	{}
+};
+
+static int clk_mt2701_bdp_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_BDP_NR);
+
+	mtk_clk_register_gates(node, bdp_clks, ARRAY_SIZE(bdp_clks),
+						clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (r)
+		dev_err(&pdev->dev,
+			"could not register clock provider: %s: %d\n",
+			pdev->name, r);
+
+	return r;
+}
+
+static struct platform_driver clk_mt2701_bdp_drv = {
+	.probe = clk_mt2701_bdp_probe,
+	.driver = {
+		.name = "clk-mt2701-bdp",
+		.of_match_table = of_match_clk_mt2701_bdp,
+	},
+};
+
+builtin_platform_driver(clk_mt2701_bdp_drv);
diff --git a/drivers/clk/mediatek/clk-mt2701-eth.c b/drivers/clk/mediatek/clk-mt2701-eth.c
new file mode 100644
index 000000000000..877be8715afa
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt2701-eth.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Shunli Wang <shunli.wang@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt2701-clk.h>
+
+static const struct mtk_gate_regs eth_cg_regs = {
+	.sta_ofs = 0x0030,
+};
+
+#define GATE_ETH(_id, _name, _parent, _shift) {		\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &eth_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_no_setclr_inv,	\
+	}
+
+static const struct mtk_gate eth_clks[] = {
+	GATE_ETH(CLK_ETHSYS_HSDMA, "hsdma_clk", "ethif_sel", 5),
+	GATE_ETH(CLK_ETHSYS_ESW, "esw_clk", "ethpll_500m_ck", 6),
+	GATE_ETH(CLK_ETHSYS_GP2, "gp2_clk", "trgpll", 7),
+	GATE_ETH(CLK_ETHSYS_GP1, "gp1_clk", "ethpll_500m_ck", 8),
+	GATE_ETH(CLK_ETHSYS_PCM, "pcm_clk", "ethif_sel", 11),
+	GATE_ETH(CLK_ETHSYS_GDMA, "gdma_clk", "ethif_sel", 14),
+	GATE_ETH(CLK_ETHSYS_I2S, "i2s_clk", "ethif_sel", 17),
+	GATE_ETH(CLK_ETHSYS_CRYPTO, "crypto_clk", "ethif_sel", 29),
+};
+
+static const struct of_device_id of_match_clk_mt2701_eth[] = {
+	{ .compatible = "mediatek,mt2701-ethsys", },
+	{}
+};
+
+static int clk_mt2701_eth_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_ETHSYS_NR);
+
+	mtk_clk_register_gates(node, eth_clks, ARRAY_SIZE(eth_clks),
+						clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (r)
+		dev_err(&pdev->dev,
+			"could not register clock provider: %s: %d\n",
+			pdev->name, r);
+
+	return r;
+}
+
+static struct platform_driver clk_mt2701_eth_drv = {
+	.probe = clk_mt2701_eth_probe,
+	.driver = {
+		.name = "clk-mt2701-eth",
+		.of_match_table = of_match_clk_mt2701_eth,
+	},
+};
+
+builtin_platform_driver(clk_mt2701_eth_drv);
diff --git a/drivers/clk/mediatek/clk-mt2701-hif.c b/drivers/clk/mediatek/clk-mt2701-hif.c
new file mode 100644
index 000000000000..18f3723be3e8
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt2701-hif.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Shunli Wang <shunli.wang@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt2701-clk.h>
+
+static const struct mtk_gate_regs hif_cg_regs = {
+	.sta_ofs = 0x0030,
+};
+
+#define GATE_HIF(_id, _name, _parent, _shift) {		\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &hif_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_no_setclr_inv,	\
+	}
+
+static const struct mtk_gate hif_clks[] = {
+	GATE_HIF(CLK_HIFSYS_USB0PHY, "usb0_phy_clk", "ethpll_500m_ck", 21),
+	GATE_HIF(CLK_HIFSYS_USB1PHY, "usb1_phy_clk", "ethpll_500m_ck", 22),
+	GATE_HIF(CLK_HIFSYS_PCIE0, "pcie0_clk", "ethpll_500m_ck", 24),
+	GATE_HIF(CLK_HIFSYS_PCIE1, "pcie1_clk", "ethpll_500m_ck", 25),
+	GATE_HIF(CLK_HIFSYS_PCIE2, "pcie2_clk", "ethpll_500m_ck", 26),
+};
+
+static const struct of_device_id of_match_clk_mt2701_hif[] = {
+	{ .compatible = "mediatek,mt2701-hifsys", },
+	{}
+};
+
+static int clk_mt2701_hif_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_HIFSYS_NR);
+
+	mtk_clk_register_gates(node, hif_clks, ARRAY_SIZE(hif_clks),
+						clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (r) {
+		dev_err(&pdev->dev,
+			"could not register clock provider: %s: %d\n",
+			pdev->name, r);
+		return r;
+	}
+
+	mtk_register_reset_controller(node, 1, 0x34);
+
+	return 0;
+}
+
+static struct platform_driver clk_mt2701_hif_drv = {
+	.probe = clk_mt2701_hif_probe,
+	.driver = {
+		.name = "clk-mt2701-hif",
+		.of_match_table = of_match_clk_mt2701_hif,
+	},
+};
+
+builtin_platform_driver(clk_mt2701_hif_drv);
diff --git a/drivers/clk/mediatek/clk-mt2701-img.c b/drivers/clk/mediatek/clk-mt2701-img.c
new file mode 100644
index 000000000000..b7441c98bda8
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt2701-img.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Shunli Wang <shunli.wang@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt2701-clk.h>
+
+static const struct mtk_gate_regs img_cg_regs = {
+	.set_ofs = 0x0004,
+	.clr_ofs = 0x0008,
+	.sta_ofs = 0x0000,
+};
+
+#define GATE_IMG(_id, _name, _parent, _shift) {		\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &img_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+static const struct mtk_gate img_clks[] = {
+	GATE_IMG(CLK_IMG_SMI_COMM, "img_smi_comm", "mm_sel", 0),
+	GATE_IMG(CLK_IMG_RESZ, "img_resz", "mm_sel", 1),
+	GATE_IMG(CLK_IMG_JPGDEC_SMI, "img_jpgdec_smi", "mm_sel", 5),
+	GATE_IMG(CLK_IMG_JPGDEC, "img_jpgdec", "mm_sel", 6),
+	GATE_IMG(CLK_IMG_VENC_LT, "img_venc_lt", "mm_sel", 8),
+	GATE_IMG(CLK_IMG_VENC, "img_venc", "mm_sel", 9),
+};
+
+static const struct of_device_id of_match_clk_mt2701_img[] = {
+	{ .compatible = "mediatek,mt2701-imgsys", },
+	{}
+};
+
+static int clk_mt2701_img_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_IMG_NR);
+
+	mtk_clk_register_gates(node, img_clks, ARRAY_SIZE(img_clks),
+						clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (r)
+		dev_err(&pdev->dev,
+			"could not register clock provider: %s: %d\n",
+			pdev->name, r);
+
+	return r;
+}
+
+static struct platform_driver clk_mt2701_img_drv = {
+	.probe = clk_mt2701_img_probe,
+	.driver = {
+		.name = "clk-mt2701-img",
+		.of_match_table = of_match_clk_mt2701_img,
+	},
+};
+
+builtin_platform_driver(clk_mt2701_img_drv);
diff --git a/drivers/clk/mediatek/clk-mt2701-mm.c b/drivers/clk/mediatek/clk-mt2701-mm.c
new file mode 100644
index 000000000000..fe1f85072fc5
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt2701-mm.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Shunli Wang <shunli.wang@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt2701-clk.h>
+
+static const struct mtk_gate_regs disp0_cg_regs = {
+	.set_ofs = 0x0104,
+	.clr_ofs = 0x0108,
+	.sta_ofs = 0x0100,
+};
+
+static const struct mtk_gate_regs disp1_cg_regs = {
+	.set_ofs = 0x0114,
+	.clr_ofs = 0x0118,
+	.sta_ofs = 0x0110,
+};
+
+#define GATE_DISP0(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &disp0_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+#define GATE_DISP1(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &disp1_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+static const struct mtk_gate mm_clks[] = {
+	GATE_DISP0(CLK_MM_SMI_COMMON, "mm_smi_comm", "mm_sel", 0),
+	GATE_DISP0(CLK_MM_SMI_LARB0, "mm_smi_larb0", "mm_sel", 1),
+	GATE_DISP0(CLK_MM_CMDQ, "mm_cmdq", "mm_sel", 2),
+	GATE_DISP0(CLK_MM_MUTEX, "mm_mutex", "mm_sel", 3),
+	GATE_DISP0(CLK_MM_DISP_COLOR, "mm_disp_color", "mm_sel", 4),
+	GATE_DISP0(CLK_MM_DISP_BLS, "mm_disp_bls", "mm_sel", 5),
+	GATE_DISP0(CLK_MM_DISP_WDMA, "mm_disp_wdma", "mm_sel", 6),
+	GATE_DISP0(CLK_MM_DISP_RDMA, "mm_disp_rdma", "mm_sel", 7),
+	GATE_DISP0(CLK_MM_DISP_OVL, "mm_disp_ovl", "mm_sel", 8),
+	GATE_DISP0(CLK_MM_MDP_TDSHP, "mm_mdp_tdshp", "mm_sel", 9),
+	GATE_DISP0(CLK_MM_MDP_WROT, "mm_mdp_wrot", "mm_sel", 10),
+	GATE_DISP0(CLK_MM_MDP_WDMA, "mm_mdp_wdma", "mm_sel", 11),
+	GATE_DISP0(CLK_MM_MDP_RSZ1, "mm_mdp_rsz1", "mm_sel", 12),
+	GATE_DISP0(CLK_MM_MDP_RSZ0, "mm_mdp_rsz0", "mm_sel", 13),
+	GATE_DISP0(CLK_MM_MDP_RDMA, "mm_mdp_rdma", "mm_sel", 14),
+	GATE_DISP0(CLK_MM_MDP_BLS_26M, "mm_mdp_bls_26m", "pwm_sel", 15),
+	GATE_DISP0(CLK_MM_CAM_MDP, "mm_cam_mdp", "mm_sel", 16),
+	GATE_DISP0(CLK_MM_FAKE_ENG, "mm_fake_eng", "mm_sel", 17),
+	GATE_DISP0(CLK_MM_MUTEX_32K, "mm_mutex_32k", "rtc_sel", 18),
+	GATE_DISP0(CLK_MM_DISP_RDMA1, "mm_disp_rdma1", "mm_sel", 19),
+	GATE_DISP0(CLK_MM_DISP_UFOE, "mm_disp_ufoe", "mm_sel", 20),
+	GATE_DISP1(CLK_MM_DSI_ENGINE, "mm_dsi_eng", "mm_sel", 0),
+	GATE_DISP1(CLK_MM_DSI_DIG, "mm_dsi_dig", "dsi0_lntc_dsi", 1),
+	GATE_DISP1(CLK_MM_DPI_DIGL, "mm_dpi_digl", "dpi0_sel", 2),
+	GATE_DISP1(CLK_MM_DPI_ENGINE, "mm_dpi_eng", "mm_sel", 3),
+	GATE_DISP1(CLK_MM_DPI1_DIGL, "mm_dpi1_digl", "dpi1_sel", 4),
+	GATE_DISP1(CLK_MM_DPI1_ENGINE, "mm_dpi1_eng", "mm_sel", 5),
+	GATE_DISP1(CLK_MM_TVE_OUTPUT, "mm_tve_output", "tve_sel", 6),
+	GATE_DISP1(CLK_MM_TVE_INPUT, "mm_tve_input", "dpi0_sel", 7),
+	GATE_DISP1(CLK_MM_HDMI_PIXEL, "mm_hdmi_pixel", "dpi1_sel", 8),
+	GATE_DISP1(CLK_MM_HDMI_PLL, "mm_hdmi_pll", "hdmi_sel", 9),
+	GATE_DISP1(CLK_MM_HDMI_AUDIO, "mm_hdmi_audio", "apll_sel", 10),
+	GATE_DISP1(CLK_MM_HDMI_SPDIF, "mm_hdmi_spdif", "apll_sel", 11),
+	GATE_DISP1(CLK_MM_TVE_FMM, "mm_tve_fmm", "mm_sel", 14),
+};
+
+static const struct of_device_id of_match_clk_mt2701_mm[] = {
+	{ .compatible = "mediatek,mt2701-mmsys", },
+	{}
+};
+
+static int clk_mt2701_mm_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_MM_NR);
+
+	mtk_clk_register_gates(node, mm_clks, ARRAY_SIZE(mm_clks),
+						clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (r)
+		dev_err(&pdev->dev,
+			"could not register clock provider: %s: %d\n",
+			pdev->name, r);
+
+	return r;
+}
+
+static struct platform_driver clk_mt2701_mm_drv = {
+	.probe = clk_mt2701_mm_probe,
+	.driver = {
+		.name = "clk-mt2701-mm",
+		.of_match_table = of_match_clk_mt2701_mm,
+	},
+};
+
+builtin_platform_driver(clk_mt2701_mm_drv);
diff --git a/drivers/clk/mediatek/clk-mt2701-vdec.c b/drivers/clk/mediatek/clk-mt2701-vdec.c
new file mode 100644
index 000000000000..d3c0fc9d6f02
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt2701-vdec.c
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Shunli Wang <shunli.wang@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt2701-clk.h>
+
+static const struct mtk_gate_regs vdec0_cg_regs = {
+	.set_ofs = 0x0000,
+	.clr_ofs = 0x0004,
+	.sta_ofs = 0x0000,
+};
+
+static const struct mtk_gate_regs vdec1_cg_regs = {
+	.set_ofs = 0x0008,
+	.clr_ofs = 0x000c,
+	.sta_ofs = 0x0008,
+};
+
+#define GATE_VDEC0(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &vdec0_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr_inv,	\
+	}
+
+#define GATE_VDEC1(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &vdec1_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr_inv,	\
+	}
+
+static const struct mtk_gate vdec_clks[] = {
+	GATE_VDEC0(CLK_VDEC_CKGEN, "vdec_cken", "vdec_sel", 0),
+	GATE_VDEC1(CLK_VDEC_LARB, "vdec_larb_cken", "mm_sel", 0),
+};
+
+static const struct of_device_id of_match_clk_mt2701_vdec[] = {
+	{ .compatible = "mediatek,mt2701-vdecsys", },
+	{}
+};
+
+static int clk_mt2701_vdec_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_VDEC_NR);
+
+	mtk_clk_register_gates(node, vdec_clks, ARRAY_SIZE(vdec_clks),
+						clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (r)
+		dev_err(&pdev->dev,
+			"could not register clock provider: %s: %d\n",
+			pdev->name, r);
+
+	return r;
+}
+
+static struct platform_driver clk_mt2701_vdec_drv = {
+	.probe = clk_mt2701_vdec_probe,
+	.driver = {
+		.name = "clk-mt2701-vdec",
+		.of_match_table = of_match_clk_mt2701_vdec,
+	},
+};
+
+builtin_platform_driver(clk_mt2701_vdec_drv);
diff --git a/drivers/clk/mediatek/clk-mt2701.c b/drivers/clk/mediatek/clk-mt2701.c
new file mode 100644
index 000000000000..6f26e6a37a6b
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt2701.c
@@ -0,0 +1,1035 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Shunli Wang <shunli.wang@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt2701-clk.h>
+
+/*
+ * For some clocks, we don't care what their actual rates are. And these
+ * clocks may change their rate on different products or different scenarios.
+ * So we model these clocks' rate as 0, to denote it's not an actual rate.
+ */
+#define DUMMY_RATE		0
+
+static DEFINE_SPINLOCK(mt2701_clk_lock);
+
+static const struct mtk_fixed_clk top_fixed_clks[] = {
+	FIXED_CLK(CLK_TOP_DPI, "dpi_ck", "clk26m",
+		108 * MHZ),
+	FIXED_CLK(CLK_TOP_DMPLL, "dmpll_ck", "clk26m",
+		400 * MHZ),
+	FIXED_CLK(CLK_TOP_VENCPLL, "vencpll_ck", "clk26m",
+		295750000),
+	FIXED_CLK(CLK_TOP_HDMI_0_PIX340M, "hdmi_0_pix340m", "clk26m",
+		340 * MHZ),
+	FIXED_CLK(CLK_TOP_HDMI_0_DEEP340M, "hdmi_0_deep340m", "clk26m",
+		340 * MHZ),
+	FIXED_CLK(CLK_TOP_HDMI_0_PLL340M, "hdmi_0_pll340m", "clk26m",
+		340 * MHZ),
+	FIXED_CLK(CLK_TOP_HDMITX_CLKDIG_CTS, "hdmitx_dig_cts", "clk26m",
+		300 * MHZ),
+	FIXED_CLK(CLK_TOP_HADDS2_FB, "hadds2_fbclk", "clk26m",
+		27 * MHZ),
+	FIXED_CLK(CLK_TOP_WBG_DIG_416M, "wbg_dig_ck_416m", "clk26m",
+		416 * MHZ),
+	FIXED_CLK(CLK_TOP_DSI0_LNTC_DSI, "dsi0_lntc_dsi", "clk26m",
+		143 * MHZ),
+	FIXED_CLK(CLK_TOP_HDMI_SCL_RX, "hdmi_scl_rx", "clk26m",
+		27 * MHZ),
+	FIXED_CLK(CLK_TOP_AUD_EXT1, "aud_ext1", "clk26m",
+		DUMMY_RATE),
+	FIXED_CLK(CLK_TOP_AUD_EXT2, "aud_ext2", "clk26m",
+		DUMMY_RATE),
+	FIXED_CLK(CLK_TOP_NFI1X_PAD, "nfi1x_pad", "clk26m",
+		DUMMY_RATE),
+};
+
+static const struct mtk_fixed_factor top_fixed_divs[] = {
+	FACTOR(CLK_TOP_SYSPLL, "syspll_ck", "mainpll", 1, 1),
+	FACTOR(CLK_TOP_SYSPLL_D2, "syspll_d2", "mainpll", 1, 2),
+	FACTOR(CLK_TOP_SYSPLL_D3, "syspll_d3", "mainpll", 1, 3),
+	FACTOR(CLK_TOP_SYSPLL_D5, "syspll_d5", "mainpll", 1, 5),
+	FACTOR(CLK_TOP_SYSPLL_D7, "syspll_d7", "mainpll", 1, 7),
+	FACTOR(CLK_TOP_SYSPLL1_D2, "syspll1_d2", "syspll_d2", 1, 2),
+	FACTOR(CLK_TOP_SYSPLL1_D4, "syspll1_d4", "syspll_d2", 1, 4),
+	FACTOR(CLK_TOP_SYSPLL1_D8, "syspll1_d8", "syspll_d2", 1, 8),
+	FACTOR(CLK_TOP_SYSPLL1_D16, "syspll1_d16", "syspll_d2", 1, 16),
+	FACTOR(CLK_TOP_SYSPLL2_D2, "syspll2_d2", "syspll_d3", 1, 2),
+	FACTOR(CLK_TOP_SYSPLL2_D4, "syspll2_d4", "syspll_d3", 1, 4),
+	FACTOR(CLK_TOP_SYSPLL2_D8, "syspll2_d8", "syspll_d3", 1, 8),
+	FACTOR(CLK_TOP_SYSPLL3_D2, "syspll3_d2", "syspll_d5", 1, 2),
+	FACTOR(CLK_TOP_SYSPLL3_D4, "syspll3_d4", "syspll_d5", 1, 4),
+	FACTOR(CLK_TOP_SYSPLL4_D2, "syspll4_d2", "syspll_d7", 1, 2),
+	FACTOR(CLK_TOP_SYSPLL4_D4, "syspll4_d4", "syspll_d7", 1, 4),
+
+	FACTOR(CLK_TOP_UNIVPLL, "univpll_ck", "univpll", 1, 1),
+	FACTOR(CLK_TOP_UNIVPLL_D2, "univpll_d2", "univpll", 1, 2),
+	FACTOR(CLK_TOP_UNIVPLL_D3, "univpll_d3", "univpll", 1, 3),
+	FACTOR(CLK_TOP_UNIVPLL_D5, "univpll_d5", "univpll", 1, 5),
+	FACTOR(CLK_TOP_UNIVPLL_D7, "univpll_d7", "univpll", 1, 7),
+	FACTOR(CLK_TOP_UNIVPLL_D26, "univpll_d26", "univpll", 1, 26),
+	FACTOR(CLK_TOP_UNIVPLL_D52, "univpll_d52", "univpll", 1, 52),
+	FACTOR(CLK_TOP_UNIVPLL_D108, "univpll_d108", "univpll", 1, 108),
+	FACTOR(CLK_TOP_USB_PHY48M, "usb_phy48m_ck", "univpll", 1, 26),
+	FACTOR(CLK_TOP_UNIVPLL1_D2, "univpll1_d2", "univpll_d2", 1, 2),
+	FACTOR(CLK_TOP_UNIVPLL1_D4, "univpll1_d4", "univpll_d2", 1, 4),
+	FACTOR(CLK_TOP_UNIVPLL1_D8, "univpll1_d8", "univpll_d2", 1, 8),
+	FACTOR(CLK_TOP_8BDAC, "8bdac_ck", "univpll_d2", 1, 1),
+	FACTOR(CLK_TOP_UNIVPLL2_D2, "univpll2_d2", "univpll_d3", 1, 2),
+	FACTOR(CLK_TOP_UNIVPLL2_D4, "univpll2_d4", "univpll_d3", 1, 4),
+	FACTOR(CLK_TOP_UNIVPLL2_D8, "univpll2_d8", "univpll_d3", 1, 8),
+	FACTOR(CLK_TOP_UNIVPLL2_D16, "univpll2_d16", "univpll_d3", 1, 16),
+	FACTOR(CLK_TOP_UNIVPLL2_D32, "univpll2_d32", "univpll_d3", 1, 32),
+	FACTOR(CLK_TOP_UNIVPLL3_D2, "univpll3_d2", "univpll_d5", 1, 2),
+	FACTOR(CLK_TOP_UNIVPLL3_D4, "univpll3_d4", "univpll_d5", 1, 4),
+	FACTOR(CLK_TOP_UNIVPLL3_D8, "univpll3_d8", "univpll_d5", 1, 8),
+
+	FACTOR(CLK_TOP_MSDCPLL, "msdcpll_ck", "msdcpll", 1, 1),
+	FACTOR(CLK_TOP_MSDCPLL_D2, "msdcpll_d2", "msdcpll", 1, 2),
+	FACTOR(CLK_TOP_MSDCPLL_D4, "msdcpll_d4", "msdcpll", 1, 4),
+	FACTOR(CLK_TOP_MSDCPLL_D8, "msdcpll_d8", "msdcpll", 1, 8),
+
+	FACTOR(CLK_TOP_MMPLL, "mmpll_ck", "mmpll", 1, 1),
+	FACTOR(CLK_TOP_MMPLL_D2, "mmpll_d2", "mmpll", 1, 2),
+
+	FACTOR(CLK_TOP_DMPLL_D2, "dmpll_d2", "dmpll_ck", 1, 2),
+	FACTOR(CLK_TOP_DMPLL_D4, "dmpll_d4", "dmpll_ck", 1, 4),
+	FACTOR(CLK_TOP_DMPLL_X2, "dmpll_x2", "dmpll_ck", 1, 1),
+
+	FACTOR(CLK_TOP_TVDPLL, "tvdpll_ck", "tvdpll", 1, 1),
+	FACTOR(CLK_TOP_TVDPLL_D2, "tvdpll_d2", "tvdpll", 1, 2),
+	FACTOR(CLK_TOP_TVDPLL_D4, "tvdpll_d4", "tvdpll", 1, 4),
+
+	FACTOR(CLK_TOP_VDECPLL, "vdecpll_ck", "vdecpll", 1, 1),
+	FACTOR(CLK_TOP_TVD2PLL, "tvd2pll_ck", "tvd2pll", 1, 1),
+	FACTOR(CLK_TOP_TVD2PLL_D2, "tvd2pll_d2", "tvd2pll", 1, 2),
+
+	FACTOR(CLK_TOP_MIPIPLL, "mipipll", "dpi_ck", 1, 1),
+	FACTOR(CLK_TOP_MIPIPLL_D2, "mipipll_d2", "dpi_ck", 1, 2),
+	FACTOR(CLK_TOP_MIPIPLL_D4, "mipipll_d4", "dpi_ck", 1, 4),
+
+	FACTOR(CLK_TOP_HDMIPLL, "hdmipll_ck", "hdmitx_dig_cts", 1, 1),
+	FACTOR(CLK_TOP_HDMIPLL_D2, "hdmipll_d2", "hdmitx_dig_cts", 1, 2),
+	FACTOR(CLK_TOP_HDMIPLL_D3, "hdmipll_d3", "hdmitx_dig_cts", 1, 3),
+
+	FACTOR(CLK_TOP_ARMPLL_1P3G, "armpll_1p3g_ck", "armpll", 1, 1),
+
+	FACTOR(CLK_TOP_AUDPLL, "audpll", "audpll_sel", 1, 1),
+	FACTOR(CLK_TOP_AUDPLL_D4, "audpll_d4", "audpll_sel", 1, 4),
+	FACTOR(CLK_TOP_AUDPLL_D8, "audpll_d8", "audpll_sel", 1, 8),
+	FACTOR(CLK_TOP_AUDPLL_D16, "audpll_d16", "audpll_sel", 1, 16),
+	FACTOR(CLK_TOP_AUDPLL_D24, "audpll_d24", "audpll_sel", 1, 24),
+
+	FACTOR(CLK_TOP_AUD1PLL_98M, "aud1pll_98m_ck", "aud1pll", 1, 3),
+	FACTOR(CLK_TOP_AUD2PLL_90M, "aud2pll_90m_ck", "aud2pll", 1, 3),
+	FACTOR(CLK_TOP_HADDS2PLL_98M, "hadds2pll_98m", "hadds2pll", 1, 3),
+	FACTOR(CLK_TOP_HADDS2PLL_294M, "hadds2pll_294m", "hadds2pll", 1, 1),
+	FACTOR(CLK_TOP_ETHPLL_500M, "ethpll_500m_ck", "ethpll", 1, 1),
+	FACTOR(CLK_TOP_CLK26M_D8, "clk26m_d8", "clk26m", 1, 8),
+	FACTOR(CLK_TOP_32K_INTERNAL, "32k_internal", "clk26m", 1, 793),
+	FACTOR(CLK_TOP_32K_EXTERNAL, "32k_external", "rtc32k", 1, 1),
+};
+
+static const char * const axi_parents[] = {
+	"clk26m",
+	"syspll1_d2",
+	"syspll_d5",
+	"syspll1_d4",
+	"univpll_d5",
+	"univpll2_d2",
+	"mmpll_d2",
+	"dmpll_d2"
+};
+
+static const char * const mem_parents[] = {
+	"clk26m",
+	"dmpll_ck"
+};
+
+static const char * const ddrphycfg_parents[] = {
+	"clk26m",
+	"syspll1_d8"
+};
+
+static const char * const mm_parents[] = {
+	"clk26m",
+	"vencpll_ck",
+	"syspll1_d2",
+	"syspll1_d4",
+	"univpll_d5",
+	"univpll1_d2",
+	"univpll2_d2",
+	"dmpll_ck"
+};
+
+static const char * const pwm_parents[] = {
+	"clk26m",
+	"univpll2_d4",
+	"univpll3_d2",
+	"univpll1_d4",
+};
+
+static const char * const vdec_parents[] = {
+	"clk26m",
+	"vdecpll_ck",
+	"syspll_d5",
+	"syspll1_d4",
+	"univpll_d5",
+	"univpll2_d2",
+	"vencpll_ck",
+	"msdcpll_d2",
+	"mmpll_d2"
+};
+
+static const char * const mfg_parents[] = {
+	"clk26m",
+	"mmpll_ck",
+	"dmpll_x2_ck",
+	"msdcpll_ck",
+	"clk26m",
+	"syspll_d3",
+	"univpll_d3",
+	"univpll1_d2"
+};
+
+static const char * const camtg_parents[] = {
+	"clk26m",
+	"univpll_d26",
+	"univpll2_d2",
+	"syspll3_d2",
+	"syspll3_d4",
+	"msdcpll_d2",
+	"mmpll_d2"
+};
+
+static const char * const uart_parents[] = {
+	"clk26m",
+	"univpll2_d8"
+};
+
+static const char * const spi_parents[] = {
+	"clk26m",
+	"syspll3_d2",
+	"syspll4_d2",
+	"univpll2_d4",
+	"univpll1_d8"
+};
+
+static const char * const usb20_parents[] = {
+	"clk26m",
+	"univpll1_d8",
+	"univpll3_d4"
+};
+
+static const char * const msdc30_parents[] = {
+	"clk26m",
+	"msdcpll_d2",
+	"syspll2_d2",
+	"syspll1_d4",
+	"univpll1_d4",
+	"univpll2_d4"
+};
+
+static const char * const audio_parents[] = {
+	"clk26m",
+	"syspll1_d16"
+};
+
+static const char * const aud_intbus_parents[] = {
+	"clk26m",
+	"syspll1_d4",
+	"syspll3_d2",
+	"syspll4_d2",
+	"univpll3_d2",
+	"univpll2_d4"
+};
+
+static const char * const pmicspi_parents[] = {
+	"clk26m",
+	"syspll1_d8",
+	"syspll2_d4",
+	"syspll4_d2",
+	"syspll3_d4",
+	"syspll2_d8",
+	"syspll1_d16",
+	"univpll3_d4",
+	"univpll_d26",
+	"dmpll_d2",
+	"dmpll_d4"
+};
+
+static const char * const scp_parents[] = {
+	"clk26m",
+	"syspll1_d8",
+	"dmpll_d2",
+	"dmpll_d4"
+};
+
+static const char * const dpi0_parents[] = {
+	"clk26m",
+	"mipipll",
+	"mipipll_d2",
+	"mipipll_d4",
+	"clk26m",
+	"tvdpll_ck",
+	"tvdpll_d2",
+	"tvdpll_d4"
+};
+
+static const char * const dpi1_parents[] = {
+	"clk26m",
+	"tvdpll_ck",
+	"tvdpll_d2",
+	"tvdpll_d4"
+};
+
+static const char * const tve_parents[] = {
+	"clk26m",
+	"mipipll",
+	"mipipll_d2",
+	"mipipll_d4",
+	"clk26m",
+	"tvdpll_ck",
+	"tvdpll_d2",
+	"tvdpll_d4"
+};
+
+static const char * const hdmi_parents[] = {
+	"clk26m",
+	"hdmipll_ck",
+	"hdmipll_d2",
+	"hdmipll_d3"
+};
+
+static const char * const apll_parents[] = {
+	"clk26m",
+	"audpll",
+	"audpll_d4",
+	"audpll_d8",
+	"audpll_d16",
+	"audpll_d24",
+	"clk26m",
+	"clk26m"
+};
+
+static const char * const rtc_parents[] = {
+	"32k_internal",
+	"32k_external",
+	"clk26m",
+	"univpll3_d8"
+};
+
+static const char * const nfi2x_parents[] = {
+	"clk26m",
+	"syspll2_d2",
+	"syspll_d7",
+	"univpll3_d2",
+	"syspll2_d4",
+	"univpll3_d4",
+	"syspll4_d4",
+	"clk26m"
+};
+
+static const char * const emmc_hclk_parents[] = {
+	"clk26m",
+	"syspll1_d2",
+	"syspll1_d4",
+	"syspll2_d2"
+};
+
+static const char * const flash_parents[] = {
+	"clk26m_d8",
+	"clk26m",
+	"syspll2_d8",
+	"syspll3_d4",
+	"univpll3_d4",
+	"syspll4_d2",
+	"syspll2_d4",
+	"univpll2_d4"
+};
+
+static const char * const di_parents[] = {
+	"clk26m",
+	"tvd2pll_ck",
+	"tvd2pll_d2",
+	"clk26m"
+};
+
+static const char * const nr_osd_parents[] = {
+	"clk26m",
+	"vencpll_ck",
+	"syspll1_d2",
+	"syspll1_d4",
+	"univpll_d5",
+	"univpll1_d2",
+	"univpll2_d2",
+	"dmpll_ck"
+};
+
+static const char * const hdmirx_bist_parents[] = {
+	"clk26m",
+	"syspll_d3",
+	"clk26m",
+	"syspll1_d16",
+	"syspll4_d2",
+	"syspll1_d4",
+	"vencpll_ck",
+	"clk26m"
+};
+
+static const char * const intdir_parents[] = {
+	"clk26m",
+	"mmpll_ck",
+	"syspll_d2",
+	"univpll_d2"
+};
+
+static const char * const asm_parents[] = {
+	"clk26m",
+	"univpll2_d4",
+	"univpll2_d2",
+	"syspll_d5"
+};
+
+static const char * const ms_card_parents[] = {
+	"clk26m",
+	"univpll3_d8",
+	"syspll4_d4"
+};
+
+static const char * const ethif_parents[] = {
+	"clk26m",
+	"syspll1_d2",
+	"syspll_d5",
+	"syspll1_d4",
+	"univpll_d5",
+	"univpll1_d2",
+	"dmpll_ck",
+	"dmpll_d2"
+};
+
+static const char * const hdmirx_parents[] = {
+	"clk26m",
+	"univpll_d52"
+};
+
+static const char * const cmsys_parents[] = {
+	"clk26m",
+	"syspll1_d2",
+	"univpll1_d2",
+	"univpll_d5",
+	"syspll_d5",
+	"syspll2_d2",
+	"syspll1_d4",
+	"syspll3_d2",
+	"syspll2_d4",
+	"syspll1_d8",
+	"clk26m",
+	"clk26m",
+	"clk26m",
+	"clk26m",
+	"clk26m"
+};
+
+static const char * const clk_8bdac_parents[] = {
+	"32k_internal",
+	"8bdac_ck",
+	"clk26m",
+	"clk26m"
+};
+
+static const char * const aud2dvd_parents[] = {
+	"a1sys_hp_ck",
+	"a2sys_hp_ck"
+};
+
+static const char * const padmclk_parents[] = {
+	"clk26m",
+	"univpll_d26",
+	"univpll_d52",
+	"univpll_d108",
+	"univpll2_d8",
+	"univpll2_d16",
+	"univpll2_d32"
+};
+
+static const char * const aud_mux_parents[] = {
+	"clk26m",
+	"aud1pll_98m_ck",
+	"aud2pll_90m_ck",
+	"hadds2pll_98m",
+	"audio_ext1_ck",
+	"audio_ext2_ck"
+};
+
+static const char * const aud_src_parents[] = {
+	"aud_mux1_sel",
+	"aud_mux2_sel"
+};
+
+static const char * const cpu_parents[] = {
+	"clk26m",
+	"armpll",
+	"mainpll",
+	"mmpll"
+};
+
+static const struct mtk_composite top_muxes[] = {
+	MUX_GATE_FLAGS(CLK_TOP_AXI_SEL, "axi_sel", axi_parents,
+		0x0040, 0, 3, 7, CLK_IS_CRITICAL),
+	MUX_GATE_FLAGS(CLK_TOP_MEM_SEL, "mem_sel", mem_parents,
+		0x0040, 8, 1, 15, CLK_IS_CRITICAL),
+	MUX_GATE_FLAGS(CLK_TOP_DDRPHYCFG_SEL, "ddrphycfg_sel",
+		ddrphycfg_parents, 0x0040, 16, 1, 23, CLK_IS_CRITICAL),
+	MUX_GATE(CLK_TOP_MM_SEL, "mm_sel", mm_parents,
+		0x0040, 24, 3, 31),
+
+	MUX_GATE(CLK_TOP_PWM_SEL, "pwm_sel", pwm_parents,
+		0x0050, 0, 2, 7),
+	MUX_GATE(CLK_TOP_VDEC_SEL, "vdec_sel", vdec_parents,
+		0x0050, 8, 4, 15),
+	MUX_GATE(CLK_TOP_MFG_SEL, "mfg_sel", mfg_parents,
+		0x0050, 16, 3, 23),
+	MUX_GATE(CLK_TOP_CAMTG_SEL, "camtg_sel", camtg_parents,
+		0x0050, 24, 3, 31),
+	MUX_GATE(CLK_TOP_UART_SEL, "uart_sel", uart_parents,
+		0x0060, 0, 1, 7),
+
+	MUX_GATE(CLK_TOP_SPI0_SEL, "spi0_sel", spi_parents,
+		0x0060, 8, 3, 15),
+	MUX_GATE(CLK_TOP_USB20_SEL, "usb20_sel", usb20_parents,
+		0x0060, 16, 2, 23),
+	MUX_GATE(CLK_TOP_MSDC30_0_SEL, "msdc30_0_sel", msdc30_parents,
+		0x0060, 24, 3, 31),
+
+	MUX_GATE(CLK_TOP_MSDC30_1_SEL, "msdc30_1_sel", msdc30_parents,
+		0x0070, 0, 3, 7),
+	MUX_GATE(CLK_TOP_MSDC30_2_SEL, "msdc30_2_sel", msdc30_parents,
+		0x0070, 8, 3, 15),
+	MUX_GATE(CLK_TOP_AUDIO_SEL, "audio_sel", msdc30_parents,
+		0x0070, 16, 1, 23),
+	MUX_GATE(CLK_TOP_AUDINTBUS_SEL, "aud_intbus_sel", aud_intbus_parents,
+		0x0070, 24, 3, 31),
+
+	MUX_GATE(CLK_TOP_PMICSPI_SEL, "pmicspi_sel", pmicspi_parents,
+		0x0080, 0, 4, 7),
+	MUX_GATE(CLK_TOP_SCP_SEL, "scp_sel", scp_parents,
+		0x0080, 8, 2, 15),
+	MUX_GATE(CLK_TOP_DPI0_SEL, "dpi0_sel", dpi0_parents,
+		0x0080, 16, 3, 23),
+	MUX_GATE(CLK_TOP_DPI1_SEL, "dpi1_sel", dpi1_parents,
+		0x0080, 24, 2, 31),
+
+	MUX_GATE(CLK_TOP_TVE_SEL, "tve_sel", tve_parents,
+		0x0090, 0, 3, 7),
+	MUX_GATE(CLK_TOP_HDMI_SEL, "hdmi_sel", hdmi_parents,
+		0x0090, 8, 2, 15),
+	MUX_GATE(CLK_TOP_APLL_SEL, "apll_sel", apll_parents,
+		0x0090, 16, 3, 23),
+
+	MUX_GATE_FLAGS(CLK_TOP_RTC_SEL, "rtc_sel", rtc_parents,
+		0x00A0, 0, 2, 7, CLK_IS_CRITICAL),
+	MUX_GATE(CLK_TOP_NFI2X_SEL, "nfi2x_sel", nfi2x_parents,
+		0x00A0, 8, 3, 15),
+	MUX_GATE(CLK_TOP_EMMC_HCLK_SEL, "emmc_hclk_sel", emmc_hclk_parents,
+		0x00A0, 24, 2, 31),
+
+	MUX_GATE(CLK_TOP_FLASH_SEL, "flash_sel", flash_parents,
+		0x00B0, 0, 3, 7),
+	MUX_GATE(CLK_TOP_DI_SEL, "di_sel", di_parents,
+		0x00B0, 8, 2, 15),
+	MUX_GATE(CLK_TOP_NR_SEL, "nr_sel", nr_osd_parents,
+		0x00B0, 16, 3, 23),
+	MUX_GATE(CLK_TOP_OSD_SEL, "osd_sel", nr_osd_parents,
+		0x00B0, 24, 3, 31),
+
+	MUX_GATE(CLK_TOP_HDMIRX_BIST_SEL, "hdmirx_bist_sel",
+		hdmirx_bist_parents, 0x00C0, 0, 3, 7),
+	MUX_GATE(CLK_TOP_INTDIR_SEL, "intdir_sel", intdir_parents,
+		0x00C0, 8, 2, 15),
+	MUX_GATE(CLK_TOP_ASM_I_SEL, "asm_i_sel", asm_parents,
+		0x00C0, 16, 2, 23),
+	MUX_GATE(CLK_TOP_ASM_M_SEL, "asm_m_sel", asm_parents,
+		0x00C0, 24, 3, 31),
+
+	MUX_GATE(CLK_TOP_ASM_H_SEL, "asm_h_sel", asm_parents,
+		0x00D0, 0, 2, 7),
+	MUX_GATE(CLK_TOP_MS_CARD_SEL, "ms_card_sel", ms_card_parents,
+		0x00D0, 16, 2, 23),
+	MUX_GATE(CLK_TOP_ETHIF_SEL, "ethif_sel", ethif_parents,
+		0x00D0, 24, 3, 31),
+
+	MUX_GATE(CLK_TOP_HDMIRX26_24_SEL, "hdmirx26_24_sel", hdmirx_parents,
+		0x00E0, 0, 1, 7),
+	MUX_GATE(CLK_TOP_MSDC30_3_SEL, "msdc30_3_sel", msdc30_parents,
+		0x00E0, 8, 3, 15),
+	MUX_GATE(CLK_TOP_CMSYS_SEL, "cmsys_sel", cmsys_parents,
+		0x00E0, 16, 4, 23),
+
+	MUX_GATE(CLK_TOP_SPI1_SEL, "spi2_sel", spi_parents,
+		0x00E0, 24, 3, 31),
+	MUX_GATE(CLK_TOP_SPI2_SEL, "spi1_sel", spi_parents,
+		0x00F0, 0, 3, 7),
+	MUX_GATE(CLK_TOP_8BDAC_SEL, "8bdac_sel", clk_8bdac_parents,
+		0x00F0, 8, 2, 15),
+	MUX_GATE(CLK_TOP_AUD2DVD_SEL, "aud2dvd_sel", aud2dvd_parents,
+		0x00F0, 16, 1, 23),
+
+	MUX(CLK_TOP_PADMCLK_SEL, "padmclk_sel", padmclk_parents,
+		0x0100, 0, 3),
+
+	MUX(CLK_TOP_AUD_MUX1_SEL, "aud_mux1_sel", aud_mux_parents,
+		0x012c, 0, 3),
+	MUX(CLK_TOP_AUD_MUX2_SEL, "aud_mux2_sel", aud_mux_parents,
+		0x012c, 3, 3),
+	MUX(CLK_TOP_AUDPLL_MUX_SEL, "audpll_sel", aud_mux_parents,
+		0x012c, 6, 3),
+	MUX_GATE(CLK_TOP_AUD_K1_SRC_SEL, "aud_k1_src_sel", aud_src_parents,
+		0x012c, 15, 1, 23),
+	MUX_GATE(CLK_TOP_AUD_K2_SRC_SEL, "aud_k2_src_sel", aud_src_parents,
+		0x012c, 16, 1, 24),
+	MUX_GATE(CLK_TOP_AUD_K3_SRC_SEL, "aud_k3_src_sel", aud_src_parents,
+		0x012c, 17, 1, 25),
+	MUX_GATE(CLK_TOP_AUD_K4_SRC_SEL, "aud_k4_src_sel", aud_src_parents,
+		0x012c, 18, 1, 26),
+	MUX_GATE(CLK_TOP_AUD_K5_SRC_SEL, "aud_k5_src_sel", aud_src_parents,
+		0x012c, 19, 1, 27),
+	MUX_GATE(CLK_TOP_AUD_K6_SRC_SEL, "aud_k6_src_sel", aud_src_parents,
+		0x012c, 20, 1, 28),
+};
+
+static const struct mtk_clk_divider top_adj_divs[] = {
+	DIV_ADJ(CLK_TOP_AUD_EXTCK1_DIV, "audio_ext1_ck", "aud_ext1",
+		0x0120, 0, 8),
+	DIV_ADJ(CLK_TOP_AUD_EXTCK2_DIV, "audio_ext2_ck", "aud_ext2",
+		0x0120, 8, 8),
+	DIV_ADJ(CLK_TOP_AUD_MUX1_DIV, "aud_mux1_div", "aud_mux1_sel",
+		0x0120, 16, 8),
+	DIV_ADJ(CLK_TOP_AUD_MUX2_DIV, "aud_mux2_div", "aud_mux2_sel",
+		0x0120, 24, 8),
+	DIV_ADJ(CLK_TOP_AUD_K1_SRC_DIV, "aud_k1_src_div", "aud_k1_src_sel",
+		0x0124, 0, 8),
+	DIV_ADJ(CLK_TOP_AUD_K2_SRC_DIV, "aud_k2_src_div", "aud_k2_src_sel",
+		0x0124, 8, 8),
+	DIV_ADJ(CLK_TOP_AUD_K3_SRC_DIV, "aud_k3_src_div", "aud_k3_src_sel",
+		0x0124, 16, 8),
+	DIV_ADJ(CLK_TOP_AUD_K4_SRC_DIV, "aud_k4_src_div", "aud_k4_src_sel",
+		0x0124, 24, 8),
+	DIV_ADJ(CLK_TOP_AUD_K5_SRC_DIV, "aud_k5_src_div", "aud_k5_src_sel",
+		0x0128, 0, 8),
+	DIV_ADJ(CLK_TOP_AUD_K6_SRC_DIV, "aud_k6_src_div", "aud_k6_src_sel",
+		0x0128, 8, 8),
+};
+
+static const struct mtk_gate_regs top_aud_cg_regs = {
+	.sta_ofs = 0x012C,
+};
+
+#define GATE_TOP_AUD(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &top_aud_cg_regs,		\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_no_setclr,	\
+	}
+
+static const struct mtk_gate top_clks[] = {
+	GATE_TOP_AUD(CLK_TOP_AUD_48K_TIMING, "a1sys_hp_ck", "aud_mux1_div",
+		21),
+	GATE_TOP_AUD(CLK_TOP_AUD_44K_TIMING, "a2sys_hp_ck", "aud_mux2_div",
+		22),
+	GATE_TOP_AUD(CLK_TOP_AUD_I2S1_MCLK, "aud_i2s1_mclk", "aud_k1_src_div",
+		23),
+	GATE_TOP_AUD(CLK_TOP_AUD_I2S2_MCLK, "aud_i2s2_mclk", "aud_k2_src_div",
+		24),
+	GATE_TOP_AUD(CLK_TOP_AUD_I2S3_MCLK, "aud_i2s3_mclk", "aud_k3_src_div",
+		25),
+	GATE_TOP_AUD(CLK_TOP_AUD_I2S4_MCLK, "aud_i2s4_mclk", "aud_k4_src_div",
+		26),
+	GATE_TOP_AUD(CLK_TOP_AUD_I2S5_MCLK, "aud_i2s5_mclk", "aud_k5_src_div",
+		27),
+	GATE_TOP_AUD(CLK_TOP_AUD_I2S6_MCLK, "aud_i2s6_mclk", "aud_k6_src_div",
+		28),
+};
+
+static int mtk_topckgen_init(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	void __iomem *base;
+	struct device_node *node = pdev->dev.of_node;
+	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	clk_data = mtk_alloc_clk_data(CLK_TOP_NR);
+
+	mtk_clk_register_fixed_clks(top_fixed_clks, ARRAY_SIZE(top_fixed_clks),
+								clk_data);
+
+	mtk_clk_register_factors(top_fixed_divs, ARRAY_SIZE(top_fixed_divs),
+								clk_data);
+
+	mtk_clk_register_composites(top_muxes, ARRAY_SIZE(top_muxes),
+				base, &mt2701_clk_lock, clk_data);
+
+	mtk_clk_register_dividers(top_adj_divs, ARRAY_SIZE(top_adj_divs),
+				base, &mt2701_clk_lock, clk_data);
+
+	mtk_clk_register_gates(node, top_clks, ARRAY_SIZE(top_clks),
+						clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct mtk_gate_regs infra_cg_regs = {
+	.set_ofs = 0x0040,
+	.clr_ofs = 0x0044,
+	.sta_ofs = 0x0048,
+};
+
+#define GATE_ICG(_id, _name, _parent, _shift) {		\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &infra_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+static const struct mtk_gate infra_clks[] = {
+	GATE_ICG(CLK_INFRA_DBG, "dbgclk", "axi_sel", 0),
+	GATE_ICG(CLK_INFRA_SMI, "smi_ck", "mm_sel", 1),
+	GATE_ICG(CLK_INFRA_QAXI_CM4, "cm4_ck", "axi_sel", 2),
+	GATE_ICG(CLK_INFRA_AUD_SPLIN_B, "audio_splin_bck", "hadds2pll_294m", 4),
+	GATE_ICG(CLK_INFRA_AUDIO, "audio_ck", "clk26m", 5),
+	GATE_ICG(CLK_INFRA_EFUSE, "efuse_ck", "clk26m", 6),
+	GATE_ICG(CLK_INFRA_L2C_SRAM, "l2c_sram_ck", "mm_sel", 7),
+	GATE_ICG(CLK_INFRA_M4U, "m4u_ck", "mem_sel", 8),
+	GATE_ICG(CLK_INFRA_CONNMCU, "connsys_bus", "wbg_dig_ck_416m", 12),
+	GATE_ICG(CLK_INFRA_TRNG, "trng_ck", "axi_sel", 13),
+	GATE_ICG(CLK_INFRA_RAMBUFIF, "rambufif_ck", "mem_sel", 14),
+	GATE_ICG(CLK_INFRA_CPUM, "cpum_ck", "mem_sel", 15),
+	GATE_ICG(CLK_INFRA_KP, "kp_ck", "axi_sel", 16),
+	GATE_ICG(CLK_INFRA_CEC, "cec_ck", "rtc_sel", 18),
+	GATE_ICG(CLK_INFRA_IRRX, "irrx_ck", "axi_sel", 19),
+	GATE_ICG(CLK_INFRA_PMICSPI, "pmicspi_ck", "pmicspi_sel", 22),
+	GATE_ICG(CLK_INFRA_PMICWRAP, "pmicwrap_ck", "axi_sel", 23),
+	GATE_ICG(CLK_INFRA_DDCCI, "ddcci_ck", "axi_sel", 24),
+};
+
+static const struct mtk_fixed_factor infra_fixed_divs[] = {
+	FACTOR(CLK_INFRA_CLK_13M, "clk13m", "clk26m", 1, 2),
+};
+
+static struct clk_onecell_data *infra_clk_data;
+
+static void mtk_infrasys_init_early(struct device_node *node)
+{
+	int r, i;
+
+	if (!infra_clk_data) {
+		infra_clk_data = mtk_alloc_clk_data(CLK_INFRA_NR);
+
+		for (i = 0; i < CLK_INFRA_NR; i++)
+			infra_clk_data->clks[i] = ERR_PTR(-EPROBE_DEFER);
+	}
+
+	mtk_clk_register_factors(infra_fixed_divs, ARRAY_SIZE(infra_fixed_divs),
+						infra_clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, infra_clk_data);
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+			__func__, r);
+}
+CLK_OF_DECLARE_DRIVER(mtk_infra, "mediatek,mt2701-infracfg",
+			mtk_infrasys_init_early);
+
+static int mtk_infrasys_init(struct platform_device *pdev)
+{
+	int r, i;
+	struct device_node *node = pdev->dev.of_node;
+
+	if (!infra_clk_data) {
+		infra_clk_data = mtk_alloc_clk_data(CLK_INFRA_NR);
+	} else {
+		for (i = 0; i < CLK_INFRA_NR; i++) {
+			if (infra_clk_data->clks[i] == ERR_PTR(-EPROBE_DEFER))
+				infra_clk_data->clks[i] = ERR_PTR(-ENOENT);
+		}
+	}
+
+	mtk_clk_register_gates(node, infra_clks, ARRAY_SIZE(infra_clks),
+						infra_clk_data);
+	mtk_clk_register_factors(infra_fixed_divs, ARRAY_SIZE(infra_fixed_divs),
+						infra_clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, infra_clk_data);
+	if (r)
+		return r;
+
+	mtk_register_reset_controller(node, 2, 0x30);
+
+	return 0;
+}
+
+static const struct mtk_gate_regs peri0_cg_regs = {
+	.set_ofs = 0x0008,
+	.clr_ofs = 0x0010,
+	.sta_ofs = 0x0018,
+};
+
+static const struct mtk_gate_regs peri1_cg_regs = {
+	.set_ofs = 0x000c,
+	.clr_ofs = 0x0014,
+	.sta_ofs = 0x001c,
+};
+
+#define GATE_PERI0(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &peri0_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+#define GATE_PERI1(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &peri1_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+static const struct mtk_gate peri_clks[] = {
+	GATE_PERI0(CLK_PERI_USB0_MCU, "usb0_mcu_ck", "axi_sel", 31),
+	GATE_PERI0(CLK_PERI_ETH, "eth_ck", "clk26m", 30),
+	GATE_PERI0(CLK_PERI_SPI0, "spi0_ck", "spi0_sel", 29),
+	GATE_PERI0(CLK_PERI_AUXADC, "auxadc_ck", "clk26m", 28),
+	GATE_PERI0(CLK_PERI_I2C3, "i2c3_ck", "clk26m", 27),
+	GATE_PERI0(CLK_PERI_I2C2, "i2c2_ck", "axi_sel", 26),
+	GATE_PERI0(CLK_PERI_I2C1, "i2c1_ck", "axi_sel", 25),
+	GATE_PERI0(CLK_PERI_I2C0, "i2c0_ck", "axi_sel", 24),
+	GATE_PERI0(CLK_PERI_BTIF, "bitif_ck", "axi_sel", 23),
+	GATE_PERI0(CLK_PERI_UART3, "uart3_ck", "axi_sel", 22),
+	GATE_PERI0(CLK_PERI_UART2, "uart2_ck", "axi_sel", 21),
+	GATE_PERI0(CLK_PERI_UART1, "uart1_ck", "axi_sel", 20),
+	GATE_PERI0(CLK_PERI_UART0, "uart0_ck", "axi_sel", 19),
+	GATE_PERI0(CLK_PERI_NLI, "nli_ck", "axi_sel", 18),
+	GATE_PERI0(CLK_PERI_MSDC50_3, "msdc50_3_ck", "emmc_hclk_sel", 17),
+	GATE_PERI0(CLK_PERI_MSDC30_3, "msdc30_3_ck", "msdc30_3_sel", 16),
+	GATE_PERI0(CLK_PERI_MSDC30_2, "msdc30_2_ck", "msdc30_2_sel", 15),
+	GATE_PERI0(CLK_PERI_MSDC30_1, "msdc30_1_ck", "msdc30_1_sel", 14),
+	GATE_PERI0(CLK_PERI_MSDC30_0, "msdc30_0_ck", "msdc30_0_sel", 13),
+	GATE_PERI0(CLK_PERI_AP_DMA, "ap_dma_ck", "axi_sel", 12),
+	GATE_PERI0(CLK_PERI_USB1, "usb1_ck", "usb20_sel", 11),
+	GATE_PERI0(CLK_PERI_USB0, "usb0_ck", "usb20_sel", 10),
+	GATE_PERI0(CLK_PERI_PWM, "pwm_ck", "axi_sel", 9),
+	GATE_PERI0(CLK_PERI_PWM7, "pwm7_ck", "axi_sel", 8),
+	GATE_PERI0(CLK_PERI_PWM6, "pwm6_ck", "axi_sel", 7),
+	GATE_PERI0(CLK_PERI_PWM5, "pwm5_ck", "axi_sel", 6),
+	GATE_PERI0(CLK_PERI_PWM4, "pwm4_ck", "axi_sel", 5),
+	GATE_PERI0(CLK_PERI_PWM3, "pwm3_ck", "axi_sel", 4),
+	GATE_PERI0(CLK_PERI_PWM2, "pwm2_ck", "axi_sel", 3),
+	GATE_PERI0(CLK_PERI_PWM1, "pwm1_ck", "axi_sel", 2),
+	GATE_PERI0(CLK_PERI_THERM, "therm_ck", "axi_sel", 1),
+	GATE_PERI0(CLK_PERI_NFI, "nfi_ck", "nfi2x_sel", 0),
+
+	GATE_PERI1(CLK_PERI_FCI, "fci_ck", "ms_card_sel", 11),
+	GATE_PERI1(CLK_PERI_SPI2, "spi2_ck", "spi2_sel", 10),
+	GATE_PERI1(CLK_PERI_SPI1, "spi1_ck", "spi1_sel", 9),
+	GATE_PERI1(CLK_PERI_HOST89_DVD, "host89_dvd_ck", "aud2dvd_sel", 8),
+	GATE_PERI1(CLK_PERI_HOST89_SPI, "host89_spi_ck", "spi0_sel", 7),
+	GATE_PERI1(CLK_PERI_HOST89_INT, "host89_int_ck", "axi_sel", 6),
+	GATE_PERI1(CLK_PERI_FLASH, "flash_ck", "nfi2x_sel", 5),
+	GATE_PERI1(CLK_PERI_NFI_PAD, "nfi_pad_ck", "nfi1x_pad", 4),
+	GATE_PERI1(CLK_PERI_NFI_ECC, "nfi_ecc_ck", "nfi1x_pad", 3),
+	GATE_PERI1(CLK_PERI_GCPU, "gcpu_ck", "axi_sel", 2),
+	GATE_PERI1(CLK_PERI_USB_SLV, "usbslv_ck", "axi_sel", 1),
+	GATE_PERI1(CLK_PERI_USB1_MCU, "usb1_mcu_ck", "axi_sel", 0),
+};
+
+static const char * const uart_ck_sel_parents[] = {
+	"clk26m",
+	"uart_sel",
+};
+
+static const struct mtk_composite peri_muxs[] = {
+	MUX(CLK_PERI_UART0_SEL, "uart0_ck_sel", uart_ck_sel_parents,
+		0x40c, 0, 1),
+	MUX(CLK_PERI_UART1_SEL, "uart1_ck_sel", uart_ck_sel_parents,
+		0x40c, 1, 1),
+	MUX(CLK_PERI_UART2_SEL, "uart2_ck_sel", uart_ck_sel_parents,
+		0x40c, 2, 1),
+	MUX(CLK_PERI_UART3_SEL, "uart3_ck_sel", uart_ck_sel_parents,
+		0x40c, 3, 1),
+};
+
+static int mtk_pericfg_init(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	void __iomem *base;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	clk_data = mtk_alloc_clk_data(CLK_PERI_NR);
+
+	mtk_clk_register_gates(node, peri_clks, ARRAY_SIZE(peri_clks),
+						clk_data);
+
+	mtk_clk_register_composites(peri_muxs, ARRAY_SIZE(peri_muxs), base,
+			&mt2701_clk_lock, clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (r)
+		return r;
+
+	mtk_register_reset_controller(node, 2, 0x0);
+
+	return 0;
+}
+
+#define MT8590_PLL_FMAX		(2000 * MHZ)
+#define CON0_MT8590_RST_BAR	BIT(27)
+
+#define PLL(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits, _pd_reg, \
+			_pd_shift, _tuner_reg, _pcw_reg, _pcw_shift) {	\
+		.id = _id,						\
+		.name = _name,						\
+		.reg = _reg,						\
+		.pwr_reg = _pwr_reg,					\
+		.en_mask = _en_mask,					\
+		.flags = _flags,					\
+		.rst_bar_mask = CON0_MT8590_RST_BAR,			\
+		.fmax = MT8590_PLL_FMAX,				\
+		.pcwbits = _pcwbits,					\
+		.pd_reg = _pd_reg,					\
+		.pd_shift = _pd_shift,					\
+		.tuner_reg = _tuner_reg,				\
+		.pcw_reg = _pcw_reg,					\
+		.pcw_shift = _pcw_shift,				\
+	}
+
+static const struct mtk_pll_data apmixed_plls[] = {
+	PLL(CLK_APMIXED_ARMPLL, "armpll", 0x200, 0x20c, 0x80000001,
+			PLL_AO, 21, 0x204, 24, 0x0, 0x204, 0),
+	PLL(CLK_APMIXED_MAINPLL, "mainpll", 0x210, 0x21c, 0xf0000001,
+		  HAVE_RST_BAR, 21, 0x210, 4, 0x0, 0x214, 0),
+	PLL(CLK_APMIXED_UNIVPLL, "univpll", 0x220, 0x22c, 0xf3000001,
+		  HAVE_RST_BAR, 7, 0x220, 4, 0x0, 0x224, 14),
+	PLL(CLK_APMIXED_MMPLL, "mmpll", 0x230, 0x23c, 0x00000001, 0,
+				21, 0x230, 4, 0x0, 0x234, 0),
+	PLL(CLK_APMIXED_MSDCPLL, "msdcpll", 0x240, 0x24c, 0x00000001, 0,
+				21, 0x240, 4, 0x0, 0x244, 0),
+	PLL(CLK_APMIXED_TVDPLL, "tvdpll", 0x250, 0x25c, 0x00000001, 0,
+				21, 0x250, 4, 0x0, 0x254, 0),
+	PLL(CLK_APMIXED_AUD1PLL, "aud1pll", 0x270, 0x27c, 0x00000001, 0,
+				31, 0x270, 4, 0x0, 0x274, 0),
+	PLL(CLK_APMIXED_TRGPLL, "trgpll", 0x280, 0x28c, 0x00000001, 0,
+				31, 0x280, 4, 0x0, 0x284, 0),
+	PLL(CLK_APMIXED_ETHPLL, "ethpll", 0x290, 0x29c, 0x00000001, 0,
+				31, 0x290, 4, 0x0, 0x294, 0),
+	PLL(CLK_APMIXED_VDECPLL, "vdecpll", 0x2a0, 0x2ac, 0x00000001, 0,
+				31, 0x2a0, 4, 0x0, 0x2a4, 0),
+	PLL(CLK_APMIXED_HADDS2PLL, "hadds2pll", 0x2b0, 0x2bc, 0x00000001, 0,
+				31, 0x2b0, 4, 0x0, 0x2b4, 0),
+	PLL(CLK_APMIXED_AUD2PLL, "aud2pll", 0x2c0, 0x2cc, 0x00000001, 0,
+				31, 0x2c0, 4, 0x0, 0x2c4, 0),
+	PLL(CLK_APMIXED_TVD2PLL, "tvd2pll", 0x2d0, 0x2dc, 0x00000001, 0,
+				21, 0x2d0, 4, 0x0, 0x2d4, 0),
+};
+
+static int mtk_apmixedsys_init(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_APMIXED_NR);
+	if (!clk_data)
+		return -ENOMEM;
+
+	mtk_clk_register_plls(node, apmixed_plls, ARRAY_SIZE(apmixed_plls),
+								clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct of_device_id of_match_clk_mt2701[] = {
+	{
+		.compatible = "mediatek,mt2701-topckgen",
+		.data = mtk_topckgen_init,
+	}, {
+		.compatible = "mediatek,mt2701-infracfg",
+		.data = mtk_infrasys_init,
+	}, {
+		.compatible = "mediatek,mt2701-pericfg",
+		.data = mtk_pericfg_init,
+	}, {
+		.compatible = "mediatek,mt2701-apmixedsys",
+		.data = mtk_apmixedsys_init,
+	}, {
+		/* sentinel */
+	}
+};
+
+static int clk_mt2701_probe(struct platform_device *pdev)
+{
+	int (*clk_init)(struct platform_device *);
+	int r;
+
+	clk_init = of_device_get_match_data(&pdev->dev);
+	if (!clk_init)
+		return -EINVAL;
+
+	r = clk_init(pdev);
+	if (r)
+		dev_err(&pdev->dev,
+			"could not register clock provider: %s: %d\n",
+			pdev->name, r);
+
+	return r;
+}
+
+static struct platform_driver clk_mt2701_drv = {
+	.probe = clk_mt2701_probe,
+	.driver = {
+		.name = "clk-mt2701",
+		.of_match_table = of_match_clk_mt2701,
+	},
+};
+
+static int __init clk_mt2701_init(void)
+{
+	return platform_driver_register(&clk_mt2701_drv);
+}
+
+arch_initcall(clk_mt2701_init);
diff --git a/drivers/clk/mediatek/clk-mtk.c b/drivers/clk/mediatek/clk-mtk.c
index bb30f7063569..0541df78141c 100644
--- a/drivers/clk/mediatek/clk-mtk.c
+++ b/drivers/clk/mediatek/clk-mtk.c
@@ -58,6 +58,9 @@ void mtk_clk_register_fixed_clks(const struct mtk_fixed_clk *clks,
 	for (i = 0; i < num; i++) {
 		const struct mtk_fixed_clk *rc = &clks[i];
 
+		if (clk_data && !IS_ERR_OR_NULL(clk_data->clks[rc->id]))
+			continue;
+
 		clk = clk_register_fixed_rate(NULL, rc->name, rc->parent, 0,
 					      rc->rate);
 
@@ -81,6 +84,9 @@ void mtk_clk_register_factors(const struct mtk_fixed_factor *clks,
 	for (i = 0; i < num; i++) {
 		const struct mtk_fixed_factor *ff = &clks[i];
 
+		if (clk_data && !IS_ERR_OR_NULL(clk_data->clks[ff->id]))
+			continue;
+
 		clk = clk_register_fixed_factor(NULL, ff->name, ff->parent_name,
 				CLK_SET_RATE_PARENT, ff->mult, ff->div);
 
@@ -116,6 +122,9 @@ int mtk_clk_register_gates(struct device_node *node,
 	for (i = 0; i < num; i++) {
 		const struct mtk_gate *gate = &clks[i];
 
+		if (!IS_ERR_OR_NULL(clk_data->clks[gate->id]))
+			continue;
+
 		clk = mtk_clk_register_gate(gate->name, gate->parent_name,
 				regmap,
 				gate->regs->set_ofs,
@@ -232,6 +241,9 @@ void mtk_clk_register_composites(const struct mtk_composite *mcs,
 	for (i = 0; i < num; i++) {
 		const struct mtk_composite *mc = &mcs[i];
 
+		if (clk_data && !IS_ERR_OR_NULL(clk_data->clks[mc->id]))
+			continue;
+
 		clk = mtk_clk_register_composite(mc, base, lock);
 
 		if (IS_ERR(clk)) {
@@ -244,3 +256,31 @@ void mtk_clk_register_composites(const struct mtk_composite *mcs,
 			clk_data->clks[mc->id] = clk;
 	}
 }
+
+void mtk_clk_register_dividers(const struct mtk_clk_divider *mcds,
+			int num, void __iomem *base, spinlock_t *lock,
+				struct clk_onecell_data *clk_data)
+{
+	struct clk *clk;
+	int i;
+
+	for (i = 0; i <  num; i++) {
+		const struct mtk_clk_divider *mcd = &mcds[i];
+
+		if (clk_data && !IS_ERR_OR_NULL(clk_data->clks[mcd->id]))
+			continue;
+
+		clk = clk_register_divider(NULL, mcd->name, mcd->parent_name,
+			mcd->flags, base +  mcd->div_reg, mcd->div_shift,
+			mcd->div_width, mcd->clk_divider_flags, lock);
+
+		if (IS_ERR(clk)) {
+			pr_err("Failed to register clk %s: %ld\n",
+				mcd->name, PTR_ERR(clk));
+			continue;
+		}
+
+		if (clk_data)
+			clk_data->clks[mcd->id] = clk;
+	}
+}
diff --git a/drivers/clk/mediatek/clk-mtk.h b/drivers/clk/mediatek/clk-mtk.h
index 9f24fcfa304f..f5d6b70ce189 100644
--- a/drivers/clk/mediatek/clk-mtk.h
+++ b/drivers/clk/mediatek/clk-mtk.h
@@ -87,7 +87,8 @@ struct mtk_composite {
  * In case the rate change propagation to parent clocks is undesirable,
  * this macro allows to specify the clock flags manually.
  */
-#define MUX_GATE_FLAGS(_id, _name, _parents, _reg, _shift, _width, _gate, _flags) {	\
+#define MUX_GATE_FLAGS(_id, _name, _parents, _reg, _shift, _width,	\
+			_gate, _flags) {				\
 		.id = _id,						\
 		.name = _name,						\
 		.mux_reg = _reg,					\
@@ -106,7 +107,8 @@ struct mtk_composite {
  * parent clock by default.
  */
 #define MUX_GATE(_id, _name, _parents, _reg, _shift, _width, _gate)	\
-	MUX_GATE_FLAGS(_id, _name, _parents, _reg, _shift, _width, _gate, CLK_SET_RATE_PARENT)
+	MUX_GATE_FLAGS(_id, _name, _parents, _reg, _shift, _width,	\
+		_gate, CLK_SET_RATE_PARENT)
 
 #define MUX(_id, _name, _parents, _reg, _shift, _width) {		\
 		.id = _id,						\
@@ -121,7 +123,8 @@ struct mtk_composite {
 		.flags = CLK_SET_RATE_PARENT,				\
 	}
 
-#define DIV_GATE(_id, _name, _parent, _gate_reg, _gate_shift, _div_reg, _div_width, _div_shift) {	\
+#define DIV_GATE(_id, _name, _parent, _gate_reg, _gate_shift, _div_reg,	\
+					_div_width, _div_shift) {	\
 		.id = _id,						\
 		.parent = _parent,					\
 		.name = _name,						\
@@ -156,12 +159,40 @@ struct mtk_gate {
 	const struct clk_ops *ops;
 };
 
-int mtk_clk_register_gates(struct device_node *node, const struct mtk_gate *clks,
-		int num, struct clk_onecell_data *clk_data);
+int mtk_clk_register_gates(struct device_node *node,
+			const struct mtk_gate *clks, int num,
+			struct clk_onecell_data *clk_data);
+
+struct mtk_clk_divider {
+	int id;
+	const char *name;
+	const char *parent_name;
+	unsigned long flags;
+
+	u32 div_reg;
+	unsigned char div_shift;
+	unsigned char div_width;
+	unsigned char clk_divider_flags;
+	const struct clk_div_table *clk_div_table;
+};
+
+#define DIV_ADJ(_id, _name, _parent, _reg, _shift, _width) {	\
+		.id = _id,					\
+		.name = _name,					\
+		.parent_name = _parent,				\
+		.div_reg = _reg,				\
+		.div_shift = _shift,				\
+		.div_width = _width,				\
+}
+
+void mtk_clk_register_dividers(const struct mtk_clk_divider *mcds,
+			int num, void __iomem *base, spinlock_t *lock,
+				struct clk_onecell_data *clk_data);
 
 struct clk_onecell_data *mtk_alloc_clk_data(unsigned int clk_num);
 
 #define HAVE_RST_BAR	BIT(0)
+#define PLL_AO		BIT(1)
 
 struct mtk_pll_div_table {
 	u32 div;
diff --git a/drivers/clk/mediatek/clk-pll.c b/drivers/clk/mediatek/clk-pll.c
index 0c2deac17ce9..a409142e9346 100644
--- a/drivers/clk/mediatek/clk-pll.c
+++ b/drivers/clk/mediatek/clk-pll.c
@@ -301,6 +301,7 @@ static struct clk *mtk_clk_register_pll(const struct mtk_pll_data *data,
 	pll->data = data;
 
 	init.name = data->name;
+	init.flags = (data->flags & PLL_AO) ? CLK_IS_CRITICAL : 0;
 	init.ops = &mtk_pll_ops;
 	init.parent_names = &parent_name;
 	init.num_parents = 1;
diff --git a/drivers/clk/mmp/clk-of-mmp2.c b/drivers/clk/mmp/clk-of-mmp2.c
index 9adaf48aea23..0fc75c395957 100644
--- a/drivers/clk/mmp/clk-of-mmp2.c
+++ b/drivers/clk/mmp/clk-of-mmp2.c
@@ -309,19 +309,19 @@ static void __init mmp2_clk_init(struct device_node *np)
 	pxa_unit->mpmu_base = of_iomap(np, 0);
 	if (!pxa_unit->mpmu_base) {
 		pr_err("failed to map mpmu registers\n");
-		return;
+		goto free_memory;
 	}
 
 	pxa_unit->apmu_base = of_iomap(np, 1);
 	if (!pxa_unit->apmu_base) {
 		pr_err("failed to map apmu registers\n");
-		return;
+		goto unmap_mpmu_region;
 	}
 
 	pxa_unit->apbc_base = of_iomap(np, 2);
 	if (!pxa_unit->apbc_base) {
 		pr_err("failed to map apbc registers\n");
-		return;
+		goto unmap_apmu_region;
 	}
 
 	mmp_clk_init(np, &pxa_unit->unit, MMP2_NR_CLKS);
@@ -333,6 +333,15 @@ static void __init mmp2_clk_init(struct device_node *np)
 	mmp2_axi_periph_clk_init(pxa_unit);
 
 	mmp2_clk_reset_init(np, pxa_unit);
+
+	return;
+
+unmap_apmu_region:
+	iounmap(pxa_unit->apmu_base);
+unmap_mpmu_region:
+	iounmap(pxa_unit->mpmu_base);
+free_memory:
+	kfree(pxa_unit);
 }
 
 CLK_OF_DECLARE(mmp2_clk, "marvell,mmp2-clock", mmp2_clk_init);
diff --git a/drivers/clk/mmp/clk-of-pxa1928.c b/drivers/clk/mmp/clk-of-pxa1928.c
index e478ff44e170..cede7b4ca3b9 100644
--- a/drivers/clk/mmp/clk-of-pxa1928.c
+++ b/drivers/clk/mmp/clk-of-pxa1928.c
@@ -216,6 +216,7 @@ static void __init pxa1928_mpmu_clk_init(struct device_node *np)
 	pxa_unit->mpmu_base = of_iomap(np, 0);
 	if (!pxa_unit->mpmu_base) {
 		pr_err("failed to map mpmu registers\n");
+		kfree(pxa_unit);
 		return;
 	}
 
@@ -234,6 +235,7 @@ static void __init pxa1928_apmu_clk_init(struct device_node *np)
 	pxa_unit->apmu_base = of_iomap(np, 0);
 	if (!pxa_unit->apmu_base) {
 		pr_err("failed to map apmu registers\n");
+		kfree(pxa_unit);
 		return;
 	}
 
@@ -254,6 +256,7 @@ static void __init pxa1928_apbc_clk_init(struct device_node *np)
 	pxa_unit->apbc_base = of_iomap(np, 0);
 	if (!pxa_unit->apbc_base) {
 		pr_err("failed to map apbc registers\n");
+		kfree(pxa_unit);
 		return;
 	}
 
diff --git a/drivers/clk/mmp/clk-of-pxa910.c b/drivers/clk/mmp/clk-of-pxa910.c
index 64d1ef49caeb..1dcabe95cb67 100644
--- a/drivers/clk/mmp/clk-of-pxa910.c
+++ b/drivers/clk/mmp/clk-of-pxa910.c
@@ -278,25 +278,25 @@ static void __init pxa910_clk_init(struct device_node *np)
 	pxa_unit->mpmu_base = of_iomap(np, 0);
 	if (!pxa_unit->mpmu_base) {
 		pr_err("failed to map mpmu registers\n");
-		return;
+		goto free_memory;
 	}
 
 	pxa_unit->apmu_base = of_iomap(np, 1);
 	if (!pxa_unit->apmu_base) {
 		pr_err("failed to map apmu registers\n");
-		return;
+		goto unmap_mpmu_region;
 	}
 
 	pxa_unit->apbc_base = of_iomap(np, 2);
 	if (!pxa_unit->apbc_base) {
 		pr_err("failed to map apbc registers\n");
-		return;
+		goto unmap_apmu_region;
 	}
 
 	pxa_unit->apbcp_base = of_iomap(np, 3);
 	if (!pxa_unit->apbcp_base) {
 		pr_err("failed to map apbcp registers\n");
-		return;
+		goto unmap_apbc_region;
 	}
 
 	mmp_clk_init(np, &pxa_unit->unit, PXA910_NR_CLKS);
@@ -308,6 +308,17 @@ static void __init pxa910_clk_init(struct device_node *np)
 	pxa910_axi_periph_clk_init(pxa_unit);
 
 	pxa910_clk_reset_init(np, pxa_unit);
+
+	return;
+
+unmap_apbc_region:
+	iounmap(pxa_unit->apbc_base);
+unmap_apmu_region:
+	iounmap(pxa_unit->apmu_base);
+unmap_mpmu_region:
+	iounmap(pxa_unit->mpmu_base);
+free_memory:
+	kfree(pxa_unit);
 }
 
 CLK_OF_DECLARE(pxa910_clk, "marvell,pxa910-clock", pxa910_clk_init);
diff --git a/drivers/clk/mvebu/ap806-system-controller.c b/drivers/clk/mvebu/ap806-system-controller.c
index 02023baf86c9..8181b919f062 100644
--- a/drivers/clk/mvebu/ap806-system-controller.c
+++ b/drivers/clk/mvebu/ap806-system-controller.c
@@ -14,7 +14,7 @@
 
 #include <linux/clk-provider.h>
 #include <linux/mfd/syscon.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
@@ -135,34 +135,17 @@ fail0:
 	return ret;
 }
 
-static int ap806_syscon_clk_remove(struct platform_device *pdev)
-{
-	of_clk_del_provider(pdev->dev.of_node);
-	clk_unregister_fixed_factor(ap806_clks[3]);
-	clk_unregister_fixed_rate(ap806_clks[2]);
-	clk_unregister_fixed_rate(ap806_clks[1]);
-	clk_unregister_fixed_rate(ap806_clks[0]);
-
-	return 0;
-}
-
 static const struct of_device_id ap806_syscon_of_match[] = {
 	{ .compatible = "marvell,ap806-system-controller", },
 	{ }
 };
-MODULE_DEVICE_TABLE(of, armada8k_pcie_of_match);
 
 static struct platform_driver ap806_syscon_driver = {
 	.probe = ap806_syscon_clk_probe,
-	.remove = ap806_syscon_clk_remove,
 	.driver		= {
 		.name	= "marvell-ap806-system-controller",
 		.of_match_table = ap806_syscon_of_match,
+		.suppress_bind_attrs = true,
 	},
 };
-
-module_platform_driver(ap806_syscon_driver);
-
-MODULE_DESCRIPTION("Marvell AP806 System Controller driver");
-MODULE_AUTHOR("Thomas Petazzoni <thomas.petazzoni@free-electrons.com>");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(ap806_syscon_driver);
diff --git a/drivers/clk/mvebu/cp110-system-controller.c b/drivers/clk/mvebu/cp110-system-controller.c
index f2303da7fda7..32e5b43c086f 100644
--- a/drivers/clk/mvebu/cp110-system-controller.c
+++ b/drivers/clk/mvebu/cp110-system-controller.c
@@ -30,7 +30,7 @@
 
 #include <linux/clk-provider.h>
 #include <linux/mfd/syscon.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
@@ -87,7 +87,7 @@ struct cp110_gate_clk {
 	u8 bit_idx;
 };
 
-#define to_cp110_gate_clk(clk) container_of(clk, struct cp110_gate_clk, hw)
+#define to_cp110_gate_clk(hw) container_of(hw, struct cp110_gate_clk, hw)
 
 static int cp110_gate_enable(struct clk_hw *hw)
 {
@@ -123,13 +123,14 @@ static const struct clk_ops cp110_gate_ops = {
 	.is_enabled = cp110_gate_is_enabled,
 };
 
-static struct clk *cp110_register_gate(const char *name,
-				       const char *parent_name,
-				       struct regmap *regmap, u8 bit_idx)
+static struct clk_hw *cp110_register_gate(const char *name,
+					  const char *parent_name,
+					  struct regmap *regmap, u8 bit_idx)
 {
 	struct cp110_gate_clk *gate;
-	struct clk *clk;
+	struct clk_hw *hw;
 	struct clk_init_data init;
+	int ret;
 
 	gate = kzalloc(sizeof(*gate), GFP_KERNEL);
 	if (!gate)
@@ -146,39 +147,37 @@ static struct clk *cp110_register_gate(const char *name,
 	gate->bit_idx = bit_idx;
 	gate->hw.init = &init;
 
-	clk = clk_register(NULL, &gate->hw);
-	if (IS_ERR(clk))
+	hw = &gate->hw;
+	ret = clk_hw_register(NULL, hw);
+	if (ret) {
 		kfree(gate);
+		hw = ERR_PTR(ret);
+	}
 
-	return clk;
+	return hw;
 }
 
-static void cp110_unregister_gate(struct clk *clk)
+static void cp110_unregister_gate(struct clk_hw *hw)
 {
-	struct clk_hw *hw;
-
-	hw = __clk_get_hw(clk);
-	if (!hw)
-		return;
-
-	clk_unregister(clk);
+	clk_hw_unregister(hw);
 	kfree(to_cp110_gate_clk(hw));
 }
 
-static struct clk *cp110_of_clk_get(struct of_phandle_args *clkspec, void *data)
+static struct clk_hw *cp110_of_clk_get(struct of_phandle_args *clkspec,
+				       void *data)
 {
-	struct clk_onecell_data *clk_data = data;
+	struct clk_hw_onecell_data *clk_data = data;
 	unsigned int type = clkspec->args[0];
 	unsigned int idx = clkspec->args[1];
 
 	if (type == CP110_CLK_TYPE_CORE) {
 		if (idx > CP110_MAX_CORE_CLOCKS)
 			return ERR_PTR(-EINVAL);
-		return clk_data->clks[idx];
+		return clk_data->hws[idx];
 	} else if (type == CP110_CLK_TYPE_GATABLE) {
 		if (idx > CP110_MAX_GATABLE_CLOCKS)
 			return ERR_PTR(-EINVAL);
-		return clk_data->clks[CP110_MAX_CORE_CLOCKS + idx];
+		return clk_data->hws[CP110_MAX_CORE_CLOCKS + idx];
 	}
 
 	return ERR_PTR(-EINVAL);
@@ -189,8 +188,8 @@ static int cp110_syscon_clk_probe(struct platform_device *pdev)
 	struct regmap *regmap;
 	struct device_node *np = pdev->dev.of_node;
 	const char *ppv2_name, *apll_name, *core_name, *eip_name, *nand_name;
-	struct clk_onecell_data *cp110_clk_data;
-	struct clk *clk, **cp110_clks;
+	struct clk_hw_onecell_data *cp110_clk_data;
+	struct clk_hw *hw, **cp110_clks;
 	u32 nand_clk_ctrl;
 	int i, ret;
 
@@ -203,80 +202,75 @@ static int cp110_syscon_clk_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	cp110_clks = devm_kcalloc(&pdev->dev, sizeof(struct clk *),
-				  CP110_CLK_NUM, GFP_KERNEL);
-	if (!cp110_clks)
-		return -ENOMEM;
-
-	cp110_clk_data = devm_kzalloc(&pdev->dev,
-				      sizeof(*cp110_clk_data),
+	cp110_clk_data = devm_kzalloc(&pdev->dev, sizeof(*cp110_clk_data) +
+				      sizeof(struct clk_hw *) * CP110_CLK_NUM,
 				      GFP_KERNEL);
 	if (!cp110_clk_data)
 		return -ENOMEM;
 
-	cp110_clk_data->clks = cp110_clks;
-	cp110_clk_data->clk_num = CP110_CLK_NUM;
+	cp110_clks = cp110_clk_data->hws;
+	cp110_clk_data->num = CP110_CLK_NUM;
 
-	/* Register the APLL which is the root of the clk tree */
+	/* Register the APLL which is the root of the hw tree */
 	of_property_read_string_index(np, "core-clock-output-names",
 				      CP110_CORE_APLL, &apll_name);
-	clk = clk_register_fixed_rate(NULL, apll_name, NULL, 0,
-				      1000 * 1000 * 1000);
-	if (IS_ERR(clk)) {
-		ret = PTR_ERR(clk);
+	hw = clk_hw_register_fixed_rate(NULL, apll_name, NULL, 0,
+					1000 * 1000 * 1000);
+	if (IS_ERR(hw)) {
+		ret = PTR_ERR(hw);
 		goto fail0;
 	}
 
-	cp110_clks[CP110_CORE_APLL] = clk;
+	cp110_clks[CP110_CORE_APLL] = hw;
 
 	/* PPv2 is APLL/3 */
 	of_property_read_string_index(np, "core-clock-output-names",
 				      CP110_CORE_PPV2, &ppv2_name);
-	clk = clk_register_fixed_factor(NULL, ppv2_name, apll_name, 0, 1, 3);
-	if (IS_ERR(clk)) {
-		ret = PTR_ERR(clk);
+	hw = clk_hw_register_fixed_factor(NULL, ppv2_name, apll_name, 0, 1, 3);
+	if (IS_ERR(hw)) {
+		ret = PTR_ERR(hw);
 		goto fail1;
 	}
 
-	cp110_clks[CP110_CORE_PPV2] = clk;
+	cp110_clks[CP110_CORE_PPV2] = hw;
 
 	/* EIP clock is APLL/2 */
 	of_property_read_string_index(np, "core-clock-output-names",
 				      CP110_CORE_EIP, &eip_name);
-	clk = clk_register_fixed_factor(NULL, eip_name, apll_name, 0, 1, 2);
-	if (IS_ERR(clk)) {
-		ret = PTR_ERR(clk);
+	hw = clk_hw_register_fixed_factor(NULL, eip_name, apll_name, 0, 1, 2);
+	if (IS_ERR(hw)) {
+		ret = PTR_ERR(hw);
 		goto fail2;
 	}
 
-	cp110_clks[CP110_CORE_EIP] = clk;
+	cp110_clks[CP110_CORE_EIP] = hw;
 
 	/* Core clock is EIP/2 */
 	of_property_read_string_index(np, "core-clock-output-names",
 				      CP110_CORE_CORE, &core_name);
-	clk = clk_register_fixed_factor(NULL, core_name, eip_name, 0, 1, 2);
-	if (IS_ERR(clk)) {
-		ret = PTR_ERR(clk);
+	hw = clk_hw_register_fixed_factor(NULL, core_name, eip_name, 0, 1, 2);
+	if (IS_ERR(hw)) {
+		ret = PTR_ERR(hw);
 		goto fail3;
 	}
 
-	cp110_clks[CP110_CORE_CORE] = clk;
+	cp110_clks[CP110_CORE_CORE] = hw;
 
 	/* NAND can be either APLL/2.5 or core clock */
 	of_property_read_string_index(np, "core-clock-output-names",
 				      CP110_CORE_NAND, &nand_name);
 	if (nand_clk_ctrl & NF_CLOCK_SEL_400_MASK)
-		clk = clk_register_fixed_factor(NULL, nand_name,
-						apll_name, 0, 2, 5);
+		hw = clk_hw_register_fixed_factor(NULL, nand_name,
+						   apll_name, 0, 2, 5);
 	else
-		clk = clk_register_fixed_factor(NULL, nand_name,
-						core_name, 0, 1, 1);
-	if (IS_ERR(clk)) {
-		ret = PTR_ERR(clk);
+		hw = clk_hw_register_fixed_factor(NULL, nand_name,
+						   core_name, 0, 1, 1);
+	if (IS_ERR(hw)) {
+		ret = PTR_ERR(hw);
 		goto fail4;
 	}
 
-	cp110_clks[CP110_CORE_NAND] = clk;
+	cp110_clks[CP110_CORE_NAND] = hw;
 
 	for (i = 0; i < CP110_MAX_GATABLE_CLOCKS; i++) {
 		const char *parent, *name;
@@ -335,16 +329,16 @@ static int cp110_syscon_clk_probe(struct platform_device *pdev)
 			break;
 		}
 
-		clk = cp110_register_gate(name, parent, regmap, i);
-		if (IS_ERR(clk)) {
-			ret = PTR_ERR(clk);
+		hw = cp110_register_gate(name, parent, regmap, i);
+		if (IS_ERR(hw)) {
+			ret = PTR_ERR(hw);
 			goto fail_gate;
 		}
 
-		cp110_clks[CP110_MAX_CORE_CLOCKS + i] = clk;
+		cp110_clks[CP110_MAX_CORE_CLOCKS + i] = hw;
 	}
 
-	ret = of_clk_add_provider(np, cp110_of_clk_get, cp110_clk_data);
+	ret = of_clk_add_hw_provider(np, cp110_of_clk_get, cp110_clk_data);
 	if (ret)
 		goto fail_clk_add;
 
@@ -355,65 +349,36 @@ static int cp110_syscon_clk_probe(struct platform_device *pdev)
 fail_clk_add:
 fail_gate:
 	for (i = 0; i < CP110_MAX_GATABLE_CLOCKS; i++) {
-		clk = cp110_clks[CP110_MAX_CORE_CLOCKS + i];
+		hw = cp110_clks[CP110_MAX_CORE_CLOCKS + i];
 
-		if (clk)
-			cp110_unregister_gate(clk);
+		if (hw)
+			cp110_unregister_gate(hw);
 	}
 
-	clk_unregister_fixed_factor(cp110_clks[CP110_CORE_NAND]);
+	clk_hw_unregister_fixed_factor(cp110_clks[CP110_CORE_NAND]);
 fail4:
-	clk_unregister_fixed_factor(cp110_clks[CP110_CORE_CORE]);
+	clk_hw_unregister_fixed_factor(cp110_clks[CP110_CORE_CORE]);
 fail3:
-	clk_unregister_fixed_factor(cp110_clks[CP110_CORE_EIP]);
+	clk_hw_unregister_fixed_factor(cp110_clks[CP110_CORE_EIP]);
 fail2:
-	clk_unregister_fixed_factor(cp110_clks[CP110_CORE_PPV2]);
+	clk_hw_unregister_fixed_factor(cp110_clks[CP110_CORE_PPV2]);
 fail1:
-	clk_unregister_fixed_rate(cp110_clks[CP110_CORE_APLL]);
+	clk_hw_unregister_fixed_rate(cp110_clks[CP110_CORE_APLL]);
 fail0:
 	return ret;
 }
 
-static int cp110_syscon_clk_remove(struct platform_device *pdev)
-{
-	struct clk **cp110_clks = platform_get_drvdata(pdev);
-	int i;
-
-	of_clk_del_provider(pdev->dev.of_node);
-
-	for (i = 0; i < CP110_MAX_GATABLE_CLOCKS; i++) {
-		struct clk *clk = cp110_clks[CP110_MAX_CORE_CLOCKS + i];
-
-		if (clk)
-			cp110_unregister_gate(clk);
-	}
-
-	clk_unregister_fixed_factor(cp110_clks[CP110_CORE_NAND]);
-	clk_unregister_fixed_factor(cp110_clks[CP110_CORE_CORE]);
-	clk_unregister_fixed_factor(cp110_clks[CP110_CORE_EIP]);
-	clk_unregister_fixed_factor(cp110_clks[CP110_CORE_PPV2]);
-	clk_unregister_fixed_rate(cp110_clks[CP110_CORE_APLL]);
-
-	return 0;
-}
-
 static const struct of_device_id cp110_syscon_of_match[] = {
 	{ .compatible = "marvell,cp110-system-controller0", },
 	{ }
 };
-MODULE_DEVICE_TABLE(of, armada8k_pcie_of_match);
 
 static struct platform_driver cp110_syscon_driver = {
 	.probe = cp110_syscon_clk_probe,
-	.remove = cp110_syscon_clk_remove,
 	.driver		= {
 		.name	= "marvell-cp110-system-controller0",
 		.of_match_table = cp110_syscon_of_match,
+		.suppress_bind_attrs = true,
 	},
 };
-
-module_platform_driver(cp110_syscon_driver);
-
-MODULE_DESCRIPTION("Marvell CP110 System Controller 0 driver");
-MODULE_AUTHOR("Thomas Petazzoni <thomas.petazzoni@free-electrons.com>");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(cp110_syscon_driver);
diff --git a/drivers/clk/nxp/clk-lpc18xx-ccu.c b/drivers/clk/nxp/clk-lpc18xx-ccu.c
index f7136b94fd0e..27781b49eb82 100644
--- a/drivers/clk/nxp/clk-lpc18xx-ccu.c
+++ b/drivers/clk/nxp/clk-lpc18xx-ccu.c
@@ -277,12 +277,15 @@ static void __init lpc18xx_ccu_init(struct device_node *np)
 	}
 
 	clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
-	if (!clk_data)
+	if (!clk_data) {
+		iounmap(reg_base);
 		return;
+	}
 
 	clk_data->num = of_property_count_strings(np, "clock-names");
 	clk_data->name = kcalloc(clk_data->num, sizeof(char *), GFP_KERNEL);
 	if (!clk_data->name) {
+		iounmap(reg_base);
 		kfree(clk_data);
 		return;
 	}
diff --git a/drivers/clk/nxp/clk-lpc32xx.c b/drivers/clk/nxp/clk-lpc32xx.c
index 34c97353cdeb..5b98ff9076f3 100644
--- a/drivers/clk/nxp/clk-lpc32xx.c
+++ b/drivers/clk/nxp/clk-lpc32xx.c
@@ -1282,13 +1282,13 @@ static struct clk_hw_proto clk_hw_proto[LPC32XX_CLK_HW_MAX] = {
 
 	LPC32XX_DEFINE_MUX(PWM1_MUX, PWMCLK_CTRL, 1, 0x1, NULL, 0),
 	LPC32XX_DEFINE_DIV(PWM1_DIV, PWMCLK_CTRL, 4, 4, NULL,
-			   CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO),
+			   CLK_DIVIDER_ONE_BASED),
 	LPC32XX_DEFINE_GATE(PWM1_GATE, PWMCLK_CTRL, 0, 0),
 	LPC32XX_DEFINE_COMPOSITE(PWM1, PWM1_MUX, PWM1_DIV, PWM1_GATE),
 
 	LPC32XX_DEFINE_MUX(PWM2_MUX, PWMCLK_CTRL, 3, 0x1, NULL, 0),
 	LPC32XX_DEFINE_DIV(PWM2_DIV, PWMCLK_CTRL, 8, 4, NULL,
-			   CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO),
+			   CLK_DIVIDER_ONE_BASED),
 	LPC32XX_DEFINE_GATE(PWM2_GATE, PWMCLK_CTRL, 2, 0),
 	LPC32XX_DEFINE_COMPOSITE(PWM2, PWM2_MUX, PWM2_DIV, PWM2_GATE),
 
@@ -1335,8 +1335,7 @@ static struct clk_hw_proto clk_hw_proto[LPC32XX_CLK_HW_MAX] = {
 	LPC32XX_DEFINE_GATE(USB_DIV_GATE, USB_CTRL, 17, 0),
 	LPC32XX_DEFINE_COMPOSITE(USB_DIV, _NULL, USB_DIV_DIV, USB_DIV_GATE),
 
-	LPC32XX_DEFINE_DIV(SD_DIV, MS_CTRL, 0, 4, NULL,
-			   CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO),
+	LPC32XX_DEFINE_DIV(SD_DIV, MS_CTRL, 0, 4, NULL, CLK_DIVIDER_ONE_BASED),
 	LPC32XX_DEFINE_CLK(SD_GATE, MS_CTRL, BIT(5) | BIT(9), BIT(5) | BIT(9),
 			   0x0, BIT(5) | BIT(9), 0x0, 0x0, clk_mask_ops),
 	LPC32XX_DEFINE_COMPOSITE(SD, _NULL, SD_DIV, SD_GATE),
@@ -1478,6 +1477,20 @@ static struct clk * __init lpc32xx_clk_register(u32 id)
 	return clk;
 }
 
+static void __init lpc32xx_clk_div_quirk(u32 reg, u32 div_mask, u32 gate)
+{
+	u32 val;
+
+	regmap_read(clk_regmap, reg, &val);
+
+	if (!(val & div_mask)) {
+		val &= ~gate;
+		val |= BIT(__ffs(div_mask));
+	}
+
+	regmap_update_bits(clk_regmap, reg, gate | div_mask, val);
+}
+
 static void __init lpc32xx_clk_init(struct device_node *np)
 {
 	unsigned int i;
@@ -1517,6 +1530,17 @@ static void __init lpc32xx_clk_init(struct device_node *np)
 		return;
 	}
 
+	/*
+	 * Divider part of PWM and MS clocks requires a quirk to avoid
+	 * a misinterpretation of formally valid zero value in register
+	 * bitfield, which indicates another clock gate. Instead of
+	 * adding complexity to a gate clock ensure that zero value in
+	 * divider clock is never met in runtime.
+	 */
+	lpc32xx_clk_div_quirk(LPC32XX_CLKPWR_PWMCLK_CTRL, 0xf0, BIT(0));
+	lpc32xx_clk_div_quirk(LPC32XX_CLKPWR_PWMCLK_CTRL, 0xf00, BIT(2));
+	lpc32xx_clk_div_quirk(LPC32XX_CLKPWR_MS_CTRL, 0xf, BIT(5) | BIT(9));
+
 	for (i = 1; i < LPC32XX_CLK_MAX; i++) {
 		clk[i] = lpc32xx_clk_register(i);
 		if (IS_ERR(clk[i])) {
diff --git a/drivers/clk/pxa/clk-pxa.c b/drivers/clk/pxa/clk-pxa.c
index 29cee9e8d4d9..74f64c3c4290 100644
--- a/drivers/clk/pxa/clk-pxa.c
+++ b/drivers/clk/pxa/clk-pxa.c
@@ -18,7 +18,27 @@
 #include <dt-bindings/clock/pxa-clock.h>
 #include "clk-pxa.h"
 
-DEFINE_SPINLOCK(lock);
+#define KHz 1000
+#define MHz (1000 * 1000)
+
+#define MDREFR_K0DB4	(1 << 29)	/* SDCLK0 Divide by 4 Control/Status */
+#define MDREFR_K2FREE	(1 << 25)	/* SDRAM Free-Running Control */
+#define MDREFR_K1FREE	(1 << 24)	/* SDRAM Free-Running Control */
+#define MDREFR_K0FREE	(1 << 23)	/* SDRAM Free-Running Control */
+#define MDREFR_SLFRSH	(1 << 22)	/* SDRAM Self-Refresh Control/Status */
+#define MDREFR_APD	(1 << 20)	/* SDRAM/SSRAM Auto-Power-Down Enable */
+#define MDREFR_K2DB2	(1 << 19)	/* SDCLK2 Divide by 2 Control/Status */
+#define MDREFR_K2RUN	(1 << 18)	/* SDCLK2 Run Control/Status */
+#define MDREFR_K1DB2	(1 << 17)	/* SDCLK1 Divide by 2 Control/Status */
+#define MDREFR_K1RUN	(1 << 16)	/* SDCLK1 Run Control/Status */
+#define MDREFR_E1PIN	(1 << 15)	/* SDCKE1 Level Control/Status */
+#define MDREFR_K0DB2	(1 << 14)	/* SDCLK0 Divide by 2 Control/Status */
+#define MDREFR_K0RUN	(1 << 13)	/* SDCLK0 Run Control/Status */
+#define MDREFR_E0PIN	(1 << 12)	/* SDCKE0 Level Control/Status */
+#define MDREFR_DB2_MASK	(MDREFR_K2DB2 | MDREFR_K1DB2)
+#define MDREFR_DRI_MASK	0xFFF
+
+static DEFINE_SPINLOCK(pxa_clk_lock);
 
 static struct clk *pxa_clocks[CLK_MAX];
 static struct clk_onecell_data onecell_data = {
@@ -89,7 +109,7 @@ int __init clk_pxa_cken_init(const struct desc_clk_cken *clks, int nb_clks)
 		pxa_clk->lp = clks[i].lp;
 		pxa_clk->hp = clks[i].hp;
 		pxa_clk->gate = clks[i].gate;
-		pxa_clk->gate.lock = &lock;
+		pxa_clk->gate.lock = &pxa_clk_lock;
 		clk = clk_register_composite(NULL, clks[i].name,
 					     clks[i].parent_names, 2,
 					     &pxa_clk->hw, &cken_mux_ops,
@@ -106,3 +126,124 @@ void __init clk_pxa_dt_common_init(struct device_node *np)
 {
 	of_clk_add_provider(np, of_clk_src_onecell_get, &onecell_data);
 }
+
+void pxa2xx_core_turbo_switch(bool on)
+{
+	unsigned long flags;
+	unsigned int unused, clkcfg;
+
+	local_irq_save(flags);
+
+	asm("mrc p14, 0, %0, c6, c0, 0" : "=r" (clkcfg));
+	clkcfg &= ~CLKCFG_TURBO & ~CLKCFG_HALFTURBO;
+	if (on)
+		clkcfg |= CLKCFG_TURBO;
+	clkcfg |= CLKCFG_FCS;
+
+	asm volatile(
+	"	b	2f\n"
+	"	.align	5\n"
+	"1:	mcr	p14, 0, %1, c6, c0, 0\n"
+	"	b	3f\n"
+	"2:	b	1b\n"
+	"3:	nop\n"
+		: "=&r" (unused)
+		: "r" (clkcfg)
+		: );
+
+	local_irq_restore(flags);
+}
+
+void pxa2xx_cpll_change(struct pxa2xx_freq *freq,
+			u32 (*mdrefr_dri)(unsigned int), void __iomem *mdrefr,
+			void __iomem *cccr)
+{
+	unsigned int clkcfg = freq->clkcfg;
+	unsigned int unused, preset_mdrefr, postset_mdrefr;
+	unsigned long flags;
+
+	local_irq_save(flags);
+
+	/* Calculate the next MDREFR.  If we're slowing down the SDRAM clock
+	 * we need to preset the smaller DRI before the change.	 If we're
+	 * speeding up we need to set the larger DRI value after the change.
+	 */
+	preset_mdrefr = postset_mdrefr = readl(mdrefr);
+	if ((preset_mdrefr & MDREFR_DRI_MASK) > mdrefr_dri(freq->membus_khz)) {
+		preset_mdrefr = (preset_mdrefr & ~MDREFR_DRI_MASK);
+		preset_mdrefr |= mdrefr_dri(freq->membus_khz);
+	}
+	postset_mdrefr =
+		(postset_mdrefr & ~MDREFR_DRI_MASK) |
+		mdrefr_dri(freq->membus_khz);
+
+	/* If we're dividing the memory clock by two for the SDRAM clock, this
+	 * must be set prior to the change.  Clearing the divide must be done
+	 * after the change.
+	 */
+	if (freq->div2) {
+		preset_mdrefr  |= MDREFR_DB2_MASK;
+		postset_mdrefr |= MDREFR_DB2_MASK;
+	} else {
+		postset_mdrefr &= ~MDREFR_DB2_MASK;
+	}
+
+	/* Set new the CCCR and prepare CLKCFG */
+	writel(freq->cccr, cccr);
+
+	asm volatile(
+	"	ldr	r4, [%1]\n"
+	"	b	2f\n"
+	"	.align	5\n"
+	"1:	str	%3, [%1]		/* preset the MDREFR */\n"
+	"	mcr	p14, 0, %2, c6, c0, 0	/* set CLKCFG[FCS] */\n"
+	"	str	%4, [%1]		/* postset the MDREFR */\n"
+	"	b	3f\n"
+	"2:	b	1b\n"
+	"3:	nop\n"
+	     : "=&r" (unused)
+	     : "r" (mdrefr), "r" (clkcfg), "r" (preset_mdrefr),
+	       "r" (postset_mdrefr)
+	     : "r4", "r5");
+
+	local_irq_restore(flags);
+}
+
+int pxa2xx_determine_rate(struct clk_rate_request *req,
+			  struct pxa2xx_freq *freqs, int nb_freqs)
+{
+	int i, closest_below = -1, closest_above = -1;
+	unsigned long rate;
+
+	for (i = 0; i < nb_freqs; i++) {
+		rate = freqs[i].cpll;
+		if (rate == req->rate)
+			break;
+		if (rate < req->min_rate)
+			continue;
+		if (rate > req->max_rate)
+			continue;
+		if (rate <= req->rate)
+			closest_below = i;
+		if ((rate >= req->rate) && (closest_above == -1))
+			closest_above = i;
+	}
+
+	req->best_parent_hw = NULL;
+
+	if (i < nb_freqs) {
+		rate = req->rate;
+	} else if (closest_below >= 0) {
+		rate = freqs[closest_below].cpll;
+	} else if (closest_above >= 0) {
+		rate = freqs[closest_above].cpll;
+	} else {
+		pr_debug("%s(rate=%lu) no match\n", __func__, req->rate);
+		return -EINVAL;
+	}
+
+	pr_debug("%s(rate=%lu) rate=%lu\n", __func__, req->rate, rate);
+	req->rate = rate;
+
+	return 0;
+}
diff --git a/drivers/clk/pxa/clk-pxa.h b/drivers/clk/pxa/clk-pxa.h
index d1de805df867..2b90c5917b32 100644
--- a/drivers/clk/pxa/clk-pxa.h
+++ b/drivers/clk/pxa/clk-pxa.h
@@ -13,6 +13,11 @@
 #ifndef _CLK_PXA_
 #define _CLK_PXA_
 
+#define CLKCFG_TURBO		0x1
+#define CLKCFG_FCS		0x2
+#define CLKCFG_HALFTURBO	0x4
+#define CLKCFG_FASTBUS		0x8
+
 #define PARENTS(name) \
 	static const char *const name ## _parents[] __initconst
 #define MUX_RO_RATE_RO_OPS(name, clk_name)			\
@@ -35,10 +40,27 @@
 			NULL, NULL, CLK_GET_RATE_NOCACHE);	\
 	}
 
-#define RATE_RO_OPS(name, clk_name)			\
+#define RATE_RO_OPS(name, clk_name)				\
+	static struct clk_hw name ## _rate_hw;			\
+	static const struct clk_ops name ## _rate_ops = {		\
+		.recalc_rate = name ## _get_rate,		\
+	};							\
+	static struct clk * __init clk_register_ ## name(void)	\
+	{							\
+		return clk_register_composite(NULL, clk_name,	\
+			name ## _parents,			\
+			ARRAY_SIZE(name ## _parents),		\
+			NULL, NULL,				\
+			&name ## _rate_hw, &name ## _rate_ops,	\
+			NULL, NULL, CLK_GET_RATE_NOCACHE);	\
+	}
+
+#define RATE_OPS(name, clk_name)				\
 	static struct clk_hw name ## _rate_hw;			\
 	static struct clk_ops name ## _rate_ops = {		\
 		.recalc_rate = name ## _get_rate,		\
+		.set_rate = name ## _set_rate,			\
+		.determine_rate = name ## _determine_rate,	\
 	};							\
 	static struct clk * __init clk_register_ ## name(void)	\
 	{							\
@@ -50,6 +72,24 @@
 			NULL, NULL, CLK_GET_RATE_NOCACHE);	\
 	}
 
+#define MUX_OPS(name, clk_name, flags)				\
+	static struct clk_hw name ## _mux_hw;			\
+	static const struct clk_ops name ## _mux_ops = {	\
+		.get_parent = name ## _get_parent,		\
+		.set_parent = name ## _set_parent,		\
+		.determine_rate = name ## _determine_rate,	\
+	};							\
+	static struct clk * __init clk_register_ ## name(void)	\
+	{							\
+		return clk_register_composite(NULL, clk_name,	\
+			name ## _parents,			\
+			ARRAY_SIZE(name ## _parents),		\
+			&name ## _mux_hw, &name ## _mux_ops,	\
+			NULL, NULL,				\
+			NULL, NULL,				\
+			CLK_GET_RATE_NOCACHE | flags); \
+	}
+
 /*
  * CKEN clock type
  * This clock takes it source from 2 possible parents :
@@ -95,7 +135,15 @@ struct desc_clk_cken {
 	PXA_CKEN(dev_id, con_id, name, parents, 1, 1, 1, 1,		\
 		 NULL, cken_reg, cken_bit, flag)
 
-static int dummy_clk_set_parent(struct clk_hw *hw, u8 index)
+struct pxa2xx_freq {
+	unsigned long cpll;
+	unsigned int membus_khz;
+	unsigned int cccr;
+	unsigned int div2;
+	unsigned int clkcfg;
+};
+
+static inline int dummy_clk_set_parent(struct clk_hw *hw, u8 index)
 {
 	return 0;
 }
@@ -105,4 +153,11 @@ extern void clkdev_pxa_register(int ckid, const char *con_id,
 extern int clk_pxa_cken_init(const struct desc_clk_cken *clks, int nb_clks);
 void clk_pxa_dt_common_init(struct device_node *np);
 
+void pxa2xx_core_turbo_switch(bool on);
+void pxa2xx_cpll_change(struct pxa2xx_freq *freq,
+			u32 (*mdrefr_dri)(unsigned int), void __iomem *mdrefr,
+			void __iomem *cccr);
+int pxa2xx_determine_rate(struct clk_rate_request *req,
+			  struct pxa2xx_freq *freqs,  int nb_freqs);
+
 #endif
diff --git a/drivers/clk/pxa/clk-pxa25x.c b/drivers/clk/pxa/clk-pxa25x.c
index a98b98e2a9e4..6416c1f8e632 100644
--- a/drivers/clk/pxa/clk-pxa25x.c
+++ b/drivers/clk/pxa/clk-pxa25x.c
@@ -18,6 +18,7 @@
 #include <linux/io.h>
 #include <linux/of.h>
 #include <mach/pxa2xx-regs.h>
+#include <mach/smemc.h>
 
 #include <dt-bindings/clock/pxa-clock.h>
 #include "clk-pxa.h"
@@ -30,6 +31,17 @@ enum {
 	PXA_CORE_TURBO,
 };
 
+#define PXA25x_CLKCFG(T)			\
+	(CLKCFG_FCS |				\
+	 ((T) ? CLKCFG_TURBO : 0))
+#define PXA25x_CCCR(N2, M, L) (N2 << 7 | M << 5 | L)
+
+#define MDCNFG_DRAC2(mdcnfg)	(((mdcnfg) >> 21) & 0x3)
+#define MDCNFG_DRAC0(mdcnfg)	(((mdcnfg) >> 5) & 0x3)
+
+/* Define the refresh period in mSec for the SDRAM and the number of rows */
+#define SDRAM_TREF	64	/* standard 64ms SDRAM */
+
 /*
  * Various clock factors driven by the CCCR register.
  */
@@ -48,6 +60,34 @@ static const char * const get_freq_khz[] = {
 	"core", "run", "cpll", "memory"
 };
 
+static int get_sdram_rows(void)
+{
+	static int sdram_rows;
+	unsigned int drac2 = 0, drac0 = 0;
+	u32 mdcnfg;
+
+	if (sdram_rows)
+		return sdram_rows;
+
+	mdcnfg = readl_relaxed(MDCNFG);
+
+	if (mdcnfg & (MDCNFG_DE2 | MDCNFG_DE3))
+		drac2 = MDCNFG_DRAC2(mdcnfg);
+
+	if (mdcnfg & (MDCNFG_DE0 | MDCNFG_DE1))
+		drac0 = MDCNFG_DRAC0(mdcnfg);
+
+	sdram_rows = 1 << (11 + max(drac0, drac2));
+	return sdram_rows;
+}
+
+static u32 mdrefr_dri(unsigned int freq_khz)
+{
+	u32 interval = freq_khz * SDRAM_TREF / get_sdram_rows();
+
+	return interval / 32;
+}
+
 /*
  * Get the clock frequency as reflected by CCCR and the turbo flag.
  * We assume these values have been applied via a fcs.
@@ -139,6 +179,21 @@ static struct desc_clk_cken pxa25x_clocks[] __initdata = {
 			     clk_pxa25x_memory_parents, 0),
 };
 
+/*
+ * In this table, PXA25x_CCCR(N2, M, L) has the following meaning, where :
+ *   - freq_cpll = n * m * L * 3.6864 MHz
+ *   - n = N2 / 2
+ *   - m = 2^(M - 1), where 1 <= M <= 3
+ *   - l = L_clk_mult[L], ie. { 0, 27, 32, 36, 40, 45, 0, }[L]
+ */
+static struct pxa2xx_freq pxa25x_freqs[] = {
+	/* CPU  MEMBUS  CCCR                  DIV2 CCLKCFG      */
+	{ 99532800, 99500, PXA25x_CCCR(2,  1, 1),  1, PXA25x_CLKCFG(1)},
+	{199065600, 99500, PXA25x_CCCR(4,  1, 1),  0, PXA25x_CLKCFG(1)},
+	{298598400, 99500, PXA25x_CCCR(3,  2, 1),  0, PXA25x_CLKCFG(1)},
+	{398131200, 99500, PXA25x_CCCR(4,  2, 1),  0, PXA25x_CLKCFG(1)},
+};
+
 static u8 clk_pxa25x_core_get_parent(struct clk_hw *hw)
 {
 	unsigned long clkcfg;
@@ -151,13 +206,24 @@ static u8 clk_pxa25x_core_get_parent(struct clk_hw *hw)
 	return PXA_CORE_RUN;
 }
 
-static unsigned long clk_pxa25x_core_get_rate(struct clk_hw *hw,
-					      unsigned long parent_rate)
+static int clk_pxa25x_core_set_parent(struct clk_hw *hw, u8 index)
 {
-	return parent_rate;
+	if (index > PXA_CORE_TURBO)
+		return -EINVAL;
+
+	pxa2xx_core_turbo_switch(index == PXA_CORE_TURBO);
+
+	return 0;
+}
+
+static int clk_pxa25x_core_determine_rate(struct clk_hw *hw,
+					  struct clk_rate_request *req)
+{
+	return __clk_mux_determine_rate(hw, req);
 }
+
 PARENTS(clk_pxa25x_core) = { "run", "cpll" };
-MUX_RO_RATE_RO_OPS(clk_pxa25x_core, "core");
+MUX_OPS(clk_pxa25x_core, "core", CLK_SET_RATE_PARENT);
 
 static unsigned long clk_pxa25x_run_get_rate(struct clk_hw *hw,
 					     unsigned long parent_rate)
@@ -182,17 +248,42 @@ static unsigned long clk_pxa25x_cpll_get_rate(struct clk_hw *hw,
 	m = M_clk_mult[(cccr >> 5) & 0x03];
 	n2 = N2_clk_mult[(cccr >> 7) & 0x07];
 
-	if (t)
-		return m * l * n2 * parent_rate / 2;
-	return m * l * parent_rate;
+	return m * l * n2 * parent_rate / 2;
+}
+
+static int clk_pxa25x_cpll_determine_rate(struct clk_hw *hw,
+					  struct clk_rate_request *req)
+{
+	return pxa2xx_determine_rate(req, pxa25x_freqs,
+				     ARRAY_SIZE(pxa25x_freqs));
+}
+
+static int clk_pxa25x_cpll_set_rate(struct clk_hw *hw, unsigned long rate,
+				    unsigned long parent_rate)
+{
+	int i;
+
+	pr_debug("%s(rate=%lu parent_rate=%lu)\n", __func__, rate, parent_rate);
+	for (i = 0; i < ARRAY_SIZE(pxa25x_freqs); i++)
+		if (pxa25x_freqs[i].cpll == rate)
+			break;
+
+	if (i >= ARRAY_SIZE(pxa25x_freqs))
+		return -EINVAL;
+
+	pxa2xx_cpll_change(&pxa25x_freqs[i], mdrefr_dri, MDREFR, CCCR);
+
+	return 0;
 }
 PARENTS(clk_pxa25x_cpll) = { "osc_3_6864mhz" };
-RATE_RO_OPS(clk_pxa25x_cpll, "cpll");
+RATE_OPS(clk_pxa25x_cpll, "cpll");
 
 static void __init pxa25x_register_core(void)
 {
-	clk_register_clk_pxa25x_cpll();
-	clk_register_clk_pxa25x_run();
+	clkdev_pxa_register(CLK_NONE, "cpll", NULL,
+			    clk_register_clk_pxa25x_cpll());
+	clkdev_pxa_register(CLK_NONE, "run", NULL,
+			    clk_register_clk_pxa25x_run());
 	clkdev_pxa_register(CLK_CORE, "core", NULL,
 			    clk_register_clk_pxa25x_core());
 }
@@ -214,7 +305,8 @@ static void __init pxa25x_base_clocks_init(void)
 {
 	pxa25x_register_plls();
 	pxa25x_register_core();
-	clk_register_clk_pxa25x_memory();
+	clkdev_pxa_register(CLK_NONE, "system_bus", NULL,
+			    clk_register_clk_pxa25x_memory());
 }
 
 #define DUMMY_CLK(_con_id, _dev_id, _parent) \
@@ -230,7 +322,7 @@ static struct dummy_clk dummy_clks[] __initdata = {
 	DUMMY_CLK("GPIO11_CLK", NULL, "osc_3_6864mhz"),
 	DUMMY_CLK("GPIO12_CLK", NULL, "osc_32_768khz"),
 	DUMMY_CLK(NULL, "sa1100-rtc", "osc_32_768khz"),
-	DUMMY_CLK("OSTIMER0", NULL, "osc_32_768khz"),
+	DUMMY_CLK("OSTIMER0", NULL, "osc_3_6864mhz"),
 	DUMMY_CLK("UARTCLK", "pxa2xx-ir", "STUART"),
 };
 
diff --git a/drivers/clk/pxa/clk-pxa27x.c b/drivers/clk/pxa/clk-pxa27x.c
index c40b1804f58c..25a30194d27a 100644
--- a/drivers/clk/pxa/clk-pxa27x.c
+++ b/drivers/clk/pxa/clk-pxa27x.c
@@ -17,6 +17,8 @@
 #include <linux/clkdev.h>
 #include <linux/of.h>
 
+#include <mach/smemc.h>
+
 #include <dt-bindings/clock/pxa-clock.h>
 #include "clk-pxa.h"
 
@@ -45,11 +47,52 @@ enum {
 	PXA_MEM_RUN,
 };
 
+#define PXA27x_CLKCFG(B, HT, T)			\
+	(CLKCFG_FCS |				\
+	 ((B)  ? CLKCFG_FASTBUS : 0) |		\
+	 ((HT) ? CLKCFG_HALFTURBO : 0) |	\
+	 ((T)  ? CLKCFG_TURBO : 0))
+#define PXA27x_CCCR(A, L, N2) (A << 25 | N2 << 7 | L)
+
+#define MDCNFG_DRAC2(mdcnfg)	(((mdcnfg) >> 21) & 0x3)
+#define MDCNFG_DRAC0(mdcnfg)	(((mdcnfg) >> 5) & 0x3)
+
+/* Define the refresh period in mSec for the SDRAM and the number of rows */
+#define SDRAM_TREF	64	/* standard 64ms SDRAM */
+
 static const char * const get_freq_khz[] = {
 	"core", "run", "cpll", "memory",
 	"system_bus"
 };
 
+static int get_sdram_rows(void)
+{
+	static int sdram_rows;
+	unsigned int drac2 = 0, drac0 = 0;
+	u32 mdcnfg;
+
+	if (sdram_rows)
+		return sdram_rows;
+
+	mdcnfg = readl_relaxed(MDCNFG);
+
+	if (mdcnfg & (MDCNFG_DE2 | MDCNFG_DE3))
+		drac2 = MDCNFG_DRAC2(mdcnfg);
+
+	if (mdcnfg & (MDCNFG_DE0 | MDCNFG_DE1))
+		drac0 = MDCNFG_DRAC0(mdcnfg);
+
+	sdram_rows = 1 << (11 + max(drac0, drac2));
+	return sdram_rows;
+}
+
+static u32 mdrefr_dri(unsigned int freq_khz)
+{
+	u32 interval = freq_khz * SDRAM_TREF / get_sdram_rows();
+
+	return (interval - 31) / 32;
+}
+
 /*
  * Get the clock frequency as reflected by CCSR and the turbo flag.
  * We assume these values have been applied via a fcs.
@@ -145,6 +188,42 @@ static struct desc_clk_cken pxa27x_clocks[] __initdata = {
 
 };
 
+/*
+ * PXA270 definitions
+ *
+ * For the PXA27x:
+ * Control variables are A, L, 2N for CCCR; B, HT, T for CLKCFG.
+ *
+ * A = 0 => memory controller clock from table 3-7,
+ * A = 1 => memory controller clock = system bus clock
+ * Run mode frequency	= 13 MHz * L
+ * Turbo mode frequency = 13 MHz * L * N
+ * System bus frequency = 13 MHz * L / (B + 1)
+ *
+ * In CCCR:
+ * A = 1
+ * L = 16	  oscillator to run mode ratio
+ * 2N = 6	  2 * (turbo mode to run mode ratio)
+ *
+ * In CCLKCFG:
+ * B = 1	  Fast bus mode
+ * HT = 0	  Half-Turbo mode
+ * T = 1	  Turbo mode
+ *
+ * For now, just support some of the combinations in table 3-7 of
+ * PXA27x Processor Family Developer's Manual to simplify frequency
+ * change sequences.
+ */
+static struct pxa2xx_freq pxa27x_freqs[] = {
+	{104000000, 104000, PXA27x_CCCR(1,  8, 2), 0, PXA27x_CLKCFG(1, 0, 1) },
+	{156000000, 104000, PXA27x_CCCR(1,  8, 3), 0, PXA27x_CLKCFG(1, 0, 1) },
+	{208000000, 208000, PXA27x_CCCR(0, 16, 2), 1, PXA27x_CLKCFG(0, 0, 1) },
+	{312000000, 208000, PXA27x_CCCR(1, 16, 3), 1, PXA27x_CLKCFG(1, 0, 1) },
+	{416000000, 208000, PXA27x_CCCR(1, 16, 4), 1, PXA27x_CLKCFG(1, 0, 1) },
+	{520000000, 208000, PXA27x_CCCR(1, 16, 5), 1, PXA27x_CLKCFG(1, 0, 1) },
+	{624000000, 208000, PXA27x_CCCR(1, 16, 6), 1, PXA27x_CLKCFG(1, 0, 1) },
+};
+
 static unsigned long clk_pxa27x_cpll_get_rate(struct clk_hw *hw,
 	unsigned long parent_rate)
 {
@@ -162,10 +241,35 @@ static unsigned long clk_pxa27x_cpll_get_rate(struct clk_hw *hw,
 	L  = l * parent_rate;
 	N  = (L * n2) / 2;
 
-	return t ? N : L;
+	return N;
+}
+
+static int clk_pxa27x_cpll_determine_rate(struct clk_hw *hw,
+					  struct clk_rate_request *req)
+{
+	return pxa2xx_determine_rate(req, pxa27x_freqs,
+				     ARRAY_SIZE(pxa27x_freqs));
+}
+
+static int clk_pxa27x_cpll_set_rate(struct clk_hw *hw, unsigned long rate,
+				    unsigned long parent_rate)
+{
+	int i;
+
+	pr_debug("%s(rate=%lu parent_rate=%lu)\n", __func__, rate, parent_rate);
+	for (i = 0; i < ARRAY_SIZE(pxa27x_freqs); i++)
+		if (pxa27x_freqs[i].cpll == rate)
+			break;
+
+	if (i >= ARRAY_SIZE(pxa27x_freqs))
+		return -EINVAL;
+
+	pxa2xx_cpll_change(&pxa27x_freqs[i], mdrefr_dri, MDREFR, CCCR);
+	return 0;
 }
+
 PARENTS(clk_pxa27x_cpll) = { "osc_13mhz" };
-RATE_RO_OPS(clk_pxa27x_cpll, "cpll");
+RATE_OPS(clk_pxa27x_cpll, "cpll");
 
 static unsigned long clk_pxa27x_lcd_base_get_rate(struct clk_hw *hw,
 						  unsigned long parent_rate)
@@ -217,31 +321,10 @@ static void __init pxa27x_register_plls(void)
 	clk_register_fixed_factor(NULL, "ppll_312mhz", "osc_13mhz", 0, 24, 1);
 }
 
-static unsigned long clk_pxa27x_core_get_rate(struct clk_hw *hw,
-					      unsigned long parent_rate)
-{
-	unsigned long clkcfg;
-	unsigned int t, ht, b, osc_forced;
-	unsigned long ccsr = readl(CCSR);
-
-	osc_forced = ccsr & (1 << CCCR_CPDIS_BIT);
-	asm("mrc\tp14, 0, %0, c6, c0, 0" : "=r" (clkcfg));
-	t  = clkcfg & (1 << 0);
-	ht = clkcfg & (1 << 2);
-	b  = clkcfg & (1 << 3);
-
-	if (osc_forced)
-		return parent_rate;
-	if (ht)
-		return parent_rate / 2;
-	else
-		return parent_rate;
-}
-
 static u8 clk_pxa27x_core_get_parent(struct clk_hw *hw)
 {
 	unsigned long clkcfg;
-	unsigned int t, ht, b, osc_forced;
+	unsigned int t, ht, osc_forced;
 	unsigned long ccsr = readl(CCSR);
 
 	osc_forced = ccsr & (1 << CCCR_CPDIS_BIT);
@@ -251,14 +334,30 @@ static u8 clk_pxa27x_core_get_parent(struct clk_hw *hw)
 	asm("mrc\tp14, 0, %0, c6, c0, 0" : "=r" (clkcfg));
 	t  = clkcfg & (1 << 0);
 	ht = clkcfg & (1 << 2);
-	b  = clkcfg & (1 << 3);
 
 	if (ht || t)
 		return PXA_CORE_TURBO;
 	return PXA_CORE_RUN;
 }
+
+static int clk_pxa27x_core_set_parent(struct clk_hw *hw, u8 index)
+{
+	if (index > PXA_CORE_TURBO)
+		return -EINVAL;
+
+	pxa2xx_core_turbo_switch(index == PXA_CORE_TURBO);
+
+	return 0;
+}
+
+static int clk_pxa27x_core_determine_rate(struct clk_hw *hw,
+					  struct clk_rate_request *req)
+{
+	return __clk_mux_determine_rate(hw, req);
+}
+
 PARENTS(clk_pxa27x_core) = { "osc_13mhz", "run", "cpll" };
-MUX_RO_RATE_RO_OPS(clk_pxa27x_core, "core");
+MUX_OPS(clk_pxa27x_core, "core", CLK_SET_RATE_PARENT);
 
 static unsigned long clk_pxa27x_run_get_rate(struct clk_hw *hw,
 					     unsigned long parent_rate)
@@ -273,9 +372,10 @@ RATE_RO_OPS(clk_pxa27x_run, "run");
 
 static void __init pxa27x_register_core(void)
 {
-	clk_register_clk_pxa27x_cpll();
-	clk_register_clk_pxa27x_run();
-
+	clkdev_pxa_register(CLK_NONE, "cpll", NULL,
+			    clk_register_clk_pxa27x_cpll());
+	clkdev_pxa_register(CLK_NONE, "run", NULL,
+			    clk_register_clk_pxa27x_run());
 	clkdev_pxa_register(CLK_CORE, "core", NULL,
 			    clk_register_clk_pxa27x_core());
 }
@@ -294,9 +394,9 @@ static unsigned long clk_pxa27x_system_bus_get_rate(struct clk_hw *hw,
 	if (osc_forced)
 		return parent_rate;
 	if (b)
-		return parent_rate / 2;
-	else
 		return parent_rate;
+	else
+		return parent_rate / 2;
 }
 
 static u8 clk_pxa27x_system_bus_get_parent(struct clk_hw *hw)
@@ -385,8 +485,10 @@ static void __init pxa27x_base_clocks_init(void)
 {
 	pxa27x_register_plls();
 	pxa27x_register_core();
-	clk_register_clk_pxa27x_system_bus();
-	clk_register_clk_pxa27x_memory();
+	clkdev_pxa_register(CLK_NONE, "system_bus", NULL,
+			    clk_register_clk_pxa27x_system_bus());
+	clkdev_pxa_register(CLK_NONE, "memory", NULL,
+			    clk_register_clk_pxa27x_memory());
 	clk_register_clk_pxa27x_lcd_base();
 }
 
diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig
index 0146d3c2547f..5fb8d7430908 100644
--- a/drivers/clk/qcom/Kconfig
+++ b/drivers/clk/qcom/Kconfig
@@ -2,6 +2,9 @@ config QCOM_GDSC
 	bool
 	select PM_GENERIC_DOMAINS if PM
 
+config QCOM_RPMCC
+	bool
+
 config COMMON_CLK_QCOM
 	tristate "Support for Qualcomm's clock controllers"
 	depends on OF
@@ -9,6 +12,32 @@ config COMMON_CLK_QCOM
 	select REGMAP_MMIO
 	select RESET_CONTROLLER
 
+config QCOM_CLK_RPM
+	tristate "RPM based Clock Controller"
+	depends on COMMON_CLK_QCOM && MFD_QCOM_RPM
+	select QCOM_RPMCC
+	help
+	  The RPM (Resource Power Manager) is a dedicated hardware engine for
+	  managing the shared SoC resources in order to keep the lowest power
+	  profile. It communicates with other hardware subsystems via shared
+	  memory and accepts clock requests, aggregates the requests and turns
+	  the clocks on/off or scales them on demand.
+	  Say Y if you want to support the clocks exposed by the RPM on
+	  platforms such as apq8064, msm8660, msm8960 etc.
+
+config QCOM_CLK_SMD_RPM
+	tristate "RPM over SMD based Clock Controller"
+	depends on COMMON_CLK_QCOM && QCOM_SMD_RPM
+	select QCOM_RPMCC
+	help
+	  The RPM (Resource Power Manager) is a dedicated hardware engine for
+	  managing the shared SoC resources in order to keep the lowest power
+	  profile. It communicates with other hardware subsystems via shared
+	  memory and accepts clock requests, aggregates the requests and turns
+	  the clocks on/off or scales them on demand.
+	  Say Y if you want to support the clocks exposed by the RPM on
+	  platforms such as apq8016, apq8084, msm8974 etc.
+
 config APQ_GCC_8084
 	tristate "APQ8084 Global Clock Controller"
 	select QCOM_GDSC
@@ -132,6 +161,14 @@ config MSM_MMCC_8974
 	  Say Y if you want to support multimedia devices such as display,
 	  graphics, video encode/decode, camera, etc.
 
+config MSM_GCC_8994
+	tristate "MSM8994 Global Clock Controller"
+	depends on COMMON_CLK_QCOM
+	help
+	  Support for the global clock controller on msm8994 devices.
+	  Say Y if you want to use peripheral devices such as UART, SPI,
+	  i2c, USB, UFS, SD/eMMC, PCIe, etc.
+
 config MSM_GCC_8996
 	tristate "MSM8996 Global Clock Controller"
 	select QCOM_GDSC
diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile
index 1fb1f5476cb0..1c3e222b917b 100644
--- a/drivers/clk/qcom/Makefile
+++ b/drivers/clk/qcom/Makefile
@@ -24,8 +24,11 @@ obj-$(CONFIG_MSM_GCC_8660) += gcc-msm8660.o
 obj-$(CONFIG_MSM_GCC_8916) += gcc-msm8916.o
 obj-$(CONFIG_MSM_GCC_8960) += gcc-msm8960.o
 obj-$(CONFIG_MSM_GCC_8974) += gcc-msm8974.o
+obj-$(CONFIG_MSM_GCC_8994) += gcc-msm8994.o
 obj-$(CONFIG_MSM_GCC_8996) += gcc-msm8996.o
 obj-$(CONFIG_MSM_LCC_8960) += lcc-msm8960.o
 obj-$(CONFIG_MSM_MMCC_8960) += mmcc-msm8960.o
 obj-$(CONFIG_MSM_MMCC_8974) += mmcc-msm8974.o
 obj-$(CONFIG_MSM_MMCC_8996) += mmcc-msm8996.o
+obj-$(CONFIG_QCOM_CLK_RPM) += clk-rpm.o
+obj-$(CONFIG_QCOM_CLK_SMD_RPM) += clk-smd-rpm.o
diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c
index e6a03eaf7a93..47a1da3739ce 100644
--- a/drivers/clk/qcom/clk-alpha-pll.c
+++ b/drivers/clk/qcom/clk-alpha-pll.c
@@ -18,17 +18,21 @@
 #include <linux/delay.h>
 
 #include "clk-alpha-pll.h"
+#include "common.h"
 
 #define PLL_MODE		0x00
 # define PLL_OUTCTRL		BIT(0)
 # define PLL_BYPASSNL		BIT(1)
 # define PLL_RESET_N		BIT(2)
+# define PLL_OFFLINE_REQ	BIT(7)
 # define PLL_LOCK_COUNT_SHIFT	8
 # define PLL_LOCK_COUNT_MASK	0x3f
 # define PLL_BIAS_COUNT_SHIFT	14
 # define PLL_BIAS_COUNT_MASK	0x3f
 # define PLL_VOTE_FSM_ENA	BIT(20)
+# define PLL_FSM_ENA		BIT(20)
 # define PLL_VOTE_FSM_RESET	BIT(21)
+# define PLL_OFFLINE_ACK	BIT(28)
 # define PLL_ACTIVE_FLAG	BIT(30)
 # define PLL_LOCK_DET		BIT(31)
 
@@ -46,6 +50,7 @@
 #define PLL_USER_CTL_U		0x14
 
 #define PLL_CONFIG_CTL		0x18
+#define PLL_CONFIG_CTL_U	0x20
 #define PLL_TEST_CTL		0x1c
 #define PLL_TEST_CTL_U		0x20
 #define PLL_STATUS		0x24
@@ -55,6 +60,7 @@
  */
 #define ALPHA_REG_BITWIDTH	40
 #define ALPHA_BITWIDTH		32
+#define ALPHA_16BIT_MASK	0xffff
 
 #define to_clk_alpha_pll(_hw) container_of(to_clk_regmap(_hw), \
 					   struct clk_alpha_pll, clkr)
@@ -62,9 +68,10 @@
 #define to_clk_alpha_pll_postdiv(_hw) container_of(to_clk_regmap(_hw), \
 					   struct clk_alpha_pll_postdiv, clkr)
 
-static int wait_for_pll(struct clk_alpha_pll *pll)
+static int wait_for_pll(struct clk_alpha_pll *pll, u32 mask, bool inverse,
+			const char *action)
 {
-	u32 val, mask, off;
+	u32 val, off;
 	int count;
 	int ret;
 	const char *name = clk_hw_get_name(&pll->clkr.hw);
@@ -74,26 +81,148 @@ static int wait_for_pll(struct clk_alpha_pll *pll)
 	if (ret)
 		return ret;
 
-	if (val & PLL_VOTE_FSM_ENA)
-		mask = PLL_ACTIVE_FLAG;
-	else
-		mask = PLL_LOCK_DET;
-
-	/* Wait for pll to enable. */
 	for (count = 100; count > 0; count--) {
 		ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
 		if (ret)
 			return ret;
-		if ((val & mask) == mask)
+		if (inverse && !(val & mask))
+			return 0;
+		else if ((val & mask) == mask)
 			return 0;
 
 		udelay(1);
 	}
 
-	WARN(1, "%s didn't enable after voting for it!\n", name);
+	WARN(1, "%s failed to %s!\n", name, action);
 	return -ETIMEDOUT;
 }
 
+#define wait_for_pll_enable_active(pll) \
+	wait_for_pll(pll, PLL_ACTIVE_FLAG, 0, "enable")
+
+#define wait_for_pll_enable_lock(pll) \
+	wait_for_pll(pll, PLL_LOCK_DET, 0, "enable")
+
+#define wait_for_pll_disable(pll) \
+	wait_for_pll(pll, PLL_ACTIVE_FLAG, 1, "disable")
+
+#define wait_for_pll_offline(pll) \
+	wait_for_pll(pll, PLL_OFFLINE_ACK, 0, "offline")
+
+void clk_alpha_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap,
+			     const struct alpha_pll_config *config)
+{
+	u32 val, mask;
+	u32 off = pll->offset;
+
+	regmap_write(regmap, off + PLL_L_VAL, config->l);
+	regmap_write(regmap, off + PLL_ALPHA_VAL, config->alpha);
+	regmap_write(regmap, off + PLL_CONFIG_CTL, config->config_ctl_val);
+	regmap_write(regmap, off + PLL_CONFIG_CTL_U, config->config_ctl_hi_val);
+
+	val = config->main_output_mask;
+	val |= config->aux_output_mask;
+	val |= config->aux2_output_mask;
+	val |= config->early_output_mask;
+	val |= config->pre_div_val;
+	val |= config->post_div_val;
+	val |= config->vco_val;
+
+	mask = config->main_output_mask;
+	mask |= config->aux_output_mask;
+	mask |= config->aux2_output_mask;
+	mask |= config->early_output_mask;
+	mask |= config->pre_div_mask;
+	mask |= config->post_div_mask;
+	mask |= config->vco_mask;
+
+	regmap_update_bits(regmap, off + PLL_USER_CTL, mask, val);
+
+	if (pll->flags & SUPPORTS_FSM_MODE)
+		qcom_pll_set_fsm_mode(regmap, off + PLL_MODE, 6, 0);
+}
+
+static int clk_alpha_pll_hwfsm_enable(struct clk_hw *hw)
+{
+	int ret;
+	u32 val, off;
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+
+	off = pll->offset;
+	ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+	if (ret)
+		return ret;
+
+	val |= PLL_FSM_ENA;
+
+	if (pll->flags & SUPPORTS_OFFLINE_REQ)
+		val &= ~PLL_OFFLINE_REQ;
+
+	ret = regmap_write(pll->clkr.regmap, off + PLL_MODE, val);
+	if (ret)
+		return ret;
+
+	/* Make sure enable request goes through before waiting for update */
+	mb();
+
+	return wait_for_pll_enable_active(pll);
+}
+
+static void clk_alpha_pll_hwfsm_disable(struct clk_hw *hw)
+{
+	int ret;
+	u32 val, off;
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+
+	off = pll->offset;
+	ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+	if (ret)
+		return;
+
+	if (pll->flags & SUPPORTS_OFFLINE_REQ) {
+		ret = regmap_update_bits(pll->clkr.regmap, off + PLL_MODE,
+					 PLL_OFFLINE_REQ, PLL_OFFLINE_REQ);
+		if (ret)
+			return;
+
+		ret = wait_for_pll_offline(pll);
+		if (ret)
+			return;
+	}
+
+	/* Disable hwfsm */
+	ret = regmap_update_bits(pll->clkr.regmap, off + PLL_MODE,
+				 PLL_FSM_ENA, 0);
+	if (ret)
+		return;
+
+	wait_for_pll_disable(pll);
+}
+
+static int pll_is_enabled(struct clk_hw *hw, u32 mask)
+{
+	int ret;
+	u32 val, off;
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+
+	off = pll->offset;
+	ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+	if (ret)
+		return ret;
+
+	return !!(val & mask);
+}
+
+static int clk_alpha_pll_hwfsm_is_enabled(struct clk_hw *hw)
+{
+	return pll_is_enabled(hw, PLL_ACTIVE_FLAG);
+}
+
+static int clk_alpha_pll_is_enabled(struct clk_hw *hw)
+{
+	return pll_is_enabled(hw, PLL_LOCK_DET);
+}
+
 static int clk_alpha_pll_enable(struct clk_hw *hw)
 {
 	int ret;
@@ -112,7 +241,7 @@ static int clk_alpha_pll_enable(struct clk_hw *hw)
 		ret = clk_enable_regmap(hw);
 		if (ret)
 			return ret;
-		return wait_for_pll(pll);
+		return wait_for_pll_enable_active(pll);
 	}
 
 	/* Skip if already enabled */
@@ -136,7 +265,7 @@ static int clk_alpha_pll_enable(struct clk_hw *hw)
 	if (ret)
 		return ret;
 
-	ret = wait_for_pll(pll);
+	ret = wait_for_pll_enable_lock(pll);
 	if (ret)
 		return ret;
 
@@ -234,9 +363,14 @@ clk_alpha_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
 	regmap_read(pll->clkr.regmap, off + PLL_USER_CTL, &ctl);
 	if (ctl & PLL_ALPHA_EN) {
 		regmap_read(pll->clkr.regmap, off + PLL_ALPHA_VAL, &low);
-		regmap_read(pll->clkr.regmap, off + PLL_ALPHA_VAL_U, &high);
-		a = (u64)high << 32 | low;
-		a >>= ALPHA_REG_BITWIDTH - ALPHA_BITWIDTH;
+		if (pll->flags & SUPPORTS_16BIT_ALPHA) {
+			a = low & ALPHA_16BIT_MASK;
+		} else {
+			regmap_read(pll->clkr.regmap, off + PLL_ALPHA_VAL_U,
+				    &high);
+			a = (u64)high << 32 | low;
+			a >>= ALPHA_REG_BITWIDTH - ALPHA_BITWIDTH;
+		}
 	}
 
 	return alpha_pll_calc_rate(prate, l, a);
@@ -257,11 +391,15 @@ static int clk_alpha_pll_set_rate(struct clk_hw *hw, unsigned long rate,
 		return -EINVAL;
 	}
 
-	a <<= (ALPHA_REG_BITWIDTH - ALPHA_BITWIDTH);
-
 	regmap_write(pll->clkr.regmap, off + PLL_L_VAL, l);
-	regmap_write(pll->clkr.regmap, off + PLL_ALPHA_VAL, a);
-	regmap_write(pll->clkr.regmap, off + PLL_ALPHA_VAL_U, a >> 32);
+
+	if (pll->flags & SUPPORTS_16BIT_ALPHA) {
+		regmap_write(pll->clkr.regmap, off + PLL_ALPHA_VAL,
+			     a & ALPHA_16BIT_MASK);
+	} else {
+		a <<= (ALPHA_REG_BITWIDTH - ALPHA_BITWIDTH);
+		regmap_write(pll->clkr.regmap, off + PLL_ALPHA_VAL_U, a >> 32);
+	}
 
 	regmap_update_bits(pll->clkr.regmap, off + PLL_USER_CTL,
 			   PLL_VCO_MASK << PLL_VCO_SHIFT,
@@ -294,12 +432,23 @@ static long clk_alpha_pll_round_rate(struct clk_hw *hw, unsigned long rate,
 const struct clk_ops clk_alpha_pll_ops = {
 	.enable = clk_alpha_pll_enable,
 	.disable = clk_alpha_pll_disable,
+	.is_enabled = clk_alpha_pll_is_enabled,
 	.recalc_rate = clk_alpha_pll_recalc_rate,
 	.round_rate = clk_alpha_pll_round_rate,
 	.set_rate = clk_alpha_pll_set_rate,
 };
 EXPORT_SYMBOL_GPL(clk_alpha_pll_ops);
 
+const struct clk_ops clk_alpha_pll_hwfsm_ops = {
+	.enable = clk_alpha_pll_hwfsm_enable,
+	.disable = clk_alpha_pll_hwfsm_disable,
+	.is_enabled = clk_alpha_pll_hwfsm_is_enabled,
+	.recalc_rate = clk_alpha_pll_recalc_rate,
+	.round_rate = clk_alpha_pll_round_rate,
+	.set_rate = clk_alpha_pll_set_rate,
+};
+EXPORT_SYMBOL_GPL(clk_alpha_pll_hwfsm_ops);
+
 static unsigned long
 clk_alpha_pll_postdiv_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
 {
diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h
index 90ce2016e1a0..d6e1ee2c7348 100644
--- a/drivers/clk/qcom/clk-alpha-pll.h
+++ b/drivers/clk/qcom/clk-alpha-pll.h
@@ -34,6 +34,10 @@ struct clk_alpha_pll {
 
 	const struct pll_vco *vco_table;
 	size_t num_vco;
+#define SUPPORTS_OFFLINE_REQ	BIT(0)
+#define SUPPORTS_16BIT_ALPHA	BIT(1)
+#define SUPPORTS_FSM_MODE	BIT(2)
+	u8 flags;
 
 	struct clk_regmap clkr;
 };
@@ -51,7 +55,28 @@ struct clk_alpha_pll_postdiv {
 	struct clk_regmap clkr;
 };
 
+struct alpha_pll_config {
+	u32 l;
+	u32 alpha;
+	u32 config_ctl_val;
+	u32 config_ctl_hi_val;
+	u32 main_output_mask;
+	u32 aux_output_mask;
+	u32 aux2_output_mask;
+	u32 early_output_mask;
+	u32 pre_div_val;
+	u32 pre_div_mask;
+	u32 post_div_val;
+	u32 post_div_mask;
+	u32 vco_val;
+	u32 vco_mask;
+};
+
 extern const struct clk_ops clk_alpha_pll_ops;
+extern const struct clk_ops clk_alpha_pll_hwfsm_ops;
 extern const struct clk_ops clk_alpha_pll_postdiv_ops;
 
+void clk_alpha_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap,
+			     const struct alpha_pll_config *config);
+
 #endif
diff --git a/drivers/clk/qcom/clk-pll.c b/drivers/clk/qcom/clk-pll.c
index 5b940d629045..cb6cb8710daf 100644
--- a/drivers/clk/qcom/clk-pll.c
+++ b/drivers/clk/qcom/clk-pll.c
@@ -23,16 +23,11 @@
 #include <asm/div64.h>
 
 #include "clk-pll.h"
+#include "common.h"
 
 #define PLL_OUTCTRL		BIT(0)
 #define PLL_BYPASSNL		BIT(1)
 #define PLL_RESET_N		BIT(2)
-#define PLL_LOCK_COUNT_SHIFT	8
-#define PLL_LOCK_COUNT_MASK	0x3f
-#define PLL_BIAS_COUNT_SHIFT	14
-#define PLL_BIAS_COUNT_MASK	0x3f
-#define PLL_VOTE_FSM_ENA	BIT(20)
-#define PLL_VOTE_FSM_RESET	BIT(21)
 
 static int clk_pll_enable(struct clk_hw *hw)
 {
@@ -228,26 +223,6 @@ const struct clk_ops clk_pll_vote_ops = {
 };
 EXPORT_SYMBOL_GPL(clk_pll_vote_ops);
 
-static void
-clk_pll_set_fsm_mode(struct clk_pll *pll, struct regmap *regmap, u8 lock_count)
-{
-	u32 val;
-	u32 mask;
-
-	/* De-assert reset to FSM */
-	regmap_update_bits(regmap, pll->mode_reg, PLL_VOTE_FSM_RESET, 0);
-
-	/* Program bias count and lock count */
-	val = 1 << PLL_BIAS_COUNT_SHIFT | lock_count << PLL_LOCK_COUNT_SHIFT;
-	mask = PLL_BIAS_COUNT_MASK << PLL_BIAS_COUNT_SHIFT;
-	mask |= PLL_LOCK_COUNT_MASK << PLL_LOCK_COUNT_SHIFT;
-	regmap_update_bits(regmap, pll->mode_reg, mask, val);
-
-	/* Enable PLL FSM voting */
-	regmap_update_bits(regmap, pll->mode_reg, PLL_VOTE_FSM_ENA,
-		PLL_VOTE_FSM_ENA);
-}
-
 static void clk_pll_configure(struct clk_pll *pll, struct regmap *regmap,
 	const struct pll_config *config)
 {
@@ -280,7 +255,7 @@ void clk_pll_configure_sr(struct clk_pll *pll, struct regmap *regmap,
 {
 	clk_pll_configure(pll, regmap, config);
 	if (fsm_mode)
-		clk_pll_set_fsm_mode(pll, regmap, 8);
+		qcom_pll_set_fsm_mode(regmap, pll->mode_reg, 1, 8);
 }
 EXPORT_SYMBOL_GPL(clk_pll_configure_sr);
 
@@ -289,7 +264,7 @@ void clk_pll_configure_sr_hpm_lp(struct clk_pll *pll, struct regmap *regmap,
 {
 	clk_pll_configure(pll, regmap, config);
 	if (fsm_mode)
-		clk_pll_set_fsm_mode(pll, regmap, 0);
+		qcom_pll_set_fsm_mode(regmap, pll->mode_reg, 1, 0);
 }
 EXPORT_SYMBOL_GPL(clk_pll_configure_sr_hpm_lp);
 
diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h
index b904c335cda4..1b3e8d265bdb 100644
--- a/drivers/clk/qcom/clk-rcg.h
+++ b/drivers/clk/qcom/clk-rcg.h
@@ -173,6 +173,7 @@ struct clk_rcg2 {
 #define to_clk_rcg2(_hw) container_of(to_clk_regmap(_hw), struct clk_rcg2, clkr)
 
 extern const struct clk_ops clk_rcg2_ops;
+extern const struct clk_ops clk_rcg2_floor_ops;
 extern const struct clk_ops clk_rcg2_shared_ops;
 extern const struct clk_ops clk_edp_pixel_ops;
 extern const struct clk_ops clk_byte_ops;
diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c
index a071bba8018c..1a0985ae20d2 100644
--- a/drivers/clk/qcom/clk-rcg2.c
+++ b/drivers/clk/qcom/clk-rcg2.c
@@ -47,6 +47,11 @@
 #define N_REG			0xc
 #define D_REG			0x10
 
+enum freq_policy {
+	FLOOR,
+	CEIL,
+};
+
 static int clk_rcg2_is_enabled(struct clk_hw *hw)
 {
 	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
@@ -176,15 +181,26 @@ clk_rcg2_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
 	return calc_rate(parent_rate, m, n, mode, hid_div);
 }
 
-static int _freq_tbl_determine_rate(struct clk_hw *hw,
-		const struct freq_tbl *f, struct clk_rate_request *req)
+static int _freq_tbl_determine_rate(struct clk_hw *hw, const struct freq_tbl *f,
+				    struct clk_rate_request *req,
+				    enum freq_policy policy)
 {
 	unsigned long clk_flags, rate = req->rate;
 	struct clk_hw *p;
 	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
 	int index;
 
-	f = qcom_find_freq(f, rate);
+	switch (policy) {
+	case FLOOR:
+		f = qcom_find_freq_floor(f, rate);
+		break;
+	case CEIL:
+		f = qcom_find_freq(f, rate);
+		break;
+	default:
+		return -EINVAL;
+	};
+
 	if (!f)
 		return -EINVAL;
 
@@ -221,7 +237,15 @@ static int clk_rcg2_determine_rate(struct clk_hw *hw,
 {
 	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
 
-	return _freq_tbl_determine_rate(hw, rcg->freq_tbl, req);
+	return _freq_tbl_determine_rate(hw, rcg->freq_tbl, req, CEIL);
+}
+
+static int clk_rcg2_determine_floor_rate(struct clk_hw *hw,
+					 struct clk_rate_request *req)
+{
+	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
+
+	return _freq_tbl_determine_rate(hw, rcg->freq_tbl, req, FLOOR);
 }
 
 static int clk_rcg2_configure(struct clk_rcg2 *rcg, const struct freq_tbl *f)
@@ -265,12 +289,23 @@ static int clk_rcg2_configure(struct clk_rcg2 *rcg, const struct freq_tbl *f)
 	return update_config(rcg);
 }
 
-static int __clk_rcg2_set_rate(struct clk_hw *hw, unsigned long rate)
+static int __clk_rcg2_set_rate(struct clk_hw *hw, unsigned long rate,
+			       enum freq_policy policy)
 {
 	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
 	const struct freq_tbl *f;
 
-	f = qcom_find_freq(rcg->freq_tbl, rate);
+	switch (policy) {
+	case FLOOR:
+		f = qcom_find_freq_floor(rcg->freq_tbl, rate);
+		break;
+	case CEIL:
+		f = qcom_find_freq(rcg->freq_tbl, rate);
+		break;
+	default:
+		return -EINVAL;
+	};
+
 	if (!f)
 		return -EINVAL;
 
@@ -280,13 +315,25 @@ static int __clk_rcg2_set_rate(struct clk_hw *hw, unsigned long rate)
 static int clk_rcg2_set_rate(struct clk_hw *hw, unsigned long rate,
 			    unsigned long parent_rate)
 {
-	return __clk_rcg2_set_rate(hw, rate);
+	return __clk_rcg2_set_rate(hw, rate, CEIL);
+}
+
+static int clk_rcg2_set_floor_rate(struct clk_hw *hw, unsigned long rate,
+				   unsigned long parent_rate)
+{
+	return __clk_rcg2_set_rate(hw, rate, FLOOR);
 }
 
 static int clk_rcg2_set_rate_and_parent(struct clk_hw *hw,
 		unsigned long rate, unsigned long parent_rate, u8 index)
 {
-	return __clk_rcg2_set_rate(hw, rate);
+	return __clk_rcg2_set_rate(hw, rate, CEIL);
+}
+
+static int clk_rcg2_set_floor_rate_and_parent(struct clk_hw *hw,
+		unsigned long rate, unsigned long parent_rate, u8 index)
+{
+	return __clk_rcg2_set_rate(hw, rate, FLOOR);
 }
 
 const struct clk_ops clk_rcg2_ops = {
@@ -300,6 +347,17 @@ const struct clk_ops clk_rcg2_ops = {
 };
 EXPORT_SYMBOL_GPL(clk_rcg2_ops);
 
+const struct clk_ops clk_rcg2_floor_ops = {
+	.is_enabled = clk_rcg2_is_enabled,
+	.get_parent = clk_rcg2_get_parent,
+	.set_parent = clk_rcg2_set_parent,
+	.recalc_rate = clk_rcg2_recalc_rate,
+	.determine_rate = clk_rcg2_determine_floor_rate,
+	.set_rate = clk_rcg2_set_floor_rate,
+	.set_rate_and_parent = clk_rcg2_set_floor_rate_and_parent,
+};
+EXPORT_SYMBOL_GPL(clk_rcg2_floor_ops);
+
 static int clk_rcg2_shared_force_enable(struct clk_hw *hw, unsigned long rate)
 {
 	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
@@ -323,7 +381,7 @@ static int clk_rcg2_shared_force_enable(struct clk_hw *hw, unsigned long rate)
 		pr_err("%s: RCG did not turn on\n", name);
 
 	/* set clock rate */
-	ret = __clk_rcg2_set_rate(hw, rate);
+	ret = __clk_rcg2_set_rate(hw, rate, CEIL);
 	if (ret)
 		return ret;
 
diff --git a/drivers/clk/qcom/clk-rpm.c b/drivers/clk/qcom/clk-rpm.c
new file mode 100644
index 000000000000..df3e5fe8442a
--- /dev/null
+++ b/drivers/clk/qcom/clk-rpm.c
@@ -0,0 +1,497 @@
+/*
+ * Copyright (c) 2016, Linaro Limited
+ * Copyright (c) 2014, The Linux Foundation. All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/err.h>
+#include <linux/export.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/mfd/qcom_rpm.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+#include <dt-bindings/mfd/qcom-rpm.h>
+#include <dt-bindings/clock/qcom,rpmcc.h>
+
+#define QCOM_RPM_MISC_CLK_TYPE				0x306b6c63
+#define QCOM_RPM_SCALING_ENABLE_ID			0x2
+
+#define DEFINE_CLK_RPM(_platform, _name, _active, r_id)			      \
+	static struct clk_rpm _platform##_##_active;			      \
+	static struct clk_rpm _platform##_##_name = {			      \
+		.rpm_clk_id = (r_id),					      \
+		.peer = &_platform##_##_active,				      \
+		.rate = INT_MAX,					      \
+		.hw.init = &(struct clk_init_data){			      \
+			.ops = &clk_rpm_ops,				      \
+			.name = #_name,					      \
+			.parent_names = (const char *[]){ "pxo_board" },      \
+			.num_parents = 1,				      \
+		},							      \
+	};								      \
+	static struct clk_rpm _platform##_##_active = {			      \
+		.rpm_clk_id = (r_id),					      \
+		.peer = &_platform##_##_name,				      \
+		.active_only = true,					      \
+		.rate = INT_MAX,					      \
+		.hw.init = &(struct clk_init_data){			      \
+			.ops = &clk_rpm_ops,				      \
+			.name = #_active,				      \
+			.parent_names = (const char *[]){ "pxo_board" },      \
+			.num_parents = 1,				      \
+		},							      \
+	}
+
+#define DEFINE_CLK_RPM_PXO_BRANCH(_platform, _name, _active, r_id, r)	      \
+	static struct clk_rpm _platform##_##_active;			      \
+	static struct clk_rpm _platform##_##_name = {			      \
+		.rpm_clk_id = (r_id),					      \
+		.active_only = true,					      \
+		.peer = &_platform##_##_active,				      \
+		.rate = (r),						      \
+		.branch = true,						      \
+		.hw.init = &(struct clk_init_data){			      \
+			.ops = &clk_rpm_branch_ops,			      \
+			.name = #_name,					      \
+			.parent_names = (const char *[]){ "pxo_board" },      \
+			.num_parents = 1,				      \
+		},							      \
+	};								      \
+	static struct clk_rpm _platform##_##_active = {			      \
+		.rpm_clk_id = (r_id),					      \
+		.peer = &_platform##_##_name,				      \
+		.rate = (r),						      \
+		.branch = true,						      \
+		.hw.init = &(struct clk_init_data){			      \
+			.ops = &clk_rpm_branch_ops,			      \
+			.name = #_active,				      \
+			.parent_names = (const char *[]){ "pxo_board" },      \
+			.num_parents = 1,				      \
+		},							      \
+	}
+
+#define DEFINE_CLK_RPM_CXO_BRANCH(_platform, _name, _active, r_id, r)	      \
+	static struct clk_rpm _platform##_##_active;			      \
+	static struct clk_rpm _platform##_##_name = {			      \
+		.rpm_clk_id = (r_id),					      \
+		.peer = &_platform##_##_active,				      \
+		.rate = (r),						      \
+		.branch = true,						      \
+		.hw.init = &(struct clk_init_data){			      \
+			.ops = &clk_rpm_branch_ops,			      \
+			.name = #_name,					      \
+			.parent_names = (const char *[]){ "cxo_board" },      \
+			.num_parents = 1,				      \
+		},							      \
+	};								      \
+	static struct clk_rpm _platform##_##_active = {			      \
+		.rpm_clk_id = (r_id),					      \
+		.active_only = true,					      \
+		.peer = &_platform##_##_name,				      \
+		.rate = (r),						      \
+		.branch = true,						      \
+		.hw.init = &(struct clk_init_data){			      \
+			.ops = &clk_rpm_branch_ops,			      \
+			.name = #_active,				      \
+			.parent_names = (const char *[]){ "cxo_board" },      \
+			.num_parents = 1,				      \
+		},							      \
+	}
+
+#define to_clk_rpm(_hw) container_of(_hw, struct clk_rpm, hw)
+
+struct clk_rpm {
+	const int rpm_clk_id;
+	const bool active_only;
+	unsigned long rate;
+	bool enabled;
+	bool branch;
+	struct clk_rpm *peer;
+	struct clk_hw hw;
+	struct qcom_rpm *rpm;
+};
+
+struct rpm_cc {
+	struct qcom_rpm *rpm;
+	struct clk_rpm **clks;
+	size_t num_clks;
+};
+
+struct rpm_clk_desc {
+	struct clk_rpm **clks;
+	size_t num_clks;
+};
+
+static DEFINE_MUTEX(rpm_clk_lock);
+
+static int clk_rpm_handoff(struct clk_rpm *r)
+{
+	int ret;
+	u32 value = INT_MAX;
+
+	ret = qcom_rpm_write(r->rpm, QCOM_RPM_ACTIVE_STATE,
+			     r->rpm_clk_id, &value, 1);
+	if (ret)
+		return ret;
+	ret = qcom_rpm_write(r->rpm, QCOM_RPM_SLEEP_STATE,
+			     r->rpm_clk_id, &value, 1);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int clk_rpm_set_rate_active(struct clk_rpm *r, unsigned long rate)
+{
+	u32 value = DIV_ROUND_UP(rate, 1000); /* to kHz */
+
+	return qcom_rpm_write(r->rpm, QCOM_RPM_ACTIVE_STATE,
+			      r->rpm_clk_id, &value, 1);
+}
+
+static int clk_rpm_set_rate_sleep(struct clk_rpm *r, unsigned long rate)
+{
+	u32 value = DIV_ROUND_UP(rate, 1000); /* to kHz */
+
+	return qcom_rpm_write(r->rpm, QCOM_RPM_SLEEP_STATE,
+			      r->rpm_clk_id, &value, 1);
+}
+
+static void to_active_sleep(struct clk_rpm *r, unsigned long rate,
+			    unsigned long *active, unsigned long *sleep)
+{
+	*active = rate;
+
+	/*
+	 * Active-only clocks don't care what the rate is during sleep. So,
+	 * they vote for zero.
+	 */
+	if (r->active_only)
+		*sleep = 0;
+	else
+		*sleep = *active;
+}
+
+static int clk_rpm_prepare(struct clk_hw *hw)
+{
+	struct clk_rpm *r = to_clk_rpm(hw);
+	struct clk_rpm *peer = r->peer;
+	unsigned long this_rate = 0, this_sleep_rate = 0;
+	unsigned long peer_rate = 0, peer_sleep_rate = 0;
+	unsigned long active_rate, sleep_rate;
+	int ret = 0;
+
+	mutex_lock(&rpm_clk_lock);
+
+	/* Don't send requests to the RPM if the rate has not been set. */
+	if (!r->rate)
+		goto out;
+
+	to_active_sleep(r, r->rate, &this_rate, &this_sleep_rate);
+
+	/* Take peer clock's rate into account only if it's enabled. */
+	if (peer->enabled)
+		to_active_sleep(peer, peer->rate,
+				&peer_rate, &peer_sleep_rate);
+
+	active_rate = max(this_rate, peer_rate);
+
+	if (r->branch)
+		active_rate = !!active_rate;
+
+	ret = clk_rpm_set_rate_active(r, active_rate);
+	if (ret)
+		goto out;
+
+	sleep_rate = max(this_sleep_rate, peer_sleep_rate);
+	if (r->branch)
+		sleep_rate = !!sleep_rate;
+
+	ret = clk_rpm_set_rate_sleep(r, sleep_rate);
+	if (ret)
+		/* Undo the active set vote and restore it */
+		ret = clk_rpm_set_rate_active(r, peer_rate);
+
+out:
+	if (!ret)
+		r->enabled = true;
+
+	mutex_unlock(&rpm_clk_lock);
+
+	return ret;
+}
+
+static void clk_rpm_unprepare(struct clk_hw *hw)
+{
+	struct clk_rpm *r = to_clk_rpm(hw);
+	struct clk_rpm *peer = r->peer;
+	unsigned long peer_rate = 0, peer_sleep_rate = 0;
+	unsigned long active_rate, sleep_rate;
+	int ret;
+
+	mutex_lock(&rpm_clk_lock);
+
+	if (!r->rate)
+		goto out;
+
+	/* Take peer clock's rate into account only if it's enabled. */
+	if (peer->enabled)
+		to_active_sleep(peer, peer->rate, &peer_rate,
+				&peer_sleep_rate);
+
+	active_rate = r->branch ? !!peer_rate : peer_rate;
+	ret = clk_rpm_set_rate_active(r, active_rate);
+	if (ret)
+		goto out;
+
+	sleep_rate = r->branch ? !!peer_sleep_rate : peer_sleep_rate;
+	ret = clk_rpm_set_rate_sleep(r, sleep_rate);
+	if (ret)
+		goto out;
+
+	r->enabled = false;
+
+out:
+	mutex_unlock(&rpm_clk_lock);
+}
+
+static int clk_rpm_set_rate(struct clk_hw *hw,
+			    unsigned long rate, unsigned long parent_rate)
+{
+	struct clk_rpm *r = to_clk_rpm(hw);
+	struct clk_rpm *peer = r->peer;
+	unsigned long active_rate, sleep_rate;
+	unsigned long this_rate = 0, this_sleep_rate = 0;
+	unsigned long peer_rate = 0, peer_sleep_rate = 0;
+	int ret = 0;
+
+	mutex_lock(&rpm_clk_lock);
+
+	if (!r->enabled)
+		goto out;
+
+	to_active_sleep(r, rate, &this_rate, &this_sleep_rate);
+
+	/* Take peer clock's rate into account only if it's enabled. */
+	if (peer->enabled)
+		to_active_sleep(peer, peer->rate,
+				&peer_rate, &peer_sleep_rate);
+
+	active_rate = max(this_rate, peer_rate);
+	ret = clk_rpm_set_rate_active(r, active_rate);
+	if (ret)
+		goto out;
+
+	sleep_rate = max(this_sleep_rate, peer_sleep_rate);
+	ret = clk_rpm_set_rate_sleep(r, sleep_rate);
+	if (ret)
+		goto out;
+
+	r->rate = rate;
+
+out:
+	mutex_unlock(&rpm_clk_lock);
+
+	return ret;
+}
+
+static long clk_rpm_round_rate(struct clk_hw *hw, unsigned long rate,
+			       unsigned long *parent_rate)
+{
+	/*
+	 * RPM handles rate rounding and we don't have a way to
+	 * know what the rate will be, so just return whatever
+	 * rate is requested.
+	 */
+	return rate;
+}
+
+static unsigned long clk_rpm_recalc_rate(struct clk_hw *hw,
+					 unsigned long parent_rate)
+{
+	struct clk_rpm *r = to_clk_rpm(hw);
+
+	/*
+	 * RPM handles rate rounding and we don't have a way to
+	 * know what the rate will be, so just return whatever
+	 * rate was set.
+	 */
+	return r->rate;
+}
+
+static const struct clk_ops clk_rpm_ops = {
+	.prepare	= clk_rpm_prepare,
+	.unprepare	= clk_rpm_unprepare,
+	.set_rate	= clk_rpm_set_rate,
+	.round_rate	= clk_rpm_round_rate,
+	.recalc_rate	= clk_rpm_recalc_rate,
+};
+
+static const struct clk_ops clk_rpm_branch_ops = {
+	.prepare	= clk_rpm_prepare,
+	.unprepare	= clk_rpm_unprepare,
+	.round_rate	= clk_rpm_round_rate,
+	.recalc_rate	= clk_rpm_recalc_rate,
+};
+
+/* apq8064 */
+DEFINE_CLK_RPM(apq8064, afab_clk, afab_a_clk, QCOM_RPM_APPS_FABRIC_CLK);
+DEFINE_CLK_RPM(apq8064, cfpb_clk, cfpb_a_clk, QCOM_RPM_CFPB_CLK);
+DEFINE_CLK_RPM(apq8064, daytona_clk, daytona_a_clk, QCOM_RPM_DAYTONA_FABRIC_CLK);
+DEFINE_CLK_RPM(apq8064, ebi1_clk, ebi1_a_clk, QCOM_RPM_EBI1_CLK);
+DEFINE_CLK_RPM(apq8064, mmfab_clk, mmfab_a_clk, QCOM_RPM_MM_FABRIC_CLK);
+DEFINE_CLK_RPM(apq8064, mmfpb_clk, mmfpb_a_clk, QCOM_RPM_MMFPB_CLK);
+DEFINE_CLK_RPM(apq8064, sfab_clk, sfab_a_clk, QCOM_RPM_SYS_FABRIC_CLK);
+DEFINE_CLK_RPM(apq8064, sfpb_clk, sfpb_a_clk, QCOM_RPM_SFPB_CLK);
+DEFINE_CLK_RPM(apq8064, qdss_clk, qdss_a_clk, QCOM_RPM_QDSS_CLK);
+
+static struct clk_rpm *apq8064_clks[] = {
+	[RPM_APPS_FABRIC_CLK] = &apq8064_afab_clk,
+	[RPM_APPS_FABRIC_A_CLK] = &apq8064_afab_a_clk,
+	[RPM_CFPB_CLK] = &apq8064_cfpb_clk,
+	[RPM_CFPB_A_CLK] = &apq8064_cfpb_a_clk,
+	[RPM_DAYTONA_FABRIC_CLK] = &apq8064_daytona_clk,
+	[RPM_DAYTONA_FABRIC_A_CLK] = &apq8064_daytona_a_clk,
+	[RPM_EBI1_CLK] = &apq8064_ebi1_clk,
+	[RPM_EBI1_A_CLK] = &apq8064_ebi1_a_clk,
+	[RPM_MM_FABRIC_CLK] = &apq8064_mmfab_clk,
+	[RPM_MM_FABRIC_A_CLK] = &apq8064_mmfab_a_clk,
+	[RPM_MMFPB_CLK] = &apq8064_mmfpb_clk,
+	[RPM_MMFPB_A_CLK] = &apq8064_mmfpb_a_clk,
+	[RPM_SYS_FABRIC_CLK] = &apq8064_sfab_clk,
+	[RPM_SYS_FABRIC_A_CLK] = &apq8064_sfab_a_clk,
+	[RPM_SFPB_CLK] = &apq8064_sfpb_clk,
+	[RPM_SFPB_A_CLK] = &apq8064_sfpb_a_clk,
+	[RPM_QDSS_CLK] = &apq8064_qdss_clk,
+	[RPM_QDSS_A_CLK] = &apq8064_qdss_a_clk,
+};
+
+static const struct rpm_clk_desc rpm_clk_apq8064 = {
+	.clks = apq8064_clks,
+	.num_clks = ARRAY_SIZE(apq8064_clks),
+};
+
+static const struct of_device_id rpm_clk_match_table[] = {
+	{ .compatible = "qcom,rpmcc-apq8064", .data = &rpm_clk_apq8064 },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, rpm_clk_match_table);
+
+static struct clk_hw *qcom_rpm_clk_hw_get(struct of_phandle_args *clkspec,
+					  void *data)
+{
+	struct rpm_cc *rcc = data;
+	unsigned int idx = clkspec->args[0];
+
+	if (idx >= rcc->num_clks) {
+		pr_err("%s: invalid index %u\n", __func__, idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return rcc->clks[idx] ? &rcc->clks[idx]->hw : ERR_PTR(-ENOENT);
+}
+
+static int rpm_clk_probe(struct platform_device *pdev)
+{
+	struct rpm_cc *rcc;
+	int ret;
+	size_t num_clks, i;
+	struct qcom_rpm *rpm;
+	struct clk_rpm **rpm_clks;
+	const struct rpm_clk_desc *desc;
+
+	rpm = dev_get_drvdata(pdev->dev.parent);
+	if (!rpm) {
+		dev_err(&pdev->dev, "Unable to retrieve handle to RPM\n");
+		return -ENODEV;
+	}
+
+	desc = of_device_get_match_data(&pdev->dev);
+	if (!desc)
+		return -EINVAL;
+
+	rpm_clks = desc->clks;
+	num_clks = desc->num_clks;
+
+	rcc = devm_kzalloc(&pdev->dev, sizeof(*rcc), GFP_KERNEL);
+	if (!rcc)
+		return -ENOMEM;
+
+	rcc->clks = rpm_clks;
+	rcc->num_clks = num_clks;
+
+	for (i = 0; i < num_clks; i++) {
+		if (!rpm_clks[i])
+			continue;
+
+		rpm_clks[i]->rpm = rpm;
+
+		ret = clk_rpm_handoff(rpm_clks[i]);
+		if (ret)
+			goto err;
+	}
+
+	for (i = 0; i < num_clks; i++) {
+		if (!rpm_clks[i])
+			continue;
+
+		ret = devm_clk_hw_register(&pdev->dev, &rpm_clks[i]->hw);
+		if (ret)
+			goto err;
+	}
+
+	ret = of_clk_add_hw_provider(pdev->dev.of_node, qcom_rpm_clk_hw_get,
+				     rcc);
+	if (ret)
+		goto err;
+
+	return 0;
+err:
+	dev_err(&pdev->dev, "Error registering RPM Clock driver (%d)\n", ret);
+	return ret;
+}
+
+static int rpm_clk_remove(struct platform_device *pdev)
+{
+	of_clk_del_provider(pdev->dev.of_node);
+	return 0;
+}
+
+static struct platform_driver rpm_clk_driver = {
+	.driver = {
+		.name = "qcom-clk-rpm",
+		.of_match_table = rpm_clk_match_table,
+	},
+	.probe = rpm_clk_probe,
+	.remove = rpm_clk_remove,
+};
+
+static int __init rpm_clk_init(void)
+{
+	return platform_driver_register(&rpm_clk_driver);
+}
+core_initcall(rpm_clk_init);
+
+static void __exit rpm_clk_exit(void)
+{
+	platform_driver_unregister(&rpm_clk_driver);
+}
+module_exit(rpm_clk_exit);
+
+MODULE_DESCRIPTION("Qualcomm RPM Clock Controller Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:qcom-clk-rpm");
diff --git a/drivers/clk/qcom/clk-smd-rpm.c b/drivers/clk/qcom/clk-smd-rpm.c
new file mode 100644
index 000000000000..07e2cc6ed781
--- /dev/null
+++ b/drivers/clk/qcom/clk-smd-rpm.c
@@ -0,0 +1,578 @@
+/*
+ * Copyright (c) 2016, Linaro Limited
+ * Copyright (c) 2014, The Linux Foundation. All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/err.h>
+#include <linux/export.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/soc/qcom/smd-rpm.h>
+
+#include <dt-bindings/clock/qcom,rpmcc.h>
+#include <dt-bindings/mfd/qcom-rpm.h>
+
+#define QCOM_RPM_KEY_SOFTWARE_ENABLE			0x6e657773
+#define QCOM_RPM_KEY_PIN_CTRL_CLK_BUFFER_ENABLE_KEY	0x62636370
+#define QCOM_RPM_SMD_KEY_RATE				0x007a484b
+#define QCOM_RPM_SMD_KEY_ENABLE				0x62616e45
+#define QCOM_RPM_SMD_KEY_STATE				0x54415453
+#define QCOM_RPM_SCALING_ENABLE_ID			0x2
+
+#define __DEFINE_CLK_SMD_RPM(_platform, _name, _active, type, r_id, stat_id,  \
+			     key)					      \
+	static struct clk_smd_rpm _platform##_##_active;		      \
+	static struct clk_smd_rpm _platform##_##_name = {		      \
+		.rpm_res_type = (type),					      \
+		.rpm_clk_id = (r_id),					      \
+		.rpm_status_id = (stat_id),				      \
+		.rpm_key = (key),					      \
+		.peer = &_platform##_##_active,				      \
+		.rate = INT_MAX,					      \
+		.hw.init = &(struct clk_init_data){			      \
+			.ops = &clk_smd_rpm_ops,			      \
+			.name = #_name,					      \
+			.parent_names = (const char *[]){ "xo_board" },       \
+			.num_parents = 1,				      \
+		},							      \
+	};								      \
+	static struct clk_smd_rpm _platform##_##_active = {		      \
+		.rpm_res_type = (type),					      \
+		.rpm_clk_id = (r_id),					      \
+		.rpm_status_id = (stat_id),				      \
+		.active_only = true,					      \
+		.rpm_key = (key),					      \
+		.peer = &_platform##_##_name,				      \
+		.rate = INT_MAX,					      \
+		.hw.init = &(struct clk_init_data){			      \
+			.ops = &clk_smd_rpm_ops,			      \
+			.name = #_active,				      \
+			.parent_names = (const char *[]){ "xo_board" },	      \
+			.num_parents = 1,				      \
+		},							      \
+	}
+
+#define __DEFINE_CLK_SMD_RPM_BRANCH(_platform, _name, _active, type, r_id,    \
+				    stat_id, r, key)			      \
+	static struct clk_smd_rpm _platform##_##_active;		      \
+	static struct clk_smd_rpm _platform##_##_name = {		      \
+		.rpm_res_type = (type),					      \
+		.rpm_clk_id = (r_id),					      \
+		.rpm_status_id = (stat_id),				      \
+		.rpm_key = (key),					      \
+		.branch = true,						      \
+		.peer = &_platform##_##_active,				      \
+		.rate = (r),						      \
+		.hw.init = &(struct clk_init_data){			      \
+			.ops = &clk_smd_rpm_branch_ops,			      \
+			.name = #_name,					      \
+			.parent_names = (const char *[]){ "xo_board" },	      \
+			.num_parents = 1,				      \
+		},							      \
+	};								      \
+	static struct clk_smd_rpm _platform##_##_active = {		      \
+		.rpm_res_type = (type),					      \
+		.rpm_clk_id = (r_id),					      \
+		.rpm_status_id = (stat_id),				      \
+		.active_only = true,					      \
+		.rpm_key = (key),					      \
+		.branch = true,						      \
+		.peer = &_platform##_##_name,				      \
+		.rate = (r),						      \
+		.hw.init = &(struct clk_init_data){			      \
+			.ops = &clk_smd_rpm_branch_ops,			      \
+			.name = #_active,				      \
+			.parent_names = (const char *[]){ "xo_board" },	      \
+			.num_parents = 1,				      \
+		},							      \
+	}
+
+#define DEFINE_CLK_SMD_RPM(_platform, _name, _active, type, r_id)	      \
+		__DEFINE_CLK_SMD_RPM(_platform, _name, _active, type, r_id,   \
+		0, QCOM_RPM_SMD_KEY_RATE)
+
+#define DEFINE_CLK_SMD_RPM_BRANCH(_platform, _name, _active, type, r_id, r)   \
+		__DEFINE_CLK_SMD_RPM_BRANCH(_platform, _name, _active, type,  \
+		r_id, 0, r, QCOM_RPM_SMD_KEY_ENABLE)
+
+#define DEFINE_CLK_SMD_RPM_QDSS(_platform, _name, _active, type, r_id)	      \
+		__DEFINE_CLK_SMD_RPM(_platform, _name, _active, type, r_id,   \
+		0, QCOM_RPM_SMD_KEY_STATE)
+
+#define DEFINE_CLK_SMD_RPM_XO_BUFFER(_platform, _name, _active, r_id)	      \
+		__DEFINE_CLK_SMD_RPM_BRANCH(_platform, _name, _active,	      \
+		QCOM_SMD_RPM_CLK_BUF_A, r_id, 0, 1000,			      \
+		QCOM_RPM_KEY_SOFTWARE_ENABLE)
+
+#define DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(_platform, _name, _active, r_id) \
+		__DEFINE_CLK_SMD_RPM_BRANCH(_platform, _name, _active,	      \
+		QCOM_SMD_RPM_CLK_BUF_A, r_id, 0, 1000,			      \
+		QCOM_RPM_KEY_PIN_CTRL_CLK_BUFFER_ENABLE_KEY)
+
+#define to_clk_smd_rpm(_hw) container_of(_hw, struct clk_smd_rpm, hw)
+
+struct clk_smd_rpm {
+	const int rpm_res_type;
+	const int rpm_key;
+	const int rpm_clk_id;
+	const int rpm_status_id;
+	const bool active_only;
+	bool enabled;
+	bool branch;
+	struct clk_smd_rpm *peer;
+	struct clk_hw hw;
+	unsigned long rate;
+	struct qcom_smd_rpm *rpm;
+};
+
+struct clk_smd_rpm_req {
+	__le32 key;
+	__le32 nbytes;
+	__le32 value;
+};
+
+struct rpm_cc {
+	struct qcom_rpm *rpm;
+	struct clk_smd_rpm **clks;
+	size_t num_clks;
+};
+
+struct rpm_smd_clk_desc {
+	struct clk_smd_rpm **clks;
+	size_t num_clks;
+};
+
+static DEFINE_MUTEX(rpm_smd_clk_lock);
+
+static int clk_smd_rpm_handoff(struct clk_smd_rpm *r)
+{
+	int ret;
+	struct clk_smd_rpm_req req = {
+		.key = cpu_to_le32(r->rpm_key),
+		.nbytes = cpu_to_le32(sizeof(u32)),
+		.value = cpu_to_le32(INT_MAX),
+	};
+
+	ret = qcom_rpm_smd_write(r->rpm, QCOM_SMD_RPM_ACTIVE_STATE,
+				 r->rpm_res_type, r->rpm_clk_id, &req,
+				 sizeof(req));
+	if (ret)
+		return ret;
+	ret = qcom_rpm_smd_write(r->rpm, QCOM_SMD_RPM_SLEEP_STATE,
+				 r->rpm_res_type, r->rpm_clk_id, &req,
+				 sizeof(req));
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int clk_smd_rpm_set_rate_active(struct clk_smd_rpm *r,
+				       unsigned long rate)
+{
+	struct clk_smd_rpm_req req = {
+		.key = cpu_to_le32(r->rpm_key),
+		.nbytes = cpu_to_le32(sizeof(u32)),
+		.value = cpu_to_le32(DIV_ROUND_UP(rate, 1000)), /* to kHz */
+	};
+
+	return qcom_rpm_smd_write(r->rpm, QCOM_SMD_RPM_ACTIVE_STATE,
+				  r->rpm_res_type, r->rpm_clk_id, &req,
+				  sizeof(req));
+}
+
+static int clk_smd_rpm_set_rate_sleep(struct clk_smd_rpm *r,
+				      unsigned long rate)
+{
+	struct clk_smd_rpm_req req = {
+		.key = cpu_to_le32(r->rpm_key),
+		.nbytes = cpu_to_le32(sizeof(u32)),
+		.value = cpu_to_le32(DIV_ROUND_UP(rate, 1000)), /* to kHz */
+	};
+
+	return qcom_rpm_smd_write(r->rpm, QCOM_SMD_RPM_SLEEP_STATE,
+				  r->rpm_res_type, r->rpm_clk_id, &req,
+				  sizeof(req));
+}
+
+static void to_active_sleep(struct clk_smd_rpm *r, unsigned long rate,
+			    unsigned long *active, unsigned long *sleep)
+{
+	*active = rate;
+
+	/*
+	 * Active-only clocks don't care what the rate is during sleep. So,
+	 * they vote for zero.
+	 */
+	if (r->active_only)
+		*sleep = 0;
+	else
+		*sleep = *active;
+}
+
+static int clk_smd_rpm_prepare(struct clk_hw *hw)
+{
+	struct clk_smd_rpm *r = to_clk_smd_rpm(hw);
+	struct clk_smd_rpm *peer = r->peer;
+	unsigned long this_rate = 0, this_sleep_rate = 0;
+	unsigned long peer_rate = 0, peer_sleep_rate = 0;
+	unsigned long active_rate, sleep_rate;
+	int ret = 0;
+
+	mutex_lock(&rpm_smd_clk_lock);
+
+	/* Don't send requests to the RPM if the rate has not been set. */
+	if (!r->rate)
+		goto out;
+
+	to_active_sleep(r, r->rate, &this_rate, &this_sleep_rate);
+
+	/* Take peer clock's rate into account only if it's enabled. */
+	if (peer->enabled)
+		to_active_sleep(peer, peer->rate,
+				&peer_rate, &peer_sleep_rate);
+
+	active_rate = max(this_rate, peer_rate);
+
+	if (r->branch)
+		active_rate = !!active_rate;
+
+	ret = clk_smd_rpm_set_rate_active(r, active_rate);
+	if (ret)
+		goto out;
+
+	sleep_rate = max(this_sleep_rate, peer_sleep_rate);
+	if (r->branch)
+		sleep_rate = !!sleep_rate;
+
+	ret = clk_smd_rpm_set_rate_sleep(r, sleep_rate);
+	if (ret)
+		/* Undo the active set vote and restore it */
+		ret = clk_smd_rpm_set_rate_active(r, peer_rate);
+
+out:
+	if (!ret)
+		r->enabled = true;
+
+	mutex_unlock(&rpm_smd_clk_lock);
+
+	return ret;
+}
+
+static void clk_smd_rpm_unprepare(struct clk_hw *hw)
+{
+	struct clk_smd_rpm *r = to_clk_smd_rpm(hw);
+	struct clk_smd_rpm *peer = r->peer;
+	unsigned long peer_rate = 0, peer_sleep_rate = 0;
+	unsigned long active_rate, sleep_rate;
+	int ret;
+
+	mutex_lock(&rpm_smd_clk_lock);
+
+	if (!r->rate)
+		goto out;
+
+	/* Take peer clock's rate into account only if it's enabled. */
+	if (peer->enabled)
+		to_active_sleep(peer, peer->rate, &peer_rate,
+				&peer_sleep_rate);
+
+	active_rate = r->branch ? !!peer_rate : peer_rate;
+	ret = clk_smd_rpm_set_rate_active(r, active_rate);
+	if (ret)
+		goto out;
+
+	sleep_rate = r->branch ? !!peer_sleep_rate : peer_sleep_rate;
+	ret = clk_smd_rpm_set_rate_sleep(r, sleep_rate);
+	if (ret)
+		goto out;
+
+	r->enabled = false;
+
+out:
+	mutex_unlock(&rpm_smd_clk_lock);
+}
+
+static int clk_smd_rpm_set_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long parent_rate)
+{
+	struct clk_smd_rpm *r = to_clk_smd_rpm(hw);
+	struct clk_smd_rpm *peer = r->peer;
+	unsigned long active_rate, sleep_rate;
+	unsigned long this_rate = 0, this_sleep_rate = 0;
+	unsigned long peer_rate = 0, peer_sleep_rate = 0;
+	int ret = 0;
+
+	mutex_lock(&rpm_smd_clk_lock);
+
+	if (!r->enabled)
+		goto out;
+
+	to_active_sleep(r, rate, &this_rate, &this_sleep_rate);
+
+	/* Take peer clock's rate into account only if it's enabled. */
+	if (peer->enabled)
+		to_active_sleep(peer, peer->rate,
+				&peer_rate, &peer_sleep_rate);
+
+	active_rate = max(this_rate, peer_rate);
+	ret = clk_smd_rpm_set_rate_active(r, active_rate);
+	if (ret)
+		goto out;
+
+	sleep_rate = max(this_sleep_rate, peer_sleep_rate);
+	ret = clk_smd_rpm_set_rate_sleep(r, sleep_rate);
+	if (ret)
+		goto out;
+
+	r->rate = rate;
+
+out:
+	mutex_unlock(&rpm_smd_clk_lock);
+
+	return ret;
+}
+
+static long clk_smd_rpm_round_rate(struct clk_hw *hw, unsigned long rate,
+				   unsigned long *parent_rate)
+{
+	/*
+	 * RPM handles rate rounding and we don't have a way to
+	 * know what the rate will be, so just return whatever
+	 * rate is requested.
+	 */
+	return rate;
+}
+
+static unsigned long clk_smd_rpm_recalc_rate(struct clk_hw *hw,
+					     unsigned long parent_rate)
+{
+	struct clk_smd_rpm *r = to_clk_smd_rpm(hw);
+
+	/*
+	 * RPM handles rate rounding and we don't have a way to
+	 * know what the rate will be, so just return whatever
+	 * rate was set.
+	 */
+	return r->rate;
+}
+
+static int clk_smd_rpm_enable_scaling(struct qcom_smd_rpm *rpm)
+{
+	int ret;
+	struct clk_smd_rpm_req req = {
+		.key = cpu_to_le32(QCOM_RPM_SMD_KEY_ENABLE),
+		.nbytes = cpu_to_le32(sizeof(u32)),
+		.value = cpu_to_le32(1),
+	};
+
+	ret = qcom_rpm_smd_write(rpm, QCOM_SMD_RPM_SLEEP_STATE,
+				 QCOM_SMD_RPM_MISC_CLK,
+				 QCOM_RPM_SCALING_ENABLE_ID, &req, sizeof(req));
+	if (ret) {
+		pr_err("RPM clock scaling (sleep set) not enabled!\n");
+		return ret;
+	}
+
+	ret = qcom_rpm_smd_write(rpm, QCOM_SMD_RPM_ACTIVE_STATE,
+				 QCOM_SMD_RPM_MISC_CLK,
+				 QCOM_RPM_SCALING_ENABLE_ID, &req, sizeof(req));
+	if (ret) {
+		pr_err("RPM clock scaling (active set) not enabled!\n");
+		return ret;
+	}
+
+	pr_debug("%s: RPM clock scaling is enabled\n", __func__);
+	return 0;
+}
+
+static const struct clk_ops clk_smd_rpm_ops = {
+	.prepare	= clk_smd_rpm_prepare,
+	.unprepare	= clk_smd_rpm_unprepare,
+	.set_rate	= clk_smd_rpm_set_rate,
+	.round_rate	= clk_smd_rpm_round_rate,
+	.recalc_rate	= clk_smd_rpm_recalc_rate,
+};
+
+static const struct clk_ops clk_smd_rpm_branch_ops = {
+	.prepare	= clk_smd_rpm_prepare,
+	.unprepare	= clk_smd_rpm_unprepare,
+	.round_rate	= clk_smd_rpm_round_rate,
+	.recalc_rate	= clk_smd_rpm_recalc_rate,
+};
+
+/* msm8916 */
+DEFINE_CLK_SMD_RPM(msm8916, pcnoc_clk, pcnoc_a_clk, QCOM_SMD_RPM_BUS_CLK, 0);
+DEFINE_CLK_SMD_RPM(msm8916, snoc_clk, snoc_a_clk, QCOM_SMD_RPM_BUS_CLK, 1);
+DEFINE_CLK_SMD_RPM(msm8916, bimc_clk, bimc_a_clk, QCOM_SMD_RPM_MEM_CLK, 0);
+DEFINE_CLK_SMD_RPM_QDSS(msm8916, qdss_clk, qdss_a_clk, QCOM_SMD_RPM_MISC_CLK, 1);
+DEFINE_CLK_SMD_RPM_XO_BUFFER(msm8916, bb_clk1, bb_clk1_a, 1);
+DEFINE_CLK_SMD_RPM_XO_BUFFER(msm8916, bb_clk2, bb_clk2_a, 2);
+DEFINE_CLK_SMD_RPM_XO_BUFFER(msm8916, rf_clk1, rf_clk1_a, 4);
+DEFINE_CLK_SMD_RPM_XO_BUFFER(msm8916, rf_clk2, rf_clk2_a, 5);
+DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(msm8916, bb_clk1_pin, bb_clk1_a_pin, 1);
+DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(msm8916, bb_clk2_pin, bb_clk2_a_pin, 2);
+DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(msm8916, rf_clk1_pin, rf_clk1_a_pin, 4);
+DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(msm8916, rf_clk2_pin, rf_clk2_a_pin, 5);
+
+static struct clk_smd_rpm *msm8916_clks[] = {
+	[RPM_SMD_PCNOC_CLK]		= &msm8916_pcnoc_clk,
+	[RPM_SMD_PCNOC_A_CLK]		= &msm8916_pcnoc_a_clk,
+	[RPM_SMD_SNOC_CLK]		= &msm8916_snoc_clk,
+	[RPM_SMD_SNOC_A_CLK]		= &msm8916_snoc_a_clk,
+	[RPM_SMD_BIMC_CLK]		= &msm8916_bimc_clk,
+	[RPM_SMD_BIMC_A_CLK]		= &msm8916_bimc_a_clk,
+	[RPM_SMD_QDSS_CLK]		= &msm8916_qdss_clk,
+	[RPM_SMD_QDSS_A_CLK]		= &msm8916_qdss_a_clk,
+	[RPM_SMD_BB_CLK1]		= &msm8916_bb_clk1,
+	[RPM_SMD_BB_CLK1_A]		= &msm8916_bb_clk1_a,
+	[RPM_SMD_BB_CLK2]		= &msm8916_bb_clk2,
+	[RPM_SMD_BB_CLK2_A]		= &msm8916_bb_clk2_a,
+	[RPM_SMD_RF_CLK1]		= &msm8916_rf_clk1,
+	[RPM_SMD_RF_CLK1_A]		= &msm8916_rf_clk1_a,
+	[RPM_SMD_RF_CLK2]		= &msm8916_rf_clk2,
+	[RPM_SMD_RF_CLK2_A]		= &msm8916_rf_clk2_a,
+	[RPM_SMD_BB_CLK1_PIN]		= &msm8916_bb_clk1_pin,
+	[RPM_SMD_BB_CLK1_A_PIN]		= &msm8916_bb_clk1_a_pin,
+	[RPM_SMD_BB_CLK2_PIN]		= &msm8916_bb_clk2_pin,
+	[RPM_SMD_BB_CLK2_A_PIN]		= &msm8916_bb_clk2_a_pin,
+	[RPM_SMD_RF_CLK1_PIN]		= &msm8916_rf_clk1_pin,
+	[RPM_SMD_RF_CLK1_A_PIN]		= &msm8916_rf_clk1_a_pin,
+	[RPM_SMD_RF_CLK2_PIN]		= &msm8916_rf_clk2_pin,
+	[RPM_SMD_RF_CLK2_A_PIN]		= &msm8916_rf_clk2_a_pin,
+};
+
+static const struct rpm_smd_clk_desc rpm_clk_msm8916 = {
+	.clks = msm8916_clks,
+	.num_clks = ARRAY_SIZE(msm8916_clks),
+};
+
+static const struct of_device_id rpm_smd_clk_match_table[] = {
+	{ .compatible = "qcom,rpmcc-msm8916", .data = &rpm_clk_msm8916 },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, rpm_smd_clk_match_table);
+
+static struct clk_hw *qcom_smdrpm_clk_hw_get(struct of_phandle_args *clkspec,
+					     void *data)
+{
+	struct rpm_cc *rcc = data;
+	unsigned int idx = clkspec->args[0];
+
+	if (idx >= rcc->num_clks) {
+		pr_err("%s: invalid index %u\n", __func__, idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return rcc->clks[idx] ? &rcc->clks[idx]->hw : ERR_PTR(-ENOENT);
+}
+
+static int rpm_smd_clk_probe(struct platform_device *pdev)
+{
+	struct rpm_cc *rcc;
+	int ret;
+	size_t num_clks, i;
+	struct qcom_smd_rpm *rpm;
+	struct clk_smd_rpm **rpm_smd_clks;
+	const struct rpm_smd_clk_desc *desc;
+
+	rpm = dev_get_drvdata(pdev->dev.parent);
+	if (!rpm) {
+		dev_err(&pdev->dev, "Unable to retrieve handle to RPM\n");
+		return -ENODEV;
+	}
+
+	desc = of_device_get_match_data(&pdev->dev);
+	if (!desc)
+		return -EINVAL;
+
+	rpm_smd_clks = desc->clks;
+	num_clks = desc->num_clks;
+
+	rcc = devm_kzalloc(&pdev->dev, sizeof(*rcc), GFP_KERNEL);
+	if (!rcc)
+		return -ENOMEM;
+
+	rcc->clks = rpm_smd_clks;
+	rcc->num_clks = num_clks;
+
+	for (i = 0; i < num_clks; i++) {
+		if (!rpm_smd_clks[i])
+			continue;
+
+		rpm_smd_clks[i]->rpm = rpm;
+
+		ret = clk_smd_rpm_handoff(rpm_smd_clks[i]);
+		if (ret)
+			goto err;
+	}
+
+	ret = clk_smd_rpm_enable_scaling(rpm);
+	if (ret)
+		goto err;
+
+	for (i = 0; i < num_clks; i++) {
+		if (!rpm_smd_clks[i])
+			continue;
+
+		ret = devm_clk_hw_register(&pdev->dev, &rpm_smd_clks[i]->hw);
+		if (ret)
+			goto err;
+	}
+
+	ret = of_clk_add_hw_provider(pdev->dev.of_node, qcom_smdrpm_clk_hw_get,
+				     rcc);
+	if (ret)
+		goto err;
+
+	return 0;
+err:
+	dev_err(&pdev->dev, "Error registering SMD clock driver (%d)\n", ret);
+	return ret;
+}
+
+static int rpm_smd_clk_remove(struct platform_device *pdev)
+{
+	of_clk_del_provider(pdev->dev.of_node);
+	return 0;
+}
+
+static struct platform_driver rpm_smd_clk_driver = {
+	.driver = {
+		.name = "qcom-clk-smd-rpm",
+		.of_match_table = rpm_smd_clk_match_table,
+	},
+	.probe = rpm_smd_clk_probe,
+	.remove = rpm_smd_clk_remove,
+};
+
+static int __init rpm_smd_clk_init(void)
+{
+	return platform_driver_register(&rpm_smd_clk_driver);
+}
+core_initcall(rpm_smd_clk_init);
+
+static void __exit rpm_smd_clk_exit(void)
+{
+	platform_driver_unregister(&rpm_smd_clk_driver);
+}
+module_exit(rpm_smd_clk_exit);
+
+MODULE_DESCRIPTION("Qualcomm RPM over SMD Clock Controller Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:qcom-clk-smd-rpm");
diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c
index fffcbaf0fba7..cfab7b400381 100644
--- a/drivers/clk/qcom/common.c
+++ b/drivers/clk/qcom/common.c
@@ -46,6 +46,22 @@ struct freq_tbl *qcom_find_freq(const struct freq_tbl *f, unsigned long rate)
 }
 EXPORT_SYMBOL_GPL(qcom_find_freq);
 
+const struct freq_tbl *qcom_find_freq_floor(const struct freq_tbl *f,
+					    unsigned long rate)
+{
+	const struct freq_tbl *best = NULL;
+
+	for ( ; f->freq; f++) {
+		if (rate >= f->freq)
+			best = f;
+		else
+			break;
+	}
+
+	return best;
+}
+EXPORT_SYMBOL_GPL(qcom_find_freq_floor);
+
 int qcom_find_src_index(struct clk_hw *hw, const struct parent_map *map, u8 src)
 {
 	int i, num_parents = clk_hw_get_num_parents(hw);
@@ -74,6 +90,27 @@ qcom_cc_map(struct platform_device *pdev, const struct qcom_cc_desc *desc)
 }
 EXPORT_SYMBOL_GPL(qcom_cc_map);
 
+void
+qcom_pll_set_fsm_mode(struct regmap *map, u32 reg, u8 bias_count, u8 lock_count)
+{
+	u32 val;
+	u32 mask;
+
+	/* De-assert reset to FSM */
+	regmap_update_bits(map, reg, PLL_VOTE_FSM_RESET, 0);
+
+	/* Program bias count and lock count */
+	val = bias_count << PLL_BIAS_COUNT_SHIFT |
+		lock_count << PLL_LOCK_COUNT_SHIFT;
+	mask = PLL_BIAS_COUNT_MASK << PLL_BIAS_COUNT_SHIFT;
+	mask |= PLL_LOCK_COUNT_MASK << PLL_LOCK_COUNT_SHIFT;
+	regmap_update_bits(map, reg, mask, val);
+
+	/* Enable PLL FSM voting */
+	regmap_update_bits(map, reg, PLL_VOTE_FSM_ENA, PLL_VOTE_FSM_ENA);
+}
+EXPORT_SYMBOL_GPL(qcom_pll_set_fsm_mode);
+
 static void qcom_cc_del_clk_provider(void *data)
 {
 	of_clk_del_provider(data);
@@ -153,15 +190,12 @@ int qcom_cc_register_board_clk(struct device *dev, const char *path,
 			       const char *name, unsigned long rate)
 {
 	bool add_factor = true;
-	struct device_node *node;
-
-	/* The RPM clock driver will add the factor clock if present */
-	if (IS_ENABLED(CONFIG_QCOM_RPMCC)) {
-		node = of_find_compatible_node(NULL, NULL, "qcom,rpmcc");
-		if (of_device_is_available(node))
-			add_factor = false;
-		of_node_put(node);
-	}
+
+	/*
+	 * TODO: The RPM clock driver currently does not support the xo clock.
+	 * When xo is added to the RPM clock driver, we should change this
+	 * function to skip registration of xo factor clocks.
+	 */
 
 	return _qcom_cc_register_board_clk(dev, path, name, rate, add_factor);
 }
diff --git a/drivers/clk/qcom/common.h b/drivers/clk/qcom/common.h
index ae9bdeb21f29..23c1927669ba 100644
--- a/drivers/clk/qcom/common.h
+++ b/drivers/clk/qcom/common.h
@@ -22,6 +22,13 @@ struct freq_tbl;
 struct clk_hw;
 struct parent_map;
 
+#define PLL_LOCK_COUNT_SHIFT	8
+#define PLL_LOCK_COUNT_MASK	0x3f
+#define PLL_BIAS_COUNT_SHIFT	14
+#define PLL_BIAS_COUNT_MASK	0x3f
+#define PLL_VOTE_FSM_ENA	BIT(20)
+#define PLL_VOTE_FSM_RESET	BIT(21)
+
 struct qcom_cc_desc {
 	const struct regmap_config *config;
 	struct clk_regmap **clks;
@@ -34,6 +41,10 @@ struct qcom_cc_desc {
 
 extern const struct freq_tbl *qcom_find_freq(const struct freq_tbl *f,
 					     unsigned long rate);
+extern const struct freq_tbl *qcom_find_freq_floor(const struct freq_tbl *f,
+						   unsigned long rate);
+extern void
+qcom_pll_set_fsm_mode(struct regmap *m, u32 reg, u8 bias_count, u8 lock_count);
 extern int qcom_find_src_index(struct clk_hw *hw, const struct parent_map *map,
 			       u8 src);
 
diff --git a/drivers/clk/qcom/gcc-apq8084.c b/drivers/clk/qcom/gcc-apq8084.c
index 070037a29ea5..486d9610355c 100644
--- a/drivers/clk/qcom/gcc-apq8084.c
+++ b/drivers/clk/qcom/gcc-apq8084.c
@@ -1142,7 +1142,7 @@ static struct clk_rcg2 sdcc1_apps_clk_src = {
 		.name = "sdcc1_apps_clk_src",
 		.parent_names = gcc_xo_gpll0_gpll4,
 		.num_parents = 3,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
@@ -1156,7 +1156,7 @@ static struct clk_rcg2 sdcc2_apps_clk_src = {
 		.name = "sdcc2_apps_clk_src",
 		.parent_names = gcc_xo_gpll0,
 		.num_parents = 2,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
@@ -1170,7 +1170,7 @@ static struct clk_rcg2 sdcc3_apps_clk_src = {
 		.name = "sdcc3_apps_clk_src",
 		.parent_names = gcc_xo_gpll0,
 		.num_parents = 2,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
@@ -1184,7 +1184,7 @@ static struct clk_rcg2 sdcc4_apps_clk_src = {
 		.name = "sdcc4_apps_clk_src",
 		.parent_names = gcc_xo_gpll0,
 		.num_parents = 2,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/gcc-ipq4019.c b/drivers/clk/qcom/gcc-ipq4019.c
index b593065de8db..33d09138f5e5 100644
--- a/drivers/clk/qcom/gcc-ipq4019.c
+++ b/drivers/clk/qcom/gcc-ipq4019.c
@@ -185,8 +185,7 @@ static struct clk_branch gcc_audio_pwm_clk = {
 };
 
 static const struct freq_tbl ftbl_gcc_blsp1_qup1_2_i2c_apps_clk[] = {
-	F(19200000, P_XO, 1, 2, 5),
-	F(24000000, P_XO, 1, 1, 2),
+	F(19050000, P_FEPLL200, 10.5, 1, 1),
 	{ }
 };
 
diff --git a/drivers/clk/qcom/gcc-ipq806x.c b/drivers/clk/qcom/gcc-ipq806x.c
index 52a7d3959875..28eb200d0f1e 100644
--- a/drivers/clk/qcom/gcc-ipq806x.c
+++ b/drivers/clk/qcom/gcc-ipq806x.c
@@ -2990,11 +2990,11 @@ static int gcc_ipq806x_probe(struct platform_device *pdev)
 	struct regmap *regmap;
 	int ret;
 
-	ret = qcom_cc_register_board_clk(dev, "cxo_board", "cxo", 19200000);
+	ret = qcom_cc_register_board_clk(dev, "cxo_board", "cxo", 25000000);
 	if (ret)
 		return ret;
 
-	ret = qcom_cc_register_board_clk(dev, "pxo_board", "pxo", 27000000);
+	ret = qcom_cc_register_board_clk(dev, "pxo_board", "pxo", 25000000);
 	if (ret)
 		return ret;
 
diff --git a/drivers/clk/qcom/gcc-msm8916.c b/drivers/clk/qcom/gcc-msm8916.c
index 5c4e193164d4..628e6ca276ec 100644
--- a/drivers/clk/qcom/gcc-msm8916.c
+++ b/drivers/clk/qcom/gcc-msm8916.c
@@ -1107,7 +1107,7 @@ static struct clk_rcg2 sdcc1_apps_clk_src = {
 		.name = "sdcc1_apps_clk_src",
 		.parent_names = gcc_xo_gpll0,
 		.num_parents = 2,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
@@ -1132,7 +1132,7 @@ static struct clk_rcg2 sdcc2_apps_clk_src = {
 		.name = "sdcc2_apps_clk_src",
 		.parent_names = gcc_xo_gpll0,
 		.num_parents = 2,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/gcc-msm8974.c b/drivers/clk/qcom/gcc-msm8974.c
index 00915209e7c5..348e30da4f18 100644
--- a/drivers/clk/qcom/gcc-msm8974.c
+++ b/drivers/clk/qcom/gcc-msm8974.c
@@ -872,7 +872,7 @@ static struct clk_init_data sdcc1_apps_clk_src_init = {
 	.name = "sdcc1_apps_clk_src",
 	.parent_names = gcc_xo_gpll0,
 	.num_parents = 2,
-	.ops = &clk_rcg2_ops,
+	.ops = &clk_rcg2_floor_ops,
 };
 
 static struct clk_rcg2 sdcc1_apps_clk_src = {
@@ -894,7 +894,7 @@ static struct clk_rcg2 sdcc2_apps_clk_src = {
 		.name = "sdcc2_apps_clk_src",
 		.parent_names = gcc_xo_gpll0,
 		.num_parents = 2,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
@@ -908,7 +908,7 @@ static struct clk_rcg2 sdcc3_apps_clk_src = {
 		.name = "sdcc3_apps_clk_src",
 		.parent_names = gcc_xo_gpll0,
 		.num_parents = 2,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
@@ -922,7 +922,7 @@ static struct clk_rcg2 sdcc4_apps_clk_src = {
 		.name = "sdcc4_apps_clk_src",
 		.parent_names = gcc_xo_gpll0,
 		.num_parents = 2,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/gcc-msm8994.c b/drivers/clk/qcom/gcc-msm8994.c
new file mode 100644
index 000000000000..8afd8304a070
--- /dev/null
+++ b/drivers/clk/qcom/gcc-msm8994.c
@@ -0,0 +1,2300 @@
+/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/ctype.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,gcc-msm8994.h>
+
+#include "common.h"
+#include "clk-regmap.h"
+#include "clk-alpha-pll.h"
+#include "clk-rcg.h"
+#include "clk-branch.h"
+#include "reset.h"
+
+enum {
+	P_XO,
+	P_GPLL0,
+	P_GPLL4,
+};
+
+static const struct parent_map gcc_xo_gpll0_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+};
+
+static const char * const gcc_xo_gpll0[] = {
+	"xo",
+	"gpll0",
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll4_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL4, 5 },
+};
+
+static const char * const gcc_xo_gpll0_gpll4[] = {
+	"xo",
+	"gpll0",
+	"gpll4",
+};
+
+#define F(f, s, h, m, n) { (f), (s), (2 * (h) - 1), (m), (n) }
+
+static struct clk_fixed_factor xo = {
+	.mult = 1,
+	.div = 1,
+	.hw.init = &(struct clk_init_data)
+	{
+		.name = "xo",
+		.parent_names = (const char *[]) { "xo_board" },
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll0_early = {
+	.offset = 0x00000,
+	.clkr = {
+		.enable_reg = 0x1480,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gpll0_early",
+			.parent_names = (const char *[]) { "xo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll0 = {
+	.offset = 0x00000,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "gpll0",
+		.parent_names = (const char *[]) { "gpll0_early" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll4_early = {
+	.offset = 0x1dc0,
+	.clkr = {
+		.enable_reg = 0x1480,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gpll4_early",
+			.parent_names = (const char *[]) { "xo" },
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll4 = {
+	.offset = 0x1dc0,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "gpll4",
+		.parent_names = (const char *[]) { "gpll4_early" },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+	},
+};
+
+static struct freq_tbl ftbl_ufs_axi_clk_src[] = {
+	F(50000000, P_GPLL0, 12, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	F(150000000, P_GPLL0, 4, 0, 0),
+	F(171430000, P_GPLL0, 3.5, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	F(240000000, P_GPLL0, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 ufs_axi_clk_src = {
+	.cmd_rcgr = 0x1d68,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_ufs_axi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "ufs_axi_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_usb30_master_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(125000000, P_GPLL0, 1, 5, 24),
+	{ }
+};
+
+static struct clk_rcg2 usb30_master_clk_src = {
+	.cmd_rcgr = 0x03d4,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_usb30_master_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "usb30_master_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_blsp_i2c_apps_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0, 12, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0660,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_qup1_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_blspqup_spi_apps_clk_src[] = {
+	F(960000, P_XO, 10, 1, 2),
+	F(4800000, P_XO, 4, 0, 0),
+	F(9600000, P_XO, 2, 0, 0),
+	F(15000000, P_GPLL0, 10, 1, 4),
+	F(19200000, P_XO, 1, 0, 0),
+	F(24000000, P_GPLL0, 12.5, 1, 2),
+	F(25000000, P_GPLL0, 12, 1, 2),
+	F(48000000, P_GPLL0, 12.5, 0, 0),
+	F(50000000, P_GPLL0, 12, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_qup1_spi_apps_clk_src = {
+	.cmd_rcgr = 0x064c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blspqup_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_qup1_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup2_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x06e0,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_qup2_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup2_spi_apps_clk_src = {
+	.cmd_rcgr = 0x06cc,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blspqup_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_qup2_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup3_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0760,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_qup3_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup3_spi_apps_clk_src = {
+	.cmd_rcgr = 0x074c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blspqup_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_qup3_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup4_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x07e0,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_qup4_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup4_spi_apps_clk_src = {
+	.cmd_rcgr = 0x07cc,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blspqup_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_qup4_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup5_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0860,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_qup5_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup5_spi_apps_clk_src = {
+	.cmd_rcgr = 0x084c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blspqup_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_qup5_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup6_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x08e0,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_qup6_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup6_spi_apps_clk_src = {
+	.cmd_rcgr = 0x08cc,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blspqup_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_qup6_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_blsp_uart_apps_clk_src[] = {
+	F(3686400, P_GPLL0, 1, 96, 15625),
+	F(7372800, P_GPLL0, 1, 192, 15625),
+	F(14745600, P_GPLL0, 1, 384, 15625),
+	F(16000000, P_GPLL0, 5, 2, 15),
+	F(19200000, P_XO, 1, 0, 0),
+	F(24000000, P_GPLL0, 5, 1, 5),
+	F(32000000, P_GPLL0, 1, 4, 75),
+	F(40000000, P_GPLL0, 15, 0, 0),
+	F(46400000, P_GPLL0, 1, 29, 375),
+	F(48000000, P_GPLL0, 12.5, 0, 0),
+	F(51200000, P_GPLL0, 1, 32, 375),
+	F(56000000, P_GPLL0, 1, 7, 75),
+	F(58982400, P_GPLL0, 1, 1536, 15625),
+	F(60000000, P_GPLL0, 10, 0, 0),
+	F(63160000, P_GPLL0, 9.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_uart1_apps_clk_src = {
+	.cmd_rcgr = 0x068c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_uart_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_uart1_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart2_apps_clk_src = {
+	.cmd_rcgr = 0x070c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_uart_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_uart2_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart3_apps_clk_src = {
+	.cmd_rcgr = 0x078c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_uart_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_uart3_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart4_apps_clk_src = {
+	.cmd_rcgr = 0x080c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_uart_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_uart4_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart5_apps_clk_src = {
+	.cmd_rcgr = 0x088c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_uart_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_uart5_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart6_apps_clk_src = {
+	.cmd_rcgr = 0x090c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_uart_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp1_uart6_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup1_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x09a0,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_qup1_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup1_spi_apps_clk_src = {
+	.cmd_rcgr = 0x098c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blspqup_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_qup1_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup2_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0a20,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_qup2_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup2_spi_apps_clk_src = {
+	.cmd_rcgr = 0x0a0c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blspqup_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_qup2_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup3_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0aa0,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_qup3_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup3_spi_apps_clk_src = {
+	.cmd_rcgr = 0x0a8c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blspqup_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_qup3_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup4_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0b20,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_qup4_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup4_spi_apps_clk_src = {
+	.cmd_rcgr = 0x0b0c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blspqup_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_qup4_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup5_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0ba0,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_qup5_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup5_spi_apps_clk_src = {
+	.cmd_rcgr = 0x0b8c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blspqup_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_qup5_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup6_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0c20,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_i2c_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_qup6_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup6_spi_apps_clk_src = {
+	.cmd_rcgr = 0x0c0c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blspqup_spi_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_qup6_spi_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart1_apps_clk_src = {
+	.cmd_rcgr = 0x09cc,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_uart_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_uart1_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart2_apps_clk_src = {
+	.cmd_rcgr = 0x0a4c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_uart_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_uart2_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart3_apps_clk_src = {
+	.cmd_rcgr = 0x0acc,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_uart_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_uart3_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart4_apps_clk_src = {
+	.cmd_rcgr = 0x0b4c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_uart_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_uart4_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart5_apps_clk_src = {
+	.cmd_rcgr = 0x0bcc,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_uart_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_uart5_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart6_apps_clk_src = {
+	.cmd_rcgr = 0x0c4c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_blsp_uart_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "blsp2_uart6_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_gp1_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gp1_clk_src = {
+	.cmd_rcgr = 0x1904,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gp1_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "gp1_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_gp2_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gp2_clk_src = {
+	.cmd_rcgr = 0x1944,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gp2_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "gp2_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_gp3_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gp3_clk_src = {
+	.cmd_rcgr = 0x1984,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gp3_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "gp3_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_pcie_0_aux_clk_src[] = {
+	F(1011000, P_XO, 1, 1, 19),
+	{ }
+};
+
+static struct clk_rcg2 pcie_0_aux_clk_src = {
+	.cmd_rcgr = 0x1b00,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.freq_tbl = ftbl_pcie_0_aux_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "pcie_0_aux_clk_src",
+		.parent_names = (const char *[]) { "xo" },
+		.num_parents = 1,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_pcie_pipe_clk_src[] = {
+	F(125000000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 pcie_0_pipe_clk_src = {
+	.cmd_rcgr = 0x1adc,
+	.hid_width = 5,
+	.freq_tbl = ftbl_pcie_pipe_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "pcie_0_pipe_clk_src",
+		.parent_names = (const char *[]) { "xo" },
+		.num_parents = 1,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_pcie_1_aux_clk_src[] = {
+	F(1011000, P_XO, 1, 1, 19),
+	{ }
+};
+
+static struct clk_rcg2 pcie_1_aux_clk_src = {
+	.cmd_rcgr = 0x1b80,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.freq_tbl = ftbl_pcie_1_aux_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "pcie_1_aux_clk_src",
+		.parent_names = (const char *[]) { "xo" },
+		.num_parents = 1,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 pcie_1_pipe_clk_src = {
+	.cmd_rcgr = 0x1b5c,
+	.hid_width = 5,
+	.freq_tbl = ftbl_pcie_pipe_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "pcie_1_pipe_clk_src",
+		.parent_names = (const char *[]) { "xo" },
+		.num_parents = 1,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_pdm2_clk_src[] = {
+	F(60000000, P_GPLL0, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 pdm2_clk_src = {
+	.cmd_rcgr = 0x0cd0,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_pdm2_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "pdm2_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_sdcc1_apps_clk_src[] = {
+	F(144000, P_XO, 16, 3, 25),
+	F(400000, P_XO, 12, 1, 4),
+	F(20000000, P_GPLL0, 15, 1, 2),
+	F(25000000, P_GPLL0, 12, 1, 2),
+	F(50000000, P_GPLL0, 12, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	F(192000000, P_GPLL4, 2, 0, 0),
+	F(384000000, P_GPLL4, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 sdcc1_apps_clk_src = {
+	.cmd_rcgr = 0x04d0,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll4_map,
+	.freq_tbl = ftbl_sdcc1_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "sdcc1_apps_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll4,
+		.num_parents = 3,
+		.ops = &clk_rcg2_floor_ops,
+	},
+};
+
+static struct freq_tbl ftbl_sdcc2_4_apps_clk_src[] = {
+	F(144000, P_XO, 16, 3, 25),
+	F(400000, P_XO, 12, 1, 4),
+	F(20000000, P_GPLL0, 15, 1, 2),
+	F(25000000, P_GPLL0, 12, 1, 2),
+	F(50000000, P_GPLL0, 12, 0, 0),
+	F(100000000, P_GPLL0, 6, 0, 0),
+	F(200000000, P_GPLL0, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 sdcc2_apps_clk_src = {
+	.cmd_rcgr = 0x0510,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_sdcc2_4_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "sdcc2_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_floor_ops,
+	},
+};
+
+static struct clk_rcg2 sdcc3_apps_clk_src = {
+	.cmd_rcgr = 0x0550,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_sdcc2_4_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "sdcc3_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_floor_ops,
+	},
+};
+
+static struct clk_rcg2 sdcc4_apps_clk_src = {
+	.cmd_rcgr = 0x0590,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_sdcc2_4_apps_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "sdcc4_apps_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_floor_ops,
+	},
+};
+
+static struct freq_tbl ftbl_tsif_ref_clk_src[] = {
+	F(105500, P_XO, 1, 1, 182),
+	{ }
+};
+
+static struct clk_rcg2 tsif_ref_clk_src = {
+	.cmd_rcgr = 0x0d90,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.freq_tbl = ftbl_tsif_ref_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "tsif_ref_clk_src",
+		.parent_names = (const char *[]) { "xo" },
+		.num_parents = 1,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_usb30_mock_utmi_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(60000000, P_GPLL0, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb30_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x03e8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_usb30_mock_utmi_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "usb30_mock_utmi_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_usb3_phy_aux_clk_src[] = {
+	F(1200000, P_XO, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb3_phy_aux_clk_src = {
+	.cmd_rcgr = 0x1414,
+	.hid_width = 5,
+	.freq_tbl = ftbl_usb3_phy_aux_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "usb3_phy_aux_clk_src",
+		.parent_names = (const char *[]) { "xo" },
+		.num_parents = 1,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_usb_hs_system_clk_src[] = {
+	F(75000000, P_GPLL0, 8, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb_hs_system_clk_src = {
+	.cmd_rcgr = 0x0490,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_usb_hs_system_clk_src,
+	.clkr.hw.init = &(struct clk_init_data)
+	{
+		.name = "usb_hs_system_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_blsp1_ahb_clk = {
+	.halt_reg = 0x05c4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x1484,
+		.enable_mask = BIT(17),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = {
+	.halt_reg = 0x0648,
+	.clkr = {
+		.enable_reg = 0x0648,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_qup1_i2c_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_qup1_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = {
+	.halt_reg = 0x0644,
+	.clkr = {
+		.enable_reg = 0x0644,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_qup1_spi_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_qup1_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = {
+	.halt_reg = 0x06c8,
+	.clkr = {
+		.enable_reg = 0x06c8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_qup2_i2c_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_qup2_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = {
+	.halt_reg = 0x06c4,
+	.clkr = {
+		.enable_reg = 0x06c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_qup2_spi_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_qup2_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = {
+	.halt_reg = 0x0748,
+	.clkr = {
+		.enable_reg = 0x0748,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_qup3_i2c_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_qup3_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = {
+	.halt_reg = 0x0744,
+	.clkr = {
+		.enable_reg = 0x0744,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_qup3_spi_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_qup3_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = {
+	.halt_reg = 0x07c8,
+	.clkr = {
+		.enable_reg = 0x07c8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_qup4_i2c_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_qup4_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = {
+	.halt_reg = 0x07c4,
+	.clkr = {
+		.enable_reg = 0x07c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_qup4_spi_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_qup4_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = {
+	.halt_reg = 0x0848,
+	.clkr = {
+		.enable_reg = 0x0848,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_qup5_i2c_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_qup5_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = {
+	.halt_reg = 0x0844,
+	.clkr = {
+		.enable_reg = 0x0844,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_qup5_spi_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_qup5_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup6_i2c_apps_clk = {
+	.halt_reg = 0x08c8,
+	.clkr = {
+		.enable_reg = 0x08c8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_qup6_i2c_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_qup6_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = {
+	.halt_reg = 0x08c4,
+	.clkr = {
+		.enable_reg = 0x08c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_qup6_spi_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_qup6_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart1_apps_clk = {
+	.halt_reg = 0x0684,
+	.clkr = {
+		.enable_reg = 0x0684,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_uart1_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_uart1_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart2_apps_clk = {
+	.halt_reg = 0x0704,
+	.clkr = {
+		.enable_reg = 0x0704,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_uart2_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_uart2_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart3_apps_clk = {
+	.halt_reg = 0x0784,
+	.clkr = {
+		.enable_reg = 0x0784,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_uart3_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_uart3_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart4_apps_clk = {
+	.halt_reg = 0x0804,
+	.clkr = {
+		.enable_reg = 0x0804,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_uart4_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_uart4_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart5_apps_clk = {
+	.halt_reg = 0x0884,
+	.clkr = {
+		.enable_reg = 0x0884,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_uart5_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_uart5_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart6_apps_clk = {
+	.halt_reg = 0x0904,
+	.clkr = {
+		.enable_reg = 0x0904,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp1_uart6_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp1_uart6_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_ahb_clk = {
+	.halt_reg = 0x0944,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x1484,
+		.enable_mask = BIT(15),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup1_i2c_apps_clk = {
+	.halt_reg = 0x0988,
+	.clkr = {
+		.enable_reg = 0x0988,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_qup1_i2c_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_qup1_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup1_spi_apps_clk = {
+	.halt_reg = 0x0984,
+	.clkr = {
+		.enable_reg = 0x0984,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_qup1_spi_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_qup1_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup2_i2c_apps_clk = {
+	.halt_reg = 0x0a08,
+	.clkr = {
+		.enable_reg = 0x0a08,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_qup2_i2c_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_qup2_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup2_spi_apps_clk = {
+	.halt_reg = 0x0a04,
+	.clkr = {
+		.enable_reg = 0x0a04,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_qup2_spi_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_qup2_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup3_i2c_apps_clk = {
+	.halt_reg = 0x0a88,
+	.clkr = {
+		.enable_reg = 0x0a88,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_qup3_i2c_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_qup3_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup3_spi_apps_clk = {
+	.halt_reg = 0x0a84,
+	.clkr = {
+		.enable_reg = 0x0a84,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_qup3_spi_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_qup3_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup4_i2c_apps_clk = {
+	.halt_reg = 0x0b08,
+	.clkr = {
+		.enable_reg = 0x0b08,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_qup4_i2c_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_qup4_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup4_spi_apps_clk = {
+	.halt_reg = 0x0b04,
+	.clkr = {
+		.enable_reg = 0x0b04,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_qup4_spi_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_qup4_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup5_i2c_apps_clk = {
+	.halt_reg = 0x0b88,
+	.clkr = {
+		.enable_reg = 0x0b88,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_qup5_i2c_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_qup5_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup5_spi_apps_clk = {
+	.halt_reg = 0x0b84,
+	.clkr = {
+		.enable_reg = 0x0b84,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_qup5_spi_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_qup5_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup6_i2c_apps_clk = {
+	.halt_reg = 0x0c08,
+	.clkr = {
+		.enable_reg = 0x0c08,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_qup6_i2c_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_qup6_i2c_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup6_spi_apps_clk = {
+	.halt_reg = 0x0c04,
+	.clkr = {
+		.enable_reg = 0x0c04,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_qup6_spi_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_qup6_spi_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart1_apps_clk = {
+	.halt_reg = 0x09c4,
+	.clkr = {
+		.enable_reg = 0x09c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_uart1_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_uart1_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart2_apps_clk = {
+	.halt_reg = 0x0a44,
+	.clkr = {
+		.enable_reg = 0x0a44,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_uart2_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_uart2_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart3_apps_clk = {
+	.halt_reg = 0x0ac4,
+	.clkr = {
+		.enable_reg = 0x0ac4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_uart3_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_uart3_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart4_apps_clk = {
+	.halt_reg = 0x0b44,
+	.clkr = {
+		.enable_reg = 0x0b44,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_uart4_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_uart4_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart5_apps_clk = {
+	.halt_reg = 0x0bc4,
+	.clkr = {
+		.enable_reg = 0x0bc4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_uart5_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_uart5_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart6_apps_clk = {
+	.halt_reg = 0x0c44,
+	.clkr = {
+		.enable_reg = 0x0c44,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_blsp2_uart6_apps_clk",
+			.parent_names = (const char *[]) {
+				"blsp2_uart6_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp1_clk = {
+	.halt_reg = 0x1900,
+	.clkr = {
+		.enable_reg = 0x1900,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_gp1_clk",
+			.parent_names = (const char *[]) {
+				"gp1_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp2_clk = {
+	.halt_reg = 0x1940,
+	.clkr = {
+		.enable_reg = 0x1940,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_gp2_clk",
+			.parent_names = (const char *[]) {
+				"gp2_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp3_clk = {
+	.halt_reg = 0x1980,
+	.clkr = {
+		.enable_reg = 0x1980,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_gp3_clk",
+			.parent_names = (const char *[]) {
+				"gp3_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_aux_clk = {
+	.halt_reg = 0x1ad4,
+	.clkr = {
+		.enable_reg = 0x1ad4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_pcie_0_aux_clk",
+			.parent_names = (const char *[]) {
+				"pcie_0_aux_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_pipe_clk = {
+	.halt_reg = 0x1ad8,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x1ad8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_pcie_0_pipe_clk",
+			.parent_names = (const char *[]) {
+				"pcie_0_pipe_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_aux_clk = {
+	.halt_reg = 0x1b54,
+	.clkr = {
+		.enable_reg = 0x1b54,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_pcie_1_aux_clk",
+			.parent_names = (const char *[]) {
+				"pcie_1_aux_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_pipe_clk = {
+	.halt_reg = 0x1b58,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x1b58,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_pcie_1_pipe_clk",
+			.parent_names = (const char *[]) {
+				"pcie_1_pipe_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm2_clk = {
+	.halt_reg = 0x0ccc,
+	.clkr = {
+		.enable_reg = 0x0ccc,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_pdm2_clk",
+			.parent_names = (const char *[]) {
+				"pdm2_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_apps_clk = {
+	.halt_reg = 0x04c4,
+	.clkr = {
+		.enable_reg = 0x04c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_sdcc1_apps_clk",
+			.parent_names = (const char *[]) {
+				"sdcc1_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_apps_clk = {
+	.halt_reg = 0x0504,
+	.clkr = {
+		.enable_reg = 0x0504,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_sdcc2_apps_clk",
+			.parent_names = (const char *[]) {
+				"sdcc2_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc3_apps_clk = {
+	.halt_reg = 0x0544,
+	.clkr = {
+		.enable_reg = 0x0544,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_sdcc3_apps_clk",
+			.parent_names = (const char *[]) {
+				"sdcc3_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_apps_clk = {
+	.halt_reg = 0x0584,
+	.clkr = {
+		.enable_reg = 0x0584,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_sdcc4_apps_clk",
+			.parent_names = (const char *[]) {
+				"sdcc4_apps_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sys_noc_ufs_axi_clk = {
+	.halt_reg = 0x1d7c,
+	.clkr = {
+		.enable_reg = 0x1d7c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_sys_noc_ufs_axi_clk",
+			.parent_names = (const char *[]) {
+				"ufs_axi_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sys_noc_usb3_axi_clk = {
+	.halt_reg = 0x03fc,
+	.clkr = {
+		.enable_reg = 0x03fc,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_sys_noc_usb3_axi_clk",
+			.parent_names = (const char *[]) {
+				"usb30_master_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tsif_ref_clk = {
+	.halt_reg = 0x0d88,
+	.clkr = {
+		.enable_reg = 0x0d88,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_tsif_ref_clk",
+			.parent_names = (const char *[]) {
+				"tsif_ref_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_axi_clk = {
+	.halt_reg = 0x1d48,
+	.clkr = {
+		.enable_reg = 0x1d48,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_ufs_axi_clk",
+			.parent_names = (const char *[]) {
+				"ufs_axi_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_rx_cfg_clk = {
+	.halt_reg = 0x1d54,
+	.clkr = {
+		.enable_reg = 0x1d54,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_ufs_rx_cfg_clk",
+			.parent_names = (const char *[]) {
+				"ufs_axi_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_tx_cfg_clk = {
+	.halt_reg = 0x1d50,
+	.clkr = {
+		.enable_reg = 0x1d50,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_ufs_tx_cfg_clk",
+			.parent_names = (const char *[]) {
+				"ufs_axi_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_master_clk = {
+	.halt_reg = 0x03c8,
+	.clkr = {
+		.enable_reg = 0x03c8,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_usb30_master_clk",
+			.parent_names = (const char *[]) {
+				"usb30_master_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_mock_utmi_clk = {
+	.halt_reg = 0x03d0,
+	.clkr = {
+		.enable_reg = 0x03d0,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_usb30_mock_utmi_clk",
+			.parent_names = (const char *[]) {
+				"usb30_mock_utmi_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_phy_aux_clk = {
+	.halt_reg = 0x1408,
+	.clkr = {
+		.enable_reg = 0x1408,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_usb3_phy_aux_clk",
+			.parent_names = (const char *[]) {
+				"usb3_phy_aux_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_hs_system_clk = {
+	.halt_reg = 0x0484,
+	.clkr = {
+		.enable_reg = 0x0484,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data)
+		{
+			.name = "gcc_usb_hs_system_clk",
+			.parent_names = (const char *[]) {
+				"usb_hs_system_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap *gcc_msm8994_clocks[] = {
+	[GPLL0_EARLY] = &gpll0_early.clkr,
+	[GPLL0] = &gpll0.clkr,
+	[GPLL4_EARLY] = &gpll4_early.clkr,
+	[GPLL4] = &gpll4.clkr,
+	[UFS_AXI_CLK_SRC] = &ufs_axi_clk_src.clkr,
+	[USB30_MASTER_CLK_SRC] = &usb30_master_clk_src.clkr,
+	[BLSP1_QUP1_I2C_APPS_CLK_SRC] = &blsp1_qup1_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP1_SPI_APPS_CLK_SRC] = &blsp1_qup1_spi_apps_clk_src.clkr,
+	[BLSP1_QUP2_I2C_APPS_CLK_SRC] = &blsp1_qup2_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP2_SPI_APPS_CLK_SRC] = &blsp1_qup2_spi_apps_clk_src.clkr,
+	[BLSP1_QUP3_I2C_APPS_CLK_SRC] = &blsp1_qup3_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP3_SPI_APPS_CLK_SRC] = &blsp1_qup3_spi_apps_clk_src.clkr,
+	[BLSP1_QUP4_I2C_APPS_CLK_SRC] = &blsp1_qup4_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP4_SPI_APPS_CLK_SRC] = &blsp1_qup4_spi_apps_clk_src.clkr,
+	[BLSP1_QUP5_I2C_APPS_CLK_SRC] = &blsp1_qup5_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP5_SPI_APPS_CLK_SRC] = &blsp1_qup5_spi_apps_clk_src.clkr,
+	[BLSP1_QUP6_I2C_APPS_CLK_SRC] = &blsp1_qup6_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP6_SPI_APPS_CLK_SRC] = &blsp1_qup6_spi_apps_clk_src.clkr,
+	[BLSP1_UART1_APPS_CLK_SRC] = &blsp1_uart1_apps_clk_src.clkr,
+	[BLSP1_UART2_APPS_CLK_SRC] = &blsp1_uart2_apps_clk_src.clkr,
+	[BLSP1_UART3_APPS_CLK_SRC] = &blsp1_uart3_apps_clk_src.clkr,
+	[BLSP1_UART4_APPS_CLK_SRC] = &blsp1_uart4_apps_clk_src.clkr,
+	[BLSP1_UART5_APPS_CLK_SRC] = &blsp1_uart5_apps_clk_src.clkr,
+	[BLSP1_UART6_APPS_CLK_SRC] = &blsp1_uart6_apps_clk_src.clkr,
+	[BLSP2_QUP1_I2C_APPS_CLK_SRC] = &blsp2_qup1_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP1_SPI_APPS_CLK_SRC] = &blsp2_qup1_spi_apps_clk_src.clkr,
+	[BLSP2_QUP2_I2C_APPS_CLK_SRC] = &blsp2_qup2_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP2_SPI_APPS_CLK_SRC] = &blsp2_qup2_spi_apps_clk_src.clkr,
+	[BLSP2_QUP3_I2C_APPS_CLK_SRC] = &blsp2_qup3_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP3_SPI_APPS_CLK_SRC] = &blsp2_qup3_spi_apps_clk_src.clkr,
+	[BLSP2_QUP4_I2C_APPS_CLK_SRC] = &blsp2_qup4_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP4_SPI_APPS_CLK_SRC] = &blsp2_qup4_spi_apps_clk_src.clkr,
+	[BLSP2_QUP5_I2C_APPS_CLK_SRC] = &blsp2_qup5_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP5_SPI_APPS_CLK_SRC] = &blsp2_qup5_spi_apps_clk_src.clkr,
+	[BLSP2_QUP6_I2C_APPS_CLK_SRC] = &blsp2_qup6_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP6_SPI_APPS_CLK_SRC] = &blsp2_qup6_spi_apps_clk_src.clkr,
+	[BLSP2_UART1_APPS_CLK_SRC] = &blsp2_uart1_apps_clk_src.clkr,
+	[BLSP2_UART2_APPS_CLK_SRC] = &blsp2_uart2_apps_clk_src.clkr,
+	[BLSP2_UART3_APPS_CLK_SRC] = &blsp2_uart3_apps_clk_src.clkr,
+	[BLSP2_UART4_APPS_CLK_SRC] = &blsp2_uart4_apps_clk_src.clkr,
+	[BLSP2_UART5_APPS_CLK_SRC] = &blsp2_uart5_apps_clk_src.clkr,
+	[BLSP2_UART6_APPS_CLK_SRC] = &blsp2_uart6_apps_clk_src.clkr,
+	[GP1_CLK_SRC] = &gp1_clk_src.clkr,
+	[GP2_CLK_SRC] = &gp2_clk_src.clkr,
+	[GP3_CLK_SRC] = &gp3_clk_src.clkr,
+	[PCIE_0_AUX_CLK_SRC] = &pcie_0_aux_clk_src.clkr,
+	[PCIE_0_PIPE_CLK_SRC] = &pcie_0_pipe_clk_src.clkr,
+	[PCIE_1_AUX_CLK_SRC] = &pcie_1_aux_clk_src.clkr,
+	[PCIE_1_PIPE_CLK_SRC] = &pcie_1_pipe_clk_src.clkr,
+	[PDM2_CLK_SRC] = &pdm2_clk_src.clkr,
+	[SDCC1_APPS_CLK_SRC] = &sdcc1_apps_clk_src.clkr,
+	[SDCC2_APPS_CLK_SRC] = &sdcc2_apps_clk_src.clkr,
+	[SDCC3_APPS_CLK_SRC] = &sdcc3_apps_clk_src.clkr,
+	[SDCC4_APPS_CLK_SRC] = &sdcc4_apps_clk_src.clkr,
+	[TSIF_REF_CLK_SRC] = &tsif_ref_clk_src.clkr,
+	[USB30_MOCK_UTMI_CLK_SRC] = &usb30_mock_utmi_clk_src.clkr,
+	[USB3_PHY_AUX_CLK_SRC] = &usb3_phy_aux_clk_src.clkr,
+	[USB_HS_SYSTEM_CLK_SRC] = &usb_hs_system_clk_src.clkr,
+	[GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr,
+	[GCC_BLSP1_QUP1_I2C_APPS_CLK] = &gcc_blsp1_qup1_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP1_SPI_APPS_CLK] = &gcc_blsp1_qup1_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP2_I2C_APPS_CLK] = &gcc_blsp1_qup2_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP2_SPI_APPS_CLK] = &gcc_blsp1_qup2_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP3_I2C_APPS_CLK] = &gcc_blsp1_qup3_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP3_SPI_APPS_CLK] = &gcc_blsp1_qup3_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP4_I2C_APPS_CLK] = &gcc_blsp1_qup4_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP4_SPI_APPS_CLK] = &gcc_blsp1_qup4_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP5_I2C_APPS_CLK] = &gcc_blsp1_qup5_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP5_SPI_APPS_CLK] = &gcc_blsp1_qup5_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP6_I2C_APPS_CLK] = &gcc_blsp1_qup6_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP6_SPI_APPS_CLK] = &gcc_blsp1_qup6_spi_apps_clk.clkr,
+	[GCC_BLSP1_UART1_APPS_CLK] = &gcc_blsp1_uart1_apps_clk.clkr,
+	[GCC_BLSP1_UART2_APPS_CLK] = &gcc_blsp1_uart2_apps_clk.clkr,
+	[GCC_BLSP1_UART3_APPS_CLK] = &gcc_blsp1_uart3_apps_clk.clkr,
+	[GCC_BLSP1_UART4_APPS_CLK] = &gcc_blsp1_uart4_apps_clk.clkr,
+	[GCC_BLSP1_UART5_APPS_CLK] = &gcc_blsp1_uart5_apps_clk.clkr,
+	[GCC_BLSP1_UART6_APPS_CLK] = &gcc_blsp1_uart6_apps_clk.clkr,
+	[GCC_BLSP2_AHB_CLK] = &gcc_blsp2_ahb_clk.clkr,
+	[GCC_BLSP2_QUP1_I2C_APPS_CLK] = &gcc_blsp2_qup1_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP1_SPI_APPS_CLK] = &gcc_blsp2_qup1_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP2_I2C_APPS_CLK] = &gcc_blsp2_qup2_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP2_SPI_APPS_CLK] = &gcc_blsp2_qup2_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP3_I2C_APPS_CLK] = &gcc_blsp2_qup3_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP3_SPI_APPS_CLK] = &gcc_blsp2_qup3_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP4_I2C_APPS_CLK] = &gcc_blsp2_qup4_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP4_SPI_APPS_CLK] = &gcc_blsp2_qup4_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP5_I2C_APPS_CLK] = &gcc_blsp2_qup5_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP5_SPI_APPS_CLK] = &gcc_blsp2_qup5_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP6_I2C_APPS_CLK] = &gcc_blsp2_qup6_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP6_SPI_APPS_CLK] = &gcc_blsp2_qup6_spi_apps_clk.clkr,
+	[GCC_BLSP2_UART1_APPS_CLK] = &gcc_blsp2_uart1_apps_clk.clkr,
+	[GCC_BLSP2_UART2_APPS_CLK] = &gcc_blsp2_uart2_apps_clk.clkr,
+	[GCC_BLSP2_UART3_APPS_CLK] = &gcc_blsp2_uart3_apps_clk.clkr,
+	[GCC_BLSP2_UART4_APPS_CLK] = &gcc_blsp2_uart4_apps_clk.clkr,
+	[GCC_BLSP2_UART5_APPS_CLK] = &gcc_blsp2_uart5_apps_clk.clkr,
+	[GCC_BLSP2_UART6_APPS_CLK] = &gcc_blsp2_uart6_apps_clk.clkr,
+	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
+	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
+	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
+	[GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr,
+	[GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr,
+	[GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr,
+	[GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr,
+	[GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr,
+	[GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr,
+	[GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr,
+	[GCC_SDCC3_APPS_CLK] = &gcc_sdcc3_apps_clk.clkr,
+	[GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr,
+	[GCC_SYS_NOC_UFS_AXI_CLK] = &gcc_sys_noc_ufs_axi_clk.clkr,
+	[GCC_SYS_NOC_USB3_AXI_CLK] = &gcc_sys_noc_usb3_axi_clk.clkr,
+	[GCC_TSIF_REF_CLK] = &gcc_tsif_ref_clk.clkr,
+	[GCC_UFS_AXI_CLK] = &gcc_ufs_axi_clk.clkr,
+	[GCC_UFS_RX_CFG_CLK] = &gcc_ufs_rx_cfg_clk.clkr,
+	[GCC_UFS_TX_CFG_CLK] = &gcc_ufs_tx_cfg_clk.clkr,
+	[GCC_USB30_MASTER_CLK] = &gcc_usb30_master_clk.clkr,
+	[GCC_USB30_MOCK_UTMI_CLK] = &gcc_usb30_mock_utmi_clk.clkr,
+	[GCC_USB3_PHY_AUX_CLK] = &gcc_usb3_phy_aux_clk.clkr,
+	[GCC_USB_HS_SYSTEM_CLK] = &gcc_usb_hs_system_clk.clkr,
+};
+
+static const struct regmap_config gcc_msm8994_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0x2000,
+	.fast_io	= true,
+};
+
+static const struct qcom_cc_desc gcc_msm8994_desc = {
+	.config = &gcc_msm8994_regmap_config,
+	.clks = gcc_msm8994_clocks,
+	.num_clks = ARRAY_SIZE(gcc_msm8994_clocks),
+};
+
+static const struct of_device_id gcc_msm8994_match_table[] = {
+	{ .compatible = "qcom,gcc-msm8994" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, gcc_msm8994_match_table);
+
+static int gcc_msm8994_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct clk *clk;
+
+	clk = devm_clk_register(dev, &xo.hw);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	return qcom_cc_probe(pdev, &gcc_msm8994_desc);
+}
+
+static struct platform_driver gcc_msm8994_driver = {
+	.probe		= gcc_msm8994_probe,
+	.driver		= {
+		.name	= "gcc-msm8994",
+		.of_match_table = gcc_msm8994_match_table,
+	},
+};
+
+static int __init gcc_msm8994_init(void)
+{
+	return platform_driver_register(&gcc_msm8994_driver);
+}
+core_initcall(gcc_msm8994_init);
+
+static void __exit gcc_msm8994_exit(void)
+{
+	platform_driver_unregister(&gcc_msm8994_driver);
+}
+module_exit(gcc_msm8994_exit);
+
+MODULE_DESCRIPTION("Qualcomm GCC MSM8994 Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:gcc-msm8994");
diff --git a/drivers/clk/qcom/gcc-msm8996.c b/drivers/clk/qcom/gcc-msm8996.c
index fe03e6fbc7df..4b1fc1730d29 100644
--- a/drivers/clk/qcom/gcc-msm8996.c
+++ b/drivers/clk/qcom/gcc-msm8996.c
@@ -460,14 +460,22 @@ static struct clk_rcg2 sdcc1_apps_clk_src = {
 		.name = "sdcc1_apps_clk_src",
 		.parent_names = gcc_xo_gpll0_gpll4_gpll0_early_div,
 		.num_parents = 4,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
+static struct freq_tbl ftbl_sdcc1_ice_core_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(150000000, P_GPLL0, 4, 0, 0),
+	F(300000000, P_GPLL0, 2, 0, 0),
+	{ }
+};
+
 static struct clk_rcg2 sdcc1_ice_core_clk_src = {
 	.cmd_rcgr = 0x13024,
 	.hid_width = 5,
 	.parent_map = gcc_xo_gpll0_gpll4_gpll0_early_div_map,
+	.freq_tbl = ftbl_sdcc1_ice_core_clk_src,
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "sdcc1_ice_core_clk_src",
 		.parent_names = gcc_xo_gpll0_gpll4_gpll0_early_div,
@@ -497,7 +505,7 @@ static struct clk_rcg2 sdcc2_apps_clk_src = {
 		.name = "sdcc2_apps_clk_src",
 		.parent_names = gcc_xo_gpll0_gpll4,
 		.num_parents = 3,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
@@ -511,7 +519,7 @@ static struct clk_rcg2 sdcc3_apps_clk_src = {
 		.name = "sdcc3_apps_clk_src",
 		.parent_names = gcc_xo_gpll0_gpll4,
 		.num_parents = 3,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
@@ -535,7 +543,7 @@ static struct clk_rcg2 sdcc4_apps_clk_src = {
 		.name = "sdcc4_apps_clk_src",
 		.parent_names = gcc_xo_gpll0,
 		.num_parents = 2,
-		.ops = &clk_rcg2_ops,
+		.ops = &clk_rcg2_floor_ops,
 	},
 };
 
@@ -1230,10 +1238,18 @@ static struct clk_rcg2 ufs_axi_clk_src = {
 	},
 };
 
+static const struct freq_tbl ftbl_ufs_ice_core_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(150000000, P_GPLL0, 4, 0, 0),
+	F(300000000, P_GPLL0, 2, 0, 0),
+	{ }
+};
+
 static struct clk_rcg2 ufs_ice_core_clk_src = {
 	.cmd_rcgr = 0x76014,
 	.hid_width = 5,
 	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_ufs_ice_core_clk_src,
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "ufs_ice_core_clk_src",
 		.parent_names = gcc_xo_gpll0,
@@ -1242,10 +1258,19 @@ static struct clk_rcg2 ufs_ice_core_clk_src = {
 	},
 };
 
+static const struct freq_tbl ftbl_qspi_ser_clk_src[] = {
+	F(75000000, P_GPLL0, 8, 0, 0),
+	F(150000000, P_GPLL0, 4, 0, 0),
+	F(256000000, P_GPLL4, 1.5, 0, 0),
+	F(300000000, P_GPLL0, 2, 0, 0),
+	{ }
+};
+
 static struct clk_rcg2 qspi_ser_clk_src = {
 	.cmd_rcgr = 0x8b00c,
 	.hid_width = 5,
 	.parent_map = gcc_xo_gpll0_gpll1_early_div_gpll1_gpll4_gpll0_early_div_map,
+	.freq_tbl = ftbl_qspi_ser_clk_src,
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "qspi_ser_clk_src",
 		.parent_names = gcc_xo_gpll0_gpll1_early_div_gpll1_gpll4_gpll0_early_div,
diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c
index f12d7b2bddd7..288186cce0ae 100644
--- a/drivers/clk/qcom/gdsc.c
+++ b/drivers/clk/qcom/gdsc.c
@@ -30,6 +30,7 @@
 #define SW_OVERRIDE_MASK	BIT(2)
 #define HW_CONTROL_MASK		BIT(1)
 #define SW_COLLAPSE_MASK	BIT(0)
+#define GMEM_CLAMP_IO_MASK	BIT(0)
 
 /* Wait 2^n CXO cycles between all states. Here, n=2 (4 cycles). */
 #define EN_REST_WAIT_VAL	(0x2 << 20)
@@ -55,6 +56,13 @@ static int gdsc_is_enabled(struct gdsc *sc, unsigned int reg)
 	return !!(val & PWR_ON_MASK);
 }
 
+static int gdsc_hwctrl(struct gdsc *sc, bool en)
+{
+	u32 val = en ? HW_CONTROL_MASK : 0;
+
+	return regmap_update_bits(sc->regmap, sc->gdscr, HW_CONTROL_MASK, val);
+}
+
 static int gdsc_toggle_logic(struct gdsc *sc, bool en)
 {
 	int ret;
@@ -140,6 +148,18 @@ static inline void gdsc_clear_mem_on(struct gdsc *sc)
 		regmap_update_bits(sc->regmap, sc->cxcs[i], mask, 0);
 }
 
+static inline void gdsc_deassert_clamp_io(struct gdsc *sc)
+{
+	regmap_update_bits(sc->regmap, sc->clamp_io_ctrl,
+			   GMEM_CLAMP_IO_MASK, 0);
+}
+
+static inline void gdsc_assert_clamp_io(struct gdsc *sc)
+{
+	regmap_update_bits(sc->regmap, sc->clamp_io_ctrl,
+			   GMEM_CLAMP_IO_MASK, 1);
+}
+
 static int gdsc_enable(struct generic_pm_domain *domain)
 {
 	struct gdsc *sc = domain_to_gdsc(domain);
@@ -148,6 +168,9 @@ static int gdsc_enable(struct generic_pm_domain *domain)
 	if (sc->pwrsts == PWRSTS_ON)
 		return gdsc_deassert_reset(sc);
 
+	if (sc->flags & CLAMP_IO)
+		gdsc_deassert_clamp_io(sc);
+
 	ret = gdsc_toggle_logic(sc, true);
 	if (ret)
 		return ret;
@@ -164,20 +187,39 @@ static int gdsc_enable(struct generic_pm_domain *domain)
 	 */
 	udelay(1);
 
+	/* Turn on HW trigger mode if supported */
+	if (sc->flags & HW_CTRL)
+		return gdsc_hwctrl(sc, true);
+
 	return 0;
 }
 
 static int gdsc_disable(struct generic_pm_domain *domain)
 {
 	struct gdsc *sc = domain_to_gdsc(domain);
+	int ret;
 
 	if (sc->pwrsts == PWRSTS_ON)
 		return gdsc_assert_reset(sc);
 
+	/* Turn off HW trigger mode if supported */
+	if (sc->flags & HW_CTRL) {
+		ret = gdsc_hwctrl(sc, false);
+		if (ret < 0)
+			return ret;
+	}
+
 	if (sc->pwrsts & PWRSTS_OFF)
 		gdsc_clear_mem_on(sc);
 
-	return gdsc_toggle_logic(sc, false);
+	ret = gdsc_toggle_logic(sc, false);
+	if (ret)
+		return ret;
+
+	if (sc->flags & CLAMP_IO)
+		gdsc_assert_clamp_io(sc);
+
+	return 0;
 }
 
 static int gdsc_init(struct gdsc *sc)
diff --git a/drivers/clk/qcom/gdsc.h b/drivers/clk/qcom/gdsc.h
index 3bf497c36bdf..39648348e5ec 100644
--- a/drivers/clk/qcom/gdsc.h
+++ b/drivers/clk/qcom/gdsc.h
@@ -39,6 +39,7 @@ struct gdsc {
 	struct regmap			*regmap;
 	unsigned int			gdscr;
 	unsigned int			gds_hw_ctrl;
+	unsigned int			clamp_io_ctrl;
 	unsigned int			*cxcs;
 	unsigned int			cxc_count;
 	const u8			pwrsts;
@@ -50,6 +51,8 @@ struct gdsc {
 #define PWRSTS_RET_ON		(PWRSTS_RET | PWRSTS_ON)
 	const u8			flags;
 #define VOTABLE		BIT(0)
+#define CLAMP_IO	BIT(1)
+#define HW_CTRL		BIT(2)
 	struct reset_controller_dev	*rcdev;
 	unsigned int			*resets;
 	unsigned int			reset_count;
diff --git a/drivers/clk/qcom/lcc-ipq806x.c b/drivers/clk/qcom/lcc-ipq806x.c
index db3998e5e2d8..977e98eadbeb 100644
--- a/drivers/clk/qcom/lcc-ipq806x.c
+++ b/drivers/clk/qcom/lcc-ipq806x.c
@@ -443,7 +443,7 @@ static int lcc_ipq806x_probe(struct platform_device *pdev)
 		return PTR_ERR(regmap);
 
 	/* Configure the rate of PLL4 if the bootloader hasn't already */
-	val = regmap_read(regmap, 0x0, &val);
+	regmap_read(regmap, 0x0, &val);
 	if (!val)
 		clk_pll_configure_sr(&pll4, regmap, &pll4_config, true);
 	/* Enable PLL4 source on the LPASS Primary PLL Mux */
diff --git a/drivers/clk/qcom/mmcc-msm8996.c b/drivers/clk/qcom/mmcc-msm8996.c
index ca97e1151797..9b97246287a7 100644
--- a/drivers/clk/qcom/mmcc-msm8996.c
+++ b/drivers/clk/qcom/mmcc-msm8996.c
@@ -2945,6 +2945,7 @@ static struct gdsc venus_core0_gdsc = {
 		.name = "venus_core0",
 	},
 	.pwrsts = PWRSTS_OFF_ON,
+	.flags = HW_CTRL,
 };
 
 static struct gdsc venus_core1_gdsc = {
@@ -2955,6 +2956,7 @@ static struct gdsc venus_core1_gdsc = {
 		.name = "venus_core1",
 	},
 	.pwrsts = PWRSTS_OFF_ON,
+	.flags = HW_CTRL,
 };
 
 static struct gdsc camss_gdsc = {
@@ -3034,6 +3036,28 @@ static struct gdsc mdss_gdsc = {
 	.pwrsts = PWRSTS_OFF_ON,
 };
 
+static struct gdsc gpu_gdsc = {
+	.gdscr = 0x4034,
+	.gds_hw_ctrl = 0x4038,
+	.pd = {
+		.name = "gpu",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = VOTABLE,
+};
+
+static struct gdsc gpu_gx_gdsc = {
+	.gdscr = 0x4024,
+	.clamp_io_ctrl = 0x4300,
+	.cxcs = (unsigned int []){ 0x4028 },
+	.cxc_count = 1,
+	.pd = {
+		.name = "gpu_gx",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = CLAMP_IO,
+};
+
 static struct clk_regmap *mmcc_msm8996_clocks[] = {
 	[MMPLL0_EARLY] = &mmpll0_early.clkr,
 	[MMPLL0_PLL] = &mmpll0.clkr,
@@ -3223,6 +3247,8 @@ static struct gdsc *mmcc_msm8996_gdscs[] = {
 	[CPP_GDSC] = &cpp_gdsc,
 	[FD_GDSC] = &fd_gdsc,
 	[MDSS_GDSC] = &mdss_gdsc,
+	[GPU_GDSC] = &gpu_gdsc,
+	[GPU_GX_GDSC] = &gpu_gx_gdsc,
 };
 
 static const struct qcom_reset_map mmcc_msm8996_resets[] = {
diff --git a/drivers/clk/renesas/Kconfig b/drivers/clk/renesas/Kconfig
index 41a12d376799..2586dfa0026b 100644
--- a/drivers/clk/renesas/Kconfig
+++ b/drivers/clk/renesas/Kconfig
@@ -1,5 +1,7 @@
 config CLK_RENESAS_CPG_MSSR
 	bool
+	default y if ARCH_R8A7743
+	default y if ARCH_R8A7745
 	default y if ARCH_R8A7795
 	default y if ARCH_R8A7796
 
diff --git a/drivers/clk/renesas/Makefile b/drivers/clk/renesas/Makefile
index 90dd0db7d9c6..1072f7653c0c 100644
--- a/drivers/clk/renesas/Makefile
+++ b/drivers/clk/renesas/Makefile
@@ -2,6 +2,8 @@ obj-$(CONFIG_ARCH_EMEV2)		+= clk-emev2.o
 obj-$(CONFIG_ARCH_R7S72100)		+= clk-rz.o
 obj-$(CONFIG_ARCH_R8A73A4)		+= clk-r8a73a4.o clk-div6.o
 obj-$(CONFIG_ARCH_R8A7740)		+= clk-r8a7740.o clk-div6.o
+obj-$(CONFIG_ARCH_R8A7743)		+= r8a7743-cpg-mssr.o rcar-gen2-cpg.o
+obj-$(CONFIG_ARCH_R8A7745)		+= r8a7745-cpg-mssr.o rcar-gen2-cpg.o
 obj-$(CONFIG_ARCH_R8A7778)		+= clk-r8a7778.o
 obj-$(CONFIG_ARCH_R8A7779)		+= clk-r8a7779.o
 obj-$(CONFIG_ARCH_R8A7790)		+= clk-rcar-gen2.o clk-div6.o
diff --git a/drivers/clk/renesas/clk-r8a7778.c b/drivers/clk/renesas/clk-r8a7778.c
index 40e3a501a50e..886a8380e912 100644
--- a/drivers/clk/renesas/clk-r8a7778.c
+++ b/drivers/clk/renesas/clk-r8a7778.c
@@ -12,6 +12,7 @@
 #include <linux/clk/renesas.h>
 #include <linux/of_address.h>
 #include <linux/slab.h>
+#include <linux/soc/renesas/rcar-rst.h>
 
 struct r8a7778_cpg {
 	struct clk_onecell_data data;
@@ -83,6 +84,18 @@ static void __init r8a7778_cpg_clocks_init(struct device_node *np)
 	struct clk **clks;
 	unsigned int i;
 	int num_clks;
+	u32 mode;
+
+	if (rcar_rst_read_mode_pins(&mode))
+		return;
+
+	BUG_ON(!(mode & BIT(19)));
+
+	cpg_mode_rates = (!!(mode & BIT(18)) << 2) |
+			 (!!(mode & BIT(12)) << 1) |
+			 (!!(mode & BIT(11)));
+	cpg_mode_divs = (!!(mode & BIT(2)) << 1) |
+			(!!(mode & BIT(1)));
 
 	num_clks = of_property_count_strings(np, "clock-output-names");
 	if (num_clks < 0) {
@@ -130,16 +143,3 @@ static void __init r8a7778_cpg_clocks_init(struct device_node *np)
 
 CLK_OF_DECLARE(r8a7778_cpg_clks, "renesas,r8a7778-cpg-clocks",
 	       r8a7778_cpg_clocks_init);
-
-void __init r8a7778_clocks_init(u32 mode)
-{
-	BUG_ON(!(mode & BIT(19)));
-
-	cpg_mode_rates = (!!(mode & BIT(18)) << 2) |
-			 (!!(mode & BIT(12)) << 1) |
-			 (!!(mode & BIT(11)));
-	cpg_mode_divs = (!!(mode & BIT(2)) << 1) |
-			(!!(mode & BIT(1)));
-
-	of_clk_init(NULL);
-}
diff --git a/drivers/clk/renesas/clk-r8a7779.c b/drivers/clk/renesas/clk-r8a7779.c
index cf2a37df03b1..27fbfafaf2cd 100644
--- a/drivers/clk/renesas/clk-r8a7779.c
+++ b/drivers/clk/renesas/clk-r8a7779.c
@@ -18,6 +18,7 @@
 #include <linux/of_address.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/soc/renesas/rcar-rst.h>
 
 #include <dt-bindings/clock/r8a7779-clock.h>
 
@@ -88,8 +89,6 @@ static const unsigned int cpg_plla_mult[4] __initconst = { 42, 48, 56, 64 };
  * Initialization
  */
 
-static u32 cpg_mode __initdata;
-
 static struct clk * __init
 r8a7779_cpg_register_clock(struct device_node *np, struct r8a7779_cpg *cpg,
 			   const struct cpg_clk_config *config,
@@ -127,6 +126,10 @@ static void __init r8a7779_cpg_clocks_init(struct device_node *np)
 	struct clk **clks;
 	unsigned int i, plla_mult;
 	int num_clks;
+	u32 mode;
+
+	if (rcar_rst_read_mode_pins(&mode))
+		return;
 
 	num_clks = of_property_count_strings(np, "clock-output-names");
 	if (num_clks < 0) {
@@ -148,8 +151,8 @@ static void __init r8a7779_cpg_clocks_init(struct device_node *np)
 	cpg->data.clks = clks;
 	cpg->data.clk_num = num_clks;
 
-	config = &cpg_clk_configs[CPG_CLK_CONFIG_INDEX(cpg_mode)];
-	plla_mult = cpg_plla_mult[CPG_PLLA_MULT_INDEX(cpg_mode)];
+	config = &cpg_clk_configs[CPG_CLK_CONFIG_INDEX(mode)];
+	plla_mult = cpg_plla_mult[CPG_PLLA_MULT_INDEX(mode)];
 
 	for (i = 0; i < num_clks; ++i) {
 		const char *name;
@@ -173,10 +176,3 @@ static void __init r8a7779_cpg_clocks_init(struct device_node *np)
 }
 CLK_OF_DECLARE(r8a7779_cpg_clks, "renesas,r8a7779-cpg-clocks",
 	       r8a7779_cpg_clocks_init);
-
-void __init r8a7779_clocks_init(u32 mode)
-{
-	cpg_mode = mode;
-
-	of_clk_init(NULL);
-}
diff --git a/drivers/clk/renesas/clk-rcar-gen2.c b/drivers/clk/renesas/clk-rcar-gen2.c
index 00e6aba4b9c0..f39519edc645 100644
--- a/drivers/clk/renesas/clk-rcar-gen2.c
+++ b/drivers/clk/renesas/clk-rcar-gen2.c
@@ -19,6 +19,7 @@
 #include <linux/of_address.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/soc/renesas/rcar-rst.h>
 
 struct rcar_gen2_cpg {
 	struct clk_onecell_data data;
@@ -364,6 +365,23 @@ rcar_gen2_cpg_register_clock(struct device_node *np, struct rcar_gen2_cpg *cpg,
 						 4, 0, table, &cpg->lock);
 }
 
+/*
+ * Reset register definitions.
+ */
+#define MODEMR	0xe6160060
+
+static u32 __init rcar_gen2_read_mode_pins(void)
+{
+	void __iomem *modemr = ioremap_nocache(MODEMR, 4);
+	u32 mode;
+
+	BUG_ON(!modemr);
+	mode = ioread32(modemr);
+	iounmap(modemr);
+
+	return mode;
+}
+
 static void __init rcar_gen2_cpg_clocks_init(struct device_node *np)
 {
 	const struct cpg_pll_config *config;
@@ -372,6 +390,13 @@ static void __init rcar_gen2_cpg_clocks_init(struct device_node *np)
 	unsigned int i;
 	int num_clks;
 
+	if (rcar_rst_read_mode_pins(&cpg_mode)) {
+		/* Backward-compatibility with old DT */
+		pr_warn("%s: failed to obtain mode pins from RST\n",
+			np->full_name);
+		cpg_mode = rcar_gen2_read_mode_pins();
+	}
+
 	num_clks = of_property_count_strings(np, "clock-output-names");
 	if (num_clks < 0) {
 		pr_err("%s: failed to count clocks\n", __func__);
@@ -420,10 +445,3 @@ static void __init rcar_gen2_cpg_clocks_init(struct device_node *np)
 }
 CLK_OF_DECLARE(rcar_gen2_cpg_clks, "renesas,rcar-gen2-cpg-clocks",
 	       rcar_gen2_cpg_clocks_init);
-
-void __init rcar_gen2_clocks_init(u32 mode)
-{
-	cpg_mode = mode;
-
-	of_clk_init(NULL);
-}
diff --git a/drivers/clk/renesas/r8a7743-cpg-mssr.c b/drivers/clk/renesas/r8a7743-cpg-mssr.c
new file mode 100644
index 000000000000..6dc0b3082aa6
--- /dev/null
+++ b/drivers/clk/renesas/r8a7743-cpg-mssr.c
@@ -0,0 +1,270 @@
+/*
+ * r8a7743 Clock Pulse Generator / Module Standby and Software Reset
+ *
+ * Copyright (C) 2016 Cogent Embedded Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation; of the License.
+ */
+
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/soc/renesas/rcar-rst.h>
+
+#include <dt-bindings/clock/r8a7743-cpg-mssr.h>
+
+#include "renesas-cpg-mssr.h"
+#include "rcar-gen2-cpg.h"
+
+enum clk_ids {
+	/* Core Clock Outputs exported to DT */
+	LAST_DT_CORE_CLK = R8A7743_CLK_OSC,
+
+	/* External Input Clocks */
+	CLK_EXTAL,
+	CLK_USB_EXTAL,
+
+	/* Internal Core Clocks */
+	CLK_MAIN,
+	CLK_PLL0,
+	CLK_PLL1,
+	CLK_PLL3,
+	CLK_PLL1_DIV2,
+
+	/* Module Clocks */
+	MOD_CLK_BASE
+};
+
+static const struct cpg_core_clk r8a7743_core_clks[] __initconst = {
+	/* External Clock Inputs */
+	DEF_INPUT("extal",	CLK_EXTAL),
+	DEF_INPUT("usb_extal",	CLK_USB_EXTAL),
+
+	/* Internal Core Clocks */
+	DEF_BASE(".main",	CLK_MAIN, CLK_TYPE_GEN2_MAIN, CLK_EXTAL),
+	DEF_BASE(".pll0",	CLK_PLL0, CLK_TYPE_GEN2_PLL0, CLK_MAIN),
+	DEF_BASE(".pll1",	CLK_PLL1, CLK_TYPE_GEN2_PLL1, CLK_MAIN),
+	DEF_BASE(".pll3",	CLK_PLL3, CLK_TYPE_GEN2_PLL3, CLK_MAIN),
+
+	DEF_FIXED(".pll1_div2",	CLK_PLL1_DIV2, CLK_PLL1, 2, 1),
+
+	/* Core Clock Outputs */
+	DEF_BASE("z",    R8A7743_CLK_Z,    CLK_TYPE_GEN2_Z,	CLK_PLL0),
+	DEF_BASE("lb",   R8A7743_CLK_LB,   CLK_TYPE_GEN2_LB,	CLK_PLL1),
+	DEF_BASE("sdh",  R8A7743_CLK_SDH,  CLK_TYPE_GEN2_SDH,	CLK_PLL1),
+	DEF_BASE("sd0",  R8A7743_CLK_SD0,  CLK_TYPE_GEN2_SD0,	CLK_PLL1),
+	DEF_BASE("qspi", R8A7743_CLK_QSPI, CLK_TYPE_GEN2_QSPI,	CLK_PLL1_DIV2),
+	DEF_BASE("rcan", R8A7743_CLK_RCAN, CLK_TYPE_GEN2_RCAN,	CLK_USB_EXTAL),
+
+	DEF_FIXED("zg",    R8A7743_CLK_ZG,	CLK_PLL1,	    3, 1),
+	DEF_FIXED("zx",    R8A7743_CLK_ZX,	CLK_PLL1,	    3, 1),
+	DEF_FIXED("zs",    R8A7743_CLK_ZS,	CLK_PLL1,	    6, 1),
+	DEF_FIXED("hp",    R8A7743_CLK_HP,	CLK_PLL1,	   12, 1),
+	DEF_FIXED("b",     R8A7743_CLK_B,	CLK_PLL1,	   12, 1),
+	DEF_FIXED("p",     R8A7743_CLK_P,	CLK_PLL1,	   24, 1),
+	DEF_FIXED("cl",    R8A7743_CLK_CL,	CLK_PLL1,	   48, 1),
+	DEF_FIXED("m2",    R8A7743_CLK_M2,	CLK_PLL1,	    8, 1),
+	DEF_FIXED("zb3",   R8A7743_CLK_ZB3,	CLK_PLL3,	    4, 1),
+	DEF_FIXED("zb3d2", R8A7743_CLK_ZB3D2,	CLK_PLL3,	    8, 1),
+	DEF_FIXED("ddr",   R8A7743_CLK_DDR,	CLK_PLL3,	    8, 1),
+	DEF_FIXED("mp",    R8A7743_CLK_MP,	CLK_PLL1_DIV2,	   15, 1),
+	DEF_FIXED("cp",    R8A7743_CLK_CP,	CLK_EXTAL,	    2, 1),
+	DEF_FIXED("r",     R8A7743_CLK_R,	CLK_PLL1,	49152, 1),
+	DEF_FIXED("osc",   R8A7743_CLK_OSC,	CLK_PLL1,	12288, 1),
+
+	DEF_DIV6P1("sd2",  R8A7743_CLK_SD2,	CLK_PLL1_DIV2,	0x078),
+	DEF_DIV6P1("sd3",  R8A7743_CLK_SD3,	CLK_PLL1_DIV2,	0x26c),
+	DEF_DIV6P1("mmc0", R8A7743_CLK_MMC0,	CLK_PLL1_DIV2,	0x240),
+};
+
+static const struct mssr_mod_clk r8a7743_mod_clks[] __initconst = {
+	DEF_MOD("msiof0",		   0,	R8A7743_CLK_MP),
+	DEF_MOD("vcp0",			 101,	R8A7743_CLK_ZS),
+	DEF_MOD("vpc0",			 103,	R8A7743_CLK_ZS),
+	DEF_MOD("tmu1",			 111,	R8A7743_CLK_P),
+	DEF_MOD("3dg",			 112,	R8A7743_CLK_ZG),
+	DEF_MOD("2d-dmac",		 115,	R8A7743_CLK_ZS),
+	DEF_MOD("fdp1-1",		 118,	R8A7743_CLK_ZS),
+	DEF_MOD("fdp1-0",		 119,	R8A7743_CLK_ZS),
+	DEF_MOD("tmu3",			 121,	R8A7743_CLK_P),
+	DEF_MOD("tmu2",			 122,	R8A7743_CLK_P),
+	DEF_MOD("cmt0",			 124,	R8A7743_CLK_R),
+	DEF_MOD("tmu0",			 125,	R8A7743_CLK_CP),
+	DEF_MOD("vsp1du1",		 127,	R8A7743_CLK_ZS),
+	DEF_MOD("vsp1du0",		 128,	R8A7743_CLK_ZS),
+	DEF_MOD("vsp1-sy",		 131,	R8A7743_CLK_ZS),
+	DEF_MOD("scifa2",		 202,	R8A7743_CLK_MP),
+	DEF_MOD("scifa1",		 203,	R8A7743_CLK_MP),
+	DEF_MOD("scifa0",		 204,	R8A7743_CLK_MP),
+	DEF_MOD("msiof2",		 205,	R8A7743_CLK_MP),
+	DEF_MOD("scifb0",		 206,	R8A7743_CLK_MP),
+	DEF_MOD("scifb1",		 207,	R8A7743_CLK_MP),
+	DEF_MOD("msiof1",		 208,	R8A7743_CLK_MP),
+	DEF_MOD("scifb2",		 216,	R8A7743_CLK_MP),
+	DEF_MOD("sys-dmac1",		 218,	R8A7743_CLK_ZS),
+	DEF_MOD("sys-dmac0",		 219,	R8A7743_CLK_ZS),
+	DEF_MOD("tpu0",			 304,	R8A7743_CLK_CP),
+	DEF_MOD("sdhi3",		 311,	R8A7743_CLK_SD3),
+	DEF_MOD("sdhi2",		 312,	R8A7743_CLK_SD2),
+	DEF_MOD("sdhi0",		 314,	R8A7743_CLK_SD0),
+	DEF_MOD("mmcif0",		 315,	R8A7743_CLK_MMC0),
+	DEF_MOD("iic0",			 318,	R8A7743_CLK_HP),
+	DEF_MOD("pciec",		 319,	R8A7743_CLK_MP),
+	DEF_MOD("iic1",			 323,	R8A7743_CLK_HP),
+	DEF_MOD("usb3.0",		 328,	R8A7743_CLK_MP),
+	DEF_MOD("cmt1",			 329,	R8A7743_CLK_R),
+	DEF_MOD("usbhs-dmac0",		 330,	R8A7743_CLK_HP),
+	DEF_MOD("usbhs-dmac1",		 331,	R8A7743_CLK_HP),
+	DEF_MOD("irqc",			 407,	R8A7743_CLK_CP),
+	DEF_MOD("intc-sys",		 408,	R8A7743_CLK_ZS),
+	DEF_MOD("audio-dmac1",		 501,	R8A7743_CLK_HP),
+	DEF_MOD("audio-dmac0",		 502,	R8A7743_CLK_HP),
+	DEF_MOD("thermal",		 522,	CLK_EXTAL),
+	DEF_MOD("pwm",			 523,	R8A7743_CLK_P),
+	DEF_MOD("usb-ehci",		 703,	R8A7743_CLK_MP),
+	DEF_MOD("usbhs",		 704,	R8A7743_CLK_HP),
+	DEF_MOD("hscif2",		 713,	R8A7743_CLK_ZS),
+	DEF_MOD("scif5",		 714,	R8A7743_CLK_P),
+	DEF_MOD("scif4",		 715,	R8A7743_CLK_P),
+	DEF_MOD("hscif1",		 716,	R8A7743_CLK_ZS),
+	DEF_MOD("hscif0",		 717,	R8A7743_CLK_ZS),
+	DEF_MOD("scif3",		 718,	R8A7743_CLK_P),
+	DEF_MOD("scif2",		 719,	R8A7743_CLK_P),
+	DEF_MOD("scif1",		 720,	R8A7743_CLK_P),
+	DEF_MOD("scif0",		 721,	R8A7743_CLK_P),
+	DEF_MOD("du1",			 723,	R8A7743_CLK_ZX),
+	DEF_MOD("du0",			 724,	R8A7743_CLK_ZX),
+	DEF_MOD("lvds0",		 726,	R8A7743_CLK_ZX),
+	DEF_MOD("ipmmu-sgx",		 800,	R8A7743_CLK_ZX),
+	DEF_MOD("vin2",			 809,	R8A7743_CLK_ZG),
+	DEF_MOD("vin1",			 810,	R8A7743_CLK_ZG),
+	DEF_MOD("vin0",			 811,	R8A7743_CLK_ZG),
+	DEF_MOD("etheravb",		 812,	R8A7743_CLK_HP),
+	DEF_MOD("ether",		 813,	R8A7743_CLK_P),
+	DEF_MOD("sata1",		 814,	R8A7743_CLK_ZS),
+	DEF_MOD("sata0",		 815,	R8A7743_CLK_ZS),
+	DEF_MOD("gpio7",		 904,	R8A7743_CLK_CP),
+	DEF_MOD("gpio6",		 905,	R8A7743_CLK_CP),
+	DEF_MOD("gpio5",		 907,	R8A7743_CLK_CP),
+	DEF_MOD("gpio4",		 908,	R8A7743_CLK_CP),
+	DEF_MOD("gpio3",		 909,	R8A7743_CLK_CP),
+	DEF_MOD("gpio2",		 910,	R8A7743_CLK_CP),
+	DEF_MOD("gpio1",		 911,	R8A7743_CLK_CP),
+	DEF_MOD("gpio0",		 912,	R8A7743_CLK_CP),
+	DEF_MOD("can1",			 915,	R8A7743_CLK_P),
+	DEF_MOD("can0",			 916,	R8A7743_CLK_P),
+	DEF_MOD("qspi_mod",		 917,	R8A7743_CLK_QSPI),
+	DEF_MOD("i2c5",			 925,	R8A7743_CLK_HP),
+	DEF_MOD("iicdvfs",		 926,	R8A7743_CLK_CP),
+	DEF_MOD("i2c4",			 927,	R8A7743_CLK_HP),
+	DEF_MOD("i2c3",			 928,	R8A7743_CLK_HP),
+	DEF_MOD("i2c2",			 929,	R8A7743_CLK_HP),
+	DEF_MOD("i2c1",			 930,	R8A7743_CLK_HP),
+	DEF_MOD("i2c0",			 931,	R8A7743_CLK_HP),
+	DEF_MOD("ssi-all",		1005,	R8A7743_CLK_P),
+	DEF_MOD("ssi9",			1006,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi8",			1007,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi7",			1008,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi6",			1009,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi5",			1010,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi4",			1011,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi3",			1012,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi2",			1013,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi1",			1014,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi0",			1015,	MOD_CLK_ID(1005)),
+	DEF_MOD("scu-all",		1017,	R8A7743_CLK_P),
+	DEF_MOD("scu-dvc1",		1018,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-dvc0",		1019,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu1-mix1",	1020,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu0-mix0",	1021,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src9",		1022,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src8",		1023,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src7",		1024,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src6",		1025,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src5",		1026,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src4",		1027,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src3",		1028,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src2",		1029,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src1",		1030,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src0",		1031,	MOD_CLK_ID(1017)),
+	DEF_MOD("scifa3",		1106,	R8A7743_CLK_MP),
+	DEF_MOD("scifa4",		1107,	R8A7743_CLK_MP),
+	DEF_MOD("scifa5",		1108,	R8A7743_CLK_MP),
+};
+
+static const unsigned int r8a7743_crit_mod_clks[] __initconst = {
+	MOD_CLK_ID(408),	/* INTC-SYS (GIC) */
+};
+
+/*
+ * CPG Clock Data
+ */
+
+/*
+ *    MD	EXTAL		PLL0	PLL1	PLL3
+ * 14 13 19	(MHz)		*1	*1
+ *---------------------------------------------------
+ * 0  0  0	15		x172/2	x208/2	x106
+ * 0  0  1	15		x172/2	x208/2	x88
+ * 0  1  0	20		x130/2	x156/2	x80
+ * 0  1  1	20		x130/2	x156/2	x66
+ * 1  0  0	26 / 2		x200/2	x240/2	x122
+ * 1  0  1	26 / 2		x200/2	x240/2	x102
+ * 1  1  0	30 / 2		x172/2	x208/2	x106
+ * 1  1  1	30 / 2		x172/2	x208/2	x88
+ *
+ * *1 :	Table 7.5a indicates VCO output (PLLx = VCO/2)
+ */
+#define CPG_PLL_CONFIG_INDEX(md)	((((md) & BIT(14)) >> 12) | \
+					 (((md) & BIT(13)) >> 12) | \
+					 (((md) & BIT(19)) >> 19))
+
+static const struct rcar_gen2_cpg_pll_config cpg_pll_configs[8] __initconst = {
+	/* EXTAL div	PLL1 mult	PLL3 mult */
+	{ 1,		208,		106,	},
+	{ 1,		208,		88,	},
+	{ 1,		156,		80,	},
+	{ 1,		156,		66,	},
+	{ 2,		240,		122,	},
+	{ 2,		240,		102,	},
+	{ 2,		208,		106,	},
+	{ 2,		208,		88,	},
+};
+
+static int __init r8a7743_cpg_mssr_init(struct device *dev)
+{
+	const struct rcar_gen2_cpg_pll_config *cpg_pll_config;
+	u32 cpg_mode;
+	int error;
+
+	error = rcar_rst_read_mode_pins(&cpg_mode);
+	if (error)
+		return error;
+
+	cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)];
+
+	return rcar_gen2_cpg_init(cpg_pll_config, 2, cpg_mode);
+}
+
+const struct cpg_mssr_info r8a7743_cpg_mssr_info __initconst = {
+	/* Core Clocks */
+	.core_clks = r8a7743_core_clks,
+	.num_core_clks = ARRAY_SIZE(r8a7743_core_clks),
+	.last_dt_core_clk = LAST_DT_CORE_CLK,
+	.num_total_core_clks = MOD_CLK_BASE,
+
+	/* Module Clocks */
+	.mod_clks = r8a7743_mod_clks,
+	.num_mod_clks = ARRAY_SIZE(r8a7743_mod_clks),
+	.num_hw_mod_clks = 12 * 32,
+
+	/* Critical Module Clocks */
+	.crit_mod_clks = r8a7743_crit_mod_clks,
+	.num_crit_mod_clks = ARRAY_SIZE(r8a7743_crit_mod_clks),
+
+	/* Callbacks */
+	.init = r8a7743_cpg_mssr_init,
+	.cpg_clk_register = rcar_gen2_cpg_clk_register,
+};
diff --git a/drivers/clk/renesas/r8a7745-cpg-mssr.c b/drivers/clk/renesas/r8a7745-cpg-mssr.c
new file mode 100644
index 000000000000..2f15ba786c3b
--- /dev/null
+++ b/drivers/clk/renesas/r8a7745-cpg-mssr.c
@@ -0,0 +1,259 @@
+/*
+ * r8a7745 Clock Pulse Generator / Module Standby and Software Reset
+ *
+ * Copyright (C) 2016 Cogent Embedded Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation; of the License.
+ */
+
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/soc/renesas/rcar-rst.h>
+
+#include <dt-bindings/clock/r8a7745-cpg-mssr.h>
+
+#include "renesas-cpg-mssr.h"
+#include "rcar-gen2-cpg.h"
+
+enum clk_ids {
+	/* Core Clock Outputs exported to DT */
+	LAST_DT_CORE_CLK = R8A7745_CLK_OSC,
+
+	/* External Input Clocks */
+	CLK_EXTAL,
+	CLK_USB_EXTAL,
+
+	/* Internal Core Clocks */
+	CLK_MAIN,
+	CLK_PLL0,
+	CLK_PLL1,
+	CLK_PLL3,
+	CLK_PLL1_DIV2,
+
+	/* Module Clocks */
+	MOD_CLK_BASE
+};
+
+static const struct cpg_core_clk r8a7745_core_clks[] __initconst = {
+	/* External Clock Inputs */
+	DEF_INPUT("extal",	CLK_EXTAL),
+	DEF_INPUT("usb_extal",	CLK_USB_EXTAL),
+
+	/* Internal Core Clocks */
+	DEF_BASE(".main",	CLK_MAIN, CLK_TYPE_GEN2_MAIN, CLK_EXTAL),
+	DEF_BASE(".pll0",	CLK_PLL0, CLK_TYPE_GEN2_PLL0, CLK_MAIN),
+	DEF_BASE(".pll1",	CLK_PLL1, CLK_TYPE_GEN2_PLL1, CLK_MAIN),
+	DEF_BASE(".pll3",	CLK_PLL3, CLK_TYPE_GEN2_PLL3, CLK_MAIN),
+
+	DEF_FIXED(".pll1_div2",	CLK_PLL1_DIV2, CLK_PLL1, 2, 1),
+
+	/* Core Clock Outputs */
+	DEF_BASE("lb",   R8A7745_CLK_LB,   CLK_TYPE_GEN2_LB,	CLK_PLL1),
+	DEF_BASE("sdh",  R8A7745_CLK_SDH,  CLK_TYPE_GEN2_SDH,	CLK_PLL1),
+	DEF_BASE("sd0",  R8A7745_CLK_SD0,  CLK_TYPE_GEN2_SD0,	CLK_PLL1),
+	DEF_BASE("qspi", R8A7745_CLK_QSPI, CLK_TYPE_GEN2_QSPI,	CLK_PLL1_DIV2),
+	DEF_BASE("rcan", R8A7745_CLK_RCAN, CLK_TYPE_GEN2_RCAN,	CLK_USB_EXTAL),
+
+	DEF_FIXED("z2",    R8A7745_CLK_Z2,	CLK_PLL0,	    1, 1),
+	DEF_FIXED("zg",    R8A7745_CLK_ZG,	CLK_PLL1,	    6, 1),
+	DEF_FIXED("zx",    R8A7745_CLK_ZX,	CLK_PLL1,	    3, 1),
+	DEF_FIXED("zs",    R8A7745_CLK_ZS,	CLK_PLL1,	    6, 1),
+	DEF_FIXED("hp",    R8A7745_CLK_HP,	CLK_PLL1,	   12, 1),
+	DEF_FIXED("b",     R8A7745_CLK_B,	CLK_PLL1,	   12, 1),
+	DEF_FIXED("p",     R8A7745_CLK_P,	CLK_PLL1,	   24, 1),
+	DEF_FIXED("cl",    R8A7745_CLK_CL,	CLK_PLL1,	   48, 1),
+	DEF_FIXED("cp",    R8A7745_CLK_CP,	CLK_PLL1,	   48, 1),
+	DEF_FIXED("m2",    R8A7745_CLK_M2,	CLK_PLL1,	    8, 1),
+	DEF_FIXED("zb3",   R8A7745_CLK_ZB3,	CLK_PLL3,	    4, 1),
+	DEF_FIXED("zb3d2", R8A7745_CLK_ZB3D2,	CLK_PLL3,	    8, 1),
+	DEF_FIXED("ddr",   R8A7745_CLK_DDR,	CLK_PLL3,	    8, 1),
+	DEF_FIXED("mp",    R8A7745_CLK_MP,	CLK_PLL1_DIV2,	   15, 1),
+	DEF_FIXED("cpex",  R8A7745_CLK_CPEX,	CLK_EXTAL,	    2, 1),
+	DEF_FIXED("r",     R8A7745_CLK_R,	CLK_PLL1,	49152, 1),
+	DEF_FIXED("osc",   R8A7745_CLK_OSC,	CLK_PLL1,	12288, 1),
+
+	DEF_DIV6P1("sd2",  R8A7745_CLK_SD2,	CLK_PLL1_DIV2,	0x078),
+	DEF_DIV6P1("sd3",  R8A7745_CLK_SD3,	CLK_PLL1_DIV2,	0x26c),
+	DEF_DIV6P1("mmc0", R8A7745_CLK_MMC0,	CLK_PLL1_DIV2,	0x240),
+};
+
+static const struct mssr_mod_clk r8a7745_mod_clks[] __initconst = {
+	DEF_MOD("msiof0",		   0,	R8A7745_CLK_MP),
+	DEF_MOD("vcp0",			 101,	R8A7745_CLK_ZS),
+	DEF_MOD("vpc0",			 103,	R8A7745_CLK_ZS),
+	DEF_MOD("tmu1",			 111,	R8A7745_CLK_P),
+	DEF_MOD("3dg",			 112,	R8A7745_CLK_ZG),
+	DEF_MOD("2d-dmac",		 115,	R8A7745_CLK_ZS),
+	DEF_MOD("fdp1-0",		 119,	R8A7745_CLK_ZS),
+	DEF_MOD("tmu3",			 121,	R8A7745_CLK_P),
+	DEF_MOD("tmu2",			 122,	R8A7745_CLK_P),
+	DEF_MOD("cmt0",			 124,	R8A7745_CLK_R),
+	DEF_MOD("tmu0",			 125,	R8A7745_CLK_CP),
+	DEF_MOD("vsp1du0",		 128,	R8A7745_CLK_ZS),
+	DEF_MOD("vsp1-sy",		 131,	R8A7745_CLK_ZS),
+	DEF_MOD("scifa2",		 202,	R8A7745_CLK_MP),
+	DEF_MOD("scifa1",		 203,	R8A7745_CLK_MP),
+	DEF_MOD("scifa0",		 204,	R8A7745_CLK_MP),
+	DEF_MOD("msiof2",		 205,	R8A7745_CLK_MP),
+	DEF_MOD("scifb0",		 206,	R8A7745_CLK_MP),
+	DEF_MOD("scifb1",		 207,	R8A7745_CLK_MP),
+	DEF_MOD("msiof1",		 208,	R8A7745_CLK_MP),
+	DEF_MOD("scifb2",		 216,	R8A7745_CLK_MP),
+	DEF_MOD("sys-dmac1",		 218,	R8A7745_CLK_ZS),
+	DEF_MOD("sys-dmac0",		 219,	R8A7745_CLK_ZS),
+	DEF_MOD("tpu0",			 304,	R8A7745_CLK_CP),
+	DEF_MOD("sdhi3",		 311,	R8A7745_CLK_SD3),
+	DEF_MOD("sdhi2",		 312,	R8A7745_CLK_SD2),
+	DEF_MOD("sdhi0",		 314,	R8A7745_CLK_SD0),
+	DEF_MOD("mmcif0",		 315,	R8A7745_CLK_MMC0),
+	DEF_MOD("iic0",			 318,	R8A7745_CLK_HP),
+	DEF_MOD("iic1",			 323,	R8A7745_CLK_HP),
+	DEF_MOD("cmt1",			 329,	R8A7745_CLK_R),
+	DEF_MOD("usbhs-dmac0",		 330,	R8A7745_CLK_HP),
+	DEF_MOD("usbhs-dmac1",		 331,	R8A7745_CLK_HP),
+	DEF_MOD("irqc",			 407,	R8A7745_CLK_CP),
+	DEF_MOD("intc-sys",		 408,	R8A7745_CLK_ZS),
+	DEF_MOD("audio-dmac0",		 502,	R8A7745_CLK_HP),
+	DEF_MOD("pwm",			 523,	R8A7745_CLK_P),
+	DEF_MOD("usb-ehci",		 703,	R8A7745_CLK_MP),
+	DEF_MOD("usbhs",		 704,	R8A7745_CLK_HP),
+	DEF_MOD("hscif2",		 713,	R8A7745_CLK_ZS),
+	DEF_MOD("scif5",		 714,	R8A7745_CLK_P),
+	DEF_MOD("scif4",		 715,	R8A7745_CLK_P),
+	DEF_MOD("hscif1",		 716,	R8A7745_CLK_ZS),
+	DEF_MOD("hscif0",		 717,	R8A7745_CLK_ZS),
+	DEF_MOD("scif3",		 718,	R8A7745_CLK_P),
+	DEF_MOD("scif2",		 719,	R8A7745_CLK_P),
+	DEF_MOD("scif1",		 720,	R8A7745_CLK_P),
+	DEF_MOD("scif0",		 721,	R8A7745_CLK_P),
+	DEF_MOD("du0",			 724,	R8A7745_CLK_ZX),
+	DEF_MOD("ipmmu-sgx",		 800,	R8A7745_CLK_ZX),
+	DEF_MOD("vin1",			 810,	R8A7745_CLK_ZG),
+	DEF_MOD("vin0",			 811,	R8A7745_CLK_ZG),
+	DEF_MOD("etheravb",		 812,	R8A7745_CLK_HP),
+	DEF_MOD("ether",		 813,	R8A7745_CLK_P),
+	DEF_MOD("gpio6",		 905,	R8A7745_CLK_CP),
+	DEF_MOD("gpio5",		 907,	R8A7745_CLK_CP),
+	DEF_MOD("gpio4",		 908,	R8A7745_CLK_CP),
+	DEF_MOD("gpio3",		 909,	R8A7745_CLK_CP),
+	DEF_MOD("gpio2",		 910,	R8A7745_CLK_CP),
+	DEF_MOD("gpio1",		 911,	R8A7745_CLK_CP),
+	DEF_MOD("gpio0",		 912,	R8A7745_CLK_CP),
+	DEF_MOD("can1",			 915,	R8A7745_CLK_P),
+	DEF_MOD("can0",			 916,	R8A7745_CLK_P),
+	DEF_MOD("qspi_mod",		 917,	R8A7745_CLK_QSPI),
+	DEF_MOD("i2c5",			 925,	R8A7745_CLK_HP),
+	DEF_MOD("i2c4",			 927,	R8A7745_CLK_HP),
+	DEF_MOD("i2c3",			 928,	R8A7745_CLK_HP),
+	DEF_MOD("i2c2",			 929,	R8A7745_CLK_HP),
+	DEF_MOD("i2c1",			 930,	R8A7745_CLK_HP),
+	DEF_MOD("i2c0",			 931,	R8A7745_CLK_HP),
+	DEF_MOD("ssi-all",		1005,	R8A7745_CLK_P),
+	DEF_MOD("ssi9",			1006,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi8",			1007,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi7",			1008,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi6",			1009,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi5",			1010,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi4",			1011,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi3",			1012,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi2",			1013,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi1",			1014,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi0",			1015,	MOD_CLK_ID(1005)),
+	DEF_MOD("scu-all",		1017,	R8A7745_CLK_P),
+	DEF_MOD("scu-dvc1",		1018,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-dvc0",		1019,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu1-mix1",	1020,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu0-mix0",	1021,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src9",		1022,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src8",		1023,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src7",		1024,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src6",		1025,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src5",		1026,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src4",		1027,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src3",		1028,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src2",		1029,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src1",		1030,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src0",		1031,	MOD_CLK_ID(1017)),
+	DEF_MOD("scifa3",		1106,	R8A7745_CLK_MP),
+	DEF_MOD("scifa4",		1107,	R8A7745_CLK_MP),
+	DEF_MOD("scifa5",		1108,	R8A7745_CLK_MP),
+};
+
+static const unsigned int r8a7745_crit_mod_clks[] __initconst = {
+	MOD_CLK_ID(408),	/* INTC-SYS (GIC) */
+};
+
+/*
+ * CPG Clock Data
+ */
+
+/*
+ *    MD	EXTAL		PLL0	PLL1	PLL3
+ * 14 13 19	(MHz)		*1	*2
+ *---------------------------------------------------
+ * 0  0  0	15		x200/3	x208/2	x106
+ * 0  0  1	15		x200/3	x208/2	x88
+ * 0  1  0	20		x150/3	x156/2	x80
+ * 0  1  1	20		x150/3	x156/2	x66
+ * 1  0  0	26 / 2		x230/3	x240/2	x122
+ * 1  0  1	26 / 2		x230/3	x240/2	x102
+ * 1  1  0	30 / 2		x200/3	x208/2	x106
+ * 1  1  1	30 / 2		x200/3	x208/2	x88
+ *
+ * *1 :	Table 7.5b indicates VCO output (PLL0 = VCO/3)
+ * *2 :	Table 7.5b indicates VCO output (PLL1 = VCO/2)
+ */
+#define CPG_PLL_CONFIG_INDEX(md)	((((md) & BIT(14)) >> 12) | \
+					 (((md) & BIT(13)) >> 12) | \
+					 (((md) & BIT(19)) >> 19))
+
+static const struct rcar_gen2_cpg_pll_config cpg_pll_configs[8] __initconst = {
+	/* EXTAL div	PLL1 mult	PLL3 mult	PLL0 mult */
+	{ 1,		208,		106,		200	},
+	{ 1,		208,		88,		200	},
+	{ 1,		156,		80,		150	},
+	{ 1,		156,		66,		150	},
+	{ 2,		240,		122,		230	},
+	{ 2,		240,		102,		230	},
+	{ 2,		208,		106,		200	},
+	{ 2,		208,		88,		200	},
+};
+
+static int __init r8a7745_cpg_mssr_init(struct device *dev)
+{
+	const struct rcar_gen2_cpg_pll_config *cpg_pll_config;
+	u32 cpg_mode;
+	int error;
+
+	error = rcar_rst_read_mode_pins(&cpg_mode);
+	if (error)
+		return error;
+
+	cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)];
+
+	return rcar_gen2_cpg_init(cpg_pll_config, 3, cpg_mode);
+}
+
+const struct cpg_mssr_info r8a7745_cpg_mssr_info __initconst = {
+	/* Core Clocks */
+	.core_clks = r8a7745_core_clks,
+	.num_core_clks = ARRAY_SIZE(r8a7745_core_clks),
+	.last_dt_core_clk = LAST_DT_CORE_CLK,
+	.num_total_core_clks = MOD_CLK_BASE,
+
+	/* Module Clocks */
+	.mod_clks = r8a7745_mod_clks,
+	.num_mod_clks = ARRAY_SIZE(r8a7745_mod_clks),
+	.num_hw_mod_clks = 12 * 32,
+
+	/* Critical Module Clocks */
+	.crit_mod_clks = r8a7745_crit_mod_clks,
+	.num_crit_mod_clks = ARRAY_SIZE(r8a7745_crit_mod_clks),
+
+	/* Callbacks */
+	.init = r8a7745_cpg_mssr_init,
+	.cpg_clk_register = rcar_gen2_cpg_clk_register,
+};
diff --git a/drivers/clk/renesas/r8a7795-cpg-mssr.c b/drivers/clk/renesas/r8a7795-cpg-mssr.c
index f255e451e8ca..50698a7d9074 100644
--- a/drivers/clk/renesas/r8a7795-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7795-cpg-mssr.c
@@ -15,6 +15,7 @@
 #include <linux/device.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
+#include <linux/soc/renesas/rcar-rst.h>
 
 #include <dt-bindings/clock/r8a7795-cpg-mssr.h>
 
@@ -97,7 +98,7 @@ static const struct cpg_core_clk r8a7795_core_clks[] __initconst = {
 	DEF_FIXED("cp",         R8A7795_CLK_CP,    CLK_EXTAL,      2, 1),
 
 	DEF_DIV6P1("mso",       R8A7795_CLK_MSO,   CLK_PLL1_DIV4, 0x014),
-	DEF_DIV6P1("hdmi",      R8A7795_CLK_HDMI,  CLK_PLL1_DIV2, 0x250),
+	DEF_DIV6P1("hdmi",      R8A7795_CLK_HDMI,  CLK_PLL1_DIV4, 0x250),
 	DEF_DIV6P1("canfd",     R8A7795_CLK_CANFD, CLK_PLL1_DIV4, 0x244),
 	DEF_DIV6P1("csi0",      R8A7795_CLK_CSI0,  CLK_PLL1_DIV4, 0x00c),
 
@@ -311,7 +312,12 @@ static const struct rcar_gen3_cpg_pll_config cpg_pll_configs[16] __initconst = {
 static int __init r8a7795_cpg_mssr_init(struct device *dev)
 {
 	const struct rcar_gen3_cpg_pll_config *cpg_pll_config;
-	u32 cpg_mode = rcar_gen3_read_mode_pins();
+	u32 cpg_mode;
+	int error;
+
+	error = rcar_rst_read_mode_pins(&cpg_mode);
+	if (error)
+		return error;
 
 	cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)];
 	if (!cpg_pll_config->extal_div) {
diff --git a/drivers/clk/renesas/r8a7796-cpg-mssr.c b/drivers/clk/renesas/r8a7796-cpg-mssr.c
index eb347ed265f2..7d298c57a3e0 100644
--- a/drivers/clk/renesas/r8a7796-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7796-cpg-mssr.c
@@ -16,6 +16,7 @@
 #include <linux/device.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
+#include <linux/soc/renesas/rcar-rst.h>
 
 #include <dt-bindings/clock/r8a7796-cpg-mssr.h>
 
@@ -102,6 +103,8 @@ static const struct cpg_core_clk r8a7796_core_clks[] __initconst = {
 	DEF_FIXED("cl",         R8A7796_CLK_CL,    CLK_PLL1_DIV2, 48, 1),
 	DEF_FIXED("cp",         R8A7796_CLK_CP,    CLK_EXTAL,      2, 1),
 
+	DEF_DIV6P1("csi0",      R8A7796_CLK_CSI0,  CLK_PLL1_DIV4, 0x00c),
+
 	DEF_DIV6_RO("osc",      R8A7796_CLK_OSC,   CLK_EXTAL, CPG_RCKCR,  8),
 	DEF_DIV6_RO("r_int",    CLK_RINT,          CLK_EXTAL, CPG_RCKCR, 32),
 
@@ -109,6 +112,14 @@ static const struct cpg_core_clk r8a7796_core_clks[] __initconst = {
 };
 
 static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = {
+	DEF_MOD("scif5",		 202,	R8A7796_CLK_S3D4),
+	DEF_MOD("scif4",		 203,	R8A7796_CLK_S3D4),
+	DEF_MOD("scif3",		 204,	R8A7796_CLK_S3D4),
+	DEF_MOD("scif1",		 206,	R8A7796_CLK_S3D4),
+	DEF_MOD("scif0",		 207,	R8A7796_CLK_S3D4),
+	DEF_MOD("sys-dmac2",		 217,	R8A7796_CLK_S0D3),
+	DEF_MOD("sys-dmac1",		 218,	R8A7796_CLK_S0D3),
+	DEF_MOD("sys-dmac0",		 219,	R8A7796_CLK_S0D3),
 	DEF_MOD("cmt3",			 300,	R8A7796_CLK_R),
 	DEF_MOD("cmt2",			 301,	R8A7796_CLK_R),
 	DEF_MOD("cmt1",			 302,	R8A7796_CLK_R),
@@ -120,7 +131,47 @@ static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = {
 	DEF_MOD("sdif0",		 314,	R8A7796_CLK_SD0),
 	DEF_MOD("rwdt0",		 402,	R8A7796_CLK_R),
 	DEF_MOD("intc-ap",		 408,	R8A7796_CLK_S3D1),
+	DEF_MOD("drif7",		 508,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif6",		 509,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif5",		 510,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif4",		 511,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif3",		 512,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif2",		 513,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif1",		 514,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif0",		 515,	R8A7796_CLK_S3D2),
+	DEF_MOD("hscif4",		 516,	R8A7796_CLK_S3D1),
+	DEF_MOD("hscif3",		 517,	R8A7796_CLK_S3D1),
+	DEF_MOD("hscif2",		 518,	R8A7796_CLK_S3D1),
+	DEF_MOD("hscif1",		 519,	R8A7796_CLK_S3D1),
+	DEF_MOD("hscif0",		 520,	R8A7796_CLK_S3D1),
 	DEF_MOD("thermal",		 522,	R8A7796_CLK_CP),
+	DEF_MOD("fcpvd2",		 601,	R8A7796_CLK_S0D2),
+	DEF_MOD("fcpvd1",		 602,	R8A7796_CLK_S0D2),
+	DEF_MOD("fcpvd0",		 603,	R8A7796_CLK_S0D2),
+	DEF_MOD("fcpvb0",		 607,	R8A7796_CLK_S0D1),
+	DEF_MOD("fcpvi0",		 611,	R8A7796_CLK_S0D1),
+	DEF_MOD("fcpf0",		 615,	R8A7796_CLK_S0D1),
+	DEF_MOD("fcpci0",		 617,	R8A7796_CLK_S0D2),
+	DEF_MOD("fcpcs",		 619,	R8A7796_CLK_S0D2),
+	DEF_MOD("vspd2",		 621,	R8A7796_CLK_S0D2),
+	DEF_MOD("vspd1",		 622,	R8A7796_CLK_S0D2),
+	DEF_MOD("vspd0",		 623,	R8A7796_CLK_S0D2),
+	DEF_MOD("vspb",			 626,	R8A7796_CLK_S0D1),
+	DEF_MOD("vspi0",		 631,	R8A7796_CLK_S0D1),
+	DEF_MOD("csi20",		 714,	R8A7796_CLK_CSI0),
+	DEF_MOD("csi40",		 716,	R8A7796_CLK_CSI0),
+	DEF_MOD("du2",			 722,	R8A7796_CLK_S2D1),
+	DEF_MOD("du1",			 723,	R8A7796_CLK_S2D1),
+	DEF_MOD("du0",			 724,	R8A7796_CLK_S2D1),
+	DEF_MOD("lvds",			 727,	R8A7796_CLK_S2D1),
+	DEF_MOD("vin7",			 804,	R8A7796_CLK_S0D2),
+	DEF_MOD("vin6",			 805,	R8A7796_CLK_S0D2),
+	DEF_MOD("vin5",			 806,	R8A7796_CLK_S0D2),
+	DEF_MOD("vin4",			 807,	R8A7796_CLK_S0D2),
+	DEF_MOD("vin3",			 808,	R8A7796_CLK_S0D2),
+	DEF_MOD("vin2",			 809,	R8A7796_CLK_S0D2),
+	DEF_MOD("vin1",			 810,	R8A7796_CLK_S0D2),
+	DEF_MOD("vin0",			 811,	R8A7796_CLK_S0D2),
 	DEF_MOD("etheravb",		 812,	R8A7796_CLK_S0D6),
 	DEF_MOD("gpio7",		 905,	R8A7796_CLK_S3D4),
 	DEF_MOD("gpio6",		 906,	R8A7796_CLK_S3D4),
@@ -130,6 +181,13 @@ static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = {
 	DEF_MOD("gpio2",		 910,	R8A7796_CLK_S3D4),
 	DEF_MOD("gpio1",		 911,	R8A7796_CLK_S3D4),
 	DEF_MOD("gpio0",		 912,	R8A7796_CLK_S3D4),
+	DEF_MOD("i2c6",			 918,	R8A7796_CLK_S0D6),
+	DEF_MOD("i2c5",			 919,	R8A7796_CLK_S0D6),
+	DEF_MOD("i2c4",			 927,	R8A7796_CLK_S0D6),
+	DEF_MOD("i2c3",			 928,	R8A7796_CLK_S0D6),
+	DEF_MOD("i2c2",			 929,	R8A7796_CLK_S3D2),
+	DEF_MOD("i2c1",			 930,	R8A7796_CLK_S3D2),
+	DEF_MOD("i2c0",			 931,	R8A7796_CLK_S3D2),
 };
 
 static const unsigned int r8a7796_crit_mod_clks[] __initconst = {
@@ -190,7 +248,12 @@ static const struct rcar_gen3_cpg_pll_config cpg_pll_configs[16] __initconst = {
 static int __init r8a7796_cpg_mssr_init(struct device *dev)
 {
 	const struct rcar_gen3_cpg_pll_config *cpg_pll_config;
-	u32 cpg_mode = rcar_gen3_read_mode_pins();
+	u32 cpg_mode;
+	int error;
+
+	error = rcar_rst_read_mode_pins(&cpg_mode);
+	if (error)
+		return error;
 
 	cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)];
 	if (!cpg_pll_config->extal_div) {
diff --git a/drivers/clk/renesas/rcar-gen2-cpg.c b/drivers/clk/renesas/rcar-gen2-cpg.c
new file mode 100644
index 000000000000..123b1e622179
--- /dev/null
+++ b/drivers/clk/renesas/rcar-gen2-cpg.c
@@ -0,0 +1,371 @@
+/*
+ * R-Car Gen2 Clock Pulse Generator
+ *
+ * Copyright (C) 2016 Cogent Embedded Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+
+#include <linux/bug.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+
+#include "renesas-cpg-mssr.h"
+#include "rcar-gen2-cpg.h"
+
+#define CPG_FRQCRB		0x0004
+#define CPG_FRQCRB_KICK		BIT(31)
+#define CPG_SDCKCR		0x0074
+#define CPG_PLL0CR		0x00d8
+#define CPG_PLL0CR_STC_SHIFT	24
+#define CPG_PLL0CR_STC_MASK	(0x7f << CPG_PLL0CR_STC_SHIFT)
+#define CPG_FRQCRC		0x00e0
+#define CPG_FRQCRC_ZFC_SHIFT	8
+#define CPG_FRQCRC_ZFC_MASK	(0x1f << CPG_FRQCRC_ZFC_SHIFT)
+#define CPG_ADSPCKCR		0x025c
+#define CPG_RCANCKCR		0x0270
+
+static spinlock_t cpg_lock;
+
+/*
+ * Z Clock
+ *
+ * Traits of this clock:
+ * prepare - clk_prepare only ensures that parents are prepared
+ * enable - clk_enable only ensures that parents are enabled
+ * rate - rate is adjustable.  clk->rate = parent->rate * mult / 32
+ * parent - fixed parent.  No clk_set_parent support
+ */
+
+struct cpg_z_clk {
+	struct clk_hw hw;
+	void __iomem *reg;
+	void __iomem *kick_reg;
+};
+
+#define to_z_clk(_hw)	container_of(_hw, struct cpg_z_clk, hw)
+
+static unsigned long cpg_z_clk_recalc_rate(struct clk_hw *hw,
+					   unsigned long parent_rate)
+{
+	struct cpg_z_clk *zclk = to_z_clk(hw);
+	unsigned int mult;
+	unsigned int val;
+
+	val = (readl(zclk->reg) & CPG_FRQCRC_ZFC_MASK) >> CPG_FRQCRC_ZFC_SHIFT;
+	mult = 32 - val;
+
+	return div_u64((u64)parent_rate * mult, 32);
+}
+
+static long cpg_z_clk_round_rate(struct clk_hw *hw, unsigned long rate,
+				 unsigned long *parent_rate)
+{
+	unsigned long prate  = *parent_rate;
+	unsigned int mult;
+
+	if (!prate)
+		prate = 1;
+
+	mult = div_u64((u64)rate * 32, prate);
+	mult = clamp(mult, 1U, 32U);
+
+	return *parent_rate / 32 * mult;
+}
+
+static int cpg_z_clk_set_rate(struct clk_hw *hw, unsigned long rate,
+			      unsigned long parent_rate)
+{
+	struct cpg_z_clk *zclk = to_z_clk(hw);
+	unsigned int mult;
+	u32 val, kick;
+	unsigned int i;
+
+	mult = div_u64((u64)rate * 32, parent_rate);
+	mult = clamp(mult, 1U, 32U);
+
+	if (readl(zclk->kick_reg) & CPG_FRQCRB_KICK)
+		return -EBUSY;
+
+	val = readl(zclk->reg);
+	val &= ~CPG_FRQCRC_ZFC_MASK;
+	val |= (32 - mult) << CPG_FRQCRC_ZFC_SHIFT;
+	writel(val, zclk->reg);
+
+	/*
+	 * Set KICK bit in FRQCRB to update hardware setting and wait for
+	 * clock change completion.
+	 */
+	kick = readl(zclk->kick_reg);
+	kick |= CPG_FRQCRB_KICK;
+	writel(kick, zclk->kick_reg);
+
+	/*
+	 * Note: There is no HW information about the worst case latency.
+	 *
+	 * Using experimental measurements, it seems that no more than
+	 * ~10 iterations are needed, independently of the CPU rate.
+	 * Since this value might be dependent on external xtal rate, pll1
+	 * rate or even the other emulation clocks rate, use 1000 as a
+	 * "super" safe value.
+	 */
+	for (i = 1000; i; i--) {
+		if (!(readl(zclk->kick_reg) & CPG_FRQCRB_KICK))
+			return 0;
+
+		cpu_relax();
+	}
+
+	return -ETIMEDOUT;
+}
+
+static const struct clk_ops cpg_z_clk_ops = {
+	.recalc_rate = cpg_z_clk_recalc_rate,
+	.round_rate = cpg_z_clk_round_rate,
+	.set_rate = cpg_z_clk_set_rate,
+};
+
+static struct clk * __init cpg_z_clk_register(const char *name,
+					      const char *parent_name,
+					      void __iomem *base)
+{
+	struct clk_init_data init;
+	struct cpg_z_clk *zclk;
+	struct clk *clk;
+
+	zclk = kzalloc(sizeof(*zclk), GFP_KERNEL);
+	if (!zclk)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &cpg_z_clk_ops;
+	init.flags = 0;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+
+	zclk->reg = base + CPG_FRQCRC;
+	zclk->kick_reg = base + CPG_FRQCRB;
+	zclk->hw.init = &init;
+
+	clk = clk_register(NULL, &zclk->hw);
+	if (IS_ERR(clk))
+		kfree(zclk);
+
+	return clk;
+}
+
+static struct clk * __init cpg_rcan_clk_register(const char *name,
+						 const char *parent_name,
+						 void __iomem *base)
+{
+	struct clk_fixed_factor *fixed;
+	struct clk_gate *gate;
+	struct clk *clk;
+
+	fixed = kzalloc(sizeof(*fixed), GFP_KERNEL);
+	if (!fixed)
+		return ERR_PTR(-ENOMEM);
+
+	fixed->mult = 1;
+	fixed->div = 6;
+
+	gate = kzalloc(sizeof(*gate), GFP_KERNEL);
+	if (!gate) {
+		kfree(fixed);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	gate->reg = base + CPG_RCANCKCR;
+	gate->bit_idx = 8;
+	gate->flags = CLK_GATE_SET_TO_DISABLE;
+	gate->lock = &cpg_lock;
+
+	clk = clk_register_composite(NULL, name, &parent_name, 1, NULL, NULL,
+				     &fixed->hw, &clk_fixed_factor_ops,
+				     &gate->hw, &clk_gate_ops, 0);
+	if (IS_ERR(clk)) {
+		kfree(gate);
+		kfree(fixed);
+	}
+
+	return clk;
+}
+
+/* ADSP divisors */
+static const struct clk_div_table cpg_adsp_div_table[] = {
+	{  1,  3 }, {  2,  4 }, {  3,  6 }, {  4,  8 },
+	{  5, 12 }, {  6, 16 }, {  7, 18 }, {  8, 24 },
+	{ 10, 36 }, { 11, 48 }, {  0,  0 },
+};
+
+static struct clk * __init cpg_adsp_clk_register(const char *name,
+						 const char *parent_name,
+						 void __iomem *base)
+{
+	struct clk_divider *div;
+	struct clk_gate *gate;
+	struct clk *clk;
+
+	div = kzalloc(sizeof(*div), GFP_KERNEL);
+	if (!div)
+		return ERR_PTR(-ENOMEM);
+
+	div->reg = base + CPG_ADSPCKCR;
+	div->width = 4;
+	div->table = cpg_adsp_div_table;
+	div->lock = &cpg_lock;
+
+	gate = kzalloc(sizeof(*gate), GFP_KERNEL);
+	if (!gate) {
+		kfree(div);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	gate->reg = base + CPG_ADSPCKCR;
+	gate->bit_idx = 8;
+	gate->flags = CLK_GATE_SET_TO_DISABLE;
+	gate->lock = &cpg_lock;
+
+	clk = clk_register_composite(NULL, name, &parent_name, 1, NULL, NULL,
+				     &div->hw, &clk_divider_ops,
+				     &gate->hw, &clk_gate_ops, 0);
+	if (IS_ERR(clk)) {
+		kfree(gate);
+		kfree(div);
+	}
+
+	return clk;
+}
+
+/* SDHI divisors */
+static const struct clk_div_table cpg_sdh_div_table[] = {
+	{  0,  2 }, {  1,  3 }, {  2,  4 }, {  3,  6 },
+	{  4,  8 }, {  5, 12 }, {  6, 16 }, {  7, 18 },
+	{  8, 24 }, { 10, 36 }, { 11, 48 }, {  0,  0 },
+};
+
+static const struct clk_div_table cpg_sd01_div_table[] = {
+	{  4,  8 }, {  5, 12 }, {  6, 16 }, {  7, 18 },
+	{  8, 24 }, { 10, 36 }, { 11, 48 }, { 12, 10 },
+	{  0,  0 },
+};
+
+static const struct rcar_gen2_cpg_pll_config *cpg_pll_config __initdata;
+static unsigned int cpg_pll0_div __initdata;
+static u32 cpg_mode __initdata;
+
+struct clk * __init rcar_gen2_cpg_clk_register(struct device *dev,
+					       const struct cpg_core_clk *core,
+					       const struct cpg_mssr_info *info,
+					       struct clk **clks,
+					       void __iomem *base)
+{
+	const struct clk_div_table *table = NULL;
+	const struct clk *parent;
+	const char *parent_name;
+	unsigned int mult = 1;
+	unsigned int div = 1;
+	unsigned int shift;
+
+	parent = clks[core->parent];
+	if (IS_ERR(parent))
+		return ERR_CAST(parent);
+
+	parent_name = __clk_get_name(parent);
+
+	switch (core->type) {
+	/* R-Car Gen2 */
+	case CLK_TYPE_GEN2_MAIN:
+		div = cpg_pll_config->extal_div;
+		break;
+
+	case CLK_TYPE_GEN2_PLL0:
+		/*
+		 * PLL0 is a  configurable multiplier clock except on R-Car
+		 * V2H/E2. Register the PLL0 clock as a fixed factor clock for
+		 * now as there's no generic multiplier clock implementation and
+		 * we  currently  have no need to change  the multiplier value.
+		 */
+		mult = cpg_pll_config->pll0_mult;
+		div  = cpg_pll0_div;
+		if (!mult) {
+			u32 pll0cr = readl(base + CPG_PLL0CR);
+
+			mult = (((pll0cr & CPG_PLL0CR_STC_MASK) >>
+				 CPG_PLL0CR_STC_SHIFT) + 1) * 2;
+		}
+		break;
+
+	case CLK_TYPE_GEN2_PLL1:
+		mult = cpg_pll_config->pll1_mult / 2;
+		break;
+
+	case CLK_TYPE_GEN2_PLL3:
+		mult = cpg_pll_config->pll3_mult;
+		break;
+
+	case CLK_TYPE_GEN2_Z:
+		return cpg_z_clk_register(core->name, parent_name, base);
+
+	case CLK_TYPE_GEN2_LB:
+		div = cpg_mode & BIT(18) ? 36 : 24;
+		break;
+
+	case CLK_TYPE_GEN2_ADSP:
+		return cpg_adsp_clk_register(core->name, parent_name, base);
+
+	case CLK_TYPE_GEN2_SDH:
+		table = cpg_sdh_div_table;
+		shift = 8;
+		break;
+
+	case CLK_TYPE_GEN2_SD0:
+		table = cpg_sd01_div_table;
+		shift = 4;
+		break;
+
+	case CLK_TYPE_GEN2_SD1:
+		table = cpg_sd01_div_table;
+		shift = 0;
+		break;
+
+	case CLK_TYPE_GEN2_QSPI:
+		div = (cpg_mode & (BIT(3) | BIT(2) | BIT(1))) == BIT(2) ?
+		      8 : 10;
+		break;
+
+	case CLK_TYPE_GEN2_RCAN:
+		return cpg_rcan_clk_register(core->name, parent_name, base);
+
+	default:
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (!table)
+		return clk_register_fixed_factor(NULL, core->name, parent_name,
+						 0, mult, div);
+	else
+		return clk_register_divider_table(NULL, core->name,
+						  parent_name, 0,
+						  base + CPG_SDCKCR, shift, 4,
+						  0, table, &cpg_lock);
+}
+
+int __init rcar_gen2_cpg_init(const struct rcar_gen2_cpg_pll_config *config,
+			      unsigned int pll0_div, u32 mode)
+{
+	cpg_pll_config = config;
+	cpg_pll0_div = pll0_div;
+	cpg_mode = mode;
+
+	spin_lock_init(&cpg_lock);
+
+	return 0;
+}
diff --git a/drivers/clk/renesas/rcar-gen2-cpg.h b/drivers/clk/renesas/rcar-gen2-cpg.h
new file mode 100644
index 000000000000..9eba07ff8b11
--- /dev/null
+++ b/drivers/clk/renesas/rcar-gen2-cpg.h
@@ -0,0 +1,43 @@
+/*
+ * R-Car Gen2 Clock Pulse Generator
+ *
+ * Copyright (C) 2016 Cogent Embedded Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation; version 2 of the License.
+ */
+
+#ifndef __CLK_RENESAS_RCAR_GEN2_CPG_H__
+#define __CLK_RENESAS_RCAR_GEN2_CPG_H__
+
+enum rcar_gen2_clk_types {
+	CLK_TYPE_GEN2_MAIN = CLK_TYPE_CUSTOM,
+	CLK_TYPE_GEN2_PLL0,
+	CLK_TYPE_GEN2_PLL1,
+	CLK_TYPE_GEN2_PLL3,
+	CLK_TYPE_GEN2_Z,
+	CLK_TYPE_GEN2_LB,
+	CLK_TYPE_GEN2_ADSP,
+	CLK_TYPE_GEN2_SDH,
+	CLK_TYPE_GEN2_SD0,
+	CLK_TYPE_GEN2_SD1,
+	CLK_TYPE_GEN2_QSPI,
+	CLK_TYPE_GEN2_RCAN,
+};
+
+struct rcar_gen2_cpg_pll_config {
+	unsigned int extal_div;
+	unsigned int pll1_mult;
+	unsigned int pll3_mult;
+	unsigned int pll0_mult;		/* leave as zero if PLL0CR exists */
+};
+
+struct clk *rcar_gen2_cpg_clk_register(struct device *dev,
+				       const struct cpg_core_clk *core,
+				       const struct cpg_mssr_info *info,
+				       struct clk **clks, void __iomem *base);
+int rcar_gen2_cpg_init(const struct rcar_gen2_cpg_pll_config *config,
+		       unsigned int pll0_div, u32 mode);
+
+#endif
diff --git a/drivers/clk/renesas/rcar-gen3-cpg.c b/drivers/clk/renesas/rcar-gen3-cpg.c
index bb4f2f9a8c2f..742f6dc7c156 100644
--- a/drivers/clk/renesas/rcar-gen3-cpg.c
+++ b/drivers/clk/renesas/rcar-gen3-cpg.c
@@ -98,7 +98,7 @@ static int cpg_sd_clock_enable(struct clk_hw *hw)
 	u32 val, sd_fc;
 	unsigned int i;
 
-	val = clk_readl(clock->reg);
+	val = readl(clock->reg);
 
 	sd_fc = val & CPG_SD_FC_MASK;
 	for (i = 0; i < clock->div_num; i++)
@@ -111,7 +111,7 @@ static int cpg_sd_clock_enable(struct clk_hw *hw)
 	val &= ~(CPG_SD_STP_MASK);
 	val |= clock->div_table[i].val & CPG_SD_STP_MASK;
 
-	clk_writel(val, clock->reg);
+	writel(val, clock->reg);
 
 	return 0;
 }
@@ -120,14 +120,14 @@ static void cpg_sd_clock_disable(struct clk_hw *hw)
 {
 	struct sd_clock *clock = to_sd_clock(hw);
 
-	clk_writel(clk_readl(clock->reg) | CPG_SD_STP_MASK, clock->reg);
+	writel(readl(clock->reg) | CPG_SD_STP_MASK, clock->reg);
 }
 
 static int cpg_sd_clock_is_enabled(struct clk_hw *hw)
 {
 	struct sd_clock *clock = to_sd_clock(hw);
 
-	return !(clk_readl(clock->reg) & CPG_SD_STP_MASK);
+	return !(readl(clock->reg) & CPG_SD_STP_MASK);
 }
 
 static unsigned long cpg_sd_clock_recalc_rate(struct clk_hw *hw,
@@ -138,7 +138,7 @@ static unsigned long cpg_sd_clock_recalc_rate(struct clk_hw *hw,
 	u32 val, sd_fc;
 	unsigned int i;
 
-	val = clk_readl(clock->reg);
+	val = readl(clock->reg);
 
 	sd_fc = val & CPG_SD_FC_MASK;
 	for (i = 0; i < clock->div_num; i++)
@@ -189,10 +189,10 @@ static int cpg_sd_clock_set_rate(struct clk_hw *hw, unsigned long rate,
 	if (i >= clock->div_num)
 		return -EINVAL;
 
-	val = clk_readl(clock->reg);
+	val = readl(clock->reg);
 	val &= ~(CPG_SD_STP_MASK | CPG_SD_FC_MASK);
 	val |= clock->div_table[i].val & (CPG_SD_STP_MASK | CPG_SD_FC_MASK);
-	clk_writel(val, clock->reg);
+	writel(val, clock->reg);
 
 	return 0;
 }
@@ -333,23 +333,6 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev,
 					 __clk_get_name(parent), 0, mult, div);
 }
 
-/*
- * Reset register definitions.
- */
-#define MODEMR	0xe6160060
-
-u32 __init rcar_gen3_read_mode_pins(void)
-{
-	void __iomem *modemr = ioremap_nocache(MODEMR, 4);
-	u32 mode;
-
-	BUG_ON(!modemr);
-	mode = ioread32(modemr);
-	iounmap(modemr);
-
-	return mode;
-}
-
 int __init rcar_gen3_cpg_init(const struct rcar_gen3_cpg_pll_config *config,
 			      unsigned int clk_extalr)
 {
diff --git a/drivers/clk/renesas/rcar-gen3-cpg.h b/drivers/clk/renesas/rcar-gen3-cpg.h
index f699085147d1..f788f481dd42 100644
--- a/drivers/clk/renesas/rcar-gen3-cpg.h
+++ b/drivers/clk/renesas/rcar-gen3-cpg.h
@@ -33,7 +33,6 @@ struct rcar_gen3_cpg_pll_config {
 
 #define CPG_RCKCR	0x240
 
-u32 rcar_gen3_read_mode_pins(void);
 struct clk *rcar_gen3_cpg_clk_register(struct device *dev,
 	const struct cpg_core_clk *core, const struct cpg_mssr_info *info,
 	struct clk **clks, void __iomem *base);
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c
index e1365e7491ae..8359ce75db7a 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.c
+++ b/drivers/clk/renesas/renesas-cpg-mssr.c
@@ -33,9 +33,9 @@
 #include "clk-div6.h"
 
 #ifdef DEBUG
-#define WARN_DEBUG(x)	do { } while (0)
-#else
 #define WARN_DEBUG(x)	WARN_ON(x)
+#else
+#define WARN_DEBUG(x)	do { } while (0)
 #endif
 
 
@@ -146,12 +146,12 @@ static int cpg_mstp_clock_endisable(struct clk_hw *hw, bool enable)
 		enable ? "ON" : "OFF");
 	spin_lock_irqsave(&priv->mstp_lock, flags);
 
-	value = clk_readl(priv->base + SMSTPCR(reg));
+	value = readl(priv->base + SMSTPCR(reg));
 	if (enable)
 		value &= ~bitmask;
 	else
 		value |= bitmask;
-	clk_writel(value, priv->base + SMSTPCR(reg));
+	writel(value, priv->base + SMSTPCR(reg));
 
 	spin_unlock_irqrestore(&priv->mstp_lock, flags);
 
@@ -159,8 +159,7 @@ static int cpg_mstp_clock_endisable(struct clk_hw *hw, bool enable)
 		return 0;
 
 	for (i = 1000; i > 0; --i) {
-		if (!(clk_readl(priv->base + MSTPSR(reg)) &
-		      bitmask))
+		if (!(readl(priv->base + MSTPSR(reg)) & bitmask))
 			break;
 		cpu_relax();
 	}
@@ -190,7 +189,7 @@ static int cpg_mstp_clock_is_enabled(struct clk_hw *hw)
 	struct cpg_mssr_priv *priv = clock->priv;
 	u32 value;
 
-	value = clk_readl(priv->base + MSTPSR(clock->index / 32));
+	value = readl(priv->base + MSTPSR(clock->index / 32));
 
 	return !(value & BIT(clock->index % 32));
 }
@@ -309,7 +308,7 @@ static void __init cpg_mssr_register_core_clk(const struct cpg_core_clk *core,
 	return;
 
 fail:
-	dev_err(dev, "Failed to register %s clock %s: %ld\n", "core,",
+	dev_err(dev, "Failed to register %s clock %s: %ld\n", "core",
 		core->name, PTR_ERR(clk));
 }
 
@@ -377,7 +376,7 @@ static void __init cpg_mssr_register_mod_clk(const struct mssr_mod_clk *mod,
 	return;
 
 fail:
-	dev_err(dev, "Failed to register %s clock %s: %ld\n", "module,",
+	dev_err(dev, "Failed to register %s clock %s: %ld\n", "module",
 		mod->name, PTR_ERR(clk));
 	kfree(clock);
 }
@@ -503,6 +502,18 @@ static int __init cpg_mssr_add_clk_domain(struct device *dev,
 }
 
 static const struct of_device_id cpg_mssr_match[] = {
+#ifdef CONFIG_ARCH_R8A7743
+	{
+		.compatible = "renesas,r8a7743-cpg-mssr",
+		.data = &r8a7743_cpg_mssr_info,
+	},
+#endif
+#ifdef CONFIG_ARCH_R8A7745
+	{
+		.compatible = "renesas,r8a7745-cpg-mssr",
+		.data = &r8a7745_cpg_mssr_info,
+	},
+#endif
 #ifdef CONFIG_ARCH_R8A7795
 	{
 		.compatible = "renesas,r8a7795-cpg-mssr",
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.h b/drivers/clk/renesas/renesas-cpg-mssr.h
index ee7edfaf1408..4bb7a80c6469 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.h
+++ b/drivers/clk/renesas/renesas-cpg-mssr.h
@@ -130,6 +130,8 @@ struct cpg_mssr_info {
 					struct clk **clks, void __iomem *base);
 };
 
+extern const struct cpg_mssr_info r8a7743_cpg_mssr_info;
+extern const struct cpg_mssr_info r8a7745_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a7795_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a7796_cpg_mssr_info;
 #endif
diff --git a/drivers/clk/rockchip/Makefile b/drivers/clk/rockchip/Makefile
index b5f2c8ed12e1..16e098c36f90 100644
--- a/drivers/clk/rockchip/Makefile
+++ b/drivers/clk/rockchip/Makefile
@@ -11,6 +11,7 @@ obj-y	+= clk-mmc-phase.o
 obj-y	+= clk-ddr.o
 obj-$(CONFIG_RESET_CONTROLLER)	+= softrst.o
 
+obj-y	+= clk-rk1108.o
 obj-y	+= clk-rk3036.o
 obj-y	+= clk-rk3188.o
 obj-y	+= clk-rk3228.o
diff --git a/drivers/clk/rockchip/clk-cpu.c b/drivers/clk/rockchip/clk-cpu.c
index 05b3d73bfefa..0e09684d43a5 100644
--- a/drivers/clk/rockchip/clk-cpu.c
+++ b/drivers/clk/rockchip/clk-cpu.c
@@ -124,9 +124,18 @@ static int rockchip_cpuclk_pre_rate_change(struct rockchip_cpuclk *cpuclk,
 					   struct clk_notifier_data *ndata)
 {
 	const struct rockchip_cpuclk_reg_data *reg_data = cpuclk->reg_data;
+	const struct rockchip_cpuclk_rate_table *rate;
 	unsigned long alt_prate, alt_div;
 	unsigned long flags;
 
+	/* check validity of the new rate */
+	rate = rockchip_get_cpuclk_settings(cpuclk, ndata->new_rate);
+	if (!rate) {
+		pr_err("%s: Invalid rate : %lu for cpuclk\n",
+		       __func__, ndata->new_rate);
+		return -EINVAL;
+	}
+
 	alt_prate = clk_get_rate(cpuclk->alt_parent);
 
 	spin_lock_irqsave(cpuclk->lock, flags);
diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c
index 9c1373e81683..6ed605776abd 100644
--- a/drivers/clk/rockchip/clk-pll.c
+++ b/drivers/clk/rockchip/clk-pll.c
@@ -319,7 +319,8 @@ static void rockchip_rk3036_pll_init(struct clk_hw *hw)
 
 	if (rate->fbdiv != cur.fbdiv || rate->postdiv1 != cur.postdiv1 ||
 		rate->refdiv != cur.refdiv || rate->postdiv2 != cur.postdiv2 ||
-		rate->dsmpd != cur.dsmpd || rate->frac != cur.frac) {
+		rate->dsmpd != cur.dsmpd ||
+		(!cur.dsmpd && (rate->frac != cur.frac))) {
 		struct clk *parent = clk_get_parent(hw->clk);
 
 		if (!parent) {
@@ -795,7 +796,8 @@ static void rockchip_rk3399_pll_init(struct clk_hw *hw)
 
 	if (rate->fbdiv != cur.fbdiv || rate->postdiv1 != cur.postdiv1 ||
 		rate->refdiv != cur.refdiv || rate->postdiv2 != cur.postdiv2 ||
-		rate->dsmpd != cur.dsmpd || rate->frac != cur.frac) {
+		rate->dsmpd != cur.dsmpd ||
+		(!cur.dsmpd && (rate->frac != cur.frac))) {
 		struct clk *parent = clk_get_parent(hw->clk);
 
 		if (!parent) {
diff --git a/drivers/clk/rockchip/clk-rk1108.c b/drivers/clk/rockchip/clk-rk1108.c
new file mode 100644
index 000000000000..92750d798e5d
--- /dev/null
+++ b/drivers/clk/rockchip/clk-rk1108.c
@@ -0,0 +1,531 @@
+/*
+ * Copyright (c) 2016 Rockchip Electronics Co. Ltd.
+ * Author: Shawn Lin <shawn.lin@rock-chips.com>
+ *         Andy Yan <andy.yan@rock-chips.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/syscore_ops.h>
+#include <dt-bindings/clock/rk1108-cru.h>
+#include "clk.h"
+
+#define RK1108_GRF_SOC_STATUS0	0x480
+
+enum rk1108_plls {
+	apll, dpll, gpll,
+};
+
+static struct rockchip_pll_rate_table rk1108_pll_rates[] = {
+	/* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */
+	RK3036_PLL_RATE(1608000000, 1, 67, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1584000000, 1, 66, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1560000000, 1, 65, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1536000000, 1, 64, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1512000000, 1, 63, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1488000000, 1, 62, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1464000000, 1, 61, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1440000000, 1, 60, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1416000000, 1, 59, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1392000000, 1, 58, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1368000000, 1, 57, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1344000000, 1, 56, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1320000000, 1, 55, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1296000000, 1, 54, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1272000000, 1, 53, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1248000000, 1, 52, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1200000000, 1, 50, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1188000000, 2, 99, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1104000000, 1, 46, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1100000000, 12, 550, 1, 1, 1, 0),
+	RK3036_PLL_RATE(1008000000, 1, 84, 2, 1, 1, 0),
+	RK3036_PLL_RATE(1000000000, 6, 500, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 984000000, 1, 82, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 960000000, 1, 80, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 936000000, 1, 78, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 912000000, 1, 76, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 900000000, 4, 300, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 888000000, 1, 74, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 864000000, 1, 72, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 840000000, 1, 70, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 816000000, 1, 68, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 800000000, 6, 400, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 700000000, 6, 350, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 696000000, 1, 58, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 600000000, 1, 75, 3, 1, 1, 0),
+	RK3036_PLL_RATE( 594000000, 2, 99, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 504000000, 1, 63, 3, 1, 1, 0),
+	RK3036_PLL_RATE( 500000000, 6, 250, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 408000000, 1, 68, 2, 2, 1, 0),
+	RK3036_PLL_RATE( 312000000, 1, 52, 2, 2, 1, 0),
+	RK3036_PLL_RATE( 216000000, 1, 72, 4, 2, 1, 0),
+	RK3036_PLL_RATE(  96000000, 1, 64, 4, 4, 1, 0),
+	{ /* sentinel */ },
+};
+
+#define RK1108_DIV_CORE_MASK		0xf
+#define RK1108_DIV_CORE_SHIFT		4
+
+#define RK1108_CLKSEL0(_core_peri_div)	\
+	{				\
+		.reg = RK1108_CLKSEL_CON(1),	\
+		.val = HIWORD_UPDATE(_core_peri_div, RK1108_DIV_CORE_MASK,\
+				RK1108_DIV_CORE_SHIFT)	\
+	}
+
+#define RK1108_CPUCLK_RATE(_prate, _core_peri_div)			\
+	{								\
+		.prate = _prate,					\
+		.divs = {						\
+			RK1108_CLKSEL0(_core_peri_div),		\
+		},							\
+	}
+
+static struct rockchip_cpuclk_rate_table rk1108_cpuclk_rates[] __initdata = {
+	RK1108_CPUCLK_RATE(816000000, 4),
+	RK1108_CPUCLK_RATE(600000000, 4),
+	RK1108_CPUCLK_RATE(312000000, 4),
+};
+
+static const struct rockchip_cpuclk_reg_data rk1108_cpuclk_data = {
+	.core_reg = RK1108_CLKSEL_CON(0),
+	.div_core_shift = 0,
+	.div_core_mask = 0x1f,
+	.mux_core_alt = 1,
+	.mux_core_main = 0,
+	.mux_core_shift = 8,
+	.mux_core_mask = 0x1,
+};
+
+PNAME(mux_pll_p)		= { "xin24m", "xin24m"};
+PNAME(mux_ddrphy_p)		= { "dpll_ddr", "gpll_ddr", "apll_ddr" };
+PNAME(mux_armclk_p)		= { "apll_core", "gpll_core", "dpll_core" };
+PNAME(mux_usb480m_pre_p)	= { "usbphy", "xin24m" };
+PNAME(mux_hdmiphy_phy_p)	= { "hdmiphy", "xin24m" };
+PNAME(mux_dclk_hdmiphy_pre_p)	= { "dclk_hdmiphy_src_gpll", "dclk_hdmiphy_src_dpll" };
+PNAME(mux_pll_src_4plls_p)	= { "dpll", "hdmiphy", "gpll", "usb480m" };
+PNAME(mux_pll_src_3plls_p)	= { "apll", "gpll", "dpll" };
+PNAME(mux_pll_src_2plls_p)	= { "dpll", "gpll" };
+PNAME(mux_pll_src_apll_gpll_p)	= { "apll", "gpll" };
+PNAME(mux_aclk_peri_src_p)	= { "aclk_peri_src_dpll", "aclk_peri_src_gpll" };
+PNAME(mux_aclk_bus_src_p)	= { "aclk_bus_src_gpll", "aclk_bus_src_apll", "aclk_bus_src_dpll" };
+PNAME(mux_mmc_src_p)		= { "dpll", "gpll", "xin24m", "usb480m" };
+PNAME(mux_pll_src_dpll_gpll_usb480m_p)	= { "dpll", "gpll", "usb480m" };
+PNAME(mux_uart0_p)		= { "uart0_src", "uart0_frac", "xin24m" };
+PNAME(mux_uart1_p)		= { "uart1_src", "uart1_frac", "xin24m" };
+PNAME(mux_uart2_p)		= { "uart2_src", "uart2_frac", "xin24m" };
+PNAME(mux_sclk_macphy_p)	= { "sclk_macphy_pre", "ext_gmac" };
+PNAME(mux_i2s0_pre_p)		= { "i2s0_src", "i2s0_frac", "ext_i2s", "xin12m" };
+PNAME(mux_i2s_out_p)		= { "i2s0_pre", "xin12m" };
+PNAME(mux_i2s1_p)		= { "i2s1_src", "i2s1_frac", "xin12m" };
+PNAME(mux_i2s2_p)		= { "i2s2_src", "i2s2_frac", "xin12m" };
+
+static struct rockchip_pll_clock rk1108_pll_clks[] __initdata = {
+	[apll] = PLL(pll_rk3399, PLL_APLL, "apll", mux_pll_p, 0, RK1108_PLL_CON(0),
+		     RK1108_PLL_CON(3), 8, 31, 0, rk1108_pll_rates),
+	[dpll] = PLL(pll_rk3399, PLL_DPLL, "dpll", mux_pll_p, 0, RK1108_PLL_CON(8),
+		     RK1108_PLL_CON(11), 8, 31, 0, NULL),
+	[gpll] = PLL(pll_rk3399, PLL_GPLL, "gpll", mux_pll_p, 0, RK1108_PLL_CON(16),
+		     RK1108_PLL_CON(19), 8, 31, ROCKCHIP_PLL_SYNC_RATE, rk1108_pll_rates),
+};
+
+#define MFLAGS CLK_MUX_HIWORD_MASK
+#define DFLAGS CLK_DIVIDER_HIWORD_MASK
+#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE)
+#define IFLAGS ROCKCHIP_INVERTER_HIWORD_MASK
+
+static struct rockchip_clk_branch rk1108_uart0_fracmux __initdata =
+	MUX(SCLK_UART0, "sclk_uart0", mux_uart0_p, CLK_SET_RATE_PARENT,
+			RK1108_CLKSEL_CON(13), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk1108_uart1_fracmux __initdata =
+	MUX(SCLK_UART1, "sclk_uart1", mux_uart1_p, CLK_SET_RATE_PARENT,
+			RK1108_CLKSEL_CON(14), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk1108_uart2_fracmux __initdata =
+	MUX(SCLK_UART2, "sclk_uart2", mux_uart2_p, CLK_SET_RATE_PARENT,
+			RK1108_CLKSEL_CON(15), 8, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk1108_i2s0_fracmux __initdata =
+	MUX(0, "i2s0_pre", mux_i2s0_pre_p, CLK_SET_RATE_PARENT,
+			RK1108_CLKSEL_CON(5), 12, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk1108_i2s1_fracmux __initdata =
+	MUX(0, "i2s1_pre", mux_i2s1_p, CLK_SET_RATE_PARENT,
+			RK1108_CLKSEL_CON(6), 12, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk1108_i2s2_fracmux __initdata =
+	MUX(0, "i2s2_pre", mux_i2s2_p, CLK_SET_RATE_PARENT,
+			RK1108_CLKSEL_CON(7), 12, 2, MFLAGS);
+
+static struct rockchip_clk_branch rk1108_clk_branches[] __initdata = {
+	MUX(0, "hdmi_phy", mux_hdmiphy_phy_p, CLK_SET_RATE_PARENT,
+			RK1108_MISC_CON, 13, 2, MFLAGS),
+	MUX(0, "usb480m", mux_usb480m_pre_p, CLK_SET_RATE_PARENT,
+			RK1108_MISC_CON, 15, 2, MFLAGS),
+	/*
+	 * Clock-Architecture Diagram 2
+	 */
+
+	/* PD_CORE */
+	GATE(0, "dpll_core", "dpll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(0), 1, GFLAGS),
+	GATE(0, "apll_core", "apll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(0), 0, GFLAGS),
+	GATE(0, "gpll_core", "gpll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(0), 2, GFLAGS),
+	COMPOSITE_NOMUX(0, "pclken_dbg", "armclk", CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(1), 4, 4, DFLAGS | CLK_DIVIDER_READ_ONLY,
+			RK1108_CLKGATE_CON(0), 5, GFLAGS),
+	COMPOSITE_NOMUX(ACLK_ENMCORE, "aclkenm_core", "armclk", CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(1), 0, 3, DFLAGS | CLK_DIVIDER_READ_ONLY,
+			RK1108_CLKGATE_CON(0), 4, GFLAGS),
+	GATE(ACLK_CORE, "aclk_core", "aclkenm_core", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(11), 0, GFLAGS),
+	GATE(0, "pclk_dbg", "pclken_dbg", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(11), 1, GFLAGS),
+
+	/* PD_RKVENC */
+
+	/* PD_RKVDEC */
+
+	/* PD_PMU_wrapper */
+	COMPOSITE_NOMUX(0, "pmu_24m_ena", "gpll", CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(38), 0, 5, DFLAGS,
+			RK1108_CLKGATE_CON(8), 12, GFLAGS),
+	GATE(0, "pmu", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(10), 0, GFLAGS),
+	GATE(0, "intmem1", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(10), 1, GFLAGS),
+	GATE(0, "gpio0_pmu", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(10), 2, GFLAGS),
+	GATE(0, "pmugrf", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(10), 3, GFLAGS),
+	GATE(0, "pmu_noc", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(10), 4, GFLAGS),
+	GATE(0, "i2c0_pmu_pclk", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(10), 5, GFLAGS),
+	GATE(0, "pwm0_pmu_pclk", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(10), 6, GFLAGS),
+	COMPOSITE(0, "pwm0_pmu_clk", mux_pll_src_2plls_p, CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(12), 7, 1, MFLAGS, 0, 7, DFLAGS,
+			RK1108_CLKGATE_CON(8), 15, GFLAGS),
+	COMPOSITE(0, "i2c0_pmu_clk", mux_pll_src_2plls_p, CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(19), 7, 1, MFLAGS, 0, 7, DFLAGS,
+			RK1108_CLKGATE_CON(8), 14, GFLAGS),
+	GATE(0, "pvtm_pmu", "xin24m", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(8), 13, GFLAGS),
+
+	/*
+	 * Clock-Architecture Diagram 4
+	 */
+	COMPOSITE(0, "aclk_vio0_2wrap_occ", mux_pll_src_4plls_p, CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(28), 6, 2, MFLAGS, 0, 5, DFLAGS,
+			RK1108_CLKGATE_CON(6), 0, GFLAGS),
+	GATE(0, "aclk_vio0_pre", "aclk_vio0_2wrap_occ", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(17), 0, GFLAGS),
+	COMPOSITE_NOMUX(0, "hclk_vio_pre", "aclk_vio0_pre", 0,
+			RK1108_CLKSEL_CON(29), 0, 5, DFLAGS,
+			RK1108_CLKGATE_CON(7), 2, GFLAGS),
+	COMPOSITE_NOMUX(0, "pclk_vio_pre", "aclk_vio0_pre", 0,
+			RK1108_CLKSEL_CON(29), 8, 5, DFLAGS,
+			RK1108_CLKGATE_CON(7), 3, GFLAGS),
+
+	INVERTER(0, "pclk_vip", "ext_vip",
+			RK1108_CLKSEL_CON(31), 8, IFLAGS),
+	GATE(0, "pclk_isp_pre", "pclk_vip", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(7), 6, GFLAGS),
+	GATE(0, "pclk_isp", "pclk_isp_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(18), 10, GFLAGS),
+	GATE(0, "dclk_hdmiphy_src_gpll", "gpll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(6), 5, GFLAGS),
+	GATE(0, "dclk_hdmiphy_src_dpll", "dpll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(6), 4, GFLAGS),
+	COMPOSITE_NOGATE(0, "dclk_hdmiphy", mux_dclk_hdmiphy_pre_p, 0,
+			RK1108_CLKSEL_CON(32), 6, 2, MFLAGS, 8, 6, DFLAGS),
+
+	/*
+	 * Clock-Architecture Diagram 5
+	 */
+
+	FACTOR(0, "xin12m", "xin24m", 0, 1, 2),
+
+	COMPOSITE(0, "i2s0_src", mux_pll_src_2plls_p, 0,
+			RK1108_CLKSEL_CON(5), 8, 1, MFLAGS, 0, 7, DFLAGS,
+			RK1108_CLKGATE_CON(2), 0, GFLAGS),
+	COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT,
+			RK1108_CLKSEL_CON(8), 0,
+			RK1108_CLKGATE_CON(2), 1, GFLAGS,
+			&rk1108_i2s0_fracmux),
+	GATE(SCLK_I2S0, "sclk_i2s0", "i2s0_pre", CLK_SET_RATE_PARENT,
+			RK1108_CLKGATE_CON(2), 2, GFLAGS),
+	COMPOSITE_NODIV(0, "i2s_out", mux_i2s_out_p, 0,
+			RK1108_CLKSEL_CON(5), 15, 1, MFLAGS,
+			RK1108_CLKGATE_CON(2), 3, GFLAGS),
+
+	COMPOSITE(0, "i2s1_src", mux_pll_src_2plls_p, 0,
+			RK1108_CLKSEL_CON(6), 8, 1, MFLAGS, 0, 7, DFLAGS,
+			RK1108_CLKGATE_CON(2), 4, GFLAGS),
+	COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT,
+			RK2928_CLKSEL_CON(9), 0,
+			RK2928_CLKGATE_CON(2), 5, GFLAGS,
+			&rk1108_i2s1_fracmux),
+	GATE(SCLK_I2S1, "sclk_i2s1", "i2s1_pre", CLK_SET_RATE_PARENT,
+			RK1108_CLKGATE_CON(2), 6, GFLAGS),
+
+	COMPOSITE(0, "i2s2_src", mux_pll_src_2plls_p, 0,
+			RK1108_CLKSEL_CON(7), 8, 1, MFLAGS, 0, 7, DFLAGS,
+			RK1108_CLKGATE_CON(3), 8, GFLAGS),
+	COMPOSITE_FRACMUX(0, "i2s2_frac", "i2s2_src", CLK_SET_RATE_PARENT,
+			RK1108_CLKSEL_CON(10), 0,
+			RK1108_CLKGATE_CON(2), 9, GFLAGS,
+			&rk1108_i2s2_fracmux),
+	GATE(SCLK_I2S2, "sclk_i2s2", "i2s2_pre", CLK_SET_RATE_PARENT,
+			RK1108_CLKGATE_CON(2), 10, GFLAGS),
+
+	/* PD_BUS */
+	GATE(0, "aclk_bus_src_gpll", "gpll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(1), 0, GFLAGS),
+	GATE(0, "aclk_bus_src_apll", "apll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(1), 1, GFLAGS),
+	GATE(0, "aclk_bus_src_dpll", "dpll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(1), 2, GFLAGS),
+	COMPOSITE_NOGATE(ACLK_PRE, "aclk_bus_pre", mux_aclk_bus_src_p, 0,
+			RK1108_CLKSEL_CON(2), 8, 2, MFLAGS, 0, 5, DFLAGS),
+	COMPOSITE_NOMUX(0, "hclk_bus_pre", "aclk_bus_2wrap_occ", 0,
+			RK1108_CLKSEL_CON(3), 0, 5, DFLAGS,
+			RK1108_CLKGATE_CON(1), 4, GFLAGS),
+	COMPOSITE_NOMUX(0, "pclken_bus", "aclk_bus_2wrap_occ", 0,
+			RK1108_CLKSEL_CON(3), 8, 5, DFLAGS,
+			RK1108_CLKGATE_CON(1), 5, GFLAGS),
+	GATE(0, "pclk_bus_pre", "pclken_bus", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(1), 6, GFLAGS),
+	GATE(0, "pclk_top_pre", "pclken_bus", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(1), 7, GFLAGS),
+	GATE(0, "pclk_ddr_pre", "pclken_bus", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(1), 8, GFLAGS),
+	GATE(0, "clk_timer0", "mux_pll_p", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(1), 9, GFLAGS),
+	GATE(0, "clk_timer1", "mux_pll_p", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(1), 10, GFLAGS),
+	GATE(0, "pclk_timer", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(13), 4, GFLAGS),
+
+	COMPOSITE(0, "uart0_src", mux_pll_src_dpll_gpll_usb480m_p, CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(13), 12, 2, MFLAGS, 0, 7, DFLAGS,
+			RK1108_CLKGATE_CON(3), 1, GFLAGS),
+	COMPOSITE(0, "uart1_src", mux_pll_src_dpll_gpll_usb480m_p, CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(14), 12, 2, MFLAGS, 0, 7, DFLAGS,
+			RK1108_CLKGATE_CON(3), 3, GFLAGS),
+	COMPOSITE(0, "uart21_src", mux_pll_src_dpll_gpll_usb480m_p, CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(15), 12, 2, MFLAGS, 0, 7, DFLAGS,
+			RK1108_CLKGATE_CON(3), 5, GFLAGS),
+
+	COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT,
+			RK1108_CLKSEL_CON(16), 0,
+			RK1108_CLKGATE_CON(3), 2, GFLAGS,
+			&rk1108_uart0_fracmux),
+	COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT,
+			RK1108_CLKSEL_CON(17), 0,
+			RK1108_CLKGATE_CON(3), 4, GFLAGS,
+			&rk1108_uart1_fracmux),
+	COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT,
+			RK1108_CLKSEL_CON(18), 0,
+			RK1108_CLKGATE_CON(3), 6, GFLAGS,
+			&rk1108_uart2_fracmux),
+	GATE(PCLK_UART0, "pclk_uart0", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(13), 10, GFLAGS),
+	GATE(PCLK_UART1, "pclk_uart1", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(13), 11, GFLAGS),
+	GATE(PCLK_UART2, "pclk_uart2", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(13), 12, GFLAGS),
+
+	COMPOSITE(0, "clk_i2c1", mux_pll_src_2plls_p, CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(19), 15, 2, MFLAGS, 8, 7, DFLAGS,
+			RK1108_CLKGATE_CON(3), 7, GFLAGS),
+	COMPOSITE(0, "clk_i2c2", mux_pll_src_2plls_p, CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(20), 7, 2, MFLAGS, 0, 7, DFLAGS,
+			RK1108_CLKGATE_CON(3), 8, GFLAGS),
+	COMPOSITE(0, "clk_i2c3", mux_pll_src_2plls_p, CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(20), 15, 2, MFLAGS, 8, 7, DFLAGS,
+			RK1108_CLKGATE_CON(3), 9, GFLAGS),
+	GATE(0, "pclk_i2c1", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(13), 0, GFLAGS),
+	GATE(0, "pclk_i2c2", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(13), 1, GFLAGS),
+	GATE(0, "pclk_i2c3", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(13), 2, GFLAGS),
+	COMPOSITE(0, "clk_pwm1", mux_pll_src_2plls_p, CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(12), 15, 2, MFLAGS, 8, 7, DFLAGS,
+			RK1108_CLKGATE_CON(3), 10, GFLAGS),
+	GATE(0, "pclk_pwm1", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(13), 6, GFLAGS),
+	GATE(0, "pclk_wdt", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(13), 3, GFLAGS),
+	GATE(0, "pclk_gpio1", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(13), 7, GFLAGS),
+	GATE(0, "pclk_gpio2", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(13), 8, GFLAGS),
+	GATE(0, "pclk_gpio3", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(13), 9, GFLAGS),
+
+	GATE(0, "pclk_grf", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(14), 0, GFLAGS),
+
+	GATE(ACLK_DMAC, "aclk_dmac", "aclk_bus_pre", 0,
+	     RK1108_CLKGATE_CON(12), 2, GFLAGS),
+	GATE(0, "hclk_rom", "hclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(12), 3, GFLAGS),
+	GATE(0, "aclk_intmem", "aclk_bus_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(12), 1, GFLAGS),
+
+	/* PD_DDR */
+	GATE(0, "apll_ddr", "apll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(0), 8, GFLAGS),
+	GATE(0, "dpll_ddr", "dpll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(0), 9, GFLAGS),
+	GATE(0, "gpll_ddr", "gpll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(0), 10, GFLAGS),
+	COMPOSITE(0, "ddrphy4x", mux_ddrphy_p, CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(4), 8, 2, MFLAGS, 0, 3,
+			DFLAGS | CLK_DIVIDER_POWER_OF_TWO,
+			RK1108_CLKGATE_CON(10), 9, GFLAGS),
+	GATE(0, "ddrupctl", "ddrphy_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(12), 4, GFLAGS),
+	GATE(0, "ddrc", "ddrphy", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(12), 5, GFLAGS),
+	GATE(0, "ddrmon", "ddrphy_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(12), 6, GFLAGS),
+	GATE(0, "timer_clk", "xin24m", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(0), 11, GFLAGS),
+
+	/*
+	 * Clock-Architecture Diagram 6
+	 */
+
+	/* PD_PERI */
+	COMPOSITE_NOMUX(0, "pclk_periph_pre", "gpll", 0,
+			RK1108_CLKSEL_CON(23), 10, 5, DFLAGS,
+			RK1108_CLKGATE_CON(4), 5, GFLAGS),
+	GATE(0, "pclk_periph", "pclk_periph_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(15), 13, GFLAGS),
+	COMPOSITE_NOMUX(0, "hclk_periph_pre", "gpll", 0,
+			RK1108_CLKSEL_CON(23), 5, 5, DFLAGS,
+			RK1108_CLKGATE_CON(4), 4, GFLAGS),
+	GATE(0, "hclk_periph", "hclk_periph_pre", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(15), 12, GFLAGS),
+
+	GATE(0, "aclk_peri_src_dpll", "dpll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(4), 1, GFLAGS),
+	GATE(0, "aclk_peri_src_gpll", "gpll", CLK_IGNORE_UNUSED,
+			RK1108_CLKGATE_CON(4), 2, GFLAGS),
+	COMPOSITE(0, "aclk_periph", mux_aclk_peri_src_p, CLK_IGNORE_UNUSED,
+			RK1108_CLKSEL_CON(23), 15, 2, MFLAGS, 0, 5, DFLAGS,
+			RK1108_CLKGATE_CON(15), 11, GFLAGS),
+
+	COMPOSITE(SCLK_SDMMC, "sclk_sdmmc0", mux_mmc_src_p, 0,
+			RK1108_CLKSEL_CON(25), 8, 2, MFLAGS, 0, 8, DFLAGS,
+			RK1108_CLKGATE_CON(5), 0, GFLAGS),
+
+	COMPOSITE_NODIV(0, "sclk_sdio_src", mux_mmc_src_p, 0,
+			RK1108_CLKSEL_CON(25), 10, 2, MFLAGS,
+			RK1108_CLKGATE_CON(5), 2, GFLAGS),
+	DIV(SCLK_SDIO, "sclk_sdio", "sclk_sdio_src", 0,
+			RK1108_CLKSEL_CON(26), 0, 8, DFLAGS),
+
+	COMPOSITE_NODIV(0, "sclk_emmc_src", mux_mmc_src_p, 0,
+			RK1108_CLKSEL_CON(25), 12, 2, MFLAGS,
+			RK1108_CLKGATE_CON(5), 1, GFLAGS),
+	DIV(SCLK_EMMC, "sclk_emmc", "sclk_emmc_src", 0,
+			RK2928_CLKSEL_CON(26), 8, 8, DFLAGS),
+	GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_periph", 0, RK1108_CLKGATE_CON(15), 0, GFLAGS),
+	GATE(HCLK_SDIO, "hclk_sdio", "hclk_periph", 0, RK1108_CLKGATE_CON(15), 1, GFLAGS),
+	GATE(HCLK_EMMC, "hclk_emmc", "hclk_periph", 0, RK1108_CLKGATE_CON(15), 2, GFLAGS),
+
+	COMPOSITE(SCLK_NANDC, "sclk_nandc", mux_pll_src_2plls_p, 0,
+			RK1108_CLKSEL_CON(27), 14, 2, MFLAGS, 8, 5, DFLAGS,
+			RK1108_CLKGATE_CON(5), 3, GFLAGS),
+	GATE(HCLK_NANDC, "hclk_nandc", "hclk_periph", 0, RK1108_CLKGATE_CON(15), 3, GFLAGS),
+
+	COMPOSITE(SCLK_SFC, "sclk_sfc", mux_pll_src_2plls_p, 0,
+			RK1108_CLKSEL_CON(27), 7, 2, MFLAGS, 0, 7, DFLAGS,
+			RK1108_CLKGATE_CON(5), 4, GFLAGS),
+	GATE(HCLK_SFC, "hclk_sfc", "hclk_periph", 0, RK1108_CLKGATE_CON(15), 10, GFLAGS),
+
+	COMPOSITE(0, "sclk_macphy_pre", mux_pll_src_apll_gpll_p, 0,
+			RK1108_CLKSEL_CON(24), 12, 2, MFLAGS, 0, 5, DFLAGS,
+			RK1108_CLKGATE_CON(4), 10, GFLAGS),
+	MUX(0, "sclk_macphy", mux_sclk_macphy_p, CLK_SET_RATE_PARENT,
+			RK1108_CLKSEL_CON(24), 8, 2, MFLAGS),
+	GATE(0, "sclk_macphy_rx", "sclk_macphy", 0, RK1108_CLKGATE_CON(4), 8, GFLAGS),
+	GATE(0, "sclk_mac_ref", "sclk_macphy", 0, RK1108_CLKGATE_CON(4), 6, GFLAGS),
+	GATE(0, "sclk_mac_refout", "sclk_macphy", 0, RK1108_CLKGATE_CON(4), 7, GFLAGS),
+
+	MMC(SCLK_SDMMC_DRV,    "sdmmc_drv",    "sclk_sdmmc", RK1108_SDMMC_CON0, 1),
+	MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "sclk_sdmmc", RK1108_SDMMC_CON1, 1),
+
+	MMC(SCLK_SDIO_DRV,     "sdio_drv",     "sclk_sdio",  RK1108_SDIO_CON0,  1),
+	MMC(SCLK_SDIO_SAMPLE,  "sdio_sample",  "sclk_sdio",  RK1108_SDIO_CON1,  1),
+
+	MMC(SCLK_EMMC_DRV,     "emmc_drv",     "sclk_emmc",  RK1108_EMMC_CON0,  1),
+	MMC(SCLK_EMMC_SAMPLE,  "emmc_sample",  "sclk_emmc",  RK1108_EMMC_CON1,  1),
+};
+
+static const char *const rk1108_critical_clocks[] __initconst = {
+	"aclk_core",
+	"aclk_bus_src_gpll",
+	"aclk_periph",
+	"hclk_periph",
+	"pclk_periph",
+};
+
+static void __init rk1108_clk_init(struct device_node *np)
+{
+	struct rockchip_clk_provider *ctx;
+	void __iomem *reg_base;
+
+	reg_base = of_iomap(np, 0);
+	if (!reg_base) {
+		pr_err("%s: could not map cru region\n", __func__);
+		return;
+	}
+
+	ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS);
+	if (IS_ERR(ctx)) {
+		pr_err("%s: rockchip clk init failed\n", __func__);
+		iounmap(reg_base);
+		return;
+	}
+
+	rockchip_clk_register_plls(ctx, rk1108_pll_clks,
+				   ARRAY_SIZE(rk1108_pll_clks),
+				   RK1108_GRF_SOC_STATUS0);
+	rockchip_clk_register_branches(ctx, rk1108_clk_branches,
+				  ARRAY_SIZE(rk1108_clk_branches));
+	rockchip_clk_protect_critical(rk1108_critical_clocks,
+				      ARRAY_SIZE(rk1108_critical_clocks));
+
+	rockchip_clk_register_armclk(ctx, ARMCLK, "armclk",
+			mux_armclk_p, ARRAY_SIZE(mux_armclk_p),
+			&rk1108_cpuclk_data, rk1108_cpuclk_rates,
+			ARRAY_SIZE(rk1108_cpuclk_rates));
+
+	rockchip_register_softrst(np, 13, reg_base + RK1108_SOFTRST_CON(0),
+				  ROCKCHIP_SOFTRST_HIWORD_MASK);
+
+	rockchip_register_restart_notifier(ctx, RK1108_GLB_SRST_FST, NULL);
+
+	rockchip_clk_of_add_provider(np, ctx);
+}
+CLK_OF_DECLARE(rk1108_cru, "rockchip,rk1108-cru", rk1108_clk_init);
diff --git a/drivers/clk/rockchip/clk-rk3188.c b/drivers/clk/rockchip/clk-rk3188.c
index d0e722a0e8cf..062ef4960244 100644
--- a/drivers/clk/rockchip/clk-rk3188.c
+++ b/drivers/clk/rockchip/clk-rk3188.c
@@ -89,6 +89,7 @@ static struct rockchip_pll_rate_table rk3188_pll_rates[] = {
 	RK3066_PLL_RATE( 504000000, 1, 84, 4),
 	RK3066_PLL_RATE( 456000000, 1, 76, 4),
 	RK3066_PLL_RATE( 408000000, 1, 68, 4),
+	RK3066_PLL_RATE( 400000000, 3, 100, 2),
 	RK3066_PLL_RATE( 384000000, 2, 128, 4),
 	RK3066_PLL_RATE( 360000000, 1, 60, 4),
 	RK3066_PLL_RATE( 312000000, 1, 52, 4),
@@ -306,14 +307,14 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = {
 			RK2928_CLKSEL_CON(26), 8, 1, MFLAGS, 0, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO,
 			RK2928_CLKGATE_CON(0), 2, GFLAGS),
 
-	GATE(0, "aclk_cpu", "aclk_cpu_pre", 0,
+	GATE(ACLK_CPU, "aclk_cpu", "aclk_cpu_pre", 0,
 			RK2928_CLKGATE_CON(0), 3, GFLAGS),
 
 	GATE(0, "atclk_cpu", "pclk_cpu_pre", 0,
 			RK2928_CLKGATE_CON(0), 6, GFLAGS),
-	GATE(0, "pclk_cpu", "pclk_cpu_pre", 0,
+	GATE(PCLK_CPU, "pclk_cpu", "pclk_cpu_pre", 0,
 			RK2928_CLKGATE_CON(0), 5, GFLAGS),
-	GATE(0, "hclk_cpu", "hclk_cpu_pre", CLK_IGNORE_UNUSED,
+	GATE(HCLK_CPU, "hclk_cpu", "hclk_cpu_pre", CLK_IGNORE_UNUSED,
 			RK2928_CLKGATE_CON(0), 4, GFLAGS),
 
 	COMPOSITE(0, "aclk_lcdc0_pre", mux_pll_src_cpll_gpll_p, CLK_IGNORE_UNUSED,
@@ -323,12 +324,12 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = {
 			RK2928_CLKSEL_CON(31), 15, 1, MFLAGS, 8, 5, DFLAGS,
 			RK2928_CLKGATE_CON(1), 4, GFLAGS),
 
-	GATE(0, "aclk_peri", "aclk_peri_pre", 0,
+	GATE(ACLK_PERI, "aclk_peri", "aclk_peri_pre", 0,
 			RK2928_CLKGATE_CON(2), 1, GFLAGS),
-	COMPOSITE_NOMUX(0, "hclk_peri", "aclk_peri_pre", 0,
+	COMPOSITE_NOMUX(HCLK_PERI, "hclk_peri", "aclk_peri_pre", 0,
 			RK2928_CLKSEL_CON(10), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO,
 			RK2928_CLKGATE_CON(2), 2, GFLAGS),
-	COMPOSITE_NOMUX(0, "pclk_peri", "aclk_peri_pre", 0,
+	COMPOSITE_NOMUX(PCLK_PERI, "pclk_peri", "aclk_peri_pre", 0,
 			RK2928_CLKSEL_CON(10), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO,
 			RK2928_CLKGATE_CON(2), 3, GFLAGS),
 
diff --git a/drivers/clk/rockchip/clk-rk3399.c b/drivers/clk/rockchip/clk-rk3399.c
index 8387c7a40bda..3490887b0579 100644
--- a/drivers/clk/rockchip/clk-rk3399.c
+++ b/drivers/clk/rockchip/clk-rk3399.c
@@ -77,7 +77,7 @@ static struct rockchip_pll_rate_table rk3399_pll_rates[] = {
 	RK3036_PLL_RATE(1104000000, 1, 46, 1, 1, 1, 0),
 	RK3036_PLL_RATE(1100000000, 12, 550, 1, 1, 1, 0),
 	RK3036_PLL_RATE(1008000000, 1, 84, 2, 1, 1, 0),
-	RK3036_PLL_RATE(1000000000, 6, 500, 2, 1, 1, 0),
+	RK3036_PLL_RATE(1000000000, 1, 125, 3, 1, 1, 0),
 	RK3036_PLL_RATE( 984000000, 1, 82, 2, 1, 1, 0),
 	RK3036_PLL_RATE( 960000000, 1, 80, 2, 1, 1, 0),
 	RK3036_PLL_RATE( 936000000, 1, 78, 2, 1, 1, 0),
@@ -87,12 +87,13 @@ static struct rockchip_pll_rate_table rk3399_pll_rates[] = {
 	RK3036_PLL_RATE( 864000000, 1, 72, 2, 1, 1, 0),
 	RK3036_PLL_RATE( 840000000, 1, 70, 2, 1, 1, 0),
 	RK3036_PLL_RATE( 816000000, 1, 68, 2, 1, 1, 0),
-	RK3036_PLL_RATE( 800000000, 6, 400, 2, 1, 1, 0),
+	RK3036_PLL_RATE( 800000000, 1, 100, 3, 1, 1, 0),
 	RK3036_PLL_RATE( 700000000, 6, 350, 2, 1, 1, 0),
 	RK3036_PLL_RATE( 696000000, 1, 58, 2, 1, 1, 0),
 	RK3036_PLL_RATE( 676000000, 3, 169, 2, 1, 1, 0),
 	RK3036_PLL_RATE( 600000000, 1, 75, 3, 1, 1, 0),
 	RK3036_PLL_RATE( 594000000, 1, 99, 4, 1, 1, 0),
+	RK3036_PLL_RATE( 533250000, 8, 711, 4, 1, 1, 0),
 	RK3036_PLL_RATE( 504000000, 1, 63, 3, 1, 1, 0),
 	RK3036_PLL_RATE( 500000000, 6, 250, 2, 1, 1, 0),
 	RK3036_PLL_RATE( 408000000, 1, 68, 2, 2, 1, 0),
@@ -410,11 +411,11 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	GATE(SCLK_USB2PHY1_REF, "clk_usb2phy1_ref", "xin24m", CLK_IGNORE_UNUSED,
 			RK3399_CLKGATE_CON(6), 6, GFLAGS),
 
-	GATE(0, "clk_usbphy0_480m_src", "clk_usbphy0_480m", CLK_IGNORE_UNUSED,
+	GATE(0, "clk_usbphy0_480m_src", "clk_usbphy0_480m", 0,
 			RK3399_CLKGATE_CON(13), 12, GFLAGS),
-	GATE(0, "clk_usbphy1_480m_src", "clk_usbphy1_480m", CLK_IGNORE_UNUSED,
+	GATE(0, "clk_usbphy1_480m_src", "clk_usbphy1_480m", 0,
 			RK3399_CLKGATE_CON(13), 12, GFLAGS),
-	MUX(0, "clk_usbphy_480m", mux_usbphy_480m_p, CLK_IGNORE_UNUSED,
+	MUX(0, "clk_usbphy_480m", mux_usbphy_480m_p, 0,
 			RK3399_CLKSEL_CON(14), 6, 1, MFLAGS),
 
 	MUX(0, "upll", mux_pll_src_24m_usbphy480m_p, 0,
@@ -498,7 +499,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 			RK3399_CLKGATE_CON(14), 10, GFLAGS),
 	GATE(ACLK_GIC_ADB400_CORE_L_2_GIC, "aclk_core_adb400_core_l_2_gic", "armclkl", CLK_IGNORE_UNUSED,
 			RK3399_CLKGATE_CON(14), 11, GFLAGS),
-	GATE(SCLK_PVTM_CORE_L, "clk_pvtm_core_l", "xin24m", CLK_IGNORE_UNUSED,
+	GATE(SCLK_PVTM_CORE_L, "clk_pvtm_core_l", "xin24m", 0,
 			RK3399_CLKGATE_CON(0), 7, GFLAGS),
 
 	/* big core */
@@ -539,7 +540,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	GATE(0, "pclk_dbg_cxcs_pd_core_b", "pclk_dbg_core_b", CLK_IGNORE_UNUSED,
 			RK3399_CLKGATE_CON(14), 2, GFLAGS),
 
-	GATE(SCLK_PVTM_CORE_B, "clk_pvtm_core_b", "xin24m", CLK_IGNORE_UNUSED,
+	GATE(SCLK_PVTM_CORE_B, "clk_pvtm_core_b", "xin24m", 0,
 			RK3399_CLKGATE_CON(1), 7, GFLAGS),
 
 	/* gmac */
@@ -675,18 +676,18 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 
 	GATE(PCLK_CENTER_MAIN_NOC, "pclk_center_main_noc", "pclk_ddr", CLK_IGNORE_UNUSED,
 			RK3399_CLKGATE_CON(18), 10, GFLAGS),
-	GATE(PCLK_DDR_MON, "pclk_ddr_mon", "pclk_ddr", CLK_IGNORE_UNUSED,
+	GATE(PCLK_DDR_MON, "pclk_ddr_mon", "pclk_ddr", 0,
 			RK3399_CLKGATE_CON(18), 12, GFLAGS),
 	GATE(PCLK_CIC, "pclk_cic", "pclk_ddr", CLK_IGNORE_UNUSED,
 			RK3399_CLKGATE_CON(18), 15, GFLAGS),
 	GATE(PCLK_DDR_SGRF, "pclk_ddr_sgrf", "pclk_ddr", CLK_IGNORE_UNUSED,
 			RK3399_CLKGATE_CON(19), 2, GFLAGS),
 
-	GATE(SCLK_PVTM_DDR, "clk_pvtm_ddr", "xin24m", CLK_IGNORE_UNUSED,
+	GATE(SCLK_PVTM_DDR, "clk_pvtm_ddr", "xin24m", 0,
 			RK3399_CLKGATE_CON(4), 11, GFLAGS),
-	GATE(SCLK_DFIMON0_TIMER, "clk_dfimon0_timer", "xin24m", CLK_IGNORE_UNUSED,
+	GATE(SCLK_DFIMON0_TIMER, "clk_dfimon0_timer", "xin24m", 0,
 			RK3399_CLKGATE_CON(3), 5, GFLAGS),
-	GATE(SCLK_DFIMON1_TIMER, "clk_dfimon1_timer", "xin24m", CLK_IGNORE_UNUSED,
+	GATE(SCLK_DFIMON1_TIMER, "clk_dfimon1_timer", "xin24m", 0,
 			RK3399_CLKGATE_CON(3), 6, GFLAGS),
 
 	/* cci */
@@ -966,7 +967,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	GATE(SCLK_INTMEM3, "clk_intmem3", "aclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(23), 5, GFLAGS),
 	GATE(SCLK_INTMEM4, "clk_intmem4", "aclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(23), 6, GFLAGS),
 	GATE(SCLK_INTMEM5, "clk_intmem5", "aclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(23), 7, GFLAGS),
-	GATE(ACLK_DCF, "aclk_dcf", "aclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(23), 8, GFLAGS),
+	GATE(ACLK_DCF, "aclk_dcf", "aclk_perilp0", 0, RK3399_CLKGATE_CON(23), 8, GFLAGS),
 	GATE(ACLK_DMAC0_PERILP, "aclk_dmac0_perilp", "aclk_perilp0", 0, RK3399_CLKGATE_CON(25), 5, GFLAGS),
 	GATE(ACLK_DMAC1_PERILP, "aclk_dmac1_perilp", "aclk_perilp0", 0, RK3399_CLKGATE_CON(25), 6, GFLAGS),
 	GATE(ACLK_PERILP0_NOC, "aclk_perilp0_noc", "aclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(25), 7, GFLAGS),
@@ -980,7 +981,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	GATE(HCLK_PERILP0_NOC, "hclk_perilp0_noc", "hclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(25), 8, GFLAGS),
 
 	/* pclk_perilp0 gates */
-	GATE(PCLK_DCF, "pclk_dcf", "pclk_perilp0", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(23), 9, GFLAGS),
+	GATE(PCLK_DCF, "pclk_dcf", "pclk_perilp0", 0, RK3399_CLKGATE_CON(23), 9, GFLAGS),
 
 	/* crypto */
 	COMPOSITE(SCLK_CRYPTO0, "clk_crypto0", mux_pll_src_cpll_gpll_ppll_p, 0,
@@ -1382,8 +1383,8 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = {
 	/* clk_test */
 	/* clk_test_pre is controlled by CRU_MISC_CON[3] */
 	COMPOSITE_NOMUX(0, "clk_test", "clk_test_pre", CLK_IGNORE_UNUSED,
-			RK3368_CLKSEL_CON(58), 0, 5, DFLAGS,
-			RK3368_CLKGATE_CON(13), 11, GFLAGS),
+			RK3399_CLKSEL_CON(58), 0, 5, DFLAGS,
+			RK3399_CLKGATE_CON(13), 11, GFLAGS),
 
 	/* ddrc */
 	GATE(0, "clk_ddrc_lpll_src", "lpll", 0, RK3399_CLKGATE_CON(3),
diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h
index 1653edd792a5..d67eecc4ade9 100644
--- a/drivers/clk/rockchip/clk.h
+++ b/drivers/clk/rockchip/clk.h
@@ -34,6 +34,21 @@ struct clk;
 #define HIWORD_UPDATE(val, mask, shift) \
 		((val) << (shift) | (mask) << ((shift) + 16))
 
+/* register positions shared by RK1108, RK2928, RK3036, RK3066, RK3188 and RK3228 */
+#define RK1108_PLL_CON(x)		((x) * 0x4)
+#define RK1108_CLKSEL_CON(x)		((x) * 0x4 + 0x60)
+#define RK1108_CLKGATE_CON(x)		((x) * 0x4 + 0x120)
+#define RK1108_SOFTRST_CON(x)		((x) * 0x4 + 0x180)
+#define RK1108_GLB_SRST_FST		0x1c0
+#define RK1108_GLB_SRST_SND		0x1c4
+#define RK1108_MISC_CON			0x1cc
+#define RK1108_SDMMC_CON0		0x1d8
+#define RK1108_SDMMC_CON1		0x1dc
+#define RK1108_SDIO_CON0		0x1e0
+#define RK1108_SDIO_CON1		0x1e4
+#define RK1108_EMMC_CON0		0x1e8
+#define RK1108_EMMC_CON1		0x1ec
+
 #define RK2928_PLL_CON(x)		((x) * 0x4)
 #define RK2928_MODE_CON		0x40
 #define RK2928_CLKSEL_CON(x)	((x) * 0x4 + 0x44)
diff --git a/drivers/clk/samsung/clk-exynos5433.c b/drivers/clk/samsung/clk-exynos5433.c
index ea1608682d7f..f096bd7df40c 100644
--- a/drivers/clk/samsung/clk-exynos5433.c
+++ b/drivers/clk/samsung/clk-exynos5433.c
@@ -543,7 +543,7 @@ static const struct samsung_div_clock top_div_clks[] __initconst = {
 static const struct samsung_gate_clock top_gate_clks[] __initconst = {
 	/* ENABLE_ACLK_TOP */
 	GATE(CLK_ACLK_G3D_400, "aclk_g3d_400", "div_aclk_g3d_400",
-			ENABLE_ACLK_TOP, 30, 0, 0),
+			ENABLE_ACLK_TOP, 30, CLK_IS_CRITICAL, 0),
 	GATE(CLK_ACLK_IMEM_SSX_266, "aclk_imem_ssx_266",
 			"div_aclk_imem_sssx_266", ENABLE_ACLK_TOP,
 			29, CLK_IGNORE_UNUSED, 0),
@@ -555,25 +555,25 @@ static const struct samsung_gate_clock top_gate_clks[] __initconst = {
 			CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0),
 	GATE(CLK_ACLK_IMEM_200, "aclk_imem_200", "div_aclk_imem_266",
 			ENABLE_ACLK_TOP, 24,
-			CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0),
+			CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, 0),
 	GATE(CLK_ACLK_IMEM_266, "aclk_imem_266", "div_aclk_imem_200",
 			ENABLE_ACLK_TOP, 23,
 			CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0),
 	GATE(CLK_ACLK_PERIC_66, "aclk_peric_66", "div_aclk_peric_66_b",
 			ENABLE_ACLK_TOP, 22,
-			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+			CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
 	GATE(CLK_ACLK_PERIS_66, "aclk_peris_66", "div_aclk_peris_66_b",
 			ENABLE_ACLK_TOP, 21,
-			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+			CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
 	GATE(CLK_ACLK_MSCL_400, "aclk_mscl_400", "div_aclk_mscl_400",
 			ENABLE_ACLK_TOP, 19,
-			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+			CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
 	GATE(CLK_ACLK_FSYS_200, "aclk_fsys_200", "div_aclk_fsys_200",
 			ENABLE_ACLK_TOP, 18,
-			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+			CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
 	GATE(CLK_ACLK_GSCL_111, "aclk_gscl_111", "div_aclk_gscl_111",
 			ENABLE_ACLK_TOP, 15,
-			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+			CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
 	GATE(CLK_ACLK_GSCL_333, "aclk_gscl_333", "div_aclk_gscl_333",
 			ENABLE_ACLK_TOP, 14,
 			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
@@ -582,7 +582,7 @@ static const struct samsung_gate_clock top_gate_clks[] __initconst = {
 			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
 	GATE(CLK_ACLK_CAM1_400, "aclk_cam1_400", "div_aclk_cam1_400",
 			ENABLE_ACLK_TOP, 12,
-			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+			CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
 	GATE(CLK_ACLK_CAM1_552, "aclk_cam1_552", "div_aclk_cam1_552",
 			ENABLE_ACLK_TOP, 11,
 			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
@@ -591,7 +591,7 @@ static const struct samsung_gate_clock top_gate_clks[] __initconst = {
 			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
 	GATE(CLK_ACLK_CAM0_400, "aclk_cam0_400", "div_aclk_cam0_400",
 			ENABLE_ACLK_TOP, 9,
-			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+			CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
 	GATE(CLK_ACLK_CAM0_552, "aclk_cam0_552", "div_aclk_cam0_552",
 			ENABLE_ACLK_TOP, 8,
 			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
@@ -600,19 +600,19 @@ static const struct samsung_gate_clock top_gate_clks[] __initconst = {
 			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
 	GATE(CLK_ACLK_ISP_400, "aclk_isp_400", "div_aclk_isp_400",
 			ENABLE_ACLK_TOP, 6,
-			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+			CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
 	GATE(CLK_ACLK_HEVC_400, "aclk_hevc_400", "div_aclk_hevc_400",
 			ENABLE_ACLK_TOP, 5,
-			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+			CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
 	GATE(CLK_ACLK_MFC_400, "aclk_mfc_400", "div_aclk_mfc_400",
 			ENABLE_ACLK_TOP, 3,
-			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+			CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
 	GATE(CLK_ACLK_G2D_266, "aclk_g2d_266", "div_aclk_g2d_266",
 			ENABLE_ACLK_TOP, 2,
 			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
 	GATE(CLK_ACLK_G2D_400, "aclk_g2d_400", "div_aclk_g2d_400",
 			ENABLE_ACLK_TOP, 0,
-			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
+			CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0),
 
 	/* ENABLE_SCLK_TOP_MSCL */
 	GATE(CLK_SCLK_JPEG_MSCL, "sclk_jpeg_mscl", "div_sclk_jpeg",
@@ -1385,7 +1385,7 @@ static const struct samsung_gate_clock mif_gate_clks[] __initconst = {
 			CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0),
 	GATE(CLK_ACLK_DISP_333, "aclk_disp_333", "div_aclk_disp_333",
 			ENABLE_ACLK_MIF3, 1,
-			CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0),
+			CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, 0),
 	GATE(CLK_ACLK_CPIF_200, "aclk_cpif_200", "div_aclk_cpif_200",
 			ENABLE_ACLK_MIF3, 0,
 			CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0),
@@ -1929,7 +1929,7 @@ CLK_OF_DECLARE(exynos5433_cmu_peris, "samsung,exynos5433-cmu-peris",
 
 /* list of all parent clock list */
 PNAME(mout_sclk_ufs_mphy_user_p)	= { "oscclk", "sclk_ufs_mphy", };
-PNAME(mout_aclk_fsys_200_user_p)	= { "oscclk", "div_aclk_fsys_200", };
+PNAME(mout_aclk_fsys_200_user_p)	= { "oscclk", "aclk_fsys_200", };
 PNAME(mout_sclk_pcie_100_user_p)	= { "oscclk", "sclk_pcie_100_fsys",};
 PNAME(mout_sclk_ufsunipro_user_p)	= { "oscclk", "sclk_ufsunipro_fsys",};
 PNAME(mout_sclk_mmc2_user_p)		= { "oscclk", "sclk_mmc2_fsys", };
diff --git a/drivers/clk/st/clk-flexgen.c b/drivers/clk/st/clk-flexgen.c
index a485f3b284b9..918ba3164da9 100644
--- a/drivers/clk/st/clk-flexgen.c
+++ b/drivers/clk/st/clk-flexgen.c
@@ -329,8 +329,10 @@ static void __init st_of_flexgen_setup(struct device_node *np)
 		return;
 
 	parents = flexgen_get_parents(np, &num_parents);
-	if (!parents)
+	if (!parents) {
+		iounmap(reg);
 		return;
+	}
 
 	match = of_match_node(flexgen_of_match, np);
 	if (match) {
@@ -394,6 +396,7 @@ static void __init st_of_flexgen_setup(struct device_node *np)
 	return;
 
 err:
+	iounmap(reg);
 	if (clk_data)
 		kfree(clk_data->clks);
 	kfree(clk_data);
diff --git a/drivers/clk/sunxi-ng/Kconfig b/drivers/clk/sunxi-ng/Kconfig
index 254d9526c018..8454c6e3dd65 100644
--- a/drivers/clk/sunxi-ng/Kconfig
+++ b/drivers/clk/sunxi-ng/Kconfig
@@ -35,17 +35,14 @@ config SUNXI_CCU_NK
 
 config SUNXI_CCU_NKM
 	bool
-	select RATIONAL
 	select SUNXI_CCU_GATE
 
 config SUNXI_CCU_NKMP
 	bool
-	select RATIONAL
 	select SUNXI_CCU_GATE
 
 config SUNXI_CCU_NM
 	bool
-	select RATIONAL
 	select SUNXI_CCU_FRAC
 	select SUNXI_CCU_GATE
 
@@ -56,6 +53,17 @@ config SUNXI_CCU_MP
 
 # SoC Drivers
 
+config SUN50I_A64_CCU
+	bool "Support for the Allwinner A64 CCU"
+	select SUNXI_CCU_DIV
+	select SUNXI_CCU_NK
+	select SUNXI_CCU_NKM
+	select SUNXI_CCU_NKMP
+	select SUNXI_CCU_NM
+	select SUNXI_CCU_MP
+	select SUNXI_CCU_PHASE
+	default ARM64 && ARCH_SUNXI
+
 config SUN6I_A31_CCU
 	bool "Support for the Allwinner A31/A31s CCU"
 	select SUNXI_CCU_DIV
diff --git a/drivers/clk/sunxi-ng/Makefile b/drivers/clk/sunxi-ng/Makefile
index 106cba27c331..24fbc6e5deb8 100644
--- a/drivers/clk/sunxi-ng/Makefile
+++ b/drivers/clk/sunxi-ng/Makefile
@@ -18,6 +18,7 @@ obj-$(CONFIG_SUNXI_CCU_NM)	+= ccu_nm.o
 obj-$(CONFIG_SUNXI_CCU_MP)	+= ccu_mp.o
 
 # SoC support
+obj-$(CONFIG_SUN50I_A64_CCU)	+= ccu-sun50i-a64.o
 obj-$(CONFIG_SUN6I_A31_CCU)	+= ccu-sun6i-a31.o
 obj-$(CONFIG_SUN8I_A23_CCU)	+= ccu-sun8i-a23.o
 obj-$(CONFIG_SUN8I_A33_CCU)	+= ccu-sun8i-a33.o
diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
new file mode 100644
index 000000000000..e3c084cc6da5
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
@@ -0,0 +1,915 @@
+/*
+ * Copyright (c) 2016 Maxime Ripard. All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+
+#include "ccu_common.h"
+#include "ccu_reset.h"
+
+#include "ccu_div.h"
+#include "ccu_gate.h"
+#include "ccu_mp.h"
+#include "ccu_mult.h"
+#include "ccu_nk.h"
+#include "ccu_nkm.h"
+#include "ccu_nkmp.h"
+#include "ccu_nm.h"
+#include "ccu_phase.h"
+
+#include "ccu-sun50i-a64.h"
+
+static struct ccu_nkmp pll_cpux_clk = {
+	.enable		= BIT(31),
+	.lock		= BIT(28),
+	.n		= _SUNXI_CCU_MULT(8, 5),
+	.k		= _SUNXI_CCU_MULT(4, 2),
+	.m		= _SUNXI_CCU_DIV(0, 2),
+	.p		= _SUNXI_CCU_DIV_MAX(16, 2, 4),
+	.common		= {
+		.reg		= 0x000,
+		.hw.init	= CLK_HW_INIT("pll-cpux",
+					      "osc24M",
+					      &ccu_nkmp_ops,
+					      CLK_SET_RATE_UNGATE),
+	},
+};
+
+/*
+ * The Audio PLL is supposed to have 4 outputs: 3 fixed factors from
+ * the base (2x, 4x and 8x), and one variable divider (the one true
+ * pll audio).
+ *
+ * We don't have any need for the variable divider for now, so we just
+ * hardcode it to match with the clock names
+ */
+#define SUN50I_A64_PLL_AUDIO_REG	0x008
+
+static SUNXI_CCU_NM_WITH_GATE_LOCK(pll_audio_base_clk, "pll-audio-base",
+				   "osc24M", 0x008,
+				   8, 7,	/* N */
+				   0, 5,	/* M */
+				   BIT(31),	/* gate */
+				   BIT(28),	/* lock */
+				   CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_video0_clk, "pll-video0",
+					"osc24M", 0x010,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_ve_clk, "pll-ve",
+					"osc24M", 0x018,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NKM_WITH_GATE_LOCK(pll_ddr0_clk, "pll-ddr0",
+				    "osc24M", 0x020,
+				    8, 5,	/* N */
+				    4, 2,	/* K */
+				    0, 2,	/* M */
+				    BIT(31),	/* gate */
+				    BIT(28),	/* lock */
+				    CLK_SET_RATE_UNGATE);
+
+static struct ccu_nk pll_periph0_clk = {
+	.enable		= BIT(31),
+	.lock		= BIT(28),
+	.n		= _SUNXI_CCU_MULT(8, 5),
+	.k		= _SUNXI_CCU_MULT_MIN(4, 2, 2),
+	.fixed_post_div	= 2,
+	.common		= {
+		.reg		= 0x028,
+		.features	= CCU_FEATURE_FIXED_POSTDIV,
+		.hw.init	= CLK_HW_INIT("pll-periph0", "osc24M",
+					      &ccu_nk_ops, CLK_SET_RATE_UNGATE),
+	},
+};
+
+static struct ccu_nk pll_periph1_clk = {
+	.enable		= BIT(31),
+	.lock		= BIT(28),
+	.n		= _SUNXI_CCU_MULT(8, 5),
+	.k		= _SUNXI_CCU_MULT_MIN(4, 2, 2),
+	.fixed_post_div	= 2,
+	.common		= {
+		.reg		= 0x02c,
+		.features	= CCU_FEATURE_FIXED_POSTDIV,
+		.hw.init	= CLK_HW_INIT("pll-periph1", "osc24M",
+					      &ccu_nk_ops, CLK_SET_RATE_UNGATE),
+	},
+};
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_video1_clk, "pll-video1",
+					"osc24M", 0x030,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_gpu_clk, "pll-gpu",
+					"osc24M", 0x038,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+/*
+ * The output function can be changed to something more complex that
+ * we do not handle yet.
+ *
+ * Hardcode the mode so that we don't fall in that case.
+ */
+#define SUN50I_A64_PLL_MIPI_REG		0x040
+
+static struct ccu_nkm pll_mipi_clk = {
+	.enable		= BIT(31),
+	.lock		= BIT(28),
+	.n		= _SUNXI_CCU_MULT(8, 4),
+	.k		= _SUNXI_CCU_MULT_MIN(4, 2, 2),
+	.m		= _SUNXI_CCU_DIV(0, 4),
+	.common		= {
+		.reg		= 0x040,
+		.hw.init	= CLK_HW_INIT("pll-mipi", "pll-video0",
+					      &ccu_nkm_ops, CLK_SET_RATE_UNGATE),
+	},
+};
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_hsic_clk, "pll-hsic",
+					"osc24M", 0x044,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_de_clk, "pll-de",
+					"osc24M", 0x048,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static SUNXI_CCU_NM_WITH_GATE_LOCK(pll_ddr1_clk, "pll-ddr1",
+				   "osc24M", 0x04c,
+				   8, 7,	/* N */
+				   0, 2,	/* M */
+				   BIT(31),	/* gate */
+				   BIT(28),	/* lock */
+				   CLK_SET_RATE_UNGATE);
+
+static const char * const cpux_parents[] = { "osc32k", "osc24M",
+					     "pll-cpux", "pll-cpux" };
+static SUNXI_CCU_MUX(cpux_clk, "cpux", cpux_parents,
+		     0x050, 16, 2, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL);
+
+static SUNXI_CCU_M(axi_clk, "axi", "cpux", 0x050, 0, 2, 0);
+
+static const char * const ahb1_parents[] = { "osc32k", "osc24M",
+					     "axi", "pll-periph0" };
+static struct ccu_div ahb1_clk = {
+	.div		= _SUNXI_CCU_DIV_FLAGS(4, 2, CLK_DIVIDER_POWER_OF_TWO),
+
+	.mux		= {
+		.shift	= 12,
+		.width	= 2,
+
+		.variable_prediv	= {
+			.index	= 3,
+			.shift	= 6,
+			.width	= 2,
+		},
+	},
+
+	.common		= {
+		.reg		= 0x054,
+		.features	= CCU_FEATURE_VARIABLE_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("ahb1",
+						      ahb1_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static struct clk_div_table apb1_div_table[] = {
+	{ .val = 0, .div = 2 },
+	{ .val = 1, .div = 2 },
+	{ .val = 2, .div = 4 },
+	{ .val = 3, .div = 8 },
+	{ /* Sentinel */ },
+};
+static SUNXI_CCU_DIV_TABLE(apb1_clk, "apb1", "ahb1",
+			   0x054, 8, 2, apb1_div_table, 0);
+
+static const char * const apb2_parents[] = { "osc32k", "osc24M",
+					     "pll-periph0-2x",
+					     "pll-periph0-2x" };
+static SUNXI_CCU_MP_WITH_MUX(apb2_clk, "apb2", apb2_parents, 0x058,
+			     0, 5,	/* M */
+			     16, 2,	/* P */
+			     24, 2,	/* mux */
+			     0);
+
+static const char * const ahb2_parents[] = { "ahb1", "pll-periph0" };
+static const struct ccu_mux_fixed_prediv ahb2_fixed_predivs[] = {
+	{ .index = 1, .div = 2 },
+};
+static struct ccu_mux ahb2_clk = {
+	.mux		= {
+		.shift	= 0,
+		.width	= 1,
+		.fixed_predivs	= ahb2_fixed_predivs,
+		.n_predivs	= ARRAY_SIZE(ahb2_fixed_predivs),
+	},
+
+	.common		= {
+		.reg		= 0x05c,
+		.features	= CCU_FEATURE_FIXED_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("ahb2",
+						      ahb2_parents,
+						      &ccu_mux_ops,
+						      0),
+	},
+};
+
+static SUNXI_CCU_GATE(bus_mipi_dsi_clk,	"bus-mipi-dsi",	"ahb1",
+		      0x060, BIT(1), 0);
+static SUNXI_CCU_GATE(bus_ce_clk,	"bus-ce",	"ahb1",
+		      0x060, BIT(5), 0);
+static SUNXI_CCU_GATE(bus_dma_clk,	"bus-dma",	"ahb1",
+		      0x060, BIT(6), 0);
+static SUNXI_CCU_GATE(bus_mmc0_clk,	"bus-mmc0",	"ahb1",
+		      0x060, BIT(8), 0);
+static SUNXI_CCU_GATE(bus_mmc1_clk,	"bus-mmc1",	"ahb1",
+		      0x060, BIT(9), 0);
+static SUNXI_CCU_GATE(bus_mmc2_clk,	"bus-mmc2",	"ahb1",
+		      0x060, BIT(10), 0);
+static SUNXI_CCU_GATE(bus_nand_clk,	"bus-nand",	"ahb1",
+		      0x060, BIT(13), 0);
+static SUNXI_CCU_GATE(bus_dram_clk,	"bus-dram",	"ahb1",
+		      0x060, BIT(14), 0);
+static SUNXI_CCU_GATE(bus_emac_clk,	"bus-emac",	"ahb2",
+		      0x060, BIT(17), 0);
+static SUNXI_CCU_GATE(bus_ts_clk,	"bus-ts",	"ahb1",
+		      0x060, BIT(18), 0);
+static SUNXI_CCU_GATE(bus_hstimer_clk,	"bus-hstimer",	"ahb1",
+		      0x060, BIT(19), 0);
+static SUNXI_CCU_GATE(bus_spi0_clk,	"bus-spi0",	"ahb1",
+		      0x060, BIT(20), 0);
+static SUNXI_CCU_GATE(bus_spi1_clk,	"bus-spi1",	"ahb1",
+		      0x060, BIT(21), 0);
+static SUNXI_CCU_GATE(bus_otg_clk,	"bus-otg",	"ahb1",
+		      0x060, BIT(23), 0);
+static SUNXI_CCU_GATE(bus_ehci0_clk,	"bus-ehci0",	"ahb1",
+		      0x060, BIT(24), 0);
+static SUNXI_CCU_GATE(bus_ehci1_clk,	"bus-ehci1",	"ahb2",
+		      0x060, BIT(25), 0);
+static SUNXI_CCU_GATE(bus_ohci0_clk,	"bus-ohci0",	"ahb1",
+		      0x060, BIT(28), 0);
+static SUNXI_CCU_GATE(bus_ohci1_clk,	"bus-ohci1",	"ahb2",
+		      0x060, BIT(29), 0);
+
+static SUNXI_CCU_GATE(bus_ve_clk,	"bus-ve",	"ahb1",
+		      0x064, BIT(0), 0);
+static SUNXI_CCU_GATE(bus_tcon0_clk,	"bus-tcon0",	"ahb1",
+		      0x064, BIT(3), 0);
+static SUNXI_CCU_GATE(bus_tcon1_clk,	"bus-tcon1",	"ahb1",
+		      0x064, BIT(4), 0);
+static SUNXI_CCU_GATE(bus_deinterlace_clk,	"bus-deinterlace",	"ahb1",
+		      0x064, BIT(5), 0);
+static SUNXI_CCU_GATE(bus_csi_clk,	"bus-csi",	"ahb1",
+		      0x064, BIT(8), 0);
+static SUNXI_CCU_GATE(bus_hdmi_clk,	"bus-hdmi",	"ahb1",
+		      0x064, BIT(11), 0);
+static SUNXI_CCU_GATE(bus_de_clk,	"bus-de",	"ahb1",
+		      0x064, BIT(12), 0);
+static SUNXI_CCU_GATE(bus_gpu_clk,	"bus-gpu",	"ahb1",
+		      0x064, BIT(20), 0);
+static SUNXI_CCU_GATE(bus_msgbox_clk,	"bus-msgbox",	"ahb1",
+		      0x064, BIT(21), 0);
+static SUNXI_CCU_GATE(bus_spinlock_clk,	"bus-spinlock",	"ahb1",
+		      0x064, BIT(22), 0);
+
+static SUNXI_CCU_GATE(bus_codec_clk,	"bus-codec",	"apb1",
+		      0x068, BIT(0), 0);
+static SUNXI_CCU_GATE(bus_spdif_clk,	"bus-spdif",	"apb1",
+		      0x068, BIT(1), 0);
+static SUNXI_CCU_GATE(bus_pio_clk,	"bus-pio",	"apb1",
+		      0x068, BIT(5), 0);
+static SUNXI_CCU_GATE(bus_ths_clk,	"bus-ths",	"apb1",
+		      0x068, BIT(8), 0);
+static SUNXI_CCU_GATE(bus_i2s0_clk,	"bus-i2s0",	"apb1",
+		      0x068, BIT(12), 0);
+static SUNXI_CCU_GATE(bus_i2s1_clk,	"bus-i2s1",	"apb1",
+		      0x068, BIT(13), 0);
+static SUNXI_CCU_GATE(bus_i2s2_clk,	"bus-i2s2",	"apb1",
+		      0x068, BIT(14), 0);
+
+static SUNXI_CCU_GATE(bus_i2c0_clk,	"bus-i2c0",	"apb2",
+		      0x06c, BIT(0), 0);
+static SUNXI_CCU_GATE(bus_i2c1_clk,	"bus-i2c1",	"apb2",
+		      0x06c, BIT(1), 0);
+static SUNXI_CCU_GATE(bus_i2c2_clk,	"bus-i2c2",	"apb2",
+		      0x06c, BIT(2), 0);
+static SUNXI_CCU_GATE(bus_scr_clk,	"bus-scr",	"apb2",
+		      0x06c, BIT(5), 0);
+static SUNXI_CCU_GATE(bus_uart0_clk,	"bus-uart0",	"apb2",
+		      0x06c, BIT(16), 0);
+static SUNXI_CCU_GATE(bus_uart1_clk,	"bus-uart1",	"apb2",
+		      0x06c, BIT(17), 0);
+static SUNXI_CCU_GATE(bus_uart2_clk,	"bus-uart2",	"apb2",
+		      0x06c, BIT(18), 0);
+static SUNXI_CCU_GATE(bus_uart3_clk,	"bus-uart3",	"apb2",
+		      0x06c, BIT(19), 0);
+static SUNXI_CCU_GATE(bus_uart4_clk,	"bus-uart4",	"apb2",
+		      0x06c, BIT(20), 0);
+
+static SUNXI_CCU_GATE(bus_dbg_clk,	"bus-dbg",	"ahb1",
+		      0x070, BIT(7), 0);
+
+static struct clk_div_table ths_div_table[] = {
+	{ .val = 0, .div = 1 },
+	{ .val = 1, .div = 2 },
+	{ .val = 2, .div = 4 },
+	{ .val = 3, .div = 6 },
+};
+static const char * const ths_parents[] = { "osc24M" };
+static struct ccu_div ths_clk = {
+	.enable	= BIT(31),
+	.div	= _SUNXI_CCU_DIV_TABLE(0, 2, ths_div_table),
+	.mux	= _SUNXI_CCU_MUX(24, 2),
+	.common	= {
+		.reg		= 0x074,
+		.hw.init	= CLK_HW_INIT_PARENTS("ths",
+						      ths_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static const char * const mod0_default_parents[] = { "osc24M", "pll-periph0",
+						     "pll-periph1" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(nand_clk, "nand", mod0_default_parents, 0x080,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static const char * const mmc_default_parents[] = { "osc24M", "pll-periph0-2x",
+						    "pll-periph1-2x" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc0_clk, "mmc0", mmc_default_parents, 0x088,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc1_clk, "mmc1", mmc_default_parents, 0x08c,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc2_clk, "mmc2", mmc_default_parents, 0x090,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static const char * const ts_parents[] = { "osc24M", "pll-periph0", };
+static SUNXI_CCU_MP_WITH_MUX_GATE(ts_clk, "ts", ts_parents, 0x098,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 4,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(ce_clk, "ce", mmc_default_parents, 0x09c,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi0_clk, "spi0", mod0_default_parents, 0x0a0,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", mod0_default_parents, 0x0a4,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static const char * const i2s_parents[] = { "pll-audio-8x", "pll-audio-4x",
+					    "pll-audio-2x", "pll-audio" };
+static SUNXI_CCU_MUX_WITH_GATE(i2s0_clk, "i2s0", i2s_parents,
+			       0x0b0, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_MUX_WITH_GATE(i2s1_clk, "i2s1", i2s_parents,
+			       0x0b4, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_MUX_WITH_GATE(i2s2_clk, "i2s2", i2s_parents,
+			       0x0b8, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_M_WITH_GATE(spdif_clk, "spdif", "pll-audio",
+			     0x0c0, 0, 4, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_GATE(usb_phy0_clk,	"usb-phy0",	"osc24M",
+		      0x0cc, BIT(8), 0);
+static SUNXI_CCU_GATE(usb_phy1_clk,	"usb-phy1",	"osc24M",
+		      0x0cc, BIT(9), 0);
+static SUNXI_CCU_GATE(usb_hsic_clk,	"usb-hsic",	"pll-hsic",
+		      0x0cc, BIT(10), 0);
+static SUNXI_CCU_GATE(usb_hsic_12m_clk,	"usb-hsic-12M",	"osc12M",
+		      0x0cc, BIT(11), 0);
+static SUNXI_CCU_GATE(usb_ohci0_clk,	"usb-ohci0",	"osc12M",
+		      0x0cc, BIT(16), 0);
+static SUNXI_CCU_GATE(usb_ohci1_clk,	"usb-ohci1",	"usb-ohci0",
+		      0x0cc, BIT(17), 0);
+
+static const char * const dram_parents[] = { "pll-ddr0", "pll-ddr1" };
+static SUNXI_CCU_M_WITH_MUX(dram_clk, "dram", dram_parents,
+			    0x0f4, 0, 4, 20, 2, CLK_IS_CRITICAL);
+
+static SUNXI_CCU_GATE(dram_ve_clk,	"dram-ve",	"dram",
+		      0x100, BIT(0), 0);
+static SUNXI_CCU_GATE(dram_csi_clk,	"dram-csi",	"dram",
+		      0x100, BIT(1), 0);
+static SUNXI_CCU_GATE(dram_deinterlace_clk,	"dram-deinterlace",	"dram",
+		      0x100, BIT(2), 0);
+static SUNXI_CCU_GATE(dram_ts_clk,	"dram-ts",	"dram",
+		      0x100, BIT(3), 0);
+
+static const char * const de_parents[] = { "pll-periph0-2x", "pll-de" };
+static SUNXI_CCU_M_WITH_MUX_GATE(de_clk, "de", de_parents,
+				 0x104, 0, 4, 24, 3, BIT(31), 0);
+
+static const char * const tcon0_parents[] = { "pll-mipi", "pll-video0-2x" };
+static const u8 tcon0_table[] = { 0, 2, };
+static SUNXI_CCU_MUX_TABLE_WITH_GATE(tcon0_clk, "tcon0", tcon0_parents,
+				     tcon0_table, 0x118, 24, 3, BIT(31),
+				     CLK_SET_RATE_PARENT);
+
+static const char * const tcon1_parents[] = { "pll-video0", "pll-video1" };
+static const u8 tcon1_table[] = { 0, 2, };
+static struct ccu_div tcon1_clk = {
+	.enable		= BIT(31),
+	.div		= _SUNXI_CCU_DIV(0, 4),
+	.mux		= _SUNXI_CCU_MUX_TABLE(24, 2, tcon1_table),
+	.common		= {
+		.reg		= 0x11c,
+		.hw.init	= CLK_HW_INIT_PARENTS("tcon1",
+						      tcon1_parents,
+						      &ccu_div_ops,
+						      CLK_SET_RATE_PARENT),
+	},
+};
+
+static const char * const deinterlace_parents[] = { "pll-periph0", "pll-periph1" };
+static SUNXI_CCU_M_WITH_MUX_GATE(deinterlace_clk, "deinterlace", deinterlace_parents,
+				 0x124, 0, 4, 24, 3, BIT(31), 0);
+
+static SUNXI_CCU_GATE(csi_misc_clk,	"csi-misc",	"osc24M",
+		      0x130, BIT(31), 0);
+
+static const char * const csi_sclk_parents[] = { "pll-periph0", "pll-periph1" };
+static SUNXI_CCU_M_WITH_MUX_GATE(csi_sclk_clk, "csi-sclk", csi_sclk_parents,
+				 0x134, 16, 4, 24, 3, BIT(31), 0);
+
+static const char * const csi_mclk_parents[] = { "osc24M", "pll-video1", "pll-periph1" };
+static SUNXI_CCU_M_WITH_MUX_GATE(csi_mclk_clk, "csi-mclk", csi_mclk_parents,
+				 0x134, 0, 5, 8, 3, BIT(15), 0);
+
+static SUNXI_CCU_M_WITH_GATE(ve_clk, "ve", "pll-ve",
+			     0x13c, 16, 3, BIT(31), 0);
+
+static SUNXI_CCU_GATE(ac_dig_clk,	"ac-dig",	"pll-audio",
+		      0x140, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_GATE(ac_dig_4x_clk,	"ac-dig-4x",	"pll-audio-4x",
+		      0x140, BIT(30), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_GATE(avs_clk,		"avs",		"osc24M",
+		      0x144, BIT(31), 0);
+
+static const char * const hdmi_parents[] = { "pll-video0", "pll-video1" };
+static SUNXI_CCU_M_WITH_MUX_GATE(hdmi_clk, "hdmi", hdmi_parents,
+				 0x150, 0, 4, 24, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_GATE(hdmi_ddc_clk,	"hdmi-ddc",	"osc24M",
+		      0x154, BIT(31), 0);
+
+static const char * const mbus_parents[] = { "osc24M", "pll-periph0-2x",
+						 "pll-ddr0", "pll-ddr1" };
+static SUNXI_CCU_M_WITH_MUX_GATE(mbus_clk, "mbus", mbus_parents,
+				 0x15c, 0, 3, 24, 2, BIT(31), CLK_IS_CRITICAL);
+
+static const char * const dsi_dphy_parents[] = { "pll-video0", "pll-periph0" };
+static const u8 dsi_dphy_table[] = { 0, 2, };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(dsi_dphy_clk, "dsi-dphy",
+				       dsi_dphy_parents, dsi_dphy_table,
+				       0x168, 0, 4, 8, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_M_WITH_GATE(gpu_clk, "gpu", "pll-gpu",
+			     0x1a0, 0, 3, BIT(31), CLK_SET_RATE_PARENT);
+
+/* Fixed Factor clocks */
+static CLK_FIXED_FACTOR(osc12M_clk, "osc12M", "osc24M", 1, 2, 0);
+
+/* We hardcode the divider to 4 for now */
+static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
+			"pll-audio-base", 4, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
+			"pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
+			"pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
+			"pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x",
+			"pll-periph0", 1, 2, 0);
+static CLK_FIXED_FACTOR(pll_periph1_2x_clk, "pll-periph1-2x",
+			"pll-periph1", 1, 2, 0);
+static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x",
+			"pll-video0", 1, 2, CLK_SET_RATE_PARENT);
+
+static struct ccu_common *sun50i_a64_ccu_clks[] = {
+	&pll_cpux_clk.common,
+	&pll_audio_base_clk.common,
+	&pll_video0_clk.common,
+	&pll_ve_clk.common,
+	&pll_ddr0_clk.common,
+	&pll_periph0_clk.common,
+	&pll_periph1_clk.common,
+	&pll_video1_clk.common,
+	&pll_gpu_clk.common,
+	&pll_mipi_clk.common,
+	&pll_hsic_clk.common,
+	&pll_de_clk.common,
+	&pll_ddr1_clk.common,
+	&cpux_clk.common,
+	&axi_clk.common,
+	&ahb1_clk.common,
+	&apb1_clk.common,
+	&apb2_clk.common,
+	&ahb2_clk.common,
+	&bus_mipi_dsi_clk.common,
+	&bus_ce_clk.common,
+	&bus_dma_clk.common,
+	&bus_mmc0_clk.common,
+	&bus_mmc1_clk.common,
+	&bus_mmc2_clk.common,
+	&bus_nand_clk.common,
+	&bus_dram_clk.common,
+	&bus_emac_clk.common,
+	&bus_ts_clk.common,
+	&bus_hstimer_clk.common,
+	&bus_spi0_clk.common,
+	&bus_spi1_clk.common,
+	&bus_otg_clk.common,
+	&bus_ehci0_clk.common,
+	&bus_ehci1_clk.common,
+	&bus_ohci0_clk.common,
+	&bus_ohci1_clk.common,
+	&bus_ve_clk.common,
+	&bus_tcon0_clk.common,
+	&bus_tcon1_clk.common,
+	&bus_deinterlace_clk.common,
+	&bus_csi_clk.common,
+	&bus_hdmi_clk.common,
+	&bus_de_clk.common,
+	&bus_gpu_clk.common,
+	&bus_msgbox_clk.common,
+	&bus_spinlock_clk.common,
+	&bus_codec_clk.common,
+	&bus_spdif_clk.common,
+	&bus_pio_clk.common,
+	&bus_ths_clk.common,
+	&bus_i2s0_clk.common,
+	&bus_i2s1_clk.common,
+	&bus_i2s2_clk.common,
+	&bus_i2c0_clk.common,
+	&bus_i2c1_clk.common,
+	&bus_i2c2_clk.common,
+	&bus_scr_clk.common,
+	&bus_uart0_clk.common,
+	&bus_uart1_clk.common,
+	&bus_uart2_clk.common,
+	&bus_uart3_clk.common,
+	&bus_uart4_clk.common,
+	&bus_dbg_clk.common,
+	&ths_clk.common,
+	&nand_clk.common,
+	&mmc0_clk.common,
+	&mmc1_clk.common,
+	&mmc2_clk.common,
+	&ts_clk.common,
+	&ce_clk.common,
+	&spi0_clk.common,
+	&spi1_clk.common,
+	&i2s0_clk.common,
+	&i2s1_clk.common,
+	&i2s2_clk.common,
+	&spdif_clk.common,
+	&usb_phy0_clk.common,
+	&usb_phy1_clk.common,
+	&usb_hsic_clk.common,
+	&usb_hsic_12m_clk.common,
+	&usb_ohci0_clk.common,
+	&usb_ohci1_clk.common,
+	&dram_clk.common,
+	&dram_ve_clk.common,
+	&dram_csi_clk.common,
+	&dram_deinterlace_clk.common,
+	&dram_ts_clk.common,
+	&de_clk.common,
+	&tcon0_clk.common,
+	&tcon1_clk.common,
+	&deinterlace_clk.common,
+	&csi_misc_clk.common,
+	&csi_sclk_clk.common,
+	&csi_mclk_clk.common,
+	&ve_clk.common,
+	&ac_dig_clk.common,
+	&ac_dig_4x_clk.common,
+	&avs_clk.common,
+	&hdmi_clk.common,
+	&hdmi_ddc_clk.common,
+	&mbus_clk.common,
+	&dsi_dphy_clk.common,
+	&gpu_clk.common,
+};
+
+static struct clk_hw_onecell_data sun50i_a64_hw_clks = {
+	.hws	= {
+		[CLK_OSC_12M]		= &osc12M_clk.hw,
+		[CLK_PLL_CPUX]		= &pll_cpux_clk.common.hw,
+		[CLK_PLL_AUDIO_BASE]	= &pll_audio_base_clk.common.hw,
+		[CLK_PLL_AUDIO]		= &pll_audio_clk.hw,
+		[CLK_PLL_AUDIO_2X]	= &pll_audio_2x_clk.hw,
+		[CLK_PLL_AUDIO_4X]	= &pll_audio_4x_clk.hw,
+		[CLK_PLL_AUDIO_8X]	= &pll_audio_8x_clk.hw,
+		[CLK_PLL_VIDEO0]	= &pll_video0_clk.common.hw,
+		[CLK_PLL_VIDEO0_2X]	= &pll_video0_2x_clk.hw,
+		[CLK_PLL_VE]		= &pll_ve_clk.common.hw,
+		[CLK_PLL_DDR0]		= &pll_ddr0_clk.common.hw,
+		[CLK_PLL_PERIPH0]	= &pll_periph0_clk.common.hw,
+		[CLK_PLL_PERIPH0_2X]	= &pll_periph0_2x_clk.hw,
+		[CLK_PLL_PERIPH1]	= &pll_periph1_clk.common.hw,
+		[CLK_PLL_PERIPH1_2X]	= &pll_periph1_2x_clk.hw,
+		[CLK_PLL_VIDEO1]	= &pll_video1_clk.common.hw,
+		[CLK_PLL_GPU]		= &pll_gpu_clk.common.hw,
+		[CLK_PLL_MIPI]  	= &pll_mipi_clk.common.hw,
+		[CLK_PLL_HSIC]		= &pll_hsic_clk.common.hw,
+		[CLK_PLL_DE]		= &pll_de_clk.common.hw,
+		[CLK_PLL_DDR1]		= &pll_ddr1_clk.common.hw,
+		[CLK_CPUX]		= &cpux_clk.common.hw,
+		[CLK_AXI]		= &axi_clk.common.hw,
+		[CLK_AHB1]		= &ahb1_clk.common.hw,
+		[CLK_APB1]		= &apb1_clk.common.hw,
+		[CLK_APB2]		= &apb2_clk.common.hw,
+		[CLK_AHB2]		= &ahb2_clk.common.hw,
+		[CLK_BUS_MIPI_DSI]	= &bus_mipi_dsi_clk.common.hw,
+		[CLK_BUS_CE]		= &bus_ce_clk.common.hw,
+		[CLK_BUS_DMA]		= &bus_dma_clk.common.hw,
+		[CLK_BUS_MMC0]		= &bus_mmc0_clk.common.hw,
+		[CLK_BUS_MMC1]		= &bus_mmc1_clk.common.hw,
+		[CLK_BUS_MMC2]		= &bus_mmc2_clk.common.hw,
+		[CLK_BUS_NAND]		= &bus_nand_clk.common.hw,
+		[CLK_BUS_DRAM]		= &bus_dram_clk.common.hw,
+		[CLK_BUS_EMAC]		= &bus_emac_clk.common.hw,
+		[CLK_BUS_TS]		= &bus_ts_clk.common.hw,
+		[CLK_BUS_HSTIMER]	= &bus_hstimer_clk.common.hw,
+		[CLK_BUS_SPI0]		= &bus_spi0_clk.common.hw,
+		[CLK_BUS_SPI1]		= &bus_spi1_clk.common.hw,
+		[CLK_BUS_OTG]		= &bus_otg_clk.common.hw,
+		[CLK_BUS_EHCI0]		= &bus_ehci0_clk.common.hw,
+		[CLK_BUS_EHCI1]		= &bus_ehci1_clk.common.hw,
+		[CLK_BUS_OHCI0]		= &bus_ohci0_clk.common.hw,
+		[CLK_BUS_OHCI1]		= &bus_ohci1_clk.common.hw,
+		[CLK_BUS_VE]		= &bus_ve_clk.common.hw,
+		[CLK_BUS_TCON0]		= &bus_tcon0_clk.common.hw,
+		[CLK_BUS_TCON1]		= &bus_tcon1_clk.common.hw,
+		[CLK_BUS_DEINTERLACE]	= &bus_deinterlace_clk.common.hw,
+		[CLK_BUS_CSI]		= &bus_csi_clk.common.hw,
+		[CLK_BUS_HDMI]		= &bus_hdmi_clk.common.hw,
+		[CLK_BUS_DE]		= &bus_de_clk.common.hw,
+		[CLK_BUS_GPU]		= &bus_gpu_clk.common.hw,
+		[CLK_BUS_MSGBOX]	= &bus_msgbox_clk.common.hw,
+		[CLK_BUS_SPINLOCK]	= &bus_spinlock_clk.common.hw,
+		[CLK_BUS_CODEC]		= &bus_codec_clk.common.hw,
+		[CLK_BUS_SPDIF]		= &bus_spdif_clk.common.hw,
+		[CLK_BUS_PIO]		= &bus_pio_clk.common.hw,
+		[CLK_BUS_THS]		= &bus_ths_clk.common.hw,
+		[CLK_BUS_I2S0]		= &bus_i2s0_clk.common.hw,
+		[CLK_BUS_I2S1]		= &bus_i2s1_clk.common.hw,
+		[CLK_BUS_I2S2]		= &bus_i2s2_clk.common.hw,
+		[CLK_BUS_I2C0]		= &bus_i2c0_clk.common.hw,
+		[CLK_BUS_I2C1]		= &bus_i2c1_clk.common.hw,
+		[CLK_BUS_I2C2]		= &bus_i2c2_clk.common.hw,
+		[CLK_BUS_UART0]		= &bus_uart0_clk.common.hw,
+		[CLK_BUS_UART1]		= &bus_uart1_clk.common.hw,
+		[CLK_BUS_UART2]		= &bus_uart2_clk.common.hw,
+		[CLK_BUS_UART3]		= &bus_uart3_clk.common.hw,
+		[CLK_BUS_UART4]		= &bus_uart4_clk.common.hw,
+		[CLK_BUS_SCR]		= &bus_scr_clk.common.hw,
+		[CLK_BUS_DBG]		= &bus_dbg_clk.common.hw,
+		[CLK_THS]		= &ths_clk.common.hw,
+		[CLK_NAND]		= &nand_clk.common.hw,
+		[CLK_MMC0]		= &mmc0_clk.common.hw,
+		[CLK_MMC1]		= &mmc1_clk.common.hw,
+		[CLK_MMC2]		= &mmc2_clk.common.hw,
+		[CLK_TS]		= &ts_clk.common.hw,
+		[CLK_CE]		= &ce_clk.common.hw,
+		[CLK_SPI0]		= &spi0_clk.common.hw,
+		[CLK_SPI1]		= &spi1_clk.common.hw,
+		[CLK_I2S0]		= &i2s0_clk.common.hw,
+		[CLK_I2S1]		= &i2s1_clk.common.hw,
+		[CLK_I2S2]		= &i2s2_clk.common.hw,
+		[CLK_SPDIF]		= &spdif_clk.common.hw,
+		[CLK_USB_PHY0]		= &usb_phy0_clk.common.hw,
+		[CLK_USB_PHY1]		= &usb_phy1_clk.common.hw,
+		[CLK_USB_HSIC]		= &usb_hsic_clk.common.hw,
+		[CLK_USB_HSIC_12M]	= &usb_hsic_12m_clk.common.hw,
+		[CLK_USB_OHCI0]		= &usb_ohci0_clk.common.hw,
+		[CLK_USB_OHCI1]		= &usb_ohci1_clk.common.hw,
+		[CLK_DRAM]		= &dram_clk.common.hw,
+		[CLK_DRAM_VE]		= &dram_ve_clk.common.hw,
+		[CLK_DRAM_CSI]		= &dram_csi_clk.common.hw,
+		[CLK_DRAM_DEINTERLACE]	= &dram_deinterlace_clk.common.hw,
+		[CLK_DRAM_TS]		= &dram_ts_clk.common.hw,
+		[CLK_DE]		= &de_clk.common.hw,
+		[CLK_TCON0]		= &tcon0_clk.common.hw,
+		[CLK_TCON1]		= &tcon1_clk.common.hw,
+		[CLK_DEINTERLACE]	= &deinterlace_clk.common.hw,
+		[CLK_CSI_MISC]		= &csi_misc_clk.common.hw,
+		[CLK_CSI_SCLK]		= &csi_sclk_clk.common.hw,
+		[CLK_CSI_MCLK]		= &csi_mclk_clk.common.hw,
+		[CLK_VE]		= &ve_clk.common.hw,
+		[CLK_AC_DIG]		= &ac_dig_clk.common.hw,
+		[CLK_AC_DIG_4X]		= &ac_dig_4x_clk.common.hw,
+		[CLK_AVS]		= &avs_clk.common.hw,
+		[CLK_HDMI]		= &hdmi_clk.common.hw,
+		[CLK_HDMI_DDC]		= &hdmi_ddc_clk.common.hw,
+		[CLK_MBUS]		= &mbus_clk.common.hw,
+		[CLK_DSI_DPHY]		= &dsi_dphy_clk.common.hw,
+		[CLK_GPU]		= &gpu_clk.common.hw,
+	},
+	.num	= CLK_NUMBER,
+};
+
+static struct ccu_reset_map sun50i_a64_ccu_resets[] = {
+	[RST_USB_PHY0]		=  { 0x0cc, BIT(0) },
+	[RST_USB_PHY1]		=  { 0x0cc, BIT(1) },
+	[RST_USB_HSIC]		=  { 0x0cc, BIT(2) },
+
+	[RST_DRAM]		=  { 0x0f4, BIT(31) },
+	[RST_MBUS]		=  { 0x0fc, BIT(31) },
+
+	[RST_BUS_MIPI_DSI]	=  { 0x2c0, BIT(1) },
+	[RST_BUS_CE]		=  { 0x2c0, BIT(5) },
+	[RST_BUS_DMA]		=  { 0x2c0, BIT(6) },
+	[RST_BUS_MMC0]		=  { 0x2c0, BIT(8) },
+	[RST_BUS_MMC1]		=  { 0x2c0, BIT(9) },
+	[RST_BUS_MMC2]		=  { 0x2c0, BIT(10) },
+	[RST_BUS_NAND]		=  { 0x2c0, BIT(13) },
+	[RST_BUS_DRAM]		=  { 0x2c0, BIT(14) },
+	[RST_BUS_EMAC]		=  { 0x2c0, BIT(17) },
+	[RST_BUS_TS]		=  { 0x2c0, BIT(18) },
+	[RST_BUS_HSTIMER]	=  { 0x2c0, BIT(19) },
+	[RST_BUS_SPI0]		=  { 0x2c0, BIT(20) },
+	[RST_BUS_SPI1]		=  { 0x2c0, BIT(21) },
+	[RST_BUS_OTG]		=  { 0x2c0, BIT(23) },
+	[RST_BUS_EHCI0]		=  { 0x2c0, BIT(24) },
+	[RST_BUS_EHCI1]		=  { 0x2c0, BIT(25) },
+	[RST_BUS_OHCI0]		=  { 0x2c0, BIT(28) },
+	[RST_BUS_OHCI1]		=  { 0x2c0, BIT(29) },
+
+	[RST_BUS_VE]		=  { 0x2c4, BIT(0) },
+	[RST_BUS_TCON0]		=  { 0x2c4, BIT(3) },
+	[RST_BUS_TCON1]		=  { 0x2c4, BIT(4) },
+	[RST_BUS_DEINTERLACE]	=  { 0x2c4, BIT(5) },
+	[RST_BUS_CSI]		=  { 0x2c4, BIT(8) },
+	[RST_BUS_HDMI0]		=  { 0x2c4, BIT(10) },
+	[RST_BUS_HDMI1]		=  { 0x2c4, BIT(11) },
+	[RST_BUS_DE]		=  { 0x2c4, BIT(12) },
+	[RST_BUS_GPU]		=  { 0x2c4, BIT(20) },
+	[RST_BUS_MSGBOX]	=  { 0x2c4, BIT(21) },
+	[RST_BUS_SPINLOCK]	=  { 0x2c4, BIT(22) },
+	[RST_BUS_DBG]		=  { 0x2c4, BIT(31) },
+
+	[RST_BUS_LVDS]		=  { 0x2c8, BIT(0) },
+
+	[RST_BUS_CODEC]		=  { 0x2d0, BIT(0) },
+	[RST_BUS_SPDIF]		=  { 0x2d0, BIT(1) },
+	[RST_BUS_THS]		=  { 0x2d0, BIT(8) },
+	[RST_BUS_I2S0]		=  { 0x2d0, BIT(12) },
+	[RST_BUS_I2S1]		=  { 0x2d0, BIT(13) },
+	[RST_BUS_I2S2]		=  { 0x2d0, BIT(14) },
+
+	[RST_BUS_I2C0]		=  { 0x2d8, BIT(0) },
+	[RST_BUS_I2C1]		=  { 0x2d8, BIT(1) },
+	[RST_BUS_I2C2]		=  { 0x2d8, BIT(2) },
+	[RST_BUS_SCR]		=  { 0x2d8, BIT(5) },
+	[RST_BUS_UART0]		=  { 0x2d8, BIT(16) },
+	[RST_BUS_UART1]		=  { 0x2d8, BIT(17) },
+	[RST_BUS_UART2]		=  { 0x2d8, BIT(18) },
+	[RST_BUS_UART3]		=  { 0x2d8, BIT(19) },
+	[RST_BUS_UART4]		=  { 0x2d8, BIT(20) },
+};
+
+static const struct sunxi_ccu_desc sun50i_a64_ccu_desc = {
+	.ccu_clks	= sun50i_a64_ccu_clks,
+	.num_ccu_clks	= ARRAY_SIZE(sun50i_a64_ccu_clks),
+
+	.hw_clks	= &sun50i_a64_hw_clks,
+
+	.resets		= sun50i_a64_ccu_resets,
+	.num_resets	= ARRAY_SIZE(sun50i_a64_ccu_resets),
+};
+
+static int sun50i_a64_ccu_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	void __iomem *reg;
+	u32 val;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	reg = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(reg))
+		return PTR_ERR(reg);
+
+	/* Force the PLL-Audio-1x divider to 4 */
+	val = readl(reg + SUN50I_A64_PLL_AUDIO_REG);
+	val &= ~GENMASK(19, 16);
+	writel(val | (3 << 16), reg + SUN50I_A64_PLL_AUDIO_REG);
+
+	writel(0x515, reg + SUN50I_A64_PLL_MIPI_REG);
+
+	return sunxi_ccu_probe(pdev->dev.of_node, reg, &sun50i_a64_ccu_desc);
+}
+
+static const struct of_device_id sun50i_a64_ccu_ids[] = {
+	{ .compatible = "allwinner,sun50i-a64-ccu" },
+	{ }
+};
+
+static struct platform_driver sun50i_a64_ccu_driver = {
+	.probe	= sun50i_a64_ccu_probe,
+	.driver	= {
+		.name	= "sun50i-a64-ccu",
+		.of_match_table	= sun50i_a64_ccu_ids,
+	},
+};
+builtin_platform_driver(sun50i_a64_ccu_driver);
diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.h b/drivers/clk/sunxi-ng/ccu-sun50i-a64.h
new file mode 100644
index 000000000000..9b3cd24b78d2
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2016 Maxime Ripard
+ *
+ * Maxime Ripard <maxime.ripard@free-electrons.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _CCU_SUN50I_A64_H_
+#define _CCU_SUN50I_A64_H_
+
+#include <dt-bindings/clock/sun50i-a64-ccu.h>
+#include <dt-bindings/reset/sun50i-a64-ccu.h>
+
+#define CLK_OSC_12M			0
+#define CLK_PLL_CPUX			1
+#define CLK_PLL_AUDIO_BASE		2
+#define CLK_PLL_AUDIO			3
+#define CLK_PLL_AUDIO_2X		4
+#define CLK_PLL_AUDIO_4X		5
+#define CLK_PLL_AUDIO_8X		6
+#define CLK_PLL_VIDEO0			7
+#define CLK_PLL_VIDEO0_2X		8
+#define CLK_PLL_VE			9
+#define CLK_PLL_DDR0			10
+#define CLK_PLL_PERIPH0			11
+#define CLK_PLL_PERIPH0_2X		12
+#define CLK_PLL_PERIPH1			13
+#define CLK_PLL_PERIPH1_2X		14
+#define CLK_PLL_VIDEO1			15
+#define CLK_PLL_GPU			16
+#define CLK_PLL_MIPI			17
+#define CLK_PLL_HSIC			18
+#define CLK_PLL_DE			19
+#define CLK_PLL_DDR1			20
+#define CLK_CPUX			21
+#define CLK_AXI				22
+#define CLK_APB				23
+#define CLK_AHB1			24
+#define CLK_APB1			25
+#define CLK_APB2			26
+#define CLK_AHB2			27
+
+/* All the bus gates are exported */
+
+/* The first bunch of module clocks are exported */
+
+#define CLK_USB_OHCI0_12M		90
+
+#define CLK_USB_OHCI1_12M		92
+
+#define CLK_DRAM			94
+
+/* All the DRAM gates are exported */
+
+/* Some more module clocks are exported */
+
+#define CLK_MBUS			112
+
+/* And the DSI and GPU module clock is exported */
+
+#define CLK_NUMBER			(CLK_GPU + 1)
+
+#endif /* _CCU_SUN50I_A64_H_ */
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
index 2646d980087b..5c6d37bdf247 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
@@ -344,10 +344,10 @@ static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", mod0_default_parents, 0x0a4,
 static const char * const i2s_parents[] = { "pll-audio-8x", "pll-audio-4x",
 					    "pll-audio-2x", "pll-audio" };
 static SUNXI_CCU_MUX_WITH_GATE(i2s0_clk, "i2s0", i2s_parents,
-			       0x0b0, 16, 2, BIT(31), 0);
+			       0x0b0, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
 
 static SUNXI_CCU_MUX_WITH_GATE(i2s1_clk, "i2s1", i2s_parents,
-			       0x0b4, 16, 2, BIT(31), 0);
+			       0x0b4, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
 
 /* TODO: the parent for most of the USB clocks is not known */
 static SUNXI_CCU_GATE(usb_phy0_clk,	"usb-phy0",	"osc24M",
@@ -415,7 +415,7 @@ static SUNXI_CCU_M_WITH_GATE(ve_clk, "ve", "pll-ve",
 			     0x13c, 16, 3, BIT(31), CLK_SET_RATE_PARENT);
 
 static SUNXI_CCU_GATE(ac_dig_clk,	"ac-dig",	"pll-audio",
-		      0x140, BIT(31), 0);
+		      0x140, BIT(31), CLK_SET_RATE_PARENT);
 static SUNXI_CCU_GATE(avs_clk,		"avs",		"osc24M",
 		      0x144, BIT(31), 0);
 
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
index 4d70590f05e3..21c427d86f28 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
@@ -394,16 +394,16 @@ static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", mod0_default_parents, 0x0a4,
 static const char * const i2s_parents[] = { "pll-audio-8x", "pll-audio-4x",
 					    "pll-audio-2x", "pll-audio" };
 static SUNXI_CCU_MUX_WITH_GATE(i2s0_clk, "i2s0", i2s_parents,
-			       0x0b0, 16, 2, BIT(31), 0);
+			       0x0b0, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
 
 static SUNXI_CCU_MUX_WITH_GATE(i2s1_clk, "i2s1", i2s_parents,
-			       0x0b4, 16, 2, BIT(31), 0);
+			       0x0b4, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
 
 static SUNXI_CCU_MUX_WITH_GATE(i2s2_clk, "i2s2", i2s_parents,
-			       0x0b8, 16, 2, BIT(31), 0);
+			       0x0b8, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
 
 static SUNXI_CCU_M_WITH_GATE(spdif_clk, "spdif", "pll-audio",
-			     0x0c0, 0, 4, BIT(31), 0);
+			     0x0c0, 0, 4, BIT(31), CLK_SET_RATE_PARENT);
 
 static SUNXI_CCU_GATE(usb_phy0_clk,	"usb-phy0",	"osc24M",
 		      0x0cc, BIT(8), 0);
@@ -466,7 +466,7 @@ static SUNXI_CCU_M_WITH_GATE(ve_clk, "ve", "pll-ve",
 			     0x13c, 16, 3, BIT(31), 0);
 
 static SUNXI_CCU_GATE(ac_dig_clk,	"ac-dig",	"pll-audio",
-		      0x140, BIT(31), 0);
+		      0x140, BIT(31), CLK_SET_RATE_PARENT);
 static SUNXI_CCU_GATE(avs_clk,		"avs",		"osc24M",
 		      0x144, BIT(31), 0);
 
diff --git a/drivers/clk/sunxi-ng/ccu_div.h b/drivers/clk/sunxi-ng/ccu_div.h
index 34c338832c0d..06540f7cf41c 100644
--- a/drivers/clk/sunxi-ng/ccu_div.h
+++ b/drivers/clk/sunxi-ng/ccu_div.h
@@ -20,7 +20,7 @@
 #include "ccu_mux.h"
 
 /**
- * struct _ccu_div - Internal divider description
+ * struct ccu_div_internal - Internal divider description
  * @shift: Bit offset of the divider in its register
  * @width: Width of the divider field in its register
  * @max: Maximum value allowed for that divider. This is the
@@ -36,7 +36,7 @@
  * It is basically a wrapper around the clk_divider functions
  * arguments.
  */
-struct _ccu_div {
+struct ccu_div_internal {
 	u8			shift;
 	u8			width;
 
@@ -78,7 +78,7 @@ struct _ccu_div {
 struct ccu_div {
 	u32			enable;
 
-	struct _ccu_div		div;
+	struct ccu_div_internal		div;
 	struct ccu_mux_internal	mux;
 	struct ccu_common	common;
 };
diff --git a/drivers/clk/sunxi-ng/ccu_frac.c b/drivers/clk/sunxi-ng/ccu_frac.c
index 5c4b10cd15b5..8b5eb7756bf7 100644
--- a/drivers/clk/sunxi-ng/ccu_frac.c
+++ b/drivers/clk/sunxi-ng/ccu_frac.c
@@ -14,7 +14,7 @@
 #include "ccu_frac.h"
 
 bool ccu_frac_helper_is_enabled(struct ccu_common *common,
-				struct _ccu_frac *cf)
+				struct ccu_frac_internal *cf)
 {
 	if (!(common->features & CCU_FEATURE_FRACTIONAL))
 		return false;
@@ -23,7 +23,7 @@ bool ccu_frac_helper_is_enabled(struct ccu_common *common,
 }
 
 void ccu_frac_helper_enable(struct ccu_common *common,
-			    struct _ccu_frac *cf)
+			    struct ccu_frac_internal *cf)
 {
 	unsigned long flags;
 	u32 reg;
@@ -38,7 +38,7 @@ void ccu_frac_helper_enable(struct ccu_common *common,
 }
 
 void ccu_frac_helper_disable(struct ccu_common *common,
-			     struct _ccu_frac *cf)
+			     struct ccu_frac_internal *cf)
 {
 	unsigned long flags;
 	u32 reg;
@@ -53,7 +53,7 @@ void ccu_frac_helper_disable(struct ccu_common *common,
 }
 
 bool ccu_frac_helper_has_rate(struct ccu_common *common,
-			      struct _ccu_frac *cf,
+			      struct ccu_frac_internal *cf,
 			      unsigned long rate)
 {
 	if (!(common->features & CCU_FEATURE_FRACTIONAL))
@@ -63,7 +63,7 @@ bool ccu_frac_helper_has_rate(struct ccu_common *common,
 }
 
 unsigned long ccu_frac_helper_read_rate(struct ccu_common *common,
-					struct _ccu_frac *cf)
+					struct ccu_frac_internal *cf)
 {
 	u32 reg;
 
@@ -84,7 +84,7 @@ unsigned long ccu_frac_helper_read_rate(struct ccu_common *common,
 }
 
 int ccu_frac_helper_set_rate(struct ccu_common *common,
-			     struct _ccu_frac *cf,
+			     struct ccu_frac_internal *cf,
 			     unsigned long rate)
 {
 	unsigned long flags;
diff --git a/drivers/clk/sunxi-ng/ccu_frac.h b/drivers/clk/sunxi-ng/ccu_frac.h
index e4c670b1cdfe..7b1ee380156f 100644
--- a/drivers/clk/sunxi-ng/ccu_frac.h
+++ b/drivers/clk/sunxi-ng/ccu_frac.h
@@ -18,7 +18,7 @@
 
 #include "ccu_common.h"
 
-struct _ccu_frac {
+struct ccu_frac_internal {
 	u32		enable;
 	u32		select;
 
@@ -33,21 +33,21 @@ struct _ccu_frac {
 	}
 
 bool ccu_frac_helper_is_enabled(struct ccu_common *common,
-				struct _ccu_frac *cf);
+				struct ccu_frac_internal *cf);
 void ccu_frac_helper_enable(struct ccu_common *common,
-			    struct _ccu_frac *cf);
+			    struct ccu_frac_internal *cf);
 void ccu_frac_helper_disable(struct ccu_common *common,
-			     struct _ccu_frac *cf);
+			     struct ccu_frac_internal *cf);
 
 bool ccu_frac_helper_has_rate(struct ccu_common *common,
-			      struct _ccu_frac *cf,
+			      struct ccu_frac_internal *cf,
 			      unsigned long rate);
 
 unsigned long ccu_frac_helper_read_rate(struct ccu_common *common,
-					struct _ccu_frac *cf);
+					struct ccu_frac_internal *cf);
 
 int ccu_frac_helper_set_rate(struct ccu_common *common,
-			     struct _ccu_frac *cf,
+			     struct ccu_frac_internal *cf,
 			     unsigned long rate);
 
 #endif /* _CCU_FRAC_H_ */
diff --git a/drivers/clk/sunxi-ng/ccu_mp.h b/drivers/clk/sunxi-ng/ccu_mp.h
index edf9215ea8cc..915625e97d98 100644
--- a/drivers/clk/sunxi-ng/ccu_mp.h
+++ b/drivers/clk/sunxi-ng/ccu_mp.h
@@ -29,8 +29,8 @@
 struct ccu_mp {
 	u32			enable;
 
-	struct _ccu_div		m;
-	struct _ccu_div		p;
+	struct ccu_div_internal		m;
+	struct ccu_div_internal		p;
 	struct ccu_mux_internal	mux;
 	struct ccu_common	common;
 };
diff --git a/drivers/clk/sunxi-ng/ccu_mult.c b/drivers/clk/sunxi-ng/ccu_mult.c
index 010e9424691d..678b6cb49f01 100644
--- a/drivers/clk/sunxi-ng/ccu_mult.c
+++ b/drivers/clk/sunxi-ng/ccu_mult.c
@@ -13,10 +13,23 @@
 #include "ccu_gate.h"
 #include "ccu_mult.h"
 
+struct _ccu_mult {
+	unsigned long	mult, min, max;
+};
+
 static void ccu_mult_find_best(unsigned long parent, unsigned long rate,
-			       unsigned int max_n, unsigned int *n)
+			       struct _ccu_mult *mult)
 {
-	*n = rate / parent;
+	int _mult;
+
+	_mult = rate / parent;
+	if (_mult < mult->min)
+		_mult = mult->min;
+
+	if (_mult > mult->max)
+		_mult = mult->max;
+
+	mult->mult = _mult;
 }
 
 static unsigned long ccu_mult_round_rate(struct ccu_mux_internal *mux,
@@ -25,11 +38,13 @@ static unsigned long ccu_mult_round_rate(struct ccu_mux_internal *mux,
 					void *data)
 {
 	struct ccu_mult *cm = data;
-	unsigned int n;
+	struct _ccu_mult _cm;
 
-	ccu_mult_find_best(parent_rate, rate, 1 << cm->mult.width, &n);
+	_cm.min = 1;
+	_cm.max = 1 << cm->mult.width;
+	ccu_mult_find_best(parent_rate, rate, &_cm);
 
-	return parent_rate * n;
+	return parent_rate * _cm.mult;
 }
 
 static void ccu_mult_disable(struct clk_hw *hw)
@@ -83,21 +98,23 @@ static int ccu_mult_set_rate(struct clk_hw *hw, unsigned long rate,
 			   unsigned long parent_rate)
 {
 	struct ccu_mult *cm = hw_to_ccu_mult(hw);
+	struct _ccu_mult _cm;
 	unsigned long flags;
-	unsigned int n;
 	u32 reg;
 
 	ccu_mux_helper_adjust_parent_for_prediv(&cm->common, &cm->mux, -1,
 						&parent_rate);
 
-	ccu_mult_find_best(parent_rate, rate, 1 << cm->mult.width, &n);
+	_cm.min = cm->mult.min;
+	_cm.max = 1 << cm->mult.width;
+	ccu_mult_find_best(parent_rate, rate, &_cm);
 
 	spin_lock_irqsave(cm->common.lock, flags);
 
 	reg = readl(cm->common.base + cm->common.reg);
 	reg &= ~GENMASK(cm->mult.width + cm->mult.shift - 1, cm->mult.shift);
 
-	writel(reg | ((n - 1) << cm->mult.shift),
+	writel(reg | ((_cm.mult - 1) << cm->mult.shift),
 	       cm->common.base + cm->common.reg);
 
 	spin_unlock_irqrestore(cm->common.lock, flags);
diff --git a/drivers/clk/sunxi-ng/ccu_mult.h b/drivers/clk/sunxi-ng/ccu_mult.h
index 5d2c8dc14073..c1a2134bdc71 100644
--- a/drivers/clk/sunxi-ng/ccu_mult.h
+++ b/drivers/clk/sunxi-ng/ccu_mult.h
@@ -4,21 +4,26 @@
 #include "ccu_common.h"
 #include "ccu_mux.h"
 
-struct _ccu_mult {
+struct ccu_mult_internal {
 	u8	shift;
 	u8	width;
+	u8	min;
 };
 
-#define _SUNXI_CCU_MULT(_shift, _width)		\
-	{					\
-		.shift	= _shift,		\
-		.width	= _width,		\
+#define _SUNXI_CCU_MULT_MIN(_shift, _width, _min)	\
+	{						\
+		.shift	= _shift,			\
+		.width	= _width,			\
+		.min	= _min,				\
 	}
 
+#define _SUNXI_CCU_MULT(_shift, _width)		\
+	_SUNXI_CCU_MULT_MIN(_shift, _width, 1)
+
 struct ccu_mult {
 	u32			enable;
 
-	struct _ccu_mult	mult;
+	struct ccu_mult_internal	mult;
 	struct ccu_mux_internal	mux;
 	struct ccu_common	common;
 };
diff --git a/drivers/clk/sunxi-ng/ccu_nk.c b/drivers/clk/sunxi-ng/ccu_nk.c
index d6fafb397489..eaf0fdf78d2b 100644
--- a/drivers/clk/sunxi-ng/ccu_nk.c
+++ b/drivers/clk/sunxi-ng/ccu_nk.c
@@ -9,21 +9,24 @@
  */
 
 #include <linux/clk-provider.h>
-#include <linux/rational.h>
 
 #include "ccu_gate.h"
 #include "ccu_nk.h"
 
+struct _ccu_nk {
+	unsigned long	n, min_n, max_n;
+	unsigned long	k, min_k, max_k;
+};
+
 static void ccu_nk_find_best(unsigned long parent, unsigned long rate,
-			     unsigned int max_n, unsigned int max_k,
-			     unsigned int *n, unsigned int *k)
+			     struct _ccu_nk *nk)
 {
 	unsigned long best_rate = 0;
 	unsigned int best_k = 0, best_n = 0;
 	unsigned int _k, _n;
 
-	for (_k = 1; _k <= max_k; _k++) {
-		for (_n = 1; _n <= max_n; _n++) {
+	for (_k = nk->min_k; _k <= nk->max_k; _k++) {
+		for (_n = nk->min_n; _n <= nk->max_n; _n++) {
 			unsigned long tmp_rate = parent * _n * _k;
 
 			if (tmp_rate > rate)
@@ -37,8 +40,8 @@ static void ccu_nk_find_best(unsigned long parent, unsigned long rate,
 		}
 	}
 
-	*k = best_k;
-	*n = best_n;
+	nk->k = best_k;
+	nk->n = best_n;
 }
 
 static void ccu_nk_disable(struct clk_hw *hw)
@@ -89,16 +92,19 @@ static long ccu_nk_round_rate(struct clk_hw *hw, unsigned long rate,
 			      unsigned long *parent_rate)
 {
 	struct ccu_nk *nk = hw_to_ccu_nk(hw);
-	unsigned int n, k;
+	struct _ccu_nk _nk;
 
 	if (nk->common.features & CCU_FEATURE_FIXED_POSTDIV)
 		rate *= nk->fixed_post_div;
 
-	ccu_nk_find_best(*parent_rate, rate,
-			 1 << nk->n.width, 1 << nk->k.width,
-			 &n, &k);
+	_nk.min_n = nk->n.min;
+	_nk.max_n = 1 << nk->n.width;
+	_nk.min_k = nk->k.min;
+	_nk.max_k = 1 << nk->k.width;
+
+	ccu_nk_find_best(*parent_rate, rate, &_nk);
+	rate = *parent_rate * _nk.n * _nk.k;
 
-	rate = *parent_rate * n * k;
 	if (nk->common.features & CCU_FEATURE_FIXED_POSTDIV)
 		rate = rate / nk->fixed_post_div;
 
@@ -110,15 +116,18 @@ static int ccu_nk_set_rate(struct clk_hw *hw, unsigned long rate,
 {
 	struct ccu_nk *nk = hw_to_ccu_nk(hw);
 	unsigned long flags;
-	unsigned int n, k;
+	struct _ccu_nk _nk;
 	u32 reg;
 
 	if (nk->common.features & CCU_FEATURE_FIXED_POSTDIV)
 		rate = rate * nk->fixed_post_div;
 
-	ccu_nk_find_best(parent_rate, rate,
-			 1 << nk->n.width, 1 << nk->k.width,
-			 &n, &k);
+	_nk.min_n = nk->n.min;
+	_nk.max_n = 1 << nk->n.width;
+	_nk.min_k = nk->k.min;
+	_nk.max_k = 1 << nk->k.width;
+
+	ccu_nk_find_best(parent_rate, rate, &_nk);
 
 	spin_lock_irqsave(nk->common.lock, flags);
 
@@ -126,7 +135,7 @@ static int ccu_nk_set_rate(struct clk_hw *hw, unsigned long rate,
 	reg &= ~GENMASK(nk->n.width + nk->n.shift - 1, nk->n.shift);
 	reg &= ~GENMASK(nk->k.width + nk->k.shift - 1, nk->k.shift);
 
-	writel(reg | ((k - 1) << nk->k.shift) | ((n - 1) << nk->n.shift),
+	writel(reg | ((_nk.k - 1) << nk->k.shift) | ((_nk.n - 1) << nk->n.shift),
 	       nk->common.base + nk->common.reg);
 
 	spin_unlock_irqrestore(nk->common.lock, flags);
diff --git a/drivers/clk/sunxi-ng/ccu_nk.h b/drivers/clk/sunxi-ng/ccu_nk.h
index 4b52da0c29fe..437836b80696 100644
--- a/drivers/clk/sunxi-ng/ccu_nk.h
+++ b/drivers/clk/sunxi-ng/ccu_nk.h
@@ -30,8 +30,8 @@ struct ccu_nk {
 	u32			enable;
 	u32			lock;
 
-	struct _ccu_mult	n;
-	struct _ccu_mult	k;
+	struct ccu_mult_internal	n;
+	struct ccu_mult_internal	k;
 
 	unsigned int		fixed_post_div;
 
diff --git a/drivers/clk/sunxi-ng/ccu_nkm.c b/drivers/clk/sunxi-ng/ccu_nkm.c
index 059fdc3b4f96..9b840a47a94d 100644
--- a/drivers/clk/sunxi-ng/ccu_nkm.c
+++ b/drivers/clk/sunxi-ng/ccu_nkm.c
@@ -9,15 +9,14 @@
  */
 
 #include <linux/clk-provider.h>
-#include <linux/rational.h>
 
 #include "ccu_gate.h"
 #include "ccu_nkm.h"
 
 struct _ccu_nkm {
-	unsigned long	n, max_n;
-	unsigned long	k, max_k;
-	unsigned long	m, max_m;
+	unsigned long	n, min_n, max_n;
+	unsigned long	k, min_k, max_k;
+	unsigned long	m, min_m, max_m;
 };
 
 static void ccu_nkm_find_best(unsigned long parent, unsigned long rate,
@@ -27,22 +26,22 @@ static void ccu_nkm_find_best(unsigned long parent, unsigned long rate,
 	unsigned long best_n = 0, best_k = 0, best_m = 0;
 	unsigned long _n, _k, _m;
 
-	for (_k = 1; _k <= nkm->max_k; _k++) {
-		unsigned long tmp_rate;
-
-		rational_best_approximation(rate / _k, parent,
-					    nkm->max_n, nkm->max_m, &_n, &_m);
-
-		tmp_rate = parent * _n * _k / _m;
-
-		if (tmp_rate > rate)
-			continue;
-
-		if ((rate - tmp_rate) < (rate - best_rate)) {
-			best_rate = tmp_rate;
-			best_n = _n;
-			best_k = _k;
-			best_m = _m;
+	for (_k = nkm->min_k; _k <= nkm->max_k; _k++) {
+		for (_n = nkm->min_n; _n <= nkm->max_n; _n++) {
+			for (_m = nkm->min_m; _m <= nkm->max_m; _m++) {
+				unsigned long tmp_rate;
+
+				tmp_rate = parent * _n * _k / _m;
+
+				if (tmp_rate > rate)
+					continue;
+				if ((rate - tmp_rate) < (rate - best_rate)) {
+					best_rate = tmp_rate;
+					best_n = _n;
+					best_k = _k;
+					best_m = _m;
+				}
+			}
 		}
 	}
 
@@ -101,8 +100,11 @@ static unsigned long ccu_nkm_round_rate(struct ccu_mux_internal *mux,
 	struct ccu_nkm *nkm = data;
 	struct _ccu_nkm _nkm;
 
+	_nkm.min_n = nkm->n.min;
 	_nkm.max_n = 1 << nkm->n.width;
+	_nkm.min_k = nkm->k.min;
 	_nkm.max_k = 1 << nkm->k.width;
+	_nkm.min_m = 1;
 	_nkm.max_m = nkm->m.max ?: 1 << nkm->m.width;
 
 	ccu_nkm_find_best(parent_rate, rate, &_nkm);
@@ -127,8 +129,11 @@ static int ccu_nkm_set_rate(struct clk_hw *hw, unsigned long rate,
 	unsigned long flags;
 	u32 reg;
 
+	_nkm.min_n = nkm->n.min;
 	_nkm.max_n = 1 << nkm->n.width;
+	_nkm.min_k = nkm->k.min;
 	_nkm.max_k = 1 << nkm->k.width;
+	_nkm.min_m = 1;
 	_nkm.max_m = nkm->m.max ?: 1 << nkm->m.width;
 
 	ccu_nkm_find_best(parent_rate, rate, &_nkm);
diff --git a/drivers/clk/sunxi-ng/ccu_nkm.h b/drivers/clk/sunxi-ng/ccu_nkm.h
index 35493fddd8ab..34580894f4d1 100644
--- a/drivers/clk/sunxi-ng/ccu_nkm.h
+++ b/drivers/clk/sunxi-ng/ccu_nkm.h
@@ -29,9 +29,9 @@ struct ccu_nkm {
 	u32			enable;
 	u32			lock;
 
-	struct _ccu_mult	n;
-	struct _ccu_mult	k;
-	struct _ccu_div		m;
+	struct ccu_mult_internal	n;
+	struct ccu_mult_internal	k;
+	struct ccu_div_internal		m;
 	struct ccu_mux_internal	mux;
 
 	struct ccu_common	common;
diff --git a/drivers/clk/sunxi-ng/ccu_nkmp.c b/drivers/clk/sunxi-ng/ccu_nkmp.c
index 9769dee99511..684c42da3ebb 100644
--- a/drivers/clk/sunxi-ng/ccu_nkmp.c
+++ b/drivers/clk/sunxi-ng/ccu_nkmp.c
@@ -9,16 +9,15 @@
  */
 
 #include <linux/clk-provider.h>
-#include <linux/rational.h>
 
 #include "ccu_gate.h"
 #include "ccu_nkmp.h"
 
 struct _ccu_nkmp {
-	unsigned long	n, max_n;
-	unsigned long	k, max_k;
-	unsigned long	m, max_m;
-	unsigned long	p, max_p;
+	unsigned long	n, min_n, max_n;
+	unsigned long	k, min_k, max_k;
+	unsigned long	m, min_m, max_m;
+	unsigned long	p, min_p, max_p;
 };
 
 static void ccu_nkmp_find_best(unsigned long parent, unsigned long rate,
@@ -28,25 +27,25 @@ static void ccu_nkmp_find_best(unsigned long parent, unsigned long rate,
 	unsigned long best_n = 0, best_k = 0, best_m = 0, best_p = 0;
 	unsigned long _n, _k, _m, _p;
 
-	for (_k = 1; _k <= nkmp->max_k; _k++) {
-		for (_p = 1; _p <= nkmp->max_p; _p <<= 1) {
-			unsigned long tmp_rate;
-
-			rational_best_approximation(rate / _k, parent / _p,
-						    nkmp->max_n, nkmp->max_m,
-						    &_n, &_m);
-
-			tmp_rate = parent * _n * _k / (_m * _p);
-
-			if (tmp_rate > rate)
-				continue;
-
-			if ((rate - tmp_rate) < (rate - best_rate)) {
-				best_rate = tmp_rate;
-				best_n = _n;
-				best_k = _k;
-				best_m = _m;
-				best_p = _p;
+	for (_k = nkmp->min_k; _k <= nkmp->max_k; _k++) {
+		for (_n = nkmp->min_n; _n <= nkmp->max_n; _n++) {
+			for (_m = nkmp->min_m; _m <= nkmp->max_m; _m++) {
+				for (_p = nkmp->min_p; _p <= nkmp->max_p; _p <<= 1) {
+					unsigned long tmp_rate;
+
+					tmp_rate = parent * _n * _k / (_m * _p);
+
+					if (tmp_rate > rate)
+						continue;
+
+					if ((rate - tmp_rate) < (rate - best_rate)) {
+						best_rate = tmp_rate;
+						best_n = _n;
+						best_k = _k;
+						best_m = _m;
+						best_p = _p;
+					}
+				}
 			}
 		}
 	}
@@ -108,9 +107,13 @@ static long ccu_nkmp_round_rate(struct clk_hw *hw, unsigned long rate,
 	struct ccu_nkmp *nkmp = hw_to_ccu_nkmp(hw);
 	struct _ccu_nkmp _nkmp;
 
+	_nkmp.min_n = nkmp->n.min;
 	_nkmp.max_n = 1 << nkmp->n.width;
+	_nkmp.min_k = nkmp->k.min;
 	_nkmp.max_k = 1 << nkmp->k.width;
+	_nkmp.min_m = 1;
 	_nkmp.max_m = nkmp->m.max ?: 1 << nkmp->m.width;
+	_nkmp.min_p = 1;
 	_nkmp.max_p = nkmp->p.max ?: 1 << ((1 << nkmp->p.width) - 1);
 
 	ccu_nkmp_find_best(*parent_rate, rate, &_nkmp);
@@ -126,9 +129,13 @@ static int ccu_nkmp_set_rate(struct clk_hw *hw, unsigned long rate,
 	unsigned long flags;
 	u32 reg;
 
+	_nkmp.min_n = 1;
 	_nkmp.max_n = 1 << nkmp->n.width;
+	_nkmp.min_k = 1;
 	_nkmp.max_k = 1 << nkmp->k.width;
+	_nkmp.min_m = 1;
 	_nkmp.max_m = nkmp->m.max ?: 1 << nkmp->m.width;
+	_nkmp.min_p = 1;
 	_nkmp.max_p = nkmp->p.max ?: 1 << ((1 << nkmp->p.width) - 1);
 
 	ccu_nkmp_find_best(parent_rate, rate, &_nkmp);
diff --git a/drivers/clk/sunxi-ng/ccu_nkmp.h b/drivers/clk/sunxi-ng/ccu_nkmp.h
index 5adb0c92a614..a82facbc6144 100644
--- a/drivers/clk/sunxi-ng/ccu_nkmp.h
+++ b/drivers/clk/sunxi-ng/ccu_nkmp.h
@@ -29,10 +29,10 @@ struct ccu_nkmp {
 	u32			enable;
 	u32			lock;
 
-	struct _ccu_mult	n;
-	struct _ccu_mult	k;
-	struct _ccu_div		m;
-	struct _ccu_div		p;
+	struct ccu_mult_internal	n;
+	struct ccu_mult_internal	k;
+	struct ccu_div_internal		m;
+	struct ccu_div_internal		p;
 
 	struct ccu_common	common;
 };
diff --git a/drivers/clk/sunxi-ng/ccu_nm.c b/drivers/clk/sunxi-ng/ccu_nm.c
index b61bdd8c7a7f..c9f3b6c982f0 100644
--- a/drivers/clk/sunxi-ng/ccu_nm.c
+++ b/drivers/clk/sunxi-ng/ccu_nm.c
@@ -9,12 +9,42 @@
  */
 
 #include <linux/clk-provider.h>
-#include <linux/rational.h>
 
 #include "ccu_frac.h"
 #include "ccu_gate.h"
 #include "ccu_nm.h"
 
+struct _ccu_nm {
+	unsigned long	n, min_n, max_n;
+	unsigned long	m, min_m, max_m;
+};
+
+static void ccu_nm_find_best(unsigned long parent, unsigned long rate,
+			     struct _ccu_nm *nm)
+{
+	unsigned long best_rate = 0;
+	unsigned long best_n = 0, best_m = 0;
+	unsigned long _n, _m;
+
+	for (_n = nm->min_n; _n <= nm->max_n; _n++) {
+		for (_m = nm->min_m; _m <= nm->max_m; _m++) {
+			unsigned long tmp_rate = parent * _n  / _m;
+
+			if (tmp_rate > rate)
+				continue;
+
+			if ((rate - tmp_rate) < (rate - best_rate)) {
+				best_rate = tmp_rate;
+				best_n = _n;
+				best_m = _m;
+			}
+		}
+	}
+
+	nm->n = best_n;
+	nm->m = best_m;
+}
+
 static void ccu_nm_disable(struct clk_hw *hw)
 {
 	struct ccu_nm *nm = hw_to_ccu_nm(hw);
@@ -61,24 +91,24 @@ static long ccu_nm_round_rate(struct clk_hw *hw, unsigned long rate,
 			      unsigned long *parent_rate)
 {
 	struct ccu_nm *nm = hw_to_ccu_nm(hw);
-	unsigned long max_n, max_m;
-	unsigned long n, m;
+	struct _ccu_nm _nm;
 
-	max_n = 1 << nm->n.width;
-	max_m = nm->m.max ?: 1 << nm->m.width;
+	_nm.min_n = nm->n.min;
+	_nm.max_n = 1 << nm->n.width;
+	_nm.min_m = 1;
+	_nm.max_m = nm->m.max ?: 1 << nm->m.width;
 
-	rational_best_approximation(rate, *parent_rate, max_n, max_m, &n, &m);
+	ccu_nm_find_best(*parent_rate, rate, &_nm);
 
-	return *parent_rate * n / m;
+	return *parent_rate * _nm.n / _nm.m;
 }
 
 static int ccu_nm_set_rate(struct clk_hw *hw, unsigned long rate,
 			   unsigned long parent_rate)
 {
 	struct ccu_nm *nm = hw_to_ccu_nm(hw);
+	struct _ccu_nm _nm;
 	unsigned long flags;
-	unsigned long max_n, max_m;
-	unsigned long n, m;
 	u32 reg;
 
 	if (ccu_frac_helper_has_rate(&nm->common, &nm->frac, rate))
@@ -86,10 +116,12 @@ static int ccu_nm_set_rate(struct clk_hw *hw, unsigned long rate,
 	else
 		ccu_frac_helper_disable(&nm->common, &nm->frac);
 
-	max_n = 1 << nm->n.width;
-	max_m = nm->m.max ?: 1 << nm->m.width;
+	_nm.min_n = 1;
+	_nm.max_n = 1 << nm->n.width;
+	_nm.min_m = 1;
+	_nm.max_m = nm->m.max ?: 1 << nm->m.width;
 
-	rational_best_approximation(rate, parent_rate, max_n, max_m, &n, &m);
+	ccu_nm_find_best(parent_rate, rate, &_nm);
 
 	spin_lock_irqsave(nm->common.lock, flags);
 
@@ -97,7 +129,7 @@ static int ccu_nm_set_rate(struct clk_hw *hw, unsigned long rate,
 	reg &= ~GENMASK(nm->n.width + nm->n.shift - 1, nm->n.shift);
 	reg &= ~GENMASK(nm->m.width + nm->m.shift - 1, nm->m.shift);
 
-	writel(reg | ((m - 1) << nm->m.shift) | ((n - 1) << nm->n.shift),
+	writel(reg | ((_nm.m - 1) << nm->m.shift) | ((_nm.n - 1) << nm->n.shift),
 	       nm->common.base + nm->common.reg);
 
 	spin_unlock_irqrestore(nm->common.lock, flags);
diff --git a/drivers/clk/sunxi-ng/ccu_nm.h b/drivers/clk/sunxi-ng/ccu_nm.h
index 0b7bcd33a2df..e87fd186da78 100644
--- a/drivers/clk/sunxi-ng/ccu_nm.h
+++ b/drivers/clk/sunxi-ng/ccu_nm.h
@@ -30,9 +30,9 @@ struct ccu_nm {
 	u32			enable;
 	u32			lock;
 
-	struct _ccu_mult	n;
-	struct _ccu_div		m;
-	struct _ccu_frac	frac;
+	struct ccu_mult_internal	n;
+	struct ccu_div_internal		m;
+	struct ccu_frac_internal	frac;
 
 	struct ccu_common	common;
 };
diff --git a/drivers/clk/sunxi/clk-mod0.c b/drivers/clk/sunxi/clk-mod0.c
index e54266cc1c51..4417ae129ac7 100644
--- a/drivers/clk/sunxi/clk-mod0.c
+++ b/drivers/clk/sunxi/clk-mod0.c
@@ -24,7 +24,7 @@
 #include "clk-factors.h"
 
 /**
- * sun4i_get_mod0_factors() - calculates m, n factors for MOD0-style clocks
+ * sun4i_a10_get_mod0_factors() - calculates m, n factors for MOD0-style clocks
  * MOD0 rate is calculated as follows
  * rate = (parent_rate >> p) / (m + 1);
  */
diff --git a/drivers/clk/tegra/clk-tegra124-dfll-fcpu.c b/drivers/clk/tegra/clk-tegra124-dfll-fcpu.c
index c205809ba580..ad1c1cc829cb 100644
--- a/drivers/clk/tegra/clk-tegra124-dfll-fcpu.c
+++ b/drivers/clk/tegra/clk-tegra124-dfll-fcpu.c
@@ -20,7 +20,7 @@
 #include <linux/cpu.h>
 #include <linux/err.h>
 #include <linux/kernel.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <soc/tegra/fuse.h>
 
@@ -148,7 +148,6 @@ static const struct of_device_id tegra124_dfll_fcpu_of_match[] = {
 	{ .compatible = "nvidia,tegra124-dfll", },
 	{ },
 };
-MODULE_DEVICE_TABLE(of, tegra124_dfll_fcpu_of_match);
 
 static const struct dev_pm_ops tegra124_dfll_pm_ops = {
 	SET_RUNTIME_PM_OPS(tegra_dfll_runtime_suspend,
@@ -164,20 +163,4 @@ static struct platform_driver tegra124_dfll_fcpu_driver = {
 		.pm = &tegra124_dfll_pm_ops,
 	},
 };
-
-static int __init tegra124_dfll_fcpu_init(void)
-{
-	return platform_driver_register(&tegra124_dfll_fcpu_driver);
-}
-module_init(tegra124_dfll_fcpu_init);
-
-static void __exit tegra124_dfll_fcpu_exit(void)
-{
-	platform_driver_unregister(&tegra124_dfll_fcpu_driver);
-}
-module_exit(tegra124_dfll_fcpu_exit);
-
-MODULE_DESCRIPTION("Tegra124 DFLL clock source driver");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Aleksandr Frid <afrid@nvidia.com>");
-MODULE_AUTHOR("Paul Walmsley <pwalmsley@nvidia.com>");
+builtin_platform_driver(tegra124_dfll_fcpu_driver);
diff --git a/drivers/clk/tegra/cvb.c b/drivers/clk/tegra/cvb.c
index 624115e82ff9..da9e8e7b5ce5 100644
--- a/drivers/clk/tegra/cvb.c
+++ b/drivers/clk/tegra/cvb.c
@@ -92,19 +92,19 @@ static int build_opp_table(struct device *dev, const struct cvb_table *table,
 
 /**
  * tegra_cvb_add_opp_table - build OPP table from Tegra CVB tables
- * @cvb_tables: array of CVB tables
- * @sz: size of the previously mentioned array
+ * @dev: the struct device * for which the OPP table is built
+ * @tables: array of CVB tables
+ * @count: size of the previously mentioned array
  * @process_id: process id of the HW module
  * @speedo_id: speedo id of the HW module
  * @speedo_value: speedo value of the HW module
- * @max_rate: highest safe clock rate
- * @opp_dev: the struct device * for which the OPP table is built
+ * @max_freq: highest safe clock rate
  *
  * On Tegra, a CVB table encodes the relationship between operating voltage
  * and safe maximal frequency for a given module (e.g. GPU or CPU). This
  * function calculates the optimal voltage-frequency operating points
  * for the given arguments and exports them via the OPP library for the
- * given @opp_dev. Returns a pointer to the struct cvb_table that matched
+ * given @dev. Returns a pointer to the struct cvb_table that matched
  * or an ERR_PTR on failure.
  */
 const struct cvb_table *
diff --git a/drivers/clk/ti/clk-3xxx.c b/drivers/clk/ti/clk-3xxx.c
index 8831e1a05367..11d8aa3ec186 100644
--- a/drivers/clk/ti/clk-3xxx.c
+++ b/drivers/clk/ti/clk-3xxx.c
@@ -22,13 +22,6 @@
 
 #include "clock.h"
 
-/*
- * DPLL5_FREQ_FOR_USBHOST: USBHOST and USBTLL are the only clocks
- * that are sourced by DPLL5, and both of these require this clock
- * to be at 120 MHz for proper operation.
- */
-#define DPLL5_FREQ_FOR_USBHOST		120000000
-
 #define OMAP3430ES2_ST_DSS_IDLE_SHIFT			1
 #define OMAP3430ES2_ST_HSOTGUSB_IDLE_SHIFT		5
 #define OMAP3430ES2_ST_SSI_IDLE_SHIFT			8
@@ -546,14 +539,21 @@ void __init omap3_clk_lock_dpll5(void)
 	struct clk *dpll5_clk;
 	struct clk *dpll5_m2_clk;
 
+	/*
+	 * Errata sprz319f advisory 2.1 documents a USB host clock drift issue
+	 * that can be worked around using specially crafted dpll5 settings
+	 * with a dpll5_m2 divider set to 8. Set the dpll5 rate to 8x the USB
+	 * host clock rate, its .set_rate handler() will detect that frequency
+	 * and use the errata settings.
+	 */
 	dpll5_clk = clk_get(NULL, "dpll5_ck");
-	clk_set_rate(dpll5_clk, DPLL5_FREQ_FOR_USBHOST);
+	clk_set_rate(dpll5_clk, OMAP3_DPLL5_FREQ_FOR_USBHOST * 8);
 	clk_prepare_enable(dpll5_clk);
 
-	/* Program dpll5_m2_clk divider for no division */
+	/* Program dpll5_m2_clk divider */
 	dpll5_m2_clk = clk_get(NULL, "dpll5_m2_ck");
 	clk_prepare_enable(dpll5_m2_clk);
-	clk_set_rate(dpll5_m2_clk, DPLL5_FREQ_FOR_USBHOST);
+	clk_set_rate(dpll5_m2_clk, OMAP3_DPLL5_FREQ_FOR_USBHOST);
 
 	clk_disable_unprepare(dpll5_m2_clk);
 	clk_disable_unprepare(dpll5_clk);
diff --git a/drivers/clk/ti/clk-7xx.c b/drivers/clk/ti/clk-7xx.c
index bfa17d33ef3b..9fd6043314eb 100644
--- a/drivers/clk/ti/clk-7xx.c
+++ b/drivers/clk/ti/clk-7xx.c
@@ -201,7 +201,6 @@ static struct ti_dt_clk dra7xx_clks[] = {
 	DT_CLK(NULL, "atl_dpll_clk_mux", "atl_dpll_clk_mux"),
 	DT_CLK(NULL, "atl_gfclk_mux", "atl_gfclk_mux"),
 	DT_CLK(NULL, "dcan1_sys_clk_mux", "dcan1_sys_clk_mux"),
-	DT_CLK(NULL, "gmac_gmii_ref_clk_div", "gmac_gmii_ref_clk_div"),
 	DT_CLK(NULL, "gmac_rft_clk_mux", "gmac_rft_clk_mux"),
 	DT_CLK(NULL, "gpu_core_gclk_mux", "gpu_core_gclk_mux"),
 	DT_CLK(NULL, "gpu_hyd_gclk_mux", "gpu_hyd_gclk_mux"),
diff --git a/drivers/clk/ti/clk-dra7-atl.c b/drivers/clk/ti/clk-dra7-atl.c
index c77333230bdf..45d05339d583 100644
--- a/drivers/clk/ti/clk-dra7-atl.c
+++ b/drivers/clk/ti/clk-dra7-atl.c
@@ -15,7 +15,7 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/slab.h>
@@ -295,31 +295,17 @@ static int of_dra7_atl_clk_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int of_dra7_atl_clk_remove(struct platform_device *pdev)
-{
-	pm_runtime_disable(&pdev->dev);
-
-	return 0;
-}
-
 static const struct of_device_id of_dra7_atl_clk_match_tbl[] = {
 	{ .compatible = "ti,dra7-atl", },
 	{},
 };
-MODULE_DEVICE_TABLE(of, of_dra7_atl_clk_match_tbl);
 
 static struct platform_driver dra7_atl_clk_driver = {
 	.driver = {
 		.name = "dra7-atl",
+		.suppress_bind_attrs = true,
 		.of_match_table = of_dra7_atl_clk_match_tbl,
 	},
 	.probe = of_dra7_atl_clk_probe,
-	.remove = of_dra7_atl_clk_remove,
 };
-
-module_platform_driver(dra7_atl_clk_driver);
-
-MODULE_DESCRIPTION("Clock driver for DRA7 Audio Tracking Logic");
-MODULE_ALIAS("platform:dra7-atl-clock");
-MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
-MODULE_LICENSE("GPL v2");
+builtin_platform_driver(dra7_atl_clk_driver);
diff --git a/drivers/clk/ti/clock.h b/drivers/clk/ti/clock.h
index 90f3f472ae1c..13c37f48d9d6 100644
--- a/drivers/clk/ti/clock.h
+++ b/drivers/clk/ti/clock.h
@@ -257,11 +257,20 @@ long omap2_dpll_round_rate(struct clk_hw *hw, unsigned long target_rate,
 unsigned long omap3_clkoutx2_recalc(struct clk_hw *hw,
 				    unsigned long parent_rate);
 
+/*
+ * OMAP3_DPLL5_FREQ_FOR_USBHOST: USBHOST and USBTLL are the only clocks
+ * that are sourced by DPLL5, and both of these require this clock
+ * to be at 120 MHz for proper operation.
+ */
+#define OMAP3_DPLL5_FREQ_FOR_USBHOST	120000000
+
 unsigned long omap3_dpll_recalc(struct clk_hw *hw, unsigned long parent_rate);
 int omap3_dpll4_set_rate(struct clk_hw *clk, unsigned long rate,
 			 unsigned long parent_rate);
 int omap3_dpll4_set_rate_and_parent(struct clk_hw *hw, unsigned long rate,
 				    unsigned long parent_rate, u8 index);
+int omap3_dpll5_set_rate(struct clk_hw *hw, unsigned long rate,
+			 unsigned long parent_rate);
 void omap3_clk_lock_dpll5(void);
 
 unsigned long omap4_dpll_regm4xen_recalc(struct clk_hw *hw,
diff --git a/drivers/clk/ti/dpll.c b/drivers/clk/ti/dpll.c
index 9fc8754a6e61..4b9a419d8e14 100644
--- a/drivers/clk/ti/dpll.c
+++ b/drivers/clk/ti/dpll.c
@@ -114,6 +114,18 @@ static const struct clk_ops omap3_dpll_ck_ops = {
 	.round_rate	= &omap2_dpll_round_rate,
 };
 
+static const struct clk_ops omap3_dpll5_ck_ops = {
+	.enable		= &omap3_noncore_dpll_enable,
+	.disable	= &omap3_noncore_dpll_disable,
+	.get_parent	= &omap2_init_dpll_parent,
+	.recalc_rate	= &omap3_dpll_recalc,
+	.set_rate	= &omap3_dpll5_set_rate,
+	.set_parent	= &omap3_noncore_dpll_set_parent,
+	.set_rate_and_parent	= &omap3_noncore_dpll_set_rate_and_parent,
+	.determine_rate	= &omap3_noncore_dpll_determine_rate,
+	.round_rate	= &omap2_dpll_round_rate,
+};
+
 static const struct clk_ops omap3_dpll_per_ck_ops = {
 	.enable		= &omap3_noncore_dpll_enable,
 	.disable	= &omap3_noncore_dpll_disable,
@@ -474,7 +486,12 @@ static void __init of_ti_omap3_dpll_setup(struct device_node *node)
 		.modes = (1 << DPLL_LOW_POWER_BYPASS) | (1 << DPLL_LOCKED),
 	};
 
-	of_ti_dpll_setup(node, &omap3_dpll_ck_ops, &dd);
+	if ((of_machine_is_compatible("ti,omap3630") ||
+	     of_machine_is_compatible("ti,omap36xx")) &&
+	    !strcmp(node->name, "dpll5_ck"))
+		of_ti_dpll_setup(node, &omap3_dpll5_ck_ops, &dd);
+	else
+		of_ti_dpll_setup(node, &omap3_dpll_ck_ops, &dd);
 }
 CLK_OF_DECLARE(ti_omap3_dpll_clock, "ti,omap3-dpll-clock",
 	       of_ti_omap3_dpll_setup);
diff --git a/drivers/clk/ti/dpll3xxx.c b/drivers/clk/ti/dpll3xxx.c
index 88f2ce81ba55..4cdd28a25584 100644
--- a/drivers/clk/ti/dpll3xxx.c
+++ b/drivers/clk/ti/dpll3xxx.c
@@ -838,3 +838,70 @@ int omap3_dpll4_set_rate_and_parent(struct clk_hw *hw, unsigned long rate,
 	return omap3_noncore_dpll_set_rate_and_parent(hw, rate, parent_rate,
 						      index);
 }
+
+/* Apply DM3730 errata sprz319 advisory 2.1. */
+static bool omap3_dpll5_apply_errata(struct clk_hw *hw,
+				     unsigned long parent_rate)
+{
+	struct omap3_dpll5_settings {
+		unsigned int rate, m, n;
+	};
+
+	static const struct omap3_dpll5_settings precomputed[] = {
+		/*
+		 * From DM3730 errata advisory 2.1, table 35 and 36.
+		 * The N value is increased by 1 compared to the tables as the
+		 * errata lists register values while last_rounded_field is the
+		 * real divider value.
+		 */
+		{ 12000000,  80,  0 + 1 },
+		{ 13000000, 443,  5 + 1 },
+		{ 19200000,  50,  0 + 1 },
+		{ 26000000, 443, 11 + 1 },
+		{ 38400000,  25,  0 + 1 }
+	};
+
+	const struct omap3_dpll5_settings *d;
+	struct clk_hw_omap *clk = to_clk_hw_omap(hw);
+	struct dpll_data *dd;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(precomputed); ++i) {
+		if (parent_rate == precomputed[i].rate)
+			break;
+	}
+
+	if (i == ARRAY_SIZE(precomputed))
+		return false;
+
+	d = &precomputed[i];
+
+	/* Update the M, N and rounded rate values and program the DPLL. */
+	dd = clk->dpll_data;
+	dd->last_rounded_m = d->m;
+	dd->last_rounded_n = d->n;
+	dd->last_rounded_rate = div_u64((u64)parent_rate * d->m, d->n);
+	omap3_noncore_dpll_program(clk, 0);
+
+	return true;
+}
+
+/**
+ * omap3_dpll5_set_rate - set rate for omap3 dpll5
+ * @hw: clock to change
+ * @rate: target rate for clock
+ * @parent_rate: rate of the parent clock
+ *
+ * Set rate for the DPLL5 clock. Apply the sprz319 advisory 2.1 on OMAP36xx if
+ * the DPLL is used for USB host (detected through the requested rate).
+ */
+int omap3_dpll5_set_rate(struct clk_hw *hw, unsigned long rate,
+			 unsigned long parent_rate)
+{
+	if (rate == OMAP3_DPLL5_FREQ_FOR_USBHOST * 8) {
+		if (omap3_dpll5_apply_errata(hw, parent_rate))
+			return 0;
+	}
+
+	return omap3_noncore_dpll_set_rate(hw, rate, parent_rate);
+}
diff --git a/drivers/clk/uniphier/Makefile b/drivers/clk/uniphier/Makefile
index f27b360329ca..665d1d65a90e 100644
--- a/drivers/clk/uniphier/Makefile
+++ b/drivers/clk/uniphier/Makefile
@@ -1,8 +1,11 @@
 obj-y	+= clk-uniphier-core.o
+
+obj-y	+= clk-uniphier-cpugear.o
 obj-y	+= clk-uniphier-fixed-factor.o
 obj-y	+= clk-uniphier-fixed-rate.o
 obj-y	+= clk-uniphier-gate.o
 obj-y	+= clk-uniphier-mux.o
+
 obj-y	+= clk-uniphier-sys.o
 obj-y	+= clk-uniphier-mio.o
 obj-y	+= clk-uniphier-peri.o
diff --git a/drivers/clk/uniphier/clk-uniphier-core.c b/drivers/clk/uniphier/clk-uniphier-core.c
index 26c53f7963a4..0007218ce6a0 100644
--- a/drivers/clk/uniphier/clk-uniphier-core.c
+++ b/drivers/clk/uniphier/clk-uniphier-core.c
@@ -27,6 +27,9 @@ static struct clk_hw *uniphier_clk_register(struct device *dev,
 					const struct uniphier_clk_data *data)
 {
 	switch (data->type) {
+	case UNIPHIER_CLK_TYPE_CPUGEAR:
+		return uniphier_clk_register_cpugear(dev, regmap, data->name,
+						     &data->data.cpugear);
 	case UNIPHIER_CLK_TYPE_FIXED_FACTOR:
 		return uniphier_clk_register_fixed_factor(dev, data->name,
 							  &data->data.factor);
diff --git a/drivers/clk/uniphier/clk-uniphier-cpugear.c b/drivers/clk/uniphier/clk-uniphier-cpugear.c
new file mode 100644
index 000000000000..9bff26e0cbb0
--- /dev/null
+++ b/drivers/clk/uniphier/clk-uniphier-cpugear.c
@@ -0,0 +1,115 @@
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/regmap.h>
+
+#include "clk-uniphier.h"
+
+#define UNIPHIER_CLK_CPUGEAR_STAT	0	/* status */
+#define UNIPHIER_CLK_CPUGEAR_SET	4	/* set */
+#define UNIPHIER_CLK_CPUGEAR_UPD	8	/* update */
+#define   UNIPHIER_CLK_CPUGEAR_UPD_BIT	BIT(0)
+
+struct uniphier_clk_cpugear {
+	struct clk_hw hw;
+	struct regmap *regmap;
+	unsigned int regbase;
+	unsigned int mask;
+};
+
+#define to_uniphier_clk_cpugear(_hw) \
+			container_of(_hw, struct uniphier_clk_cpugear, hw)
+
+static int uniphier_clk_cpugear_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct uniphier_clk_cpugear *gear = to_uniphier_clk_cpugear(hw);
+	int ret;
+	unsigned int val;
+
+	ret = regmap_write_bits(gear->regmap,
+				gear->regbase + UNIPHIER_CLK_CPUGEAR_SET,
+				gear->mask, index);
+	if (ret)
+		return ret;
+
+	ret = regmap_write_bits(gear->regmap,
+				gear->regbase + UNIPHIER_CLK_CPUGEAR_SET,
+				UNIPHIER_CLK_CPUGEAR_UPD_BIT,
+				UNIPHIER_CLK_CPUGEAR_UPD_BIT);
+	if (ret)
+		return ret;
+
+	return regmap_read_poll_timeout(gear->regmap,
+				gear->regbase + UNIPHIER_CLK_CPUGEAR_UPD,
+				val, !(val & UNIPHIER_CLK_CPUGEAR_UPD_BIT),
+				0, 1);
+}
+
+static u8 uniphier_clk_cpugear_get_parent(struct clk_hw *hw)
+{
+	struct uniphier_clk_cpugear *gear = to_uniphier_clk_cpugear(hw);
+	int num_parents = clk_hw_get_num_parents(hw);
+	int ret;
+	unsigned int val;
+
+	ret = regmap_read(gear->regmap,
+			  gear->regbase + UNIPHIER_CLK_CPUGEAR_STAT, &val);
+	if (ret)
+		return ret;
+
+	val &= gear->mask;
+
+	return val < num_parents ? val : -EINVAL;
+}
+
+static const struct clk_ops uniphier_clk_cpugear_ops = {
+	.determine_rate = __clk_mux_determine_rate,
+	.set_parent = uniphier_clk_cpugear_set_parent,
+	.get_parent = uniphier_clk_cpugear_get_parent,
+};
+
+struct clk_hw *uniphier_clk_register_cpugear(struct device *dev,
+					 struct regmap *regmap,
+					 const char *name,
+				const struct uniphier_clk_cpugear_data *data)
+{
+	struct uniphier_clk_cpugear *gear;
+	struct clk_init_data init;
+	int ret;
+
+	gear = devm_kzalloc(dev, sizeof(*gear), GFP_KERNEL);
+	if (!gear)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &uniphier_clk_cpugear_ops;
+	init.flags = CLK_SET_RATE_PARENT;
+	init.parent_names = data->parent_names;
+	init.num_parents = data->num_parents,
+
+	gear->regmap = regmap;
+	gear->regbase = data->regbase;
+	gear->mask = data->mask;
+	gear->hw.init = &init;
+
+	ret = devm_clk_hw_register(dev, &gear->hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return &gear->hw;
+}
diff --git a/drivers/clk/uniphier/clk-uniphier-sys.c b/drivers/clk/uniphier/clk-uniphier-sys.c
index 5d029991047d..d049316c1c0f 100644
--- a/drivers/clk/uniphier/clk-uniphier-sys.c
+++ b/drivers/clk/uniphier/clk-uniphier-sys.c
@@ -125,16 +125,35 @@ const struct uniphier_clk_data uniphier_pxs2_sys_clk_data[] = {
 };
 
 const struct uniphier_clk_data uniphier_ld11_sys_clk_data[] = {
+	UNIPHIER_CLK_FACTOR("cpll", -1, "ref", 392, 5),		/* 1960 MHz */
+	UNIPHIER_CLK_FACTOR("mpll", -1, "ref", 64, 1),		/* 1600 MHz */
 	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 80, 1),		/* 2000 MHz */
+	UNIPHIER_CLK_FACTOR("vspll", -1, "ref", 80, 1),		/* 2000 MHz */
 	UNIPHIER_CLK_FACTOR("uart", 0, "spll", 1, 34),
 	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 40),
 	UNIPHIER_LD11_SYS_CLK_STDMAC(8),			/* HSC, MIO */
 	UNIPHIER_CLK_FACTOR("usb2", -1, "ref", 24, 25),
+	/* CPU gears */
+	UNIPHIER_CLK_DIV4("cpll", 2, 3, 4, 8),
+	UNIPHIER_CLK_DIV4("mpll", 2, 3, 4, 8),
+	UNIPHIER_CLK_DIV3("spll", 3, 4, 8),
+	/* Note: both gear1 and gear4 are spll/4.  This is not a bug. */
+	UNIPHIER_CLK_CPUGEAR("cpu-ca53", 33, 0x8080, 0xf, 8,
+			     "cpll/2", "spll/4", "cpll/3", "spll/3",
+			     "spll/4", "spll/8", "cpll/4", "cpll/8"),
+	UNIPHIER_CLK_CPUGEAR("cpu-ipp", 34, 0x8100, 0xf, 8,
+			     "mpll/2", "spll/4", "mpll/3", "spll/3",
+			     "spll/4", "spll/8", "mpll/4", "mpll/8"),
 	{ /* sentinel */ }
 };
 
 const struct uniphier_clk_data uniphier_ld20_sys_clk_data[] = {
+	UNIPHIER_CLK_FACTOR("cpll", -1, "ref", 88, 1),		/* ARM: 2200 MHz */
+	UNIPHIER_CLK_FACTOR("gppll", -1, "ref", 52, 1),		/* Mali: 1300 MHz */
+	UNIPHIER_CLK_FACTOR("mpll", -1, "ref", 64, 1),		/* Codec: 1600 MHz */
 	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 80, 1),		/* 2000 MHz */
+	UNIPHIER_CLK_FACTOR("s2pll", -1, "ref", 88, 1),		/* IPP: 2200 MHz */
+	UNIPHIER_CLK_FACTOR("vppll", -1, "ref", 504, 5),	/* 2520 MHz */
 	UNIPHIER_CLK_FACTOR("uart", 0, "spll", 1, 34),
 	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 40),
 	UNIPHIER_LD20_SYS_CLK_SD,
@@ -147,5 +166,18 @@ const struct uniphier_clk_data uniphier_ld20_sys_clk_data[] = {
 	UNIPHIER_CLK_GATE("usb30", 14, NULL, 0x210c, 14),
 	UNIPHIER_CLK_GATE("usb30-phy0", 16, NULL, 0x210c, 12),
 	UNIPHIER_CLK_GATE("usb30-phy1", 17, NULL, 0x210c, 13),
+	/* CPU gears */
+	UNIPHIER_CLK_DIV4("cpll", 2, 3, 4, 8),
+	UNIPHIER_CLK_DIV4("spll", 2, 3, 4, 8),
+	UNIPHIER_CLK_DIV4("s2pll", 2, 3, 4, 8),
+	UNIPHIER_CLK_CPUGEAR("cpu-ca72", 32, 0x8000, 0xf, 8,
+			     "cpll/2", "spll/2", "cpll/3", "spll/3",
+			     "spll/4", "spll/8", "cpll/4", "cpll/8"),
+	UNIPHIER_CLK_CPUGEAR("cpu-ca53", 33, 0x8080, 0xf, 8,
+			     "cpll/2", "spll/2", "cpll/3", "spll/3",
+			     "spll/4", "spll/8", "cpll/4", "cpll/8"),
+	UNIPHIER_CLK_CPUGEAR("cpu-ipp", 34, 0x8100, 0xf, 8,
+			     "s2pll/2", "spll/2", "s2pll/3", "spll/3",
+			     "spll/4", "spll/8", "s2pll/4", "s2pll/8"),
 	{ /* sentinel */ }
 };
diff --git a/drivers/clk/uniphier/clk-uniphier.h b/drivers/clk/uniphier/clk-uniphier.h
index 0244dba1f4cf..01c16ecec48f 100644
--- a/drivers/clk/uniphier/clk-uniphier.h
+++ b/drivers/clk/uniphier/clk-uniphier.h
@@ -20,15 +20,24 @@ struct clk_hw;
 struct device;
 struct regmap;
 
-#define UNIPHIER_CLK_MUX_MAX_PARENTS	8
+#define UNIPHIER_CLK_CPUGEAR_MAX_PARENTS	16
+#define UNIPHIER_CLK_MUX_MAX_PARENTS		8
 
 enum uniphier_clk_type {
+	UNIPHIER_CLK_TYPE_CPUGEAR,
 	UNIPHIER_CLK_TYPE_FIXED_FACTOR,
 	UNIPHIER_CLK_TYPE_FIXED_RATE,
 	UNIPHIER_CLK_TYPE_GATE,
 	UNIPHIER_CLK_TYPE_MUX,
 };
 
+struct uniphier_clk_cpugear_data {
+	const char *parent_names[UNIPHIER_CLK_CPUGEAR_MAX_PARENTS];
+	unsigned int num_parents;
+	unsigned int regbase;
+	unsigned int mask;
+};
+
 struct uniphier_clk_fixed_factor_data {
 	const char *parent_name;
 	unsigned int mult;
@@ -58,6 +67,7 @@ struct uniphier_clk_data {
 	enum uniphier_clk_type type;
 	int idx;
 	union {
+		struct uniphier_clk_cpugear_data cpugear;
 		struct uniphier_clk_fixed_factor_data factor;
 		struct uniphier_clk_fixed_rate_data rate;
 		struct uniphier_clk_gate_data gate;
@@ -65,6 +75,20 @@ struct uniphier_clk_data {
 	} data;
 };
 
+#define UNIPHIER_CLK_CPUGEAR(_name, _idx, _regbase, _mask,	\
+			     _num_parents, ...)			\
+	{							\
+		.name = (_name),				\
+		.type = UNIPHIER_CLK_TYPE_CPUGEAR,		\
+		.idx = (_idx),					\
+		.data.cpugear = {				\
+			.parent_names = { __VA_ARGS__ },	\
+			.num_parents = (_num_parents),		\
+			.regbase = (_regbase),			\
+			.mask = (_mask)				\
+		 },						\
+	}
+
 #define UNIPHIER_CLK_FACTOR(_name, _idx, _parent, _mult, _div)	\
 	{							\
 		.name = (_name),				\
@@ -77,7 +101,6 @@ struct uniphier_clk_data {
 		},						\
 	}
 
-
 #define UNIPHIER_CLK_GATE(_name, _idx, _parent, _reg, _bit)	\
 	{							\
 		.name = (_name),				\
@@ -90,7 +113,25 @@ struct uniphier_clk_data {
 		},						\
 	}
 
+#define UNIPHIER_CLK_DIV(parent, div)				\
+	UNIPHIER_CLK_FACTOR(parent "/" #div, -1, parent, 1, div)
+
+#define UNIPHIER_CLK_DIV2(parent, div0, div1)			\
+	UNIPHIER_CLK_DIV(parent, div0),				\
+	UNIPHIER_CLK_DIV(parent, div1)
+
+#define UNIPHIER_CLK_DIV3(parent, div0, div1, div2)		\
+	UNIPHIER_CLK_DIV2(parent, div0, div1),			\
+	UNIPHIER_CLK_DIV(parent, div2)
+
+#define UNIPHIER_CLK_DIV4(parent, div0, div1, div2, div3)	\
+	UNIPHIER_CLK_DIV2(parent, div0, div1),			\
+	UNIPHIER_CLK_DIV2(parent, div2, div3)
 
+struct clk_hw *uniphier_clk_register_cpugear(struct device *dev,
+					     struct regmap *regmap,
+					     const char *name,
+				const struct uniphier_clk_cpugear_data *data);
 struct clk_hw *uniphier_clk_register_fixed_factor(struct device *dev,
 						  const char *name,
 			const struct uniphier_clk_fixed_factor_data *data);
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index e2c6e43cf8ca..4866f7aa32e6 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -282,6 +282,26 @@ config CLKSRC_MPS2
 	select CLKSRC_MMIO
 	select CLKSRC_OF
 
+config ARC_TIMERS
+	bool "Support for 32-bit TIMERn counters in ARC Cores" if COMPILE_TEST
+	depends on GENERIC_CLOCKEVENTS
+	select CLKSRC_OF
+	help
+	  These are legacy 32-bit TIMER0 and TIMER1 counters found on all ARC cores
+	  (ARC700 as well as ARC HS38).
+	  TIMER0 serves as clockevent while TIMER1 provides clocksource
+
+config ARC_TIMERS_64BIT
+	bool "Support for 64-bit counters in ARC HS38 cores" if COMPILE_TEST
+	depends on GENERIC_CLOCKEVENTS
+	depends on ARC_TIMERS
+	select CLKSRC_OF
+	help
+	  This enables 2 different 64-bit timers: RTC (for UP) and GFRC (for SMP)
+	  RTC is implemented inside the core, while GFRC sits outside the core in
+	  ARConnect IP block. Driver automatically picks one of them for clocksource
+	  as appropriate.
+
 config ARM_ARCH_TIMER
 	bool
 	select CLKSRC_OF if OF
diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile
index cf87f407f1ad..a14111e1f087 100644
--- a/drivers/clocksource/Makefile
+++ b/drivers/clocksource/Makefile
@@ -51,6 +51,7 @@ obj-$(CONFIG_CLKSRC_TI_32K)	+= timer-ti-32k.o
 obj-$(CONFIG_CLKSRC_NPS)	+= timer-nps.o
 obj-$(CONFIG_OXNAS_RPS_TIMER)	+= timer-oxnas-rps.o
 
+obj-$(CONFIG_ARC_TIMERS)		+= arc_timer.o
 obj-$(CONFIG_ARM_ARCH_TIMER)		+= arm_arch_timer.o
 obj-$(CONFIG_ARM_GLOBAL_TIMER)		+= arm_global_timer.o
 obj-$(CONFIG_ARMV7M_SYSTICK)		+= armv7m_systick.o
diff --git a/arch/arc/kernel/time.c b/drivers/clocksource/arc_timer.c
index c10390d1ddb6..a49748d826c0 100644
--- a/arch/arc/kernel/time.c
+++ b/drivers/clocksource/arc_timer.c
@@ -1,32 +1,18 @@
 /*
+ * Copyright (C) 2016-17 Synopsys, Inc. (www.synopsys.com)
  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
- *
- * vineetg: Jan 1011
- *  -sched_clock( ) no longer jiffies based. Uses the same clocksource
- *   as gtod
- *
- * Rajeshwarr/Vineetg: Mar 2008
- *  -Implemented CONFIG_GENERIC_TIME (rather deleted arch specific code)
- *   for arch independent gettimeofday()
- *  -Implemented CONFIG_GENERIC_CLOCKEVENTS as base for hrtimers
- *
- * Vineetg: Mar 2008: Forked off from time.c which now is time-jiff.c
  */
 
-/* ARC700 has two 32bit independent prog Timers: TIMER0 and TIMER1
- * Each can programmed to go from @count to @limit and optionally
- * interrupt when that happens.
- * A write to Control Register clears the Interrupt
+/* ARC700 has two 32bit independent prog Timers: TIMER0 and TIMER1, Each can be
+ * programmed to go from @count to @limit and optionally interrupt.
+ * We've designated TIMER0 for clockevents and TIMER1 for clocksource
  *
- * We've designated TIMER0 for events (clockevents)
- * while TIMER1 for free running (clocksource)
- *
- * Newer ARC700 cores have 64bit clk fetching RTSC insn, preferred over TIMER1
- * which however is currently broken
+ * ARCv2 based HS38 cores have RTC (in-core) and GFRC (inside ARConnect/MCIP)
+ * which are suitable for UP and SMP based clocksources respectively
  */
 
 #include <linux/interrupt.h>
@@ -37,23 +23,10 @@
 #include <linux/cpu.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
-#include <asm/irq.h>
-#include <asm/arcregs.h>
-
-#include <asm/mcip.h>
 
-/* Timer related Aux registers */
-#define ARC_REG_TIMER0_LIMIT	0x23	/* timer 0 limit */
-#define ARC_REG_TIMER0_CTRL	0x22	/* timer 0 control */
-#define ARC_REG_TIMER0_CNT	0x21	/* timer 0 count */
-#define ARC_REG_TIMER1_LIMIT	0x102	/* timer 1 limit */
-#define ARC_REG_TIMER1_CTRL	0x101	/* timer 1 control */
-#define ARC_REG_TIMER1_CNT	0x100	/* timer 1 count */
+#include <soc/arc/timers.h>
+#include <soc/arc/mcip.h>
 
-#define TIMER_CTRL_IE	(1 << 0) /* Interrupt when Count reaches limit */
-#define TIMER_CTRL_NH	(1 << 1) /* Count only when CPU NOT halted */
-
-#define ARC_TIMER_MAX	0xFFFFFFFF
 
 static unsigned long arc_timer_freq;
 
@@ -81,31 +54,24 @@ static int noinline arc_get_timer_clk(struct device_node *node)
 
 /********** Clock Source Device *********/
 
-#ifdef CONFIG_ARC_HAS_GFRC
+#ifdef CONFIG_ARC_TIMERS_64BIT
 
 static cycle_t arc_read_gfrc(struct clocksource *cs)
 {
 	unsigned long flags;
-	union {
-#ifdef CONFIG_CPU_BIG_ENDIAN
-		struct { u32 h, l; };
-#else
-		struct { u32 l, h; };
-#endif
-		cycle_t  full;
-	} stamp;
+	u32 l, h;
 
 	local_irq_save(flags);
 
 	__mcip_cmd(CMD_GFRC_READ_LO, 0);
-	stamp.l = read_aux_reg(ARC_REG_MCIP_READBACK);
+	l = read_aux_reg(ARC_REG_MCIP_READBACK);
 
 	__mcip_cmd(CMD_GFRC_READ_HI, 0);
-	stamp.h = read_aux_reg(ARC_REG_MCIP_READBACK);
+	h = read_aux_reg(ARC_REG_MCIP_READBACK);
 
 	local_irq_restore(flags);
 
-	return stamp.full;
+	return (((cycle_t)h) << 32) | l;
 }
 
 static struct clocksource arc_counter_gfrc = {
@@ -118,11 +84,14 @@ static struct clocksource arc_counter_gfrc = {
 
 static int __init arc_cs_setup_gfrc(struct device_node *node)
 {
-	int exists = cpuinfo_arc700[0].extn.gfrc;
+	struct mcip_bcr mp;
 	int ret;
 
-	if (WARN(!exists, "Global-64-bit-Ctr clocksource not detected"))
+	READ_BCR(ARC_REG_MCIP_BCR, mp);
+	if (!mp.gfrc) {
+		pr_warn("Global-64-bit-Ctr clocksource not detected");
 		return -ENXIO;
+	}
 
 	ret = arc_get_timer_clk(node);
 	if (ret)
@@ -132,10 +101,6 @@ static int __init arc_cs_setup_gfrc(struct device_node *node)
 }
 CLOCKSOURCE_OF_DECLARE(arc_gfrc, "snps,archs-timer-gfrc", arc_cs_setup_gfrc);
 
-#endif
-
-#ifdef CONFIG_ARC_HAS_RTC
-
 #define AUX_RTC_CTRL	0x103
 #define AUX_RTC_LOW	0x104
 #define AUX_RTC_HIGH	0x105
@@ -143,14 +108,7 @@ CLOCKSOURCE_OF_DECLARE(arc_gfrc, "snps,archs-timer-gfrc", arc_cs_setup_gfrc);
 static cycle_t arc_read_rtc(struct clocksource *cs)
 {
 	unsigned long status;
-	union {
-#ifdef CONFIG_CPU_BIG_ENDIAN
-		struct { u32 high, low; };
-#else
-		struct { u32 low, high; };
-#endif
-		cycle_t  full;
-	} stamp;
+	u32 l, h;
 
 	/*
 	 * hardware has an internal state machine which tracks readout of
@@ -159,12 +117,12 @@ static cycle_t arc_read_rtc(struct clocksource *cs)
 	 *  - high increments after low has been read
 	 */
 	do {
-		stamp.low = read_aux_reg(AUX_RTC_LOW);
-		stamp.high = read_aux_reg(AUX_RTC_HIGH);
+		l = read_aux_reg(AUX_RTC_LOW);
+		h = read_aux_reg(AUX_RTC_HIGH);
 		status = read_aux_reg(AUX_RTC_CTRL);
 	} while (!(status & _BITUL(31)));
 
-	return stamp.full;
+	return (((cycle_t)h) << 32) | l;
 }
 
 static struct clocksource arc_counter_rtc = {
@@ -177,15 +135,20 @@ static struct clocksource arc_counter_rtc = {
 
 static int __init arc_cs_setup_rtc(struct device_node *node)
 {
-	int exists = cpuinfo_arc700[smp_processor_id()].extn.rtc;
+	struct bcr_timer timer;
 	int ret;
 
-	if (WARN(!exists, "Local-64-bit-Ctr clocksource not detected"))
+	READ_BCR(ARC_REG_TIMERS_BCR, timer);
+	if (!timer.rtc) {
+		pr_warn("Local-64-bit-Ctr clocksource not detected");
 		return -ENXIO;
+	}
 
 	/* Local to CPU hence not usable in SMP */
-	if (WARN(IS_ENABLED(CONFIG_SMP), "Local-64-bit-Ctr not usable in SMP"))
+	if (IS_ENABLED(CONFIG_SMP)) {
+		pr_warn("Local-64-bit-Ctr not usable in SMP");
 		return -EINVAL;
+	}
 
 	ret = arc_get_timer_clk(node);
 	if (ret)
@@ -228,7 +191,7 @@ static int __init arc_cs_setup_timer1(struct device_node *node)
 	if (ret)
 		return ret;
 
-	write_aux_reg(ARC_REG_TIMER1_LIMIT, ARC_TIMER_MAX);
+	write_aux_reg(ARC_REG_TIMER1_LIMIT, ARC_TIMERN_MAX);
 	write_aux_reg(ARC_REG_TIMER1_CNT, 0);
 	write_aux_reg(ARC_REG_TIMER1_CTRL, TIMER_CTRL_NH);
 
@@ -306,7 +269,7 @@ static int arc_timer_starting_cpu(unsigned int cpu)
 
 	evt->cpumask = cpumask_of(smp_processor_id());
 
-	clockevents_config_and_register(evt, arc_timer_freq, 0, ARC_TIMER_MAX);
+	clockevents_config_and_register(evt, arc_timer_freq, 0, ARC_TIMERN_MAX);
 	enable_percpu_irq(arc_timer_irq, 0);
 	return 0;
 }
@@ -371,12 +334,3 @@ static int __init arc_of_timer_init(struct device_node *np)
 	return ret;
 }
 CLOCKSOURCE_OF_DECLARE(arc_clkevt, "snps,arc-timer", arc_of_timer_init);
-
-/*
- * Called from start_kernel() - boot CPU only
- */
-void __init time_init(void)
-{
-	of_clk_init(NULL);
-	clocksource_probe();
-}
diff --git a/drivers/clocksource/pxa_timer.c b/drivers/clocksource/pxa_timer.c
index 3e1cb512f3ce..9cae38eebec2 100644
--- a/drivers/clocksource/pxa_timer.c
+++ b/drivers/clocksource/pxa_timer.c
@@ -220,17 +220,16 @@ CLOCKSOURCE_OF_DECLARE(pxa_timer, "marvell,pxa-timer", pxa_timer_dt_init);
 /*
  * Legacy timer init for non device-tree boards.
  */
-void __init pxa_timer_nodt_init(int irq, void __iomem *base,
-	unsigned long clock_tick_rate)
+void __init pxa_timer_nodt_init(int irq, void __iomem *base)
 {
 	struct clk *clk;
 
 	timer_base = base;
 	clk = clk_get(NULL, "OSTIMER0");
-	if (clk && !IS_ERR(clk))
+	if (clk && !IS_ERR(clk)) {
 		clk_prepare_enable(clk);
-	else
+		pxa_timer_common_init(irq, clk_get_rate(clk));
+	} else {
 		pr_crit("%s: unable to get clk\n", __func__);
-
-	pxa_timer_common_init(irq, clock_tick_rate);
+	}
 }
diff --git a/drivers/clocksource/timer-nps.c b/drivers/clocksource/timer-nps.c
index 70c149af8ee0..8da5e93b6810 100644
--- a/drivers/clocksource/timer-nps.c
+++ b/drivers/clocksource/timer-nps.c
@@ -46,7 +46,36 @@
 /* This array is per cluster of CPUs (Each NPS400 cluster got 256 CPUs) */
 static void *nps_msu_reg_low_addr[NPS_CLUSTER_NUM] __read_mostly;
 
-static unsigned long nps_timer_rate;
+static int __init nps_get_timer_clk(struct device_node *node,
+			     unsigned long *timer_freq,
+			     struct clk **clk)
+{
+	int ret;
+
+	*clk = of_clk_get(node, 0);
+	ret = PTR_ERR_OR_ZERO(*clk);
+	if (ret) {
+		pr_err("timer missing clk");
+		return ret;
+	}
+
+	ret = clk_prepare_enable(*clk);
+	if (ret) {
+		pr_err("Couldn't enable parent clk\n");
+		clk_put(*clk);
+		return ret;
+	}
+
+	*timer_freq = clk_get_rate(*clk);
+	if (!(*timer_freq)) {
+		pr_err("Couldn't get clk rate\n");
+		clk_disable_unprepare(*clk);
+		clk_put(*clk);
+		return -EINVAL;
+	}
+
+	return 0;
+}
 
 static cycle_t nps_clksrc_read(struct clocksource *clksrc)
 {
@@ -55,26 +84,24 @@ static cycle_t nps_clksrc_read(struct clocksource *clksrc)
 	return (cycle_t)ioread32be(nps_msu_reg_low_addr[cluster]);
 }
 
-static int __init nps_setup_clocksource(struct device_node *node,
-					struct clk *clk)
+static int __init nps_setup_clocksource(struct device_node *node)
 {
 	int ret, cluster;
+	struct clk *clk;
+	unsigned long nps_timer1_freq;
+
 
 	for (cluster = 0; cluster < NPS_CLUSTER_NUM; cluster++)
 		nps_msu_reg_low_addr[cluster] =
 			nps_host_reg((cluster << NPS_CLUSTER_OFFSET),
-				 NPS_MSU_BLKID, NPS_MSU_TICK_LOW);
+				     NPS_MSU_BLKID, NPS_MSU_TICK_LOW);
 
-	ret = clk_prepare_enable(clk);
-	if (ret) {
-		pr_err("Couldn't enable parent clock\n");
+	ret = nps_get_timer_clk(node, &nps_timer1_freq, &clk);
+	if (ret)
 		return ret;
-	}
-
-	nps_timer_rate = clk_get_rate(clk);
 
-	ret = clocksource_mmio_init(nps_msu_reg_low_addr, "EZnps-tick",
-				    nps_timer_rate, 301, 32, nps_clksrc_read);
+	ret = clocksource_mmio_init(nps_msu_reg_low_addr, "nps-tick",
+				    nps_timer1_freq, 300, 32, nps_clksrc_read);
 	if (ret) {
 		pr_err("Couldn't register clock source.\n");
 		clk_disable_unprepare(clk);
@@ -83,18 +110,175 @@ static int __init nps_setup_clocksource(struct device_node *node,
 	return ret;
 }
 
-static int __init nps_timer_init(struct device_node *node)
+CLOCKSOURCE_OF_DECLARE(ezchip_nps400_clksrc, "ezchip,nps400-timer",
+		       nps_setup_clocksource);
+CLOCKSOURCE_OF_DECLARE(ezchip_nps400_clk_src, "ezchip,nps400-timer1",
+		       nps_setup_clocksource);
+
+#ifdef CONFIG_EZNPS_MTM_EXT
+#include <soc/nps/mtm.h>
+
+/* Timer related Aux registers */
+#define NPS_REG_TIMER0_TSI	0xFFFFF850
+#define NPS_REG_TIMER0_LIMIT	0x23
+#define NPS_REG_TIMER0_CTRL	0x22
+#define NPS_REG_TIMER0_CNT	0x21
+
+/*
+ * Interrupt Enabled (IE) - re-arm the timer
+ * Not Halted (NH) - is cleared when working with JTAG (for debug)
+ */
+#define TIMER0_CTRL_IE		BIT(0)
+#define TIMER0_CTRL_NH		BIT(1)
+
+static unsigned long nps_timer0_freq;
+static unsigned long nps_timer0_irq;
+
+static void nps_clkevent_rm_thread(void)
+{
+	int thread;
+	unsigned int cflags, enabled_threads;
+
+	hw_schd_save(&cflags);
+
+	enabled_threads = read_aux_reg(NPS_REG_TIMER0_TSI);
+
+	/* remove thread from TSI1 */
+	thread = read_aux_reg(CTOP_AUX_THREAD_ID);
+	enabled_threads &= ~(1 << thread);
+	write_aux_reg(NPS_REG_TIMER0_TSI, enabled_threads);
+
+	/* Acknowledge and if needed re-arm the timer */
+	if (!enabled_threads)
+		write_aux_reg(NPS_REG_TIMER0_CTRL, TIMER0_CTRL_NH);
+	else
+		write_aux_reg(NPS_REG_TIMER0_CTRL,
+			      TIMER0_CTRL_IE | TIMER0_CTRL_NH);
+
+	hw_schd_restore(cflags);
+}
+
+static void nps_clkevent_add_thread(unsigned long delta)
+{
+	int thread;
+	unsigned int cflags, enabled_threads;
+
+	hw_schd_save(&cflags);
+
+	/* add thread to TSI1 */
+	thread = read_aux_reg(CTOP_AUX_THREAD_ID);
+	enabled_threads = read_aux_reg(NPS_REG_TIMER0_TSI);
+	enabled_threads |= (1 << thread);
+	write_aux_reg(NPS_REG_TIMER0_TSI, enabled_threads);
+
+	/* set next timer event */
+	write_aux_reg(NPS_REG_TIMER0_LIMIT, delta);
+	write_aux_reg(NPS_REG_TIMER0_CNT, 0);
+	write_aux_reg(NPS_REG_TIMER0_CTRL,
+		      TIMER0_CTRL_IE | TIMER0_CTRL_NH);
+
+	hw_schd_restore(cflags);
+}
+
+/*
+ * Whenever anyone tries to change modes, we just mask interrupts
+ * and wait for the next event to get set.
+ */
+static int nps_clkevent_set_state(struct clock_event_device *dev)
+{
+	nps_clkevent_rm_thread();
+	disable_percpu_irq(nps_timer0_irq);
+
+	return 0;
+}
+
+static int nps_clkevent_set_next_event(unsigned long delta,
+				       struct clock_event_device *dev)
+{
+	nps_clkevent_add_thread(delta);
+	enable_percpu_irq(nps_timer0_irq, IRQ_TYPE_NONE);
+
+	return 0;
+}
+
+static DEFINE_PER_CPU(struct clock_event_device, nps_clockevent_device) = {
+	.name				=	"NPS Timer0",
+	.features			=	CLOCK_EVT_FEAT_ONESHOT,
+	.rating				=	300,
+	.set_next_event			=	nps_clkevent_set_next_event,
+	.set_state_oneshot		=	nps_clkevent_set_state,
+	.set_state_oneshot_stopped	=	nps_clkevent_set_state,
+	.set_state_shutdown		=	nps_clkevent_set_state,
+	.tick_resume			=	nps_clkevent_set_state,
+};
+
+static irqreturn_t timer_irq_handler(int irq, void *dev_id)
+{
+	struct clock_event_device *evt = dev_id;
+
+	nps_clkevent_rm_thread();
+	evt->event_handler(evt);
+
+	return IRQ_HANDLED;
+}
+
+static int nps_timer_starting_cpu(unsigned int cpu)
+{
+	struct clock_event_device *evt = this_cpu_ptr(&nps_clockevent_device);
+
+	evt->cpumask = cpumask_of(smp_processor_id());
+
+	clockevents_config_and_register(evt, nps_timer0_freq, 0, ULONG_MAX);
+	enable_percpu_irq(nps_timer0_irq, IRQ_TYPE_NONE);
+
+	return 0;
+}
+
+static int nps_timer_dying_cpu(unsigned int cpu)
+{
+	disable_percpu_irq(nps_timer0_irq);
+	return 0;
+}
+
+static int __init nps_setup_clockevent(struct device_node *node)
 {
 	struct clk *clk;
+	int ret;
 
-	clk = of_clk_get(node, 0);
-	if (IS_ERR(clk)) {
-		pr_err("Can't get timer clock.\n");
-		return PTR_ERR(clk);
+	nps_timer0_irq = irq_of_parse_and_map(node, 0);
+	if (nps_timer0_irq <= 0) {
+		pr_err("clockevent: missing irq");
+		return -EINVAL;
 	}
 
-	return nps_setup_clocksource(node, clk);
+	ret = nps_get_timer_clk(node, &nps_timer0_freq, &clk);
+	if (ret)
+		return ret;
+
+	/* Needs apriori irq_set_percpu_devid() done in intc map function */
+	ret = request_percpu_irq(nps_timer0_irq, timer_irq_handler,
+				 "Timer0 (per-cpu-tick)",
+				 &nps_clockevent_device);
+	if (ret) {
+		pr_err("Couldn't request irq\n");
+		clk_disable_unprepare(clk);
+		return ret;
+	}
+
+	ret = cpuhp_setup_state(CPUHP_AP_ARC_TIMER_STARTING,
+				"clockevents/nps:starting",
+				nps_timer_starting_cpu,
+				nps_timer_dying_cpu);
+	if (ret) {
+		pr_err("Failed to setup hotplug state");
+		clk_disable_unprepare(clk);
+		free_percpu_irq(nps_timer0_irq, &nps_clockevent_device);
+		return ret;
+	}
+
+	return 0;
 }
 
-CLOCKSOURCE_OF_DECLARE(ezchip_nps400_clksrc, "ezchip,nps400-timer",
-		       nps_timer_init);
+CLOCKSOURCE_OF_DECLARE(ezchip_nps400_clk_evt, "ezchip,nps400-timer0",
+		       nps_setup_clockevent);
+#endif /* CONFIG_EZNPS_MTM_EXT */
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
index d89b8afe23b6..920c469f3953 100644
--- a/drivers/cpufreq/Kconfig.arm
+++ b/drivers/cpufreq/Kconfig.arm
@@ -12,6 +12,27 @@ config ARM_BIG_LITTLE_CPUFREQ
 	help
 	  This enables the Generic CPUfreq driver for ARM big.LITTLE platforms.
 
+config ARM_BRCMSTB_AVS_CPUFREQ
+	tristate "Broadcom STB AVS CPUfreq driver"
+	depends on ARCH_BRCMSTB || COMPILE_TEST
+	default y
+	help
+	  Some Broadcom STB SoCs use a co-processor running proprietary firmware
+	  ("AVS") to handle voltage and frequency scaling. This driver provides
+	  a standard CPUfreq interface to to the firmware.
+
+	  Say Y, if you have a Broadcom SoC with AVS support for DFS or DVFS.
+
+config ARM_BRCMSTB_AVS_CPUFREQ_DEBUG
+	bool "Broadcom STB AVS CPUfreq driver sysfs debug capability"
+	depends on ARM_BRCMSTB_AVS_CPUFREQ
+	help
+	  Enabling this option turns on debug support via sysfs under
+	  /sys/kernel/debug/brcmstb-avs-cpufreq. It is possible to read all and
+	  write some AVS mailbox registers through sysfs entries.
+
+	  If in doubt, say N.
+
 config ARM_DT_BL_CPUFREQ
 	tristate "Generic probing via DT for ARM big LITTLE CPUfreq driver"
 	depends on ARM_BIG_LITTLE_CPUFREQ && OF
@@ -60,14 +81,6 @@ config ARM_IMX6Q_CPUFREQ
 
 	  If in doubt, say N.
 
-config ARM_INTEGRATOR
-	tristate "CPUfreq driver for ARM Integrator CPUs"
-	depends on ARCH_INTEGRATOR
-	default y
-	help
-	  This enables the CPUfreq driver for ARM Integrator CPUs.
-	  If in doubt, say Y.
-
 config ARM_KIRKWOOD_CPUFREQ
 	def_bool MACH_KIRKWOOD
 	help
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index 0a9b6a093646..1e46c3918e7a 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -51,12 +51,12 @@ obj-$(CONFIG_ARM_BIG_LITTLE_CPUFREQ)	+= arm_big_little.o
 # LITTLE drivers, so that it is probed last.
 obj-$(CONFIG_ARM_DT_BL_CPUFREQ)		+= arm_big_little_dt.o
 
+obj-$(CONFIG_ARM_BRCMSTB_AVS_CPUFREQ)	+= brcmstb-avs-cpufreq.o
 obj-$(CONFIG_ARCH_DAVINCI)		+= davinci-cpufreq.o
 obj-$(CONFIG_UX500_SOC_DB8500)		+= dbx500-cpufreq.o
 obj-$(CONFIG_ARM_EXYNOS5440_CPUFREQ)	+= exynos5440-cpufreq.o
 obj-$(CONFIG_ARM_HIGHBANK_CPUFREQ)	+= highbank-cpufreq.o
 obj-$(CONFIG_ARM_IMX6Q_CPUFREQ)		+= imx6q-cpufreq.o
-obj-$(CONFIG_ARM_INTEGRATOR)		+= integrator-cpufreq.o
 obj-$(CONFIG_ARM_KIRKWOOD_CPUFREQ)	+= kirkwood-cpufreq.o
 obj-$(CONFIG_ARM_MT8173_CPUFREQ)	+= mt8173-cpufreq.o
 obj-$(CONFIG_ARM_OMAP2PLUS_CPUFREQ)	+= omap-cpufreq.o
diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c
index 297e9128fe9f..3a98702b7445 100644
--- a/drivers/cpufreq/acpi-cpufreq.c
+++ b/drivers/cpufreq/acpi-cpufreq.c
@@ -84,7 +84,6 @@ static inline struct acpi_processor_performance *to_perf_data(struct acpi_cpufre
 static struct cpufreq_driver acpi_cpufreq_driver;
 
 static unsigned int acpi_pstate_strict;
-static struct msr __percpu *msrs;
 
 static bool boost_state(unsigned int cpu)
 {
@@ -104,11 +103,10 @@ static bool boost_state(unsigned int cpu)
 	return false;
 }
 
-static void boost_set_msrs(bool enable, const struct cpumask *cpumask)
+static int boost_set_msr(bool enable)
 {
-	u32 cpu;
 	u32 msr_addr;
-	u64 msr_mask;
+	u64 msr_mask, val;
 
 	switch (boot_cpu_data.x86_vendor) {
 	case X86_VENDOR_INTEL:
@@ -120,26 +118,31 @@ static void boost_set_msrs(bool enable, const struct cpumask *cpumask)
 		msr_mask = MSR_K7_HWCR_CPB_DIS;
 		break;
 	default:
-		return;
+		return -EINVAL;
 	}
 
-	rdmsr_on_cpus(cpumask, msr_addr, msrs);
+	rdmsrl(msr_addr, val);
 
-	for_each_cpu(cpu, cpumask) {
-		struct msr *reg = per_cpu_ptr(msrs, cpu);
-		if (enable)
-			reg->q &= ~msr_mask;
-		else
-			reg->q |= msr_mask;
-	}
+	if (enable)
+		val &= ~msr_mask;
+	else
+		val |= msr_mask;
+
+	wrmsrl(msr_addr, val);
+	return 0;
+}
+
+static void boost_set_msr_each(void *p_en)
+{
+	bool enable = (bool) p_en;
 
-	wrmsr_on_cpus(cpumask, msr_addr, msrs);
+	boost_set_msr(enable);
 }
 
 static int set_boost(int val)
 {
 	get_online_cpus();
-	boost_set_msrs(val, cpu_online_mask);
+	on_each_cpu(boost_set_msr_each, (void *)(long)val, 1);
 	put_online_cpus();
 	pr_debug("Core Boosting %sabled.\n", val ? "en" : "dis");
 
@@ -536,46 +539,24 @@ static void free_acpi_perf_data(void)
 	free_percpu(acpi_perf_data);
 }
 
-static int boost_notify(struct notifier_block *nb, unsigned long action,
-		      void *hcpu)
+static int cpufreq_boost_online(unsigned int cpu)
 {
-	unsigned cpu = (long)hcpu;
-	const struct cpumask *cpumask;
-
-	cpumask = get_cpu_mask(cpu);
+	/*
+	 * On the CPU_UP path we simply keep the boost-disable flag
+	 * in sync with the current global state.
+	 */
+	return boost_set_msr(acpi_cpufreq_driver.boost_enabled);
+}
 
+static int cpufreq_boost_down_prep(unsigned int cpu)
+{
 	/*
 	 * Clear the boost-disable bit on the CPU_DOWN path so that
-	 * this cpu cannot block the remaining ones from boosting. On
-	 * the CPU_UP path we simply keep the boost-disable flag in
-	 * sync with the current global state.
+	 * this cpu cannot block the remaining ones from boosting.
 	 */
-
-	switch (action) {
-	case CPU_DOWN_FAILED:
-	case CPU_DOWN_FAILED_FROZEN:
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		boost_set_msrs(acpi_cpufreq_driver.boost_enabled, cpumask);
-		break;
-
-	case CPU_DOWN_PREPARE:
-	case CPU_DOWN_PREPARE_FROZEN:
-		boost_set_msrs(1, cpumask);
-		break;
-
-	default:
-		break;
-	}
-
-	return NOTIFY_OK;
+	return boost_set_msr(1);
 }
 
-
-static struct notifier_block boost_nb = {
-	.notifier_call          = boost_notify,
-};
-
 /*
  * acpi_cpufreq_early_init - initialize ACPI P-States library
  *
@@ -922,37 +903,35 @@ static struct cpufreq_driver acpi_cpufreq_driver = {
 	.attr		= acpi_cpufreq_attr,
 };
 
+static enum cpuhp_state acpi_cpufreq_online;
+
 static void __init acpi_cpufreq_boost_init(void)
 {
-	if (boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA)) {
-		msrs = msrs_alloc();
-
-		if (!msrs)
-			return;
-
-		acpi_cpufreq_driver.set_boost = set_boost;
-		acpi_cpufreq_driver.boost_enabled = boost_state(0);
-
-		cpu_notifier_register_begin();
+	int ret;
 
-		/* Force all MSRs to the same value */
-		boost_set_msrs(acpi_cpufreq_driver.boost_enabled,
-			       cpu_online_mask);
+	if (!(boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA)))
+		return;
 
-		__register_cpu_notifier(&boost_nb);
+	acpi_cpufreq_driver.set_boost = set_boost;
+	acpi_cpufreq_driver.boost_enabled = boost_state(0);
 
-		cpu_notifier_register_done();
+	/*
+	 * This calls the online callback on all online cpu and forces all
+	 * MSRs to the same value.
+	 */
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "cpufreq/acpi:online",
+				cpufreq_boost_online, cpufreq_boost_down_prep);
+	if (ret < 0) {
+		pr_err("acpi_cpufreq: failed to register hotplug callbacks\n");
+		return;
 	}
+	acpi_cpufreq_online = ret;
 }
 
 static void acpi_cpufreq_boost_exit(void)
 {
-	if (msrs) {
-		unregister_cpu_notifier(&boost_nb);
-
-		msrs_free(msrs);
-		msrs = NULL;
-	}
+	if (acpi_cpufreq_online >= 0)
+		cpuhp_remove_state_nocalls(acpi_cpufreq_online);
 }
 
 static int __init acpi_cpufreq_init(void)
diff --git a/drivers/cpufreq/brcmstb-avs-cpufreq.c b/drivers/cpufreq/brcmstb-avs-cpufreq.c
new file mode 100644
index 000000000000..4fda623e55bb
--- /dev/null
+++ b/drivers/cpufreq/brcmstb-avs-cpufreq.c
@@ -0,0 +1,1057 @@
+/*
+ * CPU frequency scaling for Broadcom SoCs with AVS firmware that
+ * supports DVS or DVFS
+ *
+ * Copyright (c) 2016 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+/*
+ * "AVS" is the name of a firmware developed at Broadcom. It derives
+ * its name from the technique called "Adaptive Voltage Scaling".
+ * Adaptive voltage scaling was the original purpose of this firmware.
+ * The AVS firmware still supports "AVS mode", where all it does is
+ * adaptive voltage scaling. However, on some newer Broadcom SoCs, the
+ * AVS Firmware, despite its unchanged name, also supports DFS mode and
+ * DVFS mode.
+ *
+ * In the context of this document and the related driver, "AVS" by
+ * itself always means the Broadcom firmware and never refers to the
+ * technique called "Adaptive Voltage Scaling".
+ *
+ * The Broadcom STB AVS CPUfreq driver provides voltage and frequency
+ * scaling on Broadcom SoCs using AVS firmware with support for DFS and
+ * DVFS. The AVS firmware is running on its own co-processor. The
+ * driver supports both uniprocessor (UP) and symmetric multiprocessor
+ * (SMP) systems which share clock and voltage across all CPUs.
+ *
+ * Actual voltage and frequency scaling is done solely by the AVS
+ * firmware. This driver does not change frequency or voltage itself.
+ * It provides a standard CPUfreq interface to the rest of the kernel
+ * and to userland. It interfaces with the AVS firmware to effect the
+ * requested changes and to report back the current system status in a
+ * way that is expected by existing tools.
+ */
+
+#include <linux/cpufreq.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/semaphore.h>
+
+#ifdef CONFIG_ARM_BRCMSTB_AVS_CPUFREQ_DEBUG
+#include <linux/ctype.h>
+#include <linux/debugfs.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#endif
+
+/* Max number of arguments AVS calls take */
+#define AVS_MAX_CMD_ARGS	4
+/*
+ * This macro is used to generate AVS parameter register offsets. For
+ * x >= AVS_MAX_CMD_ARGS, it returns 0 to protect against accidental memory
+ * access outside of the parameter range. (Offset 0 is the first parameter.)
+ */
+#define AVS_PARAM_MULT(x)	((x) < AVS_MAX_CMD_ARGS ? (x) : 0)
+
+/* AVS Mailbox Register offsets */
+#define AVS_MBOX_COMMAND	0x00
+#define AVS_MBOX_STATUS		0x04
+#define AVS_MBOX_VOLTAGE0	0x08
+#define AVS_MBOX_TEMP0		0x0c
+#define AVS_MBOX_PV0		0x10
+#define AVS_MBOX_MV0		0x14
+#define AVS_MBOX_PARAM(x)	(0x18 + AVS_PARAM_MULT(x) * sizeof(u32))
+#define AVS_MBOX_REVISION	0x28
+#define AVS_MBOX_PSTATE		0x2c
+#define AVS_MBOX_HEARTBEAT	0x30
+#define AVS_MBOX_MAGIC		0x34
+#define AVS_MBOX_SIGMA_HVT	0x38
+#define AVS_MBOX_SIGMA_SVT	0x3c
+#define AVS_MBOX_VOLTAGE1	0x40
+#define AVS_MBOX_TEMP1		0x44
+#define AVS_MBOX_PV1		0x48
+#define AVS_MBOX_MV1		0x4c
+#define AVS_MBOX_FREQUENCY	0x50
+
+/* AVS Commands */
+#define AVS_CMD_AVAILABLE	0x00
+#define AVS_CMD_DISABLE		0x10
+#define AVS_CMD_ENABLE		0x11
+#define AVS_CMD_S2_ENTER	0x12
+#define AVS_CMD_S2_EXIT		0x13
+#define AVS_CMD_BBM_ENTER	0x14
+#define AVS_CMD_BBM_EXIT	0x15
+#define AVS_CMD_S3_ENTER	0x16
+#define AVS_CMD_S3_EXIT		0x17
+#define AVS_CMD_BALANCE		0x18
+/* PMAP and P-STATE commands */
+#define AVS_CMD_GET_PMAP	0x30
+#define AVS_CMD_SET_PMAP	0x31
+#define AVS_CMD_GET_PSTATE	0x40
+#define AVS_CMD_SET_PSTATE	0x41
+
+/* Different modes AVS supports (for GET_PMAP/SET_PMAP) */
+#define AVS_MODE_AVS		0x0
+#define AVS_MODE_DFS		0x1
+#define AVS_MODE_DVS		0x2
+#define AVS_MODE_DVFS		0x3
+
+/*
+ * PMAP parameter p1
+ * unused:31-24, mdiv_p0:23-16, unused:15-14, pdiv:13-10 , ndiv_int:9-0
+ */
+#define NDIV_INT_SHIFT		0
+#define NDIV_INT_MASK		0x3ff
+#define PDIV_SHIFT		10
+#define PDIV_MASK		0xf
+#define MDIV_P0_SHIFT		16
+#define MDIV_P0_MASK		0xff
+/*
+ * PMAP parameter p2
+ * mdiv_p4:31-24, mdiv_p3:23-16, mdiv_p2:15:8, mdiv_p1:7:0
+ */
+#define MDIV_P1_SHIFT		0
+#define MDIV_P1_MASK		0xff
+#define MDIV_P2_SHIFT		8
+#define MDIV_P2_MASK		0xff
+#define MDIV_P3_SHIFT		16
+#define MDIV_P3_MASK		0xff
+#define MDIV_P4_SHIFT		24
+#define MDIV_P4_MASK		0xff
+
+/* Different P-STATES AVS supports (for GET_PSTATE/SET_PSTATE) */
+#define AVS_PSTATE_P0		0x0
+#define AVS_PSTATE_P1		0x1
+#define AVS_PSTATE_P2		0x2
+#define AVS_PSTATE_P3		0x3
+#define AVS_PSTATE_P4		0x4
+#define AVS_PSTATE_MAX		AVS_PSTATE_P4
+
+/* CPU L2 Interrupt Controller Registers */
+#define AVS_CPU_L2_SET0		0x04
+#define AVS_CPU_L2_INT_MASK	BIT(31)
+
+/* AVS Command Status Values */
+#define AVS_STATUS_CLEAR	0x00
+/* Command/notification accepted */
+#define AVS_STATUS_SUCCESS	0xf0
+/* Command/notification rejected */
+#define AVS_STATUS_FAILURE	0xff
+/* Invalid command/notification (unknown) */
+#define AVS_STATUS_INVALID	0xf1
+/* Non-AVS modes are not supported */
+#define AVS_STATUS_NO_SUPP	0xf2
+/* Cannot set P-State until P-Map supplied */
+#define AVS_STATUS_NO_MAP	0xf3
+/* Cannot change P-Map after initial P-Map set */
+#define AVS_STATUS_MAP_SET	0xf4
+/* Max AVS status; higher numbers are used for debugging */
+#define AVS_STATUS_MAX		0xff
+
+/* Other AVS related constants */
+#define AVS_LOOP_LIMIT		10000
+#define AVS_TIMEOUT		300 /* in ms; expected completion is < 10ms */
+#define AVS_FIRMWARE_MAGIC	0xa11600d1
+
+#define BRCM_AVS_CPUFREQ_PREFIX	"brcmstb-avs"
+#define BRCM_AVS_CPUFREQ_NAME	BRCM_AVS_CPUFREQ_PREFIX "-cpufreq"
+#define BRCM_AVS_CPU_DATA	"brcm,avs-cpu-data-mem"
+#define BRCM_AVS_CPU_INTR	"brcm,avs-cpu-l2-intr"
+#define BRCM_AVS_HOST_INTR	"sw_intr"
+
+struct pmap {
+	unsigned int mode;
+	unsigned int p1;
+	unsigned int p2;
+	unsigned int state;
+};
+
+struct private_data {
+	void __iomem *base;
+	void __iomem *avs_intr_base;
+	struct device *dev;
+#ifdef CONFIG_ARM_BRCMSTB_AVS_CPUFREQ_DEBUG
+	struct dentry *debugfs;
+#endif
+	struct completion done;
+	struct semaphore sem;
+	struct pmap pmap;
+};
+
+#ifdef CONFIG_ARM_BRCMSTB_AVS_CPUFREQ_DEBUG
+
+enum debugfs_format {
+	DEBUGFS_NORMAL,
+	DEBUGFS_FLOAT,
+	DEBUGFS_REV,
+};
+
+struct debugfs_data {
+	struct debugfs_entry *entry;
+	struct private_data *priv;
+};
+
+struct debugfs_entry {
+	char *name;
+	u32 offset;
+	fmode_t mode;
+	enum debugfs_format format;
+};
+
+#define DEBUGFS_ENTRY(name, mode, format)	{ \
+	#name, AVS_MBOX_##name, mode, format \
+}
+
+/*
+ * These are used for debugfs only. Otherwise we use AVS_MBOX_PARAM() directly.
+ */
+#define AVS_MBOX_PARAM1		AVS_MBOX_PARAM(0)
+#define AVS_MBOX_PARAM2		AVS_MBOX_PARAM(1)
+#define AVS_MBOX_PARAM3		AVS_MBOX_PARAM(2)
+#define AVS_MBOX_PARAM4		AVS_MBOX_PARAM(3)
+
+/*
+ * This table stores the name, access permissions and offset for each hardware
+ * register and is used to generate debugfs entries.
+ */
+static struct debugfs_entry debugfs_entries[] = {
+	DEBUGFS_ENTRY(COMMAND, S_IWUSR, DEBUGFS_NORMAL),
+	DEBUGFS_ENTRY(STATUS, S_IWUSR, DEBUGFS_NORMAL),
+	DEBUGFS_ENTRY(VOLTAGE0, 0, DEBUGFS_FLOAT),
+	DEBUGFS_ENTRY(TEMP0, 0, DEBUGFS_FLOAT),
+	DEBUGFS_ENTRY(PV0, 0, DEBUGFS_FLOAT),
+	DEBUGFS_ENTRY(MV0, 0, DEBUGFS_FLOAT),
+	DEBUGFS_ENTRY(PARAM1, S_IWUSR, DEBUGFS_NORMAL),
+	DEBUGFS_ENTRY(PARAM2, S_IWUSR, DEBUGFS_NORMAL),
+	DEBUGFS_ENTRY(PARAM3, S_IWUSR, DEBUGFS_NORMAL),
+	DEBUGFS_ENTRY(PARAM4, S_IWUSR, DEBUGFS_NORMAL),
+	DEBUGFS_ENTRY(REVISION, 0, DEBUGFS_REV),
+	DEBUGFS_ENTRY(PSTATE, 0, DEBUGFS_NORMAL),
+	DEBUGFS_ENTRY(HEARTBEAT, 0, DEBUGFS_NORMAL),
+	DEBUGFS_ENTRY(MAGIC, S_IWUSR, DEBUGFS_NORMAL),
+	DEBUGFS_ENTRY(SIGMA_HVT, 0, DEBUGFS_NORMAL),
+	DEBUGFS_ENTRY(SIGMA_SVT, 0, DEBUGFS_NORMAL),
+	DEBUGFS_ENTRY(VOLTAGE1, 0, DEBUGFS_FLOAT),
+	DEBUGFS_ENTRY(TEMP1, 0, DEBUGFS_FLOAT),
+	DEBUGFS_ENTRY(PV1, 0, DEBUGFS_FLOAT),
+	DEBUGFS_ENTRY(MV1, 0, DEBUGFS_FLOAT),
+	DEBUGFS_ENTRY(FREQUENCY, 0, DEBUGFS_NORMAL),
+};
+
+static int brcm_avs_target_index(struct cpufreq_policy *, unsigned int);
+
+static char *__strtolower(char *s)
+{
+	char *p;
+
+	for (p = s; *p; p++)
+		*p = tolower(*p);
+
+	return s;
+}
+
+#endif /* CONFIG_ARM_BRCMSTB_AVS_CPUFREQ_DEBUG */
+
+static void __iomem *__map_region(const char *name)
+{
+	struct device_node *np;
+	void __iomem *ptr;
+
+	np = of_find_compatible_node(NULL, NULL, name);
+	if (!np)
+		return NULL;
+
+	ptr = of_iomap(np, 0);
+	of_node_put(np);
+
+	return ptr;
+}
+
+static int __issue_avs_command(struct private_data *priv, int cmd, bool is_send,
+			       u32 args[])
+{
+	unsigned long time_left = msecs_to_jiffies(AVS_TIMEOUT);
+	void __iomem *base = priv->base;
+	unsigned int i;
+	int ret;
+	u32 val;
+
+	ret = down_interruptible(&priv->sem);
+	if (ret)
+		return ret;
+
+	/*
+	 * Make sure no other command is currently running: cmd is 0 if AVS
+	 * co-processor is idle. Due to the guard above, we should almost never
+	 * have to wait here.
+	 */
+	for (i = 0, val = 1; val != 0 && i < AVS_LOOP_LIMIT; i++)
+		val = readl(base + AVS_MBOX_COMMAND);
+
+	/* Give the caller a chance to retry if AVS is busy. */
+	if (i == AVS_LOOP_LIMIT) {
+		ret = -EAGAIN;
+		goto out;
+	}
+
+	/* Clear status before we begin. */
+	writel(AVS_STATUS_CLEAR, base + AVS_MBOX_STATUS);
+
+	/* We need to send arguments for this command. */
+	if (args && is_send) {
+		for (i = 0; i < AVS_MAX_CMD_ARGS; i++)
+			writel(args[i], base + AVS_MBOX_PARAM(i));
+	}
+
+	/* Protect from spurious interrupts. */
+	reinit_completion(&priv->done);
+
+	/* Now issue the command & tell firmware to wake up to process it. */
+	writel(cmd, base + AVS_MBOX_COMMAND);
+	writel(AVS_CPU_L2_INT_MASK, priv->avs_intr_base + AVS_CPU_L2_SET0);
+
+	/* Wait for AVS co-processor to finish processing the command. */
+	time_left = wait_for_completion_timeout(&priv->done, time_left);
+
+	/*
+	 * If the AVS status is not in the expected range, it means AVS didn't
+	 * complete our command in time, and we return an error. Also, if there
+	 * is no "time left", we timed out waiting for the interrupt.
+	 */
+	val = readl(base + AVS_MBOX_STATUS);
+	if (time_left == 0 || val == 0 || val > AVS_STATUS_MAX) {
+		dev_err(priv->dev, "AVS command %#x didn't complete in time\n",
+			cmd);
+		dev_err(priv->dev, "    Time left: %u ms, AVS status: %#x\n",
+			jiffies_to_msecs(time_left), val);
+		ret = -ETIMEDOUT;
+		goto out;
+	}
+
+	/* This command returned arguments, so we read them back. */
+	if (args && !is_send) {
+		for (i = 0; i < AVS_MAX_CMD_ARGS; i++)
+			args[i] = readl(base + AVS_MBOX_PARAM(i));
+	}
+
+	/* Clear status to tell AVS co-processor we are done. */
+	writel(AVS_STATUS_CLEAR, base + AVS_MBOX_STATUS);
+
+	/* Convert firmware errors to errno's as much as possible. */
+	switch (val) {
+	case AVS_STATUS_INVALID:
+		ret = -EINVAL;
+		break;
+	case AVS_STATUS_NO_SUPP:
+		ret = -ENOTSUPP;
+		break;
+	case AVS_STATUS_NO_MAP:
+		ret = -ENOENT;
+		break;
+	case AVS_STATUS_MAP_SET:
+		ret = -EEXIST;
+		break;
+	case AVS_STATUS_FAILURE:
+		ret = -EIO;
+		break;
+	}
+
+out:
+	up(&priv->sem);
+
+	return ret;
+}
+
+static irqreturn_t irq_handler(int irq, void *data)
+{
+	struct private_data *priv = data;
+
+	/* AVS command completed execution. Wake up __issue_avs_command(). */
+	complete(&priv->done);
+
+	return IRQ_HANDLED;
+}
+
+static char *brcm_avs_mode_to_string(unsigned int mode)
+{
+	switch (mode) {
+	case AVS_MODE_AVS:
+		return "AVS";
+	case AVS_MODE_DFS:
+		return "DFS";
+	case AVS_MODE_DVS:
+		return "DVS";
+	case AVS_MODE_DVFS:
+		return "DVFS";
+	}
+	return NULL;
+}
+
+static void brcm_avs_parse_p1(u32 p1, unsigned int *mdiv_p0, unsigned int *pdiv,
+			      unsigned int *ndiv)
+{
+	*mdiv_p0 = (p1 >> MDIV_P0_SHIFT) & MDIV_P0_MASK;
+	*pdiv = (p1 >> PDIV_SHIFT) & PDIV_MASK;
+	*ndiv = (p1 >> NDIV_INT_SHIFT) & NDIV_INT_MASK;
+}
+
+static void brcm_avs_parse_p2(u32 p2, unsigned int *mdiv_p1,
+			      unsigned int *mdiv_p2, unsigned int *mdiv_p3,
+			      unsigned int *mdiv_p4)
+{
+	*mdiv_p4 = (p2 >> MDIV_P4_SHIFT) & MDIV_P4_MASK;
+	*mdiv_p3 = (p2 >> MDIV_P3_SHIFT) & MDIV_P3_MASK;
+	*mdiv_p2 = (p2 >> MDIV_P2_SHIFT) & MDIV_P2_MASK;
+	*mdiv_p1 = (p2 >> MDIV_P1_SHIFT) & MDIV_P1_MASK;
+}
+
+static int brcm_avs_get_pmap(struct private_data *priv, struct pmap *pmap)
+{
+	u32 args[AVS_MAX_CMD_ARGS];
+	int ret;
+
+	ret = __issue_avs_command(priv, AVS_CMD_GET_PMAP, false, args);
+	if (ret || !pmap)
+		return ret;
+
+	pmap->mode = args[0];
+	pmap->p1 = args[1];
+	pmap->p2 = args[2];
+	pmap->state = args[3];
+
+	return 0;
+}
+
+static int brcm_avs_set_pmap(struct private_data *priv, struct pmap *pmap)
+{
+	u32 args[AVS_MAX_CMD_ARGS];
+
+	args[0] = pmap->mode;
+	args[1] = pmap->p1;
+	args[2] = pmap->p2;
+	args[3] = pmap->state;
+
+	return __issue_avs_command(priv, AVS_CMD_SET_PMAP, true, args);
+}
+
+static int brcm_avs_get_pstate(struct private_data *priv, unsigned int *pstate)
+{
+	u32 args[AVS_MAX_CMD_ARGS];
+	int ret;
+
+	ret = __issue_avs_command(priv, AVS_CMD_GET_PSTATE, false, args);
+	if (ret)
+		return ret;
+	*pstate = args[0];
+
+	return 0;
+}
+
+static int brcm_avs_set_pstate(struct private_data *priv, unsigned int pstate)
+{
+	u32 args[AVS_MAX_CMD_ARGS];
+
+	args[0] = pstate;
+
+	return __issue_avs_command(priv, AVS_CMD_SET_PSTATE, true, args);
+}
+
+static unsigned long brcm_avs_get_voltage(void __iomem *base)
+{
+	return readl(base + AVS_MBOX_VOLTAGE1);
+}
+
+static unsigned long brcm_avs_get_frequency(void __iomem *base)
+{
+	return readl(base + AVS_MBOX_FREQUENCY) * 1000;	/* in kHz */
+}
+
+/*
+ * We determine which frequencies are supported by cycling through all P-states
+ * and reading back what frequency we are running at for each P-state.
+ */
+static struct cpufreq_frequency_table *
+brcm_avs_get_freq_table(struct device *dev, struct private_data *priv)
+{
+	struct cpufreq_frequency_table *table;
+	unsigned int pstate;
+	int i, ret;
+
+	/* Remember P-state for later */
+	ret = brcm_avs_get_pstate(priv, &pstate);
+	if (ret)
+		return ERR_PTR(ret);
+
+	table = devm_kzalloc(dev, (AVS_PSTATE_MAX + 1) * sizeof(*table),
+			     GFP_KERNEL);
+	if (!table)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = AVS_PSTATE_P0; i <= AVS_PSTATE_MAX; i++) {
+		ret = brcm_avs_set_pstate(priv, i);
+		if (ret)
+			return ERR_PTR(ret);
+		table[i].frequency = brcm_avs_get_frequency(priv->base);
+		table[i].driver_data = i;
+	}
+	table[i].frequency = CPUFREQ_TABLE_END;
+
+	/* Restore P-state */
+	ret = brcm_avs_set_pstate(priv, pstate);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return table;
+}
+
+#ifdef CONFIG_ARM_BRCMSTB_AVS_CPUFREQ_DEBUG
+
+#define MANT(x)	(unsigned int)(abs((x)) / 1000)
+#define FRAC(x)	(unsigned int)(abs((x)) - abs((x)) / 1000 * 1000)
+
+static int brcm_avs_debug_show(struct seq_file *s, void *data)
+{
+	struct debugfs_data *dbgfs = s->private;
+	void __iomem *base;
+	u32 val, offset;
+
+	if (!dbgfs) {
+		seq_puts(s, "No device pointer\n");
+		return 0;
+	}
+
+	base = dbgfs->priv->base;
+	offset = dbgfs->entry->offset;
+	val = readl(base + offset);
+	switch (dbgfs->entry->format) {
+	case DEBUGFS_NORMAL:
+		seq_printf(s, "%u\n", val);
+		break;
+	case DEBUGFS_FLOAT:
+		seq_printf(s, "%d.%03d\n", MANT(val), FRAC(val));
+		break;
+	case DEBUGFS_REV:
+		seq_printf(s, "%c.%c.%c.%c\n", (val >> 24 & 0xff),
+			   (val >> 16 & 0xff), (val >> 8 & 0xff),
+			   val & 0xff);
+		break;
+	}
+	seq_printf(s, "0x%08x\n", val);
+
+	return 0;
+}
+
+#undef MANT
+#undef FRAC
+
+static ssize_t brcm_avs_seq_write(struct file *file, const char __user *buf,
+				  size_t size, loff_t *ppos)
+{
+	struct seq_file *s = file->private_data;
+	struct debugfs_data *dbgfs = s->private;
+	struct private_data *priv = dbgfs->priv;
+	void __iomem *base, *avs_intr_base;
+	bool use_issue_command = false;
+	unsigned long val, offset;
+	char str[128];
+	int ret;
+	char *str_ptr = str;
+
+	if (size >= sizeof(str))
+		return -E2BIG;
+
+	memset(str, 0, sizeof(str));
+	ret = copy_from_user(str, buf, size);
+	if (ret)
+		return ret;
+
+	base = priv->base;
+	avs_intr_base = priv->avs_intr_base;
+	offset = dbgfs->entry->offset;
+	/*
+	 * Special case writing to "command" entry only: if the string starts
+	 * with a 'c', we use the driver's __issue_avs_command() function.
+	 * Otherwise, we perform a raw write. This should allow testing of raw
+	 * access as well as using the higher level function. (Raw access
+	 * doesn't clear the firmware return status after issuing the command.)
+	 */
+	if (str_ptr[0] == 'c' && offset == AVS_MBOX_COMMAND) {
+		use_issue_command = true;
+		str_ptr++;
+	}
+	if (kstrtoul(str_ptr, 0, &val) != 0)
+		return -EINVAL;
+
+	/*
+	 * Setting the P-state is a special case. We need to update the CPU
+	 * frequency we report.
+	 */
+	if (val == AVS_CMD_SET_PSTATE) {
+		struct cpufreq_policy *policy;
+		unsigned int pstate;
+
+		policy = cpufreq_cpu_get(smp_processor_id());
+		/* Read back the P-state we are about to set */
+		pstate = readl(base + AVS_MBOX_PARAM(0));
+		if (use_issue_command) {
+			ret = brcm_avs_target_index(policy, pstate);
+			return ret ? ret : size;
+		}
+		policy->cur = policy->freq_table[pstate].frequency;
+	}
+
+	if (use_issue_command) {
+		ret = __issue_avs_command(priv, val, false, NULL);
+	} else {
+		/* Locking here is not perfect, but is only for debug. */
+		ret = down_interruptible(&priv->sem);
+		if (ret)
+			return ret;
+
+		writel(val, base + offset);
+		/* We have to wake up the firmware to process a command. */
+		if (offset == AVS_MBOX_COMMAND)
+			writel(AVS_CPU_L2_INT_MASK,
+			       avs_intr_base + AVS_CPU_L2_SET0);
+		up(&priv->sem);
+	}
+
+	return ret ? ret : size;
+}
+
+static struct debugfs_entry *__find_debugfs_entry(const char *name)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(debugfs_entries); i++)
+		if (strcasecmp(debugfs_entries[i].name, name) == 0)
+			return &debugfs_entries[i];
+
+	return NULL;
+}
+
+static int brcm_avs_debug_open(struct inode *inode, struct file *file)
+{
+	struct debugfs_data *data;
+	fmode_t fmode;
+	int ret;
+
+	/*
+	 * seq_open(), which is called by single_open(), clears "write" access.
+	 * We need write access to some files, so we preserve our access mode
+	 * and restore it.
+	 */
+	fmode = file->f_mode;
+	/*
+	 * Check access permissions even for root. We don't want to be writing
+	 * to read-only registers. Access for regular users has already been
+	 * checked by the VFS layer.
+	 */
+	if ((fmode & FMODE_WRITER) && !(inode->i_mode & S_IWUSR))
+		return -EACCES;
+
+	data = kmalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+	/*
+	 * We use the same file system operations for all our debug files. To
+	 * produce specific output, we look up the file name upon opening a
+	 * debugfs entry and map it to a memory offset. This offset is then used
+	 * in the generic "show" function to read a specific register.
+	 */
+	data->entry = __find_debugfs_entry(file->f_path.dentry->d_iname);
+	data->priv = inode->i_private;
+
+	ret = single_open(file, brcm_avs_debug_show, data);
+	if (ret)
+		kfree(data);
+	file->f_mode = fmode;
+
+	return ret;
+}
+
+static int brcm_avs_debug_release(struct inode *inode, struct file *file)
+{
+	struct seq_file *seq_priv = file->private_data;
+	struct debugfs_data *data = seq_priv->private;
+
+	kfree(data);
+	return single_release(inode, file);
+}
+
+static const struct file_operations brcm_avs_debug_ops = {
+	.open		= brcm_avs_debug_open,
+	.read		= seq_read,
+	.write		= brcm_avs_seq_write,
+	.llseek		= seq_lseek,
+	.release	= brcm_avs_debug_release,
+};
+
+static void brcm_avs_cpufreq_debug_init(struct platform_device *pdev)
+{
+	struct private_data *priv = platform_get_drvdata(pdev);
+	struct dentry *dir;
+	int i;
+
+	if (!priv)
+		return;
+
+	dir = debugfs_create_dir(BRCM_AVS_CPUFREQ_NAME, NULL);
+	if (IS_ERR_OR_NULL(dir))
+		return;
+	priv->debugfs = dir;
+
+	for (i = 0; i < ARRAY_SIZE(debugfs_entries); i++) {
+		/*
+		 * The DEBUGFS_ENTRY macro generates uppercase strings. We
+		 * convert them to lowercase before creating the debugfs
+		 * entries.
+		 */
+		char *entry = __strtolower(debugfs_entries[i].name);
+		fmode_t mode = debugfs_entries[i].mode;
+
+		if (!debugfs_create_file(entry, S_IFREG | S_IRUGO | mode,
+					 dir, priv, &brcm_avs_debug_ops)) {
+			priv->debugfs = NULL;
+			debugfs_remove_recursive(dir);
+			break;
+		}
+	}
+}
+
+static void brcm_avs_cpufreq_debug_exit(struct platform_device *pdev)
+{
+	struct private_data *priv = platform_get_drvdata(pdev);
+
+	if (priv && priv->debugfs) {
+		debugfs_remove_recursive(priv->debugfs);
+		priv->debugfs = NULL;
+	}
+}
+
+#else
+
+static void brcm_avs_cpufreq_debug_init(struct platform_device *pdev) {}
+static void brcm_avs_cpufreq_debug_exit(struct platform_device *pdev) {}
+
+#endif /* CONFIG_ARM_BRCMSTB_AVS_CPUFREQ_DEBUG */
+
+/*
+ * To ensure the right firmware is running we need to
+ *    - check the MAGIC matches what we expect
+ *    - brcm_avs_get_pmap() doesn't return -ENOTSUPP or -EINVAL
+ * We need to set up our interrupt handling before calling brcm_avs_get_pmap()!
+ */
+static bool brcm_avs_is_firmware_loaded(struct private_data *priv)
+{
+	u32 magic;
+	int rc;
+
+	rc = brcm_avs_get_pmap(priv, NULL);
+	magic = readl(priv->base + AVS_MBOX_MAGIC);
+
+	return (magic == AVS_FIRMWARE_MAGIC) && (rc != -ENOTSUPP) &&
+		(rc != -EINVAL);
+}
+
+static unsigned int brcm_avs_cpufreq_get(unsigned int cpu)
+{
+	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
+	struct private_data *priv = policy->driver_data;
+
+	return brcm_avs_get_frequency(priv->base);
+}
+
+static int brcm_avs_target_index(struct cpufreq_policy *policy,
+				 unsigned int index)
+{
+	return brcm_avs_set_pstate(policy->driver_data,
+				  policy->freq_table[index].driver_data);
+}
+
+static int brcm_avs_suspend(struct cpufreq_policy *policy)
+{
+	struct private_data *priv = policy->driver_data;
+
+	return brcm_avs_get_pmap(priv, &priv->pmap);
+}
+
+static int brcm_avs_resume(struct cpufreq_policy *policy)
+{
+	struct private_data *priv = policy->driver_data;
+	int ret;
+
+	ret = brcm_avs_set_pmap(priv, &priv->pmap);
+	if (ret == -EEXIST) {
+		struct platform_device *pdev  = cpufreq_get_driver_data();
+		struct device *dev = &pdev->dev;
+
+		dev_warn(dev, "PMAP was already set\n");
+		ret = 0;
+	}
+
+	return ret;
+}
+
+/*
+ * All initialization code that we only want to execute once goes here. Setup
+ * code that can be re-tried on every core (if it failed before) can go into
+ * brcm_avs_cpufreq_init().
+ */
+static int brcm_avs_prepare_init(struct platform_device *pdev)
+{
+	struct private_data *priv;
+	struct device *dev;
+	int host_irq, ret;
+
+	dev = &pdev->dev;
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+	sema_init(&priv->sem, 1);
+	init_completion(&priv->done);
+	platform_set_drvdata(pdev, priv);
+
+	priv->base = __map_region(BRCM_AVS_CPU_DATA);
+	if (!priv->base) {
+		dev_err(dev, "Couldn't find property %s in device tree.\n",
+			BRCM_AVS_CPU_DATA);
+		return -ENOENT;
+	}
+
+	priv->avs_intr_base = __map_region(BRCM_AVS_CPU_INTR);
+	if (!priv->avs_intr_base) {
+		dev_err(dev, "Couldn't find property %s in device tree.\n",
+			BRCM_AVS_CPU_INTR);
+		ret = -ENOENT;
+		goto unmap_base;
+	}
+
+	host_irq = platform_get_irq_byname(pdev, BRCM_AVS_HOST_INTR);
+	if (host_irq < 0) {
+		dev_err(dev, "Couldn't find interrupt %s -- %d\n",
+			BRCM_AVS_HOST_INTR, host_irq);
+		ret = host_irq;
+		goto unmap_intr_base;
+	}
+
+	ret = devm_request_irq(dev, host_irq, irq_handler, IRQF_TRIGGER_RISING,
+			       BRCM_AVS_HOST_INTR, priv);
+	if (ret) {
+		dev_err(dev, "IRQ request failed: %s (%d) -- %d\n",
+			BRCM_AVS_HOST_INTR, host_irq, ret);
+		goto unmap_intr_base;
+	}
+
+	if (brcm_avs_is_firmware_loaded(priv))
+		return 0;
+
+	dev_err(dev, "AVS firmware is not loaded or doesn't support DVFS\n");
+	ret = -ENODEV;
+
+unmap_intr_base:
+	iounmap(priv->avs_intr_base);
+unmap_base:
+	iounmap(priv->base);
+	platform_set_drvdata(pdev, NULL);
+
+	return ret;
+}
+
+static int brcm_avs_cpufreq_init(struct cpufreq_policy *policy)
+{
+	struct cpufreq_frequency_table *freq_table;
+	struct platform_device *pdev;
+	struct private_data *priv;
+	struct device *dev;
+	int ret;
+
+	pdev = cpufreq_get_driver_data();
+	priv = platform_get_drvdata(pdev);
+	policy->driver_data = priv;
+	dev = &pdev->dev;
+
+	freq_table = brcm_avs_get_freq_table(dev, priv);
+	if (IS_ERR(freq_table)) {
+		ret = PTR_ERR(freq_table);
+		dev_err(dev, "Couldn't determine frequency table (%d).\n", ret);
+		return ret;
+	}
+
+	ret = cpufreq_table_validate_and_show(policy, freq_table);
+	if (ret) {
+		dev_err(dev, "invalid frequency table: %d\n", ret);
+		return ret;
+	}
+
+	/* All cores share the same clock and thus the same policy. */
+	cpumask_setall(policy->cpus);
+
+	ret = __issue_avs_command(priv, AVS_CMD_ENABLE, false, NULL);
+	if (!ret) {
+		unsigned int pstate;
+
+		ret = brcm_avs_get_pstate(priv, &pstate);
+		if (!ret) {
+			policy->cur = freq_table[pstate].frequency;
+			dev_info(dev, "registered\n");
+			return 0;
+		}
+	}
+
+	dev_err(dev, "couldn't initialize driver (%d)\n", ret);
+
+	return ret;
+}
+
+static ssize_t show_brcm_avs_pstate(struct cpufreq_policy *policy, char *buf)
+{
+	struct private_data *priv = policy->driver_data;
+	unsigned int pstate;
+
+	if (brcm_avs_get_pstate(priv, &pstate))
+		return sprintf(buf, "<unknown>\n");
+
+	return sprintf(buf, "%u\n", pstate);
+}
+
+static ssize_t show_brcm_avs_mode(struct cpufreq_policy *policy, char *buf)
+{
+	struct private_data *priv = policy->driver_data;
+	struct pmap pmap;
+
+	if (brcm_avs_get_pmap(priv, &pmap))
+		return sprintf(buf, "<unknown>\n");
+
+	return sprintf(buf, "%s %u\n", brcm_avs_mode_to_string(pmap.mode),
+		pmap.mode);
+}
+
+static ssize_t show_brcm_avs_pmap(struct cpufreq_policy *policy, char *buf)
+{
+	unsigned int mdiv_p0, mdiv_p1, mdiv_p2, mdiv_p3, mdiv_p4;
+	struct private_data *priv = policy->driver_data;
+	unsigned int ndiv, pdiv;
+	struct pmap pmap;
+
+	if (brcm_avs_get_pmap(priv, &pmap))
+		return sprintf(buf, "<unknown>\n");
+
+	brcm_avs_parse_p1(pmap.p1, &mdiv_p0, &pdiv, &ndiv);
+	brcm_avs_parse_p2(pmap.p2, &mdiv_p1, &mdiv_p2, &mdiv_p3, &mdiv_p4);
+
+	return sprintf(buf, "0x%08x 0x%08x %u %u %u %u %u %u %u\n",
+		pmap.p1, pmap.p2, ndiv, pdiv, mdiv_p0, mdiv_p1, mdiv_p2,
+		mdiv_p3, mdiv_p4);
+}
+
+static ssize_t show_brcm_avs_voltage(struct cpufreq_policy *policy, char *buf)
+{
+	struct private_data *priv = policy->driver_data;
+
+	return sprintf(buf, "0x%08lx\n", brcm_avs_get_voltage(priv->base));
+}
+
+static ssize_t show_brcm_avs_frequency(struct cpufreq_policy *policy, char *buf)
+{
+	struct private_data *priv = policy->driver_data;
+
+	return sprintf(buf, "0x%08lx\n", brcm_avs_get_frequency(priv->base));
+}
+
+cpufreq_freq_attr_ro(brcm_avs_pstate);
+cpufreq_freq_attr_ro(brcm_avs_mode);
+cpufreq_freq_attr_ro(brcm_avs_pmap);
+cpufreq_freq_attr_ro(brcm_avs_voltage);
+cpufreq_freq_attr_ro(brcm_avs_frequency);
+
+static struct freq_attr *brcm_avs_cpufreq_attr[] = {
+	&cpufreq_freq_attr_scaling_available_freqs,
+	&brcm_avs_pstate,
+	&brcm_avs_mode,
+	&brcm_avs_pmap,
+	&brcm_avs_voltage,
+	&brcm_avs_frequency,
+	NULL
+};
+
+static struct cpufreq_driver brcm_avs_driver = {
+	.flags		= CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+	.verify		= cpufreq_generic_frequency_table_verify,
+	.target_index	= brcm_avs_target_index,
+	.get		= brcm_avs_cpufreq_get,
+	.suspend	= brcm_avs_suspend,
+	.resume		= brcm_avs_resume,
+	.init		= brcm_avs_cpufreq_init,
+	.attr		= brcm_avs_cpufreq_attr,
+	.name		= BRCM_AVS_CPUFREQ_PREFIX,
+};
+
+static int brcm_avs_cpufreq_probe(struct platform_device *pdev)
+{
+	int ret;
+
+	ret = brcm_avs_prepare_init(pdev);
+	if (ret)
+		return ret;
+
+	brcm_avs_driver.driver_data = pdev;
+	ret = cpufreq_register_driver(&brcm_avs_driver);
+	if (!ret)
+		brcm_avs_cpufreq_debug_init(pdev);
+
+	return ret;
+}
+
+static int brcm_avs_cpufreq_remove(struct platform_device *pdev)
+{
+	struct private_data *priv;
+	int ret;
+
+	ret = cpufreq_unregister_driver(&brcm_avs_driver);
+	if (ret)
+		return ret;
+
+	brcm_avs_cpufreq_debug_exit(pdev);
+
+	priv = platform_get_drvdata(pdev);
+	iounmap(priv->base);
+	iounmap(priv->avs_intr_base);
+	platform_set_drvdata(pdev, NULL);
+
+	return 0;
+}
+
+static const struct of_device_id brcm_avs_cpufreq_match[] = {
+	{ .compatible = BRCM_AVS_CPU_DATA },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, brcm_avs_cpufreq_match);
+
+static struct platform_driver brcm_avs_cpufreq_platdrv = {
+	.driver = {
+		.name	= BRCM_AVS_CPUFREQ_NAME,
+		.of_match_table = brcm_avs_cpufreq_match,
+	},
+	.probe		= brcm_avs_cpufreq_probe,
+	.remove		= brcm_avs_cpufreq_remove,
+};
+module_platform_driver(brcm_avs_cpufreq_platdrv);
+
+MODULE_AUTHOR("Markus Mayer <mmayer@broadcom.com>");
+MODULE_DESCRIPTION("CPUfreq driver for Broadcom STB AVS");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
index 4852d9efe74e..e82bb3c30b92 100644
--- a/drivers/cpufreq/cppc_cpufreq.c
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -247,3 +247,10 @@ MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec");
 MODULE_LICENSE("GPL");
 
 late_initcall(cppc_cpufreq_init);
+
+static const struct acpi_device_id cppc_acpi_ids[] = {
+	{ACPI_PROCESSOR_DEVICE_HID, },
+	{}
+};
+
+MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids);
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index 71267626456b..bc97b6a4b1cf 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -26,6 +26,9 @@ static const struct of_device_id machines[] __initconst = {
 	{ .compatible = "allwinner,sun8i-a83t", },
 	{ .compatible = "allwinner,sun8i-h3", },
 
+	{ .compatible = "arm,integrator-ap", },
+	{ .compatible = "arm,integrator-cp", },
+
 	{ .compatible = "hisilicon,hi6220", },
 
 	{ .compatible = "fsl,imx27", },
@@ -34,6 +37,8 @@ static const struct of_device_id machines[] __initconst = {
 	{ .compatible = "fsl,imx7d", },
 
 	{ .compatible = "marvell,berlin", },
+	{ .compatible = "marvell,pxa250", },
+	{ .compatible = "marvell,pxa270", },
 
 	{ .compatible = "samsung,exynos3250", },
 	{ .compatible = "samsung,exynos4210", },
@@ -50,6 +55,8 @@ static const struct of_device_id machines[] __initconst = {
 	{ .compatible = "renesas,r7s72100", },
 	{ .compatible = "renesas,r8a73a4", },
 	{ .compatible = "renesas,r8a7740", },
+	{ .compatible = "renesas,r8a7743", },
+	{ .compatible = "renesas,r8a7745", },
 	{ .compatible = "renesas,r8a7778", },
 	{ .compatible = "renesas,r8a7779", },
 	{ .compatible = "renesas,r8a7790", },
@@ -72,6 +79,12 @@ static const struct of_device_id machines[] __initconst = {
 
 	{ .compatible = "sigma,tango4" },
 
+	{ .compatible = "socionext,uniphier-pro5", },
+	{ .compatible = "socionext,uniphier-pxs2", },
+	{ .compatible = "socionext,uniphier-ld6b", },
+	{ .compatible = "socionext,uniphier-ld11", },
+	{ .compatible = "socionext,uniphier-ld20", },
+
 	{ .compatible = "ti,am33xx", },
 	{ .compatible = "ti,dra7", },
 	{ .compatible = "ti,omap2", },
@@ -81,6 +94,8 @@ static const struct of_device_id machines[] __initconst = {
 
 	{ .compatible = "xlnx,zynq-7000", },
 
+	{ .compatible = "zte,zx296718", },
+
 	{ }
 };
 
diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c
index 5c07ae05d69a..269013311e79 100644
--- a/drivers/cpufreq/cpufreq-dt.c
+++ b/drivers/cpufreq/cpufreq-dt.c
@@ -28,6 +28,7 @@
 #include "cpufreq-dt.h"
 
 struct private_data {
+	struct opp_table *opp_table;
 	struct device *cpu_dev;
 	struct thermal_cooling_device *cdev;
 	const char *reg_name;
@@ -143,6 +144,7 @@ static int resources_available(void)
 static int cpufreq_init(struct cpufreq_policy *policy)
 {
 	struct cpufreq_frequency_table *freq_table;
+	struct opp_table *opp_table = NULL;
 	struct private_data *priv;
 	struct device *cpu_dev;
 	struct clk *cpu_clk;
@@ -186,8 +188,9 @@ static int cpufreq_init(struct cpufreq_policy *policy)
 	 */
 	name = find_supply_name(cpu_dev);
 	if (name) {
-		ret = dev_pm_opp_set_regulator(cpu_dev, name);
-		if (ret) {
+		opp_table = dev_pm_opp_set_regulators(cpu_dev, &name, 1);
+		if (IS_ERR(opp_table)) {
+			ret = PTR_ERR(opp_table);
 			dev_err(cpu_dev, "Failed to set regulator for cpu%d: %d\n",
 				policy->cpu, ret);
 			goto out_put_clk;
@@ -237,6 +240,7 @@ static int cpufreq_init(struct cpufreq_policy *policy)
 	}
 
 	priv->reg_name = name;
+	priv->opp_table = opp_table;
 
 	ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
 	if (ret) {
@@ -285,7 +289,7 @@ out_free_priv:
 out_free_opp:
 	dev_pm_opp_of_cpumask_remove_table(policy->cpus);
 	if (name)
-		dev_pm_opp_put_regulator(cpu_dev);
+		dev_pm_opp_put_regulators(opp_table);
 out_put_clk:
 	clk_put(cpu_clk);
 
@@ -300,7 +304,7 @@ static int cpufreq_exit(struct cpufreq_policy *policy)
 	dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
 	dev_pm_opp_of_cpumask_remove_table(policy->related_cpus);
 	if (priv->reg_name)
-		dev_pm_opp_put_regulator(priv->cpu_dev);
+		dev_pm_opp_put_regulators(priv->opp_table);
 
 	clk_put(policy->clk);
 	kfree(priv);
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 6e6c1fb60fbc..cc475eff90b3 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -1526,7 +1526,10 @@ unsigned int cpufreq_get(unsigned int cpu)
 
 	if (policy) {
 		down_read(&policy->rwsem);
-		ret_freq = __cpufreq_get(policy);
+
+		if (!policy_is_inactive(policy))
+			ret_freq = __cpufreq_get(policy);
+
 		up_read(&policy->rwsem);
 
 		cpufreq_cpu_put(policy);
@@ -2254,17 +2257,19 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
  *	Useful for policy notifiers which have different necessities
  *	at different times.
  */
-int cpufreq_update_policy(unsigned int cpu)
+void cpufreq_update_policy(unsigned int cpu)
 {
 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
 	struct cpufreq_policy new_policy;
-	int ret;
 
 	if (!policy)
-		return -ENODEV;
+		return;
 
 	down_write(&policy->rwsem);
 
+	if (policy_is_inactive(policy))
+		goto unlock;
+
 	pr_debug("updating policy for CPU %u\n", cpu);
 	memcpy(&new_policy, policy, sizeof(*policy));
 	new_policy.min = policy->user_policy.min;
@@ -2275,24 +2280,20 @@ int cpufreq_update_policy(unsigned int cpu)
 	 * -> ask driver for current freq and notify governors about a change
 	 */
 	if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
-		if (cpufreq_suspended) {
-			ret = -EAGAIN;
+		if (cpufreq_suspended)
 			goto unlock;
-		}
+
 		new_policy.cur = cpufreq_update_current_freq(policy);
-		if (WARN_ON(!new_policy.cur)) {
-			ret = -EIO;
+		if (WARN_ON(!new_policy.cur))
 			goto unlock;
-		}
 	}
 
-	ret = cpufreq_set_policy(policy, &new_policy);
+	cpufreq_set_policy(policy, &new_policy);
 
 unlock:
 	up_write(&policy->rwsem);
 
 	cpufreq_cpu_put(policy);
-	return ret;
 }
 EXPORT_SYMBOL(cpufreq_update_policy);
 
diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c
index 13475890d792..992f7c20760f 100644
--- a/drivers/cpufreq/cpufreq_conservative.c
+++ b/drivers/cpufreq/cpufreq_conservative.c
@@ -37,16 +37,16 @@ struct cs_dbs_tuners {
 #define DEF_SAMPLING_DOWN_FACTOR		(1)
 #define MAX_SAMPLING_DOWN_FACTOR		(10)
 
-static inline unsigned int get_freq_target(struct cs_dbs_tuners *cs_tuners,
-					   struct cpufreq_policy *policy)
+static inline unsigned int get_freq_step(struct cs_dbs_tuners *cs_tuners,
+					 struct cpufreq_policy *policy)
 {
-	unsigned int freq_target = (cs_tuners->freq_step * policy->max) / 100;
+	unsigned int freq_step = (cs_tuners->freq_step * policy->max) / 100;
 
 	/* max freq cannot be less than 100. But who knows... */
-	if (unlikely(freq_target == 0))
-		freq_target = DEF_FREQUENCY_STEP;
+	if (unlikely(freq_step == 0))
+		freq_step = DEF_FREQUENCY_STEP;
 
-	return freq_target;
+	return freq_step;
 }
 
 /*
@@ -55,10 +55,10 @@ static inline unsigned int get_freq_target(struct cs_dbs_tuners *cs_tuners,
  * sampling_down_factor, we check, if current idle time is more than 80%
  * (default), then we try to decrease frequency
  *
- * Any frequency increase takes it to the maximum frequency. Frequency reduction
- * happens at minimum steps of 5% (default) of maximum frequency
+ * Frequency updates happen at minimum steps of 5% (default) of maximum
+ * frequency
  */
-static unsigned int cs_dbs_timer(struct cpufreq_policy *policy)
+static unsigned int cs_dbs_update(struct cpufreq_policy *policy)
 {
 	struct policy_dbs_info *policy_dbs = policy->governor_data;
 	struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
@@ -66,6 +66,7 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy)
 	struct dbs_data *dbs_data = policy_dbs->dbs_data;
 	struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
 	unsigned int load = dbs_update(policy);
+	unsigned int freq_step;
 
 	/*
 	 * break out if we 'cannot' reduce the speed as the user might
@@ -82,6 +83,23 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy)
 	if (requested_freq > policy->max || requested_freq < policy->min)
 		requested_freq = policy->cur;
 
+	freq_step = get_freq_step(cs_tuners, policy);
+
+	/*
+	 * Decrease requested_freq one freq_step for each idle period that
+	 * we didn't update the frequency.
+	 */
+	if (policy_dbs->idle_periods < UINT_MAX) {
+		unsigned int freq_steps = policy_dbs->idle_periods * freq_step;
+
+		if (requested_freq > freq_steps)
+			requested_freq -= freq_steps;
+		else
+			requested_freq = policy->min;
+
+		policy_dbs->idle_periods = UINT_MAX;
+	}
+
 	/* Check for frequency increase */
 	if (load > dbs_data->up_threshold) {
 		dbs_info->down_skip = 0;
@@ -90,7 +108,7 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy)
 		if (requested_freq == policy->max)
 			goto out;
 
-		requested_freq += get_freq_target(cs_tuners, policy);
+		requested_freq += freq_step;
 		if (requested_freq > policy->max)
 			requested_freq = policy->max;
 
@@ -106,16 +124,14 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy)
 
 	/* Check for frequency decrease */
 	if (load < cs_tuners->down_threshold) {
-		unsigned int freq_target;
 		/*
 		 * if we cannot reduce the frequency anymore, break out early
 		 */
 		if (requested_freq == policy->min)
 			goto out;
 
-		freq_target = get_freq_target(cs_tuners, policy);
-		if (requested_freq > freq_target)
-			requested_freq -= freq_target;
+		if (requested_freq > freq_step)
+			requested_freq -= freq_step;
 		else
 			requested_freq = policy->min;
 
@@ -305,7 +321,7 @@ static void cs_start(struct cpufreq_policy *policy)
 static struct dbs_governor cs_governor = {
 	.gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("conservative"),
 	.kobj_type = { .default_attrs = cs_attributes },
-	.gov_dbs_timer = cs_dbs_timer,
+	.gov_dbs_update = cs_dbs_update,
 	.alloc = cs_alloc,
 	.free = cs_free,
 	.init = cs_init,
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index 642dd0f183a8..0196467280bd 100644
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -61,7 +61,7 @@ ssize_t store_sampling_rate(struct gov_attr_set *attr_set, const char *buf,
 	 * entries can't be freed concurrently.
 	 */
 	list_for_each_entry(policy_dbs, &attr_set->policy_list, list) {
-		mutex_lock(&policy_dbs->timer_mutex);
+		mutex_lock(&policy_dbs->update_mutex);
 		/*
 		 * On 32-bit architectures this may race with the
 		 * sample_delay_ns read in dbs_update_util_handler(), but that
@@ -76,7 +76,7 @@ ssize_t store_sampling_rate(struct gov_attr_set *attr_set, const char *buf,
 		 * taken, so it shouldn't be significant.
 		 */
 		gov_update_sample_delay(policy_dbs, 0);
-		mutex_unlock(&policy_dbs->timer_mutex);
+		mutex_unlock(&policy_dbs->update_mutex);
 	}
 
 	return count;
@@ -117,7 +117,7 @@ unsigned int dbs_update(struct cpufreq_policy *policy)
 	struct policy_dbs_info *policy_dbs = policy->governor_data;
 	struct dbs_data *dbs_data = policy_dbs->dbs_data;
 	unsigned int ignore_nice = dbs_data->ignore_nice_load;
-	unsigned int max_load = 0;
+	unsigned int max_load = 0, idle_periods = UINT_MAX;
 	unsigned int sampling_rate, io_busy, j;
 
 	/*
@@ -215,9 +215,19 @@ unsigned int dbs_update(struct cpufreq_policy *policy)
 			j_cdbs->prev_load = load;
 		}
 
+		if (time_elapsed > 2 * sampling_rate) {
+			unsigned int periods = time_elapsed / sampling_rate;
+
+			if (periods < idle_periods)
+				idle_periods = periods;
+		}
+
 		if (load > max_load)
 			max_load = load;
 	}
+
+	policy_dbs->idle_periods = idle_periods;
+
 	return max_load;
 }
 EXPORT_SYMBOL_GPL(dbs_update);
@@ -236,9 +246,9 @@ static void dbs_work_handler(struct work_struct *work)
 	 * Make sure cpufreq_governor_limits() isn't evaluating load or the
 	 * ondemand governor isn't updating the sampling rate in parallel.
 	 */
-	mutex_lock(&policy_dbs->timer_mutex);
-	gov_update_sample_delay(policy_dbs, gov->gov_dbs_timer(policy));
-	mutex_unlock(&policy_dbs->timer_mutex);
+	mutex_lock(&policy_dbs->update_mutex);
+	gov_update_sample_delay(policy_dbs, gov->gov_dbs_update(policy));
+	mutex_unlock(&policy_dbs->update_mutex);
 
 	/* Allow the utilization update handler to queue up more work. */
 	atomic_set(&policy_dbs->work_count, 0);
@@ -348,7 +358,7 @@ static struct policy_dbs_info *alloc_policy_dbs_info(struct cpufreq_policy *poli
 		return NULL;
 
 	policy_dbs->policy = policy;
-	mutex_init(&policy_dbs->timer_mutex);
+	mutex_init(&policy_dbs->update_mutex);
 	atomic_set(&policy_dbs->work_count, 0);
 	init_irq_work(&policy_dbs->irq_work, dbs_irq_work);
 	INIT_WORK(&policy_dbs->work, dbs_work_handler);
@@ -367,7 +377,7 @@ static void free_policy_dbs_info(struct policy_dbs_info *policy_dbs,
 {
 	int j;
 
-	mutex_destroy(&policy_dbs->timer_mutex);
+	mutex_destroy(&policy_dbs->update_mutex);
 
 	for_each_cpu(j, policy_dbs->policy->related_cpus) {
 		struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
@@ -547,10 +557,10 @@ void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy)
 {
 	struct policy_dbs_info *policy_dbs = policy->governor_data;
 
-	mutex_lock(&policy_dbs->timer_mutex);
+	mutex_lock(&policy_dbs->update_mutex);
 	cpufreq_policy_apply_limits(policy);
 	gov_update_sample_delay(policy_dbs, 0);
 
-	mutex_unlock(&policy_dbs->timer_mutex);
+	mutex_unlock(&policy_dbs->update_mutex);
 }
 EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_limits);
diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h
index ef1037e9c92b..f5717ca070cc 100644
--- a/drivers/cpufreq/cpufreq_governor.h
+++ b/drivers/cpufreq/cpufreq_governor.h
@@ -85,7 +85,7 @@ struct policy_dbs_info {
 	 * Per policy mutex that serializes load evaluation from limit-change
 	 * and work-handler.
 	 */
-	struct mutex timer_mutex;
+	struct mutex update_mutex;
 
 	u64 last_sample_time;
 	s64 sample_delay_ns;
@@ -97,6 +97,7 @@ struct policy_dbs_info {
 	struct list_head list;
 	/* Multiplier for increasing sample delay temporarily. */
 	unsigned int rate_mult;
+	unsigned int idle_periods;	/* For conservative */
 	/* Status indicators */
 	bool is_shared;		/* This object is used by multiple CPUs */
 	bool work_in_progress;	/* Work is being queued up or in progress */
@@ -135,7 +136,7 @@ struct dbs_governor {
 	 */
 	struct dbs_data *gdbs_data;
 
-	unsigned int (*gov_dbs_timer)(struct cpufreq_policy *policy);
+	unsigned int (*gov_dbs_update)(struct cpufreq_policy *policy);
 	struct policy_dbs_info *(*alloc)(void);
 	void (*free)(struct policy_dbs_info *policy_dbs);
 	int (*init)(struct dbs_data *dbs_data);
diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c
index 3a1f49f5f4c6..4a017e895296 100644
--- a/drivers/cpufreq/cpufreq_ondemand.c
+++ b/drivers/cpufreq/cpufreq_ondemand.c
@@ -25,7 +25,7 @@
 #define MAX_SAMPLING_DOWN_FACTOR		(100000)
 #define MICRO_FREQUENCY_UP_THRESHOLD		(95)
 #define MICRO_FREQUENCY_MIN_SAMPLE_RATE		(10000)
-#define MIN_FREQUENCY_UP_THRESHOLD		(11)
+#define MIN_FREQUENCY_UP_THRESHOLD		(1)
 #define MAX_FREQUENCY_UP_THRESHOLD		(100)
 
 static struct od_ops od_ops;
@@ -169,7 +169,7 @@ static void od_update(struct cpufreq_policy *policy)
 	}
 }
 
-static unsigned int od_dbs_timer(struct cpufreq_policy *policy)
+static unsigned int od_dbs_update(struct cpufreq_policy *policy)
 {
 	struct policy_dbs_info *policy_dbs = policy->governor_data;
 	struct dbs_data *dbs_data = policy_dbs->dbs_data;
@@ -191,7 +191,7 @@ static unsigned int od_dbs_timer(struct cpufreq_policy *policy)
 	od_update(policy);
 
 	if (dbs_info->freq_lo) {
-		/* Setup timer for SUB_SAMPLE */
+		/* Setup SUB_SAMPLE */
 		dbs_info->sample_type = OD_SUB_SAMPLE;
 		return dbs_info->freq_hi_delay_us;
 	}
@@ -255,11 +255,11 @@ static ssize_t store_sampling_down_factor(struct gov_attr_set *attr_set,
 	list_for_each_entry(policy_dbs, &attr_set->policy_list, list) {
 		/*
 		 * Doing this without locking might lead to using different
-		 * rate_mult values in od_update() and od_dbs_timer().
+		 * rate_mult values in od_update() and od_dbs_update().
 		 */
-		mutex_lock(&policy_dbs->timer_mutex);
+		mutex_lock(&policy_dbs->update_mutex);
 		policy_dbs->rate_mult = 1;
-		mutex_unlock(&policy_dbs->timer_mutex);
+		mutex_unlock(&policy_dbs->update_mutex);
 	}
 
 	return count;
@@ -374,8 +374,7 @@ static int od_init(struct dbs_data *dbs_data)
 		dbs_data->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
 		/*
 		 * In nohz/micro accounting case we set the minimum frequency
-		 * not depending on HZ, but fixed (very low). The deferred
-		 * timer might skip some samples if idle/sleeping as needed.
+		 * not depending on HZ, but fixed (very low).
 		*/
 		dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE;
 	} else {
@@ -415,7 +414,7 @@ static struct od_ops od_ops = {
 static struct dbs_governor od_dbs_gov = {
 	.gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("ondemand"),
 	.kobj_type = { .default_attrs = od_attributes },
-	.gov_dbs_timer = od_dbs_timer,
+	.gov_dbs_update = od_dbs_update,
 	.alloc = od_alloc,
 	.free = od_free,
 	.init = od_init,
diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c
index 06d3abdffd3a..ac284e66839c 100644
--- a/drivers/cpufreq/cpufreq_stats.c
+++ b/drivers/cpufreq/cpufreq_stats.c
@@ -41,6 +41,18 @@ static int cpufreq_stats_update(struct cpufreq_stats *stats)
 	return 0;
 }
 
+static void cpufreq_stats_clear_table(struct cpufreq_stats *stats)
+{
+	unsigned int count = stats->max_state;
+
+	memset(stats->time_in_state, 0, count * sizeof(u64));
+#ifdef CONFIG_CPU_FREQ_STAT_DETAILS
+	memset(stats->trans_table, 0, count * count * sizeof(int));
+#endif
+	stats->last_time = get_jiffies_64();
+	stats->total_trans = 0;
+}
+
 static ssize_t show_total_trans(struct cpufreq_policy *policy, char *buf)
 {
 	return sprintf(buf, "%d\n", policy->stats->total_trans);
@@ -64,6 +76,14 @@ static ssize_t show_time_in_state(struct cpufreq_policy *policy, char *buf)
 	return len;
 }
 
+static ssize_t store_reset(struct cpufreq_policy *policy, const char *buf,
+			   size_t count)
+{
+	/* We don't care what is written to the attribute. */
+	cpufreq_stats_clear_table(policy->stats);
+	return count;
+}
+
 #ifdef CONFIG_CPU_FREQ_STAT_DETAILS
 static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf)
 {
@@ -113,10 +133,12 @@ cpufreq_freq_attr_ro(trans_table);
 
 cpufreq_freq_attr_ro(total_trans);
 cpufreq_freq_attr_ro(time_in_state);
+cpufreq_freq_attr_wo(reset);
 
 static struct attribute *default_attrs[] = {
 	&total_trans.attr,
 	&time_in_state.attr,
+	&reset.attr,
 #ifdef CONFIG_CPU_FREQ_STAT_DETAILS
 	&trans_table.attr,
 #endif
diff --git a/drivers/cpufreq/integrator-cpufreq.c b/drivers/cpufreq/integrator-cpufreq.c
deleted file mode 100644
index 79e3ff2771a6..000000000000
--- a/drivers/cpufreq/integrator-cpufreq.c
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- *  Copyright (C) 2001-2002 Deep Blue Solutions Ltd.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * CPU support functions
- */
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/cpufreq.h>
-#include <linux/sched.h>
-#include <linux/smp.h>
-#include <linux/init.h>
-#include <linux/io.h>
-#include <linux/platform_device.h>
-#include <linux/of.h>
-#include <linux/of_address.h>
-
-#include <asm/mach-types.h>
-#include <asm/hardware/icst.h>
-
-static void __iomem *cm_base;
-/* The cpufreq driver only use the OSC register */
-#define INTEGRATOR_HDR_OSC_OFFSET       0x08
-#define INTEGRATOR_HDR_LOCK_OFFSET      0x14
-
-static struct cpufreq_driver integrator_driver;
-
-static const struct icst_params lclk_params = {
-	.ref		= 24000000,
-	.vco_max	= ICST525_VCO_MAX_5V,
-	.vco_min	= ICST525_VCO_MIN,
-	.vd_min		= 8,
-	.vd_max		= 132,
-	.rd_min		= 24,
-	.rd_max		= 24,
-	.s2div		= icst525_s2div,
-	.idx2s		= icst525_idx2s,
-};
-
-static const struct icst_params cclk_params = {
-	.ref		= 24000000,
-	.vco_max	= ICST525_VCO_MAX_5V,
-	.vco_min	= ICST525_VCO_MIN,
-	.vd_min		= 12,
-	.vd_max		= 160,
-	.rd_min		= 24,
-	.rd_max		= 24,
-	.s2div		= icst525_s2div,
-	.idx2s		= icst525_idx2s,
-};
-
-/*
- * Validate the speed policy.
- */
-static int integrator_verify_policy(struct cpufreq_policy *policy)
-{
-	struct icst_vco vco;
-
-	cpufreq_verify_within_cpu_limits(policy);
-
-	vco = icst_hz_to_vco(&cclk_params, policy->max * 1000);
-	policy->max = icst_hz(&cclk_params, vco) / 1000;
-
-	vco = icst_hz_to_vco(&cclk_params, policy->min * 1000);
-	policy->min = icst_hz(&cclk_params, vco) / 1000;
-
-	cpufreq_verify_within_cpu_limits(policy);
-	return 0;
-}
-
-
-static int integrator_set_target(struct cpufreq_policy *policy,
-				 unsigned int target_freq,
-				 unsigned int relation)
-{
-	cpumask_t cpus_allowed;
-	int cpu = policy->cpu;
-	struct icst_vco vco;
-	struct cpufreq_freqs freqs;
-	u_int cm_osc;
-
-	/*
-	 * Save this threads cpus_allowed mask.
-	 */
-	cpus_allowed = current->cpus_allowed;
-
-	/*
-	 * Bind to the specified CPU.  When this call returns,
-	 * we should be running on the right CPU.
-	 */
-	set_cpus_allowed_ptr(current, cpumask_of(cpu));
-	BUG_ON(cpu != smp_processor_id());
-
-	/* get current setting */
-	cm_osc = __raw_readl(cm_base + INTEGRATOR_HDR_OSC_OFFSET);
-
-	if (machine_is_integrator())
-		vco.s = (cm_osc >> 8) & 7;
-	else if (machine_is_cintegrator())
-		vco.s = 1;
-	vco.v = cm_osc & 255;
-	vco.r = 22;
-	freqs.old = icst_hz(&cclk_params, vco) / 1000;
-
-	/* icst_hz_to_vco rounds down -- so we need the next
-	 * larger freq in case of CPUFREQ_RELATION_L.
-	 */
-	if (relation == CPUFREQ_RELATION_L)
-		target_freq += 999;
-	if (target_freq > policy->max)
-		target_freq = policy->max;
-	vco = icst_hz_to_vco(&cclk_params, target_freq * 1000);
-	freqs.new = icst_hz(&cclk_params, vco) / 1000;
-
-	if (freqs.old == freqs.new) {
-		set_cpus_allowed_ptr(current, &cpus_allowed);
-		return 0;
-	}
-
-	cpufreq_freq_transition_begin(policy, &freqs);
-
-	cm_osc = __raw_readl(cm_base + INTEGRATOR_HDR_OSC_OFFSET);
-
-	if (machine_is_integrator()) {
-		cm_osc &= 0xfffff800;
-		cm_osc |= vco.s << 8;
-	} else if (machine_is_cintegrator()) {
-		cm_osc &= 0xffffff00;
-	}
-	cm_osc |= vco.v;
-
-	__raw_writel(0xa05f, cm_base + INTEGRATOR_HDR_LOCK_OFFSET);
-	__raw_writel(cm_osc, cm_base + INTEGRATOR_HDR_OSC_OFFSET);
-	__raw_writel(0, cm_base + INTEGRATOR_HDR_LOCK_OFFSET);
-
-	/*
-	 * Restore the CPUs allowed mask.
-	 */
-	set_cpus_allowed_ptr(current, &cpus_allowed);
-
-	cpufreq_freq_transition_end(policy, &freqs, 0);
-
-	return 0;
-}
-
-static unsigned int integrator_get(unsigned int cpu)
-{
-	cpumask_t cpus_allowed;
-	unsigned int current_freq;
-	u_int cm_osc;
-	struct icst_vco vco;
-
-	cpus_allowed = current->cpus_allowed;
-
-	set_cpus_allowed_ptr(current, cpumask_of(cpu));
-	BUG_ON(cpu != smp_processor_id());
-
-	/* detect memory etc. */
-	cm_osc = __raw_readl(cm_base + INTEGRATOR_HDR_OSC_OFFSET);
-
-	if (machine_is_integrator())
-		vco.s = (cm_osc >> 8) & 7;
-	else
-		vco.s = 1;
-	vco.v = cm_osc & 255;
-	vco.r = 22;
-
-	current_freq = icst_hz(&cclk_params, vco) / 1000; /* current freq */
-
-	set_cpus_allowed_ptr(current, &cpus_allowed);
-
-	return current_freq;
-}
-
-static int integrator_cpufreq_init(struct cpufreq_policy *policy)
-{
-
-	/* set default policy and cpuinfo */
-	policy->max = policy->cpuinfo.max_freq = 160000;
-	policy->min = policy->cpuinfo.min_freq = 12000;
-	policy->cpuinfo.transition_latency = 1000000; /* 1 ms, assumed */
-
-	return 0;
-}
-
-static struct cpufreq_driver integrator_driver = {
-	.flags		= CPUFREQ_NEED_INITIAL_FREQ_CHECK,
-	.verify		= integrator_verify_policy,
-	.target		= integrator_set_target,
-	.get		= integrator_get,
-	.init		= integrator_cpufreq_init,
-	.name		= "integrator",
-};
-
-static int __init integrator_cpufreq_probe(struct platform_device *pdev)
-{
-	struct resource *res;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -ENODEV;
-
-	cm_base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
-	if (!cm_base)
-		return -ENODEV;
-
-	return cpufreq_register_driver(&integrator_driver);
-}
-
-static int __exit integrator_cpufreq_remove(struct platform_device *pdev)
-{
-	return cpufreq_unregister_driver(&integrator_driver);
-}
-
-static const struct of_device_id integrator_cpufreq_match[] = {
-	{ .compatible = "arm,core-module-integrator"},
-	{ },
-};
-
-MODULE_DEVICE_TABLE(of, integrator_cpufreq_match);
-
-static struct platform_driver integrator_cpufreq_driver = {
-	.driver = {
-		.name = "integrator-cpufreq",
-		.of_match_table = integrator_cpufreq_match,
-	},
-	.remove = __exit_p(integrator_cpufreq_remove),
-};
-
-module_platform_driver_probe(integrator_cpufreq_driver,
-			     integrator_cpufreq_probe);
-
-MODULE_AUTHOR("Russell M. King");
-MODULE_DESCRIPTION("cpufreq driver for ARM Integrator CPUs");
-MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index e8dc42fc0915..6acbd4af632e 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -37,6 +37,8 @@
 #include <asm/cpufeature.h>
 #include <asm/intel-family.h>
 
+#define INTEL_CPUFREQ_TRANSITION_LATENCY	20000
+
 #define ATOM_RATIOS		0x66a
 #define ATOM_VIDS		0x66b
 #define ATOM_TURBO_RATIOS	0x66c
@@ -53,6 +55,8 @@
 
 #define EXT_BITS 6
 #define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)
+#define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS)
+#define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS)
 
 static inline int32_t mul_fp(int32_t x, int32_t y)
 {
@@ -123,6 +127,8 @@ struct sample {
  * @scaling:		Scaling factor to  convert frequency to cpufreq
  *			frequency units
  * @turbo_pstate:	Max Turbo P state possible for this platform
+ * @max_freq:		@max_pstate frequency in cpufreq units
+ * @turbo_freq:		@turbo_pstate frequency in cpufreq units
  *
  * Stores the per cpu model P state limits and current P state.
  */
@@ -133,6 +139,8 @@ struct pstate_data {
 	int	max_pstate_physical;
 	int	scaling;
 	int	turbo_pstate;
+	unsigned int max_freq;
+	unsigned int turbo_freq;
 };
 
 /**
@@ -178,6 +186,48 @@ struct _pid {
 };
 
 /**
+ * struct perf_limits - Store user and policy limits
+ * @no_turbo:		User requested turbo state from intel_pstate sysfs
+ * @turbo_disabled:	Platform turbo status either from msr
+ *			MSR_IA32_MISC_ENABLE or when maximum available pstate
+ *			matches the maximum turbo pstate
+ * @max_perf_pct:	Effective maximum performance limit in percentage, this
+ *			is minimum of either limits enforced by cpufreq policy
+ *			or limits from user set limits via intel_pstate sysfs
+ * @min_perf_pct:	Effective minimum performance limit in percentage, this
+ *			is maximum of either limits enforced by cpufreq policy
+ *			or limits from user set limits via intel_pstate sysfs
+ * @max_perf:		This is a scaled value between 0 to 255 for max_perf_pct
+ *			This value is used to limit max pstate
+ * @min_perf:		This is a scaled value between 0 to 255 for min_perf_pct
+ *			This value is used to limit min pstate
+ * @max_policy_pct:	The maximum performance in percentage enforced by
+ *			cpufreq setpolicy interface
+ * @max_sysfs_pct:	The maximum performance in percentage enforced by
+ *			intel pstate sysfs interface, unused when per cpu
+ *			controls are enforced
+ * @min_policy_pct:	The minimum performance in percentage enforced by
+ *			cpufreq setpolicy interface
+ * @min_sysfs_pct:	The minimum performance in percentage enforced by
+ *			intel pstate sysfs interface, unused when per cpu
+ *			controls are enforced
+ *
+ * Storage for user and policy defined limits.
+ */
+struct perf_limits {
+	int no_turbo;
+	int turbo_disabled;
+	int max_perf_pct;
+	int min_perf_pct;
+	int32_t max_perf;
+	int32_t min_perf;
+	int max_policy_pct;
+	int max_sysfs_pct;
+	int min_policy_pct;
+	int min_sysfs_pct;
+};
+
+/**
  * struct cpudata -	Per CPU instance data storage
  * @cpu:		CPU number for this instance data
  * @policy:		CPUFreq policy value
@@ -195,8 +245,19 @@ struct _pid {
  * @prev_cummulative_iowait: IO Wait time difference from last and
  *			current sample
  * @sample:		Storage for storing last Sample data
+ * @perf_limits:	Pointer to perf_limit unique to this CPU
+ *			Not all field in the structure are applicable
+ *			when per cpu controls are enforced
  * @acpi_perf_data:	Stores ACPI perf information read from _PSS
  * @valid_pss_table:	Set to true for valid ACPI _PSS entries found
+ * @epp_powersave:	Last saved HWP energy performance preference
+ *			(EPP) or energy performance bias (EPB),
+ *			when policy switched to performance
+ * @epp_policy:		Last saved policy used to set EPP/EPB
+ * @epp_default:	Power on default HWP energy performance
+ *			preference/bias
+ * @epp_saved:		Saved EPP/EPB during system suspend or CPU offline
+ *			operation
  *
  * This structure stores per CPU instance data for all CPUs.
  */
@@ -218,11 +279,16 @@ struct cpudata {
 	u64	prev_tsc;
 	u64	prev_cummulative_iowait;
 	struct sample sample;
+	struct perf_limits *perf_limits;
 #ifdef CONFIG_ACPI
 	struct acpi_processor_performance acpi_perf_data;
 	bool valid_pss_table;
 #endif
 	unsigned int iowait_boost;
+	s16 epp_powersave;
+	s16 epp_policy;
+	s16 epp_default;
+	s16 epp_saved;
 };
 
 static struct cpudata **all_cpu_data;
@@ -236,7 +302,6 @@ static struct cpudata **all_cpu_data;
  * @p_gain_pct:		PID proportional gain
  * @i_gain_pct:		PID integral gain
  * @d_gain_pct:		PID derivative gain
- * @boost_iowait:	Whether or not to use iowait boosting.
  *
  * Stores per CPU model static PID configuration data.
  */
@@ -248,7 +313,6 @@ struct pstate_adjust_policy {
 	int p_gain_pct;
 	int d_gain_pct;
 	int i_gain_pct;
-	bool boost_iowait;
 };
 
 /**
@@ -292,58 +356,19 @@ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu);
 static struct pstate_adjust_policy pid_params __read_mostly;
 static struct pstate_funcs pstate_funcs __read_mostly;
 static int hwp_active __read_mostly;
+static bool per_cpu_limits __read_mostly;
 
 #ifdef CONFIG_ACPI
 static bool acpi_ppc;
 #endif
 
-/**
- * struct perf_limits - Store user and policy limits
- * @no_turbo:		User requested turbo state from intel_pstate sysfs
- * @turbo_disabled:	Platform turbo status either from msr
- *			MSR_IA32_MISC_ENABLE or when maximum available pstate
- *			matches the maximum turbo pstate
- * @max_perf_pct:	Effective maximum performance limit in percentage, this
- *			is minimum of either limits enforced by cpufreq policy
- *			or limits from user set limits via intel_pstate sysfs
- * @min_perf_pct:	Effective minimum performance limit in percentage, this
- *			is maximum of either limits enforced by cpufreq policy
- *			or limits from user set limits via intel_pstate sysfs
- * @max_perf:		This is a scaled value between 0 to 255 for max_perf_pct
- *			This value is used to limit max pstate
- * @min_perf:		This is a scaled value between 0 to 255 for min_perf_pct
- *			This value is used to limit min pstate
- * @max_policy_pct:	The maximum performance in percentage enforced by
- *			cpufreq setpolicy interface
- * @max_sysfs_pct:	The maximum performance in percentage enforced by
- *			intel pstate sysfs interface
- * @min_policy_pct:	The minimum performance in percentage enforced by
- *			cpufreq setpolicy interface
- * @min_sysfs_pct:	The minimum performance in percentage enforced by
- *			intel pstate sysfs interface
- *
- * Storage for user and policy defined limits.
- */
-struct perf_limits {
-	int no_turbo;
-	int turbo_disabled;
-	int max_perf_pct;
-	int min_perf_pct;
-	int32_t max_perf;
-	int32_t min_perf;
-	int max_policy_pct;
-	int max_sysfs_pct;
-	int min_policy_pct;
-	int min_sysfs_pct;
-};
-
 static struct perf_limits performance_limits = {
 	.no_turbo = 0,
 	.turbo_disabled = 0,
 	.max_perf_pct = 100,
-	.max_perf = int_tofp(1),
+	.max_perf = int_ext_tofp(1),
 	.min_perf_pct = 100,
-	.min_perf = int_tofp(1),
+	.min_perf = int_ext_tofp(1),
 	.max_policy_pct = 100,
 	.max_sysfs_pct = 100,
 	.min_policy_pct = 0,
@@ -354,7 +379,7 @@ static struct perf_limits powersave_limits = {
 	.no_turbo = 0,
 	.turbo_disabled = 0,
 	.max_perf_pct = 100,
-	.max_perf = int_tofp(1),
+	.max_perf = int_ext_tofp(1),
 	.min_perf_pct = 0,
 	.min_perf = 0,
 	.max_policy_pct = 100,
@@ -369,6 +394,8 @@ static struct perf_limits *limits = &performance_limits;
 static struct perf_limits *limits = &powersave_limits;
 #endif
 
+static DEFINE_MUTEX(intel_pstate_limits_lock);
+
 #ifdef CONFIG_ACPI
 
 static bool intel_pstate_get_ppc_enable_status(void)
@@ -513,11 +540,11 @@ static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
 }
 
 #else
-static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
+static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
 {
 }
 
-static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
+static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
 {
 }
 #endif
@@ -613,24 +640,252 @@ static inline void update_turbo_state(void)
 		 cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
 }
 
+static s16 intel_pstate_get_epb(struct cpudata *cpu_data)
+{
+	u64 epb;
+	int ret;
+
+	if (!static_cpu_has(X86_FEATURE_EPB))
+		return -ENXIO;
+
+	ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb);
+	if (ret)
+		return (s16)ret;
+
+	return (s16)(epb & 0x0f);
+}
+
+static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data)
+{
+	s16 epp;
+
+	if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
+		/*
+		 * When hwp_req_data is 0, means that caller didn't read
+		 * MSR_HWP_REQUEST, so need to read and get EPP.
+		 */
+		if (!hwp_req_data) {
+			epp = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST,
+					    &hwp_req_data);
+			if (epp)
+				return epp;
+		}
+		epp = (hwp_req_data >> 24) & 0xff;
+	} else {
+		/* When there is no EPP present, HWP uses EPB settings */
+		epp = intel_pstate_get_epb(cpu_data);
+	}
+
+	return epp;
+}
+
+static int intel_pstate_set_epb(int cpu, s16 pref)
+{
+	u64 epb;
+	int ret;
+
+	if (!static_cpu_has(X86_FEATURE_EPB))
+		return -ENXIO;
+
+	ret = rdmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb);
+	if (ret)
+		return ret;
+
+	epb = (epb & ~0x0f) | pref;
+	wrmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, epb);
+
+	return 0;
+}
+
+/*
+ * EPP/EPB display strings corresponding to EPP index in the
+ * energy_perf_strings[]
+ *	index		String
+ *-------------------------------------
+ *	0		default
+ *	1		performance
+ *	2		balance_performance
+ *	3		balance_power
+ *	4		power
+ */
+static const char * const energy_perf_strings[] = {
+	"default",
+	"performance",
+	"balance_performance",
+	"balance_power",
+	"power",
+	NULL
+};
+
+static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data)
+{
+	s16 epp;
+	int index = -EINVAL;
+
+	epp = intel_pstate_get_epp(cpu_data, 0);
+	if (epp < 0)
+		return epp;
+
+	if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
+		/*
+		 * Range:
+		 *	0x00-0x3F	:	Performance
+		 *	0x40-0x7F	:	Balance performance
+		 *	0x80-0xBF	:	Balance power
+		 *	0xC0-0xFF	:	Power
+		 * The EPP is a 8 bit value, but our ranges restrict the
+		 * value which can be set. Here only using top two bits
+		 * effectively.
+		 */
+		index = (epp >> 6) + 1;
+	} else if (static_cpu_has(X86_FEATURE_EPB)) {
+		/*
+		 * Range:
+		 *	0x00-0x03	:	Performance
+		 *	0x04-0x07	:	Balance performance
+		 *	0x08-0x0B	:	Balance power
+		 *	0x0C-0x0F	:	Power
+		 * The EPB is a 4 bit value, but our ranges restrict the
+		 * value which can be set. Here only using top two bits
+		 * effectively.
+		 */
+		index = (epp >> 2) + 1;
+	}
+
+	return index;
+}
+
+static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data,
+					      int pref_index)
+{
+	int epp = -EINVAL;
+	int ret;
+
+	if (!pref_index)
+		epp = cpu_data->epp_default;
+
+	mutex_lock(&intel_pstate_limits_lock);
+
+	if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
+		u64 value;
+
+		ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, &value);
+		if (ret)
+			goto return_pref;
+
+		value &= ~GENMASK_ULL(31, 24);
+
+		/*
+		 * If epp is not default, convert from index into
+		 * energy_perf_strings to epp value, by shifting 6
+		 * bits left to use only top two bits in epp.
+		 * The resultant epp need to shifted by 24 bits to
+		 * epp position in MSR_HWP_REQUEST.
+		 */
+		if (epp == -EINVAL)
+			epp = (pref_index - 1) << 6;
+
+		value |= (u64)epp << 24;
+		ret = wrmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, value);
+	} else {
+		if (epp == -EINVAL)
+			epp = (pref_index - 1) << 2;
+		ret = intel_pstate_set_epb(cpu_data->cpu, epp);
+	}
+return_pref:
+	mutex_unlock(&intel_pstate_limits_lock);
+
+	return ret;
+}
+
+static ssize_t show_energy_performance_available_preferences(
+				struct cpufreq_policy *policy, char *buf)
+{
+	int i = 0;
+	int ret = 0;
+
+	while (energy_perf_strings[i] != NULL)
+		ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]);
+
+	ret += sprintf(&buf[ret], "\n");
+
+	return ret;
+}
+
+cpufreq_freq_attr_ro(energy_performance_available_preferences);
+
+static ssize_t store_energy_performance_preference(
+		struct cpufreq_policy *policy, const char *buf, size_t count)
+{
+	struct cpudata *cpu_data = all_cpu_data[policy->cpu];
+	char str_preference[21];
+	int ret, i = 0;
+
+	ret = sscanf(buf, "%20s", str_preference);
+	if (ret != 1)
+		return -EINVAL;
+
+	while (energy_perf_strings[i] != NULL) {
+		if (!strcmp(str_preference, energy_perf_strings[i])) {
+			intel_pstate_set_energy_pref_index(cpu_data, i);
+			return count;
+		}
+		++i;
+	}
+
+	return -EINVAL;
+}
+
+static ssize_t show_energy_performance_preference(
+				struct cpufreq_policy *policy, char *buf)
+{
+	struct cpudata *cpu_data = all_cpu_data[policy->cpu];
+	int preference;
+
+	preference = intel_pstate_get_energy_pref_index(cpu_data);
+	if (preference < 0)
+		return preference;
+
+	return  sprintf(buf, "%s\n", energy_perf_strings[preference]);
+}
+
+cpufreq_freq_attr_rw(energy_performance_preference);
+
+static struct freq_attr *hwp_cpufreq_attrs[] = {
+	&energy_performance_preference,
+	&energy_performance_available_preferences,
+	NULL,
+};
+
 static void intel_pstate_hwp_set(const struct cpumask *cpumask)
 {
 	int min, hw_min, max, hw_max, cpu, range, adj_range;
+	struct perf_limits *perf_limits = limits;
 	u64 value, cap;
 
 	for_each_cpu(cpu, cpumask) {
+		int max_perf_pct, min_perf_pct;
+		struct cpudata *cpu_data = all_cpu_data[cpu];
+		s16 epp;
+
+		if (per_cpu_limits)
+			perf_limits = all_cpu_data[cpu]->perf_limits;
+
 		rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
 		hw_min = HWP_LOWEST_PERF(cap);
 		hw_max = HWP_HIGHEST_PERF(cap);
 		range = hw_max - hw_min;
 
+		max_perf_pct = perf_limits->max_perf_pct;
+		min_perf_pct = perf_limits->min_perf_pct;
+
 		rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
-		adj_range = limits->min_perf_pct * range / 100;
+		adj_range = min_perf_pct * range / 100;
 		min = hw_min + adj_range;
 		value &= ~HWP_MIN_PERF(~0L);
 		value |= HWP_MIN_PERF(min);
 
-		adj_range = limits->max_perf_pct * range / 100;
+		adj_range = max_perf_pct * range / 100;
 		max = hw_min + adj_range;
 		if (limits->no_turbo) {
 			hw_max = HWP_GUARANTEED_PERF(cap);
@@ -640,6 +895,53 @@ static void intel_pstate_hwp_set(const struct cpumask *cpumask)
 
 		value &= ~HWP_MAX_PERF(~0L);
 		value |= HWP_MAX_PERF(max);
+
+		if (cpu_data->epp_policy == cpu_data->policy)
+			goto skip_epp;
+
+		cpu_data->epp_policy = cpu_data->policy;
+
+		if (cpu_data->epp_saved >= 0) {
+			epp = cpu_data->epp_saved;
+			cpu_data->epp_saved = -EINVAL;
+			goto update_epp;
+		}
+
+		if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) {
+			epp = intel_pstate_get_epp(cpu_data, value);
+			cpu_data->epp_powersave = epp;
+			/* If EPP read was failed, then don't try to write */
+			if (epp < 0)
+				goto skip_epp;
+
+
+			epp = 0;
+		} else {
+			/* skip setting EPP, when saved value is invalid */
+			if (cpu_data->epp_powersave < 0)
+				goto skip_epp;
+
+			/*
+			 * No need to restore EPP when it is not zero. This
+			 * means:
+			 *  - Policy is not changed
+			 *  - user has manually changed
+			 *  - Error reading EPB
+			 */
+			epp = intel_pstate_get_epp(cpu_data, value);
+			if (epp)
+				goto skip_epp;
+
+			epp = cpu_data->epp_powersave;
+		}
+update_epp:
+		if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
+			value &= ~GENMASK_ULL(31, 24);
+			value |= (u64)epp << 24;
+		} else {
+			intel_pstate_set_epb(cpu, epp);
+		}
+skip_epp:
 		wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
 	}
 }
@@ -652,6 +954,28 @@ static int intel_pstate_hwp_set_policy(struct cpufreq_policy *policy)
 	return 0;
 }
 
+static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy)
+{
+	struct cpudata *cpu_data = all_cpu_data[policy->cpu];
+
+	if (!hwp_active)
+		return 0;
+
+	cpu_data->epp_saved = intel_pstate_get_epp(cpu_data, 0);
+
+	return 0;
+}
+
+static int intel_pstate_resume(struct cpufreq_policy *policy)
+{
+	if (!hwp_active)
+		return 0;
+
+	all_cpu_data[policy->cpu]->epp_policy = 0;
+
+	return intel_pstate_hwp_set_policy(policy);
+}
+
 static void intel_pstate_hwp_set_online_cpus(void)
 {
 	get_online_cpus();
@@ -694,8 +1018,10 @@ static void __init intel_pstate_debug_expose_params(void)
 	struct dentry *debugfs_parent;
 	int i = 0;
 
-	if (hwp_active)
+	if (hwp_active ||
+	    pstate_funcs.get_target_pstate == get_target_pstate_use_cpu_load)
 		return;
+
 	debugfs_parent = debugfs_create_dir("pstate_snb", NULL);
 	if (IS_ERR_OR_NULL(debugfs_parent))
 		return;
@@ -768,9 +1094,12 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
 	if (ret != 1)
 		return -EINVAL;
 
+	mutex_lock(&intel_pstate_limits_lock);
+
 	update_turbo_state();
 	if (limits->turbo_disabled) {
 		pr_warn("Turbo disabled by BIOS or unavailable on processor\n");
+		mutex_unlock(&intel_pstate_limits_lock);
 		return -EPERM;
 	}
 
@@ -779,6 +1108,8 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
 	if (hwp_active)
 		intel_pstate_hwp_set_online_cpus();
 
+	mutex_unlock(&intel_pstate_limits_lock);
+
 	return count;
 }
 
@@ -792,6 +1123,8 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
 	if (ret != 1)
 		return -EINVAL;
 
+	mutex_lock(&intel_pstate_limits_lock);
+
 	limits->max_sysfs_pct = clamp_t(int, input, 0 , 100);
 	limits->max_perf_pct = min(limits->max_policy_pct,
 				   limits->max_sysfs_pct);
@@ -799,10 +1132,13 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
 				   limits->max_perf_pct);
 	limits->max_perf_pct = max(limits->min_perf_pct,
 				   limits->max_perf_pct);
-	limits->max_perf = div_fp(limits->max_perf_pct, 100);
+	limits->max_perf = div_ext_fp(limits->max_perf_pct, 100);
 
 	if (hwp_active)
 		intel_pstate_hwp_set_online_cpus();
+
+	mutex_unlock(&intel_pstate_limits_lock);
+
 	return count;
 }
 
@@ -816,6 +1152,8 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
 	if (ret != 1)
 		return -EINVAL;
 
+	mutex_lock(&intel_pstate_limits_lock);
+
 	limits->min_sysfs_pct = clamp_t(int, input, 0 , 100);
 	limits->min_perf_pct = max(limits->min_policy_pct,
 				   limits->min_sysfs_pct);
@@ -823,10 +1161,13 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
 				   limits->min_perf_pct);
 	limits->min_perf_pct = min(limits->max_perf_pct,
 				   limits->min_perf_pct);
-	limits->min_perf = div_fp(limits->min_perf_pct, 100);
+	limits->min_perf = div_ext_fp(limits->min_perf_pct, 100);
 
 	if (hwp_active)
 		intel_pstate_hwp_set_online_cpus();
+
+	mutex_unlock(&intel_pstate_limits_lock);
+
 	return count;
 }
 
@@ -841,8 +1182,6 @@ define_one_global_ro(num_pstates);
 
 static struct attribute *intel_pstate_attributes[] = {
 	&no_turbo.attr,
-	&max_perf_pct.attr,
-	&min_perf_pct.attr,
 	&turbo_pct.attr,
 	&num_pstates.attr,
 	NULL
@@ -859,9 +1198,26 @@ static void __init intel_pstate_sysfs_expose_params(void)
 
 	intel_pstate_kobject = kobject_create_and_add("intel_pstate",
 						&cpu_subsys.dev_root->kobj);
-	BUG_ON(!intel_pstate_kobject);
+	if (WARN_ON(!intel_pstate_kobject))
+		return;
+
 	rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
-	BUG_ON(rc);
+	if (WARN_ON(rc))
+		return;
+
+	/*
+	 * If per cpu limits are enforced there are no global limits, so
+	 * return without creating max/min_perf_pct attributes
+	 */
+	if (per_cpu_limits)
+		return;
+
+	rc = sysfs_create_file(intel_pstate_kobject, &max_perf_pct.attr);
+	WARN_ON(rc);
+
+	rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr);
+	WARN_ON(rc);
+
 }
 /************************** sysfs end ************************/
 
@@ -872,6 +1228,9 @@ static void intel_pstate_hwp_enable(struct cpudata *cpudata)
 		wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
 
 	wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
+	cpudata->epp_policy = 0;
+	if (cpudata->epp_default == -EINVAL)
+		cpudata->epp_default = intel_pstate_get_epp(cpudata, 0);
 }
 
 static int atom_get_min_pstate(void)
@@ -1099,7 +1458,6 @@ static const struct cpu_defaults silvermont_params = {
 		.p_gain_pct = 14,
 		.d_gain_pct = 0,
 		.i_gain_pct = 4,
-		.boost_iowait = true,
 	},
 	.funcs = {
 		.get_max = atom_get_max_pstate,
@@ -1121,7 +1479,6 @@ static const struct cpu_defaults airmont_params = {
 		.p_gain_pct = 14,
 		.d_gain_pct = 0,
 		.i_gain_pct = 4,
-		.boost_iowait = true,
 	},
 	.funcs = {
 		.get_max = atom_get_max_pstate,
@@ -1163,7 +1520,6 @@ static const struct cpu_defaults bxt_params = {
 		.p_gain_pct = 14,
 		.d_gain_pct = 0,
 		.i_gain_pct = 4,
-		.boost_iowait = true,
 	},
 	.funcs = {
 		.get_max = core_get_max_pstate,
@@ -1181,20 +1537,24 @@ static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
 	int max_perf = cpu->pstate.turbo_pstate;
 	int max_perf_adj;
 	int min_perf;
+	struct perf_limits *perf_limits = limits;
 
 	if (limits->no_turbo || limits->turbo_disabled)
 		max_perf = cpu->pstate.max_pstate;
 
+	if (per_cpu_limits)
+		perf_limits = cpu->perf_limits;
+
 	/*
 	 * performance can be limited by user through sysfs, by cpufreq
 	 * policy, or by cpu specific default values determined through
 	 * experimentation.
 	 */
-	max_perf_adj = fp_toint(max_perf * limits->max_perf);
+	max_perf_adj = fp_ext_toint(max_perf * perf_limits->max_perf);
 	*max = clamp_t(int, max_perf_adj,
 			cpu->pstate.min_pstate, cpu->pstate.turbo_pstate);
 
-	min_perf = fp_toint(max_perf * limits->min_perf);
+	min_perf = fp_ext_toint(max_perf * perf_limits->min_perf);
 	*min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
 }
 
@@ -1232,6 +1592,8 @@ static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
 	cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical();
 	cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
 	cpu->pstate.scaling = pstate_funcs.get_scaling();
+	cpu->pstate.max_freq = cpu->pstate.max_pstate * cpu->pstate.scaling;
+	cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
 
 	if (pstate_funcs.get_vid)
 		pstate_funcs.get_vid(cpu);
@@ -1370,15 +1732,19 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
 	return cpu->pstate.current_pstate - pid_calc(&cpu->pid, perf_scaled);
 }
 
-static inline void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
+static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
 {
 	int max_perf, min_perf;
 
-	update_turbo_state();
-
 	intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
 	pstate = clamp_t(int, pstate, min_perf, max_perf);
 	trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
+	return pstate;
+}
+
+static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
+{
+	pstate = intel_pstate_prepare_request(cpu, pstate);
 	if (pstate == cpu->pstate.current_pstate)
 		return;
 
@@ -1396,6 +1762,8 @@ static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
 	target_pstate = cpu->policy == CPUFREQ_POLICY_PERFORMANCE ?
 		cpu->pstate.turbo_pstate : pstate_funcs.get_target_pstate(cpu);
 
+	update_turbo_state();
+
 	intel_pstate_update_pstate(cpu, target_pstate);
 
 	sample = &cpu->sample;
@@ -1416,7 +1784,7 @@ static void intel_pstate_update_util(struct update_util_data *data, u64 time,
 	struct cpudata *cpu = container_of(data, struct cpudata, update_util);
 	u64 delta_ns;
 
-	if (pid_params.boost_iowait) {
+	if (pstate_funcs.get_target_pstate == get_target_pstate_use_cpu_load) {
 		if (flags & SCHED_CPUFREQ_IOWAIT) {
 			cpu->iowait_boost = int_tofp(1);
 		} else if (cpu->iowait_boost) {
@@ -1462,6 +1830,7 @@ static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
 	ICPU(INTEL_FAM6_SKYLAKE_DESKTOP,	core_params),
 	ICPU(INTEL_FAM6_BROADWELL_XEON_D,	core_params),
 	ICPU(INTEL_FAM6_XEON_PHI_KNL,		knl_params),
+	ICPU(INTEL_FAM6_XEON_PHI_KNM,		knl_params),
 	ICPU(INTEL_FAM6_ATOM_GOLDMONT,		bxt_params),
 	{}
 };
@@ -1478,11 +1847,26 @@ static int intel_pstate_init_cpu(unsigned int cpunum)
 {
 	struct cpudata *cpu;
 
-	if (!all_cpu_data[cpunum])
-		all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata),
-					       GFP_KERNEL);
-	if (!all_cpu_data[cpunum])
-		return -ENOMEM;
+	cpu = all_cpu_data[cpunum];
+
+	if (!cpu) {
+		unsigned int size = sizeof(struct cpudata);
+
+		if (per_cpu_limits)
+			size += sizeof(struct perf_limits);
+
+		cpu = kzalloc(size, GFP_KERNEL);
+		if (!cpu)
+			return -ENOMEM;
+
+		all_cpu_data[cpunum] = cpu;
+		if (per_cpu_limits)
+			cpu->perf_limits = (struct perf_limits *)(cpu + 1);
+
+		cpu->epp_default = -EINVAL;
+		cpu->epp_powersave = -EINVAL;
+		cpu->epp_saved = -EINVAL;
+	}
 
 	cpu = all_cpu_data[cpunum];
 
@@ -1541,18 +1925,57 @@ static void intel_pstate_set_performance_limits(struct perf_limits *limits)
 	limits->no_turbo = 0;
 	limits->turbo_disabled = 0;
 	limits->max_perf_pct = 100;
-	limits->max_perf = int_tofp(1);
+	limits->max_perf = int_ext_tofp(1);
 	limits->min_perf_pct = 100;
-	limits->min_perf = int_tofp(1);
+	limits->min_perf = int_ext_tofp(1);
 	limits->max_policy_pct = 100;
 	limits->max_sysfs_pct = 100;
 	limits->min_policy_pct = 0;
 	limits->min_sysfs_pct = 0;
 }
 
+static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy,
+					    struct perf_limits *limits)
+{
+
+	limits->max_policy_pct = DIV_ROUND_UP(policy->max * 100,
+					      policy->cpuinfo.max_freq);
+	limits->max_policy_pct = clamp_t(int, limits->max_policy_pct, 0, 100);
+	if (policy->max == policy->min) {
+		limits->min_policy_pct = limits->max_policy_pct;
+	} else {
+		limits->min_policy_pct = DIV_ROUND_UP(policy->min * 100,
+						      policy->cpuinfo.max_freq);
+		limits->min_policy_pct = clamp_t(int, limits->min_policy_pct,
+						 0, 100);
+	}
+
+	/* Normalize user input to [min_policy_pct, max_policy_pct] */
+	limits->min_perf_pct = max(limits->min_policy_pct,
+				   limits->min_sysfs_pct);
+	limits->min_perf_pct = min(limits->max_policy_pct,
+				   limits->min_perf_pct);
+	limits->max_perf_pct = min(limits->max_policy_pct,
+				   limits->max_sysfs_pct);
+	limits->max_perf_pct = max(limits->min_policy_pct,
+				   limits->max_perf_pct);
+
+	/* Make sure min_perf_pct <= max_perf_pct */
+	limits->min_perf_pct = min(limits->max_perf_pct, limits->min_perf_pct);
+
+	limits->min_perf = div_ext_fp(limits->min_perf_pct, 100);
+	limits->max_perf = div_ext_fp(limits->max_perf_pct, 100);
+	limits->max_perf = round_up(limits->max_perf, EXT_FRAC_BITS);
+	limits->min_perf = round_up(limits->min_perf, EXT_FRAC_BITS);
+
+	pr_debug("cpu:%d max_perf_pct:%d min_perf_pct:%d\n", policy->cpu,
+		 limits->max_perf_pct, limits->min_perf_pct);
+}
+
 static int intel_pstate_set_policy(struct cpufreq_policy *policy)
 {
 	struct cpudata *cpu;
+	struct perf_limits *perf_limits = NULL;
 
 	if (!policy->cpuinfo.max_freq)
 		return -ENODEV;
@@ -1570,41 +1993,31 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy)
 		policy->max = policy->cpuinfo.max_freq;
 	}
 
-	if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
-		limits = &performance_limits;
+	if (per_cpu_limits)
+		perf_limits = cpu->perf_limits;
+
+	mutex_lock(&intel_pstate_limits_lock);
+
+	if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) {
+		if (!perf_limits) {
+			limits = &performance_limits;
+			perf_limits = limits;
+		}
 		if (policy->max >= policy->cpuinfo.max_freq) {
 			pr_debug("set performance\n");
-			intel_pstate_set_performance_limits(limits);
+			intel_pstate_set_performance_limits(perf_limits);
 			goto out;
 		}
 	} else {
 		pr_debug("set powersave\n");
-		limits = &powersave_limits;
-	}
-
-	limits->min_policy_pct = (policy->min * 100) / policy->cpuinfo.max_freq;
-	limits->min_policy_pct = clamp_t(int, limits->min_policy_pct, 0 , 100);
-	limits->max_policy_pct = DIV_ROUND_UP(policy->max * 100,
-					      policy->cpuinfo.max_freq);
-	limits->max_policy_pct = clamp_t(int, limits->max_policy_pct, 0 , 100);
-
-	/* Normalize user input to [min_policy_pct, max_policy_pct] */
-	limits->min_perf_pct = max(limits->min_policy_pct,
-				   limits->min_sysfs_pct);
-	limits->min_perf_pct = min(limits->max_policy_pct,
-				   limits->min_perf_pct);
-	limits->max_perf_pct = min(limits->max_policy_pct,
-				   limits->max_sysfs_pct);
-	limits->max_perf_pct = max(limits->min_policy_pct,
-				   limits->max_perf_pct);
-
-	/* Make sure min_perf_pct <= max_perf_pct */
-	limits->min_perf_pct = min(limits->max_perf_pct, limits->min_perf_pct);
+		if (!perf_limits) {
+			limits = &powersave_limits;
+			perf_limits = limits;
+		}
 
-	limits->min_perf = div_fp(limits->min_perf_pct, 100);
-	limits->max_perf = div_fp(limits->max_perf_pct, 100);
-	limits->max_perf = round_up(limits->max_perf, FRAC_BITS);
+	}
 
+	intel_pstate_update_perf_limits(policy, perf_limits);
  out:
 	if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
 		/*
@@ -1619,6 +2032,8 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy)
 
 	intel_pstate_hwp_set_policy(policy);
 
+	mutex_unlock(&intel_pstate_limits_lock);
+
 	return 0;
 }
 
@@ -1633,22 +2048,32 @@ static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
 	return 0;
 }
 
+static void intel_cpufreq_stop_cpu(struct cpufreq_policy *policy)
+{
+	intel_pstate_set_min_pstate(all_cpu_data[policy->cpu]);
+}
+
 static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
 {
-	int cpu_num = policy->cpu;
-	struct cpudata *cpu = all_cpu_data[cpu_num];
+	pr_debug("CPU %d exiting\n", policy->cpu);
 
-	pr_debug("CPU %d exiting\n", cpu_num);
+	intel_pstate_clear_update_util_hook(policy->cpu);
+	if (hwp_active)
+		intel_pstate_hwp_save_state(policy);
+	else
+		intel_cpufreq_stop_cpu(policy);
+}
 
-	intel_pstate_clear_update_util_hook(cpu_num);
+static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
+{
+	intel_pstate_exit_perf_limits(policy);
 
-	if (hwp_active)
-		return;
+	policy->fast_switch_possible = false;
 
-	intel_pstate_set_min_pstate(cpu);
+	return 0;
 }
 
-static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
+static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
 {
 	struct cpudata *cpu;
 	int rc;
@@ -1659,10 +2084,13 @@ static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
 
 	cpu = all_cpu_data[policy->cpu];
 
-	if (limits->min_perf_pct == 100 && limits->max_perf_pct == 100)
-		policy->policy = CPUFREQ_POLICY_PERFORMANCE;
-	else
-		policy->policy = CPUFREQ_POLICY_POWERSAVE;
+	/*
+	 * We need sane value in the cpu->perf_limits, so inherit from global
+	 * perf_limits limits, which are seeded with values based on the
+	 * CONFIG_CPU_FREQ_DEFAULT_GOV_*, during boot up.
+	 */
+	if (per_cpu_limits)
+		memcpy(cpu->perf_limits, limits, sizeof(struct perf_limits));
 
 	policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
 	policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
@@ -1675,24 +2103,35 @@ static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
 	policy->cpuinfo.max_freq *= cpu->pstate.scaling;
 
 	intel_pstate_init_acpi_perf_limits(policy);
-	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
 	cpumask_set_cpu(policy->cpu, policy->cpus);
 
+	policy->fast_switch_possible = true;
+
 	return 0;
 }
 
-static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
+static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
 {
-	intel_pstate_exit_perf_limits(policy);
+	int ret = __intel_pstate_cpu_init(policy);
+
+	if (ret)
+		return ret;
+
+	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
+	if (limits->min_perf_pct == 100 && limits->max_perf_pct == 100)
+		policy->policy = CPUFREQ_POLICY_PERFORMANCE;
+	else
+		policy->policy = CPUFREQ_POLICY_POWERSAVE;
 
 	return 0;
 }
 
-static struct cpufreq_driver intel_pstate_driver = {
+static struct cpufreq_driver intel_pstate = {
 	.flags		= CPUFREQ_CONST_LOOPS,
 	.verify		= intel_pstate_verify_policy,
 	.setpolicy	= intel_pstate_set_policy,
-	.resume		= intel_pstate_hwp_set_policy,
+	.suspend	= intel_pstate_hwp_save_state,
+	.resume		= intel_pstate_resume,
 	.get		= intel_pstate_get,
 	.init		= intel_pstate_cpu_init,
 	.exit		= intel_pstate_cpu_exit,
@@ -1700,6 +2139,118 @@ static struct cpufreq_driver intel_pstate_driver = {
 	.name		= "intel_pstate",
 };
 
+static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy)
+{
+	struct cpudata *cpu = all_cpu_data[policy->cpu];
+	struct perf_limits *perf_limits = limits;
+
+	update_turbo_state();
+	policy->cpuinfo.max_freq = limits->turbo_disabled ?
+			cpu->pstate.max_freq : cpu->pstate.turbo_freq;
+
+	cpufreq_verify_within_cpu_limits(policy);
+
+	if (per_cpu_limits)
+		perf_limits = cpu->perf_limits;
+
+	intel_pstate_update_perf_limits(policy, perf_limits);
+
+	return 0;
+}
+
+static unsigned int intel_cpufreq_turbo_update(struct cpudata *cpu,
+					       struct cpufreq_policy *policy,
+					       unsigned int target_freq)
+{
+	unsigned int max_freq;
+
+	update_turbo_state();
+
+	max_freq = limits->no_turbo || limits->turbo_disabled ?
+			cpu->pstate.max_freq : cpu->pstate.turbo_freq;
+	policy->cpuinfo.max_freq = max_freq;
+	if (policy->max > max_freq)
+		policy->max = max_freq;
+
+	if (target_freq > max_freq)
+		target_freq = max_freq;
+
+	return target_freq;
+}
+
+static int intel_cpufreq_target(struct cpufreq_policy *policy,
+				unsigned int target_freq,
+				unsigned int relation)
+{
+	struct cpudata *cpu = all_cpu_data[policy->cpu];
+	struct cpufreq_freqs freqs;
+	int target_pstate;
+
+	freqs.old = policy->cur;
+	freqs.new = intel_cpufreq_turbo_update(cpu, policy, target_freq);
+
+	cpufreq_freq_transition_begin(policy, &freqs);
+	switch (relation) {
+	case CPUFREQ_RELATION_L:
+		target_pstate = DIV_ROUND_UP(freqs.new, cpu->pstate.scaling);
+		break;
+	case CPUFREQ_RELATION_H:
+		target_pstate = freqs.new / cpu->pstate.scaling;
+		break;
+	default:
+		target_pstate = DIV_ROUND_CLOSEST(freqs.new, cpu->pstate.scaling);
+		break;
+	}
+	target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
+	if (target_pstate != cpu->pstate.current_pstate) {
+		cpu->pstate.current_pstate = target_pstate;
+		wrmsrl_on_cpu(policy->cpu, MSR_IA32_PERF_CTL,
+			      pstate_funcs.get_val(cpu, target_pstate));
+	}
+	cpufreq_freq_transition_end(policy, &freqs, false);
+
+	return 0;
+}
+
+static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy,
+					      unsigned int target_freq)
+{
+	struct cpudata *cpu = all_cpu_data[policy->cpu];
+	int target_pstate;
+
+	target_freq = intel_cpufreq_turbo_update(cpu, policy, target_freq);
+	target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling);
+	intel_pstate_update_pstate(cpu, target_pstate);
+	return target_freq;
+}
+
+static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
+{
+	int ret = __intel_pstate_cpu_init(policy);
+
+	if (ret)
+		return ret;
+
+	policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY;
+	/* This reflects the intel_pstate_get_cpu_pstates() setting. */
+	policy->cur = policy->cpuinfo.min_freq;
+
+	return 0;
+}
+
+static struct cpufreq_driver intel_cpufreq = {
+	.flags		= CPUFREQ_CONST_LOOPS,
+	.verify		= intel_cpufreq_verify_policy,
+	.target		= intel_cpufreq_target,
+	.fast_switch	= intel_cpufreq_fast_switch,
+	.init		= intel_cpufreq_cpu_init,
+	.exit		= intel_pstate_cpu_exit,
+	.stop_cpu	= intel_cpufreq_stop_cpu,
+	.name		= "intel_cpufreq",
+};
+
+static struct cpufreq_driver *intel_pstate_driver = &intel_pstate;
+
 static int no_load __initdata;
 static int no_hwp __initdata;
 static int hwp_only __initdata;
@@ -1726,6 +2277,19 @@ static void __init copy_pid_params(struct pstate_adjust_policy *policy)
 	pid_params.setpoint = policy->setpoint;
 }
 
+#ifdef CONFIG_ACPI
+static void intel_pstate_use_acpi_profile(void)
+{
+	if (acpi_gbl_FADT.preferred_profile == PM_MOBILE)
+		pstate_funcs.get_target_pstate =
+				get_target_pstate_use_cpu_load;
+}
+#else
+static void intel_pstate_use_acpi_profile(void)
+{
+}
+#endif
+
 static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
 {
 	pstate_funcs.get_max   = funcs->get_max;
@@ -1737,6 +2301,7 @@ static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
 	pstate_funcs.get_vid   = funcs->get_vid;
 	pstate_funcs.get_target_pstate = funcs->get_target_pstate;
 
+	intel_pstate_use_acpi_profile();
 }
 
 #ifdef CONFIG_ACPI
@@ -1850,9 +2415,20 @@ static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
 
 	return false;
 }
+
+static void intel_pstate_request_control_from_smm(void)
+{
+	/*
+	 * It may be unsafe to request P-states control from SMM if _PPC support
+	 * has not been enabled.
+	 */
+	if (acpi_ppc)
+		acpi_processor_pstate_control();
+}
 #else /* CONFIG_ACPI not enabled */
 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
 static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
+static inline void intel_pstate_request_control_from_smm(void) {}
 #endif /* CONFIG_ACPI */
 
 static const struct x86_cpu_id hwp_support_ids[] __initconst = {
@@ -1872,6 +2448,7 @@ static int __init intel_pstate_init(void)
 	if (x86_match_cpu(hwp_support_ids) && !no_hwp) {
 		copy_cpu_funcs(&core_params.funcs);
 		hwp_active++;
+		intel_pstate.attr = hwp_cpufreq_attrs;
 		goto hwp_cpu_matched;
 	}
 
@@ -1904,7 +2481,9 @@ hwp_cpu_matched:
 	if (!hwp_active && hwp_only)
 		goto out;
 
-	rc = cpufreq_register_driver(&intel_pstate_driver);
+	intel_pstate_request_control_from_smm();
+
+	rc = cpufreq_register_driver(intel_pstate_driver);
 	if (rc)
 		goto out;
 
@@ -1919,7 +2498,9 @@ out:
 	get_online_cpus();
 	for_each_online_cpu(cpu) {
 		if (all_cpu_data[cpu]) {
-			intel_pstate_clear_update_util_hook(cpu);
+			if (intel_pstate_driver == &intel_pstate)
+				intel_pstate_clear_update_util_hook(cpu);
+
 			kfree(all_cpu_data[cpu]);
 		}
 	}
@@ -1935,8 +2516,13 @@ static int __init intel_pstate_setup(char *str)
 	if (!str)
 		return -EINVAL;
 
-	if (!strcmp(str, "disable"))
+	if (!strcmp(str, "disable")) {
 		no_load = 1;
+	} else if (!strcmp(str, "passive")) {
+		pr_info("Passive mode enabled\n");
+		intel_pstate_driver = &intel_cpufreq;
+		no_hwp = 1;
+	}
 	if (!strcmp(str, "no_hwp")) {
 		pr_info("HWP disabled\n");
 		no_hwp = 1;
@@ -1945,6 +2531,8 @@ static int __init intel_pstate_setup(char *str)
 		force_load = 1;
 	if (!strcmp(str, "hwp_only"))
 		hwp_only = 1;
+	if (!strcmp(str, "per_cpu_perf_limits"))
+		per_cpu_limits = true;
 
 #ifdef CONFIG_ACPI
 	if (!strcmp(str, "support_acpi_ppc"))
diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c
index d3ffde806629..37671b545880 100644
--- a/drivers/cpufreq/powernv-cpufreq.c
+++ b/drivers/cpufreq/powernv-cpufreq.c
@@ -42,6 +42,10 @@
 #define PMSR_PSAFE_ENABLE	(1UL << 30)
 #define PMSR_SPR_EM_DISABLE	(1UL << 31)
 #define PMSR_MAX(x)		((x >> 32) & 0xFF)
+#define LPSTATE_SHIFT		48
+#define GPSTATE_SHIFT		56
+#define GET_LPSTATE(x)		(((x) >> LPSTATE_SHIFT) & 0xFF)
+#define GET_GPSTATE(x)		(((x) >> GPSTATE_SHIFT) & 0xFF)
 
 #define MAX_RAMP_DOWN_TIME				5120
 /*
@@ -592,7 +596,8 @@ void gpstate_timer_handler(unsigned long data)
 {
 	struct cpufreq_policy *policy = (struct cpufreq_policy *)data;
 	struct global_pstate_info *gpstates = policy->driver_data;
-	int gpstate_idx;
+	int gpstate_idx, lpstate_idx;
+	unsigned long val;
 	unsigned int time_diff = jiffies_to_msecs(jiffies)
 					- gpstates->last_sampled_time;
 	struct powernv_smp_call_data freq_data;
@@ -600,21 +605,37 @@ void gpstate_timer_handler(unsigned long data)
 	if (!spin_trylock(&gpstates->gpstate_lock))
 		return;
 
+	/*
+	 * If PMCR was last updated was using fast_swtich then
+	 * We may have wrong in gpstate->last_lpstate_idx
+	 * value. Hence, read from PMCR to get correct data.
+	 */
+	val = get_pmspr(SPRN_PMCR);
+	freq_data.gpstate_id = (s8)GET_GPSTATE(val);
+	freq_data.pstate_id = (s8)GET_LPSTATE(val);
+	if (freq_data.gpstate_id  == freq_data.pstate_id) {
+		reset_gpstates(policy);
+		spin_unlock(&gpstates->gpstate_lock);
+		return;
+	}
+
 	gpstates->last_sampled_time += time_diff;
 	gpstates->elapsed_time += time_diff;
-	freq_data.pstate_id = idx_to_pstate(gpstates->last_lpstate_idx);
 
-	if ((gpstates->last_gpstate_idx == gpstates->last_lpstate_idx) ||
-	    (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME)) {
+	if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {
 		gpstate_idx = pstate_to_idx(freq_data.pstate_id);
+		lpstate_idx = gpstate_idx;
 		reset_gpstates(policy);
 		gpstates->highest_lpstate_idx = gpstate_idx;
 	} else {
+		lpstate_idx = pstate_to_idx(freq_data.pstate_id);
 		gpstate_idx = calc_global_pstate(gpstates->elapsed_time,
 						 gpstates->highest_lpstate_idx,
-						 gpstates->last_lpstate_idx);
+						 lpstate_idx);
 	}
-
+	freq_data.gpstate_id = idx_to_pstate(gpstate_idx);
+	gpstates->last_gpstate_idx = gpstate_idx;
+	gpstates->last_lpstate_idx = lpstate_idx;
 	/*
 	 * If local pstate is equal to global pstate, rampdown is over
 	 * So timer is not required to be queued.
@@ -622,10 +643,6 @@ void gpstate_timer_handler(unsigned long data)
 	if (gpstate_idx != gpstates->last_lpstate_idx)
 		queue_gpstate_timer(gpstates);
 
-	freq_data.gpstate_id = idx_to_pstate(gpstate_idx);
-	gpstates->last_gpstate_idx = pstate_to_idx(freq_data.gpstate_id);
-	gpstates->last_lpstate_idx = pstate_to_idx(freq_data.pstate_id);
-
 	spin_unlock(&gpstates->gpstate_lock);
 
 	/* Timer may get migrated to a different cpu on cpu hot unplug */
@@ -647,8 +664,14 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy,
 	if (unlikely(rebooting) && new_index != get_nominal_index())
 		return 0;
 
-	if (!throttled)
+	if (!throttled) {
+		/* we don't want to be preempted while
+		 * checking if the CPU frequency has been throttled
+		 */
+		preempt_disable();
 		powernv_cpufreq_throttle_check(NULL);
+		preempt_enable();
+	}
 
 	cur_msec = jiffies_to_msecs(get_jiffies_64());
 
@@ -752,9 +775,12 @@ static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy)
 	spin_lock_init(&gpstates->gpstate_lock);
 	ret = cpufreq_table_validate_and_show(policy, powernv_freqs);
 
-	if (ret < 0)
+	if (ret < 0) {
 		kfree(policy->driver_data);
+		return ret;
+	}
 
+	policy->fast_switch_possible = true;
 	return ret;
 }
 
@@ -897,6 +923,20 @@ static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy)
 	del_timer_sync(&gpstates->timer);
 }
 
+static unsigned int powernv_fast_switch(struct cpufreq_policy *policy,
+					unsigned int target_freq)
+{
+	int index;
+	struct powernv_smp_call_data freq_data;
+
+	index = cpufreq_table_find_index_dl(policy, target_freq);
+	freq_data.pstate_id = powernv_freqs[index].driver_data;
+	freq_data.gpstate_id = powernv_freqs[index].driver_data;
+	set_pstate(&freq_data);
+
+	return powernv_freqs[index].frequency;
+}
+
 static struct cpufreq_driver powernv_cpufreq_driver = {
 	.name		= "powernv-cpufreq",
 	.flags		= CPUFREQ_CONST_LOOPS,
@@ -904,6 +944,7 @@ static struct cpufreq_driver powernv_cpufreq_driver = {
 	.exit		= powernv_cpufreq_cpu_exit,
 	.verify		= cpufreq_generic_frequency_table_verify,
 	.target_index	= powernv_cpufreq_target_index,
+	.fast_switch	= powernv_fast_switch,
 	.get		= powernv_cpufreq_get,
 	.stop_cpu	= powernv_cpufreq_stop_cpu,
 	.attr		= powernv_cpu_freq_attr,
diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c
index 7fe442ca38f4..0835a37a5f3a 100644
--- a/drivers/cpuidle/cpuidle-powernv.c
+++ b/drivers/cpuidle/cpuidle-powernv.c
@@ -22,7 +22,7 @@
 
 #define POWERNV_THRESHOLD_LATENCY_NS 200000
 
-struct cpuidle_driver powernv_idle_driver = {
+static struct cpuidle_driver powernv_idle_driver = {
 	.name             = "powernv_idle",
 	.owner            = THIS_MODULE,
 };
diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c
index c73207abb5a4..62810ff3b00f 100644
--- a/drivers/cpuidle/cpuidle.c
+++ b/drivers/cpuidle/cpuidle.c
@@ -97,7 +97,23 @@ static int find_deepest_state(struct cpuidle_driver *drv,
 	return ret;
 }
 
-#ifdef CONFIG_SUSPEND
+/**
+ * cpuidle_use_deepest_state - Set/clear governor override flag.
+ * @enable: New value of the flag.
+ *
+ * Set/unset the current CPU to use the deepest idle state (override governors
+ * going forward if set).
+ */
+void cpuidle_use_deepest_state(bool enable)
+{
+	struct cpuidle_device *dev;
+
+	preempt_disable();
+	dev = cpuidle_get_device();
+	dev->use_deepest_state = enable;
+	preempt_enable();
+}
+
 /**
  * cpuidle_find_deepest_state - Find the deepest available idle state.
  * @drv: cpuidle driver for the given CPU.
@@ -109,6 +125,7 @@ int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
 	return find_deepest_state(drv, dev, UINT_MAX, 0, false);
 }
 
+#ifdef CONFIG_SUSPEND
 static void enter_freeze_proper(struct cpuidle_driver *drv,
 				struct cpuidle_device *dev, int index)
 {
diff --git a/drivers/cpuidle/dt_idle_states.c b/drivers/cpuidle/dt_idle_states.c
index a5c111b67f37..ffca4fc0061d 100644
--- a/drivers/cpuidle/dt_idle_states.c
+++ b/drivers/cpuidle/dt_idle_states.c
@@ -38,6 +38,12 @@ static int init_state_node(struct cpuidle_state *idle_state,
 	 * state enter function.
 	 */
 	idle_state->enter = match_id->data;
+	/*
+	 * Since this is not a "coupled" state, it's safe to assume interrupts
+	 * won't be enabled when it exits allowing the tick to be frozen
+	 * safely. So enter() can be also enter_freeze() callback.
+	 */
+	idle_state->enter_freeze = match_id->data;
 
 	err = of_property_read_u32(state_node, "wakeup-latency-us",
 				   &idle_state->exit_latency);
diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c
index fb9f511cca23..4e78263e34a4 100644
--- a/drivers/cpuidle/governor.c
+++ b/drivers/cpuidle/governor.c
@@ -9,7 +9,6 @@
  */
 
 #include <linux/mutex.h>
-#include <linux/module.h>
 #include <linux/cpuidle.h>
 
 #include "cpuidle.h"
@@ -53,14 +52,11 @@ int cpuidle_switch_governor(struct cpuidle_governor *gov)
 	if (cpuidle_curr_governor) {
 		list_for_each_entry(dev, &cpuidle_detected_devices, device_list)
 			cpuidle_disable_device(dev);
-		module_put(cpuidle_curr_governor->owner);
 	}
 
 	cpuidle_curr_governor = gov;
 
 	if (gov) {
-		if (!try_module_get(cpuidle_curr_governor->owner))
-			return -EINVAL;
 		list_for_each_entry(dev, &cpuidle_detected_devices, device_list)
 			cpuidle_enable_device(dev);
 		cpuidle_install_idle_handler();
diff --git a/drivers/cpuidle/governors/ladder.c b/drivers/cpuidle/governors/ladder.c
index 63bd5a403e22..fe8f08948fcb 100644
--- a/drivers/cpuidle/governors/ladder.c
+++ b/drivers/cpuidle/governors/ladder.c
@@ -15,7 +15,6 @@
 #include <linux/kernel.h>
 #include <linux/cpuidle.h>
 #include <linux/pm_qos.h>
-#include <linux/module.h>
 #include <linux/jiffies.h>
 #include <linux/tick.h>
 
@@ -177,7 +176,6 @@ static struct cpuidle_governor ladder_governor = {
 	.enable =	ladder_enable_device,
 	.select =	ladder_select_state,
 	.reflect =	ladder_reflect,
-	.owner =	THIS_MODULE,
 };
 
 /**
diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index 03d38c291de6..d9b5b9398a0f 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -19,7 +19,6 @@
 #include <linux/tick.h>
 #include <linux/sched.h>
 #include <linux/math64.h>
-#include <linux/module.h>
 
 /*
  * Please note when changing the tuning values:
@@ -484,7 +483,6 @@ static struct cpuidle_governor menu_governor = {
 	.enable =	menu_enable_device,
 	.select =	menu_select,
 	.reflect =	menu_reflect,
-	.owner =	THIS_MODULE,
 };
 
 /**
diff --git a/drivers/cpuidle/sysfs.c b/drivers/cpuidle/sysfs.c
index 832a2c3f01ff..c5adc8c9ac43 100644
--- a/drivers/cpuidle/sysfs.c
+++ b/drivers/cpuidle/sysfs.c
@@ -403,8 +403,10 @@ static int cpuidle_add_state_sysfs(struct cpuidle_device *device)
 	/* state statistics */
 	for (i = 0; i < drv->state_count; i++) {
 		kobj = kzalloc(sizeof(struct cpuidle_state_kobj), GFP_KERNEL);
-		if (!kobj)
+		if (!kobj) {
+			ret = -ENOMEM;
 			goto error_state;
+		}
 		kobj->state = &drv->states[i];
 		kobj->state_usage = &device->states_usage[i];
 		init_completion(&kobj->kobj_unregister);
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 4d2b81f2b223..79564785ae30 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -555,4 +555,6 @@ config CRYPTO_DEV_ROCKCHIP
 
 source "drivers/crypto/chelsio/Kconfig"
 
+source "drivers/crypto/virtio/Kconfig"
+
 endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index ad7250fa1348..bc53cb833a06 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -32,3 +32,4 @@ obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
 obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/
 obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/
 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/
+obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
index dae1e39139e9..d10b4ae5e0da 100644
--- a/drivers/crypto/amcc/crypto4xx_core.c
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -135,8 +135,7 @@ int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size)
 	ctx->sa_out = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4,
 					 &ctx->sa_out_dma_addr, GFP_ATOMIC);
 	if (ctx->sa_out == NULL) {
-		dma_free_coherent(ctx->dev->core_dev->device,
-				  ctx->sa_len * 4,
+		dma_free_coherent(ctx->dev->core_dev->device, size * 4,
 				  ctx->sa_in, ctx->sa_in_dma_addr);
 		return -ENOMEM;
 	}
diff --git a/drivers/crypto/atmel-aes-regs.h b/drivers/crypto/atmel-aes-regs.h
index 6c2951bb70b1..0ec04407b533 100644
--- a/drivers/crypto/atmel-aes-regs.h
+++ b/drivers/crypto/atmel-aes-regs.h
@@ -28,6 +28,7 @@
 #define AES_MR_OPMOD_CFB		(0x3 << 12)
 #define AES_MR_OPMOD_CTR		(0x4 << 12)
 #define AES_MR_OPMOD_GCM		(0x5 << 12)
+#define AES_MR_OPMOD_XTS		(0x6 << 12)
 #define AES_MR_LOD				(0x1 << 15)
 #define AES_MR_CFBS_MASK		(0x7 << 16)
 #define AES_MR_CFBS_128b		(0x0 << 16)
@@ -67,6 +68,9 @@
 #define AES_CTRR	0x98
 #define AES_GCMHR(x)	(0x9c + ((x) * 0x04))
 
+#define AES_TWR(x)	(0xc0 + ((x) * 0x04))
+#define AES_ALPHAR(x)	(0xd0 + ((x) * 0x04))
+
 #define AES_HW_VERSION	0xFC
 
 #endif /* __ATMEL_AES_REGS_H__ */
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index e3d40a8dfffb..0e3d0d655b96 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -36,6 +36,7 @@
 #include <crypto/scatterwalk.h>
 #include <crypto/algapi.h>
 #include <crypto/aes.h>
+#include <crypto/xts.h>
 #include <crypto/internal/aead.h>
 #include <linux/platform_data/crypto-atmel.h>
 #include <dt-bindings/dma/at91.h>
@@ -68,6 +69,7 @@
 #define AES_FLAGS_CFB8		(AES_MR_OPMOD_CFB | AES_MR_CFBS_8b)
 #define AES_FLAGS_CTR		AES_MR_OPMOD_CTR
 #define AES_FLAGS_GCM		AES_MR_OPMOD_GCM
+#define AES_FLAGS_XTS		AES_MR_OPMOD_XTS
 
 #define AES_FLAGS_MODE_MASK	(AES_FLAGS_OPMODE_MASK |	\
 				 AES_FLAGS_ENCRYPT |		\
@@ -89,6 +91,7 @@ struct atmel_aes_caps {
 	bool			has_cfb64;
 	bool			has_ctr32;
 	bool			has_gcm;
+	bool			has_xts;
 	u32			max_burst_size;
 };
 
@@ -135,6 +138,12 @@ struct atmel_aes_gcm_ctx {
 	atmel_aes_fn_t		ghash_resume;
 };
 
+struct atmel_aes_xts_ctx {
+	struct atmel_aes_base_ctx	base;
+
+	u32			key2[AES_KEYSIZE_256 / sizeof(u32)];
+};
+
 struct atmel_aes_reqctx {
 	unsigned long		mode;
 };
@@ -282,6 +291,20 @@ static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz)
 		snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2);
 		break;
 
+	case AES_TWR(0):
+	case AES_TWR(1):
+	case AES_TWR(2):
+	case AES_TWR(3):
+		snprintf(tmp, sz, "TWR[%u]", (offset - AES_TWR(0)) >> 2);
+		break;
+
+	case AES_ALPHAR(0):
+	case AES_ALPHAR(1):
+	case AES_ALPHAR(2):
+	case AES_ALPHAR(3):
+		snprintf(tmp, sz, "ALPHAR[%u]", (offset - AES_ALPHAR(0)) >> 2);
+		break;
+
 	default:
 		snprintf(tmp, sz, "0x%02x", offset);
 		break;
@@ -317,7 +340,7 @@ static inline void atmel_aes_write(struct atmel_aes_dev *dd,
 		char tmp[16];
 
 		dev_vdbg(dd->dev, "write 0x%08x into %s\n", value,
-			 atmel_aes_reg_name(offset, tmp));
+			 atmel_aes_reg_name(offset, tmp, sizeof(tmp)));
 	}
 #endif /* VERBOSE_DEBUG */
 
@@ -453,15 +476,15 @@ static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err)
 	return err;
 }
 
-static void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma,
-				 const u32 *iv)
+static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma,
+				     const u32 *iv, const u32 *key, int keylen)
 {
 	u32 valmr = 0;
 
 	/* MR register must be set before IV registers */
-	if (dd->ctx->keylen == AES_KEYSIZE_128)
+	if (keylen == AES_KEYSIZE_128)
 		valmr |= AES_MR_KEYSIZE_128;
-	else if (dd->ctx->keylen == AES_KEYSIZE_192)
+	else if (keylen == AES_KEYSIZE_192)
 		valmr |= AES_MR_KEYSIZE_192;
 	else
 		valmr |= AES_MR_KEYSIZE_256;
@@ -478,13 +501,19 @@ static void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma,
 
 	atmel_aes_write(dd, AES_MR, valmr);
 
-	atmel_aes_write_n(dd, AES_KEYWR(0), dd->ctx->key,
-			  SIZE_IN_WORDS(dd->ctx->keylen));
+	atmel_aes_write_n(dd, AES_KEYWR(0), key, SIZE_IN_WORDS(keylen));
 
 	if (iv && (valmr & AES_MR_OPMOD_MASK) != AES_MR_OPMOD_ECB)
 		atmel_aes_write_block(dd, AES_IVR(0), iv);
 }
 
+static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma,
+					const u32 *iv)
+
+{
+	atmel_aes_write_ctrl_key(dd, use_dma, iv,
+				 dd->ctx->key, dd->ctx->keylen);
+}
 
 /* CPU transfer */
 
@@ -1769,6 +1798,137 @@ static struct aead_alg aes_gcm_alg = {
 };
 
 
+/* xts functions */
+
+static inline struct atmel_aes_xts_ctx *
+atmel_aes_xts_ctx_cast(struct atmel_aes_base_ctx *ctx)
+{
+	return container_of(ctx, struct atmel_aes_xts_ctx, base);
+}
+
+static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd);
+
+static int atmel_aes_xts_start(struct atmel_aes_dev *dd)
+{
+	struct atmel_aes_xts_ctx *ctx = atmel_aes_xts_ctx_cast(dd->ctx);
+	struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq);
+	struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req);
+	unsigned long flags;
+	int err;
+
+	atmel_aes_set_mode(dd, rctx);
+
+	err = atmel_aes_hw_init(dd);
+	if (err)
+		return atmel_aes_complete(dd, err);
+
+	/* Compute the tweak value from req->info with ecb(aes). */
+	flags = dd->flags;
+	dd->flags &= ~AES_FLAGS_MODE_MASK;
+	dd->flags |= (AES_FLAGS_ECB | AES_FLAGS_ENCRYPT);
+	atmel_aes_write_ctrl_key(dd, false, NULL,
+				 ctx->key2, ctx->base.keylen);
+	dd->flags = flags;
+
+	atmel_aes_write_block(dd, AES_IDATAR(0), req->info);
+	return atmel_aes_wait_for_data_ready(dd, atmel_aes_xts_process_data);
+}
+
+static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd)
+{
+	struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq);
+	bool use_dma = (req->nbytes >= ATMEL_AES_DMA_THRESHOLD);
+	u32 tweak[AES_BLOCK_SIZE / sizeof(u32)];
+	static const u32 one[AES_BLOCK_SIZE / sizeof(u32)] = {cpu_to_le32(1), };
+	u8 *tweak_bytes = (u8 *)tweak;
+	int i;
+
+	/* Read the computed ciphered tweak value. */
+	atmel_aes_read_block(dd, AES_ODATAR(0), tweak);
+	/*
+	 * Hardware quirk:
+	 * the order of the ciphered tweak bytes need to be reversed before
+	 * writing them into the ODATARx registers.
+	 */
+	for (i = 0; i < AES_BLOCK_SIZE/2; ++i) {
+		u8 tmp = tweak_bytes[AES_BLOCK_SIZE - 1 - i];
+
+		tweak_bytes[AES_BLOCK_SIZE - 1 - i] = tweak_bytes[i];
+		tweak_bytes[i] = tmp;
+	}
+
+	/* Process the data. */
+	atmel_aes_write_ctrl(dd, use_dma, NULL);
+	atmel_aes_write_block(dd, AES_TWR(0), tweak);
+	atmel_aes_write_block(dd, AES_ALPHAR(0), one);
+	if (use_dma)
+		return atmel_aes_dma_start(dd, req->src, req->dst, req->nbytes,
+					   atmel_aes_transfer_complete);
+
+	return atmel_aes_cpu_start(dd, req->src, req->dst, req->nbytes,
+				   atmel_aes_transfer_complete);
+}
+
+static int atmel_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+				unsigned int keylen)
+{
+	struct atmel_aes_xts_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+	int err;
+
+	err = xts_check_key(crypto_ablkcipher_tfm(tfm), key, keylen);
+	if (err)
+		return err;
+
+	memcpy(ctx->base.key, key, keylen/2);
+	memcpy(ctx->key2, key + keylen/2, keylen/2);
+	ctx->base.keylen = keylen/2;
+
+	return 0;
+}
+
+static int atmel_aes_xts_encrypt(struct ablkcipher_request *req)
+{
+	return atmel_aes_crypt(req, AES_FLAGS_XTS | AES_FLAGS_ENCRYPT);
+}
+
+static int atmel_aes_xts_decrypt(struct ablkcipher_request *req)
+{
+	return atmel_aes_crypt(req, AES_FLAGS_XTS);
+}
+
+static int atmel_aes_xts_cra_init(struct crypto_tfm *tfm)
+{
+	struct atmel_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_aes_reqctx);
+	ctx->base.start = atmel_aes_xts_start;
+
+	return 0;
+}
+
+static struct crypto_alg aes_xts_alg = {
+	.cra_name		= "xts(aes)",
+	.cra_driver_name	= "atmel-xts-aes",
+	.cra_priority		= ATMEL_AES_PRIORITY,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= AES_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct atmel_aes_xts_ctx),
+	.cra_alignmask		= 0xf,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= atmel_aes_xts_cra_init,
+	.cra_exit		= atmel_aes_cra_exit,
+	.cra_u.ablkcipher = {
+		.min_keysize	= 2 * AES_MIN_KEY_SIZE,
+		.max_keysize	= 2 * AES_MAX_KEY_SIZE,
+		.ivsize		= AES_BLOCK_SIZE,
+		.setkey		= atmel_aes_xts_setkey,
+		.encrypt	= atmel_aes_xts_encrypt,
+		.decrypt	= atmel_aes_xts_decrypt,
+	}
+};
+
+
 /* Probe functions */
 
 static int atmel_aes_buff_init(struct atmel_aes_dev *dd)
@@ -1877,6 +2037,9 @@ static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd)
 {
 	int i;
 
+	if (dd->caps.has_xts)
+		crypto_unregister_alg(&aes_xts_alg);
+
 	if (dd->caps.has_gcm)
 		crypto_unregister_aead(&aes_gcm_alg);
 
@@ -1909,8 +2072,16 @@ static int atmel_aes_register_algs(struct atmel_aes_dev *dd)
 			goto err_aes_gcm_alg;
 	}
 
+	if (dd->caps.has_xts) {
+		err = crypto_register_alg(&aes_xts_alg);
+		if (err)
+			goto err_aes_xts_alg;
+	}
+
 	return 0;
 
+err_aes_xts_alg:
+	crypto_unregister_aead(&aes_gcm_alg);
 err_aes_gcm_alg:
 	crypto_unregister_alg(&aes_cfb64_alg);
 err_aes_cfb64_alg:
@@ -1928,6 +2099,7 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd)
 	dd->caps.has_cfb64 = 0;
 	dd->caps.has_ctr32 = 0;
 	dd->caps.has_gcm = 0;
+	dd->caps.has_xts = 0;
 	dd->caps.max_burst_size = 1;
 
 	/* keep only major version number */
@@ -1937,6 +2109,7 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd)
 		dd->caps.has_cfb64 = 1;
 		dd->caps.has_ctr32 = 1;
 		dd->caps.has_gcm = 1;
+		dd->caps.has_xts = 1;
 		dd->caps.max_burst_size = 4;
 		break;
 	case 0x200:
@@ -2138,7 +2311,7 @@ aes_dd_err:
 
 static int atmel_aes_remove(struct platform_device *pdev)
 {
-	static struct atmel_aes_dev *aes_dd;
+	struct atmel_aes_dev *aes_dd;
 
 	aes_dd = platform_get_drvdata(pdev);
 	if (!aes_dd)
diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig
index 64bf3024b680..bc0d3569f8d9 100644
--- a/drivers/crypto/caam/Kconfig
+++ b/drivers/crypto/caam/Kconfig
@@ -74,7 +74,7 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD
 
 config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
 	tristate "Register algorithm implementations with the Crypto API"
-	depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
+	depends on CRYPTO_DEV_FSL_CAAM_JR
 	default y
 	select CRYPTO_AEAD
 	select CRYPTO_AUTHENC
@@ -89,7 +89,7 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
 
 config CRYPTO_DEV_FSL_CAAM_AHASH_API
 	tristate "Register hash algorithm implementations with Crypto API"
-	depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
+	depends on CRYPTO_DEV_FSL_CAAM_JR
 	default y
 	select CRYPTO_HASH
 	help
@@ -101,7 +101,7 @@ config CRYPTO_DEV_FSL_CAAM_AHASH_API
 
 config CRYPTO_DEV_FSL_CAAM_PKC_API
         tristate "Register public key cryptography implementations with Crypto API"
-        depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
+        depends on CRYPTO_DEV_FSL_CAAM_JR
         default y
         select CRYPTO_RSA
         help
@@ -113,7 +113,7 @@ config CRYPTO_DEV_FSL_CAAM_PKC_API
 
 config CRYPTO_DEV_FSL_CAAM_RNG_API
 	tristate "Register caam device for hwrng API"
-	depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR
+	depends on CRYPTO_DEV_FSL_CAAM_JR
 	default y
 	select CRYPTO_RNG
 	select HW_RANDOM
@@ -134,3 +134,6 @@ config CRYPTO_DEV_FSL_CAAM_DEBUG
 	help
 	  Selecting this will enable printing of various debug
 	  information in the CAAM driver.
+
+config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
+	def_tristate CRYPTO_DEV_FSL_CAAM_CRYPTO_API
diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile
index 08bf5515ae8a..6554742f357e 100644
--- a/drivers/crypto/caam/Makefile
+++ b/drivers/crypto/caam/Makefile
@@ -8,6 +8,7 @@ endif
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 954a64c7757b..662fe94cb2f8 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -2,6 +2,7 @@
  * caam - Freescale FSL CAAM support for crypto API
  *
  * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ * Copyright 2016 NXP
  *
  * Based on talitos crypto API driver.
  *
@@ -53,6 +54,7 @@
 #include "error.h"
 #include "sg_sw_sec4.h"
 #include "key_gen.h"
+#include "caamalg_desc.h"
 
 /*
  * crypto alg
@@ -62,8 +64,6 @@
 #define CAAM_MAX_KEY_SIZE		(AES_MAX_KEY_SIZE + \
 					 CTR_RFC3686_NONCE_SIZE + \
 					 SHA512_DIGEST_SIZE * 2)
-/* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
-#define CAAM_MAX_IV_LENGTH		16
 
 #define AEAD_DESC_JOB_IO_LEN		(DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
 #define GCM_DESC_JOB_IO_LEN		(AEAD_DESC_JOB_IO_LEN + \
@@ -71,37 +71,6 @@
 #define AUTHENC_DESC_JOB_IO_LEN		(AEAD_DESC_JOB_IO_LEN + \
 					 CAAM_CMD_SZ * 5)
 
-/* length of descriptors text */
-#define DESC_AEAD_BASE			(4 * CAAM_CMD_SZ)
-#define DESC_AEAD_ENC_LEN		(DESC_AEAD_BASE + 11 * CAAM_CMD_SZ)
-#define DESC_AEAD_DEC_LEN		(DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
-#define DESC_AEAD_GIVENC_LEN		(DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ)
-
-/* Note: Nonce is counted in enckeylen */
-#define DESC_AEAD_CTR_RFC3686_LEN	(4 * CAAM_CMD_SZ)
-
-#define DESC_AEAD_NULL_BASE		(3 * CAAM_CMD_SZ)
-#define DESC_AEAD_NULL_ENC_LEN		(DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ)
-#define DESC_AEAD_NULL_DEC_LEN		(DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ)
-
-#define DESC_GCM_BASE			(3 * CAAM_CMD_SZ)
-#define DESC_GCM_ENC_LEN		(DESC_GCM_BASE + 16 * CAAM_CMD_SZ)
-#define DESC_GCM_DEC_LEN		(DESC_GCM_BASE + 12 * CAAM_CMD_SZ)
-
-#define DESC_RFC4106_BASE		(3 * CAAM_CMD_SZ)
-#define DESC_RFC4106_ENC_LEN		(DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
-#define DESC_RFC4106_DEC_LEN		(DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
-
-#define DESC_RFC4543_BASE		(3 * CAAM_CMD_SZ)
-#define DESC_RFC4543_ENC_LEN		(DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ)
-#define DESC_RFC4543_DEC_LEN		(DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ)
-
-#define DESC_ABLKCIPHER_BASE		(3 * CAAM_CMD_SZ)
-#define DESC_ABLKCIPHER_ENC_LEN		(DESC_ABLKCIPHER_BASE + \
-					 20 * CAAM_CMD_SZ)
-#define DESC_ABLKCIPHER_DEC_LEN		(DESC_ABLKCIPHER_BASE + \
-					 15 * CAAM_CMD_SZ)
-
 #define DESC_MAX_USED_BYTES		(CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
 #define DESC_MAX_USED_LEN		(DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
 
@@ -117,8 +86,7 @@
 
 static void dbg_dump_sg(const char *level, const char *prefix_str,
 			int prefix_type, int rowsize, int groupsize,
-			struct scatterlist *sg, size_t tlen, bool ascii,
-			bool may_sleep)
+			struct scatterlist *sg, size_t tlen, bool ascii)
 {
 	struct scatterlist *it;
 	void *it_page;
@@ -152,7 +120,6 @@ static struct list_head alg_list;
 struct caam_alg_entry {
 	int class1_alg_type;
 	int class2_alg_type;
-	int alg_op;
 	bool rfc3686;
 	bool geniv;
 };
@@ -163,52 +130,6 @@ struct caam_aead_alg {
 	bool registered;
 };
 
-/* Set DK bit in class 1 operation if shared */
-static inline void append_dec_op1(u32 *desc, u32 type)
-{
-	u32 *jump_cmd, *uncond_jump_cmd;
-
-	/* DK bit is valid only for AES */
-	if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
-		append_operation(desc, type | OP_ALG_AS_INITFINAL |
-				 OP_ALG_DECRYPT);
-		return;
-	}
-
-	jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
-	append_operation(desc, type | OP_ALG_AS_INITFINAL |
-			 OP_ALG_DECRYPT);
-	uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
-	set_jump_tgt_here(desc, jump_cmd);
-	append_operation(desc, type | OP_ALG_AS_INITFINAL |
-			 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
-	set_jump_tgt_here(desc, uncond_jump_cmd);
-}
-
-/*
- * For aead functions, read payload and write payload,
- * both of which are specified in req->src and req->dst
- */
-static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
-{
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
-			     KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
-}
-
-/*
- * For ablkcipher encrypt and decrypt, read from req->src and
- * write to req->dst
- */
-static inline void ablkcipher_append_src_dst(u32 *desc)
-{
-	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
-	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
-			     KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
-}
-
 /*
  * per-session context
  */
@@ -220,147 +141,36 @@ struct caam_ctx {
 	dma_addr_t sh_desc_enc_dma;
 	dma_addr_t sh_desc_dec_dma;
 	dma_addr_t sh_desc_givenc_dma;
-	u32 class1_alg_type;
-	u32 class2_alg_type;
-	u32 alg_op;
 	u8 key[CAAM_MAX_KEY_SIZE];
 	dma_addr_t key_dma;
-	unsigned int enckeylen;
-	unsigned int split_key_len;
-	unsigned int split_key_pad_len;
+	struct alginfo adata;
+	struct alginfo cdata;
 	unsigned int authsize;
 };
 
-static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
-			    int keys_fit_inline, bool is_rfc3686)
-{
-	u32 *nonce;
-	unsigned int enckeylen = ctx->enckeylen;
-
-	/*
-	 * RFC3686 specific:
-	 *	| ctx->key = {AUTH_KEY, ENC_KEY, NONCE}
-	 *	| enckeylen = encryption key size + nonce size
-	 */
-	if (is_rfc3686)
-		enckeylen -= CTR_RFC3686_NONCE_SIZE;
-
-	if (keys_fit_inline) {
-		append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
-				  ctx->split_key_len, CLASS_2 |
-				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
-		append_key_as_imm(desc, (void *)ctx->key +
-				  ctx->split_key_pad_len, enckeylen,
-				  enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
-	} else {
-		append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
-			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
-		append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
-			   enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
-	}
-
-	/* Load Counter into CONTEXT1 reg */
-	if (is_rfc3686) {
-		nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len +
-			       enckeylen);
-		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
-				   LDST_CLASS_IND_CCB |
-				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
-		append_move(desc,
-			    MOVE_SRC_OUTFIFO |
-			    MOVE_DEST_CLASS1CTX |
-			    (16 << MOVE_OFFSET_SHIFT) |
-			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
-	}
-}
-
-static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
-				  int keys_fit_inline, bool is_rfc3686)
-{
-	u32 *key_jump_cmd;
-
-	/* Note: Context registers are saved. */
-	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
-
-	/* Skip if already shared */
-	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-				   JUMP_COND_SHRD);
-
-	append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
-
-	set_jump_tgt_here(desc, key_jump_cmd);
-}
-
 static int aead_null_set_sh_desc(struct crypto_aead *aead)
 {
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
 	struct device *jrdev = ctx->jrdev;
-	bool keys_fit_inline = false;
-	u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd;
 	u32 *desc;
+	int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN -
+			ctx->adata.keylen_pad;
 
 	/*
 	 * Job Descriptor and Shared Descriptors
 	 * must all fit into the 64-word Descriptor h/w Buffer
 	 */
-	if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN +
-	    ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = true;
+	if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) {
+		ctx->adata.key_inline = true;
+		ctx->adata.key_virt = ctx->key;
+	} else {
+		ctx->adata.key_inline = false;
+		ctx->adata.key_dma = ctx->key_dma;
+	}
 
 	/* aead_encrypt shared descriptor */
 	desc = ctx->sh_desc_enc;
-
-	init_sh_desc(desc, HDR_SHARE_SERIAL);
-
-	/* Skip if already shared */
-	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-				   JUMP_COND_SHRD);
-	if (keys_fit_inline)
-		append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
-				  ctx->split_key_len, CLASS_2 |
-				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
-	else
-		append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
-			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/* assoclen + cryptlen = seqinlen */
-	append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
-
-	/* Prepare to read and write cryptlen + assoclen bytes */
-	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
-	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
-
-	/*
-	 * MOVE_LEN opcode is not available in all SEC HW revisions,
-	 * thus need to do some magic, i.e. self-patch the descriptor
-	 * buffer.
-	 */
-	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
-				    MOVE_DEST_MATH3 |
-				    (0x6 << MOVE_LEN_SHIFT));
-	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
-				     MOVE_DEST_DESCBUF |
-				     MOVE_WAITCOMP |
-				     (0x8 << MOVE_LEN_SHIFT));
-
-	/* Class 2 operation */
-	append_operation(desc, ctx->class2_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
-
-	/* Read and write cryptlen bytes */
-	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
-
-	set_move_tgt_here(desc, read_move_cmd);
-	set_move_tgt_here(desc, write_move_cmd);
-	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
-	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
-		    MOVE_AUX_LS);
-
-	/* Write ICV */
-	append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
-			 LDST_SRCDST_BYTE_CONTEXT);
-
+	cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize);
 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -368,84 +178,22 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead)
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR,
-		       "aead null enc shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 
 	/*
 	 * Job Descriptor and Shared Descriptors
 	 * must all fit into the 64-word Descriptor h/w Buffer
 	 */
-	keys_fit_inline = false;
-	if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN +
-	    ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = true;
-
-	desc = ctx->sh_desc_dec;
+	if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) {
+		ctx->adata.key_inline = true;
+		ctx->adata.key_virt = ctx->key;
+	} else {
+		ctx->adata.key_inline = false;
+		ctx->adata.key_dma = ctx->key_dma;
+	}
 
 	/* aead_decrypt shared descriptor */
-	init_sh_desc(desc, HDR_SHARE_SERIAL);
-
-	/* Skip if already shared */
-	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-				   JUMP_COND_SHRD);
-	if (keys_fit_inline)
-		append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
-				  ctx->split_key_len, CLASS_2 |
-				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
-	else
-		append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
-			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/* Class 2 operation */
-	append_operation(desc, ctx->class2_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
-
-	/* assoclen + cryptlen = seqoutlen */
-	append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
-
-	/* Prepare to read and write cryptlen + assoclen bytes */
-	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
-	append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
-
-	/*
-	 * MOVE_LEN opcode is not available in all SEC HW revisions,
-	 * thus need to do some magic, i.e. self-patch the descriptor
-	 * buffer.
-	 */
-	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
-				    MOVE_DEST_MATH2 |
-				    (0x6 << MOVE_LEN_SHIFT));
-	write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
-				     MOVE_DEST_DESCBUF |
-				     MOVE_WAITCOMP |
-				     (0x8 << MOVE_LEN_SHIFT));
-
-	/* Read and write cryptlen bytes */
-	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
-
-	/*
-	 * Insert a NOP here, since we need at least 4 instructions between
-	 * code patching the descriptor buffer and the location being patched.
-	 */
-	jump_cmd = append_jump(desc, JUMP_TEST_ALL);
-	set_jump_tgt_here(desc, jump_cmd);
-
-	set_move_tgt_here(desc, read_move_cmd);
-	set_move_tgt_here(desc, write_move_cmd);
-	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
-	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
-		    MOVE_AUX_LS);
-	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
-
-	/* Load ICV */
-	append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
-			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
-
+	desc = ctx->sh_desc_dec;
+	cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize);
 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -453,12 +201,6 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead)
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR,
-		       "aead null dec shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 
 	return 0;
 }
@@ -470,11 +212,11 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 	unsigned int ivsize = crypto_aead_ivsize(aead);
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
 	struct device *jrdev = ctx->jrdev;
-	bool keys_fit_inline;
-	u32 geniv, moveiv;
 	u32 ctx1_iv_off = 0;
-	u32 *desc;
-	const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
+	u32 *desc, *nonce = NULL;
+	u32 inl_mask;
+	unsigned int data_len[2];
+	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 			       OP_ALG_AAI_CTR_MOD128);
 	const bool is_rfc3686 = alg->caam.rfc3686;
 
@@ -482,7 +224,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 		return 0;
 
 	/* NULL encryption / decryption */
-	if (!ctx->enckeylen)
+	if (!ctx->cdata.keylen)
 		return aead_null_set_sh_desc(aead);
 
 	/*
@@ -497,8 +239,14 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 	 * RFC3686 specific:
 	 *	CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 	 */
-	if (is_rfc3686)
+	if (is_rfc3686) {
 		ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
+		nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
+				ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
+	}
+
+	data_len[0] = ctx->adata.keylen_pad;
+	data_len[1] = ctx->cdata.keylen;
 
 	if (alg->caam.geniv)
 		goto skip_enc;
@@ -507,54 +255,29 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 	 * Job Descriptor and Shared Descriptors
 	 * must all fit into the 64-word Descriptor h/w Buffer
 	 */
-	keys_fit_inline = false;
-	if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN +
-	    ctx->split_key_pad_len + ctx->enckeylen +
-	    (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
-	    CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = true;
-
-	/* aead_encrypt shared descriptor */
-	desc = ctx->sh_desc_enc;
-
-	/* Note: Context registers are saved. */
-	init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
-
-	/* Class 2 operation */
-	append_operation(desc, ctx->class2_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
-
-	/* Read and write assoclen bytes */
-	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
-	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
-
-	/* Skip assoc data */
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
-
-	/* read assoc before reading payload */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
-				      FIFOLDST_VLF);
-
-	/* Load Counter into CONTEXT1 reg */
-	if (is_rfc3686)
-		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
-				     LDST_SRCDST_BYTE_CONTEXT |
-				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-				      LDST_OFFSET_SHIFT));
+	if (desc_inline_query(DESC_AEAD_ENC_LEN +
+			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
+			      AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
+			      ARRAY_SIZE(data_len)) < 0)
+		return -EINVAL;
 
-	/* Class 1 operation */
-	append_operation(desc, ctx->class1_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
+	if (inl_mask & 1)
+		ctx->adata.key_virt = ctx->key;
+	else
+		ctx->adata.key_dma = ctx->key_dma;
 
-	/* Read and write cryptlen bytes */
-	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
-	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
-	aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
+	if (inl_mask & 2)
+		ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
+	else
+		ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 
-	/* Write ICV */
-	append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
-			 LDST_SRCDST_BYTE_CONTEXT);
+	ctx->adata.key_inline = !!(inl_mask & 1);
+	ctx->cdata.key_inline = !!(inl_mask & 2);
 
+	/* aead_encrypt shared descriptor */
+	desc = ctx->sh_desc_enc;
+	cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ctx->authsize,
+			       is_rfc3686, nonce, ctx1_iv_off);
 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -562,79 +285,36 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 
 skip_enc:
 	/*
 	 * Job Descriptor and Shared Descriptors
 	 * must all fit into the 64-word Descriptor h/w Buffer
 	 */
-	keys_fit_inline = false;
-	if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN +
-	    ctx->split_key_pad_len + ctx->enckeylen +
-	    (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
-	    CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = true;
-
-	/* aead_decrypt shared descriptor */
-	desc = ctx->sh_desc_dec;
-
-	/* Note: Context registers are saved. */
-	init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
-
-	/* Class 2 operation */
-	append_operation(desc, ctx->class2_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
+	if (desc_inline_query(DESC_AEAD_DEC_LEN +
+			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
+			      AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
+			      ARRAY_SIZE(data_len)) < 0)
+		return -EINVAL;
 
-	/* Read and write assoclen bytes */
-	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
-	if (alg->caam.geniv)
-		append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize);
+	if (inl_mask & 1)
+		ctx->adata.key_virt = ctx->key;
 	else
-		append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
-
-	/* Skip assoc data */
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
-
-	/* read assoc before reading payload */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
-			     KEY_VLF);
+		ctx->adata.key_dma = ctx->key_dma;
 
-	if (alg->caam.geniv) {
-		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
-				LDST_SRCDST_BYTE_CONTEXT |
-				(ctx1_iv_off << LDST_OFFSET_SHIFT));
-		append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
-			    (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
-	}
-
-	/* Load Counter into CONTEXT1 reg */
-	if (is_rfc3686)
-		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
-				     LDST_SRCDST_BYTE_CONTEXT |
-				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-				      LDST_OFFSET_SHIFT));
-
-	/* Choose operation */
-	if (ctr_mode)
-		append_operation(desc, ctx->class1_alg_type |
-				 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
+	if (inl_mask & 2)
+		ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 	else
-		append_dec_op1(desc, ctx->class1_alg_type);
-
-	/* Read and write cryptlen bytes */
-	append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
-	append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
-	aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
+		ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 
-	/* Load ICV */
-	append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
-			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
+	ctx->adata.key_inline = !!(inl_mask & 1);
+	ctx->cdata.key_inline = !!(inl_mask & 2);
 
+	/* aead_decrypt shared descriptor */
+	desc = ctx->sh_desc_dec;
+	cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize,
+			       ctx->authsize, alg->caam.geniv, is_rfc3686,
+			       nonce, ctx1_iv_off);
 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -642,11 +322,6 @@ skip_enc:
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 
 	if (!alg->caam.geniv)
 		goto skip_givenc;
@@ -655,93 +330,30 @@ skip_enc:
 	 * Job Descriptor and Shared Descriptors
 	 * must all fit into the 64-word Descriptor h/w Buffer
 	 */
-	keys_fit_inline = false;
-	if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN +
-	    ctx->split_key_pad_len + ctx->enckeylen +
-	    (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
-	    CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = true;
-
-	/* aead_givencrypt shared descriptor */
-	desc = ctx->sh_desc_enc;
-
-	/* Note: Context registers are saved. */
-	init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
-
-	if (is_rfc3686)
-		goto copy_iv;
-
-	/* Generate IV */
-	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
-		NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
-		NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
-	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
-			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
-	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
-	append_move(desc, MOVE_WAITCOMP |
-		    MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
-		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
-		    (ivsize << MOVE_LEN_SHIFT));
-	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
-
-copy_iv:
-	/* Copy IV to class 1 context */
-	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
-		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
-		    (ivsize << MOVE_LEN_SHIFT));
-
-	/* Return to encryption */
-	append_operation(desc, ctx->class2_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
-
-	/* Read and write assoclen bytes */
-	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
-	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
-
-	/* ivsize + cryptlen = seqoutlen - authsize */
-	append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
-
-	/* Skip assoc data */
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
-
-	/* read assoc before reading payload */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
-			     KEY_VLF);
-
-	/* Copy iv from outfifo to class 2 fifo */
-	moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
-		 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
-	append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
-			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
-	append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
-			    LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
-
-	/* Load Counter into CONTEXT1 reg */
-	if (is_rfc3686)
-		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
-				     LDST_SRCDST_BYTE_CONTEXT |
-				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-				      LDST_OFFSET_SHIFT));
-
-	/* Class 1 operation */
-	append_operation(desc, ctx->class1_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
-
-	/* Will write ivsize + cryptlen */
-	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+	if (desc_inline_query(DESC_AEAD_GIVENC_LEN +
+			      (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
+			      AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
+			      ARRAY_SIZE(data_len)) < 0)
+		return -EINVAL;
 
-	/* Not need to reload iv */
-	append_seq_fifo_load(desc, ivsize,
-			     FIFOLD_CLASS_SKIP);
+	if (inl_mask & 1)
+		ctx->adata.key_virt = ctx->key;
+	else
+		ctx->adata.key_dma = ctx->key_dma;
 
-	/* Will read cryptlen */
-	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
-	aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
+	if (inl_mask & 2)
+		ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
+	else
+		ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 
-	/* Write ICV */
-	append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
-			 LDST_SRCDST_BYTE_CONTEXT);
+	ctx->adata.key_inline = !!(inl_mask & 1);
+	ctx->cdata.key_inline = !!(inl_mask & 2);
 
+	/* aead_givencrypt shared descriptor */
+	desc = ctx->sh_desc_enc;
+	cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize,
+				  ctx->authsize, is_rfc3686, nonce,
+				  ctx1_iv_off);
 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -749,11 +361,6 @@ copy_iv:
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 
 skip_givenc:
 	return 0;
@@ -774,12 +381,11 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
 {
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
 	struct device *jrdev = ctx->jrdev;
-	bool keys_fit_inline = false;
-	u32 *key_jump_cmd, *zero_payload_jump_cmd,
-	    *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2;
 	u32 *desc;
+	int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
+			ctx->cdata.keylen;
 
-	if (!ctx->enckeylen || !ctx->authsize)
+	if (!ctx->cdata.keylen || !ctx->authsize)
 		return 0;
 
 	/*
@@ -787,82 +393,16 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
 	 * Job Descriptor and Shared Descriptor
 	 * must fit into the 64-word Descriptor h/w Buffer
 	 */
-	if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN +
-	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = true;
+	if (rem_bytes >= DESC_GCM_ENC_LEN) {
+		ctx->cdata.key_inline = true;
+		ctx->cdata.key_virt = ctx->key;
+	} else {
+		ctx->cdata.key_inline = false;
+		ctx->cdata.key_dma = ctx->key_dma;
+	}
 
 	desc = ctx->sh_desc_enc;
-
-	init_sh_desc(desc, HDR_SHARE_SERIAL);
-
-	/* skip key loading if they are loaded due to sharing */
-	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-				   JUMP_COND_SHRD | JUMP_COND_SELF);
-	if (keys_fit_inline)
-		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
-				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
-	else
-		append_key(desc, ctx->key_dma, ctx->enckeylen,
-			   CLASS_1 | KEY_DEST_CLASS_REG);
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/* class 1 operation */
-	append_operation(desc, ctx->class1_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
-
-	/* if assoclen + cryptlen is ZERO, skip to ICV write */
-	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
-	zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
-						 JUMP_COND_MATH_Z);
-
-	/* if assoclen is ZERO, skip reading the assoc data */
-	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
-	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
-						 JUMP_COND_MATH_Z);
-
-	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
-
-	/* skip assoc data */
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
-
-	/* cryptlen = seqinlen - assoclen */
-	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
-
-	/* if cryptlen is ZERO jump to zero-payload commands */
-	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
-					    JUMP_COND_MATH_Z);
-
-	/* read assoc data */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
-			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
-	set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
-
-	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
-
-	/* write encrypted data */
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
-
-	/* read payload data */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
-			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
-
-	/* jump the zero-payload commands */
-	append_jump(desc, JUMP_TEST_ALL | 2);
-
-	/* zero-payload commands */
-	set_jump_tgt_here(desc, zero_payload_jump_cmd);
-
-	/* read assoc data */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
-			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
-
-	/* There is no input data */
-	set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
-
-	/* write ICV */
-	append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
-			 LDST_SRCDST_BYTE_CONTEXT);
-
+	cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ctx->authsize);
 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -870,80 +410,21 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 
 	/*
 	 * Job Descriptor and Shared Descriptors
 	 * must all fit into the 64-word Descriptor h/w Buffer
 	 */
-	keys_fit_inline = false;
-	if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN +
-	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = true;
+	if (rem_bytes >= DESC_GCM_DEC_LEN) {
+		ctx->cdata.key_inline = true;
+		ctx->cdata.key_virt = ctx->key;
+	} else {
+		ctx->cdata.key_inline = false;
+		ctx->cdata.key_dma = ctx->key_dma;
+	}
 
 	desc = ctx->sh_desc_dec;
-
-	init_sh_desc(desc, HDR_SHARE_SERIAL);
-
-	/* skip key loading if they are loaded due to sharing */
-	key_jump_cmd = append_jump(desc, JUMP_JSL |
-				   JUMP_TEST_ALL | JUMP_COND_SHRD |
-				   JUMP_COND_SELF);
-	if (keys_fit_inline)
-		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
-				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
-	else
-		append_key(desc, ctx->key_dma, ctx->enckeylen,
-			   CLASS_1 | KEY_DEST_CLASS_REG);
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/* class 1 operation */
-	append_operation(desc, ctx->class1_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
-
-	/* if assoclen is ZERO, skip reading the assoc data */
-	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
-	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
-						 JUMP_COND_MATH_Z);
-
-	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
-
-	/* skip assoc data */
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
-
-	/* read assoc data */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
-			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
-
-	set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
-
-	/* cryptlen = seqoutlen - assoclen */
-	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
-
-	/* jump to zero-payload command if cryptlen is zero */
-	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
-					    JUMP_COND_MATH_Z);
-
-	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
-
-	/* store encrypted data */
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
-
-	/* read payload data */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
-			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
-
-	/* zero-payload command */
-	set_jump_tgt_here(desc, zero_payload_jump_cmd);
-
-	/* read ICV */
-	append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
-			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
-
+	cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ctx->authsize);
 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -951,11 +432,6 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 
 	return 0;
 }
@@ -974,11 +450,11 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 {
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
 	struct device *jrdev = ctx->jrdev;
-	bool keys_fit_inline = false;
-	u32 *key_jump_cmd;
 	u32 *desc;
+	int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
+			ctx->cdata.keylen;
 
-	if (!ctx->enckeylen || !ctx->authsize)
+	if (!ctx->cdata.keylen || !ctx->authsize)
 		return 0;
 
 	/*
@@ -986,62 +462,16 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 	 * Job Descriptor and Shared Descriptor
 	 * must fit into the 64-word Descriptor h/w Buffer
 	 */
-	if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN +
-	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = true;
+	if (rem_bytes >= DESC_RFC4106_ENC_LEN) {
+		ctx->cdata.key_inline = true;
+		ctx->cdata.key_virt = ctx->key;
+	} else {
+		ctx->cdata.key_inline = false;
+		ctx->cdata.key_dma = ctx->key_dma;
+	}
 
 	desc = ctx->sh_desc_enc;
-
-	init_sh_desc(desc, HDR_SHARE_SERIAL);
-
-	/* Skip key loading if it is loaded due to sharing */
-	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-				   JUMP_COND_SHRD);
-	if (keys_fit_inline)
-		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
-				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
-	else
-		append_key(desc, ctx->key_dma, ctx->enckeylen,
-			   CLASS_1 | KEY_DEST_CLASS_REG);
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/* Class 1 operation */
-	append_operation(desc, ctx->class1_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
-
-	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
-	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
-
-	/* Read assoc data */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
-			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
-
-	/* Skip IV */
-	append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
-
-	/* Will read cryptlen bytes */
-	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
-
-	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
-
-	/* Skip assoc data */
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
-
-	/* cryptlen = seqoutlen - assoclen */
-	append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
-
-	/* Write encrypted data */
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
-
-	/* Read payload data */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
-			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
-
-	/* Write ICV */
-	append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
-			 LDST_SRCDST_BYTE_CONTEXT);
-
+	cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ctx->authsize);
 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -1049,73 +479,21 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 
 	/*
 	 * Job Descriptor and Shared Descriptors
 	 * must all fit into the 64-word Descriptor h/w Buffer
 	 */
-	keys_fit_inline = false;
-	if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN +
-	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = true;
+	if (rem_bytes >= DESC_RFC4106_DEC_LEN) {
+		ctx->cdata.key_inline = true;
+		ctx->cdata.key_virt = ctx->key;
+	} else {
+		ctx->cdata.key_inline = false;
+		ctx->cdata.key_dma = ctx->key_dma;
+	}
 
 	desc = ctx->sh_desc_dec;
-
-	init_sh_desc(desc, HDR_SHARE_SERIAL);
-
-	/* Skip key loading if it is loaded due to sharing */
-	key_jump_cmd = append_jump(desc, JUMP_JSL |
-				   JUMP_TEST_ALL | JUMP_COND_SHRD);
-	if (keys_fit_inline)
-		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
-				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
-	else
-		append_key(desc, ctx->key_dma, ctx->enckeylen,
-			   CLASS_1 | KEY_DEST_CLASS_REG);
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/* Class 1 operation */
-	append_operation(desc, ctx->class1_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
-
-	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
-	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
-
-	/* Read assoc data */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
-			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
-
-	/* Skip IV */
-	append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
-
-	/* Will read cryptlen bytes */
-	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
-
-	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
-
-	/* Skip assoc data */
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
-
-	/* Will write cryptlen bytes */
-	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
-
-	/* Store payload data */
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
-
-	/* Read encrypted data */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
-			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
-
-	/* Read ICV */
-	append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
-			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
-
+	cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ctx->authsize);
 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -1123,11 +501,6 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 
 	return 0;
 }
@@ -1147,12 +520,11 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 {
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
 	struct device *jrdev = ctx->jrdev;
-	bool keys_fit_inline = false;
-	u32 *key_jump_cmd;
-	u32 *read_move_cmd, *write_move_cmd;
 	u32 *desc;
+	int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
+			ctx->cdata.keylen;
 
-	if (!ctx->enckeylen || !ctx->authsize)
+	if (!ctx->cdata.keylen || !ctx->authsize)
 		return 0;
 
 	/*
@@ -1160,61 +532,16 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 	 * Job Descriptor and Shared Descriptor
 	 * must fit into the 64-word Descriptor h/w Buffer
 	 */
-	if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN +
-	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = true;
+	if (rem_bytes >= DESC_RFC4543_ENC_LEN) {
+		ctx->cdata.key_inline = true;
+		ctx->cdata.key_virt = ctx->key;
+	} else {
+		ctx->cdata.key_inline = false;
+		ctx->cdata.key_dma = ctx->key_dma;
+	}
 
 	desc = ctx->sh_desc_enc;
-
-	init_sh_desc(desc, HDR_SHARE_SERIAL);
-
-	/* Skip key loading if it is loaded due to sharing */
-	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-				   JUMP_COND_SHRD);
-	if (keys_fit_inline)
-		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
-				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
-	else
-		append_key(desc, ctx->key_dma, ctx->enckeylen,
-			   CLASS_1 | KEY_DEST_CLASS_REG);
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/* Class 1 operation */
-	append_operation(desc, ctx->class1_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
-
-	/* assoclen + cryptlen = seqinlen */
-	append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
-
-	/*
-	 * MOVE_LEN opcode is not available in all SEC HW revisions,
-	 * thus need to do some magic, i.e. self-patch the descriptor
-	 * buffer.
-	 */
-	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
-				    (0x6 << MOVE_LEN_SHIFT));
-	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
-				     (0x8 << MOVE_LEN_SHIFT));
-
-	/* Will read assoclen + cryptlen bytes */
-	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
-
-	/* Will write assoclen + cryptlen bytes */
-	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
-
-	/* Read and write assoclen + cryptlen bytes */
-	aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
-
-	set_move_tgt_here(desc, read_move_cmd);
-	set_move_tgt_here(desc, write_move_cmd);
-	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
-	/* Move payload data to OFIFO */
-	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
-
-	/* Write ICV */
-	append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
-			 LDST_SRCDST_BYTE_CONTEXT);
-
+	cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ctx->authsize);
 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -1222,77 +549,21 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 
 	/*
 	 * Job Descriptor and Shared Descriptors
 	 * must all fit into the 64-word Descriptor h/w Buffer
 	 */
-	keys_fit_inline = false;
-	if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN +
-	    ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = true;
+	if (rem_bytes >= DESC_RFC4543_DEC_LEN) {
+		ctx->cdata.key_inline = true;
+		ctx->cdata.key_virt = ctx->key;
+	} else {
+		ctx->cdata.key_inline = false;
+		ctx->cdata.key_dma = ctx->key_dma;
+	}
 
 	desc = ctx->sh_desc_dec;
-
-	init_sh_desc(desc, HDR_SHARE_SERIAL);
-
-	/* Skip key loading if it is loaded due to sharing */
-	key_jump_cmd = append_jump(desc, JUMP_JSL |
-				   JUMP_TEST_ALL | JUMP_COND_SHRD);
-	if (keys_fit_inline)
-		append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
-				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
-	else
-		append_key(desc, ctx->key_dma, ctx->enckeylen,
-			   CLASS_1 | KEY_DEST_CLASS_REG);
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/* Class 1 operation */
-	append_operation(desc, ctx->class1_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
-
-	/* assoclen + cryptlen = seqoutlen */
-	append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
-
-	/*
-	 * MOVE_LEN opcode is not available in all SEC HW revisions,
-	 * thus need to do some magic, i.e. self-patch the descriptor
-	 * buffer.
-	 */
-	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
-				    (0x6 << MOVE_LEN_SHIFT));
-	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
-				     (0x8 << MOVE_LEN_SHIFT));
-
-	/* Will read assoclen + cryptlen bytes */
-	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
-
-	/* Will write assoclen + cryptlen bytes */
-	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
-
-	/* Store payload data */
-	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
-
-	/* In-snoop assoclen + cryptlen data */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
-			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
-
-	set_move_tgt_here(desc, read_move_cmd);
-	set_move_tgt_here(desc, write_move_cmd);
-	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
-	/* Move payload data to OFIFO */
-	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
-	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
-
-	/* Read ICV */
-	append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
-			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
-
+	cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ctx->authsize);
 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -1300,11 +571,6 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 
 	return 0;
 }
@@ -1320,19 +586,9 @@ static int rfc4543_setauthsize(struct crypto_aead *authenc,
 	return 0;
 }
 
-static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in,
-			      u32 authkeylen)
-{
-	return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
-			       ctx->split_key_pad_len, key_in, authkeylen,
-			       ctx->alg_op);
-}
-
 static int aead_setkey(struct crypto_aead *aead,
 			       const u8 *key, unsigned int keylen)
 {
-	/* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
-	static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
 	struct device *jrdev = ctx->jrdev;
 	struct crypto_authenc_keys keys;
@@ -1341,33 +597,25 @@ static int aead_setkey(struct crypto_aead *aead,
 	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 		goto badkey;
 
-	/* Pick class 2 key length from algorithm submask */
-	ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
-				      OP_ALG_ALGSEL_SHIFT] * 2;
-	ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
-
-	if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE)
-		goto badkey;
-
 #ifdef DEBUG
 	printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
 	       keys.authkeylen + keys.enckeylen, keys.enckeylen,
 	       keys.authkeylen);
-	printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
-	       ctx->split_key_len, ctx->split_key_pad_len);
 	print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 #endif
 
-	ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen);
+	ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey,
+			    keys.authkeylen, CAAM_MAX_KEY_SIZE -
+			    keys.enckeylen);
 	if (ret) {
 		goto badkey;
 	}
 
 	/* postpend encryption key to auth split key */
-	memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen);
+	memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 
-	ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
+	ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->adata.keylen_pad +
 				      keys.enckeylen, DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, ctx->key_dma)) {
 		dev_err(jrdev, "unable to map key i/o memory\n");
@@ -1376,14 +624,14 @@ static int aead_setkey(struct crypto_aead *aead,
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
-		       ctx->split_key_pad_len + keys.enckeylen, 1);
+		       ctx->adata.keylen_pad + keys.enckeylen, 1);
 #endif
 
-	ctx->enckeylen = keys.enckeylen;
+	ctx->cdata.keylen = keys.enckeylen;
 
 	ret = aead_set_sh_desc(aead);
 	if (ret) {
-		dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
+		dma_unmap_single(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
 				 keys.enckeylen, DMA_TO_DEVICE);
 	}
 
@@ -1412,11 +660,11 @@ static int gcm_setkey(struct crypto_aead *aead,
 		dev_err(jrdev, "unable to map key i/o memory\n");
 		return -ENOMEM;
 	}
-	ctx->enckeylen = keylen;
+	ctx->cdata.keylen = keylen;
 
 	ret = gcm_set_sh_desc(aead);
 	if (ret) {
-		dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
+		dma_unmap_single(jrdev, ctx->key_dma, ctx->cdata.keylen,
 				 DMA_TO_DEVICE);
 	}
 
@@ -1444,9 +692,9 @@ static int rfc4106_setkey(struct crypto_aead *aead,
 	 * The last four bytes of the key material are used as the salt value
 	 * in the nonce. Update the AES key length.
 	 */
-	ctx->enckeylen = keylen - 4;
+	ctx->cdata.keylen = keylen - 4;
 
-	ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
+	ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->cdata.keylen,
 				      DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, ctx->key_dma)) {
 		dev_err(jrdev, "unable to map key i/o memory\n");
@@ -1455,7 +703,7 @@ static int rfc4106_setkey(struct crypto_aead *aead,
 
 	ret = rfc4106_set_sh_desc(aead);
 	if (ret) {
-		dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
+		dma_unmap_single(jrdev, ctx->key_dma, ctx->cdata.keylen,
 				 DMA_TO_DEVICE);
 	}
 
@@ -1483,9 +731,9 @@ static int rfc4543_setkey(struct crypto_aead *aead,
 	 * The last four bytes of the key material are used as the salt value
 	 * in the nonce. Update the AES key length.
 	 */
-	ctx->enckeylen = keylen - 4;
+	ctx->cdata.keylen = keylen - 4;
 
-	ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
+	ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->cdata.keylen,
 				      DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, ctx->key_dma)) {
 		dev_err(jrdev, "unable to map key i/o memory\n");
@@ -1494,7 +742,7 @@ static int rfc4543_setkey(struct crypto_aead *aead,
 
 	ret = rfc4543_set_sh_desc(aead);
 	if (ret) {
-		dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
+		dma_unmap_single(jrdev, ctx->key_dma, ctx->cdata.keylen,
 				 DMA_TO_DEVICE);
 	}
 
@@ -1505,21 +753,18 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 			     const u8 *key, unsigned int keylen)
 {
 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
-	struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher;
 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher);
 	const char *alg_name = crypto_tfm_alg_name(tfm);
 	struct device *jrdev = ctx->jrdev;
-	int ret = 0;
-	u32 *key_jump_cmd;
+	unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 	u32 *desc;
-	u8 *nonce;
-	u32 geniv;
 	u32 ctx1_iv_off = 0;
-	const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
+	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 			       OP_ALG_AAI_CTR_MOD128);
 	const bool is_rfc3686 = (ctr_mode &&
 				 (strstr(alg_name, "rfc3686") != NULL));
 
+	memcpy(ctx->key, key, keylen);
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@@ -1542,60 +787,20 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 		keylen -= CTR_RFC3686_NONCE_SIZE;
 	}
 
-	memcpy(ctx->key, key, keylen);
 	ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
 				      DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, ctx->key_dma)) {
 		dev_err(jrdev, "unable to map key i/o memory\n");
 		return -ENOMEM;
 	}
-	ctx->enckeylen = keylen;
+	ctx->cdata.keylen = keylen;
+	ctx->cdata.key_virt = ctx->key;
+	ctx->cdata.key_inline = true;
 
 	/* ablkcipher_encrypt shared descriptor */
 	desc = ctx->sh_desc_enc;
-	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
-	/* Skip if already shared */
-	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-				   JUMP_COND_SHRD);
-
-	/* Load class1 key only */
-	append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
-			  ctx->enckeylen, CLASS_1 |
-			  KEY_DEST_CLASS_REG);
-
-	/* Load nonce into CONTEXT1 reg */
-	if (is_rfc3686) {
-		nonce = (u8 *)key + keylen;
-		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
-				   LDST_CLASS_IND_CCB |
-				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
-		append_move(desc, MOVE_WAITCOMP |
-			    MOVE_SRC_OUTFIFO |
-			    MOVE_DEST_CLASS1CTX |
-			    (16 << MOVE_OFFSET_SHIFT) |
-			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
-	}
-
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/* Load iv */
-	append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
-			LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
-
-	/* Load counter into CONTEXT1 reg */
-	if (is_rfc3686)
-		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
-				     LDST_SRCDST_BYTE_CONTEXT |
-				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-				      LDST_OFFSET_SHIFT));
-
-	/* Load operation */
-	append_operation(desc, ctx->class1_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
-
-	/* Perform operation */
-	ablkcipher_append_src_dst(desc);
-
+	cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
+				     ctx1_iv_off);
 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -1603,61 +808,11 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR,
-		       "ablkcipher enc shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
+
 	/* ablkcipher_decrypt shared descriptor */
 	desc = ctx->sh_desc_dec;
-
-	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
-	/* Skip if already shared */
-	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-				   JUMP_COND_SHRD);
-
-	/* Load class1 key only */
-	append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
-			  ctx->enckeylen, CLASS_1 |
-			  KEY_DEST_CLASS_REG);
-
-	/* Load nonce into CONTEXT1 reg */
-	if (is_rfc3686) {
-		nonce = (u8 *)key + keylen;
-		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
-				   LDST_CLASS_IND_CCB |
-				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
-		append_move(desc, MOVE_WAITCOMP |
-			    MOVE_SRC_OUTFIFO |
-			    MOVE_DEST_CLASS1CTX |
-			    (16 << MOVE_OFFSET_SHIFT) |
-			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
-	}
-
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/* load IV */
-	append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
-			LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
-
-	/* Load counter into CONTEXT1 reg */
-	if (is_rfc3686)
-		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
-				     LDST_SRCDST_BYTE_CONTEXT |
-				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-				      LDST_OFFSET_SHIFT));
-
-	/* Choose operation */
-	if (ctr_mode)
-		append_operation(desc, ctx->class1_alg_type |
-				 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
-	else
-		append_dec_op1(desc, ctx->class1_alg_type);
-
-	/* Perform operation */
-	ablkcipher_append_src_dst(desc);
-
+	cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
+				     ctx1_iv_off);
 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 					      desc_bytes(desc),
 					      DMA_TO_DEVICE);
@@ -1666,76 +821,10 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 		return -ENOMEM;
 	}
 
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR,
-		       "ablkcipher dec shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 	/* ablkcipher_givencrypt shared descriptor */
 	desc = ctx->sh_desc_givenc;
-
-	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
-	/* Skip if already shared */
-	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-				   JUMP_COND_SHRD);
-
-	/* Load class1 key only */
-	append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
-			  ctx->enckeylen, CLASS_1 |
-			  KEY_DEST_CLASS_REG);
-
-	/* Load Nonce into CONTEXT1 reg */
-	if (is_rfc3686) {
-		nonce = (u8 *)key + keylen;
-		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
-				   LDST_CLASS_IND_CCB |
-				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
-		append_move(desc, MOVE_WAITCOMP |
-			    MOVE_SRC_OUTFIFO |
-			    MOVE_DEST_CLASS1CTX |
-			    (16 << MOVE_OFFSET_SHIFT) |
-			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
-	}
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/* Generate IV */
-	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
-		NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
-		NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT);
-	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
-			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
-	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
-	append_move(desc, MOVE_WAITCOMP |
-		    MOVE_SRC_INFIFO |
-		    MOVE_DEST_CLASS1CTX |
-		    (crt->ivsize << MOVE_LEN_SHIFT) |
-		    (ctx1_iv_off << MOVE_OFFSET_SHIFT));
-	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
-
-	/* Copy generated IV to memory */
-	append_seq_store(desc, crt->ivsize,
-			 LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
-			 (ctx1_iv_off << LDST_OFFSET_SHIFT));
-
-	/* Load Counter into CONTEXT1 reg */
-	if (is_rfc3686)
-		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
-				     LDST_SRCDST_BYTE_CONTEXT |
-				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
-				      LDST_OFFSET_SHIFT));
-
-	if (ctx1_iv_off)
-		append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
-			    (1 << JUMP_OFFSET_SHIFT));
-
-	/* Load operation */
-	append_operation(desc, ctx->class1_alg_type |
-			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
-
-	/* Perform operation */
-	ablkcipher_append_src_dst(desc);
-
+	cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686,
+					ctx1_iv_off);
 	ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
 						 desc_bytes(desc),
 						 DMA_TO_DEVICE);
@@ -1743,14 +832,8 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR,
-		       "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
 
-	return ret;
+	return 0;
 }
 
 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
@@ -1758,8 +841,7 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 {
 	struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
 	struct device *jrdev = ctx->jrdev;
-	u32 *key_jump_cmd, *desc;
-	__be64 sector_size = cpu_to_be64(512);
+	u32 *desc;
 
 	if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {
 		crypto_ablkcipher_set_flags(ablkcipher,
@@ -1774,88 +856,23 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 		dev_err(jrdev, "unable to map key i/o memory\n");
 		return -ENOMEM;
 	}
-	ctx->enckeylen = keylen;
+	ctx->cdata.keylen = keylen;
+	ctx->cdata.key_virt = ctx->key;
+	ctx->cdata.key_inline = true;
 
 	/* xts_ablkcipher_encrypt shared descriptor */
 	desc = ctx->sh_desc_enc;
-	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
-	/* Skip if already shared */
-	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-				   JUMP_COND_SHRD);
-
-	/* Load class1 keys only */
-	append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
-			  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
-
-	/* Load sector size with index 40 bytes (0x28) */
-	append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT |
-		   LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8);
-	append_data(desc, (void *)&sector_size, 8);
-
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/*
-	 * create sequence for loading the sector index
-	 * Upper 8B of IV - will be used as sector index
-	 * Lower 8B of IV - will be discarded
-	 */
-	append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
-		   LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8);
-	append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
-
-	/* Load operation */
-	append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL |
-			 OP_ALG_ENCRYPT);
-
-	/* Perform operation */
-	ablkcipher_append_src_dst(desc);
-
+	cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata);
 	ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
 					      DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR,
-		       "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
 
 	/* xts_ablkcipher_decrypt shared descriptor */
 	desc = ctx->sh_desc_dec;
-
-	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
-	/* Skip if already shared */
-	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-				   JUMP_COND_SHRD);
-
-	/* Load class1 key only */
-	append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
-			  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
-
-	/* Load sector size with index 40 bytes (0x28) */
-	append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT |
-		   LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8);
-	append_data(desc, (void *)&sector_size, 8);
-
-	set_jump_tgt_here(desc, key_jump_cmd);
-
-	/*
-	 * create sequence for loading the sector index
-	 * Upper 8B of IV - will be used as sector index
-	 * Lower 8B of IV - will be discarded
-	 */
-	append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
-		   LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8);
-	append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
-
-	/* Load operation */
-	append_dec_op1(desc, ctx->class1_alg_type);
-
-	/* Perform operation */
-	ablkcipher_append_src_dst(desc);
-
+	cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata);
 	ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
 					      DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
@@ -1864,31 +881,22 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 		dev_err(jrdev, "unable to map shared descriptor\n");
 		return -ENOMEM;
 	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR,
-		       "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
 
 	return 0;
 }
 
 /*
  * aead_edesc - s/w-extended aead descriptor
- * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
  * @src_nents: number of segments in input scatterlist
  * @dst_nents: number of segments in output scatterlist
- * @iv_dma: dma address of iv for checking continuity and link table
- * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
  * @sec4_sg_bytes: length of dma mapped sec4_sg space
  * @sec4_sg_dma: bus physical mapped address of h/w link table
+ * @sec4_sg: pointer to h/w link table
  * @hw_desc: the h/w job descriptor followed by any referenced link tables
  */
 struct aead_edesc {
-	int assoc_nents;
 	int src_nents;
 	int dst_nents;
-	dma_addr_t iv_dma;
 	int sec4_sg_bytes;
 	dma_addr_t sec4_sg_dma;
 	struct sec4_sg_entry *sec4_sg;
@@ -1900,9 +908,9 @@ struct aead_edesc {
  * @src_nents: number of segments in input scatterlist
  * @dst_nents: number of segments in output scatterlist
  * @iv_dma: dma address of iv for checking continuity and link table
- * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
  * @sec4_sg_bytes: length of dma mapped sec4_sg space
  * @sec4_sg_dma: bus physical mapped address of h/w link table
+ * @sec4_sg: pointer to h/w link table
  * @hw_desc: the h/w job descriptor followed by any referenced link tables
  */
 struct ablkcipher_edesc {
@@ -2019,8 +1027,7 @@ static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 #endif
 
-	edesc = (struct ablkcipher_edesc *)((char *)desc -
-		 offsetof(struct ablkcipher_edesc, hw_desc));
+	edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]);
 
 	if (err)
 		caam_jr_strstatus(jrdev, err);
@@ -2031,7 +1038,7 @@ static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 		       edesc->src_nents > 1 ? 100 : ivsize, 1);
 	dbg_dump_sg(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
 		    DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
-		    edesc->dst_nents > 1 ? 100 : req->nbytes, 1, true);
+		    edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
 #endif
 
 	ablkcipher_unmap(jrdev, edesc, req);
@@ -2052,8 +1059,7 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 #endif
 
-	edesc = (struct ablkcipher_edesc *)((char *)desc -
-		 offsetof(struct ablkcipher_edesc, hw_desc));
+	edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]);
 	if (err)
 		caam_jr_strstatus(jrdev, err);
 
@@ -2063,7 +1069,7 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 		       ivsize, 1);
 	dbg_dump_sg(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
 		    DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
-		    edesc->dst_nents > 1 ? 100 : req->nbytes, 1, true);
+		    edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
 #endif
 
 	ablkcipher_unmap(jrdev, edesc, req);
@@ -2157,7 +1163,7 @@ static void init_gcm_job(struct aead_request *req,
 			 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last);
 	/* Append Salt */
 	if (!generic_gcm)
-		append_data(desc, ctx->key + ctx->enckeylen, 4);
+		append_data(desc, ctx->key + ctx->cdata.keylen, 4);
 	/* Append IV */
 	append_data(desc, req->iv, ivsize);
 	/* End of blank commands */
@@ -2172,7 +1178,7 @@ static void init_authenc_job(struct aead_request *req,
 						 struct caam_aead_alg, aead);
 	unsigned int ivsize = crypto_aead_ivsize(aead);
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
-	const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
+	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 			       OP_ALG_AAI_CTR_MOD128);
 	const bool is_rfc3686 = alg->caam.rfc3686;
 	u32 *desc = edesc->hw_desc;
@@ -2218,15 +1224,13 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
 	int len, sec4_sg_index = 0;
 
 #ifdef DEBUG
-	bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-					      CRYPTO_TFM_REQ_MAY_SLEEP)) != 0);
 	print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 		       ivsize, 1);
 	printk(KERN_ERR "asked=%d, nbytes%d\n", (int)edesc->src_nents ? 100 : req->nbytes, req->nbytes);
 	dbg_dump_sg(KERN_ERR, "src    @"__stringify(__LINE__)": ",
 		    DUMP_PREFIX_ADDRESS, 16, 4, req->src,
-		    edesc->src_nents ? 100 : req->nbytes, 1, may_sleep);
+		    edesc->src_nents ? 100 : req->nbytes, 1);
 #endif
 
 	len = desc_len(sh_desc);
@@ -2278,14 +1282,12 @@ static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
 	int len, sec4_sg_index = 0;
 
 #ifdef DEBUG
-	bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-					      CRYPTO_TFM_REQ_MAY_SLEEP)) != 0);
 	print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 		       ivsize, 1);
 	dbg_dump_sg(KERN_ERR, "src    @" __stringify(__LINE__) ": ",
 		    DUMP_PREFIX_ADDRESS, 16, 4, req->src,
-		    edesc->src_nents ? 100 : req->nbytes, 1, may_sleep);
+		    edesc->src_nents ? 100 : req->nbytes, 1);
 #endif
 
 	len = desc_len(sh_desc);
@@ -2344,10 +1346,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 
 	/* Check if data are contiguous. */
 	all_contig = !src_nents;
-	if (!all_contig) {
-		src_nents = src_nents ? : 1;
+	if (!all_contig)
 		sec4_sg_len = src_nents;
-	}
 
 	sec4_sg_len += dst_nents;
 
@@ -2556,11 +1556,9 @@ static int aead_decrypt(struct aead_request *req)
 	int ret = 0;
 
 #ifdef DEBUG
-	bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-					      CRYPTO_TFM_REQ_MAY_SLEEP)) != 0);
 	dbg_dump_sg(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
 		    DUMP_PREFIX_ADDRESS, 16, 4, req->src,
-		    req->assoclen + req->cryptlen, 1, may_sleep);
+		    req->assoclen + req->cryptlen, 1);
 #endif
 
 	/* allocate extended descriptor */
@@ -2618,16 +1616,33 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 	if (likely(req->src == req->dst)) {
 		sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
 				 DMA_BIDIRECTIONAL);
+		if (unlikely(!sgc)) {
+			dev_err(jrdev, "unable to map source\n");
+			return ERR_PTR(-ENOMEM);
+		}
 	} else {
 		sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
 				 DMA_TO_DEVICE);
+		if (unlikely(!sgc)) {
+			dev_err(jrdev, "unable to map source\n");
+			return ERR_PTR(-ENOMEM);
+		}
+
 		sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
 				 DMA_FROM_DEVICE);
+		if (unlikely(!sgc)) {
+			dev_err(jrdev, "unable to map destination\n");
+			dma_unmap_sg(jrdev, req->src, src_nents ? : 1,
+				     DMA_TO_DEVICE);
+			return ERR_PTR(-ENOMEM);
+		}
 	}
 
 	iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, iv_dma)) {
 		dev_err(jrdev, "unable to map IV\n");
+		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, 0, 0);
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -2647,6 +1662,8 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 			GFP_DMA | flags);
 	if (!edesc) {
 		dev_err(jrdev, "could not allocate extended descriptor\n");
+		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
+			   iv_dma, ivsize, 0, 0);
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -2673,6 +1690,9 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 					    sec4_sg_bytes, DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 		dev_err(jrdev, "unable to map S/G table\n");
+		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
+			   iv_dma, ivsize, 0, 0);
+		kfree(edesc);
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -2794,11 +1814,26 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 	if (likely(req->src == req->dst)) {
 		sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
 				 DMA_BIDIRECTIONAL);
+		if (unlikely(!sgc)) {
+			dev_err(jrdev, "unable to map source\n");
+			return ERR_PTR(-ENOMEM);
+		}
 	} else {
 		sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
 				 DMA_TO_DEVICE);
+		if (unlikely(!sgc)) {
+			dev_err(jrdev, "unable to map source\n");
+			return ERR_PTR(-ENOMEM);
+		}
+
 		sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
 				 DMA_FROM_DEVICE);
+		if (unlikely(!sgc)) {
+			dev_err(jrdev, "unable to map destination\n");
+			dma_unmap_sg(jrdev, req->src, src_nents ? : 1,
+				     DMA_TO_DEVICE);
+			return ERR_PTR(-ENOMEM);
+		}
 	}
 
 	/*
@@ -2808,6 +1843,8 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 	iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, iv_dma)) {
 		dev_err(jrdev, "unable to map IV\n");
+		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
+			   0, 0, 0);
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -2823,6 +1860,8 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 			GFP_DMA | flags);
 	if (!edesc) {
 		dev_err(jrdev, "could not allocate extended descriptor\n");
+		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
+			   iv_dma, ivsize, 0, 0);
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -2850,6 +1889,9 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 					    sec4_sg_bytes, DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
 		dev_err(jrdev, "unable to map S/G table\n");
+		caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
+			   iv_dma, ivsize, 0, 0);
+		kfree(edesc);
 		return ERR_PTR(-ENOMEM);
 	}
 	edesc->iv_dma = iv_dma;
@@ -2916,7 +1958,6 @@ struct caam_alg_template {
 	} template_u;
 	u32 class1_alg_type;
 	u32 class2_alg_type;
-	u32 alg_op;
 };
 
 static struct caam_alg_template driver_algs[] = {
@@ -3101,7 +2142,6 @@ static struct caam_aead_alg driver_aeads[] = {
 		.caam = {
 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3123,7 +2163,6 @@ static struct caam_aead_alg driver_aeads[] = {
 		.caam = {
 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3145,7 +2184,6 @@ static struct caam_aead_alg driver_aeads[] = {
 		.caam = {
 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3167,7 +2205,6 @@ static struct caam_aead_alg driver_aeads[] = {
 		.caam = {
 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3189,7 +2226,6 @@ static struct caam_aead_alg driver_aeads[] = {
 		.caam = {
 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3211,7 +2247,6 @@ static struct caam_aead_alg driver_aeads[] = {
 		.caam = {
 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3233,7 +2268,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3256,7 +2290,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3279,7 +2312,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3302,7 +2334,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3325,7 +2356,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3348,7 +2378,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3371,7 +2400,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3394,7 +2422,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3417,7 +2444,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3440,7 +2466,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3463,7 +2488,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3486,7 +2510,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3509,7 +2532,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
 		}
 	},
 	{
@@ -3532,7 +2554,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		}
 	},
@@ -3556,7 +2577,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3580,7 +2600,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3604,7 +2623,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3628,7 +2646,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3652,7 +2669,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3676,7 +2692,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3700,7 +2715,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3724,7 +2738,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3748,7 +2761,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3772,7 +2784,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3795,7 +2806,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3818,7 +2828,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3841,7 +2850,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3864,7 +2872,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3887,7 +2894,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3910,7 +2916,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3933,7 +2938,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -3956,7 +2960,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -3979,7 +2982,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -4002,7 +3004,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -4025,7 +3026,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
 		},
 	},
 	{
@@ -4048,7 +3048,6 @@ static struct caam_aead_alg driver_aeads[] = {
 			.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
 			.geniv = true,
 		},
 	},
@@ -4073,7 +3072,6 @@ static struct caam_aead_alg driver_aeads[] = {
 					   OP_ALG_AAI_CTR_MOD128,
 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
 			.rfc3686 = true,
 		},
 	},
@@ -4098,7 +3096,6 @@ static struct caam_aead_alg driver_aeads[] = {
 					   OP_ALG_AAI_CTR_MOD128,
 			.class2_alg_type = OP_ALG_ALGSEL_MD5 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
 			.rfc3686 = true,
 			.geniv = true,
 		},
@@ -4124,7 +3121,6 @@ static struct caam_aead_alg driver_aeads[] = {
 					   OP_ALG_AAI_CTR_MOD128,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 			.rfc3686 = true,
 		},
 	},
@@ -4149,7 +3145,6 @@ static struct caam_aead_alg driver_aeads[] = {
 					   OP_ALG_AAI_CTR_MOD128,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 			.rfc3686 = true,
 			.geniv = true,
 		},
@@ -4175,7 +3170,6 @@ static struct caam_aead_alg driver_aeads[] = {
 					   OP_ALG_AAI_CTR_MOD128,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
 			.rfc3686 = true,
 		},
 	},
@@ -4200,7 +3194,6 @@ static struct caam_aead_alg driver_aeads[] = {
 					   OP_ALG_AAI_CTR_MOD128,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
 			.rfc3686 = true,
 			.geniv = true,
 		},
@@ -4226,7 +3219,6 @@ static struct caam_aead_alg driver_aeads[] = {
 					   OP_ALG_AAI_CTR_MOD128,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 			.rfc3686 = true,
 		},
 	},
@@ -4251,7 +3243,6 @@ static struct caam_aead_alg driver_aeads[] = {
 					   OP_ALG_AAI_CTR_MOD128,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 			.rfc3686 = true,
 			.geniv = true,
 		},
@@ -4277,7 +3268,6 @@ static struct caam_aead_alg driver_aeads[] = {
 					   OP_ALG_AAI_CTR_MOD128,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
 			.rfc3686 = true,
 		},
 	},
@@ -4302,7 +3292,6 @@ static struct caam_aead_alg driver_aeads[] = {
 					   OP_ALG_AAI_CTR_MOD128,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
 			.rfc3686 = true,
 			.geniv = true,
 		},
@@ -4328,7 +3317,6 @@ static struct caam_aead_alg driver_aeads[] = {
 					   OP_ALG_AAI_CTR_MOD128,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
 			.rfc3686 = true,
 		},
 	},
@@ -4353,7 +3341,6 @@ static struct caam_aead_alg driver_aeads[] = {
 					   OP_ALG_AAI_CTR_MOD128,
 			.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
 					   OP_ALG_AAI_HMAC_PRECOMP,
-			.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
 			.rfc3686 = true,
 			.geniv = true,
 		},
@@ -4375,9 +3362,8 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
 	}
 
 	/* copy descriptor header template value */
-	ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
-	ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
-	ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op;
+	ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
+	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
 
 	return 0;
 }
@@ -4420,7 +3406,7 @@ static void caam_exit_common(struct caam_ctx *ctx)
 	if (ctx->key_dma &&
 	    !dma_mapping_error(ctx->jrdev, ctx->key_dma))
 		dma_unmap_single(ctx->jrdev, ctx->key_dma,
-				 ctx->enckeylen + ctx->split_key_pad_len,
+				 ctx->cdata.keylen + ctx->adata.keylen_pad,
 				 DMA_TO_DEVICE);
 
 	caam_jr_free(ctx->jrdev);
@@ -4498,7 +3484,6 @@ static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
 
 	t_alg->caam.class1_alg_type = template->class1_alg_type;
 	t_alg->caam.class2_alg_type = template->class2_alg_type;
-	t_alg->caam.alg_op = template->alg_op;
 
 	return t_alg;
 }
diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c
new file mode 100644
index 000000000000..f3f48c10b9d6
--- /dev/null
+++ b/drivers/crypto/caam/caamalg_desc.c
@@ -0,0 +1,1306 @@
+/*
+ * Shared descriptors for aead, ablkcipher algorithms
+ *
+ * Copyright 2016 NXP
+ */
+
+#include "compat.h"
+#include "desc_constr.h"
+#include "caamalg_desc.h"
+
+/*
+ * For aead functions, read payload and write payload,
+ * both of which are specified in req->src and req->dst
+ */
+static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
+{
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
+			     KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
+}
+
+/* Set DK bit in class 1 operation if shared */
+static inline void append_dec_op1(u32 *desc, u32 type)
+{
+	u32 *jump_cmd, *uncond_jump_cmd;
+
+	/* DK bit is valid only for AES */
+	if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
+		append_operation(desc, type | OP_ALG_AS_INITFINAL |
+				 OP_ALG_DECRYPT);
+		return;
+	}
+
+	jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
+	append_operation(desc, type | OP_ALG_AS_INITFINAL |
+			 OP_ALG_DECRYPT);
+	uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
+	set_jump_tgt_here(desc, jump_cmd);
+	append_operation(desc, type | OP_ALG_AS_INITFINAL |
+			 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
+	set_jump_tgt_here(desc, uncond_jump_cmd);
+}
+
+/**
+ * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
+ *                               (non-protocol) with no (null) encryption.
+ * @desc: pointer to buffer used for descriptor construction
+ * @adata: pointer to authentication transform definitions. Note that since a
+ *         split key is to be used, the size of the split key itself is
+ *         specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
+ *         SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
+ * @icvsize: integrity check value (ICV) size (truncated or full)
+ *
+ * Note: Requires an MDHA split key.
+ */
+void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
+				 unsigned int icvsize)
+{
+	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
+
+	/* Skip if already shared */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD);
+	if (adata->key_inline)
+		append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
+				  adata->keylen, CLASS_2 | KEY_DEST_MDHA_SPLIT |
+				  KEY_ENC);
+	else
+		append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
+			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/* assoclen + cryptlen = seqinlen */
+	append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
+
+	/* Prepare to read and write cryptlen + assoclen bytes */
+	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
+	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+
+	/*
+	 * MOVE_LEN opcode is not available in all SEC HW revisions,
+	 * thus need to do some magic, i.e. self-patch the descriptor
+	 * buffer.
+	 */
+	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
+				    MOVE_DEST_MATH3 |
+				    (0x6 << MOVE_LEN_SHIFT));
+	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
+				     MOVE_DEST_DESCBUF |
+				     MOVE_WAITCOMP |
+				     (0x8 << MOVE_LEN_SHIFT));
+
+	/* Class 2 operation */
+	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_ENCRYPT);
+
+	/* Read and write cryptlen bytes */
+	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
+
+	set_move_tgt_here(desc, read_move_cmd);
+	set_move_tgt_here(desc, write_move_cmd);
+	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
+	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
+		    MOVE_AUX_LS);
+
+	/* Write ICV */
+	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
+			 LDST_SRCDST_BYTE_CONTEXT);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR,
+		       "aead null enc shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
+
+/**
+ * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
+ *                               (non-protocol) with no (null) decryption.
+ * @desc: pointer to buffer used for descriptor construction
+ * @adata: pointer to authentication transform definitions. Note that since a
+ *         split key is to be used, the size of the split key itself is
+ *         specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
+ *         SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
+ * @icvsize: integrity check value (ICV) size (truncated or full)
+ *
+ * Note: Requires an MDHA split key.
+ */
+void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
+				 unsigned int icvsize)
+{
+	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
+
+	/* Skip if already shared */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD);
+	if (adata->key_inline)
+		append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
+				  adata->keylen, CLASS_2 |
+				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
+	else
+		append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
+			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/* Class 2 operation */
+	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
+
+	/* assoclen + cryptlen = seqoutlen */
+	append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
+
+	/* Prepare to read and write cryptlen + assoclen bytes */
+	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
+	append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
+
+	/*
+	 * MOVE_LEN opcode is not available in all SEC HW revisions,
+	 * thus need to do some magic, i.e. self-patch the descriptor
+	 * buffer.
+	 */
+	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
+				    MOVE_DEST_MATH2 |
+				    (0x6 << MOVE_LEN_SHIFT));
+	write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
+				     MOVE_DEST_DESCBUF |
+				     MOVE_WAITCOMP |
+				     (0x8 << MOVE_LEN_SHIFT));
+
+	/* Read and write cryptlen bytes */
+	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
+
+	/*
+	 * Insert a NOP here, since we need at least 4 instructions between
+	 * code patching the descriptor buffer and the location being patched.
+	 */
+	jump_cmd = append_jump(desc, JUMP_TEST_ALL);
+	set_jump_tgt_here(desc, jump_cmd);
+
+	set_move_tgt_here(desc, read_move_cmd);
+	set_move_tgt_here(desc, write_move_cmd);
+	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
+	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
+		    MOVE_AUX_LS);
+	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
+
+	/* Load ICV */
+	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
+			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR,
+		       "aead null dec shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
+
+static void init_sh_desc_key_aead(u32 * const desc,
+				  struct alginfo * const cdata,
+				  struct alginfo * const adata,
+				  const bool is_rfc3686, u32 *nonce)
+{
+	u32 *key_jump_cmd;
+	unsigned int enckeylen = cdata->keylen;
+
+	/* Note: Context registers are saved. */
+	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
+
+	/* Skip if already shared */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD);
+
+	/*
+	 * RFC3686 specific:
+	 *	| key = {AUTH_KEY, ENC_KEY, NONCE}
+	 *	| enckeylen = encryption key size + nonce size
+	 */
+	if (is_rfc3686)
+		enckeylen -= CTR_RFC3686_NONCE_SIZE;
+
+	if (adata->key_inline)
+		append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
+				  adata->keylen, CLASS_2 |
+				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
+	else
+		append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
+			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
+
+	if (cdata->key_inline)
+		append_key_as_imm(desc, cdata->key_virt, enckeylen,
+				  enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
+	else
+		append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
+			   KEY_DEST_CLASS_REG);
+
+	/* Load Counter into CONTEXT1 reg */
+	if (is_rfc3686) {
+		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
+				   LDST_CLASS_IND_CCB |
+				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
+		append_move(desc,
+			    MOVE_SRC_OUTFIFO |
+			    MOVE_DEST_CLASS1CTX |
+			    (16 << MOVE_OFFSET_SHIFT) |
+			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
+	}
+
+	set_jump_tgt_here(desc, key_jump_cmd);
+}
+
+/**
+ * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
+ *                          (non-protocol).
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
+ *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
+ * @adata: pointer to authentication transform definitions. Note that since a
+ *         split key is to be used, the size of the split key itself is
+ *         specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
+ *         SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
+ * @icvsize: integrity check value (ICV) size (truncated or full)
+ * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
+ * @nonce: pointer to rfc3686 nonce
+ * @ctx1_iv_off: IV offset in CONTEXT1 register
+ *
+ * Note: Requires an MDHA split key.
+ */
+void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
+			    struct alginfo *adata, unsigned int icvsize,
+			    const bool is_rfc3686, u32 *nonce,
+			    const u32 ctx1_iv_off)
+{
+	/* Note: Context registers are saved. */
+	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
+
+	/* Class 2 operation */
+	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_ENCRYPT);
+
+	/* Read and write assoclen bytes */
+	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
+	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+
+	/* Skip assoc data */
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
+
+	/* read assoc before reading payload */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
+				      FIFOLDST_VLF);
+
+	/* Load Counter into CONTEXT1 reg */
+	if (is_rfc3686)
+		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
+				     LDST_SRCDST_BYTE_CONTEXT |
+				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				      LDST_OFFSET_SHIFT));
+
+	/* Class 1 operation */
+	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_ENCRYPT);
+
+	/* Read and write cryptlen bytes */
+	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+	aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
+
+	/* Write ICV */
+	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
+			 LDST_SRCDST_BYTE_CONTEXT);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
+
+/**
+ * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
+ *                          (non-protocol).
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
+ *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
+ * @adata: pointer to authentication transform definitions. Note that since a
+ *         split key is to be used, the size of the split key itself is
+ *         specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
+ *         SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
+ * @ivsize: initialization vector size
+ * @icvsize: integrity check value (ICV) size (truncated or full)
+ * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
+ * @nonce: pointer to rfc3686 nonce
+ * @ctx1_iv_off: IV offset in CONTEXT1 register
+ *
+ * Note: Requires an MDHA split key.
+ */
+void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
+			    struct alginfo *adata, unsigned int ivsize,
+			    unsigned int icvsize, const bool geniv,
+			    const bool is_rfc3686, u32 *nonce,
+			    const u32 ctx1_iv_off)
+{
+	/* Note: Context registers are saved. */
+	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
+
+	/* Class 2 operation */
+	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
+
+	/* Read and write assoclen bytes */
+	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
+	if (geniv)
+		append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize);
+	else
+		append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+
+	/* Skip assoc data */
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
+
+	/* read assoc before reading payload */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
+			     KEY_VLF);
+
+	if (geniv) {
+		append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
+				LDST_SRCDST_BYTE_CONTEXT |
+				(ctx1_iv_off << LDST_OFFSET_SHIFT));
+		append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
+			    (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
+	}
+
+	/* Load Counter into CONTEXT1 reg */
+	if (is_rfc3686)
+		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
+				     LDST_SRCDST_BYTE_CONTEXT |
+				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				      LDST_OFFSET_SHIFT));
+
+	/* Choose operation */
+	if (ctx1_iv_off)
+		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+				 OP_ALG_DECRYPT);
+	else
+		append_dec_op1(desc, cdata->algtype);
+
+	/* Read and write cryptlen bytes */
+	append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
+	append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
+	aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
+
+	/* Load ICV */
+	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
+			     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
+
+/**
+ * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
+ *                             (non-protocol) with HW-generated initialization
+ *                             vector.
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
+ *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
+ * @adata: pointer to authentication transform definitions. Note that since a
+ *         split key is to be used, the size of the split key itself is
+ *         specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
+ *         SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
+ * @ivsize: initialization vector size
+ * @icvsize: integrity check value (ICV) size (truncated or full)
+ * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
+ * @nonce: pointer to rfc3686 nonce
+ * @ctx1_iv_off: IV offset in CONTEXT1 register
+ *
+ * Note: Requires an MDHA split key.
+ */
+void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
+			       struct alginfo *adata, unsigned int ivsize,
+			       unsigned int icvsize, const bool is_rfc3686,
+			       u32 *nonce, const u32 ctx1_iv_off)
+{
+	u32 geniv, moveiv;
+
+	/* Note: Context registers are saved. */
+	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
+
+	if (is_rfc3686)
+		goto copy_iv;
+
+	/* Generate IV */
+	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
+		NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
+		NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
+	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
+			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
+	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
+	append_move(desc, MOVE_WAITCOMP |
+		    MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
+		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
+		    (ivsize << MOVE_LEN_SHIFT));
+	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
+
+copy_iv:
+	/* Copy IV to class 1 context */
+	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
+		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
+		    (ivsize << MOVE_LEN_SHIFT));
+
+	/* Return to encryption */
+	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_ENCRYPT);
+
+	/* Read and write assoclen bytes */
+	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
+	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+
+	/* Skip assoc data */
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
+
+	/* read assoc before reading payload */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
+			     KEY_VLF);
+
+	/* Copy iv from outfifo to class 2 fifo */
+	moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
+		 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
+	append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
+			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
+	append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
+			    LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
+
+	/* Load Counter into CONTEXT1 reg */
+	if (is_rfc3686)
+		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
+				     LDST_SRCDST_BYTE_CONTEXT |
+				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				      LDST_OFFSET_SHIFT));
+
+	/* Class 1 operation */
+	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_ENCRYPT);
+
+	/* Will write ivsize + cryptlen */
+	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+
+	/* Not need to reload iv */
+	append_seq_fifo_load(desc, ivsize,
+			     FIFOLD_CLASS_SKIP);
+
+	/* Will read cryptlen */
+	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
+			     FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
+
+	/* Write ICV */
+	append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
+			 LDST_SRCDST_BYTE_CONTEXT);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR,
+		       "aead givenc shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
+
+/**
+ * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
+ * @icvsize: integrity check value (ICV) size (truncated or full)
+ */
+void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
+			   unsigned int icvsize)
+{
+	u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
+	    *zero_assoc_jump_cmd2;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
+
+	/* skip key loading if they are loaded due to sharing */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD | JUMP_COND_SELF);
+	if (cdata->key_inline)
+		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
+				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
+	else
+		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
+			   KEY_DEST_CLASS_REG);
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/* class 1 operation */
+	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_ENCRYPT);
+
+	/* if assoclen + cryptlen is ZERO, skip to ICV write */
+	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+	zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
+						 JUMP_COND_MATH_Z);
+
+	/* if assoclen is ZERO, skip reading the assoc data */
+	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
+	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
+					   JUMP_COND_MATH_Z);
+
+	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+
+	/* skip assoc data */
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
+
+	/* cryptlen = seqinlen - assoclen */
+	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
+
+	/* if cryptlen is ZERO jump to zero-payload commands */
+	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
+					    JUMP_COND_MATH_Z);
+
+	/* read assoc data */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
+			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
+	set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
+
+	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+
+	/* write encrypted data */
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
+
+	/* read payload data */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
+			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
+
+	/* jump the zero-payload commands */
+	append_jump(desc, JUMP_TEST_ALL | 2);
+
+	/* zero-payload commands */
+	set_jump_tgt_here(desc, zero_payload_jump_cmd);
+
+	/* read assoc data */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
+			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
+
+	/* There is no input data */
+	set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
+
+	/* write ICV */
+	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
+			 LDST_SRCDST_BYTE_CONTEXT);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
+
+/**
+ * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
+ * @icvsize: integrity check value (ICV) size (truncated or full)
+ */
+void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
+			   unsigned int icvsize)
+{
+	u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
+
+	/* skip key loading if they are loaded due to sharing */
+	key_jump_cmd = append_jump(desc, JUMP_JSL |
+				   JUMP_TEST_ALL | JUMP_COND_SHRD |
+				   JUMP_COND_SELF);
+	if (cdata->key_inline)
+		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
+				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
+	else
+		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
+			   KEY_DEST_CLASS_REG);
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/* class 1 operation */
+	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
+
+	/* if assoclen is ZERO, skip reading the assoc data */
+	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
+	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
+						 JUMP_COND_MATH_Z);
+
+	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+
+	/* skip assoc data */
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
+
+	/* read assoc data */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
+			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
+
+	set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
+
+	/* cryptlen = seqoutlen - assoclen */
+	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
+
+	/* jump to zero-payload command if cryptlen is zero */
+	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
+					    JUMP_COND_MATH_Z);
+
+	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
+
+	/* store encrypted data */
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
+
+	/* read payload data */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
+			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
+
+	/* zero-payload command */
+	set_jump_tgt_here(desc, zero_payload_jump_cmd);
+
+	/* read ICV */
+	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
+			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
+
+/**
+ * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
+ *                             (non-protocol).
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
+ * @icvsize: integrity check value (ICV) size (truncated or full)
+ */
+void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
+			       unsigned int icvsize)
+{
+	u32 *key_jump_cmd;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
+
+	/* Skip key loading if it is loaded due to sharing */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD);
+	if (cdata->key_inline)
+		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
+				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
+	else
+		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
+			   KEY_DEST_CLASS_REG);
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/* Class 1 operation */
+	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_ENCRYPT);
+
+	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
+	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+
+	/* Read assoc data */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
+			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
+
+	/* Skip IV */
+	append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
+
+	/* Will read cryptlen bytes */
+	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+
+	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
+
+	/* Skip assoc data */
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
+
+	/* cryptlen = seqoutlen - assoclen */
+	append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
+
+	/* Write encrypted data */
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
+
+	/* Read payload data */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
+			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
+
+	/* Write ICV */
+	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
+			 LDST_SRCDST_BYTE_CONTEXT);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR,
+		       "rfc4106 enc shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
+
+/**
+ * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
+ *                             (non-protocol).
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
+ * @icvsize: integrity check value (ICV) size (truncated or full)
+ */
+void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
+			       unsigned int icvsize)
+{
+	u32 *key_jump_cmd;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
+
+	/* Skip key loading if it is loaded due to sharing */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD);
+	if (cdata->key_inline)
+		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
+				  cdata->keylen, CLASS_1 |
+				  KEY_DEST_CLASS_REG);
+	else
+		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
+			   KEY_DEST_CLASS_REG);
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/* Class 1 operation */
+	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
+
+	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
+	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+
+	/* Read assoc data */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
+			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
+
+	/* Skip IV */
+	append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
+
+	/* Will read cryptlen bytes */
+	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
+
+	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
+
+	/* Skip assoc data */
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
+
+	/* Will write cryptlen bytes */
+	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
+
+	/* Store payload data */
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
+
+	/* Read encrypted data */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
+			     FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
+
+	/* Read ICV */
+	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
+			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR,
+		       "rfc4106 dec shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
+
+/**
+ * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
+ *                             (non-protocol).
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
+ * @icvsize: integrity check value (ICV) size (truncated or full)
+ */
+void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
+			       unsigned int icvsize)
+{
+	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
+
+	/* Skip key loading if it is loaded due to sharing */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD);
+	if (cdata->key_inline)
+		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
+				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
+	else
+		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
+			   KEY_DEST_CLASS_REG);
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/* Class 1 operation */
+	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_ENCRYPT);
+
+	/* assoclen + cryptlen = seqinlen */
+	append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
+
+	/*
+	 * MOVE_LEN opcode is not available in all SEC HW revisions,
+	 * thus need to do some magic, i.e. self-patch the descriptor
+	 * buffer.
+	 */
+	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
+				    (0x6 << MOVE_LEN_SHIFT));
+	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
+				     (0x8 << MOVE_LEN_SHIFT));
+
+	/* Will read assoclen + cryptlen bytes */
+	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+
+	/* Will write assoclen + cryptlen bytes */
+	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+
+	/* Read and write assoclen + cryptlen bytes */
+	aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
+
+	set_move_tgt_here(desc, read_move_cmd);
+	set_move_tgt_here(desc, write_move_cmd);
+	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
+	/* Move payload data to OFIFO */
+	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
+
+	/* Write ICV */
+	append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
+			 LDST_SRCDST_BYTE_CONTEXT);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR,
+		       "rfc4543 enc shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
+
+/**
+ * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
+ *                             (non-protocol).
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
+ * @icvsize: integrity check value (ICV) size (truncated or full)
+ */
+void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
+			       unsigned int icvsize)
+{
+	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
+
+	/* Skip key loading if it is loaded due to sharing */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD);
+	if (cdata->key_inline)
+		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
+				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
+	else
+		append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
+			   KEY_DEST_CLASS_REG);
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/* Class 1 operation */
+	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
+
+	/* assoclen + cryptlen = seqoutlen */
+	append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
+
+	/*
+	 * MOVE_LEN opcode is not available in all SEC HW revisions,
+	 * thus need to do some magic, i.e. self-patch the descriptor
+	 * buffer.
+	 */
+	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
+				    (0x6 << MOVE_LEN_SHIFT));
+	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
+				     (0x8 << MOVE_LEN_SHIFT));
+
+	/* Will read assoclen + cryptlen bytes */
+	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
+
+	/* Will write assoclen + cryptlen bytes */
+	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
+
+	/* Store payload data */
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
+
+	/* In-snoop assoclen + cryptlen data */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
+			     FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
+
+	set_move_tgt_here(desc, read_move_cmd);
+	set_move_tgt_here(desc, write_move_cmd);
+	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
+	/* Move payload data to OFIFO */
+	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
+	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
+
+	/* Read ICV */
+	append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
+			     FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR,
+		       "rfc4543 dec shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
+
+/*
+ * For ablkcipher encrypt and decrypt, read from req->src and
+ * write to req->dst
+ */
+static inline void ablkcipher_append_src_dst(u32 *desc)
+{
+	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
+			     KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
+	append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
+}
+
+/**
+ * cnstr_shdsc_ablkcipher_encap - ablkcipher encapsulation shared descriptor
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
+ *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
+ * @ivsize: initialization vector size
+ * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
+ * @ctx1_iv_off: IV offset in CONTEXT1 register
+ */
+void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata,
+				  unsigned int ivsize, const bool is_rfc3686,
+				  const u32 ctx1_iv_off)
+{
+	u32 *key_jump_cmd;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
+	/* Skip if already shared */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD);
+
+	/* Load class1 key only */
+	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
+			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
+
+	/* Load nonce into CONTEXT1 reg */
+	if (is_rfc3686) {
+		u8 *nonce = cdata->key_virt + cdata->keylen;
+
+		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
+				   LDST_CLASS_IND_CCB |
+				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
+		append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
+			    MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
+			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
+	}
+
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/* Load iv */
+	append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
+			LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
+
+	/* Load counter into CONTEXT1 reg */
+	if (is_rfc3686)
+		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
+				     LDST_SRCDST_BYTE_CONTEXT |
+				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				      LDST_OFFSET_SHIFT));
+
+	/* Load operation */
+	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_ENCRYPT);
+
+	/* Perform operation */
+	ablkcipher_append_src_dst(desc);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR,
+		       "ablkcipher enc shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap);
+
+/**
+ * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
+ *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
+ * @ivsize: initialization vector size
+ * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
+ * @ctx1_iv_off: IV offset in CONTEXT1 register
+ */
+void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata,
+				  unsigned int ivsize, const bool is_rfc3686,
+				  const u32 ctx1_iv_off)
+{
+	u32 *key_jump_cmd;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
+	/* Skip if already shared */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD);
+
+	/* Load class1 key only */
+	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
+			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
+
+	/* Load nonce into CONTEXT1 reg */
+	if (is_rfc3686) {
+		u8 *nonce = cdata->key_virt + cdata->keylen;
+
+		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
+				   LDST_CLASS_IND_CCB |
+				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
+		append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
+			    MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
+			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
+	}
+
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/* load IV */
+	append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
+			LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
+
+	/* Load counter into CONTEXT1 reg */
+	if (is_rfc3686)
+		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
+				     LDST_SRCDST_BYTE_CONTEXT |
+				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				      LDST_OFFSET_SHIFT));
+
+	/* Choose operation */
+	if (ctx1_iv_off)
+		append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+				 OP_ALG_DECRYPT);
+	else
+		append_dec_op1(desc, cdata->algtype);
+
+	/* Perform operation */
+	ablkcipher_append_src_dst(desc);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR,
+		       "ablkcipher dec shdesc@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_decap);
+
+/**
+ * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor
+ *                                   with HW-generated initialization vector.
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
+ *         with OP_ALG_AAI_CBC.
+ * @ivsize: initialization vector size
+ * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
+ * @ctx1_iv_off: IV offset in CONTEXT1 register
+ */
+void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata,
+				     unsigned int ivsize, const bool is_rfc3686,
+				     const u32 ctx1_iv_off)
+{
+	u32 *key_jump_cmd, geniv;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
+	/* Skip if already shared */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD);
+
+	/* Load class1 key only */
+	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
+			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
+
+	/* Load Nonce into CONTEXT1 reg */
+	if (is_rfc3686) {
+		u8 *nonce = cdata->key_virt + cdata->keylen;
+
+		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
+				   LDST_CLASS_IND_CCB |
+				   LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
+		append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
+			    MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
+			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
+	}
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/* Generate IV */
+	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
+		NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND |
+		(ivsize << NFIFOENTRY_DLEN_SHIFT);
+	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
+			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
+	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
+	append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO |
+		    MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) |
+		    (ctx1_iv_off << MOVE_OFFSET_SHIFT));
+	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
+
+	/* Copy generated IV to memory */
+	append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
+			 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
+
+	/* Load Counter into CONTEXT1 reg */
+	if (is_rfc3686)
+		append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
+				     LDST_SRCDST_BYTE_CONTEXT |
+				     ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				      LDST_OFFSET_SHIFT));
+
+	if (ctx1_iv_off)
+		append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
+			    (1 << JUMP_OFFSET_SHIFT));
+
+	/* Load operation */
+	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_ENCRYPT);
+
+	/* Perform operation */
+	ablkcipher_append_src_dst(desc);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR,
+		       "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_givencap);
+
+/**
+ * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared
+ *                                    descriptor
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
+ */
+void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata)
+{
+	__be64 sector_size = cpu_to_be64(512);
+	u32 *key_jump_cmd;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
+	/* Skip if already shared */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD);
+
+	/* Load class1 keys only */
+	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
+			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
+
+	/* Load sector size with index 40 bytes (0x28) */
+	append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
+			   LDST_SRCDST_BYTE_CONTEXT |
+			   (0x28 << LDST_OFFSET_SHIFT));
+
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/*
+	 * create sequence for loading the sector index
+	 * Upper 8B of IV - will be used as sector index
+	 * Lower 8B of IV - will be discarded
+	 */
+	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
+			(0x20 << LDST_OFFSET_SHIFT));
+	append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
+
+	/* Load operation */
+	append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+			 OP_ALG_ENCRYPT);
+
+	/* Perform operation */
+	ablkcipher_append_src_dst(desc);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR,
+		       "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_encap);
+
+/**
+ * cnstr_shdsc_xts_ablkcipher_decap - xts ablkcipher decapsulation shared
+ *                                    descriptor
+ * @desc: pointer to buffer used for descriptor construction
+ * @cdata: pointer to block cipher transform definitions
+ *         Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
+ */
+void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata)
+{
+	__be64 sector_size = cpu_to_be64(512);
+	u32 *key_jump_cmd;
+
+	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
+	/* Skip if already shared */
+	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+				   JUMP_COND_SHRD);
+
+	/* Load class1 key only */
+	append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
+			  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
+
+	/* Load sector size with index 40 bytes (0x28) */
+	append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
+			   LDST_SRCDST_BYTE_CONTEXT |
+			   (0x28 << LDST_OFFSET_SHIFT));
+
+	set_jump_tgt_here(desc, key_jump_cmd);
+
+	/*
+	 * create sequence for loading the sector index
+	 * Upper 8B of IV - will be used as sector index
+	 * Lower 8B of IV - will be discarded
+	 */
+	append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
+			(0x20 << LDST_OFFSET_SHIFT));
+	append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
+
+	/* Load operation */
+	append_dec_op1(desc, cdata->algtype);
+
+	/* Perform operation */
+	ablkcipher_append_src_dst(desc);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR,
+		       "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+}
+EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_decap);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("FSL CAAM descriptor support");
+MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h
new file mode 100644
index 000000000000..95551737333a
--- /dev/null
+++ b/drivers/crypto/caam/caamalg_desc.h
@@ -0,0 +1,97 @@
+/*
+ * Shared descriptors for aead, ablkcipher algorithms
+ *
+ * Copyright 2016 NXP
+ */
+
+#ifndef _CAAMALG_DESC_H_
+#define _CAAMALG_DESC_H_
+
+/* length of descriptors text */
+#define DESC_AEAD_BASE			(4 * CAAM_CMD_SZ)
+#define DESC_AEAD_ENC_LEN		(DESC_AEAD_BASE + 11 * CAAM_CMD_SZ)
+#define DESC_AEAD_DEC_LEN		(DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
+#define DESC_AEAD_GIVENC_LEN		(DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
+
+/* Note: Nonce is counted in cdata.keylen */
+#define DESC_AEAD_CTR_RFC3686_LEN	(4 * CAAM_CMD_SZ)
+
+#define DESC_AEAD_NULL_BASE		(3 * CAAM_CMD_SZ)
+#define DESC_AEAD_NULL_ENC_LEN		(DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ)
+#define DESC_AEAD_NULL_DEC_LEN		(DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ)
+
+#define DESC_GCM_BASE			(3 * CAAM_CMD_SZ)
+#define DESC_GCM_ENC_LEN		(DESC_GCM_BASE + 16 * CAAM_CMD_SZ)
+#define DESC_GCM_DEC_LEN		(DESC_GCM_BASE + 12 * CAAM_CMD_SZ)
+
+#define DESC_RFC4106_BASE		(3 * CAAM_CMD_SZ)
+#define DESC_RFC4106_ENC_LEN		(DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
+#define DESC_RFC4106_DEC_LEN		(DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
+
+#define DESC_RFC4543_BASE		(3 * CAAM_CMD_SZ)
+#define DESC_RFC4543_ENC_LEN		(DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ)
+#define DESC_RFC4543_DEC_LEN		(DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ)
+
+#define DESC_ABLKCIPHER_BASE		(3 * CAAM_CMD_SZ)
+#define DESC_ABLKCIPHER_ENC_LEN		(DESC_ABLKCIPHER_BASE + \
+					 20 * CAAM_CMD_SZ)
+#define DESC_ABLKCIPHER_DEC_LEN		(DESC_ABLKCIPHER_BASE + \
+					 15 * CAAM_CMD_SZ)
+
+void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
+				 unsigned int icvsize);
+
+void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
+				 unsigned int icvsize);
+
+void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
+			    struct alginfo *adata, unsigned int icvsize,
+			    const bool is_rfc3686, u32 *nonce,
+			    const u32 ctx1_iv_off);
+
+void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
+			    struct alginfo *adata, unsigned int ivsize,
+			    unsigned int icvsize, const bool geniv,
+			    const bool is_rfc3686, u32 *nonce,
+			    const u32 ctx1_iv_off);
+
+void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
+			       struct alginfo *adata, unsigned int ivsize,
+			       unsigned int icvsize, const bool is_rfc3686,
+			       u32 *nonce, const u32 ctx1_iv_off);
+
+void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
+			   unsigned int icvsize);
+
+void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
+			   unsigned int icvsize);
+
+void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
+			       unsigned int icvsize);
+
+void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
+			       unsigned int icvsize);
+
+void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
+			       unsigned int icvsize);
+
+void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
+			       unsigned int icvsize);
+
+void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata,
+				  unsigned int ivsize, const bool is_rfc3686,
+				  const u32 ctx1_iv_off);
+
+void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata,
+				  unsigned int ivsize, const bool is_rfc3686,
+				  const u32 ctx1_iv_off);
+
+void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata,
+				     unsigned int ivsize, const bool is_rfc3686,
+				     const u32 ctx1_iv_off);
+
+void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata);
+
+void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata);
+
+#endif /* _CAAMALG_DESC_H_ */
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 660dc206969f..e58639ea53b1 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -72,7 +72,7 @@
 #define CAAM_MAX_HASH_DIGEST_SIZE	SHA512_DIGEST_SIZE
 
 /* length of descriptors text */
-#define DESC_AHASH_BASE			(4 * CAAM_CMD_SZ)
+#define DESC_AHASH_BASE			(3 * CAAM_CMD_SZ)
 #define DESC_AHASH_UPDATE_LEN		(6 * CAAM_CMD_SZ)
 #define DESC_AHASH_UPDATE_FIRST_LEN	(DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
 #define DESC_AHASH_FINAL_LEN		(DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
@@ -103,20 +103,15 @@ struct caam_hash_ctx {
 	u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 	u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 	u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
-	u32 sh_desc_finup[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
 	dma_addr_t sh_desc_update_dma ____cacheline_aligned;
 	dma_addr_t sh_desc_update_first_dma;
 	dma_addr_t sh_desc_fin_dma;
 	dma_addr_t sh_desc_digest_dma;
-	dma_addr_t sh_desc_finup_dma;
 	struct device *jrdev;
-	u32 alg_type;
-	u32 alg_op;
 	u8 key[CAAM_MAX_HASH_KEY_SIZE];
 	dma_addr_t key_dma;
 	int ctx_len;
-	unsigned int split_key_len;
-	unsigned int split_key_pad_len;
+	struct alginfo adata;
 };
 
 /* ahash state */
@@ -222,89 +217,54 @@ static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev,
 	return 0;
 }
 
-/* Common shared descriptor commands */
-static inline void append_key_ahash(u32 *desc, struct caam_hash_ctx *ctx)
-{
-	append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
-			  ctx->split_key_len, CLASS_2 |
-			  KEY_DEST_MDHA_SPLIT | KEY_ENC);
-}
-
-/* Append key if it has been set */
-static inline void init_sh_desc_key_ahash(u32 *desc, struct caam_hash_ctx *ctx)
-{
-	u32 *key_jump_cmd;
-
-	init_sh_desc(desc, HDR_SHARE_SERIAL);
-
-	if (ctx->split_key_len) {
-		/* Skip if already shared */
-		key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-					   JUMP_COND_SHRD);
-
-		append_key_ahash(desc, ctx);
-
-		set_jump_tgt_here(desc, key_jump_cmd);
-	}
-
-	/* Propagate errors from shared to job descriptor */
-	append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
-}
-
 /*
- * For ahash read data from seqin following state->caam_ctx,
- * and write resulting class2 context to seqout, which may be state->caam_ctx
- * or req->result
+ * For ahash update, final and finup (import_ctx = true)
+ *     import context, read and write to seqout
+ * For ahash firsts and digest (import_ctx = false)
+ *     read and write to seqout
  */
-static inline void ahash_append_load_str(u32 *desc, int digestsize)
+static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize,
+				     struct caam_hash_ctx *ctx, bool import_ctx)
 {
-	/* Calculate remaining bytes to read */
-	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
-
-	/* Read remaining bytes */
-	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 |
-			     FIFOLD_TYPE_MSG | KEY_VLF);
+	u32 op = ctx->adata.algtype;
+	u32 *skip_key_load;
 
-	/* Store class2 context bytes */
-	append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
-			 LDST_SRCDST_BYTE_CONTEXT);
-}
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
 
-/*
- * For ahash update, final and finup, import context, read and write to seqout
- */
-static inline void ahash_ctx_data_to_out(u32 *desc, u32 op, u32 state,
-					 int digestsize,
-					 struct caam_hash_ctx *ctx)
-{
-	init_sh_desc_key_ahash(desc, ctx);
+	/* Append key if it has been set; ahash update excluded */
+	if ((state != OP_ALG_AS_UPDATE) && (ctx->adata.keylen)) {
+		/* Skip key loading if already shared */
+		skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+					    JUMP_COND_SHRD);
 
-	/* Import context from software */
-	append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
-		   LDST_CLASS_2_CCB | ctx->ctx_len);
+		append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad,
+				  ctx->adata.keylen, CLASS_2 |
+				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
 
-	/* Class 2 operation */
-	append_operation(desc, op | state | OP_ALG_ENCRYPT);
+		set_jump_tgt_here(desc, skip_key_load);
 
-	/*
-	 * Load from buf and/or src and write to req->result or state->context
-	 */
-	ahash_append_load_str(desc, digestsize);
-}
+		op |= OP_ALG_AAI_HMAC_PRECOMP;
+	}
 
-/* For ahash firsts and digest, read and write to seqout */
-static inline void ahash_data_to_out(u32 *desc, u32 op, u32 state,
-				     int digestsize, struct caam_hash_ctx *ctx)
-{
-	init_sh_desc_key_ahash(desc, ctx);
+	/* If needed, import context from software */
+	if (import_ctx)
+		append_seq_load(desc, ctx->ctx_len, LDST_CLASS_2_CCB |
+				LDST_SRCDST_BYTE_CONTEXT);
 
 	/* Class 2 operation */
 	append_operation(desc, op | state | OP_ALG_ENCRYPT);
 
 	/*
 	 * Load from buf and/or src and write to req->result or state->context
+	 * Calculate remaining bytes to read
 	 */
-	ahash_append_load_str(desc, digestsize);
+	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+	/* Read remaining bytes */
+	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 |
+			     FIFOLD_TYPE_MSG | KEY_VLF);
+	/* Store class2 context bytes */
+	append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
+			 LDST_SRCDST_BYTE_CONTEXT);
 }
 
 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
@@ -312,28 +272,11 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 	int digestsize = crypto_ahash_digestsize(ahash);
 	struct device *jrdev = ctx->jrdev;
-	u32 have_key = 0;
 	u32 *desc;
 
-	if (ctx->split_key_len)
-		have_key = OP_ALG_AAI_HMAC_PRECOMP;
-
 	/* ahash_update shared descriptor */
 	desc = ctx->sh_desc_update;
-
-	init_sh_desc(desc, HDR_SHARE_SERIAL);
-
-	/* Import context from software */
-	append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
-		   LDST_CLASS_2_CCB | ctx->ctx_len);
-
-	/* Class 2 operation */
-	append_operation(desc, ctx->alg_type | OP_ALG_AS_UPDATE |
-			 OP_ALG_ENCRYPT);
-
-	/* Load data and write to result or context */
-	ahash_append_load_str(desc, ctx->ctx_len);
-
+	ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true);
 	ctx->sh_desc_update_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
 						 DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, ctx->sh_desc_update_dma)) {
@@ -348,10 +291,7 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 
 	/* ahash_update_first shared descriptor */
 	desc = ctx->sh_desc_update_first;
-
-	ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INIT,
-			  ctx->ctx_len, ctx);
-
+	ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false);
 	ctx->sh_desc_update_first_dma = dma_map_single(jrdev, desc,
 						       desc_bytes(desc),
 						       DMA_TO_DEVICE);
@@ -367,10 +307,7 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 
 	/* ahash_final shared descriptor */
 	desc = ctx->sh_desc_fin;
-
-	ahash_ctx_data_to_out(desc, have_key | ctx->alg_type,
-			      OP_ALG_AS_FINALIZE, digestsize, ctx);
-
+	ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true);
 	ctx->sh_desc_fin_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
 					      DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, ctx->sh_desc_fin_dma)) {
@@ -383,30 +320,9 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 		       desc_bytes(desc), 1);
 #endif
 
-	/* ahash_finup shared descriptor */
-	desc = ctx->sh_desc_finup;
-
-	ahash_ctx_data_to_out(desc, have_key | ctx->alg_type,
-			      OP_ALG_AS_FINALIZE, digestsize, ctx);
-
-	ctx->sh_desc_finup_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
-						DMA_TO_DEVICE);
-	if (dma_mapping_error(jrdev, ctx->sh_desc_finup_dma)) {
-		dev_err(jrdev, "unable to map shared descriptor\n");
-		return -ENOMEM;
-	}
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR, "ahash finup shdesc@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-		       desc_bytes(desc), 1);
-#endif
-
 	/* ahash_digest shared descriptor */
 	desc = ctx->sh_desc_digest;
-
-	ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INITFINAL,
-			  digestsize, ctx);
-
+	ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false);
 	ctx->sh_desc_digest_dma = dma_map_single(jrdev, desc,
 						 desc_bytes(desc),
 						 DMA_TO_DEVICE);
@@ -424,14 +340,6 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 	return 0;
 }
 
-static int gen_split_hash_key(struct caam_hash_ctx *ctx, const u8 *key_in,
-			      u32 keylen)
-{
-	return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
-			       ctx->split_key_pad_len, key_in, keylen,
-			       ctx->alg_op);
-}
-
 /* Digest hash size if it is too large */
 static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
 			   u32 *keylen, u8 *key_out, u32 digestsize)
@@ -467,7 +375,7 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
 	}
 
 	/* Job descriptor to perform unkeyed hash on key_in */
-	append_operation(desc, ctx->alg_type | OP_ALG_ENCRYPT |
+	append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
 			 OP_ALG_AS_INITFINAL);
 	append_seq_in_ptr(desc, src_dma, *keylen, 0);
 	append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
@@ -511,8 +419,6 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
 static int ahash_setkey(struct crypto_ahash *ahash,
 			const u8 *key, unsigned int keylen)
 {
-	/* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
-	static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 	struct device *jrdev = ctx->jrdev;
 	int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
@@ -537,23 +443,12 @@ static int ahash_setkey(struct crypto_ahash *ahash,
 		key = hashed_key;
 	}
 
-	/* Pick class 2 key length from algorithm submask */
-	ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
-				      OP_ALG_ALGSEL_SHIFT] * 2;
-	ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
-
-#ifdef DEBUG
-	printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
-	       ctx->split_key_len, ctx->split_key_pad_len);
-	print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
-
-	ret = gen_split_hash_key(ctx, key, keylen);
+	ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, keylen,
+			    CAAM_MAX_HASH_KEY_SIZE);
 	if (ret)
 		goto bad_free_key;
 
-	ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len,
+	ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->adata.keylen_pad,
 				      DMA_TO_DEVICE);
 	if (dma_mapping_error(jrdev, ctx->key_dma)) {
 		dev_err(jrdev, "unable to map key i/o memory\n");
@@ -563,14 +458,15 @@ static int ahash_setkey(struct crypto_ahash *ahash,
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
-		       ctx->split_key_pad_len, 1);
+		       ctx->adata.keylen_pad, 1);
 #endif
 
 	ret = ahash_set_sh_desc(ahash);
 	if (ret) {
-		dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len,
+		dma_unmap_single(jrdev, ctx->key_dma, ctx->adata.keylen_pad,
 				 DMA_TO_DEVICE);
 	}
+
  error_free_key:
 	kfree(hashed_key);
 	return ret;
@@ -639,8 +535,7 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
 	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 #endif
 
-	edesc = (struct ahash_edesc *)((char *)desc -
-		 offsetof(struct ahash_edesc, hw_desc));
+	edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 	if (err)
 		caam_jr_strstatus(jrdev, err);
 
@@ -674,8 +569,7 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
 	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 #endif
 
-	edesc = (struct ahash_edesc *)((char *)desc -
-		 offsetof(struct ahash_edesc, hw_desc));
+	edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 	if (err)
 		caam_jr_strstatus(jrdev, err);
 
@@ -709,8 +603,7 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
 	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 #endif
 
-	edesc = (struct ahash_edesc *)((char *)desc -
-		 offsetof(struct ahash_edesc, hw_desc));
+	edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 	if (err)
 		caam_jr_strstatus(jrdev, err);
 
@@ -744,8 +637,7 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
 	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 #endif
 
-	edesc = (struct ahash_edesc *)((char *)desc -
-		 offsetof(struct ahash_edesc, hw_desc));
+	edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
 	if (err)
 		caam_jr_strstatus(jrdev, err);
 
@@ -1078,7 +970,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
 
 	/* allocate space for base edesc and hw desc commands, link tables */
 	edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
-				  ctx->sh_desc_finup, ctx->sh_desc_finup_dma,
+				  ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
 				  flags);
 	if (!edesc) {
 		dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
@@ -1683,7 +1575,6 @@ struct caam_hash_template {
 	unsigned int blocksize;
 	struct ahash_alg template_ahash;
 	u32 alg_type;
-	u32 alg_op;
 };
 
 /* ahash descriptors */
@@ -1709,7 +1600,6 @@ static struct caam_hash_template driver_hash[] = {
 			},
 		},
 		.alg_type = OP_ALG_ALGSEL_SHA1,
-		.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
 	}, {
 		.name = "sha224",
 		.driver_name = "sha224-caam",
@@ -1731,7 +1621,6 @@ static struct caam_hash_template driver_hash[] = {
 			},
 		},
 		.alg_type = OP_ALG_ALGSEL_SHA224,
-		.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
 	}, {
 		.name = "sha256",
 		.driver_name = "sha256-caam",
@@ -1753,7 +1642,6 @@ static struct caam_hash_template driver_hash[] = {
 			},
 		},
 		.alg_type = OP_ALG_ALGSEL_SHA256,
-		.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
 	}, {
 		.name = "sha384",
 		.driver_name = "sha384-caam",
@@ -1775,7 +1663,6 @@ static struct caam_hash_template driver_hash[] = {
 			},
 		},
 		.alg_type = OP_ALG_ALGSEL_SHA384,
-		.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
 	}, {
 		.name = "sha512",
 		.driver_name = "sha512-caam",
@@ -1797,7 +1684,6 @@ static struct caam_hash_template driver_hash[] = {
 			},
 		},
 		.alg_type = OP_ALG_ALGSEL_SHA512,
-		.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
 	}, {
 		.name = "md5",
 		.driver_name = "md5-caam",
@@ -1819,14 +1705,12 @@ static struct caam_hash_template driver_hash[] = {
 			},
 		},
 		.alg_type = OP_ALG_ALGSEL_MD5,
-		.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
 	},
 };
 
 struct caam_hash_alg {
 	struct list_head entry;
 	int alg_type;
-	int alg_op;
 	struct ahash_alg ahash_alg;
 };
 
@@ -1859,10 +1743,10 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
 		return PTR_ERR(ctx->jrdev);
 	}
 	/* copy descriptor header template value */
-	ctx->alg_type = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
-	ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_hash->alg_op;
+	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
 
-	ctx->ctx_len = runninglen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
+	ctx->ctx_len = runninglen[(ctx->adata.algtype &
+				   OP_ALG_ALGSEL_SUBMASK) >>
 				  OP_ALG_ALGSEL_SHIFT];
 
 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
@@ -1893,10 +1777,6 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm)
 		dma_unmap_single(ctx->jrdev, ctx->sh_desc_digest_dma,
 				 desc_bytes(ctx->sh_desc_digest),
 				 DMA_TO_DEVICE);
-	if (ctx->sh_desc_finup_dma &&
-	    !dma_mapping_error(ctx->jrdev, ctx->sh_desc_finup_dma))
-		dma_unmap_single(ctx->jrdev, ctx->sh_desc_finup_dma,
-				 desc_bytes(ctx->sh_desc_finup), DMA_TO_DEVICE);
 
 	caam_jr_free(ctx->jrdev);
 }
@@ -1956,7 +1836,6 @@ caam_hash_alloc(struct caam_hash_template *template,
 	alg->cra_type = &crypto_ahash_type;
 
 	t_alg->alg_type = template->alg_type;
-	t_alg->alg_op = template->alg_op;
 
 	return t_alg;
 }
diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c
index 851015e652b8..32100c4851dd 100644
--- a/drivers/crypto/caam/caampkc.c
+++ b/drivers/crypto/caam/caampkc.c
@@ -395,7 +395,7 @@ static int caam_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key,
 				unsigned int keylen)
 {
 	struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
-	struct rsa_key raw_key = {0};
+	struct rsa_key raw_key = {NULL};
 	struct caam_rsa_key *rsa_key = &ctx->key;
 	int ret;
 
@@ -441,7 +441,7 @@ static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
 				 unsigned int keylen)
 {
 	struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
-	struct rsa_key raw_key = {0};
+	struct rsa_key raw_key = {NULL};
 	struct caam_rsa_key *rsa_key = &ctx->key;
 	int ret;
 
diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c
index 9b92af2c7241..41398da3edf4 100644
--- a/drivers/crypto/caam/caamrng.c
+++ b/drivers/crypto/caam/caamrng.c
@@ -52,7 +52,7 @@
 
 /* length of descriptors */
 #define DESC_JOB_O_LEN			(CAAM_CMD_SZ * 2 + CAAM_PTR_SZ * 2)
-#define DESC_RNG_LEN			(4 * CAAM_CMD_SZ)
+#define DESC_RNG_LEN			(3 * CAAM_CMD_SZ)
 
 /* Buffer, its dma address and lock */
 struct buf_data {
@@ -100,8 +100,7 @@ static void rng_done(struct device *jrdev, u32 *desc, u32 err, void *context)
 {
 	struct buf_data *bd;
 
-	bd = (struct buf_data *)((char *)desc -
-	      offsetof(struct buf_data, hw_desc));
+	bd = container_of(desc, struct buf_data, hw_desc[0]);
 
 	if (err)
 		caam_jr_strstatus(jrdev, err);
@@ -196,9 +195,6 @@ static inline int rng_create_sh_desc(struct caam_rng_ctx *ctx)
 
 	init_sh_desc(desc, HDR_SHARE_SERIAL);
 
-	/* Propagate errors from shared to job descriptor */
-	append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
-
 	/* Generate random bytes */
 	append_operation(desc, OP_ALG_ALGSEL_RNG | OP_TYPE_CLASS1_ALG);
 
@@ -351,7 +347,7 @@ static int __init caam_rng_init(void)
 		pr_err("Job Ring Device allocation for transform failed\n");
 		return PTR_ERR(dev);
 	}
-	rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA);
+	rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA | GFP_KERNEL);
 	if (!rng_ctx) {
 		err = -ENOMEM;
 		goto free_caam_alloc;
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index e483b78c6343..755109841cfd 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -330,8 +330,8 @@ static int caam_remove(struct platform_device *pdev)
 	clk_disable_unprepare(ctrlpriv->caam_ipg);
 	clk_disable_unprepare(ctrlpriv->caam_mem);
 	clk_disable_unprepare(ctrlpriv->caam_aclk);
-	clk_disable_unprepare(ctrlpriv->caam_emi_slow);
-
+	if (ctrlpriv->caam_emi_slow)
+		clk_disable_unprepare(ctrlpriv->caam_emi_slow);
 	return 0;
 }
 
@@ -365,11 +365,8 @@ static void kick_trng(struct platform_device *pdev, int ent_delay)
 	 */
 	val = (rd_reg32(&r4tst->rtsdctl) & RTSDCTL_ENT_DLY_MASK)
 	      >> RTSDCTL_ENT_DLY_SHIFT;
-	if (ent_delay <= val) {
-		/* put RNG4 into run mode */
-		clrsetbits_32(&r4tst->rtmctl, RTMCTL_PRGM, 0);
-		return;
-	}
+	if (ent_delay <= val)
+		goto start_rng;
 
 	val = rd_reg32(&r4tst->rtsdctl);
 	val = (val & ~RTSDCTL_ENT_DLY_MASK) |
@@ -381,15 +378,12 @@ static void kick_trng(struct platform_device *pdev, int ent_delay)
 	wr_reg32(&r4tst->rtfrqmax, RTFRQMAX_DISABLE);
 	/* read the control register */
 	val = rd_reg32(&r4tst->rtmctl);
+start_rng:
 	/*
 	 * select raw sampling in both entropy shifter
-	 * and statistical checker
+	 * and statistical checker; ; put RNG4 into run mode
 	 */
-	clrsetbits_32(&val, 0, RTMCTL_SAMP_MODE_RAW_ES_SC);
-	/* put RNG4 into run mode */
-	clrsetbits_32(&val, RTMCTL_PRGM, 0);
-	/* write back the control register */
-	wr_reg32(&r4tst->rtmctl, val);
+	clrsetbits_32(&r4tst->rtmctl, RTMCTL_PRGM, RTMCTL_SAMP_MODE_RAW_ES_SC);
 }
 
 /**
@@ -482,14 +476,16 @@ static int caam_probe(struct platform_device *pdev)
 	}
 	ctrlpriv->caam_aclk = clk;
 
-	clk = caam_drv_identify_clk(&pdev->dev, "emi_slow");
-	if (IS_ERR(clk)) {
-		ret = PTR_ERR(clk);
-		dev_err(&pdev->dev,
-			"can't identify CAAM emi_slow clk: %d\n", ret);
-		return ret;
+	if (!of_machine_is_compatible("fsl,imx6ul")) {
+		clk = caam_drv_identify_clk(&pdev->dev, "emi_slow");
+		if (IS_ERR(clk)) {
+			ret = PTR_ERR(clk);
+			dev_err(&pdev->dev,
+				"can't identify CAAM emi_slow clk: %d\n", ret);
+			return ret;
+		}
+		ctrlpriv->caam_emi_slow = clk;
 	}
-	ctrlpriv->caam_emi_slow = clk;
 
 	ret = clk_prepare_enable(ctrlpriv->caam_ipg);
 	if (ret < 0) {
@@ -510,11 +506,13 @@ static int caam_probe(struct platform_device *pdev)
 		goto disable_caam_mem;
 	}
 
-	ret = clk_prepare_enable(ctrlpriv->caam_emi_slow);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "can't enable CAAM emi slow clock: %d\n",
-			ret);
-		goto disable_caam_aclk;
+	if (ctrlpriv->caam_emi_slow) {
+		ret = clk_prepare_enable(ctrlpriv->caam_emi_slow);
+		if (ret < 0) {
+			dev_err(&pdev->dev, "can't enable CAAM emi slow clock: %d\n",
+				ret);
+			goto disable_caam_aclk;
+		}
 	}
 
 	/* Get configuration properties from device tree */
@@ -541,13 +539,13 @@ static int caam_probe(struct platform_device *pdev)
 	else
 		BLOCK_OFFSET = PG_SIZE_64K;
 
-	ctrlpriv->ctrl = (struct caam_ctrl __force *)ctrl;
-	ctrlpriv->assure = (struct caam_assurance __force *)
-			   ((uint8_t *)ctrl +
+	ctrlpriv->ctrl = (struct caam_ctrl __iomem __force *)ctrl;
+	ctrlpriv->assure = (struct caam_assurance __iomem __force *)
+			   ((__force uint8_t *)ctrl +
 			    BLOCK_OFFSET * ASSURE_BLOCK_NUMBER
 			   );
-	ctrlpriv->deco = (struct caam_deco __force *)
-			 ((uint8_t *)ctrl +
+	ctrlpriv->deco = (struct caam_deco __iomem __force *)
+			 ((__force uint8_t *)ctrl +
 			 BLOCK_OFFSET * DECO_BLOCK_NUMBER
 			 );
 
@@ -627,8 +625,8 @@ static int caam_probe(struct platform_device *pdev)
 					ring);
 				continue;
 			}
-			ctrlpriv->jr[ring] = (struct caam_job_ring __force *)
-					     ((uint8_t *)ctrl +
+			ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *)
+					     ((__force uint8_t *)ctrl +
 					     (ring + JR_BLOCK_NUMBER) *
 					      BLOCK_OFFSET
 					     );
@@ -641,8 +639,8 @@ static int caam_probe(struct platform_device *pdev)
 			!!(rd_reg32(&ctrl->perfmon.comp_parms_ms) &
 			   CTPR_MS_QI_MASK);
 	if (ctrlpriv->qi_present) {
-		ctrlpriv->qi = (struct caam_queue_if __force *)
-			       ((uint8_t *)ctrl +
+		ctrlpriv->qi = (struct caam_queue_if __iomem __force *)
+			       ((__force uint8_t *)ctrl +
 				 BLOCK_OFFSET * QI_BLOCK_NUMBER
 			       );
 		/* This is all that's required to physically enable QI */
@@ -800,7 +798,7 @@ static int caam_probe(struct platform_device *pdev)
 				    &caam_fops_u32_ro);
 
 	/* Internal covering keys (useful in non-secure mode only) */
-	ctrlpriv->ctl_kek_wrap.data = &ctrlpriv->ctrl->kek[0];
+	ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0];
 	ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
 	ctrlpriv->ctl_kek = debugfs_create_blob("kek",
 						S_IRUSR |
@@ -808,7 +806,7 @@ static int caam_probe(struct platform_device *pdev)
 						ctrlpriv->ctl,
 						&ctrlpriv->ctl_kek_wrap);
 
-	ctrlpriv->ctl_tkek_wrap.data = &ctrlpriv->ctrl->tkek[0];
+	ctrlpriv->ctl_tkek_wrap.data = (__force void *)&ctrlpriv->ctrl->tkek[0];
 	ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
 	ctrlpriv->ctl_tkek = debugfs_create_blob("tkek",
 						 S_IRUSR |
@@ -816,7 +814,7 @@ static int caam_probe(struct platform_device *pdev)
 						 ctrlpriv->ctl,
 						 &ctrlpriv->ctl_tkek_wrap);
 
-	ctrlpriv->ctl_tdsk_wrap.data = &ctrlpriv->ctrl->tdsk[0];
+	ctrlpriv->ctl_tdsk_wrap.data = (__force void *)&ctrlpriv->ctrl->tdsk[0];
 	ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32);
 	ctrlpriv->ctl_tdsk = debugfs_create_blob("tdsk",
 						 S_IRUSR |
@@ -833,7 +831,8 @@ caam_remove:
 iounmap_ctrl:
 	iounmap(ctrl);
 disable_caam_emi_slow:
-	clk_disable_unprepare(ctrlpriv->caam_emi_slow);
+	if (ctrlpriv->caam_emi_slow)
+		clk_disable_unprepare(ctrlpriv->caam_emi_slow);
 disable_caam_aclk:
 	clk_disable_unprepare(ctrlpriv->caam_aclk);
 disable_caam_mem:
diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h
index 513b6646bb36..2e6766a1573f 100644
--- a/drivers/crypto/caam/desc.h
+++ b/drivers/crypto/caam/desc.h
@@ -22,12 +22,6 @@
 #define SEC4_SG_LEN_MASK	0x3fffffff	/* Excludes EXT and FINAL */
 #define SEC4_SG_OFFSET_MASK	0x00001fff
 
-struct sec4_sg_entry {
-	u64 ptr;
-	u32 len;
-	u32 bpid_offset;
-};
-
 /* Max size of any CAAM descriptor in 32-bit words, inclusive of header */
 #define MAX_CAAM_DESCSIZE	64
 
@@ -90,8 +84,8 @@ struct sec4_sg_entry {
 #define HDR_ZRO			0x00008000
 
 /* Start Index or SharedDesc Length */
-#define HDR_START_IDX_MASK	0x3f
 #define HDR_START_IDX_SHIFT	16
+#define HDR_START_IDX_MASK	(0x3f << HDR_START_IDX_SHIFT)
 
 /* If shared descriptor header, 6-bit length */
 #define HDR_DESCLEN_SHR_MASK	0x3f
@@ -121,10 +115,10 @@ struct sec4_sg_entry {
 #define HDR_PROP_DNR		0x00000800
 
 /* JobDesc/SharedDesc share property */
-#define HDR_SD_SHARE_MASK	0x03
 #define HDR_SD_SHARE_SHIFT	8
-#define HDR_JD_SHARE_MASK	0x07
+#define HDR_SD_SHARE_MASK	(0x03 << HDR_SD_SHARE_SHIFT)
 #define HDR_JD_SHARE_SHIFT	8
+#define HDR_JD_SHARE_MASK	(0x07 << HDR_JD_SHARE_SHIFT)
 
 #define HDR_SHARE_NEVER		(0x00 << HDR_SD_SHARE_SHIFT)
 #define HDR_SHARE_WAIT		(0x01 << HDR_SD_SHARE_SHIFT)
@@ -235,7 +229,7 @@ struct sec4_sg_entry {
 #define LDST_SRCDST_WORD_DECO_MATH2	(0x0a << LDST_SRCDST_SHIFT)
 #define LDST_SRCDST_WORD_DECO_AAD_SZ	(0x0b << LDST_SRCDST_SHIFT)
 #define LDST_SRCDST_WORD_DECO_MATH3	(0x0b << LDST_SRCDST_SHIFT)
-#define LDST_SRCDST_WORD_CLASS1_ICV_SZ	(0x0c << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_CLASS1_IV_SZ	(0x0c << LDST_SRCDST_SHIFT)
 #define LDST_SRCDST_WORD_ALTDS_CLASS1	(0x0f << LDST_SRCDST_SHIFT)
 #define LDST_SRCDST_WORD_PKHA_A_SZ	(0x10 << LDST_SRCDST_SHIFT)
 #define LDST_SRCDST_WORD_PKHA_B_SZ	(0x11 << LDST_SRCDST_SHIFT)
@@ -400,7 +394,7 @@ struct sec4_sg_entry {
 #define FIFOST_TYPE_PKHA_N	 (0x08 << FIFOST_TYPE_SHIFT)
 #define FIFOST_TYPE_PKHA_A	 (0x0c << FIFOST_TYPE_SHIFT)
 #define FIFOST_TYPE_PKHA_B	 (0x0d << FIFOST_TYPE_SHIFT)
-#define FIFOST_TYPE_AF_SBOX_JKEK (0x10 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_AF_SBOX_JKEK (0x20 << FIFOST_TYPE_SHIFT)
 #define FIFOST_TYPE_AF_SBOX_TKEK (0x21 << FIFOST_TYPE_SHIFT)
 #define FIFOST_TYPE_PKHA_E_JKEK	 (0x22 << FIFOST_TYPE_SHIFT)
 #define FIFOST_TYPE_PKHA_E_TKEK	 (0x23 << FIFOST_TYPE_SHIFT)
@@ -1107,8 +1101,8 @@ struct sec4_sg_entry {
 /* For non-protocol/alg-only op commands */
 #define OP_ALG_TYPE_SHIFT	24
 #define OP_ALG_TYPE_MASK	(0x7 << OP_ALG_TYPE_SHIFT)
-#define OP_ALG_TYPE_CLASS1	2
-#define OP_ALG_TYPE_CLASS2	4
+#define OP_ALG_TYPE_CLASS1	(2 << OP_ALG_TYPE_SHIFT)
+#define OP_ALG_TYPE_CLASS2	(4 << OP_ALG_TYPE_SHIFT)
 
 #define OP_ALG_ALGSEL_SHIFT	16
 #define OP_ALG_ALGSEL_MASK	(0xff << OP_ALG_ALGSEL_SHIFT)
@@ -1249,7 +1243,7 @@ struct sec4_sg_entry {
 #define OP_ALG_PKMODE_MOD_PRIMALITY	0x00f
 
 /* PKHA mode copy-memory functions */
-#define OP_ALG_PKMODE_SRC_REG_SHIFT	13
+#define OP_ALG_PKMODE_SRC_REG_SHIFT	17
 #define OP_ALG_PKMODE_SRC_REG_MASK	(7 << OP_ALG_PKMODE_SRC_REG_SHIFT)
 #define OP_ALG_PKMODE_DST_REG_SHIFT	10
 #define OP_ALG_PKMODE_DST_REG_MASK	(7 << OP_ALG_PKMODE_DST_REG_SHIFT)
diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h
index a8cd8a78ec1f..b9c8d98ef826 100644
--- a/drivers/crypto/caam/desc_constr.h
+++ b/drivers/crypto/caam/desc_constr.h
@@ -33,38 +33,39 @@
 
 extern bool caam_little_end;
 
-static inline int desc_len(u32 *desc)
+static inline int desc_len(u32 * const desc)
 {
 	return caam32_to_cpu(*desc) & HDR_DESCLEN_MASK;
 }
 
-static inline int desc_bytes(void *desc)
+static inline int desc_bytes(void * const desc)
 {
 	return desc_len(desc) * CAAM_CMD_SZ;
 }
 
-static inline u32 *desc_end(u32 *desc)
+static inline u32 *desc_end(u32 * const desc)
 {
 	return desc + desc_len(desc);
 }
 
-static inline void *sh_desc_pdb(u32 *desc)
+static inline void *sh_desc_pdb(u32 * const desc)
 {
 	return desc + 1;
 }
 
-static inline void init_desc(u32 *desc, u32 options)
+static inline void init_desc(u32 * const desc, u32 options)
 {
 	*desc = cpu_to_caam32((options | HDR_ONE) + 1);
 }
 
-static inline void init_sh_desc(u32 *desc, u32 options)
+static inline void init_sh_desc(u32 * const desc, u32 options)
 {
 	PRINT_POS;
 	init_desc(desc, CMD_SHARED_DESC_HDR | options);
 }
 
-static inline void init_sh_desc_pdb(u32 *desc, u32 options, size_t pdb_bytes)
+static inline void init_sh_desc_pdb(u32 * const desc, u32 options,
+				    size_t pdb_bytes)
 {
 	u32 pdb_len = (pdb_bytes + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ;
 
@@ -72,19 +73,20 @@ static inline void init_sh_desc_pdb(u32 *desc, u32 options, size_t pdb_bytes)
 		     options);
 }
 
-static inline void init_job_desc(u32 *desc, u32 options)
+static inline void init_job_desc(u32 * const desc, u32 options)
 {
 	init_desc(desc, CMD_DESC_HDR | options);
 }
 
-static inline void init_job_desc_pdb(u32 *desc, u32 options, size_t pdb_bytes)
+static inline void init_job_desc_pdb(u32 * const desc, u32 options,
+				     size_t pdb_bytes)
 {
 	u32 pdb_len = (pdb_bytes + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ;
 
 	init_job_desc(desc, (((pdb_len + 1) << HDR_START_IDX_SHIFT)) | options);
 }
 
-static inline void append_ptr(u32 *desc, dma_addr_t ptr)
+static inline void append_ptr(u32 * const desc, dma_addr_t ptr)
 {
 	dma_addr_t *offset = (dma_addr_t *)desc_end(desc);
 
@@ -94,8 +96,8 @@ static inline void append_ptr(u32 *desc, dma_addr_t ptr)
 				CAAM_PTR_SZ / CAAM_CMD_SZ);
 }
 
-static inline void init_job_desc_shared(u32 *desc, dma_addr_t ptr, int len,
-					u32 options)
+static inline void init_job_desc_shared(u32 * const desc, dma_addr_t ptr,
+					int len, u32 options)
 {
 	PRINT_POS;
 	init_job_desc(desc, HDR_SHARED | options |
@@ -103,7 +105,7 @@ static inline void init_job_desc_shared(u32 *desc, dma_addr_t ptr, int len,
 	append_ptr(desc, ptr);
 }
 
-static inline void append_data(u32 *desc, void *data, int len)
+static inline void append_data(u32 * const desc, void *data, int len)
 {
 	u32 *offset = desc_end(desc);
 
@@ -114,7 +116,7 @@ static inline void append_data(u32 *desc, void *data, int len)
 				(len + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ);
 }
 
-static inline void append_cmd(u32 *desc, u32 command)
+static inline void append_cmd(u32 * const desc, u32 command)
 {
 	u32 *cmd = desc_end(desc);
 
@@ -125,7 +127,7 @@ static inline void append_cmd(u32 *desc, u32 command)
 
 #define append_u32 append_cmd
 
-static inline void append_u64(u32 *desc, u64 data)
+static inline void append_u64(u32 * const desc, u64 data)
 {
 	u32 *offset = desc_end(desc);
 
@@ -142,14 +144,14 @@ static inline void append_u64(u32 *desc, u64 data)
 }
 
 /* Write command without affecting header, and return pointer to next word */
-static inline u32 *write_cmd(u32 *desc, u32 command)
+static inline u32 *write_cmd(u32 * const desc, u32 command)
 {
 	*desc = cpu_to_caam32(command);
 
 	return desc + 1;
 }
 
-static inline void append_cmd_ptr(u32 *desc, dma_addr_t ptr, int len,
+static inline void append_cmd_ptr(u32 * const desc, dma_addr_t ptr, int len,
 				  u32 command)
 {
 	append_cmd(desc, command | len);
@@ -157,7 +159,7 @@ static inline void append_cmd_ptr(u32 *desc, dma_addr_t ptr, int len,
 }
 
 /* Write length after pointer, rather than inside command */
-static inline void append_cmd_ptr_extlen(u32 *desc, dma_addr_t ptr,
+static inline void append_cmd_ptr_extlen(u32 * const desc, dma_addr_t ptr,
 					 unsigned int len, u32 command)
 {
 	append_cmd(desc, command);
@@ -166,7 +168,7 @@ static inline void append_cmd_ptr_extlen(u32 *desc, dma_addr_t ptr,
 	append_cmd(desc, len);
 }
 
-static inline void append_cmd_data(u32 *desc, void *data, int len,
+static inline void append_cmd_data(u32 * const desc, void *data, int len,
 				   u32 command)
 {
 	append_cmd(desc, command | IMMEDIATE | len);
@@ -174,7 +176,7 @@ static inline void append_cmd_data(u32 *desc, void *data, int len,
 }
 
 #define APPEND_CMD_RET(cmd, op) \
-static inline u32 *append_##cmd(u32 *desc, u32 options) \
+static inline u32 *append_##cmd(u32 * const desc, u32 options) \
 { \
 	u32 *cmd = desc_end(desc); \
 	PRINT_POS; \
@@ -184,13 +186,13 @@ static inline u32 *append_##cmd(u32 *desc, u32 options) \
 APPEND_CMD_RET(jump, JUMP)
 APPEND_CMD_RET(move, MOVE)
 
-static inline void set_jump_tgt_here(u32 *desc, u32 *jump_cmd)
+static inline void set_jump_tgt_here(u32 * const desc, u32 *jump_cmd)
 {
 	*jump_cmd = cpu_to_caam32(caam32_to_cpu(*jump_cmd) |
 				  (desc_len(desc) - (jump_cmd - desc)));
 }
 
-static inline void set_move_tgt_here(u32 *desc, u32 *move_cmd)
+static inline void set_move_tgt_here(u32 * const desc, u32 *move_cmd)
 {
 	u32 val = caam32_to_cpu(*move_cmd);
 
@@ -200,7 +202,7 @@ static inline void set_move_tgt_here(u32 *desc, u32 *move_cmd)
 }
 
 #define APPEND_CMD(cmd, op) \
-static inline void append_##cmd(u32 *desc, u32 options) \
+static inline void append_##cmd(u32 * const desc, u32 options) \
 { \
 	PRINT_POS; \
 	append_cmd(desc, CMD_##op | options); \
@@ -208,7 +210,8 @@ static inline void append_##cmd(u32 *desc, u32 options) \
 APPEND_CMD(operation, OPERATION)
 
 #define APPEND_CMD_LEN(cmd, op) \
-static inline void append_##cmd(u32 *desc, unsigned int len, u32 options) \
+static inline void append_##cmd(u32 * const desc, unsigned int len, \
+				u32 options) \
 { \
 	PRINT_POS; \
 	append_cmd(desc, CMD_##op | len | options); \
@@ -220,8 +223,8 @@ APPEND_CMD_LEN(seq_fifo_load, SEQ_FIFO_LOAD)
 APPEND_CMD_LEN(seq_fifo_store, SEQ_FIFO_STORE)
 
 #define APPEND_CMD_PTR(cmd, op) \
-static inline void append_##cmd(u32 *desc, dma_addr_t ptr, unsigned int len, \
-				u32 options) \
+static inline void append_##cmd(u32 * const desc, dma_addr_t ptr, \
+				unsigned int len, u32 options) \
 { \
 	PRINT_POS; \
 	append_cmd_ptr(desc, ptr, len, CMD_##op | options); \
@@ -231,8 +234,8 @@ APPEND_CMD_PTR(load, LOAD)
 APPEND_CMD_PTR(fifo_load, FIFO_LOAD)
 APPEND_CMD_PTR(fifo_store, FIFO_STORE)
 
-static inline void append_store(u32 *desc, dma_addr_t ptr, unsigned int len,
-				u32 options)
+static inline void append_store(u32 * const desc, dma_addr_t ptr,
+				unsigned int len, u32 options)
 {
 	u32 cmd_src;
 
@@ -249,7 +252,8 @@ static inline void append_store(u32 *desc, dma_addr_t ptr, unsigned int len,
 }
 
 #define APPEND_SEQ_PTR_INTLEN(cmd, op) \
-static inline void append_seq_##cmd##_ptr_intlen(u32 *desc, dma_addr_t ptr, \
+static inline void append_seq_##cmd##_ptr_intlen(u32 * const desc, \
+						 dma_addr_t ptr, \
 						 unsigned int len, \
 						 u32 options) \
 { \
@@ -263,7 +267,7 @@ APPEND_SEQ_PTR_INTLEN(in, IN)
 APPEND_SEQ_PTR_INTLEN(out, OUT)
 
 #define APPEND_CMD_PTR_TO_IMM(cmd, op) \
-static inline void append_##cmd##_as_imm(u32 *desc, void *data, \
+static inline void append_##cmd##_as_imm(u32 * const desc, void *data, \
 					 unsigned int len, u32 options) \
 { \
 	PRINT_POS; \
@@ -273,7 +277,7 @@ APPEND_CMD_PTR_TO_IMM(load, LOAD);
 APPEND_CMD_PTR_TO_IMM(fifo_load, FIFO_LOAD);
 
 #define APPEND_CMD_PTR_EXTLEN(cmd, op) \
-static inline void append_##cmd##_extlen(u32 *desc, dma_addr_t ptr, \
+static inline void append_##cmd##_extlen(u32 * const desc, dma_addr_t ptr, \
 					 unsigned int len, u32 options) \
 { \
 	PRINT_POS; \
@@ -287,7 +291,7 @@ APPEND_CMD_PTR_EXTLEN(seq_out_ptr, SEQ_OUT_PTR)
  * the size of its type
  */
 #define APPEND_CMD_PTR_LEN(cmd, op, type) \
-static inline void append_##cmd(u32 *desc, dma_addr_t ptr, \
+static inline void append_##cmd(u32 * const desc, dma_addr_t ptr, \
 				type len, u32 options) \
 { \
 	PRINT_POS; \
@@ -304,7 +308,7 @@ APPEND_CMD_PTR_LEN(seq_out_ptr, SEQ_OUT_PTR, u32)
  * from length of immediate data provided, e.g., split keys
  */
 #define APPEND_CMD_PTR_TO_IMM2(cmd, op) \
-static inline void append_##cmd##_as_imm(u32 *desc, void *data, \
+static inline void append_##cmd##_as_imm(u32 * const desc, void *data, \
 					 unsigned int data_len, \
 					 unsigned int len, u32 options) \
 { \
@@ -315,7 +319,7 @@ static inline void append_##cmd##_as_imm(u32 *desc, void *data, \
 APPEND_CMD_PTR_TO_IMM2(key, KEY);
 
 #define APPEND_CMD_RAW_IMM(cmd, op, type) \
-static inline void append_##cmd##_imm_##type(u32 *desc, type immediate, \
+static inline void append_##cmd##_imm_##type(u32 * const desc, type immediate, \
 					     u32 options) \
 { \
 	PRINT_POS; \
@@ -426,3 +430,64 @@ do { \
 	APPEND_MATH_IMM_u64(LSHIFT, desc, dest, src0, src1, data)
 #define append_math_rshift_imm_u64(desc, dest, src0, src1, data) \
 	APPEND_MATH_IMM_u64(RSHIFT, desc, dest, src0, src1, data)
+
+/**
+ * struct alginfo - Container for algorithm details
+ * @algtype: algorithm selector; for valid values, see documentation of the
+ *           functions where it is used.
+ * @keylen: length of the provided algorithm key, in bytes
+ * @keylen_pad: padded length of the provided algorithm key, in bytes
+ * @key: address where algorithm key resides; virtual address if key_inline
+ *       is true, dma (bus) address if key_inline is false.
+ * @key_inline: true - key can be inlined in the descriptor; false - key is
+ *              referenced by the descriptor
+ */
+struct alginfo {
+	u32 algtype;
+	unsigned int keylen;
+	unsigned int keylen_pad;
+	union {
+		dma_addr_t key_dma;
+		void *key_virt;
+	};
+	bool key_inline;
+};
+
+/**
+ * desc_inline_query() - Provide indications on which data items can be inlined
+ *                       and which shall be referenced in a shared descriptor.
+ * @sd_base_len: Shared descriptor base length - bytes consumed by the commands,
+ *               excluding the data items to be inlined (or corresponding
+ *               pointer if an item is not inlined). Each cnstr_* function that
+ *               generates descriptors should have a define mentioning
+ *               corresponding length.
+ * @jd_len: Maximum length of the job descriptor(s) that will be used
+ *          together with the shared descriptor.
+ * @data_len: Array of lengths of the data items trying to be inlined
+ * @inl_mask: 32bit mask with bit x = 1 if data item x can be inlined, 0
+ *            otherwise.
+ * @count: Number of data items (size of @data_len array); must be <= 32
+ *
+ * Return: 0 if data can be inlined / referenced, negative value if not. If 0,
+ *         check @inl_mask for details.
+ */
+static inline int desc_inline_query(unsigned int sd_base_len,
+				    unsigned int jd_len, unsigned int *data_len,
+				    u32 *inl_mask, unsigned int count)
+{
+	int rem_bytes = (int)(CAAM_DESC_BYTES_MAX - sd_base_len - jd_len);
+	unsigned int i;
+
+	*inl_mask = 0;
+	for (i = 0; (i < count) && (rem_bytes > 0); i++) {
+		if (rem_bytes - (int)(data_len[i] +
+			(count - i - 1) * CAAM_PTR_SZ) >= 0) {
+			rem_bytes -= data_len[i];
+			*inl_mask |= (1 << i);
+		} else {
+			rem_bytes -= CAAM_PTR_SZ;
+		}
+	}
+
+	return (rem_bytes >= 0) ? 0 : -1;
+}
diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c
index 33e41ea83fcc..79a0cc70717f 100644
--- a/drivers/crypto/caam/error.c
+++ b/drivers/crypto/caam/error.c
@@ -146,10 +146,9 @@ static void report_ccb_status(struct device *jrdev, const u32 status,
 	    strlen(rng_err_id_list[err_id])) {
 		/* RNG-only error */
 		err_str = rng_err_id_list[err_id];
-	} else if (err_id < ARRAY_SIZE(err_id_list))
+	} else {
 		err_str = err_id_list[err_id];
-	else
-		snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id);
+	}
 
 	/*
 	 * CCB ICV check failures are part of normal operation life;
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index 5d4c05074a5c..e2bcacc1a921 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -41,6 +41,7 @@ struct caam_drv_private_jr {
 	struct device		*dev;
 	int ridx;
 	struct caam_job_ring __iomem *rregs;	/* JobR's register space */
+	struct tasklet_struct irqtask;
 	int irq;			/* One per queue */
 
 	/* Number of scatterlist crypt transforms active on the JobR */
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index 757c27f9953d..c8604dfadbf5 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -73,6 +73,8 @@ static int caam_jr_shutdown(struct device *dev)
 
 	ret = caam_reset_hw_jr(dev);
 
+	tasklet_kill(&jrp->irqtask);
+
 	/* Release interrupt */
 	free_irq(jrp->irq, dev);
 
@@ -128,7 +130,7 @@ static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
 
 	/*
 	 * Check the output ring for ready responses, kick
-	 * the threaded irq if jobs done.
+	 * tasklet if jobs done.
 	 */
 	irqstate = rd_reg32(&jrp->rregs->jrintstatus);
 	if (!irqstate)
@@ -150,13 +152,18 @@ static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
 	/* Have valid interrupt at this point, just ACK and trigger */
 	wr_reg32(&jrp->rregs->jrintstatus, irqstate);
 
-	return IRQ_WAKE_THREAD;
+	preempt_disable();
+	tasklet_schedule(&jrp->irqtask);
+	preempt_enable();
+
+	return IRQ_HANDLED;
 }
 
-static irqreturn_t caam_jr_threadirq(int irq, void *st_dev)
+/* Deferred service handler, run as interrupt-fired tasklet */
+static void caam_jr_dequeue(unsigned long devarg)
 {
 	int hw_idx, sw_idx, i, head, tail;
-	struct device *dev = st_dev;
+	struct device *dev = (struct device *)devarg;
 	struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 	void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
 	u32 *userdesc, userstatus;
@@ -230,8 +237,6 @@ static irqreturn_t caam_jr_threadirq(int irq, void *st_dev)
 
 	/* reenable / unmask IRQs */
 	clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
-
-	return IRQ_HANDLED;
 }
 
 /**
@@ -389,10 +394,11 @@ static int caam_jr_init(struct device *dev)
 
 	jrp = dev_get_drvdata(dev);
 
+	tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
+
 	/* Connect job ring interrupt handler. */
-	error = request_threaded_irq(jrp->irq, caam_jr_interrupt,
-				     caam_jr_threadirq, IRQF_SHARED,
-				     dev_name(dev), dev);
+	error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
+			    dev_name(dev), dev);
 	if (error) {
 		dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
 			jrp->ridx, jrp->irq);
@@ -454,6 +460,7 @@ out_free_inpring:
 out_free_irq:
 	free_irq(jrp->irq, dev);
 out_kill_deq:
+	tasklet_kill(&jrp->irqtask);
 	return error;
 }
 
@@ -489,7 +496,7 @@ static int caam_jr_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	}
 
-	jrpriv->rregs = (struct caam_job_ring __force *)ctrl;
+	jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl;
 
 	if (sizeof(dma_addr_t) == sizeof(u64))
 		if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring"))
diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c
index e1eaf4ff9762..1bb2816a9b4d 100644
--- a/drivers/crypto/caam/key_gen.c
+++ b/drivers/crypto/caam/key_gen.c
@@ -10,6 +10,36 @@
 #include "desc_constr.h"
 #include "key_gen.h"
 
+/**
+ * split_key_len - Compute MDHA split key length for a given algorithm
+ * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1,
+ *        SHA224, SHA384, SHA512.
+ *
+ * Return: MDHA split key length
+ */
+static inline u32 split_key_len(u32 hash)
+{
+	/* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
+	static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
+	u32 idx;
+
+	idx = (hash & OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT;
+
+	return (u32)(mdpadlen[idx] * 2);
+}
+
+/**
+ * split_key_pad_len - Compute MDHA split key pad length for a given algorithm
+ * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1,
+ *        SHA224, SHA384, SHA512.
+ *
+ * Return: MDHA split key pad length
+ */
+static inline u32 split_key_pad_len(u32 hash)
+{
+	return ALIGN(split_key_len(hash), 16);
+}
+
 void split_key_done(struct device *dev, u32 *desc, u32 err,
 			   void *context)
 {
@@ -41,15 +71,29 @@ Split key generation-----------------------------------------------
 [06] 0x64260028    fifostr: class2 mdsplit-jdk len=40
 			@0xffe04000
 */
-int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len,
-		  int split_key_pad_len, const u8 *key_in, u32 keylen,
-		  u32 alg_op)
+int gen_split_key(struct device *jrdev, u8 *key_out,
+		  struct alginfo * const adata, const u8 *key_in, u32 keylen,
+		  int max_keylen)
 {
 	u32 *desc;
 	struct split_key_result result;
 	dma_addr_t dma_addr_in, dma_addr_out;
 	int ret = -ENOMEM;
 
+	adata->keylen = split_key_len(adata->algtype & OP_ALG_ALGSEL_MASK);
+	adata->keylen_pad = split_key_pad_len(adata->algtype &
+					      OP_ALG_ALGSEL_MASK);
+
+#ifdef DEBUG
+	dev_err(jrdev, "split keylen %d split keylen padded %d\n",
+		adata->keylen, adata->keylen_pad);
+	print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
+#endif
+
+	if (adata->keylen_pad > max_keylen)
+		return -EINVAL;
+
 	desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
 	if (!desc) {
 		dev_err(jrdev, "unable to allocate key input memory\n");
@@ -63,7 +107,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len,
 		goto out_free;
 	}
 
-	dma_addr_out = dma_map_single(jrdev, key_out, split_key_pad_len,
+	dma_addr_out = dma_map_single(jrdev, key_out, adata->keylen_pad,
 				      DMA_FROM_DEVICE);
 	if (dma_mapping_error(jrdev, dma_addr_out)) {
 		dev_err(jrdev, "unable to map key output memory\n");
@@ -74,7 +118,9 @@ int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len,
 	append_key(desc, dma_addr_in, keylen, CLASS_2 | KEY_DEST_CLASS_REG);
 
 	/* Sets MDHA up into an HMAC-INIT */
-	append_operation(desc, alg_op | OP_ALG_DECRYPT | OP_ALG_AS_INIT);
+	append_operation(desc, (adata->algtype & OP_ALG_ALGSEL_MASK) |
+			 OP_ALG_AAI_HMAC | OP_TYPE_CLASS2_ALG | OP_ALG_DECRYPT |
+			 OP_ALG_AS_INIT);
 
 	/*
 	 * do a FIFO_LOAD of zero, this will trigger the internal key expansion
@@ -87,7 +133,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len,
 	 * FIFO_STORE with the explicit split-key content store
 	 * (0x26 output type)
 	 */
-	append_fifo_store(desc, dma_addr_out, split_key_len,
+	append_fifo_store(desc, dma_addr_out, adata->keylen,
 			  LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
 
 #ifdef DEBUG
@@ -108,11 +154,11 @@ int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len,
 #ifdef DEBUG
 		print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
 			       DUMP_PREFIX_ADDRESS, 16, 4, key_out,
-			       split_key_pad_len, 1);
+			       adata->keylen_pad, 1);
 #endif
 	}
 
-	dma_unmap_single(jrdev, dma_addr_out, split_key_pad_len,
+	dma_unmap_single(jrdev, dma_addr_out, adata->keylen_pad,
 			 DMA_FROM_DEVICE);
 out_unmap_in:
 	dma_unmap_single(jrdev, dma_addr_in, keylen, DMA_TO_DEVICE);
diff --git a/drivers/crypto/caam/key_gen.h b/drivers/crypto/caam/key_gen.h
index c5588f6d8109..4628f389eb64 100644
--- a/drivers/crypto/caam/key_gen.h
+++ b/drivers/crypto/caam/key_gen.h
@@ -12,6 +12,6 @@ struct split_key_result {
 
 void split_key_done(struct device *dev, u32 *desc, u32 err, void *context);
 
-int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len,
-		    int split_key_pad_len, const u8 *key_in, u32 keylen,
-		    u32 alg_op);
+int gen_split_key(struct device *jrdev, u8 *key_out,
+		  struct alginfo * const adata, const u8 *key_in, u32 keylen,
+		  int max_keylen);
diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h
index 41cd5a356d05..6afa20c4a013 100644
--- a/drivers/crypto/caam/sg_sw_sec4.h
+++ b/drivers/crypto/caam/sg_sw_sec4.h
@@ -7,7 +7,11 @@
 
 #include "regs.h"
 
-struct sec4_sg_entry;
+struct sec4_sg_entry {
+	u64 ptr;
+	u32 len;
+	u32 bpid_offset;
+};
 
 /*
  * convert single dma address to h/w link table format
diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c
index 8d2dbacc6161..7bc09989e18a 100644
--- a/drivers/crypto/ccp/ccp-dev-v3.c
+++ b/drivers/crypto/ccp/ccp-dev-v3.c
@@ -404,10 +404,6 @@ static int ccp_init(struct ccp_device *ccp)
 		goto e_pool;
 	}
 
-	/* Initialize the queues used to wait for KSB space and suspend */
-	init_waitqueue_head(&ccp->sb_queue);
-	init_waitqueue_head(&ccp->suspend_queue);
-
 	dev_dbg(dev, "Starting threads...\n");
 	/* Create a kthread for each queue */
 	for (i = 0; i < ccp->cmd_q_count; i++) {
diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c
index faf3cb3ddce2..e2ce8190ecc9 100644
--- a/drivers/crypto/ccp/ccp-dev-v5.c
+++ b/drivers/crypto/ccp/ccp-dev-v5.c
@@ -21,6 +21,12 @@
 
 #include "ccp-dev.h"
 
+/* Allocate the requested number of contiguous LSB slots
+ * from the LSB bitmap. Look in the private range for this
+ * queue first; failing that, check the public area.
+ * If no space is available, wait around.
+ * Return: first slot number
+ */
 static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
 {
 	struct ccp_device *ccp;
@@ -50,7 +56,7 @@ static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
 			bitmap_set(ccp->lsbmap, start, count);
 
 			mutex_unlock(&ccp->sb_mutex);
-			return start * LSB_ITEM_SIZE;
+			return start;
 		}
 
 		ccp->sb_avail = 0;
@@ -63,17 +69,18 @@ static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
 	}
 }
 
+/* Free a number of LSB slots from the bitmap, starting at
+ * the indicated starting slot number.
+ */
 static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
 			 unsigned int count)
 {
-	int lsbno = start / LSB_SIZE;
-
 	if (!start)
 		return;
 
-	if (cmd_q->lsb == lsbno) {
+	if (cmd_q->lsb == start) {
 		/* An entry from the private LSB */
-		bitmap_clear(cmd_q->lsbmap, start % LSB_SIZE, count);
+		bitmap_clear(cmd_q->lsbmap, start, count);
 	} else {
 		/* From the shared LSBs */
 		struct ccp_device *ccp = cmd_q->ccp;
@@ -396,7 +403,7 @@ static int ccp5_perform_rsa(struct ccp_op *op)
 	CCP5_CMD_PROT(&desc) = 0;
 
 	function.raw = 0;
-	CCP_RSA_SIZE(&function) = op->u.rsa.mod_size;
+	CCP_RSA_SIZE(&function) = op->u.rsa.mod_size >> 3;
 	CCP5_CMD_FUNCTION(&desc) = function.raw;
 
 	CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
@@ -411,10 +418,10 @@ static int ccp5_perform_rsa(struct ccp_op *op)
 	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
 	CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 
-	/* Key (Exponent) is in external memory */
-	CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
-	CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
-	CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
+	/* Exponent is in LSB memory */
+	CCP5_CMD_KEY_LO(&desc) = op->sb_key * LSB_ITEM_SIZE;
+	CCP5_CMD_KEY_HI(&desc) = 0;
+	CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
 
 	return ccp5_do_cmd(&desc, op->cmd_q);
 }
@@ -751,9 +758,6 @@ static int ccp5_init(struct ccp_device *ccp)
 		goto e_pool;
 	}
 
-	/* Initialize the queue used to suspend */
-	init_waitqueue_head(&ccp->suspend_queue);
-
 	dev_dbg(dev, "Loading LSB map...\n");
 	/* Copy the private LSB mask to the public registers */
 	status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c
index cafa633aae10..511ab042b5e7 100644
--- a/drivers/crypto/ccp/ccp-dev.c
+++ b/drivers/crypto/ccp/ccp-dev.c
@@ -41,7 +41,7 @@ struct ccp_tasklet_data {
 };
 
 /* Human-readable error strings */
-char *ccp_error_codes[] = {
+static char *ccp_error_codes[] = {
 	"",
 	"ERR 01: ILLEGAL_ENGINE",
 	"ERR 02: ILLEGAL_KEY_ID",
@@ -478,6 +478,10 @@ struct ccp_device *ccp_alloc_struct(struct device *dev)
 	ccp->sb_count = KSB_COUNT;
 	ccp->sb_start = 0;
 
+	/* Initialize the wait queues */
+	init_waitqueue_head(&ccp->sb_queue);
+	init_waitqueue_head(&ccp->suspend_queue);
+
 	ccp->ord = ccp_increment_unit_ordinal();
 	snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", ccp->ord);
 	snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", ccp->ord);
diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h
index da5f4a678083..830f35e6005f 100644
--- a/drivers/crypto/ccp/ccp-dev.h
+++ b/drivers/crypto/ccp/ccp-dev.h
@@ -278,7 +278,7 @@ struct ccp_cmd_queue {
 	/* Private LSB that is assigned to this queue, or -1 if none.
 	 * Bitmap for my private LSB, unused otherwise
 	 */
-	unsigned int lsb;
+	int lsb;
 	DECLARE_BITMAP(lsbmap, PLSB_MAP_SIZE);
 
 	/* Queue processing thread */
@@ -515,7 +515,6 @@ struct ccp_op {
 		struct ccp_passthru_op passthru;
 		struct ccp_ecc_op ecc;
 	} u;
-	struct ccp_mem key;
 };
 
 static inline u32 ccp_addr_lo(struct ccp_dma_info *info)
@@ -541,23 +540,23 @@ static inline u32 ccp_addr_hi(struct ccp_dma_info *info)
  * word 7: upper 16 bits of key pointer; key memory type
  */
 struct dword0 {
-	__le32 soc:1;
-	__le32 ioc:1;
-	__le32 rsvd1:1;
-	__le32 init:1;
-	__le32 eom:1;		/* AES/SHA only */
-	__le32 function:15;
-	__le32 engine:4;
-	__le32 prot:1;
-	__le32 rsvd2:7;
+	unsigned int soc:1;
+	unsigned int ioc:1;
+	unsigned int rsvd1:1;
+	unsigned int init:1;
+	unsigned int eom:1;		/* AES/SHA only */
+	unsigned int function:15;
+	unsigned int engine:4;
+	unsigned int prot:1;
+	unsigned int rsvd2:7;
 };
 
 struct dword3 {
-	__le32 src_hi:16;
-	__le32 src_mem:2;
-	__le32 lsb_cxt_id:8;
-	__le32 rsvd1:5;
-	__le32 fixed:1;
+	unsigned int  src_hi:16;
+	unsigned int  src_mem:2;
+	unsigned int  lsb_cxt_id:8;
+	unsigned int  rsvd1:5;
+	unsigned int  fixed:1;
 };
 
 union dword4 {
@@ -567,18 +566,18 @@ union dword4 {
 
 union dword5 {
 	struct {
-		__le32 dst_hi:16;
-		__le32 dst_mem:2;
-		__le32 rsvd1:13;
-		__le32 fixed:1;
+		unsigned int  dst_hi:16;
+		unsigned int  dst_mem:2;
+		unsigned int  rsvd1:13;
+		unsigned int  fixed:1;
 	} fields;
 	__le32 sha_len_hi;
 };
 
 struct dword7 {
-	__le32 key_hi:16;
-	__le32 key_mem:2;
-	__le32 rsvd1:14;
+	unsigned int  key_hi:16;
+	unsigned int  key_mem:2;
+	unsigned int  rsvd1:14;
 };
 
 struct ccp5_desc {
diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig
index 4ce67fb9a880..3e104f5aa0c2 100644
--- a/drivers/crypto/chelsio/Kconfig
+++ b/drivers/crypto/chelsio/Kconfig
@@ -4,6 +4,7 @@ config CRYPTO_DEV_CHELSIO
 	select CRYPTO_SHA1
 	select CRYPTO_SHA256
 	select CRYPTO_SHA512
+	select CRYPTO_AUTHENC
 	---help---
 	  The Chelsio Crypto Co-processor driver for T6 adapters.
 
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index 56b153805462..2ed1e24b44a8 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -54,6 +54,12 @@
 #include <crypto/algapi.h>
 #include <crypto/hash.h>
 #include <crypto/sha.h>
+#include <crypto/authenc.h>
+#include <crypto/internal/aead.h>
+#include <crypto/null.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/aead.h>
+#include <crypto/scatterwalk.h>
 #include <crypto/internal/hash.h>
 
 #include "t4fw_api.h"
@@ -62,6 +68,11 @@
 #include "chcr_algo.h"
 #include "chcr_crypto.h"
 
+static inline  struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
+{
+	return ctx->crypto_ctx->aeadctx;
+}
+
 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
 {
 	return ctx->crypto_ctx->ablkctx;
@@ -72,6 +83,16 @@ static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
 	return ctx->crypto_ctx->hmacctx;
 }
 
+static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
+{
+	return gctx->ctx->gcm;
+}
+
+static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
+{
+	return gctx->ctx->authenc;
+}
+
 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
 {
 	return ctx->dev->u_ctx;
@@ -94,12 +115,37 @@ static inline unsigned int sgl_len(unsigned int n)
 	return (3 * n) / 2 + (n & 1) + 2;
 }
 
+static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
+{
+	u8 temp[SHA512_DIGEST_SIZE];
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	int authsize = crypto_aead_authsize(tfm);
+	struct cpl_fw6_pld *fw6_pld;
+	int cmp = 0;
+
+	fw6_pld = (struct cpl_fw6_pld *)input;
+	if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
+	    (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
+		cmp = memcmp(&fw6_pld->data[2], (fw6_pld + 1), authsize);
+	} else {
+
+		sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
+				authsize, req->assoclen +
+				req->cryptlen - authsize);
+		cmp = memcmp(temp, (fw6_pld + 1), authsize);
+	}
+	if (cmp)
+		*err = -EBADMSG;
+	else
+		*err = 0;
+}
+
 /*
  *	chcr_handle_resp - Unmap the DMA buffers associated with the request
  *	@req: crypto request
  */
 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
-		     int error_status)
+			 int err)
 {
 	struct crypto_tfm *tfm = req->tfm;
 	struct chcr_context *ctx = crypto_tfm_ctx(tfm);
@@ -109,17 +155,33 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
 	unsigned int digestsize, updated_digestsize;
 
 	switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
+	case CRYPTO_ALG_TYPE_AEAD:
+		ctx_req.req.aead_req = (struct aead_request *)req;
+		ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req);
+		dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.aead_req->dst,
+			     ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE);
+		if (ctx_req.ctx.reqctx->skb) {
+			kfree_skb(ctx_req.ctx.reqctx->skb);
+			ctx_req.ctx.reqctx->skb = NULL;
+		}
+		if (ctx_req.ctx.reqctx->verify == VERIFY_SW) {
+			chcr_verify_tag(ctx_req.req.aead_req, input,
+					&err);
+			ctx_req.ctx.reqctx->verify = VERIFY_HW;
+		}
+		break;
+
 	case CRYPTO_ALG_TYPE_BLKCIPHER:
 		ctx_req.req.ablk_req = (struct ablkcipher_request *)req;
 		ctx_req.ctx.ablk_ctx =
 			ablkcipher_request_ctx(ctx_req.req.ablk_req);
-		if (!error_status) {
+		if (!err) {
 			fw6_pld = (struct cpl_fw6_pld *)input;
 			memcpy(ctx_req.req.ablk_req->info, &fw6_pld->data[2],
 			       AES_BLOCK_SIZE);
 		}
 		dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.ablk_req->dst,
-			     ABLK_CTX(ctx)->dst_nents, DMA_FROM_DEVICE);
+			     ctx_req.ctx.ablk_ctx->dst_nents, DMA_FROM_DEVICE);
 		if (ctx_req.ctx.ablk_ctx->skb) {
 			kfree_skb(ctx_req.ctx.ablk_ctx->skb);
 			ctx_req.ctx.ablk_ctx->skb = NULL;
@@ -138,8 +200,10 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
 			updated_digestsize = SHA256_DIGEST_SIZE;
 		else if (digestsize == SHA384_DIGEST_SIZE)
 			updated_digestsize = SHA512_DIGEST_SIZE;
-		if (ctx_req.ctx.ahash_ctx->skb)
+		if (ctx_req.ctx.ahash_ctx->skb) {
+			kfree_skb(ctx_req.ctx.ahash_ctx->skb);
 			ctx_req.ctx.ahash_ctx->skb = NULL;
+		}
 		if (ctx_req.ctx.ahash_ctx->result == 1) {
 			ctx_req.ctx.ahash_ctx->result = 0;
 			memcpy(ctx_req.req.ahash_req->result, input +
@@ -150,11 +214,9 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
 			       sizeof(struct cpl_fw6_pld),
 			       updated_digestsize);
 		}
-		kfree(ctx_req.ctx.ahash_ctx->dummy_payload_ptr);
-		ctx_req.ctx.ahash_ctx->dummy_payload_ptr = NULL;
 		break;
 	}
-	return 0;
+	return err;
 }
 
 /*
@@ -178,40 +240,81 @@ static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
 	return flits + sgl_len(cnt);
 }
 
-static struct shash_desc *chcr_alloc_shash(unsigned int ds)
+static inline void get_aes_decrypt_key(unsigned char *dec_key,
+				       const unsigned char *key,
+				       unsigned int keylength)
+{
+	u32 temp;
+	u32 w_ring[MAX_NK];
+	int i, j, k;
+	u8  nr, nk;
+
+	switch (keylength) {
+	case AES_KEYLENGTH_128BIT:
+		nk = KEYLENGTH_4BYTES;
+		nr = NUMBER_OF_ROUNDS_10;
+		break;
+	case AES_KEYLENGTH_192BIT:
+		nk = KEYLENGTH_6BYTES;
+		nr = NUMBER_OF_ROUNDS_12;
+		break;
+	case AES_KEYLENGTH_256BIT:
+		nk = KEYLENGTH_8BYTES;
+		nr = NUMBER_OF_ROUNDS_14;
+		break;
+	default:
+		return;
+	}
+	for (i = 0; i < nk; i++)
+		w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
+
+	i = 0;
+	temp = w_ring[nk - 1];
+	while (i + nk < (nr + 1) * 4) {
+		if (!(i % nk)) {
+			/* RotWord(temp) */
+			temp = (temp << 8) | (temp >> 24);
+			temp = aes_ks_subword(temp);
+			temp ^= round_constant[i / nk];
+		} else if (nk == 8 && (i % 4 == 0)) {
+			temp = aes_ks_subword(temp);
+		}
+		w_ring[i % nk] ^= temp;
+		temp = w_ring[i % nk];
+		i++;
+	}
+	i--;
+	for (k = 0, j = i % nk; k < nk; k++) {
+		*((u32 *)dec_key + k) = htonl(w_ring[j]);
+		j--;
+		if (j < 0)
+			j += nk;
+	}
+}
+
+static struct crypto_shash *chcr_alloc_shash(unsigned int ds)
 {
 	struct crypto_shash *base_hash = NULL;
-	struct shash_desc *desc;
 
 	switch (ds) {
 	case SHA1_DIGEST_SIZE:
-		base_hash = crypto_alloc_shash("sha1-generic", 0, 0);
+		base_hash = crypto_alloc_shash("sha1", 0, 0);
 		break;
 	case SHA224_DIGEST_SIZE:
-		base_hash = crypto_alloc_shash("sha224-generic", 0, 0);
+		base_hash = crypto_alloc_shash("sha224", 0, 0);
 		break;
 	case SHA256_DIGEST_SIZE:
-		base_hash = crypto_alloc_shash("sha256-generic", 0, 0);
+		base_hash = crypto_alloc_shash("sha256", 0, 0);
 		break;
 	case SHA384_DIGEST_SIZE:
-		base_hash = crypto_alloc_shash("sha384-generic", 0, 0);
+		base_hash = crypto_alloc_shash("sha384", 0, 0);
 		break;
 	case SHA512_DIGEST_SIZE:
-		base_hash = crypto_alloc_shash("sha512-generic", 0, 0);
+		base_hash = crypto_alloc_shash("sha512", 0, 0);
 		break;
 	}
-	if (IS_ERR(base_hash)) {
-		pr_err("Can not allocate sha-generic algo.\n");
-		return (void *)base_hash;
-	}
 
-	desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(base_hash),
-		       GFP_KERNEL);
-	if (!desc)
-		return ERR_PTR(-ENOMEM);
-	desc->tfm = base_hash;
-	desc->flags = crypto_shash_get_flags(base_hash);
-	return desc;
+	return base_hash;
 }
 
 static int chcr_compute_partial_hash(struct shash_desc *desc,
@@ -279,31 +382,18 @@ static inline int is_hmac(struct crypto_tfm *tfm)
 	struct chcr_alg_template *chcr_crypto_alg =
 		container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
 			     alg.hash);
-	if ((chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK) ==
-	    CRYPTO_ALG_SUB_TYPE_HASH_HMAC)
+	if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
 		return 1;
 	return 0;
 }
 
-static inline unsigned int ch_nents(struct scatterlist *sg,
-				    unsigned int *total_size)
-{
-	unsigned int nents;
-
-	for (nents = 0, *total_size = 0; sg; sg = sg_next(sg)) {
-		nents++;
-		*total_size += sg->length;
-	}
-	return nents;
-}
-
 static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
 			   struct scatterlist *sg,
 			   struct phys_sge_parm *sg_param)
 {
 	struct phys_sge_pairs *to;
-	unsigned int out_buf_size = sg_param->obsize;
-	unsigned int nents = sg_param->nents, i, j, tot_len = 0;
+	int out_buf_size = sg_param->obsize;
+	unsigned int nents = sg_param->nents, i, j = 0;
 
 	phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
 				    | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
@@ -321,25 +411,24 @@ static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
 				       sizeof(struct cpl_rx_phys_dsgl));
 
 	for (i = 0; nents; to++) {
-		for (j = i; (nents && (j < (8 + i))); j++, nents--) {
-			to->len[j] = htons(sg->length);
+		for (j = 0; j < 8 && nents; j++, nents--) {
+			out_buf_size -= sg_dma_len(sg);
+			to->len[j] = htons(sg_dma_len(sg));
 			to->addr[j] = cpu_to_be64(sg_dma_address(sg));
-			if (out_buf_size) {
-				if (tot_len + sg_dma_len(sg) >= out_buf_size) {
-					to->len[j] = htons(out_buf_size -
-							   tot_len);
-					return;
-				}
-				tot_len += sg_dma_len(sg);
-			}
 			sg = sg_next(sg);
 		}
 	}
+	if (out_buf_size) {
+		j--;
+		to--;
+		to->len[j] = htons(ntohs(to->len[j]) + (out_buf_size));
+	}
 }
 
-static inline unsigned
-int map_writesg_phys_cpl(struct device *dev, struct cpl_rx_phys_dsgl *phys_cpl,
-			 struct scatterlist *sg, struct phys_sge_parm *sg_param)
+static inline int map_writesg_phys_cpl(struct device *dev,
+					struct cpl_rx_phys_dsgl *phys_cpl,
+					struct scatterlist *sg,
+					struct phys_sge_parm *sg_param)
 {
 	if (!sg || !sg_param->nents)
 		return 0;
@@ -353,6 +442,14 @@ int map_writesg_phys_cpl(struct device *dev, struct cpl_rx_phys_dsgl *phys_cpl,
 	return 0;
 }
 
+static inline int get_aead_subtype(struct crypto_aead *aead)
+{
+	struct aead_alg *alg = crypto_aead_alg(aead);
+	struct chcr_alg_template *chcr_crypto_alg =
+		container_of(alg, struct chcr_alg_template, alg.aead);
+	return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
+}
+
 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
 {
 	struct crypto_alg *alg = tfm->__crt_alg;
@@ -362,8 +459,23 @@ static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
 	return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
 }
 
+static inline void write_buffer_to_skb(struct sk_buff *skb,
+					unsigned int *frags,
+					char *bfr,
+					u8 bfr_len)
+{
+	skb->len += bfr_len;
+	skb->data_len += bfr_len;
+	skb->truesize += bfr_len;
+	get_page(virt_to_page(bfr));
+	skb_fill_page_desc(skb, *frags, virt_to_page(bfr),
+			   offset_in_page(bfr), bfr_len);
+	(*frags)++;
+}
+
+
 static inline void
-write_sg_data_page_desc(struct sk_buff *skb, unsigned int *frags,
+write_sg_to_skb(struct sk_buff *skb, unsigned int *frags,
 			struct scatterlist *sg, unsigned int count)
 {
 	struct page *spage;
@@ -372,8 +484,9 @@ write_sg_data_page_desc(struct sk_buff *skb, unsigned int *frags,
 	skb->len += count;
 	skb->data_len += count;
 	skb->truesize += count;
+
 	while (count > 0) {
-		if (sg && (!(sg->length)))
+		if (!sg || (!(sg->length)))
 			break;
 		spage = sg_page(sg);
 		get_page(spage);
@@ -389,29 +502,25 @@ static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
 			       struct _key_ctx *key_ctx)
 {
 	if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
-		get_aes_decrypt_key(key_ctx->key, ablkctx->key,
-				    ablkctx->enckey_len << 3);
-		memset(key_ctx->key + ablkctx->enckey_len, 0,
-		       CHCR_AES_MAX_KEY_LEN - ablkctx->enckey_len);
+		memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
 	} else {
 		memcpy(key_ctx->key,
 		       ablkctx->key + (ablkctx->enckey_len >> 1),
 		       ablkctx->enckey_len >> 1);
-		get_aes_decrypt_key(key_ctx->key + (ablkctx->enckey_len >> 1),
-				    ablkctx->key, ablkctx->enckey_len << 2);
+		memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
+		       ablkctx->rrkey, ablkctx->enckey_len >> 1);
 	}
 	return 0;
 }
 
 static inline void create_wreq(struct chcr_context *ctx,
-			       struct fw_crypto_lookaside_wr *wreq,
+			       struct chcr_wr *chcr_req,
 			       void *req, struct sk_buff *skb,
 			       int kctx_len, int hash_sz,
-			       unsigned int phys_dsgl)
+			       int is_iv,
+			       unsigned int sc_len)
 {
 	struct uld_ctx *u_ctx = ULD_CTX(ctx);
-	struct ulp_txpkt *ulptx = (struct ulp_txpkt *)(wreq + 1);
-	struct ulptx_idata *sc_imm = (struct ulptx_idata *)(ulptx + 1);
 	int iv_loc = IV_DSGL;
 	int qid = u_ctx->lldi.rxq_ids[ctx->tx_channel_id];
 	unsigned int immdatalen = 0, nr_frags = 0;
@@ -423,27 +532,27 @@ static inline void create_wreq(struct chcr_context *ctx,
 		nr_frags = skb_shinfo(skb)->nr_frags;
 	}
 
-	wreq->op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen,
-						     (kctx_len >> 4));
-	wreq->pld_size_hash_size =
+	chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen,
+				((sizeof(chcr_req->key_ctx) + kctx_len) >> 4));
+	chcr_req->wreq.pld_size_hash_size =
 		htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) |
 		      FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
-	wreq->len16_pkd = htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(
+	chcr_req->wreq.len16_pkd =
+		htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(
 				    (calc_tx_flits_ofld(skb) * 8), 16)));
-	wreq->cookie = cpu_to_be64((uintptr_t)req);
-	wreq->rx_chid_to_rx_q_id =
+	chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
+	chcr_req->wreq.rx_chid_to_rx_q_id =
 		FILL_WR_RX_Q_ID(ctx->dev->tx_channel_id, qid,
-				(hash_sz) ? IV_NOP : iv_loc);
+				is_iv ? iv_loc : IV_NOP);
 
-	ulptx->cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id);
-	ulptx->len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8),
-					 16) - ((sizeof(*wreq)) >> 4)));
+	chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id);
+	chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8),
+					16) - ((sizeof(chcr_req->wreq)) >> 4)));
 
-	sc_imm->cmd_more = FILL_CMD_MORE(immdatalen);
-	sc_imm->len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + kctx_len +
-				  ((hash_sz) ? DUMMY_BYTES :
-				  (sizeof(struct cpl_rx_phys_dsgl) +
-				   phys_dsgl)) + immdatalen);
+	chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen);
+	chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
+				   sizeof(chcr_req->key_ctx) +
+				   kctx_len + sc_len + immdatalen);
 }
 
 /**
@@ -454,86 +563,83 @@ static inline void create_wreq(struct chcr_context *ctx,
  *	@op_type:	encryption or decryption
  */
 static struct sk_buff
-*create_cipher_wr(struct crypto_async_request *req_base,
-		  struct chcr_context *ctx, unsigned short qid,
+*create_cipher_wr(struct ablkcipher_request *req,
+		  unsigned short qid,
 		  unsigned short op_type)
 {
-	struct ablkcipher_request *req = (struct ablkcipher_request *)req_base;
 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+	struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
 	struct uld_ctx *u_ctx = ULD_CTX(ctx);
 	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
 	struct sk_buff *skb = NULL;
-	struct _key_ctx *key_ctx;
-	struct fw_crypto_lookaside_wr *wreq;
-	struct cpl_tx_sec_pdu *sec_cpl;
+	struct chcr_wr *chcr_req;
 	struct cpl_rx_phys_dsgl *phys_cpl;
-	struct chcr_blkcipher_req_ctx *req_ctx = ablkcipher_request_ctx(req);
+	struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
 	struct phys_sge_parm sg_param;
-	unsigned int frags = 0, transhdr_len, phys_dsgl, dst_bufsize = 0;
+	unsigned int frags = 0, transhdr_len, phys_dsgl;
 	unsigned int ivsize = crypto_ablkcipher_ivsize(tfm), kctx_len;
+	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
+			GFP_ATOMIC;
 
 	if (!req->info)
 		return ERR_PTR(-EINVAL);
-	ablkctx->dst_nents = ch_nents(req->dst, &dst_bufsize);
-	ablkctx->enc = op_type;
-
+	reqctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes);
+	if (reqctx->dst_nents <= 0) {
+		pr_err("AES:Invalid Destination sg lists\n");
+		return ERR_PTR(-EINVAL);
+	}
 	if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
-	    (req->nbytes <= 0) || (req->nbytes % AES_BLOCK_SIZE))
+	    (req->nbytes <= 0) || (req->nbytes % AES_BLOCK_SIZE)) {
+		pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
+		       ablkctx->enckey_len, req->nbytes, ivsize);
 		return ERR_PTR(-EINVAL);
+	}
 
-	phys_dsgl = get_space_for_phys_dsgl(ablkctx->dst_nents);
+	phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents);
 
-	kctx_len = sizeof(*key_ctx) +
-		(DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
+	kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
 	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
-	skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)),
-			GFP_ATOMIC);
+	skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
 	if (!skb)
 		return ERR_PTR(-ENOMEM);
 	skb_reserve(skb, sizeof(struct sge_opaque_hdr));
-	wreq = (struct fw_crypto_lookaside_wr *)__skb_put(skb, transhdr_len);
-
-	sec_cpl = (struct cpl_tx_sec_pdu *)((u8 *)wreq + SEC_CPL_OFFSET);
-	sec_cpl->op_ivinsrtofst =
-		FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 1, 1);
-
-	sec_cpl->pldlen = htonl(ivsize + req->nbytes);
-	sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(0, 0,
-								ivsize + 1, 0);
-
-	sec_cpl->cipherstop_lo_authinsert =  FILL_SEC_CPL_AUTHINSERT(0, 0,
-								     0, 0);
-	sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 0,
+	chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
+	memset(chcr_req, 0, transhdr_len);
+	chcr_req->sec_cpl.op_ivinsrtofst =
+		FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 1);
+
+	chcr_req->sec_cpl.pldlen = htonl(ivsize + req->nbytes);
+	chcr_req->sec_cpl.aadstart_cipherstop_hi =
+			FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0);
+
+	chcr_req->sec_cpl.cipherstop_lo_authinsert =
+			FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
+	chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 0,
 							 ablkctx->ciph_mode,
-							 0, 0, ivsize >> 1, 1);
-	sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
+							 0, 0, ivsize >> 1);
+	chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
 							  0, 1, phys_dsgl);
 
-	key_ctx = (struct _key_ctx *)((u8 *)sec_cpl + sizeof(*sec_cpl));
-	key_ctx->ctx_hdr = ablkctx->key_ctx_hdr;
+	chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
 	if (op_type == CHCR_DECRYPT_OP) {
-		if (generate_copy_rrkey(ablkctx, key_ctx))
-			goto map_fail1;
+		generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
 	} else {
 		if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
-			memcpy(key_ctx->key, ablkctx->key, ablkctx->enckey_len);
+			memcpy(chcr_req->key_ctx.key, ablkctx->key,
+			       ablkctx->enckey_len);
 		} else {
-			memcpy(key_ctx->key, ablkctx->key +
+			memcpy(chcr_req->key_ctx.key, ablkctx->key +
 			       (ablkctx->enckey_len >> 1),
 			       ablkctx->enckey_len >> 1);
-			memcpy(key_ctx->key +
+			memcpy(chcr_req->key_ctx.key +
 			       (ablkctx->enckey_len >> 1),
 			       ablkctx->key,
 			       ablkctx->enckey_len >> 1);
 		}
 	}
-	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)key_ctx + kctx_len);
-
-	memcpy(ablkctx->iv, req->info, ivsize);
-	sg_init_table(&ablkctx->iv_sg, 1);
-	sg_set_buf(&ablkctx->iv_sg, ablkctx->iv, ivsize);
-	sg_param.nents = ablkctx->dst_nents;
-	sg_param.obsize = dst_bufsize;
+	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
+	sg_param.nents = reqctx->dst_nents;
+	sg_param.obsize = req->nbytes;
 	sg_param.qid = qid;
 	sg_param.align = 1;
 	if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, req->dst,
@@ -541,10 +647,12 @@ static struct sk_buff
 		goto map_fail1;
 
 	skb_set_transport_header(skb, transhdr_len);
-	write_sg_data_page_desc(skb, &frags, &ablkctx->iv_sg, ivsize);
-	write_sg_data_page_desc(skb, &frags, req->src, req->nbytes);
-	create_wreq(ctx, wreq, req, skb, kctx_len, 0, phys_dsgl);
-	req_ctx->skb = skb;
+	memcpy(reqctx->iv, req->info, ivsize);
+	write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
+	write_sg_to_skb(skb, &frags, req->src, req->nbytes);
+	create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1,
+			sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl);
+	reqctx->skb = skb;
 	skb_get(skb);
 	return skb;
 map_fail1:
@@ -557,15 +665,9 @@ static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 {
 	struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
 	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
-	struct ablkcipher_alg *alg = crypto_ablkcipher_alg(tfm);
 	unsigned int ck_size, context_size;
 	u16 alignment = 0;
 
-	if ((keylen < alg->min_keysize) || (keylen > alg->max_keysize))
-		goto badkey_err;
-
-	memcpy(ablkctx->key, key, keylen);
-	ablkctx->enckey_len = keylen;
 	if (keylen == AES_KEYSIZE_128) {
 		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
 	} else if (keylen == AES_KEYSIZE_192) {
@@ -576,7 +678,9 @@ static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 	} else {
 		goto badkey_err;
 	}
-
+	memcpy(ablkctx->key, key, keylen);
+	ablkctx->enckey_len = keylen;
+	get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
 	context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
 			keylen + alignment) >> 4;
 
@@ -612,7 +716,6 @@ static int chcr_aes_encrypt(struct ablkcipher_request *req)
 {
 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 	struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
-	struct crypto_async_request *req_base = &req->base;
 	struct uld_ctx *u_ctx = ULD_CTX(ctx);
 	struct sk_buff *skb;
 
@@ -622,8 +725,7 @@ static int chcr_aes_encrypt(struct ablkcipher_request *req)
 			return -EBUSY;
 	}
 
-	skb = create_cipher_wr(req_base, ctx,
-			       u_ctx->lldi.rxq_ids[ctx->tx_channel_id],
+	skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->tx_channel_id],
 			       CHCR_ENCRYPT_OP);
 	if (IS_ERR(skb)) {
 		pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
@@ -639,7 +741,6 @@ static int chcr_aes_decrypt(struct ablkcipher_request *req)
 {
 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 	struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
-	struct crypto_async_request *req_base = &req->base;
 	struct uld_ctx *u_ctx = ULD_CTX(ctx);
 	struct sk_buff *skb;
 
@@ -649,7 +750,7 @@ static int chcr_aes_decrypt(struct ablkcipher_request *req)
 			return -EBUSY;
 	}
 
-	skb = create_cipher_wr(req_base, ctx, u_ctx->lldi.rxq_ids[0],
+	skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[0],
 			       CHCR_DECRYPT_OP);
 	if (IS_ERR(skb)) {
 		pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
@@ -729,50 +830,33 @@ static int get_alg_config(struct algo_param *params,
 	return 0;
 }
 
-static inline int
-write_buffer_data_page_desc(struct chcr_ahash_req_ctx *req_ctx,
-			    struct sk_buff *skb, unsigned int *frags, char *bfr,
-			    u8 bfr_len)
+static inline void chcr_free_shash(struct crypto_shash *base_hash)
 {
-	void *page_ptr = NULL;
-
-	skb->len += bfr_len;
-	skb->data_len += bfr_len;
-	skb->truesize += bfr_len;
-	page_ptr = kmalloc(CHCR_HASH_MAX_BLOCK_SIZE_128, GFP_ATOMIC | GFP_DMA);
-	if (!page_ptr)
-		return -ENOMEM;
-	get_page(virt_to_page(page_ptr));
-	req_ctx->dummy_payload_ptr = page_ptr;
-	memcpy(page_ptr, bfr, bfr_len);
-	skb_fill_page_desc(skb, *frags, virt_to_page(page_ptr),
-			   offset_in_page(page_ptr), bfr_len);
-	(*frags)++;
-	return 0;
+		crypto_free_shash(base_hash);
 }
 
 /**
- *	create_final_hash_wr - Create hash work request
+ *	create_hash_wr - Create hash work request
  *	@req - Cipher req base
  */
-static struct sk_buff *create_final_hash_wr(struct ahash_request *req,
-					    struct hash_wr_param *param)
+static struct sk_buff *create_hash_wr(struct ahash_request *req,
+				      struct hash_wr_param *param)
 {
 	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
 	struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
 	struct sk_buff *skb = NULL;
-	struct _key_ctx *key_ctx;
-	struct fw_crypto_lookaside_wr *wreq;
-	struct cpl_tx_sec_pdu *sec_cpl;
+	struct chcr_wr *chcr_req;
 	unsigned int frags = 0, transhdr_len, iopad_alignment = 0;
 	unsigned int digestsize = crypto_ahash_digestsize(tfm);
-	unsigned int kctx_len = sizeof(*key_ctx);
+	unsigned int kctx_len = 0;
 	u8 hash_size_in_response = 0;
+	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
+		GFP_ATOMIC;
 
 	iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
-	kctx_len += param->alg_prm.result_size + iopad_alignment;
+	kctx_len = param->alg_prm.result_size + iopad_alignment;
 	if (param->opad_needed)
 		kctx_len += param->alg_prm.result_size + iopad_alignment;
 
@@ -781,54 +865,54 @@ static struct sk_buff *create_final_hash_wr(struct ahash_request *req,
 	else
 		hash_size_in_response = param->alg_prm.result_size;
 	transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
-	skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)),
-			GFP_ATOMIC);
+	skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
 	if (!skb)
 		return skb;
 
 	skb_reserve(skb, sizeof(struct sge_opaque_hdr));
-	wreq = (struct fw_crypto_lookaside_wr *)__skb_put(skb, transhdr_len);
-	memset(wreq, 0, transhdr_len);
+	chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
+	memset(chcr_req, 0, transhdr_len);
 
-	sec_cpl = (struct cpl_tx_sec_pdu *)((u8 *)wreq + SEC_CPL_OFFSET);
-	sec_cpl->op_ivinsrtofst =
-		FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 0, 0);
-	sec_cpl->pldlen = htonl(param->bfr_len + param->sg_len);
+	chcr_req->sec_cpl.op_ivinsrtofst =
+		FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 0);
+	chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
 
-	sec_cpl->aadstart_cipherstop_hi =
+	chcr_req->sec_cpl.aadstart_cipherstop_hi =
 		FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
-	sec_cpl->cipherstop_lo_authinsert =
+	chcr_req->sec_cpl.cipherstop_lo_authinsert =
 		FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
-	sec_cpl->seqno_numivs =
+	chcr_req->sec_cpl.seqno_numivs =
 		FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
-					 param->opad_needed, 0, 0);
+					 param->opad_needed, 0);
 
-	sec_cpl->ivgen_hdrlen =
+	chcr_req->sec_cpl.ivgen_hdrlen =
 		FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
 
-	key_ctx = (struct _key_ctx *)((u8 *)sec_cpl + sizeof(*sec_cpl));
-	memcpy(key_ctx->key, req_ctx->partial_hash, param->alg_prm.result_size);
+	memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
+	       param->alg_prm.result_size);
 
 	if (param->opad_needed)
-		memcpy(key_ctx->key + ((param->alg_prm.result_size <= 32) ? 32 :
-				       CHCR_HASH_MAX_DIGEST_SIZE),
+		memcpy(chcr_req->key_ctx.key +
+		       ((param->alg_prm.result_size <= 32) ? 32 :
+			CHCR_HASH_MAX_DIGEST_SIZE),
 		       hmacctx->opad, param->alg_prm.result_size);
 
-	key_ctx->ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
+	chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
 					    param->alg_prm.mk_size, 0,
 					    param->opad_needed,
-					    (kctx_len >> 4));
-	sec_cpl->scmd1 = cpu_to_be64((u64)param->scmd1);
+					    ((kctx_len +
+					     sizeof(chcr_req->key_ctx)) >> 4));
+	chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
 
 	skb_set_transport_header(skb, transhdr_len);
 	if (param->bfr_len != 0)
-		write_buffer_data_page_desc(req_ctx, skb, &frags, req_ctx->bfr,
-					    param->bfr_len);
+		write_buffer_to_skb(skb, &frags, req_ctx->reqbfr,
+				    param->bfr_len);
 	if (param->sg_len != 0)
-		write_sg_data_page_desc(skb, &frags, req->src, param->sg_len);
+		write_sg_to_skb(skb, &frags, req->src, param->sg_len);
 
-	create_wreq(ctx, wreq, req, skb, kctx_len, hash_size_in_response,
-		    0);
+	create_wreq(ctx, chcr_req, req, skb, kctx_len, hash_size_in_response, 0,
+			DUMMY_BYTES);
 	req_ctx->skb = skb;
 	skb_get(skb);
 	return skb;
@@ -854,34 +938,40 @@ static int chcr_ahash_update(struct ahash_request *req)
 			return -EBUSY;
 	}
 
-	if (nbytes + req_ctx->bfr_len >= bs) {
-		remainder = (nbytes + req_ctx->bfr_len) % bs;
-		nbytes = nbytes + req_ctx->bfr_len - remainder;
+	if (nbytes + req_ctx->reqlen >= bs) {
+		remainder = (nbytes + req_ctx->reqlen) % bs;
+		nbytes = nbytes + req_ctx->reqlen - remainder;
 	} else {
-		sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->bfr +
-				   req_ctx->bfr_len, nbytes, 0);
-		req_ctx->bfr_len += nbytes;
+		sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
+				   + req_ctx->reqlen, nbytes, 0);
+		req_ctx->reqlen += nbytes;
 		return 0;
 	}
 
 	params.opad_needed = 0;
 	params.more = 1;
 	params.last = 0;
-	params.sg_len = nbytes - req_ctx->bfr_len;
-	params.bfr_len = req_ctx->bfr_len;
+	params.sg_len = nbytes - req_ctx->reqlen;
+	params.bfr_len = req_ctx->reqlen;
 	params.scmd1 = 0;
 	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
 	req_ctx->result = 0;
 	req_ctx->data_len += params.sg_len + params.bfr_len;
-	skb = create_final_hash_wr(req, &params);
+	skb = create_hash_wr(req, &params);
 	if (!skb)
 		return -ENOMEM;
 
-	req_ctx->bfr_len = remainder;
-	if (remainder)
+	if (remainder) {
+		u8 *temp;
+		/* Swap buffers */
+		temp = req_ctx->reqbfr;
+		req_ctx->reqbfr = req_ctx->skbfr;
+		req_ctx->skbfr = temp;
 		sg_pcopy_to_buffer(req->src, sg_nents(req->src),
-				   req_ctx->bfr, remainder, req->nbytes -
+				   req_ctx->reqbfr, remainder, req->nbytes -
 				   remainder);
+	}
+	req_ctx->reqlen = remainder;
 	skb->dev = u_ctx->lldi.ports[0];
 	set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
 	chcr_send_wr(skb);
@@ -917,10 +1007,10 @@ static int chcr_ahash_final(struct ahash_request *req)
 	params.sg_len = 0;
 	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
 	req_ctx->result = 1;
-	params.bfr_len = req_ctx->bfr_len;
+	params.bfr_len = req_ctx->reqlen;
 	req_ctx->data_len += params.bfr_len + params.sg_len;
-	if (req_ctx->bfr && (req_ctx->bfr_len == 0)) {
-		create_last_hash_block(req_ctx->bfr, bs, req_ctx->data_len);
+	if (req_ctx->reqlen == 0) {
+		create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
 		params.last = 0;
 		params.more = 1;
 		params.scmd1 = 0;
@@ -931,7 +1021,10 @@ static int chcr_ahash_final(struct ahash_request *req)
 		params.last = 1;
 		params.more = 0;
 	}
-	skb = create_final_hash_wr(req, &params);
+	skb = create_hash_wr(req, &params);
+	if (!skb)
+		return -ENOMEM;
+
 	skb->dev = u_ctx->lldi.ports[0];
 	set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
 	chcr_send_wr(skb);
@@ -963,12 +1056,12 @@ static int chcr_ahash_finup(struct ahash_request *req)
 		params.opad_needed = 0;
 
 	params.sg_len = req->nbytes;
-	params.bfr_len = req_ctx->bfr_len;
+	params.bfr_len = req_ctx->reqlen;
 	get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
 	req_ctx->data_len += params.bfr_len + params.sg_len;
 	req_ctx->result = 1;
-	if (req_ctx->bfr && (req_ctx->bfr_len + req->nbytes) == 0) {
-		create_last_hash_block(req_ctx->bfr, bs, req_ctx->data_len);
+	if ((req_ctx->reqlen + req->nbytes) == 0) {
+		create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
 		params.last = 0;
 		params.more = 1;
 		params.scmd1 = 0;
@@ -979,9 +1072,10 @@ static int chcr_ahash_finup(struct ahash_request *req)
 		params.more = 0;
 	}
 
-	skb = create_final_hash_wr(req, &params);
+	skb = create_hash_wr(req, &params);
 	if (!skb)
 		return -ENOMEM;
+
 	skb->dev = u_ctx->lldi.ports[0];
 	set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
 	chcr_send_wr(skb);
@@ -1023,13 +1117,13 @@ static int chcr_ahash_digest(struct ahash_request *req)
 	req_ctx->result = 1;
 	req_ctx->data_len += params.bfr_len + params.sg_len;
 
-	if (req_ctx->bfr && req->nbytes == 0) {
-		create_last_hash_block(req_ctx->bfr, bs, 0);
+	if (req->nbytes == 0) {
+		create_last_hash_block(req_ctx->reqbfr, bs, 0);
 		params.more = 1;
 		params.bfr_len = bs;
 	}
 
-	skb = create_final_hash_wr(req, &params);
+	skb = create_hash_wr(req, &params);
 	if (!skb)
 		return -ENOMEM;
 
@@ -1044,12 +1138,12 @@ static int chcr_ahash_export(struct ahash_request *areq, void *out)
 	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
 	struct chcr_ahash_req_ctx *state = out;
 
-	state->bfr_len = req_ctx->bfr_len;
+	state->reqlen = req_ctx->reqlen;
 	state->data_len = req_ctx->data_len;
-	memcpy(state->bfr, req_ctx->bfr, CHCR_HASH_MAX_BLOCK_SIZE_128);
+	memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
 	memcpy(state->partial_hash, req_ctx->partial_hash,
 	       CHCR_HASH_MAX_DIGEST_SIZE);
-	return 0;
+		return 0;
 }
 
 static int chcr_ahash_import(struct ahash_request *areq, const void *in)
@@ -1057,10 +1151,11 @@ static int chcr_ahash_import(struct ahash_request *areq, const void *in)
 	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
 	struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
 
-	req_ctx->bfr_len = state->bfr_len;
+	req_ctx->reqlen = state->reqlen;
 	req_ctx->data_len = state->data_len;
-	req_ctx->dummy_payload_ptr = NULL;
-	memcpy(req_ctx->bfr, state->bfr, CHCR_HASH_MAX_BLOCK_SIZE_128);
+	req_ctx->reqbfr = req_ctx->bfr1;
+	req_ctx->skbfr = req_ctx->bfr2;
+	memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
 	memcpy(req_ctx->partial_hash, state->partial_hash,
 	       CHCR_HASH_MAX_DIGEST_SIZE);
 	return 0;
@@ -1075,15 +1170,16 @@ static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
 	unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
 	unsigned int i, err = 0, updated_digestsize;
 
-	/*
-	 * use the key to calculate the ipad and opad. ipad will sent with the
+	SHASH_DESC_ON_STACK(shash, hmacctx->base_hash);
+
+	/* use the key to calculate the ipad and opad. ipad will sent with the
 	 * first request's data. opad will be sent with the final hash result
 	 * ipad in hmacctx->ipad and opad in hmacctx->opad location
 	 */
-	if (!hmacctx->desc)
-		return -EINVAL;
+	shash->tfm = hmacctx->base_hash;
+	shash->flags = crypto_shash_get_flags(hmacctx->base_hash);
 	if (keylen > bs) {
-		err = crypto_shash_digest(hmacctx->desc, key, keylen,
+		err = crypto_shash_digest(shash, key, keylen,
 					  hmacctx->ipad);
 		if (err)
 			goto out;
@@ -1104,13 +1200,13 @@ static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
 		updated_digestsize = SHA256_DIGEST_SIZE;
 	else if (digestsize == SHA384_DIGEST_SIZE)
 		updated_digestsize = SHA512_DIGEST_SIZE;
-	err = chcr_compute_partial_hash(hmacctx->desc, hmacctx->ipad,
+	err = chcr_compute_partial_hash(shash, hmacctx->ipad,
 					hmacctx->ipad, digestsize);
 	if (err)
 		goto out;
 	chcr_change_order(hmacctx->ipad, updated_digestsize);
 
-	err = chcr_compute_partial_hash(hmacctx->desc, hmacctx->opad,
+	err = chcr_compute_partial_hash(shash, hmacctx->opad,
 					hmacctx->opad, digestsize);
 	if (err)
 		goto out;
@@ -1124,28 +1220,29 @@ static int chcr_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 {
 	struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
 	struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
-	int status = 0;
 	unsigned short context_size = 0;
 
-	if ((key_len == (AES_KEYSIZE_128 << 1)) ||
-	    (key_len == (AES_KEYSIZE_256 << 1))) {
-		memcpy(ablkctx->key, key, key_len);
-		ablkctx->enckey_len = key_len;
-		context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
-		ablkctx->key_ctx_hdr =
-			FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
-					 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
-					 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
-					 CHCR_KEYCTX_NO_KEY, 1,
-					 0, context_size);
-		ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
-	} else {
+	if ((key_len != (AES_KEYSIZE_128 << 1)) &&
+	    (key_len != (AES_KEYSIZE_256 << 1))) {
 		crypto_tfm_set_flags((struct crypto_tfm *)tfm,
 				     CRYPTO_TFM_RES_BAD_KEY_LEN);
 		ablkctx->enckey_len = 0;
-		status = -EINVAL;
+		return -EINVAL;
+
 	}
-	return status;
+
+	memcpy(ablkctx->key, key, key_len);
+	ablkctx->enckey_len = key_len;
+	get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
+	context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
+	ablkctx->key_ctx_hdr =
+		FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
+				 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
+				 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
+				 CHCR_KEYCTX_NO_KEY, 1,
+				 0, context_size);
+	ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
+	return 0;
 }
 
 static int chcr_sha_init(struct ahash_request *areq)
@@ -1155,8 +1252,9 @@ static int chcr_sha_init(struct ahash_request *areq)
 	int digestsize =  crypto_ahash_digestsize(tfm);
 
 	req_ctx->data_len = 0;
-	req_ctx->dummy_payload_ptr = NULL;
-	req_ctx->bfr_len = 0;
+	req_ctx->reqlen = 0;
+	req_ctx->reqbfr = req_ctx->bfr1;
+	req_ctx->skbfr = req_ctx->bfr2;
 	req_ctx->skb = NULL;
 	req_ctx->result = 0;
 	copy_hash_init_values(req_ctx->partial_hash, digestsize);
@@ -1204,29 +1302,1184 @@ static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
 
 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 				 sizeof(struct chcr_ahash_req_ctx));
-	hmacctx->desc = chcr_alloc_shash(digestsize);
-	if (IS_ERR(hmacctx->desc))
-		return PTR_ERR(hmacctx->desc);
+	hmacctx->base_hash = chcr_alloc_shash(digestsize);
+	if (IS_ERR(hmacctx->base_hash))
+		return PTR_ERR(hmacctx->base_hash);
 	return chcr_device_init(crypto_tfm_ctx(tfm));
 }
 
-static void chcr_free_shash(struct shash_desc *desc)
-{
-	crypto_free_shash(desc->tfm);
-	kfree(desc);
-}
-
 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
 {
 	struct chcr_context *ctx = crypto_tfm_ctx(tfm);
 	struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
 
-	if (hmacctx->desc) {
-		chcr_free_shash(hmacctx->desc);
-		hmacctx->desc = NULL;
+	if (hmacctx->base_hash) {
+		chcr_free_shash(hmacctx->base_hash);
+		hmacctx->base_hash = NULL;
+	}
+}
+
+static int chcr_copy_assoc(struct aead_request *req,
+				struct chcr_aead_ctx *ctx)
+{
+	SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
+
+	skcipher_request_set_tfm(skreq, ctx->null);
+	skcipher_request_set_callback(skreq, aead_request_flags(req),
+			NULL, NULL);
+	skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen,
+			NULL);
+
+	return crypto_skcipher_encrypt(skreq);
+}
+
+static unsigned char get_hmac(unsigned int authsize)
+{
+	switch (authsize) {
+	case ICV_8:
+		return CHCR_SCMD_HMAC_CTRL_PL1;
+	case ICV_10:
+		return CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
+	case ICV_12:
+		return CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
+	}
+	return CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
+}
+
+
+static struct sk_buff *create_authenc_wr(struct aead_request *req,
+					 unsigned short qid,
+					 int size,
+					 unsigned short op_type)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct chcr_context *ctx = crypto_aead_ctx(tfm);
+	struct uld_ctx *u_ctx = ULD_CTX(ctx);
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+	struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
+	struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+	struct sk_buff *skb = NULL;
+	struct chcr_wr *chcr_req;
+	struct cpl_rx_phys_dsgl *phys_cpl;
+	struct phys_sge_parm sg_param;
+	struct scatterlist *src, *dst;
+	struct scatterlist src_sg[2], dst_sg[2];
+	unsigned int frags = 0, transhdr_len;
+	unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0;
+	unsigned int   kctx_len = 0;
+	unsigned short stop_offset = 0;
+	unsigned int  assoclen = req->assoclen;
+	unsigned int  authsize = crypto_aead_authsize(tfm);
+	int err = 0;
+	int null = 0;
+	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
+		GFP_ATOMIC;
+
+	if (aeadctx->enckey_len == 0 || (req->cryptlen == 0))
+		goto err;
+
+	if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
+		goto err;
+
+	if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0)
+		goto err;
+	src = scatterwalk_ffwd(src_sg, req->src, req->assoclen);
+	dst = src;
+	if (req->src != req->dst) {
+		err = chcr_copy_assoc(req, aeadctx);
+		if (err)
+			return ERR_PTR(err);
+		dst = scatterwalk_ffwd(dst_sg, req->dst, req->assoclen);
+	}
+	if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) {
+		null = 1;
+		assoclen = 0;
+	}
+	reqctx->dst_nents = sg_nents_for_len(dst, req->cryptlen +
+					     (op_type ? -authsize : authsize));
+	if (reqctx->dst_nents <= 0) {
+		pr_err("AUTHENC:Invalid Destination sg entries\n");
+		goto err;
+	}
+	dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
+	kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4)
+		- sizeof(chcr_req->key_ctx);
+	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
+	skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
+	if (!skb)
+		goto err;
+
+	/* LLD is going to write the sge hdr. */
+	skb_reserve(skb, sizeof(struct sge_opaque_hdr));
+
+	/* Write WR */
+	chcr_req = (struct chcr_wr *) __skb_put(skb, transhdr_len);
+	memset(chcr_req, 0, transhdr_len);
+
+	stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
+
+	/*
+	 * Input order	is AAD,IV and Payload. where IV should be included as
+	 * the part of authdata. All other fields should be filled according
+	 * to the hardware spec
+	 */
+	chcr_req->sec_cpl.op_ivinsrtofst =
+		FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2,
+				       (ivsize ? (assoclen + 1) : 0));
+	chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen);
+	chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
+					assoclen ? 1 : 0, assoclen,
+					assoclen + ivsize + 1,
+					(stop_offset & 0x1F0) >> 4);
+	chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
+					stop_offset & 0xF,
+					null ? 0 : assoclen + ivsize + 1,
+					stop_offset, stop_offset);
+	chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
+					(op_type == CHCR_ENCRYPT_OP) ? 1 : 0,
+					CHCR_SCMD_CIPHER_MODE_AES_CBC,
+					actx->auth_mode, aeadctx->hmac_ctrl,
+					ivsize >> 1);
+	chcr_req->sec_cpl.ivgen_hdrlen =  FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
+					 0, 1, dst_size);
+
+	chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
+	if (op_type == CHCR_ENCRYPT_OP)
+		memcpy(chcr_req->key_ctx.key, aeadctx->key,
+		       aeadctx->enckey_len);
+	else
+		memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
+		       aeadctx->enckey_len);
+
+	memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) <<
+					4), actx->h_iopad, kctx_len -
+				(DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4));
+
+	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
+	sg_param.nents = reqctx->dst_nents;
+	sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
+	sg_param.qid = qid;
+	sg_param.align = 0;
+	if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, dst,
+				  &sg_param))
+		goto dstmap_fail;
+
+	skb_set_transport_header(skb, transhdr_len);
+
+	if (assoclen) {
+		/* AAD buffer in */
+		write_sg_to_skb(skb, &frags, req->src, assoclen);
+
+	}
+	write_buffer_to_skb(skb, &frags, req->iv, ivsize);
+	write_sg_to_skb(skb, &frags, src, req->cryptlen);
+	create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1,
+		   sizeof(struct cpl_rx_phys_dsgl) + dst_size);
+	reqctx->skb = skb;
+	skb_get(skb);
+
+	return skb;
+dstmap_fail:
+	/* ivmap_fail: */
+	kfree_skb(skb);
+err:
+	return ERR_PTR(-EINVAL);
+}
+
+static void aes_gcm_empty_pld_pad(struct scatterlist *sg,
+				  unsigned short offset)
+{
+	struct page *spage;
+	unsigned char *addr;
+
+	spage = sg_page(sg);
+	get_page(spage); /* so that it is not freed by NIC */
+#ifdef KMAP_ATOMIC_ARGS
+	addr = kmap_atomic(spage, KM_SOFTIRQ0);
+#else
+	addr = kmap_atomic(spage);
+#endif
+	memset(addr + sg->offset, 0, offset + 1);
+
+	kunmap_atomic(addr);
+}
+
+static int set_msg_len(u8 *block, unsigned int msglen, int csize)
+{
+	__be32 data;
+
+	memset(block, 0, csize);
+	block += csize;
+
+	if (csize >= 4)
+		csize = 4;
+	else if (msglen > (unsigned int)(1 << (8 * csize)))
+		return -EOVERFLOW;
+
+	data = cpu_to_be32(msglen);
+	memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
+
+	return 0;
+}
+
+static void generate_b0(struct aead_request *req,
+			struct chcr_aead_ctx *aeadctx,
+			unsigned short op_type)
+{
+	unsigned int l, lp, m;
+	int rc;
+	struct crypto_aead *aead = crypto_aead_reqtfm(req);
+	struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+	u8 *b0 = reqctx->scratch_pad;
+
+	m = crypto_aead_authsize(aead);
+
+	memcpy(b0, reqctx->iv, 16);
+
+	lp = b0[0];
+	l = lp + 1;
+
+	/* set m, bits 3-5 */
+	*b0 |= (8 * ((m - 2) / 2));
+
+	/* set adata, bit 6, if associated data is used */
+	if (req->assoclen)
+		*b0 |= 64;
+	rc = set_msg_len(b0 + 16 - l,
+			 (op_type == CHCR_DECRYPT_OP) ?
+			 req->cryptlen - m : req->cryptlen, l);
+}
+
+static inline int crypto_ccm_check_iv(const u8 *iv)
+{
+	/* 2 <= L <= 8, so 1 <= L' <= 7. */
+	if (iv[0] < 1 || iv[0] > 7)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int ccm_format_packet(struct aead_request *req,
+			     struct chcr_aead_ctx *aeadctx,
+			     unsigned int sub_type,
+			     unsigned short op_type)
+{
+	struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+	int rc = 0;
+
+	if (req->assoclen > T5_MAX_AAD_SIZE) {
+		pr_err("CCM: Unsupported AAD data. It should be < %d\n",
+		       T5_MAX_AAD_SIZE);
+		return -EINVAL;
+	}
+	if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
+		reqctx->iv[0] = 3;
+		memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3);
+		memcpy(reqctx->iv + 4, req->iv, 8);
+		memset(reqctx->iv + 12, 0, 4);
+		*((unsigned short *)(reqctx->scratch_pad + 16)) =
+			htons(req->assoclen - 8);
+	} else {
+		memcpy(reqctx->iv, req->iv, 16);
+		*((unsigned short *)(reqctx->scratch_pad + 16)) =
+			htons(req->assoclen);
+	}
+	generate_b0(req, aeadctx, op_type);
+	/* zero the ctr value */
+	memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1);
+	return rc;
+}
+
+static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
+				  unsigned int dst_size,
+				  struct aead_request *req,
+				  unsigned short op_type,
+					  struct chcr_context *chcrctx)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	unsigned int ivsize = AES_BLOCK_SIZE;
+	unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
+	unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
+	unsigned int c_id = chcrctx->dev->tx_channel_id;
+	unsigned int ccm_xtra;
+	unsigned char tag_offset = 0, auth_offset = 0;
+	unsigned char hmac_ctrl = get_hmac(crypto_aead_authsize(tfm));
+	unsigned int assoclen;
+
+	if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
+		assoclen = req->assoclen - 8;
+	else
+		assoclen = req->assoclen;
+	ccm_xtra = CCM_B0_SIZE +
+		((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
+
+	auth_offset = req->cryptlen ?
+		(assoclen + ivsize + 1 + ccm_xtra) : 0;
+	if (op_type == CHCR_DECRYPT_OP) {
+		if (crypto_aead_authsize(tfm) != req->cryptlen)
+			tag_offset = crypto_aead_authsize(tfm);
+		else
+			auth_offset = 0;
+	}
+
+
+	sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id,
+					 2, (ivsize ?  (assoclen + 1) :  0) +
+					 ccm_xtra);
+	sec_cpl->pldlen =
+		htonl(assoclen + ivsize + req->cryptlen + ccm_xtra);
+	/* For CCM there wil be b0 always. So AAD start will be 1 always */
+	sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
+					1, assoclen + ccm_xtra, assoclen
+					+ ivsize + 1 + ccm_xtra, 0);
+
+	sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
+					auth_offset, tag_offset,
+					(op_type == CHCR_ENCRYPT_OP) ? 0 :
+					crypto_aead_authsize(tfm));
+	sec_cpl->seqno_numivs =  FILL_SEC_CPL_SCMD0_SEQNO(op_type,
+					(op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
+					cipher_mode, mac_mode, hmac_ctrl,
+					ivsize >> 1);
+
+	sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
+					1, dst_size);
+}
+
+int aead_ccm_validate_input(unsigned short op_type,
+			    struct aead_request *req,
+			    struct chcr_aead_ctx *aeadctx,
+			    unsigned int sub_type)
+{
+	if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
+		if (crypto_ccm_check_iv(req->iv)) {
+			pr_err("CCM: IV check fails\n");
+			return -EINVAL;
+		}
+	} else {
+		if (req->assoclen != 16 && req->assoclen != 20) {
+			pr_err("RFC4309: Invalid AAD length %d\n",
+			       req->assoclen);
+			return -EINVAL;
+		}
+	}
+	if (aeadctx->enckey_len == 0) {
+		pr_err("CCM: Encryption key not set\n");
+		return -EINVAL;
+	}
+	return 0;
+}
+
+unsigned int fill_aead_req_fields(struct sk_buff *skb,
+				  struct aead_request *req,
+				  struct scatterlist *src,
+				  unsigned int ivsize,
+				  struct chcr_aead_ctx *aeadctx)
+{
+	unsigned int frags = 0;
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+	/* b0 and aad length(if available) */
+
+	write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE +
+				(req->assoclen ?  CCM_AAD_FIELD_SIZE : 0));
+	if (req->assoclen) {
+		if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
+			write_sg_to_skb(skb, &frags, req->src,
+					req->assoclen - 8);
+		else
+			write_sg_to_skb(skb, &frags, req->src, req->assoclen);
+	}
+	write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
+	if (req->cryptlen)
+		write_sg_to_skb(skb, &frags, src, req->cryptlen);
+
+	return frags;
+}
+
+static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
+					  unsigned short qid,
+					  int size,
+					  unsigned short op_type)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct chcr_context *ctx = crypto_aead_ctx(tfm);
+	struct uld_ctx *u_ctx = ULD_CTX(ctx);
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+	struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+	struct sk_buff *skb = NULL;
+	struct chcr_wr *chcr_req;
+	struct cpl_rx_phys_dsgl *phys_cpl;
+	struct phys_sge_parm sg_param;
+	struct scatterlist *src, *dst;
+	struct scatterlist src_sg[2], dst_sg[2];
+	unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE;
+	unsigned int dst_size = 0, kctx_len;
+	unsigned int sub_type;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	int err = 0;
+	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
+		GFP_ATOMIC;
+
+
+	if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
+		goto err;
+
+	if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0)
+		goto err;
+	sub_type = get_aead_subtype(tfm);
+	src = scatterwalk_ffwd(src_sg, req->src, req->assoclen);
+	dst = src;
+	if (req->src != req->dst) {
+		err = chcr_copy_assoc(req, aeadctx);
+		if (err) {
+			pr_err("AAD copy to destination buffer fails\n");
+			return ERR_PTR(err);
+		}
+		dst = scatterwalk_ffwd(dst_sg, req->dst, req->assoclen);
+	}
+	reqctx->dst_nents = sg_nents_for_len(dst, req->cryptlen +
+					     (op_type ? -authsize : authsize));
+	if (reqctx->dst_nents <= 0) {
+		pr_err("CCM:Invalid Destination sg entries\n");
+		goto err;
+	}
+
+
+	if (aead_ccm_validate_input(op_type, req, aeadctx, sub_type))
+		goto err;
+
+	dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
+	kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2;
+	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
+	skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)),  flags);
+
+	if (!skb)
+		goto err;
+
+	skb_reserve(skb, sizeof(struct sge_opaque_hdr));
+
+	chcr_req = (struct chcr_wr *) __skb_put(skb, transhdr_len);
+	memset(chcr_req, 0, transhdr_len);
+
+	fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx);
+
+	chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
+	memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
+	memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
+					16), aeadctx->key, aeadctx->enckey_len);
+
+	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
+	if (ccm_format_packet(req, aeadctx, sub_type, op_type))
+		goto dstmap_fail;
+
+	sg_param.nents = reqctx->dst_nents;
+	sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
+	sg_param.qid = qid;
+	sg_param.align = 0;
+	if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, dst,
+				  &sg_param))
+		goto dstmap_fail;
+
+	skb_set_transport_header(skb, transhdr_len);
+	frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx);
+	create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1,
+		    sizeof(struct cpl_rx_phys_dsgl) + dst_size);
+	reqctx->skb = skb;
+	skb_get(skb);
+	return skb;
+dstmap_fail:
+	kfree_skb(skb);
+	skb = NULL;
+err:
+	return ERR_PTR(-EINVAL);
+}
+
+static struct sk_buff *create_gcm_wr(struct aead_request *req,
+				     unsigned short qid,
+				     int size,
+				     unsigned short op_type)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct chcr_context *ctx = crypto_aead_ctx(tfm);
+	struct uld_ctx *u_ctx = ULD_CTX(ctx);
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+	struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
+	struct sk_buff *skb = NULL;
+	struct chcr_wr *chcr_req;
+	struct cpl_rx_phys_dsgl *phys_cpl;
+	struct phys_sge_parm sg_param;
+	struct scatterlist *src, *dst;
+	struct scatterlist src_sg[2], dst_sg[2];
+	unsigned int frags = 0, transhdr_len;
+	unsigned int ivsize = AES_BLOCK_SIZE;
+	unsigned int dst_size = 0, kctx_len;
+	unsigned char tag_offset = 0;
+	unsigned int crypt_len = 0;
+	unsigned int authsize = crypto_aead_authsize(tfm);
+	unsigned char hmac_ctrl = get_hmac(authsize);
+	int err = 0;
+	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
+		GFP_ATOMIC;
+
+	/* validate key size */
+	if (aeadctx->enckey_len == 0)
+		goto err;
+
+	if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
+		goto err;
+
+	if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0)
+		goto err;
+
+	src = scatterwalk_ffwd(src_sg, req->src, req->assoclen);
+	dst = src;
+	if (req->src != req->dst) {
+		err = chcr_copy_assoc(req, aeadctx);
+		if (err)
+			return	ERR_PTR(err);
+		dst = scatterwalk_ffwd(dst_sg, req->dst, req->assoclen);
+	}
+
+	if (!req->cryptlen)
+		/* null-payload is not supported in the hardware.
+		 * software is sending block size
+		 */
+		crypt_len = AES_BLOCK_SIZE;
+	else
+		crypt_len = req->cryptlen;
+	reqctx->dst_nents = sg_nents_for_len(dst, req->cryptlen +
+					     (op_type ? -authsize : authsize));
+	if (reqctx->dst_nents <= 0) {
+		pr_err("GCM:Invalid Destination sg entries\n");
+		goto err;
+	}
+
+
+	dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
+	kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) +
+		AEAD_H_SIZE;
+	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
+	skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
+	if (!skb)
+		goto err;
+
+	/* NIC driver is going to write the sge hdr. */
+	skb_reserve(skb, sizeof(struct sge_opaque_hdr));
+
+	chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
+	memset(chcr_req, 0, transhdr_len);
+
+	if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
+		req->assoclen -= 8;
+
+	tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
+	chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
+					ctx->dev->tx_channel_id, 2, (ivsize ?
+					(req->assoclen + 1) : 0));
+	chcr_req->sec_cpl.pldlen = htonl(req->assoclen + ivsize + crypt_len);
+	chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
+					req->assoclen ? 1 : 0, req->assoclen,
+					req->assoclen + ivsize + 1, 0);
+	if (req->cryptlen) {
+		chcr_req->sec_cpl.cipherstop_lo_authinsert =
+			FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + ivsize + 1,
+						tag_offset, tag_offset);
+		chcr_req->sec_cpl.seqno_numivs =
+			FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type ==
+					CHCR_ENCRYPT_OP) ? 1 : 0,
+					CHCR_SCMD_CIPHER_MODE_AES_GCM,
+					CHCR_SCMD_AUTH_MODE_GHASH, hmac_ctrl,
+					ivsize >> 1);
+	} else {
+		chcr_req->sec_cpl.cipherstop_lo_authinsert =
+			FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
+		chcr_req->sec_cpl.seqno_numivs =
+			FILL_SEC_CPL_SCMD0_SEQNO(op_type,
+					(op_type ==  CHCR_ENCRYPT_OP) ?
+					1 : 0, CHCR_SCMD_CIPHER_MODE_AES_CBC,
+					0, 0, ivsize >> 1);
+	}
+	chcr_req->sec_cpl.ivgen_hdrlen =  FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
+					0, 1, dst_size);
+	chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
+	memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
+	memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
+				16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
+
+	/* prepare a 16 byte iv */
+	/* S   A   L  T |  IV | 0x00000001 */
+	if (get_aead_subtype(tfm) ==
+	    CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
+		memcpy(reqctx->iv, aeadctx->salt, 4);
+		memcpy(reqctx->iv + 4, req->iv, 8);
+	} else {
+		memcpy(reqctx->iv, req->iv, 12);
+	}
+	*((unsigned int *)(reqctx->iv + 12)) = htonl(0x01);
+
+	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
+	sg_param.nents = reqctx->dst_nents;
+	sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
+	sg_param.qid = qid;
+	sg_param.align = 0;
+	if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, dst,
+				  &sg_param))
+		goto dstmap_fail;
+
+	skb_set_transport_header(skb, transhdr_len);
+
+	write_sg_to_skb(skb, &frags, req->src, req->assoclen);
+
+	write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
+
+	if (req->cryptlen) {
+		write_sg_to_skb(skb, &frags, src, req->cryptlen);
+	} else {
+		aes_gcm_empty_pld_pad(req->dst, authsize - 1);
+		write_sg_to_skb(skb, &frags, dst, crypt_len);
+	}
+
+	create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1,
+			sizeof(struct cpl_rx_phys_dsgl) + dst_size);
+	reqctx->skb = skb;
+	skb_get(skb);
+	return skb;
+
+dstmap_fail:
+	/* ivmap_fail: */
+	kfree_skb(skb);
+	skb = NULL;
+err:
+	return skb;
+}
+
+
+
+static int chcr_aead_cra_init(struct crypto_aead *tfm)
+{
+	struct chcr_context *ctx = crypto_aead_ctx(tfm);
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+
+	crypto_aead_set_reqsize(tfm, sizeof(struct chcr_aead_reqctx));
+	aeadctx->null = crypto_get_default_null_skcipher();
+	if (IS_ERR(aeadctx->null))
+		return PTR_ERR(aeadctx->null);
+	return chcr_device_init(ctx);
+}
+
+static void chcr_aead_cra_exit(struct crypto_aead *tfm)
+{
+	crypto_put_default_null_skcipher();
+}
+
+static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
+					unsigned int authsize)
+{
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
+
+	aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
+	aeadctx->mayverify = VERIFY_HW;
+	return 0;
+}
+static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
+				    unsigned int authsize)
+{
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
+	u32 maxauth = crypto_aead_maxauthsize(tfm);
+
+	/*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
+	 * true for sha1. authsize == 12 condition should be before
+	 * authsize == (maxauth >> 1)
+	 */
+	if (authsize == ICV_4) {
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
+		aeadctx->mayverify = VERIFY_HW;
+	} else if (authsize == ICV_6) {
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
+		aeadctx->mayverify = VERIFY_HW;
+	} else if (authsize == ICV_10) {
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
+		aeadctx->mayverify = VERIFY_HW;
+	} else if (authsize == ICV_12) {
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
+		aeadctx->mayverify = VERIFY_HW;
+	} else if (authsize == ICV_14) {
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
+		aeadctx->mayverify = VERIFY_HW;
+	} else if (authsize == (maxauth >> 1)) {
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
+		aeadctx->mayverify = VERIFY_HW;
+	} else if (authsize == maxauth) {
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
+		aeadctx->mayverify = VERIFY_HW;
+	} else {
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
+		aeadctx->mayverify = VERIFY_SW;
+	}
+	return 0;
+}
+
+
+static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
+{
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
+
+	switch (authsize) {
+	case ICV_4:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_8:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_12:
+		 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
+		 aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_14:
+		 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
+		 aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_16:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_13:
+	case ICV_15:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
+		aeadctx->mayverify = VERIFY_SW;
+		break;
+	default:
+
+		  crypto_tfm_set_flags((struct crypto_tfm *) tfm,
+			CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
+					  unsigned int authsize)
+{
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
+
+	switch (authsize) {
+	case ICV_8:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_12:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_16:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	default:
+		crypto_tfm_set_flags((struct crypto_tfm *)tfm,
+				     CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
+				unsigned int authsize)
+{
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
+
+	switch (authsize) {
+	case ICV_4:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_6:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_8:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_10:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_12:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_14:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	case ICV_16:
+		aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
+		aeadctx->mayverify = VERIFY_HW;
+		break;
+	default:
+		crypto_tfm_set_flags((struct crypto_tfm *)tfm,
+				     CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
+				const u8 *key,
+				unsigned int keylen)
+{
+	struct chcr_context *ctx = crypto_aead_ctx(aead);
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+	unsigned char ck_size, mk_size;
+	int key_ctx_size = 0;
+
+	memcpy(aeadctx->key, key, keylen);
+	aeadctx->enckey_len = keylen;
+	key_ctx_size = sizeof(struct _key_ctx) +
+		((DIV_ROUND_UP(keylen, 16)) << 4)  * 2;
+	if (keylen == AES_KEYSIZE_128) {
+		mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
+	} else if (keylen == AES_KEYSIZE_192) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
+		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
+	} else if (keylen == AES_KEYSIZE_256) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
+		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
+	} else {
+		crypto_tfm_set_flags((struct crypto_tfm *)aead,
+				     CRYPTO_TFM_RES_BAD_KEY_LEN);
+		aeadctx->enckey_len = 0;
+		return	-EINVAL;
+	}
+	aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
+						key_ctx_size >> 4);
+	return 0;
+}
+
+static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
+				    unsigned int keylen)
+{
+	struct chcr_context *ctx = crypto_aead_ctx(aead);
+	 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+
+	if (keylen < 3) {
+		crypto_tfm_set_flags((struct crypto_tfm *)aead,
+				     CRYPTO_TFM_RES_BAD_KEY_LEN);
+		aeadctx->enckey_len = 0;
+		return	-EINVAL;
+	}
+	keylen -= 3;
+	memcpy(aeadctx->salt, key + keylen, 3);
+	return chcr_aead_ccm_setkey(aead, key, keylen);
+}
+
+static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
+			   unsigned int keylen)
+{
+	struct chcr_context *ctx = crypto_aead_ctx(aead);
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+	struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
+	struct blkcipher_desc h_desc;
+	struct scatterlist src[1];
+	unsigned int ck_size;
+	int ret = 0, key_ctx_size = 0;
+
+	if (get_aead_subtype(aead) ==
+	    CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
+		keylen -= 4;  /* nonce/salt is present in the last 4 bytes */
+		memcpy(aeadctx->salt, key + keylen, 4);
+	}
+	if (keylen == AES_KEYSIZE_128) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
+	} else if (keylen == AES_KEYSIZE_192) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
+	} else if (keylen == AES_KEYSIZE_256) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
+	} else {
+		crypto_tfm_set_flags((struct crypto_tfm *)aead,
+				     CRYPTO_TFM_RES_BAD_KEY_LEN);
+		aeadctx->enckey_len = 0;
+		pr_err("GCM: Invalid key length %d", keylen);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	memcpy(aeadctx->key, key, keylen);
+	aeadctx->enckey_len = keylen;
+	key_ctx_size = sizeof(struct _key_ctx) +
+		((DIV_ROUND_UP(keylen, 16)) << 4) +
+		AEAD_H_SIZE;
+		aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
+						CHCR_KEYCTX_MAC_KEY_SIZE_128,
+						0, 0,
+						key_ctx_size >> 4);
+	/* Calculate the H = CIPH(K, 0 repeated 16 times) using sync aes
+	 * blkcipher It will go on key context
+	 */
+	h_desc.tfm = crypto_alloc_blkcipher("cbc(aes-generic)", 0, 0);
+	if (IS_ERR(h_desc.tfm)) {
+		aeadctx->enckey_len = 0;
+		ret = -ENOMEM;
+		goto out;
+	}
+	h_desc.flags = 0;
+	ret = crypto_blkcipher_setkey(h_desc.tfm, key, keylen);
+	if (ret) {
+		aeadctx->enckey_len = 0;
+		goto out1;
+	}
+	memset(gctx->ghash_h, 0, AEAD_H_SIZE);
+	sg_init_one(&src[0], gctx->ghash_h, AEAD_H_SIZE);
+	ret = crypto_blkcipher_encrypt(&h_desc, &src[0], &src[0], AEAD_H_SIZE);
+
+out1:
+	crypto_free_blkcipher(h_desc.tfm);
+out:
+	return ret;
+}
+
+static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
+				   unsigned int keylen)
+{
+	struct chcr_context *ctx = crypto_aead_ctx(authenc);
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+	struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
+	/* it contains auth and cipher key both*/
+	struct crypto_authenc_keys keys;
+	unsigned int bs;
+	unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
+	int err = 0, i, key_ctx_len = 0;
+	unsigned char ck_size = 0;
+	unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 };
+	struct crypto_shash *base_hash = NULL;
+	struct algo_param param;
+	int align;
+	u8 *o_ptr = NULL;
+
+	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
+		crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		goto out;
+	}
+
+	if (get_alg_config(&param, max_authsize)) {
+		pr_err("chcr : Unsupported digest size\n");
+		goto out;
+	}
+	if (keys.enckeylen == AES_KEYSIZE_128) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
+	} else if (keys.enckeylen == AES_KEYSIZE_192) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
+	} else if (keys.enckeylen == AES_KEYSIZE_256) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
+	} else {
+		pr_err("chcr : Unsupported cipher key\n");
+		goto out;
+	}
+
+	/* Copy only encryption key. We use authkey to generate h(ipad) and
+	 * h(opad) so authkey is not needed again. authkeylen size have the
+	 * size of the hash digest size.
+	 */
+	memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
+	aeadctx->enckey_len = keys.enckeylen;
+	get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
+			    aeadctx->enckey_len << 3);
+
+	base_hash  = chcr_alloc_shash(max_authsize);
+	if (IS_ERR(base_hash)) {
+		pr_err("chcr : Base driver cannot be loaded\n");
+		goto out;
 	}
+	{
+		SHASH_DESC_ON_STACK(shash, base_hash);
+		shash->tfm = base_hash;
+		shash->flags = crypto_shash_get_flags(base_hash);
+		bs = crypto_shash_blocksize(base_hash);
+		align = KEYCTX_ALIGN_PAD(max_authsize);
+		o_ptr =  actx->h_iopad + param.result_size + align;
+
+		if (keys.authkeylen > bs) {
+			err = crypto_shash_digest(shash, keys.authkey,
+						  keys.authkeylen,
+						  o_ptr);
+			if (err) {
+				pr_err("chcr : Base driver cannot be loaded\n");
+				goto out;
+			}
+			keys.authkeylen = max_authsize;
+		} else
+			memcpy(o_ptr, keys.authkey, keys.authkeylen);
+
+		/* Compute the ipad-digest*/
+		memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
+		memcpy(pad, o_ptr, keys.authkeylen);
+		for (i = 0; i < bs >> 2; i++)
+			*((unsigned int *)pad + i) ^= IPAD_DATA;
+
+		if (chcr_compute_partial_hash(shash, pad, actx->h_iopad,
+					      max_authsize))
+			goto out;
+		/* Compute the opad-digest */
+		memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
+		memcpy(pad, o_ptr, keys.authkeylen);
+		for (i = 0; i < bs >> 2; i++)
+			*((unsigned int *)pad + i) ^= OPAD_DATA;
+
+		if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize))
+			goto out;
+
+		/* convert the ipad and opad digest to network order */
+		chcr_change_order(actx->h_iopad, param.result_size);
+		chcr_change_order(o_ptr, param.result_size);
+		key_ctx_len = sizeof(struct _key_ctx) +
+			((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) +
+			(param.result_size + align) * 2;
+		aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size,
+						0, 1, key_ctx_len >> 4);
+		actx->auth_mode = param.auth_mode;
+		chcr_free_shash(base_hash);
+
+		return 0;
+	}
+out:
+	aeadctx->enckey_len = 0;
+	if (base_hash)
+		chcr_free_shash(base_hash);
+	return -EINVAL;
 }
 
+static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
+					const u8 *key, unsigned int keylen)
+{
+	struct chcr_context *ctx = crypto_aead_ctx(authenc);
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+	struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
+	struct crypto_authenc_keys keys;
+
+	/* it contains auth and cipher key both*/
+	int key_ctx_len = 0;
+	unsigned char ck_size = 0;
+
+	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
+		crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		goto out;
+	}
+	if (keys.enckeylen == AES_KEYSIZE_128) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
+	} else if (keys.enckeylen == AES_KEYSIZE_192) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
+	} else if (keys.enckeylen == AES_KEYSIZE_256) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
+	} else {
+		pr_err("chcr : Unsupported cipher key\n");
+		goto out;
+	}
+	memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
+	aeadctx->enckey_len = keys.enckeylen;
+	get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
+				    aeadctx->enckey_len << 3);
+	key_ctx_len =  sizeof(struct _key_ctx)
+		+ ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4);
+
+	aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
+						0, key_ctx_len >> 4);
+	actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
+	return 0;
+out:
+	aeadctx->enckey_len = 0;
+	return -EINVAL;
+}
+static int chcr_aead_encrypt(struct aead_request *req)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+
+	reqctx->verify = VERIFY_HW;
+
+	switch (get_aead_subtype(tfm)) {
+	case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
+	case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
+		return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
+				    create_authenc_wr);
+	case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
+	case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
+		return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
+				    create_aead_ccm_wr);
+	default:
+		return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
+				    create_gcm_wr);
+	}
+}
+
+static int chcr_aead_decrypt(struct aead_request *req)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
+	struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+	int size;
+
+	if (aeadctx->mayverify == VERIFY_SW) {
+		size = crypto_aead_maxauthsize(tfm);
+		reqctx->verify = VERIFY_SW;
+	} else {
+		size = 0;
+		reqctx->verify = VERIFY_HW;
+	}
+
+	switch (get_aead_subtype(tfm)) {
+	case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
+	case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
+		return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
+				    create_authenc_wr);
+	case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
+	case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
+		return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
+				    create_aead_ccm_wr);
+	default:
+		return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
+				    create_gcm_wr);
+	}
+}
+
+static int chcr_aead_op(struct aead_request *req,
+			  unsigned short op_type,
+			  int size,
+			  create_wr_t create_wr_fn)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct chcr_context *ctx = crypto_aead_ctx(tfm);
+	struct uld_ctx *u_ctx = ULD_CTX(ctx);
+	struct sk_buff *skb;
+
+	if (ctx && !ctx->dev) {
+		pr_err("chcr : %s : No crypto device.\n", __func__);
+		return -ENXIO;
+	}
+	if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
+				   ctx->tx_channel_id)) {
+		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+			return -EBUSY;
+	}
+
+	/* Form a WR from req */
+	skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->tx_channel_id], size,
+			   op_type);
+
+	if (IS_ERR(skb) || skb == NULL) {
+		pr_err("chcr : %s : failed to form WR. No memory\n", __func__);
+		return PTR_ERR(skb);
+	}
+
+	skb->dev = u_ctx->lldi.ports[0];
+	set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
+	chcr_send_wr(skb);
+	return -EINPROGRESS;
+}
 static struct chcr_alg_template driver_algs[] = {
 	/* AES-CBC */
 	{
@@ -1234,7 +2487,7 @@ static struct chcr_alg_template driver_algs[] = {
 		.is_registered = 0,
 		.alg.crypto = {
 			.cra_name		= "cbc(aes)",
-			.cra_driver_name	= "cbc(aes-chcr)",
+			.cra_driver_name	= "cbc-aes-chcr",
 			.cra_priority		= CHCR_CRA_PRIORITY,
 			.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
 				CRYPTO_ALG_ASYNC,
@@ -1261,7 +2514,7 @@ static struct chcr_alg_template driver_algs[] = {
 		.is_registered = 0,
 		.alg.crypto =   {
 			.cra_name		= "xts(aes)",
-			.cra_driver_name	= "xts(aes-chcr)",
+			.cra_driver_name	= "xts-aes-chcr",
 			.cra_priority		= CHCR_CRA_PRIORITY,
 			.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER |
 				CRYPTO_ALG_ASYNC,
@@ -1354,7 +2607,7 @@ static struct chcr_alg_template driver_algs[] = {
 			.halg.digestsize = SHA1_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "hmac(sha1)",
-				.cra_driver_name = "hmac(sha1-chcr)",
+				.cra_driver_name = "hmac-sha1-chcr",
 				.cra_blocksize = SHA1_BLOCK_SIZE,
 			}
 		}
@@ -1366,7 +2619,7 @@ static struct chcr_alg_template driver_algs[] = {
 			.halg.digestsize = SHA224_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "hmac(sha224)",
-				.cra_driver_name = "hmac(sha224-chcr)",
+				.cra_driver_name = "hmac-sha224-chcr",
 				.cra_blocksize = SHA224_BLOCK_SIZE,
 			}
 		}
@@ -1378,7 +2631,7 @@ static struct chcr_alg_template driver_algs[] = {
 			.halg.digestsize = SHA256_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "hmac(sha256)",
-				.cra_driver_name = "hmac(sha256-chcr)",
+				.cra_driver_name = "hmac-sha256-chcr",
 				.cra_blocksize = SHA256_BLOCK_SIZE,
 			}
 		}
@@ -1390,7 +2643,7 @@ static struct chcr_alg_template driver_algs[] = {
 			.halg.digestsize = SHA384_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "hmac(sha384)",
-				.cra_driver_name = "hmac(sha384-chcr)",
+				.cra_driver_name = "hmac-sha384-chcr",
 				.cra_blocksize = SHA384_BLOCK_SIZE,
 			}
 		}
@@ -1402,11 +2655,205 @@ static struct chcr_alg_template driver_algs[] = {
 			.halg.digestsize = SHA512_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "hmac(sha512)",
-				.cra_driver_name = "hmac(sha512-chcr)",
+				.cra_driver_name = "hmac-sha512-chcr",
 				.cra_blocksize = SHA512_BLOCK_SIZE,
 			}
 		}
 	},
+	/* Add AEAD Algorithms */
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "gcm(aes)",
+				.cra_driver_name = "gcm-aes-chcr",
+				.cra_blocksize	= 1,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_gcm_ctx),
+			},
+			.ivsize = 12,
+			.maxauthsize = GHASH_DIGEST_SIZE,
+			.setkey = chcr_gcm_setkey,
+			.setauthsize = chcr_gcm_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "rfc4106(gcm(aes))",
+				.cra_driver_name = "rfc4106-gcm-aes-chcr",
+				.cra_blocksize	 = 1,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_gcm_ctx),
+
+			},
+			.ivsize = 8,
+			.maxauthsize	= GHASH_DIGEST_SIZE,
+			.setkey = chcr_gcm_setkey,
+			.setauthsize	= chcr_4106_4309_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "ccm(aes)",
+				.cra_driver_name = "ccm-aes-chcr",
+				.cra_blocksize	 = 1,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx),
+
+			},
+			.ivsize = AES_BLOCK_SIZE,
+			.maxauthsize	= GHASH_DIGEST_SIZE,
+			.setkey = chcr_aead_ccm_setkey,
+			.setauthsize	= chcr_ccm_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "rfc4309(ccm(aes))",
+				.cra_driver_name = "rfc4309-ccm-aes-chcr",
+				.cra_blocksize	 = 1,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx),
+
+			},
+			.ivsize = 8,
+			.maxauthsize	= GHASH_DIGEST_SIZE,
+			.setkey = chcr_aead_rfc4309_setkey,
+			.setauthsize = chcr_4106_4309_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "authenc(hmac(sha1),cbc(aes))",
+				.cra_driver_name =
+					"authenc-hmac-sha1-cbc-aes-chcr",
+				.cra_blocksize	 = AES_BLOCK_SIZE,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_authenc_ctx),
+
+			},
+			.ivsize = AES_BLOCK_SIZE,
+			.maxauthsize = SHA1_DIGEST_SIZE,
+			.setkey = chcr_authenc_setkey,
+			.setauthsize = chcr_authenc_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+
+				.cra_name = "authenc(hmac(sha256),cbc(aes))",
+				.cra_driver_name =
+					"authenc-hmac-sha256-cbc-aes-chcr",
+				.cra_blocksize	 = AES_BLOCK_SIZE,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_authenc_ctx),
+
+			},
+			.ivsize = AES_BLOCK_SIZE,
+			.maxauthsize	= SHA256_DIGEST_SIZE,
+			.setkey = chcr_authenc_setkey,
+			.setauthsize = chcr_authenc_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "authenc(hmac(sha224),cbc(aes))",
+				.cra_driver_name =
+					"authenc-hmac-sha224-cbc-aes-chcr",
+				.cra_blocksize	 = AES_BLOCK_SIZE,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_authenc_ctx),
+			},
+			.ivsize = AES_BLOCK_SIZE,
+			.maxauthsize = SHA224_DIGEST_SIZE,
+			.setkey = chcr_authenc_setkey,
+			.setauthsize = chcr_authenc_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "authenc(hmac(sha384),cbc(aes))",
+				.cra_driver_name =
+					"authenc-hmac-sha384-cbc-aes-chcr",
+				.cra_blocksize	 = AES_BLOCK_SIZE,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_authenc_ctx),
+
+			},
+			.ivsize = AES_BLOCK_SIZE,
+			.maxauthsize = SHA384_DIGEST_SIZE,
+			.setkey = chcr_authenc_setkey,
+			.setauthsize = chcr_authenc_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "authenc(hmac(sha512),cbc(aes))",
+				.cra_driver_name =
+					"authenc-hmac-sha512-cbc-aes-chcr",
+				.cra_blocksize	 = AES_BLOCK_SIZE,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_authenc_ctx),
+
+			},
+			.ivsize = AES_BLOCK_SIZE,
+			.maxauthsize = SHA512_DIGEST_SIZE,
+			.setkey = chcr_authenc_setkey,
+			.setauthsize = chcr_authenc_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "authenc(digest_null,cbc(aes))",
+				.cra_driver_name =
+					"authenc-digest_null-cbc-aes-chcr",
+				.cra_blocksize	 = AES_BLOCK_SIZE,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_authenc_ctx),
+
+			},
+			.ivsize  = AES_BLOCK_SIZE,
+			.maxauthsize = 0,
+			.setkey  = chcr_aead_digest_null_setkey,
+			.setauthsize = chcr_authenc_null_setauthsize,
+		}
+	},
 };
 
 /*
@@ -1424,6 +2871,11 @@ static int chcr_unregister_alg(void)
 				crypto_unregister_alg(
 						&driver_algs[i].alg.crypto);
 			break;
+		case CRYPTO_ALG_TYPE_AEAD:
+			if (driver_algs[i].is_registered)
+				crypto_unregister_aead(
+						&driver_algs[i].alg.aead);
+			break;
 		case CRYPTO_ALG_TYPE_AHASH:
 			if (driver_algs[i].is_registered)
 				crypto_unregister_ahash(
@@ -1458,6 +2910,19 @@ static int chcr_register_alg(void)
 			err = crypto_register_alg(&driver_algs[i].alg.crypto);
 			name = driver_algs[i].alg.crypto.cra_driver_name;
 			break;
+		case CRYPTO_ALG_TYPE_AEAD:
+			driver_algs[i].alg.aead.base.cra_priority =
+				CHCR_CRA_PRIORITY;
+			driver_algs[i].alg.aead.base.cra_flags =
+				CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
+			driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
+			driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
+			driver_algs[i].alg.aead.init = chcr_aead_cra_init;
+			driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
+			driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
+			err = crypto_register_aead(&driver_algs[i].alg.aead);
+			name = driver_algs[i].alg.aead.base.cra_driver_name;
+			break;
 		case CRYPTO_ALG_TYPE_AHASH:
 			a_hash = &driver_algs[i].alg.hash;
 			a_hash->update = chcr_ahash_update;
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
index 199b0bb69b89..3c7c51f7bedf 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -108,30 +108,24 @@
 #define IPAD_DATA 0x36363636
 #define OPAD_DATA 0x5c5c5c5c
 
-#define TRANSHDR_SIZE(alignedkctx_len)\
-	(sizeof(struct ulptx_idata) +\
-	 sizeof(struct ulp_txpkt) +\
-	 sizeof(struct fw_crypto_lookaside_wr) +\
-	 sizeof(struct cpl_tx_sec_pdu) +\
-	 (alignedkctx_len))
-#define CIPHER_TRANSHDR_SIZE(alignedkctx_len, sge_pairs) \
-	(TRANSHDR_SIZE(alignedkctx_len) + sge_pairs +\
+#define TRANSHDR_SIZE(kctx_len)\
+	(sizeof(struct chcr_wr) +\
+	 kctx_len)
+#define CIPHER_TRANSHDR_SIZE(kctx_len, sge_pairs) \
+	(TRANSHDR_SIZE((kctx_len)) + (sge_pairs) +\
 	 sizeof(struct cpl_rx_phys_dsgl))
-#define HASH_TRANSHDR_SIZE(alignedkctx_len)\
-	(TRANSHDR_SIZE(alignedkctx_len) + DUMMY_BYTES)
+#define HASH_TRANSHDR_SIZE(kctx_len)\
+	(TRANSHDR_SIZE(kctx_len) + DUMMY_BYTES)
 
-#define SEC_CPL_OFFSET (sizeof(struct fw_crypto_lookaside_wr) + \
-			sizeof(struct ulp_txpkt) + \
-			sizeof(struct ulptx_idata))
 
-#define FILL_SEC_CPL_OP_IVINSR(id, len, hldr, ofst)      \
+#define FILL_SEC_CPL_OP_IVINSR(id, len, ofst)      \
 	htonl( \
 	       CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | \
 	       CPL_TX_SEC_PDU_RXCHID_V((id)) | \
 	       CPL_TX_SEC_PDU_ACKFOLLOWS_V(0) | \
 	       CPL_TX_SEC_PDU_ULPTXLPBK_V(1) | \
 	       CPL_TX_SEC_PDU_CPLLEN_V((len)) | \
-	       CPL_TX_SEC_PDU_PLACEHOLDER_V((hldr)) | \
+	       CPL_TX_SEC_PDU_PLACEHOLDER_V(0) | \
 	       CPL_TX_SEC_PDU_IVINSRTOFST_V((ofst)))
 
 #define  FILL_SEC_CPL_CIPHERSTOP_HI(a_start, a_stop, c_start, c_stop_hi) \
@@ -148,7 +142,7 @@
 		CPL_TX_SEC_PDU_AUTHSTOP_V((a_stop)) | \
 		CPL_TX_SEC_PDU_AUTHINSERT_V((a_inst)))
 
-#define  FILL_SEC_CPL_SCMD0_SEQNO(ctrl, seq, cmode, amode, opad, size, nivs)  \
+#define  FILL_SEC_CPL_SCMD0_SEQNO(ctrl, seq, cmode, amode, opad, size)  \
 		htonl( \
 		SCMD_SEQ_NO_CTRL_V(0) | \
 		SCMD_STATUS_PRESENT_V(0) | \
@@ -159,7 +153,7 @@
 		SCMD_AUTH_MODE_V((amode)) | \
 		SCMD_HMAC_CTRL_V((opad)) | \
 		SCMD_IV_SIZE_V((size)) | \
-		SCMD_NUM_IVS_V((nivs)))
+		SCMD_NUM_IVS_V(0))
 
 #define FILL_SEC_CPL_IVGEN_HDRLEN(last, more, ctx_in, mac, ivdrop, len) htonl( \
 		SCMD_ENB_DBGID_V(0) | \
@@ -264,13 +258,15 @@ enum {
  * where they indicate the size of the integrity check value (ICV)
  */
 enum {
-	AES_CCM_ICV_4   = 4,
-	AES_CCM_ICV_6   = 6,
-	AES_CCM_ICV_8   = 8,
-	AES_CCM_ICV_10  = 10,
-	AES_CCM_ICV_12  = 12,
-	AES_CCM_ICV_14  = 14,
-	AES_CCM_ICV_16 = 16
+	ICV_4  = 4,
+	ICV_6  = 6,
+	ICV_8  = 8,
+	ICV_10 = 10,
+	ICV_12 = 12,
+	ICV_13 = 13,
+	ICV_14 = 14,
+	ICV_15 = 15,
+	ICV_16 = 16
 };
 
 struct hash_op_params {
@@ -394,7 +390,7 @@ static const u8 aes_sbox[256] = {
 	187, 22
 };
 
-static u32 aes_ks_subword(const u32 w)
+static inline u32 aes_ks_subword(const u32 w)
 {
 	u8 bytes[4];
 
@@ -412,61 +408,4 @@ static u32 round_constant[11] = {
 	0x1B000000, 0x36000000, 0x6C000000
 };
 
-/* dec_key - OUTPUT - Reverse round key
- * key - INPUT - key
- * keylength - INPUT - length of the key in number of bits
- */
-static inline void get_aes_decrypt_key(unsigned char *dec_key,
-				       const unsigned char *key,
-				       unsigned int keylength)
-{
-	u32 temp;
-	u32 w_ring[MAX_NK];
-	int i, j, k;
-	u8  nr, nk;
-
-	switch (keylength) {
-	case AES_KEYLENGTH_128BIT:
-		nk = KEYLENGTH_4BYTES;
-		nr = NUMBER_OF_ROUNDS_10;
-		break;
-
-	case AES_KEYLENGTH_192BIT:
-		nk = KEYLENGTH_6BYTES;
-		nr = NUMBER_OF_ROUNDS_12;
-		break;
-	case AES_KEYLENGTH_256BIT:
-		nk = KEYLENGTH_8BYTES;
-		nr = NUMBER_OF_ROUNDS_14;
-		break;
-	default:
-		return;
-	}
-	for (i = 0; i < nk; i++ )
-		w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
-
-	i = 0;
-	temp = w_ring[nk - 1];
-	while(i + nk < (nr + 1) * 4) {
-		if(!(i % nk)) {
-			/* RotWord(temp) */
-			temp = (temp << 8) | (temp >> 24);
-			temp = aes_ks_subword(temp);
-			temp ^= round_constant[i / nk];
-		}
-		else if (nk == 8 && (i % 4 == 0))
-			temp = aes_ks_subword(temp);
-		w_ring[i % nk] ^= temp;
-		temp = w_ring[i % nk];
-		i++;
-	}
-	i--;
-	for (k = 0, j = i % nk; k < nk; k++) {
-		*((u32 *)dec_key + k) = htonl(w_ring[j]);
-		j--;
-		if(j < 0)
-			j += nk;
-	}
-}
-
 #endif /* __CHCR_ALGO_H__ */
diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c
index 4d7f6700fd7e..918da8e6e2d8 100644
--- a/drivers/crypto/chelsio/chcr_core.c
+++ b/drivers/crypto/chelsio/chcr_core.c
@@ -110,14 +110,12 @@ static int cpl_fw6_pld_handler(struct chcr_dev *dev,
 	if (ack_err_status) {
 		if (CHK_MAC_ERR_BIT(ack_err_status) ||
 		    CHK_PAD_ERR_BIT(ack_err_status))
-			error_status = -EINVAL;
+			error_status = -EBADMSG;
 	}
 	/* call completion callback with failure status */
 	if (req) {
-		if (!chcr_handle_resp(req, input, error_status))
-			req->complete(req, error_status);
-		else
-			return -EINVAL;
+		error_status = chcr_handle_resp(req, input, error_status);
+		req->complete(req, error_status);
 	} else {
 		pr_err("Incorrect request address from the firmware\n");
 		return -EFAULT;
diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h
index 2a5c671a4232..c7088a4e0a49 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -52,13 +52,27 @@
 
 #define MAC_ERROR_BIT		0
 #define CHK_MAC_ERR_BIT(x)	(((x) >> MAC_ERROR_BIT) & 1)
+#define MAX_SALT                4
 
 struct uld_ctx;
 
+struct _key_ctx {
+	__be32 ctx_hdr;
+	u8 salt[MAX_SALT];
+	__be64 reserverd;
+	unsigned char key[0];
+};
+
+struct chcr_wr {
+	struct fw_crypto_lookaside_wr wreq;
+	struct ulp_txpkt ulptx;
+	struct ulptx_idata sc_imm;
+	struct cpl_tx_sec_pdu sec_cpl;
+	struct _key_ctx key_ctx;
+};
+
 struct chcr_dev {
-	/* Request submited to h/w and waiting for response. */
 	spinlock_t lock_chcr_dev;
-	struct crypto_queue pending_queue;
 	struct uld_ctx *u_ctx;
 	unsigned char tx_channel_id;
 };
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
index d7d75605da8b..d5af7d64a763 100644
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -36,6 +36,14 @@
 #ifndef __CHCR_CRYPTO_H__
 #define __CHCR_CRYPTO_H__
 
+#define GHASH_BLOCK_SIZE    16
+#define GHASH_DIGEST_SIZE   16
+
+#define CCM_B0_SIZE             16
+#define CCM_AAD_FIELD_SIZE      2
+#define T5_MAX_AAD_SIZE 512
+
+
 /* Define following if h/w is not dropping the AAD and IV data before
  * giving the processed data
  */
@@ -63,22 +71,36 @@
 #define CHCR_SCMD_AUTH_CTRL_AUTH_CIPHER 0
 #define CHCR_SCMD_AUTH_CTRL_CIPHER_AUTH 1
 
-#define CHCR_SCMD_CIPHER_MODE_NOP           0
-#define CHCR_SCMD_CIPHER_MODE_AES_CBC       1
-#define CHCR_SCMD_CIPHER_MODE_GENERIC_AES   4
-#define CHCR_SCMD_CIPHER_MODE_AES_XTS       6
+#define CHCR_SCMD_CIPHER_MODE_NOP               0
+#define CHCR_SCMD_CIPHER_MODE_AES_CBC           1
+#define CHCR_SCMD_CIPHER_MODE_AES_GCM           2
+#define CHCR_SCMD_CIPHER_MODE_AES_CTR           3
+#define CHCR_SCMD_CIPHER_MODE_GENERIC_AES       4
+#define CHCR_SCMD_CIPHER_MODE_AES_XTS           6
+#define CHCR_SCMD_CIPHER_MODE_AES_CCM           7
 
 #define CHCR_SCMD_AUTH_MODE_NOP             0
 #define CHCR_SCMD_AUTH_MODE_SHA1            1
 #define CHCR_SCMD_AUTH_MODE_SHA224          2
 #define CHCR_SCMD_AUTH_MODE_SHA256          3
+#define CHCR_SCMD_AUTH_MODE_GHASH           4
 #define CHCR_SCMD_AUTH_MODE_SHA512_224      5
 #define CHCR_SCMD_AUTH_MODE_SHA512_256      6
 #define CHCR_SCMD_AUTH_MODE_SHA512_384      7
 #define CHCR_SCMD_AUTH_MODE_SHA512_512      8
+#define CHCR_SCMD_AUTH_MODE_CBCMAC          9
+#define CHCR_SCMD_AUTH_MODE_CMAC            10
 
 #define CHCR_SCMD_HMAC_CTRL_NOP             0
 #define CHCR_SCMD_HMAC_CTRL_NO_TRUNC        1
+#define CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366   2
+#define CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT     3
+#define CHCR_SCMD_HMAC_CTRL_PL1		    4
+#define CHCR_SCMD_HMAC_CTRL_PL2		    5
+#define CHCR_SCMD_HMAC_CTRL_PL3		    6
+#define CHCR_SCMD_HMAC_CTRL_DIV2	    7
+#define VERIFY_HW 0
+#define VERIFY_SW 1
 
 #define CHCR_SCMD_IVGEN_CTRL_HW             0
 #define CHCR_SCMD_IVGEN_CTRL_SW             1
@@ -106,39 +128,74 @@
 #define IV_IMMEDIATE            1
 #define IV_DSGL			2
 
+#define AEAD_H_SIZE             16
+
 #define CRYPTO_ALG_SUB_TYPE_MASK            0x0f000000
 #define CRYPTO_ALG_SUB_TYPE_HASH_HMAC       0x01000000
+#define CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106    0x02000000
+#define CRYPTO_ALG_SUB_TYPE_AEAD_GCM	    0x03000000
+#define CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC    0x04000000
+#define CRYPTO_ALG_SUB_TYPE_AEAD_CCM        0x05000000
+#define CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309    0x06000000
+#define CRYPTO_ALG_SUB_TYPE_AEAD_NULL       0x07000000
+#define CRYPTO_ALG_SUB_TYPE_CTR             0x08000000
 #define CRYPTO_ALG_TYPE_HMAC (CRYPTO_ALG_TYPE_AHASH |\
 			      CRYPTO_ALG_SUB_TYPE_HASH_HMAC)
 
-#define MAX_SALT                4
 #define MAX_SCRATCH_PAD_SIZE    32
 
 #define CHCR_HASH_MAX_BLOCK_SIZE_64  64
 #define CHCR_HASH_MAX_BLOCK_SIZE_128 128
 
 /* Aligned to 128 bit boundary */
-struct _key_ctx {
-	__be32 ctx_hdr;
-	u8 salt[MAX_SALT];
-	__be64 reserverd;
-	unsigned char key[0];
-};
 
 struct ablk_ctx {
-	u8 enc;
-	unsigned int processed_len;
 	__be32 key_ctx_hdr;
 	unsigned int enckey_len;
-	unsigned int dst_nents;
-	struct scatterlist iv_sg;
 	u8 key[CHCR_AES_MAX_KEY_LEN];
-	u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
 	unsigned char ciph_mode;
+	u8 rrkey[AES_MAX_KEY_SIZE];
+};
+struct chcr_aead_reqctx {
+	struct	sk_buff	*skb;
+	short int dst_nents;
+	u16 verify;
+	u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
+	unsigned char scratch_pad[MAX_SCRATCH_PAD_SIZE];
+};
+
+struct chcr_gcm_ctx {
+	u8 ghash_h[AEAD_H_SIZE];
 };
 
+struct chcr_authenc_ctx {
+	u8 dec_rrkey[AES_MAX_KEY_SIZE];
+	u8 h_iopad[2 * CHCR_HASH_MAX_DIGEST_SIZE];
+	unsigned char auth_mode;
+};
+
+struct __aead_ctx {
+	struct chcr_gcm_ctx gcm[0];
+	struct chcr_authenc_ctx authenc[0];
+};
+
+
+
+struct chcr_aead_ctx {
+	__be32 key_ctx_hdr;
+	unsigned int enckey_len;
+	struct crypto_skcipher *null;
+	u8 salt[MAX_SALT];
+	u8 key[CHCR_AES_MAX_KEY_LEN];
+	u16 hmac_ctrl;
+	u16 mayverify;
+	struct	__aead_ctx ctx[0];
+};
+
+
+
 struct hmac_ctx {
-	struct shash_desc *desc;
+	struct crypto_shash *base_hash;
 	u8 ipad[CHCR_HASH_MAX_BLOCK_SIZE_128];
 	u8 opad[CHCR_HASH_MAX_BLOCK_SIZE_128];
 };
@@ -146,6 +203,7 @@ struct hmac_ctx {
 struct __crypto_ctx {
 	struct hmac_ctx hmacctx[0];
 	struct ablk_ctx ablkctx[0];
+	struct chcr_aead_ctx aeadctx[0];
 };
 
 struct chcr_context {
@@ -156,18 +214,22 @@ struct chcr_context {
 
 struct chcr_ahash_req_ctx {
 	u32 result;
-	char bfr[CHCR_HASH_MAX_BLOCK_SIZE_128];
-	u8 bfr_len;
+	u8 bfr1[CHCR_HASH_MAX_BLOCK_SIZE_128];
+	u8 bfr2[CHCR_HASH_MAX_BLOCK_SIZE_128];
+	u8 *reqbfr;
+	u8 *skbfr;
+	u8 reqlen;
 	/* DMA the partial hash in it */
 	u8 partial_hash[CHCR_HASH_MAX_DIGEST_SIZE];
 	u64 data_len;  /* Data len till time */
-	void *dummy_payload_ptr;
 	/* SKB which is being sent to the hardware for processing */
 	struct sk_buff *skb;
 };
 
 struct chcr_blkcipher_req_ctx {
 	struct sk_buff *skb;
+	unsigned int dst_nents;
+	u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
 };
 
 struct chcr_alg_template {
@@ -176,16 +238,19 @@ struct chcr_alg_template {
 	union {
 		struct crypto_alg crypto;
 		struct ahash_alg hash;
+		struct aead_alg aead;
 	} alg;
 };
 
 struct chcr_req_ctx {
 	union {
 		struct ahash_request *ahash_req;
+		struct aead_request *aead_req;
 		struct ablkcipher_request *ablk_req;
 	} req;
 	union {
 		struct chcr_ahash_req_ctx *ahash_ctx;
+		struct chcr_aead_reqctx *reqctx;
 		struct chcr_blkcipher_req_ctx *ablk_ctx;
 	} ctx;
 };
@@ -195,9 +260,15 @@ struct sge_opaque_hdr {
 	dma_addr_t addr[MAX_SKB_FRAGS + 1];
 };
 
-typedef struct sk_buff *(*create_wr_t)(struct crypto_async_request *req,
-				       struct chcr_context *ctx,
+typedef struct sk_buff *(*create_wr_t)(struct aead_request *req,
 				       unsigned short qid,
+				       int size,
 				       unsigned short op_type);
 
+static int chcr_aead_op(struct aead_request *req_base,
+			  unsigned short op_type,
+			  int size,
+			  create_wr_t create_wr_fn);
+static inline int get_aead_subtype(struct crypto_aead *aead);
+
 #endif /* __CHCR_CRYPTO_H__ */
diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c
index 37dadb2a4feb..6e7a5c77a00a 100644
--- a/drivers/crypto/marvell/cesa.c
+++ b/drivers/crypto/marvell/cesa.c
@@ -375,10 +375,6 @@ static int mv_cesa_dev_dma_init(struct mv_cesa_dev *cesa)
 	if (!dma->padding_pool)
 		return -ENOMEM;
 
-	dma->iv_pool = dmam_pool_create("cesa_iv", dev, 16, 1, 0);
-	if (!dma->iv_pool)
-		return -ENOMEM;
-
 	cesa->dma = dma;
 
 	return 0;
diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h
index e423d33decd4..a768da7138a1 100644
--- a/drivers/crypto/marvell/cesa.h
+++ b/drivers/crypto/marvell/cesa.h
@@ -277,7 +277,7 @@ struct mv_cesa_op_ctx {
 #define CESA_TDMA_DUMMY				0
 #define CESA_TDMA_DATA				1
 #define CESA_TDMA_OP				2
-#define CESA_TDMA_IV				3
+#define CESA_TDMA_RESULT			3
 
 /**
  * struct mv_cesa_tdma_desc - TDMA descriptor
@@ -393,7 +393,6 @@ struct mv_cesa_dev_dma {
 	struct dma_pool *op_pool;
 	struct dma_pool *cache_pool;
 	struct dma_pool *padding_pool;
-	struct dma_pool *iv_pool;
 };
 
 /**
@@ -839,7 +838,7 @@ mv_cesa_tdma_desc_iter_init(struct mv_cesa_tdma_chain *chain)
 	memset(chain, 0, sizeof(*chain));
 }
 
-int mv_cesa_dma_add_iv_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src,
+int mv_cesa_dma_add_result_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src,
 			  u32 size, u32 flags, gfp_t gfp_flags);
 
 struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain,
diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c
index d19dc9614e6e..098871a22a54 100644
--- a/drivers/crypto/marvell/cipher.c
+++ b/drivers/crypto/marvell/cipher.c
@@ -212,7 +212,8 @@ mv_cesa_ablkcipher_complete(struct crypto_async_request *req)
 		struct mv_cesa_req *basereq;
 
 		basereq = &creq->base;
-		memcpy(ablkreq->info, basereq->chain.last->data, ivsize);
+		memcpy(ablkreq->info, basereq->chain.last->op->ctx.blkcipher.iv,
+		       ivsize);
 	} else {
 		memcpy_fromio(ablkreq->info,
 			      engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET,
@@ -373,8 +374,9 @@ static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req,
 
 	/* Add output data for IV */
 	ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req));
-	ret = mv_cesa_dma_add_iv_op(&basereq->chain, CESA_SA_CRYPT_IV_SRAM_OFFSET,
-				    ivsize, CESA_TDMA_SRC_IN_SRAM, flags);
+	ret = mv_cesa_dma_add_result_op(&basereq->chain, CESA_SA_CFG_SRAM_OFFSET,
+				    CESA_SA_DATA_SRAM_OFFSET,
+				    CESA_TDMA_SRC_IN_SRAM, flags);
 
 	if (ret)
 		goto err_free_tdma;
diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c
index 77712b375b84..317cf029c0cf 100644
--- a/drivers/crypto/marvell/hash.c
+++ b/drivers/crypto/marvell/hash.c
@@ -311,24 +311,40 @@ static void mv_cesa_ahash_complete(struct crypto_async_request *req)
 	int i;
 
 	digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq));
-	for (i = 0; i < digsize / 4; i++)
-		creq->state[i] = readl_relaxed(engine->regs + CESA_IVDIG(i));
 
-	if (creq->last_req) {
+	if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ &&
+	    (creq->base.chain.last->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_RESULT) {
+		__le32 *data = NULL;
+
 		/*
-		 * Hardware's MD5 digest is in little endian format, but
-		 * SHA in big endian format
+		 * Result is already in the correct endianess when the SA is
+		 * used
 		 */
-		if (creq->algo_le) {
-			__le32 *result = (void *)ahashreq->result;
+		data = creq->base.chain.last->op->ctx.hash.hash;
+		for (i = 0; i < digsize / 4; i++)
+			creq->state[i] = cpu_to_le32(data[i]);
 
-			for (i = 0; i < digsize / 4; i++)
-				result[i] = cpu_to_le32(creq->state[i]);
-		} else {
-			__be32 *result = (void *)ahashreq->result;
+		memcpy(ahashreq->result, data, digsize);
+	} else {
+		for (i = 0; i < digsize / 4; i++)
+			creq->state[i] = readl_relaxed(engine->regs +
+						       CESA_IVDIG(i));
+		if (creq->last_req) {
+			/*
+			* Hardware's MD5 digest is in little endian format, but
+			* SHA in big endian format
+			*/
+			if (creq->algo_le) {
+				__le32 *result = (void *)ahashreq->result;
+
+				for (i = 0; i < digsize / 4; i++)
+					result[i] = cpu_to_le32(creq->state[i]);
+			} else {
+				__be32 *result = (void *)ahashreq->result;
 
-			for (i = 0; i < digsize / 4; i++)
-				result[i] = cpu_to_be32(creq->state[i]);
+				for (i = 0; i < digsize / 4; i++)
+					result[i] = cpu_to_be32(creq->state[i]);
+			}
 		}
 	}
 
@@ -503,6 +519,12 @@ mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain *chain,
 						CESA_SA_DESC_CFG_LAST_FRAG,
 				      CESA_SA_DESC_CFG_FRAG_MSK);
 
+		ret = mv_cesa_dma_add_result_op(chain,
+						CESA_SA_CFG_SRAM_OFFSET,
+						CESA_SA_DATA_SRAM_OFFSET,
+						CESA_TDMA_SRC_IN_SRAM, flags);
+		if (ret)
+			return ERR_PTR(-ENOMEM);
 		return op;
 	}
 
@@ -563,6 +585,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req)
 	struct mv_cesa_op_ctx *op = NULL;
 	unsigned int frag_len;
 	int ret;
+	u32 type;
 
 	basereq->chain.first = NULL;
 	basereq->chain.last = NULL;
@@ -634,7 +657,15 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req)
 		goto err_free_tdma;
 	}
 
-	if (op) {
+	/*
+	 * If results are copied via DMA, this means that this
+	 * request can be directly processed by the engine,
+	 * without partial updates. So we can chain it at the
+	 * DMA level with other requests.
+	 */
+	type = basereq->chain.last->flags & CESA_TDMA_TYPE_MSK;
+
+	if (op && type != CESA_TDMA_RESULT) {
 		/* Add dummy desc to wait for crypto operation end */
 		ret = mv_cesa_dma_add_dummy_end(&basereq->chain, flags);
 		if (ret)
@@ -647,8 +678,10 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req)
 	else
 		creq->cache_ptr = 0;
 
-	basereq->chain.last->flags |= (CESA_TDMA_END_OF_REQ |
-				       CESA_TDMA_BREAK_CHAIN);
+	basereq->chain.last->flags |= CESA_TDMA_END_OF_REQ;
+
+	if (type != CESA_TDMA_RESULT)
+		basereq->chain.last->flags |= CESA_TDMA_BREAK_CHAIN;
 
 	return 0;
 
diff --git a/drivers/crypto/marvell/tdma.c b/drivers/crypto/marvell/tdma.c
index 9fd7a5fbaa1b..4416b88eca70 100644
--- a/drivers/crypto/marvell/tdma.c
+++ b/drivers/crypto/marvell/tdma.c
@@ -69,9 +69,6 @@ void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq)
 		if (type == CESA_TDMA_OP)
 			dma_pool_free(cesa_dev->dma->op_pool, tdma->op,
 				      le32_to_cpu(tdma->src));
-		else if (type == CESA_TDMA_IV)
-			dma_pool_free(cesa_dev->dma->iv_pool, tdma->data,
-				      le32_to_cpu(tdma->dst));
 
 		tdma = tdma->next;
 		dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma,
@@ -209,29 +206,37 @@ mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags)
 	return new_tdma;
 }
 
-int mv_cesa_dma_add_iv_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src,
+int mv_cesa_dma_add_result_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src,
 			  u32 size, u32 flags, gfp_t gfp_flags)
 {
-
-	struct mv_cesa_tdma_desc *tdma;
-	u8 *iv;
-	dma_addr_t dma_handle;
+	struct mv_cesa_tdma_desc *tdma, *op_desc;
 
 	tdma = mv_cesa_dma_add_desc(chain, gfp_flags);
 	if (IS_ERR(tdma))
 		return PTR_ERR(tdma);
 
-	iv = dma_pool_alloc(cesa_dev->dma->iv_pool, gfp_flags, &dma_handle);
-	if (!iv)
-		return -ENOMEM;
+	/* We re-use an existing op_desc object to retrieve the context
+	 * and result instead of allocating a new one.
+	 * There is at least one object of this type in a CESA crypto
+	 * req, just pick the first one in the chain.
+	 */
+	for (op_desc = chain->first; op_desc; op_desc = op_desc->next) {
+		u32 type = op_desc->flags & CESA_TDMA_TYPE_MSK;
+
+		if (type == CESA_TDMA_OP)
+			break;
+	}
+
+	if (!op_desc)
+		return -EIO;
 
 	tdma->byte_cnt = cpu_to_le32(size | BIT(31));
 	tdma->src = src;
-	tdma->dst = cpu_to_le32(dma_handle);
-	tdma->data = iv;
+	tdma->dst = op_desc->src;
+	tdma->op = op_desc->op;
 
 	flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM);
-	tdma->flags = flags | CESA_TDMA_IV;
+	tdma->flags = flags | CESA_TDMA_RESULT;
 	return 0;
 }
 
diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c
index 104e9ce9400a..451fa18c1c7b 100644
--- a/drivers/crypto/mv_cesa.c
+++ b/drivers/crypto/mv_cesa.c
@@ -1073,7 +1073,7 @@ static int mv_probe(struct platform_device *pdev)
 	if (!res)
 		return -ENXIO;
 
-	cp = kzalloc(sizeof(*cp), GFP_KERNEL);
+	cp = devm_kzalloc(&pdev->dev, sizeof(*cp), GFP_KERNEL);
 	if (!cp)
 		return -ENOMEM;
 
@@ -1163,7 +1163,6 @@ err_irq:
 err_thread:
 	kthread_stop(cp->queue_th);
 err:
-	kfree(cp);
 	cpg = NULL;
 	return ret;
 }
@@ -1187,7 +1186,6 @@ static int mv_remove(struct platform_device *pdev)
 		clk_put(cp->clk);
 	}
 
-	kfree(cp);
 	cpg = NULL;
 	return 0;
 }
diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c
index 42f0f229f7f7..036057abb257 100644
--- a/drivers/crypto/nx/nx.c
+++ b/drivers/crypto/nx/nx.c
@@ -32,7 +32,6 @@
 #include <linux/scatterlist.h>
 #include <linux/device.h>
 #include <linux/of.h>
-#include <linux/types.h>
 #include <asm/hvcall.h>
 #include <asm/vio.h>
 
diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c
index 0c49956ee0ce..1d9ecd368b5b 100644
--- a/drivers/crypto/sahara.c
+++ b/drivers/crypto/sahara.c
@@ -390,7 +390,7 @@ static void sahara_decode_status(struct sahara_dev *dev, unsigned int status)
 	if (status & SAHARA_STATUS_MODE_BATCH)
 		dev_dbg(dev->device, "	- Batch Mode.\n");
 	else if (status & SAHARA_STATUS_MODE_DEDICATED)
-		dev_dbg(dev->device, "	- Decidated Mode.\n");
+		dev_dbg(dev->device, "	- Dedicated Mode.\n");
 	else if (status & SAHARA_STATUS_MODE_DEBUG)
 		dev_dbg(dev->device, "	- Debug Mode.\n");
 
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index 0418a2f41dc0..0bba6a19d36a 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -590,7 +590,7 @@ static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
 		if (v_lo & TALITOS_CCPSR_LO_MDTE)
 			dev_err(dev, "master data transfer error\n");
 		if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
-			dev_err(dev, is_sec1 ? "pointeur not complete error\n"
+			dev_err(dev, is_sec1 ? "pointer not complete error\n"
 					     : "s/g data length zero error\n");
 		if (v_lo & TALITOS_CCPSR_LO_FPZ)
 			dev_err(dev, is_sec1 ? "parity error\n"
diff --git a/drivers/crypto/virtio/Kconfig b/drivers/crypto/virtio/Kconfig
new file mode 100644
index 000000000000..d80f73366ae2
--- /dev/null
+++ b/drivers/crypto/virtio/Kconfig
@@ -0,0 +1,10 @@
+config CRYPTO_DEV_VIRTIO
+	tristate "VirtIO crypto driver"
+	depends on VIRTIO
+	select CRYPTO_AEAD
+	select CRYPTO_AUTHENC
+	select CRYPTO_BLKCIPHER
+	default m
+	help
+	  This driver provides support for virtio crypto device. If you
+	  choose 'M' here, this module will be called virtio_crypto.
diff --git a/drivers/crypto/virtio/Makefile b/drivers/crypto/virtio/Makefile
new file mode 100644
index 000000000000..dd342c947ff9
--- /dev/null
+++ b/drivers/crypto/virtio/Makefile
@@ -0,0 +1,5 @@
+obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio_crypto.o
+virtio_crypto-objs := \
+	virtio_crypto_algs.o \
+	virtio_crypto_mgr.o \
+	virtio_crypto_core.o
diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c
new file mode 100644
index 000000000000..c2374df9abae
--- /dev/null
+++ b/drivers/crypto/virtio/virtio_crypto_algs.c
@@ -0,0 +1,540 @@
+ /* Algorithms supported by virtio crypto device
+  *
+  * Authors: Gonglei <arei.gonglei@huawei.com>
+  *
+  * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD.
+  *
+  * This program is free software; you can redistribute it and/or modify
+  * it under the terms of the GNU General Public License as published by
+  * the Free Software Foundation; either version 2 of the License, or
+  * (at your option) any later version.
+  *
+  * This program is distributed in the hope that it will be useful,
+  * but WITHOUT ANY WARRANTY; without even the implied warranty of
+  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  * GNU General Public License for more details.
+  *
+  * You should have received a copy of the GNU General Public License
+  * along with this program; if not, see <http://www.gnu.org/licenses/>.
+  */
+
+#include <linux/scatterlist.h>
+#include <crypto/algapi.h>
+#include <linux/err.h>
+#include <crypto/scatterwalk.h>
+#include <linux/atomic.h>
+
+#include <uapi/linux/virtio_crypto.h>
+#include "virtio_crypto_common.h"
+
+/*
+ * The algs_lock protects the below global virtio_crypto_active_devs
+ * and crypto algorithms registion.
+ */
+static DEFINE_MUTEX(algs_lock);
+static unsigned int virtio_crypto_active_devs;
+
+static u64 virtio_crypto_alg_sg_nents_length(struct scatterlist *sg)
+{
+	u64 total = 0;
+
+	for (total = 0; sg; sg = sg_next(sg))
+		total += sg->length;
+
+	return total;
+}
+
+static int
+virtio_crypto_alg_validate_key(int key_len, uint32_t *alg)
+{
+	switch (key_len) {
+	case AES_KEYSIZE_128:
+	case AES_KEYSIZE_192:
+	case AES_KEYSIZE_256:
+		*alg = VIRTIO_CRYPTO_CIPHER_AES_CBC;
+		break;
+	default:
+		pr_err("virtio_crypto: Unsupported key length: %d\n",
+			key_len);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int virtio_crypto_alg_ablkcipher_init_session(
+		struct virtio_crypto_ablkcipher_ctx *ctx,
+		uint32_t alg, const uint8_t *key,
+		unsigned int keylen,
+		int encrypt)
+{
+	struct scatterlist outhdr, key_sg, inhdr, *sgs[3];
+	unsigned int tmp;
+	struct virtio_crypto *vcrypto = ctx->vcrypto;
+	int op = encrypt ? VIRTIO_CRYPTO_OP_ENCRYPT : VIRTIO_CRYPTO_OP_DECRYPT;
+	int err;
+	unsigned int num_out = 0, num_in = 0;
+
+	/*
+	 * Avoid to do DMA from the stack, switch to using
+	 * dynamically-allocated for the key
+	 */
+	uint8_t *cipher_key = kmalloc(keylen, GFP_ATOMIC);
+
+	if (!cipher_key)
+		return -ENOMEM;
+
+	memcpy(cipher_key, key, keylen);
+
+	spin_lock(&vcrypto->ctrl_lock);
+	/* Pad ctrl header */
+	vcrypto->ctrl.header.opcode =
+		cpu_to_le32(VIRTIO_CRYPTO_CIPHER_CREATE_SESSION);
+	vcrypto->ctrl.header.algo = cpu_to_le32(alg);
+	/* Set the default dataqueue id to 0 */
+	vcrypto->ctrl.header.queue_id = 0;
+
+	vcrypto->input.status = cpu_to_le32(VIRTIO_CRYPTO_ERR);
+	/* Pad cipher's parameters */
+	vcrypto->ctrl.u.sym_create_session.op_type =
+		cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER);
+	vcrypto->ctrl.u.sym_create_session.u.cipher.para.algo =
+		vcrypto->ctrl.header.algo;
+	vcrypto->ctrl.u.sym_create_session.u.cipher.para.keylen =
+		cpu_to_le32(keylen);
+	vcrypto->ctrl.u.sym_create_session.u.cipher.para.op =
+		cpu_to_le32(op);
+
+	sg_init_one(&outhdr, &vcrypto->ctrl, sizeof(vcrypto->ctrl));
+	sgs[num_out++] = &outhdr;
+
+	/* Set key */
+	sg_init_one(&key_sg, cipher_key, keylen);
+	sgs[num_out++] = &key_sg;
+
+	/* Return status and session id back */
+	sg_init_one(&inhdr, &vcrypto->input, sizeof(vcrypto->input));
+	sgs[num_out + num_in++] = &inhdr;
+
+	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out,
+				num_in, vcrypto, GFP_ATOMIC);
+	if (err < 0) {
+		spin_unlock(&vcrypto->ctrl_lock);
+		kzfree(cipher_key);
+		return err;
+	}
+	virtqueue_kick(vcrypto->ctrl_vq);
+
+	/*
+	 * Trapping into the hypervisor, so the request should be
+	 * handled immediately.
+	 */
+	while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) &&
+	       !virtqueue_is_broken(vcrypto->ctrl_vq))
+		cpu_relax();
+
+	if (le32_to_cpu(vcrypto->input.status) != VIRTIO_CRYPTO_OK) {
+		spin_unlock(&vcrypto->ctrl_lock);
+		pr_err("virtio_crypto: Create session failed status: %u\n",
+			le32_to_cpu(vcrypto->input.status));
+		kzfree(cipher_key);
+		return -EINVAL;
+	}
+
+	if (encrypt)
+		ctx->enc_sess_info.session_id =
+			le64_to_cpu(vcrypto->input.session_id);
+	else
+		ctx->dec_sess_info.session_id =
+			le64_to_cpu(vcrypto->input.session_id);
+
+	spin_unlock(&vcrypto->ctrl_lock);
+
+	kzfree(cipher_key);
+	return 0;
+}
+
+static int virtio_crypto_alg_ablkcipher_close_session(
+		struct virtio_crypto_ablkcipher_ctx *ctx,
+		int encrypt)
+{
+	struct scatterlist outhdr, status_sg, *sgs[2];
+	unsigned int tmp;
+	struct virtio_crypto_destroy_session_req *destroy_session;
+	struct virtio_crypto *vcrypto = ctx->vcrypto;
+	int err;
+	unsigned int num_out = 0, num_in = 0;
+
+	spin_lock(&vcrypto->ctrl_lock);
+	vcrypto->ctrl_status.status = VIRTIO_CRYPTO_ERR;
+	/* Pad ctrl header */
+	vcrypto->ctrl.header.opcode =
+		cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION);
+	/* Set the default virtqueue id to 0 */
+	vcrypto->ctrl.header.queue_id = 0;
+
+	destroy_session = &vcrypto->ctrl.u.destroy_session;
+
+	if (encrypt)
+		destroy_session->session_id =
+			cpu_to_le64(ctx->enc_sess_info.session_id);
+	else
+		destroy_session->session_id =
+			cpu_to_le64(ctx->dec_sess_info.session_id);
+
+	sg_init_one(&outhdr, &vcrypto->ctrl, sizeof(vcrypto->ctrl));
+	sgs[num_out++] = &outhdr;
+
+	/* Return status and session id back */
+	sg_init_one(&status_sg, &vcrypto->ctrl_status.status,
+		sizeof(vcrypto->ctrl_status.status));
+	sgs[num_out + num_in++] = &status_sg;
+
+	err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out,
+			num_in, vcrypto, GFP_ATOMIC);
+	if (err < 0) {
+		spin_unlock(&vcrypto->ctrl_lock);
+		return err;
+	}
+	virtqueue_kick(vcrypto->ctrl_vq);
+
+	while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) &&
+	       !virtqueue_is_broken(vcrypto->ctrl_vq))
+		cpu_relax();
+
+	if (vcrypto->ctrl_status.status != VIRTIO_CRYPTO_OK) {
+		spin_unlock(&vcrypto->ctrl_lock);
+		pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n",
+			vcrypto->ctrl_status.status,
+			destroy_session->session_id);
+
+		return -EINVAL;
+	}
+	spin_unlock(&vcrypto->ctrl_lock);
+
+	return 0;
+}
+
+static int virtio_crypto_alg_ablkcipher_init_sessions(
+		struct virtio_crypto_ablkcipher_ctx *ctx,
+		const uint8_t *key, unsigned int keylen)
+{
+	uint32_t alg;
+	int ret;
+	struct virtio_crypto *vcrypto = ctx->vcrypto;
+
+	if (keylen > vcrypto->max_cipher_key_len) {
+		pr_err("virtio_crypto: the key is too long\n");
+		goto bad_key;
+	}
+
+	if (virtio_crypto_alg_validate_key(keylen, &alg))
+		goto bad_key;
+
+	/* Create encryption session */
+	ret = virtio_crypto_alg_ablkcipher_init_session(ctx,
+			alg, key, keylen, 1);
+	if (ret)
+		return ret;
+	/* Create decryption session */
+	ret = virtio_crypto_alg_ablkcipher_init_session(ctx,
+			alg, key, keylen, 0);
+	if (ret) {
+		virtio_crypto_alg_ablkcipher_close_session(ctx, 1);
+		return ret;
+	}
+	return 0;
+
+bad_key:
+	crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+	return -EINVAL;
+}
+
+/* Note: kernel crypto API realization */
+static int virtio_crypto_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
+					 const uint8_t *key,
+					 unsigned int keylen)
+{
+	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+	int ret;
+
+	if (!ctx->vcrypto) {
+		/* New key */
+		int node = virtio_crypto_get_current_node();
+		struct virtio_crypto *vcrypto =
+				      virtcrypto_get_dev_node(node);
+		if (!vcrypto) {
+			pr_err("virtio_crypto: Could not find a virtio device in the system");
+			return -ENODEV;
+		}
+
+		ctx->vcrypto = vcrypto;
+	} else {
+		/* Rekeying, we should close the created sessions previously */
+		virtio_crypto_alg_ablkcipher_close_session(ctx, 1);
+		virtio_crypto_alg_ablkcipher_close_session(ctx, 0);
+	}
+
+	ret = virtio_crypto_alg_ablkcipher_init_sessions(ctx, key, keylen);
+	if (ret) {
+		virtcrypto_dev_put(ctx->vcrypto);
+		ctx->vcrypto = NULL;
+
+		return ret;
+	}
+
+	return 0;
+}
+
+static int
+__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req,
+		struct ablkcipher_request *req,
+		struct data_queue *data_vq,
+		__u8 op)
+{
+	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+	unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
+	struct virtio_crypto_ablkcipher_ctx *ctx = vc_req->ablkcipher_ctx;
+	struct virtio_crypto *vcrypto = ctx->vcrypto;
+	struct virtio_crypto_op_data_req *req_data;
+	int src_nents, dst_nents;
+	int err;
+	unsigned long flags;
+	struct scatterlist outhdr, iv_sg, status_sg, **sgs;
+	int i;
+	u64 dst_len;
+	unsigned int num_out = 0, num_in = 0;
+	int sg_total;
+	uint8_t *iv;
+
+	src_nents = sg_nents_for_len(req->src, req->nbytes);
+	dst_nents = sg_nents(req->dst);
+
+	pr_debug("virtio_crypto: Number of sgs (src_nents: %d, dst_nents: %d)\n",
+			src_nents, dst_nents);
+
+	/* Why 3?  outhdr + iv + inhdr */
+	sg_total = src_nents + dst_nents + 3;
+	sgs = kzalloc_node(sg_total * sizeof(*sgs), GFP_ATOMIC,
+				dev_to_node(&vcrypto->vdev->dev));
+	if (!sgs)
+		return -ENOMEM;
+
+	req_data = kzalloc_node(sizeof(*req_data), GFP_ATOMIC,
+				dev_to_node(&vcrypto->vdev->dev));
+	if (!req_data) {
+		kfree(sgs);
+		return -ENOMEM;
+	}
+
+	vc_req->req_data = req_data;
+	vc_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER;
+	/* Head of operation */
+	if (op) {
+		req_data->header.session_id =
+			cpu_to_le64(ctx->enc_sess_info.session_id);
+		req_data->header.opcode =
+			cpu_to_le32(VIRTIO_CRYPTO_CIPHER_ENCRYPT);
+	} else {
+		req_data->header.session_id =
+			cpu_to_le64(ctx->dec_sess_info.session_id);
+	    req_data->header.opcode =
+			cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DECRYPT);
+	}
+	req_data->u.sym_req.op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER);
+	req_data->u.sym_req.u.cipher.para.iv_len = cpu_to_le32(ivsize);
+	req_data->u.sym_req.u.cipher.para.src_data_len =
+			cpu_to_le32(req->nbytes);
+
+	dst_len = virtio_crypto_alg_sg_nents_length(req->dst);
+	if (unlikely(dst_len > U32_MAX)) {
+		pr_err("virtio_crypto: The dst_len is beyond U32_MAX\n");
+		err = -EINVAL;
+		goto free;
+	}
+
+	pr_debug("virtio_crypto: src_len: %u, dst_len: %llu\n",
+			req->nbytes, dst_len);
+
+	if (unlikely(req->nbytes + dst_len + ivsize +
+		sizeof(vc_req->status) > vcrypto->max_size)) {
+		pr_err("virtio_crypto: The length is too big\n");
+		err = -EINVAL;
+		goto free;
+	}
+
+	req_data->u.sym_req.u.cipher.para.dst_data_len =
+			cpu_to_le32((uint32_t)dst_len);
+
+	/* Outhdr */
+	sg_init_one(&outhdr, req_data, sizeof(*req_data));
+	sgs[num_out++] = &outhdr;
+
+	/* IV */
+
+	/*
+	 * Avoid to do DMA from the stack, switch to using
+	 * dynamically-allocated for the IV
+	 */
+	iv = kzalloc_node(ivsize, GFP_ATOMIC,
+				dev_to_node(&vcrypto->vdev->dev));
+	if (!iv) {
+		err = -ENOMEM;
+		goto free;
+	}
+	memcpy(iv, req->info, ivsize);
+	sg_init_one(&iv_sg, iv, ivsize);
+	sgs[num_out++] = &iv_sg;
+	vc_req->iv = iv;
+
+	/* Source data */
+	for (i = 0; i < src_nents; i++)
+		sgs[num_out++] = &req->src[i];
+
+	/* Destination data */
+	for (i = 0; i < dst_nents; i++)
+		sgs[num_out + num_in++] = &req->dst[i];
+
+	/* Status */
+	sg_init_one(&status_sg, &vc_req->status, sizeof(vc_req->status));
+	sgs[num_out + num_in++] = &status_sg;
+
+	vc_req->sgs = sgs;
+
+	spin_lock_irqsave(&data_vq->lock, flags);
+	err = virtqueue_add_sgs(data_vq->vq, sgs, num_out,
+				num_in, vc_req, GFP_ATOMIC);
+	virtqueue_kick(data_vq->vq);
+	spin_unlock_irqrestore(&data_vq->lock, flags);
+	if (unlikely(err < 0))
+		goto free_iv;
+
+	return 0;
+
+free_iv:
+	kzfree(iv);
+free:
+	kzfree(req_data);
+	kfree(sgs);
+	return err;
+}
+
+static int virtio_crypto_ablkcipher_encrypt(struct ablkcipher_request *req)
+{
+	struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
+	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
+	struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req);
+	struct virtio_crypto *vcrypto = ctx->vcrypto;
+	int ret;
+	/* Use the first data virtqueue as default */
+	struct data_queue *data_vq = &vcrypto->data_vq[0];
+
+	vc_req->ablkcipher_ctx = ctx;
+	vc_req->ablkcipher_req = req;
+	ret = __virtio_crypto_ablkcipher_do_req(vc_req, req, data_vq, 1);
+	if (ret < 0) {
+		pr_err("virtio_crypto: Encryption failed!\n");
+		return ret;
+	}
+
+	return -EINPROGRESS;
+}
+
+static int virtio_crypto_ablkcipher_decrypt(struct ablkcipher_request *req)
+{
+	struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
+	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
+	struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req);
+	struct virtio_crypto *vcrypto = ctx->vcrypto;
+	int ret;
+	/* Use the first data virtqueue as default */
+	struct data_queue *data_vq = &vcrypto->data_vq[0];
+
+	vc_req->ablkcipher_ctx = ctx;
+	vc_req->ablkcipher_req = req;
+
+	ret = __virtio_crypto_ablkcipher_do_req(vc_req, req, data_vq, 0);
+	if (ret < 0) {
+		pr_err("virtio_crypto: Decryption failed!\n");
+		return ret;
+	}
+
+	return -EINPROGRESS;
+}
+
+static int virtio_crypto_ablkcipher_init(struct crypto_tfm *tfm)
+{
+	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	tfm->crt_ablkcipher.reqsize = sizeof(struct virtio_crypto_request);
+	ctx->tfm = tfm;
+
+	return 0;
+}
+
+static void virtio_crypto_ablkcipher_exit(struct crypto_tfm *tfm)
+{
+	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	if (!ctx->vcrypto)
+		return;
+
+	virtio_crypto_alg_ablkcipher_close_session(ctx, 1);
+	virtio_crypto_alg_ablkcipher_close_session(ctx, 0);
+	virtcrypto_dev_put(ctx->vcrypto);
+	ctx->vcrypto = NULL;
+}
+
+static struct crypto_alg virtio_crypto_algs[] = { {
+	.cra_name = "cbc(aes)",
+	.cra_driver_name = "virtio_crypto_aes_cbc",
+	.cra_priority = 501,
+	.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+	.cra_blocksize = AES_BLOCK_SIZE,
+	.cra_ctxsize  = sizeof(struct virtio_crypto_ablkcipher_ctx),
+	.cra_alignmask = 0,
+	.cra_module = THIS_MODULE,
+	.cra_type = &crypto_ablkcipher_type,
+	.cra_init = virtio_crypto_ablkcipher_init,
+	.cra_exit = virtio_crypto_ablkcipher_exit,
+	.cra_u = {
+	   .ablkcipher = {
+			.setkey = virtio_crypto_ablkcipher_setkey,
+			.decrypt = virtio_crypto_ablkcipher_decrypt,
+			.encrypt = virtio_crypto_ablkcipher_encrypt,
+			.min_keysize = AES_MIN_KEY_SIZE,
+			.max_keysize = AES_MAX_KEY_SIZE,
+			.ivsize = AES_BLOCK_SIZE,
+		},
+	},
+} };
+
+int virtio_crypto_algs_register(void)
+{
+	int ret = 0;
+
+	mutex_lock(&algs_lock);
+	if (++virtio_crypto_active_devs != 1)
+		goto unlock;
+
+	ret = crypto_register_algs(virtio_crypto_algs,
+			ARRAY_SIZE(virtio_crypto_algs));
+	if (ret)
+		virtio_crypto_active_devs--;
+
+unlock:
+	mutex_unlock(&algs_lock);
+	return ret;
+}
+
+void virtio_crypto_algs_unregister(void)
+{
+	mutex_lock(&algs_lock);
+	if (--virtio_crypto_active_devs != 0)
+		goto unlock;
+
+	crypto_unregister_algs(virtio_crypto_algs,
+			ARRAY_SIZE(virtio_crypto_algs));
+
+unlock:
+	mutex_unlock(&algs_lock);
+}
diff --git a/drivers/crypto/virtio/virtio_crypto_common.h b/drivers/crypto/virtio/virtio_crypto_common.h
new file mode 100644
index 000000000000..3d6566b02876
--- /dev/null
+++ b/drivers/crypto/virtio/virtio_crypto_common.h
@@ -0,0 +1,128 @@
+/* Common header for Virtio crypto device.
+ *
+ * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _VIRTIO_CRYPTO_COMMON_H
+#define _VIRTIO_CRYPTO_COMMON_H
+
+#include <linux/virtio.h>
+#include <linux/crypto.h>
+#include <linux/spinlock.h>
+#include <crypto/aead.h>
+#include <crypto/aes.h>
+#include <crypto/authenc.h>
+
+
+/* Internal representation of a data virtqueue */
+struct data_queue {
+	/* Virtqueue associated with this send _queue */
+	struct virtqueue *vq;
+
+	/* To protect the vq operations for the dataq */
+	spinlock_t lock;
+
+	/* Name of the tx queue: dataq.$index */
+	char name[32];
+};
+
+struct virtio_crypto {
+	struct virtio_device *vdev;
+	struct virtqueue *ctrl_vq;
+	struct data_queue *data_vq;
+
+	/* To protect the vq operations for the controlq */
+	spinlock_t ctrl_lock;
+
+	/* Maximum of data queues supported by the device */
+	u32 max_data_queues;
+
+	/* Number of queue currently used by the driver */
+	u32 curr_queue;
+
+	/* Maximum length of cipher key */
+	u32 max_cipher_key_len;
+	/* Maximum length of authenticated key */
+	u32 max_auth_key_len;
+	/* Maximum size of per request */
+	u64 max_size;
+
+	/* Control VQ buffers: protected by the ctrl_lock */
+	struct virtio_crypto_op_ctrl_req ctrl;
+	struct virtio_crypto_session_input input;
+	struct virtio_crypto_inhdr ctrl_status;
+
+	unsigned long status;
+	atomic_t ref_count;
+	struct list_head list;
+	struct module *owner;
+	uint8_t dev_id;
+
+	/* Does the affinity hint is set for virtqueues? */
+	bool affinity_hint_set;
+};
+
+struct virtio_crypto_sym_session_info {
+	/* Backend session id, which come from the host side */
+	__u64 session_id;
+};
+
+struct virtio_crypto_ablkcipher_ctx {
+	struct virtio_crypto *vcrypto;
+	struct crypto_tfm *tfm;
+
+	struct virtio_crypto_sym_session_info enc_sess_info;
+	struct virtio_crypto_sym_session_info dec_sess_info;
+};
+
+struct virtio_crypto_request {
+	/* Cipher or aead */
+	uint32_t type;
+	uint8_t status;
+	struct virtio_crypto_ablkcipher_ctx *ablkcipher_ctx;
+	struct ablkcipher_request *ablkcipher_req;
+	struct virtio_crypto_op_data_req *req_data;
+	struct scatterlist **sgs;
+	uint8_t *iv;
+};
+
+int virtcrypto_devmgr_add_dev(struct virtio_crypto *vcrypto_dev);
+struct list_head *virtcrypto_devmgr_get_head(void);
+void virtcrypto_devmgr_rm_dev(struct virtio_crypto *vcrypto_dev);
+struct virtio_crypto *virtcrypto_devmgr_get_first(void);
+int virtcrypto_dev_in_use(struct virtio_crypto *vcrypto_dev);
+int virtcrypto_dev_get(struct virtio_crypto *vcrypto_dev);
+void virtcrypto_dev_put(struct virtio_crypto *vcrypto_dev);
+int virtcrypto_dev_started(struct virtio_crypto *vcrypto_dev);
+struct virtio_crypto *virtcrypto_get_dev_node(int node);
+int virtcrypto_dev_start(struct virtio_crypto *vcrypto);
+void virtcrypto_dev_stop(struct virtio_crypto *vcrypto);
+
+static inline int virtio_crypto_get_current_node(void)
+{
+	int cpu, node;
+
+	cpu = get_cpu();
+	node = topology_physical_package_id(cpu);
+	put_cpu();
+
+	return node;
+}
+
+int virtio_crypto_algs_register(void);
+void virtio_crypto_algs_unregister(void);
+
+#endif /* _VIRTIO_CRYPTO_COMMON_H */
diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c
new file mode 100644
index 000000000000..fe70ec823b27
--- /dev/null
+++ b/drivers/crypto/virtio/virtio_crypto_core.c
@@ -0,0 +1,476 @@
+ /* Driver for Virtio crypto device.
+  *
+  * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD.
+  *
+  * This program is free software; you can redistribute it and/or modify
+  * it under the terms of the GNU General Public License as published by
+  * the Free Software Foundation; either version 2 of the License, or
+  * (at your option) any later version.
+  *
+  * This program is distributed in the hope that it will be useful,
+  * but WITHOUT ANY WARRANTY; without even the implied warranty of
+  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  * GNU General Public License for more details.
+  *
+  * You should have received a copy of the GNU General Public License
+  * along with this program; if not, see <http://www.gnu.org/licenses/>.
+  */
+
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/virtio_config.h>
+#include <linux/cpu.h>
+
+#include <uapi/linux/virtio_crypto.h>
+#include "virtio_crypto_common.h"
+
+
+static void
+virtcrypto_clear_request(struct virtio_crypto_request *vc_req)
+{
+	if (vc_req) {
+		kzfree(vc_req->iv);
+		kzfree(vc_req->req_data);
+		kfree(vc_req->sgs);
+	}
+}
+
+static void virtcrypto_dataq_callback(struct virtqueue *vq)
+{
+	struct virtio_crypto *vcrypto = vq->vdev->priv;
+	struct virtio_crypto_request *vc_req;
+	unsigned long flags;
+	unsigned int len;
+	struct ablkcipher_request *ablk_req;
+	int error;
+	unsigned int qid = vq->index;
+
+	spin_lock_irqsave(&vcrypto->data_vq[qid].lock, flags);
+	do {
+		virtqueue_disable_cb(vq);
+		while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) {
+			if (vc_req->type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
+				switch (vc_req->status) {
+				case VIRTIO_CRYPTO_OK:
+					error = 0;
+					break;
+				case VIRTIO_CRYPTO_INVSESS:
+				case VIRTIO_CRYPTO_ERR:
+					error = -EINVAL;
+					break;
+				case VIRTIO_CRYPTO_BADMSG:
+					error = -EBADMSG;
+					break;
+				default:
+					error = -EIO;
+					break;
+				}
+				ablk_req = vc_req->ablkcipher_req;
+				virtcrypto_clear_request(vc_req);
+
+				spin_unlock_irqrestore(
+					&vcrypto->data_vq[qid].lock, flags);
+				/* Finish the encrypt or decrypt process */
+				ablk_req->base.complete(&ablk_req->base, error);
+				spin_lock_irqsave(
+					&vcrypto->data_vq[qid].lock, flags);
+			}
+		}
+	} while (!virtqueue_enable_cb(vq));
+	spin_unlock_irqrestore(&vcrypto->data_vq[qid].lock, flags);
+}
+
+static int virtcrypto_find_vqs(struct virtio_crypto *vi)
+{
+	vq_callback_t **callbacks;
+	struct virtqueue **vqs;
+	int ret = -ENOMEM;
+	int i, total_vqs;
+	const char **names;
+
+	/*
+	 * We expect 1 data virtqueue, followed by
+	 * possible N-1 data queues used in multiqueue mode,
+	 * followed by control vq.
+	 */
+	total_vqs = vi->max_data_queues + 1;
+
+	/* Allocate space for find_vqs parameters */
+	vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
+	if (!vqs)
+		goto err_vq;
+	callbacks = kcalloc(total_vqs, sizeof(*callbacks), GFP_KERNEL);
+	if (!callbacks)
+		goto err_callback;
+	names = kcalloc(total_vqs, sizeof(*names), GFP_KERNEL);
+	if (!names)
+		goto err_names;
+
+	/* Parameters for control virtqueue */
+	callbacks[total_vqs - 1] = NULL;
+	names[total_vqs - 1] = "controlq";
+
+	/* Allocate/initialize parameters for data virtqueues */
+	for (i = 0; i < vi->max_data_queues; i++) {
+		callbacks[i] = virtcrypto_dataq_callback;
+		snprintf(vi->data_vq[i].name, sizeof(vi->data_vq[i].name),
+				"dataq.%d", i);
+		names[i] = vi->data_vq[i].name;
+	}
+
+	ret = vi->vdev->config->find_vqs(vi->vdev, total_vqs, vqs, callbacks,
+					 names);
+	if (ret)
+		goto err_find;
+
+	vi->ctrl_vq = vqs[total_vqs - 1];
+
+	for (i = 0; i < vi->max_data_queues; i++) {
+		spin_lock_init(&vi->data_vq[i].lock);
+		vi->data_vq[i].vq = vqs[i];
+	}
+
+	kfree(names);
+	kfree(callbacks);
+	kfree(vqs);
+
+	return 0;
+
+err_find:
+	kfree(names);
+err_names:
+	kfree(callbacks);
+err_callback:
+	kfree(vqs);
+err_vq:
+	return ret;
+}
+
+static int virtcrypto_alloc_queues(struct virtio_crypto *vi)
+{
+	vi->data_vq = kcalloc(vi->max_data_queues, sizeof(*vi->data_vq),
+				GFP_KERNEL);
+	if (!vi->data_vq)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void virtcrypto_clean_affinity(struct virtio_crypto *vi, long hcpu)
+{
+	int i;
+
+	if (vi->affinity_hint_set) {
+		for (i = 0; i < vi->max_data_queues; i++)
+			virtqueue_set_affinity(vi->data_vq[i].vq, -1);
+
+		vi->affinity_hint_set = false;
+	}
+}
+
+static void virtcrypto_set_affinity(struct virtio_crypto *vcrypto)
+{
+	int i = 0;
+	int cpu;
+
+	/*
+	 * In single queue mode, we don't set the cpu affinity.
+	 */
+	if (vcrypto->curr_queue == 1 || vcrypto->max_data_queues == 1) {
+		virtcrypto_clean_affinity(vcrypto, -1);
+		return;
+	}
+
+	/*
+	 * In multiqueue mode, we let the queue to be private to one cpu
+	 * by setting the affinity hint to eliminate the contention.
+	 *
+	 * TODO: adds cpu hotplug support by register cpu notifier.
+	 *
+	 */
+	for_each_online_cpu(cpu) {
+		virtqueue_set_affinity(vcrypto->data_vq[i].vq, cpu);
+		if (++i >= vcrypto->max_data_queues)
+			break;
+	}
+
+	vcrypto->affinity_hint_set = true;
+}
+
+static void virtcrypto_free_queues(struct virtio_crypto *vi)
+{
+	kfree(vi->data_vq);
+}
+
+static int virtcrypto_init_vqs(struct virtio_crypto *vi)
+{
+	int ret;
+
+	/* Allocate send & receive queues */
+	ret = virtcrypto_alloc_queues(vi);
+	if (ret)
+		goto err;
+
+	ret = virtcrypto_find_vqs(vi);
+	if (ret)
+		goto err_free;
+
+	get_online_cpus();
+	virtcrypto_set_affinity(vi);
+	put_online_cpus();
+
+	return 0;
+
+err_free:
+	virtcrypto_free_queues(vi);
+err:
+	return ret;
+}
+
+static int virtcrypto_update_status(struct virtio_crypto *vcrypto)
+{
+	u32 status;
+	int err;
+
+	virtio_cread(vcrypto->vdev,
+	    struct virtio_crypto_config, status, &status);
+
+	/*
+	 * Unknown status bits would be a host error and the driver
+	 * should consider the device to be broken.
+	 */
+	if (status & (~VIRTIO_CRYPTO_S_HW_READY)) {
+		dev_warn(&vcrypto->vdev->dev,
+				"Unknown status bits: 0x%x\n", status);
+
+		virtio_break_device(vcrypto->vdev);
+		return -EPERM;
+	}
+
+	if (vcrypto->status == status)
+		return 0;
+
+	vcrypto->status = status;
+
+	if (vcrypto->status & VIRTIO_CRYPTO_S_HW_READY) {
+		err = virtcrypto_dev_start(vcrypto);
+		if (err) {
+			dev_err(&vcrypto->vdev->dev,
+				"Failed to start virtio crypto device.\n");
+
+			return -EPERM;
+		}
+		dev_info(&vcrypto->vdev->dev, "Accelerator is ready\n");
+	} else {
+		virtcrypto_dev_stop(vcrypto);
+		dev_info(&vcrypto->vdev->dev, "Accelerator is not ready\n");
+	}
+
+	return 0;
+}
+
+static void virtcrypto_del_vqs(struct virtio_crypto *vcrypto)
+{
+	struct virtio_device *vdev = vcrypto->vdev;
+
+	virtcrypto_clean_affinity(vcrypto, -1);
+
+	vdev->config->del_vqs(vdev);
+
+	virtcrypto_free_queues(vcrypto);
+}
+
+static int virtcrypto_probe(struct virtio_device *vdev)
+{
+	int err = -EFAULT;
+	struct virtio_crypto *vcrypto;
+	u32 max_data_queues = 0, max_cipher_key_len = 0;
+	u32 max_auth_key_len = 0;
+	u64 max_size = 0;
+
+	if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
+		return -ENODEV;
+
+	if (!vdev->config->get) {
+		dev_err(&vdev->dev, "%s failure: config access disabled\n",
+			__func__);
+		return -EINVAL;
+	}
+
+	if (num_possible_nodes() > 1 && dev_to_node(&vdev->dev) < 0) {
+		/*
+		 * If the accelerator is connected to a node with no memory
+		 * there is no point in using the accelerator since the remote
+		 * memory transaction will be very slow.
+		 */
+		dev_err(&vdev->dev, "Invalid NUMA configuration.\n");
+		return -EINVAL;
+	}
+
+	vcrypto = kzalloc_node(sizeof(*vcrypto), GFP_KERNEL,
+					dev_to_node(&vdev->dev));
+	if (!vcrypto)
+		return -ENOMEM;
+
+	virtio_cread(vdev, struct virtio_crypto_config,
+			max_dataqueues, &max_data_queues);
+	if (max_data_queues < 1)
+		max_data_queues = 1;
+
+	virtio_cread(vdev, struct virtio_crypto_config,
+		max_cipher_key_len, &max_cipher_key_len);
+	virtio_cread(vdev, struct virtio_crypto_config,
+		max_auth_key_len, &max_auth_key_len);
+	virtio_cread(vdev, struct virtio_crypto_config,
+		max_size, &max_size);
+
+	/* Add virtio crypto device to global table */
+	err = virtcrypto_devmgr_add_dev(vcrypto);
+	if (err) {
+		dev_err(&vdev->dev, "Failed to add new virtio crypto device.\n");
+		goto free;
+	}
+	vcrypto->owner = THIS_MODULE;
+	vcrypto = vdev->priv = vcrypto;
+	vcrypto->vdev = vdev;
+
+	spin_lock_init(&vcrypto->ctrl_lock);
+
+	/* Use single data queue as default */
+	vcrypto->curr_queue = 1;
+	vcrypto->max_data_queues = max_data_queues;
+	vcrypto->max_cipher_key_len = max_cipher_key_len;
+	vcrypto->max_auth_key_len = max_auth_key_len;
+	vcrypto->max_size = max_size;
+
+	dev_info(&vdev->dev,
+		"max_queues: %u, max_cipher_key_len: %u, max_auth_key_len: %u, max_size 0x%llx\n",
+		vcrypto->max_data_queues,
+		vcrypto->max_cipher_key_len,
+		vcrypto->max_auth_key_len,
+		vcrypto->max_size);
+
+	err = virtcrypto_init_vqs(vcrypto);
+	if (err) {
+		dev_err(&vdev->dev, "Failed to initialize vqs.\n");
+		goto free_dev;
+	}
+	virtio_device_ready(vdev);
+
+	err = virtcrypto_update_status(vcrypto);
+	if (err)
+		goto free_vqs;
+
+	return 0;
+
+free_vqs:
+	vcrypto->vdev->config->reset(vdev);
+	virtcrypto_del_vqs(vcrypto);
+free_dev:
+	virtcrypto_devmgr_rm_dev(vcrypto);
+free:
+	kfree(vcrypto);
+	return err;
+}
+
+static void virtcrypto_free_unused_reqs(struct virtio_crypto *vcrypto)
+{
+	struct virtio_crypto_request *vc_req;
+	int i;
+	struct virtqueue *vq;
+
+	for (i = 0; i < vcrypto->max_data_queues; i++) {
+		vq = vcrypto->data_vq[i].vq;
+		while ((vc_req = virtqueue_detach_unused_buf(vq)) != NULL) {
+			kfree(vc_req->req_data);
+			kfree(vc_req->sgs);
+		}
+	}
+}
+
+static void virtcrypto_remove(struct virtio_device *vdev)
+{
+	struct virtio_crypto *vcrypto = vdev->priv;
+
+	dev_info(&vdev->dev, "Start virtcrypto_remove.\n");
+
+	if (virtcrypto_dev_started(vcrypto))
+		virtcrypto_dev_stop(vcrypto);
+	vdev->config->reset(vdev);
+	virtcrypto_free_unused_reqs(vcrypto);
+	virtcrypto_del_vqs(vcrypto);
+	virtcrypto_devmgr_rm_dev(vcrypto);
+	kfree(vcrypto);
+}
+
+static void virtcrypto_config_changed(struct virtio_device *vdev)
+{
+	struct virtio_crypto *vcrypto = vdev->priv;
+
+	virtcrypto_update_status(vcrypto);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int virtcrypto_freeze(struct virtio_device *vdev)
+{
+	struct virtio_crypto *vcrypto = vdev->priv;
+
+	vdev->config->reset(vdev);
+	virtcrypto_free_unused_reqs(vcrypto);
+	if (virtcrypto_dev_started(vcrypto))
+		virtcrypto_dev_stop(vcrypto);
+
+	virtcrypto_del_vqs(vcrypto);
+	return 0;
+}
+
+static int virtcrypto_restore(struct virtio_device *vdev)
+{
+	struct virtio_crypto *vcrypto = vdev->priv;
+	int err;
+
+	err = virtcrypto_init_vqs(vcrypto);
+	if (err)
+		return err;
+
+	virtio_device_ready(vdev);
+	err = virtcrypto_dev_start(vcrypto);
+	if (err) {
+		dev_err(&vdev->dev, "Failed to start virtio crypto device.\n");
+		return -EFAULT;
+	}
+
+	return 0;
+}
+#endif
+
+static unsigned int features[] = {
+	/* none */
+};
+
+static struct virtio_device_id id_table[] = {
+	{ VIRTIO_ID_CRYPTO, VIRTIO_DEV_ANY_ID },
+	{ 0 },
+};
+
+static struct virtio_driver virtio_crypto_driver = {
+	.driver.name         = KBUILD_MODNAME,
+	.driver.owner        = THIS_MODULE,
+	.feature_table       = features,
+	.feature_table_size  = ARRAY_SIZE(features),
+	.id_table            = id_table,
+	.probe               = virtcrypto_probe,
+	.remove              = virtcrypto_remove,
+	.config_changed = virtcrypto_config_changed,
+#ifdef CONFIG_PM_SLEEP
+	.freeze = virtcrypto_freeze,
+	.restore = virtcrypto_restore,
+#endif
+};
+
+module_virtio_driver(virtio_crypto_driver);
+
+MODULE_DEVICE_TABLE(virtio, id_table);
+MODULE_DESCRIPTION("virtio crypto device driver");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Gonglei <arei.gonglei@huawei.com>");
diff --git a/drivers/crypto/virtio/virtio_crypto_mgr.c b/drivers/crypto/virtio/virtio_crypto_mgr.c
new file mode 100644
index 000000000000..a69ff71de2c4
--- /dev/null
+++ b/drivers/crypto/virtio/virtio_crypto_mgr.c
@@ -0,0 +1,264 @@
+ /* Management for virtio crypto devices (refer to adf_dev_mgr.c)
+  *
+  * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD.
+  *
+  * This program is free software; you can redistribute it and/or modify
+  * it under the terms of the GNU General Public License as published by
+  * the Free Software Foundation; either version 2 of the License, or
+  * (at your option) any later version.
+  *
+  * This program is distributed in the hope that it will be useful,
+  * but WITHOUT ANY WARRANTY; without even the implied warranty of
+  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  * GNU General Public License for more details.
+  *
+  * You should have received a copy of the GNU General Public License
+  * along with this program; if not, see <http://www.gnu.org/licenses/>.
+  */
+
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/module.h>
+
+#include <uapi/linux/virtio_crypto.h>
+#include "virtio_crypto_common.h"
+
+static LIST_HEAD(virtio_crypto_table);
+static uint32_t num_devices;
+
+/* The table_lock protects the above global list and num_devices */
+static DEFINE_MUTEX(table_lock);
+
+#define VIRTIO_CRYPTO_MAX_DEVICES 32
+
+
+/*
+ * virtcrypto_devmgr_add_dev() - Add vcrypto_dev to the acceleration
+ * framework.
+ * @vcrypto_dev:  Pointer to virtio crypto device.
+ *
+ * Function adds virtio crypto device to the global list.
+ * To be used by virtio crypto device specific drivers.
+ *
+ * Return: 0 on success, error code othewise.
+ */
+int virtcrypto_devmgr_add_dev(struct virtio_crypto *vcrypto_dev)
+{
+	struct list_head *itr;
+
+	mutex_lock(&table_lock);
+	if (num_devices == VIRTIO_CRYPTO_MAX_DEVICES) {
+		pr_info("virtio_crypto: only support up to %d devices\n",
+			    VIRTIO_CRYPTO_MAX_DEVICES);
+		mutex_unlock(&table_lock);
+		return -EFAULT;
+	}
+
+	list_for_each(itr, &virtio_crypto_table) {
+		struct virtio_crypto *ptr =
+				list_entry(itr, struct virtio_crypto, list);
+
+		if (ptr == vcrypto_dev) {
+			mutex_unlock(&table_lock);
+			return -EEXIST;
+		}
+	}
+	atomic_set(&vcrypto_dev->ref_count, 0);
+	list_add_tail(&vcrypto_dev->list, &virtio_crypto_table);
+	vcrypto_dev->dev_id = num_devices++;
+	mutex_unlock(&table_lock);
+	return 0;
+}
+
+struct list_head *virtcrypto_devmgr_get_head(void)
+{
+	return &virtio_crypto_table;
+}
+
+/*
+ * virtcrypto_devmgr_rm_dev() - Remove vcrypto_dev from the acceleration
+ * framework.
+ * @vcrypto_dev:  Pointer to virtio crypto device.
+ *
+ * Function removes virtio crypto device from the acceleration framework.
+ * To be used by virtio crypto device specific drivers.
+ *
+ * Return: void
+ */
+void virtcrypto_devmgr_rm_dev(struct virtio_crypto *vcrypto_dev)
+{
+	mutex_lock(&table_lock);
+	list_del(&vcrypto_dev->list);
+	num_devices--;
+	mutex_unlock(&table_lock);
+}
+
+/*
+ * virtcrypto_devmgr_get_first()
+ *
+ * Function returns the first virtio crypto device from the acceleration
+ * framework.
+ *
+ * To be used by virtio crypto device specific drivers.
+ *
+ * Return: pointer to vcrypto_dev or NULL if not found.
+ */
+struct virtio_crypto *virtcrypto_devmgr_get_first(void)
+{
+	struct virtio_crypto *dev = NULL;
+
+	mutex_lock(&table_lock);
+	if (!list_empty(&virtio_crypto_table))
+		dev = list_first_entry(&virtio_crypto_table,
+					struct virtio_crypto,
+				    list);
+	mutex_unlock(&table_lock);
+	return dev;
+}
+
+/*
+ * virtcrypto_dev_in_use() - Check whether vcrypto_dev is currently in use
+ * @vcrypto_dev: Pointer to virtio crypto device.
+ *
+ * To be used by virtio crypto device specific drivers.
+ *
+ * Return: 1 when device is in use, 0 otherwise.
+ */
+int virtcrypto_dev_in_use(struct virtio_crypto *vcrypto_dev)
+{
+	return atomic_read(&vcrypto_dev->ref_count) != 0;
+}
+
+/*
+ * virtcrypto_dev_get() - Increment vcrypto_dev reference count
+ * @vcrypto_dev: Pointer to virtio crypto device.
+ *
+ * Increment the vcrypto_dev refcount and if this is the first time
+ * incrementing it during this period the vcrypto_dev is in use,
+ * increment the module refcount too.
+ * To be used by virtio crypto device specific drivers.
+ *
+ * Return: 0 when successful, EFAULT when fail to bump module refcount
+ */
+int virtcrypto_dev_get(struct virtio_crypto *vcrypto_dev)
+{
+	if (atomic_add_return(1, &vcrypto_dev->ref_count) == 1)
+		if (!try_module_get(vcrypto_dev->owner))
+			return -EFAULT;
+	return 0;
+}
+
+/*
+ * virtcrypto_dev_put() - Decrement vcrypto_dev reference count
+ * @vcrypto_dev: Pointer to virtio crypto device.
+ *
+ * Decrement the vcrypto_dev refcount and if this is the last time
+ * decrementing it during this period the vcrypto_dev is in use,
+ * decrement the module refcount too.
+ * To be used by virtio crypto device specific drivers.
+ *
+ * Return: void
+ */
+void virtcrypto_dev_put(struct virtio_crypto *vcrypto_dev)
+{
+	if (atomic_sub_return(1, &vcrypto_dev->ref_count) == 0)
+		module_put(vcrypto_dev->owner);
+}
+
+/*
+ * virtcrypto_dev_started() - Check whether device has started
+ * @vcrypto_dev: Pointer to virtio crypto device.
+ *
+ * To be used by virtio crypto device specific drivers.
+ *
+ * Return: 1 when the device has started, 0 otherwise
+ */
+int virtcrypto_dev_started(struct virtio_crypto *vcrypto_dev)
+{
+	return (vcrypto_dev->status & VIRTIO_CRYPTO_S_HW_READY);
+}
+
+/*
+ * virtcrypto_get_dev_node() - Get vcrypto_dev on the node.
+ * @node:  Node id the driver works.
+ *
+ * Function returns the virtio crypto device used fewest on the node.
+ *
+ * To be used by virtio crypto device specific drivers.
+ *
+ * Return: pointer to vcrypto_dev or NULL if not found.
+ */
+struct virtio_crypto *virtcrypto_get_dev_node(int node)
+{
+	struct virtio_crypto *vcrypto_dev = NULL, *tmp_dev;
+	unsigned long best = ~0;
+	unsigned long ctr;
+
+	mutex_lock(&table_lock);
+	list_for_each_entry(tmp_dev, virtcrypto_devmgr_get_head(), list) {
+
+		if ((node == dev_to_node(&tmp_dev->vdev->dev) ||
+		     dev_to_node(&tmp_dev->vdev->dev) < 0) &&
+		    virtcrypto_dev_started(tmp_dev)) {
+			ctr = atomic_read(&tmp_dev->ref_count);
+			if (best > ctr) {
+				vcrypto_dev = tmp_dev;
+				best = ctr;
+			}
+		}
+	}
+
+	if (!vcrypto_dev) {
+		pr_info("virtio_crypto: Could not find a device on node %d\n",
+				node);
+		/* Get any started device */
+		list_for_each_entry(tmp_dev,
+				virtcrypto_devmgr_get_head(), list) {
+			if (virtcrypto_dev_started(tmp_dev)) {
+				vcrypto_dev = tmp_dev;
+				break;
+			}
+		}
+	}
+	mutex_unlock(&table_lock);
+	if (!vcrypto_dev)
+		return NULL;
+
+	virtcrypto_dev_get(vcrypto_dev);
+	return vcrypto_dev;
+}
+
+/*
+ * virtcrypto_dev_start() - Start virtio crypto device
+ * @vcrypto:    Pointer to virtio crypto device.
+ *
+ * Function notifies all the registered services that the virtio crypto device
+ * is ready to be used.
+ * To be used by virtio crypto device specific drivers.
+ *
+ * Return: 0 on success, EFAULT when fail to register algorithms
+ */
+int virtcrypto_dev_start(struct virtio_crypto *vcrypto)
+{
+	if (virtio_crypto_algs_register()) {
+		pr_err("virtio_crypto: Failed to register crypto algs\n");
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/*
+ * virtcrypto_dev_stop() - Stop virtio crypto device
+ * @vcrypto:    Pointer to virtio crypto device.
+ *
+ * Function notifies all the registered services that the virtio crypto device
+ * is ready to be used.
+ * To be used by virtio crypto device specific drivers.
+ *
+ * Return: void
+ */
+void virtcrypto_dev_stop(struct virtio_crypto *vcrypto)
+{
+	virtio_crypto_algs_unregister();
+}
diff --git a/drivers/crypto/vmx/Makefile b/drivers/crypto/vmx/Makefile
index de6e241b0866..55f7c392582f 100644
--- a/drivers/crypto/vmx/Makefile
+++ b/drivers/crypto/vmx/Makefile
@@ -10,10 +10,12 @@ endif
 quiet_cmd_perl = PERL $@
       cmd_perl = $(PERL) $(<) $(TARGET) > $(@)
 
-$(src)/aesp8-ppc.S: $(src)/aesp8-ppc.pl
-	$(call cmd,perl)
+targets += aesp8-ppc.S ghashp8-ppc.S
+
+$(obj)/aesp8-ppc.S: $(src)/aesp8-ppc.pl FORCE
+	$(call if_changed,perl)
   
-$(src)/ghashp8-ppc.S: $(src)/ghashp8-ppc.pl
-	$(call cmd,perl)
+$(obj)/ghashp8-ppc.S: $(src)/ghashp8-ppc.pl FORCE
+	$(call if_changed,perl)
 
-.PRECIOUS: $(obj)/aesp8-ppc.S $(obj)/ghashp8-ppc.S
+clean-files := aesp8-ppc.S ghashp8-ppc.S
diff --git a/drivers/dax/dax.c b/drivers/dax/dax.c
index 286447a83dab..26ec39ddf21f 100644
--- a/drivers/dax/dax.c
+++ b/drivers/dax/dax.c
@@ -328,7 +328,6 @@ static phys_addr_t pgoff_to_phys(struct dax_dev *dax_dev, pgoff_t pgoff,
 static int __dax_dev_fault(struct dax_dev *dax_dev, struct vm_area_struct *vma,
 		struct vm_fault *vmf)
 {
-	unsigned long vaddr = (unsigned long) vmf->virtual_address;
 	struct device *dev = &dax_dev->dev;
 	struct dax_region *dax_region;
 	int rc = VM_FAULT_SIGBUS;
@@ -353,7 +352,7 @@ static int __dax_dev_fault(struct dax_dev *dax_dev, struct vm_area_struct *vma,
 
 	pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
 
-	rc = vm_insert_mixed(vma, vaddr, pfn);
+	rc = vm_insert_mixed(vma, vmf->address, pfn);
 
 	if (rc == -ENOMEM)
 		return VM_FAULT_OOM;
diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c
index bf3ea7603a58..a324801d6a66 100644
--- a/drivers/devfreq/devfreq.c
+++ b/drivers/devfreq/devfreq.c
@@ -850,7 +850,7 @@ err_out:
 EXPORT_SYMBOL(devfreq_add_governor);
 
 /**
- * devfreq_remove_device() - Remove devfreq feature from a device.
+ * devfreq_remove_governor() - Remove devfreq feature from a device.
  * @governor:	the devfreq governor to be removed
  */
 int devfreq_remove_governor(struct devfreq_governor *governor)
diff --git a/drivers/devfreq/event/exynos-nocp.c b/drivers/devfreq/event/exynos-nocp.c
index 49e712aca0c1..5c3e7b11e8a6 100644
--- a/drivers/devfreq/event/exynos-nocp.c
+++ b/drivers/devfreq/event/exynos-nocp.c
@@ -190,6 +190,7 @@ static const struct of_device_id exynos_nocp_id_match[] = {
 	{ .compatible = "samsung,exynos5420-nocp", },
 	{ /* sentinel */ },
 };
+MODULE_DEVICE_TABLE(of, exynos_nocp_id_match);
 
 static struct regmap_config exynos_nocp_regmap_config = {
 	.reg_bits = 32,
diff --git a/drivers/devfreq/event/exynos-ppmu.c b/drivers/devfreq/event/exynos-ppmu.c
index f55cf0eb2a66..107eb91a9415 100644
--- a/drivers/devfreq/event/exynos-ppmu.c
+++ b/drivers/devfreq/event/exynos-ppmu.c
@@ -15,7 +15,6 @@
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/mutex.h>
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
 #include <linux/suspend.h>
@@ -34,7 +33,6 @@ struct exynos_ppmu {
 	unsigned int num_events;
 
 	struct device *dev;
-	struct mutex lock;
 
 	struct exynos_ppmu_data ppmu;
 };
@@ -90,8 +88,6 @@ struct __exynos_ppmu_events {
 	PPMU_EVENT(d1-cpu),
 	PPMU_EVENT(d1-general),
 	PPMU_EVENT(d1-rt),
-
-	{ /* sentinel */ },
 };
 
 static int exynos_ppmu_find_ppmu_id(struct devfreq_event_dev *edev)
@@ -351,6 +347,7 @@ static const struct of_device_id exynos_ppmu_id_match[] = {
 	},
 	{ /* sentinel */ },
 };
+MODULE_DEVICE_TABLE(of, exynos_ppmu_id_match);
 
 static struct devfreq_event_ops *exynos_bus_get_ops(struct device_node *np)
 {
@@ -463,7 +460,6 @@ static int exynos_ppmu_probe(struct platform_device *pdev)
 	if (!info)
 		return -ENOMEM;
 
-	mutex_init(&info->lock);
 	info->dev = &pdev->dev;
 
 	/* Parse dt data to get resource */
diff --git a/drivers/devfreq/event/rockchip-dfi.c b/drivers/devfreq/event/rockchip-dfi.c
index 43fcc5a7f515..22b113363ffc 100644
--- a/drivers/devfreq/event/rockchip-dfi.c
+++ b/drivers/devfreq/event/rockchip-dfi.c
@@ -188,6 +188,7 @@ static const struct of_device_id rockchip_dfi_id_match[] = {
 	{ .compatible = "rockchip,rk3399-dfi" },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, rockchip_dfi_id_match);
 
 static int rockchip_dfi_probe(struct platform_device *pdev)
 {
diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c
index 29866f7e6d7e..a8ed7792ece2 100644
--- a/drivers/devfreq/exynos-bus.c
+++ b/drivers/devfreq/exynos-bus.c
@@ -35,7 +35,7 @@ struct exynos_bus {
 	unsigned int edev_count;
 	struct mutex lock;
 
-	struct dev_pm_opp *curr_opp;
+	unsigned long curr_freq;
 
 	struct regulator *regulator;
 	struct clk *clk;
@@ -99,7 +99,7 @@ static int exynos_bus_target(struct device *dev, unsigned long *freq, u32 flags)
 {
 	struct exynos_bus *bus = dev_get_drvdata(dev);
 	struct dev_pm_opp *new_opp;
-	unsigned long old_freq, new_freq, old_volt, new_volt, tol;
+	unsigned long old_freq, new_freq, new_volt, tol;
 	int ret = 0;
 
 	/* Get new opp-bus instance according to new bus clock */
@@ -113,8 +113,7 @@ static int exynos_bus_target(struct device *dev, unsigned long *freq, u32 flags)
 
 	new_freq = dev_pm_opp_get_freq(new_opp);
 	new_volt = dev_pm_opp_get_voltage(new_opp);
-	old_freq = dev_pm_opp_get_freq(bus->curr_opp);
-	old_volt = dev_pm_opp_get_voltage(bus->curr_opp);
+	old_freq = bus->curr_freq;
 	rcu_read_unlock();
 
 	if (old_freq == new_freq)
@@ -146,7 +145,7 @@ static int exynos_bus_target(struct device *dev, unsigned long *freq, u32 flags)
 			goto out;
 		}
 	}
-	bus->curr_opp = new_opp;
+	bus->curr_freq = new_freq;
 
 	dev_dbg(dev, "Set the frequency of bus (%lukHz -> %lukHz)\n",
 			old_freq/1000, new_freq/1000);
@@ -163,9 +162,7 @@ static int exynos_bus_get_dev_status(struct device *dev,
 	struct devfreq_event_data edata;
 	int ret;
 
-	rcu_read_lock();
-	stat->current_frequency = dev_pm_opp_get_freq(bus->curr_opp);
-	rcu_read_unlock();
+	stat->current_frequency = bus->curr_freq;
 
 	ret = exynos_bus_get_event(bus, &edata);
 	if (ret < 0) {
@@ -226,7 +223,7 @@ static int exynos_bus_passive_target(struct device *dev, unsigned long *freq,
 	}
 
 	new_freq = dev_pm_opp_get_freq(new_opp);
-	old_freq = dev_pm_opp_get_freq(bus->curr_opp);
+	old_freq = bus->curr_freq;
 	rcu_read_unlock();
 
 	if (old_freq == new_freq)
@@ -242,7 +239,7 @@ static int exynos_bus_passive_target(struct device *dev, unsigned long *freq,
 	}
 
 	*freq = new_freq;
-	bus->curr_opp = new_opp;
+	bus->curr_freq = new_freq;
 
 	dev_dbg(dev, "Set the frequency of bus (%lukHz -> %lukHz)\n",
 			old_freq/1000, new_freq/1000);
@@ -335,6 +332,7 @@ static int exynos_bus_parse_of(struct device_node *np,
 			      struct exynos_bus *bus)
 {
 	struct device *dev = bus->dev;
+	struct dev_pm_opp *opp;
 	unsigned long rate;
 	int ret;
 
@@ -352,22 +350,23 @@ static int exynos_bus_parse_of(struct device_node *np,
 	}
 
 	/* Get the freq and voltage from OPP table to scale the bus freq */
-	rcu_read_lock();
 	ret = dev_pm_opp_of_add_table(dev);
 	if (ret < 0) {
 		dev_err(dev, "failed to get OPP table\n");
-		rcu_read_unlock();
 		goto err_clk;
 	}
 
 	rate = clk_get_rate(bus->clk);
-	bus->curr_opp = devfreq_recommended_opp(dev, &rate, 0);
-	if (IS_ERR(bus->curr_opp)) {
+
+	rcu_read_lock();
+	opp = devfreq_recommended_opp(dev, &rate, 0);
+	if (IS_ERR(opp)) {
 		dev_err(dev, "failed to find dev_pm_opp\n");
 		rcu_read_unlock();
-		ret = PTR_ERR(bus->curr_opp);
+		ret = PTR_ERR(opp);
 		goto err_opp;
 	}
+	bus->curr_freq = dev_pm_opp_get_freq(opp);
 	rcu_read_unlock();
 
 	return 0;
diff --git a/drivers/devfreq/rk3399_dmc.c b/drivers/devfreq/rk3399_dmc.c
index e24b73d66659..27d2f349b53c 100644
--- a/drivers/devfreq/rk3399_dmc.c
+++ b/drivers/devfreq/rk3399_dmc.c
@@ -80,7 +80,6 @@ struct rk3399_dmcfreq {
 	struct regulator *vdd_center;
 	unsigned long rate, target_rate;
 	unsigned long volt, target_volt;
-	struct dev_pm_opp *curr_opp;
 };
 
 static int rk3399_dmcfreq_target(struct device *dev, unsigned long *freq,
@@ -102,9 +101,6 @@ static int rk3399_dmcfreq_target(struct device *dev, unsigned long *freq,
 	target_rate = dev_pm_opp_get_freq(opp);
 	target_volt = dev_pm_opp_get_voltage(opp);
 
-	dmcfreq->rate = dev_pm_opp_get_freq(dmcfreq->curr_opp);
-	dmcfreq->volt = dev_pm_opp_get_voltage(dmcfreq->curr_opp);
-
 	rcu_read_unlock();
 
 	if (dmcfreq->rate == target_rate)
@@ -165,7 +161,9 @@ static int rk3399_dmcfreq_target(struct device *dev, unsigned long *freq,
 	if (err)
 		dev_err(dev, "Cannot to set vol %lu uV\n", target_volt);
 
-	dmcfreq->curr_opp = opp;
+	dmcfreq->rate = target_rate;
+	dmcfreq->volt = target_volt;
+
 out:
 	mutex_unlock(&dmcfreq->lock);
 	return err;
@@ -414,7 +412,6 @@ static int rk3399_dmcfreq_probe(struct platform_device *pdev)
 	 */
 	if (dev_pm_opp_of_add_table(dev)) {
 		dev_err(dev, "Invalid operating-points in device tree.\n");
-		rcu_read_unlock();
 		return -EINVAL;
 	}
 
@@ -431,12 +428,13 @@ static int rk3399_dmcfreq_probe(struct platform_device *pdev)
 		rcu_read_unlock();
 		return PTR_ERR(opp);
 	}
+	data->rate = dev_pm_opp_get_freq(opp);
+	data->volt = dev_pm_opp_get_voltage(opp);
 	rcu_read_unlock();
-	data->curr_opp = opp;
 
 	rk3399_devfreq_dmc_profile.initial_freq = data->rate;
 
-	data->devfreq = devfreq_add_device(dev,
+	data->devfreq = devm_devfreq_add_device(dev,
 					   &rk3399_devfreq_dmc_profile,
 					   "simple_ondemand",
 					   &data->ondemand_data);
@@ -454,6 +452,7 @@ static const struct of_device_id rk3399dmc_devfreq_of_match[] = {
 	{ .compatible = "rockchip,rk3399-dmc" },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, rk3399dmc_devfreq_of_match);
 
 static struct platform_driver rk3399_dmcfreq_driver = {
 	.probe	= rk3399_dmcfreq_probe,
diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig
index 2585821b24ab..ed3b785bae37 100644
--- a/drivers/dma-buf/Kconfig
+++ b/drivers/dma-buf/Kconfig
@@ -7,7 +7,7 @@ config SYNC_FILE
 	select DMA_SHARED_BUFFER
 	---help---
 	  The Sync File Framework adds explicit syncronization via
-	  userspace. It enables send/receive 'struct fence' objects to/from
+	  userspace. It enables send/receive 'struct dma_fence' objects to/from
 	  userspace via Sync File fds for synchronization between drivers via
 	  userspace components. It has been ported from Android.
 
diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
index 210a10bfad2b..c33bf8863147 100644
--- a/drivers/dma-buf/Makefile
+++ b/drivers/dma-buf/Makefile
@@ -1,3 +1,3 @@
-obj-y := dma-buf.o fence.o reservation.o seqno-fence.o fence-array.o
+obj-y := dma-buf.o dma-fence.o dma-fence-array.o reservation.o seqno-fence.o
 obj-$(CONFIG_SYNC_FILE)		+= sync_file.o
 obj-$(CONFIG_SW_SYNC)		+= sw_sync.o sync_debug.o
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index cf04d249a6a4..e72e64484131 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -25,7 +25,7 @@
 #include <linux/fs.h>
 #include <linux/slab.h>
 #include <linux/dma-buf.h>
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 #include <linux/anon_inodes.h>
 #include <linux/export.h>
 #include <linux/debugfs.h>
@@ -124,7 +124,7 @@ static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence)
 	return base + offset;
 }
 
-static void dma_buf_poll_cb(struct fence *fence, struct fence_cb *cb)
+static void dma_buf_poll_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 {
 	struct dma_buf_poll_cb_t *dcb = (struct dma_buf_poll_cb_t *)cb;
 	unsigned long flags;
@@ -140,7 +140,7 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll)
 	struct dma_buf *dmabuf;
 	struct reservation_object *resv;
 	struct reservation_object_list *fobj;
-	struct fence *fence_excl;
+	struct dma_fence *fence_excl;
 	unsigned long events;
 	unsigned shared_count, seq;
 
@@ -187,20 +187,20 @@ retry:
 		spin_unlock_irq(&dmabuf->poll.lock);
 
 		if (events & pevents) {
-			if (!fence_get_rcu(fence_excl)) {
+			if (!dma_fence_get_rcu(fence_excl)) {
 				/* force a recheck */
 				events &= ~pevents;
 				dma_buf_poll_cb(NULL, &dcb->cb);
-			} else if (!fence_add_callback(fence_excl, &dcb->cb,
-						       dma_buf_poll_cb)) {
+			} else if (!dma_fence_add_callback(fence_excl, &dcb->cb,
+							   dma_buf_poll_cb)) {
 				events &= ~pevents;
-				fence_put(fence_excl);
+				dma_fence_put(fence_excl);
 			} else {
 				/*
 				 * No callback queued, wake up any additional
 				 * waiters.
 				 */
-				fence_put(fence_excl);
+				dma_fence_put(fence_excl);
 				dma_buf_poll_cb(NULL, &dcb->cb);
 			}
 		}
@@ -222,9 +222,9 @@ retry:
 			goto out;
 
 		for (i = 0; i < shared_count; ++i) {
-			struct fence *fence = rcu_dereference(fobj->shared[i]);
+			struct dma_fence *fence = rcu_dereference(fobj->shared[i]);
 
-			if (!fence_get_rcu(fence)) {
+			if (!dma_fence_get_rcu(fence)) {
 				/*
 				 * fence refcount dropped to zero, this means
 				 * that fobj has been freed
@@ -235,13 +235,13 @@ retry:
 				dma_buf_poll_cb(NULL, &dcb->cb);
 				break;
 			}
-			if (!fence_add_callback(fence, &dcb->cb,
-						dma_buf_poll_cb)) {
-				fence_put(fence);
+			if (!dma_fence_add_callback(fence, &dcb->cb,
+						    dma_buf_poll_cb)) {
+				dma_fence_put(fence);
 				events &= ~POLLOUT;
 				break;
 			}
-			fence_put(fence);
+			dma_fence_put(fence);
 		}
 
 		/* No callback queued, wake up any additional waiters. */
diff --git a/drivers/dma-buf/fence-array.c b/drivers/dma-buf/dma-fence-array.c
index f1989fcaf354..67eb7c8fb88c 100644
--- a/drivers/dma-buf/fence-array.c
+++ b/drivers/dma-buf/dma-fence-array.c
@@ -1,5 +1,5 @@
 /*
- * fence-array: aggregate fences to be waited together
+ * dma-fence-array: aggregate fences to be waited together
  *
  * Copyright (C) 2016 Collabora Ltd
  * Copyright (C) 2016 Advanced Micro Devices, Inc.
@@ -19,35 +19,34 @@
 
 #include <linux/export.h>
 #include <linux/slab.h>
-#include <linux/fence-array.h>
+#include <linux/dma-fence-array.h>
 
-static void fence_array_cb_func(struct fence *f, struct fence_cb *cb);
-
-static const char *fence_array_get_driver_name(struct fence *fence)
+static const char *dma_fence_array_get_driver_name(struct dma_fence *fence)
 {
-	return "fence_array";
+	return "dma_fence_array";
 }
 
-static const char *fence_array_get_timeline_name(struct fence *fence)
+static const char *dma_fence_array_get_timeline_name(struct dma_fence *fence)
 {
 	return "unbound";
 }
 
-static void fence_array_cb_func(struct fence *f, struct fence_cb *cb)
+static void dma_fence_array_cb_func(struct dma_fence *f,
+				    struct dma_fence_cb *cb)
 {
-	struct fence_array_cb *array_cb =
-		container_of(cb, struct fence_array_cb, cb);
-	struct fence_array *array = array_cb->array;
+	struct dma_fence_array_cb *array_cb =
+		container_of(cb, struct dma_fence_array_cb, cb);
+	struct dma_fence_array *array = array_cb->array;
 
 	if (atomic_dec_and_test(&array->num_pending))
-		fence_signal(&array->base);
-	fence_put(&array->base);
+		dma_fence_signal(&array->base);
+	dma_fence_put(&array->base);
 }
 
-static bool fence_array_enable_signaling(struct fence *fence)
+static bool dma_fence_array_enable_signaling(struct dma_fence *fence)
 {
-	struct fence_array *array = to_fence_array(fence);
-	struct fence_array_cb *cb = (void *)(&array[1]);
+	struct dma_fence_array *array = to_dma_fence_array(fence);
+	struct dma_fence_array_cb *cb = (void *)(&array[1]);
 	unsigned i;
 
 	for (i = 0; i < array->num_fences; ++i) {
@@ -60,10 +59,10 @@ static bool fence_array_enable_signaling(struct fence *fence)
 		 * until we signal the array as complete (but that is now
 		 * insufficient).
 		 */
-		fence_get(&array->base);
-		if (fence_add_callback(array->fences[i], &cb[i].cb,
-				       fence_array_cb_func)) {
-			fence_put(&array->base);
+		dma_fence_get(&array->base);
+		if (dma_fence_add_callback(array->fences[i], &cb[i].cb,
+					   dma_fence_array_cb_func)) {
+			dma_fence_put(&array->base);
 			if (atomic_dec_and_test(&array->num_pending))
 				return false;
 		}
@@ -72,69 +71,71 @@ static bool fence_array_enable_signaling(struct fence *fence)
 	return true;
 }
 
-static bool fence_array_signaled(struct fence *fence)
+static bool dma_fence_array_signaled(struct dma_fence *fence)
 {
-	struct fence_array *array = to_fence_array(fence);
+	struct dma_fence_array *array = to_dma_fence_array(fence);
 
 	return atomic_read(&array->num_pending) <= 0;
 }
 
-static void fence_array_release(struct fence *fence)
+static void dma_fence_array_release(struct dma_fence *fence)
 {
-	struct fence_array *array = to_fence_array(fence);
+	struct dma_fence_array *array = to_dma_fence_array(fence);
 	unsigned i;
 
 	for (i = 0; i < array->num_fences; ++i)
-		fence_put(array->fences[i]);
+		dma_fence_put(array->fences[i]);
 
 	kfree(array->fences);
-	fence_free(fence);
+	dma_fence_free(fence);
 }
 
-const struct fence_ops fence_array_ops = {
-	.get_driver_name = fence_array_get_driver_name,
-	.get_timeline_name = fence_array_get_timeline_name,
-	.enable_signaling = fence_array_enable_signaling,
-	.signaled = fence_array_signaled,
-	.wait = fence_default_wait,
-	.release = fence_array_release,
+const struct dma_fence_ops dma_fence_array_ops = {
+	.get_driver_name = dma_fence_array_get_driver_name,
+	.get_timeline_name = dma_fence_array_get_timeline_name,
+	.enable_signaling = dma_fence_array_enable_signaling,
+	.signaled = dma_fence_array_signaled,
+	.wait = dma_fence_default_wait,
+	.release = dma_fence_array_release,
 };
-EXPORT_SYMBOL(fence_array_ops);
+EXPORT_SYMBOL(dma_fence_array_ops);
 
 /**
- * fence_array_create - Create a custom fence array
+ * dma_fence_array_create - Create a custom fence array
  * @num_fences:		[in]	number of fences to add in the array
  * @fences:		[in]	array containing the fences
  * @context:		[in]	fence context to use
  * @seqno:		[in]	sequence number to use
  * @signal_on_any:	[in]	signal on any fence in the array
  *
- * Allocate a fence_array object and initialize the base fence with fence_init().
+ * Allocate a dma_fence_array object and initialize the base fence with
+ * dma_fence_init().
  * In case of error it returns NULL.
  *
  * The caller should allocate the fences array with num_fences size
  * and fill it with the fences it wants to add to the object. Ownership of this
- * array is taken and fence_put() is used on each fence on release.
+ * array is taken and dma_fence_put() is used on each fence on release.
  *
  * If @signal_on_any is true the fence array signals if any fence in the array
  * signals, otherwise it signals when all fences in the array signal.
  */
-struct fence_array *fence_array_create(int num_fences, struct fence **fences,
-				       u64 context, unsigned seqno,
-				       bool signal_on_any)
+struct dma_fence_array *dma_fence_array_create(int num_fences,
+					       struct dma_fence **fences,
+					       u64 context, unsigned seqno,
+					       bool signal_on_any)
 {
-	struct fence_array *array;
+	struct dma_fence_array *array;
 	size_t size = sizeof(*array);
 
 	/* Allocate the callback structures behind the array. */
-	size += num_fences * sizeof(struct fence_array_cb);
+	size += num_fences * sizeof(struct dma_fence_array_cb);
 	array = kzalloc(size, GFP_KERNEL);
 	if (!array)
 		return NULL;
 
 	spin_lock_init(&array->lock);
-	fence_init(&array->base, &fence_array_ops, &array->lock,
-		   context, seqno);
+	dma_fence_init(&array->base, &dma_fence_array_ops, &array->lock,
+		       context, seqno);
 
 	array->num_fences = num_fences;
 	atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences);
@@ -142,4 +143,4 @@ struct fence_array *fence_array_create(int num_fences, struct fence **fences,
 
 	return array;
 }
-EXPORT_SYMBOL(fence_array_create);
+EXPORT_SYMBOL(dma_fence_array_create);
diff --git a/drivers/dma-buf/fence.c b/drivers/dma-buf/dma-fence.c
index 4d51f9e83fa8..0212af7997d9 100644
--- a/drivers/dma-buf/fence.c
+++ b/drivers/dma-buf/dma-fence.c
@@ -21,13 +21,13 @@
 #include <linux/slab.h>
 #include <linux/export.h>
 #include <linux/atomic.h>
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 
 #define CREATE_TRACE_POINTS
-#include <trace/events/fence.h>
+#include <trace/events/dma_fence.h>
 
-EXPORT_TRACEPOINT_SYMBOL(fence_annotate_wait_on);
-EXPORT_TRACEPOINT_SYMBOL(fence_emit);
+EXPORT_TRACEPOINT_SYMBOL(dma_fence_annotate_wait_on);
+EXPORT_TRACEPOINT_SYMBOL(dma_fence_emit);
 
 /*
  * fence context counter: each execution context should have its own
@@ -35,39 +35,41 @@ EXPORT_TRACEPOINT_SYMBOL(fence_emit);
  * context or not. One device can have multiple separate contexts,
  * and they're used if some engine can run independently of another.
  */
-static atomic64_t fence_context_counter = ATOMIC64_INIT(0);
+static atomic64_t dma_fence_context_counter = ATOMIC64_INIT(0);
 
 /**
- * fence_context_alloc - allocate an array of fence contexts
+ * dma_fence_context_alloc - allocate an array of fence contexts
  * @num:	[in]	amount of contexts to allocate
  *
  * This function will return the first index of the number of fences allocated.
  * The fence context is used for setting fence->context to a unique number.
  */
-u64 fence_context_alloc(unsigned num)
+u64 dma_fence_context_alloc(unsigned num)
 {
 	BUG_ON(!num);
-	return atomic64_add_return(num, &fence_context_counter) - num;
+	return atomic64_add_return(num, &dma_fence_context_counter) - num;
 }
-EXPORT_SYMBOL(fence_context_alloc);
+EXPORT_SYMBOL(dma_fence_context_alloc);
 
 /**
- * fence_signal_locked - signal completion of a fence
+ * dma_fence_signal_locked - signal completion of a fence
  * @fence: the fence to signal
  *
  * Signal completion for software callbacks on a fence, this will unblock
- * fence_wait() calls and run all the callbacks added with
- * fence_add_callback(). Can be called multiple times, but since a fence
+ * dma_fence_wait() calls and run all the callbacks added with
+ * dma_fence_add_callback(). Can be called multiple times, but since a fence
  * can only go from unsignaled to signaled state, it will only be effective
  * the first time.
  *
- * Unlike fence_signal, this function must be called with fence->lock held.
+ * Unlike dma_fence_signal, this function must be called with fence->lock held.
  */
-int fence_signal_locked(struct fence *fence)
+int dma_fence_signal_locked(struct dma_fence *fence)
 {
-	struct fence_cb *cur, *tmp;
+	struct dma_fence_cb *cur, *tmp;
 	int ret = 0;
 
+	lockdep_assert_held(fence->lock);
+
 	if (WARN_ON(!fence))
 		return -EINVAL;
 
@@ -76,15 +78,15 @@ int fence_signal_locked(struct fence *fence)
 		smp_mb__before_atomic();
 	}
 
-	if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
+	if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
 		ret = -EINVAL;
 
 		/*
-		 * we might have raced with the unlocked fence_signal,
+		 * we might have raced with the unlocked dma_fence_signal,
 		 * still run through all callbacks
 		 */
 	} else
-		trace_fence_signaled(fence);
+		trace_dma_fence_signaled(fence);
 
 	list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {
 		list_del_init(&cur->node);
@@ -92,19 +94,19 @@ int fence_signal_locked(struct fence *fence)
 	}
 	return ret;
 }
-EXPORT_SYMBOL(fence_signal_locked);
+EXPORT_SYMBOL(dma_fence_signal_locked);
 
 /**
- * fence_signal - signal completion of a fence
+ * dma_fence_signal - signal completion of a fence
  * @fence: the fence to signal
  *
  * Signal completion for software callbacks on a fence, this will unblock
- * fence_wait() calls and run all the callbacks added with
- * fence_add_callback(). Can be called multiple times, but since a fence
+ * dma_fence_wait() calls and run all the callbacks added with
+ * dma_fence_add_callback(). Can be called multiple times, but since a fence
  * can only go from unsignaled to signaled state, it will only be effective
  * the first time.
  */
-int fence_signal(struct fence *fence)
+int dma_fence_signal(struct dma_fence *fence)
 {
 	unsigned long flags;
 
@@ -116,13 +118,13 @@ int fence_signal(struct fence *fence)
 		smp_mb__before_atomic();
 	}
 
-	if (test_and_set_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+	if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
 		return -EINVAL;
 
-	trace_fence_signaled(fence);
+	trace_dma_fence_signaled(fence);
 
-	if (test_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) {
-		struct fence_cb *cur, *tmp;
+	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) {
+		struct dma_fence_cb *cur, *tmp;
 
 		spin_lock_irqsave(fence->lock, flags);
 		list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {
@@ -133,10 +135,10 @@ int fence_signal(struct fence *fence)
 	}
 	return 0;
 }
-EXPORT_SYMBOL(fence_signal);
+EXPORT_SYMBOL(dma_fence_signal);
 
 /**
- * fence_wait_timeout - sleep until the fence gets signaled
+ * dma_fence_wait_timeout - sleep until the fence gets signaled
  * or until timeout elapses
  * @fence:	[in]	the fence to wait on
  * @intr:	[in]	if true, do an interruptible wait
@@ -152,78 +154,76 @@ EXPORT_SYMBOL(fence_signal);
  * freed before return, resulting in undefined behavior.
  */
 signed long
-fence_wait_timeout(struct fence *fence, bool intr, signed long timeout)
+dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
 {
 	signed long ret;
 
 	if (WARN_ON(timeout < 0))
 		return -EINVAL;
 
-	if (timeout == 0)
-		return fence_is_signaled(fence);
-
-	trace_fence_wait_start(fence);
+	trace_dma_fence_wait_start(fence);
 	ret = fence->ops->wait(fence, intr, timeout);
-	trace_fence_wait_end(fence);
+	trace_dma_fence_wait_end(fence);
 	return ret;
 }
-EXPORT_SYMBOL(fence_wait_timeout);
+EXPORT_SYMBOL(dma_fence_wait_timeout);
 
-void fence_release(struct kref *kref)
+void dma_fence_release(struct kref *kref)
 {
-	struct fence *fence =
-			container_of(kref, struct fence, refcount);
+	struct dma_fence *fence =
+		container_of(kref, struct dma_fence, refcount);
 
-	trace_fence_destroy(fence);
+	trace_dma_fence_destroy(fence);
 
 	BUG_ON(!list_empty(&fence->cb_list));
 
 	if (fence->ops->release)
 		fence->ops->release(fence);
 	else
-		fence_free(fence);
+		dma_fence_free(fence);
 }
-EXPORT_SYMBOL(fence_release);
+EXPORT_SYMBOL(dma_fence_release);
 
-void fence_free(struct fence *fence)
+void dma_fence_free(struct dma_fence *fence)
 {
 	kfree_rcu(fence, rcu);
 }
-EXPORT_SYMBOL(fence_free);
+EXPORT_SYMBOL(dma_fence_free);
 
 /**
- * fence_enable_sw_signaling - enable signaling on fence
+ * dma_fence_enable_sw_signaling - enable signaling on fence
  * @fence:	[in]	the fence to enable
  *
  * this will request for sw signaling to be enabled, to make the fence
  * complete as soon as possible
  */
-void fence_enable_sw_signaling(struct fence *fence)
+void dma_fence_enable_sw_signaling(struct dma_fence *fence)
 {
 	unsigned long flags;
 
-	if (!test_and_set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags) &&
-	    !test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
-		trace_fence_enable_signal(fence);
+	if (!test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
+			      &fence->flags) &&
+	    !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
+		trace_dma_fence_enable_signal(fence);
 
 		spin_lock_irqsave(fence->lock, flags);
 
 		if (!fence->ops->enable_signaling(fence))
-			fence_signal_locked(fence);
+			dma_fence_signal_locked(fence);
 
 		spin_unlock_irqrestore(fence->lock, flags);
 	}
 }
-EXPORT_SYMBOL(fence_enable_sw_signaling);
+EXPORT_SYMBOL(dma_fence_enable_sw_signaling);
 
 /**
- * fence_add_callback - add a callback to be called when the fence
+ * dma_fence_add_callback - add a callback to be called when the fence
  * is signaled
  * @fence:	[in]	the fence to wait on
  * @cb:		[in]	the callback to register
  * @func:	[in]	the function to call
  *
- * cb will be initialized by fence_add_callback, no initialization
+ * cb will be initialized by dma_fence_add_callback, no initialization
  * by the caller is required. Any number of callbacks can be registered
  * to a fence, but a callback can only be registered to one fence at a time.
  *
@@ -232,15 +232,15 @@ EXPORT_SYMBOL(fence_enable_sw_signaling);
  * *not* call the callback)
  *
  * Add a software callback to the fence. Same restrictions apply to
- * refcount as it does to fence_wait, however the caller doesn't need to
+ * refcount as it does to dma_fence_wait, however the caller doesn't need to
  * keep a refcount to fence afterwards: when software access is enabled,
  * the creator of the fence is required to keep the fence alive until
- * after it signals with fence_signal. The callback itself can be called
+ * after it signals with dma_fence_signal. The callback itself can be called
  * from irq context.
  *
  */
-int fence_add_callback(struct fence *fence, struct fence_cb *cb,
-		       fence_func_t func)
+int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
+			   dma_fence_func_t func)
 {
 	unsigned long flags;
 	int ret = 0;
@@ -249,22 +249,23 @@ int fence_add_callback(struct fence *fence, struct fence_cb *cb,
 	if (WARN_ON(!fence || !func))
 		return -EINVAL;
 
-	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
 		INIT_LIST_HEAD(&cb->node);
 		return -ENOENT;
 	}
 
 	spin_lock_irqsave(fence->lock, flags);
 
-	was_set = test_and_set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags);
+	was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
+				   &fence->flags);
 
-	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
 		ret = -ENOENT;
 	else if (!was_set) {
-		trace_fence_enable_signal(fence);
+		trace_dma_fence_enable_signal(fence);
 
 		if (!fence->ops->enable_signaling(fence)) {
-			fence_signal_locked(fence);
+			dma_fence_signal_locked(fence);
 			ret = -ENOENT;
 		}
 	}
@@ -278,10 +279,10 @@ int fence_add_callback(struct fence *fence, struct fence_cb *cb,
 
 	return ret;
 }
-EXPORT_SYMBOL(fence_add_callback);
+EXPORT_SYMBOL(dma_fence_add_callback);
 
 /**
- * fence_remove_callback - remove a callback from the signaling list
+ * dma_fence_remove_callback - remove a callback from the signaling list
  * @fence:	[in]	the fence to wait on
  * @cb:		[in]	the callback to remove
  *
@@ -296,7 +297,7 @@ EXPORT_SYMBOL(fence_add_callback);
  * with a reference held to the fence.
  */
 bool
-fence_remove_callback(struct fence *fence, struct fence_cb *cb)
+dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
 {
 	unsigned long flags;
 	bool ret;
@@ -311,15 +312,15 @@ fence_remove_callback(struct fence *fence, struct fence_cb *cb)
 
 	return ret;
 }
-EXPORT_SYMBOL(fence_remove_callback);
+EXPORT_SYMBOL(dma_fence_remove_callback);
 
 struct default_wait_cb {
-	struct fence_cb base;
+	struct dma_fence_cb base;
 	struct task_struct *task;
 };
 
 static void
-fence_default_wait_cb(struct fence *fence, struct fence_cb *cb)
+dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 {
 	struct default_wait_cb *wait =
 		container_of(cb, struct default_wait_cb, base);
@@ -328,25 +329,27 @@ fence_default_wait_cb(struct fence *fence, struct fence_cb *cb)
 }
 
 /**
- * fence_default_wait - default sleep until the fence gets signaled
+ * dma_fence_default_wait - default sleep until the fence gets signaled
  * or until timeout elapses
  * @fence:	[in]	the fence to wait on
  * @intr:	[in]	if true, do an interruptible wait
  * @timeout:	[in]	timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
  *
  * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
- * remaining timeout in jiffies on success.
+ * remaining timeout in jiffies on success. If timeout is zero the value one is
+ * returned if the fence is already signaled for consistency with other
+ * functions taking a jiffies timeout.
  */
 signed long
-fence_default_wait(struct fence *fence, bool intr, signed long timeout)
+dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
 {
 	struct default_wait_cb cb;
 	unsigned long flags;
-	signed long ret = timeout;
+	signed long ret = timeout ? timeout : 1;
 	bool was_set;
 
-	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
-		return timeout;
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+		return ret;
 
 	spin_lock_irqsave(fence->lock, flags);
 
@@ -355,25 +358,26 @@ fence_default_wait(struct fence *fence, bool intr, signed long timeout)
 		goto out;
 	}
 
-	was_set = test_and_set_bit(FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags);
+	was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
+				   &fence->flags);
 
-	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
 		goto out;
 
 	if (!was_set) {
-		trace_fence_enable_signal(fence);
+		trace_dma_fence_enable_signal(fence);
 
 		if (!fence->ops->enable_signaling(fence)) {
-			fence_signal_locked(fence);
+			dma_fence_signal_locked(fence);
 			goto out;
 		}
 	}
 
-	cb.base.func = fence_default_wait_cb;
+	cb.base.func = dma_fence_default_wait_cb;
 	cb.task = current;
 	list_add(&cb.base.node, &fence->cb_list);
 
-	while (!test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {
+	while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {
 		if (intr)
 			__set_current_state(TASK_INTERRUPTIBLE);
 		else
@@ -395,28 +399,34 @@ out:
 	spin_unlock_irqrestore(fence->lock, flags);
 	return ret;
 }
-EXPORT_SYMBOL(fence_default_wait);
+EXPORT_SYMBOL(dma_fence_default_wait);
 
 static bool
-fence_test_signaled_any(struct fence **fences, uint32_t count)
+dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count,
+			    uint32_t *idx)
 {
 	int i;
 
 	for (i = 0; i < count; ++i) {
-		struct fence *fence = fences[i];
-		if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+		struct dma_fence *fence = fences[i];
+		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
+			if (idx)
+				*idx = i;
 			return true;
+		}
 	}
 	return false;
 }
 
 /**
- * fence_wait_any_timeout - sleep until any fence gets signaled
+ * dma_fence_wait_any_timeout - sleep until any fence gets signaled
  * or until timeout elapses
  * @fences:	[in]	array of fences to wait on
  * @count:	[in]	number of fences to wait on
  * @intr:	[in]	if true, do an interruptible wait
  * @timeout:	[in]	timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
+ * @idx:       [out]	the first signaled fence index, meaningful only on
+ *			positive return
  *
  * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
  * interrupted, 0 if the wait timed out, or the remaining timeout in jiffies
@@ -427,8 +437,8 @@ fence_test_signaled_any(struct fence **fences, uint32_t count)
  * fence might be freed before return, resulting in undefined behavior.
  */
 signed long
-fence_wait_any_timeout(struct fence **fences, uint32_t count,
-		       bool intr, signed long timeout)
+dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count,
+			   bool intr, signed long timeout, uint32_t *idx)
 {
 	struct default_wait_cb *cb;
 	signed long ret = timeout;
@@ -439,8 +449,11 @@ fence_wait_any_timeout(struct fence **fences, uint32_t count,
 
 	if (timeout == 0) {
 		for (i = 0; i < count; ++i)
-			if (fence_is_signaled(fences[i]))
+			if (dma_fence_is_signaled(fences[i])) {
+				if (idx)
+					*idx = i;
 				return 1;
+			}
 
 		return 0;
 	}
@@ -452,17 +465,19 @@ fence_wait_any_timeout(struct fence **fences, uint32_t count,
 	}
 
 	for (i = 0; i < count; ++i) {
-		struct fence *fence = fences[i];
+		struct dma_fence *fence = fences[i];
 
-		if (fence->ops->wait != fence_default_wait) {
+		if (fence->ops->wait != dma_fence_default_wait) {
 			ret = -EINVAL;
 			goto fence_rm_cb;
 		}
 
 		cb[i].task = current;
-		if (fence_add_callback(fence, &cb[i].base,
-				       fence_default_wait_cb)) {
+		if (dma_fence_add_callback(fence, &cb[i].base,
+					   dma_fence_default_wait_cb)) {
 			/* This fence is already signaled */
+			if (idx)
+				*idx = i;
 			goto fence_rm_cb;
 		}
 	}
@@ -473,7 +488,7 @@ fence_wait_any_timeout(struct fence **fences, uint32_t count,
 		else
 			set_current_state(TASK_UNINTERRUPTIBLE);
 
-		if (fence_test_signaled_any(fences, count))
+		if (dma_fence_test_signaled_any(fences, count, idx))
 			break;
 
 		ret = schedule_timeout(ret);
@@ -486,34 +501,34 @@ fence_wait_any_timeout(struct fence **fences, uint32_t count,
 
 fence_rm_cb:
 	while (i-- > 0)
-		fence_remove_callback(fences[i], &cb[i].base);
+		dma_fence_remove_callback(fences[i], &cb[i].base);
 
 err_free_cb:
 	kfree(cb);
 
 	return ret;
 }
-EXPORT_SYMBOL(fence_wait_any_timeout);
+EXPORT_SYMBOL(dma_fence_wait_any_timeout);
 
 /**
- * fence_init - Initialize a custom fence.
+ * dma_fence_init - Initialize a custom fence.
  * @fence:	[in]	the fence to initialize
- * @ops:	[in]	the fence_ops for operations on this fence
+ * @ops:	[in]	the dma_fence_ops for operations on this fence
  * @lock:	[in]	the irqsafe spinlock to use for locking this fence
  * @context:	[in]	the execution context this fence is run on
  * @seqno:	[in]	a linear increasing sequence number for this context
  *
  * Initializes an allocated fence, the caller doesn't have to keep its
  * refcount after committing with this fence, but it will need to hold a
- * refcount again if fence_ops.enable_signaling gets called. This can
+ * refcount again if dma_fence_ops.enable_signaling gets called. This can
  * be used for other implementing other types of fence.
  *
  * context and seqno are used for easy comparison between fences, allowing
- * to check which fence is later by simply using fence_later.
+ * to check which fence is later by simply using dma_fence_later.
  */
 void
-fence_init(struct fence *fence, const struct fence_ops *ops,
-	     spinlock_t *lock, u64 context, unsigned seqno)
+dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
+	       spinlock_t *lock, u64 context, unsigned seqno)
 {
 	BUG_ON(!lock);
 	BUG_ON(!ops || !ops->wait || !ops->enable_signaling ||
@@ -527,6 +542,6 @@ fence_init(struct fence *fence, const struct fence_ops *ops,
 	fence->seqno = seqno;
 	fence->flags = 0UL;
 
-	trace_fence_init(fence);
+	trace_dma_fence_init(fence);
 }
-EXPORT_SYMBOL(fence_init);
+EXPORT_SYMBOL(dma_fence_init);
diff --git a/drivers/dma-buf/reservation.c b/drivers/dma-buf/reservation.c
index 723d8af988e5..393817e849ed 100644
--- a/drivers/dma-buf/reservation.c
+++ b/drivers/dma-buf/reservation.c
@@ -102,17 +102,17 @@ EXPORT_SYMBOL(reservation_object_reserve_shared);
 static void
 reservation_object_add_shared_inplace(struct reservation_object *obj,
 				      struct reservation_object_list *fobj,
-				      struct fence *fence)
+				      struct dma_fence *fence)
 {
 	u32 i;
 
-	fence_get(fence);
+	dma_fence_get(fence);
 
 	preempt_disable();
 	write_seqcount_begin(&obj->seq);
 
 	for (i = 0; i < fobj->shared_count; ++i) {
-		struct fence *old_fence;
+		struct dma_fence *old_fence;
 
 		old_fence = rcu_dereference_protected(fobj->shared[i],
 						reservation_object_held(obj));
@@ -123,7 +123,7 @@ reservation_object_add_shared_inplace(struct reservation_object *obj,
 			write_seqcount_end(&obj->seq);
 			preempt_enable();
 
-			fence_put(old_fence);
+			dma_fence_put(old_fence);
 			return;
 		}
 	}
@@ -143,12 +143,12 @@ static void
 reservation_object_add_shared_replace(struct reservation_object *obj,
 				      struct reservation_object_list *old,
 				      struct reservation_object_list *fobj,
-				      struct fence *fence)
+				      struct dma_fence *fence)
 {
 	unsigned i;
-	struct fence *old_fence = NULL;
+	struct dma_fence *old_fence = NULL;
 
-	fence_get(fence);
+	dma_fence_get(fence);
 
 	if (!old) {
 		RCU_INIT_POINTER(fobj->shared[0], fence);
@@ -165,7 +165,7 @@ reservation_object_add_shared_replace(struct reservation_object *obj,
 	fobj->shared_count = old->shared_count;
 
 	for (i = 0; i < old->shared_count; ++i) {
-		struct fence *check;
+		struct dma_fence *check;
 
 		check = rcu_dereference_protected(old->shared[i],
 						reservation_object_held(obj));
@@ -196,7 +196,7 @@ done:
 		kfree_rcu(old, rcu);
 
 	if (old_fence)
-		fence_put(old_fence);
+		dma_fence_put(old_fence);
 }
 
 /**
@@ -208,7 +208,7 @@ done:
  * reservation_object_reserve_shared() has been called.
  */
 void reservation_object_add_shared_fence(struct reservation_object *obj,
-					 struct fence *fence)
+					 struct dma_fence *fence)
 {
 	struct reservation_object_list *old, *fobj = obj->staged;
 
@@ -231,9 +231,9 @@ EXPORT_SYMBOL(reservation_object_add_shared_fence);
  * Add a fence to the exclusive slot.  The obj->lock must be held.
  */
 void reservation_object_add_excl_fence(struct reservation_object *obj,
-				       struct fence *fence)
+				       struct dma_fence *fence)
 {
-	struct fence *old_fence = reservation_object_get_excl(obj);
+	struct dma_fence *old_fence = reservation_object_get_excl(obj);
 	struct reservation_object_list *old;
 	u32 i = 0;
 
@@ -242,7 +242,7 @@ void reservation_object_add_excl_fence(struct reservation_object *obj,
 		i = old->shared_count;
 
 	if (fence)
-		fence_get(fence);
+		dma_fence_get(fence);
 
 	preempt_disable();
 	write_seqcount_begin(&obj->seq);
@@ -255,11 +255,11 @@ void reservation_object_add_excl_fence(struct reservation_object *obj,
 
 	/* inplace update, no shared fences */
 	while (i--)
-		fence_put(rcu_dereference_protected(old->shared[i],
+		dma_fence_put(rcu_dereference_protected(old->shared[i],
 						reservation_object_held(obj)));
 
 	if (old_fence)
-		fence_put(old_fence);
+		dma_fence_put(old_fence);
 }
 EXPORT_SYMBOL(reservation_object_add_excl_fence);
 
@@ -276,26 +276,32 @@ EXPORT_SYMBOL(reservation_object_add_excl_fence);
  * Zero or -errno
  */
 int reservation_object_get_fences_rcu(struct reservation_object *obj,
-				      struct fence **pfence_excl,
+				      struct dma_fence **pfence_excl,
 				      unsigned *pshared_count,
-				      struct fence ***pshared)
+				      struct dma_fence ***pshared)
 {
-	unsigned shared_count = 0;
-	unsigned retry = 1;
-	struct fence **shared = NULL, *fence_excl = NULL;
-	int ret = 0;
+	struct dma_fence **shared = NULL;
+	struct dma_fence *fence_excl;
+	unsigned int shared_count;
+	int ret = 1;
 
-	while (retry) {
+	do {
 		struct reservation_object_list *fobj;
 		unsigned seq;
+		unsigned int i;
 
-		seq = read_seqcount_begin(&obj->seq);
+		shared_count = i = 0;
 
 		rcu_read_lock();
+		seq = read_seqcount_begin(&obj->seq);
+
+		fence_excl = rcu_dereference(obj->fence_excl);
+		if (fence_excl && !dma_fence_get_rcu(fence_excl))
+			goto unlock;
 
 		fobj = rcu_dereference(obj->fence);
 		if (fobj) {
-			struct fence **nshared;
+			struct dma_fence **nshared;
 			size_t sz = sizeof(*shared) * fobj->shared_max;
 
 			nshared = krealloc(shared, sz,
@@ -309,52 +315,37 @@ int reservation_object_get_fences_rcu(struct reservation_object *obj,
 				}
 
 				ret = -ENOMEM;
-				shared_count = 0;
 				break;
 			}
 			shared = nshared;
-			memcpy(shared, fobj->shared, sz);
 			shared_count = fobj->shared_count;
-		} else
-			shared_count = 0;
-		fence_excl = rcu_dereference(obj->fence_excl);
-
-		retry = read_seqcount_retry(&obj->seq, seq);
-		if (retry)
-			goto unlock;
-
-		if (!fence_excl || fence_get_rcu(fence_excl)) {
-			unsigned i;
 
 			for (i = 0; i < shared_count; ++i) {
-				if (fence_get_rcu(shared[i]))
-					continue;
-
-				/* uh oh, refcount failed, abort and retry */
-				while (i--)
-					fence_put(shared[i]);
-
-				if (fence_excl) {
-					fence_put(fence_excl);
-					fence_excl = NULL;
-				}
-
-				retry = 1;
-				break;
+				shared[i] = rcu_dereference(fobj->shared[i]);
+				if (!dma_fence_get_rcu(shared[i]))
+					break;
 			}
-		} else
-			retry = 1;
+		}
 
+		if (i != shared_count || read_seqcount_retry(&obj->seq, seq)) {
+			while (i--)
+				dma_fence_put(shared[i]);
+			dma_fence_put(fence_excl);
+			goto unlock;
+		}
+
+		ret = 0;
 unlock:
 		rcu_read_unlock();
-	}
-	*pshared_count = shared_count;
-	if (shared_count)
-		*pshared = shared;
-	else {
-		*pshared = NULL;
+	} while (ret);
+
+	if (!shared_count) {
 		kfree(shared);
+		shared = NULL;
 	}
+
+	*pshared_count = shared_count;
+	*pshared = shared;
 	*pfence_excl = fence_excl;
 
 	return ret;
@@ -377,12 +368,9 @@ long reservation_object_wait_timeout_rcu(struct reservation_object *obj,
 					 bool wait_all, bool intr,
 					 unsigned long timeout)
 {
-	struct fence *fence;
+	struct dma_fence *fence;
 	unsigned seq, shared_count, i = 0;
-	long ret = timeout;
-
-	if (!timeout)
-		return reservation_object_test_signaled_rcu(obj, wait_all);
+	long ret = timeout ? timeout : 1;
 
 retry:
 	fence = NULL;
@@ -397,20 +385,18 @@ retry:
 		if (fobj)
 			shared_count = fobj->shared_count;
 
-		if (read_seqcount_retry(&obj->seq, seq))
-			goto unlock_retry;
-
 		for (i = 0; i < shared_count; ++i) {
-			struct fence *lfence = rcu_dereference(fobj->shared[i]);
+			struct dma_fence *lfence = rcu_dereference(fobj->shared[i]);
 
-			if (test_bit(FENCE_FLAG_SIGNALED_BIT, &lfence->flags))
+			if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+				     &lfence->flags))
 				continue;
 
-			if (!fence_get_rcu(lfence))
+			if (!dma_fence_get_rcu(lfence))
 				goto unlock_retry;
 
-			if (fence_is_signaled(lfence)) {
-				fence_put(lfence);
+			if (dma_fence_is_signaled(lfence)) {
+				dma_fence_put(lfence);
 				continue;
 			}
 
@@ -420,18 +406,16 @@ retry:
 	}
 
 	if (!shared_count) {
-		struct fence *fence_excl = rcu_dereference(obj->fence_excl);
-
-		if (read_seqcount_retry(&obj->seq, seq))
-			goto unlock_retry;
+		struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl);
 
 		if (fence_excl &&
-		    !test_bit(FENCE_FLAG_SIGNALED_BIT, &fence_excl->flags)) {
-			if (!fence_get_rcu(fence_excl))
+		    !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+			      &fence_excl->flags)) {
+			if (!dma_fence_get_rcu(fence_excl))
 				goto unlock_retry;
 
-			if (fence_is_signaled(fence_excl))
-				fence_put(fence_excl);
+			if (dma_fence_is_signaled(fence_excl))
+				dma_fence_put(fence_excl);
 			else
 				fence = fence_excl;
 		}
@@ -439,8 +423,13 @@ retry:
 
 	rcu_read_unlock();
 	if (fence) {
-		ret = fence_wait_timeout(fence, intr, ret);
-		fence_put(fence);
+		if (read_seqcount_retry(&obj->seq, seq)) {
+			dma_fence_put(fence);
+			goto retry;
+		}
+
+		ret = dma_fence_wait_timeout(fence, intr, ret);
+		dma_fence_put(fence);
 		if (ret > 0 && wait_all && (i + 1 < shared_count))
 			goto retry;
 	}
@@ -454,18 +443,18 @@ EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu);
 
 
 static inline int
-reservation_object_test_signaled_single(struct fence *passed_fence)
+reservation_object_test_signaled_single(struct dma_fence *passed_fence)
 {
-	struct fence *fence, *lfence = passed_fence;
+	struct dma_fence *fence, *lfence = passed_fence;
 	int ret = 1;
 
-	if (!test_bit(FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) {
-		fence = fence_get_rcu(lfence);
+	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) {
+		fence = dma_fence_get_rcu(lfence);
 		if (!fence)
 			return -1;
 
-		ret = !!fence_is_signaled(fence);
-		fence_put(fence);
+		ret = !!dma_fence_is_signaled(fence);
+		dma_fence_put(fence);
 	}
 	return ret;
 }
@@ -484,12 +473,13 @@ bool reservation_object_test_signaled_rcu(struct reservation_object *obj,
 					  bool test_all)
 {
 	unsigned seq, shared_count;
-	int ret = true;
+	int ret;
 
+	rcu_read_lock();
 retry:
+	ret = true;
 	shared_count = 0;
 	seq = read_seqcount_begin(&obj->seq);
-	rcu_read_lock();
 
 	if (test_all) {
 		unsigned i;
@@ -500,46 +490,35 @@ retry:
 		if (fobj)
 			shared_count = fobj->shared_count;
 
-		if (read_seqcount_retry(&obj->seq, seq))
-			goto unlock_retry;
-
 		for (i = 0; i < shared_count; ++i) {
-			struct fence *fence = rcu_dereference(fobj->shared[i]);
+			struct dma_fence *fence = rcu_dereference(fobj->shared[i]);
 
 			ret = reservation_object_test_signaled_single(fence);
 			if (ret < 0)
-				goto unlock_retry;
+				goto retry;
 			else if (!ret)
 				break;
 		}
 
-		/*
-		 * There could be a read_seqcount_retry here, but nothing cares
-		 * about whether it's the old or newer fence pointers that are
-		 * signaled. That race could still have happened after checking
-		 * read_seqcount_retry. If you care, use ww_mutex_lock.
-		 */
+		if (read_seqcount_retry(&obj->seq, seq))
+			goto retry;
 	}
 
 	if (!shared_count) {
-		struct fence *fence_excl = rcu_dereference(obj->fence_excl);
-
-		if (read_seqcount_retry(&obj->seq, seq))
-			goto unlock_retry;
+		struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl);
 
 		if (fence_excl) {
 			ret = reservation_object_test_signaled_single(
 								fence_excl);
 			if (ret < 0)
-				goto unlock_retry;
+				goto retry;
+
+			if (read_seqcount_retry(&obj->seq, seq))
+				goto retry;
 		}
 	}
 
 	rcu_read_unlock();
 	return ret;
-
-unlock_retry:
-	rcu_read_unlock();
-	goto retry;
 }
 EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu);
diff --git a/drivers/dma-buf/seqno-fence.c b/drivers/dma-buf/seqno-fence.c
index 71127f8f1626..f47112a64763 100644
--- a/drivers/dma-buf/seqno-fence.c
+++ b/drivers/dma-buf/seqno-fence.c
@@ -21,35 +21,35 @@
 #include <linux/export.h>
 #include <linux/seqno-fence.h>
 
-static const char *seqno_fence_get_driver_name(struct fence *fence)
+static const char *seqno_fence_get_driver_name(struct dma_fence *fence)
 {
 	struct seqno_fence *seqno_fence = to_seqno_fence(fence);
 
 	return seqno_fence->ops->get_driver_name(fence);
 }
 
-static const char *seqno_fence_get_timeline_name(struct fence *fence)
+static const char *seqno_fence_get_timeline_name(struct dma_fence *fence)
 {
 	struct seqno_fence *seqno_fence = to_seqno_fence(fence);
 
 	return seqno_fence->ops->get_timeline_name(fence);
 }
 
-static bool seqno_enable_signaling(struct fence *fence)
+static bool seqno_enable_signaling(struct dma_fence *fence)
 {
 	struct seqno_fence *seqno_fence = to_seqno_fence(fence);
 
 	return seqno_fence->ops->enable_signaling(fence);
 }
 
-static bool seqno_signaled(struct fence *fence)
+static bool seqno_signaled(struct dma_fence *fence)
 {
 	struct seqno_fence *seqno_fence = to_seqno_fence(fence);
 
 	return seqno_fence->ops->signaled && seqno_fence->ops->signaled(fence);
 }
 
-static void seqno_release(struct fence *fence)
+static void seqno_release(struct dma_fence *fence)
 {
 	struct seqno_fence *f = to_seqno_fence(fence);
 
@@ -57,18 +57,18 @@ static void seqno_release(struct fence *fence)
 	if (f->ops->release)
 		f->ops->release(fence);
 	else
-		fence_free(&f->base);
+		dma_fence_free(&f->base);
 }
 
-static signed long seqno_wait(struct fence *fence, bool intr,
-				signed long timeout)
+static signed long seqno_wait(struct dma_fence *fence, bool intr,
+			      signed long timeout)
 {
 	struct seqno_fence *f = to_seqno_fence(fence);
 
 	return f->ops->wait(fence, intr, timeout);
 }
 
-const struct fence_ops seqno_fence_ops = {
+const struct dma_fence_ops seqno_fence_ops = {
 	.get_driver_name = seqno_fence_get_driver_name,
 	.get_timeline_name = seqno_fence_get_timeline_name,
 	.enable_signaling = seqno_enable_signaling,
diff --git a/drivers/dma-buf/sw_sync.c b/drivers/dma-buf/sw_sync.c
index 62e8e6dc7953..69c5ff36e2f9 100644
--- a/drivers/dma-buf/sw_sync.c
+++ b/drivers/dma-buf/sw_sync.c
@@ -68,9 +68,9 @@ struct sw_sync_create_fence_data {
 
 #define SW_SYNC_IOC_INC			_IOW(SW_SYNC_IOC_MAGIC, 1, __u32)
 
-static const struct fence_ops timeline_fence_ops;
+static const struct dma_fence_ops timeline_fence_ops;
 
-static inline struct sync_pt *fence_to_sync_pt(struct fence *fence)
+static inline struct sync_pt *dma_fence_to_sync_pt(struct dma_fence *fence)
 {
 	if (fence->ops != &timeline_fence_ops)
 		return NULL;
@@ -84,7 +84,7 @@ static inline struct sync_pt *fence_to_sync_pt(struct fence *fence)
  * Creates a new sync_timeline. Returns the sync_timeline object or NULL in
  * case of error.
  */
-struct sync_timeline *sync_timeline_create(const char *name)
+static struct sync_timeline *sync_timeline_create(const char *name)
 {
 	struct sync_timeline *obj;
 
@@ -93,7 +93,7 @@ struct sync_timeline *sync_timeline_create(const char *name)
 		return NULL;
 
 	kref_init(&obj->kref);
-	obj->context = fence_context_alloc(1);
+	obj->context = dma_fence_context_alloc(1);
 	strlcpy(obj->name, name, sizeof(obj->name));
 
 	INIT_LIST_HEAD(&obj->child_list_head);
@@ -146,7 +146,7 @@ static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc)
 
 	list_for_each_entry_safe(pt, next, &obj->active_list_head,
 				 active_list) {
-		if (fence_is_signaled_locked(&pt->base))
+		if (dma_fence_is_signaled_locked(&pt->base))
 			list_del_init(&pt->active_list);
 	}
 
@@ -179,30 +179,30 @@ static struct sync_pt *sync_pt_create(struct sync_timeline *obj, int size,
 
 	spin_lock_irqsave(&obj->child_list_lock, flags);
 	sync_timeline_get(obj);
-	fence_init(&pt->base, &timeline_fence_ops, &obj->child_list_lock,
-		   obj->context, value);
+	dma_fence_init(&pt->base, &timeline_fence_ops, &obj->child_list_lock,
+		       obj->context, value);
 	list_add_tail(&pt->child_list, &obj->child_list_head);
 	INIT_LIST_HEAD(&pt->active_list);
 	spin_unlock_irqrestore(&obj->child_list_lock, flags);
 	return pt;
 }
 
-static const char *timeline_fence_get_driver_name(struct fence *fence)
+static const char *timeline_fence_get_driver_name(struct dma_fence *fence)
 {
 	return "sw_sync";
 }
 
-static const char *timeline_fence_get_timeline_name(struct fence *fence)
+static const char *timeline_fence_get_timeline_name(struct dma_fence *fence)
 {
-	struct sync_timeline *parent = fence_parent(fence);
+	struct sync_timeline *parent = dma_fence_parent(fence);
 
 	return parent->name;
 }
 
-static void timeline_fence_release(struct fence *fence)
+static void timeline_fence_release(struct dma_fence *fence)
 {
-	struct sync_pt *pt = fence_to_sync_pt(fence);
-	struct sync_timeline *parent = fence_parent(fence);
+	struct sync_pt *pt = dma_fence_to_sync_pt(fence);
+	struct sync_timeline *parent = dma_fence_parent(fence);
 	unsigned long flags;
 
 	spin_lock_irqsave(fence->lock, flags);
@@ -212,20 +212,20 @@ static void timeline_fence_release(struct fence *fence)
 	spin_unlock_irqrestore(fence->lock, flags);
 
 	sync_timeline_put(parent);
-	fence_free(fence);
+	dma_fence_free(fence);
 }
 
-static bool timeline_fence_signaled(struct fence *fence)
+static bool timeline_fence_signaled(struct dma_fence *fence)
 {
-	struct sync_timeline *parent = fence_parent(fence);
+	struct sync_timeline *parent = dma_fence_parent(fence);
 
 	return (fence->seqno > parent->value) ? false : true;
 }
 
-static bool timeline_fence_enable_signaling(struct fence *fence)
+static bool timeline_fence_enable_signaling(struct dma_fence *fence)
 {
-	struct sync_pt *pt = fence_to_sync_pt(fence);
-	struct sync_timeline *parent = fence_parent(fence);
+	struct sync_pt *pt = dma_fence_to_sync_pt(fence);
+	struct sync_timeline *parent = dma_fence_parent(fence);
 
 	if (timeline_fence_signaled(fence))
 		return false;
@@ -234,26 +234,26 @@ static bool timeline_fence_enable_signaling(struct fence *fence)
 	return true;
 }
 
-static void timeline_fence_value_str(struct fence *fence,
+static void timeline_fence_value_str(struct dma_fence *fence,
 				    char *str, int size)
 {
 	snprintf(str, size, "%d", fence->seqno);
 }
 
-static void timeline_fence_timeline_value_str(struct fence *fence,
+static void timeline_fence_timeline_value_str(struct dma_fence *fence,
 					     char *str, int size)
 {
-	struct sync_timeline *parent = fence_parent(fence);
+	struct sync_timeline *parent = dma_fence_parent(fence);
 
 	snprintf(str, size, "%d", parent->value);
 }
 
-static const struct fence_ops timeline_fence_ops = {
+static const struct dma_fence_ops timeline_fence_ops = {
 	.get_driver_name = timeline_fence_get_driver_name,
 	.get_timeline_name = timeline_fence_get_timeline_name,
 	.enable_signaling = timeline_fence_enable_signaling,
 	.signaled = timeline_fence_signaled,
-	.wait = fence_default_wait,
+	.wait = dma_fence_default_wait,
 	.release = timeline_fence_release,
 	.fence_value_str = timeline_fence_value_str,
 	.timeline_value_str = timeline_fence_timeline_value_str,
@@ -316,8 +316,8 @@ static long sw_sync_ioctl_create_fence(struct sync_timeline *obj,
 	}
 
 	sync_file = sync_file_create(&pt->base);
+	dma_fence_put(&pt->base);
 	if (!sync_file) {
-		fence_put(&pt->base);
 		err = -ENOMEM;
 		goto err;
 	}
diff --git a/drivers/dma-buf/sync_debug.c b/drivers/dma-buf/sync_debug.c
index 2dd4c3db6caa..48b20e34fb6d 100644
--- a/drivers/dma-buf/sync_debug.c
+++ b/drivers/dma-buf/sync_debug.c
@@ -71,12 +71,13 @@ static const char *sync_status_str(int status)
 	return "error";
 }
 
-static void sync_print_fence(struct seq_file *s, struct fence *fence, bool show)
+static void sync_print_fence(struct seq_file *s,
+			     struct dma_fence *fence, bool show)
 {
 	int status = 1;
-	struct sync_timeline *parent = fence_parent(fence);
+	struct sync_timeline *parent = dma_fence_parent(fence);
 
-	if (fence_is_signaled_locked(fence))
+	if (dma_fence_is_signaled_locked(fence))
 		status = fence->status;
 
 	seq_printf(s, "  %s%sfence %s",
@@ -135,10 +136,10 @@ static void sync_print_sync_file(struct seq_file *s,
 	int i;
 
 	seq_printf(s, "[%p] %s: %s\n", sync_file, sync_file->name,
-		   sync_status_str(!fence_is_signaled(sync_file->fence)));
+		   sync_status_str(!dma_fence_is_signaled(sync_file->fence)));
 
-	if (fence_is_array(sync_file->fence)) {
-		struct fence_array *array = to_fence_array(sync_file->fence);
+	if (dma_fence_is_array(sync_file->fence)) {
+		struct dma_fence_array *array = to_dma_fence_array(sync_file->fence);
 
 		for (i = 0; i < array->num_fences; ++i)
 			sync_print_fence(s, array->fences[i], true);
diff --git a/drivers/dma-buf/sync_debug.h b/drivers/dma-buf/sync_debug.h
index d269aa6783aa..26fe8b9907b3 100644
--- a/drivers/dma-buf/sync_debug.h
+++ b/drivers/dma-buf/sync_debug.h
@@ -15,7 +15,7 @@
 
 #include <linux/list.h>
 #include <linux/spinlock.h>
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 
 #include <linux/sync_file.h>
 #include <uapi/linux/sync_file.h>
@@ -45,10 +45,9 @@ struct sync_timeline {
 	struct list_head	sync_timeline_list;
 };
 
-static inline struct sync_timeline *fence_parent(struct fence *fence)
+static inline struct sync_timeline *dma_fence_parent(struct dma_fence *fence)
 {
-	return container_of(fence->lock, struct sync_timeline,
-			    child_list_lock);
+	return container_of(fence->lock, struct sync_timeline, child_list_lock);
 }
 
 /**
@@ -58,7 +57,7 @@ static inline struct sync_timeline *fence_parent(struct fence *fence)
  * @active_list: sync timeline active child's list
  */
 struct sync_pt {
-	struct fence base;
+	struct dma_fence base;
 	struct list_head child_list;
 	struct list_head active_list;
 };
diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c
index b29a9e817320..6d802f2d2881 100644
--- a/drivers/dma-buf/sync_file.c
+++ b/drivers/dma-buf/sync_file.c
@@ -54,7 +54,7 @@ err:
 	return NULL;
 }
 
-static void fence_check_cb_func(struct fence *f, struct fence_cb *cb)
+static void fence_check_cb_func(struct dma_fence *f, struct dma_fence_cb *cb)
 {
 	struct sync_file *sync_file;
 
@@ -71,7 +71,7 @@ static void fence_check_cb_func(struct fence *f, struct fence_cb *cb)
  * takes ownership of @fence. The sync_file can be released with
  * fput(sync_file->file). Returns the sync_file or NULL in case of error.
  */
-struct sync_file *sync_file_create(struct fence *fence)
+struct sync_file *sync_file_create(struct dma_fence *fence)
 {
 	struct sync_file *sync_file;
 
@@ -79,7 +79,7 @@ struct sync_file *sync_file_create(struct fence *fence)
 	if (!sync_file)
 		return NULL;
 
-	sync_file->fence = fence;
+	sync_file->fence = dma_fence_get(fence);
 
 	snprintf(sync_file->name, sizeof(sync_file->name), "%s-%s%llu-%d",
 		 fence->ops->get_driver_name(fence),
@@ -121,16 +121,16 @@ err:
  * Ensures @fd references a valid sync_file and returns a fence that
  * represents all fence in the sync_file. On error NULL is returned.
  */
-struct fence *sync_file_get_fence(int fd)
+struct dma_fence *sync_file_get_fence(int fd)
 {
 	struct sync_file *sync_file;
-	struct fence *fence;
+	struct dma_fence *fence;
 
 	sync_file = sync_file_fdget(fd);
 	if (!sync_file)
 		return NULL;
 
-	fence = fence_get(sync_file->fence);
+	fence = dma_fence_get(sync_file->fence);
 	fput(sync_file->file);
 
 	return fence;
@@ -138,22 +138,23 @@ struct fence *sync_file_get_fence(int fd)
 EXPORT_SYMBOL(sync_file_get_fence);
 
 static int sync_file_set_fence(struct sync_file *sync_file,
-			       struct fence **fences, int num_fences)
+			       struct dma_fence **fences, int num_fences)
 {
-	struct fence_array *array;
+	struct dma_fence_array *array;
 
 	/*
 	 * The reference for the fences in the new sync_file and held
 	 * in add_fence() during the merge procedure, so for num_fences == 1
 	 * we already own a new reference to the fence. For num_fence > 1
-	 * we own the reference of the fence_array creation.
+	 * we own the reference of the dma_fence_array creation.
 	 */
 	if (num_fences == 1) {
 		sync_file->fence = fences[0];
 		kfree(fences);
 	} else {
-		array = fence_array_create(num_fences, fences,
-					   fence_context_alloc(1), 1, false);
+		array = dma_fence_array_create(num_fences, fences,
+					       dma_fence_context_alloc(1),
+					       1, false);
 		if (!array)
 			return -ENOMEM;
 
@@ -163,10 +164,11 @@ static int sync_file_set_fence(struct sync_file *sync_file,
 	return 0;
 }
 
-static struct fence **get_fences(struct sync_file *sync_file, int *num_fences)
+static struct dma_fence **get_fences(struct sync_file *sync_file,
+				     int *num_fences)
 {
-	if (fence_is_array(sync_file->fence)) {
-		struct fence_array *array = to_fence_array(sync_file->fence);
+	if (dma_fence_is_array(sync_file->fence)) {
+		struct dma_fence_array *array = to_dma_fence_array(sync_file->fence);
 
 		*num_fences = array->num_fences;
 		return array->fences;
@@ -176,12 +178,13 @@ static struct fence **get_fences(struct sync_file *sync_file, int *num_fences)
 	return &sync_file->fence;
 }
 
-static void add_fence(struct fence **fences, int *i, struct fence *fence)
+static void add_fence(struct dma_fence **fences,
+		      int *i, struct dma_fence *fence)
 {
 	fences[*i] = fence;
 
-	if (!fence_is_signaled(fence)) {
-		fence_get(fence);
+	if (!dma_fence_is_signaled(fence)) {
+		dma_fence_get(fence);
 		(*i)++;
 	}
 }
@@ -200,7 +203,7 @@ static struct sync_file *sync_file_merge(const char *name, struct sync_file *a,
 					 struct sync_file *b)
 {
 	struct sync_file *sync_file;
-	struct fence **fences, **nfences, **a_fences, **b_fences;
+	struct dma_fence **fences, **nfences, **a_fences, **b_fences;
 	int i, i_a, i_b, num_fences, a_num_fences, b_num_fences;
 
 	sync_file = sync_file_alloc();
@@ -226,8 +229,8 @@ static struct sync_file *sync_file_merge(const char *name, struct sync_file *a,
 	 * and sync_file_create, this is a reasonable assumption.
 	 */
 	for (i = i_a = i_b = 0; i_a < a_num_fences && i_b < b_num_fences; ) {
-		struct fence *pt_a = a_fences[i_a];
-		struct fence *pt_b = b_fences[i_b];
+		struct dma_fence *pt_a = a_fences[i_a];
+		struct dma_fence *pt_b = b_fences[i_b];
 
 		if (pt_a->context < pt_b->context) {
 			add_fence(fences, &i, pt_a);
@@ -255,7 +258,7 @@ static struct sync_file *sync_file_merge(const char *name, struct sync_file *a,
 		add_fence(fences, &i, b_fences[i_b]);
 
 	if (i == 0)
-		fences[i++] = fence_get(a_fences[0]);
+		fences[i++] = dma_fence_get(a_fences[0]);
 
 	if (num_fences > i) {
 		nfences = krealloc(fences, i * sizeof(*fences),
@@ -286,8 +289,8 @@ static void sync_file_free(struct kref *kref)
 						     kref);
 
 	if (test_bit(POLL_ENABLED, &sync_file->fence->flags))
-		fence_remove_callback(sync_file->fence, &sync_file->cb);
-	fence_put(sync_file->fence);
+		dma_fence_remove_callback(sync_file->fence, &sync_file->cb);
+	dma_fence_put(sync_file->fence);
 	kfree(sync_file);
 }
 
@@ -305,14 +308,13 @@ static unsigned int sync_file_poll(struct file *file, poll_table *wait)
 
 	poll_wait(file, &sync_file->wq, wait);
 
-	if (!poll_does_not_wait(wait) &&
-	    !test_and_set_bit(POLL_ENABLED, &sync_file->fence->flags)) {
-		if (fence_add_callback(sync_file->fence, &sync_file->cb,
-				       fence_check_cb_func) < 0)
+	if (!test_and_set_bit(POLL_ENABLED, &sync_file->fence->flags)) {
+		if (dma_fence_add_callback(sync_file->fence, &sync_file->cb,
+					   fence_check_cb_func) < 0)
 			wake_up_all(&sync_file->wq);
 	}
 
-	return fence_is_signaled(sync_file->fence) ? POLLIN : 0;
+	return dma_fence_is_signaled(sync_file->fence) ? POLLIN : 0;
 }
 
 static long sync_file_ioctl_merge(struct sync_file *sync_file,
@@ -370,14 +372,14 @@ err_put_fd:
 	return err;
 }
 
-static void sync_fill_fence_info(struct fence *fence,
+static void sync_fill_fence_info(struct dma_fence *fence,
 				 struct sync_fence_info *info)
 {
 	strlcpy(info->obj_name, fence->ops->get_timeline_name(fence),
 		sizeof(info->obj_name));
 	strlcpy(info->driver_name, fence->ops->get_driver_name(fence),
 		sizeof(info->driver_name));
-	if (fence_is_signaled(fence))
+	if (dma_fence_is_signaled(fence))
 		info->status = fence->status >= 0 ? 1 : fence->status;
 	else
 		info->status = 0;
@@ -389,7 +391,7 @@ static long sync_file_ioctl_fence_info(struct sync_file *sync_file,
 {
 	struct sync_file_info info;
 	struct sync_fence_info *fence_info = NULL;
-	struct fence **fences;
+	struct dma_fence **fences;
 	__u32 size;
 	int num_fences, ret, i;
 
@@ -429,7 +431,7 @@ static long sync_file_ioctl_fence_info(struct sync_file *sync_file,
 
 no_fences:
 	strlcpy(info.name, sync_file->name, sizeof(info.name));
-	info.status = fence_is_signaled(sync_file->fence);
+	info.status = dma_fence_is_signaled(sync_file->fence);
 	info.num_fences = num_fences;
 
 	if (copy_to_user((void __user *)arg, &info, sizeof(info)))
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 141aefbe37ec..263495d0adbd 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -436,6 +436,20 @@ config STE_DMA40
 	help
 	  Support for ST-Ericsson DMA40 controller
 
+config ST_FDMA
+	tristate "ST FDMA dmaengine support"
+	depends on ARCH_STI
+	depends on REMOTEPROC
+	select ST_SLIM_REMOTEPROC
+	select DMA_ENGINE
+	select DMA_VIRTUAL_CHANNELS
+	help
+	  Enable support for ST FDMA controller.
+	  It supports 16 independent DMA channels, accepts up to 32 DMA requests
+
+	  Say Y here if you have such a chipset.
+	  If unsure, say N.
+
 config STM32_DMA
 	bool "STMicroelectronics STM32 DMA support"
 	depends on ARCH_STM32 || COMPILE_TEST
@@ -480,7 +494,7 @@ config TEGRA20_APB_DMA
 	  or vice versa. It does not support memory to memory data transfer.
 
 config TEGRA210_ADMA
-	bool "NVIDIA Tegra210 ADMA support"
+	tristate "NVIDIA Tegra210 ADMA support"
 	depends on (ARCH_TEGRA_210_SOC || COMPILE_TEST) && PM_CLK
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index e4dc9cac7ee8..a4fa3360e609 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -67,6 +67,7 @@ obj-$(CONFIG_TI_DMA_CROSSBAR) += ti-dma-crossbar.o
 obj-$(CONFIG_TI_EDMA) += edma.o
 obj-$(CONFIG_XGENE_DMA) += xgene-dma.o
 obj-$(CONFIG_ZX_DMA) += zx296702_dma.o
+obj-$(CONFIG_ST_FDMA) += st_fdma.o
 
 obj-y += qcom/
 obj-y += xilinx/
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index 939a7c31f760..0b7c6ce629a6 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -1793,6 +1793,13 @@ bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)
 }
 EXPORT_SYMBOL_GPL(pl08x_filter_id);
 
+static bool pl08x_filter_fn(struct dma_chan *chan, void *chan_id)
+{
+	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+
+	return plchan->cd == chan_id;
+}
+
 /*
  * Just check that the device is there and active
  * TODO: turn this bit on/off depending on the number of physical channels
@@ -2307,6 +2314,10 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
 			ret = -EINVAL;
 			goto out_no_platdata;
 		}
+	} else {
+		pl08x->slave.filter.map = pl08x->pd->slave_map;
+		pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len;
+		pl08x->slave.filter.fn = pl08x_filter_fn;
 	}
 
 	/* By default, AHB1 only.  If dualmaster, from platform */
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index a4c8f80db29d..1baf3404a365 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -111,9 +111,8 @@ static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan,
 	struct at_dma	*atdma = to_at_dma(chan->device);
 	dma_addr_t phys;
 
-	desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);
+	desc = dma_pool_zalloc(atdma->dma_desc_pool, gfp_flags, &phys);
 	if (desc) {
-		memset(desc, 0, sizeof(struct at_desc));
 		INIT_LIST_HEAD(&desc->tx_list);
 		dma_async_tx_descriptor_init(&desc->txd, chan);
 		/* txd.flags will be overwritten in prep functions */
diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c
index b7d7f2d443a1..7d4e0bcda9af 100644
--- a/drivers/dma/at_xdmac.c
+++ b/drivers/dma/at_xdmac.c
@@ -221,7 +221,6 @@ struct at_xdmac {
 	int			irq;
 	struct clk		*clk;
 	u32			save_gim;
-	u32			save_gs;
 	struct dma_pool		*at_xdmac_desc_pool;
 	struct at_xdmac_chan	chan[0];
 };
@@ -444,9 +443,8 @@ static struct at_xdmac_desc *at_xdmac_alloc_desc(struct dma_chan *chan,
 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);
 	dma_addr_t		phys;
 
-	desc = dma_pool_alloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys);
+	desc = dma_pool_zalloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys);
 	if (desc) {
-		memset(desc, 0, sizeof(*desc));
 		INIT_LIST_HEAD(&desc->descs_list);
 		dma_async_tx_descriptor_init(&desc->tx_dma_desc, chan);
 		desc->tx_dma_desc.tx_submit = at_xdmac_tx_submit;
@@ -1896,7 +1894,6 @@ static int atmel_xdmac_resume(struct device *dev)
 	}
 
 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim);
-	at_xdmac_write(atxdmac, AT_XDMAC_GE, atxdmac->save_gs);
 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {
 		atchan = to_at_xdmac_chan(chan);
 		at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc);
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index cf76fc6149e5..451f899f74e4 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -164,7 +164,9 @@ struct dmatest_thread {
 	struct task_struct	*task;
 	struct dma_chan		*chan;
 	u8			**srcs;
+	u8			**usrcs;
 	u8			**dsts;
+	u8			**udsts;
 	enum dma_transaction_type type;
 	bool			done;
 };
@@ -431,6 +433,7 @@ static int dmatest_func(void *data)
 	ktime_t			comparetime = ktime_set(0, 0);
 	s64			runtime = 0;
 	unsigned long long	total_len = 0;
+	u8			align = 0;
 
 	set_freezable();
 
@@ -441,20 +444,24 @@ static int dmatest_func(void *data)
 	params = &info->params;
 	chan = thread->chan;
 	dev = chan->device;
-	if (thread->type == DMA_MEMCPY)
+	if (thread->type == DMA_MEMCPY) {
+		align = dev->copy_align;
 		src_cnt = dst_cnt = 1;
-	else if (thread->type == DMA_SG)
+	} else if (thread->type == DMA_SG) {
+		align = dev->copy_align;
 		src_cnt = dst_cnt = sg_buffers;
-	else if (thread->type == DMA_XOR) {
+	} else if (thread->type == DMA_XOR) {
 		/* force odd to ensure dst = src */
 		src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
 		dst_cnt = 1;
+		align = dev->xor_align;
 	} else if (thread->type == DMA_PQ) {
 		/* force odd to ensure dst = src */
 		src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
 		dst_cnt = 2;
+		align = dev->pq_align;
 
-		pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
+		pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
 		if (!pq_coefs)
 			goto err_thread_type;
 
@@ -463,23 +470,47 @@ static int dmatest_func(void *data)
 	} else
 		goto err_thread_type;
 
-	thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
+	thread->srcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL);
 	if (!thread->srcs)
 		goto err_srcs;
+
+	thread->usrcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL);
+	if (!thread->usrcs)
+		goto err_usrcs;
+
 	for (i = 0; i < src_cnt; i++) {
-		thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
-		if (!thread->srcs[i])
+		thread->usrcs[i] = kmalloc(params->buf_size + align,
+					   GFP_KERNEL);
+		if (!thread->usrcs[i])
 			goto err_srcbuf;
+
+		/* align srcs to alignment restriction */
+		if (align)
+			thread->srcs[i] = PTR_ALIGN(thread->usrcs[i], align);
+		else
+			thread->srcs[i] = thread->usrcs[i];
 	}
 	thread->srcs[i] = NULL;
 
-	thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
+	thread->dsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL);
 	if (!thread->dsts)
 		goto err_dsts;
+
+	thread->udsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL);
+	if (!thread->udsts)
+		goto err_udsts;
+
 	for (i = 0; i < dst_cnt; i++) {
-		thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
-		if (!thread->dsts[i])
+		thread->udsts[i] = kmalloc(params->buf_size + align,
+					   GFP_KERNEL);
+		if (!thread->udsts[i])
 			goto err_dstbuf;
+
+		/* align dsts to alignment restriction */
+		if (align)
+			thread->dsts[i] = PTR_ALIGN(thread->udsts[i], align);
+		else
+			thread->dsts[i] = thread->udsts[i];
 	}
 	thread->dsts[i] = NULL;
 
@@ -498,20 +529,11 @@ static int dmatest_func(void *data)
 		dma_addr_t srcs[src_cnt];
 		dma_addr_t *dsts;
 		unsigned int src_off, dst_off, len;
-		u8 align = 0;
 		struct scatterlist tx_sg[src_cnt];
 		struct scatterlist rx_sg[src_cnt];
 
 		total_tests++;
 
-		/* honor alignment restrictions */
-		if (thread->type == DMA_MEMCPY || thread->type == DMA_SG)
-			align = dev->copy_align;
-		else if (thread->type == DMA_XOR)
-			align = dev->xor_align;
-		else if (thread->type == DMA_PQ)
-			align = dev->pq_align;
-
 		if (1 << align > params->buf_size) {
 			pr_err("%u-byte buffer too small for %d-byte alignment\n",
 			       params->buf_size, 1 << align);
@@ -549,7 +571,7 @@ static int dmatest_func(void *data)
 			filltime = ktime_add(filltime, diff);
 		}
 
-		um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt,
+		um = dmaengine_get_unmap_data(dev->dev, src_cnt + dst_cnt,
 					      GFP_KERNEL);
 		if (!um) {
 			failed_tests++;
@@ -729,13 +751,17 @@ static int dmatest_func(void *data)
 
 	ret = 0;
 err_dstbuf:
-	for (i = 0; thread->dsts[i]; i++)
-		kfree(thread->dsts[i]);
+	for (i = 0; thread->udsts[i]; i++)
+		kfree(thread->udsts[i]);
+	kfree(thread->udsts);
+err_udsts:
 	kfree(thread->dsts);
 err_dsts:
 err_srcbuf:
-	for (i = 0; thread->srcs[i]; i++)
-		kfree(thread->srcs[i]);
+	for (i = 0; thread->usrcs[i]; i++)
+		kfree(thread->usrcs[i]);
+	kfree(thread->usrcs);
+err_usrcs:
 	kfree(thread->srcs);
 err_srcs:
 	kfree(pq_coefs);
diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c
index c2c0a613cb7a..e5adf5d1c34f 100644
--- a/drivers/dma/dw/core.c
+++ b/drivers/dma/dw/core.c
@@ -1569,7 +1569,7 @@ int dw_dma_probe(struct dw_dma_chip *chip)
 				(dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0;
 		} else {
 			dwc->block_size = pdata->block_size;
-			dwc->nollp = pdata->is_nollp;
+			dwc->nollp = !pdata->multi_block[i];
 		}
 	}
 
diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c
index 5bda0eb9f393..b1655e40cfa2 100644
--- a/drivers/dma/dw/platform.c
+++ b/drivers/dma/dw/platform.c
@@ -102,7 +102,7 @@ dw_dma_parse_dt(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
 	struct dw_dma_platform_data *pdata;
-	u32 tmp, arr[DW_DMA_MAX_NR_MASTERS];
+	u32 tmp, arr[DW_DMA_MAX_NR_MASTERS], mb[DW_DMA_MAX_NR_CHANNELS];
 	u32 nr_masters;
 	u32 nr_channels;
 
@@ -118,6 +118,8 @@ dw_dma_parse_dt(struct platform_device *pdev)
 
 	if (of_property_read_u32(np, "dma-channels", &nr_channels))
 		return NULL;
+	if (nr_channels > DW_DMA_MAX_NR_CHANNELS)
+		return NULL;
 
 	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
 	if (!pdata)
@@ -129,6 +131,12 @@ dw_dma_parse_dt(struct platform_device *pdev)
 	if (of_property_read_bool(np, "is_private"))
 		pdata->is_private = true;
 
+	/*
+	 * All known devices, which use DT for configuration, support
+	 * memory-to-memory transfers. So enable it by default.
+	 */
+	pdata->is_memcpy = true;
+
 	if (!of_property_read_u32(np, "chan_allocation_order", &tmp))
 		pdata->chan_allocation_order = (unsigned char)tmp;
 
@@ -146,6 +154,14 @@ dw_dma_parse_dt(struct platform_device *pdev)
 			pdata->data_width[tmp] = BIT(arr[tmp] & 0x07);
 	}
 
+	if (!of_property_read_u32_array(np, "multi-block", mb, nr_channels)) {
+		for (tmp = 0; tmp < nr_channels; tmp++)
+			pdata->multi_block[tmp] = mb[tmp];
+	} else {
+		for (tmp = 0; tmp < nr_channels; tmp++)
+			pdata->multi_block[tmp] = 1;
+	}
+
 	return pdata;
 }
 #else
diff --git a/drivers/dma/dw/regs.h b/drivers/dma/dw/regs.h
index f65dd104479f..4e0128c62704 100644
--- a/drivers/dma/dw/regs.h
+++ b/drivers/dma/dw/regs.h
@@ -12,7 +12,8 @@
 #include <linux/interrupt.h>
 #include <linux/dmaengine.h>
 
-#define DW_DMA_MAX_NR_CHANNELS	8
+#include "internal.h"
+
 #define DW_DMA_MAX_NR_REQUESTS	16
 
 /* flow controller */
diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c
index 77242b37ef87..3879f80a4815 100644
--- a/drivers/dma/edma.c
+++ b/drivers/dma/edma.c
@@ -2451,6 +2451,9 @@ static int edma_pm_resume(struct device *dev)
 	int i;
 	s8 (*queue_priority_mapping)[2];
 
+	/* re initialize dummy slot to dummy param set */
+	edma_write_slot(ecc, ecc->dummy_slot, &dummy_paramset);
+
 	queue_priority_mapping = ecc->info->queue_priority_mapping;
 
 	/* Event queue priority mapping */
diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c
index db2f9e1653a2..90d29f90acfb 100644
--- a/drivers/dma/fsl_raid.c
+++ b/drivers/dma/fsl_raid.c
@@ -881,6 +881,7 @@ static struct of_device_id fsl_re_ids[] = {
 	{ .compatible = "fsl,raideng-v1.0", },
 	{}
 };
+MODULE_DEVICE_TABLE(of, fsl_re_ids);
 
 static struct platform_driver fsl_re_driver = {
 	.driver = {
diff --git a/drivers/dma/hsu/pci.c b/drivers/dma/hsu/pci.c
index b51639f045ed..4875fa428e81 100644
--- a/drivers/dma/hsu/pci.c
+++ b/drivers/dma/hsu/pci.c
@@ -77,13 +77,15 @@ static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (!chip)
 		return -ENOMEM;
 
+	ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
+	if (ret < 0)
+		return ret;
+
 	chip->dev = &pdev->dev;
 	chip->regs = pcim_iomap_table(pdev)[0];
 	chip->length = pci_resource_len(pdev, 0);
 	chip->offset = HSU_PCI_CHAN_OFFSET;
-	chip->irq = pdev->irq;
-
-	pci_enable_msi(pdev);
+	chip->irq = pci_irq_vector(pdev, 0);
 
 	ret = hsu_dma_probe(chip);
 	if (ret)
diff --git a/drivers/dma/img-mdc-dma.c b/drivers/dma/img-mdc-dma.c
index 624f1e1e9c55..54db1411ce73 100644
--- a/drivers/dma/img-mdc-dma.c
+++ b/drivers/dma/img-mdc-dma.c
@@ -292,7 +292,7 @@ static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(
 	struct mdc_dma *mdma = mchan->mdma;
 	struct mdc_tx_desc *mdesc;
 	struct mdc_hw_list_desc *curr, *prev = NULL;
-	dma_addr_t curr_phys, prev_phys;
+	dma_addr_t curr_phys;
 
 	if (!len)
 		return NULL;
@@ -324,7 +324,6 @@ static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(
 				     xfer_size);
 
 		prev = curr;
-		prev_phys = curr_phys;
 
 		mdesc->list_len++;
 		src += xfer_size;
@@ -375,7 +374,7 @@ static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
 	struct mdc_dma *mdma = mchan->mdma;
 	struct mdc_tx_desc *mdesc;
 	struct mdc_hw_list_desc *curr, *prev = NULL;
-	dma_addr_t curr_phys, prev_phys;
+	dma_addr_t curr_phys;
 
 	if (!buf_len && !period_len)
 		return NULL;
@@ -430,7 +429,6 @@ static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
 			}
 
 			prev = curr;
-			prev_phys = curr_phys;
 
 			mdesc->list_len++;
 			buf_addr += xfer_size;
@@ -458,7 +456,7 @@ static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
 	struct mdc_tx_desc *mdesc;
 	struct scatterlist *sg;
 	struct mdc_hw_list_desc *curr, *prev = NULL;
-	dma_addr_t curr_phys, prev_phys;
+	dma_addr_t curr_phys;
 	unsigned int i;
 
 	if (!sgl)
@@ -509,7 +507,6 @@ static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
 			}
 
 			prev = curr;
-			prev_phys = curr_phys;
 
 			mdesc->list_len++;
 			mdesc->list_xfer_size += xfer_size;
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c
index b9629b2bfc05..d1651a50c349 100644
--- a/drivers/dma/imx-sdma.c
+++ b/drivers/dma/imx-sdma.c
@@ -298,6 +298,7 @@ struct sdma_engine;
  * @event_id1		for channels that use 2 events
  * @word_size		peripheral access size
  * @buf_tail		ID of the buffer that was processed
+ * @buf_ptail		ID of the previous buffer that was processed
  * @num_bd		max NUM_BD. number of descriptors currently handling
  */
 struct sdma_channel {
@@ -309,6 +310,7 @@ struct sdma_channel {
 	unsigned int			event_id1;
 	enum dma_slave_buswidth		word_size;
 	unsigned int			buf_tail;
+	unsigned int			buf_ptail;
 	unsigned int			num_bd;
 	unsigned int			period_len;
 	struct sdma_buffer_descriptor	*bd;
@@ -700,6 +702,8 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac)
 		sdmac->chn_real_count = bd->mode.count;
 		bd->mode.status |= BD_DONE;
 		bd->mode.count = sdmac->period_len;
+		sdmac->buf_ptail = sdmac->buf_tail;
+		sdmac->buf_tail = (sdmac->buf_tail + 1) % sdmac->num_bd;
 
 		/*
 		 * The callback is called from the interrupt context in order
@@ -710,9 +714,6 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac)
 
 		dmaengine_desc_get_callback_invoke(&sdmac->desc, NULL);
 
-		sdmac->buf_tail++;
-		sdmac->buf_tail %= sdmac->num_bd;
-
 		if (error)
 			sdmac->status = old_status;
 	}
@@ -1186,6 +1187,8 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
 	sdmac->flags = 0;
 
 	sdmac->buf_tail = 0;
+	sdmac->buf_ptail = 0;
+	sdmac->chn_real_count = 0;
 
 	dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n",
 			sg_len, channel);
@@ -1288,6 +1291,8 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
 	sdmac->status = DMA_IN_PROGRESS;
 
 	sdmac->buf_tail = 0;
+	sdmac->buf_ptail = 0;
+	sdmac->chn_real_count = 0;
 	sdmac->period_len = period_len;
 
 	sdmac->flags |= IMX_DMA_SG_LOOP;
@@ -1385,7 +1390,7 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan,
 	u32 residue;
 
 	if (sdmac->flags & IMX_DMA_SG_LOOP)
-		residue = (sdmac->num_bd - sdmac->buf_tail) *
+		residue = (sdmac->num_bd - sdmac->buf_ptail) *
 			   sdmac->period_len - sdmac->chn_real_count;
 	else
 		residue = sdmac->chn_count - sdmac->chn_real_count;
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 49386ce04bf5..a371b07a0981 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -39,6 +39,7 @@
 #include "../dmaengine.h"
 
 static char *chanerr_str[] = {
+	"DMA Transfer Source Address Error",
 	"DMA Transfer Destination Address Error",
 	"Next Descriptor Address Error",
 	"Descriptor Error",
@@ -66,7 +67,6 @@ static char *chanerr_str[] = {
 	"Result Guard Tag verification Error",
 	"Result Application Tag verification Error",
 	"Result Reference Tag verification Error",
-	NULL
 };
 
 static void ioat_eh(struct ioatdma_chan *ioat_chan);
@@ -75,13 +75,10 @@ static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr)
 {
 	int i;
 
-	for (i = 0; i < 32; i++) {
+	for (i = 0; i < ARRAY_SIZE(chanerr_str); i++) {
 		if ((chanerr >> i) & 1) {
-			if (chanerr_str[i]) {
-				dev_err(to_dev(ioat_chan), "Err(%d): %s\n",
-					i, chanerr_str[i]);
-			} else
-				break;
+			dev_err(to_dev(ioat_chan), "Err(%d): %s\n",
+				i, chanerr_str[i]);
 		}
 	}
 }
@@ -341,15 +338,12 @@ ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags)
 {
 	struct ioat_dma_descriptor *hw;
 	struct ioat_ring_ent *desc;
-	struct ioatdma_device *ioat_dma;
 	struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
 	int chunk;
 	dma_addr_t phys;
 	u8 *pos;
 	off_t offs;
 
-	ioat_dma = to_ioatdma_device(chan->device);
-
 	chunk = idx / IOAT_DESCS_PER_2M;
 	idx &= (IOAT_DESCS_PER_2M - 1);
 	offs = idx * IOAT_DESC_SZ;
@@ -614,11 +608,8 @@ static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
 
 		tx = &desc->txd;
 		if (tx->cookie) {
-			struct dmaengine_result res;
-
 			dma_cookie_complete(tx);
 			dma_descriptor_unmap(tx);
-			res.result = DMA_TRANS_NOERROR;
 			dmaengine_desc_get_callback_invoke(tx, NULL);
 			tx->callback = NULL;
 			tx->callback_result = NULL;
diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c
index 015f7110b96d..90eddd9f07e4 100644
--- a/drivers/dma/ioat/init.c
+++ b/drivers/dma/ioat/init.c
@@ -340,11 +340,13 @@ static int ioat_dma_self_test(struct ioatdma_device *ioat_dma)
 	dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
 	if (dma_mapping_error(dev, dma_src)) {
 		dev_err(dev, "mapping src buffer failed\n");
+		err = -ENOMEM;
 		goto free_resources;
 	}
 	dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
 	if (dma_mapping_error(dev, dma_dest)) {
 		dev_err(dev, "mapping dest buffer failed\n");
+		err = -ENOMEM;
 		goto unmap_src;
 	}
 	flags = DMA_PREP_INTERRUPT;
@@ -827,16 +829,20 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)
 	op = IOAT_OP_XOR;
 
 	dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE);
-	if (dma_mapping_error(dev, dest_dma))
+	if (dma_mapping_error(dev, dest_dma)) {
+		err = -ENOMEM;
 		goto free_resources;
+	}
 
 	for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
 		dma_srcs[i] = DMA_ERROR_CODE;
 	for (i = 0; i < IOAT_NUM_SRC_TEST; i++) {
 		dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE,
 					   DMA_TO_DEVICE);
-		if (dma_mapping_error(dev, dma_srcs[i]))
+		if (dma_mapping_error(dev, dma_srcs[i])) {
+			err = -ENOMEM;
 			goto dma_unmap;
+		}
 	}
 	tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
 				      IOAT_NUM_SRC_TEST, PAGE_SIZE,
@@ -904,8 +910,10 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)
 	for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) {
 		dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
 					   DMA_TO_DEVICE);
-		if (dma_mapping_error(dev, dma_srcs[i]))
+		if (dma_mapping_error(dev, dma_srcs[i])) {
+			err = -ENOMEM;
 			goto dma_unmap;
+		}
 	}
 	tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
 					  IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
@@ -957,8 +965,10 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)
 	for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) {
 		dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
 					   DMA_TO_DEVICE);
-		if (dma_mapping_error(dev, dma_srcs[i]))
+		if (dma_mapping_error(dev, dma_srcs[i])) {
+			err = -ENOMEM;
 			goto dma_unmap;
+		}
 	}
 	tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
 					  IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
@@ -1071,7 +1081,6 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)
 	struct dma_device *dma;
 	struct dma_chan *c;
 	struct ioatdma_chan *ioat_chan;
-	bool is_raid_device = false;
 	int err;
 	u16 val16;
 
@@ -1095,7 +1104,6 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)
 		ioat_dma->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ);
 
 	if (ioat_dma->cap & IOAT_CAP_XOR) {
-		is_raid_device = true;
 		dma->max_xor = 8;
 
 		dma_cap_set(DMA_XOR, dma->cap_mask);
@@ -1106,7 +1114,6 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)
 	}
 
 	if (ioat_dma->cap & IOAT_CAP_PQ) {
-		is_raid_device = true;
 
 		dma->device_prep_dma_pq = ioat_prep_pq;
 		dma->device_prep_dma_pq_val = ioat_prep_pq_val;
diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c
index aabcb7934b05..01e25c68dd5a 100644
--- a/drivers/dma/k3dma.c
+++ b/drivers/dma/k3dma.c
@@ -458,13 +458,12 @@ static struct k3_dma_desc_sw *k3_dma_alloc_desc_resource(int num,
 	if (!ds)
 		return NULL;
 
-	ds->desc_hw = dma_pool_alloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli);
+	ds->desc_hw = dma_pool_zalloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli);
 	if (!ds->desc_hw) {
 		dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc);
 		kfree(ds);
 		return NULL;
 	}
-	memset(ds->desc_hw, 0, sizeof(struct k3_desc_hw) * num);
 	ds->desc_num = num;
 	return ds;
 }
diff --git a/drivers/dma/mic_x100_dma.c b/drivers/dma/mic_x100_dma.c
index 818255844a3c..5ba5714d0b7c 100644
--- a/drivers/dma/mic_x100_dma.c
+++ b/drivers/dma/mic_x100_dma.c
@@ -554,9 +554,7 @@ static int mic_dma_init(struct mic_dma_device *mic_dma_dev,
 	int ret;
 
 	for (i = first_chan; i < first_chan + MIC_DMA_NUM_CHAN; i++) {
-		unsigned long data;
 		ch = &mic_dma_dev->mic_ch[i];
-		data = (unsigned long)ch;
 		ch->ch_num = i;
 		ch->owner = owner;
 		spin_lock_init(&ch->cleanup_lock);
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index 23f75285a4d9..0cb951b743a6 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -68,6 +68,36 @@ static void mv_desc_init(struct mv_xor_desc_slot *desc,
 	hw_desc->byte_count = byte_count;
 }
 
+/* Populate the descriptor */
+static void mv_xor_config_sg_ll_desc(struct mv_xor_desc_slot *desc,
+				     dma_addr_t dma_src, dma_addr_t dma_dst,
+				     u32 len, struct mv_xor_desc_slot *prev)
+{
+	struct mv_xor_desc *hw_desc = desc->hw_desc;
+
+	hw_desc->status = XOR_DESC_DMA_OWNED;
+	hw_desc->phy_next_desc = 0;
+	/* Configure for XOR with only one src address -> MEMCPY */
+	hw_desc->desc_command = XOR_DESC_OPERATION_XOR | (0x1 << 0);
+	hw_desc->phy_dest_addr = dma_dst;
+	hw_desc->phy_src_addr[0] = dma_src;
+	hw_desc->byte_count = len;
+
+	if (prev) {
+		struct mv_xor_desc *hw_prev = prev->hw_desc;
+
+		hw_prev->phy_next_desc = desc->async_tx.phys;
+	}
+}
+
+static void mv_xor_desc_config_eod(struct mv_xor_desc_slot *desc)
+{
+	struct mv_xor_desc *hw_desc = desc->hw_desc;
+
+	/* Enable end-of-descriptor interrupt */
+	hw_desc->desc_command |= XOR_DESC_EOD_INT_EN;
+}
+
 static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)
 {
 	struct mv_xor_desc *hw_desc = desc->hw_desc;
@@ -228,8 +258,13 @@ mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan)
 	list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
 				 node) {
 
-		if (async_tx_test_ack(&iter->async_tx))
+		if (async_tx_test_ack(&iter->async_tx)) {
 			list_move_tail(&iter->node, &mv_chan->free_slots);
+			if (!list_empty(&iter->sg_tx_list)) {
+				list_splice_tail_init(&iter->sg_tx_list,
+							&mv_chan->free_slots);
+			}
+		}
 	}
 	return 0;
 }
@@ -244,11 +279,20 @@ mv_desc_clean_slot(struct mv_xor_desc_slot *desc,
 	/* the client is allowed to attach dependent operations
 	 * until 'ack' is set
 	 */
-	if (!async_tx_test_ack(&desc->async_tx))
+	if (!async_tx_test_ack(&desc->async_tx)) {
 		/* move this slot to the completed_slots */
 		list_move_tail(&desc->node, &mv_chan->completed_slots);
-	else
+		if (!list_empty(&desc->sg_tx_list)) {
+			list_splice_tail_init(&desc->sg_tx_list,
+					      &mv_chan->completed_slots);
+		}
+	} else {
 		list_move_tail(&desc->node, &mv_chan->free_slots);
+		if (!list_empty(&desc->sg_tx_list)) {
+			list_splice_tail_init(&desc->sg_tx_list,
+					      &mv_chan->free_slots);
+		}
+	}
 
 	return 0;
 }
@@ -450,6 +494,7 @@ static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
 		dma_async_tx_descriptor_init(&slot->async_tx, chan);
 		slot->async_tx.tx_submit = mv_xor_tx_submit;
 		INIT_LIST_HEAD(&slot->node);
+		INIT_LIST_HEAD(&slot->sg_tx_list);
 		dma_desc = mv_chan->dma_desc_pool;
 		slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE;
 		slot->idx = idx++;
@@ -617,6 +662,132 @@ mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
 	return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
 }
 
+/**
+ * mv_xor_prep_dma_sg - prepare descriptors for a memory sg transaction
+ * @chan: DMA channel
+ * @dst_sg: Destination scatter list
+ * @dst_sg_len: Number of entries in destination scatter list
+ * @src_sg: Source scatter list
+ * @src_sg_len: Number of entries in source scatter list
+ * @flags: transfer ack flags
+ *
+ * Return: Async transaction descriptor on success and NULL on failure
+ */
+static struct dma_async_tx_descriptor *
+mv_xor_prep_dma_sg(struct dma_chan *chan, struct scatterlist *dst_sg,
+		   unsigned int dst_sg_len, struct scatterlist *src_sg,
+		   unsigned int src_sg_len, unsigned long flags)
+{
+	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
+	struct mv_xor_desc_slot *new;
+	struct mv_xor_desc_slot *first = NULL;
+	struct mv_xor_desc_slot *prev = NULL;
+	size_t len, dst_avail, src_avail;
+	dma_addr_t dma_dst, dma_src;
+	int desc_cnt = 0;
+	int ret;
+
+	dev_dbg(mv_chan_to_devp(mv_chan),
+		"%s dst_sg_len: %d src_sg_len: %d flags: %ld\n",
+		__func__, dst_sg_len, src_sg_len, flags);
+
+	dst_avail = sg_dma_len(dst_sg);
+	src_avail = sg_dma_len(src_sg);
+
+	/* Run until we are out of scatterlist entries */
+	while (true) {
+		/* Allocate and populate the descriptor */
+		desc_cnt++;
+		new = mv_chan_alloc_slot(mv_chan);
+		if (!new) {
+			dev_err(mv_chan_to_devp(mv_chan),
+				"Out of descriptors (desc_cnt=%d)!\n",
+				desc_cnt);
+			goto err;
+		}
+
+		len = min_t(size_t, src_avail, dst_avail);
+		len = min_t(size_t, len, MV_XOR_MAX_BYTE_COUNT);
+		if (len == 0)
+			goto fetch;
+
+		if (len < MV_XOR_MIN_BYTE_COUNT) {
+			dev_err(mv_chan_to_devp(mv_chan),
+				"Transfer size of %zu too small!\n", len);
+			goto err;
+		}
+
+		dma_dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) -
+			dst_avail;
+		dma_src = sg_dma_address(src_sg) + sg_dma_len(src_sg) -
+			src_avail;
+
+		/* Check if a new window needs to get added for 'dst' */
+		ret = mv_xor_add_io_win(mv_chan, dma_dst);
+		if (ret)
+			goto err;
+
+		/* Check if a new window needs to get added for 'src' */
+		ret = mv_xor_add_io_win(mv_chan, dma_src);
+		if (ret)
+			goto err;
+
+		/* Populate the descriptor */
+		mv_xor_config_sg_ll_desc(new, dma_src, dma_dst, len, prev);
+		prev = new;
+		dst_avail -= len;
+		src_avail -= len;
+
+		if (!first)
+			first = new;
+		else
+			list_move_tail(&new->node, &first->sg_tx_list);
+
+fetch:
+		/* Fetch the next dst scatterlist entry */
+		if (dst_avail == 0) {
+			if (dst_sg_len == 0)
+				break;
+
+			/* Fetch the next entry: if there are no more: done */
+			dst_sg = sg_next(dst_sg);
+			if (dst_sg == NULL)
+				break;
+
+			dst_sg_len--;
+			dst_avail = sg_dma_len(dst_sg);
+		}
+
+		/* Fetch the next src scatterlist entry */
+		if (src_avail == 0) {
+			if (src_sg_len == 0)
+				break;
+
+			/* Fetch the next entry: if there are no more: done */
+			src_sg = sg_next(src_sg);
+			if (src_sg == NULL)
+				break;
+
+			src_sg_len--;
+			src_avail = sg_dma_len(src_sg);
+		}
+	}
+
+	/* Set the EOD flag in the last descriptor */
+	mv_xor_desc_config_eod(new);
+	first->async_tx.flags = flags;
+
+	return &first->async_tx;
+
+err:
+	/* Cleanup: Move all descriptors back into the free list */
+	spin_lock_bh(&mv_chan->lock);
+	mv_desc_clean_slot(first, mv_chan);
+	spin_unlock_bh(&mv_chan->lock);
+
+	return NULL;
+}
+
 static void mv_xor_free_chan_resources(struct dma_chan *chan)
 {
 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
@@ -1083,6 +1254,8 @@ mv_xor_channel_add(struct mv_xor_device *xordev,
 		dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;
 	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
 		dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;
+	if (dma_has_cap(DMA_SG, dma_dev->cap_mask))
+		dma_dev->device_prep_dma_sg = mv_xor_prep_dma_sg;
 	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
 		dma_dev->max_xor = 8;
 		dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
@@ -1132,10 +1305,11 @@ mv_xor_channel_add(struct mv_xor_device *xordev,
 			goto err_free_irq;
 	}
 
-	dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",
+	dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s%s)\n",
 		 mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",
 		 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
 		 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
+		 dma_has_cap(DMA_SG, dma_dev->cap_mask) ? "sg " : "",
 		 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
 
 	dma_async_device_register(dma_dev);
@@ -1378,6 +1552,7 @@ static int mv_xor_probe(struct platform_device *pdev)
 
 			dma_cap_zero(cap_mask);
 			dma_cap_set(DMA_MEMCPY, cap_mask);
+			dma_cap_set(DMA_SG, cap_mask);
 			dma_cap_set(DMA_XOR, cap_mask);
 			dma_cap_set(DMA_INTERRUPT, cap_mask);
 
@@ -1455,12 +1630,7 @@ static struct platform_driver mv_xor_driver = {
 	},
 };
 
-
-static int __init mv_xor_init(void)
-{
-	return platform_driver_register(&mv_xor_driver);
-}
-device_initcall(mv_xor_init);
+builtin_platform_driver(mv_xor_driver);
 
 /*
 MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h
index 88eeab222a23..cf921dd6af73 100644
--- a/drivers/dma/mv_xor.h
+++ b/drivers/dma/mv_xor.h
@@ -148,6 +148,7 @@ struct mv_xor_chan {
  */
 struct mv_xor_desc_slot {
 	struct list_head	node;
+	struct list_head	sg_tx_list;
 	enum dma_transaction_type	type;
 	void			*hw_desc;
 	u16			idx;
diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c
index 09de71519d37..3f45b9bdf201 100644
--- a/drivers/dma/nbpfaxi.c
+++ b/drivers/dma/nbpfaxi.c
@@ -225,6 +225,8 @@ struct nbpf_channel {
 struct nbpf_device {
 	struct dma_device dma_dev;
 	void __iomem *base;
+	u32 max_burst_mem_read;
+	u32 max_burst_mem_write;
 	struct clk *clk;
 	const struct nbpf_config *config;
 	unsigned int eirq;
@@ -425,10 +427,33 @@ static void nbpf_chan_configure(struct nbpf_channel *chan)
 	nbpf_chan_write(chan, NBPF_CHAN_CFG, NBPF_CHAN_CFG_DMS | chan->dmarq_cfg);
 }
 
-static u32 nbpf_xfer_ds(struct nbpf_device *nbpf, size_t size)
+static u32 nbpf_xfer_ds(struct nbpf_device *nbpf, size_t size,
+			enum dma_transfer_direction direction)
 {
+	int max_burst = nbpf->config->buffer_size * 8;
+
+	if (nbpf->max_burst_mem_read || nbpf->max_burst_mem_write) {
+		switch (direction) {
+		case DMA_MEM_TO_MEM:
+			max_burst = min_not_zero(nbpf->max_burst_mem_read,
+						 nbpf->max_burst_mem_write);
+			break;
+		case DMA_MEM_TO_DEV:
+			if (nbpf->max_burst_mem_read)
+				max_burst = nbpf->max_burst_mem_read;
+			break;
+		case DMA_DEV_TO_MEM:
+			if (nbpf->max_burst_mem_write)
+				max_burst = nbpf->max_burst_mem_write;
+			break;
+		case DMA_DEV_TO_DEV:
+		default:
+			break;
+		}
+	}
+
 	/* Maximum supported bursts depend on the buffer size */
-	return min_t(int, __ffs(size), ilog2(nbpf->config->buffer_size * 8));
+	return min_t(int, __ffs(size), ilog2(max_burst));
 }
 
 static size_t nbpf_xfer_size(struct nbpf_device *nbpf,
@@ -458,7 +483,7 @@ static size_t nbpf_xfer_size(struct nbpf_device *nbpf,
 		size = burst;
 	}
 
-	return nbpf_xfer_ds(nbpf, size);
+	return nbpf_xfer_ds(nbpf, size, DMA_TRANS_NONE);
 }
 
 /*
@@ -507,7 +532,7 @@ static int nbpf_prep_one(struct nbpf_link_desc *ldesc,
 	 * transfers we enable the SBE bit and terminate the transfer in our
 	 * .device_pause handler.
 	 */
-	mem_xfer = nbpf_xfer_ds(chan->nbpf, size);
+	mem_xfer = nbpf_xfer_ds(chan->nbpf, size, direction);
 
 	switch (direction) {
 	case DMA_DEV_TO_MEM:
@@ -1313,6 +1338,11 @@ static int nbpf_probe(struct platform_device *pdev)
 	if (IS_ERR(nbpf->clk))
 		return PTR_ERR(nbpf->clk);
 
+	of_property_read_u32(np, "max-burst-mem-read",
+			     &nbpf->max_burst_mem_read);
+	of_property_read_u32(np, "max-burst-mem-write",
+			     &nbpf->max_burst_mem_write);
+
 	nbpf->config = cfg;
 
 	for (i = 0; irqs < ARRAY_SIZE(irqbuf); i++) {
diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c
index 7ca27d4b1c54..ac68666cd3f4 100644
--- a/drivers/dma/omap-dma.c
+++ b/drivers/dma/omap-dma.c
@@ -166,6 +166,9 @@ enum {
 	CSDP_DST_BURST_16	= 1 << 14,
 	CSDP_DST_BURST_32	= 2 << 14,
 	CSDP_DST_BURST_64	= 3 << 14,
+	CSDP_WRITE_NON_POSTED	= 0 << 16,
+	CSDP_WRITE_POSTED	= 1 << 16,
+	CSDP_WRITE_LAST_NON_POSTED = 2 << 16,
 
 	CICR_TOUT_IE		= BIT(0),	/* OMAP1 only */
 	CICR_DROP_IE		= BIT(1),
@@ -422,7 +425,30 @@ static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
 	c->running = true;
 }
 
-static void omap_dma_stop(struct omap_chan *c)
+static void omap_dma_drain_chan(struct omap_chan *c)
+{
+	int i;
+	u32 val;
+
+	/* Wait for sDMA FIFO to drain */
+	for (i = 0; ; i++) {
+		val = omap_dma_chan_read(c, CCR);
+		if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
+			break;
+
+		if (i > 100)
+			break;
+
+		udelay(5);
+	}
+
+	if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
+		dev_err(c->vc.chan.device->dev,
+			"DMA drain did not complete on lch %d\n",
+			c->dma_ch);
+}
+
+static int omap_dma_stop(struct omap_chan *c)
 {
 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 	uint32_t val;
@@ -435,7 +461,6 @@ static void omap_dma_stop(struct omap_chan *c)
 	val = omap_dma_chan_read(c, CCR);
 	if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
 		uint32_t sysconfig;
-		unsigned i;
 
 		sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
 		val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
@@ -446,27 +471,19 @@ static void omap_dma_stop(struct omap_chan *c)
 		val &= ~CCR_ENABLE;
 		omap_dma_chan_write(c, CCR, val);
 
-		/* Wait for sDMA FIFO to drain */
-		for (i = 0; ; i++) {
-			val = omap_dma_chan_read(c, CCR);
-			if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
-				break;
-
-			if (i > 100)
-				break;
-
-			udelay(5);
-		}
-
-		if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
-			dev_err(c->vc.chan.device->dev,
-				"DMA drain did not complete on lch %d\n",
-			        c->dma_ch);
+		if (!(c->ccr & CCR_BUFFERING_DISABLE))
+			omap_dma_drain_chan(c);
 
 		omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
 	} else {
+		if (!(val & CCR_ENABLE))
+			return -EINVAL;
+
 		val &= ~CCR_ENABLE;
 		omap_dma_chan_write(c, CCR, val);
+
+		if (!(c->ccr & CCR_BUFFERING_DISABLE))
+			omap_dma_drain_chan(c);
 	}
 
 	mb();
@@ -481,8 +498,8 @@ static void omap_dma_stop(struct omap_chan *c)
 
 		omap_dma_chan_write(c, CLNK_CTRL, val);
 	}
-
 	c->running = false;
+	return 0;
 }
 
 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
@@ -836,6 +853,8 @@ static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
 	} else {
 		txstate->residue = 0;
 	}
+	if (ret == DMA_IN_PROGRESS && c->paused)
+		ret = DMA_PAUSED;
 	spin_unlock_irqrestore(&c->vc.lock, flags);
 
 	return ret;
@@ -865,15 +884,18 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 	unsigned i, es, en, frame_bytes;
 	bool ll_failed = false;
 	u32 burst;
+	u32 port_window, port_window_bytes;
 
 	if (dir == DMA_DEV_TO_MEM) {
 		dev_addr = c->cfg.src_addr;
 		dev_width = c->cfg.src_addr_width;
 		burst = c->cfg.src_maxburst;
+		port_window = c->cfg.src_port_window_size;
 	} else if (dir == DMA_MEM_TO_DEV) {
 		dev_addr = c->cfg.dst_addr;
 		dev_width = c->cfg.dst_addr_width;
 		burst = c->cfg.dst_maxburst;
+		port_window = c->cfg.dst_port_window_size;
 	} else {
 		dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
 		return NULL;
@@ -894,6 +916,12 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 		return NULL;
 	}
 
+	/* When the port_window is used, one frame must cover the window */
+	if (port_window) {
+		burst = port_window;
+		port_window_bytes = port_window * es_bytes[es];
+	}
+
 	/* Now allocate and setup the descriptor. */
 	d = kzalloc(sizeof(*d) + sglen * sizeof(d->sg[0]), GFP_ATOMIC);
 	if (!d)
@@ -905,11 +933,45 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 
 	d->ccr = c->ccr | CCR_SYNC_FRAME;
 	if (dir == DMA_DEV_TO_MEM) {
-		d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
 		d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
+
+		d->ccr |= CCR_DST_AMODE_POSTINC;
+		if (port_window) {
+			d->ccr |= CCR_SRC_AMODE_DBLIDX;
+			d->ei = 1;
+			/*
+			 * One frame covers the port_window and by  configure
+			 * the source frame index to be -1 * (port_window - 1)
+			 * we instruct the sDMA that after a frame is processed
+			 * it should move back to the start of the window.
+			 */
+			d->fi = -(port_window_bytes - 1);
+
+			if (port_window_bytes >= 64)
+				d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
+			else if (port_window_bytes >= 32)
+				d->csdp = CSDP_SRC_BURST_32 | CSDP_SRC_PACKED;
+			else if (port_window_bytes >= 16)
+				d->csdp = CSDP_SRC_BURST_16 | CSDP_SRC_PACKED;
+		} else {
+			d->ccr |= CCR_SRC_AMODE_CONSTANT;
+		}
 	} else {
-		d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
 		d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
+
+		d->ccr |= CCR_SRC_AMODE_POSTINC;
+		if (port_window) {
+			d->ccr |= CCR_DST_AMODE_DBLIDX;
+
+			if (port_window_bytes >= 64)
+				d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
+			else if (port_window_bytes >= 32)
+				d->csdp = CSDP_DST_BURST_32 | CSDP_DST_PACKED;
+			else if (port_window_bytes >= 16)
+				d->csdp = CSDP_DST_BURST_16 | CSDP_DST_PACKED;
+		} else {
+			d->ccr |= CCR_DST_AMODE_CONSTANT;
+		}
 	}
 
 	d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
@@ -927,6 +989,9 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 			d->ccr |= CCR_TRIGGER_SRC;
 
 		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
+
+		if (port_window)
+			d->csdp |= CSDP_WRITE_LAST_NON_POSTED;
 	}
 	if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
 		d->clnk_ctrl = c->dma_ch;
@@ -952,6 +1017,16 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 		osg->addr = sg_dma_address(sgent);
 		osg->en = en;
 		osg->fn = sg_dma_len(sgent) / frame_bytes;
+		if (port_window && dir == DMA_MEM_TO_DEV) {
+			osg->ei = 1;
+			/*
+			 * One frame covers the port_window and by  configure
+			 * the source frame index to be -1 * (port_window - 1)
+			 * we instruct the sDMA that after a frame is processed
+			 * it should move back to the start of the window.
+			 */
+			osg->fi = -(port_window_bytes - 1);
+		}
 
 		if (d->using_ll) {
 			osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
@@ -1247,10 +1322,8 @@ static int omap_dma_terminate_all(struct dma_chan *chan)
 			omap_dma_stop(c);
 	}
 
-	if (c->cyclic) {
-		c->cyclic = false;
-		c->paused = false;
-	}
+	c->cyclic = false;
+	c->paused = false;
 
 	vchan_get_all_descriptors(&c->vc, &head);
 	spin_unlock_irqrestore(&c->vc.lock, flags);
@@ -1269,28 +1342,66 @@ static void omap_dma_synchronize(struct dma_chan *chan)
 static int omap_dma_pause(struct dma_chan *chan)
 {
 	struct omap_chan *c = to_omap_dma_chan(chan);
+	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
+	unsigned long flags;
+	int ret = -EINVAL;
+	bool can_pause = false;
 
-	/* Pause/Resume only allowed with cyclic mode */
-	if (!c->cyclic)
-		return -EINVAL;
+	spin_lock_irqsave(&od->irq_lock, flags);
 
-	if (!c->paused) {
-		omap_dma_stop(c);
-		c->paused = true;
+	if (!c->desc)
+		goto out;
+
+	if (c->cyclic)
+		can_pause = true;
+
+	/*
+	 * We do not allow DMA_MEM_TO_DEV transfers to be paused.
+	 * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer:
+	 * "When a channel is disabled during a transfer, the channel undergoes
+	 * an abort, unless it is hardware-source-synchronized …".
+	 * A source-synchronised channel is one where the fetching of data is
+	 * under control of the device. In other words, a device-to-memory
+	 * transfer. So, a destination-synchronised channel (which would be a
+	 * memory-to-device transfer) undergoes an abort if the the CCR_ENABLE
+	 * bit is cleared.
+	 * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel
+	 * aborts immediately after completion of current read/write
+	 * transactions and then the FIFO is cleaned up." The term "cleaned up"
+	 * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE
+	 * are both clear _before_ disabling the channel, otherwise data loss
+	 * will occur.
+	 * The problem is that if the channel is active, then device activity
+	 * can result in DMA activity starting between reading those as both
+	 * clear and the write to DMA_CCR to clear the enable bit hitting the
+	 * hardware. If the DMA hardware can't drain the data in its FIFO to the
+	 * destination, then data loss "might" occur (say if we write to an UART
+	 * and the UART is not accepting any further data).
+	 */
+	else if (c->desc->dir == DMA_DEV_TO_MEM)
+		can_pause = true;
+
+	if (can_pause && !c->paused) {
+		ret = omap_dma_stop(c);
+		if (!ret)
+			c->paused = true;
 	}
+out:
+	spin_unlock_irqrestore(&od->irq_lock, flags);
 
-	return 0;
+	return ret;
 }
 
 static int omap_dma_resume(struct dma_chan *chan)
 {
 	struct omap_chan *c = to_omap_dma_chan(chan);
+	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
+	unsigned long flags;
+	int ret = -EINVAL;
 
-	/* Pause/Resume only allowed with cyclic mode */
-	if (!c->cyclic)
-		return -EINVAL;
+	spin_lock_irqsave(&od->irq_lock, flags);
 
-	if (c->paused) {
+	if (c->paused && c->desc) {
 		mb();
 
 		/* Restore channel link register */
@@ -1298,9 +1409,11 @@ static int omap_dma_resume(struct dma_chan *chan)
 
 		omap_dma_start(c, c->desc);
 		c->paused = false;
+		ret = 0;
 	}
+	spin_unlock_irqrestore(&od->irq_lock, flags);
 
-	return 0;
+	return ret;
 }
 
 static int omap_dma_chan_init(struct omap_dmadev *od)
diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c
index df95727dc2fb..f9028e9d0dfc 100644
--- a/drivers/dma/pch_dma.c
+++ b/drivers/dma/pch_dma.c
@@ -417,10 +417,8 @@ static dma_cookie_t pd_tx_submit(struct dma_async_tx_descriptor *txd)
 {
 	struct pch_dma_desc *desc = to_pd_desc(txd);
 	struct pch_dma_chan *pd_chan = to_pd_chan(txd->chan);
-	dma_cookie_t cookie;
 
 	spin_lock(&pd_chan->lock);
-	cookie = dma_cookie_assign(txd);
 
 	if (list_empty(&pd_chan->active_list)) {
 		list_add_tail(&desc->desc_node, &pd_chan->active_list);
@@ -439,9 +437,8 @@ static struct pch_dma_desc *pdc_alloc_desc(struct dma_chan *chan, gfp_t flags)
 	struct pch_dma *pd = to_pd(chan->device);
 	dma_addr_t addr;
 
-	desc = pci_pool_alloc(pd->pool, flags, &addr);
+	desc = pci_pool_zalloc(pd->pool, flags, &addr);
 	if (desc) {
-		memset(desc, 0, sizeof(struct pch_dma_desc));
 		INIT_LIST_HEAD(&desc->tx_list);
 		dma_async_tx_descriptor_init(&desc->txd, chan);
 		desc->txd.tx_submit = pd_tx_submit;
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index 030fe05ed43b..87fd01539fcb 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -570,7 +570,8 @@ static inline u32 _emit_ADDH(unsigned dry_run, u8 buf[],
 
 	buf[0] = CMD_DMAADDH;
 	buf[0] |= (da << 1);
-	*((__le16 *)&buf[1]) = cpu_to_le16(val);
+	buf[1] = val;
+	buf[2] = val >> 8;
 
 	PL330_DBGCMD_DUMP(SZ_DMAADDH, "\tDMAADDH %s %u\n",
 		da == 1 ? "DA" : "SA", val);
@@ -724,7 +725,10 @@ static inline u32 _emit_MOV(unsigned dry_run, u8 buf[],
 
 	buf[0] = CMD_DMAMOV;
 	buf[1] = dst;
-	*((__le32 *)&buf[2]) = cpu_to_le32(val);
+	buf[2] = val;
+	buf[3] = val >> 8;
+	buf[4] = val >> 16;
+	buf[5] = val >> 24;
 
 	PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n",
 		dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val);
@@ -899,10 +903,11 @@ static inline u32 _emit_GO(unsigned dry_run, u8 buf[],
 
 	buf[0] = CMD_DMAGO;
 	buf[0] |= (ns << 1);
-
 	buf[1] = chan & 0x7;
-
-	*((__le32 *)&buf[2]) = cpu_to_le32(addr);
+	buf[2] = addr;
+	buf[3] = addr >> 8;
+	buf[4] = addr >> 16;
+	buf[5] = addr >> 24;
 
 	return SZ_DMAGO;
 }
@@ -1883,11 +1888,8 @@ static int dmac_alloc_resources(struct pl330_dmac *pl330)
 
 static int pl330_add(struct pl330_dmac *pl330)
 {
-	void __iomem *regs;
 	int i, ret;
 
-	regs = pl330->base;
-
 	/* Check if we can handle this DMAC */
 	if ((pl330->pcfg.periph_id & 0xfffff) != PERIPH_ID_VAL) {
 		dev_err(pl330->ddma.dev, "PERIPH_ID 0x%x !\n",
@@ -2263,6 +2265,11 @@ static int pl330_get_current_xferred_count(struct dma_pl330_chan *pch,
 	}
 	pm_runtime_mark_last_busy(pch->dmac->ddma.dev);
 	pm_runtime_put_autosuspend(pl330->ddma.dev);
+
+	/* If DMAMOV hasn't finished yet, SAR/DAR can be zero */
+	if (!val)
+		return 0;
+
 	return val - addr;
 }
 
diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c
index 3f56f9ca4482..b53fb618bbf6 100644
--- a/drivers/dma/pxa_dma.c
+++ b/drivers/dma/pxa_dma.c
@@ -413,15 +413,6 @@ static inline void pxad_init_debugfs(struct pxad_device *pdev) {}
 static inline void pxad_cleanup_debugfs(struct pxad_device *pdev) {}
 #endif
 
-/*
- * In the transition phase where legacy pxa handling is done at the same time as
- * mmp_dma, the DMA physical channel split between the 2 DMA providers is done
- * through legacy_reserved. Legacy code reserves DMA channels by settings
- * corresponding bits in legacy_reserved.
- */
-static u32 legacy_reserved;
-static u32 legacy_unavailable;
-
 static struct pxad_phy *lookup_phy(struct pxad_chan *pchan)
 {
 	int prio, i;
@@ -442,14 +433,10 @@ static struct pxad_phy *lookup_phy(struct pxad_chan *pchan)
 		for (i = 0; i < pdev->nr_chans; i++) {
 			if (prio != (i & 0xf) >> 2)
 				continue;
-			if ((i < 32) && (legacy_reserved & BIT(i)))
-				continue;
 			phy = &pdev->phys[i];
 			if (!phy->vchan) {
 				phy->vchan = pchan;
 				found = phy;
-				if (i < 32)
-					legacy_unavailable |= BIT(i);
 				goto out_unlock;
 			}
 		}
@@ -469,7 +456,6 @@ static void pxad_free_phy(struct pxad_chan *chan)
 	struct pxad_device *pdev = to_pxad_dev(chan->vc.chan.device);
 	unsigned long flags;
 	u32 reg;
-	int i;
 
 	dev_dbg(&chan->vc.chan.dev->device,
 		"%s(): freeing\n", __func__);
@@ -483,9 +469,6 @@ static void pxad_free_phy(struct pxad_chan *chan)
 	}
 
 	spin_lock_irqsave(&pdev->phy_lock, flags);
-	for (i = 0; i < 32; i++)
-		if (chan->phy == &pdev->phys[i])
-			legacy_unavailable &= ~BIT(i);
 	chan->phy->vchan = NULL;
 	chan->phy = NULL;
 	spin_unlock_irqrestore(&pdev->phy_lock, flags);
@@ -739,8 +722,6 @@ static irqreturn_t pxad_int_handler(int irq, void *dev_id)
 		i = __ffs(dint);
 		dint &= (dint - 1);
 		phy = &pdev->phys[i];
-		if ((i < 32) && (legacy_reserved & BIT(i)))
-			continue;
 		if (pxad_chan_handler(irq, phy) == IRQ_HANDLED)
 			ret = IRQ_HANDLED;
 	}
@@ -1522,15 +1503,6 @@ bool pxad_filter_fn(struct dma_chan *chan, void *param)
 }
 EXPORT_SYMBOL_GPL(pxad_filter_fn);
 
-int pxad_toggle_reserved_channel(int legacy_channel)
-{
-	if (legacy_unavailable & (BIT(legacy_channel)))
-		return -EBUSY;
-	legacy_reserved ^= BIT(legacy_channel);
-	return 0;
-}
-EXPORT_SYMBOL_GPL(pxad_toggle_reserved_channel);
-
 module_platform_driver(pxad_driver);
 
 MODULE_DESCRIPTION("Marvell PXA Peripheral DMA Driver");
diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c
index e244e10a94b5..3c982c96b4b7 100644
--- a/drivers/dma/qcom/hidma.c
+++ b/drivers/dma/qcom/hidma.c
@@ -56,6 +56,7 @@
 #include <linux/irq.h>
 #include <linux/atomic.h>
 #include <linux/pm_runtime.h>
+#include <linux/msi.h>
 
 #include "../dmaengine.h"
 #include "hidma.h"
@@ -70,6 +71,7 @@
 #define HIDMA_ERR_INFO_SW			0xFF
 #define HIDMA_ERR_CODE_UNEXPECTED_TERMINATE	0x0
 #define HIDMA_NR_DEFAULT_DESC			10
+#define HIDMA_MSI_INTS				11
 
 static inline struct hidma_dev *to_hidma_dev(struct dma_device *dmadev)
 {
@@ -553,6 +555,17 @@ static irqreturn_t hidma_chirq_handler(int chirq, void *arg)
 	return hidma_ll_inthandler(chirq, lldev);
 }
 
+#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
+static irqreturn_t hidma_chirq_handler_msi(int chirq, void *arg)
+{
+	struct hidma_lldev **lldevp = arg;
+	struct hidma_dev *dmadev = to_hidma_dev_from_lldev(lldevp);
+
+	return hidma_ll_inthandler_msi(chirq, *lldevp,
+				       1 << (chirq - dmadev->msi_virqbase));
+}
+#endif
+
 static ssize_t hidma_show_values(struct device *dev,
 				 struct device_attribute *attr, char *buf)
 {
@@ -567,8 +580,13 @@ static ssize_t hidma_show_values(struct device *dev,
 	return strlen(buf);
 }
 
-static int hidma_create_sysfs_entry(struct hidma_dev *dev, char *name,
-				    int mode)
+static inline void  hidma_sysfs_uninit(struct hidma_dev *dev)
+{
+	device_remove_file(dev->ddev.dev, dev->chid_attrs);
+}
+
+static struct device_attribute*
+hidma_create_sysfs_entry(struct hidma_dev *dev, char *name, int mode)
 {
 	struct device_attribute *attrs;
 	char *name_copy;
@@ -576,18 +594,125 @@ static int hidma_create_sysfs_entry(struct hidma_dev *dev, char *name,
 	attrs = devm_kmalloc(dev->ddev.dev, sizeof(struct device_attribute),
 			     GFP_KERNEL);
 	if (!attrs)
-		return -ENOMEM;
+		return NULL;
 
 	name_copy = devm_kstrdup(dev->ddev.dev, name, GFP_KERNEL);
 	if (!name_copy)
-		return -ENOMEM;
+		return NULL;
 
 	attrs->attr.name = name_copy;
 	attrs->attr.mode = mode;
 	attrs->show = hidma_show_values;
 	sysfs_attr_init(&attrs->attr);
 
-	return device_create_file(dev->ddev.dev, attrs);
+	return attrs;
+}
+
+static int hidma_sysfs_init(struct hidma_dev *dev)
+{
+	dev->chid_attrs = hidma_create_sysfs_entry(dev, "chid", S_IRUGO);
+	if (!dev->chid_attrs)
+		return -ENOMEM;
+
+	return device_create_file(dev->ddev.dev, dev->chid_attrs);
+}
+
+#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
+static void hidma_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
+{
+	struct device *dev = msi_desc_to_dev(desc);
+	struct hidma_dev *dmadev = dev_get_drvdata(dev);
+
+	if (!desc->platform.msi_index) {
+		writel(msg->address_lo, dmadev->dev_evca + 0x118);
+		writel(msg->address_hi, dmadev->dev_evca + 0x11C);
+		writel(msg->data, dmadev->dev_evca + 0x120);
+	}
+}
+#endif
+
+static void hidma_free_msis(struct hidma_dev *dmadev)
+{
+#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
+	struct device *dev = dmadev->ddev.dev;
+	struct msi_desc *desc;
+
+	/* free allocated MSI interrupts above */
+	for_each_msi_entry(desc, dev)
+		devm_free_irq(dev, desc->irq, &dmadev->lldev);
+
+	platform_msi_domain_free_irqs(dev);
+#endif
+}
+
+static int hidma_request_msi(struct hidma_dev *dmadev,
+			     struct platform_device *pdev)
+{
+#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
+	int rc;
+	struct msi_desc *desc;
+	struct msi_desc *failed_desc = NULL;
+
+	rc = platform_msi_domain_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS,
+					    hidma_write_msi_msg);
+	if (rc)
+		return rc;
+
+	for_each_msi_entry(desc, &pdev->dev) {
+		if (!desc->platform.msi_index)
+			dmadev->msi_virqbase = desc->irq;
+
+		rc = devm_request_irq(&pdev->dev, desc->irq,
+				       hidma_chirq_handler_msi,
+				       0, "qcom-hidma-msi",
+				       &dmadev->lldev);
+		if (rc) {
+			failed_desc = desc;
+			break;
+		}
+	}
+
+	if (rc) {
+		/* free allocated MSI interrupts above */
+		for_each_msi_entry(desc, &pdev->dev) {
+			if (desc == failed_desc)
+				break;
+			devm_free_irq(&pdev->dev, desc->irq,
+				      &dmadev->lldev);
+		}
+	} else {
+		/* Add callback to free MSIs on teardown */
+		hidma_ll_setup_irq(dmadev->lldev, true);
+
+	}
+	if (rc)
+		dev_warn(&pdev->dev,
+			 "failed to request MSI irq, falling back to wired IRQ\n");
+	return rc;
+#else
+	return -EINVAL;
+#endif
+}
+
+static bool hidma_msi_capable(struct device *dev)
+{
+	struct acpi_device *adev = ACPI_COMPANION(dev);
+	const char *of_compat;
+	int ret = -EINVAL;
+
+	if (!adev || acpi_disabled) {
+		ret = device_property_read_string(dev, "compatible",
+						  &of_compat);
+		if (ret)
+			return false;
+
+		ret = strcmp(of_compat, "qcom,hidma-1.1");
+	} else {
+#ifdef CONFIG_ACPI
+		ret = strcmp(acpi_device_hid(adev), "QCOM8062");
+#endif
+	}
+	return ret == 0;
 }
 
 static int hidma_probe(struct platform_device *pdev)
@@ -599,6 +724,7 @@ static int hidma_probe(struct platform_device *pdev)
 	void __iomem *evca;
 	void __iomem *trca;
 	int rc;
+	bool msi;
 
 	pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT);
 	pm_runtime_use_autosuspend(&pdev->dev);
@@ -660,6 +786,12 @@ static int hidma_probe(struct platform_device *pdev)
 	dmadev->ddev.device_terminate_all = hidma_terminate_all;
 	dmadev->ddev.copy_align = 8;
 
+	/*
+	 * Determine the MSI capability of the platform. Old HW doesn't
+	 * support MSI.
+	 */
+	msi = hidma_msi_capable(&pdev->dev);
+
 	device_property_read_u32(&pdev->dev, "desc-count",
 				 &dmadev->nr_descriptors);
 
@@ -688,10 +820,17 @@ static int hidma_probe(struct platform_device *pdev)
 		goto dmafree;
 	}
 
-	rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler, 0,
-			      "qcom-hidma", dmadev->lldev);
-	if (rc)
-		goto uninit;
+	platform_set_drvdata(pdev, dmadev);
+	if (msi)
+		rc = hidma_request_msi(dmadev, pdev);
+
+	if (!msi || rc) {
+		hidma_ll_setup_irq(dmadev->lldev, false);
+		rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler,
+				      0, "qcom-hidma", dmadev->lldev);
+		if (rc)
+			goto uninit;
+	}
 
 	INIT_LIST_HEAD(&dmadev->ddev.channels);
 	rc = hidma_chan_init(dmadev, 0);
@@ -705,14 +844,16 @@ static int hidma_probe(struct platform_device *pdev)
 	dmadev->irq = chirq;
 	tasklet_init(&dmadev->task, hidma_issue_task, (unsigned long)dmadev);
 	hidma_debug_init(dmadev);
-	hidma_create_sysfs_entry(dmadev, "chid", S_IRUGO);
+	hidma_sysfs_init(dmadev);
 	dev_info(&pdev->dev, "HI-DMA engine driver registration complete\n");
-	platform_set_drvdata(pdev, dmadev);
 	pm_runtime_mark_last_busy(dmadev->ddev.dev);
 	pm_runtime_put_autosuspend(dmadev->ddev.dev);
 	return 0;
 
 uninit:
+	if (msi)
+		hidma_free_msis(dmadev);
+
 	hidma_debug_uninit(dmadev);
 	hidma_ll_uninit(dmadev->lldev);
 dmafree:
@@ -730,8 +871,13 @@ static int hidma_remove(struct platform_device *pdev)
 
 	pm_runtime_get_sync(dmadev->ddev.dev);
 	dma_async_device_unregister(&dmadev->ddev);
-	devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev);
+	if (!dmadev->lldev->msi_support)
+		devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev);
+	else
+		hidma_free_msis(dmadev);
+
 	tasklet_kill(&dmadev->task);
+	hidma_sysfs_uninit(dmadev);
 	hidma_debug_uninit(dmadev);
 	hidma_ll_uninit(dmadev->lldev);
 	hidma_free(dmadev);
@@ -746,12 +892,15 @@ static int hidma_remove(struct platform_device *pdev)
 #if IS_ENABLED(CONFIG_ACPI)
 static const struct acpi_device_id hidma_acpi_ids[] = {
 	{"QCOM8061"},
+	{"QCOM8062"},
 	{},
 };
+MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids);
 #endif
 
 static const struct of_device_id hidma_match[] = {
 	{.compatible = "qcom,hidma-1.0",},
+	{.compatible = "qcom,hidma-1.1",},
 	{},
 };
 MODULE_DEVICE_TABLE(of, hidma_match);
diff --git a/drivers/dma/qcom/hidma.h b/drivers/dma/qcom/hidma.h
index e52e20716303..c7d014235c32 100644
--- a/drivers/dma/qcom/hidma.h
+++ b/drivers/dma/qcom/hidma.h
@@ -46,6 +46,7 @@ struct hidma_tre {
 };
 
 struct hidma_lldev {
+	bool msi_support;		/* flag indicating MSI support    */
 	bool initialized;		/* initialized flag               */
 	u8 trch_state;			/* trch_state of the device	  */
 	u8 evch_state;			/* evch_state of the device	  */
@@ -58,7 +59,7 @@ struct hidma_lldev {
 	void __iomem *evca;		/* Event Channel address          */
 	struct hidma_tre
 		**pending_tre_list;	/* Pointers to pending TREs	  */
-	s32 pending_tre_count;		/* Number of TREs pending	  */
+	atomic_t pending_tre_count;	/* Number of TREs pending	  */
 
 	void *tre_ring;			/* TRE ring			  */
 	dma_addr_t tre_dma;		/* TRE ring to be shared with HW  */
@@ -114,6 +115,7 @@ struct hidma_dev {
 	int				irq;
 	int				chidx;
 	u32				nr_descriptors;
+	int				msi_virqbase;
 
 	struct hidma_lldev		*lldev;
 	void				__iomem *dev_trca;
@@ -128,6 +130,9 @@ struct hidma_dev {
 	struct dentry			*debugfs;
 	struct dentry			*stats;
 
+	/* sysfs entry for the channel id */
+	struct device_attribute		*chid_attrs;
+
 	/* Task delivering issue_pending */
 	struct tasklet_struct		task;
 };
@@ -145,12 +150,14 @@ int hidma_ll_disable(struct hidma_lldev *lldev);
 int hidma_ll_enable(struct hidma_lldev *llhndl);
 void hidma_ll_set_transfer_params(struct hidma_lldev *llhndl, u32 tre_ch,
 	dma_addr_t src, dma_addr_t dest, u32 len, u32 flags);
+void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi);
 int hidma_ll_setup(struct hidma_lldev *lldev);
 struct hidma_lldev *hidma_ll_init(struct device *dev, u32 max_channels,
 			void __iomem *trca, void __iomem *evca,
 			u8 chidx);
 int hidma_ll_uninit(struct hidma_lldev *llhndl);
 irqreturn_t hidma_ll_inthandler(int irq, void *arg);
+irqreturn_t hidma_ll_inthandler_msi(int irq, void *arg, int cause);
 void hidma_cleanup_pending_tre(struct hidma_lldev *llhndl, u8 err_info,
 				u8 err_code);
 int hidma_debug_init(struct hidma_dev *dmadev);
diff --git a/drivers/dma/qcom/hidma_dbg.c b/drivers/dma/qcom/hidma_dbg.c
index fa827e5ffd68..3bdcb8056a36 100644
--- a/drivers/dma/qcom/hidma_dbg.c
+++ b/drivers/dma/qcom/hidma_dbg.c
@@ -74,7 +74,8 @@ static void hidma_ll_devstats(struct seq_file *s, void *llhndl)
 	seq_printf(s, "tre_ring_handle=%pap\n", &lldev->tre_dma);
 	seq_printf(s, "tre_ring_size = 0x%x\n", lldev->tre_ring_size);
 	seq_printf(s, "tre_processed_off = 0x%x\n", lldev->tre_processed_off);
-	seq_printf(s, "pending_tre_count=%d\n", lldev->pending_tre_count);
+	seq_printf(s, "pending_tre_count=%d\n",
+			atomic_read(&lldev->pending_tre_count));
 	seq_printf(s, "evca=%p\n", lldev->evca);
 	seq_printf(s, "evre_ring=%p\n", lldev->evre_ring);
 	seq_printf(s, "evre_ring_handle=%pap\n", &lldev->evre_dma);
@@ -164,7 +165,6 @@ static const struct file_operations hidma_dma_fops = {
 void hidma_debug_uninit(struct hidma_dev *dmadev)
 {
 	debugfs_remove_recursive(dmadev->debugfs);
-	debugfs_remove_recursive(dmadev->stats);
 }
 
 int hidma_debug_init(struct hidma_dev *dmadev)
diff --git a/drivers/dma/qcom/hidma_ll.c b/drivers/dma/qcom/hidma_ll.c
index 3224f24c577b..6645bdf0d151 100644
--- a/drivers/dma/qcom/hidma_ll.c
+++ b/drivers/dma/qcom/hidma_ll.c
@@ -198,13 +198,16 @@ static void hidma_ll_tre_complete(unsigned long arg)
 	}
 }
 
-static int hidma_post_completed(struct hidma_lldev *lldev, int tre_iterator,
-				u8 err_info, u8 err_code)
+static int hidma_post_completed(struct hidma_lldev *lldev, u8 err_info,
+				u8 err_code)
 {
 	struct hidma_tre *tre;
 	unsigned long flags;
+	u32 tre_iterator;
 
 	spin_lock_irqsave(&lldev->lock, flags);
+
+	tre_iterator = lldev->tre_processed_off;
 	tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE];
 	if (!tre) {
 		spin_unlock_irqrestore(&lldev->lock, flags);
@@ -218,12 +221,14 @@ static int hidma_post_completed(struct hidma_lldev *lldev, int tre_iterator,
 	 * Keep track of pending TREs that SW is expecting to receive
 	 * from HW. We got one now. Decrement our counter.
 	 */
-	lldev->pending_tre_count--;
-	if (lldev->pending_tre_count < 0) {
+	if (atomic_dec_return(&lldev->pending_tre_count) < 0) {
 		dev_warn(lldev->dev, "tre count mismatch on completion");
-		lldev->pending_tre_count = 0;
+		atomic_set(&lldev->pending_tre_count, 0);
 	}
 
+	HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE,
+				 lldev->tre_ring_size);
+	lldev->tre_processed_off = tre_iterator;
 	spin_unlock_irqrestore(&lldev->lock, flags);
 
 	tre->err_info = err_info;
@@ -245,13 +250,11 @@ static int hidma_post_completed(struct hidma_lldev *lldev, int tre_iterator,
 static int hidma_handle_tre_completion(struct hidma_lldev *lldev)
 {
 	u32 evre_ring_size = lldev->evre_ring_size;
-	u32 tre_ring_size = lldev->tre_ring_size;
 	u32 err_info, err_code, evre_write_off;
-	u32 tre_iterator, evre_iterator;
+	u32 evre_iterator;
 	u32 num_completed = 0;
 
 	evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);
-	tre_iterator = lldev->tre_processed_off;
 	evre_iterator = lldev->evre_processed_off;
 
 	if ((evre_write_off > evre_ring_size) ||
@@ -274,12 +277,9 @@ static int hidma_handle_tre_completion(struct hidma_lldev *lldev)
 		err_code =
 		    (cfg >> HIDMA_EVRE_CODE_BIT_POS) & HIDMA_EVRE_CODE_MASK;
 
-		if (hidma_post_completed(lldev, tre_iterator, err_info,
-					 err_code))
+		if (hidma_post_completed(lldev, err_info, err_code))
 			break;
 
-		HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE,
-					 tre_ring_size);
 		HIDMA_INCREMENT_ITERATOR(evre_iterator, HIDMA_EVRE_SIZE,
 					 evre_ring_size);
 
@@ -291,21 +291,22 @@ static int hidma_handle_tre_completion(struct hidma_lldev *lldev)
 		evre_write_off =
 		    readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);
 		num_completed++;
+
+		/*
+		 * An error interrupt might have arrived while we are processing
+		 * the completed interrupt.
+		 */
+		if (!hidma_ll_isenabled(lldev))
+			break;
 	}
 
 	if (num_completed) {
 		u32 evre_read_off = (lldev->evre_processed_off +
 				     HIDMA_EVRE_SIZE * num_completed);
-		u32 tre_read_off = (lldev->tre_processed_off +
-				    HIDMA_TRE_SIZE * num_completed);
-
 		evre_read_off = evre_read_off % evre_ring_size;
-		tre_read_off = tre_read_off % tre_ring_size;
-
 		writel(evre_read_off, lldev->evca + HIDMA_EVCA_DOORBELL_REG);
 
 		/* record the last processed tre offset */
-		lldev->tre_processed_off = tre_read_off;
 		lldev->evre_processed_off = evre_read_off;
 	}
 
@@ -315,27 +316,10 @@ static int hidma_handle_tre_completion(struct hidma_lldev *lldev)
 void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info,
 			       u8 err_code)
 {
-	u32 tre_iterator;
-	u32 tre_ring_size = lldev->tre_ring_size;
-	int num_completed = 0;
-	u32 tre_read_off;
-
-	tre_iterator = lldev->tre_processed_off;
-	while (lldev->pending_tre_count) {
-		if (hidma_post_completed(lldev, tre_iterator, err_info,
-					 err_code))
+	while (atomic_read(&lldev->pending_tre_count)) {
+		if (hidma_post_completed(lldev, err_info, err_code))
 			break;
-		HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE,
-					 tre_ring_size);
-		num_completed++;
 	}
-	tre_read_off = (lldev->tre_processed_off +
-			HIDMA_TRE_SIZE * num_completed);
-
-	tre_read_off = tre_read_off % tre_ring_size;
-
-	/* record the last processed tre offset */
-	lldev->tre_processed_off = tre_read_off;
 }
 
 static int hidma_ll_reset(struct hidma_lldev *lldev)
@@ -412,12 +396,24 @@ static int hidma_ll_reset(struct hidma_lldev *lldev)
  * requests traditionally to the destination, this concept does not apply
  * here for this HW.
  */
-irqreturn_t hidma_ll_inthandler(int chirq, void *arg)
+static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause)
 {
-	struct hidma_lldev *lldev = arg;
-	u32 status;
-	u32 enable;
-	u32 cause;
+	if (cause & HIDMA_ERR_INT_MASK) {
+		dev_err(lldev->dev, "error 0x%x, disabling...\n",
+				cause);
+
+		/* Clear out pending interrupts */
+		writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
+
+		/* No further submissions. */
+		hidma_ll_disable(lldev);
+
+		/* Driver completes the txn and intimates the client.*/
+		hidma_cleanup_pending_tre(lldev, 0xFF,
+					  HIDMA_EVRE_STATUS_ERROR);
+
+		return;
+	}
 
 	/*
 	 * Fine tuned for this HW...
@@ -426,35 +422,28 @@ irqreturn_t hidma_ll_inthandler(int chirq, void *arg)
 	 * read and write accessors are used for performance reasons due to
 	 * interrupt delivery guarantees. Do not copy this code blindly and
 	 * expect that to work.
+	 *
+	 * Try to consume as many EVREs as possible.
 	 */
+	hidma_handle_tre_completion(lldev);
+
+	/* We consumed TREs or there are pending TREs or EVREs. */
+	writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
+}
+
+irqreturn_t hidma_ll_inthandler(int chirq, void *arg)
+{
+	struct hidma_lldev *lldev = arg;
+	u32 status;
+	u32 enable;
+	u32 cause;
+
 	status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
 	enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
 	cause = status & enable;
 
 	while (cause) {
-		if (cause & HIDMA_ERR_INT_MASK) {
-			dev_err(lldev->dev, "error 0x%x, disabling...\n",
-					cause);
-
-			/* Clear out pending interrupts */
-			writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
-
-			/* No further submissions. */
-			hidma_ll_disable(lldev);
-
-			/* Driver completes the txn and intimates the client.*/
-			hidma_cleanup_pending_tre(lldev, 0xFF,
-						  HIDMA_EVRE_STATUS_ERROR);
-			goto out;
-		}
-
-		/*
-		 * Try to consume as many EVREs as possible.
-		 */
-		hidma_handle_tre_completion(lldev);
-
-		/* We consumed TREs or there are pending TREs or EVREs. */
-		writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
+		hidma_ll_int_handler_internal(lldev, cause);
 
 		/*
 		 * Another interrupt might have arrived while we are
@@ -465,7 +454,14 @@ irqreturn_t hidma_ll_inthandler(int chirq, void *arg)
 		cause = status & enable;
 	}
 
-out:
+	return IRQ_HANDLED;
+}
+
+irqreturn_t hidma_ll_inthandler_msi(int chirq, void *arg, int cause)
+{
+	struct hidma_lldev *lldev = arg;
+
+	hidma_ll_int_handler_internal(lldev, cause);
 	return IRQ_HANDLED;
 }
 
@@ -548,7 +544,7 @@ void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch)
 	tre->err_code = 0;
 	tre->err_info = 0;
 	tre->queued = 1;
-	lldev->pending_tre_count++;
+	atomic_inc(&lldev->pending_tre_count);
 	lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE)
 					% lldev->tre_ring_size;
 	spin_unlock_irqrestore(&lldev->lock, flags);
@@ -564,19 +560,8 @@ int hidma_ll_disable(struct hidma_lldev *lldev)
 	u32 val;
 	int ret;
 
-	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
-	lldev->evch_state = HIDMA_CH_STATE(val);
-	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
-	lldev->trch_state = HIDMA_CH_STATE(val);
-
-	/* already suspended by this OS */
-	if ((lldev->trch_state == HIDMA_CH_SUSPENDED) ||
-	    (lldev->evch_state == HIDMA_CH_SUSPENDED))
-		return 0;
-
-	/* already stopped by the manager */
-	if ((lldev->trch_state == HIDMA_CH_STOPPED) ||
-	    (lldev->evch_state == HIDMA_CH_STOPPED))
+	/* The channel needs to be in working state */
+	if (!hidma_ll_isenabled(lldev))
 		return 0;
 
 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
@@ -654,7 +639,7 @@ int hidma_ll_setup(struct hidma_lldev *lldev)
 	u32 val;
 	u32 nr_tres = lldev->nr_tres;
 
-	lldev->pending_tre_count = 0;
+	atomic_set(&lldev->pending_tre_count, 0);
 	lldev->tre_processed_off = 0;
 	lldev->evre_processed_off = 0;
 	lldev->tre_write_offset = 0;
@@ -691,17 +676,36 @@ int hidma_ll_setup(struct hidma_lldev *lldev)
 	writel(HIDMA_EVRE_SIZE * nr_tres,
 			lldev->evca + HIDMA_EVCA_RING_LEN_REG);
 
-	/* support IRQ only for now */
+	/* configure interrupts */
+	hidma_ll_setup_irq(lldev, lldev->msi_support);
+
+	rc = hidma_ll_enable(lldev);
+	if (rc)
+		return rc;
+
+	return rc;
+}
+
+void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi)
+{
+	u32 val;
+
+	lldev->msi_support = msi;
+
+	/* disable interrupts again after reset */
+	writel(0, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
+	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
+
+	/* support IRQ by default */
 	val = readl(lldev->evca + HIDMA_EVCA_INTCTRL_REG);
 	val &= ~0xF;
-	val |= 0x1;
+	if (!lldev->msi_support)
+		val = val | 0x1;
 	writel(val, lldev->evca + HIDMA_EVCA_INTCTRL_REG);
 
 	/* clear all pending interrupts and enable them */
 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
-
-	return hidma_ll_enable(lldev);
 }
 
 struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres,
@@ -816,7 +820,7 @@ int hidma_ll_uninit(struct hidma_lldev *lldev)
 	tasklet_kill(&lldev->task);
 	memset(lldev->trepool, 0, required_bytes);
 	lldev->trepool = NULL;
-	lldev->pending_tre_count = 0;
+	atomic_set(&lldev->pending_tre_count, 0);
 	lldev->tre_write_offset = 0;
 
 	rc = hidma_ll_reset(lldev);
diff --git a/drivers/dma/qcom/hidma_mgmt.c b/drivers/dma/qcom/hidma_mgmt.c
index 82f36e466083..f847d32cc4b5 100644
--- a/drivers/dma/qcom/hidma_mgmt.c
+++ b/drivers/dma/qcom/hidma_mgmt.c
@@ -282,6 +282,7 @@ static const struct acpi_device_id hidma_mgmt_acpi_ids[] = {
 	{"QCOM8060"},
 	{},
 };
+MODULE_DEVICE_TABLE(acpi, hidma_mgmt_acpi_ids);
 #endif
 
 static const struct of_device_id hidma_mgmt_match[] = {
@@ -375,8 +376,15 @@ static int __init hidma_mgmt_of_populate_channels(struct device_node *np)
 			ret = PTR_ERR(new_pdev);
 			goto out;
 		}
+		of_node_get(child);
+		new_pdev->dev.of_node = child;
 		of_dma_configure(&new_pdev->dev, child);
-
+		/*
+		 * It is assumed that calling of_msi_configure is safe on
+		 * platforms with or without MSI support.
+		 */
+		of_msi_configure(&new_pdev->dev, child);
+		of_node_put(child);
 		kfree(res);
 		res = NULL;
 	}
@@ -395,7 +403,6 @@ static int __init hidma_mgmt_init(void)
 	for_each_matching_node(child, hidma_mgmt_match) {
 		/* device tree based firmware here */
 		hidma_mgmt_of_populate_channels(child);
-		of_node_put(child);
 	}
 #endif
 	platform_driver_register(&hidma_mgmt_driver);
diff --git a/drivers/dma/s3c24xx-dma.c b/drivers/dma/s3c24xx-dma.c
index 3c579abbabb7..f04c4702d98b 100644
--- a/drivers/dma/s3c24xx-dma.c
+++ b/drivers/dma/s3c24xx-dma.c
@@ -289,16 +289,11 @@ static
 struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan)
 {
 	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
-	const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
-	struct s3c24xx_dma_channel *cdata;
 	struct s3c24xx_dma_phy *phy = NULL;
 	unsigned long flags;
 	int i;
 	int ret;
 
-	if (s3cchan->slave)
-		cdata = &pdata->channels[s3cchan->id];
-
 	for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
 		phy = &s3cdma->phy_chans[i];
 
diff --git a/drivers/dma/sh/usb-dmac.c b/drivers/dma/sh/usb-dmac.c
index 06ecdc38cee0..72c649713ace 100644
--- a/drivers/dma/sh/usb-dmac.c
+++ b/drivers/dma/sh/usb-dmac.c
@@ -652,7 +652,6 @@ static bool usb_dmac_chan_filter(struct dma_chan *chan, void *arg)
 static struct dma_chan *usb_dmac_of_xlate(struct of_phandle_args *dma_spec,
 					  struct of_dma *ofdma)
 {
-	struct usb_dmac_chan *uchan;
 	struct dma_chan *chan;
 	dma_cap_mask_t mask;
 
@@ -667,8 +666,6 @@ static struct dma_chan *usb_dmac_of_xlate(struct of_phandle_args *dma_spec,
 	if (!chan)
 		return NULL;
 
-	uchan = to_usb_dmac_chan(chan);
-
 	return chan;
 }
 
diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c
index 8f62edad51be..a0733ac3edb1 100644
--- a/drivers/dma/sirf-dma.c
+++ b/drivers/dma/sirf-dma.c
@@ -1011,7 +1011,6 @@ static int __maybe_unused sirfsoc_dma_pm_suspend(struct device *dev)
 {
 	struct sirfsoc_dma *sdma = dev_get_drvdata(dev);
 	struct sirfsoc_dma_regs *save = &sdma->regs_save;
-	struct sirfsoc_dma_desc *sdesc;
 	struct sirfsoc_dma_chan *schan;
 	int ch;
 	int ret;
@@ -1044,9 +1043,6 @@ static int __maybe_unused sirfsoc_dma_pm_suspend(struct device *dev)
 		schan = &sdma->channels[ch];
 		if (list_empty(&schan->active))
 			continue;
-		sdesc = list_first_entry(&schan->active,
-			struct sirfsoc_dma_desc,
-			node);
 		save->ctrl[ch] = readl_relaxed(sdma->base +
 			ch * 0x10 + SIRFSOC_DMA_CH_CTRL);
 	}
diff --git a/drivers/dma/st_fdma.c b/drivers/dma/st_fdma.c
new file mode 100644
index 000000000000..bfb79bd0c6de
--- /dev/null
+++ b/drivers/dma/st_fdma.c
@@ -0,0 +1,889 @@
+/*
+ * DMA driver for STMicroelectronics STi FDMA controller
+ *
+ * Copyright (C) 2014 STMicroelectronics
+ *
+ * Author: Ludovic Barre <Ludovic.barre@st.com>
+ *	   Peter Griffin <peter.griffin@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/remoteproc.h>
+
+#include "st_fdma.h"
+
+static inline struct st_fdma_chan *to_st_fdma_chan(struct dma_chan *c)
+{
+	return container_of(c, struct st_fdma_chan, vchan.chan);
+}
+
+static struct st_fdma_desc *to_st_fdma_desc(struct virt_dma_desc *vd)
+{
+	return container_of(vd, struct st_fdma_desc, vdesc);
+}
+
+static int st_fdma_dreq_get(struct st_fdma_chan *fchan)
+{
+	struct st_fdma_dev *fdev = fchan->fdev;
+	u32 req_line_cfg = fchan->cfg.req_line;
+	u32 dreq_line;
+	int try = 0;
+
+	/*
+	 * dreq_mask is shared for n channels of fdma, so all accesses must be
+	 * atomic. if the dreq_mask is changed between ffz and set_bit,
+	 * we retry
+	 */
+	do {
+		if (fdev->dreq_mask == ~0L) {
+			dev_err(fdev->dev, "No req lines available\n");
+			return -EINVAL;
+		}
+
+		if (try || req_line_cfg >= ST_FDMA_NR_DREQS) {
+			dev_err(fdev->dev, "Invalid or used req line\n");
+			return -EINVAL;
+		} else {
+			dreq_line = req_line_cfg;
+		}
+
+		try++;
+	} while (test_and_set_bit(dreq_line, &fdev->dreq_mask));
+
+	dev_dbg(fdev->dev, "get dreq_line:%d mask:%#lx\n",
+		dreq_line, fdev->dreq_mask);
+
+	return dreq_line;
+}
+
+static void st_fdma_dreq_put(struct st_fdma_chan *fchan)
+{
+	struct st_fdma_dev *fdev = fchan->fdev;
+
+	dev_dbg(fdev->dev, "put dreq_line:%#x\n", fchan->dreq_line);
+	clear_bit(fchan->dreq_line, &fdev->dreq_mask);
+}
+
+static void st_fdma_xfer_desc(struct st_fdma_chan *fchan)
+{
+	struct virt_dma_desc *vdesc;
+	unsigned long nbytes, ch_cmd, cmd;
+
+	vdesc = vchan_next_desc(&fchan->vchan);
+	if (!vdesc)
+		return;
+
+	fchan->fdesc = to_st_fdma_desc(vdesc);
+	nbytes = fchan->fdesc->node[0].desc->nbytes;
+	cmd = FDMA_CMD_START(fchan->vchan.chan.chan_id);
+	ch_cmd = fchan->fdesc->node[0].pdesc | FDMA_CH_CMD_STA_START;
+
+	/* start the channel for the descriptor */
+	fnode_write(fchan, nbytes, FDMA_CNTN_OFST);
+	fchan_write(fchan, ch_cmd, FDMA_CH_CMD_OFST);
+	writel(cmd,
+		fchan->fdev->slim_rproc->peri + FDMA_CMD_SET_OFST);
+
+	dev_dbg(fchan->fdev->dev, "start chan:%d\n", fchan->vchan.chan.chan_id);
+}
+
+static void st_fdma_ch_sta_update(struct st_fdma_chan *fchan,
+				  unsigned long int_sta)
+{
+	unsigned long ch_sta, ch_err;
+	int ch_id = fchan->vchan.chan.chan_id;
+	struct st_fdma_dev *fdev = fchan->fdev;
+
+	ch_sta = fchan_read(fchan, FDMA_CH_CMD_OFST);
+	ch_err = ch_sta & FDMA_CH_CMD_ERR_MASK;
+	ch_sta &= FDMA_CH_CMD_STA_MASK;
+
+	if (int_sta & FDMA_INT_STA_ERR) {
+		dev_warn(fdev->dev, "chan:%d, error:%ld\n", ch_id, ch_err);
+		fchan->status = DMA_ERROR;
+		return;
+	}
+
+	switch (ch_sta) {
+	case FDMA_CH_CMD_STA_PAUSED:
+		fchan->status = DMA_PAUSED;
+		break;
+
+	case FDMA_CH_CMD_STA_RUNNING:
+		fchan->status = DMA_IN_PROGRESS;
+		break;
+	}
+}
+
+static irqreturn_t st_fdma_irq_handler(int irq, void *dev_id)
+{
+	struct st_fdma_dev *fdev = dev_id;
+	irqreturn_t ret = IRQ_NONE;
+	struct st_fdma_chan *fchan = &fdev->chans[0];
+	unsigned long int_sta, clr;
+
+	int_sta = fdma_read(fdev, FDMA_INT_STA_OFST);
+	clr = int_sta;
+
+	for (; int_sta != 0 ; int_sta >>= 2, fchan++) {
+		if (!(int_sta & (FDMA_INT_STA_CH | FDMA_INT_STA_ERR)))
+			continue;
+
+		spin_lock(&fchan->vchan.lock);
+		st_fdma_ch_sta_update(fchan, int_sta);
+
+		if (fchan->fdesc) {
+			if (!fchan->fdesc->iscyclic) {
+				list_del(&fchan->fdesc->vdesc.node);
+				vchan_cookie_complete(&fchan->fdesc->vdesc);
+				fchan->fdesc = NULL;
+				fchan->status = DMA_COMPLETE;
+			} else {
+				vchan_cyclic_callback(&fchan->fdesc->vdesc);
+			}
+
+			/* Start the next descriptor (if available) */
+			if (!fchan->fdesc)
+				st_fdma_xfer_desc(fchan);
+		}
+
+		spin_unlock(&fchan->vchan.lock);
+		ret = IRQ_HANDLED;
+	}
+
+	fdma_write(fdev, clr, FDMA_INT_CLR_OFST);
+
+	return ret;
+}
+
+static struct dma_chan *st_fdma_of_xlate(struct of_phandle_args *dma_spec,
+					 struct of_dma *ofdma)
+{
+	struct st_fdma_dev *fdev = ofdma->of_dma_data;
+	struct dma_chan *chan;
+	struct st_fdma_chan *fchan;
+	int ret;
+
+	if (dma_spec->args_count < 1)
+		return ERR_PTR(-EINVAL);
+
+	if (fdev->dma_device.dev->of_node != dma_spec->np)
+		return ERR_PTR(-EINVAL);
+
+	ret = rproc_boot(fdev->slim_rproc->rproc);
+	if (ret == -ENOENT)
+		return ERR_PTR(-EPROBE_DEFER);
+	else if (ret)
+		return ERR_PTR(ret);
+
+	chan = dma_get_any_slave_channel(&fdev->dma_device);
+	if (!chan)
+		goto err_chan;
+
+	fchan = to_st_fdma_chan(chan);
+
+	fchan->cfg.of_node = dma_spec->np;
+	fchan->cfg.req_line = dma_spec->args[0];
+	fchan->cfg.req_ctrl = 0;
+	fchan->cfg.type = ST_FDMA_TYPE_FREE_RUN;
+
+	if (dma_spec->args_count > 1)
+		fchan->cfg.req_ctrl = dma_spec->args[1]
+			& FDMA_REQ_CTRL_CFG_MASK;
+
+	if (dma_spec->args_count > 2)
+		fchan->cfg.type = dma_spec->args[2];
+
+	if (fchan->cfg.type == ST_FDMA_TYPE_FREE_RUN) {
+		fchan->dreq_line = 0;
+	} else {
+		fchan->dreq_line = st_fdma_dreq_get(fchan);
+		if (IS_ERR_VALUE(fchan->dreq_line)) {
+			chan = ERR_PTR(fchan->dreq_line);
+			goto err_chan;
+		}
+	}
+
+	dev_dbg(fdev->dev, "xlate req_line:%d type:%d req_ctrl:%#lx\n",
+		fchan->cfg.req_line, fchan->cfg.type, fchan->cfg.req_ctrl);
+
+	return chan;
+
+err_chan:
+	rproc_shutdown(fdev->slim_rproc->rproc);
+	return chan;
+
+}
+
+static void st_fdma_free_desc(struct virt_dma_desc *vdesc)
+{
+	struct st_fdma_desc *fdesc;
+	int i;
+
+	fdesc = to_st_fdma_desc(vdesc);
+	for (i = 0; i < fdesc->n_nodes; i++)
+		dma_pool_free(fdesc->fchan->node_pool, fdesc->node[i].desc,
+			      fdesc->node[i].pdesc);
+	kfree(fdesc);
+}
+
+static struct st_fdma_desc *st_fdma_alloc_desc(struct st_fdma_chan *fchan,
+					       int sg_len)
+{
+	struct st_fdma_desc *fdesc;
+	int i;
+
+	fdesc = kzalloc(sizeof(*fdesc) +
+			sizeof(struct st_fdma_sw_node) * sg_len, GFP_NOWAIT);
+	if (!fdesc)
+		return NULL;
+
+	fdesc->fchan = fchan;
+	fdesc->n_nodes = sg_len;
+	for (i = 0; i < sg_len; i++) {
+		fdesc->node[i].desc = dma_pool_alloc(fchan->node_pool,
+				GFP_NOWAIT, &fdesc->node[i].pdesc);
+		if (!fdesc->node[i].desc)
+			goto err;
+	}
+	return fdesc;
+
+err:
+	while (--i >= 0)
+		dma_pool_free(fchan->node_pool, fdesc->node[i].desc,
+			      fdesc->node[i].pdesc);
+	kfree(fdesc);
+	return NULL;
+}
+
+static int st_fdma_alloc_chan_res(struct dma_chan *chan)
+{
+	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
+
+	/* Create the dma pool for descriptor allocation */
+	fchan->node_pool = dma_pool_create(dev_name(&chan->dev->device),
+					    fchan->fdev->dev,
+					    sizeof(struct st_fdma_hw_node),
+					    __alignof__(struct st_fdma_hw_node),
+					    0);
+
+	if (!fchan->node_pool) {
+		dev_err(fchan->fdev->dev, "unable to allocate desc pool\n");
+		return -ENOMEM;
+	}
+
+	dev_dbg(fchan->fdev->dev, "alloc ch_id:%d type:%d\n",
+		fchan->vchan.chan.chan_id, fchan->cfg.type);
+
+	return 0;
+}
+
+static void st_fdma_free_chan_res(struct dma_chan *chan)
+{
+	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
+	struct rproc *rproc = fchan->fdev->slim_rproc->rproc;
+	unsigned long flags;
+
+	LIST_HEAD(head);
+
+	dev_dbg(fchan->fdev->dev, "%s: freeing chan:%d\n",
+		__func__, fchan->vchan.chan.chan_id);
+
+	if (fchan->cfg.type != ST_FDMA_TYPE_FREE_RUN)
+		st_fdma_dreq_put(fchan);
+
+	spin_lock_irqsave(&fchan->vchan.lock, flags);
+	fchan->fdesc = NULL;
+	spin_unlock_irqrestore(&fchan->vchan.lock, flags);
+
+	dma_pool_destroy(fchan->node_pool);
+	fchan->node_pool = NULL;
+	memset(&fchan->cfg, 0, sizeof(struct st_fdma_cfg));
+
+	rproc_shutdown(rproc);
+}
+
+static struct dma_async_tx_descriptor *st_fdma_prep_dma_memcpy(
+	struct dma_chan *chan,	dma_addr_t dst, dma_addr_t src,
+	size_t len, unsigned long flags)
+{
+	struct st_fdma_chan *fchan;
+	struct st_fdma_desc *fdesc;
+	struct st_fdma_hw_node *hw_node;
+
+	if (!len)
+		return NULL;
+
+	fchan = to_st_fdma_chan(chan);
+
+	/* We only require a single descriptor */
+	fdesc = st_fdma_alloc_desc(fchan, 1);
+	if (!fdesc) {
+		dev_err(fchan->fdev->dev, "no memory for desc\n");
+		return NULL;
+	}
+
+	hw_node = fdesc->node[0].desc;
+	hw_node->next = 0;
+	hw_node->control = FDMA_NODE_CTRL_REQ_MAP_FREE_RUN;
+	hw_node->control |= FDMA_NODE_CTRL_SRC_INCR;
+	hw_node->control |= FDMA_NODE_CTRL_DST_INCR;
+	hw_node->control |= FDMA_NODE_CTRL_INT_EON;
+	hw_node->nbytes = len;
+	hw_node->saddr = src;
+	hw_node->daddr = dst;
+	hw_node->generic.length = len;
+	hw_node->generic.sstride = 0;
+	hw_node->generic.dstride = 0;
+
+	return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags);
+}
+
+static int config_reqctrl(struct st_fdma_chan *fchan,
+			  enum dma_transfer_direction direction)
+{
+	u32 maxburst = 0, addr = 0;
+	enum dma_slave_buswidth width;
+	int ch_id = fchan->vchan.chan.chan_id;
+	struct st_fdma_dev *fdev = fchan->fdev;
+
+	switch (direction) {
+
+	case DMA_DEV_TO_MEM:
+		fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_WNR;
+		maxburst = fchan->scfg.src_maxburst;
+		width = fchan->scfg.src_addr_width;
+		addr = fchan->scfg.src_addr;
+		break;
+
+	case DMA_MEM_TO_DEV:
+		fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_WNR;
+		maxburst = fchan->scfg.dst_maxburst;
+		width = fchan->scfg.dst_addr_width;
+		addr = fchan->scfg.dst_addr;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_OPCODE_MASK;
+
+	switch (width) {
+
+	case DMA_SLAVE_BUSWIDTH_1_BYTE:
+		fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST1;
+		break;
+
+	case DMA_SLAVE_BUSWIDTH_2_BYTES:
+		fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST2;
+		break;
+
+	case DMA_SLAVE_BUSWIDTH_4_BYTES:
+		fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST4;
+		break;
+
+	case DMA_SLAVE_BUSWIDTH_8_BYTES:
+		fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST8;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_NUM_OPS_MASK;
+	fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_NUM_OPS(maxburst-1);
+	dreq_write(fchan, fchan->cfg.req_ctrl, FDMA_REQ_CTRL_OFST);
+
+	fchan->cfg.dev_addr = addr;
+	fchan->cfg.dir = direction;
+
+	dev_dbg(fdev->dev, "chan:%d config_reqctrl:%#x req_ctrl:%#lx\n",
+		ch_id, addr, fchan->cfg.req_ctrl);
+
+	return 0;
+}
+
+static void fill_hw_node(struct st_fdma_hw_node *hw_node,
+			struct st_fdma_chan *fchan,
+			enum dma_transfer_direction direction)
+{
+	if (direction == DMA_MEM_TO_DEV) {
+		hw_node->control |= FDMA_NODE_CTRL_SRC_INCR;
+		hw_node->control |= FDMA_NODE_CTRL_DST_STATIC;
+		hw_node->daddr = fchan->cfg.dev_addr;
+	} else {
+		hw_node->control |= FDMA_NODE_CTRL_SRC_STATIC;
+		hw_node->control |= FDMA_NODE_CTRL_DST_INCR;
+		hw_node->saddr = fchan->cfg.dev_addr;
+	}
+
+	hw_node->generic.sstride = 0;
+	hw_node->generic.dstride = 0;
+}
+
+static inline struct st_fdma_chan *st_fdma_prep_common(struct dma_chan *chan,
+		size_t len, enum dma_transfer_direction direction)
+{
+	struct st_fdma_chan *fchan;
+
+	if (!chan || !len)
+		return NULL;
+
+	fchan = to_st_fdma_chan(chan);
+
+	if (!is_slave_direction(direction)) {
+		dev_err(fchan->fdev->dev, "bad direction?\n");
+		return NULL;
+	}
+
+	return fchan;
+}
+
+static struct dma_async_tx_descriptor *st_fdma_prep_dma_cyclic(
+		struct dma_chan *chan, dma_addr_t buf_addr, size_t len,
+		size_t period_len, enum dma_transfer_direction direction,
+		unsigned long flags)
+{
+	struct st_fdma_chan *fchan;
+	struct st_fdma_desc *fdesc;
+	int sg_len, i;
+
+	fchan = st_fdma_prep_common(chan, len, direction);
+	if (!fchan)
+		return NULL;
+
+	if (!period_len)
+		return NULL;
+
+	if (config_reqctrl(fchan, direction)) {
+		dev_err(fchan->fdev->dev, "bad width or direction\n");
+		return NULL;
+	}
+
+	/* the buffer length must be a multiple of period_len */
+	if (len % period_len != 0) {
+		dev_err(fchan->fdev->dev, "len is not multiple of period\n");
+		return NULL;
+	}
+
+	sg_len = len / period_len;
+	fdesc = st_fdma_alloc_desc(fchan, sg_len);
+	if (!fdesc) {
+		dev_err(fchan->fdev->dev, "no memory for desc\n");
+		return NULL;
+	}
+
+	fdesc->iscyclic = true;
+
+	for (i = 0; i < sg_len; i++) {
+		struct st_fdma_hw_node *hw_node = fdesc->node[i].desc;
+
+		hw_node->next = fdesc->node[(i + 1) % sg_len].pdesc;
+
+		hw_node->control =
+			FDMA_NODE_CTRL_REQ_MAP_DREQ(fchan->dreq_line);
+		hw_node->control |= FDMA_NODE_CTRL_INT_EON;
+
+		fill_hw_node(hw_node, fchan, direction);
+
+		if (direction == DMA_MEM_TO_DEV)
+			hw_node->saddr = buf_addr + (i * period_len);
+		else
+			hw_node->daddr = buf_addr + (i * period_len);
+
+		hw_node->nbytes = period_len;
+		hw_node->generic.length = period_len;
+	}
+
+	return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags);
+}
+
+static struct dma_async_tx_descriptor *st_fdma_prep_slave_sg(
+		struct dma_chan *chan, struct scatterlist *sgl,
+		unsigned int sg_len, enum dma_transfer_direction direction,
+		unsigned long flags, void *context)
+{
+	struct st_fdma_chan *fchan;
+	struct st_fdma_desc *fdesc;
+	struct st_fdma_hw_node *hw_node;
+	struct scatterlist *sg;
+	int i;
+
+	fchan = st_fdma_prep_common(chan, sg_len, direction);
+	if (!fchan)
+		return NULL;
+
+	if (!sgl)
+		return NULL;
+
+	fdesc = st_fdma_alloc_desc(fchan, sg_len);
+	if (!fdesc) {
+		dev_err(fchan->fdev->dev, "no memory for desc\n");
+		return NULL;
+	}
+
+	fdesc->iscyclic = false;
+
+	for_each_sg(sgl, sg, sg_len, i) {
+		hw_node = fdesc->node[i].desc;
+
+		hw_node->next = fdesc->node[(i + 1) % sg_len].pdesc;
+		hw_node->control = FDMA_NODE_CTRL_REQ_MAP_DREQ(fchan->dreq_line);
+
+		fill_hw_node(hw_node, fchan, direction);
+
+		if (direction == DMA_MEM_TO_DEV)
+			hw_node->saddr = sg_dma_address(sg);
+		else
+			hw_node->daddr = sg_dma_address(sg);
+
+		hw_node->nbytes = sg_dma_len(sg);
+		hw_node->generic.length = sg_dma_len(sg);
+	}
+
+	/* interrupt at end of last node */
+	hw_node->control |= FDMA_NODE_CTRL_INT_EON;
+
+	return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags);
+}
+
+static size_t st_fdma_desc_residue(struct st_fdma_chan *fchan,
+				   struct virt_dma_desc *vdesc,
+				   bool in_progress)
+{
+	struct st_fdma_desc *fdesc = fchan->fdesc;
+	size_t residue = 0;
+	dma_addr_t cur_addr = 0;
+	int i;
+
+	if (in_progress) {
+		cur_addr = fchan_read(fchan, FDMA_CH_CMD_OFST);
+		cur_addr &= FDMA_CH_CMD_DATA_MASK;
+	}
+
+	for (i = fchan->fdesc->n_nodes - 1 ; i >= 0; i--) {
+		if (cur_addr == fdesc->node[i].pdesc) {
+			residue += fnode_read(fchan, FDMA_CNTN_OFST);
+			break;
+		}
+		residue += fdesc->node[i].desc->nbytes;
+	}
+
+	return residue;
+}
+
+static enum dma_status st_fdma_tx_status(struct dma_chan *chan,
+					 dma_cookie_t cookie,
+					 struct dma_tx_state *txstate)
+{
+	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
+	struct virt_dma_desc *vd;
+	enum dma_status ret;
+	unsigned long flags;
+
+	ret = dma_cookie_status(chan, cookie, txstate);
+	if (ret == DMA_COMPLETE || !txstate)
+		return ret;
+
+	spin_lock_irqsave(&fchan->vchan.lock, flags);
+	vd = vchan_find_desc(&fchan->vchan, cookie);
+	if (fchan->fdesc && cookie == fchan->fdesc->vdesc.tx.cookie)
+		txstate->residue = st_fdma_desc_residue(fchan, vd, true);
+	else if (vd)
+		txstate->residue = st_fdma_desc_residue(fchan, vd, false);
+	else
+		txstate->residue = 0;
+
+	spin_unlock_irqrestore(&fchan->vchan.lock, flags);
+
+	return ret;
+}
+
+static void st_fdma_issue_pending(struct dma_chan *chan)
+{
+	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
+	unsigned long flags;
+
+	spin_lock_irqsave(&fchan->vchan.lock, flags);
+
+	if (vchan_issue_pending(&fchan->vchan) && !fchan->fdesc)
+		st_fdma_xfer_desc(fchan);
+
+	spin_unlock_irqrestore(&fchan->vchan.lock, flags);
+}
+
+static int st_fdma_pause(struct dma_chan *chan)
+{
+	unsigned long flags;
+	LIST_HEAD(head);
+	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
+	int ch_id = fchan->vchan.chan.chan_id;
+	unsigned long cmd = FDMA_CMD_PAUSE(ch_id);
+
+	dev_dbg(fchan->fdev->dev, "pause chan:%d\n", ch_id);
+
+	spin_lock_irqsave(&fchan->vchan.lock, flags);
+	if (fchan->fdesc)
+		fdma_write(fchan->fdev, cmd, FDMA_CMD_SET_OFST);
+	spin_unlock_irqrestore(&fchan->vchan.lock, flags);
+
+	return 0;
+}
+
+static int st_fdma_resume(struct dma_chan *chan)
+{
+	unsigned long flags;
+	unsigned long val;
+	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
+	int ch_id = fchan->vchan.chan.chan_id;
+
+	dev_dbg(fchan->fdev->dev, "resume chan:%d\n", ch_id);
+
+	spin_lock_irqsave(&fchan->vchan.lock, flags);
+	if (fchan->fdesc) {
+		val = fchan_read(fchan, FDMA_CH_CMD_OFST);
+		val &= FDMA_CH_CMD_DATA_MASK;
+		fchan_write(fchan, val, FDMA_CH_CMD_OFST);
+	}
+	spin_unlock_irqrestore(&fchan->vchan.lock, flags);
+
+	return 0;
+}
+
+static int st_fdma_terminate_all(struct dma_chan *chan)
+{
+	unsigned long flags;
+	LIST_HEAD(head);
+	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
+	int ch_id = fchan->vchan.chan.chan_id;
+	unsigned long cmd = FDMA_CMD_PAUSE(ch_id);
+
+	dev_dbg(fchan->fdev->dev, "terminate chan:%d\n", ch_id);
+
+	spin_lock_irqsave(&fchan->vchan.lock, flags);
+	fdma_write(fchan->fdev, cmd, FDMA_CMD_SET_OFST);
+	fchan->fdesc = NULL;
+	vchan_get_all_descriptors(&fchan->vchan, &head);
+	spin_unlock_irqrestore(&fchan->vchan.lock, flags);
+	vchan_dma_desc_free_list(&fchan->vchan, &head);
+
+	return 0;
+}
+
+static int st_fdma_slave_config(struct dma_chan *chan,
+				struct dma_slave_config *slave_cfg)
+{
+	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
+
+	memcpy(&fchan->scfg, slave_cfg, sizeof(fchan->scfg));
+	return 0;
+}
+
+static const struct st_fdma_driverdata fdma_mpe31_stih407_11 = {
+	.name = "STiH407",
+	.id = 0,
+};
+
+static const struct st_fdma_driverdata fdma_mpe31_stih407_12 = {
+	.name = "STiH407",
+	.id = 1,
+};
+
+static const struct st_fdma_driverdata fdma_mpe31_stih407_13 = {
+	.name = "STiH407",
+	.id = 2,
+};
+
+static const struct of_device_id st_fdma_match[] = {
+	{ .compatible = "st,stih407-fdma-mpe31-11"
+	  , .data = &fdma_mpe31_stih407_11 },
+	{ .compatible = "st,stih407-fdma-mpe31-12"
+	  , .data = &fdma_mpe31_stih407_12 },
+	{ .compatible = "st,stih407-fdma-mpe31-13"
+	  , .data = &fdma_mpe31_stih407_13 },
+	{},
+};
+MODULE_DEVICE_TABLE(of, st_fdma_match);
+
+static int st_fdma_parse_dt(struct platform_device *pdev,
+			const struct st_fdma_driverdata *drvdata,
+			struct st_fdma_dev *fdev)
+{
+	snprintf(fdev->fw_name, FW_NAME_SIZE, "fdma_%s_%d.elf",
+		drvdata->name, drvdata->id);
+
+	return of_property_read_u32(pdev->dev.of_node, "dma-channels",
+				    &fdev->nr_channels);
+}
+#define FDMA_DMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
+				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
+
+static void st_fdma_free(struct st_fdma_dev *fdev)
+{
+	struct st_fdma_chan *fchan;
+	int i;
+
+	for (i = 0; i < fdev->nr_channels; i++) {
+		fchan = &fdev->chans[i];
+		list_del(&fchan->vchan.chan.device_node);
+		tasklet_kill(&fchan->vchan.task);
+	}
+}
+
+static int st_fdma_probe(struct platform_device *pdev)
+{
+	struct st_fdma_dev *fdev;
+	const struct of_device_id *match;
+	struct device_node *np = pdev->dev.of_node;
+	const struct st_fdma_driverdata *drvdata;
+	int ret, i;
+
+	match = of_match_device((st_fdma_match), &pdev->dev);
+	if (!match || !match->data) {
+		dev_err(&pdev->dev, "No device match found\n");
+		return -ENODEV;
+	}
+
+	drvdata = match->data;
+
+	fdev = devm_kzalloc(&pdev->dev, sizeof(*fdev), GFP_KERNEL);
+	if (!fdev)
+		return -ENOMEM;
+
+	ret = st_fdma_parse_dt(pdev, drvdata, fdev);
+	if (ret) {
+		dev_err(&pdev->dev, "unable to find platform data\n");
+		goto err;
+	}
+
+	fdev->chans = devm_kcalloc(&pdev->dev, fdev->nr_channels,
+				   sizeof(struct st_fdma_chan), GFP_KERNEL);
+	if (!fdev->chans)
+		return -ENOMEM;
+
+	fdev->dev = &pdev->dev;
+	fdev->drvdata = drvdata;
+	platform_set_drvdata(pdev, fdev);
+
+	fdev->irq = platform_get_irq(pdev, 0);
+	if (fdev->irq < 0) {
+		dev_err(&pdev->dev, "Failed to get irq resource\n");
+		return -EINVAL;
+	}
+
+	ret = devm_request_irq(&pdev->dev, fdev->irq, st_fdma_irq_handler, 0,
+			       dev_name(&pdev->dev), fdev);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to request irq (%d)\n", ret);
+		goto err;
+	}
+
+	fdev->slim_rproc = st_slim_rproc_alloc(pdev, fdev->fw_name);
+	if (IS_ERR(fdev->slim_rproc)) {
+		ret = PTR_ERR(fdev->slim_rproc);
+		dev_err(&pdev->dev, "slim_rproc_alloc failed (%d)\n", ret);
+		goto err;
+	}
+
+	/* Initialise list of FDMA channels */
+	INIT_LIST_HEAD(&fdev->dma_device.channels);
+	for (i = 0; i < fdev->nr_channels; i++) {
+		struct st_fdma_chan *fchan = &fdev->chans[i];
+
+		fchan->fdev = fdev;
+		fchan->vchan.desc_free = st_fdma_free_desc;
+		vchan_init(&fchan->vchan, &fdev->dma_device);
+	}
+
+	/* Initialise the FDMA dreq (reserve 0 & 31 for FDMA use) */
+	fdev->dreq_mask = BIT(0) | BIT(31);
+
+	dma_cap_set(DMA_SLAVE, fdev->dma_device.cap_mask);
+	dma_cap_set(DMA_CYCLIC, fdev->dma_device.cap_mask);
+	dma_cap_set(DMA_MEMCPY, fdev->dma_device.cap_mask);
+
+	fdev->dma_device.dev = &pdev->dev;
+	fdev->dma_device.device_alloc_chan_resources = st_fdma_alloc_chan_res;
+	fdev->dma_device.device_free_chan_resources = st_fdma_free_chan_res;
+	fdev->dma_device.device_prep_dma_cyclic	= st_fdma_prep_dma_cyclic;
+	fdev->dma_device.device_prep_slave_sg = st_fdma_prep_slave_sg;
+	fdev->dma_device.device_prep_dma_memcpy = st_fdma_prep_dma_memcpy;
+	fdev->dma_device.device_tx_status = st_fdma_tx_status;
+	fdev->dma_device.device_issue_pending = st_fdma_issue_pending;
+	fdev->dma_device.device_terminate_all = st_fdma_terminate_all;
+	fdev->dma_device.device_config = st_fdma_slave_config;
+	fdev->dma_device.device_pause = st_fdma_pause;
+	fdev->dma_device.device_resume = st_fdma_resume;
+
+	fdev->dma_device.src_addr_widths = FDMA_DMA_BUSWIDTHS;
+	fdev->dma_device.dst_addr_widths = FDMA_DMA_BUSWIDTHS;
+	fdev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+	fdev->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
+
+	ret = dma_async_device_register(&fdev->dma_device);
+	if (ret) {
+		dev_err(&pdev->dev,
+			"Failed to register DMA device (%d)\n", ret);
+		goto err_rproc;
+	}
+
+	ret = of_dma_controller_register(np, st_fdma_of_xlate, fdev);
+	if (ret) {
+		dev_err(&pdev->dev,
+			"Failed to register controller (%d)\n", ret);
+		goto err_dma_dev;
+	}
+
+	dev_info(&pdev->dev, "ST FDMA engine driver, irq:%d\n", fdev->irq);
+
+	return 0;
+
+err_dma_dev:
+	dma_async_device_unregister(&fdev->dma_device);
+err_rproc:
+	st_fdma_free(fdev);
+	st_slim_rproc_put(fdev->slim_rproc);
+err:
+	return ret;
+}
+
+static int st_fdma_remove(struct platform_device *pdev)
+{
+	struct st_fdma_dev *fdev = platform_get_drvdata(pdev);
+
+	devm_free_irq(&pdev->dev, fdev->irq, fdev);
+	st_slim_rproc_put(fdev->slim_rproc);
+	of_dma_controller_free(pdev->dev.of_node);
+	dma_async_device_unregister(&fdev->dma_device);
+
+	return 0;
+}
+
+static struct platform_driver st_fdma_platform_driver = {
+	.driver = {
+		.name = DRIVER_NAME,
+		.of_match_table = st_fdma_match,
+	},
+	.probe = st_fdma_probe,
+	.remove = st_fdma_remove,
+};
+module_platform_driver(st_fdma_platform_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("STMicroelectronics FDMA engine driver");
+MODULE_AUTHOR("Ludovic.barre <Ludovic.barre@st.com>");
+MODULE_AUTHOR("Peter Griffin <peter.griffin@linaro.org>");
+MODULE_ALIAS("platform: " DRIVER_NAME);
diff --git a/drivers/dma/st_fdma.h b/drivers/dma/st_fdma.h
new file mode 100644
index 000000000000..c58e00d4ab37
--- /dev/null
+++ b/drivers/dma/st_fdma.h
@@ -0,0 +1,249 @@
+/*
+ * DMA driver header for STMicroelectronics STi FDMA controller
+ *
+ * Copyright (C) 2014 STMicroelectronics
+ *
+ * Author: Ludovic Barre <Ludovic.barre@st.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+#ifndef __DMA_ST_FDMA_H
+#define __DMA_ST_FDMA_H
+
+#include <linux/dmaengine.h>
+#include <linux/dmapool.h>
+#include <linux/io.h>
+#include <linux/remoteproc/st_slim_rproc.h>
+#include "virt-dma.h"
+
+#define ST_FDMA_NR_DREQS 32
+#define FW_NAME_SIZE 30
+#define DRIVER_NAME "st-fdma"
+
+/**
+ * struct st_fdma_generic_node - Free running/paced generic node
+ *
+ * @length: Length in bytes of a line in a 2D mem to mem
+ * @sstride: Stride, in bytes, between source lines in a 2D data move
+ * @dstride: Stride, in bytes, between destination lines in a 2D data move
+ */
+struct st_fdma_generic_node {
+	u32 length;
+	u32 sstride;
+	u32 dstride;
+};
+
+/**
+ * struct st_fdma_hw_node - Node structure used by fdma hw
+ *
+ * @next: Pointer to next node
+ * @control: Transfer Control Parameters
+ * @nbytes: Number of Bytes to read
+ * @saddr: Source address
+ * @daddr: Destination address
+ *
+ * @generic: generic node for free running/paced transfert type
+ * 2 others transfert type are possible, but not yet implemented
+ *
+ * The NODE structures must be aligned to a 32 byte boundary
+ */
+struct st_fdma_hw_node {
+	u32 next;
+	u32 control;
+	u32 nbytes;
+	u32 saddr;
+	u32 daddr;
+	union {
+		struct st_fdma_generic_node generic;
+	};
+} __aligned(32);
+
+/*
+ * node control parameters
+ */
+#define FDMA_NODE_CTRL_REQ_MAP_MASK	GENMASK(4, 0)
+#define FDMA_NODE_CTRL_REQ_MAP_FREE_RUN	0x0
+#define FDMA_NODE_CTRL_REQ_MAP_DREQ(n)	((n)&FDMA_NODE_CTRL_REQ_MAP_MASK)
+#define FDMA_NODE_CTRL_REQ_MAP_EXT		FDMA_NODE_CTRL_REQ_MAP_MASK
+#define FDMA_NODE_CTRL_SRC_MASK		GENMASK(6, 5)
+#define FDMA_NODE_CTRL_SRC_STATIC	BIT(5)
+#define FDMA_NODE_CTRL_SRC_INCR		BIT(6)
+#define FDMA_NODE_CTRL_DST_MASK		GENMASK(8, 7)
+#define FDMA_NODE_CTRL_DST_STATIC	BIT(7)
+#define FDMA_NODE_CTRL_DST_INCR		BIT(8)
+#define FDMA_NODE_CTRL_SECURE		BIT(15)
+#define FDMA_NODE_CTRL_PAUSE_EON	BIT(30)
+#define FDMA_NODE_CTRL_INT_EON		BIT(31)
+
+/**
+ * struct st_fdma_sw_node - descriptor structure for link list
+ *
+ * @pdesc: Physical address of desc
+ * @node: link used for putting this into a channel queue
+ */
+struct st_fdma_sw_node {
+	dma_addr_t pdesc;
+	struct st_fdma_hw_node *desc;
+};
+
+#define NAME_SZ 10
+
+struct st_fdma_driverdata {
+	u32 id;
+	char name[NAME_SZ];
+};
+
+struct st_fdma_desc {
+	struct virt_dma_desc vdesc;
+	struct st_fdma_chan *fchan;
+	bool iscyclic;
+	unsigned int n_nodes;
+	struct st_fdma_sw_node node[];
+};
+
+enum st_fdma_type {
+	ST_FDMA_TYPE_FREE_RUN,
+	ST_FDMA_TYPE_PACED,
+};
+
+struct st_fdma_cfg {
+	struct device_node *of_node;
+	enum st_fdma_type type;
+	dma_addr_t dev_addr;
+	enum dma_transfer_direction dir;
+	int req_line; /* request line */
+	long req_ctrl; /* Request control */
+};
+
+struct st_fdma_chan {
+	struct st_fdma_dev *fdev;
+	struct dma_pool *node_pool;
+	struct dma_slave_config scfg;
+	struct st_fdma_cfg cfg;
+
+	int dreq_line;
+
+	struct virt_dma_chan vchan;
+	struct st_fdma_desc *fdesc;
+	enum dma_status	status;
+};
+
+struct st_fdma_dev {
+	struct device *dev;
+	const struct st_fdma_driverdata *drvdata;
+	struct dma_device dma_device;
+
+	struct st_slim_rproc *slim_rproc;
+
+	int irq;
+
+	struct st_fdma_chan *chans;
+
+	spinlock_t dreq_lock;
+	unsigned long dreq_mask;
+
+	u32 nr_channels;
+	char fw_name[FW_NAME_SIZE];
+};
+
+/* Peripheral Registers*/
+
+#define FDMA_CMD_STA_OFST	0xFC0
+#define FDMA_CMD_SET_OFST	0xFC4
+#define FDMA_CMD_CLR_OFST	0xFC8
+#define FDMA_CMD_MASK_OFST	0xFCC
+#define FDMA_CMD_START(ch)		(0x1 << (ch << 1))
+#define FDMA_CMD_PAUSE(ch)		(0x2 << (ch << 1))
+#define FDMA_CMD_FLUSH(ch)		(0x3 << (ch << 1))
+
+#define FDMA_INT_STA_OFST	0xFD0
+#define FDMA_INT_STA_CH			0x1
+#define FDMA_INT_STA_ERR		0x2
+
+#define FDMA_INT_SET_OFST	0xFD4
+#define FDMA_INT_CLR_OFST	0xFD8
+#define FDMA_INT_MASK_OFST	0xFDC
+
+#define fdma_read(fdev, name) \
+	readl((fdev)->slim_rproc->peri + name)
+
+#define fdma_write(fdev, val, name) \
+	writel((val), (fdev)->slim_rproc->peri + name)
+
+/* fchan interface (dmem) */
+#define FDMA_CH_CMD_OFST	0x200
+#define FDMA_CH_CMD_STA_MASK		GENMASK(1, 0)
+#define FDMA_CH_CMD_STA_IDLE		(0x0)
+#define FDMA_CH_CMD_STA_START		(0x1)
+#define FDMA_CH_CMD_STA_RUNNING		(0x2)
+#define FDMA_CH_CMD_STA_PAUSED		(0x3)
+#define FDMA_CH_CMD_ERR_MASK		GENMASK(4, 2)
+#define FDMA_CH_CMD_ERR_INT		(0x0 << 2)
+#define FDMA_CH_CMD_ERR_NAND		(0x1 << 2)
+#define FDMA_CH_CMD_ERR_MCHI		(0x2 << 2)
+#define FDMA_CH_CMD_DATA_MASK		GENMASK(31, 5)
+#define fchan_read(fchan, name) \
+	readl((fchan)->fdev->slim_rproc->mem[ST_SLIM_DMEM].cpu_addr \
+			+ (fchan)->vchan.chan.chan_id * 0x4 \
+			+ name)
+
+#define fchan_write(fchan, val, name) \
+	writel((val), (fchan)->fdev->slim_rproc->mem[ST_SLIM_DMEM].cpu_addr \
+			+ (fchan)->vchan.chan.chan_id * 0x4 \
+			+ name)
+
+/* req interface */
+#define FDMA_REQ_CTRL_OFST	0x240
+#define dreq_write(fchan, val, name) \
+	writel((val), (fchan)->fdev->slim_rproc->mem[ST_SLIM_DMEM].cpu_addr \
+			+ fchan->dreq_line * 0x04 \
+			+ name)
+/* node interface */
+#define FDMA_NODE_SZ 128
+#define FDMA_PTRN_OFST		0x800
+#define FDMA_CNTN_OFST		0x808
+#define FDMA_SADDRN_OFST	0x80c
+#define FDMA_DADDRN_OFST	0x810
+#define fnode_read(fchan, name) \
+	readl((fchan)->fdev->slim_rproc->mem[ST_SLIM_DMEM].cpu_addr \
+			+ (fchan)->vchan.chan.chan_id * FDMA_NODE_SZ \
+			+ name)
+
+#define fnode_write(fchan, val, name) \
+	writel((val), (fchan)->fdev->slim_rproc->mem[ST_SLIM_DMEM].cpu_addr \
+			+ (fchan)->vchan.chan.chan_id * FDMA_NODE_SZ \
+			+ name)
+
+/*
+ * request control bits
+ */
+#define FDMA_REQ_CTRL_NUM_OPS_MASK	GENMASK(31, 24)
+#define FDMA_REQ_CTRL_NUM_OPS(n)	(FDMA_REQ_CTRL_NUM_OPS_MASK & \
+					((n) << 24))
+#define FDMA_REQ_CTRL_INITIATOR_MASK	BIT(22)
+#define FDMA_REQ_CTRL_INIT0		(0x0 << 22)
+#define FDMA_REQ_CTRL_INIT1		(0x1 << 22)
+#define FDMA_REQ_CTRL_INC_ADDR_ON	BIT(21)
+#define FDMA_REQ_CTRL_DATA_SWAP_ON	BIT(17)
+#define FDMA_REQ_CTRL_WNR		BIT(14)
+#define FDMA_REQ_CTRL_OPCODE_MASK	GENMASK(7, 4)
+#define FDMA_REQ_CTRL_OPCODE_LD_ST1	(0x0 << 4)
+#define FDMA_REQ_CTRL_OPCODE_LD_ST2	(0x1 << 4)
+#define FDMA_REQ_CTRL_OPCODE_LD_ST4	(0x2 << 4)
+#define FDMA_REQ_CTRL_OPCODE_LD_ST8	(0x3 << 4)
+#define FDMA_REQ_CTRL_OPCODE_LD_ST16	(0x4 << 4)
+#define FDMA_REQ_CTRL_OPCODE_LD_ST32	(0x5 << 4)
+#define FDMA_REQ_CTRL_OPCODE_LD_ST64	(0x6 << 4)
+#define FDMA_REQ_CTRL_HOLDOFF_MASK	GENMASK(2, 0)
+#define FDMA_REQ_CTRL_HOLDOFF(n)	((n) & FDMA_REQ_CTRL_HOLDOFF_MASK)
+
+/* bits used by client to configure request control */
+#define FDMA_REQ_CTRL_CFG_MASK (FDMA_REQ_CTRL_HOLDOFF_MASK | \
+				FDMA_REQ_CTRL_DATA_SWAP_ON | \
+				FDMA_REQ_CTRL_INC_ADDR_ON | \
+				FDMA_REQ_CTRL_INITIATOR_MASK)
+
+#endif	/* __DMA_ST_FDMA_H */
diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c
index 307547f4848d..3688d0873a3e 100644
--- a/drivers/dma/stm32-dma.c
+++ b/drivers/dma/stm32-dma.c
@@ -527,13 +527,12 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid)
 {
 	struct stm32_dma_chan *chan = devid;
 	struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);
-	u32 status, scr, sfcr;
+	u32 status, scr;
 
 	spin_lock(&chan->vchan.lock);
 
 	status = stm32_dma_irq_status(chan);
 	scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id));
-	sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id));
 
 	if ((status & STM32_DMA_TCI) && (scr & STM32_DMA_SCR_TCIE)) {
 		stm32_dma_irq_clear(chan, STM32_DMA_TCI);
@@ -574,15 +573,12 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
 	int src_bus_width, dst_bus_width;
 	int src_burst_size, dst_burst_size;
 	u32 src_maxburst, dst_maxburst;
-	dma_addr_t src_addr, dst_addr;
 	u32 dma_scr = 0;
 
 	src_addr_width = chan->dma_sconfig.src_addr_width;
 	dst_addr_width = chan->dma_sconfig.dst_addr_width;
 	src_maxburst = chan->dma_sconfig.src_maxburst;
 	dst_maxburst = chan->dma_sconfig.dst_maxburst;
-	src_addr = chan->dma_sconfig.src_addr;
-	dst_addr = chan->dma_sconfig.dst_addr;
 
 	switch (direction) {
 	case DMA_MEM_TO_DEV:
diff --git a/drivers/dma/zx296702_dma.c b/drivers/dma/zx296702_dma.c
index 245d759d5ffc..380276d078b2 100644
--- a/drivers/dma/zx296702_dma.c
+++ b/drivers/dma/zx296702_dma.c
@@ -435,13 +435,12 @@ static struct zx_dma_desc_sw *zx_alloc_desc_resource(int num,
 	if (!ds)
 		return NULL;
 
-	ds->desc_hw = dma_pool_alloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli);
+	ds->desc_hw = dma_pool_zalloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli);
 	if (!ds->desc_hw) {
 		dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc);
 		kfree(ds);
 		return NULL;
 	}
-	memset(ds->desc_hw, 0, sizeof(struct zx_desc_hw) * num);
 	ds->desc_num = num;
 	return ds;
 }
diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c
index 58d3e2b39b5b..c5a5b91f37f0 100644
--- a/drivers/edac/altera_edac.c
+++ b/drivers/edac/altera_edac.c
@@ -35,7 +35,6 @@
 #include <linux/uaccess.h>
 
 #include "altera_edac.h"
-#include "edac_core.h"
 #include "edac_module.h"
 
 #define EDAC_MOD_STR		"altera_edac"
@@ -153,13 +152,17 @@ static ssize_t altr_sdr_mc_err_inject_write(struct file *file,
 	if (count == 3) {
 		edac_printk(KERN_ALERT, EDAC_MC,
 			    "Inject Double bit error\n");
+		local_irq_disable();
 		regmap_write(drvdata->mc_vbase, priv->ce_ue_trgr_offset,
 			     (read_reg | priv->ue_set_mask));
+		local_irq_enable();
 	} else {
 		edac_printk(KERN_ALERT, EDAC_MC,
 			    "Inject Single bit error\n");
+		local_irq_disable();
 		regmap_write(drvdata->mc_vbase,	priv->ce_ue_trgr_offset,
 			     (read_reg | priv->ce_set_mask));
+		local_irq_enable();
 	}
 
 	ptemp[0] = 0x5A5A5A5A;
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c
index ee181c53626f..260251177830 100644
--- a/drivers/edac/amd64_edac.c
+++ b/drivers/edac/amd64_edac.c
@@ -164,8 +164,23 @@ static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
  * other archs, we might not have access to the caches directly.
  */
 
+static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval)
+{
+	/*
+	 * Fam17h supports scrub values between 0x5 and 0x14. Also, the values
+	 * are shifted down by 0x5, so scrubval 0x5 is written to the register
+	 * as 0x0, scrubval 0x6 as 0x1, etc.
+	 */
+	if (scrubval >= 0x5 && scrubval <= 0x14) {
+		scrubval -= 0x5;
+		pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF);
+		pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1);
+	} else {
+		pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1);
+	}
+}
 /*
- * scan the scrub rate mapping table for a close or matching bandwidth value to
+ * Scan the scrub rate mapping table for a close or matching bandwidth value to
  * issue. If requested is too big, then use last maximum value found.
  */
 static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
@@ -196,7 +211,9 @@ static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
 
 	scrubval = scrubrates[i].scrubval;
 
-	if (pvt->fam == 0x15 && pvt->model == 0x60) {
+	if (pvt->fam == 0x17) {
+		__f17h_set_scrubval(pvt, scrubval);
+	} else if (pvt->fam == 0x15 && pvt->model == 0x60) {
 		f15h_select_dct(pvt, 0);
 		pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
 		f15h_select_dct(pvt, 1);
@@ -233,18 +250,34 @@ static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
 static int get_scrub_rate(struct mem_ctl_info *mci)
 {
 	struct amd64_pvt *pvt = mci->pvt_info;
-	u32 scrubval = 0;
 	int i, retval = -EINVAL;
+	u32 scrubval = 0;
 
-	if (pvt->fam == 0x15) {
+	switch (pvt->fam) {
+	case 0x15:
 		/* Erratum #505 */
 		if (pvt->model < 0x10)
 			f15h_select_dct(pvt, 0);
 
 		if (pvt->model == 0x60)
 			amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
-	} else
+		break;
+
+	case 0x17:
+		amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval);
+		if (scrubval & BIT(0)) {
+			amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval);
+			scrubval &= 0xF;
+			scrubval += 0x5;
+		} else {
+			scrubval = 0;
+		}
+		break;
+
+	default:
 		amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
+		break;
+	}
 
 	scrubval = scrubval & 0x001F;
 
@@ -682,15 +715,33 @@ static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
  */
 static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
 {
-	u8 bit;
 	unsigned long edac_cap = EDAC_FLAG_NONE;
+	u8 bit;
 
-	bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
-		? 19
-		: 17;
+	if (pvt->umc) {
+		u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
 
-	if (pvt->dclr0 & BIT(bit))
-		edac_cap = EDAC_FLAG_SECDED;
+		for (i = 0; i < NUM_UMCS; i++) {
+			if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
+				continue;
+
+			umc_en_mask |= BIT(i);
+
+			/* UMC Configuration bit 12 (DimmEccEn) */
+			if (pvt->umc[i].umc_cfg & BIT(12))
+				dimm_ecc_en_mask |= BIT(i);
+		}
+
+		if (umc_en_mask == dimm_ecc_en_mask)
+			edac_cap = EDAC_FLAG_SECDED;
+	} else {
+		bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
+			? 19
+			: 17;
+
+		if (pvt->dclr0 & BIT(bit))
+			edac_cap = EDAC_FLAG_SECDED;
+	}
 
 	return edac_cap;
 }
@@ -729,8 +780,75 @@ static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
 		 (dclr & BIT(15)) ?  "yes" : "no");
 }
 
+static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
+{
+	u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
+	int dimm, size0, size1;
+
+	edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
+
+	for (dimm = 0; dimm < 4; dimm++) {
+		size0 = 0;
+
+		if (dcsb[dimm*2] & DCSB_CS_ENABLE)
+			size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, dimm);
+
+		size1 = 0;
+		if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
+			size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, dimm);
+
+		amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
+				dimm * 2,     size0,
+				dimm * 2 + 1, size1);
+	}
+}
+
+static void __dump_misc_regs_df(struct amd64_pvt *pvt)
+{
+	struct amd64_umc *umc;
+	u32 i, tmp, umc_base;
+
+	for (i = 0; i < NUM_UMCS; i++) {
+		umc_base = get_umc_base(i);
+		umc = &pvt->umc[i];
+
+		edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
+		edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
+		edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
+		edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
+
+		amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
+		edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
+
+		amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
+		edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
+		edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
+
+		edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
+				i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
+				    (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
+		edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
+				i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
+		edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
+				i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
+		edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
+				i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
+
+		if (pvt->dram_type == MEM_LRDDR4) {
+			amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp);
+			edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
+					i, 1 << ((tmp >> 4) & 0x3));
+		}
+
+		debug_display_dimm_sizes_df(pvt, i);
+	}
+
+	edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n",
+		 pvt->dhar, dhar_base(pvt));
+}
+
 /* Display and decode various NB registers for debug purposes. */
-static void dump_misc_regs(struct amd64_pvt *pvt)
+static void __dump_misc_regs(struct amd64_pvt *pvt)
 {
 	edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
 
@@ -750,8 +868,6 @@ static void dump_misc_regs(struct amd64_pvt *pvt)
 		 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
 				   : f10_dhar_offset(pvt));
 
-	edac_dbg(1, "  DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
-
 	debug_display_dimm_sizes(pvt, 0);
 
 	/* everything below this point is Fam10h and above */
@@ -760,13 +876,25 @@ static void dump_misc_regs(struct amd64_pvt *pvt)
 
 	debug_display_dimm_sizes(pvt, 1);
 
-	amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
-
 	/* Only if NOT ganged does dclr1 have valid info */
 	if (!dct_ganging_enabled(pvt))
 		debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
 }
 
+/* Display and decode various NB registers for debug purposes. */
+static void dump_misc_regs(struct amd64_pvt *pvt)
+{
+	if (pvt->umc)
+		__dump_misc_regs_df(pvt);
+	else
+		__dump_misc_regs(pvt);
+
+	edac_dbg(1, "  DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
+
+	amd64_info("using %s syndromes.\n",
+			((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
+}
+
 /*
  * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
  */
@@ -789,46 +917,78 @@ static void prep_chip_selects(struct amd64_pvt *pvt)
  */
 static void read_dct_base_mask(struct amd64_pvt *pvt)
 {
-	int cs;
+	int base_reg0, base_reg1, mask_reg0, mask_reg1, cs;
 
 	prep_chip_selects(pvt);
 
+	if (pvt->umc) {
+		base_reg0 = get_umc_base(0) + UMCCH_BASE_ADDR;
+		base_reg1 = get_umc_base(1) + UMCCH_BASE_ADDR;
+		mask_reg0 = get_umc_base(0) + UMCCH_ADDR_MASK;
+		mask_reg1 = get_umc_base(1) + UMCCH_ADDR_MASK;
+	} else {
+		base_reg0 = DCSB0;
+		base_reg1 = DCSB1;
+		mask_reg0 = DCSM0;
+		mask_reg1 = DCSM1;
+	}
+
 	for_each_chip_select(cs, 0, pvt) {
-		int reg0   = DCSB0 + (cs * 4);
-		int reg1   = DCSB1 + (cs * 4);
+		int reg0   = base_reg0 + (cs * 4);
+		int reg1   = base_reg1 + (cs * 4);
 		u32 *base0 = &pvt->csels[0].csbases[cs];
 		u32 *base1 = &pvt->csels[1].csbases[cs];
 
-		if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
-			edac_dbg(0, "  DCSB0[%d]=0x%08x reg: F2x%x\n",
-				 cs, *base0, reg0);
+		if (pvt->umc) {
+			if (!amd_smn_read(pvt->mc_node_id, reg0, base0))
+				edac_dbg(0, "  DCSB0[%d]=0x%08x reg: 0x%x\n",
+					 cs, *base0, reg0);
 
-		if (pvt->fam == 0xf)
-			continue;
+			if (!amd_smn_read(pvt->mc_node_id, reg1, base1))
+				edac_dbg(0, "  DCSB1[%d]=0x%08x reg: 0x%x\n",
+					 cs, *base1, reg1);
+		} else {
+			if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
+				edac_dbg(0, "  DCSB0[%d]=0x%08x reg: F2x%x\n",
+					 cs, *base0, reg0);
+
+			if (pvt->fam == 0xf)
+				continue;
 
-		if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
-			edac_dbg(0, "  DCSB1[%d]=0x%08x reg: F2x%x\n",
-				 cs, *base1, (pvt->fam == 0x10) ? reg1
+			if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
+				edac_dbg(0, "  DCSB1[%d]=0x%08x reg: F2x%x\n",
+					 cs, *base1, (pvt->fam == 0x10) ? reg1
 								: reg0);
+		}
 	}
 
 	for_each_chip_select_mask(cs, 0, pvt) {
-		int reg0   = DCSM0 + (cs * 4);
-		int reg1   = DCSM1 + (cs * 4);
+		int reg0   = mask_reg0 + (cs * 4);
+		int reg1   = mask_reg1 + (cs * 4);
 		u32 *mask0 = &pvt->csels[0].csmasks[cs];
 		u32 *mask1 = &pvt->csels[1].csmasks[cs];
 
-		if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
-			edac_dbg(0, "    DCSM0[%d]=0x%08x reg: F2x%x\n",
-				 cs, *mask0, reg0);
+		if (pvt->umc) {
+			if (!amd_smn_read(pvt->mc_node_id, reg0, mask0))
+				edac_dbg(0, "    DCSM0[%d]=0x%08x reg: 0x%x\n",
+					 cs, *mask0, reg0);
 
-		if (pvt->fam == 0xf)
-			continue;
+			if (!amd_smn_read(pvt->mc_node_id, reg1, mask1))
+				edac_dbg(0, "    DCSM1[%d]=0x%08x reg: 0x%x\n",
+					 cs, *mask1, reg1);
+		} else {
+			if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
+				edac_dbg(0, "    DCSM0[%d]=0x%08x reg: F2x%x\n",
+					 cs, *mask0, reg0);
+
+			if (pvt->fam == 0xf)
+				continue;
 
-		if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
-			edac_dbg(0, "    DCSM1[%d]=0x%08x reg: F2x%x\n",
-				 cs, *mask1, (pvt->fam == 0x10) ? reg1
+			if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
+				edac_dbg(0, "    DCSM1[%d]=0x%08x reg: F2x%x\n",
+					 cs, *mask1, (pvt->fam == 0x10) ? reg1
 								: reg0);
+		}
 	}
 }
 
@@ -881,6 +1041,15 @@ static void determine_memory_type(struct amd64_pvt *pvt)
 	case 0x16:
 		goto ddr3;
 
+	case 0x17:
+		if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5))
+			pvt->dram_type = MEM_LRDDR4;
+		else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4))
+			pvt->dram_type = MEM_RDDR4;
+		else
+			pvt->dram_type = MEM_DDR4;
+		return;
+
 	default:
 		WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
 		pvt->dram_type = MEM_EMPTY;
@@ -1210,6 +1379,19 @@ static int f1x_early_channel_count(struct amd64_pvt *pvt)
 	return channels;
 }
 
+static int f17_early_channel_count(struct amd64_pvt *pvt)
+{
+	int i, channels = 0;
+
+	/* SDP Control bit 31 (SdpInit) is clear for unused UMC channels */
+	for (i = 0; i < NUM_UMCS; i++)
+		channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT);
+
+	amd64_info("MCT channel count: %d\n", channels);
+
+	return channels;
+}
+
 static int ddr3_cs_size(unsigned i, bool dct_width)
 {
 	unsigned shift = 0;
@@ -1337,6 +1519,23 @@ static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
 		return ddr3_cs_size(cs_mode, false);
 }
 
+static int f17_base_addr_to_cs_size(struct amd64_pvt *pvt, u8 umc,
+				    unsigned int cs_mode, int csrow_nr)
+{
+	u32 base_addr = pvt->csels[umc].csbases[csrow_nr];
+
+	/*  Each mask is used for every two base addresses. */
+	u32 addr_mask = pvt->csels[umc].csmasks[csrow_nr >> 1];
+
+	/*  Register [31:1] = Address [39:9]. Size is in kBs here. */
+	u32 size = ((addr_mask >> 1) - (base_addr >> 1) + 1) >> 1;
+
+	edac_dbg(1, "BaseAddr: 0x%x, AddrMask: 0x%x\n", base_addr, addr_mask);
+
+	/* Return size in MBs. */
+	return size >> 10;
+}
+
 static void read_dram_ctl_register(struct amd64_pvt *pvt)
 {
 
@@ -1897,8 +2096,9 @@ static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
 
 		size0 = 0;
 		if (dcsb[dimm*2] & DCSB_CS_ENABLE)
-			/* For f15m60h, need multiplier for LRDIMM cs_size
-			 * calculation. We pass 'dimm' value to the dbam_to_cs
+			/*
+			 * For F15m60h, we need multiplier for LRDIMM cs_size
+			 * calculation. We pass dimm value to the dbam_to_cs
 			 * mapper so we can find the multiplier from the
 			 * corresponding DCSM.
 			 */
@@ -1989,6 +2189,15 @@ static struct amd64_family_type family_types[] = {
 			.dbam_to_cs		= f16_dbam_to_chip_select,
 		}
 	},
+	[F17_CPUS] = {
+		.ctl_name = "F17h",
+		.f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0,
+		.f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6,
+		.ops = {
+			.early_channel_count	= f17_early_channel_count,
+			.dbam_to_cs		= f17_base_addr_to_cs_size,
+		}
+	},
 };
 
 /*
@@ -2155,7 +2364,7 @@ static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
 	return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
 }
 
-static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err,
+static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
 			    u8 ecc_type)
 {
 	enum hw_event_mc_err_type err_type;
@@ -2165,6 +2374,8 @@ static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err,
 		err_type = HW_EVENT_ERR_CORRECTED;
 	else if (ecc_type == 1)
 		err_type = HW_EVENT_ERR_UNCORRECTED;
+	else if (ecc_type == 3)
+		err_type = HW_EVENT_ERR_DEFERRED;
 	else {
 		WARN(1, "Something is rotten in the state of Denmark.\n");
 		return;
@@ -2181,7 +2392,13 @@ static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err,
 		string = "Failed to map error addr to a csrow";
 		break;
 	case ERR_CHANNEL:
-		string = "unknown syndrome - possible error reporting race";
+		string = "Unknown syndrome - possible error reporting race";
+		break;
+	case ERR_SYND:
+		string = "MCA_SYND not valid - unknown syndrome and csrow";
+		break;
+	case ERR_NORM_ADDR:
+		string = "Cannot decode normalized address";
 		break;
 	default:
 		string = "WTF error";
@@ -2227,36 +2444,127 @@ static inline void decode_bus_error(int node_id, struct mce *m)
 
 	pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
 
-	__log_bus_error(mci, &err, ecc_type);
+	__log_ecc_error(mci, &err, ecc_type);
+}
+
+/*
+ * To find the UMC channel represented by this bank we need to match on its
+ * instance_id. The instance_id of a bank is held in the lower 32 bits of its
+ * IPID.
+ */
+static int find_umc_channel(struct amd64_pvt *pvt, struct mce *m)
+{
+	u32 umc_instance_id[] = {0x50f00, 0x150f00};
+	u32 instance_id = m->ipid & GENMASK(31, 0);
+	int i, channel = -1;
+
+	for (i = 0; i < ARRAY_SIZE(umc_instance_id); i++)
+		if (umc_instance_id[i] == instance_id)
+			channel = i;
+
+	return channel;
+}
+
+static void decode_umc_error(int node_id, struct mce *m)
+{
+	u8 ecc_type = (m->status >> 45) & 0x3;
+	struct mem_ctl_info *mci;
+	struct amd64_pvt *pvt;
+	struct err_info err;
+	u64 sys_addr;
+
+	mci = edac_mc_find(node_id);
+	if (!mci)
+		return;
+
+	pvt = mci->pvt_info;
+
+	memset(&err, 0, sizeof(err));
+
+	if (m->status & MCI_STATUS_DEFERRED)
+		ecc_type = 3;
+
+	err.channel = find_umc_channel(pvt, m);
+	if (err.channel < 0) {
+		err.err_code = ERR_CHANNEL;
+		goto log_error;
+	}
+
+	if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
+		err.err_code = ERR_NORM_ADDR;
+		goto log_error;
+	}
+
+	error_address_to_page_and_offset(sys_addr, &err);
+
+	if (!(m->status & MCI_STATUS_SYNDV)) {
+		err.err_code = ERR_SYND;
+		goto log_error;
+	}
+
+	if (ecc_type == 2) {
+		u8 length = (m->synd >> 18) & 0x3f;
+
+		if (length)
+			err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0);
+		else
+			err.err_code = ERR_CHANNEL;
+	}
+
+	err.csrow = m->synd & 0x7;
+
+log_error:
+	__log_ecc_error(mci, &err, ecc_type);
 }
 
 /*
  * Use pvt->F3 which contains the F3 CPU PCI device to get the related
  * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error.
+ * Reserve F0 and F6 on systems with a UMC.
  */
-static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f2_id)
-{
+static int
+reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
+{
+	if (pvt->umc) {
+		pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
+		if (!pvt->F0) {
+			amd64_err("F0 not found, device 0x%x (broken BIOS?)\n", pci_id1);
+			return -ENODEV;
+		}
+
+		pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
+		if (!pvt->F6) {
+			pci_dev_put(pvt->F0);
+			pvt->F0 = NULL;
+
+			amd64_err("F6 not found: device 0x%x (broken BIOS?)\n", pci_id2);
+			return -ENODEV;
+		}
+
+		edac_dbg(1, "F0: %s\n", pci_name(pvt->F0));
+		edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
+		edac_dbg(1, "F6: %s\n", pci_name(pvt->F6));
+
+		return 0;
+	}
+
 	/* Reserve the ADDRESS MAP Device */
-	pvt->F1 = pci_get_related_function(pvt->F3->vendor, f1_id, pvt->F3);
+	pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
 	if (!pvt->F1) {
-		amd64_err("error address map device not found: "
-			  "vendor %x device 0x%x (broken BIOS?)\n",
-			  PCI_VENDOR_ID_AMD, f1_id);
+		amd64_err("F1 not found: device 0x%x (broken BIOS?)\n", pci_id1);
 		return -ENODEV;
 	}
 
 	/* Reserve the DCT Device */
-	pvt->F2 = pci_get_related_function(pvt->F3->vendor, f2_id, pvt->F3);
+	pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
 	if (!pvt->F2) {
 		pci_dev_put(pvt->F1);
 		pvt->F1 = NULL;
 
-		amd64_err("error F2 device not found: "
-			  "vendor %x device 0x%x (broken BIOS?)\n",
-			  PCI_VENDOR_ID_AMD, f2_id);
-
+		amd64_err("F2 not found: device 0x%x (broken BIOS?)\n", pci_id2);
 		return -ENODEV;
 	}
+
 	edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
 	edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
 	edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
@@ -2266,8 +2574,69 @@ static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f2_id)
 
 static void free_mc_sibling_devs(struct amd64_pvt *pvt)
 {
-	pci_dev_put(pvt->F1);
-	pci_dev_put(pvt->F2);
+	if (pvt->umc) {
+		pci_dev_put(pvt->F0);
+		pci_dev_put(pvt->F6);
+	} else {
+		pci_dev_put(pvt->F1);
+		pci_dev_put(pvt->F2);
+	}
+}
+
+static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
+{
+	pvt->ecc_sym_sz = 4;
+
+	if (pvt->umc) {
+		u8 i;
+
+		for (i = 0; i < NUM_UMCS; i++) {
+			/* Check enabled channels only: */
+			if ((pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) &&
+			    (pvt->umc[i].ecc_ctrl & BIT(7))) {
+				pvt->ecc_sym_sz = 8;
+				break;
+			}
+		}
+
+		return;
+	}
+
+	if (pvt->fam >= 0x10) {
+		u32 tmp;
+
+		amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
+		/* F16h has only DCT0, so no need to read dbam1. */
+		if (pvt->fam != 0x16)
+			amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
+
+		/* F10h, revD and later can do x8 ECC too. */
+		if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
+			pvt->ecc_sym_sz = 8;
+	}
+}
+
+/*
+ * Retrieve the hardware registers of the memory controller.
+ */
+static void __read_mc_regs_df(struct amd64_pvt *pvt)
+{
+	u8 nid = pvt->mc_node_id;
+	struct amd64_umc *umc;
+	u32 i, umc_base;
+
+	/* Read registers from each UMC */
+	for (i = 0; i < NUM_UMCS; i++) {
+
+		umc_base = get_umc_base(i);
+		umc = &pvt->umc[i];
+
+		amd_smn_read(nid, umc_base + UMCCH_DIMM_CFG, &umc->dimm_cfg);
+		amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
+		amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
+		amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
+		amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
+	}
 }
 
 /*
@@ -2276,24 +2645,31 @@ static void free_mc_sibling_devs(struct amd64_pvt *pvt)
  */
 static void read_mc_regs(struct amd64_pvt *pvt)
 {
-	unsigned range;
+	unsigned int range;
 	u64 msr_val;
-	u32 tmp;
 
 	/*
 	 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
-	 * those are Read-As-Zero
+	 * those are Read-As-Zero.
 	 */
 	rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
 	edac_dbg(0, "  TOP_MEM:  0x%016llx\n", pvt->top_mem);
 
-	/* check first whether TOP_MEM2 is enabled */
+	/* Check first whether TOP_MEM2 is enabled: */
 	rdmsrl(MSR_K8_SYSCFG, msr_val);
-	if (msr_val & (1U << 21)) {
+	if (msr_val & BIT(21)) {
 		rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
 		edac_dbg(0, "  TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
-	} else
+	} else {
 		edac_dbg(0, "  TOP_MEM2 disabled\n");
+	}
+
+	if (pvt->umc) {
+		__read_mc_regs_df(pvt);
+		amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar);
+
+		goto skip;
+	}
 
 	amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
 
@@ -2322,8 +2698,6 @@ static void read_mc_regs(struct amd64_pvt *pvt)
 			 dram_dst_node(pvt, range));
 	}
 
-	read_dct_base_mask(pvt);
-
 	amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
 	amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
 
@@ -2337,20 +2711,14 @@ static void read_mc_regs(struct amd64_pvt *pvt)
 		amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
 	}
 
-	pvt->ecc_sym_sz = 4;
+skip:
+	read_dct_base_mask(pvt);
+
 	determine_memory_type(pvt);
 	edac_dbg(1, "  DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
 
-	if (pvt->fam >= 0x10) {
-		amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
-		/* F16h has only DCT0, so no need to read dbam1 */
-		if (pvt->fam != 0x16)
-			amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
+	determine_ecc_sym_sz(pvt);
 
-		/* F10h, revD and later can do x8 ECC too */
-		if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
-			pvt->ecc_sym_sz = 8;
-	}
 	dump_misc_regs(pvt);
 }
 
@@ -2420,20 +2788,22 @@ static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
 static int init_csrows(struct mem_ctl_info *mci)
 {
 	struct amd64_pvt *pvt = mci->pvt_info;
+	enum edac_type edac_mode = EDAC_NONE;
 	struct csrow_info *csrow;
 	struct dimm_info *dimm;
-	enum edac_type edac_mode;
 	int i, j, empty = 1;
 	int nr_pages = 0;
 	u32 val;
 
-	amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
+	if (!pvt->umc) {
+		amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
 
-	pvt->nbcfg = val;
+		pvt->nbcfg = val;
 
-	edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
-		 pvt->mc_node_id, val,
-		 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
+		edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
+			 pvt->mc_node_id, val,
+			 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
+	}
 
 	/*
 	 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
@@ -2469,14 +2839,18 @@ static int init_csrows(struct mem_ctl_info *mci)
 
 		edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
 
-		/*
-		 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
-		 */
-		if (pvt->nbcfg & NBCFG_ECC_ENABLE)
-			edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) ?
-				    EDAC_S4ECD4ED : EDAC_SECDED;
-		else
-			edac_mode = EDAC_NONE;
+		/* Determine DIMM ECC mode: */
+		if (pvt->umc) {
+			if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED)
+				edac_mode = EDAC_S4ECD4ED;
+			else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED)
+				edac_mode = EDAC_SECDED;
+
+		} else if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
+			edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
+					? EDAC_S4ECD4ED
+					: EDAC_SECDED;
+		}
 
 		for (j = 0; j < pvt->channel_count; j++) {
 			dimm = csrow->channels[j]->dimm;
@@ -2539,7 +2913,7 @@ static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
 
 	if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
 		amd64_warn("%s: error allocating mask\n", __func__);
-		return false;
+		return -ENOMEM;
 	}
 
 	get_cpus_on_this_dct_cpumask(cmask, nid);
@@ -2627,7 +3001,6 @@ static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
 {
 	u32 value, mask = 0x3;		/* UECC/CECC enable */
 
-
 	if (!s->nbctl_valid)
 		return;
 
@@ -2663,20 +3036,50 @@ static const char *ecc_msg =
 
 static bool ecc_enabled(struct pci_dev *F3, u16 nid)
 {
-	u32 value;
-	u8 ecc_en = 0;
 	bool nb_mce_en = false;
+	u8 ecc_en = 0, i;
+	u32 value;
 
-	amd64_read_pci_cfg(F3, NBCFG, &value);
+	if (boot_cpu_data.x86 >= 0x17) {
+		u8 umc_en_mask = 0, ecc_en_mask = 0;
 
-	ecc_en = !!(value & NBCFG_ECC_ENABLE);
-	amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
+		for (i = 0; i < NUM_UMCS; i++) {
+			u32 base = get_umc_base(i);
+
+			/* Only check enabled UMCs. */
+			if (amd_smn_read(nid, base + UMCCH_SDP_CTRL, &value))
+				continue;
+
+			if (!(value & UMC_SDP_INIT))
+				continue;
+
+			umc_en_mask |= BIT(i);
 
-	nb_mce_en = nb_mce_bank_enabled_on_node(nid);
-	if (!nb_mce_en)
-		amd64_notice("NB MCE bank disabled, set MSR "
-			     "0x%08x[4] on node %d to enable.\n",
-			     MSR_IA32_MCG_CTL, nid);
+			if (amd_smn_read(nid, base + UMCCH_UMC_CAP_HI, &value))
+				continue;
+
+			if (value & UMC_ECC_ENABLED)
+				ecc_en_mask |= BIT(i);
+		}
+
+		/* Check whether at least one UMC is enabled: */
+		if (umc_en_mask)
+			ecc_en = umc_en_mask == ecc_en_mask;
+
+		/* Assume UMC MCA banks are enabled. */
+		nb_mce_en = true;
+	} else {
+		amd64_read_pci_cfg(F3, NBCFG, &value);
+
+		ecc_en = !!(value & NBCFG_ECC_ENABLE);
+
+		nb_mce_en = nb_mce_bank_enabled_on_node(nid);
+		if (!nb_mce_en)
+			amd64_notice("NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
+				     MSR_IA32_MCG_CTL, nid);
+	}
+
+	amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
 
 	if (!ecc_en || !nb_mce_en) {
 		amd64_notice("%s", ecc_msg);
@@ -2685,6 +3088,27 @@ static bool ecc_enabled(struct pci_dev *F3, u16 nid)
 	return true;
 }
 
+static inline void
+f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
+{
+	u8 i, ecc_en = 1, cpk_en = 1;
+
+	for (i = 0; i < NUM_UMCS; i++) {
+		if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
+			ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
+			cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
+		}
+	}
+
+	/* Set chipkill only if ECC is enabled: */
+	if (ecc_en) {
+		mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
+
+		if (cpk_en)
+			mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
+	}
+}
+
 static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
 				 struct amd64_family_type *fam)
 {
@@ -2693,17 +3117,21 @@ static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
 	mci->mtype_cap		= MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
 	mci->edac_ctl_cap	= EDAC_FLAG_NONE;
 
-	if (pvt->nbcap & NBCAP_SECDED)
-		mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
+	if (pvt->umc) {
+		f17h_determine_edac_ctl_cap(mci, pvt);
+	} else {
+		if (pvt->nbcap & NBCAP_SECDED)
+			mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
 
-	if (pvt->nbcap & NBCAP_CHIPKILL)
-		mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
+		if (pvt->nbcap & NBCAP_CHIPKILL)
+			mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
+	}
 
 	mci->edac_cap		= determine_edac_cap(pvt);
 	mci->mod_name		= EDAC_MOD_STR;
 	mci->mod_ver		= EDAC_AMD64_VERSION;
 	mci->ctl_name		= fam->ctl_name;
-	mci->dev_name		= pci_name(pvt->F2);
+	mci->dev_name		= pci_name(pvt->F3);
 	mci->ctl_page_to_phys	= NULL;
 
 	/* memory scrubber interface */
@@ -2759,6 +3187,11 @@ static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
 		pvt->ops	= &family_types[F16_CPUS].ops;
 		break;
 
+	case 0x17:
+		fam_type	= &family_types[F17_CPUS];
+		pvt->ops	= &family_types[F17_CPUS].ops;
+		break;
+
 	default:
 		amd64_err("Unsupported family!\n");
 		return NULL;
@@ -2789,6 +3222,7 @@ static int init_one_instance(unsigned int nid)
 	struct mem_ctl_info *mci = NULL;
 	struct edac_mc_layer layers[2];
 	struct amd64_pvt *pvt = NULL;
+	u16 pci_id1, pci_id2;
 	int err = 0, ret;
 
 	ret = -ENOMEM;
@@ -2804,10 +3238,23 @@ static int init_one_instance(unsigned int nid)
 	if (!fam_type)
 		goto err_free;
 
-	ret = -ENODEV;
-	err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f2_id);
+	if (pvt->fam >= 0x17) {
+		pvt->umc = kcalloc(NUM_UMCS, sizeof(struct amd64_umc), GFP_KERNEL);
+		if (!pvt->umc) {
+			ret = -ENOMEM;
+			goto err_free;
+		}
+
+		pci_id1 = fam_type->f0_id;
+		pci_id2 = fam_type->f6_id;
+	} else {
+		pci_id1 = fam_type->f1_id;
+		pci_id2 = fam_type->f2_id;
+	}
+
+	err = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2);
 	if (err)
-		goto err_free;
+		goto err_post_init;
 
 	read_mc_regs(pvt);
 
@@ -2857,7 +3304,10 @@ static int init_one_instance(unsigned int nid)
 	if (report_gart_errors)
 		amd_report_gart_errors(true);
 
-	amd_register_ecc_decoder(decode_bus_error);
+	if (pvt->umc)
+		amd_register_ecc_decoder(decode_umc_error);
+	else
+		amd_register_ecc_decoder(decode_bus_error);
 
 	return 0;
 
@@ -2867,6 +3317,10 @@ err_add_mc:
 err_siblings:
 	free_mc_sibling_devs(pvt);
 
+err_post_init:
+	if (pvt->fam >= 0x17)
+		kfree(pvt->umc);
+
 err_free:
 	kfree(pvt);
 
@@ -2893,7 +3347,11 @@ static int probe_one_instance(unsigned int nid)
 		if (!ecc_enable_override)
 			goto err_enable;
 
-		amd64_warn("Forcing ECC on!\n");
+		if (boot_cpu_data.x86 >= 0x17) {
+			amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
+			goto err_enable;
+		} else
+			amd64_warn("Forcing ECC on!\n");
 
 		if (!enable_ecc_error_reporting(s, nid, F3))
 			goto err_enable;
@@ -2902,7 +3360,9 @@ static int probe_one_instance(unsigned int nid)
 	ret = init_one_instance(nid);
 	if (ret < 0) {
 		amd64_err("Error probing instance: %d\n", nid);
-		restore_ecc_error_reporting(s, nid, F3);
+
+		if (boot_cpu_data.x86 < 0x17)
+			restore_ecc_error_reporting(s, nid, F3);
 	}
 
 	return ret;
@@ -2938,7 +3398,11 @@ static void remove_one_instance(unsigned int nid)
 
 	/* unregister from EDAC MCE */
 	amd_report_gart_errors(false);
-	amd_unregister_ecc_decoder(decode_bus_error);
+
+	if (pvt->umc)
+		amd_unregister_ecc_decoder(decode_umc_error);
+	else
+		amd_unregister_ecc_decoder(decode_bus_error);
 
 	kfree(ecc_stngs[nid]);
 	ecc_stngs[nid] = NULL;
@@ -2963,7 +3427,10 @@ static void setup_pci_device(void)
 		return;
 
 	pvt = mci->pvt_info;
-	pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
+	if (pvt->umc)
+		pci_ctl = edac_pci_create_generic_ctl(&pvt->F0->dev, EDAC_MOD_STR);
+	else
+		pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
 	if (!pci_ctl) {
 		pr_warn("%s(): Unable to create PCI control\n", __func__);
 		pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
@@ -2975,6 +3442,7 @@ static const struct x86_cpu_id amd64_cpuids[] = {
 	{ X86_VENDOR_AMD, 0x10, X86_MODEL_ANY,	X86_FEATURE_ANY, 0 },
 	{ X86_VENDOR_AMD, 0x15, X86_MODEL_ANY,	X86_FEATURE_ANY, 0 },
 	{ X86_VENDOR_AMD, 0x16, X86_MODEL_ANY,	X86_FEATURE_ANY, 0 },
+	{ X86_VENDOR_AMD, 0x17, X86_MODEL_ANY,	X86_FEATURE_ANY, 0 },
 	{ }
 };
 MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
diff --git a/drivers/edac/amd64_edac.h b/drivers/edac/amd64_edac.h
index c08870479054..496603d8f3d2 100644
--- a/drivers/edac/amd64_edac.h
+++ b/drivers/edac/amd64_edac.h
@@ -17,7 +17,7 @@
 #include <linux/mmzone.h>
 #include <linux/edac.h>
 #include <asm/msr.h>
-#include "edac_core.h"
+#include "edac_module.h"
 #include "mce_amd.h"
 
 #define amd64_debug(fmt, arg...) \
@@ -30,10 +30,10 @@
 	edac_printk(KERN_NOTICE, "amd64", fmt, ##arg)
 
 #define amd64_warn(fmt, arg...) \
-	edac_printk(KERN_WARNING, "amd64", fmt, ##arg)
+	edac_printk(KERN_WARNING, "amd64", "Warning: " fmt, ##arg)
 
 #define amd64_err(fmt, arg...) \
-	edac_printk(KERN_ERR, "amd64", fmt, ##arg)
+	edac_printk(KERN_ERR, "amd64", "Error: " fmt, ##arg)
 
 #define amd64_mc_warn(mci, fmt, arg...) \
 	edac_mc_chipset_printk(mci, KERN_WARNING, "amd64", fmt, ##arg)
@@ -118,6 +118,8 @@
 #define PCI_DEVICE_ID_AMD_16H_NB_F2	0x1532
 #define PCI_DEVICE_ID_AMD_16H_M30H_NB_F1 0x1581
 #define PCI_DEVICE_ID_AMD_16H_M30H_NB_F2 0x1582
+#define PCI_DEVICE_ID_AMD_17H_DF_F0	0x1460
+#define PCI_DEVICE_ID_AMD_17H_DF_F6	0x1466
 
 /*
  * Function 1 - Address Map
@@ -202,6 +204,8 @@
 #define DCT_SEL_HI			0x114
 
 #define F15H_M60H_SCRCTRL		0x1C8
+#define F17H_SCR_BASE_ADDR		0x48
+#define F17H_SCR_LIMIT_ADDR		0x4C
 
 /*
  * Function 3 - Misc Control
@@ -248,6 +252,31 @@
 /* MSRs */
 #define MSR_MCGCTL_NBE			BIT(4)
 
+/* F17h */
+
+/* F0: */
+#define DF_DHAR				0x104
+
+/* UMC CH register offsets */
+#define UMCCH_BASE_ADDR			0x0
+#define UMCCH_ADDR_MASK			0x20
+#define UMCCH_ADDR_CFG			0x30
+#define UMCCH_DIMM_CFG			0x80
+#define UMCCH_UMC_CFG			0x100
+#define UMCCH_SDP_CTRL			0x104
+#define UMCCH_ECC_CTRL			0x14C
+#define UMCCH_ECC_BAD_SYMBOL		0xD90
+#define UMCCH_UMC_CAP			0xDF0
+#define UMCCH_UMC_CAP_HI		0xDF4
+
+/* UMC CH bitfields */
+#define UMC_ECC_CHIPKILL_CAP		BIT(31)
+#define UMC_ECC_ENABLED			BIT(30)
+
+#define UMC_SDP_INIT			BIT(31)
+
+#define NUM_UMCS			2
+
 enum amd_families {
 	K8_CPUS = 0,
 	F10_CPUS,
@@ -256,6 +285,7 @@ enum amd_families {
 	F15_M60H_CPUS,
 	F16_CPUS,
 	F16_M30H_CPUS,
+	F17_CPUS,
 	NUM_FAMILIES,
 };
 
@@ -288,11 +318,19 @@ struct chip_select {
 	u8 m_cnt;
 };
 
+struct amd64_umc {
+	u32 dimm_cfg;		/* DIMM Configuration reg */
+	u32 umc_cfg;		/* Configuration reg */
+	u32 sdp_ctrl;		/* SDP Control reg */
+	u32 ecc_ctrl;		/* DRAM ECC Control reg */
+	u32 umc_cap_hi;		/* Capabilities High reg */
+};
+
 struct amd64_pvt {
 	struct low_ops *ops;
 
 	/* pci_device handles which we utilize */
-	struct pci_dev *F1, *F2, *F3;
+	struct pci_dev *F0, *F1, *F2, *F3, *F6;
 
 	u16 mc_node_id;		/* MC index of this MC node */
 	u8 fam;			/* CPU family */
@@ -335,6 +373,8 @@ struct amd64_pvt {
 
 	/* cache the dram_type */
 	enum mem_type dram_type;
+
+	struct amd64_umc *umc;	/* UMC registers */
 };
 
 enum err_codes {
@@ -342,6 +382,8 @@ enum err_codes {
 	ERR_NODE	= -1,
 	ERR_CSROW	= -2,
 	ERR_CHANNEL	= -3,
+	ERR_SYND	= -4,
+	ERR_NORM_ADDR	= -5,
 };
 
 struct err_info {
@@ -354,6 +396,12 @@ struct err_info {
 	u32 offset;
 };
 
+static inline u32 get_umc_base(u8 channel)
+{
+	/* ch0: 0x50000, ch1: 0x150000 */
+	return 0x50000 + (!!channel << 20);
+}
+
 static inline u64 get_dram_base(struct amd64_pvt *pvt, u8 i)
 {
 	u64 addr = ((u64)pvt->ranges[i].base.lo & 0xffff0000) << 8;
@@ -422,7 +470,7 @@ struct low_ops {
 
 struct amd64_family_type {
 	const char *ctl_name;
-	u16 f1_id, f2_id;
+	u16 f0_id, f1_id, f2_id, f6_id;
 	struct low_ops ops;
 };
 
diff --git a/drivers/edac/amd76x_edac.c b/drivers/edac/amd76x_edac.c
index 3a501b530e11..a7450275ad28 100644
--- a/drivers/edac/amd76x_edac.c
+++ b/drivers/edac/amd76x_edac.c
@@ -17,7 +17,7 @@
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
 #include <linux/edac.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define AMD76X_REVISION	" Ver: 2.0.2"
 #define EDAC_MOD_STR	"amd76x_edac"
diff --git a/drivers/edac/amd8111_edac.c b/drivers/edac/amd8111_edac.c
index 2b63f7c2d6d2..b5786cfded3a 100644
--- a/drivers/edac/amd8111_edac.c
+++ b/drivers/edac/amd8111_edac.c
@@ -29,7 +29,6 @@
 #include <linux/pci_ids.h>
 #include <asm/io.h>
 
-#include "edac_core.h"
 #include "edac_module.h"
 #include "amd8111_edac.h"
 
diff --git a/drivers/edac/amd8131_edac.c b/drivers/edac/amd8131_edac.c
index a5c680561c73..8851c33d7d24 100644
--- a/drivers/edac/amd8131_edac.c
+++ b/drivers/edac/amd8131_edac.c
@@ -29,7 +29,6 @@
 #include <linux/edac.h>
 #include <linux/pci_ids.h>
 
-#include "edac_core.h"
 #include "edac_module.h"
 #include "amd8131_edac.h"
 
diff --git a/drivers/edac/cell_edac.c b/drivers/edac/cell_edac.c
index a9259b069dcd..bc1f3416400e 100644
--- a/drivers/edac/cell_edac.c
+++ b/drivers/edac/cell_edac.c
@@ -19,7 +19,7 @@
 #include <asm/machdep.h>
 #include <asm/cell-regs.h>
 
-#include "edac_core.h"
+#include "edac_module.h"
 
 struct cell_edac_priv
 {
diff --git a/drivers/edac/cpc925_edac.c b/drivers/edac/cpc925_edac.c
index 682288ced4ac..837b62c4993d 100644
--- a/drivers/edac/cpc925_edac.c
+++ b/drivers/edac/cpc925_edac.c
@@ -27,7 +27,6 @@
 #include <linux/platform_device.h>
 #include <linux/gfp.h>
 
-#include "edac_core.h"
 #include "edac_module.h"
 
 #define CPC925_EDAC_REVISION	" Ver: 1.0.0"
diff --git a/drivers/edac/e752x_edac.c b/drivers/edac/e752x_edac.c
index b2d71388172b..1a352cae1f52 100644
--- a/drivers/edac/e752x_edac.c
+++ b/drivers/edac/e752x_edac.c
@@ -24,7 +24,7 @@
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
 #include <linux/edac.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define E752X_REVISION	" Ver: 2.0.2"
 #define EDAC_MOD_STR	"e752x_edac"
diff --git a/drivers/edac/e7xxx_edac.c b/drivers/edac/e7xxx_edac.c
index ece3aef16bb1..67ef07aed923 100644
--- a/drivers/edac/e7xxx_edac.c
+++ b/drivers/edac/e7xxx_edac.c
@@ -30,7 +30,7 @@
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
 #include <linux/edac.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define	E7XXX_REVISION " Ver: 2.0.2"
 #define	EDAC_MOD_STR	"e7xxx_edac"
diff --git a/drivers/edac/edac_device.c b/drivers/edac/edac_device.c
index a97900333e2d..de4d5d08af9e 100644
--- a/drivers/edac/edac_device.c
+++ b/drivers/edac/edac_device.c
@@ -12,23 +12,20 @@
  * 19 Jan 2007
  */
 
+#include <asm/page.h>
+#include <asm/uaccess.h>
+#include <linux/ctype.h>
+#include <linux/highmem.h>
+#include <linux/init.h>
+#include <linux/jiffies.h>
 #include <linux/module.h>
-#include <linux/types.h>
+#include <linux/slab.h>
 #include <linux/smp.h>
-#include <linux/init.h>
+#include <linux/spinlock.h>
 #include <linux/sysctl.h>
-#include <linux/highmem.h>
 #include <linux/timer.h>
-#include <linux/slab.h>
-#include <linux/jiffies.h>
-#include <linux/spinlock.h>
-#include <linux/list.h>
-#include <linux/ctype.h>
-#include <linux/workqueue.h>
-#include <asm/uaccess.h>
-#include <asm/page.h>
 
-#include "edac_core.h"
+#include "edac_device.h"
 #include "edac_module.h"
 
 /* lock for the list: 'edac_device_list', manipulation of this list
@@ -50,21 +47,6 @@ static void edac_device_dump_device(struct edac_device_ctl_info *edac_dev)
 }
 #endif				/* CONFIG_EDAC_DEBUG */
 
-
-/*
- * edac_device_alloc_ctl_info()
- *	Allocate a new edac device control info structure
- *
- *	The control structure is allocated in complete chunk
- *	from the OS. It is in turn sub allocated to the
- *	various objects that compose the structure
- *
- *	The structure has a 'nr_instance' array within itself.
- *	Each instance represents a major component
- *		Example:  L1 cache and L2 cache are 2 instance components
- *
- *	Within each instance is an array of 'nr_blocks' blockoffsets
- */
 struct edac_device_ctl_info *edac_device_alloc_ctl_info(
 	unsigned sz_private,
 	char *edac_device_name, unsigned nr_instances,
@@ -244,11 +226,6 @@ struct edac_device_ctl_info *edac_device_alloc_ctl_info(
 }
 EXPORT_SYMBOL_GPL(edac_device_alloc_ctl_info);
 
-/*
- * edac_device_free_ctl_info()
- *	frees the memory allocated by the edac_device_alloc_ctl_info()
- *	function
- */
 void edac_device_free_ctl_info(struct edac_device_ctl_info *ctl_info)
 {
 	edac_device_unregister_sysfs_main_kobj(ctl_info);
@@ -460,12 +437,6 @@ void edac_device_reset_delay_period(struct edac_device_ctl_info *edac_dev,
 	edac_mod_work(&edac_dev->work, jiffs);
 }
 
-/*
- * edac_device_alloc_index: Allocate a unique device index number
- *
- * Return:
- *	allocated index number
- */
 int edac_device_alloc_index(void)
 {
 	static atomic_t device_indexes = ATOMIC_INIT(0);
@@ -474,17 +445,6 @@ int edac_device_alloc_index(void)
 }
 EXPORT_SYMBOL_GPL(edac_device_alloc_index);
 
-/**
- * edac_device_add_device: Insert the 'edac_dev' structure into the
- * edac_device global list and create sysfs entries associated with
- * edac_device structure.
- * @edac_device: pointer to the edac_device structure to be added to the list
- * 'edac_device' structure.
- *
- * Return:
- *	0	Success
- *	!0	Failure
- */
 int edac_device_add_device(struct edac_device_ctl_info *edac_dev)
 {
 	edac_dbg(0, "\n");
@@ -541,19 +501,6 @@ fail0:
 }
 EXPORT_SYMBOL_GPL(edac_device_add_device);
 
-/**
- * edac_device_del_device:
- *	Remove sysfs entries for specified edac_device structure and
- *	then remove edac_device structure from global list
- *
- * @dev:
- *	Pointer to 'struct device' representing edac_device
- *	structure to remove.
- *
- * Return:
- *	Pointer to removed edac_device structure,
- *	OR NULL if device not found.
- */
 struct edac_device_ctl_info *edac_device_del_device(struct device *dev)
 {
 	struct edac_device_ctl_info *edac_dev;
@@ -608,10 +555,6 @@ static inline int edac_device_get_panic_on_ue(struct edac_device_ctl_info
 	return edac_dev->panic_on_ue;
 }
 
-/*
- * edac_device_handle_ce
- *	perform a common output and handling of an 'edac_dev' CE event
- */
 void edac_device_handle_ce(struct edac_device_ctl_info *edac_dev,
 			int inst_nr, int block_nr, const char *msg)
 {
@@ -654,10 +597,6 @@ void edac_device_handle_ce(struct edac_device_ctl_info *edac_dev,
 }
 EXPORT_SYMBOL_GPL(edac_device_handle_ce);
 
-/*
- * edac_device_handle_ue
- *	perform a common output and handling of an 'edac_dev' UE event
- */
 void edac_device_handle_ue(struct edac_device_ctl_info *edac_dev,
 			int inst_nr, int block_nr, const char *msg)
 {
diff --git a/drivers/edac/edac_core.h b/drivers/edac/edac_device.h
index 4861542163d7..1aaba74ae411 100644
--- a/drivers/edac/edac_core.h
+++ b/drivers/edac/edac_device.h
@@ -1,5 +1,5 @@
 /*
- * Defines, structures, APIs for edac_core module
+ * Defines, structures, APIs for edac_device
  *
  * (C) 2007 Linux Networx (http://lnxi.com)
  * This file may be distributed under the terms of the
@@ -15,86 +15,22 @@
  * Refactored for multi-source files:
  *	Doug Thompson <norsk5@xmission.com>
  *
+ * Please look at Documentation/driver-api/edac.rst for more info about
+ * EDAC core structs and functions.
  */
 
-#ifndef _EDAC_CORE_H_
-#define _EDAC_CORE_H_
+#ifndef _EDAC_DEVICE_H_
+#define _EDAC_DEVICE_H_
 
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/module.h>
-#include <linux/spinlock.h>
-#include <linux/smp.h>
-#include <linux/pci.h>
-#include <linux/time.h>
-#include <linux/nmi.h>
-#include <linux/rcupdate.h>
 #include <linux/completion.h>
+#include <linux/device.h>
+#include <linux/edac.h>
 #include <linux/kobject.h>
-#include <linux/platform_device.h>
+#include <linux/list.h>
+#include <linux/types.h>
+#include <linux/sysfs.h>
 #include <linux/workqueue.h>
-#include <linux/edac.h>
-
-#define EDAC_DEVICE_NAME_LEN	31
-#define EDAC_ATTRIB_VALUE_LEN	15
-
-#if PAGE_SHIFT < 20
-#define PAGES_TO_MiB(pages)	((pages) >> (20 - PAGE_SHIFT))
-#define MiB_TO_PAGES(mb)	((mb) << (20 - PAGE_SHIFT))
-#else				/* PAGE_SHIFT > 20 */
-#define PAGES_TO_MiB(pages)	((pages) << (PAGE_SHIFT - 20))
-#define MiB_TO_PAGES(mb)	((mb) >> (PAGE_SHIFT - 20))
-#endif
-
-#define edac_printk(level, prefix, fmt, arg...) \
-	printk(level "EDAC " prefix ": " fmt, ##arg)
-
-#define edac_mc_printk(mci, level, fmt, arg...) \
-	printk(level "EDAC MC%d: " fmt, mci->mc_idx, ##arg)
-
-#define edac_mc_chipset_printk(mci, level, prefix, fmt, arg...) \
-	printk(level "EDAC " prefix " MC%d: " fmt, mci->mc_idx, ##arg)
-
-#define edac_device_printk(ctl, level, fmt, arg...) \
-	printk(level "EDAC DEVICE%d: " fmt, ctl->dev_idx, ##arg)
-
-#define edac_pci_printk(ctl, level, fmt, arg...) \
-	printk(level "EDAC PCI%d: " fmt, ctl->pci_idx, ##arg)
 
-/* prefixes for edac_printk() and edac_mc_printk() */
-#define EDAC_MC "MC"
-#define EDAC_PCI "PCI"
-#define EDAC_DEBUG "DEBUG"
-
-extern const char * const edac_mem_types[];
-
-#ifdef CONFIG_EDAC_DEBUG
-extern int edac_debug_level;
-
-#define edac_dbg(level, fmt, ...)					\
-do {									\
-	if (level <= edac_debug_level)					\
-		edac_printk(KERN_DEBUG, EDAC_DEBUG,			\
-			    "%s: " fmt, __func__, ##__VA_ARGS__);	\
-} while (0)
-
-#else				/* !CONFIG_EDAC_DEBUG */
-
-#define edac_dbg(level, fmt, ...)					\
-do {									\
-	if (0)								\
-		edac_printk(KERN_DEBUG, EDAC_DEBUG,			\
-			    "%s: " fmt, __func__, ##__VA_ARGS__);	\
-} while (0)
-
-#endif				/* !CONFIG_EDAC_DEBUG */
-
-#define PCI_VEND_DEV(vend, dev) PCI_VENDOR_ID_ ## vend, \
-	PCI_DEVICE_ID_ ## vend ## _ ## dev
-
-#define edac_dev_name(dev) (dev)->dev_name
-
-#define to_mci(k) container_of(k, struct mem_ctl_info, dev)
 
 /*
  * The following are the structures to provide for a generic
@@ -321,197 +257,64 @@ extern struct edac_device_ctl_info *edac_device_alloc_ctl_info(
 
 extern void edac_device_free_ctl_info(struct edac_device_ctl_info *ctl_info);
 
-#ifdef CONFIG_PCI
-
-struct edac_pci_counter {
-	atomic_t pe_count;
-	atomic_t npe_count;
-};
-
-/*
- * Abstract edac_pci control info structure
+/**
+ * edac_device_add_device: Insert the 'edac_dev' structure into the
+ *	 edac_device global list and create sysfs entries associated with
+ *	 edac_device structure.
+ *
+ * @edac_dev: pointer to edac_device structure to be added to the list
+ *	'edac_device' structure.
  *
+ * Returns:
+ *	0 on Success, or an error code on failure
  */
-struct edac_pci_ctl_info {
-	/* for global list of edac_pci_ctl_info structs */
-	struct list_head link;
-
-	int pci_idx;
-
-	struct bus_type *edac_subsys;	/* pointer to subsystem */
-
-	/* the internal state of this controller instance */
-	int op_state;
-	/* work struct for this instance */
-	struct delayed_work work;
-
-	/* pointer to edac polling checking routine:
-	 *      If NOT NULL: points to polling check routine
-	 *      If NULL: Then assumes INTERRUPT operation, where
-	 *              MC driver will receive events
-	 */
-	void (*edac_check) (struct edac_pci_ctl_info * edac_dev);
-
-	struct device *dev;	/* pointer to device structure */
-
-	const char *mod_name;	/* module name */
-	const char *ctl_name;	/* edac controller  name */
-	const char *dev_name;	/* pci/platform/etc... name */
-
-	void *pvt_info;		/* pointer to 'private driver' info */
-
-	unsigned long start_time;	/* edac_pci load start time (jiffies) */
-
-	struct completion complete;
-
-	/* sysfs top name under 'edac' directory
-	 * and instance name:
-	 *      cpu/cpu0/...
-	 *      cpu/cpu1/...
-	 *      cpu/cpu2/...
-	 *      ...
-	 */
-	char name[EDAC_DEVICE_NAME_LEN + 1];
-
-	/* Event counters for the this whole EDAC Device */
-	struct edac_pci_counter counters;
-
-	/* edac sysfs device control for the 'name'
-	 * device this structure controls
-	 */
-	struct kobject kobj;
-	struct completion kobj_complete;
-};
-
-#define to_edac_pci_ctl_work(w) \
-		container_of(w, struct edac_pci_ctl_info,work)
-
-/* write all or some bits in a byte-register*/
-static inline void pci_write_bits8(struct pci_dev *pdev, int offset, u8 value,
-				   u8 mask)
-{
-	if (mask != 0xff) {
-		u8 buf;
-
-		pci_read_config_byte(pdev, offset, &buf);
-		value &= mask;
-		buf &= ~mask;
-		value |= buf;
-	}
-
-	pci_write_config_byte(pdev, offset, value);
-}
-
-/* write all or some bits in a word-register*/
-static inline void pci_write_bits16(struct pci_dev *pdev, int offset,
-				    u16 value, u16 mask)
-{
-	if (mask != 0xffff) {
-		u16 buf;
-
-		pci_read_config_word(pdev, offset, &buf);
-		value &= mask;
-		buf &= ~mask;
-		value |= buf;
-	}
-
-	pci_write_config_word(pdev, offset, value);
-}
+extern int edac_device_add_device(struct edac_device_ctl_info *edac_dev);
 
-/*
- * pci_write_bits32
+/**
+ * edac_device_del_device:
+ *	Remove sysfs entries for specified edac_device structure and
+ *	then remove edac_device structure from global list
  *
- * edac local routine to do pci_write_config_dword, but adds
- * a mask parameter. If mask is all ones, ignore the mask.
- * Otherwise utilize the mask to isolate specified bits
+ * @dev:
+ *	Pointer to struct &device representing the edac device
+ *	structure to remove.
  *
- * write all or some bits in a dword-register
+ * Returns:
+ *	Pointer to removed edac_device structure,
+ *	or %NULL if device not found.
  */
-static inline void pci_write_bits32(struct pci_dev *pdev, int offset,
-				    u32 value, u32 mask)
-{
-	if (mask != 0xffffffff) {
-		u32 buf;
-
-		pci_read_config_dword(pdev, offset, &buf);
-		value &= mask;
-		buf &= ~mask;
-		value |= buf;
-	}
-
-	pci_write_config_dword(pdev, offset, value);
-}
-
-#endif				/* CONFIG_PCI */
-
-struct mem_ctl_info *edac_mc_alloc(unsigned mc_num,
-				   unsigned n_layers,
-				   struct edac_mc_layer *layers,
-				   unsigned sz_pvt);
-extern int edac_mc_add_mc_with_groups(struct mem_ctl_info *mci,
-				      const struct attribute_group **groups);
-#define edac_mc_add_mc(mci)	edac_mc_add_mc_with_groups(mci, NULL)
-extern void edac_mc_free(struct mem_ctl_info *mci);
-extern struct mem_ctl_info *edac_mc_find(int idx);
-extern struct mem_ctl_info *find_mci_by_dev(struct device *dev);
-extern struct mem_ctl_info *edac_mc_del_mc(struct device *dev);
-extern int edac_mc_find_csrow_by_page(struct mem_ctl_info *mci,
-				      unsigned long page);
-
-void edac_raw_mc_handle_error(const enum hw_event_mc_err_type type,
-			      struct mem_ctl_info *mci,
-			      struct edac_raw_error_desc *e);
-
-void edac_mc_handle_error(const enum hw_event_mc_err_type type,
-			  struct mem_ctl_info *mci,
-			  const u16 error_count,
-			  const unsigned long page_frame_number,
-			  const unsigned long offset_in_page,
-			  const unsigned long syndrome,
-			  const int top_layer,
-			  const int mid_layer,
-			  const int low_layer,
-			  const char *msg,
-			  const char *other_detail);
+extern struct edac_device_ctl_info *edac_device_del_device(struct device *dev);
 
-/*
- * edac_device APIs
+/**
+ * edac_device_handle_ue():
+ *	perform a common output and handling of an 'edac_dev' UE event
+ *
+ * @edac_dev: pointer to struct &edac_device_ctl_info
+ * @inst_nr: number of the instance where the UE error happened
+ * @block_nr: number of the block where the UE error happened
+ * @msg: message to be printed
  */
-extern int edac_device_add_device(struct edac_device_ctl_info *edac_dev);
-extern struct edac_device_ctl_info *edac_device_del_device(struct device *dev);
 extern void edac_device_handle_ue(struct edac_device_ctl_info *edac_dev,
 				int inst_nr, int block_nr, const char *msg);
+/**
+ * edac_device_handle_ce():
+ *	perform a common output and handling of an 'edac_dev' CE event
+ *
+ * @edac_dev: pointer to struct &edac_device_ctl_info
+ * @inst_nr: number of the instance where the CE error happened
+ * @block_nr: number of the block where the CE error happened
+ * @msg: message to be printed
+ */
 extern void edac_device_handle_ce(struct edac_device_ctl_info *edac_dev,
 				int inst_nr, int block_nr, const char *msg);
-extern int edac_device_alloc_index(void);
-extern const char *edac_layer_name[];
 
-/*
- * edac_pci APIs
- */
-extern struct edac_pci_ctl_info *edac_pci_alloc_ctl_info(unsigned int sz_pvt,
-				const char *edac_pci_name);
-
-extern void edac_pci_free_ctl_info(struct edac_pci_ctl_info *pci);
-
-extern void edac_pci_reset_delay_period(struct edac_pci_ctl_info *pci,
-				unsigned long value);
-
-extern int edac_pci_alloc_index(void);
-extern int edac_pci_add_device(struct edac_pci_ctl_info *pci, int edac_idx);
-extern struct edac_pci_ctl_info *edac_pci_del_device(struct device *dev);
-
-extern struct edac_pci_ctl_info *edac_pci_create_generic_ctl(
-				struct device *dev,
-				const char *mod_name);
-
-extern void edac_pci_release_generic_ctl(struct edac_pci_ctl_info *pci);
-extern int edac_pci_create_sysfs(struct edac_pci_ctl_info *pci);
-extern void edac_pci_remove_sysfs(struct edac_pci_ctl_info *pci);
-
-/*
- * edac misc APIs
+/**
+ * edac_device_alloc_index: Allocate a unique device index number
+ *
+ * Returns:
+ *	allocated index number
  */
-extern char *edac_op_state_to_string(int op_state);
+extern int edac_device_alloc_index(void);
+extern const char *edac_layer_name[];
 
-#endif				/* _EDAC_CORE_H_ */
+#endif
diff --git a/drivers/edac/edac_device_sysfs.c b/drivers/edac/edac_device_sysfs.c
index 93da1a45c716..0e7ea3591b78 100644
--- a/drivers/edac/edac_device_sysfs.c
+++ b/drivers/edac/edac_device_sysfs.c
@@ -1,7 +1,7 @@
 /*
  * file for managing the edac_device subsystem of devices for EDAC
  *
- * (C) 2007 SoftwareBitMaker 
+ * (C) 2007 SoftwareBitMaker
  *
  * This file may be distributed under the terms of the
  * GNU General Public License.
@@ -15,7 +15,7 @@
 #include <linux/slab.h>
 #include <linux/edac.h>
 
-#include "edac_core.h"
+#include "edac_device.h"
 #include "edac_module.h"
 
 #define EDAC_DEVICE_SYMLINK	"device"
diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c
index c3ee3ad98a63..5f2c717f8053 100644
--- a/drivers/edac/edac_mc.c
+++ b/drivers/edac/edac_mc.c
@@ -30,7 +30,7 @@
 #include <linux/bitops.h>
 #include <asm/uaccess.h>
 #include <asm/page.h>
-#include "edac_core.h"
+#include "edac_mc.h"
 #include "edac_module.h"
 #include <ras/ras_event.h>
 
@@ -239,30 +239,6 @@ static void _edac_mc_free(struct mem_ctl_info *mci)
 	kfree(mci);
 }
 
-/**
- * edac_mc_alloc: Allocate and partially fill a struct mem_ctl_info structure
- * @mc_num:		Memory controller number
- * @n_layers:		Number of MC hierarchy layers
- * layers:		Describes each layer as seen by the Memory Controller
- * @size_pvt:		size of private storage needed
- *
- *
- * Everything is kmalloc'ed as one big chunk - more efficient.
- * Only can be used if all structures have the same lifetime - otherwise
- * you have to allocate and initialize your own structures.
- *
- * Use edac_mc_free() to free mc structures allocated by this function.
- *
- * NOTE: drivers handle multi-rank memories in different ways: in some
- * drivers, one multi-rank memory stick is mapped as one entry, while, in
- * others, a single multi-rank memory stick would be mapped into several
- * entries. Currently, this function will allocate multiple struct dimm_info
- * on such scenarios, as grouping the multiple ranks require drivers change.
- *
- * Returns:
- *	On failure: NULL
- *	On success: struct mem_ctl_info pointer
- */
 struct mem_ctl_info *edac_mc_alloc(unsigned mc_num,
 				   unsigned n_layers,
 				   struct edac_mc_layer *layers,
@@ -460,11 +436,6 @@ error:
 }
 EXPORT_SYMBOL_GPL(edac_mc_alloc);
 
-/**
- * edac_mc_free
- *	'Free' a previously allocated 'mci' structure
- * @mci: pointer to a struct mem_ctl_info structure
- */
 void edac_mc_free(struct mem_ctl_info *mci)
 {
 	edac_dbg(1, "\n");
@@ -482,15 +453,8 @@ void edac_mc_free(struct mem_ctl_info *mci)
 }
 EXPORT_SYMBOL_GPL(edac_mc_free);
 
-
-/**
- * find_mci_by_dev
- *
- *	scan list of controllers looking for the one that manages
- *	the 'dev' device
- * @dev: pointer to a struct device related with the MCI
- */
-struct mem_ctl_info *find_mci_by_dev(struct device *dev)
+/* Caller must hold mem_ctls_mutex */
+static struct mem_ctl_info *__find_mci_by_dev(struct device *dev)
 {
 	struct mem_ctl_info *mci;
 	struct list_head *item;
@@ -506,6 +470,24 @@ struct mem_ctl_info *find_mci_by_dev(struct device *dev)
 
 	return NULL;
 }
+
+/**
+ * find_mci_by_dev
+ *
+ *	scan list of controllers looking for the one that manages
+ *	the 'dev' device
+ * @dev: pointer to a struct device related with the MCI
+ */
+struct mem_ctl_info *find_mci_by_dev(struct device *dev)
+{
+	struct mem_ctl_info *ret;
+
+	mutex_lock(&mem_ctls_mutex);
+	ret = __find_mci_by_dev(dev);
+	mutex_unlock(&mem_ctls_mutex);
+
+	return ret;
+}
 EXPORT_SYMBOL_GPL(find_mci_by_dev);
 
 /*
@@ -588,7 +570,7 @@ static int add_mc_to_global_list(struct mem_ctl_info *mci)
 
 	insert_before = &mc_devices;
 
-	p = find_mci_by_dev(mci->pdev);
+	p = __find_mci_by_dev(mci->pdev);
 	if (unlikely(p != NULL))
 		goto fail0;
 
@@ -635,44 +617,30 @@ static int del_mc_from_global_list(struct mem_ctl_info *mci)
 	return handlers;
 }
 
-/**
- * edac_mc_find: Search for a mem_ctl_info structure whose index is 'idx'.
- *
- * If found, return a pointer to the structure.
- * Else return NULL.
- *
- * Caller must hold mem_ctls_mutex.
- */
 struct mem_ctl_info *edac_mc_find(int idx)
 {
+	struct mem_ctl_info *mci = NULL;
 	struct list_head *item;
-	struct mem_ctl_info *mci;
+
+	mutex_lock(&mem_ctls_mutex);
 
 	list_for_each(item, &mc_devices) {
 		mci = list_entry(item, struct mem_ctl_info, link);
 
 		if (mci->mc_idx >= idx) {
-			if (mci->mc_idx == idx)
-				return mci;
-
+			if (mci->mc_idx == idx) {
+				goto unlock;
+			}
 			break;
 		}
 	}
 
-	return NULL;
+unlock:
+	mutex_unlock(&mem_ctls_mutex);
+	return mci;
 }
 EXPORT_SYMBOL(edac_mc_find);
 
-/**
- * edac_mc_add_mc_with_groups: Insert the 'mci' structure into the mci
- *	global list and create sysfs entries associated with mci structure
- * @mci: pointer to the mci structure to be added to the list
- * @groups: optional attribute groups for the driver-specific sysfs entries
- *
- * Return:
- *	0	Success
- *	!0	Failure
- */
 
 /* FIXME - should a warning be printed if no error detection? correction? */
 int edac_mc_add_mc_with_groups(struct mem_ctl_info *mci,
@@ -763,13 +731,6 @@ fail0:
 }
 EXPORT_SYMBOL_GPL(edac_mc_add_mc_with_groups);
 
-/**
- * edac_mc_del_mc: Remove sysfs entries for specified mci structure and
- *                 remove mci structure from global list
- * @pdev: Pointer to 'struct device' representing mci structure to remove.
- *
- * Return pointer to removed mci structure, or NULL if device not found.
- */
 struct mem_ctl_info *edac_mc_del_mc(struct device *dev)
 {
 	struct mem_ctl_info *mci;
@@ -779,7 +740,7 @@ struct mem_ctl_info *edac_mc_del_mc(struct device *dev)
 	mutex_lock(&mem_ctls_mutex);
 
 	/* find the requested mci struct in the global list */
-	mci = find_mci_by_dev(dev);
+	mci = __find_mci_by_dev(dev);
 	if (mci == NULL) {
 		mutex_unlock(&mem_ctls_mutex);
 		return NULL;
@@ -1033,18 +994,6 @@ static void edac_ue_error(struct mem_ctl_info *mci,
 	edac_inc_ue_error(mci, enable_per_layer_report, pos, error_count);
 }
 
-/**
- * edac_raw_mc_handle_error - reports a memory event to userspace without doing
- *			      anything to discover the error location
- *
- * @type:		severity of the error (CE/UE/Fatal)
- * @mci:		a struct mem_ctl_info pointer
- * @e:			error description
- *
- * This raw function is used internally by edac_mc_handle_error(). It should
- * only be called directly when the hardware error come directly from BIOS,
- * like in the case of APEI GHES driver.
- */
 void edac_raw_mc_handle_error(const enum hw_event_mc_err_type type,
 			      struct mem_ctl_info *mci,
 			      struct edac_raw_error_desc *e)
@@ -1074,24 +1023,6 @@ void edac_raw_mc_handle_error(const enum hw_event_mc_err_type type,
 }
 EXPORT_SYMBOL_GPL(edac_raw_mc_handle_error);
 
-/**
- * edac_mc_handle_error - reports a memory event to userspace
- *
- * @type:		severity of the error (CE/UE/Fatal)
- * @mci:		a struct mem_ctl_info pointer
- * @error_count:	Number of errors of the same type
- * @page_frame_number:	mem page where the error occurred
- * @offset_in_page:	offset of the error inside the page
- * @syndrome:		ECC syndrome
- * @top_layer:		Memory layer[0] position
- * @mid_layer:		Memory layer[1] position
- * @low_layer:		Memory layer[2] position
- * @msg:		Message meaningful to the end users that
- *			explains the event
- * @other_detail:	Technical details about the event that
- *			may help hardware manufacturers and
- *			EDAC developers to analyse the event
- */
 void edac_mc_handle_error(const enum hw_event_mc_err_type type,
 			  struct mem_ctl_info *mci,
 			  const u16 error_count,
diff --git a/drivers/edac/edac_mc.h b/drivers/edac/edac_mc.h
new file mode 100644
index 000000000000..50fc1dc9c0d8
--- /dev/null
+++ b/drivers/edac/edac_mc.h
@@ -0,0 +1,245 @@
+/*
+ * Defines, structures, APIs for edac_mc module
+ *
+ * (C) 2007 Linux Networx (http://lnxi.com)
+ * This file may be distributed under the terms of the
+ * GNU General Public License.
+ *
+ * Written by Thayne Harbaugh
+ * Based on work by Dan Hollis <goemon at anime dot net> and others.
+ *	http://www.anime.net/~goemon/linux-ecc/
+ *
+ * NMI handling support added by
+ *     Dave Peterson <dsp@llnl.gov> <dave_peterson@pobox.com>
+ *
+ * Refactored for multi-source files:
+ *	Doug Thompson <norsk5@xmission.com>
+ *
+ * Please look at Documentation/driver-api/edac.rst for more info about
+ * EDAC core structs and functions.
+ */
+
+#ifndef _EDAC_MC_H_
+#define _EDAC_MC_H_
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/spinlock.h>
+#include <linux/smp.h>
+#include <linux/pci.h>
+#include <linux/time.h>
+#include <linux/nmi.h>
+#include <linux/rcupdate.h>
+#include <linux/completion.h>
+#include <linux/kobject.h>
+#include <linux/platform_device.h>
+#include <linux/workqueue.h>
+#include <linux/edac.h>
+
+#if PAGE_SHIFT < 20
+#define PAGES_TO_MiB(pages)	((pages) >> (20 - PAGE_SHIFT))
+#define MiB_TO_PAGES(mb)	((mb) << (20 - PAGE_SHIFT))
+#else				/* PAGE_SHIFT > 20 */
+#define PAGES_TO_MiB(pages)	((pages) << (PAGE_SHIFT - 20))
+#define MiB_TO_PAGES(mb)	((mb) >> (PAGE_SHIFT - 20))
+#endif
+
+#define edac_printk(level, prefix, fmt, arg...) \
+	printk(level "EDAC " prefix ": " fmt, ##arg)
+
+#define edac_mc_printk(mci, level, fmt, arg...) \
+	printk(level "EDAC MC%d: " fmt, mci->mc_idx, ##arg)
+
+#define edac_mc_chipset_printk(mci, level, prefix, fmt, arg...) \
+	printk(level "EDAC " prefix " MC%d: " fmt, mci->mc_idx, ##arg)
+
+#define edac_device_printk(ctl, level, fmt, arg...) \
+	printk(level "EDAC DEVICE%d: " fmt, ctl->dev_idx, ##arg)
+
+#define edac_pci_printk(ctl, level, fmt, arg...) \
+	printk(level "EDAC PCI%d: " fmt, ctl->pci_idx, ##arg)
+
+/* prefixes for edac_printk() and edac_mc_printk() */
+#define EDAC_MC "MC"
+#define EDAC_PCI "PCI"
+#define EDAC_DEBUG "DEBUG"
+
+extern const char * const edac_mem_types[];
+
+#ifdef CONFIG_EDAC_DEBUG
+extern int edac_debug_level;
+
+#define edac_dbg(level, fmt, ...)					\
+do {									\
+	if (level <= edac_debug_level)					\
+		edac_printk(KERN_DEBUG, EDAC_DEBUG,			\
+			    "%s: " fmt, __func__, ##__VA_ARGS__);	\
+} while (0)
+
+#else				/* !CONFIG_EDAC_DEBUG */
+
+#define edac_dbg(level, fmt, ...)					\
+do {									\
+	if (0)								\
+		edac_printk(KERN_DEBUG, EDAC_DEBUG,			\
+			    "%s: " fmt, __func__, ##__VA_ARGS__);	\
+} while (0)
+
+#endif				/* !CONFIG_EDAC_DEBUG */
+
+#define PCI_VEND_DEV(vend, dev) PCI_VENDOR_ID_ ## vend, \
+	PCI_DEVICE_ID_ ## vend ## _ ## dev
+
+#define edac_dev_name(dev) (dev)->dev_name
+
+#define to_mci(k) container_of(k, struct mem_ctl_info, dev)
+
+/**
+ * edac_mc_alloc() - Allocate and partially fill a struct &mem_ctl_info.
+ *
+ * @mc_num:		Memory controller number
+ * @n_layers:		Number of MC hierarchy layers
+ * @layers:		Describes each layer as seen by the Memory Controller
+ * @sz_pvt:		size of private storage needed
+ *
+ *
+ * Everything is kmalloc'ed as one big chunk - more efficient.
+ * Only can be used if all structures have the same lifetime - otherwise
+ * you have to allocate and initialize your own structures.
+ *
+ * Use edac_mc_free() to free mc structures allocated by this function.
+ *
+ * .. note::
+ *
+ *   drivers handle multi-rank memories in different ways: in some
+ *   drivers, one multi-rank memory stick is mapped as one entry, while, in
+ *   others, a single multi-rank memory stick would be mapped into several
+ *   entries. Currently, this function will allocate multiple struct dimm_info
+ *   on such scenarios, as grouping the multiple ranks require drivers change.
+ *
+ * Returns:
+ *	On success, return a pointer to struct mem_ctl_info pointer;
+ *	%NULL otherwise
+ */
+struct mem_ctl_info *edac_mc_alloc(unsigned mc_num,
+				   unsigned n_layers,
+				   struct edac_mc_layer *layers,
+				   unsigned sz_pvt);
+
+/**
+ * edac_mc_add_mc_with_groups() - Insert the @mci structure into the mci
+ *	global list and create sysfs entries associated with @mci structure.
+ *
+ * @mci: pointer to the mci structure to be added to the list
+ * @groups: optional attribute groups for the driver-specific sysfs entries
+ *
+ * Returns:
+ *	0 on Success, or an error code on failure
+ */
+extern int edac_mc_add_mc_with_groups(struct mem_ctl_info *mci,
+				      const struct attribute_group **groups);
+#define edac_mc_add_mc(mci)	edac_mc_add_mc_with_groups(mci, NULL)
+
+/**
+ * edac_mc_free() -  Frees a previously allocated @mci structure
+ *
+ * @mci: pointer to a struct mem_ctl_info structure
+ */
+extern void edac_mc_free(struct mem_ctl_info *mci);
+
+/**
+ * edac_mc_find() - Search for a mem_ctl_info structure whose index is @idx.
+ *
+ * @idx: index to be seek
+ *
+ * If found, return a pointer to the structure.
+ * Else return NULL.
+ */
+extern struct mem_ctl_info *edac_mc_find(int idx);
+
+/**
+ * find_mci_by_dev() - Scan list of controllers looking for the one that
+ *	manages the @dev device.
+ *
+ * @dev: pointer to a struct device related with the MCI
+ *
+ * Returns: on success, returns a pointer to struct &mem_ctl_info;
+ * %NULL otherwise.
+ */
+extern struct mem_ctl_info *find_mci_by_dev(struct device *dev);
+
+/**
+ * edac_mc_del_mc() - Remove sysfs entries for mci structure associated with
+ *	@dev and remove mci structure from global list.
+ *
+ * @dev: Pointer to struct &device representing mci structure to remove.
+ *
+ * Returns: pointer to removed mci structure, or %NULL if device not found.
+ */
+extern struct mem_ctl_info *edac_mc_del_mc(struct device *dev);
+
+/**
+ * edac_mc_find_csrow_by_page() - Ancillary routine to identify what csrow
+ *	contains a memory page.
+ *
+ * @mci: pointer to a struct mem_ctl_info structure
+ * @page: memory page to find
+ *
+ * Returns: on success, returns the csrow. -1 if not found.
+ */
+extern int edac_mc_find_csrow_by_page(struct mem_ctl_info *mci,
+				      unsigned long page);
+
+/**
+ * edac_raw_mc_handle_error() - Reports a memory event to userspace without
+ *	doing anything to discover the error location.
+ *
+ * @type:		severity of the error (CE/UE/Fatal)
+ * @mci:		a struct mem_ctl_info pointer
+ * @e:			error description
+ *
+ * This raw function is used internally by edac_mc_handle_error(). It should
+ * only be called directly when the hardware error come directly from BIOS,
+ * like in the case of APEI GHES driver.
+ */
+void edac_raw_mc_handle_error(const enum hw_event_mc_err_type type,
+			      struct mem_ctl_info *mci,
+			      struct edac_raw_error_desc *e);
+
+/**
+ * edac_mc_handle_error() - Reports a memory event to userspace.
+ *
+ * @type:		severity of the error (CE/UE/Fatal)
+ * @mci:		a struct mem_ctl_info pointer
+ * @error_count:	Number of errors of the same type
+ * @page_frame_number:	mem page where the error occurred
+ * @offset_in_page:	offset of the error inside the page
+ * @syndrome:		ECC syndrome
+ * @top_layer:		Memory layer[0] position
+ * @mid_layer:		Memory layer[1] position
+ * @low_layer:		Memory layer[2] position
+ * @msg:		Message meaningful to the end users that
+ *			explains the event
+ * @other_detail:	Technical details about the event that
+ *			may help hardware manufacturers and
+ *			EDAC developers to analyse the event
+ */
+void edac_mc_handle_error(const enum hw_event_mc_err_type type,
+			  struct mem_ctl_info *mci,
+			  const u16 error_count,
+			  const unsigned long page_frame_number,
+			  const unsigned long offset_in_page,
+			  const unsigned long syndrome,
+			  const int top_layer,
+			  const int mid_layer,
+			  const int low_layer,
+			  const char *msg,
+			  const char *other_detail);
+
+/*
+ * edac misc APIs
+ */
+extern char *edac_op_state_to_string(int op_state);
+
+#endif				/* _EDAC_MC_H_ */
diff --git a/drivers/edac/edac_mc_sysfs.c b/drivers/edac/edac_mc_sysfs.c
index 4e0f8e720ad9..39dbab7d62f1 100644
--- a/drivers/edac/edac_mc_sysfs.c
+++ b/drivers/edac/edac_mc_sysfs.c
@@ -19,7 +19,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/uaccess.h>
 
-#include "edac_core.h"
+#include "edac_mc.h"
 #include "edac_module.h"
 
 /* MC EDAC Controls, setable by module parameter, and sysfs */
diff --git a/drivers/edac/edac_module.c b/drivers/edac/edac_module.c
index 5f8543be995a..172598a27d7d 100644
--- a/drivers/edac/edac_module.c
+++ b/drivers/edac/edac_module.c
@@ -12,7 +12,7 @@
  */
 #include <linux/edac.h>
 
-#include "edac_core.h"
+#include "edac_mc.h"
 #include "edac_module.h"
 
 #define EDAC_VERSION "Ver: 3.0.0"
diff --git a/drivers/edac/edac_module.h b/drivers/edac/edac_module.h
index cfaacb99c973..014871e169cc 100644
--- a/drivers/edac/edac_module.h
+++ b/drivers/edac/edac_module.h
@@ -10,7 +10,9 @@
 #ifndef	__EDAC_MODULE_H__
 #define	__EDAC_MODULE_H__
 
-#include "edac_core.h"
+#include "edac_mc.h"
+#include "edac_pci.h"
+#include "edac_device.h"
 
 /*
  * INTERNAL EDAC MODULE:
diff --git a/drivers/edac/edac_pci.c b/drivers/edac/edac_pci.c
index 8f2f2899a7a2..4e9d5632041a 100644
--- a/drivers/edac/edac_pci.c
+++ b/drivers/edac/edac_pci.c
@@ -9,35 +9,25 @@
  * or implied.
  *
  */
+#include <asm/page.h>
+#include <asm/uaccess.h>
+#include <linux/ctype.h>
+#include <linux/highmem.h>
+#include <linux/init.h>
 #include <linux/module.h>
-#include <linux/types.h>
+#include <linux/slab.h>
 #include <linux/smp.h>
-#include <linux/init.h>
+#include <linux/spinlock.h>
 #include <linux/sysctl.h>
-#include <linux/highmem.h>
 #include <linux/timer.h>
-#include <linux/slab.h>
-#include <linux/spinlock.h>
-#include <linux/list.h>
-#include <linux/ctype.h>
-#include <linux/workqueue.h>
-#include <asm/uaccess.h>
-#include <asm/page.h>
 
-#include "edac_core.h"
+#include "edac_pci.h"
 #include "edac_module.h"
 
 static DEFINE_MUTEX(edac_pci_ctls_mutex);
 static LIST_HEAD(edac_pci_list);
 static atomic_t pci_indexes = ATOMIC_INIT(0);
 
-/*
- * edac_pci_alloc_ctl_info
- *
- *	The alloc() function for the 'edac_pci' control info
- *	structure. The chip driver will allocate one of these for each
- *	edac_pci it is going to control/register with the EDAC CORE.
- */
 struct edac_pci_ctl_info *edac_pci_alloc_ctl_info(unsigned int sz_pvt,
 						const char *edac_pci_name)
 {
@@ -68,16 +58,6 @@ struct edac_pci_ctl_info *edac_pci_alloc_ctl_info(unsigned int sz_pvt,
 }
 EXPORT_SYMBOL_GPL(edac_pci_alloc_ctl_info);
 
-/*
- * edac_pci_free_ctl_info()
- *
- *	Last action on the pci control structure.
- *
- *	call the remove sysfs information, which will unregister
- *	this control struct's kobj. When that kobj's ref count
- *	goes to zero, its release function will be call and then
- *	kfree() the memory.
- */
 void edac_pci_free_ctl_info(struct edac_pci_ctl_info *pci)
 {
 	edac_dbg(1, "\n");
@@ -215,31 +195,12 @@ static void edac_pci_workq_function(struct work_struct *work_req)
 	mutex_unlock(&edac_pci_ctls_mutex);
 }
 
-/*
- * edac_pci_alloc_index: Allocate a unique PCI index number
- *
- * Return:
- *      allocated index number
- *
- */
 int edac_pci_alloc_index(void)
 {
 	return atomic_inc_return(&pci_indexes) - 1;
 }
 EXPORT_SYMBOL_GPL(edac_pci_alloc_index);
 
-/*
- * edac_pci_add_device: Insert the 'edac_dev' structure into the
- * edac_pci global list and create sysfs entries associated with
- * edac_pci structure.
- * @pci: pointer to the edac_device structure to be added to the list
- * @edac_idx: A unique numeric identifier to be assigned to the
- * 'edac_pci' structure.
- *
- * Return:
- *      0       Success
- *      !0      Failure
- */
 int edac_pci_add_device(struct edac_pci_ctl_info *pci, int edac_idx)
 {
 	edac_dbg(0, "\n");
@@ -285,19 +246,6 @@ fail0:
 }
 EXPORT_SYMBOL_GPL(edac_pci_add_device);
 
-/*
- * edac_pci_del_device()
- * 	Remove sysfs entries for specified edac_pci structure and
- * 	then remove edac_pci structure from global list
- *
- * @dev:
- * 	Pointer to 'struct device' representing edac_pci structure
- * 	to remove
- *
- * Return:
- * 	Pointer to removed edac_pci structure,
- * 	or NULL if device not found
- */
 struct edac_pci_ctl_info *edac_pci_del_device(struct device *dev)
 {
 	struct edac_pci_ctl_info *pci;
@@ -351,17 +299,6 @@ struct edac_pci_gen_data {
 	int edac_idx;
 };
 
-/*
- * edac_pci_create_generic_ctl
- *
- *	A generic constructor for a PCI parity polling device
- *	Some systems have more than one domain of PCI busses.
- *	For systems with one domain, then this API will
- *	provide for a generic poller.
- *
- *	This routine calls the edac_pci_alloc_ctl_info() for
- *	the generic device, with default values
- */
 struct edac_pci_ctl_info *edac_pci_create_generic_ctl(struct device *dev,
 						const char *mod_name)
 {
@@ -394,11 +331,6 @@ struct edac_pci_ctl_info *edac_pci_create_generic_ctl(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(edac_pci_create_generic_ctl);
 
-/*
- * edac_pci_release_generic_ctl
- *
- *	The release function of a generic EDAC PCI polling device
- */
 void edac_pci_release_generic_ctl(struct edac_pci_ctl_info *pci)
 {
 	edac_dbg(0, "pci mod=%s\n", pci->mod_name);
diff --git a/drivers/edac/edac_pci.h b/drivers/edac/edac_pci.h
new file mode 100644
index 000000000000..5175f5724cfa
--- /dev/null
+++ b/drivers/edac/edac_pci.h
@@ -0,0 +1,271 @@
+/*
+ * Defines, structures, APIs for edac_pci and edac_pci_sysfs
+ *
+ * (C) 2007 Linux Networx (http://lnxi.com)
+ * This file may be distributed under the terms of the
+ * GNU General Public License.
+ *
+ * Written by Thayne Harbaugh
+ * Based on work by Dan Hollis <goemon at anime dot net> and others.
+ *	http://www.anime.net/~goemon/linux-ecc/
+ *
+ * NMI handling support added by
+ *     Dave Peterson <dsp@llnl.gov> <dave_peterson@pobox.com>
+ *
+ * Refactored for multi-source files:
+ *	Doug Thompson <norsk5@xmission.com>
+ *
+ * Please look at Documentation/driver-api/edac.rst for more info about
+ * EDAC core structs and functions.
+ */
+
+#ifndef _EDAC_PCI_H_
+#define _EDAC_PCI_H_
+
+#include <linux/completion.h>
+#include <linux/device.h>
+#include <linux/edac.h>
+#include <linux/kobject.h>
+#include <linux/list.h>
+#include <linux/pci.h>
+#include <linux/types.h>
+#include <linux/workqueue.h>
+
+#ifdef CONFIG_PCI
+
+struct edac_pci_counter {
+	atomic_t pe_count;
+	atomic_t npe_count;
+};
+
+/*
+ * Abstract edac_pci control info structure
+ *
+ */
+struct edac_pci_ctl_info {
+	/* for global list of edac_pci_ctl_info structs */
+	struct list_head link;
+
+	int pci_idx;
+
+	struct bus_type *edac_subsys;	/* pointer to subsystem */
+
+	/* the internal state of this controller instance */
+	int op_state;
+	/* work struct for this instance */
+	struct delayed_work work;
+
+	/* pointer to edac polling checking routine:
+	 *      If NOT NULL: points to polling check routine
+	 *      If NULL: Then assumes INTERRUPT operation, where
+	 *              MC driver will receive events
+	 */
+	void (*edac_check) (struct edac_pci_ctl_info * edac_dev);
+
+	struct device *dev;	/* pointer to device structure */
+
+	const char *mod_name;	/* module name */
+	const char *ctl_name;	/* edac controller  name */
+	const char *dev_name;	/* pci/platform/etc... name */
+
+	void *pvt_info;		/* pointer to 'private driver' info */
+
+	unsigned long start_time;	/* edac_pci load start time (jiffies) */
+
+	struct completion complete;
+
+	/* sysfs top name under 'edac' directory
+	 * and instance name:
+	 *      cpu/cpu0/...
+	 *      cpu/cpu1/...
+	 *      cpu/cpu2/...
+	 *      ...
+	 */
+	char name[EDAC_DEVICE_NAME_LEN + 1];
+
+	/* Event counters for the this whole EDAC Device */
+	struct edac_pci_counter counters;
+
+	/* edac sysfs device control for the 'name'
+	 * device this structure controls
+	 */
+	struct kobject kobj;
+};
+
+#define to_edac_pci_ctl_work(w) \
+		container_of(w, struct edac_pci_ctl_info,work)
+
+/* write all or some bits in a byte-register*/
+static inline void pci_write_bits8(struct pci_dev *pdev, int offset, u8 value,
+				   u8 mask)
+{
+	if (mask != 0xff) {
+		u8 buf;
+
+		pci_read_config_byte(pdev, offset, &buf);
+		value &= mask;
+		buf &= ~mask;
+		value |= buf;
+	}
+
+	pci_write_config_byte(pdev, offset, value);
+}
+
+/* write all or some bits in a word-register*/
+static inline void pci_write_bits16(struct pci_dev *pdev, int offset,
+				    u16 value, u16 mask)
+{
+	if (mask != 0xffff) {
+		u16 buf;
+
+		pci_read_config_word(pdev, offset, &buf);
+		value &= mask;
+		buf &= ~mask;
+		value |= buf;
+	}
+
+	pci_write_config_word(pdev, offset, value);
+}
+
+/*
+ * pci_write_bits32
+ *
+ * edac local routine to do pci_write_config_dword, but adds
+ * a mask parameter. If mask is all ones, ignore the mask.
+ * Otherwise utilize the mask to isolate specified bits
+ *
+ * write all or some bits in a dword-register
+ */
+static inline void pci_write_bits32(struct pci_dev *pdev, int offset,
+				    u32 value, u32 mask)
+{
+	if (mask != 0xffffffff) {
+		u32 buf;
+
+		pci_read_config_dword(pdev, offset, &buf);
+		value &= mask;
+		buf &= ~mask;
+		value |= buf;
+	}
+
+	pci_write_config_dword(pdev, offset, value);
+}
+
+#endif				/* CONFIG_PCI */
+
+/*
+ * edac_pci APIs
+ */
+
+/**
+ * edac_pci_alloc_ctl_info:
+ *	The alloc() function for the 'edac_pci' control info
+ *	structure.
+ *
+ * @sz_pvt: size of the private info at struct &edac_pci_ctl_info
+ * @edac_pci_name: name of the PCI device
+ *
+ * The chip driver will allocate one of these for each
+ * edac_pci it is going to control/register with the EDAC CORE.
+ *
+ * Returns: a pointer to struct &edac_pci_ctl_info on success; %NULL otherwise.
+ */
+extern struct edac_pci_ctl_info *edac_pci_alloc_ctl_info(unsigned int sz_pvt,
+				const char *edac_pci_name);
+
+/**
+ * edac_pci_free_ctl_info():
+ *	Last action on the pci control structure.
+ *
+ * @pci: pointer to struct &edac_pci_ctl_info
+ *
+ * Calls the remove sysfs information, which will unregister
+ * this control struct's kobj. When that kobj's ref count
+ * goes to zero, its release function will be call and then
+ * kfree() the memory.
+ */
+extern void edac_pci_free_ctl_info(struct edac_pci_ctl_info *pci);
+
+/**
+ * edac_pci_alloc_index: Allocate a unique PCI index number
+ *
+ * Returns:
+ *      allocated index number
+ *
+ */
+extern int edac_pci_alloc_index(void);
+
+/**
+ * edac_pci_add_device(): Insert the 'edac_dev' structure into the
+ *	edac_pci global list and create sysfs entries associated with
+ *	edac_pci structure.
+ *
+ * @pci: pointer to the edac_device structure to be added to the list
+ * @edac_idx: A unique numeric identifier to be assigned to the
+ *	'edac_pci' structure.
+ *
+ * Returns:
+ *	0 on Success, or an error code on failure
+ */
+extern int edac_pci_add_device(struct edac_pci_ctl_info *pci, int edac_idx);
+
+/**
+ * edac_pci_del_device()
+ *	Remove sysfs entries for specified edac_pci structure and
+ *	then remove edac_pci structure from global list
+ *
+ * @dev:
+ *	Pointer to 'struct device' representing edac_pci structure
+ *	to remove
+ *
+ * Returns:
+ *	Pointer to removed edac_pci structure,
+ *	or %NULL if device not found
+ */
+extern struct edac_pci_ctl_info *edac_pci_del_device(struct device *dev);
+
+/**
+ * edac_pci_create_generic_ctl()
+ *	A generic constructor for a PCI parity polling device
+ *	Some systems have more than one domain of PCI busses.
+ *	For systems with one domain, then this API will
+ *	provide for a generic poller.
+ *
+ * @dev: pointer to struct &device;
+ * @mod_name: name of the PCI device
+ *
+ * This routine calls the edac_pci_alloc_ctl_info() for
+ * the generic device, with default values
+ *
+ * Returns: Pointer to struct &edac_pci_ctl_info on success, %NULL on
+ *	failure.
+ */
+extern struct edac_pci_ctl_info *edac_pci_create_generic_ctl(
+				struct device *dev,
+				const char *mod_name);
+
+/**
+ * edac_pci_release_generic_ctl
+ *	The release function of a generic EDAC PCI polling device
+ *
+ * @pci: pointer to struct &edac_pci_ctl_info
+ */
+extern void edac_pci_release_generic_ctl(struct edac_pci_ctl_info *pci);
+
+/**
+ * edac_pci_create_sysfs
+ *	Create the controls/attributes for the specified EDAC PCI device
+ *
+ * @pci: pointer to struct &edac_pci_ctl_info
+ */
+extern int edac_pci_create_sysfs(struct edac_pci_ctl_info *pci);
+
+/**
+ * edac_pci_remove_sysfs()
+ *	remove the controls and attributes for this EDAC PCI device
+ *
+ * @pci: pointer to struct &edac_pci_ctl_info
+ */
+extern void edac_pci_remove_sysfs(struct edac_pci_ctl_info *pci);
+
+#endif
diff --git a/drivers/edac/edac_pci_sysfs.c b/drivers/edac/edac_pci_sysfs.c
index 6e3428ba400f..72c9eb9fdffb 100644
--- a/drivers/edac/edac_pci_sysfs.c
+++ b/drivers/edac/edac_pci_sysfs.c
@@ -11,7 +11,7 @@
 #include <linux/slab.h>
 #include <linux/ctype.h>
 
-#include "edac_core.h"
+#include "edac_pci.h"
 #include "edac_module.h"
 
 #define EDAC_PCI_SYMLINK	"device"
@@ -418,12 +418,6 @@ static void edac_pci_main_kobj_teardown(void)
 	}
 }
 
-/*
- *
- * edac_pci_create_sysfs
- *
- *	Create the controls/attributes for the specified EDAC PCI device
- */
 int edac_pci_create_sysfs(struct edac_pci_ctl_info *pci)
 {
 	int err;
@@ -459,11 +453,6 @@ unregister_cleanup:
 	return err;
 }
 
-/*
- * edac_pci_remove_sysfs
- *
- *	remove the controls and attributes for this EDAC PCI device
- */
 void edac_pci_remove_sysfs(struct edac_pci_ctl_info *pci)
 {
 	edac_dbg(0, "index=%d\n", pci->pci_idx);
diff --git a/drivers/edac/fsl_ddr_edac.c b/drivers/edac/fsl_ddr_edac.c
index 9774f52f0c3e..4e9608a958e7 100644
--- a/drivers/edac/fsl_ddr_edac.c
+++ b/drivers/edac/fsl_ddr_edac.c
@@ -28,7 +28,6 @@
 #include <linux/of_device.h>
 #include <linux/of_address.h>
 #include "edac_module.h"
-#include "edac_core.h"
 #include "fsl_ddr_edac.h"
 
 #define EDAC_MOD_STR	"fsl_ddr_edac"
diff --git a/drivers/edac/ghes_edac.c b/drivers/edac/ghes_edac.c
index e3fa4390f846..4e61a6229dd2 100644
--- a/drivers/edac/ghes_edac.c
+++ b/drivers/edac/ghes_edac.c
@@ -14,7 +14,7 @@
 #include <acpi/ghes.h>
 #include <linux/edac.h>
 #include <linux/dmi.h>
-#include "edac_core.h"
+#include "edac_module.h"
 #include <ras/ras_event.h>
 
 #define GHES_EDAC_REVISION " Ver: 1.0.0"
diff --git a/drivers/edac/highbank_l2_edac.c b/drivers/edac/highbank_l2_edac.c
index 2f193668ebc7..cd9a2bb7c548 100644
--- a/drivers/edac/highbank_l2_edac.c
+++ b/drivers/edac/highbank_l2_edac.c
@@ -21,7 +21,6 @@
 #include <linux/platform_device.h>
 #include <linux/of_platform.h>
 
-#include "edac_core.h"
 #include "edac_module.h"
 
 #define SR_CLR_SB_ECC_INTR	0x0
diff --git a/drivers/edac/highbank_mc_edac.c b/drivers/edac/highbank_mc_edac.c
index 11260cc3360e..0e7e0a404d89 100644
--- a/drivers/edac/highbank_mc_edac.c
+++ b/drivers/edac/highbank_mc_edac.c
@@ -22,7 +22,6 @@
 #include <linux/of_platform.h>
 #include <linux/uaccess.h>
 
-#include "edac_core.h"
 #include "edac_module.h"
 
 /* DDR Ctrlr Error Registers */
diff --git a/drivers/edac/i3000_edac.c b/drivers/edac/i3000_edac.c
index 5cb36a6022cc..5306240570d7 100644
--- a/drivers/edac/i3000_edac.c
+++ b/drivers/edac/i3000_edac.c
@@ -14,7 +14,7 @@
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
 #include <linux/edac.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define I3000_REVISION		"1.1"
 
diff --git a/drivers/edac/i3200_edac.c b/drivers/edac/i3200_edac.c
index 1f453382258a..77c58d201a30 100644
--- a/drivers/edac/i3200_edac.c
+++ b/drivers/edac/i3200_edac.c
@@ -13,7 +13,7 @@
 #include <linux/pci_ids.h>
 #include <linux/edac.h>
 #include <linux/io.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #include <linux/io-64-nonatomic-lo-hi.h>
 
diff --git a/drivers/edac/i5000_edac.c b/drivers/edac/i5000_edac.c
index 72e07e3cf718..1670d27bcac8 100644
--- a/drivers/edac/i5000_edac.c
+++ b/drivers/edac/i5000_edac.c
@@ -22,7 +22,7 @@
 #include <linux/edac.h>
 #include <asm/mmzone.h>
 
-#include "edac_core.h"
+#include "edac_module.h"
 
 /*
  * Alter this version for the I5000 module when modifications are made
diff --git a/drivers/edac/i5100_edac.c b/drivers/edac/i5100_edac.c
index c655162caf08..a8334c4acea7 100644
--- a/drivers/edac/i5100_edac.c
+++ b/drivers/edac/i5100_edac.c
@@ -29,7 +29,6 @@
 #include <linux/mmzone.h>
 #include <linux/debugfs.h>
 
-#include "edac_core.h"
 #include "edac_module.h"
 
 /* register addresses */
diff --git a/drivers/edac/i5400_edac.c b/drivers/edac/i5400_edac.c
index 6ef6ad1ba16e..abf6ef22e220 100644
--- a/drivers/edac/i5400_edac.c
+++ b/drivers/edac/i5400_edac.c
@@ -32,7 +32,7 @@
 #include <linux/edac.h>
 #include <linux/mmzone.h>
 
-#include "edac_core.h"
+#include "edac_module.h"
 
 /*
  * Alter this version for the I5400 module when modifications are made
diff --git a/drivers/edac/i7300_edac.c b/drivers/edac/i7300_edac.c
index dcac982fdc7a..0a912bf6de00 100644
--- a/drivers/edac/i7300_edac.c
+++ b/drivers/edac/i7300_edac.c
@@ -26,7 +26,7 @@
 #include <linux/edac.h>
 #include <linux/mmzone.h>
 
-#include "edac_core.h"
+#include "edac_module.h"
 
 /*
  * Alter this version for the I7300 module when modifications are made
diff --git a/drivers/edac/i7core_edac.c b/drivers/edac/i7core_edac.c
index 8a68a5e943ea..69b5adead0ad 100644
--- a/drivers/edac/i7core_edac.c
+++ b/drivers/edac/i7core_edac.c
@@ -39,7 +39,7 @@
 #include <asm/processor.h>
 #include <asm/div64.h>
 
-#include "edac_core.h"
+#include "edac_module.h"
 
 /* Static vars */
 static LIST_HEAD(i7core_edac_list);
diff --git a/drivers/edac/i82443bxgx_edac.c b/drivers/edac/i82443bxgx_edac.c
index 4d4110364f02..cb61a5b7d080 100644
--- a/drivers/edac/i82443bxgx_edac.c
+++ b/drivers/edac/i82443bxgx_edac.c
@@ -29,7 +29,7 @@
 
 
 #include <linux/edac.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define I82443_REVISION	"0.1"
 
diff --git a/drivers/edac/i82860_edac.c b/drivers/edac/i82860_edac.c
index ee1078cd3b96..236c813227fc 100644
--- a/drivers/edac/i82860_edac.c
+++ b/drivers/edac/i82860_edac.c
@@ -14,7 +14,7 @@
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
 #include <linux/edac.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define  I82860_REVISION " Ver: 2.0.2"
 #define EDAC_MOD_STR	"i82860_edac"
diff --git a/drivers/edac/i82875p_edac.c b/drivers/edac/i82875p_edac.c
index c26a513f8869..e286b7e74c7a 100644
--- a/drivers/edac/i82875p_edac.c
+++ b/drivers/edac/i82875p_edac.c
@@ -18,7 +18,7 @@
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
 #include <linux/edac.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define I82875P_REVISION	" Ver: 2.0.2"
 #define EDAC_MOD_STR		"i82875p_edac"
diff --git a/drivers/edac/i82975x_edac.c b/drivers/edac/i82975x_edac.c
index 35ab66c623a3..7baa8ace267b 100644
--- a/drivers/edac/i82975x_edac.c
+++ b/drivers/edac/i82975x_edac.c
@@ -14,7 +14,7 @@
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
 #include <linux/edac.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define I82975X_REVISION	" Ver: 1.0.0"
 #define EDAC_MOD_STR		"i82975x_edac"
diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c
index 1c88d9707495..2733fb5938a4 100644
--- a/drivers/edac/ie31200_edac.c
+++ b/drivers/edac/ie31200_edac.c
@@ -41,7 +41,7 @@
 #include <linux/edac.h>
 
 #include <linux/io-64-nonatomic-lo-hi.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define IE31200_REVISION "1.0"
 #define EDAC_MOD_STR "ie31200_edac"
diff --git a/drivers/edac/layerscape_edac.c b/drivers/edac/layerscape_edac.c
index 6c59d897ad12..94cac7686a56 100644
--- a/drivers/edac/layerscape_edac.c
+++ b/drivers/edac/layerscape_edac.c
@@ -16,7 +16,7 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include "edac_core.h"
+#include "edac_module.h"
 #include "fsl_ddr_edac.h"
 
 static const struct of_device_id fsl_ddr_mc_err_of_match[] = {
diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c
index 80762acd8cc8..34208f38c5b1 100644
--- a/drivers/edac/mce_amd.c
+++ b/drivers/edac/mce_amd.c
@@ -8,7 +8,7 @@ static struct amd_decoder_ops *fam_ops;
 static u8 xec_mask	 = 0xf;
 
 static bool report_gart_errors;
-static void (*nb_bus_decoder)(int node_id, struct mce *m);
+static void (*decode_dram_ecc)(int node_id, struct mce *m);
 
 void amd_report_gart_errors(bool v)
 {
@@ -18,16 +18,16 @@ EXPORT_SYMBOL_GPL(amd_report_gart_errors);
 
 void amd_register_ecc_decoder(void (*f)(int, struct mce *))
 {
-	nb_bus_decoder = f;
+	decode_dram_ecc = f;
 }
 EXPORT_SYMBOL_GPL(amd_register_ecc_decoder);
 
 void amd_unregister_ecc_decoder(void (*f)(int, struct mce *))
 {
-	if (nb_bus_decoder) {
-		WARN_ON(nb_bus_decoder != f);
+	if (decode_dram_ecc) {
+		WARN_ON(decode_dram_ecc != f);
 
-		nb_bus_decoder = NULL;
+		decode_dram_ecc = NULL;
 	}
 }
 EXPORT_SYMBOL_GPL(amd_unregister_ecc_decoder);
@@ -763,8 +763,8 @@ static void decode_mc4_mce(struct mce *m)
 
 			pr_cont("%s.\n", mc4_mce_desc[xec]);
 
-			if (nb_bus_decoder)
-				nb_bus_decoder(node_id, m);
+			if (decode_dram_ecc)
+				decode_dram_ecc(node_id, m);
 			return;
 		}
 		break;
@@ -877,6 +877,13 @@ static void decode_smca_errors(struct mce *m)
 		pr_emerg(HW_ERR "%s Error: ", ip_name);
 		pr_cont("%s.\n", smca_mce_descs[bank_type].descs[xec]);
 	}
+
+	/*
+	 * amd_get_nb_id() returns the last level cache id.
+	 * The last level cache on Fam17h is 1 level below the node.
+	 */
+	if (bank_type == SMCA_UMC && xec == 0 && decode_dram_ecc)
+		decode_dram_ecc(amd_get_nb_id(m->extcpu) >> 1, m);
 }
 
 static inline void amd_decode_err_code(u16 ec)
@@ -957,10 +964,13 @@ int amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data)
 		((m->status & MCI_STATUS_PCC)	? "PCC"	  : "-"),
 		((m->status & MCI_STATUS_ADDRV)	? "AddrV" : "-"));
 
-	if (c->x86 >= 0x15)
-		pr_cont("|%s|%s",
-			((m->status & MCI_STATUS_DEFERRED) ? "Deferred" : "-"),
-			((m->status & MCI_STATUS_POISON)   ? "Poison"   : "-"));
+	if (c->x86 >= 0x15) {
+		pr_cont("|%s", (m->status & MCI_STATUS_DEFERRED ? "Deferred" : "-"));
+
+		/* F15h, bank4, bit 43 is part of McaStatSubCache. */
+		if (c->x86 != 0x15 || m->bank != 4)
+			pr_cont("|%s", (m->status & MCI_STATUS_POISON ? "Poison" : "-"));
+	}
 
 	if (boot_cpu_has(X86_FEATURE_SMCA)) {
 		u32 low, high;
diff --git a/drivers/edac/mpc85xx_edac.c b/drivers/edac/mpc85xx_edac.c
index ff0567526ee3..8f66cbed70b7 100644
--- a/drivers/edac/mpc85xx_edac.c
+++ b/drivers/edac/mpc85xx_edac.c
@@ -25,7 +25,6 @@
 #include <linux/of_platform.h>
 #include <linux/of_device.h>
 #include "edac_module.h"
-#include "edac_core.h"
 #include "mpc85xx_edac.h"
 #include "fsl_ddr_edac.h"
 
@@ -300,6 +299,22 @@ err:
 	return res;
 }
 
+static int mpc85xx_pci_err_remove(struct platform_device *op)
+{
+	struct edac_pci_ctl_info *pci = dev_get_drvdata(&op->dev);
+	struct mpc85xx_pci_pdata *pdata = pci->pvt_info;
+
+	edac_dbg(0, "\n");
+
+	out_be32(pdata->pci_vbase + MPC85XX_PCI_ERR_ADDR, orig_pci_err_cap_dr);
+	out_be32(pdata->pci_vbase + MPC85XX_PCI_ERR_EN, orig_pci_err_en);
+
+	edac_pci_del_device(&op->dev);
+	edac_pci_free_ctl_info(pci);
+
+	return 0;
+}
+
 static const struct platform_device_id mpc85xx_pci_err_match[] = {
 	{
 		.name = "mpc85xx-pci-edac"
@@ -309,6 +324,7 @@ static const struct platform_device_id mpc85xx_pci_err_match[] = {
 
 static struct platform_driver mpc85xx_pci_err_driver = {
 	.probe = mpc85xx_pci_err_probe,
+	.remove = mpc85xx_pci_err_remove,
 	.id_table = mpc85xx_pci_err_match,
 	.driver = {
 		.name = "mpc85xx_pci_err",
diff --git a/drivers/edac/mv64x60_edac.c b/drivers/edac/mv64x60_edac.c
index cb9b8577acbc..14b7e7b71eaa 100644
--- a/drivers/edac/mv64x60_edac.c
+++ b/drivers/edac/mv64x60_edac.c
@@ -17,7 +17,6 @@
 #include <linux/edac.h>
 #include <linux/gfp.h>
 
-#include "edac_core.h"
 #include "edac_module.h"
 #include "mv64x60_edac.h"
 
diff --git a/drivers/edac/octeon_edac-l2c.c b/drivers/edac/octeon_edac-l2c.c
index afea7fc625cc..c33059e9b0be 100644
--- a/drivers/edac/octeon_edac-l2c.c
+++ b/drivers/edac/octeon_edac-l2c.c
@@ -16,7 +16,6 @@
 
 #include <asm/octeon/cvmx.h>
 
-#include "edac_core.h"
 #include "edac_module.h"
 
 #define EDAC_MOD_STR "octeon-l2c"
diff --git a/drivers/edac/octeon_edac-lmc.c b/drivers/edac/octeon_edac-lmc.c
index cda6dab5067a..9c1ffe3e912b 100644
--- a/drivers/edac/octeon_edac-lmc.c
+++ b/drivers/edac/octeon_edac-lmc.c
@@ -19,7 +19,6 @@
 #include <asm/octeon/octeon.h>
 #include <asm/octeon/cvmx-lmcx-defs.h>
 
-#include "edac_core.h"
 #include "edac_module.h"
 
 #define OCTEON_MAX_MC 4
diff --git a/drivers/edac/octeon_edac-pc.c b/drivers/edac/octeon_edac-pc.c
index 2ab6cf24c959..754eced59c32 100644
--- a/drivers/edac/octeon_edac-pc.c
+++ b/drivers/edac/octeon_edac-pc.c
@@ -15,7 +15,6 @@
 #include <linux/io.h>
 #include <linux/edac.h>
 
-#include "edac_core.h"
 #include "edac_module.h"
 
 #include <asm/octeon/cvmx.h>
diff --git a/drivers/edac/octeon_edac-pci.c b/drivers/edac/octeon_edac-pci.c
index 9ca73cec74e7..28b238eecefc 100644
--- a/drivers/edac/octeon_edac-pci.c
+++ b/drivers/edac/octeon_edac-pci.c
@@ -18,7 +18,6 @@
 #include <asm/octeon/cvmx-pci-defs.h>
 #include <asm/octeon/octeon.h>
 
-#include "edac_core.h"
 #include "edac_module.h"
 
 static void octeon_pci_poll(struct edac_pci_ctl_info *pci)
diff --git a/drivers/edac/pasemi_edac.c b/drivers/edac/pasemi_edac.c
index 9c971b575530..199f2c80480d 100644
--- a/drivers/edac/pasemi_edac.c
+++ b/drivers/edac/pasemi_edac.c
@@ -26,7 +26,7 @@
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
 #include <linux/edac.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define MODULE_NAME "pasemi_edac"
 
diff --git a/drivers/edac/ppc4xx_edac.c b/drivers/edac/ppc4xx_edac.c
index 691ce25e9010..e55e92590106 100644
--- a/drivers/edac/ppc4xx_edac.c
+++ b/drivers/edac/ppc4xx_edac.c
@@ -21,7 +21,7 @@
 
 #include <asm/dcr.h>
 
-#include "edac_core.h"
+#include "edac_module.h"
 #include "ppc4xx_edac.h"
 
 /*
diff --git a/drivers/edac/r82600_edac.c b/drivers/edac/r82600_edac.c
index 8f936bc7a010..978916625ced 100644
--- a/drivers/edac/r82600_edac.c
+++ b/drivers/edac/r82600_edac.c
@@ -20,7 +20,7 @@
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
 #include <linux/edac.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define R82600_REVISION	" Ver: 2.0.2"
 #define EDAC_MOD_STR	"r82600_edac"
diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c
index 54775221a01f..54ae6dc45ab2 100644
--- a/drivers/edac/sb_edac.c
+++ b/drivers/edac/sb_edac.c
@@ -23,10 +23,11 @@
 #include <linux/math64.h>
 #include <linux/mod_devicetable.h>
 #include <asm/cpu_device_id.h>
+#include <asm/intel-family.h>
 #include <asm/processor.h>
 #include <asm/mce.h>
 
-#include "edac_core.h"
+#include "edac_module.h"
 
 /* Static vars */
 static LIST_HEAD(sbridge_edac_list);
@@ -3365,12 +3366,13 @@ fail0:
 	{ X86_VENDOR_INTEL, 6, model, 0, (unsigned long)&table }
 
 static const struct x86_cpu_id sbridge_cpuids[] = {
-	ICPU(0x2d, pci_dev_descr_sbridge_table),	/* SANDY_BRIDGE */
-	ICPU(0x3e, pci_dev_descr_ibridge_table),	/* IVY_BRIDGE */
-	ICPU(0x3f, pci_dev_descr_haswell_table),	/* HASWELL */
-	ICPU(0x4f, pci_dev_descr_broadwell_table),	/* BROADWELL */
-	ICPU(0x56, pci_dev_descr_broadwell_table),	/* BROADWELL-DE */
-	ICPU(0x57, pci_dev_descr_knl_table),		/* KNIGHTS_LANDING */
+	ICPU(INTEL_FAM6_SANDYBRIDGE_X,	  pci_dev_descr_sbridge_table),
+	ICPU(INTEL_FAM6_IVYBRIDGE_X,	  pci_dev_descr_ibridge_table),
+	ICPU(INTEL_FAM6_HASWELL_X,	  pci_dev_descr_haswell_table),
+	ICPU(INTEL_FAM6_BROADWELL_X,	  pci_dev_descr_broadwell_table),
+	ICPU(INTEL_FAM6_BROADWELL_XEON_D, pci_dev_descr_broadwell_table),
+	ICPU(INTEL_FAM6_XEON_PHI_KNL,	  pci_dev_descr_knl_table),
+	ICPU(INTEL_FAM6_XEON_PHI_KNM,	  pci_dev_descr_knl_table),
 	{ }
 };
 MODULE_DEVICE_TABLE(x86cpu, sbridge_cpuids);
diff --git a/drivers/edac/skx_edac.c b/drivers/edac/skx_edac.c
index 0ff4878c2aa1..79ef675e4d6f 100644
--- a/drivers/edac/skx_edac.c
+++ b/drivers/edac/skx_edac.c
@@ -25,10 +25,11 @@
 #include <linux/math64.h>
 #include <linux/mod_devicetable.h>
 #include <asm/cpu_device_id.h>
+#include <asm/intel-family.h>
 #include <asm/processor.h>
 #include <asm/mce.h>
 
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define SKX_REVISION    " Ver: 1.0 "
 
@@ -262,8 +263,8 @@ fail:
 	return -ENODEV;
 }
 
-const struct x86_cpu_id skx_cpuids[] = {
-	{ X86_VENDOR_INTEL, 6, 0x55, 0, 0 },	/* Skylake */
+static const struct x86_cpu_id skx_cpuids[] = {
+	{ X86_VENDOR_INTEL, 6, INTEL_FAM6_SKYLAKE_X, 0, 0 },
 	{ }
 };
 MODULE_DEVICE_TABLE(x86cpu, skx_cpuids);
@@ -1036,7 +1037,7 @@ static void skx_remove(void)
  *	search for all the devices we need
  *	check which DIMMs are present.
  */
-int __init skx_init(void)
+static int __init skx_init(void)
 {
 	const struct x86_cpu_id *id;
 	const struct munit *m;
diff --git a/drivers/edac/synopsys_edac.c b/drivers/edac/synopsys_edac.c
index fc153aea2f6c..1c01dec78ec3 100644
--- a/drivers/edac/synopsys_edac.c
+++ b/drivers/edac/synopsys_edac.c
@@ -23,7 +23,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 
-#include "edac_core.h"
+#include "edac_module.h"
 
 /* Number of cs_rows needed per memory controller */
 #define SYNPS_EDAC_NR_CSROWS	1
diff --git a/drivers/edac/tile_edac.c b/drivers/edac/tile_edac.c
index 71381642ce2a..8a33a87e67f1 100644
--- a/drivers/edac/tile_edac.c
+++ b/drivers/edac/tile_edac.c
@@ -30,7 +30,7 @@
 #include <hv/hypervisor.h>
 #include <hv/drv_mshim_intf.h>
 
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define DRV_NAME	"tile-edac"
 
diff --git a/drivers/edac/x38_edac.c b/drivers/edac/x38_edac.c
index 314cf5cf268c..03c97a4bf590 100644
--- a/drivers/edac/x38_edac.c
+++ b/drivers/edac/x38_edac.c
@@ -16,7 +16,7 @@
 #include <linux/edac.h>
 
 #include <linux/io-64-nonatomic-lo-hi.h>
-#include "edac_core.h"
+#include "edac_module.h"
 
 #define X38_REVISION		"1.1"
 
diff --git a/drivers/edac/xgene_edac.c b/drivers/edac/xgene_edac.c
index bf19b6e3bd12..6c270d9d304a 100644
--- a/drivers/edac/xgene_edac.c
+++ b/drivers/edac/xgene_edac.c
@@ -28,7 +28,6 @@
 #include <linux/of_address.h>
 #include <linux/regmap.h>
 
-#include "edac_core.h"
 #include "edac_module.h"
 
 #define EDAC_MOD_STR			"xgene_edac"
@@ -1602,16 +1601,16 @@ static void xgene_edac_pa_report(struct edac_device_ctl_info *edac_dev)
 		dev_err(edac_dev->dev, "IOB PA read data RAM error\n");
 	if (reg & IOBPA_M_RDATA_CORRUPT_MASK)
 		dev_err(edac_dev->dev,
-			"Mutilple IOB PA read data RAM error\n");
+			"Multiple IOB PA read data RAM error\n");
 	if (reg & IOBPA_WDATA_CORRUPT_MASK)
 		dev_err(edac_dev->dev, "IOB PA write data RAM error\n");
 	if (reg & IOBPA_M_WDATA_CORRUPT_MASK)
 		dev_err(edac_dev->dev,
-			"Mutilple IOB PA write data RAM error\n");
+			"Multiple IOB PA write data RAM error\n");
 	if (reg & IOBPA_TRANS_CORRUPT_MASK)
 		dev_err(edac_dev->dev, "IOB PA transaction error\n");
 	if (reg & IOBPA_M_TRANS_CORRUPT_MASK)
-		dev_err(edac_dev->dev, "Mutilple IOB PA transaction error\n");
+		dev_err(edac_dev->dev, "Multiple IOB PA transaction error\n");
 	if (reg & IOBPA_REQIDRAM_CORRUPT_MASK)
 		dev_err(edac_dev->dev, "IOB PA transaction ID RAM error\n");
 	if (reg & IOBPA_M_REQIDRAM_CORRUPT_MASK)
diff --git a/drivers/extcon/extcon-arizona.c b/drivers/extcon/extcon-arizona.c
index 56e6c4c7c60d..d836d4ce5ee4 100644
--- a/drivers/extcon/extcon-arizona.c
+++ b/drivers/extcon/extcon-arizona.c
@@ -274,9 +274,10 @@ static void arizona_extcon_pulse_micbias(struct arizona_extcon_info *info)
 	struct arizona *arizona = info->arizona;
 	const char *widget = arizona_extcon_get_micbias(info);
 	struct snd_soc_dapm_context *dapm = arizona->dapm;
+	struct snd_soc_component *component = snd_soc_dapm_to_component(dapm);
 	int ret;
 
-	ret = snd_soc_dapm_force_enable_pin(dapm, widget);
+	ret = snd_soc_component_force_enable_pin(component, widget);
 	if (ret != 0)
 		dev_warn(arizona->dev, "Failed to enable %s: %d\n",
 			 widget, ret);
@@ -284,7 +285,7 @@ static void arizona_extcon_pulse_micbias(struct arizona_extcon_info *info)
 	snd_soc_dapm_sync(dapm);
 
 	if (!arizona->pdata.micd_force_micbias) {
-		ret = snd_soc_dapm_disable_pin(arizona->dapm, widget);
+		ret = snd_soc_component_disable_pin(component, widget);
 		if (ret != 0)
 			dev_warn(arizona->dev, "Failed to disable %s: %d\n",
 				 widget, ret);
@@ -349,6 +350,7 @@ static void arizona_stop_mic(struct arizona_extcon_info *info)
 	struct arizona *arizona = info->arizona;
 	const char *widget = arizona_extcon_get_micbias(info);
 	struct snd_soc_dapm_context *dapm = arizona->dapm;
+	struct snd_soc_component *component = snd_soc_dapm_to_component(dapm);
 	bool change;
 	int ret;
 
@@ -356,7 +358,7 @@ static void arizona_stop_mic(struct arizona_extcon_info *info)
 				 ARIZONA_MICD_ENA, 0,
 				 &change);
 
-	ret = snd_soc_dapm_disable_pin(dapm, widget);
+	ret = snd_soc_component_disable_pin(component, widget);
 	if (ret != 0)
 		dev_warn(arizona->dev,
 			 "Failed to disable %s: %d\n",
diff --git a/drivers/extcon/extcon-usb-gpio.c b/drivers/extcon/extcon-usb-gpio.c
index a27d350f69e3..d589c5feff3d 100644
--- a/drivers/extcon/extcon-usb-gpio.c
+++ b/drivers/extcon/extcon-usb-gpio.c
@@ -24,7 +24,6 @@
 #include <linux/module.h>
 #include <linux/of_gpio.h>
 #include <linux/platform_device.h>
-#include <linux/pm_wakeirq.h>
 #include <linux/slab.h>
 #include <linux/workqueue.h>
 #include <linux/acpi.h>
@@ -36,7 +35,9 @@ struct usb_extcon_info {
 	struct extcon_dev *edev;
 
 	struct gpio_desc *id_gpiod;
+	struct gpio_desc *vbus_gpiod;
 	int id_irq;
+	int vbus_irq;
 
 	unsigned long debounce_jiffies;
 	struct delayed_work wq_detcable;
@@ -48,31 +49,47 @@ static const unsigned int usb_extcon_cable[] = {
 	EXTCON_NONE,
 };
 
+/*
+ * "USB" = VBUS and "USB-HOST" = !ID, so we have:
+ * Both "USB" and "USB-HOST" can't be set as active at the
+ * same time so if "USB-HOST" is active (i.e. ID is 0)  we keep "USB" inactive
+ * even if VBUS is on.
+ *
+ *  State              |    ID   |   VBUS
+ * ----------------------------------------
+ *  [1] USB            |    H    |    H
+ *  [2] none           |    H    |    L
+ *  [3] USB-HOST       |    L    |    H
+ *  [4] USB-HOST       |    L    |    L
+ *
+ * In case we have only one of these signals:
+ * - VBUS only - we want to distinguish between [1] and [2], so ID is always 1.
+ * - ID only - we want to distinguish between [1] and [4], so VBUS = ID.
+*/
 static void usb_extcon_detect_cable(struct work_struct *work)
 {
-	int id;
+	int id, vbus;
 	struct usb_extcon_info *info = container_of(to_delayed_work(work),
 						    struct usb_extcon_info,
 						    wq_detcable);
 
-	/* check ID and update cable state */
-	id = gpiod_get_value_cansleep(info->id_gpiod);
-	if (id) {
-		/*
-		 * ID = 1 means USB HOST cable detached.
-		 * As we don't have event for USB peripheral cable attached,
-		 * we simulate USB peripheral attach here.
-		 */
+	/* check ID and VBUS and update cable state */
+	id = info->id_gpiod ?
+		gpiod_get_value_cansleep(info->id_gpiod) : 1;
+	vbus = info->vbus_gpiod ?
+		gpiod_get_value_cansleep(info->vbus_gpiod) : id;
+
+	/* at first we clean states which are no longer active */
+	if (id)
 		extcon_set_state_sync(info->edev, EXTCON_USB_HOST, false);
-		extcon_set_state_sync(info->edev, EXTCON_USB, true);
-	} else {
-		/*
-		 * ID = 0 means USB HOST cable attached.
-		 * As we don't have event for USB peripheral cable detached,
-		 * we simulate USB peripheral detach here.
-		 */
+	if (!vbus)
 		extcon_set_state_sync(info->edev, EXTCON_USB, false);
+
+	if (!id) {
 		extcon_set_state_sync(info->edev, EXTCON_USB_HOST, true);
+	} else {
+		if (vbus)
+			extcon_set_state_sync(info->edev, EXTCON_USB, true);
 	}
 }
 
@@ -101,12 +118,21 @@ static int usb_extcon_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	info->dev = dev;
-	info->id_gpiod = devm_gpiod_get(&pdev->dev, "id", GPIOD_IN);
-	if (IS_ERR(info->id_gpiod)) {
-		dev_err(dev, "failed to get ID GPIO\n");
-		return PTR_ERR(info->id_gpiod);
+	info->id_gpiod = devm_gpiod_get_optional(&pdev->dev, "id", GPIOD_IN);
+	info->vbus_gpiod = devm_gpiod_get_optional(&pdev->dev, "vbus",
+						   GPIOD_IN);
+
+	if (!info->id_gpiod && !info->vbus_gpiod) {
+		dev_err(dev, "failed to get gpios\n");
+		return -ENODEV;
 	}
 
+	if (IS_ERR(info->id_gpiod))
+		return PTR_ERR(info->id_gpiod);
+
+	if (IS_ERR(info->vbus_gpiod))
+		return PTR_ERR(info->vbus_gpiod);
+
 	info->edev = devm_extcon_dev_allocate(dev, usb_extcon_cable);
 	if (IS_ERR(info->edev)) {
 		dev_err(dev, "failed to allocate extcon device\n");
@@ -119,32 +145,56 @@ static int usb_extcon_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	ret = gpiod_set_debounce(info->id_gpiod,
-				 USB_GPIO_DEBOUNCE_MS * 1000);
+	if (info->id_gpiod)
+		ret = gpiod_set_debounce(info->id_gpiod,
+					 USB_GPIO_DEBOUNCE_MS * 1000);
+	if (!ret && info->vbus_gpiod)
+		ret = gpiod_set_debounce(info->vbus_gpiod,
+					 USB_GPIO_DEBOUNCE_MS * 1000);
+
 	if (ret < 0)
 		info->debounce_jiffies = msecs_to_jiffies(USB_GPIO_DEBOUNCE_MS);
 
 	INIT_DELAYED_WORK(&info->wq_detcable, usb_extcon_detect_cable);
 
-	info->id_irq = gpiod_to_irq(info->id_gpiod);
-	if (info->id_irq < 0) {
-		dev_err(dev, "failed to get ID IRQ\n");
-		return info->id_irq;
+	if (info->id_gpiod) {
+		info->id_irq = gpiod_to_irq(info->id_gpiod);
+		if (info->id_irq < 0) {
+			dev_err(dev, "failed to get ID IRQ\n");
+			return info->id_irq;
+		}
+
+		ret = devm_request_threaded_irq(dev, info->id_irq, NULL,
+						usb_irq_handler,
+						IRQF_TRIGGER_RISING |
+						IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+						pdev->name, info);
+		if (ret < 0) {
+			dev_err(dev, "failed to request handler for ID IRQ\n");
+			return ret;
+		}
 	}
 
-	ret = devm_request_threaded_irq(dev, info->id_irq, NULL,
-					usb_irq_handler,
-					IRQF_TRIGGER_RISING |
-					IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
-					pdev->name, info);
-	if (ret < 0) {
-		dev_err(dev, "failed to request handler for ID IRQ\n");
-		return ret;
+	if (info->vbus_gpiod) {
+		info->vbus_irq = gpiod_to_irq(info->vbus_gpiod);
+		if (info->vbus_irq < 0) {
+			dev_err(dev, "failed to get VBUS IRQ\n");
+			return info->vbus_irq;
+		}
+
+		ret = devm_request_threaded_irq(dev, info->vbus_irq, NULL,
+						usb_irq_handler,
+						IRQF_TRIGGER_RISING |
+						IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+						pdev->name, info);
+		if (ret < 0) {
+			dev_err(dev, "failed to request handler for VBUS IRQ\n");
+			return ret;
+		}
 	}
 
 	platform_set_drvdata(pdev, info);
 	device_init_wakeup(dev, true);
-	dev_pm_set_wake_irq(dev, info->id_irq);
 
 	/* Perform initial detection */
 	usb_extcon_detect_cable(&info->wq_detcable.work);
@@ -157,8 +207,6 @@ static int usb_extcon_remove(struct platform_device *pdev)
 	struct usb_extcon_info *info = platform_get_drvdata(pdev);
 
 	cancel_delayed_work_sync(&info->wq_detcable);
-
-	dev_pm_clear_wake_irq(&pdev->dev);
 	device_init_wakeup(&pdev->dev, false);
 
 	return 0;
@@ -170,12 +218,32 @@ static int usb_extcon_suspend(struct device *dev)
 	struct usb_extcon_info *info = dev_get_drvdata(dev);
 	int ret = 0;
 
+	if (device_may_wakeup(dev)) {
+		if (info->id_gpiod) {
+			ret = enable_irq_wake(info->id_irq);
+			if (ret)
+				return ret;
+		}
+		if (info->vbus_gpiod) {
+			ret = enable_irq_wake(info->vbus_irq);
+			if (ret) {
+				if (info->id_gpiod)
+					disable_irq_wake(info->id_irq);
+
+				return ret;
+			}
+		}
+	}
+
 	/*
 	 * We don't want to process any IRQs after this point
 	 * as GPIOs used behind I2C subsystem might not be
 	 * accessible until resume completes. So disable IRQ.
 	 */
-	disable_irq(info->id_irq);
+	if (info->id_gpiod)
+		disable_irq(info->id_irq);
+	if (info->vbus_gpiod)
+		disable_irq(info->vbus_irq);
 
 	return ret;
 }
@@ -185,7 +253,28 @@ static int usb_extcon_resume(struct device *dev)
 	struct usb_extcon_info *info = dev_get_drvdata(dev);
 	int ret = 0;
 
-	enable_irq(info->id_irq);
+	if (device_may_wakeup(dev)) {
+		if (info->id_gpiod) {
+			ret = disable_irq_wake(info->id_irq);
+			if (ret)
+				return ret;
+		}
+		if (info->vbus_gpiod) {
+			ret = disable_irq_wake(info->vbus_irq);
+			if (ret) {
+				if (info->id_gpiod)
+					enable_irq_wake(info->id_irq);
+
+				return ret;
+			}
+		}
+	}
+
+	if (info->id_gpiod)
+		enable_irq(info->id_irq);
+	if (info->vbus_gpiod)
+		enable_irq(info->vbus_irq);
+
 	if (!device_may_wakeup(dev))
 		queue_delayed_work(system_power_efficient_wq,
 				   &info->wq_detcable, 0);
diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig
index bca172d42c74..1867f0d1389b 100644
--- a/drivers/firmware/Kconfig
+++ b/drivers/firmware/Kconfig
@@ -8,6 +8,17 @@ menu "Firmware Drivers"
 config ARM_PSCI_FW
 	bool
 
+config ARM_PSCI_CHECKER
+	bool "ARM PSCI checker"
+	depends on ARM_PSCI_FW && HOTPLUG_CPU && !TORTURE_TEST
+	help
+	  Run the PSCI checker during startup. This checks that hotplug and
+	  suspend operations work correctly when using PSCI.
+
+	  The torture tests may interfere with the PSCI checker by turning CPUs
+	  on and off through hotplug, so for now torture tests and PSCI checker
+	  are mutually exclusive.
+
 config ARM_SCPI_PROTOCOL
 	tristate "ARM System Control and Power Interface (SCPI) Message Protocol"
 	depends on MAILBOX
@@ -203,6 +214,21 @@ config QCOM_SCM_64
 	def_bool y
 	depends on QCOM_SCM && ARM64
 
+config TI_SCI_PROTOCOL
+	tristate "TI System Control Interface (TISCI) Message Protocol"
+	depends on TI_MESSAGE_MANAGER
+	help
+	  TI System Control Interface (TISCI) Message Protocol is used to manage
+	  compute systems such as ARM, DSP etc with the system controller in
+	  complex System on Chip(SoC) such as those found on certain keystone
+	  generation SoC from TI.
+
+	  System controller provides various facilities including power
+	  management function support.
+
+	  This protocol library is used by client drivers to use the features
+	  provided by the system controller.
+
 config HAVE_ARM_SMCCC
 	bool
 
@@ -210,5 +236,6 @@ source "drivers/firmware/broadcom/Kconfig"
 source "drivers/firmware/google/Kconfig"
 source "drivers/firmware/efi/Kconfig"
 source "drivers/firmware/meson/Kconfig"
+source "drivers/firmware/tegra/Kconfig"
 
 endmenu
diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile
index 898ac41fa8b3..a37f12e8d137 100644
--- a/drivers/firmware/Makefile
+++ b/drivers/firmware/Makefile
@@ -2,6 +2,7 @@
 # Makefile for the linux kernel.
 #
 obj-$(CONFIG_ARM_PSCI_FW)	+= psci.o
+obj-$(CONFIG_ARM_PSCI_CHECKER)	+= psci_checker.o
 obj-$(CONFIG_ARM_SCPI_PROTOCOL)	+= arm_scpi.o
 obj-$(CONFIG_ARM_SCPI_POWER_DOMAIN) += scpi_pm_domain.o
 obj-$(CONFIG_DMI)		+= dmi_scan.o
@@ -20,9 +21,11 @@ obj-$(CONFIG_QCOM_SCM)		+= qcom_scm.o
 obj-$(CONFIG_QCOM_SCM_64)	+= qcom_scm-64.o
 obj-$(CONFIG_QCOM_SCM_32)	+= qcom_scm-32.o
 CFLAGS_qcom_scm-32.o :=$(call as-instr,.arch armv7-a\n.arch_extension sec,-DREQUIRES_SEC=1) -march=armv7-a
+obj-$(CONFIG_TI_SCI_PROTOCOL)	+= ti_sci.o
 
 obj-y				+= broadcom/
 obj-y				+= meson/
 obj-$(CONFIG_GOOGLE_FIRMWARE)	+= google/
 obj-$(CONFIG_EFI)		+= efi/
 obj-$(CONFIG_UEFI_CPER)		+= efi/
+obj-y				+= tegra/
diff --git a/drivers/firmware/arm_scpi.c b/drivers/firmware/arm_scpi.c
index ce2bc2a38101..70e13230d8db 100644
--- a/drivers/firmware/arm_scpi.c
+++ b/drivers/firmware/arm_scpi.c
@@ -50,20 +50,27 @@
 #define CMD_TOKEN_ID_MASK	0xff
 #define CMD_DATA_SIZE_SHIFT	16
 #define CMD_DATA_SIZE_MASK	0x1ff
+#define CMD_LEGACY_DATA_SIZE_SHIFT	20
+#define CMD_LEGACY_DATA_SIZE_MASK	0x1ff
 #define PACK_SCPI_CMD(cmd_id, tx_sz)			\
 	((((cmd_id) & CMD_ID_MASK) << CMD_ID_SHIFT) |	\
 	(((tx_sz) & CMD_DATA_SIZE_MASK) << CMD_DATA_SIZE_SHIFT))
 #define ADD_SCPI_TOKEN(cmd, token)			\
 	((cmd) |= (((token) & CMD_TOKEN_ID_MASK) << CMD_TOKEN_ID_SHIFT))
+#define PACK_LEGACY_SCPI_CMD(cmd_id, tx_sz)				\
+	((((cmd_id) & CMD_ID_MASK) << CMD_ID_SHIFT) |			       \
+	(((tx_sz) & CMD_LEGACY_DATA_SIZE_MASK) << CMD_LEGACY_DATA_SIZE_SHIFT))
 
 #define CMD_SIZE(cmd)	(((cmd) >> CMD_DATA_SIZE_SHIFT) & CMD_DATA_SIZE_MASK)
+#define CMD_LEGACY_SIZE(cmd)	(((cmd) >> CMD_LEGACY_DATA_SIZE_SHIFT) & \
+					CMD_LEGACY_DATA_SIZE_MASK)
 #define CMD_UNIQ_MASK	(CMD_TOKEN_ID_MASK << CMD_TOKEN_ID_SHIFT | CMD_ID_MASK)
 #define CMD_XTRACT_UNIQ(cmd)	((cmd) & CMD_UNIQ_MASK)
 
 #define SCPI_SLOT		0
 
 #define MAX_DVFS_DOMAINS	8
-#define MAX_DVFS_OPPS		8
+#define MAX_DVFS_OPPS		16
 #define DVFS_LATENCY(hdr)	(le32_to_cpu(hdr) >> 16)
 #define DVFS_OPP_COUNT(hdr)	((le32_to_cpu(hdr) >> 8) & 0xff)
 
@@ -99,6 +106,7 @@ enum scpi_error_codes {
 	SCPI_ERR_MAX
 };
 
+/* SCPI Standard commands */
 enum scpi_std_cmd {
 	SCPI_CMD_INVALID		= 0x00,
 	SCPI_CMD_SCPI_READY		= 0x01,
@@ -132,6 +140,108 @@ enum scpi_std_cmd {
 	SCPI_CMD_COUNT
 };
 
+/* SCPI Legacy Commands */
+enum legacy_scpi_std_cmd {
+	LEGACY_SCPI_CMD_INVALID			= 0x00,
+	LEGACY_SCPI_CMD_SCPI_READY		= 0x01,
+	LEGACY_SCPI_CMD_SCPI_CAPABILITIES	= 0x02,
+	LEGACY_SCPI_CMD_EVENT			= 0x03,
+	LEGACY_SCPI_CMD_SET_CSS_PWR_STATE	= 0x04,
+	LEGACY_SCPI_CMD_GET_CSS_PWR_STATE	= 0x05,
+	LEGACY_SCPI_CMD_CFG_PWR_STATE_STAT	= 0x06,
+	LEGACY_SCPI_CMD_GET_PWR_STATE_STAT	= 0x07,
+	LEGACY_SCPI_CMD_SYS_PWR_STATE		= 0x08,
+	LEGACY_SCPI_CMD_L2_READY		= 0x09,
+	LEGACY_SCPI_CMD_SET_AP_TIMER		= 0x0a,
+	LEGACY_SCPI_CMD_CANCEL_AP_TIME		= 0x0b,
+	LEGACY_SCPI_CMD_DVFS_CAPABILITIES	= 0x0c,
+	LEGACY_SCPI_CMD_GET_DVFS_INFO		= 0x0d,
+	LEGACY_SCPI_CMD_SET_DVFS		= 0x0e,
+	LEGACY_SCPI_CMD_GET_DVFS		= 0x0f,
+	LEGACY_SCPI_CMD_GET_DVFS_STAT		= 0x10,
+	LEGACY_SCPI_CMD_SET_RTC			= 0x11,
+	LEGACY_SCPI_CMD_GET_RTC			= 0x12,
+	LEGACY_SCPI_CMD_CLOCK_CAPABILITIES	= 0x13,
+	LEGACY_SCPI_CMD_SET_CLOCK_INDEX		= 0x14,
+	LEGACY_SCPI_CMD_SET_CLOCK_VALUE		= 0x15,
+	LEGACY_SCPI_CMD_GET_CLOCK_VALUE		= 0x16,
+	LEGACY_SCPI_CMD_PSU_CAPABILITIES	= 0x17,
+	LEGACY_SCPI_CMD_SET_PSU			= 0x18,
+	LEGACY_SCPI_CMD_GET_PSU			= 0x19,
+	LEGACY_SCPI_CMD_SENSOR_CAPABILITIES	= 0x1a,
+	LEGACY_SCPI_CMD_SENSOR_INFO		= 0x1b,
+	LEGACY_SCPI_CMD_SENSOR_VALUE		= 0x1c,
+	LEGACY_SCPI_CMD_SENSOR_CFG_PERIODIC	= 0x1d,
+	LEGACY_SCPI_CMD_SENSOR_CFG_BOUNDS	= 0x1e,
+	LEGACY_SCPI_CMD_SENSOR_ASYNC_VALUE	= 0x1f,
+	LEGACY_SCPI_CMD_COUNT
+};
+
+/* List all commands that are required to go through the high priority link */
+static int legacy_hpriority_cmds[] = {
+	LEGACY_SCPI_CMD_GET_CSS_PWR_STATE,
+	LEGACY_SCPI_CMD_CFG_PWR_STATE_STAT,
+	LEGACY_SCPI_CMD_GET_PWR_STATE_STAT,
+	LEGACY_SCPI_CMD_SET_DVFS,
+	LEGACY_SCPI_CMD_GET_DVFS,
+	LEGACY_SCPI_CMD_SET_RTC,
+	LEGACY_SCPI_CMD_GET_RTC,
+	LEGACY_SCPI_CMD_SET_CLOCK_INDEX,
+	LEGACY_SCPI_CMD_SET_CLOCK_VALUE,
+	LEGACY_SCPI_CMD_GET_CLOCK_VALUE,
+	LEGACY_SCPI_CMD_SET_PSU,
+	LEGACY_SCPI_CMD_GET_PSU,
+	LEGACY_SCPI_CMD_SENSOR_CFG_PERIODIC,
+	LEGACY_SCPI_CMD_SENSOR_CFG_BOUNDS,
+};
+
+/* List all commands used by this driver, used as indexes */
+enum scpi_drv_cmds {
+	CMD_SCPI_CAPABILITIES = 0,
+	CMD_GET_CLOCK_INFO,
+	CMD_GET_CLOCK_VALUE,
+	CMD_SET_CLOCK_VALUE,
+	CMD_GET_DVFS,
+	CMD_SET_DVFS,
+	CMD_GET_DVFS_INFO,
+	CMD_SENSOR_CAPABILITIES,
+	CMD_SENSOR_INFO,
+	CMD_SENSOR_VALUE,
+	CMD_SET_DEVICE_PWR_STATE,
+	CMD_GET_DEVICE_PWR_STATE,
+	CMD_MAX_COUNT,
+};
+
+static int scpi_std_commands[CMD_MAX_COUNT] = {
+	SCPI_CMD_SCPI_CAPABILITIES,
+	SCPI_CMD_GET_CLOCK_INFO,
+	SCPI_CMD_GET_CLOCK_VALUE,
+	SCPI_CMD_SET_CLOCK_VALUE,
+	SCPI_CMD_GET_DVFS,
+	SCPI_CMD_SET_DVFS,
+	SCPI_CMD_GET_DVFS_INFO,
+	SCPI_CMD_SENSOR_CAPABILITIES,
+	SCPI_CMD_SENSOR_INFO,
+	SCPI_CMD_SENSOR_VALUE,
+	SCPI_CMD_SET_DEVICE_PWR_STATE,
+	SCPI_CMD_GET_DEVICE_PWR_STATE,
+};
+
+static int scpi_legacy_commands[CMD_MAX_COUNT] = {
+	LEGACY_SCPI_CMD_SCPI_CAPABILITIES,
+	-1, /* GET_CLOCK_INFO */
+	LEGACY_SCPI_CMD_GET_CLOCK_VALUE,
+	LEGACY_SCPI_CMD_SET_CLOCK_VALUE,
+	LEGACY_SCPI_CMD_GET_DVFS,
+	LEGACY_SCPI_CMD_SET_DVFS,
+	LEGACY_SCPI_CMD_GET_DVFS_INFO,
+	LEGACY_SCPI_CMD_SENSOR_CAPABILITIES,
+	LEGACY_SCPI_CMD_SENSOR_INFO,
+	LEGACY_SCPI_CMD_SENSOR_VALUE,
+	-1, /* SET_DEVICE_PWR_STATE */
+	-1, /* GET_DEVICE_PWR_STATE */
+};
+
 struct scpi_xfer {
 	u32 slot; /* has to be first element */
 	u32 cmd;
@@ -160,7 +270,10 @@ struct scpi_chan {
 struct scpi_drvinfo {
 	u32 protocol_version;
 	u32 firmware_version;
+	bool is_legacy;
 	int num_chans;
+	int *commands;
+	DECLARE_BITMAP(cmd_priority, LEGACY_SCPI_CMD_COUNT);
 	atomic_t next_chan;
 	struct scpi_ops *scpi_ops;
 	struct scpi_chan *channels;
@@ -177,6 +290,11 @@ struct scpi_shared_mem {
 	u8 payload[0];
 } __packed;
 
+struct legacy_scpi_shared_mem {
+	__le32 status;
+	u8 payload[0];
+} __packed;
+
 struct scp_capabilities {
 	__le32 protocol_version;
 	__le32 event_version;
@@ -202,6 +320,12 @@ struct clk_set_value {
 	__le32 rate;
 } __packed;
 
+struct legacy_clk_set_value {
+	__le32 rate;
+	__le16 id;
+	__le16 reserved;
+} __packed;
+
 struct dvfs_info {
 	__le32 header;
 	struct {
@@ -273,19 +397,43 @@ static void scpi_process_cmd(struct scpi_chan *ch, u32 cmd)
 		return;
 	}
 
-	list_for_each_entry(t, &ch->rx_pending, node)
-		if (CMD_XTRACT_UNIQ(t->cmd) == CMD_XTRACT_UNIQ(cmd)) {
-			list_del(&t->node);
-			match = t;
-			break;
-		}
+	/* Command type is not replied by the SCP Firmware in legacy Mode
+	 * We should consider that command is the head of pending RX commands
+	 * if the list is not empty. In TX only mode, the list would be empty.
+	 */
+	if (scpi_info->is_legacy) {
+		match = list_first_entry(&ch->rx_pending, struct scpi_xfer,
+					 node);
+		list_del(&match->node);
+	} else {
+		list_for_each_entry(t, &ch->rx_pending, node)
+			if (CMD_XTRACT_UNIQ(t->cmd) == CMD_XTRACT_UNIQ(cmd)) {
+				list_del(&t->node);
+				match = t;
+				break;
+			}
+	}
 	/* check if wait_for_completion is in progress or timed-out */
 	if (match && !completion_done(&match->done)) {
-		struct scpi_shared_mem *mem = ch->rx_payload;
-		unsigned int len = min(match->rx_len, CMD_SIZE(cmd));
+		unsigned int len;
+
+		if (scpi_info->is_legacy) {
+			struct legacy_scpi_shared_mem *mem = ch->rx_payload;
+
+			/* RX Length is not replied by the legacy Firmware */
+			len = match->rx_len;
+
+			match->status = le32_to_cpu(mem->status);
+			memcpy_fromio(match->rx_buf, mem->payload, len);
+		} else {
+			struct scpi_shared_mem *mem = ch->rx_payload;
+
+			len = min(match->rx_len, CMD_SIZE(cmd));
+
+			match->status = le32_to_cpu(mem->status);
+			memcpy_fromio(match->rx_buf, mem->payload, len);
+		}
 
-		match->status = le32_to_cpu(mem->status);
-		memcpy_fromio(match->rx_buf, mem->payload, len);
 		if (match->rx_len > len)
 			memset(match->rx_buf + len, 0, match->rx_len - len);
 		complete(&match->done);
@@ -297,7 +445,10 @@ static void scpi_handle_remote_msg(struct mbox_client *c, void *msg)
 {
 	struct scpi_chan *ch = container_of(c, struct scpi_chan, cl);
 	struct scpi_shared_mem *mem = ch->rx_payload;
-	u32 cmd = le32_to_cpu(mem->command);
+	u32 cmd = 0;
+
+	if (!scpi_info->is_legacy)
+		cmd = le32_to_cpu(mem->command);
 
 	scpi_process_cmd(ch, cmd);
 }
@@ -309,8 +460,13 @@ static void scpi_tx_prepare(struct mbox_client *c, void *msg)
 	struct scpi_chan *ch = container_of(c, struct scpi_chan, cl);
 	struct scpi_shared_mem *mem = (struct scpi_shared_mem *)ch->tx_payload;
 
-	if (t->tx_buf)
-		memcpy_toio(mem->payload, t->tx_buf, t->tx_len);
+	if (t->tx_buf) {
+		if (scpi_info->is_legacy)
+			memcpy_toio(ch->tx_payload, t->tx_buf, t->tx_len);
+		else
+			memcpy_toio(mem->payload, t->tx_buf, t->tx_len);
+	}
+
 	if (t->rx_buf) {
 		if (!(++ch->token))
 			++ch->token;
@@ -319,7 +475,9 @@ static void scpi_tx_prepare(struct mbox_client *c, void *msg)
 		list_add_tail(&t->node, &ch->rx_pending);
 		spin_unlock_irqrestore(&ch->rx_lock, flags);
 	}
-	mem->command = cpu_to_le32(t->cmd);
+
+	if (!scpi_info->is_legacy)
+		mem->command = cpu_to_le32(t->cmd);
 }
 
 static struct scpi_xfer *get_scpi_xfer(struct scpi_chan *ch)
@@ -344,23 +502,38 @@ static void put_scpi_xfer(struct scpi_xfer *t, struct scpi_chan *ch)
 	mutex_unlock(&ch->xfers_lock);
 }
 
-static int scpi_send_message(u8 cmd, void *tx_buf, unsigned int tx_len,
+static int scpi_send_message(u8 idx, void *tx_buf, unsigned int tx_len,
 			     void *rx_buf, unsigned int rx_len)
 {
 	int ret;
 	u8 chan;
+	u8 cmd;
 	struct scpi_xfer *msg;
 	struct scpi_chan *scpi_chan;
 
-	chan = atomic_inc_return(&scpi_info->next_chan) % scpi_info->num_chans;
+	if (scpi_info->commands[idx] < 0)
+		return -EOPNOTSUPP;
+
+	cmd = scpi_info->commands[idx];
+
+	if (scpi_info->is_legacy)
+		chan = test_bit(cmd, scpi_info->cmd_priority) ? 1 : 0;
+	else
+		chan = atomic_inc_return(&scpi_info->next_chan) %
+			scpi_info->num_chans;
 	scpi_chan = scpi_info->channels + chan;
 
 	msg = get_scpi_xfer(scpi_chan);
 	if (!msg)
 		return -ENOMEM;
 
-	msg->slot = BIT(SCPI_SLOT);
-	msg->cmd = PACK_SCPI_CMD(cmd, tx_len);
+	if (scpi_info->is_legacy) {
+		msg->cmd = PACK_LEGACY_SCPI_CMD(cmd, tx_len);
+		msg->slot = msg->cmd;
+	} else {
+		msg->slot = BIT(SCPI_SLOT);
+		msg->cmd = PACK_SCPI_CMD(cmd, tx_len);
+	}
 	msg->tx_buf = tx_buf;
 	msg->tx_len = tx_len;
 	msg->rx_buf = rx_buf;
@@ -397,7 +570,7 @@ scpi_clk_get_range(u16 clk_id, unsigned long *min, unsigned long *max)
 	struct clk_get_info clk;
 	__le16 le_clk_id = cpu_to_le16(clk_id);
 
-	ret = scpi_send_message(SCPI_CMD_GET_CLOCK_INFO, &le_clk_id,
+	ret = scpi_send_message(CMD_GET_CLOCK_INFO, &le_clk_id,
 				sizeof(le_clk_id), &clk, sizeof(clk));
 	if (!ret) {
 		*min = le32_to_cpu(clk.min_rate);
@@ -412,8 +585,9 @@ static unsigned long scpi_clk_get_val(u16 clk_id)
 	struct clk_get_value clk;
 	__le16 le_clk_id = cpu_to_le16(clk_id);
 
-	ret = scpi_send_message(SCPI_CMD_GET_CLOCK_VALUE, &le_clk_id,
+	ret = scpi_send_message(CMD_GET_CLOCK_VALUE, &le_clk_id,
 				sizeof(le_clk_id), &clk, sizeof(clk));
+
 	return ret ? ret : le32_to_cpu(clk.rate);
 }
 
@@ -425,7 +599,19 @@ static int scpi_clk_set_val(u16 clk_id, unsigned long rate)
 		.rate = cpu_to_le32(rate)
 	};
 
-	return scpi_send_message(SCPI_CMD_SET_CLOCK_VALUE, &clk, sizeof(clk),
+	return scpi_send_message(CMD_SET_CLOCK_VALUE, &clk, sizeof(clk),
+				 &stat, sizeof(stat));
+}
+
+static int legacy_scpi_clk_set_val(u16 clk_id, unsigned long rate)
+{
+	int stat;
+	struct legacy_clk_set_value clk = {
+		.id = cpu_to_le16(clk_id),
+		.rate = cpu_to_le32(rate)
+	};
+
+	return scpi_send_message(CMD_SET_CLOCK_VALUE, &clk, sizeof(clk),
 				 &stat, sizeof(stat));
 }
 
@@ -434,8 +620,9 @@ static int scpi_dvfs_get_idx(u8 domain)
 	int ret;
 	u8 dvfs_idx;
 
-	ret = scpi_send_message(SCPI_CMD_GET_DVFS, &domain, sizeof(domain),
+	ret = scpi_send_message(CMD_GET_DVFS, &domain, sizeof(domain),
 				&dvfs_idx, sizeof(dvfs_idx));
+
 	return ret ? ret : dvfs_idx;
 }
 
@@ -444,7 +631,7 @@ static int scpi_dvfs_set_idx(u8 domain, u8 index)
 	int stat;
 	struct dvfs_set dvfs = {domain, index};
 
-	return scpi_send_message(SCPI_CMD_SET_DVFS, &dvfs, sizeof(dvfs),
+	return scpi_send_message(CMD_SET_DVFS, &dvfs, sizeof(dvfs),
 				 &stat, sizeof(stat));
 }
 
@@ -468,9 +655,8 @@ static struct scpi_dvfs_info *scpi_dvfs_get_info(u8 domain)
 	if (scpi_info->dvfs[domain])	/* data already populated */
 		return scpi_info->dvfs[domain];
 
-	ret = scpi_send_message(SCPI_CMD_GET_DVFS_INFO, &domain, sizeof(domain),
+	ret = scpi_send_message(CMD_GET_DVFS_INFO, &domain, sizeof(domain),
 				&buf, sizeof(buf));
-
 	if (ret)
 		return ERR_PTR(ret);
 
@@ -503,7 +689,7 @@ static int scpi_sensor_get_capability(u16 *sensors)
 	struct sensor_capabilities cap_buf;
 	int ret;
 
-	ret = scpi_send_message(SCPI_CMD_SENSOR_CAPABILITIES, NULL, 0, &cap_buf,
+	ret = scpi_send_message(CMD_SENSOR_CAPABILITIES, NULL, 0, &cap_buf,
 				sizeof(cap_buf));
 	if (!ret)
 		*sensors = le16_to_cpu(cap_buf.sensors);
@@ -517,7 +703,7 @@ static int scpi_sensor_get_info(u16 sensor_id, struct scpi_sensor_info *info)
 	struct _scpi_sensor_info _info;
 	int ret;
 
-	ret = scpi_send_message(SCPI_CMD_SENSOR_INFO, &id, sizeof(id),
+	ret = scpi_send_message(CMD_SENSOR_INFO, &id, sizeof(id),
 				&_info, sizeof(_info));
 	if (!ret) {
 		memcpy(info, &_info, sizeof(*info));
@@ -533,7 +719,7 @@ static int scpi_sensor_get_value(u16 sensor, u64 *val)
 	struct sensor_value buf;
 	int ret;
 
-	ret = scpi_send_message(SCPI_CMD_SENSOR_VALUE, &id, sizeof(id),
+	ret = scpi_send_message(CMD_SENSOR_VALUE, &id, sizeof(id),
 				&buf, sizeof(buf));
 	if (!ret)
 		*val = (u64)le32_to_cpu(buf.hi_val) << 32 |
@@ -548,7 +734,7 @@ static int scpi_device_get_power_state(u16 dev_id)
 	u8 pstate;
 	__le16 id = cpu_to_le16(dev_id);
 
-	ret = scpi_send_message(SCPI_CMD_GET_DEVICE_PWR_STATE, &id,
+	ret = scpi_send_message(CMD_GET_DEVICE_PWR_STATE, &id,
 				sizeof(id), &pstate, sizeof(pstate));
 	return ret ? ret : pstate;
 }
@@ -561,7 +747,7 @@ static int scpi_device_set_power_state(u16 dev_id, u8 pstate)
 		.pstate = pstate,
 	};
 
-	return scpi_send_message(SCPI_CMD_SET_DEVICE_PWR_STATE, &dev_set,
+	return scpi_send_message(CMD_SET_DEVICE_PWR_STATE, &dev_set,
 				 sizeof(dev_set), &stat, sizeof(stat));
 }
 
@@ -591,12 +777,16 @@ static int scpi_init_versions(struct scpi_drvinfo *info)
 	int ret;
 	struct scp_capabilities caps;
 
-	ret = scpi_send_message(SCPI_CMD_SCPI_CAPABILITIES, NULL, 0,
+	ret = scpi_send_message(CMD_SCPI_CAPABILITIES, NULL, 0,
 				&caps, sizeof(caps));
 	if (!ret) {
 		info->protocol_version = le32_to_cpu(caps.protocol_version);
 		info->firmware_version = le32_to_cpu(caps.platform_version);
 	}
+	/* Ignore error if not implemented */
+	if (scpi_info->is_legacy && ret == -EOPNOTSUPP)
+		return 0;
+
 	return ret;
 }
 
@@ -681,6 +871,11 @@ static int scpi_alloc_xfer_list(struct device *dev, struct scpi_chan *ch)
 	return 0;
 }
 
+static const struct of_device_id legacy_scpi_of_match[] = {
+	{.compatible = "arm,scpi-pre-1.0"},
+	{},
+};
+
 static int scpi_probe(struct platform_device *pdev)
 {
 	int count, idx, ret;
@@ -693,6 +888,9 @@ static int scpi_probe(struct platform_device *pdev)
 	if (!scpi_info)
 		return -ENOMEM;
 
+	if (of_match_device(legacy_scpi_of_match, &pdev->dev))
+		scpi_info->is_legacy = true;
+
 	count = of_count_phandle_with_args(np, "mboxes", "#mbox-cells");
 	if (count < 0) {
 		dev_err(dev, "no mboxes property in '%s'\n", np->full_name);
@@ -755,8 +953,21 @@ err:
 
 	scpi_info->channels = scpi_chan;
 	scpi_info->num_chans = count;
+	scpi_info->commands = scpi_std_commands;
+
 	platform_set_drvdata(pdev, scpi_info);
 
+	if (scpi_info->is_legacy) {
+		/* Replace with legacy variants */
+		scpi_ops.clk_set_val = legacy_scpi_clk_set_val;
+		scpi_info->commands = scpi_legacy_commands;
+
+		/* Fill priority bitmap */
+		for (idx = 0; idx < ARRAY_SIZE(legacy_hpriority_cmds); idx++)
+			set_bit(legacy_hpriority_cmds[idx],
+				scpi_info->cmd_priority);
+	}
+
 	ret = scpi_init_versions(scpi_info);
 	if (ret) {
 		dev_err(dev, "incorrect or no SCP firmware found\n");
@@ -781,6 +992,7 @@ err:
 
 static const struct of_device_id scpi_of_match[] = {
 	{.compatible = "arm,scpi"},
+	{.compatible = "arm,scpi-pre-1.0"},
 	{},
 };
 
diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c
index 7c75a8d9091a..349dc3e1e52e 100644
--- a/drivers/firmware/efi/arm-runtime.c
+++ b/drivers/firmware/efi/arm-runtime.c
@@ -39,7 +39,7 @@ static struct mm_struct efi_mm = {
 	.mmlist			= LIST_HEAD_INIT(efi_mm.mmlist),
 };
 
-#ifdef CONFIG_ARM64_PTDUMP
+#ifdef CONFIG_ARM64_PTDUMP_DEBUGFS
 #include <asm/ptdump.h>
 
 static struct ptdump_info efi_ptdump_info = {
@@ -53,7 +53,7 @@ static struct ptdump_info efi_ptdump_info = {
 
 static int __init ptdump_init(void)
 {
-	return ptdump_register(&efi_ptdump_info, "efi_page_tables");
+	return ptdump_debugfs_register(&efi_ptdump_info, "efi_page_tables");
 }
 device_initcall(ptdump_init);
 
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index 6621b13c370f..d564d25df8ab 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -6,7 +6,7 @@
 #
 cflags-$(CONFIG_X86_32)		:= -march=i386
 cflags-$(CONFIG_X86_64)		:= -mcmodel=small
-cflags-$(CONFIG_X86)		+= -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2 \
+cflags-$(CONFIG_X86)		+= -m$(BITS) -D__KERNEL__ -O2 \
 				   -fPIC -fno-strict-aliasing -mno-red-zone \
 				   -mno-mmx -mno-sse
 
diff --git a/drivers/firmware/psci.c b/drivers/firmware/psci.c
index 8263429e21b8..6c60a5087caf 100644
--- a/drivers/firmware/psci.c
+++ b/drivers/firmware/psci.c
@@ -630,7 +630,7 @@ int __init psci_dt_init(void)
 
 	np = of_find_matching_node_and_match(NULL, psci_of_match, &matched_np);
 
-	if (!np)
+	if (!np || !of_device_is_available(np))
 		return -ENODEV;
 
 	init_fn = (psci_initcall_t)matched_np->data;
diff --git a/drivers/firmware/psci_checker.c b/drivers/firmware/psci_checker.c
new file mode 100644
index 000000000000..44bdb78f837b
--- /dev/null
+++ b/drivers/firmware/psci_checker.c
@@ -0,0 +1,490 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * Copyright (C) 2016 ARM Limited
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/atomic.h>
+#include <linux/completion.h>
+#include <linux/cpu.h>
+#include <linux/cpuidle.h>
+#include <linux/cpu_pm.h>
+#include <linux/kernel.h>
+#include <linux/kthread.h>
+#include <linux/module.h>
+#include <linux/preempt.h>
+#include <linux/psci.h>
+#include <linux/slab.h>
+#include <linux/tick.h>
+#include <linux/topology.h>
+
+#include <asm/cpuidle.h>
+
+#include <uapi/linux/psci.h>
+
+#define NUM_SUSPEND_CYCLE (10)
+
+static unsigned int nb_available_cpus;
+static int tos_resident_cpu = -1;
+
+static atomic_t nb_active_threads;
+static struct completion suspend_threads_started =
+	COMPLETION_INITIALIZER(suspend_threads_started);
+static struct completion suspend_threads_done =
+	COMPLETION_INITIALIZER(suspend_threads_done);
+
+/*
+ * We assume that PSCI operations are used if they are available. This is not
+ * necessarily true on arm64, since the decision is based on the
+ * "enable-method" property of each CPU in the DT, but given that there is no
+ * arch-specific way to check this, we assume that the DT is sensible.
+ */
+static int psci_ops_check(void)
+{
+	int migrate_type = -1;
+	int cpu;
+
+	if (!(psci_ops.cpu_off && psci_ops.cpu_on && psci_ops.cpu_suspend)) {
+		pr_warn("Missing PSCI operations, aborting tests\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (psci_ops.migrate_info_type)
+		migrate_type = psci_ops.migrate_info_type();
+
+	if (migrate_type == PSCI_0_2_TOS_UP_MIGRATE ||
+	    migrate_type == PSCI_0_2_TOS_UP_NO_MIGRATE) {
+		/* There is a UP Trusted OS, find on which core it resides. */
+		for_each_online_cpu(cpu)
+			if (psci_tos_resident_on(cpu)) {
+				tos_resident_cpu = cpu;
+				break;
+			}
+		if (tos_resident_cpu == -1)
+			pr_warn("UP Trusted OS resides on no online CPU\n");
+	}
+
+	return 0;
+}
+
+static int find_clusters(const struct cpumask *cpus,
+			 const struct cpumask **clusters)
+{
+	unsigned int nb = 0;
+	cpumask_var_t tmp;
+
+	if (!alloc_cpumask_var(&tmp, GFP_KERNEL))
+		return -ENOMEM;
+	cpumask_copy(tmp, cpus);
+
+	while (!cpumask_empty(tmp)) {
+		const struct cpumask *cluster =
+			topology_core_cpumask(cpumask_any(tmp));
+
+		clusters[nb++] = cluster;
+		cpumask_andnot(tmp, tmp, cluster);
+	}
+
+	free_cpumask_var(tmp);
+	return nb;
+}
+
+/*
+ * offlined_cpus is a temporary array but passing it as an argument avoids
+ * multiple allocations.
+ */
+static unsigned int down_and_up_cpus(const struct cpumask *cpus,
+				     struct cpumask *offlined_cpus)
+{
+	int cpu;
+	int err = 0;
+
+	cpumask_clear(offlined_cpus);
+
+	/* Try to power down all CPUs in the mask. */
+	for_each_cpu(cpu, cpus) {
+		int ret = cpu_down(cpu);
+
+		/*
+		 * cpu_down() checks the number of online CPUs before the TOS
+		 * resident CPU.
+		 */
+		if (cpumask_weight(offlined_cpus) + 1 == nb_available_cpus) {
+			if (ret != -EBUSY) {
+				pr_err("Unexpected return code %d while trying "
+				       "to power down last online CPU %d\n",
+				       ret, cpu);
+				++err;
+			}
+		} else if (cpu == tos_resident_cpu) {
+			if (ret != -EPERM) {
+				pr_err("Unexpected return code %d while trying "
+				       "to power down TOS resident CPU %d\n",
+				       ret, cpu);
+				++err;
+			}
+		} else if (ret != 0) {
+			pr_err("Error occurred (%d) while trying "
+			       "to power down CPU %d\n", ret, cpu);
+			++err;
+		}
+
+		if (ret == 0)
+			cpumask_set_cpu(cpu, offlined_cpus);
+	}
+
+	/* Try to power up all the CPUs that have been offlined. */
+	for_each_cpu(cpu, offlined_cpus) {
+		int ret = cpu_up(cpu);
+
+		if (ret != 0) {
+			pr_err("Error occurred (%d) while trying "
+			       "to power up CPU %d\n", ret, cpu);
+			++err;
+		} else {
+			cpumask_clear_cpu(cpu, offlined_cpus);
+		}
+	}
+
+	/*
+	 * Something went bad at some point and some CPUs could not be turned
+	 * back on.
+	 */
+	WARN_ON(!cpumask_empty(offlined_cpus) ||
+		num_online_cpus() != nb_available_cpus);
+
+	return err;
+}
+
+static int hotplug_tests(void)
+{
+	int err;
+	cpumask_var_t offlined_cpus;
+	int i, nb_cluster;
+	const struct cpumask **clusters;
+	char *page_buf;
+
+	err = -ENOMEM;
+	if (!alloc_cpumask_var(&offlined_cpus, GFP_KERNEL))
+		return err;
+	/* We may have up to nb_available_cpus clusters. */
+	clusters = kmalloc_array(nb_available_cpus, sizeof(*clusters),
+				 GFP_KERNEL);
+	if (!clusters)
+		goto out_free_cpus;
+	page_buf = (char *)__get_free_page(GFP_KERNEL);
+	if (!page_buf)
+		goto out_free_clusters;
+
+	err = 0;
+	nb_cluster = find_clusters(cpu_online_mask, clusters);
+
+	/*
+	 * Of course the last CPU cannot be powered down and cpu_down() should
+	 * refuse doing that.
+	 */
+	pr_info("Trying to turn off and on again all CPUs\n");
+	err += down_and_up_cpus(cpu_online_mask, offlined_cpus);
+
+	/*
+	 * Take down CPUs by cluster this time. When the last CPU is turned
+	 * off, the cluster itself should shut down.
+	 */
+	for (i = 0; i < nb_cluster; ++i) {
+		int cluster_id =
+			topology_physical_package_id(cpumask_any(clusters[i]));
+		ssize_t len = cpumap_print_to_pagebuf(true, page_buf,
+						      clusters[i]);
+		/* Remove trailing newline. */
+		page_buf[len - 1] = '\0';
+		pr_info("Trying to turn off and on again cluster %d "
+			"(CPUs %s)\n", cluster_id, page_buf);
+		err += down_and_up_cpus(clusters[i], offlined_cpus);
+	}
+
+	free_page((unsigned long)page_buf);
+out_free_clusters:
+	kfree(clusters);
+out_free_cpus:
+	free_cpumask_var(offlined_cpus);
+	return err;
+}
+
+static void dummy_callback(unsigned long ignored) {}
+
+static int suspend_cpu(int index, bool broadcast)
+{
+	int ret;
+
+	arch_cpu_idle_enter();
+
+	if (broadcast) {
+		/*
+		 * The local timer will be shut down, we need to enter tick
+		 * broadcast.
+		 */
+		ret = tick_broadcast_enter();
+		if (ret) {
+			/*
+			 * In the absence of hardware broadcast mechanism,
+			 * this CPU might be used to broadcast wakeups, which
+			 * may be why entering tick broadcast has failed.
+			 * There is little the kernel can do to work around
+			 * that, so enter WFI instead (idle state 0).
+			 */
+			cpu_do_idle();
+			ret = 0;
+			goto out_arch_exit;
+		}
+	}
+
+	/*
+	 * Replicate the common ARM cpuidle enter function
+	 * (arm_enter_idle_state).
+	 */
+	ret = CPU_PM_CPU_IDLE_ENTER(arm_cpuidle_suspend, index);
+
+	if (broadcast)
+		tick_broadcast_exit();
+
+out_arch_exit:
+	arch_cpu_idle_exit();
+
+	return ret;
+}
+
+static int suspend_test_thread(void *arg)
+{
+	int cpu = (long)arg;
+	int i, nb_suspend = 0, nb_shallow_sleep = 0, nb_err = 0;
+	struct sched_param sched_priority = { .sched_priority = MAX_RT_PRIO-1 };
+	struct cpuidle_device *dev;
+	struct cpuidle_driver *drv;
+	/* No need for an actual callback, we just want to wake up the CPU. */
+	struct timer_list wakeup_timer =
+		TIMER_INITIALIZER(dummy_callback, 0, 0);
+
+	/* Wait for the main thread to give the start signal. */
+	wait_for_completion(&suspend_threads_started);
+
+	/* Set maximum priority to preempt all other threads on this CPU. */
+	if (sched_setscheduler_nocheck(current, SCHED_FIFO, &sched_priority))
+		pr_warn("Failed to set suspend thread scheduler on CPU %d\n",
+			cpu);
+
+	dev = this_cpu_read(cpuidle_devices);
+	drv = cpuidle_get_cpu_driver(dev);
+
+	pr_info("CPU %d entering suspend cycles, states 1 through %d\n",
+		cpu, drv->state_count - 1);
+
+	for (i = 0; i < NUM_SUSPEND_CYCLE; ++i) {
+		int index;
+		/*
+		 * Test all possible states, except 0 (which is usually WFI and
+		 * doesn't use PSCI).
+		 */
+		for (index = 1; index < drv->state_count; ++index) {
+			struct cpuidle_state *state = &drv->states[index];
+			bool broadcast = state->flags & CPUIDLE_FLAG_TIMER_STOP;
+			int ret;
+
+			/*
+			 * Set the timer to wake this CPU up in some time (which
+			 * should be largely sufficient for entering suspend).
+			 * If the local tick is disabled when entering suspend,
+			 * suspend_cpu() takes care of switching to a broadcast
+			 * tick, so the timer will still wake us up.
+			 */
+			mod_timer(&wakeup_timer, jiffies +
+				  usecs_to_jiffies(state->target_residency));
+
+			/* IRQs must be disabled during suspend operations. */
+			local_irq_disable();
+
+			ret = suspend_cpu(index, broadcast);
+
+			/*
+			 * We have woken up. Re-enable IRQs to handle any
+			 * pending interrupt, do not wait until the end of the
+			 * loop.
+			 */
+			local_irq_enable();
+
+			if (ret == index) {
+				++nb_suspend;
+			} else if (ret >= 0) {
+				/* We did not enter the expected state. */
+				++nb_shallow_sleep;
+			} else {
+				pr_err("Failed to suspend CPU %d: error %d "
+				       "(requested state %d, cycle %d)\n",
+				       cpu, ret, index, i);
+				++nb_err;
+			}
+		}
+	}
+
+	/*
+	 * Disable the timer to make sure that the timer will not trigger
+	 * later.
+	 */
+	del_timer(&wakeup_timer);
+
+	if (atomic_dec_return_relaxed(&nb_active_threads) == 0)
+		complete(&suspend_threads_done);
+
+	/* Give up on RT scheduling and wait for termination. */
+	sched_priority.sched_priority = 0;
+	if (sched_setscheduler_nocheck(current, SCHED_NORMAL, &sched_priority))
+		pr_warn("Failed to set suspend thread scheduler on CPU %d\n",
+			cpu);
+	for (;;) {
+		/* Needs to be set first to avoid missing a wakeup. */
+		set_current_state(TASK_INTERRUPTIBLE);
+		if (kthread_should_stop()) {
+			__set_current_state(TASK_RUNNING);
+			break;
+		}
+		schedule();
+	}
+
+	pr_info("CPU %d suspend test results: success %d, shallow states %d, errors %d\n",
+		cpu, nb_suspend, nb_shallow_sleep, nb_err);
+
+	return nb_err;
+}
+
+static int suspend_tests(void)
+{
+	int i, cpu, err = 0;
+	struct task_struct **threads;
+	int nb_threads = 0;
+
+	threads = kmalloc_array(nb_available_cpus, sizeof(*threads),
+				GFP_KERNEL);
+	if (!threads)
+		return -ENOMEM;
+
+	/*
+	 * Stop cpuidle to prevent the idle tasks from entering a deep sleep
+	 * mode, as it might interfere with the suspend threads on other CPUs.
+	 * This does not prevent the suspend threads from using cpuidle (only
+	 * the idle tasks check this status). Take the idle lock so that
+	 * the cpuidle driver and device look-up can be carried out safely.
+	 */
+	cpuidle_pause_and_lock();
+
+	for_each_online_cpu(cpu) {
+		struct task_struct *thread;
+		/* Check that cpuidle is available on that CPU. */
+		struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
+		struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
+
+		if (!dev || !drv) {
+			pr_warn("cpuidle not available on CPU %d, ignoring\n",
+				cpu);
+			continue;
+		}
+
+		thread = kthread_create_on_cpu(suspend_test_thread,
+					       (void *)(long)cpu, cpu,
+					       "psci_suspend_test");
+		if (IS_ERR(thread))
+			pr_err("Failed to create kthread on CPU %d\n", cpu);
+		else
+			threads[nb_threads++] = thread;
+	}
+
+	if (nb_threads < 1) {
+		err = -ENODEV;
+		goto out;
+	}
+
+	atomic_set(&nb_active_threads, nb_threads);
+
+	/*
+	 * Wake up the suspend threads. To avoid the main thread being preempted
+	 * before all the threads have been unparked, the suspend threads will
+	 * wait for the completion of suspend_threads_started.
+	 */
+	for (i = 0; i < nb_threads; ++i)
+		wake_up_process(threads[i]);
+	complete_all(&suspend_threads_started);
+
+	wait_for_completion(&suspend_threads_done);
+
+
+	/* Stop and destroy all threads, get return status. */
+	for (i = 0; i < nb_threads; ++i)
+		err += kthread_stop(threads[i]);
+ out:
+	cpuidle_resume_and_unlock();
+	kfree(threads);
+	return err;
+}
+
+static int __init psci_checker(void)
+{
+	int ret;
+
+	/*
+	 * Since we're in an initcall, we assume that all the CPUs that all
+	 * CPUs that can be onlined have been onlined.
+	 *
+	 * The tests assume that hotplug is enabled but nobody else is using it,
+	 * otherwise the results will be unpredictable. However, since there
+	 * is no userspace yet in initcalls, that should be fine, as long as
+	 * no torture test is running at the same time (see Kconfig).
+	 */
+	nb_available_cpus = num_online_cpus();
+
+	/* Check PSCI operations are set up and working. */
+	ret = psci_ops_check();
+	if (ret)
+		return ret;
+
+	pr_info("PSCI checker started using %u CPUs\n", nb_available_cpus);
+
+	pr_info("Starting hotplug tests\n");
+	ret = hotplug_tests();
+	if (ret == 0)
+		pr_info("Hotplug tests passed OK\n");
+	else if (ret > 0)
+		pr_err("%d error(s) encountered in hotplug tests\n", ret);
+	else {
+		pr_err("Out of memory\n");
+		return ret;
+	}
+
+	pr_info("Starting suspend tests (%d cycles per state)\n",
+		NUM_SUSPEND_CYCLE);
+	ret = suspend_tests();
+	if (ret == 0)
+		pr_info("Suspend tests passed OK\n");
+	else if (ret > 0)
+		pr_err("%d error(s) encountered in suspend tests\n", ret);
+	else {
+		switch (ret) {
+		case -ENOMEM:
+			pr_err("Out of memory\n");
+			break;
+		case -ENODEV:
+			pr_warn("Could not start suspend tests on any CPU\n");
+			break;
+		}
+	}
+
+	pr_info("PSCI checker completed\n");
+	return ret < 0 ? ret : 0;
+}
+late_initcall(psci_checker);
diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c
index d95c70227c05..893f953eaccf 100644
--- a/drivers/firmware/qcom_scm.c
+++ b/drivers/firmware/qcom_scm.c
@@ -28,6 +28,10 @@
 
 #include "qcom_scm.h"
 
+#define SCM_HAS_CORE_CLK	BIT(0)
+#define SCM_HAS_IFACE_CLK	BIT(1)
+#define SCM_HAS_BUS_CLK		BIT(2)
+
 struct qcom_scm {
 	struct device *dev;
 	struct clk *core_clk;
@@ -323,32 +327,40 @@ EXPORT_SYMBOL(qcom_scm_is_available);
 static int qcom_scm_probe(struct platform_device *pdev)
 {
 	struct qcom_scm *scm;
+	unsigned long clks;
 	int ret;
 
 	scm = devm_kzalloc(&pdev->dev, sizeof(*scm), GFP_KERNEL);
 	if (!scm)
 		return -ENOMEM;
 
-	scm->core_clk = devm_clk_get(&pdev->dev, "core");
-	if (IS_ERR(scm->core_clk)) {
-		if (PTR_ERR(scm->core_clk) == -EPROBE_DEFER)
+	clks = (unsigned long)of_device_get_match_data(&pdev->dev);
+	if (clks & SCM_HAS_CORE_CLK) {
+		scm->core_clk = devm_clk_get(&pdev->dev, "core");
+		if (IS_ERR(scm->core_clk)) {
+			if (PTR_ERR(scm->core_clk) != -EPROBE_DEFER)
+				dev_err(&pdev->dev,
+					"failed to acquire core clk\n");
 			return PTR_ERR(scm->core_clk);
-
-		scm->core_clk = NULL;
+		}
 	}
 
-	if (of_device_is_compatible(pdev->dev.of_node, "qcom,scm")) {
+	if (clks & SCM_HAS_IFACE_CLK) {
 		scm->iface_clk = devm_clk_get(&pdev->dev, "iface");
 		if (IS_ERR(scm->iface_clk)) {
 			if (PTR_ERR(scm->iface_clk) != -EPROBE_DEFER)
-				dev_err(&pdev->dev, "failed to acquire iface clk\n");
+				dev_err(&pdev->dev,
+					"failed to acquire iface clk\n");
 			return PTR_ERR(scm->iface_clk);
 		}
+	}
 
+	if (clks & SCM_HAS_BUS_CLK) {
 		scm->bus_clk = devm_clk_get(&pdev->dev, "bus");
 		if (IS_ERR(scm->bus_clk)) {
 			if (PTR_ERR(scm->bus_clk) != -EPROBE_DEFER)
-				dev_err(&pdev->dev, "failed to acquire bus clk\n");
+				dev_err(&pdev->dev,
+					"failed to acquire bus clk\n");
 			return PTR_ERR(scm->bus_clk);
 		}
 	}
@@ -356,7 +368,9 @@ static int qcom_scm_probe(struct platform_device *pdev)
 	scm->reset.ops = &qcom_scm_pas_reset_ops;
 	scm->reset.nr_resets = 1;
 	scm->reset.of_node = pdev->dev.of_node;
-	reset_controller_register(&scm->reset);
+	ret = devm_reset_controller_register(&pdev->dev, &scm->reset);
+	if (ret)
+		return ret;
 
 	/* vote for max clk rate for highest performance */
 	ret = clk_set_rate(scm->core_clk, INT_MAX);
@@ -372,10 +386,23 @@ static int qcom_scm_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id qcom_scm_dt_match[] = {
-	{ .compatible = "qcom,scm-apq8064",},
-	{ .compatible = "qcom,scm-msm8660",},
-	{ .compatible = "qcom,scm-msm8960",},
-	{ .compatible = "qcom,scm",},
+	{ .compatible = "qcom,scm-apq8064",
+	  .data = (void *) SCM_HAS_CORE_CLK,
+	},
+	{ .compatible = "qcom,scm-msm8660",
+	  .data = (void *) SCM_HAS_CORE_CLK,
+	},
+	{ .compatible = "qcom,scm-msm8960",
+	  .data = (void *) SCM_HAS_CORE_CLK,
+	},
+	{ .compatible = "qcom,scm-msm8996",
+	  .data = NULL, /* no clocks */
+	},
+	{ .compatible = "qcom,scm",
+	  .data = (void *)(SCM_HAS_CORE_CLK
+			   | SCM_HAS_IFACE_CLK
+			   | SCM_HAS_BUS_CLK),
+	},
 	{}
 };
 
diff --git a/drivers/firmware/tegra/Kconfig b/drivers/firmware/tegra/Kconfig
new file mode 100644
index 000000000000..ff2730d5c468
--- /dev/null
+++ b/drivers/firmware/tegra/Kconfig
@@ -0,0 +1,25 @@
+menu "Tegra firmware driver"
+
+config TEGRA_IVC
+	bool "Tegra IVC protocol"
+	depends on ARCH_TEGRA
+	help
+	  IVC (Inter-VM Communication) protocol is part of the IPC
+	  (Inter Processor Communication) framework on Tegra. It maintains the
+	  data and the different commuication channels in SysRAM or RAM and
+	  keeps the content is synchronization between host CPU and remote
+	  processors.
+
+config TEGRA_BPMP
+	bool "Tegra BPMP driver"
+	depends on ARCH_TEGRA && TEGRA_HSP_MBOX && TEGRA_IVC
+	help
+	  BPMP (Boot and Power Management Processor) is designed to off-loading
+	  the PM functions which include clock/DVFS/thermal/power from the CPU.
+	  It needs HSP as the HW synchronization and notification module and
+	  IVC module as the message communication protocol.
+
+	  This driver manages the IPC interface between host CPU and the
+	  firmware running on BPMP.
+
+endmenu
diff --git a/drivers/firmware/tegra/Makefile b/drivers/firmware/tegra/Makefile
new file mode 100644
index 000000000000..e34a2f79e1ad
--- /dev/null
+++ b/drivers/firmware/tegra/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_TEGRA_BPMP)	+= bpmp.o
+obj-$(CONFIG_TEGRA_IVC)		+= ivc.o
diff --git a/drivers/firmware/tegra/bpmp.c b/drivers/firmware/tegra/bpmp.c
new file mode 100644
index 000000000000..4ff02d310868
--- /dev/null
+++ b/drivers/firmware/tegra/bpmp.c
@@ -0,0 +1,868 @@
+/*
+ * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/clk/tegra.h>
+#include <linux/genalloc.h>
+#include <linux/mailbox_client.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/semaphore.h>
+
+#include <soc/tegra/bpmp.h>
+#include <soc/tegra/bpmp-abi.h>
+#include <soc/tegra/ivc.h>
+
+#define MSG_ACK		BIT(0)
+#define MSG_RING	BIT(1)
+
+static inline struct tegra_bpmp *
+mbox_client_to_bpmp(struct mbox_client *client)
+{
+	return container_of(client, struct tegra_bpmp, mbox.client);
+}
+
+struct tegra_bpmp *tegra_bpmp_get(struct device *dev)
+{
+	struct platform_device *pdev;
+	struct tegra_bpmp *bpmp;
+	struct device_node *np;
+
+	np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);
+	if (!np)
+		return ERR_PTR(-ENOENT);
+
+	pdev = of_find_device_by_node(np);
+	if (!pdev) {
+		bpmp = ERR_PTR(-ENODEV);
+		goto put;
+	}
+
+	bpmp = platform_get_drvdata(pdev);
+	if (!bpmp) {
+		bpmp = ERR_PTR(-EPROBE_DEFER);
+		put_device(&pdev->dev);
+		goto put;
+	}
+
+put:
+	of_node_put(np);
+	return bpmp;
+}
+EXPORT_SYMBOL_GPL(tegra_bpmp_get);
+
+void tegra_bpmp_put(struct tegra_bpmp *bpmp)
+{
+	if (bpmp)
+		put_device(bpmp->dev);
+}
+EXPORT_SYMBOL_GPL(tegra_bpmp_put);
+
+static int tegra_bpmp_channel_get_index(struct tegra_bpmp_channel *channel)
+{
+	return channel - channel->bpmp->channels;
+}
+
+static int
+tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)
+{
+	struct tegra_bpmp *bpmp = channel->bpmp;
+	unsigned int offset, count;
+	int index;
+
+	offset = bpmp->soc->channels.thread.offset;
+	count = bpmp->soc->channels.thread.count;
+
+	index = tegra_bpmp_channel_get_index(channel);
+	if (index < 0)
+		return index;
+
+	if (index < offset || index >= offset + count)
+		return -EINVAL;
+
+	return index - offset;
+}
+
+static struct tegra_bpmp_channel *
+tegra_bpmp_channel_get_thread(struct tegra_bpmp *bpmp, unsigned int index)
+{
+	unsigned int offset = bpmp->soc->channels.thread.offset;
+	unsigned int count = bpmp->soc->channels.thread.count;
+
+	if (index >= count)
+		return NULL;
+
+	return &bpmp->channels[offset + index];
+}
+
+static struct tegra_bpmp_channel *
+tegra_bpmp_channel_get_tx(struct tegra_bpmp *bpmp)
+{
+	unsigned int offset = bpmp->soc->channels.cpu_tx.offset;
+
+	return &bpmp->channels[offset + smp_processor_id()];
+}
+
+static struct tegra_bpmp_channel *
+tegra_bpmp_channel_get_rx(struct tegra_bpmp *bpmp)
+{
+	unsigned int offset = bpmp->soc->channels.cpu_rx.offset;
+
+	return &bpmp->channels[offset];
+}
+
+static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg)
+{
+	return (msg->tx.size <= MSG_DATA_MIN_SZ) &&
+	       (msg->rx.size <= MSG_DATA_MIN_SZ) &&
+	       (msg->tx.size == 0 || msg->tx.data) &&
+	       (msg->rx.size == 0 || msg->rx.data);
+}
+
+static bool tegra_bpmp_master_acked(struct tegra_bpmp_channel *channel)
+{
+	void *frame;
+
+	frame = tegra_ivc_read_get_next_frame(channel->ivc);
+	if (IS_ERR(frame)) {
+		channel->ib = NULL;
+		return false;
+	}
+
+	channel->ib = frame;
+
+	return true;
+}
+
+static int tegra_bpmp_wait_ack(struct tegra_bpmp_channel *channel)
+{
+	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
+	ktime_t end;
+
+	end = ktime_add_us(ktime_get(), timeout);
+
+	do {
+		if (tegra_bpmp_master_acked(channel))
+			return 0;
+	} while (ktime_before(ktime_get(), end));
+
+	return -ETIMEDOUT;
+}
+
+static bool tegra_bpmp_master_free(struct tegra_bpmp_channel *channel)
+{
+	void *frame;
+
+	frame = tegra_ivc_write_get_next_frame(channel->ivc);
+	if (IS_ERR(frame)) {
+		channel->ob = NULL;
+		return false;
+	}
+
+	channel->ob = frame;
+
+	return true;
+}
+
+static int tegra_bpmp_wait_master_free(struct tegra_bpmp_channel *channel)
+{
+	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
+	ktime_t start, now;
+
+	start = ns_to_ktime(local_clock());
+
+	do {
+		if (tegra_bpmp_master_free(channel))
+			return 0;
+
+		now = ns_to_ktime(local_clock());
+	} while (ktime_us_delta(now, start) < timeout);
+
+	return -ETIMEDOUT;
+}
+
+static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
+					 void *data, size_t size)
+{
+	if (data && size > 0)
+		memcpy(data, channel->ib->data, size);
+
+	return tegra_ivc_read_advance(channel->ivc);
+}
+
+static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
+				       void *data, size_t size)
+{
+	struct tegra_bpmp *bpmp = channel->bpmp;
+	unsigned long flags;
+	ssize_t err;
+	int index;
+
+	index = tegra_bpmp_channel_get_thread_index(channel);
+	if (index < 0)
+		return index;
+
+	spin_lock_irqsave(&bpmp->lock, flags);
+	err = __tegra_bpmp_channel_read(channel, data, size);
+	clear_bit(index, bpmp->threaded.allocated);
+	spin_unlock_irqrestore(&bpmp->lock, flags);
+
+	up(&bpmp->threaded.lock);
+
+	return err;
+}
+
+static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
+					  unsigned int mrq, unsigned long flags,
+					  const void *data, size_t size)
+{
+	channel->ob->code = mrq;
+	channel->ob->flags = flags;
+
+	if (data && size > 0)
+		memcpy(channel->ob->data, data, size);
+
+	return tegra_ivc_write_advance(channel->ivc);
+}
+
+static struct tegra_bpmp_channel *
+tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,
+			  const void *data, size_t size)
+{
+	unsigned long timeout = bpmp->soc->channels.thread.timeout;
+	unsigned int count = bpmp->soc->channels.thread.count;
+	struct tegra_bpmp_channel *channel;
+	unsigned long flags;
+	unsigned int index;
+	int err;
+
+	err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));
+	if (err < 0)
+		return ERR_PTR(err);
+
+	spin_lock_irqsave(&bpmp->lock, flags);
+
+	index = find_first_zero_bit(bpmp->threaded.allocated, count);
+	if (index == count) {
+		channel = ERR_PTR(-EBUSY);
+		goto unlock;
+	}
+
+	channel = tegra_bpmp_channel_get_thread(bpmp, index);
+	if (!channel) {
+		channel = ERR_PTR(-EINVAL);
+		goto unlock;
+	}
+
+	if (!tegra_bpmp_master_free(channel)) {
+		channel = ERR_PTR(-EBUSY);
+		goto unlock;
+	}
+
+	set_bit(index, bpmp->threaded.allocated);
+
+	err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,
+					 data, size);
+	if (err < 0) {
+		clear_bit(index, bpmp->threaded.allocated);
+		goto unlock;
+	}
+
+	set_bit(index, bpmp->threaded.busy);
+
+unlock:
+	spin_unlock_irqrestore(&bpmp->lock, flags);
+	return channel;
+}
+
+static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
+					unsigned int mrq, unsigned long flags,
+					const void *data, size_t size)
+{
+	int err;
+
+	err = tegra_bpmp_wait_master_free(channel);
+	if (err < 0)
+		return err;
+
+	return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);
+}
+
+int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
+			       struct tegra_bpmp_message *msg)
+{
+	struct tegra_bpmp_channel *channel;
+	int err;
+
+	if (WARN_ON(!irqs_disabled()))
+		return -EPERM;
+
+	if (!tegra_bpmp_message_valid(msg))
+		return -EINVAL;
+
+	channel = tegra_bpmp_channel_get_tx(bpmp);
+
+	err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,
+				       msg->tx.data, msg->tx.size);
+	if (err < 0)
+		return err;
+
+	err = mbox_send_message(bpmp->mbox.channel, NULL);
+	if (err < 0)
+		return err;
+
+	mbox_client_txdone(bpmp->mbox.channel, 0);
+
+	err = tegra_bpmp_wait_ack(channel);
+	if (err < 0)
+		return err;
+
+	return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size);
+}
+EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic);
+
+int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
+			struct tegra_bpmp_message *msg)
+{
+	struct tegra_bpmp_channel *channel;
+	unsigned long timeout;
+	int err;
+
+	if (WARN_ON(irqs_disabled()))
+		return -EPERM;
+
+	if (!tegra_bpmp_message_valid(msg))
+		return -EINVAL;
+
+	channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,
+					    msg->tx.size);
+	if (IS_ERR(channel))
+		return PTR_ERR(channel);
+
+	err = mbox_send_message(bpmp->mbox.channel, NULL);
+	if (err < 0)
+		return err;
+
+	mbox_client_txdone(bpmp->mbox.channel, 0);
+
+	timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);
+
+	err = wait_for_completion_timeout(&channel->completion, timeout);
+	if (err == 0)
+		return -ETIMEDOUT;
+
+	return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size);
+}
+EXPORT_SYMBOL_GPL(tegra_bpmp_transfer);
+
+static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,
+						  unsigned int mrq)
+{
+	struct tegra_bpmp_mrq *entry;
+
+	list_for_each_entry(entry, &bpmp->mrqs, list)
+		if (entry->mrq == mrq)
+			return entry;
+
+	return NULL;
+}
+
+static void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel,
+				  int code, const void *data, size_t size)
+{
+	unsigned long flags = channel->ib->flags;
+	struct tegra_bpmp *bpmp = channel->bpmp;
+	struct tegra_bpmp_mb_data *frame;
+	int err;
+
+	if (WARN_ON(size > MSG_DATA_MIN_SZ))
+		return;
+
+	err = tegra_ivc_read_advance(channel->ivc);
+	if (WARN_ON(err < 0))
+		return;
+
+	if ((flags & MSG_ACK) == 0)
+		return;
+
+	frame = tegra_ivc_write_get_next_frame(channel->ivc);
+	if (WARN_ON(IS_ERR(frame)))
+		return;
+
+	frame->code = code;
+
+	if (data && size > 0)
+		memcpy(frame->data, data, size);
+
+	err = tegra_ivc_write_advance(channel->ivc);
+	if (WARN_ON(err < 0))
+		return;
+
+	if (flags & MSG_RING) {
+		err = mbox_send_message(bpmp->mbox.channel, NULL);
+		if (WARN_ON(err < 0))
+			return;
+
+		mbox_client_txdone(bpmp->mbox.channel, 0);
+	}
+}
+
+static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,
+				  unsigned int mrq,
+				  struct tegra_bpmp_channel *channel)
+{
+	struct tegra_bpmp_mrq *entry;
+	u32 zero = 0;
+
+	spin_lock(&bpmp->lock);
+
+	entry = tegra_bpmp_find_mrq(bpmp, mrq);
+	if (!entry) {
+		spin_unlock(&bpmp->lock);
+		tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));
+		return;
+	}
+
+	entry->handler(mrq, channel, entry->data);
+
+	spin_unlock(&bpmp->lock);
+}
+
+int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
+			   tegra_bpmp_mrq_handler_t handler, void *data)
+{
+	struct tegra_bpmp_mrq *entry;
+	unsigned long flags;
+
+	if (!handler)
+		return -EINVAL;
+
+	entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);
+	if (!entry)
+		return -ENOMEM;
+
+	spin_lock_irqsave(&bpmp->lock, flags);
+
+	entry->mrq = mrq;
+	entry->handler = handler;
+	entry->data = data;
+	list_add(&entry->list, &bpmp->mrqs);
+
+	spin_unlock_irqrestore(&bpmp->lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq);
+
+void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)
+{
+	struct tegra_bpmp_mrq *entry;
+	unsigned long flags;
+
+	spin_lock_irqsave(&bpmp->lock, flags);
+
+	entry = tegra_bpmp_find_mrq(bpmp, mrq);
+	if (!entry)
+		goto unlock;
+
+	list_del(&entry->list);
+	devm_kfree(bpmp->dev, entry);
+
+unlock:
+	spin_unlock_irqrestore(&bpmp->lock, flags);
+}
+EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq);
+
+static void tegra_bpmp_mrq_handle_ping(unsigned int mrq,
+				       struct tegra_bpmp_channel *channel,
+				       void *data)
+{
+	struct mrq_ping_request *request;
+	struct mrq_ping_response response;
+
+	request = (struct mrq_ping_request *)channel->ib->data;
+
+	memset(&response, 0, sizeof(response));
+	response.reply = request->challenge << 1;
+
+	tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));
+}
+
+static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)
+{
+	struct mrq_ping_response response;
+	struct mrq_ping_request request;
+	struct tegra_bpmp_message msg;
+	unsigned long flags;
+	ktime_t start, end;
+	int err;
+
+	memset(&request, 0, sizeof(request));
+	request.challenge = 1;
+
+	memset(&response, 0, sizeof(response));
+
+	memset(&msg, 0, sizeof(msg));
+	msg.mrq = MRQ_PING;
+	msg.tx.data = &request;
+	msg.tx.size = sizeof(request);
+	msg.rx.data = &response;
+	msg.rx.size = sizeof(response);
+
+	local_irq_save(flags);
+	start = ktime_get();
+	err = tegra_bpmp_transfer_atomic(bpmp, &msg);
+	end = ktime_get();
+	local_irq_restore(flags);
+
+	if (!err)
+		dev_dbg(bpmp->dev,
+			"ping ok: challenge: %u, response: %u, time: %lld\n",
+			request.challenge, response.reply,
+			ktime_to_us(ktime_sub(end, start)));
+
+	return err;
+}
+
+static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,
+				       size_t size)
+{
+	struct mrq_query_tag_request request;
+	struct tegra_bpmp_message msg;
+	unsigned long flags;
+	dma_addr_t phys;
+	void *virt;
+	int err;
+
+	virt = dma_alloc_coherent(bpmp->dev, MSG_DATA_MIN_SZ, &phys,
+				  GFP_KERNEL | GFP_DMA32);
+	if (!virt)
+		return -ENOMEM;
+
+	memset(&request, 0, sizeof(request));
+	request.addr = phys;
+
+	memset(&msg, 0, sizeof(msg));
+	msg.mrq = MRQ_QUERY_TAG;
+	msg.tx.data = &request;
+	msg.tx.size = sizeof(request);
+
+	local_irq_save(flags);
+	err = tegra_bpmp_transfer_atomic(bpmp, &msg);
+	local_irq_restore(flags);
+
+	if (err == 0)
+		strlcpy(tag, virt, size);
+
+	dma_free_coherent(bpmp->dev, MSG_DATA_MIN_SZ, virt, phys);
+
+	return err;
+}
+
+static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)
+{
+	unsigned long flags = channel->ob->flags;
+
+	if ((flags & MSG_RING) == 0)
+		return;
+
+	complete(&channel->completion);
+}
+
+static void tegra_bpmp_handle_rx(struct mbox_client *client, void *data)
+{
+	struct tegra_bpmp *bpmp = mbox_client_to_bpmp(client);
+	struct tegra_bpmp_channel *channel;
+	unsigned int i, count;
+	unsigned long *busy;
+
+	channel = tegra_bpmp_channel_get_rx(bpmp);
+	count = bpmp->soc->channels.thread.count;
+	busy = bpmp->threaded.busy;
+
+	if (tegra_bpmp_master_acked(channel))
+		tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);
+
+	spin_lock(&bpmp->lock);
+
+	for_each_set_bit(i, busy, count) {
+		struct tegra_bpmp_channel *channel;
+
+		channel = tegra_bpmp_channel_get_thread(bpmp, i);
+		if (!channel)
+			continue;
+
+		if (tegra_bpmp_master_acked(channel)) {
+			tegra_bpmp_channel_signal(channel);
+			clear_bit(i, busy);
+		}
+	}
+
+	spin_unlock(&bpmp->lock);
+}
+
+static void tegra_bpmp_ivc_notify(struct tegra_ivc *ivc, void *data)
+{
+	struct tegra_bpmp *bpmp = data;
+	int err;
+
+	if (WARN_ON(bpmp->mbox.channel == NULL))
+		return;
+
+	err = mbox_send_message(bpmp->mbox.channel, NULL);
+	if (err < 0)
+		return;
+
+	mbox_client_txdone(bpmp->mbox.channel, 0);
+}
+
+static int tegra_bpmp_channel_init(struct tegra_bpmp_channel *channel,
+				   struct tegra_bpmp *bpmp,
+				   unsigned int index)
+{
+	size_t message_size, queue_size;
+	unsigned int offset;
+	int err;
+
+	channel->ivc = devm_kzalloc(bpmp->dev, sizeof(*channel->ivc),
+				    GFP_KERNEL);
+	if (!channel->ivc)
+		return -ENOMEM;
+
+	message_size = tegra_ivc_align(MSG_MIN_SZ);
+	queue_size = tegra_ivc_total_queue_size(message_size);
+	offset = queue_size * index;
+
+	err = tegra_ivc_init(channel->ivc, NULL,
+			     bpmp->rx.virt + offset, bpmp->rx.phys + offset,
+			     bpmp->tx.virt + offset, bpmp->tx.phys + offset,
+			     1, message_size, tegra_bpmp_ivc_notify,
+			     bpmp);
+	if (err < 0) {
+		dev_err(bpmp->dev, "failed to setup IVC for channel %u: %d\n",
+			index, err);
+		return err;
+	}
+
+	init_completion(&channel->completion);
+	channel->bpmp = bpmp;
+
+	return 0;
+}
+
+static void tegra_bpmp_channel_reset(struct tegra_bpmp_channel *channel)
+{
+	/* reset the channel state */
+	tegra_ivc_reset(channel->ivc);
+
+	/* sync the channel state with BPMP */
+	while (tegra_ivc_notified(channel->ivc))
+		;
+}
+
+static void tegra_bpmp_channel_cleanup(struct tegra_bpmp_channel *channel)
+{
+	tegra_ivc_cleanup(channel->ivc);
+}
+
+static int tegra_bpmp_probe(struct platform_device *pdev)
+{
+	struct tegra_bpmp_channel *channel;
+	struct tegra_bpmp *bpmp;
+	unsigned int i;
+	char tag[32];
+	size_t size;
+	int err;
+
+	bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);
+	if (!bpmp)
+		return -ENOMEM;
+
+	bpmp->soc = of_device_get_match_data(&pdev->dev);
+	bpmp->dev = &pdev->dev;
+
+	bpmp->tx.pool = of_gen_pool_get(pdev->dev.of_node, "shmem", 0);
+	if (!bpmp->tx.pool) {
+		dev_err(&pdev->dev, "TX shmem pool not found\n");
+		return -ENOMEM;
+	}
+
+	bpmp->tx.virt = gen_pool_dma_alloc(bpmp->tx.pool, 4096, &bpmp->tx.phys);
+	if (!bpmp->tx.virt) {
+		dev_err(&pdev->dev, "failed to allocate from TX pool\n");
+		return -ENOMEM;
+	}
+
+	bpmp->rx.pool = of_gen_pool_get(pdev->dev.of_node, "shmem", 1);
+	if (!bpmp->rx.pool) {
+		dev_err(&pdev->dev, "RX shmem pool not found\n");
+		err = -ENOMEM;
+		goto free_tx;
+	}
+
+	bpmp->rx.virt = gen_pool_dma_alloc(bpmp->rx.pool, 4096, &bpmp->rx.phys);
+	if (!bpmp->rx.pool) {
+		dev_err(&pdev->dev, "failed to allocate from RX pool\n");
+		err = -ENOMEM;
+		goto free_tx;
+	}
+
+	INIT_LIST_HEAD(&bpmp->mrqs);
+	spin_lock_init(&bpmp->lock);
+
+	bpmp->threaded.count = bpmp->soc->channels.thread.count;
+	sema_init(&bpmp->threaded.lock, bpmp->threaded.count);
+
+	size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);
+
+	bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
+	if (!bpmp->threaded.allocated) {
+		err = -ENOMEM;
+		goto free_rx;
+	}
+
+	bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
+	if (!bpmp->threaded.busy) {
+		err = -ENOMEM;
+		goto free_rx;
+	}
+
+	bpmp->num_channels = bpmp->soc->channels.cpu_tx.count +
+			     bpmp->soc->channels.thread.count +
+			     bpmp->soc->channels.cpu_rx.count;
+
+	bpmp->channels = devm_kcalloc(&pdev->dev, bpmp->num_channels,
+				      sizeof(*channel), GFP_KERNEL);
+	if (!bpmp->channels) {
+		err = -ENOMEM;
+		goto free_rx;
+	}
+
+	/* message channel initialization */
+	for (i = 0; i < bpmp->num_channels; i++) {
+		struct tegra_bpmp_channel *channel = &bpmp->channels[i];
+
+		err = tegra_bpmp_channel_init(channel, bpmp, i);
+		if (err < 0)
+			goto cleanup_channels;
+	}
+
+	/* mbox registration */
+	bpmp->mbox.client.dev = &pdev->dev;
+	bpmp->mbox.client.rx_callback = tegra_bpmp_handle_rx;
+	bpmp->mbox.client.tx_block = false;
+	bpmp->mbox.client.knows_txdone = false;
+
+	bpmp->mbox.channel = mbox_request_channel(&bpmp->mbox.client, 0);
+	if (IS_ERR(bpmp->mbox.channel)) {
+		err = PTR_ERR(bpmp->mbox.channel);
+		dev_err(&pdev->dev, "failed to get HSP mailbox: %d\n", err);
+		goto cleanup_channels;
+	}
+
+	/* reset message channels */
+	for (i = 0; i < bpmp->num_channels; i++) {
+		struct tegra_bpmp_channel *channel = &bpmp->channels[i];
+
+		tegra_bpmp_channel_reset(channel);
+	}
+
+	err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,
+				     tegra_bpmp_mrq_handle_ping, bpmp);
+	if (err < 0)
+		goto free_mbox;
+
+	err = tegra_bpmp_ping(bpmp);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);
+		goto free_mrq;
+	}
+
+	err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag) - 1);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);
+		goto free_mrq;
+	}
+
+	dev_info(&pdev->dev, "firmware: %s\n", tag);
+
+	err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
+	if (err < 0)
+		goto free_mrq;
+
+	err = tegra_bpmp_init_clocks(bpmp);
+	if (err < 0)
+		goto free_mrq;
+
+	err = tegra_bpmp_init_resets(bpmp);
+	if (err < 0)
+		goto free_mrq;
+
+	platform_set_drvdata(pdev, bpmp);
+
+	return 0;
+
+free_mrq:
+	tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);
+free_mbox:
+	mbox_free_channel(bpmp->mbox.channel);
+cleanup_channels:
+	while (i--)
+		tegra_bpmp_channel_cleanup(&bpmp->channels[i]);
+free_rx:
+	gen_pool_free(bpmp->rx.pool, (unsigned long)bpmp->rx.virt, 4096);
+free_tx:
+	gen_pool_free(bpmp->tx.pool, (unsigned long)bpmp->tx.virt, 4096);
+	return err;
+}
+
+static const struct tegra_bpmp_soc tegra186_soc = {
+	.channels = {
+		.cpu_tx = {
+			.offset = 0,
+			.count = 6,
+			.timeout = 60 * USEC_PER_SEC,
+		},
+		.thread = {
+			.offset = 6,
+			.count = 7,
+			.timeout = 600 * USEC_PER_SEC,
+		},
+		.cpu_rx = {
+			.offset = 13,
+			.count = 1,
+			.timeout = 0,
+		},
+	},
+	.num_resets = 193,
+};
+
+static const struct of_device_id tegra_bpmp_match[] = {
+	{ .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
+	{ }
+};
+
+static struct platform_driver tegra_bpmp_driver = {
+	.driver = {
+		.name = "tegra-bpmp",
+		.of_match_table = tegra_bpmp_match,
+	},
+	.probe = tegra_bpmp_probe,
+};
+
+static int __init tegra_bpmp_init(void)
+{
+	return platform_driver_register(&tegra_bpmp_driver);
+}
+core_initcall(tegra_bpmp_init);
diff --git a/drivers/firmware/tegra/ivc.c b/drivers/firmware/tegra/ivc.c
new file mode 100644
index 000000000000..29ecfd815320
--- /dev/null
+++ b/drivers/firmware/tegra/ivc.c
@@ -0,0 +1,695 @@
+/*
+ * Copyright (c) 2014-2016, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <soc/tegra/ivc.h>
+
+#define TEGRA_IVC_ALIGN 64
+
+/*
+ * IVC channel reset protocol.
+ *
+ * Each end uses its tx_channel.state to indicate its synchronization state.
+ */
+enum tegra_ivc_state {
+	/*
+	 * This value is zero for backwards compatibility with services that
+	 * assume channels to be initially zeroed. Such channels are in an
+	 * initially valid state, but cannot be asynchronously reset, and must
+	 * maintain a valid state at all times.
+	 *
+	 * The transmitting end can enter the established state from the sync or
+	 * ack state when it observes the receiving endpoint in the ack or
+	 * established state, indicating that has cleared the counters in our
+	 * rx_channel.
+	 */
+	TEGRA_IVC_STATE_ESTABLISHED = 0,
+
+	/*
+	 * If an endpoint is observed in the sync state, the remote endpoint is
+	 * allowed to clear the counters it owns asynchronously with respect to
+	 * the current endpoint. Therefore, the current endpoint is no longer
+	 * allowed to communicate.
+	 */
+	TEGRA_IVC_STATE_SYNC,
+
+	/*
+	 * When the transmitting end observes the receiving end in the sync
+	 * state, it can clear the w_count and r_count and transition to the ack
+	 * state. If the remote endpoint observes us in the ack state, it can
+	 * return to the established state once it has cleared its counters.
+	 */
+	TEGRA_IVC_STATE_ACK
+};
+
+/*
+ * This structure is divided into two-cache aligned parts, the first is only
+ * written through the tx.channel pointer, while the second is only written
+ * through the rx.channel pointer. This delineates ownership of the cache
+ * lines, which is critical to performance and necessary in non-cache coherent
+ * implementations.
+ */
+struct tegra_ivc_header {
+	union {
+		struct {
+			/* fields owned by the transmitting end */
+			u32 count;
+			u32 state;
+		};
+
+		u8 pad[TEGRA_IVC_ALIGN];
+	} tx;
+
+	union {
+		/* fields owned by the receiving end */
+		u32 count;
+		u8 pad[TEGRA_IVC_ALIGN];
+	} rx;
+};
+
+static inline void tegra_ivc_invalidate(struct tegra_ivc *ivc, dma_addr_t phys)
+{
+	if (!ivc->peer)
+		return;
+
+	dma_sync_single_for_cpu(ivc->peer, phys, TEGRA_IVC_ALIGN,
+				DMA_FROM_DEVICE);
+}
+
+static inline void tegra_ivc_flush(struct tegra_ivc *ivc, dma_addr_t phys)
+{
+	if (!ivc->peer)
+		return;
+
+	dma_sync_single_for_device(ivc->peer, phys, TEGRA_IVC_ALIGN,
+				   DMA_TO_DEVICE);
+}
+
+static inline bool tegra_ivc_empty(struct tegra_ivc *ivc,
+				   struct tegra_ivc_header *header)
+{
+	/*
+	 * This function performs multiple checks on the same values with
+	 * security implications, so create snapshots with ACCESS_ONCE() to
+	 * ensure that these checks use the same values.
+	 */
+	u32 tx = ACCESS_ONCE(header->tx.count);
+	u32 rx = ACCESS_ONCE(header->rx.count);
+
+	/*
+	 * Perform an over-full check to prevent denial of service attacks
+	 * where a server could be easily fooled into believing that there's
+	 * an extremely large number of frames ready, since receivers are not
+	 * expected to check for full or over-full conditions.
+	 *
+	 * Although the channel isn't empty, this is an invalid case caused by
+	 * a potentially malicious peer, so returning empty is safer, because
+	 * it gives the impression that the channel has gone silent.
+	 */
+	if (tx - rx > ivc->num_frames)
+		return true;
+
+	return tx == rx;
+}
+
+static inline bool tegra_ivc_full(struct tegra_ivc *ivc,
+				  struct tegra_ivc_header *header)
+{
+	u32 tx = ACCESS_ONCE(header->tx.count);
+	u32 rx = ACCESS_ONCE(header->rx.count);
+
+	/*
+	 * Invalid cases where the counters indicate that the queue is over
+	 * capacity also appear full.
+	 */
+	return tx - rx >= ivc->num_frames;
+}
+
+static inline u32 tegra_ivc_available(struct tegra_ivc *ivc,
+				      struct tegra_ivc_header *header)
+{
+	u32 tx = ACCESS_ONCE(header->tx.count);
+	u32 rx = ACCESS_ONCE(header->rx.count);
+
+	/*
+	 * This function isn't expected to be used in scenarios where an
+	 * over-full situation can lead to denial of service attacks. See the
+	 * comment in tegra_ivc_empty() for an explanation about special
+	 * over-full considerations.
+	 */
+	return tx - rx;
+}
+
+static inline void tegra_ivc_advance_tx(struct tegra_ivc *ivc)
+{
+	ACCESS_ONCE(ivc->tx.channel->tx.count) =
+		ACCESS_ONCE(ivc->tx.channel->tx.count) + 1;
+
+	if (ivc->tx.position == ivc->num_frames - 1)
+		ivc->tx.position = 0;
+	else
+		ivc->tx.position++;
+}
+
+static inline void tegra_ivc_advance_rx(struct tegra_ivc *ivc)
+{
+	ACCESS_ONCE(ivc->rx.channel->rx.count) =
+		ACCESS_ONCE(ivc->rx.channel->rx.count) + 1;
+
+	if (ivc->rx.position == ivc->num_frames - 1)
+		ivc->rx.position = 0;
+	else
+		ivc->rx.position++;
+}
+
+static inline int tegra_ivc_check_read(struct tegra_ivc *ivc)
+{
+	unsigned int offset = offsetof(struct tegra_ivc_header, tx.count);
+
+	/*
+	 * tx.channel->state is set locally, so it is not synchronized with
+	 * state from the remote peer. The remote peer cannot reset its
+	 * transmit counters until we've acknowledged its synchronization
+	 * request, so no additional synchronization is required because an
+	 * asynchronous transition of rx.channel->state to
+	 * TEGRA_IVC_STATE_ACK is not allowed.
+	 */
+	if (ivc->tx.channel->tx.state != TEGRA_IVC_STATE_ESTABLISHED)
+		return -ECONNRESET;
+
+	/*
+	 * Avoid unnecessary invalidations when performing repeated accesses
+	 * to an IVC channel by checking the old queue pointers first.
+	 *
+	 * Synchronization is only necessary when these pointers indicate
+	 * empty or full.
+	 */
+	if (!tegra_ivc_empty(ivc, ivc->rx.channel))
+		return 0;
+
+	tegra_ivc_invalidate(ivc, ivc->rx.phys + offset);
+
+	if (tegra_ivc_empty(ivc, ivc->rx.channel))
+		return -ENOSPC;
+
+	return 0;
+}
+
+static inline int tegra_ivc_check_write(struct tegra_ivc *ivc)
+{
+	unsigned int offset = offsetof(struct tegra_ivc_header, rx.count);
+
+	if (ivc->tx.channel->tx.state != TEGRA_IVC_STATE_ESTABLISHED)
+		return -ECONNRESET;
+
+	if (!tegra_ivc_full(ivc, ivc->tx.channel))
+		return 0;
+
+	tegra_ivc_invalidate(ivc, ivc->tx.phys + offset);
+
+	if (tegra_ivc_full(ivc, ivc->tx.channel))
+		return -ENOSPC;
+
+	return 0;
+}
+
+static void *tegra_ivc_frame_virt(struct tegra_ivc *ivc,
+				  struct tegra_ivc_header *header,
+				  unsigned int frame)
+{
+	if (WARN_ON(frame >= ivc->num_frames))
+		return ERR_PTR(-EINVAL);
+
+	return (void *)(header + 1) + ivc->frame_size * frame;
+}
+
+static inline dma_addr_t tegra_ivc_frame_phys(struct tegra_ivc *ivc,
+					      dma_addr_t phys,
+					      unsigned int frame)
+{
+	unsigned long offset;
+
+	offset = sizeof(struct tegra_ivc_header) + ivc->frame_size * frame;
+
+	return phys + offset;
+}
+
+static inline void tegra_ivc_invalidate_frame(struct tegra_ivc *ivc,
+					      dma_addr_t phys,
+					      unsigned int frame,
+					      unsigned int offset,
+					      size_t size)
+{
+	if (!ivc->peer || WARN_ON(frame >= ivc->num_frames))
+		return;
+
+	phys = tegra_ivc_frame_phys(ivc, phys, frame) + offset;
+
+	dma_sync_single_for_cpu(ivc->peer, phys, size, DMA_FROM_DEVICE);
+}
+
+static inline void tegra_ivc_flush_frame(struct tegra_ivc *ivc,
+					 dma_addr_t phys,
+					 unsigned int frame,
+					 unsigned int offset,
+					 size_t size)
+{
+	if (!ivc->peer || WARN_ON(frame >= ivc->num_frames))
+		return;
+
+	phys = tegra_ivc_frame_phys(ivc, phys, frame) + offset;
+
+	dma_sync_single_for_device(ivc->peer, phys, size, DMA_TO_DEVICE);
+}
+
+/* directly peek at the next frame rx'ed */
+void *tegra_ivc_read_get_next_frame(struct tegra_ivc *ivc)
+{
+	int err;
+
+	if (WARN_ON(ivc == NULL))
+		return ERR_PTR(-EINVAL);
+
+	err = tegra_ivc_check_read(ivc);
+	if (err < 0)
+		return ERR_PTR(err);
+
+	/*
+	 * Order observation of ivc->rx.position potentially indicating new
+	 * data before data read.
+	 */
+	smp_rmb();
+
+	tegra_ivc_invalidate_frame(ivc, ivc->rx.phys, ivc->rx.position, 0,
+				   ivc->frame_size);
+
+	return tegra_ivc_frame_virt(ivc, ivc->rx.channel, ivc->rx.position);
+}
+EXPORT_SYMBOL(tegra_ivc_read_get_next_frame);
+
+int tegra_ivc_read_advance(struct tegra_ivc *ivc)
+{
+	unsigned int rx = offsetof(struct tegra_ivc_header, rx.count);
+	unsigned int tx = offsetof(struct tegra_ivc_header, tx.count);
+	int err;
+
+	/*
+	 * No read barriers or synchronization here: the caller is expected to
+	 * have already observed the channel non-empty. This check is just to
+	 * catch programming errors.
+	 */
+	err = tegra_ivc_check_read(ivc);
+	if (err < 0)
+		return err;
+
+	tegra_ivc_advance_rx(ivc);
+
+	tegra_ivc_flush(ivc, ivc->rx.phys + rx);
+
+	/*
+	 * Ensure our write to ivc->rx.position occurs before our read from
+	 * ivc->tx.position.
+	 */
+	smp_mb();
+
+	/*
+	 * Notify only upon transition from full to non-full. The available
+	 * count can only asynchronously increase, so the worst possible
+	 * side-effect will be a spurious notification.
+	 */
+	tegra_ivc_invalidate(ivc, ivc->rx.phys + tx);
+
+	if (tegra_ivc_available(ivc, ivc->rx.channel) == ivc->num_frames - 1)
+		ivc->notify(ivc, ivc->notify_data);
+
+	return 0;
+}
+EXPORT_SYMBOL(tegra_ivc_read_advance);
+
+/* directly poke at the next frame to be tx'ed */
+void *tegra_ivc_write_get_next_frame(struct tegra_ivc *ivc)
+{
+	int err;
+
+	err = tegra_ivc_check_write(ivc);
+	if (err < 0)
+		return ERR_PTR(err);
+
+	return tegra_ivc_frame_virt(ivc, ivc->tx.channel, ivc->tx.position);
+}
+EXPORT_SYMBOL(tegra_ivc_write_get_next_frame);
+
+/* advance the tx buffer */
+int tegra_ivc_write_advance(struct tegra_ivc *ivc)
+{
+	unsigned int tx = offsetof(struct tegra_ivc_header, tx.count);
+	unsigned int rx = offsetof(struct tegra_ivc_header, rx.count);
+	int err;
+
+	err = tegra_ivc_check_write(ivc);
+	if (err < 0)
+		return err;
+
+	tegra_ivc_flush_frame(ivc, ivc->tx.phys, ivc->tx.position, 0,
+			      ivc->frame_size);
+
+	/*
+	 * Order any possible stores to the frame before update of
+	 * ivc->tx.position.
+	 */
+	smp_wmb();
+
+	tegra_ivc_advance_tx(ivc);
+	tegra_ivc_flush(ivc, ivc->tx.phys + tx);
+
+	/*
+	 * Ensure our write to ivc->tx.position occurs before our read from
+	 * ivc->rx.position.
+	 */
+	smp_mb();
+
+	/*
+	 * Notify only upon transition from empty to non-empty. The available
+	 * count can only asynchronously decrease, so the worst possible
+	 * side-effect will be a spurious notification.
+	 */
+	tegra_ivc_invalidate(ivc, ivc->tx.phys + rx);
+
+	if (tegra_ivc_available(ivc, ivc->tx.channel) == 1)
+		ivc->notify(ivc, ivc->notify_data);
+
+	return 0;
+}
+EXPORT_SYMBOL(tegra_ivc_write_advance);
+
+void tegra_ivc_reset(struct tegra_ivc *ivc)
+{
+	unsigned int offset = offsetof(struct tegra_ivc_header, tx.count);
+
+	ivc->tx.channel->tx.state = TEGRA_IVC_STATE_SYNC;
+	tegra_ivc_flush(ivc, ivc->tx.phys + offset);
+	ivc->notify(ivc, ivc->notify_data);
+}
+EXPORT_SYMBOL(tegra_ivc_reset);
+
+/*
+ * =======================================================
+ *  IVC State Transition Table - see tegra_ivc_notified()
+ * =======================================================
+ *
+ *	local	remote	action
+ *	-----	------	-----------------------------------
+ *	SYNC	EST	<none>
+ *	SYNC	ACK	reset counters; move to EST; notify
+ *	SYNC	SYNC	reset counters; move to ACK; notify
+ *	ACK	EST	move to EST; notify
+ *	ACK	ACK	move to EST; notify
+ *	ACK	SYNC	reset counters; move to ACK; notify
+ *	EST	EST	<none>
+ *	EST	ACK	<none>
+ *	EST	SYNC	reset counters; move to ACK; notify
+ *
+ * ===============================================================
+ */
+
+int tegra_ivc_notified(struct tegra_ivc *ivc)
+{
+	unsigned int offset = offsetof(struct tegra_ivc_header, tx.count);
+	enum tegra_ivc_state state;
+
+	/* Copy the receiver's state out of shared memory. */
+	tegra_ivc_invalidate(ivc, ivc->rx.phys + offset);
+	state = ACCESS_ONCE(ivc->rx.channel->tx.state);
+
+	if (state == TEGRA_IVC_STATE_SYNC) {
+		offset = offsetof(struct tegra_ivc_header, tx.count);
+
+		/*
+		 * Order observation of TEGRA_IVC_STATE_SYNC before stores
+		 * clearing tx.channel.
+		 */
+		smp_rmb();
+
+		/*
+		 * Reset tx.channel counters. The remote end is in the SYNC
+		 * state and won't make progress until we change our state,
+		 * so the counters are not in use at this time.
+		 */
+		ivc->tx.channel->tx.count = 0;
+		ivc->rx.channel->rx.count = 0;
+
+		ivc->tx.position = 0;
+		ivc->rx.position = 0;
+
+		/*
+		 * Ensure that counters appear cleared before new state can be
+		 * observed.
+		 */
+		smp_wmb();
+
+		/*
+		 * Move to ACK state. We have just cleared our counters, so it
+		 * is now safe for the remote end to start using these values.
+		 */
+		ivc->tx.channel->tx.state = TEGRA_IVC_STATE_ACK;
+		tegra_ivc_flush(ivc, ivc->tx.phys + offset);
+
+		/*
+		 * Notify remote end to observe state transition.
+		 */
+		ivc->notify(ivc, ivc->notify_data);
+
+	} else if (ivc->tx.channel->tx.state == TEGRA_IVC_STATE_SYNC &&
+		   state == TEGRA_IVC_STATE_ACK) {
+		offset = offsetof(struct tegra_ivc_header, tx.count);
+
+		/*
+		 * Order observation of ivc_state_sync before stores clearing
+		 * tx_channel.
+		 */
+		smp_rmb();
+
+		/*
+		 * Reset tx.channel counters. The remote end is in the ACK
+		 * state and won't make progress until we change our state,
+		 * so the counters are not in use at this time.
+		 */
+		ivc->tx.channel->tx.count = 0;
+		ivc->rx.channel->rx.count = 0;
+
+		ivc->tx.position = 0;
+		ivc->rx.position = 0;
+
+		/*
+		 * Ensure that counters appear cleared before new state can be
+		 * observed.
+		 */
+		smp_wmb();
+
+		/*
+		 * Move to ESTABLISHED state. We know that the remote end has
+		 * already cleared its counters, so it is safe to start
+		 * writing/reading on this channel.
+		 */
+		ivc->tx.channel->tx.state = TEGRA_IVC_STATE_ESTABLISHED;
+		tegra_ivc_flush(ivc, ivc->tx.phys + offset);
+
+		/*
+		 * Notify remote end to observe state transition.
+		 */
+		ivc->notify(ivc, ivc->notify_data);
+
+	} else if (ivc->tx.channel->tx.state == TEGRA_IVC_STATE_ACK) {
+		offset = offsetof(struct tegra_ivc_header, tx.count);
+
+		/*
+		 * At this point, we have observed the peer to be in either
+		 * the ACK or ESTABLISHED state. Next, order observation of
+		 * peer state before storing to tx.channel.
+		 */
+		smp_rmb();
+
+		/*
+		 * Move to ESTABLISHED state. We know that we have previously
+		 * cleared our counters, and we know that the remote end has
+		 * cleared its counters, so it is safe to start writing/reading
+		 * on this channel.
+		 */
+		ivc->tx.channel->tx.state = TEGRA_IVC_STATE_ESTABLISHED;
+		tegra_ivc_flush(ivc, ivc->tx.phys + offset);
+
+		/*
+		 * Notify remote end to observe state transition.
+		 */
+		ivc->notify(ivc, ivc->notify_data);
+
+	} else {
+		/*
+		 * There is no need to handle any further action. Either the
+		 * channel is already fully established, or we are waiting for
+		 * the remote end to catch up with our current state. Refer
+		 * to the diagram in "IVC State Transition Table" above.
+		 */
+	}
+
+	if (ivc->tx.channel->tx.state != TEGRA_IVC_STATE_ESTABLISHED)
+		return -EAGAIN;
+
+	return 0;
+}
+EXPORT_SYMBOL(tegra_ivc_notified);
+
+size_t tegra_ivc_align(size_t size)
+{
+	return ALIGN(size, TEGRA_IVC_ALIGN);
+}
+EXPORT_SYMBOL(tegra_ivc_align);
+
+unsigned tegra_ivc_total_queue_size(unsigned queue_size)
+{
+	if (!IS_ALIGNED(queue_size, TEGRA_IVC_ALIGN)) {
+		pr_err("%s: queue_size (%u) must be %u-byte aligned\n",
+		       __func__, queue_size, TEGRA_IVC_ALIGN);
+		return 0;
+	}
+
+	return queue_size + sizeof(struct tegra_ivc_header);
+}
+EXPORT_SYMBOL(tegra_ivc_total_queue_size);
+
+static int tegra_ivc_check_params(unsigned long rx, unsigned long tx,
+				  unsigned int num_frames, size_t frame_size)
+{
+	BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct tegra_ivc_header, tx.count),
+				 TEGRA_IVC_ALIGN));
+	BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct tegra_ivc_header, rx.count),
+				 TEGRA_IVC_ALIGN));
+	BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct tegra_ivc_header),
+				 TEGRA_IVC_ALIGN));
+
+	if ((uint64_t)num_frames * (uint64_t)frame_size >= 0x100000000UL) {
+		pr_err("num_frames * frame_size overflows\n");
+		return -EINVAL;
+	}
+
+	if (!IS_ALIGNED(frame_size, TEGRA_IVC_ALIGN)) {
+		pr_err("frame size not adequately aligned: %zu\n", frame_size);
+		return -EINVAL;
+	}
+
+	/*
+	 * The headers must at least be aligned enough for counters
+	 * to be accessed atomically.
+	 */
+	if (!IS_ALIGNED(rx, TEGRA_IVC_ALIGN)) {
+		pr_err("IVC channel start not aligned: %#lx\n", rx);
+		return -EINVAL;
+	}
+
+	if (!IS_ALIGNED(tx, TEGRA_IVC_ALIGN)) {
+		pr_err("IVC channel start not aligned: %#lx\n", tx);
+		return -EINVAL;
+	}
+
+	if (rx < tx) {
+		if (rx + frame_size * num_frames > tx) {
+			pr_err("queue regions overlap: %#lx + %zx > %#lx\n",
+			       rx, frame_size * num_frames, tx);
+			return -EINVAL;
+		}
+	} else {
+		if (tx + frame_size * num_frames > rx) {
+			pr_err("queue regions overlap: %#lx + %zx > %#lx\n",
+			       tx, frame_size * num_frames, rx);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+int tegra_ivc_init(struct tegra_ivc *ivc, struct device *peer, void *rx,
+		   dma_addr_t rx_phys, void *tx, dma_addr_t tx_phys,
+		   unsigned int num_frames, size_t frame_size,
+		   void (*notify)(struct tegra_ivc *ivc, void *data),
+		   void *data)
+{
+	size_t queue_size;
+	int err;
+
+	if (WARN_ON(!ivc || !notify))
+		return -EINVAL;
+
+	/*
+	 * All sizes that can be returned by communication functions should
+	 * fit in an int.
+	 */
+	if (frame_size > INT_MAX)
+		return -E2BIG;
+
+	err = tegra_ivc_check_params((unsigned long)rx, (unsigned long)tx,
+				     num_frames, frame_size);
+	if (err < 0)
+		return err;
+
+	queue_size = tegra_ivc_total_queue_size(num_frames * frame_size);
+
+	if (peer) {
+		ivc->rx.phys = dma_map_single(peer, rx, queue_size,
+					      DMA_BIDIRECTIONAL);
+		if (ivc->rx.phys == DMA_ERROR_CODE)
+			return -ENOMEM;
+
+		ivc->tx.phys = dma_map_single(peer, tx, queue_size,
+					      DMA_BIDIRECTIONAL);
+		if (ivc->tx.phys == DMA_ERROR_CODE) {
+			dma_unmap_single(peer, ivc->rx.phys, queue_size,
+					 DMA_BIDIRECTIONAL);
+			return -ENOMEM;
+		}
+	} else {
+		ivc->rx.phys = rx_phys;
+		ivc->tx.phys = tx_phys;
+	}
+
+	ivc->rx.channel = rx;
+	ivc->tx.channel = tx;
+	ivc->peer = peer;
+	ivc->notify = notify;
+	ivc->notify_data = data;
+	ivc->frame_size = frame_size;
+	ivc->num_frames = num_frames;
+
+	/*
+	 * These values aren't necessarily correct until the channel has been
+	 * reset.
+	 */
+	ivc->tx.position = 0;
+	ivc->rx.position = 0;
+
+	return 0;
+}
+EXPORT_SYMBOL(tegra_ivc_init);
+
+void tegra_ivc_cleanup(struct tegra_ivc *ivc)
+{
+	if (ivc->peer) {
+		size_t size = tegra_ivc_total_queue_size(ivc->num_frames *
+							 ivc->frame_size);
+
+		dma_unmap_single(ivc->peer, ivc->rx.phys, size,
+				 DMA_BIDIRECTIONAL);
+		dma_unmap_single(ivc->peer, ivc->tx.phys, size,
+				 DMA_BIDIRECTIONAL);
+	}
+}
+EXPORT_SYMBOL(tegra_ivc_cleanup);
diff --git a/drivers/firmware/ti_sci.c b/drivers/firmware/ti_sci.c
new file mode 100644
index 000000000000..874ff32db366
--- /dev/null
+++ b/drivers/firmware/ti_sci.c
@@ -0,0 +1,1991 @@
+/*
+ * Texas Instruments System Control Interface Protocol Driver
+ *
+ * Copyright (C) 2015-2016 Texas Instruments Incorporated - http://www.ti.com/
+ *	Nishanth Menon
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) "%s: " fmt, __func__
+
+#include <linux/bitmap.h>
+#include <linux/debugfs.h>
+#include <linux/export.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/semaphore.h>
+#include <linux/slab.h>
+#include <linux/soc/ti/ti-msgmgr.h>
+#include <linux/soc/ti/ti_sci_protocol.h>
+#include <linux/reboot.h>
+
+#include "ti_sci.h"
+
+/* List of all TI SCI devices active in system */
+static LIST_HEAD(ti_sci_list);
+/* Protection for the entire list */
+static DEFINE_MUTEX(ti_sci_list_mutex);
+
+/**
+ * struct ti_sci_xfer - Structure representing a message flow
+ * @tx_message:	Transmit message
+ * @rx_len:	Receive message length
+ * @xfer_buf:	Preallocated buffer to store receive message
+ *		Since we work with request-ACK protocol, we can
+ *		reuse the same buffer for the rx path as we
+ *		use for the tx path.
+ * @done:	completion event
+ */
+struct ti_sci_xfer {
+	struct ti_msgmgr_message tx_message;
+	u8 rx_len;
+	u8 *xfer_buf;
+	struct completion done;
+};
+
+/**
+ * struct ti_sci_xfers_info - Structure to manage transfer information
+ * @sem_xfer_count:	Counting Semaphore for managing max simultaneous
+ *			Messages.
+ * @xfer_block:		Preallocated Message array
+ * @xfer_alloc_table:	Bitmap table for allocated messages.
+ *			Index of this bitmap table is also used for message
+ *			sequence identifier.
+ * @xfer_lock:		Protection for message allocation
+ */
+struct ti_sci_xfers_info {
+	struct semaphore sem_xfer_count;
+	struct ti_sci_xfer *xfer_block;
+	unsigned long *xfer_alloc_table;
+	/* protect transfer allocation */
+	spinlock_t xfer_lock;
+};
+
+/**
+ * struct ti_sci_desc - Description of SoC integration
+ * @host_id:		Host identifier representing the compute entity
+ * @max_rx_timeout_ms:	Timeout for communication with SoC (in Milliseconds)
+ * @max_msgs: Maximum number of messages that can be pending
+ *		  simultaneously in the system
+ * @max_msg_size: Maximum size of data per message that can be handled.
+ */
+struct ti_sci_desc {
+	u8 host_id;
+	int max_rx_timeout_ms;
+	int max_msgs;
+	int max_msg_size;
+};
+
+/**
+ * struct ti_sci_info - Structure representing a TI SCI instance
+ * @dev:	Device pointer
+ * @desc:	SoC description for this instance
+ * @nb:	Reboot Notifier block
+ * @d:		Debugfs file entry
+ * @debug_region: Memory region where the debug message are available
+ * @debug_region_size: Debug region size
+ * @debug_buffer: Buffer allocated to copy debug messages.
+ * @handle:	Instance of TI SCI handle to send to clients.
+ * @cl:		Mailbox Client
+ * @chan_tx:	Transmit mailbox channel
+ * @chan_rx:	Receive mailbox channel
+ * @minfo:	Message info
+ * @node:	list head
+ * @users:	Number of users of this instance
+ */
+struct ti_sci_info {
+	struct device *dev;
+	struct notifier_block nb;
+	const struct ti_sci_desc *desc;
+	struct dentry *d;
+	void __iomem *debug_region;
+	char *debug_buffer;
+	size_t debug_region_size;
+	struct ti_sci_handle handle;
+	struct mbox_client cl;
+	struct mbox_chan *chan_tx;
+	struct mbox_chan *chan_rx;
+	struct ti_sci_xfers_info minfo;
+	struct list_head node;
+	/* protected by ti_sci_list_mutex */
+	int users;
+
+};
+
+#define cl_to_ti_sci_info(c)	container_of(c, struct ti_sci_info, cl)
+#define handle_to_ti_sci_info(h) container_of(h, struct ti_sci_info, handle)
+#define reboot_to_ti_sci_info(n) container_of(n, struct ti_sci_info, nb)
+
+#ifdef CONFIG_DEBUG_FS
+
+/**
+ * ti_sci_debug_show() - Helper to dump the debug log
+ * @s:	sequence file pointer
+ * @unused:	unused.
+ *
+ * Return: 0
+ */
+static int ti_sci_debug_show(struct seq_file *s, void *unused)
+{
+	struct ti_sci_info *info = s->private;
+
+	memcpy_fromio(info->debug_buffer, info->debug_region,
+		      info->debug_region_size);
+	/*
+	 * We don't trust firmware to leave NULL terminated last byte (hence
+	 * we have allocated 1 extra 0 byte). Since we cannot guarantee any
+	 * specific data format for debug messages, We just present the data
+	 * in the buffer as is - we expect the messages to be self explanatory.
+	 */
+	seq_puts(s, info->debug_buffer);
+	return 0;
+}
+
+/**
+ * ti_sci_debug_open() - debug file open
+ * @inode:	inode pointer
+ * @file:	file pointer
+ *
+ * Return: result of single_open
+ */
+static int ti_sci_debug_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, ti_sci_debug_show, inode->i_private);
+}
+
+/* log file operations */
+static const struct file_operations ti_sci_debug_fops = {
+	.open = ti_sci_debug_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+/**
+ * ti_sci_debugfs_create() - Create log debug file
+ * @pdev:	platform device pointer
+ * @info:	Pointer to SCI entity information
+ *
+ * Return: 0 if all went fine, else corresponding error.
+ */
+static int ti_sci_debugfs_create(struct platform_device *pdev,
+				 struct ti_sci_info *info)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	char debug_name[50] = "ti_sci_debug@";
+
+	/* Debug region is optional */
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+					   "debug_messages");
+	info->debug_region = devm_ioremap_resource(dev, res);
+	if (IS_ERR(info->debug_region))
+		return 0;
+	info->debug_region_size = resource_size(res);
+
+	info->debug_buffer = devm_kcalloc(dev, info->debug_region_size + 1,
+					  sizeof(char), GFP_KERNEL);
+	if (!info->debug_buffer)
+		return -ENOMEM;
+	/* Setup NULL termination */
+	info->debug_buffer[info->debug_region_size] = 0;
+
+	info->d = debugfs_create_file(strncat(debug_name, dev_name(dev),
+					      sizeof(debug_name)),
+				      0444, NULL, info, &ti_sci_debug_fops);
+	if (IS_ERR(info->d))
+		return PTR_ERR(info->d);
+
+	dev_dbg(dev, "Debug region => %p, size = %zu bytes, resource: %pr\n",
+		info->debug_region, info->debug_region_size, res);
+	return 0;
+}
+
+/**
+ * ti_sci_debugfs_destroy() - clean up log debug file
+ * @pdev:	platform device pointer
+ * @info:	Pointer to SCI entity information
+ */
+static void ti_sci_debugfs_destroy(struct platform_device *pdev,
+				   struct ti_sci_info *info)
+{
+	if (IS_ERR(info->debug_region))
+		return;
+
+	debugfs_remove(info->d);
+}
+#else /* CONFIG_DEBUG_FS */
+static inline int ti_sci_debugfs_create(struct platform_device *dev,
+					struct ti_sci_info *info)
+{
+	return 0;
+}
+
+static inline void ti_sci_debugfs_destroy(struct platform_device *dev,
+					  struct ti_sci_info *info)
+{
+}
+#endif /* CONFIG_DEBUG_FS */
+
+/**
+ * ti_sci_dump_header_dbg() - Helper to dump a message header.
+ * @dev:	Device pointer corresponding to the SCI entity
+ * @hdr:	pointer to header.
+ */
+static inline void ti_sci_dump_header_dbg(struct device *dev,
+					  struct ti_sci_msg_hdr *hdr)
+{
+	dev_dbg(dev, "MSGHDR:type=0x%04x host=0x%02x seq=0x%02x flags=0x%08x\n",
+		hdr->type, hdr->host, hdr->seq, hdr->flags);
+}
+
+/**
+ * ti_sci_rx_callback() - mailbox client callback for receive messages
+ * @cl:	client pointer
+ * @m:	mailbox message
+ *
+ * Processes one received message to appropriate transfer information and
+ * signals completion of the transfer.
+ *
+ * NOTE: This function will be invoked in IRQ context, hence should be
+ * as optimal as possible.
+ */
+static void ti_sci_rx_callback(struct mbox_client *cl, void *m)
+{
+	struct ti_sci_info *info = cl_to_ti_sci_info(cl);
+	struct device *dev = info->dev;
+	struct ti_sci_xfers_info *minfo = &info->minfo;
+	struct ti_msgmgr_message *mbox_msg = m;
+	struct ti_sci_msg_hdr *hdr = (struct ti_sci_msg_hdr *)mbox_msg->buf;
+	struct ti_sci_xfer *xfer;
+	u8 xfer_id;
+
+	xfer_id = hdr->seq;
+
+	/*
+	 * Are we even expecting this?
+	 * NOTE: barriers were implicit in locks used for modifying the bitmap
+	 */
+	if (!test_bit(xfer_id, minfo->xfer_alloc_table)) {
+		dev_err(dev, "Message for %d is not expected!\n", xfer_id);
+		return;
+	}
+
+	xfer = &minfo->xfer_block[xfer_id];
+
+	/* Is the message of valid length? */
+	if (mbox_msg->len > info->desc->max_msg_size) {
+		dev_err(dev, "Unable to handle %d xfer(max %d)\n",
+			mbox_msg->len, info->desc->max_msg_size);
+		ti_sci_dump_header_dbg(dev, hdr);
+		return;
+	}
+	if (mbox_msg->len < xfer->rx_len) {
+		dev_err(dev, "Recv xfer %d < expected %d length\n",
+			mbox_msg->len, xfer->rx_len);
+		ti_sci_dump_header_dbg(dev, hdr);
+		return;
+	}
+
+	ti_sci_dump_header_dbg(dev, hdr);
+	/* Take a copy to the rx buffer.. */
+	memcpy(xfer->xfer_buf, mbox_msg->buf, xfer->rx_len);
+	complete(&xfer->done);
+}
+
+/**
+ * ti_sci_get_one_xfer() - Allocate one message
+ * @info:	Pointer to SCI entity information
+ * @msg_type:	Message type
+ * @msg_flags:	Flag to set for the message
+ * @tx_message_size: transmit message size
+ * @rx_message_size: receive message size
+ *
+ * Helper function which is used by various command functions that are
+ * exposed to clients of this driver for allocating a message traffic event.
+ *
+ * This function can sleep depending on pending requests already in the system
+ * for the SCI entity. Further, this also holds a spinlock to maintain integrity
+ * of internal data structures.
+ *
+ * Return: 0 if all went fine, else corresponding error.
+ */
+static struct ti_sci_xfer *ti_sci_get_one_xfer(struct ti_sci_info *info,
+					       u16 msg_type, u32 msg_flags,
+					       size_t tx_message_size,
+					       size_t rx_message_size)
+{
+	struct ti_sci_xfers_info *minfo = &info->minfo;
+	struct ti_sci_xfer *xfer;
+	struct ti_sci_msg_hdr *hdr;
+	unsigned long flags;
+	unsigned long bit_pos;
+	u8 xfer_id;
+	int ret;
+	int timeout;
+
+	/* Ensure we have sane transfer sizes */
+	if (rx_message_size > info->desc->max_msg_size ||
+	    tx_message_size > info->desc->max_msg_size ||
+	    rx_message_size < sizeof(*hdr) || tx_message_size < sizeof(*hdr))
+		return ERR_PTR(-ERANGE);
+
+	/*
+	 * Ensure we have only controlled number of pending messages.
+	 * Ideally, we might just have to wait a single message, be
+	 * conservative and wait 5 times that..
+	 */
+	timeout = msecs_to_jiffies(info->desc->max_rx_timeout_ms) * 5;
+	ret = down_timeout(&minfo->sem_xfer_count, timeout);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	/* Keep the locked section as small as possible */
+	spin_lock_irqsave(&minfo->xfer_lock, flags);
+	bit_pos = find_first_zero_bit(minfo->xfer_alloc_table,
+				      info->desc->max_msgs);
+	set_bit(bit_pos, minfo->xfer_alloc_table);
+	spin_unlock_irqrestore(&minfo->xfer_lock, flags);
+
+	/*
+	 * We already ensured in probe that we can have max messages that can
+	 * fit in  hdr.seq - NOTE: this improves access latencies
+	 * to predictable O(1) access, BUT, it opens us to risk if
+	 * remote misbehaves with corrupted message sequence responses.
+	 * If that happens, we are going to be messed up anyways..
+	 */
+	xfer_id = (u8)bit_pos;
+
+	xfer = &minfo->xfer_block[xfer_id];
+
+	hdr = (struct ti_sci_msg_hdr *)xfer->tx_message.buf;
+	xfer->tx_message.len = tx_message_size;
+	xfer->rx_len = (u8)rx_message_size;
+
+	reinit_completion(&xfer->done);
+
+	hdr->seq = xfer_id;
+	hdr->type = msg_type;
+	hdr->host = info->desc->host_id;
+	hdr->flags = msg_flags;
+
+	return xfer;
+}
+
+/**
+ * ti_sci_put_one_xfer() - Release a message
+ * @minfo:	transfer info pointer
+ * @xfer:	message that was reserved by ti_sci_get_one_xfer
+ *
+ * This holds a spinlock to maintain integrity of internal data structures.
+ */
+static void ti_sci_put_one_xfer(struct ti_sci_xfers_info *minfo,
+				struct ti_sci_xfer *xfer)
+{
+	unsigned long flags;
+	struct ti_sci_msg_hdr *hdr;
+	u8 xfer_id;
+
+	hdr = (struct ti_sci_msg_hdr *)xfer->tx_message.buf;
+	xfer_id = hdr->seq;
+
+	/*
+	 * Keep the locked section as small as possible
+	 * NOTE: we might escape with smp_mb and no lock here..
+	 * but just be conservative and symmetric.
+	 */
+	spin_lock_irqsave(&minfo->xfer_lock, flags);
+	clear_bit(xfer_id, minfo->xfer_alloc_table);
+	spin_unlock_irqrestore(&minfo->xfer_lock, flags);
+
+	/* Increment the count for the next user to get through */
+	up(&minfo->sem_xfer_count);
+}
+
+/**
+ * ti_sci_do_xfer() - Do one transfer
+ * @info:	Pointer to SCI entity information
+ * @xfer:	Transfer to initiate and wait for response
+ *
+ * Return: -ETIMEDOUT in case of no response, if transmit error,
+ *	   return corresponding error, else if all goes well,
+ *	   return 0.
+ */
+static inline int ti_sci_do_xfer(struct ti_sci_info *info,
+				 struct ti_sci_xfer *xfer)
+{
+	int ret;
+	int timeout;
+	struct device *dev = info->dev;
+
+	ret = mbox_send_message(info->chan_tx, &xfer->tx_message);
+	if (ret < 0)
+		return ret;
+
+	ret = 0;
+
+	/* And we wait for the response. */
+	timeout = msecs_to_jiffies(info->desc->max_rx_timeout_ms);
+	if (!wait_for_completion_timeout(&xfer->done, timeout)) {
+		dev_err(dev, "Mbox timedout in resp(caller: %pF)\n",
+			(void *)_RET_IP_);
+		ret = -ETIMEDOUT;
+	}
+	/*
+	 * NOTE: we might prefer not to need the mailbox ticker to manage the
+	 * transfer queueing since the protocol layer queues things by itself.
+	 * Unfortunately, we have to kick the mailbox framework after we have
+	 * received our message.
+	 */
+	mbox_client_txdone(info->chan_tx, ret);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_get_revision() - command to get the revision of the SCI entity
+ * @info:	Pointer to SCI entity information
+ *
+ * Updates the SCI information in the internal data structure.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_get_revision(struct ti_sci_info *info)
+{
+	struct device *dev = info->dev;
+	struct ti_sci_handle *handle = &info->handle;
+	struct ti_sci_version_info *ver = &handle->version;
+	struct ti_sci_msg_resp_version *rev_info;
+	struct ti_sci_xfer *xfer;
+	int ret;
+
+	/* No need to setup flags since it is expected to respond */
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_VERSION,
+				   0x0, sizeof(struct ti_sci_msg_hdr),
+				   sizeof(*rev_info));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+
+	rev_info = (struct ti_sci_msg_resp_version *)xfer->xfer_buf;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	ver->abi_major = rev_info->abi_major;
+	ver->abi_minor = rev_info->abi_minor;
+	ver->firmware_revision = rev_info->firmware_revision;
+	strncpy(ver->firmware_description, rev_info->firmware_description,
+		sizeof(ver->firmware_description));
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+	return ret;
+}
+
+/**
+ * ti_sci_is_response_ack() - Generic ACK/NACK message checkup
+ * @r:	pointer to response buffer
+ *
+ * Return: true if the response was an ACK, else returns false.
+ */
+static inline bool ti_sci_is_response_ack(void *r)
+{
+	struct ti_sci_msg_hdr *hdr = r;
+
+	return hdr->flags & TI_SCI_FLAG_RESP_GENERIC_ACK ? true : false;
+}
+
+/**
+ * ti_sci_set_device_state() - Set device state helper
+ * @handle:	pointer to TI SCI handle
+ * @id:		Device identifier
+ * @flags:	flags to setup for the device
+ * @state:	State to move the device to
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_set_device_state(const struct ti_sci_handle *handle,
+				   u32 id, u32 flags, u8 state)
+{
+	struct ti_sci_info *info;
+	struct ti_sci_msg_req_set_device_state *req;
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_SET_DEVICE_STATE,
+				   flags | TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_set_device_state *)xfer->xfer_buf;
+	req->id = id;
+	req->state = state;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+
+	ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_get_device_state() - Get device state helper
+ * @handle:	Handle to the device
+ * @id:		Device Identifier
+ * @clcnt:	Pointer to Context Loss Count
+ * @resets:	pointer to resets
+ * @p_state:	pointer to p_state
+ * @c_state:	pointer to c_state
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_get_device_state(const struct ti_sci_handle *handle,
+				   u32 id,  u32 *clcnt,  u32 *resets,
+				    u8 *p_state,  u8 *c_state)
+{
+	struct ti_sci_info *info;
+	struct ti_sci_msg_req_get_device_state *req;
+	struct ti_sci_msg_resp_get_device_state *resp;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	if (!clcnt && !resets && !p_state && !c_state)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	/* Response is expected, so need of any flags */
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_GET_DEVICE_STATE,
+				   0, sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_get_device_state *)xfer->xfer_buf;
+	req->id = id;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_resp_get_device_state *)xfer->xfer_buf;
+	if (!ti_sci_is_response_ack(resp)) {
+		ret = -ENODEV;
+		goto fail;
+	}
+
+	if (clcnt)
+		*clcnt = resp->context_loss_count;
+	if (resets)
+		*resets = resp->resets;
+	if (p_state)
+		*p_state = resp->programmed_state;
+	if (c_state)
+		*c_state = resp->current_state;
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_get_device() - command to request for device managed by TISCI
+ * @handle:	Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * @id:		Device Identifier
+ *
+ * Request for the device - NOTE: the client MUST maintain integrity of
+ * usage count by balancing get_device with put_device. No refcounting is
+ * managed by driver for that purpose.
+ *
+ * NOTE: The request is for exclusive access for the processor.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_get_device(const struct ti_sci_handle *handle, u32 id)
+{
+	return ti_sci_set_device_state(handle, id,
+				       MSG_FLAG_DEVICE_EXCLUSIVE,
+				       MSG_DEVICE_SW_STATE_ON);
+}
+
+/**
+ * ti_sci_cmd_idle_device() - Command to idle a device managed by TISCI
+ * @handle:	Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * @id:		Device Identifier
+ *
+ * Request for the device - NOTE: the client MUST maintain integrity of
+ * usage count by balancing get_device with put_device. No refcounting is
+ * managed by driver for that purpose.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_idle_device(const struct ti_sci_handle *handle, u32 id)
+{
+	return ti_sci_set_device_state(handle, id,
+				       MSG_FLAG_DEVICE_EXCLUSIVE,
+				       MSG_DEVICE_SW_STATE_RETENTION);
+}
+
+/**
+ * ti_sci_cmd_put_device() - command to release a device managed by TISCI
+ * @handle:	Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * @id:		Device Identifier
+ *
+ * Request for the device - NOTE: the client MUST maintain integrity of
+ * usage count by balancing get_device with put_device. No refcounting is
+ * managed by driver for that purpose.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_put_device(const struct ti_sci_handle *handle, u32 id)
+{
+	return ti_sci_set_device_state(handle, id,
+				       0, MSG_DEVICE_SW_STATE_AUTO_OFF);
+}
+
+/**
+ * ti_sci_cmd_dev_is_valid() - Is the device valid
+ * @handle:	Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * @id:		Device Identifier
+ *
+ * Return: 0 if all went fine and the device ID is valid, else return
+ * appropriate error.
+ */
+static int ti_sci_cmd_dev_is_valid(const struct ti_sci_handle *handle, u32 id)
+{
+	u8 unused;
+
+	/* check the device state which will also tell us if the ID is valid */
+	return ti_sci_get_device_state(handle, id, NULL, NULL, NULL, &unused);
+}
+
+/**
+ * ti_sci_cmd_dev_get_clcnt() - Get context loss counter
+ * @handle:	Pointer to TISCI handle
+ * @id:		Device Identifier
+ * @count:	Pointer to Context Loss counter to populate
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_dev_get_clcnt(const struct ti_sci_handle *handle, u32 id,
+				    u32 *count)
+{
+	return ti_sci_get_device_state(handle, id, count, NULL, NULL, NULL);
+}
+
+/**
+ * ti_sci_cmd_dev_is_idle() - Check if the device is requested to be idle
+ * @handle:	Pointer to TISCI handle
+ * @id:		Device Identifier
+ * @r_state:	true if requested to be idle
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_dev_is_idle(const struct ti_sci_handle *handle, u32 id,
+				  bool *r_state)
+{
+	int ret;
+	u8 state;
+
+	if (!r_state)
+		return -EINVAL;
+
+	ret = ti_sci_get_device_state(handle, id, NULL, NULL, &state, NULL);
+	if (ret)
+		return ret;
+
+	*r_state = (state == MSG_DEVICE_SW_STATE_RETENTION);
+
+	return 0;
+}
+
+/**
+ * ti_sci_cmd_dev_is_stop() - Check if the device is requested to be stopped
+ * @handle:	Pointer to TISCI handle
+ * @id:		Device Identifier
+ * @r_state:	true if requested to be stopped
+ * @curr_state:	true if currently stopped.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_dev_is_stop(const struct ti_sci_handle *handle, u32 id,
+				  bool *r_state,  bool *curr_state)
+{
+	int ret;
+	u8 p_state, c_state;
+
+	if (!r_state && !curr_state)
+		return -EINVAL;
+
+	ret =
+	    ti_sci_get_device_state(handle, id, NULL, NULL, &p_state, &c_state);
+	if (ret)
+		return ret;
+
+	if (r_state)
+		*r_state = (p_state == MSG_DEVICE_SW_STATE_AUTO_OFF);
+	if (curr_state)
+		*curr_state = (c_state == MSG_DEVICE_HW_STATE_OFF);
+
+	return 0;
+}
+
+/**
+ * ti_sci_cmd_dev_is_on() - Check if the device is requested to be ON
+ * @handle:	Pointer to TISCI handle
+ * @id:		Device Identifier
+ * @r_state:	true if requested to be ON
+ * @curr_state:	true if currently ON and active
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_dev_is_on(const struct ti_sci_handle *handle, u32 id,
+				bool *r_state,  bool *curr_state)
+{
+	int ret;
+	u8 p_state, c_state;
+
+	if (!r_state && !curr_state)
+		return -EINVAL;
+
+	ret =
+	    ti_sci_get_device_state(handle, id, NULL, NULL, &p_state, &c_state);
+	if (ret)
+		return ret;
+
+	if (r_state)
+		*r_state = (p_state == MSG_DEVICE_SW_STATE_ON);
+	if (curr_state)
+		*curr_state = (c_state == MSG_DEVICE_HW_STATE_ON);
+
+	return 0;
+}
+
+/**
+ * ti_sci_cmd_dev_is_trans() - Check if the device is currently transitioning
+ * @handle:	Pointer to TISCI handle
+ * @id:		Device Identifier
+ * @curr_state:	true if currently transitioning.
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_dev_is_trans(const struct ti_sci_handle *handle, u32 id,
+				   bool *curr_state)
+{
+	int ret;
+	u8 state;
+
+	if (!curr_state)
+		return -EINVAL;
+
+	ret = ti_sci_get_device_state(handle, id, NULL, NULL, NULL, &state);
+	if (ret)
+		return ret;
+
+	*curr_state = (state == MSG_DEVICE_HW_STATE_TRANS);
+
+	return 0;
+}
+
+/**
+ * ti_sci_cmd_set_device_resets() - command to set resets for device managed
+ *				    by TISCI
+ * @handle:	Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * @id:		Device Identifier
+ * @reset_state: Device specific reset bit field
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_set_device_resets(const struct ti_sci_handle *handle,
+					u32 id, u32 reset_state)
+{
+	struct ti_sci_info *info;
+	struct ti_sci_msg_req_set_device_resets *req;
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_SET_DEVICE_RESETS,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_set_device_resets *)xfer->xfer_buf;
+	req->id = id;
+	req->resets = reset_state;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+
+	ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_get_device_resets() - Get reset state for device managed
+ *				    by TISCI
+ * @handle:		Pointer to TISCI handle
+ * @id:			Device Identifier
+ * @reset_state:	Pointer to reset state to populate
+ *
+ * Return: 0 if all went fine, else return appropriate error.
+ */
+static int ti_sci_cmd_get_device_resets(const struct ti_sci_handle *handle,
+					u32 id, u32 *reset_state)
+{
+	return ti_sci_get_device_state(handle, id, NULL, reset_state, NULL,
+				       NULL);
+}
+
+/**
+ * ti_sci_set_clock_state() - Set clock state helper
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @flags:	Header flags as needed
+ * @state:	State to request for the clock.
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_set_clock_state(const struct ti_sci_handle *handle,
+				  u32 dev_id, u8 clk_id,
+				  u32 flags, u8 state)
+{
+	struct ti_sci_info *info;
+	struct ti_sci_msg_req_set_clock_state *req;
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_SET_CLOCK_STATE,
+				   flags | TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_set_clock_state *)xfer->xfer_buf;
+	req->dev_id = dev_id;
+	req->clk_id = clk_id;
+	req->request_state = state;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+
+	ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_get_clock_state() - Get clock state helper
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @programmed_state:	State requested for clock to move to
+ * @current_state:	State that the clock is currently in
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_get_clock_state(const struct ti_sci_handle *handle,
+				      u32 dev_id, u8 clk_id,
+				      u8 *programmed_state, u8 *current_state)
+{
+	struct ti_sci_info *info;
+	struct ti_sci_msg_req_get_clock_state *req;
+	struct ti_sci_msg_resp_get_clock_state *resp;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	if (!programmed_state && !current_state)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_GET_CLOCK_STATE,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_get_clock_state *)xfer->xfer_buf;
+	req->dev_id = dev_id;
+	req->clk_id = clk_id;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_resp_get_clock_state *)xfer->xfer_buf;
+
+	if (!ti_sci_is_response_ack(resp)) {
+		ret = -ENODEV;
+		goto fail;
+	}
+
+	if (programmed_state)
+		*programmed_state = resp->programmed_state;
+	if (current_state)
+		*current_state = resp->current_state;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_get_clock() - Get control of a clock from TI SCI
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @needs_ssc: 'true' if Spread Spectrum clock is desired, else 'false'
+ * @can_change_freq: 'true' if frequency change is desired, else 'false'
+ * @enable_input_term: 'true' if input termination is desired, else 'false'
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_get_clock(const struct ti_sci_handle *handle, u32 dev_id,
+				u8 clk_id, bool needs_ssc, bool can_change_freq,
+				bool enable_input_term)
+{
+	u32 flags = 0;
+
+	flags |= needs_ssc ? MSG_FLAG_CLOCK_ALLOW_SSC : 0;
+	flags |= can_change_freq ? MSG_FLAG_CLOCK_ALLOW_FREQ_CHANGE : 0;
+	flags |= enable_input_term ? MSG_FLAG_CLOCK_INPUT_TERM : 0;
+
+	return ti_sci_set_clock_state(handle, dev_id, clk_id, flags,
+				      MSG_CLOCK_SW_STATE_REQ);
+}
+
+/**
+ * ti_sci_cmd_idle_clock() - Idle a clock which is in our control
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ *
+ * NOTE: This clock must have been requested by get_clock previously.
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_idle_clock(const struct ti_sci_handle *handle,
+				 u32 dev_id, u8 clk_id)
+{
+	return ti_sci_set_clock_state(handle, dev_id, clk_id, 0,
+				      MSG_CLOCK_SW_STATE_UNREQ);
+}
+
+/**
+ * ti_sci_cmd_put_clock() - Release a clock from our control back to TISCI
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ *
+ * NOTE: This clock must have been requested by get_clock previously.
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_put_clock(const struct ti_sci_handle *handle,
+				u32 dev_id, u8 clk_id)
+{
+	return ti_sci_set_clock_state(handle, dev_id, clk_id, 0,
+				      MSG_CLOCK_SW_STATE_AUTO);
+}
+
+/**
+ * ti_sci_cmd_clk_is_auto() - Is the clock being auto managed
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @req_state: state indicating if the clock is auto managed
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_clk_is_auto(const struct ti_sci_handle *handle,
+				  u32 dev_id, u8 clk_id, bool *req_state)
+{
+	u8 state = 0;
+	int ret;
+
+	if (!req_state)
+		return -EINVAL;
+
+	ret = ti_sci_cmd_get_clock_state(handle, dev_id, clk_id, &state, NULL);
+	if (ret)
+		return ret;
+
+	*req_state = (state == MSG_CLOCK_SW_STATE_AUTO);
+	return 0;
+}
+
+/**
+ * ti_sci_cmd_clk_is_on() - Is the clock ON
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @req_state: state indicating if the clock is managed by us and enabled
+ * @curr_state: state indicating if the clock is ready for operation
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_clk_is_on(const struct ti_sci_handle *handle, u32 dev_id,
+				u8 clk_id, bool *req_state, bool *curr_state)
+{
+	u8 c_state = 0, r_state = 0;
+	int ret;
+
+	if (!req_state && !curr_state)
+		return -EINVAL;
+
+	ret = ti_sci_cmd_get_clock_state(handle, dev_id, clk_id,
+					 &r_state, &c_state);
+	if (ret)
+		return ret;
+
+	if (req_state)
+		*req_state = (r_state == MSG_CLOCK_SW_STATE_REQ);
+	if (curr_state)
+		*curr_state = (c_state == MSG_CLOCK_HW_STATE_READY);
+	return 0;
+}
+
+/**
+ * ti_sci_cmd_clk_is_off() - Is the clock OFF
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @req_state: state indicating if the clock is managed by us and disabled
+ * @curr_state: state indicating if the clock is NOT ready for operation
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_clk_is_off(const struct ti_sci_handle *handle, u32 dev_id,
+				 u8 clk_id, bool *req_state, bool *curr_state)
+{
+	u8 c_state = 0, r_state = 0;
+	int ret;
+
+	if (!req_state && !curr_state)
+		return -EINVAL;
+
+	ret = ti_sci_cmd_get_clock_state(handle, dev_id, clk_id,
+					 &r_state, &c_state);
+	if (ret)
+		return ret;
+
+	if (req_state)
+		*req_state = (r_state == MSG_CLOCK_SW_STATE_UNREQ);
+	if (curr_state)
+		*curr_state = (c_state == MSG_CLOCK_HW_STATE_NOT_READY);
+	return 0;
+}
+
+/**
+ * ti_sci_cmd_clk_set_parent() - Set the clock source of a specific device clock
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @parent_id:	Parent clock identifier to set
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_clk_set_parent(const struct ti_sci_handle *handle,
+				     u32 dev_id, u8 clk_id, u8 parent_id)
+{
+	struct ti_sci_info *info;
+	struct ti_sci_msg_req_set_clock_parent *req;
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_SET_CLOCK_PARENT,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_set_clock_parent *)xfer->xfer_buf;
+	req->dev_id = dev_id;
+	req->clk_id = clk_id;
+	req->parent_id = parent_id;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+
+	ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_clk_get_parent() - Get current parent clock source
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @parent_id:	Current clock parent
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_clk_get_parent(const struct ti_sci_handle *handle,
+				     u32 dev_id, u8 clk_id, u8 *parent_id)
+{
+	struct ti_sci_info *info;
+	struct ti_sci_msg_req_get_clock_parent *req;
+	struct ti_sci_msg_resp_get_clock_parent *resp;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle || !parent_id)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_GET_CLOCK_PARENT,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_get_clock_parent *)xfer->xfer_buf;
+	req->dev_id = dev_id;
+	req->clk_id = clk_id;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_resp_get_clock_parent *)xfer->xfer_buf;
+
+	if (!ti_sci_is_response_ack(resp))
+		ret = -ENODEV;
+	else
+		*parent_id = resp->parent_id;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_clk_get_num_parents() - Get num parents of the current clk source
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @num_parents: Returns he number of parents to the current clock.
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_clk_get_num_parents(const struct ti_sci_handle *handle,
+					  u32 dev_id, u8 clk_id,
+					  u8 *num_parents)
+{
+	struct ti_sci_info *info;
+	struct ti_sci_msg_req_get_clock_num_parents *req;
+	struct ti_sci_msg_resp_get_clock_num_parents *resp;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle || !num_parents)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_GET_NUM_CLOCK_PARENTS,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_get_clock_num_parents *)xfer->xfer_buf;
+	req->dev_id = dev_id;
+	req->clk_id = clk_id;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_resp_get_clock_num_parents *)xfer->xfer_buf;
+
+	if (!ti_sci_is_response_ack(resp))
+		ret = -ENODEV;
+	else
+		*num_parents = resp->num_parents;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_clk_get_match_freq() - Find a good match for frequency
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @min_freq:	The minimum allowable frequency in Hz. This is the minimum
+ *		allowable programmed frequency and does not account for clock
+ *		tolerances and jitter.
+ * @target_freq: The target clock frequency in Hz. A frequency will be
+ *		processed as close to this target frequency as possible.
+ * @max_freq:	The maximum allowable frequency in Hz. This is the maximum
+ *		allowable programmed frequency and does not account for clock
+ *		tolerances and jitter.
+ * @match_freq:	Frequency match in Hz response.
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_clk_get_match_freq(const struct ti_sci_handle *handle,
+					 u32 dev_id, u8 clk_id, u64 min_freq,
+					 u64 target_freq, u64 max_freq,
+					 u64 *match_freq)
+{
+	struct ti_sci_info *info;
+	struct ti_sci_msg_req_query_clock_freq *req;
+	struct ti_sci_msg_resp_query_clock_freq *resp;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle || !match_freq)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_QUERY_CLOCK_FREQ,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_query_clock_freq *)xfer->xfer_buf;
+	req->dev_id = dev_id;
+	req->clk_id = clk_id;
+	req->min_freq_hz = min_freq;
+	req->target_freq_hz = target_freq;
+	req->max_freq_hz = max_freq;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_resp_query_clock_freq *)xfer->xfer_buf;
+
+	if (!ti_sci_is_response_ack(resp))
+		ret = -ENODEV;
+	else
+		*match_freq = resp->freq_hz;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_clk_set_freq() - Set a frequency for clock
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @min_freq:	The minimum allowable frequency in Hz. This is the minimum
+ *		allowable programmed frequency and does not account for clock
+ *		tolerances and jitter.
+ * @target_freq: The target clock frequency in Hz. A frequency will be
+ *		processed as close to this target frequency as possible.
+ * @max_freq:	The maximum allowable frequency in Hz. This is the maximum
+ *		allowable programmed frequency and does not account for clock
+ *		tolerances and jitter.
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_clk_set_freq(const struct ti_sci_handle *handle,
+				   u32 dev_id, u8 clk_id, u64 min_freq,
+				   u64 target_freq, u64 max_freq)
+{
+	struct ti_sci_info *info;
+	struct ti_sci_msg_req_set_clock_freq *req;
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_SET_CLOCK_FREQ,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_set_clock_freq *)xfer->xfer_buf;
+	req->dev_id = dev_id;
+	req->clk_id = clk_id;
+	req->min_freq_hz = min_freq;
+	req->target_freq_hz = target_freq;
+	req->max_freq_hz = max_freq;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+
+	ret = ti_sci_is_response_ack(resp) ? 0 : -ENODEV;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/**
+ * ti_sci_cmd_clk_get_freq() - Get current frequency
+ * @handle:	pointer to TI SCI handle
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @freq:	Currently frequency in Hz
+ *
+ * Return: 0 if all went well, else returns appropriate error value.
+ */
+static int ti_sci_cmd_clk_get_freq(const struct ti_sci_handle *handle,
+				   u32 dev_id, u8 clk_id, u64 *freq)
+{
+	struct ti_sci_info *info;
+	struct ti_sci_msg_req_get_clock_freq *req;
+	struct ti_sci_msg_resp_get_clock_freq *resp;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle || !freq)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_GET_CLOCK_FREQ,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_get_clock_freq *)xfer->xfer_buf;
+	req->dev_id = dev_id;
+	req->clk_id = clk_id;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_resp_get_clock_freq *)xfer->xfer_buf;
+
+	if (!ti_sci_is_response_ack(resp))
+		ret = -ENODEV;
+	else
+		*freq = resp->freq_hz;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+static int ti_sci_cmd_core_reboot(const struct ti_sci_handle *handle)
+{
+	struct ti_sci_info *info;
+	struct ti_sci_msg_req_reboot *req;
+	struct ti_sci_msg_hdr *resp;
+	struct ti_sci_xfer *xfer;
+	struct device *dev;
+	int ret = 0;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	dev = info->dev;
+
+	xfer = ti_sci_get_one_xfer(info, TI_SCI_MSG_SYS_RESET,
+				   TI_SCI_FLAG_REQ_ACK_ON_PROCESSED,
+				   sizeof(*req), sizeof(*resp));
+	if (IS_ERR(xfer)) {
+		ret = PTR_ERR(xfer);
+		dev_err(dev, "Message alloc failed(%d)\n", ret);
+		return ret;
+	}
+	req = (struct ti_sci_msg_req_reboot *)xfer->xfer_buf;
+
+	ret = ti_sci_do_xfer(info, xfer);
+	if (ret) {
+		dev_err(dev, "Mbox send fail %d\n", ret);
+		goto fail;
+	}
+
+	resp = (struct ti_sci_msg_hdr *)xfer->xfer_buf;
+
+	if (!ti_sci_is_response_ack(resp))
+		ret = -ENODEV;
+	else
+		ret = 0;
+
+fail:
+	ti_sci_put_one_xfer(&info->minfo, xfer);
+
+	return ret;
+}
+
+/*
+ * ti_sci_setup_ops() - Setup the operations structures
+ * @info:	pointer to TISCI pointer
+ */
+static void ti_sci_setup_ops(struct ti_sci_info *info)
+{
+	struct ti_sci_ops *ops = &info->handle.ops;
+	struct ti_sci_core_ops *core_ops = &ops->core_ops;
+	struct ti_sci_dev_ops *dops = &ops->dev_ops;
+	struct ti_sci_clk_ops *cops = &ops->clk_ops;
+
+	core_ops->reboot_device = ti_sci_cmd_core_reboot;
+
+	dops->get_device = ti_sci_cmd_get_device;
+	dops->idle_device = ti_sci_cmd_idle_device;
+	dops->put_device = ti_sci_cmd_put_device;
+
+	dops->is_valid = ti_sci_cmd_dev_is_valid;
+	dops->get_context_loss_count = ti_sci_cmd_dev_get_clcnt;
+	dops->is_idle = ti_sci_cmd_dev_is_idle;
+	dops->is_stop = ti_sci_cmd_dev_is_stop;
+	dops->is_on = ti_sci_cmd_dev_is_on;
+	dops->is_transitioning = ti_sci_cmd_dev_is_trans;
+	dops->set_device_resets = ti_sci_cmd_set_device_resets;
+	dops->get_device_resets = ti_sci_cmd_get_device_resets;
+
+	cops->get_clock = ti_sci_cmd_get_clock;
+	cops->idle_clock = ti_sci_cmd_idle_clock;
+	cops->put_clock = ti_sci_cmd_put_clock;
+	cops->is_auto = ti_sci_cmd_clk_is_auto;
+	cops->is_on = ti_sci_cmd_clk_is_on;
+	cops->is_off = ti_sci_cmd_clk_is_off;
+
+	cops->set_parent = ti_sci_cmd_clk_set_parent;
+	cops->get_parent = ti_sci_cmd_clk_get_parent;
+	cops->get_num_parents = ti_sci_cmd_clk_get_num_parents;
+
+	cops->get_best_match_freq = ti_sci_cmd_clk_get_match_freq;
+	cops->set_freq = ti_sci_cmd_clk_set_freq;
+	cops->get_freq = ti_sci_cmd_clk_get_freq;
+}
+
+/**
+ * ti_sci_get_handle() - Get the TI SCI handle for a device
+ * @dev:	Pointer to device for which we want SCI handle
+ *
+ * NOTE: The function does not track individual clients of the framework
+ * and is expected to be maintained by caller of TI SCI protocol library.
+ * ti_sci_put_handle must be balanced with successful ti_sci_get_handle
+ * Return: pointer to handle if successful, else:
+ * -EPROBE_DEFER if the instance is not ready
+ * -ENODEV if the required node handler is missing
+ * -EINVAL if invalid conditions are encountered.
+ */
+const struct ti_sci_handle *ti_sci_get_handle(struct device *dev)
+{
+	struct device_node *ti_sci_np;
+	struct list_head *p;
+	struct ti_sci_handle *handle = NULL;
+	struct ti_sci_info *info;
+
+	if (!dev) {
+		pr_err("I need a device pointer\n");
+		return ERR_PTR(-EINVAL);
+	}
+	ti_sci_np = of_get_parent(dev->of_node);
+	if (!ti_sci_np) {
+		dev_err(dev, "No OF information\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	mutex_lock(&ti_sci_list_mutex);
+	list_for_each(p, &ti_sci_list) {
+		info = list_entry(p, struct ti_sci_info, node);
+		if (ti_sci_np == info->dev->of_node) {
+			handle = &info->handle;
+			info->users++;
+			break;
+		}
+	}
+	mutex_unlock(&ti_sci_list_mutex);
+	of_node_put(ti_sci_np);
+
+	if (!handle)
+		return ERR_PTR(-EPROBE_DEFER);
+
+	return handle;
+}
+EXPORT_SYMBOL_GPL(ti_sci_get_handle);
+
+/**
+ * ti_sci_put_handle() - Release the handle acquired by ti_sci_get_handle
+ * @handle:	Handle acquired by ti_sci_get_handle
+ *
+ * NOTE: The function does not track individual clients of the framework
+ * and is expected to be maintained by caller of TI SCI protocol library.
+ * ti_sci_put_handle must be balanced with successful ti_sci_get_handle
+ *
+ * Return: 0 is successfully released
+ * if an error pointer was passed, it returns the error value back,
+ * if null was passed, it returns -EINVAL;
+ */
+int ti_sci_put_handle(const struct ti_sci_handle *handle)
+{
+	struct ti_sci_info *info;
+
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
+	if (!handle)
+		return -EINVAL;
+
+	info = handle_to_ti_sci_info(handle);
+	mutex_lock(&ti_sci_list_mutex);
+	if (!WARN_ON(!info->users))
+		info->users--;
+	mutex_unlock(&ti_sci_list_mutex);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ti_sci_put_handle);
+
+static void devm_ti_sci_release(struct device *dev, void *res)
+{
+	const struct ti_sci_handle **ptr = res;
+	const struct ti_sci_handle *handle = *ptr;
+	int ret;
+
+	ret = ti_sci_put_handle(handle);
+	if (ret)
+		dev_err(dev, "failed to put handle %d\n", ret);
+}
+
+/**
+ * devm_ti_sci_get_handle() - Managed get handle
+ * @dev:	device for which we want SCI handle for.
+ *
+ * NOTE: This releases the handle once the device resources are
+ * no longer needed. MUST NOT BE released with ti_sci_put_handle.
+ * The function does not track individual clients of the framework
+ * and is expected to be maintained by caller of TI SCI protocol library.
+ *
+ * Return: 0 if all went fine, else corresponding error.
+ */
+const struct ti_sci_handle *devm_ti_sci_get_handle(struct device *dev)
+{
+	const struct ti_sci_handle **ptr;
+	const struct ti_sci_handle *handle;
+
+	ptr = devres_alloc(devm_ti_sci_release, sizeof(*ptr), GFP_KERNEL);
+	if (!ptr)
+		return ERR_PTR(-ENOMEM);
+	handle = ti_sci_get_handle(dev);
+
+	if (!IS_ERR(handle)) {
+		*ptr = handle;
+		devres_add(dev, ptr);
+	} else {
+		devres_free(ptr);
+	}
+
+	return handle;
+}
+EXPORT_SYMBOL_GPL(devm_ti_sci_get_handle);
+
+static int tisci_reboot_handler(struct notifier_block *nb, unsigned long mode,
+				void *cmd)
+{
+	struct ti_sci_info *info = reboot_to_ti_sci_info(nb);
+	const struct ti_sci_handle *handle = &info->handle;
+
+	ti_sci_cmd_core_reboot(handle);
+
+	/* call fail OR pass, we should not be here in the first place */
+	return NOTIFY_BAD;
+}
+
+/* Description for K2G */
+static const struct ti_sci_desc ti_sci_pmmc_k2g_desc = {
+	.host_id = 2,
+	/* Conservative duration */
+	.max_rx_timeout_ms = 1000,
+	/* Limited by MBOX_TX_QUEUE_LEN. K2G can handle upto 128 messages! */
+	.max_msgs = 20,
+	.max_msg_size = 64,
+};
+
+static const struct of_device_id ti_sci_of_match[] = {
+	{.compatible = "ti,k2g-sci", .data = &ti_sci_pmmc_k2g_desc},
+	{ /* Sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, ti_sci_of_match);
+
+static int ti_sci_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	const struct of_device_id *of_id;
+	const struct ti_sci_desc *desc;
+	struct ti_sci_xfer *xfer;
+	struct ti_sci_info *info = NULL;
+	struct ti_sci_xfers_info *minfo;
+	struct mbox_client *cl;
+	int ret = -EINVAL;
+	int i;
+	int reboot = 0;
+
+	of_id = of_match_device(ti_sci_of_match, dev);
+	if (!of_id) {
+		dev_err(dev, "OF data missing\n");
+		return -EINVAL;
+	}
+	desc = of_id->data;
+
+	info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	info->dev = dev;
+	info->desc = desc;
+	reboot = of_property_read_bool(dev->of_node,
+				       "ti,system-reboot-controller");
+	INIT_LIST_HEAD(&info->node);
+	minfo = &info->minfo;
+
+	/*
+	 * Pre-allocate messages
+	 * NEVER allocate more than what we can indicate in hdr.seq
+	 * if we have data description bug, force a fix..
+	 */
+	if (WARN_ON(desc->max_msgs >=
+		    1 << 8 * sizeof(((struct ti_sci_msg_hdr *)0)->seq)))
+		return -EINVAL;
+
+	minfo->xfer_block = devm_kcalloc(dev,
+					 desc->max_msgs,
+					 sizeof(*minfo->xfer_block),
+					 GFP_KERNEL);
+	if (!minfo->xfer_block)
+		return -ENOMEM;
+
+	minfo->xfer_alloc_table = devm_kzalloc(dev,
+					       BITS_TO_LONGS(desc->max_msgs)
+					       * sizeof(unsigned long),
+					       GFP_KERNEL);
+	if (!minfo->xfer_alloc_table)
+		return -ENOMEM;
+	bitmap_zero(minfo->xfer_alloc_table, desc->max_msgs);
+
+	/* Pre-initialize the buffer pointer to pre-allocated buffers */
+	for (i = 0, xfer = minfo->xfer_block; i < desc->max_msgs; i++, xfer++) {
+		xfer->xfer_buf = devm_kcalloc(dev, 1, desc->max_msg_size,
+					      GFP_KERNEL);
+		if (!xfer->xfer_buf)
+			return -ENOMEM;
+
+		xfer->tx_message.buf = xfer->xfer_buf;
+		init_completion(&xfer->done);
+	}
+
+	ret = ti_sci_debugfs_create(pdev, info);
+	if (ret)
+		dev_warn(dev, "Failed to create debug file\n");
+
+	platform_set_drvdata(pdev, info);
+
+	cl = &info->cl;
+	cl->dev = dev;
+	cl->tx_block = false;
+	cl->rx_callback = ti_sci_rx_callback;
+	cl->knows_txdone = true;
+
+	spin_lock_init(&minfo->xfer_lock);
+	sema_init(&minfo->sem_xfer_count, desc->max_msgs);
+
+	info->chan_rx = mbox_request_channel_byname(cl, "rx");
+	if (IS_ERR(info->chan_rx)) {
+		ret = PTR_ERR(info->chan_rx);
+		goto out;
+	}
+
+	info->chan_tx = mbox_request_channel_byname(cl, "tx");
+	if (IS_ERR(info->chan_tx)) {
+		ret = PTR_ERR(info->chan_tx);
+		goto out;
+	}
+	ret = ti_sci_cmd_get_revision(info);
+	if (ret) {
+		dev_err(dev, "Unable to communicate with TISCI(%d)\n", ret);
+		goto out;
+	}
+
+	ti_sci_setup_ops(info);
+
+	if (reboot) {
+		info->nb.notifier_call = tisci_reboot_handler;
+		info->nb.priority = 128;
+
+		ret = register_restart_handler(&info->nb);
+		if (ret) {
+			dev_err(dev, "reboot registration fail(%d)\n", ret);
+			return ret;
+		}
+	}
+
+	dev_info(dev, "ABI: %d.%d (firmware rev 0x%04x '%s')\n",
+		 info->handle.version.abi_major, info->handle.version.abi_minor,
+		 info->handle.version.firmware_revision,
+		 info->handle.version.firmware_description);
+
+	mutex_lock(&ti_sci_list_mutex);
+	list_add_tail(&info->node, &ti_sci_list);
+	mutex_unlock(&ti_sci_list_mutex);
+
+	return of_platform_populate(dev->of_node, NULL, NULL, dev);
+out:
+	if (!IS_ERR(info->chan_tx))
+		mbox_free_channel(info->chan_tx);
+	if (!IS_ERR(info->chan_rx))
+		mbox_free_channel(info->chan_rx);
+	debugfs_remove(info->d);
+	return ret;
+}
+
+static int ti_sci_remove(struct platform_device *pdev)
+{
+	struct ti_sci_info *info;
+	struct device *dev = &pdev->dev;
+	int ret = 0;
+
+	of_platform_depopulate(dev);
+
+	info = platform_get_drvdata(pdev);
+
+	if (info->nb.notifier_call)
+		unregister_restart_handler(&info->nb);
+
+	mutex_lock(&ti_sci_list_mutex);
+	if (info->users)
+		ret = -EBUSY;
+	else
+		list_del(&info->node);
+	mutex_unlock(&ti_sci_list_mutex);
+
+	if (!ret) {
+		ti_sci_debugfs_destroy(pdev, info);
+
+		/* Safe to free channels since no more users */
+		mbox_free_channel(info->chan_tx);
+		mbox_free_channel(info->chan_rx);
+	}
+
+	return ret;
+}
+
+static struct platform_driver ti_sci_driver = {
+	.probe = ti_sci_probe,
+	.remove = ti_sci_remove,
+	.driver = {
+		   .name = "ti-sci",
+		   .of_match_table = of_match_ptr(ti_sci_of_match),
+	},
+};
+module_platform_driver(ti_sci_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("TI System Control Interface(SCI) driver");
+MODULE_AUTHOR("Nishanth Menon");
+MODULE_ALIAS("platform:ti-sci");
diff --git a/drivers/firmware/ti_sci.h b/drivers/firmware/ti_sci.h
new file mode 100644
index 000000000000..9b611e9e6f6d
--- /dev/null
+++ b/drivers/firmware/ti_sci.h
@@ -0,0 +1,492 @@
+/*
+ * Texas Instruments System Control Interface (TISCI) Protocol
+ *
+ * Communication protocol with TI SCI hardware
+ * The system works in a message response protocol
+ * See: http://processors.wiki.ti.com/index.php/TISCI for details
+ *
+ * Copyright (C)  2015-2016 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *   Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ *   Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the
+ *   distribution.
+ *
+ *   Neither the name of Texas Instruments Incorporated nor the names of
+ *   its contributors may be used to endorse or promote products derived
+ *   from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef __TI_SCI_H
+#define __TI_SCI_H
+
+/* Generic Messages */
+#define TI_SCI_MSG_ENABLE_WDT	0x0000
+#define TI_SCI_MSG_WAKE_RESET	0x0001
+#define TI_SCI_MSG_VERSION	0x0002
+#define TI_SCI_MSG_WAKE_REASON	0x0003
+#define TI_SCI_MSG_GOODBYE	0x0004
+#define TI_SCI_MSG_SYS_RESET	0x0005
+
+/* Device requests */
+#define TI_SCI_MSG_SET_DEVICE_STATE	0x0200
+#define TI_SCI_MSG_GET_DEVICE_STATE	0x0201
+#define TI_SCI_MSG_SET_DEVICE_RESETS	0x0202
+
+/* Clock requests */
+#define TI_SCI_MSG_SET_CLOCK_STATE	0x0100
+#define TI_SCI_MSG_GET_CLOCK_STATE	0x0101
+#define TI_SCI_MSG_SET_CLOCK_PARENT	0x0102
+#define TI_SCI_MSG_GET_CLOCK_PARENT	0x0103
+#define TI_SCI_MSG_GET_NUM_CLOCK_PARENTS 0x0104
+#define TI_SCI_MSG_SET_CLOCK_FREQ	0x010c
+#define TI_SCI_MSG_QUERY_CLOCK_FREQ	0x010d
+#define TI_SCI_MSG_GET_CLOCK_FREQ	0x010e
+
+/**
+ * struct ti_sci_msg_hdr - Generic Message Header for All messages and responses
+ * @type:	Type of messages: One of TI_SCI_MSG* values
+ * @host:	Host of the message
+ * @seq:	Message identifier indicating a transfer sequence
+ * @flags:	Flag for the message
+ */
+struct ti_sci_msg_hdr {
+	u16 type;
+	u8 host;
+	u8 seq;
+#define TI_SCI_MSG_FLAG(val)			(1 << (val))
+#define TI_SCI_FLAG_REQ_GENERIC_NORESPONSE	0x0
+#define TI_SCI_FLAG_REQ_ACK_ON_RECEIVED		TI_SCI_MSG_FLAG(0)
+#define TI_SCI_FLAG_REQ_ACK_ON_PROCESSED	TI_SCI_MSG_FLAG(1)
+#define TI_SCI_FLAG_RESP_GENERIC_NACK		0x0
+#define TI_SCI_FLAG_RESP_GENERIC_ACK		TI_SCI_MSG_FLAG(1)
+	/* Additional Flags */
+	u32 flags;
+} __packed;
+
+/**
+ * struct ti_sci_msg_resp_version - Response for a message
+ * @hdr:		Generic header
+ * @firmware_description: String describing the firmware
+ * @firmware_revision:	Firmware revision
+ * @abi_major:		Major version of the ABI that firmware supports
+ * @abi_minor:		Minor version of the ABI that firmware supports
+ *
+ * In general, ABI version changes follow the rule that minor version increments
+ * are backward compatible. Major revision changes in ABI may not be
+ * backward compatible.
+ *
+ * Response to a generic message with message type TI_SCI_MSG_VERSION
+ */
+struct ti_sci_msg_resp_version {
+	struct ti_sci_msg_hdr hdr;
+	char firmware_description[32];
+	u16 firmware_revision;
+	u8 abi_major;
+	u8 abi_minor;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_reboot - Reboot the SoC
+ * @hdr:	Generic Header
+ *
+ * Request type is TI_SCI_MSG_SYS_RESET, responded with a generic
+ * ACK/NACK message.
+ */
+struct ti_sci_msg_req_reboot {
+	struct ti_sci_msg_hdr hdr;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_set_device_state - Set the desired state of the device
+ * @hdr:		Generic header
+ * @id:	Indicates which device to modify
+ * @reserved: Reserved space in message, must be 0 for backward compatibility
+ * @state: The desired state of the device.
+ *
+ * Certain flags can also be set to alter the device state:
+ * + MSG_FLAG_DEVICE_WAKE_ENABLED - Configure the device to be a wake source.
+ * The meaning of this flag will vary slightly from device to device and from
+ * SoC to SoC but it generally allows the device to wake the SoC out of deep
+ * suspend states.
+ * + MSG_FLAG_DEVICE_RESET_ISO - Enable reset isolation for this device.
+ * + MSG_FLAG_DEVICE_EXCLUSIVE - Claim this device exclusively. When passed
+ * with STATE_RETENTION or STATE_ON, it will claim the device exclusively.
+ * If another host already has this device set to STATE_RETENTION or STATE_ON,
+ * the message will fail. Once successful, other hosts attempting to set
+ * STATE_RETENTION or STATE_ON will fail.
+ *
+ * Request type is TI_SCI_MSG_SET_DEVICE_STATE, responded with a generic
+ * ACK/NACK message.
+ */
+struct ti_sci_msg_req_set_device_state {
+	/* Additional hdr->flags options */
+#define MSG_FLAG_DEVICE_WAKE_ENABLED	TI_SCI_MSG_FLAG(8)
+#define MSG_FLAG_DEVICE_RESET_ISO	TI_SCI_MSG_FLAG(9)
+#define MSG_FLAG_DEVICE_EXCLUSIVE	TI_SCI_MSG_FLAG(10)
+	struct ti_sci_msg_hdr hdr;
+	u32 id;
+	u32 reserved;
+
+#define MSG_DEVICE_SW_STATE_AUTO_OFF	0
+#define MSG_DEVICE_SW_STATE_RETENTION	1
+#define MSG_DEVICE_SW_STATE_ON		2
+	u8 state;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_get_device_state - Request to get device.
+ * @hdr:		Generic header
+ * @id:		Device Identifier
+ *
+ * Request type is TI_SCI_MSG_GET_DEVICE_STATE, responded device state
+ * information
+ */
+struct ti_sci_msg_req_get_device_state {
+	struct ti_sci_msg_hdr hdr;
+	u32 id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_resp_get_device_state - Response to get device request.
+ * @hdr:		Generic header
+ * @context_loss_count: Indicates how many times the device has lost context. A
+ *	driver can use this monotonic counter to determine if the device has
+ *	lost context since the last time this message was exchanged.
+ * @resets: Programmed state of the reset lines.
+ * @programmed_state:	The state as programmed by set_device.
+ *			- Uses the MSG_DEVICE_SW_* macros
+ * @current_state:	The actual state of the hardware.
+ *
+ * Response to request TI_SCI_MSG_GET_DEVICE_STATE.
+ */
+struct ti_sci_msg_resp_get_device_state {
+	struct ti_sci_msg_hdr hdr;
+	u32 context_loss_count;
+	u32 resets;
+	u8 programmed_state;
+#define MSG_DEVICE_HW_STATE_OFF		0
+#define MSG_DEVICE_HW_STATE_ON		1
+#define MSG_DEVICE_HW_STATE_TRANS	2
+	u8 current_state;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_set_device_resets - Set the desired resets
+ *				configuration of the device
+ * @hdr:		Generic header
+ * @id:	Indicates which device to modify
+ * @resets: A bit field of resets for the device. The meaning, behavior,
+ *	and usage of the reset flags are device specific. 0 for a bit
+ *	indicates releasing the reset represented by that bit while 1
+ *	indicates keeping it held.
+ *
+ * Request type is TI_SCI_MSG_SET_DEVICE_RESETS, responded with a generic
+ * ACK/NACK message.
+ */
+struct ti_sci_msg_req_set_device_resets {
+	struct ti_sci_msg_hdr hdr;
+	u32 id;
+	u32 resets;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_set_clock_state - Request to setup a Clock state
+ * @hdr:	Generic Header, Certain flags can be set specific to the clocks:
+ *		MSG_FLAG_CLOCK_ALLOW_SSC: Allow this clock to be modified
+ *		via spread spectrum clocking.
+ *		MSG_FLAG_CLOCK_ALLOW_FREQ_CHANGE: Allow this clock's
+ *		frequency to be changed while it is running so long as it
+ *		is within the min/max limits.
+ *		MSG_FLAG_CLOCK_INPUT_TERM: Enable input termination, this
+ *		is only applicable to clock inputs on the SoC pseudo-device.
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @request_state: Request the state for the clock to be set to.
+ *		MSG_CLOCK_SW_STATE_UNREQ: The IP does not require this clock,
+ *		it can be disabled, regardless of the state of the device
+ *		MSG_CLOCK_SW_STATE_AUTO: Allow the System Controller to
+ *		automatically manage the state of this clock. If the device
+ *		is enabled, then the clock is enabled. If the device is set
+ *		to off or retention, then the clock is internally set as not
+ *		being required by the device.(default)
+ *		MSG_CLOCK_SW_STATE_REQ:  Configure the clock to be enabled,
+ *		regardless of the state of the device.
+ *
+ * Normally, all required clocks are managed by TISCI entity, this is used
+ * only for specific control *IF* required. Auto managed state is
+ * MSG_CLOCK_SW_STATE_AUTO, in other states, TISCI entity assume remote
+ * will explicitly control.
+ *
+ * Request type is TI_SCI_MSG_SET_CLOCK_STATE, response is a generic
+ * ACK or NACK message.
+ */
+struct ti_sci_msg_req_set_clock_state {
+	/* Additional hdr->flags options */
+#define MSG_FLAG_CLOCK_ALLOW_SSC		TI_SCI_MSG_FLAG(8)
+#define MSG_FLAG_CLOCK_ALLOW_FREQ_CHANGE	TI_SCI_MSG_FLAG(9)
+#define MSG_FLAG_CLOCK_INPUT_TERM		TI_SCI_MSG_FLAG(10)
+	struct ti_sci_msg_hdr hdr;
+	u32 dev_id;
+	u8 clk_id;
+#define MSG_CLOCK_SW_STATE_UNREQ	0
+#define MSG_CLOCK_SW_STATE_AUTO		1
+#define MSG_CLOCK_SW_STATE_REQ		2
+	u8 request_state;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_get_clock_state - Request for clock state
+ * @hdr:	Generic Header
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to get state of.
+ *
+ * Request type is TI_SCI_MSG_GET_CLOCK_STATE, response is state
+ * of the clock
+ */
+struct ti_sci_msg_req_get_clock_state {
+	struct ti_sci_msg_hdr hdr;
+	u32 dev_id;
+	u8 clk_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_resp_get_clock_state - Response to get clock state
+ * @hdr:	Generic Header
+ * @programmed_state: Any programmed state of the clock. This is one of
+ *		MSG_CLOCK_SW_STATE* values.
+ * @current_state: Current state of the clock. This is one of:
+ *		MSG_CLOCK_HW_STATE_NOT_READY: Clock is not ready
+ *		MSG_CLOCK_HW_STATE_READY: Clock is ready
+ *
+ * Response to TI_SCI_MSG_GET_CLOCK_STATE.
+ */
+struct ti_sci_msg_resp_get_clock_state {
+	struct ti_sci_msg_hdr hdr;
+	u8 programmed_state;
+#define MSG_CLOCK_HW_STATE_NOT_READY	0
+#define MSG_CLOCK_HW_STATE_READY	1
+	u8 current_state;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_set_clock_parent - Set the clock parent
+ * @hdr:	Generic Header
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * @parent_id:	The new clock parent is selectable by an index via this
+ *		parameter.
+ *
+ * Request type is TI_SCI_MSG_SET_CLOCK_PARENT, response is generic
+ * ACK / NACK message.
+ */
+struct ti_sci_msg_req_set_clock_parent {
+	struct ti_sci_msg_hdr hdr;
+	u32 dev_id;
+	u8 clk_id;
+	u8 parent_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_get_clock_parent - Get the clock parent
+ * @hdr:	Generic Header
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to get the parent for.
+ *
+ * Request type is TI_SCI_MSG_GET_CLOCK_PARENT, response is parent information
+ */
+struct ti_sci_msg_req_get_clock_parent {
+	struct ti_sci_msg_hdr hdr;
+	u32 dev_id;
+	u8 clk_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_resp_get_clock_parent - Response with clock parent
+ * @hdr:	Generic Header
+ * @parent_id:	The current clock parent
+ *
+ * Response to TI_SCI_MSG_GET_CLOCK_PARENT.
+ */
+struct ti_sci_msg_resp_get_clock_parent {
+	struct ti_sci_msg_hdr hdr;
+	u8 parent_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_get_clock_num_parents - Request to get clock parents
+ * @hdr:	Generic header
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *
+ * This request provides information about how many clock parent options
+ * are available for a given clock to a device. This is typically used
+ * for input clocks.
+ *
+ * Request type is TI_SCI_MSG_GET_NUM_CLOCK_PARENTS, response is appropriate
+ * message, or NACK in case of inability to satisfy request.
+ */
+struct ti_sci_msg_req_get_clock_num_parents {
+	struct ti_sci_msg_hdr hdr;
+	u32 dev_id;
+	u8 clk_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_resp_get_clock_num_parents - Response for get clk parents
+ * @hdr:		Generic header
+ * @num_parents:	Number of clock parents
+ *
+ * Response to TI_SCI_MSG_GET_NUM_CLOCK_PARENTS
+ */
+struct ti_sci_msg_resp_get_clock_num_parents {
+	struct ti_sci_msg_hdr hdr;
+	u8 num_parents;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_query_clock_freq - Request to query a frequency
+ * @hdr:	Generic Header
+ * @dev_id:	Device identifier this request is for
+ * @min_freq_hz: The minimum allowable frequency in Hz. This is the minimum
+ *		allowable programmed frequency and does not account for clock
+ *		tolerances and jitter.
+ * @target_freq_hz: The target clock frequency. A frequency will be found
+ *		as close to this target frequency as possible.
+ * @max_freq_hz: The maximum allowable frequency in Hz. This is the maximum
+ *		allowable programmed frequency and does not account for clock
+ *		tolerances and jitter.
+ * @clk_id:	Clock identifier for the device for this request.
+ *
+ * NOTE: Normally clock frequency management is automatically done by TISCI
+ * entity. In case of specific requests, TISCI evaluates capability to achieve
+ * requested frequency within provided range and responds with
+ * result message.
+ *
+ * Request type is TI_SCI_MSG_QUERY_CLOCK_FREQ, response is appropriate message,
+ * or NACK in case of inability to satisfy request.
+ */
+struct ti_sci_msg_req_query_clock_freq {
+	struct ti_sci_msg_hdr hdr;
+	u32 dev_id;
+	u64 min_freq_hz;
+	u64 target_freq_hz;
+	u64 max_freq_hz;
+	u8 clk_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_resp_query_clock_freq - Response to a clock frequency query
+ * @hdr:	Generic Header
+ * @freq_hz:	Frequency that is the best match in Hz.
+ *
+ * Response to request type TI_SCI_MSG_QUERY_CLOCK_FREQ. NOTE: if the request
+ * cannot be satisfied, the message will be of type NACK.
+ */
+struct ti_sci_msg_resp_query_clock_freq {
+	struct ti_sci_msg_hdr hdr;
+	u64 freq_hz;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_set_clock_freq - Request to setup a clock frequency
+ * @hdr:	Generic Header
+ * @dev_id:	Device identifier this request is for
+ * @min_freq_hz: The minimum allowable frequency in Hz. This is the minimum
+ *		allowable programmed frequency and does not account for clock
+ *		tolerances and jitter.
+ * @target_freq_hz: The target clock frequency. The clock will be programmed
+ *		at a rate as close to this target frequency as possible.
+ * @max_freq_hz: The maximum allowable frequency in Hz. This is the maximum
+ *		allowable programmed frequency and does not account for clock
+ *		tolerances and jitter.
+ * @clk_id:	Clock identifier for the device for this request.
+ *
+ * NOTE: Normally clock frequency management is automatically done by TISCI
+ * entity. In case of specific requests, TISCI evaluates capability to achieve
+ * requested range and responds with success/failure message.
+ *
+ * This sets the desired frequency for a clock within an allowable
+ * range. This message will fail on an enabled clock unless
+ * MSG_FLAG_CLOCK_ALLOW_FREQ_CHANGE is set for the clock. Additionally,
+ * if other clocks have their frequency modified due to this message,
+ * they also must have the MSG_FLAG_CLOCK_ALLOW_FREQ_CHANGE or be disabled.
+ *
+ * Calling set frequency on a clock input to the SoC pseudo-device will
+ * inform the PMMC of that clock's frequency. Setting a frequency of
+ * zero will indicate the clock is disabled.
+ *
+ * Calling set frequency on clock outputs from the SoC pseudo-device will
+ * function similarly to setting the clock frequency on a device.
+ *
+ * Request type is TI_SCI_MSG_SET_CLOCK_FREQ, response is a generic ACK/NACK
+ * message.
+ */
+struct ti_sci_msg_req_set_clock_freq {
+	struct ti_sci_msg_hdr hdr;
+	u32 dev_id;
+	u64 min_freq_hz;
+	u64 target_freq_hz;
+	u64 max_freq_hz;
+	u8 clk_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_req_get_clock_freq - Request to get the clock frequency
+ * @hdr:	Generic Header
+ * @dev_id:	Device identifier this request is for
+ * @clk_id:	Clock identifier for the device for this request.
+ *
+ * NOTE: Normally clock frequency management is automatically done by TISCI
+ * entity. In some cases, clock frequencies are configured by host.
+ *
+ * Request type is TI_SCI_MSG_GET_CLOCK_FREQ, responded with clock frequency
+ * that the clock is currently at.
+ */
+struct ti_sci_msg_req_get_clock_freq {
+	struct ti_sci_msg_hdr hdr;
+	u32 dev_id;
+	u8 clk_id;
+} __packed;
+
+/**
+ * struct ti_sci_msg_resp_get_clock_freq - Response of clock frequency request
+ * @hdr:	Generic Header
+ * @freq_hz:	Frequency that the clock is currently on, in Hz.
+ *
+ * Response to request type TI_SCI_MSG_GET_CLOCK_FREQ.
+ */
+struct ti_sci_msg_resp_get_clock_freq {
+	struct ti_sci_msg_hdr hdr;
+	u64 freq_hz;
+} __packed;
+
+#endif /* __TI_SCI_H */
diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig
index cd84934774cc..ce861a2853a4 100644
--- a/drivers/fpga/Kconfig
+++ b/drivers/fpga/Kconfig
@@ -13,12 +13,26 @@ config FPGA
 
 if FPGA
 
+config FPGA_REGION
+	tristate "FPGA Region"
+	depends on OF && FPGA_BRIDGE
+	help
+	  FPGA Regions allow loading FPGA images under control of
+	  the Device Tree.
+
 config FPGA_MGR_SOCFPGA
 	tristate "Altera SOCFPGA FPGA Manager"
-	depends on ARCH_SOCFPGA
+	depends on ARCH_SOCFPGA || COMPILE_TEST
 	help
 	  FPGA manager driver support for Altera SOCFPGA.
 
+config FPGA_MGR_SOCFPGA_A10
+	tristate "Altera SoCFPGA Arria10"
+	depends on ARCH_SOCFPGA || COMPILE_TEST
+	select REGMAP_MMIO
+	help
+	  FPGA manager driver support for Altera Arria10 SoCFPGA.
+
 config FPGA_MGR_ZYNQ_FPGA
 	tristate "Xilinx Zynq FPGA"
 	depends on ARCH_ZYNQ || COMPILE_TEST
@@ -26,6 +40,29 @@ config FPGA_MGR_ZYNQ_FPGA
 	help
 	  FPGA manager driver support for Xilinx Zynq FPGAs.
 
+config FPGA_BRIDGE
+	tristate "FPGA Bridge Framework"
+	depends on OF
+	help
+	  Say Y here if you want to support bridges connected between host
+	  processors and FPGAs or between FPGAs.
+
+config SOCFPGA_FPGA_BRIDGE
+	tristate "Altera SoCFPGA FPGA Bridges"
+	depends on ARCH_SOCFPGA && FPGA_BRIDGE
+	help
+	  Say Y to enable drivers for FPGA bridges for Altera SOCFPGA
+	  devices.
+
+config ALTERA_FREEZE_BRIDGE
+	tristate "Altera FPGA Freeze Bridge"
+	depends on ARCH_SOCFPGA && FPGA_BRIDGE
+	help
+	  Say Y to enable drivers for Altera FPGA Freeze bridges.  A
+	  freeze bridge is a bridge that exists in the FPGA fabric to
+	  isolate one region of the FPGA from the busses while that
+	  region is being reprogrammed.
+
 endif # FPGA
 
 endmenu
diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile
index 8d83fc6b1613..8df07bcf42a6 100644
--- a/drivers/fpga/Makefile
+++ b/drivers/fpga/Makefile
@@ -7,4 +7,13 @@ obj-$(CONFIG_FPGA)			+= fpga-mgr.o
 
 # FPGA Manager Drivers
 obj-$(CONFIG_FPGA_MGR_SOCFPGA)		+= socfpga.o
+obj-$(CONFIG_FPGA_MGR_SOCFPGA_A10)	+= socfpga-a10.o
 obj-$(CONFIG_FPGA_MGR_ZYNQ_FPGA)	+= zynq-fpga.o
+
+# FPGA Bridge Drivers
+obj-$(CONFIG_FPGA_BRIDGE)		+= fpga-bridge.o
+obj-$(CONFIG_SOCFPGA_FPGA_BRIDGE)	+= altera-hps2fpga.o altera-fpga2sdram.o
+obj-$(CONFIG_ALTERA_FREEZE_BRIDGE)	+= altera-freeze-bridge.o
+
+# High Level Interfaces
+obj-$(CONFIG_FPGA_REGION)		+= fpga-region.o
diff --git a/drivers/fpga/altera-fpga2sdram.c b/drivers/fpga/altera-fpga2sdram.c
new file mode 100644
index 000000000000..d4eeb74388da
--- /dev/null
+++ b/drivers/fpga/altera-fpga2sdram.c
@@ -0,0 +1,180 @@
+/*
+ * FPGA to SDRAM Bridge Driver for Altera SoCFPGA Devices
+ *
+ *  Copyright (C) 2013-2016 Altera Corporation, All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * This driver manages a bridge between an FPGA and the SDRAM used by the ARM
+ * host processor system (HPS).
+ *
+ * The bridge contains 4 read ports, 4 write ports, and 6 command ports.
+ * Reconfiguring these ports requires that no SDRAM transactions occur during
+ * reconfiguration.  The code reconfiguring the ports cannot run out of SDRAM
+ * nor can the FPGA access the SDRAM during reconfiguration.  This driver does
+ * not support reconfiguring the ports.  The ports are configured by code
+ * running out of on chip ram before Linux is started and the configuration
+ * is passed in a handoff register in the system manager.
+ *
+ * This driver supports enabling and disabling of the configured ports, which
+ * allows for safe reprogramming of the FPGA, assuming that the new FPGA image
+ * uses the same port configuration.  Bridges must be disabled before
+ * reprogramming the FPGA and re-enabled after the FPGA has been programmed.
+ */
+
+#include <linux/fpga/fpga-bridge.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/regmap.h>
+
+#define ALT_SDR_CTL_FPGAPORTRST_OFST		0x80
+#define ALT_SDR_CTL_FPGAPORTRST_PORTRSTN_MSK	0x00003fff
+#define ALT_SDR_CTL_FPGAPORTRST_RD_SHIFT	0
+#define ALT_SDR_CTL_FPGAPORTRST_WR_SHIFT	4
+#define ALT_SDR_CTL_FPGAPORTRST_CTRL_SHIFT	8
+
+/*
+ * From the Cyclone V HPS Memory Map document:
+ *   These registers are used to store handoff information between the
+ *   preloader and the OS. These 8 registers can be used to store any
+ *   information. The contents of these registers have no impact on
+ *   the state of the HPS hardware.
+ */
+#define SYSMGR_ISWGRP_HANDOFF3          (0x8C)
+
+#define F2S_BRIDGE_NAME "fpga2sdram"
+
+struct alt_fpga2sdram_data {
+	struct device *dev;
+	struct regmap *sdrctl;
+	int mask;
+};
+
+static int alt_fpga2sdram_enable_show(struct fpga_bridge *bridge)
+{
+	struct alt_fpga2sdram_data *priv = bridge->priv;
+	int value;
+
+	regmap_read(priv->sdrctl, ALT_SDR_CTL_FPGAPORTRST_OFST, &value);
+
+	return (value & priv->mask) == priv->mask;
+}
+
+static inline int _alt_fpga2sdram_enable_set(struct alt_fpga2sdram_data *priv,
+					     bool enable)
+{
+	return regmap_update_bits(priv->sdrctl, ALT_SDR_CTL_FPGAPORTRST_OFST,
+				  priv->mask, enable ? priv->mask : 0);
+}
+
+static int alt_fpga2sdram_enable_set(struct fpga_bridge *bridge, bool enable)
+{
+	return _alt_fpga2sdram_enable_set(bridge->priv, enable);
+}
+
+struct prop_map {
+	char *prop_name;
+	u32 *prop_value;
+	u32 prop_max;
+};
+
+static const struct fpga_bridge_ops altera_fpga2sdram_br_ops = {
+	.enable_set = alt_fpga2sdram_enable_set,
+	.enable_show = alt_fpga2sdram_enable_show,
+};
+
+static const struct of_device_id altera_fpga_of_match[] = {
+	{ .compatible = "altr,socfpga-fpga2sdram-bridge" },
+	{},
+};
+
+static int alt_fpga_bridge_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct alt_fpga2sdram_data *priv;
+	u32 enable;
+	struct regmap *sysmgr;
+	int ret = 0;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+
+	priv->sdrctl = syscon_regmap_lookup_by_compatible("altr,sdr-ctl");
+	if (IS_ERR(priv->sdrctl)) {
+		dev_err(dev, "regmap for altr,sdr-ctl lookup failed.\n");
+		return PTR_ERR(priv->sdrctl);
+	}
+
+	sysmgr = syscon_regmap_lookup_by_compatible("altr,sys-mgr");
+	if (IS_ERR(sysmgr)) {
+		dev_err(dev, "regmap for altr,sys-mgr lookup failed.\n");
+		return PTR_ERR(sysmgr);
+	}
+
+	/* Get f2s bridge configuration saved in handoff register */
+	regmap_read(sysmgr, SYSMGR_ISWGRP_HANDOFF3, &priv->mask);
+
+	ret = fpga_bridge_register(dev, F2S_BRIDGE_NAME,
+				   &altera_fpga2sdram_br_ops, priv);
+	if (ret)
+		return ret;
+
+	dev_info(dev, "driver initialized with handoff %08x\n", priv->mask);
+
+	if (!of_property_read_u32(dev->of_node, "bridge-enable", &enable)) {
+		if (enable > 1) {
+			dev_warn(dev, "invalid bridge-enable %u > 1\n", enable);
+		} else {
+			dev_info(dev, "%s bridge\n",
+				 (enable ? "enabling" : "disabling"));
+			ret = _alt_fpga2sdram_enable_set(priv, enable);
+			if (ret) {
+				fpga_bridge_unregister(&pdev->dev);
+				return ret;
+			}
+		}
+	}
+
+	return ret;
+}
+
+static int alt_fpga_bridge_remove(struct platform_device *pdev)
+{
+	fpga_bridge_unregister(&pdev->dev);
+
+	return 0;
+}
+
+MODULE_DEVICE_TABLE(of, altera_fpga_of_match);
+
+static struct platform_driver altera_fpga_driver = {
+	.probe = alt_fpga_bridge_probe,
+	.remove = alt_fpga_bridge_remove,
+	.driver = {
+		.name	= "altera_fpga2sdram_bridge",
+		.of_match_table = of_match_ptr(altera_fpga_of_match),
+	},
+};
+
+module_platform_driver(altera_fpga_driver);
+
+MODULE_DESCRIPTION("Altera SoCFPGA FPGA to SDRAM Bridge");
+MODULE_AUTHOR("Alan Tull <atull@opensource.altera.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/fpga/altera-freeze-bridge.c b/drivers/fpga/altera-freeze-bridge.c
new file mode 100644
index 000000000000..8dcd9fb22cb9
--- /dev/null
+++ b/drivers/fpga/altera-freeze-bridge.c
@@ -0,0 +1,273 @@
+/*
+ * FPGA Freeze Bridge Controller
+ *
+ *  Copyright (C) 2016 Altera Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/of_device.h>
+#include <linux/module.h>
+#include <linux/fpga/fpga-bridge.h>
+
+#define FREEZE_CSR_STATUS_OFFSET		0
+#define FREEZE_CSR_CTRL_OFFSET			4
+#define FREEZE_CSR_ILLEGAL_REQ_OFFSET		8
+#define FREEZE_CSR_REG_VERSION			12
+
+#define FREEZE_CSR_SUPPORTED_VERSION		2
+
+#define FREEZE_CSR_STATUS_FREEZE_REQ_DONE	BIT(0)
+#define FREEZE_CSR_STATUS_UNFREEZE_REQ_DONE	BIT(1)
+
+#define FREEZE_CSR_CTRL_FREEZE_REQ		BIT(0)
+#define FREEZE_CSR_CTRL_RESET_REQ		BIT(1)
+#define FREEZE_CSR_CTRL_UNFREEZE_REQ		BIT(2)
+
+#define FREEZE_BRIDGE_NAME			"freeze"
+
+struct altera_freeze_br_data {
+	struct device *dev;
+	void __iomem *base_addr;
+	bool enable;
+};
+
+/*
+ * Poll status until status bit is set or we have a timeout.
+ */
+static int altera_freeze_br_req_ack(struct altera_freeze_br_data *priv,
+				    u32 timeout, u32 req_ack)
+{
+	struct device *dev = priv->dev;
+	void __iomem *csr_illegal_req_addr = priv->base_addr +
+					     FREEZE_CSR_ILLEGAL_REQ_OFFSET;
+	u32 status, illegal, ctrl;
+	int ret = -ETIMEDOUT;
+
+	do {
+		illegal = readl(csr_illegal_req_addr);
+		if (illegal) {
+			dev_err(dev, "illegal request detected 0x%x", illegal);
+
+			writel(1, csr_illegal_req_addr);
+
+			illegal = readl(csr_illegal_req_addr);
+			if (illegal)
+				dev_err(dev, "illegal request not cleared 0x%x",
+					illegal);
+
+			ret = -EINVAL;
+			break;
+		}
+
+		status = readl(priv->base_addr + FREEZE_CSR_STATUS_OFFSET);
+		dev_dbg(dev, "%s %x %x\n", __func__, status, req_ack);
+		status &= req_ack;
+		if (status) {
+			ctrl = readl(priv->base_addr + FREEZE_CSR_CTRL_OFFSET);
+			dev_dbg(dev, "%s request %x acknowledged %x %x\n",
+				__func__, req_ack, status, ctrl);
+			ret = 0;
+			break;
+		}
+
+		udelay(1);
+	} while (timeout--);
+
+	if (ret == -ETIMEDOUT)
+		dev_err(dev, "%s timeout waiting for 0x%x\n",
+			__func__, req_ack);
+
+	return ret;
+}
+
+static int altera_freeze_br_do_freeze(struct altera_freeze_br_data *priv,
+				      u32 timeout)
+{
+	struct device *dev = priv->dev;
+	void __iomem *csr_ctrl_addr = priv->base_addr +
+				      FREEZE_CSR_CTRL_OFFSET;
+	u32 status;
+	int ret;
+
+	status = readl(priv->base_addr + FREEZE_CSR_STATUS_OFFSET);
+
+	dev_dbg(dev, "%s %d %d\n", __func__, status, readl(csr_ctrl_addr));
+
+	if (status & FREEZE_CSR_STATUS_FREEZE_REQ_DONE) {
+		dev_dbg(dev, "%s bridge already disabled %d\n",
+			__func__, status);
+		return 0;
+	} else if (!(status & FREEZE_CSR_STATUS_UNFREEZE_REQ_DONE)) {
+		dev_err(dev, "%s bridge not enabled %d\n", __func__, status);
+		return -EINVAL;
+	}
+
+	writel(FREEZE_CSR_CTRL_FREEZE_REQ, csr_ctrl_addr);
+
+	ret = altera_freeze_br_req_ack(priv, timeout,
+				       FREEZE_CSR_STATUS_FREEZE_REQ_DONE);
+
+	if (ret)
+		writel(0, csr_ctrl_addr);
+	else
+		writel(FREEZE_CSR_CTRL_RESET_REQ, csr_ctrl_addr);
+
+	return ret;
+}
+
+static int altera_freeze_br_do_unfreeze(struct altera_freeze_br_data *priv,
+					u32 timeout)
+{
+	struct device *dev = priv->dev;
+	void __iomem *csr_ctrl_addr = priv->base_addr +
+				      FREEZE_CSR_CTRL_OFFSET;
+	u32 status;
+	int ret;
+
+	writel(0, csr_ctrl_addr);
+
+	status = readl(priv->base_addr + FREEZE_CSR_STATUS_OFFSET);
+
+	dev_dbg(dev, "%s %d %d\n", __func__, status, readl(csr_ctrl_addr));
+
+	if (status & FREEZE_CSR_STATUS_UNFREEZE_REQ_DONE) {
+		dev_dbg(dev, "%s bridge already enabled %d\n",
+			__func__, status);
+		return 0;
+	} else if (!(status & FREEZE_CSR_STATUS_FREEZE_REQ_DONE)) {
+		dev_err(dev, "%s bridge not frozen %d\n", __func__, status);
+		return -EINVAL;
+	}
+
+	writel(FREEZE_CSR_CTRL_UNFREEZE_REQ, csr_ctrl_addr);
+
+	ret = altera_freeze_br_req_ack(priv, timeout,
+				       FREEZE_CSR_STATUS_UNFREEZE_REQ_DONE);
+
+	status = readl(priv->base_addr + FREEZE_CSR_STATUS_OFFSET);
+
+	dev_dbg(dev, "%s %d %d\n", __func__, status, readl(csr_ctrl_addr));
+
+	writel(0, csr_ctrl_addr);
+
+	return ret;
+}
+
+/*
+ * enable = 1 : allow traffic through the bridge
+ * enable = 0 : disable traffic through the bridge
+ */
+static int altera_freeze_br_enable_set(struct fpga_bridge *bridge,
+				       bool enable)
+{
+	struct altera_freeze_br_data *priv = bridge->priv;
+	struct fpga_image_info *info = bridge->info;
+	u32 timeout = 0;
+	int ret;
+
+	if (enable) {
+		if (info)
+			timeout = info->enable_timeout_us;
+
+		ret = altera_freeze_br_do_unfreeze(bridge->priv, timeout);
+	} else {
+		if (info)
+			timeout = info->disable_timeout_us;
+
+		ret = altera_freeze_br_do_freeze(bridge->priv, timeout);
+	}
+
+	if (!ret)
+		priv->enable = enable;
+
+	return ret;
+}
+
+static int altera_freeze_br_enable_show(struct fpga_bridge *bridge)
+{
+	struct altera_freeze_br_data *priv = bridge->priv;
+
+	return priv->enable;
+}
+
+static struct fpga_bridge_ops altera_freeze_br_br_ops = {
+	.enable_set = altera_freeze_br_enable_set,
+	.enable_show = altera_freeze_br_enable_show,
+};
+
+static const struct of_device_id altera_freeze_br_of_match[] = {
+	{ .compatible = "altr,freeze-bridge-controller", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, altera_freeze_br_of_match);
+
+static int altera_freeze_br_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = pdev->dev.of_node;
+	struct altera_freeze_br_data *priv;
+	struct resource *res;
+	u32 status, revision;
+
+	if (!np)
+		return -ENODEV;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->base_addr = devm_ioremap_resource(dev, res);
+	if (IS_ERR(priv->base_addr))
+		return PTR_ERR(priv->base_addr);
+
+	status = readl(priv->base_addr + FREEZE_CSR_STATUS_OFFSET);
+	if (status & FREEZE_CSR_STATUS_UNFREEZE_REQ_DONE)
+		priv->enable = 1;
+
+	revision = readl(priv->base_addr + FREEZE_CSR_REG_VERSION);
+	if (revision != FREEZE_CSR_SUPPORTED_VERSION)
+		dev_warn(dev,
+			 "%s Freeze Controller unexpected revision %d != %d\n",
+			 __func__, revision, FREEZE_CSR_SUPPORTED_VERSION);
+
+	return fpga_bridge_register(dev, FREEZE_BRIDGE_NAME,
+				    &altera_freeze_br_br_ops, priv);
+}
+
+static int altera_freeze_br_remove(struct platform_device *pdev)
+{
+	fpga_bridge_unregister(&pdev->dev);
+
+	return 0;
+}
+
+static struct platform_driver altera_freeze_br_driver = {
+	.probe = altera_freeze_br_probe,
+	.remove = altera_freeze_br_remove,
+	.driver = {
+		.name	= "altera_freeze_br",
+		.of_match_table = of_match_ptr(altera_freeze_br_of_match),
+	},
+};
+
+module_platform_driver(altera_freeze_br_driver);
+
+MODULE_DESCRIPTION("Altera Freeze Bridge");
+MODULE_AUTHOR("Alan Tull <atull@opensource.altera.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/fpga/altera-hps2fpga.c b/drivers/fpga/altera-hps2fpga.c
new file mode 100644
index 000000000000..4b354c79be31
--- /dev/null
+++ b/drivers/fpga/altera-hps2fpga.c
@@ -0,0 +1,222 @@
+/*
+ * FPGA to/from HPS Bridge Driver for Altera SoCFPGA Devices
+ *
+ *  Copyright (C) 2013-2016 Altera Corporation, All Rights Reserved.
+ *
+ * Includes this patch from the mailing list:
+ *   fpga: altera-hps2fpga: fix HPS2FPGA bridge visibility to L3 masters
+ *   Signed-off-by: Anatolij Gustschin <agust@denx.de>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * This driver manages bridges on a Altera SOCFPGA between the ARM host
+ * processor system (HPS) and the embedded FPGA.
+ *
+ * This driver supports enabling and disabling of the configured ports, which
+ * allows for safe reprogramming of the FPGA, assuming that the new FPGA image
+ * uses the same port configuration.  Bridges must be disabled before
+ * reprogramming the FPGA and re-enabled after the FPGA has been programmed.
+ */
+
+#include <linux/clk.h>
+#include <linux/fpga/fpga-bridge.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include <linux/spinlock.h>
+
+#define ALT_L3_REMAP_OFST			0x0
+#define ALT_L3_REMAP_MPUZERO_MSK		0x00000001
+#define ALT_L3_REMAP_H2F_MSK			0x00000008
+#define ALT_L3_REMAP_LWH2F_MSK			0x00000010
+
+#define HPS2FPGA_BRIDGE_NAME			"hps2fpga"
+#define LWHPS2FPGA_BRIDGE_NAME			"lwhps2fpga"
+#define FPGA2HPS_BRIDGE_NAME			"fpga2hps"
+
+struct altera_hps2fpga_data {
+	const char *name;
+	struct reset_control *bridge_reset;
+	struct regmap *l3reg;
+	unsigned int remap_mask;
+	struct clk *clk;
+};
+
+static int alt_hps2fpga_enable_show(struct fpga_bridge *bridge)
+{
+	struct altera_hps2fpga_data *priv = bridge->priv;
+
+	return reset_control_status(priv->bridge_reset);
+}
+
+/* The L3 REMAP register is write only, so keep a cached value. */
+static unsigned int l3_remap_shadow;
+static spinlock_t l3_remap_lock;
+
+static int _alt_hps2fpga_enable_set(struct altera_hps2fpga_data *priv,
+				    bool enable)
+{
+	unsigned long flags;
+	int ret;
+
+	/* bring bridge out of reset */
+	if (enable)
+		ret = reset_control_deassert(priv->bridge_reset);
+	else
+		ret = reset_control_assert(priv->bridge_reset);
+	if (ret)
+		return ret;
+
+	/* Allow bridge to be visible to L3 masters or not */
+	if (priv->remap_mask) {
+		spin_lock_irqsave(&l3_remap_lock, flags);
+		l3_remap_shadow |= ALT_L3_REMAP_MPUZERO_MSK;
+
+		if (enable)
+			l3_remap_shadow |= priv->remap_mask;
+		else
+			l3_remap_shadow &= ~priv->remap_mask;
+
+		ret = regmap_write(priv->l3reg, ALT_L3_REMAP_OFST,
+				   l3_remap_shadow);
+		spin_unlock_irqrestore(&l3_remap_lock, flags);
+	}
+
+	return ret;
+}
+
+static int alt_hps2fpga_enable_set(struct fpga_bridge *bridge, bool enable)
+{
+	return _alt_hps2fpga_enable_set(bridge->priv, enable);
+}
+
+static const struct fpga_bridge_ops altera_hps2fpga_br_ops = {
+	.enable_set = alt_hps2fpga_enable_set,
+	.enable_show = alt_hps2fpga_enable_show,
+};
+
+static struct altera_hps2fpga_data hps2fpga_data  = {
+	.name = HPS2FPGA_BRIDGE_NAME,
+	.remap_mask = ALT_L3_REMAP_H2F_MSK,
+};
+
+static struct altera_hps2fpga_data lwhps2fpga_data  = {
+	.name = LWHPS2FPGA_BRIDGE_NAME,
+	.remap_mask = ALT_L3_REMAP_LWH2F_MSK,
+};
+
+static struct altera_hps2fpga_data fpga2hps_data  = {
+	.name = FPGA2HPS_BRIDGE_NAME,
+};
+
+static const struct of_device_id altera_fpga_of_match[] = {
+	{ .compatible = "altr,socfpga-hps2fpga-bridge",
+	  .data = &hps2fpga_data },
+	{ .compatible = "altr,socfpga-lwhps2fpga-bridge",
+	  .data = &lwhps2fpga_data },
+	{ .compatible = "altr,socfpga-fpga2hps-bridge",
+	  .data = &fpga2hps_data },
+	{},
+};
+
+static int alt_fpga_bridge_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct altera_hps2fpga_data *priv;
+	const struct of_device_id *of_id;
+	u32 enable;
+	int ret;
+
+	of_id = of_match_device(altera_fpga_of_match, dev);
+	priv = (struct altera_hps2fpga_data *)of_id->data;
+
+	priv->bridge_reset = of_reset_control_get_by_index(dev->of_node, 0);
+	if (IS_ERR(priv->bridge_reset)) {
+		dev_err(dev, "Could not get %s reset control\n", priv->name);
+		return PTR_ERR(priv->bridge_reset);
+	}
+
+	if (priv->remap_mask) {
+		priv->l3reg = syscon_regmap_lookup_by_compatible("altr,l3regs");
+		if (IS_ERR(priv->l3reg)) {
+			dev_err(dev, "regmap for altr,l3regs lookup failed\n");
+			return PTR_ERR(priv->l3reg);
+		}
+	}
+
+	priv->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(priv->clk)) {
+		dev_err(dev, "no clock specified\n");
+		return PTR_ERR(priv->clk);
+	}
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret) {
+		dev_err(dev, "could not enable clock\n");
+		return -EBUSY;
+	}
+
+	spin_lock_init(&l3_remap_lock);
+
+	if (!of_property_read_u32(dev->of_node, "bridge-enable", &enable)) {
+		if (enable > 1) {
+			dev_warn(dev, "invalid bridge-enable %u > 1\n", enable);
+		} else {
+			dev_info(dev, "%s bridge\n",
+				 (enable ? "enabling" : "disabling"));
+
+			ret = _alt_hps2fpga_enable_set(priv, enable);
+			if (ret) {
+				fpga_bridge_unregister(&pdev->dev);
+				return ret;
+			}
+		}
+	}
+
+	return fpga_bridge_register(dev, priv->name, &altera_hps2fpga_br_ops,
+				    priv);
+}
+
+static int alt_fpga_bridge_remove(struct platform_device *pdev)
+{
+	struct fpga_bridge *bridge = platform_get_drvdata(pdev);
+	struct altera_hps2fpga_data *priv = bridge->priv;
+
+	fpga_bridge_unregister(&pdev->dev);
+
+	clk_disable_unprepare(priv->clk);
+
+	return 0;
+}
+
+MODULE_DEVICE_TABLE(of, altera_fpga_of_match);
+
+static struct platform_driver alt_fpga_bridge_driver = {
+	.probe = alt_fpga_bridge_probe,
+	.remove = alt_fpga_bridge_remove,
+	.driver = {
+		.name	= "altera_hps2fpga_bridge",
+		.of_match_table = of_match_ptr(altera_fpga_of_match),
+	},
+};
+
+module_platform_driver(alt_fpga_bridge_driver);
+
+MODULE_DESCRIPTION("Altera SoCFPGA HPS to FPGA Bridge");
+MODULE_AUTHOR("Alan Tull <atull@opensource.altera.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/fpga/fpga-bridge.c b/drivers/fpga/fpga-bridge.c
new file mode 100644
index 000000000000..33ee83e6373c
--- /dev/null
+++ b/drivers/fpga/fpga-bridge.c
@@ -0,0 +1,395 @@
+/*
+ * FPGA Bridge Framework Driver
+ *
+ *  Copyright (C) 2013-2016 Altera Corporation, All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <linux/fpga/fpga-bridge.h>
+#include <linux/idr.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+static DEFINE_IDA(fpga_bridge_ida);
+static struct class *fpga_bridge_class;
+
+/* Lock for adding/removing bridges to linked lists*/
+spinlock_t bridge_list_lock;
+
+static int fpga_bridge_of_node_match(struct device *dev, const void *data)
+{
+	return dev->of_node == data;
+}
+
+/**
+ * fpga_bridge_enable - Enable transactions on the bridge
+ *
+ * @bridge: FPGA bridge
+ *
+ * Return: 0 for success, error code otherwise.
+ */
+int fpga_bridge_enable(struct fpga_bridge *bridge)
+{
+	dev_dbg(&bridge->dev, "enable\n");
+
+	if (bridge->br_ops && bridge->br_ops->enable_set)
+		return bridge->br_ops->enable_set(bridge, 1);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(fpga_bridge_enable);
+
+/**
+ * fpga_bridge_disable - Disable transactions on the bridge
+ *
+ * @bridge: FPGA bridge
+ *
+ * Return: 0 for success, error code otherwise.
+ */
+int fpga_bridge_disable(struct fpga_bridge *bridge)
+{
+	dev_dbg(&bridge->dev, "disable\n");
+
+	if (bridge->br_ops && bridge->br_ops->enable_set)
+		return bridge->br_ops->enable_set(bridge, 0);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(fpga_bridge_disable);
+
+/**
+ * of_fpga_bridge_get - get an exclusive reference to a fpga bridge
+ *
+ * @np: node pointer of a FPGA bridge
+ * @info: fpga image specific information
+ *
+ * Return fpga_bridge struct if successful.
+ * Return -EBUSY if someone already has a reference to the bridge.
+ * Return -ENODEV if @np is not a FPGA Bridge.
+ */
+struct fpga_bridge *of_fpga_bridge_get(struct device_node *np,
+				       struct fpga_image_info *info)
+
+{
+	struct device *dev;
+	struct fpga_bridge *bridge;
+	int ret = -ENODEV;
+
+	dev = class_find_device(fpga_bridge_class, NULL, np,
+				fpga_bridge_of_node_match);
+	if (!dev)
+		goto err_dev;
+
+	bridge = to_fpga_bridge(dev);
+	if (!bridge)
+		goto err_dev;
+
+	bridge->info = info;
+
+	if (!mutex_trylock(&bridge->mutex)) {
+		ret = -EBUSY;
+		goto err_dev;
+	}
+
+	if (!try_module_get(dev->parent->driver->owner))
+		goto err_ll_mod;
+
+	dev_dbg(&bridge->dev, "get\n");
+
+	return bridge;
+
+err_ll_mod:
+	mutex_unlock(&bridge->mutex);
+err_dev:
+	put_device(dev);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(of_fpga_bridge_get);
+
+/**
+ * fpga_bridge_put - release a reference to a bridge
+ *
+ * @bridge: FPGA bridge
+ */
+void fpga_bridge_put(struct fpga_bridge *bridge)
+{
+	dev_dbg(&bridge->dev, "put\n");
+
+	bridge->info = NULL;
+	module_put(bridge->dev.parent->driver->owner);
+	mutex_unlock(&bridge->mutex);
+	put_device(&bridge->dev);
+}
+EXPORT_SYMBOL_GPL(fpga_bridge_put);
+
+/**
+ * fpga_bridges_enable - enable bridges in a list
+ * @bridge_list: list of FPGA bridges
+ *
+ * Enable each bridge in the list.  If list is empty, do nothing.
+ *
+ * Return 0 for success or empty bridge list; return error code otherwise.
+ */
+int fpga_bridges_enable(struct list_head *bridge_list)
+{
+	struct fpga_bridge *bridge;
+	struct list_head *node;
+	int ret;
+
+	list_for_each(node, bridge_list) {
+		bridge = list_entry(node, struct fpga_bridge, node);
+		ret = fpga_bridge_enable(bridge);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(fpga_bridges_enable);
+
+/**
+ * fpga_bridges_disable - disable bridges in a list
+ *
+ * @bridge_list: list of FPGA bridges
+ *
+ * Disable each bridge in the list.  If list is empty, do nothing.
+ *
+ * Return 0 for success or empty bridge list; return error code otherwise.
+ */
+int fpga_bridges_disable(struct list_head *bridge_list)
+{
+	struct fpga_bridge *bridge;
+	struct list_head *node;
+	int ret;
+
+	list_for_each(node, bridge_list) {
+		bridge = list_entry(node, struct fpga_bridge, node);
+		ret = fpga_bridge_disable(bridge);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(fpga_bridges_disable);
+
+/**
+ * fpga_bridges_put - put bridges
+ *
+ * @bridge_list: list of FPGA bridges
+ *
+ * For each bridge in the list, put the bridge and remove it from the list.
+ * If list is empty, do nothing.
+ */
+void fpga_bridges_put(struct list_head *bridge_list)
+{
+	struct fpga_bridge *bridge;
+	struct list_head *node, *next;
+	unsigned long flags;
+
+	list_for_each_safe(node, next, bridge_list) {
+		bridge = list_entry(node, struct fpga_bridge, node);
+
+		fpga_bridge_put(bridge);
+
+		spin_lock_irqsave(&bridge_list_lock, flags);
+		list_del(&bridge->node);
+		spin_unlock_irqrestore(&bridge_list_lock, flags);
+	}
+}
+EXPORT_SYMBOL_GPL(fpga_bridges_put);
+
+/**
+ * fpga_bridges_get_to_list - get a bridge, add it to a list
+ *
+ * @np: node pointer of a FPGA bridge
+ * @info: fpga image specific information
+ * @bridge_list: list of FPGA bridges
+ *
+ * Get an exclusive reference to the bridge and and it to the list.
+ *
+ * Return 0 for success, error code from of_fpga_bridge_get() othewise.
+ */
+int fpga_bridge_get_to_list(struct device_node *np,
+			    struct fpga_image_info *info,
+			    struct list_head *bridge_list)
+{
+	struct fpga_bridge *bridge;
+	unsigned long flags;
+
+	bridge = of_fpga_bridge_get(np, info);
+	if (IS_ERR(bridge))
+		return PTR_ERR(bridge);
+
+	spin_lock_irqsave(&bridge_list_lock, flags);
+	list_add(&bridge->node, bridge_list);
+	spin_unlock_irqrestore(&bridge_list_lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(fpga_bridge_get_to_list);
+
+static ssize_t name_show(struct device *dev,
+			 struct device_attribute *attr, char *buf)
+{
+	struct fpga_bridge *bridge = to_fpga_bridge(dev);
+
+	return sprintf(buf, "%s\n", bridge->name);
+}
+
+static ssize_t state_show(struct device *dev,
+			  struct device_attribute *attr, char *buf)
+{
+	struct fpga_bridge *bridge = to_fpga_bridge(dev);
+	int enable = 1;
+
+	if (bridge->br_ops && bridge->br_ops->enable_show)
+		enable = bridge->br_ops->enable_show(bridge);
+
+	return sprintf(buf, "%s\n", enable ? "enabled" : "disabled");
+}
+
+static DEVICE_ATTR_RO(name);
+static DEVICE_ATTR_RO(state);
+
+static struct attribute *fpga_bridge_attrs[] = {
+	&dev_attr_name.attr,
+	&dev_attr_state.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(fpga_bridge);
+
+/**
+ * fpga_bridge_register - register a fpga bridge driver
+ * @dev:	FPGA bridge device from pdev
+ * @name:	FPGA bridge name
+ * @br_ops:	pointer to structure of fpga bridge ops
+ * @priv:	FPGA bridge private data
+ *
+ * Return: 0 for success, error code otherwise.
+ */
+int fpga_bridge_register(struct device *dev, const char *name,
+			 const struct fpga_bridge_ops *br_ops, void *priv)
+{
+	struct fpga_bridge *bridge;
+	int id, ret = 0;
+
+	if (!name || !strlen(name)) {
+		dev_err(dev, "Attempt to register with no name!\n");
+		return -EINVAL;
+	}
+
+	bridge = kzalloc(sizeof(*bridge), GFP_KERNEL);
+	if (!bridge)
+		return -ENOMEM;
+
+	id = ida_simple_get(&fpga_bridge_ida, 0, 0, GFP_KERNEL);
+	if (id < 0) {
+		ret = id;
+		goto error_kfree;
+	}
+
+	mutex_init(&bridge->mutex);
+	INIT_LIST_HEAD(&bridge->node);
+
+	bridge->name = name;
+	bridge->br_ops = br_ops;
+	bridge->priv = priv;
+
+	device_initialize(&bridge->dev);
+	bridge->dev.class = fpga_bridge_class;
+	bridge->dev.parent = dev;
+	bridge->dev.of_node = dev->of_node;
+	bridge->dev.id = id;
+	dev_set_drvdata(dev, bridge);
+
+	ret = dev_set_name(&bridge->dev, "br%d", id);
+	if (ret)
+		goto error_device;
+
+	ret = device_add(&bridge->dev);
+	if (ret)
+		goto error_device;
+
+	of_platform_populate(dev->of_node, NULL, NULL, dev);
+
+	dev_info(bridge->dev.parent, "fpga bridge [%s] registered\n",
+		 bridge->name);
+
+	return 0;
+
+error_device:
+	ida_simple_remove(&fpga_bridge_ida, id);
+error_kfree:
+	kfree(bridge);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(fpga_bridge_register);
+
+/**
+ * fpga_bridge_unregister - unregister a fpga bridge driver
+ * @dev: FPGA bridge device from pdev
+ */
+void fpga_bridge_unregister(struct device *dev)
+{
+	struct fpga_bridge *bridge = dev_get_drvdata(dev);
+
+	/*
+	 * If the low level driver provides a method for putting bridge into
+	 * a desired state upon unregister, do it.
+	 */
+	if (bridge->br_ops && bridge->br_ops->fpga_bridge_remove)
+		bridge->br_ops->fpga_bridge_remove(bridge);
+
+	device_unregister(&bridge->dev);
+}
+EXPORT_SYMBOL_GPL(fpga_bridge_unregister);
+
+static void fpga_bridge_dev_release(struct device *dev)
+{
+	struct fpga_bridge *bridge = to_fpga_bridge(dev);
+
+	ida_simple_remove(&fpga_bridge_ida, bridge->dev.id);
+	kfree(bridge);
+}
+
+static int __init fpga_bridge_dev_init(void)
+{
+	spin_lock_init(&bridge_list_lock);
+
+	fpga_bridge_class = class_create(THIS_MODULE, "fpga_bridge");
+	if (IS_ERR(fpga_bridge_class))
+		return PTR_ERR(fpga_bridge_class);
+
+	fpga_bridge_class->dev_groups = fpga_bridge_groups;
+	fpga_bridge_class->dev_release = fpga_bridge_dev_release;
+
+	return 0;
+}
+
+static void __exit fpga_bridge_dev_exit(void)
+{
+	class_destroy(fpga_bridge_class);
+	ida_destroy(&fpga_bridge_ida);
+}
+
+MODULE_DESCRIPTION("FPGA Bridge Driver");
+MODULE_AUTHOR("Alan Tull <atull@opensource.altera.com>");
+MODULE_LICENSE("GPL v2");
+
+subsys_initcall(fpga_bridge_dev_init);
+module_exit(fpga_bridge_dev_exit);
diff --git a/drivers/fpga/fpga-mgr.c b/drivers/fpga/fpga-mgr.c
index 953dc9195937..f0a69d3e60a5 100644
--- a/drivers/fpga/fpga-mgr.c
+++ b/drivers/fpga/fpga-mgr.c
@@ -32,19 +32,20 @@ static struct class *fpga_mgr_class;
 /**
  * fpga_mgr_buf_load - load fpga from image in buffer
  * @mgr:	fpga manager
- * @flags:	flags setting fpga confuration modes
+ * @info:	fpga image specific information
  * @buf:	buffer contain fpga image
  * @count:	byte count of buf
  *
  * Step the low level fpga manager through the device-specific steps of getting
  * an FPGA ready to be configured, writing the image to it, then doing whatever
  * post-configuration steps necessary.  This code assumes the caller got the
- * mgr pointer from of_fpga_mgr_get() and checked that it is not an error code.
+ * mgr pointer from of_fpga_mgr_get() or fpga_mgr_get() and checked that it is
+ * not an error code.
  *
  * Return: 0 on success, negative error code otherwise.
  */
-int fpga_mgr_buf_load(struct fpga_manager *mgr, u32 flags, const char *buf,
-		      size_t count)
+int fpga_mgr_buf_load(struct fpga_manager *mgr, struct fpga_image_info *info,
+		      const char *buf, size_t count)
 {
 	struct device *dev = &mgr->dev;
 	int ret;
@@ -52,10 +53,12 @@ int fpga_mgr_buf_load(struct fpga_manager *mgr, u32 flags, const char *buf,
 	/*
 	 * Call the low level driver's write_init function.  This will do the
 	 * device-specific things to get the FPGA into the state where it is
-	 * ready to receive an FPGA image.
+	 * ready to receive an FPGA image. The low level driver only gets to
+	 * see the first initial_header_size bytes in the buffer.
 	 */
 	mgr->state = FPGA_MGR_STATE_WRITE_INIT;
-	ret = mgr->mops->write_init(mgr, flags, buf, count);
+	ret = mgr->mops->write_init(mgr, info, buf,
+				    min(mgr->mops->initial_header_size, count));
 	if (ret) {
 		dev_err(dev, "Error preparing FPGA for writing\n");
 		mgr->state = FPGA_MGR_STATE_WRITE_INIT_ERR;
@@ -78,7 +81,7 @@ int fpga_mgr_buf_load(struct fpga_manager *mgr, u32 flags, const char *buf,
 	 * steps to finish and set the FPGA into operating mode.
 	 */
 	mgr->state = FPGA_MGR_STATE_WRITE_COMPLETE;
-	ret = mgr->mops->write_complete(mgr, flags);
+	ret = mgr->mops->write_complete(mgr, info);
 	if (ret) {
 		dev_err(dev, "Error after writing image data to FPGA\n");
 		mgr->state = FPGA_MGR_STATE_WRITE_COMPLETE_ERR;
@@ -93,17 +96,19 @@ EXPORT_SYMBOL_GPL(fpga_mgr_buf_load);
 /**
  * fpga_mgr_firmware_load - request firmware and load to fpga
  * @mgr:	fpga manager
- * @flags:	flags setting fpga confuration modes
+ * @info:	fpga image specific information
  * @image_name:	name of image file on the firmware search path
  *
  * Request an FPGA image using the firmware class, then write out to the FPGA.
  * Update the state before each step to provide info on what step failed if
  * there is a failure.  This code assumes the caller got the mgr pointer
- * from of_fpga_mgr_get() and checked that it is not an error code.
+ * from of_fpga_mgr_get() or fpga_mgr_get() and checked that it is not an error
+ * code.
  *
  * Return: 0 on success, negative error code otherwise.
  */
-int fpga_mgr_firmware_load(struct fpga_manager *mgr, u32 flags,
+int fpga_mgr_firmware_load(struct fpga_manager *mgr,
+			   struct fpga_image_info *info,
 			   const char *image_name)
 {
 	struct device *dev = &mgr->dev;
@@ -121,7 +126,7 @@ int fpga_mgr_firmware_load(struct fpga_manager *mgr, u32 flags,
 		return ret;
 	}
 
-	ret = fpga_mgr_buf_load(mgr, flags, fw->data, fw->size);
+	ret = fpga_mgr_buf_load(mgr, info, fw->data, fw->size);
 
 	release_firmware(fw);
 
@@ -181,30 +186,11 @@ static struct attribute *fpga_mgr_attrs[] = {
 };
 ATTRIBUTE_GROUPS(fpga_mgr);
 
-static int fpga_mgr_of_node_match(struct device *dev, const void *data)
-{
-	return dev->of_node == data;
-}
-
-/**
- * of_fpga_mgr_get - get an exclusive reference to a fpga mgr
- * @node:	device node
- *
- * Given a device node, get an exclusive reference to a fpga mgr.
- *
- * Return: fpga manager struct or IS_ERR() condition containing error code.
- */
-struct fpga_manager *of_fpga_mgr_get(struct device_node *node)
+struct fpga_manager *__fpga_mgr_get(struct device *dev)
 {
 	struct fpga_manager *mgr;
-	struct device *dev;
 	int ret = -ENODEV;
 
-	dev = class_find_device(fpga_mgr_class, NULL, node,
-				fpga_mgr_of_node_match);
-	if (!dev)
-		return ERR_PTR(-ENODEV);
-
 	mgr = to_fpga_manager(dev);
 	if (!mgr)
 		goto err_dev;
@@ -226,6 +212,55 @@ err_dev:
 	put_device(dev);
 	return ERR_PTR(ret);
 }
+
+static int fpga_mgr_dev_match(struct device *dev, const void *data)
+{
+	return dev->parent == data;
+}
+
+/**
+ * fpga_mgr_get - get an exclusive reference to a fpga mgr
+ * @dev:	parent device that fpga mgr was registered with
+ *
+ * Given a device, get an exclusive reference to a fpga mgr.
+ *
+ * Return: fpga manager struct or IS_ERR() condition containing error code.
+ */
+struct fpga_manager *fpga_mgr_get(struct device *dev)
+{
+	struct device *mgr_dev = class_find_device(fpga_mgr_class, NULL, dev,
+						   fpga_mgr_dev_match);
+	if (!mgr_dev)
+		return ERR_PTR(-ENODEV);
+
+	return __fpga_mgr_get(mgr_dev);
+}
+EXPORT_SYMBOL_GPL(fpga_mgr_get);
+
+static int fpga_mgr_of_node_match(struct device *dev, const void *data)
+{
+	return dev->of_node == data;
+}
+
+/**
+ * of_fpga_mgr_get - get an exclusive reference to a fpga mgr
+ * @node:	device node
+ *
+ * Given a device node, get an exclusive reference to a fpga mgr.
+ *
+ * Return: fpga manager struct or IS_ERR() condition containing error code.
+ */
+struct fpga_manager *of_fpga_mgr_get(struct device_node *node)
+{
+	struct device *dev;
+
+	dev = class_find_device(fpga_mgr_class, NULL, node,
+				fpga_mgr_of_node_match);
+	if (!dev)
+		return ERR_PTR(-ENODEV);
+
+	return __fpga_mgr_get(dev);
+}
 EXPORT_SYMBOL_GPL(of_fpga_mgr_get);
 
 /**
diff --git a/drivers/fpga/fpga-region.c b/drivers/fpga/fpga-region.c
new file mode 100644
index 000000000000..3222fdbad75a
--- /dev/null
+++ b/drivers/fpga/fpga-region.c
@@ -0,0 +1,603 @@
+/*
+ * FPGA Region - Device Tree support for FPGA programming under Linux
+ *
+ *  Copyright (C) 2013-2016 Altera Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/fpga/fpga-bridge.h>
+#include <linux/fpga/fpga-mgr.h>
+#include <linux/idr.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+/**
+ * struct fpga_region - FPGA Region structure
+ * @dev: FPGA Region device
+ * @mutex: enforces exclusive reference to region
+ * @bridge_list: list of FPGA bridges specified in region
+ * @info: fpga image specific information
+ */
+struct fpga_region {
+	struct device dev;
+	struct mutex mutex; /* for exclusive reference to region */
+	struct list_head bridge_list;
+	struct fpga_image_info *info;
+};
+
+#define to_fpga_region(d) container_of(d, struct fpga_region, dev)
+
+static DEFINE_IDA(fpga_region_ida);
+static struct class *fpga_region_class;
+
+static const struct of_device_id fpga_region_of_match[] = {
+	{ .compatible = "fpga-region", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, fpga_region_of_match);
+
+static int fpga_region_of_node_match(struct device *dev, const void *data)
+{
+	return dev->of_node == data;
+}
+
+/**
+ * fpga_region_find - find FPGA region
+ * @np: device node of FPGA Region
+ * Caller will need to put_device(&region->dev) when done.
+ * Returns FPGA Region struct or NULL
+ */
+static struct fpga_region *fpga_region_find(struct device_node *np)
+{
+	struct device *dev;
+
+	dev = class_find_device(fpga_region_class, NULL, np,
+				fpga_region_of_node_match);
+	if (!dev)
+		return NULL;
+
+	return to_fpga_region(dev);
+}
+
+/**
+ * fpga_region_get - get an exclusive reference to a fpga region
+ * @region: FPGA Region struct
+ *
+ * Caller should call fpga_region_put() when done with region.
+ *
+ * Return fpga_region struct if successful.
+ * Return -EBUSY if someone already has a reference to the region.
+ * Return -ENODEV if @np is not a FPGA Region.
+ */
+static struct fpga_region *fpga_region_get(struct fpga_region *region)
+{
+	struct device *dev = &region->dev;
+
+	if (!mutex_trylock(&region->mutex)) {
+		dev_dbg(dev, "%s: FPGA Region already in use\n", __func__);
+		return ERR_PTR(-EBUSY);
+	}
+
+	get_device(dev);
+	of_node_get(dev->of_node);
+	if (!try_module_get(dev->parent->driver->owner)) {
+		of_node_put(dev->of_node);
+		put_device(dev);
+		mutex_unlock(&region->mutex);
+		return ERR_PTR(-ENODEV);
+	}
+
+	dev_dbg(&region->dev, "get\n");
+
+	return region;
+}
+
+/**
+ * fpga_region_put - release a reference to a region
+ *
+ * @region: FPGA region
+ */
+static void fpga_region_put(struct fpga_region *region)
+{
+	struct device *dev = &region->dev;
+
+	dev_dbg(&region->dev, "put\n");
+
+	module_put(dev->parent->driver->owner);
+	of_node_put(dev->of_node);
+	put_device(dev);
+	mutex_unlock(&region->mutex);
+}
+
+/**
+ * fpga_region_get_manager - get exclusive reference for FPGA manager
+ * @region: FPGA region
+ *
+ * Get FPGA Manager from "fpga-mgr" property or from ancestor region.
+ *
+ * Caller should call fpga_mgr_put() when done with manager.
+ *
+ * Return: fpga manager struct or IS_ERR() condition containing error code.
+ */
+static struct fpga_manager *fpga_region_get_manager(struct fpga_region *region)
+{
+	struct device *dev = &region->dev;
+	struct device_node *np = dev->of_node;
+	struct device_node  *mgr_node;
+	struct fpga_manager *mgr;
+
+	of_node_get(np);
+	while (np) {
+		if (of_device_is_compatible(np, "fpga-region")) {
+			mgr_node = of_parse_phandle(np, "fpga-mgr", 0);
+			if (mgr_node) {
+				mgr = of_fpga_mgr_get(mgr_node);
+				of_node_put(np);
+				return mgr;
+			}
+		}
+		np = of_get_next_parent(np);
+	}
+	of_node_put(np);
+
+	return ERR_PTR(-EINVAL);
+}
+
+/**
+ * fpga_region_get_bridges - create a list of bridges
+ * @region: FPGA region
+ * @overlay: device node of the overlay
+ *
+ * Create a list of bridges including the parent bridge and the bridges
+ * specified by "fpga-bridges" property.  Note that the
+ * fpga_bridges_enable/disable/put functions are all fine with an empty list
+ * if that happens.
+ *
+ * Caller should call fpga_bridges_put(&region->bridge_list) when
+ * done with the bridges.
+ *
+ * Return 0 for success (even if there are no bridges specified)
+ * or -EBUSY if any of the bridges are in use.
+ */
+static int fpga_region_get_bridges(struct fpga_region *region,
+				   struct device_node *overlay)
+{
+	struct device *dev = &region->dev;
+	struct device_node *region_np = dev->of_node;
+	struct device_node *br, *np, *parent_br = NULL;
+	int i, ret;
+
+	/* If parent is a bridge, add to list */
+	ret = fpga_bridge_get_to_list(region_np->parent, region->info,
+				      &region->bridge_list);
+	if (ret == -EBUSY)
+		return ret;
+
+	if (!ret)
+		parent_br = region_np->parent;
+
+	/* If overlay has a list of bridges, use it. */
+	if (of_parse_phandle(overlay, "fpga-bridges", 0))
+		np = overlay;
+	else
+		np = region_np;
+
+	for (i = 0; ; i++) {
+		br = of_parse_phandle(np, "fpga-bridges", i);
+		if (!br)
+			break;
+
+		/* If parent bridge is in list, skip it. */
+		if (br == parent_br)
+			continue;
+
+		/* If node is a bridge, get it and add to list */
+		ret = fpga_bridge_get_to_list(br, region->info,
+					      &region->bridge_list);
+
+		/* If any of the bridges are in use, give up */
+		if (ret == -EBUSY) {
+			fpga_bridges_put(&region->bridge_list);
+			return -EBUSY;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * fpga_region_program_fpga - program FPGA
+ * @region: FPGA region
+ * @firmware_name: name of FPGA image firmware file
+ * @overlay: device node of the overlay
+ * Program an FPGA using information in the device tree.
+ * Function assumes that there is a firmware-name property.
+ * Return 0 for success or negative error code.
+ */
+static int fpga_region_program_fpga(struct fpga_region *region,
+				    const char *firmware_name,
+				    struct device_node *overlay)
+{
+	struct fpga_manager *mgr;
+	int ret;
+
+	region = fpga_region_get(region);
+	if (IS_ERR(region)) {
+		pr_err("failed to get fpga region\n");
+		return PTR_ERR(region);
+	}
+
+	mgr = fpga_region_get_manager(region);
+	if (IS_ERR(mgr)) {
+		pr_err("failed to get fpga region manager\n");
+		return PTR_ERR(mgr);
+	}
+
+	ret = fpga_region_get_bridges(region, overlay);
+	if (ret) {
+		pr_err("failed to get fpga region bridges\n");
+		goto err_put_mgr;
+	}
+
+	ret = fpga_bridges_disable(&region->bridge_list);
+	if (ret) {
+		pr_err("failed to disable region bridges\n");
+		goto err_put_br;
+	}
+
+	ret = fpga_mgr_firmware_load(mgr, region->info, firmware_name);
+	if (ret) {
+		pr_err("failed to load fpga image\n");
+		goto err_put_br;
+	}
+
+	ret = fpga_bridges_enable(&region->bridge_list);
+	if (ret) {
+		pr_err("failed to enable region bridges\n");
+		goto err_put_br;
+	}
+
+	fpga_mgr_put(mgr);
+	fpga_region_put(region);
+
+	return 0;
+
+err_put_br:
+	fpga_bridges_put(&region->bridge_list);
+err_put_mgr:
+	fpga_mgr_put(mgr);
+	fpga_region_put(region);
+
+	return ret;
+}
+
+/**
+ * child_regions_with_firmware
+ * @overlay: device node of the overlay
+ *
+ * If the overlay adds child FPGA regions, they are not allowed to have
+ * firmware-name property.
+ *
+ * Return 0 for OK or -EINVAL if child FPGA region adds firmware-name.
+ */
+static int child_regions_with_firmware(struct device_node *overlay)
+{
+	struct device_node *child_region;
+	const char *child_firmware_name;
+	int ret = 0;
+
+	of_node_get(overlay);
+
+	child_region = of_find_matching_node(overlay, fpga_region_of_match);
+	while (child_region) {
+		if (!of_property_read_string(child_region, "firmware-name",
+					     &child_firmware_name)) {
+			ret = -EINVAL;
+			break;
+		}
+		child_region = of_find_matching_node(child_region,
+						     fpga_region_of_match);
+	}
+
+	of_node_put(child_region);
+
+	if (ret)
+		pr_err("firmware-name not allowed in child FPGA region: %s",
+		       child_region->full_name);
+
+	return ret;
+}
+
+/**
+ * fpga_region_notify_pre_apply - pre-apply overlay notification
+ *
+ * @region: FPGA region that the overlay was applied to
+ * @nd: overlay notification data
+ *
+ * Called after when an overlay targeted to a FPGA Region is about to be
+ * applied.  Function will check the properties that will be added to the FPGA
+ * region.  If the checks pass, it will program the FPGA.
+ *
+ * The checks are:
+ * The overlay must add either firmware-name or external-fpga-config property
+ * to the FPGA Region.
+ *
+ *   firmware-name        : program the FPGA
+ *   external-fpga-config : FPGA is already programmed
+ *
+ * The overlay can add other FPGA regions, but child FPGA regions cannot have a
+ * firmware-name property since those regions don't exist yet.
+ *
+ * If the overlay that breaks the rules, notifier returns an error and the
+ * overlay is rejected before it goes into the main tree.
+ *
+ * Returns 0 for success or negative error code for failure.
+ */
+static int fpga_region_notify_pre_apply(struct fpga_region *region,
+					struct of_overlay_notify_data *nd)
+{
+	const char *firmware_name = NULL;
+	struct fpga_image_info *info;
+	int ret;
+
+	info = devm_kzalloc(&region->dev, sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	region->info = info;
+
+	/* Reject overlay if child FPGA Regions have firmware-name property */
+	ret = child_regions_with_firmware(nd->overlay);
+	if (ret)
+		return ret;
+
+	/* Read FPGA region properties from the overlay */
+	if (of_property_read_bool(nd->overlay, "partial-fpga-config"))
+		info->flags |= FPGA_MGR_PARTIAL_RECONFIG;
+
+	if (of_property_read_bool(nd->overlay, "external-fpga-config"))
+		info->flags |= FPGA_MGR_EXTERNAL_CONFIG;
+
+	of_property_read_string(nd->overlay, "firmware-name", &firmware_name);
+
+	of_property_read_u32(nd->overlay, "region-unfreeze-timeout-us",
+			     &info->enable_timeout_us);
+
+	of_property_read_u32(nd->overlay, "region-freeze-timeout-us",
+			     &info->disable_timeout_us);
+
+	/* If FPGA was externally programmed, don't specify firmware */
+	if ((info->flags & FPGA_MGR_EXTERNAL_CONFIG) && firmware_name) {
+		pr_err("error: specified firmware and external-fpga-config");
+		return -EINVAL;
+	}
+
+	/* FPGA is already configured externally.  We're done. */
+	if (info->flags & FPGA_MGR_EXTERNAL_CONFIG)
+		return 0;
+
+	/* If we got this far, we should be programming the FPGA */
+	if (!firmware_name) {
+		pr_err("should specify firmware-name or external-fpga-config\n");
+		return -EINVAL;
+	}
+
+	return fpga_region_program_fpga(region, firmware_name, nd->overlay);
+}
+
+/**
+ * fpga_region_notify_post_remove - post-remove overlay notification
+ *
+ * @region: FPGA region that was targeted by the overlay that was removed
+ * @nd: overlay notification data
+ *
+ * Called after an overlay has been removed if the overlay's target was a
+ * FPGA region.
+ */
+static void fpga_region_notify_post_remove(struct fpga_region *region,
+					   struct of_overlay_notify_data *nd)
+{
+	fpga_bridges_disable(&region->bridge_list);
+	fpga_bridges_put(&region->bridge_list);
+	devm_kfree(&region->dev, region->info);
+	region->info = NULL;
+}
+
+/**
+ * of_fpga_region_notify - reconfig notifier for dynamic DT changes
+ * @nb:		notifier block
+ * @action:	notifier action
+ * @arg:	reconfig data
+ *
+ * This notifier handles programming a FPGA when a "firmware-name" property is
+ * added to a fpga-region.
+ *
+ * Returns NOTIFY_OK or error if FPGA programming fails.
+ */
+static int of_fpga_region_notify(struct notifier_block *nb,
+				 unsigned long action, void *arg)
+{
+	struct of_overlay_notify_data *nd = arg;
+	struct fpga_region *region;
+	int ret;
+
+	switch (action) {
+	case OF_OVERLAY_PRE_APPLY:
+		pr_debug("%s OF_OVERLAY_PRE_APPLY\n", __func__);
+		break;
+	case OF_OVERLAY_POST_APPLY:
+		pr_debug("%s OF_OVERLAY_POST_APPLY\n", __func__);
+		return NOTIFY_OK;       /* not for us */
+	case OF_OVERLAY_PRE_REMOVE:
+		pr_debug("%s OF_OVERLAY_PRE_REMOVE\n", __func__);
+		return NOTIFY_OK;       /* not for us */
+	case OF_OVERLAY_POST_REMOVE:
+		pr_debug("%s OF_OVERLAY_POST_REMOVE\n", __func__);
+		break;
+	default:			/* should not happen */
+		return NOTIFY_OK;
+	}
+
+	region = fpga_region_find(nd->target);
+	if (!region)
+		return NOTIFY_OK;
+
+	ret = 0;
+	switch (action) {
+	case OF_OVERLAY_PRE_APPLY:
+		ret = fpga_region_notify_pre_apply(region, nd);
+		break;
+
+	case OF_OVERLAY_POST_REMOVE:
+		fpga_region_notify_post_remove(region, nd);
+		break;
+	}
+
+	put_device(&region->dev);
+
+	if (ret)
+		return notifier_from_errno(ret);
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block fpga_region_of_nb = {
+	.notifier_call = of_fpga_region_notify,
+};
+
+static int fpga_region_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct fpga_region *region;
+	int id, ret = 0;
+
+	region = kzalloc(sizeof(*region), GFP_KERNEL);
+	if (!region)
+		return -ENOMEM;
+
+	id = ida_simple_get(&fpga_region_ida, 0, 0, GFP_KERNEL);
+	if (id < 0) {
+		ret = id;
+		goto err_kfree;
+	}
+
+	mutex_init(&region->mutex);
+	INIT_LIST_HEAD(&region->bridge_list);
+
+	device_initialize(&region->dev);
+	region->dev.class = fpga_region_class;
+	region->dev.parent = dev;
+	region->dev.of_node = np;
+	region->dev.id = id;
+	dev_set_drvdata(dev, region);
+
+	ret = dev_set_name(&region->dev, "region%d", id);
+	if (ret)
+		goto err_remove;
+
+	ret = device_add(&region->dev);
+	if (ret)
+		goto err_remove;
+
+	of_platform_populate(np, fpga_region_of_match, NULL, &region->dev);
+
+	dev_info(dev, "FPGA Region probed\n");
+
+	return 0;
+
+err_remove:
+	ida_simple_remove(&fpga_region_ida, id);
+err_kfree:
+	kfree(region);
+
+	return ret;
+}
+
+static int fpga_region_remove(struct platform_device *pdev)
+{
+	struct fpga_region *region = platform_get_drvdata(pdev);
+
+	device_unregister(&region->dev);
+
+	return 0;
+}
+
+static struct platform_driver fpga_region_driver = {
+	.probe = fpga_region_probe,
+	.remove = fpga_region_remove,
+	.driver = {
+		.name	= "fpga-region",
+		.of_match_table = of_match_ptr(fpga_region_of_match),
+	},
+};
+
+static void fpga_region_dev_release(struct device *dev)
+{
+	struct fpga_region *region = to_fpga_region(dev);
+
+	ida_simple_remove(&fpga_region_ida, region->dev.id);
+	kfree(region);
+}
+
+/**
+ * fpga_region_init - init function for fpga_region class
+ * Creates the fpga_region class and registers a reconfig notifier.
+ */
+static int __init fpga_region_init(void)
+{
+	int ret;
+
+	fpga_region_class = class_create(THIS_MODULE, "fpga_region");
+	if (IS_ERR(fpga_region_class))
+		return PTR_ERR(fpga_region_class);
+
+	fpga_region_class->dev_release = fpga_region_dev_release;
+
+	ret = of_overlay_notifier_register(&fpga_region_of_nb);
+	if (ret)
+		goto err_class;
+
+	ret = platform_driver_register(&fpga_region_driver);
+	if (ret)
+		goto err_plat;
+
+	return 0;
+
+err_plat:
+	of_overlay_notifier_unregister(&fpga_region_of_nb);
+err_class:
+	class_destroy(fpga_region_class);
+	ida_destroy(&fpga_region_ida);
+	return ret;
+}
+
+static void __exit fpga_region_exit(void)
+{
+	platform_driver_unregister(&fpga_region_driver);
+	of_overlay_notifier_unregister(&fpga_region_of_nb);
+	class_destroy(fpga_region_class);
+	ida_destroy(&fpga_region_ida);
+}
+
+subsys_initcall(fpga_region_init);
+module_exit(fpga_region_exit);
+
+MODULE_DESCRIPTION("FPGA Region");
+MODULE_AUTHOR("Alan Tull <atull@opensource.altera.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/fpga/socfpga-a10.c b/drivers/fpga/socfpga-a10.c
new file mode 100644
index 000000000000..f8770af0f6b5
--- /dev/null
+++ b/drivers/fpga/socfpga-a10.c
@@ -0,0 +1,557 @@
+/*
+ * FPGA Manager Driver for Altera Arria10 SoCFPGA
+ *
+ * Copyright (C) 2015-2016 Altera Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/delay.h>
+#include <linux/fpga/fpga-mgr.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/regmap.h>
+
+#define A10_FPGAMGR_DCLKCNT_OFST				0x08
+#define A10_FPGAMGR_DCLKSTAT_OFST				0x0c
+#define A10_FPGAMGR_IMGCFG_CTL_00_OFST				0x70
+#define A10_FPGAMGR_IMGCFG_CTL_01_OFST				0x74
+#define A10_FPGAMGR_IMGCFG_CTL_02_OFST				0x78
+#define A10_FPGAMGR_IMGCFG_STAT_OFST				0x80
+
+#define A10_FPGAMGR_DCLKSTAT_DCLKDONE				BIT(0)
+
+#define A10_FPGAMGR_IMGCFG_CTL_00_S2F_NENABLE_NCONFIG		BIT(0)
+#define A10_FPGAMGR_IMGCFG_CTL_00_S2F_NENABLE_NSTATUS		BIT(1)
+#define A10_FPGAMGR_IMGCFG_CTL_00_S2F_NENABLE_CONDONE		BIT(2)
+#define A10_FPGAMGR_IMGCFG_CTL_00_S2F_NCONFIG			BIT(8)
+#define A10_FPGAMGR_IMGCFG_CTL_00_S2F_NSTATUS_OE		BIT(16)
+#define A10_FPGAMGR_IMGCFG_CTL_00_S2F_CONDONE_OE		BIT(24)
+
+#define A10_FPGAMGR_IMGCFG_CTL_01_S2F_NENABLE_CONFIG		BIT(0)
+#define A10_FPGAMGR_IMGCFG_CTL_01_S2F_PR_REQUEST		BIT(16)
+#define A10_FPGAMGR_IMGCFG_CTL_01_S2F_NCE			BIT(24)
+
+#define A10_FPGAMGR_IMGCFG_CTL_02_EN_CFG_CTRL			BIT(0)
+#define A10_FPGAMGR_IMGCFG_CTL_02_CDRATIO_MASK		(BIT(16) | BIT(17))
+#define A10_FPGAMGR_IMGCFG_CTL_02_CDRATIO_SHIFT			16
+#define A10_FPGAMGR_IMGCFG_CTL_02_CFGWIDTH			BIT(24)
+#define A10_FPGAMGR_IMGCFG_CTL_02_CFGWIDTH_SHIFT		24
+
+#define A10_FPGAMGR_IMGCFG_STAT_F2S_CRC_ERROR			BIT(0)
+#define A10_FPGAMGR_IMGCFG_STAT_F2S_EARLY_USERMODE		BIT(1)
+#define A10_FPGAMGR_IMGCFG_STAT_F2S_USERMODE			BIT(2)
+#define A10_FPGAMGR_IMGCFG_STAT_F2S_NSTATUS_PIN			BIT(4)
+#define A10_FPGAMGR_IMGCFG_STAT_F2S_CONDONE_PIN			BIT(6)
+#define A10_FPGAMGR_IMGCFG_STAT_F2S_PR_READY			BIT(9)
+#define A10_FPGAMGR_IMGCFG_STAT_F2S_PR_DONE			BIT(10)
+#define A10_FPGAMGR_IMGCFG_STAT_F2S_PR_ERROR			BIT(11)
+#define A10_FPGAMGR_IMGCFG_STAT_F2S_NCONFIG_PIN			BIT(12)
+#define A10_FPGAMGR_IMGCFG_STAT_F2S_MSEL_MASK	(BIT(16) | BIT(17) | BIT(18))
+#define A10_FPGAMGR_IMGCFG_STAT_F2S_MSEL_SHIFT		        16
+
+/* FPGA CD Ratio Value */
+#define CDRATIO_x1						0x0
+#define CDRATIO_x2						0x1
+#define CDRATIO_x4						0x2
+#define CDRATIO_x8						0x3
+
+/* Configuration width 16/32 bit */
+#define CFGWDTH_32						1
+#define CFGWDTH_16						0
+
+/*
+ * struct a10_fpga_priv - private data for fpga manager
+ * @regmap: regmap for register access
+ * @fpga_data_addr: iomap for single address data register to FPGA
+ * @clk: clock
+ */
+struct a10_fpga_priv {
+	struct regmap *regmap;
+	void __iomem *fpga_data_addr;
+	struct clk *clk;
+};
+
+static bool socfpga_a10_fpga_writeable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case A10_FPGAMGR_DCLKCNT_OFST:
+	case A10_FPGAMGR_DCLKSTAT_OFST:
+	case A10_FPGAMGR_IMGCFG_CTL_00_OFST:
+	case A10_FPGAMGR_IMGCFG_CTL_01_OFST:
+	case A10_FPGAMGR_IMGCFG_CTL_02_OFST:
+		return true;
+	}
+	return false;
+}
+
+static bool socfpga_a10_fpga_readable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case A10_FPGAMGR_DCLKCNT_OFST:
+	case A10_FPGAMGR_DCLKSTAT_OFST:
+	case A10_FPGAMGR_IMGCFG_CTL_00_OFST:
+	case A10_FPGAMGR_IMGCFG_CTL_01_OFST:
+	case A10_FPGAMGR_IMGCFG_CTL_02_OFST:
+	case A10_FPGAMGR_IMGCFG_STAT_OFST:
+		return true;
+	}
+	return false;
+}
+
+static const struct regmap_config socfpga_a10_fpga_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.writeable_reg = socfpga_a10_fpga_writeable_reg,
+	.readable_reg = socfpga_a10_fpga_readable_reg,
+	.max_register = A10_FPGAMGR_IMGCFG_STAT_OFST,
+	.cache_type = REGCACHE_NONE,
+};
+
+/*
+ * from the register map description of cdratio in imgcfg_ctrl_02:
+ *  Normal Configuration    : 32bit Passive Parallel
+ *  Partial Reconfiguration : 16bit Passive Parallel
+ */
+static void socfpga_a10_fpga_set_cfg_width(struct a10_fpga_priv *priv,
+					   int width)
+{
+	width <<= A10_FPGAMGR_IMGCFG_CTL_02_CFGWIDTH_SHIFT;
+
+	regmap_update_bits(priv->regmap, A10_FPGAMGR_IMGCFG_CTL_02_OFST,
+			   A10_FPGAMGR_IMGCFG_CTL_02_CFGWIDTH, width);
+}
+
+static void socfpga_a10_fpga_generate_dclks(struct a10_fpga_priv *priv,
+					    u32 count)
+{
+	u32 val;
+
+	/* Clear any existing DONE status. */
+	regmap_write(priv->regmap, A10_FPGAMGR_DCLKSTAT_OFST,
+		     A10_FPGAMGR_DCLKSTAT_DCLKDONE);
+
+	/* Issue the DCLK regmap. */
+	regmap_write(priv->regmap, A10_FPGAMGR_DCLKCNT_OFST, count);
+
+	/* wait till the dclkcnt done */
+	regmap_read_poll_timeout(priv->regmap, A10_FPGAMGR_DCLKSTAT_OFST, val,
+				 val, 1, 100);
+
+	/* Clear DONE status. */
+	regmap_write(priv->regmap, A10_FPGAMGR_DCLKSTAT_OFST,
+		     A10_FPGAMGR_DCLKSTAT_DCLKDONE);
+}
+
+#define RBF_ENCRYPTION_MODE_OFFSET		69
+#define RBF_DECOMPRESS_OFFSET			229
+
+static int socfpga_a10_fpga_encrypted(u32 *buf32, size_t buf32_size)
+{
+	if (buf32_size < RBF_ENCRYPTION_MODE_OFFSET + 1)
+		return -EINVAL;
+
+	/* Is the bitstream encrypted? */
+	return ((buf32[RBF_ENCRYPTION_MODE_OFFSET] >> 2) & 3) != 0;
+}
+
+static int socfpga_a10_fpga_compressed(u32 *buf32, size_t buf32_size)
+{
+	if (buf32_size < RBF_DECOMPRESS_OFFSET + 1)
+		return -EINVAL;
+
+	/* Is the bitstream compressed? */
+	return !((buf32[RBF_DECOMPRESS_OFFSET] >> 1) & 1);
+}
+
+static unsigned int socfpga_a10_fpga_get_cd_ratio(unsigned int cfg_width,
+						  bool encrypt, bool compress)
+{
+	unsigned int cd_ratio;
+
+	/*
+	 * cd ratio is dependent on cfg width and whether the bitstream
+	 * is encrypted and/or compressed.
+	 *
+	 * | width | encr. | compr. | cd ratio |
+	 * |  16   |   0   |   0    |     1    |
+	 * |  16   |   0   |   1    |     4    |
+	 * |  16   |   1   |   0    |     2    |
+	 * |  16   |   1   |   1    |     4    |
+	 * |  32   |   0   |   0    |     1    |
+	 * |  32   |   0   |   1    |     8    |
+	 * |  32   |   1   |   0    |     4    |
+	 * |  32   |   1   |   1    |     8    |
+	 */
+	if (!compress && !encrypt)
+		return CDRATIO_x1;
+
+	if (compress)
+		cd_ratio = CDRATIO_x4;
+	else
+		cd_ratio = CDRATIO_x2;
+
+	/* If 32 bit, double the cd ratio by incrementing the field  */
+	if (cfg_width == CFGWDTH_32)
+		cd_ratio += 1;
+
+	return cd_ratio;
+}
+
+static int socfpga_a10_fpga_set_cdratio(struct fpga_manager *mgr,
+					unsigned int cfg_width,
+					const char *buf, size_t count)
+{
+	struct a10_fpga_priv *priv = mgr->priv;
+	unsigned int cd_ratio;
+	int encrypt, compress;
+
+	encrypt = socfpga_a10_fpga_encrypted((u32 *)buf, count / 4);
+	if (encrypt < 0)
+		return -EINVAL;
+
+	compress = socfpga_a10_fpga_compressed((u32 *)buf, count / 4);
+	if (compress < 0)
+		return -EINVAL;
+
+	cd_ratio = socfpga_a10_fpga_get_cd_ratio(cfg_width, encrypt, compress);
+
+	regmap_update_bits(priv->regmap, A10_FPGAMGR_IMGCFG_CTL_02_OFST,
+			   A10_FPGAMGR_IMGCFG_CTL_02_CDRATIO_MASK,
+			   cd_ratio << A10_FPGAMGR_IMGCFG_CTL_02_CDRATIO_SHIFT);
+
+	return 0;
+}
+
+static u32 socfpga_a10_fpga_read_stat(struct a10_fpga_priv *priv)
+{
+	u32 val;
+
+	regmap_read(priv->regmap, A10_FPGAMGR_IMGCFG_STAT_OFST, &val);
+
+	return val;
+}
+
+static int socfpga_a10_fpga_wait_for_pr_ready(struct a10_fpga_priv *priv)
+{
+	u32 reg, i;
+
+	for (i = 0; i < 10 ; i++) {
+		reg = socfpga_a10_fpga_read_stat(priv);
+
+		if (reg & A10_FPGAMGR_IMGCFG_STAT_F2S_PR_ERROR)
+			return -EINVAL;
+
+		if (reg & A10_FPGAMGR_IMGCFG_STAT_F2S_PR_READY)
+			return 0;
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int socfpga_a10_fpga_wait_for_pr_done(struct a10_fpga_priv *priv)
+{
+	u32 reg, i;
+
+	for (i = 0; i < 10 ; i++) {
+		reg = socfpga_a10_fpga_read_stat(priv);
+
+		if (reg & A10_FPGAMGR_IMGCFG_STAT_F2S_PR_ERROR)
+			return -EINVAL;
+
+		if (reg & A10_FPGAMGR_IMGCFG_STAT_F2S_PR_DONE)
+			return 0;
+	}
+
+	return -ETIMEDOUT;
+}
+
+/* Start the FPGA programming by initialize the FPGA Manager */
+static int socfpga_a10_fpga_write_init(struct fpga_manager *mgr,
+				       struct fpga_image_info *info,
+				       const char *buf, size_t count)
+{
+	struct a10_fpga_priv *priv = mgr->priv;
+	unsigned int cfg_width;
+	u32 msel, stat, mask;
+	int ret;
+
+	if (info->flags & FPGA_MGR_PARTIAL_RECONFIG)
+		cfg_width = CFGWDTH_16;
+	else
+		return -EINVAL;
+
+	/* Check for passive parallel (msel == 000 or 001) */
+	msel = socfpga_a10_fpga_read_stat(priv);
+	msel &= A10_FPGAMGR_IMGCFG_STAT_F2S_MSEL_MASK;
+	msel >>= A10_FPGAMGR_IMGCFG_STAT_F2S_MSEL_SHIFT;
+	if ((msel != 0) && (msel != 1)) {
+		dev_dbg(&mgr->dev, "Fail: invalid msel=%d\n", msel);
+		return -EINVAL;
+	}
+
+	/* Make sure no external devices are interfering */
+	stat = socfpga_a10_fpga_read_stat(priv);
+	mask = A10_FPGAMGR_IMGCFG_STAT_F2S_NCONFIG_PIN |
+	       A10_FPGAMGR_IMGCFG_STAT_F2S_NSTATUS_PIN;
+	if ((stat & mask) != mask)
+		return -EINVAL;
+
+	/* Set cfg width */
+	socfpga_a10_fpga_set_cfg_width(priv, cfg_width);
+
+	/* Determine cd ratio from bitstream header and set cd ratio */
+	ret = socfpga_a10_fpga_set_cdratio(mgr, cfg_width, buf, count);
+	if (ret)
+		return ret;
+
+	/*
+	 * Clear s2f_nce to enable chip select.  Leave pr_request
+	 * unasserted and override disabled.
+	 */
+	regmap_write(priv->regmap, A10_FPGAMGR_IMGCFG_CTL_01_OFST,
+		     A10_FPGAMGR_IMGCFG_CTL_01_S2F_NENABLE_CONFIG);
+
+	/* Set cfg_ctrl to enable s2f dclk and data */
+	regmap_update_bits(priv->regmap, A10_FPGAMGR_IMGCFG_CTL_02_OFST,
+			   A10_FPGAMGR_IMGCFG_CTL_02_EN_CFG_CTRL,
+			   A10_FPGAMGR_IMGCFG_CTL_02_EN_CFG_CTRL);
+
+	/*
+	 * Disable overrides not needed for pr.
+	 * s2f_config==1 leaves reset deasseted.
+	 */
+	regmap_write(priv->regmap, A10_FPGAMGR_IMGCFG_CTL_00_OFST,
+		     A10_FPGAMGR_IMGCFG_CTL_00_S2F_NENABLE_NCONFIG |
+		     A10_FPGAMGR_IMGCFG_CTL_00_S2F_NENABLE_NSTATUS |
+		     A10_FPGAMGR_IMGCFG_CTL_00_S2F_NENABLE_CONDONE |
+		     A10_FPGAMGR_IMGCFG_CTL_00_S2F_NCONFIG);
+
+	/* Enable override for data, dclk, nce, and pr_request to CSS */
+	regmap_update_bits(priv->regmap, A10_FPGAMGR_IMGCFG_CTL_01_OFST,
+			   A10_FPGAMGR_IMGCFG_CTL_01_S2F_NENABLE_CONFIG, 0);
+
+	/* Send some clocks to clear out any errors */
+	socfpga_a10_fpga_generate_dclks(priv, 256);
+
+	/* Assert pr_request */
+	regmap_update_bits(priv->regmap, A10_FPGAMGR_IMGCFG_CTL_01_OFST,
+			   A10_FPGAMGR_IMGCFG_CTL_01_S2F_PR_REQUEST,
+			   A10_FPGAMGR_IMGCFG_CTL_01_S2F_PR_REQUEST);
+
+	/* Provide 2048 DCLKs before starting the config data streaming. */
+	socfpga_a10_fpga_generate_dclks(priv, 0x7ff);
+
+	/* Wait for pr_ready */
+	return socfpga_a10_fpga_wait_for_pr_ready(priv);
+}
+
+/*
+ * write data to the FPGA data register
+ */
+static int socfpga_a10_fpga_write(struct fpga_manager *mgr, const char *buf,
+				  size_t count)
+{
+	struct a10_fpga_priv *priv = mgr->priv;
+	u32 *buffer_32 = (u32 *)buf;
+	size_t i = 0;
+
+	if (count <= 0)
+		return -EINVAL;
+
+	/* Write out the complete 32-bit chunks */
+	while (count >= sizeof(u32)) {
+		writel(buffer_32[i++], priv->fpga_data_addr);
+		count -= sizeof(u32);
+	}
+
+	/* Write out remaining non 32-bit chunks */
+	switch (count) {
+	case 3:
+		writel(buffer_32[i++] & 0x00ffffff, priv->fpga_data_addr);
+		break;
+	case 2:
+		writel(buffer_32[i++] & 0x0000ffff, priv->fpga_data_addr);
+		break;
+	case 1:
+		writel(buffer_32[i++] & 0x000000ff, priv->fpga_data_addr);
+		break;
+	case 0:
+		break;
+	default:
+		/* This will never happen */
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+static int socfpga_a10_fpga_write_complete(struct fpga_manager *mgr,
+					   struct fpga_image_info *info)
+{
+	struct a10_fpga_priv *priv = mgr->priv;
+	u32 reg;
+	int ret;
+
+	/* Wait for pr_done */
+	ret = socfpga_a10_fpga_wait_for_pr_done(priv);
+
+	/* Clear pr_request */
+	regmap_update_bits(priv->regmap, A10_FPGAMGR_IMGCFG_CTL_01_OFST,
+			   A10_FPGAMGR_IMGCFG_CTL_01_S2F_PR_REQUEST, 0);
+
+	/* Send some clocks to clear out any errors */
+	socfpga_a10_fpga_generate_dclks(priv, 256);
+
+	/* Disable s2f dclk and data */
+	regmap_update_bits(priv->regmap, A10_FPGAMGR_IMGCFG_CTL_02_OFST,
+			   A10_FPGAMGR_IMGCFG_CTL_02_EN_CFG_CTRL, 0);
+
+	/* Deassert chip select */
+	regmap_update_bits(priv->regmap, A10_FPGAMGR_IMGCFG_CTL_01_OFST,
+			   A10_FPGAMGR_IMGCFG_CTL_01_S2F_NCE,
+			   A10_FPGAMGR_IMGCFG_CTL_01_S2F_NCE);
+
+	/* Disable data, dclk, nce, and pr_request override to CSS */
+	regmap_update_bits(priv->regmap, A10_FPGAMGR_IMGCFG_CTL_01_OFST,
+			   A10_FPGAMGR_IMGCFG_CTL_01_S2F_NENABLE_CONFIG,
+			   A10_FPGAMGR_IMGCFG_CTL_01_S2F_NENABLE_CONFIG);
+
+	/* Return any errors regarding pr_done or pr_error */
+	if (ret)
+		return ret;
+
+	/* Final check */
+	reg = socfpga_a10_fpga_read_stat(priv);
+
+	if (((reg & A10_FPGAMGR_IMGCFG_STAT_F2S_USERMODE) == 0) ||
+	    ((reg & A10_FPGAMGR_IMGCFG_STAT_F2S_CONDONE_PIN) == 0) ||
+	    ((reg & A10_FPGAMGR_IMGCFG_STAT_F2S_NSTATUS_PIN) == 0)) {
+		dev_dbg(&mgr->dev,
+			"Timeout in final check. Status=%08xf\n", reg);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static enum fpga_mgr_states socfpga_a10_fpga_state(struct fpga_manager *mgr)
+{
+	struct a10_fpga_priv *priv = mgr->priv;
+	u32 reg = socfpga_a10_fpga_read_stat(priv);
+
+	if (reg & A10_FPGAMGR_IMGCFG_STAT_F2S_USERMODE)
+		return FPGA_MGR_STATE_OPERATING;
+
+	if (reg & A10_FPGAMGR_IMGCFG_STAT_F2S_PR_READY)
+		return FPGA_MGR_STATE_WRITE;
+
+	if (reg & A10_FPGAMGR_IMGCFG_STAT_F2S_CRC_ERROR)
+		return FPGA_MGR_STATE_WRITE_COMPLETE_ERR;
+
+	if ((reg & A10_FPGAMGR_IMGCFG_STAT_F2S_NSTATUS_PIN) == 0)
+		return FPGA_MGR_STATE_RESET;
+
+	return FPGA_MGR_STATE_UNKNOWN;
+}
+
+static const struct fpga_manager_ops socfpga_a10_fpga_mgr_ops = {
+	.initial_header_size = (RBF_DECOMPRESS_OFFSET + 1) * 4,
+	.state = socfpga_a10_fpga_state,
+	.write_init = socfpga_a10_fpga_write_init,
+	.write = socfpga_a10_fpga_write,
+	.write_complete = socfpga_a10_fpga_write_complete,
+};
+
+static int socfpga_a10_fpga_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct a10_fpga_priv *priv;
+	void __iomem *reg_base;
+	struct resource *res;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	/* First mmio base is for register access */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	reg_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(reg_base))
+		return PTR_ERR(reg_base);
+
+	/* Second mmio base is for writing FPGA image data */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	priv->fpga_data_addr = devm_ioremap_resource(dev, res);
+	if (IS_ERR(priv->fpga_data_addr))
+		return PTR_ERR(priv->fpga_data_addr);
+
+	/* regmap for register access */
+	priv->regmap = devm_regmap_init_mmio(dev, reg_base,
+					     &socfpga_a10_fpga_regmap_config);
+	if (IS_ERR(priv->regmap))
+		return -ENODEV;
+
+	priv->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(priv->clk)) {
+		dev_err(dev, "no clock specified\n");
+		return PTR_ERR(priv->clk);
+	}
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret) {
+		dev_err(dev, "could not enable clock\n");
+		return -EBUSY;
+	}
+
+	return fpga_mgr_register(dev, "SoCFPGA Arria10 FPGA Manager",
+				 &socfpga_a10_fpga_mgr_ops, priv);
+}
+
+static int socfpga_a10_fpga_remove(struct platform_device *pdev)
+{
+	struct fpga_manager *mgr = platform_get_drvdata(pdev);
+	struct a10_fpga_priv *priv = mgr->priv;
+
+	fpga_mgr_unregister(&pdev->dev);
+	clk_disable_unprepare(priv->clk);
+
+	return 0;
+}
+
+static const struct of_device_id socfpga_a10_fpga_of_match[] = {
+	{ .compatible = "altr,socfpga-a10-fpga-mgr", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, socfpga_a10_fpga_of_match);
+
+static struct platform_driver socfpga_a10_fpga_driver = {
+	.probe = socfpga_a10_fpga_probe,
+	.remove = socfpga_a10_fpga_remove,
+	.driver = {
+		.name	= "socfpga_a10_fpga_manager",
+		.of_match_table = socfpga_a10_fpga_of_match,
+	},
+};
+
+module_platform_driver(socfpga_a10_fpga_driver);
+
+MODULE_AUTHOR("Alan Tull <atull@opensource.altera.com>");
+MODULE_DESCRIPTION("SoCFPGA Arria10 FPGA Manager");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/fpga/socfpga.c b/drivers/fpga/socfpga.c
index 27d2ff28132c..b6672e66cda6 100644
--- a/drivers/fpga/socfpga.c
+++ b/drivers/fpga/socfpga.c
@@ -407,13 +407,14 @@ static int socfpga_fpga_reset(struct fpga_manager *mgr)
 /*
  * Prepare the FPGA to receive the configuration data.
  */
-static int socfpga_fpga_ops_configure_init(struct fpga_manager *mgr, u32 flags,
+static int socfpga_fpga_ops_configure_init(struct fpga_manager *mgr,
+					   struct fpga_image_info *info,
 					   const char *buf, size_t count)
 {
 	struct socfpga_fpga_priv *priv = mgr->priv;
 	int ret;
 
-	if (flags & FPGA_MGR_PARTIAL_RECONFIG) {
+	if (info->flags & FPGA_MGR_PARTIAL_RECONFIG) {
 		dev_err(&mgr->dev, "Partial reconfiguration not supported.\n");
 		return -EINVAL;
 	}
@@ -478,7 +479,7 @@ static int socfpga_fpga_ops_configure_write(struct fpga_manager *mgr,
 }
 
 static int socfpga_fpga_ops_configure_complete(struct fpga_manager *mgr,
-					       u32 flags)
+					       struct fpga_image_info *info)
 {
 	struct socfpga_fpga_priv *priv = mgr->priv;
 	u32 status;
diff --git a/drivers/fpga/zynq-fpga.c b/drivers/fpga/zynq-fpga.c
index c2fb4120bd62..1812bf7614e1 100644
--- a/drivers/fpga/zynq-fpga.c
+++ b/drivers/fpga/zynq-fpga.c
@@ -118,7 +118,6 @@
 #define FPGA_RST_NONE_MASK		0x0
 
 struct zynq_fpga_priv {
-	struct device *dev;
 	int irq;
 	struct clk *clk;
 
@@ -175,7 +174,8 @@ static irqreturn_t zynq_fpga_isr(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static int zynq_fpga_ops_write_init(struct fpga_manager *mgr, u32 flags,
+static int zynq_fpga_ops_write_init(struct fpga_manager *mgr,
+				    struct fpga_image_info *info,
 				    const char *buf, size_t count)
 {
 	struct zynq_fpga_priv *priv;
@@ -189,7 +189,7 @@ static int zynq_fpga_ops_write_init(struct fpga_manager *mgr, u32 flags,
 		return err;
 
 	/* don't globally reset PL if we're doing partial reconfig */
-	if (!(flags & FPGA_MGR_PARTIAL_RECONFIG)) {
+	if (!(info->flags & FPGA_MGR_PARTIAL_RECONFIG)) {
 		/* assert AXI interface resets */
 		regmap_write(priv->slcr, SLCR_FPGA_RST_CTRL_OFFSET,
 			     FPGA_RST_ALL_MASK);
@@ -217,7 +217,7 @@ static int zynq_fpga_ops_write_init(struct fpga_manager *mgr, u32 flags,
 					     INIT_POLL_DELAY,
 					     INIT_POLL_TIMEOUT);
 		if (err) {
-			dev_err(priv->dev, "Timeout waiting for PCFG_INIT");
+			dev_err(&mgr->dev, "Timeout waiting for PCFG_INIT\n");
 			goto out_err;
 		}
 
@@ -231,7 +231,7 @@ static int zynq_fpga_ops_write_init(struct fpga_manager *mgr, u32 flags,
 					     INIT_POLL_DELAY,
 					     INIT_POLL_TIMEOUT);
 		if (err) {
-			dev_err(priv->dev, "Timeout waiting for !PCFG_INIT");
+			dev_err(&mgr->dev, "Timeout waiting for !PCFG_INIT\n");
 			goto out_err;
 		}
 
@@ -245,7 +245,7 @@ static int zynq_fpga_ops_write_init(struct fpga_manager *mgr, u32 flags,
 					     INIT_POLL_DELAY,
 					     INIT_POLL_TIMEOUT);
 		if (err) {
-			dev_err(priv->dev, "Timeout waiting for PCFG_INIT");
+			dev_err(&mgr->dev, "Timeout waiting for PCFG_INIT\n");
 			goto out_err;
 		}
 	}
@@ -262,7 +262,7 @@ static int zynq_fpga_ops_write_init(struct fpga_manager *mgr, u32 flags,
 	/* check that we have room in the command queue */
 	status = zynq_fpga_read(priv, STATUS_OFFSET);
 	if (status & STATUS_DMA_Q_F) {
-		dev_err(priv->dev, "DMA command queue full");
+		dev_err(&mgr->dev, "DMA command queue full\n");
 		err = -EBUSY;
 		goto out_err;
 	}
@@ -295,7 +295,8 @@ static int zynq_fpga_ops_write(struct fpga_manager *mgr,
 	in_count = count;
 	priv = mgr->priv;
 
-	kbuf = dma_alloc_coherent(priv->dev, count, &dma_addr, GFP_KERNEL);
+	kbuf =
+	    dma_alloc_coherent(mgr->dev.parent, count, &dma_addr, GFP_KERNEL);
 	if (!kbuf)
 		return -ENOMEM;
 
@@ -331,19 +332,19 @@ static int zynq_fpga_ops_write(struct fpga_manager *mgr,
 	zynq_fpga_write(priv, INT_STS_OFFSET, intr_status);
 
 	if (!((intr_status & IXR_D_P_DONE_MASK) == IXR_D_P_DONE_MASK)) {
-		dev_err(priv->dev, "Error configuring FPGA");
+		dev_err(&mgr->dev, "Error configuring FPGA\n");
 		err = -EFAULT;
 	}
 
 	clk_disable(priv->clk);
 
 out_free:
-	dma_free_coherent(priv->dev, in_count, kbuf, dma_addr);
-
+	dma_free_coherent(mgr->dev.parent, count, kbuf, dma_addr);
 	return err;
 }
 
-static int zynq_fpga_ops_write_complete(struct fpga_manager *mgr, u32 flags)
+static int zynq_fpga_ops_write_complete(struct fpga_manager *mgr,
+					struct fpga_image_info *info)
 {
 	struct zynq_fpga_priv *priv = mgr->priv;
 	int err;
@@ -364,7 +365,7 @@ static int zynq_fpga_ops_write_complete(struct fpga_manager *mgr, u32 flags)
 		return err;
 
 	/* for the partial reconfig case we didn't touch the level shifters */
-	if (!(flags & FPGA_MGR_PARTIAL_RECONFIG)) {
+	if (!(info->flags & FPGA_MGR_PARTIAL_RECONFIG)) {
 		/* enable level shifters from PL to PS */
 		regmap_write(priv->slcr, SLCR_LVL_SHFTR_EN_OFFSET,
 			     LVL_SHFTR_ENABLE_PL_TO_PS);
@@ -416,8 +417,6 @@ static int zynq_fpga_probe(struct platform_device *pdev)
 	if (!priv)
 		return -ENOMEM;
 
-	priv->dev = dev;
-
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	priv->io_base = devm_ioremap_resource(dev, res);
 	if (IS_ERR(priv->io_base))
@@ -426,7 +425,7 @@ static int zynq_fpga_probe(struct platform_device *pdev)
 	priv->slcr = syscon_regmap_lookup_by_phandle(dev->of_node,
 		"syscon");
 	if (IS_ERR(priv->slcr)) {
-		dev_err(dev, "unable to get zynq-slcr regmap");
+		dev_err(dev, "unable to get zynq-slcr regmap\n");
 		return PTR_ERR(priv->slcr);
 	}
 
@@ -434,38 +433,41 @@ static int zynq_fpga_probe(struct platform_device *pdev)
 
 	priv->irq = platform_get_irq(pdev, 0);
 	if (priv->irq < 0) {
-		dev_err(dev, "No IRQ available");
+		dev_err(dev, "No IRQ available\n");
 		return priv->irq;
 	}
 
-	err = devm_request_irq(dev, priv->irq, zynq_fpga_isr, 0,
-			       dev_name(dev), priv);
-	if (err) {
-		dev_err(dev, "unable to request IRQ");
-		return err;
-	}
-
 	priv->clk = devm_clk_get(dev, "ref_clk");
 	if (IS_ERR(priv->clk)) {
-		dev_err(dev, "input clock not found");
+		dev_err(dev, "input clock not found\n");
 		return PTR_ERR(priv->clk);
 	}
 
 	err = clk_prepare_enable(priv->clk);
 	if (err) {
-		dev_err(dev, "unable to enable clock");
+		dev_err(dev, "unable to enable clock\n");
 		return err;
 	}
 
 	/* unlock the device */
 	zynq_fpga_write(priv, UNLOCK_OFFSET, UNLOCK_MASK);
 
+	zynq_fpga_write(priv, INT_MASK_OFFSET, 0xFFFFFFFF);
+	zynq_fpga_write(priv, INT_STS_OFFSET, IXR_ALL_MASK);
+	err = devm_request_irq(dev, priv->irq, zynq_fpga_isr, 0, dev_name(dev),
+			       priv);
+	if (err) {
+		dev_err(dev, "unable to request IRQ\n");
+		clk_disable_unprepare(priv->clk);
+		return err;
+	}
+
 	clk_disable(priv->clk);
 
 	err = fpga_mgr_register(dev, "Xilinx Zynq FPGA Manager",
 				&zynq_fpga_ops, priv);
 	if (err) {
-		dev_err(dev, "unable to register FPGA manager");
+		dev_err(dev, "unable to register FPGA manager\n");
 		clk_unprepare(priv->clk);
 		return err;
 	}
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index ed37e5908b91..d5d36549ecc1 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -167,7 +167,7 @@ config GPIO_DWAPB
 
 config GPIO_EM
 	tristate "Emma Mobile GPIO"
-	depends on ARM && OF_GPIO
+	depends on (ARCH_EMEV2 || COMPILE_TEST) && OF_GPIO
 	help
 	  Say yes here to support GPIO on Renesas Emma Mobile SoCs.
 
@@ -451,7 +451,7 @@ config GPIO_VR41XX
 
 config GPIO_VX855
 	tristate "VIA VX855/VX875 GPIO"
-	depends on PCI
+	depends on (X86 || COMPILE_TEST) && PCI
 	select MFD_CORE
 	select MFD_VX855
 	help
@@ -520,6 +520,7 @@ config GPIO_ZYNQ
 
 config GPIO_ZX
 	bool "ZTE ZX GPIO support"
+	depends on ARCH_ZX || COMPILE_TEST
 	select GPIOLIB_IRQCHIP
 	help
 	  Say yes here to support the GPIO device on ZTE ZX SoCs.
@@ -603,7 +604,7 @@ config GPIO_IT87
 
 config GPIO_SCH
 	tristate "Intel SCH/TunnelCreek/Centerton/Quark X1000 GPIO"
-	depends on PCI
+	depends on (X86 || COMPILE_TEST) && PCI
 	select MFD_CORE
 	select LPC_SCH
 	help
@@ -777,16 +778,13 @@ config GPIO_PCF857X
 	  platform-neutral GPIO calls.
 
 config GPIO_SX150X
-	bool "Semtech SX150x I2C GPIO expander"
-	depends on I2C=y
-	select GPIOLIB_IRQCHIP
+	bool "Semtech SX150x I2C GPIO expander (deprecated)"
+	depends on PINCTRL && I2C=y
+	select PINCTRL_SX150X
 	default n
 	help
-	  Say yes here to provide support for Semtech SX150-series I2C
-	  GPIO expanders. Compatible models include:
-
-	  8 bits:  sx1508q
-	  16 bits: sx1509q
+	  Say yes here to provide support for Semtech SX150x-series I2C
+	  GPIO expanders. The GPIO driver was replaced by a Pinctrl version.
 
 config GPIO_TPIC2810
 	tristate "TPIC2810 8-Bit I2C GPO expander"
@@ -798,6 +796,7 @@ config GPIO_TPIC2810
 
 config GPIO_TS4900
 	tristate "Technologic Systems FPGA I2C GPIO"
+	depends on SOC_IMX6 || COMPILE_TEST
 	select REGMAP_I2C
 	help
 	  Say yes here to enabled the GPIO driver for Technologic's FPGA core.
@@ -814,6 +813,14 @@ config GPIO_ADP5520
 	  This option enables support for on-chip GPIO found
 	  on Analog Devices ADP5520 PMICs.
 
+config GPIO_ALTERA_A10SR
+	tristate "Altera Arria10 System Resource GPIO"
+	depends on MFD_ALTERA_A10SR
+	help
+	  Driver for Arria10 Development Kit GPIO expansion which
+	  includes reads of pushbuttons and DIP switches as well
+	  as writes to LEDs.
+
 config GPIO_ARIZONA
 	tristate "Wolfson Microelectronics Arizona class devices"
 	depends on MFD_ARIZONA
@@ -822,7 +829,7 @@ config GPIO_ARIZONA
 
 config GPIO_CRYSTAL_COVE
 	tristate "GPIO support for Crystal Cove PMIC"
-	depends on INTEL_SOC_PMIC
+	depends on (X86 || COMPILE_TEST) && INTEL_SOC_PMIC
 	select GPIOLIB_IRQCHIP
 	help
 	  Support for GPIO pins on Crystal Cove PMIC.
@@ -835,6 +842,7 @@ config GPIO_CRYSTAL_COVE
 
 config GPIO_CS5535
 	tristate "AMD CS5535/CS5536 GPIO support"
+	depends on X86 || MIPS || COMPILE_TEST
 	depends on MFD_CS5535
 	help
 	  The AMD CS5535 and CS5536 southbridges support 28 GPIO pins that
@@ -927,7 +935,7 @@ config GPIO_MAX77620
 
 config GPIO_MSIC
 	bool "Intel MSIC mixed signal gpio support"
-	depends on MFD_INTEL_MSIC
+	depends on (X86 || COMPILE_TEST) && MFD_INTEL_MSIC
 	help
 	  Enable support for GPIO on intel MSIC controllers found in
 	  intel MID devices
@@ -1028,7 +1036,7 @@ config GPIO_UCB1400
 
 config GPIO_WHISKEY_COVE
 	tristate "GPIO support for Whiskey Cove PMIC"
-	depends on INTEL_SOC_PMIC
+	depends on (X86 || COMPILE_TEST) && INTEL_SOC_PMIC
 	select GPIOLIB_IRQCHIP
 	help
 	  Support for GPIO pins on Whiskey Cove PMIC.
@@ -1067,6 +1075,7 @@ menu "PCI GPIO expanders"
 
 config GPIO_AMD8111
 	tristate "AMD 8111 GPIO driver"
+	depends on X86 || COMPILE_TEST
 	help
 	  The AMD 8111 south bridge contains 32 GPIO pins which can be used.
 
@@ -1108,6 +1117,7 @@ config GPIO_MERRIFIELD
 
 config GPIO_ML_IOH
 	tristate "OKI SEMICONDUCTOR ML7213 IOH GPIO support"
+	depends on X86 || COMPILE_TEST
 	select GENERIC_IRQ_CHIP
 	help
 	  ML7213 is companion chip for Intel Atom E6xx series.
diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile
index d074c2299393..a7676b82de6f 100644
--- a/drivers/gpio/Makefile
+++ b/drivers/gpio/Makefile
@@ -5,6 +5,7 @@ ccflags-$(CONFIG_DEBUG_GPIO)	+= -DDEBUG
 obj-$(CONFIG_GPIOLIB)		+= devres.o
 obj-$(CONFIG_GPIOLIB)		+= gpiolib.o
 obj-$(CONFIG_GPIOLIB)		+= gpiolib-legacy.o
+obj-$(CONFIG_GPIOLIB)		+= gpiolib-devprop.o
 obj-$(CONFIG_OF_GPIO)		+= gpiolib-of.o
 obj-$(CONFIG_GPIO_SYSFS)	+= gpiolib-sysfs.o
 obj-$(CONFIG_GPIO_ACPI)		+= gpiolib-acpi.o
@@ -24,6 +25,7 @@ obj-$(CONFIG_GPIO_ADNP)		+= gpio-adnp.o
 obj-$(CONFIG_GPIO_ADP5520)	+= gpio-adp5520.o
 obj-$(CONFIG_GPIO_ADP5588)	+= gpio-adp5588.o
 obj-$(CONFIG_GPIO_ALTERA)  	+= gpio-altera.o
+obj-$(CONFIG_GPIO_ALTERA_A10SR)	+= gpio-altera-a10sr.o
 obj-$(CONFIG_GPIO_AMD8111)	+= gpio-amd8111.o
 obj-$(CONFIG_GPIO_AMDPT)	+= gpio-amdpt.o
 obj-$(CONFIG_GPIO_ARIZONA)	+= gpio-arizona.o
@@ -102,7 +104,6 @@ obj-$(CONFIG_GPIO_SPEAR_SPICS)	+= gpio-spear-spics.o
 obj-$(CONFIG_GPIO_STA2X11)	+= gpio-sta2x11.o
 obj-$(CONFIG_GPIO_STMPE)	+= gpio-stmpe.o
 obj-$(CONFIG_GPIO_STP_XWAY)	+= gpio-stp-xway.o
-obj-$(CONFIG_GPIO_SX150X)	+= gpio-sx150x.o
 obj-$(CONFIG_GPIO_SYSCON)	+= gpio-syscon.o
 obj-$(CONFIG_GPIO_TB10X)	+= gpio-tb10x.o
 obj-$(CONFIG_GPIO_TC3589X)	+= gpio-tc3589x.o
diff --git a/drivers/gpio/gpio-adnp.c b/drivers/gpio/gpio-adnp.c
index 8ff7b0d3eac6..89863ea25de1 100644
--- a/drivers/gpio/gpio-adnp.c
+++ b/drivers/gpio/gpio-adnp.c
@@ -468,17 +468,19 @@ static int adnp_irq_setup(struct adnp *adnp)
 		return err;
 	}
 
-	err = gpiochip_irqchip_add(chip,
-				   &adnp_irq_chip,
-				   0,
-				   handle_simple_irq,
-				   IRQ_TYPE_NONE);
+	err = gpiochip_irqchip_add_nested(chip,
+					  &adnp_irq_chip,
+					  0,
+					  handle_simple_irq,
+					  IRQ_TYPE_NONE);
 	if (err) {
 		dev_err(chip->parent,
 			"could not connect irqchip to gpiochip\n");
 		return err;
 	}
 
+	gpiochip_set_nested_irqchip(chip, &adnp_irq_chip, adnp->client->irq);
+
 	return 0;
 }
 
diff --git a/drivers/gpio/gpio-altera-a10sr.c b/drivers/gpio/gpio-altera-a10sr.c
new file mode 100644
index 000000000000..9e1a138fed53
--- /dev/null
+++ b/drivers/gpio/gpio-altera-a10sr.c
@@ -0,0 +1,130 @@
+/*
+ *  Copyright Intel Corporation (C) 2014-2016. All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * GPIO driver for  Altera Arria10 MAX5 System Resource Chip
+ *
+ * Adapted from gpio-tps65910.c
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/mfd/altera-a10sr.h>
+#include <linux/module.h>
+
+/**
+ * struct altr_a10sr_gpio - Altera Max5 GPIO device private data structure
+ * @gp:   : instance of the gpio_chip
+ * @regmap: the regmap from the parent device.
+ */
+struct altr_a10sr_gpio {
+	struct gpio_chip gp;
+	struct regmap *regmap;
+};
+
+static int altr_a10sr_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+	struct altr_a10sr_gpio *gpio = gpiochip_get_data(chip);
+	int ret, val;
+
+	ret = regmap_read(gpio->regmap, ALTR_A10SR_PBDSW_REG, &val);
+	if (ret < 0)
+		return ret;
+
+	return !!(val & BIT(offset - ALTR_A10SR_LED_VALID_SHIFT));
+}
+
+static void altr_a10sr_gpio_set(struct gpio_chip *chip, unsigned int offset,
+				int value)
+{
+	struct altr_a10sr_gpio *gpio = gpiochip_get_data(chip);
+
+	regmap_update_bits(gpio->regmap, ALTR_A10SR_LED_REG,
+			   BIT(ALTR_A10SR_LED_VALID_SHIFT + offset),
+			   value ? BIT(ALTR_A10SR_LED_VALID_SHIFT + offset)
+			   : 0);
+}
+
+static int altr_a10sr_gpio_direction_input(struct gpio_chip *gc,
+					   unsigned int nr)
+{
+	if (nr >= (ALTR_A10SR_IN_VALID_RANGE_LO - ALTR_A10SR_LED_VALID_SHIFT))
+		return 0;
+	return -EINVAL;
+}
+
+static int altr_a10sr_gpio_direction_output(struct gpio_chip *gc,
+					    unsigned int nr, int value)
+{
+	if (nr <= (ALTR_A10SR_OUT_VALID_RANGE_HI - ALTR_A10SR_LED_VALID_SHIFT))
+		return 0;
+	return -EINVAL;
+}
+
+static struct gpio_chip altr_a10sr_gc = {
+	.label = "altr_a10sr_gpio",
+	.owner = THIS_MODULE,
+	.get = altr_a10sr_gpio_get,
+	.set = altr_a10sr_gpio_set,
+	.direction_input = altr_a10sr_gpio_direction_input,
+	.direction_output = altr_a10sr_gpio_direction_output,
+	.can_sleep = true,
+	.ngpio = 12,
+	.base = -1,
+};
+
+static int altr_a10sr_gpio_probe(struct platform_device *pdev)
+{
+	struct altr_a10sr_gpio *gpio;
+	int ret;
+	struct altr_a10sr *a10sr = dev_get_drvdata(pdev->dev.parent);
+
+	gpio = devm_kzalloc(&pdev->dev, sizeof(*gpio), GFP_KERNEL);
+	if (!gpio)
+		return -ENOMEM;
+
+	gpio->regmap = a10sr->regmap;
+
+	gpio->gp = altr_a10sr_gc;
+
+	gpio->gp.of_node = pdev->dev.of_node;
+
+	ret = devm_gpiochip_add_data(&pdev->dev, &gpio->gp, gpio);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Could not register gpiochip, %d\n", ret);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, gpio);
+
+	return 0;
+}
+
+static const struct of_device_id altr_a10sr_gpio_of_match[] = {
+	{ .compatible = "altr,a10sr-gpio" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, altr_a10sr_gpio_of_match);
+
+static struct platform_driver altr_a10sr_gpio_driver = {
+	.probe = altr_a10sr_gpio_probe,
+	.driver = {
+		.name	= "altr_a10sr_gpio",
+		.of_match_table = of_match_ptr(altr_a10sr_gpio_of_match),
+	},
+};
+module_platform_driver(altr_a10sr_gpio_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Thor Thayer <tthayer@opensource.altera.com>");
+MODULE_DESCRIPTION("Altera Arria10 System Resource Chip GPIO");
diff --git a/drivers/gpio/gpio-arizona.c b/drivers/gpio/gpio-arizona.c
index 482462889c8f..1f91557717a6 100644
--- a/drivers/gpio/gpio-arizona.c
+++ b/drivers/gpio/gpio-arizona.c
@@ -137,15 +137,10 @@ static int arizona_gpio_probe(struct platform_device *pdev)
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Could not register gpiochip, %d\n",
 			ret);
-		goto err;
+		return ret;
 	}
 
-	platform_set_drvdata(pdev, arizona_gpio);
-
-	return ret;
-
-err:
-	return ret;
+	return 0;
 }
 
 static struct platform_driver arizona_gpio_driver = {
diff --git a/drivers/gpio/gpio-axp209.c b/drivers/gpio/gpio-axp209.c
index d9c2a517c6df..4a346b7b4172 100644
--- a/drivers/gpio/gpio-axp209.c
+++ b/drivers/gpio/gpio-axp209.c
@@ -64,13 +64,9 @@ static int axp20x_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
 	struct axp20x_gpio *gpio = gpiochip_get_data(chip);
 	unsigned int val;
-	int reg, ret;
-
-	reg = axp20x_gpio_get_reg(offset);
-	if (reg < 0)
-		return reg;
+	int ret;
 
-	ret = regmap_read(gpio->regmap, reg, &val);
+	ret = regmap_read(gpio->regmap, AXP20X_GPIO20_SS, &val);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpio/gpio-crystalcove.c b/drivers/gpio/gpio-crystalcove.c
index 7c446d118cd6..2197368cc899 100644
--- a/drivers/gpio/gpio-crystalcove.c
+++ b/drivers/gpio/gpio-crystalcove.c
@@ -351,8 +351,8 @@ static int crystalcove_gpio_probe(struct platform_device *pdev)
 		return retval;
 	}
 
-	gpiochip_irqchip_add(&cg->chip, &crystalcove_irqchip, 0,
-			     handle_simple_irq, IRQ_TYPE_NONE);
+	gpiochip_irqchip_add_nested(&cg->chip, &crystalcove_irqchip, 0,
+				    handle_simple_irq, IRQ_TYPE_NONE);
 
 	retval = request_threaded_irq(irq, NULL, crystalcove_gpio_irq_handler,
 				      IRQF_ONESHOT, KBUILD_MODNAME, cg);
@@ -362,6 +362,8 @@ static int crystalcove_gpio_probe(struct platform_device *pdev)
 		return retval;
 	}
 
+	gpiochip_set_nested_irqchip(&cg->chip, &crystalcove_irqchip, irq);
+
 	return 0;
 }
 
diff --git a/drivers/gpio/gpio-davinci.c b/drivers/gpio/gpio-davinci.c
index dd262f00295d..9191056548fe 100644
--- a/drivers/gpio/gpio-davinci.c
+++ b/drivers/gpio/gpio-davinci.c
@@ -40,6 +40,7 @@ struct davinci_gpio_regs {
 typedef struct irq_chip *(*gpio_get_irq_chip_cb_t)(unsigned int irq);
 
 #define BINTEN	0x8 /* GPIO Interrupt Per-Bank Enable Register */
+#define MAX_LABEL_SIZE 20
 
 static void __iomem *gpio_base;
 
@@ -201,6 +202,7 @@ static int davinci_gpio_probe(struct platform_device *pdev)
 	struct davinci_gpio_regs __iomem *regs;
 	struct device *dev = &pdev->dev;
 	struct resource *res;
+	char label[MAX_LABEL_SIZE];
 
 	pdata = davinci_gpio_get_pdata(pdev);
 	if (!pdata) {
@@ -237,7 +239,10 @@ static int davinci_gpio_probe(struct platform_device *pdev)
 		return PTR_ERR(gpio_base);
 
 	for (i = 0, base = 0; base < ngpio; i++, base += 32) {
-		chips[i].chip.label = "DaVinci";
+		snprintf(label, MAX_LABEL_SIZE, "davinci_gpio.%d", i);
+		chips[i].chip.label = devm_kstrdup(dev, label, GFP_KERNEL);
+		if (!chips[i].chip.label)
+			return -ENOMEM;
 
 		chips[i].chip.direction_input = davinci_direction_in;
 		chips[i].chip.get = davinci_gpio_get;
diff --git a/drivers/gpio/gpio-dln2.c b/drivers/gpio/gpio-dln2.c
index f7a60a441e95..5d38b08d1ee2 100644
--- a/drivers/gpio/gpio-dln2.c
+++ b/drivers/gpio/gpio-dln2.c
@@ -467,7 +467,6 @@ static int dln2_gpio_probe(struct platform_device *pdev)
 	dln2->gpio.base = -1;
 	dln2->gpio.ngpio = pins;
 	dln2->gpio.can_sleep = true;
-	dln2->gpio.irq_not_threaded = true;
 	dln2->gpio.set = dln2_gpio_set;
 	dln2->gpio.get = dln2_gpio_get;
 	dln2->gpio.request = dln2_gpio_request;
diff --git a/drivers/gpio/gpio-etraxfs.c b/drivers/gpio/gpio-etraxfs.c
index 00b022c9acb3..a254d5b07b94 100644
--- a/drivers/gpio/gpio-etraxfs.c
+++ b/drivers/gpio/gpio-etraxfs.c
@@ -471,9 +471,4 @@ static struct platform_driver etraxfs_gpio_driver = {
 	.probe	= etraxfs_gpio_probe,
 };
 
-static int __init etraxfs_gpio_init(void)
-{
-	return platform_driver_register(&etraxfs_gpio_driver);
-}
-
-device_initcall(etraxfs_gpio_init);
+builtin_platform_driver(etraxfs_gpio_driver);
diff --git a/drivers/gpio/gpio-htc-egpio.c b/drivers/gpio/gpio-htc-egpio.c
index 0b4df6051097..271356effb2e 100644
--- a/drivers/gpio/gpio-htc-egpio.c
+++ b/drivers/gpio/gpio-htc-egpio.c
@@ -17,7 +17,7 @@
 #include <linux/platform_data/gpio-htc-egpio.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
-#include <linux/module.h>
+#include <linux/init.h>
 
 struct egpio_chip {
 	int              reg_start;
@@ -160,10 +160,14 @@ static int egpio_get(struct gpio_chip *chip, unsigned offset)
 	bit   = egpio_bit(ei, offset);
 	reg   = egpio->reg_start + egpio_pos(ei, offset);
 
-	value = egpio_readw(ei, reg);
-	pr_debug("readw(%p + %x) = %x\n",
-			ei->base_addr, reg << ei->bus_shift, value);
-	return !!(value & bit);
+	if (test_bit(offset, &egpio->is_out)) {
+		return !!(egpio->cached_values & (1 << offset));
+	} else {
+		value = egpio_readw(ei, reg);
+		pr_debug("readw(%p + %x) = %x\n",
+			 ei->base_addr, reg << ei->bus_shift, value);
+		return !!(value & bit);
+	}
 }
 
 static int egpio_direction_input(struct gpio_chip *chip, unsigned offset)
@@ -225,6 +229,15 @@ static int egpio_direction_output(struct gpio_chip *chip,
 	}
 }
 
+static int egpio_get_direction(struct gpio_chip *chip, unsigned offset)
+{
+	struct egpio_chip *egpio;
+
+	egpio = gpiochip_get_data(chip);
+
+	return !test_bit(offset, &egpio->is_out);
+}
+
 static void egpio_write_cache(struct egpio_info *ei)
 {
 	int               i;
@@ -327,6 +340,7 @@ static int __init egpio_probe(struct platform_device *pdev)
 		chip->set             = egpio_set;
 		chip->direction_input = egpio_direction_input;
 		chip->direction_output = egpio_direction_output;
+		chip->get_direction   = egpio_get_direction;
 		chip->base            = pdata->chip[i].gpio_base;
 		chip->ngpio           = pdata->chip[i].num_gpios;
 
@@ -367,24 +381,6 @@ fail:
 	return ret;
 }
 
-static int __exit egpio_remove(struct platform_device *pdev)
-{
-	struct egpio_info *ei = platform_get_drvdata(pdev);
-	unsigned int      irq, irq_end;
-
-	if (ei->chained_irq) {
-		irq_end = ei->irq_start + ei->nirqs;
-		for (irq = ei->irq_start; irq < irq_end; irq++) {
-			irq_set_chip_and_handler(irq, NULL, NULL);
-			irq_set_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE);
-		}
-		irq_set_chained_handler(ei->chained_irq, NULL);
-		device_init_wakeup(&pdev->dev, 0);
-	}
-
-	return 0;
-}
-
 #ifdef CONFIG_PM
 static int egpio_suspend(struct platform_device *pdev, pm_message_t state)
 {
@@ -416,8 +412,8 @@ static int egpio_resume(struct platform_device *pdev)
 static struct platform_driver egpio_driver = {
 	.driver = {
 		.name = "htc-egpio",
+		.suppress_bind_attrs = true,
 	},
-	.remove       = __exit_p(egpio_remove),
 	.suspend      = egpio_suspend,
 	.resume       = egpio_resume,
 };
@@ -426,15 +422,5 @@ static int __init egpio_init(void)
 {
 	return platform_driver_probe(&egpio_driver, egpio_probe);
 }
-
-static void __exit egpio_exit(void)
-{
-	platform_driver_unregister(&egpio_driver);
-}
-
 /* start early for dependencies */
 subsys_initcall(egpio_init);
-module_exit(egpio_exit)
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Kevin O'Connor <kevin@koconnor.net>");
diff --git a/drivers/gpio/gpio-intel-mid.c b/drivers/gpio/gpio-intel-mid.c
index 164de64b11fc..a1e44c221f66 100644
--- a/drivers/gpio/gpio-intel-mid.c
+++ b/drivers/gpio/gpio-intel-mid.c
@@ -421,9 +421,4 @@ static struct pci_driver intel_gpio_driver = {
 	},
 };
 
-static int __init intel_gpio_init(void)
-{
-	return pci_register_driver(&intel_gpio_driver);
-}
-
-device_initcall(intel_gpio_init);
+builtin_pci_driver(intel_gpio_driver);
diff --git a/drivers/gpio/gpio-max732x.c b/drivers/gpio/gpio-max732x.c
index a9aaf9d822b4..4ea4c6a1313b 100644
--- a/drivers/gpio/gpio-max732x.c
+++ b/drivers/gpio/gpio-max732x.c
@@ -520,20 +520,19 @@ static int max732x_irq_setup(struct max732x_chip *chip,
 				client->irq);
 			return ret;
 		}
-		ret =  gpiochip_irqchip_add(&chip->gpio_chip,
-					    &max732x_irq_chip,
-					    irq_base,
-					    handle_simple_irq,
-					    IRQ_TYPE_NONE);
+		ret =  gpiochip_irqchip_add_nested(&chip->gpio_chip,
+						   &max732x_irq_chip,
+						   irq_base,
+						   handle_simple_irq,
+						   IRQ_TYPE_NONE);
 		if (ret) {
 			dev_err(&client->dev,
 				"could not connect irqchip to gpiochip\n");
 			return ret;
 		}
-		gpiochip_set_chained_irqchip(&chip->gpio_chip,
-					     &max732x_irq_chip,
-					     client->irq,
-					     NULL);
+		gpiochip_set_nested_irqchip(&chip->gpio_chip,
+					    &max732x_irq_chip,
+					    client->irq);
 	}
 
 	return 0;
diff --git a/drivers/gpio/gpio-max77620.c b/drivers/gpio/gpio-max77620.c
index b46b436cb97f..ec8de4190db9 100644
--- a/drivers/gpio/gpio-max77620.c
+++ b/drivers/gpio/gpio-max77620.c
@@ -21,9 +21,6 @@ struct max77620_gpio {
 	struct gpio_chip	gpio_chip;
 	struct regmap		*rmap;
 	struct device		*dev;
-	int			gpio_irq;
-	int			irq_base;
-	int			gpio_base;
 };
 
 static const struct regmap_irq max77620_gpio_irqs[] = {
@@ -254,7 +251,6 @@ static int max77620_gpio_probe(struct platform_device *pdev)
 
 	mgpio->rmap = chip->rmap;
 	mgpio->dev = &pdev->dev;
-	mgpio->gpio_irq = gpio_irq;
 
 	mgpio->gpio_chip.label = pdev->name;
 	mgpio->gpio_chip.parent = &pdev->dev;
@@ -268,7 +264,6 @@ static int max77620_gpio_probe(struct platform_device *pdev)
 	mgpio->gpio_chip.ngpio = MAX77620_GPIO_NR;
 	mgpio->gpio_chip.can_sleep = 1;
 	mgpio->gpio_chip.base = -1;
-	mgpio->irq_base = -1;
 #ifdef CONFIG_OF_GPIO
 	mgpio->gpio_chip.of_node = pdev->dev.parent->of_node;
 #endif
@@ -281,9 +276,8 @@ static int max77620_gpio_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	mgpio->gpio_base = mgpio->gpio_chip.base;
-	ret = devm_regmap_add_irq_chip(&pdev->dev, chip->rmap, mgpio->gpio_irq,
-				       IRQF_ONESHOT, mgpio->irq_base,
+	ret = devm_regmap_add_irq_chip(&pdev->dev, chip->rmap, gpio_irq,
+				       IRQF_ONESHOT, -1,
 				       &max77620_gpio_irq_chip,
 				       &chip->gpio_irq_data);
 	if (ret < 0) {
@@ -296,6 +290,7 @@ static int max77620_gpio_probe(struct platform_device *pdev)
 
 static const struct platform_device_id max77620_gpio_devtype[] = {
 	{ .name = "max77620-gpio", },
+	{ .name = "max20024-gpio", },
 	{},
 };
 MODULE_DEVICE_TABLE(platform, max77620_gpio_devtype);
diff --git a/drivers/gpio/gpio-mb86s7x.c b/drivers/gpio/gpio-mb86s7x.c
index d55af50e7034..ffb73f688ae1 100644
--- a/drivers/gpio/gpio-mb86s7x.c
+++ b/drivers/gpio/gpio-mb86s7x.c
@@ -217,8 +217,4 @@ static struct platform_driver mb86s70_gpio_driver = {
 	.remove = mb86s70_gpio_remove,
 };
 
-static int __init mb86s70_gpio_init(void)
-{
-	return platform_driver_register(&mb86s70_gpio_driver);
-}
-device_initcall(mb86s70_gpio_init);
+builtin_platform_driver(mb86s70_gpio_driver);
diff --git a/drivers/gpio/gpio-mcp23s08.c b/drivers/gpio/gpio-mcp23s08.c
index 99d37b56c258..504550665091 100644
--- a/drivers/gpio/gpio-mcp23s08.c
+++ b/drivers/gpio/gpio-mcp23s08.c
@@ -473,21 +473,20 @@ static int mcp23s08_irq_setup(struct mcp23s08 *mcp)
 		return err;
 	}
 
-	err =  gpiochip_irqchip_add(chip,
-				    &mcp23s08_irq_chip,
-				    0,
-				    handle_simple_irq,
-				    IRQ_TYPE_NONE);
+	err =  gpiochip_irqchip_add_nested(chip,
+					   &mcp23s08_irq_chip,
+					   0,
+					   handle_simple_irq,
+					   IRQ_TYPE_NONE);
 	if (err) {
 		dev_err(chip->parent,
 			"could not connect irqchip to gpiochip: %d\n", err);
 		return err;
 	}
 
-	gpiochip_set_chained_irqchip(chip,
-				     &mcp23s08_irq_chip,
-				     mcp->irq,
-				     NULL);
+	gpiochip_set_nested_irqchip(chip,
+				    &mcp23s08_irq_chip,
+				    mcp->irq);
 
 	return 0;
 }
diff --git a/drivers/gpio/gpio-merrifield.c b/drivers/gpio/gpio-merrifield.c
index 45b51278b8ee..69e0f4ace465 100644
--- a/drivers/gpio/gpio-merrifield.c
+++ b/drivers/gpio/gpio-merrifield.c
@@ -11,6 +11,7 @@
 
 #include <linux/bitops.h>
 #include <linux/gpio/driver.h>
+#include <linux/gpio.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
@@ -161,6 +162,34 @@ static int mrfld_gpio_direction_output(struct gpio_chip *chip,
 	return 0;
 }
 
+static int mrfld_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
+{
+	void __iomem *gpdr = gpio_reg(chip, offset, GPDR);
+
+	return (readl(gpdr) & BIT(offset % 32)) ? GPIOF_DIR_OUT : GPIOF_DIR_IN;
+}
+
+static int mrfld_gpio_set_debounce(struct gpio_chip *chip, unsigned int offset,
+				   unsigned int debounce)
+{
+	struct mrfld_gpio *priv = gpiochip_get_data(chip);
+	void __iomem *gfbr = gpio_reg(chip, offset, GFBR);
+	unsigned long flags;
+	u32 value;
+
+	raw_spin_lock_irqsave(&priv->lock, flags);
+
+	if (debounce)
+		value = readl(gfbr) & ~BIT(offset % 32);
+	else
+		value = readl(gfbr) | BIT(offset % 32);
+	writel(value, gfbr);
+
+	raw_spin_unlock_irqrestore(&priv->lock, flags);
+
+	return 0;
+}
+
 static void mrfld_irq_ack(struct irq_data *d)
 {
 	struct mrfld_gpio *priv = irq_data_get_irq_chip_data(d);
@@ -384,6 +413,8 @@ static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id
 	priv->chip.direction_output = mrfld_gpio_direction_output;
 	priv->chip.get = mrfld_gpio_get;
 	priv->chip.set = mrfld_gpio_set;
+	priv->chip.get_direction = mrfld_gpio_get_direction;
+	priv->chip.set_debounce = mrfld_gpio_set_debounce;
 	priv->chip.base = gpio_base;
 	priv->chip.ngpio = MRFLD_NGPIO;
 	priv->chip.can_sleep = false;
@@ -411,7 +442,7 @@ static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id
 	}
 
 	retval = gpiochip_irqchip_add(&priv->chip, &mrfld_irqchip, irq_base,
-				      handle_simple_irq, IRQ_TYPE_NONE);
+				      handle_bad_irq, IRQ_TYPE_NONE);
 	if (retval) {
 		dev_err(&pdev->dev, "could not connect irqchip to gpiochip\n");
 		return retval;
diff --git a/drivers/gpio/gpio-mxs.c b/drivers/gpio/gpio-mxs.c
index ee1724806f46..1e8fde8cb803 100644
--- a/drivers/gpio/gpio-mxs.c
+++ b/drivers/gpio/gpio-mxs.c
@@ -87,10 +87,15 @@ static int mxs_gpio_set_irq_type(struct irq_data *d, unsigned int type)
 	u32 val;
 	u32 pin_mask = 1 << d->hwirq;
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
+	struct irq_chip_type *ct = irq_data_get_chip_type(d);
 	struct mxs_gpio_port *port = gc->private;
 	void __iomem *pin_addr;
 	int edge;
 
+	if (!(ct->type & type))
+		if (irq_setup_alt_chip(d, type))
+			return -EINVAL;
+
 	port->both_edges &= ~pin_mask;
 	switch (type) {
 	case IRQ_TYPE_EDGE_BOTH:
@@ -119,10 +124,13 @@ static int mxs_gpio_set_irq_type(struct irq_data *d, unsigned int type)
 
 	/* set level or edge */
 	pin_addr = port->base + PINCTRL_IRQLEV(port);
-	if (edge & GPIO_INT_LEV_MASK)
+	if (edge & GPIO_INT_LEV_MASK) {
 		writel(pin_mask, pin_addr + MXS_SET);
-	else
+		writel(pin_mask, port->base + PINCTRL_IRQEN(port) + MXS_SET);
+	} else {
 		writel(pin_mask, pin_addr + MXS_CLR);
+		writel(pin_mask, port->base + PINCTRL_PIN2IRQ(port) + MXS_SET);
+	}
 
 	/* set polarity */
 	pin_addr = port->base + PINCTRL_IRQPOL(port);
@@ -202,21 +210,37 @@ static int __init mxs_gpio_init_gc(struct mxs_gpio_port *port, int irq_base)
 	struct irq_chip_generic *gc;
 	struct irq_chip_type *ct;
 
-	gc = irq_alloc_generic_chip("gpio-mxs", 1, irq_base,
+	gc = irq_alloc_generic_chip("gpio-mxs", 2, irq_base,
 				    port->base, handle_level_irq);
 	if (!gc)
 		return -ENOMEM;
 
 	gc->private = port;
 
-	ct = gc->chip_types;
+	ct = &gc->chip_types[0];
+	ct->type = IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW;
+	ct->chip.irq_ack = irq_gc_ack_set_bit;
+	ct->chip.irq_mask = irq_gc_mask_disable_reg;
+	ct->chip.irq_unmask = irq_gc_unmask_enable_reg;
+	ct->chip.irq_set_type = mxs_gpio_set_irq_type;
+	ct->chip.irq_set_wake = mxs_gpio_set_wake_irq;
+	ct->chip.flags = IRQCHIP_SET_TYPE_MASKED;
+	ct->regs.ack = PINCTRL_IRQSTAT(port) + MXS_CLR;
+	ct->regs.enable = PINCTRL_PIN2IRQ(port) + MXS_SET;
+	ct->regs.disable = PINCTRL_PIN2IRQ(port) + MXS_CLR;
+
+	ct = &gc->chip_types[1];
+	ct->type = IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING;
 	ct->chip.irq_ack = irq_gc_ack_set_bit;
-	ct->chip.irq_mask = irq_gc_mask_clr_bit;
-	ct->chip.irq_unmask = irq_gc_mask_set_bit;
+	ct->chip.irq_mask = irq_gc_mask_disable_reg;
+	ct->chip.irq_unmask = irq_gc_unmask_enable_reg;
 	ct->chip.irq_set_type = mxs_gpio_set_irq_type;
 	ct->chip.irq_set_wake = mxs_gpio_set_wake_irq;
+	ct->chip.flags = IRQCHIP_SET_TYPE_MASKED;
 	ct->regs.ack = PINCTRL_IRQSTAT(port) + MXS_CLR;
-	ct->regs.mask = PINCTRL_IRQEN(port);
+	ct->regs.enable = PINCTRL_IRQEN(port) + MXS_SET;
+	ct->regs.disable = PINCTRL_IRQEN(port) + MXS_CLR;
+	ct->handler = handle_level_irq;
 
 	irq_setup_generic_chip(gc, IRQ_MSK(32), IRQ_GC_INIT_NESTED_LOCK,
 			       IRQ_NOREQUEST, 0);
@@ -297,11 +321,8 @@ static int mxs_gpio_probe(struct platform_device *pdev)
 	}
 	port->base = base;
 
-	/*
-	 * select the pin interrupt functionality but initially
-	 * disable the interrupts
-	 */
-	writel(~0U, port->base + PINCTRL_PIN2IRQ(port));
+	/* initially disable the interrupts */
+	writel(0, port->base + PINCTRL_PIN2IRQ(port));
 	writel(0, port->base + PINCTRL_IRQEN(port));
 
 	/* clear address has to be used to clear IRQSTAT bits */
diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
index fe731f094257..d5d72d84b719 100644
--- a/drivers/gpio/gpio-pca953x.c
+++ b/drivers/gpio/gpio-pca953x.c
@@ -74,6 +74,7 @@ static const struct i2c_device_id pca953x_id[] = {
 	{ "max7312", 16 | PCA953X_TYPE | PCA_INT, },
 	{ "max7313", 16 | PCA953X_TYPE | PCA_INT, },
 	{ "max7315", 8  | PCA953X_TYPE | PCA_INT, },
+	{ "max7318", 16 | PCA953X_TYPE | PCA_INT, },
 	{ "pca6107", 8  | PCA953X_TYPE | PCA_INT, },
 	{ "tca6408", 8  | PCA953X_TYPE | PCA_INT, },
 	{ "tca6416", 16 | PCA953X_TYPE | PCA_INT, },
@@ -635,20 +636,20 @@ static int pca953x_irq_setup(struct pca953x_chip *chip,
 			return ret;
 		}
 
-		ret =  gpiochip_irqchip_add(&chip->gpio_chip,
-					    &pca953x_irq_chip,
-					    irq_base,
-					    handle_simple_irq,
-					    IRQ_TYPE_NONE);
+		ret =  gpiochip_irqchip_add_nested(&chip->gpio_chip,
+						   &pca953x_irq_chip,
+						   irq_base,
+						   handle_simple_irq,
+						   IRQ_TYPE_NONE);
 		if (ret) {
 			dev_err(&client->dev,
 				"could not connect irqchip to gpiochip\n");
 			return ret;
 		}
 
-		gpiochip_set_chained_irqchip(&chip->gpio_chip,
-					     &pca953x_irq_chip,
-					     client->irq, NULL);
+		gpiochip_set_nested_irqchip(&chip->gpio_chip,
+					    &pca953x_irq_chip,
+					    client->irq);
 	}
 
 	return 0;
@@ -907,6 +908,7 @@ static const struct of_device_id pca953x_dt_ids[] = {
 	{ .compatible = "maxim,max7312", .data = OF_953X(16, PCA_INT), },
 	{ .compatible = "maxim,max7313", .data = OF_953X(16, PCA_INT), },
 	{ .compatible = "maxim,max7315", .data = OF_953X( 8, PCA_INT), },
+	{ .compatible = "maxim,max7318", .data = OF_953X(16, PCA_INT), },
 
 	{ .compatible = "ti,pca6107", .data = OF_953X( 8, PCA_INT), },
 	{ .compatible = "ti,pca9536", .data = OF_953X( 4, 0), },
diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c
index d168410e2338..895af42a4513 100644
--- a/drivers/gpio/gpio-pcf857x.c
+++ b/drivers/gpio/gpio-pcf857x.c
@@ -378,9 +378,10 @@ static int pcf857x_probe(struct i2c_client *client,
 
 	/* Enable irqchip if we have an interrupt */
 	if (client->irq) {
-		status = gpiochip_irqchip_add(&gpio->chip, &pcf857x_irq_chip,
-					      0, handle_level_irq,
-					      IRQ_TYPE_NONE);
+		status = gpiochip_irqchip_add_nested(&gpio->chip,
+						     &pcf857x_irq_chip,
+						     0, handle_level_irq,
+						     IRQ_TYPE_NONE);
 		if (status) {
 			dev_err(&client->dev, "cannot add irqchip\n");
 			goto fail;
@@ -393,8 +394,8 @@ static int pcf857x_probe(struct i2c_client *client,
 		if (status)
 			goto fail;
 
-		gpiochip_set_chained_irqchip(&gpio->chip, &pcf857x_irq_chip,
-					     client->irq, NULL);
+		gpiochip_set_nested_irqchip(&gpio->chip, &pcf857x_irq_chip,
+					    client->irq);
 		gpio->irq_parent = client->irq;
 	}
 
diff --git a/drivers/gpio/gpio-pl061.c b/drivers/gpio/gpio-pl061.c
index 6e3c1430616f..0a6bfd2b06e5 100644
--- a/drivers/gpio/gpio-pl061.c
+++ b/drivers/gpio/gpio-pl061.c
@@ -23,7 +23,6 @@
 #include <linux/gpio.h>
 #include <linux/device.h>
 #include <linux/amba/bus.h>
-#include <linux/amba/pl061.h>
 #include <linux/slab.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/pm.h>
@@ -50,11 +49,12 @@ struct pl061_context_save_regs {
 };
 #endif
 
-struct pl061_gpio {
+struct pl061 {
 	spinlock_t		lock;
 
 	void __iomem		*base;
 	struct gpio_chip	gc;
+	int			parent_irq;
 
 #ifdef CONFIG_PM
 	struct pl061_context_save_regs csave_regs;
@@ -63,22 +63,22 @@ struct pl061_gpio {
 
 static int pl061_get_direction(struct gpio_chip *gc, unsigned offset)
 {
-	struct pl061_gpio *chip = gpiochip_get_data(gc);
+	struct pl061 *pl061 = gpiochip_get_data(gc);
 
-	return !(readb(chip->base + GPIODIR) & BIT(offset));
+	return !(readb(pl061->base + GPIODIR) & BIT(offset));
 }
 
 static int pl061_direction_input(struct gpio_chip *gc, unsigned offset)
 {
-	struct pl061_gpio *chip = gpiochip_get_data(gc);
+	struct pl061 *pl061 = gpiochip_get_data(gc);
 	unsigned long flags;
 	unsigned char gpiodir;
 
-	spin_lock_irqsave(&chip->lock, flags);
-	gpiodir = readb(chip->base + GPIODIR);
+	spin_lock_irqsave(&pl061->lock, flags);
+	gpiodir = readb(pl061->base + GPIODIR);
 	gpiodir &= ~(BIT(offset));
-	writeb(gpiodir, chip->base + GPIODIR);
-	spin_unlock_irqrestore(&chip->lock, flags);
+	writeb(gpiodir, pl061->base + GPIODIR);
+	spin_unlock_irqrestore(&pl061->lock, flags);
 
 	return 0;
 }
@@ -86,44 +86,44 @@ static int pl061_direction_input(struct gpio_chip *gc, unsigned offset)
 static int pl061_direction_output(struct gpio_chip *gc, unsigned offset,
 		int value)
 {
-	struct pl061_gpio *chip = gpiochip_get_data(gc);
+	struct pl061 *pl061 = gpiochip_get_data(gc);
 	unsigned long flags;
 	unsigned char gpiodir;
 
-	spin_lock_irqsave(&chip->lock, flags);
-	writeb(!!value << offset, chip->base + (BIT(offset + 2)));
-	gpiodir = readb(chip->base + GPIODIR);
+	spin_lock_irqsave(&pl061->lock, flags);
+	writeb(!!value << offset, pl061->base + (BIT(offset + 2)));
+	gpiodir = readb(pl061->base + GPIODIR);
 	gpiodir |= BIT(offset);
-	writeb(gpiodir, chip->base + GPIODIR);
+	writeb(gpiodir, pl061->base + GPIODIR);
 
 	/*
 	 * gpio value is set again, because pl061 doesn't allow to set value of
 	 * a gpio pin before configuring it in OUT mode.
 	 */
-	writeb(!!value << offset, chip->base + (BIT(offset + 2)));
-	spin_unlock_irqrestore(&chip->lock, flags);
+	writeb(!!value << offset, pl061->base + (BIT(offset + 2)));
+	spin_unlock_irqrestore(&pl061->lock, flags);
 
 	return 0;
 }
 
 static int pl061_get_value(struct gpio_chip *gc, unsigned offset)
 {
-	struct pl061_gpio *chip = gpiochip_get_data(gc);
+	struct pl061 *pl061 = gpiochip_get_data(gc);
 
-	return !!readb(chip->base + (BIT(offset + 2)));
+	return !!readb(pl061->base + (BIT(offset + 2)));
 }
 
 static void pl061_set_value(struct gpio_chip *gc, unsigned offset, int value)
 {
-	struct pl061_gpio *chip = gpiochip_get_data(gc);
+	struct pl061 *pl061 = gpiochip_get_data(gc);
 
-	writeb(!!value << offset, chip->base + (BIT(offset + 2)));
+	writeb(!!value << offset, pl061->base + (BIT(offset + 2)));
 }
 
 static int pl061_irq_type(struct irq_data *d, unsigned trigger)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pl061_gpio *chip = gpiochip_get_data(gc);
+	struct pl061 *pl061 = gpiochip_get_data(gc);
 	int offset = irqd_to_hwirq(d);
 	unsigned long flags;
 	u8 gpiois, gpioibe, gpioiev;
@@ -143,11 +143,11 @@ static int pl061_irq_type(struct irq_data *d, unsigned trigger)
 	}
 
 
-	spin_lock_irqsave(&chip->lock, flags);
+	spin_lock_irqsave(&pl061->lock, flags);
 
-	gpioiev = readb(chip->base + GPIOIEV);
-	gpiois = readb(chip->base + GPIOIS);
-	gpioibe = readb(chip->base + GPIOIBE);
+	gpioiev = readb(pl061->base + GPIOIEV);
+	gpiois = readb(pl061->base + GPIOIS);
+	gpioibe = readb(pl061->base + GPIOIBE);
 
 	if (trigger & (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW)) {
 		bool polarity = trigger & IRQ_TYPE_LEVEL_HIGH;
@@ -199,11 +199,11 @@ static int pl061_irq_type(struct irq_data *d, unsigned trigger)
 			 offset);
 	}
 
-	writeb(gpiois, chip->base + GPIOIS);
-	writeb(gpioibe, chip->base + GPIOIBE);
-	writeb(gpioiev, chip->base + GPIOIEV);
+	writeb(gpiois, pl061->base + GPIOIS);
+	writeb(gpioibe, pl061->base + GPIOIBE);
+	writeb(gpioiev, pl061->base + GPIOIEV);
 
-	spin_unlock_irqrestore(&chip->lock, flags);
+	spin_unlock_irqrestore(&pl061->lock, flags);
 
 	return 0;
 }
@@ -213,12 +213,12 @@ static void pl061_irq_handler(struct irq_desc *desc)
 	unsigned long pending;
 	int offset;
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
-	struct pl061_gpio *chip = gpiochip_get_data(gc);
+	struct pl061 *pl061 = gpiochip_get_data(gc);
 	struct irq_chip *irqchip = irq_desc_get_chip(desc);
 
 	chained_irq_enter(irqchip, desc);
 
-	pending = readb(chip->base + GPIOMIS);
+	pending = readb(pl061->base + GPIOMIS);
 	if (pending) {
 		for_each_set_bit(offset, &pending, PL061_GPIO_NR)
 			generic_handle_irq(irq_find_mapping(gc->irqdomain,
@@ -231,27 +231,27 @@ static void pl061_irq_handler(struct irq_desc *desc)
 static void pl061_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pl061_gpio *chip = gpiochip_get_data(gc);
+	struct pl061 *pl061 = gpiochip_get_data(gc);
 	u8 mask = BIT(irqd_to_hwirq(d) % PL061_GPIO_NR);
 	u8 gpioie;
 
-	spin_lock(&chip->lock);
-	gpioie = readb(chip->base + GPIOIE) & ~mask;
-	writeb(gpioie, chip->base + GPIOIE);
-	spin_unlock(&chip->lock);
+	spin_lock(&pl061->lock);
+	gpioie = readb(pl061->base + GPIOIE) & ~mask;
+	writeb(gpioie, pl061->base + GPIOIE);
+	spin_unlock(&pl061->lock);
 }
 
 static void pl061_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pl061_gpio *chip = gpiochip_get_data(gc);
+	struct pl061 *pl061 = gpiochip_get_data(gc);
 	u8 mask = BIT(irqd_to_hwirq(d) % PL061_GPIO_NR);
 	u8 gpioie;
 
-	spin_lock(&chip->lock);
-	gpioie = readb(chip->base + GPIOIE) | mask;
-	writeb(gpioie, chip->base + GPIOIE);
-	spin_unlock(&chip->lock);
+	spin_lock(&pl061->lock);
+	gpioie = readb(pl061->base + GPIOIE) | mask;
+	writeb(gpioie, pl061->base + GPIOIE);
+	spin_unlock(&pl061->lock);
 }
 
 /**
@@ -265,19 +265,20 @@ static void pl061_irq_unmask(struct irq_data *d)
 static void pl061_irq_ack(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct pl061_gpio *chip = gpiochip_get_data(gc);
+	struct pl061 *pl061 = gpiochip_get_data(gc);
 	u8 mask = BIT(irqd_to_hwirq(d) % PL061_GPIO_NR);
 
-	spin_lock(&chip->lock);
-	writeb(mask, chip->base + GPIOIC);
-	spin_unlock(&chip->lock);
+	spin_lock(&pl061->lock);
+	writeb(mask, pl061->base + GPIOIC);
+	spin_unlock(&pl061->lock);
 }
 
 static int pl061_irq_set_wake(struct irq_data *d, unsigned int state)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct pl061 *pl061 = gpiochip_get_data(gc);
 
-	return irq_set_irq_wake(gc->irq_parent, state);
+	return irq_set_irq_wake(pl061->parent_irq, state);
 }
 
 static struct irq_chip pl061_irqchip = {
@@ -292,81 +293,60 @@ static struct irq_chip pl061_irqchip = {
 static int pl061_probe(struct amba_device *adev, const struct amba_id *id)
 {
 	struct device *dev = &adev->dev;
-	struct pl061_platform_data *pdata = dev_get_platdata(dev);
-	struct pl061_gpio *chip;
-	int ret, irq, i, irq_base;
+	struct pl061 *pl061;
+	int ret, irq;
 
-	chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
-	if (chip == NULL)
+	pl061 = devm_kzalloc(dev, sizeof(*pl061), GFP_KERNEL);
+	if (pl061 == NULL)
 		return -ENOMEM;
 
-	if (pdata) {
-		chip->gc.base = pdata->gpio_base;
-		irq_base = pdata->irq_base;
-		if (irq_base <= 0) {
-			dev_err(&adev->dev, "invalid IRQ base in pdata\n");
-			return -ENODEV;
-		}
-	} else {
-		chip->gc.base = -1;
-		irq_base = 0;
-	}
-
-	chip->base = devm_ioremap_resource(dev, &adev->res);
-	if (IS_ERR(chip->base))
-		return PTR_ERR(chip->base);
+	pl061->base = devm_ioremap_resource(dev, &adev->res);
+	if (IS_ERR(pl061->base))
+		return PTR_ERR(pl061->base);
 
-	spin_lock_init(&chip->lock);
+	spin_lock_init(&pl061->lock);
 	if (of_property_read_bool(dev->of_node, "gpio-ranges")) {
-		chip->gc.request = gpiochip_generic_request;
-		chip->gc.free = gpiochip_generic_free;
+		pl061->gc.request = gpiochip_generic_request;
+		pl061->gc.free = gpiochip_generic_free;
 	}
 
-	chip->gc.get_direction = pl061_get_direction;
-	chip->gc.direction_input = pl061_direction_input;
-	chip->gc.direction_output = pl061_direction_output;
-	chip->gc.get = pl061_get_value;
-	chip->gc.set = pl061_set_value;
-	chip->gc.ngpio = PL061_GPIO_NR;
-	chip->gc.label = dev_name(dev);
-	chip->gc.parent = dev;
-	chip->gc.owner = THIS_MODULE;
-
-	ret = gpiochip_add_data(&chip->gc, chip);
+	pl061->gc.base = -1;
+	pl061->gc.get_direction = pl061_get_direction;
+	pl061->gc.direction_input = pl061_direction_input;
+	pl061->gc.direction_output = pl061_direction_output;
+	pl061->gc.get = pl061_get_value;
+	pl061->gc.set = pl061_set_value;
+	pl061->gc.ngpio = PL061_GPIO_NR;
+	pl061->gc.label = dev_name(dev);
+	pl061->gc.parent = dev;
+	pl061->gc.owner = THIS_MODULE;
+
+	ret = gpiochip_add_data(&pl061->gc, pl061);
 	if (ret)
 		return ret;
 
 	/*
 	 * irq_chip support
 	 */
-	writeb(0, chip->base + GPIOIE); /* disable irqs */
+	writeb(0, pl061->base + GPIOIE); /* disable irqs */
 	irq = adev->irq[0];
 	if (irq < 0) {
 		dev_err(&adev->dev, "invalid IRQ\n");
 		return -ENODEV;
 	}
+	pl061->parent_irq = irq;
 
-	ret = gpiochip_irqchip_add(&chip->gc, &pl061_irqchip,
-				   irq_base, handle_bad_irq,
+	ret = gpiochip_irqchip_add(&pl061->gc, &pl061_irqchip,
+				   0, handle_bad_irq,
 				   IRQ_TYPE_NONE);
 	if (ret) {
 		dev_info(&adev->dev, "could not add irqchip\n");
 		return ret;
 	}
-	gpiochip_set_chained_irqchip(&chip->gc, &pl061_irqchip,
+	gpiochip_set_chained_irqchip(&pl061->gc, &pl061_irqchip,
 				     irq, pl061_irq_handler);
 
-	for (i = 0; i < PL061_GPIO_NR; i++) {
-		if (pdata) {
-			if (pdata->directions & (BIT(i)))
-				pl061_direction_output(&chip->gc, i,
-						pdata->values & (BIT(i)));
-			else
-				pl061_direction_input(&chip->gc, i);
-		}
-	}
-
-	amba_set_drvdata(adev, chip);
+	amba_set_drvdata(adev, pl061);
 	dev_info(&adev->dev, "PL061 GPIO chip @%pa registered\n",
 		 &adev->res.start);
 
@@ -376,20 +356,20 @@ static int pl061_probe(struct amba_device *adev, const struct amba_id *id)
 #ifdef CONFIG_PM
 static int pl061_suspend(struct device *dev)
 {
-	struct pl061_gpio *chip = dev_get_drvdata(dev);
+	struct pl061 *pl061 = dev_get_drvdata(dev);
 	int offset;
 
-	chip->csave_regs.gpio_data = 0;
-	chip->csave_regs.gpio_dir = readb(chip->base + GPIODIR);
-	chip->csave_regs.gpio_is = readb(chip->base + GPIOIS);
-	chip->csave_regs.gpio_ibe = readb(chip->base + GPIOIBE);
-	chip->csave_regs.gpio_iev = readb(chip->base + GPIOIEV);
-	chip->csave_regs.gpio_ie = readb(chip->base + GPIOIE);
+	pl061->csave_regs.gpio_data = 0;
+	pl061->csave_regs.gpio_dir = readb(pl061->base + GPIODIR);
+	pl061->csave_regs.gpio_is = readb(pl061->base + GPIOIS);
+	pl061->csave_regs.gpio_ibe = readb(pl061->base + GPIOIBE);
+	pl061->csave_regs.gpio_iev = readb(pl061->base + GPIOIEV);
+	pl061->csave_regs.gpio_ie = readb(pl061->base + GPIOIE);
 
 	for (offset = 0; offset < PL061_GPIO_NR; offset++) {
-		if (chip->csave_regs.gpio_dir & (BIT(offset)))
-			chip->csave_regs.gpio_data |=
-				pl061_get_value(&chip->gc, offset) << offset;
+		if (pl061->csave_regs.gpio_dir & (BIT(offset)))
+			pl061->csave_regs.gpio_data |=
+				pl061_get_value(&pl061->gc, offset) << offset;
 	}
 
 	return 0;
@@ -397,22 +377,22 @@ static int pl061_suspend(struct device *dev)
 
 static int pl061_resume(struct device *dev)
 {
-	struct pl061_gpio *chip = dev_get_drvdata(dev);
+	struct pl061 *pl061 = dev_get_drvdata(dev);
 	int offset;
 
 	for (offset = 0; offset < PL061_GPIO_NR; offset++) {
-		if (chip->csave_regs.gpio_dir & (BIT(offset)))
-			pl061_direction_output(&chip->gc, offset,
-					chip->csave_regs.gpio_data &
+		if (pl061->csave_regs.gpio_dir & (BIT(offset)))
+			pl061_direction_output(&pl061->gc, offset,
+					pl061->csave_regs.gpio_data &
 					(BIT(offset)));
 		else
-			pl061_direction_input(&chip->gc, offset);
+			pl061_direction_input(&pl061->gc, offset);
 	}
 
-	writeb(chip->csave_regs.gpio_is, chip->base + GPIOIS);
-	writeb(chip->csave_regs.gpio_ibe, chip->base + GPIOIBE);
-	writeb(chip->csave_regs.gpio_iev, chip->base + GPIOIEV);
-	writeb(chip->csave_regs.gpio_ie, chip->base + GPIOIE);
+	writeb(pl061->csave_regs.gpio_is, pl061->base + GPIOIS);
+	writeb(pl061->csave_regs.gpio_ibe, pl061->base + GPIOIBE);
+	writeb(pl061->csave_regs.gpio_iev, pl061->base + GPIOIEV);
+	writeb(pl061->csave_regs.gpio_ie, pl061->base + GPIOIE);
 
 	return 0;
 }
diff --git a/drivers/gpio/gpio-stmpe.c b/drivers/gpio/gpio-stmpe.c
index 5b0042776ec7..16cbc5702865 100644
--- a/drivers/gpio/gpio-stmpe.c
+++ b/drivers/gpio/gpio-stmpe.c
@@ -413,7 +413,7 @@ static irqreturn_t stmpe_gpio_irq(int irq, void *dev)
 		    stmpe->partnum != STMPE1801) {
 			stmpe_reg_write(stmpe, statmsbreg + i, status[i]);
 			stmpe_reg_write(stmpe,
-					stmpe->regs[STMPE_IDX_GPEDR_LSB + i],
+					stmpe->regs[STMPE_IDX_GPEDR_MSB] + i,
 					status[i]);
 		}
 	}
@@ -484,21 +484,20 @@ static int stmpe_gpio_probe(struct platform_device *pdev)
 				if (stmpe_gpio->norequest_mask & BIT(i))
 					clear_bit(i, stmpe_gpio->chip.irq_valid_mask);
 		}
-		ret =  gpiochip_irqchip_add(&stmpe_gpio->chip,
-					    &stmpe_gpio_irq_chip,
-					    0,
-					    handle_simple_irq,
-					    IRQ_TYPE_NONE);
+		ret =  gpiochip_irqchip_add_nested(&stmpe_gpio->chip,
+						   &stmpe_gpio_irq_chip,
+						   0,
+						   handle_simple_irq,
+						   IRQ_TYPE_NONE);
 		if (ret) {
 			dev_err(&pdev->dev,
 				"could not connect irqchip to gpiochip\n");
 			goto out_disable;
 		}
 
-		gpiochip_set_chained_irqchip(&stmpe_gpio->chip,
-					     &stmpe_gpio_irq_chip,
-					     irq,
-					     NULL);
+		gpiochip_set_nested_irqchip(&stmpe_gpio->chip,
+					    &stmpe_gpio_irq_chip,
+					    irq);
 	}
 
 	platform_set_drvdata(pdev, stmpe_gpio);
diff --git a/drivers/gpio/gpio-sx150x.c b/drivers/gpio/gpio-sx150x.c
deleted file mode 100644
index af95de89db01..000000000000
--- a/drivers/gpio/gpio-sx150x.c
+++ /dev/null
@@ -1,792 +0,0 @@
-/* Copyright (c) 2010, Code Aurora Forum. All rights reserved.
- *
- * Driver for Semtech SX150X I2C GPIO Expanders
- *
- * Author: Gregory Bean <gbean@codeaurora.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- */
-#include <linux/gpio.h>
-#include <linux/i2c.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/mutex.h>
-#include <linux/slab.h>
-#include <linux/of.h>
-#include <linux/of_address.h>
-#include <linux/of_irq.h>
-#include <linux/of_gpio.h>
-#include <linux/of_device.h>
-
-#define NO_UPDATE_PENDING	-1
-
-/* The chip models of sx150x */
-#define SX150X_123 0
-#define SX150X_456 1
-#define SX150X_789 2
-
-struct sx150x_123_pri {
-	u8 reg_pld_mode;
-	u8 reg_pld_table0;
-	u8 reg_pld_table1;
-	u8 reg_pld_table2;
-	u8 reg_pld_table3;
-	u8 reg_pld_table4;
-	u8 reg_advance;
-};
-
-struct sx150x_456_pri {
-	u8 reg_pld_mode;
-	u8 reg_pld_table0;
-	u8 reg_pld_table1;
-	u8 reg_pld_table2;
-	u8 reg_pld_table3;
-	u8 reg_pld_table4;
-	u8 reg_advance;
-};
-
-struct sx150x_789_pri {
-	u8 reg_drain;
-	u8 reg_polarity;
-	u8 reg_clock;
-	u8 reg_misc;
-	u8 reg_reset;
-	u8 ngpios;
-};
-
-struct sx150x_device_data {
-	u8 model;
-	u8 reg_pullup;
-	u8 reg_pulldn;
-	u8 reg_dir;
-	u8 reg_data;
-	u8 reg_irq_mask;
-	u8 reg_irq_src;
-	u8 reg_sense;
-	u8 ngpios;
-	union {
-		struct sx150x_123_pri x123;
-		struct sx150x_456_pri x456;
-		struct sx150x_789_pri x789;
-	} pri;
-};
-
-/**
- * struct sx150x_platform_data - config data for SX150x driver
- * @gpio_base: The index number of the first GPIO assigned to this
- *             GPIO expander.  The expander will create a block of
- *             consecutively numbered gpios beginning at the given base,
- *             with the size of the block depending on the model of the
- *             expander chip.
- * @oscio_is_gpo: If set to true, the driver will configure OSCIO as a GPO
- *                instead of as an oscillator, increasing the size of the
- *                GP(I)O pool created by this expander by one.  The
- *                output-only GPO pin will be added at the end of the block.
- * @io_pullup_ena: A bit-mask which enables or disables the pull-up resistor
- *                 for each IO line in the expander.  Setting the bit at
- *                 position n will enable the pull-up for the IO at
- *                 the corresponding offset.  For chips with fewer than
- *                 16 IO pins, high-end bits are ignored.
- * @io_pulldn_ena: A bit-mask which enables-or disables the pull-down
- *                 resistor for each IO line in the expander. Setting the
- *                 bit at position n will enable the pull-down for the IO at
- *                 the corresponding offset.  For chips with fewer than
- *                 16 IO pins, high-end bits are ignored.
- * @io_polarity: A bit-mask which enables polarity inversion for each IO line
- *               in the expander.  Setting the bit at position n inverts
- *               the polarity of that IO line, while clearing it results
- *               in normal polarity. For chips with fewer than 16 IO pins,
- *               high-end bits are ignored.
- * @irq_summary: The 'summary IRQ' line to which the GPIO expander's INT line
- *               is connected, via which it reports interrupt events
- *               across all GPIO lines.  This must be a real,
- *               pre-existing IRQ line.
- *               Setting this value < 0 disables the irq_chip functionality
- *               of the driver.
- * @irq_base: The first 'virtual IRQ' line at which our block of GPIO-based
- *            IRQ lines will appear.  Similarly to gpio_base, the expander
- *            will create a block of irqs beginning at this number.
- *            This value is ignored if irq_summary is < 0.
- * @reset_during_probe: If set to true, the driver will trigger a full
- *                      reset of the chip at the beginning of the probe
- *                      in order to place it in a known state.
- */
-struct sx150x_platform_data {
-	unsigned gpio_base;
-	bool     oscio_is_gpo;
-	u16      io_pullup_ena;
-	u16      io_pulldn_ena;
-	u16      io_polarity;
-	int      irq_summary;
-	unsigned irq_base;
-	bool     reset_during_probe;
-};
-
-struct sx150x_chip {
-	struct gpio_chip                 gpio_chip;
-	struct i2c_client               *client;
-	const struct sx150x_device_data *dev_cfg;
-	int                              irq_summary;
-	int                              irq_base;
-	int				 irq_update;
-	u32                              irq_sense;
-	u32				 irq_masked;
-	u32				 dev_sense;
-	u32				 dev_masked;
-	struct irq_chip                  irq_chip;
-	struct mutex                     lock;
-};
-
-static const struct sx150x_device_data sx150x_devices[] = {
-	[0] = { /* sx1508q */
-		.model = SX150X_789,
-		.reg_pullup	= 0x03,
-		.reg_pulldn	= 0x04,
-		.reg_dir	= 0x07,
-		.reg_data	= 0x08,
-		.reg_irq_mask	= 0x09,
-		.reg_irq_src	= 0x0c,
-		.reg_sense	= 0x0b,
-		.pri.x789 = {
-			.reg_drain	= 0x05,
-			.reg_polarity	= 0x06,
-			.reg_clock	= 0x0f,
-			.reg_misc	= 0x10,
-			.reg_reset	= 0x7d,
-		},
-		.ngpios = 8,
-	},
-	[1] = { /* sx1509q */
-		.model = SX150X_789,
-		.reg_pullup	= 0x07,
-		.reg_pulldn	= 0x09,
-		.reg_dir	= 0x0f,
-		.reg_data	= 0x11,
-		.reg_irq_mask	= 0x13,
-		.reg_irq_src	= 0x19,
-		.reg_sense	= 0x17,
-		.pri.x789 = {
-			.reg_drain	= 0x0b,
-			.reg_polarity	= 0x0d,
-			.reg_clock	= 0x1e,
-			.reg_misc	= 0x1f,
-			.reg_reset	= 0x7d,
-		},
-		.ngpios	= 16
-	},
-	[2] = { /* sx1506q */
-		.model = SX150X_456,
-		.reg_pullup	= 0x05,
-		.reg_pulldn	= 0x07,
-		.reg_dir	= 0x03,
-		.reg_data	= 0x01,
-		.reg_irq_mask	= 0x09,
-		.reg_irq_src	= 0x0f,
-		.reg_sense	= 0x0d,
-		.pri.x456 = {
-			.reg_pld_mode	= 0x21,
-			.reg_pld_table0	= 0x23,
-			.reg_pld_table1	= 0x25,
-			.reg_pld_table2	= 0x27,
-			.reg_pld_table3	= 0x29,
-			.reg_pld_table4	= 0x2b,
-			.reg_advance	= 0xad,
-		},
-		.ngpios	= 16
-	},
-	[3] = { /* sx1502q */
-		.model = SX150X_123,
-		.reg_pullup	= 0x02,
-		.reg_pulldn	= 0x03,
-		.reg_dir	= 0x01,
-		.reg_data	= 0x00,
-		.reg_irq_mask	= 0x05,
-		.reg_irq_src	= 0x08,
-		.reg_sense	= 0x07,
-		.pri.x123 = {
-			.reg_pld_mode	= 0x10,
-			.reg_pld_table0	= 0x11,
-			.reg_pld_table1	= 0x12,
-			.reg_pld_table2	= 0x13,
-			.reg_pld_table3	= 0x14,
-			.reg_pld_table4	= 0x15,
-			.reg_advance	= 0xad,
-		},
-		.ngpios	= 8,
-	},
-};
-
-static const struct i2c_device_id sx150x_id[] = {
-	{"sx1508q", 0},
-	{"sx1509q", 1},
-	{"sx1506q", 2},
-	{"sx1502q", 3},
-	{}
-};
-
-static const struct of_device_id sx150x_of_match[] = {
-	{ .compatible = "semtech,sx1508q" },
-	{ .compatible = "semtech,sx1509q" },
-	{ .compatible = "semtech,sx1506q" },
-	{ .compatible = "semtech,sx1502q" },
-	{},
-};
-
-static s32 sx150x_i2c_write(struct i2c_client *client, u8 reg, u8 val)
-{
-	s32 err = i2c_smbus_write_byte_data(client, reg, val);
-
-	if (err < 0)
-		dev_warn(&client->dev,
-			"i2c write fail: can't write %02x to %02x: %d\n",
-			val, reg, err);
-	return err;
-}
-
-static s32 sx150x_i2c_read(struct i2c_client *client, u8 reg, u8 *val)
-{
-	s32 err = i2c_smbus_read_byte_data(client, reg);
-
-	if (err >= 0)
-		*val = err;
-	else
-		dev_warn(&client->dev,
-			"i2c read fail: can't read from %02x: %d\n",
-			reg, err);
-	return err;
-}
-
-static inline bool offset_is_oscio(struct sx150x_chip *chip, unsigned offset)
-{
-	return (chip->dev_cfg->ngpios == offset);
-}
-
-/*
- * These utility functions solve the common problem of locating and setting
- * configuration bits.  Configuration bits are grouped into registers
- * whose indexes increase downwards.  For example, with eight-bit registers,
- * sixteen gpios would have their config bits grouped in the following order:
- * REGISTER N-1 [ f e d c b a 9 8 ]
- *          N   [ 7 6 5 4 3 2 1 0 ]
- *
- * For multi-bit configurations, the pattern gets wider:
- * REGISTER N-3 [ f f e e d d c c ]
- *          N-2 [ b b a a 9 9 8 8 ]
- *          N-1 [ 7 7 6 6 5 5 4 4 ]
- *          N   [ 3 3 2 2 1 1 0 0 ]
- *
- * Given the address of the starting register 'N', the index of the gpio
- * whose configuration we seek to change, and the width in bits of that
- * configuration, these functions allow us to locate the correct
- * register and mask the correct bits.
- */
-static inline void sx150x_find_cfg(u8 offset, u8 width,
-				u8 *reg, u8 *mask, u8 *shift)
-{
-	*reg   -= offset * width / 8;
-	*mask   = (1 << width) - 1;
-	*shift  = (offset * width) % 8;
-	*mask <<= *shift;
-}
-
-static s32 sx150x_write_cfg(struct sx150x_chip *chip,
-			u8 offset, u8 width, u8 reg, u8 val)
-{
-	u8  mask;
-	u8  data;
-	u8  shift;
-	s32 err;
-
-	sx150x_find_cfg(offset, width, &reg, &mask, &shift);
-	err = sx150x_i2c_read(chip->client, reg, &data);
-	if (err < 0)
-		return err;
-
-	data &= ~mask;
-	data |= (val << shift) & mask;
-	return sx150x_i2c_write(chip->client, reg, data);
-}
-
-static int sx150x_get_io(struct sx150x_chip *chip, unsigned offset)
-{
-	u8  reg = chip->dev_cfg->reg_data;
-	u8  mask;
-	u8  data;
-	u8  shift;
-	s32 err;
-
-	sx150x_find_cfg(offset, 1, &reg, &mask, &shift);
-	err = sx150x_i2c_read(chip->client, reg, &data);
-	if (err >= 0)
-		err = (data & mask) != 0 ? 1 : 0;
-
-	return err;
-}
-
-static void sx150x_set_oscio(struct sx150x_chip *chip, int val)
-{
-	sx150x_i2c_write(chip->client,
-			chip->dev_cfg->pri.x789.reg_clock,
-			(val ? 0x1f : 0x10));
-}
-
-static void sx150x_set_io(struct sx150x_chip *chip, unsigned offset, int val)
-{
-	sx150x_write_cfg(chip,
-			offset,
-			1,
-			chip->dev_cfg->reg_data,
-			(val ? 1 : 0));
-}
-
-static int sx150x_io_input(struct sx150x_chip *chip, unsigned offset)
-{
-	return sx150x_write_cfg(chip,
-				offset,
-				1,
-				chip->dev_cfg->reg_dir,
-				1);
-}
-
-static int sx150x_io_output(struct sx150x_chip *chip, unsigned offset, int val)
-{
-	int err;
-
-	err = sx150x_write_cfg(chip,
-			offset,
-			1,
-			chip->dev_cfg->reg_data,
-			(val ? 1 : 0));
-	if (err >= 0)
-		err = sx150x_write_cfg(chip,
-				offset,
-				1,
-				chip->dev_cfg->reg_dir,
-				0);
-	return err;
-}
-
-static int sx150x_gpio_get(struct gpio_chip *gc, unsigned offset)
-{
-	struct sx150x_chip *chip = gpiochip_get_data(gc);
-	int status = -EINVAL;
-
-	if (!offset_is_oscio(chip, offset)) {
-		mutex_lock(&chip->lock);
-		status = sx150x_get_io(chip, offset);
-		mutex_unlock(&chip->lock);
-	}
-
-	return (status < 0) ? status : !!status;
-}
-
-static void sx150x_gpio_set(struct gpio_chip *gc, unsigned offset, int val)
-{
-	struct sx150x_chip *chip = gpiochip_get_data(gc);
-
-	mutex_lock(&chip->lock);
-	if (offset_is_oscio(chip, offset))
-		sx150x_set_oscio(chip, val);
-	else
-		sx150x_set_io(chip, offset, val);
-	mutex_unlock(&chip->lock);
-}
-
-static int sx150x_gpio_set_single_ended(struct gpio_chip *gc,
-					unsigned offset,
-                                        enum single_ended_mode mode)
-{
-	struct sx150x_chip *chip = gpiochip_get_data(gc);
-
-	/* On the SX160X 789 we can set open drain */
-	if (chip->dev_cfg->model != SX150X_789)
-		return -ENOTSUPP;
-
-	if (mode == LINE_MODE_PUSH_PULL)
-		return sx150x_write_cfg(chip,
-					offset,
-					1,
-					chip->dev_cfg->pri.x789.reg_drain,
-					0);
-
-	if (mode == LINE_MODE_OPEN_DRAIN)
-		return sx150x_write_cfg(chip,
-					offset,
-					1,
-					chip->dev_cfg->pri.x789.reg_drain,
-					1);
-	return -ENOTSUPP;
-}
-
-static int sx150x_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
-{
-	struct sx150x_chip *chip = gpiochip_get_data(gc);
-	int status = -EINVAL;
-
-	if (!offset_is_oscio(chip, offset)) {
-		mutex_lock(&chip->lock);
-		status = sx150x_io_input(chip, offset);
-		mutex_unlock(&chip->lock);
-	}
-	return status;
-}
-
-static int sx150x_gpio_direction_output(struct gpio_chip *gc,
-					unsigned offset,
-					int val)
-{
-	struct sx150x_chip *chip = gpiochip_get_data(gc);
-	int status = 0;
-
-	if (!offset_is_oscio(chip, offset)) {
-		mutex_lock(&chip->lock);
-		status = sx150x_io_output(chip, offset, val);
-		mutex_unlock(&chip->lock);
-	}
-	return status;
-}
-
-static void sx150x_irq_mask(struct irq_data *d)
-{
-	struct sx150x_chip *chip = gpiochip_get_data(irq_data_get_irq_chip_data(d));
-	unsigned n = d->hwirq;
-
-	chip->irq_masked |= (1 << n);
-	chip->irq_update = n;
-}
-
-static void sx150x_irq_unmask(struct irq_data *d)
-{
-	struct sx150x_chip *chip = gpiochip_get_data(irq_data_get_irq_chip_data(d));
-	unsigned n = d->hwirq;
-
-	chip->irq_masked &= ~(1 << n);
-	chip->irq_update = n;
-}
-
-static int sx150x_irq_set_type(struct irq_data *d, unsigned int flow_type)
-{
-	struct sx150x_chip *chip = gpiochip_get_data(irq_data_get_irq_chip_data(d));
-	unsigned n, val = 0;
-
-	if (flow_type & (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW))
-		return -EINVAL;
-
-	n = d->hwirq;
-
-	if (flow_type & IRQ_TYPE_EDGE_RISING)
-		val |= 0x1;
-	if (flow_type & IRQ_TYPE_EDGE_FALLING)
-		val |= 0x2;
-
-	chip->irq_sense &= ~(3UL << (n * 2));
-	chip->irq_sense |= val << (n * 2);
-	chip->irq_update = n;
-	return 0;
-}
-
-static irqreturn_t sx150x_irq_thread_fn(int irq, void *dev_id)
-{
-	struct sx150x_chip *chip = (struct sx150x_chip *)dev_id;
-	unsigned nhandled = 0;
-	unsigned sub_irq;
-	unsigned n;
-	s32 err;
-	u8 val;
-	int i;
-
-	for (i = (chip->dev_cfg->ngpios / 8) - 1; i >= 0; --i) {
-		err = sx150x_i2c_read(chip->client,
-				      chip->dev_cfg->reg_irq_src - i,
-				      &val);
-		if (err < 0)
-			continue;
-
-		sx150x_i2c_write(chip->client,
-				chip->dev_cfg->reg_irq_src - i,
-				val);
-		for (n = 0; n < 8; ++n) {
-			if (val & (1 << n)) {
-				sub_irq = irq_find_mapping(
-					chip->gpio_chip.irqdomain,
-					(i * 8) + n);
-				handle_nested_irq(sub_irq);
-				++nhandled;
-			}
-		}
-	}
-
-	return (nhandled > 0 ? IRQ_HANDLED : IRQ_NONE);
-}
-
-static void sx150x_irq_bus_lock(struct irq_data *d)
-{
-	struct sx150x_chip *chip = gpiochip_get_data(irq_data_get_irq_chip_data(d));
-
-	mutex_lock(&chip->lock);
-}
-
-static void sx150x_irq_bus_sync_unlock(struct irq_data *d)
-{
-	struct sx150x_chip *chip = gpiochip_get_data(irq_data_get_irq_chip_data(d));
-	unsigned n;
-
-	if (chip->irq_update == NO_UPDATE_PENDING)
-		goto out;
-
-	n = chip->irq_update;
-	chip->irq_update = NO_UPDATE_PENDING;
-
-	/* Avoid updates if nothing changed */
-	if (chip->dev_sense == chip->irq_sense &&
-	    chip->dev_masked == chip->irq_masked)
-		goto out;
-
-	chip->dev_sense = chip->irq_sense;
-	chip->dev_masked = chip->irq_masked;
-
-	if (chip->irq_masked & (1 << n)) {
-		sx150x_write_cfg(chip, n, 1, chip->dev_cfg->reg_irq_mask, 1);
-		sx150x_write_cfg(chip, n, 2, chip->dev_cfg->reg_sense, 0);
-	} else {
-		sx150x_write_cfg(chip, n, 1, chip->dev_cfg->reg_irq_mask, 0);
-		sx150x_write_cfg(chip, n, 2, chip->dev_cfg->reg_sense,
-				 chip->irq_sense >> (n * 2));
-	}
-out:
-	mutex_unlock(&chip->lock);
-}
-
-static void sx150x_init_chip(struct sx150x_chip *chip,
-			struct i2c_client *client,
-			kernel_ulong_t driver_data,
-			struct sx150x_platform_data *pdata)
-{
-	mutex_init(&chip->lock);
-
-	chip->client                     = client;
-	chip->dev_cfg                    = &sx150x_devices[driver_data];
-	chip->gpio_chip.parent              = &client->dev;
-	chip->gpio_chip.label            = client->name;
-	chip->gpio_chip.direction_input  = sx150x_gpio_direction_input;
-	chip->gpio_chip.direction_output = sx150x_gpio_direction_output;
-	chip->gpio_chip.get              = sx150x_gpio_get;
-	chip->gpio_chip.set              = sx150x_gpio_set;
-	chip->gpio_chip.set_single_ended = sx150x_gpio_set_single_ended;
-	chip->gpio_chip.base             = pdata->gpio_base;
-	chip->gpio_chip.can_sleep        = true;
-	chip->gpio_chip.ngpio            = chip->dev_cfg->ngpios;
-#ifdef CONFIG_OF_GPIO
-	chip->gpio_chip.of_node          = client->dev.of_node;
-	chip->gpio_chip.of_gpio_n_cells  = 2;
-#endif
-	if (pdata->oscio_is_gpo)
-		++chip->gpio_chip.ngpio;
-
-	chip->irq_chip.name                = client->name;
-	chip->irq_chip.irq_mask            = sx150x_irq_mask;
-	chip->irq_chip.irq_unmask          = sx150x_irq_unmask;
-	chip->irq_chip.irq_set_type        = sx150x_irq_set_type;
-	chip->irq_chip.irq_bus_lock        = sx150x_irq_bus_lock;
-	chip->irq_chip.irq_bus_sync_unlock = sx150x_irq_bus_sync_unlock;
-	chip->irq_summary                  = -1;
-	chip->irq_base                     = -1;
-	chip->irq_masked                   = ~0;
-	chip->irq_sense                    = 0;
-	chip->dev_masked                   = ~0;
-	chip->dev_sense                    = 0;
-	chip->irq_update		   = NO_UPDATE_PENDING;
-}
-
-static int sx150x_init_io(struct sx150x_chip *chip, u8 base, u16 cfg)
-{
-	int err = 0;
-	unsigned n;
-
-	for (n = 0; err >= 0 && n < (chip->dev_cfg->ngpios / 8); ++n)
-		err = sx150x_i2c_write(chip->client, base - n, cfg >> (n * 8));
-	return err;
-}
-
-static int sx150x_reset(struct sx150x_chip *chip)
-{
-	int err;
-
-	err = i2c_smbus_write_byte_data(chip->client,
-					chip->dev_cfg->pri.x789.reg_reset,
-					0x12);
-	if (err < 0)
-		return err;
-
-	err = i2c_smbus_write_byte_data(chip->client,
-					chip->dev_cfg->pri.x789.reg_reset,
-					0x34);
-	return err;
-}
-
-static int sx150x_init_hw(struct sx150x_chip *chip,
-			struct sx150x_platform_data *pdata)
-{
-	int err = 0;
-
-	if (pdata->reset_during_probe) {
-		err = sx150x_reset(chip);
-		if (err < 0)
-			return err;
-	}
-
-	if (chip->dev_cfg->model == SX150X_789)
-		err = sx150x_i2c_write(chip->client,
-				chip->dev_cfg->pri.x789.reg_misc,
-				0x01);
-	else if (chip->dev_cfg->model == SX150X_456)
-		err = sx150x_i2c_write(chip->client,
-				chip->dev_cfg->pri.x456.reg_advance,
-				0x04);
-	else
-		err = sx150x_i2c_write(chip->client,
-				chip->dev_cfg->pri.x123.reg_advance,
-				0x00);
-	if (err < 0)
-		return err;
-
-	err = sx150x_init_io(chip, chip->dev_cfg->reg_pullup,
-			pdata->io_pullup_ena);
-	if (err < 0)
-		return err;
-
-	err = sx150x_init_io(chip, chip->dev_cfg->reg_pulldn,
-			pdata->io_pulldn_ena);
-	if (err < 0)
-		return err;
-
-	if (chip->dev_cfg->model == SX150X_789) {
-		err = sx150x_init_io(chip,
-				chip->dev_cfg->pri.x789.reg_polarity,
-				pdata->io_polarity);
-		if (err < 0)
-			return err;
-	} else if (chip->dev_cfg->model == SX150X_456) {
-		/* Set all pins to work in normal mode */
-		err = sx150x_init_io(chip,
-				chip->dev_cfg->pri.x456.reg_pld_mode,
-				0);
-		if (err < 0)
-			return err;
-	} else {
-		/* Set all pins to work in normal mode */
-		err = sx150x_init_io(chip,
-				chip->dev_cfg->pri.x123.reg_pld_mode,
-				0);
-		if (err < 0)
-			return err;
-	}
-
-
-	if (pdata->oscio_is_gpo)
-		sx150x_set_oscio(chip, 0);
-
-	return err;
-}
-
-static int sx150x_install_irq_chip(struct sx150x_chip *chip,
-				int irq_summary,
-				int irq_base)
-{
-	int err;
-
-	chip->irq_summary = irq_summary;
-	chip->irq_base    = irq_base;
-
-	/* Add gpio chip to irq subsystem */
-	err = gpiochip_irqchip_add(&chip->gpio_chip,
-		&chip->irq_chip, chip->irq_base,
-		handle_edge_irq, IRQ_TYPE_EDGE_BOTH);
-	if (err) {
-		dev_err(&chip->client->dev,
-			"could not connect irqchip to gpiochip\n");
-		return  err;
-	}
-
-	err = devm_request_threaded_irq(&chip->client->dev,
-			irq_summary, NULL, sx150x_irq_thread_fn,
-			IRQF_ONESHOT | IRQF_SHARED | IRQF_TRIGGER_FALLING,
-			chip->irq_chip.name, chip);
-	if (err < 0) {
-		chip->irq_summary = -1;
-		chip->irq_base    = -1;
-	}
-
-	return err;
-}
-
-static int sx150x_probe(struct i2c_client *client,
-				const struct i2c_device_id *id)
-{
-	static const u32 i2c_funcs = I2C_FUNC_SMBUS_BYTE_DATA |
-				     I2C_FUNC_SMBUS_WRITE_WORD_DATA;
-	struct sx150x_platform_data *pdata;
-	struct sx150x_chip *chip;
-	int rc;
-
-	pdata = dev_get_platdata(&client->dev);
-	if (!pdata)
-		return -EINVAL;
-
-	if (!i2c_check_functionality(client->adapter, i2c_funcs))
-		return -ENOSYS;
-
-	chip = devm_kzalloc(&client->dev,
-		sizeof(struct sx150x_chip), GFP_KERNEL);
-	if (!chip)
-		return -ENOMEM;
-
-	sx150x_init_chip(chip, client, id->driver_data, pdata);
-	rc = sx150x_init_hw(chip, pdata);
-	if (rc < 0)
-		return rc;
-
-	rc = devm_gpiochip_add_data(&client->dev, &chip->gpio_chip, chip);
-	if (rc)
-		return rc;
-
-	if (pdata->irq_summary >= 0) {
-		rc = sx150x_install_irq_chip(chip,
-					pdata->irq_summary,
-					pdata->irq_base);
-		if (rc < 0)
-			return rc;
-	}
-
-	i2c_set_clientdata(client, chip);
-
-	return 0;
-}
-
-static struct i2c_driver sx150x_driver = {
-	.driver = {
-		.name = "sx150x",
-		.of_match_table = of_match_ptr(sx150x_of_match),
-	},
-	.probe    = sx150x_probe,
-	.id_table = sx150x_id,
-};
-
-static int __init sx150x_init(void)
-{
-	return i2c_add_driver(&sx150x_driver);
-}
-subsys_initcall(sx150x_init);
diff --git a/drivers/gpio/gpio-tc3589x.c b/drivers/gpio/gpio-tc3589x.c
index d6e21f1a70a9..be97101c2c9a 100644
--- a/drivers/gpio/gpio-tc3589x.c
+++ b/drivers/gpio/gpio-tc3589x.c
@@ -337,21 +337,20 @@ static int tc3589x_gpio_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	ret =  gpiochip_irqchip_add(&tc3589x_gpio->chip,
-				    &tc3589x_gpio_irq_chip,
-				    0,
-				    handle_simple_irq,
-				    IRQ_TYPE_NONE);
+	ret =  gpiochip_irqchip_add_nested(&tc3589x_gpio->chip,
+					   &tc3589x_gpio_irq_chip,
+					   0,
+					   handle_simple_irq,
+					   IRQ_TYPE_NONE);
 	if (ret) {
 		dev_err(&pdev->dev,
 			"could not connect irqchip to gpiochip\n");
 		return ret;
 	}
 
-	gpiochip_set_chained_irqchip(&tc3589x_gpio->chip,
-				     &tc3589x_gpio_irq_chip,
-				     irq,
-				     NULL);
+	gpiochip_set_nested_irqchip(&tc3589x_gpio->chip,
+				    &tc3589x_gpio_irq_chip,
+				    irq);
 
 	platform_set_drvdata(pdev, tc3589x_gpio);
 
diff --git a/drivers/gpio/gpio-vf610.c b/drivers/gpio/gpio-vf610.c
index 3edb09cb9ee0..521fbe338589 100644
--- a/drivers/gpio/gpio-vf610.c
+++ b/drivers/gpio/gpio-vf610.c
@@ -283,8 +283,4 @@ static struct platform_driver vf610_gpio_driver = {
 	.probe		= vf610_gpio_probe,
 };
 
-static int __init gpio_vf610_init(void)
-{
-	return platform_driver_register(&vf610_gpio_driver);
-}
-device_initcall(gpio_vf610_init);
+builtin_platform_driver(vf610_gpio_driver);
diff --git a/drivers/gpio/gpio-wcove.c b/drivers/gpio/gpio-wcove.c
index d0ddba7a9d08..34baee5b1dd6 100644
--- a/drivers/gpio/gpio-wcove.c
+++ b/drivers/gpio/gpio-wcove.c
@@ -426,8 +426,8 @@ static int wcove_gpio_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	ret = gpiochip_irqchip_add(&wg->chip, &wcove_irqchip, 0,
-			     handle_simple_irq, IRQ_TYPE_NONE);
+	ret = gpiochip_irqchip_add_nested(&wg->chip, &wcove_irqchip, 0,
+					  handle_simple_irq, IRQ_TYPE_NONE);
 	if (ret) {
 		dev_err(dev, "Failed to add irqchip: %d\n", ret);
 		return ret;
@@ -446,6 +446,8 @@ static int wcove_gpio_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	gpiochip_set_nested_irqchip(&wg->chip, &wcove_irqchip, virq);
+
 	return 0;
 }
 
diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c
index 72a4b326fd0d..a3faefa44f68 100644
--- a/drivers/gpio/gpiolib-acpi.c
+++ b/drivers/gpio/gpiolib-acpi.c
@@ -468,7 +468,8 @@ static int acpi_gpio_property_lookup(struct fwnode_handle *fwnode,
 	int ret;
 
 	memset(&args, 0, sizeof(args));
-	ret = acpi_node_get_property_reference(fwnode, propname, index, &args);
+	ret = __acpi_node_get_property_reference(fwnode, propname, index, 3,
+						 &args);
 	if (ret) {
 		struct acpi_device *adev = to_acpi_device_node(fwnode);
 
@@ -483,13 +484,13 @@ static int acpi_gpio_property_lookup(struct fwnode_handle *fwnode,
 	 * on returned args.
 	 */
 	lookup->adev = args.adev;
-	if (args.nargs >= 2) {
-		lookup->index = args.args[0];
-		lookup->pin_index = args.args[1];
-		/* 3rd argument, if present is used to specify active_low. */
-		if (args.nargs >= 3)
-			lookup->active_low = !!args.args[2];
-	}
+	if (args.nargs != 3)
+		return -EPROTO;
+
+	lookup->index = args.args[0];
+	lookup->pin_index = args.args[1];
+	lookup->active_low = !!args.args[2];
+
 	return 0;
 }
 
@@ -859,6 +860,77 @@ static void acpi_gpiochip_free_regions(struct acpi_gpio_chip *achip)
 	}
 }
 
+static struct gpio_desc *acpi_gpiochip_parse_own_gpio(
+	struct acpi_gpio_chip *achip, struct fwnode_handle *fwnode,
+	const char **name, unsigned int *lflags, unsigned int *dflags)
+{
+	struct gpio_chip *chip = achip->chip;
+	struct gpio_desc *desc;
+	u32 gpios[2];
+	int ret;
+
+	*lflags = 0;
+	*dflags = 0;
+	*name = NULL;
+
+	ret = fwnode_property_read_u32_array(fwnode, "gpios", gpios,
+					     ARRAY_SIZE(gpios));
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	ret = acpi_gpiochip_pin_to_gpio_offset(chip->gpiodev, gpios[0]);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	desc = gpiochip_get_desc(chip, ret);
+	if (IS_ERR(desc))
+		return desc;
+
+	if (gpios[1])
+		*lflags |= GPIO_ACTIVE_LOW;
+
+	if (fwnode_property_present(fwnode, "input"))
+		*dflags |= GPIOD_IN;
+	else if (fwnode_property_present(fwnode, "output-low"))
+		*dflags |= GPIOD_OUT_LOW;
+	else if (fwnode_property_present(fwnode, "output-high"))
+		*dflags |= GPIOD_OUT_HIGH;
+	else
+		return ERR_PTR(-EINVAL);
+
+	fwnode_property_read_string(fwnode, "line-name", name);
+
+	return desc;
+}
+
+static void acpi_gpiochip_scan_gpios(struct acpi_gpio_chip *achip)
+{
+	struct gpio_chip *chip = achip->chip;
+	struct fwnode_handle *fwnode;
+
+	device_for_each_child_node(chip->parent, fwnode) {
+		unsigned int lflags, dflags;
+		struct gpio_desc *desc;
+		const char *name;
+		int ret;
+
+		if (!fwnode_property_present(fwnode, "gpio-hog"))
+			continue;
+
+		desc = acpi_gpiochip_parse_own_gpio(achip, fwnode, &name,
+						    &lflags, &dflags);
+		if (IS_ERR(desc))
+			continue;
+
+		ret = gpiod_hog(desc, name, lflags, dflags);
+		if (ret) {
+			dev_err(chip->parent, "Failed to hog GPIO\n");
+			fwnode_handle_put(fwnode);
+			return;
+		}
+	}
+}
+
 void acpi_gpiochip_add(struct gpio_chip *chip)
 {
 	struct acpi_gpio_chip *acpi_gpio;
@@ -889,7 +961,11 @@ void acpi_gpiochip_add(struct gpio_chip *chip)
 		return;
 	}
 
+	if (!chip->names)
+		devprop_gpiochip_set_names(chip);
+
 	acpi_gpiochip_request_regions(acpi_gpio);
+	acpi_gpiochip_scan_gpios(acpi_gpio);
 	acpi_walk_dep_device_list(handle);
 }
 
@@ -918,18 +994,27 @@ void acpi_gpiochip_remove(struct gpio_chip *chip)
 	kfree(acpi_gpio);
 }
 
-static unsigned int acpi_gpio_package_count(const union acpi_object *obj)
+static int acpi_gpio_package_count(const union acpi_object *obj)
 {
 	const union acpi_object *element = obj->package.elements;
 	const union acpi_object *end = element + obj->package.count;
 	unsigned int count = 0;
 
 	while (element < end) {
-		if (element->type == ACPI_TYPE_LOCAL_REFERENCE)
+		switch (element->type) {
+		case ACPI_TYPE_LOCAL_REFERENCE:
+			element += 3;
+			/* Fallthrough */
+		case ACPI_TYPE_INTEGER:
+			element++;
 			count++;
+			break;
 
-		element++;
+		default:
+			return -EPROTO;
+		}
 	}
+
 	return count;
 }
 
diff --git a/drivers/gpio/gpiolib-devprop.c b/drivers/gpio/gpiolib-devprop.c
new file mode 100644
index 000000000000..27f383bda7d9
--- /dev/null
+++ b/drivers/gpio/gpiolib-devprop.c
@@ -0,0 +1,67 @@
+/*
+ * Device property helpers for GPIO chips.
+ *
+ * Copyright (C) 2016, Intel Corporation
+ * Author: Mika Westerberg <mika.westerberg@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/property.h>
+#include <linux/slab.h>
+#include <linux/gpio/consumer.h>
+#include <linux/gpio/driver.h>
+
+#include "gpiolib.h"
+
+/**
+ * devprop_gpiochip_set_names - Set GPIO line names using device properties
+ * @chip: GPIO chip whose lines should be named, if possible
+ *
+ * Looks for device property "gpio-line-names" and if it exists assigns
+ * GPIO line names for the chip. The memory allocated for the assigned
+ * names belong to the underlying firmware node and should not be released
+ * by the caller.
+ */
+void devprop_gpiochip_set_names(struct gpio_chip *chip)
+{
+	struct gpio_device *gdev = chip->gpiodev;
+	const char **names;
+	int ret, i;
+
+	if (!chip->parent) {
+		dev_warn(&gdev->dev, "GPIO chip parent is NULL\n");
+		return;
+	}
+
+	ret = device_property_read_string_array(chip->parent, "gpio-line-names",
+						NULL, 0);
+	if (ret < 0)
+		return;
+
+	if (ret != gdev->ngpio) {
+		dev_warn(chip->parent,
+			 "names %d do not match number of GPIOs %d\n", ret,
+			 gdev->ngpio);
+		return;
+	}
+
+	names = kcalloc(gdev->ngpio, sizeof(*names), GFP_KERNEL);
+	if (!names)
+		return;
+
+	ret = device_property_read_string_array(chip->parent, "gpio-line-names",
+						names, gdev->ngpio);
+	if (ret < 0) {
+		dev_warn(chip->parent, "failed to read GPIO line names\n");
+		kfree(names);
+		return;
+	}
+
+	for (i = 0; i < gdev->ngpio; i++)
+		gdev->descs[i].name = names[i];
+
+	kfree(names);
+}
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index 193f15d50bba..92b185f19232 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -226,51 +226,6 @@ static struct gpio_desc *of_parse_own_gpio(struct device_node *np,
 }
 
 /**
- * of_gpiochip_set_names() - set up the names of the lines
- * @chip: GPIO chip whose lines should be named, if possible
- */
-static void of_gpiochip_set_names(struct gpio_chip *gc)
-{
-	struct gpio_device *gdev = gc->gpiodev;
-	struct device_node *np = gc->of_node;
-	int i;
-	int nstrings;
-
-	nstrings = of_property_count_strings(np, "gpio-line-names");
-	if (nstrings <= 0)
-		/* Lines names not present */
-		return;
-
-	/* This is normally not what you want */
-	if (gdev->ngpio != nstrings)
-		dev_info(&gdev->dev, "gpio-line-names specifies %d line "
-			 "names but there are %d lines on the chip\n",
-			 nstrings, gdev->ngpio);
-
-	/*
-	 * Make sure to not index beyond the end of the number of descriptors
-	 * of the GPIO device.
-	 */
-	for (i = 0; i < gdev->ngpio; i++) {
-		const char *name;
-		int ret;
-
-		ret = of_property_read_string_index(np,
-						    "gpio-line-names",
-						    i,
-						    &name);
-		if (ret) {
-			if (ret != -ENODATA)
-                                dev_err(&gdev->dev,
-                                        "unable to name line %d: %d\n",
-                                        i, ret);
-			break;
-		}
-		gdev->descs[i].name = name;
-	}
-}
-
-/**
  * of_gpiochip_scan_gpios - Scan gpio-controller for gpio definitions
  * @chip:	gpio chip to act on
  *
@@ -296,8 +251,10 @@ static int of_gpiochip_scan_gpios(struct gpio_chip *chip)
 			continue;
 
 		ret = gpiod_hog(desc, name, lflags, dflags);
-		if (ret < 0)
+		if (ret < 0) {
+			of_node_put(np);
 			return ret;
+		}
 	}
 
 	return 0;
@@ -526,7 +483,7 @@ int of_gpiochip_add(struct gpio_chip *chip)
 
 	/* If the chip defines names itself, these take precedence */
 	if (!chip->names)
-		of_gpiochip_set_names(chip);
+		devprop_gpiochip_set_names(chip);
 
 	of_node_get(chip->of_node);
 
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index 868128a676ba..f4c26c7826cd 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -986,7 +986,8 @@ static int gpio_chrdev_open(struct inode *inode, struct file *filp)
 		return -ENODEV;
 	get_device(&gdev->dev);
 	filp->private_data = gdev;
-	return 0;
+
+	return nonseekable_open(inode, filp);
 }
 
 /**
@@ -1011,7 +1012,7 @@ static const struct file_operations gpio_fileops = {
 	.release = gpio_chrdev_release,
 	.open = gpio_chrdev_open,
 	.owner = THIS_MODULE,
-	.llseek = noop_llseek,
+	.llseek = no_llseek,
 	.unlocked_ioctl = gpio_ioctl,
 #ifdef CONFIG_COMPAT
 	.compat_ioctl = gpio_ioctl_compat,
@@ -1512,7 +1513,7 @@ static bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gpiochip,
 }
 
 /**
- * gpiochip_set_chained_irqchip() - sets a chained irqchip to a gpiochip
+ * gpiochip_set_cascaded_irqchip() - connects a cascaded irqchip to a gpiochip
  * @gpiochip: the gpiochip to set the irqchip chain to
  * @irqchip: the irqchip to chain to the gpiochip
  * @parent_irq: the irq number corresponding to the parent IRQ for this
@@ -1521,10 +1522,10 @@ static bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gpiochip,
  * coming out of the gpiochip. If the interrupt is nested rather than
  * cascaded, pass NULL in this handler argument
  */
-void gpiochip_set_chained_irqchip(struct gpio_chip *gpiochip,
-				  struct irq_chip *irqchip,
-				  int parent_irq,
-				  irq_flow_handler_t parent_handler)
+static void gpiochip_set_cascaded_irqchip(struct gpio_chip *gpiochip,
+					  struct irq_chip *irqchip,
+					  int parent_irq,
+					  irq_flow_handler_t parent_handler)
 {
 	unsigned int offset;
 
@@ -1548,7 +1549,7 @@ void gpiochip_set_chained_irqchip(struct gpio_chip *gpiochip,
 		irq_set_chained_handler_and_data(parent_irq, parent_handler,
 						 gpiochip);
 
-		gpiochip->irq_parent = parent_irq;
+		gpiochip->irq_chained_parent = parent_irq;
 	}
 
 	/* Set the parent IRQ for all affected IRQs */
@@ -1559,9 +1560,48 @@ void gpiochip_set_chained_irqchip(struct gpio_chip *gpiochip,
 			       parent_irq);
 	}
 }
+
+/**
+ * gpiochip_set_chained_irqchip() - connects a chained irqchip to a gpiochip
+ * @gpiochip: the gpiochip to set the irqchip chain to
+ * @irqchip: the irqchip to chain to the gpiochip
+ * @parent_irq: the irq number corresponding to the parent IRQ for this
+ * chained irqchip
+ * @parent_handler: the parent interrupt handler for the accumulated IRQ
+ * coming out of the gpiochip. If the interrupt is nested rather than
+ * cascaded, pass NULL in this handler argument
+ */
+void gpiochip_set_chained_irqchip(struct gpio_chip *gpiochip,
+				  struct irq_chip *irqchip,
+				  int parent_irq,
+				  irq_flow_handler_t parent_handler)
+{
+	gpiochip_set_cascaded_irqchip(gpiochip, irqchip, parent_irq,
+				      parent_handler);
+}
 EXPORT_SYMBOL_GPL(gpiochip_set_chained_irqchip);
 
 /**
+ * gpiochip_set_nested_irqchip() - connects a nested irqchip to a gpiochip
+ * @gpiochip: the gpiochip to set the irqchip nested handler to
+ * @irqchip: the irqchip to nest to the gpiochip
+ * @parent_irq: the irq number corresponding to the parent IRQ for this
+ * nested irqchip
+ */
+void gpiochip_set_nested_irqchip(struct gpio_chip *gpiochip,
+				 struct irq_chip *irqchip,
+				 int parent_irq)
+{
+	if (!gpiochip->irq_nested) {
+		chip_err(gpiochip, "tried to nest a chained gpiochip\n");
+		return;
+	}
+	gpiochip_set_cascaded_irqchip(gpiochip, irqchip, parent_irq,
+				      NULL);
+}
+EXPORT_SYMBOL_GPL(gpiochip_set_nested_irqchip);
+
+/**
  * gpiochip_irq_map() - maps an IRQ into a GPIO irqchip
  * @d: the irqdomain used by this irqchip
  * @irq: the global irq number used by this GPIO irqchip irq
@@ -1583,8 +1623,8 @@ static int gpiochip_irq_map(struct irq_domain *d, unsigned int irq,
 	 */
 	irq_set_lockdep_class(irq, chip->lock_key);
 	irq_set_chip_and_handler(irq, chip->irqchip, chip->irq_handler);
-	/* Chips that can sleep need nested thread handlers */
-	if (chip->can_sleep && !chip->irq_not_threaded)
+	/* Chips that use nested thread handlers have them marked */
+	if (chip->irq_nested)
 		irq_set_nested_thread(irq, 1);
 	irq_set_noprobe(irq);
 
@@ -1602,7 +1642,7 @@ static void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq)
 {
 	struct gpio_chip *chip = d->host_data;
 
-	if (chip->can_sleep)
+	if (chip->irq_nested)
 		irq_set_nested_thread(irq, 0);
 	irq_set_chip_and_handler(irq, NULL, NULL);
 	irq_set_chip_data(irq, NULL);
@@ -1657,9 +1697,9 @@ static void gpiochip_irqchip_remove(struct gpio_chip *gpiochip)
 
 	acpi_gpiochip_free_interrupts(gpiochip);
 
-	if (gpiochip->irq_parent) {
-		irq_set_chained_handler(gpiochip->irq_parent, NULL);
-		irq_set_handler_data(gpiochip->irq_parent, NULL);
+	if (gpiochip->irq_chained_parent) {
+		irq_set_chained_handler(gpiochip->irq_chained_parent, NULL);
+		irq_set_handler_data(gpiochip->irq_chained_parent, NULL);
 	}
 
 	/* Remove all IRQ mappings and delete the domain */
@@ -1683,7 +1723,7 @@ static void gpiochip_irqchip_remove(struct gpio_chip *gpiochip)
 }
 
 /**
- * gpiochip_irqchip_add() - adds an irqchip to a gpiochip
+ * _gpiochip_irqchip_add() - adds an irqchip to a gpiochip
  * @gpiochip: the gpiochip to add the irqchip to
  * @irqchip: the irqchip to add to the gpiochip
  * @first_irq: if not dynamically assigned, the base (first) IRQ to
@@ -1691,6 +1731,8 @@ static void gpiochip_irqchip_remove(struct gpio_chip *gpiochip)
  * @handler: the irq handler to use (often a predefined irq core function)
  * @type: the default type for IRQs on this irqchip, pass IRQ_TYPE_NONE
  * to have the core avoid setting up any default type in the hardware.
+ * @nested: whether this is a nested irqchip calling handle_nested_irq()
+ * in its IRQ handler
  * @lock_key: lockdep class
  *
  * This function closely associates a certain irqchip with a certain
@@ -1712,6 +1754,7 @@ int _gpiochip_irqchip_add(struct gpio_chip *gpiochip,
 			  unsigned int first_irq,
 			  irq_flow_handler_t handler,
 			  unsigned int type,
+			  bool nested,
 			  struct lock_class_key *lock_key)
 {
 	struct device_node *of_node;
@@ -1726,6 +1769,7 @@ int _gpiochip_irqchip_add(struct gpio_chip *gpiochip,
 		pr_err("missing gpiochip .dev parent pointer\n");
 		return -EINVAL;
 	}
+	gpiochip->irq_nested = nested;
 	of_node = gpiochip->parent->of_node;
 #ifdef CONFIG_OF_GPIO
 	/*
@@ -2223,6 +2267,7 @@ EXPORT_SYMBOL_GPL(gpiod_direction_input);
 static int _gpiod_direction_output_raw(struct gpio_desc *desc, int value)
 {
 	struct gpio_chip *gc = desc->gdev->chip;
+	int val = !!value;
 	int ret;
 
 	/* GPIOs used for IRQs shall not be set as output */
@@ -2242,7 +2287,7 @@ static int _gpiod_direction_output_raw(struct gpio_desc *desc, int value)
 				goto set_output_value;
 		}
 		/* Emulate open drain by not actively driving the line high */
-		if (value)
+		if (val)
 			return gpiod_direction_input(desc);
 	}
 	else if (test_bit(FLAG_OPEN_SOURCE, &desc->flags)) {
@@ -2253,7 +2298,7 @@ static int _gpiod_direction_output_raw(struct gpio_desc *desc, int value)
 				goto set_output_value;
 		}
 		/* Emulate open source by not actively driving the line low */
-		if (!value)
+		if (!val)
 			return gpiod_direction_input(desc);
 	} else {
 		/* Make sure to disable open drain/source hardware, if any */
@@ -2271,10 +2316,10 @@ set_output_value:
 		return -EIO;
 	}
 
-	ret = gc->direction_output(gc, gpio_chip_hwgpio(desc), value);
+	ret = gc->direction_output(gc, gpio_chip_hwgpio(desc), val);
 	if (!ret)
 		set_bit(FLAG_IS_OUT, &desc->flags);
-	trace_gpio_value(desc_to_gpio(desc), 0, value);
+	trace_gpio_value(desc_to_gpio(desc), 0, val);
 	trace_gpio_direction(desc_to_gpio(desc), 0, ret);
 	return ret;
 }
@@ -2314,6 +2359,8 @@ int gpiod_direction_output(struct gpio_desc *desc, int value)
 	VALIDATE_DESC(desc);
 	if (test_bit(FLAG_ACTIVE_LOW, &desc->flags))
 		value = !value;
+	else
+		value = !!value;
 	return _gpiod_direction_output_raw(desc, value);
 }
 EXPORT_SYMBOL_GPL(gpiod_direction_output);
@@ -2758,6 +2805,15 @@ int gpiochip_lock_as_irq(struct gpio_chip *chip, unsigned int offset)
 	}
 
 	set_bit(FLAG_USED_AS_IRQ, &desc->flags);
+
+	/*
+	 * If the consumer has not set up a label (such as when the
+	 * IRQ is referenced from .to_irq()) we set up a label here
+	 * so it is clear this is used as an interrupt.
+	 */
+	if (!desc->label)
+		desc_set_label(desc, "interrupt");
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(gpiochip_lock_as_irq);
@@ -2772,10 +2828,17 @@ EXPORT_SYMBOL_GPL(gpiochip_lock_as_irq);
  */
 void gpiochip_unlock_as_irq(struct gpio_chip *chip, unsigned int offset)
 {
-	if (offset >= chip->ngpio)
+	struct gpio_desc *desc;
+
+	desc = gpiochip_get_desc(chip, offset);
+	if (IS_ERR(desc))
 		return;
 
-	clear_bit(FLAG_USED_AS_IRQ, &chip->gpiodev->descs[offset].flags);
+	clear_bit(FLAG_USED_AS_IRQ, &desc->flags);
+
+	/* If we only had this marking, erase it */
+	if (desc->label && !strcmp(desc->label, "interrupt"))
+		desc_set_label(desc, NULL);
 }
 EXPORT_SYMBOL_GPL(gpiochip_unlock_as_irq);
 
@@ -3170,7 +3233,7 @@ static int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id,
 	/* Process flags */
 	if (dflags & GPIOD_FLAGS_BIT_DIR_OUT)
 		status = gpiod_direction_output(desc,
-					      dflags & GPIOD_FLAGS_BIT_DIR_VAL);
+				!!(dflags & GPIOD_FLAGS_BIT_DIR_VAL));
 	else
 		status = gpiod_direction_input(desc);
 
diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h
index 346fbda39220..d10eaf520860 100644
--- a/drivers/gpio/gpiolib.h
+++ b/drivers/gpio/gpiolib.h
@@ -209,6 +209,8 @@ static int __maybe_unused gpio_chip_hwgpio(const struct gpio_desc *desc)
 	return desc - &desc->gdev->descs[0];
 }
 
+void devprop_gpiochip_set_names(struct gpio_chip *chip);
+
 /* With descriptor prefix */
 
 #define gpiod_emerg(desc, fmt, ...)					       \
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 483059a22b1b..ebfe8404c25f 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -12,6 +12,7 @@ menuconfig DRM
 	select I2C
 	select I2C_ALGOBIT
 	select DMA_SHARED_BUFFER
+	select SYNC_FILE
 	help
 	  Kernel-level support for the Direct Rendering Infrastructure (DRI)
 	  introduced in XFree86 4.0. If you say Y here, you need to select
@@ -33,6 +34,20 @@ config DRM_DP_AUX_CHARDEV
 	  read and write values to arbitrary DPCD registers on the DP aux
 	  channel.
 
+config DRM_DEBUG_MM
+	bool "Insert extra checks and debug info into the DRM range managers"
+	default n
+	depends on DRM=y
+	depends on STACKTRACE_SUPPORT
+	select STACKDEPOT
+	help
+	  Enable allocation tracking of memory manager and leak detection on
+	  shutdown.
+
+	  Recommended for driver developers only.
+
+	  If in doubt, say "N".
+
 config DRM_KMS_HELPER
 	tristate
 	depends on DRM
@@ -223,6 +238,12 @@ source "drivers/gpu/drm/hisilicon/Kconfig"
 
 source "drivers/gpu/drm/mediatek/Kconfig"
 
+source "drivers/gpu/drm/zte/Kconfig"
+
+source "drivers/gpu/drm/mxsfb/Kconfig"
+
+source "drivers/gpu/drm/meson/Kconfig"
+
 # Keep legacy drivers last
 
 menuconfig DRM_LEGACY
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 25c720454017..b9ae4280de9d 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -9,13 +9,14 @@ drm-y       :=	drm_auth.o drm_bufs.o drm_cache.o \
 		drm_scatter.o drm_pci.o \
 		drm_platform.o drm_sysfs.o drm_hashtab.o drm_mm.o \
 		drm_crtc.o drm_fourcc.o drm_modes.o drm_edid.o \
-		drm_info.o drm_debugfs.o drm_encoder_slave.o \
+		drm_info.o drm_encoder_slave.o \
 		drm_trace_points.o drm_global.o drm_prime.o \
 		drm_rect.o drm_vma_manager.o drm_flip_work.o \
 		drm_modeset_lock.o drm_atomic.o drm_bridge.o \
 		drm_framebuffer.o drm_connector.o drm_blend.o \
 		drm_encoder.o drm_mode_object.o drm_property.o \
-		drm_plane.o drm_color_mgmt.o
+		drm_plane.o drm_color_mgmt.o drm_print.o \
+		drm_dumb_buffers.o drm_mode_config.o
 
 drm-$(CONFIG_COMPAT) += drm_ioc32.o
 drm-$(CONFIG_DRM_GEM_CMA_HELPER) += drm_gem_cma_helper.o
@@ -23,6 +24,7 @@ drm-$(CONFIG_PCI) += ati_pcigart.o
 drm-$(CONFIG_DRM_PANEL) += drm_panel.o
 drm-$(CONFIG_OF) += drm_of.o
 drm-$(CONFIG_AGP) += drm_agpsupport.o
+drm-$(CONFIG_DEBUG_FS) += drm_debugfs.o drm_debugfs_crc.o
 
 drm_kms_helper-y := drm_crtc_helper.o drm_dp_helper.o drm_probe_helper.o \
 		drm_plane_helper.o drm_dp_mst_topology.o drm_atomic_helper.o \
@@ -79,6 +81,7 @@ obj-$(CONFIG_DRM_TEGRA) += tegra/
 obj-$(CONFIG_DRM_STI) += sti/
 obj-$(CONFIG_DRM_IMX) += imx/
 obj-$(CONFIG_DRM_MEDIATEK) += mediatek/
+obj-$(CONFIG_DRM_MESON)	+= meson/
 obj-y			+= i2c/
 obj-y			+= panel/
 obj-y			+= bridge/
@@ -86,3 +89,5 @@ obj-$(CONFIG_DRM_FSL_DCU) += fsl-dcu/
 obj-$(CONFIG_DRM_ETNAVIV) += etnaviv/
 obj-$(CONFIG_DRM_ARCPGU)+= arc/
 obj-y			+= hisilicon/
+obj-$(CONFIG_DRM_ZTE)	+= zte/
+obj-$(CONFIG_DRM_MXSFB)	+= mxsfb/
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 248a05d02917..41bd2bf28f4c 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -24,7 +24,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 	atombios_encoders.o amdgpu_sa.o atombios_i2c.o \
 	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
 	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
-	amdgpu_gtt_mgr.o
+	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
diff --git a/drivers/gpu/drm/amd/amdgpu/ObjectID.h b/drivers/gpu/drm/amd/amdgpu/ObjectID.h
index b8d66670bb17..06192698bd96 100644
--- a/drivers/gpu/drm/amd/amdgpu/ObjectID.h
+++ b/drivers/gpu/drm/amd/amdgpu/ObjectID.h
@@ -90,7 +90,6 @@
 #define ENCODER_OBJECT_ID_INTERNAL_VCE            0x24
 #define ENCODER_OBJECT_ID_INTERNAL_UNIPHY3        0x25
 #define ENCODER_OBJECT_ID_INTERNAL_AMCLK          0x27
-#define ENCODER_OBJECT_ID_VIRTUAL                 0x28
 
 #define ENCODER_OBJECT_ID_GENERAL_EXTERNAL_DVO    0xFF
 
@@ -120,7 +119,6 @@
 #define CONNECTOR_OBJECT_ID_eDP                   0x14
 #define CONNECTOR_OBJECT_ID_MXM                   0x15
 #define CONNECTOR_OBJECT_ID_LVDS_eDP              0x16
-#define CONNECTOR_OBJECT_ID_VIRTUAL               0x17
 
 /* deleted */
 
@@ -149,7 +147,6 @@
 #define GRAPH_OBJECT_ENUM_ID5                     0x05
 #define GRAPH_OBJECT_ENUM_ID6                     0x06
 #define GRAPH_OBJECT_ENUM_ID7                     0x07
-#define GRAPH_OBJECT_ENUM_VIRTUAL                 0x08
 
 /****************************************************/
 /* Graphics Object ID Bit definition                */
@@ -411,10 +408,6 @@
                                                   GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
                                                   ENCODER_OBJECT_ID_HDMI_ANX9805 << OBJECT_ID_SHIFT)
 
-#define ENCODER_VIRTUAL_ENUM_VIRTUAL            ( GRAPH_OBJECT_TYPE_ENCODER << OBJECT_TYPE_SHIFT |\
-                                                  GRAPH_OBJECT_ENUM_VIRTUAL << ENUM_ID_SHIFT |\
-                                                  ENCODER_OBJECT_ID_VIRTUAL << OBJECT_ID_SHIFT)
-
 /****************************************************/
 /* Connector Object ID definition - Shared with BIOS */
 /****************************************************/
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 05c2850c04b0..63208e5c1588 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -34,7 +34,7 @@
 #include <linux/kref.h>
 #include <linux/interval_tree.h>
 #include <linux/hashtable.h>
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 
 #include <ttm/ttm_bo_api.h>
 #include <ttm/ttm_bo_driver.h>
@@ -53,7 +53,11 @@
 #include "amdgpu_ucode.h"
 #include "amdgpu_ttm.h"
 #include "amdgpu_gds.h"
+#include "amdgpu_sync.h"
+#include "amdgpu_ring.h"
+#include "amdgpu_vm.h"
 #include "amd_powerplay.h"
+#include "amdgpu_dpm.h"
 #include "amdgpu_acp.h"
 
 #include "gpu_scheduler.h"
@@ -88,15 +92,16 @@ extern int amdgpu_vm_debug;
 extern int amdgpu_sched_jobs;
 extern int amdgpu_sched_hw_submission;
 extern int amdgpu_powerplay;
-extern int amdgpu_powercontainment;
+extern int amdgpu_no_evict;
+extern int amdgpu_direct_gma_size;
 extern unsigned amdgpu_pcie_gen_cap;
 extern unsigned amdgpu_pcie_lane_cap;
 extern unsigned amdgpu_cg_mask;
 extern unsigned amdgpu_pg_mask;
 extern char *amdgpu_disable_cu;
-extern int amdgpu_sclk_deep_sleep_en;
 extern char *amdgpu_virtual_display;
 extern unsigned amdgpu_pp_feature_mask;
+extern int amdgpu_vram_page_split;
 
 #define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS	        3000
 #define AMDGPU_MAX_USEC_TIMEOUT			100000	/* 100 ms */
@@ -107,12 +112,6 @@ extern unsigned amdgpu_pp_feature_mask;
 #define AMDGPUFB_CONN_LIMIT			4
 #define AMDGPU_BIOS_NUM_SCRATCH			8
 
-/* max number of rings */
-#define AMDGPU_MAX_RINGS			16
-#define AMDGPU_MAX_GFX_RINGS			1
-#define AMDGPU_MAX_COMPUTE_RINGS		8
-#define AMDGPU_MAX_VCE_RINGS			3
-
 /* max number of IP instances */
 #define AMDGPU_MAX_SDMA_INSTANCES		2
 
@@ -152,8 +151,6 @@ extern unsigned amdgpu_pp_feature_mask;
 
 struct amdgpu_device;
 struct amdgpu_ib;
-struct amdgpu_vm;
-struct amdgpu_ring;
 struct amdgpu_cs_parser;
 struct amdgpu_job;
 struct amdgpu_irq_src;
@@ -198,21 +195,38 @@ int amdgpu_wait_for_idle(struct amdgpu_device *adev,
 bool amdgpu_is_idle(struct amdgpu_device *adev,
 		    enum amd_ip_block_type block_type);
 
+#define AMDGPU_MAX_IP_NUM 16
+
+struct amdgpu_ip_block_status {
+	bool valid;
+	bool sw;
+	bool hw;
+	bool late_initialized;
+	bool hang;
+};
+
 struct amdgpu_ip_block_version {
-	enum amd_ip_block_type type;
-	u32 major;
-	u32 minor;
-	u32 rev;
+	const enum amd_ip_block_type type;
+	const u32 major;
+	const u32 minor;
+	const u32 rev;
 	const struct amd_ip_funcs *funcs;
 };
 
+struct amdgpu_ip_block {
+	struct amdgpu_ip_block_status status;
+	const struct amdgpu_ip_block_version *version;
+};
+
 int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
 				enum amd_ip_block_type type,
 				u32 major, u32 minor);
 
-const struct amdgpu_ip_block_version * amdgpu_get_ip_block(
-					struct amdgpu_device *adev,
-					enum amd_ip_block_type type);
+struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
+					     enum amd_ip_block_type type);
+
+int amdgpu_ip_block_add(struct amdgpu_device *adev,
+			const struct amdgpu_ip_block_version *ip_block_version);
 
 /* provided by hw blocks that can move/clear data.  e.g., gfx or sdma */
 struct amdgpu_buffer_funcs {
@@ -286,47 +300,6 @@ struct amdgpu_ih_funcs {
 	void (*set_rptr)(struct amdgpu_device *adev);
 };
 
-/* provided by hw blocks that expose a ring buffer for commands */
-struct amdgpu_ring_funcs {
-	/* ring read/write ptr handling */
-	u32 (*get_rptr)(struct amdgpu_ring *ring);
-	u32 (*get_wptr)(struct amdgpu_ring *ring);
-	void (*set_wptr)(struct amdgpu_ring *ring);
-	/* validating and patching of IBs */
-	int (*parse_cs)(struct amdgpu_cs_parser *p, uint32_t ib_idx);
-	/* command emit functions */
-	void (*emit_ib)(struct amdgpu_ring *ring,
-			struct amdgpu_ib *ib,
-			unsigned vm_id, bool ctx_switch);
-	void (*emit_fence)(struct amdgpu_ring *ring, uint64_t addr,
-			   uint64_t seq, unsigned flags);
-	void (*emit_pipeline_sync)(struct amdgpu_ring *ring);
-	void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vm_id,
-			      uint64_t pd_addr);
-	void (*emit_hdp_flush)(struct amdgpu_ring *ring);
-	void (*emit_hdp_invalidate)(struct amdgpu_ring *ring);
-	void (*emit_gds_switch)(struct amdgpu_ring *ring, uint32_t vmid,
-				uint32_t gds_base, uint32_t gds_size,
-				uint32_t gws_base, uint32_t gws_size,
-				uint32_t oa_base, uint32_t oa_size);
-	/* testing functions */
-	int (*test_ring)(struct amdgpu_ring *ring);
-	int (*test_ib)(struct amdgpu_ring *ring, long timeout);
-	/* insert NOP packets */
-	void (*insert_nop)(struct amdgpu_ring *ring, uint32_t count);
-	/* pad the indirect buffer to the necessary number of dw */
-	void (*pad_ib)(struct amdgpu_ring *ring, struct amdgpu_ib *ib);
-	unsigned (*init_cond_exec)(struct amdgpu_ring *ring);
-	void (*patch_cond_exec)(struct amdgpu_ring *ring, unsigned offset);
-	/* note usage for clock and power gating */
-	void (*begin_use)(struct amdgpu_ring *ring);
-	void (*end_use)(struct amdgpu_ring *ring);
-	void (*emit_switch_buffer) (struct amdgpu_ring *ring);
-	void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags);
-	unsigned (*get_emit_ib_size) (struct amdgpu_ring *ring);
-	unsigned (*get_dma_frame_size) (struct amdgpu_ring *ring);
-};
-
 /*
  * BIOS.
  */
@@ -364,47 +337,6 @@ struct amdgpu_clock {
 };
 
 /*
- * Fences.
- */
-struct amdgpu_fence_driver {
-	uint64_t			gpu_addr;
-	volatile uint32_t		*cpu_addr;
-	/* sync_seq is protected by ring emission lock */
-	uint32_t			sync_seq;
-	atomic_t			last_seq;
-	bool				initialized;
-	struct amdgpu_irq_src		*irq_src;
-	unsigned			irq_type;
-	struct timer_list		fallback_timer;
-	unsigned			num_fences_mask;
-	spinlock_t			lock;
-	struct fence			**fences;
-};
-
-/* some special values for the owner field */
-#define AMDGPU_FENCE_OWNER_UNDEFINED	((void*)0ul)
-#define AMDGPU_FENCE_OWNER_VM		((void*)1ul)
-
-#define AMDGPU_FENCE_FLAG_64BIT         (1 << 0)
-#define AMDGPU_FENCE_FLAG_INT           (1 << 1)
-
-int amdgpu_fence_driver_init(struct amdgpu_device *adev);
-void amdgpu_fence_driver_fini(struct amdgpu_device *adev);
-void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev);
-
-int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
-				  unsigned num_hw_submission);
-int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
-				   struct amdgpu_irq_src *irq_src,
-				   unsigned irq_type);
-void amdgpu_fence_driver_suspend(struct amdgpu_device *adev);
-void amdgpu_fence_driver_resume(struct amdgpu_device *adev);
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct fence **fence);
-void amdgpu_fence_process(struct amdgpu_ring *ring);
-int amdgpu_fence_wait_empty(struct amdgpu_ring *ring);
-unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring);
-
-/*
  * BO.
  */
 struct amdgpu_bo_list_entry {
@@ -427,7 +359,7 @@ struct amdgpu_bo_va_mapping {
 struct amdgpu_bo_va {
 	/* protected by bo being reserved */
 	struct list_head		bo_list;
-	struct fence		        *last_pt_update;
+	struct dma_fence	        *last_pt_update;
 	unsigned			ref_count;
 
 	/* protected by vm mutex and spinlock */
@@ -465,7 +397,6 @@ struct amdgpu_bo {
 	 */
 	struct list_head		va;
 	/* Constant after initialization */
-	struct amdgpu_device		*adev;
 	struct drm_gem_object		gem_base;
 	struct amdgpu_bo		*parent;
 	struct amdgpu_bo		*shadow;
@@ -544,7 +475,7 @@ struct amdgpu_sa_bo {
 	struct amdgpu_sa_manager	*manager;
 	unsigned			soffset;
 	unsigned			eoffset;
-	struct fence		        *fence;
+	struct dma_fence	        *fence;
 };
 
 /*
@@ -562,27 +493,6 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 int amdgpu_mode_dumb_mmap(struct drm_file *filp,
 			  struct drm_device *dev,
 			  uint32_t handle, uint64_t *offset_p);
-/*
- * Synchronization
- */
-struct amdgpu_sync {
-	DECLARE_HASHTABLE(fences, 4);
-	struct fence	        *last_vm_update;
-};
-
-void amdgpu_sync_create(struct amdgpu_sync *sync);
-int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
-		      struct fence *f);
-int amdgpu_sync_resv(struct amdgpu_device *adev,
-		     struct amdgpu_sync *sync,
-		     struct reservation_object *resv,
-		     void *owner);
-struct fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
-				     struct amdgpu_ring *ring);
-struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync);
-void amdgpu_sync_free(struct amdgpu_sync *sync);
-int amdgpu_sync_init(void);
-void amdgpu_sync_fini(void);
 int amdgpu_fence_slab_init(void);
 void amdgpu_fence_slab_fini(void);
 
@@ -704,10 +614,10 @@ struct amdgpu_flip_work {
 	uint64_t			base;
 	struct drm_pending_vblank_event *event;
 	struct amdgpu_bo		*old_abo;
-	struct fence			*excl;
+	struct dma_fence		*excl;
 	unsigned			shared_count;
-	struct fence			**shared;
-	struct fence_cb			cb;
+	struct dma_fence		**shared;
+	struct dma_fence_cb		cb;
 	bool				async;
 };
 
@@ -724,14 +634,6 @@ struct amdgpu_ib {
 	uint32_t			flags;
 };
 
-enum amdgpu_ring_type {
-	AMDGPU_RING_TYPE_GFX,
-	AMDGPU_RING_TYPE_COMPUTE,
-	AMDGPU_RING_TYPE_SDMA,
-	AMDGPU_RING_TYPE_UVD,
-	AMDGPU_RING_TYPE_VCE
-};
-
 extern const struct amd_sched_backend_ops amdgpu_sched_ops;
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
@@ -743,214 +645,7 @@ void amdgpu_job_free_resources(struct amdgpu_job *job);
 void amdgpu_job_free(struct amdgpu_job *job);
 int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
 		      struct amd_sched_entity *entity, void *owner,
-		      struct fence **f);
-
-struct amdgpu_ring {
-	struct amdgpu_device		*adev;
-	const struct amdgpu_ring_funcs	*funcs;
-	struct amdgpu_fence_driver	fence_drv;
-	struct amd_gpu_scheduler	sched;
-
-	struct amdgpu_bo	*ring_obj;
-	volatile uint32_t	*ring;
-	unsigned		rptr_offs;
-	unsigned		wptr;
-	unsigned		wptr_old;
-	unsigned		ring_size;
-	unsigned		max_dw;
-	int			count_dw;
-	uint64_t		gpu_addr;
-	uint32_t		align_mask;
-	uint32_t		ptr_mask;
-	bool			ready;
-	u32			nop;
-	u32			idx;
-	u32			me;
-	u32			pipe;
-	u32			queue;
-	struct amdgpu_bo	*mqd_obj;
-	u32			doorbell_index;
-	bool			use_doorbell;
-	unsigned		wptr_offs;
-	unsigned		fence_offs;
-	uint64_t		current_ctx;
-	enum amdgpu_ring_type	type;
-	char			name[16];
-	unsigned		cond_exe_offs;
-	u64			cond_exe_gpu_addr;
-	volatile u32		*cond_exe_cpu_addr;
-#if defined(CONFIG_DEBUG_FS)
-	struct dentry *ent;
-#endif
-};
-
-/*
- * VM
- */
-
-/* maximum number of VMIDs */
-#define AMDGPU_NUM_VM	16
-
-/* Maximum number of PTEs the hardware can write with one command */
-#define AMDGPU_VM_MAX_UPDATE_SIZE	0x3FFFF
-
-/* number of entries in page table */
-#define AMDGPU_VM_PTE_COUNT (1 << amdgpu_vm_block_size)
-
-/* PTBs (Page Table Blocks) need to be aligned to 32K */
-#define AMDGPU_VM_PTB_ALIGN_SIZE   32768
-
-/* LOG2 number of continuous pages for the fragment field */
-#define AMDGPU_LOG2_PAGES_PER_FRAG 4
-
-#define AMDGPU_PTE_VALID	(1 << 0)
-#define AMDGPU_PTE_SYSTEM	(1 << 1)
-#define AMDGPU_PTE_SNOOPED	(1 << 2)
-
-/* VI only */
-#define AMDGPU_PTE_EXECUTABLE	(1 << 4)
-
-#define AMDGPU_PTE_READABLE	(1 << 5)
-#define AMDGPU_PTE_WRITEABLE	(1 << 6)
-
-#define AMDGPU_PTE_FRAG(x)	((x & 0x1f) << 7)
-
-/* How to programm VM fault handling */
-#define AMDGPU_VM_FAULT_STOP_NEVER	0
-#define AMDGPU_VM_FAULT_STOP_FIRST	1
-#define AMDGPU_VM_FAULT_STOP_ALWAYS	2
-
-struct amdgpu_vm_pt {
-	struct amdgpu_bo_list_entry	entry;
-	uint64_t			addr;
-	uint64_t			shadow_addr;
-};
-
-struct amdgpu_vm {
-	/* tree of virtual addresses mapped */
-	struct rb_root		va;
-
-	/* protecting invalidated */
-	spinlock_t		status_lock;
-
-	/* BOs moved, but not yet updated in the PT */
-	struct list_head	invalidated;
-
-	/* BOs cleared in the PT because of a move */
-	struct list_head	cleared;
-
-	/* BO mappings freed, but not yet updated in the PT */
-	struct list_head	freed;
-
-	/* contains the page directory */
-	struct amdgpu_bo	*page_directory;
-	unsigned		max_pde_used;
-	struct fence		*page_directory_fence;
-	uint64_t		last_eviction_counter;
-
-	/* array of page tables, one for each page directory entry */
-	struct amdgpu_vm_pt	*page_tables;
-
-	/* for id and flush management per ring */
-	struct amdgpu_vm_id	*ids[AMDGPU_MAX_RINGS];
-
-	/* protecting freed */
-	spinlock_t		freed_lock;
-
-	/* Scheduler entity for page table updates */
-	struct amd_sched_entity	entity;
-
-	/* client id */
-	u64                     client_id;
-};
-
-struct amdgpu_vm_id {
-	struct list_head	list;
-	struct fence		*first;
-	struct amdgpu_sync	active;
-	struct fence		*last_flush;
-	atomic64_t		owner;
-
-	uint64_t		pd_gpu_addr;
-	/* last flushed PD/PT update */
-	struct fence		*flushed_updates;
-
-	uint32_t                current_gpu_reset_count;
-
-	uint32_t		gds_base;
-	uint32_t		gds_size;
-	uint32_t		gws_base;
-	uint32_t		gws_size;
-	uint32_t		oa_base;
-	uint32_t		oa_size;
-};
-
-struct amdgpu_vm_manager {
-	/* Handling of VMIDs */
-	struct mutex				lock;
-	unsigned				num_ids;
-	struct list_head			ids_lru;
-	struct amdgpu_vm_id			ids[AMDGPU_NUM_VM];
-
-	/* Handling of VM fences */
-	u64					fence_context;
-	unsigned				seqno[AMDGPU_MAX_RINGS];
-
-	uint32_t				max_pfn;
-	/* vram base address for page table entry  */
-	u64					vram_base_offset;
-	/* is vm enabled? */
-	bool					enabled;
-	/* vm pte handling */
-	const struct amdgpu_vm_pte_funcs        *vm_pte_funcs;
-	struct amdgpu_ring                      *vm_pte_rings[AMDGPU_MAX_RINGS];
-	unsigned				vm_pte_num_rings;
-	atomic_t				vm_pte_next_ring;
-	/* client id counter */
-	atomic64_t				client_counter;
-};
-
-void amdgpu_vm_manager_init(struct amdgpu_device *adev);
-void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
-int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm);
-void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
-void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
-			 struct list_head *validated,
-			 struct amdgpu_bo_list_entry *entry);
-void amdgpu_vm_get_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			  struct list_head *duplicates);
-void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
-				  struct amdgpu_vm *vm);
-int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
-		      struct amdgpu_sync *sync, struct fence *fence,
-		      struct amdgpu_job *job);
-int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job);
-void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id);
-int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
-				    struct amdgpu_vm *vm);
-int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
-			  struct amdgpu_vm *vm);
-int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			     struct amdgpu_sync *sync);
-int amdgpu_vm_bo_update(struct amdgpu_device *adev,
-			struct amdgpu_bo_va *bo_va,
-			bool clear);
-void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
-			     struct amdgpu_bo *bo);
-struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
-				       struct amdgpu_bo *bo);
-struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
-				      struct amdgpu_vm *vm,
-				      struct amdgpu_bo *bo);
-int amdgpu_vm_bo_map(struct amdgpu_device *adev,
-		     struct amdgpu_bo_va *bo_va,
-		     uint64_t addr, uint64_t offset,
-		     uint64_t size, uint32_t flags);
-int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
-		       struct amdgpu_bo_va *bo_va,
-		       uint64_t addr);
-void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
-		      struct amdgpu_bo_va *bo_va);
+		      struct dma_fence **f);
 
 /*
  * context related structures
@@ -958,7 +653,7 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
 
 struct amdgpu_ctx_ring {
 	uint64_t		sequence;
-	struct fence		**fences;
+	struct dma_fence	**fences;
 	struct amd_sched_entity	entity;
 };
 
@@ -967,7 +662,7 @@ struct amdgpu_ctx {
 	struct amdgpu_device    *adev;
 	unsigned		reset_counter;
 	spinlock_t		ring_lock;
-	struct fence            **fences;
+	struct dma_fence	**fences;
 	struct amdgpu_ctx_ring	rings[AMDGPU_MAX_RINGS];
 	bool preamble_presented;
 };
@@ -983,8 +678,8 @@ struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id);
 int amdgpu_ctx_put(struct amdgpu_ctx *ctx);
 
 uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
-			      struct fence *fence);
-struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
+			      struct dma_fence *fence);
+struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
 				   struct amdgpu_ring *ring, uint64_t seq);
 
 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
@@ -1094,6 +789,16 @@ struct amdgpu_scratch {
 /*
  * GFX configurations
  */
+#define AMDGPU_GFX_MAX_SE 4
+#define AMDGPU_GFX_MAX_SH_PER_SE 2
+
+struct amdgpu_rb_config {
+	uint32_t rb_backend_disable;
+	uint32_t user_rb_backend_disable;
+	uint32_t raster_config;
+	uint32_t raster_config_1;
+};
+
 struct amdgpu_gca_config {
 	unsigned max_shader_engines;
 	unsigned max_tile_pipes;
@@ -1122,6 +827,8 @@ struct amdgpu_gca_config {
 
 	uint32_t tile_mode_array[32];
 	uint32_t macrotile_mode_array[16];
+
+	struct amdgpu_rb_config rb_config[AMDGPU_GFX_MAX_SE][AMDGPU_GFX_MAX_SH_PER_SE];
 };
 
 struct amdgpu_cu_info {
@@ -1134,6 +841,9 @@ struct amdgpu_gfx_funcs {
 	/* get the gpu clock counter */
 	uint64_t (*get_gpu_clock_counter)(struct amdgpu_device *adev);
 	void (*select_se_sh)(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance);
+	void (*read_wave_data)(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields);
+	void (*read_wave_vgprs)(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t thread, uint32_t start, uint32_t size, uint32_t *dst);
+	void (*read_wave_sgprs)(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t start, uint32_t size, uint32_t *dst);
 };
 
 struct amdgpu_gfx {
@@ -1182,23 +892,13 @@ struct amdgpu_gfx {
 int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		  unsigned size, struct amdgpu_ib *ib);
 void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib,
-		    struct fence *f);
+		    struct dma_fence *f);
 int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
-		       struct amdgpu_ib *ib, struct fence *last_vm_update,
-		       struct amdgpu_job *job, struct fence **f);
+		       struct amdgpu_ib *ib, struct dma_fence *last_vm_update,
+		       struct amdgpu_job *job, struct dma_fence **f);
 int amdgpu_ib_pool_init(struct amdgpu_device *adev);
 void amdgpu_ib_pool_fini(struct amdgpu_device *adev);
 int amdgpu_ib_ring_tests(struct amdgpu_device *adev);
-int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw);
-void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count);
-void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib);
-void amdgpu_ring_commit(struct amdgpu_ring *ring);
-void amdgpu_ring_undo(struct amdgpu_ring *ring);
-int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
-		     unsigned ring_size, u32 nop, u32 align_mask,
-		     struct amdgpu_irq_src *irq_src, unsigned irq_type,
-		     enum amdgpu_ring_type ring_type);
-void amdgpu_ring_fini(struct amdgpu_ring *ring);
 
 /*
  * CS.
@@ -1226,7 +926,7 @@ struct amdgpu_cs_parser {
 	struct amdgpu_bo_list		*bo_list;
 	struct amdgpu_bo_list_entry	vm_pd;
 	struct list_head		validated;
-	struct fence			*fence;
+	struct dma_fence		*fence;
 	uint64_t			bytes_moved_threshold;
 	uint64_t			bytes_moved;
 	struct amdgpu_bo_list_entry	*evictable;
@@ -1246,7 +946,7 @@ struct amdgpu_job {
 	struct amdgpu_ring	*ring;
 	struct amdgpu_sync	sync;
 	struct amdgpu_ib	*ibs;
-	struct fence		*fence; /* the hw fence */
+	struct dma_fence	*fence; /* the hw fence */
 	uint32_t		preamble_status;
 	uint32_t		num_ibs;
 	void			*owner;
@@ -1295,354 +995,6 @@ struct amdgpu_wb {
 int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb);
 void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb);
 
-
-
-enum amdgpu_int_thermal_type {
-	THERMAL_TYPE_NONE,
-	THERMAL_TYPE_EXTERNAL,
-	THERMAL_TYPE_EXTERNAL_GPIO,
-	THERMAL_TYPE_RV6XX,
-	THERMAL_TYPE_RV770,
-	THERMAL_TYPE_ADT7473_WITH_INTERNAL,
-	THERMAL_TYPE_EVERGREEN,
-	THERMAL_TYPE_SUMO,
-	THERMAL_TYPE_NI,
-	THERMAL_TYPE_SI,
-	THERMAL_TYPE_EMC2103_WITH_INTERNAL,
-	THERMAL_TYPE_CI,
-	THERMAL_TYPE_KV,
-};
-
-enum amdgpu_dpm_auto_throttle_src {
-	AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL,
-	AMDGPU_DPM_AUTO_THROTTLE_SRC_EXTERNAL
-};
-
-enum amdgpu_dpm_event_src {
-	AMDGPU_DPM_EVENT_SRC_ANALOG = 0,
-	AMDGPU_DPM_EVENT_SRC_EXTERNAL = 1,
-	AMDGPU_DPM_EVENT_SRC_DIGITAL = 2,
-	AMDGPU_DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3,
-	AMDGPU_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL = 4
-};
-
-#define AMDGPU_MAX_VCE_LEVELS 6
-
-enum amdgpu_vce_level {
-	AMDGPU_VCE_LEVEL_AC_ALL = 0,     /* AC, All cases */
-	AMDGPU_VCE_LEVEL_DC_EE = 1,      /* DC, entropy encoding */
-	AMDGPU_VCE_LEVEL_DC_LL_LOW = 2,  /* DC, low latency queue, res <= 720 */
-	AMDGPU_VCE_LEVEL_DC_LL_HIGH = 3, /* DC, low latency queue, 1080 >= res > 720 */
-	AMDGPU_VCE_LEVEL_DC_GP_LOW = 4,  /* DC, general purpose queue, res <= 720 */
-	AMDGPU_VCE_LEVEL_DC_GP_HIGH = 5, /* DC, general purpose queue, 1080 >= res > 720 */
-};
-
-struct amdgpu_ps {
-	u32 caps; /* vbios flags */
-	u32 class; /* vbios flags */
-	u32 class2; /* vbios flags */
-	/* UVD clocks */
-	u32 vclk;
-	u32 dclk;
-	/* VCE clocks */
-	u32 evclk;
-	u32 ecclk;
-	bool vce_active;
-	enum amdgpu_vce_level vce_level;
-	/* asic priv */
-	void *ps_priv;
-};
-
-struct amdgpu_dpm_thermal {
-	/* thermal interrupt work */
-	struct work_struct work;
-	/* low temperature threshold */
-	int                min_temp;
-	/* high temperature threshold */
-	int                max_temp;
-	/* was last interrupt low to high or high to low */
-	bool               high_to_low;
-	/* interrupt source */
-	struct amdgpu_irq_src	irq;
-};
-
-enum amdgpu_clk_action
-{
-	AMDGPU_SCLK_UP = 1,
-	AMDGPU_SCLK_DOWN
-};
-
-struct amdgpu_blacklist_clocks
-{
-	u32 sclk;
-	u32 mclk;
-	enum amdgpu_clk_action action;
-};
-
-struct amdgpu_clock_and_voltage_limits {
-	u32 sclk;
-	u32 mclk;
-	u16 vddc;
-	u16 vddci;
-};
-
-struct amdgpu_clock_array {
-	u32 count;
-	u32 *values;
-};
-
-struct amdgpu_clock_voltage_dependency_entry {
-	u32 clk;
-	u16 v;
-};
-
-struct amdgpu_clock_voltage_dependency_table {
-	u32 count;
-	struct amdgpu_clock_voltage_dependency_entry *entries;
-};
-
-union amdgpu_cac_leakage_entry {
-	struct {
-		u16 vddc;
-		u32 leakage;
-	};
-	struct {
-		u16 vddc1;
-		u16 vddc2;
-		u16 vddc3;
-	};
-};
-
-struct amdgpu_cac_leakage_table {
-	u32 count;
-	union amdgpu_cac_leakage_entry *entries;
-};
-
-struct amdgpu_phase_shedding_limits_entry {
-	u16 voltage;
-	u32 sclk;
-	u32 mclk;
-};
-
-struct amdgpu_phase_shedding_limits_table {
-	u32 count;
-	struct amdgpu_phase_shedding_limits_entry *entries;
-};
-
-struct amdgpu_uvd_clock_voltage_dependency_entry {
-	u32 vclk;
-	u32 dclk;
-	u16 v;
-};
-
-struct amdgpu_uvd_clock_voltage_dependency_table {
-	u8 count;
-	struct amdgpu_uvd_clock_voltage_dependency_entry *entries;
-};
-
-struct amdgpu_vce_clock_voltage_dependency_entry {
-	u32 ecclk;
-	u32 evclk;
-	u16 v;
-};
-
-struct amdgpu_vce_clock_voltage_dependency_table {
-	u8 count;
-	struct amdgpu_vce_clock_voltage_dependency_entry *entries;
-};
-
-struct amdgpu_ppm_table {
-	u8 ppm_design;
-	u16 cpu_core_number;
-	u32 platform_tdp;
-	u32 small_ac_platform_tdp;
-	u32 platform_tdc;
-	u32 small_ac_platform_tdc;
-	u32 apu_tdp;
-	u32 dgpu_tdp;
-	u32 dgpu_ulv_power;
-	u32 tj_max;
-};
-
-struct amdgpu_cac_tdp_table {
-	u16 tdp;
-	u16 configurable_tdp;
-	u16 tdc;
-	u16 battery_power_limit;
-	u16 small_power_limit;
-	u16 low_cac_leakage;
-	u16 high_cac_leakage;
-	u16 maximum_power_delivery_limit;
-};
-
-struct amdgpu_dpm_dynamic_state {
-	struct amdgpu_clock_voltage_dependency_table vddc_dependency_on_sclk;
-	struct amdgpu_clock_voltage_dependency_table vddci_dependency_on_mclk;
-	struct amdgpu_clock_voltage_dependency_table vddc_dependency_on_mclk;
-	struct amdgpu_clock_voltage_dependency_table mvdd_dependency_on_mclk;
-	struct amdgpu_clock_voltage_dependency_table vddc_dependency_on_dispclk;
-	struct amdgpu_uvd_clock_voltage_dependency_table uvd_clock_voltage_dependency_table;
-	struct amdgpu_vce_clock_voltage_dependency_table vce_clock_voltage_dependency_table;
-	struct amdgpu_clock_voltage_dependency_table samu_clock_voltage_dependency_table;
-	struct amdgpu_clock_voltage_dependency_table acp_clock_voltage_dependency_table;
-	struct amdgpu_clock_voltage_dependency_table vddgfx_dependency_on_sclk;
-	struct amdgpu_clock_array valid_sclk_values;
-	struct amdgpu_clock_array valid_mclk_values;
-	struct amdgpu_clock_and_voltage_limits max_clock_voltage_on_dc;
-	struct amdgpu_clock_and_voltage_limits max_clock_voltage_on_ac;
-	u32 mclk_sclk_ratio;
-	u32 sclk_mclk_delta;
-	u16 vddc_vddci_delta;
-	u16 min_vddc_for_pcie_gen2;
-	struct amdgpu_cac_leakage_table cac_leakage_table;
-	struct amdgpu_phase_shedding_limits_table phase_shedding_limits_table;
-	struct amdgpu_ppm_table *ppm_table;
-	struct amdgpu_cac_tdp_table *cac_tdp_table;
-};
-
-struct amdgpu_dpm_fan {
-	u16 t_min;
-	u16 t_med;
-	u16 t_high;
-	u16 pwm_min;
-	u16 pwm_med;
-	u16 pwm_high;
-	u8 t_hyst;
-	u32 cycle_delay;
-	u16 t_max;
-	u8 control_mode;
-	u16 default_max_fan_pwm;
-	u16 default_fan_output_sensitivity;
-	u16 fan_output_sensitivity;
-	bool ucode_fan_control;
-};
-
-enum amdgpu_pcie_gen {
-	AMDGPU_PCIE_GEN1 = 0,
-	AMDGPU_PCIE_GEN2 = 1,
-	AMDGPU_PCIE_GEN3 = 2,
-	AMDGPU_PCIE_GEN_INVALID = 0xffff
-};
-
-enum amdgpu_dpm_forced_level {
-	AMDGPU_DPM_FORCED_LEVEL_AUTO = 0,
-	AMDGPU_DPM_FORCED_LEVEL_LOW = 1,
-	AMDGPU_DPM_FORCED_LEVEL_HIGH = 2,
-	AMDGPU_DPM_FORCED_LEVEL_MANUAL = 3,
-};
-
-struct amdgpu_vce_state {
-	/* vce clocks */
-	u32 evclk;
-	u32 ecclk;
-	/* gpu clocks */
-	u32 sclk;
-	u32 mclk;
-	u8 clk_idx;
-	u8 pstate;
-};
-
-struct amdgpu_dpm_funcs {
-	int (*get_temperature)(struct amdgpu_device *adev);
-	int (*pre_set_power_state)(struct amdgpu_device *adev);
-	int (*set_power_state)(struct amdgpu_device *adev);
-	void (*post_set_power_state)(struct amdgpu_device *adev);
-	void (*display_configuration_changed)(struct amdgpu_device *adev);
-	u32 (*get_sclk)(struct amdgpu_device *adev, bool low);
-	u32 (*get_mclk)(struct amdgpu_device *adev, bool low);
-	void (*print_power_state)(struct amdgpu_device *adev, struct amdgpu_ps *ps);
-	void (*debugfs_print_current_performance_level)(struct amdgpu_device *adev, struct seq_file *m);
-	int (*force_performance_level)(struct amdgpu_device *adev, enum amdgpu_dpm_forced_level level);
-	bool (*vblank_too_short)(struct amdgpu_device *adev);
-	void (*powergate_uvd)(struct amdgpu_device *adev, bool gate);
-	void (*powergate_vce)(struct amdgpu_device *adev, bool gate);
-	void (*enable_bapm)(struct amdgpu_device *adev, bool enable);
-	void (*set_fan_control_mode)(struct amdgpu_device *adev, u32 mode);
-	u32 (*get_fan_control_mode)(struct amdgpu_device *adev);
-	int (*set_fan_speed_percent)(struct amdgpu_device *adev, u32 speed);
-	int (*get_fan_speed_percent)(struct amdgpu_device *adev, u32 *speed);
-	int (*force_clock_level)(struct amdgpu_device *adev, enum pp_clock_type type, uint32_t mask);
-	int (*print_clock_levels)(struct amdgpu_device *adev, enum pp_clock_type type, char *buf);
-	int (*get_sclk_od)(struct amdgpu_device *adev);
-	int (*set_sclk_od)(struct amdgpu_device *adev, uint32_t value);
-	int (*get_mclk_od)(struct amdgpu_device *adev);
-	int (*set_mclk_od)(struct amdgpu_device *adev, uint32_t value);
-};
-
-struct amdgpu_dpm {
-	struct amdgpu_ps        *ps;
-	/* number of valid power states */
-	int                     num_ps;
-	/* current power state that is active */
-	struct amdgpu_ps        *current_ps;
-	/* requested power state */
-	struct amdgpu_ps        *requested_ps;
-	/* boot up power state */
-	struct amdgpu_ps        *boot_ps;
-	/* default uvd power state */
-	struct amdgpu_ps        *uvd_ps;
-	/* vce requirements */
-	struct amdgpu_vce_state vce_states[AMDGPU_MAX_VCE_LEVELS];
-	enum amdgpu_vce_level vce_level;
-	enum amd_pm_state_type state;
-	enum amd_pm_state_type user_state;
-	u32                     platform_caps;
-	u32                     voltage_response_time;
-	u32                     backbias_response_time;
-	void                    *priv;
-	u32			new_active_crtcs;
-	int			new_active_crtc_count;
-	u32			current_active_crtcs;
-	int			current_active_crtc_count;
-	struct amdgpu_dpm_dynamic_state dyn_state;
-	struct amdgpu_dpm_fan fan;
-	u32 tdp_limit;
-	u32 near_tdp_limit;
-	u32 near_tdp_limit_adjusted;
-	u32 sq_ramping_threshold;
-	u32 cac_leakage;
-	u16 tdp_od_limit;
-	u32 tdp_adjustment;
-	u16 load_line_slope;
-	bool power_control;
-	bool ac_power;
-	/* special states active */
-	bool                    thermal_active;
-	bool                    uvd_active;
-	bool                    vce_active;
-	/* thermal handling */
-	struct amdgpu_dpm_thermal thermal;
-	/* forced levels */
-	enum amdgpu_dpm_forced_level forced_level;
-};
-
-struct amdgpu_pm {
-	struct mutex		mutex;
-	u32                     current_sclk;
-	u32                     current_mclk;
-	u32                     default_sclk;
-	u32                     default_mclk;
-	struct amdgpu_i2c_chan *i2c_bus;
-	/* internal thermal controller on rv6xx+ */
-	enum amdgpu_int_thermal_type int_thermal_type;
-	struct device	        *int_hwmon_dev;
-	/* fan control parameters */
-	bool                    no_fan;
-	u8                      fan_pulses_per_revolution;
-	u8                      fan_min_rpm;
-	u8                      fan_max_rpm;
-	/* dpm */
-	bool                    dpm_enabled;
-	bool                    sysfs_initialized;
-	struct amdgpu_dpm       dpm;
-	const struct firmware	*fw;	/* SMC firmware */
-	uint32_t                fw_version;
-	const struct amdgpu_dpm_funcs *funcs;
-	uint32_t                pcie_gen_mask;
-	uint32_t                pcie_mlw_mask;
-	struct amd_pp_display_configuration pm_display_cfg;/* set by DAL */
-};
-
 void amdgpu_get_pcie_info(struct amdgpu_device *adev);
 
 /*
@@ -1863,6 +1215,8 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *filp);
 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
+int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
+				struct drm_file *filp);
 
 int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *filp);
@@ -1940,14 +1294,6 @@ typedef void (*amdgpu_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
 typedef uint32_t (*amdgpu_block_rreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
 typedef void (*amdgpu_block_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t, uint32_t);
 
-struct amdgpu_ip_block_status {
-	bool valid;
-	bool sw;
-	bool hw;
-	bool late_initialized;
-	bool hang;
-};
-
 struct amdgpu_device {
 	struct device			*dev;
 	struct drm_device		*ddev;
@@ -1986,6 +1332,7 @@ struct amdgpu_device {
 
 	/* BIOS */
 	uint8_t				*bios;
+	uint32_t			bios_size;
 	bool				is_atom_bios;
 	struct amdgpu_bo		*stollen_vga_memory;
 	uint32_t			bios_scratch[AMDGPU_BIOS_NUM_SCRATCH];
@@ -2103,9 +1450,8 @@ struct amdgpu_device {
 	/* GDS */
 	struct amdgpu_gds		gds;
 
-	const struct amdgpu_ip_block_version *ip_blocks;
+	struct amdgpu_ip_block          ip_blocks[AMDGPU_MAX_IP_NUM];
 	int				num_ip_blocks;
-	struct amdgpu_ip_block_status	*ip_block_status;
 	struct mutex	mn_lock;
 	DECLARE_HASHTABLE(mn_hash, 7);
 
@@ -2128,6 +1474,11 @@ struct amdgpu_device {
 
 };
 
+static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
+{
+	return container_of(bdev, struct amdgpu_device, mman.bdev);
+}
+
 bool amdgpu_device_is_px(struct drm_device *dev);
 int amdgpu_device_init(struct amdgpu_device *adev,
 		       struct drm_device *ddev,
@@ -2279,15 +1630,12 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_ring_pad_ib(r, ib) ((r)->funcs->pad_ib((r), (ib)))
 #define amdgpu_ring_init_cond_exec(r) (r)->funcs->init_cond_exec((r))
 #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
-#define amdgpu_ring_get_emit_ib_size(r) (r)->funcs->get_emit_ib_size((r))
-#define amdgpu_ring_get_dma_frame_size(r) (r)->funcs->get_dma_frame_size((r))
 #define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev))
 #define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv))
 #define amdgpu_ih_set_rptr(adev) (adev)->irq.ih_funcs->set_rptr((adev))
 #define amdgpu_display_set_vga_render_state(adev, r) (adev)->mode_info.funcs->set_vga_render_state((adev), (r))
 #define amdgpu_display_vblank_get_counter(adev, crtc) (adev)->mode_info.funcs->vblank_get_counter((adev), (crtc))
 #define amdgpu_display_vblank_wait(adev, crtc) (adev)->mode_info.funcs->vblank_wait((adev), (crtc))
-#define amdgpu_display_is_display_hung(adev) (adev)->mode_info.funcs->is_display_hung((adev))
 #define amdgpu_display_backlight_set_level(adev, e, l) (adev)->mode_info.funcs->backlight_set_level((e), (l))
 #define amdgpu_display_backlight_get_level(adev, e) (adev)->mode_info.funcs->backlight_get_level((e))
 #define amdgpu_display_hpd_sense(adev, h) (adev)->mode_info.funcs->hpd_sense((adev), (h))
@@ -2302,108 +1650,8 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_display_resume_mc_access(adev, s) (adev)->mode_info.funcs->resume_mc_access((adev), (s))
 #define amdgpu_emit_copy_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((ib),  (s), (d), (b))
 #define amdgpu_emit_fill_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b))
-#define amdgpu_dpm_pre_set_power_state(adev) (adev)->pm.funcs->pre_set_power_state((adev))
-#define amdgpu_dpm_set_power_state(adev) (adev)->pm.funcs->set_power_state((adev))
-#define amdgpu_dpm_post_set_power_state(adev) (adev)->pm.funcs->post_set_power_state((adev))
-#define amdgpu_dpm_display_configuration_changed(adev) (adev)->pm.funcs->display_configuration_changed((adev))
-#define amdgpu_dpm_print_power_state(adev, ps) (adev)->pm.funcs->print_power_state((adev), (ps))
-#define amdgpu_dpm_vblank_too_short(adev) (adev)->pm.funcs->vblank_too_short((adev))
-#define amdgpu_dpm_enable_bapm(adev, e) (adev)->pm.funcs->enable_bapm((adev), (e))
 #define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev))
 #define amdgpu_gfx_select_se_sh(adev, se, sh, instance) (adev)->gfx.funcs->select_se_sh((adev), (se), (sh), (instance))
-
-#define amdgpu_dpm_read_sensor(adev, idx, value) \
-	((adev)->pp_enabled ? \
-		(adev)->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, (idx), (value)) : \
-		-EINVAL)
-
-#define amdgpu_dpm_get_temperature(adev) \
-	((adev)->pp_enabled ?						\
-	      (adev)->powerplay.pp_funcs->get_temperature((adev)->powerplay.pp_handle) : \
-	      (adev)->pm.funcs->get_temperature((adev)))
-
-#define amdgpu_dpm_set_fan_control_mode(adev, m) \
-	((adev)->pp_enabled ?						\
-	      (adev)->powerplay.pp_funcs->set_fan_control_mode((adev)->powerplay.pp_handle, (m)) : \
-	      (adev)->pm.funcs->set_fan_control_mode((adev), (m)))
-
-#define amdgpu_dpm_get_fan_control_mode(adev) \
-	((adev)->pp_enabled ?						\
-	      (adev)->powerplay.pp_funcs->get_fan_control_mode((adev)->powerplay.pp_handle) : \
-	      (adev)->pm.funcs->get_fan_control_mode((adev)))
-
-#define amdgpu_dpm_set_fan_speed_percent(adev, s) \
-	((adev)->pp_enabled ?						\
-	      (adev)->powerplay.pp_funcs->set_fan_speed_percent((adev)->powerplay.pp_handle, (s)) : \
-	      (adev)->pm.funcs->set_fan_speed_percent((adev), (s)))
-
-#define amdgpu_dpm_get_fan_speed_percent(adev, s) \
-	((adev)->pp_enabled ?						\
-	      (adev)->powerplay.pp_funcs->get_fan_speed_percent((adev)->powerplay.pp_handle, (s)) : \
-	      (adev)->pm.funcs->get_fan_speed_percent((adev), (s)))
-
-#define amdgpu_dpm_get_sclk(adev, l) \
-	((adev)->pp_enabled ?						\
-	      (adev)->powerplay.pp_funcs->get_sclk((adev)->powerplay.pp_handle, (l)) : \
-		(adev)->pm.funcs->get_sclk((adev), (l)))
-
-#define amdgpu_dpm_get_mclk(adev, l)  \
-	((adev)->pp_enabled ?						\
-	      (adev)->powerplay.pp_funcs->get_mclk((adev)->powerplay.pp_handle, (l)) : \
-	      (adev)->pm.funcs->get_mclk((adev), (l)))
-
-
-#define amdgpu_dpm_force_performance_level(adev, l) \
-	((adev)->pp_enabled ?						\
-	      (adev)->powerplay.pp_funcs->force_performance_level((adev)->powerplay.pp_handle, (l)) : \
-	      (adev)->pm.funcs->force_performance_level((adev), (l)))
-
-#define amdgpu_dpm_powergate_uvd(adev, g) \
-	((adev)->pp_enabled ?						\
-	      (adev)->powerplay.pp_funcs->powergate_uvd((adev)->powerplay.pp_handle, (g)) : \
-	      (adev)->pm.funcs->powergate_uvd((adev), (g)))
-
-#define amdgpu_dpm_powergate_vce(adev, g) \
-	((adev)->pp_enabled ?						\
-	      (adev)->powerplay.pp_funcs->powergate_vce((adev)->powerplay.pp_handle, (g)) : \
-	      (adev)->pm.funcs->powergate_vce((adev), (g)))
-
-#define amdgpu_dpm_get_current_power_state(adev) \
-	(adev)->powerplay.pp_funcs->get_current_power_state((adev)->powerplay.pp_handle)
-
-#define amdgpu_dpm_get_performance_level(adev) \
-	(adev)->powerplay.pp_funcs->get_performance_level((adev)->powerplay.pp_handle)
-
-#define amdgpu_dpm_get_pp_num_states(adev, data) \
-	(adev)->powerplay.pp_funcs->get_pp_num_states((adev)->powerplay.pp_handle, data)
-
-#define amdgpu_dpm_get_pp_table(adev, table) \
-	(adev)->powerplay.pp_funcs->get_pp_table((adev)->powerplay.pp_handle, table)
-
-#define amdgpu_dpm_set_pp_table(adev, buf, size) \
-	(adev)->powerplay.pp_funcs->set_pp_table((adev)->powerplay.pp_handle, buf, size)
-
-#define amdgpu_dpm_print_clock_levels(adev, type, buf) \
-	(adev)->powerplay.pp_funcs->print_clock_levels((adev)->powerplay.pp_handle, type, buf)
-
-#define amdgpu_dpm_force_clock_level(adev, type, level) \
-		(adev)->powerplay.pp_funcs->force_clock_level((adev)->powerplay.pp_handle, type, level)
-
-#define amdgpu_dpm_get_sclk_od(adev) \
-	(adev)->powerplay.pp_funcs->get_sclk_od((adev)->powerplay.pp_handle)
-
-#define amdgpu_dpm_set_sclk_od(adev, value) \
-	(adev)->powerplay.pp_funcs->set_sclk_od((adev)->powerplay.pp_handle, value)
-
-#define amdgpu_dpm_get_mclk_od(adev) \
-	((adev)->powerplay.pp_funcs->get_mclk_od((adev)->powerplay.pp_handle))
-
-#define amdgpu_dpm_set_mclk_od(adev, value) \
-	((adev)->powerplay.pp_funcs->set_mclk_od((adev)->powerplay.pp_handle, value))
-
-#define amdgpu_dpm_dispatch_task(adev, event_id, input, output)		\
-	(adev)->powerplay.pp_funcs->dispatch_tasks((adev)->powerplay.pp_handle, (event_id), (input), (output))
-
 #define amdgpu_gds_switch(adev, r, v, d, w, a) (adev)->gds.funcs->patch_gds_switch((r), (v), (d), (w), (a))
 
 /* Common functions */
@@ -2434,8 +1682,6 @@ uint32_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base);
 void amdgpu_gtt_location(struct amdgpu_device *adev, struct amdgpu_mc *mc);
 void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size);
-u64 amdgpu_ttm_get_gtt_mem_size(struct amdgpu_device *adev);
-int amdgpu_ttm_global_init(struct amdgpu_device *adev);
 int amdgpu_ttm_init(struct amdgpu_device *adev);
 void amdgpu_ttm_fini(struct amdgpu_device *adev);
 void amdgpu_program_register_sequence(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
index 2057683f7b59..06879d1dcabd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
@@ -265,14 +265,14 @@ static int acp_hw_init(void *handle)
 
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	const struct amdgpu_ip_block_version *ip_version =
+	const struct amdgpu_ip_block *ip_block =
 		amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_ACP);
 
-	if (!ip_version)
+	if (!ip_block)
 		return -EINVAL;
 
 	r = amd_acp_hw_init(adev->acp.cgs_device,
-			    ip_version->major, ip_version->minor);
+			    ip_block->version->major, ip_block->version->minor);
 	/* -ENODEV means board uses AZ rather than ACP */
 	if (r == -ENODEV)
 		return 0;
@@ -459,7 +459,7 @@ static int acp_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs acp_ip_funcs = {
+static const struct amd_ip_funcs acp_ip_funcs = {
 	.name = "acp_ip",
 	.early_init = acp_early_init,
 	.late_init = NULL,
@@ -475,3 +475,12 @@ const struct amd_ip_funcs acp_ip_funcs = {
 	.set_clockgating_state = acp_set_clockgating_state,
 	.set_powergating_state = acp_set_powergating_state,
 };
+
+const struct amdgpu_ip_block_version acp_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_ACP,
+	.major = 2,
+	.minor = 2,
+	.rev = 0,
+	.funcs = &acp_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h
index 8a396313c86f..a288ce25c176 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h
@@ -37,6 +37,6 @@ struct amdgpu_acp {
 	struct acp_pm_domain *acp_genpd;
 };
 
-extern const struct amd_ip_funcs acp_ip_funcs;
+extern const struct amdgpu_ip_block_version acp_ip_block;
 
 #endif /* __AMDGPU_ACP_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index 8e6bf548d689..56a86dd5789e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -1115,49 +1115,6 @@ int amdgpu_atombios_get_memory_pll_dividers(struct amdgpu_device *adev,
 	return 0;
 }
 
-uint32_t amdgpu_atombios_get_engine_clock(struct amdgpu_device *adev)
-{
-	GET_ENGINE_CLOCK_PS_ALLOCATION args;
-	int index = GetIndexIntoMasterTable(COMMAND, GetEngineClock);
-
-	amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args);
-	return le32_to_cpu(args.ulReturnEngineClock);
-}
-
-uint32_t amdgpu_atombios_get_memory_clock(struct amdgpu_device *adev)
-{
-	GET_MEMORY_CLOCK_PS_ALLOCATION args;
-	int index = GetIndexIntoMasterTable(COMMAND, GetMemoryClock);
-
-	amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args);
-	return le32_to_cpu(args.ulReturnMemoryClock);
-}
-
-void amdgpu_atombios_set_engine_clock(struct amdgpu_device *adev,
-				      uint32_t eng_clock)
-{
-	SET_ENGINE_CLOCK_PS_ALLOCATION args;
-	int index = GetIndexIntoMasterTable(COMMAND, SetEngineClock);
-
-	args.ulTargetEngineClock = cpu_to_le32(eng_clock);	/* 10 khz */
-
-	amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args);
-}
-
-void amdgpu_atombios_set_memory_clock(struct amdgpu_device *adev,
-				      uint32_t mem_clock)
-{
-	SET_MEMORY_CLOCK_PS_ALLOCATION args;
-	int index = GetIndexIntoMasterTable(COMMAND, SetMemoryClock);
-
-	if (adev->flags & AMD_IS_APU)
-		return;
-
-	args.ulTargetMemoryClock = cpu_to_le32(mem_clock);	/* 10 khz */
-
-	amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args);
-}
-
 void amdgpu_atombios_set_engine_dram_timings(struct amdgpu_device *adev,
 					     u32 eng_clock, u32 mem_clock)
 {
@@ -1256,45 +1213,6 @@ int amdgpu_atombios_get_leakage_vddc_based_on_leakage_idx(struct amdgpu_device *
 	return amdgpu_atombios_get_max_vddc(adev, VOLTAGE_TYPE_VDDC, leakage_idx, voltage);
 }
 
-void amdgpu_atombios_set_voltage(struct amdgpu_device *adev,
-				 u16 voltage_level,
-				 u8 voltage_type)
-{
-	union set_voltage args;
-	int index = GetIndexIntoMasterTable(COMMAND, SetVoltage);
-	u8 frev, crev, volt_index = voltage_level;
-
-	if (!amdgpu_atom_parse_cmd_header(adev->mode_info.atom_context, index, &frev, &crev))
-		return;
-
-	/* 0xff01 is a flag rather then an actual voltage */
-	if (voltage_level == 0xff01)
-		return;
-
-	switch (crev) {
-	case 1:
-		args.v1.ucVoltageType = voltage_type;
-		args.v1.ucVoltageMode = SET_ASIC_VOLTAGE_MODE_ALL_SOURCE;
-		args.v1.ucVoltageIndex = volt_index;
-		break;
-	case 2:
-		args.v2.ucVoltageType = voltage_type;
-		args.v2.ucVoltageMode = SET_ASIC_VOLTAGE_MODE_SET_VOLTAGE;
-		args.v2.usVoltageLevel = cpu_to_le16(voltage_level);
-		break;
-	case 3:
-		args.v3.ucVoltageType = voltage_type;
-		args.v3.ucVoltageMode = ATOM_SET_VOLTAGE;
-		args.v3.usVoltageLevel = cpu_to_le16(voltage_level);
-		break;
-	default:
-		DRM_ERROR("Unknown table version %d, %d\n", frev, crev);
-		return;
-	}
-
-	amdgpu_atom_execute_table(adev->mode_info.atom_context, index, (uint32_t *)&args);
-}
-
 int amdgpu_atombios_get_leakage_id_from_vbios(struct amdgpu_device *adev,
 					      u16 *leakage_id)
 {
@@ -1784,6 +1702,19 @@ void amdgpu_atombios_scratch_regs_restore(struct amdgpu_device *adev)
 		WREG32(mmBIOS_SCRATCH_0 + i, adev->bios_scratch[i]);
 }
 
+void amdgpu_atombios_scratch_regs_engine_hung(struct amdgpu_device *adev,
+					      bool hung)
+{
+	u32 tmp = RREG32(mmBIOS_SCRATCH_3);
+
+	if (hung)
+		tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
+	else
+		tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
+
+	WREG32(mmBIOS_SCRATCH_3, tmp);
+}
+
 /* Atom needs data in little endian format
  * so swap as appropriate when copying data to
  * or from atom. Note that atom operates on
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
index 17356151db38..70e9acef5d9c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
@@ -163,16 +163,6 @@ int amdgpu_atombios_get_memory_pll_dividers(struct amdgpu_device *adev,
 					    bool strobe_mode,
 					    struct atom_mpll_param *mpll_param);
 
-uint32_t amdgpu_atombios_get_engine_clock(struct amdgpu_device *adev);
-uint32_t amdgpu_atombios_get_memory_clock(struct amdgpu_device *adev);
-void amdgpu_atombios_set_engine_clock(struct amdgpu_device *adev,
-				      uint32_t eng_clock);
-void amdgpu_atombios_set_memory_clock(struct amdgpu_device *adev,
-				      uint32_t mem_clock);
-void amdgpu_atombios_set_voltage(struct amdgpu_device *adev,
-				 u16 voltage_level,
-				 u8 voltage_type);
-
 void amdgpu_atombios_set_engine_dram_timings(struct amdgpu_device *adev,
 					     u32 eng_clock, u32 mem_clock);
 
@@ -206,6 +196,8 @@ void amdgpu_atombios_scratch_regs_lock(struct amdgpu_device *adev, bool lock);
 void amdgpu_atombios_scratch_regs_init(struct amdgpu_device *adev);
 void amdgpu_atombios_scratch_regs_save(struct amdgpu_device *adev);
 void amdgpu_atombios_scratch_regs_restore(struct amdgpu_device *adev);
+void amdgpu_atombios_scratch_regs_engine_hung(struct amdgpu_device *adev,
+					      bool hung);
 
 void amdgpu_atombios_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le);
 int amdgpu_atombios_get_max_vddc(struct amdgpu_device *adev, u8 voltage_type,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
index 345305235349..cc97eee93226 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
@@ -33,7 +33,7 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size,
 {
 	unsigned long start_jiffies;
 	unsigned long end_jiffies;
-	struct fence *fence = NULL;
+	struct dma_fence *fence = NULL;
 	int i, r;
 
 	start_jiffies = jiffies;
@@ -43,17 +43,17 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size,
 				       false);
 		if (r)
 			goto exit_do_move;
-		r = fence_wait(fence, false);
+		r = dma_fence_wait(fence, false);
 		if (r)
 			goto exit_do_move;
-		fence_put(fence);
+		dma_fence_put(fence);
 	}
 	end_jiffies = jiffies;
 	r = jiffies_to_msecs(end_jiffies - start_jiffies);
 
 exit_do_move:
 	if (fence)
-		fence_put(fence);
+		dma_fence_put(fence);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
index 2b6afe123f3d..4f973a9c7b87 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
@@ -70,10 +70,11 @@ static bool igp_read_bios_from_vram(struct amdgpu_device *adev)
 		return false;
 	}
 	adev->bios = kmalloc(size, GFP_KERNEL);
-	if (adev->bios == NULL) {
+	if (!adev->bios) {
 		iounmap(bios);
 		return false;
 	}
+	adev->bios_size = size;
 	memcpy_fromio(adev->bios, bios, size);
 	iounmap(bios);
 	return true;
@@ -103,6 +104,7 @@ bool amdgpu_read_bios(struct amdgpu_device *adev)
 		pci_unmap_rom(adev->pdev, bios);
 		return false;
 	}
+	adev->bios_size = size;
 	memcpy_fromio(adev->bios, bios, size);
 	pci_unmap_rom(adev->pdev, bios);
 	return true;
@@ -135,6 +137,7 @@ static bool amdgpu_read_bios_from_rom(struct amdgpu_device *adev)
 		DRM_ERROR("no memory to allocate for BIOS\n");
 		return false;
 	}
+	adev->bios_size = len;
 
 	/* read complete BIOS */
 	return amdgpu_asic_read_bios_from_rom(adev, adev->bios, len);
@@ -159,6 +162,7 @@ static bool amdgpu_read_platform_bios(struct amdgpu_device *adev)
 	if (adev->bios == NULL) {
 		return false;
 	}
+	adev->bios_size = size;
 
 	return true;
 }
@@ -273,6 +277,7 @@ static bool amdgpu_atrm_get_bios(struct amdgpu_device *adev)
 		kfree(adev->bios);
 		return false;
 	}
+	adev->bios_size = size;
 	return true;
 }
 #else
@@ -334,6 +339,7 @@ static bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev)
 	}
 
 	adev->bios = kmemdup(&vbios->VbiosContent, vhdr->ImageLength, GFP_KERNEL);
+	adev->bios_size = vhdr->ImageLength;
 	ret = !!adev->bios;
 
 out_unmap:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index 662976292535..9ada56c16a58 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -146,7 +146,8 @@ static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
 	switch(type) {
 	case CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB:
 	case CGS_GPU_MEM_TYPE__VISIBLE_FB:
-		flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+		flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+			AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
 		domain = AMDGPU_GEM_DOMAIN_VRAM;
 		if (max_offset > adev->mc.real_vram_size)
 			return -EINVAL;
@@ -157,7 +158,8 @@ static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
 		break;
 	case CGS_GPU_MEM_TYPE__INVISIBLE_CONTIG_FB:
 	case CGS_GPU_MEM_TYPE__INVISIBLE_FB:
-		flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
+		flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
+			AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
 		domain = AMDGPU_GEM_DOMAIN_VRAM;
 		if (adev->mc.visible_vram_size < adev->mc.real_vram_size) {
 			place.fpfn =
@@ -240,7 +242,7 @@ static int amdgpu_cgs_gmap_gpu_mem(struct cgs_device *cgs_device, cgs_handle_t h
 	r = amdgpu_bo_reserve(obj, false);
 	if (unlikely(r != 0))
 		return r;
-	r = amdgpu_bo_pin_restricted(obj, AMDGPU_GEM_DOMAIN_GTT,
+	r = amdgpu_bo_pin_restricted(obj, obj->prefered_domains,
 				     min_offset, max_offset, mcaddr);
 	amdgpu_bo_unreserve(obj);
 	return r;
@@ -624,11 +626,11 @@ static int amdgpu_cgs_set_clockgating_state(struct cgs_device *cgs_device,
 	int i, r = -1;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
 
-		if (adev->ip_blocks[i].type == block_type) {
-			r = adev->ip_blocks[i].funcs->set_clockgating_state(
+		if (adev->ip_blocks[i].version->type == block_type) {
+			r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
 								(void *)adev,
 									state);
 			break;
@@ -645,11 +647,11 @@ static int amdgpu_cgs_set_powergating_state(struct cgs_device *cgs_device,
 	int i, r = -1;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
 
-		if (adev->ip_blocks[i].type == block_type) {
-			r = adev->ip_blocks[i].funcs->set_powergating_state(
+		if (adev->ip_blocks[i].version->type == block_type) {
+			r = adev->ip_blocks[i].version->funcs->set_powergating_state(
 								(void *)adev,
 									state);
 			break;
@@ -685,15 +687,21 @@ static uint32_t fw_type_convert(struct cgs_device *cgs_device, uint32_t fw_type)
 		result = AMDGPU_UCODE_ID_CP_MEC1;
 		break;
 	case CGS_UCODE_ID_CP_MEC_JT2:
-		if (adev->asic_type == CHIP_TONGA || adev->asic_type == CHIP_POLARIS11
-		  || adev->asic_type == CHIP_POLARIS10)
-			result = AMDGPU_UCODE_ID_CP_MEC2;
-		else
+		/* for VI. JT2 should be the same as JT1, because:
+			1, MEC2 and MEC1 use exactly same FW.
+			2, JT2 is not pached but JT1 is.
+		*/
+		if (adev->asic_type >= CHIP_TOPAZ)
 			result = AMDGPU_UCODE_ID_CP_MEC1;
+		else
+			result = AMDGPU_UCODE_ID_CP_MEC2;
 		break;
 	case CGS_UCODE_ID_RLC_G:
 		result = AMDGPU_UCODE_ID_RLC_G;
 		break;
+	case CGS_UCODE_ID_STORAGE:
+		result = AMDGPU_UCODE_ID_STORAGE;
+		break;
 	default:
 		DRM_ERROR("Firmware type not supported\n");
 	}
@@ -715,7 +723,7 @@ static uint16_t amdgpu_get_firmware_version(struct cgs_device *cgs_device,
 					enum cgs_ucode_id type)
 {
 	CGS_FUNC_ADEV;
-	uint16_t fw_version;
+	uint16_t fw_version = 0;
 
 	switch (type) {
 		case CGS_UCODE_ID_SDMA0:
@@ -745,9 +753,11 @@ static uint16_t amdgpu_get_firmware_version(struct cgs_device *cgs_device,
 		case CGS_UCODE_ID_RLC_G:
 			fw_version = adev->gfx.rlc_fw_version;
 			break;
+		case CGS_UCODE_ID_STORAGE:
+			break;
 		default:
 			DRM_ERROR("firmware type %d do not have version\n", type);
-			fw_version = 0;
+			break;
 	}
 	return fw_version;
 }
@@ -776,12 +786,18 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 
 		if ((type == CGS_UCODE_ID_CP_MEC_JT1) ||
 		    (type == CGS_UCODE_ID_CP_MEC_JT2)) {
-			gpu_addr += le32_to_cpu(header->jt_offset) << 2;
+			gpu_addr += ALIGN(le32_to_cpu(header->header.ucode_size_bytes), PAGE_SIZE);
 			data_size = le32_to_cpu(header->jt_size) << 2;
 		}
-		info->mc_addr = gpu_addr;
+
+		info->kptr = ucode->kaddr;
 		info->image_size = data_size;
+		info->mc_addr = gpu_addr;
 		info->version = (uint16_t)le32_to_cpu(header->header.ucode_version);
+
+		if (CGS_UCODE_ID_CP_MEC == type)
+			info->image_size = (header->jt_offset) << 2;
+
 		info->fw_version = amdgpu_get_firmware_version(cgs_device, type);
 		info->feature_version = (uint16_t)le32_to_cpu(header->ucode_feature_version);
 	} else {
@@ -860,6 +876,12 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 	return 0;
 }
 
+static int amdgpu_cgs_is_virtualization_enabled(void *cgs_device)
+{
+	CGS_FUNC_ADEV;
+	return amdgpu_sriov_vf(adev);
+}
+
 static int amdgpu_cgs_query_system_info(struct cgs_device *cgs_device,
 					struct cgs_system_info *sys_info)
 {
@@ -1213,6 +1235,7 @@ static const struct cgs_ops amdgpu_cgs_ops = {
 	amdgpu_cgs_notify_dpm_enabled,
 	amdgpu_cgs_call_acpi_method,
 	amdgpu_cgs_query_system_info,
+	amdgpu_cgs_is_virtualization_enabled
 };
 
 static const struct cgs_os_ops amdgpu_cgs_os_ops = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index 086aa5c9c634..8d1cf2d3e663 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -1517,88 +1517,6 @@ static const struct drm_connector_funcs amdgpu_connector_edp_funcs = {
 	.force = amdgpu_connector_dvi_force,
 };
 
-static struct drm_encoder *
-amdgpu_connector_virtual_encoder(struct drm_connector *connector)
-{
-	int enc_id = connector->encoder_ids[0];
-	struct drm_encoder *encoder;
-	int i;
-	for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
-		if (connector->encoder_ids[i] == 0)
-			break;
-
-		encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]);
-		if (!encoder)
-			continue;
-
-		if (encoder->encoder_type == DRM_MODE_ENCODER_VIRTUAL)
-			return encoder;
-	}
-
-	/* pick the first one */
-	if (enc_id)
-		return drm_encoder_find(connector->dev, enc_id);
-	return NULL;
-}
-
-static int amdgpu_connector_virtual_get_modes(struct drm_connector *connector)
-{
-	struct drm_encoder *encoder = amdgpu_connector_best_single_encoder(connector);
-
-	if (encoder) {
-		amdgpu_connector_add_common_modes(encoder, connector);
-	}
-
-	return 0;
-}
-
-static int amdgpu_connector_virtual_mode_valid(struct drm_connector *connector,
-					   struct drm_display_mode *mode)
-{
-	return MODE_OK;
-}
-
-static int
-amdgpu_connector_virtual_dpms(struct drm_connector *connector, int mode)
-{
-	return 0;
-}
-
-static enum drm_connector_status
-
-amdgpu_connector_virtual_detect(struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
-static int
-amdgpu_connector_virtual_set_property(struct drm_connector *connector,
-				  struct drm_property *property,
-				  uint64_t val)
-{
-	return 0;
-}
-
-static void amdgpu_connector_virtual_force(struct drm_connector *connector)
-{
-	return;
-}
-
-static const struct drm_connector_helper_funcs amdgpu_connector_virtual_helper_funcs = {
-	.get_modes = amdgpu_connector_virtual_get_modes,
-	.mode_valid = amdgpu_connector_virtual_mode_valid,
-	.best_encoder = amdgpu_connector_virtual_encoder,
-};
-
-static const struct drm_connector_funcs amdgpu_connector_virtual_funcs = {
-	.dpms = amdgpu_connector_virtual_dpms,
-	.detect = amdgpu_connector_virtual_detect,
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = amdgpu_connector_virtual_set_property,
-	.destroy = amdgpu_connector_destroy,
-	.force = amdgpu_connector_virtual_force,
-};
-
 void
 amdgpu_connector_add(struct amdgpu_device *adev,
 		      uint32_t connector_id,
@@ -1983,17 +1901,6 @@ amdgpu_connector_add(struct amdgpu_device *adev,
 			connector->interlace_allowed = false;
 			connector->doublescan_allowed = false;
 			break;
-		case DRM_MODE_CONNECTOR_VIRTUAL:
-			amdgpu_dig_connector = kzalloc(sizeof(struct amdgpu_connector_atom_dig), GFP_KERNEL);
-			if (!amdgpu_dig_connector)
-				goto failed;
-			amdgpu_connector->con_priv = amdgpu_dig_connector;
-			drm_connector_init(dev, &amdgpu_connector->base, &amdgpu_connector_virtual_funcs, connector_type);
-			drm_connector_helper_add(&amdgpu_connector->base, &amdgpu_connector_virtual_helper_funcs);
-			subpixel_order = SubPixelHorizontalRGB;
-			connector->interlace_allowed = false;
-			connector->doublescan_allowed = false;
-			break;
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 82dc8d20e28a..29d6d84d1c28 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -355,6 +355,7 @@ static void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev,
 static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
 				 struct amdgpu_bo *bo)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	u64 initial_bytes_moved;
 	uint32_t domain;
 	int r;
@@ -372,9 +373,9 @@ static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
 
 retry:
 	amdgpu_ttm_placement_from_domain(bo, domain);
-	initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
+	initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
 	r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
-	p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
+	p->bytes_moved += atomic64_read(&adev->num_bytes_moved) -
 		initial_bytes_moved;
 
 	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
@@ -387,9 +388,9 @@ retry:
 
 /* Last resort, try to evict something from the current working set */
 static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
-				struct amdgpu_bo_list_entry *lobj)
+				struct amdgpu_bo *validated)
 {
-	uint32_t domain = lobj->robj->allowed_domains;
+	uint32_t domain = validated->allowed_domains;
 	int r;
 
 	if (!p->evictable)
@@ -400,11 +401,12 @@ static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
 
 		struct amdgpu_bo_list_entry *candidate = p->evictable;
 		struct amdgpu_bo *bo = candidate->robj;
+		struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 		u64 initial_bytes_moved;
 		uint32_t other;
 
 		/* If we reached our current BO we can forget it */
-		if (candidate == lobj)
+		if (candidate->robj == validated)
 			break;
 
 		other = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
@@ -420,9 +422,9 @@ static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
 
 		/* Good we can try to move this BO somewhere else */
 		amdgpu_ttm_placement_from_domain(bo, other);
-		initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
+		initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
 		r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
-		p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
+		p->bytes_moved += atomic64_read(&adev->num_bytes_moved) -
 			initial_bytes_moved;
 
 		if (unlikely(r))
@@ -437,6 +439,23 @@ static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
 	return false;
 }
 
+static int amdgpu_cs_validate(void *param, struct amdgpu_bo *bo)
+{
+	struct amdgpu_cs_parser *p = param;
+	int r;
+
+	do {
+		r = amdgpu_cs_bo_validate(p, bo);
+	} while (r == -ENOMEM && amdgpu_cs_try_evict(p, bo));
+	if (r)
+		return r;
+
+	if (bo->shadow)
+		r = amdgpu_cs_bo_validate(p, bo->shadow);
+
+	return r;
+}
+
 static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
 			    struct list_head *validated)
 {
@@ -464,18 +483,10 @@ static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
 		if (p->evictable == lobj)
 			p->evictable = NULL;
 
-		do {
-			r = amdgpu_cs_bo_validate(p, bo);
-		} while (r == -ENOMEM && amdgpu_cs_try_evict(p, lobj));
+		r = amdgpu_cs_validate(p, bo);
 		if (r)
 			return r;
 
-		if (bo->shadow) {
-			r = amdgpu_cs_bo_validate(p, bo);
-			if (r)
-				return r;
-		}
-
 		if (binding_userptr) {
 			drm_free_large(lobj->user_pages);
 			lobj->user_pages = NULL;
@@ -594,14 +605,19 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 		list_splice(&need_pages, &p->validated);
 	}
 
-	amdgpu_vm_get_pt_bos(p->adev, &fpriv->vm, &duplicates);
-
 	p->bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(p->adev);
 	p->bytes_moved = 0;
 	p->evictable = list_last_entry(&p->validated,
 				       struct amdgpu_bo_list_entry,
 				       tv.head);
 
+	r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm,
+				      amdgpu_cs_validate, p);
+	if (r) {
+		DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n");
+		goto error_validate;
+	}
+
 	r = amdgpu_cs_list_validate(p, &duplicates);
 	if (r) {
 		DRM_ERROR("amdgpu_cs_list_validate(duplicates) failed.\n");
@@ -720,7 +736,7 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bo
 		ttm_eu_backoff_reservation(&parser->ticket,
 					   &parser->validated);
 	}
-	fence_put(parser->fence);
+	dma_fence_put(parser->fence);
 
 	if (parser->ctx)
 		amdgpu_ctx_put(parser->ctx);
@@ -757,7 +773,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
 
 	if (p->bo_list) {
 		for (i = 0; i < p->bo_list->num_entries; i++) {
-			struct fence *f;
+			struct dma_fence *f;
 
 			/* ignore duplicates */
 			bo = p->bo_list->array[i].robj;
@@ -807,13 +823,14 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
 
 	/* Only for UVD/VCE VM emulation */
 	if (ring->funcs->parse_cs) {
-		p->job->vm = NULL;
 		for (i = 0; i < p->job->num_ibs; i++) {
 			r = amdgpu_ring_parse_cs(ring, p, i);
 			if (r)
 				return r;
 		}
-	} else {
+	}
+
+	if (p->job->vm) {
 		p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
 
 		r = amdgpu_bo_vm_update_pte(p, vm);
@@ -824,16 +841,6 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
 	return amdgpu_cs_sync_rings(p);
 }
 
-static int amdgpu_cs_handle_lockup(struct amdgpu_device *adev, int r)
-{
-	if (r == -EDEADLK) {
-		r = amdgpu_gpu_reset(adev);
-		if (!r)
-			r = -EAGAIN;
-	}
-	return r;
-}
-
 static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
 			     struct amdgpu_cs_parser *parser)
 {
@@ -902,7 +909,7 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
 			offset = ((uint64_t)m->it.start) * AMDGPU_GPU_PAGE_SIZE;
 			kptr += chunk_ib->va_start - offset;
 
-			r =  amdgpu_ib_get(adev, NULL, chunk_ib->ib_bytes, ib);
+			r =  amdgpu_ib_get(adev, vm, chunk_ib->ib_bytes, ib);
 			if (r) {
 				DRM_ERROR("Failed to get ib !\n");
 				return r;
@@ -917,9 +924,9 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
 				return r;
 			}
 
-			ib->gpu_addr = chunk_ib->va_start;
 		}
 
+		ib->gpu_addr = chunk_ib->va_start;
 		ib->length_dw = chunk_ib->ib_bytes / 4;
 		ib->flags = chunk_ib->flags;
 		j++;
@@ -927,8 +934,8 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
 
 	/* UVD & VCE fw doesn't support user fences */
 	if (parser->job->uf_addr && (
-	    parser->job->ring->type == AMDGPU_RING_TYPE_UVD ||
-	    parser->job->ring->type == AMDGPU_RING_TYPE_VCE))
+	    parser->job->ring->funcs->type == AMDGPU_RING_TYPE_UVD ||
+	    parser->job->ring->funcs->type == AMDGPU_RING_TYPE_VCE))
 		return -EINVAL;
 
 	return 0;
@@ -957,7 +964,7 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
 		for (j = 0; j < num_deps; ++j) {
 			struct amdgpu_ring *ring;
 			struct amdgpu_ctx *ctx;
-			struct fence *fence;
+			struct dma_fence *fence;
 
 			r = amdgpu_cs_get_ring(adev, deps[j].ip_type,
 					       deps[j].ip_instance,
@@ -979,7 +986,7 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
 			} else if (fence) {
 				r = amdgpu_sync_fence(adev, &p->job->sync,
 						      fence);
-				fence_put(fence);
+				dma_fence_put(fence);
 				amdgpu_ctx_put(ctx);
 				if (r)
 					return r;
@@ -1009,7 +1016,7 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 
 	job->owner = p->filp;
 	job->fence_ctx = entity->fence_context;
-	p->fence = fence_get(&job->base.s_fence->finished);
+	p->fence = dma_fence_get(&job->base.s_fence->finished);
 	cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, p->fence);
 	job->uf_sequence = cs->out.handle;
 	amdgpu_job_free_resources(job);
@@ -1037,29 +1044,29 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 	r = amdgpu_cs_parser_init(&parser, data);
 	if (r) {
 		DRM_ERROR("Failed to initialize parser !\n");
-		amdgpu_cs_parser_fini(&parser, r, false);
-		r = amdgpu_cs_handle_lockup(adev, r);
-		return r;
-	}
-	r = amdgpu_cs_parser_bos(&parser, data);
-	if (r == -ENOMEM)
-		DRM_ERROR("Not enough memory for command submission!\n");
-	else if (r && r != -ERESTARTSYS)
-		DRM_ERROR("Failed to process the buffer list %d!\n", r);
-	else if (!r) {
-		reserved_buffers = true;
-		r = amdgpu_cs_ib_fill(adev, &parser);
+		goto out;
 	}
 
-	if (!r) {
-		r = amdgpu_cs_dependencies(adev, &parser);
-		if (r)
-			DRM_ERROR("Failed in the dependencies handling %d!\n", r);
+	r = amdgpu_cs_parser_bos(&parser, data);
+	if (r) {
+		if (r == -ENOMEM)
+			DRM_ERROR("Not enough memory for command submission!\n");
+		else if (r != -ERESTARTSYS)
+			DRM_ERROR("Failed to process the buffer list %d!\n", r);
+		goto out;
 	}
 
+	reserved_buffers = true;
+	r = amdgpu_cs_ib_fill(adev, &parser);
 	if (r)
 		goto out;
 
+	r = amdgpu_cs_dependencies(adev, &parser);
+	if (r) {
+		DRM_ERROR("Failed in the dependencies handling %d!\n", r);
+		goto out;
+	}
+
 	for (i = 0; i < parser.job->num_ibs; i++)
 		trace_amdgpu_cs(&parser, i);
 
@@ -1071,7 +1078,6 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 
 out:
 	amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
-	r = amdgpu_cs_handle_lockup(adev, r);
 	return r;
 }
 
@@ -1092,7 +1098,7 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
 	unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
 	struct amdgpu_ring *ring = NULL;
 	struct amdgpu_ctx *ctx;
-	struct fence *fence;
+	struct dma_fence *fence;
 	long r;
 
 	r = amdgpu_cs_get_ring(adev, wait->in.ip_type, wait->in.ip_instance,
@@ -1108,8 +1114,8 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
 	if (IS_ERR(fence))
 		r = PTR_ERR(fence);
 	else if (fence) {
-		r = fence_wait_timeout(fence, true, timeout);
-		fence_put(fence);
+		r = dma_fence_wait_timeout(fence, true, timeout);
+		dma_fence_put(fence);
 	} else
 		r = 1;
 
@@ -1124,6 +1130,180 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
 }
 
 /**
+ * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence
+ *
+ * @adev: amdgpu device
+ * @filp: file private
+ * @user: drm_amdgpu_fence copied from user space
+ */
+static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev,
+					     struct drm_file *filp,
+					     struct drm_amdgpu_fence *user)
+{
+	struct amdgpu_ring *ring;
+	struct amdgpu_ctx *ctx;
+	struct dma_fence *fence;
+	int r;
+
+	r = amdgpu_cs_get_ring(adev, user->ip_type, user->ip_instance,
+			       user->ring, &ring);
+	if (r)
+		return ERR_PTR(r);
+
+	ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id);
+	if (ctx == NULL)
+		return ERR_PTR(-EINVAL);
+
+	fence = amdgpu_ctx_get_fence(ctx, ring, user->seq_no);
+	amdgpu_ctx_put(ctx);
+
+	return fence;
+}
+
+/**
+ * amdgpu_cs_wait_all_fence - wait on all fences to signal
+ *
+ * @adev: amdgpu device
+ * @filp: file private
+ * @wait: wait parameters
+ * @fences: array of drm_amdgpu_fence
+ */
+static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev,
+				     struct drm_file *filp,
+				     union drm_amdgpu_wait_fences *wait,
+				     struct drm_amdgpu_fence *fences)
+{
+	uint32_t fence_count = wait->in.fence_count;
+	unsigned int i;
+	long r = 1;
+
+	for (i = 0; i < fence_count; i++) {
+		struct dma_fence *fence;
+		unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
+
+		fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
+		if (IS_ERR(fence))
+			return PTR_ERR(fence);
+		else if (!fence)
+			continue;
+
+		r = dma_fence_wait_timeout(fence, true, timeout);
+		if (r < 0)
+			return r;
+
+		if (r == 0)
+			break;
+	}
+
+	memset(wait, 0, sizeof(*wait));
+	wait->out.status = (r > 0);
+
+	return 0;
+}
+
+/**
+ * amdgpu_cs_wait_any_fence - wait on any fence to signal
+ *
+ * @adev: amdgpu device
+ * @filp: file private
+ * @wait: wait parameters
+ * @fences: array of drm_amdgpu_fence
+ */
+static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev,
+				    struct drm_file *filp,
+				    union drm_amdgpu_wait_fences *wait,
+				    struct drm_amdgpu_fence *fences)
+{
+	unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
+	uint32_t fence_count = wait->in.fence_count;
+	uint32_t first = ~0;
+	struct dma_fence **array;
+	unsigned int i;
+	long r;
+
+	/* Prepare the fence array */
+	array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL);
+
+	if (array == NULL)
+		return -ENOMEM;
+
+	for (i = 0; i < fence_count; i++) {
+		struct dma_fence *fence;
+
+		fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
+		if (IS_ERR(fence)) {
+			r = PTR_ERR(fence);
+			goto err_free_fence_array;
+		} else if (fence) {
+			array[i] = fence;
+		} else { /* NULL, the fence has been already signaled */
+			r = 1;
+			goto out;
+		}
+	}
+
+	r = dma_fence_wait_any_timeout(array, fence_count, true, timeout,
+				       &first);
+	if (r < 0)
+		goto err_free_fence_array;
+
+out:
+	memset(wait, 0, sizeof(*wait));
+	wait->out.status = (r > 0);
+	wait->out.first_signaled = first;
+	/* set return value 0 to indicate success */
+	r = 0;
+
+err_free_fence_array:
+	for (i = 0; i < fence_count; i++)
+		dma_fence_put(array[i]);
+	kfree(array);
+
+	return r;
+}
+
+/**
+ * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish
+ *
+ * @dev: drm device
+ * @data: data from userspace
+ * @filp: file private
+ */
+int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
+				struct drm_file *filp)
+{
+	struct amdgpu_device *adev = dev->dev_private;
+	union drm_amdgpu_wait_fences *wait = data;
+	uint32_t fence_count = wait->in.fence_count;
+	struct drm_amdgpu_fence *fences_user;
+	struct drm_amdgpu_fence *fences;
+	int r;
+
+	/* Get the fences from userspace */
+	fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence),
+			GFP_KERNEL);
+	if (fences == NULL)
+		return -ENOMEM;
+
+	fences_user = (void __user *)(unsigned long)(wait->in.fences);
+	if (copy_from_user(fences, fences_user,
+		sizeof(struct drm_amdgpu_fence) * fence_count)) {
+		r = -EFAULT;
+		goto err_free_fences;
+	}
+
+	if (wait->in.wait_all)
+		r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences);
+	else
+		r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences);
+
+err_free_fences:
+	kfree(fences);
+
+	return r;
+}
+
+/**
  * amdgpu_cs_find_bo_va - find bo_va for VM address
  *
  * @parser: command submission parser context
@@ -1196,6 +1376,15 @@ int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser *parser)
 		r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
 		if (unlikely(r))
 			return r;
+
+		if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
+			continue;
+
+		bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+		amdgpu_ttm_placement_from_domain(bo, bo->allowed_domains);
+		r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+		if (unlikely(r))
+			return r;
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index a5e2fcbef0f0..400c66ba4c6b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -35,7 +35,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, struct amdgpu_ctx *ctx)
 	kref_init(&ctx->refcount);
 	spin_lock_init(&ctx->ring_lock);
 	ctx->fences = kcalloc(amdgpu_sched_jobs * AMDGPU_MAX_RINGS,
-			      sizeof(struct fence*), GFP_KERNEL);
+			      sizeof(struct dma_fence*), GFP_KERNEL);
 	if (!ctx->fences)
 		return -ENOMEM;
 
@@ -55,18 +55,18 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, struct amdgpu_ctx *ctx)
 		r = amd_sched_entity_init(&ring->sched, &ctx->rings[i].entity,
 					  rq, amdgpu_sched_jobs);
 		if (r)
-			break;
+			goto failed;
 	}
 
-	if (i < adev->num_rings) {
-		for (j = 0; j < i; j++)
-			amd_sched_entity_fini(&adev->rings[j]->sched,
-					      &ctx->rings[j].entity);
-		kfree(ctx->fences);
-		ctx->fences = NULL;
-		return r;
-	}
 	return 0;
+
+failed:
+	for (j = 0; j < i; j++)
+		amd_sched_entity_fini(&adev->rings[j]->sched,
+				      &ctx->rings[j].entity);
+	kfree(ctx->fences);
+	ctx->fences = NULL;
+	return r;
 }
 
 static void amdgpu_ctx_fini(struct amdgpu_ctx *ctx)
@@ -79,7 +79,7 @@ static void amdgpu_ctx_fini(struct amdgpu_ctx *ctx)
 
 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
 		for (j = 0; j < amdgpu_sched_jobs; ++j)
-			fence_put(ctx->rings[i].fences[j]);
+			dma_fence_put(ctx->rings[i].fences[j]);
 	kfree(ctx->fences);
 	ctx->fences = NULL;
 
@@ -241,39 +241,39 @@ int amdgpu_ctx_put(struct amdgpu_ctx *ctx)
 }
 
 uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
-			      struct fence *fence)
+			      struct dma_fence *fence)
 {
 	struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx];
 	uint64_t seq = cring->sequence;
 	unsigned idx = 0;
-	struct fence *other = NULL;
+	struct dma_fence *other = NULL;
 
 	idx = seq & (amdgpu_sched_jobs - 1);
 	other = cring->fences[idx];
 	if (other) {
 		signed long r;
-		r = fence_wait_timeout(other, false, MAX_SCHEDULE_TIMEOUT);
+		r = dma_fence_wait_timeout(other, false, MAX_SCHEDULE_TIMEOUT);
 		if (r < 0)
 			DRM_ERROR("Error (%ld) waiting for fence!\n", r);
 	}
 
-	fence_get(fence);
+	dma_fence_get(fence);
 
 	spin_lock(&ctx->ring_lock);
 	cring->fences[idx] = fence;
 	cring->sequence++;
 	spin_unlock(&ctx->ring_lock);
 
-	fence_put(other);
+	dma_fence_put(other);
 
 	return seq;
 }
 
-struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
-				   struct amdgpu_ring *ring, uint64_t seq)
+struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
+				       struct amdgpu_ring *ring, uint64_t seq)
 {
 	struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx];
-	struct fence *fence;
+	struct dma_fence *fence;
 
 	spin_lock(&ctx->ring_lock);
 
@@ -288,7 +288,7 @@ struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
 		return NULL;
 	}
 
-	fence = fence_get(cring->fences[seq & (amdgpu_sched_jobs - 1)]);
+	fence = dma_fence_get(cring->fences[seq & (amdgpu_sched_jobs - 1)]);
 	spin_unlock(&ctx->ring_lock);
 
 	return fence;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index e41d4baebf86..cc8aafd9cb0d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -264,7 +264,8 @@ static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
 	if (adev->vram_scratch.robj == NULL) {
 		r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE,
 				     PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
-				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+				     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 				     NULL, NULL, &adev->vram_scratch.robj);
 		if (r) {
 			return r;
@@ -442,13 +443,9 @@ void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
 static void amdgpu_wb_fini(struct amdgpu_device *adev)
 {
 	if (adev->wb.wb_obj) {
-		if (!amdgpu_bo_reserve(adev->wb.wb_obj, false)) {
-			amdgpu_bo_kunmap(adev->wb.wb_obj);
-			amdgpu_bo_unpin(adev->wb.wb_obj);
-			amdgpu_bo_unreserve(adev->wb.wb_obj);
-		}
-		amdgpu_bo_unref(&adev->wb.wb_obj);
-		adev->wb.wb = NULL;
+		amdgpu_bo_free_kernel(&adev->wb.wb_obj,
+				      &adev->wb.gpu_addr,
+				      (void **)&adev->wb.wb);
 		adev->wb.wb_obj = NULL;
 	}
 }
@@ -467,33 +464,14 @@ static int amdgpu_wb_init(struct amdgpu_device *adev)
 	int r;
 
 	if (adev->wb.wb_obj == NULL) {
-		r = amdgpu_bo_create(adev, AMDGPU_MAX_WB * 4, PAGE_SIZE, true,
-				     AMDGPU_GEM_DOMAIN_GTT, 0,  NULL, NULL,
-				     &adev->wb.wb_obj);
+		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * 4,
+					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
+					    &adev->wb.wb_obj, &adev->wb.gpu_addr,
+					    (void **)&adev->wb.wb);
 		if (r) {
 			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
 			return r;
 		}
-		r = amdgpu_bo_reserve(adev->wb.wb_obj, false);
-		if (unlikely(r != 0)) {
-			amdgpu_wb_fini(adev);
-			return r;
-		}
-		r = amdgpu_bo_pin(adev->wb.wb_obj, AMDGPU_GEM_DOMAIN_GTT,
-				&adev->wb.gpu_addr);
-		if (r) {
-			amdgpu_bo_unreserve(adev->wb.wb_obj);
-			dev_warn(adev->dev, "(%d) pin WB bo failed\n", r);
-			amdgpu_wb_fini(adev);
-			return r;
-		}
-		r = amdgpu_bo_kmap(adev->wb.wb_obj, (void **)&adev->wb.wb);
-		amdgpu_bo_unreserve(adev->wb.wb_obj);
-		if (r) {
-			dev_warn(adev->dev, "(%d) map WB bo failed\n", r);
-			amdgpu_wb_fini(adev);
-			return r;
-		}
 
 		adev->wb.num_wb = AMDGPU_MAX_WB;
 		memset(&adev->wb.used, 0, sizeof(adev->wb.used));
@@ -1038,6 +1016,13 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev)
 			 amdgpu_vm_block_size);
 		amdgpu_vm_block_size = 9;
 	}
+
+	if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
+	    !amdgpu_check_pot_argument(amdgpu_vram_page_split))) {
+		dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
+			 amdgpu_vram_page_split);
+		amdgpu_vram_page_split = 1024;
+	}
 }
 
 /**
@@ -1112,11 +1097,11 @@ int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
 	int i, r = 0;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
-		if (adev->ip_blocks[i].type == block_type) {
-			r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
-									    state);
+		if (adev->ip_blocks[i].version->type == block_type) {
+			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
+										     state);
 			if (r)
 				return r;
 			break;
@@ -1132,11 +1117,11 @@ int amdgpu_set_powergating_state(struct amdgpu_device *adev,
 	int i, r = 0;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
-		if (adev->ip_blocks[i].type == block_type) {
-			r = adev->ip_blocks[i].funcs->set_powergating_state((void *)adev,
-									    state);
+		if (adev->ip_blocks[i].version->type == block_type) {
+			r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
+										     state);
 			if (r)
 				return r;
 			break;
@@ -1151,10 +1136,10 @@ int amdgpu_wait_for_idle(struct amdgpu_device *adev,
 	int i, r;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
-		if (adev->ip_blocks[i].type == block_type) {
-			r = adev->ip_blocks[i].funcs->wait_for_idle((void *)adev);
+		if (adev->ip_blocks[i].version->type == block_type) {
+			r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
 			if (r)
 				return r;
 			break;
@@ -1170,23 +1155,22 @@ bool amdgpu_is_idle(struct amdgpu_device *adev,
 	int i;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
-		if (adev->ip_blocks[i].type == block_type)
-			return adev->ip_blocks[i].funcs->is_idle((void *)adev);
+		if (adev->ip_blocks[i].version->type == block_type)
+			return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
 	}
 	return true;
 
 }
 
-const struct amdgpu_ip_block_version * amdgpu_get_ip_block(
-					struct amdgpu_device *adev,
-					enum amd_ip_block_type type)
+struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
+					     enum amd_ip_block_type type)
 {
 	int i;
 
 	for (i = 0; i < adev->num_ip_blocks; i++)
-		if (adev->ip_blocks[i].type == type)
+		if (adev->ip_blocks[i].version->type == type)
 			return &adev->ip_blocks[i];
 
 	return NULL;
@@ -1207,38 +1191,75 @@ int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
 				enum amd_ip_block_type type,
 				u32 major, u32 minor)
 {
-	const struct amdgpu_ip_block_version *ip_block;
-	ip_block = amdgpu_get_ip_block(adev, type);
+	struct amdgpu_ip_block *ip_block = amdgpu_get_ip_block(adev, type);
 
-	if (ip_block && ((ip_block->major > major) ||
-			((ip_block->major == major) &&
-			(ip_block->minor >= minor))))
+	if (ip_block && ((ip_block->version->major > major) ||
+			((ip_block->version->major == major) &&
+			(ip_block->version->minor >= minor))))
 		return 0;
 
 	return 1;
 }
 
-static void amdgpu_whether_enable_virtual_display(struct amdgpu_device *adev)
+/**
+ * amdgpu_ip_block_add
+ *
+ * @adev: amdgpu_device pointer
+ * @ip_block_version: pointer to the IP to add
+ *
+ * Adds the IP block driver information to the collection of IPs
+ * on the asic.
+ */
+int amdgpu_ip_block_add(struct amdgpu_device *adev,
+			const struct amdgpu_ip_block_version *ip_block_version)
+{
+	if (!ip_block_version)
+		return -EINVAL;
+
+	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
+
+	return 0;
+}
+
+static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
 {
 	adev->enable_virtual_display = false;
 
 	if (amdgpu_virtual_display) {
 		struct drm_device *ddev = adev->ddev;
 		const char *pci_address_name = pci_name(ddev->pdev);
-		char *pciaddstr, *pciaddstr_tmp, *pciaddname;
+		char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
 
 		pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
 		pciaddstr_tmp = pciaddstr;
-		while ((pciaddname = strsep(&pciaddstr_tmp, ";"))) {
+		while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
+			pciaddname = strsep(&pciaddname_tmp, ",");
 			if (!strcmp(pci_address_name, pciaddname)) {
+				long num_crtc;
+				int res = -1;
+
 				adev->enable_virtual_display = true;
+
+				if (pciaddname_tmp)
+					res = kstrtol(pciaddname_tmp, 10,
+						      &num_crtc);
+
+				if (!res) {
+					if (num_crtc < 1)
+						num_crtc = 1;
+					if (num_crtc > 6)
+						num_crtc = 6;
+					adev->mode_info.num_crtc = num_crtc;
+				} else {
+					adev->mode_info.num_crtc = 1;
+				}
 				break;
 			}
 		}
 
-		DRM_INFO("virtual display string:%s, %s:virtual_display:%d\n",
-				 amdgpu_virtual_display, pci_address_name,
-				 adev->enable_virtual_display);
+		DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
+			 amdgpu_virtual_display, pci_address_name,
+			 adev->enable_virtual_display, adev->mode_info.num_crtc);
 
 		kfree(pciaddstr);
 	}
@@ -1248,7 +1269,7 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
 {
 	int i, r;
 
-	amdgpu_whether_enable_virtual_display(adev);
+	amdgpu_device_enable_virtual_display(adev);
 
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
@@ -1300,33 +1321,24 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
 		return -EINVAL;
 	}
 
-	adev->ip_block_status = kcalloc(adev->num_ip_blocks,
-					sizeof(struct amdgpu_ip_block_status), GFP_KERNEL);
-	if (adev->ip_block_status == NULL)
-		return -ENOMEM;
-
-	if (adev->ip_blocks == NULL) {
-		DRM_ERROR("No IP blocks found!\n");
-		return r;
-	}
-
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
 			DRM_ERROR("disabled ip block: %d\n", i);
-			adev->ip_block_status[i].valid = false;
+			adev->ip_blocks[i].status.valid = false;
 		} else {
-			if (adev->ip_blocks[i].funcs->early_init) {
-				r = adev->ip_blocks[i].funcs->early_init((void *)adev);
+			if (adev->ip_blocks[i].version->funcs->early_init) {
+				r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
 				if (r == -ENOENT) {
-					adev->ip_block_status[i].valid = false;
+					adev->ip_blocks[i].status.valid = false;
 				} else if (r) {
-					DRM_ERROR("early_init of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
+					DRM_ERROR("early_init of IP block <%s> failed %d\n",
+						  adev->ip_blocks[i].version->funcs->name, r);
 					return r;
 				} else {
-					adev->ip_block_status[i].valid = true;
+					adev->ip_blocks[i].status.valid = true;
 				}
 			} else {
-				adev->ip_block_status[i].valid = true;
+				adev->ip_blocks[i].status.valid = true;
 			}
 		}
 	}
@@ -1342,22 +1354,23 @@ static int amdgpu_init(struct amdgpu_device *adev)
 	int i, r;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
-		r = adev->ip_blocks[i].funcs->sw_init((void *)adev);
+		r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
 		if (r) {
-			DRM_ERROR("sw_init of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
+			DRM_ERROR("sw_init of IP block <%s> failed %d\n",
+				  adev->ip_blocks[i].version->funcs->name, r);
 			return r;
 		}
-		adev->ip_block_status[i].sw = true;
+		adev->ip_blocks[i].status.sw = true;
 		/* need to do gmc hw init early so we can allocate gpu mem */
-		if (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_GMC) {
+		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
 			r = amdgpu_vram_scratch_init(adev);
 			if (r) {
 				DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
 				return r;
 			}
-			r = adev->ip_blocks[i].funcs->hw_init((void *)adev);
+			r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
 			if (r) {
 				DRM_ERROR("hw_init %d failed %d\n", i, r);
 				return r;
@@ -1367,22 +1380,23 @@ static int amdgpu_init(struct amdgpu_device *adev)
 				DRM_ERROR("amdgpu_wb_init failed %d\n", r);
 				return r;
 			}
-			adev->ip_block_status[i].hw = true;
+			adev->ip_blocks[i].status.hw = true;
 		}
 	}
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].sw)
+		if (!adev->ip_blocks[i].status.sw)
 			continue;
 		/* gmc hw init is done early */
-		if (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_GMC)
+		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC)
 			continue;
-		r = adev->ip_blocks[i].funcs->hw_init((void *)adev);
+		r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
 		if (r) {
-			DRM_ERROR("hw_init of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
+			DRM_ERROR("hw_init of IP block <%s> failed %d\n",
+				  adev->ip_blocks[i].version->funcs->name, r);
 			return r;
 		}
-		adev->ip_block_status[i].hw = true;
+		adev->ip_blocks[i].status.hw = true;
 	}
 
 	return 0;
@@ -1393,25 +1407,26 @@ static int amdgpu_late_init(struct amdgpu_device *adev)
 	int i = 0, r;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
-		if (adev->ip_blocks[i].funcs->late_init) {
-			r = adev->ip_blocks[i].funcs->late_init((void *)adev);
+		if (adev->ip_blocks[i].version->funcs->late_init) {
+			r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
 			if (r) {
-				DRM_ERROR("late_init of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
+				DRM_ERROR("late_init of IP block <%s> failed %d\n",
+					  adev->ip_blocks[i].version->funcs->name, r);
 				return r;
 			}
-			adev->ip_block_status[i].late_initialized = true;
+			adev->ip_blocks[i].status.late_initialized = true;
 		}
 		/* skip CG for VCE/UVD, it's handled specially */
-		if (adev->ip_blocks[i].type != AMD_IP_BLOCK_TYPE_UVD &&
-		    adev->ip_blocks[i].type != AMD_IP_BLOCK_TYPE_VCE) {
+		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
+		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
 			/* enable clockgating to save power */
-			r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
-									    AMD_CG_STATE_GATE);
+			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
+										     AMD_CG_STATE_GATE);
 			if (r) {
 				DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
-					  adev->ip_blocks[i].funcs->name, r);
+					  adev->ip_blocks[i].version->funcs->name, r);
 				return r;
 			}
 		}
@@ -1426,68 +1441,77 @@ static int amdgpu_fini(struct amdgpu_device *adev)
 
 	/* need to disable SMC first */
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].hw)
+		if (!adev->ip_blocks[i].status.hw)
 			continue;
-		if (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_SMC) {
+		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
 			/* ungate blocks before hw fini so that we can shutdown the blocks safely */
-			r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
-									    AMD_CG_STATE_UNGATE);
+			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
+										     AMD_CG_STATE_UNGATE);
 			if (r) {
 				DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
-					  adev->ip_blocks[i].funcs->name, r);
+					  adev->ip_blocks[i].version->funcs->name, r);
 				return r;
 			}
-			r = adev->ip_blocks[i].funcs->hw_fini((void *)adev);
+			r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
 			/* XXX handle errors */
 			if (r) {
 				DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
-					  adev->ip_blocks[i].funcs->name, r);
+					  adev->ip_blocks[i].version->funcs->name, r);
 			}
-			adev->ip_block_status[i].hw = false;
+			adev->ip_blocks[i].status.hw = false;
 			break;
 		}
 	}
 
 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
-		if (!adev->ip_block_status[i].hw)
+		if (!adev->ip_blocks[i].status.hw)
 			continue;
-		if (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_GMC) {
+		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
 			amdgpu_wb_fini(adev);
 			amdgpu_vram_scratch_fini(adev);
 		}
-		/* ungate blocks before hw fini so that we can shutdown the blocks safely */
-		r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
-								    AMD_CG_STATE_UNGATE);
-		if (r) {
-			DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
-			return r;
+
+		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
+			adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
+			/* ungate blocks before hw fini so that we can shutdown the blocks safely */
+			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
+										     AMD_CG_STATE_UNGATE);
+			if (r) {
+				DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
+					  adev->ip_blocks[i].version->funcs->name, r);
+				return r;
+			}
 		}
-		r = adev->ip_blocks[i].funcs->hw_fini((void *)adev);
+
+		r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
 		/* XXX handle errors */
 		if (r) {
-			DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
+			DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
+				  adev->ip_blocks[i].version->funcs->name, r);
 		}
-		adev->ip_block_status[i].hw = false;
+
+		adev->ip_blocks[i].status.hw = false;
 	}
 
 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
-		if (!adev->ip_block_status[i].sw)
+		if (!adev->ip_blocks[i].status.sw)
 			continue;
-		r = adev->ip_blocks[i].funcs->sw_fini((void *)adev);
+		r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
 		/* XXX handle errors */
 		if (r) {
-			DRM_DEBUG("sw_fini of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
+			DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
+				  adev->ip_blocks[i].version->funcs->name, r);
 		}
-		adev->ip_block_status[i].sw = false;
-		adev->ip_block_status[i].valid = false;
+		adev->ip_blocks[i].status.sw = false;
+		adev->ip_blocks[i].status.valid = false;
 	}
 
 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
-		if (!adev->ip_block_status[i].late_initialized)
+		if (!adev->ip_blocks[i].status.late_initialized)
 			continue;
-		if (adev->ip_blocks[i].funcs->late_fini)
-			adev->ip_blocks[i].funcs->late_fini((void *)adev);
-		adev->ip_block_status[i].late_initialized = false;
+		if (adev->ip_blocks[i].version->funcs->late_fini)
+			adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
+		adev->ip_blocks[i].status.late_initialized = false;
 	}
 
 	return 0;
@@ -1505,21 +1529,23 @@ int amdgpu_suspend(struct amdgpu_device *adev)
 	}
 
 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
 		/* ungate blocks so that suspend can properly shut them down */
 		if (i != AMD_IP_BLOCK_TYPE_SMC) {
-			r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
-									    AMD_CG_STATE_UNGATE);
+			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
+										     AMD_CG_STATE_UNGATE);
 			if (r) {
-				DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
+				DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
+					  adev->ip_blocks[i].version->funcs->name, r);
 			}
 		}
 		/* XXX handle errors */
-		r = adev->ip_blocks[i].funcs->suspend(adev);
+		r = adev->ip_blocks[i].version->funcs->suspend(adev);
 		/* XXX handle errors */
 		if (r) {
-			DRM_ERROR("suspend of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
+			DRM_ERROR("suspend of IP block <%s> failed %d\n",
+				  adev->ip_blocks[i].version->funcs->name, r);
 		}
 	}
 
@@ -1531,11 +1557,12 @@ static int amdgpu_resume(struct amdgpu_device *adev)
 	int i, r;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
-		r = adev->ip_blocks[i].funcs->resume(adev);
+		r = adev->ip_blocks[i].version->funcs->resume(adev);
 		if (r) {
-			DRM_ERROR("resume of IP block <%s> failed %d\n", adev->ip_blocks[i].funcs->name, r);
+			DRM_ERROR("resume of IP block <%s> failed %d\n",
+				  adev->ip_blocks[i].version->funcs->name, r);
 			return r;
 		}
 	}
@@ -1586,7 +1613,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	adev->vm_manager.vm_pte_funcs = NULL;
 	adev->vm_manager.vm_pte_num_rings = 0;
 	adev->gart.gart_funcs = NULL;
-	adev->fence_context = fence_context_alloc(AMDGPU_MAX_RINGS);
+	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
 
 	adev->smc_rreg = &amdgpu_invalid_rreg;
 	adev->smc_wreg = &amdgpu_invalid_wreg;
@@ -1846,8 +1873,6 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
 	amdgpu_fence_driver_fini(adev);
 	amdgpu_fbdev_fini(adev);
 	r = amdgpu_fini(adev);
-	kfree(adev->ip_block_status);
-	adev->ip_block_status = NULL;
 	adev->accel_working = false;
 	/* free i2c buses */
 	amdgpu_i2c_fini(adev);
@@ -1943,7 +1968,10 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
 
 	r = amdgpu_suspend(adev);
 
-	/* evict remaining vram memory */
+	/* evict remaining vram memory
+	 * This second call to evict vram is to evict the gart page table
+	 * using the CPU.
+	 */
 	amdgpu_bo_evict_vram(adev);
 
 	amdgpu_atombios_scratch_regs_save(adev);
@@ -2085,13 +2113,13 @@ static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
 	bool asic_hang = false;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
-		if (adev->ip_blocks[i].funcs->check_soft_reset)
-			adev->ip_block_status[i].hang =
-				adev->ip_blocks[i].funcs->check_soft_reset(adev);
-		if (adev->ip_block_status[i].hang) {
-			DRM_INFO("IP block:%d is hang!\n", i);
+		if (adev->ip_blocks[i].version->funcs->check_soft_reset)
+			adev->ip_blocks[i].status.hang =
+				adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
+		if (adev->ip_blocks[i].status.hang) {
+			DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
 			asic_hang = true;
 		}
 	}
@@ -2103,11 +2131,11 @@ static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
 	int i, r = 0;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
-		if (adev->ip_block_status[i].hang &&
-		    adev->ip_blocks[i].funcs->pre_soft_reset) {
-			r = adev->ip_blocks[i].funcs->pre_soft_reset(adev);
+		if (adev->ip_blocks[i].status.hang &&
+		    adev->ip_blocks[i].version->funcs->pre_soft_reset) {
+			r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
 			if (r)
 				return r;
 		}
@@ -2121,13 +2149,13 @@ static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
 	int i;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
-		if ((adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_GMC) ||
-		    (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_SMC) ||
-		    (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_ACP) ||
-		    (adev->ip_blocks[i].type == AMD_IP_BLOCK_TYPE_DCE)) {
-			if (adev->ip_block_status[i].hang) {
+		if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
+		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
+		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
+		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)) {
+			if (adev->ip_blocks[i].status.hang) {
 				DRM_INFO("Some block need full reset!\n");
 				return true;
 			}
@@ -2141,11 +2169,11 @@ static int amdgpu_soft_reset(struct amdgpu_device *adev)
 	int i, r = 0;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
-		if (adev->ip_block_status[i].hang &&
-		    adev->ip_blocks[i].funcs->soft_reset) {
-			r = adev->ip_blocks[i].funcs->soft_reset(adev);
+		if (adev->ip_blocks[i].status.hang &&
+		    adev->ip_blocks[i].version->funcs->soft_reset) {
+			r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
 			if (r)
 				return r;
 		}
@@ -2159,11 +2187,11 @@ static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
 	int i, r = 0;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_block_status[i].valid)
+		if (!adev->ip_blocks[i].status.valid)
 			continue;
-		if (adev->ip_block_status[i].hang &&
-		    adev->ip_blocks[i].funcs->post_soft_reset)
-			r = adev->ip_blocks[i].funcs->post_soft_reset(adev);
+		if (adev->ip_blocks[i].status.hang &&
+		    adev->ip_blocks[i].version->funcs->post_soft_reset)
+			r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
 		if (r)
 			return r;
 	}
@@ -2182,7 +2210,7 @@ bool amdgpu_need_backup(struct amdgpu_device *adev)
 static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
 					   struct amdgpu_ring *ring,
 					   struct amdgpu_bo *bo,
-					   struct fence **fence)
+					   struct dma_fence **fence)
 {
 	uint32_t domain;
 	int r;
@@ -2298,30 +2326,30 @@ retry:
 		if (need_full_reset && amdgpu_need_backup(adev)) {
 			struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 			struct amdgpu_bo *bo, *tmp;
-			struct fence *fence = NULL, *next = NULL;
+			struct dma_fence *fence = NULL, *next = NULL;
 
 			DRM_INFO("recover vram bo from shadow\n");
 			mutex_lock(&adev->shadow_list_lock);
 			list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
 				amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
 				if (fence) {
-					r = fence_wait(fence, false);
+					r = dma_fence_wait(fence, false);
 					if (r) {
 						WARN(r, "recovery from shadow isn't comleted\n");
 						break;
 					}
 				}
 
-				fence_put(fence);
+				dma_fence_put(fence);
 				fence = next;
 			}
 			mutex_unlock(&adev->shadow_list_lock);
 			if (fence) {
-				r = fence_wait(fence, false);
+				r = dma_fence_wait(fence, false);
 				if (r)
 					WARN(r, "recovery from shadow isn't comleted\n");
 			}
-			fence_put(fence);
+			dma_fence_put(fence);
 		}
 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 			struct amdgpu_ring *ring = adev->rings[i];
@@ -2471,9 +2499,6 @@ int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
 	adev->debugfs_count = i;
 #if defined(CONFIG_DEBUG_FS)
 	drm_debugfs_create_files(files, nfiles,
-				 adev->ddev->control->debugfs_root,
-				 adev->ddev->control);
-	drm_debugfs_create_files(files, nfiles,
 				 adev->ddev->primary->debugfs_root,
 				 adev->ddev->primary);
 #endif
@@ -2488,9 +2513,6 @@ static void amdgpu_debugfs_remove_files(struct amdgpu_device *adev)
 	for (i = 0; i < adev->debugfs_count; i++) {
 		drm_debugfs_remove_files(adev->debugfs[i].files,
 					 adev->debugfs[i].num_files,
-					 adev->ddev->control);
-		drm_debugfs_remove_files(adev->debugfs[i].files,
-					 adev->debugfs[i].num_files,
 					 adev->ddev->primary);
 	}
 #endif
@@ -2517,6 +2539,13 @@ static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 		se_bank = (*pos >> 24) & 0x3FF;
 		sh_bank = (*pos >> 34) & 0x3FF;
 		instance_bank = (*pos >> 44) & 0x3FF;
+
+		if (se_bank == 0x3FF)
+			se_bank = 0xFFFFFFFF;
+		if (sh_bank == 0x3FF)
+			sh_bank = 0xFFFFFFFF;
+		if (instance_bank == 0x3FF)
+			instance_bank = 0xFFFFFFFF;
 		use_bank = 1;
 	} else {
 		use_bank = 0;
@@ -2525,8 +2554,8 @@ static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 	*pos &= 0x3FFFF;
 
 	if (use_bank) {
-		if (sh_bank >= adev->gfx.config.max_sh_per_se ||
-		    se_bank >= adev->gfx.config.max_shader_engines)
+		if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
+		    (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
 			return -EINVAL;
 		mutex_lock(&adev->grbm_idx_mutex);
 		amdgpu_gfx_select_se_sh(adev, se_bank,
@@ -2573,10 +2602,45 @@ static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 	struct amdgpu_device *adev = f->f_inode->i_private;
 	ssize_t result = 0;
 	int r;
+	bool pm_pg_lock, use_bank;
+	unsigned instance_bank, sh_bank, se_bank;
 
 	if (size & 0x3 || *pos & 0x3)
 		return -EINVAL;
 
+	/* are we reading registers for which a PG lock is necessary? */
+	pm_pg_lock = (*pos >> 23) & 1;
+
+	if (*pos & (1ULL << 62)) {
+		se_bank = (*pos >> 24) & 0x3FF;
+		sh_bank = (*pos >> 34) & 0x3FF;
+		instance_bank = (*pos >> 44) & 0x3FF;
+
+		if (se_bank == 0x3FF)
+			se_bank = 0xFFFFFFFF;
+		if (sh_bank == 0x3FF)
+			sh_bank = 0xFFFFFFFF;
+		if (instance_bank == 0x3FF)
+			instance_bank = 0xFFFFFFFF;
+		use_bank = 1;
+	} else {
+		use_bank = 0;
+	}
+
+	*pos &= 0x3FFFF;
+
+	if (use_bank) {
+		if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
+		    (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
+			return -EINVAL;
+		mutex_lock(&adev->grbm_idx_mutex);
+		amdgpu_gfx_select_se_sh(adev, se_bank,
+					sh_bank, instance_bank);
+	}
+
+	if (pm_pg_lock)
+		mutex_lock(&adev->pm.mutex);
+
 	while (size) {
 		uint32_t value;
 
@@ -2595,6 +2659,14 @@ static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 		size -= 4;
 	}
 
+	if (use_bank) {
+		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+		mutex_unlock(&adev->grbm_idx_mutex);
+	}
+
+	if (pm_pg_lock)
+		mutex_unlock(&adev->pm.mutex);
+
 	return result;
 }
 
@@ -2857,6 +2929,116 @@ static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 	return !r ? 4 : r;
 }
 
+static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
+					size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = f->f_inode->i_private;
+	int r, x;
+	ssize_t result=0;
+	uint32_t offset, se, sh, cu, wave, simd, data[32];
+
+	if (size & 3 || *pos & 3)
+		return -EINVAL;
+
+	/* decode offset */
+	offset = (*pos & 0x7F);
+	se = ((*pos >> 7) & 0xFF);
+	sh = ((*pos >> 15) & 0xFF);
+	cu = ((*pos >> 23) & 0xFF);
+	wave = ((*pos >> 31) & 0xFF);
+	simd = ((*pos >> 37) & 0xFF);
+
+	/* switch to the specific se/sh/cu */
+	mutex_lock(&adev->grbm_idx_mutex);
+	amdgpu_gfx_select_se_sh(adev, se, sh, cu);
+
+	x = 0;
+	if (adev->gfx.funcs->read_wave_data)
+		adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
+
+	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
+	mutex_unlock(&adev->grbm_idx_mutex);
+
+	if (!x)
+		return -EINVAL;
+
+	while (size && (offset < x * 4)) {
+		uint32_t value;
+
+		value = data[offset >> 2];
+		r = put_user(value, (uint32_t *)buf);
+		if (r)
+			return r;
+
+		result += 4;
+		buf += 4;
+		offset += 4;
+		size -= 4;
+	}
+
+	return result;
+}
+
+static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
+					size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = f->f_inode->i_private;
+	int r;
+	ssize_t result = 0;
+	uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
+
+	if (size & 3 || *pos & 3)
+		return -EINVAL;
+
+	/* decode offset */
+	offset = (*pos & 0xFFF);       /* in dwords */
+	se = ((*pos >> 12) & 0xFF);
+	sh = ((*pos >> 20) & 0xFF);
+	cu = ((*pos >> 28) & 0xFF);
+	wave = ((*pos >> 36) & 0xFF);
+	simd = ((*pos >> 44) & 0xFF);
+	thread = ((*pos >> 52) & 0xFF);
+	bank = ((*pos >> 60) & 1);
+
+	data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	/* switch to the specific se/sh/cu */
+	mutex_lock(&adev->grbm_idx_mutex);
+	amdgpu_gfx_select_se_sh(adev, se, sh, cu);
+
+	if (bank == 0) {
+		if (adev->gfx.funcs->read_wave_vgprs)
+			adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
+	} else {
+		if (adev->gfx.funcs->read_wave_sgprs)
+			adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
+	}
+
+	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
+	mutex_unlock(&adev->grbm_idx_mutex);
+
+	while (size) {
+		uint32_t value;
+
+		value = data[offset++];
+		r = put_user(value, (uint32_t *)buf);
+		if (r) {
+			result = r;
+			goto err;
+		}
+
+		result += 4;
+		buf += 4;
+		size -= 4;
+	}
+
+err:
+	kfree(data);
+	return result;
+}
+
 static const struct file_operations amdgpu_debugfs_regs_fops = {
 	.owner = THIS_MODULE,
 	.read = amdgpu_debugfs_regs_read,
@@ -2894,6 +3076,17 @@ static const struct file_operations amdgpu_debugfs_sensors_fops = {
 	.llseek = default_llseek
 };
 
+static const struct file_operations amdgpu_debugfs_wave_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_wave_read,
+	.llseek = default_llseek
+};
+static const struct file_operations amdgpu_debugfs_gpr_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_gpr_read,
+	.llseek = default_llseek
+};
+
 static const struct file_operations *debugfs_regs[] = {
 	&amdgpu_debugfs_regs_fops,
 	&amdgpu_debugfs_regs_didt_fops,
@@ -2901,6 +3094,8 @@ static const struct file_operations *debugfs_regs[] = {
 	&amdgpu_debugfs_regs_smc_fops,
 	&amdgpu_debugfs_gca_config_fops,
 	&amdgpu_debugfs_sensors_fops,
+	&amdgpu_debugfs_wave_fops,
+	&amdgpu_debugfs_gpr_fops,
 };
 
 static const char *debugfs_regs_names[] = {
@@ -2910,6 +3105,8 @@ static const char *debugfs_regs_names[] = {
 	"amdgpu_regs_smc",
 	"amdgpu_gca_config",
 	"amdgpu_sensors",
+	"amdgpu_wave",
+	"amdgpu_gpr",
 };
 
 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index 083e2b429872..581601ca6b89 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -35,29 +35,29 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
 
-static void amdgpu_flip_callback(struct fence *f, struct fence_cb *cb)
+static void amdgpu_flip_callback(struct dma_fence *f, struct dma_fence_cb *cb)
 {
 	struct amdgpu_flip_work *work =
 		container_of(cb, struct amdgpu_flip_work, cb);
 
-	fence_put(f);
+	dma_fence_put(f);
 	schedule_work(&work->flip_work.work);
 }
 
 static bool amdgpu_flip_handle_fence(struct amdgpu_flip_work *work,
-				     struct fence **f)
+				     struct dma_fence **f)
 {
-	struct fence *fence= *f;
+	struct dma_fence *fence= *f;
 
 	if (fence == NULL)
 		return false;
 
 	*f = NULL;
 
-	if (!fence_add_callback(fence, &work->cb, amdgpu_flip_callback))
+	if (!dma_fence_add_callback(fence, &work->cb, amdgpu_flip_callback))
 		return true;
 
-	fence_put(fence);
+	dma_fence_put(fence);
 	return false;
 }
 
@@ -68,9 +68,9 @@ static void amdgpu_flip_work_func(struct work_struct *__work)
 	struct amdgpu_flip_work *work =
 		container_of(delayed_work, struct amdgpu_flip_work, flip_work);
 	struct amdgpu_device *adev = work->adev;
-	struct amdgpu_crtc *amdgpuCrtc = adev->mode_info.crtcs[work->crtc_id];
+	struct amdgpu_crtc *amdgpu_crtc = adev->mode_info.crtcs[work->crtc_id];
 
-	struct drm_crtc *crtc = &amdgpuCrtc->base;
+	struct drm_crtc *crtc = &amdgpu_crtc->base;
 	unsigned long flags;
 	unsigned i;
 	int vpos, hpos;
@@ -85,14 +85,14 @@ static void amdgpu_flip_work_func(struct work_struct *__work)
 	/* Wait until we're out of the vertical blank period before the one
 	 * targeted by the flip
 	 */
-	if (amdgpuCrtc->enabled &&
+	if (amdgpu_crtc->enabled &&
 	    (amdgpu_get_crtc_scanoutpos(adev->ddev, work->crtc_id, 0,
 					&vpos, &hpos, NULL, NULL,
 					&crtc->hwmode)
 	     & (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK)) ==
 	    (DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_IN_VBLANK) &&
 	    (int)(work->target_vblank -
-		  amdgpu_get_vblank_counter_kms(adev->ddev, amdgpuCrtc->crtc_id)) > 0) {
+		  amdgpu_get_vblank_counter_kms(adev->ddev, amdgpu_crtc->crtc_id)) > 0) {
 		schedule_delayed_work(&work->flip_work, usecs_to_jiffies(1000));
 		return;
 	}
@@ -104,12 +104,12 @@ static void amdgpu_flip_work_func(struct work_struct *__work)
 	adev->mode_info.funcs->page_flip(adev, work->crtc_id, work->base, work->async);
 
 	/* Set the flip status */
-	amdgpuCrtc->pflip_status = AMDGPU_FLIP_SUBMITTED;
+	amdgpu_crtc->pflip_status = AMDGPU_FLIP_SUBMITTED;
 	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
 
 
 	DRM_DEBUG_DRIVER("crtc:%d[%p], pflip_stat:AMDGPU_FLIP_SUBMITTED, work: %p,\n",
-					 amdgpuCrtc->crtc_id, amdgpuCrtc, work);
+					 amdgpu_crtc->crtc_id, amdgpu_crtc, work);
 
 }
 
@@ -187,7 +187,7 @@ int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc,
 		goto cleanup;
 	}
 
-	r = amdgpu_bo_pin_restricted(new_abo, AMDGPU_GEM_DOMAIN_VRAM, 0, 0, &base);
+	r = amdgpu_bo_pin(new_abo, AMDGPU_GEM_DOMAIN_VRAM, &base);
 	if (unlikely(r != 0)) {
 		r = -EINVAL;
 		DRM_ERROR("failed to pin new abo buffer before flip\n");
@@ -244,9 +244,9 @@ unreserve:
 
 cleanup:
 	amdgpu_bo_unref(&work->old_abo);
-	fence_put(work->excl);
+	dma_fence_put(work->excl);
 	for (i = 0; i < work->shared_count; ++i)
-		fence_put(work->shared[i]);
+		dma_fence_put(work->shared[i]);
 	kfree(work->shared);
 	kfree(work);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
index 14f57d9915e3..6ca0333ca4c0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
@@ -553,9 +553,10 @@ int amdgpu_parse_extended_power_table(struct amdgpu_device *adev)
 				entry = (ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record *)
 					((u8 *)entry + sizeof(ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record));
 			}
-			for (i = 0; i < states->numEntries; i++) {
-				if (i >= AMDGPU_MAX_VCE_LEVELS)
-					break;
+			adev->pm.dpm.num_of_vce_states =
+					states->numEntries > AMD_MAX_VCE_LEVELS ?
+					AMD_MAX_VCE_LEVELS : states->numEntries;
+			for (i = 0; i < adev->pm.dpm.num_of_vce_states; i++) {
 				vce_clk = (VCEClockInfo *)
 					((u8 *)&array->entries[0] +
 					 (state_entry->ucVCEClockInfoIndex * sizeof(VCEClockInfo)));
@@ -955,3 +956,12 @@ u8 amdgpu_encode_pci_lane_width(u32 lanes)
 
 	return encoded_lanes[lanes];
 }
+
+struct amd_vce_state*
+amdgpu_get_vce_clock_state(struct amdgpu_device *adev, unsigned idx)
+{
+	if (idx < adev->pm.dpm.num_of_vce_states)
+		return &adev->pm.dpm.vce_states[idx];
+
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
index 3738a96c2619..955d6f21e2b3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
@@ -23,6 +23,453 @@
 #ifndef __AMDGPU_DPM_H__
 #define __AMDGPU_DPM_H__
 
+enum amdgpu_int_thermal_type {
+	THERMAL_TYPE_NONE,
+	THERMAL_TYPE_EXTERNAL,
+	THERMAL_TYPE_EXTERNAL_GPIO,
+	THERMAL_TYPE_RV6XX,
+	THERMAL_TYPE_RV770,
+	THERMAL_TYPE_ADT7473_WITH_INTERNAL,
+	THERMAL_TYPE_EVERGREEN,
+	THERMAL_TYPE_SUMO,
+	THERMAL_TYPE_NI,
+	THERMAL_TYPE_SI,
+	THERMAL_TYPE_EMC2103_WITH_INTERNAL,
+	THERMAL_TYPE_CI,
+	THERMAL_TYPE_KV,
+};
+
+enum amdgpu_dpm_auto_throttle_src {
+	AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL,
+	AMDGPU_DPM_AUTO_THROTTLE_SRC_EXTERNAL
+};
+
+enum amdgpu_dpm_event_src {
+	AMDGPU_DPM_EVENT_SRC_ANALOG = 0,
+	AMDGPU_DPM_EVENT_SRC_EXTERNAL = 1,
+	AMDGPU_DPM_EVENT_SRC_DIGITAL = 2,
+	AMDGPU_DPM_EVENT_SRC_ANALOG_OR_EXTERNAL = 3,
+	AMDGPU_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL = 4
+};
+
+#define SCLK_DEEP_SLEEP_MASK 0x8
+
+struct amdgpu_ps {
+	u32 caps; /* vbios flags */
+	u32 class; /* vbios flags */
+	u32 class2; /* vbios flags */
+	/* UVD clocks */
+	u32 vclk;
+	u32 dclk;
+	/* VCE clocks */
+	u32 evclk;
+	u32 ecclk;
+	bool vce_active;
+	enum amd_vce_level vce_level;
+	/* asic priv */
+	void *ps_priv;
+};
+
+struct amdgpu_dpm_thermal {
+	/* thermal interrupt work */
+	struct work_struct work;
+	/* low temperature threshold */
+	int                min_temp;
+	/* high temperature threshold */
+	int                max_temp;
+	/* was last interrupt low to high or high to low */
+	bool               high_to_low;
+	/* interrupt source */
+	struct amdgpu_irq_src	irq;
+};
+
+enum amdgpu_clk_action
+{
+	AMDGPU_SCLK_UP = 1,
+	AMDGPU_SCLK_DOWN
+};
+
+struct amdgpu_blacklist_clocks
+{
+	u32 sclk;
+	u32 mclk;
+	enum amdgpu_clk_action action;
+};
+
+struct amdgpu_clock_and_voltage_limits {
+	u32 sclk;
+	u32 mclk;
+	u16 vddc;
+	u16 vddci;
+};
+
+struct amdgpu_clock_array {
+	u32 count;
+	u32 *values;
+};
+
+struct amdgpu_clock_voltage_dependency_entry {
+	u32 clk;
+	u16 v;
+};
+
+struct amdgpu_clock_voltage_dependency_table {
+	u32 count;
+	struct amdgpu_clock_voltage_dependency_entry *entries;
+};
+
+union amdgpu_cac_leakage_entry {
+	struct {
+		u16 vddc;
+		u32 leakage;
+	};
+	struct {
+		u16 vddc1;
+		u16 vddc2;
+		u16 vddc3;
+	};
+};
+
+struct amdgpu_cac_leakage_table {
+	u32 count;
+	union amdgpu_cac_leakage_entry *entries;
+};
+
+struct amdgpu_phase_shedding_limits_entry {
+	u16 voltage;
+	u32 sclk;
+	u32 mclk;
+};
+
+struct amdgpu_phase_shedding_limits_table {
+	u32 count;
+	struct amdgpu_phase_shedding_limits_entry *entries;
+};
+
+struct amdgpu_uvd_clock_voltage_dependency_entry {
+	u32 vclk;
+	u32 dclk;
+	u16 v;
+};
+
+struct amdgpu_uvd_clock_voltage_dependency_table {
+	u8 count;
+	struct amdgpu_uvd_clock_voltage_dependency_entry *entries;
+};
+
+struct amdgpu_vce_clock_voltage_dependency_entry {
+	u32 ecclk;
+	u32 evclk;
+	u16 v;
+};
+
+struct amdgpu_vce_clock_voltage_dependency_table {
+	u8 count;
+	struct amdgpu_vce_clock_voltage_dependency_entry *entries;
+};
+
+struct amdgpu_ppm_table {
+	u8 ppm_design;
+	u16 cpu_core_number;
+	u32 platform_tdp;
+	u32 small_ac_platform_tdp;
+	u32 platform_tdc;
+	u32 small_ac_platform_tdc;
+	u32 apu_tdp;
+	u32 dgpu_tdp;
+	u32 dgpu_ulv_power;
+	u32 tj_max;
+};
+
+struct amdgpu_cac_tdp_table {
+	u16 tdp;
+	u16 configurable_tdp;
+	u16 tdc;
+	u16 battery_power_limit;
+	u16 small_power_limit;
+	u16 low_cac_leakage;
+	u16 high_cac_leakage;
+	u16 maximum_power_delivery_limit;
+};
+
+struct amdgpu_dpm_dynamic_state {
+	struct amdgpu_clock_voltage_dependency_table vddc_dependency_on_sclk;
+	struct amdgpu_clock_voltage_dependency_table vddci_dependency_on_mclk;
+	struct amdgpu_clock_voltage_dependency_table vddc_dependency_on_mclk;
+	struct amdgpu_clock_voltage_dependency_table mvdd_dependency_on_mclk;
+	struct amdgpu_clock_voltage_dependency_table vddc_dependency_on_dispclk;
+	struct amdgpu_uvd_clock_voltage_dependency_table uvd_clock_voltage_dependency_table;
+	struct amdgpu_vce_clock_voltage_dependency_table vce_clock_voltage_dependency_table;
+	struct amdgpu_clock_voltage_dependency_table samu_clock_voltage_dependency_table;
+	struct amdgpu_clock_voltage_dependency_table acp_clock_voltage_dependency_table;
+	struct amdgpu_clock_voltage_dependency_table vddgfx_dependency_on_sclk;
+	struct amdgpu_clock_array valid_sclk_values;
+	struct amdgpu_clock_array valid_mclk_values;
+	struct amdgpu_clock_and_voltage_limits max_clock_voltage_on_dc;
+	struct amdgpu_clock_and_voltage_limits max_clock_voltage_on_ac;
+	u32 mclk_sclk_ratio;
+	u32 sclk_mclk_delta;
+	u16 vddc_vddci_delta;
+	u16 min_vddc_for_pcie_gen2;
+	struct amdgpu_cac_leakage_table cac_leakage_table;
+	struct amdgpu_phase_shedding_limits_table phase_shedding_limits_table;
+	struct amdgpu_ppm_table *ppm_table;
+	struct amdgpu_cac_tdp_table *cac_tdp_table;
+};
+
+struct amdgpu_dpm_fan {
+	u16 t_min;
+	u16 t_med;
+	u16 t_high;
+	u16 pwm_min;
+	u16 pwm_med;
+	u16 pwm_high;
+	u8 t_hyst;
+	u32 cycle_delay;
+	u16 t_max;
+	u8 control_mode;
+	u16 default_max_fan_pwm;
+	u16 default_fan_output_sensitivity;
+	u16 fan_output_sensitivity;
+	bool ucode_fan_control;
+};
+
+enum amdgpu_pcie_gen {
+	AMDGPU_PCIE_GEN1 = 0,
+	AMDGPU_PCIE_GEN2 = 1,
+	AMDGPU_PCIE_GEN3 = 2,
+	AMDGPU_PCIE_GEN_INVALID = 0xffff
+};
+
+enum amdgpu_dpm_forced_level {
+	AMDGPU_DPM_FORCED_LEVEL_AUTO = 0,
+	AMDGPU_DPM_FORCED_LEVEL_LOW = 1,
+	AMDGPU_DPM_FORCED_LEVEL_HIGH = 2,
+	AMDGPU_DPM_FORCED_LEVEL_MANUAL = 3,
+};
+
+struct amdgpu_dpm_funcs {
+	int (*get_temperature)(struct amdgpu_device *adev);
+	int (*pre_set_power_state)(struct amdgpu_device *adev);
+	int (*set_power_state)(struct amdgpu_device *adev);
+	void (*post_set_power_state)(struct amdgpu_device *adev);
+	void (*display_configuration_changed)(struct amdgpu_device *adev);
+	u32 (*get_sclk)(struct amdgpu_device *adev, bool low);
+	u32 (*get_mclk)(struct amdgpu_device *adev, bool low);
+	void (*print_power_state)(struct amdgpu_device *adev, struct amdgpu_ps *ps);
+	void (*debugfs_print_current_performance_level)(struct amdgpu_device *adev, struct seq_file *m);
+	int (*force_performance_level)(struct amdgpu_device *adev, enum amdgpu_dpm_forced_level level);
+	bool (*vblank_too_short)(struct amdgpu_device *adev);
+	void (*powergate_uvd)(struct amdgpu_device *adev, bool gate);
+	void (*powergate_vce)(struct amdgpu_device *adev, bool gate);
+	void (*enable_bapm)(struct amdgpu_device *adev, bool enable);
+	void (*set_fan_control_mode)(struct amdgpu_device *adev, u32 mode);
+	u32 (*get_fan_control_mode)(struct amdgpu_device *adev);
+	int (*set_fan_speed_percent)(struct amdgpu_device *adev, u32 speed);
+	int (*get_fan_speed_percent)(struct amdgpu_device *adev, u32 *speed);
+	int (*force_clock_level)(struct amdgpu_device *adev, enum pp_clock_type type, uint32_t mask);
+	int (*print_clock_levels)(struct amdgpu_device *adev, enum pp_clock_type type, char *buf);
+	int (*get_sclk_od)(struct amdgpu_device *adev);
+	int (*set_sclk_od)(struct amdgpu_device *adev, uint32_t value);
+	int (*get_mclk_od)(struct amdgpu_device *adev);
+	int (*set_mclk_od)(struct amdgpu_device *adev, uint32_t value);
+	int (*check_state_equal)(struct amdgpu_device *adev,
+				struct amdgpu_ps *cps,
+				struct amdgpu_ps *rps,
+				bool *equal);
+
+	struct amd_vce_state* (*get_vce_clock_state)(struct amdgpu_device *adev, unsigned idx);
+};
+
+#define amdgpu_dpm_pre_set_power_state(adev) (adev)->pm.funcs->pre_set_power_state((adev))
+#define amdgpu_dpm_set_power_state(adev) (adev)->pm.funcs->set_power_state((adev))
+#define amdgpu_dpm_post_set_power_state(adev) (adev)->pm.funcs->post_set_power_state((adev))
+#define amdgpu_dpm_display_configuration_changed(adev) (adev)->pm.funcs->display_configuration_changed((adev))
+#define amdgpu_dpm_print_power_state(adev, ps) (adev)->pm.funcs->print_power_state((adev), (ps))
+#define amdgpu_dpm_vblank_too_short(adev) (adev)->pm.funcs->vblank_too_short((adev))
+#define amdgpu_dpm_enable_bapm(adev, e) (adev)->pm.funcs->enable_bapm((adev), (e))
+
+#define amdgpu_dpm_read_sensor(adev, idx, value) \
+	((adev)->pp_enabled ? \
+		(adev)->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, (idx), (value)) : \
+		-EINVAL)
+
+#define amdgpu_dpm_get_temperature(adev) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->get_temperature((adev)->powerplay.pp_handle) : \
+	      (adev)->pm.funcs->get_temperature((adev)))
+
+#define amdgpu_dpm_set_fan_control_mode(adev, m) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->set_fan_control_mode((adev)->powerplay.pp_handle, (m)) : \
+	      (adev)->pm.funcs->set_fan_control_mode((adev), (m)))
+
+#define amdgpu_dpm_get_fan_control_mode(adev) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->get_fan_control_mode((adev)->powerplay.pp_handle) : \
+	      (adev)->pm.funcs->get_fan_control_mode((adev)))
+
+#define amdgpu_dpm_set_fan_speed_percent(adev, s) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->set_fan_speed_percent((adev)->powerplay.pp_handle, (s)) : \
+	      (adev)->pm.funcs->set_fan_speed_percent((adev), (s)))
+
+#define amdgpu_dpm_get_fan_speed_percent(adev, s) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->get_fan_speed_percent((adev)->powerplay.pp_handle, (s)) : \
+	      (adev)->pm.funcs->get_fan_speed_percent((adev), (s)))
+
+#define amdgpu_dpm_get_fan_speed_rpm(adev, s) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->get_fan_speed_rpm((adev)->powerplay.pp_handle, (s)) : \
+	      -EINVAL)
+
+#define amdgpu_dpm_get_sclk(adev, l) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->get_sclk((adev)->powerplay.pp_handle, (l)) : \
+		(adev)->pm.funcs->get_sclk((adev), (l)))
+
+#define amdgpu_dpm_get_mclk(adev, l)  \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->get_mclk((adev)->powerplay.pp_handle, (l)) : \
+	      (adev)->pm.funcs->get_mclk((adev), (l)))
+
+
+#define amdgpu_dpm_force_performance_level(adev, l) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->force_performance_level((adev)->powerplay.pp_handle, (l)) : \
+	      (adev)->pm.funcs->force_performance_level((adev), (l)))
+
+#define amdgpu_dpm_powergate_uvd(adev, g) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->powergate_uvd((adev)->powerplay.pp_handle, (g)) : \
+	      (adev)->pm.funcs->powergate_uvd((adev), (g)))
+
+#define amdgpu_dpm_powergate_vce(adev, g) \
+	((adev)->pp_enabled ?						\
+	      (adev)->powerplay.pp_funcs->powergate_vce((adev)->powerplay.pp_handle, (g)) : \
+	      (adev)->pm.funcs->powergate_vce((adev), (g)))
+
+#define amdgpu_dpm_get_current_power_state(adev) \
+	(adev)->powerplay.pp_funcs->get_current_power_state((adev)->powerplay.pp_handle)
+
+#define amdgpu_dpm_get_performance_level(adev) \
+	(adev)->powerplay.pp_funcs->get_performance_level((adev)->powerplay.pp_handle)
+
+#define amdgpu_dpm_get_pp_num_states(adev, data) \
+	(adev)->powerplay.pp_funcs->get_pp_num_states((adev)->powerplay.pp_handle, data)
+
+#define amdgpu_dpm_get_pp_table(adev, table) \
+	(adev)->powerplay.pp_funcs->get_pp_table((adev)->powerplay.pp_handle, table)
+
+#define amdgpu_dpm_set_pp_table(adev, buf, size) \
+	(adev)->powerplay.pp_funcs->set_pp_table((adev)->powerplay.pp_handle, buf, size)
+
+#define amdgpu_dpm_print_clock_levels(adev, type, buf) \
+	(adev)->powerplay.pp_funcs->print_clock_levels((adev)->powerplay.pp_handle, type, buf)
+
+#define amdgpu_dpm_force_clock_level(adev, type, level) \
+		(adev)->powerplay.pp_funcs->force_clock_level((adev)->powerplay.pp_handle, type, level)
+
+#define amdgpu_dpm_get_sclk_od(adev) \
+	(adev)->powerplay.pp_funcs->get_sclk_od((adev)->powerplay.pp_handle)
+
+#define amdgpu_dpm_set_sclk_od(adev, value) \
+	(adev)->powerplay.pp_funcs->set_sclk_od((adev)->powerplay.pp_handle, value)
+
+#define amdgpu_dpm_get_mclk_od(adev) \
+	((adev)->powerplay.pp_funcs->get_mclk_od((adev)->powerplay.pp_handle))
+
+#define amdgpu_dpm_set_mclk_od(adev, value) \
+	((adev)->powerplay.pp_funcs->set_mclk_od((adev)->powerplay.pp_handle, value))
+
+#define amdgpu_dpm_dispatch_task(adev, event_id, input, output)		\
+	(adev)->powerplay.pp_funcs->dispatch_tasks((adev)->powerplay.pp_handle, (event_id), (input), (output))
+
+#define amgdpu_dpm_check_state_equal(adev, cps, rps, equal) (adev)->pm.funcs->check_state_equal((adev), (cps),(rps),(equal))
+
+#define amdgpu_dpm_get_vce_clock_state(adev, i)				\
+	((adev)->pp_enabled ?						\
+	 (adev)->powerplay.pp_funcs->get_vce_clock_state((adev)->powerplay.pp_handle, (i)) : \
+	 (adev)->pm.funcs->get_vce_clock_state((adev), (i)))
+
+struct amdgpu_dpm {
+	struct amdgpu_ps        *ps;
+	/* number of valid power states */
+	int                     num_ps;
+	/* current power state that is active */
+	struct amdgpu_ps        *current_ps;
+	/* requested power state */
+	struct amdgpu_ps        *requested_ps;
+	/* boot up power state */
+	struct amdgpu_ps        *boot_ps;
+	/* default uvd power state */
+	struct amdgpu_ps        *uvd_ps;
+	/* vce requirements */
+	u32                  num_of_vce_states;
+	struct amd_vce_state vce_states[AMD_MAX_VCE_LEVELS];
+	enum amd_vce_level vce_level;
+	enum amd_pm_state_type state;
+	enum amd_pm_state_type user_state;
+	enum amd_pm_state_type last_state;
+	enum amd_pm_state_type last_user_state;
+	u32                     platform_caps;
+	u32                     voltage_response_time;
+	u32                     backbias_response_time;
+	void                    *priv;
+	u32			new_active_crtcs;
+	int			new_active_crtc_count;
+	u32			current_active_crtcs;
+	int			current_active_crtc_count;
+	struct amdgpu_dpm_dynamic_state dyn_state;
+	struct amdgpu_dpm_fan fan;
+	u32 tdp_limit;
+	u32 near_tdp_limit;
+	u32 near_tdp_limit_adjusted;
+	u32 sq_ramping_threshold;
+	u32 cac_leakage;
+	u16 tdp_od_limit;
+	u32 tdp_adjustment;
+	u16 load_line_slope;
+	bool power_control;
+	bool ac_power;
+	/* special states active */
+	bool                    thermal_active;
+	bool                    uvd_active;
+	bool                    vce_active;
+	/* thermal handling */
+	struct amdgpu_dpm_thermal thermal;
+	/* forced levels */
+	enum amdgpu_dpm_forced_level forced_level;
+};
+
+struct amdgpu_pm {
+	struct mutex		mutex;
+	u32                     current_sclk;
+	u32                     current_mclk;
+	u32                     default_sclk;
+	u32                     default_mclk;
+	struct amdgpu_i2c_chan *i2c_bus;
+	/* internal thermal controller on rv6xx+ */
+	enum amdgpu_int_thermal_type int_thermal_type;
+	struct device	        *int_hwmon_dev;
+	/* fan control parameters */
+	bool                    no_fan;
+	u8                      fan_pulses_per_revolution;
+	u8                      fan_min_rpm;
+	u8                      fan_max_rpm;
+	/* dpm */
+	bool                    dpm_enabled;
+	bool                    sysfs_initialized;
+	struct amdgpu_dpm       dpm;
+	const struct firmware	*fw;	/* SMC firmware */
+	uint32_t                fw_version;
+	const struct amdgpu_dpm_funcs *funcs;
+	uint32_t                pcie_gen_mask;
+	uint32_t                pcie_mlw_mask;
+	struct amd_pp_display_configuration pm_display_cfg;/* set by DAL */
+};
+
 #define R600_SSTU_DFLT                               0
 #define R600_SST_DFLT                                0x00C8
 
@@ -82,4 +529,7 @@ u16 amdgpu_get_pcie_lane_support(struct amdgpu_device *adev,
 				 u16 default_lanes);
 u8 amdgpu_encode_pci_lane_width(u32 lanes);
 
+struct amd_vce_state*
+amdgpu_get_vce_clock_state(struct amdgpu_device *adev, unsigned idx);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index e0890deccb2f..8cb937b2bfcc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -58,9 +58,10 @@
  * - 3.6.0 - kmd involves use CONTEXT_CONTROL in ring buffer.
  * - 3.7.0 - Add support for VCE clock list packet
  * - 3.8.0 - Add support raster config init in the kernel
+ * - 3.9.0 - Add support for memory query info about VRAM and GTT.
  */
 #define KMS_DRIVER_MAJOR	3
-#define KMS_DRIVER_MINOR	8
+#define KMS_DRIVER_MINOR	9
 #define KMS_DRIVER_PATCHLEVEL	0
 
 int amdgpu_vram_limit = 0;
@@ -85,12 +86,13 @@ int amdgpu_vm_size = 64;
 int amdgpu_vm_block_size = -1;
 int amdgpu_vm_fault_stop = 0;
 int amdgpu_vm_debug = 0;
+int amdgpu_vram_page_split = 1024;
 int amdgpu_exp_hw_support = 0;
 int amdgpu_sched_jobs = 32;
 int amdgpu_sched_hw_submission = 2;
 int amdgpu_powerplay = -1;
-int amdgpu_powercontainment = 1;
-int amdgpu_sclk_deep_sleep_en = 1;
+int amdgpu_no_evict = 0;
+int amdgpu_direct_gma_size = 0;
 unsigned amdgpu_pcie_gen_cap = 0;
 unsigned amdgpu_pcie_lane_cap = 0;
 unsigned amdgpu_cg_mask = 0xffffffff;
@@ -165,6 +167,9 @@ module_param_named(vm_fault_stop, amdgpu_vm_fault_stop, int, 0444);
 MODULE_PARM_DESC(vm_debug, "Debug VM handling (0 = disabled (default), 1 = enabled)");
 module_param_named(vm_debug, amdgpu_vm_debug, int, 0644);
 
+MODULE_PARM_DESC(vram_page_split, "Number of pages after we split VRAM allocations (default 1024, -1 = disable)");
+module_param_named(vram_page_split, amdgpu_vram_page_split, int, 0444);
+
 MODULE_PARM_DESC(exp_hw_support, "experimental hw support (1 = enable, 0 = disable (default))");
 module_param_named(exp_hw_support, amdgpu_exp_hw_support, int, 0444);
 
@@ -177,14 +182,14 @@ module_param_named(sched_hw_submission, amdgpu_sched_hw_submission, int, 0444);
 MODULE_PARM_DESC(powerplay, "Powerplay component (1 = enable, 0 = disable, -1 = auto (default))");
 module_param_named(powerplay, amdgpu_powerplay, int, 0444);
 
-MODULE_PARM_DESC(powercontainment, "Power Containment (1 = enable (default), 0 = disable)");
-module_param_named(powercontainment, amdgpu_powercontainment, int, 0444);
-
 MODULE_PARM_DESC(ppfeaturemask, "all power features enabled (default))");
 module_param_named(ppfeaturemask, amdgpu_pp_feature_mask, int, 0444);
 
-MODULE_PARM_DESC(sclkdeepsleep, "SCLK Deep Sleep (1 = enable (default), 0 = disable)");
-module_param_named(sclkdeepsleep, amdgpu_sclk_deep_sleep_en, int, 0444);
+MODULE_PARM_DESC(no_evict, "Support pinning request from user space (1 = enable, 0 = disable (default))");
+module_param_named(no_evict, amdgpu_no_evict, int, 0444);
+
+MODULE_PARM_DESC(direct_gma_size, "Direct GMA size in megabytes (max 96MB)");
+module_param_named(direct_gma_size, amdgpu_direct_gma_size, int, 0444);
 
 MODULE_PARM_DESC(pcie_gen_cap, "PCIE Gen Caps (0: autodetect (default))");
 module_param_named(pcie_gen_cap, amdgpu_pcie_gen_cap, uint, 0444);
@@ -201,7 +206,8 @@ module_param_named(pg_mask, amdgpu_pg_mask, uint, 0444);
 MODULE_PARM_DESC(disable_cu, "Disable CUs (se.sh.cu,...)");
 module_param_named(disable_cu, amdgpu_disable_cu, charp, 0444);
 
-MODULE_PARM_DESC(virtual_display, "Enable virtual display feature (the virtual_display will be set like xxxx:xx:xx.x;xxxx:xx:xx.x)");
+MODULE_PARM_DESC(virtual_display,
+		 "Enable virtual display feature (the virtual_display will be set like xxxx:xx:xx.x,x;xxxx:xx:xx.x,x)");
 module_param_named(virtual_display, amdgpu_virtual_display, charp, 0444);
 
 static const struct pci_device_id pciidlist[] = {
@@ -381,6 +387,7 @@ static const struct pci_device_id pciidlist[] = {
 	{0x1002, 0x6939, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA},
 	/* fiji */
 	{0x1002, 0x7300, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_FIJI},
+	{0x1002, 0x730F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_FIJI},
 	/* carrizo */
 	{0x1002, 0x9870, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU},
 	{0x1002, 0x9874, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU},
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index 9fb8aa4d6bae..24629bec181a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -75,27 +75,21 @@ amdgpufb_release(struct fb_info *info, int user)
 
 static struct fb_ops amdgpufb_ops = {
 	.owner = THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_open = amdgpufb_open,
 	.fb_release = amdgpufb_release,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
 	.fb_fillrect = drm_fb_helper_cfb_fillrect,
 	.fb_copyarea = drm_fb_helper_cfb_copyarea,
 	.fb_imageblit = drm_fb_helper_cfb_imageblit,
-	.fb_pan_display = drm_fb_helper_pan_display,
-	.fb_blank = drm_fb_helper_blank,
-	.fb_setcmap = drm_fb_helper_setcmap,
-	.fb_debug_enter = drm_fb_helper_debug_enter,
-	.fb_debug_leave = drm_fb_helper_debug_leave,
 };
 
 
-int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int bpp, bool tiled)
+int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int cpp, bool tiled)
 {
 	int aligned = width;
 	int pitch_mask = 0;
 
-	switch (bpp / 8) {
+	switch (cpp) {
 	case 1:
 		pitch_mask = 255;
 		break;
@@ -110,7 +104,7 @@ int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int bpp, bool tile
 
 	aligned += pitch_mask;
 	aligned &= ~pitch_mask;
-	return aligned;
+	return aligned * cpp;
 }
 
 static void amdgpufb_destroy_pinned_object(struct drm_gem_object *gobj)
@@ -139,20 +133,21 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
 	int ret;
 	int aligned_size, size;
 	int height = mode_cmd->height;
-	u32 bpp, depth;
+	u32 cpp;
 
-	drm_fb_get_bpp_depth(mode_cmd->pixel_format, &depth, &bpp);
+	cpp = drm_format_plane_cpp(mode_cmd->pixel_format, 0);
 
 	/* need to align pitch with crtc limits */
-	mode_cmd->pitches[0] = amdgpu_align_pitch(adev, mode_cmd->width, bpp,
-						  fb_tiled) * ((bpp + 1) / 8);
+	mode_cmd->pitches[0] = amdgpu_align_pitch(adev, mode_cmd->width, cpp,
+						  fb_tiled);
 
 	height = ALIGN(mode_cmd->height, 8);
 	size = mode_cmd->pitches[0] * height;
 	aligned_size = ALIGN(size, PAGE_SIZE);
 	ret = amdgpu_gem_object_create(adev, aligned_size, 0,
 				       AMDGPU_GEM_DOMAIN_VRAM,
-				       AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+				       AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+				       AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 				       true, &gobj);
 	if (ret) {
 		printk(KERN_ERR "failed to allocate framebuffer (%d)\n",
@@ -176,7 +171,7 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
 	}
 
 
-	ret = amdgpu_bo_pin_restricted(abo, AMDGPU_GEM_DOMAIN_VRAM, 0, 0, NULL);
+	ret = amdgpu_bo_pin(abo, AMDGPU_GEM_DOMAIN_VRAM, NULL);
 	if (ret) {
 		amdgpu_bo_unreserve(abo);
 		goto out_unref;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 77b34ec92632..7b60fb79c3a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -48,7 +48,7 @@
  */
 
 struct amdgpu_fence {
-	struct fence base;
+	struct dma_fence base;
 
 	/* RB, DMA, etc. */
 	struct amdgpu_ring		*ring;
@@ -74,8 +74,8 @@ void amdgpu_fence_slab_fini(void)
 /*
  * Cast helper
  */
-static const struct fence_ops amdgpu_fence_ops;
-static inline struct amdgpu_fence *to_amdgpu_fence(struct fence *f)
+static const struct dma_fence_ops amdgpu_fence_ops;
+static inline struct amdgpu_fence *to_amdgpu_fence(struct dma_fence *f)
 {
 	struct amdgpu_fence *__f = container_of(f, struct amdgpu_fence, base);
 
@@ -131,11 +131,11 @@ static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
  * Emits a fence command on the requested ring (all asics).
  * Returns 0 on success, -ENOMEM on failure.
  */
-int amdgpu_fence_emit(struct amdgpu_ring *ring, struct fence **f)
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_fence *fence;
-	struct fence *old, **ptr;
+	struct dma_fence *old, **ptr;
 	uint32_t seq;
 
 	fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
@@ -144,10 +144,10 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct fence **f)
 
 	seq = ++ring->fence_drv.sync_seq;
 	fence->ring = ring;
-	fence_init(&fence->base, &amdgpu_fence_ops,
-		   &ring->fence_drv.lock,
-		   adev->fence_context + ring->idx,
-		   seq);
+	dma_fence_init(&fence->base, &amdgpu_fence_ops,
+		       &ring->fence_drv.lock,
+		       adev->fence_context + ring->idx,
+		       seq);
 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
 			       seq, AMDGPU_FENCE_FLAG_INT);
 
@@ -156,12 +156,12 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct fence **f)
 	 * emitting the fence would mess up the hardware ring buffer.
 	 */
 	old = rcu_dereference_protected(*ptr, 1);
-	if (old && !fence_is_signaled(old)) {
+	if (old && !dma_fence_is_signaled(old)) {
 		DRM_INFO("rcu slot is busy\n");
-		fence_wait(old, false);
+		dma_fence_wait(old, false);
 	}
 
-	rcu_assign_pointer(*ptr, fence_get(&fence->base));
+	rcu_assign_pointer(*ptr, dma_fence_get(&fence->base));
 
 	*f = &fence->base;
 
@@ -212,7 +212,7 @@ void amdgpu_fence_process(struct amdgpu_ring *ring)
 	seq &= drv->num_fences_mask;
 
 	do {
-		struct fence *fence, **ptr;
+		struct dma_fence *fence, **ptr;
 
 		++last_seq;
 		last_seq &= drv->num_fences_mask;
@@ -225,13 +225,13 @@ void amdgpu_fence_process(struct amdgpu_ring *ring)
 		if (!fence)
 			continue;
 
-		r = fence_signal(fence);
+		r = dma_fence_signal(fence);
 		if (!r)
-			FENCE_TRACE(fence, "signaled from irq context\n");
+			DMA_FENCE_TRACE(fence, "signaled from irq context\n");
 		else
 			BUG();
 
-		fence_put(fence);
+		dma_fence_put(fence);
 	} while (last_seq != seq);
 }
 
@@ -261,7 +261,7 @@ static void amdgpu_fence_fallback(unsigned long arg)
 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
 {
 	uint64_t seq = ACCESS_ONCE(ring->fence_drv.sync_seq);
-	struct fence *fence, **ptr;
+	struct dma_fence *fence, **ptr;
 	int r;
 
 	if (!seq)
@@ -270,14 +270,14 @@ int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
 	rcu_read_lock();
 	fence = rcu_dereference(*ptr);
-	if (!fence || !fence_get_rcu(fence)) {
+	if (!fence || !dma_fence_get_rcu(fence)) {
 		rcu_read_unlock();
 		return 0;
 	}
 	rcu_read_unlock();
 
-	r = fence_wait(fence, false);
-	fence_put(fence);
+	r = dma_fence_wait(fence, false);
+	dma_fence_put(fence);
 	return r;
 }
 
@@ -382,24 +382,27 @@ int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
 	if (!ring->fence_drv.fences)
 		return -ENOMEM;
 
-	timeout = msecs_to_jiffies(amdgpu_lockup_timeout);
-	if (timeout == 0) {
-		/*
-		 * FIXME:
-		 * Delayed workqueue cannot use it directly,
-		 * so the scheduler will not use delayed workqueue if
-		 * MAX_SCHEDULE_TIMEOUT is set.
-		 * Currently keep it simple and silly.
-		 */
-		timeout = MAX_SCHEDULE_TIMEOUT;
-	}
-	r = amd_sched_init(&ring->sched, &amdgpu_sched_ops,
-			   num_hw_submission,
-			   timeout, ring->name);
-	if (r) {
-		DRM_ERROR("Failed to create scheduler on ring %s.\n",
-			  ring->name);
-		return r;
+	/* No need to setup the GPU scheduler for KIQ ring */
+	if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ) {
+		timeout = msecs_to_jiffies(amdgpu_lockup_timeout);
+		if (timeout == 0) {
+			/*
+			 * FIXME:
+			 * Delayed workqueue cannot use it directly,
+			 * so the scheduler will not use delayed workqueue if
+			 * MAX_SCHEDULE_TIMEOUT is set.
+			 * Currently keep it simple and silly.
+			 */
+			timeout = MAX_SCHEDULE_TIMEOUT;
+		}
+		r = amd_sched_init(&ring->sched, &amdgpu_sched_ops,
+				   num_hw_submission,
+				   timeout, ring->name);
+		if (r) {
+			DRM_ERROR("Failed to create scheduler on ring %s.\n",
+				  ring->name);
+			return r;
+		}
 	}
 
 	return 0;
@@ -453,7 +456,7 @@ void amdgpu_fence_driver_fini(struct amdgpu_device *adev)
 		amd_sched_fini(&ring->sched);
 		del_timer_sync(&ring->fence_drv.fallback_timer);
 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)
-			fence_put(ring->fence_drv.fences[j]);
+			dma_fence_put(ring->fence_drv.fences[j]);
 		kfree(ring->fence_drv.fences);
 		ring->fence_drv.fences = NULL;
 		ring->fence_drv.initialized = false;
@@ -542,12 +545,12 @@ void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev)
  * Common fence implementation
  */
 
-static const char *amdgpu_fence_get_driver_name(struct fence *fence)
+static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
 {
 	return "amdgpu";
 }
 
-static const char *amdgpu_fence_get_timeline_name(struct fence *f)
+static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
 {
 	struct amdgpu_fence *fence = to_amdgpu_fence(f);
 	return (const char *)fence->ring->name;
@@ -561,7 +564,7 @@ static const char *amdgpu_fence_get_timeline_name(struct fence *f)
  * to fence_queue that checks if this fence is signaled, and if so it
  * signals the fence and removes itself.
  */
-static bool amdgpu_fence_enable_signaling(struct fence *f)
+static bool amdgpu_fence_enable_signaling(struct dma_fence *f)
 {
 	struct amdgpu_fence *fence = to_amdgpu_fence(f);
 	struct amdgpu_ring *ring = fence->ring;
@@ -569,7 +572,7 @@ static bool amdgpu_fence_enable_signaling(struct fence *f)
 	if (!timer_pending(&ring->fence_drv.fallback_timer))
 		amdgpu_fence_schedule_fallback(ring);
 
-	FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);
+	DMA_FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx);
 
 	return true;
 }
@@ -583,7 +586,7 @@ static bool amdgpu_fence_enable_signaling(struct fence *f)
  */
 static void amdgpu_fence_free(struct rcu_head *rcu)
 {
-	struct fence *f = container_of(rcu, struct fence, rcu);
+	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
 	struct amdgpu_fence *fence = to_amdgpu_fence(f);
 	kmem_cache_free(amdgpu_fence_slab, fence);
 }
@@ -596,16 +599,16 @@ static void amdgpu_fence_free(struct rcu_head *rcu)
  * This function is called when the reference count becomes zero.
  * It just RCU schedules freeing up the fence.
  */
-static void amdgpu_fence_release(struct fence *f)
+static void amdgpu_fence_release(struct dma_fence *f)
 {
 	call_rcu(&f->rcu, amdgpu_fence_free);
 }
 
-static const struct fence_ops amdgpu_fence_ops = {
+static const struct dma_fence_ops amdgpu_fence_ops = {
 	.get_driver_name = amdgpu_fence_get_driver_name,
 	.get_timeline_name = amdgpu_fence_get_timeline_name,
 	.enable_signaling = amdgpu_fence_enable_signaling,
-	.wait = fence_default_wait,
+	.wait = dma_fence_default_wait,
 	.release = amdgpu_fence_release,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index 21a1242fc13b..964d2a946ed5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -126,7 +126,8 @@ int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev)
 	if (adev->gart.robj == NULL) {
 		r = amdgpu_bo_create(adev, adev->gart.table_size,
 				     PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
-				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+				     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 				     NULL, NULL, &adev->gart.robj);
 		if (r) {
 			return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index a7ea9a3b454e..cd62f6ffde2a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -116,10 +116,11 @@ void amdgpu_gem_force_release(struct amdgpu_device *adev)
  * Call from drm_gem_handle_create which appear in both new and open ioctl
  * case.
  */
-int amdgpu_gem_object_open(struct drm_gem_object *obj, struct drm_file *file_priv)
+int amdgpu_gem_object_open(struct drm_gem_object *obj,
+			   struct drm_file *file_priv)
 {
 	struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj);
-	struct amdgpu_device *adev = abo->adev;
+	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
 	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
 	struct amdgpu_vm *vm = &fpriv->vm;
 	struct amdgpu_bo_va *bo_va;
@@ -142,7 +143,7 @@ void amdgpu_gem_object_close(struct drm_gem_object *obj,
 			     struct drm_file *file_priv)
 {
 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
-	struct amdgpu_device *adev = bo->adev;
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
 	struct amdgpu_vm *vm = &fpriv->vm;
 
@@ -407,10 +408,8 @@ int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
 		return -ENOENT;
 	}
 	robj = gem_to_amdgpu_bo(gobj);
-	if (timeout == 0)
-		ret = reservation_object_test_signaled_rcu(robj->tbo.resv, true);
-	else
-		ret = reservation_object_wait_timeout_rcu(robj->tbo.resv, true, true, timeout);
+	ret = reservation_object_wait_timeout_rcu(robj->tbo.resv, true, true,
+						  timeout);
 
 	/* ret == 0 means not signaled,
 	 * ret > 0 means signaled
@@ -470,6 +469,16 @@ out:
 	return r;
 }
 
+static int amdgpu_gem_va_check(void *param, struct amdgpu_bo *bo)
+{
+	unsigned domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
+
+	/* if anything is swapped out don't swap it in here,
+	   just abort and wait for the next CS */
+
+	return domain == AMDGPU_GEM_DOMAIN_CPU ? -ERESTARTSYS : 0;
+}
+
 /**
  * amdgpu_gem_va_update_vm -update the bo_va in its VM
  *
@@ -480,7 +489,8 @@ out:
  * vital here, so they are not reported back to userspace.
  */
 static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
-				    struct amdgpu_bo_va *bo_va, uint32_t operation)
+				    struct amdgpu_bo_va *bo_va,
+				    uint32_t operation)
 {
 	struct ttm_validate_buffer tv, *entry;
 	struct amdgpu_bo_list_entry vm_pd;
@@ -503,7 +513,6 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
 	if (r)
 		goto error_print;
 
-	amdgpu_vm_get_pt_bos(adev, bo_va->vm, &duplicates);
 	list_for_each_entry(entry, &list, head) {
 		domain = amdgpu_mem_type_to_domain(entry->bo->mem.mem_type);
 		/* if anything is swapped out don't swap it in here,
@@ -511,13 +520,10 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
 		if (domain == AMDGPU_GEM_DOMAIN_CPU)
 			goto error_unreserve;
 	}
-	list_for_each_entry(entry, &duplicates, head) {
-		domain = amdgpu_mem_type_to_domain(entry->bo->mem.mem_type);
-		/* if anything is swapped out don't swap it in here,
-		   just abort and wait for the next CS */
-		if (domain == AMDGPU_GEM_DOMAIN_CPU)
-			goto error_unreserve;
-	}
+	r = amdgpu_vm_validate_pt_bos(adev, bo_va->vm, amdgpu_gem_va_check,
+				      NULL);
+	if (r)
+		goto error_unreserve;
 
 	r = amdgpu_vm_update_page_directory(adev, bo_va->vm);
 	if (r)
@@ -538,8 +544,6 @@ error_print:
 		DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
 }
 
-
-
 int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 			  struct drm_file *filp)
 {
@@ -549,7 +553,8 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 	struct amdgpu_bo *abo;
 	struct amdgpu_bo_va *bo_va;
-	struct ttm_validate_buffer tv, tv_pd;
+	struct amdgpu_bo_list_entry vm_pd;
+	struct ttm_validate_buffer tv;
 	struct ww_acquire_ctx ticket;
 	struct list_head list, duplicates;
 	uint32_t invalid_flags, va_flags = 0;
@@ -594,9 +599,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 	tv.shared = true;
 	list_add(&tv.head, &list);
 
-	tv_pd.bo = &fpriv->vm.page_directory->tbo;
-	tv_pd.shared = true;
-	list_add(&tv_pd.head, &list);
+	amdgpu_vm_get_pd_bo(&fpriv->vm, &list, &vm_pd);
 
 	r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates);
 	if (r) {
@@ -704,7 +707,8 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 	uint32_t handle;
 	int r;
 
-	args->pitch = amdgpu_align_pitch(adev, args->width, args->bpp, 0) * ((args->bpp + 1) / 8);
+	args->pitch = amdgpu_align_pitch(adev, args->width,
+					 DIV_ROUND_UP(args->bpp, 8), 0);
 	args->size = (u64)args->pitch * args->height;
 	args->size = ALIGN(args->size, PAGE_SIZE);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index a074edd95c70..01a42b6a69a4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -24,6 +24,7 @@
  */
 #include <drm/drmP.h>
 #include "amdgpu.h"
+#include "amdgpu_gfx.h"
 
 /*
  * GPU scratch registers helpers function.
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index 51321e154c09..e02044086445 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -27,6 +27,7 @@
 int amdgpu_gfx_scratch_get(struct amdgpu_device *adev, uint32_t *reg);
 void amdgpu_gfx_scratch_free(struct amdgpu_device *adev, uint32_t reg);
 
-unsigned amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_sh);
+void amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se,
+		unsigned max_sh);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index f86c84427778..00f46b0e076d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -164,10 +164,13 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
 	spin_unlock(&mgr->lock);
 
 	node = kzalloc(sizeof(*node), GFP_KERNEL);
-	if (!node)
-		return -ENOMEM;
+	if (!node) {
+		r = -ENOMEM;
+		goto err_out;
+	}
 
 	node->start = AMDGPU_BO_INVALID_OFFSET;
+	node->size = mem->num_pages;
 	mem->mm_node = node;
 
 	if (place->fpfn || place->lpfn || place->flags & TTM_PL_FLAG_TOPDOWN) {
@@ -175,12 +178,20 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
 		if (unlikely(r)) {
 			kfree(node);
 			mem->mm_node = NULL;
+			r = 0;
+			goto err_out;
 		}
 	} else {
 		mem->start = node->start;
 	}
 
 	return 0;
+err_out:
+	spin_lock(&mgr->lock);
+	mgr->available += mem->num_pages;
+	spin_unlock(&mgr->lock);
+
+	return r;
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 6a6c86c9c169..216a9572d946 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -89,7 +89,7 @@ int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  * Free an IB (all asics).
  */
 void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib,
-		    struct fence *f)
+		    struct dma_fence *f)
 {
 	amdgpu_sa_bo_free(adev, &ib->sa_bo, f);
 }
@@ -116,8 +116,8 @@ void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib,
  * to SI there was just a DE IB.
  */
 int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
-		       struct amdgpu_ib *ibs, struct fence *last_vm_update,
-		       struct amdgpu_job *job, struct fence **f)
+		       struct amdgpu_ib *ibs, struct dma_fence *last_vm_update,
+		       struct amdgpu_job *job, struct dma_fence **f)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_ib *ib = &ibs[0];
@@ -152,8 +152,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 		return -EINVAL;
 	}
 
-	alloc_size = amdgpu_ring_get_dma_frame_size(ring) +
-		num_ibs * amdgpu_ring_get_emit_ib_size(ring);
+	alloc_size = ring->funcs->emit_frame_size + num_ibs *
+		ring->funcs->emit_ib_size;
 
 	r = amdgpu_ring_alloc(ring, alloc_size);
 	if (r) {
@@ -161,7 +161,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 		return r;
 	}
 
-	if (ring->type == AMDGPU_RING_TYPE_SDMA && ring->funcs->init_cond_exec)
+	if (ring->funcs->init_cond_exec)
 		patch_offset = amdgpu_ring_init_cond_exec(ring);
 
 	if (vm) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 9fa809876339..fb902932f571 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -424,15 +424,6 @@ int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
 	return 0;
 }
 
-bool amdgpu_irq_get_delayed(struct amdgpu_device *adev,
-			struct amdgpu_irq_src *src,
-			unsigned type)
-{
-	if ((type >= src->num_types) || !src->enabled_types)
-		return false;
-	return atomic_inc_return(&src->enabled_types[type]) == 1;
-}
-
 /**
  * amdgpu_irq_put - disable interrupt
  *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
index f016464035b8..1642f4108297 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
@@ -88,9 +88,6 @@ int amdgpu_irq_update(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
 		      unsigned type);
 int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
 		   unsigned type);
-bool amdgpu_irq_get_delayed(struct amdgpu_device *adev,
-			    struct amdgpu_irq_src *src,
-			    unsigned type);
 int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
 		   unsigned type);
 bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 8c5807994073..a0de6286c453 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -81,7 +81,7 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
 
 void amdgpu_job_free_resources(struct amdgpu_job *job)
 {
-	struct fence *f;
+	struct dma_fence *f;
 	unsigned i;
 
 	/* use sched fence if available */
@@ -95,7 +95,7 @@ static void amdgpu_job_free_cb(struct amd_sched_job *s_job)
 {
 	struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base);
 
-	fence_put(job->fence);
+	dma_fence_put(job->fence);
 	amdgpu_sync_free(&job->sync);
 	kfree(job);
 }
@@ -104,14 +104,14 @@ void amdgpu_job_free(struct amdgpu_job *job)
 {
 	amdgpu_job_free_resources(job);
 
-	fence_put(job->fence);
+	dma_fence_put(job->fence);
 	amdgpu_sync_free(&job->sync);
 	kfree(job);
 }
 
 int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
 		      struct amd_sched_entity *entity, void *owner,
-		      struct fence **f)
+		      struct dma_fence **f)
 {
 	int r;
 	job->ring = ring;
@@ -125,19 +125,19 @@ int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
 
 	job->owner = owner;
 	job->fence_ctx = entity->fence_context;
-	*f = fence_get(&job->base.s_fence->finished);
+	*f = dma_fence_get(&job->base.s_fence->finished);
 	amdgpu_job_free_resources(job);
 	amd_sched_entity_push_job(&job->base);
 
 	return 0;
 }
 
-static struct fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
+static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
 {
 	struct amdgpu_job *job = to_amdgpu_job(sched_job);
 	struct amdgpu_vm *vm = job->vm;
 
-	struct fence *fence = amdgpu_sync_get_fence(&job->sync);
+	struct dma_fence *fence = amdgpu_sync_get_fence(&job->sync);
 
 	if (fence == NULL && vm && !job->vm_id) {
 		struct amdgpu_ring *ring = job->ring;
@@ -155,9 +155,9 @@ static struct fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
 	return fence;
 }
 
-static struct fence *amdgpu_job_run(struct amd_sched_job *sched_job)
+static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
 {
-	struct fence *fence = NULL;
+	struct dma_fence *fence = NULL;
 	struct amdgpu_job *job;
 	int r;
 
@@ -176,8 +176,8 @@ static struct fence *amdgpu_job_run(struct amd_sched_job *sched_job)
 		DRM_ERROR("Error scheduling IBs (%d)\n", r);
 
 	/* if gpu reset, hw fence will be replaced here */
-	fence_put(job->fence);
-	job->fence = fence_get(fence);
+	dma_fence_put(job->fence);
+	job->fence = dma_fence_get(fence);
 	amdgpu_job_free_resources(job);
 	return fence;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 3938fca1ea8e..9af87eaf8ee3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -308,10 +308,10 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 		}
 
 		for (i = 0; i < adev->num_ip_blocks; i++) {
-			if (adev->ip_blocks[i].type == type &&
-			    adev->ip_block_status[i].valid) {
-				ip.hw_ip_version_major = adev->ip_blocks[i].major;
-				ip.hw_ip_version_minor = adev->ip_blocks[i].minor;
+			if (adev->ip_blocks[i].version->type == type &&
+			    adev->ip_blocks[i].status.valid) {
+				ip.hw_ip_version_major = adev->ip_blocks[i].version->major;
+				ip.hw_ip_version_minor = adev->ip_blocks[i].version->minor;
 				ip.capabilities_flags = 0;
 				ip.available_rings = ring_mask;
 				ip.ib_start_alignment = ib_start_alignment;
@@ -347,8 +347,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 		}
 
 		for (i = 0; i < adev->num_ip_blocks; i++)
-			if (adev->ip_blocks[i].type == type &&
-			    adev->ip_block_status[i].valid &&
+			if (adev->ip_blocks[i].version->type == type &&
+			    adev->ip_blocks[i].status.valid &&
 			    count < AMDGPU_HW_IP_INSTANCE_MAX_COUNT)
 				count++;
 
@@ -413,6 +413,36 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 		return copy_to_user(out, &vram_gtt,
 				    min((size_t)size, sizeof(vram_gtt))) ? -EFAULT : 0;
 	}
+	case AMDGPU_INFO_MEMORY: {
+		struct drm_amdgpu_memory_info mem;
+
+		memset(&mem, 0, sizeof(mem));
+		mem.vram.total_heap_size = adev->mc.real_vram_size;
+		mem.vram.usable_heap_size =
+			adev->mc.real_vram_size - adev->vram_pin_size;
+		mem.vram.heap_usage = atomic64_read(&adev->vram_usage);
+		mem.vram.max_allocation = mem.vram.usable_heap_size * 3 / 4;
+
+		mem.cpu_accessible_vram.total_heap_size =
+			adev->mc.visible_vram_size;
+		mem.cpu_accessible_vram.usable_heap_size =
+			adev->mc.visible_vram_size -
+			(adev->vram_pin_size - adev->invisible_pin_size);
+		mem.cpu_accessible_vram.heap_usage =
+			atomic64_read(&adev->vram_vis_usage);
+		mem.cpu_accessible_vram.max_allocation =
+			mem.cpu_accessible_vram.usable_heap_size * 3 / 4;
+
+		mem.gtt.total_heap_size = adev->mc.gtt_size;
+		mem.gtt.usable_heap_size =
+			adev->mc.gtt_size - adev->gart_pin_size;
+		mem.gtt.heap_usage = atomic64_read(&adev->gtt_usage);
+		mem.gtt.max_allocation = mem.gtt.usable_heap_size * 3 / 4;
+
+		return copy_to_user(out, &mem,
+				    min((size_t)size, sizeof(mem)))
+				    ? -EFAULT : 0;
+	}
 	case AMDGPU_INFO_READ_MMR_REG: {
 		unsigned n, alloc_size;
 		uint32_t *regs;
@@ -475,6 +505,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 		dev_info.ids_flags = 0;
 		if (adev->flags & AMD_IS_APU)
 			dev_info.ids_flags |= AMDGPU_IDS_FLAGS_FUSION;
+		if (amdgpu_sriov_vf(adev))
+			dev_info.ids_flags |= AMDGPU_IDS_FLAGS_PREEMPTION;
 		dev_info.virtual_address_offset = AMDGPU_VA_RESERVED_SIZE;
 		dev_info.virtual_address_max = (uint64_t)adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
 		dev_info.virtual_address_alignment = max((int)PAGE_SIZE, AMDGPU_GPU_PAGE_SIZE);
@@ -494,6 +526,50 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 		return copy_to_user(out, &dev_info,
 				    min((size_t)size, sizeof(dev_info))) ? -EFAULT : 0;
 	}
+	case AMDGPU_INFO_VCE_CLOCK_TABLE: {
+		unsigned i;
+		struct drm_amdgpu_info_vce_clock_table vce_clk_table = {};
+		struct amd_vce_state *vce_state;
+
+		for (i = 0; i < AMDGPU_VCE_CLOCK_TABLE_ENTRIES; i++) {
+			vce_state = amdgpu_dpm_get_vce_clock_state(adev, i);
+			if (vce_state) {
+				vce_clk_table.entries[i].sclk = vce_state->sclk;
+				vce_clk_table.entries[i].mclk = vce_state->mclk;
+				vce_clk_table.entries[i].eclk = vce_state->evclk;
+				vce_clk_table.num_valid_entries++;
+			}
+		}
+
+		return copy_to_user(out, &vce_clk_table,
+				    min((size_t)size, sizeof(vce_clk_table))) ? -EFAULT : 0;
+	}
+	case AMDGPU_INFO_VBIOS: {
+		uint32_t bios_size = adev->bios_size;
+
+		switch (info->vbios_info.type) {
+		case AMDGPU_INFO_VBIOS_SIZE:
+			return copy_to_user(out, &bios_size,
+					min((size_t)size, sizeof(bios_size)))
+					? -EFAULT : 0;
+		case AMDGPU_INFO_VBIOS_IMAGE: {
+			uint8_t *bios;
+			uint32_t bios_offset = info->vbios_info.offset;
+
+			if (bios_offset >= bios_size)
+				return -EINVAL;
+
+			bios = adev->bios + bios_offset;
+			return copy_to_user(out, bios,
+					    min((size_t)size, (size_t)(bios_size - bios_offset)))
+					? -EFAULT : 0;
+		}
+		default:
+			DRM_DEBUG_KMS("Invalid request %d\n",
+					info->vbios_info.type);
+			return -EINVAL;
+		}
+	}
 	default:
 		DRM_DEBUG_KMS("Invalid request %d\n", info->query);
 		return -EINVAL;
@@ -775,6 +851,7 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = {
 	DRM_IOCTL_DEF_DRV(AMDGPU_CS, amdgpu_cs_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(AMDGPU_INFO, amdgpu_info_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(AMDGPU_WAIT_CS, amdgpu_cs_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(AMDGPU_WAIT_FENCES, amdgpu_cs_wait_fences_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(AMDGPU_GEM_METADATA, amdgpu_gem_metadata_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(AMDGPU_GEM_VA, amdgpu_gem_va_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(AMDGPU_GEM_OP, amdgpu_gem_op_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
index 32fa7b7913f7..7ea3cacf9f9f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
@@ -285,7 +285,7 @@ free_rmn:
 int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
 {
 	unsigned long end = addr + amdgpu_bo_size(bo) - 1;
-	struct amdgpu_device *adev = bo->adev;
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	struct amdgpu_mn *rmn;
 	struct amdgpu_mn_node *node = NULL;
 	struct list_head bos;
@@ -340,7 +340,7 @@ int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
  */
 void amdgpu_mn_unregister(struct amdgpu_bo *bo)
 {
-	struct amdgpu_device *adev = bo->adev;
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	struct amdgpu_mn *rmn;
 	struct list_head *head;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index 7b0eff7d060b..202b4176b74e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -271,8 +271,6 @@ struct amdgpu_display_funcs {
 	u32 (*vblank_get_counter)(struct amdgpu_device *adev, int crtc);
 	/* wait for vblank */
 	void (*vblank_wait)(struct amdgpu_device *adev, int crtc);
-	/* is dce hung */
-	bool (*is_display_hung)(struct amdgpu_device *adev);
 	/* set backlight level */
 	void (*backlight_set_level)(struct amdgpu_encoder *amdgpu_encoder,
 				    u8 level);
@@ -341,8 +339,6 @@ struct amdgpu_mode_info {
 	int			num_dig; /* number of dig blocks */
 	int			disp_priority;
 	const struct amdgpu_display_funcs *funcs;
-	struct hrtimer vblank_timer;
-	enum amdgpu_interrupt_state vsync_timer_enabled;
 };
 
 #define AMDGPU_MAX_BL_LEVEL 0xFF
@@ -413,6 +409,9 @@ struct amdgpu_crtc {
 	u32 wm_high;
 	u32 lb_vblank_lead_lines;
 	struct drm_display_mode hw_mode;
+	/* for virtual dce */
+	struct hrtimer vblank_timer;
+	enum amdgpu_interrupt_state vsync_timer_enabled;
 };
 
 struct amdgpu_encoder_atom_dig {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index f3efb1c5dae9..bf79b73e1538 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -88,18 +88,19 @@ static void amdgpu_update_memory_usage(struct amdgpu_device *adev,
 
 static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
 	struct amdgpu_bo *bo;
 
 	bo = container_of(tbo, struct amdgpu_bo, tbo);
 
-	amdgpu_update_memory_usage(bo->adev, &bo->tbo.mem, NULL);
+	amdgpu_update_memory_usage(adev, &bo->tbo.mem, NULL);
 
 	drm_gem_object_release(&bo->gem_base);
 	amdgpu_bo_unref(&bo->parent);
 	if (!list_empty(&bo->shadow_list)) {
-		mutex_lock(&bo->adev->shadow_list_lock);
+		mutex_lock(&adev->shadow_list_lock);
 		list_del_init(&bo->shadow_list);
-		mutex_unlock(&bo->adev->shadow_list_lock);
+		mutex_unlock(&adev->shadow_list_lock);
 	}
 	kfree(bo->metadata);
 	kfree(bo);
@@ -121,20 +122,14 @@ static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
 
 	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
 		unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
+		unsigned lpfn = 0;
 
-		if (flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS &&
-		    !(flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
-		    adev->mc.visible_vram_size < adev->mc.real_vram_size) {
-			places[c].fpfn = visible_pfn;
-			places[c].lpfn = 0;
-			places[c].flags = TTM_PL_FLAG_WC |
-				TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_VRAM |
-				TTM_PL_FLAG_TOPDOWN;
-			c++;
-		}
+		/* This forces a reallocation if the flag wasn't set before */
+		if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
+			lpfn = adev->mc.real_vram_size >> PAGE_SHIFT;
 
 		places[c].fpfn = 0;
-		places[c].lpfn = 0;
+		places[c].lpfn = lpfn;
 		places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
 			TTM_PL_FLAG_VRAM;
 		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
@@ -205,8 +200,10 @@ static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
 
 void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 {
-	amdgpu_ttm_placement_init(abo->adev, &abo->placement,
-				  abo->placements, domain, abo->flags);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
+
+	amdgpu_ttm_placement_init(adev, &abo->placement, abo->placements,
+				  domain, abo->flags);
 }
 
 static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
@@ -245,7 +242,8 @@ int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
 	int r;
 
 	r = amdgpu_bo_create(adev, size, align, true, domain,
-			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 			     NULL, NULL, bo_ptr);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", r);
@@ -351,7 +349,6 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 		kfree(bo);
 		return r;
 	}
-	bo->adev = adev;
 	INIT_LIST_HEAD(&bo->shadow_list);
 	INIT_LIST_HEAD(&bo->va);
 	bo->prefered_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
@@ -374,39 +371,36 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 
 	amdgpu_fill_placement_to_bo(bo, placement);
 	/* Kernel allocation are uninterruptible */
+
+	if (!resv) {
+		bool locked;
+
+		reservation_object_init(&bo->tbo.ttm_resv);
+		locked = ww_mutex_trylock(&bo->tbo.ttm_resv.lock);
+		WARN_ON(!locked);
+	}
 	r = ttm_bo_init(&adev->mman.bdev, &bo->tbo, size, type,
 			&bo->placement, page_align, !kernel, NULL,
-			acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
-	if (unlikely(r != 0)) {
+			acc_size, sg, resv ? resv : &bo->tbo.ttm_resv,
+			&amdgpu_ttm_bo_destroy);
+	if (unlikely(r != 0))
 		return r;
-	}
 
 	if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
 	    bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
-		struct fence *fence;
-
-		if (adev->mman.buffer_funcs_ring == NULL ||
-		   !adev->mman.buffer_funcs_ring->ready) {
-			r = -EBUSY;
-			goto fail_free;
-		}
-
-		r = amdgpu_bo_reserve(bo, false);
-		if (unlikely(r != 0))
-			goto fail_free;
+		struct dma_fence *fence;
 
-		amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);
-		r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
-		if (unlikely(r != 0))
+		r = amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence);
+		if (unlikely(r))
 			goto fail_unreserve;
 
-		amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence);
 		amdgpu_bo_fence(bo, fence, false);
-		amdgpu_bo_unreserve(bo);
-		fence_put(bo->tbo.moving);
-		bo->tbo.moving = fence_get(fence);
-		fence_put(fence);
+		dma_fence_put(bo->tbo.moving);
+		bo->tbo.moving = dma_fence_get(fence);
+		dma_fence_put(fence);
 	}
+	if (!resv)
+		ww_mutex_unlock(&bo->tbo.resv->lock);
 	*bo_ptr = bo;
 
 	trace_amdgpu_bo_create(bo);
@@ -414,8 +408,7 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 	return 0;
 
 fail_unreserve:
-	amdgpu_bo_unreserve(bo);
-fail_free:
+	ww_mutex_unlock(&bo->tbo.resv->lock);
 	amdgpu_bo_unref(&bo);
 	return r;
 }
@@ -491,7 +484,7 @@ int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
 			       struct amdgpu_ring *ring,
 			       struct amdgpu_bo *bo,
 			       struct reservation_object *resv,
-			       struct fence **fence,
+			       struct dma_fence **fence,
 			       bool direct)
 
 {
@@ -523,7 +516,7 @@ int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
 				  struct amdgpu_ring *ring,
 				  struct amdgpu_bo *bo,
 				  struct reservation_object *resv,
-				  struct fence **fence,
+				  struct dma_fence **fence,
 				  bool direct)
 
 {
@@ -616,6 +609,7 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 			     u64 min_offset, u64 max_offset,
 			     u64 *gpu_addr)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	int r, i;
 	unsigned fpfn, lpfn;
 
@@ -643,18 +637,20 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 
 		return 0;
 	}
+
+	bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
 	amdgpu_ttm_placement_from_domain(bo, domain);
 	for (i = 0; i < bo->placement.num_placement; i++) {
 		/* force to pin into visible video ram */
 		if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
 		    !(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) &&
 		    (!max_offset || max_offset >
-		     bo->adev->mc.visible_vram_size)) {
+		     adev->mc.visible_vram_size)) {
 			if (WARN_ON_ONCE(min_offset >
-					 bo->adev->mc.visible_vram_size))
+					 adev->mc.visible_vram_size))
 				return -EINVAL;
 			fpfn = min_offset >> PAGE_SHIFT;
-			lpfn = bo->adev->mc.visible_vram_size >> PAGE_SHIFT;
+			lpfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
 		} else {
 			fpfn = min_offset >> PAGE_SHIFT;
 			lpfn = max_offset >> PAGE_SHIFT;
@@ -669,12 +665,12 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 
 	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
 	if (unlikely(r)) {
-		dev_err(bo->adev->dev, "%p pin failed\n", bo);
+		dev_err(adev->dev, "%p pin failed\n", bo);
 		goto error;
 	}
 	r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
 	if (unlikely(r)) {
-		dev_err(bo->adev->dev, "%p bind failed\n", bo);
+		dev_err(adev->dev, "%p bind failed\n", bo);
 		goto error;
 	}
 
@@ -682,11 +678,11 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 	if (gpu_addr != NULL)
 		*gpu_addr = amdgpu_bo_gpu_offset(bo);
 	if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
-		bo->adev->vram_pin_size += amdgpu_bo_size(bo);
+		adev->vram_pin_size += amdgpu_bo_size(bo);
 		if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
-			bo->adev->invisible_pin_size += amdgpu_bo_size(bo);
+			adev->invisible_pin_size += amdgpu_bo_size(bo);
 	} else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
-		bo->adev->gart_pin_size += amdgpu_bo_size(bo);
+		adev->gart_pin_size += amdgpu_bo_size(bo);
 	}
 
 error:
@@ -700,10 +696,11 @@ int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr)
 
 int amdgpu_bo_unpin(struct amdgpu_bo *bo)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	int r, i;
 
 	if (!bo->pin_count) {
-		dev_warn(bo->adev->dev, "%p unpin not necessary\n", bo);
+		dev_warn(adev->dev, "%p unpin not necessary\n", bo);
 		return 0;
 	}
 	bo->pin_count--;
@@ -715,16 +712,16 @@ int amdgpu_bo_unpin(struct amdgpu_bo *bo)
 	}
 	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
 	if (unlikely(r)) {
-		dev_err(bo->adev->dev, "%p validate failed for unpin\n", bo);
+		dev_err(adev->dev, "%p validate failed for unpin\n", bo);
 		goto error;
 	}
 
 	if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
-		bo->adev->vram_pin_size -= amdgpu_bo_size(bo);
+		adev->vram_pin_size -= amdgpu_bo_size(bo);
 		if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
-			bo->adev->invisible_pin_size -= amdgpu_bo_size(bo);
+			adev->invisible_pin_size -= amdgpu_bo_size(bo);
 	} else if (bo->tbo.mem.mem_type == TTM_PL_TT) {
-		bo->adev->gart_pin_size -= amdgpu_bo_size(bo);
+		adev->gart_pin_size -= amdgpu_bo_size(bo);
 	}
 
 error:
@@ -854,6 +851,7 @@ int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 			   struct ttm_mem_reg *new_mem)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 	struct amdgpu_bo *abo;
 	struct ttm_mem_reg *old_mem = &bo->mem;
 
@@ -861,21 +859,21 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 		return;
 
 	abo = container_of(bo, struct amdgpu_bo, tbo);
-	amdgpu_vm_bo_invalidate(abo->adev, abo);
+	amdgpu_vm_bo_invalidate(adev, abo);
 
 	/* update statistics */
 	if (!new_mem)
 		return;
 
 	/* move_notify is called before move happens */
-	amdgpu_update_memory_usage(abo->adev, &bo->mem, new_mem);
+	amdgpu_update_memory_usage(adev, &bo->mem, new_mem);
 
 	trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
 }
 
 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 {
-	struct amdgpu_device *adev;
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 	struct amdgpu_bo *abo;
 	unsigned long offset, size, lpfn;
 	int i, r;
@@ -884,13 +882,14 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 		return 0;
 
 	abo = container_of(bo, struct amdgpu_bo, tbo);
-	adev = abo->adev;
 	if (bo->mem.mem_type != TTM_PL_VRAM)
 		return 0;
 
 	size = bo->mem.num_pages << PAGE_SHIFT;
 	offset = bo->mem.start << PAGE_SHIFT;
-	if ((offset + size) <= adev->mc.visible_vram_size)
+	/* TODO: figure out how to map scattered VRAM to the CPU */
+	if ((offset + size) <= adev->mc.visible_vram_size &&
+	    (abo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS))
 		return 0;
 
 	/* Can't move a pinned BO to visible VRAM */
@@ -898,6 +897,7 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 		return -EINVAL;
 
 	/* hurrah the memory is not visible ! */
+	abo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
 	amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM);
 	lpfn =	adev->mc.visible_vram_size >> PAGE_SHIFT;
 	for (i = 0; i < abo->placement.num_placement; i++) {
@@ -931,7 +931,7 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
  * @shared: true if fence should be added shared
  *
  */
-void amdgpu_bo_fence(struct amdgpu_bo *bo, struct fence *fence,
+void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
 		     bool shared)
 {
 	struct reservation_object *resv = bo->tbo.resv;
@@ -959,6 +959,8 @@ u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
 	WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
 		     !bo->pin_count);
 	WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);
+	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM &&
+		     !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
 
 	return bo->tbo.offset;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 8255034d73eb..5cbf59ec0f68 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -71,12 +71,13 @@ static inline unsigned amdgpu_mem_type_to_domain(u32 mem_type)
  */
 static inline int amdgpu_bo_reserve(struct amdgpu_bo *bo, bool no_intr)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	int r;
 
 	r = ttm_bo_reserve(&bo->tbo, !no_intr, false, NULL);
 	if (unlikely(r != 0)) {
 		if (r != -ERESTARTSYS)
-			dev_err(bo->adev->dev, "%p reserve failed\n", bo);
+			dev_err(adev->dev, "%p reserve failed\n", bo);
 		return r;
 	}
 	return 0;
@@ -156,19 +157,19 @@ int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 				  struct ttm_mem_reg *new_mem);
 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo);
-void amdgpu_bo_fence(struct amdgpu_bo *bo, struct fence *fence,
+void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
 		     bool shared);
 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo);
 int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
 			       struct amdgpu_ring *ring,
 			       struct amdgpu_bo *bo,
 			       struct reservation_object *resv,
-			       struct fence **fence, bool direct);
+			       struct dma_fence **fence, bool direct);
 int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
 				  struct amdgpu_ring *ring,
 				  struct amdgpu_bo *bo,
 				  struct reservation_object *resv,
-				  struct fence **fence,
+				  struct dma_fence **fence,
 				  bool direct);
 
 
@@ -200,7 +201,7 @@ int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
 		     unsigned size, unsigned align);
 void amdgpu_sa_bo_free(struct amdgpu_device *adev,
 			      struct amdgpu_sa_bo **sa_bo,
-			      struct fence *fence);
+			      struct dma_fence *fence);
 #if defined(CONFIG_DEBUG_FS)
 void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
 					 struct seq_file *m);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index accc908bdc88..723ae682bf25 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -737,6 +737,21 @@ static ssize_t amdgpu_hwmon_get_pwm1(struct device *dev,
 	return sprintf(buf, "%i\n", speed);
 }
 
+static ssize_t amdgpu_hwmon_get_fan1_input(struct device *dev,
+					   struct device_attribute *attr,
+					   char *buf)
+{
+	struct amdgpu_device *adev = dev_get_drvdata(dev);
+	int err;
+	u32 speed;
+
+	err = amdgpu_dpm_get_fan_speed_rpm(adev, &speed);
+	if (err)
+		return err;
+
+	return sprintf(buf, "%i\n", speed);
+}
+
 static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, amdgpu_hwmon_show_temp, NULL, 0);
 static SENSOR_DEVICE_ATTR(temp1_crit, S_IRUGO, amdgpu_hwmon_show_temp_thresh, NULL, 0);
 static SENSOR_DEVICE_ATTR(temp1_crit_hyst, S_IRUGO, amdgpu_hwmon_show_temp_thresh, NULL, 1);
@@ -744,6 +759,7 @@ static SENSOR_DEVICE_ATTR(pwm1, S_IRUGO | S_IWUSR, amdgpu_hwmon_get_pwm1, amdgpu
 static SENSOR_DEVICE_ATTR(pwm1_enable, S_IRUGO | S_IWUSR, amdgpu_hwmon_get_pwm1_enable, amdgpu_hwmon_set_pwm1_enable, 0);
 static SENSOR_DEVICE_ATTR(pwm1_min, S_IRUGO, amdgpu_hwmon_get_pwm1_min, NULL, 0);
 static SENSOR_DEVICE_ATTR(pwm1_max, S_IRUGO, amdgpu_hwmon_get_pwm1_max, NULL, 0);
+static SENSOR_DEVICE_ATTR(fan1_input, S_IRUGO, amdgpu_hwmon_get_fan1_input, NULL, 0);
 
 static struct attribute *hwmon_attributes[] = {
 	&sensor_dev_attr_temp1_input.dev_attr.attr,
@@ -753,6 +769,7 @@ static struct attribute *hwmon_attributes[] = {
 	&sensor_dev_attr_pwm1_enable.dev_attr.attr,
 	&sensor_dev_attr_pwm1_min.dev_attr.attr,
 	&sensor_dev_attr_pwm1_max.dev_attr.attr,
+	&sensor_dev_attr_fan1_input.dev_attr.attr,
 	NULL
 };
 
@@ -804,6 +821,10 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
 	     attr == &sensor_dev_attr_pwm1_min.dev_attr.attr))
 		return 0;
 
+	/* requires powerplay */
+	if (attr == &sensor_dev_attr_fan1_input.dev_attr.attr)
+		return 0;
+
 	return effective_mode;
 }
 
@@ -986,10 +1007,10 @@ restart_search:
 
 static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
 {
-	int i;
 	struct amdgpu_ps *ps;
 	enum amd_pm_state_type dpm_state;
 	int ret;
+	bool equal;
 
 	/* if dpm init failed */
 	if (!adev->pm.dpm_enabled)
@@ -1009,46 +1030,6 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
 	else
 		return;
 
-	/* no need to reprogram if nothing changed unless we are on BTC+ */
-	if (adev->pm.dpm.current_ps == adev->pm.dpm.requested_ps) {
-		/* vce just modifies an existing state so force a change */
-		if (ps->vce_active != adev->pm.dpm.vce_active)
-			goto force;
-		if (adev->flags & AMD_IS_APU) {
-			/* for APUs if the num crtcs changed but state is the same,
-			 * all we need to do is update the display configuration.
-			 */
-			if (adev->pm.dpm.new_active_crtcs != adev->pm.dpm.current_active_crtcs) {
-				/* update display watermarks based on new power state */
-				amdgpu_display_bandwidth_update(adev);
-				/* update displays */
-				amdgpu_dpm_display_configuration_changed(adev);
-				adev->pm.dpm.current_active_crtcs = adev->pm.dpm.new_active_crtcs;
-				adev->pm.dpm.current_active_crtc_count = adev->pm.dpm.new_active_crtc_count;
-			}
-			return;
-		} else {
-			/* for BTC+ if the num crtcs hasn't changed and state is the same,
-			 * nothing to do, if the num crtcs is > 1 and state is the same,
-			 * update display configuration.
-			 */
-			if (adev->pm.dpm.new_active_crtcs ==
-			    adev->pm.dpm.current_active_crtcs) {
-				return;
-			} else if ((adev->pm.dpm.current_active_crtc_count > 1) &&
-				   (adev->pm.dpm.new_active_crtc_count > 1)) {
-				/* update display watermarks based on new power state */
-				amdgpu_display_bandwidth_update(adev);
-				/* update displays */
-				amdgpu_dpm_display_configuration_changed(adev);
-				adev->pm.dpm.current_active_crtcs = adev->pm.dpm.new_active_crtcs;
-				adev->pm.dpm.current_active_crtc_count = adev->pm.dpm.new_active_crtc_count;
-				return;
-			}
-		}
-	}
-
-force:
 	if (amdgpu_dpm == 1) {
 		printk("switching from power state:\n");
 		amdgpu_dpm_print_power_state(adev, adev->pm.dpm.current_ps);
@@ -1059,31 +1040,21 @@ force:
 	/* update whether vce is active */
 	ps->vce_active = adev->pm.dpm.vce_active;
 
+	amdgpu_dpm_display_configuration_changed(adev);
+
 	ret = amdgpu_dpm_pre_set_power_state(adev);
 	if (ret)
 		return;
 
-	/* update display watermarks based on new power state */
-	amdgpu_display_bandwidth_update(adev);
+	if ((0 != amgdpu_dpm_check_state_equal(adev, adev->pm.dpm.current_ps, adev->pm.dpm.requested_ps, &equal)))
+		equal = false;
 
-	/* wait for the rings to drain */
-	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
-		struct amdgpu_ring *ring = adev->rings[i];
-		if (ring && ring->ready)
-			amdgpu_fence_wait_empty(ring);
-	}
+	if (equal)
+		return;
 
-	/* program the new power state */
 	amdgpu_dpm_set_power_state(adev);
-
-	/* update current power state */
-	adev->pm.dpm.current_ps = adev->pm.dpm.requested_ps;
-
 	amdgpu_dpm_post_set_power_state(adev);
 
-	/* update displays */
-	amdgpu_dpm_display_configuration_changed(adev);
-
 	adev->pm.dpm.current_active_crtcs = adev->pm.dpm.new_active_crtcs;
 	adev->pm.dpm.current_active_crtc_count = adev->pm.dpm.new_active_crtc_count;
 
@@ -1135,7 +1106,7 @@ void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable)
 			mutex_lock(&adev->pm.mutex);
 			adev->pm.dpm.vce_active = true;
 			/* XXX select vce level based on ring/task */
-			adev->pm.dpm.vce_level = AMDGPU_VCE_LEVEL_AC_ALL;
+			adev->pm.dpm.vce_level = AMD_VCE_LEVEL_AC_ALL;
 			mutex_unlock(&adev->pm.mutex);
 		} else {
 			mutex_lock(&adev->pm.mutex);
@@ -1276,20 +1247,20 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
 	struct drm_device *ddev = adev->ddev;
 	struct drm_crtc *crtc;
 	struct amdgpu_crtc *amdgpu_crtc;
+	int i = 0;
 
 	if (!adev->pm.dpm_enabled)
 		return;
 
-	if (adev->pp_enabled) {
-		int i = 0;
+	amdgpu_display_bandwidth_update(adev);
 
-		amdgpu_display_bandwidth_update(adev);
-		for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
-			struct amdgpu_ring *ring = adev->rings[i];
-			if (ring && ring->ready)
-				amdgpu_fence_wait_empty(ring);
-		}
+	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
+		struct amdgpu_ring *ring = adev->rings[i];
+		if (ring && ring->ready)
+			amdgpu_fence_wait_empty(ring);
+	}
 
+	if (adev->pp_enabled) {
 		amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_DISPLAY_CONFIG_CHANGE, NULL, NULL);
 	} else {
 		mutex_lock(&adev->pm.mutex);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
index 7532ff822aa7..fc592c2b0e16 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
@@ -155,9 +155,6 @@ static int amdgpu_pp_sw_init(void *handle)
 		ret = adev->powerplay.ip_funcs->sw_init(
 					adev->powerplay.pp_handle);
 
-	if (adev->pp_enabled)
-		adev->pm.dpm_enabled = true;
-
 	return ret;
 }
 
@@ -187,6 +184,9 @@ static int amdgpu_pp_hw_init(void *handle)
 		ret = adev->powerplay.ip_funcs->hw_init(
 					adev->powerplay.pp_handle);
 
+	if ((amdgpu_dpm != 0) && !amdgpu_sriov_vf(adev))
+		adev->pm.dpm_enabled = true;
+
 	return ret;
 }
 
@@ -299,7 +299,7 @@ static int amdgpu_pp_soft_reset(void *handle)
 	return ret;
 }
 
-const struct amd_ip_funcs amdgpu_pp_ip_funcs = {
+static const struct amd_ip_funcs amdgpu_pp_ip_funcs = {
 	.name = "amdgpu_powerplay",
 	.early_init = amdgpu_pp_early_init,
 	.late_init = amdgpu_pp_late_init,
@@ -316,3 +316,12 @@ const struct amd_ip_funcs amdgpu_pp_ip_funcs = {
 	.set_clockgating_state = amdgpu_pp_set_clockgating_state,
 	.set_powergating_state = amdgpu_pp_set_powergating_state,
 };
+
+const struct amdgpu_ip_block_version amdgpu_pp_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_SMC,
+	.major = 1,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &amdgpu_pp_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.h
index da5cf47cfd99..c0c4bfdcdb14 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.h
@@ -23,11 +23,11 @@
  *
  */
 
-#ifndef __AMDGPU_POPWERPLAY_H__
-#define __AMDGPU_POPWERPLAY_H__
+#ifndef __AMDGPU_POWERPLAY_H__
+#define __AMDGPU_POWERPLAY_H__
 
 #include "amd_shared.h"
 
-extern const struct amd_ip_funcs amdgpu_pp_ip_funcs;
+extern const struct amdgpu_ip_block_version amdgpu_pp_ip_block;
 
-#endif /* __AMDSOC_DM_H__ */
+#endif /* __AMDGPU_POWERPLAY_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 3cb5e903cd62..4c992826d2d6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -65,7 +65,7 @@ int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw)
 {
 	/* Align requested size with padding so unlock_commit can
 	 * pad safely */
-	ndw = (ndw + ring->align_mask) & ~ring->align_mask;
+	ndw = (ndw + ring->funcs->align_mask) & ~ring->funcs->align_mask;
 
 	/* Make sure we aren't trying to allocate more space
 	 * than the maximum for one submission
@@ -94,7 +94,7 @@ void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 	int i;
 
 	for (i = 0; i < count; i++)
-		amdgpu_ring_write(ring, ring->nop);
+		amdgpu_ring_write(ring, ring->funcs->nop);
 }
 
 /** amdgpu_ring_generic_pad_ib - pad IB with NOP packets
@@ -106,8 +106,8 @@ void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
  */
 void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
 {
-	while (ib->length_dw & ring->align_mask)
-		ib->ptr[ib->length_dw++] = ring->nop;
+	while (ib->length_dw & ring->funcs->align_mask)
+		ib->ptr[ib->length_dw++] = ring->funcs->nop;
 }
 
 /**
@@ -125,8 +125,9 @@ void amdgpu_ring_commit(struct amdgpu_ring *ring)
 	uint32_t count;
 
 	/* We pad to match fetch size */
-	count = ring->align_mask + 1 - (ring->wptr & ring->align_mask);
-	count %= ring->align_mask + 1;
+	count = ring->funcs->align_mask + 1 -
+		(ring->wptr & ring->funcs->align_mask);
+	count %= ring->funcs->align_mask + 1;
 	ring->funcs->insert_nop(ring, count);
 
 	mb();
@@ -163,9 +164,8 @@ void amdgpu_ring_undo(struct amdgpu_ring *ring)
  * Returns 0 on success, error on failure.
  */
 int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
-		     unsigned max_dw, u32 nop, u32 align_mask,
-		     struct amdgpu_irq_src *irq_src, unsigned irq_type,
-		     enum amdgpu_ring_type ring_type)
+		     unsigned max_dw, struct amdgpu_irq_src *irq_src,
+		     unsigned irq_type)
 {
 	int r;
 
@@ -216,9 +216,6 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 
 	ring->ring_size = roundup_pow_of_two(max_dw * 4 *
 					     amdgpu_sched_hw_submission);
-	ring->align_mask = align_mask;
-	ring->nop = nop;
-	ring->type = ring_type;
 
 	/* Allocate ring buffer */
 	if (ring->ring_obj == NULL) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
new file mode 100644
index 000000000000..574f0b79c690
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -0,0 +1,186 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Christian König
+ */
+#ifndef __AMDGPU_RING_H__
+#define __AMDGPU_RING_H__
+
+#include "gpu_scheduler.h"
+
+/* max number of rings */
+#define AMDGPU_MAX_RINGS		16
+#define AMDGPU_MAX_GFX_RINGS		1
+#define AMDGPU_MAX_COMPUTE_RINGS	8
+#define AMDGPU_MAX_VCE_RINGS		3
+
+/* some special values for the owner field */
+#define AMDGPU_FENCE_OWNER_UNDEFINED	((void*)0ul)
+#define AMDGPU_FENCE_OWNER_VM		((void*)1ul)
+
+#define AMDGPU_FENCE_FLAG_64BIT         (1 << 0)
+#define AMDGPU_FENCE_FLAG_INT           (1 << 1)
+
+enum amdgpu_ring_type {
+	AMDGPU_RING_TYPE_GFX,
+	AMDGPU_RING_TYPE_COMPUTE,
+	AMDGPU_RING_TYPE_SDMA,
+	AMDGPU_RING_TYPE_UVD,
+	AMDGPU_RING_TYPE_VCE,
+	AMDGPU_RING_TYPE_KIQ
+};
+
+struct amdgpu_device;
+struct amdgpu_ring;
+struct amdgpu_ib;
+struct amdgpu_cs_parser;
+
+/*
+ * Fences.
+ */
+struct amdgpu_fence_driver {
+	uint64_t			gpu_addr;
+	volatile uint32_t		*cpu_addr;
+	/* sync_seq is protected by ring emission lock */
+	uint32_t			sync_seq;
+	atomic_t			last_seq;
+	bool				initialized;
+	struct amdgpu_irq_src		*irq_src;
+	unsigned			irq_type;
+	struct timer_list		fallback_timer;
+	unsigned			num_fences_mask;
+	spinlock_t			lock;
+	struct dma_fence		**fences;
+};
+
+int amdgpu_fence_driver_init(struct amdgpu_device *adev);
+void amdgpu_fence_driver_fini(struct amdgpu_device *adev);
+void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev);
+
+int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
+				  unsigned num_hw_submission);
+int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
+				   struct amdgpu_irq_src *irq_src,
+				   unsigned irq_type);
+void amdgpu_fence_driver_suspend(struct amdgpu_device *adev);
+void amdgpu_fence_driver_resume(struct amdgpu_device *adev);
+int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **fence);
+void amdgpu_fence_process(struct amdgpu_ring *ring);
+int amdgpu_fence_wait_empty(struct amdgpu_ring *ring);
+unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring);
+
+/*
+ * Rings.
+ */
+
+/* provided by hw blocks that expose a ring buffer for commands */
+struct amdgpu_ring_funcs {
+	enum amdgpu_ring_type	type;
+	uint32_t		align_mask;
+	u32			nop;
+
+	/* ring read/write ptr handling */
+	u32 (*get_rptr)(struct amdgpu_ring *ring);
+	u32 (*get_wptr)(struct amdgpu_ring *ring);
+	void (*set_wptr)(struct amdgpu_ring *ring);
+	/* validating and patching of IBs */
+	int (*parse_cs)(struct amdgpu_cs_parser *p, uint32_t ib_idx);
+	/* constants to calculate how many DW are needed for an emit */
+	unsigned emit_frame_size;
+	unsigned emit_ib_size;
+	/* command emit functions */
+	void (*emit_ib)(struct amdgpu_ring *ring,
+			struct amdgpu_ib *ib,
+			unsigned vm_id, bool ctx_switch);
+	void (*emit_fence)(struct amdgpu_ring *ring, uint64_t addr,
+			   uint64_t seq, unsigned flags);
+	void (*emit_pipeline_sync)(struct amdgpu_ring *ring);
+	void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vm_id,
+			      uint64_t pd_addr);
+	void (*emit_hdp_flush)(struct amdgpu_ring *ring);
+	void (*emit_hdp_invalidate)(struct amdgpu_ring *ring);
+	void (*emit_gds_switch)(struct amdgpu_ring *ring, uint32_t vmid,
+				uint32_t gds_base, uint32_t gds_size,
+				uint32_t gws_base, uint32_t gws_size,
+				uint32_t oa_base, uint32_t oa_size);
+	/* testing functions */
+	int (*test_ring)(struct amdgpu_ring *ring);
+	int (*test_ib)(struct amdgpu_ring *ring, long timeout);
+	/* insert NOP packets */
+	void (*insert_nop)(struct amdgpu_ring *ring, uint32_t count);
+	/* pad the indirect buffer to the necessary number of dw */
+	void (*pad_ib)(struct amdgpu_ring *ring, struct amdgpu_ib *ib);
+	unsigned (*init_cond_exec)(struct amdgpu_ring *ring);
+	void (*patch_cond_exec)(struct amdgpu_ring *ring, unsigned offset);
+	/* note usage for clock and power gating */
+	void (*begin_use)(struct amdgpu_ring *ring);
+	void (*end_use)(struct amdgpu_ring *ring);
+	void (*emit_switch_buffer) (struct amdgpu_ring *ring);
+	void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags);
+};
+
+struct amdgpu_ring {
+	struct amdgpu_device		*adev;
+	const struct amdgpu_ring_funcs	*funcs;
+	struct amdgpu_fence_driver	fence_drv;
+	struct amd_gpu_scheduler	sched;
+
+	struct amdgpu_bo	*ring_obj;
+	volatile uint32_t	*ring;
+	unsigned		rptr_offs;
+	unsigned		wptr;
+	unsigned		wptr_old;
+	unsigned		ring_size;
+	unsigned		max_dw;
+	int			count_dw;
+	uint64_t		gpu_addr;
+	uint32_t		ptr_mask;
+	bool			ready;
+	u32			idx;
+	u32			me;
+	u32			pipe;
+	u32			queue;
+	struct amdgpu_bo	*mqd_obj;
+	u32			doorbell_index;
+	bool			use_doorbell;
+	unsigned		wptr_offs;
+	unsigned		fence_offs;
+	uint64_t		current_ctx;
+	char			name[16];
+	unsigned		cond_exe_offs;
+	u64			cond_exe_gpu_addr;
+	volatile u32		*cond_exe_cpu_addr;
+#if defined(CONFIG_DEBUG_FS)
+	struct dentry *ent;
+#endif
+};
+
+int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw);
+void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count);
+void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib);
+void amdgpu_ring_commit(struct amdgpu_ring *ring);
+void amdgpu_ring_undo(struct amdgpu_ring *ring);
+int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
+		     unsigned ring_size, struct amdgpu_irq_src *irq_src,
+		     unsigned irq_type);
+void amdgpu_ring_fini(struct amdgpu_ring *ring);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
index d8af37a845f4..de9f919ae336 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
@@ -147,7 +147,7 @@ static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo)
 	}
 	list_del_init(&sa_bo->olist);
 	list_del_init(&sa_bo->flist);
-	fence_put(sa_bo->fence);
+	dma_fence_put(sa_bo->fence);
 	kfree(sa_bo);
 }
 
@@ -161,7 +161,7 @@ static void amdgpu_sa_bo_try_free(struct amdgpu_sa_manager *sa_manager)
 	sa_bo = list_entry(sa_manager->hole->next, struct amdgpu_sa_bo, olist);
 	list_for_each_entry_safe_from(sa_bo, tmp, &sa_manager->olist, olist) {
 		if (sa_bo->fence == NULL ||
-		    !fence_is_signaled(sa_bo->fence)) {
+		    !dma_fence_is_signaled(sa_bo->fence)) {
 			return;
 		}
 		amdgpu_sa_bo_remove_locked(sa_bo);
@@ -244,7 +244,7 @@ static bool amdgpu_sa_event(struct amdgpu_sa_manager *sa_manager,
 }
 
 static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager,
-				   struct fence **fences,
+				   struct dma_fence **fences,
 				   unsigned *tries)
 {
 	struct amdgpu_sa_bo *best_bo = NULL;
@@ -272,7 +272,7 @@ static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager,
 		sa_bo = list_first_entry(&sa_manager->flist[i],
 					 struct amdgpu_sa_bo, flist);
 
-		if (!fence_is_signaled(sa_bo->fence)) {
+		if (!dma_fence_is_signaled(sa_bo->fence)) {
 			fences[i] = sa_bo->fence;
 			continue;
 		}
@@ -314,7 +314,7 @@ int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
 		     struct amdgpu_sa_bo **sa_bo,
 		     unsigned size, unsigned align)
 {
-	struct fence *fences[AMDGPU_SA_NUM_FENCE_LISTS];
+	struct dma_fence *fences[AMDGPU_SA_NUM_FENCE_LISTS];
 	unsigned tries[AMDGPU_SA_NUM_FENCE_LISTS];
 	unsigned count;
 	int i, r;
@@ -327,9 +327,8 @@ int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
 		return -EINVAL;
 
 	*sa_bo = kmalloc(sizeof(struct amdgpu_sa_bo), GFP_KERNEL);
-	if ((*sa_bo) == NULL) {
+	if (!(*sa_bo))
 		return -ENOMEM;
-	}
 	(*sa_bo)->manager = sa_manager;
 	(*sa_bo)->fence = NULL;
 	INIT_LIST_HEAD(&(*sa_bo)->olist);
@@ -356,14 +355,15 @@ int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
 
 		for (i = 0, count = 0; i < AMDGPU_SA_NUM_FENCE_LISTS; ++i)
 			if (fences[i])
-				fences[count++] = fence_get(fences[i]);
+				fences[count++] = dma_fence_get(fences[i]);
 
 		if (count) {
 			spin_unlock(&sa_manager->wq.lock);
-			t = fence_wait_any_timeout(fences, count, false,
-						   MAX_SCHEDULE_TIMEOUT);
+			t = dma_fence_wait_any_timeout(fences, count, false,
+						       MAX_SCHEDULE_TIMEOUT,
+						       NULL);
 			for (i = 0; i < count; ++i)
-				fence_put(fences[i]);
+				dma_fence_put(fences[i]);
 
 			r = (t > 0) ? 0 : t;
 			spin_lock(&sa_manager->wq.lock);
@@ -384,7 +384,7 @@ int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
 }
 
 void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo,
-		       struct fence *fence)
+		       struct dma_fence *fence)
 {
 	struct amdgpu_sa_manager *sa_manager;
 
@@ -394,10 +394,10 @@ void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo,
 
 	sa_manager = (*sa_bo)->manager;
 	spin_lock(&sa_manager->wq.lock);
-	if (fence && !fence_is_signaled(fence)) {
+	if (fence && !dma_fence_is_signaled(fence)) {
 		uint32_t idx;
 
-		(*sa_bo)->fence = fence_get(fence);
+		(*sa_bo)->fence = dma_fence_get(fence);
 		idx = fence->context % AMDGPU_SA_NUM_FENCE_LISTS;
 		list_add_tail(&(*sa_bo)->flist, &sa_manager->flist[idx]);
 	} else {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index 5c8d3022fb87..ed814e6d0207 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -34,7 +34,7 @@
 
 struct amdgpu_sync_entry {
 	struct hlist_node	node;
-	struct fence		*fence;
+	struct dma_fence	*fence;
 };
 
 static struct kmem_cache *amdgpu_sync_slab;
@@ -60,7 +60,8 @@ void amdgpu_sync_create(struct amdgpu_sync *sync)
  *
  * Test if the fence was issued by us.
  */
-static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, struct fence *f)
+static bool amdgpu_sync_same_dev(struct amdgpu_device *adev,
+				 struct dma_fence *f)
 {
 	struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
 
@@ -81,7 +82,7 @@ static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, struct fence *f)
  *
  * Extract who originally created the fence.
  */
-static void *amdgpu_sync_get_owner(struct fence *f)
+static void *amdgpu_sync_get_owner(struct dma_fence *f)
 {
 	struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
 
@@ -99,13 +100,14 @@ static void *amdgpu_sync_get_owner(struct fence *f)
  *
  * Either keep the existing fence or the new one, depending which one is later.
  */
-static void amdgpu_sync_keep_later(struct fence **keep, struct fence *fence)
+static void amdgpu_sync_keep_later(struct dma_fence **keep,
+				   struct dma_fence *fence)
 {
-	if (*keep && fence_is_later(*keep, fence))
+	if (*keep && dma_fence_is_later(*keep, fence))
 		return;
 
-	fence_put(*keep);
-	*keep = fence_get(fence);
+	dma_fence_put(*keep);
+	*keep = dma_fence_get(fence);
 }
 
 /**
@@ -117,7 +119,7 @@ static void amdgpu_sync_keep_later(struct fence **keep, struct fence *fence)
  * Tries to add the fence to an existing hash entry. Returns true when an entry
  * was found, false otherwise.
  */
-static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct fence *f)
+static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f)
 {
 	struct amdgpu_sync_entry *e;
 
@@ -139,7 +141,7 @@ static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct fence *f)
  *
  */
 int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
-		      struct fence *f)
+		      struct dma_fence *f)
 {
 	struct amdgpu_sync_entry *e;
 
@@ -158,7 +160,7 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
 		return -ENOMEM;
 
 	hash_add(sync->fences, &e->node, f->context);
-	e->fence = fence_get(f);
+	e->fence = dma_fence_get(f);
 	return 0;
 }
 
@@ -177,7 +179,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
 		     void *owner)
 {
 	struct reservation_object_list *flist;
-	struct fence *f;
+	struct dma_fence *f;
 	void *fence_owner;
 	unsigned i;
 	int r = 0;
@@ -231,15 +233,15 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
  * Returns the next fence not signaled yet without removing it from the sync
  * object.
  */
-struct fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
-				     struct amdgpu_ring *ring)
+struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
+					 struct amdgpu_ring *ring)
 {
 	struct amdgpu_sync_entry *e;
 	struct hlist_node *tmp;
 	int i;
 
 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
-		struct fence *f = e->fence;
+		struct dma_fence *f = e->fence;
 		struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
 
 		if (ring && s_fence) {
@@ -247,16 +249,16 @@ struct fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
 			 * when they are scheduled.
 			 */
 			if (s_fence->sched == &ring->sched) {
-				if (fence_is_signaled(&s_fence->scheduled))
+				if (dma_fence_is_signaled(&s_fence->scheduled))
 					continue;
 
 				return &s_fence->scheduled;
 			}
 		}
 
-		if (fence_is_signaled(f)) {
+		if (dma_fence_is_signaled(f)) {
 			hash_del(&e->node);
-			fence_put(f);
+			dma_fence_put(f);
 			kmem_cache_free(amdgpu_sync_slab, e);
 			continue;
 		}
@@ -274,11 +276,11 @@ struct fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
  *
  * Get and removes the next fence from the sync object not signaled yet.
  */
-struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
+struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
 {
 	struct amdgpu_sync_entry *e;
 	struct hlist_node *tmp;
-	struct fence *f;
+	struct dma_fence *f;
 	int i;
 
 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
@@ -288,10 +290,10 @@ struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
 		hash_del(&e->node);
 		kmem_cache_free(amdgpu_sync_slab, e);
 
-		if (!fence_is_signaled(f))
+		if (!dma_fence_is_signaled(f))
 			return f;
 
-		fence_put(f);
+		dma_fence_put(f);
 	}
 	return NULL;
 }
@@ -311,11 +313,11 @@ void amdgpu_sync_free(struct amdgpu_sync *sync)
 
 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
 		hash_del(&e->node);
-		fence_put(e->fence);
+		dma_fence_put(e->fence);
 		kmem_cache_free(amdgpu_sync_slab, e);
 	}
 
-	fence_put(sync->last_vm_update);
+	dma_fence_put(sync->last_vm_update);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
new file mode 100644
index 000000000000..605be266e07f
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Christian König
+ */
+#ifndef __AMDGPU_SYNC_H__
+#define __AMDGPU_SYNC_H__
+
+#include <linux/hashtable.h>
+
+struct dma_fence;
+struct reservation_object;
+struct amdgpu_device;
+struct amdgpu_ring;
+
+/*
+ * Container for fences used to sync command submissions.
+ */
+struct amdgpu_sync {
+	DECLARE_HASHTABLE(fences, 4);
+	struct dma_fence	*last_vm_update;
+};
+
+void amdgpu_sync_create(struct amdgpu_sync *sync);
+int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
+		      struct dma_fence *f);
+int amdgpu_sync_resv(struct amdgpu_device *adev,
+		     struct amdgpu_sync *sync,
+		     struct reservation_object *resv,
+		     void *owner);
+struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
+				     struct amdgpu_ring *ring);
+struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync);
+void amdgpu_sync_free(struct amdgpu_sync *sync);
+int amdgpu_sync_init(void);
+void amdgpu_sync_fini(void);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
index b827c75e95de..e05a24325eeb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
@@ -78,7 +78,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 		void *gtt_map, *vram_map;
 		void **gtt_start, **gtt_end;
 		void **vram_start, **vram_end;
-		struct fence *fence = NULL;
+		struct dma_fence *fence = NULL;
 
 		r = amdgpu_bo_create(adev, size, PAGE_SIZE, true,
 				     AMDGPU_GEM_DOMAIN_GTT, 0, NULL,
@@ -118,13 +118,13 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 			goto out_lclean_unpin;
 		}
 
-		r = fence_wait(fence, false);
+		r = dma_fence_wait(fence, false);
 		if (r) {
 			DRM_ERROR("Failed to wait for GTT->VRAM fence %d\n", i);
 			goto out_lclean_unpin;
 		}
 
-		fence_put(fence);
+		dma_fence_put(fence);
 
 		r = amdgpu_bo_kmap(vram_obj, &vram_map);
 		if (r) {
@@ -163,13 +163,13 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 			goto out_lclean_unpin;
 		}
 
-		r = fence_wait(fence, false);
+		r = dma_fence_wait(fence, false);
 		if (r) {
 			DRM_ERROR("Failed to wait for VRAM->GTT fence %d\n", i);
 			goto out_lclean_unpin;
 		}
 
-		fence_put(fence);
+		dma_fence_put(fence);
 
 		r = amdgpu_bo_kmap(gtt_obj[i], &gtt_map);
 		if (r) {
@@ -216,7 +216,7 @@ out_lclean:
 			amdgpu_bo_unref(&gtt_obj[i]);
 		}
 		if (fence)
-			fence_put(fence);
+			dma_fence_put(fence);
 		break;
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 067e5e683bb3..bb964a8ff938 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -104,7 +104,7 @@ TRACE_EVENT(amdgpu_cs_ioctl,
 			     __field(struct amdgpu_device *, adev)
 			     __field(struct amd_sched_job *, sched_job)
 			     __field(struct amdgpu_ib *, ib)
-			     __field(struct fence *, fence)
+			     __field(struct dma_fence *, fence)
 			     __field(char *, ring_name)
 			     __field(u32, num_ibs)
 			     ),
@@ -129,7 +129,7 @@ TRACE_EVENT(amdgpu_sched_run_job,
 			     __field(struct amdgpu_device *, adev)
 			     __field(struct amd_sched_job *, sched_job)
 			     __field(struct amdgpu_ib *, ib)
-			     __field(struct fence *, fence)
+			     __field(struct dma_fence *, fence)
 			     __field(char *, ring_name)
 			     __field(u32, num_ibs)
 			     ),
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index dcaf691f56b5..bc70f80260d8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -34,7 +34,6 @@
 #include <ttm/ttm_placement.h>
 #include <ttm/ttm_module.h>
 #include <ttm/ttm_page_alloc.h>
-#include <ttm/ttm_memory.h>
 #include <drm/drmP.h>
 #include <drm/amdgpu_drm.h>
 #include <linux/seq_file.h>
@@ -51,16 +50,6 @@
 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);
 
-static struct amdgpu_device *amdgpu_get_adev(struct ttm_bo_device *bdev)
-{
-	struct amdgpu_mman *mman;
-	struct amdgpu_device *adev;
-
-	mman = container_of(bdev, struct amdgpu_mman, bdev);
-	adev = container_of(mman, struct amdgpu_device, mman);
-	return adev;
-}
-
 
 /*
  * Global memory.
@@ -75,7 +64,7 @@ static void amdgpu_ttm_mem_global_release(struct drm_global_reference *ref)
 	ttm_mem_global_release(ref->object);
 }
 
-int amdgpu_ttm_global_init(struct amdgpu_device *adev)
+static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 {
 	struct drm_global_reference *global_ref;
 	struct amdgpu_ring *ring;
@@ -150,7 +139,7 @@ static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 {
 	struct amdgpu_device *adev;
 
-	adev = amdgpu_get_adev(bdev);
+	adev = amdgpu_ttm_adev(bdev);
 
 	switch (type) {
 	case TTM_PL_SYSTEM:
@@ -168,7 +157,7 @@ static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 		break;
 	case TTM_PL_VRAM:
 		/* "On-card" video ram */
-		man->func = &ttm_bo_manager_func;
+		man->func = &amdgpu_vram_mgr_func;
 		man->gpu_offset = adev->mc.vram_start;
 		man->flags = TTM_MEMTYPE_FLAG_FIXED |
 			     TTM_MEMTYPE_FLAG_MAPPABLE;
@@ -195,6 +184,7 @@ static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 				struct ttm_placement *placement)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 	struct amdgpu_bo *abo;
 	static struct ttm_place placements = {
 		.fpfn = 0,
@@ -213,7 +203,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 	abo = container_of(bo, struct amdgpu_bo, tbo);
 	switch (bo->mem.mem_type) {
 	case TTM_PL_VRAM:
-		if (abo->adev->mman.buffer_funcs_ring->ready == false) {
+		if (adev->mman.buffer_funcs_ring->ready == false) {
 			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
 		} else {
 			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
@@ -229,7 +219,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 				 * allocating address space for the BO.
 				 */
 				abo->placements[i].lpfn =
-					abo->adev->mc.gtt_size >> PAGE_SHIFT;
+					adev->mc.gtt_size >> PAGE_SHIFT;
 			}
 		}
 		break;
@@ -260,63 +250,115 @@ static void amdgpu_move_null(struct ttm_buffer_object *bo,
 	new_mem->mm_node = NULL;
 }
 
-static int amdgpu_move_blit(struct ttm_buffer_object *bo,
-			bool evict, bool no_wait_gpu,
-			struct ttm_mem_reg *new_mem,
-			struct ttm_mem_reg *old_mem)
+static int amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
+			       struct drm_mm_node *mm_node,
+			       struct ttm_mem_reg *mem,
+			       uint64_t *addr)
 {
-	struct amdgpu_device *adev;
-	struct amdgpu_ring *ring;
-	uint64_t old_start, new_start;
-	struct fence *fence;
 	int r;
 
-	adev = amdgpu_get_adev(bo->bdev);
-	ring = adev->mman.buffer_funcs_ring;
-
-	switch (old_mem->mem_type) {
+	switch (mem->mem_type) {
 	case TTM_PL_TT:
-		r = amdgpu_ttm_bind(bo, old_mem);
+		r = amdgpu_ttm_bind(bo, mem);
 		if (r)
 			return r;
 
 	case TTM_PL_VRAM:
-		old_start = (u64)old_mem->start << PAGE_SHIFT;
-		old_start += bo->bdev->man[old_mem->mem_type].gpu_offset;
+		*addr = mm_node->start << PAGE_SHIFT;
+		*addr += bo->bdev->man[mem->mem_type].gpu_offset;
 		break;
 	default:
-		DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
+		DRM_ERROR("Unknown placement %d\n", mem->mem_type);
 		return -EINVAL;
 	}
-	switch (new_mem->mem_type) {
-	case TTM_PL_TT:
-		r = amdgpu_ttm_bind(bo, new_mem);
-		if (r)
-			return r;
 
-	case TTM_PL_VRAM:
-		new_start = (u64)new_mem->start << PAGE_SHIFT;
-		new_start += bo->bdev->man[new_mem->mem_type].gpu_offset;
-		break;
-	default:
-		DRM_ERROR("Unknown placement %d\n", old_mem->mem_type);
-		return -EINVAL;
-	}
+	return 0;
+}
+
+static int amdgpu_move_blit(struct ttm_buffer_object *bo,
+			    bool evict, bool no_wait_gpu,
+			    struct ttm_mem_reg *new_mem,
+			    struct ttm_mem_reg *old_mem)
+{
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
+	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+
+	struct drm_mm_node *old_mm, *new_mm;
+	uint64_t old_start, old_size, new_start, new_size;
+	unsigned long num_pages;
+	struct dma_fence *fence = NULL;
+	int r;
+
+	BUILD_BUG_ON((PAGE_SIZE % AMDGPU_GPU_PAGE_SIZE) != 0);
+
 	if (!ring->ready) {
 		DRM_ERROR("Trying to move memory with ring turned off.\n");
 		return -EINVAL;
 	}
 
-	BUILD_BUG_ON((PAGE_SIZE % AMDGPU_GPU_PAGE_SIZE) != 0);
+	old_mm = old_mem->mm_node;
+	r = amdgpu_mm_node_addr(bo, old_mm, old_mem, &old_start);
+	if (r)
+		return r;
+	old_size = old_mm->size;
+
 
-	r = amdgpu_copy_buffer(ring, old_start, new_start,
-			       new_mem->num_pages * PAGE_SIZE, /* bytes */
-			       bo->resv, &fence, false);
+	new_mm = new_mem->mm_node;
+	r = amdgpu_mm_node_addr(bo, new_mm, new_mem, &new_start);
 	if (r)
 		return r;
+	new_size = new_mm->size;
+
+	num_pages = new_mem->num_pages;
+	while (num_pages) {
+		unsigned long cur_pages = min(old_size, new_size);
+		struct dma_fence *next;
+
+		r = amdgpu_copy_buffer(ring, old_start, new_start,
+				       cur_pages * PAGE_SIZE,
+				       bo->resv, &next, false);
+		if (r)
+			goto error;
+
+		dma_fence_put(fence);
+		fence = next;
+
+		num_pages -= cur_pages;
+		if (!num_pages)
+			break;
+
+		old_size -= cur_pages;
+		if (!old_size) {
+			r = amdgpu_mm_node_addr(bo, ++old_mm, old_mem,
+						&old_start);
+			if (r)
+				goto error;
+			old_size = old_mm->size;
+		} else {
+			old_start += cur_pages * PAGE_SIZE;
+		}
+
+		new_size -= cur_pages;
+		if (!new_size) {
+			r = amdgpu_mm_node_addr(bo, ++new_mm, new_mem,
+						&new_start);
+			if (r)
+				goto error;
+
+			new_size = new_mm->size;
+		} else {
+			new_start += cur_pages * PAGE_SIZE;
+		}
+	}
 
 	r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
-	fence_put(fence);
+	dma_fence_put(fence);
+	return r;
+
+error:
+	if (fence)
+		dma_fence_wait(fence, false);
+	dma_fence_put(fence);
 	return r;
 }
 
@@ -332,7 +374,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
 	struct ttm_placement placement;
 	int r;
 
-	adev = amdgpu_get_adev(bo->bdev);
+	adev = amdgpu_ttm_adev(bo->bdev);
 	tmp_mem = *new_mem;
 	tmp_mem.mm_node = NULL;
 	placement.num_placement = 1;
@@ -379,7 +421,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
 	struct ttm_place placements;
 	int r;
 
-	adev = amdgpu_get_adev(bo->bdev);
+	adev = amdgpu_ttm_adev(bo->bdev);
 	tmp_mem = *new_mem;
 	tmp_mem.mm_node = NULL;
 	placement.num_placement = 1;
@@ -422,7 +464,7 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo,
 	if (WARN_ON_ONCE(abo->pin_count > 0))
 		return -EINVAL;
 
-	adev = amdgpu_get_adev(bo->bdev);
+	adev = amdgpu_ttm_adev(bo->bdev);
 
 	/* remember the eviction */
 	if (evict)
@@ -475,7 +517,7 @@ memcpy:
 static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 {
 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
-	struct amdgpu_device *adev = amdgpu_get_adev(bdev);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
 
 	mem->bus.addr = NULL;
 	mem->bus.offset = 0;
@@ -607,7 +649,7 @@ release_pages:
 /* prepare the sg table with the user pages */
 static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
 {
-	struct amdgpu_device *adev = amdgpu_get_adev(ttm->bdev);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
 	unsigned nents;
 	int r;
@@ -639,7 +681,7 @@ release_sg:
 
 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
 {
-	struct amdgpu_device *adev = amdgpu_get_adev(ttm->bdev);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
 	struct sg_page_iter sg_iter;
 
@@ -799,7 +841,7 @@ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_bo_device *bdev,
 	struct amdgpu_device *adev;
 	struct amdgpu_ttm_tt *gtt;
 
-	adev = amdgpu_get_adev(bdev);
+	adev = amdgpu_ttm_adev(bdev);
 
 	gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);
 	if (gtt == NULL) {
@@ -843,7 +885,7 @@ static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm)
 		return 0;
 	}
 
-	adev = amdgpu_get_adev(ttm->bdev);
+	adev = amdgpu_ttm_adev(ttm->bdev);
 
 #ifdef CONFIG_SWIOTLB
 	if (swiotlb_nr_tbl()) {
@@ -889,7 +931,7 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
 	if (slave)
 		return;
 
-	adev = amdgpu_get_adev(ttm->bdev);
+	adev = amdgpu_ttm_adev(ttm->bdev);
 
 #ifdef CONFIG_SWIOTLB
 	if (swiotlb_nr_tbl()) {
@@ -1012,7 +1054,7 @@ uint32_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
 
 static void amdgpu_ttm_lru_removal(struct ttm_buffer_object *tbo)
 {
-	struct amdgpu_device *adev = amdgpu_get_adev(tbo->bdev);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
 	unsigned i, j;
 
 	for (i = 0; i < AMDGPU_TTM_LRU_SIZE; ++i) {
@@ -1029,7 +1071,7 @@ static void amdgpu_ttm_lru_removal(struct ttm_buffer_object *tbo)
 
 static struct amdgpu_mman_lru *amdgpu_ttm_lru(struct ttm_buffer_object *tbo)
 {
-	struct amdgpu_device *adev = amdgpu_get_adev(tbo->bdev);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
 	unsigned log2_size = min(ilog2(tbo->num_pages),
 				 AMDGPU_TTM_LRU_SIZE - 1);
 
@@ -1060,12 +1102,37 @@ static struct list_head *amdgpu_ttm_swap_lru_tail(struct ttm_buffer_object *tbo)
 	return res;
 }
 
+static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
+					    const struct ttm_place *place)
+{
+	if (bo->mem.mem_type == TTM_PL_VRAM &&
+	    bo->mem.start == AMDGPU_BO_INVALID_OFFSET) {
+		unsigned long num_pages = bo->mem.num_pages;
+		struct drm_mm_node *node = bo->mem.mm_node;
+
+		/* Check each drm MM node individually */
+		while (num_pages) {
+			if (place->fpfn < (node->start + node->size) &&
+			    !(place->lpfn && place->lpfn <= node->start))
+				return true;
+
+			num_pages -= node->size;
+			++node;
+		}
+
+		return false;
+	}
+
+	return ttm_bo_eviction_valuable(bo, place);
+}
+
 static struct ttm_bo_driver amdgpu_bo_driver = {
 	.ttm_tt_create = &amdgpu_ttm_tt_create,
 	.ttm_tt_populate = &amdgpu_ttm_tt_populate,
 	.ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
 	.invalidate_caches = &amdgpu_invalidate_caches,
 	.init_mem_type = &amdgpu_init_mem_type,
+	.eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
 	.evict_flags = &amdgpu_evict_flags,
 	.move = &amdgpu_bo_move,
 	.verify_access = &amdgpu_verify_access,
@@ -1083,6 +1150,10 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	unsigned i, j;
 	int r;
 
+	r = amdgpu_ttm_global_init(adev);
+	if (r) {
+		return r;
+	}
 	/* No others user of address space so set it to 0 */
 	r = ttm_bo_device_init(&adev->mman.bdev,
 			       adev->mman.bo_global_ref.ref.object,
@@ -1119,7 +1190,8 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 
 	r = amdgpu_bo_create(adev, 256 * 1024, PAGE_SIZE, true,
 			     AMDGPU_GEM_DOMAIN_VRAM,
-			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 			     NULL, NULL, &adev->stollen_vga_memory);
 	if (r) {
 		return r;
@@ -1247,7 +1319,7 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring,
 		       uint64_t dst_offset,
 		       uint32_t byte_count,
 		       struct reservation_object *resv,
-		       struct fence **fence, bool direct_submit)
+		       struct dma_fence **fence, bool direct_submit)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_job *job;
@@ -1294,7 +1366,7 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring,
 	if (direct_submit) {
 		r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs,
 				       NULL, NULL, fence);
-		job->fence = fence_get(*fence);
+		job->fence = dma_fence_get(*fence);
 		if (r)
 			DRM_ERROR("Error scheduling IBs (%d)\n", r);
 		amdgpu_job_free(job);
@@ -1313,28 +1385,40 @@ error_free:
 }
 
 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
-		uint32_t src_data,
-		struct reservation_object *resv,
-		struct fence **fence)
+		       uint32_t src_data,
+		       struct reservation_object *resv,
+		       struct dma_fence **fence)
 {
-	struct amdgpu_device *adev = bo->adev;
-	struct amdgpu_job *job;
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 
-	uint32_t max_bytes, byte_count;
-	uint64_t dst_offset;
+	struct drm_mm_node *mm_node;
+	unsigned long num_pages;
 	unsigned int num_loops, num_dw;
-	unsigned int i;
+
+	struct amdgpu_job *job;
 	int r;
 
-	byte_count = bo->tbo.num_pages << PAGE_SHIFT;
-	max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
-	num_loops = DIV_ROUND_UP(byte_count, max_bytes);
+	if (!ring->ready) {
+		DRM_ERROR("Trying to clear memory with ring turned off.\n");
+		return -EINVAL;
+	}
+
+	num_pages = bo->tbo.num_pages;
+	mm_node = bo->tbo.mem.mm_node;
+	num_loops = 0;
+	while (num_pages) {
+		uint32_t byte_count = mm_node->size << PAGE_SHIFT;
+
+		num_loops += DIV_ROUND_UP(byte_count, max_bytes);
+		num_pages -= mm_node->size;
+		++mm_node;
+	}
 	num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;
 
 	/* for IB padding */
-	while (num_dw & 0x7)
-		num_dw++;
+	num_dw += 64;
 
 	r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
 	if (r)
@@ -1342,28 +1426,43 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 
 	if (resv) {
 		r = amdgpu_sync_resv(adev, &job->sync, resv,
-				AMDGPU_FENCE_OWNER_UNDEFINED);
+				     AMDGPU_FENCE_OWNER_UNDEFINED);
 		if (r) {
 			DRM_ERROR("sync failed (%d).\n", r);
 			goto error_free;
 		}
 	}
 
-	dst_offset = bo->tbo.mem.start << PAGE_SHIFT;
-	for (i = 0; i < num_loops; i++) {
-		uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
+	num_pages = bo->tbo.num_pages;
+	mm_node = bo->tbo.mem.mm_node;
 
-		amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
-				dst_offset, cur_size_in_bytes);
+	while (num_pages) {
+		uint32_t byte_count = mm_node->size << PAGE_SHIFT;
+		uint64_t dst_addr;
 
-		dst_offset += cur_size_in_bytes;
-		byte_count -= cur_size_in_bytes;
+		r = amdgpu_mm_node_addr(&bo->tbo, mm_node,
+					&bo->tbo.mem, &dst_addr);
+		if (r)
+			return r;
+
+		while (byte_count) {
+			uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
+
+			amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
+						dst_addr, cur_size_in_bytes);
+
+			dst_addr += cur_size_in_bytes;
+			byte_count -= cur_size_in_bytes;
+		}
+
+		num_pages -= mm_node->size;
+		++mm_node;
 	}
 
 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 	WARN_ON(job->ibs[0].length_dw > num_dw);
 	r = amdgpu_job_submit(job, ring, &adev->mman.entity,
-			AMDGPU_FENCE_OWNER_UNDEFINED, fence);
+			      AMDGPU_FENCE_OWNER_UNDEFINED, fence);
 	if (r)
 		goto error_free;
 
@@ -1554,8 +1653,3 @@ static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
 
 #endif
 }
-
-u64 amdgpu_ttm_get_gtt_mem_size(struct amdgpu_device *adev)
-{
-	return ttm_get_kernel_zone_memory_size(adev->mman.mem_global_ref.object);
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 9812c805326c..98ee384f0fca 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -66,6 +66,7 @@ struct amdgpu_mman {
 };
 
 extern const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func;
+extern const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func;
 
 int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
 			 struct ttm_buffer_object *tbo,
@@ -77,11 +78,11 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring,
 		       uint64_t dst_offset,
 		       uint32_t byte_count,
 		       struct reservation_object *resv,
-		       struct fence **fence, bool direct_submit);
+		       struct dma_fence **fence, bool direct_submit);
 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 			uint32_t src_data,
 			struct reservation_object *resv,
-			struct fence **fence);
+			struct dma_fence **fence);
 
 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
 bool amdgpu_ttm_is_bound(struct ttm_tt *ttm);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index cb3d252f3c78..0f0b38191fac 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -228,6 +228,9 @@ static int amdgpu_ucode_init_single_fw(struct amdgpu_firmware_info *ucode,
 	ucode->mc_addr = mc_addr;
 	ucode->kaddr = kptr;
 
+	if (ucode->ucode_id == AMDGPU_UCODE_ID_STORAGE)
+		return 0;
+
 	header = (const struct common_firmware_header *)ucode->fw->data;
 	memcpy(ucode->kaddr, (void *)((uint8_t *)ucode->fw->data +
 		le32_to_cpu(header->ucode_array_offset_bytes)),
@@ -236,6 +239,31 @@ static int amdgpu_ucode_init_single_fw(struct amdgpu_firmware_info *ucode,
 	return 0;
 }
 
+static int amdgpu_ucode_patch_jt(struct amdgpu_firmware_info *ucode,
+				uint64_t mc_addr, void *kptr)
+{
+	const struct gfx_firmware_header_v1_0 *header = NULL;
+	const struct common_firmware_header *comm_hdr = NULL;
+	uint8_t* src_addr = NULL;
+	uint8_t* dst_addr = NULL;
+
+	if (NULL == ucode->fw)
+		return 0;
+
+	comm_hdr = (const struct common_firmware_header *)ucode->fw->data;
+	header = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data;
+	dst_addr = ucode->kaddr +
+			   ALIGN(le32_to_cpu(comm_hdr->ucode_size_bytes),
+			   PAGE_SIZE);
+	src_addr = (uint8_t *)ucode->fw->data +
+			   le32_to_cpu(comm_hdr->ucode_array_offset_bytes) +
+			   (le32_to_cpu(header->jt_offset) * 4);
+	memcpy(dst_addr, src_addr, le32_to_cpu(header->jt_size) * 4);
+
+	return 0;
+}
+
+
 int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 {
 	struct amdgpu_bo **bo = &adev->firmware.fw_buf;
@@ -247,7 +275,8 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 	const struct common_firmware_header *header = NULL;
 
 	err = amdgpu_bo_create(adev, adev->firmware.fw_size, PAGE_SIZE, true,
-			       AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL, bo);
+				amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
+				0, NULL, NULL, bo);
 	if (err) {
 		dev_err(adev->dev, "(%d) Firmware buffer allocate failed\n", err);
 		goto failed;
@@ -259,7 +288,8 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 		goto failed_reserve;
 	}
 
-	err = amdgpu_bo_pin(*bo, AMDGPU_GEM_DOMAIN_GTT, &fw_mc_addr);
+	err = amdgpu_bo_pin(*bo, amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
+				&fw_mc_addr);
 	if (err) {
 		dev_err(adev->dev, "(%d) Firmware buffer pin failed\n", err);
 		goto failed_pin;
@@ -279,6 +309,13 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 			header = (const struct common_firmware_header *)ucode->fw->data;
 			amdgpu_ucode_init_single_fw(ucode, fw_mc_addr + fw_offset,
 						    fw_buf_ptr + fw_offset);
+			if (i == AMDGPU_UCODE_ID_CP_MEC1) {
+				const struct gfx_firmware_header_v1_0 *cp_hdr;
+				cp_hdr = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data;
+				amdgpu_ucode_patch_jt(ucode, fw_mc_addr + fw_offset,
+						    fw_buf_ptr + fw_offset);
+				fw_offset += ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE);
+			}
 			fw_offset += ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 		}
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
index e468be4e28fa..a8a4230729f9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
@@ -130,6 +130,7 @@ enum AMDGPU_UCODE_ID {
 	AMDGPU_UCODE_ID_CP_MEC1,
 	AMDGPU_UCODE_ID_CP_MEC2,
 	AMDGPU_UCODE_ID_RLC_G,
+	AMDGPU_UCODE_ID_STORAGE,
 	AMDGPU_UCODE_ID_MAXIMUM,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index e3281cacc586..a81dfaeeb8c0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -333,7 +333,7 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
 	for (i = 0; i < adev->uvd.max_handles; ++i) {
 		uint32_t handle = atomic_read(&adev->uvd.handles[i]);
 		if (handle != 0 && adev->uvd.filp[i] == filp) {
-			struct fence *fence;
+			struct dma_fence *fence;
 
 			r = amdgpu_uvd_get_destroy_msg(ring, handle,
 						       false, &fence);
@@ -342,8 +342,8 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
 				continue;
 			}
 
-			fence_wait(fence, false);
-			fence_put(fence);
+			dma_fence_wait(fence, false);
+			dma_fence_put(fence);
 
 			adev->uvd.filp[i] = NULL;
 			atomic_set(&adev->uvd.handles[i], 0);
@@ -360,6 +360,18 @@ static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *abo)
 	}
 }
 
+static u64 amdgpu_uvd_get_addr_from_ctx(struct amdgpu_uvd_cs_ctx *ctx)
+{
+	uint32_t lo, hi;
+	uint64_t addr;
+
+	lo = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data0);
+	hi = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data1);
+	addr = ((uint64_t)lo) | (((uint64_t)hi) << 32);
+
+	return addr;
+}
+
 /**
  * amdgpu_uvd_cs_pass1 - first parsing round
  *
@@ -372,14 +384,10 @@ static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx)
 {
 	struct amdgpu_bo_va_mapping *mapping;
 	struct amdgpu_bo *bo;
-	uint32_t cmd, lo, hi;
-	uint64_t addr;
+	uint32_t cmd;
+	uint64_t addr = amdgpu_uvd_get_addr_from_ctx(ctx);
 	int r = 0;
 
-	lo = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data0);
-	hi = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data1);
-	addr = ((uint64_t)lo) | (((uint64_t)hi) << 32);
-
 	mapping = amdgpu_cs_find_mapping(ctx->parser, addr, &bo);
 	if (mapping == NULL) {
 		DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
@@ -698,18 +706,16 @@ static int amdgpu_uvd_cs_pass2(struct amdgpu_uvd_cs_ctx *ctx)
 {
 	struct amdgpu_bo_va_mapping *mapping;
 	struct amdgpu_bo *bo;
-	uint32_t cmd, lo, hi;
+	uint32_t cmd;
 	uint64_t start, end;
-	uint64_t addr;
+	uint64_t addr = amdgpu_uvd_get_addr_from_ctx(ctx);
 	int r;
 
-	lo = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data0);
-	hi = amdgpu_get_ib_value(ctx->parser, ctx->ib_idx, ctx->data1);
-	addr = ((uint64_t)lo) | (((uint64_t)hi) << 32);
-
 	mapping = amdgpu_cs_find_mapping(ctx->parser, addr, &bo);
-	if (mapping == NULL)
+	if (mapping == NULL) {
+		DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
 		return -EINVAL;
+	}
 
 	start = amdgpu_bo_gpu_offset(bo);
 
@@ -876,6 +882,9 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx)
 	struct amdgpu_ib *ib = &parser->job->ibs[ib_idx];
 	int r;
 
+	parser->job->vm = NULL;
+	ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);
+
 	if (ib->length_dw % 16) {
 		DRM_ERROR("UVD IB length (%d) not 16 dwords aligned!\n",
 			  ib->length_dw);
@@ -890,10 +899,13 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx)
 	ctx.buf_sizes = buf_sizes;
 	ctx.ib_idx = ib_idx;
 
-	/* first round, make sure the buffers are actually in the UVD segment */
-	r = amdgpu_uvd_cs_packets(&ctx, amdgpu_uvd_cs_pass1);
-	if (r)
-		return r;
+	/* first round only required on chips without UVD 64 bit address support */
+	if (!parser->adev->uvd.address_64_bit) {
+		/* first round, make sure the buffers are actually in the UVD segment */
+		r = amdgpu_uvd_cs_packets(&ctx, amdgpu_uvd_cs_pass1);
+		if (r)
+			return r;
+	}
 
 	/* second round, patch buffer addresses into the command stream */
 	r = amdgpu_uvd_cs_packets(&ctx, amdgpu_uvd_cs_pass2);
@@ -909,14 +921,14 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx)
 }
 
 static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
-			       bool direct, struct fence **fence)
+			       bool direct, struct dma_fence **fence)
 {
 	struct ttm_validate_buffer tv;
 	struct ww_acquire_ctx ticket;
 	struct list_head head;
 	struct amdgpu_job *job;
 	struct amdgpu_ib *ib;
-	struct fence *f = NULL;
+	struct dma_fence *f = NULL;
 	struct amdgpu_device *adev = ring->adev;
 	uint64_t addr;
 	int i, r;
@@ -931,7 +943,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 	if (r)
 		return r;
 
-	if (!bo->adev->uvd.address_64_bit) {
+	if (!ring->adev->uvd.address_64_bit) {
 		amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);
 		amdgpu_uvd_force_into_uvd_segment(bo);
 	}
@@ -960,7 +972,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 
 	if (direct) {
 		r = amdgpu_ib_schedule(ring, 1, ib, NULL, NULL, &f);
-		job->fence = fence_get(f);
+		job->fence = dma_fence_get(f);
 		if (r)
 			goto err_free;
 
@@ -975,9 +987,9 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 	ttm_eu_fence_buffer_objects(&ticket, &head, f);
 
 	if (fence)
-		*fence = fence_get(f);
+		*fence = dma_fence_get(f);
 	amdgpu_bo_unref(&bo);
-	fence_put(f);
+	dma_fence_put(f);
 
 	return 0;
 
@@ -993,7 +1005,7 @@ err:
    crash the vcpu so just try to emmit a dummy create/destroy msg to
    avoid this */
 int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
-			      struct fence **fence)
+			      struct dma_fence **fence)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_bo *bo;
@@ -1002,7 +1014,8 @@ int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
 
 	r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true,
 			     AMDGPU_GEM_DOMAIN_VRAM,
-			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 			     NULL, NULL, &bo);
 	if (r)
 		return r;
@@ -1042,7 +1055,7 @@ int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
 }
 
 int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
-			       bool direct, struct fence **fence)
+			       bool direct, struct dma_fence **fence)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_bo *bo;
@@ -1051,7 +1064,8 @@ int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 
 	r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true,
 			     AMDGPU_GEM_DOMAIN_VRAM,
-			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 			     NULL, NULL, &bo);
 	if (r)
 		return r;
@@ -1128,7 +1142,7 @@ void amdgpu_uvd_ring_end_use(struct amdgpu_ring *ring)
  */
 int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
-	struct fence *fence;
+	struct dma_fence *fence;
 	long r;
 
 	r = amdgpu_uvd_get_create_msg(ring, 1, NULL);
@@ -1143,7 +1157,7 @@ int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		goto error;
 	}
 
-	r = fence_wait_timeout(fence, false, timeout);
+	r = dma_fence_wait_timeout(fence, false, timeout);
 	if (r == 0) {
 		DRM_ERROR("amdgpu: IB test timed out.\n");
 		r = -ETIMEDOUT;
@@ -1154,7 +1168,7 @@ int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = 0;
 	}
 
-	fence_put(fence);
+	dma_fence_put(fence);
 
 error:
 	return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
index c850009602d1..6249ba1bde2a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
@@ -29,9 +29,9 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev);
 int amdgpu_uvd_suspend(struct amdgpu_device *adev);
 int amdgpu_uvd_resume(struct amdgpu_device *adev);
 int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
-			      struct fence **fence);
+			      struct dma_fence **fence);
 int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
-			       bool direct, struct fence **fence);
+			       bool direct, struct dma_fence **fence);
 void amdgpu_uvd_free_handles(struct amdgpu_device *adev,
 			     struct drm_file *filp);
 int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 7fe8fd884f06..69b66b9e7f57 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -157,7 +157,8 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
 
 	r = amdgpu_bo_create(adev, size, PAGE_SIZE, true,
 			     AMDGPU_GEM_DOMAIN_VRAM,
-			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 			     NULL, NULL, &adev->vce.vcpu_bo);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate VCE bo\n", r);
@@ -395,12 +396,12 @@ void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
  * Open up a stream for HW test
  */
 int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
-			      struct fence **fence)
+			      struct dma_fence **fence)
 {
 	const unsigned ib_size_dw = 1024;
 	struct amdgpu_job *job;
 	struct amdgpu_ib *ib;
-	struct fence *f = NULL;
+	struct dma_fence *f = NULL;
 	uint64_t dummy;
 	int i, r;
 
@@ -450,14 +451,14 @@ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
 		ib->ptr[i] = 0x0;
 
 	r = amdgpu_ib_schedule(ring, 1, ib, NULL, NULL, &f);
-	job->fence = fence_get(f);
+	job->fence = dma_fence_get(f);
 	if (r)
 		goto err;
 
 	amdgpu_job_free(job);
 	if (fence)
-		*fence = fence_get(f);
-	fence_put(f);
+		*fence = dma_fence_get(f);
+	dma_fence_put(f);
 	return 0;
 
 err:
@@ -476,12 +477,12 @@ err:
  * Close up a stream for HW test or if userspace failed to do so
  */
 int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
-			       bool direct, struct fence **fence)
+			       bool direct, struct dma_fence **fence)
 {
 	const unsigned ib_size_dw = 1024;
 	struct amdgpu_job *job;
 	struct amdgpu_ib *ib;
-	struct fence *f = NULL;
+	struct dma_fence *f = NULL;
 	int i, r;
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
@@ -513,7 +514,7 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 
 	if (direct) {
 		r = amdgpu_ib_schedule(ring, 1, ib, NULL, NULL, &f);
-		job->fence = fence_get(f);
+		job->fence = dma_fence_get(f);
 		if (r)
 			goto err;
 
@@ -526,8 +527,8 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 	}
 
 	if (fence)
-		*fence = fence_get(f);
-	fence_put(f);
+		*fence = dma_fence_get(f);
+	dma_fence_put(f);
 	return 0;
 
 err:
@@ -641,6 +642,9 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
 	uint32_t *size = &tmp;
 	int i, r, idx = 0;
 
+	p->job->vm = NULL;
+	ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);
+
 	r = amdgpu_cs_sysvm_access_required(p);
 	if (r)
 		return r;
@@ -788,6 +792,96 @@ out:
 }
 
 /**
+ * amdgpu_vce_cs_parse_vm - parse the command stream in VM mode
+ *
+ * @p: parser context
+ *
+ */
+int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser *p, uint32_t ib_idx)
+{
+	struct amdgpu_ib *ib = &p->job->ibs[ib_idx];
+	int session_idx = -1;
+	uint32_t destroyed = 0;
+	uint32_t created = 0;
+	uint32_t allocated = 0;
+	uint32_t tmp, handle = 0;
+	int i, r = 0, idx = 0;
+
+	while (idx < ib->length_dw) {
+		uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
+		uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
+
+		if ((len < 8) || (len & 3)) {
+			DRM_ERROR("invalid VCE command length (%d)!\n", len);
+			r = -EINVAL;
+			goto out;
+		}
+
+		switch (cmd) {
+		case 0x00000001: /* session */
+			handle = amdgpu_get_ib_value(p, ib_idx, idx + 2);
+			session_idx = amdgpu_vce_validate_handle(p, handle,
+								 &allocated);
+			if (session_idx < 0) {
+				r = session_idx;
+				goto out;
+			}
+			break;
+
+		case 0x01000001: /* create */
+			created |= 1 << session_idx;
+			if (destroyed & (1 << session_idx)) {
+				destroyed &= ~(1 << session_idx);
+				allocated |= 1 << session_idx;
+
+			} else if (!(allocated & (1 << session_idx))) {
+				DRM_ERROR("Handle already in use!\n");
+				r = -EINVAL;
+				goto out;
+			}
+
+			break;
+
+		case 0x02000001: /* destroy */
+			destroyed |= 1 << session_idx;
+			break;
+
+		default:
+			break;
+		}
+
+		if (session_idx == -1) {
+			DRM_ERROR("no session command at start of IB\n");
+			r = -EINVAL;
+			goto out;
+		}
+
+		idx += len / 4;
+	}
+
+	if (allocated & ~created) {
+		DRM_ERROR("New session without create command!\n");
+		r = -ENOENT;
+	}
+
+out:
+	if (!r) {
+		/* No error, free all destroyed handle slots */
+		tmp = destroyed;
+		amdgpu_ib_free(p->adev, ib, NULL);
+	} else {
+		/* Error during parsing, free all allocated handle slots */
+		tmp = allocated;
+	}
+
+	for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
+		if (tmp & (1 << i))
+			atomic_set(&p->adev->vce.handles[i], 0);
+
+	return r;
+}
+
+/**
  * amdgpu_vce_ring_emit_ib - execute indirect buffer
  *
  * @ring: engine to use
@@ -823,18 +917,6 @@ void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 	amdgpu_ring_write(ring, VCE_CMD_END);
 }
 
-unsigned amdgpu_vce_ring_get_emit_ib_size(struct amdgpu_ring *ring)
-{
-	return
-		4; /* amdgpu_vce_ring_emit_ib */
-}
-
-unsigned amdgpu_vce_ring_get_dma_frame_size(struct amdgpu_ring *ring)
-{
-	return
-		6; /* amdgpu_vce_ring_emit_fence  x1 no user fence */
-}
-
 /**
  * amdgpu_vce_ring_test_ring - test if VCE ring is working
  *
@@ -883,7 +965,7 @@ int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
  */
 int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
-	struct fence *fence = NULL;
+	struct dma_fence *fence = NULL;
 	long r;
 
 	/* skip vce ring1/2 ib test for now, since it's not reliable */
@@ -902,7 +984,7 @@ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		goto error;
 	}
 
-	r = fence_wait_timeout(fence, false, timeout);
+	r = dma_fence_wait_timeout(fence, false, timeout);
 	if (r == 0) {
 		DRM_ERROR("amdgpu: IB test timed out.\n");
 		r = -ETIMEDOUT;
@@ -913,6 +995,6 @@ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = 0;
 	}
 error:
-	fence_put(fence);
+	dma_fence_put(fence);
 	return r;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
index 12729d2852df..d98041f7508d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
@@ -29,11 +29,12 @@ int amdgpu_vce_sw_fini(struct amdgpu_device *adev);
 int amdgpu_vce_suspend(struct amdgpu_device *adev);
 int amdgpu_vce_resume(struct amdgpu_device *adev);
 int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
-			      struct fence **fence);
+			      struct dma_fence **fence);
 int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
-			       bool direct, struct fence **fence);
+			       bool direct, struct dma_fence **fence);
 void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp);
 int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx);
+int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser *p, uint32_t ib_idx);
 void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib,
 			     unsigned vm_id, bool ctx_switch);
 void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 968c4260d7a7..1dda9321bd5a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -25,7 +25,7 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
-#include <linux/fence-array.h>
+#include <linux/dma-fence-array.h>
 #include <drm/drmP.h>
 #include <drm/amdgpu_drm.h>
 #include "amdgpu.h"
@@ -116,38 +116,43 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 }
 
 /**
- * amdgpu_vm_get_bos - add the vm BOs to a duplicates list
+ * amdgpu_vm_validate_pt_bos - validate the page table BOs
  *
  * @adev: amdgpu device pointer
  * @vm: vm providing the BOs
- * @duplicates: head of duplicates list
+ * @validate: callback to do the validation
+ * @param: parameter for the validation callback
  *
- * Add the page directory to the BO duplicates list
- * for command submission.
+ * Validate the page table BOs on command submission if neccessary.
  */
-void amdgpu_vm_get_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			  struct list_head *duplicates)
+int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			      int (*validate)(void *p, struct amdgpu_bo *bo),
+			      void *param)
 {
 	uint64_t num_evictions;
 	unsigned i;
+	int r;
 
 	/* We only need to validate the page tables
 	 * if they aren't already valid.
 	 */
 	num_evictions = atomic64_read(&adev->num_evictions);
 	if (num_evictions == vm->last_eviction_counter)
-		return;
+		return 0;
 
 	/* add the vm page table to the list */
 	for (i = 0; i <= vm->max_pde_used; ++i) {
-		struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;
+		struct amdgpu_bo *bo = vm->page_tables[i].bo;
 
-		if (!entry->robj)
+		if (!bo)
 			continue;
 
-		list_add(&entry->tv.head, duplicates);
+		r = validate(param, bo);
+		if (r)
+			return r;
 	}
 
+	return 0;
 }
 
 /**
@@ -166,12 +171,12 @@ void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
 
 	spin_lock(&glob->lru_lock);
 	for (i = 0; i <= vm->max_pde_used; ++i) {
-		struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;
+		struct amdgpu_bo *bo = vm->page_tables[i].bo;
 
-		if (!entry->robj)
+		if (!bo)
 			continue;
 
-		ttm_bo_move_to_lru_tail(&entry->robj->tbo);
+		ttm_bo_move_to_lru_tail(&bo->tbo);
 	}
 	spin_unlock(&glob->lru_lock);
 }
@@ -194,14 +199,14 @@ static bool amdgpu_vm_is_gpu_reset(struct amdgpu_device *adev,
  * Allocate an id for the vm, adding fences to the sync obj as necessary.
  */
 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
-		      struct amdgpu_sync *sync, struct fence *fence,
+		      struct amdgpu_sync *sync, struct dma_fence *fence,
 		      struct amdgpu_job *job)
 {
 	struct amdgpu_device *adev = ring->adev;
 	uint64_t fence_context = adev->fence_context + ring->idx;
-	struct fence *updates = sync->last_vm_update;
+	struct dma_fence *updates = sync->last_vm_update;
 	struct amdgpu_vm_id *id, *idle;
-	struct fence **fences;
+	struct dma_fence **fences;
 	unsigned i;
 	int r = 0;
 
@@ -225,17 +230,17 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 	if (&idle->list == &adev->vm_manager.ids_lru) {
 		u64 fence_context = adev->vm_manager.fence_context + ring->idx;
 		unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
-		struct fence_array *array;
+		struct dma_fence_array *array;
 		unsigned j;
 
 		for (j = 0; j < i; ++j)
-			fence_get(fences[j]);
+			dma_fence_get(fences[j]);
 
-		array = fence_array_create(i, fences, fence_context,
+		array = dma_fence_array_create(i, fences, fence_context,
 					   seqno, true);
 		if (!array) {
 			for (j = 0; j < i; ++j)
-				fence_put(fences[j]);
+				dma_fence_put(fences[j]);
 			kfree(fences);
 			r = -ENOMEM;
 			goto error;
@@ -243,7 +248,7 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 
 
 		r = amdgpu_sync_fence(ring->adev, sync, &array->base);
-		fence_put(&array->base);
+		dma_fence_put(&array->base);
 		if (r)
 			goto error;
 
@@ -257,7 +262,7 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 	/* Check if we can use a VMID already assigned to this VM */
 	i = ring->idx;
 	do {
-		struct fence *flushed;
+		struct dma_fence *flushed;
 
 		id = vm->ids[i++];
 		if (i == AMDGPU_MAX_RINGS)
@@ -279,12 +284,12 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 			continue;
 
 		if (id->last_flush->context != fence_context &&
-		    !fence_is_signaled(id->last_flush))
+		    !dma_fence_is_signaled(id->last_flush))
 			continue;
 
 		flushed  = id->flushed_updates;
 		if (updates &&
-		    (!flushed || fence_is_later(updates, flushed)))
+		    (!flushed || dma_fence_is_later(updates, flushed)))
 			continue;
 
 		/* Good we can use this VMID. Remember this submission as
@@ -315,14 +320,14 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 	if (r)
 		goto error;
 
-	fence_put(id->first);
-	id->first = fence_get(fence);
+	dma_fence_put(id->first);
+	id->first = dma_fence_get(fence);
 
-	fence_put(id->last_flush);
+	dma_fence_put(id->last_flush);
 	id->last_flush = NULL;
 
-	fence_put(id->flushed_updates);
-	id->flushed_updates = fence_get(updates);
+	dma_fence_put(id->flushed_updates);
+	id->flushed_updates = dma_fence_get(updates);
 
 	id->pd_gpu_addr = job->vm_pd_addr;
 	id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
@@ -341,9 +346,9 @@ error:
 static bool amdgpu_vm_ring_has_compute_vm_bug(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
-	const struct amdgpu_ip_block_version *ip_block;
+	const struct amdgpu_ip_block *ip_block;
 
-	if (ring->type != AMDGPU_RING_TYPE_COMPUTE)
+	if (ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
 		/* only compute rings */
 		return false;
 
@@ -351,10 +356,10 @@ static bool amdgpu_vm_ring_has_compute_vm_bug(struct amdgpu_ring *ring)
 	if (!ip_block)
 		return false;
 
-	if (ip_block->major <= 7) {
+	if (ip_block->version->major <= 7) {
 		/* gfx7 has no workaround */
 		return true;
-	} else if (ip_block->major == 8) {
+	} else if (ip_block->version->major == 8) {
 		if (adev->gfx.mec_fw_version >= 673)
 			/* gfx8 is fixed in MEC firmware 673 */
 			return false;
@@ -393,7 +398,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
 
 	if (ring->funcs->emit_vm_flush && (job->vm_needs_flush ||
 	    amdgpu_vm_is_gpu_reset(adev, id))) {
-		struct fence *fence;
+		struct dma_fence *fence;
 
 		trace_amdgpu_vm_flush(job->vm_pd_addr, ring->idx, job->vm_id);
 		amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr);
@@ -403,7 +408,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
 			return r;
 
 		mutex_lock(&adev->vm_manager.lock);
-		fence_put(id->last_flush);
+		dma_fence_put(id->last_flush);
 		id->last_flush = fence;
 		mutex_unlock(&adev->vm_manager.lock);
 	}
@@ -525,70 +530,6 @@ static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
 }
 
 /**
- * amdgpu_vm_clear_bo - initially clear the page dir/table
- *
- * @adev: amdgpu_device pointer
- * @bo: bo to clear
- *
- * need to reserve bo first before calling it.
- */
-static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
-			      struct amdgpu_vm *vm,
-			      struct amdgpu_bo *bo)
-{
-	struct amdgpu_ring *ring;
-	struct fence *fence = NULL;
-	struct amdgpu_job *job;
-	struct amdgpu_pte_update_params params;
-	unsigned entries;
-	uint64_t addr;
-	int r;
-
-	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
-
-	r = reservation_object_reserve_shared(bo->tbo.resv);
-	if (r)
-		return r;
-
-	r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
-	if (r)
-		goto error;
-
-	r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
-	if (r)
-		goto error;
-
-	addr = amdgpu_bo_gpu_offset(bo);
-	entries = amdgpu_bo_size(bo) / 8;
-
-	r = amdgpu_job_alloc_with_ib(adev, 64, &job);
-	if (r)
-		goto error;
-
-	memset(&params, 0, sizeof(params));
-	params.adev = adev;
-	params.ib = &job->ibs[0];
-	amdgpu_vm_do_set_ptes(&params, addr, 0, entries, 0, 0);
-	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
-
-	WARN_ON(job->ibs[0].length_dw > 64);
-	r = amdgpu_job_submit(job, ring, &vm->entity,
-			      AMDGPU_FENCE_OWNER_VM, &fence);
-	if (r)
-		goto error_free;
-
-	amdgpu_bo_fence(bo, fence, true);
-	fence_put(fence);
-	return 0;
-
-error_free:
-	amdgpu_job_free(job);
-
-error:
-	return r;
-}
-
-/**
  * amdgpu_vm_map_gart - Resolve gart mapping of addr
  *
  * @pages_addr: optional DMA address to use for lookup
@@ -612,32 +553,35 @@ static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
 	return result;
 }
 
-static int amdgpu_vm_update_pd_or_shadow(struct amdgpu_device *adev,
-					 struct amdgpu_vm *vm,
-					 bool shadow)
+/*
+ * amdgpu_vm_update_pdes - make sure that page directory is valid
+ *
+ * @adev: amdgpu_device pointer
+ * @vm: requested vm
+ * @start: start of GPU address range
+ * @end: end of GPU address range
+ *
+ * Allocates new page tables if necessary
+ * and updates the page directory.
+ * Returns 0 for success, error for failure.
+ */
+int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
+				    struct amdgpu_vm *vm)
 {
+	struct amdgpu_bo *shadow;
 	struct amdgpu_ring *ring;
-	struct amdgpu_bo *pd = shadow ? vm->page_directory->shadow :
-		vm->page_directory;
-	uint64_t pd_addr;
+	uint64_t pd_addr, shadow_addr;
 	uint32_t incr = AMDGPU_VM_PTE_COUNT * 8;
-	uint64_t last_pde = ~0, last_pt = ~0;
+	uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
 	unsigned count = 0, pt_idx, ndw;
 	struct amdgpu_job *job;
 	struct amdgpu_pte_update_params params;
-	struct fence *fence = NULL;
+	struct dma_fence *fence = NULL;
 
 	int r;
 
-	if (!pd)
-		return 0;
-
-	r = amdgpu_ttm_bind(&pd->tbo, &pd->tbo.mem);
-	if (r)
-		return r;
-
-	pd_addr = amdgpu_bo_gpu_offset(pd);
 	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
+	shadow = vm->page_directory->shadow;
 
 	/* padding, etc. */
 	ndw = 64;
@@ -645,6 +589,17 @@ static int amdgpu_vm_update_pd_or_shadow(struct amdgpu_device *adev,
 	/* assume the worst case */
 	ndw += vm->max_pde_used * 6;
 
+	pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
+	if (shadow) {
+		r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
+		if (r)
+			return r;
+		shadow_addr = amdgpu_bo_gpu_offset(shadow);
+		ndw *= 2;
+	} else {
+		shadow_addr = 0;
+	}
+
 	r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
 	if (r)
 		return r;
@@ -655,30 +610,26 @@ static int amdgpu_vm_update_pd_or_shadow(struct amdgpu_device *adev,
 
 	/* walk over the address space and update the page directory */
 	for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
-		struct amdgpu_bo *bo = vm->page_tables[pt_idx].entry.robj;
+		struct amdgpu_bo *bo = vm->page_tables[pt_idx].bo;
 		uint64_t pde, pt;
 
 		if (bo == NULL)
 			continue;
 
 		if (bo->shadow) {
-			struct amdgpu_bo *shadow = bo->shadow;
+			struct amdgpu_bo *pt_shadow = bo->shadow;
 
-			r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
+			r = amdgpu_ttm_bind(&pt_shadow->tbo,
+					    &pt_shadow->tbo.mem);
 			if (r)
 				return r;
 		}
 
 		pt = amdgpu_bo_gpu_offset(bo);
-		if (!shadow) {
-			if (vm->page_tables[pt_idx].addr == pt)
-				continue;
-			vm->page_tables[pt_idx].addr = pt;
-		} else {
-			if (vm->page_tables[pt_idx].shadow_addr == pt)
-				continue;
-			vm->page_tables[pt_idx].shadow_addr = pt;
-		}
+		if (vm->page_tables[pt_idx].addr == pt)
+			continue;
+
+		vm->page_tables[pt_idx].addr = pt;
 
 		pde = pd_addr + pt_idx * 8;
 		if (((last_pde + 8 * count) != pde) ||
@@ -686,6 +637,13 @@ static int amdgpu_vm_update_pd_or_shadow(struct amdgpu_device *adev,
 		    (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
 
 			if (count) {
+				if (shadow)
+					amdgpu_vm_do_set_ptes(&params,
+							      last_shadow,
+							      last_pt, count,
+							      incr,
+							      AMDGPU_PTE_VALID);
+
 				amdgpu_vm_do_set_ptes(&params, last_pde,
 						      last_pt, count, incr,
 						      AMDGPU_PTE_VALID);
@@ -693,34 +651,44 @@ static int amdgpu_vm_update_pd_or_shadow(struct amdgpu_device *adev,
 
 			count = 1;
 			last_pde = pde;
+			last_shadow = shadow_addr + pt_idx * 8;
 			last_pt = pt;
 		} else {
 			++count;
 		}
 	}
 
-	if (count)
+	if (count) {
+		if (vm->page_directory->shadow)
+			amdgpu_vm_do_set_ptes(&params, last_shadow, last_pt,
+					      count, incr, AMDGPU_PTE_VALID);
+
 		amdgpu_vm_do_set_ptes(&params, last_pde, last_pt,
 				      count, incr, AMDGPU_PTE_VALID);
+	}
 
-	if (params.ib->length_dw != 0) {
-		amdgpu_ring_pad_ib(ring, params.ib);
-		amdgpu_sync_resv(adev, &job->sync, pd->tbo.resv,
+	if (params.ib->length_dw == 0) {
+		amdgpu_job_free(job);
+		return 0;
+	}
+
+	amdgpu_ring_pad_ib(ring, params.ib);
+	amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
+			 AMDGPU_FENCE_OWNER_VM);
+	if (shadow)
+		amdgpu_sync_resv(adev, &job->sync, shadow->tbo.resv,
 				 AMDGPU_FENCE_OWNER_VM);
-		WARN_ON(params.ib->length_dw > ndw);
-		r = amdgpu_job_submit(job, ring, &vm->entity,
-				      AMDGPU_FENCE_OWNER_VM, &fence);
-		if (r)
-			goto error_free;
 
-		amdgpu_bo_fence(pd, fence, true);
-		fence_put(vm->page_directory_fence);
-		vm->page_directory_fence = fence_get(fence);
-		fence_put(fence);
+	WARN_ON(params.ib->length_dw > ndw);
+	r = amdgpu_job_submit(job, ring, &vm->entity,
+			      AMDGPU_FENCE_OWNER_VM, &fence);
+	if (r)
+		goto error_free;
 
-	} else {
-		amdgpu_job_free(job);
-	}
+	amdgpu_bo_fence(vm->page_directory, fence, true);
+	dma_fence_put(vm->page_directory_fence);
+	vm->page_directory_fence = dma_fence_get(fence);
+	dma_fence_put(fence);
 
 	return 0;
 
@@ -729,29 +697,6 @@ error_free:
 	return r;
 }
 
-/*
- * amdgpu_vm_update_pdes - make sure that page directory is valid
- *
- * @adev: amdgpu_device pointer
- * @vm: requested vm
- * @start: start of GPU address range
- * @end: end of GPU address range
- *
- * Allocates new page tables if necessary
- * and updates the page directory.
- * Returns 0 for success, error for failure.
- */
-int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
-                                   struct amdgpu_vm *vm)
-{
-	int r;
-
-	r = amdgpu_vm_update_pd_or_shadow(adev, vm, true);
-	if (r)
-		return r;
-	return amdgpu_vm_update_pd_or_shadow(adev, vm, false);
-}
-
 /**
  * amdgpu_vm_update_ptes - make sure that page tables are valid
  *
@@ -781,11 +726,11 @@ static void amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
 	/* initialize the variables */
 	addr = start;
 	pt_idx = addr >> amdgpu_vm_block_size;
-	pt = vm->page_tables[pt_idx].entry.robj;
+	pt = vm->page_tables[pt_idx].bo;
 	if (params->shadow) {
 		if (!pt->shadow)
 			return;
-		pt = vm->page_tables[pt_idx].entry.robj->shadow;
+		pt = pt->shadow;
 	}
 	if ((addr & ~mask) == (end & ~mask))
 		nptes = end - addr;
@@ -804,11 +749,11 @@ static void amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
 	/* walk over the address space and update the page tables */
 	while (addr < end) {
 		pt_idx = addr >> amdgpu_vm_block_size;
-		pt = vm->page_tables[pt_idx].entry.robj;
+		pt = vm->page_tables[pt_idx].bo;
 		if (params->shadow) {
 			if (!pt->shadow)
 				return;
-			pt = vm->page_tables[pt_idx].entry.robj->shadow;
+			pt = pt->shadow;
 		}
 
 		if ((addr & ~mask) == (end & ~mask))
@@ -929,20 +874,20 @@ static void amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
  * Returns 0 for success, -EINVAL for failure.
  */
 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
-				       struct fence *exclusive,
+				       struct dma_fence *exclusive,
 				       uint64_t src,
 				       dma_addr_t *pages_addr,
 				       struct amdgpu_vm *vm,
 				       uint64_t start, uint64_t last,
 				       uint32_t flags, uint64_t addr,
-				       struct fence **fence)
+				       struct dma_fence **fence)
 {
 	struct amdgpu_ring *ring;
 	void *owner = AMDGPU_FENCE_OWNER_VM;
 	unsigned nptes, ncmds, ndw;
 	struct amdgpu_job *job;
 	struct amdgpu_pte_update_params params;
-	struct fence *f = NULL;
+	struct dma_fence *f = NULL;
 	int r;
 
 	memset(&params, 0, sizeof(params));
@@ -1045,10 +990,10 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 
 	amdgpu_bo_fence(vm->page_directory, f, true);
 	if (fence) {
-		fence_put(*fence);
-		*fence = fence_get(f);
+		dma_fence_put(*fence);
+		*fence = dma_fence_get(f);
 	}
-	fence_put(f);
+	dma_fence_put(f);
 	return 0;
 
 error_free:
@@ -1065,8 +1010,8 @@ error_free:
  * @pages_addr: DMA addresses to use for mapping
  * @vm: requested vm
  * @mapping: mapped range and flags to use for the update
- * @addr: addr to set the area to
  * @flags: HW flags for the mapping
+ * @nodes: array of drm_mm_nodes with the MC addresses
  * @fence: optional resulting fence
  *
  * Split the mapping into smaller chunks so that each update fits
@@ -1074,17 +1019,16 @@ error_free:
  * Returns 0 for success, -EINVAL for failure.
  */
 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
-				      struct fence *exclusive,
+				      struct dma_fence *exclusive,
 				      uint32_t gtt_flags,
 				      dma_addr_t *pages_addr,
 				      struct amdgpu_vm *vm,
 				      struct amdgpu_bo_va_mapping *mapping,
-				      uint32_t flags, uint64_t addr,
-				      struct fence **fence)
+				      uint32_t flags,
+				      struct drm_mm_node *nodes,
+				      struct dma_fence **fence)
 {
-	const uint64_t max_size = 64ULL * 1024ULL * 1024ULL / AMDGPU_GPU_PAGE_SIZE;
-
-	uint64_t src = 0, start = mapping->it.start;
+	uint64_t pfn, src = 0, start = mapping->it.start;
 	int r;
 
 	/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
@@ -1097,23 +1041,40 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
 
 	trace_amdgpu_vm_bo_update(mapping);
 
-	if (pages_addr) {
-		if (flags == gtt_flags)
-			src = adev->gart.table_addr + (addr >> 12) * 8;
-		addr = 0;
+	pfn = mapping->offset >> PAGE_SHIFT;
+	if (nodes) {
+		while (pfn >= nodes->size) {
+			pfn -= nodes->size;
+			++nodes;
+		}
 	}
-	addr += mapping->offset;
 
-	if (!pages_addr || src)
-		return amdgpu_vm_bo_update_mapping(adev, exclusive,
-						   src, pages_addr, vm,
-						   start, mapping->it.last,
-						   flags, addr, fence);
+	do {
+		uint64_t max_entries;
+		uint64_t addr, last;
 
-	while (start != mapping->it.last + 1) {
-		uint64_t last;
+		if (nodes) {
+			addr = nodes->start << PAGE_SHIFT;
+			max_entries = (nodes->size - pfn) *
+				(PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
+		} else {
+			addr = 0;
+			max_entries = S64_MAX;
+		}
 
-		last = min((uint64_t)mapping->it.last, start + max_size - 1);
+		if (pages_addr) {
+			if (flags == gtt_flags)
+				src = adev->gart.table_addr +
+					(addr >> AMDGPU_GPU_PAGE_SHIFT) * 8;
+			else
+				max_entries = min(max_entries, 16ull * 1024ull);
+			addr = 0;
+		} else if (flags & AMDGPU_PTE_VALID) {
+			addr += adev->vm_manager.vram_base_offset;
+		}
+		addr += pfn << PAGE_SHIFT;
+
+		last = min((uint64_t)mapping->it.last, start + max_entries - 1);
 		r = amdgpu_vm_bo_update_mapping(adev, exclusive,
 						src, pages_addr, vm,
 						start, last, flags, addr,
@@ -1121,9 +1082,14 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
 		if (r)
 			return r;
 
+		pfn += last - start + 1;
+		if (nodes && nodes->size == pfn) {
+			pfn = 0;
+			++nodes;
+		}
 		start = last + 1;
-		addr += max_size * AMDGPU_GPU_PAGE_SIZE;
-	}
+
+	} while (unlikely(start != mapping->it.last + 1));
 
 	return 0;
 }
@@ -1147,40 +1113,30 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 	dma_addr_t *pages_addr = NULL;
 	uint32_t gtt_flags, flags;
 	struct ttm_mem_reg *mem;
-	struct fence *exclusive;
-	uint64_t addr;
+	struct drm_mm_node *nodes;
+	struct dma_fence *exclusive;
 	int r;
 
 	if (clear) {
 		mem = NULL;
-		addr = 0;
+		nodes = NULL;
 		exclusive = NULL;
 	} else {
 		struct ttm_dma_tt *ttm;
 
 		mem = &bo_va->bo->tbo.mem;
-		addr = (u64)mem->start << PAGE_SHIFT;
-		switch (mem->mem_type) {
-		case TTM_PL_TT:
+		nodes = mem->mm_node;
+		if (mem->mem_type == TTM_PL_TT) {
 			ttm = container_of(bo_va->bo->tbo.ttm, struct
 					   ttm_dma_tt, ttm);
 			pages_addr = ttm->dma_address;
-			break;
-
-		case TTM_PL_VRAM:
-			addr += adev->vm_manager.vram_base_offset;
-			break;
-
-		default:
-			break;
 		}
-
 		exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv);
 	}
 
 	flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
 	gtt_flags = (amdgpu_ttm_is_bound(bo_va->bo->tbo.ttm) &&
-		adev == bo_va->bo->adev) ? flags : 0;
+		adev == amdgpu_ttm_adev(bo_va->bo->tbo.bdev)) ? flags : 0;
 
 	spin_lock(&vm->status_lock);
 	if (!list_empty(&bo_va->vm_status))
@@ -1190,7 +1146,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 	list_for_each_entry(mapping, &bo_va->invalids, list) {
 		r = amdgpu_vm_bo_split_mapping(adev, exclusive,
 					       gtt_flags, pages_addr, vm,
-					       mapping, flags, addr,
+					       mapping, flags, nodes,
 					       &bo_va->last_pt_update);
 		if (r)
 			return r;
@@ -1405,18 +1361,18 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 	/* walk over the address space and allocate the page tables */
 	for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) {
 		struct reservation_object *resv = vm->page_directory->tbo.resv;
-		struct amdgpu_bo_list_entry *entry;
 		struct amdgpu_bo *pt;
 
-		entry = &vm->page_tables[pt_idx].entry;
-		if (entry->robj)
+		if (vm->page_tables[pt_idx].bo)
 			continue;
 
 		r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
 				     AMDGPU_GPU_PAGE_SIZE, true,
 				     AMDGPU_GEM_DOMAIN_VRAM,
 				     AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
-				     AMDGPU_GEM_CREATE_SHADOW,
+				     AMDGPU_GEM_CREATE_SHADOW |
+				     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
+				     AMDGPU_GEM_CREATE_VRAM_CLEARED,
 				     NULL, resv, &pt);
 		if (r)
 			goto error_free;
@@ -1426,27 +1382,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 		 */
 		pt->parent = amdgpu_bo_ref(vm->page_directory);
 
-		r = amdgpu_vm_clear_bo(adev, vm, pt);
-		if (r) {
-			amdgpu_bo_unref(&pt->shadow);
-			amdgpu_bo_unref(&pt);
-			goto error_free;
-		}
-
-		if (pt->shadow) {
-			r = amdgpu_vm_clear_bo(adev, vm, pt->shadow);
-			if (r) {
-				amdgpu_bo_unref(&pt->shadow);
-				amdgpu_bo_unref(&pt);
-				goto error_free;
-			}
-		}
-
-		entry->robj = pt;
-		entry->priority = 0;
-		entry->tv.bo = &entry->robj->tbo;
-		entry->tv.shared = true;
-		entry->user_pages = NULL;
+		vm->page_tables[pt_idx].bo = pt;
 		vm->page_tables[pt_idx].addr = 0;
 	}
 
@@ -1547,7 +1483,7 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
 		kfree(mapping);
 	}
 
-	fence_put(bo_va->last_pt_update);
+	dma_fence_put(bo_va->last_pt_update);
 	kfree(bo_va);
 }
 
@@ -1626,7 +1562,9 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	r = amdgpu_bo_create(adev, pd_size, align, true,
 			     AMDGPU_GEM_DOMAIN_VRAM,
 			     AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
-			     AMDGPU_GEM_CREATE_SHADOW,
+			     AMDGPU_GEM_CREATE_SHADOW |
+			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
+			     AMDGPU_GEM_CREATE_VRAM_CLEARED,
 			     NULL, NULL, &vm->page_directory);
 	if (r)
 		goto error_free_sched_entity;
@@ -1635,24 +1573,11 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	if (r)
 		goto error_free_page_directory;
 
-	r = amdgpu_vm_clear_bo(adev, vm, vm->page_directory);
-	if (r)
-		goto error_unreserve;
-
-	if (vm->page_directory->shadow) {
-		r = amdgpu_vm_clear_bo(adev, vm, vm->page_directory->shadow);
-		if (r)
-			goto error_unreserve;
-	}
-
 	vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
 	amdgpu_bo_unreserve(vm->page_directory);
 
 	return 0;
 
-error_unreserve:
-	amdgpu_bo_unreserve(vm->page_directory);
-
 error_free_page_directory:
 	amdgpu_bo_unref(&vm->page_directory->shadow);
 	amdgpu_bo_unref(&vm->page_directory);
@@ -1697,7 +1622,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	}
 
 	for (i = 0; i < amdgpu_vm_num_pdes(adev); i++) {
-		struct amdgpu_bo *pt = vm->page_tables[i].entry.robj;
+		struct amdgpu_bo *pt = vm->page_tables[i].bo;
 
 		if (!pt)
 			continue;
@@ -1709,7 +1634,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 
 	amdgpu_bo_unref(&vm->page_directory->shadow);
 	amdgpu_bo_unref(&vm->page_directory);
-	fence_put(vm->page_directory_fence);
+	dma_fence_put(vm->page_directory_fence);
 }
 
 /**
@@ -1733,7 +1658,8 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
 			      &adev->vm_manager.ids_lru);
 	}
 
-	adev->vm_manager.fence_context = fence_context_alloc(AMDGPU_MAX_RINGS);
+	adev->vm_manager.fence_context =
+		dma_fence_context_alloc(AMDGPU_MAX_RINGS);
 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
 		adev->vm_manager.seqno[i] = 0;
 
@@ -1755,9 +1681,9 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
 	for (i = 0; i < AMDGPU_NUM_VM; ++i) {
 		struct amdgpu_vm_id *id = &adev->vm_manager.ids[i];
 
-		fence_put(adev->vm_manager.ids[i].first);
+		dma_fence_put(adev->vm_manager.ids[i].first);
 		amdgpu_sync_free(&adev->vm_manager.ids[i].active);
-		fence_put(id->flushed_updates);
-		fence_put(id->last_flush);
+		dma_fence_put(id->flushed_updates);
+		dma_fence_put(id->last_flush);
 	}
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
new file mode 100644
index 000000000000..adbc2f5e5c7f
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -0,0 +1,205 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Christian König
+ */
+#ifndef __AMDGPU_VM_H__
+#define __AMDGPU_VM_H__
+
+#include <linux/rbtree.h>
+
+#include "gpu_scheduler.h"
+#include "amdgpu_sync.h"
+#include "amdgpu_ring.h"
+
+struct amdgpu_bo_va;
+struct amdgpu_job;
+struct amdgpu_bo_list_entry;
+
+/*
+ * GPUVM handling
+ */
+
+/* maximum number of VMIDs */
+#define AMDGPU_NUM_VM	16
+
+/* Maximum number of PTEs the hardware can write with one command */
+#define AMDGPU_VM_MAX_UPDATE_SIZE	0x3FFFF
+
+/* number of entries in page table */
+#define AMDGPU_VM_PTE_COUNT (1 << amdgpu_vm_block_size)
+
+/* PTBs (Page Table Blocks) need to be aligned to 32K */
+#define AMDGPU_VM_PTB_ALIGN_SIZE   32768
+
+/* LOG2 number of continuous pages for the fragment field */
+#define AMDGPU_LOG2_PAGES_PER_FRAG 4
+
+#define AMDGPU_PTE_VALID	(1 << 0)
+#define AMDGPU_PTE_SYSTEM	(1 << 1)
+#define AMDGPU_PTE_SNOOPED	(1 << 2)
+
+/* VI only */
+#define AMDGPU_PTE_EXECUTABLE	(1 << 4)
+
+#define AMDGPU_PTE_READABLE	(1 << 5)
+#define AMDGPU_PTE_WRITEABLE	(1 << 6)
+
+#define AMDGPU_PTE_FRAG(x)	((x & 0x1f) << 7)
+
+/* How to programm VM fault handling */
+#define AMDGPU_VM_FAULT_STOP_NEVER	0
+#define AMDGPU_VM_FAULT_STOP_FIRST	1
+#define AMDGPU_VM_FAULT_STOP_ALWAYS	2
+
+struct amdgpu_vm_pt {
+	struct amdgpu_bo	*bo;
+	uint64_t		addr;
+};
+
+struct amdgpu_vm {
+	/* tree of virtual addresses mapped */
+	struct rb_root		va;
+
+	/* protecting invalidated */
+	spinlock_t		status_lock;
+
+	/* BOs moved, but not yet updated in the PT */
+	struct list_head	invalidated;
+
+	/* BOs cleared in the PT because of a move */
+	struct list_head	cleared;
+
+	/* BO mappings freed, but not yet updated in the PT */
+	struct list_head	freed;
+
+	/* contains the page directory */
+	struct amdgpu_bo	*page_directory;
+	unsigned		max_pde_used;
+	struct dma_fence		*page_directory_fence;
+	uint64_t		last_eviction_counter;
+
+	/* array of page tables, one for each page directory entry */
+	struct amdgpu_vm_pt	*page_tables;
+
+	/* for id and flush management per ring */
+	struct amdgpu_vm_id	*ids[AMDGPU_MAX_RINGS];
+
+	/* protecting freed */
+	spinlock_t		freed_lock;
+
+	/* Scheduler entity for page table updates */
+	struct amd_sched_entity	entity;
+
+	/* client id */
+	u64                     client_id;
+};
+
+struct amdgpu_vm_id {
+	struct list_head	list;
+	struct dma_fence		*first;
+	struct amdgpu_sync	active;
+	struct dma_fence		*last_flush;
+	atomic64_t		owner;
+
+	uint64_t		pd_gpu_addr;
+	/* last flushed PD/PT update */
+	struct dma_fence		*flushed_updates;
+
+	uint32_t                current_gpu_reset_count;
+
+	uint32_t		gds_base;
+	uint32_t		gds_size;
+	uint32_t		gws_base;
+	uint32_t		gws_size;
+	uint32_t		oa_base;
+	uint32_t		oa_size;
+};
+
+struct amdgpu_vm_manager {
+	/* Handling of VMIDs */
+	struct mutex				lock;
+	unsigned				num_ids;
+	struct list_head			ids_lru;
+	struct amdgpu_vm_id			ids[AMDGPU_NUM_VM];
+
+	/* Handling of VM fences */
+	u64					fence_context;
+	unsigned				seqno[AMDGPU_MAX_RINGS];
+
+	uint32_t				max_pfn;
+	/* vram base address for page table entry  */
+	u64					vram_base_offset;
+	/* is vm enabled? */
+	bool					enabled;
+	/* vm pte handling */
+	const struct amdgpu_vm_pte_funcs        *vm_pte_funcs;
+	struct amdgpu_ring                      *vm_pte_rings[AMDGPU_MAX_RINGS];
+	unsigned				vm_pte_num_rings;
+	atomic_t				vm_pte_next_ring;
+	/* client id counter */
+	atomic64_t				client_counter;
+};
+
+void amdgpu_vm_manager_init(struct amdgpu_device *adev);
+void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
+int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
+			 struct list_head *validated,
+			 struct amdgpu_bo_list_entry *entry);
+int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			      int (*callback)(void *p, struct amdgpu_bo *bo),
+			      void *param);
+void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
+				  struct amdgpu_vm *vm);
+int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
+		      struct amdgpu_sync *sync, struct dma_fence *fence,
+		      struct amdgpu_job *job);
+int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job);
+void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id);
+int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
+				    struct amdgpu_vm *vm);
+int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
+			  struct amdgpu_vm *vm);
+int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			     struct amdgpu_sync *sync);
+int amdgpu_vm_bo_update(struct amdgpu_device *adev,
+			struct amdgpu_bo_va *bo_va,
+			bool clear);
+void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
+			     struct amdgpu_bo *bo);
+struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
+				       struct amdgpu_bo *bo);
+struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
+				      struct amdgpu_vm *vm,
+				      struct amdgpu_bo *bo);
+int amdgpu_vm_bo_map(struct amdgpu_device *adev,
+		     struct amdgpu_bo_va *bo_va,
+		     uint64_t addr, uint64_t offset,
+		     uint64_t size, uint32_t flags);
+int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
+		       struct amdgpu_bo_va *bo_va,
+		       uint64_t addr);
+void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
+		      struct amdgpu_bo_va *bo_va);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
new file mode 100644
index 000000000000..d710226a0fff
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -0,0 +1,222 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Christian König
+ */
+
+#include <drm/drmP.h>
+#include "amdgpu.h"
+
+struct amdgpu_vram_mgr {
+	struct drm_mm mm;
+	spinlock_t lock;
+};
+
+/**
+ * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
+ *
+ * @man: TTM memory type manager
+ * @p_size: maximum size of VRAM
+ *
+ * Allocate and initialize the VRAM manager.
+ */
+static int amdgpu_vram_mgr_init(struct ttm_mem_type_manager *man,
+				unsigned long p_size)
+{
+	struct amdgpu_vram_mgr *mgr;
+
+	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
+	if (!mgr)
+		return -ENOMEM;
+
+	drm_mm_init(&mgr->mm, 0, p_size);
+	spin_lock_init(&mgr->lock);
+	man->priv = mgr;
+	return 0;
+}
+
+/**
+ * amdgpu_vram_mgr_fini - free and destroy VRAM manager
+ *
+ * @man: TTM memory type manager
+ *
+ * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
+ * allocated inside it.
+ */
+static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man)
+{
+	struct amdgpu_vram_mgr *mgr = man->priv;
+
+	spin_lock(&mgr->lock);
+	if (!drm_mm_clean(&mgr->mm)) {
+		spin_unlock(&mgr->lock);
+		return -EBUSY;
+	}
+
+	drm_mm_takedown(&mgr->mm);
+	spin_unlock(&mgr->lock);
+	kfree(mgr);
+	man->priv = NULL;
+	return 0;
+}
+
+/**
+ * amdgpu_vram_mgr_new - allocate new ranges
+ *
+ * @man: TTM memory type manager
+ * @tbo: TTM BO we need this range for
+ * @place: placement flags and restrictions
+ * @mem: the resulting mem object
+ *
+ * Allocate VRAM for the given BO.
+ */
+static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
+			       struct ttm_buffer_object *tbo,
+			       const struct ttm_place *place,
+			       struct ttm_mem_reg *mem)
+{
+	struct amdgpu_bo *bo = container_of(tbo, struct amdgpu_bo, tbo);
+	struct amdgpu_vram_mgr *mgr = man->priv;
+	struct drm_mm *mm = &mgr->mm;
+	struct drm_mm_node *nodes;
+	enum drm_mm_search_flags sflags = DRM_MM_SEARCH_DEFAULT;
+	enum drm_mm_allocator_flags aflags = DRM_MM_CREATE_DEFAULT;
+	unsigned long lpfn, num_nodes, pages_per_node, pages_left;
+	unsigned i;
+	int r;
+
+	lpfn = place->lpfn;
+	if (!lpfn)
+		lpfn = man->size;
+
+	if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS ||
+	    place->lpfn || amdgpu_vram_page_split == -1) {
+		pages_per_node = ~0ul;
+		num_nodes = 1;
+	} else {
+		pages_per_node = max((uint32_t)amdgpu_vram_page_split,
+				     mem->page_alignment);
+		num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node);
+	}
+
+	nodes = kcalloc(num_nodes, sizeof(*nodes), GFP_KERNEL);
+	if (!nodes)
+		return -ENOMEM;
+
+	if (place->flags & TTM_PL_FLAG_TOPDOWN) {
+		sflags = DRM_MM_SEARCH_BELOW;
+		aflags = DRM_MM_CREATE_TOP;
+	}
+
+	pages_left = mem->num_pages;
+
+	spin_lock(&mgr->lock);
+	for (i = 0; i < num_nodes; ++i) {
+		unsigned long pages = min(pages_left, pages_per_node);
+		uint32_t alignment = mem->page_alignment;
+
+		if (pages == pages_per_node)
+			alignment = pages_per_node;
+		else
+			sflags |= DRM_MM_SEARCH_BEST;
+
+		r = drm_mm_insert_node_in_range_generic(mm, &nodes[i], pages,
+							alignment, 0,
+							place->fpfn, lpfn,
+							sflags, aflags);
+		if (unlikely(r))
+			goto error;
+
+		pages_left -= pages;
+	}
+	spin_unlock(&mgr->lock);
+
+	mem->start = num_nodes == 1 ? nodes[0].start : AMDGPU_BO_INVALID_OFFSET;
+	mem->mm_node = nodes;
+
+	return 0;
+
+error:
+	while (i--)
+		drm_mm_remove_node(&nodes[i]);
+	spin_unlock(&mgr->lock);
+
+	kfree(nodes);
+	return r == -ENOSPC ? 0 : r;
+}
+
+/**
+ * amdgpu_vram_mgr_del - free ranges
+ *
+ * @man: TTM memory type manager
+ * @tbo: TTM BO we need this range for
+ * @place: placement flags and restrictions
+ * @mem: TTM memory object
+ *
+ * Free the allocated VRAM again.
+ */
+static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man,
+				struct ttm_mem_reg *mem)
+{
+	struct amdgpu_vram_mgr *mgr = man->priv;
+	struct drm_mm_node *nodes = mem->mm_node;
+	unsigned pages = mem->num_pages;
+
+	if (!mem->mm_node)
+		return;
+
+	spin_lock(&mgr->lock);
+	while (pages) {
+		pages -= nodes->size;
+		drm_mm_remove_node(nodes);
+		++nodes;
+	}
+	spin_unlock(&mgr->lock);
+
+	kfree(mem->mm_node);
+	mem->mm_node = NULL;
+}
+
+/**
+ * amdgpu_vram_mgr_debug - dump VRAM table
+ *
+ * @man: TTM memory type manager
+ * @prefix: text prefix
+ *
+ * Dump the table content using printk.
+ */
+static void amdgpu_vram_mgr_debug(struct ttm_mem_type_manager *man,
+				  const char *prefix)
+{
+	struct amdgpu_vram_mgr *mgr = man->priv;
+
+	spin_lock(&mgr->lock);
+	drm_mm_debug_table(&mgr->mm, prefix);
+	spin_unlock(&mgr->lock);
+}
+
+const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func = {
+	amdgpu_vram_mgr_init,
+	amdgpu_vram_mgr_fini,
+	amdgpu_vram_mgr_new,
+	amdgpu_vram_mgr_del,
+	amdgpu_vram_mgr_debug
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c b/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c
index f7d236f95e74..8c9bc75a9c2d 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_crtc.c
@@ -31,6 +31,7 @@
 #include "atom.h"
 #include "atom-bits.h"
 #include "atombios_encoders.h"
+#include "atombios_crtc.h"
 #include "amdgpu_atombios.h"
 #include "amdgpu_pll.h"
 #include "amdgpu_connectors.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index 5be788b269e2..bda9e3de191e 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -887,9 +887,6 @@ static void ci_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
 {
 	struct ci_power_info *pi = ci_get_pi(adev);
 
-	if (pi->uvd_power_gated == gate)
-		return;
-
 	pi->uvd_power_gated = gate;
 
 	ci_update_uvd_dpm(adev, gate);
@@ -960,6 +957,12 @@ static void ci_apply_state_adjust_rules(struct amdgpu_device *adev,
 		sclk = ps->performance_levels[0].sclk;
 	}
 
+	if (adev->pm.pm_display_cfg.min_core_set_clock > sclk)
+		sclk = adev->pm.pm_display_cfg.min_core_set_clock;
+
+	if (adev->pm.pm_display_cfg.min_mem_set_clock > mclk)
+		mclk = adev->pm.pm_display_cfg.min_mem_set_clock;
+
 	if (rps->vce_active) {
 		if (sclk < adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].sclk)
 			sclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].sclk;
@@ -2201,6 +2204,11 @@ static int ci_upload_firmware(struct amdgpu_device *adev)
 	struct ci_power_info *pi = ci_get_pi(adev);
 	int i, ret;
 
+	if (amdgpu_ci_is_smc_running(adev)) {
+		DRM_INFO("smc is running, no need to load smc firmware\n");
+		return 0;
+	}
+
 	for (i = 0; i < adev->usec_timeout; i++) {
 		if (RREG32_SMC(ixRCU_UC_EVENTS) & RCU_UC_EVENTS__boot_seq_done_MASK)
 			break;
@@ -4190,8 +4198,10 @@ static int ci_update_uvd_dpm(struct amdgpu_device *adev, bool gate)
 {
 	struct ci_power_info *pi = ci_get_pi(adev);
 	u32 tmp;
+	int ret = 0;
 
 	if (!gate) {
+		/* turn the clocks on when decoding */
 		if (pi->caps_uvd_dpm ||
 		    (adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count <= 0))
 			pi->smc_state_table.UvdBootLevel = 0;
@@ -4203,9 +4213,14 @@ static int ci_update_uvd_dpm(struct amdgpu_device *adev, bool gate)
 		tmp &= ~DPM_TABLE_475__UvdBootLevel_MASK;
 		tmp |= (pi->smc_state_table.UvdBootLevel << DPM_TABLE_475__UvdBootLevel__SHIFT);
 		WREG32_SMC(ixDPM_TABLE_475, tmp);
+		ret = ci_enable_uvd_dpm(adev, true);
+	} else {
+		ret = ci_enable_uvd_dpm(adev, false);
+		if (ret)
+			return ret;
 	}
 
-	return ci_enable_uvd_dpm(adev, !gate);
+	return ret;
 }
 
 static u8 ci_get_vce_boot_level(struct amdgpu_device *adev)
@@ -4247,13 +4262,12 @@ static int ci_update_vce_dpm(struct amdgpu_device *adev,
 
 			ret = ci_enable_vce_dpm(adev, true);
 		} else {
+			ret = ci_enable_vce_dpm(adev, false);
+			if (ret)
+				return ret;
 			/* turn the clocks off when not encoding */
 			ret = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
 							    AMD_CG_STATE_GATE);
-			if (ret)
-				return ret;
-
-			ret = ci_enable_vce_dpm(adev, false);
 		}
 	}
 	return ret;
@@ -5219,6 +5233,7 @@ static void ci_update_current_ps(struct amdgpu_device *adev,
 	pi->current_rps = *rps;
 	pi->current_ps = *new_ps;
 	pi->current_rps.ps_priv = &pi->current_ps;
+	adev->pm.dpm.current_ps = &pi->current_rps;
 }
 
 static void ci_update_requested_ps(struct amdgpu_device *adev,
@@ -5230,6 +5245,7 @@ static void ci_update_requested_ps(struct amdgpu_device *adev,
 	pi->requested_rps = *rps;
 	pi->requested_ps = *new_ps;
 	pi->requested_rps.ps_priv = &pi->requested_ps;
+	adev->pm.dpm.requested_ps = &pi->requested_rps;
 }
 
 static int ci_dpm_pre_set_power_state(struct amdgpu_device *adev)
@@ -5267,8 +5283,6 @@ static int ci_dpm_enable(struct amdgpu_device *adev)
 	struct amdgpu_ps *boot_ps = adev->pm.dpm.boot_ps;
 	int ret;
 
-	if (amdgpu_ci_is_smc_running(adev))
-		return -EINVAL;
 	if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_NONE) {
 		ci_enable_voltage_control(adev);
 		ret = ci_construct_voltage_tables(adev);
@@ -5689,7 +5703,7 @@ static int ci_parse_power_table(struct amdgpu_device *adev)
 	adev->pm.dpm.num_ps = state_array->ucNumEntries;
 
 	/* fill in the vce power states */
-	for (i = 0; i < AMDGPU_MAX_VCE_LEVELS; i++) {
+	for (i = 0; i < adev->pm.dpm.num_of_vce_states; i++) {
 		u32 sclk, mclk;
 		clock_array_index = adev->pm.dpm.vce_states[i].clk_idx;
 		clock_info = (union pplib_clock_info *)
@@ -5874,7 +5888,7 @@ static int ci_dpm_init(struct amdgpu_device *adev)
 	pi->pcie_dpm_key_disabled = 0;
 	pi->thermal_sclk_dpm_enabled = 0;
 
-	if (amdgpu_sclk_deep_sleep_en)
+	if (amdgpu_pp_feature_mask & SCLK_DEEP_SLEEP_MASK)
 		pi->caps_sclk_ds = true;
 	else
 		pi->caps_sclk_ds = false;
@@ -5977,7 +5991,7 @@ static int ci_dpm_init(struct amdgpu_device *adev)
 			tmp |= CNB_PWRMGT_CNTL__DPM_ENABLED_MASK;
 			break;
 		default:
-			DRM_ERROR("Invalid PCC GPIO: %u!\n", gpio.shift);
+			DRM_INFO("Invalid PCC GPIO: %u!\n", gpio.shift);
 			break;
 		}
 		WREG32_SMC(ixCNB_PWRMGT_CNTL, tmp);
@@ -6069,7 +6083,7 @@ ci_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
 		activity_percent = activity_percent > 100 ? 100 : activity_percent;
 	}
 
-	seq_printf(m, "uvd %sabled\n", pi->uvd_enabled ? "en" : "dis");
+	seq_printf(m, "uvd %sabled\n", pi->uvd_power_gated ? "dis" : "en");
 	seq_printf(m, "vce %sabled\n", rps->vce_active ? "en" : "dis");
 	seq_printf(m, "power level avg    sclk: %u mclk: %u\n",
 		   sclk, mclk);
@@ -6094,6 +6108,56 @@ static void ci_dpm_print_power_state(struct amdgpu_device *adev,
 	amdgpu_dpm_print_ps_status(adev, rps);
 }
 
+static inline bool ci_are_power_levels_equal(const struct ci_pl *ci_cpl1,
+						const struct ci_pl *ci_cpl2)
+{
+	return ((ci_cpl1->mclk == ci_cpl2->mclk) &&
+		  (ci_cpl1->sclk == ci_cpl2->sclk) &&
+		  (ci_cpl1->pcie_gen == ci_cpl2->pcie_gen) &&
+		  (ci_cpl1->pcie_lane == ci_cpl2->pcie_lane));
+}
+
+static int ci_check_state_equal(struct amdgpu_device *adev,
+				struct amdgpu_ps *cps,
+				struct amdgpu_ps *rps,
+				bool *equal)
+{
+	struct ci_ps *ci_cps;
+	struct ci_ps *ci_rps;
+	int i;
+
+	if (adev == NULL || cps == NULL || rps == NULL || equal == NULL)
+		return -EINVAL;
+
+	ci_cps = ci_get_ps(cps);
+	ci_rps = ci_get_ps(rps);
+
+	if (ci_cps == NULL) {
+		*equal = false;
+		return 0;
+	}
+
+	if (ci_cps->performance_level_count != ci_rps->performance_level_count) {
+
+		*equal = false;
+		return 0;
+	}
+
+	for (i = 0; i < ci_cps->performance_level_count; i++) {
+		if (!ci_are_power_levels_equal(&(ci_cps->performance_levels[i]),
+					&(ci_rps->performance_levels[i]))) {
+			*equal = false;
+			return 0;
+		}
+	}
+
+	/* If all performance levels are the same try to use the UVD clocks to break the tie.*/
+	*equal = ((cps->vclk == rps->vclk) && (cps->dclk == rps->dclk));
+	*equal &= ((cps->evclk == rps->evclk) && (cps->ecclk == rps->ecclk));
+
+	return 0;
+}
+
 static u32 ci_dpm_get_sclk(struct amdgpu_device *adev, bool low)
 {
 	struct ci_power_info *pi = ci_get_pi(adev);
@@ -6289,12 +6353,19 @@ static int ci_dpm_suspend(void *handle)
 
 	if (adev->pm.dpm_enabled) {
 		mutex_lock(&adev->pm.mutex);
-		/* disable dpm */
-		ci_dpm_disable(adev);
-		/* reset the power state */
-		adev->pm.dpm.current_ps = adev->pm.dpm.requested_ps = adev->pm.dpm.boot_ps;
+		amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq,
+			       AMDGPU_THERMAL_IRQ_LOW_TO_HIGH);
+		amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq,
+			       AMDGPU_THERMAL_IRQ_HIGH_TO_LOW);
+		adev->pm.dpm.last_user_state = adev->pm.dpm.user_state;
+		adev->pm.dpm.last_state = adev->pm.dpm.state;
+		adev->pm.dpm.user_state = POWER_STATE_TYPE_INTERNAL_BOOT;
+		adev->pm.dpm.state = POWER_STATE_TYPE_INTERNAL_BOOT;
 		mutex_unlock(&adev->pm.mutex);
+		amdgpu_pm_compute_clocks(adev);
+
 	}
+
 	return 0;
 }
 
@@ -6312,6 +6383,8 @@ static int ci_dpm_resume(void *handle)
 			adev->pm.dpm_enabled = false;
 		else
 			adev->pm.dpm_enabled = true;
+		adev->pm.dpm.user_state = adev->pm.dpm.last_user_state;
+		adev->pm.dpm.state = adev->pm.dpm.last_state;
 		mutex_unlock(&adev->pm.mutex);
 		if (adev->pm.dpm_enabled)
 			amdgpu_pm_compute_clocks(adev);
@@ -6646,6 +6719,8 @@ static const struct amdgpu_dpm_funcs ci_dpm_funcs = {
 	.set_sclk_od = ci_dpm_set_sclk_od,
 	.get_mclk_od = ci_dpm_get_mclk_od,
 	.set_mclk_od = ci_dpm_set_mclk_od,
+	.check_state_equal = ci_check_state_equal,
+	.get_vce_clock_state = amdgpu_get_vce_clock_state,
 };
 
 static void ci_dpm_set_dpm_funcs(struct amdgpu_device *adev)
@@ -6664,3 +6739,12 @@ static void ci_dpm_set_irq_funcs(struct amdgpu_device *adev)
 	adev->pm.dpm.thermal.irq.num_types = AMDGPU_THERMAL_IRQ_LAST;
 	adev->pm.dpm.thermal.irq.funcs = &ci_dpm_irq_funcs;
 }
+
+const struct amdgpu_ip_block_version ci_dpm_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_SMC,
+	.major = 7,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &ci_dpm_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index a845b6a93b79..302df85893ab 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -1189,18 +1189,6 @@ static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
 	return r;
 }
 
-static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
-{
-	u32 tmp = RREG32(mmBIOS_SCRATCH_3);
-
-	if (hung)
-		tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
-	else
-		tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
-
-	WREG32(mmBIOS_SCRATCH_3, tmp);
-}
-
 /**
  * cik_asic_reset - soft reset GPU
  *
@@ -1213,11 +1201,12 @@ static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hu
 static int cik_asic_reset(struct amdgpu_device *adev)
 {
 	int r;
-	cik_set_bios_scratch_engine_hung(adev, true);
+
+	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
 
 	r = cik_gpu_pci_config_reset(adev);
 
-	cik_set_bios_scratch_engine_hung(adev, false);
+	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
 
 	return r;
 }
@@ -1641,745 +1630,6 @@ static void cik_detect_hw_virtualization(struct amdgpu_device *adev)
 		adev->virtualization.virtual_caps |= AMDGPU_PASSTHROUGH_MODE;
 }
 
-static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 8,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &dce_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 7,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &gfx_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_sdma_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 4,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &uvd_v4_2_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v2_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version bonaire_ip_blocks_vd[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 8,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &dce_virtual_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 7,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &gfx_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_sdma_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 4,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &uvd_v4_2_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v2_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 8,
-		.minor = 5,
-		.rev = 0,
-		.funcs = &dce_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 7,
-		.minor = 3,
-		.rev = 0,
-		.funcs = &gfx_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_sdma_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 4,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &uvd_v4_2_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v2_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version hawaii_ip_blocks_vd[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 8,
-		.minor = 5,
-		.rev = 0,
-		.funcs = &dce_virtual_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 7,
-		.minor = 3,
-		.rev = 0,
-		.funcs = &gfx_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_sdma_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 4,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &uvd_v4_2_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v2_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 8,
-		.minor = 3,
-		.rev = 0,
-		.funcs = &dce_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 7,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &gfx_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_sdma_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 4,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &uvd_v4_2_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v2_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version kabini_ip_blocks_vd[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 8,
-		.minor = 3,
-		.rev = 0,
-		.funcs = &dce_virtual_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 7,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &gfx_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_sdma_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 4,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &uvd_v4_2_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v2_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 8,
-		.minor = 3,
-		.rev = 0,
-		.funcs = &dce_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 7,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &gfx_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_sdma_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 4,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &uvd_v4_2_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v2_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version mullins_ip_blocks_vd[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 8,
-		.minor = 3,
-		.rev = 0,
-		.funcs = &dce_virtual_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 7,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &gfx_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_sdma_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 4,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &uvd_v4_2_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v2_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 8,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &dce_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 7,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &gfx_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_sdma_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 4,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &uvd_v4_2_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v2_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version kaveri_ip_blocks_vd[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 8,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &dce_virtual_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 7,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &gfx_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cik_sdma_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 4,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &uvd_v4_2_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v2_0_ip_funcs,
-	},
-};
-
-int cik_set_ip_blocks(struct amdgpu_device *adev)
-{
-	if (adev->enable_virtual_display) {
-		switch (adev->asic_type) {
-		case CHIP_BONAIRE:
-			adev->ip_blocks = bonaire_ip_blocks_vd;
-			adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks_vd);
-			break;
-		case CHIP_HAWAII:
-			adev->ip_blocks = hawaii_ip_blocks_vd;
-			adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks_vd);
-			break;
-		case CHIP_KAVERI:
-			adev->ip_blocks = kaveri_ip_blocks_vd;
-			adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks_vd);
-			break;
-		case CHIP_KABINI:
-			adev->ip_blocks = kabini_ip_blocks_vd;
-			adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks_vd);
-			break;
-		case CHIP_MULLINS:
-			adev->ip_blocks = mullins_ip_blocks_vd;
-			adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks_vd);
-			break;
-		default:
-			/* FIXME: not supported yet */
-			return -EINVAL;
-		}
-	} else {
-		switch (adev->asic_type) {
-		case CHIP_BONAIRE:
-			adev->ip_blocks = bonaire_ip_blocks;
-			adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
-			break;
-		case CHIP_HAWAII:
-			adev->ip_blocks = hawaii_ip_blocks;
-			adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
-			break;
-		case CHIP_KAVERI:
-			adev->ip_blocks = kaveri_ip_blocks;
-			adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
-			break;
-		case CHIP_KABINI:
-			adev->ip_blocks = kabini_ip_blocks;
-			adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
-			break;
-		case CHIP_MULLINS:
-			adev->ip_blocks = mullins_ip_blocks;
-			adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
-			break;
-		default:
-			/* FIXME: not supported yet */
-			return -EINVAL;
-		}
-	}
-
-	return 0;
-}
-
 static const struct amdgpu_asic_funcs cik_asic_funcs =
 {
 	.read_disabled_bios = &cik_read_disabled_bios,
@@ -2612,7 +1862,7 @@ static int cik_common_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs cik_common_ip_funcs = {
+static const struct amd_ip_funcs cik_common_ip_funcs = {
 	.name = "cik_common",
 	.early_init = cik_common_early_init,
 	.late_init = NULL,
@@ -2628,3 +1878,79 @@ const struct amd_ip_funcs cik_common_ip_funcs = {
 	.set_clockgating_state = cik_common_set_clockgating_state,
 	.set_powergating_state = cik_common_set_powergating_state,
 };
+
+static const struct amdgpu_ip_block_version cik_common_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_COMMON,
+	.major = 1,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &cik_common_ip_funcs,
+};
+
+int cik_set_ip_blocks(struct amdgpu_device *adev)
+{
+	switch (adev->asic_type) {
+	case CHIP_BONAIRE:
+		amdgpu_ip_block_add(adev, &cik_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v7_0_ip_block);
+		amdgpu_ip_block_add(adev, &cik_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		else
+			amdgpu_ip_block_add(adev, &dce_v8_2_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v7_2_ip_block);
+		amdgpu_ip_block_add(adev, &cik_sdma_ip_block);
+		amdgpu_ip_block_add(adev, &uvd_v4_2_ip_block);
+		amdgpu_ip_block_add(adev, &vce_v2_0_ip_block);
+		break;
+	case CHIP_HAWAII:
+		amdgpu_ip_block_add(adev, &cik_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v7_0_ip_block);
+		amdgpu_ip_block_add(adev, &cik_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		else
+			amdgpu_ip_block_add(adev, &dce_v8_5_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v7_3_ip_block);
+		amdgpu_ip_block_add(adev, &cik_sdma_ip_block);
+		amdgpu_ip_block_add(adev, &uvd_v4_2_ip_block);
+		amdgpu_ip_block_add(adev, &vce_v2_0_ip_block);
+		break;
+	case CHIP_KAVERI:
+		amdgpu_ip_block_add(adev, &cik_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v7_0_ip_block);
+		amdgpu_ip_block_add(adev, &cik_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		else
+			amdgpu_ip_block_add(adev, &dce_v8_1_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v7_1_ip_block);
+		amdgpu_ip_block_add(adev, &cik_sdma_ip_block);
+		amdgpu_ip_block_add(adev, &uvd_v4_2_ip_block);
+		amdgpu_ip_block_add(adev, &vce_v2_0_ip_block);
+		break;
+	case CHIP_KABINI:
+	case CHIP_MULLINS:
+		amdgpu_ip_block_add(adev, &cik_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v7_0_ip_block);
+		amdgpu_ip_block_add(adev, &cik_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		else
+			amdgpu_ip_block_add(adev, &dce_v8_3_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v7_2_ip_block);
+		amdgpu_ip_block_add(adev, &cik_sdma_ip_block);
+		amdgpu_ip_block_add(adev, &uvd_v4_2_ip_block);
+		amdgpu_ip_block_add(adev, &vce_v2_0_ip_block);
+		break;
+	default:
+		/* FIXME: not supported yet */
+		return -EINVAL;
+	}
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.h b/drivers/gpu/drm/amd/amdgpu/cik.h
index 5ebd2d7a0327..c4989f51ecef 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.h
+++ b/drivers/gpu/drm/amd/amdgpu/cik.h
@@ -24,8 +24,6 @@
 #ifndef __CIK_H__
 #define __CIK_H__
 
-extern const struct amd_ip_funcs cik_common_ip_funcs;
-
 void cik_srbm_select(struct amdgpu_device *adev,
 		     u32 me, u32 pipe, u32 queue, u32 vmid);
 int cik_set_ip_blocks(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
index be3d6f79a864..319b32cdea84 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
@@ -413,7 +413,7 @@ static int cik_ih_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs cik_ih_ip_funcs = {
+static const struct amd_ip_funcs cik_ih_ip_funcs = {
 	.name = "cik_ih",
 	.early_init = cik_ih_early_init,
 	.late_init = NULL,
@@ -441,3 +441,12 @@ static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev)
 	if (adev->irq.ih_funcs == NULL)
 		adev->irq.ih_funcs = &cik_ih_funcs;
 }
+
+const struct amdgpu_ip_block_version cik_ih_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_IH,
+	.major = 2,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &cik_ih_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.h b/drivers/gpu/drm/amd/amdgpu/cik_ih.h
index 6b0f375ec244..1d9ddee2868e 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.h
@@ -24,6 +24,6 @@
 #ifndef __CIK_IH_H__
 #define __CIK_IH_H__
 
-extern const struct amd_ip_funcs cik_ih_ip_funcs;
+extern const struct amdgpu_ip_block_version cik_ih_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index cb952acc7133..4c34dbc7a254 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -206,10 +206,10 @@ static void cik_sdma_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 
 	for (i = 0; i < count; i++)
 		if (sdma && sdma->burst_nop && (i == 0))
-			amdgpu_ring_write(ring, ring->nop |
+			amdgpu_ring_write(ring, ring->funcs->nop |
 					  SDMA_NOP_COUNT(count - 1));
 		else
-			amdgpu_ring_write(ring, ring->nop);
+			amdgpu_ring_write(ring, ring->funcs->nop);
 }
 
 /**
@@ -622,7 +622,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_ib ib;
-	struct fence *f = NULL;
+	struct dma_fence *f = NULL;
 	unsigned index;
 	u32 tmp = 0;
 	u64 gpu_addr;
@@ -655,7 +655,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	if (r)
 		goto err1;
 
-	r = fence_wait_timeout(f, false, timeout);
+	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
 		DRM_ERROR("amdgpu: IB test timed out\n");
 		r = -ETIMEDOUT;
@@ -675,7 +675,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 err1:
 	amdgpu_ib_free(adev, &ib, NULL);
-	fence_put(f);
+	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
 	return r;
@@ -848,22 +848,6 @@ static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */
 }
 
-static unsigned cik_sdma_ring_get_emit_ib_size(struct amdgpu_ring *ring)
-{
-	return
-		7 + 4; /* cik_sdma_ring_emit_ib */
-}
-
-static unsigned cik_sdma_ring_get_dma_frame_size(struct amdgpu_ring *ring)
-{
-	return
-		6 + /* cik_sdma_ring_emit_hdp_flush */
-		3 + /* cik_sdma_ring_emit_hdp_invalidate */
-		6 + /* cik_sdma_ring_emit_pipeline_sync */
-		12 + /* cik_sdma_ring_emit_vm_flush */
-		9 + 9 + 9; /* cik_sdma_ring_emit_fence x3 for user fence, vm fence */
-}
-
 static void cik_enable_sdma_mgcg(struct amdgpu_device *adev,
 				 bool enable)
 {
@@ -959,11 +943,10 @@ static int cik_sdma_sw_init(void *handle)
 		ring->ring_obj = NULL;
 		sprintf(ring->name, "sdma%d", i);
 		r = amdgpu_ring_init(adev, ring, 1024,
-				     SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0), 0xf,
 				     &adev->sdma.trap_irq,
 				     (i == 0) ?
-				     AMDGPU_SDMA_IRQ_TRAP0 : AMDGPU_SDMA_IRQ_TRAP1,
-				     AMDGPU_RING_TYPE_SDMA);
+				     AMDGPU_SDMA_IRQ_TRAP0 :
+				     AMDGPU_SDMA_IRQ_TRAP1);
 		if (r)
 			return r;
 	}
@@ -1207,7 +1190,7 @@ static int cik_sdma_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs cik_sdma_ip_funcs = {
+static const struct amd_ip_funcs cik_sdma_ip_funcs = {
 	.name = "cik_sdma",
 	.early_init = cik_sdma_early_init,
 	.late_init = NULL,
@@ -1225,10 +1208,19 @@ const struct amd_ip_funcs cik_sdma_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = {
+	.type = AMDGPU_RING_TYPE_SDMA,
+	.align_mask = 0xf,
+	.nop = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0),
 	.get_rptr = cik_sdma_ring_get_rptr,
 	.get_wptr = cik_sdma_ring_get_wptr,
 	.set_wptr = cik_sdma_ring_set_wptr,
-	.parse_cs = NULL,
+	.emit_frame_size =
+		6 + /* cik_sdma_ring_emit_hdp_flush */
+		3 + /* cik_sdma_ring_emit_hdp_invalidate */
+		6 + /* cik_sdma_ring_emit_pipeline_sync */
+		12 + /* cik_sdma_ring_emit_vm_flush */
+		9 + 9 + 9, /* cik_sdma_ring_emit_fence x3 for user fence, vm fence */
+	.emit_ib_size = 7 + 4, /* cik_sdma_ring_emit_ib */
 	.emit_ib = cik_sdma_ring_emit_ib,
 	.emit_fence = cik_sdma_ring_emit_fence,
 	.emit_pipeline_sync = cik_sdma_ring_emit_pipeline_sync,
@@ -1239,8 +1231,6 @@ static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = {
 	.test_ib = cik_sdma_ring_test_ib,
 	.insert_nop = cik_sdma_ring_insert_nop,
 	.pad_ib = cik_sdma_ring_pad_ib,
-	.get_emit_ib_size = cik_sdma_ring_get_emit_ib_size,
-	.get_dma_frame_size = cik_sdma_ring_get_dma_frame_size,
 };
 
 static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev)
@@ -1352,3 +1342,12 @@ static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev)
 		adev->vm_manager.vm_pte_num_rings = adev->sdma.num_instances;
 	}
 }
+
+const struct amdgpu_ip_block_version cik_sdma_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_SDMA,
+	.major = 2,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &cik_sdma_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.h b/drivers/gpu/drm/amd/amdgpu/cik_sdma.h
index 027727c677b8..a4a8fe01410b 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.h
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.h
@@ -24,6 +24,6 @@
 #ifndef __CIK_SDMA_H__
 #define __CIK_SDMA_H__
 
-extern const struct amd_ip_funcs cik_sdma_ip_funcs;
+extern const struct amdgpu_ip_block_version cik_sdma_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/cikd.h b/drivers/gpu/drm/amd/amdgpu/cikd.h
index 8659852aea9e..6cbd913fd12e 100644
--- a/drivers/gpu/drm/amd/amdgpu/cikd.h
+++ b/drivers/gpu/drm/amd/amdgpu/cikd.h
@@ -43,6 +43,14 @@
 #define CRTC4_REGISTER_OFFSET                 (0x477c - 0x1b7c)
 #define CRTC5_REGISTER_OFFSET                 (0x4a7c - 0x1b7c)
 
+/* hpd instance offsets */
+#define HPD0_REGISTER_OFFSET                 (0x1807 - 0x1807)
+#define HPD1_REGISTER_OFFSET                 (0x180a - 0x1807)
+#define HPD2_REGISTER_OFFSET                 (0x180d - 0x1807)
+#define HPD3_REGISTER_OFFSET                 (0x1810 - 0x1807)
+#define HPD4_REGISTER_OFFSET                 (0x1813 - 0x1807)
+#define HPD5_REGISTER_OFFSET                 (0x1816 - 0x1807)
+
 #define BONAIRE_GB_ADDR_CONFIG_GOLDEN        0x12010001
 #define HAWAII_GB_ADDR_CONFIG_GOLDEN         0x12011003
 
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
index 3c082e143730..ba2b66be9022 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
@@ -438,7 +438,7 @@ static int cz_dpm_init(struct amdgpu_device *adev)
 		pi->caps_td_ramping = true;
 		pi->caps_tcp_ramping = true;
 	}
-	if (amdgpu_sclk_deep_sleep_en)
+	if (amdgpu_pp_feature_mask & SCLK_DEEP_SLEEP_MASK)
 		pi->caps_sclk_ds = true;
 	else
 		pi->caps_sclk_ds = false;
@@ -1250,7 +1250,8 @@ static void cz_update_current_ps(struct amdgpu_device *adev,
 
 	pi->current_ps = *ps;
 	pi->current_rps = *rps;
-	pi->current_rps.ps_priv = ps;
+	pi->current_rps.ps_priv = &pi->current_ps;
+	adev->pm.dpm.current_ps = &pi->current_rps;
 
 }
 
@@ -1262,7 +1263,8 @@ static void cz_update_requested_ps(struct amdgpu_device *adev,
 
 	pi->requested_ps = *ps;
 	pi->requested_rps = *rps;
-	pi->requested_rps.ps_priv = ps;
+	pi->requested_rps.ps_priv = &pi->requested_ps;
+	adev->pm.dpm.requested_ps = &pi->requested_rps;
 
 }
 
@@ -2109,9 +2111,8 @@ static void cz_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
 
 	if (gate) {
 		if (pi->caps_uvd_pg) {
-			/* disable clockgating so we can properly shut down the block */
 			ret = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-							    AMD_CG_STATE_UNGATE);
+							    AMD_CG_STATE_GATE);
 			if (ret) {
 				DRM_ERROR("UVD DPM Power Gating failed to set clockgating state\n");
 				return;
@@ -2157,9 +2158,8 @@ static void cz_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
 				return;
 			}
 
-			/* enable clockgating. hw will dynamically gate/ungate clocks on the fly */
 			ret = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-							    AMD_CG_STATE_GATE);
+							    AMD_CG_STATE_UNGATE);
 			if (ret) {
 				DRM_ERROR("UVD DPM Power Gating Failed to set clockgating state\n");
 				return;
@@ -2257,6 +2257,18 @@ static void cz_dpm_powergate_vce(struct amdgpu_device *adev, bool gate)
 	}
 }
 
+static int cz_check_state_equal(struct amdgpu_device *adev,
+				struct amdgpu_ps *cps,
+				struct amdgpu_ps *rps,
+				bool *equal)
+{
+	if (equal == NULL)
+		return -EINVAL;
+
+	*equal = false;
+	return 0;
+}
+
 const struct amd_ip_funcs cz_dpm_ip_funcs = {
 	.name = "cz_dpm",
 	.early_init = cz_dpm_early_init,
@@ -2289,6 +2301,7 @@ static const struct amdgpu_dpm_funcs cz_dpm_funcs = {
 	.vblank_too_short = NULL,
 	.powergate_uvd = cz_dpm_powergate_uvd,
 	.powergate_vce = cz_dpm_powergate_vce,
+	.check_state_equal = cz_check_state_equal,
 };
 
 static void cz_dpm_set_funcs(struct amdgpu_device *adev)
@@ -2296,3 +2309,12 @@ static void cz_dpm_set_funcs(struct amdgpu_device *adev)
 	if (NULL == adev->pm.funcs)
 		adev->pm.funcs = &cz_dpm_funcs;
 }
+
+const struct amdgpu_ip_block_version cz_dpm_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_SMC,
+	.major = 8,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &cz_dpm_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
index 3d23a70b6432..fe7cbb24da7b 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
@@ -394,7 +394,7 @@ static int cz_ih_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs cz_ih_ip_funcs = {
+static const struct amd_ip_funcs cz_ih_ip_funcs = {
 	.name = "cz_ih",
 	.early_init = cz_ih_early_init,
 	.late_init = NULL,
@@ -423,3 +423,11 @@ static void cz_ih_set_interrupt_funcs(struct amdgpu_device *adev)
 		adev->irq.ih_funcs = &cz_ih_funcs;
 }
 
+const struct amdgpu_ip_block_version cz_ih_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_IH,
+	.major = 3,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &cz_ih_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.h b/drivers/gpu/drm/amd/amdgpu/cz_ih.h
index fc4057a2ecb9..14be7753221b 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.h
@@ -24,6 +24,6 @@
 #ifndef __CZ_IH_H__
 #define __CZ_IH_H__
 
-extern const struct amd_ip_funcs cz_ih_ip_funcs;
+extern const struct amdgpu_ip_block_version cz_ih_ip_block;
 
 #endif /* __CZ_IH_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index 9260caef74fa..9999dc71b998 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -31,6 +31,7 @@
 #include "atombios_encoders.h"
 #include "amdgpu_pll.h"
 #include "amdgpu_connectors.h"
+#include "dce_v10_0.h"
 
 #include "dce/dce_10_0_d.h"
 #include "dce/dce_10_0_sh_mask.h"
@@ -330,33 +331,12 @@ static int dce_v10_0_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc,
 static bool dce_v10_0_hpd_sense(struct amdgpu_device *adev,
 			       enum amdgpu_hpd_id hpd)
 {
-	int idx;
 	bool connected = false;
 
-	switch (hpd) {
-	case AMDGPU_HPD_1:
-		idx = 0;
-		break;
-	case AMDGPU_HPD_2:
-		idx = 1;
-		break;
-	case AMDGPU_HPD_3:
-		idx = 2;
-		break;
-	case AMDGPU_HPD_4:
-		idx = 3;
-		break;
-	case AMDGPU_HPD_5:
-		idx = 4;
-		break;
-	case AMDGPU_HPD_6:
-		idx = 5;
-		break;
-	default:
+	if (hpd >= adev->mode_info.num_hpd)
 		return connected;
-	}
 
-	if (RREG32(mmDC_HPD_INT_STATUS + hpd_offsets[idx]) &
+	if (RREG32(mmDC_HPD_INT_STATUS + hpd_offsets[hpd]) &
 	    DC_HPD_INT_STATUS__DC_HPD_SENSE_MASK)
 		connected = true;
 
@@ -376,37 +356,16 @@ static void dce_v10_0_hpd_set_polarity(struct amdgpu_device *adev,
 {
 	u32 tmp;
 	bool connected = dce_v10_0_hpd_sense(adev, hpd);
-	int idx;
 
-	switch (hpd) {
-	case AMDGPU_HPD_1:
-		idx = 0;
-		break;
-	case AMDGPU_HPD_2:
-		idx = 1;
-		break;
-	case AMDGPU_HPD_3:
-		idx = 2;
-		break;
-	case AMDGPU_HPD_4:
-		idx = 3;
-		break;
-	case AMDGPU_HPD_5:
-		idx = 4;
-		break;
-	case AMDGPU_HPD_6:
-		idx = 5;
-		break;
-	default:
+	if (hpd >= adev->mode_info.num_hpd)
 		return;
-	}
 
-	tmp = RREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx]);
+	tmp = RREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[hpd]);
 	if (connected)
 		tmp = REG_SET_FIELD(tmp, DC_HPD_INT_CONTROL, DC_HPD_INT_POLARITY, 0);
 	else
 		tmp = REG_SET_FIELD(tmp, DC_HPD_INT_CONTROL, DC_HPD_INT_POLARITY, 1);
-	WREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx], tmp);
+	WREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[hpd], tmp);
 }
 
 /**
@@ -422,33 +381,12 @@ static void dce_v10_0_hpd_init(struct amdgpu_device *adev)
 	struct drm_device *dev = adev->ddev;
 	struct drm_connector *connector;
 	u32 tmp;
-	int idx;
 
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-		switch (amdgpu_connector->hpd.hpd) {
-		case AMDGPU_HPD_1:
-			idx = 0;
-			break;
-		case AMDGPU_HPD_2:
-			idx = 1;
-			break;
-		case AMDGPU_HPD_3:
-			idx = 2;
-			break;
-		case AMDGPU_HPD_4:
-			idx = 3;
-			break;
-		case AMDGPU_HPD_5:
-			idx = 4;
-			break;
-		case AMDGPU_HPD_6:
-			idx = 5;
-			break;
-		default:
+		if (amdgpu_connector->hpd.hpd >= adev->mode_info.num_hpd)
 			continue;
-		}
 
 		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
 		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
@@ -457,24 +395,24 @@ static void dce_v10_0_hpd_init(struct amdgpu_device *adev)
 			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
 			 * also avoid interrupt storms during dpms.
 			 */
-			tmp = RREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx]);
+			tmp = RREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd]);
 			tmp = REG_SET_FIELD(tmp, DC_HPD_INT_CONTROL, DC_HPD_INT_EN, 0);
-			WREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx], tmp);
+			WREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd], tmp);
 			continue;
 		}
 
-		tmp = RREG32(mmDC_HPD_CONTROL + hpd_offsets[idx]);
+		tmp = RREG32(mmDC_HPD_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd]);
 		tmp = REG_SET_FIELD(tmp, DC_HPD_CONTROL, DC_HPD_EN, 1);
-		WREG32(mmDC_HPD_CONTROL + hpd_offsets[idx], tmp);
+		WREG32(mmDC_HPD_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd], tmp);
 
-		tmp = RREG32(mmDC_HPD_TOGGLE_FILT_CNTL + hpd_offsets[idx]);
+		tmp = RREG32(mmDC_HPD_TOGGLE_FILT_CNTL + hpd_offsets[amdgpu_connector->hpd.hpd]);
 		tmp = REG_SET_FIELD(tmp, DC_HPD_TOGGLE_FILT_CNTL,
 				    DC_HPD_CONNECT_INT_DELAY,
 				    AMDGPU_HPD_CONNECT_INT_DELAY_IN_MS);
 		tmp = REG_SET_FIELD(tmp, DC_HPD_TOGGLE_FILT_CNTL,
 				    DC_HPD_DISCONNECT_INT_DELAY,
 				    AMDGPU_HPD_DISCONNECT_INT_DELAY_IN_MS);
-		WREG32(mmDC_HPD_TOGGLE_FILT_CNTL + hpd_offsets[idx], tmp);
+		WREG32(mmDC_HPD_TOGGLE_FILT_CNTL + hpd_offsets[amdgpu_connector->hpd.hpd], tmp);
 
 		dce_v10_0_hpd_set_polarity(adev, amdgpu_connector->hpd.hpd);
 		amdgpu_irq_get(adev, &adev->hpd_irq,
@@ -495,37 +433,16 @@ static void dce_v10_0_hpd_fini(struct amdgpu_device *adev)
 	struct drm_device *dev = adev->ddev;
 	struct drm_connector *connector;
 	u32 tmp;
-	int idx;
 
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-		switch (amdgpu_connector->hpd.hpd) {
-		case AMDGPU_HPD_1:
-			idx = 0;
-			break;
-		case AMDGPU_HPD_2:
-			idx = 1;
-			break;
-		case AMDGPU_HPD_3:
-			idx = 2;
-			break;
-		case AMDGPU_HPD_4:
-			idx = 3;
-			break;
-		case AMDGPU_HPD_5:
-			idx = 4;
-			break;
-		case AMDGPU_HPD_6:
-			idx = 5;
-			break;
-		default:
+		if (amdgpu_connector->hpd.hpd >= adev->mode_info.num_hpd)
 			continue;
-		}
 
-		tmp = RREG32(mmDC_HPD_CONTROL + hpd_offsets[idx]);
+		tmp = RREG32(mmDC_HPD_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd]);
 		tmp = REG_SET_FIELD(tmp, DC_HPD_CONTROL, DC_HPD_EN, 0);
-		WREG32(mmDC_HPD_CONTROL + hpd_offsets[idx], tmp);
+		WREG32(mmDC_HPD_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd], tmp);
 
 		amdgpu_irq_put(adev, &adev->hpd_irq,
 			       amdgpu_connector->hpd.hpd);
@@ -2115,7 +2032,7 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 	u32 tmp, viewport_w, viewport_h;
 	int r;
 	bool bypass_lut = false;
-	char *format_name;
+	struct drm_format_name_buf format_name;
 
 	/* no fb bound */
 	if (!atomic && !crtc->primary->fb) {
@@ -2227,9 +2144,8 @@ static int dce_v10_0_crtc_do_set_base(struct drm_crtc *crtc,
 		bypass_lut = true;
 		break;
 	default:
-		format_name = drm_get_format_name(target_fb->pixel_format);
-		DRM_ERROR("Unsupported screen format %s\n", format_name);
-		kfree(format_name);
+		DRM_ERROR("Unsupported screen format %s\n",
+		          drm_get_format_name(target_fb->pixel_format, &format_name));
 		return -EINVAL;
 	}
 
@@ -2577,6 +2493,9 @@ static int dce_v10_0_cursor_move_locked(struct drm_crtc *crtc,
 	struct amdgpu_device *adev = crtc->dev->dev_private;
 	int xorigin = 0, yorigin = 0;
 
+	amdgpu_crtc->cursor_x = x;
+	amdgpu_crtc->cursor_y = y;
+
 	/* avivo cursor are offset into the total surface */
 	x += crtc->x;
 	y += crtc->y;
@@ -2593,11 +2512,6 @@ static int dce_v10_0_cursor_move_locked(struct drm_crtc *crtc,
 
 	WREG32(mmCUR_POSITION + amdgpu_crtc->crtc_offset, (x << 16) | y);
 	WREG32(mmCUR_HOT_SPOT + amdgpu_crtc->crtc_offset, (xorigin << 16) | yorigin);
-	WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
-	       ((amdgpu_crtc->cursor_width - 1) << 16) | (amdgpu_crtc->cursor_height - 1));
-
-	amdgpu_crtc->cursor_x = x;
-	amdgpu_crtc->cursor_y = y;
 
 	return 0;
 }
@@ -2623,6 +2537,7 @@ static int dce_v10_0_crtc_cursor_set2(struct drm_crtc *crtc,
 				      int32_t hot_y)
 {
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
 	struct drm_gem_object *obj;
 	struct amdgpu_bo *aobj;
 	int ret;
@@ -2661,9 +2576,6 @@ static int dce_v10_0_crtc_cursor_set2(struct drm_crtc *crtc,
 		return ret;
 	}
 
-	amdgpu_crtc->cursor_width = width;
-	amdgpu_crtc->cursor_height = height;
-
 	dce_v10_0_lock_cursor(crtc, true);
 
 	if (hot_x != amdgpu_crtc->cursor_hot_x ||
@@ -2679,6 +2591,14 @@ static int dce_v10_0_crtc_cursor_set2(struct drm_crtc *crtc,
 		amdgpu_crtc->cursor_hot_y = hot_y;
 	}
 
+	if (width != amdgpu_crtc->cursor_width ||
+	    height != amdgpu_crtc->cursor_height) {
+		WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
+		       (width - 1) << 16 | (height - 1));
+		amdgpu_crtc->cursor_width = width;
+		amdgpu_crtc->cursor_height = height;
+	}
+
 	dce_v10_0_show_cursor(crtc);
 	dce_v10_0_lock_cursor(crtc, false);
 
@@ -2700,6 +2620,7 @@ unpin:
 static void dce_v10_0_cursor_reset(struct drm_crtc *crtc)
 {
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
 
 	if (amdgpu_crtc->cursor_bo) {
 		dce_v10_0_lock_cursor(crtc, true);
@@ -2707,6 +2628,10 @@ static void dce_v10_0_cursor_reset(struct drm_crtc *crtc)
 		dce_v10_0_cursor_move_locked(crtc, amdgpu_crtc->cursor_x,
 					     amdgpu_crtc->cursor_y);
 
+		WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
+		       (amdgpu_crtc->cursor_width - 1) << 16 |
+		       (amdgpu_crtc->cursor_height - 1));
+
 		dce_v10_0_show_cursor(crtc);
 
 		dce_v10_0_lock_cursor(crtc, false);
@@ -3548,7 +3473,7 @@ static int dce_v10_0_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs dce_v10_0_ip_funcs = {
+static const struct amd_ip_funcs dce_v10_0_ip_funcs = {
 	.name = "dce_v10_0",
 	.early_init = dce_v10_0_early_init,
 	.late_init = NULL,
@@ -3833,7 +3758,6 @@ static const struct amdgpu_display_funcs dce_v10_0_display_funcs = {
 	.bandwidth_update = &dce_v10_0_bandwidth_update,
 	.vblank_get_counter = &dce_v10_0_vblank_get_counter,
 	.vblank_wait = &dce_v10_0_vblank_wait,
-	.is_display_hung = &dce_v10_0_is_display_hung,
 	.backlight_set_level = &amdgpu_atombios_encoder_set_backlight_level,
 	.backlight_get_level = &amdgpu_atombios_encoder_get_backlight_level,
 	.hpd_sense = &dce_v10_0_hpd_sense,
@@ -3879,3 +3803,21 @@ static void dce_v10_0_set_irq_funcs(struct amdgpu_device *adev)
 	adev->hpd_irq.num_types = AMDGPU_HPD_LAST;
 	adev->hpd_irq.funcs = &dce_v10_0_hpd_irq_funcs;
 }
+
+const struct amdgpu_ip_block_version dce_v10_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 10,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &dce_v10_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version dce_v10_1_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 10,
+	.minor = 1,
+	.rev = 0,
+	.funcs = &dce_v10_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.h b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.h
index e3dc04d293e4..7a0747789f1d 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.h
@@ -24,7 +24,9 @@
 #ifndef __DCE_V10_0_H__
 #define __DCE_V10_0_H__
 
-extern const struct amd_ip_funcs dce_v10_0_ip_funcs;
+
+extern const struct amdgpu_ip_block_version dce_v10_0_ip_block;
+extern const struct amdgpu_ip_block_version dce_v10_1_ip_block;
 
 void dce_v10_0_disable_dce(struct amdgpu_device *adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index 367739bd1927..b3d62b909f43 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -31,6 +31,7 @@
 #include "atombios_encoders.h"
 #include "amdgpu_pll.h"
 #include "amdgpu_connectors.h"
+#include "dce_v11_0.h"
 
 #include "dce/dce_11_0_d.h"
 #include "dce/dce_11_0_sh_mask.h"
@@ -346,33 +347,12 @@ static int dce_v11_0_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc,
 static bool dce_v11_0_hpd_sense(struct amdgpu_device *adev,
 			       enum amdgpu_hpd_id hpd)
 {
-	int idx;
 	bool connected = false;
 
-	switch (hpd) {
-	case AMDGPU_HPD_1:
-		idx = 0;
-		break;
-	case AMDGPU_HPD_2:
-		idx = 1;
-		break;
-	case AMDGPU_HPD_3:
-		idx = 2;
-		break;
-	case AMDGPU_HPD_4:
-		idx = 3;
-		break;
-	case AMDGPU_HPD_5:
-		idx = 4;
-		break;
-	case AMDGPU_HPD_6:
-		idx = 5;
-		break;
-	default:
+	if (hpd >= adev->mode_info.num_hpd)
 		return connected;
-	}
 
-	if (RREG32(mmDC_HPD_INT_STATUS + hpd_offsets[idx]) &
+	if (RREG32(mmDC_HPD_INT_STATUS + hpd_offsets[hpd]) &
 	    DC_HPD_INT_STATUS__DC_HPD_SENSE_MASK)
 		connected = true;
 
@@ -392,37 +372,16 @@ static void dce_v11_0_hpd_set_polarity(struct amdgpu_device *adev,
 {
 	u32 tmp;
 	bool connected = dce_v11_0_hpd_sense(adev, hpd);
-	int idx;
 
-	switch (hpd) {
-	case AMDGPU_HPD_1:
-		idx = 0;
-		break;
-	case AMDGPU_HPD_2:
-		idx = 1;
-		break;
-	case AMDGPU_HPD_3:
-		idx = 2;
-		break;
-	case AMDGPU_HPD_4:
-		idx = 3;
-		break;
-	case AMDGPU_HPD_5:
-		idx = 4;
-		break;
-	case AMDGPU_HPD_6:
-		idx = 5;
-		break;
-	default:
+	if (hpd >= adev->mode_info.num_hpd)
 		return;
-	}
 
-	tmp = RREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx]);
+	tmp = RREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[hpd]);
 	if (connected)
 		tmp = REG_SET_FIELD(tmp, DC_HPD_INT_CONTROL, DC_HPD_INT_POLARITY, 0);
 	else
 		tmp = REG_SET_FIELD(tmp, DC_HPD_INT_CONTROL, DC_HPD_INT_POLARITY, 1);
-	WREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx], tmp);
+	WREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[hpd], tmp);
 }
 
 /**
@@ -438,33 +397,12 @@ static void dce_v11_0_hpd_init(struct amdgpu_device *adev)
 	struct drm_device *dev = adev->ddev;
 	struct drm_connector *connector;
 	u32 tmp;
-	int idx;
 
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-		switch (amdgpu_connector->hpd.hpd) {
-		case AMDGPU_HPD_1:
-			idx = 0;
-			break;
-		case AMDGPU_HPD_2:
-			idx = 1;
-			break;
-		case AMDGPU_HPD_3:
-			idx = 2;
-			break;
-		case AMDGPU_HPD_4:
-			idx = 3;
-			break;
-		case AMDGPU_HPD_5:
-			idx = 4;
-			break;
-		case AMDGPU_HPD_6:
-			idx = 5;
-			break;
-		default:
+		if (amdgpu_connector->hpd.hpd >= adev->mode_info.num_hpd)
 			continue;
-		}
 
 		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
 		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
@@ -473,24 +411,24 @@ static void dce_v11_0_hpd_init(struct amdgpu_device *adev)
 			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
 			 * also avoid interrupt storms during dpms.
 			 */
-			tmp = RREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx]);
+			tmp = RREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd]);
 			tmp = REG_SET_FIELD(tmp, DC_HPD_INT_CONTROL, DC_HPD_INT_EN, 0);
-			WREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[idx], tmp);
+			WREG32(mmDC_HPD_INT_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd], tmp);
 			continue;
 		}
 
-		tmp = RREG32(mmDC_HPD_CONTROL + hpd_offsets[idx]);
+		tmp = RREG32(mmDC_HPD_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd]);
 		tmp = REG_SET_FIELD(tmp, DC_HPD_CONTROL, DC_HPD_EN, 1);
-		WREG32(mmDC_HPD_CONTROL + hpd_offsets[idx], tmp);
+		WREG32(mmDC_HPD_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd], tmp);
 
-		tmp = RREG32(mmDC_HPD_TOGGLE_FILT_CNTL + hpd_offsets[idx]);
+		tmp = RREG32(mmDC_HPD_TOGGLE_FILT_CNTL + hpd_offsets[amdgpu_connector->hpd.hpd]);
 		tmp = REG_SET_FIELD(tmp, DC_HPD_TOGGLE_FILT_CNTL,
 				    DC_HPD_CONNECT_INT_DELAY,
 				    AMDGPU_HPD_CONNECT_INT_DELAY_IN_MS);
 		tmp = REG_SET_FIELD(tmp, DC_HPD_TOGGLE_FILT_CNTL,
 				    DC_HPD_DISCONNECT_INT_DELAY,
 				    AMDGPU_HPD_DISCONNECT_INT_DELAY_IN_MS);
-		WREG32(mmDC_HPD_TOGGLE_FILT_CNTL + hpd_offsets[idx], tmp);
+		WREG32(mmDC_HPD_TOGGLE_FILT_CNTL + hpd_offsets[amdgpu_connector->hpd.hpd], tmp);
 
 		dce_v11_0_hpd_set_polarity(adev, amdgpu_connector->hpd.hpd);
 		amdgpu_irq_get(adev, &adev->hpd_irq, amdgpu_connector->hpd.hpd);
@@ -510,37 +448,16 @@ static void dce_v11_0_hpd_fini(struct amdgpu_device *adev)
 	struct drm_device *dev = adev->ddev;
 	struct drm_connector *connector;
 	u32 tmp;
-	int idx;
 
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-		switch (amdgpu_connector->hpd.hpd) {
-		case AMDGPU_HPD_1:
-			idx = 0;
-			break;
-		case AMDGPU_HPD_2:
-			idx = 1;
-			break;
-		case AMDGPU_HPD_3:
-			idx = 2;
-			break;
-		case AMDGPU_HPD_4:
-			idx = 3;
-			break;
-		case AMDGPU_HPD_5:
-			idx = 4;
-			break;
-		case AMDGPU_HPD_6:
-			idx = 5;
-			break;
-		default:
+		if (amdgpu_connector->hpd.hpd >= adev->mode_info.num_hpd)
 			continue;
-		}
 
-		tmp = RREG32(mmDC_HPD_CONTROL + hpd_offsets[idx]);
+		tmp = RREG32(mmDC_HPD_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd]);
 		tmp = REG_SET_FIELD(tmp, DC_HPD_CONTROL, DC_HPD_EN, 0);
-		WREG32(mmDC_HPD_CONTROL + hpd_offsets[idx], tmp);
+		WREG32(mmDC_HPD_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd], tmp);
 
 		amdgpu_irq_put(adev, &adev->hpd_irq, amdgpu_connector->hpd.hpd);
 	}
@@ -2096,7 +2013,7 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
 	u32 tmp, viewport_w, viewport_h;
 	int r;
 	bool bypass_lut = false;
-	char *format_name;
+	struct drm_format_name_buf format_name;
 
 	/* no fb bound */
 	if (!atomic && !crtc->primary->fb) {
@@ -2208,9 +2125,8 @@ static int dce_v11_0_crtc_do_set_base(struct drm_crtc *crtc,
 		bypass_lut = true;
 		break;
 	default:
-		format_name = drm_get_format_name(target_fb->pixel_format);
-		DRM_ERROR("Unsupported screen format %s\n", format_name);
-		kfree(format_name);
+		DRM_ERROR("Unsupported screen format %s\n",
+		          drm_get_format_name(target_fb->pixel_format, &format_name));
 		return -EINVAL;
 	}
 
@@ -2593,6 +2509,9 @@ static int dce_v11_0_cursor_move_locked(struct drm_crtc *crtc,
 	struct amdgpu_device *adev = crtc->dev->dev_private;
 	int xorigin = 0, yorigin = 0;
 
+	amdgpu_crtc->cursor_x = x;
+	amdgpu_crtc->cursor_y = y;
+
 	/* avivo cursor are offset into the total surface */
 	x += crtc->x;
 	y += crtc->y;
@@ -2609,11 +2528,6 @@ static int dce_v11_0_cursor_move_locked(struct drm_crtc *crtc,
 
 	WREG32(mmCUR_POSITION + amdgpu_crtc->crtc_offset, (x << 16) | y);
 	WREG32(mmCUR_HOT_SPOT + amdgpu_crtc->crtc_offset, (xorigin << 16) | yorigin);
-	WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
-	       ((amdgpu_crtc->cursor_width - 1) << 16) | (amdgpu_crtc->cursor_height - 1));
-
-	amdgpu_crtc->cursor_x = x;
-	amdgpu_crtc->cursor_y = y;
 
 	return 0;
 }
@@ -2639,6 +2553,7 @@ static int dce_v11_0_crtc_cursor_set2(struct drm_crtc *crtc,
 				      int32_t hot_y)
 {
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
 	struct drm_gem_object *obj;
 	struct amdgpu_bo *aobj;
 	int ret;
@@ -2677,9 +2592,6 @@ static int dce_v11_0_crtc_cursor_set2(struct drm_crtc *crtc,
 		return ret;
 	}
 
-	amdgpu_crtc->cursor_width = width;
-	amdgpu_crtc->cursor_height = height;
-
 	dce_v11_0_lock_cursor(crtc, true);
 
 	if (hot_x != amdgpu_crtc->cursor_hot_x ||
@@ -2695,6 +2607,14 @@ static int dce_v11_0_crtc_cursor_set2(struct drm_crtc *crtc,
 		amdgpu_crtc->cursor_hot_y = hot_y;
 	}
 
+	if (width != amdgpu_crtc->cursor_width ||
+	    height != amdgpu_crtc->cursor_height) {
+		WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
+		       (width - 1) << 16 | (height - 1));
+		amdgpu_crtc->cursor_width = width;
+		amdgpu_crtc->cursor_height = height;
+	}
+
 	dce_v11_0_show_cursor(crtc);
 	dce_v11_0_lock_cursor(crtc, false);
 
@@ -2716,6 +2636,7 @@ unpin:
 static void dce_v11_0_cursor_reset(struct drm_crtc *crtc)
 {
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
 
 	if (amdgpu_crtc->cursor_bo) {
 		dce_v11_0_lock_cursor(crtc, true);
@@ -2723,6 +2644,10 @@ static void dce_v11_0_cursor_reset(struct drm_crtc *crtc)
 		dce_v11_0_cursor_move_locked(crtc, amdgpu_crtc->cursor_x,
 					     amdgpu_crtc->cursor_y);
 
+		WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
+		       (amdgpu_crtc->cursor_width - 1) << 16 |
+		       (amdgpu_crtc->cursor_height - 1));
+
 		dce_v11_0_show_cursor(crtc);
 
 		dce_v11_0_lock_cursor(crtc, false);
@@ -3605,7 +3530,7 @@ static int dce_v11_0_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs dce_v11_0_ip_funcs = {
+static const struct amd_ip_funcs dce_v11_0_ip_funcs = {
 	.name = "dce_v11_0",
 	.early_init = dce_v11_0_early_init,
 	.late_init = NULL,
@@ -3889,7 +3814,6 @@ static const struct amdgpu_display_funcs dce_v11_0_display_funcs = {
 	.bandwidth_update = &dce_v11_0_bandwidth_update,
 	.vblank_get_counter = &dce_v11_0_vblank_get_counter,
 	.vblank_wait = &dce_v11_0_vblank_wait,
-	.is_display_hung = &dce_v11_0_is_display_hung,
 	.backlight_set_level = &amdgpu_atombios_encoder_set_backlight_level,
 	.backlight_get_level = &amdgpu_atombios_encoder_get_backlight_level,
 	.hpd_sense = &dce_v11_0_hpd_sense,
@@ -3935,3 +3859,21 @@ static void dce_v11_0_set_irq_funcs(struct amdgpu_device *adev)
 	adev->hpd_irq.num_types = AMDGPU_HPD_LAST;
 	adev->hpd_irq.funcs = &dce_v11_0_hpd_irq_funcs;
 }
+
+const struct amdgpu_ip_block_version dce_v11_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 11,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &dce_v11_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version dce_v11_2_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 11,
+	.minor = 2,
+	.rev = 0,
+	.funcs = &dce_v11_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.h b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.h
index 1f58a65ba2ef..0d878ca3acba 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.h
@@ -24,7 +24,8 @@
 #ifndef __DCE_V11_0_H__
 #define __DCE_V11_0_H__
 
-extern const struct amd_ip_funcs dce_v11_0_ip_funcs;
+extern const struct amdgpu_ip_block_version dce_v11_0_ip_block;
+extern const struct amdgpu_ip_block_version dce_v11_2_ip_block;
 
 void dce_v11_0_disable_dce(struct amdgpu_device *adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index 15f9fc0514b2..e564442b6393 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -30,8 +30,19 @@
 #include "atombios_encoders.h"
 #include "amdgpu_pll.h"
 #include "amdgpu_connectors.h"
-#include "si/si_reg.h"
-#include "si/sid.h"
+
+#include "bif/bif_3_0_d.h"
+#include "bif/bif_3_0_sh_mask.h"
+#include "oss/oss_1_0_d.h"
+#include "oss/oss_1_0_sh_mask.h"
+#include "gca/gfx_6_0_d.h"
+#include "gca/gfx_6_0_sh_mask.h"
+#include "gmc/gmc_6_0_d.h"
+#include "gmc/gmc_6_0_sh_mask.h"
+#include "dce/dce_6_0_d.h"
+#include "dce/dce_6_0_sh_mask.h"
+#include "gca/gfx_7_2_enum.h"
+#include "si_enums.h"
 
 static void dce_v6_0_set_display_funcs(struct amdgpu_device *adev);
 static void dce_v6_0_set_irq_funcs(struct amdgpu_device *adev);
@@ -46,6 +57,16 @@ static const u32 crtc_offsets[6] =
 	SI_CRTC5_REGISTER_OFFSET
 };
 
+static const u32 hpd_offsets[] =
+{
+	mmDC_HPD1_INT_STATUS - mmDC_HPD1_INT_STATUS,
+	mmDC_HPD2_INT_STATUS - mmDC_HPD1_INT_STATUS,
+	mmDC_HPD3_INT_STATUS - mmDC_HPD1_INT_STATUS,
+	mmDC_HPD4_INT_STATUS - mmDC_HPD1_INT_STATUS,
+	mmDC_HPD5_INT_STATUS - mmDC_HPD1_INT_STATUS,
+	mmDC_HPD6_INT_STATUS - mmDC_HPD1_INT_STATUS,
+};
+
 static const uint32_t dig_offsets[] = {
 	SI_CRTC0_REGISTER_OFFSET,
 	SI_CRTC1_REGISTER_OFFSET,
@@ -63,46 +84,37 @@ static const struct {
 	uint32_t	hpd;
 
 } interrupt_status_offsets[6] = { {
-	.reg = DISP_INTERRUPT_STATUS,
+	.reg = mmDISP_INTERRUPT_STATUS,
 	.vblank = DISP_INTERRUPT_STATUS__LB_D1_VBLANK_INTERRUPT_MASK,
 	.vline = DISP_INTERRUPT_STATUS__LB_D1_VLINE_INTERRUPT_MASK,
 	.hpd = DISP_INTERRUPT_STATUS__DC_HPD1_INTERRUPT_MASK
 }, {
-	.reg = DISP_INTERRUPT_STATUS_CONTINUE,
+	.reg = mmDISP_INTERRUPT_STATUS_CONTINUE,
 	.vblank = DISP_INTERRUPT_STATUS_CONTINUE__LB_D2_VBLANK_INTERRUPT_MASK,
 	.vline = DISP_INTERRUPT_STATUS_CONTINUE__LB_D2_VLINE_INTERRUPT_MASK,
 	.hpd = DISP_INTERRUPT_STATUS_CONTINUE__DC_HPD2_INTERRUPT_MASK
 }, {
-	.reg = DISP_INTERRUPT_STATUS_CONTINUE2,
+	.reg = mmDISP_INTERRUPT_STATUS_CONTINUE2,
 	.vblank = DISP_INTERRUPT_STATUS_CONTINUE2__LB_D3_VBLANK_INTERRUPT_MASK,
 	.vline = DISP_INTERRUPT_STATUS_CONTINUE2__LB_D3_VLINE_INTERRUPT_MASK,
 	.hpd = DISP_INTERRUPT_STATUS_CONTINUE2__DC_HPD3_INTERRUPT_MASK
 }, {
-	.reg = DISP_INTERRUPT_STATUS_CONTINUE3,
+	.reg = mmDISP_INTERRUPT_STATUS_CONTINUE3,
 	.vblank = DISP_INTERRUPT_STATUS_CONTINUE3__LB_D4_VBLANK_INTERRUPT_MASK,
 	.vline = DISP_INTERRUPT_STATUS_CONTINUE3__LB_D4_VLINE_INTERRUPT_MASK,
 	.hpd = DISP_INTERRUPT_STATUS_CONTINUE3__DC_HPD4_INTERRUPT_MASK
 }, {
-	.reg = DISP_INTERRUPT_STATUS_CONTINUE4,
+	.reg = mmDISP_INTERRUPT_STATUS_CONTINUE4,
 	.vblank = DISP_INTERRUPT_STATUS_CONTINUE4__LB_D5_VBLANK_INTERRUPT_MASK,
 	.vline = DISP_INTERRUPT_STATUS_CONTINUE4__LB_D5_VLINE_INTERRUPT_MASK,
 	.hpd = DISP_INTERRUPT_STATUS_CONTINUE4__DC_HPD5_INTERRUPT_MASK
 }, {
-	.reg = DISP_INTERRUPT_STATUS_CONTINUE5,
+	.reg = mmDISP_INTERRUPT_STATUS_CONTINUE5,
 	.vblank = DISP_INTERRUPT_STATUS_CONTINUE5__LB_D6_VBLANK_INTERRUPT_MASK,
 	.vline = DISP_INTERRUPT_STATUS_CONTINUE5__LB_D6_VLINE_INTERRUPT_MASK,
 	.hpd = DISP_INTERRUPT_STATUS_CONTINUE5__DC_HPD6_INTERRUPT_MASK
 } };
 
-static const uint32_t hpd_int_control_offsets[6] = {
-	DC_HPD1_INT_CONTROL,
-	DC_HPD2_INT_CONTROL,
-	DC_HPD3_INT_CONTROL,
-	DC_HPD4_INT_CONTROL,
-	DC_HPD5_INT_CONTROL,
-	DC_HPD6_INT_CONTROL,
-};
-
 static u32 dce_v6_0_audio_endpt_rreg(struct amdgpu_device *adev,
 				     u32 block_offset, u32 reg)
 {
@@ -118,7 +130,7 @@ static void dce_v6_0_audio_endpt_wreg(struct amdgpu_device *adev,
 
 static bool dce_v6_0_is_in_vblank(struct amdgpu_device *adev, int crtc)
 {
-	if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
+	if (RREG32(mmCRTC_STATUS + crtc_offsets[crtc]) & CRTC_STATUS__CRTC_V_BLANK_MASK)
 		return true;
 	else
 		return false;
@@ -128,8 +140,8 @@ static bool dce_v6_0_is_counter_moving(struct amdgpu_device *adev, int crtc)
 {
 	u32 pos1, pos2;
 
-	pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
-	pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
+	pos1 = RREG32(mmCRTC_STATUS_POSITION + crtc_offsets[crtc]);
+	pos2 = RREG32(mmCRTC_STATUS_POSITION + crtc_offsets[crtc]);
 
 	if (pos1 != pos2)
 		return true;
@@ -151,7 +163,7 @@ static void dce_v6_0_vblank_wait(struct amdgpu_device *adev, int crtc)
 	if (crtc >= adev->mode_info.num_crtc)
 		return;
 
-	if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
+	if (!(RREG32(mmCRTC_CONTROL + crtc_offsets[crtc]) & CRTC_CONTROL__CRTC_MASTER_EN_MASK))
 		return;
 
 	/* depending on when we hit vblank, we may be close to active; if so,
@@ -179,7 +191,7 @@ static u32 dce_v6_0_vblank_get_counter(struct amdgpu_device *adev, int crtc)
 	if (crtc >= adev->mode_info.num_crtc)
 		return 0;
 	else
-		return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
+		return RREG32(mmCRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
 }
 
 static void dce_v6_0_pageflip_interrupt_init(struct amdgpu_device *adev)
@@ -219,16 +231,16 @@ static void dce_v6_0_page_flip(struct amdgpu_device *adev,
 	struct amdgpu_crtc *amdgpu_crtc = adev->mode_info.crtcs[crtc_id];
 
 	/* flip at hsync for async, default is vsync */
-	WREG32(EVERGREEN_GRPH_FLIP_CONTROL + amdgpu_crtc->crtc_offset, async ?
-	       EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
+	WREG32(mmGRPH_FLIP_CONTROL + amdgpu_crtc->crtc_offset, async ?
+	       GRPH_FLIP_CONTROL__GRPH_SURFACE_UPDATE_H_RETRACE_EN_MASK : 0);
 	/* update the scanout addresses */
-	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
+	WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
 	       upper_32_bits(crtc_base));
-	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
+	WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
 	       (u32)crtc_base);
 
 	/* post the write */
-	RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset);
+	RREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset);
 }
 
 static int dce_v6_0_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc,
@@ -236,8 +248,8 @@ static int dce_v6_0_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc,
 {
 	if ((crtc < 0) || (crtc >= adev->mode_info.num_crtc))
 		return -EINVAL;
-	*vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END + crtc_offsets[crtc]);
-	*position = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
+	*vbl = RREG32(mmCRTC_V_BLANK_START_END + crtc_offsets[crtc]);
+	*position = RREG32(mmCRTC_STATUS_POSITION + crtc_offsets[crtc]);
 
 	return 0;
 
@@ -257,34 +269,11 @@ static bool dce_v6_0_hpd_sense(struct amdgpu_device *adev,
 {
 	bool connected = false;
 
-	switch (hpd) {
-	case AMDGPU_HPD_1:
-		if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
-			connected = true;
-		break;
-	case AMDGPU_HPD_2:
-		if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
-			connected = true;
-		break;
-	case AMDGPU_HPD_3:
-		if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
-			connected = true;
-		break;
-	case AMDGPU_HPD_4:
-		if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
-			connected = true;
-		break;
-	case AMDGPU_HPD_5:
-		if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
-			connected = true;
-		break;
-	case AMDGPU_HPD_6:
-		if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
-			connected = true;
-		break;
-	default:
-		break;
-	}
+	if (hpd >= adev->mode_info.num_hpd)
+		return connected;
+
+	if (RREG32(mmDC_HPD1_INT_STATUS + hpd_offsets[hpd]) & DC_HPD1_INT_STATUS__DC_HPD1_SENSE_MASK)
+		connected = true;
 
 	return connected;
 }
@@ -303,58 +292,15 @@ static void dce_v6_0_hpd_set_polarity(struct amdgpu_device *adev,
 	u32 tmp;
 	bool connected = dce_v6_0_hpd_sense(adev, hpd);
 
-	switch (hpd) {
-	case AMDGPU_HPD_1:
-		tmp = RREG32(DC_HPD1_INT_CONTROL);
-		if (connected)
-			tmp &= ~DC_HPDx_INT_POLARITY;
-		else
-			tmp |= DC_HPDx_INT_POLARITY;
-		WREG32(DC_HPD1_INT_CONTROL, tmp);
-		break;
-	case AMDGPU_HPD_2:
-		tmp = RREG32(DC_HPD2_INT_CONTROL);
-		if (connected)
-			tmp &= ~DC_HPDx_INT_POLARITY;
-		else
-			tmp |= DC_HPDx_INT_POLARITY;
-		WREG32(DC_HPD2_INT_CONTROL, tmp);
-		break;
-	case AMDGPU_HPD_3:
-		tmp = RREG32(DC_HPD3_INT_CONTROL);
-		if (connected)
-			tmp &= ~DC_HPDx_INT_POLARITY;
-		else
-			tmp |= DC_HPDx_INT_POLARITY;
-		WREG32(DC_HPD3_INT_CONTROL, tmp);
-		break;
-	case AMDGPU_HPD_4:
-		tmp = RREG32(DC_HPD4_INT_CONTROL);
-		if (connected)
-			tmp &= ~DC_HPDx_INT_POLARITY;
-		else
-			tmp |= DC_HPDx_INT_POLARITY;
-		WREG32(DC_HPD4_INT_CONTROL, tmp);
-		break;
-	case AMDGPU_HPD_5:
-		tmp = RREG32(DC_HPD5_INT_CONTROL);
-		if (connected)
-			tmp &= ~DC_HPDx_INT_POLARITY;
-		else
-			tmp |= DC_HPDx_INT_POLARITY;
-		WREG32(DC_HPD5_INT_CONTROL, tmp);
-			break;
-	case AMDGPU_HPD_6:
-		tmp = RREG32(DC_HPD6_INT_CONTROL);
-		if (connected)
-			tmp &= ~DC_HPDx_INT_POLARITY;
-		else
-			tmp |= DC_HPDx_INT_POLARITY;
-		WREG32(DC_HPD6_INT_CONTROL, tmp);
-		break;
-	default:
-		break;
-	}
+	if (hpd >= adev->mode_info.num_hpd)
+		return;
+
+	tmp = RREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[hpd]);
+	if (connected)
+		tmp &= ~DC_HPD1_INT_CONTROL__DC_HPD1_INT_POLARITY_MASK;
+	else
+		tmp |= DC_HPD1_INT_CONTROL__DC_HPD1_INT_POLARITY_MASK;
+	WREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[hpd], tmp);
 }
 
 /**
@@ -369,34 +315,17 @@ static void dce_v6_0_hpd_init(struct amdgpu_device *adev)
 {
 	struct drm_device *dev = adev->ddev;
 	struct drm_connector *connector;
-	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
-		DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
+	u32 tmp;
 
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-		switch (amdgpu_connector->hpd.hpd) {
-		case AMDGPU_HPD_1:
-			WREG32(DC_HPD1_CONTROL, tmp);
-			break;
-		case AMDGPU_HPD_2:
-			WREG32(DC_HPD2_CONTROL, tmp);
-			break;
-		case AMDGPU_HPD_3:
-			WREG32(DC_HPD3_CONTROL, tmp);
-			break;
-		case AMDGPU_HPD_4:
-			WREG32(DC_HPD4_CONTROL, tmp);
-			break;
-		case AMDGPU_HPD_5:
-			WREG32(DC_HPD5_CONTROL, tmp);
-			break;
-		case AMDGPU_HPD_6:
-			WREG32(DC_HPD6_CONTROL, tmp);
-			break;
-		default:
-			break;
-		}
+		if (amdgpu_connector->hpd.hpd >= adev->mode_info.num_hpd)
+			continue;
+
+		tmp = RREG32(mmDC_HPD1_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd]);
+		tmp |= DC_HPD1_CONTROL__DC_HPD1_EN_MASK;
+		WREG32(mmDC_HPD1_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd], tmp);
 
 		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
 		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
@@ -405,34 +334,9 @@ static void dce_v6_0_hpd_init(struct amdgpu_device *adev)
 			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
 			 * also avoid interrupt storms during dpms.
 			 */
-			u32 dc_hpd_int_cntl_reg, dc_hpd_int_cntl;
-
-			switch (amdgpu_connector->hpd.hpd) {
-			case AMDGPU_HPD_1:
-				dc_hpd_int_cntl_reg = DC_HPD1_INT_CONTROL;
-				break;
-			case AMDGPU_HPD_2:
-				dc_hpd_int_cntl_reg = DC_HPD2_INT_CONTROL;
-				break;
-			case AMDGPU_HPD_3:
-				dc_hpd_int_cntl_reg = DC_HPD3_INT_CONTROL;
-				break;
-			case AMDGPU_HPD_4:
-				dc_hpd_int_cntl_reg = DC_HPD4_INT_CONTROL;
-				break;
-			case AMDGPU_HPD_5:
-				dc_hpd_int_cntl_reg = DC_HPD5_INT_CONTROL;
-				break;
-			case AMDGPU_HPD_6:
-				dc_hpd_int_cntl_reg = DC_HPD6_INT_CONTROL;
-				break;
-			default:
-				continue;
-			}
-
-			dc_hpd_int_cntl = RREG32(dc_hpd_int_cntl_reg);
-			dc_hpd_int_cntl &= ~DC_HPDx_INT_EN;
-			WREG32(dc_hpd_int_cntl_reg, dc_hpd_int_cntl);
+			tmp = RREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd]);
+			tmp &= ~DC_HPD1_INT_CONTROL__DC_HPD1_INT_EN_MASK;
+			WREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd], tmp);
 			continue;
 		}
 
@@ -454,46 +358,25 @@ static void dce_v6_0_hpd_fini(struct amdgpu_device *adev)
 {
 	struct drm_device *dev = adev->ddev;
 	struct drm_connector *connector;
+	u32 tmp;
 
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-		switch (amdgpu_connector->hpd.hpd) {
-		case AMDGPU_HPD_1:
-			WREG32(DC_HPD1_CONTROL, 0);
-			break;
-		case AMDGPU_HPD_2:
-			WREG32(DC_HPD2_CONTROL, 0);
-			break;
-		case AMDGPU_HPD_3:
-			WREG32(DC_HPD3_CONTROL, 0);
-			break;
-		case AMDGPU_HPD_4:
-			WREG32(DC_HPD4_CONTROL, 0);
-			break;
-		case AMDGPU_HPD_5:
-			WREG32(DC_HPD5_CONTROL, 0);
-			break;
-		case AMDGPU_HPD_6:
-			WREG32(DC_HPD6_CONTROL, 0);
-			break;
-		default:
-			break;
-		}
+		if (amdgpu_connector->hpd.hpd >= adev->mode_info.num_hpd)
+			continue;
+
+		tmp = RREG32(mmDC_HPD1_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd]);
+		tmp &= ~DC_HPD1_CONTROL__DC_HPD1_EN_MASK;
+		WREG32(mmDC_HPD1_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd], 0);
+
 		amdgpu_irq_put(adev, &adev->hpd_irq, amdgpu_connector->hpd.hpd);
 	}
 }
 
 static u32 dce_v6_0_hpd_get_gpio_reg(struct amdgpu_device *adev)
 {
-	return SI_DC_GPIO_HPD_A;
-}
-
-static bool dce_v6_0_is_display_hung(struct amdgpu_device *adev)
-{
-	DRM_INFO("xxxx: dce_v6_0_is_display_hung ----no imp!!!!!\n");
-
-	return true;
+	return mmDC_GPIO_HPD_A;
 }
 
 static u32 evergreen_get_vblank_counter(struct amdgpu_device* adev, int crtc)
@@ -501,7 +384,7 @@ static u32 evergreen_get_vblank_counter(struct amdgpu_device* adev, int crtc)
 	if (crtc >= adev->mode_info.num_crtc)
 		return 0;
 	else
-		return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
+		return RREG32(mmCRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
 }
 
 static void dce_v6_0_stop_mc_access(struct amdgpu_device *adev,
@@ -510,25 +393,25 @@ static void dce_v6_0_stop_mc_access(struct amdgpu_device *adev,
 	u32 crtc_enabled, tmp, frame_count;
 	int i, j;
 
-	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
-	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
+	save->vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
+	save->vga_hdp_control = RREG32(mmVGA_HDP_CONTROL);
 
 	/* disable VGA render */
-	WREG32(VGA_RENDER_CONTROL, 0);
+	WREG32(mmVGA_RENDER_CONTROL, 0);
 
 	/* blank the display controllers */
 	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
+		crtc_enabled = RREG32(mmCRTC_CONTROL + crtc_offsets[i]) & CRTC_CONTROL__CRTC_MASTER_EN_MASK;
 		if (crtc_enabled) {
 			save->crtc_enabled[i] = true;
-			tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
+			tmp = RREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i]);
 
-			if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
+			if (!(tmp & CRTC_BLANK_CONTROL__CRTC_BLANK_DATA_EN_MASK)) {
 				dce_v6_0_vblank_wait(adev, i);
-				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
-				tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
-				WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
-				WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
+				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
+				tmp |= CRTC_BLANK_CONTROL__CRTC_BLANK_DATA_EN_MASK;
+				WREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
+				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
 			}
 			/* wait for the next frame */
 			frame_count = evergreen_get_vblank_counter(adev, i);
@@ -539,11 +422,11 @@ static void dce_v6_0_stop_mc_access(struct amdgpu_device *adev,
 			}
 
 			/* XXX this is a hack to avoid strange behavior with EFI on certain systems */
-			WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
-			tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
-			tmp &= ~EVERGREEN_CRTC_MASTER_EN;
-			WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
-			WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
+			WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
+			tmp = RREG32(mmCRTC_CONTROL + crtc_offsets[i]);
+			tmp &= ~CRTC_CONTROL__CRTC_MASTER_EN_MASK;
+			WREG32(mmCRTC_CONTROL + crtc_offsets[i], tmp);
+			WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
 			save->crtc_enabled[i] = false;
 			/* ***** */
 		} else {
@@ -560,41 +443,40 @@ static void dce_v6_0_resume_mc_access(struct amdgpu_device *adev,
 
 	/* update crtc base addresses */
 	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
+		WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
 		       upper_32_bits(adev->mc.vram_start));
-		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
+		WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
 		       upper_32_bits(adev->mc.vram_start));
-		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
+		WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
 		       (u32)adev->mc.vram_start);
-		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
+		WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
 		       (u32)adev->mc.vram_start);
 	}
 
-	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(adev->mc.vram_start));
-	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)adev->mc.vram_start);
+	WREG32(mmVGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(adev->mc.vram_start));
+	WREG32(mmVGA_MEMORY_BASE_ADDRESS, (u32)adev->mc.vram_start);
 
 	/* unlock regs and wait for update */
 	for (i = 0; i < adev->mode_info.num_crtc; i++) {
 		if (save->crtc_enabled[i]) {
-			tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
-			if ((tmp & 0x7) != 3) {
+			tmp = RREG32(mmMASTER_UPDATE_MODE + crtc_offsets[i]);
+			if ((tmp & 0x7) != 0) {
 				tmp &= ~0x7;
-				tmp |= 0x3;
-				WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
+				WREG32(mmMASTER_UPDATE_MODE + crtc_offsets[i], tmp);
 			}
-			tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
-			if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
-				tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
-				WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
+			tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]);
+			if (tmp & GRPH_UPDATE__GRPH_UPDATE_LOCK_MASK) {
+				tmp &= ~GRPH_UPDATE__GRPH_UPDATE_LOCK_MASK;
+				WREG32(mmGRPH_UPDATE + crtc_offsets[i], tmp);
 			}
-			tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
+			tmp = RREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i]);
 			if (tmp & 1) {
 				tmp &= ~1;
-				WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
+				WREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
 			}
 			for (j = 0; j < adev->usec_timeout; j++) {
-				tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
-				if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
+				tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]);
+				if ((tmp & GRPH_UPDATE__GRPH_SURFACE_UPDATE_PENDING_MASK) == 0)
 					break;
 				udelay(1);
 			}
@@ -602,19 +484,62 @@ static void dce_v6_0_resume_mc_access(struct amdgpu_device *adev,
 	}
 
 	/* Unlock vga access */
-	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
+	WREG32(mmVGA_HDP_CONTROL, save->vga_hdp_control);
 	mdelay(1);
-	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
+	WREG32(mmVGA_RENDER_CONTROL, save->vga_render_control);
 
 }
 
 static void dce_v6_0_set_vga_render_state(struct amdgpu_device *adev,
 					  bool render)
 {
-	if (!render) 
-		WREG32(R_000300_VGA_RENDER_CONTROL,
-			RREG32(R_000300_VGA_RENDER_CONTROL) & C_000300_VGA_VSTATUS_CNTL);
+	if (!render)
+		WREG32(mmVGA_RENDER_CONTROL,
+			RREG32(mmVGA_RENDER_CONTROL) & VGA_VSTATUS_CNTL);
+
+}
+
+static int dce_v6_0_get_num_crtc(struct amdgpu_device *adev)
+{
+	int num_crtc = 0;
+
+	switch (adev->asic_type) {
+	case CHIP_TAHITI:
+	case CHIP_PITCAIRN:
+	case CHIP_VERDE:
+		num_crtc = 6;
+		break;
+	case CHIP_OLAND:
+		num_crtc = 2;
+		break;
+	default:
+		num_crtc = 0;
+	}
+	return num_crtc;
+}
+
+void dce_v6_0_disable_dce(struct amdgpu_device *adev)
+{
+	/*Disable VGA render and enabled crtc, if has DCE engine*/
+	if (amdgpu_atombios_has_dce_engine_info(adev)) {
+		u32 tmp;
+		int crtc_enabled, i;
+
+		dce_v6_0_set_vga_render_state(adev, false);
 
+		/*Disable crtc*/
+		for (i = 0; i < dce_v6_0_get_num_crtc(adev); i++) {
+			crtc_enabled = RREG32(mmCRTC_CONTROL + crtc_offsets[i]) &
+				CRTC_CONTROL__CRTC_MASTER_EN_MASK;
+			if (crtc_enabled) {
+				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
+				tmp = RREG32(mmCRTC_CONTROL + crtc_offsets[i]);
+				tmp &= ~CRTC_CONTROL__CRTC_MASTER_EN_MASK;
+				WREG32(mmCRTC_CONTROL + crtc_offsets[i], tmp);
+				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
+			}
+		}
+	}
 }
 
 static void dce_v6_0_program_fmt(struct drm_encoder *encoder)
@@ -647,19 +572,23 @@ static void dce_v6_0_program_fmt(struct drm_encoder *encoder)
 	case 6:
 		if (dither == AMDGPU_FMT_DITHER_ENABLE)
 			/* XXX sort out optimal dither settings */
-			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
-				FMT_SPATIAL_DITHER_EN);
+			tmp |= (FMT_BIT_DEPTH_CONTROL__FMT_FRAME_RANDOM_ENABLE_MASK |
+				FMT_BIT_DEPTH_CONTROL__FMT_HIGHPASS_RANDOM_ENABLE_MASK |
+				FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_EN_MASK);
 		else
-			tmp |= FMT_TRUNCATE_EN;
+			tmp |= FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_EN_MASK;
 		break;
 	case 8:
 		if (dither == AMDGPU_FMT_DITHER_ENABLE)
 			/* XXX sort out optimal dither settings */
-			tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
-				FMT_RGB_RANDOM_ENABLE |
-				FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
+			tmp |= (FMT_BIT_DEPTH_CONTROL__FMT_FRAME_RANDOM_ENABLE_MASK |
+				FMT_BIT_DEPTH_CONTROL__FMT_HIGHPASS_RANDOM_ENABLE_MASK |
+				FMT_BIT_DEPTH_CONTROL__FMT_RGB_RANDOM_ENABLE_MASK |
+				FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_EN_MASK |
+				FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_DEPTH_MASK);
 		else
-			tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
+			tmp |= (FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_EN_MASK |
+				FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_DEPTH_MASK);
 		break;
 	case 10:
 	default:
@@ -667,7 +596,7 @@ static void dce_v6_0_program_fmt(struct drm_encoder *encoder)
 		break;
 	}
 
-	WREG32(FMT_BIT_DEPTH_CONTROL + amdgpu_crtc->crtc_offset, tmp);
+	WREG32(mmFMT_BIT_DEPTH_CONTROL + amdgpu_crtc->crtc_offset, tmp);
 }
 
 /**
@@ -681,7 +610,7 @@ static void dce_v6_0_program_fmt(struct drm_encoder *encoder)
  */
 static u32 si_get_number_of_dram_channels(struct amdgpu_device *adev)
 {
-	u32 tmp = RREG32(MC_SHARED_CHMAP);
+	u32 tmp = RREG32(mmMC_SHARED_CHMAP);
 
 	switch ((tmp & MC_SHARED_CHMAP__NOOFCHAN_MASK) >> MC_SHARED_CHMAP__NOOFCHAN__SHIFT) {
 	case 0:
@@ -1178,28 +1107,28 @@ static void dce_v6_0_program_watermarks(struct amdgpu_device *adev,
 	}
 
 	/* select wm A */
-	arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset);
+	arb_control3 = RREG32(mmDPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset);
 	tmp = arb_control3;
 	tmp &= ~LATENCY_WATERMARK_MASK(3);
 	tmp |= LATENCY_WATERMARK_MASK(1);
-	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset, tmp);
-	WREG32(DPG_PIPE_LATENCY_CONTROL + amdgpu_crtc->crtc_offset,
-	       (LATENCY_LOW_WATERMARK(latency_watermark_a) |
-		LATENCY_HIGH_WATERMARK(line_time)));
+	WREG32(mmDPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset, tmp);
+	WREG32(mmDPG_PIPE_URGENCY_CONTROL + amdgpu_crtc->crtc_offset,
+	       ((latency_watermark_a << DPG_PIPE_URGENCY_CONTROL__URGENCY_LOW_WATERMARK__SHIFT)  |
+		(line_time << DPG_PIPE_URGENCY_CONTROL__URGENCY_HIGH_WATERMARK__SHIFT)));
 	/* select wm B */
-	tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset);
+	tmp = RREG32(mmDPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset);
 	tmp &= ~LATENCY_WATERMARK_MASK(3);
 	tmp |= LATENCY_WATERMARK_MASK(2);
-	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset, tmp);
-	WREG32(DPG_PIPE_LATENCY_CONTROL + amdgpu_crtc->crtc_offset,
-	       (LATENCY_LOW_WATERMARK(latency_watermark_b) |
-		LATENCY_HIGH_WATERMARK(line_time)));
+	WREG32(mmDPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset, tmp);
+	WREG32(mmDPG_PIPE_URGENCY_CONTROL + amdgpu_crtc->crtc_offset,
+	       ((latency_watermark_b << DPG_PIPE_URGENCY_CONTROL__URGENCY_LOW_WATERMARK__SHIFT) |
+		(line_time << DPG_PIPE_URGENCY_CONTROL__URGENCY_HIGH_WATERMARK__SHIFT)));
 	/* restore original selection */
-	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset, arb_control3);
+	WREG32(mmDPG_PIPE_ARBITRATION_CONTROL3 + amdgpu_crtc->crtc_offset, arb_control3);
 
 	/* write the priority marks */
-	WREG32(PRIORITY_A_CNT + amdgpu_crtc->crtc_offset, priority_a_cnt);
-	WREG32(PRIORITY_B_CNT + amdgpu_crtc->crtc_offset, priority_b_cnt);
+	WREG32(mmPRIORITY_A_CNT + amdgpu_crtc->crtc_offset, priority_a_cnt);
+	WREG32(mmPRIORITY_B_CNT + amdgpu_crtc->crtc_offset, priority_b_cnt);
 
 	/* save values for DPM */
 	amdgpu_crtc->line_time = line_time;
@@ -1217,7 +1146,7 @@ static u32 dce_v6_0_line_buffer_adjust(struct amdgpu_device *adev,
 	/*
 	 * Line Buffer Setup
 	 * There are 3 line buffers, each one shared by 2 display controllers.
-	 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
+	 * mmDC_LB_MEMORY_SPLIT controls how that line buffer is shared between
 	 * the display controllers.  The paritioning is done via one of four
 	 * preset allocations specified in bits 21:20:
 	 *  0 - half lb
@@ -1240,14 +1169,14 @@ static u32 dce_v6_0_line_buffer_adjust(struct amdgpu_device *adev,
 		buffer_alloc = 0;
 	}
 
-	WREG32(DC_LB_MEMORY_SPLIT + amdgpu_crtc->crtc_offset,
+	WREG32(mmDC_LB_MEMORY_SPLIT + amdgpu_crtc->crtc_offset,
 	       DC_LB_MEMORY_CONFIG(tmp));
 
-	WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
-	       DMIF_BUFFERS_ALLOCATED(buffer_alloc));
+	WREG32(mmPIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
+	       (buffer_alloc << PIPE0_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED__SHIFT));
 	for (i = 0; i < adev->usec_timeout; i++) {
-		if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
-		    DMIF_BUFFERS_ALLOCATED_COMPLETED)
+		if (RREG32(mmPIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
+		    PIPE0_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED_MASK)
 			break;
 		udelay(1);
 	}
@@ -1489,12 +1418,12 @@ static void dce_v6_0_afmt_fini(struct amdgpu_device *adev)
 
 static const u32 vga_control_regs[6] =
 {
-	AVIVO_D1VGA_CONTROL,
-	AVIVO_D2VGA_CONTROL,
-	EVERGREEN_D3VGA_CONTROL,
-	EVERGREEN_D4VGA_CONTROL,
-	EVERGREEN_D5VGA_CONTROL,
-	EVERGREEN_D6VGA_CONTROL,
+	mmD1VGA_CONTROL,
+	mmD2VGA_CONTROL,
+	mmD3VGA_CONTROL,
+	mmD4VGA_CONTROL,
+	mmD5VGA_CONTROL,
+	mmD6VGA_CONTROL,
 };
 
 static void dce_v6_0_vga_enable(struct drm_crtc *crtc, bool enable)
@@ -1514,7 +1443,7 @@ static void dce_v6_0_grph_enable(struct drm_crtc *crtc, bool enable)
 	struct drm_device *dev = crtc->dev;
 	struct amdgpu_device *adev = dev->dev_private;
 
-	WREG32(EVERGREEN_GRPH_ENABLE + amdgpu_crtc->crtc_offset, enable ? 1 : 0);
+	WREG32(mmGRPH_ENABLE + amdgpu_crtc->crtc_offset, enable ? 1 : 0);
 }
 
 static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
@@ -1530,10 +1459,11 @@ static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
 	struct amdgpu_bo *abo;
 	uint64_t fb_location, tiling_flags;
 	uint32_t fb_format, fb_pitch_pixels, pipe_config;
-	u32 fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_NONE);
+	u32 fb_swap = GRPH_ENDIAN_SWAP(GRPH_ENDIAN_NONE);
 	u32 viewport_w, viewport_h;
 	int r;
 	bool bypass_lut = false;
+	struct drm_format_name_buf format_name;
 
 	/* no fb bound */
 	if (!atomic && !crtc->primary->fb) {
@@ -1573,71 +1503,71 @@ static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
 
 	switch (target_fb->pixel_format) {
 	case DRM_FORMAT_C8:
-		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_8BPP) |
-			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_INDEXED));
+		fb_format = (GRPH_DEPTH(GRPH_DEPTH_8BPP) |
+			     GRPH_FORMAT(GRPH_FORMAT_INDEXED));
 		break;
 	case DRM_FORMAT_XRGB4444:
 	case DRM_FORMAT_ARGB4444:
-		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) |
-			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB4444));
+		fb_format = (GRPH_DEPTH(GRPH_DEPTH_16BPP) |
+			     GRPH_FORMAT(GRPH_FORMAT_ARGB4444));
 #ifdef __BIG_ENDIAN
-		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16);
+		fb_swap = GRPH_ENDIAN_SWAP(GRPH_ENDIAN_8IN16);
 #endif
 		break;
 	case DRM_FORMAT_XRGB1555:
 	case DRM_FORMAT_ARGB1555:
-		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) |
-			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB1555));
+		fb_format = (GRPH_DEPTH(GRPH_DEPTH_16BPP) |
+			     GRPH_FORMAT(GRPH_FORMAT_ARGB1555));
 #ifdef __BIG_ENDIAN
-		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16);
+		fb_swap = GRPH_ENDIAN_SWAP(GRPH_ENDIAN_8IN16);
 #endif
 		break;
 	case DRM_FORMAT_BGRX5551:
 	case DRM_FORMAT_BGRA5551:
-		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) |
-			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_BGRA5551));
+		fb_format = (GRPH_DEPTH(GRPH_DEPTH_16BPP) |
+			     GRPH_FORMAT(GRPH_FORMAT_BGRA5551));
 #ifdef __BIG_ENDIAN
-		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16);
+		fb_swap = GRPH_ENDIAN_SWAP(GRPH_ENDIAN_8IN16);
 #endif
 		break;
 	case DRM_FORMAT_RGB565:
-		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_16BPP) |
-			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB565));
+		fb_format = (GRPH_DEPTH(GRPH_DEPTH_16BPP) |
+			     GRPH_FORMAT(GRPH_FORMAT_ARGB565));
 #ifdef __BIG_ENDIAN
-		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN16);
+		fb_swap = GRPH_ENDIAN_SWAP(GRPH_ENDIAN_8IN16);
 #endif
 		break;
 	case DRM_FORMAT_XRGB8888:
 	case DRM_FORMAT_ARGB8888:
-		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_32BPP) |
-			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB8888));
+		fb_format = (GRPH_DEPTH(GRPH_DEPTH_32BPP) |
+			     GRPH_FORMAT(GRPH_FORMAT_ARGB8888));
 #ifdef __BIG_ENDIAN
-		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN32);
+		fb_swap = GRPH_ENDIAN_SWAP(GRPH_ENDIAN_8IN32);
 #endif
 		break;
 	case DRM_FORMAT_XRGB2101010:
 	case DRM_FORMAT_ARGB2101010:
-		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_32BPP) |
-			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_ARGB2101010));
+		fb_format = (GRPH_DEPTH(GRPH_DEPTH_32BPP) |
+			     GRPH_FORMAT(GRPH_FORMAT_ARGB2101010));
 #ifdef __BIG_ENDIAN
-		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN32);
+		fb_swap = GRPH_ENDIAN_SWAP(GRPH_ENDIAN_8IN32);
 #endif
 		/* Greater 8 bpc fb needs to bypass hw-lut to retain precision */
 		bypass_lut = true;
 		break;
 	case DRM_FORMAT_BGRX1010102:
 	case DRM_FORMAT_BGRA1010102:
-		fb_format = (EVERGREEN_GRPH_DEPTH(EVERGREEN_GRPH_DEPTH_32BPP) |
-			     EVERGREEN_GRPH_FORMAT(EVERGREEN_GRPH_FORMAT_BGRA1010102));
+		fb_format = (GRPH_DEPTH(GRPH_DEPTH_32BPP) |
+			     GRPH_FORMAT(GRPH_FORMAT_BGRA1010102));
 #ifdef __BIG_ENDIAN
-		fb_swap = EVERGREEN_GRPH_ENDIAN_SWAP(EVERGREEN_GRPH_ENDIAN_8IN32);
+		fb_swap = GRPH_ENDIAN_SWAP(GRPH_ENDIAN_8IN32);
 #endif
 		/* Greater 8 bpc fb needs to bypass hw-lut to retain precision */
 		bypass_lut = true;
 		break;
 	default:
 		DRM_ERROR("Unsupported screen format %s\n",
-			  drm_get_format_name(target_fb->pixel_format));
+		          drm_get_format_name(target_fb->pixel_format, &format_name));
 		return -EINVAL;
 	}
 
@@ -1650,75 +1580,75 @@ static int dce_v6_0_crtc_do_set_base(struct drm_crtc *crtc,
 		tile_split = AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT);
 		num_banks = AMDGPU_TILING_GET(tiling_flags, NUM_BANKS);
 
-		fb_format |= EVERGREEN_GRPH_NUM_BANKS(num_banks);
-		fb_format |= EVERGREEN_GRPH_ARRAY_MODE(EVERGREEN_GRPH_ARRAY_2D_TILED_THIN1);
-		fb_format |= EVERGREEN_GRPH_TILE_SPLIT(tile_split);
-		fb_format |= EVERGREEN_GRPH_BANK_WIDTH(bankw);
-		fb_format |= EVERGREEN_GRPH_BANK_HEIGHT(bankh);
-		fb_format |= EVERGREEN_GRPH_MACRO_TILE_ASPECT(mtaspect);
+		fb_format |= GRPH_NUM_BANKS(num_banks);
+		fb_format |= GRPH_ARRAY_MODE(GRPH_ARRAY_2D_TILED_THIN1);
+		fb_format |= GRPH_TILE_SPLIT(tile_split);
+		fb_format |= GRPH_BANK_WIDTH(bankw);
+		fb_format |= GRPH_BANK_HEIGHT(bankh);
+		fb_format |= GRPH_MACRO_TILE_ASPECT(mtaspect);
 	} else if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == ARRAY_1D_TILED_THIN1) {
-		fb_format |= EVERGREEN_GRPH_ARRAY_MODE(EVERGREEN_GRPH_ARRAY_1D_TILED_THIN1);
+		fb_format |= GRPH_ARRAY_MODE(GRPH_ARRAY_1D_TILED_THIN1);
 	}
 
 	pipe_config = AMDGPU_TILING_GET(tiling_flags, PIPE_CONFIG);
-	fb_format |= SI_GRPH_PIPE_CONFIG(pipe_config);
+	fb_format |= GRPH_PIPE_CONFIG(pipe_config);
 
 	dce_v6_0_vga_enable(crtc, false);
 
 	/* Make sure surface address is updated at vertical blank rather than
 	 * horizontal blank
 	 */
-	WREG32(EVERGREEN_GRPH_FLIP_CONTROL + amdgpu_crtc->crtc_offset, 0);
+	WREG32(mmGRPH_FLIP_CONTROL + amdgpu_crtc->crtc_offset, 0);
 
-	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
+	WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
 	       upper_32_bits(fb_location));
-	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
+	WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
 	       upper_32_bits(fb_location));
-	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
-	       (u32)fb_location & EVERGREEN_GRPH_SURFACE_ADDRESS_MASK);
-	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
-	       (u32) fb_location & EVERGREEN_GRPH_SURFACE_ADDRESS_MASK);
-	WREG32(EVERGREEN_GRPH_CONTROL + amdgpu_crtc->crtc_offset, fb_format);
-	WREG32(EVERGREEN_GRPH_SWAP_CONTROL + amdgpu_crtc->crtc_offset, fb_swap);
+	WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
+	       (u32)fb_location & GRPH_PRIMARY_SURFACE_ADDRESS__GRPH_PRIMARY_SURFACE_ADDRESS_MASK);
+	WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
+	       (u32) fb_location & GRPH_PRIMARY_SURFACE_ADDRESS__GRPH_PRIMARY_SURFACE_ADDRESS_MASK);
+	WREG32(mmGRPH_CONTROL + amdgpu_crtc->crtc_offset, fb_format);
+	WREG32(mmGRPH_SWAP_CNTL + amdgpu_crtc->crtc_offset, fb_swap);
 
 	/*
 	 * The LUT only has 256 slots for indexing by a 8 bpc fb. Bypass the LUT
 	 * for > 8 bpc scanout to avoid truncation of fb indices to 8 msb's, to
 	 * retain the full precision throughout the pipeline.
 	 */
-	WREG32_P(EVERGREEN_GRPH_LUT_10BIT_BYPASS_CONTROL + amdgpu_crtc->crtc_offset,
-		 (bypass_lut ? EVERGREEN_LUT_10BIT_BYPASS_EN : 0),
-		 ~EVERGREEN_LUT_10BIT_BYPASS_EN);
+	WREG32_P(mmGRPH_LUT_10BIT_BYPASS + amdgpu_crtc->crtc_offset,
+		 (bypass_lut ? GRPH_LUT_10BIT_BYPASS__GRPH_LUT_10BIT_BYPASS_EN_MASK : 0),
+		 ~GRPH_LUT_10BIT_BYPASS__GRPH_LUT_10BIT_BYPASS_EN_MASK);
 
 	if (bypass_lut)
 		DRM_DEBUG_KMS("Bypassing hardware LUT due to 10 bit fb scanout.\n");
 
-	WREG32(EVERGREEN_GRPH_SURFACE_OFFSET_X + amdgpu_crtc->crtc_offset, 0);
-	WREG32(EVERGREEN_GRPH_SURFACE_OFFSET_Y + amdgpu_crtc->crtc_offset, 0);
-	WREG32(EVERGREEN_GRPH_X_START + amdgpu_crtc->crtc_offset, 0);
-	WREG32(EVERGREEN_GRPH_Y_START + amdgpu_crtc->crtc_offset, 0);
-	WREG32(EVERGREEN_GRPH_X_END + amdgpu_crtc->crtc_offset, target_fb->width);
-	WREG32(EVERGREEN_GRPH_Y_END + amdgpu_crtc->crtc_offset, target_fb->height);
+	WREG32(mmGRPH_SURFACE_OFFSET_X + amdgpu_crtc->crtc_offset, 0);
+	WREG32(mmGRPH_SURFACE_OFFSET_Y + amdgpu_crtc->crtc_offset, 0);
+	WREG32(mmGRPH_X_START + amdgpu_crtc->crtc_offset, 0);
+	WREG32(mmGRPH_Y_START + amdgpu_crtc->crtc_offset, 0);
+	WREG32(mmGRPH_X_END + amdgpu_crtc->crtc_offset, target_fb->width);
+	WREG32(mmGRPH_Y_END + amdgpu_crtc->crtc_offset, target_fb->height);
 
 	fb_pitch_pixels = target_fb->pitches[0] / (target_fb->bits_per_pixel / 8);
-	WREG32(EVERGREEN_GRPH_PITCH + amdgpu_crtc->crtc_offset, fb_pitch_pixels);
+	WREG32(mmGRPH_PITCH + amdgpu_crtc->crtc_offset, fb_pitch_pixels);
 
 	dce_v6_0_grph_enable(crtc, true);
 
-	WREG32(EVERGREEN_DESKTOP_HEIGHT + amdgpu_crtc->crtc_offset,
+	WREG32(mmDESKTOP_HEIGHT + amdgpu_crtc->crtc_offset,
 		       target_fb->height);
 	x &= ~3;
 	y &= ~1;
-	WREG32(EVERGREEN_VIEWPORT_START + amdgpu_crtc->crtc_offset,
+	WREG32(mmVIEWPORT_START + amdgpu_crtc->crtc_offset,
 	       (x << 16) | y);
 	viewport_w = crtc->mode.hdisplay;
 	viewport_h = (crtc->mode.vdisplay + 1) & ~1;
 
-	WREG32(EVERGREEN_VIEWPORT_SIZE + amdgpu_crtc->crtc_offset,
+	WREG32(mmVIEWPORT_SIZE + amdgpu_crtc->crtc_offset,
 	       (viewport_w << 16) | viewport_h);
 
 	/* set pageflip to happen anywhere in vblank interval */
-	WREG32(EVERGREEN_MASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
+	WREG32(mmMASTER_UPDATE_MODE + amdgpu_crtc->crtc_offset, 0);
 
 	if (!atomic && fb && fb != crtc->primary->fb) {
 		amdgpu_fb = to_amdgpu_framebuffer(fb);
@@ -1745,10 +1675,10 @@ static void dce_v6_0_set_interleave(struct drm_crtc *crtc,
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 
 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
-		WREG32(EVERGREEN_DATA_FORMAT + amdgpu_crtc->crtc_offset,
-		       EVERGREEN_INTERLEAVE_EN);
+		WREG32(mmDATA_FORMAT + amdgpu_crtc->crtc_offset,
+		       INTERLEAVE_EN);
 	else
-		WREG32(EVERGREEN_DATA_FORMAT + amdgpu_crtc->crtc_offset, 0);
+		WREG32(mmDATA_FORMAT + amdgpu_crtc->crtc_offset, 0);
 }
 
 static void dce_v6_0_crtc_load_lut(struct drm_crtc *crtc)
@@ -1761,54 +1691,52 @@ static void dce_v6_0_crtc_load_lut(struct drm_crtc *crtc)
 
 	DRM_DEBUG_KMS("%d\n", amdgpu_crtc->crtc_id);
 
-	WREG32(NI_INPUT_CSC_CONTROL + amdgpu_crtc->crtc_offset,
-	       (NI_INPUT_CSC_GRPH_MODE(NI_INPUT_CSC_BYPASS) |
-		NI_INPUT_CSC_OVL_MODE(NI_INPUT_CSC_BYPASS)));
-	WREG32(NI_PRESCALE_GRPH_CONTROL + amdgpu_crtc->crtc_offset,
-	       NI_GRPH_PRESCALE_BYPASS);
-	WREG32(NI_PRESCALE_OVL_CONTROL + amdgpu_crtc->crtc_offset,
-	       NI_OVL_PRESCALE_BYPASS);
-	WREG32(NI_INPUT_GAMMA_CONTROL + amdgpu_crtc->crtc_offset,
-	       (NI_GRPH_INPUT_GAMMA_MODE(NI_INPUT_GAMMA_USE_LUT) |
-		NI_OVL_INPUT_GAMMA_MODE(NI_INPUT_GAMMA_USE_LUT)));
-
+	WREG32(mmINPUT_CSC_CONTROL + amdgpu_crtc->crtc_offset,
+	       ((0 << INPUT_CSC_CONTROL__INPUT_CSC_GRPH_MODE__SHIFT) |
+		(0 << INPUT_CSC_CONTROL__INPUT_CSC_OVL_MODE__SHIFT)));
+	WREG32(mmPRESCALE_GRPH_CONTROL + amdgpu_crtc->crtc_offset,
+	       PRESCALE_GRPH_CONTROL__GRPH_PRESCALE_BYPASS_MASK);
+	WREG32(mmPRESCALE_OVL_CONTROL + amdgpu_crtc->crtc_offset,
+	       PRESCALE_OVL_CONTROL__OVL_PRESCALE_BYPASS_MASK);
+	WREG32(mmINPUT_GAMMA_CONTROL + amdgpu_crtc->crtc_offset,
+	       ((0 << INPUT_GAMMA_CONTROL__GRPH_INPUT_GAMMA_MODE__SHIFT) |
+		(0 << INPUT_GAMMA_CONTROL__OVL_INPUT_GAMMA_MODE__SHIFT)));
 
+	WREG32(mmDC_LUT_CONTROL + amdgpu_crtc->crtc_offset, 0);
 
-	WREG32(EVERGREEN_DC_LUT_CONTROL + amdgpu_crtc->crtc_offset, 0);
+	WREG32(mmDC_LUT_BLACK_OFFSET_BLUE + amdgpu_crtc->crtc_offset, 0);
+	WREG32(mmDC_LUT_BLACK_OFFSET_GREEN + amdgpu_crtc->crtc_offset, 0);
+	WREG32(mmDC_LUT_BLACK_OFFSET_RED + amdgpu_crtc->crtc_offset, 0);
 
-	WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_BLUE + amdgpu_crtc->crtc_offset, 0);
-	WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_GREEN + amdgpu_crtc->crtc_offset, 0);
-	WREG32(EVERGREEN_DC_LUT_BLACK_OFFSET_RED + amdgpu_crtc->crtc_offset, 0);
+	WREG32(mmDC_LUT_WHITE_OFFSET_BLUE + amdgpu_crtc->crtc_offset, 0xffff);
+	WREG32(mmDC_LUT_WHITE_OFFSET_GREEN + amdgpu_crtc->crtc_offset, 0xffff);
+	WREG32(mmDC_LUT_WHITE_OFFSET_RED + amdgpu_crtc->crtc_offset, 0xffff);
 
-	WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_BLUE + amdgpu_crtc->crtc_offset, 0xffff);
-	WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_GREEN + amdgpu_crtc->crtc_offset, 0xffff);
-	WREG32(EVERGREEN_DC_LUT_WHITE_OFFSET_RED + amdgpu_crtc->crtc_offset, 0xffff);
+	WREG32(mmDC_LUT_RW_MODE + amdgpu_crtc->crtc_offset, 0);
+	WREG32(mmDC_LUT_WRITE_EN_MASK + amdgpu_crtc->crtc_offset, 0x00000007);
 
-	WREG32(EVERGREEN_DC_LUT_RW_MODE + amdgpu_crtc->crtc_offset, 0);
-	WREG32(EVERGREEN_DC_LUT_WRITE_EN_MASK + amdgpu_crtc->crtc_offset, 0x00000007);
-
-	WREG32(EVERGREEN_DC_LUT_RW_INDEX + amdgpu_crtc->crtc_offset, 0);
+	WREG32(mmDC_LUT_RW_INDEX + amdgpu_crtc->crtc_offset, 0);
 	for (i = 0; i < 256; i++) {
-		WREG32(EVERGREEN_DC_LUT_30_COLOR + amdgpu_crtc->crtc_offset,
+		WREG32(mmDC_LUT_30_COLOR + amdgpu_crtc->crtc_offset,
 		       (amdgpu_crtc->lut_r[i] << 20) |
 		       (amdgpu_crtc->lut_g[i] << 10) |
 		       (amdgpu_crtc->lut_b[i] << 0));
 	}
 
-	WREG32(NI_DEGAMMA_CONTROL + amdgpu_crtc->crtc_offset,
-	       (NI_GRPH_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
-		NI_OVL_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
-		NI_ICON_DEGAMMA_MODE(NI_DEGAMMA_BYPASS) |
-		NI_CURSOR_DEGAMMA_MODE(NI_DEGAMMA_BYPASS)));
-	WREG32(NI_GAMUT_REMAP_CONTROL + amdgpu_crtc->crtc_offset,
-	       (NI_GRPH_GAMUT_REMAP_MODE(NI_GAMUT_REMAP_BYPASS) |
-		NI_OVL_GAMUT_REMAP_MODE(NI_GAMUT_REMAP_BYPASS)));
-	WREG32(NI_REGAMMA_CONTROL + amdgpu_crtc->crtc_offset,
-	       (NI_GRPH_REGAMMA_MODE(NI_REGAMMA_BYPASS) |
-		NI_OVL_REGAMMA_MODE(NI_REGAMMA_BYPASS)));
-	WREG32(NI_OUTPUT_CSC_CONTROL + amdgpu_crtc->crtc_offset,
-	       (NI_OUTPUT_CSC_GRPH_MODE(0) |
-		NI_OUTPUT_CSC_OVL_MODE(NI_OUTPUT_CSC_BYPASS)));
+	WREG32(mmDEGAMMA_CONTROL + amdgpu_crtc->crtc_offset,
+	       ((0 << DEGAMMA_CONTROL__GRPH_DEGAMMA_MODE__SHIFT) |
+		(0 << DEGAMMA_CONTROL__OVL_DEGAMMA_MODE__SHIFT) |
+		ICON_DEGAMMA_MODE(0) |
+		(0 << DEGAMMA_CONTROL__CURSOR_DEGAMMA_MODE__SHIFT)));
+	WREG32(mmGAMUT_REMAP_CONTROL + amdgpu_crtc->crtc_offset,
+	       ((0 << GAMUT_REMAP_CONTROL__GRPH_GAMUT_REMAP_MODE__SHIFT) |
+		(0 << GAMUT_REMAP_CONTROL__OVL_GAMUT_REMAP_MODE__SHIFT)));
+	WREG32(mmREGAMMA_CONTROL + amdgpu_crtc->crtc_offset,
+	       ((0 << REGAMMA_CONTROL__GRPH_REGAMMA_MODE__SHIFT) |
+		(0 << REGAMMA_CONTROL__OVL_REGAMMA_MODE__SHIFT)));
+	WREG32(mmOUTPUT_CSC_CONTROL + amdgpu_crtc->crtc_offset,
+	       ((0 << OUTPUT_CSC_CONTROL__OUTPUT_CSC_GRPH_MODE__SHIFT) |
+		(0 << OUTPUT_CSC_CONTROL__OUTPUT_CSC_OVL_MODE__SHIFT)));
 	/* XXX match this to the depth of the crtc fmt block, move to modeset? */
 	WREG32(0x1a50 + amdgpu_crtc->crtc_offset, 0);
 
@@ -1887,12 +1815,12 @@ static void dce_v6_0_lock_cursor(struct drm_crtc *crtc, bool lock)
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	uint32_t cur_lock;
 
-	cur_lock = RREG32(EVERGREEN_CUR_UPDATE + amdgpu_crtc->crtc_offset);
+	cur_lock = RREG32(mmCUR_UPDATE + amdgpu_crtc->crtc_offset);
 	if (lock)
-		cur_lock |= EVERGREEN_CURSOR_UPDATE_LOCK;
+		cur_lock |= CUR_UPDATE__CURSOR_UPDATE_LOCK_MASK;
 	else
-		cur_lock &= ~EVERGREEN_CURSOR_UPDATE_LOCK;
-	WREG32(EVERGREEN_CUR_UPDATE + amdgpu_crtc->crtc_offset, cur_lock);
+		cur_lock &= ~CUR_UPDATE__CURSOR_UPDATE_LOCK_MASK;
+	WREG32(mmCUR_UPDATE + amdgpu_crtc->crtc_offset, cur_lock);
 }
 
 static void dce_v6_0_hide_cursor(struct drm_crtc *crtc)
@@ -1900,9 +1828,9 @@ static void dce_v6_0_hide_cursor(struct drm_crtc *crtc)
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct amdgpu_device *adev = crtc->dev->dev_private;
 
-	WREG32_IDX(EVERGREEN_CUR_CONTROL + amdgpu_crtc->crtc_offset,
-		   EVERGREEN_CURSOR_MODE(EVERGREEN_CURSOR_24_8_PRE_MULT) |
-		   EVERGREEN_CURSOR_URGENT_CONTROL(EVERGREEN_CURSOR_URGENT_1_2));
+	WREG32_IDX(mmCUR_CONTROL + amdgpu_crtc->crtc_offset,
+		   (CURSOR_24_8_PRE_MULT << CUR_CONTROL__CURSOR_MODE__SHIFT) |
+		   (CURSOR_URGENT_1_2 << CUR_CONTROL__CURSOR_URGENT_CONTROL__SHIFT));
 
 
 }
@@ -1912,15 +1840,15 @@ static void dce_v6_0_show_cursor(struct drm_crtc *crtc)
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct amdgpu_device *adev = crtc->dev->dev_private;
 
-	WREG32(EVERGREEN_CUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
+	WREG32(mmCUR_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
 	       upper_32_bits(amdgpu_crtc->cursor_addr));
-	WREG32(EVERGREEN_CUR_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
+	WREG32(mmCUR_SURFACE_ADDRESS + amdgpu_crtc->crtc_offset,
 	       lower_32_bits(amdgpu_crtc->cursor_addr));
 
-	WREG32_IDX(EVERGREEN_CUR_CONTROL + amdgpu_crtc->crtc_offset,
-		   EVERGREEN_CURSOR_EN |
-		   EVERGREEN_CURSOR_MODE(EVERGREEN_CURSOR_24_8_PRE_MULT) |
-		   EVERGREEN_CURSOR_URGENT_CONTROL(EVERGREEN_CURSOR_URGENT_1_2));
+	WREG32_IDX(mmCUR_CONTROL + amdgpu_crtc->crtc_offset,
+		   CUR_CONTROL__CURSOR_EN_MASK |
+		   (CURSOR_24_8_PRE_MULT << CUR_CONTROL__CURSOR_MODE__SHIFT) |
+		   (CURSOR_URGENT_1_2 << CUR_CONTROL__CURSOR_URGENT_CONTROL__SHIFT));
 
 }
 
@@ -1931,7 +1859,8 @@ static int dce_v6_0_cursor_move_locked(struct drm_crtc *crtc,
 	struct amdgpu_device *adev = crtc->dev->dev_private;
 	int xorigin = 0, yorigin = 0;
 
-	int w = amdgpu_crtc->cursor_width;
+	amdgpu_crtc->cursor_x = x;
+	amdgpu_crtc->cursor_y = y;
 
 	/* avivo cursor are offset into the total surface */
 	x += crtc->x;
@@ -1947,13 +1876,9 @@ static int dce_v6_0_cursor_move_locked(struct drm_crtc *crtc,
 		y = 0;
 	}
 
-	WREG32(EVERGREEN_CUR_POSITION + amdgpu_crtc->crtc_offset, (x << 16) | y);
-	WREG32(EVERGREEN_CUR_HOT_SPOT + amdgpu_crtc->crtc_offset, (xorigin << 16) | yorigin);
-	WREG32(EVERGREEN_CUR_SIZE + amdgpu_crtc->crtc_offset,
-	       ((w - 1) << 16) | (amdgpu_crtc->cursor_height - 1));
+	WREG32(mmCUR_POSITION + amdgpu_crtc->crtc_offset, (x << 16) | y);
+	WREG32(mmCUR_HOT_SPOT + amdgpu_crtc->crtc_offset, (xorigin << 16) | yorigin);
 
-	amdgpu_crtc->cursor_x = x;
-	amdgpu_crtc->cursor_y = y;
 	return 0;
 }
 
@@ -1978,6 +1903,7 @@ static int dce_v6_0_crtc_cursor_set2(struct drm_crtc *crtc,
 				     int32_t hot_y)
 {
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
 	struct drm_gem_object *obj;
 	struct amdgpu_bo *aobj;
 	int ret;
@@ -2016,12 +1942,11 @@ static int dce_v6_0_crtc_cursor_set2(struct drm_crtc *crtc,
 		return ret;
 	}
 
-	amdgpu_crtc->cursor_width = width;
-	amdgpu_crtc->cursor_height = height;
-
 	dce_v6_0_lock_cursor(crtc, true);
 
-	if (hot_x != amdgpu_crtc->cursor_hot_x ||
+	if (width != amdgpu_crtc->cursor_width ||
+	    height != amdgpu_crtc->cursor_height ||
+	    hot_x != amdgpu_crtc->cursor_hot_x ||
 	    hot_y != amdgpu_crtc->cursor_hot_y) {
 		int x, y;
 
@@ -2030,10 +1955,20 @@ static int dce_v6_0_crtc_cursor_set2(struct drm_crtc *crtc,
 
 		dce_v6_0_cursor_move_locked(crtc, x, y);
 
+		amdgpu_crtc->cursor_width = width;
+		amdgpu_crtc->cursor_height = height;
 		amdgpu_crtc->cursor_hot_x = hot_x;
 		amdgpu_crtc->cursor_hot_y = hot_y;
 	}
 
+	if (width != amdgpu_crtc->cursor_width ||
+	    height != amdgpu_crtc->cursor_height) {
+		WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
+		       (width - 1) << 16 | (height - 1));
+		amdgpu_crtc->cursor_width = width;
+		amdgpu_crtc->cursor_height = height;
+	}
+
 	dce_v6_0_show_cursor(crtc);
 	dce_v6_0_lock_cursor(crtc, false);
 
@@ -2055,6 +1990,7 @@ unpin:
 static void dce_v6_0_cursor_reset(struct drm_crtc *crtc)
 {
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
 
 	if (amdgpu_crtc->cursor_bo) {
 		dce_v6_0_lock_cursor(crtc, true);
@@ -2062,6 +1998,10 @@ static void dce_v6_0_cursor_reset(struct drm_crtc *crtc)
 		dce_v6_0_cursor_move_locked(crtc, amdgpu_crtc->cursor_x,
 					    amdgpu_crtc->cursor_y);
 
+		WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
+		       (amdgpu_crtc->cursor_width - 1) << 16 |
+		       (amdgpu_crtc->cursor_height - 1));
+
 		dce_v6_0_show_cursor(crtc);
 		dce_v6_0_lock_cursor(crtc, false);
 	}
@@ -2117,13 +2057,13 @@ static void dce_v6_0_crtc_dpms(struct drm_crtc *crtc, int mode)
 		type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id);
 		amdgpu_irq_update(adev, &adev->crtc_irq, type);
 		amdgpu_irq_update(adev, &adev->pageflip_irq, type);
-		drm_vblank_post_modeset(dev, amdgpu_crtc->crtc_id);
+		drm_crtc_vblank_on(crtc);
 		dce_v6_0_crtc_load_lut(crtc);
 		break;
 	case DRM_MODE_DPMS_STANDBY:
 	case DRM_MODE_DPMS_SUSPEND:
 	case DRM_MODE_DPMS_OFF:
-		drm_vblank_pre_modeset(dev, amdgpu_crtc->crtc_id);
+		drm_crtc_vblank_off(crtc);
 		if (amdgpu_crtc->enabled)
 			amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE);
 		amdgpu_atombios_crtc_enable(crtc, ATOM_DISABLE);
@@ -2338,21 +2278,20 @@ static int dce_v6_0_early_init(void *handle)
 	dce_v6_0_set_display_funcs(adev);
 	dce_v6_0_set_irq_funcs(adev);
 
+	adev->mode_info.num_crtc = dce_v6_0_get_num_crtc(adev);
+
 	switch (adev->asic_type) {
 	case CHIP_TAHITI:
 	case CHIP_PITCAIRN:
 	case CHIP_VERDE:
-		adev->mode_info.num_crtc = 6;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 6;
 		break;
 	case CHIP_OLAND:
-		adev->mode_info.num_crtc = 2;
 		adev->mode_info.num_hpd = 2;
 		adev->mode_info.num_dig = 2;
 		break;
 	default:
-		/* FIXME: not supported yet */
 		return -EINVAL;
 	}
 
@@ -2556,14 +2495,14 @@ static void dce_v6_0_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
 
 	switch (state) {
 	case AMDGPU_IRQ_STATE_DISABLE:
-		interrupt_mask = RREG32(INT_MASK + reg_block);
+		interrupt_mask = RREG32(mmINT_MASK + reg_block);
 		interrupt_mask &= ~VBLANK_INT_MASK;
-		WREG32(INT_MASK + reg_block, interrupt_mask);
+		WREG32(mmINT_MASK + reg_block, interrupt_mask);
 		break;
 	case AMDGPU_IRQ_STATE_ENABLE:
-		interrupt_mask = RREG32(INT_MASK + reg_block);
+		interrupt_mask = RREG32(mmINT_MASK + reg_block);
 		interrupt_mask |= VBLANK_INT_MASK;
-		WREG32(INT_MASK + reg_block, interrupt_mask);
+		WREG32(mmINT_MASK + reg_block, interrupt_mask);
 		break;
 	default:
 		break;
@@ -2582,42 +2521,23 @@ static int dce_v6_0_set_hpd_interrupt_state(struct amdgpu_device *adev,
 					    unsigned type,
 					    enum amdgpu_interrupt_state state)
 {
-	u32 dc_hpd_int_cntl_reg, dc_hpd_int_cntl;
+	u32 dc_hpd_int_cntl;
 
-	switch (type) {
-	case AMDGPU_HPD_1:
-		dc_hpd_int_cntl_reg = DC_HPD1_INT_CONTROL;
-		break;
-	case AMDGPU_HPD_2:
-		dc_hpd_int_cntl_reg = DC_HPD2_INT_CONTROL;
-		break;
-	case AMDGPU_HPD_3:
-		dc_hpd_int_cntl_reg = DC_HPD3_INT_CONTROL;
-		break;
-	case AMDGPU_HPD_4:
-		dc_hpd_int_cntl_reg = DC_HPD4_INT_CONTROL;
-		break;
-	case AMDGPU_HPD_5:
-		dc_hpd_int_cntl_reg = DC_HPD5_INT_CONTROL;
-		break;
-	case AMDGPU_HPD_6:
-		dc_hpd_int_cntl_reg = DC_HPD6_INT_CONTROL;
-		break;
-	default:
+	if (type >= adev->mode_info.num_hpd) {
 		DRM_DEBUG("invalid hdp %d\n", type);
 		return 0;
 	}
 
 	switch (state) {
 	case AMDGPU_IRQ_STATE_DISABLE:
-		dc_hpd_int_cntl = RREG32(dc_hpd_int_cntl_reg);
-		dc_hpd_int_cntl &= ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
-		WREG32(dc_hpd_int_cntl_reg, dc_hpd_int_cntl);
+		dc_hpd_int_cntl = RREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[type]);
+		dc_hpd_int_cntl &= ~DC_HPDx_INT_EN;
+		WREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[type], dc_hpd_int_cntl);
 		break;
 	case AMDGPU_IRQ_STATE_ENABLE:
-		dc_hpd_int_cntl = RREG32(dc_hpd_int_cntl_reg);
-		dc_hpd_int_cntl |= (DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
-		WREG32(dc_hpd_int_cntl_reg, dc_hpd_int_cntl);
+		dc_hpd_int_cntl = RREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[type]);
+		dc_hpd_int_cntl |= DC_HPDx_INT_EN;
+		WREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[type], dc_hpd_int_cntl);
 		break;
 	default:
 		break;
@@ -2685,7 +2605,7 @@ static int dce_v6_0_crtc_irq(struct amdgpu_device *adev,
 	switch (entry->src_data) {
 	case 0: /* vblank */
 		if (disp_int & interrupt_status_offsets[crtc].vblank)
-			WREG32(VBLANK_STATUS + crtc_offsets[crtc], VBLANK_ACK);
+			WREG32(mmVBLANK_STATUS + crtc_offsets[crtc], VBLANK_ACK);
 		else
 			DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
 
@@ -2696,7 +2616,7 @@ static int dce_v6_0_crtc_irq(struct amdgpu_device *adev,
 		break;
 	case 1: /* vline */
 		if (disp_int & interrupt_status_offsets[crtc].vline)
-			WREG32(VLINE_STATUS + crtc_offsets[crtc], VLINE_ACK);
+			WREG32(mmVLINE_STATUS + crtc_offsets[crtc], VLINE_ACK);
 		else
 			DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
 
@@ -2722,12 +2642,12 @@ static int dce_v6_0_set_pageflip_interrupt_state(struct amdgpu_device *adev,
 		return -EINVAL;
 	}
 
-	reg = RREG32(GRPH_INT_CONTROL + crtc_offsets[type]);
+	reg = RREG32(mmGRPH_INTERRUPT_CONTROL + crtc_offsets[type]);
 	if (state == AMDGPU_IRQ_STATE_DISABLE)
-		WREG32(GRPH_INT_CONTROL + crtc_offsets[type],
+		WREG32(mmGRPH_INTERRUPT_CONTROL + crtc_offsets[type],
 		       reg & ~GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
 	else
-		WREG32(GRPH_INT_CONTROL + crtc_offsets[type],
+		WREG32(mmGRPH_INTERRUPT_CONTROL + crtc_offsets[type],
 		       reg | GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK);
 
 	return 0;
@@ -2750,9 +2670,9 @@ static int dce_v6_0_pageflip_irq(struct amdgpu_device *adev,
 		return -EINVAL;
 	}
 
-	if (RREG32(GRPH_INT_STATUS + crtc_offsets[crtc_id]) &
+	if (RREG32(mmGRPH_INTERRUPT_STATUS + crtc_offsets[crtc_id]) &
 	    GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_OCCURRED_MASK)
-		WREG32(GRPH_INT_STATUS + crtc_offsets[crtc_id],
+		WREG32(mmGRPH_INTERRUPT_STATUS + crtc_offsets[crtc_id],
 		       GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR_MASK);
 
 	/* IRQ could occur when in initial stage */
@@ -2790,7 +2710,7 @@ static int dce_v6_0_hpd_irq(struct amdgpu_device *adev,
 			    struct amdgpu_irq_src *source,
 			    struct amdgpu_iv_entry *entry)
 {
-	uint32_t disp_int, mask, int_control, tmp;
+	uint32_t disp_int, mask, tmp;
 	unsigned hpd;
 
 	if (entry->src_data >= adev->mode_info.num_hpd) {
@@ -2801,12 +2721,11 @@ static int dce_v6_0_hpd_irq(struct amdgpu_device *adev,
 	hpd = entry->src_data;
 	disp_int = RREG32(interrupt_status_offsets[hpd].reg);
 	mask = interrupt_status_offsets[hpd].hpd;
-	int_control = hpd_int_control_offsets[hpd];
 
 	if (disp_int & mask) {
-		tmp = RREG32(int_control);
+		tmp = RREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[hpd]);
 		tmp |= DC_HPD1_INT_CONTROL__DC_HPD1_INT_ACK_MASK;
-		WREG32(int_control, tmp);
+		WREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[hpd], tmp);
 		schedule_work(&adev->hotplug_work);
 		DRM_INFO("IH: HPD%d\n", hpd + 1);
 	}
@@ -2827,7 +2746,7 @@ static int dce_v6_0_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs dce_v6_0_ip_funcs = {
+static const struct amd_ip_funcs dce_v6_0_ip_funcs = {
 	.name = "dce_v6_0",
 	.early_init = dce_v6_0_early_init,
 	.late_init = NULL,
@@ -3122,7 +3041,6 @@ static const struct amdgpu_display_funcs dce_v6_0_display_funcs = {
 	.bandwidth_update = &dce_v6_0_bandwidth_update,
 	.vblank_get_counter = &dce_v6_0_vblank_get_counter,
 	.vblank_wait = &dce_v6_0_vblank_wait,
-	.is_display_hung = &dce_v6_0_is_display_hung,
 	.backlight_set_level = &amdgpu_atombios_encoder_set_backlight_level,
 	.backlight_get_level = &amdgpu_atombios_encoder_get_backlight_level,
 	.hpd_sense = &dce_v6_0_hpd_sense,
@@ -3168,3 +3086,21 @@ static void dce_v6_0_set_irq_funcs(struct amdgpu_device *adev)
 	adev->hpd_irq.num_types = AMDGPU_HPD_LAST;
 	adev->hpd_irq.funcs = &dce_v6_0_hpd_irq_funcs;
 }
+
+const struct amdgpu_ip_block_version dce_v6_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 6,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &dce_v6_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version dce_v6_4_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 6,
+	.minor = 4,
+	.rev = 0,
+	.funcs = &dce_v6_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.h b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.h
index 6a5528105bb6..7b546b596de1 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.h
@@ -24,6 +24,9 @@
 #ifndef __DCE_V6_0_H__
 #define __DCE_V6_0_H__
 
-extern const struct amd_ip_funcs dce_v6_0_ip_funcs;
+extern const struct amdgpu_ip_block_version dce_v6_0_ip_block;
+extern const struct amdgpu_ip_block_version dce_v6_4_ip_block;
+
+void dce_v6_0_disable_dce(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 8c4d808db0f1..6ce7fb42dbef 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -31,6 +31,7 @@
 #include "atombios_encoders.h"
 #include "amdgpu_pll.h"
 #include "amdgpu_connectors.h"
+#include "dce_v8_0.h"
 
 #include "dce/dce_8_0_d.h"
 #include "dce/dce_8_0_sh_mask.h"
@@ -56,6 +57,16 @@ static const u32 crtc_offsets[6] =
 	CRTC5_REGISTER_OFFSET
 };
 
+static const u32 hpd_offsets[] =
+{
+	HPD0_REGISTER_OFFSET,
+	HPD1_REGISTER_OFFSET,
+	HPD2_REGISTER_OFFSET,
+	HPD3_REGISTER_OFFSET,
+	HPD4_REGISTER_OFFSET,
+	HPD5_REGISTER_OFFSET
+};
+
 static const uint32_t dig_offsets[] = {
 	CRTC0_REGISTER_OFFSET,
 	CRTC1_REGISTER_OFFSET,
@@ -104,15 +115,6 @@ static const struct {
 	.hpd = DISP_INTERRUPT_STATUS_CONTINUE5__DC_HPD6_INTERRUPT_MASK
 } };
 
-static const uint32_t hpd_int_control_offsets[6] = {
-	mmDC_HPD1_INT_CONTROL,
-	mmDC_HPD2_INT_CONTROL,
-	mmDC_HPD3_INT_CONTROL,
-	mmDC_HPD4_INT_CONTROL,
-	mmDC_HPD5_INT_CONTROL,
-	mmDC_HPD6_INT_CONTROL,
-};
-
 static u32 dce_v8_0_audio_endpt_rreg(struct amdgpu_device *adev,
 				     u32 block_offset, u32 reg)
 {
@@ -278,34 +280,12 @@ static bool dce_v8_0_hpd_sense(struct amdgpu_device *adev,
 {
 	bool connected = false;
 
-	switch (hpd) {
-	case AMDGPU_HPD_1:
-		if (RREG32(mmDC_HPD1_INT_STATUS) & DC_HPD1_INT_STATUS__DC_HPD1_SENSE_MASK)
-			connected = true;
-		break;
-	case AMDGPU_HPD_2:
-		if (RREG32(mmDC_HPD2_INT_STATUS) & DC_HPD2_INT_STATUS__DC_HPD2_SENSE_MASK)
-			connected = true;
-		break;
-	case AMDGPU_HPD_3:
-		if (RREG32(mmDC_HPD3_INT_STATUS) & DC_HPD3_INT_STATUS__DC_HPD3_SENSE_MASK)
-			connected = true;
-		break;
-	case AMDGPU_HPD_4:
-		if (RREG32(mmDC_HPD4_INT_STATUS) & DC_HPD4_INT_STATUS__DC_HPD4_SENSE_MASK)
-			connected = true;
-		break;
-	case AMDGPU_HPD_5:
-		if (RREG32(mmDC_HPD5_INT_STATUS) & DC_HPD5_INT_STATUS__DC_HPD5_SENSE_MASK)
-			connected = true;
-		break;
-	case AMDGPU_HPD_6:
-		if (RREG32(mmDC_HPD6_INT_STATUS) & DC_HPD6_INT_STATUS__DC_HPD6_SENSE_MASK)
-			connected = true;
-		break;
-	default:
-		break;
-	}
+	if (hpd >= adev->mode_info.num_hpd)
+		return connected;
+
+	if (RREG32(mmDC_HPD1_INT_STATUS + hpd_offsets[hpd]) &
+	    DC_HPD1_INT_STATUS__DC_HPD1_SENSE_MASK)
+		connected = true;
 
 	return connected;
 }
@@ -324,58 +304,15 @@ static void dce_v8_0_hpd_set_polarity(struct amdgpu_device *adev,
 	u32 tmp;
 	bool connected = dce_v8_0_hpd_sense(adev, hpd);
 
-	switch (hpd) {
-	case AMDGPU_HPD_1:
-		tmp = RREG32(mmDC_HPD1_INT_CONTROL);
-		if (connected)
-			tmp &= ~DC_HPD1_INT_CONTROL__DC_HPD1_INT_POLARITY_MASK;
-		else
-			tmp |= DC_HPD1_INT_CONTROL__DC_HPD1_INT_POLARITY_MASK;
-		WREG32(mmDC_HPD1_INT_CONTROL, tmp);
-		break;
-	case AMDGPU_HPD_2:
-		tmp = RREG32(mmDC_HPD2_INT_CONTROL);
-		if (connected)
-			tmp &= ~DC_HPD2_INT_CONTROL__DC_HPD2_INT_POLARITY_MASK;
-		else
-			tmp |= DC_HPD2_INT_CONTROL__DC_HPD2_INT_POLARITY_MASK;
-		WREG32(mmDC_HPD2_INT_CONTROL, tmp);
-		break;
-	case AMDGPU_HPD_3:
-		tmp = RREG32(mmDC_HPD3_INT_CONTROL);
-		if (connected)
-			tmp &= ~DC_HPD3_INT_CONTROL__DC_HPD3_INT_POLARITY_MASK;
-		else
-			tmp |= DC_HPD3_INT_CONTROL__DC_HPD3_INT_POLARITY_MASK;
-		WREG32(mmDC_HPD3_INT_CONTROL, tmp);
-		break;
-	case AMDGPU_HPD_4:
-		tmp = RREG32(mmDC_HPD4_INT_CONTROL);
-		if (connected)
-			tmp &= ~DC_HPD4_INT_CONTROL__DC_HPD4_INT_POLARITY_MASK;
-		else
-			tmp |= DC_HPD4_INT_CONTROL__DC_HPD4_INT_POLARITY_MASK;
-		WREG32(mmDC_HPD4_INT_CONTROL, tmp);
-		break;
-	case AMDGPU_HPD_5:
-		tmp = RREG32(mmDC_HPD5_INT_CONTROL);
-		if (connected)
-			tmp &= ~DC_HPD5_INT_CONTROL__DC_HPD5_INT_POLARITY_MASK;
-		else
-			tmp |= DC_HPD5_INT_CONTROL__DC_HPD5_INT_POLARITY_MASK;
-		WREG32(mmDC_HPD5_INT_CONTROL, tmp);
-			break;
-	case AMDGPU_HPD_6:
-		tmp = RREG32(mmDC_HPD6_INT_CONTROL);
-		if (connected)
-			tmp &= ~DC_HPD6_INT_CONTROL__DC_HPD6_INT_POLARITY_MASK;
-		else
-			tmp |= DC_HPD6_INT_CONTROL__DC_HPD6_INT_POLARITY_MASK;
-		WREG32(mmDC_HPD6_INT_CONTROL, tmp);
-		break;
-	default:
-		break;
-	}
+	if (hpd >= adev->mode_info.num_hpd)
+		return;
+
+	tmp = RREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[hpd]);
+	if (connected)
+		tmp &= ~DC_HPD1_INT_CONTROL__DC_HPD1_INT_POLARITY_MASK;
+	else
+		tmp |= DC_HPD1_INT_CONTROL__DC_HPD1_INT_POLARITY_MASK;
+	WREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[hpd], tmp);
 }
 
 /**
@@ -390,35 +327,17 @@ static void dce_v8_0_hpd_init(struct amdgpu_device *adev)
 {
 	struct drm_device *dev = adev->ddev;
 	struct drm_connector *connector;
-	u32 tmp = (0x9c4 << DC_HPD1_CONTROL__DC_HPD1_CONNECTION_TIMER__SHIFT) |
-		(0xfa << DC_HPD1_CONTROL__DC_HPD1_RX_INT_TIMER__SHIFT) |
-		DC_HPD1_CONTROL__DC_HPD1_EN_MASK;
+	u32 tmp;
 
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-		switch (amdgpu_connector->hpd.hpd) {
-		case AMDGPU_HPD_1:
-			WREG32(mmDC_HPD1_CONTROL, tmp);
-			break;
-		case AMDGPU_HPD_2:
-			WREG32(mmDC_HPD2_CONTROL, tmp);
-			break;
-		case AMDGPU_HPD_3:
-			WREG32(mmDC_HPD3_CONTROL, tmp);
-			break;
-		case AMDGPU_HPD_4:
-			WREG32(mmDC_HPD4_CONTROL, tmp);
-			break;
-		case AMDGPU_HPD_5:
-			WREG32(mmDC_HPD5_CONTROL, tmp);
-			break;
-		case AMDGPU_HPD_6:
-			WREG32(mmDC_HPD6_CONTROL, tmp);
-			break;
-		default:
-			break;
-		}
+		if (amdgpu_connector->hpd.hpd >= adev->mode_info.num_hpd)
+			continue;
+
+		tmp = RREG32(mmDC_HPD1_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd]);
+		tmp |= DC_HPD1_CONTROL__DC_HPD1_EN_MASK;
+		WREG32(mmDC_HPD1_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd], tmp);
 
 		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
 		    connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
@@ -427,34 +346,9 @@ static void dce_v8_0_hpd_init(struct amdgpu_device *adev)
 			 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
 			 * also avoid interrupt storms during dpms.
 			 */
-			u32 dc_hpd_int_cntl_reg, dc_hpd_int_cntl;
-
-			switch (amdgpu_connector->hpd.hpd) {
-			case AMDGPU_HPD_1:
-				dc_hpd_int_cntl_reg = mmDC_HPD1_INT_CONTROL;
-				break;
-			case AMDGPU_HPD_2:
-				dc_hpd_int_cntl_reg = mmDC_HPD2_INT_CONTROL;
-				break;
-			case AMDGPU_HPD_3:
-				dc_hpd_int_cntl_reg = mmDC_HPD3_INT_CONTROL;
-				break;
-			case AMDGPU_HPD_4:
-				dc_hpd_int_cntl_reg = mmDC_HPD4_INT_CONTROL;
-				break;
-			case AMDGPU_HPD_5:
-				dc_hpd_int_cntl_reg = mmDC_HPD5_INT_CONTROL;
-				break;
-			case AMDGPU_HPD_6:
-				dc_hpd_int_cntl_reg = mmDC_HPD6_INT_CONTROL;
-				break;
-			default:
-				continue;
-			}
-
-			dc_hpd_int_cntl = RREG32(dc_hpd_int_cntl_reg);
-			dc_hpd_int_cntl &= ~DC_HPD1_INT_CONTROL__DC_HPD1_INT_EN_MASK;
-			WREG32(dc_hpd_int_cntl_reg, dc_hpd_int_cntl);
+			tmp = RREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd]);
+			tmp &= ~DC_HPD1_INT_CONTROL__DC_HPD1_INT_EN_MASK;
+			WREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd], tmp);
 			continue;
 		}
 
@@ -475,32 +369,18 @@ static void dce_v8_0_hpd_fini(struct amdgpu_device *adev)
 {
 	struct drm_device *dev = adev->ddev;
 	struct drm_connector *connector;
+	u32 tmp;
 
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 
-		switch (amdgpu_connector->hpd.hpd) {
-		case AMDGPU_HPD_1:
-			WREG32(mmDC_HPD1_CONTROL, 0);
-			break;
-		case AMDGPU_HPD_2:
-			WREG32(mmDC_HPD2_CONTROL, 0);
-			break;
-		case AMDGPU_HPD_3:
-			WREG32(mmDC_HPD3_CONTROL, 0);
-			break;
-		case AMDGPU_HPD_4:
-			WREG32(mmDC_HPD4_CONTROL, 0);
-			break;
-		case AMDGPU_HPD_5:
-			WREG32(mmDC_HPD5_CONTROL, 0);
-			break;
-		case AMDGPU_HPD_6:
-			WREG32(mmDC_HPD6_CONTROL, 0);
-			break;
-		default:
-			break;
-		}
+		if (amdgpu_connector->hpd.hpd >= adev->mode_info.num_hpd)
+			continue;
+
+		tmp = RREG32(mmDC_HPD1_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd]);
+		tmp &= ~DC_HPD1_CONTROL__DC_HPD1_EN_MASK;
+		WREG32(mmDC_HPD1_CONTROL + hpd_offsets[amdgpu_connector->hpd.hpd], 0);
+
 		amdgpu_irq_put(adev, &adev->hpd_irq, amdgpu_connector->hpd.hpd);
 	}
 }
@@ -2030,7 +1910,7 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 	u32 viewport_w, viewport_h;
 	int r;
 	bool bypass_lut = false;
-	char *format_name;
+	struct drm_format_name_buf format_name;
 
 	/* no fb bound */
 	if (!atomic && !crtc->primary->fb) {
@@ -2135,9 +2015,8 @@ static int dce_v8_0_crtc_do_set_base(struct drm_crtc *crtc,
 		bypass_lut = true;
 		break;
 	default:
-		format_name = drm_get_format_name(target_fb->pixel_format);
-		DRM_ERROR("Unsupported screen format %s\n", format_name);
-		kfree(format_name);
+		DRM_ERROR("Unsupported screen format %s\n",
+		          drm_get_format_name(target_fb->pixel_format, &format_name));
 		return -EINVAL;
 	}
 
@@ -2465,6 +2344,9 @@ static int dce_v8_0_cursor_move_locked(struct drm_crtc *crtc,
 	struct amdgpu_device *adev = crtc->dev->dev_private;
 	int xorigin = 0, yorigin = 0;
 
+	amdgpu_crtc->cursor_x = x;
+	amdgpu_crtc->cursor_y = y;
+
 	/* avivo cursor are offset into the total surface */
 	x += crtc->x;
 	y += crtc->y;
@@ -2481,11 +2363,6 @@ static int dce_v8_0_cursor_move_locked(struct drm_crtc *crtc,
 
 	WREG32(mmCUR_POSITION + amdgpu_crtc->crtc_offset, (x << 16) | y);
 	WREG32(mmCUR_HOT_SPOT + amdgpu_crtc->crtc_offset, (xorigin << 16) | yorigin);
-	WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
-	       ((amdgpu_crtc->cursor_width - 1) << 16) | (amdgpu_crtc->cursor_height - 1));
-
-	amdgpu_crtc->cursor_x = x;
-	amdgpu_crtc->cursor_y = y;
 
 	return 0;
 }
@@ -2511,6 +2388,7 @@ static int dce_v8_0_crtc_cursor_set2(struct drm_crtc *crtc,
 				     int32_t hot_y)
 {
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
 	struct drm_gem_object *obj;
 	struct amdgpu_bo *aobj;
 	int ret;
@@ -2549,9 +2427,6 @@ static int dce_v8_0_crtc_cursor_set2(struct drm_crtc *crtc,
 		return ret;
 	}
 
-	amdgpu_crtc->cursor_width = width;
-	amdgpu_crtc->cursor_height = height;
-
 	dce_v8_0_lock_cursor(crtc, true);
 
 	if (hot_x != amdgpu_crtc->cursor_hot_x ||
@@ -2563,10 +2438,20 @@ static int dce_v8_0_crtc_cursor_set2(struct drm_crtc *crtc,
 
 		dce_v8_0_cursor_move_locked(crtc, x, y);
 
+		amdgpu_crtc->cursor_width = width;
+		amdgpu_crtc->cursor_height = height;
 		amdgpu_crtc->cursor_hot_x = hot_x;
 		amdgpu_crtc->cursor_hot_y = hot_y;
 	}
 
+	if (width != amdgpu_crtc->cursor_width ||
+	    height != amdgpu_crtc->cursor_height) {
+		WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
+		       (width - 1) << 16 | (height - 1));
+		amdgpu_crtc->cursor_width = width;
+		amdgpu_crtc->cursor_height = height;
+	}
+
 	dce_v8_0_show_cursor(crtc);
 	dce_v8_0_lock_cursor(crtc, false);
 
@@ -2588,6 +2473,7 @@ unpin:
 static void dce_v8_0_cursor_reset(struct drm_crtc *crtc)
 {
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
 
 	if (amdgpu_crtc->cursor_bo) {
 		dce_v8_0_lock_cursor(crtc, true);
@@ -2595,6 +2481,10 @@ static void dce_v8_0_cursor_reset(struct drm_crtc *crtc)
 		dce_v8_0_cursor_move_locked(crtc, amdgpu_crtc->cursor_x,
 					    amdgpu_crtc->cursor_y);
 
+		WREG32(mmCUR_SIZE + amdgpu_crtc->crtc_offset,
+		       (amdgpu_crtc->cursor_width - 1) << 16 |
+		       (amdgpu_crtc->cursor_height - 1));
+
 		dce_v8_0_show_cursor(crtc);
 
 		dce_v8_0_lock_cursor(crtc, false);
@@ -3198,42 +3088,23 @@ static int dce_v8_0_set_hpd_interrupt_state(struct amdgpu_device *adev,
 					    unsigned type,
 					    enum amdgpu_interrupt_state state)
 {
-	u32 dc_hpd_int_cntl_reg, dc_hpd_int_cntl;
+	u32 dc_hpd_int_cntl;
 
-	switch (type) {
-	case AMDGPU_HPD_1:
-		dc_hpd_int_cntl_reg = mmDC_HPD1_INT_CONTROL;
-		break;
-	case AMDGPU_HPD_2:
-		dc_hpd_int_cntl_reg = mmDC_HPD2_INT_CONTROL;
-		break;
-	case AMDGPU_HPD_3:
-		dc_hpd_int_cntl_reg = mmDC_HPD3_INT_CONTROL;
-		break;
-	case AMDGPU_HPD_4:
-		dc_hpd_int_cntl_reg = mmDC_HPD4_INT_CONTROL;
-		break;
-	case AMDGPU_HPD_5:
-		dc_hpd_int_cntl_reg = mmDC_HPD5_INT_CONTROL;
-		break;
-	case AMDGPU_HPD_6:
-		dc_hpd_int_cntl_reg = mmDC_HPD6_INT_CONTROL;
-		break;
-	default:
+	if (type >= adev->mode_info.num_hpd) {
 		DRM_DEBUG("invalid hdp %d\n", type);
 		return 0;
 	}
 
 	switch (state) {
 	case AMDGPU_IRQ_STATE_DISABLE:
-		dc_hpd_int_cntl = RREG32(dc_hpd_int_cntl_reg);
+		dc_hpd_int_cntl = RREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[type]);
 		dc_hpd_int_cntl &= ~DC_HPD1_INT_CONTROL__DC_HPD1_INT_EN_MASK;
-		WREG32(dc_hpd_int_cntl_reg, dc_hpd_int_cntl);
+		WREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[type], dc_hpd_int_cntl);
 		break;
 	case AMDGPU_IRQ_STATE_ENABLE:
-		dc_hpd_int_cntl = RREG32(dc_hpd_int_cntl_reg);
+		dc_hpd_int_cntl = RREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[type]);
 		dc_hpd_int_cntl |= DC_HPD1_INT_CONTROL__DC_HPD1_INT_EN_MASK;
-		WREG32(dc_hpd_int_cntl_reg, dc_hpd_int_cntl);
+		WREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[type], dc_hpd_int_cntl);
 		break;
 	default:
 		break;
@@ -3406,7 +3277,7 @@ static int dce_v8_0_hpd_irq(struct amdgpu_device *adev,
 			    struct amdgpu_irq_src *source,
 			    struct amdgpu_iv_entry *entry)
 {
-	uint32_t disp_int, mask, int_control, tmp;
+	uint32_t disp_int, mask, tmp;
 	unsigned hpd;
 
 	if (entry->src_data >= adev->mode_info.num_hpd) {
@@ -3417,12 +3288,11 @@ static int dce_v8_0_hpd_irq(struct amdgpu_device *adev,
 	hpd = entry->src_data;
 	disp_int = RREG32(interrupt_status_offsets[hpd].reg);
 	mask = interrupt_status_offsets[hpd].hpd;
-	int_control = hpd_int_control_offsets[hpd];
 
 	if (disp_int & mask) {
-		tmp = RREG32(int_control);
+		tmp = RREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[hpd]);
 		tmp |= DC_HPD1_INT_CONTROL__DC_HPD1_INT_ACK_MASK;
-		WREG32(int_control, tmp);
+		WREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[hpd], tmp);
 		schedule_work(&adev->hotplug_work);
 		DRM_DEBUG("IH: HPD%d\n", hpd + 1);
 	}
@@ -3443,7 +3313,7 @@ static int dce_v8_0_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs dce_v8_0_ip_funcs = {
+static const struct amd_ip_funcs dce_v8_0_ip_funcs = {
 	.name = "dce_v8_0",
 	.early_init = dce_v8_0_early_init,
 	.late_init = NULL,
@@ -3727,7 +3597,6 @@ static const struct amdgpu_display_funcs dce_v8_0_display_funcs = {
 	.bandwidth_update = &dce_v8_0_bandwidth_update,
 	.vblank_get_counter = &dce_v8_0_vblank_get_counter,
 	.vblank_wait = &dce_v8_0_vblank_wait,
-	.is_display_hung = &dce_v8_0_is_display_hung,
 	.backlight_set_level = &amdgpu_atombios_encoder_set_backlight_level,
 	.backlight_get_level = &amdgpu_atombios_encoder_get_backlight_level,
 	.hpd_sense = &dce_v8_0_hpd_sense,
@@ -3773,3 +3642,48 @@ static void dce_v8_0_set_irq_funcs(struct amdgpu_device *adev)
 	adev->hpd_irq.num_types = AMDGPU_HPD_LAST;
 	adev->hpd_irq.funcs = &dce_v8_0_hpd_irq_funcs;
 }
+
+const struct amdgpu_ip_block_version dce_v8_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 8,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &dce_v8_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version dce_v8_1_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 8,
+	.minor = 1,
+	.rev = 0,
+	.funcs = &dce_v8_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version dce_v8_2_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 8,
+	.minor = 2,
+	.rev = 0,
+	.funcs = &dce_v8_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version dce_v8_3_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 8,
+	.minor = 3,
+	.rev = 0,
+	.funcs = &dce_v8_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version dce_v8_5_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 8,
+	.minor = 5,
+	.rev = 0,
+	.funcs = &dce_v8_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.h b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.h
index 7d0770c3a49b..13b802dd946a 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.h
@@ -24,7 +24,11 @@
 #ifndef __DCE_V8_0_H__
 #define __DCE_V8_0_H__
 
-extern const struct amd_ip_funcs dce_v8_0_ip_funcs;
+extern const struct amdgpu_ip_block_version dce_v8_0_ip_block;
+extern const struct amdgpu_ip_block_version dce_v8_1_ip_block;
+extern const struct amdgpu_ip_block_version dce_v8_2_ip_block;
+extern const struct amdgpu_ip_block_version dce_v8_3_ip_block;
+extern const struct amdgpu_ip_block_version dce_v8_5_ip_block;
 
 void dce_v8_0_disable_dce(struct amdgpu_device *adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
index c2bd9f045532..e4a5a5ac0ff3 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
@@ -27,6 +27,9 @@
 #include "atom.h"
 #include "amdgpu_pll.h"
 #include "amdgpu_connectors.h"
+#ifdef CONFIG_DRM_AMDGPU_SI
+#include "dce_v6_0.h"
+#endif
 #ifdef CONFIG_DRM_AMDGPU_CIK
 #include "dce_v8_0.h"
 #endif
@@ -34,11 +37,13 @@
 #include "dce_v11_0.h"
 #include "dce_virtual.h"
 
+#define DCE_VIRTUAL_VBLANK_PERIOD 16666666
+
+
 static void dce_virtual_set_display_funcs(struct amdgpu_device *adev);
 static void dce_virtual_set_irq_funcs(struct amdgpu_device *adev);
-static int dce_virtual_pageflip_irq(struct amdgpu_device *adev,
-				  struct amdgpu_irq_src *source,
-				  struct amdgpu_iv_entry *entry);
+static int dce_virtual_connector_encoder_init(struct amdgpu_device *adev,
+					      int index);
 
 /**
  * dce_virtual_vblank_wait - vblank wait asic callback.
@@ -90,15 +95,18 @@ static u32 dce_virtual_hpd_get_gpio_reg(struct amdgpu_device *adev)
 	return 0;
 }
 
-static bool dce_virtual_is_display_hung(struct amdgpu_device *adev)
-{
-	return false;
-}
-
 static void dce_virtual_stop_mc_access(struct amdgpu_device *adev,
 			      struct amdgpu_mode_mc_save *save)
 {
 	switch (adev->asic_type) {
+#ifdef CONFIG_DRM_AMDGPU_SI
+	case CHIP_TAHITI:
+	case CHIP_PITCAIRN:
+	case CHIP_VERDE:
+	case CHIP_OLAND:
+		dce_v6_0_disable_dce(adev);
+		break;
+#endif
 #ifdef CONFIG_DRM_AMDGPU_CIK
 	case CHIP_BONAIRE:
 	case CHIP_HAWAII:
@@ -119,6 +127,9 @@ static void dce_virtual_stop_mc_access(struct amdgpu_device *adev,
 		dce_v11_0_disable_dce(adev);
 		break;
 	case CHIP_TOPAZ:
+#ifdef CONFIG_DRM_AMDGPU_SI
+	case CHIP_HAINAN:
+#endif
 		/* no DCE */
 		return;
 	default:
@@ -195,16 +206,15 @@ static void dce_virtual_crtc_dpms(struct drm_crtc *crtc, int mode)
 	switch (mode) {
 	case DRM_MODE_DPMS_ON:
 		amdgpu_crtc->enabled = true;
-		/* Make sure VBLANK and PFLIP interrupts are still enabled */
+		/* Make sure VBLANK interrupts are still enabled */
 		type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id);
 		amdgpu_irq_update(adev, &adev->crtc_irq, type);
-		amdgpu_irq_update(adev, &adev->pageflip_irq, type);
-		drm_vblank_on(dev, amdgpu_crtc->crtc_id);
+		drm_crtc_vblank_on(crtc);
 		break;
 	case DRM_MODE_DPMS_STANDBY:
 	case DRM_MODE_DPMS_SUSPEND:
 	case DRM_MODE_DPMS_OFF:
-		drm_vblank_off(dev, amdgpu_crtc->crtc_id);
+		drm_crtc_vblank_off(crtc);
 		amdgpu_crtc->enabled = false;
 		break;
 	}
@@ -264,24 +274,6 @@ static bool dce_virtual_crtc_mode_fixup(struct drm_crtc *crtc,
 				     const struct drm_display_mode *mode,
 				     struct drm_display_mode *adjusted_mode)
 {
-	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-	struct drm_device *dev = crtc->dev;
-	struct drm_encoder *encoder;
-
-	/* assign the encoder to the amdgpu crtc to avoid repeated lookups later */
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		if (encoder->crtc == crtc) {
-			amdgpu_crtc->encoder = encoder;
-			amdgpu_crtc->connector = amdgpu_get_connector_for_encoder(encoder);
-			break;
-		}
-	}
-	if ((amdgpu_crtc->encoder == NULL) || (amdgpu_crtc->connector == NULL)) {
-		amdgpu_crtc->encoder = NULL;
-		amdgpu_crtc->connector = NULL;
-		return false;
-	}
-
 	return true;
 }
 
@@ -341,6 +333,7 @@ static int dce_virtual_crtc_init(struct amdgpu_device *adev, int index)
 	amdgpu_crtc->pll_id = ATOM_PPLL_INVALID;
 	amdgpu_crtc->encoder = NULL;
 	amdgpu_crtc->connector = NULL;
+	amdgpu_crtc->vsync_timer_enabled = AMDGPU_IRQ_STATE_DISABLE;
 	drm_crtc_helper_add(&amdgpu_crtc->base, &dce_virtual_crtc_helper_funcs);
 
 	return 0;
@@ -350,48 +343,121 @@ static int dce_virtual_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	adev->mode_info.vsync_timer_enabled = AMDGPU_IRQ_STATE_DISABLE;
 	dce_virtual_set_display_funcs(adev);
 	dce_virtual_set_irq_funcs(adev);
 
-	adev->mode_info.num_crtc = 1;
 	adev->mode_info.num_hpd = 1;
 	adev->mode_info.num_dig = 1;
 	return 0;
 }
 
-static bool dce_virtual_get_connector_info(struct amdgpu_device *adev)
+static struct drm_encoder *
+dce_virtual_encoder(struct drm_connector *connector)
 {
-	struct amdgpu_i2c_bus_rec ddc_bus;
-	struct amdgpu_router router;
-	struct amdgpu_hpd hpd;
+	int enc_id = connector->encoder_ids[0];
+	struct drm_encoder *encoder;
+	int i;
 
-	/* look up gpio for ddc, hpd */
-	ddc_bus.valid = false;
-	hpd.hpd = AMDGPU_HPD_NONE;
-	/* needed for aux chan transactions */
-	ddc_bus.hpd = hpd.hpd;
+	for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
+		if (connector->encoder_ids[i] == 0)
+			break;
 
-	memset(&router, 0, sizeof(router));
-	router.ddc_valid = false;
-	router.cd_valid = false;
-	amdgpu_display_add_connector(adev,
-				      0,
-				      ATOM_DEVICE_CRT1_SUPPORT,
-				      DRM_MODE_CONNECTOR_VIRTUAL, &ddc_bus,
-				      CONNECTOR_OBJECT_ID_VIRTUAL,
-				      &hpd,
-				      &router);
+		encoder = drm_encoder_find(connector->dev, connector->encoder_ids[i]);
+		if (!encoder)
+			continue;
 
-	amdgpu_display_add_encoder(adev, ENCODER_VIRTUAL_ENUM_VIRTUAL,
-							ATOM_DEVICE_CRT1_SUPPORT,
-							0);
+		if (encoder->encoder_type == DRM_MODE_ENCODER_VIRTUAL)
+			return encoder;
+	}
 
-	amdgpu_link_encoder_connector(adev->ddev);
+	/* pick the first one */
+	if (enc_id)
+		return drm_encoder_find(connector->dev, enc_id);
+	return NULL;
+}
+
+static int dce_virtual_get_modes(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_display_mode *mode = NULL;
+	unsigned i;
+	static const struct mode_size {
+		int w;
+		int h;
+	} common_modes[17] = {
+		{ 640,  480},
+		{ 720,  480},
+		{ 800,  600},
+		{ 848,  480},
+		{1024,  768},
+		{1152,  768},
+		{1280,  720},
+		{1280,  800},
+		{1280,  854},
+		{1280,  960},
+		{1280, 1024},
+		{1440,  900},
+		{1400, 1050},
+		{1680, 1050},
+		{1600, 1200},
+		{1920, 1080},
+		{1920, 1200}
+	};
+
+	for (i = 0; i < 17; i++) {
+		mode = drm_cvt_mode(dev, common_modes[i].w, common_modes[i].h, 60, false, false, false);
+		drm_mode_probed_add(connector, mode);
+	}
 
-	return true;
+	return 0;
+}
+
+static int dce_virtual_mode_valid(struct drm_connector *connector,
+				  struct drm_display_mode *mode)
+{
+	return MODE_OK;
 }
 
+static int
+dce_virtual_dpms(struct drm_connector *connector, int mode)
+{
+	return 0;
+}
+
+static int
+dce_virtual_set_property(struct drm_connector *connector,
+			 struct drm_property *property,
+			 uint64_t val)
+{
+	return 0;
+}
+
+static void dce_virtual_destroy(struct drm_connector *connector)
+{
+	drm_connector_unregister(connector);
+	drm_connector_cleanup(connector);
+	kfree(connector);
+}
+
+static void dce_virtual_force(struct drm_connector *connector)
+{
+	return;
+}
+
+static const struct drm_connector_helper_funcs dce_virtual_connector_helper_funcs = {
+	.get_modes = dce_virtual_get_modes,
+	.mode_valid = dce_virtual_mode_valid,
+	.best_encoder = dce_virtual_encoder,
+};
+
+static const struct drm_connector_funcs dce_virtual_connector_funcs = {
+	.dpms = dce_virtual_dpms,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.set_property = dce_virtual_set_property,
+	.destroy = dce_virtual_destroy,
+	.force = dce_virtual_force,
+};
+
 static int dce_virtual_sw_init(void *handle)
 {
 	int r, i;
@@ -420,16 +486,16 @@ static int dce_virtual_sw_init(void *handle)
 	adev->ddev->mode_config.max_width = 16384;
 	adev->ddev->mode_config.max_height = 16384;
 
-	/* allocate crtcs */
+	/* allocate crtcs, encoders, connectors */
 	for (i = 0; i < adev->mode_info.num_crtc; i++) {
 		r = dce_virtual_crtc_init(adev, i);
 		if (r)
 			return r;
+		r = dce_virtual_connector_encoder_init(adev, i);
+		if (r)
+			return r;
 	}
 
-	dce_virtual_get_connector_info(adev);
-	amdgpu_print_display_setup(adev->ddev);
-
 	drm_kms_helper_poll_init(adev->ddev);
 
 	adev->mode_info.mode_config_initialized = true;
@@ -496,7 +562,7 @@ static int dce_virtual_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs dce_virtual_ip_funcs = {
+static const struct amd_ip_funcs dce_virtual_ip_funcs = {
 	.name = "dce_virtual",
 	.early_init = dce_virtual_early_init,
 	.late_init = NULL,
@@ -526,8 +592,8 @@ static void dce_virtual_encoder_commit(struct drm_encoder *encoder)
 
 static void
 dce_virtual_encoder_mode_set(struct drm_encoder *encoder,
-		      struct drm_display_mode *mode,
-		      struct drm_display_mode *adjusted_mode)
+			     struct drm_display_mode *mode,
+			     struct drm_display_mode *adjusted_mode)
 {
 	return;
 }
@@ -547,10 +613,6 @@ static bool dce_virtual_encoder_mode_fixup(struct drm_encoder *encoder,
 				    const struct drm_display_mode *mode,
 				    struct drm_display_mode *adjusted_mode)
 {
-
-	/* set the active encoder to connector routing */
-	amdgpu_encoder_set_active_device(encoder);
-
 	return true;
 }
 
@@ -576,45 +638,40 @@ static const struct drm_encoder_funcs dce_virtual_encoder_funcs = {
 	.destroy = dce_virtual_encoder_destroy,
 };
 
-static void dce_virtual_encoder_add(struct amdgpu_device *adev,
-				 uint32_t encoder_enum,
-				 uint32_t supported_device,
-				 u16 caps)
+static int dce_virtual_connector_encoder_init(struct amdgpu_device *adev,
+					      int index)
 {
-	struct drm_device *dev = adev->ddev;
 	struct drm_encoder *encoder;
-	struct amdgpu_encoder *amdgpu_encoder;
-
-	/* see if we already added it */
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		amdgpu_encoder = to_amdgpu_encoder(encoder);
-		if (amdgpu_encoder->encoder_enum == encoder_enum) {
-			amdgpu_encoder->devices |= supported_device;
-			return;
-		}
+	struct drm_connector *connector;
+
+	/* add a new encoder */
+	encoder = kzalloc(sizeof(struct drm_encoder), GFP_KERNEL);
+	if (!encoder)
+		return -ENOMEM;
+	encoder->possible_crtcs = 1 << index;
+	drm_encoder_init(adev->ddev, encoder, &dce_virtual_encoder_funcs,
+			 DRM_MODE_ENCODER_VIRTUAL, NULL);
+	drm_encoder_helper_add(encoder, &dce_virtual_encoder_helper_funcs);
 
+	connector = kzalloc(sizeof(struct drm_connector), GFP_KERNEL);
+	if (!connector) {
+		kfree(encoder);
+		return -ENOMEM;
 	}
 
-	/* add a new one */
-	amdgpu_encoder = kzalloc(sizeof(struct amdgpu_encoder), GFP_KERNEL);
-	if (!amdgpu_encoder)
-		return;
+	/* add a new connector */
+	drm_connector_init(adev->ddev, connector, &dce_virtual_connector_funcs,
+			   DRM_MODE_CONNECTOR_VIRTUAL);
+	drm_connector_helper_add(connector, &dce_virtual_connector_helper_funcs);
+	connector->display_info.subpixel_order = SubPixelHorizontalRGB;
+	connector->interlace_allowed = false;
+	connector->doublescan_allowed = false;
+	drm_connector_register(connector);
 
-	encoder = &amdgpu_encoder->base;
-	encoder->possible_crtcs = 0x1;
-	amdgpu_encoder->enc_priv = NULL;
-	amdgpu_encoder->encoder_enum = encoder_enum;
-	amdgpu_encoder->encoder_id = (encoder_enum & OBJECT_ID_MASK) >> OBJECT_ID_SHIFT;
-	amdgpu_encoder->devices = supported_device;
-	amdgpu_encoder->rmx_type = RMX_OFF;
-	amdgpu_encoder->underscan_type = UNDERSCAN_OFF;
-	amdgpu_encoder->is_ext_encoder = false;
-	amdgpu_encoder->caps = caps;
-
-	drm_encoder_init(dev, encoder, &dce_virtual_encoder_funcs,
-					 DRM_MODE_ENCODER_VIRTUAL, NULL);
-	drm_encoder_helper_add(encoder, &dce_virtual_encoder_helper_funcs);
-	DRM_INFO("[FM]encoder: %d is VIRTUAL\n", amdgpu_encoder->encoder_id);
+	/* link them */
+	drm_mode_connector_attach_encoder(connector, encoder);
+
+	return 0;
 }
 
 static const struct amdgpu_display_funcs dce_virtual_display_funcs = {
@@ -622,7 +679,6 @@ static const struct amdgpu_display_funcs dce_virtual_display_funcs = {
 	.bandwidth_update = &dce_virtual_bandwidth_update,
 	.vblank_get_counter = &dce_virtual_vblank_get_counter,
 	.vblank_wait = &dce_virtual_vblank_wait,
-	.is_display_hung = &dce_virtual_is_display_hung,
 	.backlight_set_level = NULL,
 	.backlight_get_level = NULL,
 	.hpd_sense = &dce_virtual_hpd_sense,
@@ -630,8 +686,8 @@ static const struct amdgpu_display_funcs dce_virtual_display_funcs = {
 	.hpd_get_gpio_reg = &dce_virtual_hpd_get_gpio_reg,
 	.page_flip = &dce_virtual_page_flip,
 	.page_flip_get_scanoutpos = &dce_virtual_crtc_get_scanoutpos,
-	.add_encoder = &dce_virtual_encoder_add,
-	.add_connector = &amdgpu_connector_add,
+	.add_encoder = NULL,
+	.add_connector = NULL,
 	.stop_mc_access = &dce_virtual_stop_mc_access,
 	.resume_mc_access = &dce_virtual_resume_mc_access,
 };
@@ -642,107 +698,13 @@ static void dce_virtual_set_display_funcs(struct amdgpu_device *adev)
 		adev->mode_info.funcs = &dce_virtual_display_funcs;
 }
 
-static enum hrtimer_restart dce_virtual_vblank_timer_handle(struct hrtimer *vblank_timer)
-{
-	struct amdgpu_mode_info *mode_info = container_of(vblank_timer, struct amdgpu_mode_info ,vblank_timer);
-	struct amdgpu_device *adev = container_of(mode_info, struct amdgpu_device ,mode_info);
-	unsigned crtc = 0;
-	drm_handle_vblank(adev->ddev, crtc);
-	dce_virtual_pageflip_irq(adev, NULL, NULL);
-	hrtimer_start(vblank_timer, ktime_set(0, DCE_VIRTUAL_VBLANK_PERIOD), HRTIMER_MODE_REL);
-	return HRTIMER_NORESTART;
-}
-
-static void dce_virtual_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
-						     int crtc,
-						     enum amdgpu_interrupt_state state)
-{
-	if (crtc >= adev->mode_info.num_crtc) {
-		DRM_DEBUG("invalid crtc %d\n", crtc);
-		return;
-	}
-
-	if (state && !adev->mode_info.vsync_timer_enabled) {
-		DRM_DEBUG("Enable software vsync timer\n");
-		hrtimer_init(&adev->mode_info.vblank_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
-		hrtimer_set_expires(&adev->mode_info.vblank_timer, ktime_set(0, DCE_VIRTUAL_VBLANK_PERIOD));
-		adev->mode_info.vblank_timer.function = dce_virtual_vblank_timer_handle;
-		hrtimer_start(&adev->mode_info.vblank_timer, ktime_set(0, DCE_VIRTUAL_VBLANK_PERIOD), HRTIMER_MODE_REL);
-	} else if (!state && adev->mode_info.vsync_timer_enabled) {
-		DRM_DEBUG("Disable software vsync timer\n");
-		hrtimer_cancel(&adev->mode_info.vblank_timer);
-	}
-
-	adev->mode_info.vsync_timer_enabled = state;
-	DRM_DEBUG("[FM]set crtc %d vblank interrupt state %d\n", crtc, state);
-}
-
-
-static int dce_virtual_set_crtc_irq_state(struct amdgpu_device *adev,
-                                       struct amdgpu_irq_src *source,
-                                       unsigned type,
-                                       enum amdgpu_interrupt_state state)
-{
-	switch (type) {
-	case AMDGPU_CRTC_IRQ_VBLANK1:
-		dce_virtual_set_crtc_vblank_interrupt_state(adev, 0, state);
-		break;
-	default:
-		break;
-	}
-	return 0;
-}
-
-static void dce_virtual_crtc_vblank_int_ack(struct amdgpu_device *adev,
-					  int crtc)
-{
-	if (crtc >= adev->mode_info.num_crtc) {
-		DRM_DEBUG("invalid crtc %d\n", crtc);
-		return;
-	}
-}
-
-static int dce_virtual_crtc_irq(struct amdgpu_device *adev,
-			      struct amdgpu_irq_src *source,
-			      struct amdgpu_iv_entry *entry)
-{
-	unsigned crtc = 0;
-	unsigned irq_type = AMDGPU_CRTC_IRQ_VBLANK1;
-
-	dce_virtual_crtc_vblank_int_ack(adev, crtc);
-
-	if (amdgpu_irq_enabled(adev, source, irq_type)) {
-		drm_handle_vblank(adev->ddev, crtc);
-	}
-	dce_virtual_pageflip_irq(adev, NULL, NULL);
-	DRM_DEBUG("IH: D%d vblank\n", crtc + 1);
-	return 0;
-}
-
-static int dce_virtual_set_pageflip_irq_state(struct amdgpu_device *adev,
-					    struct amdgpu_irq_src *src,
-					    unsigned type,
-					    enum amdgpu_interrupt_state state)
-{
-	if (type >= adev->mode_info.num_crtc) {
-		DRM_ERROR("invalid pageflip crtc %d\n", type);
-		return -EINVAL;
-	}
-	DRM_DEBUG("[FM]set pageflip irq type %d state %d\n", type, state);
-
-	return 0;
-}
-
-static int dce_virtual_pageflip_irq(struct amdgpu_device *adev,
-				  struct amdgpu_irq_src *source,
-				  struct amdgpu_iv_entry *entry)
+static int dce_virtual_pageflip(struct amdgpu_device *adev,
+				unsigned crtc_id)
 {
 	unsigned long flags;
-	unsigned crtc_id = 0;
 	struct amdgpu_crtc *amdgpu_crtc;
 	struct amdgpu_flip_work *works;
 
-	crtc_id = 0;
 	amdgpu_crtc = adev->mode_info.crtcs[crtc_id];
 
 	if (crtc_id >= adev->mode_info.num_crtc) {
@@ -781,22 +743,79 @@ static int dce_virtual_pageflip_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
+static enum hrtimer_restart dce_virtual_vblank_timer_handle(struct hrtimer *vblank_timer)
+{
+	struct amdgpu_crtc *amdgpu_crtc = container_of(vblank_timer,
+						       struct amdgpu_crtc, vblank_timer);
+	struct drm_device *ddev = amdgpu_crtc->base.dev;
+	struct amdgpu_device *adev = ddev->dev_private;
+
+	drm_handle_vblank(ddev, amdgpu_crtc->crtc_id);
+	dce_virtual_pageflip(adev, amdgpu_crtc->crtc_id);
+	hrtimer_start(vblank_timer, ktime_set(0, DCE_VIRTUAL_VBLANK_PERIOD),
+		      HRTIMER_MODE_REL);
+
+	return HRTIMER_NORESTART;
+}
+
+static void dce_virtual_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
+							int crtc,
+							enum amdgpu_interrupt_state state)
+{
+	if (crtc >= adev->mode_info.num_crtc) {
+		DRM_DEBUG("invalid crtc %d\n", crtc);
+		return;
+	}
+
+	if (state && !adev->mode_info.crtcs[crtc]->vsync_timer_enabled) {
+		DRM_DEBUG("Enable software vsync timer\n");
+		hrtimer_init(&adev->mode_info.crtcs[crtc]->vblank_timer,
+			     CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+		hrtimer_set_expires(&adev->mode_info.crtcs[crtc]->vblank_timer,
+				    ktime_set(0, DCE_VIRTUAL_VBLANK_PERIOD));
+		adev->mode_info.crtcs[crtc]->vblank_timer.function =
+			dce_virtual_vblank_timer_handle;
+		hrtimer_start(&adev->mode_info.crtcs[crtc]->vblank_timer,
+			      ktime_set(0, DCE_VIRTUAL_VBLANK_PERIOD), HRTIMER_MODE_REL);
+	} else if (!state && adev->mode_info.crtcs[crtc]->vsync_timer_enabled) {
+		DRM_DEBUG("Disable software vsync timer\n");
+		hrtimer_cancel(&adev->mode_info.crtcs[crtc]->vblank_timer);
+	}
+
+	adev->mode_info.crtcs[crtc]->vsync_timer_enabled = state;
+	DRM_DEBUG("[FM]set crtc %d vblank interrupt state %d\n", crtc, state);
+}
+
+
+static int dce_virtual_set_crtc_irq_state(struct amdgpu_device *adev,
+					  struct amdgpu_irq_src *source,
+					  unsigned type,
+					  enum amdgpu_interrupt_state state)
+{
+	if (type > AMDGPU_CRTC_IRQ_VBLANK6)
+		return -EINVAL;
+
+	dce_virtual_set_crtc_vblank_interrupt_state(adev, type, state);
+
+	return 0;
+}
+
 static const struct amdgpu_irq_src_funcs dce_virtual_crtc_irq_funcs = {
 	.set = dce_virtual_set_crtc_irq_state,
-	.process = dce_virtual_crtc_irq,
-};
-
-static const struct amdgpu_irq_src_funcs dce_virtual_pageflip_irq_funcs = {
-	.set = dce_virtual_set_pageflip_irq_state,
-	.process = dce_virtual_pageflip_irq,
+	.process = NULL,
 };
 
 static void dce_virtual_set_irq_funcs(struct amdgpu_device *adev)
 {
 	adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST;
 	adev->crtc_irq.funcs = &dce_virtual_crtc_irq_funcs;
-
-	adev->pageflip_irq.num_types = AMDGPU_PAGEFLIP_IRQ_LAST;
-	adev->pageflip_irq.funcs = &dce_virtual_pageflip_irq_funcs;
 }
 
+const struct amdgpu_ip_block_version dce_virtual_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 1,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &dce_virtual_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.h b/drivers/gpu/drm/amd/amdgpu/dce_virtual.h
index e239243f6ebc..ed422012c8c6 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.h
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.h
@@ -24,8 +24,7 @@
 #ifndef __DCE_VIRTUAL_H__
 #define __DCE_VIRTUAL_H__
 
-extern const struct amd_ip_funcs dce_virtual_ip_funcs;
-#define DCE_VIRTUAL_VBLANK_PERIOD 16666666
+extern const struct amdgpu_ip_block_version dce_virtual_ip_block;
 
 #endif
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index 40abb6b81c09..558640aee15a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -26,15 +26,18 @@
 #include "amdgpu_gfx.h"
 #include "amdgpu_ucode.h"
 #include "si/clearstate_si.h"
-#include "si/sid.h"
-
-#define GFX6_NUM_GFX_RINGS     1
-#define GFX6_NUM_COMPUTE_RINGS 2
-#define STATIC_PER_CU_PG_ENABLE                    (1 << 3)
-#define DYN_PER_CU_PG_ENABLE                       (1 << 2)
-#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
-#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
-
+#include "bif/bif_3_0_d.h"
+#include "bif/bif_3_0_sh_mask.h"
+#include "oss/oss_1_0_d.h"
+#include "oss/oss_1_0_sh_mask.h"
+#include "gca/gfx_6_0_d.h"
+#include "gca/gfx_6_0_sh_mask.h"
+#include "gmc/gmc_6_0_d.h"
+#include "gmc/gmc_6_0_sh_mask.h"
+#include "dce/dce_6_0_d.h"
+#include "dce/dce_6_0_sh_mask.h"
+#include "gca/gfx_7_2_enum.h"
+#include "si_enums.h"
 
 static void gfx_v6_0_set_ring_funcs(struct amdgpu_device *adev);
 static void gfx_v6_0_set_irq_funcs(struct amdgpu_device *adev);
@@ -70,6 +73,15 @@ static void gfx_v6_0_get_csb_buffer(struct amdgpu_device *adev, volatile u32 *bu
 //static void gfx_v6_0_init_cp_pg_table(struct amdgpu_device *adev);
 static void gfx_v6_0_init_pg(struct amdgpu_device *adev);
 
+#define ARRAY_MODE(x)					((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT)
+#define PIPE_CONFIG(x)					((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT)
+#define TILE_SPLIT(x)					((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT)
+#define MICRO_TILE_MODE(x)				((x) << 0)
+#define SAMPLE_SPLIT(x)					((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT)
+#define BANK_WIDTH(x)					((x) << 14)
+#define BANK_HEIGHT(x)					((x) << 16)
+#define MACRO_TILE_ASPECT(x)				((x) << 18)
+#define NUM_BANKS(x)					((x) << 20)
 
 static const u32 verde_rlc_save_restore_register_list[] =
 {
@@ -400,8 +412,8 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 	}
 
 	if (adev->asic_type == CHIP_VERDE ||
-		adev->asic_type == CHIP_OLAND ||
-		adev->asic_type == CHIP_HAINAN) {
+	    adev->asic_type == CHIP_OLAND ||
+	    adev->asic_type == CHIP_HAINAN) {
 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
 			switch (reg_offset) {
 			case 0:
@@ -414,7 +426,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
 				break;
-			case 1: 
+			case 1:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -434,7 +446,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
 				break;
-			case 3:  
+			case 3:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -444,7 +456,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
 				break;
-			case 4:  
+			case 4:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -454,7 +466,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 5:  
+			case 5:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -464,7 +476,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 6:  
+			case 6:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -474,7 +486,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 7:  
+			case 7:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -484,7 +496,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
 				break;
-			case 8: 
+			case 8:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -494,7 +506,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 9:  
+			case 9:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -504,7 +516,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 10:  
+			case 10:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -514,7 +526,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
 				break;
-			case 11:  
+			case 11:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -524,7 +536,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 12:  
+			case 12:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -534,7 +546,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 13:  
+			case 13:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -544,7 +556,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 14:  
+			case 14:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -554,7 +566,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 15:  
+			case 15:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -564,7 +576,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 16:  
+			case 16:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -574,7 +586,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 17:  
+			case 17:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
@@ -584,7 +596,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 21:  
+			case 21:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
@@ -594,7 +606,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 22:  
+			case 22:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
@@ -604,7 +616,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
 				break;
-			case 23: 
+			case 23:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
@@ -614,7 +626,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 24: 
+			case 24:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
@@ -624,7 +636,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
 				break;
-			case 25: 
+			case 25:
 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
@@ -639,7 +651,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 				break;
 			}
 			adev->gfx.config.tile_mode_array[reg_offset] = gb_tile_moden;
-			WREG32(GB_TILE_MODE0 + reg_offset, gb_tile_moden);
+			WREG32(mmGB_TILE_MODE0 + reg_offset, gb_tile_moden);
 		}
 	} else if ((adev->asic_type == CHIP_TAHITI) || (adev->asic_type == CHIP_PITCAIRN)) {
 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
@@ -879,7 +891,7 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 				break;
 			}
 			adev->gfx.config.tile_mode_array[reg_offset] = gb_tile_moden;
-			WREG32(GB_TILE_MODE0 + reg_offset, gb_tile_moden);
+			WREG32(mmGB_TILE_MODE0 + reg_offset, gb_tile_moden);
 		}
 	} else{
 
@@ -894,19 +906,23 @@ static void gfx_v6_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
 	u32 data;
 
 	if (instance == 0xffffffff)
-		data = INSTANCE_BROADCAST_WRITES;
+		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
 	else
-		data = INSTANCE_INDEX(instance);
+		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
 
 	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
-		data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
+		data |= GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
+			GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK;
 	else if (se_num == 0xffffffff)
-		data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
+		data |= GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK |
+			(sh_num << GRBM_GFX_INDEX__SH_INDEX__SHIFT);
 	else if (sh_num == 0xffffffff)
-		data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
+		data |= GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
+			(se_num << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
 	else
-		data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
-	WREG32(GRBM_GFX_INDEX, data);
+		data |= (sh_num << GRBM_GFX_INDEX__SH_INDEX__SHIFT) |
+			(se_num << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
+	WREG32(mmGRBM_GFX_INDEX, data);
 }
 
 static u32 gfx_v6_0_create_bitmask(u32 bit_width)
@@ -920,11 +936,11 @@ static u32 gfx_v6_0_get_rb_disabled(struct amdgpu_device *adev,
 {
 	u32 data, mask;
 
-	data = RREG32(CC_RB_BACKEND_DISABLE);
-	data &= BACKEND_DISABLE_MASK;
-	data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
+	data = RREG32(mmCC_RB_BACKEND_DISABLE);
+	data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
+	data |= RREG32(mmGC_USER_RB_BACKEND_DISABLE);
 
-	data >>= BACKEND_DISABLE_SHIFT;
+	data >>= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
 
 	mask = gfx_v6_0_create_bitmask(max_rb_num_per_se / sh_per_se);
 
@@ -936,14 +952,23 @@ static void gfx_v6_0_raster_config(struct amdgpu_device *adev, u32 *rconf)
 	switch (adev->asic_type) {
 	case CHIP_TAHITI:
 	case CHIP_PITCAIRN:
-		*rconf |= RB_XSEL2(2) | RB_XSEL | PKR_MAP(2) | PKR_YSEL(1) |
-			  SE_MAP(2) | SE_XSEL(2) | SE_YSEL(2);
+		*rconf |=
+			   (2 << PA_SC_RASTER_CONFIG__RB_XSEL2__SHIFT) |
+			   (1 << PA_SC_RASTER_CONFIG__RB_XSEL__SHIFT) |
+			   (2 << PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT) |
+			   (1 << PA_SC_RASTER_CONFIG__PKR_YSEL__SHIFT) |
+			   (2 << PA_SC_RASTER_CONFIG__SE_MAP__SHIFT) |
+			   (2 << PA_SC_RASTER_CONFIG__SE_XSEL__SHIFT) |
+			   (2 << PA_SC_RASTER_CONFIG__SE_YSEL__SHIFT);
 		break;
 	case CHIP_VERDE:
-		*rconf |= RB_XSEL | PKR_MAP(2) | PKR_YSEL(1);
+		*rconf |=
+			   (1 << PA_SC_RASTER_CONFIG__RB_XSEL__SHIFT) |
+			   (2 << PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT) |
+			   (1 << PA_SC_RASTER_CONFIG__PKR_YSEL__SHIFT);
 		break;
 	case CHIP_OLAND:
-		*rconf |= RB_YSEL;
+		*rconf |= (1 << PA_SC_RASTER_CONFIG__RB_YSEL__SHIFT);
 		break;
 	case CHIP_HAINAN:
 		*rconf |= 0x0;
@@ -981,24 +1006,24 @@ static void gfx_v6_0_write_harvested_raster_configs(struct amdgpu_device *adev,
 		int idx = (se / 2) * 2;
 
 		if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
-			raster_config_se &= ~SE_MAP_MASK;
+			raster_config_se &= ~PA_SC_RASTER_CONFIG__SE_MAP_MASK;
 
 			if (!se_mask[idx]) {
-				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
+				raster_config_se |= RASTER_CONFIG_SE_MAP_3 << PA_SC_RASTER_CONFIG__SE_MAP__SHIFT;
 			} else {
-				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
+				raster_config_se |= RASTER_CONFIG_SE_MAP_0 << PA_SC_RASTER_CONFIG__SE_MAP__SHIFT;
 			}
 		}
 
 		pkr0_mask &= rb_mask;
 		pkr1_mask &= rb_mask;
 		if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
-			raster_config_se &= ~PKR_MAP_MASK;
+			raster_config_se &= ~PA_SC_RASTER_CONFIG__PKR_MAP_MASK;
 
 			if (!pkr0_mask) {
-				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
+				raster_config_se |= RASTER_CONFIG_PKR_MAP_3 << PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT;
 			} else {
-				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
+				raster_config_se |= RASTER_CONFIG_PKR_MAP_0 << PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT;
 			}
 		}
 
@@ -1009,14 +1034,14 @@ static void gfx_v6_0_write_harvested_raster_configs(struct amdgpu_device *adev,
 			rb0_mask &= rb_mask;
 			rb1_mask &= rb_mask;
 			if (!rb0_mask || !rb1_mask) {
-				raster_config_se &= ~RB_MAP_PKR0_MASK;
+				raster_config_se &= ~PA_SC_RASTER_CONFIG__RB_MAP_PKR0_MASK;
 
 				if (!rb0_mask) {
 					raster_config_se |=
-						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
+						RASTER_CONFIG_RB_MAP_3 << PA_SC_RASTER_CONFIG__RB_MAP_PKR0__SHIFT;
 				} else {
 					raster_config_se |=
-						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
+						RASTER_CONFIG_RB_MAP_0 << PA_SC_RASTER_CONFIG__RB_MAP_PKR0__SHIFT;
 				}
 			}
 
@@ -1026,14 +1051,14 @@ static void gfx_v6_0_write_harvested_raster_configs(struct amdgpu_device *adev,
 				rb0_mask &= rb_mask;
 				rb1_mask &= rb_mask;
 				if (!rb0_mask || !rb1_mask) {
-					raster_config_se &= ~RB_MAP_PKR1_MASK;
+					raster_config_se &= ~PA_SC_RASTER_CONFIG__RB_MAP_PKR1_MASK;
 
 					if (!rb0_mask) {
 						raster_config_se |=
-							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
+							RASTER_CONFIG_RB_MAP_3 << PA_SC_RASTER_CONFIG__RB_MAP_PKR1__SHIFT;
 					} else {
 						raster_config_se |=
-							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
+							RASTER_CONFIG_RB_MAP_0 << PA_SC_RASTER_CONFIG__RB_MAP_PKR1__SHIFT;
 					}
 				}
 			}
@@ -1041,7 +1066,7 @@ static void gfx_v6_0_write_harvested_raster_configs(struct amdgpu_device *adev,
 
 		/* GRBM_GFX_INDEX has a different offset on SI */
 		gfx_v6_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff);
-		WREG32(PA_SC_RASTER_CONFIG, raster_config_se);
+		WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
 	}
 
 	/* GRBM_GFX_INDEX has a different offset on SI */
@@ -1063,7 +1088,7 @@ static void gfx_v6_0_setup_rb(struct amdgpu_device *adev,
 		for (j = 0; j < sh_per_se; j++) {
 			gfx_v6_0_select_se_sh(adev, i, j, 0xffffffff);
 			data = gfx_v6_0_get_rb_disabled(adev, max_rb_num_per_se, sh_per_se);
-			disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
+			disabled_rbs |= data << ((i * sh_per_se + j) * 2);
 		}
 	}
 	gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
@@ -1105,7 +1130,7 @@ static void gfx_v6_0_setup_rb(struct amdgpu_device *adev,
 
 		if (!adev->gfx.config.backend_enable_mask ||
 				adev->gfx.config.num_rbs >= num_rb_pipes)
-			WREG32(PA_SC_RASTER_CONFIG, data);
+			WREG32(mmPA_SC_RASTER_CONFIG, data);
 		else
 			gfx_v6_0_write_harvested_raster_configs(adev, data,
 								adev->gfx.config.backend_enable_mask,
@@ -1124,11 +1149,11 @@ static u32 gfx_v6_0_get_cu_enabled(struct amdgpu_device *adev, u32 cu_per_sh)
 {
 	u32 data, mask;
 
-	data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
-	data &= INACTIVE_CUS_MASK;
-	data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
+	data = RREG32(mmCC_GC_SHADER_ARRAY_CONFIG);
+	data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
+	data |= RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
 
-	data >>= INACTIVE_CUS_SHIFT;
+	data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
 
 	mask = gfx_v6_0_create_bitmask(cu_per_sh);
 
@@ -1148,7 +1173,7 @@ static void gfx_v6_0_setup_spi(struct amdgpu_device *adev,
 	for (i = 0; i < se_num; i++) {
 		for (j = 0; j < sh_per_se; j++) {
 			gfx_v6_0_select_se_sh(adev, i, j, 0xffffffff);
-			data = RREG32(SPI_STATIC_THREAD_MGMT_3);
+			data = RREG32(mmSPI_STATIC_THREAD_MGMT_3);
 			active_cu = gfx_v6_0_get_cu_enabled(adev, cu_per_sh);
 
 			mask = 1;
@@ -1156,7 +1181,7 @@ static void gfx_v6_0_setup_spi(struct amdgpu_device *adev,
 				mask <<= k;
 				if (active_cu & mask) {
 					data &= ~mask;
-					WREG32(SPI_STATIC_THREAD_MGMT_3, data);
+					WREG32(mmSPI_STATIC_THREAD_MGMT_3, data);
 					break;
 				}
 			}
@@ -1209,7 +1234,6 @@ static void gfx_v6_0_gpu_init(struct amdgpu_device *adev)
 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
 		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
 		break;
-
 	case CHIP_VERDE:
 		adev->gfx.config.max_shader_engines = 1;
 		adev->gfx.config.max_tile_pipes = 4;
@@ -1266,18 +1290,18 @@ static void gfx_v6_0_gpu_init(struct amdgpu_device *adev)
 		break;
 	}
 
-	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
-	WREG32(SRBM_INT_CNTL, 1);
-	WREG32(SRBM_INT_ACK, 1);
+	WREG32(mmGRBM_CNTL, (0xff << GRBM_CNTL__READ_TIMEOUT__SHIFT));
+	WREG32(mmSRBM_INT_CNTL, 1);
+	WREG32(mmSRBM_INT_ACK, 1);
 
-	WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
+	WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
 
-	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
-	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
+	mc_shared_chmap = RREG32(mmMC_SHARED_CHMAP);
+	mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);
 
 	adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
 	adev->gfx.config.mem_max_burst_length_bytes = 256;
-	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
+	tmp = (mc_arb_ramcfg & MC_ARB_RAMCFG__NOOFCOLS_MASK) >> MC_ARB_RAMCFG__NOOFCOLS__SHIFT;
 	adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
 	if (adev->gfx.config.mem_row_size_in_kb > 4)
 		adev->gfx.config.mem_row_size_in_kb = 4;
@@ -1285,32 +1309,33 @@ static void gfx_v6_0_gpu_init(struct amdgpu_device *adev)
 	adev->gfx.config.num_gpus = 1;
 	adev->gfx.config.multi_gpu_tile_size = 64;
 
-	gb_addr_config &= ~ROW_SIZE_MASK;
+	gb_addr_config &= ~GB_ADDR_CONFIG__ROW_SIZE_MASK;
 	switch (adev->gfx.config.mem_row_size_in_kb) {
 	case 1:
 	default:
-		gb_addr_config |= ROW_SIZE(0);
+		gb_addr_config |= 0 << GB_ADDR_CONFIG__ROW_SIZE__SHIFT;
 		break;
 	case 2:
-		gb_addr_config |= ROW_SIZE(1);
+		gb_addr_config |= 1 << GB_ADDR_CONFIG__ROW_SIZE__SHIFT;
 		break;
 	case 4:
-		gb_addr_config |= ROW_SIZE(2);
+		gb_addr_config |= 2 << GB_ADDR_CONFIG__ROW_SIZE__SHIFT;
 		break;
 	}
 	adev->gfx.config.gb_addr_config = gb_addr_config;
 
-	WREG32(GB_ADDR_CONFIG, gb_addr_config);
-	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
-	WREG32(DMIF_ADDR_CALC, gb_addr_config);
-	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
-	WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
-	WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
+	WREG32(mmGB_ADDR_CONFIG, gb_addr_config);
+	WREG32(mmDMIF_ADDR_CONFIG, gb_addr_config);
+	WREG32(mmDMIF_ADDR_CALC, gb_addr_config);
+	WREG32(mmHDP_ADDR_CONFIG, gb_addr_config);
+	WREG32(mmDMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
+	WREG32(mmDMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
+
 #if 0
 	if (adev->has_uvd) {
-		WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
-		WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
-		WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
+		WREG32(mmUVD_UDEC_ADDR_CONFIG, gb_addr_config);
+		WREG32(mmUVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
+		WREG32(mmUVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
 	}
 #endif
 	gfx_v6_0_tiling_mode_table_init(adev);
@@ -1325,45 +1350,48 @@ static void gfx_v6_0_gpu_init(struct amdgpu_device *adev)
 
 	gfx_v6_0_get_cu_info(adev);
 
-	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
-				     ROQ_IB2_START(0x2b)));
-	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
+	WREG32(mmCP_QUEUE_THRESHOLDS, ((0x16 << CP_QUEUE_THRESHOLDS__ROQ_IB1_START__SHIFT) |
+				       (0x2b << CP_QUEUE_THRESHOLDS__ROQ_IB2_START__SHIFT)));
+	WREG32(mmCP_MEQ_THRESHOLDS, (0x30 << CP_MEQ_THRESHOLDS__MEQ1_START__SHIFT) |
+				    (0x60 << CP_MEQ_THRESHOLDS__MEQ2_START__SHIFT));
 
-	sx_debug_1 = RREG32(SX_DEBUG_1);
-	WREG32(SX_DEBUG_1, sx_debug_1);
+	sx_debug_1 = RREG32(mmSX_DEBUG_1);
+	WREG32(mmSX_DEBUG_1, sx_debug_1);
 
-	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
+	WREG32(mmSPI_CONFIG_CNTL_1, (4 << SPI_CONFIG_CNTL_1__VTX_DONE_DELAY__SHIFT));
 
-	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(adev->gfx.config.sc_prim_fifo_size_frontend) |
-				 SC_BACKEND_PRIM_FIFO_SIZE(adev->gfx.config.sc_prim_fifo_size_backend) |
-				 SC_HIZ_TILE_FIFO_SIZE(adev->gfx.config.sc_hiz_tile_fifo_size) |
-				 SC_EARLYZ_TILE_FIFO_SIZE(adev->gfx.config.sc_earlyz_tile_fifo_size)));
+	WREG32(mmPA_SC_FIFO_SIZE, ((adev->gfx.config.sc_prim_fifo_size_frontend << PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
+				   (adev->gfx.config.sc_prim_fifo_size_backend << PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
+				   (adev->gfx.config.sc_hiz_tile_fifo_size << PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
+				   (adev->gfx.config.sc_earlyz_tile_fifo_size << PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT)));
 
-	WREG32(VGT_NUM_INSTANCES, 1);
-	WREG32(CP_PERFMON_CNTL, 0);
-	WREG32(SQ_CONFIG, 0);
-	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
-					  FORCE_EOV_MAX_REZ_CNT(255)));
+	WREG32(mmVGT_NUM_INSTANCES, 1);
+	WREG32(mmCP_PERFMON_CNTL, 0);
+	WREG32(mmSQ_CONFIG, 0);
+	WREG32(mmPA_SC_FORCE_EOV_MAX_CNTS, ((4095 << PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT__SHIFT) |
+					  (255 << PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT__SHIFT)));
 
-	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
-	       AUTO_INVLD_EN(ES_AND_GS_AUTO));
+	WREG32(mmVGT_CACHE_INVALIDATION,
+		(VC_AND_TC << VGT_CACHE_INVALIDATION__CACHE_INVALIDATION__SHIFT) |
+		(ES_AND_GS_AUTO << VGT_CACHE_INVALIDATION__AUTO_INVLD_EN__SHIFT));
 
-	WREG32(VGT_GS_VERTEX_REUSE, 16);
-	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
+	WREG32(mmVGT_GS_VERTEX_REUSE, 16);
+	WREG32(mmPA_SC_LINE_STIPPLE_STATE, 0);
 
-	WREG32(CB_PERFCOUNTER0_SELECT0, 0);
-	WREG32(CB_PERFCOUNTER0_SELECT1, 0);
-	WREG32(CB_PERFCOUNTER1_SELECT0, 0);
-	WREG32(CB_PERFCOUNTER1_SELECT1, 0);
-	WREG32(CB_PERFCOUNTER2_SELECT0, 0);
-	WREG32(CB_PERFCOUNTER2_SELECT1, 0);
-	WREG32(CB_PERFCOUNTER3_SELECT0, 0);
-	WREG32(CB_PERFCOUNTER3_SELECT1, 0);
+	WREG32(mmCB_PERFCOUNTER0_SELECT0, 0);
+	WREG32(mmCB_PERFCOUNTER0_SELECT1, 0);
+	WREG32(mmCB_PERFCOUNTER1_SELECT0, 0);
+	WREG32(mmCB_PERFCOUNTER1_SELECT1, 0);
+	WREG32(mmCB_PERFCOUNTER2_SELECT0, 0);
+	WREG32(mmCB_PERFCOUNTER2_SELECT1, 0);
+	WREG32(mmCB_PERFCOUNTER3_SELECT0, 0);
+	WREG32(mmCB_PERFCOUNTER3_SELECT1, 0);
 
-	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
-	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
+	hdp_host_path_cntl = RREG32(mmHDP_HOST_PATH_CNTL);
+	WREG32(mmHDP_HOST_PATH_CNTL, hdp_host_path_cntl);
 
-	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
+	WREG32(mmPA_CL_ENHANCE, PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA_MASK |
+				(3 << PA_CL_ENHANCE__NUM_CLIP_SEQ__SHIFT));
 
 	udelay(50);
 }
@@ -1374,7 +1402,7 @@ static void gfx_v6_0_scratch_init(struct amdgpu_device *adev)
 	int i;
 
 	adev->gfx.scratch.num_reg = 7;
-	adev->gfx.scratch.reg_base = SCRATCH_REG0;
+	adev->gfx.scratch.reg_base = mmSCRATCH_REG0;
 	for (i = 0; i < adev->gfx.scratch.num_reg; i++) {
 		adev->gfx.scratch.free[i] = true;
 		adev->gfx.scratch.reg[i] = adev->gfx.scratch.reg_base + i;
@@ -1430,11 +1458,18 @@ static void gfx_v6_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
 				 WRITE_DATA_DST_SEL(0)));
-	amdgpu_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL);
+	amdgpu_ring_write(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL);
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, 0x1);
 }
 
+static void gfx_v6_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
+{
+	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
+	amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
+		EVENT_INDEX(0));
+}
+
 /**
  * gfx_v6_0_ring_emit_hdp_invalidate - emit an hdp invalidate on the cp
  *
@@ -1448,7 +1483,7 @@ static void gfx_v6_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
 				 WRITE_DATA_DST_SEL(0)));
-	amdgpu_ring_write(ring, HDP_DEBUG0);
+	amdgpu_ring_write(ring, mmHDP_DEBUG0);
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, 0x1);
 }
@@ -1460,7 +1495,7 @@ static void gfx_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
 	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
 	/* flush read cache over gart */
 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
-	amdgpu_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START));
+	amdgpu_ring_write(ring, (mmCP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START));
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
 	amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
@@ -1475,7 +1510,8 @@ static void gfx_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
 	amdgpu_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
 	amdgpu_ring_write(ring, addr & 0xfffffffc);
 	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
-				DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
+				((write64bit ? 2 : 1) << CP_EOP_DONE_DATA_CNTL__DATA_SEL__SHIFT) |
+				((int_sel ? 2 : 0) << CP_EOP_DONE_DATA_CNTL__INT_SEL__SHIFT));
 	amdgpu_ring_write(ring, lower_32_bits(seq));
 	amdgpu_ring_write(ring, upper_32_bits(seq));
 }
@@ -1522,7 +1558,7 @@ static int gfx_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_ib ib;
-	struct fence *f = NULL;
+	struct dma_fence *f = NULL;
 	uint32_t scratch;
 	uint32_t tmp = 0;
 	long r;
@@ -1548,7 +1584,7 @@ static int gfx_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	if (r)
 		goto err2;
 
-	r = fence_wait_timeout(f, false, timeout);
+	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
 		DRM_ERROR("amdgpu: IB test timed out\n");
 		r = -ETIMEDOUT;
@@ -1569,7 +1605,7 @@ static int gfx_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 err2:
 	amdgpu_ib_free(adev, &ib, NULL);
-	fence_put(f);
+	dma_fence_put(f);
 err1:
 	amdgpu_gfx_scratch_free(adev, scratch);
 	return r;
@@ -1578,11 +1614,13 @@ err1:
 static void gfx_v6_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
 {
 	int i;
-	if (enable)
-		WREG32(CP_ME_CNTL, 0);
-	else {
-		WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
-		WREG32(SCRATCH_UMSK, 0);
+	if (enable) {
+		WREG32(mmCP_ME_CNTL, 0);
+	} else {
+		WREG32(mmCP_ME_CNTL, (CP_ME_CNTL__ME_HALT_MASK |
+				      CP_ME_CNTL__PFP_HALT_MASK |
+				      CP_ME_CNTL__CE_HALT_MASK));
+		WREG32(mmSCRATCH_UMSK, 0);
 		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
 			adev->gfx.gfx_ring[i].ready = false;
 		for (i = 0; i < adev->gfx.num_compute_rings; i++)
@@ -1616,34 +1654,33 @@ static int gfx_v6_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
 	fw_data = (const __le32 *)
 		(adev->gfx.pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
 	fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
-	WREG32(CP_PFP_UCODE_ADDR, 0);
+	WREG32(mmCP_PFP_UCODE_ADDR, 0);
 	for (i = 0; i < fw_size; i++)
-		WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
-	WREG32(CP_PFP_UCODE_ADDR, 0);
+		WREG32(mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
+	WREG32(mmCP_PFP_UCODE_ADDR, 0);
 
 	/* CE */
 	fw_data = (const __le32 *)
 		(adev->gfx.ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
 	fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
-	WREG32(CP_CE_UCODE_ADDR, 0);
+	WREG32(mmCP_CE_UCODE_ADDR, 0);
 	for (i = 0; i < fw_size; i++)
-		WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
-	WREG32(CP_CE_UCODE_ADDR, 0);
+		WREG32(mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
+	WREG32(mmCP_CE_UCODE_ADDR, 0);
 
 	/* ME */
 	fw_data = (const __be32 *)
 		(adev->gfx.me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
 	fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
-	WREG32(CP_ME_RAM_WADDR, 0);
+	WREG32(mmCP_ME_RAM_WADDR, 0);
 	for (i = 0; i < fw_size; i++)
-		WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
-	WREG32(CP_ME_RAM_WADDR, 0);
+		WREG32(mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
+	WREG32(mmCP_ME_RAM_WADDR, 0);
 
-
-	WREG32(CP_PFP_UCODE_ADDR, 0);
-	WREG32(CP_CE_UCODE_ADDR, 0);
-	WREG32(CP_ME_RAM_WADDR, 0);
-	WREG32(CP_ME_RAM_RADDR, 0);
+	WREG32(mmCP_PFP_UCODE_ADDR, 0);
+	WREG32(mmCP_CE_UCODE_ADDR, 0);
+	WREG32(mmCP_ME_RAM_WADDR, 0);
+	WREG32(mmCP_ME_RAM_RADDR, 0);
 	return 0;
 }
 
@@ -1720,14 +1757,14 @@ static int gfx_v6_0_cp_gfx_resume(struct amdgpu_device *adev)
 	int r;
 	u64 rptr_addr;
 
-	WREG32(CP_SEM_WAIT_TIMER, 0x0);
-	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
+	WREG32(mmCP_SEM_WAIT_TIMER, 0x0);
+	WREG32(mmCP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
 
 	/* Set the write pointer delay */
-	WREG32(CP_RB_WPTR_DELAY, 0);
+	WREG32(mmCP_RB_WPTR_DELAY, 0);
 
-	WREG32(CP_DEBUG, 0);
-	WREG32(SCRATCH_ADDR, 0);
+	WREG32(mmCP_DEBUG, 0);
+	WREG32(mmSCRATCH_ADDR, 0);
 
 	/* ring 0 - compute and gfx */
 	/* Set ring buffer size */
@@ -1738,24 +1775,24 @@ static int gfx_v6_0_cp_gfx_resume(struct amdgpu_device *adev)
 #ifdef __BIG_ENDIAN
 	tmp |= BUF_SWAP_32BIT;
 #endif
-	WREG32(CP_RB0_CNTL, tmp);
+	WREG32(mmCP_RB0_CNTL, tmp);
 
 	/* Initialize the ring buffer's read and write pointers */
-	WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
+	WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK);
 	ring->wptr = 0;
-	WREG32(CP_RB0_WPTR, ring->wptr);
+	WREG32(mmCP_RB0_WPTR, ring->wptr);
 
 	/* set the wb address whether it's enabled or not */
 	rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
-	WREG32(CP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
-	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
+	WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
+	WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
 
-	WREG32(SCRATCH_UMSK, 0);
+	WREG32(mmSCRATCH_UMSK, 0);
 
 	mdelay(1);
-	WREG32(CP_RB0_CNTL, tmp);
+	WREG32(mmCP_RB0_CNTL, tmp);
 
-	WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
+	WREG32(mmCP_RB0_BASE, ring->gpu_addr >> 8);
 
 	/* start the rings */
 	gfx_v6_0_cp_gfx_start(adev);
@@ -1779,11 +1816,11 @@ static u32 gfx_v6_0_ring_get_wptr(struct amdgpu_ring *ring)
 	struct amdgpu_device *adev = ring->adev;
 
 	if (ring == &adev->gfx.gfx_ring[0])
-		return RREG32(CP_RB0_WPTR);
+		return RREG32(mmCP_RB0_WPTR);
 	else if (ring == &adev->gfx.compute_ring[0])
-		return RREG32(CP_RB1_WPTR);
+		return RREG32(mmCP_RB1_WPTR);
 	else if (ring == &adev->gfx.compute_ring[1])
-		return RREG32(CP_RB2_WPTR);
+		return RREG32(mmCP_RB2_WPTR);
 	else
 		BUG();
 }
@@ -1792,8 +1829,8 @@ static void gfx_v6_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	WREG32(CP_RB0_WPTR, ring->wptr);
-	(void)RREG32(CP_RB0_WPTR);
+	WREG32(mmCP_RB0_WPTR, ring->wptr);
+	(void)RREG32(mmCP_RB0_WPTR);
 }
 
 static void gfx_v6_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
@@ -1801,11 +1838,11 @@ static void gfx_v6_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
 	struct amdgpu_device *adev = ring->adev;
 
 	if (ring == &adev->gfx.compute_ring[0]) {
-		WREG32(CP_RB1_WPTR, ring->wptr);
-		(void)RREG32(CP_RB1_WPTR);
+		WREG32(mmCP_RB1_WPTR, ring->wptr);
+		(void)RREG32(mmCP_RB1_WPTR);
 	} else if (ring == &adev->gfx.compute_ring[1]) {
-		WREG32(CP_RB2_WPTR, ring->wptr);
-		(void)RREG32(CP_RB2_WPTR);
+		WREG32(mmCP_RB2_WPTR, ring->wptr);
+		(void)RREG32(mmCP_RB2_WPTR);
 	} else {
 		BUG();
 	}
@@ -1817,7 +1854,7 @@ static int gfx_v6_0_cp_compute_resume(struct amdgpu_device *adev)
 	struct amdgpu_ring *ring;
 	u32 tmp;
 	u32 rb_bufsz;
-	int r;
+	int i, r;
 	u64 rptr_addr;
 
 	/* ring1  - compute only */
@@ -1829,19 +1866,19 @@ static int gfx_v6_0_cp_compute_resume(struct amdgpu_device *adev)
 #ifdef __BIG_ENDIAN
 	tmp |= BUF_SWAP_32BIT;
 #endif
-	WREG32(CP_RB1_CNTL, tmp);
+	WREG32(mmCP_RB1_CNTL, tmp);
 
-	WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
+	WREG32(mmCP_RB1_CNTL, tmp | CP_RB1_CNTL__RB_RPTR_WR_ENA_MASK);
 	ring->wptr = 0;
-	WREG32(CP_RB1_WPTR, ring->wptr);
+	WREG32(mmCP_RB1_WPTR, ring->wptr);
 
 	rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
-	WREG32(CP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
-	WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
+	WREG32(mmCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
+	WREG32(mmCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
 
 	mdelay(1);
-	WREG32(CP_RB1_CNTL, tmp);
-	WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
+	WREG32(mmCP_RB1_CNTL, tmp);
+	WREG32(mmCP_RB1_BASE, ring->gpu_addr >> 8);
 
 	ring = &adev->gfx.compute_ring[1];
 	rb_bufsz = order_base_2(ring->ring_size / 8);
@@ -1849,32 +1886,27 @@ static int gfx_v6_0_cp_compute_resume(struct amdgpu_device *adev)
 #ifdef __BIG_ENDIAN
 	tmp |= BUF_SWAP_32BIT;
 #endif
-	WREG32(CP_RB2_CNTL, tmp);
+	WREG32(mmCP_RB2_CNTL, tmp);
 
-	WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
+	WREG32(mmCP_RB2_CNTL, tmp | CP_RB2_CNTL__RB_RPTR_WR_ENA_MASK);
 	ring->wptr = 0;
-	WREG32(CP_RB2_WPTR, ring->wptr);
+	WREG32(mmCP_RB2_WPTR, ring->wptr);
 	rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
-	WREG32(CP_RB2_RPTR_ADDR, lower_32_bits(rptr_addr));
-	WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
+	WREG32(mmCP_RB2_RPTR_ADDR, lower_32_bits(rptr_addr));
+	WREG32(mmCP_RB2_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
 
 	mdelay(1);
-	WREG32(CP_RB2_CNTL, tmp);
-	WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
+	WREG32(mmCP_RB2_CNTL, tmp);
+	WREG32(mmCP_RB2_BASE, ring->gpu_addr >> 8);
 
-	adev->gfx.compute_ring[0].ready = true;
-	adev->gfx.compute_ring[1].ready = true;
+	adev->gfx.compute_ring[0].ready = false;
+	adev->gfx.compute_ring[1].ready = false;
 
-	r = amdgpu_ring_test_ring(&adev->gfx.compute_ring[0]);
-	if (r) {
-		adev->gfx.compute_ring[0].ready = false;
-		return r;
-	}
-
-	r = amdgpu_ring_test_ring(&adev->gfx.compute_ring[1]);
-	if (r) {
-		adev->gfx.compute_ring[1].ready = false;
-		return r;
+	for (i = 0; i < 2; i++) {
+		r = amdgpu_ring_test_ring(&adev->gfx.compute_ring[i]);
+		if (r)
+			return r;
+		adev->gfx.compute_ring[i].ready = true;
 	}
 
 	return 0;
@@ -1892,24 +1924,26 @@ static int gfx_v6_0_cp_load_microcode(struct amdgpu_device *adev)
 
 static void gfx_v6_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
 					       bool enable)
-{	
-	u32 tmp = RREG32(CP_INT_CNTL_RING0);
+{
+	u32 tmp = RREG32(mmCP_INT_CNTL_RING0);
 	u32 mask;
 	int i;
 
 	if (enable)
-		tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
+		tmp |= (CP_INT_CNTL__CNTX_BUSY_INT_ENABLE_MASK |
+			CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE_MASK);
 	else
-		tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
-	WREG32(CP_INT_CNTL_RING0, tmp);
+		tmp &= ~(CP_INT_CNTL__CNTX_BUSY_INT_ENABLE_MASK |
+			 CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE_MASK);
+	WREG32(mmCP_INT_CNTL_RING0, tmp);
 
 	if (!enable) {
 		/* read a gfx register */
-		tmp = RREG32(DB_DEPTH_INFO);
+		tmp = RREG32(mmDB_DEPTH_INFO);
 
 		mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
 		for (i = 0; i < adev->usec_timeout; i++) {
-			if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
+			if ((RREG32(mmRLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
 				break;
 			udelay(1);
 		}
@@ -1940,7 +1974,7 @@ static int gfx_v6_0_cp_resume(struct amdgpu_device *adev)
 
 static void gfx_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 {
-	int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
+	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
 	uint32_t seq = ring->fence_drv.sync_seq;
 	uint64_t addr = ring->fence_drv.gpu_addr;
 
@@ -1966,16 +2000,16 @@ static void gfx_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 					unsigned vm_id, uint64_t pd_addr)
 {
-	int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
+	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
 
 	/* write new base address */
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
 				 WRITE_DATA_DST_SEL(0)));
 	if (vm_id < 8) {
-		amdgpu_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id ));
+		amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id ));
 	} else {
-		amdgpu_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (vm_id - 8)));
+		amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (vm_id - 8)));
 	}
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, pd_addr >> 12);
@@ -1984,7 +2018,7 @@ static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
 				 WRITE_DATA_DST_SEL(0)));
-	amdgpu_ring_write(ring, VM_INVALIDATE_REQUEST);
+	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, 1 << vm_id);
 
@@ -1992,7 +2026,7 @@ static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 	amdgpu_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
 				 WAIT_REG_MEM_ENGINE(0))); /* me */
-	amdgpu_ring_write(ring, VM_INVALIDATE_REQUEST);
+	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, 0); /* ref */
 	amdgpu_ring_write(ring, 0); /* mask */
@@ -2071,7 +2105,6 @@ static int gfx_v6_0_rlc_init(struct amdgpu_device *adev)
 	if (src_ptr) {
 		/* save restore block */
 		if (adev->gfx.rlc.save_restore_obj == NULL) {
-
 			r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
 					     AMDGPU_GEM_DOMAIN_VRAM,
 					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
@@ -2166,20 +2199,12 @@ static int gfx_v6_0_rlc_init(struct amdgpu_device *adev)
 
 static void gfx_v6_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
 {
-	u32 tmp;
-
-	tmp = RREG32(RLC_LB_CNTL);
-	if (enable)
-		tmp |= LOAD_BALANCE_ENABLE;
-	else
-		tmp &= ~LOAD_BALANCE_ENABLE;
-	WREG32(RLC_LB_CNTL, tmp);
+	WREG32_FIELD(RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
 
 	if (!enable) {
 		gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
-		WREG32(SPI_LB_CU_MASK, 0x00ff);
+		WREG32(mmSPI_LB_CU_MASK, 0x00ff);
 	}
-
 }
 
 static void gfx_v6_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
@@ -2187,13 +2212,13 @@ static void gfx_v6_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
 	int i;
 
 	for (i = 0; i < adev->usec_timeout; i++) {
-		if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
+		if (RREG32(mmRLC_SERDES_MASTER_BUSY_0) == 0)
 			break;
 		udelay(1);
 	}
 
 	for (i = 0; i < adev->usec_timeout; i++) {
-		if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
+		if (RREG32(mmRLC_SERDES_MASTER_BUSY_1) == 0)
 			break;
 		udelay(1);
 	}
@@ -2203,20 +2228,20 @@ static void gfx_v6_0_update_rlc(struct amdgpu_device *adev, u32 rlc)
 {
 	u32 tmp;
 
-	tmp = RREG32(RLC_CNTL);
+	tmp = RREG32(mmRLC_CNTL);
 	if (tmp != rlc)
-		WREG32(RLC_CNTL, rlc);
+		WREG32(mmRLC_CNTL, rlc);
 }
 
 static u32 gfx_v6_0_halt_rlc(struct amdgpu_device *adev)
 {
 	u32 data, orig;
 
-	orig = data = RREG32(RLC_CNTL);
+	orig = data = RREG32(mmRLC_CNTL);
 
-	if (data & RLC_ENABLE) {
-		data &= ~RLC_ENABLE;
-		WREG32(RLC_CNTL, data);
+	if (data & RLC_CNTL__RLC_ENABLE_F32_MASK) {
+		data &= ~RLC_CNTL__RLC_ENABLE_F32_MASK;
+		WREG32(mmRLC_CNTL, data);
 
 		gfx_v6_0_wait_for_rlc_serdes(adev);
 	}
@@ -2226,7 +2251,7 @@ static u32 gfx_v6_0_halt_rlc(struct amdgpu_device *adev)
 
 static void gfx_v6_0_rlc_stop(struct amdgpu_device *adev)
 {
-	WREG32(RLC_CNTL, 0);
+	WREG32(mmRLC_CNTL, 0);
 
 	gfx_v6_0_enable_gui_idle_interrupt(adev, false);
 	gfx_v6_0_wait_for_rlc_serdes(adev);
@@ -2234,7 +2259,7 @@ static void gfx_v6_0_rlc_stop(struct amdgpu_device *adev)
 
 static void gfx_v6_0_rlc_start(struct amdgpu_device *adev)
 {
-	WREG32(RLC_CNTL, RLC_ENABLE);
+	WREG32(mmRLC_CNTL, RLC_CNTL__RLC_ENABLE_F32_MASK);
 
 	gfx_v6_0_enable_gui_idle_interrupt(adev, true);
 
@@ -2243,13 +2268,9 @@ static void gfx_v6_0_rlc_start(struct amdgpu_device *adev)
 
 static void gfx_v6_0_rlc_reset(struct amdgpu_device *adev)
 {
-	u32 tmp = RREG32(GRBM_SOFT_RESET);
-
-	tmp |= SOFT_RESET_RLC;
-	WREG32(GRBM_SOFT_RESET, tmp);
+	WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
 	udelay(50);
-	tmp &= ~SOFT_RESET_RLC;
-	WREG32(GRBM_SOFT_RESET, tmp);
+	WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
 	udelay(50);
 }
 
@@ -2258,11 +2279,12 @@ static bool gfx_v6_0_lbpw_supported(struct amdgpu_device *adev)
 	u32 tmp;
 
 	/* Enable LBPW only for DDR3 */
-	tmp = RREG32(MC_SEQ_MISC0);
+	tmp = RREG32(mmMC_SEQ_MISC0);
 	if ((tmp & 0xF0000000) == 0xB0000000)
 		return true;
 	return false;
 }
+
 static void gfx_v6_0_init_cg(struct amdgpu_device *adev)
 {
 }
@@ -2283,15 +2305,15 @@ static int gfx_v6_0_rlc_resume(struct amdgpu_device *adev)
 	gfx_v6_0_init_pg(adev);
 	gfx_v6_0_init_cg(adev);
 
-	WREG32(RLC_RL_BASE, 0);
-	WREG32(RLC_RL_SIZE, 0);
-	WREG32(RLC_LB_CNTL, 0);
-	WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
-	WREG32(RLC_LB_CNTR_INIT, 0);
-	WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
+	WREG32(mmRLC_RL_BASE, 0);
+	WREG32(mmRLC_RL_SIZE, 0);
+	WREG32(mmRLC_LB_CNTL, 0);
+	WREG32(mmRLC_LB_CNTR_MAX, 0xffffffff);
+	WREG32(mmRLC_LB_CNTR_INIT, 0);
+	WREG32(mmRLC_LB_INIT_CU_MASK, 0xffffffff);
 
-	WREG32(RLC_MC_CNTL, 0);
-	WREG32(RLC_UCODE_CNTL, 0);
+	WREG32(mmRLC_MC_CNTL, 0);
+	WREG32(mmRLC_UCODE_CNTL, 0);
 
 	hdr = (const struct rlc_firmware_header_v1_0 *)adev->gfx.rlc_fw->data;
 	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
@@ -2301,10 +2323,10 @@ static int gfx_v6_0_rlc_resume(struct amdgpu_device *adev)
 	amdgpu_ucode_print_rlc_hdr(&hdr->header);
 
 	for (i = 0; i < fw_size; i++) {
-		WREG32(RLC_UCODE_ADDR, i);
-		WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
+		WREG32(mmRLC_UCODE_ADDR, i);
+		WREG32(mmRLC_UCODE_DATA, le32_to_cpup(fw_data++));
 	}
-	WREG32(RLC_UCODE_ADDR, 0);
+	WREG32(mmRLC_UCODE_ADDR, 0);
 
 	gfx_v6_0_enable_lbpw(adev, gfx_v6_0_lbpw_supported(adev));
 	gfx_v6_0_rlc_start(adev);
@@ -2316,38 +2338,38 @@ static void gfx_v6_0_enable_cgcg(struct amdgpu_device *adev, bool enable)
 {
 	u32 data, orig, tmp;
 
-	orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
+	orig = data = RREG32(mmRLC_CGCG_CGLS_CTRL);
 
 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
 		gfx_v6_0_enable_gui_idle_interrupt(adev, true);
 
-		WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
+		WREG32(mmRLC_GCPM_GENERAL_3, 0x00000080);
 
 		tmp = gfx_v6_0_halt_rlc(adev);
 
-		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
-		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
-		WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
+		WREG32(mmRLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
+		WREG32(mmRLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
+		WREG32(mmRLC_SERDES_WR_CTRL, 0x00b000ff);
 
 		gfx_v6_0_wait_for_rlc_serdes(adev);
 		gfx_v6_0_update_rlc(adev, tmp);
 
-		WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
+		WREG32(mmRLC_SERDES_WR_CTRL, 0x007000ff);
 
-		data |= CGCG_EN | CGLS_EN;
+		data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
 	} else {
 		gfx_v6_0_enable_gui_idle_interrupt(adev, false);
 
-		RREG32(CB_CGTT_SCLK_CTRL);
-		RREG32(CB_CGTT_SCLK_CTRL);
-		RREG32(CB_CGTT_SCLK_CTRL);
-		RREG32(CB_CGTT_SCLK_CTRL);
+		RREG32(mmCB_CGTT_SCLK_CTRL);
+		RREG32(mmCB_CGTT_SCLK_CTRL);
+		RREG32(mmCB_CGTT_SCLK_CTRL);
+		RREG32(mmCB_CGTT_SCLK_CTRL);
 
-		data &= ~(CGCG_EN | CGLS_EN);
+		data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
 	}
 
 	if (orig != data)
-		WREG32(RLC_CGCG_CGLS_CTRL, data);
+		WREG32(mmRLC_CGCG_CGLS_CTRL, data);
 
 }
 
@@ -2357,51 +2379,51 @@ static void gfx_v6_0_enable_mgcg(struct amdgpu_device *adev, bool enable)
 	u32 data, orig, tmp = 0;
 
 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
-		orig = data = RREG32(CGTS_SM_CTRL_REG);
+		orig = data = RREG32(mmCGTS_SM_CTRL_REG);
 		data = 0x96940200;
 		if (orig != data)
-			WREG32(CGTS_SM_CTRL_REG, data);
+			WREG32(mmCGTS_SM_CTRL_REG, data);
 
 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
-			orig = data = RREG32(CP_MEM_SLP_CNTL);
-			data |= CP_MEM_LS_EN;
+			orig = data = RREG32(mmCP_MEM_SLP_CNTL);
+			data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
 			if (orig != data)
-				WREG32(CP_MEM_SLP_CNTL, data);
+				WREG32(mmCP_MEM_SLP_CNTL, data);
 		}
 
-		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
+		orig = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
 		data &= 0xffffffc0;
 		if (orig != data)
-			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
+			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
 
 		tmp = gfx_v6_0_halt_rlc(adev);
 
-		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
-		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
-		WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
+		WREG32(mmRLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
+		WREG32(mmRLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
+		WREG32(mmRLC_SERDES_WR_CTRL, 0x00d000ff);
 
 		gfx_v6_0_update_rlc(adev, tmp);
 	} else {
-		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
+		orig = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
 		data |= 0x00000003;
 		if (orig != data)
-			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
+			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
 
-		data = RREG32(CP_MEM_SLP_CNTL);
-		if (data & CP_MEM_LS_EN) {
-			data &= ~CP_MEM_LS_EN;
-			WREG32(CP_MEM_SLP_CNTL, data);
+		data = RREG32(mmCP_MEM_SLP_CNTL);
+		if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
+			data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
+			WREG32(mmCP_MEM_SLP_CNTL, data);
 		}
-		orig = data = RREG32(CGTS_SM_CTRL_REG);
-		data |= LS_OVERRIDE | OVERRIDE;
+		orig = data = RREG32(mmCGTS_SM_CTRL_REG);
+		data |= CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK | CGTS_SM_CTRL_REG__OVERRIDE_MASK;
 		if (orig != data)
-			WREG32(CGTS_SM_CTRL_REG, data);
+			WREG32(mmCGTS_SM_CTRL_REG, data);
 
 		tmp = gfx_v6_0_halt_rlc(adev);
 
-		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
-		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
-		WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
+		WREG32(mmRLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
+		WREG32(mmRLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
+		WREG32(mmRLC_SERDES_WR_CTRL, 0x00e000ff);
 
 		gfx_v6_0_update_rlc(adev, tmp);
 	}
@@ -2421,6 +2443,7 @@ static void gfx_v6_0_update_cg(struct amdgpu_device *adev,
 	gfx_v6_0_enable_gui_idle_interrupt(adev, true);
 }
 */
+
 static void gfx_v6_0_enable_sclk_slowdown_on_pu(struct amdgpu_device *adev,
 						bool enable)
 {
@@ -2435,13 +2458,13 @@ static void gfx_v6_0_enable_cp_pg(struct amdgpu_device *adev, bool enable)
 {
 	u32 data, orig;
 
-	orig = data = RREG32(RLC_PG_CNTL);
+	orig = data = RREG32(mmRLC_PG_CNTL);
 	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_CP))
 		data &= ~0x8000;
 	else
 		data |= 0x8000;
 	if (orig != data)
-		WREG32(RLC_PG_CNTL, data);
+		WREG32(mmRLC_PG_CNTL, data);
 }
 
 static void gfx_v6_0_enable_gds_pg(struct amdgpu_device *adev, bool enable)
@@ -2518,26 +2541,13 @@ static void gfx_v6_0_init_cp_pg_table(struct amdgpu_device *adev)
 static void gfx_v6_0_enable_gfx_cgpg(struct amdgpu_device *adev,
 				     bool enable)
 {
-
-	u32 tmp;
-
 	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
-		tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
-		WREG32(RLC_TTOP_D, tmp);
-
-		tmp = RREG32(RLC_PG_CNTL);
-		tmp |= GFX_PG_ENABLE;
-		WREG32(RLC_PG_CNTL, tmp);
-
-		tmp = RREG32(RLC_AUTO_PG_CTRL);
-		tmp |= AUTO_PG_EN;
-		WREG32(RLC_AUTO_PG_CTRL, tmp);
+		WREG32(mmRLC_TTOP_D, RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10));
+		WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, 1);
+		WREG32_FIELD(RLC_AUTO_PG_CTRL, AUTO_PG_EN, 1);
 	} else {
-		tmp = RREG32(RLC_AUTO_PG_CTRL);
-		tmp &= ~AUTO_PG_EN;
-		WREG32(RLC_AUTO_PG_CTRL, tmp);
-
-		tmp = RREG32(DB_RENDER_CONTROL);
+		WREG32_FIELD(RLC_AUTO_PG_CTRL, AUTO_PG_EN, 0);
+		(void)RREG32(mmDB_RENDER_CONTROL);
 	}
 }
 
@@ -2550,8 +2560,8 @@ static u32 gfx_v6_0_get_cu_active_bitmap(struct amdgpu_device *adev,
 
 	mutex_lock(&adev->grbm_idx_mutex);
 	gfx_v6_0_select_se_sh(adev, se, sh, 0xffffffff);
-	tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
-	tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
+	tmp = RREG32(mmCC_GC_SHADER_ARRAY_CONFIG);
+	tmp1 = RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
 	gfx_v6_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 	mutex_unlock(&adev->grbm_idx_mutex);
 
@@ -2594,12 +2604,8 @@ static void gfx_v6_0_init_ao_cu_mask(struct amdgpu_device *adev)
 		}
 	}
 
-	WREG32(RLC_PG_AO_CU_MASK, tmp);
-
-	tmp = RREG32(RLC_MAX_PG_CU);
-	tmp &= ~MAX_PU_CU_MASK;
-	tmp |= MAX_PU_CU(active_cu_number);
-	WREG32(RLC_MAX_PG_CU, tmp);
+	WREG32(mmRLC_PG_AO_CU_MASK, tmp);
+	WREG32_FIELD(RLC_MAX_PG_CU, MAX_POWERED_UP_CU, active_cu_number);
 }
 
 static void gfx_v6_0_enable_gfx_static_mgpg(struct amdgpu_device *adev,
@@ -2607,13 +2613,13 @@ static void gfx_v6_0_enable_gfx_static_mgpg(struct amdgpu_device *adev,
 {
 	u32 data, orig;
 
-	orig = data = RREG32(RLC_PG_CNTL);
+	orig = data = RREG32(mmRLC_PG_CNTL);
 	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG))
-		data |= STATIC_PER_CU_PG_ENABLE;
+		data |= RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
 	else
-		data &= ~STATIC_PER_CU_PG_ENABLE;
+		data &= ~RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
 	if (orig != data)
-		WREG32(RLC_PG_CNTL, data);
+		WREG32(mmRLC_PG_CNTL, data);
 }
 
 static void gfx_v6_0_enable_gfx_dynamic_mgpg(struct amdgpu_device *adev,
@@ -2621,33 +2627,28 @@ static void gfx_v6_0_enable_gfx_dynamic_mgpg(struct amdgpu_device *adev,
 {
 	u32 data, orig;
 
-	orig = data = RREG32(RLC_PG_CNTL);
+	orig = data = RREG32(mmRLC_PG_CNTL);
 	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG))
-		data |= DYN_PER_CU_PG_ENABLE;
+		data |= RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
 	else
-		data &= ~DYN_PER_CU_PG_ENABLE;
+		data &= ~RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
 	if (orig != data)
-		WREG32(RLC_PG_CNTL, data);
+		WREG32(mmRLC_PG_CNTL, data);
 }
 
 static void gfx_v6_0_init_gfx_cgpg(struct amdgpu_device *adev)
 {
 	u32 tmp;
 
-	WREG32(RLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
+	WREG32(mmRLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
+	WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_SRC, 1);
+	WREG32(mmRLC_CLEAR_STATE_RESTORE_BASE, adev->gfx.rlc.clear_state_gpu_addr >> 8);
 
-	tmp = RREG32(RLC_PG_CNTL);
-	tmp |= GFX_PG_SRC;
-	WREG32(RLC_PG_CNTL, tmp);
-
-	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, adev->gfx.rlc.clear_state_gpu_addr >> 8);
-
-	tmp = RREG32(RLC_AUTO_PG_CTRL);
-
-	tmp &= ~GRBM_REG_SGIT_MASK;
-	tmp |= GRBM_REG_SGIT(0x700);
-	tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
-	WREG32(RLC_AUTO_PG_CTRL, tmp);
+	tmp = RREG32(mmRLC_AUTO_PG_CTRL);
+	tmp &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
+	tmp |= (0x700 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
+	tmp &= ~RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD_MASK;
+	WREG32(mmRLC_AUTO_PG_CTRL, tmp);
 }
 
 static void gfx_v6_0_update_gfx_pg(struct amdgpu_device *adev, bool enable)
@@ -2703,7 +2704,6 @@ static void gfx_v6_0_get_csb_buffer(struct amdgpu_device *adev,
 
 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
-
 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
 	buffer[count++] = cpu_to_le32(0x80000000);
 	buffer[count++] = cpu_to_le32(0x80000000);
@@ -2723,7 +2723,7 @@ static void gfx_v6_0_get_csb_buffer(struct amdgpu_device *adev,
 	}
 
 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
-	buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
+	buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
 
 	switch (adev->asic_type) {
 	case CHIP_TAHITI:
@@ -2766,16 +2766,16 @@ static void gfx_v6_0_init_pg(struct amdgpu_device *adev)
 			gfx_v6_0_enable_cp_pg(adev, true);
 			gfx_v6_0_enable_gds_pg(adev, true);
 		} else {
-			WREG32(RLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
-			WREG32(RLC_CLEAR_STATE_RESTORE_BASE, adev->gfx.rlc.clear_state_gpu_addr >> 8);
+			WREG32(mmRLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
+			WREG32(mmRLC_CLEAR_STATE_RESTORE_BASE, adev->gfx.rlc.clear_state_gpu_addr >> 8);
 
 		}
 		gfx_v6_0_init_ao_cu_mask(adev);
 		gfx_v6_0_update_gfx_pg(adev, true);
 	} else {
 
-		WREG32(RLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
-		WREG32(RLC_CLEAR_STATE_RESTORE_BASE, adev->gfx.rlc.clear_state_gpu_addr >> 8);
+		WREG32(mmRLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
+		WREG32(mmRLC_CLEAR_STATE_RESTORE_BASE, adev->gfx.rlc.clear_state_gpu_addr >> 8);
 	}
 }
 
@@ -2800,50 +2800,86 @@ static uint64_t gfx_v6_0_get_gpu_clock_counter(struct amdgpu_device *adev)
 	uint64_t clock;
 
 	mutex_lock(&adev->gfx.gpu_clock_mutex);
-	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
-	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
-	        ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
+	WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
+	clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) |
+	        ((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
 	mutex_unlock(&adev->gfx.gpu_clock_mutex);
 	return clock;
 }
 
 static void gfx_v6_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
 {
+	if (flags & AMDGPU_HAVE_CTX_SWITCH)
+		gfx_v6_0_ring_emit_vgt_flush(ring);
 	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
 	amdgpu_ring_write(ring, 0x80000000);
 	amdgpu_ring_write(ring, 0);
 }
 
-static unsigned gfx_v6_0_ring_get_emit_ib_size(struct amdgpu_ring *ring)
+
+static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
 {
-	return
-		6; /* gfx_v6_0_ring_emit_ib */
+	WREG32(mmSQ_IND_INDEX,
+		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
+		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
+		(address << SQ_IND_INDEX__INDEX__SHIFT) |
+		(SQ_IND_INDEX__FORCE_READ_MASK));
+	return RREG32(mmSQ_IND_DATA);
 }
 
-static unsigned gfx_v6_0_ring_get_dma_frame_size_gfx(struct amdgpu_ring *ring)
+static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
+			   uint32_t wave, uint32_t thread,
+			   uint32_t regno, uint32_t num, uint32_t *out)
 {
-	return
-		5 + /* gfx_v6_0_ring_emit_hdp_flush */
-		5 + /* gfx_v6_0_ring_emit_hdp_invalidate */
-		14 + 14 + 14 + /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */
-		7 + 4 + /* gfx_v6_0_ring_emit_pipeline_sync */
-		17 + 6 + /* gfx_v6_0_ring_emit_vm_flush */
-		3; /* gfx_v6_ring_emit_cntxcntl */
+	WREG32(mmSQ_IND_INDEX,
+		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
+		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
+		(regno << SQ_IND_INDEX__INDEX__SHIFT) |
+		(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
+		(SQ_IND_INDEX__FORCE_READ_MASK) |
+		(SQ_IND_INDEX__AUTO_INCR_MASK));
+	while (num--)
+		*(out++) = RREG32(mmSQ_IND_DATA);
 }
 
-static unsigned gfx_v6_0_ring_get_dma_frame_size_compute(struct amdgpu_ring *ring)
+static void gfx_v6_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
 {
-	return
-		5 + /* gfx_v6_0_ring_emit_hdp_flush */
-		5 + /* gfx_v6_0_ring_emit_hdp_invalidate */
-		7 + /* gfx_v6_0_ring_emit_pipeline_sync */
-		17 + /* gfx_v6_0_ring_emit_vm_flush */
-		14 + 14 + 14; /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */
+	/* type 0 wave data */
+	dst[(*no_fields)++] = 0;
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
+}
+
+static void gfx_v6_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
+				     uint32_t wave, uint32_t start,
+				     uint32_t size, uint32_t *dst)
+{
+	wave_read_regs(
+		adev, simd, wave, 0,
+		start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
 }
 
 static const struct amdgpu_gfx_funcs gfx_v6_0_gfx_funcs = {
 	.get_gpu_clock_counter = &gfx_v6_0_get_gpu_clock_counter,
 	.select_se_sh = &gfx_v6_0_select_se_sh,
+	.read_wave_data = &gfx_v6_0_read_wave_data,
+	.read_wave_sgprs = &gfx_v6_0_read_wave_sgprs,
 };
 
 static int gfx_v6_0_early_init(void *handle)
@@ -2896,9 +2932,7 @@ static int gfx_v6_0_sw_init(void *handle)
 		ring->ring_obj = NULL;
 		sprintf(ring->name, "gfx");
 		r = amdgpu_ring_init(adev, ring, 1024,
-				     0x80000000, 0xf,
-				     &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_EOP,
-				     AMDGPU_RING_TYPE_GFX);
+				     &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_EOP);
 		if (r)
 			return r;
 	}
@@ -2920,9 +2954,7 @@ static int gfx_v6_0_sw_init(void *handle)
 		sprintf(ring->name, "comp %d.%d.%d", ring->me, ring->pipe, ring->queue);
 		irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ring->pipe;
 		r = amdgpu_ring_init(adev, ring, 1024,
-				     0x80000000, 0xf,
-				     &adev->gfx.eop_irq, irq_type,
-				     AMDGPU_RING_TYPE_COMPUTE);
+				     &adev->gfx.eop_irq, irq_type);
 		if (r)
 			return r;
 	}
@@ -2998,7 +3030,7 @@ static bool gfx_v6_0_is_idle(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	if (RREG32(GRBM_STATUS) & GRBM_STATUS__GUI_ACTIVE_MASK)
+	if (RREG32(mmGRBM_STATUS) & GRBM_STATUS__GUI_ACTIVE_MASK)
 		return false;
 	else
 		return true;
@@ -3029,14 +3061,14 @@ static void gfx_v6_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
 
 	switch (state) {
 	case AMDGPU_IRQ_STATE_DISABLE:
-		cp_int_cntl = RREG32(CP_INT_CNTL_RING0);
-		cp_int_cntl &= ~CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK;
-		WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
+		cp_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
+		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
 		break;
 	case AMDGPU_IRQ_STATE_ENABLE:
-		cp_int_cntl = RREG32(CP_INT_CNTL_RING0);
-		cp_int_cntl |= CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK;
-		WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
+		cp_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
+		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
 		break;
 	default:
 		break;
@@ -3051,27 +3083,27 @@ static void gfx_v6_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
 	switch (state){
 	case AMDGPU_IRQ_STATE_DISABLE:
 		if (ring == 0) {
-			cp_int_cntl = RREG32(CP_INT_CNTL_RING1);
-			cp_int_cntl &= ~CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK;
-			WREG32(CP_INT_CNTL_RING1, cp_int_cntl);
+			cp_int_cntl = RREG32(mmCP_INT_CNTL_RING1);
+			cp_int_cntl &= ~CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE_MASK;
+			WREG32(mmCP_INT_CNTL_RING1, cp_int_cntl);
 			break;
 		} else {
-			cp_int_cntl = RREG32(CP_INT_CNTL_RING2);
-			cp_int_cntl &= ~CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK;
-			WREG32(CP_INT_CNTL_RING2, cp_int_cntl);
+			cp_int_cntl = RREG32(mmCP_INT_CNTL_RING2);
+			cp_int_cntl &= ~CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE_MASK;
+			WREG32(mmCP_INT_CNTL_RING2, cp_int_cntl);
 			break;
 
 		}
 	case AMDGPU_IRQ_STATE_ENABLE:
 		if (ring == 0) {
-			cp_int_cntl = RREG32(CP_INT_CNTL_RING1);
-			cp_int_cntl |= CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK;
-			WREG32(CP_INT_CNTL_RING1, cp_int_cntl);
+			cp_int_cntl = RREG32(mmCP_INT_CNTL_RING1);
+			cp_int_cntl |= CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE_MASK;
+			WREG32(mmCP_INT_CNTL_RING1, cp_int_cntl);
 			break;
 		} else {
-			cp_int_cntl = RREG32(CP_INT_CNTL_RING2);
-			cp_int_cntl |= CP_INT_CNTL_RING__TIME_STAMP_INT_ENABLE_MASK;
-			WREG32(CP_INT_CNTL_RING2, cp_int_cntl);
+			cp_int_cntl = RREG32(mmCP_INT_CNTL_RING2);
+			cp_int_cntl |= CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE_MASK;
+			WREG32(mmCP_INT_CNTL_RING2, cp_int_cntl);
 			break;
 
 		}
@@ -3092,14 +3124,14 @@ static int gfx_v6_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
 
 	switch (state) {
 	case AMDGPU_IRQ_STATE_DISABLE:
-		cp_int_cntl = RREG32(CP_INT_CNTL_RING0);
+		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
 		cp_int_cntl &= ~CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK;
-		WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
 		break;
 	case AMDGPU_IRQ_STATE_ENABLE:
-		cp_int_cntl = RREG32(CP_INT_CNTL_RING0);
+		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
 		cp_int_cntl |= CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK;
-		WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
 		break;
 	default:
 		break;
@@ -3117,14 +3149,14 @@ static int gfx_v6_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
 
 	switch (state) {
 	case AMDGPU_IRQ_STATE_DISABLE:
-		cp_int_cntl = RREG32(CP_INT_CNTL_RING0);
+		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
 		cp_int_cntl &= ~CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK;
-		WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
 		break;
 	case AMDGPU_IRQ_STATE_ENABLE:
-		cp_int_cntl = RREG32(CP_INT_CNTL_RING0);
+		cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
 		cp_int_cntl |= CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK;
-		WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
+		WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
 		break;
 	default:
 		break;
@@ -3164,7 +3196,7 @@ static int gfx_v6_0_eop_irq(struct amdgpu_device *adev,
 		break;
 	case 1:
 	case 2:
-		amdgpu_fence_process(&adev->gfx.compute_ring[entry->ring_id -1]);
+		amdgpu_fence_process(&adev->gfx.compute_ring[entry->ring_id - 1]);
 		break;
 	default:
 		break;
@@ -3237,7 +3269,7 @@ static int gfx_v6_0_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs gfx_v6_0_ip_funcs = {
+static const struct amd_ip_funcs gfx_v6_0_ip_funcs = {
 	.name = "gfx_v6_0",
 	.early_init = gfx_v6_0_early_init,
 	.late_init = NULL,
@@ -3255,10 +3287,20 @@ const struct amd_ip_funcs gfx_v6_0_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_gfx = {
+	.type = AMDGPU_RING_TYPE_GFX,
+	.align_mask = 0xff,
+	.nop = 0x80000000,
 	.get_rptr = gfx_v6_0_ring_get_rptr,
 	.get_wptr = gfx_v6_0_ring_get_wptr,
 	.set_wptr = gfx_v6_0_ring_set_wptr_gfx,
-	.parse_cs = NULL,
+	.emit_frame_size =
+		5 + /* gfx_v6_0_ring_emit_hdp_flush */
+		5 + /* gfx_v6_0_ring_emit_hdp_invalidate */
+		14 + 14 + 14 + /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */
+		7 + 4 + /* gfx_v6_0_ring_emit_pipeline_sync */
+		17 + 6 + /* gfx_v6_0_ring_emit_vm_flush */
+		3 + 2, /* gfx_v6_ring_emit_cntxcntl including vgt flush */
+	.emit_ib_size = 6, /* gfx_v6_0_ring_emit_ib */
 	.emit_ib = gfx_v6_0_ring_emit_ib,
 	.emit_fence = gfx_v6_0_ring_emit_fence,
 	.emit_pipeline_sync = gfx_v6_0_ring_emit_pipeline_sync,
@@ -3269,15 +3311,22 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_gfx = {
 	.test_ib = gfx_v6_0_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
 	.emit_cntxcntl = gfx_v6_ring_emit_cntxcntl,
-	.get_emit_ib_size = gfx_v6_0_ring_get_emit_ib_size,
-	.get_dma_frame_size = gfx_v6_0_ring_get_dma_frame_size_gfx,
 };
 
 static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_compute = {
+	.type = AMDGPU_RING_TYPE_COMPUTE,
+	.align_mask = 0xff,
+	.nop = 0x80000000,
 	.get_rptr = gfx_v6_0_ring_get_rptr,
 	.get_wptr = gfx_v6_0_ring_get_wptr,
 	.set_wptr = gfx_v6_0_ring_set_wptr_compute,
-	.parse_cs = NULL,
+	.emit_frame_size =
+		5 + /* gfx_v6_0_ring_emit_hdp_flush */
+		5 + /* gfx_v6_0_ring_emit_hdp_invalidate */
+		7 + /* gfx_v6_0_ring_emit_pipeline_sync */
+		17 + /* gfx_v6_0_ring_emit_vm_flush */
+		14 + 14 + 14, /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */
+	.emit_ib_size = 6, /* gfx_v6_0_ring_emit_ib */
 	.emit_ib = gfx_v6_0_ring_emit_ib,
 	.emit_fence = gfx_v6_0_ring_emit_fence,
 	.emit_pipeline_sync = gfx_v6_0_ring_emit_pipeline_sync,
@@ -3287,8 +3336,6 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_compute = {
 	.test_ring = gfx_v6_0_ring_test_ring,
 	.test_ib = gfx_v6_0_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
-	.get_emit_ib_size = gfx_v6_0_ring_get_emit_ib_size,
-	.get_dma_frame_size = gfx_v6_0_ring_get_dma_frame_size_compute,
 };
 
 static void gfx_v6_0_set_ring_funcs(struct amdgpu_device *adev)
@@ -3360,3 +3407,12 @@ static void gfx_v6_0_get_cu_info(struct amdgpu_device *adev)
 	cu_info->number = active_cu_number;
 	cu_info->ao_cu_mask = ao_cu_mask;
 }
+
+const struct amdgpu_ip_block_version gfx_v6_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GFX,
+	.major = 6,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &gfx_v6_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.h b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.h
index b9657e72b248..ced6fc42f688 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.h
@@ -24,6 +24,6 @@
 #ifndef __GFX_V6_0_H__
 #define __GFX_V6_0_H__
 
-extern const struct amd_ip_funcs gfx_v6_0_ip_funcs;
+extern const struct amdgpu_ip_block_version gfx_v6_0_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 71116da9e782..c4e14015ec5b 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -2077,9 +2077,9 @@ static int gfx_v7_0_ring_test_ring(struct amdgpu_ring *ring)
 static void gfx_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
 	u32 ref_and_mask;
-	int usepfp = ring->type == AMDGPU_RING_TYPE_COMPUTE ? 0 : 1;
+	int usepfp = ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE ? 0 : 1;
 
-	if (ring->type == AMDGPU_RING_TYPE_COMPUTE) {
+	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
 		switch (ring->me) {
 		case 1:
 			ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
@@ -2105,6 +2105,18 @@ static void gfx_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, 0x20); /* poll interval */
 }
 
+static void gfx_v7_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
+{
+	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
+	amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) |
+		EVENT_INDEX(4));
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
+	amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
+		EVENT_INDEX(0));
+}
+
+
 /**
  * gfx_v7_0_ring_emit_hdp_invalidate - emit an hdp invalidate on the cp
  *
@@ -2260,6 +2272,7 @@ static void gfx_v7_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
 
 	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
 	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
+		gfx_v7_0_ring_emit_vgt_flush(ring);
 		/* set load_global_config & load_global_uconfig */
 		dw2 |= 0x8001;
 		/* set load_cs_sh_regs */
@@ -2286,7 +2299,7 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_ib ib;
-	struct fence *f = NULL;
+	struct dma_fence *f = NULL;
 	uint32_t scratch;
 	uint32_t tmp = 0;
 	long r;
@@ -2312,7 +2325,7 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	if (r)
 		goto err2;
 
-	r = fence_wait_timeout(f, false, timeout);
+	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
 		DRM_ERROR("amdgpu: IB test timed out\n");
 		r = -ETIMEDOUT;
@@ -2333,7 +2346,7 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 err2:
 	amdgpu_ib_free(adev, &ib, NULL);
-	fence_put(f);
+	dma_fence_put(f);
 err1:
 	amdgpu_gfx_scratch_free(adev, scratch);
 	return r;
@@ -3222,7 +3235,7 @@ static int gfx_v7_0_cp_resume(struct amdgpu_device *adev)
  */
 static void gfx_v7_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 {
-	int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
+	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
 	uint32_t seq = ring->fence_drv.sync_seq;
 	uint64_t addr = ring->fence_drv.gpu_addr;
 
@@ -3262,7 +3275,7 @@ static void gfx_v7_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 static void gfx_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 					unsigned vm_id, uint64_t pd_addr)
 {
-	int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
+	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
 
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
@@ -3391,7 +3404,8 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
 		if (adev->gfx.rlc.save_restore_obj == NULL) {
 			r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
 					     AMDGPU_GEM_DOMAIN_VRAM,
-					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+					     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 					     NULL, NULL,
 					     &adev->gfx.rlc.save_restore_obj);
 			if (r) {
@@ -3435,7 +3449,8 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
 		if (adev->gfx.rlc.clear_state_obj == NULL) {
 			r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
 					     AMDGPU_GEM_DOMAIN_VRAM,
-					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+					     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 					     NULL, NULL,
 					     &adev->gfx.rlc.clear_state_obj);
 			if (r) {
@@ -3475,7 +3490,8 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
 		if (adev->gfx.rlc.cp_table_obj == NULL) {
 			r = amdgpu_bo_create(adev, adev->gfx.rlc.cp_table_size, PAGE_SIZE, true,
 					     AMDGPU_GEM_DOMAIN_VRAM,
-					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+					     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 					     NULL, NULL,
 					     &adev->gfx.rlc.cp_table_obj);
 			if (r) {
@@ -4354,44 +4370,69 @@ static void gfx_v7_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
 }
 
-static unsigned gfx_v7_0_ring_get_emit_ib_size_gfx(struct amdgpu_ring *ring)
+static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
 {
-	return
-		4; /* gfx_v7_0_ring_emit_ib_gfx */
+	WREG32(mmSQ_IND_INDEX,
+		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
+		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
+		(address << SQ_IND_INDEX__INDEX__SHIFT) |
+		(SQ_IND_INDEX__FORCE_READ_MASK));
+	return RREG32(mmSQ_IND_DATA);
 }
 
-static unsigned gfx_v7_0_ring_get_dma_frame_size_gfx(struct amdgpu_ring *ring)
+static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
+			   uint32_t wave, uint32_t thread,
+			   uint32_t regno, uint32_t num, uint32_t *out)
 {
-	return
-		20 + /* gfx_v7_0_ring_emit_gds_switch */
-		7 + /* gfx_v7_0_ring_emit_hdp_flush */
-		5 + /* gfx_v7_0_ring_emit_hdp_invalidate */
-		12 + 12 + 12 + /* gfx_v7_0_ring_emit_fence_gfx x3 for user fence, vm fence */
-		7 + 4 + /* gfx_v7_0_ring_emit_pipeline_sync */
-		17 + 6 + /* gfx_v7_0_ring_emit_vm_flush */
-		3; /* gfx_v7_ring_emit_cntxcntl */
+	WREG32(mmSQ_IND_INDEX,
+		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
+		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
+		(regno << SQ_IND_INDEX__INDEX__SHIFT) |
+		(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
+		(SQ_IND_INDEX__FORCE_READ_MASK) |
+		(SQ_IND_INDEX__AUTO_INCR_MASK));
+	while (num--)
+		*(out++) = RREG32(mmSQ_IND_DATA);
 }
 
-static unsigned gfx_v7_0_ring_get_emit_ib_size_compute(struct amdgpu_ring *ring)
+static void gfx_v7_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
 {
-	return
-		4; /* gfx_v7_0_ring_emit_ib_compute */
+	/* type 0 wave data */
+	dst[(*no_fields)++] = 0;
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
 }
 
-static unsigned gfx_v7_0_ring_get_dma_frame_size_compute(struct amdgpu_ring *ring)
+static void gfx_v7_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
+				     uint32_t wave, uint32_t start,
+				     uint32_t size, uint32_t *dst)
 {
-	return
-		20 + /* gfx_v7_0_ring_emit_gds_switch */
-		7 + /* gfx_v7_0_ring_emit_hdp_flush */
-		5 + /* gfx_v7_0_ring_emit_hdp_invalidate */
-		7 + /* gfx_v7_0_ring_emit_pipeline_sync */
-		17 + /* gfx_v7_0_ring_emit_vm_flush */
-		7 + 7 + 7; /* gfx_v7_0_ring_emit_fence_compute x3 for user fence, vm fence */
+	wave_read_regs(
+		adev, simd, wave, 0,
+		start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
 }
 
 static const struct amdgpu_gfx_funcs gfx_v7_0_gfx_funcs = {
 	.get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
 	.select_se_sh = &gfx_v7_0_select_se_sh,
+	.read_wave_data = &gfx_v7_0_read_wave_data,
+	.read_wave_sgprs = &gfx_v7_0_read_wave_sgprs,
 };
 
 static const struct amdgpu_rlc_funcs gfx_v7_0_rlc_funcs = {
@@ -4643,9 +4684,7 @@ static int gfx_v7_0_sw_init(void *handle)
 		ring->ring_obj = NULL;
 		sprintf(ring->name, "gfx");
 		r = amdgpu_ring_init(adev, ring, 1024,
-				     PACKET3(PACKET3_NOP, 0x3FFF), 0xf,
-				     &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_EOP,
-				     AMDGPU_RING_TYPE_GFX);
+				     &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_EOP);
 		if (r)
 			return r;
 	}
@@ -4670,9 +4709,7 @@ static int gfx_v7_0_sw_init(void *handle)
 		irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ring->pipe;
 		/* type-2 packets are deprecated on MEC, use type-3 instead */
 		r = amdgpu_ring_init(adev, ring, 1024,
-				     PACKET3(PACKET3_NOP, 0x3FFF), 0xf,
-				     &adev->gfx.eop_irq, irq_type,
-				     AMDGPU_RING_TYPE_COMPUTE);
+				     &adev->gfx.eop_irq, irq_type);
 		if (r)
 			return r;
 	}
@@ -5123,7 +5160,7 @@ static int gfx_v7_0_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs gfx_v7_0_ip_funcs = {
+static const struct amd_ip_funcs gfx_v7_0_ip_funcs = {
 	.name = "gfx_v7_0",
 	.early_init = gfx_v7_0_early_init,
 	.late_init = gfx_v7_0_late_init,
@@ -5141,10 +5178,21 @@ const struct amd_ip_funcs gfx_v7_0_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = {
+	.type = AMDGPU_RING_TYPE_GFX,
+	.align_mask = 0xff,
+	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.get_rptr = gfx_v7_0_ring_get_rptr,
 	.get_wptr = gfx_v7_0_ring_get_wptr_gfx,
 	.set_wptr = gfx_v7_0_ring_set_wptr_gfx,
-	.parse_cs = NULL,
+	.emit_frame_size =
+		20 + /* gfx_v7_0_ring_emit_gds_switch */
+		7 + /* gfx_v7_0_ring_emit_hdp_flush */
+		5 + /* gfx_v7_0_ring_emit_hdp_invalidate */
+		12 + 12 + 12 + /* gfx_v7_0_ring_emit_fence_gfx x3 for user fence, vm fence */
+		7 + 4 + /* gfx_v7_0_ring_emit_pipeline_sync */
+		17 + 6 + /* gfx_v7_0_ring_emit_vm_flush */
+		3 + 4, /* gfx_v7_ring_emit_cntxcntl including vgt flush*/
+	.emit_ib_size = 4, /* gfx_v7_0_ring_emit_ib_gfx */
 	.emit_ib = gfx_v7_0_ring_emit_ib_gfx,
 	.emit_fence = gfx_v7_0_ring_emit_fence_gfx,
 	.emit_pipeline_sync = gfx_v7_0_ring_emit_pipeline_sync,
@@ -5157,15 +5205,23 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = {
 	.insert_nop = amdgpu_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.emit_cntxcntl = gfx_v7_ring_emit_cntxcntl,
-	.get_emit_ib_size = gfx_v7_0_ring_get_emit_ib_size_gfx,
-	.get_dma_frame_size = gfx_v7_0_ring_get_dma_frame_size_gfx,
 };
 
 static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = {
+	.type = AMDGPU_RING_TYPE_COMPUTE,
+	.align_mask = 0xff,
+	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.get_rptr = gfx_v7_0_ring_get_rptr,
 	.get_wptr = gfx_v7_0_ring_get_wptr_compute,
 	.set_wptr = gfx_v7_0_ring_set_wptr_compute,
-	.parse_cs = NULL,
+	.emit_frame_size =
+		20 + /* gfx_v7_0_ring_emit_gds_switch */
+		7 + /* gfx_v7_0_ring_emit_hdp_flush */
+		5 + /* gfx_v7_0_ring_emit_hdp_invalidate */
+		7 + /* gfx_v7_0_ring_emit_pipeline_sync */
+		17 + /* gfx_v7_0_ring_emit_vm_flush */
+		7 + 7 + 7, /* gfx_v7_0_ring_emit_fence_compute x3 for user fence, vm fence */
+	.emit_ib_size =	4, /* gfx_v7_0_ring_emit_ib_compute */
 	.emit_ib = gfx_v7_0_ring_emit_ib_compute,
 	.emit_fence = gfx_v7_0_ring_emit_fence_compute,
 	.emit_pipeline_sync = gfx_v7_0_ring_emit_pipeline_sync,
@@ -5177,8 +5233,6 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = {
 	.test_ib = gfx_v7_0_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
-	.get_emit_ib_size = gfx_v7_0_ring_get_emit_ib_size_compute,
-	.get_dma_frame_size = gfx_v7_0_ring_get_dma_frame_size_compute,
 };
 
 static void gfx_v7_0_set_ring_funcs(struct amdgpu_device *adev)
@@ -5289,3 +5343,39 @@ static void gfx_v7_0_get_cu_info(struct amdgpu_device *adev)
 	cu_info->number = active_cu_number;
 	cu_info->ao_cu_mask = ao_cu_mask;
 }
+
+const struct amdgpu_ip_block_version gfx_v7_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GFX,
+	.major = 7,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &gfx_v7_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version gfx_v7_1_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GFX,
+	.major = 7,
+	.minor = 1,
+	.rev = 0,
+	.funcs = &gfx_v7_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version gfx_v7_2_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GFX,
+	.major = 7,
+	.minor = 2,
+	.rev = 0,
+	.funcs = &gfx_v7_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version gfx_v7_3_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GFX,
+	.major = 7,
+	.minor = 3,
+	.rev = 0,
+	.funcs = &gfx_v7_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h
index 94e3ea147c26..2f5164cc0e53 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h
@@ -24,6 +24,9 @@
 #ifndef __GFX_V7_0_H__
 #define __GFX_V7_0_H__
 
-extern const struct amd_ip_funcs gfx_v7_0_ip_funcs;
+extern const struct amdgpu_ip_block_version gfx_v7_0_ip_block;
+extern const struct amdgpu_ip_block_version gfx_v7_1_ip_block;
+extern const struct amdgpu_ip_block_version gfx_v7_2_ip_block;
+extern const struct amdgpu_ip_block_version gfx_v7_3_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index bb97182dc749..6324f67bdb1f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -25,6 +25,7 @@
 #include "amdgpu.h"
 #include "amdgpu_gfx.h"
 #include "vi.h"
+#include "vi_structs.h"
 #include "vid.h"
 #include "amdgpu_ucode.h"
 #include "amdgpu_atombios.h"
@@ -167,6 +168,7 @@ static const u32 golden_settings_tonga_a11[] =
 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
 	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
+	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
 	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
@@ -797,7 +799,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_ib ib;
-	struct fence *f = NULL;
+	struct dma_fence *f = NULL;
 	uint32_t scratch;
 	uint32_t tmp = 0;
 	long r;
@@ -823,7 +825,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	if (r)
 		goto err2;
 
-	r = fence_wait_timeout(f, false, timeout);
+	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
 		DRM_ERROR("amdgpu: IB test timed out.\n");
 		r = -ETIMEDOUT;
@@ -843,7 +845,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	}
 err2:
 	amdgpu_ib_free(adev, &ib, NULL);
-	fence_put(f);
+	dma_fence_put(f);
 err1:
 	amdgpu_gfx_scratch_free(adev, scratch);
 	return r;
@@ -1057,6 +1059,19 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
 		adev->firmware.fw_size +=
 			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
 
+		/* we need account JT in */
+		cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
+		adev->firmware.fw_size +=
+			ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE);
+
+		if (amdgpu_sriov_vf(adev)) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE];
+			info->ucode_id = AMDGPU_UCODE_ID_STORAGE;
+			info->fw = adev->gfx.mec_fw;
+			adev->firmware.fw_size +=
+				ALIGN(le32_to_cpu(64 * PAGE_SIZE), PAGE_SIZE);
+		}
+
 		if (adev->gfx.mec2_fw) {
 			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
 			info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
@@ -1126,34 +1141,8 @@ static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev,
 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
 	buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG -
 			PACKET3_SET_CONTEXT_REG_START);
-	switch (adev->asic_type) {
-	case CHIP_TONGA:
-	case CHIP_POLARIS10:
-		buffer[count++] = cpu_to_le32(0x16000012);
-		buffer[count++] = cpu_to_le32(0x0000002A);
-		break;
-	case CHIP_POLARIS11:
-		buffer[count++] = cpu_to_le32(0x16000012);
-		buffer[count++] = cpu_to_le32(0x00000000);
-		break;
-	case CHIP_FIJI:
-		buffer[count++] = cpu_to_le32(0x3a00161a);
-		buffer[count++] = cpu_to_le32(0x0000002e);
-		break;
-	case CHIP_TOPAZ:
-	case CHIP_CARRIZO:
-		buffer[count++] = cpu_to_le32(0x00000002);
-		buffer[count++] = cpu_to_le32(0x00000000);
-		break;
-	case CHIP_STONEY:
-		buffer[count++] = cpu_to_le32(0x00000000);
-		buffer[count++] = cpu_to_le32(0x00000000);
-		break;
-	default:
-		buffer[count++] = cpu_to_le32(0x00000000);
-		buffer[count++] = cpu_to_le32(0x00000000);
-		break;
-	}
+	buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config);
+	buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1);
 
 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
@@ -1272,7 +1261,8 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
 		if (adev->gfx.rlc.clear_state_obj == NULL) {
 			r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
 					     AMDGPU_GEM_DOMAIN_VRAM,
-					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+					     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 					     NULL, NULL,
 					     &adev->gfx.rlc.clear_state_obj);
 			if (r) {
@@ -1314,7 +1304,8 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
 		if (adev->gfx.rlc.cp_table_obj == NULL) {
 			r = amdgpu_bo_create(adev, adev->gfx.rlc.cp_table_size, PAGE_SIZE, true,
 					     AMDGPU_GEM_DOMAIN_VRAM,
-					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+					     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 					     NULL, NULL,
 					     &adev->gfx.rlc.cp_table_obj);
 			if (r) {
@@ -1382,7 +1373,7 @@ static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
 
 	if (adev->gfx.mec.hpd_eop_obj == NULL) {
 		r = amdgpu_bo_create(adev,
-				     adev->gfx.mec.num_mec *adev->gfx.mec.num_pipe * MEC_HPD_SIZE * 2,
+				     adev->gfx.mec.num_queue * MEC_HPD_SIZE,
 				     PAGE_SIZE, true,
 				     AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
 				     &adev->gfx.mec.hpd_eop_obj);
@@ -1411,7 +1402,7 @@ static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
 		return r;
 	}
 
-	memset(hpd, 0, adev->gfx.mec.num_mec *adev->gfx.mec.num_pipe * MEC_HPD_SIZE * 2);
+	memset(hpd, 0, adev->gfx.mec.num_queue * MEC_HPD_SIZE);
 
 	amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
 	amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
@@ -1574,7 +1565,7 @@ static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
 {
 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
 	struct amdgpu_ib ib;
-	struct fence *f = NULL;
+	struct dma_fence *f = NULL;
 	int r, i;
 	u32 tmp;
 	unsigned total_size, vgpr_offset, sgpr_offset;
@@ -1707,7 +1698,7 @@ static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
 	}
 
 	/* wait for the GPU to finish processing the IB */
-	r = fence_wait(f, false);
+	r = dma_fence_wait(f, false);
 	if (r) {
 		DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
 		goto fail;
@@ -1728,7 +1719,7 @@ static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
 
 fail:
 	amdgpu_ib_free(adev, &ib, NULL);
-	fence_put(f);
+	dma_fence_put(f);
 
 	return r;
 }
@@ -2044,10 +2035,8 @@ static int gfx_v8_0_sw_init(void *handle)
 			ring->doorbell_index = AMDGPU_DOORBELL_GFX_RING0;
 		}
 
-		r = amdgpu_ring_init(adev, ring, 1024,
-				     PACKET3(PACKET3_NOP, 0x3FFF), 0xf,
-				     &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_EOP,
-				     AMDGPU_RING_TYPE_GFX);
+		r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
+				     AMDGPU_CP_IRQ_GFX_EOP);
 		if (r)
 			return r;
 	}
@@ -2071,10 +2060,8 @@ static int gfx_v8_0_sw_init(void *handle)
 		sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
 		irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ring->pipe;
 		/* type-2 packets are deprecated on MEC, use type-3 instead */
-		r = amdgpu_ring_init(adev, ring, 1024,
-				     PACKET3(PACKET3_NOP, 0x3FFF), 0xf,
-				     &adev->gfx.eop_irq, irq_type,
-				     AMDGPU_RING_TYPE_COMPUTE);
+		r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
+				     irq_type);
 		if (r)
 			return r;
 	}
@@ -3678,6 +3665,21 @@ static void gfx_v8_0_setup_rb(struct amdgpu_device *adev)
 							num_rb_pipes);
 	}
 
+	/* cache the values for userspace */
+	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
+		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
+			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
+			adev->gfx.config.rb_config[i][j].rb_backend_disable =
+				RREG32(mmCC_RB_BACKEND_DISABLE);
+			adev->gfx.config.rb_config[i][j].user_rb_backend_disable =
+				RREG32(mmGC_USER_RB_BACKEND_DISABLE);
+			adev->gfx.config.rb_config[i][j].raster_config =
+				RREG32(mmPA_SC_RASTER_CONFIG);
+			adev->gfx.config.rb_config[i][j].raster_config_1 =
+				RREG32(mmPA_SC_RASTER_CONFIG_1);
+		}
+	}
+	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 	mutex_unlock(&adev->grbm_idx_mutex);
 }
 
@@ -3904,7 +3906,7 @@ static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
 	int list_size;
 	unsigned int *register_list_format =
 		kmalloc(adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
-	if (register_list_format == NULL)
+	if (!register_list_format)
 		return -ENOMEM;
 	memcpy(register_list_format, adev->gfx.rlc.register_list_format,
 			adev->gfx.rlc.reg_list_format_size_bytes);
@@ -4330,7 +4332,7 @@ static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev)
 	struct amdgpu_ring *ring;
 	u32 tmp;
 	u32 rb_bufsz;
-	u64 rb_addr, rptr_addr;
+	u64 rb_addr, rptr_addr, wptr_gpu_addr;
 	int r;
 
 	/* Set the write pointer delay */
@@ -4361,6 +4363,9 @@ static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev)
 	WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
 	WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
 
+	wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+	WREG32(mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
+	WREG32(mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
 	mdelay(1);
 	WREG32(mmCP_RB0_CNTL, tmp);
 
@@ -4466,267 +4471,6 @@ static int gfx_v8_0_cp_compute_load_microcode(struct amdgpu_device *adev)
 	return 0;
 }
 
-struct vi_mqd {
-	uint32_t header;  /* ordinal0 */
-	uint32_t compute_dispatch_initiator;  /* ordinal1 */
-	uint32_t compute_dim_x;  /* ordinal2 */
-	uint32_t compute_dim_y;  /* ordinal3 */
-	uint32_t compute_dim_z;  /* ordinal4 */
-	uint32_t compute_start_x;  /* ordinal5 */
-	uint32_t compute_start_y;  /* ordinal6 */
-	uint32_t compute_start_z;  /* ordinal7 */
-	uint32_t compute_num_thread_x;  /* ordinal8 */
-	uint32_t compute_num_thread_y;  /* ordinal9 */
-	uint32_t compute_num_thread_z;  /* ordinal10 */
-	uint32_t compute_pipelinestat_enable;  /* ordinal11 */
-	uint32_t compute_perfcount_enable;  /* ordinal12 */
-	uint32_t compute_pgm_lo;  /* ordinal13 */
-	uint32_t compute_pgm_hi;  /* ordinal14 */
-	uint32_t compute_tba_lo;  /* ordinal15 */
-	uint32_t compute_tba_hi;  /* ordinal16 */
-	uint32_t compute_tma_lo;  /* ordinal17 */
-	uint32_t compute_tma_hi;  /* ordinal18 */
-	uint32_t compute_pgm_rsrc1;  /* ordinal19 */
-	uint32_t compute_pgm_rsrc2;  /* ordinal20 */
-	uint32_t compute_vmid;  /* ordinal21 */
-	uint32_t compute_resource_limits;  /* ordinal22 */
-	uint32_t compute_static_thread_mgmt_se0;  /* ordinal23 */
-	uint32_t compute_static_thread_mgmt_se1;  /* ordinal24 */
-	uint32_t compute_tmpring_size;  /* ordinal25 */
-	uint32_t compute_static_thread_mgmt_se2;  /* ordinal26 */
-	uint32_t compute_static_thread_mgmt_se3;  /* ordinal27 */
-	uint32_t compute_restart_x;  /* ordinal28 */
-	uint32_t compute_restart_y;  /* ordinal29 */
-	uint32_t compute_restart_z;  /* ordinal30 */
-	uint32_t compute_thread_trace_enable;  /* ordinal31 */
-	uint32_t compute_misc_reserved;  /* ordinal32 */
-	uint32_t compute_dispatch_id;  /* ordinal33 */
-	uint32_t compute_threadgroup_id;  /* ordinal34 */
-	uint32_t compute_relaunch;  /* ordinal35 */
-	uint32_t compute_wave_restore_addr_lo;  /* ordinal36 */
-	uint32_t compute_wave_restore_addr_hi;  /* ordinal37 */
-	uint32_t compute_wave_restore_control;  /* ordinal38 */
-	uint32_t reserved9;  /* ordinal39 */
-	uint32_t reserved10;  /* ordinal40 */
-	uint32_t reserved11;  /* ordinal41 */
-	uint32_t reserved12;  /* ordinal42 */
-	uint32_t reserved13;  /* ordinal43 */
-	uint32_t reserved14;  /* ordinal44 */
-	uint32_t reserved15;  /* ordinal45 */
-	uint32_t reserved16;  /* ordinal46 */
-	uint32_t reserved17;  /* ordinal47 */
-	uint32_t reserved18;  /* ordinal48 */
-	uint32_t reserved19;  /* ordinal49 */
-	uint32_t reserved20;  /* ordinal50 */
-	uint32_t reserved21;  /* ordinal51 */
-	uint32_t reserved22;  /* ordinal52 */
-	uint32_t reserved23;  /* ordinal53 */
-	uint32_t reserved24;  /* ordinal54 */
-	uint32_t reserved25;  /* ordinal55 */
-	uint32_t reserved26;  /* ordinal56 */
-	uint32_t reserved27;  /* ordinal57 */
-	uint32_t reserved28;  /* ordinal58 */
-	uint32_t reserved29;  /* ordinal59 */
-	uint32_t reserved30;  /* ordinal60 */
-	uint32_t reserved31;  /* ordinal61 */
-	uint32_t reserved32;  /* ordinal62 */
-	uint32_t reserved33;  /* ordinal63 */
-	uint32_t reserved34;  /* ordinal64 */
-	uint32_t compute_user_data_0;  /* ordinal65 */
-	uint32_t compute_user_data_1;  /* ordinal66 */
-	uint32_t compute_user_data_2;  /* ordinal67 */
-	uint32_t compute_user_data_3;  /* ordinal68 */
-	uint32_t compute_user_data_4;  /* ordinal69 */
-	uint32_t compute_user_data_5;  /* ordinal70 */
-	uint32_t compute_user_data_6;  /* ordinal71 */
-	uint32_t compute_user_data_7;  /* ordinal72 */
-	uint32_t compute_user_data_8;  /* ordinal73 */
-	uint32_t compute_user_data_9;  /* ordinal74 */
-	uint32_t compute_user_data_10;  /* ordinal75 */
-	uint32_t compute_user_data_11;  /* ordinal76 */
-	uint32_t compute_user_data_12;  /* ordinal77 */
-	uint32_t compute_user_data_13;  /* ordinal78 */
-	uint32_t compute_user_data_14;  /* ordinal79 */
-	uint32_t compute_user_data_15;  /* ordinal80 */
-	uint32_t cp_compute_csinvoc_count_lo;  /* ordinal81 */
-	uint32_t cp_compute_csinvoc_count_hi;  /* ordinal82 */
-	uint32_t reserved35;  /* ordinal83 */
-	uint32_t reserved36;  /* ordinal84 */
-	uint32_t reserved37;  /* ordinal85 */
-	uint32_t cp_mqd_query_time_lo;  /* ordinal86 */
-	uint32_t cp_mqd_query_time_hi;  /* ordinal87 */
-	uint32_t cp_mqd_connect_start_time_lo;  /* ordinal88 */
-	uint32_t cp_mqd_connect_start_time_hi;  /* ordinal89 */
-	uint32_t cp_mqd_connect_end_time_lo;  /* ordinal90 */
-	uint32_t cp_mqd_connect_end_time_hi;  /* ordinal91 */
-	uint32_t cp_mqd_connect_end_wf_count;  /* ordinal92 */
-	uint32_t cp_mqd_connect_end_pq_rptr;  /* ordinal93 */
-	uint32_t cp_mqd_connect_end_pq_wptr;  /* ordinal94 */
-	uint32_t cp_mqd_connect_end_ib_rptr;  /* ordinal95 */
-	uint32_t reserved38;  /* ordinal96 */
-	uint32_t reserved39;  /* ordinal97 */
-	uint32_t cp_mqd_save_start_time_lo;  /* ordinal98 */
-	uint32_t cp_mqd_save_start_time_hi;  /* ordinal99 */
-	uint32_t cp_mqd_save_end_time_lo;  /* ordinal100 */
-	uint32_t cp_mqd_save_end_time_hi;  /* ordinal101 */
-	uint32_t cp_mqd_restore_start_time_lo;  /* ordinal102 */
-	uint32_t cp_mqd_restore_start_time_hi;  /* ordinal103 */
-	uint32_t cp_mqd_restore_end_time_lo;  /* ordinal104 */
-	uint32_t cp_mqd_restore_end_time_hi;  /* ordinal105 */
-	uint32_t reserved40;  /* ordinal106 */
-	uint32_t reserved41;  /* ordinal107 */
-	uint32_t gds_cs_ctxsw_cnt0;  /* ordinal108 */
-	uint32_t gds_cs_ctxsw_cnt1;  /* ordinal109 */
-	uint32_t gds_cs_ctxsw_cnt2;  /* ordinal110 */
-	uint32_t gds_cs_ctxsw_cnt3;  /* ordinal111 */
-	uint32_t reserved42;  /* ordinal112 */
-	uint32_t reserved43;  /* ordinal113 */
-	uint32_t cp_pq_exe_status_lo;  /* ordinal114 */
-	uint32_t cp_pq_exe_status_hi;  /* ordinal115 */
-	uint32_t cp_packet_id_lo;  /* ordinal116 */
-	uint32_t cp_packet_id_hi;  /* ordinal117 */
-	uint32_t cp_packet_exe_status_lo;  /* ordinal118 */
-	uint32_t cp_packet_exe_status_hi;  /* ordinal119 */
-	uint32_t gds_save_base_addr_lo;  /* ordinal120 */
-	uint32_t gds_save_base_addr_hi;  /* ordinal121 */
-	uint32_t gds_save_mask_lo;  /* ordinal122 */
-	uint32_t gds_save_mask_hi;  /* ordinal123 */
-	uint32_t ctx_save_base_addr_lo;  /* ordinal124 */
-	uint32_t ctx_save_base_addr_hi;  /* ordinal125 */
-	uint32_t reserved44;  /* ordinal126 */
-	uint32_t reserved45;  /* ordinal127 */
-	uint32_t cp_mqd_base_addr_lo;  /* ordinal128 */
-	uint32_t cp_mqd_base_addr_hi;  /* ordinal129 */
-	uint32_t cp_hqd_active;  /* ordinal130 */
-	uint32_t cp_hqd_vmid;  /* ordinal131 */
-	uint32_t cp_hqd_persistent_state;  /* ordinal132 */
-	uint32_t cp_hqd_pipe_priority;  /* ordinal133 */
-	uint32_t cp_hqd_queue_priority;  /* ordinal134 */
-	uint32_t cp_hqd_quantum;  /* ordinal135 */
-	uint32_t cp_hqd_pq_base_lo;  /* ordinal136 */
-	uint32_t cp_hqd_pq_base_hi;  /* ordinal137 */
-	uint32_t cp_hqd_pq_rptr;  /* ordinal138 */
-	uint32_t cp_hqd_pq_rptr_report_addr_lo;  /* ordinal139 */
-	uint32_t cp_hqd_pq_rptr_report_addr_hi;  /* ordinal140 */
-	uint32_t cp_hqd_pq_wptr_poll_addr;  /* ordinal141 */
-	uint32_t cp_hqd_pq_wptr_poll_addr_hi;  /* ordinal142 */
-	uint32_t cp_hqd_pq_doorbell_control;  /* ordinal143 */
-	uint32_t cp_hqd_pq_wptr;  /* ordinal144 */
-	uint32_t cp_hqd_pq_control;  /* ordinal145 */
-	uint32_t cp_hqd_ib_base_addr_lo;  /* ordinal146 */
-	uint32_t cp_hqd_ib_base_addr_hi;  /* ordinal147 */
-	uint32_t cp_hqd_ib_rptr;  /* ordinal148 */
-	uint32_t cp_hqd_ib_control;  /* ordinal149 */
-	uint32_t cp_hqd_iq_timer;  /* ordinal150 */
-	uint32_t cp_hqd_iq_rptr;  /* ordinal151 */
-	uint32_t cp_hqd_dequeue_request;  /* ordinal152 */
-	uint32_t cp_hqd_dma_offload;  /* ordinal153 */
-	uint32_t cp_hqd_sema_cmd;  /* ordinal154 */
-	uint32_t cp_hqd_msg_type;  /* ordinal155 */
-	uint32_t cp_hqd_atomic0_preop_lo;  /* ordinal156 */
-	uint32_t cp_hqd_atomic0_preop_hi;  /* ordinal157 */
-	uint32_t cp_hqd_atomic1_preop_lo;  /* ordinal158 */
-	uint32_t cp_hqd_atomic1_preop_hi;  /* ordinal159 */
-	uint32_t cp_hqd_hq_status0;  /* ordinal160 */
-	uint32_t cp_hqd_hq_control0;  /* ordinal161 */
-	uint32_t cp_mqd_control;  /* ordinal162 */
-	uint32_t cp_hqd_hq_status1;  /* ordinal163 */
-	uint32_t cp_hqd_hq_control1;  /* ordinal164 */
-	uint32_t cp_hqd_eop_base_addr_lo;  /* ordinal165 */
-	uint32_t cp_hqd_eop_base_addr_hi;  /* ordinal166 */
-	uint32_t cp_hqd_eop_control;  /* ordinal167 */
-	uint32_t cp_hqd_eop_rptr;  /* ordinal168 */
-	uint32_t cp_hqd_eop_wptr;  /* ordinal169 */
-	uint32_t cp_hqd_eop_done_events;  /* ordinal170 */
-	uint32_t cp_hqd_ctx_save_base_addr_lo;  /* ordinal171 */
-	uint32_t cp_hqd_ctx_save_base_addr_hi;  /* ordinal172 */
-	uint32_t cp_hqd_ctx_save_control;  /* ordinal173 */
-	uint32_t cp_hqd_cntl_stack_offset;  /* ordinal174 */
-	uint32_t cp_hqd_cntl_stack_size;  /* ordinal175 */
-	uint32_t cp_hqd_wg_state_offset;  /* ordinal176 */
-	uint32_t cp_hqd_ctx_save_size;  /* ordinal177 */
-	uint32_t cp_hqd_gds_resource_state;  /* ordinal178 */
-	uint32_t cp_hqd_error;  /* ordinal179 */
-	uint32_t cp_hqd_eop_wptr_mem;  /* ordinal180 */
-	uint32_t cp_hqd_eop_dones;  /* ordinal181 */
-	uint32_t reserved46;  /* ordinal182 */
-	uint32_t reserved47;  /* ordinal183 */
-	uint32_t reserved48;  /* ordinal184 */
-	uint32_t reserved49;  /* ordinal185 */
-	uint32_t reserved50;  /* ordinal186 */
-	uint32_t reserved51;  /* ordinal187 */
-	uint32_t reserved52;  /* ordinal188 */
-	uint32_t reserved53;  /* ordinal189 */
-	uint32_t reserved54;  /* ordinal190 */
-	uint32_t reserved55;  /* ordinal191 */
-	uint32_t iqtimer_pkt_header;  /* ordinal192 */
-	uint32_t iqtimer_pkt_dw0;  /* ordinal193 */
-	uint32_t iqtimer_pkt_dw1;  /* ordinal194 */
-	uint32_t iqtimer_pkt_dw2;  /* ordinal195 */
-	uint32_t iqtimer_pkt_dw3;  /* ordinal196 */
-	uint32_t iqtimer_pkt_dw4;  /* ordinal197 */
-	uint32_t iqtimer_pkt_dw5;  /* ordinal198 */
-	uint32_t iqtimer_pkt_dw6;  /* ordinal199 */
-	uint32_t iqtimer_pkt_dw7;  /* ordinal200 */
-	uint32_t iqtimer_pkt_dw8;  /* ordinal201 */
-	uint32_t iqtimer_pkt_dw9;  /* ordinal202 */
-	uint32_t iqtimer_pkt_dw10;  /* ordinal203 */
-	uint32_t iqtimer_pkt_dw11;  /* ordinal204 */
-	uint32_t iqtimer_pkt_dw12;  /* ordinal205 */
-	uint32_t iqtimer_pkt_dw13;  /* ordinal206 */
-	uint32_t iqtimer_pkt_dw14;  /* ordinal207 */
-	uint32_t iqtimer_pkt_dw15;  /* ordinal208 */
-	uint32_t iqtimer_pkt_dw16;  /* ordinal209 */
-	uint32_t iqtimer_pkt_dw17;  /* ordinal210 */
-	uint32_t iqtimer_pkt_dw18;  /* ordinal211 */
-	uint32_t iqtimer_pkt_dw19;  /* ordinal212 */
-	uint32_t iqtimer_pkt_dw20;  /* ordinal213 */
-	uint32_t iqtimer_pkt_dw21;  /* ordinal214 */
-	uint32_t iqtimer_pkt_dw22;  /* ordinal215 */
-	uint32_t iqtimer_pkt_dw23;  /* ordinal216 */
-	uint32_t iqtimer_pkt_dw24;  /* ordinal217 */
-	uint32_t iqtimer_pkt_dw25;  /* ordinal218 */
-	uint32_t iqtimer_pkt_dw26;  /* ordinal219 */
-	uint32_t iqtimer_pkt_dw27;  /* ordinal220 */
-	uint32_t iqtimer_pkt_dw28;  /* ordinal221 */
-	uint32_t iqtimer_pkt_dw29;  /* ordinal222 */
-	uint32_t iqtimer_pkt_dw30;  /* ordinal223 */
-	uint32_t iqtimer_pkt_dw31;  /* ordinal224 */
-	uint32_t reserved56;  /* ordinal225 */
-	uint32_t reserved57;  /* ordinal226 */
-	uint32_t reserved58;  /* ordinal227 */
-	uint32_t set_resources_header;  /* ordinal228 */
-	uint32_t set_resources_dw1;  /* ordinal229 */
-	uint32_t set_resources_dw2;  /* ordinal230 */
-	uint32_t set_resources_dw3;  /* ordinal231 */
-	uint32_t set_resources_dw4;  /* ordinal232 */
-	uint32_t set_resources_dw5;  /* ordinal233 */
-	uint32_t set_resources_dw6;  /* ordinal234 */
-	uint32_t set_resources_dw7;  /* ordinal235 */
-	uint32_t reserved59;  /* ordinal236 */
-	uint32_t reserved60;  /* ordinal237 */
-	uint32_t reserved61;  /* ordinal238 */
-	uint32_t reserved62;  /* ordinal239 */
-	uint32_t reserved63;  /* ordinal240 */
-	uint32_t reserved64;  /* ordinal241 */
-	uint32_t reserved65;  /* ordinal242 */
-	uint32_t reserved66;  /* ordinal243 */
-	uint32_t reserved67;  /* ordinal244 */
-	uint32_t reserved68;  /* ordinal245 */
-	uint32_t reserved69;  /* ordinal246 */
-	uint32_t reserved70;  /* ordinal247 */
-	uint32_t reserved71;  /* ordinal248 */
-	uint32_t reserved72;  /* ordinal249 */
-	uint32_t reserved73;  /* ordinal250 */
-	uint32_t reserved74;  /* ordinal251 */
-	uint32_t reserved75;  /* ordinal252 */
-	uint32_t reserved76;  /* ordinal253 */
-	uint32_t reserved77;  /* ordinal254 */
-	uint32_t reserved78;  /* ordinal255 */
-
-	uint32_t reserved_t[256]; /* Reserve 256 dword buffer used by ucode */
-};
-
 static void gfx_v8_0_cp_compute_fini(struct amdgpu_device *adev)
 {
 	int i, r;
@@ -4760,34 +4504,7 @@ static int gfx_v8_0_cp_compute_resume(struct amdgpu_device *adev)
 	u32 *buf;
 	struct vi_mqd *mqd;
 
-	/* init the pipes */
-	mutex_lock(&adev->srbm_mutex);
-	for (i = 0; i < (adev->gfx.mec.num_pipe * adev->gfx.mec.num_mec); i++) {
-		int me = (i < 4) ? 1 : 2;
-		int pipe = (i < 4) ? i : (i - 4);
-
-		eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE);
-		eop_gpu_addr >>= 8;
-
-		vi_srbm_select(adev, me, pipe, 0, 0);
-
-		/* write the EOP addr */
-		WREG32(mmCP_HQD_EOP_BASE_ADDR, eop_gpu_addr);
-		WREG32(mmCP_HQD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr));
-
-		/* set the VMID assigned */
-		WREG32(mmCP_HQD_VMID, 0);
-
-		/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
-		tmp = RREG32(mmCP_HQD_EOP_CONTROL);
-		tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
-				    (order_base_2(MEC_HPD_SIZE / 4) - 1));
-		WREG32(mmCP_HQD_EOP_CONTROL, tmp);
-	}
-	vi_srbm_select(adev, 0, 0, 0, 0);
-	mutex_unlock(&adev->srbm_mutex);
-
-	/* init the queues.  Just two for now. */
+	/* init the queues.  */
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
 
@@ -4839,6 +4556,22 @@ static int gfx_v8_0_cp_compute_resume(struct amdgpu_device *adev)
 			       ring->pipe,
 			       ring->queue, 0);
 
+		eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE);
+		eop_gpu_addr >>= 8;
+
+		/* write the EOP addr */
+		WREG32(mmCP_HQD_EOP_BASE_ADDR, eop_gpu_addr);
+		WREG32(mmCP_HQD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr));
+
+		/* set the VMID assigned */
+		WREG32(mmCP_HQD_VMID, 0);
+
+		/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
+		tmp = RREG32(mmCP_HQD_EOP_CONTROL);
+		tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
+				    (order_base_2(MEC_HPD_SIZE / 4) - 1));
+		WREG32(mmCP_HQD_EOP_CONTROL, tmp);
+
 		/* disable wptr polling */
 		tmp = RREG32(mmCP_PQ_WPTR_POLL_CNTL);
 		tmp = REG_SET_FIELD(tmp, CP_PQ_WPTR_POLL_CNTL, EN, 0);
@@ -4922,9 +4655,9 @@ static int gfx_v8_0_cp_compute_resume(struct amdgpu_device *adev)
 
 		/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
 		wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
-		mqd->cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
+		mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
 		mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
-		WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR, mqd->cp_hqd_pq_wptr_poll_addr);
+		WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR, mqd->cp_hqd_pq_wptr_poll_addr_lo);
 		WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
 		       mqd->cp_hqd_pq_wptr_poll_addr_hi);
 
@@ -5095,6 +4828,10 @@ static int gfx_v8_0_hw_fini(void *handle)
 
 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+	if (amdgpu_sriov_vf(adev)) {
+		pr_debug("For SRIOV client, shouldn't do anything.\n");
+		return 0;
+	}
 	gfx_v8_0_cp_enable(adev, false);
 	gfx_v8_0_rlc_stop(adev);
 	gfx_v8_0_cp_compute_fini(adev);
@@ -5437,9 +5174,70 @@ static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
 }
 
+static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
+{
+	WREG32(mmSQ_IND_INDEX,
+		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
+		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
+		(address << SQ_IND_INDEX__INDEX__SHIFT) |
+		(SQ_IND_INDEX__FORCE_READ_MASK));
+	return RREG32(mmSQ_IND_DATA);
+}
+
+static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
+			   uint32_t wave, uint32_t thread,
+			   uint32_t regno, uint32_t num, uint32_t *out)
+{
+	WREG32(mmSQ_IND_INDEX,
+		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
+		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
+		(regno << SQ_IND_INDEX__INDEX__SHIFT) |
+		(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
+		(SQ_IND_INDEX__FORCE_READ_MASK) |
+		(SQ_IND_INDEX__AUTO_INCR_MASK));
+	while (num--)
+		*(out++) = RREG32(mmSQ_IND_DATA);
+}
+
+static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
+{
+	/* type 0 wave data */
+	dst[(*no_fields)++] = 0;
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
+	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
+}
+
+static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
+				     uint32_t wave, uint32_t start,
+				     uint32_t size, uint32_t *dst)
+{
+	wave_read_regs(
+		adev, simd, wave, 0,
+		start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
+}
+
+
 static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = {
 	.get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter,
 	.select_se_sh = &gfx_v8_0_select_se_sh,
+	.read_wave_data = &gfx_v8_0_read_wave_data,
+	.read_wave_sgprs = &gfx_v8_0_read_wave_sgprs,
 };
 
 static int gfx_v8_0_early_init(void *handle)
@@ -5891,29 +5689,24 @@ static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev
 	adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
-		/* 1 enable cntx_empty_int_enable/cntx_busy_int_enable/
-		 * Cmp_busy/GFX_Idle interrupts
-		 */
-		gfx_v8_0_enable_gui_idle_interrupt(adev, true);
-
 		temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
 		data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK;
 		if (temp1 != data1)
 			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
 
-		/* 2 wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
+		/* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
 		gfx_v8_0_wait_for_rlc_serdes(adev);
 
-		/* 3 - clear cgcg override */
+		/* 2 - clear cgcg override */
 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
 
 		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
 		gfx_v8_0_wait_for_rlc_serdes(adev);
 
-		/* 4 - write cmd to set CGLS */
+		/* 3 - write cmd to set CGLS */
 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD);
 
-		/* 5 - enable cgcg */
+		/* 4 - enable cgcg */
 		data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
 
 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
@@ -5931,6 +5724,11 @@ static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev
 
 		if (temp != data)
 			WREG32(mmRLC_CGCG_CGLS_CTRL, data);
+
+		/* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/
+		 * Cmp_busy/GFX_Idle interrupts
+		 */
+		gfx_v8_0_enable_gui_idle_interrupt(adev, true);
 	} else {
 		/* disable cntx_empty_int_enable & GFX Idle interrupt */
 		gfx_v8_0_enable_gui_idle_interrupt(adev, false);
@@ -6119,7 +5917,7 @@ static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
 	u32 ref_and_mask, reg_mem_engine;
 
-	if (ring->type == AMDGPU_RING_TYPE_COMPUTE) {
+	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
 		switch (ring->me) {
 		case 1:
 			ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
@@ -6147,6 +5945,18 @@ static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, 0x20); /* poll interval */
 }
 
+static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
+{
+	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
+	amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) |
+		EVENT_INDEX(4));
+
+	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
+	amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
+		EVENT_INDEX(0));
+}
+
+
 static void gfx_v8_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
@@ -6221,7 +6031,7 @@ static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
 
 static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 {
-	int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
+	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
 	uint32_t seq = ring->fence_drv.sync_seq;
 	uint64_t addr = ring->fence_drv.gpu_addr;
 
@@ -6239,11 +6049,7 @@ static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 					unsigned vm_id, uint64_t pd_addr)
 {
-	int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX);
-
-	/* GFX8 emits 128 dw nop to prevent DE do vm_flush before CE finish CEIB */
-	if (usepfp)
-		amdgpu_ring_insert_nop(ring, 128);
+	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
 
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
@@ -6336,6 +6142,7 @@ static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
 
 	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
 	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
+		gfx_v8_0_ring_emit_vgt_flush(ring);
 		/* set load_global_config & load_global_uconfig */
 		dw2 |= 0x8001;
 		/* set load_cs_sh_regs */
@@ -6359,42 +6166,6 @@ static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
 	amdgpu_ring_write(ring, 0);
 }
 
-static unsigned gfx_v8_0_ring_get_emit_ib_size_gfx(struct amdgpu_ring *ring)
-{
-	return
-		4; /* gfx_v8_0_ring_emit_ib_gfx */
-}
-
-static unsigned gfx_v8_0_ring_get_dma_frame_size_gfx(struct amdgpu_ring *ring)
-{
-	return
-		20 + /* gfx_v8_0_ring_emit_gds_switch */
-		7 + /* gfx_v8_0_ring_emit_hdp_flush */
-		5 + /* gfx_v8_0_ring_emit_hdp_invalidate */
-		6 + 6 + 6 +/* gfx_v8_0_ring_emit_fence_gfx x3 for user fence, vm fence */
-		7 + /* gfx_v8_0_ring_emit_pipeline_sync */
-		256 + 19 + /* gfx_v8_0_ring_emit_vm_flush */
-		2 + /* gfx_v8_ring_emit_sb */
-		3; /* gfx_v8_ring_emit_cntxcntl */
-}
-
-static unsigned gfx_v8_0_ring_get_emit_ib_size_compute(struct amdgpu_ring *ring)
-{
-	return
-		4; /* gfx_v8_0_ring_emit_ib_compute */
-}
-
-static unsigned gfx_v8_0_ring_get_dma_frame_size_compute(struct amdgpu_ring *ring)
-{
-	return
-		20 + /* gfx_v8_0_ring_emit_gds_switch */
-		7 + /* gfx_v8_0_ring_emit_hdp_flush */
-		5 + /* gfx_v8_0_ring_emit_hdp_invalidate */
-		7 + /* gfx_v8_0_ring_emit_pipeline_sync */
-		17 + /* gfx_v8_0_ring_emit_vm_flush */
-		7 + 7 + 7; /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */
-}
-
 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
 						 enum amdgpu_interrupt_state state)
 {
@@ -6540,7 +6311,7 @@ static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
-const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
+static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
 	.name = "gfx_v8_0",
 	.early_init = gfx_v8_0_early_init,
 	.late_init = gfx_v8_0_late_init,
@@ -6561,10 +6332,22 @@ const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
+	.type = AMDGPU_RING_TYPE_GFX,
+	.align_mask = 0xff,
+	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.get_rptr = gfx_v8_0_ring_get_rptr,
 	.get_wptr = gfx_v8_0_ring_get_wptr_gfx,
 	.set_wptr = gfx_v8_0_ring_set_wptr_gfx,
-	.parse_cs = NULL,
+	.emit_frame_size =
+		20 + /* gfx_v8_0_ring_emit_gds_switch */
+		7 + /* gfx_v8_0_ring_emit_hdp_flush */
+		5 + /* gfx_v8_0_ring_emit_hdp_invalidate */
+		6 + 6 + 6 +/* gfx_v8_0_ring_emit_fence_gfx x3 for user fence, vm fence */
+		7 + /* gfx_v8_0_ring_emit_pipeline_sync */
+		128 + 19 + /* gfx_v8_0_ring_emit_vm_flush */
+		2 + /* gfx_v8_ring_emit_sb */
+		3 + 4, /* gfx_v8_ring_emit_cntxcntl including vgt flush */
+	.emit_ib_size =	4, /* gfx_v8_0_ring_emit_ib_gfx */
 	.emit_ib = gfx_v8_0_ring_emit_ib_gfx,
 	.emit_fence = gfx_v8_0_ring_emit_fence_gfx,
 	.emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
@@ -6578,15 +6361,23 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.emit_switch_buffer = gfx_v8_ring_emit_sb,
 	.emit_cntxcntl = gfx_v8_ring_emit_cntxcntl,
-	.get_emit_ib_size = gfx_v8_0_ring_get_emit_ib_size_gfx,
-	.get_dma_frame_size = gfx_v8_0_ring_get_dma_frame_size_gfx,
 };
 
 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
+	.type = AMDGPU_RING_TYPE_COMPUTE,
+	.align_mask = 0xff,
+	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
 	.get_rptr = gfx_v8_0_ring_get_rptr,
 	.get_wptr = gfx_v8_0_ring_get_wptr_compute,
 	.set_wptr = gfx_v8_0_ring_set_wptr_compute,
-	.parse_cs = NULL,
+	.emit_frame_size =
+		20 + /* gfx_v8_0_ring_emit_gds_switch */
+		7 + /* gfx_v8_0_ring_emit_hdp_flush */
+		5 + /* gfx_v8_0_ring_emit_hdp_invalidate */
+		7 + /* gfx_v8_0_ring_emit_pipeline_sync */
+		17 + /* gfx_v8_0_ring_emit_vm_flush */
+		7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */
+	.emit_ib_size =	4, /* gfx_v8_0_ring_emit_ib_compute */
 	.emit_ib = gfx_v8_0_ring_emit_ib_compute,
 	.emit_fence = gfx_v8_0_ring_emit_fence_compute,
 	.emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
@@ -6598,8 +6389,6 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
 	.test_ib = gfx_v8_0_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
-	.get_emit_ib_size = gfx_v8_0_ring_get_emit_ib_size_compute,
-	.get_dma_frame_size = gfx_v8_0_ring_get_dma_frame_size_compute,
 };
 
 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
@@ -6752,3 +6541,21 @@ static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
 	cu_info->number = active_cu_number;
 	cu_info->ao_cu_mask = ao_cu_mask;
 }
+
+const struct amdgpu_ip_block_version gfx_v8_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GFX,
+	.major = 8,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &gfx_v8_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version gfx_v8_1_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GFX,
+	.major = 8,
+	.minor = 1,
+	.rev = 0,
+	.funcs = &gfx_v8_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h
index ebed1f829297..788cc3ab584b 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h
@@ -24,6 +24,7 @@
 #ifndef __GFX_V8_0_H__
 #define __GFX_V8_0_H__
 
-extern const struct amd_ip_funcs gfx_v8_0_ip_funcs;
+extern const struct amdgpu_ip_block_version gfx_v8_0_ip_block;
+extern const struct amdgpu_ip_block_version gfx_v8_1_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index b13c8aaec078..45a573e63d4a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -1,4 +1,3 @@
-
 /*
  * Copyright 2014 Advanced Micro Devices, Inc.
  *
@@ -26,7 +25,16 @@
 #include "amdgpu.h"
 #include "gmc_v6_0.h"
 #include "amdgpu_ucode.h"
-#include "si/sid.h"
+
+#include "bif/bif_3_0_d.h"
+#include "bif/bif_3_0_sh_mask.h"
+#include "oss/oss_1_0_d.h"
+#include "oss/oss_1_0_sh_mask.h"
+#include "gmc/gmc_6_0_d.h"
+#include "gmc/gmc_6_0_sh_mask.h"
+#include "dce/dce_6_0_d.h"
+#include "dce/dce_6_0_sh_mask.h"
+#include "si_enums.h"
 
 static void gmc_v6_0_set_gart_funcs(struct amdgpu_device *adev);
 static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev);
@@ -37,6 +45,16 @@ MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
 MODULE_FIRMWARE("radeon/verde_mc.bin");
 MODULE_FIRMWARE("radeon/oland_mc.bin");
 
+#define MC_SEQ_MISC0__MT__MASK   0xf0000000
+#define MC_SEQ_MISC0__MT__GDDR1  0x10000000
+#define MC_SEQ_MISC0__MT__DDR2   0x20000000
+#define MC_SEQ_MISC0__MT__GDDR3  0x30000000
+#define MC_SEQ_MISC0__MT__GDDR4  0x40000000
+#define MC_SEQ_MISC0__MT__GDDR5  0x50000000
+#define MC_SEQ_MISC0__MT__HBM    0x60000000
+#define MC_SEQ_MISC0__MT__DDR3   0xB0000000
+
+
 static const u32 crtc_offsets[6] =
 {
 	SI_CRTC0_REGISTER_OFFSET,
@@ -57,14 +75,14 @@ static void gmc_v6_0_mc_stop(struct amdgpu_device *adev,
 
 	gmc_v6_0_wait_for_idle((void *)adev);
 
-	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
-	if (REG_GET_FIELD(blackout, mmMC_SHARED_BLACKOUT_CNTL, xxBLACKOUT_MODE) != 1) {
+	blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
+	if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
 		/* Block CPU access */
-		WREG32(BIF_FB_EN, 0);
+		WREG32(mmBIF_FB_EN, 0);
 		/* blackout the MC */
 		blackout = REG_SET_FIELD(blackout,
-					 mmMC_SHARED_BLACKOUT_CNTL, xxBLACKOUT_MODE, 0);
-		WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
+					 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
+		WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
 	}
 	/* wait for the MC to settle */
 	udelay(100);
@@ -77,13 +95,13 @@ static void gmc_v6_0_mc_resume(struct amdgpu_device *adev,
 	u32 tmp;
 
 	/* unblackout the MC */
-	tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
-	tmp = REG_SET_FIELD(tmp, mmMC_SHARED_BLACKOUT_CNTL, xxBLACKOUT_MODE, 0);
-	WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
+	tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
+	tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
+	WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
 	/* allow CPU access */
-	tmp = REG_SET_FIELD(0, mmBIF_FB_EN, xxFB_READ_EN, 1);
-	tmp = REG_SET_FIELD(tmp, mmBIF_FB_EN, xxFB_WRITE_EN, 1);
-	WREG32(BIF_FB_EN, tmp);
+	tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
+	tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
+	WREG32(mmBIF_FB_EN, tmp);
 
 	if (adev->mode_info.num_crtc)
 		amdgpu_display_resume_mc_access(adev, save);
@@ -158,37 +176,37 @@ static int gmc_v6_0_mc_load_microcode(struct amdgpu_device *adev)
 	new_fw_data = (const __le32 *)
 		(adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 
-	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
+	running = RREG32(mmMC_SEQ_SUP_CNTL) & MC_SEQ_SUP_CNTL__RUN_MASK;
 
 	if (running == 0) {
 
 		/* reset the engine and set to writable */
-		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
-		WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
+		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
+		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
 
 		/* load mc io regs */
 		for (i = 0; i < regs_size; i++) {
-			WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
-			WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
+			WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
+			WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
 		}
 		/* load the MC ucode */
 		for (i = 0; i < ucode_size; i++) {
-			WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
+			WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
 		}
 
 		/* put the engine back into the active state */
-		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
-		WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
-		WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
+		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
+		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
+		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
 
 		/* wait for training to complete */
 		for (i = 0; i < adev->usec_timeout; i++) {
-			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
+			if (RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL) & MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D0_MASK)
 				break;
 			udelay(1);
 		}
 		for (i = 0; i < adev->usec_timeout; i++) {
-			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
+			if (RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL) & MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D1_MASK)
 				break;
 			udelay(1);
 		}
@@ -225,7 +243,7 @@ static void gmc_v6_0_mc_program(struct amdgpu_device *adev)
 		WREG32((0xb08 + j), 0x00000000);
 		WREG32((0xb09 + j), 0x00000000);
 	}
-	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
+	WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
 
 	gmc_v6_0_mc_stop(adev, &save);
 
@@ -233,24 +251,24 @@ static void gmc_v6_0_mc_program(struct amdgpu_device *adev)
 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
 	}
 
-	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
+	WREG32(mmVGA_HDP_CONTROL, VGA_HDP_CONTROL__VGA_MEMORY_DISABLE_MASK);
 	/* Update configuration */
-	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
+	WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
 	       adev->mc.vram_start >> 12);
-	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
+	WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 	       adev->mc.vram_end >> 12);
-	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
+	WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
 	       adev->vram_scratch.gpu_addr >> 12);
 	tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
 	tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
-	WREG32(MC_VM_FB_LOCATION, tmp);
+	WREG32(mmMC_VM_FB_LOCATION, tmp);
 	/* XXX double check these! */
-	WREG32(HDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
-	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
-	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
-	WREG32(MC_VM_AGP_BASE, 0);
-	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
-	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
+	WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
+	WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
+	WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
+	WREG32(mmMC_VM_AGP_BASE, 0);
+	WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
+	WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
 
 	if (gmc_v6_0_wait_for_idle((void *)adev)) {
 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
@@ -265,16 +283,16 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
 	u32 tmp;
 	int chansize, numchan;
 
-	tmp = RREG32(MC_ARB_RAMCFG);
-	if (tmp & CHANSIZE_OVERRIDE) {
+	tmp = RREG32(mmMC_ARB_RAMCFG);
+	if (tmp & (1 << 11)) {
 		chansize = 16;
-	} else if (tmp & CHANSIZE_MASK) {
+	} else if (tmp & MC_ARB_RAMCFG__CHANSIZE_MASK) {
 		chansize = 64;
 	} else {
 		chansize = 32;
 	}
-	tmp = RREG32(MC_SHARED_CHMAP);
-	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
+	tmp = RREG32(mmMC_SHARED_CHMAP);
+	switch ((tmp & MC_SHARED_CHMAP__NOOFCHAN_MASK) >> MC_SHARED_CHMAP__NOOFCHAN__SHIFT) {
 	case 0:
 	default:
 		numchan = 1;
@@ -309,15 +327,15 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
 	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
 	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
 	/* size in MB on si */
-	adev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
-	adev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
+	adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
+	adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
 	adev->mc.visible_vram_size = adev->mc.aper_size;
 
 	/* unless the user had overridden it, set the gart
 	 * size equal to the 1024 or vram, whichever is larger.
 	 */
 	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = amdgpu_ttm_get_gtt_mem_size(adev);
+		adev->mc.gtt_size = max((1024ULL << 20), adev->mc.mc_vram_size);
 	else
 		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
 
@@ -329,9 +347,9 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
 static void gmc_v6_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
 					uint32_t vmid)
 {
-	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
+	WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
 
-	WREG32(VM_INVALIDATE_REQUEST, 1 << vmid);
+	WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
 }
 
 static int gmc_v6_0_gart_set_pte_pde(struct amdgpu_device *adev,
@@ -355,20 +373,20 @@ static void gmc_v6_0_set_fault_enable_default(struct amdgpu_device *adev,
 {
 	u32 tmp;
 
-	tmp = RREG32(VM_CONTEXT1_CNTL);
-	tmp = REG_SET_FIELD(tmp, mmVM_CONTEXT1_CNTL,
-			    xxRANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
-	tmp = REG_SET_FIELD(tmp, mmVM_CONTEXT1_CNTL,
-			    xxDUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
-	tmp = REG_SET_FIELD(tmp, mmVM_CONTEXT1_CNTL,
-			    xxPDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
-	tmp = REG_SET_FIELD(tmp, mmVM_CONTEXT1_CNTL,
-			    xxVALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
-	tmp = REG_SET_FIELD(tmp, mmVM_CONTEXT1_CNTL,
-			    xxREAD_PROTECTION_FAULT_ENABLE_DEFAULT, value);
-	tmp = REG_SET_FIELD(tmp, mmVM_CONTEXT1_CNTL,
-			    xxWRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
-	WREG32(VM_CONTEXT1_CNTL, tmp);
+	tmp = RREG32(mmVM_CONTEXT1_CNTL);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+			    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+			    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+			    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+			    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+			    READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+			    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	WREG32(mmVM_CONTEXT1_CNTL, tmp);
 }
 
 static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
@@ -383,33 +401,39 @@ static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
 	if (r)
 		return r;
 	/* Setup TLB control */
-	WREG32(MC_VM_MX_L1_TLB_CNTL,
+	WREG32(mmMC_VM_MX_L1_TLB_CNTL,
 	       (0xA << 7) |
-	       ENABLE_L1_TLB |
-	       ENABLE_L1_FRAGMENT_PROCESSING |
-	       SYSTEM_ACCESS_MODE_NOT_IN_SYS |
-	       ENABLE_ADVANCED_DRIVER_MODEL |
-	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
+	       MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK |
+	       MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_FRAGMENT_PROCESSING_MASK |
+	       MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK |
+	       MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK |
+	       (0UL << MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT));
 	/* Setup L2 cache */
-	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
-	       ENABLE_L2_FRAGMENT_PROCESSING |
-	       ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
-	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
-	       EFFECTIVE_L2_QUEUE_SIZE(7) |
-	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
-	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
-	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
-	       BANK_SELECT(4) |
-	       L2_CACHE_BIGK_FRAGMENT_SIZE(4));
+	WREG32(mmVM_L2_CNTL,
+	       VM_L2_CNTL__ENABLE_L2_CACHE_MASK |
+	       VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING_MASK |
+	       VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK |
+	       VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK |
+	       (7UL << VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT) |
+	       (1UL << VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT));
+	WREG32(mmVM_L2_CNTL2,
+	       VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK |
+	       VM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK);
+	WREG32(mmVM_L2_CNTL3,
+	       VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK |
+	       (4UL << VM_L2_CNTL3__BANK_SELECT__SHIFT) |
+	       (4UL << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
 	/* setup context0 */
-	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gtt_start >> 12);
-	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gtt_end >> 12);
-	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
-	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
+	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gtt_start >> 12);
+	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gtt_end >> 12);
+	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
+	WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 			(u32)(adev->dummy_page.addr >> 12));
-	WREG32(VM_CONTEXT0_CNTL2, 0);
-	WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
-				  RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
+	WREG32(mmVM_CONTEXT0_CNTL2, 0);
+	WREG32(mmVM_CONTEXT0_CNTL,
+	       VM_CONTEXT0_CNTL__ENABLE_CONTEXT_MASK |
+	       (0UL << VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH__SHIFT) |
+	       VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK);
 
 	WREG32(0x575, 0);
 	WREG32(0x576, 0);
@@ -417,39 +441,41 @@ static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
 
 	/* empty context1-15 */
 	/* set vm size, must be a multiple of 4 */
-	WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
-	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
+	WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
+	WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
 	/* Assign the pt base to something valid for now; the pts used for
 	 * the VMs are determined by the application and setup and assigned
 	 * on the fly in the vm part of radeon_gart.c
 	 */
 	for (i = 1; i < 16; i++) {
 		if (i < 8)
-			WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
+			WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
 			       adev->gart.table_addr >> 12);
 		else
-			WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
+			WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
 			       adev->gart.table_addr >> 12);
 	}
 
 	/* enable context1-15 */
-	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
+	WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
 	       (u32)(adev->dummy_page.addr >> 12));
-	WREG32(VM_CONTEXT1_CNTL2, 4);
-	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
-				PAGE_TABLE_BLOCK_SIZE(amdgpu_vm_block_size - 9) |
-				RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
-				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
-				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
-				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
-				PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
-				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
-				VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
-				VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
-				READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
-				READ_PROTECTION_FAULT_ENABLE_DEFAULT |
-				WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
-				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
+	WREG32(mmVM_CONTEXT1_CNTL2, 4);
+	WREG32(mmVM_CONTEXT1_CNTL,
+	       VM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK |
+	       (1UL << VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT) |
+	       ((amdgpu_vm_block_size - 9) << VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT) |
+	       VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+	       VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK |
+	       VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+	       VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK |
+	       VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+	       VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK |
+	       VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+	       VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK |
+	       VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+	       VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK |
+	       VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+	       VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK);
 
 	gmc_v6_0_gart_flush_gpu_tlb(adev, 0);
 	dev_info(adev->dev, "PCIE GART of %uM enabled (table at 0x%016llX).\n",
@@ -488,19 +514,22 @@ static void gmc_v6_0_gart_disable(struct amdgpu_device *adev)
 	}*/
 
 	/* Disable all tables */
-	WREG32(VM_CONTEXT0_CNTL, 0);
-	WREG32(VM_CONTEXT1_CNTL, 0);
+	WREG32(mmVM_CONTEXT0_CNTL, 0);
+	WREG32(mmVM_CONTEXT1_CNTL, 0);
 	/* Setup TLB control */
-	WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
-	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
+	WREG32(mmMC_VM_MX_L1_TLB_CNTL,
+	       MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK |
+	       (0UL << MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT));
 	/* Setup L2 cache */
-	WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
-	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
-	       EFFECTIVE_L2_QUEUE_SIZE(7) |
-	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
-	WREG32(VM_L2_CNTL2, 0);
-	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
-	       L2_CACHE_BIGK_FRAGMENT_SIZE(0));
+	WREG32(mmVM_L2_CNTL,
+	       VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK |
+	       VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK |
+	       (7UL << VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT) |
+	       (1UL << VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT));
+	WREG32(mmVM_L2_CNTL2, 0);
+	WREG32(mmVM_L2_CNTL3,
+	       VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK |
+	       (0UL << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
 	amdgpu_gart_table_vram_unpin(adev);
 }
 
@@ -523,7 +552,7 @@ static int gmc_v6_0_vm_init(struct amdgpu_device *adev)
 
 	/* base offset of vram pages */
 	if (adev->flags & AMD_IS_APU) {
-		u64 tmp = RREG32(MC_VM_FB_OFFSET);
+		u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
 		tmp <<= 22;
 		adev->vm_manager.vram_base_offset = tmp;
 	} else
@@ -540,19 +569,19 @@ static void gmc_v6_0_vm_decode_fault(struct amdgpu_device *adev,
 				     u32 status, u32 addr, u32 mc_client)
 {
 	u32 mc_id;
-	u32 vmid = REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS, xxVMID);
-	u32 protections = REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS,
-					xxPROTECTIONS);
+	u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
+	u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+					PROTECTIONS);
 	char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
 		(mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
 
-	mc_id = REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS,
-			      xxMEMORY_CLIENT_ID);
+	mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+			      MEMORY_CLIENT_ID);
 
 	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
 	       protections, vmid, addr,
-	       REG_GET_FIELD(status, mmVM_CONTEXT1_PROTECTION_FAULT_STATUS,
-			     xxMEMORY_CLIENT_RW) ?
+	       REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+			     MEMORY_CLIENT_RW) ?
 	       "write" : "read", block, mc_client, mc_id);
 }
 
@@ -655,7 +684,7 @@ static void gmc_v6_0_enable_hdp_mgcg(struct amdgpu_device *adev,
 {
 	u32 orig, data;
 
-	orig = data = RREG32(HDP_HOST_PATH_CNTL);
+	orig = data = RREG32(mmHDP_HOST_PATH_CNTL);
 
 	if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_HDP_MGCG))
 		data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 0);
@@ -663,7 +692,7 @@ static void gmc_v6_0_enable_hdp_mgcg(struct amdgpu_device *adev,
 		data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 1);
 
 	if (orig != data)
-		WREG32(HDP_HOST_PATH_CNTL, data);
+		WREG32(mmHDP_HOST_PATH_CNTL, data);
 }
 
 static void gmc_v6_0_enable_hdp_ls(struct amdgpu_device *adev,
@@ -671,7 +700,7 @@ static void gmc_v6_0_enable_hdp_ls(struct amdgpu_device *adev,
 {
 	u32 orig, data;
 
-	orig = data = RREG32(HDP_MEM_POWER_LS);
+	orig = data = RREG32(mmHDP_MEM_POWER_LS);
 
 	if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_HDP_LS))
 		data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 1);
@@ -679,7 +708,7 @@ static void gmc_v6_0_enable_hdp_ls(struct amdgpu_device *adev,
 		data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 0);
 
 	if (orig != data)
-		WREG32(HDP_MEM_POWER_LS, data);
+		WREG32(mmHDP_MEM_POWER_LS, data);
 }
 */
 
@@ -713,7 +742,7 @@ static int gmc_v6_0_early_init(void *handle)
 	if (adev->flags & AMD_IS_APU) {
 		adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
 	} else {
-		u32 tmp = RREG32(MC_SEQ_MISC0);
+		u32 tmp = RREG32(mmMC_SEQ_MISC0);
 		tmp &= MC_SEQ_MISC0__MT__MASK;
 		adev->mc.vram_type = gmc_v6_0_convert_vram_type(tmp);
 	}
@@ -766,11 +795,6 @@ static int gmc_v6_0_sw_init(void *handle)
 		return r;
 	}
 
-	r = amdgpu_ttm_global_init(adev);
-	if (r) {
-		return r;
-	}
-
 	r = gmc_v6_0_mc_init(adev);
 	if (r)
 		return r;
@@ -879,7 +903,7 @@ static int gmc_v6_0_resume(void *handle)
 static bool gmc_v6_0_is_idle(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	u32 tmp = RREG32(SRBM_STATUS);
+	u32 tmp = RREG32(mmSRBM_STATUS);
 
 	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
 		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
@@ -895,7 +919,7 @@ static int gmc_v6_0_wait_for_idle(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	for (i = 0; i < adev->usec_timeout; i++) {
-		tmp = RREG32(SRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
+		tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
 					       SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
 					       SRBM_STATUS__MCC_BUSY_MASK |
 					       SRBM_STATUS__MCD_BUSY_MASK |
@@ -913,17 +937,17 @@ static int gmc_v6_0_soft_reset(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	struct amdgpu_mode_mc_save save;
 	u32 srbm_soft_reset = 0;
-	u32 tmp = RREG32(SRBM_STATUS);
+	u32 tmp = RREG32(mmSRBM_STATUS);
 
 	if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
-						mmSRBM_SOFT_RESET, xxSOFT_RESET_VMC, 1);
+						SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
 
 	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
 		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
 		if (!(adev->flags & AMD_IS_APU))
 			srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
-							mmSRBM_SOFT_RESET, xxSOFT_RESET_MC, 1);
+							SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
 	}
 
 	if (srbm_soft_reset) {
@@ -933,17 +957,17 @@ static int gmc_v6_0_soft_reset(void *handle)
 		}
 
 
-		tmp = RREG32(SRBM_SOFT_RESET);
+		tmp = RREG32(mmSRBM_SOFT_RESET);
 		tmp |= srbm_soft_reset;
 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
-		WREG32(SRBM_SOFT_RESET, tmp);
-		tmp = RREG32(SRBM_SOFT_RESET);
+		WREG32(mmSRBM_SOFT_RESET, tmp);
+		tmp = RREG32(mmSRBM_SOFT_RESET);
 
 		udelay(50);
 
 		tmp &= ~srbm_soft_reset;
-		WREG32(SRBM_SOFT_RESET, tmp);
-		tmp = RREG32(SRBM_SOFT_RESET);
+		WREG32(mmSRBM_SOFT_RESET, tmp);
+		tmp = RREG32(mmSRBM_SOFT_RESET);
 
 		udelay(50);
 
@@ -969,20 +993,20 @@ static int gmc_v6_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
 
 	switch (state) {
 	case AMDGPU_IRQ_STATE_DISABLE:
-		tmp = RREG32(VM_CONTEXT0_CNTL);
+		tmp = RREG32(mmVM_CONTEXT0_CNTL);
 		tmp &= ~bits;
-		WREG32(VM_CONTEXT0_CNTL, tmp);
-		tmp = RREG32(VM_CONTEXT1_CNTL);
+		WREG32(mmVM_CONTEXT0_CNTL, tmp);
+		tmp = RREG32(mmVM_CONTEXT1_CNTL);
 		tmp &= ~bits;
-		WREG32(VM_CONTEXT1_CNTL, tmp);
+		WREG32(mmVM_CONTEXT1_CNTL, tmp);
 		break;
 	case AMDGPU_IRQ_STATE_ENABLE:
-		tmp = RREG32(VM_CONTEXT0_CNTL);
+		tmp = RREG32(mmVM_CONTEXT0_CNTL);
 		tmp |= bits;
-		WREG32(VM_CONTEXT0_CNTL, tmp);
-		tmp = RREG32(VM_CONTEXT1_CNTL);
+		WREG32(mmVM_CONTEXT0_CNTL, tmp);
+		tmp = RREG32(mmVM_CONTEXT1_CNTL);
 		tmp |= bits;
-		WREG32(VM_CONTEXT1_CNTL, tmp);
+		WREG32(mmVM_CONTEXT1_CNTL, tmp);
 		break;
 	default:
 		break;
@@ -997,9 +1021,9 @@ static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev,
 {
 	u32 addr, status;
 
-	addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
-	status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
-	WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
+	addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
+	status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
+	WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
 
 	if (!addr && !status)
 		return 0;
@@ -1007,13 +1031,15 @@ static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev,
 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
 		gmc_v6_0_set_fault_enable_default(adev, false);
 
-	dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
-		entry->src_id, entry->src_data);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
-		addr);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
-		status);
-	gmc_v6_0_vm_decode_fault(adev, status, addr, 0);
+	if (printk_ratelimit()) {
+		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
+			entry->src_id, entry->src_data);
+		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
+			addr);
+		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
+			status);
+		gmc_v6_0_vm_decode_fault(adev, status, addr, 0);
+	}
 
 	return 0;
 }
@@ -1030,7 +1056,7 @@ static int gmc_v6_0_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs gmc_v6_0_ip_funcs = {
+static const struct amd_ip_funcs gmc_v6_0_ip_funcs = {
 	.name = "gmc_v6_0",
 	.early_init = gmc_v6_0_early_init,
 	.late_init = gmc_v6_0_late_init,
@@ -1069,3 +1095,11 @@ static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev)
 	adev->mc.vm_fault.funcs = &gmc_v6_0_irq_funcs;
 }
 
+const struct amdgpu_ip_block_version gmc_v6_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GMC,
+	.major = 6,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &gmc_v6_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.h b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.h
index 42c4fc676cd4..ed2f64dec47a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.h
@@ -24,6 +24,6 @@
 #ifndef __GMC_V6_0_H__
 #define __GMC_V6_0_H__
 
-extern const struct amd_ip_funcs gmc_v6_0_ip_funcs;
+extern const struct amdgpu_ip_block_version gmc_v6_0_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index aa0c4b964621..273b16fb9459 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -385,7 +385,7 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
 	 * size equal to the 1024 or vram, whichever is larger.
 	 */
 	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = amdgpu_ttm_get_gtt_mem_size(adev);
+		adev->mc.gtt_size = max((1024ULL << 20), adev->mc.mc_vram_size);
 	else
 		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
 
@@ -711,7 +711,7 @@ static void gmc_v7_0_vm_decode_fault(struct amdgpu_device *adev,
 	mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			      MEMORY_CLIENT_ID);
 
-	printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
+	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
 	       protections, vmid, addr,
 	       REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			     MEMORY_CLIENT_RW) ?
@@ -945,11 +945,6 @@ static int gmc_v7_0_sw_init(void *handle)
 		return r;
 	}
 
-	r = amdgpu_ttm_global_init(adev);
-	if (r) {
-		return r;
-	}
-
 	r = gmc_v7_0_mc_init(adev);
 	if (r)
 		return r;
@@ -1198,13 +1193,15 @@ static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev,
 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
 		gmc_v7_0_set_fault_enable_default(adev, false);
 
-	dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
-		entry->src_id, entry->src_data);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
-		addr);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
-		status);
-	gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client);
+	if (printk_ratelimit()) {
+		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
+			entry->src_id, entry->src_data);
+		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
+			addr);
+		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
+			status);
+		gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client);
+	}
 
 	return 0;
 }
@@ -1235,7 +1232,7 @@ static int gmc_v7_0_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs gmc_v7_0_ip_funcs = {
+static const struct amd_ip_funcs gmc_v7_0_ip_funcs = {
 	.name = "gmc_v7_0",
 	.early_init = gmc_v7_0_early_init,
 	.late_init = gmc_v7_0_late_init,
@@ -1273,3 +1270,21 @@ static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev)
 	adev->mc.vm_fault.num_types = 1;
 	adev->mc.vm_fault.funcs = &gmc_v7_0_irq_funcs;
 }
+
+const struct amdgpu_ip_block_version gmc_v7_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GMC,
+	.major = 7,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &gmc_v7_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version gmc_v7_4_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GMC,
+	.major = 7,
+	.minor = 4,
+	.rev = 0,
+	.funcs = &gmc_v7_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h
index 0b386b5d2f7a..ebce2966c1c7 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h
@@ -24,6 +24,7 @@
 #ifndef __GMC_V7_0_H__
 #define __GMC_V7_0_H__
 
-extern const struct amd_ip_funcs gmc_v7_0_ip_funcs;
+extern const struct amdgpu_ip_block_version gmc_v7_0_ip_block;
+extern const struct amdgpu_ip_block_version gmc_v7_4_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index a16b2201d52c..0daac3a5be79 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -472,7 +472,7 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
 	 * size equal to the 1024 or vram, whichever is larger.
 	 */
 	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = amdgpu_ttm_get_gtt_mem_size(adev);
+		adev->mc.gtt_size = max((1024ULL << 20), adev->mc.mc_vram_size);
 	else
 		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
 
@@ -837,7 +837,7 @@ static void gmc_v8_0_vm_decode_fault(struct amdgpu_device *adev,
 	mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			      MEMORY_CLIENT_ID);
 
-	printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
+	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
 	       protections, vmid, addr,
 	       REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
 			     MEMORY_CLIENT_RW) ?
@@ -952,11 +952,6 @@ static int gmc_v8_0_sw_init(void *handle)
 		return r;
 	}
 
-	r = amdgpu_ttm_global_init(adev);
-	if (r) {
-		return r;
-	}
-
 	r = gmc_v8_0_mc_init(adev);
 	if (r)
 		return r;
@@ -1242,13 +1237,15 @@ static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev,
 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
 		gmc_v8_0_set_fault_enable_default(adev, false);
 
-	dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
-		entry->src_id, entry->src_data);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
-		addr);
-	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
-		status);
-	gmc_v8_0_vm_decode_fault(adev, status, addr, mc_client);
+	if (printk_ratelimit()) {
+		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
+			entry->src_id, entry->src_data);
+		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
+			addr);
+		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
+			status);
+		gmc_v8_0_vm_decode_fault(adev, status, addr, mc_client);
+	}
 
 	return 0;
 }
@@ -1437,7 +1434,7 @@ static int gmc_v8_0_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
+static const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
 	.name = "gmc_v8_0",
 	.early_init = gmc_v8_0_early_init,
 	.late_init = gmc_v8_0_late_init,
@@ -1478,3 +1475,30 @@ static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev)
 	adev->mc.vm_fault.num_types = 1;
 	adev->mc.vm_fault.funcs = &gmc_v8_0_irq_funcs;
 }
+
+const struct amdgpu_ip_block_version gmc_v8_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GMC,
+	.major = 8,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &gmc_v8_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version gmc_v8_1_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GMC,
+	.major = 8,
+	.minor = 1,
+	.rev = 0,
+	.funcs = &gmc_v8_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version gmc_v8_5_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_GMC,
+	.major = 8,
+	.minor = 5,
+	.rev = 0,
+	.funcs = &gmc_v8_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h
index fc5001a8119d..19b8a8aed204 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h
@@ -24,6 +24,8 @@
 #ifndef __GMC_V8_0_H__
 #define __GMC_V8_0_H__
 
-extern const struct amd_ip_funcs gmc_v8_0_ip_funcs;
+extern const struct amdgpu_ip_block_version gmc_v8_0_ip_block;
+extern const struct amdgpu_ip_block_version gmc_v8_1_ip_block;
+extern const struct amdgpu_ip_block_version gmc_v8_5_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
index 3b8906ce3511..ac21bb7bc0f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
@@ -392,7 +392,7 @@ static int iceland_ih_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs iceland_ih_ip_funcs = {
+static const struct amd_ip_funcs iceland_ih_ip_funcs = {
 	.name = "iceland_ih",
 	.early_init = iceland_ih_early_init,
 	.late_init = NULL,
@@ -421,3 +421,11 @@ static void iceland_ih_set_interrupt_funcs(struct amdgpu_device *adev)
 		adev->irq.ih_funcs = &iceland_ih_funcs;
 }
 
+const struct amdgpu_ip_block_version iceland_ih_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_IH,
+	.major = 2,
+	.minor = 4,
+	.rev = 0,
+	.funcs = &iceland_ih_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.h b/drivers/gpu/drm/amd/amdgpu/iceland_ih.h
index 57558cddfbcb..3235f4277548 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.h
@@ -24,6 +24,6 @@
 #ifndef __ICELAND_IH_H__
 #define __ICELAND_IH_H__
 
-extern const struct amd_ip_funcs iceland_ih_ip_funcs;
+extern const struct amdgpu_ip_block_version iceland_ih_ip_block;
 
 #endif /* __ICELAND_IH_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index 71d2856222fa..5a1bc358bcb1 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -2796,7 +2796,7 @@ static int kv_parse_power_table(struct amdgpu_device *adev)
 	adev->pm.dpm.num_ps = state_array->ucNumEntries;
 
 	/* fill in the vce power states */
-	for (i = 0; i < AMDGPU_MAX_VCE_LEVELS; i++) {
+	for (i = 0; i < adev->pm.dpm.num_of_vce_states; i++) {
 		u32 sclk;
 		clock_array_index = adev->pm.dpm.vce_states[i].clk_idx;
 		clock_info = (union pplib_clock_info *)
@@ -2845,7 +2845,7 @@ static int kv_dpm_init(struct amdgpu_device *adev)
 		pi->caps_tcp_ramping = true;
 	}
 
-	if (amdgpu_sclk_deep_sleep_en)
+	if (amdgpu_pp_feature_mask & SCLK_DEEP_SLEEP_MASK)
 		pi->caps_sclk_ds = true;
 	else
 		pi->caps_sclk_ds = false;
@@ -3245,6 +3245,18 @@ static int kv_dpm_set_powergating_state(void *handle,
 	return 0;
 }
 
+static int kv_check_state_equal(struct amdgpu_device *adev,
+				struct amdgpu_ps *cps,
+				struct amdgpu_ps *rps,
+				bool *equal)
+{
+	if (equal == NULL)
+		return -EINVAL;
+
+	*equal = false;
+	return 0;
+}
+
 const struct amd_ip_funcs kv_dpm_ip_funcs = {
 	.name = "kv_dpm",
 	.early_init = kv_dpm_early_init,
@@ -3275,6 +3287,8 @@ static const struct amdgpu_dpm_funcs kv_dpm_funcs = {
 	.force_performance_level = &kv_dpm_force_performance_level,
 	.powergate_uvd = &kv_dpm_powergate_uvd,
 	.enable_bapm = &kv_dpm_enable_bapm,
+	.get_vce_clock_state = amdgpu_get_vce_clock_state,
+	.check_state_equal = kv_check_state_equal,
 };
 
 static void kv_dpm_set_dpm_funcs(struct amdgpu_device *adev)
@@ -3293,3 +3307,12 @@ static void kv_dpm_set_irq_funcs(struct amdgpu_device *adev)
 	adev->pm.dpm.thermal.irq.num_types = AMDGPU_THERMAL_IRQ_LAST;
 	adev->pm.dpm.thermal.irq.funcs = &kv_dpm_irq_funcs;
 }
+
+const struct amdgpu_ip_block_version kv_dpm_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_SMC,
+	.major = 7,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &kv_dpm_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index 565dab3c7218..fbe74a33899c 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -232,10 +232,10 @@ static void sdma_v2_4_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 
 	for (i = 0; i < count; i++)
 		if (sdma && sdma->burst_nop && (i == 0))
-			amdgpu_ring_write(ring, ring->nop |
+			amdgpu_ring_write(ring, ring->funcs->nop |
 				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
 		else
-			amdgpu_ring_write(ring, ring->nop);
+			amdgpu_ring_write(ring, ring->funcs->nop);
 }
 
 /**
@@ -668,7 +668,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_ib ib;
-	struct fence *f = NULL;
+	struct dma_fence *f = NULL;
 	unsigned index;
 	u32 tmp = 0;
 	u64 gpu_addr;
@@ -705,7 +705,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	if (r)
 		goto err1;
 
-	r = fence_wait_timeout(f, false, timeout);
+	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
 		DRM_ERROR("amdgpu: IB test timed out\n");
 		r = -ETIMEDOUT;
@@ -725,7 +725,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 err1:
 	amdgpu_ib_free(adev, &ib, NULL);
-	fence_put(f);
+	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
 	return r;
@@ -775,11 +775,11 @@ static void sdma_v2_4_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
 	unsigned ndw = count * 2;
 
 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
-		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
+		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
 	ib->ptr[ib->length_dw++] = pe;
 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
 	ib->ptr[ib->length_dw++] = ndw;
-	for (; ndw > 0; ndw -= 2, --count, pe += 8) {
+	for (; ndw > 0; ndw -= 2) {
 		ib->ptr[ib->length_dw++] = lower_32_bits(value);
 		ib->ptr[ib->length_dw++] = upper_32_bits(value);
 		value += incr;
@@ -902,22 +902,6 @@ static void sdma_v2_4_ring_emit_vm_flush(struct amdgpu_ring *ring,
 			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
 }
 
-static unsigned sdma_v2_4_ring_get_emit_ib_size(struct amdgpu_ring *ring)
-{
-	return
-		7 + 6; /* sdma_v2_4_ring_emit_ib */
-}
-
-static unsigned sdma_v2_4_ring_get_dma_frame_size(struct amdgpu_ring *ring)
-{
-	return
-		6 + /* sdma_v2_4_ring_emit_hdp_flush */
-		3 + /* sdma_v2_4_ring_emit_hdp_invalidate */
-		6 + /* sdma_v2_4_ring_emit_pipeline_sync */
-		12 + /* sdma_v2_4_ring_emit_vm_flush */
-		10 + 10 + 10; /* sdma_v2_4_ring_emit_fence x3 for user fence, vm fence */
-}
-
 static int sdma_v2_4_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -965,11 +949,10 @@ static int sdma_v2_4_sw_init(void *handle)
 		ring->use_doorbell = false;
 		sprintf(ring->name, "sdma%d", i);
 		r = amdgpu_ring_init(adev, ring, 1024,
-				     SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), 0xf,
 				     &adev->sdma.trap_irq,
 				     (i == 0) ?
-				     AMDGPU_SDMA_IRQ_TRAP0 : AMDGPU_SDMA_IRQ_TRAP1,
-				     AMDGPU_RING_TYPE_SDMA);
+				     AMDGPU_SDMA_IRQ_TRAP0 :
+				     AMDGPU_SDMA_IRQ_TRAP1);
 		if (r)
 			return r;
 	}
@@ -1204,7 +1187,7 @@ static int sdma_v2_4_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs sdma_v2_4_ip_funcs = {
+static const struct amd_ip_funcs sdma_v2_4_ip_funcs = {
 	.name = "sdma_v2_4",
 	.early_init = sdma_v2_4_early_init,
 	.late_init = NULL,
@@ -1222,10 +1205,19 @@ const struct amd_ip_funcs sdma_v2_4_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs sdma_v2_4_ring_funcs = {
+	.type = AMDGPU_RING_TYPE_SDMA,
+	.align_mask = 0xf,
+	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
 	.get_rptr = sdma_v2_4_ring_get_rptr,
 	.get_wptr = sdma_v2_4_ring_get_wptr,
 	.set_wptr = sdma_v2_4_ring_set_wptr,
-	.parse_cs = NULL,
+	.emit_frame_size =
+		6 + /* sdma_v2_4_ring_emit_hdp_flush */
+		3 + /* sdma_v2_4_ring_emit_hdp_invalidate */
+		6 + /* sdma_v2_4_ring_emit_pipeline_sync */
+		12 + /* sdma_v2_4_ring_emit_vm_flush */
+		10 + 10 + 10, /* sdma_v2_4_ring_emit_fence x3 for user fence, vm fence */
+	.emit_ib_size = 7 + 6, /* sdma_v2_4_ring_emit_ib */
 	.emit_ib = sdma_v2_4_ring_emit_ib,
 	.emit_fence = sdma_v2_4_ring_emit_fence,
 	.emit_pipeline_sync = sdma_v2_4_ring_emit_pipeline_sync,
@@ -1236,8 +1228,6 @@ static const struct amdgpu_ring_funcs sdma_v2_4_ring_funcs = {
 	.test_ib = sdma_v2_4_ring_test_ib,
 	.insert_nop = sdma_v2_4_ring_insert_nop,
 	.pad_ib = sdma_v2_4_ring_pad_ib,
-	.get_emit_ib_size = sdma_v2_4_ring_get_emit_ib_size,
-	.get_dma_frame_size = sdma_v2_4_ring_get_dma_frame_size,
 };
 
 static void sdma_v2_4_set_ring_funcs(struct amdgpu_device *adev)
@@ -1350,3 +1340,12 @@ static void sdma_v2_4_set_vm_pte_funcs(struct amdgpu_device *adev)
 		adev->vm_manager.vm_pte_num_rings = adev->sdma.num_instances;
 	}
 }
+
+const struct amdgpu_ip_block_version sdma_v2_4_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_SDMA,
+	.major = 2,
+	.minor = 4,
+	.rev = 0,
+	.funcs = &sdma_v2_4_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.h b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.h
index 07349f5ee10f..28b433729216 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.h
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.h
@@ -24,6 +24,6 @@
 #ifndef __SDMA_V2_4_H__
 #define __SDMA_V2_4_H__
 
-extern const struct amd_ip_funcs sdma_v2_4_ip_funcs;
+extern const struct amdgpu_ip_block_version sdma_v2_4_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index a9d10941fb53..1170a64a3184 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -392,10 +392,10 @@ static void sdma_v3_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 
 	for (i = 0; i < count; i++)
 		if (sdma && sdma->burst_nop && (i == 0))
-			amdgpu_ring_write(ring, ring->nop |
+			amdgpu_ring_write(ring, ring->funcs->nop |
 				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
 		else
-			amdgpu_ring_write(ring, ring->nop);
+			amdgpu_ring_write(ring, ring->funcs->nop);
 }
 
 /**
@@ -871,7 +871,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_ib ib;
-	struct fence *f = NULL;
+	struct dma_fence *f = NULL;
 	unsigned index;
 	u32 tmp = 0;
 	u64 gpu_addr;
@@ -908,7 +908,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	if (r)
 		goto err1;
 
-	r = fence_wait_timeout(f, false, timeout);
+	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
 		DRM_ERROR("amdgpu: IB test timed out\n");
 		r = -ETIMEDOUT;
@@ -927,7 +927,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	}
 err1:
 	amdgpu_ib_free(adev, &ib, NULL);
-	fence_put(f);
+	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
 	return r;
@@ -977,11 +977,11 @@ static void sdma_v3_0_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
 	unsigned ndw = count * 2;
 
 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
-		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
+		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
 	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
 	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
 	ib->ptr[ib->length_dw++] = ndw;
-	for (; ndw > 0; ndw -= 2, --count, pe += 8) {
+	for (; ndw > 0; ndw -= 2) {
 		ib->ptr[ib->length_dw++] = lower_32_bits(value);
 		ib->ptr[ib->length_dw++] = upper_32_bits(value);
 		value += incr;
@@ -1104,22 +1104,6 @@ static void sdma_v3_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10)); /* retry count, poll interval */
 }
 
-static unsigned sdma_v3_0_ring_get_emit_ib_size(struct amdgpu_ring *ring)
-{
-	return
-		7 + 6; /* sdma_v3_0_ring_emit_ib */
-}
-
-static unsigned sdma_v3_0_ring_get_dma_frame_size(struct amdgpu_ring *ring)
-{
-	return
-		6 + /* sdma_v3_0_ring_emit_hdp_flush */
-		3 + /* sdma_v3_0_ring_emit_hdp_invalidate */
-		6 + /* sdma_v3_0_ring_emit_pipeline_sync */
-		12 + /* sdma_v3_0_ring_emit_vm_flush */
-		10 + 10 + 10; /* sdma_v3_0_ring_emit_fence x3 for user fence, vm fence */
-}
-
 static int sdma_v3_0_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -1177,11 +1161,10 @@ static int sdma_v3_0_sw_init(void *handle)
 
 		sprintf(ring->name, "sdma%d", i);
 		r = amdgpu_ring_init(adev, ring, 1024,
-				     SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), 0xf,
 				     &adev->sdma.trap_irq,
 				     (i == 0) ?
-				     AMDGPU_SDMA_IRQ_TRAP0 : AMDGPU_SDMA_IRQ_TRAP1,
-				     AMDGPU_RING_TYPE_SDMA);
+				     AMDGPU_SDMA_IRQ_TRAP0 :
+				     AMDGPU_SDMA_IRQ_TRAP1);
 		if (r)
 			return r;
 	}
@@ -1544,7 +1527,7 @@ static int sdma_v3_0_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs sdma_v3_0_ip_funcs = {
+static const struct amd_ip_funcs sdma_v3_0_ip_funcs = {
 	.name = "sdma_v3_0",
 	.early_init = sdma_v3_0_early_init,
 	.late_init = NULL,
@@ -1565,10 +1548,19 @@ const struct amd_ip_funcs sdma_v3_0_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs sdma_v3_0_ring_funcs = {
+	.type = AMDGPU_RING_TYPE_SDMA,
+	.align_mask = 0xf,
+	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
 	.get_rptr = sdma_v3_0_ring_get_rptr,
 	.get_wptr = sdma_v3_0_ring_get_wptr,
 	.set_wptr = sdma_v3_0_ring_set_wptr,
-	.parse_cs = NULL,
+	.emit_frame_size =
+		6 + /* sdma_v3_0_ring_emit_hdp_flush */
+		3 + /* sdma_v3_0_ring_emit_hdp_invalidate */
+		6 + /* sdma_v3_0_ring_emit_pipeline_sync */
+		12 + /* sdma_v3_0_ring_emit_vm_flush */
+		10 + 10 + 10, /* sdma_v3_0_ring_emit_fence x3 for user fence, vm fence */
+	.emit_ib_size = 7 + 6, /* sdma_v3_0_ring_emit_ib */
 	.emit_ib = sdma_v3_0_ring_emit_ib,
 	.emit_fence = sdma_v3_0_ring_emit_fence,
 	.emit_pipeline_sync = sdma_v3_0_ring_emit_pipeline_sync,
@@ -1579,8 +1571,6 @@ static const struct amdgpu_ring_funcs sdma_v3_0_ring_funcs = {
 	.test_ib = sdma_v3_0_ring_test_ib,
 	.insert_nop = sdma_v3_0_ring_insert_nop,
 	.pad_ib = sdma_v3_0_ring_pad_ib,
-	.get_emit_ib_size = sdma_v3_0_ring_get_emit_ib_size,
-	.get_dma_frame_size = sdma_v3_0_ring_get_dma_frame_size,
 };
 
 static void sdma_v3_0_set_ring_funcs(struct amdgpu_device *adev)
@@ -1693,3 +1683,21 @@ static void sdma_v3_0_set_vm_pte_funcs(struct amdgpu_device *adev)
 		adev->vm_manager.vm_pte_num_rings = adev->sdma.num_instances;
 	}
 }
+
+const struct amdgpu_ip_block_version sdma_v3_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_SDMA,
+	.major = 3,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &sdma_v3_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version sdma_v3_1_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_SDMA,
+	.major = 3,
+	.minor = 1,
+	.rev = 0,
+	.funcs = &sdma_v3_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.h b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.h
index 0cb9698a3054..7aa223d35f1b 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.h
@@ -24,6 +24,7 @@
 #ifndef __SDMA_V3_0_H__
 #define __SDMA_V3_0_H__
 
-extern const struct amd_ip_funcs sdma_v3_0_ip_funcs;
+extern const struct amdgpu_ip_block_version sdma_v3_0_ip_block;
+extern const struct amdgpu_ip_block_version sdma_v3_1_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index dc9511c5ecb8..3ed8ad8725b9 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -39,6 +39,7 @@
 #include "si_dma.h"
 #include "dce_v6_0.h"
 #include "si.h"
+#include "dce_virtual.h"
 
 static const u32 tahiti_golden_registers[] =
 {
@@ -905,7 +906,7 @@ static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 }
 
-u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
+static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
 {
 	unsigned long flags;
 	u32 r;
@@ -918,7 +919,7 @@ u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
 	return r;
 }
 
-void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
+static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags;
 
@@ -1811,7 +1812,7 @@ static int si_common_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs si_common_ip_funcs = {
+static const struct amd_ip_funcs si_common_ip_funcs = {
 	.name = "si_common",
 	.early_init = si_common_early_init,
 	.late_init = NULL,
@@ -1828,119 +1829,13 @@ const struct amd_ip_funcs si_common_ip_funcs = {
 	.set_powergating_state = si_common_set_powergating_state,
 };
 
-static const struct amdgpu_ip_block_version verde_ip_blocks[] =
+static const struct amdgpu_ip_block_version si_common_ip_block =
 {
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &si_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 6,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v6_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &si_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 6,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 6,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &dce_v6_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 6,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gfx_v6_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &si_dma_ip_funcs,
-	},
-/*	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 3,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &si_null_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &si_null_ip_funcs,
-	},
-	*/
-};
-
-
-static const struct amdgpu_ip_block_version hainan_ip_blocks[] =
-{
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &si_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 6,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v6_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &si_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 6,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 6,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gfx_v6_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &si_dma_ip_funcs,
-	},
+	.type = AMD_IP_BLOCK_TYPE_COMMON,
+	.major = 1,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &si_common_ip_funcs,
 };
 
 int si_set_ip_blocks(struct amdgpu_device *adev)
@@ -1949,13 +1844,42 @@ int si_set_ip_blocks(struct amdgpu_device *adev)
 	case CHIP_VERDE:
 	case CHIP_TAHITI:
 	case CHIP_PITCAIRN:
+		amdgpu_ip_block_add(adev, &si_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v6_0_ip_block);
+		amdgpu_ip_block_add(adev, &si_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		else
+			amdgpu_ip_block_add(adev, &dce_v6_0_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v6_0_ip_block);
+		amdgpu_ip_block_add(adev, &si_dma_ip_block);
+		/* amdgpu_ip_block_add(adev, &uvd_v3_1_ip_block); */
+		/* amdgpu_ip_block_add(adev, &vce_v1_0_ip_block); */
+		break;
 	case CHIP_OLAND:
-		adev->ip_blocks = verde_ip_blocks;
-		adev->num_ip_blocks = ARRAY_SIZE(verde_ip_blocks);
+		amdgpu_ip_block_add(adev, &si_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v6_0_ip_block);
+		amdgpu_ip_block_add(adev, &si_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		else
+			amdgpu_ip_block_add(adev, &dce_v6_4_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v6_0_ip_block);
+		amdgpu_ip_block_add(adev, &si_dma_ip_block);
+		/* amdgpu_ip_block_add(adev, &uvd_v3_1_ip_block); */
+		/* amdgpu_ip_block_add(adev, &vce_v1_0_ip_block); */
 		break;
 	case CHIP_HAINAN:
-		adev->ip_blocks = hainan_ip_blocks;
-		adev->num_ip_blocks = ARRAY_SIZE(hainan_ip_blocks);
+		amdgpu_ip_block_add(adev, &si_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v6_0_ip_block);
+		amdgpu_ip_block_add(adev, &si_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v6_0_ip_block);
+		amdgpu_ip_block_add(adev, &si_dma_ip_block);
 		break;
 	default:
 		BUG();
diff --git a/drivers/gpu/drm/amd/amdgpu/si.h b/drivers/gpu/drm/amd/amdgpu/si.h
index 959d7b63e0e5..589225080c24 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.h
+++ b/drivers/gpu/drm/amd/amdgpu/si.h
@@ -24,8 +24,6 @@
 #ifndef __SI_H__
 #define __SI_H__
 
-extern const struct amd_ip_funcs si_common_ip_funcs;
-
 void si_srbm_select(struct amdgpu_device *adev,
 		     u32 me, u32 pipe, u32 queue, u32 vmid);
 int si_set_ip_blocks(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index de358193a8f9..3dd552ae0b59 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -274,7 +274,7 @@ static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_ib ib;
-	struct fence *f = NULL;
+	struct dma_fence *f = NULL;
 	unsigned index;
 	u32 tmp = 0;
 	u64 gpu_addr;
@@ -305,7 +305,7 @@ static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	if (r)
 		goto err1;
 
-	r = fence_wait_timeout(f, false, timeout);
+	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
 		DRM_ERROR("amdgpu: IB test timed out\n");
 		r = -ETIMEDOUT;
@@ -325,7 +325,7 @@ static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 err1:
 	amdgpu_ib_free(adev, &ib, NULL);
-	fence_put(f);
+	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
 	return r;
@@ -495,22 +495,6 @@ static void si_dma_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, (0 << 28) | 0x20); /* func(always) | poll interval */
 }
 
-static unsigned si_dma_ring_get_emit_ib_size(struct amdgpu_ring *ring)
-{
-	return
-		7 + 3; /* si_dma_ring_emit_ib */
-}
-
-static unsigned si_dma_ring_get_dma_frame_size(struct amdgpu_ring *ring)
-{
-	return
-		3 + /* si_dma_ring_emit_hdp_flush */
-		3 + /* si_dma_ring_emit_hdp_invalidate */
-		6 + /* si_dma_ring_emit_pipeline_sync */
-		12 + /* si_dma_ring_emit_vm_flush */
-		9 + 9 + 9; /* si_dma_ring_emit_fence x3 for user fence, vm fence */
-}
-
 static int si_dma_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -547,11 +531,10 @@ static int si_dma_sw_init(void *handle)
 		ring->use_doorbell = false;
 		sprintf(ring->name, "sdma%d", i);
 		r = amdgpu_ring_init(adev, ring, 1024,
-				     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0), 0xf,
 				     &adev->sdma.trap_irq,
 				     (i == 0) ?
-				     AMDGPU_SDMA_IRQ_TRAP0 : AMDGPU_SDMA_IRQ_TRAP1,
-				     AMDGPU_RING_TYPE_SDMA);
+				     AMDGPU_SDMA_IRQ_TRAP0 :
+				     AMDGPU_SDMA_IRQ_TRAP1);
 		if (r)
 			return r;
 	}
@@ -762,7 +745,7 @@ static int si_dma_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs si_dma_ip_funcs = {
+static const struct amd_ip_funcs si_dma_ip_funcs = {
 	.name = "si_dma",
 	.early_init = si_dma_early_init,
 	.late_init = NULL,
@@ -780,10 +763,19 @@ const struct amd_ip_funcs si_dma_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs si_dma_ring_funcs = {
+	.type = AMDGPU_RING_TYPE_SDMA,
+	.align_mask = 0xf,
+	.nop = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0),
 	.get_rptr = si_dma_ring_get_rptr,
 	.get_wptr = si_dma_ring_get_wptr,
 	.set_wptr = si_dma_ring_set_wptr,
-	.parse_cs = NULL,
+	.emit_frame_size =
+		3 + /* si_dma_ring_emit_hdp_flush */
+		3 + /* si_dma_ring_emit_hdp_invalidate */
+		6 + /* si_dma_ring_emit_pipeline_sync */
+		12 + /* si_dma_ring_emit_vm_flush */
+		9 + 9 + 9, /* si_dma_ring_emit_fence x3 for user fence, vm fence */
+	.emit_ib_size = 7 + 3, /* si_dma_ring_emit_ib */
 	.emit_ib = si_dma_ring_emit_ib,
 	.emit_fence = si_dma_ring_emit_fence,
 	.emit_pipeline_sync = si_dma_ring_emit_pipeline_sync,
@@ -794,8 +786,6 @@ static const struct amdgpu_ring_funcs si_dma_ring_funcs = {
 	.test_ib = si_dma_ring_test_ib,
 	.insert_nop = amdgpu_ring_insert_nop,
 	.pad_ib = si_dma_ring_pad_ib,
-	.get_emit_ib_size = si_dma_ring_get_emit_ib_size,
-	.get_dma_frame_size = si_dma_ring_get_dma_frame_size,
 };
 
 static void si_dma_set_ring_funcs(struct amdgpu_device *adev)
@@ -913,3 +903,12 @@ static void si_dma_set_vm_pte_funcs(struct amdgpu_device *adev)
 		adev->vm_manager.vm_pte_num_rings = adev->sdma.num_instances;
 	}
 }
+
+const struct amdgpu_ip_block_version si_dma_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_SDMA,
+	.major = 1,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &si_dma_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.h b/drivers/gpu/drm/amd/amdgpu/si_dma.h
index 3a3e0c78a54b..5ac1b8452fb4 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.h
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.h
@@ -24,6 +24,6 @@
 #ifndef __SI_DMA_H__
 #define __SI_DMA_H__
 
-extern const struct amd_ip_funcs si_dma_ip_funcs;
+extern const struct amdgpu_ip_block_version si_dma_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index d6f85b1a0b93..6c65a1a2de79 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -3171,6 +3171,7 @@ static void ni_update_current_ps(struct amdgpu_device *adev,
 	eg_pi->current_rps = *rps;
 	ni_pi->current_ps = *new_ps;
 	eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
+	adev->pm.dpm.current_ps = &eg_pi->current_rps;
 }
 
 static void ni_update_requested_ps(struct amdgpu_device *adev,
@@ -3183,6 +3184,7 @@ static void ni_update_requested_ps(struct amdgpu_device *adev,
 	eg_pi->requested_rps = *rps;
 	ni_pi->requested_ps = *new_ps;
 	eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
+	adev->pm.dpm.requested_ps = &eg_pi->requested_rps;
 }
 
 static void ni_set_uvd_clock_before_set_eng_clock(struct amdgpu_device *adev,
@@ -3504,6 +3506,7 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev,
 		    (adev->pdev->revision == 0x80) ||
 		    (adev->pdev->revision == 0x81) ||
 		    (adev->pdev->revision == 0x83) ||
+		    (adev->pdev->revision == 0x87) ||
 		    (adev->pdev->device == 0x6604) ||
 		    (adev->pdev->device == 0x6605)) {
 			max_sclk = 75000;
@@ -7347,7 +7350,7 @@ static int si_parse_power_table(struct amdgpu_device *adev)
 	adev->pm.dpm.num_ps = state_array->ucNumEntries;
 
 	/* fill in the vce power states */
-	for (i = 0; i < AMDGPU_MAX_VCE_LEVELS; i++) {
+	for (i = 0; i < adev->pm.dpm.num_of_vce_states; i++) {
 		u32 sclk, mclk;
 		clock_array_index = adev->pm.dpm.vce_states[i].clk_idx;
 		clock_info = (union pplib_clock_info *)
@@ -7713,6 +7716,7 @@ static int si_dpm_init_microcode(struct amdgpu_device *adev)
 		    (adev->pdev->revision == 0x80) ||
 		    (adev->pdev->revision == 0x81) ||
 		    (adev->pdev->revision == 0x83) ||
+		    (adev->pdev->revision == 0x87) ||
 		    (adev->pdev->device == 0x6604) ||
 		    (adev->pdev->device == 0x6605))
 			chip_name = "oland_k";
@@ -7986,6 +7990,57 @@ static int si_dpm_early_init(void *handle)
 	return 0;
 }
 
+static inline bool si_are_power_levels_equal(const struct rv7xx_pl  *si_cpl1,
+						const struct rv7xx_pl *si_cpl2)
+{
+	return ((si_cpl1->mclk == si_cpl2->mclk) &&
+		  (si_cpl1->sclk == si_cpl2->sclk) &&
+		  (si_cpl1->pcie_gen == si_cpl2->pcie_gen) &&
+		  (si_cpl1->vddc == si_cpl2->vddc) &&
+		  (si_cpl1->vddci == si_cpl2->vddci));
+}
+
+static int si_check_state_equal(struct amdgpu_device *adev,
+				struct amdgpu_ps *cps,
+				struct amdgpu_ps *rps,
+				bool *equal)
+{
+	struct si_ps *si_cps;
+	struct si_ps *si_rps;
+	int i;
+
+	if (adev == NULL || cps == NULL || rps == NULL || equal == NULL)
+		return -EINVAL;
+
+	si_cps = si_get_ps(cps);
+	si_rps = si_get_ps(rps);
+
+	if (si_cps == NULL) {
+		printk("si_cps is NULL\n");
+		*equal = false;
+		return 0;
+	}
+
+	if (si_cps->performance_level_count != si_rps->performance_level_count) {
+		*equal = false;
+		return 0;
+	}
+
+	for (i = 0; i < si_cps->performance_level_count; i++) {
+		if (!si_are_power_levels_equal(&(si_cps->performance_levels[i]),
+					&(si_rps->performance_levels[i]))) {
+			*equal = false;
+			return 0;
+		}
+	}
+
+	/* If all performance levels are the same try to use the UVD clocks to break the tie.*/
+	*equal = ((cps->vclk == rps->vclk) && (cps->dclk == rps->dclk));
+	*equal &= ((cps->evclk == rps->evclk) && (cps->ecclk == rps->ecclk));
+
+	return 0;
+}
+
 
 const struct amd_ip_funcs si_dpm_ip_funcs = {
 	.name = "si_dpm",
@@ -8020,6 +8075,8 @@ static const struct amdgpu_dpm_funcs si_dpm_funcs = {
 	.get_fan_control_mode = &si_dpm_get_fan_control_mode,
 	.set_fan_speed_percent = &si_dpm_set_fan_speed_percent,
 	.get_fan_speed_percent = &si_dpm_get_fan_speed_percent,
+	.check_state_equal = &si_check_state_equal,
+	.get_vce_clock_state = amdgpu_get_vce_clock_state,
 };
 
 static void si_dpm_set_dpm_funcs(struct amdgpu_device *adev)
@@ -8039,3 +8096,11 @@ static void si_dpm_set_irq_funcs(struct amdgpu_device *adev)
 	adev->pm.dpm.thermal.irq.funcs = &si_dpm_irq_funcs;
 }
 
+const struct amdgpu_ip_block_version si_dpm_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_SMC,
+	.major = 6,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &si_dpm_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/si_enums.h b/drivers/gpu/drm/amd/amdgpu/si_enums.h
new file mode 100644
index 000000000000..fde2086246fa
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/si_enums.h
@@ -0,0 +1,272 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef SI_ENUMS_H
+#define SI_ENUMS_H
+
+#define VBLANK_INT_MASK                (1 << 0)
+#define DC_HPDx_INT_EN                 (1 << 16)
+#define VBLANK_ACK                     (1 << 4)
+#define VLINE_ACK                      (1 << 4)
+
+#define CURSOR_WIDTH 64
+#define CURSOR_HEIGHT 64
+
+#define VGA_VSTATUS_CNTL               0xFFFCFFFF
+#define PRIORITY_MARK_MASK             0x7fff
+#define PRIORITY_OFF                   (1 << 16)
+#define PRIORITY_ALWAYS_ON             (1 << 20)
+#define INTERLEAVE_EN                  (1 << 0)
+
+#define LATENCY_WATERMARK_MASK(x)      ((x) << 16)
+#define DC_LB_MEMORY_CONFIG(x)         ((x) << 20)
+#define ICON_DEGAMMA_MODE(x)           (((x) & 0x3) << 8)
+
+#define GRPH_ENDIAN_SWAP(x)            (((x) & 0x3) << 0)
+#define GRPH_ENDIAN_NONE               0
+#define GRPH_ENDIAN_8IN16              1
+#define GRPH_ENDIAN_8IN32              2
+#define GRPH_ENDIAN_8IN64              3
+
+#define GRPH_DEPTH(x)                  (((x) & 0x3) << 0)
+#define GRPH_DEPTH_8BPP                0
+#define GRPH_DEPTH_16BPP               1
+#define GRPH_DEPTH_32BPP               2
+
+#define GRPH_FORMAT(x)                 (((x) & 0x7) << 8)
+#define GRPH_FORMAT_INDEXED            0
+#define GRPH_FORMAT_ARGB1555           0
+#define GRPH_FORMAT_ARGB565            1
+#define GRPH_FORMAT_ARGB4444           2
+#define GRPH_FORMAT_AI88               3
+#define GRPH_FORMAT_MONO16             4
+#define GRPH_FORMAT_BGRA5551           5
+#define GRPH_FORMAT_ARGB8888           0
+#define GRPH_FORMAT_ARGB2101010        1
+#define GRPH_FORMAT_32BPP_DIG          2
+#define GRPH_FORMAT_8B_ARGB2101010     3
+#define GRPH_FORMAT_BGRA1010102        4
+#define GRPH_FORMAT_8B_BGRA1010102     5
+#define GRPH_FORMAT_RGB111110          6
+#define GRPH_FORMAT_BGR101111          7
+
+#define GRPH_NUM_BANKS(x)              (((x) & 0x3) << 2)
+#define GRPH_ARRAY_MODE(x)             (((x) & 0x7) << 20)
+#define GRPH_ARRAY_LINEAR_GENERAL      0
+#define GRPH_ARRAY_LINEAR_ALIGNED      1
+#define GRPH_ARRAY_1D_TILED_THIN1      2
+#define GRPH_ARRAY_2D_TILED_THIN1      4
+#define GRPH_TILE_SPLIT(x)             (((x) & 0x7) << 13)
+#define GRPH_BANK_WIDTH(x)             (((x) & 0x3) << 6)
+#define GRPH_BANK_HEIGHT(x)            (((x) & 0x3) << 11)
+#define GRPH_MACRO_TILE_ASPECT(x)      (((x) & 0x3) << 18)
+#define GRPH_ARRAY_MODE(x)             (((x) & 0x7) << 20)
+#define GRPH_PIPE_CONFIG(x)                   (((x) & 0x1f) << 24)
+
+#define CURSOR_EN                      (1 << 0)
+#define CURSOR_MODE(x)                 (((x) & 0x3) << 8)
+#define CURSOR_MONO                    0
+#define CURSOR_24_1                    1
+#define CURSOR_24_8_PRE_MULT           2
+#define CURSOR_24_8_UNPRE_MULT         3
+#define CURSOR_2X_MAGNIFY              (1 << 16)
+#define CURSOR_FORCE_MC_ON             (1 << 20)
+#define CURSOR_URGENT_CONTROL(x)       (((x) & 0x7) << 24)
+#define CURSOR_URGENT_ALWAYS           0
+#define CURSOR_URGENT_1_8              1
+#define CURSOR_URGENT_1_4              2
+#define CURSOR_URGENT_3_8              3
+#define CURSOR_URGENT_1_2              4
+#define CURSOR_UPDATE_PENDING          (1 << 0)
+#define CURSOR_UPDATE_TAKEN            (1 << 1)
+#define CURSOR_UPDATE_LOCK             (1 << 16)
+#define CURSOR_DISABLE_MULTIPLE_UPDATE (1 << 24)
+
+#define AMDGPU_NUM_OF_VMIDS                     8
+#define SI_CRTC0_REGISTER_OFFSET                0
+#define SI_CRTC1_REGISTER_OFFSET                0x300
+#define SI_CRTC2_REGISTER_OFFSET                0x2600
+#define SI_CRTC3_REGISTER_OFFSET                0x2900
+#define SI_CRTC4_REGISTER_OFFSET                0x2c00
+#define SI_CRTC5_REGISTER_OFFSET                0x2f00
+
+#define DMA0_REGISTER_OFFSET 0x000
+#define DMA1_REGISTER_OFFSET 0x200
+#define ES_AND_GS_AUTO       3
+#define RADEON_PACKET_TYPE3  3
+#define CE_PARTITION_BASE    3
+#define BUF_SWAP_32BIT       (2 << 16)
+
+#define GFX_POWER_STATUS                           (1 << 1)
+#define GFX_CLOCK_STATUS                           (1 << 2)
+#define GFX_LS_STATUS                              (1 << 3)
+#define RLC_BUSY_STATUS                            (1 << 0)
+
+#define RLC_PUD(x)                               ((x) << 0)
+#define RLC_PUD_MASK                             (0xff << 0)
+#define RLC_PDD(x)                               ((x) << 8)
+#define RLC_PDD_MASK                             (0xff << 8)
+#define RLC_TTPD(x)                              ((x) << 16)
+#define RLC_TTPD_MASK                            (0xff << 16)
+#define RLC_MSD(x)                               ((x) << 24)
+#define RLC_MSD_MASK                             (0xff << 24)
+#define WRITE_DATA_ENGINE_SEL(x) ((x) << 30)
+#define WRITE_DATA_DST_SEL(x) ((x) << 8)
+#define EVENT_TYPE(x) ((x) << 0)
+#define EVENT_INDEX(x) ((x) << 8)
+#define WAIT_REG_MEM_MEM_SPACE(x)               ((x) << 4)
+#define WAIT_REG_MEM_FUNCTION(x)                ((x) << 0)
+#define WAIT_REG_MEM_ENGINE(x)                  ((x) << 8)
+
+#define GFX6_NUM_GFX_RINGS     1
+#define GFX6_NUM_COMPUTE_RINGS 2
+#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
+#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
+
+#define TAHITI_GB_ADDR_CONFIG_GOLDEN        0x12011003
+#define VERDE_GB_ADDR_CONFIG_GOLDEN         0x12010002
+#define HAINAN_GB_ADDR_CONFIG_GOLDEN        0x02010001
+
+#define PACKET3(op, n)  ((RADEON_PACKET_TYPE3 << 30) |                  \
+                         (((op) & 0xFF) << 8) |                         \
+                         ((n) & 0x3FFF) << 16)
+#define PACKET3_COMPUTE(op, n) (PACKET3(op, n) | 1 << 1)
+#define	PACKET3_NOP					0x10
+#define	PACKET3_SET_BASE				0x11
+#define		PACKET3_BASE_INDEX(x)                  ((x) << 0)
+#define	PACKET3_CLEAR_STATE				0x12
+#define	PACKET3_INDEX_BUFFER_SIZE			0x13
+#define	PACKET3_DISPATCH_DIRECT				0x15
+#define	PACKET3_DISPATCH_INDIRECT			0x16
+#define	PACKET3_ALLOC_GDS				0x1B
+#define	PACKET3_WRITE_GDS_RAM				0x1C
+#define	PACKET3_ATOMIC_GDS				0x1D
+#define	PACKET3_ATOMIC					0x1E
+#define	PACKET3_OCCLUSION_QUERY				0x1F
+#define	PACKET3_SET_PREDICATION				0x20
+#define	PACKET3_REG_RMW					0x21
+#define	PACKET3_COND_EXEC				0x22
+#define	PACKET3_PRED_EXEC				0x23
+#define	PACKET3_DRAW_INDIRECT				0x24
+#define	PACKET3_DRAW_INDEX_INDIRECT			0x25
+#define	PACKET3_INDEX_BASE				0x26
+#define	PACKET3_DRAW_INDEX_2				0x27
+#define	PACKET3_CONTEXT_CONTROL				0x28
+#define	PACKET3_INDEX_TYPE				0x2A
+#define	PACKET3_DRAW_INDIRECT_MULTI			0x2C
+#define	PACKET3_DRAW_INDEX_AUTO				0x2D
+#define	PACKET3_DRAW_INDEX_IMMD				0x2E
+#define	PACKET3_NUM_INSTANCES				0x2F
+#define	PACKET3_DRAW_INDEX_MULTI_AUTO			0x30
+#define	PACKET3_INDIRECT_BUFFER_CONST			0x31
+#define	PACKET3_INDIRECT_BUFFER				0x3F
+#define	PACKET3_STRMOUT_BUFFER_UPDATE			0x34
+#define	PACKET3_DRAW_INDEX_OFFSET_2			0x35
+#define	PACKET3_DRAW_INDEX_MULTI_ELEMENT		0x36
+#define	PACKET3_WRITE_DATA				0x37
+#define	PACKET3_DRAW_INDEX_INDIRECT_MULTI		0x38
+#define	PACKET3_MEM_SEMAPHORE				0x39
+#define	PACKET3_MPEG_INDEX				0x3A
+#define	PACKET3_COPY_DW					0x3B
+#define	PACKET3_WAIT_REG_MEM				0x3C
+#define	PACKET3_MEM_WRITE				0x3D
+#define	PACKET3_COPY_DATA				0x40
+#define	PACKET3_CP_DMA					0x41
+#              define PACKET3_CP_DMA_DST_SEL(x)    ((x) << 20)
+#              define PACKET3_CP_DMA_ENGINE(x)     ((x) << 27)
+#              define PACKET3_CP_DMA_SRC_SEL(x)    ((x) << 29)
+#              define PACKET3_CP_DMA_CP_SYNC       (1 << 31)
+#              define PACKET3_CP_DMA_DIS_WC        (1 << 21)
+#              define PACKET3_CP_DMA_CMD_SRC_SWAP(x) ((x) << 22)
+#              define PACKET3_CP_DMA_CMD_DST_SWAP(x) ((x) << 24)
+#              define PACKET3_CP_DMA_CMD_SAS       (1 << 26)
+#              define PACKET3_CP_DMA_CMD_DAS       (1 << 27)
+#              define PACKET3_CP_DMA_CMD_SAIC      (1 << 28)
+#              define PACKET3_CP_DMA_CMD_DAIC      (1 << 29)
+#              define PACKET3_CP_DMA_CMD_RAW_WAIT  (1 << 30)
+#define	PACKET3_PFP_SYNC_ME				0x42
+#define	PACKET3_SURFACE_SYNC				0x43
+#              define PACKET3_DEST_BASE_0_ENA      (1 << 0)
+#              define PACKET3_DEST_BASE_1_ENA      (1 << 1)
+#              define PACKET3_CB0_DEST_BASE_ENA    (1 << 6)
+#              define PACKET3_CB1_DEST_BASE_ENA    (1 << 7)
+#              define PACKET3_CB2_DEST_BASE_ENA    (1 << 8)
+#              define PACKET3_CB3_DEST_BASE_ENA    (1 << 9)
+#              define PACKET3_CB4_DEST_BASE_ENA    (1 << 10)
+#              define PACKET3_CB5_DEST_BASE_ENA    (1 << 11)
+#              define PACKET3_CB6_DEST_BASE_ENA    (1 << 12)
+#              define PACKET3_CB7_DEST_BASE_ENA    (1 << 13)
+#              define PACKET3_DB_DEST_BASE_ENA     (1 << 14)
+#              define PACKET3_DEST_BASE_2_ENA      (1 << 19)
+#              define PACKET3_DEST_BASE_3_ENA      (1 << 21)
+#              define PACKET3_TCL1_ACTION_ENA      (1 << 22)
+#              define PACKET3_TC_ACTION_ENA        (1 << 23)
+#              define PACKET3_CB_ACTION_ENA        (1 << 25)
+#              define PACKET3_DB_ACTION_ENA        (1 << 26)
+#              define PACKET3_SH_KCACHE_ACTION_ENA (1 << 27)
+#              define PACKET3_SH_ICACHE_ACTION_ENA (1 << 29)
+#define	PACKET3_ME_INITIALIZE				0x44
+#define		PACKET3_ME_INITIALIZE_DEVICE_ID(x) ((x) << 16)
+#define	PACKET3_COND_WRITE				0x45
+#define	PACKET3_EVENT_WRITE				0x46
+#define	PACKET3_EVENT_WRITE_EOP				0x47
+#define	PACKET3_EVENT_WRITE_EOS				0x48
+#define	PACKET3_PREAMBLE_CNTL				0x4A
+#              define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE     (2 << 28)
+#              define PACKET3_PREAMBLE_END_CLEAR_STATE       (3 << 28)
+#define	PACKET3_ONE_REG_WRITE				0x57
+#define	PACKET3_LOAD_CONFIG_REG				0x5F
+#define	PACKET3_LOAD_CONTEXT_REG			0x60
+#define	PACKET3_LOAD_SH_REG				0x61
+#define	PACKET3_SET_CONFIG_REG				0x68
+#define		PACKET3_SET_CONFIG_REG_START			0x00002000
+#define		PACKET3_SET_CONFIG_REG_END			0x00002c00
+#define	PACKET3_SET_CONTEXT_REG				0x69
+#define		PACKET3_SET_CONTEXT_REG_START			0x000a000
+#define		PACKET3_SET_CONTEXT_REG_END			0x000a400
+#define	PACKET3_SET_CONTEXT_REG_INDIRECT		0x73
+#define	PACKET3_SET_RESOURCE_INDIRECT			0x74
+#define	PACKET3_SET_SH_REG				0x76
+#define		PACKET3_SET_SH_REG_START			0x00002c00
+#define		PACKET3_SET_SH_REG_END				0x00003000
+#define	PACKET3_SET_SH_REG_OFFSET			0x77
+#define	PACKET3_ME_WRITE				0x7A
+#define	PACKET3_SCRATCH_RAM_WRITE			0x7D
+#define	PACKET3_SCRATCH_RAM_READ			0x7E
+#define	PACKET3_CE_WRITE				0x7F
+#define	PACKET3_LOAD_CONST_RAM				0x80
+#define	PACKET3_WRITE_CONST_RAM				0x81
+#define	PACKET3_WRITE_CONST_RAM_OFFSET			0x82
+#define	PACKET3_DUMP_CONST_RAM				0x83
+#define	PACKET3_INCREMENT_CE_COUNTER			0x84
+#define	PACKET3_INCREMENT_DE_COUNTER			0x85
+#define	PACKET3_WAIT_ON_CE_COUNTER			0x86
+#define	PACKET3_WAIT_ON_DE_COUNTER			0x87
+#define	PACKET3_WAIT_ON_DE_COUNTER_DIFF			0x88
+#define	PACKET3_SET_CE_DE_COUNTERS			0x89
+#define	PACKET3_WAIT_ON_AVAIL_BUFFER			0x8A
+#define	PACKET3_SWITCH_BUFFER				0x8B
+#define PACKET3_SEM_WAIT_ON_SIGNAL    (0x1 << 12)
+#define PACKET3_SEM_SEL_SIGNAL	    (0x6 << 29)
+#define PACKET3_SEM_SEL_WAIT	    (0x7 << 29)
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c
index 8fae3d4a2360..db0f36846661 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c
@@ -268,7 +268,7 @@ static int si_ih_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs si_ih_ip_funcs = {
+static const struct amd_ip_funcs si_ih_ip_funcs = {
 	.name = "si_ih",
 	.early_init = si_ih_early_init,
 	.late_init = NULL,
@@ -297,3 +297,11 @@ static void si_ih_set_interrupt_funcs(struct amdgpu_device *adev)
 		adev->irq.ih_funcs = &si_ih_funcs;
 }
 
+const struct amdgpu_ip_block_version si_ih_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_IH,
+	.major = 1,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &si_ih_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.h b/drivers/gpu/drm/amd/amdgpu/si_ih.h
index f3e3a954369c..42e64a53e24f 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/si_ih.h
@@ -24,6 +24,6 @@
 #ifndef __SI_IH_H__
 #define __SI_IH_H__
 
-extern const struct amd_ip_funcs si_ih_ip_funcs;
+extern const struct amdgpu_ip_block_version si_ih_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index b4ea229bb449..52b71ee58793 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -455,7 +455,7 @@ static int tonga_ih_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs tonga_ih_ip_funcs = {
+static const struct amd_ip_funcs tonga_ih_ip_funcs = {
 	.name = "tonga_ih",
 	.early_init = tonga_ih_early_init,
 	.late_init = NULL,
@@ -487,3 +487,11 @@ static void tonga_ih_set_interrupt_funcs(struct amdgpu_device *adev)
 		adev->irq.ih_funcs = &tonga_ih_funcs;
 }
 
+const struct amdgpu_ip_block_version tonga_ih_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_IH,
+	.major = 3,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &tonga_ih_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.h b/drivers/gpu/drm/amd/amdgpu/tonga_ih.h
index 7392d70fa4a7..499027eee5c2 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.h
@@ -24,6 +24,6 @@
 #ifndef __TONGA_IH_H__
 #define __TONGA_IH_H__
 
-extern const struct amd_ip_funcs tonga_ih_ip_funcs;
+extern const struct amdgpu_ip_block_version tonga_ih_ip_block;
 
-#endif /* __CZ_IH_H__ */
+#endif /* __TONGA_IH_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
index f6c941550b8f..96444e4d862a 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
@@ -36,13 +36,17 @@
 
 #include "bif/bif_4_1_d.h"
 
+#include "smu/smu_7_0_1_d.h"
+#include "smu/smu_7_0_1_sh_mask.h"
+
 static void uvd_v4_2_mc_resume(struct amdgpu_device *adev);
 static void uvd_v4_2_init_cg(struct amdgpu_device *adev);
 static void uvd_v4_2_set_ring_funcs(struct amdgpu_device *adev);
 static void uvd_v4_2_set_irq_funcs(struct amdgpu_device *adev);
 static int uvd_v4_2_start(struct amdgpu_device *adev);
 static void uvd_v4_2_stop(struct amdgpu_device *adev);
-
+static int uvd_v4_2_set_clockgating_state(void *handle,
+				enum amd_clockgating_state state);
 /**
  * uvd_v4_2_ring_get_rptr - get read pointer
  *
@@ -116,8 +120,7 @@ static int uvd_v4_2_sw_init(void *handle)
 
 	ring = &adev->uvd.ring;
 	sprintf(ring->name, "uvd");
-	r = amdgpu_ring_init(adev, ring, 512, PACKET0(mmUVD_NO_OP, 0), 0xf,
-			     &adev->uvd.irq, 0, AMDGPU_RING_TYPE_UVD);
+	r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
 
 	return r;
 }
@@ -152,9 +155,9 @@ static int uvd_v4_2_hw_init(void *handle)
 	uint32_t tmp;
 	int r;
 
-	/* raise clocks while booting up the VCPU */
-	amdgpu_asic_set_uvd_clocks(adev, 53300, 40000);
-
+	uvd_v4_2_init_cg(adev);
+	uvd_v4_2_set_clockgating_state(adev, AMD_CG_STATE_GATE);
+	amdgpu_asic_set_uvd_clocks(adev, 10000, 10000);
 	r = uvd_v4_2_start(adev);
 	if (r)
 		goto done;
@@ -194,8 +197,6 @@ static int uvd_v4_2_hw_init(void *handle)
 	amdgpu_ring_commit(ring);
 
 done:
-	/* lower clocks again */
-	amdgpu_asic_set_uvd_clocks(adev, 0, 0);
 
 	if (!r)
 		DRM_INFO("UVD initialized successfully.\n");
@@ -272,9 +273,6 @@ static int uvd_v4_2_start(struct amdgpu_device *adev)
 
 	uvd_v4_2_mc_resume(adev);
 
-	/* disable clock gating */
-	WREG32(mmUVD_CGC_GATE, 0);
-
 	/* disable interupt */
 	WREG32_P(mmUVD_MASTINT_EN, 0, ~(1 << 1));
 
@@ -526,20 +524,6 @@ static void uvd_v4_2_ring_emit_ib(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, ib->length_dw);
 }
 
-static unsigned uvd_v4_2_ring_get_emit_ib_size(struct amdgpu_ring *ring)
-{
-	return
-		4; /* uvd_v4_2_ring_emit_ib */
-}
-
-static unsigned uvd_v4_2_ring_get_dma_frame_size(struct amdgpu_ring *ring)
-{
-	return
-		2 + /* uvd_v4_2_ring_emit_hdp_flush */
-		2 + /* uvd_v4_2_ring_emit_hdp_invalidate */
-		14; /* uvd_v4_2_ring_emit_fence  x1 no user fence */
-}
-
 /**
  * uvd_v4_2_mc_resume - memory controller programming
  *
@@ -580,8 +564,6 @@ static void uvd_v4_2_mc_resume(struct amdgpu_device *adev)
 	WREG32(mmUVD_UDEC_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
 	WREG32(mmUVD_UDEC_DB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
 	WREG32(mmUVD_UDEC_DBW_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
-
-	uvd_v4_2_init_cg(adev);
 }
 
 static void uvd_v4_2_enable_mgcg(struct amdgpu_device *adev,
@@ -591,7 +573,7 @@ static void uvd_v4_2_enable_mgcg(struct amdgpu_device *adev,
 
 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_UVD_MGCG)) {
 		data = RREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL);
-		data = 0xfff;
+		data |= 0xfff;
 		WREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL, data);
 
 		orig = data = RREG32(mmUVD_CGC_CTRL);
@@ -615,6 +597,8 @@ static void uvd_v4_2_set_dcm(struct amdgpu_device *adev,
 {
 	u32 tmp, tmp2;
 
+	WREG32_FIELD(UVD_CGC_GATE, REGS, 0);
+
 	tmp = RREG32(mmUVD_CGC_CTRL);
 	tmp &= ~(UVD_CGC_CTRL__CLK_OFF_DELAY_MASK | UVD_CGC_CTRL__CLK_GATE_DLY_TIMER_MASK);
 	tmp |= UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK |
@@ -738,7 +722,7 @@ static int uvd_v4_2_set_powergating_state(void *handle,
 	}
 }
 
-const struct amd_ip_funcs uvd_v4_2_ip_funcs = {
+static const struct amd_ip_funcs uvd_v4_2_ip_funcs = {
 	.name = "uvd_v4_2",
 	.early_init = uvd_v4_2_early_init,
 	.late_init = NULL,
@@ -756,10 +740,18 @@ const struct amd_ip_funcs uvd_v4_2_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs uvd_v4_2_ring_funcs = {
+	.type = AMDGPU_RING_TYPE_UVD,
+	.align_mask = 0xf,
+	.nop = PACKET0(mmUVD_NO_OP, 0),
 	.get_rptr = uvd_v4_2_ring_get_rptr,
 	.get_wptr = uvd_v4_2_ring_get_wptr,
 	.set_wptr = uvd_v4_2_ring_set_wptr,
 	.parse_cs = amdgpu_uvd_ring_parse_cs,
+	.emit_frame_size =
+		2 + /* uvd_v4_2_ring_emit_hdp_flush */
+		2 + /* uvd_v4_2_ring_emit_hdp_invalidate */
+		14, /* uvd_v4_2_ring_emit_fence  x1 no user fence */
+	.emit_ib_size = 4, /* uvd_v4_2_ring_emit_ib */
 	.emit_ib = uvd_v4_2_ring_emit_ib,
 	.emit_fence = uvd_v4_2_ring_emit_fence,
 	.emit_hdp_flush = uvd_v4_2_ring_emit_hdp_flush,
@@ -770,8 +762,6 @@ static const struct amdgpu_ring_funcs uvd_v4_2_ring_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
-	.get_emit_ib_size = uvd_v4_2_ring_get_emit_ib_size,
-	.get_dma_frame_size = uvd_v4_2_ring_get_dma_frame_size,
 };
 
 static void uvd_v4_2_set_ring_funcs(struct amdgpu_device *adev)
@@ -789,3 +779,12 @@ static void uvd_v4_2_set_irq_funcs(struct amdgpu_device *adev)
 	adev->uvd.irq.num_types = 1;
 	adev->uvd.irq.funcs = &uvd_v4_2_irq_funcs;
 }
+
+const struct amdgpu_ip_block_version uvd_v4_2_ip_block =
+{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 4,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &uvd_v4_2_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.h b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.h
index 0a615dd50840..8a0444bb8b95 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.h
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.h
@@ -24,6 +24,6 @@
 #ifndef __UVD_V4_2_H__
 #define __UVD_V4_2_H__
 
-extern const struct amd_ip_funcs uvd_v4_2_ip_funcs;
+extern const struct amdgpu_ip_block_version uvd_v4_2_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index 400c16fe579e..a79e283590fb 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -33,12 +33,17 @@
 #include "oss/oss_2_0_sh_mask.h"
 #include "bif/bif_5_0_d.h"
 #include "vi.h"
+#include "smu/smu_7_1_2_d.h"
+#include "smu/smu_7_1_2_sh_mask.h"
 
 static void uvd_v5_0_set_ring_funcs(struct amdgpu_device *adev);
 static void uvd_v5_0_set_irq_funcs(struct amdgpu_device *adev);
 static int uvd_v5_0_start(struct amdgpu_device *adev);
 static void uvd_v5_0_stop(struct amdgpu_device *adev);
-
+static int uvd_v5_0_set_clockgating_state(void *handle,
+					  enum amd_clockgating_state state);
+static void uvd_v5_0_enable_mgcg(struct amdgpu_device *adev,
+				 bool enable);
 /**
  * uvd_v5_0_ring_get_rptr - get read pointer
  *
@@ -112,8 +117,7 @@ static int uvd_v5_0_sw_init(void *handle)
 
 	ring = &adev->uvd.ring;
 	sprintf(ring->name, "uvd");
-	r = amdgpu_ring_init(adev, ring, 512, PACKET0(mmUVD_NO_OP, 0), 0xf,
-			     &adev->uvd.irq, 0, AMDGPU_RING_TYPE_UVD);
+	r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
 
 	return r;
 }
@@ -148,9 +152,6 @@ static int uvd_v5_0_hw_init(void *handle)
 	uint32_t tmp;
 	int r;
 
-	/* raise clocks while booting up the VCPU */
-	amdgpu_asic_set_uvd_clocks(adev, 53300, 40000);
-
 	r = uvd_v5_0_start(adev);
 	if (r)
 		goto done;
@@ -188,11 +189,7 @@ static int uvd_v5_0_hw_init(void *handle)
 	amdgpu_ring_write(ring, 3);
 
 	amdgpu_ring_commit(ring);
-
 done:
-	/* lower clocks again */
-	amdgpu_asic_set_uvd_clocks(adev, 0, 0);
-
 	if (!r)
 		DRM_INFO("UVD initialized successfully.\n");
 
@@ -225,6 +222,7 @@ static int uvd_v5_0_suspend(void *handle)
 	r = uvd_v5_0_hw_fini(adev);
 	if (r)
 		return r;
+	uvd_v5_0_set_clockgating_state(adev, AMD_CG_STATE_GATE);
 
 	r = amdgpu_uvd_suspend(adev);
 	if (r)
@@ -312,8 +310,9 @@ static int uvd_v5_0_start(struct amdgpu_device *adev)
 
 	uvd_v5_0_mc_resume(adev);
 
-	/* disable clock gating */
-	WREG32(mmUVD_CGC_GATE, 0);
+	amdgpu_asic_set_uvd_clocks(adev, 10000, 10000);
+	uvd_v5_0_set_clockgating_state(adev, AMD_CG_STATE_UNGATE);
+	uvd_v5_0_enable_mgcg(adev, true);
 
 	/* disable interupt */
 	WREG32_P(mmUVD_MASTINT_EN, 0, ~(1 << 1));
@@ -577,20 +576,6 @@ static void uvd_v5_0_ring_emit_ib(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, ib->length_dw);
 }
 
-static unsigned uvd_v5_0_ring_get_emit_ib_size(struct amdgpu_ring *ring)
-{
-	return
-		6; /* uvd_v5_0_ring_emit_ib */
-}
-
-static unsigned uvd_v5_0_ring_get_dma_frame_size(struct amdgpu_ring *ring)
-{
-	return
-		2 + /* uvd_v5_0_ring_emit_hdp_flush */
-		2 + /* uvd_v5_0_ring_emit_hdp_invalidate */
-		14; /* uvd_v5_0_ring_emit_fence  x1 no user fence */
-}
-
 static bool uvd_v5_0_is_idle(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -641,16 +626,12 @@ static int uvd_v5_0_process_interrupt(struct amdgpu_device *adev,
 	return 0;
 }
 
-static void uvd_v5_0_set_sw_clock_gating(struct amdgpu_device *adev)
+static void uvd_v5_0_enable_clock_gating(struct amdgpu_device *adev, bool enable)
 {
-	uint32_t data, data1, data2, suvd_flags;
+	uint32_t data1, data3, suvd_flags;
 
-	data = RREG32(mmUVD_CGC_CTRL);
 	data1 = RREG32(mmUVD_SUVD_CGC_GATE);
-	data2 = RREG32(mmUVD_SUVD_CGC_CTRL);
-
-	data &= ~(UVD_CGC_CTRL__CLK_OFF_DELAY_MASK |
-		  UVD_CGC_CTRL__CLK_GATE_DLY_TIMER_MASK);
+	data3 = RREG32(mmUVD_CGC_GATE);
 
 	suvd_flags = UVD_SUVD_CGC_GATE__SRE_MASK |
 		     UVD_SUVD_CGC_GATE__SIT_MASK |
@@ -658,6 +639,51 @@ static void uvd_v5_0_set_sw_clock_gating(struct amdgpu_device *adev)
 		     UVD_SUVD_CGC_GATE__SCM_MASK |
 		     UVD_SUVD_CGC_GATE__SDB_MASK;
 
+	if (enable) {
+		data3 |= (UVD_CGC_GATE__SYS_MASK     |
+			UVD_CGC_GATE__UDEC_MASK      |
+			UVD_CGC_GATE__MPEG2_MASK     |
+			UVD_CGC_GATE__RBC_MASK       |
+			UVD_CGC_GATE__LMI_MC_MASK    |
+			UVD_CGC_GATE__IDCT_MASK      |
+			UVD_CGC_GATE__MPRD_MASK      |
+			UVD_CGC_GATE__MPC_MASK       |
+			UVD_CGC_GATE__LBSI_MASK      |
+			UVD_CGC_GATE__LRBBM_MASK     |
+			UVD_CGC_GATE__UDEC_RE_MASK   |
+			UVD_CGC_GATE__UDEC_CM_MASK   |
+			UVD_CGC_GATE__UDEC_IT_MASK   |
+			UVD_CGC_GATE__UDEC_DB_MASK   |
+			UVD_CGC_GATE__UDEC_MP_MASK   |
+			UVD_CGC_GATE__WCB_MASK       |
+			UVD_CGC_GATE__JPEG_MASK      |
+			UVD_CGC_GATE__SCPU_MASK);
+		/* only in pg enabled, we can gate clock to vcpu*/
+		if (adev->pg_flags & AMD_PG_SUPPORT_UVD)
+			data3 |= UVD_CGC_GATE__VCPU_MASK;
+		data3 &= ~UVD_CGC_GATE__REGS_MASK;
+		data1 |= suvd_flags;
+	} else {
+		data3 = 0;
+		data1 = 0;
+	}
+
+	WREG32(mmUVD_SUVD_CGC_GATE, data1);
+	WREG32(mmUVD_CGC_GATE, data3);
+}
+
+static void uvd_v5_0_set_sw_clock_gating(struct amdgpu_device *adev)
+{
+	uint32_t data, data2;
+
+	data = RREG32(mmUVD_CGC_CTRL);
+	data2 = RREG32(mmUVD_SUVD_CGC_CTRL);
+
+
+	data &= ~(UVD_CGC_CTRL__CLK_OFF_DELAY_MASK |
+		  UVD_CGC_CTRL__CLK_GATE_DLY_TIMER_MASK);
+
+
 	data |= UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK |
 		(1 << REG_FIELD_SHIFT(UVD_CGC_CTRL, CLK_GATE_DLY_TIMER)) |
 		(4 << REG_FIELD_SHIFT(UVD_CGC_CTRL, CLK_OFF_DELAY));
@@ -688,11 +714,8 @@ static void uvd_v5_0_set_sw_clock_gating(struct amdgpu_device *adev)
 			UVD_SUVD_CGC_CTRL__SMP_MODE_MASK |
 			UVD_SUVD_CGC_CTRL__SCM_MODE_MASK |
 			UVD_SUVD_CGC_CTRL__SDB_MODE_MASK);
-	data1 |= suvd_flags;
 
 	WREG32(mmUVD_CGC_CTRL, data);
-	WREG32(mmUVD_CGC_GATE, 0);
-	WREG32(mmUVD_SUVD_CGC_GATE, data1);
 	WREG32(mmUVD_SUVD_CGC_CTRL, data2);
 }
 
@@ -737,6 +760,32 @@ static void uvd_v5_0_set_hw_clock_gating(struct amdgpu_device *adev)
 }
 #endif
 
+static void uvd_v5_0_enable_mgcg(struct amdgpu_device *adev,
+				 bool enable)
+{
+	u32 orig, data;
+
+	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_UVD_MGCG)) {
+		data = RREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL);
+		data |= 0xfff;
+		WREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL, data);
+
+		orig = data = RREG32(mmUVD_CGC_CTRL);
+		data |= UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
+		if (orig != data)
+			WREG32(mmUVD_CGC_CTRL, data);
+	} else {
+		data = RREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL);
+		data &= ~0xfff;
+		WREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL, data);
+
+		orig = data = RREG32(mmUVD_CGC_CTRL);
+		data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
+		if (orig != data)
+			WREG32(mmUVD_CGC_CTRL, data);
+	}
+}
+
 static int uvd_v5_0_set_clockgating_state(void *handle,
 					  enum amd_clockgating_state state)
 {
@@ -752,17 +801,18 @@ static int uvd_v5_0_set_clockgating_state(void *handle,
 
 	curstate = state;
 	if (enable) {
-		/* disable HW gating and enable Sw gating */
-		uvd_v5_0_set_sw_clock_gating(adev);
-	} else {
 		/* wait for STATUS to clear */
 		if (uvd_v5_0_wait_for_idle(handle))
 			return -EBUSY;
+		uvd_v5_0_enable_clock_gating(adev, true);
 
 		/* enable HW gates because UVD is idle */
 /*		uvd_v5_0_set_hw_clock_gating(adev); */
+	} else {
+		uvd_v5_0_enable_clock_gating(adev, false);
 	}
 
+	uvd_v5_0_set_sw_clock_gating(adev);
 	return 0;
 }
 
@@ -789,7 +839,7 @@ static int uvd_v5_0_set_powergating_state(void *handle,
 	}
 }
 
-const struct amd_ip_funcs uvd_v5_0_ip_funcs = {
+static const struct amd_ip_funcs uvd_v5_0_ip_funcs = {
 	.name = "uvd_v5_0",
 	.early_init = uvd_v5_0_early_init,
 	.late_init = NULL,
@@ -807,10 +857,18 @@ const struct amd_ip_funcs uvd_v5_0_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = {
+	.type = AMDGPU_RING_TYPE_UVD,
+	.align_mask = 0xf,
+	.nop = PACKET0(mmUVD_NO_OP, 0),
 	.get_rptr = uvd_v5_0_ring_get_rptr,
 	.get_wptr = uvd_v5_0_ring_get_wptr,
 	.set_wptr = uvd_v5_0_ring_set_wptr,
 	.parse_cs = amdgpu_uvd_ring_parse_cs,
+	.emit_frame_size =
+		2 + /* uvd_v5_0_ring_emit_hdp_flush */
+		2 + /* uvd_v5_0_ring_emit_hdp_invalidate */
+		14, /* uvd_v5_0_ring_emit_fence  x1 no user fence */
+	.emit_ib_size = 6, /* uvd_v5_0_ring_emit_ib */
 	.emit_ib = uvd_v5_0_ring_emit_ib,
 	.emit_fence = uvd_v5_0_ring_emit_fence,
 	.emit_hdp_flush = uvd_v5_0_ring_emit_hdp_flush,
@@ -821,8 +879,6 @@ static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
-	.get_emit_ib_size = uvd_v5_0_ring_get_emit_ib_size,
-	.get_dma_frame_size = uvd_v5_0_ring_get_dma_frame_size,
 };
 
 static void uvd_v5_0_set_ring_funcs(struct amdgpu_device *adev)
@@ -840,3 +896,12 @@ static void uvd_v5_0_set_irq_funcs(struct amdgpu_device *adev)
 	adev->uvd.irq.num_types = 1;
 	adev->uvd.irq.funcs = &uvd_v5_0_irq_funcs;
 }
+
+const struct amdgpu_ip_block_version uvd_v5_0_ip_block =
+{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 5,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &uvd_v5_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.h b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.h
index e3b3c49fa5de..2eaaea793ac5 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.h
@@ -24,6 +24,6 @@
 #ifndef __UVD_V5_0_H__
 #define __UVD_V5_0_H__
 
-extern const struct amd_ip_funcs uvd_v5_0_ip_funcs;
+extern const struct amdgpu_ip_block_version uvd_v5_0_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index ab3df6d75656..ba0bbf7138dc 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -42,6 +42,10 @@ static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev);
 static int uvd_v6_0_start(struct amdgpu_device *adev);
 static void uvd_v6_0_stop(struct amdgpu_device *adev);
 static void uvd_v6_0_set_sw_clock_gating(struct amdgpu_device *adev);
+static int uvd_v6_0_set_clockgating_state(void *handle,
+					  enum amd_clockgating_state state);
+static void uvd_v6_0_enable_mgcg(struct amdgpu_device *adev,
+				 bool enable);
 
 /**
  * uvd_v6_0_ring_get_rptr - get read pointer
@@ -116,8 +120,7 @@ static int uvd_v6_0_sw_init(void *handle)
 
 	ring = &adev->uvd.ring;
 	sprintf(ring->name, "uvd");
-	r = amdgpu_ring_init(adev, ring, 512, PACKET0(mmUVD_NO_OP, 0), 0xf,
-			     &adev->uvd.irq, 0, AMDGPU_RING_TYPE_UVD);
+	r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
 
 	return r;
 }
@@ -394,11 +397,11 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
 	lmi_swap_cntl = 0;
 	mp_swap_cntl = 0;
 
+	amdgpu_asic_set_uvd_clocks(adev, 10000, 10000);
+	uvd_v6_0_set_clockgating_state(adev, AMD_CG_STATE_UNGATE);
+	uvd_v6_0_enable_mgcg(adev, true);
 	uvd_v6_0_mc_resume(adev);
 
-	/* disable clock gating */
-	WREG32_FIELD(UVD_CGC_CTRL, DYN_CLOCK_MODE, 0);
-
 	/* disable interupt */
 	WREG32_FIELD(UVD_MASTINT_EN, VCPU_EN, 0);
 
@@ -725,31 +728,6 @@ static void uvd_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, 0xE);
 }
 
-static unsigned uvd_v6_0_ring_get_emit_ib_size(struct amdgpu_ring *ring)
-{
-	return
-		8; /* uvd_v6_0_ring_emit_ib */
-}
-
-static unsigned uvd_v6_0_ring_get_dma_frame_size(struct amdgpu_ring *ring)
-{
-	return
-		2 + /* uvd_v6_0_ring_emit_hdp_flush */
-		2 + /* uvd_v6_0_ring_emit_hdp_invalidate */
-		10 + /* uvd_v6_0_ring_emit_pipeline_sync */
-		14; /* uvd_v6_0_ring_emit_fence x1 no user fence */
-}
-
-static unsigned uvd_v6_0_ring_get_dma_frame_size_vm(struct amdgpu_ring *ring)
-{
-	return
-		2 + /* uvd_v6_0_ring_emit_hdp_flush */
-		2 + /* uvd_v6_0_ring_emit_hdp_invalidate */
-		10 + /* uvd_v6_0_ring_emit_pipeline_sync */
-		20 + /* uvd_v6_0_ring_emit_vm_flush */
-		14 + 14; /* uvd_v6_0_ring_emit_fence x2 vm fence */
-}
-
 static bool uvd_v6_0_is_idle(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -862,22 +840,72 @@ static int uvd_v6_0_process_interrupt(struct amdgpu_device *adev,
 	return 0;
 }
 
+static void uvd_v6_0_enable_clock_gating(struct amdgpu_device *adev, bool enable)
+{
+	uint32_t data1, data3;
+
+	data1 = RREG32(mmUVD_SUVD_CGC_GATE);
+	data3 = RREG32(mmUVD_CGC_GATE);
+
+	data1 |= UVD_SUVD_CGC_GATE__SRE_MASK |
+		     UVD_SUVD_CGC_GATE__SIT_MASK |
+		     UVD_SUVD_CGC_GATE__SMP_MASK |
+		     UVD_SUVD_CGC_GATE__SCM_MASK |
+		     UVD_SUVD_CGC_GATE__SDB_MASK |
+		     UVD_SUVD_CGC_GATE__SRE_H264_MASK |
+		     UVD_SUVD_CGC_GATE__SRE_HEVC_MASK |
+		     UVD_SUVD_CGC_GATE__SIT_H264_MASK |
+		     UVD_SUVD_CGC_GATE__SIT_HEVC_MASK |
+		     UVD_SUVD_CGC_GATE__SCM_H264_MASK |
+		     UVD_SUVD_CGC_GATE__SCM_HEVC_MASK |
+		     UVD_SUVD_CGC_GATE__SDB_H264_MASK |
+		     UVD_SUVD_CGC_GATE__SDB_HEVC_MASK;
+
+	if (enable) {
+		data3 |= (UVD_CGC_GATE__SYS_MASK       |
+			UVD_CGC_GATE__UDEC_MASK      |
+			UVD_CGC_GATE__MPEG2_MASK     |
+			UVD_CGC_GATE__RBC_MASK       |
+			UVD_CGC_GATE__LMI_MC_MASK    |
+			UVD_CGC_GATE__LMI_UMC_MASK   |
+			UVD_CGC_GATE__IDCT_MASK      |
+			UVD_CGC_GATE__MPRD_MASK      |
+			UVD_CGC_GATE__MPC_MASK       |
+			UVD_CGC_GATE__LBSI_MASK      |
+			UVD_CGC_GATE__LRBBM_MASK     |
+			UVD_CGC_GATE__UDEC_RE_MASK   |
+			UVD_CGC_GATE__UDEC_CM_MASK   |
+			UVD_CGC_GATE__UDEC_IT_MASK   |
+			UVD_CGC_GATE__UDEC_DB_MASK   |
+			UVD_CGC_GATE__UDEC_MP_MASK   |
+			UVD_CGC_GATE__WCB_MASK       |
+			UVD_CGC_GATE__JPEG_MASK      |
+			UVD_CGC_GATE__SCPU_MASK      |
+			UVD_CGC_GATE__JPEG2_MASK);
+		/* only in pg enabled, we can gate clock to vcpu*/
+		if (adev->pg_flags & AMD_PG_SUPPORT_UVD)
+			data3 |= UVD_CGC_GATE__VCPU_MASK;
+
+		data3 &= ~UVD_CGC_GATE__REGS_MASK;
+	} else {
+		data3 = 0;
+	}
+
+	WREG32(mmUVD_SUVD_CGC_GATE, data1);
+	WREG32(mmUVD_CGC_GATE, data3);
+}
+
 static void uvd_v6_0_set_sw_clock_gating(struct amdgpu_device *adev)
 {
-	uint32_t data, data1, data2, suvd_flags;
+	uint32_t data, data2;
 
 	data = RREG32(mmUVD_CGC_CTRL);
-	data1 = RREG32(mmUVD_SUVD_CGC_GATE);
 	data2 = RREG32(mmUVD_SUVD_CGC_CTRL);
 
+
 	data &= ~(UVD_CGC_CTRL__CLK_OFF_DELAY_MASK |
 		  UVD_CGC_CTRL__CLK_GATE_DLY_TIMER_MASK);
 
-	suvd_flags = UVD_SUVD_CGC_GATE__SRE_MASK |
-		     UVD_SUVD_CGC_GATE__SIT_MASK |
-		     UVD_SUVD_CGC_GATE__SMP_MASK |
-		     UVD_SUVD_CGC_GATE__SCM_MASK |
-		     UVD_SUVD_CGC_GATE__SDB_MASK;
 
 	data |= UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK |
 		(1 << REG_FIELD_SHIFT(UVD_CGC_CTRL, CLK_GATE_DLY_TIMER)) |
@@ -910,11 +938,8 @@ static void uvd_v6_0_set_sw_clock_gating(struct amdgpu_device *adev)
 			UVD_SUVD_CGC_CTRL__SMP_MODE_MASK |
 			UVD_SUVD_CGC_CTRL__SCM_MODE_MASK |
 			UVD_SUVD_CGC_CTRL__SDB_MODE_MASK);
-	data1 |= suvd_flags;
 
 	WREG32(mmUVD_CGC_CTRL, data);
-	WREG32(mmUVD_CGC_GATE, 0);
-	WREG32(mmUVD_SUVD_CGC_GATE, data1);
 	WREG32(mmUVD_SUVD_CGC_CTRL, data2);
 }
 
@@ -961,44 +986,53 @@ static void uvd_v6_0_set_hw_clock_gating(struct amdgpu_device *adev)
 }
 #endif
 
-static void uvd_v6_set_bypass_mode(struct amdgpu_device *adev, bool enable)
+static void uvd_v6_0_enable_mgcg(struct amdgpu_device *adev,
+				 bool enable)
 {
-	u32 tmp = RREG32_SMC(ixGCK_DFS_BYPASS_CNTL);
+	u32 orig, data;
 
-	if (enable)
-		tmp |= (GCK_DFS_BYPASS_CNTL__BYPASSDCLK_MASK |
-			GCK_DFS_BYPASS_CNTL__BYPASSVCLK_MASK);
-	else
-		tmp &= ~(GCK_DFS_BYPASS_CNTL__BYPASSDCLK_MASK |
-			 GCK_DFS_BYPASS_CNTL__BYPASSVCLK_MASK);
+	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_UVD_MGCG)) {
+		data = RREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL);
+		data |= 0xfff;
+		WREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL, data);
 
-	WREG32_SMC(ixGCK_DFS_BYPASS_CNTL, tmp);
+		orig = data = RREG32(mmUVD_CGC_CTRL);
+		data |= UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
+		if (orig != data)
+			WREG32(mmUVD_CGC_CTRL, data);
+	} else {
+		data = RREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL);
+		data &= ~0xfff;
+		WREG32_UVD_CTX(ixUVD_CGC_MEM_CTRL, data);
+
+		orig = data = RREG32(mmUVD_CGC_CTRL);
+		data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
+		if (orig != data)
+			WREG32(mmUVD_CGC_CTRL, data);
+	}
 }
 
 static int uvd_v6_0_set_clockgating_state(void *handle,
 					  enum amd_clockgating_state state)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-
-	if (adev->asic_type == CHIP_FIJI ||
-	    adev->asic_type == CHIP_POLARIS10)
-		uvd_v6_set_bypass_mode(adev, state == AMD_CG_STATE_GATE ? true : false);
+	bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
 
 	if (!(adev->cg_flags & AMD_CG_SUPPORT_UVD_MGCG))
 		return 0;
 
-	if (state == AMD_CG_STATE_GATE) {
-		/* disable HW gating and enable Sw gating */
-		uvd_v6_0_set_sw_clock_gating(adev);
-	} else {
+	if (enable) {
 		/* wait for STATUS to clear */
 		if (uvd_v6_0_wait_for_idle(handle))
 			return -EBUSY;
-
+		uvd_v6_0_enable_clock_gating(adev, true);
 		/* enable HW gates because UVD is idle */
 /*		uvd_v6_0_set_hw_clock_gating(adev); */
+	} else {
+		/* disable HW gating and enable Sw gating */
+		uvd_v6_0_enable_clock_gating(adev, false);
 	}
-
+	uvd_v6_0_set_sw_clock_gating(adev);
 	return 0;
 }
 
@@ -1027,7 +1061,7 @@ static int uvd_v6_0_set_powergating_state(void *handle,
 	}
 }
 
-const struct amd_ip_funcs uvd_v6_0_ip_funcs = {
+static const struct amd_ip_funcs uvd_v6_0_ip_funcs = {
 	.name = "uvd_v6_0",
 	.early_init = uvd_v6_0_early_init,
 	.late_init = NULL,
@@ -1048,10 +1082,19 @@ const struct amd_ip_funcs uvd_v6_0_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs uvd_v6_0_ring_phys_funcs = {
+	.type = AMDGPU_RING_TYPE_UVD,
+	.align_mask = 0xf,
+	.nop = PACKET0(mmUVD_NO_OP, 0),
 	.get_rptr = uvd_v6_0_ring_get_rptr,
 	.get_wptr = uvd_v6_0_ring_get_wptr,
 	.set_wptr = uvd_v6_0_ring_set_wptr,
 	.parse_cs = amdgpu_uvd_ring_parse_cs,
+	.emit_frame_size =
+		2 + /* uvd_v6_0_ring_emit_hdp_flush */
+		2 + /* uvd_v6_0_ring_emit_hdp_invalidate */
+		10 + /* uvd_v6_0_ring_emit_pipeline_sync */
+		14, /* uvd_v6_0_ring_emit_fence x1 no user fence */
+	.emit_ib_size = 8, /* uvd_v6_0_ring_emit_ib */
 	.emit_ib = uvd_v6_0_ring_emit_ib,
 	.emit_fence = uvd_v6_0_ring_emit_fence,
 	.emit_hdp_flush = uvd_v6_0_ring_emit_hdp_flush,
@@ -1062,15 +1105,22 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_phys_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
-	.get_emit_ib_size = uvd_v6_0_ring_get_emit_ib_size,
-	.get_dma_frame_size = uvd_v6_0_ring_get_dma_frame_size,
 };
 
 static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
+	.type = AMDGPU_RING_TYPE_UVD,
+	.align_mask = 0xf,
+	.nop = PACKET0(mmUVD_NO_OP, 0),
 	.get_rptr = uvd_v6_0_ring_get_rptr,
 	.get_wptr = uvd_v6_0_ring_get_wptr,
 	.set_wptr = uvd_v6_0_ring_set_wptr,
-	.parse_cs = NULL,
+	.emit_frame_size =
+		2 + /* uvd_v6_0_ring_emit_hdp_flush */
+		2 + /* uvd_v6_0_ring_emit_hdp_invalidate */
+		10 + /* uvd_v6_0_ring_emit_pipeline_sync */
+		20 + /* uvd_v6_0_ring_emit_vm_flush */
+		14 + 14, /* uvd_v6_0_ring_emit_fence x2 vm fence */
+	.emit_ib_size = 8, /* uvd_v6_0_ring_emit_ib */
 	.emit_ib = uvd_v6_0_ring_emit_ib,
 	.emit_fence = uvd_v6_0_ring_emit_fence,
 	.emit_vm_flush = uvd_v6_0_ring_emit_vm_flush,
@@ -1083,8 +1133,6 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
-	.get_emit_ib_size = uvd_v6_0_ring_get_emit_ib_size,
-	.get_dma_frame_size = uvd_v6_0_ring_get_dma_frame_size_vm,
 };
 
 static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev)
@@ -1108,3 +1156,30 @@ static void uvd_v6_0_set_irq_funcs(struct amdgpu_device *adev)
 	adev->uvd.irq.num_types = 1;
 	adev->uvd.irq.funcs = &uvd_v6_0_irq_funcs;
 }
+
+const struct amdgpu_ip_block_version uvd_v6_0_ip_block =
+{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 6,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &uvd_v6_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version uvd_v6_2_ip_block =
+{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 6,
+		.minor = 2,
+		.rev = 0,
+		.funcs = &uvd_v6_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version uvd_v6_3_ip_block =
+{
+		.type = AMD_IP_BLOCK_TYPE_UVD,
+		.major = 6,
+		.minor = 3,
+		.rev = 0,
+		.funcs = &uvd_v6_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.h b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.h
index 6b92a2352986..d3d48c6428cd 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.h
@@ -24,6 +24,8 @@
 #ifndef __UVD_V6_0_H__
 #define __UVD_V6_0_H__
 
-extern const struct amd_ip_funcs uvd_v6_0_ip_funcs;
+extern const struct amdgpu_ip_block_version uvd_v6_0_ip_block;
+extern const struct amdgpu_ip_block_version uvd_v6_2_ip_block;
+extern const struct amdgpu_ip_block_version uvd_v6_3_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
index 76e64ad04a53..38ed903dd6f8 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
@@ -224,8 +224,8 @@ static int vce_v2_0_sw_init(void *handle)
 	for (i = 0; i < adev->vce.num_rings; i++) {
 		ring = &adev->vce.ring[i];
 		sprintf(ring->name, "vce%d", i);
-		r = amdgpu_ring_init(adev, ring, 512, VCE_CMD_NO_OP, 0xf,
-				     &adev->vce.irq, 0, AMDGPU_RING_TYPE_VCE);
+		r = amdgpu_ring_init(adev, ring, 512,
+				     &adev->vce.irq, 0);
 		if (r)
 			return r;
 	}
@@ -592,7 +592,7 @@ static int vce_v2_0_set_powergating_state(void *handle,
 		return vce_v2_0_start(adev);
 }
 
-const struct amd_ip_funcs vce_v2_0_ip_funcs = {
+static const struct amd_ip_funcs vce_v2_0_ip_funcs = {
 	.name = "vce_v2_0",
 	.early_init = vce_v2_0_early_init,
 	.late_init = NULL,
@@ -610,10 +610,15 @@ const struct amd_ip_funcs vce_v2_0_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs vce_v2_0_ring_funcs = {
+	.type = AMDGPU_RING_TYPE_VCE,
+	.align_mask = 0xf,
+	.nop = VCE_CMD_NO_OP,
 	.get_rptr = vce_v2_0_ring_get_rptr,
 	.get_wptr = vce_v2_0_ring_get_wptr,
 	.set_wptr = vce_v2_0_ring_set_wptr,
 	.parse_cs = amdgpu_vce_ring_parse_cs,
+	.emit_frame_size = 6, /* amdgpu_vce_ring_emit_fence  x1 no user fence */
+	.emit_ib_size = 4, /* amdgpu_vce_ring_emit_ib */
 	.emit_ib = amdgpu_vce_ring_emit_ib,
 	.emit_fence = amdgpu_vce_ring_emit_fence,
 	.test_ring = amdgpu_vce_ring_test_ring,
@@ -622,8 +627,6 @@ static const struct amdgpu_ring_funcs vce_v2_0_ring_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_vce_ring_begin_use,
 	.end_use = amdgpu_vce_ring_end_use,
-	.get_emit_ib_size = amdgpu_vce_ring_get_emit_ib_size,
-	.get_dma_frame_size = amdgpu_vce_ring_get_dma_frame_size,
 };
 
 static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev)
@@ -644,3 +647,12 @@ static void vce_v2_0_set_irq_funcs(struct amdgpu_device *adev)
 	adev->vce.irq.num_types = 1;
 	adev->vce.irq.funcs = &vce_v2_0_irq_funcs;
 };
+
+const struct amdgpu_ip_block_version vce_v2_0_ip_block =
+{
+		.type = AMD_IP_BLOCK_TYPE_VCE,
+		.major = 2,
+		.minor = 0,
+		.rev = 0,
+		.funcs = &vce_v2_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.h b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.h
index 0d2ae8a01acd..4d15167654a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.h
@@ -24,6 +24,6 @@
 #ifndef __VCE_V2_0_H__
 #define __VCE_V2_0_H__
 
-extern const struct amd_ip_funcs vce_v2_0_ip_funcs;
+extern const struct amdgpu_ip_block_version vce_v2_0_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index 6feed726e299..6b3293a1c7b8 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -134,7 +134,7 @@ static void vce_v3_0_set_vce_sw_clock_gating(struct amdgpu_device *adev,
 	   accessible but the firmware will throttle the clocks on the
 	   fly as necessary.
 	*/
-	if (gated) {
+	if (!gated) {
 		data = RREG32(mmVCE_CLOCK_GATING_B);
 		data |= 0x1ff;
 		data &= ~0xef0000;
@@ -395,8 +395,7 @@ static int vce_v3_0_sw_init(void *handle)
 	for (i = 0; i < adev->vce.num_rings; i++) {
 		ring = &adev->vce.ring[i];
 		sprintf(ring->name, "vce%d", i);
-		r = amdgpu_ring_init(adev, ring, 512, VCE_CMD_NO_OP, 0xf,
-				     &adev->vce.irq, 0, AMDGPU_RING_TYPE_VCE);
+		r = amdgpu_ring_init(adev, ring, 512, &adev->vce.irq, 0);
 		if (r)
 			return r;
 	}
@@ -814,28 +813,7 @@ static void vce_v3_0_emit_pipeline_sync(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, seq);
 }
 
-static unsigned vce_v3_0_ring_get_emit_ib_size(struct amdgpu_ring *ring)
-{
-	return
-		5; /* vce_v3_0_ring_emit_ib */
-}
-
-static unsigned vce_v3_0_ring_get_dma_frame_size(struct amdgpu_ring *ring)
-{
-	return
-		4 + /* vce_v3_0_emit_pipeline_sync */
-		6; /* amdgpu_vce_ring_emit_fence x1 no user fence */
-}
-
-static unsigned vce_v3_0_ring_get_dma_frame_size_vm(struct amdgpu_ring *ring)
-{
-	return
-		6 + /* vce_v3_0_emit_vm_flush */
-		4 + /* vce_v3_0_emit_pipeline_sync */
-		6 + 6; /* amdgpu_vce_ring_emit_fence x2 vm fence */
-}
-
-const struct amd_ip_funcs vce_v3_0_ip_funcs = {
+static const struct amd_ip_funcs vce_v3_0_ip_funcs = {
 	.name = "vce_v3_0",
 	.early_init = vce_v3_0_early_init,
 	.late_init = NULL,
@@ -856,10 +834,17 @@ const struct amd_ip_funcs vce_v3_0_ip_funcs = {
 };
 
 static const struct amdgpu_ring_funcs vce_v3_0_ring_phys_funcs = {
+	.type = AMDGPU_RING_TYPE_VCE,
+	.align_mask = 0xf,
+	.nop = VCE_CMD_NO_OP,
 	.get_rptr = vce_v3_0_ring_get_rptr,
 	.get_wptr = vce_v3_0_ring_get_wptr,
 	.set_wptr = vce_v3_0_ring_set_wptr,
 	.parse_cs = amdgpu_vce_ring_parse_cs,
+	.emit_frame_size =
+		4 + /* vce_v3_0_emit_pipeline_sync */
+		6, /* amdgpu_vce_ring_emit_fence x1 no user fence */
+	.emit_ib_size = 5, /* vce_v3_0_ring_emit_ib */
 	.emit_ib = amdgpu_vce_ring_emit_ib,
 	.emit_fence = amdgpu_vce_ring_emit_fence,
 	.test_ring = amdgpu_vce_ring_test_ring,
@@ -868,15 +853,21 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_phys_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_vce_ring_begin_use,
 	.end_use = amdgpu_vce_ring_end_use,
-	.get_emit_ib_size = vce_v3_0_ring_get_emit_ib_size,
-	.get_dma_frame_size = vce_v3_0_ring_get_dma_frame_size,
 };
 
 static const struct amdgpu_ring_funcs vce_v3_0_ring_vm_funcs = {
+	.type = AMDGPU_RING_TYPE_VCE,
+	.align_mask = 0xf,
+	.nop = VCE_CMD_NO_OP,
 	.get_rptr = vce_v3_0_ring_get_rptr,
 	.get_wptr = vce_v3_0_ring_get_wptr,
 	.set_wptr = vce_v3_0_ring_set_wptr,
-	.parse_cs = NULL,
+	.parse_cs = amdgpu_vce_ring_parse_cs_vm,
+	.emit_frame_size =
+		6 + /* vce_v3_0_emit_vm_flush */
+		4 + /* vce_v3_0_emit_pipeline_sync */
+		6 + 6, /* amdgpu_vce_ring_emit_fence x2 vm fence */
+	.emit_ib_size = 4, /* amdgpu_vce_ring_emit_ib */
 	.emit_ib = vce_v3_0_ring_emit_ib,
 	.emit_vm_flush = vce_v3_0_emit_vm_flush,
 	.emit_pipeline_sync = vce_v3_0_emit_pipeline_sync,
@@ -887,8 +878,6 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_vm_funcs = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_vce_ring_begin_use,
 	.end_use = amdgpu_vce_ring_end_use,
-	.get_emit_ib_size = vce_v3_0_ring_get_emit_ib_size,
-	.get_dma_frame_size = vce_v3_0_ring_get_dma_frame_size_vm,
 };
 
 static void vce_v3_0_set_ring_funcs(struct amdgpu_device *adev)
@@ -916,3 +905,30 @@ static void vce_v3_0_set_irq_funcs(struct amdgpu_device *adev)
 	adev->vce.irq.num_types = 1;
 	adev->vce.irq.funcs = &vce_v3_0_irq_funcs;
 };
+
+const struct amdgpu_ip_block_version vce_v3_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_VCE,
+	.major = 3,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &vce_v3_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version vce_v3_1_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_VCE,
+	.major = 3,
+	.minor = 1,
+	.rev = 0,
+	.funcs = &vce_v3_0_ip_funcs,
+};
+
+const struct amdgpu_ip_block_version vce_v3_4_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_VCE,
+	.major = 3,
+	.minor = 4,
+	.rev = 0,
+	.funcs = &vce_v3_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.h b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.h
index b45af65da81f..08b908c7de0f 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.h
@@ -24,6 +24,8 @@
 #ifndef __VCE_V3_0_H__
 #define __VCE_V3_0_H__
 
-extern const struct amd_ip_funcs vce_v3_0_ip_funcs;
+extern const struct amdgpu_ip_block_version vce_v3_0_ip_block;
+extern const struct amdgpu_ip_block_version vce_v3_1_ip_block;
+extern const struct amdgpu_ip_block_version vce_v3_4_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index f62f1a74f890..9f771f4ffcb7 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -123,8 +123,8 @@ static u32 vi_smc_rreg(struct amdgpu_device *adev, u32 reg)
 	u32 r;
 
 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	WREG32(mmSMC_IND_INDEX_0, (reg));
-	r = RREG32(mmSMC_IND_DATA_0);
+	WREG32(mmSMC_IND_INDEX_11, (reg));
+	r = RREG32(mmSMC_IND_DATA_11);
 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 	return r;
 }
@@ -134,8 +134,8 @@ static void vi_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 	unsigned long flags;
 
 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
-	WREG32(mmSMC_IND_INDEX_0, (reg));
-	WREG32(mmSMC_IND_DATA_0, (v));
+	WREG32(mmSMC_IND_INDEX_11, (reg));
+	WREG32(mmSMC_IND_DATA_11, (v));
 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 }
 
@@ -439,12 +439,12 @@ static bool vi_read_bios_from_rom(struct amdgpu_device *adev,
 	/* take the smc lock since we are using the smc index */
 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
 	/* set rom index to 0 */
-	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
-	WREG32(mmSMC_IND_DATA_0, 0);
+	WREG32(mmSMC_IND_INDEX_11, ixROM_INDEX);
+	WREG32(mmSMC_IND_DATA_11, 0);
 	/* set index to data for continous read */
-	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
+	WREG32(mmSMC_IND_INDEX_11, ixROM_DATA);
 	for (i = 0; i < length_dw; i++)
-		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
+		dw_ptr[i] = RREG32(mmSMC_IND_DATA_11);
 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 
 	return true;
@@ -558,21 +558,100 @@ static const struct amdgpu_allowed_register_entry vi_allowed_read_registers[] =
 	{mmPA_SC_RASTER_CONFIG_1, false, true},
 };
 
-static uint32_t vi_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
-					 u32 sh_num, u32 reg_offset)
-{
-	uint32_t val;
+static uint32_t vi_get_register_value(struct amdgpu_device *adev,
+				      bool indexed, u32 se_num,
+				      u32 sh_num, u32 reg_offset)
+{
+	if (indexed) {
+		uint32_t val;
+		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
+		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
+
+		switch (reg_offset) {
+		case mmCC_RB_BACKEND_DISABLE:
+			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
+		case mmGC_USER_RB_BACKEND_DISABLE:
+			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
+		case mmPA_SC_RASTER_CONFIG:
+			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
+		case mmPA_SC_RASTER_CONFIG_1:
+			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
+		}
 
-	mutex_lock(&adev->grbm_idx_mutex);
-	if (se_num != 0xffffffff || sh_num != 0xffffffff)
-		amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
+		mutex_lock(&adev->grbm_idx_mutex);
+		if (se_num != 0xffffffff || sh_num != 0xffffffff)
+			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
 
-	val = RREG32(reg_offset);
+		val = RREG32(reg_offset);
 
-	if (se_num != 0xffffffff || sh_num != 0xffffffff)
-		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
-	mutex_unlock(&adev->grbm_idx_mutex);
-	return val;
+		if (se_num != 0xffffffff || sh_num != 0xffffffff)
+			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+		mutex_unlock(&adev->grbm_idx_mutex);
+		return val;
+	} else {
+		unsigned idx;
+
+		switch (reg_offset) {
+		case mmGB_ADDR_CONFIG:
+			return adev->gfx.config.gb_addr_config;
+		case mmMC_ARB_RAMCFG:
+			return adev->gfx.config.mc_arb_ramcfg;
+		case mmGB_TILE_MODE0:
+		case mmGB_TILE_MODE1:
+		case mmGB_TILE_MODE2:
+		case mmGB_TILE_MODE3:
+		case mmGB_TILE_MODE4:
+		case mmGB_TILE_MODE5:
+		case mmGB_TILE_MODE6:
+		case mmGB_TILE_MODE7:
+		case mmGB_TILE_MODE8:
+		case mmGB_TILE_MODE9:
+		case mmGB_TILE_MODE10:
+		case mmGB_TILE_MODE11:
+		case mmGB_TILE_MODE12:
+		case mmGB_TILE_MODE13:
+		case mmGB_TILE_MODE14:
+		case mmGB_TILE_MODE15:
+		case mmGB_TILE_MODE16:
+		case mmGB_TILE_MODE17:
+		case mmGB_TILE_MODE18:
+		case mmGB_TILE_MODE19:
+		case mmGB_TILE_MODE20:
+		case mmGB_TILE_MODE21:
+		case mmGB_TILE_MODE22:
+		case mmGB_TILE_MODE23:
+		case mmGB_TILE_MODE24:
+		case mmGB_TILE_MODE25:
+		case mmGB_TILE_MODE26:
+		case mmGB_TILE_MODE27:
+		case mmGB_TILE_MODE28:
+		case mmGB_TILE_MODE29:
+		case mmGB_TILE_MODE30:
+		case mmGB_TILE_MODE31:
+			idx = (reg_offset - mmGB_TILE_MODE0);
+			return adev->gfx.config.tile_mode_array[idx];
+		case mmGB_MACROTILE_MODE0:
+		case mmGB_MACROTILE_MODE1:
+		case mmGB_MACROTILE_MODE2:
+		case mmGB_MACROTILE_MODE3:
+		case mmGB_MACROTILE_MODE4:
+		case mmGB_MACROTILE_MODE5:
+		case mmGB_MACROTILE_MODE6:
+		case mmGB_MACROTILE_MODE7:
+		case mmGB_MACROTILE_MODE8:
+		case mmGB_MACROTILE_MODE9:
+		case mmGB_MACROTILE_MODE10:
+		case mmGB_MACROTILE_MODE11:
+		case mmGB_MACROTILE_MODE12:
+		case mmGB_MACROTILE_MODE13:
+		case mmGB_MACROTILE_MODE14:
+		case mmGB_MACROTILE_MODE15:
+			idx = (reg_offset - mmGB_MACROTILE_MODE0);
+			return adev->gfx.config.macrotile_mode_array[idx];
+		default:
+			return RREG32(reg_offset);
+		}
+	}
 }
 
 static int vi_read_register(struct amdgpu_device *adev, u32 se_num,
@@ -607,10 +686,9 @@ static int vi_read_register(struct amdgpu_device *adev, u32 se_num,
 			if (reg_offset != asic_register_entry->reg_offset)
 				continue;
 			if (!asic_register_entry->untouched)
-				*value = asic_register_entry->grbm_indexed ?
-					vi_read_indexed_register(adev, se_num,
-								 sh_num, reg_offset) :
-					RREG32(reg_offset);
+				*value = vi_get_register_value(adev,
+							       asic_register_entry->grbm_indexed,
+							       se_num, sh_num, reg_offset);
 			return 0;
 		}
 	}
@@ -620,10 +698,9 @@ static int vi_read_register(struct amdgpu_device *adev, u32 se_num,
 			continue;
 
 		if (!vi_allowed_read_registers[i].untouched)
-			*value = vi_allowed_read_registers[i].grbm_indexed ?
-				vi_read_indexed_register(adev, se_num,
-							 sh_num, reg_offset) :
-				RREG32(reg_offset);
+			*value = vi_get_register_value(adev,
+						       vi_allowed_read_registers[i].grbm_indexed,
+						       se_num, sh_num, reg_offset);
 		return 0;
 	}
 	return -EINVAL;
@@ -654,18 +731,6 @@ static int vi_gpu_pci_config_reset(struct amdgpu_device *adev)
 	return -EINVAL;
 }
 
-static void vi_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
-{
-	u32 tmp = RREG32(mmBIOS_SCRATCH_3);
-
-	if (hung)
-		tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
-	else
-		tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
-
-	WREG32(mmBIOS_SCRATCH_3, tmp);
-}
-
 /**
  * vi_asic_reset - soft reset GPU
  *
@@ -679,11 +744,11 @@ static int vi_asic_reset(struct amdgpu_device *adev)
 {
 	int r;
 
-	vi_set_bios_scratch_engine_hung(adev, true);
+	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
 
 	r = vi_gpu_pci_config_reset(adev);
 
-	vi_set_bios_scratch_engine_hung(adev, false);
+	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
 
 	return r;
 }
@@ -783,734 +848,6 @@ static void vi_enable_doorbell_aperture(struct amdgpu_device *adev,
 	WREG32(mmBIF_DOORBELL_APER_EN, tmp);
 }
 
-/* topaz has no DCE, UVD, VCE */
-static const struct amdgpu_ip_block_version topaz_ip_blocks[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vi_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 7,
-		.minor = 4,
-		.rev = 0,
-		.funcs = &gmc_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 2,
-		.minor = 4,
-		.rev = 0,
-		.funcs = &iceland_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gfx_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 2,
-		.minor = 4,
-		.rev = 0,
-		.funcs = &sdma_v2_4_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version topaz_ip_blocks_vd[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vi_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 7,
-		.minor = 4,
-		.rev = 0,
-		.funcs = &gmc_v7_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 2,
-		.minor = 4,
-		.rev = 0,
-		.funcs = &iceland_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 1,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &dce_virtual_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gfx_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 2,
-		.minor = 4,
-		.rev = 0,
-		.funcs = &sdma_v2_4_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version tonga_ip_blocks[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vi_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &tonga_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 10,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &dce_v10_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gfx_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &sdma_v3_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 5,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &uvd_v5_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v3_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version tonga_ip_blocks_vd[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vi_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &tonga_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 10,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &dce_virtual_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gfx_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &sdma_v3_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 5,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &uvd_v5_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v3_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version fiji_ip_blocks[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vi_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 8,
-		.minor = 5,
-		.rev = 0,
-		.funcs = &gmc_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &tonga_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 10,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &dce_v10_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gfx_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &sdma_v3_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 6,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &uvd_v6_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v3_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version fiji_ip_blocks_vd[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vi_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 8,
-		.minor = 5,
-		.rev = 0,
-		.funcs = &gmc_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &tonga_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 10,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &dce_virtual_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gfx_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &sdma_v3_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 6,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &uvd_v6_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v3_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version polaris11_ip_blocks[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vi_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 8,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &gmc_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 3,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &tonga_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 11,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &dce_v11_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gfx_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 3,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &sdma_v3_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 6,
-		.minor = 3,
-		.rev = 0,
-		.funcs = &uvd_v6_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 3,
-		.minor = 4,
-		.rev = 0,
-		.funcs = &vce_v3_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version polaris11_ip_blocks_vd[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vi_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 8,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &gmc_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 3,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &tonga_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 7,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 11,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &dce_virtual_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gfx_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 3,
-		.minor = 1,
-		.rev = 0,
-		.funcs = &sdma_v3_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 6,
-		.minor = 3,
-		.rev = 0,
-		.funcs = &uvd_v6_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 3,
-		.minor = 4,
-		.rev = 0,
-		.funcs = &vce_v3_0_ip_funcs,
-	},
-};
-
-static const struct amdgpu_ip_block_version cz_ip_blocks[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vi_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cz_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 11,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &dce_v11_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gfx_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &sdma_v3_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 6,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &uvd_v6_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v3_0_ip_funcs,
-	},
-#if defined(CONFIG_DRM_AMD_ACP)
-	{
-		.type = AMD_IP_BLOCK_TYPE_ACP,
-		.major = 2,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &acp_ip_funcs,
-	},
-#endif
-};
-
-static const struct amdgpu_ip_block_version cz_ip_blocks_vd[] =
-{
-	/* ORDER MATTERS! */
-	{
-		.type = AMD_IP_BLOCK_TYPE_COMMON,
-		.major = 2,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vi_common_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GMC,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gmc_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_IH,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &cz_ih_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SMC,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &amdgpu_pp_ip_funcs
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_DCE,
-		.major = 11,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &dce_virtual_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_GFX,
-		.major = 8,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &gfx_v8_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_SDMA,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &sdma_v3_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_UVD,
-		.major = 6,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &uvd_v6_0_ip_funcs,
-	},
-	{
-		.type = AMD_IP_BLOCK_TYPE_VCE,
-		.major = 3,
-		.minor = 0,
-		.rev = 0,
-		.funcs = &vce_v3_0_ip_funcs,
-	},
-#if defined(CONFIG_DRM_AMD_ACP)
-	{
-		.type = AMD_IP_BLOCK_TYPE_ACP,
-		.major = 2,
-		.minor = 2,
-		.rev = 0,
-		.funcs = &acp_ip_funcs,
-	},
-#endif
-};
-
-int vi_set_ip_blocks(struct amdgpu_device *adev)
-{
-	if (adev->enable_virtual_display) {
-		switch (adev->asic_type) {
-		case CHIP_TOPAZ:
-			adev->ip_blocks = topaz_ip_blocks_vd;
-			adev->num_ip_blocks = ARRAY_SIZE(topaz_ip_blocks_vd);
-			break;
-		case CHIP_FIJI:
-			adev->ip_blocks = fiji_ip_blocks_vd;
-			adev->num_ip_blocks = ARRAY_SIZE(fiji_ip_blocks_vd);
-			break;
-		case CHIP_TONGA:
-			adev->ip_blocks = tonga_ip_blocks_vd;
-			adev->num_ip_blocks = ARRAY_SIZE(tonga_ip_blocks_vd);
-			break;
-		case CHIP_POLARIS11:
-		case CHIP_POLARIS10:
-			adev->ip_blocks = polaris11_ip_blocks_vd;
-			adev->num_ip_blocks = ARRAY_SIZE(polaris11_ip_blocks_vd);
-			break;
-
-		case CHIP_CARRIZO:
-		case CHIP_STONEY:
-			adev->ip_blocks = cz_ip_blocks_vd;
-			adev->num_ip_blocks = ARRAY_SIZE(cz_ip_blocks_vd);
-			break;
-		default:
-			/* FIXME: not supported yet */
-			return -EINVAL;
-		}
-	} else {
-		switch (adev->asic_type) {
-		case CHIP_TOPAZ:
-			adev->ip_blocks = topaz_ip_blocks;
-			adev->num_ip_blocks = ARRAY_SIZE(topaz_ip_blocks);
-			break;
-		case CHIP_FIJI:
-			adev->ip_blocks = fiji_ip_blocks;
-			adev->num_ip_blocks = ARRAY_SIZE(fiji_ip_blocks);
-			break;
-		case CHIP_TONGA:
-			adev->ip_blocks = tonga_ip_blocks;
-			adev->num_ip_blocks = ARRAY_SIZE(tonga_ip_blocks);
-			break;
-		case CHIP_POLARIS11:
-		case CHIP_POLARIS10:
-			adev->ip_blocks = polaris11_ip_blocks;
-			adev->num_ip_blocks = ARRAY_SIZE(polaris11_ip_blocks);
-			break;
-		case CHIP_CARRIZO:
-		case CHIP_STONEY:
-			adev->ip_blocks = cz_ip_blocks;
-			adev->num_ip_blocks = ARRAY_SIZE(cz_ip_blocks);
-			break;
-		default:
-			/* FIXME: not supported yet */
-			return -EINVAL;
-		}
-	}
-
-	return 0;
-}
-
 #define ATI_REV_ID_FUSE_MACRO__ADDRESS      0xC0014044
 #define ATI_REV_ID_FUSE_MACRO__SHIFT        9
 #define ATI_REV_ID_FUSE_MACRO__MASK         0x00001E00
@@ -1589,22 +926,25 @@ static int vi_common_early_init(void *handle)
 			AMD_CG_SUPPORT_HDP_LS |
 			AMD_CG_SUPPORT_ROM_MGCG |
 			AMD_CG_SUPPORT_MC_MGCG |
-			AMD_CG_SUPPORT_MC_LS;
+			AMD_CG_SUPPORT_MC_LS |
+			AMD_CG_SUPPORT_UVD_MGCG;
 		adev->pg_flags = 0;
 		adev->external_rev_id = adev->rev_id + 0x3c;
 		break;
 	case CHIP_TONGA:
 		adev->cg_flags = AMD_CG_SUPPORT_UVD_MGCG;
-		adev->pg_flags = 0;
+		adev->pg_flags = AMD_PG_SUPPORT_UVD;
 		adev->external_rev_id = adev->rev_id + 0x14;
 		break;
 	case CHIP_POLARIS11:
-		adev->cg_flags = 0;
+		adev->cg_flags = AMD_CG_SUPPORT_UVD_MGCG |
+			AMD_CG_SUPPORT_VCE_MGCG;
 		adev->pg_flags = 0;
 		adev->external_rev_id = adev->rev_id + 0x5A;
 		break;
 	case CHIP_POLARIS10:
-		adev->cg_flags = 0;
+		adev->cg_flags = AMD_CG_SUPPORT_UVD_MGCG |
+			AMD_CG_SUPPORT_VCE_MGCG;
 		adev->pg_flags = 0;
 		adev->external_rev_id = adev->rev_id + 0x50;
 		break;
@@ -1910,7 +1250,7 @@ static int vi_common_set_powergating_state(void *handle,
 	return 0;
 }
 
-const struct amd_ip_funcs vi_common_ip_funcs = {
+static const struct amd_ip_funcs vi_common_ip_funcs = {
 	.name = "vi_common",
 	.early_init = vi_common_early_init,
 	.late_init = NULL,
@@ -1927,3 +1267,110 @@ const struct amd_ip_funcs vi_common_ip_funcs = {
 	.set_powergating_state = vi_common_set_powergating_state,
 };
 
+static const struct amdgpu_ip_block_version vi_common_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_COMMON,
+	.major = 1,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &vi_common_ip_funcs,
+};
+
+int vi_set_ip_blocks(struct amdgpu_device *adev)
+{
+	switch (adev->asic_type) {
+	case CHIP_TOPAZ:
+		/* topaz has no DCE, UVD, VCE */
+		amdgpu_ip_block_add(adev, &vi_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v7_4_ip_block);
+		amdgpu_ip_block_add(adev, &iceland_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
+		amdgpu_ip_block_add(adev, &sdma_v2_4_ip_block);
+		break;
+	case CHIP_FIJI:
+		amdgpu_ip_block_add(adev, &vi_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v8_5_ip_block);
+		amdgpu_ip_block_add(adev, &tonga_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		else
+			amdgpu_ip_block_add(adev, &dce_v10_1_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
+		amdgpu_ip_block_add(adev, &sdma_v3_0_ip_block);
+		amdgpu_ip_block_add(adev, &uvd_v6_0_ip_block);
+		amdgpu_ip_block_add(adev, &vce_v3_0_ip_block);
+		break;
+	case CHIP_TONGA:
+		amdgpu_ip_block_add(adev, &vi_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v8_0_ip_block);
+		amdgpu_ip_block_add(adev, &tonga_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		else
+			amdgpu_ip_block_add(adev, &dce_v10_0_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
+		amdgpu_ip_block_add(adev, &sdma_v3_0_ip_block);
+		amdgpu_ip_block_add(adev, &uvd_v5_0_ip_block);
+		amdgpu_ip_block_add(adev, &vce_v3_0_ip_block);
+		break;
+	case CHIP_POLARIS11:
+	case CHIP_POLARIS10:
+		amdgpu_ip_block_add(adev, &vi_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v8_1_ip_block);
+		amdgpu_ip_block_add(adev, &tonga_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		else
+			amdgpu_ip_block_add(adev, &dce_v11_2_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
+		amdgpu_ip_block_add(adev, &sdma_v3_1_ip_block);
+		amdgpu_ip_block_add(adev, &uvd_v6_3_ip_block);
+		amdgpu_ip_block_add(adev, &vce_v3_4_ip_block);
+		break;
+	case CHIP_CARRIZO:
+		amdgpu_ip_block_add(adev, &vi_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v8_0_ip_block);
+		amdgpu_ip_block_add(adev, &cz_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		else
+			amdgpu_ip_block_add(adev, &dce_v11_0_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
+		amdgpu_ip_block_add(adev, &sdma_v3_0_ip_block);
+		amdgpu_ip_block_add(adev, &uvd_v6_0_ip_block);
+		amdgpu_ip_block_add(adev, &vce_v3_1_ip_block);
+#if defined(CONFIG_DRM_AMD_ACP)
+		amdgpu_ip_block_add(adev, &acp_ip_block);
+#endif
+		break;
+	case CHIP_STONEY:
+		amdgpu_ip_block_add(adev, &vi_common_ip_block);
+		amdgpu_ip_block_add(adev, &gmc_v8_0_ip_block);
+		amdgpu_ip_block_add(adev, &cz_ih_ip_block);
+		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		if (adev->enable_virtual_display)
+			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+		else
+			amdgpu_ip_block_add(adev, &dce_v11_0_ip_block);
+		amdgpu_ip_block_add(adev, &gfx_v8_1_ip_block);
+		amdgpu_ip_block_add(adev, &sdma_v3_0_ip_block);
+		amdgpu_ip_block_add(adev, &uvd_v6_2_ip_block);
+		amdgpu_ip_block_add(adev, &vce_v3_4_ip_block);
+#if defined(CONFIG_DRM_AMD_ACP)
+		amdgpu_ip_block_add(adev, &acp_ip_block);
+#endif
+		break;
+	default:
+		/* FIXME: not supported yet */
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.h b/drivers/gpu/drm/amd/amdgpu/vi.h
index 502094042462..575d7aed5d32 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.h
+++ b/drivers/gpu/drm/amd/amdgpu/vi.h
@@ -24,8 +24,6 @@
 #ifndef __VI_H__
 #define __VI_H__
 
-extern const struct amd_ip_funcs vi_common_ip_funcs;
-
 void vi_srbm_select(struct amdgpu_device *adev,
 		    u32 me, u32 pipe, u32 queue, u32 vmid);
 int vi_set_ip_blocks(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index bec8125bceb0..d1986276dbbd 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -84,6 +84,29 @@ enum amd_powergating_state {
 	AMD_PG_STATE_UNGATE,
 };
 
+struct amd_vce_state {
+	/* vce clocks */
+	u32 evclk;
+	u32 ecclk;
+	/* gpu clocks */
+	u32 sclk;
+	u32 mclk;
+	u8 clk_idx;
+	u8 pstate;
+};
+
+
+#define AMD_MAX_VCE_LEVELS 6
+
+enum amd_vce_level {
+	AMD_VCE_LEVEL_AC_ALL = 0,     /* AC, All cases */
+	AMD_VCE_LEVEL_DC_EE = 1,      /* DC, entropy encoding */
+	AMD_VCE_LEVEL_DC_LL_LOW = 2,  /* DC, low latency queue, res <= 720 */
+	AMD_VCE_LEVEL_DC_LL_HIGH = 3, /* DC, low latency queue, 1080 >= res > 720 */
+	AMD_VCE_LEVEL_DC_GP_LOW = 4,  /* DC, general purpose queue, res <= 720 */
+	AMD_VCE_LEVEL_DC_GP_HIGH = 5, /* DC, general purpose queue, 1080 >= res > 720 */
+};
+
 /* CG flags */
 #define AMD_CG_SUPPORT_GFX_MGCG			(1 << 0)
 #define AMD_CG_SUPPORT_GFX_MGLS			(1 << 1)
diff --git a/drivers/gpu/drm/amd/include/asic_reg/bif/bif_3_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/bif/bif_3_0_d.h
new file mode 100644
index 000000000000..7138fbf7256a
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/bif/bif_3_0_d.h
@@ -0,0 +1,661 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef BIF_3_0_D_H
+#define BIF_3_0_D_H
+
+#define ixPB0_DFT_DEBUG_CTRL_REG0 0x1300C
+#define ixPB0_DFT_JIT_INJ_REG0 0x13000
+#define ixPB0_DFT_JIT_INJ_REG1 0x13004
+#define ixPB0_DFT_JIT_INJ_REG2 0x13008
+#define ixPB0_GLB_CTRL_REG0 0x10004
+#define ixPB0_GLB_CTRL_REG1 0x10008
+#define ixPB0_GLB_CTRL_REG2 0x1000C
+#define ixPB0_GLB_CTRL_REG3 0x10010
+#define ixPB0_GLB_CTRL_REG4 0x10014
+#define ixPB0_GLB_CTRL_REG5 0x10018
+#define ixPB0_GLB_OVRD_REG0 0x10030
+#define ixPB0_GLB_OVRD_REG1 0x10034
+#define ixPB0_GLB_OVRD_REG2 0x10038
+#define ixPB0_GLB_SCI_STAT_OVRD_REG0 0x1001C
+#define ixPB0_GLB_SCI_STAT_OVRD_REG1 0x10020
+#define ixPB0_GLB_SCI_STAT_OVRD_REG2 0x10024
+#define ixPB0_GLB_SCI_STAT_OVRD_REG3 0x10028
+#define ixPB0_GLB_SCI_STAT_OVRD_REG4 0x1002C
+#define ixPB0_HW_DEBUG 0x12004
+#define ixPB0_PIF_CNTL 0x0010
+#define ixPB0_PIF_CNTL2 0x0014
+#define ixPB0_PIF_HW_DEBUG 0x0002
+#define ixPB0_PIF_PAIRING 0x0011
+#define ixPB0_PIF_PDNB_OVERRIDE_0 0x0020
+#define ixPB0_PIF_PDNB_OVERRIDE_10 0x0032
+#define ixPB0_PIF_PDNB_OVERRIDE_1 0x0021
+#define ixPB0_PIF_PDNB_OVERRIDE_11 0x0033
+#define ixPB0_PIF_PDNB_OVERRIDE_12 0x0034
+#define ixPB0_PIF_PDNB_OVERRIDE_13 0x0035
+#define ixPB0_PIF_PDNB_OVERRIDE_14 0x0036
+#define ixPB0_PIF_PDNB_OVERRIDE_15 0x0037
+#define ixPB0_PIF_PDNB_OVERRIDE_2 0x0022
+#define ixPB0_PIF_PDNB_OVERRIDE_3 0x0023
+#define ixPB0_PIF_PDNB_OVERRIDE_4 0x0024
+#define ixPB0_PIF_PDNB_OVERRIDE_5 0x0025
+#define ixPB0_PIF_PDNB_OVERRIDE_6 0x0026
+#define ixPB0_PIF_PDNB_OVERRIDE_7 0x0027
+#define ixPB0_PIF_PDNB_OVERRIDE_8 0x0030
+#define ixPB0_PIF_PDNB_OVERRIDE_9 0x0031
+#define ixPB0_PIF_PWRDOWN_0 0x0012
+#define ixPB0_PIF_PWRDOWN_1 0x0013
+#define ixPB0_PIF_PWRDOWN_2 0x0017
+#define ixPB0_PIF_PWRDOWN_3 0x0018
+#define ixPB0_PIF_SC_CTL 0x0016
+#define ixPB0_PIF_SCRATCH 0x0001
+#define ixPB0_PIF_SEQ_STATUS_0 0x0028
+#define ixPB0_PIF_SEQ_STATUS_10 0x003A
+#define ixPB0_PIF_SEQ_STATUS_1 0x0029
+#define ixPB0_PIF_SEQ_STATUS_11 0x003B
+#define ixPB0_PIF_SEQ_STATUS_12 0x003C
+#define ixPB0_PIF_SEQ_STATUS_13 0x003D
+#define ixPB0_PIF_SEQ_STATUS_14 0x003E
+#define ixPB0_PIF_SEQ_STATUS_15 0x003F
+#define ixPB0_PIF_SEQ_STATUS_2 0x002A
+#define ixPB0_PIF_SEQ_STATUS_3 0x002B
+#define ixPB0_PIF_SEQ_STATUS_4 0x002C
+#define ixPB0_PIF_SEQ_STATUS_5 0x002D
+#define ixPB0_PIF_SEQ_STATUS_6 0x002E
+#define ixPB0_PIF_SEQ_STATUS_7 0x002F
+#define ixPB0_PIF_SEQ_STATUS_8 0x0038
+#define ixPB0_PIF_SEQ_STATUS_9 0x0039
+#define ixPB0_PIF_TXPHYSTATUS 0x0015
+#define ixPB0_PLL_LC0_CTRL_REG0 0x14480
+#define ixPB0_PLL_LC0_OVRD_REG0 0x14490
+#define ixPB0_PLL_LC0_OVRD_REG1 0x14494
+#define ixPB0_PLL_LC0_SCI_STAT_OVRD_REG0 0x14500
+#define ixPB0_PLL_LC1_SCI_STAT_OVRD_REG0 0x14504
+#define ixPB0_PLL_LC2_SCI_STAT_OVRD_REG0 0x14508
+#define ixPB0_PLL_LC3_SCI_STAT_OVRD_REG0 0x1450C
+#define ixPB0_PLL_RO0_CTRL_REG0 0x14440
+#define ixPB0_PLL_RO0_OVRD_REG0 0x14450
+#define ixPB0_PLL_RO0_OVRD_REG1 0x14454
+#define ixPB0_PLL_RO0_SCI_STAT_OVRD_REG0 0x14460
+#define ixPB0_PLL_RO1_SCI_STAT_OVRD_REG0 0x14464
+#define ixPB0_PLL_RO2_SCI_STAT_OVRD_REG0 0x14468
+#define ixPB0_PLL_RO3_SCI_STAT_OVRD_REG0 0x1446C
+#define ixPB0_PLL_RO_GLB_CTRL_REG0 0x14000
+#define ixPB0_PLL_RO_GLB_OVRD_REG0 0x14010
+#define ixPB0_RX_GLB_CTRL_REG0 0x16000
+#define ixPB0_RX_GLB_CTRL_REG1 0x16004
+#define ixPB0_RX_GLB_CTRL_REG2 0x16008
+#define ixPB0_RX_GLB_CTRL_REG3 0x1600C
+#define ixPB0_RX_GLB_CTRL_REG4 0x16010
+#define ixPB0_RX_GLB_CTRL_REG5 0x16014
+#define ixPB0_RX_GLB_CTRL_REG6 0x16018
+#define ixPB0_RX_GLB_CTRL_REG7 0x1601C
+#define ixPB0_RX_GLB_CTRL_REG8 0x16020
+#define ixPB0_RX_GLB_OVRD_REG0 0x16030
+#define ixPB0_RX_GLB_OVRD_REG1 0x16034
+#define ixPB0_RX_GLB_SCI_STAT_OVRD_REG0 0x16028
+#define ixPB0_RX_LANE0_CTRL_REG0 0x16440
+#define ixPB0_RX_LANE0_SCI_STAT_OVRD_REG0 0x16448
+#define ixPB0_RX_LANE10_CTRL_REG0 0x17500
+#define ixPB0_RX_LANE10_SCI_STAT_OVRD_REG0 0x17508
+#define ixPB0_RX_LANE11_CTRL_REG0 0x17600
+#define ixPB0_RX_LANE11_SCI_STAT_OVRD_REG0 0x17608
+#define ixPB0_RX_LANE12_CTRL_REG0 0x17840
+#define ixPB0_RX_LANE12_SCI_STAT_OVRD_REG0 0x17848
+#define ixPB0_RX_LANE13_CTRL_REG0 0x17880
+#define ixPB0_RX_LANE13_SCI_STAT_OVRD_REG0 0x17888
+#define ixPB0_RX_LANE14_CTRL_REG0 0x17900
+#define ixPB0_RX_LANE14_SCI_STAT_OVRD_REG0 0x17908
+#define ixPB0_RX_LANE15_CTRL_REG0 0x17A00
+#define ixPB0_RX_LANE15_SCI_STAT_OVRD_REG0 0x17A08
+#define ixPB0_RX_LANE1_CTRL_REG0 0x16480
+#define ixPB0_RX_LANE1_SCI_STAT_OVRD_REG0 0x16488
+#define ixPB0_RX_LANE2_CTRL_REG0 0x16500
+#define ixPB0_RX_LANE2_SCI_STAT_OVRD_REG0 0x16508
+#define ixPB0_RX_LANE3_CTRL_REG0 0x16600
+#define ixPB0_RX_LANE3_SCI_STAT_OVRD_REG0 0x16608
+#define ixPB0_RX_LANE4_CTRL_REG0 0x16800
+#define ixPB0_RX_LANE4_SCI_STAT_OVRD_REG0 0x16848
+#define ixPB0_RX_LANE5_CTRL_REG0 0x16880
+#define ixPB0_RX_LANE5_SCI_STAT_OVRD_REG0 0x16888
+#define ixPB0_RX_LANE6_CTRL_REG0 0x16900
+#define ixPB0_RX_LANE6_SCI_STAT_OVRD_REG0 0x16908
+#define ixPB0_RX_LANE7_CTRL_REG0 0x16A00
+#define ixPB0_RX_LANE7_SCI_STAT_OVRD_REG0 0x16A08
+#define ixPB0_RX_LANE8_CTRL_REG0 0x17440
+#define ixPB0_RX_LANE8_SCI_STAT_OVRD_REG0 0x17448
+#define ixPB0_RX_LANE9_CTRL_REG0 0x17480
+#define ixPB0_RX_LANE9_SCI_STAT_OVRD_REG0 0x17488
+#define ixPB0_STRAP_GLB_REG0 0x12020
+#define ixPB0_STRAP_PLL_REG0 0x12030
+#define ixPB0_STRAP_RX_REG0 0x12028
+#define ixPB0_STRAP_RX_REG1 0x1202C
+#define ixPB0_STRAP_TX_REG0 0x12024
+#define ixPB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0 0x18014
+#define ixPB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1 0x18018
+#define ixPB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2 0x1801C
+#define ixPB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3 0x18020
+#define ixPB0_TX_GLB_CTRL_REG0 0x18000
+#define ixPB0_TX_GLB_LANE_SKEW_CTRL 0x18004
+#define ixPB0_TX_GLB_OVRD_REG0 0x18030
+#define ixPB0_TX_GLB_OVRD_REG1 0x18034
+#define ixPB0_TX_GLB_OVRD_REG2 0x18038
+#define ixPB0_TX_GLB_OVRD_REG3 0x1803C
+#define ixPB0_TX_GLB_OVRD_REG4 0x18040
+#define ixPB0_TX_GLB_SCI_STAT_OVRD_REG0 0x18010
+#define ixPB0_TX_LANE0_CTRL_REG0 0x18440
+#define ixPB0_TX_LANE0_OVRD_REG0 0x18444
+#define ixPB0_TX_LANE0_SCI_STAT_OVRD_REG0 0x18448
+#define ixPB0_TX_LANE10_CTRL_REG0 0x19500
+#define ixPB0_TX_LANE10_OVRD_REG0 0x19504
+#define ixPB0_TX_LANE10_SCI_STAT_OVRD_REG0 0x19508
+#define ixPB0_TX_LANE11_CTRL_REG0 0x19600
+#define ixPB0_TX_LANE11_OVRD_REG0 0x19604
+#define ixPB0_TX_LANE11_SCI_STAT_OVRD_REG0 0x19608
+#define ixPB0_TX_LANE12_CTRL_REG0 0x19840
+#define ixPB0_TX_LANE12_OVRD_REG0 0x19844
+#define ixPB0_TX_LANE12_SCI_STAT_OVRD_REG0 0x19848
+#define ixPB0_TX_LANE13_CTRL_REG0 0x19880
+#define ixPB0_TX_LANE13_OVRD_REG0 0x19884
+#define ixPB0_TX_LANE13_SCI_STAT_OVRD_REG0 0x19888
+#define ixPB0_TX_LANE14_CTRL_REG0 0x19900
+#define ixPB0_TX_LANE14_OVRD_REG0 0x19904
+#define ixPB0_TX_LANE14_SCI_STAT_OVRD_REG0 0x19908
+#define ixPB0_TX_LANE15_CTRL_REG0 0x19A00
+#define ixPB0_TX_LANE15_OVRD_REG0 0x19A04
+#define ixPB0_TX_LANE15_SCI_STAT_OVRD_REG0 0x19A08
+#define ixPB0_TX_LANE1_CTRL_REG0 0x18480
+#define ixPB0_TX_LANE1_OVRD_REG0 0x18484
+#define ixPB0_TX_LANE1_SCI_STAT_OVRD_REG0 0x18488
+#define ixPB0_TX_LANE2_CTRL_REG0 0x18500
+#define ixPB0_TX_LANE2_OVRD_REG0 0x18504
+#define ixPB0_TX_LANE2_SCI_STAT_OVRD_REG0 0x18508
+#define ixPB0_TX_LANE3_CTRL_REG0 0x18600
+#define ixPB0_TX_LANE3_OVRD_REG0 0x18604
+#define ixPB0_TX_LANE3_SCI_STAT_OVRD_REG0 0x18608
+#define ixPB0_TX_LANE4_CTRL_REG0 0x18840
+#define ixPB0_TX_LANE4_OVRD_REG0 0x18844
+#define ixPB0_TX_LANE4_SCI_STAT_OVRD_REG0 0x18848
+#define ixPB0_TX_LANE5_CTRL_REG0 0x18880
+#define ixPB0_TX_LANE5_OVRD_REG0 0x18884
+#define ixPB0_TX_LANE5_SCI_STAT_OVRD_REG0 0x18888
+#define ixPB0_TX_LANE6_CTRL_REG0 0x18900
+#define ixPB0_TX_LANE6_OVRD_REG0 0x18904
+#define ixPB0_TX_LANE6_SCI_STAT_OVRD_REG0 0x18908
+#define ixPB0_TX_LANE7_CTRL_REG0 0x18A00
+#define ixPB0_TX_LANE7_OVRD_REG0 0x18A04
+#define ixPB0_TX_LANE7_SCI_STAT_OVRD_REG0 0x18A08
+#define ixPB0_TX_LANE8_CTRL_REG0 0x19440
+#define ixPB0_TX_LANE8_OVRD_REG0 0x19444
+#define ixPB0_TX_LANE8_SCI_STAT_OVRD_REG0 0x19448
+#define ixPB0_TX_LANE9_CTRL_REG0 0x19480
+#define ixPB0_TX_LANE9_OVRD_REG0 0x19484
+#define ixPB0_TX_LANE9_SCI_STAT_OVRD_REG0 0x19488
+#define ixPB1_DFT_DEBUG_CTRL_REG0 0x1300C
+#define ixPB1_DFT_JIT_INJ_REG0 0x13000
+#define ixPB1_DFT_JIT_INJ_REG1 0x13004
+#define ixPB1_DFT_JIT_INJ_REG2 0x13008
+#define ixPB1_GLB_CTRL_REG0 0x10004
+#define ixPB1_GLB_CTRL_REG1 0x10008
+#define ixPB1_GLB_CTRL_REG2 0x1000C
+#define ixPB1_GLB_CTRL_REG3 0x10010
+#define ixPB1_GLB_CTRL_REG4 0x10014
+#define ixPB1_GLB_CTRL_REG5 0x10018
+#define ixPB1_GLB_OVRD_REG0 0x10030
+#define ixPB1_GLB_OVRD_REG1 0x10034
+#define ixPB1_GLB_OVRD_REG2 0x10038
+#define ixPB1_GLB_SCI_STAT_OVRD_REG0 0x1001C
+#define ixPB1_GLB_SCI_STAT_OVRD_REG1 0x10020
+#define ixPB1_GLB_SCI_STAT_OVRD_REG2 0x10024
+#define ixPB1_GLB_SCI_STAT_OVRD_REG3 0x10028
+#define ixPB1_GLB_SCI_STAT_OVRD_REG4 0x1002C
+#define ixPB1_HW_DEBUG 0x12004
+#define ixPB1_PIF_CNTL 0x0010
+#define ixPB1_PIF_CNTL2 0x0014
+#define ixPB1_PIF_HW_DEBUG 0x0002
+#define ixPB1_PIF_PAIRING 0x0011
+#define ixPB1_PIF_PDNB_OVERRIDE_0 0x0020
+#define ixPB1_PIF_PDNB_OVERRIDE_10 0x0032
+#define ixPB1_PIF_PDNB_OVERRIDE_1 0x0021
+#define ixPB1_PIF_PDNB_OVERRIDE_11 0x0033
+#define ixPB1_PIF_PDNB_OVERRIDE_12 0x0034
+#define ixPB1_PIF_PDNB_OVERRIDE_13 0x0035
+#define ixPB1_PIF_PDNB_OVERRIDE_14 0x0036
+#define ixPB1_PIF_PDNB_OVERRIDE_15 0x0037
+#define ixPB1_PIF_PDNB_OVERRIDE_2 0x0022
+#define ixPB1_PIF_PDNB_OVERRIDE_3 0x0023
+#define ixPB1_PIF_PDNB_OVERRIDE_4 0x0024
+#define ixPB1_PIF_PDNB_OVERRIDE_5 0x0025
+#define ixPB1_PIF_PDNB_OVERRIDE_6 0x0026
+#define ixPB1_PIF_PDNB_OVERRIDE_7 0x0027
+#define ixPB1_PIF_PDNB_OVERRIDE_8 0x0030
+#define ixPB1_PIF_PDNB_OVERRIDE_9 0x0031
+#define ixPB1_PIF_PWRDOWN_0 0x0012
+#define ixPB1_PIF_PWRDOWN_1 0x0013
+#define ixPB1_PIF_PWRDOWN_2 0x0017
+#define ixPB1_PIF_PWRDOWN_3 0x0018
+#define ixPB1_PIF_SC_CTL 0x0016
+#define ixPB1_PIF_SCRATCH 0x0001
+#define ixPB1_PIF_SEQ_STATUS_0 0x0028
+#define ixPB1_PIF_SEQ_STATUS_10 0x003A
+#define ixPB1_PIF_SEQ_STATUS_1 0x0029
+#define ixPB1_PIF_SEQ_STATUS_11 0x003B
+#define ixPB1_PIF_SEQ_STATUS_12 0x003C
+#define ixPB1_PIF_SEQ_STATUS_13 0x003D
+#define ixPB1_PIF_SEQ_STATUS_14 0x003E
+#define ixPB1_PIF_SEQ_STATUS_15 0x003F
+#define ixPB1_PIF_SEQ_STATUS_2 0x002A
+#define ixPB1_PIF_SEQ_STATUS_3 0x002B
+#define ixPB1_PIF_SEQ_STATUS_4 0x002C
+#define ixPB1_PIF_SEQ_STATUS_5 0x002D
+#define ixPB1_PIF_SEQ_STATUS_6 0x002E
+#define ixPB1_PIF_SEQ_STATUS_7 0x002F
+#define ixPB1_PIF_SEQ_STATUS_8 0x0038
+#define ixPB1_PIF_SEQ_STATUS_9 0x0039
+#define ixPB1_PIF_TXPHYSTATUS 0x0015
+#define ixPB1_PLL_LC0_CTRL_REG0 0x14480
+#define ixPB1_PLL_LC0_OVRD_REG0 0x14490
+#define ixPB1_PLL_LC0_OVRD_REG1 0x14494
+#define ixPB1_PLL_LC0_SCI_STAT_OVRD_REG0 0x14500
+#define ixPB1_PLL_LC1_SCI_STAT_OVRD_REG0 0x14504
+#define ixPB1_PLL_LC2_SCI_STAT_OVRD_REG0 0x14508
+#define ixPB1_PLL_LC3_SCI_STAT_OVRD_REG0 0x1450C
+#define ixPB1_PLL_RO0_CTRL_REG0 0x14440
+#define ixPB1_PLL_RO0_OVRD_REG0 0x14450
+#define ixPB1_PLL_RO0_OVRD_REG1 0x14454
+#define ixPB1_PLL_RO0_SCI_STAT_OVRD_REG0 0x14460
+#define ixPB1_PLL_RO1_SCI_STAT_OVRD_REG0 0x14464
+#define ixPB1_PLL_RO2_SCI_STAT_OVRD_REG0 0x14468
+#define ixPB1_PLL_RO3_SCI_STAT_OVRD_REG0 0x1446C
+#define ixPB1_PLL_RO_GLB_CTRL_REG0 0x14000
+#define ixPB1_PLL_RO_GLB_OVRD_REG0 0x14010
+#define ixPB1_RX_GLB_CTRL_REG0 0x16000
+#define ixPB1_RX_GLB_CTRL_REG1 0x16004
+#define ixPB1_RX_GLB_CTRL_REG2 0x16008
+#define ixPB1_RX_GLB_CTRL_REG3 0x1600C
+#define ixPB1_RX_GLB_CTRL_REG4 0x16010
+#define ixPB1_RX_GLB_CTRL_REG5 0x16014
+#define ixPB1_RX_GLB_CTRL_REG6 0x16018
+#define ixPB1_RX_GLB_CTRL_REG7 0x1601C
+#define ixPB1_RX_GLB_CTRL_REG8 0x16020
+#define ixPB1_RX_GLB_OVRD_REG0 0x16030
+#define ixPB1_RX_GLB_OVRD_REG1 0x16034
+#define ixPB1_RX_GLB_SCI_STAT_OVRD_REG0 0x16028
+#define ixPB1_RX_LANE0_CTRL_REG0 0x16440
+#define ixPB1_RX_LANE0_SCI_STAT_OVRD_REG0 0x16448
+#define ixPB1_RX_LANE10_CTRL_REG0 0x17500
+#define ixPB1_RX_LANE10_SCI_STAT_OVRD_REG0 0x17508
+#define ixPB1_RX_LANE11_CTRL_REG0 0x17600
+#define ixPB1_RX_LANE11_SCI_STAT_OVRD_REG0 0x17608
+#define ixPB1_RX_LANE12_CTRL_REG0 0x17840
+#define ixPB1_RX_LANE12_SCI_STAT_OVRD_REG0 0x17848
+#define ixPB1_RX_LANE13_CTRL_REG0 0x17880
+#define ixPB1_RX_LANE13_SCI_STAT_OVRD_REG0 0x17888
+#define ixPB1_RX_LANE14_CTRL_REG0 0x17900
+#define ixPB1_RX_LANE14_SCI_STAT_OVRD_REG0 0x17908
+#define ixPB1_RX_LANE15_CTRL_REG0 0x17A00
+#define ixPB1_RX_LANE15_SCI_STAT_OVRD_REG0 0x17A08
+#define ixPB1_RX_LANE1_CTRL_REG0 0x16480
+#define ixPB1_RX_LANE1_SCI_STAT_OVRD_REG0 0x16488
+#define ixPB1_RX_LANE2_CTRL_REG0 0x16500
+#define ixPB1_RX_LANE2_SCI_STAT_OVRD_REG0 0x16508
+#define ixPB1_RX_LANE3_CTRL_REG0 0x16600
+#define ixPB1_RX_LANE3_SCI_STAT_OVRD_REG0 0x16608
+#define ixPB1_RX_LANE4_CTRL_REG0 0x16800
+#define ixPB1_RX_LANE4_SCI_STAT_OVRD_REG0 0x16848
+#define ixPB1_RX_LANE5_CTRL_REG0 0x16880
+#define ixPB1_RX_LANE5_SCI_STAT_OVRD_REG0 0x16888
+#define ixPB1_RX_LANE6_CTRL_REG0 0x16900
+#define ixPB1_RX_LANE6_SCI_STAT_OVRD_REG0 0x16908
+#define ixPB1_RX_LANE7_CTRL_REG0 0x16A00
+#define ixPB1_RX_LANE7_SCI_STAT_OVRD_REG0 0x16A08
+#define ixPB1_RX_LANE8_CTRL_REG0 0x17440
+#define ixPB1_RX_LANE8_SCI_STAT_OVRD_REG0 0x17448
+#define ixPB1_RX_LANE9_CTRL_REG0 0x17480
+#define ixPB1_RX_LANE9_SCI_STAT_OVRD_REG0 0x17488
+#define ixPB1_STRAP_GLB_REG0 0x12020
+#define ixPB1_STRAP_PLL_REG0 0x12030
+#define ixPB1_STRAP_RX_REG0 0x12028
+#define ixPB1_STRAP_RX_REG1 0x1202C
+#define ixPB1_STRAP_TX_REG0 0x12024
+#define ixPB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0 0x18014
+#define ixPB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1 0x18018
+#define ixPB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2 0x1801C
+#define ixPB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3 0x18020
+#define ixPB1_TX_GLB_CTRL_REG0 0x18000
+#define ixPB1_TX_GLB_LANE_SKEW_CTRL 0x18004
+#define ixPB1_TX_GLB_OVRD_REG0 0x18030
+#define ixPB1_TX_GLB_OVRD_REG1 0x18034
+#define ixPB1_TX_GLB_OVRD_REG2 0x18038
+#define ixPB1_TX_GLB_OVRD_REG3 0x1803C
+#define ixPB1_TX_GLB_OVRD_REG4 0x18040
+#define ixPB1_TX_GLB_SCI_STAT_OVRD_REG0 0x18010
+#define ixPB1_TX_LANE0_CTRL_REG0 0x18440
+#define ixPB1_TX_LANE0_OVRD_REG0 0x18444
+#define ixPB1_TX_LANE0_SCI_STAT_OVRD_REG0 0x18448
+#define ixPB1_TX_LANE10_CTRL_REG0 0x19500
+#define ixPB1_TX_LANE10_OVRD_REG0 0x19504
+#define ixPB1_TX_LANE10_SCI_STAT_OVRD_REG0 0x19508
+#define ixPB1_TX_LANE11_CTRL_REG0 0x19600
+#define ixPB1_TX_LANE11_OVRD_REG0 0x19604
+#define ixPB1_TX_LANE11_SCI_STAT_OVRD_REG0 0x19608
+#define ixPB1_TX_LANE12_CTRL_REG0 0x19840
+#define ixPB1_TX_LANE12_OVRD_REG0 0x19844
+#define ixPB1_TX_LANE12_SCI_STAT_OVRD_REG0 0x19848
+#define ixPB1_TX_LANE13_CTRL_REG0 0x19880
+#define ixPB1_TX_LANE13_OVRD_REG0 0x19884
+#define ixPB1_TX_LANE13_SCI_STAT_OVRD_REG0 0x19888
+#define ixPB1_TX_LANE14_CTRL_REG0 0x19900
+#define ixPB1_TX_LANE14_OVRD_REG0 0x19904
+#define ixPB1_TX_LANE14_SCI_STAT_OVRD_REG0 0x19908
+#define ixPB1_TX_LANE15_CTRL_REG0 0x19A00
+#define ixPB1_TX_LANE15_OVRD_REG0 0x19A04
+#define ixPB1_TX_LANE15_SCI_STAT_OVRD_REG0 0x19A08
+#define ixPB1_TX_LANE1_CTRL_REG0 0x18480
+#define ixPB1_TX_LANE1_OVRD_REG0 0x18484
+#define ixPB1_TX_LANE1_SCI_STAT_OVRD_REG0 0x18488
+#define ixPB1_TX_LANE2_CTRL_REG0 0x18500
+#define ixPB1_TX_LANE2_OVRD_REG0 0x18504
+#define ixPB1_TX_LANE2_SCI_STAT_OVRD_REG0 0x18508
+#define ixPB1_TX_LANE3_CTRL_REG0 0x18600
+#define ixPB1_TX_LANE3_OVRD_REG0 0x18604
+#define ixPB1_TX_LANE3_SCI_STAT_OVRD_REG0 0x18608
+#define ixPB1_TX_LANE4_CTRL_REG0 0x18840
+#define ixPB1_TX_LANE4_OVRD_REG0 0x18844
+#define ixPB1_TX_LANE4_SCI_STAT_OVRD_REG0 0x18848
+#define ixPB1_TX_LANE5_CTRL_REG0 0x18880
+#define ixPB1_TX_LANE5_OVRD_REG0 0x18884
+#define ixPB1_TX_LANE5_SCI_STAT_OVRD_REG0 0x18888
+#define ixPB1_TX_LANE6_CTRL_REG0 0x18900
+#define ixPB1_TX_LANE6_OVRD_REG0 0x18904
+#define ixPB1_TX_LANE6_SCI_STAT_OVRD_REG0 0x18908
+#define ixPB1_TX_LANE7_CTRL_REG0 0x18A00
+#define ixPB1_TX_LANE7_OVRD_REG0 0x18A04
+#define ixPB1_TX_LANE7_SCI_STAT_OVRD_REG0 0x18A08
+#define ixPB1_TX_LANE8_CTRL_REG0 0x19440
+#define ixPB1_TX_LANE8_OVRD_REG0 0x19444
+#define ixPB1_TX_LANE8_SCI_STAT_OVRD_REG0 0x19448
+#define ixPB1_TX_LANE9_CTRL_REG0 0x19480
+#define ixPB1_TX_LANE9_OVRD_REG0 0x19484
+#define ixPB1_TX_LANE9_SCI_STAT_OVRD_REG0 0x19488
+#define ixPCIE_BUS_CNTL 0x0021
+#define ixPCIE_CFG_CNTL 0x003C
+#define ixPCIE_CI_CNTL 0x0020
+#define ixPCIE_CNTL 0x0010
+#define ixPCIE_CNTL2 0x001C
+#define ixPCIE_CONFIG_CNTL 0x0011
+#define ixPCIE_DEBUG_CNTL 0x0012
+#define ixPCIE_ERR_CNTL 0x006A
+#define ixPCIE_F0_DPA_CAP 0x00E0
+#define ixPCIE_F0_DPA_CNTL 0x00E5
+#define ixPCIE_F0_DPA_LATENCY_INDICATOR 0x00E4
+#define ixPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0 0x00E7
+#define ixPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1 0x00E8
+#define ixPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2 0x00E9
+#define ixPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3 0x00EA
+#define ixPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4 0x00EB
+#define ixPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5 0x00EC
+#define ixPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6 0x00ED
+#define ixPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7 0x00EE
+#define ixPCIE_FC_CPL 0x0062
+#define ixPCIE_FC_NP 0x0061
+#define ixPCIE_FC_P 0x0060
+#define ixPCIE_HW_DEBUG 0x0002
+#define ixPCIE_I2C_REG_ADDR_EXPAND 0x003A
+#define ixPCIE_I2C_REG_DATA 0x003B
+#define ixPCIE_INT_CNTL 0x001A
+#define ixPCIE_INT_STATUS 0x001B
+#define ixPCIE_LC_BEST_EQ_SETTINGS 0x00B9
+#define ixPCIE_LC_BW_CHANGE_CNTL 0x00B2
+#define ixPCIE_LC_CDR_CNTL 0x00B3
+#define ixPCIE_LC_CNTL 0x00A0
+#define ixPCIE_LC_CNTL2 0x00B1
+#define ixPCIE_LC_CNTL3 0x00B5
+#define ixPCIE_LC_CNTL4 0x00B6
+#define ixPCIE_LC_CNTL5 0x00B7
+#define ixPCIE_LC_FORCE_COEFF 0x00B8
+#define ixPCIE_LC_FORCE_EQ_REQ_COEFF 0x00BA
+#define ixPCIE_LC_LANE_CNTL 0x00B4
+#define ixPCIE_LC_LINK_WIDTH_CNTL 0x00A2
+#define ixPCIE_LC_N_FTS_CNTL 0x00A3
+#define ixPCIE_LC_SPEED_CNTL 0x00A4
+#define ixPCIE_LC_STATE0 0x00A5
+#define ixPCIE_LC_STATE10 0x0026
+#define ixPCIE_LC_STATE1 0x00A6
+#define ixPCIE_LC_STATE11 0x0027
+#define ixPCIE_LC_STATE2 0x00A7
+#define ixPCIE_LC_STATE3 0x00A8
+#define ixPCIE_LC_STATE4 0x00A9
+#define ixPCIE_LC_STATE5 0x00AA
+#define ixPCIE_LC_STATE6 0x0022
+#define ixPCIE_LC_STATE7 0x0023
+#define ixPCIE_LC_STATE8 0x0024
+#define ixPCIE_LC_STATE9 0x0025
+#define ixPCIE_LC_STATUS1 0x0028
+#define ixPCIE_LC_STATUS2 0x0029
+#define ixPCIE_LC_TRAINING_CNTL 0x00A1
+#define ixPCIE_P_BUF_STATUS 0x0041
+#define ixPCIE_P_CNTL 0x0040
+#define ixPCIE_P_DECODER_STATUS 0x0042
+#define ixPCIE_PERF_CNTL_EVENT0_PORT_SEL 0x0093
+#define ixPCIE_PERF_CNTL_EVENT1_PORT_SEL 0x0094
+#define ixPCIE_PERF_CNTL_MST_C_CLK 0x0087
+#define ixPCIE_PERF_CNTL_MST_R_CLK 0x0084
+#define ixPCIE_PERF_CNTL_SLV_NS_C_CLK 0x0090
+#define ixPCIE_PERF_CNTL_SLV_R_CLK 0x008A
+#define ixPCIE_PERF_CNTL_SLV_S_C_CLK 0x008D
+#define ixPCIE_PERF_CNTL_TXCLK 0x0081
+#define ixPCIE_PERF_CNTL_TXCLK2 0x0095
+#define ixPCIE_PERF_COUNT0_MST_C_CLK 0x0088
+#define ixPCIE_PERF_COUNT0_MST_R_CLK 0x0085
+#define ixPCIE_PERF_COUNT0_SLV_NS_C_CLK 0x0091
+#define ixPCIE_PERF_COUNT0_SLV_R_CLK 0x008B
+#define ixPCIE_PERF_COUNT0_SLV_S_C_CLK 0x008E
+#define ixPCIE_PERF_COUNT0_TXCLK 0x0082
+#define ixPCIE_PERF_COUNT0_TXCLK2 0x0096
+#define ixPCIE_PERF_COUNT1_MST_C_CLK 0x0089
+#define ixPCIE_PERF_COUNT1_MST_R_CLK 0x0086
+#define ixPCIE_PERF_COUNT1_SLV_NS_C_CLK 0x0092
+#define ixPCIE_PERF_COUNT1_SLV_R_CLK 0x008C
+#define ixPCIE_PERF_COUNT1_SLV_S_C_CLK 0x008F
+#define ixPCIE_PERF_COUNT1_TXCLK 0x0083
+#define ixPCIE_PERF_COUNT1_TXCLK2 0x0097
+#define ixPCIE_PERF_COUNT_CNTL 0x0080
+#define ixPCIEP_HW_DEBUG 0x0002
+#define ixPCIE_P_MISC_STATUS 0x0043
+#define ixPCIEP_PORT_CNTL 0x0010
+#define ixPCIE_P_PORT_LANE_STATUS 0x0050
+#define ixPCIE_PRBS_CLR 0x00C8
+#define ixPCIE_PRBS_ERRCNT_0 0x00D0
+#define ixPCIE_PRBS_ERRCNT_10 0x00DA
+#define ixPCIE_PRBS_ERRCNT_1 0x00D1
+#define ixPCIE_PRBS_ERRCNT_11 0x00DB
+#define ixPCIE_PRBS_ERRCNT_12 0x00DC
+#define ixPCIE_PRBS_ERRCNT_13 0x00DD
+#define ixPCIE_PRBS_ERRCNT_14 0x00DE
+#define ixPCIE_PRBS_ERRCNT_15 0x00DF
+#define ixPCIE_PRBS_ERRCNT_2 0x00D2
+#define ixPCIE_PRBS_ERRCNT_3 0x00D3
+#define ixPCIE_PRBS_ERRCNT_4 0x00D4
+#define ixPCIE_PRBS_ERRCNT_5 0x00D5
+#define ixPCIE_PRBS_ERRCNT_6 0x00D6
+#define ixPCIE_PRBS_ERRCNT_7 0x00D7
+#define ixPCIE_PRBS_ERRCNT_8 0x00D8
+#define ixPCIE_PRBS_ERRCNT_9 0x00D9
+#define ixPCIE_PRBS_FREERUN 0x00CB
+#define ixPCIE_PRBS_HI_BITCNT 0x00CF
+#define ixPCIE_PRBS_LO_BITCNT 0x00CE
+#define ixPCIE_PRBS_MISC 0x00CC
+#define ixPCIE_PRBS_STATUS1 0x00C9
+#define ixPCIE_PRBS_STATUS2 0x00CA
+#define ixPCIE_PRBS_USER_PATTERN 0x00CD
+#define ixPCIE_P_RCV_L0S_FTS_DET 0x0050
+#define ixPCIEP_RESERVED 0x0000
+#define ixPCIEP_SCRATCH 0x0001
+#define ixPCIEP_STRAP_LC 0x00C0
+#define ixPCIEP_STRAP_MISC 0x00C1
+#define ixPCIE_RESERVED 0x0000
+#define ixPCIE_RX_CNTL 0x0070
+#define ixPCIE_RX_CNTL2 0x001D
+#define ixPCIE_RX_CNTL3 0x0074
+#define ixPCIE_RX_CREDITS_ALLOCATED_CPL 0x0082
+#define ixPCIE_RX_CREDITS_ALLOCATED_NP 0x0081
+#define ixPCIE_RX_CREDITS_ALLOCATED_P 0x0080
+#define ixPCIE_RX_EXPECTED_SEQNUM 0x0071
+#define ixPCIE_RX_LAST_TLP0 0x0031
+#define ixPCIE_RX_LAST_TLP1 0x0032
+#define ixPCIE_RX_LAST_TLP2 0x0033
+#define ixPCIE_RX_LAST_TLP3 0x0034
+#define ixPCIE_RX_NUM_NAK 0x000E
+#define ixPCIE_RX_NUM_NAK_GENERATED 0x000F
+#define ixPCIE_RX_VENDOR_SPECIFIC 0x0072
+#define ixPCIE_SCRATCH 0x0001
+#define ixPCIE_STRAP_F0 0x00B0
+#define ixPCIE_STRAP_F1 0x00B1
+#define ixPCIE_STRAP_F2 0x00B2
+#define ixPCIE_STRAP_F3 0x00B3
+#define ixPCIE_STRAP_F4 0x00B4
+#define ixPCIE_STRAP_F5 0x00B5
+#define ixPCIE_STRAP_F6 0x00B6
+#define ixPCIE_STRAP_F7 0x00B7
+#define ixPCIE_STRAP_I2C_BD 0x00C4
+#define ixPCIE_STRAP_MISC 0x00C0
+#define ixPCIE_STRAP_MISC2 0x00C1
+#define ixPCIE_STRAP_PI 0x00C2
+#define ixPCIE_TX_ACK_LATENCY_LIMIT 0x0026
+#define ixPCIE_TX_CNTL 0x0020
+#define ixPCIE_TX_CREDITS_ADVT_CPL 0x0032
+#define ixPCIE_TX_CREDITS_ADVT_NP 0x0031
+#define ixPCIE_TX_CREDITS_ADVT_P 0x0030
+#define ixPCIE_TX_CREDITS_FCU_THRESHOLD 0x0037
+#define ixPCIE_TX_CREDITS_INIT_CPL 0x0035
+#define ixPCIE_TX_CREDITS_INIT_NP 0x0034
+#define ixPCIE_TX_CREDITS_INIT_P 0x0033
+#define ixPCIE_TX_CREDITS_STATUS 0x0036
+#define ixPCIE_TX_LAST_TLP0 0x0035
+#define ixPCIE_TX_LAST_TLP1 0x0036
+#define ixPCIE_TX_LAST_TLP2 0x0037
+#define ixPCIE_TX_LAST_TLP3 0x0038
+#define ixPCIE_TX_REPLAY 0x0025
+#define ixPCIE_TX_REQUESTER_ID 0x0021
+#define ixPCIE_TX_REQUEST_NUM_CNTL 0x0023
+#define ixPCIE_TX_SEQ 0x0024
+#define ixPCIE_TX_VENDOR_SPECIFIC 0x0022
+#define ixPCIE_WPR_CNTL 0x0030
+#define mmBACO_CNTL 0x14E5
+#define mmBF_ANA_ISO_CNTL 0x14C7
+#define mmBIF_BACO_DEBUG 0x14DF
+#define mmBIF_BACO_DEBUG_LATCH 0x14DC
+#define mmBIF_BACO_MSIC 0x14DE
+#define mmBIF_BUSNUM_CNTL1 0x1525
+#define mmBIF_BUSNUM_CNTL2 0x152B
+#define mmBIF_BUSNUM_LIST0 0x1526
+#define mmBIF_BUSNUM_LIST1 0x1527
+#define mmBIF_BUSY_DELAY_CNTR 0x1529
+#define mmBIF_CLK_PDWN_DELAY_TIMER 0x151F
+#define mmBIF_DEBUG_CNTL 0x151C
+#define mmBIF_DEBUG_MUX 0x151D
+#define mmBIF_DEBUG_OUT 0x151E
+#define mmBIF_DEVFUNCNUM_LIST0 0x14E8
+#define mmBIF_DEVFUNCNUM_LIST1 0x14E7
+#define mmBIF_FB_EN 0x1524
+#define mmBIF_FEATURES_CONTROL_MISC 0x14C2
+#define mmBIF_PERFCOUNTER0_RESULT 0x152D
+#define mmBIF_PERFCOUNTER1_RESULT 0x152E
+#define mmBIF_PERFMON_CNTL 0x152C
+#define mmBIF_PIF_TXCLK_SWITCH_TIMER 0x152F
+#define mmBIF_RESET_EN 0x1511
+#define mmBIF_SCRATCH0 0x150E
+#define mmBIF_SCRATCH1 0x150F
+#define mmBIF_SSA_DISP_LOWER 0x14D2
+#define mmBIF_SSA_DISP_UPPER 0x14D3
+#define mmBIF_SSA_GFX0_LOWER 0x14CA
+#define mmBIF_SSA_GFX0_UPPER 0x14CB
+#define mmBIF_SSA_GFX1_LOWER 0x14CC
+#define mmBIF_SSA_GFX1_UPPER 0x14CD
+#define mmBIF_SSA_GFX2_LOWER 0x14CE
+#define mmBIF_SSA_GFX2_UPPER 0x14CF
+#define mmBIF_SSA_GFX3_LOWER 0x14D0
+#define mmBIF_SSA_GFX3_UPPER 0x14D1
+#define mmBIF_SSA_MC_LOWER 0x14D4
+#define mmBIF_SSA_MC_UPPER 0x14D5
+#define mmBIF_SSA_PWR_STATUS 0x14C8
+#define mmBIF_XDMA_HI 0x14C1
+#define mmBIF_XDMA_LO 0x14C0
+#define mmBIOS_SCRATCH_0 0x05C9
+#define mmBIOS_SCRATCH_10 0x05D3
+#define mmBIOS_SCRATCH_1 0x05CA
+#define mmBIOS_SCRATCH_11 0x05D4
+#define mmBIOS_SCRATCH_12 0x05D5
+#define mmBIOS_SCRATCH_13 0x05D6
+#define mmBIOS_SCRATCH_14 0x05D7
+#define mmBIOS_SCRATCH_15 0x05D8
+#define mmBIOS_SCRATCH_2 0x05CB
+#define mmBIOS_SCRATCH_3 0x05CC
+#define mmBIOS_SCRATCH_4 0x05CD
+#define mmBIOS_SCRATCH_5 0x05CE
+#define mmBIOS_SCRATCH_6 0x05CF
+#define mmBIOS_SCRATCH_7 0x05D0
+#define mmBIOS_SCRATCH_8 0x05D1
+#define mmBIOS_SCRATCH_9 0x05D2
+#define mmBUS_CNTL 0x1508
+#define mmCAPTURE_HOST_BUSNUM 0x153C
+#define mmCLKREQB_PAD_CNTL 0x1521
+#define mmCONFIG_APER_SIZE 0x150C
+#define mmCONFIG_CNTL 0x1509
+#define mmCONFIG_F0_BASE 0x150B
+#define mmCONFIG_MEMSIZE 0x150A
+#define mmCONFIG_REG_APER_SIZE 0x150D
+#define mmHDP_MEM_COHERENCY_FLUSH_CNTL 0x1520
+#define mmHDP_REG_COHERENCY_FLUSH_CNTL 0x1528
+#define mmHOST_BUSNUM 0x153D
+#define mmHW_DEBUG 0x1515
+#define mmIMPCTL_RESET 0x14F5
+#define mmINTERRUPT_CNTL 0x151A
+#define mmINTERRUPT_CNTL2 0x151B
+#define mmMASTER_CREDIT_CNTL 0x1516
+#define mmMM_CFGREGS_CNTL 0x1513
+#define mmMM_DATA 0x0001
+#define mmMM_INDEX 0x0000
+#define mmMM_INDEX_HI 0x0006
+#define mmNEW_REFCLKB_TIMER 0x14EA
+#define mmNEW_REFCLKB_TIMER_1 0x14E9
+#define mmPCIE_DATA 0x000D
+#define mmPCIE_INDEX 0x000C
+#define mmPEER0_FB_OFFSET_HI 0x14F3
+#define mmPEER0_FB_OFFSET_LO 0x14F2
+#define mmPEER1_FB_OFFSET_HI 0x14F1
+#define mmPEER1_FB_OFFSET_LO 0x14F0
+#define mmPEER2_FB_OFFSET_HI 0x14EF
+#define mmPEER2_FB_OFFSET_LO 0x14EE
+#define mmPEER3_FB_OFFSET_HI 0x14ED
+#define mmPEER3_FB_OFFSET_LO 0x14EC
+#define mmPEER_REG_RANGE0 0x153E
+#define mmPEER_REG_RANGE1 0x153F
+#define mmSLAVE_HANG_ERROR 0x153B
+#define mmSLAVE_HANG_PROTECTION_CNTL 0x1536
+#define mmSLAVE_REQ_CREDIT_CNTL 0x1517
+#define mmSMBCLK_PAD_CNTL 0x1523
+#define mmSMBDAT_PAD_CNTL 0x1522
+#define mmSMBUS_BACO_DUMMY 0x14C6
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/bif/bif_3_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/bif/bif_3_0_sh_mask.h
new file mode 100644
index 000000000000..e94445acf3c6
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/bif/bif_3_0_sh_mask.h
@@ -0,0 +1,8127 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef BIF_3_0_SH_MASK_H
+#define BIF_3_0_SH_MASK_H
+
+#define BACO_CNTL__BACO_ANA_ISO_DIS_MASK 0x00000080L
+#define BACO_CNTL__BACO_ANA_ISO_DIS__SHIFT 0x00000007
+#define BACO_CNTL__BACO_BCLK_OFF_MASK 0x00000002L
+#define BACO_CNTL__BACO_BCLK_OFF__SHIFT 0x00000001
+#define BACO_CNTL__BACO_EN_MASK 0x00000001L
+#define BACO_CNTL__BACO_EN__SHIFT 0x00000000
+#define BACO_CNTL__BACO_HANG_PROTECTION_EN_MASK 0x00000020L
+#define BACO_CNTL__BACO_HANG_PROTECTION_EN__SHIFT 0x00000005
+#define BACO_CNTL__BACO_ISO_DIS_MASK 0x00000004L
+#define BACO_CNTL__BACO_ISO_DIS__SHIFT 0x00000002
+#define BACO_CNTL__BACO_MODE_MASK 0x00000040L
+#define BACO_CNTL__BACO_MODE__SHIFT 0x00000006
+#define BACO_CNTL__BACO_POWER_OFF_MASK 0x00000008L
+#define BACO_CNTL__BACO_POWER_OFF__SHIFT 0x00000003
+#define BACO_CNTL__BACO_RESET_EN_MASK 0x00000010L
+#define BACO_CNTL__BACO_RESET_EN__SHIFT 0x00000004
+#define BACO_CNTL__PWRGOOD_BF_MASK 0x00000200L
+#define BACO_CNTL__PWRGOOD_BF__SHIFT 0x00000009
+#define BACO_CNTL__PWRGOOD_DVO_MASK 0x00001000L
+#define BACO_CNTL__PWRGOOD_DVO__SHIFT 0x0000000c
+#define BACO_CNTL__PWRGOOD_GPIO_MASK 0x00000400L
+#define BACO_CNTL__PWRGOOD_GPIO__SHIFT 0x0000000a
+#define BACO_CNTL__PWRGOOD_MEM_MASK 0x00000800L
+#define BACO_CNTL__PWRGOOD_MEM__SHIFT 0x0000000b
+#define BACO_CNTL__RCU_BIF_CONFIG_DONE_MASK 0x00000100L
+#define BACO_CNTL__RCU_BIF_CONFIG_DONE__SHIFT 0x00000008
+#define BF_ANA_ISO_CNTL__BF_ANA_ISO_DIS_MASK_MASK 0x00000001L
+#define BF_ANA_ISO_CNTL__BF_ANA_ISO_DIS_MASK__SHIFT 0x00000000
+#define BF_ANA_ISO_CNTL__BF_VDDC_ISO_DIS_MASK_MASK 0x00000002L
+#define BF_ANA_ISO_CNTL__BF_VDDC_ISO_DIS_MASK__SHIFT 0x00000001
+#define BIF_BACO_DEBUG__BIF_BACO_SCANDUMP_FLG_MASK 0x00000001L
+#define BIF_BACO_DEBUG__BIF_BACO_SCANDUMP_FLG__SHIFT 0x00000000
+#define BIF_BACO_DEBUG_LATCH__BIF_BACO_LATCH_FLG_MASK 0x00000001L
+#define BIF_BACO_DEBUG_LATCH__BIF_BACO_LATCH_FLG__SHIFT 0x00000000
+#define BIF_BACO_MSIC__BIF_XTALIN_SEL_MASK 0x00000001L
+#define BIF_BACO_MSIC__BIF_XTALIN_SEL__SHIFT 0x00000000
+#define BIF_BUSNUM_CNTL1__ID_MASK_MASK 0x000000ffL
+#define BIF_BUSNUM_CNTL1__ID_MASK__SHIFT 0x00000000
+#define BIF_BUSNUM_CNTL2__AUTOUPDATE_EN_MASK 0x00000100L
+#define BIF_BUSNUM_CNTL2__AUTOUPDATE_EN__SHIFT 0x00000008
+#define BIF_BUSNUM_CNTL2__AUTOUPDATE_SEL_MASK 0x000000ffL
+#define BIF_BUSNUM_CNTL2__AUTOUPDATE_SEL__SHIFT 0x00000000
+#define BIF_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH_MASK 0x00020000L
+#define BIF_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH__SHIFT 0x00000011
+#define BIF_BUSNUM_CNTL2__HDPREG_CNTL_MASK 0x00010000L
+#define BIF_BUSNUM_CNTL2__HDPREG_CNTL__SHIFT 0x00000010
+#define BIF_BUSNUM_LIST0__ID0_MASK 0x000000ffL
+#define BIF_BUSNUM_LIST0__ID0__SHIFT 0x00000000
+#define BIF_BUSNUM_LIST0__ID1_MASK 0x0000ff00L
+#define BIF_BUSNUM_LIST0__ID1__SHIFT 0x00000008
+#define BIF_BUSNUM_LIST0__ID2_MASK 0x00ff0000L
+#define BIF_BUSNUM_LIST0__ID2__SHIFT 0x00000010
+#define BIF_BUSNUM_LIST0__ID3_MASK 0xff000000L
+#define BIF_BUSNUM_LIST0__ID3__SHIFT 0x00000018
+#define BIF_BUSNUM_LIST1__ID4_MASK 0x000000ffL
+#define BIF_BUSNUM_LIST1__ID4__SHIFT 0x00000000
+#define BIF_BUSNUM_LIST1__ID5_MASK 0x0000ff00L
+#define BIF_BUSNUM_LIST1__ID5__SHIFT 0x00000008
+#define BIF_BUSNUM_LIST1__ID6_MASK 0x00ff0000L
+#define BIF_BUSNUM_LIST1__ID6__SHIFT 0x00000010
+#define BIF_BUSNUM_LIST1__ID7_MASK 0xff000000L
+#define BIF_BUSNUM_LIST1__ID7__SHIFT 0x00000018
+#define BIF_BUSY_DELAY_CNTR__DELAY_CNT_MASK 0x0000003fL
+#define BIF_BUSY_DELAY_CNTR__DELAY_CNT__SHIFT 0x00000000
+#define BIF_CLK_PDWN_DELAY_TIMER__TIMER_MASK 0x000003ffL
+#define BIF_CLK_PDWN_DELAY_TIMER__TIMER__SHIFT 0x00000000
+#define BIF_DEBUG_CNTL__DEBUG_BYTESEL_BLK1_MASK 0x00000010L
+#define BIF_DEBUG_CNTL__DEBUG_BYTESEL_BLK1__SHIFT 0x00000004
+#define BIF_DEBUG_CNTL__DEBUG_BYTESEL_BLK2_MASK 0x00000020L
+#define BIF_DEBUG_CNTL__DEBUG_BYTESEL_BLK2__SHIFT 0x00000005
+#define BIF_DEBUG_CNTL__DEBUG_EN_MASK 0x00000001L
+#define BIF_DEBUG_CNTL__DEBUG_EN__SHIFT 0x00000000
+#define BIF_DEBUG_CNTL__DEBUG_IDSEL_BLK1_MASK 0x00001f00L
+#define BIF_DEBUG_CNTL__DEBUG_IDSEL_BLK1__SHIFT 0x00000008
+#define BIF_DEBUG_CNTL__DEBUG_IDSEL_BLK2_MASK 0x001f0000L
+#define BIF_DEBUG_CNTL__DEBUG_IDSEL_BLK2__SHIFT 0x00000010
+#define BIF_DEBUG_CNTL__DEBUG_IDSEL_XSP_MASK 0x01000000L
+#define BIF_DEBUG_CNTL__DEBUG_IDSEL_XSP__SHIFT 0x00000018
+#define BIF_DEBUG_CNTL__DEBUG_MULTIBLOCKEN_MASK 0x00000002L
+#define BIF_DEBUG_CNTL__DEBUG_MULTIBLOCKEN__SHIFT 0x00000001
+#define BIF_DEBUG_CNTL__DEBUG_OUT_EN_MASK 0x00000004L
+#define BIF_DEBUG_CNTL__DEBUG_OUT_EN__SHIFT 0x00000002
+#define BIF_DEBUG_CNTL__DEBUG_PAD_SEL_MASK 0x00000008L
+#define BIF_DEBUG_CNTL__DEBUG_PAD_SEL__SHIFT 0x00000003
+#define BIF_DEBUG_CNTL__DEBUG_SWAP_MASK 0x00000080L
+#define BIF_DEBUG_CNTL__DEBUG_SWAP__SHIFT 0x00000007
+#define BIF_DEBUG_CNTL__DEBUG_SYNC_CLKSEL_MASK 0xc0000000L
+#define BIF_DEBUG_CNTL__DEBUG_SYNC_CLKSEL__SHIFT 0x0000001e
+#define BIF_DEBUG_CNTL__DEBUG_SYNC_EN_MASK 0x00000040L
+#define BIF_DEBUG_CNTL__DEBUG_SYNC_EN__SHIFT 0x00000006
+#define BIF_DEBUG_MUX__DEBUG_MUX_BLK1_MASK 0x0000003fL
+#define BIF_DEBUG_MUX__DEBUG_MUX_BLK1__SHIFT 0x00000000
+#define BIF_DEBUG_MUX__DEBUG_MUX_BLK2_MASK 0x00003f00L
+#define BIF_DEBUG_MUX__DEBUG_MUX_BLK2__SHIFT 0x00000008
+#define BIF_DEBUG_OUT__DEBUG_OUTPUT_MASK 0x0001ffffL
+#define BIF_DEBUG_OUT__DEBUG_OUTPUT__SHIFT 0x00000000
+#define BIF_DEVFUNCNUM_LIST0__DEVFUNC_ID0_MASK 0x000000ffL
+#define BIF_DEVFUNCNUM_LIST0__DEVFUNC_ID0__SHIFT 0x00000000
+#define BIF_DEVFUNCNUM_LIST0__DEVFUNC_ID1_MASK 0x0000ff00L
+#define BIF_DEVFUNCNUM_LIST0__DEVFUNC_ID1__SHIFT 0x00000008
+#define BIF_DEVFUNCNUM_LIST0__DEVFUNC_ID2_MASK 0x00ff0000L
+#define BIF_DEVFUNCNUM_LIST0__DEVFUNC_ID2__SHIFT 0x00000010
+#define BIF_DEVFUNCNUM_LIST0__DEVFUNC_ID3_MASK 0xff000000L
+#define BIF_DEVFUNCNUM_LIST0__DEVFUNC_ID3__SHIFT 0x00000018
+#define BIF_DEVFUNCNUM_LIST1__DEVFUNC_ID4_MASK 0x000000ffL
+#define BIF_DEVFUNCNUM_LIST1__DEVFUNC_ID4__SHIFT 0x00000000
+#define BIF_DEVFUNCNUM_LIST1__DEVFUNC_ID5_MASK 0x0000ff00L
+#define BIF_DEVFUNCNUM_LIST1__DEVFUNC_ID5__SHIFT 0x00000008
+#define BIF_DEVFUNCNUM_LIST1__DEVFUNC_ID6_MASK 0x00ff0000L
+#define BIF_DEVFUNCNUM_LIST1__DEVFUNC_ID6__SHIFT 0x00000010
+#define BIF_DEVFUNCNUM_LIST1__DEVFUNC_ID7_MASK 0xff000000L
+#define BIF_DEVFUNCNUM_LIST1__DEVFUNC_ID7__SHIFT 0x00000018
+#define BIF_FB_EN__FB_READ_EN_MASK 0x00000001L
+#define BIF_FB_EN__FB_READ_EN__SHIFT 0x00000000
+#define BIF_FB_EN__FB_WRITE_EN_MASK 0x00000002L
+#define BIF_FB_EN__FB_WRITE_EN__SHIFT 0x00000001
+#define BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS_MASK 0x00000008L
+#define BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS__SHIFT 0x00000003
+#define BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS_MASK 0x00000004L
+#define BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS__SHIFT 0x00000002
+#define BIF_FEATURES_CONTROL_MISC__IGNORE_BE_CHECK_GASKET_COMB_DIS_MASK 0x00000100L
+#define BIF_FEATURES_CONTROL_MISC__IGNORE_BE_CHECK_GASKET_COMB_DIS__SHIFT 0x00000008
+#define BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS_MASK 0x00000001L
+#define BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS__SHIFT 0x00000000
+#define BIF_FEATURES_CONTROL_MISC__PLL_SWITCH_IMPCTL_CAL_DONE_DIS_MASK 0x00000080L
+#define BIF_FEATURES_CONTROL_MISC__PLL_SWITCH_IMPCTL_CAL_DONE_DIS__SHIFT 0x00000007
+#define BIF_FEATURES_CONTROL_MISC__POST_PSN_ONLY_PKT_REPORT_UR_ALL_DIS_MASK 0x00000020L
+#define BIF_FEATURES_CONTROL_MISC__POST_PSN_ONLY_PKT_REPORT_UR_ALL_DIS__SHIFT 0x00000005
+#define BIF_FEATURES_CONTROL_MISC__POST_PSN_ONLY_PKT_REPORT_UR_PART_DIS_MASK 0x00000040L
+#define BIF_FEATURES_CONTROL_MISC__POST_PSN_ONLY_PKT_REPORT_UR_PART_DIS__SHIFT 0x00000006
+#define BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS_MASK 0x00000002L
+#define BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS__SHIFT 0x00000001
+#define BIF_FEATURES_CONTROL_MISC__UR_PSN_PKT_REPORT_POISON_DIS_MASK 0x00000010L
+#define BIF_FEATURES_CONTROL_MISC__UR_PSN_PKT_REPORT_POISON_DIS__SHIFT 0x00000004
+#define BIF_PERFCOUNTER0_RESULT__PERFCOUNTER_RESULT_MASK 0xffffffffL
+#define BIF_PERFCOUNTER0_RESULT__PERFCOUNTER_RESULT__SHIFT 0x00000000
+#define BIF_PERFCOUNTER1_RESULT__PERFCOUNTER_RESULT_MASK 0xffffffffL
+#define BIF_PERFCOUNTER1_RESULT__PERFCOUNTER_RESULT__SHIFT 0x00000000
+#define BIF_PERFMON_CNTL__PERFCOUNTER_EN_MASK 0x00000001L
+#define BIF_PERFMON_CNTL__PERFCOUNTER_EN__SHIFT 0x00000000
+#define BIF_PERFMON_CNTL__PERFCOUNTER_RESET0_MASK 0x00000002L
+#define BIF_PERFMON_CNTL__PERFCOUNTER_RESET0__SHIFT 0x00000001
+#define BIF_PERFMON_CNTL__PERFCOUNTER_RESET1_MASK 0x00000004L
+#define BIF_PERFMON_CNTL__PERFCOUNTER_RESET1__SHIFT 0x00000002
+#define BIF_PERFMON_CNTL__PERF_SEL0_MASK 0x00001f00L
+#define BIF_PERFMON_CNTL__PERF_SEL0__SHIFT 0x00000008
+#define BIF_PERFMON_CNTL__PERF_SEL1_MASK 0x0003e000L
+#define BIF_PERFMON_CNTL__PERF_SEL1__SHIFT 0x0000000d
+#define BIF_PIF_TXCLK_SWITCH_TIMER__PLL0_ACK_TIMER_MASK 0x00000007L
+#define BIF_PIF_TXCLK_SWITCH_TIMER__PLL0_ACK_TIMER__SHIFT 0x00000000
+#define BIF_PIF_TXCLK_SWITCH_TIMER__PLL1_ACK_TIMER_MASK 0x00000038L
+#define BIF_PIF_TXCLK_SWITCH_TIMER__PLL1_ACK_TIMER__SHIFT 0x00000003
+#define BIF_PIF_TXCLK_SWITCH_TIMER__PLL_SWITCH_TIMER_MASK 0x000003c0L
+#define BIF_PIF_TXCLK_SWITCH_TIMER__PLL_SWITCH_TIMER__SHIFT 0x00000006
+#define BIF_RESET_EN__BIF_COR_RESET_EN_MASK 0x00400000L
+#define BIF_RESET_EN__BIF_COR_RESET_EN__SHIFT 0x00000016
+#define BIF_RESET_EN__CFG_RESET_EN_MASK 0x00000040L
+#define BIF_RESET_EN__CFG_RESET_EN__SHIFT 0x00000006
+#define BIF_RESET_EN__CFG_RESET_PULSE_WIDTH_MASK 0x0003f000L
+#define BIF_RESET_EN__CFG_RESET_PULSE_WIDTH__SHIFT 0x0000000c
+#define BIF_RESET_EN__COR_RESET_EN_MASK 0x00000008L
+#define BIF_RESET_EN__COR_RESET_EN__SHIFT 0x00000003
+#define BIF_RESET_EN__DRV_RESET_DELAY_SEL_MASK 0x000c0000L
+#define BIF_RESET_EN__DRV_RESET_DELAY_SEL__SHIFT 0x00000012
+#define BIF_RESET_EN__DRV_RESET_EN_MASK 0x00000080L
+#define BIF_RESET_EN__DRV_RESET_EN__SHIFT 0x00000007
+#define BIF_RESET_EN__FUNC0_FLR_EN_MASK 0x00800000L
+#define BIF_RESET_EN__FUNC0_FLR_EN__SHIFT 0x00000017
+#define BIF_RESET_EN__FUNC0_RESET_DELAY_SEL_MASK 0x0c000000L
+#define BIF_RESET_EN__FUNC0_RESET_DELAY_SEL__SHIFT 0x0000001a
+#define BIF_RESET_EN__FUNC1_FLR_EN_MASK 0x01000000L
+#define BIF_RESET_EN__FUNC1_FLR_EN__SHIFT 0x00000018
+#define BIF_RESET_EN__FUNC1_RESET_DELAY_SEL_MASK 0x30000000L
+#define BIF_RESET_EN__FUNC1_RESET_DELAY_SEL__SHIFT 0x0000001c
+#define BIF_RESET_EN__FUNC2_FLR_EN_MASK 0x02000000L
+#define BIF_RESET_EN__FUNC2_FLR_EN__SHIFT 0x00000019
+#define BIF_RESET_EN__FUNC2_RESET_DELAY_SEL_MASK 0xc0000000L
+#define BIF_RESET_EN__FUNC2_RESET_DELAY_SEL__SHIFT 0x0000001e
+#define BIF_RESET_EN__HOT_RESET_EN_MASK 0x00000200L
+#define BIF_RESET_EN__HOT_RESET_EN__SHIFT 0x00000009
+#define BIF_RESET_EN__LINK_DISABLE_RESET_EN_MASK 0x00000400L
+#define BIF_RESET_EN__LINK_DISABLE_RESET_EN__SHIFT 0x0000000a
+#define BIF_RESET_EN__LINK_DOWN_RESET_EN_MASK 0x00000800L
+#define BIF_RESET_EN__LINK_DOWN_RESET_EN__SHIFT 0x0000000b
+#define BIF_RESET_EN__PHY_RESET_EN_MASK 0x00000004L
+#define BIF_RESET_EN__PHY_RESET_EN__SHIFT 0x00000002
+#define BIF_RESET_EN__PIF_RSTB_EN_MASK 0x00100000L
+#define BIF_RESET_EN__PIF_RSTB_EN__SHIFT 0x00000014
+#define BIF_RESET_EN__PIF_STRAP_ALLVALID_EN_MASK 0x00200000L
+#define BIF_RESET_EN__PIF_STRAP_ALLVALID_EN__SHIFT 0x00000015
+#define BIF_RESET_EN__REG_RESET_EN_MASK 0x00000010L
+#define BIF_RESET_EN__REG_RESET_EN__SHIFT 0x00000004
+#define BIF_RESET_EN__RESET_CFGREG_ONLY_EN_MASK 0x00000100L
+#define BIF_RESET_EN__RESET_CFGREG_ONLY_EN__SHIFT 0x00000008
+#define BIF_RESET_EN__SOFT_RST_MODE_MASK 0x00000002L
+#define BIF_RESET_EN__SOFT_RST_MODE__SHIFT 0x00000001
+#define BIF_RESET_EN__STY_RESET_EN_MASK 0x00000020L
+#define BIF_RESET_EN__STY_RESET_EN__SHIFT 0x00000005
+#define BIF_SCRATCH0__BIF_SCRATCH0_MASK 0xffffffffL
+#define BIF_SCRATCH0__BIF_SCRATCH0__SHIFT 0x00000000
+#define BIF_SCRATCH1__BIF_SCRATCH1_MASK 0xffffffffL
+#define BIF_SCRATCH1__BIF_SCRATCH1__SHIFT 0x00000000
+#define BIF_SSA_DISP_LOWER__SSA_DISP_LOWER_MASK 0x0003fffcL
+#define BIF_SSA_DISP_LOWER__SSA_DISP_LOWER__SHIFT 0x00000002
+#define BIF_SSA_DISP_LOWER__SSA_DISP_REG_CMP_EN_MASK 0x40000000L
+#define BIF_SSA_DISP_LOWER__SSA_DISP_REG_CMP_EN__SHIFT 0x0000001e
+#define BIF_SSA_DISP_LOWER__SSA_DISP_REG_STALL_EN_MASK 0x80000000L
+#define BIF_SSA_DISP_LOWER__SSA_DISP_REG_STALL_EN__SHIFT 0x0000001f
+#define BIF_SSA_DISP_UPPER__SSA_DISP_UPPER_MASK 0x0003fffcL
+#define BIF_SSA_DISP_UPPER__SSA_DISP_UPPER__SHIFT 0x00000002
+#define BIF_SSA_GFX0_LOWER__SSA_GFX0_LOWER_MASK 0x0003fffcL
+#define BIF_SSA_GFX0_LOWER__SSA_GFX0_LOWER__SHIFT 0x00000002
+#define BIF_SSA_GFX0_LOWER__SSA_GFX0_REG_CMP_EN_MASK 0x40000000L
+#define BIF_SSA_GFX0_LOWER__SSA_GFX0_REG_CMP_EN__SHIFT 0x0000001e
+#define BIF_SSA_GFX0_LOWER__SSA_GFX0_REG_STALL_EN_MASK 0x80000000L
+#define BIF_SSA_GFX0_LOWER__SSA_GFX0_REG_STALL_EN__SHIFT 0x0000001f
+#define BIF_SSA_GFX0_UPPER__SSA_GFX0_UPPER_MASK 0x0003fffcL
+#define BIF_SSA_GFX0_UPPER__SSA_GFX0_UPPER__SHIFT 0x00000002
+#define BIF_SSA_GFX1_LOWER__SSA_GFX1_LOWER_MASK 0x0003fffcL
+#define BIF_SSA_GFX1_LOWER__SSA_GFX1_LOWER__SHIFT 0x00000002
+#define BIF_SSA_GFX1_LOWER__SSA_GFX1_REG_CMP_EN_MASK 0x40000000L
+#define BIF_SSA_GFX1_LOWER__SSA_GFX1_REG_CMP_EN__SHIFT 0x0000001e
+#define BIF_SSA_GFX1_LOWER__SSA_GFX1_REG_STALL_EN_MASK 0x80000000L
+#define BIF_SSA_GFX1_LOWER__SSA_GFX1_REG_STALL_EN__SHIFT 0x0000001f
+#define BIF_SSA_GFX1_UPPER__SSA_GFX1_UPPER_MASK 0x0003fffcL
+#define BIF_SSA_GFX1_UPPER__SSA_GFX1_UPPER__SHIFT 0x00000002
+#define BIF_SSA_GFX2_LOWER__SSA_GFX2_LOWER_MASK 0x0003fffcL
+#define BIF_SSA_GFX2_LOWER__SSA_GFX2_LOWER__SHIFT 0x00000002
+#define BIF_SSA_GFX2_LOWER__SSA_GFX2_REG_CMP_EN_MASK 0x40000000L
+#define BIF_SSA_GFX2_LOWER__SSA_GFX2_REG_CMP_EN__SHIFT 0x0000001e
+#define BIF_SSA_GFX2_LOWER__SSA_GFX2_REG_STALL_EN_MASK 0x80000000L
+#define BIF_SSA_GFX2_LOWER__SSA_GFX2_REG_STALL_EN__SHIFT 0x0000001f
+#define BIF_SSA_GFX2_UPPER__SSA_GFX2_UPPER_MASK 0x0003fffcL
+#define BIF_SSA_GFX2_UPPER__SSA_GFX2_UPPER__SHIFT 0x00000002
+#define BIF_SSA_GFX3_LOWER__SSA_GFX3_LOWER_MASK 0x0003fffcL
+#define BIF_SSA_GFX3_LOWER__SSA_GFX3_LOWER__SHIFT 0x00000002
+#define BIF_SSA_GFX3_LOWER__SSA_GFX3_REG_CMP_EN_MASK 0x40000000L
+#define BIF_SSA_GFX3_LOWER__SSA_GFX3_REG_CMP_EN__SHIFT 0x0000001e
+#define BIF_SSA_GFX3_LOWER__SSA_GFX3_REG_STALL_EN_MASK 0x80000000L
+#define BIF_SSA_GFX3_LOWER__SSA_GFX3_REG_STALL_EN__SHIFT 0x0000001f
+#define BIF_SSA_GFX3_UPPER__SSA_GFX3_UPPER_MASK 0x0003fffcL
+#define BIF_SSA_GFX3_UPPER__SSA_GFX3_UPPER__SHIFT 0x00000002
+#define BIF_SSA_MC_LOWER__SSA_MC_FB_STALL_EN_MASK 0x20000000L
+#define BIF_SSA_MC_LOWER__SSA_MC_FB_STALL_EN__SHIFT 0x0000001d
+#define BIF_SSA_MC_LOWER__SSA_MC_LOWER_MASK 0x0003fffcL
+#define BIF_SSA_MC_LOWER__SSA_MC_LOWER__SHIFT 0x00000002
+#define BIF_SSA_MC_LOWER__SSA_MC_REG_CMP_EN_MASK 0x40000000L
+#define BIF_SSA_MC_LOWER__SSA_MC_REG_CMP_EN__SHIFT 0x0000001e
+#define BIF_SSA_MC_LOWER__SSA_MC_REG_STALL_EN_MASK 0x80000000L
+#define BIF_SSA_MC_LOWER__SSA_MC_REG_STALL_EN__SHIFT 0x0000001f
+#define BIF_SSA_MC_UPPER__SSA_MC_UPPER_MASK 0x0003fffcL
+#define BIF_SSA_MC_UPPER__SSA_MC_UPPER__SHIFT 0x00000002
+#define BIF_SSA_PWR_STATUS__SSA_DISP_PWR_STATUS_MASK 0x00000002L
+#define BIF_SSA_PWR_STATUS__SSA_DISP_PWR_STATUS__SHIFT 0x00000001
+#define BIF_SSA_PWR_STATUS__SSA_GFX_PWR_STATUS_MASK 0x00000001L
+#define BIF_SSA_PWR_STATUS__SSA_GFX_PWR_STATUS__SHIFT 0x00000000
+#define BIF_SSA_PWR_STATUS__SSA_MC_PWR_STATUS_MASK 0x00000004L
+#define BIF_SSA_PWR_STATUS__SSA_MC_PWR_STATUS__SHIFT 0x00000002
+#define BIF_XDMA_HI__BIF_XDMA_UPPER_BOUND_MASK 0x1fffffffL
+#define BIF_XDMA_HI__BIF_XDMA_UPPER_BOUND__SHIFT 0x00000000
+#define BIF_XDMA_LO__BIF_XDMA_APER_EN_MASK 0x80000000L
+#define BIF_XDMA_LO__BIF_XDMA_APER_EN__SHIFT 0x0000001f
+#define BIF_XDMA_LO__BIF_XDMA_LOWER_BOUND_MASK 0x1fffffffL
+#define BIF_XDMA_LO__BIF_XDMA_LOWER_BOUND__SHIFT 0x00000000
+#define BIOS_SCRATCH_0__BIOS_SCRATCH_0_MASK 0xffffffffL
+#define BIOS_SCRATCH_0__BIOS_SCRATCH_0__SHIFT 0x00000000
+#define BIOS_SCRATCH_10__BIOS_SCRATCH_10_MASK 0xffffffffL
+#define BIOS_SCRATCH_10__BIOS_SCRATCH_10__SHIFT 0x00000000
+#define BIOS_SCRATCH_11__BIOS_SCRATCH_11_MASK 0xffffffffL
+#define BIOS_SCRATCH_11__BIOS_SCRATCH_11__SHIFT 0x00000000
+#define BIOS_SCRATCH_12__BIOS_SCRATCH_12_MASK 0xffffffffL
+#define BIOS_SCRATCH_12__BIOS_SCRATCH_12__SHIFT 0x00000000
+#define BIOS_SCRATCH_13__BIOS_SCRATCH_13_MASK 0xffffffffL
+#define BIOS_SCRATCH_13__BIOS_SCRATCH_13__SHIFT 0x00000000
+#define BIOS_SCRATCH_14__BIOS_SCRATCH_14_MASK 0xffffffffL
+#define BIOS_SCRATCH_14__BIOS_SCRATCH_14__SHIFT 0x00000000
+#define BIOS_SCRATCH_15__BIOS_SCRATCH_15_MASK 0xffffffffL
+#define BIOS_SCRATCH_15__BIOS_SCRATCH_15__SHIFT 0x00000000
+#define BIOS_SCRATCH_1__BIOS_SCRATCH_1_MASK 0xffffffffL
+#define BIOS_SCRATCH_1__BIOS_SCRATCH_1__SHIFT 0x00000000
+#define BIOS_SCRATCH_2__BIOS_SCRATCH_2_MASK 0xffffffffL
+#define BIOS_SCRATCH_2__BIOS_SCRATCH_2__SHIFT 0x00000000
+#define BIOS_SCRATCH_3__BIOS_SCRATCH_3_MASK 0xffffffffL
+#define BIOS_SCRATCH_3__BIOS_SCRATCH_3__SHIFT 0x00000000
+#define BIOS_SCRATCH_4__BIOS_SCRATCH_4_MASK 0xffffffffL
+#define BIOS_SCRATCH_4__BIOS_SCRATCH_4__SHIFT 0x00000000
+#define BIOS_SCRATCH_5__BIOS_SCRATCH_5_MASK 0xffffffffL
+#define BIOS_SCRATCH_5__BIOS_SCRATCH_5__SHIFT 0x00000000
+#define BIOS_SCRATCH_6__BIOS_SCRATCH_6_MASK 0xffffffffL
+#define BIOS_SCRATCH_6__BIOS_SCRATCH_6__SHIFT 0x00000000
+#define BIOS_SCRATCH_7__BIOS_SCRATCH_7_MASK 0xffffffffL
+#define BIOS_SCRATCH_7__BIOS_SCRATCH_7__SHIFT 0x00000000
+#define BIOS_SCRATCH_8__BIOS_SCRATCH_8_MASK 0xffffffffL
+#define BIOS_SCRATCH_8__BIOS_SCRATCH_8__SHIFT 0x00000000
+#define BIOS_SCRATCH_9__BIOS_SCRATCH_9_MASK 0xffffffffL
+#define BIOS_SCRATCH_9__BIOS_SCRATCH_9__SHIFT 0x00000000
+#define BUS_CNTL__BIF_ERR_RTR_BKPRESSURE_EN_MASK 0x00000100L
+#define BUS_CNTL__BIF_ERR_RTR_BKPRESSURE_EN__SHIFT 0x00000008
+#define BUS_CNTL__BIOS_ROM_DIS_MASK 0x00000002L
+#define BUS_CNTL__BIOS_ROM_DIS__SHIFT 0x00000001
+#define BUS_CNTL__BIOS_ROM_WRT_EN_MASK 0x00000001L
+#define BUS_CNTL__BIOS_ROM_WRT_EN__SHIFT 0x00000000
+#define BUS_CNTL__PMI_BM_DIS_MASK 0x00000010L
+#define BUS_CNTL__PMI_BM_DIS__SHIFT 0x00000004
+#define BUS_CNTL__PMI_INT_DIS_MASK 0x00000020L
+#define BUS_CNTL__PMI_INT_DIS__SHIFT 0x00000005
+#define BUS_CNTL__PMI_IO_DIS_MASK 0x00000004L
+#define BUS_CNTL__PMI_IO_DIS__SHIFT 0x00000002
+#define BUS_CNTL__PMI_MEM_DIS_MASK 0x00000008L
+#define BUS_CNTL__PMI_MEM_DIS__SHIFT 0x00000003
+#define BUS_CNTL__RD_STALL_IO_WR_MASK 0x00040000L
+#define BUS_CNTL__RD_STALL_IO_WR__SHIFT 0x00000012
+#define BUS_CNTL__SET_AZ_TC_MASK 0x00001c00L
+#define BUS_CNTL__SET_AZ_TC__SHIFT 0x0000000a
+#define BUS_CNTL__SET_MC_TC_MASK 0x0000e000L
+#define BUS_CNTL__SET_MC_TC__SHIFT 0x0000000d
+#define BUS_CNTL__VGA_MEM_COHERENCY_DIS_MASK 0x00000080L
+#define BUS_CNTL__VGA_MEM_COHERENCY_DIS__SHIFT 0x00000007
+#define BUS_CNTL__VGA_REG_COHERENCY_DIS_MASK 0x00000040L
+#define BUS_CNTL__VGA_REG_COHERENCY_DIS__SHIFT 0x00000006
+#define BUS_CNTL__ZERO_BE_RD_EN_MASK 0x00020000L
+#define BUS_CNTL__ZERO_BE_RD_EN__SHIFT 0x00000011
+#define BUS_CNTL__ZERO_BE_WR_EN_MASK 0x00010000L
+#define BUS_CNTL__ZERO_BE_WR_EN__SHIFT 0x00000010
+#define CAPTURE_HOST_BUSNUM__CHECK_EN_MASK 0x00000001L
+#define CAPTURE_HOST_BUSNUM__CHECK_EN__SHIFT 0x00000000
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_A_MASK 0x00000001L
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_A__SHIFT 0x00000000
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN_MASK 0x00001000L
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN__SHIFT 0x0000000c
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE_MASK 0x00000004L
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE__SHIFT 0x00000002
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN_MASK 0x00000800L
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN__SHIFT 0x0000000b
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL_MASK 0x00000002L
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL__SHIFT 0x00000001
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN_MASK 0x00000200L
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN__SHIFT 0x00000009
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0_MASK 0x00000020L
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0__SHIFT 0x00000005
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1_MASK 0x00000040L
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1__SHIFT 0x00000006
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2_MASK 0x00000080L
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2__SHIFT 0x00000007
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3_MASK 0x00000100L
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3__SHIFT 0x00000008
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE_MASK 0x00000018L
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE__SHIFT 0x00000003
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE_MASK 0x00000400L
+#define CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE__SHIFT 0x0000000a
+#define CONFIG_APER_SIZE__APER_SIZE_MASK 0xffffffffL
+#define CONFIG_APER_SIZE__APER_SIZE__SHIFT 0x00000000
+#define CONFIG_CNTL__CFG_VGA_RAM_EN_MASK 0x00000001L
+#define CONFIG_CNTL__CFG_VGA_RAM_EN__SHIFT 0x00000000
+#define CONFIG_CNTL__GENMO_MONO_ADDRESS_B_MASK 0x00000004L
+#define CONFIG_CNTL__GENMO_MONO_ADDRESS_B__SHIFT 0x00000002
+#define CONFIG_CNTL__GRPH_ADRSEL_MASK 0x00000018L
+#define CONFIG_CNTL__GRPH_ADRSEL__SHIFT 0x00000003
+#define CONFIG_CNTL__VGA_DIS_MASK 0x00000002L
+#define CONFIG_CNTL__VGA_DIS__SHIFT 0x00000001
+#define CONFIG_F0_BASE__F0_BASE_MASK 0xffffffffL
+#define CONFIG_F0_BASE__F0_BASE__SHIFT 0x00000000
+#define CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK 0xffffffffL
+#define CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT 0x00000000
+#define CONFIG_REG_APER_SIZE__REG_APER_SIZE_MASK 0x000fffffL
+#define CONFIG_REG_APER_SIZE__REG_APER_SIZE__SHIFT 0x00000000
+#define HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK 0x00000001L
+#define HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT 0x00000000
+#define HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK 0x00000001L
+#define HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT 0x00000000
+#define HOST_BUSNUM__HOST_ID_MASK 0x0000ffffL
+#define HOST_BUSNUM__HOST_ID__SHIFT 0x00000000
+#define HW_DEBUG__HW_00_DEBUG_MASK 0x00000001L
+#define HW_DEBUG__HW_00_DEBUG__SHIFT 0x00000000
+#define HW_DEBUG__HW_01_DEBUG_MASK 0x00000002L
+#define HW_DEBUG__HW_01_DEBUG__SHIFT 0x00000001
+#define HW_DEBUG__HW_02_DEBUG_MASK 0x00000004L
+#define HW_DEBUG__HW_02_DEBUG__SHIFT 0x00000002
+#define HW_DEBUG__HW_03_DEBUG_MASK 0x00000008L
+#define HW_DEBUG__HW_03_DEBUG__SHIFT 0x00000003
+#define HW_DEBUG__HW_04_DEBUG_MASK 0x00000010L
+#define HW_DEBUG__HW_04_DEBUG__SHIFT 0x00000004
+#define HW_DEBUG__HW_05_DEBUG_MASK 0x00000020L
+#define HW_DEBUG__HW_05_DEBUG__SHIFT 0x00000005
+#define HW_DEBUG__HW_06_DEBUG_MASK 0x00000040L
+#define HW_DEBUG__HW_06_DEBUG__SHIFT 0x00000006
+#define HW_DEBUG__HW_07_DEBUG_MASK 0x00000080L
+#define HW_DEBUG__HW_07_DEBUG__SHIFT 0x00000007
+#define HW_DEBUG__HW_08_DEBUG_MASK 0x00000100L
+#define HW_DEBUG__HW_08_DEBUG__SHIFT 0x00000008
+#define HW_DEBUG__HW_09_DEBUG_MASK 0x00000200L
+#define HW_DEBUG__HW_09_DEBUG__SHIFT 0x00000009
+#define HW_DEBUG__HW_10_DEBUG_MASK 0x00000400L
+#define HW_DEBUG__HW_10_DEBUG__SHIFT 0x0000000a
+#define HW_DEBUG__HW_11_DEBUG_MASK 0x00000800L
+#define HW_DEBUG__HW_11_DEBUG__SHIFT 0x0000000b
+#define HW_DEBUG__HW_12_DEBUG_MASK 0x00001000L
+#define HW_DEBUG__HW_12_DEBUG__SHIFT 0x0000000c
+#define HW_DEBUG__HW_13_DEBUG_MASK 0x00002000L
+#define HW_DEBUG__HW_13_DEBUG__SHIFT 0x0000000d
+#define HW_DEBUG__HW_14_DEBUG_MASK 0x00004000L
+#define HW_DEBUG__HW_14_DEBUG__SHIFT 0x0000000e
+#define HW_DEBUG__HW_15_DEBUG_MASK 0x00008000L
+#define HW_DEBUG__HW_15_DEBUG__SHIFT 0x0000000f
+#define HW_DEBUG__HW_16_DEBUG_MASK 0x00010000L
+#define HW_DEBUG__HW_16_DEBUG__SHIFT 0x00000010
+#define HW_DEBUG__HW_17_DEBUG_MASK 0x00020000L
+#define HW_DEBUG__HW_17_DEBUG__SHIFT 0x00000011
+#define HW_DEBUG__HW_18_DEBUG_MASK 0x00040000L
+#define HW_DEBUG__HW_18_DEBUG__SHIFT 0x00000012
+#define HW_DEBUG__HW_19_DEBUG_MASK 0x00080000L
+#define HW_DEBUG__HW_19_DEBUG__SHIFT 0x00000013
+#define HW_DEBUG__HW_20_DEBUG_MASK 0x00100000L
+#define HW_DEBUG__HW_20_DEBUG__SHIFT 0x00000014
+#define HW_DEBUG__HW_21_DEBUG_MASK 0x00200000L
+#define HW_DEBUG__HW_21_DEBUG__SHIFT 0x00000015
+#define HW_DEBUG__HW_22_DEBUG_MASK 0x00400000L
+#define HW_DEBUG__HW_22_DEBUG__SHIFT 0x00000016
+#define HW_DEBUG__HW_23_DEBUG_MASK 0x00800000L
+#define HW_DEBUG__HW_23_DEBUG__SHIFT 0x00000017
+#define HW_DEBUG__HW_24_DEBUG_MASK 0x01000000L
+#define HW_DEBUG__HW_24_DEBUG__SHIFT 0x00000018
+#define HW_DEBUG__HW_25_DEBUG_MASK 0x02000000L
+#define HW_DEBUG__HW_25_DEBUG__SHIFT 0x00000019
+#define HW_DEBUG__HW_26_DEBUG_MASK 0x04000000L
+#define HW_DEBUG__HW_26_DEBUG__SHIFT 0x0000001a
+#define HW_DEBUG__HW_27_DEBUG_MASK 0x08000000L
+#define HW_DEBUG__HW_27_DEBUG__SHIFT 0x0000001b
+#define HW_DEBUG__HW_28_DEBUG_MASK 0x10000000L
+#define HW_DEBUG__HW_28_DEBUG__SHIFT 0x0000001c
+#define HW_DEBUG__HW_29_DEBUG_MASK 0x20000000L
+#define HW_DEBUG__HW_29_DEBUG__SHIFT 0x0000001d
+#define HW_DEBUG__HW_30_DEBUG_MASK 0x40000000L
+#define HW_DEBUG__HW_30_DEBUG__SHIFT 0x0000001e
+#define HW_DEBUG__HW_31_DEBUG_MASK 0x80000000L
+#define HW_DEBUG__HW_31_DEBUG__SHIFT 0x0000001f
+#define IMPCTL_RESET__IMP_SW_RESET_MASK 0x00000001L
+#define IMPCTL_RESET__IMP_SW_RESET__SHIFT 0x00000000
+#define INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR_MASK 0xffffffffL
+#define INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR__SHIFT 0x00000000
+#define INTERRUPT_CNTL__GEN_GPIO_INT_EN_MASK 0x00001e00L
+#define INTERRUPT_CNTL__GEN_GPIO_INT_EN__SHIFT 0x00000009
+#define INTERRUPT_CNTL__GEN_IH_INT_EN_MASK 0x00000100L
+#define INTERRUPT_CNTL__GEN_IH_INT_EN__SHIFT 0x00000008
+#define INTERRUPT_CNTL__IH_DUMMY_RD_EN_MASK 0x00000002L
+#define INTERRUPT_CNTL__IH_DUMMY_RD_EN__SHIFT 0x00000001
+#define INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK 0x00000001L
+#define INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE__SHIFT 0x00000000
+#define INTERRUPT_CNTL__IH_INTR_DLY_CNTR_MASK 0x000000f0L
+#define INTERRUPT_CNTL__IH_INTR_DLY_CNTR__SHIFT 0x00000004
+#define INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK 0x00000008L
+#define INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN__SHIFT 0x00000003
+#define INTERRUPT_CNTL__SELECT_INT_GPIO_OUTPUT_MASK 0x00006000L
+#define INTERRUPT_CNTL__SELECT_INT_GPIO_OUTPUT__SHIFT 0x0000000d
+#define MASTER_CREDIT_CNTL__BIF_AZ_RDRET_CREDIT_MASK 0x003f0000L
+#define MASTER_CREDIT_CNTL__BIF_AZ_RDRET_CREDIT__SHIFT 0x00000010
+#define MASTER_CREDIT_CNTL__BIF_MC_RDRET_CREDIT_MASK 0x0000003fL
+#define MASTER_CREDIT_CNTL__BIF_MC_RDRET_CREDIT__SHIFT 0x00000000
+#define MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL_MASK 0x00000007L
+#define MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL__SHIFT 0x00000000
+#define MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN_MASK 0x00000008L
+#define MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN__SHIFT 0x00000003
+#define MM_DATA__MM_DATA_MASK 0xffffffffL
+#define MM_DATA__MM_DATA__SHIFT 0x00000000
+#define MM_INDEX_HI__MM_OFFSET_HI_MASK 0xffffffffL
+#define MM_INDEX_HI__MM_OFFSET_HI__SHIFT 0x00000000
+#define MM_INDEX__MM_APER_MASK 0x80000000L
+#define MM_INDEX__MM_APER__SHIFT 0x0000001f
+#define MM_INDEX__MM_OFFSET_MASK 0x7fffffffL
+#define MM_INDEX__MM_OFFSET__SHIFT 0x00000000
+#define NEW_REFCLKB_TIMER_1__PHY_PLL_PDWN_TIMER_MASK 0x000003ffL
+#define NEW_REFCLKB_TIMER_1__PHY_PLL_PDWN_TIMER__SHIFT 0x00000000
+#define NEW_REFCLKB_TIMER_1__PLL0_PDNB_EN_MASK 0x00000400L
+#define NEW_REFCLKB_TIMER_1__PLL0_PDNB_EN__SHIFT 0x0000000a
+#define NEW_REFCLKB_TIMER__REFCLK_ON_MASK 0x00200000L
+#define NEW_REFCLKB_TIMER__REFCLK_ON__SHIFT 0x00000015
+#define NEW_REFCLKB_TIMER__REG_STOP_REFCLK_EN_MASK 0x00000001L
+#define NEW_REFCLKB_TIMER__REG_STOP_REFCLK_EN__SHIFT 0x00000000
+#define NEW_REFCLKB_TIMER__STOP_REFCLK_TIMER_MASK 0x001ffffeL
+#define NEW_REFCLKB_TIMER__STOP_REFCLK_TIMER__SHIFT 0x00000001
+#define PB0_DFT_DEBUG_CTRL_REG0__DFT_PHY_DEBUG_EN_MASK 0x00000001L
+#define PB0_DFT_DEBUG_CTRL_REG0__DFT_PHY_DEBUG_EN__SHIFT 0x00000000
+#define PB0_DFT_DEBUG_CTRL_REG0__DFT_PHY_DEBUG_MODE_MASK 0x0000003eL
+#define PB0_DFT_DEBUG_CTRL_REG0__DFT_PHY_DEBUG_MODE__SHIFT 0x00000001
+#define PB0_DFT_JIT_INJ_REG0__DFT_CLK_PER_STEP_MASK 0x00000f00L
+#define PB0_DFT_JIT_INJ_REG0__DFT_CLK_PER_STEP__SHIFT 0x00000008
+#define PB0_DFT_JIT_INJ_REG0__DFT_DECR_SWP_EN_MASK 0x00800000L
+#define PB0_DFT_JIT_INJ_REG0__DFT_DECR_SWP_EN__SHIFT 0x00000017
+#define PB0_DFT_JIT_INJ_REG0__DFT_INCR_SWP_EN_MASK 0x00400000L
+#define PB0_DFT_JIT_INJ_REG0__DFT_INCR_SWP_EN__SHIFT 0x00000016
+#define PB0_DFT_JIT_INJ_REG0__DFT_NUM_STEPS_MASK 0x0000001fL
+#define PB0_DFT_JIT_INJ_REG0__DFT_NUM_STEPS__SHIFT 0x00000000
+#define PB0_DFT_JIT_INJ_REG0__DFT_RECOVERY_TIME_MASK 0xff000000L
+#define PB0_DFT_JIT_INJ_REG0__DFT_RECOVERY_TIME__SHIFT 0x00000018
+#define PB0_DFT_JIT_INJ_REG1__DFT_BLOCK_EN_MASK 0x00010000L
+#define PB0_DFT_JIT_INJ_REG1__DFT_BLOCK_EN__SHIFT 0x00000010
+#define PB0_DFT_JIT_INJ_REG1__DFT_BYPASS_EN_MASK 0x00000100L
+#define PB0_DFT_JIT_INJ_REG1__DFT_BYPASS_EN__SHIFT 0x00000008
+#define PB0_DFT_JIT_INJ_REG1__DFT_BYPASS_VALUE_MASK 0x000000ffL
+#define PB0_DFT_JIT_INJ_REG1__DFT_BYPASS_VALUE__SHIFT 0x00000000
+#define PB0_DFT_JIT_INJ_REG2__DFT_LANE_EN_MASK 0x0000ffffL
+#define PB0_DFT_JIT_INJ_REG2__DFT_LANE_EN__SHIFT 0x00000000
+#define PB0_GLB_CTRL_REG0__BACKUP_MASK 0x0000ffffL
+#define PB0_GLB_CTRL_REG0__BACKUP__SHIFT 0x00000000
+#define PB0_GLB_CTRL_REG0__CFG_IDLEDET_TH_MASK 0x00030000L
+#define PB0_GLB_CTRL_REG0__CFG_IDLEDET_TH__SHIFT 0x00000010
+#define PB0_GLB_CTRL_REG0__DBG_RX2TXBYP_SEL_MASK 0x00700000L
+#define PB0_GLB_CTRL_REG0__DBG_RX2TXBYP_SEL__SHIFT 0x00000014
+#define PB0_GLB_CTRL_REG0__DBG_RXFEBYP_EN_MASK 0x00800000L
+#define PB0_GLB_CTRL_REG0__DBG_RXFEBYP_EN__SHIFT 0x00000017
+#define PB0_GLB_CTRL_REG0__DBG_RXPRBS_CLR_MASK 0x01000000L
+#define PB0_GLB_CTRL_REG0__DBG_RXPRBS_CLR__SHIFT 0x00000018
+#define PB0_GLB_CTRL_REG0__DBG_RXTOGGLE_EN_MASK 0x02000000L
+#define PB0_GLB_CTRL_REG0__DBG_RXTOGGLE_EN__SHIFT 0x00000019
+#define PB0_GLB_CTRL_REG0__DBG_TX2RXLBACK_EN_MASK 0x04000000L
+#define PB0_GLB_CTRL_REG0__DBG_TX2RXLBACK_EN__SHIFT 0x0000001a
+#define PB0_GLB_CTRL_REG0__TXCFG_CMGOOD_RANGE_MASK 0xc0000000L
+#define PB0_GLB_CTRL_REG0__TXCFG_CMGOOD_RANGE__SHIFT 0x0000001e
+#define PB0_GLB_CTRL_REG1__PLL_CFG_DISPCLK_DIV_MASK 0x80000000L
+#define PB0_GLB_CTRL_REG1__PLL_CFG_DISPCLK_DIV__SHIFT 0x0000001f
+#define PB0_GLB_CTRL_REG1__RXDBG_CDR_FR_BYP_EN_MASK 0x00000001L
+#define PB0_GLB_CTRL_REG1__RXDBG_CDR_FR_BYP_EN__SHIFT 0x00000000
+#define PB0_GLB_CTRL_REG1__RXDBG_CDR_FR_BYP_VAL_MASK 0x0000007eL
+#define PB0_GLB_CTRL_REG1__RXDBG_CDR_FR_BYP_VAL__SHIFT 0x00000001
+#define PB0_GLB_CTRL_REG1__RXDBG_CDR_PH_BYP_EN_MASK 0x00000080L
+#define PB0_GLB_CTRL_REG1__RXDBG_CDR_PH_BYP_EN__SHIFT 0x00000007
+#define PB0_GLB_CTRL_REG1__RXDBG_CDR_PH_BYP_VAL_MASK 0x00003f00L
+#define PB0_GLB_CTRL_REG1__RXDBG_CDR_PH_BYP_VAL__SHIFT 0x00000008
+#define PB0_GLB_CTRL_REG1__RXDBG_D0TH_BYP_EN_MASK 0x00004000L
+#define PB0_GLB_CTRL_REG1__RXDBG_D0TH_BYP_EN__SHIFT 0x0000000e
+#define PB0_GLB_CTRL_REG1__RXDBG_D0TH_BYP_VAL_MASK 0x003f8000L
+#define PB0_GLB_CTRL_REG1__RXDBG_D0TH_BYP_VAL__SHIFT 0x0000000f
+#define PB0_GLB_CTRL_REG1__RXDBG_D1TH_BYP_EN_MASK 0x00400000L
+#define PB0_GLB_CTRL_REG1__RXDBG_D1TH_BYP_EN__SHIFT 0x00000016
+#define PB0_GLB_CTRL_REG1__RXDBG_D1TH_BYP_VAL_MASK 0x3f800000L
+#define PB0_GLB_CTRL_REG1__RXDBG_D1TH_BYP_VAL__SHIFT 0x00000017
+#define PB0_GLB_CTRL_REG1__TST_LOSPDTST_EN_MASK 0x40000000L
+#define PB0_GLB_CTRL_REG1__TST_LOSPDTST_EN__SHIFT 0x0000001e
+#define PB0_GLB_CTRL_REG2__RXDBG_D2TH_BYP_EN_MASK 0x00000001L
+#define PB0_GLB_CTRL_REG2__RXDBG_D2TH_BYP_EN__SHIFT 0x00000000
+#define PB0_GLB_CTRL_REG2__RXDBG_D2TH_BYP_VAL_MASK 0x000000feL
+#define PB0_GLB_CTRL_REG2__RXDBG_D2TH_BYP_VAL__SHIFT 0x00000001
+#define PB0_GLB_CTRL_REG2__RXDBG_D3TH_BYP_EN_MASK 0x00000100L
+#define PB0_GLB_CTRL_REG2__RXDBG_D3TH_BYP_EN__SHIFT 0x00000008
+#define PB0_GLB_CTRL_REG2__RXDBG_D3TH_BYP_VAL_MASK 0x0000fe00L
+#define PB0_GLB_CTRL_REG2__RXDBG_D3TH_BYP_VAL__SHIFT 0x00000009
+#define PB0_GLB_CTRL_REG2__RXDBG_DXTH_BYP_EN_MASK 0x00010000L
+#define PB0_GLB_CTRL_REG2__RXDBG_DXTH_BYP_EN__SHIFT 0x00000010
+#define PB0_GLB_CTRL_REG2__RXDBG_DXTH_BYP_VAL_MASK 0x00fe0000L
+#define PB0_GLB_CTRL_REG2__RXDBG_DXTH_BYP_VAL__SHIFT 0x00000011
+#define PB0_GLB_CTRL_REG2__RXDBG_ETH_BYP_EN_MASK 0x01000000L
+#define PB0_GLB_CTRL_REG2__RXDBG_ETH_BYP_EN__SHIFT 0x00000018
+#define PB0_GLB_CTRL_REG2__RXDBG_ETH_BYP_VAL_MASK 0xfe000000L
+#define PB0_GLB_CTRL_REG2__RXDBG_ETH_BYP_VAL__SHIFT 0x00000019
+#define PB0_GLB_CTRL_REG3__BG_CFG_LC_REG_VREF0_SEL_MASK 0x00000060L
+#define PB0_GLB_CTRL_REG3__BG_CFG_LC_REG_VREF0_SEL__SHIFT 0x00000005
+#define PB0_GLB_CTRL_REG3__BG_CFG_LC_REG_VREF1_SEL_MASK 0x00000180L
+#define PB0_GLB_CTRL_REG3__BG_CFG_LC_REG_VREF1_SEL__SHIFT 0x00000007
+#define PB0_GLB_CTRL_REG3__BG_CFG_RO_REG_VREF_SEL_MASK 0x00000600L
+#define PB0_GLB_CTRL_REG3__BG_CFG_RO_REG_VREF_SEL__SHIFT 0x00000009
+#define PB0_GLB_CTRL_REG3__BG_DBG_ANALOG_SEL_MASK 0x0001c000L
+#define PB0_GLB_CTRL_REG3__BG_DBG_ANALOG_SEL__SHIFT 0x0000000e
+#define PB0_GLB_CTRL_REG3__BG_DBG_IREFBYP_EN_MASK 0x00001000L
+#define PB0_GLB_CTRL_REG3__BG_DBG_IREFBYP_EN__SHIFT 0x0000000c
+#define PB0_GLB_CTRL_REG3__BG_DBG_VREFBYP_EN_MASK 0x00000800L
+#define PB0_GLB_CTRL_REG3__BG_DBG_VREFBYP_EN__SHIFT 0x0000000b
+#define PB0_GLB_CTRL_REG3__DBG_DLL_CLK_SEL_MASK 0x001c0000L
+#define PB0_GLB_CTRL_REG3__DBG_DLL_CLK_SEL__SHIFT 0x00000012
+#define PB0_GLB_CTRL_REG3__DBG_RXLEQ_DCATTN_BYP_OVR_DISABLE_MASK 0x80000000L
+#define PB0_GLB_CTRL_REG3__DBG_RXLEQ_DCATTN_BYP_OVR_DISABLE__SHIFT 0x0000001f
+#define PB0_GLB_CTRL_REG3__DBG_RXPI_OFFSET_BYP_EN_MASK 0x00400000L
+#define PB0_GLB_CTRL_REG3__DBG_RXPI_OFFSET_BYP_EN__SHIFT 0x00000016
+#define PB0_GLB_CTRL_REG3__DBG_RXPI_OFFSET_BYP_VAL_MASK 0x07800000L
+#define PB0_GLB_CTRL_REG3__DBG_RXPI_OFFSET_BYP_VAL__SHIFT 0x00000017
+#define PB0_GLB_CTRL_REG3__DBG_RXSWAPDX_BYP_EN_MASK 0x08000000L
+#define PB0_GLB_CTRL_REG3__DBG_RXSWAPDX_BYP_EN__SHIFT 0x0000001b
+#define PB0_GLB_CTRL_REG3__DBG_RXSWAPDX_BYP_VAL_MASK 0x70000000L
+#define PB0_GLB_CTRL_REG3__DBG_RXSWAPDX_BYP_VAL__SHIFT 0x0000001c
+#define PB0_GLB_CTRL_REG3__PLL_DISPCLK_CMOS_SEL_MASK 0x00200000L
+#define PB0_GLB_CTRL_REG3__PLL_DISPCLK_CMOS_SEL__SHIFT 0x00000015
+#define PB0_GLB_CTRL_REG3__RXDBG_SEL_MASK 0x0000001fL
+#define PB0_GLB_CTRL_REG3__RXDBG_SEL__SHIFT 0x00000000
+#define PB0_GLB_CTRL_REG4__DBG_RXAPU_EXEC_MASK 0x03c00000L
+#define PB0_GLB_CTRL_REG4__DBG_RXAPU_EXEC__SHIFT 0x00000016
+#define PB0_GLB_CTRL_REG4__DBG_RXAPU_INST_MASK 0x0000ffffL
+#define PB0_GLB_CTRL_REG4__DBG_RXAPU_INST__SHIFT 0x00000000
+#define PB0_GLB_CTRL_REG4__DBG_RXDFEMUX_BYP_EN_MASK 0x00040000L
+#define PB0_GLB_CTRL_REG4__DBG_RXDFEMUX_BYP_EN__SHIFT 0x00000012
+#define PB0_GLB_CTRL_REG4__DBG_RXDFEMUX_BYP_VAL_MASK 0x00030000L
+#define PB0_GLB_CTRL_REG4__DBG_RXDFEMUX_BYP_VAL__SHIFT 0x00000010
+#define PB0_GLB_CTRL_REG4__DBG_RXDLL_VREG_REF_SEL_MASK 0x04000000L
+#define PB0_GLB_CTRL_REG4__DBG_RXDLL_VREG_REF_SEL__SHIFT 0x0000001a
+#define PB0_GLB_CTRL_REG4__DBG_RXRDATA_GATING_DISABLE_MASK 0x10000000L
+#define PB0_GLB_CTRL_REG4__DBG_RXRDATA_GATING_DISABLE__SHIFT 0x0000001c
+#define PB0_GLB_CTRL_REG4__PWRGOOD_OVRD_MASK 0x08000000L
+#define PB0_GLB_CTRL_REG4__PWRGOOD_OVRD__SHIFT 0x0000001b
+#define PB0_GLB_CTRL_REG5__DBG_RXAPU_MODE_MASK 0x000000ffL
+#define PB0_GLB_CTRL_REG5__DBG_RXAPU_MODE__SHIFT 0x00000000
+#define PB0_GLB_OVRD_REG0__TXPDTERM_VAL_OVRD_VAL_MASK 0x0000ffffL
+#define PB0_GLB_OVRD_REG0__TXPDTERM_VAL_OVRD_VAL__SHIFT 0x00000000
+#define PB0_GLB_OVRD_REG0__TXPUTERM_VAL_OVRD_VAL_MASK 0xffff0000L
+#define PB0_GLB_OVRD_REG0__TXPUTERM_VAL_OVRD_VAL__SHIFT 0x00000010
+#define PB0_GLB_OVRD_REG1__RXTERM_VAL_OVRD_EN_MASK 0x00008000L
+#define PB0_GLB_OVRD_REG1__RXTERM_VAL_OVRD_EN__SHIFT 0x0000000f
+#define PB0_GLB_OVRD_REG1__RXTERM_VAL_OVRD_VAL_MASK 0xffff0000L
+#define PB0_GLB_OVRD_REG1__RXTERM_VAL_OVRD_VAL__SHIFT 0x00000010
+#define PB0_GLB_OVRD_REG1__TST_LOSPDTST_RST_OVRD_EN_MASK 0x00000004L
+#define PB0_GLB_OVRD_REG1__TST_LOSPDTST_RST_OVRD_EN__SHIFT 0x00000002
+#define PB0_GLB_OVRD_REG1__TST_LOSPDTST_RST_OVRD_VAL_MASK 0x00000008L
+#define PB0_GLB_OVRD_REG1__TST_LOSPDTST_RST_OVRD_VAL__SHIFT 0x00000003
+#define PB0_GLB_OVRD_REG1__TXPDTERM_VAL_OVRD_EN_MASK 0x00000001L
+#define PB0_GLB_OVRD_REG1__TXPDTERM_VAL_OVRD_EN__SHIFT 0x00000000
+#define PB0_GLB_OVRD_REG1__TXPUTERM_VAL_OVRD_EN_MASK 0x00000002L
+#define PB0_GLB_OVRD_REG1__TXPUTERM_VAL_OVRD_EN__SHIFT 0x00000001
+#define PB0_GLB_OVRD_REG2__BG_PWRON_OVRD_EN_MASK 0x00000001L
+#define PB0_GLB_OVRD_REG2__BG_PWRON_OVRD_EN__SHIFT 0x00000000
+#define PB0_GLB_OVRD_REG2__BG_PWRON_OVRD_VAL_MASK 0x00000002L
+#define PB0_GLB_OVRD_REG2__BG_PWRON_OVRD_VAL__SHIFT 0x00000001
+#define PB0_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L0T3_MASK 0x00000001L
+#define PB0_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L0T3__SHIFT 0x00000000
+#define PB0_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L12T15_MASK 0x00000008L
+#define PB0_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L12T15__SHIFT 0x00000003
+#define PB0_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L4T7_MASK 0x00000002L
+#define PB0_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L4T7__SHIFT 0x00000001
+#define PB0_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L8T11_MASK 0x00000004L
+#define PB0_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L8T11__SHIFT 0x00000002
+#define PB0_GLB_SCI_STAT_OVRD_REG0__IGNR_IMPCAL_ACTIVE_SCI_UPDT_MASK 0x00000010L
+#define PB0_GLB_SCI_STAT_OVRD_REG0__IGNR_IMPCAL_ACTIVE_SCI_UPDT__SHIFT 0x00000004
+#define PB0_GLB_SCI_STAT_OVRD_REG0__IMPCAL_ACTIVE_MASK 0x00100000L
+#define PB0_GLB_SCI_STAT_OVRD_REG0__IMPCAL_ACTIVE__SHIFT 0x00000014
+#define PB0_GLB_SCI_STAT_OVRD_REG0__RXIMP_MASK 0x000f0000L
+#define PB0_GLB_SCI_STAT_OVRD_REG0__RXIMP__SHIFT 0x00000010
+#define PB0_GLB_SCI_STAT_OVRD_REG0__TXNIMP_MASK 0x00000f00L
+#define PB0_GLB_SCI_STAT_OVRD_REG0__TXNIMP__SHIFT 0x00000008
+#define PB0_GLB_SCI_STAT_OVRD_REG0__TXPIMP_MASK 0x0000f000L
+#define PB0_GLB_SCI_STAT_OVRD_REG0__TXPIMP__SHIFT 0x0000000c
+#define PB0_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_0_MASK 0x00001000L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_0__SHIFT 0x0000000c
+#define PB0_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_1_MASK 0x00002000L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_1__SHIFT 0x0000000d
+#define PB0_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_2_MASK 0x00004000L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_2__SHIFT 0x0000000e
+#define PB0_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_3_MASK 0x00008000L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_3__SHIFT 0x0000000f
+#define PB0_GLB_SCI_STAT_OVRD_REG1__FREQDIV_0_MASK 0x000c0000L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__FREQDIV_0__SHIFT 0x00000012
+#define PB0_GLB_SCI_STAT_OVRD_REG1__FREQDIV_1_MASK 0x00c00000L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__FREQDIV_1__SHIFT 0x00000016
+#define PB0_GLB_SCI_STAT_OVRD_REG1__FREQDIV_2_MASK 0x0c000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__FREQDIV_2__SHIFT 0x0000001a
+#define PB0_GLB_SCI_STAT_OVRD_REG1__FREQDIV_3_MASK 0xc0000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__FREQDIV_3__SHIFT 0x0000001e
+#define PB0_GLB_SCI_STAT_OVRD_REG1__IGNR_DLL_LOCK_SCI_UPDT_L0T3_MASK 0x00000004L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__IGNR_DLL_LOCK_SCI_UPDT_L0T3__SHIFT 0x00000002
+#define PB0_GLB_SCI_STAT_OVRD_REG1__IGNR_FREQDIV_SCI_UPDT_L0T3_MASK 0x00000002L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__IGNR_FREQDIV_SCI_UPDT_L0T3__SHIFT 0x00000001
+#define PB0_GLB_SCI_STAT_OVRD_REG1__IGNR_MODE_SCI_UPDT_L0T3_MASK 0x00000001L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__IGNR_MODE_SCI_UPDT_L0T3__SHIFT 0x00000000
+#define PB0_GLB_SCI_STAT_OVRD_REG1__MODE_0_MASK 0x00030000L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__MODE_0__SHIFT 0x00000010
+#define PB0_GLB_SCI_STAT_OVRD_REG1__MODE_1_MASK 0x00300000L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__MODE_1__SHIFT 0x00000014
+#define PB0_GLB_SCI_STAT_OVRD_REG1__MODE_2_MASK 0x03000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__MODE_2__SHIFT 0x00000018
+#define PB0_GLB_SCI_STAT_OVRD_REG1__MODE_3_MASK 0x30000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG1__MODE_3__SHIFT 0x0000001c
+#define PB0_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_4_MASK 0x00001000L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_4__SHIFT 0x0000000c
+#define PB0_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_5_MASK 0x00002000L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_5__SHIFT 0x0000000d
+#define PB0_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_6_MASK 0x00004000L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_6__SHIFT 0x0000000e
+#define PB0_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_7_MASK 0x00008000L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_7__SHIFT 0x0000000f
+#define PB0_GLB_SCI_STAT_OVRD_REG2__FREQDIV_4_MASK 0x000c0000L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__FREQDIV_4__SHIFT 0x00000012
+#define PB0_GLB_SCI_STAT_OVRD_REG2__FREQDIV_5_MASK 0x00c00000L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__FREQDIV_5__SHIFT 0x00000016
+#define PB0_GLB_SCI_STAT_OVRD_REG2__FREQDIV_6_MASK 0x0c000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__FREQDIV_6__SHIFT 0x0000001a
+#define PB0_GLB_SCI_STAT_OVRD_REG2__FREQDIV_7_MASK 0xc0000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__FREQDIV_7__SHIFT 0x0000001e
+#define PB0_GLB_SCI_STAT_OVRD_REG2__IGNR_DLL_LOCK_SCI_UPDT_L4T7_MASK 0x00000004L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__IGNR_DLL_LOCK_SCI_UPDT_L4T7__SHIFT 0x00000002
+#define PB0_GLB_SCI_STAT_OVRD_REG2__IGNR_FREQDIV_SCI_UPDT_L4T7_MASK 0x00000002L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__IGNR_FREQDIV_SCI_UPDT_L4T7__SHIFT 0x00000001
+#define PB0_GLB_SCI_STAT_OVRD_REG2__IGNR_MODE_SCI_UPDT_L4T7_MASK 0x00000001L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__IGNR_MODE_SCI_UPDT_L4T7__SHIFT 0x00000000
+#define PB0_GLB_SCI_STAT_OVRD_REG2__MODE_4_MASK 0x00030000L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__MODE_4__SHIFT 0x00000010
+#define PB0_GLB_SCI_STAT_OVRD_REG2__MODE_5_MASK 0x00300000L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__MODE_5__SHIFT 0x00000014
+#define PB0_GLB_SCI_STAT_OVRD_REG2__MODE_6_MASK 0x03000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__MODE_6__SHIFT 0x00000018
+#define PB0_GLB_SCI_STAT_OVRD_REG2__MODE_7_MASK 0x30000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG2__MODE_7__SHIFT 0x0000001c
+#define PB0_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_10_MASK 0x00004000L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_10__SHIFT 0x0000000e
+#define PB0_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_11_MASK 0x00008000L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_11__SHIFT 0x0000000f
+#define PB0_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_8_MASK 0x00001000L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_8__SHIFT 0x0000000c
+#define PB0_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_9_MASK 0x00002000L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_9__SHIFT 0x0000000d
+#define PB0_GLB_SCI_STAT_OVRD_REG3__FREQDIV_10_MASK 0x0c000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__FREQDIV_10__SHIFT 0x0000001a
+#define PB0_GLB_SCI_STAT_OVRD_REG3__FREQDIV_11_MASK 0xc0000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__FREQDIV_11__SHIFT 0x0000001e
+#define PB0_GLB_SCI_STAT_OVRD_REG3__FREQDIV_8_MASK 0x000c0000L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__FREQDIV_8__SHIFT 0x00000012
+#define PB0_GLB_SCI_STAT_OVRD_REG3__FREQDIV_9_MASK 0x00c00000L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__FREQDIV_9__SHIFT 0x00000016
+#define PB0_GLB_SCI_STAT_OVRD_REG3__IGNR_DLL_LOCK_SCI_UPDT_L8T11_MASK 0x00000004L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__IGNR_DLL_LOCK_SCI_UPDT_L8T11__SHIFT 0x00000002
+#define PB0_GLB_SCI_STAT_OVRD_REG3__IGNR_FREQDIV_SCI_UPDT_L8T11_MASK 0x00000002L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__IGNR_FREQDIV_SCI_UPDT_L8T11__SHIFT 0x00000001
+#define PB0_GLB_SCI_STAT_OVRD_REG3__IGNR_MODE_SCI_UPDT_L8T11_MASK 0x00000001L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__IGNR_MODE_SCI_UPDT_L8T11__SHIFT 0x00000000
+#define PB0_GLB_SCI_STAT_OVRD_REG3__MODE_10_MASK 0x03000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__MODE_10__SHIFT 0x00000018
+#define PB0_GLB_SCI_STAT_OVRD_REG3__MODE_11_MASK 0x30000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__MODE_11__SHIFT 0x0000001c
+#define PB0_GLB_SCI_STAT_OVRD_REG3__MODE_8_MASK 0x00030000L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__MODE_8__SHIFT 0x00000010
+#define PB0_GLB_SCI_STAT_OVRD_REG3__MODE_9_MASK 0x00300000L
+#define PB0_GLB_SCI_STAT_OVRD_REG3__MODE_9__SHIFT 0x00000014
+#define PB0_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_12_MASK 0x00001000L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_12__SHIFT 0x0000000c
+#define PB0_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_13_MASK 0x00002000L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_13__SHIFT 0x0000000d
+#define PB0_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_14_MASK 0x00004000L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_14__SHIFT 0x0000000e
+#define PB0_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_15_MASK 0x00008000L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_15__SHIFT 0x0000000f
+#define PB0_GLB_SCI_STAT_OVRD_REG4__FREQDIV_12_MASK 0x000c0000L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__FREQDIV_12__SHIFT 0x00000012
+#define PB0_GLB_SCI_STAT_OVRD_REG4__FREQDIV_13_MASK 0x00c00000L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__FREQDIV_13__SHIFT 0x00000016
+#define PB0_GLB_SCI_STAT_OVRD_REG4__FREQDIV_14_MASK 0x0c000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__FREQDIV_14__SHIFT 0x0000001a
+#define PB0_GLB_SCI_STAT_OVRD_REG4__FREQDIV_15_MASK 0xc0000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__FREQDIV_15__SHIFT 0x0000001e
+#define PB0_GLB_SCI_STAT_OVRD_REG4__IGNR_DLL_LOCK_SCI_UPDT_L12T15_MASK 0x00000004L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__IGNR_DLL_LOCK_SCI_UPDT_L12T15__SHIFT 0x00000002
+#define PB0_GLB_SCI_STAT_OVRD_REG4__IGNR_FREQDIV_SCI_UPDT_L12T15_MASK 0x00000002L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__IGNR_FREQDIV_SCI_UPDT_L12T15__SHIFT 0x00000001
+#define PB0_GLB_SCI_STAT_OVRD_REG4__IGNR_MODE_SCI_UPDT_L12T15_MASK 0x00000001L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__IGNR_MODE_SCI_UPDT_L12T15__SHIFT 0x00000000
+#define PB0_GLB_SCI_STAT_OVRD_REG4__MODE_12_MASK 0x00030000L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__MODE_12__SHIFT 0x00000010
+#define PB0_GLB_SCI_STAT_OVRD_REG4__MODE_13_MASK 0x00300000L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__MODE_13__SHIFT 0x00000014
+#define PB0_GLB_SCI_STAT_OVRD_REG4__MODE_14_MASK 0x03000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__MODE_14__SHIFT 0x00000018
+#define PB0_GLB_SCI_STAT_OVRD_REG4__MODE_15_MASK 0x30000000L
+#define PB0_GLB_SCI_STAT_OVRD_REG4__MODE_15__SHIFT 0x0000001c
+#define PB0_HW_DEBUG__PB0_HW_00_DEBUG_MASK 0x00000001L
+#define PB0_HW_DEBUG__PB0_HW_00_DEBUG__SHIFT 0x00000000
+#define PB0_HW_DEBUG__PB0_HW_01_DEBUG_MASK 0x00000002L
+#define PB0_HW_DEBUG__PB0_HW_01_DEBUG__SHIFT 0x00000001
+#define PB0_HW_DEBUG__PB0_HW_02_DEBUG_MASK 0x00000004L
+#define PB0_HW_DEBUG__PB0_HW_02_DEBUG__SHIFT 0x00000002
+#define PB0_HW_DEBUG__PB0_HW_03_DEBUG_MASK 0x00000008L
+#define PB0_HW_DEBUG__PB0_HW_03_DEBUG__SHIFT 0x00000003
+#define PB0_HW_DEBUG__PB0_HW_04_DEBUG_MASK 0x00000010L
+#define PB0_HW_DEBUG__PB0_HW_04_DEBUG__SHIFT 0x00000004
+#define PB0_HW_DEBUG__PB0_HW_05_DEBUG_MASK 0x00000020L
+#define PB0_HW_DEBUG__PB0_HW_05_DEBUG__SHIFT 0x00000005
+#define PB0_HW_DEBUG__PB0_HW_06_DEBUG_MASK 0x00000040L
+#define PB0_HW_DEBUG__PB0_HW_06_DEBUG__SHIFT 0x00000006
+#define PB0_HW_DEBUG__PB0_HW_07_DEBUG_MASK 0x00000080L
+#define PB0_HW_DEBUG__PB0_HW_07_DEBUG__SHIFT 0x00000007
+#define PB0_HW_DEBUG__PB0_HW_08_DEBUG_MASK 0x00000100L
+#define PB0_HW_DEBUG__PB0_HW_08_DEBUG__SHIFT 0x00000008
+#define PB0_HW_DEBUG__PB0_HW_09_DEBUG_MASK 0x00000200L
+#define PB0_HW_DEBUG__PB0_HW_09_DEBUG__SHIFT 0x00000009
+#define PB0_HW_DEBUG__PB0_HW_10_DEBUG_MASK 0x00000400L
+#define PB0_HW_DEBUG__PB0_HW_10_DEBUG__SHIFT 0x0000000a
+#define PB0_HW_DEBUG__PB0_HW_11_DEBUG_MASK 0x00000800L
+#define PB0_HW_DEBUG__PB0_HW_11_DEBUG__SHIFT 0x0000000b
+#define PB0_HW_DEBUG__PB0_HW_12_DEBUG_MASK 0x00001000L
+#define PB0_HW_DEBUG__PB0_HW_12_DEBUG__SHIFT 0x0000000c
+#define PB0_HW_DEBUG__PB0_HW_13_DEBUG_MASK 0x00002000L
+#define PB0_HW_DEBUG__PB0_HW_13_DEBUG__SHIFT 0x0000000d
+#define PB0_HW_DEBUG__PB0_HW_14_DEBUG_MASK 0x00004000L
+#define PB0_HW_DEBUG__PB0_HW_14_DEBUG__SHIFT 0x0000000e
+#define PB0_HW_DEBUG__PB0_HW_15_DEBUG_MASK 0x00008000L
+#define PB0_HW_DEBUG__PB0_HW_15_DEBUG__SHIFT 0x0000000f
+#define PB0_HW_DEBUG__PB0_HW_16_DEBUG_MASK 0x00010000L
+#define PB0_HW_DEBUG__PB0_HW_16_DEBUG__SHIFT 0x00000010
+#define PB0_HW_DEBUG__PB0_HW_17_DEBUG_MASK 0x00020000L
+#define PB0_HW_DEBUG__PB0_HW_17_DEBUG__SHIFT 0x00000011
+#define PB0_HW_DEBUG__PB0_HW_18_DEBUG_MASK 0x00040000L
+#define PB0_HW_DEBUG__PB0_HW_18_DEBUG__SHIFT 0x00000012
+#define PB0_HW_DEBUG__PB0_HW_19_DEBUG_MASK 0x00080000L
+#define PB0_HW_DEBUG__PB0_HW_19_DEBUG__SHIFT 0x00000013
+#define PB0_HW_DEBUG__PB0_HW_20_DEBUG_MASK 0x00100000L
+#define PB0_HW_DEBUG__PB0_HW_20_DEBUG__SHIFT 0x00000014
+#define PB0_HW_DEBUG__PB0_HW_21_DEBUG_MASK 0x00200000L
+#define PB0_HW_DEBUG__PB0_HW_21_DEBUG__SHIFT 0x00000015
+#define PB0_HW_DEBUG__PB0_HW_22_DEBUG_MASK 0x00400000L
+#define PB0_HW_DEBUG__PB0_HW_22_DEBUG__SHIFT 0x00000016
+#define PB0_HW_DEBUG__PB0_HW_23_DEBUG_MASK 0x00800000L
+#define PB0_HW_DEBUG__PB0_HW_23_DEBUG__SHIFT 0x00000017
+#define PB0_HW_DEBUG__PB0_HW_24_DEBUG_MASK 0x01000000L
+#define PB0_HW_DEBUG__PB0_HW_24_DEBUG__SHIFT 0x00000018
+#define PB0_HW_DEBUG__PB0_HW_25_DEBUG_MASK 0x02000000L
+#define PB0_HW_DEBUG__PB0_HW_25_DEBUG__SHIFT 0x00000019
+#define PB0_HW_DEBUG__PB0_HW_26_DEBUG_MASK 0x04000000L
+#define PB0_HW_DEBUG__PB0_HW_26_DEBUG__SHIFT 0x0000001a
+#define PB0_HW_DEBUG__PB0_HW_27_DEBUG_MASK 0x08000000L
+#define PB0_HW_DEBUG__PB0_HW_27_DEBUG__SHIFT 0x0000001b
+#define PB0_HW_DEBUG__PB0_HW_28_DEBUG_MASK 0x10000000L
+#define PB0_HW_DEBUG__PB0_HW_28_DEBUG__SHIFT 0x0000001c
+#define PB0_HW_DEBUG__PB0_HW_29_DEBUG_MASK 0x20000000L
+#define PB0_HW_DEBUG__PB0_HW_29_DEBUG__SHIFT 0x0000001d
+#define PB0_HW_DEBUG__PB0_HW_30_DEBUG_MASK 0x40000000L
+#define PB0_HW_DEBUG__PB0_HW_30_DEBUG__SHIFT 0x0000001e
+#define PB0_HW_DEBUG__PB0_HW_31_DEBUG_MASK 0x80000000L
+#define PB0_HW_DEBUG__PB0_HW_31_DEBUG__SHIFT 0x0000001f
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_EN_MASK 0x00000080L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_EN__SHIFT 0x00000007
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_0_MASK 0x00000100L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_0__SHIFT 0x00000008
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_10_MASK 0x00040000L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_10__SHIFT 0x00000012
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_11_MASK 0x00080000L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_11__SHIFT 0x00000013
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_12_MASK 0x00100000L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_12__SHIFT 0x00000014
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_13_MASK 0x00200000L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_13__SHIFT 0x00000015
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_14_MASK 0x00400000L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_14__SHIFT 0x00000016
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_15_MASK 0x00800000L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_15__SHIFT 0x00000017
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_1_MASK 0x00000200L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_1__SHIFT 0x00000009
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_2_MASK 0x00000400L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_2__SHIFT 0x0000000a
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_3_MASK 0x00000800L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_3__SHIFT 0x0000000b
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_4_MASK 0x00001000L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_4__SHIFT 0x0000000c
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_5_MASK 0x00002000L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_5__SHIFT 0x0000000d
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_6_MASK 0x00004000L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_6__SHIFT 0x0000000e
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_7_MASK 0x00008000L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_7__SHIFT 0x0000000f
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_8_MASK 0x00010000L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_8__SHIFT 0x00000010
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_9_MASK 0x00020000L
+#define PB0_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_9__SHIFT 0x00000011
+#define PB0_PIF_CNTL2__RXDETECT_SAMPL_TIME_MASK 0x00000006L
+#define PB0_PIF_CNTL2__RXDETECT_SAMPL_TIME__SHIFT 0x00000001
+#define PB0_PIF_CNTL2__RXPHYSTATUS_DELAY_MASK 0x07000000L
+#define PB0_PIF_CNTL2__RXPHYSTATUS_DELAY__SHIFT 0x00000018
+#define PB0_PIF_CNTL__DA_FIFO_RESET_0_MASK 0x00000002L
+#define PB0_PIF_CNTL__DA_FIFO_RESET_0__SHIFT 0x00000001
+#define PB0_PIF_CNTL__DA_FIFO_RESET_1_MASK 0x00000020L
+#define PB0_PIF_CNTL__DA_FIFO_RESET_1__SHIFT 0x00000005
+#define PB0_PIF_CNTL__DA_FIFO_RESET_2_MASK 0x00000200L
+#define PB0_PIF_CNTL__DA_FIFO_RESET_2__SHIFT 0x00000009
+#define PB0_PIF_CNTL__DA_FIFO_RESET_3_MASK 0x00002000L
+#define PB0_PIF_CNTL__DA_FIFO_RESET_3__SHIFT 0x0000000d
+#define PB0_PIF_CNTL__DIVINIT_MODE_MASK 0x00000100L
+#define PB0_PIF_CNTL__DIVINIT_MODE__SHIFT 0x00000008
+#define PB0_PIF_CNTL__EI_CYCLE_OFF_TIME_MASK 0x00700000L
+#define PB0_PIF_CNTL__EI_CYCLE_OFF_TIME__SHIFT 0x00000014
+#define PB0_PIF_CNTL__EI_DET_CYCLE_MODE_MASK 0x00000010L
+#define PB0_PIF_CNTL__EI_DET_CYCLE_MODE__SHIFT 0x00000004
+#define PB0_PIF_CNTL__EXIT_L0S_INIT_DIS_MASK 0x00800000L
+#define PB0_PIF_CNTL__EXIT_L0S_INIT_DIS__SHIFT 0x00000017
+#define PB0_PIF_CNTL__EXTEND_WAIT_FOR_RAMPUP_MASK 0x10000000L
+#define PB0_PIF_CNTL__EXTEND_WAIT_FOR_RAMPUP__SHIFT 0x0000001c
+#define PB0_PIF_CNTL__IGNORE_TxDataValid_EP_DIS_MASK 0x20000000L
+#define PB0_PIF_CNTL__IGNORE_TxDataValid_EP_DIS__SHIFT 0x0000001d
+#define PB0_PIF_CNTL__LS2_EXIT_TIME_MASK 0x000e0000L
+#define PB0_PIF_CNTL__LS2_EXIT_TIME__SHIFT 0x00000011
+#define PB0_PIF_CNTL__PHYCMD_CR_EN_MODE_MASK 0x00000008L
+#define PB0_PIF_CNTL__PHYCMD_CR_EN_MODE__SHIFT 0x00000003
+#define PB0_PIF_CNTL__PHY_CR_EN_MODE_MASK 0x00000004L
+#define PB0_PIF_CNTL__PHY_CR_EN_MODE__SHIFT 0x00000002
+#define PB0_PIF_CNTL__PLL_BINDING_ENABLE_MASK 0x00000400L
+#define PB0_PIF_CNTL__PLL_BINDING_ENABLE__SHIFT 0x0000000a
+#define PB0_PIF_CNTL__RXDETECT_FIFO_RESET_MODE_MASK 0x00000040L
+#define PB0_PIF_CNTL__RXDETECT_FIFO_RESET_MODE__SHIFT 0x00000006
+#define PB0_PIF_CNTL__RXDETECT_TX_PWR_MODE_MASK 0x00000080L
+#define PB0_PIF_CNTL__RXDETECT_TX_PWR_MODE__SHIFT 0x00000007
+#define PB0_PIF_CNTL__RXEN_GATER_MASK 0x0f000000L
+#define PB0_PIF_CNTL__RXEN_GATER__SHIFT 0x00000018
+#define PB0_PIF_CNTL__SC_CALIB_DONE_CNTL_MASK 0x00000800L
+#define PB0_PIF_CNTL__SC_CALIB_DONE_CNTL__SHIFT 0x0000000b
+#define PB0_PIF_CNTL__SERIAL_CFG_ENABLE_MASK 0x00000001L
+#define PB0_PIF_CNTL__SERIAL_CFG_ENABLE__SHIFT 0x00000000
+#define PB0_PIF_CNTL__TXGND_TIME_MASK 0x00010000L
+#define PB0_PIF_CNTL__TXGND_TIME__SHIFT 0x00000010
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_00_DEBUG_MASK 0x00000001L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_00_DEBUG__SHIFT 0x00000000
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_01_DEBUG_MASK 0x00000002L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_01_DEBUG__SHIFT 0x00000001
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_02_DEBUG_MASK 0x00000004L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_02_DEBUG__SHIFT 0x00000002
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_03_DEBUG_MASK 0x00000008L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_03_DEBUG__SHIFT 0x00000003
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_04_DEBUG_MASK 0x00000010L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_04_DEBUG__SHIFT 0x00000004
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_05_DEBUG_MASK 0x00000020L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_05_DEBUG__SHIFT 0x00000005
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_06_DEBUG_MASK 0x00000040L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_06_DEBUG__SHIFT 0x00000006
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_07_DEBUG_MASK 0x00000080L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_07_DEBUG__SHIFT 0x00000007
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_08_DEBUG_MASK 0x00000100L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_08_DEBUG__SHIFT 0x00000008
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_09_DEBUG_MASK 0x00000200L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_09_DEBUG__SHIFT 0x00000009
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_10_DEBUG_MASK 0x00000400L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_10_DEBUG__SHIFT 0x0000000a
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_11_DEBUG_MASK 0x00000800L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_11_DEBUG__SHIFT 0x0000000b
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_12_DEBUG_MASK 0x00001000L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_12_DEBUG__SHIFT 0x0000000c
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_13_DEBUG_MASK 0x00002000L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_13_DEBUG__SHIFT 0x0000000d
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_14_DEBUG_MASK 0x00004000L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_14_DEBUG__SHIFT 0x0000000e
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_15_DEBUG_MASK 0x00008000L
+#define PB0_PIF_HW_DEBUG__PB0_PIF_HW_15_DEBUG__SHIFT 0x0000000f
+#define PB0_PIF_PAIRING__MULTI_PIF_MASK 0x02000000L
+#define PB0_PIF_PAIRING__MULTI_PIF__SHIFT 0x00000019
+#define PB0_PIF_PAIRING__X16_LANE_15_0_MASK 0x00100000L
+#define PB0_PIF_PAIRING__X16_LANE_15_0__SHIFT 0x00000014
+#define PB0_PIF_PAIRING__X2_LANE_1_0_MASK 0x00000001L
+#define PB0_PIF_PAIRING__X2_LANE_1_0__SHIFT 0x00000000
+#define PB0_PIF_PAIRING__X2_LANE_11_10_MASK 0x00000020L
+#define PB0_PIF_PAIRING__X2_LANE_11_10__SHIFT 0x00000005
+#define PB0_PIF_PAIRING__X2_LANE_13_12_MASK 0x00000040L
+#define PB0_PIF_PAIRING__X2_LANE_13_12__SHIFT 0x00000006
+#define PB0_PIF_PAIRING__X2_LANE_15_14_MASK 0x00000080L
+#define PB0_PIF_PAIRING__X2_LANE_15_14__SHIFT 0x00000007
+#define PB0_PIF_PAIRING__X2_LANE_3_2_MASK 0x00000002L
+#define PB0_PIF_PAIRING__X2_LANE_3_2__SHIFT 0x00000001
+#define PB0_PIF_PAIRING__X2_LANE_5_4_MASK 0x00000004L
+#define PB0_PIF_PAIRING__X2_LANE_5_4__SHIFT 0x00000002
+#define PB0_PIF_PAIRING__X2_LANE_7_6_MASK 0x00000008L
+#define PB0_PIF_PAIRING__X2_LANE_7_6__SHIFT 0x00000003
+#define PB0_PIF_PAIRING__X2_LANE_9_8_MASK 0x00000010L
+#define PB0_PIF_PAIRING__X2_LANE_9_8__SHIFT 0x00000004
+#define PB0_PIF_PAIRING__X4_LANE_11_8_MASK 0x00000400L
+#define PB0_PIF_PAIRING__X4_LANE_11_8__SHIFT 0x0000000a
+#define PB0_PIF_PAIRING__X4_LANE_15_12_MASK 0x00000800L
+#define PB0_PIF_PAIRING__X4_LANE_15_12__SHIFT 0x0000000b
+#define PB0_PIF_PAIRING__X4_LANE_3_0_MASK 0x00000100L
+#define PB0_PIF_PAIRING__X4_LANE_3_0__SHIFT 0x00000008
+#define PB0_PIF_PAIRING__X4_LANE_7_4_MASK 0x00000200L
+#define PB0_PIF_PAIRING__X4_LANE_7_4__SHIFT 0x00000009
+#define PB0_PIF_PAIRING__X8_LANE_15_8_MASK 0x00020000L
+#define PB0_PIF_PAIRING__X8_LANE_15_8__SHIFT 0x00000011
+#define PB0_PIF_PAIRING__X8_LANE_7_0_MASK 0x00010000L
+#define PB0_PIF_PAIRING__X8_LANE_7_0__SHIFT 0x00000010
+#define PB0_PIF_PDNB_OVERRIDE_0__RXEN_OVERRIDE_EN_0_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_0__RXEN_OVERRIDE_EN_0__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_0__RXEN_OVERRIDE_VAL_0_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_0__RXEN_OVERRIDE_VAL_0__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_0__RX_PDNB_OVERRIDE_EN_0_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_0__RX_PDNB_OVERRIDE_EN_0__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_0__RX_PDNB_OVERRIDE_VAL_0_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_0__RX_PDNB_OVERRIDE_VAL_0__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_0__RXPWR_OVERRIDE_EN_0_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_0__RXPWR_OVERRIDE_EN_0__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_0__RXPWR_OVERRIDE_VAL_0_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_0__RXPWR_OVERRIDE_VAL_0__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_0__TX_PDNB_OVERRIDE_EN_0_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_0__TX_PDNB_OVERRIDE_EN_0__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_0__TX_PDNB_OVERRIDE_VAL_0_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_0__TX_PDNB_OVERRIDE_VAL_0__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_0__TXPWR_OVERRIDE_EN_0_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_0__TXPWR_OVERRIDE_EN_0__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_0__TXPWR_OVERRIDE_VAL_0_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_0__TXPWR_OVERRIDE_VAL_0__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_10__RXEN_OVERRIDE_EN_10_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_10__RXEN_OVERRIDE_EN_10__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_10__RXEN_OVERRIDE_VAL_10_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_10__RXEN_OVERRIDE_VAL_10__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_10__RX_PDNB_OVERRIDE_EN_10_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_10__RX_PDNB_OVERRIDE_EN_10__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_10__RX_PDNB_OVERRIDE_VAL_10_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_10__RX_PDNB_OVERRIDE_VAL_10__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_10__RXPWR_OVERRIDE_EN_10_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_10__RXPWR_OVERRIDE_EN_10__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_10__RXPWR_OVERRIDE_VAL_10_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_10__RXPWR_OVERRIDE_VAL_10__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_10__TX_PDNB_OVERRIDE_EN_10_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_10__TX_PDNB_OVERRIDE_EN_10__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_10__TX_PDNB_OVERRIDE_VAL_10_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_10__TX_PDNB_OVERRIDE_VAL_10__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_10__TXPWR_OVERRIDE_EN_10_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_10__TXPWR_OVERRIDE_EN_10__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_10__TXPWR_OVERRIDE_VAL_10_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_10__TXPWR_OVERRIDE_VAL_10__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_11__RXEN_OVERRIDE_EN_11_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_11__RXEN_OVERRIDE_EN_11__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_11__RXEN_OVERRIDE_VAL_11_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_11__RXEN_OVERRIDE_VAL_11__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_11__RX_PDNB_OVERRIDE_EN_11_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_11__RX_PDNB_OVERRIDE_EN_11__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_11__RX_PDNB_OVERRIDE_VAL_11_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_11__RX_PDNB_OVERRIDE_VAL_11__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_11__RXPWR_OVERRIDE_EN_11_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_11__RXPWR_OVERRIDE_EN_11__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_11__RXPWR_OVERRIDE_VAL_11_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_11__RXPWR_OVERRIDE_VAL_11__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_11__TX_PDNB_OVERRIDE_EN_11_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_11__TX_PDNB_OVERRIDE_EN_11__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_11__TX_PDNB_OVERRIDE_VAL_11_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_11__TX_PDNB_OVERRIDE_VAL_11__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_11__TXPWR_OVERRIDE_EN_11_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_11__TXPWR_OVERRIDE_EN_11__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_11__TXPWR_OVERRIDE_VAL_11_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_11__TXPWR_OVERRIDE_VAL_11__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_12__RXEN_OVERRIDE_EN_12_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_12__RXEN_OVERRIDE_EN_12__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_12__RXEN_OVERRIDE_VAL_12_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_12__RXEN_OVERRIDE_VAL_12__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_12__RX_PDNB_OVERRIDE_EN_12_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_12__RX_PDNB_OVERRIDE_EN_12__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_12__RX_PDNB_OVERRIDE_VAL_12_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_12__RX_PDNB_OVERRIDE_VAL_12__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_12__RXPWR_OVERRIDE_EN_12_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_12__RXPWR_OVERRIDE_EN_12__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_12__RXPWR_OVERRIDE_VAL_12_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_12__RXPWR_OVERRIDE_VAL_12__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_12__TX_PDNB_OVERRIDE_EN_12_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_12__TX_PDNB_OVERRIDE_EN_12__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_12__TX_PDNB_OVERRIDE_VAL_12_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_12__TX_PDNB_OVERRIDE_VAL_12__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_12__TXPWR_OVERRIDE_EN_12_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_12__TXPWR_OVERRIDE_EN_12__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_12__TXPWR_OVERRIDE_VAL_12_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_12__TXPWR_OVERRIDE_VAL_12__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_13__RXEN_OVERRIDE_EN_13_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_13__RXEN_OVERRIDE_EN_13__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_13__RXEN_OVERRIDE_VAL_13_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_13__RXEN_OVERRIDE_VAL_13__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_13__RX_PDNB_OVERRIDE_EN_13_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_13__RX_PDNB_OVERRIDE_EN_13__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_13__RX_PDNB_OVERRIDE_VAL_13_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_13__RX_PDNB_OVERRIDE_VAL_13__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_13__RXPWR_OVERRIDE_EN_13_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_13__RXPWR_OVERRIDE_EN_13__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_13__RXPWR_OVERRIDE_VAL_13_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_13__RXPWR_OVERRIDE_VAL_13__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_13__TX_PDNB_OVERRIDE_EN_13_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_13__TX_PDNB_OVERRIDE_EN_13__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_13__TX_PDNB_OVERRIDE_VAL_13_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_13__TX_PDNB_OVERRIDE_VAL_13__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_13__TXPWR_OVERRIDE_EN_13_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_13__TXPWR_OVERRIDE_EN_13__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_13__TXPWR_OVERRIDE_VAL_13_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_13__TXPWR_OVERRIDE_VAL_13__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_14__RXEN_OVERRIDE_EN_14_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_14__RXEN_OVERRIDE_EN_14__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_14__RXEN_OVERRIDE_VAL_14_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_14__RXEN_OVERRIDE_VAL_14__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_14__RX_PDNB_OVERRIDE_EN_14_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_14__RX_PDNB_OVERRIDE_EN_14__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_14__RX_PDNB_OVERRIDE_VAL_14_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_14__RX_PDNB_OVERRIDE_VAL_14__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_14__RXPWR_OVERRIDE_EN_14_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_14__RXPWR_OVERRIDE_EN_14__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_14__RXPWR_OVERRIDE_VAL_14_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_14__RXPWR_OVERRIDE_VAL_14__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_14__TX_PDNB_OVERRIDE_EN_14_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_14__TX_PDNB_OVERRIDE_EN_14__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_14__TX_PDNB_OVERRIDE_VAL_14_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_14__TX_PDNB_OVERRIDE_VAL_14__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_14__TXPWR_OVERRIDE_EN_14_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_14__TXPWR_OVERRIDE_EN_14__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_14__TXPWR_OVERRIDE_VAL_14_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_14__TXPWR_OVERRIDE_VAL_14__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_15__RXEN_OVERRIDE_EN_15_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_15__RXEN_OVERRIDE_EN_15__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_15__RXEN_OVERRIDE_VAL_15_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_15__RXEN_OVERRIDE_VAL_15__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_15__RX_PDNB_OVERRIDE_EN_15_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_15__RX_PDNB_OVERRIDE_EN_15__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_15__RX_PDNB_OVERRIDE_VAL_15_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_15__RX_PDNB_OVERRIDE_VAL_15__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_15__RXPWR_OVERRIDE_EN_15_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_15__RXPWR_OVERRIDE_EN_15__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_15__RXPWR_OVERRIDE_VAL_15_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_15__RXPWR_OVERRIDE_VAL_15__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_15__TX_PDNB_OVERRIDE_EN_15_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_15__TX_PDNB_OVERRIDE_EN_15__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_15__TX_PDNB_OVERRIDE_VAL_15_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_15__TX_PDNB_OVERRIDE_VAL_15__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_15__TXPWR_OVERRIDE_EN_15_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_15__TXPWR_OVERRIDE_EN_15__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_15__TXPWR_OVERRIDE_VAL_15_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_15__TXPWR_OVERRIDE_VAL_15__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_1__RXEN_OVERRIDE_EN_1_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_1__RXEN_OVERRIDE_EN_1__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_1__RXEN_OVERRIDE_VAL_1_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_1__RXEN_OVERRIDE_VAL_1__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_1__RX_PDNB_OVERRIDE_EN_1_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_1__RX_PDNB_OVERRIDE_EN_1__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_1__RX_PDNB_OVERRIDE_VAL_1_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_1__RX_PDNB_OVERRIDE_VAL_1__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_1__RXPWR_OVERRIDE_EN_1_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_1__RXPWR_OVERRIDE_EN_1__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_1__RXPWR_OVERRIDE_VAL_1_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_1__RXPWR_OVERRIDE_VAL_1__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_1__TX_PDNB_OVERRIDE_EN_1_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_1__TX_PDNB_OVERRIDE_EN_1__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_1__TX_PDNB_OVERRIDE_VAL_1_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_1__TX_PDNB_OVERRIDE_VAL_1__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_1__TXPWR_OVERRIDE_EN_1_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_1__TXPWR_OVERRIDE_EN_1__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_1__TXPWR_OVERRIDE_VAL_1_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_1__TXPWR_OVERRIDE_VAL_1__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_2__RXEN_OVERRIDE_EN_2_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_2__RXEN_OVERRIDE_EN_2__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_2__RXEN_OVERRIDE_VAL_2_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_2__RXEN_OVERRIDE_VAL_2__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_2__RX_PDNB_OVERRIDE_EN_2_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_2__RX_PDNB_OVERRIDE_EN_2__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_2__RX_PDNB_OVERRIDE_VAL_2_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_2__RX_PDNB_OVERRIDE_VAL_2__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_2__RXPWR_OVERRIDE_EN_2_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_2__RXPWR_OVERRIDE_EN_2__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_2__RXPWR_OVERRIDE_VAL_2_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_2__RXPWR_OVERRIDE_VAL_2__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_2__TX_PDNB_OVERRIDE_EN_2_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_2__TX_PDNB_OVERRIDE_EN_2__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_2__TX_PDNB_OVERRIDE_VAL_2_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_2__TX_PDNB_OVERRIDE_VAL_2__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_2__TXPWR_OVERRIDE_EN_2_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_2__TXPWR_OVERRIDE_EN_2__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_2__TXPWR_OVERRIDE_VAL_2_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_2__TXPWR_OVERRIDE_VAL_2__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_3__RXEN_OVERRIDE_EN_3_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_3__RXEN_OVERRIDE_EN_3__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_3__RXEN_OVERRIDE_VAL_3_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_3__RXEN_OVERRIDE_VAL_3__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_3__RX_PDNB_OVERRIDE_EN_3_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_3__RX_PDNB_OVERRIDE_EN_3__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_3__RX_PDNB_OVERRIDE_VAL_3_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_3__RX_PDNB_OVERRIDE_VAL_3__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_3__RXPWR_OVERRIDE_EN_3_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_3__RXPWR_OVERRIDE_EN_3__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_3__RXPWR_OVERRIDE_VAL_3_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_3__RXPWR_OVERRIDE_VAL_3__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_3__TX_PDNB_OVERRIDE_EN_3_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_3__TX_PDNB_OVERRIDE_EN_3__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_3__TX_PDNB_OVERRIDE_VAL_3_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_3__TX_PDNB_OVERRIDE_VAL_3__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_3__TXPWR_OVERRIDE_EN_3_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_3__TXPWR_OVERRIDE_EN_3__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_3__TXPWR_OVERRIDE_VAL_3_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_3__TXPWR_OVERRIDE_VAL_3__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_4__RXEN_OVERRIDE_EN_4_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_4__RXEN_OVERRIDE_EN_4__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_4__RXEN_OVERRIDE_VAL_4_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_4__RXEN_OVERRIDE_VAL_4__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_4__RX_PDNB_OVERRIDE_EN_4_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_4__RX_PDNB_OVERRIDE_EN_4__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_4__RX_PDNB_OVERRIDE_VAL_4_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_4__RX_PDNB_OVERRIDE_VAL_4__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_4__RXPWR_OVERRIDE_EN_4_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_4__RXPWR_OVERRIDE_EN_4__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_4__RXPWR_OVERRIDE_VAL_4_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_4__RXPWR_OVERRIDE_VAL_4__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_4__TX_PDNB_OVERRIDE_EN_4_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_4__TX_PDNB_OVERRIDE_EN_4__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_4__TX_PDNB_OVERRIDE_VAL_4_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_4__TX_PDNB_OVERRIDE_VAL_4__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_4__TXPWR_OVERRIDE_EN_4_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_4__TXPWR_OVERRIDE_EN_4__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_4__TXPWR_OVERRIDE_VAL_4_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_4__TXPWR_OVERRIDE_VAL_4__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_5__RXEN_OVERRIDE_EN_5_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_5__RXEN_OVERRIDE_EN_5__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_5__RXEN_OVERRIDE_VAL_5_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_5__RXEN_OVERRIDE_VAL_5__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_5__RX_PDNB_OVERRIDE_EN_5_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_5__RX_PDNB_OVERRIDE_EN_5__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_5__RX_PDNB_OVERRIDE_VAL_5_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_5__RX_PDNB_OVERRIDE_VAL_5__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_5__RXPWR_OVERRIDE_EN_5_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_5__RXPWR_OVERRIDE_EN_5__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_5__RXPWR_OVERRIDE_VAL_5_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_5__RXPWR_OVERRIDE_VAL_5__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_5__TX_PDNB_OVERRIDE_EN_5_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_5__TX_PDNB_OVERRIDE_EN_5__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_5__TX_PDNB_OVERRIDE_VAL_5_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_5__TX_PDNB_OVERRIDE_VAL_5__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_5__TXPWR_OVERRIDE_EN_5_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_5__TXPWR_OVERRIDE_EN_5__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_5__TXPWR_OVERRIDE_VAL_5_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_5__TXPWR_OVERRIDE_VAL_5__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_6__RXEN_OVERRIDE_EN_6_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_6__RXEN_OVERRIDE_EN_6__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_6__RXEN_OVERRIDE_VAL_6_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_6__RXEN_OVERRIDE_VAL_6__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_6__RX_PDNB_OVERRIDE_EN_6_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_6__RX_PDNB_OVERRIDE_EN_6__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_6__RX_PDNB_OVERRIDE_VAL_6_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_6__RX_PDNB_OVERRIDE_VAL_6__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_6__RXPWR_OVERRIDE_EN_6_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_6__RXPWR_OVERRIDE_EN_6__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_6__RXPWR_OVERRIDE_VAL_6_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_6__RXPWR_OVERRIDE_VAL_6__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_6__TX_PDNB_OVERRIDE_EN_6_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_6__TX_PDNB_OVERRIDE_EN_6__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_6__TX_PDNB_OVERRIDE_VAL_6_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_6__TX_PDNB_OVERRIDE_VAL_6__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_6__TXPWR_OVERRIDE_EN_6_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_6__TXPWR_OVERRIDE_EN_6__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_6__TXPWR_OVERRIDE_VAL_6_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_6__TXPWR_OVERRIDE_VAL_6__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_7__RXEN_OVERRIDE_EN_7_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_7__RXEN_OVERRIDE_EN_7__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_7__RXEN_OVERRIDE_VAL_7_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_7__RXEN_OVERRIDE_VAL_7__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_7__RX_PDNB_OVERRIDE_EN_7_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_7__RX_PDNB_OVERRIDE_EN_7__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_7__RX_PDNB_OVERRIDE_VAL_7_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_7__RX_PDNB_OVERRIDE_VAL_7__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_7__RXPWR_OVERRIDE_EN_7_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_7__RXPWR_OVERRIDE_EN_7__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_7__RXPWR_OVERRIDE_VAL_7_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_7__RXPWR_OVERRIDE_VAL_7__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_7__TX_PDNB_OVERRIDE_EN_7_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_7__TX_PDNB_OVERRIDE_EN_7__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_7__TX_PDNB_OVERRIDE_VAL_7_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_7__TX_PDNB_OVERRIDE_VAL_7__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_7__TXPWR_OVERRIDE_EN_7_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_7__TXPWR_OVERRIDE_EN_7__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_7__TXPWR_OVERRIDE_VAL_7_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_7__TXPWR_OVERRIDE_VAL_7__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_8__RXEN_OVERRIDE_EN_8_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_8__RXEN_OVERRIDE_EN_8__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_8__RXEN_OVERRIDE_VAL_8_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_8__RXEN_OVERRIDE_VAL_8__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_8__RX_PDNB_OVERRIDE_EN_8_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_8__RX_PDNB_OVERRIDE_EN_8__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_8__RX_PDNB_OVERRIDE_VAL_8_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_8__RX_PDNB_OVERRIDE_VAL_8__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_8__RXPWR_OVERRIDE_EN_8_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_8__RXPWR_OVERRIDE_EN_8__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_8__RXPWR_OVERRIDE_VAL_8_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_8__RXPWR_OVERRIDE_VAL_8__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_8__TX_PDNB_OVERRIDE_EN_8_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_8__TX_PDNB_OVERRIDE_EN_8__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_8__TX_PDNB_OVERRIDE_VAL_8_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_8__TX_PDNB_OVERRIDE_VAL_8__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_8__TXPWR_OVERRIDE_EN_8_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_8__TXPWR_OVERRIDE_EN_8__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_8__TXPWR_OVERRIDE_VAL_8_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_8__TXPWR_OVERRIDE_VAL_8__SHIFT 0x0000000b
+#define PB0_PIF_PDNB_OVERRIDE_9__RXEN_OVERRIDE_EN_9_MASK 0x00000100L
+#define PB0_PIF_PDNB_OVERRIDE_9__RXEN_OVERRIDE_EN_9__SHIFT 0x00000008
+#define PB0_PIF_PDNB_OVERRIDE_9__RXEN_OVERRIDE_VAL_9_MASK 0x00000200L
+#define PB0_PIF_PDNB_OVERRIDE_9__RXEN_OVERRIDE_VAL_9__SHIFT 0x00000009
+#define PB0_PIF_PDNB_OVERRIDE_9__RX_PDNB_OVERRIDE_EN_9_MASK 0x00000010L
+#define PB0_PIF_PDNB_OVERRIDE_9__RX_PDNB_OVERRIDE_EN_9__SHIFT 0x00000004
+#define PB0_PIF_PDNB_OVERRIDE_9__RX_PDNB_OVERRIDE_VAL_9_MASK 0x000000e0L
+#define PB0_PIF_PDNB_OVERRIDE_9__RX_PDNB_OVERRIDE_VAL_9__SHIFT 0x00000005
+#define PB0_PIF_PDNB_OVERRIDE_9__RXPWR_OVERRIDE_EN_9_MASK 0x00004000L
+#define PB0_PIF_PDNB_OVERRIDE_9__RXPWR_OVERRIDE_EN_9__SHIFT 0x0000000e
+#define PB0_PIF_PDNB_OVERRIDE_9__RXPWR_OVERRIDE_VAL_9_MASK 0x00038000L
+#define PB0_PIF_PDNB_OVERRIDE_9__RXPWR_OVERRIDE_VAL_9__SHIFT 0x0000000f
+#define PB0_PIF_PDNB_OVERRIDE_9__TX_PDNB_OVERRIDE_EN_9_MASK 0x00000001L
+#define PB0_PIF_PDNB_OVERRIDE_9__TX_PDNB_OVERRIDE_EN_9__SHIFT 0x00000000
+#define PB0_PIF_PDNB_OVERRIDE_9__TX_PDNB_OVERRIDE_VAL_9_MASK 0x0000000eL
+#define PB0_PIF_PDNB_OVERRIDE_9__TX_PDNB_OVERRIDE_VAL_9__SHIFT 0x00000001
+#define PB0_PIF_PDNB_OVERRIDE_9__TXPWR_OVERRIDE_EN_9_MASK 0x00000400L
+#define PB0_PIF_PDNB_OVERRIDE_9__TXPWR_OVERRIDE_EN_9__SHIFT 0x0000000a
+#define PB0_PIF_PDNB_OVERRIDE_9__TXPWR_OVERRIDE_VAL_9_MASK 0x00003800L
+#define PB0_PIF_PDNB_OVERRIDE_9__TXPWR_OVERRIDE_VAL_9__SHIFT 0x0000000b
+#define PB0_PIF_PWRDOWN_0__FORCE_RXEN_IN_L0s_0_MASK 0x00000008L
+#define PB0_PIF_PWRDOWN_0__FORCE_RXEN_IN_L0s_0__SHIFT 0x00000003
+#define PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK 0x00001c00L
+#define PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT 0x0000000a
+#define PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK 0x00000380L
+#define PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT 0x00000007
+#define PB0_PIF_PWRDOWN_0__PLLPWR_OVERRIDE_EN_0_MASK 0x10000000L
+#define PB0_PIF_PWRDOWN_0__PLLPWR_OVERRIDE_EN_0__SHIFT 0x0000001c
+#define PB0_PIF_PWRDOWN_0__PLLPWR_OVERRIDE_VAL_0_MASK 0xe0000000L
+#define PB0_PIF_PWRDOWN_0__PLLPWR_OVERRIDE_VAL_0__SHIFT 0x0000001d
+#define PB0_PIF_PWRDOWN_0__PLL_RAMP_UP_TIME_0_MASK 0x07000000L
+#define PB0_PIF_PWRDOWN_0__PLL_RAMP_UP_TIME_0__SHIFT 0x00000018
+#define PB0_PIF_PWRDOWN_0__RX_POWER_STATE_IN_RXS2_0_MASK 0x00000070L
+#define PB0_PIF_PWRDOWN_0__RX_POWER_STATE_IN_RXS2_0__SHIFT 0x00000004
+#define PB0_PIF_PWRDOWN_0__TX2P5CLK_CLOCK_GATING_EN_0_MASK 0x00010000L
+#define PB0_PIF_PWRDOWN_0__TX2P5CLK_CLOCK_GATING_EN_0__SHIFT 0x00000010
+#define PB0_PIF_PWRDOWN_0__TX_POWER_STATE_IN_TXS2_0_MASK 0x00000007L
+#define PB0_PIF_PWRDOWN_0__TX_POWER_STATE_IN_TXS2_0__SHIFT 0x00000000
+#define PB0_PIF_PWRDOWN_1__FORCE_RXEN_IN_L0s_1_MASK 0x00000008L
+#define PB0_PIF_PWRDOWN_1__FORCE_RXEN_IN_L0s_1__SHIFT 0x00000003
+#define PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK 0x00001c00L
+#define PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT 0x0000000a
+#define PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK 0x00000380L
+#define PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT 0x00000007
+#define PB0_PIF_PWRDOWN_1__PLLPWR_OVERRIDE_EN_1_MASK 0x10000000L
+#define PB0_PIF_PWRDOWN_1__PLLPWR_OVERRIDE_EN_1__SHIFT 0x0000001c
+#define PB0_PIF_PWRDOWN_1__PLLPWR_OVERRIDE_VAL_1_MASK 0xe0000000L
+#define PB0_PIF_PWRDOWN_1__PLLPWR_OVERRIDE_VAL_1__SHIFT 0x0000001d
+#define PB0_PIF_PWRDOWN_1__PLL_RAMP_UP_TIME_1_MASK 0x07000000L
+#define PB0_PIF_PWRDOWN_1__PLL_RAMP_UP_TIME_1__SHIFT 0x00000018
+#define PB0_PIF_PWRDOWN_1__RX_POWER_STATE_IN_RXS2_1_MASK 0x00000070L
+#define PB0_PIF_PWRDOWN_1__RX_POWER_STATE_IN_RXS2_1__SHIFT 0x00000004
+#define PB0_PIF_PWRDOWN_1__TX2P5CLK_CLOCK_GATING_EN_1_MASK 0x00010000L
+#define PB0_PIF_PWRDOWN_1__TX2P5CLK_CLOCK_GATING_EN_1__SHIFT 0x00000010
+#define PB0_PIF_PWRDOWN_1__TX_POWER_STATE_IN_TXS2_1_MASK 0x00000007L
+#define PB0_PIF_PWRDOWN_1__TX_POWER_STATE_IN_TXS2_1__SHIFT 0x00000000
+#define PB0_PIF_PWRDOWN_2__FORCE_RXEN_IN_L0s_2_MASK 0x00000008L
+#define PB0_PIF_PWRDOWN_2__FORCE_RXEN_IN_L0s_2__SHIFT 0x00000003
+#define PB0_PIF_PWRDOWN_2__PLL_POWER_STATE_IN_OFF_2_MASK 0x00001c00L
+#define PB0_PIF_PWRDOWN_2__PLL_POWER_STATE_IN_OFF_2__SHIFT 0x0000000a
+#define PB0_PIF_PWRDOWN_2__PLL_POWER_STATE_IN_TXS2_2_MASK 0x00000380L
+#define PB0_PIF_PWRDOWN_2__PLL_POWER_STATE_IN_TXS2_2__SHIFT 0x00000007
+#define PB0_PIF_PWRDOWN_2__PLLPWR_OVERRIDE_EN_2_MASK 0x10000000L
+#define PB0_PIF_PWRDOWN_2__PLLPWR_OVERRIDE_EN_2__SHIFT 0x0000001c
+#define PB0_PIF_PWRDOWN_2__PLLPWR_OVERRIDE_VAL_2_MASK 0xe0000000L
+#define PB0_PIF_PWRDOWN_2__PLLPWR_OVERRIDE_VAL_2__SHIFT 0x0000001d
+#define PB0_PIF_PWRDOWN_2__PLL_RAMP_UP_TIME_2_MASK 0x07000000L
+#define PB0_PIF_PWRDOWN_2__PLL_RAMP_UP_TIME_2__SHIFT 0x00000018
+#define PB0_PIF_PWRDOWN_2__RX_POWER_STATE_IN_RXS2_2_MASK 0x00000070L
+#define PB0_PIF_PWRDOWN_2__RX_POWER_STATE_IN_RXS2_2__SHIFT 0x00000004
+#define PB0_PIF_PWRDOWN_2__TX2P5CLK_CLOCK_GATING_EN_2_MASK 0x00010000L
+#define PB0_PIF_PWRDOWN_2__TX2P5CLK_CLOCK_GATING_EN_2__SHIFT 0x00000010
+#define PB0_PIF_PWRDOWN_2__TX_POWER_STATE_IN_TXS2_2_MASK 0x00000007L
+#define PB0_PIF_PWRDOWN_2__TX_POWER_STATE_IN_TXS2_2__SHIFT 0x00000000
+#define PB0_PIF_PWRDOWN_3__FORCE_RXEN_IN_L0s_3_MASK 0x00000008L
+#define PB0_PIF_PWRDOWN_3__FORCE_RXEN_IN_L0s_3__SHIFT 0x00000003
+#define PB0_PIF_PWRDOWN_3__PLL_POWER_STATE_IN_OFF_3_MASK 0x00001c00L
+#define PB0_PIF_PWRDOWN_3__PLL_POWER_STATE_IN_OFF_3__SHIFT 0x0000000a
+#define PB0_PIF_PWRDOWN_3__PLL_POWER_STATE_IN_TXS2_3_MASK 0x00000380L
+#define PB0_PIF_PWRDOWN_3__PLL_POWER_STATE_IN_TXS2_3__SHIFT 0x00000007
+#define PB0_PIF_PWRDOWN_3__PLLPWR_OVERRIDE_EN_3_MASK 0x10000000L
+#define PB0_PIF_PWRDOWN_3__PLLPWR_OVERRIDE_EN_3__SHIFT 0x0000001c
+#define PB0_PIF_PWRDOWN_3__PLLPWR_OVERRIDE_VAL_3_MASK 0xe0000000L
+#define PB0_PIF_PWRDOWN_3__PLLPWR_OVERRIDE_VAL_3__SHIFT 0x0000001d
+#define PB0_PIF_PWRDOWN_3__PLL_RAMP_UP_TIME_3_MASK 0x07000000L
+#define PB0_PIF_PWRDOWN_3__PLL_RAMP_UP_TIME_3__SHIFT 0x00000018
+#define PB0_PIF_PWRDOWN_3__RX_POWER_STATE_IN_RXS2_3_MASK 0x00000070L
+#define PB0_PIF_PWRDOWN_3__RX_POWER_STATE_IN_RXS2_3__SHIFT 0x00000004
+#define PB0_PIF_PWRDOWN_3__TX2P5CLK_CLOCK_GATING_EN_3_MASK 0x00010000L
+#define PB0_PIF_PWRDOWN_3__TX2P5CLK_CLOCK_GATING_EN_3__SHIFT 0x00000010
+#define PB0_PIF_PWRDOWN_3__TX_POWER_STATE_IN_TXS2_3_MASK 0x00000007L
+#define PB0_PIF_PWRDOWN_3__TX_POWER_STATE_IN_TXS2_3__SHIFT 0x00000000
+#define PB0_PIF_SC_CTL__SC_CALIBRATION_MASK 0x00000001L
+#define PB0_PIF_SC_CTL__SC_CALIBRATION__SHIFT 0x00000000
+#define PB0_PIF_SC_CTL__SC_ENTER_L1_FROM_L0_MASK 0x00000020L
+#define PB0_PIF_SC_CTL__SC_ENTER_L1_FROM_L0__SHIFT 0x00000005
+#define PB0_PIF_SC_CTL__SC_ENTER_L1_FROM_L0S_MASK 0x00000010L
+#define PB0_PIF_SC_CTL__SC_ENTER_L1_FROM_L0S__SHIFT 0x00000004
+#define PB0_PIF_SC_CTL__SC_EXIT_L1_TO_L0_MASK 0x00000008L
+#define PB0_PIF_SC_CTL__SC_EXIT_L1_TO_L0__SHIFT 0x00000003
+#define PB0_PIF_SC_CTL__SC_EXIT_L1_TO_L0S_MASK 0x00000004L
+#define PB0_PIF_SC_CTL__SC_EXIT_L1_TO_L0S__SHIFT 0x00000002
+#define PB0_PIF_SC_CTL__SC_LANE_0_RESUME_MASK 0x00010000L
+#define PB0_PIF_SC_CTL__SC_LANE_0_RESUME__SHIFT 0x00000010
+#define PB0_PIF_SC_CTL__SC_LANE_10_RESUME_MASK 0x04000000L
+#define PB0_PIF_SC_CTL__SC_LANE_10_RESUME__SHIFT 0x0000001a
+#define PB0_PIF_SC_CTL__SC_LANE_11_RESUME_MASK 0x08000000L
+#define PB0_PIF_SC_CTL__SC_LANE_11_RESUME__SHIFT 0x0000001b
+#define PB0_PIF_SC_CTL__SC_LANE_12_RESUME_MASK 0x10000000L
+#define PB0_PIF_SC_CTL__SC_LANE_12_RESUME__SHIFT 0x0000001c
+#define PB0_PIF_SC_CTL__SC_LANE_13_RESUME_MASK 0x20000000L
+#define PB0_PIF_SC_CTL__SC_LANE_13_RESUME__SHIFT 0x0000001d
+#define PB0_PIF_SC_CTL__SC_LANE_14_RESUME_MASK 0x40000000L
+#define PB0_PIF_SC_CTL__SC_LANE_14_RESUME__SHIFT 0x0000001e
+#define PB0_PIF_SC_CTL__SC_LANE_15_RESUME_MASK 0x80000000L
+#define PB0_PIF_SC_CTL__SC_LANE_15_RESUME__SHIFT 0x0000001f
+#define PB0_PIF_SC_CTL__SC_LANE_1_RESUME_MASK 0x00020000L
+#define PB0_PIF_SC_CTL__SC_LANE_1_RESUME__SHIFT 0x00000011
+#define PB0_PIF_SC_CTL__SC_LANE_2_RESUME_MASK 0x00040000L
+#define PB0_PIF_SC_CTL__SC_LANE_2_RESUME__SHIFT 0x00000012
+#define PB0_PIF_SC_CTL__SC_LANE_3_RESUME_MASK 0x00080000L
+#define PB0_PIF_SC_CTL__SC_LANE_3_RESUME__SHIFT 0x00000013
+#define PB0_PIF_SC_CTL__SC_LANE_4_RESUME_MASK 0x00100000L
+#define PB0_PIF_SC_CTL__SC_LANE_4_RESUME__SHIFT 0x00000014
+#define PB0_PIF_SC_CTL__SC_LANE_5_RESUME_MASK 0x00200000L
+#define PB0_PIF_SC_CTL__SC_LANE_5_RESUME__SHIFT 0x00000015
+#define PB0_PIF_SC_CTL__SC_LANE_6_RESUME_MASK 0x00400000L
+#define PB0_PIF_SC_CTL__SC_LANE_6_RESUME__SHIFT 0x00000016
+#define PB0_PIF_SC_CTL__SC_LANE_7_RESUME_MASK 0x00800000L
+#define PB0_PIF_SC_CTL__SC_LANE_7_RESUME__SHIFT 0x00000017
+#define PB0_PIF_SC_CTL__SC_LANE_8_RESUME_MASK 0x01000000L
+#define PB0_PIF_SC_CTL__SC_LANE_8_RESUME__SHIFT 0x00000018
+#define PB0_PIF_SC_CTL__SC_LANE_9_RESUME_MASK 0x02000000L
+#define PB0_PIF_SC_CTL__SC_LANE_9_RESUME__SHIFT 0x00000019
+#define PB0_PIF_SC_CTL__SC_PHASE_1_MASK 0x00000100L
+#define PB0_PIF_SC_CTL__SC_PHASE_1__SHIFT 0x00000008
+#define PB0_PIF_SC_CTL__SC_PHASE_2_MASK 0x00000200L
+#define PB0_PIF_SC_CTL__SC_PHASE_2__SHIFT 0x00000009
+#define PB0_PIF_SC_CTL__SC_PHASE_3_MASK 0x00000400L
+#define PB0_PIF_SC_CTL__SC_PHASE_3__SHIFT 0x0000000a
+#define PB0_PIF_SC_CTL__SC_PHASE_4_MASK 0x00000800L
+#define PB0_PIF_SC_CTL__SC_PHASE_4__SHIFT 0x0000000b
+#define PB0_PIF_SC_CTL__SC_PHASE_5_MASK 0x00001000L
+#define PB0_PIF_SC_CTL__SC_PHASE_5__SHIFT 0x0000000c
+#define PB0_PIF_SC_CTL__SC_PHASE_6_MASK 0x00002000L
+#define PB0_PIF_SC_CTL__SC_PHASE_6__SHIFT 0x0000000d
+#define PB0_PIF_SC_CTL__SC_PHASE_7_MASK 0x00004000L
+#define PB0_PIF_SC_CTL__SC_PHASE_7__SHIFT 0x0000000e
+#define PB0_PIF_SC_CTL__SC_PHASE_8_MASK 0x00008000L
+#define PB0_PIF_SC_CTL__SC_PHASE_8__SHIFT 0x0000000f
+#define PB0_PIF_SC_CTL__SC_RXDETECT_MASK 0x00000002L
+#define PB0_PIF_SC_CTL__SC_RXDETECT__SHIFT 0x00000001
+#define PB0_PIF_SC_CTL__SC_SPEED_CHANGE_MASK 0x00000040L
+#define PB0_PIF_SC_CTL__SC_SPEED_CHANGE__SHIFT 0x00000006
+#define PB0_PIF_SCRATCH__PIF_SCRATCH_MASK 0xffffffffL
+#define PB0_PIF_SCRATCH__PIF_SCRATCH__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_0__SEQ_CALIBRATION_0_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_0__SEQ_CALIBRATION_0__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_0__SEQ_ENTER_L1_FROM_L0_0_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_0__SEQ_ENTER_L1_FROM_L0_0__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_0__SEQ_ENTER_L1_FROM_L0S_0_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_0__SEQ_ENTER_L1_FROM_L0S_0__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_0__SEQ_EXIT_L1_TO_L0_0_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_0__SEQ_EXIT_L1_TO_L0_0__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_0__SEQ_EXIT_L1_TO_L0S_0_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_0__SEQ_EXIT_L1_TO_L0S_0__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_0__SEQ_PHASE_0_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_0__SEQ_PHASE_0__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_0__SEQ_RXDETECT_0_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_0__SEQ_RXDETECT_0__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_0__SEQ_SPEED_CHANGE_0_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_0__SEQ_SPEED_CHANGE_0__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_10__SEQ_CALIBRATION_10_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_10__SEQ_CALIBRATION_10__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_10__SEQ_ENTER_L1_FROM_L0_10_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_10__SEQ_ENTER_L1_FROM_L0_10__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_10__SEQ_ENTER_L1_FROM_L0S_10_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_10__SEQ_ENTER_L1_FROM_L0S_10__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_10__SEQ_EXIT_L1_TO_L0_10_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_10__SEQ_EXIT_L1_TO_L0_10__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_10__SEQ_EXIT_L1_TO_L0S_10_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_10__SEQ_EXIT_L1_TO_L0S_10__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_10__SEQ_PHASE_10_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_10__SEQ_PHASE_10__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_10__SEQ_RXDETECT_10_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_10__SEQ_RXDETECT_10__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_10__SEQ_SPEED_CHANGE_10_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_10__SEQ_SPEED_CHANGE_10__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_11__SEQ_CALIBRATION_11_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_11__SEQ_CALIBRATION_11__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_11__SEQ_ENTER_L1_FROM_L0_11_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_11__SEQ_ENTER_L1_FROM_L0_11__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_11__SEQ_ENTER_L1_FROM_L0S_11_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_11__SEQ_ENTER_L1_FROM_L0S_11__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_11__SEQ_EXIT_L1_TO_L0_11_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_11__SEQ_EXIT_L1_TO_L0_11__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_11__SEQ_EXIT_L1_TO_L0S_11_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_11__SEQ_EXIT_L1_TO_L0S_11__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_11__SEQ_PHASE_11_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_11__SEQ_PHASE_11__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_11__SEQ_RXDETECT_11_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_11__SEQ_RXDETECT_11__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_11__SEQ_SPEED_CHANGE_11_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_11__SEQ_SPEED_CHANGE_11__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_12__SEQ_CALIBRATION_12_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_12__SEQ_CALIBRATION_12__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_12__SEQ_ENTER_L1_FROM_L0_12_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_12__SEQ_ENTER_L1_FROM_L0_12__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_12__SEQ_ENTER_L1_FROM_L0S_12_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_12__SEQ_ENTER_L1_FROM_L0S_12__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_12__SEQ_EXIT_L1_TO_L0_12_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_12__SEQ_EXIT_L1_TO_L0_12__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_12__SEQ_EXIT_L1_TO_L0S_12_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_12__SEQ_EXIT_L1_TO_L0S_12__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_12__SEQ_PHASE_12_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_12__SEQ_PHASE_12__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_12__SEQ_RXDETECT_12_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_12__SEQ_RXDETECT_12__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_12__SEQ_SPEED_CHANGE_12_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_12__SEQ_SPEED_CHANGE_12__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_13__SEQ_CALIBRATION_13_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_13__SEQ_CALIBRATION_13__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_13__SEQ_ENTER_L1_FROM_L0_13_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_13__SEQ_ENTER_L1_FROM_L0_13__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_13__SEQ_ENTER_L1_FROM_L0S_13_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_13__SEQ_ENTER_L1_FROM_L0S_13__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_13__SEQ_EXIT_L1_TO_L0_13_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_13__SEQ_EXIT_L1_TO_L0_13__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_13__SEQ_EXIT_L1_TO_L0S_13_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_13__SEQ_EXIT_L1_TO_L0S_13__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_13__SEQ_PHASE_13_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_13__SEQ_PHASE_13__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_13__SEQ_RXDETECT_13_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_13__SEQ_RXDETECT_13__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_13__SEQ_SPEED_CHANGE_13_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_13__SEQ_SPEED_CHANGE_13__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_14__SEQ_CALIBRATION_14_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_14__SEQ_CALIBRATION_14__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_14__SEQ_ENTER_L1_FROM_L0_14_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_14__SEQ_ENTER_L1_FROM_L0_14__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_14__SEQ_ENTER_L1_FROM_L0S_14_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_14__SEQ_ENTER_L1_FROM_L0S_14__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_14__SEQ_EXIT_L1_TO_L0_14_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_14__SEQ_EXIT_L1_TO_L0_14__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_14__SEQ_EXIT_L1_TO_L0S_14_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_14__SEQ_EXIT_L1_TO_L0S_14__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_14__SEQ_PHASE_14_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_14__SEQ_PHASE_14__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_14__SEQ_RXDETECT_14_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_14__SEQ_RXDETECT_14__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_14__SEQ_SPEED_CHANGE_14_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_14__SEQ_SPEED_CHANGE_14__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_15__SEQ_CALIBRATION_15_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_15__SEQ_CALIBRATION_15__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_15__SEQ_ENTER_L1_FROM_L0_15_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_15__SEQ_ENTER_L1_FROM_L0_15__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_15__SEQ_ENTER_L1_FROM_L0S_15_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_15__SEQ_ENTER_L1_FROM_L0S_15__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_15__SEQ_EXIT_L1_TO_L0_15_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_15__SEQ_EXIT_L1_TO_L0_15__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_15__SEQ_EXIT_L1_TO_L0S_15_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_15__SEQ_EXIT_L1_TO_L0S_15__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_15__SEQ_PHASE_15_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_15__SEQ_PHASE_15__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_15__SEQ_RXDETECT_15_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_15__SEQ_RXDETECT_15__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_15__SEQ_SPEED_CHANGE_15_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_15__SEQ_SPEED_CHANGE_15__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_1__SEQ_CALIBRATION_1_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_1__SEQ_CALIBRATION_1__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_1__SEQ_ENTER_L1_FROM_L0_1_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_1__SEQ_ENTER_L1_FROM_L0_1__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_1__SEQ_ENTER_L1_FROM_L0S_1_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_1__SEQ_ENTER_L1_FROM_L0S_1__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_1__SEQ_EXIT_L1_TO_L0_1_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_1__SEQ_EXIT_L1_TO_L0_1__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_1__SEQ_EXIT_L1_TO_L0S_1_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_1__SEQ_EXIT_L1_TO_L0S_1__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_1__SEQ_PHASE_1_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_1__SEQ_PHASE_1__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_1__SEQ_RXDETECT_1_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_1__SEQ_RXDETECT_1__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_1__SEQ_SPEED_CHANGE_1_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_1__SEQ_SPEED_CHANGE_1__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_2__SEQ_CALIBRATION_2_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_2__SEQ_CALIBRATION_2__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_2__SEQ_ENTER_L1_FROM_L0_2_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_2__SEQ_ENTER_L1_FROM_L0_2__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_2__SEQ_ENTER_L1_FROM_L0S_2_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_2__SEQ_ENTER_L1_FROM_L0S_2__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_2__SEQ_EXIT_L1_TO_L0_2_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_2__SEQ_EXIT_L1_TO_L0_2__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_2__SEQ_EXIT_L1_TO_L0S_2_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_2__SEQ_EXIT_L1_TO_L0S_2__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_2__SEQ_PHASE_2_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_2__SEQ_PHASE_2__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_2__SEQ_RXDETECT_2_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_2__SEQ_RXDETECT_2__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_2__SEQ_SPEED_CHANGE_2_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_2__SEQ_SPEED_CHANGE_2__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_3__SEQ_CALIBRATION_3_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_3__SEQ_CALIBRATION_3__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_3__SEQ_ENTER_L1_FROM_L0_3_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_3__SEQ_ENTER_L1_FROM_L0_3__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_3__SEQ_ENTER_L1_FROM_L0S_3_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_3__SEQ_ENTER_L1_FROM_L0S_3__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_3__SEQ_EXIT_L1_TO_L0_3_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_3__SEQ_EXIT_L1_TO_L0_3__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_3__SEQ_EXIT_L1_TO_L0S_3_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_3__SEQ_EXIT_L1_TO_L0S_3__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_3__SEQ_PHASE_3_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_3__SEQ_PHASE_3__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_3__SEQ_RXDETECT_3_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_3__SEQ_RXDETECT_3__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_3__SEQ_SPEED_CHANGE_3_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_3__SEQ_SPEED_CHANGE_3__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_4__SEQ_CALIBRATION_4_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_4__SEQ_CALIBRATION_4__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_4__SEQ_ENTER_L1_FROM_L0_4_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_4__SEQ_ENTER_L1_FROM_L0_4__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_4__SEQ_ENTER_L1_FROM_L0S_4_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_4__SEQ_ENTER_L1_FROM_L0S_4__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_4__SEQ_EXIT_L1_TO_L0_4_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_4__SEQ_EXIT_L1_TO_L0_4__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_4__SEQ_EXIT_L1_TO_L0S_4_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_4__SEQ_EXIT_L1_TO_L0S_4__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_4__SEQ_PHASE_4_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_4__SEQ_PHASE_4__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_4__SEQ_RXDETECT_4_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_4__SEQ_RXDETECT_4__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_4__SEQ_SPEED_CHANGE_4_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_4__SEQ_SPEED_CHANGE_4__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_5__SEQ_CALIBRATION_5_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_5__SEQ_CALIBRATION_5__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_5__SEQ_ENTER_L1_FROM_L0_5_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_5__SEQ_ENTER_L1_FROM_L0_5__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_5__SEQ_ENTER_L1_FROM_L0S_5_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_5__SEQ_ENTER_L1_FROM_L0S_5__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_5__SEQ_EXIT_L1_TO_L0_5_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_5__SEQ_EXIT_L1_TO_L0_5__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_5__SEQ_EXIT_L1_TO_L0S_5_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_5__SEQ_EXIT_L1_TO_L0S_5__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_5__SEQ_PHASE_5_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_5__SEQ_PHASE_5__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_5__SEQ_RXDETECT_5_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_5__SEQ_RXDETECT_5__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_5__SEQ_SPEED_CHANGE_5_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_5__SEQ_SPEED_CHANGE_5__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_6__SEQ_CALIBRATION_6_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_6__SEQ_CALIBRATION_6__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_6__SEQ_ENTER_L1_FROM_L0_6_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_6__SEQ_ENTER_L1_FROM_L0_6__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_6__SEQ_ENTER_L1_FROM_L0S_6_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_6__SEQ_ENTER_L1_FROM_L0S_6__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_6__SEQ_EXIT_L1_TO_L0_6_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_6__SEQ_EXIT_L1_TO_L0_6__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_6__SEQ_EXIT_L1_TO_L0S_6_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_6__SEQ_EXIT_L1_TO_L0S_6__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_6__SEQ_PHASE_6_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_6__SEQ_PHASE_6__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_6__SEQ_RXDETECT_6_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_6__SEQ_RXDETECT_6__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_6__SEQ_SPEED_CHANGE_6_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_6__SEQ_SPEED_CHANGE_6__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_7__SEQ_CALIBRATION_7_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_7__SEQ_CALIBRATION_7__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_7__SEQ_ENTER_L1_FROM_L0_7_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_7__SEQ_ENTER_L1_FROM_L0_7__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_7__SEQ_ENTER_L1_FROM_L0S_7_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_7__SEQ_ENTER_L1_FROM_L0S_7__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_7__SEQ_EXIT_L1_TO_L0_7_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_7__SEQ_EXIT_L1_TO_L0_7__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_7__SEQ_EXIT_L1_TO_L0S_7_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_7__SEQ_EXIT_L1_TO_L0S_7__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_7__SEQ_PHASE_7_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_7__SEQ_PHASE_7__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_7__SEQ_RXDETECT_7_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_7__SEQ_RXDETECT_7__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_7__SEQ_SPEED_CHANGE_7_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_7__SEQ_SPEED_CHANGE_7__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_8__SEQ_CALIBRATION_8_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_8__SEQ_CALIBRATION_8__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_8__SEQ_ENTER_L1_FROM_L0_8_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_8__SEQ_ENTER_L1_FROM_L0_8__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_8__SEQ_ENTER_L1_FROM_L0S_8_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_8__SEQ_ENTER_L1_FROM_L0S_8__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_8__SEQ_EXIT_L1_TO_L0_8_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_8__SEQ_EXIT_L1_TO_L0_8__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_8__SEQ_EXIT_L1_TO_L0S_8_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_8__SEQ_EXIT_L1_TO_L0S_8__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_8__SEQ_PHASE_8_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_8__SEQ_PHASE_8__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_8__SEQ_RXDETECT_8_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_8__SEQ_RXDETECT_8__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_8__SEQ_SPEED_CHANGE_8_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_8__SEQ_SPEED_CHANGE_8__SHIFT 0x00000006
+#define PB0_PIF_SEQ_STATUS_9__SEQ_CALIBRATION_9_MASK 0x00000001L
+#define PB0_PIF_SEQ_STATUS_9__SEQ_CALIBRATION_9__SHIFT 0x00000000
+#define PB0_PIF_SEQ_STATUS_9__SEQ_ENTER_L1_FROM_L0_9_MASK 0x00000020L
+#define PB0_PIF_SEQ_STATUS_9__SEQ_ENTER_L1_FROM_L0_9__SHIFT 0x00000005
+#define PB0_PIF_SEQ_STATUS_9__SEQ_ENTER_L1_FROM_L0S_9_MASK 0x00000010L
+#define PB0_PIF_SEQ_STATUS_9__SEQ_ENTER_L1_FROM_L0S_9__SHIFT 0x00000004
+#define PB0_PIF_SEQ_STATUS_9__SEQ_EXIT_L1_TO_L0_9_MASK 0x00000008L
+#define PB0_PIF_SEQ_STATUS_9__SEQ_EXIT_L1_TO_L0_9__SHIFT 0x00000003
+#define PB0_PIF_SEQ_STATUS_9__SEQ_EXIT_L1_TO_L0S_9_MASK 0x00000004L
+#define PB0_PIF_SEQ_STATUS_9__SEQ_EXIT_L1_TO_L0S_9__SHIFT 0x00000002
+#define PB0_PIF_SEQ_STATUS_9__SEQ_PHASE_9_MASK 0x00000700L
+#define PB0_PIF_SEQ_STATUS_9__SEQ_PHASE_9__SHIFT 0x00000008
+#define PB0_PIF_SEQ_STATUS_9__SEQ_RXDETECT_9_MASK 0x00000002L
+#define PB0_PIF_SEQ_STATUS_9__SEQ_RXDETECT_9__SHIFT 0x00000001
+#define PB0_PIF_SEQ_STATUS_9__SEQ_SPEED_CHANGE_9_MASK 0x00000040L
+#define PB0_PIF_SEQ_STATUS_9__SEQ_SPEED_CHANGE_9__SHIFT 0x00000006
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_0_MASK 0x00000001L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_0__SHIFT 0x00000000
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_10_MASK 0x00000400L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_10__SHIFT 0x0000000a
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_11_MASK 0x00000800L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_11__SHIFT 0x0000000b
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_12_MASK 0x00001000L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_12__SHIFT 0x0000000c
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_13_MASK 0x00002000L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_13__SHIFT 0x0000000d
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_14_MASK 0x00004000L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_14__SHIFT 0x0000000e
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_15_MASK 0x00008000L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_15__SHIFT 0x0000000f
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_1_MASK 0x00000002L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_1__SHIFT 0x00000001
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_2_MASK 0x00000004L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_2__SHIFT 0x00000002
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_3_MASK 0x00000008L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_3__SHIFT 0x00000003
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_4_MASK 0x00000010L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_4__SHIFT 0x00000004
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_5_MASK 0x00000020L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_5__SHIFT 0x00000005
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_6_MASK 0x00000040L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_6__SHIFT 0x00000006
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_7_MASK 0x00000080L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_7__SHIFT 0x00000007
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_8_MASK 0x00000100L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_8__SHIFT 0x00000008
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_9_MASK 0x00000200L
+#define PB0_PIF_TXPHYSTATUS__TXPHYSTATUS_9__SHIFT 0x00000009
+#define PB0_PLL_LC0_CTRL_REG0__PLL_DBG_LC_ANALOG_SEL_0_MASK 0x00000003L
+#define PB0_PLL_LC0_CTRL_REG0__PLL_DBG_LC_ANALOG_SEL_0__SHIFT 0x00000000
+#define PB0_PLL_LC0_CTRL_REG0__PLL_DBG_LC_EXT_RESET_EN_0_MASK 0x00000004L
+#define PB0_PLL_LC0_CTRL_REG0__PLL_DBG_LC_EXT_RESET_EN_0__SHIFT 0x00000002
+#define PB0_PLL_LC0_CTRL_REG0__PLL_DBG_LC_VCTL_ADC_EN_0_MASK 0x00000008L
+#define PB0_PLL_LC0_CTRL_REG0__PLL_DBG_LC_VCTL_ADC_EN_0__SHIFT 0x00000003
+#define PB0_PLL_LC0_CTRL_REG0__PLL_TST_LC_USAMPLE_EN_0_MASK 0x00000010L
+#define PB0_PLL_LC0_CTRL_REG0__PLL_TST_LC_USAMPLE_EN_0__SHIFT 0x00000004
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_BW_CNTRL_OVRD_EN_0_MASK 0x00000008L
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_BW_CNTRL_OVRD_EN_0__SHIFT 0x00000003
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_BW_CNTRL_OVRD_VAL_0_MASK 0x00000007L
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_BW_CNTRL_OVRD_VAL_0__SHIFT 0x00000000
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_DIV_OVRD_EN_0_MASK 0x00000080L
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_DIV_OVRD_EN_0__SHIFT 0x00000007
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_DIV_OVRD_VAL_0_MASK 0x00000070L
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_DIV_OVRD_VAL_0__SHIFT 0x00000004
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_EN_OVRD_EN_0_MASK 0x00000200L
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_EN_OVRD_EN_0__SHIFT 0x00000009
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_EN_OVRD_VAL_0_MASK 0x00000100L
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_EN_OVRD_VAL_0__SHIFT 0x00000008
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_FBDIV_OVRD_EN_0_MASK 0x00040000L
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_FBDIV_OVRD_EN_0__SHIFT 0x00000012
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_FBDIV_OVRD_VAL_0_MASK 0x0003fc00L
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_FBDIV_OVRD_VAL_0__SHIFT 0x0000000a
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_LF_CNTRL_OVRD_EN_0_MASK 0x10000000L
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_LF_CNTRL_OVRD_EN_0__SHIFT 0x0000001c
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_LF_CNTRL_OVRD_VAL_0_MASK 0x0ff80000L
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_LF_CNTRL_OVRD_VAL_0__SHIFT 0x00000013
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_REFDIV_OVRD_EN_0_MASK 0x80000000L
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_REFDIV_OVRD_EN_0__SHIFT 0x0000001f
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_REFDIV_OVRD_VAL_0_MASK 0x60000000L
+#define PB0_PLL_LC0_OVRD_REG0__PLL_CFG_LC_REFDIV_OVRD_VAL_0__SHIFT 0x0000001d
+#define PB0_PLL_LC0_OVRD_REG1__PLL_CFG_LC_REFCLK_SRC_OVRD_EN_0_MASK 0x00000008L
+#define PB0_PLL_LC0_OVRD_REG1__PLL_CFG_LC_REFCLK_SRC_OVRD_EN_0__SHIFT 0x00000003
+#define PB0_PLL_LC0_OVRD_REG1__PLL_CFG_LC_REFCLK_SRC_OVRD_VAL_0_MASK 0x00000007L
+#define PB0_PLL_LC0_OVRD_REG1__PLL_CFG_LC_REFCLK_SRC_OVRD_VAL_0__SHIFT 0x00000000
+#define PB0_PLL_LC0_OVRD_REG1__PLL_CFG_LC_VCO_TUNE_OVRD_EN_0_MASK 0x00040000L
+#define PB0_PLL_LC0_OVRD_REG1__PLL_CFG_LC_VCO_TUNE_OVRD_EN_0__SHIFT 0x00000012
+#define PB0_PLL_LC0_OVRD_REG1__PLL_CFG_LC_VCO_TUNE_OVRD_VAL_0_MASK 0x0003c000L
+#define PB0_PLL_LC0_OVRD_REG1__PLL_CFG_LC_VCO_TUNE_OVRD_VAL_0__SHIFT 0x0000000e
+#define PB0_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_LEFT_EN_OVRD_EN_0_MASK 0x00000020L
+#define PB0_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_LEFT_EN_OVRD_EN_0__SHIFT 0x00000005
+#define PB0_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_LEFT_EN_OVRD_VAL_0_MASK 0x00000010L
+#define PB0_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_LEFT_EN_OVRD_VAL_0__SHIFT 0x00000004
+#define PB0_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_RIGHT_EN_OVRD_EN_0_MASK 0x00000080L
+#define PB0_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_RIGHT_EN_OVRD_EN_0__SHIFT 0x00000007
+#define PB0_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_RIGHT_EN_OVRD_VAL_0_MASK 0x00000040L
+#define PB0_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_RIGHT_EN_OVRD_VAL_0__SHIFT 0x00000006
+#define PB0_PLL_LC0_OVRD_REG1__PLL_LC_PWRON_OVRD_EN_0_MASK 0x00000200L
+#define PB0_PLL_LC0_OVRD_REG1__PLL_LC_PWRON_OVRD_EN_0__SHIFT 0x00000009
+#define PB0_PLL_LC0_OVRD_REG1__PLL_LC_PWRON_OVRD_VAL_0_MASK 0x00000100L
+#define PB0_PLL_LC0_OVRD_REG1__PLL_LC_PWRON_OVRD_VAL_0__SHIFT 0x00000008
+#define PB0_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_FREQMODE_MASK 0x00000300L
+#define PB0_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_FREQMODE__SHIFT 0x00000008
+#define PB0_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB0_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB0_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB0_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB0_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_PLLPWR_MASK 0x00000070L
+#define PB0_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_PLLPWR__SHIFT 0x00000004
+#define PB0_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_FREQMODE_MASK 0x00000300L
+#define PB0_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_FREQMODE__SHIFT 0x00000008
+#define PB0_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB0_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB0_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB0_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB0_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_PLLPWR_MASK 0x00000070L
+#define PB0_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_PLLPWR__SHIFT 0x00000004
+#define PB0_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_FREQMODE_MASK 0x00000300L
+#define PB0_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_FREQMODE__SHIFT 0x00000008
+#define PB0_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB0_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB0_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB0_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB0_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_PLLPWR_MASK 0x00000070L
+#define PB0_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_PLLPWR__SHIFT 0x00000004
+#define PB0_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_FREQMODE_MASK 0x00000300L
+#define PB0_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_FREQMODE__SHIFT 0x00000008
+#define PB0_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB0_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB0_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB0_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB0_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_PLLPWR_MASK 0x00000070L
+#define PB0_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_PLLPWR__SHIFT 0x00000004
+#define PB0_PLL_RO0_CTRL_REG0__PLL_DBG_RO_ANALOG_SEL_0_MASK 0x00000003L
+#define PB0_PLL_RO0_CTRL_REG0__PLL_DBG_RO_ANALOG_SEL_0__SHIFT 0x00000000
+#define PB0_PLL_RO0_CTRL_REG0__PLL_DBG_RO_EXT_RESET_EN_0_MASK 0x00000004L
+#define PB0_PLL_RO0_CTRL_REG0__PLL_DBG_RO_EXT_RESET_EN_0__SHIFT 0x00000002
+#define PB0_PLL_RO0_CTRL_REG0__PLL_DBG_RO_LF_CNTRL_0_MASK 0x000007f0L
+#define PB0_PLL_RO0_CTRL_REG0__PLL_DBG_RO_LF_CNTRL_0__SHIFT 0x00000004
+#define PB0_PLL_RO0_CTRL_REG0__PLL_DBG_RO_VCTL_ADC_EN_0_MASK 0x00000008L
+#define PB0_PLL_RO0_CTRL_REG0__PLL_DBG_RO_VCTL_ADC_EN_0__SHIFT 0x00000003
+#define PB0_PLL_RO0_CTRL_REG0__PLL_TST_RO_USAMPLE_EN_0_MASK 0x00000800L
+#define PB0_PLL_RO0_CTRL_REG0__PLL_TST_RO_USAMPLE_EN_0__SHIFT 0x0000000b
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_BW_CNTRL_OVRD_EN_0_MASK 0x00000100L
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_BW_CNTRL_OVRD_EN_0__SHIFT 0x00000008
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_BW_CNTRL_OVRD_VAL_0_MASK 0x000000ffL
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_BW_CNTRL_OVRD_VAL_0__SHIFT 0x00000000
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_DIV_OVRD_EN_0_MASK 0x00001000L
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_DIV_OVRD_EN_0__SHIFT 0x0000000c
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_DIV_OVRD_VAL_0_MASK 0x00000e00L
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_DIV_OVRD_VAL_0__SHIFT 0x00000009
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_EN_OVRD_EN_0_MASK 0x00004000L
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_EN_OVRD_EN_0__SHIFT 0x0000000e
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_EN_OVRD_VAL_0_MASK 0x00002000L
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_EN_OVRD_VAL_0__SHIFT 0x0000000d
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_FBDIV_OVRD_EN_0_MASK 0x10000000L
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_FBDIV_OVRD_EN_0__SHIFT 0x0000001c
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_FBDIV_OVRD_VAL_0_MASK 0x0fff8000L
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_FBDIV_OVRD_VAL_0__SHIFT 0x0000000f
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_VTOI_BIAS_CNTRL_OVRD_EN_0_MASK 0x80000000L
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_VTOI_BIAS_CNTRL_OVRD_EN_0__SHIFT 0x0000001f
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_VTOI_BIAS_CNTRL_OVRD_VAL_0_MASK 0x40000000L
+#define PB0_PLL_RO0_OVRD_REG0__PLL_CFG_RO_VTOI_BIAS_CNTRL_OVRD_VAL_0__SHIFT 0x0000001e
+#define PB0_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFCLK_SRC_OVRD_EN_0_MASK 0x00400000L
+#define PB0_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFCLK_SRC_OVRD_EN_0__SHIFT 0x00000016
+#define PB0_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFCLK_SRC_OVRD_VAL_0_MASK 0x00380000L
+#define PB0_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFCLK_SRC_OVRD_VAL_0__SHIFT 0x00000013
+#define PB0_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFDIV_OVRD_EN_0_MASK 0x00000020L
+#define PB0_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFDIV_OVRD_EN_0__SHIFT 0x00000005
+#define PB0_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFDIV_OVRD_VAL_0_MASK 0x0000001fL
+#define PB0_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFDIV_OVRD_VAL_0__SHIFT 0x00000000
+#define PB0_PLL_RO0_OVRD_REG1__PLL_CFG_RO_VCO_MODE_OVRD_EN_0_MASK 0x00000100L
+#define PB0_PLL_RO0_OVRD_REG1__PLL_CFG_RO_VCO_MODE_OVRD_EN_0__SHIFT 0x00000008
+#define PB0_PLL_RO0_OVRD_REG1__PLL_CFG_RO_VCO_MODE_OVRD_VAL_0_MASK 0x000000c0L
+#define PB0_PLL_RO0_OVRD_REG1__PLL_CFG_RO_VCO_MODE_OVRD_VAL_0__SHIFT 0x00000006
+#define PB0_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_LEFT_EN_OVRD_EN_0_MASK 0x00000400L
+#define PB0_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_LEFT_EN_OVRD_EN_0__SHIFT 0x0000000a
+#define PB0_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_LEFT_EN_OVRD_VAL_0_MASK 0x00000200L
+#define PB0_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_LEFT_EN_OVRD_VAL_0__SHIFT 0x00000009
+#define PB0_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_RIGHT_EN_OVRD_EN_0_MASK 0x00001000L
+#define PB0_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_RIGHT_EN_OVRD_EN_0__SHIFT 0x0000000c
+#define PB0_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_RIGHT_EN_OVRD_VAL_0_MASK 0x00000800L
+#define PB0_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_RIGHT_EN_OVRD_VAL_0__SHIFT 0x0000000b
+#define PB0_PLL_RO0_OVRD_REG1__PLL_RO_PWRON_OVRD_EN_0_MASK 0x00004000L
+#define PB0_PLL_RO0_OVRD_REG1__PLL_RO_PWRON_OVRD_EN_0__SHIFT 0x0000000e
+#define PB0_PLL_RO0_OVRD_REG1__PLL_RO_PWRON_OVRD_VAL_0_MASK 0x00002000L
+#define PB0_PLL_RO0_OVRD_REG1__PLL_RO_PWRON_OVRD_VAL_0__SHIFT 0x0000000d
+#define PB0_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_FREQMODE_MASK 0x00000300L
+#define PB0_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_FREQMODE__SHIFT 0x00000008
+#define PB0_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB0_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB0_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB0_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB0_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_PLLPWR_MASK 0x00000070L
+#define PB0_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_PLLPWR__SHIFT 0x00000004
+#define PB0_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_FREQMODE_MASK 0x00000300L
+#define PB0_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_FREQMODE__SHIFT 0x00000008
+#define PB0_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB0_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB0_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB0_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB0_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_PLLPWR_MASK 0x00000070L
+#define PB0_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_PLLPWR__SHIFT 0x00000004
+#define PB0_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_FREQMODE_MASK 0x00000300L
+#define PB0_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_FREQMODE__SHIFT 0x00000008
+#define PB0_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB0_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB0_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB0_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB0_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_PLLPWR_MASK 0x00000070L
+#define PB0_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_PLLPWR__SHIFT 0x00000004
+#define PB0_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_FREQMODE_MASK 0x00000300L
+#define PB0_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_FREQMODE__SHIFT 0x00000008
+#define PB0_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB0_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB0_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB0_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB0_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_PLLPWR_MASK 0x00000070L
+#define PB0_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_PLLPWR__SHIFT 0x00000004
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_EN_LUT_ENTRY_LS0_MASK 0x00000200L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_EN_LUT_ENTRY_LS0__SHIFT 0x00000009
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_EN_LUT_ENTRY_LS1_MASK 0x00000400L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_EN_LUT_ENTRY_LS1__SHIFT 0x0000000a
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_EN_LUT_ENTRY_LS2_MASK 0x00000800L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_EN_LUT_ENTRY_LS2__SHIFT 0x0000000b
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_LEFT_EN_GATING_EN_MASK 0x00100000L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_LEFT_EN_GATING_EN__SHIFT 0x00000014
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_RIGHT_EN_GATING_EN_MASK 0x00200000L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_RIGHT_EN_GATING_EN__SHIFT 0x00000015
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_EN_LUT_ENTRY_LS0_MASK 0x00001000L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_EN_LUT_ENTRY_LS0__SHIFT 0x0000000c
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_EN_LUT_ENTRY_LS1_MASK 0x00002000L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_EN_LUT_ENTRY_LS1__SHIFT 0x0000000d
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_EN_LUT_ENTRY_LS2_MASK 0x00004000L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_EN_LUT_ENTRY_LS2__SHIFT 0x0000000e
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_LEFT_EN_GATING_EN_MASK 0x00400000L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_LEFT_EN_GATING_EN__SHIFT 0x00000016
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_RIGHT_EN_GATING_EN_MASK 0x00800000L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_RIGHT_EN_GATING_EN__SHIFT 0x00000017
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_PWRON_LUT_ENTRY_LS2_MASK 0x00000100L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_LC_PWRON_LUT_ENTRY_LS2__SHIFT 0x00000008
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_EN_LUT_ENTRY_LS0_MASK 0x00000002L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_EN_LUT_ENTRY_LS0__SHIFT 0x00000001
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_EN_LUT_ENTRY_LS1_MASK 0x00000004L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_EN_LUT_ENTRY_LS1__SHIFT 0x00000002
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_EN_LUT_ENTRY_LS2_MASK 0x00000008L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_EN_LUT_ENTRY_LS2__SHIFT 0x00000003
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_LEFT_EN_GATING_EN_MASK 0x00010000L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_LEFT_EN_GATING_EN__SHIFT 0x00000010
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_RIGHT_EN_GATING_EN_MASK 0x00020000L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_RIGHT_EN_GATING_EN__SHIFT 0x00000011
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_EN_LUT_ENTRY_LS0_MASK 0x00000010L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_EN_LUT_ENTRY_LS0__SHIFT 0x00000004
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_EN_LUT_ENTRY_LS1_MASK 0x00000020L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_EN_LUT_ENTRY_LS1__SHIFT 0x00000005
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_EN_LUT_ENTRY_LS2_MASK 0x00000040L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_EN_LUT_ENTRY_LS2__SHIFT 0x00000006
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_LEFT_EN_GATING_EN_MASK 0x00040000L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_LEFT_EN_GATING_EN__SHIFT 0x00000012
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_RIGHT_EN_GATING_EN_MASK 0x00080000L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_RIGHT_EN_GATING_EN__SHIFT 0x00000013
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_PWRON_LUT_ENTRY_LS2_MASK 0x00000080L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_RO_PWRON_LUT_ENTRY_LS2__SHIFT 0x00000007
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_TST_LOSPDTST_SRC_MASK 0x00000001L
+#define PB0_PLL_RO_GLB_CTRL_REG0__PLL_TST_LOSPDTST_SRC__SHIFT 0x00000000
+#define PB0_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_MODE_GEN1_MASK 0x000003ffL
+#define PB0_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_MODE_GEN1__SHIFT 0x00000000
+#define PB0_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_MODE_GEN2_MASK 0x000ffc00L
+#define PB0_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_MODE_GEN2__SHIFT 0x0000000a
+#define PB0_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_MODE_GEN3_MASK 0x3ff00000L
+#define PB0_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_MODE_GEN3__SHIFT 0x00000014
+#define PB0_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_RST_MODE_MASK 0xc0000000L
+#define PB0_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_RST_MODE__SHIFT 0x0000001e
+#define PB0_RX_GLB_CTRL_REG1__RX_ADAPT_HLD_ASRT_TO_DCLK_EN_MASK 0xc0000000L
+#define PB0_RX_GLB_CTRL_REG1__RX_ADAPT_HLD_ASRT_TO_DCLK_EN__SHIFT 0x0000001e
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_FR_GAIN_GEN1_MASK 0x0000000fL
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_FR_GAIN_GEN1__SHIFT 0x00000000
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_FR_GAIN_GEN2_MASK 0x000000f0L
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_FR_GAIN_GEN2__SHIFT 0x00000004
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_FR_GAIN_GEN3_MASK 0x00000f00L
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_FR_GAIN_GEN3__SHIFT 0x00000008
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_PH_GAIN_GEN1_MASK 0x0000f000L
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_PH_GAIN_GEN1__SHIFT 0x0000000c
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_PH_GAIN_GEN2_MASK 0x000f0000L
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_PH_GAIN_GEN2__SHIFT 0x00000010
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_PH_GAIN_GEN3_MASK 0x00f00000L
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_PH_GAIN_GEN3__SHIFT 0x00000014
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_PI_STPSZ_GEN1_MASK 0x01000000L
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_PI_STPSZ_GEN1__SHIFT 0x00000018
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_PI_STPSZ_GEN2_MASK 0x02000000L
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_PI_STPSZ_GEN2__SHIFT 0x00000019
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_PI_STPSZ_GEN3_MASK 0x04000000L
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_CDR_PI_STPSZ_GEN3__SHIFT 0x0000001a
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_LEQ_DCATTN_BYP_EN_GEN1_MASK 0x08000000L
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_LEQ_DCATTN_BYP_EN_GEN1__SHIFT 0x0000001b
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_LEQ_DCATTN_BYP_EN_GEN2_MASK 0x10000000L
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_LEQ_DCATTN_BYP_EN_GEN2__SHIFT 0x0000001c
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_LEQ_DCATTN_BYP_EN_GEN3_MASK 0x20000000L
+#define PB0_RX_GLB_CTRL_REG1__RX_CFG_LEQ_DCATTN_BYP_EN_GEN3__SHIFT 0x0000001d
+#define PB0_RX_GLB_CTRL_REG2__RX_CFG_CDR_TIME_GEN1_MASK 0x0000f000L
+#define PB0_RX_GLB_CTRL_REG2__RX_CFG_CDR_TIME_GEN1__SHIFT 0x0000000c
+#define PB0_RX_GLB_CTRL_REG2__RX_CFG_CDR_TIME_GEN2_MASK 0x000f0000L
+#define PB0_RX_GLB_CTRL_REG2__RX_CFG_CDR_TIME_GEN2__SHIFT 0x00000010
+#define PB0_RX_GLB_CTRL_REG2__RX_CFG_CDR_TIME_GEN3_MASK 0x00f00000L
+#define PB0_RX_GLB_CTRL_REG2__RX_CFG_CDR_TIME_GEN3__SHIFT 0x00000014
+#define PB0_RX_GLB_CTRL_REG2__RX_CFG_LEQ_LOOP_GAIN_GEN1_MASK 0x03000000L
+#define PB0_RX_GLB_CTRL_REG2__RX_CFG_LEQ_LOOP_GAIN_GEN1__SHIFT 0x00000018
+#define PB0_RX_GLB_CTRL_REG2__RX_CFG_LEQ_LOOP_GAIN_GEN2_MASK 0x0c000000L
+#define PB0_RX_GLB_CTRL_REG2__RX_CFG_LEQ_LOOP_GAIN_GEN2__SHIFT 0x0000001a
+#define PB0_RX_GLB_CTRL_REG2__RX_CFG_LEQ_LOOP_GAIN_GEN3_MASK 0x30000000L
+#define PB0_RX_GLB_CTRL_REG2__RX_CFG_LEQ_LOOP_GAIN_GEN3__SHIFT 0x0000001c
+#define PB0_RX_GLB_CTRL_REG2__RX_DCLK_EN_ASRT_TO_ADAPT_HLD_MASK 0xc0000000L
+#define PB0_RX_GLB_CTRL_REG2__RX_DCLK_EN_ASRT_TO_ADAPT_HLD__SHIFT 0x0000001e
+#define PB0_RX_GLB_CTRL_REG3__RX_CFG_CDR_FR_EN_GEN1_MASK 0x00000001L
+#define PB0_RX_GLB_CTRL_REG3__RX_CFG_CDR_FR_EN_GEN1__SHIFT 0x00000000
+#define PB0_RX_GLB_CTRL_REG3__RX_CFG_CDR_FR_EN_GEN2_MASK 0x00000002L
+#define PB0_RX_GLB_CTRL_REG3__RX_CFG_CDR_FR_EN_GEN2__SHIFT 0x00000001
+#define PB0_RX_GLB_CTRL_REG3__RX_CFG_CDR_FR_EN_GEN3_MASK 0x00000004L
+#define PB0_RX_GLB_CTRL_REG3__RX_CFG_CDR_FR_EN_GEN3__SHIFT 0x00000002
+#define PB0_RX_GLB_CTRL_REG3__RX_CFG_DFE_TIME_GEN1_MASK 0x00f00000L
+#define PB0_RX_GLB_CTRL_REG3__RX_CFG_DFE_TIME_GEN1__SHIFT 0x00000014
+#define PB0_RX_GLB_CTRL_REG3__RX_CFG_DFE_TIME_GEN2_MASK 0x0f000000L
+#define PB0_RX_GLB_CTRL_REG3__RX_CFG_DFE_TIME_GEN2__SHIFT 0x00000018
+#define PB0_RX_GLB_CTRL_REG3__RX_CFG_DFE_TIME_GEN3_MASK 0xf0000000L
+#define PB0_RX_GLB_CTRL_REG3__RX_CFG_DFE_TIME_GEN3__SHIFT 0x0000001c
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_FOM_BER_GEN1_MASK 0x00000007L
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_FOM_BER_GEN1__SHIFT 0x00000000
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_FOM_BER_GEN2_MASK 0x00000038L
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_FOM_BER_GEN2__SHIFT 0x00000003
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_FOM_BER_GEN3_MASK 0x000001c0L
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_FOM_BER_GEN3__SHIFT 0x00000006
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_FOM_TIME_GEN1_MASK 0x00f00000L
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_FOM_TIME_GEN1__SHIFT 0x00000014
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_FOM_TIME_GEN2_MASK 0x0f000000L
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_FOM_TIME_GEN2__SHIFT 0x00000018
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_FOM_TIME_GEN3_MASK 0xf0000000L
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_FOM_TIME_GEN3__SHIFT 0x0000001c
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_LEQ_POLE_BYP_VAL_GEN1_MASK 0x00000e00L
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_LEQ_POLE_BYP_VAL_GEN1__SHIFT 0x00000009
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_LEQ_POLE_BYP_VAL_GEN2_MASK 0x00007000L
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_LEQ_POLE_BYP_VAL_GEN2__SHIFT 0x0000000c
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_LEQ_POLE_BYP_VAL_GEN3_MASK 0x00038000L
+#define PB0_RX_GLB_CTRL_REG4__RX_CFG_LEQ_POLE_BYP_VAL_GEN3__SHIFT 0x0000000f
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_DCATTN_BYP_VAL_GEN1_MASK 0x0000001fL
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_DCATTN_BYP_VAL_GEN1__SHIFT 0x00000000
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_DCATTN_BYP_VAL_GEN2_MASK 0x000003e0L
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_DCATTN_BYP_VAL_GEN2__SHIFT 0x00000005
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_DCATTN_BYP_VAL_GEN3_MASK 0x00007c00L
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_DCATTN_BYP_VAL_GEN3__SHIFT 0x0000000a
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_POLE_BYP_EN_GEN1_MASK 0x00008000L
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_POLE_BYP_EN_GEN1__SHIFT 0x0000000f
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_POLE_BYP_EN_GEN2_MASK 0x00010000L
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_POLE_BYP_EN_GEN2__SHIFT 0x00000010
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_POLE_BYP_EN_GEN3_MASK 0x00020000L
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_POLE_BYP_EN_GEN3__SHIFT 0x00000011
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_SHUNT_EN_GEN1_MASK 0x00040000L
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_SHUNT_EN_GEN1__SHIFT 0x00000012
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_SHUNT_EN_GEN2_MASK 0x00080000L
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_SHUNT_EN_GEN2__SHIFT 0x00000013
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_SHUNT_EN_GEN3_MASK 0x00100000L
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_LEQ_SHUNT_EN_GEN3__SHIFT 0x00000014
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_TERM_MODE_GEN1_MASK 0x08000000L
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_TERM_MODE_GEN1__SHIFT 0x0000001b
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_TERM_MODE_GEN2_MASK 0x10000000L
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_TERM_MODE_GEN2__SHIFT 0x0000001c
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_TERM_MODE_GEN3_MASK 0x20000000L
+#define PB0_RX_GLB_CTRL_REG5__RX_CFG_TERM_MODE_GEN3__SHIFT 0x0000001d
+#define PB0_RX_GLB_CTRL_REG5__RX_FORCE_DLL_RST_RXPWR_LS2OFF_TO_LS0_MASK 0x40000000L
+#define PB0_RX_GLB_CTRL_REG5__RX_FORCE_DLL_RST_RXPWR_LS2OFF_TO_LS0__SHIFT 0x0000001e
+#define PB0_RX_GLB_CTRL_REG6__RX_AUX_PWRON_LUT_ENTRY_LS2_MASK 0x08000000L
+#define PB0_RX_GLB_CTRL_REG6__RX_AUX_PWRON_LUT_ENTRY_LS2__SHIFT 0x0000001b
+#define PB0_RX_GLB_CTRL_REG6__RX_CFG_LEQ_TIME_GEN1_MASK 0x0000000fL
+#define PB0_RX_GLB_CTRL_REG6__RX_CFG_LEQ_TIME_GEN1__SHIFT 0x00000000
+#define PB0_RX_GLB_CTRL_REG6__RX_CFG_LEQ_TIME_GEN2_MASK 0x000000f0L
+#define PB0_RX_GLB_CTRL_REG6__RX_CFG_LEQ_TIME_GEN2__SHIFT 0x00000004
+#define PB0_RX_GLB_CTRL_REG6__RX_CFG_LEQ_TIME_GEN3_MASK 0x00000f00L
+#define PB0_RX_GLB_CTRL_REG6__RX_CFG_LEQ_TIME_GEN3__SHIFT 0x00000008
+#define PB0_RX_GLB_CTRL_REG6__RX_CFG_OC_TIME_GEN1_MASK 0x0000f000L
+#define PB0_RX_GLB_CTRL_REG6__RX_CFG_OC_TIME_GEN1__SHIFT 0x0000000c
+#define PB0_RX_GLB_CTRL_REG6__RX_CFG_OC_TIME_GEN2_MASK 0x000f0000L
+#define PB0_RX_GLB_CTRL_REG6__RX_CFG_OC_TIME_GEN2__SHIFT 0x00000010
+#define PB0_RX_GLB_CTRL_REG6__RX_CFG_OC_TIME_GEN3_MASK 0x00f00000L
+#define PB0_RX_GLB_CTRL_REG6__RX_CFG_OC_TIME_GEN3__SHIFT 0x00000014
+#define PB0_RX_GLB_CTRL_REG6__RX_FRONTEND_PWRON_LUT_ENTRY_LS0_CDR_EN_0_MASK 0x01000000L
+#define PB0_RX_GLB_CTRL_REG6__RX_FRONTEND_PWRON_LUT_ENTRY_LS0_CDR_EN_0__SHIFT 0x00000018
+#define PB0_RX_GLB_CTRL_REG6__RX_FRONTEND_PWRON_LUT_ENTRY_LS2_MASK 0x04000000L
+#define PB0_RX_GLB_CTRL_REG6__RX_FRONTEND_PWRON_LUT_ENTRY_LS2__SHIFT 0x0000001a
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_DLL_CPI_SEL_GEN1_MASK 0x001c0000L
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_DLL_CPI_SEL_GEN1__SHIFT 0x00000012
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_DLL_CPI_SEL_GEN2_MASK 0x00e00000L
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_DLL_CPI_SEL_GEN2__SHIFT 0x00000015
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_DLL_CPI_SEL_GEN3_MASK 0x07000000L
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_DLL_CPI_SEL_GEN3__SHIFT 0x00000018
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_DLL_FLOCK_DISABLE_GEN1_MASK 0x08000000L
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_DLL_FLOCK_DISABLE_GEN1__SHIFT 0x0000001b
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_DLL_FLOCK_DISABLE_GEN2_MASK 0x10000000L
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_DLL_FLOCK_DISABLE_GEN2__SHIFT 0x0000001c
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_DLL_FLOCK_DISABLE_GEN3_MASK 0x20000000L
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_DLL_FLOCK_DISABLE_GEN3__SHIFT 0x0000001d
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_TH_LOOP_GAIN_GEN1_MASK 0x0000000fL
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_TH_LOOP_GAIN_GEN1__SHIFT 0x00000000
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_TH_LOOP_GAIN_GEN2_MASK 0x000000f0L
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_TH_LOOP_GAIN_GEN2__SHIFT 0x00000004
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_TH_LOOP_GAIN_GEN3_MASK 0x00000f00L
+#define PB0_RX_GLB_CTRL_REG7__RX_CFG_TH_LOOP_GAIN_GEN3__SHIFT 0x00000008
+#define PB0_RX_GLB_CTRL_REG7__RX_DCLK_EN_LUT_ENTRY_LS0_CDR_EN_0_MASK 0x00001000L
+#define PB0_RX_GLB_CTRL_REG7__RX_DCLK_EN_LUT_ENTRY_LS0_CDR_EN_0__SHIFT 0x0000000c
+#define PB0_RX_GLB_CTRL_REG7__RX_DCLK_EN_LUT_ENTRY_LS2_MASK 0x00002000L
+#define PB0_RX_GLB_CTRL_REG7__RX_DCLK_EN_LUT_ENTRY_LS2__SHIFT 0x0000000d
+#define PB0_RX_GLB_CTRL_REG7__RX_DLL_PWRON_LUT_ENTRY_LS2_MASK 0x00020000L
+#define PB0_RX_GLB_CTRL_REG7__RX_DLL_PWRON_LUT_ENTRY_LS2__SHIFT 0x00000011
+#define PB0_RX_GLB_OVRD_REG0__RX_ADAPT_FOM_OVRD_EN_MASK 0x80000000L
+#define PB0_RX_GLB_OVRD_REG0__RX_ADAPT_FOM_OVRD_EN__SHIFT 0x0000001f
+#define PB0_RX_GLB_OVRD_REG0__RX_ADAPT_FOM_OVRD_VAL_MASK 0x40000000L
+#define PB0_RX_GLB_OVRD_REG0__RX_ADAPT_FOM_OVRD_VAL__SHIFT 0x0000001e
+#define PB0_RX_GLB_OVRD_REG0__RX_ADAPT_HLD_OVRD_EN_MASK 0x00000002L
+#define PB0_RX_GLB_OVRD_REG0__RX_ADAPT_HLD_OVRD_EN__SHIFT 0x00000001
+#define PB0_RX_GLB_OVRD_REG0__RX_ADAPT_HLD_OVRD_VAL_MASK 0x00000001L
+#define PB0_RX_GLB_OVRD_REG0__RX_ADAPT_HLD_OVRD_VAL__SHIFT 0x00000000
+#define PB0_RX_GLB_OVRD_REG0__RX_ADAPT_RST_OVRD_EN_MASK 0x00000008L
+#define PB0_RX_GLB_OVRD_REG0__RX_ADAPT_RST_OVRD_EN__SHIFT 0x00000003
+#define PB0_RX_GLB_OVRD_REG0__RX_ADAPT_RST_OVRD_VAL_MASK 0x00000004L
+#define PB0_RX_GLB_OVRD_REG0__RX_ADAPT_RST_OVRD_VAL__SHIFT 0x00000002
+#define PB0_RX_GLB_OVRD_REG0__RX_AUX_PWRON_OVRD_EN_MASK 0x20000000L
+#define PB0_RX_GLB_OVRD_REG0__RX_AUX_PWRON_OVRD_EN__SHIFT 0x0000001d
+#define PB0_RX_GLB_OVRD_REG0__RX_AUX_PWRON_OVRD_VAL_MASK 0x10000000L
+#define PB0_RX_GLB_OVRD_REG0__RX_AUX_PWRON_OVRD_VAL__SHIFT 0x0000001c
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_DCLK_DIV_OVRD_EN_MASK 0x00000100L
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_DCLK_DIV_OVRD_EN__SHIFT 0x00000008
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_DCLK_DIV_OVRD_VAL_MASK 0x000000c0L
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_DCLK_DIV_OVRD_VAL__SHIFT 0x00000006
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_DLL_FREQ_MODE_OVRD_EN_MASK 0x00000400L
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_DLL_FREQ_MODE_OVRD_EN__SHIFT 0x0000000a
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_DLL_FREQ_MODE_OVRD_VAL_MASK 0x00000200L
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_DLL_FREQ_MODE_OVRD_VAL__SHIFT 0x00000009
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_PLLCLK_SEL_OVRD_EN_MASK 0x00001000L
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_PLLCLK_SEL_OVRD_EN__SHIFT 0x0000000c
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_PLLCLK_SEL_OVRD_VAL_MASK 0x00000800L
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_PLLCLK_SEL_OVRD_VAL__SHIFT 0x0000000b
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_RCLK_DIV_OVRD_EN_MASK 0x00004000L
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_RCLK_DIV_OVRD_EN__SHIFT 0x0000000e
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_RCLK_DIV_OVRD_VAL_MASK 0x00002000L
+#define PB0_RX_GLB_OVRD_REG0__RX_CFG_RCLK_DIV_OVRD_VAL__SHIFT 0x0000000d
+#define PB0_RX_GLB_OVRD_REG0__RX_DCLK_EN_OVRD_EN_MASK 0x00010000L
+#define PB0_RX_GLB_OVRD_REG0__RX_DCLK_EN_OVRD_EN__SHIFT 0x00000010
+#define PB0_RX_GLB_OVRD_REG0__RX_DCLK_EN_OVRD_VAL_MASK 0x00008000L
+#define PB0_RX_GLB_OVRD_REG0__RX_DCLK_EN_OVRD_VAL__SHIFT 0x0000000f
+#define PB0_RX_GLB_OVRD_REG0__RX_DLL_PWRON_OVRD_EN_MASK 0x00040000L
+#define PB0_RX_GLB_OVRD_REG0__RX_DLL_PWRON_OVRD_EN__SHIFT 0x00000012
+#define PB0_RX_GLB_OVRD_REG0__RX_DLL_PWRON_OVRD_VAL_MASK 0x00020000L
+#define PB0_RX_GLB_OVRD_REG0__RX_DLL_PWRON_OVRD_VAL__SHIFT 0x00000011
+#define PB0_RX_GLB_OVRD_REG0__RX_FRONTEND_PWRON_OVRD_EN_MASK 0x00100000L
+#define PB0_RX_GLB_OVRD_REG0__RX_FRONTEND_PWRON_OVRD_EN__SHIFT 0x00000014
+#define PB0_RX_GLB_OVRD_REG0__RX_FRONTEND_PWRON_OVRD_VAL_MASK 0x00080000L
+#define PB0_RX_GLB_OVRD_REG0__RX_FRONTEND_PWRON_OVRD_VAL__SHIFT 0x00000013
+#define PB0_RX_GLB_OVRD_REG0__RX_IDLEDET_PWRON_OVRD_EN_MASK 0x00400000L
+#define PB0_RX_GLB_OVRD_REG0__RX_IDLEDET_PWRON_OVRD_EN__SHIFT 0x00000016
+#define PB0_RX_GLB_OVRD_REG0__RX_IDLEDET_PWRON_OVRD_VAL_MASK 0x00200000L
+#define PB0_RX_GLB_OVRD_REG0__RX_IDLEDET_PWRON_OVRD_VAL__SHIFT 0x00000015
+#define PB0_RX_GLB_OVRD_REG0__RX_TERM_EN_OVRD_EN_MASK 0x01000000L
+#define PB0_RX_GLB_OVRD_REG0__RX_TERM_EN_OVRD_EN__SHIFT 0x00000018
+#define PB0_RX_GLB_OVRD_REG0__RX_TERM_EN_OVRD_VAL_MASK 0x00800000L
+#define PB0_RX_GLB_OVRD_REG0__RX_TERM_EN_OVRD_VAL__SHIFT 0x00000017
+#define PB0_RX_GLB_OVRD_REG1__RX_ADAPT_TRK_OVRD_EN_MASK 0x00000002L
+#define PB0_RX_GLB_OVRD_REG1__RX_ADAPT_TRK_OVRD_EN__SHIFT 0x00000001
+#define PB0_RX_GLB_OVRD_REG1__RX_ADAPT_TRK_OVRD_VAL_MASK 0x00000001L
+#define PB0_RX_GLB_OVRD_REG1__RX_ADAPT_TRK_OVRD_VAL__SHIFT 0x00000000
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L0T3_MASK 0x00000010L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L0T3__SHIFT 0x00000004
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L12T15_MASK 0x00000080L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L12T15__SHIFT 0x00000007
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L4T7_MASK 0x00000020L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L4T7__SHIFT 0x00000005
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L8T11_MASK 0x00000040L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L8T11__SHIFT 0x00000006
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L0T3_MASK 0x00001000L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L0T3__SHIFT 0x0000000c
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L12T15_MASK 0x00008000L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L12T15__SHIFT 0x0000000f
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L4T7_MASK 0x00002000L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L4T7__SHIFT 0x0000000d
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L8T11_MASK 0x00004000L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L8T11__SHIFT 0x0000000e
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L0T3_MASK 0x00010000L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L0T3__SHIFT 0x00000010
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L12T15_MASK 0x00080000L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L12T15__SHIFT 0x00000013
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L4T7_MASK 0x00020000L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L4T7__SHIFT 0x00000011
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L8T11_MASK 0x00040000L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L8T11__SHIFT 0x00000012
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L0T3_MASK 0x00100000L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L0T3__SHIFT 0x00000014
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L12T15_MASK 0x00800000L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L12T15__SHIFT 0x00000017
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L4T7_MASK 0x00200000L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L4T7__SHIFT 0x00000015
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L8T11_MASK 0x00400000L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L8T11__SHIFT 0x00000016
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L0T3_MASK 0x00000100L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L0T3__SHIFT 0x00000008
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L12T15_MASK 0x00000800L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L12T15__SHIFT 0x0000000b
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L4T7_MASK 0x00000200L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L4T7__SHIFT 0x00000009
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L8T11_MASK 0x00000400L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L8T11__SHIFT 0x0000000a
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L0T3_MASK 0x00000001L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L0T3__SHIFT 0x00000000
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L12T15_MASK 0x00000008L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L12T15__SHIFT 0x00000003
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L4T7_MASK 0x00000002L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L4T7__SHIFT 0x00000001
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L8T11_MASK 0x00000004L
+#define PB0_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L8T11__SHIFT 0x00000002
+#define PB0_RX_LANE0_CTRL_REG0__RX_BACKUP_0_MASK 0x000000ffL
+#define PB0_RX_LANE0_CTRL_REG0__RX_BACKUP_0__SHIFT 0x00000000
+#define PB0_RX_LANE0_CTRL_REG0__RX_CFG_OVR_PWRSF_0_MASK 0x00002000L
+#define PB0_RX_LANE0_CTRL_REG0__RX_CFG_OVR_PWRSF_0__SHIFT 0x0000000d
+#define PB0_RX_LANE0_CTRL_REG0__RX_DBG_ANALOG_SEL_0_MASK 0x00000c00L
+#define PB0_RX_LANE0_CTRL_REG0__RX_DBG_ANALOG_SEL_0__SHIFT 0x0000000a
+#define PB0_RX_LANE0_CTRL_REG0__RX_TST_BSCAN_EN_0_MASK 0x00001000L
+#define PB0_RX_LANE0_CTRL_REG0__RX_TST_BSCAN_EN_0__SHIFT 0x0000000c
+#define PB0_RX_LANE0_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_0_MASK 0x00000008L
+#define PB0_RX_LANE0_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_0__SHIFT 0x00000003
+#define PB0_RX_LANE0_SCI_STAT_OVRD_REG0__ENABLEFOM_0_MASK 0x00000080L
+#define PB0_RX_LANE0_SCI_STAT_OVRD_REG0__ENABLEFOM_0__SHIFT 0x00000007
+#define PB0_RX_LANE0_SCI_STAT_OVRD_REG0__REQUESTFOM_0_MASK 0x00000100L
+#define PB0_RX_LANE0_SCI_STAT_OVRD_REG0__REQUESTFOM_0__SHIFT 0x00000008
+#define PB0_RX_LANE0_SCI_STAT_OVRD_REG0__RESPONSEMODE_0_MASK 0x00000200L
+#define PB0_RX_LANE0_SCI_STAT_OVRD_REG0__RESPONSEMODE_0__SHIFT 0x00000009
+#define PB0_RX_LANE0_SCI_STAT_OVRD_REG0__RXPRESETHINT_0_MASK 0x00000070L
+#define PB0_RX_LANE0_SCI_STAT_OVRD_REG0__RXPRESETHINT_0__SHIFT 0x00000004
+#define PB0_RX_LANE0_SCI_STAT_OVRD_REG0__RXPWR_0_MASK 0x00000007L
+#define PB0_RX_LANE0_SCI_STAT_OVRD_REG0__RXPWR_0__SHIFT 0x00000000
+#define PB0_RX_LANE10_CTRL_REG0__RX_BACKUP_10_MASK 0x000000ffL
+#define PB0_RX_LANE10_CTRL_REG0__RX_BACKUP_10__SHIFT 0x00000000
+#define PB0_RX_LANE10_CTRL_REG0__RX_CFG_OVR_PWRSF_10_MASK 0x00002000L
+#define PB0_RX_LANE10_CTRL_REG0__RX_CFG_OVR_PWRSF_10__SHIFT 0x0000000d
+#define PB0_RX_LANE10_CTRL_REG0__RX_DBG_ANALOG_SEL_10_MASK 0x00000c00L
+#define PB0_RX_LANE10_CTRL_REG0__RX_DBG_ANALOG_SEL_10__SHIFT 0x0000000a
+#define PB0_RX_LANE10_CTRL_REG0__RX_TST_BSCAN_EN_10_MASK 0x00001000L
+#define PB0_RX_LANE10_CTRL_REG0__RX_TST_BSCAN_EN_10__SHIFT 0x0000000c
+#define PB0_RX_LANE10_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_10_MASK 0x00000008L
+#define PB0_RX_LANE10_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_10__SHIFT 0x00000003
+#define PB0_RX_LANE10_SCI_STAT_OVRD_REG0__ENABLEFOM_10_MASK 0x00000080L
+#define PB0_RX_LANE10_SCI_STAT_OVRD_REG0__ENABLEFOM_10__SHIFT 0x00000007
+#define PB0_RX_LANE10_SCI_STAT_OVRD_REG0__REQUESTFOM_10_MASK 0x00000100L
+#define PB0_RX_LANE10_SCI_STAT_OVRD_REG0__REQUESTFOM_10__SHIFT 0x00000008
+#define PB0_RX_LANE10_SCI_STAT_OVRD_REG0__RESPONSEMODE_10_MASK 0x00000200L
+#define PB0_RX_LANE10_SCI_STAT_OVRD_REG0__RESPONSEMODE_10__SHIFT 0x00000009
+#define PB0_RX_LANE10_SCI_STAT_OVRD_REG0__RXPRESETHINT_10_MASK 0x00000070L
+#define PB0_RX_LANE10_SCI_STAT_OVRD_REG0__RXPRESETHINT_10__SHIFT 0x00000004
+#define PB0_RX_LANE10_SCI_STAT_OVRD_REG0__RXPWR_10_MASK 0x00000007L
+#define PB0_RX_LANE10_SCI_STAT_OVRD_REG0__RXPWR_10__SHIFT 0x00000000
+#define PB0_RX_LANE11_CTRL_REG0__RX_BACKUP_11_MASK 0x000000ffL
+#define PB0_RX_LANE11_CTRL_REG0__RX_BACKUP_11__SHIFT 0x00000000
+#define PB0_RX_LANE11_CTRL_REG0__RX_CFG_OVR_PWRSF_11_MASK 0x00002000L
+#define PB0_RX_LANE11_CTRL_REG0__RX_CFG_OVR_PWRSF_11__SHIFT 0x0000000d
+#define PB0_RX_LANE11_CTRL_REG0__RX_DBG_ANALOG_SEL_11_MASK 0x00000c00L
+#define PB0_RX_LANE11_CTRL_REG0__RX_DBG_ANALOG_SEL_11__SHIFT 0x0000000a
+#define PB0_RX_LANE11_CTRL_REG0__RX_TST_BSCAN_EN_11_MASK 0x00001000L
+#define PB0_RX_LANE11_CTRL_REG0__RX_TST_BSCAN_EN_11__SHIFT 0x0000000c
+#define PB0_RX_LANE11_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_11_MASK 0x00000008L
+#define PB0_RX_LANE11_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_11__SHIFT 0x00000003
+#define PB0_RX_LANE11_SCI_STAT_OVRD_REG0__ENABLEFOM_11_MASK 0x00000080L
+#define PB0_RX_LANE11_SCI_STAT_OVRD_REG0__ENABLEFOM_11__SHIFT 0x00000007
+#define PB0_RX_LANE11_SCI_STAT_OVRD_REG0__REQUESTFOM_11_MASK 0x00000100L
+#define PB0_RX_LANE11_SCI_STAT_OVRD_REG0__REQUESTFOM_11__SHIFT 0x00000008
+#define PB0_RX_LANE11_SCI_STAT_OVRD_REG0__RESPONSEMODE_11_MASK 0x00000200L
+#define PB0_RX_LANE11_SCI_STAT_OVRD_REG0__RESPONSEMODE_11__SHIFT 0x00000009
+#define PB0_RX_LANE11_SCI_STAT_OVRD_REG0__RXPRESETHINT_11_MASK 0x00000070L
+#define PB0_RX_LANE11_SCI_STAT_OVRD_REG0__RXPRESETHINT_11__SHIFT 0x00000004
+#define PB0_RX_LANE11_SCI_STAT_OVRD_REG0__RXPWR_11_MASK 0x00000007L
+#define PB0_RX_LANE11_SCI_STAT_OVRD_REG0__RXPWR_11__SHIFT 0x00000000
+#define PB0_RX_LANE12_CTRL_REG0__RX_BACKUP_12_MASK 0x000000ffL
+#define PB0_RX_LANE12_CTRL_REG0__RX_BACKUP_12__SHIFT 0x00000000
+#define PB0_RX_LANE12_CTRL_REG0__RX_CFG_OVR_PWRSF_12_MASK 0x00002000L
+#define PB0_RX_LANE12_CTRL_REG0__RX_CFG_OVR_PWRSF_12__SHIFT 0x0000000d
+#define PB0_RX_LANE12_CTRL_REG0__RX_DBG_ANALOG_SEL_12_MASK 0x00000c00L
+#define PB0_RX_LANE12_CTRL_REG0__RX_DBG_ANALOG_SEL_12__SHIFT 0x0000000a
+#define PB0_RX_LANE12_CTRL_REG0__RX_TST_BSCAN_EN_12_MASK 0x00001000L
+#define PB0_RX_LANE12_CTRL_REG0__RX_TST_BSCAN_EN_12__SHIFT 0x0000000c
+#define PB0_RX_LANE12_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_12_MASK 0x00000008L
+#define PB0_RX_LANE12_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_12__SHIFT 0x00000003
+#define PB0_RX_LANE12_SCI_STAT_OVRD_REG0__ENABLEFOM_12_MASK 0x00000080L
+#define PB0_RX_LANE12_SCI_STAT_OVRD_REG0__ENABLEFOM_12__SHIFT 0x00000007
+#define PB0_RX_LANE12_SCI_STAT_OVRD_REG0__REQUESTFOM_12_MASK 0x00000100L
+#define PB0_RX_LANE12_SCI_STAT_OVRD_REG0__REQUESTFOM_12__SHIFT 0x00000008
+#define PB0_RX_LANE12_SCI_STAT_OVRD_REG0__RESPONSEMODE_12_MASK 0x00000200L
+#define PB0_RX_LANE12_SCI_STAT_OVRD_REG0__RESPONSEMODE_12__SHIFT 0x00000009
+#define PB0_RX_LANE12_SCI_STAT_OVRD_REG0__RXPRESETHINT_12_MASK 0x00000070L
+#define PB0_RX_LANE12_SCI_STAT_OVRD_REG0__RXPRESETHINT_12__SHIFT 0x00000004
+#define PB0_RX_LANE12_SCI_STAT_OVRD_REG0__RXPWR_12_MASK 0x00000007L
+#define PB0_RX_LANE12_SCI_STAT_OVRD_REG0__RXPWR_12__SHIFT 0x00000000
+#define PB0_RX_LANE13_CTRL_REG0__RX_BACKUP_13_MASK 0x000000ffL
+#define PB0_RX_LANE13_CTRL_REG0__RX_BACKUP_13__SHIFT 0x00000000
+#define PB0_RX_LANE13_CTRL_REG0__RX_CFG_OVR_PWRSF_13_MASK 0x00002000L
+#define PB0_RX_LANE13_CTRL_REG0__RX_CFG_OVR_PWRSF_13__SHIFT 0x0000000d
+#define PB0_RX_LANE13_CTRL_REG0__RX_DBG_ANALOG_SEL_13_MASK 0x00000c00L
+#define PB0_RX_LANE13_CTRL_REG0__RX_DBG_ANALOG_SEL_13__SHIFT 0x0000000a
+#define PB0_RX_LANE13_CTRL_REG0__RX_TST_BSCAN_EN_13_MASK 0x00001000L
+#define PB0_RX_LANE13_CTRL_REG0__RX_TST_BSCAN_EN_13__SHIFT 0x0000000c
+#define PB0_RX_LANE13_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_13_MASK 0x00000008L
+#define PB0_RX_LANE13_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_13__SHIFT 0x00000003
+#define PB0_RX_LANE13_SCI_STAT_OVRD_REG0__ENABLEFOM_13_MASK 0x00000080L
+#define PB0_RX_LANE13_SCI_STAT_OVRD_REG0__ENABLEFOM_13__SHIFT 0x00000007
+#define PB0_RX_LANE13_SCI_STAT_OVRD_REG0__REQUESTFOM_13_MASK 0x00000100L
+#define PB0_RX_LANE13_SCI_STAT_OVRD_REG0__REQUESTFOM_13__SHIFT 0x00000008
+#define PB0_RX_LANE13_SCI_STAT_OVRD_REG0__RESPONSEMODE_13_MASK 0x00000200L
+#define PB0_RX_LANE13_SCI_STAT_OVRD_REG0__RESPONSEMODE_13__SHIFT 0x00000009
+#define PB0_RX_LANE13_SCI_STAT_OVRD_REG0__RXPRESETHINT_13_MASK 0x00000070L
+#define PB0_RX_LANE13_SCI_STAT_OVRD_REG0__RXPRESETHINT_13__SHIFT 0x00000004
+#define PB0_RX_LANE13_SCI_STAT_OVRD_REG0__RXPWR_13_MASK 0x00000007L
+#define PB0_RX_LANE13_SCI_STAT_OVRD_REG0__RXPWR_13__SHIFT 0x00000000
+#define PB0_RX_LANE14_CTRL_REG0__RX_BACKUP_14_MASK 0x000000ffL
+#define PB0_RX_LANE14_CTRL_REG0__RX_BACKUP_14__SHIFT 0x00000000
+#define PB0_RX_LANE14_CTRL_REG0__RX_CFG_OVR_PWRSF_14_MASK 0x00002000L
+#define PB0_RX_LANE14_CTRL_REG0__RX_CFG_OVR_PWRSF_14__SHIFT 0x0000000d
+#define PB0_RX_LANE14_CTRL_REG0__RX_DBG_ANALOG_SEL_14_MASK 0x00000c00L
+#define PB0_RX_LANE14_CTRL_REG0__RX_DBG_ANALOG_SEL_14__SHIFT 0x0000000a
+#define PB0_RX_LANE14_CTRL_REG0__RX_TST_BSCAN_EN_14_MASK 0x00001000L
+#define PB0_RX_LANE14_CTRL_REG0__RX_TST_BSCAN_EN_14__SHIFT 0x0000000c
+#define PB0_RX_LANE14_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_14_MASK 0x00000008L
+#define PB0_RX_LANE14_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_14__SHIFT 0x00000003
+#define PB0_RX_LANE14_SCI_STAT_OVRD_REG0__ENABLEFOM_14_MASK 0x00000080L
+#define PB0_RX_LANE14_SCI_STAT_OVRD_REG0__ENABLEFOM_14__SHIFT 0x00000007
+#define PB0_RX_LANE14_SCI_STAT_OVRD_REG0__REQUESTFOM_14_MASK 0x00000100L
+#define PB0_RX_LANE14_SCI_STAT_OVRD_REG0__REQUESTFOM_14__SHIFT 0x00000008
+#define PB0_RX_LANE14_SCI_STAT_OVRD_REG0__RESPONSEMODE_14_MASK 0x00000200L
+#define PB0_RX_LANE14_SCI_STAT_OVRD_REG0__RESPONSEMODE_14__SHIFT 0x00000009
+#define PB0_RX_LANE14_SCI_STAT_OVRD_REG0__RXPRESETHINT_14_MASK 0x00000070L
+#define PB0_RX_LANE14_SCI_STAT_OVRD_REG0__RXPRESETHINT_14__SHIFT 0x00000004
+#define PB0_RX_LANE14_SCI_STAT_OVRD_REG0__RXPWR_14_MASK 0x00000007L
+#define PB0_RX_LANE14_SCI_STAT_OVRD_REG0__RXPWR_14__SHIFT 0x00000000
+#define PB0_RX_LANE15_CTRL_REG0__RX_BACKUP_15_MASK 0x000000ffL
+#define PB0_RX_LANE15_CTRL_REG0__RX_BACKUP_15__SHIFT 0x00000000
+#define PB0_RX_LANE15_CTRL_REG0__RX_CFG_OVR_PWRSF_15_MASK 0x00002000L
+#define PB0_RX_LANE15_CTRL_REG0__RX_CFG_OVR_PWRSF_15__SHIFT 0x0000000d
+#define PB0_RX_LANE15_CTRL_REG0__RX_DBG_ANALOG_SEL_15_MASK 0x00000c00L
+#define PB0_RX_LANE15_CTRL_REG0__RX_DBG_ANALOG_SEL_15__SHIFT 0x0000000a
+#define PB0_RX_LANE15_CTRL_REG0__RX_TST_BSCAN_EN_15_MASK 0x00001000L
+#define PB0_RX_LANE15_CTRL_REG0__RX_TST_BSCAN_EN_15__SHIFT 0x0000000c
+#define PB0_RX_LANE15_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_15_MASK 0x00000008L
+#define PB0_RX_LANE15_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_15__SHIFT 0x00000003
+#define PB0_RX_LANE15_SCI_STAT_OVRD_REG0__ENABLEFOM_15_MASK 0x00000080L
+#define PB0_RX_LANE15_SCI_STAT_OVRD_REG0__ENABLEFOM_15__SHIFT 0x00000007
+#define PB0_RX_LANE15_SCI_STAT_OVRD_REG0__REQUESTFOM_15_MASK 0x00000100L
+#define PB0_RX_LANE15_SCI_STAT_OVRD_REG0__REQUESTFOM_15__SHIFT 0x00000008
+#define PB0_RX_LANE15_SCI_STAT_OVRD_REG0__RESPONSEMODE_15_MASK 0x00000200L
+#define PB0_RX_LANE15_SCI_STAT_OVRD_REG0__RESPONSEMODE_15__SHIFT 0x00000009
+#define PB0_RX_LANE15_SCI_STAT_OVRD_REG0__RXPRESETHINT_15_MASK 0x00000070L
+#define PB0_RX_LANE15_SCI_STAT_OVRD_REG0__RXPRESETHINT_15__SHIFT 0x00000004
+#define PB0_RX_LANE15_SCI_STAT_OVRD_REG0__RXPWR_15_MASK 0x00000007L
+#define PB0_RX_LANE15_SCI_STAT_OVRD_REG0__RXPWR_15__SHIFT 0x00000000
+#define PB0_RX_LANE1_CTRL_REG0__RX_BACKUP_1_MASK 0x000000ffL
+#define PB0_RX_LANE1_CTRL_REG0__RX_BACKUP_1__SHIFT 0x00000000
+#define PB0_RX_LANE1_CTRL_REG0__RX_CFG_OVR_PWRSF_1_MASK 0x00002000L
+#define PB0_RX_LANE1_CTRL_REG0__RX_CFG_OVR_PWRSF_1__SHIFT 0x0000000d
+#define PB0_RX_LANE1_CTRL_REG0__RX_DBG_ANALOG_SEL_1_MASK 0x00000c00L
+#define PB0_RX_LANE1_CTRL_REG0__RX_DBG_ANALOG_SEL_1__SHIFT 0x0000000a
+#define PB0_RX_LANE1_CTRL_REG0__RX_TST_BSCAN_EN_1_MASK 0x00001000L
+#define PB0_RX_LANE1_CTRL_REG0__RX_TST_BSCAN_EN_1__SHIFT 0x0000000c
+#define PB0_RX_LANE1_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_1_MASK 0x00000008L
+#define PB0_RX_LANE1_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_1__SHIFT 0x00000003
+#define PB0_RX_LANE1_SCI_STAT_OVRD_REG0__ENABLEFOM_1_MASK 0x00000080L
+#define PB0_RX_LANE1_SCI_STAT_OVRD_REG0__ENABLEFOM_1__SHIFT 0x00000007
+#define PB0_RX_LANE1_SCI_STAT_OVRD_REG0__REQUESTFOM_1_MASK 0x00000100L
+#define PB0_RX_LANE1_SCI_STAT_OVRD_REG0__REQUESTFOM_1__SHIFT 0x00000008
+#define PB0_RX_LANE1_SCI_STAT_OVRD_REG0__RESPONSEMODE_1_MASK 0x00000200L
+#define PB0_RX_LANE1_SCI_STAT_OVRD_REG0__RESPONSEMODE_1__SHIFT 0x00000009
+#define PB0_RX_LANE1_SCI_STAT_OVRD_REG0__RXPRESETHINT_1_MASK 0x00000070L
+#define PB0_RX_LANE1_SCI_STAT_OVRD_REG0__RXPRESETHINT_1__SHIFT 0x00000004
+#define PB0_RX_LANE1_SCI_STAT_OVRD_REG0__RXPWR_1_MASK 0x00000007L
+#define PB0_RX_LANE1_SCI_STAT_OVRD_REG0__RXPWR_1__SHIFT 0x00000000
+#define PB0_RX_LANE2_CTRL_REG0__RX_BACKUP_2_MASK 0x000000ffL
+#define PB0_RX_LANE2_CTRL_REG0__RX_BACKUP_2__SHIFT 0x00000000
+#define PB0_RX_LANE2_CTRL_REG0__RX_CFG_OVR_PWRSF_2_MASK 0x00002000L
+#define PB0_RX_LANE2_CTRL_REG0__RX_CFG_OVR_PWRSF_2__SHIFT 0x0000000d
+#define PB0_RX_LANE2_CTRL_REG0__RX_DBG_ANALOG_SEL_2_MASK 0x00000c00L
+#define PB0_RX_LANE2_CTRL_REG0__RX_DBG_ANALOG_SEL_2__SHIFT 0x0000000a
+#define PB0_RX_LANE2_CTRL_REG0__RX_TST_BSCAN_EN_2_MASK 0x00001000L
+#define PB0_RX_LANE2_CTRL_REG0__RX_TST_BSCAN_EN_2__SHIFT 0x0000000c
+#define PB0_RX_LANE2_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_2_MASK 0x00000008L
+#define PB0_RX_LANE2_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_2__SHIFT 0x00000003
+#define PB0_RX_LANE2_SCI_STAT_OVRD_REG0__ENABLEFOM_2_MASK 0x00000080L
+#define PB0_RX_LANE2_SCI_STAT_OVRD_REG0__ENABLEFOM_2__SHIFT 0x00000007
+#define PB0_RX_LANE2_SCI_STAT_OVRD_REG0__REQUESTFOM_2_MASK 0x00000100L
+#define PB0_RX_LANE2_SCI_STAT_OVRD_REG0__REQUESTFOM_2__SHIFT 0x00000008
+#define PB0_RX_LANE2_SCI_STAT_OVRD_REG0__RESPONSEMODE_2_MASK 0x00000200L
+#define PB0_RX_LANE2_SCI_STAT_OVRD_REG0__RESPONSEMODE_2__SHIFT 0x00000009
+#define PB0_RX_LANE2_SCI_STAT_OVRD_REG0__RXPRESETHINT_2_MASK 0x00000070L
+#define PB0_RX_LANE2_SCI_STAT_OVRD_REG0__RXPRESETHINT_2__SHIFT 0x00000004
+#define PB0_RX_LANE2_SCI_STAT_OVRD_REG0__RXPWR_2_MASK 0x00000007L
+#define PB0_RX_LANE2_SCI_STAT_OVRD_REG0__RXPWR_2__SHIFT 0x00000000
+#define PB0_RX_LANE3_CTRL_REG0__RX_BACKUP_3_MASK 0x000000ffL
+#define PB0_RX_LANE3_CTRL_REG0__RX_BACKUP_3__SHIFT 0x00000000
+#define PB0_RX_LANE3_CTRL_REG0__RX_CFG_OVR_PWRSF_3_MASK 0x00002000L
+#define PB0_RX_LANE3_CTRL_REG0__RX_CFG_OVR_PWRSF_3__SHIFT 0x0000000d
+#define PB0_RX_LANE3_CTRL_REG0__RX_DBG_ANALOG_SEL_3_MASK 0x00000c00L
+#define PB0_RX_LANE3_CTRL_REG0__RX_DBG_ANALOG_SEL_3__SHIFT 0x0000000a
+#define PB0_RX_LANE3_CTRL_REG0__RX_TST_BSCAN_EN_3_MASK 0x00001000L
+#define PB0_RX_LANE3_CTRL_REG0__RX_TST_BSCAN_EN_3__SHIFT 0x0000000c
+#define PB0_RX_LANE3_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_3_MASK 0x00000008L
+#define PB0_RX_LANE3_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_3__SHIFT 0x00000003
+#define PB0_RX_LANE3_SCI_STAT_OVRD_REG0__ENABLEFOM_3_MASK 0x00000080L
+#define PB0_RX_LANE3_SCI_STAT_OVRD_REG0__ENABLEFOM_3__SHIFT 0x00000007
+#define PB0_RX_LANE3_SCI_STAT_OVRD_REG0__REQUESTFOM_3_MASK 0x00000100L
+#define PB0_RX_LANE3_SCI_STAT_OVRD_REG0__REQUESTFOM_3__SHIFT 0x00000008
+#define PB0_RX_LANE3_SCI_STAT_OVRD_REG0__RESPONSEMODE_3_MASK 0x00000200L
+#define PB0_RX_LANE3_SCI_STAT_OVRD_REG0__RESPONSEMODE_3__SHIFT 0x00000009
+#define PB0_RX_LANE3_SCI_STAT_OVRD_REG0__RXPRESETHINT_3_MASK 0x00000070L
+#define PB0_RX_LANE3_SCI_STAT_OVRD_REG0__RXPRESETHINT_3__SHIFT 0x00000004
+#define PB0_RX_LANE3_SCI_STAT_OVRD_REG0__RXPWR_3_MASK 0x00000007L
+#define PB0_RX_LANE3_SCI_STAT_OVRD_REG0__RXPWR_3__SHIFT 0x00000000
+#define PB0_RX_LANE4_CTRL_REG0__RX_BACKUP_4_MASK 0x000000ffL
+#define PB0_RX_LANE4_CTRL_REG0__RX_BACKUP_4__SHIFT 0x00000000
+#define PB0_RX_LANE4_CTRL_REG0__RX_CFG_OVR_PWRSF_4_MASK 0x00002000L
+#define PB0_RX_LANE4_CTRL_REG0__RX_CFG_OVR_PWRSF_4__SHIFT 0x0000000d
+#define PB0_RX_LANE4_CTRL_REG0__RX_DBG_ANALOG_SEL_4_MASK 0x00000c00L
+#define PB0_RX_LANE4_CTRL_REG0__RX_DBG_ANALOG_SEL_4__SHIFT 0x0000000a
+#define PB0_RX_LANE4_CTRL_REG0__RX_TST_BSCAN_EN_4_MASK 0x00001000L
+#define PB0_RX_LANE4_CTRL_REG0__RX_TST_BSCAN_EN_4__SHIFT 0x0000000c
+#define PB0_RX_LANE4_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_4_MASK 0x00000008L
+#define PB0_RX_LANE4_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_4__SHIFT 0x00000003
+#define PB0_RX_LANE4_SCI_STAT_OVRD_REG0__ENABLEFOM_4_MASK 0x00000080L
+#define PB0_RX_LANE4_SCI_STAT_OVRD_REG0__ENABLEFOM_4__SHIFT 0x00000007
+#define PB0_RX_LANE4_SCI_STAT_OVRD_REG0__REQUESTFOM_4_MASK 0x00000100L
+#define PB0_RX_LANE4_SCI_STAT_OVRD_REG0__REQUESTFOM_4__SHIFT 0x00000008
+#define PB0_RX_LANE4_SCI_STAT_OVRD_REG0__RESPONSEMODE_4_MASK 0x00000200L
+#define PB0_RX_LANE4_SCI_STAT_OVRD_REG0__RESPONSEMODE_4__SHIFT 0x00000009
+#define PB0_RX_LANE4_SCI_STAT_OVRD_REG0__RXPRESETHINT_4_MASK 0x00000070L
+#define PB0_RX_LANE4_SCI_STAT_OVRD_REG0__RXPRESETHINT_4__SHIFT 0x00000004
+#define PB0_RX_LANE4_SCI_STAT_OVRD_REG0__RXPWR_4_MASK 0x00000007L
+#define PB0_RX_LANE4_SCI_STAT_OVRD_REG0__RXPWR_4__SHIFT 0x00000000
+#define PB0_RX_LANE5_CTRL_REG0__RX_BACKUP_5_MASK 0x000000ffL
+#define PB0_RX_LANE5_CTRL_REG0__RX_BACKUP_5__SHIFT 0x00000000
+#define PB0_RX_LANE5_CTRL_REG0__RX_CFG_OVR_PWRSF_5_MASK 0x00002000L
+#define PB0_RX_LANE5_CTRL_REG0__RX_CFG_OVR_PWRSF_5__SHIFT 0x0000000d
+#define PB0_RX_LANE5_CTRL_REG0__RX_DBG_ANALOG_SEL_5_MASK 0x00000c00L
+#define PB0_RX_LANE5_CTRL_REG0__RX_DBG_ANALOG_SEL_5__SHIFT 0x0000000a
+#define PB0_RX_LANE5_CTRL_REG0__RX_TST_BSCAN_EN_5_MASK 0x00001000L
+#define PB0_RX_LANE5_CTRL_REG0__RX_TST_BSCAN_EN_5__SHIFT 0x0000000c
+#define PB0_RX_LANE5_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_5_MASK 0x00000008L
+#define PB0_RX_LANE5_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_5__SHIFT 0x00000003
+#define PB0_RX_LANE5_SCI_STAT_OVRD_REG0__ENABLEFOM_5_MASK 0x00000080L
+#define PB0_RX_LANE5_SCI_STAT_OVRD_REG0__ENABLEFOM_5__SHIFT 0x00000007
+#define PB0_RX_LANE5_SCI_STAT_OVRD_REG0__REQUESTFOM_5_MASK 0x00000100L
+#define PB0_RX_LANE5_SCI_STAT_OVRD_REG0__REQUESTFOM_5__SHIFT 0x00000008
+#define PB0_RX_LANE5_SCI_STAT_OVRD_REG0__RESPONSEMODE_5_MASK 0x00000200L
+#define PB0_RX_LANE5_SCI_STAT_OVRD_REG0__RESPONSEMODE_5__SHIFT 0x00000009
+#define PB0_RX_LANE5_SCI_STAT_OVRD_REG0__RXPRESETHINT_5_MASK 0x00000070L
+#define PB0_RX_LANE5_SCI_STAT_OVRD_REG0__RXPRESETHINT_5__SHIFT 0x00000004
+#define PB0_RX_LANE5_SCI_STAT_OVRD_REG0__RXPWR_5_MASK 0x00000007L
+#define PB0_RX_LANE5_SCI_STAT_OVRD_REG0__RXPWR_5__SHIFT 0x00000000
+#define PB0_RX_LANE6_CTRL_REG0__RX_BACKUP_6_MASK 0x000000ffL
+#define PB0_RX_LANE6_CTRL_REG0__RX_BACKUP_6__SHIFT 0x00000000
+#define PB0_RX_LANE6_CTRL_REG0__RX_CFG_OVR_PWRSF_6_MASK 0x00002000L
+#define PB0_RX_LANE6_CTRL_REG0__RX_CFG_OVR_PWRSF_6__SHIFT 0x0000000d
+#define PB0_RX_LANE6_CTRL_REG0__RX_DBG_ANALOG_SEL_6_MASK 0x00000c00L
+#define PB0_RX_LANE6_CTRL_REG0__RX_DBG_ANALOG_SEL_6__SHIFT 0x0000000a
+#define PB0_RX_LANE6_CTRL_REG0__RX_TST_BSCAN_EN_6_MASK 0x00001000L
+#define PB0_RX_LANE6_CTRL_REG0__RX_TST_BSCAN_EN_6__SHIFT 0x0000000c
+#define PB0_RX_LANE6_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_6_MASK 0x00000008L
+#define PB0_RX_LANE6_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_6__SHIFT 0x00000003
+#define PB0_RX_LANE6_SCI_STAT_OVRD_REG0__ENABLEFOM_6_MASK 0x00000080L
+#define PB0_RX_LANE6_SCI_STAT_OVRD_REG0__ENABLEFOM_6__SHIFT 0x00000007
+#define PB0_RX_LANE6_SCI_STAT_OVRD_REG0__REQUESTFOM_6_MASK 0x00000100L
+#define PB0_RX_LANE6_SCI_STAT_OVRD_REG0__REQUESTFOM_6__SHIFT 0x00000008
+#define PB0_RX_LANE6_SCI_STAT_OVRD_REG0__RESPONSEMODE_6_MASK 0x00000200L
+#define PB0_RX_LANE6_SCI_STAT_OVRD_REG0__RESPONSEMODE_6__SHIFT 0x00000009
+#define PB0_RX_LANE6_SCI_STAT_OVRD_REG0__RXPRESETHINT_6_MASK 0x00000070L
+#define PB0_RX_LANE6_SCI_STAT_OVRD_REG0__RXPRESETHINT_6__SHIFT 0x00000004
+#define PB0_RX_LANE6_SCI_STAT_OVRD_REG0__RXPWR_6_MASK 0x00000007L
+#define PB0_RX_LANE6_SCI_STAT_OVRD_REG0__RXPWR_6__SHIFT 0x00000000
+#define PB0_RX_LANE7_CTRL_REG0__RX_BACKUP_7_MASK 0x000000ffL
+#define PB0_RX_LANE7_CTRL_REG0__RX_BACKUP_7__SHIFT 0x00000000
+#define PB0_RX_LANE7_CTRL_REG0__RX_CFG_OVR_PWRSF_7_MASK 0x00002000L
+#define PB0_RX_LANE7_CTRL_REG0__RX_CFG_OVR_PWRSF_7__SHIFT 0x0000000d
+#define PB0_RX_LANE7_CTRL_REG0__RX_DBG_ANALOG_SEL_7_MASK 0x00000c00L
+#define PB0_RX_LANE7_CTRL_REG0__RX_DBG_ANALOG_SEL_7__SHIFT 0x0000000a
+#define PB0_RX_LANE7_CTRL_REG0__RX_TST_BSCAN_EN_7_MASK 0x00001000L
+#define PB0_RX_LANE7_CTRL_REG0__RX_TST_BSCAN_EN_7__SHIFT 0x0000000c
+#define PB0_RX_LANE7_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_7_MASK 0x00000008L
+#define PB0_RX_LANE7_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_7__SHIFT 0x00000003
+#define PB0_RX_LANE7_SCI_STAT_OVRD_REG0__ENABLEFOM_7_MASK 0x00000080L
+#define PB0_RX_LANE7_SCI_STAT_OVRD_REG0__ENABLEFOM_7__SHIFT 0x00000007
+#define PB0_RX_LANE7_SCI_STAT_OVRD_REG0__REQUESTFOM_7_MASK 0x00000100L
+#define PB0_RX_LANE7_SCI_STAT_OVRD_REG0__REQUESTFOM_7__SHIFT 0x00000008
+#define PB0_RX_LANE7_SCI_STAT_OVRD_REG0__RESPONSEMODE_7_MASK 0x00000200L
+#define PB0_RX_LANE7_SCI_STAT_OVRD_REG0__RESPONSEMODE_7__SHIFT 0x00000009
+#define PB0_RX_LANE7_SCI_STAT_OVRD_REG0__RXPRESETHINT_7_MASK 0x00000070L
+#define PB0_RX_LANE7_SCI_STAT_OVRD_REG0__RXPRESETHINT_7__SHIFT 0x00000004
+#define PB0_RX_LANE7_SCI_STAT_OVRD_REG0__RXPWR_7_MASK 0x00000007L
+#define PB0_RX_LANE7_SCI_STAT_OVRD_REG0__RXPWR_7__SHIFT 0x00000000
+#define PB0_RX_LANE8_CTRL_REG0__RX_BACKUP_8_MASK 0x000000ffL
+#define PB0_RX_LANE8_CTRL_REG0__RX_BACKUP_8__SHIFT 0x00000000
+#define PB0_RX_LANE8_CTRL_REG0__RX_CFG_OVR_PWRSF_8_MASK 0x00002000L
+#define PB0_RX_LANE8_CTRL_REG0__RX_CFG_OVR_PWRSF_8__SHIFT 0x0000000d
+#define PB0_RX_LANE8_CTRL_REG0__RX_DBG_ANALOG_SEL_8_MASK 0x00000c00L
+#define PB0_RX_LANE8_CTRL_REG0__RX_DBG_ANALOG_SEL_8__SHIFT 0x0000000a
+#define PB0_RX_LANE8_CTRL_REG0__RX_TST_BSCAN_EN_8_MASK 0x00001000L
+#define PB0_RX_LANE8_CTRL_REG0__RX_TST_BSCAN_EN_8__SHIFT 0x0000000c
+#define PB0_RX_LANE8_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_8_MASK 0x00000008L
+#define PB0_RX_LANE8_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_8__SHIFT 0x00000003
+#define PB0_RX_LANE8_SCI_STAT_OVRD_REG0__ENABLEFOM_8_MASK 0x00000080L
+#define PB0_RX_LANE8_SCI_STAT_OVRD_REG0__ENABLEFOM_8__SHIFT 0x00000007
+#define PB0_RX_LANE8_SCI_STAT_OVRD_REG0__REQUESTFOM_8_MASK 0x00000100L
+#define PB0_RX_LANE8_SCI_STAT_OVRD_REG0__REQUESTFOM_8__SHIFT 0x00000008
+#define PB0_RX_LANE8_SCI_STAT_OVRD_REG0__RESPONSEMODE_8_MASK 0x00000200L
+#define PB0_RX_LANE8_SCI_STAT_OVRD_REG0__RESPONSEMODE_8__SHIFT 0x00000009
+#define PB0_RX_LANE8_SCI_STAT_OVRD_REG0__RXPRESETHINT_8_MASK 0x00000070L
+#define PB0_RX_LANE8_SCI_STAT_OVRD_REG0__RXPRESETHINT_8__SHIFT 0x00000004
+#define PB0_RX_LANE8_SCI_STAT_OVRD_REG0__RXPWR_8_MASK 0x00000007L
+#define PB0_RX_LANE8_SCI_STAT_OVRD_REG0__RXPWR_8__SHIFT 0x00000000
+#define PB0_RX_LANE9_CTRL_REG0__RX_BACKUP_9_MASK 0x000000ffL
+#define PB0_RX_LANE9_CTRL_REG0__RX_BACKUP_9__SHIFT 0x00000000
+#define PB0_RX_LANE9_CTRL_REG0__RX_CFG_OVR_PWRSF_9_MASK 0x00002000L
+#define PB0_RX_LANE9_CTRL_REG0__RX_CFG_OVR_PWRSF_9__SHIFT 0x0000000d
+#define PB0_RX_LANE9_CTRL_REG0__RX_DBG_ANALOG_SEL_9_MASK 0x00000c00L
+#define PB0_RX_LANE9_CTRL_REG0__RX_DBG_ANALOG_SEL_9__SHIFT 0x0000000a
+#define PB0_RX_LANE9_CTRL_REG0__RX_TST_BSCAN_EN_9_MASK 0x00001000L
+#define PB0_RX_LANE9_CTRL_REG0__RX_TST_BSCAN_EN_9__SHIFT 0x0000000c
+#define PB0_RX_LANE9_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_9_MASK 0x00000008L
+#define PB0_RX_LANE9_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_9__SHIFT 0x00000003
+#define PB0_RX_LANE9_SCI_STAT_OVRD_REG0__ENABLEFOM_9_MASK 0x00000080L
+#define PB0_RX_LANE9_SCI_STAT_OVRD_REG0__ENABLEFOM_9__SHIFT 0x00000007
+#define PB0_RX_LANE9_SCI_STAT_OVRD_REG0__REQUESTFOM_9_MASK 0x00000100L
+#define PB0_RX_LANE9_SCI_STAT_OVRD_REG0__REQUESTFOM_9__SHIFT 0x00000008
+#define PB0_RX_LANE9_SCI_STAT_OVRD_REG0__RESPONSEMODE_9_MASK 0x00000200L
+#define PB0_RX_LANE9_SCI_STAT_OVRD_REG0__RESPONSEMODE_9__SHIFT 0x00000009
+#define PB0_RX_LANE9_SCI_STAT_OVRD_REG0__RXPRESETHINT_9_MASK 0x00000070L
+#define PB0_RX_LANE9_SCI_STAT_OVRD_REG0__RXPRESETHINT_9__SHIFT 0x00000004
+#define PB0_RX_LANE9_SCI_STAT_OVRD_REG0__RXPWR_9_MASK 0x00000007L
+#define PB0_RX_LANE9_SCI_STAT_OVRD_REG0__RXPWR_9__SHIFT 0x00000000
+#define PB0_STRAP_GLB_REG0__STRAP_DBG_RXDLL_VREG_REF_SEL_MASK 0x00008000L
+#define PB0_STRAP_GLB_REG0__STRAP_DBG_RXDLL_VREG_REF_SEL__SHIFT 0x0000000f
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_0_MASK 0x00000001L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_0__SHIFT 0x00000000
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_10_MASK 0x00000400L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_10__SHIFT 0x0000000a
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_11_MASK 0x00000800L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_11__SHIFT 0x0000000b
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_12_MASK 0x00001000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_12__SHIFT 0x0000000c
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_13_MASK 0x00002000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_13__SHIFT 0x0000000d
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_14_MASK 0x00004000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_14__SHIFT 0x0000000e
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_15_MASK 0x00008000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_15__SHIFT 0x0000000f
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_16_MASK 0x00010000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_16__SHIFT 0x00000010
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_17_MASK 0x00020000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_17__SHIFT 0x00000011
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_18_MASK 0x00040000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_18__SHIFT 0x00000012
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_19_MASK 0x00080000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_19__SHIFT 0x00000013
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_1_MASK 0x00000002L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_1__SHIFT 0x00000001
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_20_MASK 0x00100000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_20__SHIFT 0x00000014
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_21_MASK 0x00200000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_21__SHIFT 0x00000015
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_22_MASK 0x00400000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_22__SHIFT 0x00000016
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_23_MASK 0x00800000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_23__SHIFT 0x00000017
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_24_MASK 0x01000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_24__SHIFT 0x00000018
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_25_MASK 0x02000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_25__SHIFT 0x00000019
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_26_MASK 0x04000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_26__SHIFT 0x0000001a
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_27_MASK 0x08000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_27__SHIFT 0x0000001b
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_28_MASK 0x10000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_28__SHIFT 0x0000001c
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_29_MASK 0x20000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_29__SHIFT 0x0000001d
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_2_MASK 0x00000004L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_2__SHIFT 0x00000002
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_30_MASK 0x40000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_30__SHIFT 0x0000001e
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_31_MASK 0x80000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_31__SHIFT 0x0000001f
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_3_MASK 0x00000008L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_3__SHIFT 0x00000003
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_4_MASK 0x00000010L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_4__SHIFT 0x00000004
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_5_MASK 0x00000020L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_5__SHIFT 0x00000005
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_6_MASK 0x00000040L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_6__SHIFT 0x00000006
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_7_MASK 0x00000080L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_7__SHIFT 0x00000007
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_8_MASK 0x00000100L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_8__SHIFT 0x00000008
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_9_MASK 0x00000200L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_9__SHIFT 0x00000009
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_32_MASK 0x00000001L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_32__SHIFT 0x00000000
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_33_MASK 0x00000002L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_33__SHIFT 0x00000001
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_34_MASK 0x00000004L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_34__SHIFT 0x00000002
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_35_MASK 0x00000008L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_35__SHIFT 0x00000003
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_36_MASK 0x00000010L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_36__SHIFT 0x00000004
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_37_MASK 0x00000020L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_37__SHIFT 0x00000005
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_38_MASK 0x00000040L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_38__SHIFT 0x00000006
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_39_MASK 0x00000080L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_39__SHIFT 0x00000007
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_40_MASK 0x00000100L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_40__SHIFT 0x00000008
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_41_MASK 0x00000200L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_41__SHIFT 0x00000009
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_42_MASK 0x00000400L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_42__SHIFT 0x0000000a
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_43_MASK 0x00000800L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_43__SHIFT 0x0000000b
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_44_MASK 0x00001000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_44__SHIFT 0x0000000c
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_45_MASK 0x00002000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_45__SHIFT 0x0000000d
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_46_MASK 0x00004000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_46__SHIFT 0x0000000e
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_47_MASK 0x00008000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_47__SHIFT 0x0000000f
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_48_MASK 0x00010000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_48__SHIFT 0x00000010
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_49_MASK 0x00020000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_49__SHIFT 0x00000011
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_50_MASK 0x00040000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_50__SHIFT 0x00000012
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_51_MASK 0x00080000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_51__SHIFT 0x00000013
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_52_MASK 0x00100000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_52__SHIFT 0x00000014
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_53_MASK 0x00200000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_53__SHIFT 0x00000015
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_54_MASK 0x00400000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_54__SHIFT 0x00000016
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_55_MASK 0x00800000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_55__SHIFT 0x00000017
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_56_MASK 0x01000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_56__SHIFT 0x00000018
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_57_MASK 0x02000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_57__SHIFT 0x00000019
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_58_MASK 0x04000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_58__SHIFT 0x0000001a
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_59_MASK 0x08000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_59__SHIFT 0x0000001b
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_60_MASK 0x10000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_60__SHIFT 0x0000001c
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_61_MASK 0x20000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_61__SHIFT 0x0000001d
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_62_MASK 0x40000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_62__SHIFT 0x0000001e
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_63_MASK 0x80000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_63__SHIFT 0x0000001f
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_64_MASK 0x00000001L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_64__SHIFT 0x00000000
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_65_MASK 0x00000002L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_65__SHIFT 0x00000001
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_66_MASK 0x00000004L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_66__SHIFT 0x00000002
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_67_MASK 0x00000008L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_67__SHIFT 0x00000003
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_68_MASK 0x00000010L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_68__SHIFT 0x00000004
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_69_MASK 0x00000020L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_69__SHIFT 0x00000005
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_70_MASK 0x00000040L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_70__SHIFT 0x00000006
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_71_MASK 0x00000080L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_71__SHIFT 0x00000007
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_72_MASK 0x00000100L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_72__SHIFT 0x00000008
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_73_MASK 0x00000200L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_73__SHIFT 0x00000009
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_74_MASK 0x00000400L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_74__SHIFT 0x0000000a
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_75_MASK 0x00000800L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_75__SHIFT 0x0000000b
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_76_MASK 0x00001000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_76__SHIFT 0x0000000c
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_77_MASK 0x00002000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_77__SHIFT 0x0000000d
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_78_MASK 0x00004000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_78__SHIFT 0x0000000e
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_79_MASK 0x00008000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_79__SHIFT 0x0000000f
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_80_MASK 0x00010000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_80__SHIFT 0x00000010
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_81_MASK 0x00020000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_81__SHIFT 0x00000011
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_82_MASK 0x00040000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_82__SHIFT 0x00000012
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_83_MASK 0x00080000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_83__SHIFT 0x00000013
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_84_MASK 0x00100000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_84__SHIFT 0x00000014
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_85_MASK 0x00200000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_85__SHIFT 0x00000015
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_86_MASK 0x00400000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_86__SHIFT 0x00000016
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_87_MASK 0x00800000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_87__SHIFT 0x00000017
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_88_MASK 0x01000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_88__SHIFT 0x00000018
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_89_MASK 0x02000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_89__SHIFT 0x00000019
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_90_MASK 0x04000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_90__SHIFT 0x0000001a
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_91_MASK 0x08000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_91__SHIFT 0x0000001b
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_92_MASK 0x10000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_92__SHIFT 0x0000001c
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_93_MASK 0x20000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_93__SHIFT 0x0000001d
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_94_MASK 0x40000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_94__SHIFT 0x0000001e
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_95_MASK 0x80000000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_95__SHIFT 0x0000001f
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_100_MASK 0x00000010L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_100__SHIFT 0x00000004
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_101_MASK 0x00000020L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_101__SHIFT 0x00000005
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_102_MASK 0x00000040L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_102__SHIFT 0x00000006
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_103_MASK 0x00000080L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_103__SHIFT 0x00000007
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_104_MASK 0x00000100L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_104__SHIFT 0x00000008
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_105_MASK 0x00000200L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_105__SHIFT 0x00000009
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_106_MASK 0x00000400L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_106__SHIFT 0x0000000a
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_107_MASK 0x00000800L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_107__SHIFT 0x0000000b
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_108_MASK 0x00001000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_108__SHIFT 0x0000000c
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_109_MASK 0x00002000L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_109__SHIFT 0x0000000d
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_96_MASK 0x00000001L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_96__SHIFT 0x00000000
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_97_MASK 0x00000002L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_97__SHIFT 0x00000001
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_98_MASK 0x00000004L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_98__SHIFT 0x00000002
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_99_MASK 0x00000008L
+#define PB0_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_99__SHIFT 0x00000003
+#define PB0_TX_GLB_CTRL_REG0__TX_CFG_RPTR_RST_VAL_GEN1_MASK 0x00000700L
+#define PB0_TX_GLB_CTRL_REG0__TX_CFG_RPTR_RST_VAL_GEN1__SHIFT 0x00000008
+#define PB0_TX_GLB_CTRL_REG0__TX_CFG_RPTR_RST_VAL_GEN2_MASK 0x00003800L
+#define PB0_TX_GLB_CTRL_REG0__TX_CFG_RPTR_RST_VAL_GEN2__SHIFT 0x0000000b
+#define PB0_TX_GLB_CTRL_REG0__TX_CFG_RPTR_RST_VAL_GEN3_MASK 0x0001c000L
+#define PB0_TX_GLB_CTRL_REG0__TX_CFG_RPTR_RST_VAL_GEN3__SHIFT 0x0000000e
+#define PB0_TX_GLB_CTRL_REG0__TX_COEFF_ROUND_DIR_VER_MASK 0x00400000L
+#define PB0_TX_GLB_CTRL_REG0__TX_COEFF_ROUND_DIR_VER__SHIFT 0x00000016
+#define PB0_TX_GLB_CTRL_REG0__TX_COEFF_ROUND_EN_MASK 0x00200000L
+#define PB0_TX_GLB_CTRL_REG0__TX_COEFF_ROUND_EN__SHIFT 0x00000015
+#define PB0_TX_GLB_CTRL_REG0__TX_DATA_CLK_GATING_MASK 0x00080000L
+#define PB0_TX_GLB_CTRL_REG0__TX_DATA_CLK_GATING__SHIFT 0x00000013
+#define PB0_TX_GLB_CTRL_REG0__TX_DCLK_EN_LSX_ALWAYS_ON_MASK 0x00800000L
+#define PB0_TX_GLB_CTRL_REG0__TX_DCLK_EN_LSX_ALWAYS_ON__SHIFT 0x00000017
+#define PB0_TX_GLB_CTRL_REG0__TX_DRV_DATA_ASRT_DLY_VAL_MASK 0x00000007L
+#define PB0_TX_GLB_CTRL_REG0__TX_DRV_DATA_ASRT_DLY_VAL__SHIFT 0x00000000
+#define PB0_TX_GLB_CTRL_REG0__TX_DRV_DATA_DSRT_DLY_VAL_MASK 0x00000038L
+#define PB0_TX_GLB_CTRL_REG0__TX_DRV_DATA_DSRT_DLY_VAL__SHIFT 0x00000003
+#define PB0_TX_GLB_CTRL_REG0__TX_FRONTEND_PWRON_IN_OFF_MASK 0x01000000L
+#define PB0_TX_GLB_CTRL_REG0__TX_FRONTEND_PWRON_IN_OFF__SHIFT 0x00000018
+#define PB0_TX_GLB_CTRL_REG0__TX_PRESET_TABLE_BYPASS_MASK 0x00100000L
+#define PB0_TX_GLB_CTRL_REG0__TX_PRESET_TABLE_BYPASS__SHIFT 0x00000014
+#define PB0_TX_GLB_CTRL_REG0__TX_STAGGER_CTRL_MASK 0x00060000L
+#define PB0_TX_GLB_CTRL_REG0__TX_STAGGER_CTRL__SHIFT 0x00000011
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX16_EN_L0T15_MASK 0x40000000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX16_EN_L0T15__SHIFT 0x0000001e
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_0_MASK 0x00000001L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_0__SHIFT 0x00000000
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_10_MASK 0x00000400L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_10__SHIFT 0x0000000a
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_11_MASK 0x00000800L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_11__SHIFT 0x0000000b
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_12_MASK 0x00001000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_12__SHIFT 0x0000000c
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_13_MASK 0x00002000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_13__SHIFT 0x0000000d
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_14_MASK 0x00004000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_14__SHIFT 0x0000000e
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_15_MASK 0x00008000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_15__SHIFT 0x0000000f
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_1_MASK 0x00000002L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_1__SHIFT 0x00000001
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_2_MASK 0x00000004L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_2__SHIFT 0x00000002
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_3_MASK 0x00000008L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_3__SHIFT 0x00000003
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_4_MASK 0x00000010L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_4__SHIFT 0x00000004
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_5_MASK 0x00000020L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_5__SHIFT 0x00000005
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_6_MASK 0x00000040L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_6__SHIFT 0x00000006
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_7_MASK 0x00000080L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_7__SHIFT 0x00000007
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_8_MASK 0x00000100L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_8__SHIFT 0x00000008
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_9_MASK 0x00000200L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_9__SHIFT 0x00000009
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L0T1_MASK 0x00010000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L0T1__SHIFT 0x00000010
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L10T11_MASK 0x00200000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L10T11__SHIFT 0x00000015
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L12T13_MASK 0x00400000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L12T13__SHIFT 0x00000016
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L14T15_MASK 0x00800000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L14T15__SHIFT 0x00000017
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L2T3_MASK 0x00020000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L2T3__SHIFT 0x00000011
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L4T5_MASK 0x00040000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L4T5__SHIFT 0x00000012
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L6T7_MASK 0x00080000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L6T7__SHIFT 0x00000013
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L8T9_MASK 0x00100000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L8T9__SHIFT 0x00000014
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L0T3_MASK 0x01000000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L0T3__SHIFT 0x00000018
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L12T15_MASK 0x08000000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L12T15__SHIFT 0x0000001b
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L4T7_MASK 0x02000000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L4T7__SHIFT 0x00000019
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L8T11_MASK 0x04000000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L8T11__SHIFT 0x0000001a
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX8_EN_L0T7_MASK 0x10000000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX8_EN_L0T7__SHIFT 0x0000001c
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX8_EN_L8T15_MASK 0x20000000L
+#define PB0_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX8_EN_L8T15__SHIFT 0x0000001d
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DCLK_DIV_OVRD_EN_MASK 0x00000008L
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DCLK_DIV_OVRD_EN__SHIFT 0x00000003
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DCLK_DIV_OVRD_VAL_MASK 0x00000007L
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DCLK_DIV_OVRD_VAL__SHIFT 0x00000000
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV0_EN_GEN1_OVRD_VAL_MASK 0x000000f0L
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV0_EN_GEN1_OVRD_VAL__SHIFT 0x00000004
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV0_EN_OVRD_EN_MASK 0x00000100L
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV0_EN_OVRD_EN__SHIFT 0x00000008
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV0_TAP_SEL_GEN1_OVRD_VAL_MASK 0x00001e00L
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV0_TAP_SEL_GEN1_OVRD_VAL__SHIFT 0x00000009
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV0_TAP_SEL_OVRD_EN_MASK 0x00002000L
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV0_TAP_SEL_OVRD_EN__SHIFT 0x0000000d
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV1_EN_GEN1_OVRD_VAL_MASK 0x0007c000L
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV1_EN_GEN1_OVRD_VAL__SHIFT 0x0000000e
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV1_EN_OVRD_EN_MASK 0x00080000L
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV1_EN_OVRD_EN__SHIFT 0x00000013
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV1_TAP_SEL_GEN1_OVRD_VAL_MASK 0x01f00000L
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV1_TAP_SEL_GEN1_OVRD_VAL__SHIFT 0x00000014
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV1_TAP_SEL_OVRD_EN_MASK 0x02000000L
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV1_TAP_SEL_OVRD_EN__SHIFT 0x00000019
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV2_EN_GEN1_OVRD_VAL_MASK 0x3c000000L
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV2_EN_GEN1_OVRD_VAL__SHIFT 0x0000001a
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV2_EN_OVRD_EN_MASK 0x40000000L
+#define PB0_TX_GLB_OVRD_REG0__TX_CFG_DRV2_EN_OVRD_EN__SHIFT 0x0000001e
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_DRV2_TAP_SEL_GEN1_OVRD_VAL_MASK 0x0000000fL
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_DRV2_TAP_SEL_GEN1_OVRD_VAL__SHIFT 0x00000000
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_DRV2_TAP_SEL_OVRD_EN_MASK 0x00000010L
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_DRV2_TAP_SEL_OVRD_EN__SHIFT 0x00000004
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_DRVX_EN_GEN1_OVRD_VAL_MASK 0x00000020L
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_DRVX_EN_GEN1_OVRD_VAL__SHIFT 0x00000005
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_DRVX_EN_OVRD_EN_MASK 0x00000040L
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_DRVX_EN_OVRD_EN__SHIFT 0x00000006
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_DRVX_TAP_SEL_GEN1_OVRD_VAL_MASK 0x00000080L
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_DRVX_TAP_SEL_GEN1_OVRD_VAL__SHIFT 0x00000007
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_DRVX_TAP_SEL_OVRD_EN_MASK 0x00000100L
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_DRVX_TAP_SEL_OVRD_EN__SHIFT 0x00000008
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_PLLCLK_SEL_OVRD_EN_MASK 0x00000400L
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_PLLCLK_SEL_OVRD_EN__SHIFT 0x0000000a
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_PLLCLK_SEL_OVRD_VAL_MASK 0x00000200L
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_PLLCLK_SEL_OVRD_VAL__SHIFT 0x00000009
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_TCLK_DIV_OVRD_EN_MASK 0x00001000L
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_TCLK_DIV_OVRD_EN__SHIFT 0x0000000c
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_TCLK_DIV_OVRD_VAL_MASK 0x00000800L
+#define PB0_TX_GLB_OVRD_REG1__TX_CFG_TCLK_DIV_OVRD_VAL__SHIFT 0x0000000b
+#define PB0_TX_GLB_OVRD_REG1__TX_CMDET_EN_OVRD_EN_MASK 0x00004000L
+#define PB0_TX_GLB_OVRD_REG1__TX_CMDET_EN_OVRD_EN__SHIFT 0x0000000e
+#define PB0_TX_GLB_OVRD_REG1__TX_CMDET_EN_OVRD_VAL_MASK 0x00002000L
+#define PB0_TX_GLB_OVRD_REG1__TX_CMDET_EN_OVRD_VAL__SHIFT 0x0000000d
+#define PB0_TX_GLB_OVRD_REG1__TX_DATA_IN_OVRD_EN_MASK 0x02000000L
+#define PB0_TX_GLB_OVRD_REG1__TX_DATA_IN_OVRD_EN__SHIFT 0x00000019
+#define PB0_TX_GLB_OVRD_REG1__TX_DATA_IN_OVRD_VAL_MASK 0x01ff8000L
+#define PB0_TX_GLB_OVRD_REG1__TX_DATA_IN_OVRD_VAL__SHIFT 0x0000000f
+#define PB0_TX_GLB_OVRD_REG1__TX_RPTR_RSTN_OVRD_EN_MASK 0x08000000L
+#define PB0_TX_GLB_OVRD_REG1__TX_RPTR_RSTN_OVRD_EN__SHIFT 0x0000001b
+#define PB0_TX_GLB_OVRD_REG1__TX_RPTR_RSTN_OVRD_VAL_MASK 0x04000000L
+#define PB0_TX_GLB_OVRD_REG1__TX_RPTR_RSTN_OVRD_VAL__SHIFT 0x0000001a
+#define PB0_TX_GLB_OVRD_REG1__TX_RXDET_EN_OVRD_EN_MASK 0x20000000L
+#define PB0_TX_GLB_OVRD_REG1__TX_RXDET_EN_OVRD_EN__SHIFT 0x0000001d
+#define PB0_TX_GLB_OVRD_REG1__TX_RXDET_EN_OVRD_VAL_MASK 0x10000000L
+#define PB0_TX_GLB_OVRD_REG1__TX_RXDET_EN_OVRD_VAL__SHIFT 0x0000001c
+#define PB0_TX_GLB_OVRD_REG1__TX_WPTR_RSTN_OVRD_EN_MASK 0x80000000L
+#define PB0_TX_GLB_OVRD_REG1__TX_WPTR_RSTN_OVRD_EN__SHIFT 0x0000001f
+#define PB0_TX_GLB_OVRD_REG1__TX_WPTR_RSTN_OVRD_VAL_MASK 0x40000000L
+#define PB0_TX_GLB_OVRD_REG1__TX_WPTR_RSTN_OVRD_VAL__SHIFT 0x0000001e
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_DRV0_EN_GEN2_OVRD_VAL_MASK 0x0000f000L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_DRV0_EN_GEN2_OVRD_VAL__SHIFT 0x0000000c
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_DRV0_TAP_SEL_GEN2_OVRD_VAL_MASK 0x000f0000L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_DRV0_TAP_SEL_GEN2_OVRD_VAL__SHIFT 0x00000010
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_DRV1_EN_GEN2_OVRD_VAL_MASK 0x01f00000L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_DRV1_EN_GEN2_OVRD_VAL__SHIFT 0x00000014
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_DRV1_TAP_SEL_GEN2_OVRD_VAL_MASK 0x3e000000L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_DRV1_TAP_SEL_GEN2_OVRD_VAL__SHIFT 0x00000019
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX16_EN_OVRD_EN_MASK 0x00000800L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX16_EN_OVRD_EN__SHIFT 0x0000000b
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX16_EN_OVRD_VAL_MASK 0x00000400L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX16_EN_OVRD_VAL__SHIFT 0x0000000a
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX1_EN_OVRD_EN_MASK 0x00000008L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX1_EN_OVRD_EN__SHIFT 0x00000003
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX1_EN_OVRD_VAL_MASK 0x00000004L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX1_EN_OVRD_VAL__SHIFT 0x00000002
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX2_EN_OVRD_EN_MASK 0x00000020L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX2_EN_OVRD_EN__SHIFT 0x00000005
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX2_EN_OVRD_VAL_MASK 0x00000010L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX2_EN_OVRD_VAL__SHIFT 0x00000004
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX4_EN_OVRD_EN_MASK 0x00000080L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX4_EN_OVRD_EN__SHIFT 0x00000007
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX4_EN_OVRD_VAL_MASK 0x00000040L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX4_EN_OVRD_VAL__SHIFT 0x00000006
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX8_EN_OVRD_EN_MASK 0x00000200L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX8_EN_OVRD_EN__SHIFT 0x00000009
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX8_EN_OVRD_VAL_MASK 0x00000100L
+#define PB0_TX_GLB_OVRD_REG2__TX_CFG_GROUPX8_EN_OVRD_VAL__SHIFT 0x00000008
+#define PB0_TX_GLB_OVRD_REG2__TX_WRITE_EN_OVRD_EN_MASK 0x00000002L
+#define PB0_TX_GLB_OVRD_REG2__TX_WRITE_EN_OVRD_EN__SHIFT 0x00000001
+#define PB0_TX_GLB_OVRD_REG2__TX_WRITE_EN_OVRD_VAL_MASK 0x00000001L
+#define PB0_TX_GLB_OVRD_REG2__TX_WRITE_EN_OVRD_VAL__SHIFT 0x00000000
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV0_EN_GEN3_OVRD_VAL_MASK 0x00003c00L
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV0_EN_GEN3_OVRD_VAL__SHIFT 0x0000000a
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV0_TAP_SEL_GEN3_OVRD_VAL_MASK 0x0003c000L
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV0_TAP_SEL_GEN3_OVRD_VAL__SHIFT 0x0000000e
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV1_EN_GEN3_OVRD_VAL_MASK 0x007c0000L
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV1_EN_GEN3_OVRD_VAL__SHIFT 0x00000012
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV1_TAP_SEL_GEN3_OVRD_VAL_MASK 0x0f800000L
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV1_TAP_SEL_GEN3_OVRD_VAL__SHIFT 0x00000017
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV2_EN_GEN2_OVRD_VAL_MASK 0x0000000fL
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV2_EN_GEN2_OVRD_VAL__SHIFT 0x00000000
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV2_EN_GEN3_OVRD_VAL_MASK 0xf0000000L
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV2_EN_GEN3_OVRD_VAL__SHIFT 0x0000001c
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV2_TAP_SEL_GEN2_OVRD_VAL_MASK 0x000000f0L
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRV2_TAP_SEL_GEN2_OVRD_VAL__SHIFT 0x00000004
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRVX_EN_GEN2_OVRD_VAL_MASK 0x00000100L
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRVX_EN_GEN2_OVRD_VAL__SHIFT 0x00000008
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRVX_TAP_SEL_GEN2_OVRD_VAL_MASK 0x00000200L
+#define PB0_TX_GLB_OVRD_REG3__TX_CFG_DRVX_TAP_SEL_GEN2_OVRD_VAL__SHIFT 0x00000009
+#define PB0_TX_GLB_OVRD_REG4__TX_CFG_DRV2_TAP_SEL_GEN3_OVRD_VAL_MASK 0x0000000fL
+#define PB0_TX_GLB_OVRD_REG4__TX_CFG_DRV2_TAP_SEL_GEN3_OVRD_VAL__SHIFT 0x00000000
+#define PB0_TX_GLB_OVRD_REG4__TX_CFG_DRVX_EN_GEN3_OVRD_VAL_MASK 0x00000010L
+#define PB0_TX_GLB_OVRD_REG4__TX_CFG_DRVX_EN_GEN3_OVRD_VAL__SHIFT 0x00000004
+#define PB0_TX_GLB_OVRD_REG4__TX_CFG_DRVX_TAP_SEL_GEN3_OVRD_VAL_MASK 0x00000020L
+#define PB0_TX_GLB_OVRD_REG4__TX_CFG_DRVX_TAP_SEL_GEN3_OVRD_VAL__SHIFT 0x00000005
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L0T3_MASK 0x00000100L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L0T3__SHIFT 0x00000008
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L12T15_MASK 0x00000800L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L12T15__SHIFT 0x0000000b
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L4T7_MASK 0x00000200L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L4T7__SHIFT 0x00000009
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L8T11_MASK 0x00000400L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L8T11__SHIFT 0x0000000a
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L0T3_MASK 0x00001000L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L0T3__SHIFT 0x0000000c
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L12T15_MASK 0x00008000L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L12T15__SHIFT 0x0000000f
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L4T7_MASK 0x00002000L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L4T7__SHIFT 0x0000000d
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L8T11_MASK 0x00004000L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L8T11__SHIFT 0x0000000e
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L0T3_MASK 0x00000010L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L0T3__SHIFT 0x00000004
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L12T15_MASK 0x00000080L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L12T15__SHIFT 0x00000007
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L4T7_MASK 0x00000020L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L4T7__SHIFT 0x00000005
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L8T11_MASK 0x00000040L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L8T11__SHIFT 0x00000006
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L0T3_MASK 0x00000001L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L0T3__SHIFT 0x00000000
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L12T15_MASK 0x00000008L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L12T15__SHIFT 0x00000003
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L4T7_MASK 0x00000002L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L4T7__SHIFT 0x00000001
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L8T11_MASK 0x00000004L
+#define PB0_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L8T11__SHIFT 0x00000002
+#define PB0_TX_LANE0_CTRL_REG0__TX_CFG_DISPCLK_MODE_0_MASK 0x00000001L
+#define PB0_TX_LANE0_CTRL_REG0__TX_CFG_DISPCLK_MODE_0__SHIFT 0x00000000
+#define PB0_TX_LANE0_CTRL_REG0__TX_CFG_INV_DATA_0_MASK 0x00000002L
+#define PB0_TX_LANE0_CTRL_REG0__TX_CFG_INV_DATA_0__SHIFT 0x00000001
+#define PB0_TX_LANE0_CTRL_REG0__TX_CFG_SWING_BOOST_EN_0_MASK 0x00000004L
+#define PB0_TX_LANE0_CTRL_REG0__TX_CFG_SWING_BOOST_EN_0__SHIFT 0x00000002
+#define PB0_TX_LANE0_CTRL_REG0__TX_DBG_PRBS_EN_0_MASK 0x00000008L
+#define PB0_TX_LANE0_CTRL_REG0__TX_DBG_PRBS_EN_0__SHIFT 0x00000003
+#define PB0_TX_LANE0_OVRD_REG0__TX_DCLK_EN_OVRD_EN_0_MASK 0x00000002L
+#define PB0_TX_LANE0_OVRD_REG0__TX_DCLK_EN_OVRD_EN_0__SHIFT 0x00000001
+#define PB0_TX_LANE0_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_0_MASK 0x00000001L
+#define PB0_TX_LANE0_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_0__SHIFT 0x00000000
+#define PB0_TX_LANE0_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_0_MASK 0x00000008L
+#define PB0_TX_LANE0_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_0__SHIFT 0x00000003
+#define PB0_TX_LANE0_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_0_MASK 0x00000004L
+#define PB0_TX_LANE0_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_0__SHIFT 0x00000002
+#define PB0_TX_LANE0_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_0_MASK 0x00000020L
+#define PB0_TX_LANE0_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_0__SHIFT 0x00000005
+#define PB0_TX_LANE0_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_0_MASK 0x00000010L
+#define PB0_TX_LANE0_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_0__SHIFT 0x00000004
+#define PB0_TX_LANE0_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_0_MASK 0x00000080L
+#define PB0_TX_LANE0_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_0__SHIFT 0x00000007
+#define PB0_TX_LANE0_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_0_MASK 0x00000040L
+#define PB0_TX_LANE0_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_0__SHIFT 0x00000006
+#define PB0_TX_LANE0_SCI_STAT_OVRD_REG0__COEFFICIENT_0_MASK 0x0000fc00L
+#define PB0_TX_LANE0_SCI_STAT_OVRD_REG0__COEFFICIENT_0__SHIFT 0x0000000a
+#define PB0_TX_LANE0_SCI_STAT_OVRD_REG0__COEFFICIENTID_0_MASK 0x00000300L
+#define PB0_TX_LANE0_SCI_STAT_OVRD_REG0__COEFFICIENTID_0__SHIFT 0x00000008
+#define PB0_TX_LANE0_SCI_STAT_OVRD_REG0__DEEMPH_0_MASK 0x00000080L
+#define PB0_TX_LANE0_SCI_STAT_OVRD_REG0__DEEMPH_0__SHIFT 0x00000007
+#define PB0_TX_LANE0_SCI_STAT_OVRD_REG0__INCOHERENTCK_0_MASK 0x00000008L
+#define PB0_TX_LANE0_SCI_STAT_OVRD_REG0__INCOHERENTCK_0__SHIFT 0x00000003
+#define PB0_TX_LANE0_SCI_STAT_OVRD_REG0__TXMARG_0_MASK 0x00000070L
+#define PB0_TX_LANE0_SCI_STAT_OVRD_REG0__TXMARG_0__SHIFT 0x00000004
+#define PB0_TX_LANE0_SCI_STAT_OVRD_REG0__TXPWR_0_MASK 0x00000007L
+#define PB0_TX_LANE0_SCI_STAT_OVRD_REG0__TXPWR_0__SHIFT 0x00000000
+#define PB0_TX_LANE10_CTRL_REG0__TX_CFG_DISPCLK_MODE_10_MASK 0x00000001L
+#define PB0_TX_LANE10_CTRL_REG0__TX_CFG_DISPCLK_MODE_10__SHIFT 0x00000000
+#define PB0_TX_LANE10_CTRL_REG0__TX_CFG_INV_DATA_10_MASK 0x00000002L
+#define PB0_TX_LANE10_CTRL_REG0__TX_CFG_INV_DATA_10__SHIFT 0x00000001
+#define PB0_TX_LANE10_CTRL_REG0__TX_CFG_SWING_BOOST_EN_10_MASK 0x00000004L
+#define PB0_TX_LANE10_CTRL_REG0__TX_CFG_SWING_BOOST_EN_10__SHIFT 0x00000002
+#define PB0_TX_LANE10_CTRL_REG0__TX_DBG_PRBS_EN_10_MASK 0x00000008L
+#define PB0_TX_LANE10_CTRL_REG0__TX_DBG_PRBS_EN_10__SHIFT 0x00000003
+#define PB0_TX_LANE10_OVRD_REG0__TX_DCLK_EN_OVRD_EN_10_MASK 0x00000002L
+#define PB0_TX_LANE10_OVRD_REG0__TX_DCLK_EN_OVRD_EN_10__SHIFT 0x00000001
+#define PB0_TX_LANE10_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_10_MASK 0x00000001L
+#define PB0_TX_LANE10_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_10__SHIFT 0x00000000
+#define PB0_TX_LANE10_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_10_MASK 0x00000008L
+#define PB0_TX_LANE10_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_10__SHIFT 0x00000003
+#define PB0_TX_LANE10_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_10_MASK 0x00000004L
+#define PB0_TX_LANE10_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_10__SHIFT 0x00000002
+#define PB0_TX_LANE10_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_10_MASK 0x00000020L
+#define PB0_TX_LANE10_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_10__SHIFT 0x00000005
+#define PB0_TX_LANE10_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_10_MASK 0x00000010L
+#define PB0_TX_LANE10_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_10__SHIFT 0x00000004
+#define PB0_TX_LANE10_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_10_MASK 0x00000080L
+#define PB0_TX_LANE10_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_10__SHIFT 0x00000007
+#define PB0_TX_LANE10_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_10_MASK 0x00000040L
+#define PB0_TX_LANE10_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_10__SHIFT 0x00000006
+#define PB0_TX_LANE10_SCI_STAT_OVRD_REG0__COEFFICIENT_10_MASK 0x0000fc00L
+#define PB0_TX_LANE10_SCI_STAT_OVRD_REG0__COEFFICIENT_10__SHIFT 0x0000000a
+#define PB0_TX_LANE10_SCI_STAT_OVRD_REG0__COEFFICIENTID_10_MASK 0x00000300L
+#define PB0_TX_LANE10_SCI_STAT_OVRD_REG0__COEFFICIENTID_10__SHIFT 0x00000008
+#define PB0_TX_LANE10_SCI_STAT_OVRD_REG0__DEEMPH_10_MASK 0x00000080L
+#define PB0_TX_LANE10_SCI_STAT_OVRD_REG0__DEEMPH_10__SHIFT 0x00000007
+#define PB0_TX_LANE10_SCI_STAT_OVRD_REG0__INCOHERENTCK_10_MASK 0x00000008L
+#define PB0_TX_LANE10_SCI_STAT_OVRD_REG0__INCOHERENTCK_10__SHIFT 0x00000003
+#define PB0_TX_LANE10_SCI_STAT_OVRD_REG0__TXMARG_10_MASK 0x00000070L
+#define PB0_TX_LANE10_SCI_STAT_OVRD_REG0__TXMARG_10__SHIFT 0x00000004
+#define PB0_TX_LANE10_SCI_STAT_OVRD_REG0__TXPWR_10_MASK 0x00000007L
+#define PB0_TX_LANE10_SCI_STAT_OVRD_REG0__TXPWR_10__SHIFT 0x00000000
+#define PB0_TX_LANE11_CTRL_REG0__TX_CFG_DISPCLK_MODE_11_MASK 0x00000001L
+#define PB0_TX_LANE11_CTRL_REG0__TX_CFG_DISPCLK_MODE_11__SHIFT 0x00000000
+#define PB0_TX_LANE11_CTRL_REG0__TX_CFG_INV_DATA_11_MASK 0x00000002L
+#define PB0_TX_LANE11_CTRL_REG0__TX_CFG_INV_DATA_11__SHIFT 0x00000001
+#define PB0_TX_LANE11_CTRL_REG0__TX_CFG_SWING_BOOST_EN_11_MASK 0x00000004L
+#define PB0_TX_LANE11_CTRL_REG0__TX_CFG_SWING_BOOST_EN_11__SHIFT 0x00000002
+#define PB0_TX_LANE11_CTRL_REG0__TX_DBG_PRBS_EN_11_MASK 0x00000008L
+#define PB0_TX_LANE11_CTRL_REG0__TX_DBG_PRBS_EN_11__SHIFT 0x00000003
+#define PB0_TX_LANE11_OVRD_REG0__TX_DCLK_EN_OVRD_EN_11_MASK 0x00000002L
+#define PB0_TX_LANE11_OVRD_REG0__TX_DCLK_EN_OVRD_EN_11__SHIFT 0x00000001
+#define PB0_TX_LANE11_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_11_MASK 0x00000001L
+#define PB0_TX_LANE11_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_11__SHIFT 0x00000000
+#define PB0_TX_LANE11_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_11_MASK 0x00000008L
+#define PB0_TX_LANE11_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_11__SHIFT 0x00000003
+#define PB0_TX_LANE11_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_11_MASK 0x00000004L
+#define PB0_TX_LANE11_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_11__SHIFT 0x00000002
+#define PB0_TX_LANE11_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_11_MASK 0x00000020L
+#define PB0_TX_LANE11_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_11__SHIFT 0x00000005
+#define PB0_TX_LANE11_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_11_MASK 0x00000010L
+#define PB0_TX_LANE11_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_11__SHIFT 0x00000004
+#define PB0_TX_LANE11_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_11_MASK 0x00000080L
+#define PB0_TX_LANE11_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_11__SHIFT 0x00000007
+#define PB0_TX_LANE11_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_11_MASK 0x00000040L
+#define PB0_TX_LANE11_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_11__SHIFT 0x00000006
+#define PB0_TX_LANE11_SCI_STAT_OVRD_REG0__COEFFICIENT_11_MASK 0x0000fc00L
+#define PB0_TX_LANE11_SCI_STAT_OVRD_REG0__COEFFICIENT_11__SHIFT 0x0000000a
+#define PB0_TX_LANE11_SCI_STAT_OVRD_REG0__COEFFICIENTID_11_MASK 0x00000300L
+#define PB0_TX_LANE11_SCI_STAT_OVRD_REG0__COEFFICIENTID_11__SHIFT 0x00000008
+#define PB0_TX_LANE11_SCI_STAT_OVRD_REG0__DEEMPH_11_MASK 0x00000080L
+#define PB0_TX_LANE11_SCI_STAT_OVRD_REG0__DEEMPH_11__SHIFT 0x00000007
+#define PB0_TX_LANE11_SCI_STAT_OVRD_REG0__INCOHERENTCK_11_MASK 0x00000008L
+#define PB0_TX_LANE11_SCI_STAT_OVRD_REG0__INCOHERENTCK_11__SHIFT 0x00000003
+#define PB0_TX_LANE11_SCI_STAT_OVRD_REG0__TXMARG_11_MASK 0x00000070L
+#define PB0_TX_LANE11_SCI_STAT_OVRD_REG0__TXMARG_11__SHIFT 0x00000004
+#define PB0_TX_LANE11_SCI_STAT_OVRD_REG0__TXPWR_11_MASK 0x00000007L
+#define PB0_TX_LANE11_SCI_STAT_OVRD_REG0__TXPWR_11__SHIFT 0x00000000
+#define PB0_TX_LANE12_CTRL_REG0__TX_CFG_DISPCLK_MODE_12_MASK 0x00000001L
+#define PB0_TX_LANE12_CTRL_REG0__TX_CFG_DISPCLK_MODE_12__SHIFT 0x00000000
+#define PB0_TX_LANE12_CTRL_REG0__TX_CFG_INV_DATA_12_MASK 0x00000002L
+#define PB0_TX_LANE12_CTRL_REG0__TX_CFG_INV_DATA_12__SHIFT 0x00000001
+#define PB0_TX_LANE12_CTRL_REG0__TX_CFG_SWING_BOOST_EN_12_MASK 0x00000004L
+#define PB0_TX_LANE12_CTRL_REG0__TX_CFG_SWING_BOOST_EN_12__SHIFT 0x00000002
+#define PB0_TX_LANE12_CTRL_REG0__TX_DBG_PRBS_EN_12_MASK 0x00000008L
+#define PB0_TX_LANE12_CTRL_REG0__TX_DBG_PRBS_EN_12__SHIFT 0x00000003
+#define PB0_TX_LANE12_OVRD_REG0__TX_DCLK_EN_OVRD_EN_12_MASK 0x00000002L
+#define PB0_TX_LANE12_OVRD_REG0__TX_DCLK_EN_OVRD_EN_12__SHIFT 0x00000001
+#define PB0_TX_LANE12_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_12_MASK 0x00000001L
+#define PB0_TX_LANE12_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_12__SHIFT 0x00000000
+#define PB0_TX_LANE12_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_12_MASK 0x00000008L
+#define PB0_TX_LANE12_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_12__SHIFT 0x00000003
+#define PB0_TX_LANE12_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_12_MASK 0x00000004L
+#define PB0_TX_LANE12_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_12__SHIFT 0x00000002
+#define PB0_TX_LANE12_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_12_MASK 0x00000020L
+#define PB0_TX_LANE12_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_12__SHIFT 0x00000005
+#define PB0_TX_LANE12_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_12_MASK 0x00000010L
+#define PB0_TX_LANE12_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_12__SHIFT 0x00000004
+#define PB0_TX_LANE12_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_12_MASK 0x00000080L
+#define PB0_TX_LANE12_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_12__SHIFT 0x00000007
+#define PB0_TX_LANE12_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_12_MASK 0x00000040L
+#define PB0_TX_LANE12_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_12__SHIFT 0x00000006
+#define PB0_TX_LANE12_SCI_STAT_OVRD_REG0__COEFFICIENT_12_MASK 0x0000fc00L
+#define PB0_TX_LANE12_SCI_STAT_OVRD_REG0__COEFFICIENT_12__SHIFT 0x0000000a
+#define PB0_TX_LANE12_SCI_STAT_OVRD_REG0__COEFFICIENTID_12_MASK 0x00000300L
+#define PB0_TX_LANE12_SCI_STAT_OVRD_REG0__COEFFICIENTID_12__SHIFT 0x00000008
+#define PB0_TX_LANE12_SCI_STAT_OVRD_REG0__DEEMPH_12_MASK 0x00000080L
+#define PB0_TX_LANE12_SCI_STAT_OVRD_REG0__DEEMPH_12__SHIFT 0x00000007
+#define PB0_TX_LANE12_SCI_STAT_OVRD_REG0__INCOHERENTCK_12_MASK 0x00000008L
+#define PB0_TX_LANE12_SCI_STAT_OVRD_REG0__INCOHERENTCK_12__SHIFT 0x00000003
+#define PB0_TX_LANE12_SCI_STAT_OVRD_REG0__TXMARG_12_MASK 0x00000070L
+#define PB0_TX_LANE12_SCI_STAT_OVRD_REG0__TXMARG_12__SHIFT 0x00000004
+#define PB0_TX_LANE12_SCI_STAT_OVRD_REG0__TXPWR_12_MASK 0x00000007L
+#define PB0_TX_LANE12_SCI_STAT_OVRD_REG0__TXPWR_12__SHIFT 0x00000000
+#define PB0_TX_LANE13_CTRL_REG0__TX_CFG_DISPCLK_MODE_13_MASK 0x00000001L
+#define PB0_TX_LANE13_CTRL_REG0__TX_CFG_DISPCLK_MODE_13__SHIFT 0x00000000
+#define PB0_TX_LANE13_CTRL_REG0__TX_CFG_INV_DATA_13_MASK 0x00000002L
+#define PB0_TX_LANE13_CTRL_REG0__TX_CFG_INV_DATA_13__SHIFT 0x00000001
+#define PB0_TX_LANE13_CTRL_REG0__TX_CFG_SWING_BOOST_EN_13_MASK 0x00000004L
+#define PB0_TX_LANE13_CTRL_REG0__TX_CFG_SWING_BOOST_EN_13__SHIFT 0x00000002
+#define PB0_TX_LANE13_CTRL_REG0__TX_DBG_PRBS_EN_13_MASK 0x00000008L
+#define PB0_TX_LANE13_CTRL_REG0__TX_DBG_PRBS_EN_13__SHIFT 0x00000003
+#define PB0_TX_LANE13_OVRD_REG0__TX_DCLK_EN_OVRD_EN_13_MASK 0x00000002L
+#define PB0_TX_LANE13_OVRD_REG0__TX_DCLK_EN_OVRD_EN_13__SHIFT 0x00000001
+#define PB0_TX_LANE13_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_13_MASK 0x00000001L
+#define PB0_TX_LANE13_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_13__SHIFT 0x00000000
+#define PB0_TX_LANE13_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_13_MASK 0x00000008L
+#define PB0_TX_LANE13_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_13__SHIFT 0x00000003
+#define PB0_TX_LANE13_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_13_MASK 0x00000004L
+#define PB0_TX_LANE13_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_13__SHIFT 0x00000002
+#define PB0_TX_LANE13_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_13_MASK 0x00000020L
+#define PB0_TX_LANE13_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_13__SHIFT 0x00000005
+#define PB0_TX_LANE13_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_13_MASK 0x00000010L
+#define PB0_TX_LANE13_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_13__SHIFT 0x00000004
+#define PB0_TX_LANE13_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_13_MASK 0x00000080L
+#define PB0_TX_LANE13_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_13__SHIFT 0x00000007
+#define PB0_TX_LANE13_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_13_MASK 0x00000040L
+#define PB0_TX_LANE13_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_13__SHIFT 0x00000006
+#define PB0_TX_LANE13_SCI_STAT_OVRD_REG0__COEFFICIENT_13_MASK 0x0000fc00L
+#define PB0_TX_LANE13_SCI_STAT_OVRD_REG0__COEFFICIENT_13__SHIFT 0x0000000a
+#define PB0_TX_LANE13_SCI_STAT_OVRD_REG0__COEFFICIENTID_13_MASK 0x00000300L
+#define PB0_TX_LANE13_SCI_STAT_OVRD_REG0__COEFFICIENTID_13__SHIFT 0x00000008
+#define PB0_TX_LANE13_SCI_STAT_OVRD_REG0__DEEMPH_13_MASK 0x00000080L
+#define PB0_TX_LANE13_SCI_STAT_OVRD_REG0__DEEMPH_13__SHIFT 0x00000007
+#define PB0_TX_LANE13_SCI_STAT_OVRD_REG0__INCOHERENTCK_13_MASK 0x00000008L
+#define PB0_TX_LANE13_SCI_STAT_OVRD_REG0__INCOHERENTCK_13__SHIFT 0x00000003
+#define PB0_TX_LANE13_SCI_STAT_OVRD_REG0__TXMARG_13_MASK 0x00000070L
+#define PB0_TX_LANE13_SCI_STAT_OVRD_REG0__TXMARG_13__SHIFT 0x00000004
+#define PB0_TX_LANE13_SCI_STAT_OVRD_REG0__TXPWR_13_MASK 0x00000007L
+#define PB0_TX_LANE13_SCI_STAT_OVRD_REG0__TXPWR_13__SHIFT 0x00000000
+#define PB0_TX_LANE14_CTRL_REG0__TX_CFG_DISPCLK_MODE_14_MASK 0x00000001L
+#define PB0_TX_LANE14_CTRL_REG0__TX_CFG_DISPCLK_MODE_14__SHIFT 0x00000000
+#define PB0_TX_LANE14_CTRL_REG0__TX_CFG_INV_DATA_14_MASK 0x00000002L
+#define PB0_TX_LANE14_CTRL_REG0__TX_CFG_INV_DATA_14__SHIFT 0x00000001
+#define PB0_TX_LANE14_CTRL_REG0__TX_CFG_SWING_BOOST_EN_14_MASK 0x00000004L
+#define PB0_TX_LANE14_CTRL_REG0__TX_CFG_SWING_BOOST_EN_14__SHIFT 0x00000002
+#define PB0_TX_LANE14_CTRL_REG0__TX_DBG_PRBS_EN_14_MASK 0x00000008L
+#define PB0_TX_LANE14_CTRL_REG0__TX_DBG_PRBS_EN_14__SHIFT 0x00000003
+#define PB0_TX_LANE14_OVRD_REG0__TX_DCLK_EN_OVRD_EN_14_MASK 0x00000002L
+#define PB0_TX_LANE14_OVRD_REG0__TX_DCLK_EN_OVRD_EN_14__SHIFT 0x00000001
+#define PB0_TX_LANE14_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_14_MASK 0x00000001L
+#define PB0_TX_LANE14_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_14__SHIFT 0x00000000
+#define PB0_TX_LANE14_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_14_MASK 0x00000008L
+#define PB0_TX_LANE14_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_14__SHIFT 0x00000003
+#define PB0_TX_LANE14_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_14_MASK 0x00000004L
+#define PB0_TX_LANE14_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_14__SHIFT 0x00000002
+#define PB0_TX_LANE14_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_14_MASK 0x00000020L
+#define PB0_TX_LANE14_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_14__SHIFT 0x00000005
+#define PB0_TX_LANE14_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_14_MASK 0x00000010L
+#define PB0_TX_LANE14_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_14__SHIFT 0x00000004
+#define PB0_TX_LANE14_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_14_MASK 0x00000080L
+#define PB0_TX_LANE14_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_14__SHIFT 0x00000007
+#define PB0_TX_LANE14_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_14_MASK 0x00000040L
+#define PB0_TX_LANE14_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_14__SHIFT 0x00000006
+#define PB0_TX_LANE14_SCI_STAT_OVRD_REG0__COEFFICIENT_14_MASK 0x0000fc00L
+#define PB0_TX_LANE14_SCI_STAT_OVRD_REG0__COEFFICIENT_14__SHIFT 0x0000000a
+#define PB0_TX_LANE14_SCI_STAT_OVRD_REG0__COEFFICIENTID_14_MASK 0x00000300L
+#define PB0_TX_LANE14_SCI_STAT_OVRD_REG0__COEFFICIENTID_14__SHIFT 0x00000008
+#define PB0_TX_LANE14_SCI_STAT_OVRD_REG0__DEEMPH_14_MASK 0x00000080L
+#define PB0_TX_LANE14_SCI_STAT_OVRD_REG0__DEEMPH_14__SHIFT 0x00000007
+#define PB0_TX_LANE14_SCI_STAT_OVRD_REG0__INCOHERENTCK_14_MASK 0x00000008L
+#define PB0_TX_LANE14_SCI_STAT_OVRD_REG0__INCOHERENTCK_14__SHIFT 0x00000003
+#define PB0_TX_LANE14_SCI_STAT_OVRD_REG0__TXMARG_14_MASK 0x00000070L
+#define PB0_TX_LANE14_SCI_STAT_OVRD_REG0__TXMARG_14__SHIFT 0x00000004
+#define PB0_TX_LANE14_SCI_STAT_OVRD_REG0__TXPWR_14_MASK 0x00000007L
+#define PB0_TX_LANE14_SCI_STAT_OVRD_REG0__TXPWR_14__SHIFT 0x00000000
+#define PB0_TX_LANE15_CTRL_REG0__TX_CFG_DISPCLK_MODE_15_MASK 0x00000001L
+#define PB0_TX_LANE15_CTRL_REG0__TX_CFG_DISPCLK_MODE_15__SHIFT 0x00000000
+#define PB0_TX_LANE15_CTRL_REG0__TX_CFG_INV_DATA_15_MASK 0x00000002L
+#define PB0_TX_LANE15_CTRL_REG0__TX_CFG_INV_DATA_15__SHIFT 0x00000001
+#define PB0_TX_LANE15_CTRL_REG0__TX_CFG_SWING_BOOST_EN_15_MASK 0x00000004L
+#define PB0_TX_LANE15_CTRL_REG0__TX_CFG_SWING_BOOST_EN_15__SHIFT 0x00000002
+#define PB0_TX_LANE15_CTRL_REG0__TX_DBG_PRBS_EN_15_MASK 0x00000008L
+#define PB0_TX_LANE15_CTRL_REG0__TX_DBG_PRBS_EN_15__SHIFT 0x00000003
+#define PB0_TX_LANE15_OVRD_REG0__TX_DCLK_EN_OVRD_EN_15_MASK 0x00000002L
+#define PB0_TX_LANE15_OVRD_REG0__TX_DCLK_EN_OVRD_EN_15__SHIFT 0x00000001
+#define PB0_TX_LANE15_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_15_MASK 0x00000001L
+#define PB0_TX_LANE15_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_15__SHIFT 0x00000000
+#define PB0_TX_LANE15_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_15_MASK 0x00000008L
+#define PB0_TX_LANE15_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_15__SHIFT 0x00000003
+#define PB0_TX_LANE15_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_15_MASK 0x00000004L
+#define PB0_TX_LANE15_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_15__SHIFT 0x00000002
+#define PB0_TX_LANE15_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_15_MASK 0x00000020L
+#define PB0_TX_LANE15_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_15__SHIFT 0x00000005
+#define PB0_TX_LANE15_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_15_MASK 0x00000010L
+#define PB0_TX_LANE15_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_15__SHIFT 0x00000004
+#define PB0_TX_LANE15_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_15_MASK 0x00000080L
+#define PB0_TX_LANE15_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_15__SHIFT 0x00000007
+#define PB0_TX_LANE15_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_15_MASK 0x00000040L
+#define PB0_TX_LANE15_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_15__SHIFT 0x00000006
+#define PB0_TX_LANE15_SCI_STAT_OVRD_REG0__COEFFICIENT_15_MASK 0x0000fc00L
+#define PB0_TX_LANE15_SCI_STAT_OVRD_REG0__COEFFICIENT_15__SHIFT 0x0000000a
+#define PB0_TX_LANE15_SCI_STAT_OVRD_REG0__COEFFICIENTID_15_MASK 0x00000300L
+#define PB0_TX_LANE15_SCI_STAT_OVRD_REG0__COEFFICIENTID_15__SHIFT 0x00000008
+#define PB0_TX_LANE15_SCI_STAT_OVRD_REG0__DEEMPH_15_MASK 0x00000080L
+#define PB0_TX_LANE15_SCI_STAT_OVRD_REG0__DEEMPH_15__SHIFT 0x00000007
+#define PB0_TX_LANE15_SCI_STAT_OVRD_REG0__INCOHERENTCK_15_MASK 0x00000008L
+#define PB0_TX_LANE15_SCI_STAT_OVRD_REG0__INCOHERENTCK_15__SHIFT 0x00000003
+#define PB0_TX_LANE15_SCI_STAT_OVRD_REG0__TXMARG_15_MASK 0x00000070L
+#define PB0_TX_LANE15_SCI_STAT_OVRD_REG0__TXMARG_15__SHIFT 0x00000004
+#define PB0_TX_LANE15_SCI_STAT_OVRD_REG0__TXPWR_15_MASK 0x00000007L
+#define PB0_TX_LANE15_SCI_STAT_OVRD_REG0__TXPWR_15__SHIFT 0x00000000
+#define PB0_TX_LANE1_CTRL_REG0__TX_CFG_DISPCLK_MODE_1_MASK 0x00000001L
+#define PB0_TX_LANE1_CTRL_REG0__TX_CFG_DISPCLK_MODE_1__SHIFT 0x00000000
+#define PB0_TX_LANE1_CTRL_REG0__TX_CFG_INV_DATA_1_MASK 0x00000002L
+#define PB0_TX_LANE1_CTRL_REG0__TX_CFG_INV_DATA_1__SHIFT 0x00000001
+#define PB0_TX_LANE1_CTRL_REG0__TX_CFG_SWING_BOOST_EN_1_MASK 0x00000004L
+#define PB0_TX_LANE1_CTRL_REG0__TX_CFG_SWING_BOOST_EN_1__SHIFT 0x00000002
+#define PB0_TX_LANE1_CTRL_REG0__TX_DBG_PRBS_EN_1_MASK 0x00000008L
+#define PB0_TX_LANE1_CTRL_REG0__TX_DBG_PRBS_EN_1__SHIFT 0x00000003
+#define PB0_TX_LANE1_OVRD_REG0__TX_DCLK_EN_OVRD_EN_1_MASK 0x00000002L
+#define PB0_TX_LANE1_OVRD_REG0__TX_DCLK_EN_OVRD_EN_1__SHIFT 0x00000001
+#define PB0_TX_LANE1_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_1_MASK 0x00000001L
+#define PB0_TX_LANE1_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_1__SHIFT 0x00000000
+#define PB0_TX_LANE1_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_1_MASK 0x00000008L
+#define PB0_TX_LANE1_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_1__SHIFT 0x00000003
+#define PB0_TX_LANE1_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_1_MASK 0x00000004L
+#define PB0_TX_LANE1_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_1__SHIFT 0x00000002
+#define PB0_TX_LANE1_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_1_MASK 0x00000020L
+#define PB0_TX_LANE1_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_1__SHIFT 0x00000005
+#define PB0_TX_LANE1_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_1_MASK 0x00000010L
+#define PB0_TX_LANE1_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_1__SHIFT 0x00000004
+#define PB0_TX_LANE1_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_1_MASK 0x00000080L
+#define PB0_TX_LANE1_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_1__SHIFT 0x00000007
+#define PB0_TX_LANE1_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_1_MASK 0x00000040L
+#define PB0_TX_LANE1_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_1__SHIFT 0x00000006
+#define PB0_TX_LANE1_SCI_STAT_OVRD_REG0__COEFFICIENT_1_MASK 0x0000fc00L
+#define PB0_TX_LANE1_SCI_STAT_OVRD_REG0__COEFFICIENT_1__SHIFT 0x0000000a
+#define PB0_TX_LANE1_SCI_STAT_OVRD_REG0__COEFFICIENTID_1_MASK 0x00000300L
+#define PB0_TX_LANE1_SCI_STAT_OVRD_REG0__COEFFICIENTID_1__SHIFT 0x00000008
+#define PB0_TX_LANE1_SCI_STAT_OVRD_REG0__DEEMPH_1_MASK 0x00000080L
+#define PB0_TX_LANE1_SCI_STAT_OVRD_REG0__DEEMPH_1__SHIFT 0x00000007
+#define PB0_TX_LANE1_SCI_STAT_OVRD_REG0__INCOHERENTCK_1_MASK 0x00000008L
+#define PB0_TX_LANE1_SCI_STAT_OVRD_REG0__INCOHERENTCK_1__SHIFT 0x00000003
+#define PB0_TX_LANE1_SCI_STAT_OVRD_REG0__TXMARG_1_MASK 0x00000070L
+#define PB0_TX_LANE1_SCI_STAT_OVRD_REG0__TXMARG_1__SHIFT 0x00000004
+#define PB0_TX_LANE1_SCI_STAT_OVRD_REG0__TXPWR_1_MASK 0x00000007L
+#define PB0_TX_LANE1_SCI_STAT_OVRD_REG0__TXPWR_1__SHIFT 0x00000000
+#define PB0_TX_LANE2_CTRL_REG0__TX_CFG_DISPCLK_MODE_2_MASK 0x00000001L
+#define PB0_TX_LANE2_CTRL_REG0__TX_CFG_DISPCLK_MODE_2__SHIFT 0x00000000
+#define PB0_TX_LANE2_CTRL_REG0__TX_CFG_INV_DATA_2_MASK 0x00000002L
+#define PB0_TX_LANE2_CTRL_REG0__TX_CFG_INV_DATA_2__SHIFT 0x00000001
+#define PB0_TX_LANE2_CTRL_REG0__TX_CFG_SWING_BOOST_EN_2_MASK 0x00000004L
+#define PB0_TX_LANE2_CTRL_REG0__TX_CFG_SWING_BOOST_EN_2__SHIFT 0x00000002
+#define PB0_TX_LANE2_CTRL_REG0__TX_DBG_PRBS_EN_2_MASK 0x00000008L
+#define PB0_TX_LANE2_CTRL_REG0__TX_DBG_PRBS_EN_2__SHIFT 0x00000003
+#define PB0_TX_LANE2_OVRD_REG0__TX_DCLK_EN_OVRD_EN_2_MASK 0x00000002L
+#define PB0_TX_LANE2_OVRD_REG0__TX_DCLK_EN_OVRD_EN_2__SHIFT 0x00000001
+#define PB0_TX_LANE2_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_2_MASK 0x00000001L
+#define PB0_TX_LANE2_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_2__SHIFT 0x00000000
+#define PB0_TX_LANE2_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_2_MASK 0x00000008L
+#define PB0_TX_LANE2_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_2__SHIFT 0x00000003
+#define PB0_TX_LANE2_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_2_MASK 0x00000004L
+#define PB0_TX_LANE2_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_2__SHIFT 0x00000002
+#define PB0_TX_LANE2_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_2_MASK 0x00000020L
+#define PB0_TX_LANE2_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_2__SHIFT 0x00000005
+#define PB0_TX_LANE2_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_2_MASK 0x00000010L
+#define PB0_TX_LANE2_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_2__SHIFT 0x00000004
+#define PB0_TX_LANE2_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_2_MASK 0x00000080L
+#define PB0_TX_LANE2_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_2__SHIFT 0x00000007
+#define PB0_TX_LANE2_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_2_MASK 0x00000040L
+#define PB0_TX_LANE2_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_2__SHIFT 0x00000006
+#define PB0_TX_LANE2_SCI_STAT_OVRD_REG0__COEFFICIENT_2_MASK 0x0000fc00L
+#define PB0_TX_LANE2_SCI_STAT_OVRD_REG0__COEFFICIENT_2__SHIFT 0x0000000a
+#define PB0_TX_LANE2_SCI_STAT_OVRD_REG0__COEFFICIENTID_2_MASK 0x00000300L
+#define PB0_TX_LANE2_SCI_STAT_OVRD_REG0__COEFFICIENTID_2__SHIFT 0x00000008
+#define PB0_TX_LANE2_SCI_STAT_OVRD_REG0__DEEMPH_2_MASK 0x00000080L
+#define PB0_TX_LANE2_SCI_STAT_OVRD_REG0__DEEMPH_2__SHIFT 0x00000007
+#define PB0_TX_LANE2_SCI_STAT_OVRD_REG0__INCOHERENTCK_2_MASK 0x00000008L
+#define PB0_TX_LANE2_SCI_STAT_OVRD_REG0__INCOHERENTCK_2__SHIFT 0x00000003
+#define PB0_TX_LANE2_SCI_STAT_OVRD_REG0__TXMARG_2_MASK 0x00000070L
+#define PB0_TX_LANE2_SCI_STAT_OVRD_REG0__TXMARG_2__SHIFT 0x00000004
+#define PB0_TX_LANE2_SCI_STAT_OVRD_REG0__TXPWR_2_MASK 0x00000007L
+#define PB0_TX_LANE2_SCI_STAT_OVRD_REG0__TXPWR_2__SHIFT 0x00000000
+#define PB0_TX_LANE3_CTRL_REG0__TX_CFG_DISPCLK_MODE_3_MASK 0x00000001L
+#define PB0_TX_LANE3_CTRL_REG0__TX_CFG_DISPCLK_MODE_3__SHIFT 0x00000000
+#define PB0_TX_LANE3_CTRL_REG0__TX_CFG_INV_DATA_3_MASK 0x00000002L
+#define PB0_TX_LANE3_CTRL_REG0__TX_CFG_INV_DATA_3__SHIFT 0x00000001
+#define PB0_TX_LANE3_CTRL_REG0__TX_CFG_SWING_BOOST_EN_3_MASK 0x00000004L
+#define PB0_TX_LANE3_CTRL_REG0__TX_CFG_SWING_BOOST_EN_3__SHIFT 0x00000002
+#define PB0_TX_LANE3_CTRL_REG0__TX_DBG_PRBS_EN_3_MASK 0x00000008L
+#define PB0_TX_LANE3_CTRL_REG0__TX_DBG_PRBS_EN_3__SHIFT 0x00000003
+#define PB0_TX_LANE3_OVRD_REG0__TX_DCLK_EN_OVRD_EN_3_MASK 0x00000002L
+#define PB0_TX_LANE3_OVRD_REG0__TX_DCLK_EN_OVRD_EN_3__SHIFT 0x00000001
+#define PB0_TX_LANE3_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_3_MASK 0x00000001L
+#define PB0_TX_LANE3_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_3__SHIFT 0x00000000
+#define PB0_TX_LANE3_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_3_MASK 0x00000008L
+#define PB0_TX_LANE3_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_3__SHIFT 0x00000003
+#define PB0_TX_LANE3_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_3_MASK 0x00000004L
+#define PB0_TX_LANE3_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_3__SHIFT 0x00000002
+#define PB0_TX_LANE3_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_3_MASK 0x00000020L
+#define PB0_TX_LANE3_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_3__SHIFT 0x00000005
+#define PB0_TX_LANE3_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_3_MASK 0x00000010L
+#define PB0_TX_LANE3_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_3__SHIFT 0x00000004
+#define PB0_TX_LANE3_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_3_MASK 0x00000080L
+#define PB0_TX_LANE3_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_3__SHIFT 0x00000007
+#define PB0_TX_LANE3_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_3_MASK 0x00000040L
+#define PB0_TX_LANE3_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_3__SHIFT 0x00000006
+#define PB0_TX_LANE3_SCI_STAT_OVRD_REG0__COEFFICIENT_3_MASK 0x0000fc00L
+#define PB0_TX_LANE3_SCI_STAT_OVRD_REG0__COEFFICIENT_3__SHIFT 0x0000000a
+#define PB0_TX_LANE3_SCI_STAT_OVRD_REG0__COEFFICIENTID_3_MASK 0x00000300L
+#define PB0_TX_LANE3_SCI_STAT_OVRD_REG0__COEFFICIENTID_3__SHIFT 0x00000008
+#define PB0_TX_LANE3_SCI_STAT_OVRD_REG0__DEEMPH_3_MASK 0x00000080L
+#define PB0_TX_LANE3_SCI_STAT_OVRD_REG0__DEEMPH_3__SHIFT 0x00000007
+#define PB0_TX_LANE3_SCI_STAT_OVRD_REG0__INCOHERENTCK_3_MASK 0x00000008L
+#define PB0_TX_LANE3_SCI_STAT_OVRD_REG0__INCOHERENTCK_3__SHIFT 0x00000003
+#define PB0_TX_LANE3_SCI_STAT_OVRD_REG0__TXMARG_3_MASK 0x00000070L
+#define PB0_TX_LANE3_SCI_STAT_OVRD_REG0__TXMARG_3__SHIFT 0x00000004
+#define PB0_TX_LANE3_SCI_STAT_OVRD_REG0__TXPWR_3_MASK 0x00000007L
+#define PB0_TX_LANE3_SCI_STAT_OVRD_REG0__TXPWR_3__SHIFT 0x00000000
+#define PB0_TX_LANE4_CTRL_REG0__TX_CFG_DISPCLK_MODE_4_MASK 0x00000001L
+#define PB0_TX_LANE4_CTRL_REG0__TX_CFG_DISPCLK_MODE_4__SHIFT 0x00000000
+#define PB0_TX_LANE4_CTRL_REG0__TX_CFG_INV_DATA_4_MASK 0x00000002L
+#define PB0_TX_LANE4_CTRL_REG0__TX_CFG_INV_DATA_4__SHIFT 0x00000001
+#define PB0_TX_LANE4_CTRL_REG0__TX_CFG_SWING_BOOST_EN_4_MASK 0x00000004L
+#define PB0_TX_LANE4_CTRL_REG0__TX_CFG_SWING_BOOST_EN_4__SHIFT 0x00000002
+#define PB0_TX_LANE4_CTRL_REG0__TX_DBG_PRBS_EN_4_MASK 0x00000008L
+#define PB0_TX_LANE4_CTRL_REG0__TX_DBG_PRBS_EN_4__SHIFT 0x00000003
+#define PB0_TX_LANE4_OVRD_REG0__TX_DCLK_EN_OVRD_EN_4_MASK 0x00000002L
+#define PB0_TX_LANE4_OVRD_REG0__TX_DCLK_EN_OVRD_EN_4__SHIFT 0x00000001
+#define PB0_TX_LANE4_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_4_MASK 0x00000001L
+#define PB0_TX_LANE4_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_4__SHIFT 0x00000000
+#define PB0_TX_LANE4_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_4_MASK 0x00000008L
+#define PB0_TX_LANE4_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_4__SHIFT 0x00000003
+#define PB0_TX_LANE4_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_4_MASK 0x00000004L
+#define PB0_TX_LANE4_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_4__SHIFT 0x00000002
+#define PB0_TX_LANE4_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_4_MASK 0x00000020L
+#define PB0_TX_LANE4_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_4__SHIFT 0x00000005
+#define PB0_TX_LANE4_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_4_MASK 0x00000010L
+#define PB0_TX_LANE4_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_4__SHIFT 0x00000004
+#define PB0_TX_LANE4_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_4_MASK 0x00000080L
+#define PB0_TX_LANE4_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_4__SHIFT 0x00000007
+#define PB0_TX_LANE4_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_4_MASK 0x00000040L
+#define PB0_TX_LANE4_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_4__SHIFT 0x00000006
+#define PB0_TX_LANE4_SCI_STAT_OVRD_REG0__COEFFICIENT_4_MASK 0x0000fc00L
+#define PB0_TX_LANE4_SCI_STAT_OVRD_REG0__COEFFICIENT_4__SHIFT 0x0000000a
+#define PB0_TX_LANE4_SCI_STAT_OVRD_REG0__COEFFICIENTID_4_MASK 0x00000300L
+#define PB0_TX_LANE4_SCI_STAT_OVRD_REG0__COEFFICIENTID_4__SHIFT 0x00000008
+#define PB0_TX_LANE4_SCI_STAT_OVRD_REG0__DEEMPH_4_MASK 0x00000080L
+#define PB0_TX_LANE4_SCI_STAT_OVRD_REG0__DEEMPH_4__SHIFT 0x00000007
+#define PB0_TX_LANE4_SCI_STAT_OVRD_REG0__INCOHERENTCK_4_MASK 0x00000008L
+#define PB0_TX_LANE4_SCI_STAT_OVRD_REG0__INCOHERENTCK_4__SHIFT 0x00000003
+#define PB0_TX_LANE4_SCI_STAT_OVRD_REG0__TXMARG_4_MASK 0x00000070L
+#define PB0_TX_LANE4_SCI_STAT_OVRD_REG0__TXMARG_4__SHIFT 0x00000004
+#define PB0_TX_LANE4_SCI_STAT_OVRD_REG0__TXPWR_4_MASK 0x00000007L
+#define PB0_TX_LANE4_SCI_STAT_OVRD_REG0__TXPWR_4__SHIFT 0x00000000
+#define PB0_TX_LANE5_CTRL_REG0__TX_CFG_DISPCLK_MODE_5_MASK 0x00000001L
+#define PB0_TX_LANE5_CTRL_REG0__TX_CFG_DISPCLK_MODE_5__SHIFT 0x00000000
+#define PB0_TX_LANE5_CTRL_REG0__TX_CFG_INV_DATA_5_MASK 0x00000002L
+#define PB0_TX_LANE5_CTRL_REG0__TX_CFG_INV_DATA_5__SHIFT 0x00000001
+#define PB0_TX_LANE5_CTRL_REG0__TX_CFG_SWING_BOOST_EN_5_MASK 0x00000004L
+#define PB0_TX_LANE5_CTRL_REG0__TX_CFG_SWING_BOOST_EN_5__SHIFT 0x00000002
+#define PB0_TX_LANE5_CTRL_REG0__TX_DBG_PRBS_EN_5_MASK 0x00000008L
+#define PB0_TX_LANE5_CTRL_REG0__TX_DBG_PRBS_EN_5__SHIFT 0x00000003
+#define PB0_TX_LANE5_OVRD_REG0__TX_DCLK_EN_OVRD_EN_5_MASK 0x00000002L
+#define PB0_TX_LANE5_OVRD_REG0__TX_DCLK_EN_OVRD_EN_5__SHIFT 0x00000001
+#define PB0_TX_LANE5_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_5_MASK 0x00000001L
+#define PB0_TX_LANE5_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_5__SHIFT 0x00000000
+#define PB0_TX_LANE5_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_5_MASK 0x00000008L
+#define PB0_TX_LANE5_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_5__SHIFT 0x00000003
+#define PB0_TX_LANE5_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_5_MASK 0x00000004L
+#define PB0_TX_LANE5_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_5__SHIFT 0x00000002
+#define PB0_TX_LANE5_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_5_MASK 0x00000020L
+#define PB0_TX_LANE5_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_5__SHIFT 0x00000005
+#define PB0_TX_LANE5_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_5_MASK 0x00000010L
+#define PB0_TX_LANE5_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_5__SHIFT 0x00000004
+#define PB0_TX_LANE5_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_5_MASK 0x00000080L
+#define PB0_TX_LANE5_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_5__SHIFT 0x00000007
+#define PB0_TX_LANE5_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_5_MASK 0x00000040L
+#define PB0_TX_LANE5_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_5__SHIFT 0x00000006
+#define PB0_TX_LANE5_SCI_STAT_OVRD_REG0__COEFFICIENT_5_MASK 0x0000fc00L
+#define PB0_TX_LANE5_SCI_STAT_OVRD_REG0__COEFFICIENT_5__SHIFT 0x0000000a
+#define PB0_TX_LANE5_SCI_STAT_OVRD_REG0__COEFFICIENTID_5_MASK 0x00000300L
+#define PB0_TX_LANE5_SCI_STAT_OVRD_REG0__COEFFICIENTID_5__SHIFT 0x00000008
+#define PB0_TX_LANE5_SCI_STAT_OVRD_REG0__DEEMPH_5_MASK 0x00000080L
+#define PB0_TX_LANE5_SCI_STAT_OVRD_REG0__DEEMPH_5__SHIFT 0x00000007
+#define PB0_TX_LANE5_SCI_STAT_OVRD_REG0__INCOHERENTCK_5_MASK 0x00000008L
+#define PB0_TX_LANE5_SCI_STAT_OVRD_REG0__INCOHERENTCK_5__SHIFT 0x00000003
+#define PB0_TX_LANE5_SCI_STAT_OVRD_REG0__TXMARG_5_MASK 0x00000070L
+#define PB0_TX_LANE5_SCI_STAT_OVRD_REG0__TXMARG_5__SHIFT 0x00000004
+#define PB0_TX_LANE5_SCI_STAT_OVRD_REG0__TXPWR_5_MASK 0x00000007L
+#define PB0_TX_LANE5_SCI_STAT_OVRD_REG0__TXPWR_5__SHIFT 0x00000000
+#define PB0_TX_LANE6_CTRL_REG0__TX_CFG_DISPCLK_MODE_6_MASK 0x00000001L
+#define PB0_TX_LANE6_CTRL_REG0__TX_CFG_DISPCLK_MODE_6__SHIFT 0x00000000
+#define PB0_TX_LANE6_CTRL_REG0__TX_CFG_INV_DATA_6_MASK 0x00000002L
+#define PB0_TX_LANE6_CTRL_REG0__TX_CFG_INV_DATA_6__SHIFT 0x00000001
+#define PB0_TX_LANE6_CTRL_REG0__TX_CFG_SWING_BOOST_EN_6_MASK 0x00000004L
+#define PB0_TX_LANE6_CTRL_REG0__TX_CFG_SWING_BOOST_EN_6__SHIFT 0x00000002
+#define PB0_TX_LANE6_CTRL_REG0__TX_DBG_PRBS_EN_6_MASK 0x00000008L
+#define PB0_TX_LANE6_CTRL_REG0__TX_DBG_PRBS_EN_6__SHIFT 0x00000003
+#define PB0_TX_LANE6_OVRD_REG0__TX_DCLK_EN_OVRD_EN_6_MASK 0x00000002L
+#define PB0_TX_LANE6_OVRD_REG0__TX_DCLK_EN_OVRD_EN_6__SHIFT 0x00000001
+#define PB0_TX_LANE6_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_6_MASK 0x00000001L
+#define PB0_TX_LANE6_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_6__SHIFT 0x00000000
+#define PB0_TX_LANE6_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_6_MASK 0x00000008L
+#define PB0_TX_LANE6_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_6__SHIFT 0x00000003
+#define PB0_TX_LANE6_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_6_MASK 0x00000004L
+#define PB0_TX_LANE6_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_6__SHIFT 0x00000002
+#define PB0_TX_LANE6_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_6_MASK 0x00000020L
+#define PB0_TX_LANE6_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_6__SHIFT 0x00000005
+#define PB0_TX_LANE6_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_6_MASK 0x00000010L
+#define PB0_TX_LANE6_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_6__SHIFT 0x00000004
+#define PB0_TX_LANE6_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_6_MASK 0x00000080L
+#define PB0_TX_LANE6_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_6__SHIFT 0x00000007
+#define PB0_TX_LANE6_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_6_MASK 0x00000040L
+#define PB0_TX_LANE6_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_6__SHIFT 0x00000006
+#define PB0_TX_LANE6_SCI_STAT_OVRD_REG0__COEFFICIENT_6_MASK 0x0000fc00L
+#define PB0_TX_LANE6_SCI_STAT_OVRD_REG0__COEFFICIENT_6__SHIFT 0x0000000a
+#define PB0_TX_LANE6_SCI_STAT_OVRD_REG0__COEFFICIENTID_6_MASK 0x00000300L
+#define PB0_TX_LANE6_SCI_STAT_OVRD_REG0__COEFFICIENTID_6__SHIFT 0x00000008
+#define PB0_TX_LANE6_SCI_STAT_OVRD_REG0__DEEMPH_6_MASK 0x00000080L
+#define PB0_TX_LANE6_SCI_STAT_OVRD_REG0__DEEMPH_6__SHIFT 0x00000007
+#define PB0_TX_LANE6_SCI_STAT_OVRD_REG0__INCOHERENTCK_6_MASK 0x00000008L
+#define PB0_TX_LANE6_SCI_STAT_OVRD_REG0__INCOHERENTCK_6__SHIFT 0x00000003
+#define PB0_TX_LANE6_SCI_STAT_OVRD_REG0__TXMARG_6_MASK 0x00000070L
+#define PB0_TX_LANE6_SCI_STAT_OVRD_REG0__TXMARG_6__SHIFT 0x00000004
+#define PB0_TX_LANE6_SCI_STAT_OVRD_REG0__TXPWR_6_MASK 0x00000007L
+#define PB0_TX_LANE6_SCI_STAT_OVRD_REG0__TXPWR_6__SHIFT 0x00000000
+#define PB0_TX_LANE7_CTRL_REG0__TX_CFG_DISPCLK_MODE_7_MASK 0x00000001L
+#define PB0_TX_LANE7_CTRL_REG0__TX_CFG_DISPCLK_MODE_7__SHIFT 0x00000000
+#define PB0_TX_LANE7_CTRL_REG0__TX_CFG_INV_DATA_7_MASK 0x00000002L
+#define PB0_TX_LANE7_CTRL_REG0__TX_CFG_INV_DATA_7__SHIFT 0x00000001
+#define PB0_TX_LANE7_CTRL_REG0__TX_CFG_SWING_BOOST_EN_7_MASK 0x00000004L
+#define PB0_TX_LANE7_CTRL_REG0__TX_CFG_SWING_BOOST_EN_7__SHIFT 0x00000002
+#define PB0_TX_LANE7_CTRL_REG0__TX_DBG_PRBS_EN_7_MASK 0x00000008L
+#define PB0_TX_LANE7_CTRL_REG0__TX_DBG_PRBS_EN_7__SHIFT 0x00000003
+#define PB0_TX_LANE7_OVRD_REG0__TX_DCLK_EN_OVRD_EN_7_MASK 0x00000002L
+#define PB0_TX_LANE7_OVRD_REG0__TX_DCLK_EN_OVRD_EN_7__SHIFT 0x00000001
+#define PB0_TX_LANE7_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_7_MASK 0x00000001L
+#define PB0_TX_LANE7_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_7__SHIFT 0x00000000
+#define PB0_TX_LANE7_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_7_MASK 0x00000008L
+#define PB0_TX_LANE7_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_7__SHIFT 0x00000003
+#define PB0_TX_LANE7_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_7_MASK 0x00000004L
+#define PB0_TX_LANE7_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_7__SHIFT 0x00000002
+#define PB0_TX_LANE7_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_7_MASK 0x00000020L
+#define PB0_TX_LANE7_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_7__SHIFT 0x00000005
+#define PB0_TX_LANE7_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_7_MASK 0x00000010L
+#define PB0_TX_LANE7_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_7__SHIFT 0x00000004
+#define PB0_TX_LANE7_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_7_MASK 0x00000080L
+#define PB0_TX_LANE7_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_7__SHIFT 0x00000007
+#define PB0_TX_LANE7_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_7_MASK 0x00000040L
+#define PB0_TX_LANE7_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_7__SHIFT 0x00000006
+#define PB0_TX_LANE7_SCI_STAT_OVRD_REG0__COEFFICIENT_7_MASK 0x0000fc00L
+#define PB0_TX_LANE7_SCI_STAT_OVRD_REG0__COEFFICIENT_7__SHIFT 0x0000000a
+#define PB0_TX_LANE7_SCI_STAT_OVRD_REG0__COEFFICIENTID_7_MASK 0x00000300L
+#define PB0_TX_LANE7_SCI_STAT_OVRD_REG0__COEFFICIENTID_7__SHIFT 0x00000008
+#define PB0_TX_LANE7_SCI_STAT_OVRD_REG0__DEEMPH_7_MASK 0x00000080L
+#define PB0_TX_LANE7_SCI_STAT_OVRD_REG0__DEEMPH_7__SHIFT 0x00000007
+#define PB0_TX_LANE7_SCI_STAT_OVRD_REG0__INCOHERENTCK_7_MASK 0x00000008L
+#define PB0_TX_LANE7_SCI_STAT_OVRD_REG0__INCOHERENTCK_7__SHIFT 0x00000003
+#define PB0_TX_LANE7_SCI_STAT_OVRD_REG0__TXMARG_7_MASK 0x00000070L
+#define PB0_TX_LANE7_SCI_STAT_OVRD_REG0__TXMARG_7__SHIFT 0x00000004
+#define PB0_TX_LANE7_SCI_STAT_OVRD_REG0__TXPWR_7_MASK 0x00000007L
+#define PB0_TX_LANE7_SCI_STAT_OVRD_REG0__TXPWR_7__SHIFT 0x00000000
+#define PB0_TX_LANE8_CTRL_REG0__TX_CFG_DISPCLK_MODE_8_MASK 0x00000001L
+#define PB0_TX_LANE8_CTRL_REG0__TX_CFG_DISPCLK_MODE_8__SHIFT 0x00000000
+#define PB0_TX_LANE8_CTRL_REG0__TX_CFG_INV_DATA_8_MASK 0x00000002L
+#define PB0_TX_LANE8_CTRL_REG0__TX_CFG_INV_DATA_8__SHIFT 0x00000001
+#define PB0_TX_LANE8_CTRL_REG0__TX_CFG_SWING_BOOST_EN_8_MASK 0x00000004L
+#define PB0_TX_LANE8_CTRL_REG0__TX_CFG_SWING_BOOST_EN_8__SHIFT 0x00000002
+#define PB0_TX_LANE8_CTRL_REG0__TX_DBG_PRBS_EN_8_MASK 0x00000008L
+#define PB0_TX_LANE8_CTRL_REG0__TX_DBG_PRBS_EN_8__SHIFT 0x00000003
+#define PB0_TX_LANE8_OVRD_REG0__TX_DCLK_EN_OVRD_EN_8_MASK 0x00000002L
+#define PB0_TX_LANE8_OVRD_REG0__TX_DCLK_EN_OVRD_EN_8__SHIFT 0x00000001
+#define PB0_TX_LANE8_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_8_MASK 0x00000001L
+#define PB0_TX_LANE8_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_8__SHIFT 0x00000000
+#define PB0_TX_LANE8_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_8_MASK 0x00000008L
+#define PB0_TX_LANE8_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_8__SHIFT 0x00000003
+#define PB0_TX_LANE8_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_8_MASK 0x00000004L
+#define PB0_TX_LANE8_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_8__SHIFT 0x00000002
+#define PB0_TX_LANE8_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_8_MASK 0x00000020L
+#define PB0_TX_LANE8_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_8__SHIFT 0x00000005
+#define PB0_TX_LANE8_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_8_MASK 0x00000010L
+#define PB0_TX_LANE8_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_8__SHIFT 0x00000004
+#define PB0_TX_LANE8_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_8_MASK 0x00000080L
+#define PB0_TX_LANE8_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_8__SHIFT 0x00000007
+#define PB0_TX_LANE8_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_8_MASK 0x00000040L
+#define PB0_TX_LANE8_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_8__SHIFT 0x00000006
+#define PB0_TX_LANE8_SCI_STAT_OVRD_REG0__COEFFICIENT_8_MASK 0x0000fc00L
+#define PB0_TX_LANE8_SCI_STAT_OVRD_REG0__COEFFICIENT_8__SHIFT 0x0000000a
+#define PB0_TX_LANE8_SCI_STAT_OVRD_REG0__COEFFICIENTID_8_MASK 0x00000300L
+#define PB0_TX_LANE8_SCI_STAT_OVRD_REG0__COEFFICIENTID_8__SHIFT 0x00000008
+#define PB0_TX_LANE8_SCI_STAT_OVRD_REG0__DEEMPH_8_MASK 0x00000080L
+#define PB0_TX_LANE8_SCI_STAT_OVRD_REG0__DEEMPH_8__SHIFT 0x00000007
+#define PB0_TX_LANE8_SCI_STAT_OVRD_REG0__INCOHERENTCK_8_MASK 0x00000008L
+#define PB0_TX_LANE8_SCI_STAT_OVRD_REG0__INCOHERENTCK_8__SHIFT 0x00000003
+#define PB0_TX_LANE8_SCI_STAT_OVRD_REG0__TXMARG_8_MASK 0x00000070L
+#define PB0_TX_LANE8_SCI_STAT_OVRD_REG0__TXMARG_8__SHIFT 0x00000004
+#define PB0_TX_LANE8_SCI_STAT_OVRD_REG0__TXPWR_8_MASK 0x00000007L
+#define PB0_TX_LANE8_SCI_STAT_OVRD_REG0__TXPWR_8__SHIFT 0x00000000
+#define PB0_TX_LANE9_CTRL_REG0__TX_CFG_DISPCLK_MODE_9_MASK 0x00000001L
+#define PB0_TX_LANE9_CTRL_REG0__TX_CFG_DISPCLK_MODE_9__SHIFT 0x00000000
+#define PB0_TX_LANE9_CTRL_REG0__TX_CFG_INV_DATA_9_MASK 0x00000002L
+#define PB0_TX_LANE9_CTRL_REG0__TX_CFG_INV_DATA_9__SHIFT 0x00000001
+#define PB0_TX_LANE9_CTRL_REG0__TX_CFG_SWING_BOOST_EN_9_MASK 0x00000004L
+#define PB0_TX_LANE9_CTRL_REG0__TX_CFG_SWING_BOOST_EN_9__SHIFT 0x00000002
+#define PB0_TX_LANE9_CTRL_REG0__TX_DBG_PRBS_EN_9_MASK 0x00000008L
+#define PB0_TX_LANE9_CTRL_REG0__TX_DBG_PRBS_EN_9__SHIFT 0x00000003
+#define PB0_TX_LANE9_OVRD_REG0__TX_DCLK_EN_OVRD_EN_9_MASK 0x00000002L
+#define PB0_TX_LANE9_OVRD_REG0__TX_DCLK_EN_OVRD_EN_9__SHIFT 0x00000001
+#define PB0_TX_LANE9_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_9_MASK 0x00000001L
+#define PB0_TX_LANE9_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_9__SHIFT 0x00000000
+#define PB0_TX_LANE9_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_9_MASK 0x00000008L
+#define PB0_TX_LANE9_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_9__SHIFT 0x00000003
+#define PB0_TX_LANE9_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_9_MASK 0x00000004L
+#define PB0_TX_LANE9_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_9__SHIFT 0x00000002
+#define PB0_TX_LANE9_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_9_MASK 0x00000020L
+#define PB0_TX_LANE9_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_9__SHIFT 0x00000005
+#define PB0_TX_LANE9_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_9_MASK 0x00000010L
+#define PB0_TX_LANE9_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_9__SHIFT 0x00000004
+#define PB0_TX_LANE9_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_9_MASK 0x00000080L
+#define PB0_TX_LANE9_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_9__SHIFT 0x00000007
+#define PB0_TX_LANE9_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_9_MASK 0x00000040L
+#define PB0_TX_LANE9_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_9__SHIFT 0x00000006
+#define PB0_TX_LANE9_SCI_STAT_OVRD_REG0__COEFFICIENT_9_MASK 0x0000fc00L
+#define PB0_TX_LANE9_SCI_STAT_OVRD_REG0__COEFFICIENT_9__SHIFT 0x0000000a
+#define PB0_TX_LANE9_SCI_STAT_OVRD_REG0__COEFFICIENTID_9_MASK 0x00000300L
+#define PB0_TX_LANE9_SCI_STAT_OVRD_REG0__COEFFICIENTID_9__SHIFT 0x00000008
+#define PB0_TX_LANE9_SCI_STAT_OVRD_REG0__DEEMPH_9_MASK 0x00000080L
+#define PB0_TX_LANE9_SCI_STAT_OVRD_REG0__DEEMPH_9__SHIFT 0x00000007
+#define PB0_TX_LANE9_SCI_STAT_OVRD_REG0__INCOHERENTCK_9_MASK 0x00000008L
+#define PB0_TX_LANE9_SCI_STAT_OVRD_REG0__INCOHERENTCK_9__SHIFT 0x00000003
+#define PB0_TX_LANE9_SCI_STAT_OVRD_REG0__TXMARG_9_MASK 0x00000070L
+#define PB0_TX_LANE9_SCI_STAT_OVRD_REG0__TXMARG_9__SHIFT 0x00000004
+#define PB0_TX_LANE9_SCI_STAT_OVRD_REG0__TXPWR_9_MASK 0x00000007L
+#define PB0_TX_LANE9_SCI_STAT_OVRD_REG0__TXPWR_9__SHIFT 0x00000000
+#define PB1_DFT_DEBUG_CTRL_REG0__DFT_PHY_DEBUG_EN_MASK 0x00000001L
+#define PB1_DFT_DEBUG_CTRL_REG0__DFT_PHY_DEBUG_EN__SHIFT 0x00000000
+#define PB1_DFT_DEBUG_CTRL_REG0__DFT_PHY_DEBUG_MODE_MASK 0x0000003eL
+#define PB1_DFT_DEBUG_CTRL_REG0__DFT_PHY_DEBUG_MODE__SHIFT 0x00000001
+#define PB1_DFT_JIT_INJ_REG0__DFT_CLK_PER_STEP_MASK 0x00000f00L
+#define PB1_DFT_JIT_INJ_REG0__DFT_CLK_PER_STEP__SHIFT 0x00000008
+#define PB1_DFT_JIT_INJ_REG0__DFT_DECR_SWP_EN_MASK 0x00800000L
+#define PB1_DFT_JIT_INJ_REG0__DFT_DECR_SWP_EN__SHIFT 0x00000017
+#define PB1_DFT_JIT_INJ_REG0__DFT_INCR_SWP_EN_MASK 0x00400000L
+#define PB1_DFT_JIT_INJ_REG0__DFT_INCR_SWP_EN__SHIFT 0x00000016
+#define PB1_DFT_JIT_INJ_REG0__DFT_NUM_STEPS_MASK 0x0000001fL
+#define PB1_DFT_JIT_INJ_REG0__DFT_NUM_STEPS__SHIFT 0x00000000
+#define PB1_DFT_JIT_INJ_REG0__DFT_RECOVERY_TIME_MASK 0xff000000L
+#define PB1_DFT_JIT_INJ_REG0__DFT_RECOVERY_TIME__SHIFT 0x00000018
+#define PB1_DFT_JIT_INJ_REG1__DFT_BLOCK_EN_MASK 0x00010000L
+#define PB1_DFT_JIT_INJ_REG1__DFT_BLOCK_EN__SHIFT 0x00000010
+#define PB1_DFT_JIT_INJ_REG1__DFT_BYPASS_EN_MASK 0x00000100L
+#define PB1_DFT_JIT_INJ_REG1__DFT_BYPASS_EN__SHIFT 0x00000008
+#define PB1_DFT_JIT_INJ_REG1__DFT_BYPASS_VALUE_MASK 0x000000ffL
+#define PB1_DFT_JIT_INJ_REG1__DFT_BYPASS_VALUE__SHIFT 0x00000000
+#define PB1_DFT_JIT_INJ_REG2__DFT_LANE_EN_MASK 0x0000ffffL
+#define PB1_DFT_JIT_INJ_REG2__DFT_LANE_EN__SHIFT 0x00000000
+#define PB1_GLB_CTRL_REG0__BACKUP_MASK 0x0000ffffL
+#define PB1_GLB_CTRL_REG0__BACKUP__SHIFT 0x00000000
+#define PB1_GLB_CTRL_REG0__CFG_IDLEDET_TH_MASK 0x00030000L
+#define PB1_GLB_CTRL_REG0__CFG_IDLEDET_TH__SHIFT 0x00000010
+#define PB1_GLB_CTRL_REG0__DBG_RX2TXBYP_SEL_MASK 0x00700000L
+#define PB1_GLB_CTRL_REG0__DBG_RX2TXBYP_SEL__SHIFT 0x00000014
+#define PB1_GLB_CTRL_REG0__DBG_RXFEBYP_EN_MASK 0x00800000L
+#define PB1_GLB_CTRL_REG0__DBG_RXFEBYP_EN__SHIFT 0x00000017
+#define PB1_GLB_CTRL_REG0__DBG_RXPRBS_CLR_MASK 0x01000000L
+#define PB1_GLB_CTRL_REG0__DBG_RXPRBS_CLR__SHIFT 0x00000018
+#define PB1_GLB_CTRL_REG0__DBG_RXTOGGLE_EN_MASK 0x02000000L
+#define PB1_GLB_CTRL_REG0__DBG_RXTOGGLE_EN__SHIFT 0x00000019
+#define PB1_GLB_CTRL_REG0__DBG_TX2RXLBACK_EN_MASK 0x04000000L
+#define PB1_GLB_CTRL_REG0__DBG_TX2RXLBACK_EN__SHIFT 0x0000001a
+#define PB1_GLB_CTRL_REG0__TXCFG_CMGOOD_RANGE_MASK 0xc0000000L
+#define PB1_GLB_CTRL_REG0__TXCFG_CMGOOD_RANGE__SHIFT 0x0000001e
+#define PB1_GLB_CTRL_REG1__PLL_CFG_DISPCLK_DIV_MASK 0x80000000L
+#define PB1_GLB_CTRL_REG1__PLL_CFG_DISPCLK_DIV__SHIFT 0x0000001f
+#define PB1_GLB_CTRL_REG1__RXDBG_CDR_FR_BYP_EN_MASK 0x00000001L
+#define PB1_GLB_CTRL_REG1__RXDBG_CDR_FR_BYP_EN__SHIFT 0x00000000
+#define PB1_GLB_CTRL_REG1__RXDBG_CDR_FR_BYP_VAL_MASK 0x0000007eL
+#define PB1_GLB_CTRL_REG1__RXDBG_CDR_FR_BYP_VAL__SHIFT 0x00000001
+#define PB1_GLB_CTRL_REG1__RXDBG_CDR_PH_BYP_EN_MASK 0x00000080L
+#define PB1_GLB_CTRL_REG1__RXDBG_CDR_PH_BYP_EN__SHIFT 0x00000007
+#define PB1_GLB_CTRL_REG1__RXDBG_CDR_PH_BYP_VAL_MASK 0x00003f00L
+#define PB1_GLB_CTRL_REG1__RXDBG_CDR_PH_BYP_VAL__SHIFT 0x00000008
+#define PB1_GLB_CTRL_REG1__RXDBG_D0TH_BYP_EN_MASK 0x00004000L
+#define PB1_GLB_CTRL_REG1__RXDBG_D0TH_BYP_EN__SHIFT 0x0000000e
+#define PB1_GLB_CTRL_REG1__RXDBG_D0TH_BYP_VAL_MASK 0x003f8000L
+#define PB1_GLB_CTRL_REG1__RXDBG_D0TH_BYP_VAL__SHIFT 0x0000000f
+#define PB1_GLB_CTRL_REG1__RXDBG_D1TH_BYP_EN_MASK 0x00400000L
+#define PB1_GLB_CTRL_REG1__RXDBG_D1TH_BYP_EN__SHIFT 0x00000016
+#define PB1_GLB_CTRL_REG1__RXDBG_D1TH_BYP_VAL_MASK 0x3f800000L
+#define PB1_GLB_CTRL_REG1__RXDBG_D1TH_BYP_VAL__SHIFT 0x00000017
+#define PB1_GLB_CTRL_REG1__TST_LOSPDTST_EN_MASK 0x40000000L
+#define PB1_GLB_CTRL_REG1__TST_LOSPDTST_EN__SHIFT 0x0000001e
+#define PB1_GLB_CTRL_REG2__RXDBG_D2TH_BYP_EN_MASK 0x00000001L
+#define PB1_GLB_CTRL_REG2__RXDBG_D2TH_BYP_EN__SHIFT 0x00000000
+#define PB1_GLB_CTRL_REG2__RXDBG_D2TH_BYP_VAL_MASK 0x000000feL
+#define PB1_GLB_CTRL_REG2__RXDBG_D2TH_BYP_VAL__SHIFT 0x00000001
+#define PB1_GLB_CTRL_REG2__RXDBG_D3TH_BYP_EN_MASK 0x00000100L
+#define PB1_GLB_CTRL_REG2__RXDBG_D3TH_BYP_EN__SHIFT 0x00000008
+#define PB1_GLB_CTRL_REG2__RXDBG_D3TH_BYP_VAL_MASK 0x0000fe00L
+#define PB1_GLB_CTRL_REG2__RXDBG_D3TH_BYP_VAL__SHIFT 0x00000009
+#define PB1_GLB_CTRL_REG2__RXDBG_DXTH_BYP_EN_MASK 0x00010000L
+#define PB1_GLB_CTRL_REG2__RXDBG_DXTH_BYP_EN__SHIFT 0x00000010
+#define PB1_GLB_CTRL_REG2__RXDBG_DXTH_BYP_VAL_MASK 0x00fe0000L
+#define PB1_GLB_CTRL_REG2__RXDBG_DXTH_BYP_VAL__SHIFT 0x00000011
+#define PB1_GLB_CTRL_REG2__RXDBG_ETH_BYP_EN_MASK 0x01000000L
+#define PB1_GLB_CTRL_REG2__RXDBG_ETH_BYP_EN__SHIFT 0x00000018
+#define PB1_GLB_CTRL_REG2__RXDBG_ETH_BYP_VAL_MASK 0xfe000000L
+#define PB1_GLB_CTRL_REG2__RXDBG_ETH_BYP_VAL__SHIFT 0x00000019
+#define PB1_GLB_CTRL_REG3__BG_CFG_LC_REG_VREF0_SEL_MASK 0x00000060L
+#define PB1_GLB_CTRL_REG3__BG_CFG_LC_REG_VREF0_SEL__SHIFT 0x00000005
+#define PB1_GLB_CTRL_REG3__BG_CFG_LC_REG_VREF1_SEL_MASK 0x00000180L
+#define PB1_GLB_CTRL_REG3__BG_CFG_LC_REG_VREF1_SEL__SHIFT 0x00000007
+#define PB1_GLB_CTRL_REG3__BG_CFG_RO_REG_VREF_SEL_MASK 0x00000600L
+#define PB1_GLB_CTRL_REG3__BG_CFG_RO_REG_VREF_SEL__SHIFT 0x00000009
+#define PB1_GLB_CTRL_REG3__BG_DBG_ANALOG_SEL_MASK 0x0001c000L
+#define PB1_GLB_CTRL_REG3__BG_DBG_ANALOG_SEL__SHIFT 0x0000000e
+#define PB1_GLB_CTRL_REG3__BG_DBG_IREFBYP_EN_MASK 0x00001000L
+#define PB1_GLB_CTRL_REG3__BG_DBG_IREFBYP_EN__SHIFT 0x0000000c
+#define PB1_GLB_CTRL_REG3__BG_DBG_VREFBYP_EN_MASK 0x00000800L
+#define PB1_GLB_CTRL_REG3__BG_DBG_VREFBYP_EN__SHIFT 0x0000000b
+#define PB1_GLB_CTRL_REG3__DBG_DLL_CLK_SEL_MASK 0x001c0000L
+#define PB1_GLB_CTRL_REG3__DBG_DLL_CLK_SEL__SHIFT 0x00000012
+#define PB1_GLB_CTRL_REG3__DBG_RXLEQ_DCATTN_BYP_OVR_DISABLE_MASK 0x80000000L
+#define PB1_GLB_CTRL_REG3__DBG_RXLEQ_DCATTN_BYP_OVR_DISABLE__SHIFT 0x0000001f
+#define PB1_GLB_CTRL_REG3__DBG_RXPI_OFFSET_BYP_EN_MASK 0x00400000L
+#define PB1_GLB_CTRL_REG3__DBG_RXPI_OFFSET_BYP_EN__SHIFT 0x00000016
+#define PB1_GLB_CTRL_REG3__DBG_RXPI_OFFSET_BYP_VAL_MASK 0x07800000L
+#define PB1_GLB_CTRL_REG3__DBG_RXPI_OFFSET_BYP_VAL__SHIFT 0x00000017
+#define PB1_GLB_CTRL_REG3__DBG_RXSWAPDX_BYP_EN_MASK 0x08000000L
+#define PB1_GLB_CTRL_REG3__DBG_RXSWAPDX_BYP_EN__SHIFT 0x0000001b
+#define PB1_GLB_CTRL_REG3__DBG_RXSWAPDX_BYP_VAL_MASK 0x70000000L
+#define PB1_GLB_CTRL_REG3__DBG_RXSWAPDX_BYP_VAL__SHIFT 0x0000001c
+#define PB1_GLB_CTRL_REG3__PLL_DISPCLK_CMOS_SEL_MASK 0x00200000L
+#define PB1_GLB_CTRL_REG3__PLL_DISPCLK_CMOS_SEL__SHIFT 0x00000015
+#define PB1_GLB_CTRL_REG3__RXDBG_SEL_MASK 0x0000001fL
+#define PB1_GLB_CTRL_REG3__RXDBG_SEL__SHIFT 0x00000000
+#define PB1_GLB_CTRL_REG4__DBG_RXAPU_EXEC_MASK 0x03c00000L
+#define PB1_GLB_CTRL_REG4__DBG_RXAPU_EXEC__SHIFT 0x00000016
+#define PB1_GLB_CTRL_REG4__DBG_RXAPU_INST_MASK 0x0000ffffL
+#define PB1_GLB_CTRL_REG4__DBG_RXAPU_INST__SHIFT 0x00000000
+#define PB1_GLB_CTRL_REG4__DBG_RXDFEMUX_BYP_EN_MASK 0x00040000L
+#define PB1_GLB_CTRL_REG4__DBG_RXDFEMUX_BYP_EN__SHIFT 0x00000012
+#define PB1_GLB_CTRL_REG4__DBG_RXDFEMUX_BYP_VAL_MASK 0x00030000L
+#define PB1_GLB_CTRL_REG4__DBG_RXDFEMUX_BYP_VAL__SHIFT 0x00000010
+#define PB1_GLB_CTRL_REG4__DBG_RXDLL_VREG_REF_SEL_MASK 0x04000000L
+#define PB1_GLB_CTRL_REG4__DBG_RXDLL_VREG_REF_SEL__SHIFT 0x0000001a
+#define PB1_GLB_CTRL_REG4__DBG_RXRDATA_GATING_DISABLE_MASK 0x10000000L
+#define PB1_GLB_CTRL_REG4__DBG_RXRDATA_GATING_DISABLE__SHIFT 0x0000001c
+#define PB1_GLB_CTRL_REG4__PWRGOOD_OVRD_MASK 0x08000000L
+#define PB1_GLB_CTRL_REG4__PWRGOOD_OVRD__SHIFT 0x0000001b
+#define PB1_GLB_CTRL_REG5__DBG_RXAPU_MODE_MASK 0x000000ffL
+#define PB1_GLB_CTRL_REG5__DBG_RXAPU_MODE__SHIFT 0x00000000
+#define PB1_GLB_OVRD_REG0__TXPDTERM_VAL_OVRD_VAL_MASK 0x0000ffffL
+#define PB1_GLB_OVRD_REG0__TXPDTERM_VAL_OVRD_VAL__SHIFT 0x00000000
+#define PB1_GLB_OVRD_REG0__TXPUTERM_VAL_OVRD_VAL_MASK 0xffff0000L
+#define PB1_GLB_OVRD_REG0__TXPUTERM_VAL_OVRD_VAL__SHIFT 0x00000010
+#define PB1_GLB_OVRD_REG1__RXTERM_VAL_OVRD_EN_MASK 0x00008000L
+#define PB1_GLB_OVRD_REG1__RXTERM_VAL_OVRD_EN__SHIFT 0x0000000f
+#define PB1_GLB_OVRD_REG1__RXTERM_VAL_OVRD_VAL_MASK 0xffff0000L
+#define PB1_GLB_OVRD_REG1__RXTERM_VAL_OVRD_VAL__SHIFT 0x00000010
+#define PB1_GLB_OVRD_REG1__TST_LOSPDTST_RST_OVRD_EN_MASK 0x00000004L
+#define PB1_GLB_OVRD_REG1__TST_LOSPDTST_RST_OVRD_EN__SHIFT 0x00000002
+#define PB1_GLB_OVRD_REG1__TST_LOSPDTST_RST_OVRD_VAL_MASK 0x00000008L
+#define PB1_GLB_OVRD_REG1__TST_LOSPDTST_RST_OVRD_VAL__SHIFT 0x00000003
+#define PB1_GLB_OVRD_REG1__TXPDTERM_VAL_OVRD_EN_MASK 0x00000001L
+#define PB1_GLB_OVRD_REG1__TXPDTERM_VAL_OVRD_EN__SHIFT 0x00000000
+#define PB1_GLB_OVRD_REG1__TXPUTERM_VAL_OVRD_EN_MASK 0x00000002L
+#define PB1_GLB_OVRD_REG1__TXPUTERM_VAL_OVRD_EN__SHIFT 0x00000001
+#define PB1_GLB_OVRD_REG2__BG_PWRON_OVRD_EN_MASK 0x00000001L
+#define PB1_GLB_OVRD_REG2__BG_PWRON_OVRD_EN__SHIFT 0x00000000
+#define PB1_GLB_OVRD_REG2__BG_PWRON_OVRD_VAL_MASK 0x00000002L
+#define PB1_GLB_OVRD_REG2__BG_PWRON_OVRD_VAL__SHIFT 0x00000001
+#define PB1_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L0T3_MASK 0x00000001L
+#define PB1_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L0T3__SHIFT 0x00000000
+#define PB1_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L12T15_MASK 0x00000008L
+#define PB1_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L12T15__SHIFT 0x00000003
+#define PB1_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L4T7_MASK 0x00000002L
+#define PB1_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L4T7__SHIFT 0x00000001
+#define PB1_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L8T11_MASK 0x00000004L
+#define PB1_GLB_SCI_STAT_OVRD_REG0__IGNR_ALL_SCI_UPDT_L8T11__SHIFT 0x00000002
+#define PB1_GLB_SCI_STAT_OVRD_REG0__IGNR_IMPCAL_ACTIVE_SCI_UPDT_MASK 0x00000010L
+#define PB1_GLB_SCI_STAT_OVRD_REG0__IGNR_IMPCAL_ACTIVE_SCI_UPDT__SHIFT 0x00000004
+#define PB1_GLB_SCI_STAT_OVRD_REG0__IMPCAL_ACTIVE_MASK 0x00100000L
+#define PB1_GLB_SCI_STAT_OVRD_REG0__IMPCAL_ACTIVE__SHIFT 0x00000014
+#define PB1_GLB_SCI_STAT_OVRD_REG0__RXIMP_MASK 0x000f0000L
+#define PB1_GLB_SCI_STAT_OVRD_REG0__RXIMP__SHIFT 0x00000010
+#define PB1_GLB_SCI_STAT_OVRD_REG0__TXNIMP_MASK 0x00000f00L
+#define PB1_GLB_SCI_STAT_OVRD_REG0__TXNIMP__SHIFT 0x00000008
+#define PB1_GLB_SCI_STAT_OVRD_REG0__TXPIMP_MASK 0x0000f000L
+#define PB1_GLB_SCI_STAT_OVRD_REG0__TXPIMP__SHIFT 0x0000000c
+#define PB1_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_0_MASK 0x00001000L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_0__SHIFT 0x0000000c
+#define PB1_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_1_MASK 0x00002000L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_1__SHIFT 0x0000000d
+#define PB1_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_2_MASK 0x00004000L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_2__SHIFT 0x0000000e
+#define PB1_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_3_MASK 0x00008000L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__DLL_LOCK_3__SHIFT 0x0000000f
+#define PB1_GLB_SCI_STAT_OVRD_REG1__FREQDIV_0_MASK 0x000c0000L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__FREQDIV_0__SHIFT 0x00000012
+#define PB1_GLB_SCI_STAT_OVRD_REG1__FREQDIV_1_MASK 0x00c00000L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__FREQDIV_1__SHIFT 0x00000016
+#define PB1_GLB_SCI_STAT_OVRD_REG1__FREQDIV_2_MASK 0x0c000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__FREQDIV_2__SHIFT 0x0000001a
+#define PB1_GLB_SCI_STAT_OVRD_REG1__FREQDIV_3_MASK 0xc0000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__FREQDIV_3__SHIFT 0x0000001e
+#define PB1_GLB_SCI_STAT_OVRD_REG1__IGNR_DLL_LOCK_SCI_UPDT_L0T3_MASK 0x00000004L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__IGNR_DLL_LOCK_SCI_UPDT_L0T3__SHIFT 0x00000002
+#define PB1_GLB_SCI_STAT_OVRD_REG1__IGNR_FREQDIV_SCI_UPDT_L0T3_MASK 0x00000002L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__IGNR_FREQDIV_SCI_UPDT_L0T3__SHIFT 0x00000001
+#define PB1_GLB_SCI_STAT_OVRD_REG1__IGNR_MODE_SCI_UPDT_L0T3_MASK 0x00000001L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__IGNR_MODE_SCI_UPDT_L0T3__SHIFT 0x00000000
+#define PB1_GLB_SCI_STAT_OVRD_REG1__MODE_0_MASK 0x00030000L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__MODE_0__SHIFT 0x00000010
+#define PB1_GLB_SCI_STAT_OVRD_REG1__MODE_1_MASK 0x00300000L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__MODE_1__SHIFT 0x00000014
+#define PB1_GLB_SCI_STAT_OVRD_REG1__MODE_2_MASK 0x03000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__MODE_2__SHIFT 0x00000018
+#define PB1_GLB_SCI_STAT_OVRD_REG1__MODE_3_MASK 0x30000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG1__MODE_3__SHIFT 0x0000001c
+#define PB1_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_4_MASK 0x00001000L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_4__SHIFT 0x0000000c
+#define PB1_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_5_MASK 0x00002000L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_5__SHIFT 0x0000000d
+#define PB1_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_6_MASK 0x00004000L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_6__SHIFT 0x0000000e
+#define PB1_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_7_MASK 0x00008000L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__DLL_LOCK_7__SHIFT 0x0000000f
+#define PB1_GLB_SCI_STAT_OVRD_REG2__FREQDIV_4_MASK 0x000c0000L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__FREQDIV_4__SHIFT 0x00000012
+#define PB1_GLB_SCI_STAT_OVRD_REG2__FREQDIV_5_MASK 0x00c00000L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__FREQDIV_5__SHIFT 0x00000016
+#define PB1_GLB_SCI_STAT_OVRD_REG2__FREQDIV_6_MASK 0x0c000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__FREQDIV_6__SHIFT 0x0000001a
+#define PB1_GLB_SCI_STAT_OVRD_REG2__FREQDIV_7_MASK 0xc0000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__FREQDIV_7__SHIFT 0x0000001e
+#define PB1_GLB_SCI_STAT_OVRD_REG2__IGNR_DLL_LOCK_SCI_UPDT_L4T7_MASK 0x00000004L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__IGNR_DLL_LOCK_SCI_UPDT_L4T7__SHIFT 0x00000002
+#define PB1_GLB_SCI_STAT_OVRD_REG2__IGNR_FREQDIV_SCI_UPDT_L4T7_MASK 0x00000002L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__IGNR_FREQDIV_SCI_UPDT_L4T7__SHIFT 0x00000001
+#define PB1_GLB_SCI_STAT_OVRD_REG2__IGNR_MODE_SCI_UPDT_L4T7_MASK 0x00000001L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__IGNR_MODE_SCI_UPDT_L4T7__SHIFT 0x00000000
+#define PB1_GLB_SCI_STAT_OVRD_REG2__MODE_4_MASK 0x00030000L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__MODE_4__SHIFT 0x00000010
+#define PB1_GLB_SCI_STAT_OVRD_REG2__MODE_5_MASK 0x00300000L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__MODE_5__SHIFT 0x00000014
+#define PB1_GLB_SCI_STAT_OVRD_REG2__MODE_6_MASK 0x03000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__MODE_6__SHIFT 0x00000018
+#define PB1_GLB_SCI_STAT_OVRD_REG2__MODE_7_MASK 0x30000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG2__MODE_7__SHIFT 0x0000001c
+#define PB1_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_10_MASK 0x00004000L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_10__SHIFT 0x0000000e
+#define PB1_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_11_MASK 0x00008000L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_11__SHIFT 0x0000000f
+#define PB1_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_8_MASK 0x00001000L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_8__SHIFT 0x0000000c
+#define PB1_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_9_MASK 0x00002000L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__DLL_LOCK_9__SHIFT 0x0000000d
+#define PB1_GLB_SCI_STAT_OVRD_REG3__FREQDIV_10_MASK 0x0c000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__FREQDIV_10__SHIFT 0x0000001a
+#define PB1_GLB_SCI_STAT_OVRD_REG3__FREQDIV_11_MASK 0xc0000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__FREQDIV_11__SHIFT 0x0000001e
+#define PB1_GLB_SCI_STAT_OVRD_REG3__FREQDIV_8_MASK 0x000c0000L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__FREQDIV_8__SHIFT 0x00000012
+#define PB1_GLB_SCI_STAT_OVRD_REG3__FREQDIV_9_MASK 0x00c00000L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__FREQDIV_9__SHIFT 0x00000016
+#define PB1_GLB_SCI_STAT_OVRD_REG3__IGNR_DLL_LOCK_SCI_UPDT_L8T11_MASK 0x00000004L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__IGNR_DLL_LOCK_SCI_UPDT_L8T11__SHIFT 0x00000002
+#define PB1_GLB_SCI_STAT_OVRD_REG3__IGNR_FREQDIV_SCI_UPDT_L8T11_MASK 0x00000002L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__IGNR_FREQDIV_SCI_UPDT_L8T11__SHIFT 0x00000001
+#define PB1_GLB_SCI_STAT_OVRD_REG3__IGNR_MODE_SCI_UPDT_L8T11_MASK 0x00000001L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__IGNR_MODE_SCI_UPDT_L8T11__SHIFT 0x00000000
+#define PB1_GLB_SCI_STAT_OVRD_REG3__MODE_10_MASK 0x03000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__MODE_10__SHIFT 0x00000018
+#define PB1_GLB_SCI_STAT_OVRD_REG3__MODE_11_MASK 0x30000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__MODE_11__SHIFT 0x0000001c
+#define PB1_GLB_SCI_STAT_OVRD_REG3__MODE_8_MASK 0x00030000L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__MODE_8__SHIFT 0x00000010
+#define PB1_GLB_SCI_STAT_OVRD_REG3__MODE_9_MASK 0x00300000L
+#define PB1_GLB_SCI_STAT_OVRD_REG3__MODE_9__SHIFT 0x00000014
+#define PB1_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_12_MASK 0x00001000L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_12__SHIFT 0x0000000c
+#define PB1_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_13_MASK 0x00002000L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_13__SHIFT 0x0000000d
+#define PB1_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_14_MASK 0x00004000L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_14__SHIFT 0x0000000e
+#define PB1_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_15_MASK 0x00008000L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__DLL_LOCK_15__SHIFT 0x0000000f
+#define PB1_GLB_SCI_STAT_OVRD_REG4__FREQDIV_12_MASK 0x000c0000L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__FREQDIV_12__SHIFT 0x00000012
+#define PB1_GLB_SCI_STAT_OVRD_REG4__FREQDIV_13_MASK 0x00c00000L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__FREQDIV_13__SHIFT 0x00000016
+#define PB1_GLB_SCI_STAT_OVRD_REG4__FREQDIV_14_MASK 0x0c000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__FREQDIV_14__SHIFT 0x0000001a
+#define PB1_GLB_SCI_STAT_OVRD_REG4__FREQDIV_15_MASK 0xc0000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__FREQDIV_15__SHIFT 0x0000001e
+#define PB1_GLB_SCI_STAT_OVRD_REG4__IGNR_DLL_LOCK_SCI_UPDT_L12T15_MASK 0x00000004L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__IGNR_DLL_LOCK_SCI_UPDT_L12T15__SHIFT 0x00000002
+#define PB1_GLB_SCI_STAT_OVRD_REG4__IGNR_FREQDIV_SCI_UPDT_L12T15_MASK 0x00000002L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__IGNR_FREQDIV_SCI_UPDT_L12T15__SHIFT 0x00000001
+#define PB1_GLB_SCI_STAT_OVRD_REG4__IGNR_MODE_SCI_UPDT_L12T15_MASK 0x00000001L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__IGNR_MODE_SCI_UPDT_L12T15__SHIFT 0x00000000
+#define PB1_GLB_SCI_STAT_OVRD_REG4__MODE_12_MASK 0x00030000L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__MODE_12__SHIFT 0x00000010
+#define PB1_GLB_SCI_STAT_OVRD_REG4__MODE_13_MASK 0x00300000L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__MODE_13__SHIFT 0x00000014
+#define PB1_GLB_SCI_STAT_OVRD_REG4__MODE_14_MASK 0x03000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__MODE_14__SHIFT 0x00000018
+#define PB1_GLB_SCI_STAT_OVRD_REG4__MODE_15_MASK 0x30000000L
+#define PB1_GLB_SCI_STAT_OVRD_REG4__MODE_15__SHIFT 0x0000001c
+#define PB1_HW_DEBUG__PB1_HW_00_DEBUG_MASK 0x00000001L
+#define PB1_HW_DEBUG__PB1_HW_00_DEBUG__SHIFT 0x00000000
+#define PB1_HW_DEBUG__PB1_HW_01_DEBUG_MASK 0x00000002L
+#define PB1_HW_DEBUG__PB1_HW_01_DEBUG__SHIFT 0x00000001
+#define PB1_HW_DEBUG__PB1_HW_02_DEBUG_MASK 0x00000004L
+#define PB1_HW_DEBUG__PB1_HW_02_DEBUG__SHIFT 0x00000002
+#define PB1_HW_DEBUG__PB1_HW_03_DEBUG_MASK 0x00000008L
+#define PB1_HW_DEBUG__PB1_HW_03_DEBUG__SHIFT 0x00000003
+#define PB1_HW_DEBUG__PB1_HW_04_DEBUG_MASK 0x00000010L
+#define PB1_HW_DEBUG__PB1_HW_04_DEBUG__SHIFT 0x00000004
+#define PB1_HW_DEBUG__PB1_HW_05_DEBUG_MASK 0x00000020L
+#define PB1_HW_DEBUG__PB1_HW_05_DEBUG__SHIFT 0x00000005
+#define PB1_HW_DEBUG__PB1_HW_06_DEBUG_MASK 0x00000040L
+#define PB1_HW_DEBUG__PB1_HW_06_DEBUG__SHIFT 0x00000006
+#define PB1_HW_DEBUG__PB1_HW_07_DEBUG_MASK 0x00000080L
+#define PB1_HW_DEBUG__PB1_HW_07_DEBUG__SHIFT 0x00000007
+#define PB1_HW_DEBUG__PB1_HW_08_DEBUG_MASK 0x00000100L
+#define PB1_HW_DEBUG__PB1_HW_08_DEBUG__SHIFT 0x00000008
+#define PB1_HW_DEBUG__PB1_HW_09_DEBUG_MASK 0x00000200L
+#define PB1_HW_DEBUG__PB1_HW_09_DEBUG__SHIFT 0x00000009
+#define PB1_HW_DEBUG__PB1_HW_10_DEBUG_MASK 0x00000400L
+#define PB1_HW_DEBUG__PB1_HW_10_DEBUG__SHIFT 0x0000000a
+#define PB1_HW_DEBUG__PB1_HW_11_DEBUG_MASK 0x00000800L
+#define PB1_HW_DEBUG__PB1_HW_11_DEBUG__SHIFT 0x0000000b
+#define PB1_HW_DEBUG__PB1_HW_12_DEBUG_MASK 0x00001000L
+#define PB1_HW_DEBUG__PB1_HW_12_DEBUG__SHIFT 0x0000000c
+#define PB1_HW_DEBUG__PB1_HW_13_DEBUG_MASK 0x00002000L
+#define PB1_HW_DEBUG__PB1_HW_13_DEBUG__SHIFT 0x0000000d
+#define PB1_HW_DEBUG__PB1_HW_14_DEBUG_MASK 0x00004000L
+#define PB1_HW_DEBUG__PB1_HW_14_DEBUG__SHIFT 0x0000000e
+#define PB1_HW_DEBUG__PB1_HW_15_DEBUG_MASK 0x00008000L
+#define PB1_HW_DEBUG__PB1_HW_15_DEBUG__SHIFT 0x0000000f
+#define PB1_HW_DEBUG__PB1_HW_16_DEBUG_MASK 0x00010000L
+#define PB1_HW_DEBUG__PB1_HW_16_DEBUG__SHIFT 0x00000010
+#define PB1_HW_DEBUG__PB1_HW_17_DEBUG_MASK 0x00020000L
+#define PB1_HW_DEBUG__PB1_HW_17_DEBUG__SHIFT 0x00000011
+#define PB1_HW_DEBUG__PB1_HW_18_DEBUG_MASK 0x00040000L
+#define PB1_HW_DEBUG__PB1_HW_18_DEBUG__SHIFT 0x00000012
+#define PB1_HW_DEBUG__PB1_HW_19_DEBUG_MASK 0x00080000L
+#define PB1_HW_DEBUG__PB1_HW_19_DEBUG__SHIFT 0x00000013
+#define PB1_HW_DEBUG__PB1_HW_20_DEBUG_MASK 0x00100000L
+#define PB1_HW_DEBUG__PB1_HW_20_DEBUG__SHIFT 0x00000014
+#define PB1_HW_DEBUG__PB1_HW_21_DEBUG_MASK 0x00200000L
+#define PB1_HW_DEBUG__PB1_HW_21_DEBUG__SHIFT 0x00000015
+#define PB1_HW_DEBUG__PB1_HW_22_DEBUG_MASK 0x00400000L
+#define PB1_HW_DEBUG__PB1_HW_22_DEBUG__SHIFT 0x00000016
+#define PB1_HW_DEBUG__PB1_HW_23_DEBUG_MASK 0x00800000L
+#define PB1_HW_DEBUG__PB1_HW_23_DEBUG__SHIFT 0x00000017
+#define PB1_HW_DEBUG__PB1_HW_24_DEBUG_MASK 0x01000000L
+#define PB1_HW_DEBUG__PB1_HW_24_DEBUG__SHIFT 0x00000018
+#define PB1_HW_DEBUG__PB1_HW_25_DEBUG_MASK 0x02000000L
+#define PB1_HW_DEBUG__PB1_HW_25_DEBUG__SHIFT 0x00000019
+#define PB1_HW_DEBUG__PB1_HW_26_DEBUG_MASK 0x04000000L
+#define PB1_HW_DEBUG__PB1_HW_26_DEBUG__SHIFT 0x0000001a
+#define PB1_HW_DEBUG__PB1_HW_27_DEBUG_MASK 0x08000000L
+#define PB1_HW_DEBUG__PB1_HW_27_DEBUG__SHIFT 0x0000001b
+#define PB1_HW_DEBUG__PB1_HW_28_DEBUG_MASK 0x10000000L
+#define PB1_HW_DEBUG__PB1_HW_28_DEBUG__SHIFT 0x0000001c
+#define PB1_HW_DEBUG__PB1_HW_29_DEBUG_MASK 0x20000000L
+#define PB1_HW_DEBUG__PB1_HW_29_DEBUG__SHIFT 0x0000001d
+#define PB1_HW_DEBUG__PB1_HW_30_DEBUG_MASK 0x40000000L
+#define PB1_HW_DEBUG__PB1_HW_30_DEBUG__SHIFT 0x0000001e
+#define PB1_HW_DEBUG__PB1_HW_31_DEBUG_MASK 0x80000000L
+#define PB1_HW_DEBUG__PB1_HW_31_DEBUG__SHIFT 0x0000001f
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_EN_MASK 0x00000080L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_EN__SHIFT 0x00000007
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_0_MASK 0x00000100L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_0__SHIFT 0x00000008
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_10_MASK 0x00040000L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_10__SHIFT 0x00000012
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_11_MASK 0x00080000L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_11__SHIFT 0x00000013
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_12_MASK 0x00100000L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_12__SHIFT 0x00000014
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_13_MASK 0x00200000L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_13__SHIFT 0x00000015
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_14_MASK 0x00400000L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_14__SHIFT 0x00000016
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_15_MASK 0x00800000L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_15__SHIFT 0x00000017
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_1_MASK 0x00000200L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_1__SHIFT 0x00000009
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_2_MASK 0x00000400L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_2__SHIFT 0x0000000a
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_3_MASK 0x00000800L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_3__SHIFT 0x0000000b
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_4_MASK 0x00001000L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_4__SHIFT 0x0000000c
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_5_MASK 0x00002000L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_5__SHIFT 0x0000000d
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_6_MASK 0x00004000L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_6__SHIFT 0x0000000e
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_7_MASK 0x00008000L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_7__SHIFT 0x0000000f
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_8_MASK 0x00010000L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_8__SHIFT 0x00000010
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_9_MASK 0x00020000L
+#define PB1_PIF_CNTL2__RXDETECT_OVERRIDE_VAL_9__SHIFT 0x00000011
+#define PB1_PIF_CNTL2__RXDETECT_SAMPL_TIME_MASK 0x00000006L
+#define PB1_PIF_CNTL2__RXDETECT_SAMPL_TIME__SHIFT 0x00000001
+#define PB1_PIF_CNTL2__RXPHYSTATUS_DELAY_MASK 0x07000000L
+#define PB1_PIF_CNTL2__RXPHYSTATUS_DELAY__SHIFT 0x00000018
+#define PB1_PIF_CNTL__DA_FIFO_RESET_0_MASK 0x00000002L
+#define PB1_PIF_CNTL__DA_FIFO_RESET_0__SHIFT 0x00000001
+#define PB1_PIF_CNTL__DA_FIFO_RESET_1_MASK 0x00000020L
+#define PB1_PIF_CNTL__DA_FIFO_RESET_1__SHIFT 0x00000005
+#define PB1_PIF_CNTL__DA_FIFO_RESET_2_MASK 0x00000200L
+#define PB1_PIF_CNTL__DA_FIFO_RESET_2__SHIFT 0x00000009
+#define PB1_PIF_CNTL__DA_FIFO_RESET_3_MASK 0x00002000L
+#define PB1_PIF_CNTL__DA_FIFO_RESET_3__SHIFT 0x0000000d
+#define PB1_PIF_CNTL__DIVINIT_MODE_MASK 0x00000100L
+#define PB1_PIF_CNTL__DIVINIT_MODE__SHIFT 0x00000008
+#define PB1_PIF_CNTL__EI_CYCLE_OFF_TIME_MASK 0x00700000L
+#define PB1_PIF_CNTL__EI_CYCLE_OFF_TIME__SHIFT 0x00000014
+#define PB1_PIF_CNTL__EI_DET_CYCLE_MODE_MASK 0x00000010L
+#define PB1_PIF_CNTL__EI_DET_CYCLE_MODE__SHIFT 0x00000004
+#define PB1_PIF_CNTL__EXIT_L0S_INIT_DIS_MASK 0x00800000L
+#define PB1_PIF_CNTL__EXIT_L0S_INIT_DIS__SHIFT 0x00000017
+#define PB1_PIF_CNTL__EXTEND_WAIT_FOR_RAMPUP_MASK 0x10000000L
+#define PB1_PIF_CNTL__EXTEND_WAIT_FOR_RAMPUP__SHIFT 0x0000001c
+#define PB1_PIF_CNTL__IGNORE_TxDataValid_EP_DIS_MASK 0x20000000L
+#define PB1_PIF_CNTL__IGNORE_TxDataValid_EP_DIS__SHIFT 0x0000001d
+#define PB1_PIF_CNTL__LS2_EXIT_TIME_MASK 0x000e0000L
+#define PB1_PIF_CNTL__LS2_EXIT_TIME__SHIFT 0x00000011
+#define PB1_PIF_CNTL__PHYCMD_CR_EN_MODE_MASK 0x00000008L
+#define PB1_PIF_CNTL__PHYCMD_CR_EN_MODE__SHIFT 0x00000003
+#define PB1_PIF_CNTL__PHY_CR_EN_MODE_MASK 0x00000004L
+#define PB1_PIF_CNTL__PHY_CR_EN_MODE__SHIFT 0x00000002
+#define PB1_PIF_CNTL__PLL_BINDING_ENABLE_MASK 0x00000400L
+#define PB1_PIF_CNTL__PLL_BINDING_ENABLE__SHIFT 0x0000000a
+#define PB1_PIF_CNTL__RXDETECT_FIFO_RESET_MODE_MASK 0x00000040L
+#define PB1_PIF_CNTL__RXDETECT_FIFO_RESET_MODE__SHIFT 0x00000006
+#define PB1_PIF_CNTL__RXDETECT_TX_PWR_MODE_MASK 0x00000080L
+#define PB1_PIF_CNTL__RXDETECT_TX_PWR_MODE__SHIFT 0x00000007
+#define PB1_PIF_CNTL__RXEN_GATER_MASK 0x0f000000L
+#define PB1_PIF_CNTL__RXEN_GATER__SHIFT 0x00000018
+#define PB1_PIF_CNTL__SC_CALIB_DONE_CNTL_MASK 0x00000800L
+#define PB1_PIF_CNTL__SC_CALIB_DONE_CNTL__SHIFT 0x0000000b
+#define PB1_PIF_CNTL__SERIAL_CFG_ENABLE_MASK 0x00000001L
+#define PB1_PIF_CNTL__SERIAL_CFG_ENABLE__SHIFT 0x00000000
+#define PB1_PIF_CNTL__TXGND_TIME_MASK 0x00010000L
+#define PB1_PIF_CNTL__TXGND_TIME__SHIFT 0x00000010
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_00_DEBUG_MASK 0x00000001L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_00_DEBUG__SHIFT 0x00000000
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_01_DEBUG_MASK 0x00000002L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_01_DEBUG__SHIFT 0x00000001
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_02_DEBUG_MASK 0x00000004L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_02_DEBUG__SHIFT 0x00000002
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_03_DEBUG_MASK 0x00000008L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_03_DEBUG__SHIFT 0x00000003
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_04_DEBUG_MASK 0x00000010L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_04_DEBUG__SHIFT 0x00000004
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_05_DEBUG_MASK 0x00000020L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_05_DEBUG__SHIFT 0x00000005
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_06_DEBUG_MASK 0x00000040L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_06_DEBUG__SHIFT 0x00000006
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_07_DEBUG_MASK 0x00000080L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_07_DEBUG__SHIFT 0x00000007
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_08_DEBUG_MASK 0x00000100L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_08_DEBUG__SHIFT 0x00000008
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_09_DEBUG_MASK 0x00000200L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_09_DEBUG__SHIFT 0x00000009
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_10_DEBUG_MASK 0x00000400L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_10_DEBUG__SHIFT 0x0000000a
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_11_DEBUG_MASK 0x00000800L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_11_DEBUG__SHIFT 0x0000000b
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_12_DEBUG_MASK 0x00001000L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_12_DEBUG__SHIFT 0x0000000c
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_13_DEBUG_MASK 0x00002000L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_13_DEBUG__SHIFT 0x0000000d
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_14_DEBUG_MASK 0x00004000L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_14_DEBUG__SHIFT 0x0000000e
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_15_DEBUG_MASK 0x00008000L
+#define PB1_PIF_HW_DEBUG__PB1_PIF_HW_15_DEBUG__SHIFT 0x0000000f
+#define PB1_PIF_PAIRING__MULTI_PIF_MASK 0x02000000L
+#define PB1_PIF_PAIRING__MULTI_PIF__SHIFT 0x00000019
+#define PB1_PIF_PAIRING__X16_LANE_15_0_MASK 0x00100000L
+#define PB1_PIF_PAIRING__X16_LANE_15_0__SHIFT 0x00000014
+#define PB1_PIF_PAIRING__X2_LANE_1_0_MASK 0x00000001L
+#define PB1_PIF_PAIRING__X2_LANE_1_0__SHIFT 0x00000000
+#define PB1_PIF_PAIRING__X2_LANE_11_10_MASK 0x00000020L
+#define PB1_PIF_PAIRING__X2_LANE_11_10__SHIFT 0x00000005
+#define PB1_PIF_PAIRING__X2_LANE_13_12_MASK 0x00000040L
+#define PB1_PIF_PAIRING__X2_LANE_13_12__SHIFT 0x00000006
+#define PB1_PIF_PAIRING__X2_LANE_15_14_MASK 0x00000080L
+#define PB1_PIF_PAIRING__X2_LANE_15_14__SHIFT 0x00000007
+#define PB1_PIF_PAIRING__X2_LANE_3_2_MASK 0x00000002L
+#define PB1_PIF_PAIRING__X2_LANE_3_2__SHIFT 0x00000001
+#define PB1_PIF_PAIRING__X2_LANE_5_4_MASK 0x00000004L
+#define PB1_PIF_PAIRING__X2_LANE_5_4__SHIFT 0x00000002
+#define PB1_PIF_PAIRING__X2_LANE_7_6_MASK 0x00000008L
+#define PB1_PIF_PAIRING__X2_LANE_7_6__SHIFT 0x00000003
+#define PB1_PIF_PAIRING__X2_LANE_9_8_MASK 0x00000010L
+#define PB1_PIF_PAIRING__X2_LANE_9_8__SHIFT 0x00000004
+#define PB1_PIF_PAIRING__X4_LANE_11_8_MASK 0x00000400L
+#define PB1_PIF_PAIRING__X4_LANE_11_8__SHIFT 0x0000000a
+#define PB1_PIF_PAIRING__X4_LANE_15_12_MASK 0x00000800L
+#define PB1_PIF_PAIRING__X4_LANE_15_12__SHIFT 0x0000000b
+#define PB1_PIF_PAIRING__X4_LANE_3_0_MASK 0x00000100L
+#define PB1_PIF_PAIRING__X4_LANE_3_0__SHIFT 0x00000008
+#define PB1_PIF_PAIRING__X4_LANE_7_4_MASK 0x00000200L
+#define PB1_PIF_PAIRING__X4_LANE_7_4__SHIFT 0x00000009
+#define PB1_PIF_PAIRING__X8_LANE_15_8_MASK 0x00020000L
+#define PB1_PIF_PAIRING__X8_LANE_15_8__SHIFT 0x00000011
+#define PB1_PIF_PAIRING__X8_LANE_7_0_MASK 0x00010000L
+#define PB1_PIF_PAIRING__X8_LANE_7_0__SHIFT 0x00000010
+#define PB1_PIF_PDNB_OVERRIDE_0__RXEN_OVERRIDE_EN_0_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_0__RXEN_OVERRIDE_EN_0__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_0__RXEN_OVERRIDE_VAL_0_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_0__RXEN_OVERRIDE_VAL_0__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_0__RX_PDNB_OVERRIDE_EN_0_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_0__RX_PDNB_OVERRIDE_EN_0__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_0__RX_PDNB_OVERRIDE_VAL_0_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_0__RX_PDNB_OVERRIDE_VAL_0__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_0__RXPWR_OVERRIDE_EN_0_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_0__RXPWR_OVERRIDE_EN_0__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_0__RXPWR_OVERRIDE_VAL_0_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_0__RXPWR_OVERRIDE_VAL_0__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_0__TX_PDNB_OVERRIDE_EN_0_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_0__TX_PDNB_OVERRIDE_EN_0__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_0__TX_PDNB_OVERRIDE_VAL_0_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_0__TX_PDNB_OVERRIDE_VAL_0__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_0__TXPWR_OVERRIDE_EN_0_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_0__TXPWR_OVERRIDE_EN_0__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_0__TXPWR_OVERRIDE_VAL_0_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_0__TXPWR_OVERRIDE_VAL_0__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_10__RXEN_OVERRIDE_EN_10_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_10__RXEN_OVERRIDE_EN_10__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_10__RXEN_OVERRIDE_VAL_10_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_10__RXEN_OVERRIDE_VAL_10__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_10__RX_PDNB_OVERRIDE_EN_10_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_10__RX_PDNB_OVERRIDE_EN_10__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_10__RX_PDNB_OVERRIDE_VAL_10_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_10__RX_PDNB_OVERRIDE_VAL_10__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_10__RXPWR_OVERRIDE_EN_10_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_10__RXPWR_OVERRIDE_EN_10__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_10__RXPWR_OVERRIDE_VAL_10_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_10__RXPWR_OVERRIDE_VAL_10__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_10__TX_PDNB_OVERRIDE_EN_10_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_10__TX_PDNB_OVERRIDE_EN_10__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_10__TX_PDNB_OVERRIDE_VAL_10_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_10__TX_PDNB_OVERRIDE_VAL_10__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_10__TXPWR_OVERRIDE_EN_10_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_10__TXPWR_OVERRIDE_EN_10__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_10__TXPWR_OVERRIDE_VAL_10_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_10__TXPWR_OVERRIDE_VAL_10__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_11__RXEN_OVERRIDE_EN_11_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_11__RXEN_OVERRIDE_EN_11__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_11__RXEN_OVERRIDE_VAL_11_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_11__RXEN_OVERRIDE_VAL_11__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_11__RX_PDNB_OVERRIDE_EN_11_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_11__RX_PDNB_OVERRIDE_EN_11__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_11__RX_PDNB_OVERRIDE_VAL_11_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_11__RX_PDNB_OVERRIDE_VAL_11__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_11__RXPWR_OVERRIDE_EN_11_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_11__RXPWR_OVERRIDE_EN_11__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_11__RXPWR_OVERRIDE_VAL_11_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_11__RXPWR_OVERRIDE_VAL_11__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_11__TX_PDNB_OVERRIDE_EN_11_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_11__TX_PDNB_OVERRIDE_EN_11__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_11__TX_PDNB_OVERRIDE_VAL_11_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_11__TX_PDNB_OVERRIDE_VAL_11__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_11__TXPWR_OVERRIDE_EN_11_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_11__TXPWR_OVERRIDE_EN_11__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_11__TXPWR_OVERRIDE_VAL_11_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_11__TXPWR_OVERRIDE_VAL_11__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_12__RXEN_OVERRIDE_EN_12_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_12__RXEN_OVERRIDE_EN_12__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_12__RXEN_OVERRIDE_VAL_12_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_12__RXEN_OVERRIDE_VAL_12__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_12__RX_PDNB_OVERRIDE_EN_12_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_12__RX_PDNB_OVERRIDE_EN_12__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_12__RX_PDNB_OVERRIDE_VAL_12_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_12__RX_PDNB_OVERRIDE_VAL_12__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_12__RXPWR_OVERRIDE_EN_12_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_12__RXPWR_OVERRIDE_EN_12__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_12__RXPWR_OVERRIDE_VAL_12_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_12__RXPWR_OVERRIDE_VAL_12__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_12__TX_PDNB_OVERRIDE_EN_12_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_12__TX_PDNB_OVERRIDE_EN_12__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_12__TX_PDNB_OVERRIDE_VAL_12_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_12__TX_PDNB_OVERRIDE_VAL_12__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_12__TXPWR_OVERRIDE_EN_12_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_12__TXPWR_OVERRIDE_EN_12__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_12__TXPWR_OVERRIDE_VAL_12_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_12__TXPWR_OVERRIDE_VAL_12__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_13__RXEN_OVERRIDE_EN_13_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_13__RXEN_OVERRIDE_EN_13__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_13__RXEN_OVERRIDE_VAL_13_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_13__RXEN_OVERRIDE_VAL_13__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_13__RX_PDNB_OVERRIDE_EN_13_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_13__RX_PDNB_OVERRIDE_EN_13__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_13__RX_PDNB_OVERRIDE_VAL_13_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_13__RX_PDNB_OVERRIDE_VAL_13__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_13__RXPWR_OVERRIDE_EN_13_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_13__RXPWR_OVERRIDE_EN_13__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_13__RXPWR_OVERRIDE_VAL_13_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_13__RXPWR_OVERRIDE_VAL_13__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_13__TX_PDNB_OVERRIDE_EN_13_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_13__TX_PDNB_OVERRIDE_EN_13__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_13__TX_PDNB_OVERRIDE_VAL_13_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_13__TX_PDNB_OVERRIDE_VAL_13__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_13__TXPWR_OVERRIDE_EN_13_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_13__TXPWR_OVERRIDE_EN_13__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_13__TXPWR_OVERRIDE_VAL_13_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_13__TXPWR_OVERRIDE_VAL_13__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_14__RXEN_OVERRIDE_EN_14_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_14__RXEN_OVERRIDE_EN_14__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_14__RXEN_OVERRIDE_VAL_14_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_14__RXEN_OVERRIDE_VAL_14__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_14__RX_PDNB_OVERRIDE_EN_14_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_14__RX_PDNB_OVERRIDE_EN_14__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_14__RX_PDNB_OVERRIDE_VAL_14_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_14__RX_PDNB_OVERRIDE_VAL_14__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_14__RXPWR_OVERRIDE_EN_14_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_14__RXPWR_OVERRIDE_EN_14__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_14__RXPWR_OVERRIDE_VAL_14_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_14__RXPWR_OVERRIDE_VAL_14__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_14__TX_PDNB_OVERRIDE_EN_14_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_14__TX_PDNB_OVERRIDE_EN_14__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_14__TX_PDNB_OVERRIDE_VAL_14_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_14__TX_PDNB_OVERRIDE_VAL_14__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_14__TXPWR_OVERRIDE_EN_14_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_14__TXPWR_OVERRIDE_EN_14__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_14__TXPWR_OVERRIDE_VAL_14_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_14__TXPWR_OVERRIDE_VAL_14__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_15__RXEN_OVERRIDE_EN_15_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_15__RXEN_OVERRIDE_EN_15__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_15__RXEN_OVERRIDE_VAL_15_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_15__RXEN_OVERRIDE_VAL_15__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_15__RX_PDNB_OVERRIDE_EN_15_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_15__RX_PDNB_OVERRIDE_EN_15__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_15__RX_PDNB_OVERRIDE_VAL_15_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_15__RX_PDNB_OVERRIDE_VAL_15__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_15__RXPWR_OVERRIDE_EN_15_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_15__RXPWR_OVERRIDE_EN_15__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_15__RXPWR_OVERRIDE_VAL_15_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_15__RXPWR_OVERRIDE_VAL_15__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_15__TX_PDNB_OVERRIDE_EN_15_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_15__TX_PDNB_OVERRIDE_EN_15__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_15__TX_PDNB_OVERRIDE_VAL_15_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_15__TX_PDNB_OVERRIDE_VAL_15__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_15__TXPWR_OVERRIDE_EN_15_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_15__TXPWR_OVERRIDE_EN_15__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_15__TXPWR_OVERRIDE_VAL_15_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_15__TXPWR_OVERRIDE_VAL_15__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_1__RXEN_OVERRIDE_EN_1_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_1__RXEN_OVERRIDE_EN_1__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_1__RXEN_OVERRIDE_VAL_1_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_1__RXEN_OVERRIDE_VAL_1__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_1__RX_PDNB_OVERRIDE_EN_1_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_1__RX_PDNB_OVERRIDE_EN_1__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_1__RX_PDNB_OVERRIDE_VAL_1_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_1__RX_PDNB_OVERRIDE_VAL_1__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_1__RXPWR_OVERRIDE_EN_1_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_1__RXPWR_OVERRIDE_EN_1__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_1__RXPWR_OVERRIDE_VAL_1_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_1__RXPWR_OVERRIDE_VAL_1__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_1__TX_PDNB_OVERRIDE_EN_1_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_1__TX_PDNB_OVERRIDE_EN_1__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_1__TX_PDNB_OVERRIDE_VAL_1_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_1__TX_PDNB_OVERRIDE_VAL_1__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_1__TXPWR_OVERRIDE_EN_1_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_1__TXPWR_OVERRIDE_EN_1__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_1__TXPWR_OVERRIDE_VAL_1_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_1__TXPWR_OVERRIDE_VAL_1__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_2__RXEN_OVERRIDE_EN_2_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_2__RXEN_OVERRIDE_EN_2__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_2__RXEN_OVERRIDE_VAL_2_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_2__RXEN_OVERRIDE_VAL_2__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_2__RX_PDNB_OVERRIDE_EN_2_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_2__RX_PDNB_OVERRIDE_EN_2__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_2__RX_PDNB_OVERRIDE_VAL_2_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_2__RX_PDNB_OVERRIDE_VAL_2__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_2__RXPWR_OVERRIDE_EN_2_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_2__RXPWR_OVERRIDE_EN_2__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_2__RXPWR_OVERRIDE_VAL_2_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_2__RXPWR_OVERRIDE_VAL_2__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_2__TX_PDNB_OVERRIDE_EN_2_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_2__TX_PDNB_OVERRIDE_EN_2__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_2__TX_PDNB_OVERRIDE_VAL_2_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_2__TX_PDNB_OVERRIDE_VAL_2__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_2__TXPWR_OVERRIDE_EN_2_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_2__TXPWR_OVERRIDE_EN_2__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_2__TXPWR_OVERRIDE_VAL_2_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_2__TXPWR_OVERRIDE_VAL_2__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_3__RXEN_OVERRIDE_EN_3_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_3__RXEN_OVERRIDE_EN_3__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_3__RXEN_OVERRIDE_VAL_3_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_3__RXEN_OVERRIDE_VAL_3__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_3__RX_PDNB_OVERRIDE_EN_3_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_3__RX_PDNB_OVERRIDE_EN_3__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_3__RX_PDNB_OVERRIDE_VAL_3_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_3__RX_PDNB_OVERRIDE_VAL_3__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_3__RXPWR_OVERRIDE_EN_3_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_3__RXPWR_OVERRIDE_EN_3__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_3__RXPWR_OVERRIDE_VAL_3_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_3__RXPWR_OVERRIDE_VAL_3__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_3__TX_PDNB_OVERRIDE_EN_3_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_3__TX_PDNB_OVERRIDE_EN_3__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_3__TX_PDNB_OVERRIDE_VAL_3_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_3__TX_PDNB_OVERRIDE_VAL_3__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_3__TXPWR_OVERRIDE_EN_3_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_3__TXPWR_OVERRIDE_EN_3__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_3__TXPWR_OVERRIDE_VAL_3_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_3__TXPWR_OVERRIDE_VAL_3__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_4__RXEN_OVERRIDE_EN_4_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_4__RXEN_OVERRIDE_EN_4__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_4__RXEN_OVERRIDE_VAL_4_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_4__RXEN_OVERRIDE_VAL_4__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_4__RX_PDNB_OVERRIDE_EN_4_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_4__RX_PDNB_OVERRIDE_EN_4__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_4__RX_PDNB_OVERRIDE_VAL_4_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_4__RX_PDNB_OVERRIDE_VAL_4__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_4__RXPWR_OVERRIDE_EN_4_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_4__RXPWR_OVERRIDE_EN_4__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_4__RXPWR_OVERRIDE_VAL_4_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_4__RXPWR_OVERRIDE_VAL_4__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_4__TX_PDNB_OVERRIDE_EN_4_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_4__TX_PDNB_OVERRIDE_EN_4__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_4__TX_PDNB_OVERRIDE_VAL_4_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_4__TX_PDNB_OVERRIDE_VAL_4__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_4__TXPWR_OVERRIDE_EN_4_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_4__TXPWR_OVERRIDE_EN_4__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_4__TXPWR_OVERRIDE_VAL_4_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_4__TXPWR_OVERRIDE_VAL_4__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_5__RXEN_OVERRIDE_EN_5_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_5__RXEN_OVERRIDE_EN_5__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_5__RXEN_OVERRIDE_VAL_5_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_5__RXEN_OVERRIDE_VAL_5__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_5__RX_PDNB_OVERRIDE_EN_5_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_5__RX_PDNB_OVERRIDE_EN_5__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_5__RX_PDNB_OVERRIDE_VAL_5_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_5__RX_PDNB_OVERRIDE_VAL_5__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_5__RXPWR_OVERRIDE_EN_5_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_5__RXPWR_OVERRIDE_EN_5__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_5__RXPWR_OVERRIDE_VAL_5_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_5__RXPWR_OVERRIDE_VAL_5__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_5__TX_PDNB_OVERRIDE_EN_5_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_5__TX_PDNB_OVERRIDE_EN_5__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_5__TX_PDNB_OVERRIDE_VAL_5_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_5__TX_PDNB_OVERRIDE_VAL_5__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_5__TXPWR_OVERRIDE_EN_5_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_5__TXPWR_OVERRIDE_EN_5__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_5__TXPWR_OVERRIDE_VAL_5_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_5__TXPWR_OVERRIDE_VAL_5__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_6__RXEN_OVERRIDE_EN_6_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_6__RXEN_OVERRIDE_EN_6__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_6__RXEN_OVERRIDE_VAL_6_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_6__RXEN_OVERRIDE_VAL_6__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_6__RX_PDNB_OVERRIDE_EN_6_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_6__RX_PDNB_OVERRIDE_EN_6__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_6__RX_PDNB_OVERRIDE_VAL_6_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_6__RX_PDNB_OVERRIDE_VAL_6__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_6__RXPWR_OVERRIDE_EN_6_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_6__RXPWR_OVERRIDE_EN_6__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_6__RXPWR_OVERRIDE_VAL_6_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_6__RXPWR_OVERRIDE_VAL_6__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_6__TX_PDNB_OVERRIDE_EN_6_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_6__TX_PDNB_OVERRIDE_EN_6__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_6__TX_PDNB_OVERRIDE_VAL_6_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_6__TX_PDNB_OVERRIDE_VAL_6__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_6__TXPWR_OVERRIDE_EN_6_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_6__TXPWR_OVERRIDE_EN_6__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_6__TXPWR_OVERRIDE_VAL_6_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_6__TXPWR_OVERRIDE_VAL_6__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_7__RXEN_OVERRIDE_EN_7_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_7__RXEN_OVERRIDE_EN_7__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_7__RXEN_OVERRIDE_VAL_7_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_7__RXEN_OVERRIDE_VAL_7__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_7__RX_PDNB_OVERRIDE_EN_7_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_7__RX_PDNB_OVERRIDE_EN_7__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_7__RX_PDNB_OVERRIDE_VAL_7_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_7__RX_PDNB_OVERRIDE_VAL_7__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_7__RXPWR_OVERRIDE_EN_7_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_7__RXPWR_OVERRIDE_EN_7__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_7__RXPWR_OVERRIDE_VAL_7_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_7__RXPWR_OVERRIDE_VAL_7__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_7__TX_PDNB_OVERRIDE_EN_7_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_7__TX_PDNB_OVERRIDE_EN_7__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_7__TX_PDNB_OVERRIDE_VAL_7_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_7__TX_PDNB_OVERRIDE_VAL_7__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_7__TXPWR_OVERRIDE_EN_7_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_7__TXPWR_OVERRIDE_EN_7__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_7__TXPWR_OVERRIDE_VAL_7_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_7__TXPWR_OVERRIDE_VAL_7__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_8__RXEN_OVERRIDE_EN_8_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_8__RXEN_OVERRIDE_EN_8__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_8__RXEN_OVERRIDE_VAL_8_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_8__RXEN_OVERRIDE_VAL_8__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_8__RX_PDNB_OVERRIDE_EN_8_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_8__RX_PDNB_OVERRIDE_EN_8__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_8__RX_PDNB_OVERRIDE_VAL_8_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_8__RX_PDNB_OVERRIDE_VAL_8__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_8__RXPWR_OVERRIDE_EN_8_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_8__RXPWR_OVERRIDE_EN_8__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_8__RXPWR_OVERRIDE_VAL_8_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_8__RXPWR_OVERRIDE_VAL_8__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_8__TX_PDNB_OVERRIDE_EN_8_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_8__TX_PDNB_OVERRIDE_EN_8__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_8__TX_PDNB_OVERRIDE_VAL_8_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_8__TX_PDNB_OVERRIDE_VAL_8__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_8__TXPWR_OVERRIDE_EN_8_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_8__TXPWR_OVERRIDE_EN_8__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_8__TXPWR_OVERRIDE_VAL_8_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_8__TXPWR_OVERRIDE_VAL_8__SHIFT 0x0000000b
+#define PB1_PIF_PDNB_OVERRIDE_9__RXEN_OVERRIDE_EN_9_MASK 0x00000100L
+#define PB1_PIF_PDNB_OVERRIDE_9__RXEN_OVERRIDE_EN_9__SHIFT 0x00000008
+#define PB1_PIF_PDNB_OVERRIDE_9__RXEN_OVERRIDE_VAL_9_MASK 0x00000200L
+#define PB1_PIF_PDNB_OVERRIDE_9__RXEN_OVERRIDE_VAL_9__SHIFT 0x00000009
+#define PB1_PIF_PDNB_OVERRIDE_9__RX_PDNB_OVERRIDE_EN_9_MASK 0x00000010L
+#define PB1_PIF_PDNB_OVERRIDE_9__RX_PDNB_OVERRIDE_EN_9__SHIFT 0x00000004
+#define PB1_PIF_PDNB_OVERRIDE_9__RX_PDNB_OVERRIDE_VAL_9_MASK 0x000000e0L
+#define PB1_PIF_PDNB_OVERRIDE_9__RX_PDNB_OVERRIDE_VAL_9__SHIFT 0x00000005
+#define PB1_PIF_PDNB_OVERRIDE_9__RXPWR_OVERRIDE_EN_9_MASK 0x00004000L
+#define PB1_PIF_PDNB_OVERRIDE_9__RXPWR_OVERRIDE_EN_9__SHIFT 0x0000000e
+#define PB1_PIF_PDNB_OVERRIDE_9__RXPWR_OVERRIDE_VAL_9_MASK 0x00038000L
+#define PB1_PIF_PDNB_OVERRIDE_9__RXPWR_OVERRIDE_VAL_9__SHIFT 0x0000000f
+#define PB1_PIF_PDNB_OVERRIDE_9__TX_PDNB_OVERRIDE_EN_9_MASK 0x00000001L
+#define PB1_PIF_PDNB_OVERRIDE_9__TX_PDNB_OVERRIDE_EN_9__SHIFT 0x00000000
+#define PB1_PIF_PDNB_OVERRIDE_9__TX_PDNB_OVERRIDE_VAL_9_MASK 0x0000000eL
+#define PB1_PIF_PDNB_OVERRIDE_9__TX_PDNB_OVERRIDE_VAL_9__SHIFT 0x00000001
+#define PB1_PIF_PDNB_OVERRIDE_9__TXPWR_OVERRIDE_EN_9_MASK 0x00000400L
+#define PB1_PIF_PDNB_OVERRIDE_9__TXPWR_OVERRIDE_EN_9__SHIFT 0x0000000a
+#define PB1_PIF_PDNB_OVERRIDE_9__TXPWR_OVERRIDE_VAL_9_MASK 0x00003800L
+#define PB1_PIF_PDNB_OVERRIDE_9__TXPWR_OVERRIDE_VAL_9__SHIFT 0x0000000b
+#define PB1_PIF_PWRDOWN_0__FORCE_RXEN_IN_L0s_0_MASK 0x00000008L
+#define PB1_PIF_PWRDOWN_0__FORCE_RXEN_IN_L0s_0__SHIFT 0x00000003
+#define PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK 0x00001c00L
+#define PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT 0x0000000a
+#define PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK 0x00000380L
+#define PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT 0x00000007
+#define PB1_PIF_PWRDOWN_0__PLLPWR_OVERRIDE_EN_0_MASK 0x10000000L
+#define PB1_PIF_PWRDOWN_0__PLLPWR_OVERRIDE_EN_0__SHIFT 0x0000001c
+#define PB1_PIF_PWRDOWN_0__PLLPWR_OVERRIDE_VAL_0_MASK 0xe0000000L
+#define PB1_PIF_PWRDOWN_0__PLLPWR_OVERRIDE_VAL_0__SHIFT 0x0000001d
+#define PB1_PIF_PWRDOWN_0__PLL_RAMP_UP_TIME_0_MASK 0x07000000L
+#define PB1_PIF_PWRDOWN_0__PLL_RAMP_UP_TIME_0__SHIFT 0x00000018
+#define PB1_PIF_PWRDOWN_0__RX_POWER_STATE_IN_RXS2_0_MASK 0x00000070L
+#define PB1_PIF_PWRDOWN_0__RX_POWER_STATE_IN_RXS2_0__SHIFT 0x00000004
+#define PB1_PIF_PWRDOWN_0__TX2P5CLK_CLOCK_GATING_EN_0_MASK 0x00010000L
+#define PB1_PIF_PWRDOWN_0__TX2P5CLK_CLOCK_GATING_EN_0__SHIFT 0x00000010
+#define PB1_PIF_PWRDOWN_0__TX_POWER_STATE_IN_TXS2_0_MASK 0x00000007L
+#define PB1_PIF_PWRDOWN_0__TX_POWER_STATE_IN_TXS2_0__SHIFT 0x00000000
+#define PB1_PIF_PWRDOWN_1__FORCE_RXEN_IN_L0s_1_MASK 0x00000008L
+#define PB1_PIF_PWRDOWN_1__FORCE_RXEN_IN_L0s_1__SHIFT 0x00000003
+#define PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK 0x00001c00L
+#define PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT 0x0000000a
+#define PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK 0x00000380L
+#define PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT 0x00000007
+#define PB1_PIF_PWRDOWN_1__PLLPWR_OVERRIDE_EN_1_MASK 0x10000000L
+#define PB1_PIF_PWRDOWN_1__PLLPWR_OVERRIDE_EN_1__SHIFT 0x0000001c
+#define PB1_PIF_PWRDOWN_1__PLLPWR_OVERRIDE_VAL_1_MASK 0xe0000000L
+#define PB1_PIF_PWRDOWN_1__PLLPWR_OVERRIDE_VAL_1__SHIFT 0x0000001d
+#define PB1_PIF_PWRDOWN_1__PLL_RAMP_UP_TIME_1_MASK 0x07000000L
+#define PB1_PIF_PWRDOWN_1__PLL_RAMP_UP_TIME_1__SHIFT 0x00000018
+#define PB1_PIF_PWRDOWN_1__RX_POWER_STATE_IN_RXS2_1_MASK 0x00000070L
+#define PB1_PIF_PWRDOWN_1__RX_POWER_STATE_IN_RXS2_1__SHIFT 0x00000004
+#define PB1_PIF_PWRDOWN_1__TX2P5CLK_CLOCK_GATING_EN_1_MASK 0x00010000L
+#define PB1_PIF_PWRDOWN_1__TX2P5CLK_CLOCK_GATING_EN_1__SHIFT 0x00000010
+#define PB1_PIF_PWRDOWN_1__TX_POWER_STATE_IN_TXS2_1_MASK 0x00000007L
+#define PB1_PIF_PWRDOWN_1__TX_POWER_STATE_IN_TXS2_1__SHIFT 0x00000000
+#define PB1_PIF_PWRDOWN_2__FORCE_RXEN_IN_L0s_2_MASK 0x00000008L
+#define PB1_PIF_PWRDOWN_2__FORCE_RXEN_IN_L0s_2__SHIFT 0x00000003
+#define PB1_PIF_PWRDOWN_2__PLL_POWER_STATE_IN_OFF_2_MASK 0x00001c00L
+#define PB1_PIF_PWRDOWN_2__PLL_POWER_STATE_IN_OFF_2__SHIFT 0x0000000a
+#define PB1_PIF_PWRDOWN_2__PLL_POWER_STATE_IN_TXS2_2_MASK 0x00000380L
+#define PB1_PIF_PWRDOWN_2__PLL_POWER_STATE_IN_TXS2_2__SHIFT 0x00000007
+#define PB1_PIF_PWRDOWN_2__PLLPWR_OVERRIDE_EN_2_MASK 0x10000000L
+#define PB1_PIF_PWRDOWN_2__PLLPWR_OVERRIDE_EN_2__SHIFT 0x0000001c
+#define PB1_PIF_PWRDOWN_2__PLLPWR_OVERRIDE_VAL_2_MASK 0xe0000000L
+#define PB1_PIF_PWRDOWN_2__PLLPWR_OVERRIDE_VAL_2__SHIFT 0x0000001d
+#define PB1_PIF_PWRDOWN_2__PLL_RAMP_UP_TIME_2_MASK 0x07000000L
+#define PB1_PIF_PWRDOWN_2__PLL_RAMP_UP_TIME_2__SHIFT 0x00000018
+#define PB1_PIF_PWRDOWN_2__RX_POWER_STATE_IN_RXS2_2_MASK 0x00000070L
+#define PB1_PIF_PWRDOWN_2__RX_POWER_STATE_IN_RXS2_2__SHIFT 0x00000004
+#define PB1_PIF_PWRDOWN_2__TX2P5CLK_CLOCK_GATING_EN_2_MASK 0x00010000L
+#define PB1_PIF_PWRDOWN_2__TX2P5CLK_CLOCK_GATING_EN_2__SHIFT 0x00000010
+#define PB1_PIF_PWRDOWN_2__TX_POWER_STATE_IN_TXS2_2_MASK 0x00000007L
+#define PB1_PIF_PWRDOWN_2__TX_POWER_STATE_IN_TXS2_2__SHIFT 0x00000000
+#define PB1_PIF_PWRDOWN_3__FORCE_RXEN_IN_L0s_3_MASK 0x00000008L
+#define PB1_PIF_PWRDOWN_3__FORCE_RXEN_IN_L0s_3__SHIFT 0x00000003
+#define PB1_PIF_PWRDOWN_3__PLL_POWER_STATE_IN_OFF_3_MASK 0x00001c00L
+#define PB1_PIF_PWRDOWN_3__PLL_POWER_STATE_IN_OFF_3__SHIFT 0x0000000a
+#define PB1_PIF_PWRDOWN_3__PLL_POWER_STATE_IN_TXS2_3_MASK 0x00000380L
+#define PB1_PIF_PWRDOWN_3__PLL_POWER_STATE_IN_TXS2_3__SHIFT 0x00000007
+#define PB1_PIF_PWRDOWN_3__PLLPWR_OVERRIDE_EN_3_MASK 0x10000000L
+#define PB1_PIF_PWRDOWN_3__PLLPWR_OVERRIDE_EN_3__SHIFT 0x0000001c
+#define PB1_PIF_PWRDOWN_3__PLLPWR_OVERRIDE_VAL_3_MASK 0xe0000000L
+#define PB1_PIF_PWRDOWN_3__PLLPWR_OVERRIDE_VAL_3__SHIFT 0x0000001d
+#define PB1_PIF_PWRDOWN_3__PLL_RAMP_UP_TIME_3_MASK 0x07000000L
+#define PB1_PIF_PWRDOWN_3__PLL_RAMP_UP_TIME_3__SHIFT 0x00000018
+#define PB1_PIF_PWRDOWN_3__RX_POWER_STATE_IN_RXS2_3_MASK 0x00000070L
+#define PB1_PIF_PWRDOWN_3__RX_POWER_STATE_IN_RXS2_3__SHIFT 0x00000004
+#define PB1_PIF_PWRDOWN_3__TX2P5CLK_CLOCK_GATING_EN_3_MASK 0x00010000L
+#define PB1_PIF_PWRDOWN_3__TX2P5CLK_CLOCK_GATING_EN_3__SHIFT 0x00000010
+#define PB1_PIF_PWRDOWN_3__TX_POWER_STATE_IN_TXS2_3_MASK 0x00000007L
+#define PB1_PIF_PWRDOWN_3__TX_POWER_STATE_IN_TXS2_3__SHIFT 0x00000000
+#define PB1_PIF_SC_CTL__SC_CALIBRATION_MASK 0x00000001L
+#define PB1_PIF_SC_CTL__SC_CALIBRATION__SHIFT 0x00000000
+#define PB1_PIF_SC_CTL__SC_ENTER_L1_FROM_L0_MASK 0x00000020L
+#define PB1_PIF_SC_CTL__SC_ENTER_L1_FROM_L0__SHIFT 0x00000005
+#define PB1_PIF_SC_CTL__SC_ENTER_L1_FROM_L0S_MASK 0x00000010L
+#define PB1_PIF_SC_CTL__SC_ENTER_L1_FROM_L0S__SHIFT 0x00000004
+#define PB1_PIF_SC_CTL__SC_EXIT_L1_TO_L0_MASK 0x00000008L
+#define PB1_PIF_SC_CTL__SC_EXIT_L1_TO_L0__SHIFT 0x00000003
+#define PB1_PIF_SC_CTL__SC_EXIT_L1_TO_L0S_MASK 0x00000004L
+#define PB1_PIF_SC_CTL__SC_EXIT_L1_TO_L0S__SHIFT 0x00000002
+#define PB1_PIF_SC_CTL__SC_LANE_0_RESUME_MASK 0x00010000L
+#define PB1_PIF_SC_CTL__SC_LANE_0_RESUME__SHIFT 0x00000010
+#define PB1_PIF_SC_CTL__SC_LANE_10_RESUME_MASK 0x04000000L
+#define PB1_PIF_SC_CTL__SC_LANE_10_RESUME__SHIFT 0x0000001a
+#define PB1_PIF_SC_CTL__SC_LANE_11_RESUME_MASK 0x08000000L
+#define PB1_PIF_SC_CTL__SC_LANE_11_RESUME__SHIFT 0x0000001b
+#define PB1_PIF_SC_CTL__SC_LANE_12_RESUME_MASK 0x10000000L
+#define PB1_PIF_SC_CTL__SC_LANE_12_RESUME__SHIFT 0x0000001c
+#define PB1_PIF_SC_CTL__SC_LANE_13_RESUME_MASK 0x20000000L
+#define PB1_PIF_SC_CTL__SC_LANE_13_RESUME__SHIFT 0x0000001d
+#define PB1_PIF_SC_CTL__SC_LANE_14_RESUME_MASK 0x40000000L
+#define PB1_PIF_SC_CTL__SC_LANE_14_RESUME__SHIFT 0x0000001e
+#define PB1_PIF_SC_CTL__SC_LANE_15_RESUME_MASK 0x80000000L
+#define PB1_PIF_SC_CTL__SC_LANE_15_RESUME__SHIFT 0x0000001f
+#define PB1_PIF_SC_CTL__SC_LANE_1_RESUME_MASK 0x00020000L
+#define PB1_PIF_SC_CTL__SC_LANE_1_RESUME__SHIFT 0x00000011
+#define PB1_PIF_SC_CTL__SC_LANE_2_RESUME_MASK 0x00040000L
+#define PB1_PIF_SC_CTL__SC_LANE_2_RESUME__SHIFT 0x00000012
+#define PB1_PIF_SC_CTL__SC_LANE_3_RESUME_MASK 0x00080000L
+#define PB1_PIF_SC_CTL__SC_LANE_3_RESUME__SHIFT 0x00000013
+#define PB1_PIF_SC_CTL__SC_LANE_4_RESUME_MASK 0x00100000L
+#define PB1_PIF_SC_CTL__SC_LANE_4_RESUME__SHIFT 0x00000014
+#define PB1_PIF_SC_CTL__SC_LANE_5_RESUME_MASK 0x00200000L
+#define PB1_PIF_SC_CTL__SC_LANE_5_RESUME__SHIFT 0x00000015
+#define PB1_PIF_SC_CTL__SC_LANE_6_RESUME_MASK 0x00400000L
+#define PB1_PIF_SC_CTL__SC_LANE_6_RESUME__SHIFT 0x00000016
+#define PB1_PIF_SC_CTL__SC_LANE_7_RESUME_MASK 0x00800000L
+#define PB1_PIF_SC_CTL__SC_LANE_7_RESUME__SHIFT 0x00000017
+#define PB1_PIF_SC_CTL__SC_LANE_8_RESUME_MASK 0x01000000L
+#define PB1_PIF_SC_CTL__SC_LANE_8_RESUME__SHIFT 0x00000018
+#define PB1_PIF_SC_CTL__SC_LANE_9_RESUME_MASK 0x02000000L
+#define PB1_PIF_SC_CTL__SC_LANE_9_RESUME__SHIFT 0x00000019
+#define PB1_PIF_SC_CTL__SC_PHASE_1_MASK 0x00000100L
+#define PB1_PIF_SC_CTL__SC_PHASE_1__SHIFT 0x00000008
+#define PB1_PIF_SC_CTL__SC_PHASE_2_MASK 0x00000200L
+#define PB1_PIF_SC_CTL__SC_PHASE_2__SHIFT 0x00000009
+#define PB1_PIF_SC_CTL__SC_PHASE_3_MASK 0x00000400L
+#define PB1_PIF_SC_CTL__SC_PHASE_3__SHIFT 0x0000000a
+#define PB1_PIF_SC_CTL__SC_PHASE_4_MASK 0x00000800L
+#define PB1_PIF_SC_CTL__SC_PHASE_4__SHIFT 0x0000000b
+#define PB1_PIF_SC_CTL__SC_PHASE_5_MASK 0x00001000L
+#define PB1_PIF_SC_CTL__SC_PHASE_5__SHIFT 0x0000000c
+#define PB1_PIF_SC_CTL__SC_PHASE_6_MASK 0x00002000L
+#define PB1_PIF_SC_CTL__SC_PHASE_6__SHIFT 0x0000000d
+#define PB1_PIF_SC_CTL__SC_PHASE_7_MASK 0x00004000L
+#define PB1_PIF_SC_CTL__SC_PHASE_7__SHIFT 0x0000000e
+#define PB1_PIF_SC_CTL__SC_PHASE_8_MASK 0x00008000L
+#define PB1_PIF_SC_CTL__SC_PHASE_8__SHIFT 0x0000000f
+#define PB1_PIF_SC_CTL__SC_RXDETECT_MASK 0x00000002L
+#define PB1_PIF_SC_CTL__SC_RXDETECT__SHIFT 0x00000001
+#define PB1_PIF_SC_CTL__SC_SPEED_CHANGE_MASK 0x00000040L
+#define PB1_PIF_SC_CTL__SC_SPEED_CHANGE__SHIFT 0x00000006
+#define PB1_PIF_SCRATCH__PIF_SCRATCH_MASK 0xffffffffL
+#define PB1_PIF_SCRATCH__PIF_SCRATCH__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_0__SEQ_CALIBRATION_0_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_0__SEQ_CALIBRATION_0__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_0__SEQ_ENTER_L1_FROM_L0_0_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_0__SEQ_ENTER_L1_FROM_L0_0__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_0__SEQ_ENTER_L1_FROM_L0S_0_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_0__SEQ_ENTER_L1_FROM_L0S_0__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_0__SEQ_EXIT_L1_TO_L0_0_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_0__SEQ_EXIT_L1_TO_L0_0__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_0__SEQ_EXIT_L1_TO_L0S_0_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_0__SEQ_EXIT_L1_TO_L0S_0__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_0__SEQ_PHASE_0_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_0__SEQ_PHASE_0__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_0__SEQ_RXDETECT_0_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_0__SEQ_RXDETECT_0__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_0__SEQ_SPEED_CHANGE_0_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_0__SEQ_SPEED_CHANGE_0__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_10__SEQ_CALIBRATION_10_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_10__SEQ_CALIBRATION_10__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_10__SEQ_ENTER_L1_FROM_L0_10_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_10__SEQ_ENTER_L1_FROM_L0_10__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_10__SEQ_ENTER_L1_FROM_L0S_10_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_10__SEQ_ENTER_L1_FROM_L0S_10__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_10__SEQ_EXIT_L1_TO_L0_10_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_10__SEQ_EXIT_L1_TO_L0_10__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_10__SEQ_EXIT_L1_TO_L0S_10_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_10__SEQ_EXIT_L1_TO_L0S_10__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_10__SEQ_PHASE_10_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_10__SEQ_PHASE_10__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_10__SEQ_RXDETECT_10_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_10__SEQ_RXDETECT_10__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_10__SEQ_SPEED_CHANGE_10_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_10__SEQ_SPEED_CHANGE_10__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_11__SEQ_CALIBRATION_11_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_11__SEQ_CALIBRATION_11__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_11__SEQ_ENTER_L1_FROM_L0_11_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_11__SEQ_ENTER_L1_FROM_L0_11__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_11__SEQ_ENTER_L1_FROM_L0S_11_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_11__SEQ_ENTER_L1_FROM_L0S_11__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_11__SEQ_EXIT_L1_TO_L0_11_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_11__SEQ_EXIT_L1_TO_L0_11__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_11__SEQ_EXIT_L1_TO_L0S_11_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_11__SEQ_EXIT_L1_TO_L0S_11__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_11__SEQ_PHASE_11_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_11__SEQ_PHASE_11__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_11__SEQ_RXDETECT_11_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_11__SEQ_RXDETECT_11__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_11__SEQ_SPEED_CHANGE_11_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_11__SEQ_SPEED_CHANGE_11__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_12__SEQ_CALIBRATION_12_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_12__SEQ_CALIBRATION_12__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_12__SEQ_ENTER_L1_FROM_L0_12_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_12__SEQ_ENTER_L1_FROM_L0_12__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_12__SEQ_ENTER_L1_FROM_L0S_12_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_12__SEQ_ENTER_L1_FROM_L0S_12__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_12__SEQ_EXIT_L1_TO_L0_12_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_12__SEQ_EXIT_L1_TO_L0_12__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_12__SEQ_EXIT_L1_TO_L0S_12_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_12__SEQ_EXIT_L1_TO_L0S_12__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_12__SEQ_PHASE_12_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_12__SEQ_PHASE_12__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_12__SEQ_RXDETECT_12_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_12__SEQ_RXDETECT_12__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_12__SEQ_SPEED_CHANGE_12_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_12__SEQ_SPEED_CHANGE_12__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_13__SEQ_CALIBRATION_13_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_13__SEQ_CALIBRATION_13__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_13__SEQ_ENTER_L1_FROM_L0_13_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_13__SEQ_ENTER_L1_FROM_L0_13__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_13__SEQ_ENTER_L1_FROM_L0S_13_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_13__SEQ_ENTER_L1_FROM_L0S_13__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_13__SEQ_EXIT_L1_TO_L0_13_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_13__SEQ_EXIT_L1_TO_L0_13__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_13__SEQ_EXIT_L1_TO_L0S_13_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_13__SEQ_EXIT_L1_TO_L0S_13__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_13__SEQ_PHASE_13_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_13__SEQ_PHASE_13__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_13__SEQ_RXDETECT_13_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_13__SEQ_RXDETECT_13__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_13__SEQ_SPEED_CHANGE_13_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_13__SEQ_SPEED_CHANGE_13__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_14__SEQ_CALIBRATION_14_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_14__SEQ_CALIBRATION_14__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_14__SEQ_ENTER_L1_FROM_L0_14_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_14__SEQ_ENTER_L1_FROM_L0_14__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_14__SEQ_ENTER_L1_FROM_L0S_14_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_14__SEQ_ENTER_L1_FROM_L0S_14__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_14__SEQ_EXIT_L1_TO_L0_14_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_14__SEQ_EXIT_L1_TO_L0_14__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_14__SEQ_EXIT_L1_TO_L0S_14_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_14__SEQ_EXIT_L1_TO_L0S_14__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_14__SEQ_PHASE_14_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_14__SEQ_PHASE_14__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_14__SEQ_RXDETECT_14_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_14__SEQ_RXDETECT_14__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_14__SEQ_SPEED_CHANGE_14_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_14__SEQ_SPEED_CHANGE_14__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_15__SEQ_CALIBRATION_15_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_15__SEQ_CALIBRATION_15__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_15__SEQ_ENTER_L1_FROM_L0_15_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_15__SEQ_ENTER_L1_FROM_L0_15__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_15__SEQ_ENTER_L1_FROM_L0S_15_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_15__SEQ_ENTER_L1_FROM_L0S_15__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_15__SEQ_EXIT_L1_TO_L0_15_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_15__SEQ_EXIT_L1_TO_L0_15__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_15__SEQ_EXIT_L1_TO_L0S_15_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_15__SEQ_EXIT_L1_TO_L0S_15__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_15__SEQ_PHASE_15_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_15__SEQ_PHASE_15__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_15__SEQ_RXDETECT_15_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_15__SEQ_RXDETECT_15__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_15__SEQ_SPEED_CHANGE_15_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_15__SEQ_SPEED_CHANGE_15__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_1__SEQ_CALIBRATION_1_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_1__SEQ_CALIBRATION_1__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_1__SEQ_ENTER_L1_FROM_L0_1_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_1__SEQ_ENTER_L1_FROM_L0_1__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_1__SEQ_ENTER_L1_FROM_L0S_1_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_1__SEQ_ENTER_L1_FROM_L0S_1__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_1__SEQ_EXIT_L1_TO_L0_1_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_1__SEQ_EXIT_L1_TO_L0_1__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_1__SEQ_EXIT_L1_TO_L0S_1_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_1__SEQ_EXIT_L1_TO_L0S_1__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_1__SEQ_PHASE_1_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_1__SEQ_PHASE_1__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_1__SEQ_RXDETECT_1_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_1__SEQ_RXDETECT_1__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_1__SEQ_SPEED_CHANGE_1_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_1__SEQ_SPEED_CHANGE_1__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_2__SEQ_CALIBRATION_2_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_2__SEQ_CALIBRATION_2__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_2__SEQ_ENTER_L1_FROM_L0_2_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_2__SEQ_ENTER_L1_FROM_L0_2__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_2__SEQ_ENTER_L1_FROM_L0S_2_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_2__SEQ_ENTER_L1_FROM_L0S_2__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_2__SEQ_EXIT_L1_TO_L0_2_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_2__SEQ_EXIT_L1_TO_L0_2__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_2__SEQ_EXIT_L1_TO_L0S_2_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_2__SEQ_EXIT_L1_TO_L0S_2__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_2__SEQ_PHASE_2_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_2__SEQ_PHASE_2__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_2__SEQ_RXDETECT_2_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_2__SEQ_RXDETECT_2__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_2__SEQ_SPEED_CHANGE_2_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_2__SEQ_SPEED_CHANGE_2__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_3__SEQ_CALIBRATION_3_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_3__SEQ_CALIBRATION_3__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_3__SEQ_ENTER_L1_FROM_L0_3_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_3__SEQ_ENTER_L1_FROM_L0_3__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_3__SEQ_ENTER_L1_FROM_L0S_3_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_3__SEQ_ENTER_L1_FROM_L0S_3__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_3__SEQ_EXIT_L1_TO_L0_3_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_3__SEQ_EXIT_L1_TO_L0_3__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_3__SEQ_EXIT_L1_TO_L0S_3_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_3__SEQ_EXIT_L1_TO_L0S_3__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_3__SEQ_PHASE_3_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_3__SEQ_PHASE_3__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_3__SEQ_RXDETECT_3_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_3__SEQ_RXDETECT_3__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_3__SEQ_SPEED_CHANGE_3_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_3__SEQ_SPEED_CHANGE_3__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_4__SEQ_CALIBRATION_4_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_4__SEQ_CALIBRATION_4__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_4__SEQ_ENTER_L1_FROM_L0_4_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_4__SEQ_ENTER_L1_FROM_L0_4__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_4__SEQ_ENTER_L1_FROM_L0S_4_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_4__SEQ_ENTER_L1_FROM_L0S_4__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_4__SEQ_EXIT_L1_TO_L0_4_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_4__SEQ_EXIT_L1_TO_L0_4__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_4__SEQ_EXIT_L1_TO_L0S_4_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_4__SEQ_EXIT_L1_TO_L0S_4__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_4__SEQ_PHASE_4_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_4__SEQ_PHASE_4__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_4__SEQ_RXDETECT_4_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_4__SEQ_RXDETECT_4__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_4__SEQ_SPEED_CHANGE_4_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_4__SEQ_SPEED_CHANGE_4__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_5__SEQ_CALIBRATION_5_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_5__SEQ_CALIBRATION_5__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_5__SEQ_ENTER_L1_FROM_L0_5_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_5__SEQ_ENTER_L1_FROM_L0_5__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_5__SEQ_ENTER_L1_FROM_L0S_5_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_5__SEQ_ENTER_L1_FROM_L0S_5__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_5__SEQ_EXIT_L1_TO_L0_5_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_5__SEQ_EXIT_L1_TO_L0_5__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_5__SEQ_EXIT_L1_TO_L0S_5_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_5__SEQ_EXIT_L1_TO_L0S_5__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_5__SEQ_PHASE_5_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_5__SEQ_PHASE_5__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_5__SEQ_RXDETECT_5_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_5__SEQ_RXDETECT_5__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_5__SEQ_SPEED_CHANGE_5_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_5__SEQ_SPEED_CHANGE_5__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_6__SEQ_CALIBRATION_6_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_6__SEQ_CALIBRATION_6__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_6__SEQ_ENTER_L1_FROM_L0_6_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_6__SEQ_ENTER_L1_FROM_L0_6__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_6__SEQ_ENTER_L1_FROM_L0S_6_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_6__SEQ_ENTER_L1_FROM_L0S_6__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_6__SEQ_EXIT_L1_TO_L0_6_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_6__SEQ_EXIT_L1_TO_L0_6__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_6__SEQ_EXIT_L1_TO_L0S_6_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_6__SEQ_EXIT_L1_TO_L0S_6__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_6__SEQ_PHASE_6_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_6__SEQ_PHASE_6__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_6__SEQ_RXDETECT_6_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_6__SEQ_RXDETECT_6__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_6__SEQ_SPEED_CHANGE_6_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_6__SEQ_SPEED_CHANGE_6__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_7__SEQ_CALIBRATION_7_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_7__SEQ_CALIBRATION_7__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_7__SEQ_ENTER_L1_FROM_L0_7_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_7__SEQ_ENTER_L1_FROM_L0_7__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_7__SEQ_ENTER_L1_FROM_L0S_7_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_7__SEQ_ENTER_L1_FROM_L0S_7__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_7__SEQ_EXIT_L1_TO_L0_7_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_7__SEQ_EXIT_L1_TO_L0_7__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_7__SEQ_EXIT_L1_TO_L0S_7_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_7__SEQ_EXIT_L1_TO_L0S_7__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_7__SEQ_PHASE_7_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_7__SEQ_PHASE_7__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_7__SEQ_RXDETECT_7_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_7__SEQ_RXDETECT_7__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_7__SEQ_SPEED_CHANGE_7_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_7__SEQ_SPEED_CHANGE_7__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_8__SEQ_CALIBRATION_8_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_8__SEQ_CALIBRATION_8__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_8__SEQ_ENTER_L1_FROM_L0_8_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_8__SEQ_ENTER_L1_FROM_L0_8__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_8__SEQ_ENTER_L1_FROM_L0S_8_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_8__SEQ_ENTER_L1_FROM_L0S_8__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_8__SEQ_EXIT_L1_TO_L0_8_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_8__SEQ_EXIT_L1_TO_L0_8__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_8__SEQ_EXIT_L1_TO_L0S_8_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_8__SEQ_EXIT_L1_TO_L0S_8__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_8__SEQ_PHASE_8_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_8__SEQ_PHASE_8__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_8__SEQ_RXDETECT_8_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_8__SEQ_RXDETECT_8__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_8__SEQ_SPEED_CHANGE_8_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_8__SEQ_SPEED_CHANGE_8__SHIFT 0x00000006
+#define PB1_PIF_SEQ_STATUS_9__SEQ_CALIBRATION_9_MASK 0x00000001L
+#define PB1_PIF_SEQ_STATUS_9__SEQ_CALIBRATION_9__SHIFT 0x00000000
+#define PB1_PIF_SEQ_STATUS_9__SEQ_ENTER_L1_FROM_L0_9_MASK 0x00000020L
+#define PB1_PIF_SEQ_STATUS_9__SEQ_ENTER_L1_FROM_L0_9__SHIFT 0x00000005
+#define PB1_PIF_SEQ_STATUS_9__SEQ_ENTER_L1_FROM_L0S_9_MASK 0x00000010L
+#define PB1_PIF_SEQ_STATUS_9__SEQ_ENTER_L1_FROM_L0S_9__SHIFT 0x00000004
+#define PB1_PIF_SEQ_STATUS_9__SEQ_EXIT_L1_TO_L0_9_MASK 0x00000008L
+#define PB1_PIF_SEQ_STATUS_9__SEQ_EXIT_L1_TO_L0_9__SHIFT 0x00000003
+#define PB1_PIF_SEQ_STATUS_9__SEQ_EXIT_L1_TO_L0S_9_MASK 0x00000004L
+#define PB1_PIF_SEQ_STATUS_9__SEQ_EXIT_L1_TO_L0S_9__SHIFT 0x00000002
+#define PB1_PIF_SEQ_STATUS_9__SEQ_PHASE_9_MASK 0x00000700L
+#define PB1_PIF_SEQ_STATUS_9__SEQ_PHASE_9__SHIFT 0x00000008
+#define PB1_PIF_SEQ_STATUS_9__SEQ_RXDETECT_9_MASK 0x00000002L
+#define PB1_PIF_SEQ_STATUS_9__SEQ_RXDETECT_9__SHIFT 0x00000001
+#define PB1_PIF_SEQ_STATUS_9__SEQ_SPEED_CHANGE_9_MASK 0x00000040L
+#define PB1_PIF_SEQ_STATUS_9__SEQ_SPEED_CHANGE_9__SHIFT 0x00000006
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_0_MASK 0x00000001L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_0__SHIFT 0x00000000
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_10_MASK 0x00000400L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_10__SHIFT 0x0000000a
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_11_MASK 0x00000800L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_11__SHIFT 0x0000000b
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_12_MASK 0x00001000L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_12__SHIFT 0x0000000c
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_13_MASK 0x00002000L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_13__SHIFT 0x0000000d
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_14_MASK 0x00004000L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_14__SHIFT 0x0000000e
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_15_MASK 0x00008000L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_15__SHIFT 0x0000000f
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_1_MASK 0x00000002L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_1__SHIFT 0x00000001
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_2_MASK 0x00000004L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_2__SHIFT 0x00000002
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_3_MASK 0x00000008L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_3__SHIFT 0x00000003
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_4_MASK 0x00000010L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_4__SHIFT 0x00000004
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_5_MASK 0x00000020L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_5__SHIFT 0x00000005
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_6_MASK 0x00000040L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_6__SHIFT 0x00000006
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_7_MASK 0x00000080L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_7__SHIFT 0x00000007
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_8_MASK 0x00000100L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_8__SHIFT 0x00000008
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_9_MASK 0x00000200L
+#define PB1_PIF_TXPHYSTATUS__TXPHYSTATUS_9__SHIFT 0x00000009
+#define PB1_PLL_LC0_CTRL_REG0__PLL_DBG_LC_ANALOG_SEL_0_MASK 0x00000003L
+#define PB1_PLL_LC0_CTRL_REG0__PLL_DBG_LC_ANALOG_SEL_0__SHIFT 0x00000000
+#define PB1_PLL_LC0_CTRL_REG0__PLL_DBG_LC_EXT_RESET_EN_0_MASK 0x00000004L
+#define PB1_PLL_LC0_CTRL_REG0__PLL_DBG_LC_EXT_RESET_EN_0__SHIFT 0x00000002
+#define PB1_PLL_LC0_CTRL_REG0__PLL_DBG_LC_VCTL_ADC_EN_0_MASK 0x00000008L
+#define PB1_PLL_LC0_CTRL_REG0__PLL_DBG_LC_VCTL_ADC_EN_0__SHIFT 0x00000003
+#define PB1_PLL_LC0_CTRL_REG0__PLL_TST_LC_USAMPLE_EN_0_MASK 0x00000010L
+#define PB1_PLL_LC0_CTRL_REG0__PLL_TST_LC_USAMPLE_EN_0__SHIFT 0x00000004
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_BW_CNTRL_OVRD_EN_0_MASK 0x00000008L
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_BW_CNTRL_OVRD_EN_0__SHIFT 0x00000003
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_BW_CNTRL_OVRD_VAL_0_MASK 0x00000007L
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_BW_CNTRL_OVRD_VAL_0__SHIFT 0x00000000
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_DIV_OVRD_EN_0_MASK 0x00000080L
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_DIV_OVRD_EN_0__SHIFT 0x00000007
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_DIV_OVRD_VAL_0_MASK 0x00000070L
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_DIV_OVRD_VAL_0__SHIFT 0x00000004
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_EN_OVRD_EN_0_MASK 0x00000200L
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_EN_OVRD_EN_0__SHIFT 0x00000009
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_EN_OVRD_VAL_0_MASK 0x00000100L
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_CORECLK_EN_OVRD_VAL_0__SHIFT 0x00000008
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_FBDIV_OVRD_EN_0_MASK 0x00040000L
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_FBDIV_OVRD_EN_0__SHIFT 0x00000012
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_FBDIV_OVRD_VAL_0_MASK 0x0003fc00L
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_FBDIV_OVRD_VAL_0__SHIFT 0x0000000a
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_LF_CNTRL_OVRD_EN_0_MASK 0x10000000L
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_LF_CNTRL_OVRD_EN_0__SHIFT 0x0000001c
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_LF_CNTRL_OVRD_VAL_0_MASK 0x0ff80000L
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_LF_CNTRL_OVRD_VAL_0__SHIFT 0x00000013
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_REFDIV_OVRD_EN_0_MASK 0x80000000L
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_REFDIV_OVRD_EN_0__SHIFT 0x0000001f
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_REFDIV_OVRD_VAL_0_MASK 0x60000000L
+#define PB1_PLL_LC0_OVRD_REG0__PLL_CFG_LC_REFDIV_OVRD_VAL_0__SHIFT 0x0000001d
+#define PB1_PLL_LC0_OVRD_REG1__PLL_CFG_LC_REFCLK_SRC_OVRD_EN_0_MASK 0x00000008L
+#define PB1_PLL_LC0_OVRD_REG1__PLL_CFG_LC_REFCLK_SRC_OVRD_EN_0__SHIFT 0x00000003
+#define PB1_PLL_LC0_OVRD_REG1__PLL_CFG_LC_REFCLK_SRC_OVRD_VAL_0_MASK 0x00000007L
+#define PB1_PLL_LC0_OVRD_REG1__PLL_CFG_LC_REFCLK_SRC_OVRD_VAL_0__SHIFT 0x00000000
+#define PB1_PLL_LC0_OVRD_REG1__PLL_CFG_LC_VCO_TUNE_OVRD_EN_0_MASK 0x00040000L
+#define PB1_PLL_LC0_OVRD_REG1__PLL_CFG_LC_VCO_TUNE_OVRD_EN_0__SHIFT 0x00000012
+#define PB1_PLL_LC0_OVRD_REG1__PLL_CFG_LC_VCO_TUNE_OVRD_VAL_0_MASK 0x0003c000L
+#define PB1_PLL_LC0_OVRD_REG1__PLL_CFG_LC_VCO_TUNE_OVRD_VAL_0__SHIFT 0x0000000e
+#define PB1_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_LEFT_EN_OVRD_EN_0_MASK 0x00000020L
+#define PB1_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_LEFT_EN_OVRD_EN_0__SHIFT 0x00000005
+#define PB1_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_LEFT_EN_OVRD_VAL_0_MASK 0x00000010L
+#define PB1_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_LEFT_EN_OVRD_VAL_0__SHIFT 0x00000004
+#define PB1_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_RIGHT_EN_OVRD_EN_0_MASK 0x00000080L
+#define PB1_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_RIGHT_EN_OVRD_EN_0__SHIFT 0x00000007
+#define PB1_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_RIGHT_EN_OVRD_VAL_0_MASK 0x00000040L
+#define PB1_PLL_LC0_OVRD_REG1__PLL_LC_HSCLK_RIGHT_EN_OVRD_VAL_0__SHIFT 0x00000006
+#define PB1_PLL_LC0_OVRD_REG1__PLL_LC_PWRON_OVRD_EN_0_MASK 0x00000200L
+#define PB1_PLL_LC0_OVRD_REG1__PLL_LC_PWRON_OVRD_EN_0__SHIFT 0x00000009
+#define PB1_PLL_LC0_OVRD_REG1__PLL_LC_PWRON_OVRD_VAL_0_MASK 0x00000100L
+#define PB1_PLL_LC0_OVRD_REG1__PLL_LC_PWRON_OVRD_VAL_0__SHIFT 0x00000008
+#define PB1_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_FREQMODE_MASK 0x00000300L
+#define PB1_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_FREQMODE__SHIFT 0x00000008
+#define PB1_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB1_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB1_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB1_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB1_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_PLLPWR_MASK 0x00000070L
+#define PB1_PLL_LC0_SCI_STAT_OVRD_REG0__PLL_LC0_PLLPWR__SHIFT 0x00000004
+#define PB1_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_FREQMODE_MASK 0x00000300L
+#define PB1_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_FREQMODE__SHIFT 0x00000008
+#define PB1_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB1_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB1_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB1_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB1_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_PLLPWR_MASK 0x00000070L
+#define PB1_PLL_LC1_SCI_STAT_OVRD_REG0__PLL_LC1_PLLPWR__SHIFT 0x00000004
+#define PB1_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_FREQMODE_MASK 0x00000300L
+#define PB1_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_FREQMODE__SHIFT 0x00000008
+#define PB1_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB1_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB1_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB1_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB1_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_PLLPWR_MASK 0x00000070L
+#define PB1_PLL_LC2_SCI_STAT_OVRD_REG0__PLL_LC2_PLLPWR__SHIFT 0x00000004
+#define PB1_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_FREQMODE_MASK 0x00000300L
+#define PB1_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_FREQMODE__SHIFT 0x00000008
+#define PB1_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB1_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB1_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB1_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB1_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_PLLPWR_MASK 0x00000070L
+#define PB1_PLL_LC3_SCI_STAT_OVRD_REG0__PLL_LC3_PLLPWR__SHIFT 0x00000004
+#define PB1_PLL_RO0_CTRL_REG0__PLL_DBG_RO_ANALOG_SEL_0_MASK 0x00000003L
+#define PB1_PLL_RO0_CTRL_REG0__PLL_DBG_RO_ANALOG_SEL_0__SHIFT 0x00000000
+#define PB1_PLL_RO0_CTRL_REG0__PLL_DBG_RO_EXT_RESET_EN_0_MASK 0x00000004L
+#define PB1_PLL_RO0_CTRL_REG0__PLL_DBG_RO_EXT_RESET_EN_0__SHIFT 0x00000002
+#define PB1_PLL_RO0_CTRL_REG0__PLL_DBG_RO_LF_CNTRL_0_MASK 0x000007f0L
+#define PB1_PLL_RO0_CTRL_REG0__PLL_DBG_RO_LF_CNTRL_0__SHIFT 0x00000004
+#define PB1_PLL_RO0_CTRL_REG0__PLL_DBG_RO_VCTL_ADC_EN_0_MASK 0x00000008L
+#define PB1_PLL_RO0_CTRL_REG0__PLL_DBG_RO_VCTL_ADC_EN_0__SHIFT 0x00000003
+#define PB1_PLL_RO0_CTRL_REG0__PLL_TST_RO_USAMPLE_EN_0_MASK 0x00000800L
+#define PB1_PLL_RO0_CTRL_REG0__PLL_TST_RO_USAMPLE_EN_0__SHIFT 0x0000000b
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_BW_CNTRL_OVRD_EN_0_MASK 0x00000100L
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_BW_CNTRL_OVRD_EN_0__SHIFT 0x00000008
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_BW_CNTRL_OVRD_VAL_0_MASK 0x000000ffL
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_BW_CNTRL_OVRD_VAL_0__SHIFT 0x00000000
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_DIV_OVRD_EN_0_MASK 0x00001000L
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_DIV_OVRD_EN_0__SHIFT 0x0000000c
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_DIV_OVRD_VAL_0_MASK 0x00000e00L
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_DIV_OVRD_VAL_0__SHIFT 0x00000009
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_EN_OVRD_EN_0_MASK 0x00004000L
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_EN_OVRD_EN_0__SHIFT 0x0000000e
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_EN_OVRD_VAL_0_MASK 0x00002000L
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_CORECLK_EN_OVRD_VAL_0__SHIFT 0x0000000d
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_FBDIV_OVRD_EN_0_MASK 0x10000000L
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_FBDIV_OVRD_EN_0__SHIFT 0x0000001c
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_FBDIV_OVRD_VAL_0_MASK 0x0fff8000L
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_FBDIV_OVRD_VAL_0__SHIFT 0x0000000f
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_VTOI_BIAS_CNTRL_OVRD_EN_0_MASK 0x80000000L
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_VTOI_BIAS_CNTRL_OVRD_EN_0__SHIFT 0x0000001f
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_VTOI_BIAS_CNTRL_OVRD_VAL_0_MASK 0x40000000L
+#define PB1_PLL_RO0_OVRD_REG0__PLL_CFG_RO_VTOI_BIAS_CNTRL_OVRD_VAL_0__SHIFT 0x0000001e
+#define PB1_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFCLK_SRC_OVRD_EN_0_MASK 0x00400000L
+#define PB1_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFCLK_SRC_OVRD_EN_0__SHIFT 0x00000016
+#define PB1_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFCLK_SRC_OVRD_VAL_0_MASK 0x00380000L
+#define PB1_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFCLK_SRC_OVRD_VAL_0__SHIFT 0x00000013
+#define PB1_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFDIV_OVRD_EN_0_MASK 0x00000020L
+#define PB1_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFDIV_OVRD_EN_0__SHIFT 0x00000005
+#define PB1_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFDIV_OVRD_VAL_0_MASK 0x0000001fL
+#define PB1_PLL_RO0_OVRD_REG1__PLL_CFG_RO_REFDIV_OVRD_VAL_0__SHIFT 0x00000000
+#define PB1_PLL_RO0_OVRD_REG1__PLL_CFG_RO_VCO_MODE_OVRD_EN_0_MASK 0x00000100L
+#define PB1_PLL_RO0_OVRD_REG1__PLL_CFG_RO_VCO_MODE_OVRD_EN_0__SHIFT 0x00000008
+#define PB1_PLL_RO0_OVRD_REG1__PLL_CFG_RO_VCO_MODE_OVRD_VAL_0_MASK 0x000000c0L
+#define PB1_PLL_RO0_OVRD_REG1__PLL_CFG_RO_VCO_MODE_OVRD_VAL_0__SHIFT 0x00000006
+#define PB1_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_LEFT_EN_OVRD_EN_0_MASK 0x00000400L
+#define PB1_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_LEFT_EN_OVRD_EN_0__SHIFT 0x0000000a
+#define PB1_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_LEFT_EN_OVRD_VAL_0_MASK 0x00000200L
+#define PB1_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_LEFT_EN_OVRD_VAL_0__SHIFT 0x00000009
+#define PB1_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_RIGHT_EN_OVRD_EN_0_MASK 0x00001000L
+#define PB1_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_RIGHT_EN_OVRD_EN_0__SHIFT 0x0000000c
+#define PB1_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_RIGHT_EN_OVRD_VAL_0_MASK 0x00000800L
+#define PB1_PLL_RO0_OVRD_REG1__PLL_RO_HSCLK_RIGHT_EN_OVRD_VAL_0__SHIFT 0x0000000b
+#define PB1_PLL_RO0_OVRD_REG1__PLL_RO_PWRON_OVRD_EN_0_MASK 0x00004000L
+#define PB1_PLL_RO0_OVRD_REG1__PLL_RO_PWRON_OVRD_EN_0__SHIFT 0x0000000e
+#define PB1_PLL_RO0_OVRD_REG1__PLL_RO_PWRON_OVRD_VAL_0_MASK 0x00002000L
+#define PB1_PLL_RO0_OVRD_REG1__PLL_RO_PWRON_OVRD_VAL_0__SHIFT 0x0000000d
+#define PB1_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_FREQMODE_MASK 0x00000300L
+#define PB1_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_FREQMODE__SHIFT 0x00000008
+#define PB1_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB1_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB1_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB1_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB1_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_PLLPWR_MASK 0x00000070L
+#define PB1_PLL_RO0_SCI_STAT_OVRD_REG0__PLL_RO0_PLLPWR__SHIFT 0x00000004
+#define PB1_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_FREQMODE_MASK 0x00000300L
+#define PB1_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_FREQMODE__SHIFT 0x00000008
+#define PB1_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB1_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB1_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB1_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB1_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_PLLPWR_MASK 0x00000070L
+#define PB1_PLL_RO1_SCI_STAT_OVRD_REG0__PLL_RO1_PLLPWR__SHIFT 0x00000004
+#define PB1_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_FREQMODE_MASK 0x00000300L
+#define PB1_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_FREQMODE__SHIFT 0x00000008
+#define PB1_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB1_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB1_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB1_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB1_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_PLLPWR_MASK 0x00000070L
+#define PB1_PLL_RO2_SCI_STAT_OVRD_REG0__PLL_RO2_PLLPWR__SHIFT 0x00000004
+#define PB1_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_FREQMODE_MASK 0x00000300L
+#define PB1_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_FREQMODE__SHIFT 0x00000008
+#define PB1_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_IGNR_FREQMODE_SCI_UPDT_MASK 0x00000002L
+#define PB1_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_IGNR_FREQMODE_SCI_UPDT__SHIFT 0x00000001
+#define PB1_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_IGNR_PLLPWR_SCI_UPDT_MASK 0x00000001L
+#define PB1_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_IGNR_PLLPWR_SCI_UPDT__SHIFT 0x00000000
+#define PB1_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_PLLPWR_MASK 0x00000070L
+#define PB1_PLL_RO3_SCI_STAT_OVRD_REG0__PLL_RO3_PLLPWR__SHIFT 0x00000004
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_EN_LUT_ENTRY_LS0_MASK 0x00000200L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_EN_LUT_ENTRY_LS0__SHIFT 0x00000009
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_EN_LUT_ENTRY_LS1_MASK 0x00000400L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_EN_LUT_ENTRY_LS1__SHIFT 0x0000000a
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_EN_LUT_ENTRY_LS2_MASK 0x00000800L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_EN_LUT_ENTRY_LS2__SHIFT 0x0000000b
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_LEFT_EN_GATING_EN_MASK 0x00100000L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_LEFT_EN_GATING_EN__SHIFT 0x00000014
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_RIGHT_EN_GATING_EN_MASK 0x00200000L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_LEFT_RIGHT_EN_GATING_EN__SHIFT 0x00000015
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_EN_LUT_ENTRY_LS0_MASK 0x00001000L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_EN_LUT_ENTRY_LS0__SHIFT 0x0000000c
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_EN_LUT_ENTRY_LS1_MASK 0x00002000L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_EN_LUT_ENTRY_LS1__SHIFT 0x0000000d
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_EN_LUT_ENTRY_LS2_MASK 0x00004000L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_EN_LUT_ENTRY_LS2__SHIFT 0x0000000e
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_LEFT_EN_GATING_EN_MASK 0x00400000L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_LEFT_EN_GATING_EN__SHIFT 0x00000016
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_RIGHT_EN_GATING_EN_MASK 0x00800000L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_HSCLK_RIGHT_RIGHT_EN_GATING_EN__SHIFT 0x00000017
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_PWRON_LUT_ENTRY_LS2_MASK 0x00000100L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_LC_PWRON_LUT_ENTRY_LS2__SHIFT 0x00000008
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_EN_LUT_ENTRY_LS0_MASK 0x00000002L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_EN_LUT_ENTRY_LS0__SHIFT 0x00000001
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_EN_LUT_ENTRY_LS1_MASK 0x00000004L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_EN_LUT_ENTRY_LS1__SHIFT 0x00000002
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_EN_LUT_ENTRY_LS2_MASK 0x00000008L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_EN_LUT_ENTRY_LS2__SHIFT 0x00000003
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_LEFT_EN_GATING_EN_MASK 0x00010000L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_LEFT_EN_GATING_EN__SHIFT 0x00000010
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_RIGHT_EN_GATING_EN_MASK 0x00020000L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_LEFT_RIGHT_EN_GATING_EN__SHIFT 0x00000011
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_EN_LUT_ENTRY_LS0_MASK 0x00000010L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_EN_LUT_ENTRY_LS0__SHIFT 0x00000004
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_EN_LUT_ENTRY_LS1_MASK 0x00000020L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_EN_LUT_ENTRY_LS1__SHIFT 0x00000005
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_EN_LUT_ENTRY_LS2_MASK 0x00000040L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_EN_LUT_ENTRY_LS2__SHIFT 0x00000006
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_LEFT_EN_GATING_EN_MASK 0x00040000L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_LEFT_EN_GATING_EN__SHIFT 0x00000012
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_RIGHT_EN_GATING_EN_MASK 0x00080000L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_HSCLK_RIGHT_RIGHT_EN_GATING_EN__SHIFT 0x00000013
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_PWRON_LUT_ENTRY_LS2_MASK 0x00000080L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_RO_PWRON_LUT_ENTRY_LS2__SHIFT 0x00000007
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_TST_LOSPDTST_SRC_MASK 0x00000001L
+#define PB1_PLL_RO_GLB_CTRL_REG0__PLL_TST_LOSPDTST_SRC__SHIFT 0x00000000
+#define PB1_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_MODE_GEN1_MASK 0x000003ffL
+#define PB1_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_MODE_GEN1__SHIFT 0x00000000
+#define PB1_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_MODE_GEN2_MASK 0x000ffc00L
+#define PB1_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_MODE_GEN2__SHIFT 0x0000000a
+#define PB1_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_MODE_GEN3_MASK 0x3ff00000L
+#define PB1_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_MODE_GEN3__SHIFT 0x00000014
+#define PB1_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_RST_MODE_MASK 0xc0000000L
+#define PB1_RX_GLB_CTRL_REG0__RX_CFG_ADAPT_RST_MODE__SHIFT 0x0000001e
+#define PB1_RX_GLB_CTRL_REG1__RX_ADAPT_HLD_ASRT_TO_DCLK_EN_MASK 0xc0000000L
+#define PB1_RX_GLB_CTRL_REG1__RX_ADAPT_HLD_ASRT_TO_DCLK_EN__SHIFT 0x0000001e
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_FR_GAIN_GEN1_MASK 0x0000000fL
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_FR_GAIN_GEN1__SHIFT 0x00000000
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_FR_GAIN_GEN2_MASK 0x000000f0L
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_FR_GAIN_GEN2__SHIFT 0x00000004
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_FR_GAIN_GEN3_MASK 0x00000f00L
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_FR_GAIN_GEN3__SHIFT 0x00000008
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_PH_GAIN_GEN1_MASK 0x0000f000L
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_PH_GAIN_GEN1__SHIFT 0x0000000c
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_PH_GAIN_GEN2_MASK 0x000f0000L
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_PH_GAIN_GEN2__SHIFT 0x00000010
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_PH_GAIN_GEN3_MASK 0x00f00000L
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_PH_GAIN_GEN3__SHIFT 0x00000014
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_PI_STPSZ_GEN1_MASK 0x01000000L
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_PI_STPSZ_GEN1__SHIFT 0x00000018
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_PI_STPSZ_GEN2_MASK 0x02000000L
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_PI_STPSZ_GEN2__SHIFT 0x00000019
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_PI_STPSZ_GEN3_MASK 0x04000000L
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_CDR_PI_STPSZ_GEN3__SHIFT 0x0000001a
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_LEQ_DCATTN_BYP_EN_GEN1_MASK 0x08000000L
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_LEQ_DCATTN_BYP_EN_GEN1__SHIFT 0x0000001b
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_LEQ_DCATTN_BYP_EN_GEN2_MASK 0x10000000L
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_LEQ_DCATTN_BYP_EN_GEN2__SHIFT 0x0000001c
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_LEQ_DCATTN_BYP_EN_GEN3_MASK 0x20000000L
+#define PB1_RX_GLB_CTRL_REG1__RX_CFG_LEQ_DCATTN_BYP_EN_GEN3__SHIFT 0x0000001d
+#define PB1_RX_GLB_CTRL_REG2__RX_CFG_CDR_TIME_GEN1_MASK 0x0000f000L
+#define PB1_RX_GLB_CTRL_REG2__RX_CFG_CDR_TIME_GEN1__SHIFT 0x0000000c
+#define PB1_RX_GLB_CTRL_REG2__RX_CFG_CDR_TIME_GEN2_MASK 0x000f0000L
+#define PB1_RX_GLB_CTRL_REG2__RX_CFG_CDR_TIME_GEN2__SHIFT 0x00000010
+#define PB1_RX_GLB_CTRL_REG2__RX_CFG_CDR_TIME_GEN3_MASK 0x00f00000L
+#define PB1_RX_GLB_CTRL_REG2__RX_CFG_CDR_TIME_GEN3__SHIFT 0x00000014
+#define PB1_RX_GLB_CTRL_REG2__RX_CFG_LEQ_LOOP_GAIN_GEN1_MASK 0x03000000L
+#define PB1_RX_GLB_CTRL_REG2__RX_CFG_LEQ_LOOP_GAIN_GEN1__SHIFT 0x00000018
+#define PB1_RX_GLB_CTRL_REG2__RX_CFG_LEQ_LOOP_GAIN_GEN2_MASK 0x0c000000L
+#define PB1_RX_GLB_CTRL_REG2__RX_CFG_LEQ_LOOP_GAIN_GEN2__SHIFT 0x0000001a
+#define PB1_RX_GLB_CTRL_REG2__RX_CFG_LEQ_LOOP_GAIN_GEN3_MASK 0x30000000L
+#define PB1_RX_GLB_CTRL_REG2__RX_CFG_LEQ_LOOP_GAIN_GEN3__SHIFT 0x0000001c
+#define PB1_RX_GLB_CTRL_REG2__RX_DCLK_EN_ASRT_TO_ADAPT_HLD_MASK 0xc0000000L
+#define PB1_RX_GLB_CTRL_REG2__RX_DCLK_EN_ASRT_TO_ADAPT_HLD__SHIFT 0x0000001e
+#define PB1_RX_GLB_CTRL_REG3__RX_CFG_CDR_FR_EN_GEN1_MASK 0x00000001L
+#define PB1_RX_GLB_CTRL_REG3__RX_CFG_CDR_FR_EN_GEN1__SHIFT 0x00000000
+#define PB1_RX_GLB_CTRL_REG3__RX_CFG_CDR_FR_EN_GEN2_MASK 0x00000002L
+#define PB1_RX_GLB_CTRL_REG3__RX_CFG_CDR_FR_EN_GEN2__SHIFT 0x00000001
+#define PB1_RX_GLB_CTRL_REG3__RX_CFG_CDR_FR_EN_GEN3_MASK 0x00000004L
+#define PB1_RX_GLB_CTRL_REG3__RX_CFG_CDR_FR_EN_GEN3__SHIFT 0x00000002
+#define PB1_RX_GLB_CTRL_REG3__RX_CFG_DFE_TIME_GEN1_MASK 0x00f00000L
+#define PB1_RX_GLB_CTRL_REG3__RX_CFG_DFE_TIME_GEN1__SHIFT 0x00000014
+#define PB1_RX_GLB_CTRL_REG3__RX_CFG_DFE_TIME_GEN2_MASK 0x0f000000L
+#define PB1_RX_GLB_CTRL_REG3__RX_CFG_DFE_TIME_GEN2__SHIFT 0x00000018
+#define PB1_RX_GLB_CTRL_REG3__RX_CFG_DFE_TIME_GEN3_MASK 0xf0000000L
+#define PB1_RX_GLB_CTRL_REG3__RX_CFG_DFE_TIME_GEN3__SHIFT 0x0000001c
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_FOM_BER_GEN1_MASK 0x00000007L
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_FOM_BER_GEN1__SHIFT 0x00000000
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_FOM_BER_GEN2_MASK 0x00000038L
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_FOM_BER_GEN2__SHIFT 0x00000003
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_FOM_BER_GEN3_MASK 0x000001c0L
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_FOM_BER_GEN3__SHIFT 0x00000006
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_FOM_TIME_GEN1_MASK 0x00f00000L
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_FOM_TIME_GEN1__SHIFT 0x00000014
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_FOM_TIME_GEN2_MASK 0x0f000000L
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_FOM_TIME_GEN2__SHIFT 0x00000018
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_FOM_TIME_GEN3_MASK 0xf0000000L
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_FOM_TIME_GEN3__SHIFT 0x0000001c
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_LEQ_POLE_BYP_VAL_GEN1_MASK 0x00000e00L
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_LEQ_POLE_BYP_VAL_GEN1__SHIFT 0x00000009
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_LEQ_POLE_BYP_VAL_GEN2_MASK 0x00007000L
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_LEQ_POLE_BYP_VAL_GEN2__SHIFT 0x0000000c
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_LEQ_POLE_BYP_VAL_GEN3_MASK 0x00038000L
+#define PB1_RX_GLB_CTRL_REG4__RX_CFG_LEQ_POLE_BYP_VAL_GEN3__SHIFT 0x0000000f
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_DCATTN_BYP_VAL_GEN1_MASK 0x0000001fL
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_DCATTN_BYP_VAL_GEN1__SHIFT 0x00000000
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_DCATTN_BYP_VAL_GEN2_MASK 0x000003e0L
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_DCATTN_BYP_VAL_GEN2__SHIFT 0x00000005
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_DCATTN_BYP_VAL_GEN3_MASK 0x00007c00L
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_DCATTN_BYP_VAL_GEN3__SHIFT 0x0000000a
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_POLE_BYP_EN_GEN1_MASK 0x00008000L
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_POLE_BYP_EN_GEN1__SHIFT 0x0000000f
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_POLE_BYP_EN_GEN2_MASK 0x00010000L
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_POLE_BYP_EN_GEN2__SHIFT 0x00000010
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_POLE_BYP_EN_GEN3_MASK 0x00020000L
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_POLE_BYP_EN_GEN3__SHIFT 0x00000011
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_SHUNT_EN_GEN1_MASK 0x00040000L
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_SHUNT_EN_GEN1__SHIFT 0x00000012
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_SHUNT_EN_GEN2_MASK 0x00080000L
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_SHUNT_EN_GEN2__SHIFT 0x00000013
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_SHUNT_EN_GEN3_MASK 0x00100000L
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_LEQ_SHUNT_EN_GEN3__SHIFT 0x00000014
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_TERM_MODE_GEN1_MASK 0x08000000L
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_TERM_MODE_GEN1__SHIFT 0x0000001b
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_TERM_MODE_GEN2_MASK 0x10000000L
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_TERM_MODE_GEN2__SHIFT 0x0000001c
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_TERM_MODE_GEN3_MASK 0x20000000L
+#define PB1_RX_GLB_CTRL_REG5__RX_CFG_TERM_MODE_GEN3__SHIFT 0x0000001d
+#define PB1_RX_GLB_CTRL_REG5__RX_FORCE_DLL_RST_RXPWR_LS2OFF_TO_LS0_MASK 0x40000000L
+#define PB1_RX_GLB_CTRL_REG5__RX_FORCE_DLL_RST_RXPWR_LS2OFF_TO_LS0__SHIFT 0x0000001e
+#define PB1_RX_GLB_CTRL_REG6__RX_AUX_PWRON_LUT_ENTRY_LS2_MASK 0x08000000L
+#define PB1_RX_GLB_CTRL_REG6__RX_AUX_PWRON_LUT_ENTRY_LS2__SHIFT 0x0000001b
+#define PB1_RX_GLB_CTRL_REG6__RX_CFG_LEQ_TIME_GEN1_MASK 0x0000000fL
+#define PB1_RX_GLB_CTRL_REG6__RX_CFG_LEQ_TIME_GEN1__SHIFT 0x00000000
+#define PB1_RX_GLB_CTRL_REG6__RX_CFG_LEQ_TIME_GEN2_MASK 0x000000f0L
+#define PB1_RX_GLB_CTRL_REG6__RX_CFG_LEQ_TIME_GEN2__SHIFT 0x00000004
+#define PB1_RX_GLB_CTRL_REG6__RX_CFG_LEQ_TIME_GEN3_MASK 0x00000f00L
+#define PB1_RX_GLB_CTRL_REG6__RX_CFG_LEQ_TIME_GEN3__SHIFT 0x00000008
+#define PB1_RX_GLB_CTRL_REG6__RX_CFG_OC_TIME_GEN1_MASK 0x0000f000L
+#define PB1_RX_GLB_CTRL_REG6__RX_CFG_OC_TIME_GEN1__SHIFT 0x0000000c
+#define PB1_RX_GLB_CTRL_REG6__RX_CFG_OC_TIME_GEN2_MASK 0x000f0000L
+#define PB1_RX_GLB_CTRL_REG6__RX_CFG_OC_TIME_GEN2__SHIFT 0x00000010
+#define PB1_RX_GLB_CTRL_REG6__RX_CFG_OC_TIME_GEN3_MASK 0x00f00000L
+#define PB1_RX_GLB_CTRL_REG6__RX_CFG_OC_TIME_GEN3__SHIFT 0x00000014
+#define PB1_RX_GLB_CTRL_REG6__RX_FRONTEND_PWRON_LUT_ENTRY_LS0_CDR_EN_0_MASK 0x01000000L
+#define PB1_RX_GLB_CTRL_REG6__RX_FRONTEND_PWRON_LUT_ENTRY_LS0_CDR_EN_0__SHIFT 0x00000018
+#define PB1_RX_GLB_CTRL_REG6__RX_FRONTEND_PWRON_LUT_ENTRY_LS2_MASK 0x04000000L
+#define PB1_RX_GLB_CTRL_REG6__RX_FRONTEND_PWRON_LUT_ENTRY_LS2__SHIFT 0x0000001a
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_DLL_CPI_SEL_GEN1_MASK 0x001c0000L
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_DLL_CPI_SEL_GEN1__SHIFT 0x00000012
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_DLL_CPI_SEL_GEN2_MASK 0x00e00000L
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_DLL_CPI_SEL_GEN2__SHIFT 0x00000015
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_DLL_CPI_SEL_GEN3_MASK 0x07000000L
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_DLL_CPI_SEL_GEN3__SHIFT 0x00000018
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_DLL_FLOCK_DISABLE_GEN1_MASK 0x08000000L
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_DLL_FLOCK_DISABLE_GEN1__SHIFT 0x0000001b
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_DLL_FLOCK_DISABLE_GEN2_MASK 0x10000000L
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_DLL_FLOCK_DISABLE_GEN2__SHIFT 0x0000001c
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_DLL_FLOCK_DISABLE_GEN3_MASK 0x20000000L
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_DLL_FLOCK_DISABLE_GEN3__SHIFT 0x0000001d
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_TH_LOOP_GAIN_GEN1_MASK 0x0000000fL
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_TH_LOOP_GAIN_GEN1__SHIFT 0x00000000
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_TH_LOOP_GAIN_GEN2_MASK 0x000000f0L
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_TH_LOOP_GAIN_GEN2__SHIFT 0x00000004
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_TH_LOOP_GAIN_GEN3_MASK 0x00000f00L
+#define PB1_RX_GLB_CTRL_REG7__RX_CFG_TH_LOOP_GAIN_GEN3__SHIFT 0x00000008
+#define PB1_RX_GLB_CTRL_REG7__RX_DCLK_EN_LUT_ENTRY_LS0_CDR_EN_0_MASK 0x00001000L
+#define PB1_RX_GLB_CTRL_REG7__RX_DCLK_EN_LUT_ENTRY_LS0_CDR_EN_0__SHIFT 0x0000000c
+#define PB1_RX_GLB_CTRL_REG7__RX_DCLK_EN_LUT_ENTRY_LS2_MASK 0x00002000L
+#define PB1_RX_GLB_CTRL_REG7__RX_DCLK_EN_LUT_ENTRY_LS2__SHIFT 0x0000000d
+#define PB1_RX_GLB_CTRL_REG7__RX_DLL_PWRON_LUT_ENTRY_LS2_MASK 0x00020000L
+#define PB1_RX_GLB_CTRL_REG7__RX_DLL_PWRON_LUT_ENTRY_LS2__SHIFT 0x00000011
+#define PB1_RX_GLB_OVRD_REG0__RX_ADAPT_FOM_OVRD_EN_MASK 0x80000000L
+#define PB1_RX_GLB_OVRD_REG0__RX_ADAPT_FOM_OVRD_EN__SHIFT 0x0000001f
+#define PB1_RX_GLB_OVRD_REG0__RX_ADAPT_FOM_OVRD_VAL_MASK 0x40000000L
+#define PB1_RX_GLB_OVRD_REG0__RX_ADAPT_FOM_OVRD_VAL__SHIFT 0x0000001e
+#define PB1_RX_GLB_OVRD_REG0__RX_ADAPT_HLD_OVRD_EN_MASK 0x00000002L
+#define PB1_RX_GLB_OVRD_REG0__RX_ADAPT_HLD_OVRD_EN__SHIFT 0x00000001
+#define PB1_RX_GLB_OVRD_REG0__RX_ADAPT_HLD_OVRD_VAL_MASK 0x00000001L
+#define PB1_RX_GLB_OVRD_REG0__RX_ADAPT_HLD_OVRD_VAL__SHIFT 0x00000000
+#define PB1_RX_GLB_OVRD_REG0__RX_ADAPT_RST_OVRD_EN_MASK 0x00000008L
+#define PB1_RX_GLB_OVRD_REG0__RX_ADAPT_RST_OVRD_EN__SHIFT 0x00000003
+#define PB1_RX_GLB_OVRD_REG0__RX_ADAPT_RST_OVRD_VAL_MASK 0x00000004L
+#define PB1_RX_GLB_OVRD_REG0__RX_ADAPT_RST_OVRD_VAL__SHIFT 0x00000002
+#define PB1_RX_GLB_OVRD_REG0__RX_AUX_PWRON_OVRD_EN_MASK 0x20000000L
+#define PB1_RX_GLB_OVRD_REG0__RX_AUX_PWRON_OVRD_EN__SHIFT 0x0000001d
+#define PB1_RX_GLB_OVRD_REG0__RX_AUX_PWRON_OVRD_VAL_MASK 0x10000000L
+#define PB1_RX_GLB_OVRD_REG0__RX_AUX_PWRON_OVRD_VAL__SHIFT 0x0000001c
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_DCLK_DIV_OVRD_EN_MASK 0x00000100L
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_DCLK_DIV_OVRD_EN__SHIFT 0x00000008
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_DCLK_DIV_OVRD_VAL_MASK 0x000000c0L
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_DCLK_DIV_OVRD_VAL__SHIFT 0x00000006
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_DLL_FREQ_MODE_OVRD_EN_MASK 0x00000400L
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_DLL_FREQ_MODE_OVRD_EN__SHIFT 0x0000000a
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_DLL_FREQ_MODE_OVRD_VAL_MASK 0x00000200L
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_DLL_FREQ_MODE_OVRD_VAL__SHIFT 0x00000009
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_PLLCLK_SEL_OVRD_EN_MASK 0x00001000L
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_PLLCLK_SEL_OVRD_EN__SHIFT 0x0000000c
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_PLLCLK_SEL_OVRD_VAL_MASK 0x00000800L
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_PLLCLK_SEL_OVRD_VAL__SHIFT 0x0000000b
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_RCLK_DIV_OVRD_EN_MASK 0x00004000L
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_RCLK_DIV_OVRD_EN__SHIFT 0x0000000e
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_RCLK_DIV_OVRD_VAL_MASK 0x00002000L
+#define PB1_RX_GLB_OVRD_REG0__RX_CFG_RCLK_DIV_OVRD_VAL__SHIFT 0x0000000d
+#define PB1_RX_GLB_OVRD_REG0__RX_DCLK_EN_OVRD_EN_MASK 0x00010000L
+#define PB1_RX_GLB_OVRD_REG0__RX_DCLK_EN_OVRD_EN__SHIFT 0x00000010
+#define PB1_RX_GLB_OVRD_REG0__RX_DCLK_EN_OVRD_VAL_MASK 0x00008000L
+#define PB1_RX_GLB_OVRD_REG0__RX_DCLK_EN_OVRD_VAL__SHIFT 0x0000000f
+#define PB1_RX_GLB_OVRD_REG0__RX_DLL_PWRON_OVRD_EN_MASK 0x00040000L
+#define PB1_RX_GLB_OVRD_REG0__RX_DLL_PWRON_OVRD_EN__SHIFT 0x00000012
+#define PB1_RX_GLB_OVRD_REG0__RX_DLL_PWRON_OVRD_VAL_MASK 0x00020000L
+#define PB1_RX_GLB_OVRD_REG0__RX_DLL_PWRON_OVRD_VAL__SHIFT 0x00000011
+#define PB1_RX_GLB_OVRD_REG0__RX_FRONTEND_PWRON_OVRD_EN_MASK 0x00100000L
+#define PB1_RX_GLB_OVRD_REG0__RX_FRONTEND_PWRON_OVRD_EN__SHIFT 0x00000014
+#define PB1_RX_GLB_OVRD_REG0__RX_FRONTEND_PWRON_OVRD_VAL_MASK 0x00080000L
+#define PB1_RX_GLB_OVRD_REG0__RX_FRONTEND_PWRON_OVRD_VAL__SHIFT 0x00000013
+#define PB1_RX_GLB_OVRD_REG0__RX_IDLEDET_PWRON_OVRD_EN_MASK 0x00400000L
+#define PB1_RX_GLB_OVRD_REG0__RX_IDLEDET_PWRON_OVRD_EN__SHIFT 0x00000016
+#define PB1_RX_GLB_OVRD_REG0__RX_IDLEDET_PWRON_OVRD_VAL_MASK 0x00200000L
+#define PB1_RX_GLB_OVRD_REG0__RX_IDLEDET_PWRON_OVRD_VAL__SHIFT 0x00000015
+#define PB1_RX_GLB_OVRD_REG0__RX_TERM_EN_OVRD_EN_MASK 0x01000000L
+#define PB1_RX_GLB_OVRD_REG0__RX_TERM_EN_OVRD_EN__SHIFT 0x00000018
+#define PB1_RX_GLB_OVRD_REG0__RX_TERM_EN_OVRD_VAL_MASK 0x00800000L
+#define PB1_RX_GLB_OVRD_REG0__RX_TERM_EN_OVRD_VAL__SHIFT 0x00000017
+#define PB1_RX_GLB_OVRD_REG1__RX_ADAPT_TRK_OVRD_EN_MASK 0x00000002L
+#define PB1_RX_GLB_OVRD_REG1__RX_ADAPT_TRK_OVRD_EN__SHIFT 0x00000001
+#define PB1_RX_GLB_OVRD_REG1__RX_ADAPT_TRK_OVRD_VAL_MASK 0x00000001L
+#define PB1_RX_GLB_OVRD_REG1__RX_ADAPT_TRK_OVRD_VAL__SHIFT 0x00000000
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L0T3_MASK 0x00000010L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L0T3__SHIFT 0x00000004
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L12T15_MASK 0x00000080L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L12T15__SHIFT 0x00000007
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L4T7_MASK 0x00000020L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L4T7__SHIFT 0x00000005
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L8T11_MASK 0x00000040L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ELECIDLEDETEN_SCI_UPDT_L8T11__SHIFT 0x00000006
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L0T3_MASK 0x00001000L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L0T3__SHIFT 0x0000000c
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L12T15_MASK 0x00008000L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L12T15__SHIFT 0x0000000f
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L4T7_MASK 0x00002000L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L4T7__SHIFT 0x0000000d
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L8T11_MASK 0x00004000L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_ENABLEFOM_SCI_UPDT_L8T11__SHIFT 0x0000000e
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L0T3_MASK 0x00010000L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L0T3__SHIFT 0x00000010
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L12T15_MASK 0x00080000L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L12T15__SHIFT 0x00000013
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L4T7_MASK 0x00020000L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L4T7__SHIFT 0x00000011
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L8T11_MASK 0x00040000L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_REQUESTFOM_SCI_UPDT_L8T11__SHIFT 0x00000012
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L0T3_MASK 0x00100000L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L0T3__SHIFT 0x00000014
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L12T15_MASK 0x00800000L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L12T15__SHIFT 0x00000017
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L4T7_MASK 0x00200000L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L4T7__SHIFT 0x00000015
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L8T11_MASK 0x00400000L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RESPONSEMODE_SCI_UPDT_L8T11__SHIFT 0x00000016
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L0T3_MASK 0x00000100L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L0T3__SHIFT 0x00000008
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L12T15_MASK 0x00000800L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L12T15__SHIFT 0x0000000b
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L4T7_MASK 0x00000200L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L4T7__SHIFT 0x00000009
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L8T11_MASK 0x00000400L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPRESETHINT_SCI_UPDT_L8T11__SHIFT 0x0000000a
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L0T3_MASK 0x00000001L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L0T3__SHIFT 0x00000000
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L12T15_MASK 0x00000008L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L12T15__SHIFT 0x00000003
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L4T7_MASK 0x00000002L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L4T7__SHIFT 0x00000001
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L8T11_MASK 0x00000004L
+#define PB1_RX_GLB_SCI_STAT_OVRD_REG0__IGNR_RXPWR_SCI_UPDT_L8T11__SHIFT 0x00000002
+#define PB1_RX_LANE0_CTRL_REG0__RX_BACKUP_0_MASK 0x000000ffL
+#define PB1_RX_LANE0_CTRL_REG0__RX_BACKUP_0__SHIFT 0x00000000
+#define PB1_RX_LANE0_CTRL_REG0__RX_CFG_OVR_PWRSF_0_MASK 0x00002000L
+#define PB1_RX_LANE0_CTRL_REG0__RX_CFG_OVR_PWRSF_0__SHIFT 0x0000000d
+#define PB1_RX_LANE0_CTRL_REG0__RX_DBG_ANALOG_SEL_0_MASK 0x00000c00L
+#define PB1_RX_LANE0_CTRL_REG0__RX_DBG_ANALOG_SEL_0__SHIFT 0x0000000a
+#define PB1_RX_LANE0_CTRL_REG0__RX_TST_BSCAN_EN_0_MASK 0x00001000L
+#define PB1_RX_LANE0_CTRL_REG0__RX_TST_BSCAN_EN_0__SHIFT 0x0000000c
+#define PB1_RX_LANE0_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_0_MASK 0x00000008L
+#define PB1_RX_LANE0_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_0__SHIFT 0x00000003
+#define PB1_RX_LANE0_SCI_STAT_OVRD_REG0__ENABLEFOM_0_MASK 0x00000080L
+#define PB1_RX_LANE0_SCI_STAT_OVRD_REG0__ENABLEFOM_0__SHIFT 0x00000007
+#define PB1_RX_LANE0_SCI_STAT_OVRD_REG0__REQUESTFOM_0_MASK 0x00000100L
+#define PB1_RX_LANE0_SCI_STAT_OVRD_REG0__REQUESTFOM_0__SHIFT 0x00000008
+#define PB1_RX_LANE0_SCI_STAT_OVRD_REG0__RESPONSEMODE_0_MASK 0x00000200L
+#define PB1_RX_LANE0_SCI_STAT_OVRD_REG0__RESPONSEMODE_0__SHIFT 0x00000009
+#define PB1_RX_LANE0_SCI_STAT_OVRD_REG0__RXPRESETHINT_0_MASK 0x00000070L
+#define PB1_RX_LANE0_SCI_STAT_OVRD_REG0__RXPRESETHINT_0__SHIFT 0x00000004
+#define PB1_RX_LANE0_SCI_STAT_OVRD_REG0__RXPWR_0_MASK 0x00000007L
+#define PB1_RX_LANE0_SCI_STAT_OVRD_REG0__RXPWR_0__SHIFT 0x00000000
+#define PB1_RX_LANE10_CTRL_REG0__RX_BACKUP_10_MASK 0x000000ffL
+#define PB1_RX_LANE10_CTRL_REG0__RX_BACKUP_10__SHIFT 0x00000000
+#define PB1_RX_LANE10_CTRL_REG0__RX_CFG_OVR_PWRSF_10_MASK 0x00002000L
+#define PB1_RX_LANE10_CTRL_REG0__RX_CFG_OVR_PWRSF_10__SHIFT 0x0000000d
+#define PB1_RX_LANE10_CTRL_REG0__RX_DBG_ANALOG_SEL_10_MASK 0x00000c00L
+#define PB1_RX_LANE10_CTRL_REG0__RX_DBG_ANALOG_SEL_10__SHIFT 0x0000000a
+#define PB1_RX_LANE10_CTRL_REG0__RX_TST_BSCAN_EN_10_MASK 0x00001000L
+#define PB1_RX_LANE10_CTRL_REG0__RX_TST_BSCAN_EN_10__SHIFT 0x0000000c
+#define PB1_RX_LANE10_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_10_MASK 0x00000008L
+#define PB1_RX_LANE10_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_10__SHIFT 0x00000003
+#define PB1_RX_LANE10_SCI_STAT_OVRD_REG0__ENABLEFOM_10_MASK 0x00000080L
+#define PB1_RX_LANE10_SCI_STAT_OVRD_REG0__ENABLEFOM_10__SHIFT 0x00000007
+#define PB1_RX_LANE10_SCI_STAT_OVRD_REG0__REQUESTFOM_10_MASK 0x00000100L
+#define PB1_RX_LANE10_SCI_STAT_OVRD_REG0__REQUESTFOM_10__SHIFT 0x00000008
+#define PB1_RX_LANE10_SCI_STAT_OVRD_REG0__RESPONSEMODE_10_MASK 0x00000200L
+#define PB1_RX_LANE10_SCI_STAT_OVRD_REG0__RESPONSEMODE_10__SHIFT 0x00000009
+#define PB1_RX_LANE10_SCI_STAT_OVRD_REG0__RXPRESETHINT_10_MASK 0x00000070L
+#define PB1_RX_LANE10_SCI_STAT_OVRD_REG0__RXPRESETHINT_10__SHIFT 0x00000004
+#define PB1_RX_LANE10_SCI_STAT_OVRD_REG0__RXPWR_10_MASK 0x00000007L
+#define PB1_RX_LANE10_SCI_STAT_OVRD_REG0__RXPWR_10__SHIFT 0x00000000
+#define PB1_RX_LANE11_CTRL_REG0__RX_BACKUP_11_MASK 0x000000ffL
+#define PB1_RX_LANE11_CTRL_REG0__RX_BACKUP_11__SHIFT 0x00000000
+#define PB1_RX_LANE11_CTRL_REG0__RX_CFG_OVR_PWRSF_11_MASK 0x00002000L
+#define PB1_RX_LANE11_CTRL_REG0__RX_CFG_OVR_PWRSF_11__SHIFT 0x0000000d
+#define PB1_RX_LANE11_CTRL_REG0__RX_DBG_ANALOG_SEL_11_MASK 0x00000c00L
+#define PB1_RX_LANE11_CTRL_REG0__RX_DBG_ANALOG_SEL_11__SHIFT 0x0000000a
+#define PB1_RX_LANE11_CTRL_REG0__RX_TST_BSCAN_EN_11_MASK 0x00001000L
+#define PB1_RX_LANE11_CTRL_REG0__RX_TST_BSCAN_EN_11__SHIFT 0x0000000c
+#define PB1_RX_LANE11_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_11_MASK 0x00000008L
+#define PB1_RX_LANE11_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_11__SHIFT 0x00000003
+#define PB1_RX_LANE11_SCI_STAT_OVRD_REG0__ENABLEFOM_11_MASK 0x00000080L
+#define PB1_RX_LANE11_SCI_STAT_OVRD_REG0__ENABLEFOM_11__SHIFT 0x00000007
+#define PB1_RX_LANE11_SCI_STAT_OVRD_REG0__REQUESTFOM_11_MASK 0x00000100L
+#define PB1_RX_LANE11_SCI_STAT_OVRD_REG0__REQUESTFOM_11__SHIFT 0x00000008
+#define PB1_RX_LANE11_SCI_STAT_OVRD_REG0__RESPONSEMODE_11_MASK 0x00000200L
+#define PB1_RX_LANE11_SCI_STAT_OVRD_REG0__RESPONSEMODE_11__SHIFT 0x00000009
+#define PB1_RX_LANE11_SCI_STAT_OVRD_REG0__RXPRESETHINT_11_MASK 0x00000070L
+#define PB1_RX_LANE11_SCI_STAT_OVRD_REG0__RXPRESETHINT_11__SHIFT 0x00000004
+#define PB1_RX_LANE11_SCI_STAT_OVRD_REG0__RXPWR_11_MASK 0x00000007L
+#define PB1_RX_LANE11_SCI_STAT_OVRD_REG0__RXPWR_11__SHIFT 0x00000000
+#define PB1_RX_LANE12_CTRL_REG0__RX_BACKUP_12_MASK 0x000000ffL
+#define PB1_RX_LANE12_CTRL_REG0__RX_BACKUP_12__SHIFT 0x00000000
+#define PB1_RX_LANE12_CTRL_REG0__RX_CFG_OVR_PWRSF_12_MASK 0x00002000L
+#define PB1_RX_LANE12_CTRL_REG0__RX_CFG_OVR_PWRSF_12__SHIFT 0x0000000d
+#define PB1_RX_LANE12_CTRL_REG0__RX_DBG_ANALOG_SEL_12_MASK 0x00000c00L
+#define PB1_RX_LANE12_CTRL_REG0__RX_DBG_ANALOG_SEL_12__SHIFT 0x0000000a
+#define PB1_RX_LANE12_CTRL_REG0__RX_TST_BSCAN_EN_12_MASK 0x00001000L
+#define PB1_RX_LANE12_CTRL_REG0__RX_TST_BSCAN_EN_12__SHIFT 0x0000000c
+#define PB1_RX_LANE12_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_12_MASK 0x00000008L
+#define PB1_RX_LANE12_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_12__SHIFT 0x00000003
+#define PB1_RX_LANE12_SCI_STAT_OVRD_REG0__ENABLEFOM_12_MASK 0x00000080L
+#define PB1_RX_LANE12_SCI_STAT_OVRD_REG0__ENABLEFOM_12__SHIFT 0x00000007
+#define PB1_RX_LANE12_SCI_STAT_OVRD_REG0__REQUESTFOM_12_MASK 0x00000100L
+#define PB1_RX_LANE12_SCI_STAT_OVRD_REG0__REQUESTFOM_12__SHIFT 0x00000008
+#define PB1_RX_LANE12_SCI_STAT_OVRD_REG0__RESPONSEMODE_12_MASK 0x00000200L
+#define PB1_RX_LANE12_SCI_STAT_OVRD_REG0__RESPONSEMODE_12__SHIFT 0x00000009
+#define PB1_RX_LANE12_SCI_STAT_OVRD_REG0__RXPRESETHINT_12_MASK 0x00000070L
+#define PB1_RX_LANE12_SCI_STAT_OVRD_REG0__RXPRESETHINT_12__SHIFT 0x00000004
+#define PB1_RX_LANE12_SCI_STAT_OVRD_REG0__RXPWR_12_MASK 0x00000007L
+#define PB1_RX_LANE12_SCI_STAT_OVRD_REG0__RXPWR_12__SHIFT 0x00000000
+#define PB1_RX_LANE13_CTRL_REG0__RX_BACKUP_13_MASK 0x000000ffL
+#define PB1_RX_LANE13_CTRL_REG0__RX_BACKUP_13__SHIFT 0x00000000
+#define PB1_RX_LANE13_CTRL_REG0__RX_CFG_OVR_PWRSF_13_MASK 0x00002000L
+#define PB1_RX_LANE13_CTRL_REG0__RX_CFG_OVR_PWRSF_13__SHIFT 0x0000000d
+#define PB1_RX_LANE13_CTRL_REG0__RX_DBG_ANALOG_SEL_13_MASK 0x00000c00L
+#define PB1_RX_LANE13_CTRL_REG0__RX_DBG_ANALOG_SEL_13__SHIFT 0x0000000a
+#define PB1_RX_LANE13_CTRL_REG0__RX_TST_BSCAN_EN_13_MASK 0x00001000L
+#define PB1_RX_LANE13_CTRL_REG0__RX_TST_BSCAN_EN_13__SHIFT 0x0000000c
+#define PB1_RX_LANE13_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_13_MASK 0x00000008L
+#define PB1_RX_LANE13_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_13__SHIFT 0x00000003
+#define PB1_RX_LANE13_SCI_STAT_OVRD_REG0__ENABLEFOM_13_MASK 0x00000080L
+#define PB1_RX_LANE13_SCI_STAT_OVRD_REG0__ENABLEFOM_13__SHIFT 0x00000007
+#define PB1_RX_LANE13_SCI_STAT_OVRD_REG0__REQUESTFOM_13_MASK 0x00000100L
+#define PB1_RX_LANE13_SCI_STAT_OVRD_REG0__REQUESTFOM_13__SHIFT 0x00000008
+#define PB1_RX_LANE13_SCI_STAT_OVRD_REG0__RESPONSEMODE_13_MASK 0x00000200L
+#define PB1_RX_LANE13_SCI_STAT_OVRD_REG0__RESPONSEMODE_13__SHIFT 0x00000009
+#define PB1_RX_LANE13_SCI_STAT_OVRD_REG0__RXPRESETHINT_13_MASK 0x00000070L
+#define PB1_RX_LANE13_SCI_STAT_OVRD_REG0__RXPRESETHINT_13__SHIFT 0x00000004
+#define PB1_RX_LANE13_SCI_STAT_OVRD_REG0__RXPWR_13_MASK 0x00000007L
+#define PB1_RX_LANE13_SCI_STAT_OVRD_REG0__RXPWR_13__SHIFT 0x00000000
+#define PB1_RX_LANE14_CTRL_REG0__RX_BACKUP_14_MASK 0x000000ffL
+#define PB1_RX_LANE14_CTRL_REG0__RX_BACKUP_14__SHIFT 0x00000000
+#define PB1_RX_LANE14_CTRL_REG0__RX_CFG_OVR_PWRSF_14_MASK 0x00002000L
+#define PB1_RX_LANE14_CTRL_REG0__RX_CFG_OVR_PWRSF_14__SHIFT 0x0000000d
+#define PB1_RX_LANE14_CTRL_REG0__RX_DBG_ANALOG_SEL_14_MASK 0x00000c00L
+#define PB1_RX_LANE14_CTRL_REG0__RX_DBG_ANALOG_SEL_14__SHIFT 0x0000000a
+#define PB1_RX_LANE14_CTRL_REG0__RX_TST_BSCAN_EN_14_MASK 0x00001000L
+#define PB1_RX_LANE14_CTRL_REG0__RX_TST_BSCAN_EN_14__SHIFT 0x0000000c
+#define PB1_RX_LANE14_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_14_MASK 0x00000008L
+#define PB1_RX_LANE14_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_14__SHIFT 0x00000003
+#define PB1_RX_LANE14_SCI_STAT_OVRD_REG0__ENABLEFOM_14_MASK 0x00000080L
+#define PB1_RX_LANE14_SCI_STAT_OVRD_REG0__ENABLEFOM_14__SHIFT 0x00000007
+#define PB1_RX_LANE14_SCI_STAT_OVRD_REG0__REQUESTFOM_14_MASK 0x00000100L
+#define PB1_RX_LANE14_SCI_STAT_OVRD_REG0__REQUESTFOM_14__SHIFT 0x00000008
+#define PB1_RX_LANE14_SCI_STAT_OVRD_REG0__RESPONSEMODE_14_MASK 0x00000200L
+#define PB1_RX_LANE14_SCI_STAT_OVRD_REG0__RESPONSEMODE_14__SHIFT 0x00000009
+#define PB1_RX_LANE14_SCI_STAT_OVRD_REG0__RXPRESETHINT_14_MASK 0x00000070L
+#define PB1_RX_LANE14_SCI_STAT_OVRD_REG0__RXPRESETHINT_14__SHIFT 0x00000004
+#define PB1_RX_LANE14_SCI_STAT_OVRD_REG0__RXPWR_14_MASK 0x00000007L
+#define PB1_RX_LANE14_SCI_STAT_OVRD_REG0__RXPWR_14__SHIFT 0x00000000
+#define PB1_RX_LANE15_CTRL_REG0__RX_BACKUP_15_MASK 0x000000ffL
+#define PB1_RX_LANE15_CTRL_REG0__RX_BACKUP_15__SHIFT 0x00000000
+#define PB1_RX_LANE15_CTRL_REG0__RX_CFG_OVR_PWRSF_15_MASK 0x00002000L
+#define PB1_RX_LANE15_CTRL_REG0__RX_CFG_OVR_PWRSF_15__SHIFT 0x0000000d
+#define PB1_RX_LANE15_CTRL_REG0__RX_DBG_ANALOG_SEL_15_MASK 0x00000c00L
+#define PB1_RX_LANE15_CTRL_REG0__RX_DBG_ANALOG_SEL_15__SHIFT 0x0000000a
+#define PB1_RX_LANE15_CTRL_REG0__RX_TST_BSCAN_EN_15_MASK 0x00001000L
+#define PB1_RX_LANE15_CTRL_REG0__RX_TST_BSCAN_EN_15__SHIFT 0x0000000c
+#define PB1_RX_LANE15_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_15_MASK 0x00000008L
+#define PB1_RX_LANE15_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_15__SHIFT 0x00000003
+#define PB1_RX_LANE15_SCI_STAT_OVRD_REG0__ENABLEFOM_15_MASK 0x00000080L
+#define PB1_RX_LANE15_SCI_STAT_OVRD_REG0__ENABLEFOM_15__SHIFT 0x00000007
+#define PB1_RX_LANE15_SCI_STAT_OVRD_REG0__REQUESTFOM_15_MASK 0x00000100L
+#define PB1_RX_LANE15_SCI_STAT_OVRD_REG0__REQUESTFOM_15__SHIFT 0x00000008
+#define PB1_RX_LANE15_SCI_STAT_OVRD_REG0__RESPONSEMODE_15_MASK 0x00000200L
+#define PB1_RX_LANE15_SCI_STAT_OVRD_REG0__RESPONSEMODE_15__SHIFT 0x00000009
+#define PB1_RX_LANE15_SCI_STAT_OVRD_REG0__RXPRESETHINT_15_MASK 0x00000070L
+#define PB1_RX_LANE15_SCI_STAT_OVRD_REG0__RXPRESETHINT_15__SHIFT 0x00000004
+#define PB1_RX_LANE15_SCI_STAT_OVRD_REG0__RXPWR_15_MASK 0x00000007L
+#define PB1_RX_LANE15_SCI_STAT_OVRD_REG0__RXPWR_15__SHIFT 0x00000000
+#define PB1_RX_LANE1_CTRL_REG0__RX_BACKUP_1_MASK 0x000000ffL
+#define PB1_RX_LANE1_CTRL_REG0__RX_BACKUP_1__SHIFT 0x00000000
+#define PB1_RX_LANE1_CTRL_REG0__RX_CFG_OVR_PWRSF_1_MASK 0x00002000L
+#define PB1_RX_LANE1_CTRL_REG0__RX_CFG_OVR_PWRSF_1__SHIFT 0x0000000d
+#define PB1_RX_LANE1_CTRL_REG0__RX_DBG_ANALOG_SEL_1_MASK 0x00000c00L
+#define PB1_RX_LANE1_CTRL_REG0__RX_DBG_ANALOG_SEL_1__SHIFT 0x0000000a
+#define PB1_RX_LANE1_CTRL_REG0__RX_TST_BSCAN_EN_1_MASK 0x00001000L
+#define PB1_RX_LANE1_CTRL_REG0__RX_TST_BSCAN_EN_1__SHIFT 0x0000000c
+#define PB1_RX_LANE1_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_1_MASK 0x00000008L
+#define PB1_RX_LANE1_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_1__SHIFT 0x00000003
+#define PB1_RX_LANE1_SCI_STAT_OVRD_REG0__ENABLEFOM_1_MASK 0x00000080L
+#define PB1_RX_LANE1_SCI_STAT_OVRD_REG0__ENABLEFOM_1__SHIFT 0x00000007
+#define PB1_RX_LANE1_SCI_STAT_OVRD_REG0__REQUESTFOM_1_MASK 0x00000100L
+#define PB1_RX_LANE1_SCI_STAT_OVRD_REG0__REQUESTFOM_1__SHIFT 0x00000008
+#define PB1_RX_LANE1_SCI_STAT_OVRD_REG0__RESPONSEMODE_1_MASK 0x00000200L
+#define PB1_RX_LANE1_SCI_STAT_OVRD_REG0__RESPONSEMODE_1__SHIFT 0x00000009
+#define PB1_RX_LANE1_SCI_STAT_OVRD_REG0__RXPRESETHINT_1_MASK 0x00000070L
+#define PB1_RX_LANE1_SCI_STAT_OVRD_REG0__RXPRESETHINT_1__SHIFT 0x00000004
+#define PB1_RX_LANE1_SCI_STAT_OVRD_REG0__RXPWR_1_MASK 0x00000007L
+#define PB1_RX_LANE1_SCI_STAT_OVRD_REG0__RXPWR_1__SHIFT 0x00000000
+#define PB1_RX_LANE2_CTRL_REG0__RX_BACKUP_2_MASK 0x000000ffL
+#define PB1_RX_LANE2_CTRL_REG0__RX_BACKUP_2__SHIFT 0x00000000
+#define PB1_RX_LANE2_CTRL_REG0__RX_CFG_OVR_PWRSF_2_MASK 0x00002000L
+#define PB1_RX_LANE2_CTRL_REG0__RX_CFG_OVR_PWRSF_2__SHIFT 0x0000000d
+#define PB1_RX_LANE2_CTRL_REG0__RX_DBG_ANALOG_SEL_2_MASK 0x00000c00L
+#define PB1_RX_LANE2_CTRL_REG0__RX_DBG_ANALOG_SEL_2__SHIFT 0x0000000a
+#define PB1_RX_LANE2_CTRL_REG0__RX_TST_BSCAN_EN_2_MASK 0x00001000L
+#define PB1_RX_LANE2_CTRL_REG0__RX_TST_BSCAN_EN_2__SHIFT 0x0000000c
+#define PB1_RX_LANE2_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_2_MASK 0x00000008L
+#define PB1_RX_LANE2_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_2__SHIFT 0x00000003
+#define PB1_RX_LANE2_SCI_STAT_OVRD_REG0__ENABLEFOM_2_MASK 0x00000080L
+#define PB1_RX_LANE2_SCI_STAT_OVRD_REG0__ENABLEFOM_2__SHIFT 0x00000007
+#define PB1_RX_LANE2_SCI_STAT_OVRD_REG0__REQUESTFOM_2_MASK 0x00000100L
+#define PB1_RX_LANE2_SCI_STAT_OVRD_REG0__REQUESTFOM_2__SHIFT 0x00000008
+#define PB1_RX_LANE2_SCI_STAT_OVRD_REG0__RESPONSEMODE_2_MASK 0x00000200L
+#define PB1_RX_LANE2_SCI_STAT_OVRD_REG0__RESPONSEMODE_2__SHIFT 0x00000009
+#define PB1_RX_LANE2_SCI_STAT_OVRD_REG0__RXPRESETHINT_2_MASK 0x00000070L
+#define PB1_RX_LANE2_SCI_STAT_OVRD_REG0__RXPRESETHINT_2__SHIFT 0x00000004
+#define PB1_RX_LANE2_SCI_STAT_OVRD_REG0__RXPWR_2_MASK 0x00000007L
+#define PB1_RX_LANE2_SCI_STAT_OVRD_REG0__RXPWR_2__SHIFT 0x00000000
+#define PB1_RX_LANE3_CTRL_REG0__RX_BACKUP_3_MASK 0x000000ffL
+#define PB1_RX_LANE3_CTRL_REG0__RX_BACKUP_3__SHIFT 0x00000000
+#define PB1_RX_LANE3_CTRL_REG0__RX_CFG_OVR_PWRSF_3_MASK 0x00002000L
+#define PB1_RX_LANE3_CTRL_REG0__RX_CFG_OVR_PWRSF_3__SHIFT 0x0000000d
+#define PB1_RX_LANE3_CTRL_REG0__RX_DBG_ANALOG_SEL_3_MASK 0x00000c00L
+#define PB1_RX_LANE3_CTRL_REG0__RX_DBG_ANALOG_SEL_3__SHIFT 0x0000000a
+#define PB1_RX_LANE3_CTRL_REG0__RX_TST_BSCAN_EN_3_MASK 0x00001000L
+#define PB1_RX_LANE3_CTRL_REG0__RX_TST_BSCAN_EN_3__SHIFT 0x0000000c
+#define PB1_RX_LANE3_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_3_MASK 0x00000008L
+#define PB1_RX_LANE3_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_3__SHIFT 0x00000003
+#define PB1_RX_LANE3_SCI_STAT_OVRD_REG0__ENABLEFOM_3_MASK 0x00000080L
+#define PB1_RX_LANE3_SCI_STAT_OVRD_REG0__ENABLEFOM_3__SHIFT 0x00000007
+#define PB1_RX_LANE3_SCI_STAT_OVRD_REG0__REQUESTFOM_3_MASK 0x00000100L
+#define PB1_RX_LANE3_SCI_STAT_OVRD_REG0__REQUESTFOM_3__SHIFT 0x00000008
+#define PB1_RX_LANE3_SCI_STAT_OVRD_REG0__RESPONSEMODE_3_MASK 0x00000200L
+#define PB1_RX_LANE3_SCI_STAT_OVRD_REG0__RESPONSEMODE_3__SHIFT 0x00000009
+#define PB1_RX_LANE3_SCI_STAT_OVRD_REG0__RXPRESETHINT_3_MASK 0x00000070L
+#define PB1_RX_LANE3_SCI_STAT_OVRD_REG0__RXPRESETHINT_3__SHIFT 0x00000004
+#define PB1_RX_LANE3_SCI_STAT_OVRD_REG0__RXPWR_3_MASK 0x00000007L
+#define PB1_RX_LANE3_SCI_STAT_OVRD_REG0__RXPWR_3__SHIFT 0x00000000
+#define PB1_RX_LANE4_CTRL_REG0__RX_BACKUP_4_MASK 0x000000ffL
+#define PB1_RX_LANE4_CTRL_REG0__RX_BACKUP_4__SHIFT 0x00000000
+#define PB1_RX_LANE4_CTRL_REG0__RX_CFG_OVR_PWRSF_4_MASK 0x00002000L
+#define PB1_RX_LANE4_CTRL_REG0__RX_CFG_OVR_PWRSF_4__SHIFT 0x0000000d
+#define PB1_RX_LANE4_CTRL_REG0__RX_DBG_ANALOG_SEL_4_MASK 0x00000c00L
+#define PB1_RX_LANE4_CTRL_REG0__RX_DBG_ANALOG_SEL_4__SHIFT 0x0000000a
+#define PB1_RX_LANE4_CTRL_REG0__RX_TST_BSCAN_EN_4_MASK 0x00001000L
+#define PB1_RX_LANE4_CTRL_REG0__RX_TST_BSCAN_EN_4__SHIFT 0x0000000c
+#define PB1_RX_LANE4_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_4_MASK 0x00000008L
+#define PB1_RX_LANE4_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_4__SHIFT 0x00000003
+#define PB1_RX_LANE4_SCI_STAT_OVRD_REG0__ENABLEFOM_4_MASK 0x00000080L
+#define PB1_RX_LANE4_SCI_STAT_OVRD_REG0__ENABLEFOM_4__SHIFT 0x00000007
+#define PB1_RX_LANE4_SCI_STAT_OVRD_REG0__REQUESTFOM_4_MASK 0x00000100L
+#define PB1_RX_LANE4_SCI_STAT_OVRD_REG0__REQUESTFOM_4__SHIFT 0x00000008
+#define PB1_RX_LANE4_SCI_STAT_OVRD_REG0__RESPONSEMODE_4_MASK 0x00000200L
+#define PB1_RX_LANE4_SCI_STAT_OVRD_REG0__RESPONSEMODE_4__SHIFT 0x00000009
+#define PB1_RX_LANE4_SCI_STAT_OVRD_REG0__RXPRESETHINT_4_MASK 0x00000070L
+#define PB1_RX_LANE4_SCI_STAT_OVRD_REG0__RXPRESETHINT_4__SHIFT 0x00000004
+#define PB1_RX_LANE4_SCI_STAT_OVRD_REG0__RXPWR_4_MASK 0x00000007L
+#define PB1_RX_LANE4_SCI_STAT_OVRD_REG0__RXPWR_4__SHIFT 0x00000000
+#define PB1_RX_LANE5_CTRL_REG0__RX_BACKUP_5_MASK 0x000000ffL
+#define PB1_RX_LANE5_CTRL_REG0__RX_BACKUP_5__SHIFT 0x00000000
+#define PB1_RX_LANE5_CTRL_REG0__RX_CFG_OVR_PWRSF_5_MASK 0x00002000L
+#define PB1_RX_LANE5_CTRL_REG0__RX_CFG_OVR_PWRSF_5__SHIFT 0x0000000d
+#define PB1_RX_LANE5_CTRL_REG0__RX_DBG_ANALOG_SEL_5_MASK 0x00000c00L
+#define PB1_RX_LANE5_CTRL_REG0__RX_DBG_ANALOG_SEL_5__SHIFT 0x0000000a
+#define PB1_RX_LANE5_CTRL_REG0__RX_TST_BSCAN_EN_5_MASK 0x00001000L
+#define PB1_RX_LANE5_CTRL_REG0__RX_TST_BSCAN_EN_5__SHIFT 0x0000000c
+#define PB1_RX_LANE5_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_5_MASK 0x00000008L
+#define PB1_RX_LANE5_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_5__SHIFT 0x00000003
+#define PB1_RX_LANE5_SCI_STAT_OVRD_REG0__ENABLEFOM_5_MASK 0x00000080L
+#define PB1_RX_LANE5_SCI_STAT_OVRD_REG0__ENABLEFOM_5__SHIFT 0x00000007
+#define PB1_RX_LANE5_SCI_STAT_OVRD_REG0__REQUESTFOM_5_MASK 0x00000100L
+#define PB1_RX_LANE5_SCI_STAT_OVRD_REG0__REQUESTFOM_5__SHIFT 0x00000008
+#define PB1_RX_LANE5_SCI_STAT_OVRD_REG0__RESPONSEMODE_5_MASK 0x00000200L
+#define PB1_RX_LANE5_SCI_STAT_OVRD_REG0__RESPONSEMODE_5__SHIFT 0x00000009
+#define PB1_RX_LANE5_SCI_STAT_OVRD_REG0__RXPRESETHINT_5_MASK 0x00000070L
+#define PB1_RX_LANE5_SCI_STAT_OVRD_REG0__RXPRESETHINT_5__SHIFT 0x00000004
+#define PB1_RX_LANE5_SCI_STAT_OVRD_REG0__RXPWR_5_MASK 0x00000007L
+#define PB1_RX_LANE5_SCI_STAT_OVRD_REG0__RXPWR_5__SHIFT 0x00000000
+#define PB1_RX_LANE6_CTRL_REG0__RX_BACKUP_6_MASK 0x000000ffL
+#define PB1_RX_LANE6_CTRL_REG0__RX_BACKUP_6__SHIFT 0x00000000
+#define PB1_RX_LANE6_CTRL_REG0__RX_CFG_OVR_PWRSF_6_MASK 0x00002000L
+#define PB1_RX_LANE6_CTRL_REG0__RX_CFG_OVR_PWRSF_6__SHIFT 0x0000000d
+#define PB1_RX_LANE6_CTRL_REG0__RX_DBG_ANALOG_SEL_6_MASK 0x00000c00L
+#define PB1_RX_LANE6_CTRL_REG0__RX_DBG_ANALOG_SEL_6__SHIFT 0x0000000a
+#define PB1_RX_LANE6_CTRL_REG0__RX_TST_BSCAN_EN_6_MASK 0x00001000L
+#define PB1_RX_LANE6_CTRL_REG0__RX_TST_BSCAN_EN_6__SHIFT 0x0000000c
+#define PB1_RX_LANE6_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_6_MASK 0x00000008L
+#define PB1_RX_LANE6_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_6__SHIFT 0x00000003
+#define PB1_RX_LANE6_SCI_STAT_OVRD_REG0__ENABLEFOM_6_MASK 0x00000080L
+#define PB1_RX_LANE6_SCI_STAT_OVRD_REG0__ENABLEFOM_6__SHIFT 0x00000007
+#define PB1_RX_LANE6_SCI_STAT_OVRD_REG0__REQUESTFOM_6_MASK 0x00000100L
+#define PB1_RX_LANE6_SCI_STAT_OVRD_REG0__REQUESTFOM_6__SHIFT 0x00000008
+#define PB1_RX_LANE6_SCI_STAT_OVRD_REG0__RESPONSEMODE_6_MASK 0x00000200L
+#define PB1_RX_LANE6_SCI_STAT_OVRD_REG0__RESPONSEMODE_6__SHIFT 0x00000009
+#define PB1_RX_LANE6_SCI_STAT_OVRD_REG0__RXPRESETHINT_6_MASK 0x00000070L
+#define PB1_RX_LANE6_SCI_STAT_OVRD_REG0__RXPRESETHINT_6__SHIFT 0x00000004
+#define PB1_RX_LANE6_SCI_STAT_OVRD_REG0__RXPWR_6_MASK 0x00000007L
+#define PB1_RX_LANE6_SCI_STAT_OVRD_REG0__RXPWR_6__SHIFT 0x00000000
+#define PB1_RX_LANE7_CTRL_REG0__RX_BACKUP_7_MASK 0x000000ffL
+#define PB1_RX_LANE7_CTRL_REG0__RX_BACKUP_7__SHIFT 0x00000000
+#define PB1_RX_LANE7_CTRL_REG0__RX_CFG_OVR_PWRSF_7_MASK 0x00002000L
+#define PB1_RX_LANE7_CTRL_REG0__RX_CFG_OVR_PWRSF_7__SHIFT 0x0000000d
+#define PB1_RX_LANE7_CTRL_REG0__RX_DBG_ANALOG_SEL_7_MASK 0x00000c00L
+#define PB1_RX_LANE7_CTRL_REG0__RX_DBG_ANALOG_SEL_7__SHIFT 0x0000000a
+#define PB1_RX_LANE7_CTRL_REG0__RX_TST_BSCAN_EN_7_MASK 0x00001000L
+#define PB1_RX_LANE7_CTRL_REG0__RX_TST_BSCAN_EN_7__SHIFT 0x0000000c
+#define PB1_RX_LANE7_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_7_MASK 0x00000008L
+#define PB1_RX_LANE7_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_7__SHIFT 0x00000003
+#define PB1_RX_LANE7_SCI_STAT_OVRD_REG0__ENABLEFOM_7_MASK 0x00000080L
+#define PB1_RX_LANE7_SCI_STAT_OVRD_REG0__ENABLEFOM_7__SHIFT 0x00000007
+#define PB1_RX_LANE7_SCI_STAT_OVRD_REG0__REQUESTFOM_7_MASK 0x00000100L
+#define PB1_RX_LANE7_SCI_STAT_OVRD_REG0__REQUESTFOM_7__SHIFT 0x00000008
+#define PB1_RX_LANE7_SCI_STAT_OVRD_REG0__RESPONSEMODE_7_MASK 0x00000200L
+#define PB1_RX_LANE7_SCI_STAT_OVRD_REG0__RESPONSEMODE_7__SHIFT 0x00000009
+#define PB1_RX_LANE7_SCI_STAT_OVRD_REG0__RXPRESETHINT_7_MASK 0x00000070L
+#define PB1_RX_LANE7_SCI_STAT_OVRD_REG0__RXPRESETHINT_7__SHIFT 0x00000004
+#define PB1_RX_LANE7_SCI_STAT_OVRD_REG0__RXPWR_7_MASK 0x00000007L
+#define PB1_RX_LANE7_SCI_STAT_OVRD_REG0__RXPWR_7__SHIFT 0x00000000
+#define PB1_RX_LANE8_CTRL_REG0__RX_BACKUP_8_MASK 0x000000ffL
+#define PB1_RX_LANE8_CTRL_REG0__RX_BACKUP_8__SHIFT 0x00000000
+#define PB1_RX_LANE8_CTRL_REG0__RX_CFG_OVR_PWRSF_8_MASK 0x00002000L
+#define PB1_RX_LANE8_CTRL_REG0__RX_CFG_OVR_PWRSF_8__SHIFT 0x0000000d
+#define PB1_RX_LANE8_CTRL_REG0__RX_DBG_ANALOG_SEL_8_MASK 0x00000c00L
+#define PB1_RX_LANE8_CTRL_REG0__RX_DBG_ANALOG_SEL_8__SHIFT 0x0000000a
+#define PB1_RX_LANE8_CTRL_REG0__RX_TST_BSCAN_EN_8_MASK 0x00001000L
+#define PB1_RX_LANE8_CTRL_REG0__RX_TST_BSCAN_EN_8__SHIFT 0x0000000c
+#define PB1_RX_LANE8_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_8_MASK 0x00000008L
+#define PB1_RX_LANE8_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_8__SHIFT 0x00000003
+#define PB1_RX_LANE8_SCI_STAT_OVRD_REG0__ENABLEFOM_8_MASK 0x00000080L
+#define PB1_RX_LANE8_SCI_STAT_OVRD_REG0__ENABLEFOM_8__SHIFT 0x00000007
+#define PB1_RX_LANE8_SCI_STAT_OVRD_REG0__REQUESTFOM_8_MASK 0x00000100L
+#define PB1_RX_LANE8_SCI_STAT_OVRD_REG0__REQUESTFOM_8__SHIFT 0x00000008
+#define PB1_RX_LANE8_SCI_STAT_OVRD_REG0__RESPONSEMODE_8_MASK 0x00000200L
+#define PB1_RX_LANE8_SCI_STAT_OVRD_REG0__RESPONSEMODE_8__SHIFT 0x00000009
+#define PB1_RX_LANE8_SCI_STAT_OVRD_REG0__RXPRESETHINT_8_MASK 0x00000070L
+#define PB1_RX_LANE8_SCI_STAT_OVRD_REG0__RXPRESETHINT_8__SHIFT 0x00000004
+#define PB1_RX_LANE8_SCI_STAT_OVRD_REG0__RXPWR_8_MASK 0x00000007L
+#define PB1_RX_LANE8_SCI_STAT_OVRD_REG0__RXPWR_8__SHIFT 0x00000000
+#define PB1_RX_LANE9_CTRL_REG0__RX_BACKUP_9_MASK 0x000000ffL
+#define PB1_RX_LANE9_CTRL_REG0__RX_BACKUP_9__SHIFT 0x00000000
+#define PB1_RX_LANE9_CTRL_REG0__RX_CFG_OVR_PWRSF_9_MASK 0x00002000L
+#define PB1_RX_LANE9_CTRL_REG0__RX_CFG_OVR_PWRSF_9__SHIFT 0x0000000d
+#define PB1_RX_LANE9_CTRL_REG0__RX_DBG_ANALOG_SEL_9_MASK 0x00000c00L
+#define PB1_RX_LANE9_CTRL_REG0__RX_DBG_ANALOG_SEL_9__SHIFT 0x0000000a
+#define PB1_RX_LANE9_CTRL_REG0__RX_TST_BSCAN_EN_9_MASK 0x00001000L
+#define PB1_RX_LANE9_CTRL_REG0__RX_TST_BSCAN_EN_9__SHIFT 0x0000000c
+#define PB1_RX_LANE9_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_9_MASK 0x00000008L
+#define PB1_RX_LANE9_SCI_STAT_OVRD_REG0__ELECIDLEDETEN_9__SHIFT 0x00000003
+#define PB1_RX_LANE9_SCI_STAT_OVRD_REG0__ENABLEFOM_9_MASK 0x00000080L
+#define PB1_RX_LANE9_SCI_STAT_OVRD_REG0__ENABLEFOM_9__SHIFT 0x00000007
+#define PB1_RX_LANE9_SCI_STAT_OVRD_REG0__REQUESTFOM_9_MASK 0x00000100L
+#define PB1_RX_LANE9_SCI_STAT_OVRD_REG0__REQUESTFOM_9__SHIFT 0x00000008
+#define PB1_RX_LANE9_SCI_STAT_OVRD_REG0__RESPONSEMODE_9_MASK 0x00000200L
+#define PB1_RX_LANE9_SCI_STAT_OVRD_REG0__RESPONSEMODE_9__SHIFT 0x00000009
+#define PB1_RX_LANE9_SCI_STAT_OVRD_REG0__RXPRESETHINT_9_MASK 0x00000070L
+#define PB1_RX_LANE9_SCI_STAT_OVRD_REG0__RXPRESETHINT_9__SHIFT 0x00000004
+#define PB1_RX_LANE9_SCI_STAT_OVRD_REG0__RXPWR_9_MASK 0x00000007L
+#define PB1_RX_LANE9_SCI_STAT_OVRD_REG0__RXPWR_9__SHIFT 0x00000000
+#define PB1_STRAP_GLB_REG0__STRAP_DBG_RXDLL_VREG_REF_SEL_MASK 0x00008000L
+#define PB1_STRAP_GLB_REG0__STRAP_DBG_RXDLL_VREG_REF_SEL__SHIFT 0x0000000f
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_0_MASK 0x00000001L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_0__SHIFT 0x00000000
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_10_MASK 0x00000400L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_10__SHIFT 0x0000000a
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_11_MASK 0x00000800L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_11__SHIFT 0x0000000b
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_12_MASK 0x00001000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_12__SHIFT 0x0000000c
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_13_MASK 0x00002000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_13__SHIFT 0x0000000d
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_14_MASK 0x00004000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_14__SHIFT 0x0000000e
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_15_MASK 0x00008000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_15__SHIFT 0x0000000f
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_16_MASK 0x00010000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_16__SHIFT 0x00000010
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_17_MASK 0x00020000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_17__SHIFT 0x00000011
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_18_MASK 0x00040000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_18__SHIFT 0x00000012
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_19_MASK 0x00080000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_19__SHIFT 0x00000013
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_1_MASK 0x00000002L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_1__SHIFT 0x00000001
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_20_MASK 0x00100000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_20__SHIFT 0x00000014
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_21_MASK 0x00200000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_21__SHIFT 0x00000015
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_22_MASK 0x00400000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_22__SHIFT 0x00000016
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_23_MASK 0x00800000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_23__SHIFT 0x00000017
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_24_MASK 0x01000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_24__SHIFT 0x00000018
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_25_MASK 0x02000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_25__SHIFT 0x00000019
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_26_MASK 0x04000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_26__SHIFT 0x0000001a
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_27_MASK 0x08000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_27__SHIFT 0x0000001b
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_28_MASK 0x10000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_28__SHIFT 0x0000001c
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_29_MASK 0x20000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_29__SHIFT 0x0000001d
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_2_MASK 0x00000004L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_2__SHIFT 0x00000002
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_30_MASK 0x40000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_30__SHIFT 0x0000001e
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_31_MASK 0x80000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_31__SHIFT 0x0000001f
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_3_MASK 0x00000008L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_3__SHIFT 0x00000003
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_4_MASK 0x00000010L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_4__SHIFT 0x00000004
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_5_MASK 0x00000020L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_5__SHIFT 0x00000005
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_6_MASK 0x00000040L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_6__SHIFT 0x00000006
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_7_MASK 0x00000080L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_7__SHIFT 0x00000007
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_8_MASK 0x00000100L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_8__SHIFT 0x00000008
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_9_MASK 0x00000200L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG0__ACCEPT_ENTRY_9__SHIFT 0x00000009
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_32_MASK 0x00000001L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_32__SHIFT 0x00000000
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_33_MASK 0x00000002L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_33__SHIFT 0x00000001
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_34_MASK 0x00000004L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_34__SHIFT 0x00000002
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_35_MASK 0x00000008L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_35__SHIFT 0x00000003
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_36_MASK 0x00000010L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_36__SHIFT 0x00000004
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_37_MASK 0x00000020L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_37__SHIFT 0x00000005
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_38_MASK 0x00000040L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_38__SHIFT 0x00000006
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_39_MASK 0x00000080L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_39__SHIFT 0x00000007
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_40_MASK 0x00000100L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_40__SHIFT 0x00000008
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_41_MASK 0x00000200L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_41__SHIFT 0x00000009
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_42_MASK 0x00000400L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_42__SHIFT 0x0000000a
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_43_MASK 0x00000800L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_43__SHIFT 0x0000000b
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_44_MASK 0x00001000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_44__SHIFT 0x0000000c
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_45_MASK 0x00002000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_45__SHIFT 0x0000000d
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_46_MASK 0x00004000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_46__SHIFT 0x0000000e
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_47_MASK 0x00008000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_47__SHIFT 0x0000000f
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_48_MASK 0x00010000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_48__SHIFT 0x00000010
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_49_MASK 0x00020000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_49__SHIFT 0x00000011
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_50_MASK 0x00040000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_50__SHIFT 0x00000012
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_51_MASK 0x00080000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_51__SHIFT 0x00000013
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_52_MASK 0x00100000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_52__SHIFT 0x00000014
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_53_MASK 0x00200000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_53__SHIFT 0x00000015
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_54_MASK 0x00400000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_54__SHIFT 0x00000016
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_55_MASK 0x00800000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_55__SHIFT 0x00000017
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_56_MASK 0x01000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_56__SHIFT 0x00000018
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_57_MASK 0x02000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_57__SHIFT 0x00000019
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_58_MASK 0x04000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_58__SHIFT 0x0000001a
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_59_MASK 0x08000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_59__SHIFT 0x0000001b
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_60_MASK 0x10000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_60__SHIFT 0x0000001c
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_61_MASK 0x20000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_61__SHIFT 0x0000001d
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_62_MASK 0x40000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_62__SHIFT 0x0000001e
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_63_MASK 0x80000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG1__ACCEPT_ENTRY_63__SHIFT 0x0000001f
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_64_MASK 0x00000001L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_64__SHIFT 0x00000000
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_65_MASK 0x00000002L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_65__SHIFT 0x00000001
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_66_MASK 0x00000004L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_66__SHIFT 0x00000002
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_67_MASK 0x00000008L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_67__SHIFT 0x00000003
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_68_MASK 0x00000010L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_68__SHIFT 0x00000004
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_69_MASK 0x00000020L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_69__SHIFT 0x00000005
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_70_MASK 0x00000040L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_70__SHIFT 0x00000006
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_71_MASK 0x00000080L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_71__SHIFT 0x00000007
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_72_MASK 0x00000100L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_72__SHIFT 0x00000008
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_73_MASK 0x00000200L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_73__SHIFT 0x00000009
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_74_MASK 0x00000400L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_74__SHIFT 0x0000000a
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_75_MASK 0x00000800L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_75__SHIFT 0x0000000b
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_76_MASK 0x00001000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_76__SHIFT 0x0000000c
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_77_MASK 0x00002000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_77__SHIFT 0x0000000d
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_78_MASK 0x00004000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_78__SHIFT 0x0000000e
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_79_MASK 0x00008000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_79__SHIFT 0x0000000f
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_80_MASK 0x00010000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_80__SHIFT 0x00000010
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_81_MASK 0x00020000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_81__SHIFT 0x00000011
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_82_MASK 0x00040000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_82__SHIFT 0x00000012
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_83_MASK 0x00080000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_83__SHIFT 0x00000013
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_84_MASK 0x00100000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_84__SHIFT 0x00000014
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_85_MASK 0x00200000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_85__SHIFT 0x00000015
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_86_MASK 0x00400000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_86__SHIFT 0x00000016
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_87_MASK 0x00800000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_87__SHIFT 0x00000017
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_88_MASK 0x01000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_88__SHIFT 0x00000018
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_89_MASK 0x02000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_89__SHIFT 0x00000019
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_90_MASK 0x04000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_90__SHIFT 0x0000001a
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_91_MASK 0x08000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_91__SHIFT 0x0000001b
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_92_MASK 0x10000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_92__SHIFT 0x0000001c
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_93_MASK 0x20000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_93__SHIFT 0x0000001d
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_94_MASK 0x40000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_94__SHIFT 0x0000001e
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_95_MASK 0x80000000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG2__ACCEPT_ENTRY_95__SHIFT 0x0000001f
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_100_MASK 0x00000010L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_100__SHIFT 0x00000004
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_101_MASK 0x00000020L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_101__SHIFT 0x00000005
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_102_MASK 0x00000040L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_102__SHIFT 0x00000006
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_103_MASK 0x00000080L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_103__SHIFT 0x00000007
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_104_MASK 0x00000100L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_104__SHIFT 0x00000008
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_105_MASK 0x00000200L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_105__SHIFT 0x00000009
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_106_MASK 0x00000400L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_106__SHIFT 0x0000000a
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_107_MASK 0x00000800L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_107__SHIFT 0x0000000b
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_108_MASK 0x00001000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_108__SHIFT 0x0000000c
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_109_MASK 0x00002000L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_109__SHIFT 0x0000000d
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_96_MASK 0x00000001L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_96__SHIFT 0x00000000
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_97_MASK 0x00000002L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_97__SHIFT 0x00000001
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_98_MASK 0x00000004L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_98__SHIFT 0x00000002
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_99_MASK 0x00000008L
+#define PB1_TX_GLB_COEFF_ACCEPT_TABLE_REG3__ACCEPT_ENTRY_99__SHIFT 0x00000003
+#define PB1_TX_GLB_CTRL_REG0__TX_CFG_RPTR_RST_VAL_GEN1_MASK 0x00000700L
+#define PB1_TX_GLB_CTRL_REG0__TX_CFG_RPTR_RST_VAL_GEN1__SHIFT 0x00000008
+#define PB1_TX_GLB_CTRL_REG0__TX_CFG_RPTR_RST_VAL_GEN2_MASK 0x00003800L
+#define PB1_TX_GLB_CTRL_REG0__TX_CFG_RPTR_RST_VAL_GEN2__SHIFT 0x0000000b
+#define PB1_TX_GLB_CTRL_REG0__TX_CFG_RPTR_RST_VAL_GEN3_MASK 0x0001c000L
+#define PB1_TX_GLB_CTRL_REG0__TX_CFG_RPTR_RST_VAL_GEN3__SHIFT 0x0000000e
+#define PB1_TX_GLB_CTRL_REG0__TX_COEFF_ROUND_DIR_VER_MASK 0x00400000L
+#define PB1_TX_GLB_CTRL_REG0__TX_COEFF_ROUND_DIR_VER__SHIFT 0x00000016
+#define PB1_TX_GLB_CTRL_REG0__TX_COEFF_ROUND_EN_MASK 0x00200000L
+#define PB1_TX_GLB_CTRL_REG0__TX_COEFF_ROUND_EN__SHIFT 0x00000015
+#define PB1_TX_GLB_CTRL_REG0__TX_DATA_CLK_GATING_MASK 0x00080000L
+#define PB1_TX_GLB_CTRL_REG0__TX_DATA_CLK_GATING__SHIFT 0x00000013
+#define PB1_TX_GLB_CTRL_REG0__TX_DCLK_EN_LSX_ALWAYS_ON_MASK 0x00800000L
+#define PB1_TX_GLB_CTRL_REG0__TX_DCLK_EN_LSX_ALWAYS_ON__SHIFT 0x00000017
+#define PB1_TX_GLB_CTRL_REG0__TX_DRV_DATA_ASRT_DLY_VAL_MASK 0x00000007L
+#define PB1_TX_GLB_CTRL_REG0__TX_DRV_DATA_ASRT_DLY_VAL__SHIFT 0x00000000
+#define PB1_TX_GLB_CTRL_REG0__TX_DRV_DATA_DSRT_DLY_VAL_MASK 0x00000038L
+#define PB1_TX_GLB_CTRL_REG0__TX_DRV_DATA_DSRT_DLY_VAL__SHIFT 0x00000003
+#define PB1_TX_GLB_CTRL_REG0__TX_FRONTEND_PWRON_IN_OFF_MASK 0x01000000L
+#define PB1_TX_GLB_CTRL_REG0__TX_FRONTEND_PWRON_IN_OFF__SHIFT 0x00000018
+#define PB1_TX_GLB_CTRL_REG0__TX_PRESET_TABLE_BYPASS_MASK 0x00100000L
+#define PB1_TX_GLB_CTRL_REG0__TX_PRESET_TABLE_BYPASS__SHIFT 0x00000014
+#define PB1_TX_GLB_CTRL_REG0__TX_STAGGER_CTRL_MASK 0x00060000L
+#define PB1_TX_GLB_CTRL_REG0__TX_STAGGER_CTRL__SHIFT 0x00000011
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX16_EN_L0T15_MASK 0x40000000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX16_EN_L0T15__SHIFT 0x0000001e
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_0_MASK 0x00000001L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_0__SHIFT 0x00000000
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_10_MASK 0x00000400L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_10__SHIFT 0x0000000a
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_11_MASK 0x00000800L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_11__SHIFT 0x0000000b
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_12_MASK 0x00001000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_12__SHIFT 0x0000000c
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_13_MASK 0x00002000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_13__SHIFT 0x0000000d
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_14_MASK 0x00004000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_14__SHIFT 0x0000000e
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_15_MASK 0x00008000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_15__SHIFT 0x0000000f
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_1_MASK 0x00000002L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_1__SHIFT 0x00000001
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_2_MASK 0x00000004L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_2__SHIFT 0x00000002
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_3_MASK 0x00000008L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_3__SHIFT 0x00000003
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_4_MASK 0x00000010L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_4__SHIFT 0x00000004
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_5_MASK 0x00000020L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_5__SHIFT 0x00000005
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_6_MASK 0x00000040L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_6__SHIFT 0x00000006
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_7_MASK 0x00000080L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_7__SHIFT 0x00000007
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_8_MASK 0x00000100L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_8__SHIFT 0x00000008
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_9_MASK 0x00000200L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX1_EN_9__SHIFT 0x00000009
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L0T1_MASK 0x00010000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L0T1__SHIFT 0x00000010
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L10T11_MASK 0x00200000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L10T11__SHIFT 0x00000015
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L12T13_MASK 0x00400000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L12T13__SHIFT 0x00000016
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L14T15_MASK 0x00800000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L14T15__SHIFT 0x00000017
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L2T3_MASK 0x00020000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L2T3__SHIFT 0x00000011
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L4T5_MASK 0x00040000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L4T5__SHIFT 0x00000012
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L6T7_MASK 0x00080000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L6T7__SHIFT 0x00000013
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L8T9_MASK 0x00100000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX2_EN_L8T9__SHIFT 0x00000014
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L0T3_MASK 0x01000000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L0T3__SHIFT 0x00000018
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L12T15_MASK 0x08000000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L12T15__SHIFT 0x0000001b
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L4T7_MASK 0x02000000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L4T7__SHIFT 0x00000019
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L8T11_MASK 0x04000000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX4_EN_L8T11__SHIFT 0x0000001a
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX8_EN_L0T7_MASK 0x10000000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX8_EN_L0T7__SHIFT 0x0000001c
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX8_EN_L8T15_MASK 0x20000000L
+#define PB1_TX_GLB_LANE_SKEW_CTRL__TX_CFG_GROUPX8_EN_L8T15__SHIFT 0x0000001d
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DCLK_DIV_OVRD_EN_MASK 0x00000008L
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DCLK_DIV_OVRD_EN__SHIFT 0x00000003
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DCLK_DIV_OVRD_VAL_MASK 0x00000007L
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DCLK_DIV_OVRD_VAL__SHIFT 0x00000000
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV0_EN_GEN1_OVRD_VAL_MASK 0x000000f0L
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV0_EN_GEN1_OVRD_VAL__SHIFT 0x00000004
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV0_EN_OVRD_EN_MASK 0x00000100L
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV0_EN_OVRD_EN__SHIFT 0x00000008
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV0_TAP_SEL_GEN1_OVRD_VAL_MASK 0x00001e00L
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV0_TAP_SEL_GEN1_OVRD_VAL__SHIFT 0x00000009
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV0_TAP_SEL_OVRD_EN_MASK 0x00002000L
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV0_TAP_SEL_OVRD_EN__SHIFT 0x0000000d
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV1_EN_GEN1_OVRD_VAL_MASK 0x0007c000L
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV1_EN_GEN1_OVRD_VAL__SHIFT 0x0000000e
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV1_EN_OVRD_EN_MASK 0x00080000L
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV1_EN_OVRD_EN__SHIFT 0x00000013
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV1_TAP_SEL_GEN1_OVRD_VAL_MASK 0x01f00000L
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV1_TAP_SEL_GEN1_OVRD_VAL__SHIFT 0x00000014
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV1_TAP_SEL_OVRD_EN_MASK 0x02000000L
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV1_TAP_SEL_OVRD_EN__SHIFT 0x00000019
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV2_EN_GEN1_OVRD_VAL_MASK 0x3c000000L
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV2_EN_GEN1_OVRD_VAL__SHIFT 0x0000001a
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV2_EN_OVRD_EN_MASK 0x40000000L
+#define PB1_TX_GLB_OVRD_REG0__TX_CFG_DRV2_EN_OVRD_EN__SHIFT 0x0000001e
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_DRV2_TAP_SEL_GEN1_OVRD_VAL_MASK 0x0000000fL
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_DRV2_TAP_SEL_GEN1_OVRD_VAL__SHIFT 0x00000000
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_DRV2_TAP_SEL_OVRD_EN_MASK 0x00000010L
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_DRV2_TAP_SEL_OVRD_EN__SHIFT 0x00000004
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_DRVX_EN_GEN1_OVRD_VAL_MASK 0x00000020L
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_DRVX_EN_GEN1_OVRD_VAL__SHIFT 0x00000005
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_DRVX_EN_OVRD_EN_MASK 0x00000040L
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_DRVX_EN_OVRD_EN__SHIFT 0x00000006
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_DRVX_TAP_SEL_GEN1_OVRD_VAL_MASK 0x00000080L
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_DRVX_TAP_SEL_GEN1_OVRD_VAL__SHIFT 0x00000007
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_DRVX_TAP_SEL_OVRD_EN_MASK 0x00000100L
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_DRVX_TAP_SEL_OVRD_EN__SHIFT 0x00000008
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_PLLCLK_SEL_OVRD_EN_MASK 0x00000400L
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_PLLCLK_SEL_OVRD_EN__SHIFT 0x0000000a
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_PLLCLK_SEL_OVRD_VAL_MASK 0x00000200L
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_PLLCLK_SEL_OVRD_VAL__SHIFT 0x00000009
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_TCLK_DIV_OVRD_EN_MASK 0x00001000L
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_TCLK_DIV_OVRD_EN__SHIFT 0x0000000c
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_TCLK_DIV_OVRD_VAL_MASK 0x00000800L
+#define PB1_TX_GLB_OVRD_REG1__TX_CFG_TCLK_DIV_OVRD_VAL__SHIFT 0x0000000b
+#define PB1_TX_GLB_OVRD_REG1__TX_CMDET_EN_OVRD_EN_MASK 0x00004000L
+#define PB1_TX_GLB_OVRD_REG1__TX_CMDET_EN_OVRD_EN__SHIFT 0x0000000e
+#define PB1_TX_GLB_OVRD_REG1__TX_CMDET_EN_OVRD_VAL_MASK 0x00002000L
+#define PB1_TX_GLB_OVRD_REG1__TX_CMDET_EN_OVRD_VAL__SHIFT 0x0000000d
+#define PB1_TX_GLB_OVRD_REG1__TX_DATA_IN_OVRD_EN_MASK 0x02000000L
+#define PB1_TX_GLB_OVRD_REG1__TX_DATA_IN_OVRD_EN__SHIFT 0x00000019
+#define PB1_TX_GLB_OVRD_REG1__TX_DATA_IN_OVRD_VAL_MASK 0x01ff8000L
+#define PB1_TX_GLB_OVRD_REG1__TX_DATA_IN_OVRD_VAL__SHIFT 0x0000000f
+#define PB1_TX_GLB_OVRD_REG1__TX_RPTR_RSTN_OVRD_EN_MASK 0x08000000L
+#define PB1_TX_GLB_OVRD_REG1__TX_RPTR_RSTN_OVRD_EN__SHIFT 0x0000001b
+#define PB1_TX_GLB_OVRD_REG1__TX_RPTR_RSTN_OVRD_VAL_MASK 0x04000000L
+#define PB1_TX_GLB_OVRD_REG1__TX_RPTR_RSTN_OVRD_VAL__SHIFT 0x0000001a
+#define PB1_TX_GLB_OVRD_REG1__TX_RXDET_EN_OVRD_EN_MASK 0x20000000L
+#define PB1_TX_GLB_OVRD_REG1__TX_RXDET_EN_OVRD_EN__SHIFT 0x0000001d
+#define PB1_TX_GLB_OVRD_REG1__TX_RXDET_EN_OVRD_VAL_MASK 0x10000000L
+#define PB1_TX_GLB_OVRD_REG1__TX_RXDET_EN_OVRD_VAL__SHIFT 0x0000001c
+#define PB1_TX_GLB_OVRD_REG1__TX_WPTR_RSTN_OVRD_EN_MASK 0x80000000L
+#define PB1_TX_GLB_OVRD_REG1__TX_WPTR_RSTN_OVRD_EN__SHIFT 0x0000001f
+#define PB1_TX_GLB_OVRD_REG1__TX_WPTR_RSTN_OVRD_VAL_MASK 0x40000000L
+#define PB1_TX_GLB_OVRD_REG1__TX_WPTR_RSTN_OVRD_VAL__SHIFT 0x0000001e
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_DRV0_EN_GEN2_OVRD_VAL_MASK 0x0000f000L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_DRV0_EN_GEN2_OVRD_VAL__SHIFT 0x0000000c
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_DRV0_TAP_SEL_GEN2_OVRD_VAL_MASK 0x000f0000L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_DRV0_TAP_SEL_GEN2_OVRD_VAL__SHIFT 0x00000010
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_DRV1_EN_GEN2_OVRD_VAL_MASK 0x01f00000L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_DRV1_EN_GEN2_OVRD_VAL__SHIFT 0x00000014
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_DRV1_TAP_SEL_GEN2_OVRD_VAL_MASK 0x3e000000L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_DRV1_TAP_SEL_GEN2_OVRD_VAL__SHIFT 0x00000019
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX16_EN_OVRD_EN_MASK 0x00000800L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX16_EN_OVRD_EN__SHIFT 0x0000000b
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX16_EN_OVRD_VAL_MASK 0x00000400L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX16_EN_OVRD_VAL__SHIFT 0x0000000a
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX1_EN_OVRD_EN_MASK 0x00000008L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX1_EN_OVRD_EN__SHIFT 0x00000003
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX1_EN_OVRD_VAL_MASK 0x00000004L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX1_EN_OVRD_VAL__SHIFT 0x00000002
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX2_EN_OVRD_EN_MASK 0x00000020L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX2_EN_OVRD_EN__SHIFT 0x00000005
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX2_EN_OVRD_VAL_MASK 0x00000010L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX2_EN_OVRD_VAL__SHIFT 0x00000004
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX4_EN_OVRD_EN_MASK 0x00000080L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX4_EN_OVRD_EN__SHIFT 0x00000007
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX4_EN_OVRD_VAL_MASK 0x00000040L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX4_EN_OVRD_VAL__SHIFT 0x00000006
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX8_EN_OVRD_EN_MASK 0x00000200L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX8_EN_OVRD_EN__SHIFT 0x00000009
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX8_EN_OVRD_VAL_MASK 0x00000100L
+#define PB1_TX_GLB_OVRD_REG2__TX_CFG_GROUPX8_EN_OVRD_VAL__SHIFT 0x00000008
+#define PB1_TX_GLB_OVRD_REG2__TX_WRITE_EN_OVRD_EN_MASK 0x00000002L
+#define PB1_TX_GLB_OVRD_REG2__TX_WRITE_EN_OVRD_EN__SHIFT 0x00000001
+#define PB1_TX_GLB_OVRD_REG2__TX_WRITE_EN_OVRD_VAL_MASK 0x00000001L
+#define PB1_TX_GLB_OVRD_REG2__TX_WRITE_EN_OVRD_VAL__SHIFT 0x00000000
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV0_EN_GEN3_OVRD_VAL_MASK 0x00003c00L
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV0_EN_GEN3_OVRD_VAL__SHIFT 0x0000000a
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV0_TAP_SEL_GEN3_OVRD_VAL_MASK 0x0003c000L
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV0_TAP_SEL_GEN3_OVRD_VAL__SHIFT 0x0000000e
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV1_EN_GEN3_OVRD_VAL_MASK 0x007c0000L
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV1_EN_GEN3_OVRD_VAL__SHIFT 0x00000012
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV1_TAP_SEL_GEN3_OVRD_VAL_MASK 0x0f800000L
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV1_TAP_SEL_GEN3_OVRD_VAL__SHIFT 0x00000017
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV2_EN_GEN2_OVRD_VAL_MASK 0x0000000fL
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV2_EN_GEN2_OVRD_VAL__SHIFT 0x00000000
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV2_EN_GEN3_OVRD_VAL_MASK 0xf0000000L
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV2_EN_GEN3_OVRD_VAL__SHIFT 0x0000001c
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV2_TAP_SEL_GEN2_OVRD_VAL_MASK 0x000000f0L
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRV2_TAP_SEL_GEN2_OVRD_VAL__SHIFT 0x00000004
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRVX_EN_GEN2_OVRD_VAL_MASK 0x00000100L
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRVX_EN_GEN2_OVRD_VAL__SHIFT 0x00000008
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRVX_TAP_SEL_GEN2_OVRD_VAL_MASK 0x00000200L
+#define PB1_TX_GLB_OVRD_REG3__TX_CFG_DRVX_TAP_SEL_GEN2_OVRD_VAL__SHIFT 0x00000009
+#define PB1_TX_GLB_OVRD_REG4__TX_CFG_DRV2_TAP_SEL_GEN3_OVRD_VAL_MASK 0x0000000fL
+#define PB1_TX_GLB_OVRD_REG4__TX_CFG_DRV2_TAP_SEL_GEN3_OVRD_VAL__SHIFT 0x00000000
+#define PB1_TX_GLB_OVRD_REG4__TX_CFG_DRVX_EN_GEN3_OVRD_VAL_MASK 0x00000010L
+#define PB1_TX_GLB_OVRD_REG4__TX_CFG_DRVX_EN_GEN3_OVRD_VAL__SHIFT 0x00000004
+#define PB1_TX_GLB_OVRD_REG4__TX_CFG_DRVX_TAP_SEL_GEN3_OVRD_VAL_MASK 0x00000020L
+#define PB1_TX_GLB_OVRD_REG4__TX_CFG_DRVX_TAP_SEL_GEN3_OVRD_VAL__SHIFT 0x00000005
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L0T3_MASK 0x00000100L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L0T3__SHIFT 0x00000008
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L12T15_MASK 0x00000800L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L12T15__SHIFT 0x0000000b
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L4T7_MASK 0x00000200L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L4T7__SHIFT 0x00000009
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L8T11_MASK 0x00000400L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENTID_SCI_UPDT_L8T11__SHIFT 0x0000000a
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L0T3_MASK 0x00001000L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L0T3__SHIFT 0x0000000c
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L12T15_MASK 0x00008000L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L12T15__SHIFT 0x0000000f
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L4T7_MASK 0x00002000L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L4T7__SHIFT 0x0000000d
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L8T11_MASK 0x00004000L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_COEFFICIENT_SCI_UPDT_L8T11__SHIFT 0x0000000e
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L0T3_MASK 0x00000010L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L0T3__SHIFT 0x00000004
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L12T15_MASK 0x00000080L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L12T15__SHIFT 0x00000007
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L4T7_MASK 0x00000020L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L4T7__SHIFT 0x00000005
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L8T11_MASK 0x00000040L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_INCOHERENTCK_SCI_UPDT_L8T11__SHIFT 0x00000006
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L0T3_MASK 0x00000001L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L0T3__SHIFT 0x00000000
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L12T15_MASK 0x00000008L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L12T15__SHIFT 0x00000003
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L4T7_MASK 0x00000002L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L4T7__SHIFT 0x00000001
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L8T11_MASK 0x00000004L
+#define PB1_TX_GLB_SCI_STAT_OVRD_REG0__IGNR_TXPWR_SCI_UPDT_L8T11__SHIFT 0x00000002
+#define PB1_TX_LANE0_CTRL_REG0__TX_CFG_DISPCLK_MODE_0_MASK 0x00000001L
+#define PB1_TX_LANE0_CTRL_REG0__TX_CFG_DISPCLK_MODE_0__SHIFT 0x00000000
+#define PB1_TX_LANE0_CTRL_REG0__TX_CFG_INV_DATA_0_MASK 0x00000002L
+#define PB1_TX_LANE0_CTRL_REG0__TX_CFG_INV_DATA_0__SHIFT 0x00000001
+#define PB1_TX_LANE0_CTRL_REG0__TX_CFG_SWING_BOOST_EN_0_MASK 0x00000004L
+#define PB1_TX_LANE0_CTRL_REG0__TX_CFG_SWING_BOOST_EN_0__SHIFT 0x00000002
+#define PB1_TX_LANE0_CTRL_REG0__TX_DBG_PRBS_EN_0_MASK 0x00000008L
+#define PB1_TX_LANE0_CTRL_REG0__TX_DBG_PRBS_EN_0__SHIFT 0x00000003
+#define PB1_TX_LANE0_OVRD_REG0__TX_DCLK_EN_OVRD_EN_0_MASK 0x00000002L
+#define PB1_TX_LANE0_OVRD_REG0__TX_DCLK_EN_OVRD_EN_0__SHIFT 0x00000001
+#define PB1_TX_LANE0_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_0_MASK 0x00000001L
+#define PB1_TX_LANE0_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_0__SHIFT 0x00000000
+#define PB1_TX_LANE0_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_0_MASK 0x00000008L
+#define PB1_TX_LANE0_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_0__SHIFT 0x00000003
+#define PB1_TX_LANE0_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_0_MASK 0x00000004L
+#define PB1_TX_LANE0_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_0__SHIFT 0x00000002
+#define PB1_TX_LANE0_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_0_MASK 0x00000020L
+#define PB1_TX_LANE0_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_0__SHIFT 0x00000005
+#define PB1_TX_LANE0_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_0_MASK 0x00000010L
+#define PB1_TX_LANE0_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_0__SHIFT 0x00000004
+#define PB1_TX_LANE0_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_0_MASK 0x00000080L
+#define PB1_TX_LANE0_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_0__SHIFT 0x00000007
+#define PB1_TX_LANE0_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_0_MASK 0x00000040L
+#define PB1_TX_LANE0_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_0__SHIFT 0x00000006
+#define PB1_TX_LANE0_SCI_STAT_OVRD_REG0__COEFFICIENT_0_MASK 0x0000fc00L
+#define PB1_TX_LANE0_SCI_STAT_OVRD_REG0__COEFFICIENT_0__SHIFT 0x0000000a
+#define PB1_TX_LANE0_SCI_STAT_OVRD_REG0__COEFFICIENTID_0_MASK 0x00000300L
+#define PB1_TX_LANE0_SCI_STAT_OVRD_REG0__COEFFICIENTID_0__SHIFT 0x00000008
+#define PB1_TX_LANE0_SCI_STAT_OVRD_REG0__DEEMPH_0_MASK 0x00000080L
+#define PB1_TX_LANE0_SCI_STAT_OVRD_REG0__DEEMPH_0__SHIFT 0x00000007
+#define PB1_TX_LANE0_SCI_STAT_OVRD_REG0__INCOHERENTCK_0_MASK 0x00000008L
+#define PB1_TX_LANE0_SCI_STAT_OVRD_REG0__INCOHERENTCK_0__SHIFT 0x00000003
+#define PB1_TX_LANE0_SCI_STAT_OVRD_REG0__TXMARG_0_MASK 0x00000070L
+#define PB1_TX_LANE0_SCI_STAT_OVRD_REG0__TXMARG_0__SHIFT 0x00000004
+#define PB1_TX_LANE0_SCI_STAT_OVRD_REG0__TXPWR_0_MASK 0x00000007L
+#define PB1_TX_LANE0_SCI_STAT_OVRD_REG0__TXPWR_0__SHIFT 0x00000000
+#define PB1_TX_LANE10_CTRL_REG0__TX_CFG_DISPCLK_MODE_10_MASK 0x00000001L
+#define PB1_TX_LANE10_CTRL_REG0__TX_CFG_DISPCLK_MODE_10__SHIFT 0x00000000
+#define PB1_TX_LANE10_CTRL_REG0__TX_CFG_INV_DATA_10_MASK 0x00000002L
+#define PB1_TX_LANE10_CTRL_REG0__TX_CFG_INV_DATA_10__SHIFT 0x00000001
+#define PB1_TX_LANE10_CTRL_REG0__TX_CFG_SWING_BOOST_EN_10_MASK 0x00000004L
+#define PB1_TX_LANE10_CTRL_REG0__TX_CFG_SWING_BOOST_EN_10__SHIFT 0x00000002
+#define PB1_TX_LANE10_CTRL_REG0__TX_DBG_PRBS_EN_10_MASK 0x00000008L
+#define PB1_TX_LANE10_CTRL_REG0__TX_DBG_PRBS_EN_10__SHIFT 0x00000003
+#define PB1_TX_LANE10_OVRD_REG0__TX_DCLK_EN_OVRD_EN_10_MASK 0x00000002L
+#define PB1_TX_LANE10_OVRD_REG0__TX_DCLK_EN_OVRD_EN_10__SHIFT 0x00000001
+#define PB1_TX_LANE10_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_10_MASK 0x00000001L
+#define PB1_TX_LANE10_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_10__SHIFT 0x00000000
+#define PB1_TX_LANE10_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_10_MASK 0x00000008L
+#define PB1_TX_LANE10_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_10__SHIFT 0x00000003
+#define PB1_TX_LANE10_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_10_MASK 0x00000004L
+#define PB1_TX_LANE10_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_10__SHIFT 0x00000002
+#define PB1_TX_LANE10_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_10_MASK 0x00000020L
+#define PB1_TX_LANE10_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_10__SHIFT 0x00000005
+#define PB1_TX_LANE10_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_10_MASK 0x00000010L
+#define PB1_TX_LANE10_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_10__SHIFT 0x00000004
+#define PB1_TX_LANE10_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_10_MASK 0x00000080L
+#define PB1_TX_LANE10_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_10__SHIFT 0x00000007
+#define PB1_TX_LANE10_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_10_MASK 0x00000040L
+#define PB1_TX_LANE10_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_10__SHIFT 0x00000006
+#define PB1_TX_LANE10_SCI_STAT_OVRD_REG0__COEFFICIENT_10_MASK 0x0000fc00L
+#define PB1_TX_LANE10_SCI_STAT_OVRD_REG0__COEFFICIENT_10__SHIFT 0x0000000a
+#define PB1_TX_LANE10_SCI_STAT_OVRD_REG0__COEFFICIENTID_10_MASK 0x00000300L
+#define PB1_TX_LANE10_SCI_STAT_OVRD_REG0__COEFFICIENTID_10__SHIFT 0x00000008
+#define PB1_TX_LANE10_SCI_STAT_OVRD_REG0__DEEMPH_10_MASK 0x00000080L
+#define PB1_TX_LANE10_SCI_STAT_OVRD_REG0__DEEMPH_10__SHIFT 0x00000007
+#define PB1_TX_LANE10_SCI_STAT_OVRD_REG0__INCOHERENTCK_10_MASK 0x00000008L
+#define PB1_TX_LANE10_SCI_STAT_OVRD_REG0__INCOHERENTCK_10__SHIFT 0x00000003
+#define PB1_TX_LANE10_SCI_STAT_OVRD_REG0__TXMARG_10_MASK 0x00000070L
+#define PB1_TX_LANE10_SCI_STAT_OVRD_REG0__TXMARG_10__SHIFT 0x00000004
+#define PB1_TX_LANE10_SCI_STAT_OVRD_REG0__TXPWR_10_MASK 0x00000007L
+#define PB1_TX_LANE10_SCI_STAT_OVRD_REG0__TXPWR_10__SHIFT 0x00000000
+#define PB1_TX_LANE11_CTRL_REG0__TX_CFG_DISPCLK_MODE_11_MASK 0x00000001L
+#define PB1_TX_LANE11_CTRL_REG0__TX_CFG_DISPCLK_MODE_11__SHIFT 0x00000000
+#define PB1_TX_LANE11_CTRL_REG0__TX_CFG_INV_DATA_11_MASK 0x00000002L
+#define PB1_TX_LANE11_CTRL_REG0__TX_CFG_INV_DATA_11__SHIFT 0x00000001
+#define PB1_TX_LANE11_CTRL_REG0__TX_CFG_SWING_BOOST_EN_11_MASK 0x00000004L
+#define PB1_TX_LANE11_CTRL_REG0__TX_CFG_SWING_BOOST_EN_11__SHIFT 0x00000002
+#define PB1_TX_LANE11_CTRL_REG0__TX_DBG_PRBS_EN_11_MASK 0x00000008L
+#define PB1_TX_LANE11_CTRL_REG0__TX_DBG_PRBS_EN_11__SHIFT 0x00000003
+#define PB1_TX_LANE11_OVRD_REG0__TX_DCLK_EN_OVRD_EN_11_MASK 0x00000002L
+#define PB1_TX_LANE11_OVRD_REG0__TX_DCLK_EN_OVRD_EN_11__SHIFT 0x00000001
+#define PB1_TX_LANE11_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_11_MASK 0x00000001L
+#define PB1_TX_LANE11_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_11__SHIFT 0x00000000
+#define PB1_TX_LANE11_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_11_MASK 0x00000008L
+#define PB1_TX_LANE11_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_11__SHIFT 0x00000003
+#define PB1_TX_LANE11_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_11_MASK 0x00000004L
+#define PB1_TX_LANE11_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_11__SHIFT 0x00000002
+#define PB1_TX_LANE11_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_11_MASK 0x00000020L
+#define PB1_TX_LANE11_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_11__SHIFT 0x00000005
+#define PB1_TX_LANE11_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_11_MASK 0x00000010L
+#define PB1_TX_LANE11_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_11__SHIFT 0x00000004
+#define PB1_TX_LANE11_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_11_MASK 0x00000080L
+#define PB1_TX_LANE11_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_11__SHIFT 0x00000007
+#define PB1_TX_LANE11_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_11_MASK 0x00000040L
+#define PB1_TX_LANE11_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_11__SHIFT 0x00000006
+#define PB1_TX_LANE11_SCI_STAT_OVRD_REG0__COEFFICIENT_11_MASK 0x0000fc00L
+#define PB1_TX_LANE11_SCI_STAT_OVRD_REG0__COEFFICIENT_11__SHIFT 0x0000000a
+#define PB1_TX_LANE11_SCI_STAT_OVRD_REG0__COEFFICIENTID_11_MASK 0x00000300L
+#define PB1_TX_LANE11_SCI_STAT_OVRD_REG0__COEFFICIENTID_11__SHIFT 0x00000008
+#define PB1_TX_LANE11_SCI_STAT_OVRD_REG0__DEEMPH_11_MASK 0x00000080L
+#define PB1_TX_LANE11_SCI_STAT_OVRD_REG0__DEEMPH_11__SHIFT 0x00000007
+#define PB1_TX_LANE11_SCI_STAT_OVRD_REG0__INCOHERENTCK_11_MASK 0x00000008L
+#define PB1_TX_LANE11_SCI_STAT_OVRD_REG0__INCOHERENTCK_11__SHIFT 0x00000003
+#define PB1_TX_LANE11_SCI_STAT_OVRD_REG0__TXMARG_11_MASK 0x00000070L
+#define PB1_TX_LANE11_SCI_STAT_OVRD_REG0__TXMARG_11__SHIFT 0x00000004
+#define PB1_TX_LANE11_SCI_STAT_OVRD_REG0__TXPWR_11_MASK 0x00000007L
+#define PB1_TX_LANE11_SCI_STAT_OVRD_REG0__TXPWR_11__SHIFT 0x00000000
+#define PB1_TX_LANE12_CTRL_REG0__TX_CFG_DISPCLK_MODE_12_MASK 0x00000001L
+#define PB1_TX_LANE12_CTRL_REG0__TX_CFG_DISPCLK_MODE_12__SHIFT 0x00000000
+#define PB1_TX_LANE12_CTRL_REG0__TX_CFG_INV_DATA_12_MASK 0x00000002L
+#define PB1_TX_LANE12_CTRL_REG0__TX_CFG_INV_DATA_12__SHIFT 0x00000001
+#define PB1_TX_LANE12_CTRL_REG0__TX_CFG_SWING_BOOST_EN_12_MASK 0x00000004L
+#define PB1_TX_LANE12_CTRL_REG0__TX_CFG_SWING_BOOST_EN_12__SHIFT 0x00000002
+#define PB1_TX_LANE12_CTRL_REG0__TX_DBG_PRBS_EN_12_MASK 0x00000008L
+#define PB1_TX_LANE12_CTRL_REG0__TX_DBG_PRBS_EN_12__SHIFT 0x00000003
+#define PB1_TX_LANE12_OVRD_REG0__TX_DCLK_EN_OVRD_EN_12_MASK 0x00000002L
+#define PB1_TX_LANE12_OVRD_REG0__TX_DCLK_EN_OVRD_EN_12__SHIFT 0x00000001
+#define PB1_TX_LANE12_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_12_MASK 0x00000001L
+#define PB1_TX_LANE12_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_12__SHIFT 0x00000000
+#define PB1_TX_LANE12_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_12_MASK 0x00000008L
+#define PB1_TX_LANE12_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_12__SHIFT 0x00000003
+#define PB1_TX_LANE12_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_12_MASK 0x00000004L
+#define PB1_TX_LANE12_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_12__SHIFT 0x00000002
+#define PB1_TX_LANE12_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_12_MASK 0x00000020L
+#define PB1_TX_LANE12_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_12__SHIFT 0x00000005
+#define PB1_TX_LANE12_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_12_MASK 0x00000010L
+#define PB1_TX_LANE12_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_12__SHIFT 0x00000004
+#define PB1_TX_LANE12_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_12_MASK 0x00000080L
+#define PB1_TX_LANE12_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_12__SHIFT 0x00000007
+#define PB1_TX_LANE12_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_12_MASK 0x00000040L
+#define PB1_TX_LANE12_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_12__SHIFT 0x00000006
+#define PB1_TX_LANE12_SCI_STAT_OVRD_REG0__COEFFICIENT_12_MASK 0x0000fc00L
+#define PB1_TX_LANE12_SCI_STAT_OVRD_REG0__COEFFICIENT_12__SHIFT 0x0000000a
+#define PB1_TX_LANE12_SCI_STAT_OVRD_REG0__COEFFICIENTID_12_MASK 0x00000300L
+#define PB1_TX_LANE12_SCI_STAT_OVRD_REG0__COEFFICIENTID_12__SHIFT 0x00000008
+#define PB1_TX_LANE12_SCI_STAT_OVRD_REG0__DEEMPH_12_MASK 0x00000080L
+#define PB1_TX_LANE12_SCI_STAT_OVRD_REG0__DEEMPH_12__SHIFT 0x00000007
+#define PB1_TX_LANE12_SCI_STAT_OVRD_REG0__INCOHERENTCK_12_MASK 0x00000008L
+#define PB1_TX_LANE12_SCI_STAT_OVRD_REG0__INCOHERENTCK_12__SHIFT 0x00000003
+#define PB1_TX_LANE12_SCI_STAT_OVRD_REG0__TXMARG_12_MASK 0x00000070L
+#define PB1_TX_LANE12_SCI_STAT_OVRD_REG0__TXMARG_12__SHIFT 0x00000004
+#define PB1_TX_LANE12_SCI_STAT_OVRD_REG0__TXPWR_12_MASK 0x00000007L
+#define PB1_TX_LANE12_SCI_STAT_OVRD_REG0__TXPWR_12__SHIFT 0x00000000
+#define PB1_TX_LANE13_CTRL_REG0__TX_CFG_DISPCLK_MODE_13_MASK 0x00000001L
+#define PB1_TX_LANE13_CTRL_REG0__TX_CFG_DISPCLK_MODE_13__SHIFT 0x00000000
+#define PB1_TX_LANE13_CTRL_REG0__TX_CFG_INV_DATA_13_MASK 0x00000002L
+#define PB1_TX_LANE13_CTRL_REG0__TX_CFG_INV_DATA_13__SHIFT 0x00000001
+#define PB1_TX_LANE13_CTRL_REG0__TX_CFG_SWING_BOOST_EN_13_MASK 0x00000004L
+#define PB1_TX_LANE13_CTRL_REG0__TX_CFG_SWING_BOOST_EN_13__SHIFT 0x00000002
+#define PB1_TX_LANE13_CTRL_REG0__TX_DBG_PRBS_EN_13_MASK 0x00000008L
+#define PB1_TX_LANE13_CTRL_REG0__TX_DBG_PRBS_EN_13__SHIFT 0x00000003
+#define PB1_TX_LANE13_OVRD_REG0__TX_DCLK_EN_OVRD_EN_13_MASK 0x00000002L
+#define PB1_TX_LANE13_OVRD_REG0__TX_DCLK_EN_OVRD_EN_13__SHIFT 0x00000001
+#define PB1_TX_LANE13_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_13_MASK 0x00000001L
+#define PB1_TX_LANE13_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_13__SHIFT 0x00000000
+#define PB1_TX_LANE13_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_13_MASK 0x00000008L
+#define PB1_TX_LANE13_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_13__SHIFT 0x00000003
+#define PB1_TX_LANE13_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_13_MASK 0x00000004L
+#define PB1_TX_LANE13_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_13__SHIFT 0x00000002
+#define PB1_TX_LANE13_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_13_MASK 0x00000020L
+#define PB1_TX_LANE13_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_13__SHIFT 0x00000005
+#define PB1_TX_LANE13_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_13_MASK 0x00000010L
+#define PB1_TX_LANE13_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_13__SHIFT 0x00000004
+#define PB1_TX_LANE13_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_13_MASK 0x00000080L
+#define PB1_TX_LANE13_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_13__SHIFT 0x00000007
+#define PB1_TX_LANE13_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_13_MASK 0x00000040L
+#define PB1_TX_LANE13_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_13__SHIFT 0x00000006
+#define PB1_TX_LANE13_SCI_STAT_OVRD_REG0__COEFFICIENT_13_MASK 0x0000fc00L
+#define PB1_TX_LANE13_SCI_STAT_OVRD_REG0__COEFFICIENT_13__SHIFT 0x0000000a
+#define PB1_TX_LANE13_SCI_STAT_OVRD_REG0__COEFFICIENTID_13_MASK 0x00000300L
+#define PB1_TX_LANE13_SCI_STAT_OVRD_REG0__COEFFICIENTID_13__SHIFT 0x00000008
+#define PB1_TX_LANE13_SCI_STAT_OVRD_REG0__DEEMPH_13_MASK 0x00000080L
+#define PB1_TX_LANE13_SCI_STAT_OVRD_REG0__DEEMPH_13__SHIFT 0x00000007
+#define PB1_TX_LANE13_SCI_STAT_OVRD_REG0__INCOHERENTCK_13_MASK 0x00000008L
+#define PB1_TX_LANE13_SCI_STAT_OVRD_REG0__INCOHERENTCK_13__SHIFT 0x00000003
+#define PB1_TX_LANE13_SCI_STAT_OVRD_REG0__TXMARG_13_MASK 0x00000070L
+#define PB1_TX_LANE13_SCI_STAT_OVRD_REG0__TXMARG_13__SHIFT 0x00000004
+#define PB1_TX_LANE13_SCI_STAT_OVRD_REG0__TXPWR_13_MASK 0x00000007L
+#define PB1_TX_LANE13_SCI_STAT_OVRD_REG0__TXPWR_13__SHIFT 0x00000000
+#define PB1_TX_LANE14_CTRL_REG0__TX_CFG_DISPCLK_MODE_14_MASK 0x00000001L
+#define PB1_TX_LANE14_CTRL_REG0__TX_CFG_DISPCLK_MODE_14__SHIFT 0x00000000
+#define PB1_TX_LANE14_CTRL_REG0__TX_CFG_INV_DATA_14_MASK 0x00000002L
+#define PB1_TX_LANE14_CTRL_REG0__TX_CFG_INV_DATA_14__SHIFT 0x00000001
+#define PB1_TX_LANE14_CTRL_REG0__TX_CFG_SWING_BOOST_EN_14_MASK 0x00000004L
+#define PB1_TX_LANE14_CTRL_REG0__TX_CFG_SWING_BOOST_EN_14__SHIFT 0x00000002
+#define PB1_TX_LANE14_CTRL_REG0__TX_DBG_PRBS_EN_14_MASK 0x00000008L
+#define PB1_TX_LANE14_CTRL_REG0__TX_DBG_PRBS_EN_14__SHIFT 0x00000003
+#define PB1_TX_LANE14_OVRD_REG0__TX_DCLK_EN_OVRD_EN_14_MASK 0x00000002L
+#define PB1_TX_LANE14_OVRD_REG0__TX_DCLK_EN_OVRD_EN_14__SHIFT 0x00000001
+#define PB1_TX_LANE14_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_14_MASK 0x00000001L
+#define PB1_TX_LANE14_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_14__SHIFT 0x00000000
+#define PB1_TX_LANE14_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_14_MASK 0x00000008L
+#define PB1_TX_LANE14_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_14__SHIFT 0x00000003
+#define PB1_TX_LANE14_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_14_MASK 0x00000004L
+#define PB1_TX_LANE14_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_14__SHIFT 0x00000002
+#define PB1_TX_LANE14_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_14_MASK 0x00000020L
+#define PB1_TX_LANE14_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_14__SHIFT 0x00000005
+#define PB1_TX_LANE14_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_14_MASK 0x00000010L
+#define PB1_TX_LANE14_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_14__SHIFT 0x00000004
+#define PB1_TX_LANE14_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_14_MASK 0x00000080L
+#define PB1_TX_LANE14_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_14__SHIFT 0x00000007
+#define PB1_TX_LANE14_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_14_MASK 0x00000040L
+#define PB1_TX_LANE14_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_14__SHIFT 0x00000006
+#define PB1_TX_LANE14_SCI_STAT_OVRD_REG0__COEFFICIENT_14_MASK 0x0000fc00L
+#define PB1_TX_LANE14_SCI_STAT_OVRD_REG0__COEFFICIENT_14__SHIFT 0x0000000a
+#define PB1_TX_LANE14_SCI_STAT_OVRD_REG0__COEFFICIENTID_14_MASK 0x00000300L
+#define PB1_TX_LANE14_SCI_STAT_OVRD_REG0__COEFFICIENTID_14__SHIFT 0x00000008
+#define PB1_TX_LANE14_SCI_STAT_OVRD_REG0__DEEMPH_14_MASK 0x00000080L
+#define PB1_TX_LANE14_SCI_STAT_OVRD_REG0__DEEMPH_14__SHIFT 0x00000007
+#define PB1_TX_LANE14_SCI_STAT_OVRD_REG0__INCOHERENTCK_14_MASK 0x00000008L
+#define PB1_TX_LANE14_SCI_STAT_OVRD_REG0__INCOHERENTCK_14__SHIFT 0x00000003
+#define PB1_TX_LANE14_SCI_STAT_OVRD_REG0__TXMARG_14_MASK 0x00000070L
+#define PB1_TX_LANE14_SCI_STAT_OVRD_REG0__TXMARG_14__SHIFT 0x00000004
+#define PB1_TX_LANE14_SCI_STAT_OVRD_REG0__TXPWR_14_MASK 0x00000007L
+#define PB1_TX_LANE14_SCI_STAT_OVRD_REG0__TXPWR_14__SHIFT 0x00000000
+#define PB1_TX_LANE15_CTRL_REG0__TX_CFG_DISPCLK_MODE_15_MASK 0x00000001L
+#define PB1_TX_LANE15_CTRL_REG0__TX_CFG_DISPCLK_MODE_15__SHIFT 0x00000000
+#define PB1_TX_LANE15_CTRL_REG0__TX_CFG_INV_DATA_15_MASK 0x00000002L
+#define PB1_TX_LANE15_CTRL_REG0__TX_CFG_INV_DATA_15__SHIFT 0x00000001
+#define PB1_TX_LANE15_CTRL_REG0__TX_CFG_SWING_BOOST_EN_15_MASK 0x00000004L
+#define PB1_TX_LANE15_CTRL_REG0__TX_CFG_SWING_BOOST_EN_15__SHIFT 0x00000002
+#define PB1_TX_LANE15_CTRL_REG0__TX_DBG_PRBS_EN_15_MASK 0x00000008L
+#define PB1_TX_LANE15_CTRL_REG0__TX_DBG_PRBS_EN_15__SHIFT 0x00000003
+#define PB1_TX_LANE15_OVRD_REG0__TX_DCLK_EN_OVRD_EN_15_MASK 0x00000002L
+#define PB1_TX_LANE15_OVRD_REG0__TX_DCLK_EN_OVRD_EN_15__SHIFT 0x00000001
+#define PB1_TX_LANE15_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_15_MASK 0x00000001L
+#define PB1_TX_LANE15_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_15__SHIFT 0x00000000
+#define PB1_TX_LANE15_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_15_MASK 0x00000008L
+#define PB1_TX_LANE15_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_15__SHIFT 0x00000003
+#define PB1_TX_LANE15_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_15_MASK 0x00000004L
+#define PB1_TX_LANE15_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_15__SHIFT 0x00000002
+#define PB1_TX_LANE15_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_15_MASK 0x00000020L
+#define PB1_TX_LANE15_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_15__SHIFT 0x00000005
+#define PB1_TX_LANE15_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_15_MASK 0x00000010L
+#define PB1_TX_LANE15_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_15__SHIFT 0x00000004
+#define PB1_TX_LANE15_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_15_MASK 0x00000080L
+#define PB1_TX_LANE15_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_15__SHIFT 0x00000007
+#define PB1_TX_LANE15_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_15_MASK 0x00000040L
+#define PB1_TX_LANE15_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_15__SHIFT 0x00000006
+#define PB1_TX_LANE15_SCI_STAT_OVRD_REG0__COEFFICIENT_15_MASK 0x0000fc00L
+#define PB1_TX_LANE15_SCI_STAT_OVRD_REG0__COEFFICIENT_15__SHIFT 0x0000000a
+#define PB1_TX_LANE15_SCI_STAT_OVRD_REG0__COEFFICIENTID_15_MASK 0x00000300L
+#define PB1_TX_LANE15_SCI_STAT_OVRD_REG0__COEFFICIENTID_15__SHIFT 0x00000008
+#define PB1_TX_LANE15_SCI_STAT_OVRD_REG0__DEEMPH_15_MASK 0x00000080L
+#define PB1_TX_LANE15_SCI_STAT_OVRD_REG0__DEEMPH_15__SHIFT 0x00000007
+#define PB1_TX_LANE15_SCI_STAT_OVRD_REG0__INCOHERENTCK_15_MASK 0x00000008L
+#define PB1_TX_LANE15_SCI_STAT_OVRD_REG0__INCOHERENTCK_15__SHIFT 0x00000003
+#define PB1_TX_LANE15_SCI_STAT_OVRD_REG0__TXMARG_15_MASK 0x00000070L
+#define PB1_TX_LANE15_SCI_STAT_OVRD_REG0__TXMARG_15__SHIFT 0x00000004
+#define PB1_TX_LANE15_SCI_STAT_OVRD_REG0__TXPWR_15_MASK 0x00000007L
+#define PB1_TX_LANE15_SCI_STAT_OVRD_REG0__TXPWR_15__SHIFT 0x00000000
+#define PB1_TX_LANE1_CTRL_REG0__TX_CFG_DISPCLK_MODE_1_MASK 0x00000001L
+#define PB1_TX_LANE1_CTRL_REG0__TX_CFG_DISPCLK_MODE_1__SHIFT 0x00000000
+#define PB1_TX_LANE1_CTRL_REG0__TX_CFG_INV_DATA_1_MASK 0x00000002L
+#define PB1_TX_LANE1_CTRL_REG0__TX_CFG_INV_DATA_1__SHIFT 0x00000001
+#define PB1_TX_LANE1_CTRL_REG0__TX_CFG_SWING_BOOST_EN_1_MASK 0x00000004L
+#define PB1_TX_LANE1_CTRL_REG0__TX_CFG_SWING_BOOST_EN_1__SHIFT 0x00000002
+#define PB1_TX_LANE1_CTRL_REG0__TX_DBG_PRBS_EN_1_MASK 0x00000008L
+#define PB1_TX_LANE1_CTRL_REG0__TX_DBG_PRBS_EN_1__SHIFT 0x00000003
+#define PB1_TX_LANE1_OVRD_REG0__TX_DCLK_EN_OVRD_EN_1_MASK 0x00000002L
+#define PB1_TX_LANE1_OVRD_REG0__TX_DCLK_EN_OVRD_EN_1__SHIFT 0x00000001
+#define PB1_TX_LANE1_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_1_MASK 0x00000001L
+#define PB1_TX_LANE1_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_1__SHIFT 0x00000000
+#define PB1_TX_LANE1_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_1_MASK 0x00000008L
+#define PB1_TX_LANE1_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_1__SHIFT 0x00000003
+#define PB1_TX_LANE1_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_1_MASK 0x00000004L
+#define PB1_TX_LANE1_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_1__SHIFT 0x00000002
+#define PB1_TX_LANE1_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_1_MASK 0x00000020L
+#define PB1_TX_LANE1_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_1__SHIFT 0x00000005
+#define PB1_TX_LANE1_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_1_MASK 0x00000010L
+#define PB1_TX_LANE1_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_1__SHIFT 0x00000004
+#define PB1_TX_LANE1_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_1_MASK 0x00000080L
+#define PB1_TX_LANE1_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_1__SHIFT 0x00000007
+#define PB1_TX_LANE1_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_1_MASK 0x00000040L
+#define PB1_TX_LANE1_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_1__SHIFT 0x00000006
+#define PB1_TX_LANE1_SCI_STAT_OVRD_REG0__COEFFICIENT_1_MASK 0x0000fc00L
+#define PB1_TX_LANE1_SCI_STAT_OVRD_REG0__COEFFICIENT_1__SHIFT 0x0000000a
+#define PB1_TX_LANE1_SCI_STAT_OVRD_REG0__COEFFICIENTID_1_MASK 0x00000300L
+#define PB1_TX_LANE1_SCI_STAT_OVRD_REG0__COEFFICIENTID_1__SHIFT 0x00000008
+#define PB1_TX_LANE1_SCI_STAT_OVRD_REG0__DEEMPH_1_MASK 0x00000080L
+#define PB1_TX_LANE1_SCI_STAT_OVRD_REG0__DEEMPH_1__SHIFT 0x00000007
+#define PB1_TX_LANE1_SCI_STAT_OVRD_REG0__INCOHERENTCK_1_MASK 0x00000008L
+#define PB1_TX_LANE1_SCI_STAT_OVRD_REG0__INCOHERENTCK_1__SHIFT 0x00000003
+#define PB1_TX_LANE1_SCI_STAT_OVRD_REG0__TXMARG_1_MASK 0x00000070L
+#define PB1_TX_LANE1_SCI_STAT_OVRD_REG0__TXMARG_1__SHIFT 0x00000004
+#define PB1_TX_LANE1_SCI_STAT_OVRD_REG0__TXPWR_1_MASK 0x00000007L
+#define PB1_TX_LANE1_SCI_STAT_OVRD_REG0__TXPWR_1__SHIFT 0x00000000
+#define PB1_TX_LANE2_CTRL_REG0__TX_CFG_DISPCLK_MODE_2_MASK 0x00000001L
+#define PB1_TX_LANE2_CTRL_REG0__TX_CFG_DISPCLK_MODE_2__SHIFT 0x00000000
+#define PB1_TX_LANE2_CTRL_REG0__TX_CFG_INV_DATA_2_MASK 0x00000002L
+#define PB1_TX_LANE2_CTRL_REG0__TX_CFG_INV_DATA_2__SHIFT 0x00000001
+#define PB1_TX_LANE2_CTRL_REG0__TX_CFG_SWING_BOOST_EN_2_MASK 0x00000004L
+#define PB1_TX_LANE2_CTRL_REG0__TX_CFG_SWING_BOOST_EN_2__SHIFT 0x00000002
+#define PB1_TX_LANE2_CTRL_REG0__TX_DBG_PRBS_EN_2_MASK 0x00000008L
+#define PB1_TX_LANE2_CTRL_REG0__TX_DBG_PRBS_EN_2__SHIFT 0x00000003
+#define PB1_TX_LANE2_OVRD_REG0__TX_DCLK_EN_OVRD_EN_2_MASK 0x00000002L
+#define PB1_TX_LANE2_OVRD_REG0__TX_DCLK_EN_OVRD_EN_2__SHIFT 0x00000001
+#define PB1_TX_LANE2_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_2_MASK 0x00000001L
+#define PB1_TX_LANE2_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_2__SHIFT 0x00000000
+#define PB1_TX_LANE2_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_2_MASK 0x00000008L
+#define PB1_TX_LANE2_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_2__SHIFT 0x00000003
+#define PB1_TX_LANE2_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_2_MASK 0x00000004L
+#define PB1_TX_LANE2_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_2__SHIFT 0x00000002
+#define PB1_TX_LANE2_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_2_MASK 0x00000020L
+#define PB1_TX_LANE2_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_2__SHIFT 0x00000005
+#define PB1_TX_LANE2_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_2_MASK 0x00000010L
+#define PB1_TX_LANE2_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_2__SHIFT 0x00000004
+#define PB1_TX_LANE2_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_2_MASK 0x00000080L
+#define PB1_TX_LANE2_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_2__SHIFT 0x00000007
+#define PB1_TX_LANE2_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_2_MASK 0x00000040L
+#define PB1_TX_LANE2_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_2__SHIFT 0x00000006
+#define PB1_TX_LANE2_SCI_STAT_OVRD_REG0__COEFFICIENT_2_MASK 0x0000fc00L
+#define PB1_TX_LANE2_SCI_STAT_OVRD_REG0__COEFFICIENT_2__SHIFT 0x0000000a
+#define PB1_TX_LANE2_SCI_STAT_OVRD_REG0__COEFFICIENTID_2_MASK 0x00000300L
+#define PB1_TX_LANE2_SCI_STAT_OVRD_REG0__COEFFICIENTID_2__SHIFT 0x00000008
+#define PB1_TX_LANE2_SCI_STAT_OVRD_REG0__DEEMPH_2_MASK 0x00000080L
+#define PB1_TX_LANE2_SCI_STAT_OVRD_REG0__DEEMPH_2__SHIFT 0x00000007
+#define PB1_TX_LANE2_SCI_STAT_OVRD_REG0__INCOHERENTCK_2_MASK 0x00000008L
+#define PB1_TX_LANE2_SCI_STAT_OVRD_REG0__INCOHERENTCK_2__SHIFT 0x00000003
+#define PB1_TX_LANE2_SCI_STAT_OVRD_REG0__TXMARG_2_MASK 0x00000070L
+#define PB1_TX_LANE2_SCI_STAT_OVRD_REG0__TXMARG_2__SHIFT 0x00000004
+#define PB1_TX_LANE2_SCI_STAT_OVRD_REG0__TXPWR_2_MASK 0x00000007L
+#define PB1_TX_LANE2_SCI_STAT_OVRD_REG0__TXPWR_2__SHIFT 0x00000000
+#define PB1_TX_LANE3_CTRL_REG0__TX_CFG_DISPCLK_MODE_3_MASK 0x00000001L
+#define PB1_TX_LANE3_CTRL_REG0__TX_CFG_DISPCLK_MODE_3__SHIFT 0x00000000
+#define PB1_TX_LANE3_CTRL_REG0__TX_CFG_INV_DATA_3_MASK 0x00000002L
+#define PB1_TX_LANE3_CTRL_REG0__TX_CFG_INV_DATA_3__SHIFT 0x00000001
+#define PB1_TX_LANE3_CTRL_REG0__TX_CFG_SWING_BOOST_EN_3_MASK 0x00000004L
+#define PB1_TX_LANE3_CTRL_REG0__TX_CFG_SWING_BOOST_EN_3__SHIFT 0x00000002
+#define PB1_TX_LANE3_CTRL_REG0__TX_DBG_PRBS_EN_3_MASK 0x00000008L
+#define PB1_TX_LANE3_CTRL_REG0__TX_DBG_PRBS_EN_3__SHIFT 0x00000003
+#define PB1_TX_LANE3_OVRD_REG0__TX_DCLK_EN_OVRD_EN_3_MASK 0x00000002L
+#define PB1_TX_LANE3_OVRD_REG0__TX_DCLK_EN_OVRD_EN_3__SHIFT 0x00000001
+#define PB1_TX_LANE3_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_3_MASK 0x00000001L
+#define PB1_TX_LANE3_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_3__SHIFT 0x00000000
+#define PB1_TX_LANE3_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_3_MASK 0x00000008L
+#define PB1_TX_LANE3_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_3__SHIFT 0x00000003
+#define PB1_TX_LANE3_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_3_MASK 0x00000004L
+#define PB1_TX_LANE3_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_3__SHIFT 0x00000002
+#define PB1_TX_LANE3_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_3_MASK 0x00000020L
+#define PB1_TX_LANE3_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_3__SHIFT 0x00000005
+#define PB1_TX_LANE3_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_3_MASK 0x00000010L
+#define PB1_TX_LANE3_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_3__SHIFT 0x00000004
+#define PB1_TX_LANE3_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_3_MASK 0x00000080L
+#define PB1_TX_LANE3_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_3__SHIFT 0x00000007
+#define PB1_TX_LANE3_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_3_MASK 0x00000040L
+#define PB1_TX_LANE3_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_3__SHIFT 0x00000006
+#define PB1_TX_LANE3_SCI_STAT_OVRD_REG0__COEFFICIENT_3_MASK 0x0000fc00L
+#define PB1_TX_LANE3_SCI_STAT_OVRD_REG0__COEFFICIENT_3__SHIFT 0x0000000a
+#define PB1_TX_LANE3_SCI_STAT_OVRD_REG0__COEFFICIENTID_3_MASK 0x00000300L
+#define PB1_TX_LANE3_SCI_STAT_OVRD_REG0__COEFFICIENTID_3__SHIFT 0x00000008
+#define PB1_TX_LANE3_SCI_STAT_OVRD_REG0__DEEMPH_3_MASK 0x00000080L
+#define PB1_TX_LANE3_SCI_STAT_OVRD_REG0__DEEMPH_3__SHIFT 0x00000007
+#define PB1_TX_LANE3_SCI_STAT_OVRD_REG0__INCOHERENTCK_3_MASK 0x00000008L
+#define PB1_TX_LANE3_SCI_STAT_OVRD_REG0__INCOHERENTCK_3__SHIFT 0x00000003
+#define PB1_TX_LANE3_SCI_STAT_OVRD_REG0__TXMARG_3_MASK 0x00000070L
+#define PB1_TX_LANE3_SCI_STAT_OVRD_REG0__TXMARG_3__SHIFT 0x00000004
+#define PB1_TX_LANE3_SCI_STAT_OVRD_REG0__TXPWR_3_MASK 0x00000007L
+#define PB1_TX_LANE3_SCI_STAT_OVRD_REG0__TXPWR_3__SHIFT 0x00000000
+#define PB1_TX_LANE4_CTRL_REG0__TX_CFG_DISPCLK_MODE_4_MASK 0x00000001L
+#define PB1_TX_LANE4_CTRL_REG0__TX_CFG_DISPCLK_MODE_4__SHIFT 0x00000000
+#define PB1_TX_LANE4_CTRL_REG0__TX_CFG_INV_DATA_4_MASK 0x00000002L
+#define PB1_TX_LANE4_CTRL_REG0__TX_CFG_INV_DATA_4__SHIFT 0x00000001
+#define PB1_TX_LANE4_CTRL_REG0__TX_CFG_SWING_BOOST_EN_4_MASK 0x00000004L
+#define PB1_TX_LANE4_CTRL_REG0__TX_CFG_SWING_BOOST_EN_4__SHIFT 0x00000002
+#define PB1_TX_LANE4_CTRL_REG0__TX_DBG_PRBS_EN_4_MASK 0x00000008L
+#define PB1_TX_LANE4_CTRL_REG0__TX_DBG_PRBS_EN_4__SHIFT 0x00000003
+#define PB1_TX_LANE4_OVRD_REG0__TX_DCLK_EN_OVRD_EN_4_MASK 0x00000002L
+#define PB1_TX_LANE4_OVRD_REG0__TX_DCLK_EN_OVRD_EN_4__SHIFT 0x00000001
+#define PB1_TX_LANE4_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_4_MASK 0x00000001L
+#define PB1_TX_LANE4_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_4__SHIFT 0x00000000
+#define PB1_TX_LANE4_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_4_MASK 0x00000008L
+#define PB1_TX_LANE4_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_4__SHIFT 0x00000003
+#define PB1_TX_LANE4_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_4_MASK 0x00000004L
+#define PB1_TX_LANE4_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_4__SHIFT 0x00000002
+#define PB1_TX_LANE4_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_4_MASK 0x00000020L
+#define PB1_TX_LANE4_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_4__SHIFT 0x00000005
+#define PB1_TX_LANE4_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_4_MASK 0x00000010L
+#define PB1_TX_LANE4_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_4__SHIFT 0x00000004
+#define PB1_TX_LANE4_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_4_MASK 0x00000080L
+#define PB1_TX_LANE4_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_4__SHIFT 0x00000007
+#define PB1_TX_LANE4_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_4_MASK 0x00000040L
+#define PB1_TX_LANE4_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_4__SHIFT 0x00000006
+#define PB1_TX_LANE4_SCI_STAT_OVRD_REG0__COEFFICIENT_4_MASK 0x0000fc00L
+#define PB1_TX_LANE4_SCI_STAT_OVRD_REG0__COEFFICIENT_4__SHIFT 0x0000000a
+#define PB1_TX_LANE4_SCI_STAT_OVRD_REG0__COEFFICIENTID_4_MASK 0x00000300L
+#define PB1_TX_LANE4_SCI_STAT_OVRD_REG0__COEFFICIENTID_4__SHIFT 0x00000008
+#define PB1_TX_LANE4_SCI_STAT_OVRD_REG0__DEEMPH_4_MASK 0x00000080L
+#define PB1_TX_LANE4_SCI_STAT_OVRD_REG0__DEEMPH_4__SHIFT 0x00000007
+#define PB1_TX_LANE4_SCI_STAT_OVRD_REG0__INCOHERENTCK_4_MASK 0x00000008L
+#define PB1_TX_LANE4_SCI_STAT_OVRD_REG0__INCOHERENTCK_4__SHIFT 0x00000003
+#define PB1_TX_LANE4_SCI_STAT_OVRD_REG0__TXMARG_4_MASK 0x00000070L
+#define PB1_TX_LANE4_SCI_STAT_OVRD_REG0__TXMARG_4__SHIFT 0x00000004
+#define PB1_TX_LANE4_SCI_STAT_OVRD_REG0__TXPWR_4_MASK 0x00000007L
+#define PB1_TX_LANE4_SCI_STAT_OVRD_REG0__TXPWR_4__SHIFT 0x00000000
+#define PB1_TX_LANE5_CTRL_REG0__TX_CFG_DISPCLK_MODE_5_MASK 0x00000001L
+#define PB1_TX_LANE5_CTRL_REG0__TX_CFG_DISPCLK_MODE_5__SHIFT 0x00000000
+#define PB1_TX_LANE5_CTRL_REG0__TX_CFG_INV_DATA_5_MASK 0x00000002L
+#define PB1_TX_LANE5_CTRL_REG0__TX_CFG_INV_DATA_5__SHIFT 0x00000001
+#define PB1_TX_LANE5_CTRL_REG0__TX_CFG_SWING_BOOST_EN_5_MASK 0x00000004L
+#define PB1_TX_LANE5_CTRL_REG0__TX_CFG_SWING_BOOST_EN_5__SHIFT 0x00000002
+#define PB1_TX_LANE5_CTRL_REG0__TX_DBG_PRBS_EN_5_MASK 0x00000008L
+#define PB1_TX_LANE5_CTRL_REG0__TX_DBG_PRBS_EN_5__SHIFT 0x00000003
+#define PB1_TX_LANE5_OVRD_REG0__TX_DCLK_EN_OVRD_EN_5_MASK 0x00000002L
+#define PB1_TX_LANE5_OVRD_REG0__TX_DCLK_EN_OVRD_EN_5__SHIFT 0x00000001
+#define PB1_TX_LANE5_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_5_MASK 0x00000001L
+#define PB1_TX_LANE5_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_5__SHIFT 0x00000000
+#define PB1_TX_LANE5_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_5_MASK 0x00000008L
+#define PB1_TX_LANE5_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_5__SHIFT 0x00000003
+#define PB1_TX_LANE5_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_5_MASK 0x00000004L
+#define PB1_TX_LANE5_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_5__SHIFT 0x00000002
+#define PB1_TX_LANE5_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_5_MASK 0x00000020L
+#define PB1_TX_LANE5_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_5__SHIFT 0x00000005
+#define PB1_TX_LANE5_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_5_MASK 0x00000010L
+#define PB1_TX_LANE5_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_5__SHIFT 0x00000004
+#define PB1_TX_LANE5_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_5_MASK 0x00000080L
+#define PB1_TX_LANE5_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_5__SHIFT 0x00000007
+#define PB1_TX_LANE5_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_5_MASK 0x00000040L
+#define PB1_TX_LANE5_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_5__SHIFT 0x00000006
+#define PB1_TX_LANE5_SCI_STAT_OVRD_REG0__COEFFICIENT_5_MASK 0x0000fc00L
+#define PB1_TX_LANE5_SCI_STAT_OVRD_REG0__COEFFICIENT_5__SHIFT 0x0000000a
+#define PB1_TX_LANE5_SCI_STAT_OVRD_REG0__COEFFICIENTID_5_MASK 0x00000300L
+#define PB1_TX_LANE5_SCI_STAT_OVRD_REG0__COEFFICIENTID_5__SHIFT 0x00000008
+#define PB1_TX_LANE5_SCI_STAT_OVRD_REG0__DEEMPH_5_MASK 0x00000080L
+#define PB1_TX_LANE5_SCI_STAT_OVRD_REG0__DEEMPH_5__SHIFT 0x00000007
+#define PB1_TX_LANE5_SCI_STAT_OVRD_REG0__INCOHERENTCK_5_MASK 0x00000008L
+#define PB1_TX_LANE5_SCI_STAT_OVRD_REG0__INCOHERENTCK_5__SHIFT 0x00000003
+#define PB1_TX_LANE5_SCI_STAT_OVRD_REG0__TXMARG_5_MASK 0x00000070L
+#define PB1_TX_LANE5_SCI_STAT_OVRD_REG0__TXMARG_5__SHIFT 0x00000004
+#define PB1_TX_LANE5_SCI_STAT_OVRD_REG0__TXPWR_5_MASK 0x00000007L
+#define PB1_TX_LANE5_SCI_STAT_OVRD_REG0__TXPWR_5__SHIFT 0x00000000
+#define PB1_TX_LANE6_CTRL_REG0__TX_CFG_DISPCLK_MODE_6_MASK 0x00000001L
+#define PB1_TX_LANE6_CTRL_REG0__TX_CFG_DISPCLK_MODE_6__SHIFT 0x00000000
+#define PB1_TX_LANE6_CTRL_REG0__TX_CFG_INV_DATA_6_MASK 0x00000002L
+#define PB1_TX_LANE6_CTRL_REG0__TX_CFG_INV_DATA_6__SHIFT 0x00000001
+#define PB1_TX_LANE6_CTRL_REG0__TX_CFG_SWING_BOOST_EN_6_MASK 0x00000004L
+#define PB1_TX_LANE6_CTRL_REG0__TX_CFG_SWING_BOOST_EN_6__SHIFT 0x00000002
+#define PB1_TX_LANE6_CTRL_REG0__TX_DBG_PRBS_EN_6_MASK 0x00000008L
+#define PB1_TX_LANE6_CTRL_REG0__TX_DBG_PRBS_EN_6__SHIFT 0x00000003
+#define PB1_TX_LANE6_OVRD_REG0__TX_DCLK_EN_OVRD_EN_6_MASK 0x00000002L
+#define PB1_TX_LANE6_OVRD_REG0__TX_DCLK_EN_OVRD_EN_6__SHIFT 0x00000001
+#define PB1_TX_LANE6_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_6_MASK 0x00000001L
+#define PB1_TX_LANE6_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_6__SHIFT 0x00000000
+#define PB1_TX_LANE6_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_6_MASK 0x00000008L
+#define PB1_TX_LANE6_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_6__SHIFT 0x00000003
+#define PB1_TX_LANE6_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_6_MASK 0x00000004L
+#define PB1_TX_LANE6_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_6__SHIFT 0x00000002
+#define PB1_TX_LANE6_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_6_MASK 0x00000020L
+#define PB1_TX_LANE6_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_6__SHIFT 0x00000005
+#define PB1_TX_LANE6_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_6_MASK 0x00000010L
+#define PB1_TX_LANE6_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_6__SHIFT 0x00000004
+#define PB1_TX_LANE6_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_6_MASK 0x00000080L
+#define PB1_TX_LANE6_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_6__SHIFT 0x00000007
+#define PB1_TX_LANE6_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_6_MASK 0x00000040L
+#define PB1_TX_LANE6_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_6__SHIFT 0x00000006
+#define PB1_TX_LANE6_SCI_STAT_OVRD_REG0__COEFFICIENT_6_MASK 0x0000fc00L
+#define PB1_TX_LANE6_SCI_STAT_OVRD_REG0__COEFFICIENT_6__SHIFT 0x0000000a
+#define PB1_TX_LANE6_SCI_STAT_OVRD_REG0__COEFFICIENTID_6_MASK 0x00000300L
+#define PB1_TX_LANE6_SCI_STAT_OVRD_REG0__COEFFICIENTID_6__SHIFT 0x00000008
+#define PB1_TX_LANE6_SCI_STAT_OVRD_REG0__DEEMPH_6_MASK 0x00000080L
+#define PB1_TX_LANE6_SCI_STAT_OVRD_REG0__DEEMPH_6__SHIFT 0x00000007
+#define PB1_TX_LANE6_SCI_STAT_OVRD_REG0__INCOHERENTCK_6_MASK 0x00000008L
+#define PB1_TX_LANE6_SCI_STAT_OVRD_REG0__INCOHERENTCK_6__SHIFT 0x00000003
+#define PB1_TX_LANE6_SCI_STAT_OVRD_REG0__TXMARG_6_MASK 0x00000070L
+#define PB1_TX_LANE6_SCI_STAT_OVRD_REG0__TXMARG_6__SHIFT 0x00000004
+#define PB1_TX_LANE6_SCI_STAT_OVRD_REG0__TXPWR_6_MASK 0x00000007L
+#define PB1_TX_LANE6_SCI_STAT_OVRD_REG0__TXPWR_6__SHIFT 0x00000000
+#define PB1_TX_LANE7_CTRL_REG0__TX_CFG_DISPCLK_MODE_7_MASK 0x00000001L
+#define PB1_TX_LANE7_CTRL_REG0__TX_CFG_DISPCLK_MODE_7__SHIFT 0x00000000
+#define PB1_TX_LANE7_CTRL_REG0__TX_CFG_INV_DATA_7_MASK 0x00000002L
+#define PB1_TX_LANE7_CTRL_REG0__TX_CFG_INV_DATA_7__SHIFT 0x00000001
+#define PB1_TX_LANE7_CTRL_REG0__TX_CFG_SWING_BOOST_EN_7_MASK 0x00000004L
+#define PB1_TX_LANE7_CTRL_REG0__TX_CFG_SWING_BOOST_EN_7__SHIFT 0x00000002
+#define PB1_TX_LANE7_CTRL_REG0__TX_DBG_PRBS_EN_7_MASK 0x00000008L
+#define PB1_TX_LANE7_CTRL_REG0__TX_DBG_PRBS_EN_7__SHIFT 0x00000003
+#define PB1_TX_LANE7_OVRD_REG0__TX_DCLK_EN_OVRD_EN_7_MASK 0x00000002L
+#define PB1_TX_LANE7_OVRD_REG0__TX_DCLK_EN_OVRD_EN_7__SHIFT 0x00000001
+#define PB1_TX_LANE7_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_7_MASK 0x00000001L
+#define PB1_TX_LANE7_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_7__SHIFT 0x00000000
+#define PB1_TX_LANE7_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_7_MASK 0x00000008L
+#define PB1_TX_LANE7_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_7__SHIFT 0x00000003
+#define PB1_TX_LANE7_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_7_MASK 0x00000004L
+#define PB1_TX_LANE7_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_7__SHIFT 0x00000002
+#define PB1_TX_LANE7_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_7_MASK 0x00000020L
+#define PB1_TX_LANE7_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_7__SHIFT 0x00000005
+#define PB1_TX_LANE7_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_7_MASK 0x00000010L
+#define PB1_TX_LANE7_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_7__SHIFT 0x00000004
+#define PB1_TX_LANE7_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_7_MASK 0x00000080L
+#define PB1_TX_LANE7_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_7__SHIFT 0x00000007
+#define PB1_TX_LANE7_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_7_MASK 0x00000040L
+#define PB1_TX_LANE7_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_7__SHIFT 0x00000006
+#define PB1_TX_LANE7_SCI_STAT_OVRD_REG0__COEFFICIENT_7_MASK 0x0000fc00L
+#define PB1_TX_LANE7_SCI_STAT_OVRD_REG0__COEFFICIENT_7__SHIFT 0x0000000a
+#define PB1_TX_LANE7_SCI_STAT_OVRD_REG0__COEFFICIENTID_7_MASK 0x00000300L
+#define PB1_TX_LANE7_SCI_STAT_OVRD_REG0__COEFFICIENTID_7__SHIFT 0x00000008
+#define PB1_TX_LANE7_SCI_STAT_OVRD_REG0__DEEMPH_7_MASK 0x00000080L
+#define PB1_TX_LANE7_SCI_STAT_OVRD_REG0__DEEMPH_7__SHIFT 0x00000007
+#define PB1_TX_LANE7_SCI_STAT_OVRD_REG0__INCOHERENTCK_7_MASK 0x00000008L
+#define PB1_TX_LANE7_SCI_STAT_OVRD_REG0__INCOHERENTCK_7__SHIFT 0x00000003
+#define PB1_TX_LANE7_SCI_STAT_OVRD_REG0__TXMARG_7_MASK 0x00000070L
+#define PB1_TX_LANE7_SCI_STAT_OVRD_REG0__TXMARG_7__SHIFT 0x00000004
+#define PB1_TX_LANE7_SCI_STAT_OVRD_REG0__TXPWR_7_MASK 0x00000007L
+#define PB1_TX_LANE7_SCI_STAT_OVRD_REG0__TXPWR_7__SHIFT 0x00000000
+#define PB1_TX_LANE8_CTRL_REG0__TX_CFG_DISPCLK_MODE_8_MASK 0x00000001L
+#define PB1_TX_LANE8_CTRL_REG0__TX_CFG_DISPCLK_MODE_8__SHIFT 0x00000000
+#define PB1_TX_LANE8_CTRL_REG0__TX_CFG_INV_DATA_8_MASK 0x00000002L
+#define PB1_TX_LANE8_CTRL_REG0__TX_CFG_INV_DATA_8__SHIFT 0x00000001
+#define PB1_TX_LANE8_CTRL_REG0__TX_CFG_SWING_BOOST_EN_8_MASK 0x00000004L
+#define PB1_TX_LANE8_CTRL_REG0__TX_CFG_SWING_BOOST_EN_8__SHIFT 0x00000002
+#define PB1_TX_LANE8_CTRL_REG0__TX_DBG_PRBS_EN_8_MASK 0x00000008L
+#define PB1_TX_LANE8_CTRL_REG0__TX_DBG_PRBS_EN_8__SHIFT 0x00000003
+#define PB1_TX_LANE8_OVRD_REG0__TX_DCLK_EN_OVRD_EN_8_MASK 0x00000002L
+#define PB1_TX_LANE8_OVRD_REG0__TX_DCLK_EN_OVRD_EN_8__SHIFT 0x00000001
+#define PB1_TX_LANE8_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_8_MASK 0x00000001L
+#define PB1_TX_LANE8_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_8__SHIFT 0x00000000
+#define PB1_TX_LANE8_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_8_MASK 0x00000008L
+#define PB1_TX_LANE8_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_8__SHIFT 0x00000003
+#define PB1_TX_LANE8_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_8_MASK 0x00000004L
+#define PB1_TX_LANE8_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_8__SHIFT 0x00000002
+#define PB1_TX_LANE8_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_8_MASK 0x00000020L
+#define PB1_TX_LANE8_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_8__SHIFT 0x00000005
+#define PB1_TX_LANE8_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_8_MASK 0x00000010L
+#define PB1_TX_LANE8_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_8__SHIFT 0x00000004
+#define PB1_TX_LANE8_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_8_MASK 0x00000080L
+#define PB1_TX_LANE8_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_8__SHIFT 0x00000007
+#define PB1_TX_LANE8_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_8_MASK 0x00000040L
+#define PB1_TX_LANE8_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_8__SHIFT 0x00000006
+#define PB1_TX_LANE8_SCI_STAT_OVRD_REG0__COEFFICIENT_8_MASK 0x0000fc00L
+#define PB1_TX_LANE8_SCI_STAT_OVRD_REG0__COEFFICIENT_8__SHIFT 0x0000000a
+#define PB1_TX_LANE8_SCI_STAT_OVRD_REG0__COEFFICIENTID_8_MASK 0x00000300L
+#define PB1_TX_LANE8_SCI_STAT_OVRD_REG0__COEFFICIENTID_8__SHIFT 0x00000008
+#define PB1_TX_LANE8_SCI_STAT_OVRD_REG0__DEEMPH_8_MASK 0x00000080L
+#define PB1_TX_LANE8_SCI_STAT_OVRD_REG0__DEEMPH_8__SHIFT 0x00000007
+#define PB1_TX_LANE8_SCI_STAT_OVRD_REG0__INCOHERENTCK_8_MASK 0x00000008L
+#define PB1_TX_LANE8_SCI_STAT_OVRD_REG0__INCOHERENTCK_8__SHIFT 0x00000003
+#define PB1_TX_LANE8_SCI_STAT_OVRD_REG0__TXMARG_8_MASK 0x00000070L
+#define PB1_TX_LANE8_SCI_STAT_OVRD_REG0__TXMARG_8__SHIFT 0x00000004
+#define PB1_TX_LANE8_SCI_STAT_OVRD_REG0__TXPWR_8_MASK 0x00000007L
+#define PB1_TX_LANE8_SCI_STAT_OVRD_REG0__TXPWR_8__SHIFT 0x00000000
+#define PB1_TX_LANE9_CTRL_REG0__TX_CFG_DISPCLK_MODE_9_MASK 0x00000001L
+#define PB1_TX_LANE9_CTRL_REG0__TX_CFG_DISPCLK_MODE_9__SHIFT 0x00000000
+#define PB1_TX_LANE9_CTRL_REG0__TX_CFG_INV_DATA_9_MASK 0x00000002L
+#define PB1_TX_LANE9_CTRL_REG0__TX_CFG_INV_DATA_9__SHIFT 0x00000001
+#define PB1_TX_LANE9_CTRL_REG0__TX_CFG_SWING_BOOST_EN_9_MASK 0x00000004L
+#define PB1_TX_LANE9_CTRL_REG0__TX_CFG_SWING_BOOST_EN_9__SHIFT 0x00000002
+#define PB1_TX_LANE9_CTRL_REG0__TX_DBG_PRBS_EN_9_MASK 0x00000008L
+#define PB1_TX_LANE9_CTRL_REG0__TX_DBG_PRBS_EN_9__SHIFT 0x00000003
+#define PB1_TX_LANE9_OVRD_REG0__TX_DCLK_EN_OVRD_EN_9_MASK 0x00000002L
+#define PB1_TX_LANE9_OVRD_REG0__TX_DCLK_EN_OVRD_EN_9__SHIFT 0x00000001
+#define PB1_TX_LANE9_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_9_MASK 0x00000001L
+#define PB1_TX_LANE9_OVRD_REG0__TX_DCLK_EN_OVRD_VAL_9__SHIFT 0x00000000
+#define PB1_TX_LANE9_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_9_MASK 0x00000008L
+#define PB1_TX_LANE9_OVRD_REG0__TX_DRV_DATA_EN_OVRD_EN_9__SHIFT 0x00000003
+#define PB1_TX_LANE9_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_9_MASK 0x00000004L
+#define PB1_TX_LANE9_OVRD_REG0__TX_DRV_DATA_EN_OVRD_VAL_9__SHIFT 0x00000002
+#define PB1_TX_LANE9_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_9_MASK 0x00000020L
+#define PB1_TX_LANE9_OVRD_REG0__TX_DRV_PWRON_OVRD_EN_9__SHIFT 0x00000005
+#define PB1_TX_LANE9_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_9_MASK 0x00000010L
+#define PB1_TX_LANE9_OVRD_REG0__TX_DRV_PWRON_OVRD_VAL_9__SHIFT 0x00000004
+#define PB1_TX_LANE9_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_9_MASK 0x00000080L
+#define PB1_TX_LANE9_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_EN_9__SHIFT 0x00000007
+#define PB1_TX_LANE9_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_9_MASK 0x00000040L
+#define PB1_TX_LANE9_OVRD_REG0__TX_FRONTEND_PWRON_OVRD_VAL_9__SHIFT 0x00000006
+#define PB1_TX_LANE9_SCI_STAT_OVRD_REG0__COEFFICIENT_9_MASK 0x0000fc00L
+#define PB1_TX_LANE9_SCI_STAT_OVRD_REG0__COEFFICIENT_9__SHIFT 0x0000000a
+#define PB1_TX_LANE9_SCI_STAT_OVRD_REG0__COEFFICIENTID_9_MASK 0x00000300L
+#define PB1_TX_LANE9_SCI_STAT_OVRD_REG0__COEFFICIENTID_9__SHIFT 0x00000008
+#define PB1_TX_LANE9_SCI_STAT_OVRD_REG0__DEEMPH_9_MASK 0x00000080L
+#define PB1_TX_LANE9_SCI_STAT_OVRD_REG0__DEEMPH_9__SHIFT 0x00000007
+#define PB1_TX_LANE9_SCI_STAT_OVRD_REG0__INCOHERENTCK_9_MASK 0x00000008L
+#define PB1_TX_LANE9_SCI_STAT_OVRD_REG0__INCOHERENTCK_9__SHIFT 0x00000003
+#define PB1_TX_LANE9_SCI_STAT_OVRD_REG0__TXMARG_9_MASK 0x00000070L
+#define PB1_TX_LANE9_SCI_STAT_OVRD_REG0__TXMARG_9__SHIFT 0x00000004
+#define PB1_TX_LANE9_SCI_STAT_OVRD_REG0__TXPWR_9_MASK 0x00000007L
+#define PB1_TX_LANE9_SCI_STAT_OVRD_REG0__TXPWR_9__SHIFT 0x00000000
+#define PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK 0x00000080L
+#define PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT 0x00000007
+#define PCIE_BUS_CNTL__PMI_INT_DIS_MASK 0x00000040L
+#define PCIE_BUS_CNTL__PMI_INT_DIS__SHIFT 0x00000006
+#define PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK 0x00000001L
+#define PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT 0x00000000
+#define PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK 0x00000004L
+#define PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT 0x00000002
+#define PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK 0x00000002L
+#define PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT 0x00000001
+#define PCIE_CI_CNTL__CI_MST_CMPL_DUMMY_DATA_MASK 0x00000010L
+#define PCIE_CI_CNTL__CI_MST_CMPL_DUMMY_DATA__SHIFT 0x00000004
+#define PCIE_CI_CNTL__CI_MST_IGNORE_PAGE_ALIGNED_REQUEST_MASK 0x00002000L
+#define PCIE_CI_CNTL__CI_MST_IGNORE_PAGE_ALIGNED_REQUEST__SHIFT 0x0000000d
+#define PCIE_CI_CNTL__CI_RC_ORDERING_DIS_MASK 0x00000200L
+#define PCIE_CI_CNTL__CI_RC_ORDERING_DIS__SHIFT 0x00000009
+#define PCIE_CI_CNTL__CI_SLAVE_GEN_USR_DIS_MASK 0x00000008L
+#define PCIE_CI_CNTL__CI_SLAVE_GEN_USR_DIS__SHIFT 0x00000003
+#define PCIE_CI_CNTL__CI_SLAVE_SPLIT_MODE_MASK 0x00000004L
+#define PCIE_CI_CNTL__CI_SLAVE_SPLIT_MODE__SHIFT 0x00000002
+#define PCIE_CI_CNTL__CI_SLV_CPL_ALLOC_DIS_MASK 0x00000400L
+#define PCIE_CI_CNTL__CI_SLV_CPL_ALLOC_DIS__SHIFT 0x0000000a
+#define PCIE_CI_CNTL__CI_SLV_CPL_ALLOC_MODE_MASK 0x00000800L
+#define PCIE_CI_CNTL__CI_SLV_CPL_ALLOC_MODE__SHIFT 0x0000000b
+#define PCIE_CI_CNTL__CI_SLV_CPL_ALLOC_SOR_MASK 0x00001000L
+#define PCIE_CI_CNTL__CI_SLV_CPL_ALLOC_SOR__SHIFT 0x0000000c
+#define PCIE_CI_CNTL__CI_SLV_ORDERING_DIS_MASK 0x00000100L
+#define PCIE_CI_CNTL__CI_SLV_ORDERING_DIS__SHIFT 0x00000008
+#define PCIE_CI_CNTL__CI_SLV_RC_RD_REQ_SIZE_MASK 0x000000c0L
+#define PCIE_CI_CNTL__CI_SLV_RC_RD_REQ_SIZE__SHIFT 0x00000006
+#define PCIE_CNTL2__MST_MEM_LS_EN_MASK 0x00040000L
+#define PCIE_CNTL2__MST_MEM_LS_EN__SHIFT 0x00000012
+#define PCIE_CNTL2__MST_MEM_SD_EN_MASK 0x00400000L
+#define PCIE_CNTL2__MST_MEM_SD_EN__SHIFT 0x00000016
+#define PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK 0x00080000L
+#define PCIE_CNTL2__REPLAY_MEM_LS_EN__SHIFT 0x00000013
+#define PCIE_CNTL2__REPLAY_MEM_SD_EN_MASK 0x00800000L
+#define PCIE_CNTL2__REPLAY_MEM_SD_EN__SHIFT 0x00000017
+#define PCIE_CNTL2__RX_NP_MEM_WRITE_ENCODING_MASK 0x1f000000L
+#define PCIE_CNTL2__RX_NP_MEM_WRITE_ENCODING__SHIFT 0x00000018
+#define PCIE_CNTL2__SLV_MEM_AGGRESSIVE_LS_EN_MASK 0x00020000L
+#define PCIE_CNTL2__SLV_MEM_AGGRESSIVE_LS_EN__SHIFT 0x00000011
+#define PCIE_CNTL2__SLV_MEM_AGGRESSIVE_SD_EN_MASK 0x00200000L
+#define PCIE_CNTL2__SLV_MEM_AGGRESSIVE_SD_EN__SHIFT 0x00000015
+#define PCIE_CNTL2__SLV_MEM_LS_EN_MASK 0x00010000L
+#define PCIE_CNTL2__SLV_MEM_LS_EN__SHIFT 0x00000010
+#define PCIE_CNTL2__SLV_MEM_SD_EN_MASK 0x00100000L
+#define PCIE_CNTL2__SLV_MEM_SD_EN__SHIFT 0x00000014
+#define PCIE_CNTL2__TX_ARB_MST_LIMIT_MASK 0x000007c0L
+#define PCIE_CNTL2__TX_ARB_MST_LIMIT__SHIFT 0x00000006
+#define PCIE_CNTL2__TX_ARB_ROUND_ROBIN_EN_MASK 0x00000001L
+#define PCIE_CNTL2__TX_ARB_ROUND_ROBIN_EN__SHIFT 0x00000000
+#define PCIE_CNTL2__TX_ARB_SLV_LIMIT_MASK 0x0000003eL
+#define PCIE_CNTL2__TX_ARB_SLV_LIMIT__SHIFT 0x00000001
+#define PCIE_CNTL__HWINIT_WR_LOCK_MASK 0x00000001L
+#define PCIE_CNTL__HWINIT_WR_LOCK__SHIFT 0x00000000
+#define PCIE_CNTL__LC_HOT_PLUG_DELAY_SEL_MASK 0x0000000eL
+#define PCIE_CNTL__LC_HOT_PLUG_DELAY_SEL__SHIFT 0x00000001
+#define PCIE_CNTL__PCIE_HT_NP_MEM_WRITE_MASK 0x00000200L
+#define PCIE_CNTL__PCIE_HT_NP_MEM_WRITE__SHIFT 0x00000009
+#define PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK 0x00000100L
+#define PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT 0x00000008
+#define PCIE_CNTL__RX_ATS_TRAN_CPL_SPLIT_DIS_MASK 0x00800000L
+#define PCIE_CNTL__RX_ATS_TRAN_CPL_SPLIT_DIS__SHIFT 0x00000017
+#define PCIE_CNTL__RX_CPL_POSTED_REQ_ORD_EN_MASK 0x80000000L
+#define PCIE_CNTL__RX_CPL_POSTED_REQ_ORD_EN__SHIFT 0x0000001f
+#define PCIE_CNTL__RX_RCB_ATS_UC_DIS_MASK 0x00008000L
+#define PCIE_CNTL__RX_RCB_ATS_UC_DIS__SHIFT 0x0000000f
+#define PCIE_CNTL__RX_RCB_CHANNEL_ORDERING_MASK 0x00100000L
+#define PCIE_CNTL__RX_RCB_CHANNEL_ORDERING__SHIFT 0x00000014
+#define PCIE_CNTL__RX_RCB_CPL_TIMEOUT_TEST_MODE_MASK 0x00080000L
+#define PCIE_CNTL__RX_RCB_CPL_TIMEOUT_TEST_MODE__SHIFT 0x00000013
+#define PCIE_CNTL__RX_RCB_INVALID_SIZE_DIS_MASK 0x00020000L
+#define PCIE_CNTL__RX_RCB_INVALID_SIZE_DIS__SHIFT 0x00000011
+#define PCIE_CNTL__RX_RCB_REORDER_EN_MASK 0x00010000L
+#define PCIE_CNTL__RX_RCB_REORDER_EN__SHIFT 0x00000010
+#define PCIE_CNTL__RX_RCB_UNEXP_CPL_DIS_MASK 0x00040000L
+#define PCIE_CNTL__RX_RCB_UNEXP_CPL_DIS__SHIFT 0x00000012
+#define PCIE_CNTL__RX_RCB_WRONG_ATTR_DIS_MASK 0x00200000L
+#define PCIE_CNTL__RX_RCB_WRONG_ATTR_DIS__SHIFT 0x00000015
+#define PCIE_CNTL__RX_RCB_WRONG_FUNCNUM_DIS_MASK 0x00400000L
+#define PCIE_CNTL__RX_RCB_WRONG_FUNCNUM_DIS__SHIFT 0x00000016
+#define PCIE_CNTL__RX_SB_ADJ_PAYLOAD_SIZE_MASK 0x00001c00L
+#define PCIE_CNTL__RX_SB_ADJ_PAYLOAD_SIZE__SHIFT 0x0000000a
+#define PCIE_CNTL__TX_CPL_DEBUG_MASK 0x3f000000L
+#define PCIE_CNTL__TX_CPL_DEBUG__SHIFT 0x00000018
+#define PCIE_CNTL__UR_ERR_REPORT_DIS_MASK 0x00000080L
+#define PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT 0x00000007
+#define PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK 0x02000000L
+#define PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT 0x00000019
+#define PCIE_CONFIG_CNTL__CI_MAX_PAYLOAD_SIZE_MODE_MASK 0x00010000L
+#define PCIE_CONFIG_CNTL__CI_MAX_PAYLOAD_SIZE_MODE__SHIFT 0x00000010
+#define PCIE_CONFIG_CNTL__CI_MAX_READ_REQUEST_SIZE_MODE_MASK 0x00100000L
+#define PCIE_CONFIG_CNTL__CI_MAX_READ_REQUEST_SIZE_MODE__SHIFT 0x00000014
+#define PCIE_CONFIG_CNTL__CI_MAX_READ_SAFE_MODE_MASK 0x01000000L
+#define PCIE_CONFIG_CNTL__CI_MAX_READ_SAFE_MODE__SHIFT 0x00000018
+#define PCIE_CONFIG_CNTL__CI_PRIV_MAX_PAYLOAD_SIZE_MASK 0x000e0000L
+#define PCIE_CONFIG_CNTL__CI_PRIV_MAX_PAYLOAD_SIZE__SHIFT 0x00000011
+#define PCIE_CONFIG_CNTL__CI_PRIV_MAX_READ_REQUEST_SIZE_MASK 0x00e00000L
+#define PCIE_CONFIG_CNTL__CI_PRIV_MAX_READ_REQUEST_SIZE__SHIFT 0x00000015
+#define PCIE_CONFIG_CNTL__DYN_CLK_LATENCY_MASK 0x0000000fL
+#define PCIE_CONFIG_CNTL__DYN_CLK_LATENCY__SHIFT 0x00000000
+#define PCIE_DATA__PCIE_DATA_MASK 0xffffffffL
+#define PCIE_DATA__PCIE_DATA__SHIFT 0x00000000
+#define PCIE_DEBUG_CNTL__DEBUG_LANE_EN_MASK 0xffff0000L
+#define PCIE_DEBUG_CNTL__DEBUG_LANE_EN__SHIFT 0x00000010
+#define PCIE_DEBUG_CNTL__DEBUG_PORT_EN_MASK 0x000000ffL
+#define PCIE_DEBUG_CNTL__DEBUG_PORT_EN__SHIFT 0x00000000
+#define PCIE_DEBUG_CNTL__DEBUG_SELECT_MASK 0x00000100L
+#define PCIE_DEBUG_CNTL__DEBUG_SELECT__SHIFT 0x00000008
+#define PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK 0x00000800L
+#define PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT 0x0000000b
+#define PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK 0x00001000L
+#define PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT 0x0000000c
+#define PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK 0x00002000L
+#define PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT 0x0000000d
+#define PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK 0x00000700L
+#define PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT 0x00000008
+#define PCIE_ERR_CNTL__CI_NP_SLV_BUF_RD_HALT_STATUS_MASK 0x00008000L
+#define PCIE_ERR_CNTL__CI_NP_SLV_BUF_RD_HALT_STATUS__SHIFT 0x0000000f
+#define PCIE_ERR_CNTL__CI_P_SLV_BUF_RD_HALT_STATUS_MASK 0x00004000L
+#define PCIE_ERR_CNTL__CI_P_SLV_BUF_RD_HALT_STATUS__SHIFT 0x0000000e
+#define PCIE_ERR_CNTL__CI_SLV_BUF_HALT_RESET_MASK 0x00010000L
+#define PCIE_ERR_CNTL__CI_SLV_BUF_HALT_RESET__SHIFT 0x00000010
+#define PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK 0x00000001L
+#define PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT 0x00000000
+#define PCIE_ERR_CNTL__RX_GENERATE_ECRC_ERR_MASK 0x00000080L
+#define PCIE_ERR_CNTL__RX_GENERATE_ECRC_ERR__SHIFT 0x00000007
+#define PCIE_ERR_CNTL__RX_GENERATE_LCRC_ERR_MASK 0x00000020L
+#define PCIE_ERR_CNTL__RX_GENERATE_LCRC_ERR__SHIFT 0x00000005
+#define PCIE_ERR_CNTL__STRAP_FIRST_RCVD_ERR_LOG_MASK 0x00000002L
+#define PCIE_ERR_CNTL__STRAP_FIRST_RCVD_ERR_LOG__SHIFT 0x00000001
+#define PCIE_ERR_CNTL__TX_GENERATE_ECRC_ERR_MASK 0x00000040L
+#define PCIE_ERR_CNTL__TX_GENERATE_ECRC_ERR__SHIFT 0x00000006
+#define PCIE_ERR_CNTL__TX_GENERATE_LCRC_ERR_MASK 0x00000010L
+#define PCIE_ERR_CNTL__TX_GENERATE_LCRC_ERR__SHIFT 0x00000004
+#define PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK 0x00003000L
+#define PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT 0x0000000c
+#define PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK 0x00000300L
+#define PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT 0x00000008
+#define PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK 0x00ff0000L
+#define PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT 0x00000010
+#define PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK 0xff000000L
+#define PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT 0x00000018
+#define PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK 0x0000001fL
+#define PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT 0x00000000
+#define PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK 0x000000ffL
+#define PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT 0x00000000
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK 0x000000ffL
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT 0x00000000
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK 0x000000ffL
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT 0x00000000
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK 0x000000ffL
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT 0x00000000
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK 0x000000ffL
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT 0x00000000
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK 0x000000ffL
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT 0x00000000
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK 0x000000ffL
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT 0x00000000
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK 0x000000ffL
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT 0x00000000
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK 0x000000ffL
+#define PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT 0x00000000
+#define PCIE_FC_CPL__CPLD_CREDITS_MASK 0x000000ffL
+#define PCIE_FC_CPL__CPLD_CREDITS__SHIFT 0x00000000
+#define PCIE_FC_CPL__CPLH_CREDITS_MASK 0x0000ff00L
+#define PCIE_FC_CPL__CPLH_CREDITS__SHIFT 0x00000008
+#define PCIE_FC_NP__NPD_CREDITS_MASK 0x000000ffL
+#define PCIE_FC_NP__NPD_CREDITS__SHIFT 0x00000000
+#define PCIE_FC_NP__NPH_CREDITS_MASK 0x0000ff00L
+#define PCIE_FC_NP__NPH_CREDITS__SHIFT 0x00000008
+#define PCIE_FC_P__PD_CREDITS_MASK 0x000000ffL
+#define PCIE_FC_P__PD_CREDITS__SHIFT 0x00000000
+#define PCIE_FC_P__PH_CREDITS_MASK 0x0000ff00L
+#define PCIE_FC_P__PH_CREDITS__SHIFT 0x00000008
+#define PCIE_HW_DEBUG__HW_00_DEBUG_MASK 0x00000001L
+#define PCIE_HW_DEBUG__HW_00_DEBUG__SHIFT 0x00000000
+#define PCIE_HW_DEBUG__HW_01_DEBUG_MASK 0x00000002L
+#define PCIE_HW_DEBUG__HW_01_DEBUG__SHIFT 0x00000001
+#define PCIE_HW_DEBUG__HW_02_DEBUG_MASK 0x00000004L
+#define PCIE_HW_DEBUG__HW_02_DEBUG__SHIFT 0x00000002
+#define PCIE_HW_DEBUG__HW_03_DEBUG_MASK 0x00000008L
+#define PCIE_HW_DEBUG__HW_03_DEBUG__SHIFT 0x00000003
+#define PCIE_HW_DEBUG__HW_04_DEBUG_MASK 0x00000010L
+#define PCIE_HW_DEBUG__HW_04_DEBUG__SHIFT 0x00000004
+#define PCIE_HW_DEBUG__HW_05_DEBUG_MASK 0x00000020L
+#define PCIE_HW_DEBUG__HW_05_DEBUG__SHIFT 0x00000005
+#define PCIE_HW_DEBUG__HW_06_DEBUG_MASK 0x00000040L
+#define PCIE_HW_DEBUG__HW_06_DEBUG__SHIFT 0x00000006
+#define PCIE_HW_DEBUG__HW_07_DEBUG_MASK 0x00000080L
+#define PCIE_HW_DEBUG__HW_07_DEBUG__SHIFT 0x00000007
+#define PCIE_HW_DEBUG__HW_08_DEBUG_MASK 0x00000100L
+#define PCIE_HW_DEBUG__HW_08_DEBUG__SHIFT 0x00000008
+#define PCIE_HW_DEBUG__HW_09_DEBUG_MASK 0x00000200L
+#define PCIE_HW_DEBUG__HW_09_DEBUG__SHIFT 0x00000009
+#define PCIE_HW_DEBUG__HW_10_DEBUG_MASK 0x00000400L
+#define PCIE_HW_DEBUG__HW_10_DEBUG__SHIFT 0x0000000a
+#define PCIE_HW_DEBUG__HW_11_DEBUG_MASK 0x00000800L
+#define PCIE_HW_DEBUG__HW_11_DEBUG__SHIFT 0x0000000b
+#define PCIE_HW_DEBUG__HW_12_DEBUG_MASK 0x00001000L
+#define PCIE_HW_DEBUG__HW_12_DEBUG__SHIFT 0x0000000c
+#define PCIE_HW_DEBUG__HW_13_DEBUG_MASK 0x00002000L
+#define PCIE_HW_DEBUG__HW_13_DEBUG__SHIFT 0x0000000d
+#define PCIE_HW_DEBUG__HW_14_DEBUG_MASK 0x00004000L
+#define PCIE_HW_DEBUG__HW_14_DEBUG__SHIFT 0x0000000e
+#define PCIE_HW_DEBUG__HW_15_DEBUG_MASK 0x00008000L
+#define PCIE_HW_DEBUG__HW_15_DEBUG__SHIFT 0x0000000f
+#define PCIE_I2C_REG_ADDR_EXPAND__I2C_REG_ADDR_MASK 0x0001ffffL
+#define PCIE_I2C_REG_ADDR_EXPAND__I2C_REG_ADDR__SHIFT 0x00000000
+#define PCIE_I2C_REG_DATA__I2C_REG_DATA_MASK 0xffffffffL
+#define PCIE_I2C_REG_DATA__I2C_REG_DATA__SHIFT 0x00000000
+#define PCIE_INDEX__PCIE_INDEX_MASK 0x000000ffL
+#define PCIE_INDEX__PCIE_INDEX__SHIFT 0x00000000
+#define PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK 0x00000001L
+#define PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT 0x00000000
+#define PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK 0x00000004L
+#define PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT 0x00000002
+#define PCIE_INT_CNTL__LINK_BW_INT_EN_MASK 0x00000080L
+#define PCIE_INT_CNTL__LINK_BW_INT_EN__SHIFT 0x00000007
+#define PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK 0x00000010L
+#define PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT 0x00000004
+#define PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK 0x00000002L
+#define PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT 0x00000001
+#define PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK 0x00000040L
+#define PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT 0x00000006
+#define PCIE_INT_CNTL__QUIESCE_RCVD_INT_EN_MASK 0x00000100L
+#define PCIE_INT_CNTL__QUIESCE_RCVD_INT_EN__SHIFT 0x00000008
+#define PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK 0x00000008L
+#define PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT 0x00000003
+#define PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK 0x00000001L
+#define PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT 0x00000000
+#define PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK 0x00000004L
+#define PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT 0x00000002
+#define PCIE_INT_STATUS__LINK_BW_INT_STATUS_MASK 0x00000080L
+#define PCIE_INT_STATUS__LINK_BW_INT_STATUS__SHIFT 0x00000007
+#define PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK 0x00000010L
+#define PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT 0x00000004
+#define PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK 0x00000002L
+#define PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT 0x00000001
+#define PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK 0x00000040L
+#define PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT 0x00000006
+#define PCIE_INT_STATUS__QUIESCE_RCVD_INT_STATUS_MASK 0x00000100L
+#define PCIE_INT_STATUS__QUIESCE_RCVD_INT_STATUS__SHIFT 0x00000008
+#define PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK 0x00000008L
+#define PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT 0x00000003
+#define PCIE_LC_BEST_EQ_SETTINGS__LC_BEST_CURSOR_MASK 0x0000fc00L
+#define PCIE_LC_BEST_EQ_SETTINGS__LC_BEST_CURSOR__SHIFT 0x0000000a
+#define PCIE_LC_BEST_EQ_SETTINGS__LC_BEST_FOM_MASK 0x3fc00000L
+#define PCIE_LC_BEST_EQ_SETTINGS__LC_BEST_FOM__SHIFT 0x00000016
+#define PCIE_LC_BEST_EQ_SETTINGS__LC_BEST_POSTCURSOR_MASK 0x003f0000L
+#define PCIE_LC_BEST_EQ_SETTINGS__LC_BEST_POSTCURSOR__SHIFT 0x00000010
+#define PCIE_LC_BEST_EQ_SETTINGS__LC_BEST_PRECURSOR_MASK 0x000003f0L
+#define PCIE_LC_BEST_EQ_SETTINGS__LC_BEST_PRECURSOR__SHIFT 0x00000004
+#define PCIE_LC_BEST_EQ_SETTINGS__LC_BEST_PRESET_MASK 0x0000000fL
+#define PCIE_LC_BEST_EQ_SETTINGS__LC_BEST_PRESET__SHIFT 0x00000000
+#define PCIE_LC_BW_CHANGE_CNTL__LC_BW_CHANGE_INT_EN_MASK 0x00000001L
+#define PCIE_LC_BW_CHANGE_CNTL__LC_BW_CHANGE_INT_EN__SHIFT 0x00000000
+#define PCIE_LC_BW_CHANGE_CNTL__LC_FAILED_SPEED_NEG_MASK 0x00000020L
+#define PCIE_LC_BW_CHANGE_CNTL__LC_FAILED_SPEED_NEG__SHIFT 0x00000005
+#define PCIE_LC_BW_CHANGE_CNTL__LC_HW_INIT_SPEED_CHANGE_MASK 0x00000002L
+#define PCIE_LC_BW_CHANGE_CNTL__LC_HW_INIT_SPEED_CHANGE__SHIFT 0x00000001
+#define PCIE_LC_BW_CHANGE_CNTL__LC_LINK_BW_NOTIFICATION_DETECT_MODE_MASK 0x00000400L
+#define PCIE_LC_BW_CHANGE_CNTL__LC_LINK_BW_NOTIFICATION_DETECT_MODE__SHIFT 0x0000000a
+#define PCIE_LC_BW_CHANGE_CNTL__LC_LONG_LW_CHANGE_MASK 0x00000040L
+#define PCIE_LC_BW_CHANGE_CNTL__LC_LONG_LW_CHANGE__SHIFT 0x00000006
+#define PCIE_LC_BW_CHANGE_CNTL__LC_LW_CHANGE_FAILED_MASK 0x00000200L
+#define PCIE_LC_BW_CHANGE_CNTL__LC_LW_CHANGE_FAILED__SHIFT 0x00000009
+#define PCIE_LC_BW_CHANGE_CNTL__LC_LW_CHANGE_OTHER_MASK 0x00000100L
+#define PCIE_LC_BW_CHANGE_CNTL__LC_LW_CHANGE_OTHER__SHIFT 0x00000008
+#define PCIE_LC_BW_CHANGE_CNTL__LC_OTHER_INIT_SPEED_CHANGE_MASK 0x00000008L
+#define PCIE_LC_BW_CHANGE_CNTL__LC_OTHER_INIT_SPEED_CHANGE__SHIFT 0x00000003
+#define PCIE_LC_BW_CHANGE_CNTL__LC_RELIABILITY_SPEED_CHANGE_MASK 0x00000010L
+#define PCIE_LC_BW_CHANGE_CNTL__LC_RELIABILITY_SPEED_CHANGE__SHIFT 0x00000004
+#define PCIE_LC_BW_CHANGE_CNTL__LC_SHORT_LW_CHANGE_MASK 0x00000080L
+#define PCIE_LC_BW_CHANGE_CNTL__LC_SHORT_LW_CHANGE__SHIFT 0x00000007
+#define PCIE_LC_BW_CHANGE_CNTL__LC_SW_INIT_SPEED_CHANGE_MASK 0x00000004L
+#define PCIE_LC_BW_CHANGE_CNTL__LC_SW_INIT_SPEED_CHANGE__SHIFT 0x00000002
+#define PCIE_LC_CDR_CNTL__LC_CDR_SET_TYPE_MASK 0x03000000L
+#define PCIE_LC_CDR_CNTL__LC_CDR_SET_TYPE__SHIFT 0x00000018
+#define PCIE_LC_CDR_CNTL__LC_CDR_TEST_OFF_MASK 0x00000fffL
+#define PCIE_LC_CDR_CNTL__LC_CDR_TEST_OFF__SHIFT 0x00000000
+#define PCIE_LC_CDR_CNTL__LC_CDR_TEST_SETS_MASK 0x00fff000L
+#define PCIE_LC_CDR_CNTL__LC_CDR_TEST_SETS__SHIFT 0x0000000c
+#define PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK 0x00020000L
+#define PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1__SHIFT 0x00000011
+#define PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK 0x00040000L
+#define PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23__SHIFT 0x00000012
+#define PCIE_LC_CNTL2__LC_ASSERT_INACTIVE_DURING_HOLD_MASK 0x00400000L
+#define PCIE_LC_CNTL2__LC_ASSERT_INACTIVE_DURING_HOLD__SHIFT 0x00000016
+#define PCIE_LC_CNTL2__LC_BLOCK_EL_IDLE_IN_L0_MASK 0x00100000L
+#define PCIE_LC_CNTL2__LC_BLOCK_EL_IDLE_IN_L0__SHIFT 0x00000014
+#define PCIE_LC_CNTL2__LC_DEASSERT_RX_EN_IN_L0S_MASK 0x00080000L
+#define PCIE_LC_CNTL2__LC_DEASSERT_RX_EN_IN_L0S__SHIFT 0x00000013
+#define PCIE_LC_CNTL2__LC_DISABLE_INFERRED_ELEC_IDLE_DET_MASK 0x00010000L
+#define PCIE_LC_CNTL2__LC_DISABLE_INFERRED_ELEC_IDLE_DET__SHIFT 0x00000010
+#define PCIE_LC_CNTL2__LC_DISABLE_LOST_SYM_LOCK_ARCS_MASK 0x04000000L
+#define PCIE_LC_CNTL2__LC_DISABLE_LOST_SYM_LOCK_ARCS__SHIFT 0x0000001a
+#define PCIE_LC_CNTL2__LC_ELEC_IDLE_MODE_MASK 0x0000c000L
+#define PCIE_LC_CNTL2__LC_ELEC_IDLE_MODE__SHIFT 0x0000000e
+#define PCIE_LC_CNTL2__LC_ENABLE_INFERRED_ELEC_IDLE_FOR_PI_MASK 0x80000000L
+#define PCIE_LC_CNTL2__LC_ENABLE_INFERRED_ELEC_IDLE_FOR_PI__SHIFT 0x0000001f
+#define PCIE_LC_CNTL2__LC_ILLEGAL_STATE_MASK 0x00000800L
+#define PCIE_LC_CNTL2__LC_ILLEGAL_STATE_RESTART_EN_MASK 0x00001000L
+#define PCIE_LC_CNTL2__LC_ILLEGAL_STATE_RESTART_EN__SHIFT 0x0000000c
+#define PCIE_LC_CNTL2__LC_ILLEGAL_STATE__SHIFT 0x0000000b
+#define PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK 0x08000000L
+#define PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT 0x0000001b
+#define PCIE_LC_CNTL2__LC_LINK_UP_REVERSAL_EN_MASK 0x00000400L
+#define PCIE_LC_CNTL2__LC_LINK_UP_REVERSAL_EN__SHIFT 0x0000000a
+#define PCIE_LC_CNTL2__LC_LOOK_FOR_BW_REDUCTION_MASK 0x00000080L
+#define PCIE_LC_CNTL2__LC_LOOK_FOR_BW_REDUCTION__SHIFT 0x00000007
+#define PCIE_LC_CNTL2__LC_MORE_TS2_EN_MASK 0x00000100L
+#define PCIE_LC_CNTL2__LC_MORE_TS2_EN__SHIFT 0x00000008
+#define PCIE_LC_CNTL2__LC_PMI_L1_WAIT_FOR_SLV_IDLE_MASK 0x10000000L
+#define PCIE_LC_CNTL2__LC_PMI_L1_WAIT_FOR_SLV_IDLE__SHIFT 0x0000001c
+#define PCIE_LC_CNTL2__LC_PWR_DOWN_NEG_OFF_LANES_MASK 0x02000000L
+#define PCIE_LC_CNTL2__LC_PWR_DOWN_NEG_OFF_LANES__SHIFT 0x00000019
+#define PCIE_LC_CNTL2__LC_RCV_L0_TO_RCV_L0S_DIS_MASK 0x00200000L
+#define PCIE_LC_CNTL2__LC_RCV_L0_TO_RCV_L0S_DIS__SHIFT 0x00000015
+#define PCIE_LC_CNTL2__LC_STATE_TIMED_OUT_MASK 0x00000040L
+#define PCIE_LC_CNTL2__LC_STATE_TIMED_OUT__SHIFT 0x00000006
+#define PCIE_LC_CNTL2__LC_TEST_TIMER_SEL_MASK 0x60000000L
+#define PCIE_LC_CNTL2__LC_TEST_TIMER_SEL__SHIFT 0x0000001d
+#define PCIE_LC_CNTL2__LC_TIMED_OUT_STATE_MASK 0x0000003fL
+#define PCIE_LC_CNTL2__LC_TIMED_OUT_STATE__SHIFT 0x00000000
+#define PCIE_LC_CNTL2__LC_WAIT_FOR_LANES_IN_LW_NEG_MASK 0x01800000L
+#define PCIE_LC_CNTL2__LC_WAIT_FOR_LANES_IN_LW_NEG__SHIFT 0x00000017
+#define PCIE_LC_CNTL2__LC_WAIT_FOR_OTHER_LANES_MODE_MASK 0x00002000L
+#define PCIE_LC_CNTL2__LC_WAIT_FOR_OTHER_LANES_MODE__SHIFT 0x0000000d
+#define PCIE_LC_CNTL2__LC_X12_NEGOTIATION_DIS_MASK 0x00000200L
+#define PCIE_LC_CNTL2__LC_X12_NEGOTIATION_DIS__SHIFT 0x00000009
+#define PCIE_LC_CNTL3__LC_AUTO_DISABLE_SPEED_SUPPORT_EN_MASK 0x00040000L
+#define PCIE_LC_CNTL3__LC_AUTO_DISABLE_SPEED_SUPPORT_EN__SHIFT 0x00000012
+#define PCIE_LC_CNTL3__LC_AUTO_DISABLE_SPEED_SUPPORT_MAX_FAIL_SEL_MASK 0x00180000L
+#define PCIE_LC_CNTL3__LC_AUTO_DISABLE_SPEED_SUPPORT_MAX_FAIL_SEL__SHIFT 0x00000013
+#define PCIE_LC_CNTL3__LC_AUTO_SPEED_CHANGE_ATTEMPT_FAILED_MASK 0x00000100L
+#define PCIE_LC_CNTL3__LC_AUTO_SPEED_CHANGE_ATTEMPT_FAILED__SHIFT 0x00000008
+#define PCIE_LC_CNTL3__LC_AUTO_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK 0x000000c0L
+#define PCIE_LC_CNTL3__LC_AUTO_SPEED_CHANGE_ATTEMPTS_ALLOWED__SHIFT 0x00000006
+#define PCIE_LC_CNTL3__LC_CHIP_BIF_USB_IDLE_EN_MASK 0x00010000L
+#define PCIE_LC_CNTL3__LC_CHIP_BIF_USB_IDLE_EN__SHIFT 0x00000010
+#define PCIE_LC_CNTL3__LC_CLR_FAILED_AUTO_SPD_CHANGE_CNT_MASK 0x00000200L
+#define PCIE_LC_CNTL3__LC_CLR_FAILED_AUTO_SPD_CHANGE_CNT__SHIFT 0x00000009
+#define PCIE_LC_CNTL3__LC_COMP_TO_DETECT_MASK 0x00000010L
+#define PCIE_LC_CNTL3__LC_COMP_TO_DETECT__SHIFT 0x00000004
+#define PCIE_LC_CNTL3__LC_DSC_DONT_ENTER_L23_AFTER_PME_ACK_MASK 0x00800000L
+#define PCIE_LC_CNTL3__LC_DSC_DONT_ENTER_L23_AFTER_PME_ACK__SHIFT 0x00000017
+#define PCIE_LC_CNTL3__LC_EHP_RX_PHY_CMD_MASK 0x00003000L
+#define PCIE_LC_CNTL3__LC_EHP_RX_PHY_CMD__SHIFT 0x0000000c
+#define PCIE_LC_CNTL3__LC_EHP_TX_PHY_CMD_MASK 0x0000c000L
+#define PCIE_LC_CNTL3__LC_EHP_TX_PHY_CMD__SHIFT 0x0000000e
+#define PCIE_LC_CNTL3__LC_ENHANCED_HOT_PLUG_EN_MASK 0x00000400L
+#define PCIE_LC_CNTL3__LC_ENHANCED_HOT_PLUG_EN__SHIFT 0x0000000a
+#define PCIE_LC_CNTL3__LC_FAST_L1_ENTRY_EXIT_EN_MASK 0x00200000L
+#define PCIE_LC_CNTL3__LC_FAST_L1_ENTRY_EXIT_EN__SHIFT 0x00000015
+#define PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK 0x40000000L
+#define PCIE_LC_CNTL3__LC_GO_TO_RECOVERY__SHIFT 0x0000001e
+#define PCIE_LC_CNTL3__LC_HW_VOLTAGE_IF_CONTROL_MASK 0x03000000L
+#define PCIE_LC_CNTL3__LC_HW_VOLTAGE_IF_CONTROL__SHIFT 0x00000018
+#define PCIE_LC_CNTL3__LC_L1_BLOCK_RECONFIG_EN_MASK 0x00020000L
+#define PCIE_LC_CNTL3__LC_L1_BLOCK_RECONFIG_EN__SHIFT 0x00000011
+#define PCIE_LC_CNTL3__LC_N_EIE_SEL_MASK 0x80000000L
+#define PCIE_LC_CNTL3__LC_N_EIE_SEL__SHIFT 0x0000001f
+#define PCIE_LC_CNTL3__LC_RCVD_DEEMPHASIS_MASK 0x00000008L
+#define PCIE_LC_CNTL3__LC_RCVD_DEEMPHASIS__SHIFT 0x00000003
+#define PCIE_LC_CNTL3__LC_RCVR_DET_EN_OVERRIDE_MASK 0x00000800L
+#define PCIE_LC_CNTL3__LC_RCVR_DET_EN_OVERRIDE__SHIFT 0x0000000b
+#define PCIE_LC_CNTL3__LC_RESET_TSX_CNT_IN_RLOCK_EN_MASK 0x00000020L
+#define PCIE_LC_CNTL3__LC_RESET_TSX_CNT_IN_RLOCK_EN__SHIFT 0x00000005
+#define PCIE_LC_CNTL3__LC_RXPHYCMD_INACTIVE_EN_MODE_MASK 0x00400000L
+#define PCIE_LC_CNTL3__LC_RXPHYCMD_INACTIVE_EN_MODE__SHIFT 0x00000016
+#define PCIE_LC_CNTL3__LC_SELECT_DEEMPHASIS_CNTL_MASK 0x00000006L
+#define PCIE_LC_CNTL3__LC_SELECT_DEEMPHASIS_CNTL__SHIFT 0x00000001
+#define PCIE_LC_CNTL3__LC_SELECT_DEEMPHASIS_MASK 0x00000001L
+#define PCIE_LC_CNTL3__LC_SELECT_DEEMPHASIS__SHIFT 0x00000000
+#define PCIE_LC_CNTL3__LC_VOLTAGE_TIMER_SEL_MASK 0x3c000000L
+#define PCIE_LC_CNTL3__LC_VOLTAGE_TIMER_SEL__SHIFT 0x0000001a
+#define PCIE_LC_CNTL4__LC_8GT_SKIP_ORDER_EN_MASK 0x02000000L
+#define PCIE_LC_CNTL4__LC_8GT_SKIP_ORDER_EN__SHIFT 0x00000019
+#define PCIE_LC_CNTL4__LC_BYPASS_EQ_MASK 0x00000010L
+#define PCIE_LC_CNTL4__LC_BYPASS_EQ_REQ_PHASE_MASK 0x00010000L
+#define PCIE_LC_CNTL4__LC_BYPASS_EQ_REQ_PHASE__SHIFT 0x00000010
+#define PCIE_LC_CNTL4__LC_BYPASS_EQ__SHIFT 0x00000004
+#define PCIE_LC_CNTL4__LC_DSC_CHECK_COEFFS_IN_RLOCK_MASK 0x00000400L
+#define PCIE_LC_CNTL4__LC_DSC_CHECK_COEFFS_IN_RLOCK__SHIFT 0x0000000a
+#define PCIE_LC_CNTL4__LC_EQ_SEARCH_MODE_MASK 0x00000300L
+#define PCIE_LC_CNTL4__LC_EQ_SEARCH_MODE__SHIFT 0x00000008
+#define PCIE_LC_CNTL4__LC_EQ_WAIT_FOR_EVAL_DONE_MASK 0x01000000L
+#define PCIE_LC_CNTL4__LC_EQ_WAIT_FOR_EVAL_DONE__SHIFT 0x00000018
+#define PCIE_LC_CNTL4__LC_EXTEND_EIEOS_MASK 0x00000040L
+#define PCIE_LC_CNTL4__LC_EXTEND_EIEOS__SHIFT 0x00000006
+#define PCIE_LC_CNTL4__LC_FORCE_PRESET_IN_EQ_REQ_PHASE_MASK 0x00020000L
+#define PCIE_LC_CNTL4__LC_FORCE_PRESET_IN_EQ_REQ_PHASE__SHIFT 0x00000011
+#define PCIE_LC_CNTL4__LC_FORCE_PRESET_VALUE_MASK 0x003c0000L
+#define PCIE_LC_CNTL4__LC_FORCE_PRESET_VALUE__SHIFT 0x00000012
+#define PCIE_LC_CNTL4__LC_IGNORE_PARITY_MASK 0x00000080L
+#define PCIE_LC_CNTL4__LC_IGNORE_PARITY__SHIFT 0x00000007
+#define PCIE_LC_CNTL4__LC_PCIE_TX_FULL_SWING_MASK 0x00800000L
+#define PCIE_LC_CNTL4__LC_PCIE_TX_FULL_SWING__SHIFT 0x00000017
+#define PCIE_LC_CNTL4__LC_QUIESCE_RCVD_MASK 0x00004000L
+#define PCIE_LC_CNTL4__LC_QUIESCE_RCVD__SHIFT 0x0000000e
+#define PCIE_LC_CNTL4__LC_REDO_EQ_MASK 0x00000020L
+#define PCIE_LC_CNTL4__LC_REDO_EQ__SHIFT 0x00000005
+#define PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK 0x00002000L
+#define PCIE_LC_CNTL4__LC_SET_QUIESCE__SHIFT 0x0000000d
+#define PCIE_LC_CNTL4__LC_TX_ENABLE_BEHAVIOUR_MASK 0x00000003L
+#define PCIE_LC_CNTL4__LC_TX_ENABLE_BEHAVIOUR__SHIFT 0x00000000
+#define PCIE_LC_CNTL4__LC_UNEXPECTED_COEFFS_RCVD_MASK 0x00008000L
+#define PCIE_LC_CNTL4__LC_UNEXPECTED_COEFFS_RCVD__SHIFT 0x0000000f
+#define PCIE_LC_CNTL4__LC_USC_DELAY_DLLPS_MASK 0x00400000L
+#define PCIE_LC_CNTL4__LC_USC_DELAY_DLLPS__SHIFT 0x00000016
+#define PCIE_LC_CNTL4__LC_USC_EQ_NOT_REQD_MASK 0x00000800L
+#define PCIE_LC_CNTL4__LC_USC_EQ_NOT_REQD__SHIFT 0x0000000b
+#define PCIE_LC_CNTL4__LC_USC_GO_TO_EQ_MASK 0x00001000L
+#define PCIE_LC_CNTL4__LC_USC_GO_TO_EQ__SHIFT 0x0000000c
+#define PCIE_LC_CNTL4__LC_WAIT_FOR_MORE_TS_IN_RLOCK_MASK 0xfc000000L
+#define PCIE_LC_CNTL4__LC_WAIT_FOR_MORE_TS_IN_RLOCK__SHIFT 0x0000001a
+#define PCIE_LC_CNTL5__LC_EQ_FS_0_MASK 0x0000003fL
+#define PCIE_LC_CNTL5__LC_EQ_FS_0__SHIFT 0x00000000
+#define PCIE_LC_CNTL5__LC_EQ_FS_8_MASK 0x00000fc0L
+#define PCIE_LC_CNTL5__LC_EQ_FS_8__SHIFT 0x00000006
+#define PCIE_LC_CNTL5__LC_EQ_LF_0_MASK 0x0003f000L
+#define PCIE_LC_CNTL5__LC_EQ_LF_0__SHIFT 0x0000000c
+#define PCIE_LC_CNTL5__LC_EQ_LF_8_MASK 0x00fc0000L
+#define PCIE_LC_CNTL5__LC_EQ_LF_8__SHIFT 0x00000012
+#define PCIE_LC_CNTL__LC_16X_CLEAR_TX_PIPE_MASK 0x000000f0L
+#define PCIE_LC_CNTL__LC_16X_CLEAR_TX_PIPE__SHIFT 0x00000004
+#define PCIE_LC_CNTL__LC_ASPM_TO_L1_DIS_MASK 0x01000000L
+#define PCIE_LC_CNTL__LC_ASPM_TO_L1_DIS__SHIFT 0x00000018
+#define PCIE_LC_CNTL__LC_DELAY_COUNT_MASK 0x06000000L
+#define PCIE_LC_CNTL__LC_DELAY_COUNT__SHIFT 0x00000019
+#define PCIE_LC_CNTL__LC_DELAY_L0S_EXIT_MASK 0x08000000L
+#define PCIE_LC_CNTL__LC_DELAY_L0S_EXIT__SHIFT 0x0000001b
+#define PCIE_LC_CNTL__LC_DELAY_L1_EXIT_MASK 0x10000000L
+#define PCIE_LC_CNTL__LC_DELAY_L1_EXIT__SHIFT 0x0000001c
+#define PCIE_LC_CNTL__LC_DONT_ENTER_L23_IN_D0_MASK 0x00000002L
+#define PCIE_LC_CNTL__LC_DONT_ENTER_L23_IN_D0__SHIFT 0x00000001
+#define PCIE_LC_CNTL__LC_ESCAPE_L1L23_EN_MASK 0x40000000L
+#define PCIE_LC_CNTL__LC_ESCAPE_L1L23_EN__SHIFT 0x0000001e
+#define PCIE_LC_CNTL__LC_EXTEND_WAIT_FOR_EL_IDLE_MASK 0x20000000L
+#define PCIE_LC_CNTL__LC_EXTEND_WAIT_FOR_EL_IDLE__SHIFT 0x0000001d
+#define PCIE_LC_CNTL__LC_FACTOR_IN_EXT_SYNC_MASK 0x00100000L
+#define PCIE_LC_CNTL__LC_FACTOR_IN_EXT_SYNC__SHIFT 0x00000014
+#define PCIE_LC_CNTL__LC_GATE_RCVR_IDLE_MASK 0x80000000L
+#define PCIE_LC_CNTL__LC_GATE_RCVR_IDLE__SHIFT 0x0000001f
+#define PCIE_LC_CNTL__LC_INC_N_FTS_EN_MASK 0x00020000L
+#define PCIE_LC_CNTL__LC_INC_N_FTS_EN__SHIFT 0x00000011
+#define PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK 0x00000f00L
+#define PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT 0x00000008
+#define PCIE_LC_CNTL__LC_L1_IMMEDIATE_ACK_MASK 0x00800000L
+#define PCIE_LC_CNTL__LC_L1_IMMEDIATE_ACK__SHIFT 0x00000017
+#define PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK 0x0000f000L
+#define PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT 0x0000000c
+#define PCIE_LC_CNTL__LC_LOOK_FOR_IDLE_IN_L1L23_MASK 0x000c0000L
+#define PCIE_LC_CNTL__LC_LOOK_FOR_IDLE_IN_L1L23__SHIFT 0x00000012
+#define PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK 0x00010000L
+#define PCIE_LC_CNTL__LC_PMI_TO_L1_DIS__SHIFT 0x00000010
+#define PCIE_LC_CNTL__LC_RESET_L_IDLE_COUNT_EN_MASK 0x00000004L
+#define PCIE_LC_CNTL__LC_RESET_L_IDLE_COUNT_EN__SHIFT 0x00000002
+#define PCIE_LC_CNTL__LC_RESET_LINK_MASK 0x00000008L
+#define PCIE_LC_CNTL__LC_RESET_LINK__SHIFT 0x00000003
+#define PCIE_LC_CNTL__LC_WAIT_FOR_PM_ACK_DIS_MASK 0x00200000L
+#define PCIE_LC_CNTL__LC_WAIT_FOR_PM_ACK_DIS__SHIFT 0x00000015
+#define PCIE_LC_CNTL__LC_WAKE_FROM_L23_MASK 0x00400000L
+#define PCIE_LC_CNTL__LC_WAKE_FROM_L23__SHIFT 0x00000016
+#define PCIE_LC_FORCE_COEFF__LC_3X3_COEFF_SEARCH_EN_MASK 0x00080000L
+#define PCIE_LC_FORCE_COEFF__LC_3X3_COEFF_SEARCH_EN__SHIFT 0x00000013
+#define PCIE_LC_FORCE_COEFF__LC_FORCE_COEFF_MASK 0x00000001L
+#define PCIE_LC_FORCE_COEFF__LC_FORCE_COEFF__SHIFT 0x00000000
+#define PCIE_LC_FORCE_COEFF__LC_FORCE_CURSOR_MASK 0x00001f80L
+#define PCIE_LC_FORCE_COEFF__LC_FORCE_CURSOR__SHIFT 0x00000007
+#define PCIE_LC_FORCE_COEFF__LC_FORCE_POST_CURSOR_MASK 0x0007e000L
+#define PCIE_LC_FORCE_COEFF__LC_FORCE_POST_CURSOR__SHIFT 0x0000000d
+#define PCIE_LC_FORCE_COEFF__LC_FORCE_PRE_CURSOR_MASK 0x0000007eL
+#define PCIE_LC_FORCE_COEFF__LC_FORCE_PRE_CURSOR__SHIFT 0x00000001
+#define PCIE_LC_FORCE_EQ_REQ_COEFF__LC_FORCE_COEFF_IN_EQ_REQ_PHASE_MASK 0x00000001L
+#define PCIE_LC_FORCE_EQ_REQ_COEFF__LC_FORCE_COEFF_IN_EQ_REQ_PHASE__SHIFT 0x00000000
+#define PCIE_LC_FORCE_EQ_REQ_COEFF__LC_FORCE_CURSOR_REQ_MASK 0x00001f80L
+#define PCIE_LC_FORCE_EQ_REQ_COEFF__LC_FORCE_CURSOR_REQ__SHIFT 0x00000007
+#define PCIE_LC_FORCE_EQ_REQ_COEFF__LC_FORCE_POST_CURSOR_REQ_MASK 0x0007e000L
+#define PCIE_LC_FORCE_EQ_REQ_COEFF__LC_FORCE_POST_CURSOR_REQ__SHIFT 0x0000000d
+#define PCIE_LC_FORCE_EQ_REQ_COEFF__LC_FORCE_PRE_CURSOR_REQ_MASK 0x0000007eL
+#define PCIE_LC_FORCE_EQ_REQ_COEFF__LC_FORCE_PRE_CURSOR_REQ__SHIFT 0x00000001
+#define PCIE_LC_FORCE_EQ_REQ_COEFF__LC_FS_OTHER_END_MASK 0x01f80000L
+#define PCIE_LC_FORCE_EQ_REQ_COEFF__LC_FS_OTHER_END__SHIFT 0x00000013
+#define PCIE_LC_FORCE_EQ_REQ_COEFF__LC_LF_OTHER_END_MASK 0x7e000000L
+#define PCIE_LC_FORCE_EQ_REQ_COEFF__LC_LF_OTHER_END__SHIFT 0x00000019
+#define PCIE_LC_LANE_CNTL__LC_CORRUPTED_LANES_MASK 0x0000ffffL
+#define PCIE_LC_LANE_CNTL__LC_CORRUPTED_LANES__SHIFT 0x00000000
+#define PCIE_LC_LANE_CNTL__LC_LANE_DIS_MASK 0xffff0000L
+#define PCIE_LC_LANE_CNTL__LC_LANE_DIS__SHIFT 0x00000010
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_DEASSERT_TX_PDNB_MASK 0x00010000L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_DEASSERT_TX_PDNB__SHIFT 0x00000010
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_DUAL_END_RECONFIG_EN_MASK 0x00080000L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_DUAL_END_RECONFIG_EN__SHIFT 0x00000013
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK 0x00600000L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT 0x00000015
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_DYNLINK_MST_EN_MASK 0x00040000L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_DYNLINK_MST_EN__SHIFT 0x00000012
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_EQ_REVERSAL_LOGIC_EN_MASK 0x00800000L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_EQ_REVERSAL_LOGIC_EN__SHIFT 0x00000017
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_L1_RECONFIG_EN_MASK 0x00020000L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_L1_RECONFIG_EN__SHIFT 0x00000011
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK 0x00000007L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK 0x00000070L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT 0x00000004
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT 0x00000000
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_ARC_MISSING_ESCAPE_MASK 0x00000080L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_ARC_MISSING_ESCAPE__SHIFT 0x00000007
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK 0x00000100L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW__SHIFT 0x00000008
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK 0x00000400L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN__SHIFT 0x0000000a
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK 0x00000200L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT__SHIFT 0x00000009
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_SHORT_RECONFIG_EN_MASK 0x00000800L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_SHORT_RECONFIG_EN__SHIFT 0x0000000b
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_UPCFG_TIMER_SEL_MASK 0x00008000L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_UPCFG_TIMER_SEL__SHIFT 0x0000000f
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_UPCFG_WAIT_FOR_RCVR_DIS_MASK 0x00004000L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_UPCFG_WAIT_FOR_RCVR_DIS__SHIFT 0x0000000e
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_CAPABLE_MASK 0x00100000L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_CAPABLE__SHIFT 0x00000014
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK 0x00002000L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS__SHIFT 0x0000000d
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK 0x00001000L
+#define PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT__SHIFT 0x0000000c
+#define PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK 0xff000000L
+#define PCIE_LC_N_FTS_CNTL__LC_N_FTS__SHIFT 0x00000018
+#define PCIE_LC_N_FTS_CNTL__LC_XMIT_FTS_BEFORE_RECOVERY_MASK 0x00000200L
+#define PCIE_LC_N_FTS_CNTL__LC_XMIT_FTS_BEFORE_RECOVERY__SHIFT 0x00000009
+#define PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_LIMIT_MASK 0x00ff0000L
+#define PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_LIMIT__SHIFT 0x00000010
+#define PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK 0x000000ffL
+#define PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK 0x00000100L
+#define PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN__SHIFT 0x00000008
+#define PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT 0x00000000
+#define PCIE_LC_SPEED_CNTL__LC_1_OR_MORE_TS2_SPEED_ARC_EN_MASK 0x00020000L
+#define PCIE_LC_SPEED_CNTL__LC_1_OR_MORE_TS2_SPEED_ARC_EN__SHIFT 0x00000011
+#define PCIE_LC_SPEED_CNTL__LC_AUTO_RECOVERY_DIS_MASK 0x00400000L
+#define PCIE_LC_SPEED_CNTL__LC_AUTO_RECOVERY_DIS__SHIFT 0x00000016
+#define PCIE_LC_SPEED_CNTL__LC_CHECK_DATA_RATE_MASK 0x04000000L
+#define PCIE_LC_SPEED_CNTL__LC_CHECK_DATA_RATE__SHIFT 0x0000001a
+#define PCIE_LC_SPEED_CNTL__LC_CLR_FAILED_SPD_CHANGE_CNT_MASK 0x00010000L
+#define PCIE_LC_SPEED_CNTL__LC_CLR_FAILED_SPD_CHANGE_CNT__SHIFT 0x00000010
+#define PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK 0x00006000L
+#define PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT 0x0000000d
+#define PCIE_LC_SPEED_CNTL__LC_DATA_RATE_ADVERTISED_MASK 0x03000000L
+#define PCIE_LC_SPEED_CNTL__LC_DATA_RATE_ADVERTISED__SHIFT 0x00000018
+#define PCIE_LC_SPEED_CNTL__LC_DELAY_COEFF_UPDATE_DIS_MASK 0x80000000L
+#define PCIE_LC_SPEED_CNTL__LC_DELAY_COEFF_UPDATE_DIS__SHIFT 0x0000001f
+#define PCIE_LC_SPEED_CNTL__LC_DONT_CHECK_EQTS_IN_RCFG_MASK 0x40000000L
+#define PCIE_LC_SPEED_CNTL__LC_DONT_CHECK_EQTS_IN_RCFG__SHIFT 0x0000001e
+#define PCIE_LC_SPEED_CNTL__LC_DONT_CLR_TARGET_SPD_CHANGE_STATUS_MASK 0x00008000L
+#define PCIE_LC_SPEED_CNTL__LC_DONT_CLR_TARGET_SPD_CHANGE_STATUS__SHIFT 0x0000000f
+#define PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK 0x00000100L
+#define PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE__SHIFT 0x00000008
+#define PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK 0x00000040L
+#define PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE__SHIFT 0x00000006
+#define PCIE_LC_SPEED_CNTL__LC_FORCE_EN_HW_SPEED_CHANGE_MASK 0x00000080L
+#define PCIE_LC_SPEED_CNTL__LC_FORCE_EN_HW_SPEED_CHANGE__SHIFT 0x00000007
+#define PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK 0x00000020L
+#define PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE__SHIFT 0x00000005
+#define PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK 0x00000001L
+#define PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT 0x00000000
+#define PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK 0x00000002L
+#define PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT 0x00000001
+#define PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK 0x00000200L
+#define PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE__SHIFT 0x00000009
+#define PCIE_LC_SPEED_CNTL__LC_INIT_SPEED_NEG_IN_L0s_EN_MASK 0x10000000L
+#define PCIE_LC_SPEED_CNTL__LC_INIT_SPEED_NEG_IN_L0s_EN__SHIFT 0x0000001c
+#define PCIE_LC_SPEED_CNTL__LC_INIT_SPEED_NEG_IN_L1_EN_MASK 0x20000000L
+#define PCIE_LC_SPEED_CNTL__LC_INIT_SPEED_NEG_IN_L1_EN__SHIFT 0x0000001d
+#define PCIE_LC_SPEED_CNTL__LC_MULT_UPSTREAM_AUTO_SPD_CHNG_EN_MASK 0x08000000L
+#define PCIE_LC_SPEED_CNTL__LC_MULT_UPSTREAM_AUTO_SPD_CHNG_EN__SHIFT 0x0000001b
+#define PCIE_LC_SPEED_CNTL__LC_OTHER_SIDE_EVER_SENT_GEN2_MASK 0x00040000L
+#define PCIE_LC_SPEED_CNTL__LC_OTHER_SIDE_EVER_SENT_GEN2__SHIFT 0x00000012
+#define PCIE_LC_SPEED_CNTL__LC_OTHER_SIDE_EVER_SENT_GEN3_MASK 0x00100000L
+#define PCIE_LC_SPEED_CNTL__LC_OTHER_SIDE_EVER_SENT_GEN3__SHIFT 0x00000014
+#define PCIE_LC_SPEED_CNTL__LC_OTHER_SIDE_SUPPORTS_GEN2_MASK 0x00080000L
+#define PCIE_LC_SPEED_CNTL__LC_OTHER_SIDE_SUPPORTS_GEN2__SHIFT 0x00000013
+#define PCIE_LC_SPEED_CNTL__LC_OTHER_SIDE_SUPPORTS_GEN3_MASK 0x00200000L
+#define PCIE_LC_SPEED_CNTL__LC_OTHER_SIDE_SUPPORTS_GEN3__SHIFT 0x00000015
+#define PCIE_LC_SPEED_CNTL__LC_SPEED_CHANGE_ATTEMPT_FAILED_MASK 0x00001000L
+#define PCIE_LC_SPEED_CNTL__LC_SPEED_CHANGE_ATTEMPT_FAILED__SHIFT 0x0000000c
+#define PCIE_LC_SPEED_CNTL__LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK 0x00000c00L
+#define PCIE_LC_SPEED_CNTL__LC_SPEED_CHANGE_ATTEMPTS_ALLOWED__SHIFT 0x0000000a
+#define PCIE_LC_SPEED_CNTL__LC_SPEED_CHANGE_STATUS_MASK 0x00800000L
+#define PCIE_LC_SPEED_CNTL__LC_SPEED_CHANGE_STATUS__SHIFT 0x00000017
+#define PCIE_LC_SPEED_CNTL__LC_TARGET_LINK_SPEED_OVERRIDE_EN_MASK 0x00000004L
+#define PCIE_LC_SPEED_CNTL__LC_TARGET_LINK_SPEED_OVERRIDE_EN__SHIFT 0x00000002
+#define PCIE_LC_SPEED_CNTL__LC_TARGET_LINK_SPEED_OVERRIDE_MASK 0x00000018L
+#define PCIE_LC_SPEED_CNTL__LC_TARGET_LINK_SPEED_OVERRIDE__SHIFT 0x00000003
+#define PCIE_LC_STATE0__LC_CURRENT_STATE_MASK 0x0000003fL
+#define PCIE_LC_STATE0__LC_CURRENT_STATE__SHIFT 0x00000000
+#define PCIE_LC_STATE0__LC_PREV_STATE1_MASK 0x00003f00L
+#define PCIE_LC_STATE0__LC_PREV_STATE1__SHIFT 0x00000008
+#define PCIE_LC_STATE0__LC_PREV_STATE2_MASK 0x003f0000L
+#define PCIE_LC_STATE0__LC_PREV_STATE2__SHIFT 0x00000010
+#define PCIE_LC_STATE0__LC_PREV_STATE3_MASK 0x3f000000L
+#define PCIE_LC_STATE0__LC_PREV_STATE3__SHIFT 0x00000018
+#define PCIE_LC_STATE10__LC_PREV_STATE40_MASK 0x0000003fL
+#define PCIE_LC_STATE10__LC_PREV_STATE40__SHIFT 0x00000000
+#define PCIE_LC_STATE10__LC_PREV_STATE41_MASK 0x00003f00L
+#define PCIE_LC_STATE10__LC_PREV_STATE41__SHIFT 0x00000008
+#define PCIE_LC_STATE10__LC_PREV_STATE42_MASK 0x003f0000L
+#define PCIE_LC_STATE10__LC_PREV_STATE42__SHIFT 0x00000010
+#define PCIE_LC_STATE10__LC_PREV_STATE43_MASK 0x3f000000L
+#define PCIE_LC_STATE10__LC_PREV_STATE43__SHIFT 0x00000018
+#define PCIE_LC_STATE11__LC_PREV_STATE44_MASK 0x0000003fL
+#define PCIE_LC_STATE11__LC_PREV_STATE44__SHIFT 0x00000000
+#define PCIE_LC_STATE11__LC_PREV_STATE45_MASK 0x00003f00L
+#define PCIE_LC_STATE11__LC_PREV_STATE45__SHIFT 0x00000008
+#define PCIE_LC_STATE11__LC_PREV_STATE46_MASK 0x003f0000L
+#define PCIE_LC_STATE11__LC_PREV_STATE46__SHIFT 0x00000010
+#define PCIE_LC_STATE11__LC_PREV_STATE47_MASK 0x3f000000L
+#define PCIE_LC_STATE11__LC_PREV_STATE47__SHIFT 0x00000018
+#define PCIE_LC_STATE1__LC_PREV_STATE4_MASK 0x0000003fL
+#define PCIE_LC_STATE1__LC_PREV_STATE4__SHIFT 0x00000000
+#define PCIE_LC_STATE1__LC_PREV_STATE5_MASK 0x00003f00L
+#define PCIE_LC_STATE1__LC_PREV_STATE5__SHIFT 0x00000008
+#define PCIE_LC_STATE1__LC_PREV_STATE6_MASK 0x003f0000L
+#define PCIE_LC_STATE1__LC_PREV_STATE6__SHIFT 0x00000010
+#define PCIE_LC_STATE1__LC_PREV_STATE7_MASK 0x3f000000L
+#define PCIE_LC_STATE1__LC_PREV_STATE7__SHIFT 0x00000018
+#define PCIE_LC_STATE2__LC_PREV_STATE10_MASK 0x003f0000L
+#define PCIE_LC_STATE2__LC_PREV_STATE10__SHIFT 0x00000010
+#define PCIE_LC_STATE2__LC_PREV_STATE11_MASK 0x3f000000L
+#define PCIE_LC_STATE2__LC_PREV_STATE11__SHIFT 0x00000018
+#define PCIE_LC_STATE2__LC_PREV_STATE8_MASK 0x0000003fL
+#define PCIE_LC_STATE2__LC_PREV_STATE8__SHIFT 0x00000000
+#define PCIE_LC_STATE2__LC_PREV_STATE9_MASK 0x00003f00L
+#define PCIE_LC_STATE2__LC_PREV_STATE9__SHIFT 0x00000008
+#define PCIE_LC_STATE3__LC_PREV_STATE12_MASK 0x0000003fL
+#define PCIE_LC_STATE3__LC_PREV_STATE12__SHIFT 0x00000000
+#define PCIE_LC_STATE3__LC_PREV_STATE13_MASK 0x00003f00L
+#define PCIE_LC_STATE3__LC_PREV_STATE13__SHIFT 0x00000008
+#define PCIE_LC_STATE3__LC_PREV_STATE14_MASK 0x003f0000L
+#define PCIE_LC_STATE3__LC_PREV_STATE14__SHIFT 0x00000010
+#define PCIE_LC_STATE3__LC_PREV_STATE15_MASK 0x3f000000L
+#define PCIE_LC_STATE3__LC_PREV_STATE15__SHIFT 0x00000018
+#define PCIE_LC_STATE4__LC_PREV_STATE16_MASK 0x0000003fL
+#define PCIE_LC_STATE4__LC_PREV_STATE16__SHIFT 0x00000000
+#define PCIE_LC_STATE4__LC_PREV_STATE17_MASK 0x00003f00L
+#define PCIE_LC_STATE4__LC_PREV_STATE17__SHIFT 0x00000008
+#define PCIE_LC_STATE4__LC_PREV_STATE18_MASK 0x003f0000L
+#define PCIE_LC_STATE4__LC_PREV_STATE18__SHIFT 0x00000010
+#define PCIE_LC_STATE4__LC_PREV_STATE19_MASK 0x3f000000L
+#define PCIE_LC_STATE4__LC_PREV_STATE19__SHIFT 0x00000018
+#define PCIE_LC_STATE5__LC_PREV_STATE20_MASK 0x0000003fL
+#define PCIE_LC_STATE5__LC_PREV_STATE20__SHIFT 0x00000000
+#define PCIE_LC_STATE5__LC_PREV_STATE21_MASK 0x00003f00L
+#define PCIE_LC_STATE5__LC_PREV_STATE21__SHIFT 0x00000008
+#define PCIE_LC_STATE5__LC_PREV_STATE22_MASK 0x003f0000L
+#define PCIE_LC_STATE5__LC_PREV_STATE22__SHIFT 0x00000010
+#define PCIE_LC_STATE5__LC_PREV_STATE23_MASK 0x3f000000L
+#define PCIE_LC_STATE5__LC_PREV_STATE23__SHIFT 0x00000018
+#define PCIE_LC_STATE6__LC_PREV_STATE24_MASK 0x0000003fL
+#define PCIE_LC_STATE6__LC_PREV_STATE24__SHIFT 0x00000000
+#define PCIE_LC_STATE6__LC_PREV_STATE25_MASK 0x00003f00L
+#define PCIE_LC_STATE6__LC_PREV_STATE25__SHIFT 0x00000008
+#define PCIE_LC_STATE6__LC_PREV_STATE26_MASK 0x003f0000L
+#define PCIE_LC_STATE6__LC_PREV_STATE26__SHIFT 0x00000010
+#define PCIE_LC_STATE6__LC_PREV_STATE27_MASK 0x3f000000L
+#define PCIE_LC_STATE6__LC_PREV_STATE27__SHIFT 0x00000018
+#define PCIE_LC_STATE7__LC_PREV_STATE28_MASK 0x0000003fL
+#define PCIE_LC_STATE7__LC_PREV_STATE28__SHIFT 0x00000000
+#define PCIE_LC_STATE7__LC_PREV_STATE29_MASK 0x00003f00L
+#define PCIE_LC_STATE7__LC_PREV_STATE29__SHIFT 0x00000008
+#define PCIE_LC_STATE7__LC_PREV_STATE30_MASK 0x003f0000L
+#define PCIE_LC_STATE7__LC_PREV_STATE30__SHIFT 0x00000010
+#define PCIE_LC_STATE7__LC_PREV_STATE31_MASK 0x3f000000L
+#define PCIE_LC_STATE7__LC_PREV_STATE31__SHIFT 0x00000018
+#define PCIE_LC_STATE8__LC_PREV_STATE32_MASK 0x0000003fL
+#define PCIE_LC_STATE8__LC_PREV_STATE32__SHIFT 0x00000000
+#define PCIE_LC_STATE8__LC_PREV_STATE33_MASK 0x00003f00L
+#define PCIE_LC_STATE8__LC_PREV_STATE33__SHIFT 0x00000008
+#define PCIE_LC_STATE8__LC_PREV_STATE34_MASK 0x003f0000L
+#define PCIE_LC_STATE8__LC_PREV_STATE34__SHIFT 0x00000010
+#define PCIE_LC_STATE8__LC_PREV_STATE35_MASK 0x3f000000L
+#define PCIE_LC_STATE8__LC_PREV_STATE35__SHIFT 0x00000018
+#define PCIE_LC_STATE9__LC_PREV_STATE36_MASK 0x0000003fL
+#define PCIE_LC_STATE9__LC_PREV_STATE36__SHIFT 0x00000000
+#define PCIE_LC_STATE9__LC_PREV_STATE37_MASK 0x00003f00L
+#define PCIE_LC_STATE9__LC_PREV_STATE37__SHIFT 0x00000008
+#define PCIE_LC_STATE9__LC_PREV_STATE38_MASK 0x003f0000L
+#define PCIE_LC_STATE9__LC_PREV_STATE38__SHIFT 0x00000010
+#define PCIE_LC_STATE9__LC_PREV_STATE39_MASK 0x3f000000L
+#define PCIE_LC_STATE9__LC_PREV_STATE39__SHIFT 0x00000018
+#define PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK 0x000000e0L
+#define PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT 0x00000005
+#define PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK 0x0000001cL
+#define PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT 0x00000002
+#define PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK 0x00000001L
+#define PCIE_LC_STATUS1__LC_REVERSE_RCVR__SHIFT 0x00000000
+#define PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK 0x00000002L
+#define PCIE_LC_STATUS1__LC_REVERSE_XMIT__SHIFT 0x00000001
+#define PCIE_LC_STATUS2__LC_TOTAL_INACTIVE_LANES_MASK 0x0000ffffL
+#define PCIE_LC_STATUS2__LC_TOTAL_INACTIVE_LANES__SHIFT 0x00000000
+#define PCIE_LC_STATUS2__LC_TURN_ON_LANE_MASK 0xffff0000L
+#define PCIE_LC_STATUS2__LC_TURN_ON_LANE__SHIFT 0x00000010
+#define PCIE_LC_TRAINING_CNTL__LC_ALLOW_TX_L1_CONTROL_MASK 0x10000000L
+#define PCIE_LC_TRAINING_CNTL__LC_ALLOW_TX_L1_CONTROL__SHIFT 0x0000001c
+#define PCIE_LC_TRAINING_CNTL__LC_ASPM_L1_NAK_TIMER_SEL_MASK 0x00c00000L
+#define PCIE_LC_TRAINING_CNTL__LC_ASPM_L1_NAK_TIMER_SEL__SHIFT 0x00000016
+#define PCIE_LC_TRAINING_CNTL__LC_AUTONOMOUS_CHANGE_OFF_MASK 0x00020000L
+#define PCIE_LC_TRAINING_CNTL__LC_AUTONOMOUS_CHANGE_OFF__SHIFT 0x00000011
+#define PCIE_LC_TRAINING_CNTL__LC_COMPLIANCE_RECEIVE_MASK 0x00000010L
+#define PCIE_LC_TRAINING_CNTL__LC_COMPLIANCE_RECEIVE__SHIFT 0x00000004
+#define PCIE_LC_TRAINING_CNTL__LC_DISABLE_TRAINING_BIT_ARCH_MASK 0x00002000L
+#define PCIE_LC_TRAINING_CNTL__LC_DISABLE_TRAINING_BIT_ARCH__SHIFT 0x0000000d
+#define PCIE_LC_TRAINING_CNTL__LC_DONT_DEASSERT_RX_EN_IN_R_SPEED_MASK 0x01000000L
+#define PCIE_LC_TRAINING_CNTL__LC_DONT_DEASSERT_RX_EN_IN_R_SPEED__SHIFT 0x00000018
+#define PCIE_LC_TRAINING_CNTL__LC_DONT_DEASSERT_RX_EN_IN_TEST_MASK 0x02000000L
+#define PCIE_LC_TRAINING_CNTL__LC_DONT_DEASSERT_RX_EN_IN_TEST__SHIFT 0x00000019
+#define PCIE_LC_TRAINING_CNTL__LC_DONT_GO_TO_L0S_IF_L1_ARMED_MASK 0x00000800L
+#define PCIE_LC_TRAINING_CNTL__LC_DONT_GO_TO_L0S_IF_L1_ARMED__SHIFT 0x0000000b
+#define PCIE_LC_TRAINING_CNTL__LC_EXTEND_EQ_REQ_TIME_MASK 0xc0000000L
+#define PCIE_LC_TRAINING_CNTL__LC_EXTEND_EQ_REQ_TIME__SHIFT 0x0000001e
+#define PCIE_LC_TRAINING_CNTL__LC_EXTEND_WAIT_FOR_SKP_MASK 0x00010000L
+#define PCIE_LC_TRAINING_CNTL__LC_EXTEND_WAIT_FOR_SKP__SHIFT 0x00000010
+#define PCIE_LC_TRAINING_CNTL__LC_HW_LINK_DIS_EN_MASK 0x00080000L
+#define PCIE_LC_TRAINING_CNTL__LC_HW_LINK_DIS_EN__SHIFT 0x00000013
+#define PCIE_LC_TRAINING_CNTL__LC_INIT_SPD_CHG_WITH_CSR_EN_MASK 0x00001000L
+#define PCIE_LC_TRAINING_CNTL__LC_INIT_SPD_CHG_WITH_CSR_EN__SHIFT 0x0000000c
+#define PCIE_LC_TRAINING_CNTL__LC_L0S_L1_TRAINING_CNTL_EN_MASK 0x00000040L
+#define PCIE_LC_TRAINING_CNTL__LC_L0S_L1_TRAINING_CNTL_EN__SHIFT 0x00000006
+#define PCIE_LC_TRAINING_CNTL__LC_L1_LONG_WAKE_FIX_EN_MASK 0x00000080L
+#define PCIE_LC_TRAINING_CNTL__LC_L1_LONG_WAKE_FIX_EN__SHIFT 0x00000007
+#define PCIE_LC_TRAINING_CNTL__LC_LINK_DIS_BY_HW_MASK 0x00100000L
+#define PCIE_LC_TRAINING_CNTL__LC_LINK_DIS_BY_HW__SHIFT 0x00000014
+#define PCIE_LC_TRAINING_CNTL__LC_LOOK_FOR_MORE_NON_MATCHING_TS1_MASK 0x00000020L
+#define PCIE_LC_TRAINING_CNTL__LC_LOOK_FOR_MORE_NON_MATCHING_TS1__SHIFT 0x00000005
+#define PCIE_LC_TRAINING_CNTL__LC_POWER_STATE_MASK 0x00000700L
+#define PCIE_LC_TRAINING_CNTL__LC_POWER_STATE__SHIFT 0x00000008
+#define PCIE_LC_TRAINING_CNTL__LC_RESET_ASPM_L1_NAK_TIMER_MASK 0x04000000L
+#define PCIE_LC_TRAINING_CNTL__LC_RESET_ASPM_L1_NAK_TIMER__SHIFT 0x0000001a
+#define PCIE_LC_TRAINING_CNTL__LC_SHORT_RCFG_TIMEOUT_MASK 0x08000000L
+#define PCIE_LC_TRAINING_CNTL__LC_SHORT_RCFG_TIMEOUT__SHIFT 0x0000001b
+#define PCIE_LC_TRAINING_CNTL__LC_STATIC_TX_PIPE_COUNT_EN_MASK 0x00200000L
+#define PCIE_LC_TRAINING_CNTL__LC_STATIC_TX_PIPE_COUNT_EN__SHIFT 0x00000015
+#define PCIE_LC_TRAINING_CNTL__LC_TRAINING_CNTL_MASK 0x0000000fL
+#define PCIE_LC_TRAINING_CNTL__LC_TRAINING_CNTL__SHIFT 0x00000000
+#define PCIE_LC_TRAINING_CNTL__LC_UPCONFIGURE_CAP_OFF_MASK 0x00040000L
+#define PCIE_LC_TRAINING_CNTL__LC_UPCONFIGURE_CAP_OFF__SHIFT 0x00000012
+#define PCIE_LC_TRAINING_CNTL__LC_WAIT_FOR_FOM_VALID_AFTER_TRACK_MASK 0x20000000L
+#define PCIE_LC_TRAINING_CNTL__LC_WAIT_FOR_FOM_VALID_AFTER_TRACK__SHIFT 0x0000001d
+#define PCIE_P_BUF_STATUS__P_OVERFLOW_ERR_MASK 0x0000ffffL
+#define PCIE_P_BUF_STATUS__P_OVERFLOW_ERR__SHIFT 0x00000000
+#define PCIE_P_BUF_STATUS__P_UNDERFLOW_ERR_MASK 0xffff0000L
+#define PCIE_P_BUF_STATUS__P_UNDERFLOW_ERR__SHIFT 0x00000010
+#define PCIE_P_CNTL__P_ALWAYS_USE_FAST_TXCLK_MASK 0x00002000L
+#define PCIE_P_CNTL__P_ALWAYS_USE_FAST_TXCLK__SHIFT 0x0000000d
+#define PCIE_P_CNTL__P_BLK_LOCK_MODE_MASK 0x00001000L
+#define PCIE_P_CNTL__P_BLK_LOCK_MODE__SHIFT 0x0000000c
+#define PCIE_P_CNTL__P_ELASTDESKEW_HW_DEBUG_MASK 0x00000008L
+#define PCIE_P_CNTL__P_ELASTDESKEW_HW_DEBUG__SHIFT 0x00000003
+#define PCIE_P_CNTL__P_ELEC_IDLE_MODE_MASK 0x0000c000L
+#define PCIE_P_CNTL__P_ELEC_IDLE_MODE__SHIFT 0x0000000e
+#define PCIE_P_CNTL__P_IGNORE_CRC_ERR_MASK 0x00000010L
+#define PCIE_P_CNTL__P_IGNORE_CRC_ERR__SHIFT 0x00000004
+#define PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK 0x00000040L
+#define PCIE_P_CNTL__P_IGNORE_EDB_ERR__SHIFT 0x00000006
+#define PCIE_P_CNTL__P_IGNORE_IDL_ERR_MASK 0x00000080L
+#define PCIE_P_CNTL__P_IGNORE_IDL_ERR__SHIFT 0x00000007
+#define PCIE_P_CNTL__P_IGNORE_LEN_ERR_MASK 0x00000020L
+#define PCIE_P_CNTL__P_IGNORE_LEN_ERR__SHIFT 0x00000005
+#define PCIE_P_CNTL__P_IGNORE_TOK_ERR_MASK 0x00000100L
+#define PCIE_P_CNTL__P_IGNORE_TOK_ERR__SHIFT 0x00000008
+#define PCIE_P_CNTL__P_PWRDN_EN_MASK 0x00000001L
+#define PCIE_P_CNTL__P_PWRDN_EN__SHIFT 0x00000000
+#define PCIE_P_CNTL__P_SYMALIGN_HW_DEBUG_MASK 0x00000004L
+#define PCIE_P_CNTL__P_SYMALIGN_HW_DEBUG__SHIFT 0x00000002
+#define PCIE_P_CNTL__P_SYMALIGN_MODE_MASK 0x00000002L
+#define PCIE_P_CNTL__P_SYMALIGN_MODE__SHIFT 0x00000001
+#define PCIE_P_DECODER_STATUS__P_DECODE_ERR_MASK 0x0000ffffL
+#define PCIE_P_DECODER_STATUS__P_DECODE_ERR__SHIFT 0x00000000
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_MST_C_CLK_MASK 0x00000f00L
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_MST_C_CLK__SHIFT 0x00000008
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_MST_R_CLK_MASK 0x000000f0L
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_MST_R_CLK__SHIFT 0x00000004
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_SLV_NS_C_CLK_MASK 0x00f00000L
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_SLV_NS_C_CLK__SHIFT 0x00000014
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_SLV_R_CLK_MASK 0x0000f000L
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_SLV_R_CLK__SHIFT 0x0000000c
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_SLV_S_C_CLK_MASK 0x000f0000L
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_SLV_S_C_CLK__SHIFT 0x00000010
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_TXCLK2_MASK 0x0f000000L
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_TXCLK2__SHIFT 0x00000018
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_TXCLK_MASK 0x0000000fL
+#define PCIE_PERF_CNTL_EVENT0_PORT_SEL__PERF0_PORT_SEL_TXCLK__SHIFT 0x00000000
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_MST_C_CLK_MASK 0x00000f00L
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_MST_C_CLK__SHIFT 0x00000008
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_MST_R_CLK_MASK 0x000000f0L
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_MST_R_CLK__SHIFT 0x00000004
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_SLV_NS_C_CLK_MASK 0x00f00000L
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_SLV_NS_C_CLK__SHIFT 0x00000014
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_SLV_R_CLK_MASK 0x0000f000L
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_SLV_R_CLK__SHIFT 0x0000000c
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_SLV_S_C_CLK_MASK 0x000f0000L
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_SLV_S_C_CLK__SHIFT 0x00000010
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_TXCLK2_MASK 0x0f000000L
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_TXCLK2__SHIFT 0x00000018
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_TXCLK_MASK 0x0000000fL
+#define PCIE_PERF_CNTL_EVENT1_PORT_SEL__PERF1_PORT_SEL_TXCLK__SHIFT 0x00000000
+#define PCIE_PERF_CNTL_MST_C_CLK__COUNTER0_UPPER_MASK 0x00ff0000L
+#define PCIE_PERF_CNTL_MST_C_CLK__COUNTER0_UPPER__SHIFT 0x00000010
+#define PCIE_PERF_CNTL_MST_C_CLK__COUNTER1_UPPER_MASK 0xff000000L
+#define PCIE_PERF_CNTL_MST_C_CLK__COUNTER1_UPPER__SHIFT 0x00000018
+#define PCIE_PERF_CNTL_MST_C_CLK__EVENT0_SEL_MASK 0x000000ffL
+#define PCIE_PERF_CNTL_MST_C_CLK__EVENT0_SEL__SHIFT 0x00000000
+#define PCIE_PERF_CNTL_MST_C_CLK__EVENT1_SEL_MASK 0x0000ff00L
+#define PCIE_PERF_CNTL_MST_C_CLK__EVENT1_SEL__SHIFT 0x00000008
+#define PCIE_PERF_CNTL_MST_R_CLK__COUNTER0_UPPER_MASK 0x00ff0000L
+#define PCIE_PERF_CNTL_MST_R_CLK__COUNTER0_UPPER__SHIFT 0x00000010
+#define PCIE_PERF_CNTL_MST_R_CLK__COUNTER1_UPPER_MASK 0xff000000L
+#define PCIE_PERF_CNTL_MST_R_CLK__COUNTER1_UPPER__SHIFT 0x00000018
+#define PCIE_PERF_CNTL_MST_R_CLK__EVENT0_SEL_MASK 0x000000ffL
+#define PCIE_PERF_CNTL_MST_R_CLK__EVENT0_SEL__SHIFT 0x00000000
+#define PCIE_PERF_CNTL_MST_R_CLK__EVENT1_SEL_MASK 0x0000ff00L
+#define PCIE_PERF_CNTL_MST_R_CLK__EVENT1_SEL__SHIFT 0x00000008
+#define PCIE_PERF_CNTL_SLV_NS_C_CLK__COUNTER0_UPPER_MASK 0x00ff0000L
+#define PCIE_PERF_CNTL_SLV_NS_C_CLK__COUNTER0_UPPER__SHIFT 0x00000010
+#define PCIE_PERF_CNTL_SLV_NS_C_CLK__COUNTER1_UPPER_MASK 0xff000000L
+#define PCIE_PERF_CNTL_SLV_NS_C_CLK__COUNTER1_UPPER__SHIFT 0x00000018
+#define PCIE_PERF_CNTL_SLV_NS_C_CLK__EVENT0_SEL_MASK 0x000000ffL
+#define PCIE_PERF_CNTL_SLV_NS_C_CLK__EVENT0_SEL__SHIFT 0x00000000
+#define PCIE_PERF_CNTL_SLV_NS_C_CLK__EVENT1_SEL_MASK 0x0000ff00L
+#define PCIE_PERF_CNTL_SLV_NS_C_CLK__EVENT1_SEL__SHIFT 0x00000008
+#define PCIE_PERF_CNTL_SLV_R_CLK__COUNTER0_UPPER_MASK 0x00ff0000L
+#define PCIE_PERF_CNTL_SLV_R_CLK__COUNTER0_UPPER__SHIFT 0x00000010
+#define PCIE_PERF_CNTL_SLV_R_CLK__COUNTER1_UPPER_MASK 0xff000000L
+#define PCIE_PERF_CNTL_SLV_R_CLK__COUNTER1_UPPER__SHIFT 0x00000018
+#define PCIE_PERF_CNTL_SLV_R_CLK__EVENT0_SEL_MASK 0x000000ffL
+#define PCIE_PERF_CNTL_SLV_R_CLK__EVENT0_SEL__SHIFT 0x00000000
+#define PCIE_PERF_CNTL_SLV_R_CLK__EVENT1_SEL_MASK 0x0000ff00L
+#define PCIE_PERF_CNTL_SLV_R_CLK__EVENT1_SEL__SHIFT 0x00000008
+#define PCIE_PERF_CNTL_SLV_S_C_CLK__COUNTER0_UPPER_MASK 0x00ff0000L
+#define PCIE_PERF_CNTL_SLV_S_C_CLK__COUNTER0_UPPER__SHIFT 0x00000010
+#define PCIE_PERF_CNTL_SLV_S_C_CLK__COUNTER1_UPPER_MASK 0xff000000L
+#define PCIE_PERF_CNTL_SLV_S_C_CLK__COUNTER1_UPPER__SHIFT 0x00000018
+#define PCIE_PERF_CNTL_SLV_S_C_CLK__EVENT0_SEL_MASK 0x000000ffL
+#define PCIE_PERF_CNTL_SLV_S_C_CLK__EVENT0_SEL__SHIFT 0x00000000
+#define PCIE_PERF_CNTL_SLV_S_C_CLK__EVENT1_SEL_MASK 0x0000ff00L
+#define PCIE_PERF_CNTL_SLV_S_C_CLK__EVENT1_SEL__SHIFT 0x00000008
+#define PCIE_PERF_CNTL_TXCLK2__COUNTER0_UPPER_MASK 0x00ff0000L
+#define PCIE_PERF_CNTL_TXCLK2__COUNTER0_UPPER__SHIFT 0x00000010
+#define PCIE_PERF_CNTL_TXCLK2__COUNTER1_UPPER_MASK 0xff000000L
+#define PCIE_PERF_CNTL_TXCLK2__COUNTER1_UPPER__SHIFT 0x00000018
+#define PCIE_PERF_CNTL_TXCLK2__EVENT0_SEL_MASK 0x000000ffL
+#define PCIE_PERF_CNTL_TXCLK2__EVENT0_SEL__SHIFT 0x00000000
+#define PCIE_PERF_CNTL_TXCLK2__EVENT1_SEL_MASK 0x0000ff00L
+#define PCIE_PERF_CNTL_TXCLK2__EVENT1_SEL__SHIFT 0x00000008
+#define PCIE_PERF_CNTL_TXCLK__COUNTER0_UPPER_MASK 0x00ff0000L
+#define PCIE_PERF_CNTL_TXCLK__COUNTER0_UPPER__SHIFT 0x00000010
+#define PCIE_PERF_CNTL_TXCLK__COUNTER1_UPPER_MASK 0xff000000L
+#define PCIE_PERF_CNTL_TXCLK__COUNTER1_UPPER__SHIFT 0x00000018
+#define PCIE_PERF_CNTL_TXCLK__EVENT0_SEL_MASK 0x000000ffL
+#define PCIE_PERF_CNTL_TXCLK__EVENT0_SEL__SHIFT 0x00000000
+#define PCIE_PERF_CNTL_TXCLK__EVENT1_SEL_MASK 0x0000ff00L
+#define PCIE_PERF_CNTL_TXCLK__EVENT1_SEL__SHIFT 0x00000008
+#define PCIE_PERF_COUNT0_MST_C_CLK__COUNTER0_MASK 0xffffffffL
+#define PCIE_PERF_COUNT0_MST_C_CLK__COUNTER0__SHIFT 0x00000000
+#define PCIE_PERF_COUNT0_MST_R_CLK__COUNTER0_MASK 0xffffffffL
+#define PCIE_PERF_COUNT0_MST_R_CLK__COUNTER0__SHIFT 0x00000000
+#define PCIE_PERF_COUNT0_SLV_NS_C_CLK__COUNTER0_MASK 0xffffffffL
+#define PCIE_PERF_COUNT0_SLV_NS_C_CLK__COUNTER0__SHIFT 0x00000000
+#define PCIE_PERF_COUNT0_SLV_R_CLK__COUNTER0_MASK 0xffffffffL
+#define PCIE_PERF_COUNT0_SLV_R_CLK__COUNTER0__SHIFT 0x00000000
+#define PCIE_PERF_COUNT0_SLV_S_C_CLK__COUNTER0_MASK 0xffffffffL
+#define PCIE_PERF_COUNT0_SLV_S_C_CLK__COUNTER0__SHIFT 0x00000000
+#define PCIE_PERF_COUNT0_TXCLK2__COUNTER0_MASK 0xffffffffL
+#define PCIE_PERF_COUNT0_TXCLK2__COUNTER0__SHIFT 0x00000000
+#define PCIE_PERF_COUNT0_TXCLK__COUNTER0_MASK 0xffffffffL
+#define PCIE_PERF_COUNT0_TXCLK__COUNTER0__SHIFT 0x00000000
+#define PCIE_PERF_COUNT1_MST_C_CLK__COUNTER1_MASK 0xffffffffL
+#define PCIE_PERF_COUNT1_MST_C_CLK__COUNTER1__SHIFT 0x00000000
+#define PCIE_PERF_COUNT1_MST_R_CLK__COUNTER1_MASK 0xffffffffL
+#define PCIE_PERF_COUNT1_MST_R_CLK__COUNTER1__SHIFT 0x00000000
+#define PCIE_PERF_COUNT1_SLV_NS_C_CLK__COUNTER1_MASK 0xffffffffL
+#define PCIE_PERF_COUNT1_SLV_NS_C_CLK__COUNTER1__SHIFT 0x00000000
+#define PCIE_PERF_COUNT1_SLV_R_CLK__COUNTER1_MASK 0xffffffffL
+#define PCIE_PERF_COUNT1_SLV_R_CLK__COUNTER1__SHIFT 0x00000000
+#define PCIE_PERF_COUNT1_SLV_S_C_CLK__COUNTER1_MASK 0xffffffffL
+#define PCIE_PERF_COUNT1_SLV_S_C_CLK__COUNTER1__SHIFT 0x00000000
+#define PCIE_PERF_COUNT1_TXCLK2__COUNTER1_MASK 0xffffffffL
+#define PCIE_PERF_COUNT1_TXCLK2__COUNTER1__SHIFT 0x00000000
+#define PCIE_PERF_COUNT1_TXCLK__COUNTER1_MASK 0xffffffffL
+#define PCIE_PERF_COUNT1_TXCLK__COUNTER1__SHIFT 0x00000000
+#define PCIE_PERF_COUNT_CNTL__GLOBAL_COUNT_EN_MASK 0x00000001L
+#define PCIE_PERF_COUNT_CNTL__GLOBAL_COUNT_EN__SHIFT 0x00000000
+#define PCIE_PERF_COUNT_CNTL__GLOBAL_COUNT_RESET_MASK 0x00000004L
+#define PCIE_PERF_COUNT_CNTL__GLOBAL_COUNT_RESET__SHIFT 0x00000002
+#define PCIE_PERF_COUNT_CNTL__GLOBAL_SHADOW_WR_MASK 0x00000002L
+#define PCIE_PERF_COUNT_CNTL__GLOBAL_SHADOW_WR__SHIFT 0x00000001
+#define PCIEP_HW_DEBUG__HW_00_DEBUG_MASK 0x00000001L
+#define PCIEP_HW_DEBUG__HW_00_DEBUG__SHIFT 0x00000000
+#define PCIEP_HW_DEBUG__HW_01_DEBUG_MASK 0x00000002L
+#define PCIEP_HW_DEBUG__HW_01_DEBUG__SHIFT 0x00000001
+#define PCIEP_HW_DEBUG__HW_02_DEBUG_MASK 0x00000004L
+#define PCIEP_HW_DEBUG__HW_02_DEBUG__SHIFT 0x00000002
+#define PCIEP_HW_DEBUG__HW_03_DEBUG_MASK 0x00000008L
+#define PCIEP_HW_DEBUG__HW_03_DEBUG__SHIFT 0x00000003
+#define PCIEP_HW_DEBUG__HW_04_DEBUG_MASK 0x00000010L
+#define PCIEP_HW_DEBUG__HW_04_DEBUG__SHIFT 0x00000004
+#define PCIEP_HW_DEBUG__HW_05_DEBUG_MASK 0x00000020L
+#define PCIEP_HW_DEBUG__HW_05_DEBUG__SHIFT 0x00000005
+#define PCIEP_HW_DEBUG__HW_06_DEBUG_MASK 0x00000040L
+#define PCIEP_HW_DEBUG__HW_06_DEBUG__SHIFT 0x00000006
+#define PCIEP_HW_DEBUG__HW_07_DEBUG_MASK 0x00000080L
+#define PCIEP_HW_DEBUG__HW_07_DEBUG__SHIFT 0x00000007
+#define PCIEP_HW_DEBUG__HW_08_DEBUG_MASK 0x00000100L
+#define PCIEP_HW_DEBUG__HW_08_DEBUG__SHIFT 0x00000008
+#define PCIEP_HW_DEBUG__HW_09_DEBUG_MASK 0x00000200L
+#define PCIEP_HW_DEBUG__HW_09_DEBUG__SHIFT 0x00000009
+#define PCIEP_HW_DEBUG__HW_10_DEBUG_MASK 0x00000400L
+#define PCIEP_HW_DEBUG__HW_10_DEBUG__SHIFT 0x0000000a
+#define PCIEP_HW_DEBUG__HW_11_DEBUG_MASK 0x00000800L
+#define PCIEP_HW_DEBUG__HW_11_DEBUG__SHIFT 0x0000000b
+#define PCIEP_HW_DEBUG__HW_12_DEBUG_MASK 0x00001000L
+#define PCIEP_HW_DEBUG__HW_12_DEBUG__SHIFT 0x0000000c
+#define PCIEP_HW_DEBUG__HW_13_DEBUG_MASK 0x00002000L
+#define PCIEP_HW_DEBUG__HW_13_DEBUG__SHIFT 0x0000000d
+#define PCIEP_HW_DEBUG__HW_14_DEBUG_MASK 0x00004000L
+#define PCIEP_HW_DEBUG__HW_14_DEBUG__SHIFT 0x0000000e
+#define PCIEP_HW_DEBUG__HW_15_DEBUG_MASK 0x00008000L
+#define PCIEP_HW_DEBUG__HW_15_DEBUG__SHIFT 0x0000000f
+#define PCIE_P_MISC_STATUS__P_DESKEW_ERR_MASK 0x000000ffL
+#define PCIE_P_MISC_STATUS__P_DESKEW_ERR__SHIFT 0x00000000
+#define PCIE_P_MISC_STATUS__P_SYMUNLOCK_ERR_MASK 0xffff0000L
+#define PCIE_P_MISC_STATUS__P_SYMUNLOCK_ERR__SHIFT 0x00000010
+#define PCIE_PORT_DATA__PCIE_DATA_MASK 0xffffffffL
+#define PCIE_PORT_DATA__PCIE_DATA__SHIFT 0x00000000
+#define PCIE_PORT_INDEX__PCIE_INDEX_MASK 0x000000ffL
+#define PCIE_PORT_INDEX__PCIE_INDEX__SHIFT 0x00000000
+#define PCIEP_PORT_CNTL__CI_SLV_CPL_STATIC_ALLOC_LIMIT_S_MASK 0x00007f00L
+#define PCIEP_PORT_CNTL__CI_SLV_CPL_STATIC_ALLOC_LIMIT_S__SHIFT 0x00000008
+#define PCIEP_PORT_CNTL__CI_SNOOP_OVERRIDE_MASK 0x00000002L
+#define PCIEP_PORT_CNTL__CI_SNOOP_OVERRIDE__SHIFT 0x00000001
+#define PCIEP_PORT_CNTL__HOTPLUG_MSG_EN_MASK 0x00000004L
+#define PCIEP_PORT_CNTL__HOTPLUG_MSG_EN__SHIFT 0x00000002
+#define PCIEP_PORT_CNTL__NATIVE_PME_EN_MASK 0x00000008L
+#define PCIEP_PORT_CNTL__NATIVE_PME_EN__SHIFT 0x00000003
+#define PCIEP_PORT_CNTL__PMI_BM_DIS_MASK 0x00000020L
+#define PCIEP_PORT_CNTL__PMI_BM_DIS__SHIFT 0x00000005
+#define PCIEP_PORT_CNTL__PWR_FAULT_EN_MASK 0x00000010L
+#define PCIEP_PORT_CNTL__PWR_FAULT_EN__SHIFT 0x00000004
+#define PCIEP_PORT_CNTL__SEQNUM_DEBUG_MODE_MASK 0x00000040L
+#define PCIEP_PORT_CNTL__SEQNUM_DEBUG_MODE__SHIFT 0x00000006
+#define PCIEP_PORT_CNTL__SLV_PORT_REQ_EN_MASK 0x00000001L
+#define PCIEP_PORT_CNTL__SLV_PORT_REQ_EN__SHIFT 0x00000000
+#define PCIE_P_PORT_LANE_STATUS__PHY_LINK_WIDTH_MASK 0x0000007eL
+#define PCIE_P_PORT_LANE_STATUS__PHY_LINK_WIDTH__SHIFT 0x00000001
+#define PCIE_P_PORT_LANE_STATUS__PORT_LANE_REVERSAL_MASK 0x00000001L
+#define PCIE_P_PORT_LANE_STATUS__PORT_LANE_REVERSAL__SHIFT 0x00000000
+#define PCIE_PRBS_CLR__PRBS_CHECKER_DEBUG_BUS_SELECT_MASK 0x000f0000L
+#define PCIE_PRBS_CLR__PRBS_CHECKER_DEBUG_BUS_SELECT__SHIFT 0x00000010
+#define PCIE_PRBS_CLR__PRBS_CLR_MASK 0x0000ffffL
+#define PCIE_PRBS_CLR__PRBS_CLR__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_0__PRBS_ERRCNT_0_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_0__PRBS_ERRCNT_0__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_10__PRBS_ERRCNT_10_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_10__PRBS_ERRCNT_10__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_11__PRBS_ERRCNT_11_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_11__PRBS_ERRCNT_11__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_12__PRBS_ERRCNT_12_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_12__PRBS_ERRCNT_12__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_13__PRBS_ERRCNT_13_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_13__PRBS_ERRCNT_13__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_14__PRBS_ERRCNT_14_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_14__PRBS_ERRCNT_14__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_15__PRBS_ERRCNT_15_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_15__PRBS_ERRCNT_15__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_1__PRBS_ERRCNT_1_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_1__PRBS_ERRCNT_1__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_2__PRBS_ERRCNT_2_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_2__PRBS_ERRCNT_2__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_3__PRBS_ERRCNT_3_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_3__PRBS_ERRCNT_3__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_4__PRBS_ERRCNT_4_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_4__PRBS_ERRCNT_4__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_5__PRBS_ERRCNT_5_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_5__PRBS_ERRCNT_5__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_6__PRBS_ERRCNT_6_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_6__PRBS_ERRCNT_6__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_7__PRBS_ERRCNT_7_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_7__PRBS_ERRCNT_7__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_8__PRBS_ERRCNT_8_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_8__PRBS_ERRCNT_8__SHIFT 0x00000000
+#define PCIE_PRBS_ERRCNT_9__PRBS_ERRCNT_9_MASK 0xffffffffL
+#define PCIE_PRBS_ERRCNT_9__PRBS_ERRCNT_9__SHIFT 0x00000000
+#define PCIE_PRBS_FREERUN__PRBS_FREERUN_MASK 0x0000ffffL
+#define PCIE_PRBS_FREERUN__PRBS_FREERUN__SHIFT 0x00000000
+#define PCIE_PRBS_HI_BITCNT__PRBS_HI_BITCNT_MASK 0x000000ffL
+#define PCIE_PRBS_HI_BITCNT__PRBS_HI_BITCNT__SHIFT 0x00000000
+#define PCIE_PRBS_LO_BITCNT__PRBS_LO_BITCNT_MASK 0xffffffffL
+#define PCIE_PRBS_LO_BITCNT__PRBS_LO_BITCNT__SHIFT 0x00000000
+#define PCIE_PRBS_MISC__PRBS_8BIT_SEL_MASK 0x00000010L
+#define PCIE_PRBS_MISC__PRBS_8BIT_SEL__SHIFT 0x00000004
+#define PCIE_PRBS_MISC__PRBS_CHK_ERR_MASK_MASK 0xffff0000L
+#define PCIE_PRBS_MISC__PRBS_CHK_ERR_MASK__SHIFT 0x00000010
+#define PCIE_PRBS_MISC__PRBS_COMMA_NUM_MASK 0x00000060L
+#define PCIE_PRBS_MISC__PRBS_COMMA_NUM__SHIFT 0x00000005
+#define PCIE_PRBS_MISC__PRBS_DATA_RATE_MASK 0x0000c000L
+#define PCIE_PRBS_MISC__PRBS_DATA_RATE__SHIFT 0x0000000e
+#define PCIE_PRBS_MISC__PRBS_EN_MASK 0x00000001L
+#define PCIE_PRBS_MISC__PRBS_EN__SHIFT 0x00000000
+#define PCIE_PRBS_MISC__PRBS_LOCK_CNT_MASK 0x00000f80L
+#define PCIE_PRBS_MISC__PRBS_LOCK_CNT__SHIFT 0x00000007
+#define PCIE_PRBS_MISC__PRBS_TEST_MODE_MASK 0x00000006L
+#define PCIE_PRBS_MISC__PRBS_TEST_MODE__SHIFT 0x00000001
+#define PCIE_PRBS_MISC__PRBS_USER_PATTERN_TOGGLE_MASK 0x00000008L
+#define PCIE_PRBS_MISC__PRBS_USER_PATTERN_TOGGLE__SHIFT 0x00000003
+#define PCIE_PRBS_STATUS1__PRBS_ERRSTAT_MASK 0x0000ffffL
+#define PCIE_PRBS_STATUS1__PRBS_ERRSTAT__SHIFT 0x00000000
+#define PCIE_PRBS_STATUS1__PRBS_LOCKED_MASK 0xffff0000L
+#define PCIE_PRBS_STATUS1__PRBS_LOCKED__SHIFT 0x00000010
+#define PCIE_PRBS_STATUS2__PRBS_BITCNT_DONE_MASK 0x0000ffffL
+#define PCIE_PRBS_STATUS2__PRBS_BITCNT_DONE__SHIFT 0x00000000
+#define PCIE_PRBS_USER_PATTERN__PRBS_USER_PATTERN_MASK 0x3fffffffL
+#define PCIE_PRBS_USER_PATTERN__PRBS_USER_PATTERN__SHIFT 0x00000000
+#define PCIE_P_RCV_L0S_FTS_DET__P_RCV_L0S_FTS_DET_MAX_MASK 0x0000ff00L
+#define PCIE_P_RCV_L0S_FTS_DET__P_RCV_L0S_FTS_DET_MAX__SHIFT 0x00000008
+#define PCIE_P_RCV_L0S_FTS_DET__P_RCV_L0S_FTS_DET_MIN_MASK 0x000000ffL
+#define PCIE_P_RCV_L0S_FTS_DET__P_RCV_L0S_FTS_DET_MIN__SHIFT 0x00000000
+#define PCIEP_RESERVED__PCIEP_RESERVED_MASK 0xffffffffL
+#define PCIEP_RESERVED__PCIEP_RESERVED__SHIFT 0x00000000
+#define PCIEP_SCRATCH__PCIEP_SCRATCH_MASK 0xffffffffL
+#define PCIEP_SCRATCH__PCIEP_SCRATCH__SHIFT 0x00000000
+#define PCIEP_STRAP_LC__STRAP_AUTO_RC_SPEED_NEGOTIATION_DIS_MASK 0x00008000L
+#define PCIEP_STRAP_LC__STRAP_AUTO_RC_SPEED_NEGOTIATION_DIS__SHIFT 0x0000000f
+#define PCIEP_STRAP_LC__STRAP_BYPASS_RCVR_DET_MASK 0x00000800L
+#define PCIEP_STRAP_LC__STRAP_BYPASS_RCVR_DET__SHIFT 0x0000000b
+#define PCIEP_STRAP_LC__STRAP_COMPLIANCE_DIS_MASK 0x00001000L
+#define PCIEP_STRAP_LC__STRAP_COMPLIANCE_DIS__SHIFT 0x0000000c
+#define PCIEP_STRAP_LC__STRAP_FORCE_COMPLIANCE_MASK 0x00002000L
+#define PCIEP_STRAP_LC__STRAP_FORCE_COMPLIANCE__SHIFT 0x0000000d
+#define PCIEP_STRAP_LC__STRAP_FTS_yTSx_COUNT_MASK 0x00000003L
+#define PCIEP_STRAP_LC__STRAP_FTS_yTSx_COUNT__SHIFT 0x00000000
+#define PCIEP_STRAP_LC__STRAP_LANE_NEGOTIATION_MASK 0x00070000L
+#define PCIEP_STRAP_LC__STRAP_LANE_NEGOTIATION__SHIFT 0x00000010
+#define PCIEP_STRAP_LC__STRAP_LONG_yTSx_COUNT_MASK 0x0000000cL
+#define PCIEP_STRAP_LC__STRAP_LONG_yTSx_COUNT__SHIFT 0x00000002
+#define PCIEP_STRAP_LC__STRAP_MED_yTSx_COUNT_MASK 0x00000030L
+#define PCIEP_STRAP_LC__STRAP_MED_yTSx_COUNT__SHIFT 0x00000004
+#define PCIEP_STRAP_LC__STRAP_REVERSE_LC_LANES_MASK 0x00004000L
+#define PCIEP_STRAP_LC__STRAP_REVERSE_LC_LANES__SHIFT 0x0000000e
+#define PCIEP_STRAP_LC__STRAP_SHORT_yTSx_COUNT_MASK 0x000000c0L
+#define PCIEP_STRAP_LC__STRAP_SHORT_yTSx_COUNT__SHIFT 0x00000006
+#define PCIEP_STRAP_LC__STRAP_SKIP_INTERVAL_MASK 0x00000700L
+#define PCIEP_STRAP_LC__STRAP_SKIP_INTERVAL__SHIFT 0x00000008
+#define PCIEP_STRAP_MISC__STRAP_E2E_PREFIX_EN_MASK 0x00000002L
+#define PCIEP_STRAP_MISC__STRAP_E2E_PREFIX_EN__SHIFT 0x00000001
+#define PCIEP_STRAP_MISC__STRAP_EXTENDED_FMT_SUPPORTED_MASK 0x00000004L
+#define PCIEP_STRAP_MISC__STRAP_EXTENDED_FMT_SUPPORTED__SHIFT 0x00000002
+#define PCIEP_STRAP_MISC__STRAP_OBFF_SUPPORTED_MASK 0x00000018L
+#define PCIEP_STRAP_MISC__STRAP_OBFF_SUPPORTED__SHIFT 0x00000003
+#define PCIEP_STRAP_MISC__STRAP_REVERSE_LANES_MASK 0x00000001L
+#define PCIEP_STRAP_MISC__STRAP_REVERSE_LANES__SHIFT 0x00000000
+#define PCIE_RESERVED__PCIE_RESERVED_MASK 0xffffffffL
+#define PCIE_RESERVED__PCIE_RESERVED__SHIFT 0x00000000
+#define PCIE_RX_CNTL2__RX_IGNORE_EP_ATSTRANSREQ_UR_MASK 0x00000008L
+#define PCIE_RX_CNTL2__RX_IGNORE_EP_ATSTRANSREQ_UR__SHIFT 0x00000003
+#define PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK 0x00000001L
+#define PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT 0x00000000
+#define PCIE_RX_CNTL2__RX_IGNORE_EP_INVCPL_UR_MASK 0x00000020L
+#define PCIE_RX_CNTL2__RX_IGNORE_EP_INVCPL_UR__SHIFT 0x00000005
+#define PCIE_RX_CNTL2__RX_IGNORE_EP_PAGEREQMSG_UR_MASK 0x00000010L
+#define PCIE_RX_CNTL2__RX_IGNORE_EP_PAGEREQMSG_UR__SHIFT 0x00000004
+#define PCIE_RX_CNTL2__RX_IGNORE_EP_TRANSMRD_UR_MASK 0x00000002L
+#define PCIE_RX_CNTL2__RX_IGNORE_EP_TRANSMRD_UR__SHIFT 0x00000001
+#define PCIE_RX_CNTL2__RX_IGNORE_EP_TRANSMWR_UR_MASK 0x00000004L
+#define PCIE_RX_CNTL2__RX_IGNORE_EP_TRANSMWR_UR__SHIFT 0x00000002
+#define PCIE_RX_CNTL3__RX_IGNORE_RC_INVCPLPASID_UR_MASK 0x00000010L
+#define PCIE_RX_CNTL3__RX_IGNORE_RC_INVCPLPASID_UR__SHIFT 0x00000004
+#define PCIE_RX_CNTL3__RX_IGNORE_RC_INVREQ_UR_MASK 0x00000008L
+#define PCIE_RX_CNTL3__RX_IGNORE_RC_INVREQ_UR__SHIFT 0x00000003
+#define PCIE_RX_CNTL3__RX_IGNORE_RC_PRGRESPMSG_UR_MASK 0x00000004L
+#define PCIE_RX_CNTL3__RX_IGNORE_RC_PRGRESPMSG_UR__SHIFT 0x00000002
+#define PCIE_RX_CNTL3__RX_IGNORE_RC_TRANSMRDPASID_UR_MASK 0x00000001L
+#define PCIE_RX_CNTL3__RX_IGNORE_RC_TRANSMRDPASID_UR__SHIFT 0x00000000
+#define PCIE_RX_CNTL3__RX_IGNORE_RC_TRANSMWRPASID_UR_MASK 0x00000002L
+#define PCIE_RX_CNTL3__RX_IGNORE_RC_TRANSMWRPASID_UR__SHIFT 0x00000001
+#define PCIE_RX_CNTL__RX_FC_INIT_FROM_REG_MASK 0x00008000L
+#define PCIE_RX_CNTL__RX_FC_INIT_FROM_REG__SHIFT 0x0000000f
+#define PCIE_RX_CNTL__RX_GEN_ONE_NAK_MASK 0x00004000L
+#define PCIE_RX_CNTL__RX_GEN_ONE_NAK__SHIFT 0x0000000e
+#define PCIE_RX_CNTL__RX_IGNORE_AT_ERR_MASK 0x00001000L
+#define PCIE_RX_CNTL__RX_IGNORE_AT_ERR__SHIFT 0x0000000c
+#define PCIE_RX_CNTL__RX_IGNORE_BE_ERR_MASK 0x00000002L
+#define PCIE_RX_CNTL__RX_IGNORE_BE_ERR__SHIFT 0x00000001
+#define PCIE_RX_CNTL__RX_IGNORE_CFG_ERR_MASK 0x00000010L
+#define PCIE_RX_CNTL__RX_IGNORE_CFG_ERR__SHIFT 0x00000004
+#define PCIE_RX_CNTL__RX_IGNORE_CFG_UR_MASK 0x00000400L
+#define PCIE_RX_CNTL__RX_IGNORE_CFG_UR__SHIFT 0x0000000a
+#define PCIE_RX_CNTL__RX_IGNORE_CPL_ERR_MASK 0x00000020L
+#define PCIE_RX_CNTL__RX_IGNORE_CPL_ERR__SHIFT 0x00000005
+#define PCIE_RX_CNTL__RX_IGNORE_CPLPREFIX_ERR_MASK 0x00800000L
+#define PCIE_RX_CNTL__RX_IGNORE_CPLPREFIX_ERR__SHIFT 0x00000017
+#define PCIE_RX_CNTL__RX_IGNORE_CRC_ERR_MASK 0x00000008L
+#define PCIE_RX_CNTL__RX_IGNORE_CRC_ERR__SHIFT 0x00000003
+#define PCIE_RX_CNTL__RX_IGNORE_EP_ERR_MASK 0x00000040L
+#define PCIE_RX_CNTL__RX_IGNORE_EP_ERR__SHIFT 0x00000006
+#define PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK 0x01000000L
+#define PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT 0x00000018
+#define PCIE_RX_CNTL__RX_IGNORE_IO_ERR_MASK 0x00000001L
+#define PCIE_RX_CNTL__RX_IGNORE_IO_ERR__SHIFT 0x00000000
+#define PCIE_RX_CNTL__RX_IGNORE_IO_UR_MASK 0x00000800L
+#define PCIE_RX_CNTL__RX_IGNORE_IO_UR__SHIFT 0x0000000b
+#define PCIE_RX_CNTL__RX_IGNORE_LEN_MISMATCH_ERR_MASK 0x00000080L
+#define PCIE_RX_CNTL__RX_IGNORE_LEN_MISMATCH_ERR__SHIFT 0x00000007
+#define PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK 0x00000100L
+#define PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT 0x00000008
+#define PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK 0x00400000L
+#define PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT 0x00000016
+#define PCIE_RX_CNTL__RX_IGNORE_MSG_ERR_MASK 0x00000004L
+#define PCIE_RX_CNTL__RX_IGNORE_MSG_ERR__SHIFT 0x00000002
+#define PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK 0x02000000L
+#define PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT 0x00000019
+#define PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK 0x00200000L
+#define PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT 0x00000015
+#define PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK 0x00000200L
+#define PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT 0x00000009
+#define PCIE_RX_CNTL__RX_NAK_IF_FIFO_FULL_MASK 0x00002000L
+#define PCIE_RX_CNTL__RX_NAK_IF_FIFO_FULL__SHIFT 0x0000000d
+#define PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK 0x00100000L
+#define PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT 0x00000014
+#define PCIE_RX_CNTL__RX_RCB_CPL_TIMEOUT_MASK 0x00070000L
+#define PCIE_RX_CNTL__RX_RCB_CPL_TIMEOUT_MODE_MASK 0x00080000L
+#define PCIE_RX_CNTL__RX_RCB_CPL_TIMEOUT_MODE__SHIFT 0x00000013
+#define PCIE_RX_CNTL__RX_RCB_CPL_TIMEOUT__SHIFT 0x00000010
+#define PCIE_RX_CREDITS_ALLOCATED_CPL__RX_CREDITS_ALLOCATED_CPLD_MASK 0x00000fffL
+#define PCIE_RX_CREDITS_ALLOCATED_CPL__RX_CREDITS_ALLOCATED_CPLD__SHIFT 0x00000000
+#define PCIE_RX_CREDITS_ALLOCATED_CPL__RX_CREDITS_ALLOCATED_CPLH_MASK 0x00ff0000L
+#define PCIE_RX_CREDITS_ALLOCATED_CPL__RX_CREDITS_ALLOCATED_CPLH__SHIFT 0x00000010
+#define PCIE_RX_CREDITS_ALLOCATED_NP__RX_CREDITS_ALLOCATED_NPD_MASK 0x00000fffL
+#define PCIE_RX_CREDITS_ALLOCATED_NP__RX_CREDITS_ALLOCATED_NPD__SHIFT 0x00000000
+#define PCIE_RX_CREDITS_ALLOCATED_NP__RX_CREDITS_ALLOCATED_NPH_MASK 0x00ff0000L
+#define PCIE_RX_CREDITS_ALLOCATED_NP__RX_CREDITS_ALLOCATED_NPH__SHIFT 0x00000010
+#define PCIE_RX_CREDITS_ALLOCATED_P__RX_CREDITS_ALLOCATED_PD_MASK 0x00000fffL
+#define PCIE_RX_CREDITS_ALLOCATED_P__RX_CREDITS_ALLOCATED_PD__SHIFT 0x00000000
+#define PCIE_RX_CREDITS_ALLOCATED_P__RX_CREDITS_ALLOCATED_PH_MASK 0x00ff0000L
+#define PCIE_RX_CREDITS_ALLOCATED_P__RX_CREDITS_ALLOCATED_PH__SHIFT 0x00000010
+#define PCIE_RX_EXPECTED_SEQNUM__RX_EXPECTED_SEQNUM_MASK 0x00000fffL
+#define PCIE_RX_EXPECTED_SEQNUM__RX_EXPECTED_SEQNUM__SHIFT 0x00000000
+#define PCIE_RX_LAST_TLP0__RX_LAST_TLP0_MASK 0xffffffffL
+#define PCIE_RX_LAST_TLP0__RX_LAST_TLP0__SHIFT 0x00000000
+#define PCIE_RX_LAST_TLP1__RX_LAST_TLP1_MASK 0xffffffffL
+#define PCIE_RX_LAST_TLP1__RX_LAST_TLP1__SHIFT 0x00000000
+#define PCIE_RX_LAST_TLP2__RX_LAST_TLP2_MASK 0xffffffffL
+#define PCIE_RX_LAST_TLP2__RX_LAST_TLP2__SHIFT 0x00000000
+#define PCIE_RX_LAST_TLP3__RX_LAST_TLP3_MASK 0xffffffffL
+#define PCIE_RX_LAST_TLP3__RX_LAST_TLP3__SHIFT 0x00000000
+#define PCIE_RX_NUM_NAK_GENERATED__RX_NUM_NAK_GENERATED_MASK 0xffffffffL
+#define PCIE_RX_NUM_NAK_GENERATED__RX_NUM_NAK_GENERATED__SHIFT 0x00000000
+#define PCIE_RX_NUM_NAK__RX_NUM_NAK_MASK 0xffffffffL
+#define PCIE_RX_NUM_NAK__RX_NUM_NAK__SHIFT 0x00000000
+#define PCIE_RX_VENDOR_SPECIFIC__RX_VENDOR_DATA_MASK 0x00ffffffL
+#define PCIE_RX_VENDOR_SPECIFIC__RX_VENDOR_DATA__SHIFT 0x00000000
+#define PCIE_RX_VENDOR_SPECIFIC__RX_VENDOR_STATUS_MASK 0x01000000L
+#define PCIE_RX_VENDOR_SPECIFIC__RX_VENDOR_STATUS__SHIFT 0x00000018
+#define PCIE_SCRATCH__PCIE_SCRATCH_MASK 0xffffffffL
+#define PCIE_SCRATCH__PCIE_SCRATCH__SHIFT 0x00000000
+#define PCIE_STRAP_F0__STRAP_F0_ACS_EN_MASK 0x00000040L
+#define PCIE_STRAP_F0__STRAP_F0_ACS_EN__SHIFT 0x00000006
+#define PCIE_STRAP_F0__STRAP_F0_AER_EN_MASK 0x00000020L
+#define PCIE_STRAP_F0__STRAP_F0_AER_EN__SHIFT 0x00000005
+#define PCIE_STRAP_F0__STRAP_F0_ATS_EN_MASK 0x00000400L
+#define PCIE_STRAP_F0__STRAP_F0_ATS_EN__SHIFT 0x0000000a
+#define PCIE_STRAP_F0__STRAP_F0_BAR_EN_MASK 0x00000080L
+#define PCIE_STRAP_F0__STRAP_F0_BAR_EN__SHIFT 0x00000007
+#define PCIE_STRAP_F0__STRAP_F0_DPA_EN_MASK 0x00000200L
+#define PCIE_STRAP_F0__STRAP_F0_DPA_EN__SHIFT 0x00000009
+#define PCIE_STRAP_F0__STRAP_F0_DSN_EN_MASK 0x00000010L
+#define PCIE_STRAP_F0__STRAP_F0_DSN_EN__SHIFT 0x00000004
+#define PCIE_STRAP_F0__STRAP_F0_EN_MASK 0x00000001L
+#define PCIE_STRAP_F0__STRAP_F0_EN__SHIFT 0x00000000
+#define PCIE_STRAP_F0__STRAP_F0_LEGACY_DEVICE_TYPE_EN_MASK 0x00000002L
+#define PCIE_STRAP_F0__STRAP_F0_LEGACY_DEVICE_TYPE_EN__SHIFT 0x00000001
+#define PCIE_STRAP_F0__STRAP_F0_MSI_EN_MASK 0x00000004L
+#define PCIE_STRAP_F0__STRAP_F0_MSI_EN__SHIFT 0x00000002
+#define PCIE_STRAP_F0__STRAP_F0_PAGE_REQ_EN_MASK 0x00000800L
+#define PCIE_STRAP_F0__STRAP_F0_PAGE_REQ_EN__SHIFT 0x0000000b
+#define PCIE_STRAP_F0__STRAP_F0_PASID_EN_MASK 0x00001000L
+#define PCIE_STRAP_F0__STRAP_F0_PASID_EN__SHIFT 0x0000000c
+#define PCIE_STRAP_F0__STRAP_F0_PWR_EN_MASK 0x00000100L
+#define PCIE_STRAP_F0__STRAP_F0_PWR_EN__SHIFT 0x00000008
+#define PCIE_STRAP_F0__STRAP_F0_VC_EN_MASK 0x00000008L
+#define PCIE_STRAP_F0__STRAP_F0_VC_EN__SHIFT 0x00000003
+#define PCIE_STRAP_F1__STRAP_F1_ACS_EN_MASK 0x00000040L
+#define PCIE_STRAP_F1__STRAP_F1_ACS_EN__SHIFT 0x00000006
+#define PCIE_STRAP_F1__STRAP_F1_AER_EN_MASK 0x00000020L
+#define PCIE_STRAP_F1__STRAP_F1_AER_EN__SHIFT 0x00000005
+#define PCIE_STRAP_F1__STRAP_F1_ATS_EN_MASK 0x00000400L
+#define PCIE_STRAP_F1__STRAP_F1_ATS_EN__SHIFT 0x0000000a
+#define PCIE_STRAP_F1__STRAP_F1_BAR_EN_MASK 0x00000080L
+#define PCIE_STRAP_F1__STRAP_F1_BAR_EN__SHIFT 0x00000007
+#define PCIE_STRAP_F1__STRAP_F1_DPA_EN_MASK 0x00000200L
+#define PCIE_STRAP_F1__STRAP_F1_DPA_EN__SHIFT 0x00000009
+#define PCIE_STRAP_F1__STRAP_F1_DSN_EN_MASK 0x00000010L
+#define PCIE_STRAP_F1__STRAP_F1_DSN_EN__SHIFT 0x00000004
+#define PCIE_STRAP_F1__STRAP_F1_EN_MASK 0x00000001L
+#define PCIE_STRAP_F1__STRAP_F1_EN__SHIFT 0x00000000
+#define PCIE_STRAP_F1__STRAP_F1_LEGACY_DEVICE_TYPE_EN_MASK 0x00000002L
+#define PCIE_STRAP_F1__STRAP_F1_LEGACY_DEVICE_TYPE_EN__SHIFT 0x00000001
+#define PCIE_STRAP_F1__STRAP_F1_MSI_EN_MASK 0x00000004L
+#define PCIE_STRAP_F1__STRAP_F1_MSI_EN__SHIFT 0x00000002
+#define PCIE_STRAP_F1__STRAP_F1_PAGE_REQ_EN_MASK 0x00000800L
+#define PCIE_STRAP_F1__STRAP_F1_PAGE_REQ_EN__SHIFT 0x0000000b
+#define PCIE_STRAP_F1__STRAP_F1_PASID_EN_MASK 0x00001000L
+#define PCIE_STRAP_F1__STRAP_F1_PASID_EN__SHIFT 0x0000000c
+#define PCIE_STRAP_F1__STRAP_F1_PWR_EN_MASK 0x00000100L
+#define PCIE_STRAP_F1__STRAP_F1_PWR_EN__SHIFT 0x00000008
+#define PCIE_STRAP_F1__STRAP_F1_VC_EN_MASK 0x00000008L
+#define PCIE_STRAP_F1__STRAP_F1_VC_EN__SHIFT 0x00000003
+#define PCIE_STRAP_F2__STRAP_F2_ACS_EN_MASK 0x00000040L
+#define PCIE_STRAP_F2__STRAP_F2_ACS_EN__SHIFT 0x00000006
+#define PCIE_STRAP_F2__STRAP_F2_AER_EN_MASK 0x00000020L
+#define PCIE_STRAP_F2__STRAP_F2_AER_EN__SHIFT 0x00000005
+#define PCIE_STRAP_F2__STRAP_F2_ATS_EN_MASK 0x00000400L
+#define PCIE_STRAP_F2__STRAP_F2_ATS_EN__SHIFT 0x0000000a
+#define PCIE_STRAP_F2__STRAP_F2_BAR_EN_MASK 0x00000080L
+#define PCIE_STRAP_F2__STRAP_F2_BAR_EN__SHIFT 0x00000007
+#define PCIE_STRAP_F2__STRAP_F2_DPA_EN_MASK 0x00000200L
+#define PCIE_STRAP_F2__STRAP_F2_DPA_EN__SHIFT 0x00000009
+#define PCIE_STRAP_F2__STRAP_F2_DSN_EN_MASK 0x00000010L
+#define PCIE_STRAP_F2__STRAP_F2_DSN_EN__SHIFT 0x00000004
+#define PCIE_STRAP_F2__STRAP_F2_EN_MASK 0x00000001L
+#define PCIE_STRAP_F2__STRAP_F2_EN__SHIFT 0x00000000
+#define PCIE_STRAP_F2__STRAP_F2_LEGACY_DEVICE_TYPE_EN_MASK 0x00000002L
+#define PCIE_STRAP_F2__STRAP_F2_LEGACY_DEVICE_TYPE_EN__SHIFT 0x00000001
+#define PCIE_STRAP_F2__STRAP_F2_MSI_EN_MASK 0x00000004L
+#define PCIE_STRAP_F2__STRAP_F2_MSI_EN__SHIFT 0x00000002
+#define PCIE_STRAP_F2__STRAP_F2_PAGE_REQ_EN_MASK 0x00000800L
+#define PCIE_STRAP_F2__STRAP_F2_PAGE_REQ_EN__SHIFT 0x0000000b
+#define PCIE_STRAP_F2__STRAP_F2_PASID_EN_MASK 0x00001000L
+#define PCIE_STRAP_F2__STRAP_F2_PASID_EN__SHIFT 0x0000000c
+#define PCIE_STRAP_F2__STRAP_F2_PWR_EN_MASK 0x00000100L
+#define PCIE_STRAP_F2__STRAP_F2_PWR_EN__SHIFT 0x00000008
+#define PCIE_STRAP_F2__STRAP_F2_VC_EN_MASK 0x00000008L
+#define PCIE_STRAP_F2__STRAP_F2_VC_EN__SHIFT 0x00000003
+#define PCIE_STRAP_F3__RESERVED_MASK 0xffffffffL
+#define PCIE_STRAP_F3__RESERVED__SHIFT 0x00000000
+#define PCIE_STRAP_F4__RESERVED_MASK 0xffffffffL
+#define PCIE_STRAP_F4__RESERVED__SHIFT 0x00000000
+#define PCIE_STRAP_F5__RESERVED_MASK 0xffffffffL
+#define PCIE_STRAP_F5__RESERVED__SHIFT 0x00000000
+#define PCIE_STRAP_F6__RESERVED_MASK 0xffffffffL
+#define PCIE_STRAP_F6__RESERVED__SHIFT 0x00000000
+#define PCIE_STRAP_F7__RESERVED_MASK 0xffffffffL
+#define PCIE_STRAP_F7__RESERVED__SHIFT 0x00000000
+#define PCIE_STRAP_I2C_BD__STRAP_BIF_DBG_I2C_EN_MASK 0x00000080L
+#define PCIE_STRAP_I2C_BD__STRAP_BIF_DBG_I2C_EN__SHIFT 0x00000007
+#define PCIE_STRAP_I2C_BD__STRAP_BIF_I2C_SLV_ADR_MASK 0x0000007fL
+#define PCIE_STRAP_I2C_BD__STRAP_BIF_I2C_SLV_ADR__SHIFT 0x00000000
+#define PCIE_STRAP_MISC2__STRAP_GEN2_COMPLIANCE_MASK 0x00000002L
+#define PCIE_STRAP_MISC2__STRAP_GEN2_COMPLIANCE__SHIFT 0x00000001
+#define PCIE_STRAP_MISC2__STRAP_GEN3_COMPLIANCE_MASK 0x00000008L
+#define PCIE_STRAP_MISC2__STRAP_GEN3_COMPLIANCE__SHIFT 0x00000003
+#define PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK 0x00000004L
+#define PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT 0x00000002
+#define PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK 0x01000000L
+#define PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT 0x00000018
+#define PCIE_STRAP_MISC__STRAP_ECN1P1_EN_MASK 0x02000000L
+#define PCIE_STRAP_MISC__STRAP_ECN1P1_EN__SHIFT 0x00000019
+#define PCIE_STRAP_MISC__STRAP_EXT_VC_COUNT_MASK 0x04000000L
+#define PCIE_STRAP_MISC__STRAP_EXT_VC_COUNT__SHIFT 0x0000001a
+#define PCIE_STRAP_MISC__STRAP_FLR_EN_MASK 0x40000000L
+#define PCIE_STRAP_MISC__STRAP_FLR_EN__SHIFT 0x0000001e
+#define PCIE_STRAP_MISC__STRAP_LINK_CONFIG_MASK 0x0000000fL
+#define PCIE_STRAP_MISC__STRAP_LINK_CONFIG__SHIFT 0x00000000
+#define PCIE_STRAP_MISC__STRAP_MAX_PASID_WIDTH_MASK 0x00001f00L
+#define PCIE_STRAP_MISC__STRAP_MAX_PASID_WIDTH__SHIFT 0x00000008
+#define PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK 0x20000000L
+#define PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT 0x0000001d
+#define PCIE_STRAP_MISC__STRAP_PASID_EXE_PERMISSION_SUPPORTED_MASK 0x00002000L
+#define PCIE_STRAP_MISC__STRAP_PASID_EXE_PERMISSION_SUPPORTED__SHIFT 0x0000000d
+#define PCIE_STRAP_MISC__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_MASK 0x00008000L
+#define PCIE_STRAP_MISC__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED__SHIFT 0x0000000f
+#define PCIE_STRAP_MISC__STRAP_PASID_PRIV_MODE_SUPPORTED_MASK 0x00004000L
+#define PCIE_STRAP_MISC__STRAP_PASID_PRIV_MODE_SUPPORTED__SHIFT 0x0000000e
+#define PCIE_STRAP_MISC__STRAP_REVERSE_ALL_MASK 0x10000000L
+#define PCIE_STRAP_MISC__STRAP_REVERSE_ALL__SHIFT 0x0000001c
+#define PCIE_STRAP_PI__STRAP_QUICKSIM_START_MASK 0x00000001L
+#define PCIE_STRAP_PI__STRAP_QUICKSIM_START__SHIFT 0x00000000
+#define PCIE_STRAP_PI__STRAP_TEST_TOGGLE_MODE_MASK 0x20000000L
+#define PCIE_STRAP_PI__STRAP_TEST_TOGGLE_MODE__SHIFT 0x0000001d
+#define PCIE_STRAP_PI__STRAP_TEST_TOGGLE_PATTERN_MASK 0x10000000L
+#define PCIE_STRAP_PI__STRAP_TEST_TOGGLE_PATTERN__SHIFT 0x0000001c
+#define PCIE_TX_ACK_LATENCY_LIMIT__TX_ACK_LATENCY_LIMIT_MASK 0x00000fffL
+#define PCIE_TX_ACK_LATENCY_LIMIT__TX_ACK_LATENCY_LIMIT_OVERWRITE_MASK 0x00001000L
+#define PCIE_TX_ACK_LATENCY_LIMIT__TX_ACK_LATENCY_LIMIT_OVERWRITE__SHIFT 0x0000000c
+#define PCIE_TX_ACK_LATENCY_LIMIT__TX_ACK_LATENCY_LIMIT__SHIFT 0x00000000
+#define PCIE_TX_CNTL__TX_CLEAR_EXTRA_PM_REQS_MASK 0x00400000L
+#define PCIE_TX_CNTL__TX_CLEAR_EXTRA_PM_REQS__SHIFT 0x00000016
+#define PCIE_TX_CNTL__TX_CPL_PASS_P_MASK 0x00100000L
+#define PCIE_TX_CNTL__TX_CPL_PASS_P__SHIFT 0x00000014
+#define PCIE_TX_CNTL__TX_FC_UPDATE_TIMEOUT_DIS_MASK 0x00800000L
+#define PCIE_TX_CNTL__TX_FC_UPDATE_TIMEOUT_DIS__SHIFT 0x00000017
+#define PCIE_TX_CNTL__TX_FLUSH_TLP_DIS_MASK 0x00008000L
+#define PCIE_TX_CNTL__TX_FLUSH_TLP_DIS__SHIFT 0x0000000f
+#define PCIE_TX_CNTL__TX_NP_PASS_P_MASK 0x00200000L
+#define PCIE_TX_CNTL__TX_NP_PASS_P__SHIFT 0x00000015
+#define PCIE_TX_CNTL__TX_PACK_PACKET_DIS_MASK 0x00004000L
+#define PCIE_TX_CNTL__TX_PACK_PACKET_DIS__SHIFT 0x0000000e
+#define PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK 0x00003000L
+#define PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT 0x0000000c
+#define PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK 0x00000c00L
+#define PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT 0x0000000a
+#define PCIE_TX_CREDITS_ADVT_CPL__TX_CREDITS_ADVT_CPLD_MASK 0x00000fffL
+#define PCIE_TX_CREDITS_ADVT_CPL__TX_CREDITS_ADVT_CPLD__SHIFT 0x00000000
+#define PCIE_TX_CREDITS_ADVT_CPL__TX_CREDITS_ADVT_CPLH_MASK 0x00ff0000L
+#define PCIE_TX_CREDITS_ADVT_CPL__TX_CREDITS_ADVT_CPLH__SHIFT 0x00000010
+#define PCIE_TX_CREDITS_ADVT_NP__TX_CREDITS_ADVT_NPD_MASK 0x00000fffL
+#define PCIE_TX_CREDITS_ADVT_NP__TX_CREDITS_ADVT_NPD__SHIFT 0x00000000
+#define PCIE_TX_CREDITS_ADVT_NP__TX_CREDITS_ADVT_NPH_MASK 0x00ff0000L
+#define PCIE_TX_CREDITS_ADVT_NP__TX_CREDITS_ADVT_NPH__SHIFT 0x00000010
+#define PCIE_TX_CREDITS_ADVT_P__TX_CREDITS_ADVT_PD_MASK 0x00000fffL
+#define PCIE_TX_CREDITS_ADVT_P__TX_CREDITS_ADVT_PD__SHIFT 0x00000000
+#define PCIE_TX_CREDITS_ADVT_P__TX_CREDITS_ADVT_PH_MASK 0x00ff0000L
+#define PCIE_TX_CREDITS_ADVT_P__TX_CREDITS_ADVT_PH__SHIFT 0x00000010
+#define PCIE_TX_CREDITS_FCU_THRESHOLD__TX_FCU_THRESHOLD_CPL_VC0_MASK 0x00000700L
+#define PCIE_TX_CREDITS_FCU_THRESHOLD__TX_FCU_THRESHOLD_CPL_VC0__SHIFT 0x00000008
+#define PCIE_TX_CREDITS_FCU_THRESHOLD__TX_FCU_THRESHOLD_CPL_VC1_MASK 0x07000000L
+#define PCIE_TX_CREDITS_FCU_THRESHOLD__TX_FCU_THRESHOLD_CPL_VC1__SHIFT 0x00000018
+#define PCIE_TX_CREDITS_FCU_THRESHOLD__TX_FCU_THRESHOLD_NP_VC0_MASK 0x00000070L
+#define PCIE_TX_CREDITS_FCU_THRESHOLD__TX_FCU_THRESHOLD_NP_VC0__SHIFT 0x00000004
+#define PCIE_TX_CREDITS_FCU_THRESHOLD__TX_FCU_THRESHOLD_NP_VC1_MASK 0x00700000L
+#define PCIE_TX_CREDITS_FCU_THRESHOLD__TX_FCU_THRESHOLD_NP_VC1__SHIFT 0x00000014
+#define PCIE_TX_CREDITS_FCU_THRESHOLD__TX_FCU_THRESHOLD_P_VC0_MASK 0x00000007L
+#define PCIE_TX_CREDITS_FCU_THRESHOLD__TX_FCU_THRESHOLD_P_VC0__SHIFT 0x00000000
+#define PCIE_TX_CREDITS_FCU_THRESHOLD__TX_FCU_THRESHOLD_P_VC1_MASK 0x00070000L
+#define PCIE_TX_CREDITS_FCU_THRESHOLD__TX_FCU_THRESHOLD_P_VC1__SHIFT 0x00000010
+#define PCIE_TX_CREDITS_INIT_CPL__TX_CREDITS_INIT_CPLD_MASK 0x00000fffL
+#define PCIE_TX_CREDITS_INIT_CPL__TX_CREDITS_INIT_CPLD__SHIFT 0x00000000
+#define PCIE_TX_CREDITS_INIT_CPL__TX_CREDITS_INIT_CPLH_MASK 0x00ff0000L
+#define PCIE_TX_CREDITS_INIT_CPL__TX_CREDITS_INIT_CPLH__SHIFT 0x00000010
+#define PCIE_TX_CREDITS_INIT_NP__TX_CREDITS_INIT_NPD_MASK 0x00000fffL
+#define PCIE_TX_CREDITS_INIT_NP__TX_CREDITS_INIT_NPD__SHIFT 0x00000000
+#define PCIE_TX_CREDITS_INIT_NP__TX_CREDITS_INIT_NPH_MASK 0x00ff0000L
+#define PCIE_TX_CREDITS_INIT_NP__TX_CREDITS_INIT_NPH__SHIFT 0x00000010
+#define PCIE_TX_CREDITS_INIT_P__TX_CREDITS_INIT_PD_MASK 0x00000fffL
+#define PCIE_TX_CREDITS_INIT_P__TX_CREDITS_INIT_PD__SHIFT 0x00000000
+#define PCIE_TX_CREDITS_INIT_P__TX_CREDITS_INIT_PH_MASK 0x00ff0000L
+#define PCIE_TX_CREDITS_INIT_P__TX_CREDITS_INIT_PH__SHIFT 0x00000010
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_CUR_STATUS_CPLD_MASK 0x00100000L
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_CUR_STATUS_CPLD__SHIFT 0x00000014
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_CUR_STATUS_CPLH_MASK 0x00200000L
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_CUR_STATUS_CPLH__SHIFT 0x00000015
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_CUR_STATUS_NPD_MASK 0x00040000L
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_CUR_STATUS_NPD__SHIFT 0x00000012
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_CUR_STATUS_NPH_MASK 0x00080000L
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_CUR_STATUS_NPH__SHIFT 0x00000013
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_CUR_STATUS_PD_MASK 0x00010000L
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_CUR_STATUS_PD__SHIFT 0x00000010
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_CUR_STATUS_PH_MASK 0x00020000L
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_CUR_STATUS_PH__SHIFT 0x00000011
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_ERR_CPLD_MASK 0x00000010L
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_ERR_CPLD__SHIFT 0x00000004
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_ERR_CPLH_MASK 0x00000020L
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_ERR_CPLH__SHIFT 0x00000005
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_ERR_NPD_MASK 0x00000004L
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_ERR_NPD__SHIFT 0x00000002
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_ERR_NPH_MASK 0x00000008L
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_ERR_NPH__SHIFT 0x00000003
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_ERR_PD_MASK 0x00000001L
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_ERR_PD__SHIFT 0x00000000
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_ERR_PH_MASK 0x00000002L
+#define PCIE_TX_CREDITS_STATUS__TX_CREDITS_ERR_PH__SHIFT 0x00000001
+#define PCIE_TX_LAST_TLP0__TX_LAST_TLP0_MASK 0xffffffffL
+#define PCIE_TX_LAST_TLP0__TX_LAST_TLP0__SHIFT 0x00000000
+#define PCIE_TX_LAST_TLP1__TX_LAST_TLP1_MASK 0xffffffffL
+#define PCIE_TX_LAST_TLP1__TX_LAST_TLP1__SHIFT 0x00000000
+#define PCIE_TX_LAST_TLP2__TX_LAST_TLP2_MASK 0xffffffffL
+#define PCIE_TX_LAST_TLP2__TX_LAST_TLP2__SHIFT 0x00000000
+#define PCIE_TX_LAST_TLP3__TX_LAST_TLP3_MASK 0xffffffffL
+#define PCIE_TX_LAST_TLP3__TX_LAST_TLP3__SHIFT 0x00000000
+#define PCIE_TX_REPLAY__TX_REPLAY_NUM_MASK 0x00000007L
+#define PCIE_TX_REPLAY__TX_REPLAY_NUM__SHIFT 0x00000000
+#define PCIE_TX_REPLAY__TX_REPLAY_TIMER_MASK 0xffff0000L
+#define PCIE_TX_REPLAY__TX_REPLAY_TIMER_OVERWRITE_MASK 0x00008000L
+#define PCIE_TX_REPLAY__TX_REPLAY_TIMER_OVERWRITE__SHIFT 0x0000000f
+#define PCIE_TX_REPLAY__TX_REPLAY_TIMER__SHIFT 0x00000010
+#define PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK 0x0000ff00L
+#define PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT 0x00000008
+#define PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK 0x000000f8L
+#define PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT 0x00000003
+#define PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK 0x00000007L
+#define PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT 0x00000000
+#define PCIE_TX_REQUEST_NUM_CNTL__TX_NUM_OUTSTANDING_NP_EN_MASK 0x80000000L
+#define PCIE_TX_REQUEST_NUM_CNTL__TX_NUM_OUTSTANDING_NP_EN__SHIFT 0x0000001f
+#define PCIE_TX_REQUEST_NUM_CNTL__TX_NUM_OUTSTANDING_NP_MASK 0x3f000000L
+#define PCIE_TX_REQUEST_NUM_CNTL__TX_NUM_OUTSTANDING_NP__SHIFT 0x00000018
+#define PCIE_TX_REQUEST_NUM_CNTL__TX_NUM_OUTSTANDING_NP_VC1_EN_MASK 0x40000000L
+#define PCIE_TX_REQUEST_NUM_CNTL__TX_NUM_OUTSTANDING_NP_VC1_EN__SHIFT 0x0000001e
+#define PCIE_TX_SEQ__TX_ACKD_SEQ_MASK 0x0fff0000L
+#define PCIE_TX_SEQ__TX_ACKD_SEQ__SHIFT 0x00000010
+#define PCIE_TX_SEQ__TX_NEXT_TRANSMIT_SEQ_MASK 0x00000fffL
+#define PCIE_TX_SEQ__TX_NEXT_TRANSMIT_SEQ__SHIFT 0x00000000
+#define PCIE_TX_VENDOR_SPECIFIC__TX_VENDOR_DATA_MASK 0x00ffffffL
+#define PCIE_TX_VENDOR_SPECIFIC__TX_VENDOR_DATA__SHIFT 0x00000000
+#define PCIE_WPR_CNTL__WPR_RESET_COR_EN_MASK 0x00000008L
+#define PCIE_WPR_CNTL__WPR_RESET_COR_EN__SHIFT 0x00000003
+#define PCIE_WPR_CNTL__WPR_RESET_HOT_RST_EN_MASK 0x00000001L
+#define PCIE_WPR_CNTL__WPR_RESET_HOT_RST_EN__SHIFT 0x00000000
+#define PCIE_WPR_CNTL__WPR_RESET_LNK_DIS_EN_MASK 0x00000004L
+#define PCIE_WPR_CNTL__WPR_RESET_LNK_DIS_EN__SHIFT 0x00000002
+#define PCIE_WPR_CNTL__WPR_RESET_LNK_DWN_EN_MASK 0x00000002L
+#define PCIE_WPR_CNTL__WPR_RESET_LNK_DWN_EN__SHIFT 0x00000001
+#define PCIE_WPR_CNTL__WPR_RESET_PHY_EN_MASK 0x00000040L
+#define PCIE_WPR_CNTL__WPR_RESET_PHY_EN__SHIFT 0x00000006
+#define PCIE_WPR_CNTL__WPR_RESET_REG_EN_MASK 0x00000010L
+#define PCIE_WPR_CNTL__WPR_RESET_REG_EN__SHIFT 0x00000004
+#define PCIE_WPR_CNTL__WPR_RESET_STY_EN_MASK 0x00000020L
+#define PCIE_WPR_CNTL__WPR_RESET_STY_EN__SHIFT 0x00000005
+#define PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI_MASK 0x000fffffL
+#define PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI__SHIFT 0x00000000
+#define PEER0_FB_OFFSET_LO__PEER0_FB_EN_MASK 0x80000000L
+#define PEER0_FB_OFFSET_LO__PEER0_FB_EN__SHIFT 0x0000001f
+#define PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO_MASK 0x000fffffL
+#define PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO__SHIFT 0x00000000
+#define PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI_MASK 0x000fffffL
+#define PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI__SHIFT 0x00000000
+#define PEER1_FB_OFFSET_LO__PEER1_FB_EN_MASK 0x80000000L
+#define PEER1_FB_OFFSET_LO__PEER1_FB_EN__SHIFT 0x0000001f
+#define PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO_MASK 0x000fffffL
+#define PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO__SHIFT 0x00000000
+#define PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI_MASK 0x000fffffL
+#define PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI__SHIFT 0x00000000
+#define PEER2_FB_OFFSET_LO__PEER2_FB_EN_MASK 0x80000000L
+#define PEER2_FB_OFFSET_LO__PEER2_FB_EN__SHIFT 0x0000001f
+#define PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO_MASK 0x000fffffL
+#define PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO__SHIFT 0x00000000
+#define PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI_MASK 0x000fffffL
+#define PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI__SHIFT 0x00000000
+#define PEER3_FB_OFFSET_LO__PEER3_FB_EN_MASK 0x80000000L
+#define PEER3_FB_OFFSET_LO__PEER3_FB_EN__SHIFT 0x0000001f
+#define PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO_MASK 0x000fffffL
+#define PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO__SHIFT 0x00000000
+#define PEER_REG_RANGE0__END_ADDR_MASK 0xffff0000L
+#define PEER_REG_RANGE0__END_ADDR__SHIFT 0x00000010
+#define PEER_REG_RANGE0__START_ADDR_MASK 0x0000ffffL
+#define PEER_REG_RANGE0__START_ADDR__SHIFT 0x00000000
+#define PEER_REG_RANGE1__END_ADDR_MASK 0xffff0000L
+#define PEER_REG_RANGE1__END_ADDR__SHIFT 0x00000010
+#define PEER_REG_RANGE1__START_ADDR_MASK 0x0000ffffL
+#define PEER_REG_RANGE1__START_ADDR__SHIFT 0x00000000
+#define SLAVE_HANG_ERROR__AUDIO_HANG_ERROR_MASK 0x00000010L
+#define SLAVE_HANG_ERROR__AUDIO_HANG_ERROR__SHIFT 0x00000004
+#define SLAVE_HANG_ERROR__CEC_HANG_ERROR_MASK 0x00000020L
+#define SLAVE_HANG_ERROR__CEC_HANG_ERROR__SHIFT 0x00000005
+#define SLAVE_HANG_ERROR__HDP_HANG_ERROR_MASK 0x00000002L
+#define SLAVE_HANG_ERROR__HDP_HANG_ERROR__SHIFT 0x00000001
+#define SLAVE_HANG_ERROR__ROM_HANG_ERROR_MASK 0x00000008L
+#define SLAVE_HANG_ERROR__ROM_HANG_ERROR__SHIFT 0x00000003
+#define SLAVE_HANG_ERROR__SRBM_HANG_ERROR_MASK 0x00000001L
+#define SLAVE_HANG_ERROR__SRBM_HANG_ERROR__SHIFT 0x00000000
+#define SLAVE_HANG_ERROR__VGA_HANG_ERROR_MASK 0x00000004L
+#define SLAVE_HANG_ERROR__VGA_HANG_ERROR__SHIFT 0x00000002
+#define SLAVE_HANG_PROTECTION_CNTL__HANG_PROTECTION_TIMER_SEL_MASK 0x0000000eL
+#define SLAVE_HANG_PROTECTION_CNTL__HANG_PROTECTION_TIMER_SEL__SHIFT 0x00000001
+#define SLAVE_REQ_CREDIT_CNTL__BIF_AZ_REQ_CREDIT_MASK 0x00100000L
+#define SLAVE_REQ_CREDIT_CNTL__BIF_AZ_REQ_CREDIT__SHIFT 0x00000014
+#define SLAVE_REQ_CREDIT_CNTL__BIF_HDP_REQ_CREDIT_MASK 0x00007c00L
+#define SLAVE_REQ_CREDIT_CNTL__BIF_HDP_REQ_CREDIT__SHIFT 0x0000000a
+#define SLAVE_REQ_CREDIT_CNTL__BIF_ROM_REQ_CREDIT_MASK 0x00008000L
+#define SLAVE_REQ_CREDIT_CNTL__BIF_ROM_REQ_CREDIT__SHIFT 0x0000000f
+#define SLAVE_REQ_CREDIT_CNTL__BIF_SRBM_REQ_CREDIT_MASK 0x0000001fL
+#define SLAVE_REQ_CREDIT_CNTL__BIF_SRBM_REQ_CREDIT__SHIFT 0x00000000
+#define SLAVE_REQ_CREDIT_CNTL__BIF_VGA_REQ_CREDIT_MASK 0x000001e0L
+#define SLAVE_REQ_CREDIT_CNTL__BIF_VGA_REQ_CREDIT__SHIFT 0x00000005
+#define SLAVE_REQ_CREDIT_CNTL__BIF_XDMA_REQ_CREDIT_MASK 0x7e000000L
+#define SLAVE_REQ_CREDIT_CNTL__BIF_XDMA_REQ_CREDIT__SHIFT 0x00000019
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_A_MASK 0x00000001L
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_A__SHIFT 0x00000000
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_CNTL_EN_MASK 0x00001000L
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_CNTL_EN__SHIFT 0x0000000c
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_MODE_MASK 0x00000004L
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_MODE__SHIFT 0x00000002
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SCHMEN_MASK 0x00000800L
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SCHMEN__SHIFT 0x0000000b
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SEL_MASK 0x00000002L
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SEL__SHIFT 0x00000001
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SLEWN_MASK 0x00000200L
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SLEWN__SHIFT 0x00000009
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SN0_MASK 0x00000020L
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SN0__SHIFT 0x00000005
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SN1_MASK 0x00000040L
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SN1__SHIFT 0x00000006
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SN2_MASK 0x00000080L
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SN2__SHIFT 0x00000007
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SN3_MASK 0x00000100L
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SN3__SHIFT 0x00000008
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SPARE_MASK 0x00000018L
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_SPARE__SHIFT 0x00000003
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_WAKE_MASK 0x00000400L
+#define SMBCLK_PAD_CNTL__SMBCLK_PAD_WAKE__SHIFT 0x0000000a
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_A_MASK 0x00000001L
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_A__SHIFT 0x00000000
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_CNTL_EN_MASK 0x00001000L
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_CNTL_EN__SHIFT 0x0000000c
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_MODE_MASK 0x00000004L
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_MODE__SHIFT 0x00000002
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SCHMEN_MASK 0x00000800L
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SCHMEN__SHIFT 0x0000000b
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SEL_MASK 0x00000002L
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SEL__SHIFT 0x00000001
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SLEWN_MASK 0x00000200L
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SLEWN__SHIFT 0x00000009
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SN0_MASK 0x00000020L
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SN0__SHIFT 0x00000005
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SN1_MASK 0x00000040L
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SN1__SHIFT 0x00000006
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SN2_MASK 0x00000080L
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SN2__SHIFT 0x00000007
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SN3_MASK 0x00000100L
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SN3__SHIFT 0x00000008
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SPARE_MASK 0x00000018L
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_SPARE__SHIFT 0x00000003
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_WAKE_MASK 0x00000400L
+#define SMBDAT_PAD_CNTL__SMBDAT_PAD_WAKE__SHIFT 0x0000000a
+#define SMBUS_BACO_DUMMY__SMBUS_BACO_DUMMY_DATA_MASK 0xffffffffL
+#define SMBUS_BACO_DUMMY__SMBUS_BACO_DUMMY_DATA__SHIFT 0x00000000
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_d.h
new file mode 100644
index 000000000000..ae798f768853
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_d.h
@@ -0,0 +1,4457 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef DCE_6_0_D_H
+#define DCE_6_0_D_H
+
+#define ixATTR00 0x0000
+#define ixATTR01 0x0001
+#define ixATTR02 0x0002
+#define ixATTR03 0x0003
+#define ixATTR04 0x0004
+#define ixATTR05 0x0005
+#define ixATTR06 0x0006
+#define ixATTR07 0x0007
+#define ixATTR08 0x0008
+#define ixATTR09 0x0009
+#define ixATTR0A 0x000A
+#define ixATTR0B 0x000B
+#define ixATTR0C 0x000C
+#define ixATTR0D 0x000D
+#define ixATTR0E 0x000E
+#define ixATTR0F 0x000F
+#define ixATTR10 0x0010
+#define ixATTR11 0x0011
+#define ixATTR12 0x0012
+#define ixATTR13 0x0013
+#define ixATTR14 0x0014
+#define ixAUDIO_DESCRIPTOR0 0x0001
+#define ixAUDIO_DESCRIPTOR10 0x000B
+#define ixAUDIO_DESCRIPTOR1 0x0002
+#define ixAUDIO_DESCRIPTOR11 0x000C
+#define ixAUDIO_DESCRIPTOR12 0x000D
+#define ixAUDIO_DESCRIPTOR13 0x000E
+#define ixAUDIO_DESCRIPTOR2 0x0003
+#define ixAUDIO_DESCRIPTOR3 0x0004
+#define ixAUDIO_DESCRIPTOR4 0x0005
+#define ixAUDIO_DESCRIPTOR5 0x0006
+#define ixAUDIO_DESCRIPTOR6 0x0007
+#define ixAUDIO_DESCRIPTOR7 0x0008
+#define ixAUDIO_DESCRIPTOR8 0x0009
+#define ixAUDIO_DESCRIPTOR9 0x000A
+#define ixAZALIA_CUMULATIVE_LATENCY_COUNT 0x0003
+#define ixAZALIA_CUMULATIVE_REQUEST_COUNT 0x0004
+#define ixAZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x0003
+#define ixAZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT 0x0002
+#define ixAZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x0004
+#define ixAZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING 0x0009
+#define ixAZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE 0x0008
+#define ixAZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0001
+#define ixAZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS 0x0005
+#define ixAZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x0006
+#define ixAZALIA_F0_CODEC_CONVERTER_PIN_DEBUG 0x0000
+#define ixAZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL 0x0007
+#define ixAZALIA_F0_CODEC_PIN_ASSOCIATION_INFO 0x0062
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0 0x0028
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10 0x0032
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1 0x0029
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11 0x0033
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12 0x0034
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13 0x0035
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2 0x002A
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3 0x002B
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4 0x002C
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5 0x002D
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6 0x002E
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7 0x002F
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8 0x0030
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9 0x0031
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER 0x0025
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL 0x0054
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE 0x0036
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2 0x0057
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE 0x0058
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x0056
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR 0x0038
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC 0x0037
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE 0x0023
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0 0x003A
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1 0x003B
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2 0x003C
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3 0x003D
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4 0x003E
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5 0x003F
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6 0x0040
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7 0x0041
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8 0x0042
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE 0x0022
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE 0x0055
+#define ixAZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL 0x0024
+#define ixAZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x0020
+#define ixAZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES 0x0021
+#define ixAZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0 0x0059
+#define ixAZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1 0x005A
+#define ixAZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2 0x005B
+#define ixAZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3 0x005C
+#define ixAZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4 0x005D
+#define ixAZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5 0x005E
+#define ixAZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6 0x005F
+#define ixAZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7 0x0060
+#define ixAZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8 0x0061
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID 0x2706
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT 0x2200
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER 0x270D
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_2 0x270E
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_3 0x273E
+#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_RAMP_RATE 0x2770
+#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x2F09
+#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS 0x2F0B
+#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES 0x2F0A
+#define ixAZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL 0x2724
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION 0x1770
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE 0x1705
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESET 0x17FF
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID 0x1720
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_2 0x1721
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_3 0x1722
+#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_4 0x1723
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_GROUP_TYPE 0x1F05
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES 0x1F0F
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS 0x1F0B
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT 0x1F04
+#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES 0x1F0A
+#define ixAZALIA_F2_CODEC_PIN_ASSOCIATION_INFO 0x3793
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR 0x3776
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR_DATA 0x3776
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_DATA 0x3781
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_INDEX 0x3780
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_CHANNEL_ALLOCATION 0x3771
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO 0x3772
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_HBR 0x377C
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC 0x377B
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MANUFACTURER_ID 0x0000
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE 0x3777
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE 0x3785
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE 0x3778
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE 0x3786
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE 0x3779
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE 0x3787
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE 0x377A
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE 0x3788
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL_MODE 0x3789
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_PORTID0 0x0003
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_PORTID1 0x0004
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_PRODUCT_ID 0x0001
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT 0x371C
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2 0x371D
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3 0x371E
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4 0x371F
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONNECTION_LIST_ENTRY 0x3702
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE 0x3709
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION 0x3770
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_SINK_DESCRIPTION_LEN 0x0002
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE 0x3708
+#define ixAZALIA_F2_CODEC_PIN_CONTROL_WIDGET_CONTROL 0x3707
+#define ixAZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES 0x3F09
+#define ixAZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES 0x3F0C
+#define ixAZALIA_F2_CODEC_PIN_PARAMETER_CONNECTION_LIST_LENGTH 0x3F0E
+#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_REVISION_ID 0x0F02
+#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT 0x0F04
+#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID 0x0F00
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0 0x378A
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1 0x378B
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2 0x378C
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3 0x378D
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4 0x378E
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5 0x378F
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6 0x3790
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7 0x3791
+#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8 0x3792
+#define ixAZALIA_FIFO_SIZE_CONTROL 0x0000
+#define ixAZALIA_LATENCY_COUNTER_CONTROL 0x0001
+#define ixAZALIA_STREAM_DEBUG 0x0005
+#define ixAZALIA_WORSTCASE_LATENCY_COUNT 0x0002
+#define ixCRT00 0x0000
+#define ixCRT01 0x0001
+#define ixCRT02 0x0002
+#define ixCRT03 0x0003
+#define ixCRT04 0x0004
+#define ixCRT05 0x0005
+#define ixCRT06 0x0006
+#define ixCRT07 0x0007
+#define ixCRT08 0x0008
+#define ixCRT09 0x0009
+#define ixCRT0A 0x000A
+#define ixCRT0B 0x000B
+#define ixCRT0C 0x000C
+#define ixCRT0D 0x000D
+#define ixCRT0E 0x000E
+#define ixCRT0F 0x000F
+#define ixCRT10 0x0010
+#define ixCRT11 0x0011
+#define ixCRT12 0x0012
+#define ixCRT13 0x0013
+#define ixCRT14 0x0014
+#define ixCRT15 0x0015
+#define ixCRT16 0x0016
+#define ixCRT17 0x0017
+#define ixCRT18 0x0018
+#define ixCRT1E 0x001E
+#define ixCRT1F 0x001F
+#define ixCRT22 0x0022
+#define ixDCIO_DEBUG10 0x0010
+#define ixDCIO_DEBUG1 0x0001
+#define ixDCIO_DEBUG11 0x0011
+#define ixDCIO_DEBUG12 0x0012
+#define ixDCIO_DEBUG13 0x0013
+#define ixDCIO_DEBUG2 0x0002
+#define ixDCIO_DEBUG3 0x0003
+#define ixDCIO_DEBUG4 0x0004
+#define ixDCIO_DEBUG5 0x0005
+#define ixDCIO_DEBUG6 0x0006
+#define ixDCIO_DEBUG7 0x0007
+#define ixDCIO_DEBUG8 0x0008
+#define ixDCIO_DEBUG9 0x0009
+#define ixDCIO_DEBUGA 0x000A
+#define ixDCIO_DEBUGB 0x000B
+#define ixDCIO_DEBUGC 0x000C
+#define ixDCIO_DEBUGD 0x000D
+#define ixDCIO_DEBUGE 0x000E
+#define ixDCIO_DEBUGF 0x000F
+#define ixDCIO_DEBUG_ID 0x0000
+#define ixDMIF_DEBUG02_CORE0 0x0002
+#define ixDMIF_DEBUG02_CORE1 0x000A
+#define ixDP_AUX1_DEBUG_A 0x0010
+#define ixDP_AUX1_DEBUG_B 0x0011
+#define ixDP_AUX1_DEBUG_C 0x0012
+#define ixDP_AUX1_DEBUG_D 0x0013
+#define ixDP_AUX1_DEBUG_E 0x0014
+#define ixDP_AUX1_DEBUG_F 0x0015
+#define ixDP_AUX1_DEBUG_G 0x0016
+#define ixDP_AUX1_DEBUG_H 0x0017
+#define ixDP_AUX1_DEBUG_I 0x0018
+#define ixDP_AUX2_DEBUG_A 0x0020
+#define ixDP_AUX2_DEBUG_B 0x0021
+#define ixDP_AUX2_DEBUG_C 0x0022
+#define ixDP_AUX2_DEBUG_D 0x0023
+#define ixDP_AUX2_DEBUG_E 0x0024
+#define ixDP_AUX2_DEBUG_F 0x0025
+#define ixDP_AUX2_DEBUG_G 0x0026
+#define ixDP_AUX2_DEBUG_H 0x0027
+#define ixDP_AUX2_DEBUG_I 0x0028
+#define ixDP_AUX3_DEBUG_A 0x0030
+#define ixDP_AUX3_DEBUG_B 0x0031
+#define ixDP_AUX3_DEBUG_C 0x0032
+#define ixDP_AUX3_DEBUG_D 0x0033
+#define ixDP_AUX3_DEBUG_E 0x0034
+#define ixDP_AUX3_DEBUG_F 0x0035
+#define ixDP_AUX3_DEBUG_G 0x0036
+#define ixDP_AUX3_DEBUG_H 0x0037
+#define ixDP_AUX3_DEBUG_I 0x0038
+#define ixDP_AUX4_DEBUG_A 0x0040
+#define ixDP_AUX4_DEBUG_B 0x0041
+#define ixDP_AUX4_DEBUG_C 0x0042
+#define ixDP_AUX4_DEBUG_D 0x0043
+#define ixDP_AUX4_DEBUG_E 0x0044
+#define ixDP_AUX4_DEBUG_F 0x0045
+#define ixDP_AUX4_DEBUG_G 0x0046
+#define ixDP_AUX4_DEBUG_H 0x0047
+#define ixDP_AUX4_DEBUG_I 0x0048
+#define ixDP_AUX5_DEBUG_A 0x0070
+#define ixDP_AUX5_DEBUG_B 0x0071
+#define ixDP_AUX5_DEBUG_C 0x0072
+#define ixDP_AUX5_DEBUG_D 0x0073
+#define ixDP_AUX5_DEBUG_E 0x0074
+#define ixDP_AUX5_DEBUG_F 0x0075
+#define ixDP_AUX5_DEBUG_G 0x0076
+#define ixDP_AUX5_DEBUG_H 0x0077
+#define ixDP_AUX5_DEBUG_I 0x0078
+#define ixDP_AUX6_DEBUG_A 0x0080
+#define ixDP_AUX6_DEBUG_B 0x0081
+#define ixDP_AUX6_DEBUG_C 0x0082
+#define ixDP_AUX6_DEBUG_D 0x0083
+#define ixDP_AUX6_DEBUG_E 0x0084
+#define ixDP_AUX6_DEBUG_F 0x0085
+#define ixDP_AUX6_DEBUG_G 0x0086
+#define ixDP_AUX6_DEBUG_H 0x0087
+#define ixDP_AUX6_DEBUG_I 0x0088
+#define ixFMT_DEBUG0 0x0001
+#define ixFMT_DEBUG1 0x0002
+#define ixFMT_DEBUG2 0x0003
+#define ixFMT_DEBUG_ID 0x0000
+#define ixGRA00 0x0000
+#define ixGRA01 0x0001
+#define ixGRA02 0x0002
+#define ixGRA03 0x0003
+#define ixGRA04 0x0004
+#define ixGRA05 0x0005
+#define ixGRA06 0x0006
+#define ixGRA07 0x0007
+#define ixGRA08 0x0008
+#define ixIDDCCIF02_DBG_DCCIF_C 0x0009
+#define ixIDDCCIF04_DBG_DCCIF_E 0x000B
+#define ixIDDCCIF05_DBG_DCCIF_F 0x000C
+#define ixMVP_DEBUG_12 0x000C
+#define ixMVP_DEBUG_13 0x000D
+#define ixMVP_DEBUG_14 0x000E
+#define ixMVP_DEBUG_15 0x000F
+#define ixMVP_DEBUG_16 0x0010
+#define ixMVP_DEBUG_17 0x0011
+#define ixSEQ00 0x0000
+#define ixSEQ01 0x0001
+#define ixSEQ02 0x0002
+#define ixSEQ03 0x0003
+#define ixSEQ04 0x0004
+#define ixSINK_DESCRIPTION0 0x0005
+#define ixSINK_DESCRIPTION10 0x000F
+#define ixSINK_DESCRIPTION1 0x0006
+#define ixSINK_DESCRIPTION11 0x0010
+#define ixSINK_DESCRIPTION12 0x0011
+#define ixSINK_DESCRIPTION13 0x0012
+#define ixSINK_DESCRIPTION14 0x0013
+#define ixSINK_DESCRIPTION15 0x0014
+#define ixSINK_DESCRIPTION16 0x0015
+#define ixSINK_DESCRIPTION17 0x0016
+#define ixSINK_DESCRIPTION2 0x0007
+#define ixSINK_DESCRIPTION3 0x0008
+#define ixSINK_DESCRIPTION4 0x0009
+#define ixSINK_DESCRIPTION5 0x000A
+#define ixSINK_DESCRIPTION6 0x000B
+#define ixSINK_DESCRIPTION7 0x000C
+#define ixSINK_DESCRIPTION8 0x000D
+#define ixSINK_DESCRIPTION9 0x000E
+#define ixVGADCC_DBG_DCCIF_C 0x007E
+#define mmABM_TEST_DEBUG_DATA 0x169F
+#define mmABM_TEST_DEBUG_INDEX 0x169E
+#define mmAFMT_60958_0 0x1C41
+#define mmAFMT_60958_1 0x1C42
+#define mmAFMT_60958_2 0x1C48
+#define mmAFMT_AUDIO_CRC_CONTROL 0x1C43
+#define mmAFMT_AUDIO_CRC_RESULT 0x1C49
+#define mmAFMT_AUDIO_DBG_DTO_CNTL 0x1C52
+#define mmAFMT_AUDIO_INFO0 0x1C3F
+#define mmAFMT_AUDIO_INFO1 0x1C40
+#define mmAFMT_AUDIO_PACKET_CONTROL 0x1C4B
+#define mmAFMT_AUDIO_PACKET_CONTROL2 0x1C17
+#define mmAFMT_AUDIO_SRC_CONTROL 0x1C4F
+#define mmAFMT_AVI_INFO0 0x1C21
+#define mmAFMT_AVI_INFO1 0x1C22
+#define mmAFMT_AVI_INFO2 0x1C23
+#define mmAFMT_AVI_INFO3 0x1C24
+#define mmAFMT_GENERIC_0 0x1C28
+#define mmAFMT_GENERIC_1 0x1C29
+#define mmAFMT_GENERIC_2 0x1C2A
+#define mmAFMT_GENERIC_3 0x1C2B
+#define mmAFMT_GENERIC_4 0x1C2C
+#define mmAFMT_GENERIC_5 0x1C2D
+#define mmAFMT_GENERIC_6 0x1C2E
+#define mmAFMT_GENERIC_7 0x1C2F
+#define mmAFMT_GENERIC_HDR 0x1C27
+#define mmAFMT_INFOFRAME_CONTROL0 0x1C4D
+#define mmAFMT_INTERRUPT_STATUS 0x1C14
+#define mmAFMT_ISRC1_0 0x1C18
+#define mmAFMT_ISRC1_1 0x1C19
+#define mmAFMT_ISRC1_2 0x1C1A
+#define mmAFMT_ISRC1_3 0x1C1B
+#define mmAFMT_ISRC1_4 0x1C1C
+#define mmAFMT_ISRC2_0 0x1C1D
+#define mmAFMT_ISRC2_1 0x1C1E
+#define mmAFMT_ISRC2_2 0x1C1F
+#define mmAFMT_ISRC2_3 0x1C20
+#define mmAFMT_MPEG_INFO0 0x1C25
+#define mmAFMT_MPEG_INFO1 0x1C26
+#define mmAFMT_RAMP_CONTROL0 0x1C44
+#define mmAFMT_RAMP_CONTROL1 0x1C45
+#define mmAFMT_RAMP_CONTROL2 0x1C46
+#define mmAFMT_RAMP_CONTROL3 0x1C47
+#define mmAFMT_STATUS 0x1C4A
+#define mmAFMT_VBI_PACKET_CONTROL 0x1C4C
+#define mmATTRDR 0x00F0
+#define mmATTRDW 0x00F0
+#define mmATTRX 0x00F0
+#define mmAUX_ARB_CONTROL 0x1882
+#define mmAUX_CONTROL 0x1880
+#define mmAUX_DPHY_RX_CONTROL0 0x188A
+#define mmAUX_DPHY_RX_CONTROL1 0x188B
+#define mmAUX_DPHY_RX_STATUS 0x188D
+#define mmAUX_DPHY_TX_CONTROL 0x1889
+#define mmAUX_DPHY_TX_REF_CONTROL 0x1888
+#define mmAUX_DPHY_TX_STATUS 0x188C
+#define mmAUX_GTC_SYNC_CONTROL 0x188E
+#define mmAUX_GTC_SYNC_DATA 0x1890
+#define mmAUX_INTERRUPT_CONTROL 0x1883
+#define mmAUX_LS_DATA 0x1887
+#define mmAUX_LS_STATUS 0x1885
+#define mmAUXN_IMPCAL 0x190C
+#define mmAUXP_IMPCAL 0x190B
+#define mmAUX_SW_CONTROL 0x1881
+#define mmAUX_SW_DATA 0x1886
+#define mmAUX_SW_STATUS 0x1884
+#define mmAZALIA_APPLICATION_POSITION_IN_CYCLIC_BUFFER 0x17C9
+#define mmAZALIA_AUDIO_DTO 0x17BA
+#define mmAZALIA_AUDIO_DTO_CONTROL 0x17BB
+#define mmAZALIA_BDL_DMA_CONTROL 0x17BF
+#define mmAZALIA_CONTROLLER_DEBUG 0x17CF
+#define mmAZALIA_CORB_DMA_CONTROL 0x17C1
+#define mmAZALIA_CYCLIC_BUFFER_SYNC 0x17CA
+#define mmAZALIA_DATA_DMA_CONTROL 0x17BE
+#define mmAZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL 0x17D5
+#define mmAZALIA_F0_CODEC_DEBUG 0x17DF
+#define mmAZALIA_F0_CODEC_ENDPOINT_DATA 0x1781
+#define mmAZALIA_F0_CODEC_ENDPOINT_INDEX 0x1780
+#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION 0x17DE
+#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE 0x17DB
+#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_RESET 0x17DC
+#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID 0x17DD
+#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_GROUP_TYPE 0x17D7
+#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES 0x17DA
+#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS 0x17D9
+#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES 0x17D8
+#define mmAZALIA_F0_CODEC_RESYNC_FIFO_CONTROL 0x17D6
+#define mmAZALIA_F0_CODEC_ROOT_PARAMETER_REVISION_ID 0x17D3
+#define mmAZALIA_F0_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID 0x17D2
+#define mmAZALIA_GLOBAL_CAPABILITIES 0x17CB
+#define mmAZALIA_OUTPUT_PAYLOAD_CAPABILITY 0x17CC
+#define mmAZALIA_OUTPUT_STREAM_ARBITER_CONTROL 0x17CD
+#define mmAZALIA_RIRB_AND_DP_CONTROL 0x17C0
+#define mmAZALIA_SCLK_CONTROL 0x17BC
+#define mmAZALIA_STREAM_DATA 0x17E9
+#define mmAZALIA_STREAM_INDEX 0x17E8
+#define mmAZALIA_UNDERFLOW_FILLER_SAMPLE 0x17BD
+#define mmAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA 0x1781
+#define mmAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x1780
+#define mmAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_DATA 0x1787
+#define mmAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x1786
+#define mmAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_DATA 0x178D
+#define mmAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x178C
+#define mmAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_DATA 0x1793
+#define mmAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x1792
+#define mmAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_DATA 0x1799
+#define mmAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x1798
+#define mmAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_DATA 0x179F
+#define mmAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_INDEX 0x179E
+#define mmAZF0STREAM0_AZALIA_STREAM_DATA 0x17E9
+#define mmAZF0STREAM0_AZALIA_STREAM_INDEX 0x17E8
+#define mmAZF0STREAM1_AZALIA_STREAM_DATA 0x17ED
+#define mmAZF0STREAM1_AZALIA_STREAM_INDEX 0x17EC
+#define mmAZF0STREAM2_AZALIA_STREAM_DATA 0x17F1
+#define mmAZF0STREAM2_AZALIA_STREAM_INDEX 0x17F0
+#define mmAZF0STREAM3_AZALIA_STREAM_DATA 0x17F5
+#define mmAZF0STREAM3_AZALIA_STREAM_INDEX 0x17F4
+#define mmAZF0STREAM4_AZALIA_STREAM_DATA 0x17F9
+#define mmAZF0STREAM4_AZALIA_STREAM_INDEX 0x17F8
+#define mmAZF0STREAM5_AZALIA_STREAM_DATA 0x17FD
+#define mmAZF0STREAM5_AZALIA_STREAM_INDEX 0x17FC
+#define mmAZ_TEST_DEBUG_DATA 0x17D1
+#define mmAZ_TEST_DEBUG_INDEX 0x17D0
+#define mmBL1_PWM_ABM_CNTL 0x162E
+#define mmBL1_PWM_AMBIENT_LIGHT_LEVEL 0x1628
+#define mmBL1_PWM_BL_UPDATE_SAMPLE_RATE 0x162F
+#define mmBL1_PWM_CURRENT_ABM_LEVEL 0x162B
+#define mmBL1_PWM_FINAL_DUTY_CYCLE 0x162C
+#define mmBL1_PWM_GRP2_REG_LOCK 0x1630
+#define mmBL1_PWM_MINIMUM_DUTY_CYCLE 0x162D
+#define mmBL1_PWM_TARGET_ABM_LEVEL 0x162A
+#define mmBL1_PWM_USER_LEVEL 0x1629
+#define mmBL_PWM_CNTL 0x191E
+#define mmBL_PWM_CNTL2 0x191F
+#define mmBL_PWM_GRP1_REG_LOCK 0x1921
+#define mmBL_PWM_PERIOD_CNTL 0x1920
+#define mmBPHYC_DAC_AUTO_CALIB_CONTROL 0x19FE
+#define mmBPHYC_DAC_MACRO_CNTL 0x19FD
+#define mmCC_DC_PIPE_DIS 0x177F
+#define mmCC_RCU_DC_AUDIO_PORT_CONNECTIVITY 0x17D4
+#define mmCOMM_MATRIXA_TRANS_C11_C12 0x1A43
+#define mmCOMM_MATRIXA_TRANS_C13_C14 0x1A44
+#define mmCOMM_MATRIXA_TRANS_C21_C22 0x1A45
+#define mmCOMM_MATRIXA_TRANS_C23_C24 0x1A46
+#define mmCOMM_MATRIXA_TRANS_C31_C32 0x1A47
+#define mmCOMM_MATRIXA_TRANS_C33_C34 0x1A48
+#define mmCOMM_MATRIXB_TRANS_C11_C12 0x1A49
+#define mmCOMM_MATRIXB_TRANS_C13_C14 0x1A4A
+#define mmCOMM_MATRIXB_TRANS_C21_C22 0x1A4B
+#define mmCOMM_MATRIXB_TRANS_C23_C24 0x1A4C
+#define mmCOMM_MATRIXB_TRANS_C31_C32 0x1A4D
+#define mmCOMM_MATRIXB_TRANS_C33_C34 0x1A4E
+#define mmCRTC0_CRTC_3D_STRUCTURE_CONTROL 0x1B78
+#define mmCRTC0_CRTC_ALLOW_STOP_OFF_V_CNT 0x1BC3
+#define mmCRTC0_CRTC_BLACK_COLOR 0x1BA2
+#define mmCRTC0_CRTC_BLANK_CONTROL 0x1B9D
+#define mmCRTC0_CRTC_BLANK_DATA_COLOR 0x1BA1
+#define mmCRTC0_CRTC_CONTROL 0x1B9C
+#define mmCRTC0_CRTC_COUNT_CONTROL 0x1BA9
+#define mmCRTC0_CRTC_COUNT_RESET 0x1BAA
+#define mmCRTC0_CRTC_DCFE_CLOCK_CONTROL 0x1B7C
+#define mmCRTC0_CRTC_DOUBLE_BUFFER_CONTROL 0x1BB6
+#define mmCRTC0_CRTC_DTMTEST_CNTL 0x1B92
+#define mmCRTC0_CRTC_DTMTEST_STATUS_POSITION 0x1B93
+#define mmCRTC0_CRTC_FLOW_CONTROL 0x1B99
+#define mmCRTC0_CRTC_FORCE_COUNT_NOW_CNTL 0x1B98
+#define mmCRTC0_CRTC_GSL_CONTROL 0x1B7B
+#define mmCRTC0_CRTC_GSL_VSYNC_GAP 0x1B79
+#define mmCRTC0_CRTC_GSL_WINDOW 0x1B7A
+#define mmCRTC0_CRTC_H_BLANK_EARLY_NUM 0x1B7D
+#define mmCRTC0_CRTC_H_BLANK_START_END 0x1B81
+#define mmCRTC0_CRTC_H_SYNC_A 0x1B82
+#define mmCRTC0_CRTC_H_SYNC_A_CNTL 0x1B83
+#define mmCRTC0_CRTC_H_SYNC_B 0x1B84
+#define mmCRTC0_CRTC_H_SYNC_B_CNTL 0x1B85
+#define mmCRTC0_CRTC_H_TOTAL 0x1B80
+#define mmCRTC0_CRTC_INTERLACE_CONTROL 0x1B9E
+#define mmCRTC0_CRTC_INTERLACE_STATUS 0x1B9F
+#define mmCRTC0_CRTC_INTERRUPT_CONTROL 0x1BB4
+#define mmCRTC0_CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE 0x1BAB
+#define mmCRTC0_CRTC_MASTER_EN 0x1BC2
+#define mmCRTC0_CRTC_MVP_INBAND_CNTL_INSERT 0x1BBF
+#define mmCRTC0_CRTC_MVP_INBAND_CNTL_INSERT_TIMER 0x1BC0
+#define mmCRTC0_CRTC_MVP_STATUS 0x1BC1
+#define mmCRTC0_CRTC_NOM_VERT_POSITION 0x1BA5
+#define mmCRTC0_CRTC_OVERSCAN_COLOR 0x1BA0
+#define mmCRTC0_CRTC_SNAPSHOT_CONTROL 0x1BB0
+#define mmCRTC0_CRTC_SNAPSHOT_FRAME 0x1BB2
+#define mmCRTC0_CRTC_SNAPSHOT_POSITION 0x1BB1
+#define mmCRTC0_CRTC_SNAPSHOT_STATUS 0x1BAF
+#define mmCRTC0_CRTC_START_LINE_CONTROL 0x1BB3
+#define mmCRTC0_CRTC_STATUS 0x1BA3
+#define mmCRTC0_CRTC_STATUS_FRAME_COUNT 0x1BA6
+#define mmCRTC0_CRTC_STATUS_HV_COUNT 0x1BA8
+#define mmCRTC0_CRTC_STATUS_POSITION 0x1BA4
+#define mmCRTC0_CRTC_STATUS_VF_COUNT 0x1BA7
+#define mmCRTC0_CRTC_STEREO_CONTROL 0x1BAE
+#define mmCRTC0_CRTC_STEREO_FORCE_NEXT_EYE 0x1B9B
+#define mmCRTC0_CRTC_STEREO_STATUS 0x1BAD
+#define mmCRTC0_CRTC_TEST_DEBUG_DATA 0x1BC7
+#define mmCRTC0_CRTC_TEST_DEBUG_INDEX 0x1BC6
+#define mmCRTC0_CRTC_TEST_PATTERN_COLOR 0x1BBC
+#define mmCRTC0_CRTC_TEST_PATTERN_CONTROL 0x1BBA
+#define mmCRTC0_CRTC_TEST_PATTERN_PARAMETERS 0x1BBB
+#define mmCRTC0_CRTC_TRIGA_CNTL 0x1B94
+#define mmCRTC0_CRTC_TRIGA_MANUAL_TRIG 0x1B95
+#define mmCRTC0_CRTC_TRIGB_CNTL 0x1B96
+#define mmCRTC0_CRTC_TRIGB_MANUAL_TRIG 0x1B97
+#define mmCRTC0_CRTC_UPDATE_LOCK 0x1BB5
+#define mmCRTC0_CRTC_VBI_END 0x1B86
+#define mmCRTC0_CRTC_V_BLANK_START_END 0x1B8D
+#define mmCRTC0_CRTC_VERT_SYNC_CONTROL 0x1BAC
+#define mmCRTC0_CRTC_VGA_PARAMETER_CAPTURE_MODE 0x1BB7
+#define mmCRTC0_CRTC_V_SYNC_A 0x1B8E
+#define mmCRTC0_CRTC_V_SYNC_A_CNTL 0x1B8F
+#define mmCRTC0_CRTC_V_SYNC_B 0x1B90
+#define mmCRTC0_CRTC_V_SYNC_B_CNTL 0x1B91
+#define mmCRTC0_CRTC_VSYNC_NOM_INT_STATUS 0x1B8C
+#define mmCRTC0_CRTC_V_TOTAL 0x1B87
+#define mmCRTC0_CRTC_V_TOTAL_CONTROL 0x1B8A
+#define mmCRTC0_CRTC_V_TOTAL_INT_STATUS 0x1B8B
+#define mmCRTC0_CRTC_V_TOTAL_MAX 0x1B89
+#define mmCRTC0_CRTC_V_TOTAL_MIN 0x1B88
+#define mmCRTC0_CRTC_V_UPDATE_INT_STATUS 0x1BC4
+#define mmCRTC0_DCFE_DBG_SEL 0x1B7E
+#define mmCRTC0_DCFE_MEM_LIGHT_SLEEP_CNTL 0x1B7F
+#define mmCRTC0_MASTER_UPDATE_LOCK 0x1BBD
+#define mmCRTC0_MASTER_UPDATE_MODE 0x1BBE
+#define mmCRTC0_PIXEL_RATE_CNTL 0x0140
+#define mmCRTC1_CRTC_3D_STRUCTURE_CONTROL 0x1E78
+#define mmCRTC1_CRTC_ALLOW_STOP_OFF_V_CNT 0x1EC3
+#define mmCRTC1_CRTC_BLACK_COLOR 0x1EA2
+#define mmCRTC1_CRTC_BLANK_CONTROL 0x1E9D
+#define mmCRTC1_CRTC_BLANK_DATA_COLOR 0x1EA1
+#define mmCRTC1_CRTC_CONTROL 0x1E9C
+#define mmCRTC1_CRTC_COUNT_CONTROL 0x1EA9
+#define mmCRTC1_CRTC_COUNT_RESET 0x1EAA
+#define mmCRTC1_CRTC_DCFE_CLOCK_CONTROL 0x1E7C
+#define mmCRTC1_CRTC_DOUBLE_BUFFER_CONTROL 0x1EB6
+#define mmCRTC1_CRTC_DTMTEST_CNTL 0x1E92
+#define mmCRTC1_CRTC_DTMTEST_STATUS_POSITION 0x1E93
+#define mmCRTC1_CRTC_FLOW_CONTROL 0x1E99
+#define mmCRTC1_CRTC_FORCE_COUNT_NOW_CNTL 0x1E98
+#define mmCRTC1_CRTC_GSL_CONTROL 0x1E7B
+#define mmCRTC1_CRTC_GSL_VSYNC_GAP 0x1E79
+#define mmCRTC1_CRTC_GSL_WINDOW 0x1E7A
+#define mmCRTC1_CRTC_H_BLANK_EARLY_NUM 0x1E7D
+#define mmCRTC1_CRTC_H_BLANK_START_END 0x1E81
+#define mmCRTC1_CRTC_H_SYNC_A 0x1E82
+#define mmCRTC1_CRTC_H_SYNC_A_CNTL 0x1E83
+#define mmCRTC1_CRTC_H_SYNC_B 0x1E84
+#define mmCRTC1_CRTC_H_SYNC_B_CNTL 0x1E85
+#define mmCRTC1_CRTC_H_TOTAL 0x1E80
+#define mmCRTC1_CRTC_INTERLACE_CONTROL 0x1E9E
+#define mmCRTC1_CRTC_INTERLACE_STATUS 0x1E9F
+#define mmCRTC1_CRTC_INTERRUPT_CONTROL 0x1EB4
+#define mmCRTC1_CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE 0x1EAB
+#define mmCRTC1_CRTC_MASTER_EN 0x1EC2
+#define mmCRTC1_CRTC_MVP_INBAND_CNTL_INSERT 0x1EBF
+#define mmCRTC1_CRTC_MVP_INBAND_CNTL_INSERT_TIMER 0x1EC0
+#define mmCRTC1_CRTC_MVP_STATUS 0x1EC1
+#define mmCRTC1_CRTC_NOM_VERT_POSITION 0x1EA5
+#define mmCRTC1_CRTC_OVERSCAN_COLOR 0x1EA0
+#define mmCRTC1_CRTC_SNAPSHOT_CONTROL 0x1EB0
+#define mmCRTC1_CRTC_SNAPSHOT_FRAME 0x1EB2
+#define mmCRTC1_CRTC_SNAPSHOT_POSITION 0x1EB1
+#define mmCRTC1_CRTC_SNAPSHOT_STATUS 0x1EAF
+#define mmCRTC1_CRTC_START_LINE_CONTROL 0x1EB3
+#define mmCRTC1_CRTC_STATUS 0x1EA3
+#define mmCRTC1_CRTC_STATUS_FRAME_COUNT 0x1EA6
+#define mmCRTC1_CRTC_STATUS_HV_COUNT 0x1EA8
+#define mmCRTC1_CRTC_STATUS_POSITION 0x1EA4
+#define mmCRTC1_CRTC_STATUS_VF_COUNT 0x1EA7
+#define mmCRTC1_CRTC_STEREO_CONTROL 0x1EAE
+#define mmCRTC1_CRTC_STEREO_FORCE_NEXT_EYE 0x1E9B
+#define mmCRTC1_CRTC_STEREO_STATUS 0x1EAD
+#define mmCRTC1_CRTC_TEST_DEBUG_DATA 0x1EC7
+#define mmCRTC1_CRTC_TEST_DEBUG_INDEX 0x1EC6
+#define mmCRTC1_CRTC_TEST_PATTERN_COLOR 0x1EBC
+#define mmCRTC1_CRTC_TEST_PATTERN_CONTROL 0x1EBA
+#define mmCRTC1_CRTC_TEST_PATTERN_PARAMETERS 0x1EBB
+#define mmCRTC1_CRTC_TRIGA_CNTL 0x1E94
+#define mmCRTC1_CRTC_TRIGA_MANUAL_TRIG 0x1E95
+#define mmCRTC1_CRTC_TRIGB_CNTL 0x1E96
+#define mmCRTC1_CRTC_TRIGB_MANUAL_TRIG 0x1E97
+#define mmCRTC1_CRTC_UPDATE_LOCK 0x1EB5
+#define mmCRTC1_CRTC_VBI_END 0x1E86
+#define mmCRTC1_CRTC_V_BLANK_START_END 0x1E8D
+#define mmCRTC1_CRTC_VERT_SYNC_CONTROL 0x1EAC
+#define mmCRTC1_CRTC_VGA_PARAMETER_CAPTURE_MODE 0x1EB7
+#define mmCRTC1_CRTC_V_SYNC_A 0x1E8E
+#define mmCRTC1_CRTC_V_SYNC_A_CNTL 0x1E8F
+#define mmCRTC1_CRTC_V_SYNC_B 0x1E90
+#define mmCRTC1_CRTC_V_SYNC_B_CNTL 0x1E91
+#define mmCRTC1_CRTC_VSYNC_NOM_INT_STATUS 0x1E8C
+#define mmCRTC1_CRTC_V_TOTAL 0x1E87
+#define mmCRTC1_CRTC_V_TOTAL_CONTROL 0x1E8A
+#define mmCRTC1_CRTC_V_TOTAL_INT_STATUS 0x1E8B
+#define mmCRTC1_CRTC_V_TOTAL_MAX 0x1E89
+#define mmCRTC1_CRTC_V_TOTAL_MIN 0x1E88
+#define mmCRTC1_CRTC_V_UPDATE_INT_STATUS 0x1EC4
+#define mmCRTC1_DCFE_DBG_SEL 0x1E7E
+#define mmCRTC1_DCFE_MEM_LIGHT_SLEEP_CNTL 0x1E7F
+#define mmCRTC1_MASTER_UPDATE_LOCK 0x1EBD
+#define mmCRTC1_MASTER_UPDATE_MODE 0x1EBE
+#define mmCRTC1_PIXEL_RATE_CNTL 0x0144
+#define mmCRTC2_CRTC_3D_STRUCTURE_CONTROL 0x4178
+#define mmCRTC2_CRTC_ALLOW_STOP_OFF_V_CNT 0x41C3
+#define mmCRTC2_CRTC_BLACK_COLOR 0x41A2
+#define mmCRTC2_CRTC_BLANK_CONTROL 0x419D
+#define mmCRTC2_CRTC_BLANK_DATA_COLOR 0x41A1
+#define mmCRTC2_CRTC_CONTROL 0x419C
+#define mmCRTC2_CRTC_COUNT_CONTROL 0x41A9
+#define mmCRTC2_CRTC_COUNT_RESET 0x41AA
+#define mmCRTC2_CRTC_DCFE_CLOCK_CONTROL 0x417C
+#define mmCRTC2_CRTC_DOUBLE_BUFFER_CONTROL 0x41B6
+#define mmCRTC2_CRTC_DTMTEST_CNTL 0x4192
+#define mmCRTC2_CRTC_DTMTEST_STATUS_POSITION 0x4193
+#define mmCRTC2_CRTC_FLOW_CONTROL 0x4199
+#define mmCRTC2_CRTC_FORCE_COUNT_NOW_CNTL 0x4198
+#define mmCRTC2_CRTC_GSL_CONTROL 0x417B
+#define mmCRTC2_CRTC_GSL_VSYNC_GAP 0x4179
+#define mmCRTC2_CRTC_GSL_WINDOW 0x417A
+#define mmCRTC2_CRTC_H_BLANK_EARLY_NUM 0x417D
+#define mmCRTC2_CRTC_H_BLANK_START_END 0x4181
+#define mmCRTC2_CRTC_H_SYNC_A 0x4182
+#define mmCRTC2_CRTC_H_SYNC_A_CNTL 0x4183
+#define mmCRTC2_CRTC_H_SYNC_B 0x4184
+#define mmCRTC2_CRTC_H_SYNC_B_CNTL 0x4185
+#define mmCRTC2_CRTC_H_TOTAL 0x4180
+#define mmCRTC2_CRTC_INTERLACE_CONTROL 0x419E
+#define mmCRTC2_CRTC_INTERLACE_STATUS 0x419F
+#define mmCRTC2_CRTC_INTERRUPT_CONTROL 0x41B4
+#define mmCRTC2_CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE 0x41AB
+#define mmCRTC2_CRTC_MASTER_EN 0x41C2
+#define mmCRTC2_CRTC_MVP_INBAND_CNTL_INSERT 0x41BF
+#define mmCRTC2_CRTC_MVP_INBAND_CNTL_INSERT_TIMER 0x41C0
+#define mmCRTC2_CRTC_MVP_STATUS 0x41C1
+#define mmCRTC2_CRTC_NOM_VERT_POSITION 0x41A5
+#define mmCRTC2_CRTC_OVERSCAN_COLOR 0x41A0
+#define mmCRTC2_CRTC_SNAPSHOT_CONTROL 0x41B0
+#define mmCRTC2_CRTC_SNAPSHOT_FRAME 0x41B2
+#define mmCRTC2_CRTC_SNAPSHOT_POSITION 0x41B1
+#define mmCRTC2_CRTC_SNAPSHOT_STATUS 0x41AF
+#define mmCRTC2_CRTC_START_LINE_CONTROL 0x41B3
+#define mmCRTC2_CRTC_STATUS 0x41A3
+#define mmCRTC2_CRTC_STATUS_FRAME_COUNT 0x41A6
+#define mmCRTC2_CRTC_STATUS_HV_COUNT 0x41A8
+#define mmCRTC2_CRTC_STATUS_POSITION 0x41A4
+#define mmCRTC2_CRTC_STATUS_VF_COUNT 0x41A7
+#define mmCRTC2_CRTC_STEREO_CONTROL 0x41AE
+#define mmCRTC2_CRTC_STEREO_FORCE_NEXT_EYE 0x419B
+#define mmCRTC2_CRTC_STEREO_STATUS 0x41AD
+#define mmCRTC2_CRTC_TEST_DEBUG_DATA 0x41C7
+#define mmCRTC2_CRTC_TEST_DEBUG_INDEX 0x41C6
+#define mmCRTC2_CRTC_TEST_PATTERN_COLOR 0x41BC
+#define mmCRTC2_CRTC_TEST_PATTERN_CONTROL 0x41BA
+#define mmCRTC2_CRTC_TEST_PATTERN_PARAMETERS 0x41BB
+#define mmCRTC2_CRTC_TRIGA_CNTL 0x4194
+#define mmCRTC2_CRTC_TRIGA_MANUAL_TRIG 0x4195
+#define mmCRTC2_CRTC_TRIGB_CNTL 0x4196
+#define mmCRTC2_CRTC_TRIGB_MANUAL_TRIG 0x4197
+#define mmCRTC2_CRTC_UPDATE_LOCK 0x41B5
+#define mmCRTC2_CRTC_VBI_END 0x4186
+#define mmCRTC2_CRTC_V_BLANK_START_END 0x418D
+#define mmCRTC2_CRTC_VERT_SYNC_CONTROL 0x41AC
+#define mmCRTC2_CRTC_VGA_PARAMETER_CAPTURE_MODE 0x41B7
+#define mmCRTC2_CRTC_V_SYNC_A 0x418E
+#define mmCRTC2_CRTC_V_SYNC_A_CNTL 0x418F
+#define mmCRTC2_CRTC_V_SYNC_B 0x4190
+#define mmCRTC2_CRTC_V_SYNC_B_CNTL 0x4191
+#define mmCRTC2_CRTC_VSYNC_NOM_INT_STATUS 0x418C
+#define mmCRTC2_CRTC_V_TOTAL 0x4187
+#define mmCRTC2_CRTC_V_TOTAL_CONTROL 0x418A
+#define mmCRTC2_CRTC_V_TOTAL_INT_STATUS 0x418B
+#define mmCRTC2_CRTC_V_TOTAL_MAX 0x4189
+#define mmCRTC2_CRTC_V_TOTAL_MIN 0x4188
+#define mmCRTC2_CRTC_V_UPDATE_INT_STATUS 0x41C4
+#define mmCRTC2_DCFE_DBG_SEL 0x417E
+#define mmCRTC2_DCFE_MEM_LIGHT_SLEEP_CNTL 0x417F
+#define mmCRTC2_MASTER_UPDATE_LOCK 0x41BD
+#define mmCRTC2_MASTER_UPDATE_MODE 0x41BE
+#define mmCRTC2_PIXEL_RATE_CNTL 0x0148
+#define mmCRTC3_CRTC_3D_STRUCTURE_CONTROL 0x4478
+#define mmCRTC3_CRTC_ALLOW_STOP_OFF_V_CNT 0x44C3
+#define mmCRTC3_CRTC_BLACK_COLOR 0x44A2
+#define mmCRTC3_CRTC_BLANK_CONTROL 0x449D
+#define mmCRTC3_CRTC_BLANK_DATA_COLOR 0x44A1
+#define mmCRTC3_CRTC_CONTROL 0x449C
+#define mmCRTC3_CRTC_COUNT_CONTROL 0x44A9
+#define mmCRTC3_CRTC_COUNT_RESET 0x44AA
+#define mmCRTC3_CRTC_DCFE_CLOCK_CONTROL 0x447C
+#define mmCRTC3_CRTC_DOUBLE_BUFFER_CONTROL 0x44B6
+#define mmCRTC3_CRTC_DTMTEST_CNTL 0x4492
+#define mmCRTC3_CRTC_DTMTEST_STATUS_POSITION 0x4493
+#define mmCRTC3_CRTC_FLOW_CONTROL 0x4499
+#define mmCRTC3_CRTC_FORCE_COUNT_NOW_CNTL 0x4498
+#define mmCRTC3_CRTC_GSL_CONTROL 0x447B
+#define mmCRTC3_CRTC_GSL_VSYNC_GAP 0x4479
+#define mmCRTC3_CRTC_GSL_WINDOW 0x447A
+#define mmCRTC3_CRTC_H_BLANK_EARLY_NUM 0x447D
+#define mmCRTC3_CRTC_H_BLANK_START_END 0x4481
+#define mmCRTC3_CRTC_H_SYNC_A 0x4482
+#define mmCRTC3_CRTC_H_SYNC_A_CNTL 0x4483
+#define mmCRTC3_CRTC_H_SYNC_B 0x4484
+#define mmCRTC3_CRTC_H_SYNC_B_CNTL 0x4485
+#define mmCRTC3_CRTC_H_TOTAL 0x4480
+#define mmCRTC3_CRTC_INTERLACE_CONTROL 0x449E
+#define mmCRTC3_CRTC_INTERLACE_STATUS 0x449F
+#define mmCRTC3_CRTC_INTERRUPT_CONTROL 0x44B4
+#define mmCRTC3_CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE 0x44AB
+#define mmCRTC3_CRTC_MASTER_EN 0x44C2
+#define mmCRTC3_CRTC_MVP_INBAND_CNTL_INSERT 0x44BF
+#define mmCRTC3_CRTC_MVP_INBAND_CNTL_INSERT_TIMER 0x44C0
+#define mmCRTC3_CRTC_MVP_STATUS 0x44C1
+#define mmCRTC3_CRTC_NOM_VERT_POSITION 0x44A5
+#define mmCRTC3_CRTC_OVERSCAN_COLOR 0x44A0
+#define mmCRTC3_CRTC_SNAPSHOT_CONTROL 0x44B0
+#define mmCRTC3_CRTC_SNAPSHOT_FRAME 0x44B2
+#define mmCRTC3_CRTC_SNAPSHOT_POSITION 0x44B1
+#define mmCRTC3_CRTC_SNAPSHOT_STATUS 0x44AF
+#define mmCRTC3_CRTC_START_LINE_CONTROL 0x44B3
+#define mmCRTC3_CRTC_STATUS 0x44A3
+#define mmCRTC3_CRTC_STATUS_FRAME_COUNT 0x44A6
+#define mmCRTC3_CRTC_STATUS_HV_COUNT 0x44A8
+#define mmCRTC3_CRTC_STATUS_POSITION 0x44A4
+#define mmCRTC3_CRTC_STATUS_VF_COUNT 0x44A7
+#define mmCRTC3_CRTC_STEREO_CONTROL 0x44AE
+#define mmCRTC3_CRTC_STEREO_FORCE_NEXT_EYE 0x449B
+#define mmCRTC3_CRTC_STEREO_STATUS 0x44AD
+#define mmCRTC3_CRTC_TEST_DEBUG_DATA 0x44C7
+#define mmCRTC3_CRTC_TEST_DEBUG_INDEX 0x44C6
+#define mmCRTC3_CRTC_TEST_PATTERN_COLOR 0x44BC
+#define mmCRTC3_CRTC_TEST_PATTERN_CONTROL 0x44BA
+#define mmCRTC3_CRTC_TEST_PATTERN_PARAMETERS 0x44BB
+#define mmCRTC3_CRTC_TRIGA_CNTL 0x4494
+#define mmCRTC3_CRTC_TRIGA_MANUAL_TRIG 0x4495
+#define mmCRTC3_CRTC_TRIGB_CNTL 0x4496
+#define mmCRTC3_CRTC_TRIGB_MANUAL_TRIG 0x4497
+#define mmCRTC3_CRTC_UPDATE_LOCK 0x44B5
+#define mmCRTC3_CRTC_VBI_END 0x4486
+#define mmCRTC3_CRTC_V_BLANK_START_END 0x448D
+#define mmCRTC3_CRTC_VERT_SYNC_CONTROL 0x44AC
+#define mmCRTC3_CRTC_VGA_PARAMETER_CAPTURE_MODE 0x44B7
+#define mmCRTC3_CRTC_V_SYNC_A 0x448E
+#define mmCRTC3_CRTC_V_SYNC_A_CNTL 0x448F
+#define mmCRTC3_CRTC_V_SYNC_B 0x4490
+#define mmCRTC3_CRTC_V_SYNC_B_CNTL 0x4491
+#define mmCRTC3_CRTC_VSYNC_NOM_INT_STATUS 0x448C
+#define mmCRTC3_CRTC_V_TOTAL 0x4487
+#define mmCRTC3_CRTC_V_TOTAL_CONTROL 0x448A
+#define mmCRTC3_CRTC_V_TOTAL_INT_STATUS 0x448B
+#define mmCRTC3_CRTC_V_TOTAL_MAX 0x4489
+#define mmCRTC3_CRTC_V_TOTAL_MIN 0x4488
+#define mmCRTC3_CRTC_V_UPDATE_INT_STATUS 0x44C4
+#define mmCRTC3_DCFE_DBG_SEL 0x447E
+#define mmCRTC3_DCFE_MEM_LIGHT_SLEEP_CNTL 0x447F
+#define mmCRTC_3D_STRUCTURE_CONTROL 0x1B78
+#define mmCRTC3_MASTER_UPDATE_LOCK 0x44BD
+#define mmCRTC3_MASTER_UPDATE_MODE 0x44BE
+#define mmCRTC3_PIXEL_RATE_CNTL 0x014C
+#define mmCRTC4_CRTC_3D_STRUCTURE_CONTROL 0x4778
+#define mmCRTC4_CRTC_ALLOW_STOP_OFF_V_CNT 0x47C3
+#define mmCRTC4_CRTC_BLACK_COLOR 0x47A2
+#define mmCRTC4_CRTC_BLANK_CONTROL 0x479D
+#define mmCRTC4_CRTC_BLANK_DATA_COLOR 0x47A1
+#define mmCRTC4_CRTC_CONTROL 0x479C
+#define mmCRTC4_CRTC_COUNT_CONTROL 0x47A9
+#define mmCRTC4_CRTC_COUNT_RESET 0x47AA
+#define mmCRTC4_CRTC_DCFE_CLOCK_CONTROL 0x477C
+#define mmCRTC4_CRTC_DOUBLE_BUFFER_CONTROL 0x47B6
+#define mmCRTC4_CRTC_DTMTEST_CNTL 0x4792
+#define mmCRTC4_CRTC_DTMTEST_STATUS_POSITION 0x4793
+#define mmCRTC4_CRTC_FLOW_CONTROL 0x4799
+#define mmCRTC4_CRTC_FORCE_COUNT_NOW_CNTL 0x4798
+#define mmCRTC4_CRTC_GSL_CONTROL 0x477B
+#define mmCRTC4_CRTC_GSL_VSYNC_GAP 0x4779
+#define mmCRTC4_CRTC_GSL_WINDOW 0x477A
+#define mmCRTC4_CRTC_H_BLANK_EARLY_NUM 0x477D
+#define mmCRTC4_CRTC_H_BLANK_START_END 0x4781
+#define mmCRTC4_CRTC_H_SYNC_A 0x4782
+#define mmCRTC4_CRTC_H_SYNC_A_CNTL 0x4783
+#define mmCRTC4_CRTC_H_SYNC_B 0x4784
+#define mmCRTC4_CRTC_H_SYNC_B_CNTL 0x4785
+#define mmCRTC4_CRTC_H_TOTAL 0x4780
+#define mmCRTC4_CRTC_INTERLACE_CONTROL 0x479E
+#define mmCRTC4_CRTC_INTERLACE_STATUS 0x479F
+#define mmCRTC4_CRTC_INTERRUPT_CONTROL 0x47B4
+#define mmCRTC4_CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE 0x47AB
+#define mmCRTC4_CRTC_MASTER_EN 0x47C2
+#define mmCRTC4_CRTC_MVP_INBAND_CNTL_INSERT 0x47BF
+#define mmCRTC4_CRTC_MVP_INBAND_CNTL_INSERT_TIMER 0x47C0
+#define mmCRTC4_CRTC_MVP_STATUS 0x47C1
+#define mmCRTC4_CRTC_NOM_VERT_POSITION 0x47A5
+#define mmCRTC4_CRTC_OVERSCAN_COLOR 0x47A0
+#define mmCRTC4_CRTC_SNAPSHOT_CONTROL 0x47B0
+#define mmCRTC4_CRTC_SNAPSHOT_FRAME 0x47B2
+#define mmCRTC4_CRTC_SNAPSHOT_POSITION 0x47B1
+#define mmCRTC4_CRTC_SNAPSHOT_STATUS 0x47AF
+#define mmCRTC4_CRTC_START_LINE_CONTROL 0x47B3
+#define mmCRTC4_CRTC_STATUS 0x47A3
+#define mmCRTC4_CRTC_STATUS_FRAME_COUNT 0x47A6
+#define mmCRTC4_CRTC_STATUS_HV_COUNT 0x47A8
+#define mmCRTC4_CRTC_STATUS_POSITION 0x47A4
+#define mmCRTC4_CRTC_STATUS_VF_COUNT 0x47A7
+#define mmCRTC4_CRTC_STEREO_CONTROL 0x47AE
+#define mmCRTC4_CRTC_STEREO_FORCE_NEXT_EYE 0x479B
+#define mmCRTC4_CRTC_STEREO_STATUS 0x47AD
+#define mmCRTC4_CRTC_TEST_DEBUG_DATA 0x47C7
+#define mmCRTC4_CRTC_TEST_DEBUG_INDEX 0x47C6
+#define mmCRTC4_CRTC_TEST_PATTERN_COLOR 0x47BC
+#define mmCRTC4_CRTC_TEST_PATTERN_CONTROL 0x47BA
+#define mmCRTC4_CRTC_TEST_PATTERN_PARAMETERS 0x47BB
+#define mmCRTC4_CRTC_TRIGA_CNTL 0x4794
+#define mmCRTC4_CRTC_TRIGA_MANUAL_TRIG 0x4795
+#define mmCRTC4_CRTC_TRIGB_CNTL 0x4796
+#define mmCRTC4_CRTC_TRIGB_MANUAL_TRIG 0x4797
+#define mmCRTC4_CRTC_UPDATE_LOCK 0x47B5
+#define mmCRTC4_CRTC_VBI_END 0x4786
+#define mmCRTC4_CRTC_V_BLANK_START_END 0x478D
+#define mmCRTC4_CRTC_VERT_SYNC_CONTROL 0x47AC
+#define mmCRTC4_CRTC_VGA_PARAMETER_CAPTURE_MODE 0x47B7
+#define mmCRTC4_CRTC_V_SYNC_A 0x478E
+#define mmCRTC4_CRTC_V_SYNC_A_CNTL 0x478F
+#define mmCRTC4_CRTC_V_SYNC_B 0x4790
+#define mmCRTC4_CRTC_V_SYNC_B_CNTL 0x4791
+#define mmCRTC4_CRTC_VSYNC_NOM_INT_STATUS 0x478C
+#define mmCRTC4_CRTC_V_TOTAL 0x4787
+#define mmCRTC4_CRTC_V_TOTAL_CONTROL 0x478A
+#define mmCRTC4_CRTC_V_TOTAL_INT_STATUS 0x478B
+#define mmCRTC4_CRTC_V_TOTAL_MAX 0x4789
+#define mmCRTC4_CRTC_V_TOTAL_MIN 0x4788
+#define mmCRTC4_CRTC_V_UPDATE_INT_STATUS 0x47C4
+#define mmCRTC4_DCFE_DBG_SEL 0x477E
+#define mmCRTC4_DCFE_MEM_LIGHT_SLEEP_CNTL 0x477F
+#define mmCRTC4_MASTER_UPDATE_LOCK 0x47BD
+#define mmCRTC4_MASTER_UPDATE_MODE 0x47BE
+#define mmCRTC4_PIXEL_RATE_CNTL 0x0150
+#define mmCRTC5_CRTC_3D_STRUCTURE_CONTROL 0x4A78
+#define mmCRTC5_CRTC_ALLOW_STOP_OFF_V_CNT 0x4AC3
+#define mmCRTC5_CRTC_BLACK_COLOR 0x4AA2
+#define mmCRTC5_CRTC_BLANK_CONTROL 0x4A9D
+#define mmCRTC5_CRTC_BLANK_DATA_COLOR 0x4AA1
+#define mmCRTC5_CRTC_CONTROL 0x4A9C
+#define mmCRTC5_CRTC_COUNT_CONTROL 0x4AA9
+#define mmCRTC5_CRTC_COUNT_RESET 0x4AAA
+#define mmCRTC5_CRTC_DCFE_CLOCK_CONTROL 0x4A7C
+#define mmCRTC5_CRTC_DOUBLE_BUFFER_CONTROL 0x4AB6
+#define mmCRTC5_CRTC_DTMTEST_CNTL 0x4A92
+#define mmCRTC5_CRTC_DTMTEST_STATUS_POSITION 0x4A93
+#define mmCRTC5_CRTC_FLOW_CONTROL 0x4A99
+#define mmCRTC5_CRTC_FORCE_COUNT_NOW_CNTL 0x4A98
+#define mmCRTC5_CRTC_GSL_CONTROL 0x4A7B
+#define mmCRTC5_CRTC_GSL_VSYNC_GAP 0x4A79
+#define mmCRTC5_CRTC_GSL_WINDOW 0x4A7A
+#define mmCRTC5_CRTC_H_BLANK_EARLY_NUM 0x4A7D
+#define mmCRTC5_CRTC_H_BLANK_START_END 0x4A81
+#define mmCRTC5_CRTC_H_SYNC_A 0x4A82
+#define mmCRTC5_CRTC_H_SYNC_A_CNTL 0x4A83
+#define mmCRTC5_CRTC_H_SYNC_B 0x4A84
+#define mmCRTC5_CRTC_H_SYNC_B_CNTL 0x4A85
+#define mmCRTC5_CRTC_H_TOTAL 0x4A80
+#define mmCRTC5_CRTC_INTERLACE_CONTROL 0x4A9E
+#define mmCRTC5_CRTC_INTERLACE_STATUS 0x4A9F
+#define mmCRTC5_CRTC_INTERRUPT_CONTROL 0x4AB4
+#define mmCRTC5_CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE 0x4AAB
+#define mmCRTC5_CRTC_MASTER_EN 0x4AC2
+#define mmCRTC5_CRTC_MVP_INBAND_CNTL_INSERT 0x4ABF
+#define mmCRTC5_CRTC_MVP_INBAND_CNTL_INSERT_TIMER 0x4AC0
+#define mmCRTC5_CRTC_MVP_STATUS 0x4AC1
+#define mmCRTC5_CRTC_NOM_VERT_POSITION 0x4AA5
+#define mmCRTC5_CRTC_OVERSCAN_COLOR 0x4AA0
+#define mmCRTC5_CRTC_SNAPSHOT_CONTROL 0x4AB0
+#define mmCRTC5_CRTC_SNAPSHOT_FRAME 0x4AB2
+#define mmCRTC5_CRTC_SNAPSHOT_POSITION 0x4AB1
+#define mmCRTC5_CRTC_SNAPSHOT_STATUS 0x4AAF
+#define mmCRTC5_CRTC_START_LINE_CONTROL 0x4AB3
+#define mmCRTC5_CRTC_STATUS 0x4AA3
+#define mmCRTC5_CRTC_STATUS_FRAME_COUNT 0x4AA6
+#define mmCRTC5_CRTC_STATUS_HV_COUNT 0x4AA8
+#define mmCRTC5_CRTC_STATUS_POSITION 0x4AA4
+#define mmCRTC5_CRTC_STATUS_VF_COUNT 0x4AA7
+#define mmCRTC5_CRTC_STEREO_CONTROL 0x4AAE
+#define mmCRTC5_CRTC_STEREO_FORCE_NEXT_EYE 0x4A9B
+#define mmCRTC5_CRTC_STEREO_STATUS 0x4AAD
+#define mmCRTC5_CRTC_TEST_DEBUG_DATA 0x4AC7
+#define mmCRTC5_CRTC_TEST_DEBUG_INDEX 0x4AC6
+#define mmCRTC5_CRTC_TEST_PATTERN_COLOR 0x4ABC
+#define mmCRTC5_CRTC_TEST_PATTERN_CONTROL 0x4ABA
+#define mmCRTC5_CRTC_TEST_PATTERN_PARAMETERS 0x4ABB
+#define mmCRTC5_CRTC_TRIGA_CNTL 0x4A94
+#define mmCRTC5_CRTC_TRIGA_MANUAL_TRIG 0x4A95
+#define mmCRTC5_CRTC_TRIGB_CNTL 0x4A96
+#define mmCRTC5_CRTC_TRIGB_MANUAL_TRIG 0x4A97
+#define mmCRTC5_CRTC_UPDATE_LOCK 0x4AB5
+#define mmCRTC5_CRTC_VBI_END 0x4A86
+#define mmCRTC5_CRTC_V_BLANK_START_END 0x4A8D
+#define mmCRTC5_CRTC_VERT_SYNC_CONTROL 0x4AAC
+#define mmCRTC5_CRTC_VGA_PARAMETER_CAPTURE_MODE 0x4AB7
+#define mmCRTC5_CRTC_V_SYNC_A 0x4A8E
+#define mmCRTC5_CRTC_V_SYNC_A_CNTL 0x4A8F
+#define mmCRTC5_CRTC_V_SYNC_B 0x4A90
+#define mmCRTC5_CRTC_V_SYNC_B_CNTL 0x4A91
+#define mmCRTC5_CRTC_VSYNC_NOM_INT_STATUS 0x4A8C
+#define mmCRTC5_CRTC_V_TOTAL 0x4A87
+#define mmCRTC5_CRTC_V_TOTAL_CONTROL 0x4A8A
+#define mmCRTC5_CRTC_V_TOTAL_INT_STATUS 0x4A8B
+#define mmCRTC5_CRTC_V_TOTAL_MAX 0x4A89
+#define mmCRTC5_CRTC_V_TOTAL_MIN 0x4A88
+#define mmCRTC5_CRTC_V_UPDATE_INT_STATUS 0x4AC4
+#define mmCRTC5_DCFE_DBG_SEL 0x4A7E
+#define mmCRTC5_DCFE_MEM_LIGHT_SLEEP_CNTL 0x4A7F
+#define mmCRTC5_MASTER_UPDATE_LOCK 0x4ABD
+#define mmCRTC5_MASTER_UPDATE_MODE 0x4ABE
+#define mmCRTC5_PIXEL_RATE_CNTL 0x0154
+#define mmCRTC8_DATA 0x00ED
+#define mmCRTC8_IDX 0x00ED
+#define mmCRTC_ALLOW_STOP_OFF_V_CNT 0x1BC3
+#define mmCRTC_BLACK_COLOR 0x1BA2
+#define mmCRTC_BLANK_CONTROL 0x1B9D
+#define mmCRTC_BLANK_DATA_COLOR 0x1BA1
+#define mmCRTC_CONTROL 0x1B9C
+#define mmCRTC_COUNT_CONTROL 0x1BA9
+#define mmCRTC_COUNT_RESET 0x1BAA
+#define mmCRTC_DCFE_CLOCK_CONTROL 0x1B7C
+#define mmCRTC_DOUBLE_BUFFER_CONTROL 0x1BB6
+#define mmCRTC_DTMTEST_CNTL 0x1B92
+#define mmCRTC_DTMTEST_STATUS_POSITION 0x1B93
+#define mmCRTC_FLOW_CONTROL 0x1B99
+#define mmCRTC_FORCE_COUNT_NOW_CNTL 0x1B98
+#define mmCRTC_GSL_CONTROL 0x1B7B
+#define mmCRTC_GSL_VSYNC_GAP 0x1B79
+#define mmCRTC_GSL_WINDOW 0x1B7A
+#define mmCRTC_H_BLANK_EARLY_NUM 0x1B7D
+#define mmCRTC_H_BLANK_START_END 0x1B81
+#define mmCRTC_H_SYNC_A 0x1B82
+#define mmCRTC_H_SYNC_A_CNTL 0x1B83
+#define mmCRTC_H_SYNC_B 0x1B84
+#define mmCRTC_H_SYNC_B_CNTL 0x1B85
+#define mmCRTC_H_TOTAL 0x1B80
+#define mmCRTC_INTERLACE_CONTROL 0x1B9E
+#define mmCRTC_INTERLACE_STATUS 0x1B9F
+#define mmCRTC_INTERRUPT_CONTROL 0x1BB4
+#define mmCRTC_MANUAL_FORCE_VSYNC_NEXT_LINE 0x1BAB
+#define mmCRTC_MASTER_EN 0x1BC2
+#define mmCRTC_MVP_INBAND_CNTL_INSERT 0x1BBF
+#define mmCRTC_MVP_INBAND_CNTL_INSERT_TIMER 0x1BC0
+#define mmCRTC_MVP_STATUS 0x1BC1
+#define mmCRTC_NOM_VERT_POSITION 0x1BA5
+#define mmCRTC_OVERSCAN_COLOR 0x1BA0
+#define mmCRTC_SNAPSHOT_CONTROL 0x1BB0
+#define mmCRTC_SNAPSHOT_FRAME 0x1BB2
+#define mmCRTC_SNAPSHOT_POSITION 0x1BB1
+#define mmCRTC_SNAPSHOT_STATUS 0x1BAF
+#define mmCRTC_START_LINE_CONTROL 0x1BB3
+#define mmCRTC_STATUS 0x1BA3
+#define mmCRTC_STATUS_FRAME_COUNT 0x1BA6
+#define mmCRTC_STATUS_HV_COUNT 0x1BA8
+#define mmCRTC_STATUS_POSITION 0x1BA4
+#define mmCRTC_STATUS_VF_COUNT 0x1BA7
+#define mmCRTC_STEREO_CONTROL 0x1BAE
+#define mmCRTC_STEREO_FORCE_NEXT_EYE 0x1B9B
+#define mmCRTC_STEREO_STATUS 0x1BAD
+#define mmCRTC_TEST_DEBUG_DATA 0x1BC7
+#define mmCRTC_TEST_DEBUG_INDEX 0x1BC6
+#define mmCRTC_TEST_PATTERN_COLOR 0x1BBC
+#define mmCRTC_TEST_PATTERN_CONTROL 0x1BBA
+#define mmCRTC_TEST_PATTERN_PARAMETERS 0x1BBB
+#define mmCRTC_TRIGA_CNTL 0x1B94
+#define mmCRTC_TRIGA_MANUAL_TRIG 0x1B95
+#define mmCRTC_TRIGB_CNTL 0x1B96
+#define mmCRTC_TRIGB_MANUAL_TRIG 0x1B97
+#define mmCRTC_UPDATE_LOCK 0x1BB5
+#define mmCRTC_VBI_END 0x1B86
+#define mmCRTC_V_BLANK_START_END 0x1B8D
+#define mmCRTC_VERT_SYNC_CONTROL 0x1BAC
+#define mmCRTC_VGA_PARAMETER_CAPTURE_MODE 0x1BB7
+#define mmCRTC_V_SYNC_A 0x1B8E
+#define mmCRTC_V_SYNC_A_CNTL 0x1B8F
+#define mmCRTC_V_SYNC_B 0x1B90
+#define mmCRTC_V_SYNC_B_CNTL 0x1B91
+#define mmCRTC_VSYNC_NOM_INT_STATUS 0x1B8C
+#define mmCRTC_V_TOTAL 0x1B87
+#define mmCRTC_V_TOTAL_CONTROL 0x1B8A
+#define mmCRTC_V_TOTAL_INT_STATUS 0x1B8B
+#define mmCRTC_V_TOTAL_MAX 0x1B89
+#define mmCRTC_V_TOTAL_MIN 0x1B88
+#define mmCRTC_V_UPDATE_INT_STATUS 0x1BC4
+#define mmCUR_COLOR1 0x1A6C
+#define mmCUR_COLOR2 0x1A6D
+#define mmCUR_CONTROL 0x1A66
+#define mmCUR_HOT_SPOT 0x1A6B
+#define mmCUR_POSITION 0x1A6A
+#define mmCUR_REQUEST_FILTER_CNTL 0x1A99
+#define mmCUR_SIZE 0x1A68
+#define mmCUR_SURFACE_ADDRESS 0x1A67
+#define mmCUR_SURFACE_ADDRESS_HIGH 0x1A69
+#define mmCUR_UPDATE 0x1A6E
+#define mmD1VGA_CONTROL 0x00CC
+#define mmD2VGA_CONTROL 0x00CE
+#define mmD3VGA_CONTROL 0x00F8
+#define mmD4VGA_CONTROL 0x00F9
+#define mmD5VGA_CONTROL 0x00FA
+#define mmD6VGA_CONTROL 0x00FB
+#define mmDAC_AUTODETECT_CONTROL 0x19EE
+#define mmDAC_AUTODETECT_CONTROL2 0x19EF
+#define mmDAC_AUTODETECT_CONTROL3 0x19F0
+#define mmDAC_AUTODETECT_INT_CONTROL 0x19F2
+#define mmDAC_AUTODETECT_STATUS 0x19F1
+#define mmDAC_CLK_ENABLE 0x0128
+#define mmDAC_COMPARATOR_ENABLE 0x19F7
+#define mmDAC_COMPARATOR_OUTPUT 0x19F8
+#define mmDAC_CONTROL 0x19F6
+#define mmDAC_CRC_CONTROL 0x19E7
+#define mmDAC_CRC_EN 0x19E6
+#define mmDAC_CRC_SIG_CONTROL 0x19EB
+#define mmDAC_CRC_SIG_CONTROL_MASK 0x19E9
+#define mmDAC_CRC_SIG_RGB 0x19EA
+#define mmDAC_CRC_SIG_RGB_MASK 0x19E8
+#define mmDAC_DATA 0x00F2
+#define mmDAC_DFT_CONFIG 0x19FA
+#define mmDAC_ENABLE 0x19E4
+#define mmDAC_FIFO_STATUS 0x19FB
+#define mmDAC_FORCE_DATA 0x19F4
+#define mmDAC_FORCE_OUTPUT_CNTL 0x19F3
+#define mmDAC_MACRO_CNTL_RESERVED0 0x19FC
+#define mmDAC_MACRO_CNTL_RESERVED1 0x19FD
+#define mmDAC_MACRO_CNTL_RESERVED2 0x19FE
+#define mmDAC_MACRO_CNTL_RESERVED3 0x19FF
+#define mmDAC_MASK 0x00F1
+#define mmDAC_POWERDOWN 0x19F5
+#define mmDAC_PWR_CNTL 0x19F9
+#define mmDAC_R_INDEX 0x00F1
+#define mmDAC_SOURCE_SELECT 0x19E5
+#define mmDAC_STEREOSYNC_SELECT 0x19ED
+#define mmDAC_SYNC_TRISTATE_CONTROL 0x19EC
+#define mmDAC_W_INDEX 0x00F2
+#define mmDC_ABM1_ACE_CNTL_MISC 0x1641
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_0 0x163A
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_1 0x163B
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_2 0x163C
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_3 0x163D
+#define mmDC_ABM1_ACE_OFFSET_SLOPE_4 0x163E
+#define mmDC_ABM1_ACE_THRES_12 0x163F
+#define mmDC_ABM1_ACE_THRES_34 0x1640
+#define mmDC_ABM1_BL_MASTER_LOCK 0x169C
+#define mmDC_ABM1_CNTL 0x1638
+#define mmDC_ABM1_DEBUG_MISC 0x1649
+#define mmDC_ABM1_HG_BIN_1_32_SHIFT_FLAG 0x1656
+#define mmDC_ABM1_HG_BIN_17_24_SHIFT_INDEX 0x1659
+#define mmDC_ABM1_HG_BIN_1_8_SHIFT_INDEX 0x1657
+#define mmDC_ABM1_HG_BIN_25_32_SHIFT_INDEX 0x165A
+#define mmDC_ABM1_HG_BIN_9_16_SHIFT_INDEX 0x1658
+#define mmDC_ABM1_HGLS_REG_READ_PROGRESS 0x164A
+#define mmDC_ABM1_HG_MISC_CTRL 0x164B
+#define mmDC_ABM1_HG_RESULT_10 0x1664
+#define mmDC_ABM1_HG_RESULT_1 0x165B
+#define mmDC_ABM1_HG_RESULT_11 0x1665
+#define mmDC_ABM1_HG_RESULT_12 0x1666
+#define mmDC_ABM1_HG_RESULT_13 0x1667
+#define mmDC_ABM1_HG_RESULT_14 0x1668
+#define mmDC_ABM1_HG_RESULT_15 0x1669
+#define mmDC_ABM1_HG_RESULT_16 0x166A
+#define mmDC_ABM1_HG_RESULT_17 0x166B
+#define mmDC_ABM1_HG_RESULT_18 0x166C
+#define mmDC_ABM1_HG_RESULT_19 0x166D
+#define mmDC_ABM1_HG_RESULT_20 0x166E
+#define mmDC_ABM1_HG_RESULT_2 0x165C
+#define mmDC_ABM1_HG_RESULT_21 0x166F
+#define mmDC_ABM1_HG_RESULT_22 0x1670
+#define mmDC_ABM1_HG_RESULT_23 0x1671
+#define mmDC_ABM1_HG_RESULT_24 0x1672
+#define mmDC_ABM1_HG_RESULT_3 0x165D
+#define mmDC_ABM1_HG_RESULT_4 0x165E
+#define mmDC_ABM1_HG_RESULT_5 0x165F
+#define mmDC_ABM1_HG_RESULT_6 0x1660
+#define mmDC_ABM1_HG_RESULT_7 0x1661
+#define mmDC_ABM1_HG_RESULT_8 0x1662
+#define mmDC_ABM1_HG_RESULT_9 0x1663
+#define mmDC_ABM1_HG_SAMPLE_RATE 0x1654
+#define mmDC_ABM1_IPCSC_COEFF_SEL 0x1639
+#define mmDC_ABM1_LS_FILTERED_MIN_MAX_LUMA 0x164E
+#define mmDC_ABM1_LS_MAX_PIXEL_VALUE_COUNT 0x1653
+#define mmDC_ABM1_LS_MIN_MAX_LUMA 0x164D
+#define mmDC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES 0x1651
+#define mmDC_ABM1_LS_MIN_PIXEL_VALUE_COUNT 0x1652
+#define mmDC_ABM1_LS_OVR_SCAN_BIN 0x1650
+#define mmDC_ABM1_LS_PIXEL_COUNT 0x164F
+#define mmDC_ABM1_LS_SAMPLE_RATE 0x1655
+#define mmDC_ABM1_LS_SUM_OF_LUMA 0x164C
+#define mmDC_ABM1_OVERSCAN_PIXEL_VALUE 0x169B
+#define mmDCCG_AUDIO_DTO0_MODULE 0x016D
+#define mmDCCG_AUDIO_DTO0_PHASE 0x016C
+#define mmDCCG_AUDIO_DTO1_MODULE 0x0171
+#define mmDCCG_AUDIO_DTO1_PHASE 0x0170
+#define mmDCCG_AUDIO_DTO_SOURCE 0x016B
+#define mmDCCG_CAC_STATUS 0x0137
+#define mmDCCG_GATE_DISABLE_CNTL 0x0134
+#define mmDCCG_GTC_CNTL 0x0120
+#define mmDCCG_GTC_CURRENT 0x0123
+#define mmDCCG_GTC_DTO_MODULO 0x0122
+#define mmDCCG_PERFMON_CNTL 0x0133
+#define mmDCCG_PLL0_PLL_ANALOG 0x1708
+#define mmDCCG_PLL0_PLL_CNTL 0x1707
+#define mmDCCG_PLL0_PLL_DEBUG_CNTL 0x170B
+#define mmDCCG_PLL0_PLL_DISPCLK_CURRENT_DTO_PHASE 0x170F
+#define mmDCCG_PLL0_PLL_DISPCLK_DTO_CNTL 0x170E
+#define mmDCCG_PLL0_PLL_DS_CNTL 0x1705
+#define mmDCCG_PLL0_PLL_FB_DIV 0x1701
+#define mmDCCG_PLL0_PLL_IDCLK_CNTL 0x1706
+#define mmDCCG_PLL0_PLL_POST_DIV 0x1702
+#define mmDCCG_PLL0_PLL_REF_DIV 0x1700
+#define mmDCCG_PLL0_PLL_SS_AMOUNT_DSFRAC 0x1703
+#define mmDCCG_PLL0_PLL_SS_CNTL 0x1704
+#define mmDCCG_PLL0_PLL_UNLOCK_DETECT_CNTL 0x170A
+#define mmDCCG_PLL0_PLL_UPDATE_CNTL 0x170D
+#define mmDCCG_PLL0_PLL_UPDATE_LOCK 0x170C
+#define mmDCCG_PLL0_PLL_VREG_CNTL 0x1709
+#define mmDCCG_PLL1_PLL_ANALOG 0x1718
+#define mmDCCG_PLL1_PLL_CNTL 0x1717
+#define mmDCCG_PLL1_PLL_DEBUG_CNTL 0x171B
+#define mmDCCG_PLL1_PLL_DISPCLK_CURRENT_DTO_PHASE 0x171F
+#define mmDCCG_PLL1_PLL_DISPCLK_DTO_CNTL 0x171E
+#define mmDCCG_PLL1_PLL_DS_CNTL 0x1715
+#define mmDCCG_PLL1_PLL_FB_DIV 0x1711
+#define mmDCCG_PLL1_PLL_IDCLK_CNTL 0x1716
+#define mmDCCG_PLL1_PLL_POST_DIV 0x1712
+#define mmDCCG_PLL1_PLL_REF_DIV 0x1710
+#define mmDCCG_PLL1_PLL_SS_AMOUNT_DSFRAC 0x1713
+#define mmDCCG_PLL1_PLL_SS_CNTL 0x1714
+#define mmDCCG_PLL1_PLL_UNLOCK_DETECT_CNTL 0x171A
+#define mmDCCG_PLL1_PLL_UPDATE_CNTL 0x171D
+#define mmDCCG_PLL1_PLL_UPDATE_LOCK 0x171C
+#define mmDCCG_PLL1_PLL_VREG_CNTL 0x1719
+#define mmDCCG_PLL2_PLL_ANALOG 0x1728
+#define mmDCCG_PLL2_PLL_CNTL 0x1727
+#define mmDCCG_PLL2_PLL_DEBUG_CNTL 0x172B
+#define mmDCCG_PLL2_PLL_DISPCLK_CURRENT_DTO_PHASE 0x172F
+#define mmDCCG_PLL2_PLL_DISPCLK_DTO_CNTL 0x172E
+#define mmDCCG_PLL2_PLL_DS_CNTL 0x1725
+#define mmDCCG_PLL2_PLL_FB_DIV 0x1721
+#define mmDCCG_PLL2_PLL_IDCLK_CNTL 0x1726
+#define mmDCCG_PLL2_PLL_POST_DIV 0x1722
+#define mmDCCG_PLL2_PLL_REF_DIV 0x1720
+#define mmDCCG_PLL2_PLL_SS_AMOUNT_DSFRAC 0x1723
+#define mmDCCG_PLL2_PLL_SS_CNTL 0x1724
+#define mmDCCG_PLL2_PLL_UNLOCK_DETECT_CNTL 0x172A
+#define mmDCCG_PLL2_PLL_UPDATE_CNTL 0x172D
+#define mmDCCG_PLL2_PLL_UPDATE_LOCK 0x172C
+#define mmDCCG_PLL2_PLL_VREG_CNTL 0x1729
+#define mmDCCG_SOFT_RESET 0x015F
+#define mmDCCG_TEST_CLK_SEL 0x017E
+#define mmDCCG_TEST_DEBUG_DATA 0x017D
+#define mmDCCG_TEST_DEBUG_INDEX 0x017C
+#define mmDCCG_VPCLK_CNTL 0x031F
+#define mmDCDEBUG_BUS_CLK1_SEL 0x1860
+#define mmDCDEBUG_BUS_CLK2_SEL 0x1861
+#define mmDCDEBUG_BUS_CLK3_SEL 0x1862
+#define mmDCDEBUG_BUS_CLK4_SEL 0x1863
+#define mmDCDEBUG_OUT_CNTL 0x186B
+#define mmDCDEBUG_OUT_DATA 0x186E
+#define mmDCDEBUG_OUT_PIN_OVERRIDE 0x186A
+#define mmDC_DMCU_SCRATCH 0x1618
+#define mmDC_DVODATA_CONFIG 0x1905
+#define mmDCFE0_SOFT_RESET 0x0158
+#define mmDCFE1_SOFT_RESET 0x0159
+#define mmDCFE2_SOFT_RESET 0x015A
+#define mmDCFE3_SOFT_RESET 0x015B
+#define mmDCFE4_SOFT_RESET 0x015C
+#define mmDCFE5_SOFT_RESET 0x015D
+#define mmDCFE_DBG_SEL 0x1B7E
+#define mmDCFE_MEM_LIGHT_SLEEP_CNTL 0x1B7F
+#define mmDC_GENERICA 0x1900
+#define mmDC_GENERICB 0x1901
+#define mmDC_GPIO_DDC1_A 0x194D
+#define mmDC_GPIO_DDC1_EN 0x194E
+#define mmDC_GPIO_DDC1_MASK 0x194C
+#define mmDC_GPIO_DDC1_Y 0x194F
+#define mmDC_GPIO_DDC2_A 0x1951
+#define mmDC_GPIO_DDC2_EN 0x1952
+#define mmDC_GPIO_DDC2_MASK 0x1950
+#define mmDC_GPIO_DDC2_Y 0x1953
+#define mmDC_GPIO_DDC3_A 0x1955
+#define mmDC_GPIO_DDC3_EN 0x1956
+#define mmDC_GPIO_DDC3_MASK 0x1954
+#define mmDC_GPIO_DDC3_Y 0x1957
+#define mmDC_GPIO_DDC4_A 0x1959
+#define mmDC_GPIO_DDC4_EN 0x195A
+#define mmDC_GPIO_DDC4_MASK 0x1958
+#define mmDC_GPIO_DDC4_Y 0x195B
+#define mmDC_GPIO_DDC5_A 0x195D
+#define mmDC_GPIO_DDC5_EN 0x195E
+#define mmDC_GPIO_DDC5_MASK 0x195C
+#define mmDC_GPIO_DDC5_Y 0x195F
+#define mmDC_GPIO_DDC6_A 0x1961
+#define mmDC_GPIO_DDC6_EN 0x1962
+#define mmDC_GPIO_DDC6_MASK 0x1960
+#define mmDC_GPIO_DDC6_Y 0x1963
+#define mmDC_GPIO_DDCVGA_A 0x1971
+#define mmDC_GPIO_DDCVGA_EN 0x1972
+#define mmDC_GPIO_DDCVGA_MASK 0x1970
+#define mmDC_GPIO_DDCVGA_Y 0x1973
+#define mmDC_GPIO_DEBUG 0x1904
+#define mmDC_GPIO_DVODATA_A 0x1949
+#define mmDC_GPIO_DVODATA_EN 0x194A
+#define mmDC_GPIO_DVODATA_MASK 0x1948
+#define mmDC_GPIO_DVODATA_Y 0x194B
+#define mmDC_GPIO_GENERIC_A 0x1945
+#define mmDC_GPIO_GENERIC_EN 0x1946
+#define mmDC_GPIO_GENERIC_MASK 0x1944
+#define mmDC_GPIO_GENERIC_Y 0x1947
+#define mmDC_GPIO_GENLK_A 0x1969
+#define mmDC_GPIO_GENLK_EN 0x196A
+#define mmDC_GPIO_GENLK_MASK 0x1968
+#define mmDC_GPIO_GENLK_Y 0x196B
+#define mmDC_GPIO_HPD_A 0x196D
+#define mmDC_GPIO_HPD_EN 0x196E
+#define mmDC_GPIO_HPD_MASK 0x196C
+#define mmDC_GPIO_HPD_Y 0x196F
+#define mmDC_GPIO_I2CPAD_A 0x1975
+#define mmDC_GPIO_I2CPAD_EN 0x1976
+#define mmDC_GPIO_I2CPAD_MASK 0x1974
+#define mmDC_GPIO_I2CPAD_STRENGTH 0x197A
+#define mmDC_GPIO_I2CPAD_Y 0x1977
+#define mmDC_GPIO_PAD_STRENGTH_1 0x1978
+#define mmDC_GPIO_PAD_STRENGTH_2 0x1979
+#define mmDC_GPIO_PWRSEQ_A 0x1941
+#define mmDC_GPIO_PWRSEQ_EN 0x1942
+#define mmDC_GPIO_PWRSEQ_MASK 0x1940
+#define mmDC_GPIO_PWRSEQ_Y 0x1943
+#define mmDC_GPIO_SYNCA_A 0x1965
+#define mmDC_GPIO_SYNCA_EN 0x1966
+#define mmDC_GPIO_SYNCA_MASK 0x1964
+#define mmDC_GPIO_SYNCA_Y 0x1967
+#define mmDC_GPU_TIMER_READ 0x1929
+#define mmDC_GPU_TIMER_READ_CNTL 0x192A
+#define mmDC_GPU_TIMER_START_POSITION_P_FLIP 0x1928
+#define mmDC_GPU_TIMER_START_POSITION_V_UPDATE 0x1927
+#define mmDC_HPD1_CONTROL 0x1809
+#define mmDC_HPD1_FAST_TRAIN_CNTL 0x1864
+#define mmDC_HPD1_INT_CONTROL 0x1808
+#define mmDC_HPD1_INT_STATUS 0x1807
+#define mmDC_HPD1_TOGGLE_FILT_CNTL 0x18BC
+#define mmDC_HPD2_CONTROL 0x180C
+#define mmDC_HPD2_FAST_TRAIN_CNTL 0x1865
+#define mmDC_HPD2_INT_CONTROL 0x180B
+#define mmDC_HPD2_INT_STATUS 0x180A
+#define mmDC_HPD2_TOGGLE_FILT_CNTL 0x18BD
+#define mmDC_HPD3_CONTROL 0x180F
+#define mmDC_HPD3_FAST_TRAIN_CNTL 0x1866
+#define mmDC_HPD3_INT_CONTROL 0x180E
+#define mmDC_HPD3_INT_STATUS 0x180D
+#define mmDC_HPD3_TOGGLE_FILT_CNTL 0x18BE
+#define mmDC_HPD4_CONTROL 0x1812
+#define mmDC_HPD4_FAST_TRAIN_CNTL 0x1867
+#define mmDC_HPD4_INT_CONTROL 0x1811
+#define mmDC_HPD4_INT_STATUS 0x1810
+#define mmDC_HPD4_TOGGLE_FILT_CNTL 0x18FC
+#define mmDC_HPD5_CONTROL 0x1815
+#define mmDC_HPD5_FAST_TRAIN_CNTL 0x1868
+#define mmDC_HPD5_INT_CONTROL 0x1814
+#define mmDC_HPD5_INT_STATUS 0x1813
+#define mmDC_HPD5_TOGGLE_FILT_CNTL 0x18FD
+#define mmDC_HPD6_CONTROL 0x1818
+#define mmDC_HPD6_FAST_TRAIN_CNTL 0x1869
+#define mmDC_HPD6_INT_CONTROL 0x1817
+#define mmDC_HPD6_INT_STATUS 0x1816
+#define mmDC_HPD6_TOGGLE_FILT_CNTL 0x18FE
+#define mmDC_I2C_ARBITRATION 0x181A
+#define mmDC_I2C_CONTROL 0x1819
+#define mmDC_I2C_DATA 0x1833
+#define mmDC_I2C_DDC1_HW_STATUS 0x181D
+#define mmDC_I2C_DDC1_SETUP 0x1824
+#define mmDC_I2C_DDC1_SPEED 0x1823
+#define mmDC_I2C_DDC2_HW_STATUS 0x181E
+#define mmDC_I2C_DDC2_SETUP 0x1826
+#define mmDC_I2C_DDC2_SPEED 0x1825
+#define mmDC_I2C_DDC3_HW_STATUS 0x181F
+#define mmDC_I2C_DDC3_SETUP 0x1828
+#define mmDC_I2C_DDC3_SPEED 0x1827
+#define mmDC_I2C_DDC4_HW_STATUS 0x1820
+#define mmDC_I2C_DDC4_SETUP 0x182A
+#define mmDC_I2C_DDC4_SPEED 0x1829
+#define mmDC_I2C_DDC5_HW_STATUS 0x1821
+#define mmDC_I2C_DDC5_SETUP 0x182C
+#define mmDC_I2C_DDC5_SPEED 0x182B
+#define mmDC_I2C_DDC6_HW_STATUS 0x1822
+#define mmDC_I2C_DDC6_SETUP 0x182E
+#define mmDC_I2C_DDC6_SPEED 0x182D
+#define mmDC_I2C_DDCVGA_HW_STATUS 0x1855
+#define mmDC_I2C_DDCVGA_SETUP 0x1857
+#define mmDC_I2C_DDCVGA_SPEED 0x1856
+#define mmDC_I2C_EDID_DETECT_CTRL 0x186F
+#define mmDC_I2C_INTERRUPT_CONTROL 0x181B
+#define mmDC_I2C_SW_STATUS 0x181C
+#define mmDC_I2C_TRANSACTION0 0x182F
+#define mmDC_I2C_TRANSACTION1 0x1830
+#define mmDC_I2C_TRANSACTION2 0x1831
+#define mmDC_I2C_TRANSACTION3 0x1832
+#define mmDCI_CLK_CNTL 0x031E
+#define mmDCI_CLK_RAMP_CNTL 0x0324
+#define mmDCI_DEBUG_CONFIG 0x0323
+#define mmDCI_MEM_PWR_CNTL 0x0326
+#define mmDCI_MEM_PWR_STATE 0x031B
+#define mmDCI_MEM_PWR_STATE2 0x0322
+#define mmDCIO_DEBUG 0x192E
+#define mmDCIO_GSL0_CNTL 0x1924
+#define mmDCIO_GSL1_CNTL 0x1925
+#define mmDCIO_GSL2_CNTL 0x1926
+#define mmDCIO_GSL_GENLK_PAD_CNTL 0x1922
+#define mmDCIO_GSL_SWAPLOCK_PAD_CNTL 0x1923
+#define mmDCIO_IMPCAL_CNTL_AB 0x190D
+#define mmDCIO_IMPCAL_CNTL_CD 0x1911
+#define mmDCIO_IMPCAL_CNTL_EF 0x1915
+#define mmDCIO_TEST_DEBUG_DATA 0x1930
+#define mmDCIO_TEST_DEBUG_INDEX 0x192F
+#define mmDCIO_UNIPHY0_UNIPHY_ANG_BIST_CNTL 0x198C
+#define mmDCIO_UNIPHY0_UNIPHY_CHANNEL_XBAR_CNTL 0x198E
+#define mmDCIO_UNIPHY0_UNIPHY_DATA_SYNCHRONIZATION 0x198A
+#define mmDCIO_UNIPHY0_UNIPHY_LINK_CNTL 0x198D
+#define mmDCIO_UNIPHY0_UNIPHY_PLL_CONTROL1 0x1986
+#define mmDCIO_UNIPHY0_UNIPHY_PLL_CONTROL2 0x1987
+#define mmDCIO_UNIPHY0_UNIPHY_PLL_FBDIV 0x1985
+#define mmDCIO_UNIPHY0_UNIPHY_PLL_SS_CNTL 0x1989
+#define mmDCIO_UNIPHY0_UNIPHY_PLL_SS_STEP_SIZE 0x1988
+#define mmDCIO_UNIPHY0_UNIPHY_POWER_CONTROL 0x1984
+#define mmDCIO_UNIPHY0_UNIPHY_REG_TEST_OUTPUT 0x198B
+#define mmDCIO_UNIPHY0_UNIPHY_TX_CONTROL1 0x1980
+#define mmDCIO_UNIPHY0_UNIPHY_TX_CONTROL2 0x1981
+#define mmDCIO_UNIPHY0_UNIPHY_TX_CONTROL3 0x1982
+#define mmDCIO_UNIPHY0_UNIPHY_TX_CONTROL4 0x1983
+#define mmDCIO_UNIPHY1_UNIPHY_ANG_BIST_CNTL 0x199C
+#define mmDCIO_UNIPHY1_UNIPHY_CHANNEL_XBAR_CNTL 0x199E
+#define mmDCIO_UNIPHY1_UNIPHY_DATA_SYNCHRONIZATION 0x199A
+#define mmDCIO_UNIPHY1_UNIPHY_LINK_CNTL 0x199D
+#define mmDCIO_UNIPHY1_UNIPHY_PLL_CONTROL1 0x1996
+#define mmDCIO_UNIPHY1_UNIPHY_PLL_CONTROL2 0x1997
+#define mmDCIO_UNIPHY1_UNIPHY_PLL_FBDIV 0x1995
+#define mmDCIO_UNIPHY1_UNIPHY_PLL_SS_CNTL 0x1999
+#define mmDCIO_UNIPHY1_UNIPHY_PLL_SS_STEP_SIZE 0x1998
+#define mmDCIO_UNIPHY1_UNIPHY_POWER_CONTROL 0x1994
+#define mmDCIO_UNIPHY1_UNIPHY_REG_TEST_OUTPUT 0x199B
+#define mmDCIO_UNIPHY1_UNIPHY_TX_CONTROL1 0x1990
+#define mmDCIO_UNIPHY1_UNIPHY_TX_CONTROL2 0x1991
+#define mmDCIO_UNIPHY1_UNIPHY_TX_CONTROL3 0x1992
+#define mmDCIO_UNIPHY1_UNIPHY_TX_CONTROL4 0x1993
+#define mmDCIO_UNIPHY2_UNIPHY_ANG_BIST_CNTL 0x19AC
+#define mmDCIO_UNIPHY2_UNIPHY_CHANNEL_XBAR_CNTL 0x19AE
+#define mmDCIO_UNIPHY2_UNIPHY_DATA_SYNCHRONIZATION 0x19AA
+#define mmDCIO_UNIPHY2_UNIPHY_LINK_CNTL 0x19AD
+#define mmDCIO_UNIPHY2_UNIPHY_PLL_CONTROL1 0x19A6
+#define mmDCIO_UNIPHY2_UNIPHY_PLL_CONTROL2 0x19A7
+#define mmDCIO_UNIPHY2_UNIPHY_PLL_FBDIV 0x19A5
+#define mmDCIO_UNIPHY2_UNIPHY_PLL_SS_CNTL 0x19A9
+#define mmDCIO_UNIPHY2_UNIPHY_PLL_SS_STEP_SIZE 0x19A8
+#define mmDCIO_UNIPHY2_UNIPHY_POWER_CONTROL 0x19A4
+#define mmDCIO_UNIPHY2_UNIPHY_REG_TEST_OUTPUT 0x19AB
+#define mmDCIO_UNIPHY2_UNIPHY_TX_CONTROL1 0x19A0
+#define mmDCIO_UNIPHY2_UNIPHY_TX_CONTROL2 0x19A1
+#define mmDCIO_UNIPHY2_UNIPHY_TX_CONTROL3 0x19A2
+#define mmDCIO_UNIPHY2_UNIPHY_TX_CONTROL4 0x19A3
+#define mmDCIO_UNIPHY3_UNIPHY_ANG_BIST_CNTL 0x19BC
+#define mmDCIO_UNIPHY3_UNIPHY_CHANNEL_XBAR_CNTL 0x19BE
+#define mmDCIO_UNIPHY3_UNIPHY_DATA_SYNCHRONIZATION 0x19BA
+#define mmDCIO_UNIPHY3_UNIPHY_LINK_CNTL 0x19BD
+#define mmDCIO_UNIPHY3_UNIPHY_PLL_CONTROL1 0x19B6
+#define mmDCIO_UNIPHY3_UNIPHY_PLL_CONTROL2 0x19B7
+#define mmDCIO_UNIPHY3_UNIPHY_PLL_FBDIV 0x19B5
+#define mmDCIO_UNIPHY3_UNIPHY_PLL_SS_CNTL 0x19B9
+#define mmDCIO_UNIPHY3_UNIPHY_PLL_SS_STEP_SIZE 0x19B8
+#define mmDCIO_UNIPHY3_UNIPHY_POWER_CONTROL 0x19B4
+#define mmDCIO_UNIPHY3_UNIPHY_REG_TEST_OUTPUT 0x19BB
+#define mmDCIO_UNIPHY3_UNIPHY_TX_CONTROL1 0x19B0
+#define mmDCIO_UNIPHY3_UNIPHY_TX_CONTROL2 0x19B1
+#define mmDCIO_UNIPHY3_UNIPHY_TX_CONTROL3 0x19B2
+#define mmDCIO_UNIPHY3_UNIPHY_TX_CONTROL4 0x19B3
+#define mmDCIO_UNIPHY4_UNIPHY_ANG_BIST_CNTL 0x19CC
+#define mmDCIO_UNIPHY4_UNIPHY_CHANNEL_XBAR_CNTL 0x19CE
+#define mmDCIO_UNIPHY4_UNIPHY_DATA_SYNCHRONIZATION 0x19CA
+#define mmDCIO_UNIPHY4_UNIPHY_LINK_CNTL 0x19CD
+#define mmDCIO_UNIPHY4_UNIPHY_PLL_CONTROL1 0x19C6
+#define mmDCIO_UNIPHY4_UNIPHY_PLL_CONTROL2 0x19C7
+#define mmDCIO_UNIPHY4_UNIPHY_PLL_FBDIV 0x19C5
+#define mmDCIO_UNIPHY4_UNIPHY_PLL_SS_CNTL 0x19C9
+#define mmDCIO_UNIPHY4_UNIPHY_PLL_SS_STEP_SIZE 0x19C8
+#define mmDCIO_UNIPHY4_UNIPHY_POWER_CONTROL 0x19C4
+#define mmDCIO_UNIPHY4_UNIPHY_REG_TEST_OUTPUT 0x19CB
+#define mmDCIO_UNIPHY4_UNIPHY_TX_CONTROL1 0x19C0
+#define mmDCIO_UNIPHY4_UNIPHY_TX_CONTROL2 0x19C1
+#define mmDCIO_UNIPHY4_UNIPHY_TX_CONTROL3 0x19C2
+#define mmDCIO_UNIPHY4_UNIPHY_TX_CONTROL4 0x19C3
+#define mmDCIO_UNIPHY5_UNIPHY_ANG_BIST_CNTL 0x19DC
+#define mmDCIO_UNIPHY5_UNIPHY_CHANNEL_XBAR_CNTL 0x19DE
+#define mmDCIO_UNIPHY5_UNIPHY_DATA_SYNCHRONIZATION 0x19DA
+#define mmDCIO_UNIPHY5_UNIPHY_LINK_CNTL 0x19DD
+#define mmDCIO_UNIPHY5_UNIPHY_PLL_CONTROL1 0x19D6
+#define mmDCIO_UNIPHY5_UNIPHY_PLL_CONTROL2 0x19D7
+#define mmDCIO_UNIPHY5_UNIPHY_PLL_FBDIV 0x19D5
+#define mmDCIO_UNIPHY5_UNIPHY_PLL_SS_CNTL 0x19D9
+#define mmDCIO_UNIPHY5_UNIPHY_PLL_SS_STEP_SIZE 0x19D8
+#define mmDCIO_UNIPHY5_UNIPHY_POWER_CONTROL 0x19D4
+#define mmDCIO_UNIPHY5_UNIPHY_REG_TEST_OUTPUT 0x19DB
+#define mmDCIO_UNIPHY5_UNIPHY_TX_CONTROL1 0x19D0
+#define mmDCIO_UNIPHY5_UNIPHY_TX_CONTROL2 0x19D1
+#define mmDCIO_UNIPHY5_UNIPHY_TX_CONTROL3 0x19D2
+#define mmDCIO_UNIPHY5_UNIPHY_TX_CONTROL4 0x19D3
+#define mmDCI_SOFT_RESET 0x015E
+#define mmDCI_TEST_DEBUG_DATA 0x0321
+#define mmDCI_TEST_DEBUG_INDEX 0x0320
+#define mmDC_LUT_30_COLOR 0x1A7C
+#define mmDC_LUT_AUTOFILL 0x1A7F
+#define mmDC_LUT_BLACK_OFFSET_BLUE 0x1A81
+#define mmDC_LUT_BLACK_OFFSET_GREEN 0x1A82
+#define mmDC_LUT_BLACK_OFFSET_RED 0x1A83
+#define mmDC_LUT_CONTROL 0x1A80
+#define mmDC_LUT_PWL_DATA 0x1A7B
+#define mmDC_LUT_RW_INDEX 0x1A79
+#define mmDC_LUT_RW_MODE 0x1A78
+#define mmDC_LUT_SEQ_COLOR 0x1A7A
+#define mmDC_LUT_VGA_ACCESS_ENABLE 0x1A7D
+#define mmDC_LUT_WHITE_OFFSET_BLUE 0x1A84
+#define mmDC_LUT_WHITE_OFFSET_GREEN 0x1A85
+#define mmDC_LUT_WHITE_OFFSET_RED 0x1A86
+#define mmDC_LUT_WRITE_EN_MASK 0x1A7E
+#define mmDC_MVP_LB_CONTROL 0x1ADB
+#define mmDCO_CLK_CNTL 0x192B
+#define mmDCO_CLK_RAMP_CNTL 0x192C
+#define mmDCO_LIGHT_SLEEP_DIS 0x1907
+#define mmDCO_MEM_POWER_STATE 0x1906
+#define mmDCO_SOFT_RESET 0x0167
+#define mmDCP0_COMM_MATRIXA_TRANS_C11_C12 0x1A43
+#define mmDCP0_COMM_MATRIXA_TRANS_C13_C14 0x1A44
+#define mmDCP0_COMM_MATRIXA_TRANS_C21_C22 0x1A45
+#define mmDCP0_COMM_MATRIXA_TRANS_C23_C24 0x1A46
+#define mmDCP0_COMM_MATRIXA_TRANS_C31_C32 0x1A47
+#define mmDCP0_COMM_MATRIXA_TRANS_C33_C34 0x1A48
+#define mmDCP0_COMM_MATRIXB_TRANS_C11_C12 0x1A49
+#define mmDCP0_COMM_MATRIXB_TRANS_C13_C14 0x1A4A
+#define mmDCP0_COMM_MATRIXB_TRANS_C21_C22 0x1A4B
+#define mmDCP0_COMM_MATRIXB_TRANS_C23_C24 0x1A4C
+#define mmDCP0_COMM_MATRIXB_TRANS_C31_C32 0x1A4D
+#define mmDCP0_COMM_MATRIXB_TRANS_C33_C34 0x1A4E
+#define mmDCP0_CUR_COLOR1 0x1A6C
+#define mmDCP0_CUR_COLOR2 0x1A6D
+#define mmDCP0_CUR_CONTROL 0x1A66
+#define mmDCP0_CUR_HOT_SPOT 0x1A6B
+#define mmDCP0_CUR_POSITION 0x1A6A
+#define mmDCP0_CUR_REQUEST_FILTER_CNTL 0x1A99
+#define mmDCP0_CUR_SIZE 0x1A68
+#define mmDCP0_CUR_SURFACE_ADDRESS 0x1A67
+#define mmDCP0_CUR_SURFACE_ADDRESS_HIGH 0x1A69
+#define mmDCP0_CUR_UPDATE 0x1A6E
+#define mmDCP0_DC_LUT_30_COLOR 0x1A7C
+#define mmDCP0_DC_LUT_AUTOFILL 0x1A7F
+#define mmDCP0_DC_LUT_BLACK_OFFSET_BLUE 0x1A81
+#define mmDCP0_DC_LUT_BLACK_OFFSET_GREEN 0x1A82
+#define mmDCP0_DC_LUT_BLACK_OFFSET_RED 0x1A83
+#define mmDCP0_DC_LUT_CONTROL 0x1A80
+#define mmDCP0_DC_LUT_PWL_DATA 0x1A7B
+#define mmDCP0_DC_LUT_RW_INDEX 0x1A79
+#define mmDCP0_DC_LUT_RW_MODE 0x1A78
+#define mmDCP0_DC_LUT_SEQ_COLOR 0x1A7A
+#define mmDCP0_DC_LUT_VGA_ACCESS_ENABLE 0x1A7D
+#define mmDCP0_DC_LUT_WHITE_OFFSET_BLUE 0x1A84
+#define mmDCP0_DC_LUT_WHITE_OFFSET_GREEN 0x1A85
+#define mmDCP0_DC_LUT_WHITE_OFFSET_RED 0x1A86
+#define mmDCP0_DC_LUT_WRITE_EN_MASK 0x1A7E
+#define mmDCP0_DCP_CRC_CONTROL 0x1A87
+#define mmDCP0_DCP_CRC_CURRENT 0x1A89
+#define mmDCP0_DCP_CRC_LAST 0x1A8B
+#define mmDCP0_DCP_CRC_MASK 0x1A88
+#define mmDCP0_DCP_DEBUG 0x1A8D
+#define mmDCP0_DCP_DEBUG2 0x1A98
+#define mmDCP0_DCP_FP_CONVERTED_FIELD 0x1A65
+#define mmDCP0_DCP_GSL_CONTROL 0x1A90
+#define mmDCP0_DCP_LB_DATA_GAP_BETWEEN_CHUNK 0x1A91
+#define mmDCP0_DCP_RANDOM_SEEDS 0x1A61
+#define mmDCP0_DCP_SPATIAL_DITHER_CNTL 0x1A60
+#define mmDCP0_DCP_TEST_DEBUG_DATA 0x1A96
+#define mmDCP0_DCP_TEST_DEBUG_INDEX 0x1A95
+#define mmDCP0_DEGAMMA_CONTROL 0x1A58
+#define mmDCP0_DENORM_CONTROL 0x1A50
+#define mmDCP0_GAMUT_REMAP_C11_C12 0x1A5A
+#define mmDCP0_GAMUT_REMAP_C13_C14 0x1A5B
+#define mmDCP0_GAMUT_REMAP_C21_C22 0x1A5C
+#define mmDCP0_GAMUT_REMAP_C23_C24 0x1A5D
+#define mmDCP0_GAMUT_REMAP_C31_C32 0x1A5E
+#define mmDCP0_GAMUT_REMAP_C33_C34 0x1A5F
+#define mmDCP0_GAMUT_REMAP_CONTROL 0x1A59
+#define mmDCP0_GRPH_COMPRESS_PITCH 0x1A1A
+#define mmDCP0_GRPH_COMPRESS_SURFACE_ADDRESS 0x1A19
+#define mmDCP0_GRPH_COMPRESS_SURFACE_ADDRESS_HIGH 0x1A1B
+#define mmDCP0_GRPH_CONTROL 0x1A01
+#define mmDCP0_GRPH_DFQ_CONTROL 0x1A14
+#define mmDCP0_GRPH_DFQ_STATUS 0x1A15
+#define mmDCP0_GRPH_ENABLE 0x1A00
+#define mmDCP0_GRPH_FLIP_CONTROL 0x1A12
+#define mmDCP0_GRPH_INTERRUPT_CONTROL 0x1A17
+#define mmDCP0_GRPH_INTERRUPT_STATUS 0x1A16
+#define mmDCP0_GRPH_LUT_10BIT_BYPASS 0x1A02
+#define mmDCP0_GRPH_PITCH 0x1A06
+#define mmDCP0_GRPH_PRIMARY_SURFACE_ADDRESS 0x1A04
+#define mmDCP0_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH 0x1A07
+#define mmDCP0_GRPH_SECONDARY_SURFACE_ADDRESS 0x1A05
+#define mmDCP0_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH 0x1A08
+#define mmDCP0_GRPH_STEREOSYNC_FLIP 0x1A97
+#define mmDCP0_GRPH_SURFACE_ADDRESS_HIGH_INUSE 0x1A18
+#define mmDCP0_GRPH_SURFACE_ADDRESS_INUSE 0x1A13
+#define mmDCP0_GRPH_SURFACE_OFFSET_X 0x1A09
+#define mmDCP0_GRPH_SURFACE_OFFSET_Y 0x1A0A
+#define mmDCP0_GRPH_SWAP_CNTL 0x1A03
+#define mmDCP0_GRPH_UPDATE 0x1A11
+#define mmDCP0_GRPH_X_END 0x1A0D
+#define mmDCP0_GRPH_X_START 0x1A0B
+#define mmDCP0_GRPH_Y_END 0x1A0E
+#define mmDCP0_GRPH_Y_START 0x1A0C
+#define mmDCP0_INPUT_CSC_C11_C12 0x1A36
+#define mmDCP0_INPUT_CSC_C13_C14 0x1A37
+#define mmDCP0_INPUT_CSC_C21_C22 0x1A38
+#define mmDCP0_INPUT_CSC_C23_C24 0x1A39
+#define mmDCP0_INPUT_CSC_C31_C32 0x1A3A
+#define mmDCP0_INPUT_CSC_C33_C34 0x1A3B
+#define mmDCP0_INPUT_CSC_CONTROL 0x1A35
+#define mmDCP0_INPUT_GAMMA_CONTROL 0x1A10
+#define mmDCP0_KEY_CONTROL 0x1A53
+#define mmDCP0_KEY_RANGE_ALPHA 0x1A54
+#define mmDCP0_KEY_RANGE_BLUE 0x1A57
+#define mmDCP0_KEY_RANGE_GREEN 0x1A56
+#define mmDCP0_KEY_RANGE_RED 0x1A55
+#define mmDCP0_OUTPUT_CSC_C11_C12 0x1A3D
+#define mmDCP0_OUTPUT_CSC_C13_C14 0x1A3E
+#define mmDCP0_OUTPUT_CSC_C21_C22 0x1A3F
+#define mmDCP0_OUTPUT_CSC_C23_C24 0x1A40
+#define mmDCP0_OUTPUT_CSC_C31_C32 0x1A41
+#define mmDCP0_OUTPUT_CSC_C33_C34 0x1A42
+#define mmDCP0_OUTPUT_CSC_CONTROL 0x1A3C
+#define mmDCP0_OUT_ROUND_CONTROL 0x1A51
+#define mmDCP0_OVL_CONTROL1 0x1A1D
+#define mmDCP0_OVL_CONTROL2 0x1A1E
+#define mmDCP0_OVL_DFQ_CONTROL 0x1A29
+#define mmDCP0_OVL_DFQ_STATUS 0x1A2A
+#define mmDCP0_OVL_ENABLE 0x1A1C
+#define mmDCP0_OVL_END 0x1A26
+#define mmDCP0_OVL_PITCH 0x1A21
+#define mmDCP0_OVLSCL_EDGE_PIXEL_CNTL 0x1A2C
+#define mmDCP0_OVL_SECONDARY_SURFACE_ADDRESS 0x1A92
+#define mmDCP0_OVL_SECONDARY_SURFACE_ADDRESS_HIGH 0x1A94
+#define mmDCP0_OVL_START 0x1A25
+#define mmDCP0_OVL_STEREOSYNC_FLIP 0x1A93
+#define mmDCP0_OVL_SURFACE_ADDRESS 0x1A20
+#define mmDCP0_OVL_SURFACE_ADDRESS_HIGH 0x1A22
+#define mmDCP0_OVL_SURFACE_ADDRESS_HIGH_INUSE 0x1A2B
+#define mmDCP0_OVL_SURFACE_ADDRESS_INUSE 0x1A28
+#define mmDCP0_OVL_SURFACE_OFFSET_X 0x1A23
+#define mmDCP0_OVL_SURFACE_OFFSET_Y 0x1A24
+#define mmDCP0_OVL_SWAP_CNTL 0x1A1F
+#define mmDCP0_OVL_UPDATE 0x1A27
+#define mmDCP0_PRESCALE_GRPH_CONTROL 0x1A2D
+#define mmDCP0_PRESCALE_OVL_CONTROL 0x1A31
+#define mmDCP0_PRESCALE_VALUES_GRPH_B 0x1A30
+#define mmDCP0_PRESCALE_VALUES_GRPH_G 0x1A2F
+#define mmDCP0_PRESCALE_VALUES_GRPH_R 0x1A2E
+#define mmDCP0_PRESCALE_VALUES_OVL_CB 0x1A32
+#define mmDCP0_PRESCALE_VALUES_OVL_CR 0x1A34
+#define mmDCP0_PRESCALE_VALUES_OVL_Y 0x1A33
+#define mmDCP0_REGAMMA_CNTLA_END_CNTL1 0x1AA6
+#define mmDCP0_REGAMMA_CNTLA_END_CNTL2 0x1AA7
+#define mmDCP0_REGAMMA_CNTLA_REGION_0_1 0x1AA8
+#define mmDCP0_REGAMMA_CNTLA_REGION_10_11 0x1AAD
+#define mmDCP0_REGAMMA_CNTLA_REGION_12_13 0x1AAE
+#define mmDCP0_REGAMMA_CNTLA_REGION_14_15 0x1AAF
+#define mmDCP0_REGAMMA_CNTLA_REGION_2_3 0x1AA9
+#define mmDCP0_REGAMMA_CNTLA_REGION_4_5 0x1AAA
+#define mmDCP0_REGAMMA_CNTLA_REGION_6_7 0x1AAB
+#define mmDCP0_REGAMMA_CNTLA_REGION_8_9 0x1AAC
+#define mmDCP0_REGAMMA_CNTLA_SLOPE_CNTL 0x1AA5
+#define mmDCP0_REGAMMA_CNTLA_START_CNTL 0x1AA4
+#define mmDCP0_REGAMMA_CNTLB_END_CNTL1 0x1AB2
+#define mmDCP0_REGAMMA_CNTLB_END_CNTL2 0x1AB3
+#define mmDCP0_REGAMMA_CNTLB_REGION_0_1 0x1AB4
+#define mmDCP0_REGAMMA_CNTLB_REGION_10_11 0x1AB9
+#define mmDCP0_REGAMMA_CNTLB_REGION_12_13 0x1ABA
+#define mmDCP0_REGAMMA_CNTLB_REGION_14_15 0x1ABB
+#define mmDCP0_REGAMMA_CNTLB_REGION_2_3 0x1AB5
+#define mmDCP0_REGAMMA_CNTLB_REGION_4_5 0x1AB6
+#define mmDCP0_REGAMMA_CNTLB_REGION_6_7 0x1AB7
+#define mmDCP0_REGAMMA_CNTLB_REGION_8_9 0x1AB8
+#define mmDCP0_REGAMMA_CNTLB_SLOPE_CNTL 0x1AB1
+#define mmDCP0_REGAMMA_CNTLB_START_CNTL 0x1AB0
+#define mmDCP0_REGAMMA_CONTROL 0x1AA0
+#define mmDCP0_REGAMMA_LUT_DATA 0x1AA2
+#define mmDCP0_REGAMMA_LUT_INDEX 0x1AA1
+#define mmDCP0_REGAMMA_LUT_WRITE_EN_MASK 0x1AA3
+#define mmDCP1_COMM_MATRIXA_TRANS_C11_C12 0x1D43
+#define mmDCP1_COMM_MATRIXA_TRANS_C13_C14 0x1D44
+#define mmDCP1_COMM_MATRIXA_TRANS_C21_C22 0x1D45
+#define mmDCP1_COMM_MATRIXA_TRANS_C23_C24 0x1D46
+#define mmDCP1_COMM_MATRIXA_TRANS_C31_C32 0x1D47
+#define mmDCP1_COMM_MATRIXA_TRANS_C33_C34 0x1D48
+#define mmDCP1_COMM_MATRIXB_TRANS_C11_C12 0x1D49
+#define mmDCP1_COMM_MATRIXB_TRANS_C13_C14 0x1D4A
+#define mmDCP1_COMM_MATRIXB_TRANS_C21_C22 0x1D4B
+#define mmDCP1_COMM_MATRIXB_TRANS_C23_C24 0x1D4C
+#define mmDCP1_COMM_MATRIXB_TRANS_C31_C32 0x1D4D
+#define mmDCP1_COMM_MATRIXB_TRANS_C33_C34 0x1D4E
+#define mmDCP1_CUR_COLOR1 0x1D6C
+#define mmDCP1_CUR_COLOR2 0x1D6D
+#define mmDCP1_CUR_CONTROL 0x1D66
+#define mmDCP1_CUR_HOT_SPOT 0x1D6B
+#define mmDCP1_CUR_POSITION 0x1D6A
+#define mmDCP1_CUR_REQUEST_FILTER_CNTL 0x1D99
+#define mmDCP1_CUR_SIZE 0x1D68
+#define mmDCP1_CUR_SURFACE_ADDRESS 0x1D67
+#define mmDCP1_CUR_SURFACE_ADDRESS_HIGH 0x1D69
+#define mmDCP1_CUR_UPDATE 0x1D6E
+#define mmDCP1_DC_LUT_30_COLOR 0x1D7C
+#define mmDCP1_DC_LUT_AUTOFILL 0x1D7F
+#define mmDCP1_DC_LUT_BLACK_OFFSET_BLUE 0x1D81
+#define mmDCP1_DC_LUT_BLACK_OFFSET_GREEN 0x1D82
+#define mmDCP1_DC_LUT_BLACK_OFFSET_RED 0x1D83
+#define mmDCP1_DC_LUT_CONTROL 0x1D80
+#define mmDCP1_DC_LUT_PWL_DATA 0x1D7B
+#define mmDCP1_DC_LUT_RW_INDEX 0x1D79
+#define mmDCP1_DC_LUT_RW_MODE 0x1D78
+#define mmDCP1_DC_LUT_SEQ_COLOR 0x1D7A
+#define mmDCP1_DC_LUT_VGA_ACCESS_ENABLE 0x1D7D
+#define mmDCP1_DC_LUT_WHITE_OFFSET_BLUE 0x1D84
+#define mmDCP1_DC_LUT_WHITE_OFFSET_GREEN 0x1D85
+#define mmDCP1_DC_LUT_WHITE_OFFSET_RED 0x1D86
+#define mmDCP1_DC_LUT_WRITE_EN_MASK 0x1D7E
+#define mmDCP1_DCP_CRC_CONTROL 0x1D87
+#define mmDCP1_DCP_CRC_CURRENT 0x1D89
+#define mmDCP1_DCP_CRC_LAST 0x1D8B
+#define mmDCP1_DCP_CRC_MASK 0x1D88
+#define mmDCP1_DCP_DEBUG 0x1D8D
+#define mmDCP1_DCP_DEBUG2 0x1D98
+#define mmDCP1_DCP_FP_CONVERTED_FIELD 0x1D65
+#define mmDCP1_DCP_GSL_CONTROL 0x1D90
+#define mmDCP1_DCP_LB_DATA_GAP_BETWEEN_CHUNK 0x1D91
+#define mmDCP1_DCP_RANDOM_SEEDS 0x1D61
+#define mmDCP1_DCP_SPATIAL_DITHER_CNTL 0x1D60
+#define mmDCP1_DCP_TEST_DEBUG_DATA 0x1D96
+#define mmDCP1_DCP_TEST_DEBUG_INDEX 0x1D95
+#define mmDCP1_DEGAMMA_CONTROL 0x1D58
+#define mmDCP1_DENORM_CONTROL 0x1D50
+#define mmDCP1_GAMUT_REMAP_C11_C12 0x1D5A
+#define mmDCP1_GAMUT_REMAP_C13_C14 0x1D5B
+#define mmDCP1_GAMUT_REMAP_C21_C22 0x1D5C
+#define mmDCP1_GAMUT_REMAP_C23_C24 0x1D5D
+#define mmDCP1_GAMUT_REMAP_C31_C32 0x1D5E
+#define mmDCP1_GAMUT_REMAP_C33_C34 0x1D5F
+#define mmDCP1_GAMUT_REMAP_CONTROL 0x1D59
+#define mmDCP1_GRPH_COMPRESS_PITCH 0x1D1A
+#define mmDCP1_GRPH_COMPRESS_SURFACE_ADDRESS 0x1D19
+#define mmDCP1_GRPH_COMPRESS_SURFACE_ADDRESS_HIGH 0x1D1B
+#define mmDCP1_GRPH_CONTROL 0x1D01
+#define mmDCP1_GRPH_DFQ_CONTROL 0x1D14
+#define mmDCP1_GRPH_DFQ_STATUS 0x1D15
+#define mmDCP1_GRPH_ENABLE 0x1D00
+#define mmDCP1_GRPH_FLIP_CONTROL 0x1D12
+#define mmDCP1_GRPH_INTERRUPT_CONTROL 0x1D17
+#define mmDCP1_GRPH_INTERRUPT_STATUS 0x1D16
+#define mmDCP1_GRPH_LUT_10BIT_BYPASS 0x1D02
+#define mmDCP1_GRPH_PITCH 0x1D06
+#define mmDCP1_GRPH_PRIMARY_SURFACE_ADDRESS 0x1D04
+#define mmDCP1_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH 0x1D07
+#define mmDCP1_GRPH_SECONDARY_SURFACE_ADDRESS 0x1D05
+#define mmDCP1_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH 0x1D08
+#define mmDCP1_GRPH_STEREOSYNC_FLIP 0x1D97
+#define mmDCP1_GRPH_SURFACE_ADDRESS_HIGH_INUSE 0x1D18
+#define mmDCP1_GRPH_SURFACE_ADDRESS_INUSE 0x1D13
+#define mmDCP1_GRPH_SURFACE_OFFSET_X 0x1D09
+#define mmDCP1_GRPH_SURFACE_OFFSET_Y 0x1D0A
+#define mmDCP1_GRPH_SWAP_CNTL 0x1D03
+#define mmDCP1_GRPH_UPDATE 0x1D11
+#define mmDCP1_GRPH_X_END 0x1D0D
+#define mmDCP1_GRPH_X_START 0x1D0B
+#define mmDCP1_GRPH_Y_END 0x1D0E
+#define mmDCP1_GRPH_Y_START 0x1D0C
+#define mmDCP1_INPUT_CSC_C11_C12 0x1D36
+#define mmDCP1_INPUT_CSC_C13_C14 0x1D37
+#define mmDCP1_INPUT_CSC_C21_C22 0x1D38
+#define mmDCP1_INPUT_CSC_C23_C24 0x1D39
+#define mmDCP1_INPUT_CSC_C31_C32 0x1D3A
+#define mmDCP1_INPUT_CSC_C33_C34 0x1D3B
+#define mmDCP1_INPUT_CSC_CONTROL 0x1D35
+#define mmDCP1_INPUT_GAMMA_CONTROL 0x1D10
+#define mmDCP1_KEY_CONTROL 0x1D53
+#define mmDCP1_KEY_RANGE_ALPHA 0x1D54
+#define mmDCP1_KEY_RANGE_BLUE 0x1D57
+#define mmDCP1_KEY_RANGE_GREEN 0x1D56
+#define mmDCP1_KEY_RANGE_RED 0x1D55
+#define mmDCP1_OUTPUT_CSC_C11_C12 0x1D3D
+#define mmDCP1_OUTPUT_CSC_C13_C14 0x1D3E
+#define mmDCP1_OUTPUT_CSC_C21_C22 0x1D3F
+#define mmDCP1_OUTPUT_CSC_C23_C24 0x1D40
+#define mmDCP1_OUTPUT_CSC_C31_C32 0x1D41
+#define mmDCP1_OUTPUT_CSC_C33_C34 0x1D42
+#define mmDCP1_OUTPUT_CSC_CONTROL 0x1D3C
+#define mmDCP1_OUT_ROUND_CONTROL 0x1D51
+#define mmDCP1_OVL_CONTROL1 0x1D1D
+#define mmDCP1_OVL_CONTROL2 0x1D1E
+#define mmDCP1_OVL_DFQ_CONTROL 0x1D29
+#define mmDCP1_OVL_DFQ_STATUS 0x1D2A
+#define mmDCP1_OVL_ENABLE 0x1D1C
+#define mmDCP1_OVL_END 0x1D26
+#define mmDCP1_OVL_PITCH 0x1D21
+#define mmDCP1_OVLSCL_EDGE_PIXEL_CNTL 0x1D2C
+#define mmDCP1_OVL_SECONDARY_SURFACE_ADDRESS 0x1D92
+#define mmDCP1_OVL_SECONDARY_SURFACE_ADDRESS_HIGH 0x1D94
+#define mmDCP1_OVL_START 0x1D25
+#define mmDCP1_OVL_STEREOSYNC_FLIP 0x1D93
+#define mmDCP1_OVL_SURFACE_ADDRESS 0x1D20
+#define mmDCP1_OVL_SURFACE_ADDRESS_HIGH 0x1D22
+#define mmDCP1_OVL_SURFACE_ADDRESS_HIGH_INUSE 0x1D2B
+#define mmDCP1_OVL_SURFACE_ADDRESS_INUSE 0x1D28
+#define mmDCP1_OVL_SURFACE_OFFSET_X 0x1D23
+#define mmDCP1_OVL_SURFACE_OFFSET_Y 0x1D24
+#define mmDCP1_OVL_SWAP_CNTL 0x1D1F
+#define mmDCP1_OVL_UPDATE 0x1D27
+#define mmDCP1_PRESCALE_GRPH_CONTROL 0x1D2D
+#define mmDCP1_PRESCALE_OVL_CONTROL 0x1D31
+#define mmDCP1_PRESCALE_VALUES_GRPH_B 0x1D30
+#define mmDCP1_PRESCALE_VALUES_GRPH_G 0x1D2F
+#define mmDCP1_PRESCALE_VALUES_GRPH_R 0x1D2E
+#define mmDCP1_PRESCALE_VALUES_OVL_CB 0x1D32
+#define mmDCP1_PRESCALE_VALUES_OVL_CR 0x1D34
+#define mmDCP1_PRESCALE_VALUES_OVL_Y 0x1D33
+#define mmDCP1_REGAMMA_CNTLA_END_CNTL1 0x1DA6
+#define mmDCP1_REGAMMA_CNTLA_END_CNTL2 0x1DA7
+#define mmDCP1_REGAMMA_CNTLA_REGION_0_1 0x1DA8
+#define mmDCP1_REGAMMA_CNTLA_REGION_10_11 0x1DAD
+#define mmDCP1_REGAMMA_CNTLA_REGION_12_13 0x1DAE
+#define mmDCP1_REGAMMA_CNTLA_REGION_14_15 0x1DAF
+#define mmDCP1_REGAMMA_CNTLA_REGION_2_3 0x1DA9
+#define mmDCP1_REGAMMA_CNTLA_REGION_4_5 0x1DAA
+#define mmDCP1_REGAMMA_CNTLA_REGION_6_7 0x1DAB
+#define mmDCP1_REGAMMA_CNTLA_REGION_8_9 0x1DAC
+#define mmDCP1_REGAMMA_CNTLA_SLOPE_CNTL 0x1DA5
+#define mmDCP1_REGAMMA_CNTLA_START_CNTL 0x1DA4
+#define mmDCP1_REGAMMA_CNTLB_END_CNTL1 0x1DB2
+#define mmDCP1_REGAMMA_CNTLB_END_CNTL2 0x1DB3
+#define mmDCP1_REGAMMA_CNTLB_REGION_0_1 0x1DB4
+#define mmDCP1_REGAMMA_CNTLB_REGION_10_11 0x1DB9
+#define mmDCP1_REGAMMA_CNTLB_REGION_12_13 0x1DBA
+#define mmDCP1_REGAMMA_CNTLB_REGION_14_15 0x1DBB
+#define mmDCP1_REGAMMA_CNTLB_REGION_2_3 0x1DB5
+#define mmDCP1_REGAMMA_CNTLB_REGION_4_5 0x1DB6
+#define mmDCP1_REGAMMA_CNTLB_REGION_6_7 0x1DB7
+#define mmDCP1_REGAMMA_CNTLB_REGION_8_9 0x1DB8
+#define mmDCP1_REGAMMA_CNTLB_SLOPE_CNTL 0x1DB1
+#define mmDCP1_REGAMMA_CNTLB_START_CNTL 0x1DB0
+#define mmDCP1_REGAMMA_CONTROL 0x1DA0
+#define mmDCP1_REGAMMA_LUT_DATA 0x1DA2
+#define mmDCP1_REGAMMA_LUT_INDEX 0x1DA1
+#define mmDCP1_REGAMMA_LUT_WRITE_EN_MASK 0x1DA3
+#define mmDCP2_COMM_MATRIXA_TRANS_C11_C12 0x4043
+#define mmDCP2_COMM_MATRIXA_TRANS_C13_C14 0x4044
+#define mmDCP2_COMM_MATRIXA_TRANS_C21_C22 0x4045
+#define mmDCP2_COMM_MATRIXA_TRANS_C23_C24 0x4046
+#define mmDCP2_COMM_MATRIXA_TRANS_C31_C32 0x4047
+#define mmDCP2_COMM_MATRIXA_TRANS_C33_C34 0x4048
+#define mmDCP2_COMM_MATRIXB_TRANS_C11_C12 0x4049
+#define mmDCP2_COMM_MATRIXB_TRANS_C13_C14 0x404A
+#define mmDCP2_COMM_MATRIXB_TRANS_C21_C22 0x404B
+#define mmDCP2_COMM_MATRIXB_TRANS_C23_C24 0x404C
+#define mmDCP2_COMM_MATRIXB_TRANS_C31_C32 0x404D
+#define mmDCP2_COMM_MATRIXB_TRANS_C33_C34 0x404E
+#define mmDCP2_CUR_COLOR1 0x406C
+#define mmDCP2_CUR_COLOR2 0x406D
+#define mmDCP2_CUR_CONTROL 0x4066
+#define mmDCP2_CUR_HOT_SPOT 0x406B
+#define mmDCP2_CUR_POSITION 0x406A
+#define mmDCP2_CUR_REQUEST_FILTER_CNTL 0x4099
+#define mmDCP2_CUR_SIZE 0x4068
+#define mmDCP2_CUR_SURFACE_ADDRESS 0x4067
+#define mmDCP2_CUR_SURFACE_ADDRESS_HIGH 0x4069
+#define mmDCP2_CUR_UPDATE 0x406E
+#define mmDCP2_DC_LUT_30_COLOR 0x407C
+#define mmDCP2_DC_LUT_AUTOFILL 0x407F
+#define mmDCP2_DC_LUT_BLACK_OFFSET_BLUE 0x4081
+#define mmDCP2_DC_LUT_BLACK_OFFSET_GREEN 0x4082
+#define mmDCP2_DC_LUT_BLACK_OFFSET_RED 0x4083
+#define mmDCP2_DC_LUT_CONTROL 0x4080
+#define mmDCP2_DC_LUT_PWL_DATA 0x407B
+#define mmDCP2_DC_LUT_RW_INDEX 0x4079
+#define mmDCP2_DC_LUT_RW_MODE 0x4078
+#define mmDCP2_DC_LUT_SEQ_COLOR 0x407A
+#define mmDCP2_DC_LUT_VGA_ACCESS_ENABLE 0x407D
+#define mmDCP2_DC_LUT_WHITE_OFFSET_BLUE 0x4084
+#define mmDCP2_DC_LUT_WHITE_OFFSET_GREEN 0x4085
+#define mmDCP2_DC_LUT_WHITE_OFFSET_RED 0x4086
+#define mmDCP2_DC_LUT_WRITE_EN_MASK 0x407E
+#define mmDCP2_DCP_CRC_CONTROL 0x4087
+#define mmDCP2_DCP_CRC_CURRENT 0x4089
+#define mmDCP2_DCP_CRC_LAST 0x408B
+#define mmDCP2_DCP_CRC_MASK 0x4088
+#define mmDCP2_DCP_DEBUG 0x408D
+#define mmDCP2_DCP_DEBUG2 0x4098
+#define mmDCP2_DCP_FP_CONVERTED_FIELD 0x4065
+#define mmDCP2_DCP_GSL_CONTROL 0x4090
+#define mmDCP2_DCP_LB_DATA_GAP_BETWEEN_CHUNK 0x4091
+#define mmDCP2_DCP_RANDOM_SEEDS 0x4061
+#define mmDCP2_DCP_SPATIAL_DITHER_CNTL 0x4060
+#define mmDCP2_DCP_TEST_DEBUG_DATA 0x4096
+#define mmDCP2_DCP_TEST_DEBUG_INDEX 0x4095
+#define mmDCP2_DEGAMMA_CONTROL 0x4058
+#define mmDCP2_DENORM_CONTROL 0x4050
+#define mmDCP2_GAMUT_REMAP_C11_C12 0x405A
+#define mmDCP2_GAMUT_REMAP_C13_C14 0x405B
+#define mmDCP2_GAMUT_REMAP_C21_C22 0x405C
+#define mmDCP2_GAMUT_REMAP_C23_C24 0x405D
+#define mmDCP2_GAMUT_REMAP_C31_C32 0x405E
+#define mmDCP2_GAMUT_REMAP_C33_C34 0x405F
+#define mmDCP2_GAMUT_REMAP_CONTROL 0x4059
+#define mmDCP2_GRPH_COMPRESS_PITCH 0x401A
+#define mmDCP2_GRPH_COMPRESS_SURFACE_ADDRESS 0x4019
+#define mmDCP2_GRPH_COMPRESS_SURFACE_ADDRESS_HIGH 0x401B
+#define mmDCP2_GRPH_CONTROL 0x4001
+#define mmDCP2_GRPH_DFQ_CONTROL 0x4014
+#define mmDCP2_GRPH_DFQ_STATUS 0x4015
+#define mmDCP2_GRPH_ENABLE 0x4000
+#define mmDCP2_GRPH_FLIP_CONTROL 0x4012
+#define mmDCP2_GRPH_INTERRUPT_CONTROL 0x4017
+#define mmDCP2_GRPH_INTERRUPT_STATUS 0x4016
+#define mmDCP2_GRPH_LUT_10BIT_BYPASS 0x4002
+#define mmDCP2_GRPH_PITCH 0x4006
+#define mmDCP2_GRPH_PRIMARY_SURFACE_ADDRESS 0x4004
+#define mmDCP2_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH 0x4007
+#define mmDCP2_GRPH_SECONDARY_SURFACE_ADDRESS 0x4005
+#define mmDCP2_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH 0x4008
+#define mmDCP2_GRPH_STEREOSYNC_FLIP 0x4097
+#define mmDCP2_GRPH_SURFACE_ADDRESS_HIGH_INUSE 0x4018
+#define mmDCP2_GRPH_SURFACE_ADDRESS_INUSE 0x4013
+#define mmDCP2_GRPH_SURFACE_OFFSET_X 0x4009
+#define mmDCP2_GRPH_SURFACE_OFFSET_Y 0x400A
+#define mmDCP2_GRPH_SWAP_CNTL 0x4003
+#define mmDCP2_GRPH_UPDATE 0x4011
+#define mmDCP2_GRPH_X_END 0x400D
+#define mmDCP2_GRPH_X_START 0x400B
+#define mmDCP2_GRPH_Y_END 0x400E
+#define mmDCP2_GRPH_Y_START 0x400C
+#define mmDCP2_INPUT_CSC_C11_C12 0x4036
+#define mmDCP2_INPUT_CSC_C13_C14 0x4037
+#define mmDCP2_INPUT_CSC_C21_C22 0x4038
+#define mmDCP2_INPUT_CSC_C23_C24 0x4039
+#define mmDCP2_INPUT_CSC_C31_C32 0x403A
+#define mmDCP2_INPUT_CSC_C33_C34 0x403B
+#define mmDCP2_INPUT_CSC_CONTROL 0x4035
+#define mmDCP2_INPUT_GAMMA_CONTROL 0x4010
+#define mmDCP2_KEY_CONTROL 0x4053
+#define mmDCP2_KEY_RANGE_ALPHA 0x4054
+#define mmDCP2_KEY_RANGE_BLUE 0x4057
+#define mmDCP2_KEY_RANGE_GREEN 0x4056
+#define mmDCP2_KEY_RANGE_RED 0x4055
+#define mmDCP2_OUTPUT_CSC_C11_C12 0x403D
+#define mmDCP2_OUTPUT_CSC_C13_C14 0x403E
+#define mmDCP2_OUTPUT_CSC_C21_C22 0x403F
+#define mmDCP2_OUTPUT_CSC_C23_C24 0x4040
+#define mmDCP2_OUTPUT_CSC_C31_C32 0x4041
+#define mmDCP2_OUTPUT_CSC_C33_C34 0x4042
+#define mmDCP2_OUTPUT_CSC_CONTROL 0x403C
+#define mmDCP2_OUT_ROUND_CONTROL 0x4051
+#define mmDCP2_OVL_CONTROL1 0x401D
+#define mmDCP2_OVL_CONTROL2 0x401E
+#define mmDCP2_OVL_DFQ_CONTROL 0x4029
+#define mmDCP2_OVL_DFQ_STATUS 0x402A
+#define mmDCP2_OVL_ENABLE 0x401C
+#define mmDCP2_OVL_END 0x4026
+#define mmDCP2_OVL_PITCH 0x4021
+#define mmDCP2_OVLSCL_EDGE_PIXEL_CNTL 0x402C
+#define mmDCP2_OVL_SECONDARY_SURFACE_ADDRESS 0x4092
+#define mmDCP2_OVL_SECONDARY_SURFACE_ADDRESS_HIGH 0x4094
+#define mmDCP2_OVL_START 0x4025
+#define mmDCP2_OVL_STEREOSYNC_FLIP 0x4093
+#define mmDCP2_OVL_SURFACE_ADDRESS 0x4020
+#define mmDCP2_OVL_SURFACE_ADDRESS_HIGH 0x4022
+#define mmDCP2_OVL_SURFACE_ADDRESS_HIGH_INUSE 0x402B
+#define mmDCP2_OVL_SURFACE_ADDRESS_INUSE 0x4028
+#define mmDCP2_OVL_SURFACE_OFFSET_X 0x4023
+#define mmDCP2_OVL_SURFACE_OFFSET_Y 0x4024
+#define mmDCP2_OVL_SWAP_CNTL 0x401F
+#define mmDCP2_OVL_UPDATE 0x4027
+#define mmDCP2_PRESCALE_GRPH_CONTROL 0x402D
+#define mmDCP2_PRESCALE_OVL_CONTROL 0x4031
+#define mmDCP2_PRESCALE_VALUES_GRPH_B 0x4030
+#define mmDCP2_PRESCALE_VALUES_GRPH_G 0x402F
+#define mmDCP2_PRESCALE_VALUES_GRPH_R 0x402E
+#define mmDCP2_PRESCALE_VALUES_OVL_CB 0x4032
+#define mmDCP2_PRESCALE_VALUES_OVL_CR 0x4034
+#define mmDCP2_PRESCALE_VALUES_OVL_Y 0x4033
+#define mmDCP2_REGAMMA_CNTLA_END_CNTL1 0x40A6
+#define mmDCP2_REGAMMA_CNTLA_END_CNTL2 0x40A7
+#define mmDCP2_REGAMMA_CNTLA_REGION_0_1 0x40A8
+#define mmDCP2_REGAMMA_CNTLA_REGION_10_11 0x40AD
+#define mmDCP2_REGAMMA_CNTLA_REGION_12_13 0x40AE
+#define mmDCP2_REGAMMA_CNTLA_REGION_14_15 0x40AF
+#define mmDCP2_REGAMMA_CNTLA_REGION_2_3 0x40A9
+#define mmDCP2_REGAMMA_CNTLA_REGION_4_5 0x40AA
+#define mmDCP2_REGAMMA_CNTLA_REGION_6_7 0x40AB
+#define mmDCP2_REGAMMA_CNTLA_REGION_8_9 0x40AC
+#define mmDCP2_REGAMMA_CNTLA_SLOPE_CNTL 0x40A5
+#define mmDCP2_REGAMMA_CNTLA_START_CNTL 0x40A4
+#define mmDCP2_REGAMMA_CNTLB_END_CNTL1 0x40B2
+#define mmDCP2_REGAMMA_CNTLB_END_CNTL2 0x40B3
+#define mmDCP2_REGAMMA_CNTLB_REGION_0_1 0x40B4
+#define mmDCP2_REGAMMA_CNTLB_REGION_10_11 0x40B9
+#define mmDCP2_REGAMMA_CNTLB_REGION_12_13 0x40BA
+#define mmDCP2_REGAMMA_CNTLB_REGION_14_15 0x40BB
+#define mmDCP2_REGAMMA_CNTLB_REGION_2_3 0x40B5
+#define mmDCP2_REGAMMA_CNTLB_REGION_4_5 0x40B6
+#define mmDCP2_REGAMMA_CNTLB_REGION_6_7 0x40B7
+#define mmDCP2_REGAMMA_CNTLB_REGION_8_9 0x40B8
+#define mmDCP2_REGAMMA_CNTLB_SLOPE_CNTL 0x40B1
+#define mmDCP2_REGAMMA_CNTLB_START_CNTL 0x40B0
+#define mmDCP2_REGAMMA_CONTROL 0x40A0
+#define mmDCP2_REGAMMA_LUT_DATA 0x40A2
+#define mmDCP2_REGAMMA_LUT_INDEX 0x40A1
+#define mmDCP2_REGAMMA_LUT_WRITE_EN_MASK 0x40A3
+#define mmDCP3_COMM_MATRIXA_TRANS_C11_C12 0x4343
+#define mmDCP3_COMM_MATRIXA_TRANS_C13_C14 0x4344
+#define mmDCP3_COMM_MATRIXA_TRANS_C21_C22 0x4345
+#define mmDCP3_COMM_MATRIXA_TRANS_C23_C24 0x4346
+#define mmDCP3_COMM_MATRIXA_TRANS_C31_C32 0x4347
+#define mmDCP3_COMM_MATRIXA_TRANS_C33_C34 0x4348
+#define mmDCP3_COMM_MATRIXB_TRANS_C11_C12 0x4349
+#define mmDCP3_COMM_MATRIXB_TRANS_C13_C14 0x434A
+#define mmDCP3_COMM_MATRIXB_TRANS_C21_C22 0x434B
+#define mmDCP3_COMM_MATRIXB_TRANS_C23_C24 0x434C
+#define mmDCP3_COMM_MATRIXB_TRANS_C31_C32 0x434D
+#define mmDCP3_COMM_MATRIXB_TRANS_C33_C34 0x434E
+#define mmDCP3_CUR_COLOR1 0x436C
+#define mmDCP3_CUR_COLOR2 0x436D
+#define mmDCP3_CUR_CONTROL 0x4366
+#define mmDCP3_CUR_HOT_SPOT 0x436B
+#define mmDCP3_CUR_POSITION 0x436A
+#define mmDCP3_CUR_REQUEST_FILTER_CNTL 0x4399
+#define mmDCP3_CUR_SIZE 0x4368
+#define mmDCP3_CUR_SURFACE_ADDRESS 0x4367
+#define mmDCP3_CUR_SURFACE_ADDRESS_HIGH 0x4369
+#define mmDCP3_CUR_UPDATE 0x436E
+#define mmDCP3_DC_LUT_30_COLOR 0x437C
+#define mmDCP3_DC_LUT_AUTOFILL 0x437F
+#define mmDCP3_DC_LUT_BLACK_OFFSET_BLUE 0x4381
+#define mmDCP3_DC_LUT_BLACK_OFFSET_GREEN 0x4382
+#define mmDCP3_DC_LUT_BLACK_OFFSET_RED 0x4383
+#define mmDCP3_DC_LUT_CONTROL 0x4380
+#define mmDCP3_DC_LUT_PWL_DATA 0x437B
+#define mmDCP3_DC_LUT_RW_INDEX 0x4379
+#define mmDCP3_DC_LUT_RW_MODE 0x4378
+#define mmDCP3_DC_LUT_SEQ_COLOR 0x437A
+#define mmDCP3_DC_LUT_VGA_ACCESS_ENABLE 0x437D
+#define mmDCP3_DC_LUT_WHITE_OFFSET_BLUE 0x4384
+#define mmDCP3_DC_LUT_WHITE_OFFSET_GREEN 0x4385
+#define mmDCP3_DC_LUT_WHITE_OFFSET_RED 0x4386
+#define mmDCP3_DC_LUT_WRITE_EN_MASK 0x437E
+#define mmDCP3_DCP_CRC_CONTROL 0x4387
+#define mmDCP3_DCP_CRC_CURRENT 0x4389
+#define mmDCP3_DCP_CRC_LAST 0x438B
+#define mmDCP3_DCP_CRC_MASK 0x4388
+#define mmDCP3_DCP_DEBUG 0x438D
+#define mmDCP3_DCP_DEBUG2 0x4398
+#define mmDCP3_DCP_FP_CONVERTED_FIELD 0x4365
+#define mmDCP3_DCP_GSL_CONTROL 0x4390
+#define mmDCP3_DCP_LB_DATA_GAP_BETWEEN_CHUNK 0x4391
+#define mmDCP3_DCP_RANDOM_SEEDS 0x4361
+#define mmDCP3_DCP_SPATIAL_DITHER_CNTL 0x4360
+#define mmDCP3_DCP_TEST_DEBUG_DATA 0x4396
+#define mmDCP3_DCP_TEST_DEBUG_INDEX 0x4395
+#define mmDCP3_DEGAMMA_CONTROL 0x4358
+#define mmDCP3_DENORM_CONTROL 0x4350
+#define mmDCP3_GAMUT_REMAP_C11_C12 0x435A
+#define mmDCP3_GAMUT_REMAP_C13_C14 0x435B
+#define mmDCP3_GAMUT_REMAP_C21_C22 0x435C
+#define mmDCP3_GAMUT_REMAP_C23_C24 0x435D
+#define mmDCP3_GAMUT_REMAP_C31_C32 0x435E
+#define mmDCP3_GAMUT_REMAP_C33_C34 0x435F
+#define mmDCP3_GAMUT_REMAP_CONTROL 0x4359
+#define mmDCP3_GRPH_COMPRESS_PITCH 0x431A
+#define mmDCP3_GRPH_COMPRESS_SURFACE_ADDRESS 0x4319
+#define mmDCP3_GRPH_COMPRESS_SURFACE_ADDRESS_HIGH 0x431B
+#define mmDCP3_GRPH_CONTROL 0x4301
+#define mmDCP3_GRPH_DFQ_CONTROL 0x4314
+#define mmDCP3_GRPH_DFQ_STATUS 0x4315
+#define mmDCP3_GRPH_ENABLE 0x4300
+#define mmDCP3_GRPH_FLIP_CONTROL 0x4312
+#define mmDCP3_GRPH_INTERRUPT_CONTROL 0x4317
+#define mmDCP3_GRPH_INTERRUPT_STATUS 0x4316
+#define mmDCP3_GRPH_LUT_10BIT_BYPASS 0x4302
+#define mmDCP3_GRPH_PITCH 0x4306
+#define mmDCP3_GRPH_PRIMARY_SURFACE_ADDRESS 0x4304
+#define mmDCP3_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH 0x4307
+#define mmDCP3_GRPH_SECONDARY_SURFACE_ADDRESS 0x4305
+#define mmDCP3_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH 0x4308
+#define mmDCP3_GRPH_STEREOSYNC_FLIP 0x4397
+#define mmDCP3_GRPH_SURFACE_ADDRESS_HIGH_INUSE 0x4318
+#define mmDCP3_GRPH_SURFACE_ADDRESS_INUSE 0x4313
+#define mmDCP3_GRPH_SURFACE_OFFSET_X 0x4309
+#define mmDCP3_GRPH_SURFACE_OFFSET_Y 0x430A
+#define mmDCP3_GRPH_SWAP_CNTL 0x4303
+#define mmDCP3_GRPH_UPDATE 0x4311
+#define mmDCP3_GRPH_X_END 0x430D
+#define mmDCP3_GRPH_X_START 0x430B
+#define mmDCP3_GRPH_Y_END 0x430E
+#define mmDCP3_GRPH_Y_START 0x430C
+#define mmDCP3_INPUT_CSC_C11_C12 0x4336
+#define mmDCP3_INPUT_CSC_C13_C14 0x4337
+#define mmDCP3_INPUT_CSC_C21_C22 0x4338
+#define mmDCP3_INPUT_CSC_C23_C24 0x4339
+#define mmDCP3_INPUT_CSC_C31_C32 0x433A
+#define mmDCP3_INPUT_CSC_C33_C34 0x433B
+#define mmDCP3_INPUT_CSC_CONTROL 0x4335
+#define mmDCP3_INPUT_GAMMA_CONTROL 0x4310
+#define mmDCP3_KEY_CONTROL 0x4353
+#define mmDCP3_KEY_RANGE_ALPHA 0x4354
+#define mmDCP3_KEY_RANGE_BLUE 0x4357
+#define mmDCP3_KEY_RANGE_GREEN 0x4356
+#define mmDCP3_KEY_RANGE_RED 0x4355
+#define mmDCP3_OUTPUT_CSC_C11_C12 0x433D
+#define mmDCP3_OUTPUT_CSC_C13_C14 0x433E
+#define mmDCP3_OUTPUT_CSC_C21_C22 0x433F
+#define mmDCP3_OUTPUT_CSC_C23_C24 0x4340
+#define mmDCP3_OUTPUT_CSC_C31_C32 0x4341
+#define mmDCP3_OUTPUT_CSC_C33_C34 0x4342
+#define mmDCP3_OUTPUT_CSC_CONTROL 0x433C
+#define mmDCP3_OUT_ROUND_CONTROL 0x4351
+#define mmDCP3_OVL_CONTROL1 0x431D
+#define mmDCP3_OVL_CONTROL2 0x431E
+#define mmDCP3_OVL_DFQ_CONTROL 0x4329
+#define mmDCP3_OVL_DFQ_STATUS 0x432A
+#define mmDCP3_OVL_ENABLE 0x431C
+#define mmDCP3_OVL_END 0x4326
+#define mmDCP3_OVL_PITCH 0x4321
+#define mmDCP3_OVLSCL_EDGE_PIXEL_CNTL 0x432C
+#define mmDCP3_OVL_SECONDARY_SURFACE_ADDRESS 0x4392
+#define mmDCP3_OVL_SECONDARY_SURFACE_ADDRESS_HIGH 0x4394
+#define mmDCP3_OVL_START 0x4325
+#define mmDCP3_OVL_STEREOSYNC_FLIP 0x4393
+#define mmDCP3_OVL_SURFACE_ADDRESS 0x4320
+#define mmDCP3_OVL_SURFACE_ADDRESS_HIGH 0x4322
+#define mmDCP3_OVL_SURFACE_ADDRESS_HIGH_INUSE 0x432B
+#define mmDCP3_OVL_SURFACE_ADDRESS_INUSE 0x4328
+#define mmDCP3_OVL_SURFACE_OFFSET_X 0x4323
+#define mmDCP3_OVL_SURFACE_OFFSET_Y 0x4324
+#define mmDCP3_OVL_SWAP_CNTL 0x431F
+#define mmDCP3_OVL_UPDATE 0x4327
+#define mmDCP3_PRESCALE_GRPH_CONTROL 0x432D
+#define mmDCP3_PRESCALE_OVL_CONTROL 0x4331
+#define mmDCP3_PRESCALE_VALUES_GRPH_B 0x4330
+#define mmDCP3_PRESCALE_VALUES_GRPH_G 0x432F
+#define mmDCP3_PRESCALE_VALUES_GRPH_R 0x432E
+#define mmDCP3_PRESCALE_VALUES_OVL_CB 0x4332
+#define mmDCP3_PRESCALE_VALUES_OVL_CR 0x4334
+#define mmDCP3_PRESCALE_VALUES_OVL_Y 0x4333
+#define mmDCP3_REGAMMA_CNTLA_END_CNTL1 0x43A6
+#define mmDCP3_REGAMMA_CNTLA_END_CNTL2 0x43A7
+#define mmDCP3_REGAMMA_CNTLA_REGION_0_1 0x43A8
+#define mmDCP3_REGAMMA_CNTLA_REGION_10_11 0x43AD
+#define mmDCP3_REGAMMA_CNTLA_REGION_12_13 0x43AE
+#define mmDCP3_REGAMMA_CNTLA_REGION_14_15 0x43AF
+#define mmDCP3_REGAMMA_CNTLA_REGION_2_3 0x43A9
+#define mmDCP3_REGAMMA_CNTLA_REGION_4_5 0x43AA
+#define mmDCP3_REGAMMA_CNTLA_REGION_6_7 0x43AB
+#define mmDCP3_REGAMMA_CNTLA_REGION_8_9 0x43AC
+#define mmDCP3_REGAMMA_CNTLA_SLOPE_CNTL 0x43A5
+#define mmDCP3_REGAMMA_CNTLA_START_CNTL 0x43A4
+#define mmDCP3_REGAMMA_CNTLB_END_CNTL1 0x43B2
+#define mmDCP3_REGAMMA_CNTLB_END_CNTL2 0x43B3
+#define mmDCP3_REGAMMA_CNTLB_REGION_0_1 0x43B4
+#define mmDCP3_REGAMMA_CNTLB_REGION_10_11 0x43B9
+#define mmDCP3_REGAMMA_CNTLB_REGION_12_13 0x43BA
+#define mmDCP3_REGAMMA_CNTLB_REGION_14_15 0x43BB
+#define mmDCP3_REGAMMA_CNTLB_REGION_2_3 0x43B5
+#define mmDCP3_REGAMMA_CNTLB_REGION_4_5 0x43B6
+#define mmDCP3_REGAMMA_CNTLB_REGION_6_7 0x43B7
+#define mmDCP3_REGAMMA_CNTLB_REGION_8_9 0x43B8
+#define mmDCP3_REGAMMA_CNTLB_SLOPE_CNTL 0x43B1
+#define mmDCP3_REGAMMA_CNTLB_START_CNTL 0x43B0
+#define mmDCP3_REGAMMA_CONTROL 0x43A0
+#define mmDCP3_REGAMMA_LUT_DATA 0x43A2
+#define mmDCP3_REGAMMA_LUT_INDEX 0x43A1
+#define mmDCP3_REGAMMA_LUT_WRITE_EN_MASK 0x43A3
+#define mmDCP4_COMM_MATRIXA_TRANS_C11_C12 0x4643
+#define mmDCP4_COMM_MATRIXA_TRANS_C13_C14 0x4644
+#define mmDCP4_COMM_MATRIXA_TRANS_C21_C22 0x4645
+#define mmDCP4_COMM_MATRIXA_TRANS_C23_C24 0x4646
+#define mmDCP4_COMM_MATRIXA_TRANS_C31_C32 0x4647
+#define mmDCP4_COMM_MATRIXA_TRANS_C33_C34 0x4648
+#define mmDCP4_COMM_MATRIXB_TRANS_C11_C12 0x4649
+#define mmDCP4_COMM_MATRIXB_TRANS_C13_C14 0x464A
+#define mmDCP4_COMM_MATRIXB_TRANS_C21_C22 0x464B
+#define mmDCP4_COMM_MATRIXB_TRANS_C23_C24 0x464C
+#define mmDCP4_COMM_MATRIXB_TRANS_C31_C32 0x464D
+#define mmDCP4_COMM_MATRIXB_TRANS_C33_C34 0x464E
+#define mmDCP4_CUR_COLOR1 0x466C
+#define mmDCP4_CUR_COLOR2 0x466D
+#define mmDCP4_CUR_CONTROL 0x4666
+#define mmDCP4_CUR_HOT_SPOT 0x466B
+#define mmDCP4_CUR_POSITION 0x466A
+#define mmDCP4_CUR_REQUEST_FILTER_CNTL 0x4699
+#define mmDCP4_CUR_SIZE 0x4668
+#define mmDCP4_CUR_SURFACE_ADDRESS 0x4667
+#define mmDCP4_CUR_SURFACE_ADDRESS_HIGH 0x4669
+#define mmDCP4_CUR_UPDATE 0x466E
+#define mmDCP4_DC_LUT_30_COLOR 0x467C
+#define mmDCP4_DC_LUT_AUTOFILL 0x467F
+#define mmDCP4_DC_LUT_BLACK_OFFSET_BLUE 0x4681
+#define mmDCP4_DC_LUT_BLACK_OFFSET_GREEN 0x4682
+#define mmDCP4_DC_LUT_BLACK_OFFSET_RED 0x4683
+#define mmDCP4_DC_LUT_CONTROL 0x4680
+#define mmDCP4_DC_LUT_PWL_DATA 0x467B
+#define mmDCP4_DC_LUT_RW_INDEX 0x4679
+#define mmDCP4_DC_LUT_RW_MODE 0x4678
+#define mmDCP4_DC_LUT_SEQ_COLOR 0x467A
+#define mmDCP4_DC_LUT_VGA_ACCESS_ENABLE 0x467D
+#define mmDCP4_DC_LUT_WHITE_OFFSET_BLUE 0x4684
+#define mmDCP4_DC_LUT_WHITE_OFFSET_GREEN 0x4685
+#define mmDCP4_DC_LUT_WHITE_OFFSET_RED 0x4686
+#define mmDCP4_DC_LUT_WRITE_EN_MASK 0x467E
+#define mmDCP4_DCP_CRC_CONTROL 0x4687
+#define mmDCP4_DCP_CRC_CURRENT 0x4689
+#define mmDCP4_DCP_CRC_LAST 0x468B
+#define mmDCP4_DCP_CRC_MASK 0x4688
+#define mmDCP4_DCP_DEBUG 0x468D
+#define mmDCP4_DCP_DEBUG2 0x4698
+#define mmDCP4_DCP_FP_CONVERTED_FIELD 0x4665
+#define mmDCP4_DCP_GSL_CONTROL 0x4690
+#define mmDCP4_DCP_LB_DATA_GAP_BETWEEN_CHUNK 0x4691
+#define mmDCP4_DCP_RANDOM_SEEDS 0x4661
+#define mmDCP4_DCP_SPATIAL_DITHER_CNTL 0x4660
+#define mmDCP4_DCP_TEST_DEBUG_DATA 0x4696
+#define mmDCP4_DCP_TEST_DEBUG_INDEX 0x4695
+#define mmDCP4_DEGAMMA_CONTROL 0x4658
+#define mmDCP4_DENORM_CONTROL 0x4650
+#define mmDCP4_GAMUT_REMAP_C11_C12 0x465A
+#define mmDCP4_GAMUT_REMAP_C13_C14 0x465B
+#define mmDCP4_GAMUT_REMAP_C21_C22 0x465C
+#define mmDCP4_GAMUT_REMAP_C23_C24 0x465D
+#define mmDCP4_GAMUT_REMAP_C31_C32 0x465E
+#define mmDCP4_GAMUT_REMAP_C33_C34 0x465F
+#define mmDCP4_GAMUT_REMAP_CONTROL 0x4659
+#define mmDCP4_GRPH_COMPRESS_PITCH 0x461A
+#define mmDCP4_GRPH_COMPRESS_SURFACE_ADDRESS 0x4619
+#define mmDCP4_GRPH_COMPRESS_SURFACE_ADDRESS_HIGH 0x461B
+#define mmDCP4_GRPH_CONTROL 0x4601
+#define mmDCP4_GRPH_DFQ_CONTROL 0x4614
+#define mmDCP4_GRPH_DFQ_STATUS 0x4615
+#define mmDCP4_GRPH_ENABLE 0x4600
+#define mmDCP4_GRPH_FLIP_CONTROL 0x4612
+#define mmDCP4_GRPH_INTERRUPT_CONTROL 0x4617
+#define mmDCP4_GRPH_INTERRUPT_STATUS 0x4616
+#define mmDCP4_GRPH_LUT_10BIT_BYPASS 0x4602
+#define mmDCP4_GRPH_PITCH 0x4606
+#define mmDCP4_GRPH_PRIMARY_SURFACE_ADDRESS 0x4604
+#define mmDCP4_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH 0x4607
+#define mmDCP4_GRPH_SECONDARY_SURFACE_ADDRESS 0x4605
+#define mmDCP4_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH 0x4608
+#define mmDCP4_GRPH_STEREOSYNC_FLIP 0x4697
+#define mmDCP4_GRPH_SURFACE_ADDRESS_HIGH_INUSE 0x4618
+#define mmDCP4_GRPH_SURFACE_ADDRESS_INUSE 0x4613
+#define mmDCP4_GRPH_SURFACE_OFFSET_X 0x4609
+#define mmDCP4_GRPH_SURFACE_OFFSET_Y 0x460A
+#define mmDCP4_GRPH_SWAP_CNTL 0x4603
+#define mmDCP4_GRPH_UPDATE 0x4611
+#define mmDCP4_GRPH_X_END 0x460D
+#define mmDCP4_GRPH_X_START 0x460B
+#define mmDCP4_GRPH_Y_END 0x460E
+#define mmDCP4_GRPH_Y_START 0x460C
+#define mmDCP4_INPUT_CSC_C11_C12 0x4636
+#define mmDCP4_INPUT_CSC_C13_C14 0x4637
+#define mmDCP4_INPUT_CSC_C21_C22 0x4638
+#define mmDCP4_INPUT_CSC_C23_C24 0x4639
+#define mmDCP4_INPUT_CSC_C31_C32 0x463A
+#define mmDCP4_INPUT_CSC_C33_C34 0x463B
+#define mmDCP4_INPUT_CSC_CONTROL 0x4635
+#define mmDCP4_INPUT_GAMMA_CONTROL 0x4610
+#define mmDCP4_KEY_CONTROL 0x4653
+#define mmDCP4_KEY_RANGE_ALPHA 0x4654
+#define mmDCP4_KEY_RANGE_BLUE 0x4657
+#define mmDCP4_KEY_RANGE_GREEN 0x4656
+#define mmDCP4_KEY_RANGE_RED 0x4655
+#define mmDCP4_OUTPUT_CSC_C11_C12 0x463D
+#define mmDCP4_OUTPUT_CSC_C13_C14 0x463E
+#define mmDCP4_OUTPUT_CSC_C21_C22 0x463F
+#define mmDCP4_OUTPUT_CSC_C23_C24 0x4640
+#define mmDCP4_OUTPUT_CSC_C31_C32 0x4641
+#define mmDCP4_OUTPUT_CSC_C33_C34 0x4642
+#define mmDCP4_OUTPUT_CSC_CONTROL 0x463C
+#define mmDCP4_OUT_ROUND_CONTROL 0x4651
+#define mmDCP4_OVL_CONTROL1 0x461D
+#define mmDCP4_OVL_CONTROL2 0x461E
+#define mmDCP4_OVL_DFQ_CONTROL 0x4629
+#define mmDCP4_OVL_DFQ_STATUS 0x462A
+#define mmDCP4_OVL_ENABLE 0x461C
+#define mmDCP4_OVL_END 0x4626
+#define mmDCP4_OVL_PITCH 0x4621
+#define mmDCP4_OVLSCL_EDGE_PIXEL_CNTL 0x462C
+#define mmDCP4_OVL_SECONDARY_SURFACE_ADDRESS 0x4692
+#define mmDCP4_OVL_SECONDARY_SURFACE_ADDRESS_HIGH 0x4694
+#define mmDCP4_OVL_START 0x4625
+#define mmDCP4_OVL_STEREOSYNC_FLIP 0x4693
+#define mmDCP4_OVL_SURFACE_ADDRESS 0x4620
+#define mmDCP4_OVL_SURFACE_ADDRESS_HIGH 0x4622
+#define mmDCP4_OVL_SURFACE_ADDRESS_HIGH_INUSE 0x462B
+#define mmDCP4_OVL_SURFACE_ADDRESS_INUSE 0x4628
+#define mmDCP4_OVL_SURFACE_OFFSET_X 0x4623
+#define mmDCP4_OVL_SURFACE_OFFSET_Y 0x4624
+#define mmDCP4_OVL_SWAP_CNTL 0x461F
+#define mmDCP4_OVL_UPDATE 0x4627
+#define mmDCP4_PRESCALE_GRPH_CONTROL 0x462D
+#define mmDCP4_PRESCALE_OVL_CONTROL 0x4631
+#define mmDCP4_PRESCALE_VALUES_GRPH_B 0x4630
+#define mmDCP4_PRESCALE_VALUES_GRPH_G 0x462F
+#define mmDCP4_PRESCALE_VALUES_GRPH_R 0x462E
+#define mmDCP4_PRESCALE_VALUES_OVL_CB 0x4632
+#define mmDCP4_PRESCALE_VALUES_OVL_CR 0x4634
+#define mmDCP4_PRESCALE_VALUES_OVL_Y 0x4633
+#define mmDCP4_REGAMMA_CNTLA_END_CNTL1 0x46A6
+#define mmDCP4_REGAMMA_CNTLA_END_CNTL2 0x46A7
+#define mmDCP4_REGAMMA_CNTLA_REGION_0_1 0x46A8
+#define mmDCP4_REGAMMA_CNTLA_REGION_10_11 0x46AD
+#define mmDCP4_REGAMMA_CNTLA_REGION_12_13 0x46AE
+#define mmDCP4_REGAMMA_CNTLA_REGION_14_15 0x46AF
+#define mmDCP4_REGAMMA_CNTLA_REGION_2_3 0x46A9
+#define mmDCP4_REGAMMA_CNTLA_REGION_4_5 0x46AA
+#define mmDCP4_REGAMMA_CNTLA_REGION_6_7 0x46AB
+#define mmDCP4_REGAMMA_CNTLA_REGION_8_9 0x46AC
+#define mmDCP4_REGAMMA_CNTLA_SLOPE_CNTL 0x46A5
+#define mmDCP4_REGAMMA_CNTLA_START_CNTL 0x46A4
+#define mmDCP4_REGAMMA_CNTLB_END_CNTL1 0x46B2
+#define mmDCP4_REGAMMA_CNTLB_END_CNTL2 0x46B3
+#define mmDCP4_REGAMMA_CNTLB_REGION_0_1 0x46B4
+#define mmDCP4_REGAMMA_CNTLB_REGION_10_11 0x46B9
+#define mmDCP4_REGAMMA_CNTLB_REGION_12_13 0x46BA
+#define mmDCP4_REGAMMA_CNTLB_REGION_14_15 0x46BB
+#define mmDCP4_REGAMMA_CNTLB_REGION_2_3 0x46B5
+#define mmDCP4_REGAMMA_CNTLB_REGION_4_5 0x46B6
+#define mmDCP4_REGAMMA_CNTLB_REGION_6_7 0x46B7
+#define mmDCP4_REGAMMA_CNTLB_REGION_8_9 0x46B8
+#define mmDCP4_REGAMMA_CNTLB_SLOPE_CNTL 0x46B1
+#define mmDCP4_REGAMMA_CNTLB_START_CNTL 0x46B0
+#define mmDCP4_REGAMMA_CONTROL 0x46A0
+#define mmDCP4_REGAMMA_LUT_DATA 0x46A2
+#define mmDCP4_REGAMMA_LUT_INDEX 0x46A1
+#define mmDCP4_REGAMMA_LUT_WRITE_EN_MASK 0x46A3
+#define mmDCP5_COMM_MATRIXA_TRANS_C11_C12 0x4943
+#define mmDCP5_COMM_MATRIXA_TRANS_C13_C14 0x4944
+#define mmDCP5_COMM_MATRIXA_TRANS_C21_C22 0x4945
+#define mmDCP5_COMM_MATRIXA_TRANS_C23_C24 0x4946
+#define mmDCP5_COMM_MATRIXA_TRANS_C31_C32 0x4947
+#define mmDCP5_COMM_MATRIXA_TRANS_C33_C34 0x4948
+#define mmDCP5_COMM_MATRIXB_TRANS_C11_C12 0x4949
+#define mmDCP5_COMM_MATRIXB_TRANS_C13_C14 0x494A
+#define mmDCP5_COMM_MATRIXB_TRANS_C21_C22 0x494B
+#define mmDCP5_COMM_MATRIXB_TRANS_C23_C24 0x494C
+#define mmDCP5_COMM_MATRIXB_TRANS_C31_C32 0x494D
+#define mmDCP5_COMM_MATRIXB_TRANS_C33_C34 0x494E
+#define mmDCP5_CUR_COLOR1 0x496C
+#define mmDCP5_CUR_COLOR2 0x496D
+#define mmDCP5_CUR_CONTROL 0x4966
+#define mmDCP5_CUR_HOT_SPOT 0x496B
+#define mmDCP5_CUR_POSITION 0x496A
+#define mmDCP5_CUR_REQUEST_FILTER_CNTL 0x4999
+#define mmDCP5_CUR_SIZE 0x4968
+#define mmDCP5_CUR_SURFACE_ADDRESS 0x4967
+#define mmDCP5_CUR_SURFACE_ADDRESS_HIGH 0x4969
+#define mmDCP5_CUR_UPDATE 0x496E
+#define mmDCP5_DC_LUT_30_COLOR 0x497C
+#define mmDCP5_DC_LUT_AUTOFILL 0x497F
+#define mmDCP5_DC_LUT_BLACK_OFFSET_BLUE 0x4981
+#define mmDCP5_DC_LUT_BLACK_OFFSET_GREEN 0x4982
+#define mmDCP5_DC_LUT_BLACK_OFFSET_RED 0x4983
+#define mmDCP5_DC_LUT_CONTROL 0x4980
+#define mmDCP5_DC_LUT_PWL_DATA 0x497B
+#define mmDCP5_DC_LUT_RW_INDEX 0x4979
+#define mmDCP5_DC_LUT_RW_MODE 0x4978
+#define mmDCP5_DC_LUT_SEQ_COLOR 0x497A
+#define mmDCP5_DC_LUT_VGA_ACCESS_ENABLE 0x497D
+#define mmDCP5_DC_LUT_WHITE_OFFSET_BLUE 0x4984
+#define mmDCP5_DC_LUT_WHITE_OFFSET_GREEN 0x4985
+#define mmDCP5_DC_LUT_WHITE_OFFSET_RED 0x4986
+#define mmDCP5_DC_LUT_WRITE_EN_MASK 0x497E
+#define mmDCP5_DCP_CRC_CONTROL 0x4987
+#define mmDCP5_DCP_CRC_CURRENT 0x4989
+#define mmDCP5_DCP_CRC_LAST 0x498B
+#define mmDCP5_DCP_CRC_MASK 0x4988
+#define mmDCP5_DCP_DEBUG 0x498D
+#define mmDCP5_DCP_DEBUG2 0x4998
+#define mmDCP5_DCP_FP_CONVERTED_FIELD 0x4965
+#define mmDCP5_DCP_GSL_CONTROL 0x4990
+#define mmDCP5_DCP_LB_DATA_GAP_BETWEEN_CHUNK 0x4991
+#define mmDCP5_DCP_RANDOM_SEEDS 0x4961
+#define mmDCP5_DCP_SPATIAL_DITHER_CNTL 0x4960
+#define mmDCP5_DCP_TEST_DEBUG_DATA 0x4996
+#define mmDCP5_DCP_TEST_DEBUG_INDEX 0x4995
+#define mmDCP5_DEGAMMA_CONTROL 0x4958
+#define mmDCP5_DENORM_CONTROL 0x4950
+#define mmDCP5_GAMUT_REMAP_C11_C12 0x495A
+#define mmDCP5_GAMUT_REMAP_C13_C14 0x495B
+#define mmDCP5_GAMUT_REMAP_C21_C22 0x495C
+#define mmDCP5_GAMUT_REMAP_C23_C24 0x495D
+#define mmDCP5_GAMUT_REMAP_C31_C32 0x495E
+#define mmDCP5_GAMUT_REMAP_C33_C34 0x495F
+#define mmDCP5_GAMUT_REMAP_CONTROL 0x4959
+#define mmDCP5_GRPH_COMPRESS_PITCH 0x491A
+#define mmDCP5_GRPH_COMPRESS_SURFACE_ADDRESS 0x4919
+#define mmDCP5_GRPH_COMPRESS_SURFACE_ADDRESS_HIGH 0x491B
+#define mmDCP5_GRPH_CONTROL 0x4901
+#define mmDCP5_GRPH_DFQ_CONTROL 0x4914
+#define mmDCP5_GRPH_DFQ_STATUS 0x4915
+#define mmDCP5_GRPH_ENABLE 0x4900
+#define mmDCP5_GRPH_FLIP_CONTROL 0x4912
+#define mmDCP5_GRPH_INTERRUPT_CONTROL 0x4917
+#define mmDCP5_GRPH_INTERRUPT_STATUS 0x4916
+#define mmDCP5_GRPH_LUT_10BIT_BYPASS 0x4902
+#define mmDCP5_GRPH_PITCH 0x4906
+#define mmDCP5_GRPH_PRIMARY_SURFACE_ADDRESS 0x4904
+#define mmDCP5_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH 0x4907
+#define mmDCP5_GRPH_SECONDARY_SURFACE_ADDRESS 0x4905
+#define mmDCP5_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH 0x4908
+#define mmDCP5_GRPH_STEREOSYNC_FLIP 0x4997
+#define mmDCP5_GRPH_SURFACE_ADDRESS_HIGH_INUSE 0x4918
+#define mmDCP5_GRPH_SURFACE_ADDRESS_INUSE 0x4913
+#define mmDCP5_GRPH_SURFACE_OFFSET_X 0x4909
+#define mmDCP5_GRPH_SURFACE_OFFSET_Y 0x490A
+#define mmDCP5_GRPH_SWAP_CNTL 0x4903
+#define mmDCP5_GRPH_UPDATE 0x4911
+#define mmDCP5_GRPH_X_END 0x490D
+#define mmDCP5_GRPH_X_START 0x490B
+#define mmDCP5_GRPH_Y_END 0x490E
+#define mmDCP5_GRPH_Y_START 0x490C
+#define mmDCP5_INPUT_CSC_C11_C12 0x4936
+#define mmDCP5_INPUT_CSC_C13_C14 0x4937
+#define mmDCP5_INPUT_CSC_C21_C22 0x4938
+#define mmDCP5_INPUT_CSC_C23_C24 0x4939
+#define mmDCP5_INPUT_CSC_C31_C32 0x493A
+#define mmDCP5_INPUT_CSC_C33_C34 0x493B
+#define mmDCP5_INPUT_CSC_CONTROL 0x4935
+#define mmDCP5_INPUT_GAMMA_CONTROL 0x4910
+#define mmDCP5_KEY_CONTROL 0x4953
+#define mmDCP5_KEY_RANGE_ALPHA 0x4954
+#define mmDCP5_KEY_RANGE_BLUE 0x4957
+#define mmDCP5_KEY_RANGE_GREEN 0x4956
+#define mmDCP5_KEY_RANGE_RED 0x4955
+#define mmDCP5_OUTPUT_CSC_C11_C12 0x493D
+#define mmDCP5_OUTPUT_CSC_C13_C14 0x493E
+#define mmDCP5_OUTPUT_CSC_C21_C22 0x493F
+#define mmDCP5_OUTPUT_CSC_C23_C24 0x4940
+#define mmDCP5_OUTPUT_CSC_C31_C32 0x4941
+#define mmDCP5_OUTPUT_CSC_C33_C34 0x4942
+#define mmDCP5_OUTPUT_CSC_CONTROL 0x493C
+#define mmDCP5_OUT_ROUND_CONTROL 0x4951
+#define mmDCP5_OVL_CONTROL1 0x491D
+#define mmDCP5_OVL_CONTROL2 0x491E
+#define mmDCP5_OVL_DFQ_CONTROL 0x4929
+#define mmDCP5_OVL_DFQ_STATUS 0x492A
+#define mmDCP5_OVL_ENABLE 0x491C
+#define mmDCP5_OVL_END 0x4926
+#define mmDCP5_OVL_PITCH 0x4921
+#define mmDCP5_OVLSCL_EDGE_PIXEL_CNTL 0x492C
+#define mmDCP5_OVL_SECONDARY_SURFACE_ADDRESS 0x4992
+#define mmDCP5_OVL_SECONDARY_SURFACE_ADDRESS_HIGH 0x4994
+#define mmDCP5_OVL_START 0x4925
+#define mmDCP5_OVL_STEREOSYNC_FLIP 0x4993
+#define mmDCP5_OVL_SURFACE_ADDRESS 0x4920
+#define mmDCP5_OVL_SURFACE_ADDRESS_HIGH 0x4922
+#define mmDCP5_OVL_SURFACE_ADDRESS_HIGH_INUSE 0x492B
+#define mmDCP5_OVL_SURFACE_ADDRESS_INUSE 0x4928
+#define mmDCP5_OVL_SURFACE_OFFSET_X 0x4923
+#define mmDCP5_OVL_SURFACE_OFFSET_Y 0x4924
+#define mmDCP5_OVL_SWAP_CNTL 0x491F
+#define mmDCP5_OVL_UPDATE 0x4927
+#define mmDCP5_PRESCALE_GRPH_CONTROL 0x492D
+#define mmDCP5_PRESCALE_OVL_CONTROL 0x4931
+#define mmDCP5_PRESCALE_VALUES_GRPH_B 0x4930
+#define mmDCP5_PRESCALE_VALUES_GRPH_G 0x492F
+#define mmDCP5_PRESCALE_VALUES_GRPH_R 0x492E
+#define mmDCP5_PRESCALE_VALUES_OVL_CB 0x4932
+#define mmDCP5_PRESCALE_VALUES_OVL_CR 0x4934
+#define mmDCP5_PRESCALE_VALUES_OVL_Y 0x4933
+#define mmDCP5_REGAMMA_CNTLA_END_CNTL1 0x49A6
+#define mmDCP5_REGAMMA_CNTLA_END_CNTL2 0x49A7
+#define mmDCP5_REGAMMA_CNTLA_REGION_0_1 0x49A8
+#define mmDCP5_REGAMMA_CNTLA_REGION_10_11 0x49AD
+#define mmDCP5_REGAMMA_CNTLA_REGION_12_13 0x49AE
+#define mmDCP5_REGAMMA_CNTLA_REGION_14_15 0x49AF
+#define mmDCP5_REGAMMA_CNTLA_REGION_2_3 0x49A9
+#define mmDCP5_REGAMMA_CNTLA_REGION_4_5 0x49AA
+#define mmDCP5_REGAMMA_CNTLA_REGION_6_7 0x49AB
+#define mmDCP5_REGAMMA_CNTLA_REGION_8_9 0x49AC
+#define mmDCP5_REGAMMA_CNTLA_SLOPE_CNTL 0x49A5
+#define mmDCP5_REGAMMA_CNTLA_START_CNTL 0x49A4
+#define mmDCP5_REGAMMA_CNTLB_END_CNTL1 0x49B2
+#define mmDCP5_REGAMMA_CNTLB_END_CNTL2 0x49B3
+#define mmDCP5_REGAMMA_CNTLB_REGION_0_1 0x49B4
+#define mmDCP5_REGAMMA_CNTLB_REGION_10_11 0x49B9
+#define mmDCP5_REGAMMA_CNTLB_REGION_12_13 0x49BA
+#define mmDCP5_REGAMMA_CNTLB_REGION_14_15 0x49BB
+#define mmDCP5_REGAMMA_CNTLB_REGION_2_3 0x49B5
+#define mmDCP5_REGAMMA_CNTLB_REGION_4_5 0x49B6
+#define mmDCP5_REGAMMA_CNTLB_REGION_6_7 0x49B7
+#define mmDCP5_REGAMMA_CNTLB_REGION_8_9 0x49B8
+#define mmDCP5_REGAMMA_CNTLB_SLOPE_CNTL 0x49B1
+#define mmDCP5_REGAMMA_CNTLB_START_CNTL 0x49B0
+#define mmDCP5_REGAMMA_CONTROL 0x49A0
+#define mmDCP5_REGAMMA_LUT_DATA 0x49A2
+#define mmDCP5_REGAMMA_LUT_INDEX 0x49A1
+#define mmDCP5_REGAMMA_LUT_WRITE_EN_MASK 0x49A3
+#define mmDC_PAD_EXTERN_SIG 0x1902
+#define mmDCP_CRC_CONTROL 0x1A87
+#define mmDCP_CRC_CURRENT 0x1A89
+#define mmDCP_CRC_LAST 0x1A8B
+#define mmDCP_CRC_MASK 0x1A88
+#define mmDCP_DEBUG 0x1A8D
+#define mmDCP_DEBUG2 0x1A98
+#define mmDCP_FP_CONVERTED_FIELD 0x1A65
+#define mmDC_PGCNTL_STATUS_REG 0x177E
+#define mmDC_PGFSM_CONFIG_REG 0x177C
+#define mmDC_PGFSM_WRITE_REG 0x177D
+#define mmDCP_GSL_CONTROL 0x1A90
+#define mmDCPG_TEST_DEBUG_DATA 0x177B
+#define mmDCPG_TEST_DEBUG_INDEX 0x1779
+#define mmDC_PINSTRAPS 0x1917
+#define mmDCP_LB_DATA_GAP_BETWEEN_CHUNK 0x1A91
+#define mmDCP_RANDOM_SEEDS 0x1A61
+#define mmDCP_SPATIAL_DITHER_CNTL 0x1A60
+#define mmDCP_TEST_DEBUG_DATA 0x1A96
+#define mmDCP_TEST_DEBUG_INDEX 0x1A95
+#define mmDC_RBBMIF_RDWR_CNTL1 0x031A
+#define mmDC_RBBMIF_RDWR_CNTL2 0x031D
+#define mmDC_REF_CLK_CNTL 0x1903
+#define mmDC_XDMA_INTERFACE_CNTL 0x0327
+#define mmDEGAMMA_CONTROL 0x1A58
+#define mmDENORM_CONTROL 0x1A50
+#define mmDENTIST_DISPCLK_CNTL 0x0124
+#define mmDIG0_AFMT_60958_0 0x1C41
+#define mmDIG0_AFMT_60958_1 0x1C42
+#define mmDIG0_AFMT_60958_2 0x1C48
+#define mmDIG0_AFMT_AUDIO_CRC_CONTROL 0x1C43
+#define mmDIG0_AFMT_AUDIO_CRC_RESULT 0x1C49
+#define mmDIG0_AFMT_AUDIO_DBG_DTO_CNTL 0x1C52
+#define mmDIG0_AFMT_AUDIO_INFO0 0x1C3F
+#define mmDIG0_AFMT_AUDIO_INFO1 0x1C40
+#define mmDIG0_AFMT_AUDIO_PACKET_CONTROL 0x1C4B
+#define mmDIG0_AFMT_AUDIO_PACKET_CONTROL2 0x1C17
+#define mmDIG0_AFMT_AUDIO_SRC_CONTROL 0x1C4F
+#define mmDIG0_AFMT_AVI_INFO0 0x1C21
+#define mmDIG0_AFMT_AVI_INFO1 0x1C22
+#define mmDIG0_AFMT_AVI_INFO2 0x1C23
+#define mmDIG0_AFMT_AVI_INFO3 0x1C24
+#define mmDIG0_AFMT_GENERIC_0 0x1C28
+#define mmDIG0_AFMT_GENERIC_1 0x1C29
+#define mmDIG0_AFMT_GENERIC_2 0x1C2A
+#define mmDIG0_AFMT_GENERIC_3 0x1C2B
+#define mmDIG0_AFMT_GENERIC_4 0x1C2C
+#define mmDIG0_AFMT_GENERIC_5 0x1C2D
+#define mmDIG0_AFMT_GENERIC_6 0x1C2E
+#define mmDIG0_AFMT_GENERIC_7 0x1C2F
+#define mmDIG0_AFMT_GENERIC_HDR 0x1C27
+#define mmDIG0_AFMT_INFOFRAME_CONTROL0 0x1C4D
+#define mmDIG0_AFMT_INTERRUPT_STATUS 0x1C14
+#define mmDIG0_AFMT_ISRC1_0 0x1C18
+#define mmDIG0_AFMT_ISRC1_1 0x1C19
+#define mmDIG0_AFMT_ISRC1_2 0x1C1A
+#define mmDIG0_AFMT_ISRC1_3 0x1C1B
+#define mmDIG0_AFMT_ISRC1_4 0x1C1C
+#define mmDIG0_AFMT_ISRC2_0 0x1C1D
+#define mmDIG0_AFMT_ISRC2_1 0x1C1E
+#define mmDIG0_AFMT_ISRC2_2 0x1C1F
+#define mmDIG0_AFMT_ISRC2_3 0x1C20
+#define mmDIG0_AFMT_MPEG_INFO0 0x1C25
+#define mmDIG0_AFMT_MPEG_INFO1 0x1C26
+#define mmDIG0_AFMT_RAMP_CONTROL0 0x1C44
+#define mmDIG0_AFMT_RAMP_CONTROL1 0x1C45
+#define mmDIG0_AFMT_RAMP_CONTROL2 0x1C46
+#define mmDIG0_AFMT_RAMP_CONTROL3 0x1C47
+#define mmDIG0_AFMT_STATUS 0x1C4A
+#define mmDIG0_AFMT_VBI_PACKET_CONTROL 0x1C4C
+#define mmDIG0_DIG_BE_CNTL 0x1C50
+#define mmDIG0_DIG_BE_EN_CNTL 0x1C51
+#define mmDIG0_DIG_CLOCK_PATTERN 0x1C03
+#define mmDIG0_DIG_DISPCLK_SWITCH_CNTL 0x1C08
+#define mmDIG0_DIG_DISPCLK_SWITCH_STATUS 0x1C09
+#define mmDIG0_DIG_FE_CNTL 0x1C00
+#define mmDIG0_DIG_FIFO_STATUS 0x1C0A
+#define mmDIG0_DIG_LANE_ENABLE 0x1C8D
+#define mmDIG0_DIG_OUTPUT_CRC_CNTL 0x1C01
+#define mmDIG0_DIG_OUTPUT_CRC_RESULT 0x1C02
+#define mmDIG0_DIG_RANDOM_PATTERN_SEED 0x1C05
+#define mmDIG0_DIG_TEST_PATTERN 0x1C04
+#define mmDIG0_HDMI_ACR_32_0 0x1C37
+#define mmDIG0_HDMI_ACR_32_1 0x1C38
+#define mmDIG0_HDMI_ACR_44_0 0x1C39
+#define mmDIG0_HDMI_ACR_44_1 0x1C3A
+#define mmDIG0_HDMI_ACR_48_0 0x1C3B
+#define mmDIG0_HDMI_ACR_48_1 0x1C3C
+#define mmDIG0_HDMI_ACR_PACKET_CONTROL 0x1C0F
+#define mmDIG0_HDMI_ACR_STATUS_0 0x1C3D
+#define mmDIG0_HDMI_ACR_STATUS_1 0x1C3E
+#define mmDIG0_HDMI_AUDIO_PACKET_CONTROL 0x1C0E
+#define mmDIG0_HDMI_CONTROL 0x1C0C
+#define mmDIG0_HDMI_GC 0x1C16
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL0 0x1C13
+#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL1 0x1C30
+#define mmDIG0_HDMI_INFOFRAME_CONTROL0 0x1C11
+#define mmDIG0_HDMI_INFOFRAME_CONTROL1 0x1C12
+#define mmDIG0_HDMI_STATUS 0x1C0D
+#define mmDIG0_HDMI_VBI_PACKET_CONTROL 0x1C10
+#define mmDIG0_LVDS_DATA_CNTL 0x1C8C
+#define mmDIG0_TMDS_CNTL 0x1C7C
+#define mmDIG0_TMDS_CONTROL0_FEEDBACK 0x1C7E
+#define mmDIG0_TMDS_CONTROL_CHAR 0x1C7D
+#define mmDIG0_TMDS_CTL0_1_GEN_CNTL 0x1C86
+#define mmDIG0_TMDS_CTL2_3_GEN_CNTL 0x1C87
+#define mmDIG0_TMDS_CTL_BITS 0x1C83
+#define mmDIG0_TMDS_DCBALANCER_CONTROL 0x1C84
+#define mmDIG0_TMDS_DEBUG 0x1C82
+#define mmDIG0_TMDS_STEREOSYNC_CTL_SEL 0x1C7F
+#define mmDIG0_TMDS_SYNC_CHAR_PATTERN_0_1 0x1C80
+#define mmDIG0_TMDS_SYNC_CHAR_PATTERN_2_3 0x1C81
+#define mmDIG1_AFMT_60958_0 0x1F41
+#define mmDIG1_AFMT_60958_1 0x1F42
+#define mmDIG1_AFMT_60958_2 0x1F48
+#define mmDIG1_AFMT_AUDIO_CRC_CONTROL 0x1F43
+#define mmDIG1_AFMT_AUDIO_CRC_RESULT 0x1F49
+#define mmDIG1_AFMT_AUDIO_DBG_DTO_CNTL 0x1F52
+#define mmDIG1_AFMT_AUDIO_INFO0 0x1F3F
+#define mmDIG1_AFMT_AUDIO_INFO1 0x1F40
+#define mmDIG1_AFMT_AUDIO_PACKET_CONTROL 0x1F4B
+#define mmDIG1_AFMT_AUDIO_PACKET_CONTROL2 0x1F17
+#define mmDIG1_AFMT_AUDIO_SRC_CONTROL 0x1F4F
+#define mmDIG1_AFMT_AVI_INFO0 0x1F21
+#define mmDIG1_AFMT_AVI_INFO1 0x1F22
+#define mmDIG1_AFMT_AVI_INFO2 0x1F23
+#define mmDIG1_AFMT_AVI_INFO3 0x1F24
+#define mmDIG1_AFMT_GENERIC_0 0x1F28
+#define mmDIG1_AFMT_GENERIC_1 0x1F29
+#define mmDIG1_AFMT_GENERIC_2 0x1F2A
+#define mmDIG1_AFMT_GENERIC_3 0x1F2B
+#define mmDIG1_AFMT_GENERIC_4 0x1F2C
+#define mmDIG1_AFMT_GENERIC_5 0x1F2D
+#define mmDIG1_AFMT_GENERIC_6 0x1F2E
+#define mmDIG1_AFMT_GENERIC_7 0x1F2F
+#define mmDIG1_AFMT_GENERIC_HDR 0x1F27
+#define mmDIG1_AFMT_INFOFRAME_CONTROL0 0x1F4D
+#define mmDIG1_AFMT_INTERRUPT_STATUS 0x1F14
+#define mmDIG1_AFMT_ISRC1_0 0x1F18
+#define mmDIG1_AFMT_ISRC1_1 0x1F19
+#define mmDIG1_AFMT_ISRC1_2 0x1F1A
+#define mmDIG1_AFMT_ISRC1_3 0x1F1B
+#define mmDIG1_AFMT_ISRC1_4 0x1F1C
+#define mmDIG1_AFMT_ISRC2_0 0x1F1D
+#define mmDIG1_AFMT_ISRC2_1 0x1F1E
+#define mmDIG1_AFMT_ISRC2_2 0x1F1F
+#define mmDIG1_AFMT_ISRC2_3 0x1F20
+#define mmDIG1_AFMT_MPEG_INFO0 0x1F25
+#define mmDIG1_AFMT_MPEG_INFO1 0x1F26
+#define mmDIG1_AFMT_RAMP_CONTROL0 0x1F44
+#define mmDIG1_AFMT_RAMP_CONTROL1 0x1F45
+#define mmDIG1_AFMT_RAMP_CONTROL2 0x1F46
+#define mmDIG1_AFMT_RAMP_CONTROL3 0x1F47
+#define mmDIG1_AFMT_STATUS 0x1F4A
+#define mmDIG1_AFMT_VBI_PACKET_CONTROL 0x1F4C
+#define mmDIG1_DIG_BE_CNTL 0x1F50
+#define mmDIG1_DIG_BE_EN_CNTL 0x1F51
+#define mmDIG1_DIG_CLOCK_PATTERN 0x1F03
+#define mmDIG1_DIG_DISPCLK_SWITCH_CNTL 0x1F08
+#define mmDIG1_DIG_DISPCLK_SWITCH_STATUS 0x1F09
+#define mmDIG1_DIG_FE_CNTL 0x1F00
+#define mmDIG1_DIG_FIFO_STATUS 0x1F0A
+#define mmDIG1_DIG_LANE_ENABLE 0x1F8D
+#define mmDIG1_DIG_OUTPUT_CRC_CNTL 0x1F01
+#define mmDIG1_DIG_OUTPUT_CRC_RESULT 0x1F02
+#define mmDIG1_DIG_RANDOM_PATTERN_SEED 0x1F05
+#define mmDIG1_DIG_TEST_PATTERN 0x1F04
+#define mmDIG1_HDMI_ACR_32_0 0x1F37
+#define mmDIG1_HDMI_ACR_32_1 0x1F38
+#define mmDIG1_HDMI_ACR_44_0 0x1F39
+#define mmDIG1_HDMI_ACR_44_1 0x1F3A
+#define mmDIG1_HDMI_ACR_48_0 0x1F3B
+#define mmDIG1_HDMI_ACR_48_1 0x1F3C
+#define mmDIG1_HDMI_ACR_PACKET_CONTROL 0x1F0F
+#define mmDIG1_HDMI_ACR_STATUS_0 0x1F3D
+#define mmDIG1_HDMI_ACR_STATUS_1 0x1F3E
+#define mmDIG1_HDMI_AUDIO_PACKET_CONTROL 0x1F0E
+#define mmDIG1_HDMI_CONTROL 0x1F0C
+#define mmDIG1_HDMI_GC 0x1F16
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL0 0x1F13
+#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL1 0x1F30
+#define mmDIG1_HDMI_INFOFRAME_CONTROL0 0x1F11
+#define mmDIG1_HDMI_INFOFRAME_CONTROL1 0x1F12
+#define mmDIG1_HDMI_STATUS 0x1F0D
+#define mmDIG1_HDMI_VBI_PACKET_CONTROL 0x1F10
+#define mmDIG1_LVDS_DATA_CNTL 0x1F8C
+#define mmDIG1_TMDS_CNTL 0x1F7C
+#define mmDIG1_TMDS_CONTROL0_FEEDBACK 0x1F7E
+#define mmDIG1_TMDS_CONTROL_CHAR 0x1F7D
+#define mmDIG1_TMDS_CTL0_1_GEN_CNTL 0x1F86
+#define mmDIG1_TMDS_CTL2_3_GEN_CNTL 0x1F87
+#define mmDIG1_TMDS_CTL_BITS 0x1F83
+#define mmDIG1_TMDS_DCBALANCER_CONTROL 0x1F84
+#define mmDIG1_TMDS_DEBUG 0x1F82
+#define mmDIG1_TMDS_STEREOSYNC_CTL_SEL 0x1F7F
+#define mmDIG1_TMDS_SYNC_CHAR_PATTERN_0_1 0x1F80
+#define mmDIG1_TMDS_SYNC_CHAR_PATTERN_2_3 0x1F81
+#define mmDIG2_AFMT_60958_0 0x4241
+#define mmDIG2_AFMT_60958_1 0x4242
+#define mmDIG2_AFMT_60958_2 0x4248
+#define mmDIG2_AFMT_AUDIO_CRC_CONTROL 0x4243
+#define mmDIG2_AFMT_AUDIO_CRC_RESULT 0x4249
+#define mmDIG2_AFMT_AUDIO_DBG_DTO_CNTL 0x4252
+#define mmDIG2_AFMT_AUDIO_INFO0 0x423F
+#define mmDIG2_AFMT_AUDIO_INFO1 0x4240
+#define mmDIG2_AFMT_AUDIO_PACKET_CONTROL 0x424B
+#define mmDIG2_AFMT_AUDIO_PACKET_CONTROL2 0x4217
+#define mmDIG2_AFMT_AUDIO_SRC_CONTROL 0x424F
+#define mmDIG2_AFMT_AVI_INFO0 0x4221
+#define mmDIG2_AFMT_AVI_INFO1 0x4222
+#define mmDIG2_AFMT_AVI_INFO2 0x4223
+#define mmDIG2_AFMT_AVI_INFO3 0x4224
+#define mmDIG2_AFMT_GENERIC_0 0x4228
+#define mmDIG2_AFMT_GENERIC_1 0x4229
+#define mmDIG2_AFMT_GENERIC_2 0x422A
+#define mmDIG2_AFMT_GENERIC_3 0x422B
+#define mmDIG2_AFMT_GENERIC_4 0x422C
+#define mmDIG2_AFMT_GENERIC_5 0x422D
+#define mmDIG2_AFMT_GENERIC_6 0x422E
+#define mmDIG2_AFMT_GENERIC_7 0x422F
+#define mmDIG2_AFMT_GENERIC_HDR 0x4227
+#define mmDIG2_AFMT_INFOFRAME_CONTROL0 0x424D
+#define mmDIG2_AFMT_INTERRUPT_STATUS 0x4214
+#define mmDIG2_AFMT_ISRC1_0 0x4218
+#define mmDIG2_AFMT_ISRC1_1 0x4219
+#define mmDIG2_AFMT_ISRC1_2 0x421A
+#define mmDIG2_AFMT_ISRC1_3 0x421B
+#define mmDIG2_AFMT_ISRC1_4 0x421C
+#define mmDIG2_AFMT_ISRC2_0 0x421D
+#define mmDIG2_AFMT_ISRC2_1 0x421E
+#define mmDIG2_AFMT_ISRC2_2 0x421F
+#define mmDIG2_AFMT_ISRC2_3 0x4220
+#define mmDIG2_AFMT_MPEG_INFO0 0x4225
+#define mmDIG2_AFMT_MPEG_INFO1 0x4226
+#define mmDIG2_AFMT_RAMP_CONTROL0 0x4244
+#define mmDIG2_AFMT_RAMP_CONTROL1 0x4245
+#define mmDIG2_AFMT_RAMP_CONTROL2 0x4246
+#define mmDIG2_AFMT_RAMP_CONTROL3 0x4247
+#define mmDIG2_AFMT_STATUS 0x424A
+#define mmDIG2_AFMT_VBI_PACKET_CONTROL 0x424C
+#define mmDIG2_DIG_BE_CNTL 0x4250
+#define mmDIG2_DIG_BE_EN_CNTL 0x4251
+#define mmDIG2_DIG_CLOCK_PATTERN 0x4203
+#define mmDIG2_DIG_DISPCLK_SWITCH_CNTL 0x4208
+#define mmDIG2_DIG_DISPCLK_SWITCH_STATUS 0x4209
+#define mmDIG2_DIG_FE_CNTL 0x4200
+#define mmDIG2_DIG_FIFO_STATUS 0x420A
+#define mmDIG2_DIG_LANE_ENABLE 0x428D
+#define mmDIG2_DIG_OUTPUT_CRC_CNTL 0x4201
+#define mmDIG2_DIG_OUTPUT_CRC_RESULT 0x4202
+#define mmDIG2_DIG_RANDOM_PATTERN_SEED 0x4205
+#define mmDIG2_DIG_TEST_PATTERN 0x4204
+#define mmDIG2_HDMI_ACR_32_0 0x4237
+#define mmDIG2_HDMI_ACR_32_1 0x4238
+#define mmDIG2_HDMI_ACR_44_0 0x4239
+#define mmDIG2_HDMI_ACR_44_1 0x423A
+#define mmDIG2_HDMI_ACR_48_0 0x423B
+#define mmDIG2_HDMI_ACR_48_1 0x423C
+#define mmDIG2_HDMI_ACR_PACKET_CONTROL 0x420F
+#define mmDIG2_HDMI_ACR_STATUS_0 0x423D
+#define mmDIG2_HDMI_ACR_STATUS_1 0x423E
+#define mmDIG2_HDMI_AUDIO_PACKET_CONTROL 0x420E
+#define mmDIG2_HDMI_CONTROL 0x420C
+#define mmDIG2_HDMI_GC 0x4216
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL0 0x4213
+#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL1 0x4230
+#define mmDIG2_HDMI_INFOFRAME_CONTROL0 0x4211
+#define mmDIG2_HDMI_INFOFRAME_CONTROL1 0x4212
+#define mmDIG2_HDMI_STATUS 0x420D
+#define mmDIG2_HDMI_VBI_PACKET_CONTROL 0x4210
+#define mmDIG2_LVDS_DATA_CNTL 0x428C
+#define mmDIG2_TMDS_CNTL 0x427C
+#define mmDIG2_TMDS_CONTROL0_FEEDBACK 0x427E
+#define mmDIG2_TMDS_CONTROL_CHAR 0x427D
+#define mmDIG2_TMDS_CTL0_1_GEN_CNTL 0x4286
+#define mmDIG2_TMDS_CTL2_3_GEN_CNTL 0x4287
+#define mmDIG2_TMDS_CTL_BITS 0x4283
+#define mmDIG2_TMDS_DCBALANCER_CONTROL 0x4284
+#define mmDIG2_TMDS_DEBUG 0x4282
+#define mmDIG2_TMDS_STEREOSYNC_CTL_SEL 0x427F
+#define mmDIG2_TMDS_SYNC_CHAR_PATTERN_0_1 0x4280
+#define mmDIG2_TMDS_SYNC_CHAR_PATTERN_2_3 0x4281
+#define mmDIG3_AFMT_60958_0 0x4541
+#define mmDIG3_AFMT_60958_1 0x4542
+#define mmDIG3_AFMT_60958_2 0x4548
+#define mmDIG3_AFMT_AUDIO_CRC_CONTROL 0x4543
+#define mmDIG3_AFMT_AUDIO_CRC_RESULT 0x4549
+#define mmDIG3_AFMT_AUDIO_DBG_DTO_CNTL 0x4552
+#define mmDIG3_AFMT_AUDIO_INFO0 0x453F
+#define mmDIG3_AFMT_AUDIO_INFO1 0x4540
+#define mmDIG3_AFMT_AUDIO_PACKET_CONTROL 0x454B
+#define mmDIG3_AFMT_AUDIO_PACKET_CONTROL2 0x4517
+#define mmDIG3_AFMT_AUDIO_SRC_CONTROL 0x454F
+#define mmDIG3_AFMT_AVI_INFO0 0x4521
+#define mmDIG3_AFMT_AVI_INFO1 0x4522
+#define mmDIG3_AFMT_AVI_INFO2 0x4523
+#define mmDIG3_AFMT_AVI_INFO3 0x4524
+#define mmDIG3_AFMT_GENERIC_0 0x4528
+#define mmDIG3_AFMT_GENERIC_1 0x4529
+#define mmDIG3_AFMT_GENERIC_2 0x452A
+#define mmDIG3_AFMT_GENERIC_3 0x452B
+#define mmDIG3_AFMT_GENERIC_4 0x452C
+#define mmDIG3_AFMT_GENERIC_5 0x452D
+#define mmDIG3_AFMT_GENERIC_6 0x452E
+#define mmDIG3_AFMT_GENERIC_7 0x452F
+#define mmDIG3_AFMT_GENERIC_HDR 0x4527
+#define mmDIG3_AFMT_INFOFRAME_CONTROL0 0x454D
+#define mmDIG3_AFMT_INTERRUPT_STATUS 0x4514
+#define mmDIG3_AFMT_ISRC1_0 0x4518
+#define mmDIG3_AFMT_ISRC1_1 0x4519
+#define mmDIG3_AFMT_ISRC1_2 0x451A
+#define mmDIG3_AFMT_ISRC1_3 0x451B
+#define mmDIG3_AFMT_ISRC1_4 0x451C
+#define mmDIG3_AFMT_ISRC2_0 0x451D
+#define mmDIG3_AFMT_ISRC2_1 0x451E
+#define mmDIG3_AFMT_ISRC2_2 0x451F
+#define mmDIG3_AFMT_ISRC2_3 0x4520
+#define mmDIG3_AFMT_MPEG_INFO0 0x4525
+#define mmDIG3_AFMT_MPEG_INFO1 0x4526
+#define mmDIG3_AFMT_RAMP_CONTROL0 0x4544
+#define mmDIG3_AFMT_RAMP_CONTROL1 0x4545
+#define mmDIG3_AFMT_RAMP_CONTROL2 0x4546
+#define mmDIG3_AFMT_RAMP_CONTROL3 0x4547
+#define mmDIG3_AFMT_STATUS 0x454A
+#define mmDIG3_AFMT_VBI_PACKET_CONTROL 0x454C
+#define mmDIG3_DIG_BE_CNTL 0x4550
+#define mmDIG3_DIG_BE_EN_CNTL 0x4551
+#define mmDIG3_DIG_CLOCK_PATTERN 0x4503
+#define mmDIG3_DIG_DISPCLK_SWITCH_CNTL 0x4508
+#define mmDIG3_DIG_DISPCLK_SWITCH_STATUS 0x4509
+#define mmDIG3_DIG_FE_CNTL 0x4500
+#define mmDIG3_DIG_FIFO_STATUS 0x450A
+#define mmDIG3_DIG_LANE_ENABLE 0x458D
+#define mmDIG3_DIG_OUTPUT_CRC_CNTL 0x4501
+#define mmDIG3_DIG_OUTPUT_CRC_RESULT 0x4502
+#define mmDIG3_DIG_RANDOM_PATTERN_SEED 0x4505
+#define mmDIG3_DIG_TEST_PATTERN 0x4504
+#define mmDIG3_HDMI_ACR_32_0 0x4537
+#define mmDIG3_HDMI_ACR_32_1 0x4538
+#define mmDIG3_HDMI_ACR_44_0 0x4539
+#define mmDIG3_HDMI_ACR_44_1 0x453A
+#define mmDIG3_HDMI_ACR_48_0 0x453B
+#define mmDIG3_HDMI_ACR_48_1 0x453C
+#define mmDIG3_HDMI_ACR_PACKET_CONTROL 0x450F
+#define mmDIG3_HDMI_ACR_STATUS_0 0x453D
+#define mmDIG3_HDMI_ACR_STATUS_1 0x453E
+#define mmDIG3_HDMI_AUDIO_PACKET_CONTROL 0x450E
+#define mmDIG3_HDMI_CONTROL 0x450C
+#define mmDIG3_HDMI_GC 0x4516
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL0 0x4513
+#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL1 0x4530
+#define mmDIG3_HDMI_INFOFRAME_CONTROL0 0x4511
+#define mmDIG3_HDMI_INFOFRAME_CONTROL1 0x4512
+#define mmDIG3_HDMI_STATUS 0x450D
+#define mmDIG3_HDMI_VBI_PACKET_CONTROL 0x4510
+#define mmDIG3_LVDS_DATA_CNTL 0x458C
+#define mmDIG3_TMDS_CNTL 0x457C
+#define mmDIG3_TMDS_CONTROL0_FEEDBACK 0x457E
+#define mmDIG3_TMDS_CONTROL_CHAR 0x457D
+#define mmDIG3_TMDS_CTL0_1_GEN_CNTL 0x4586
+#define mmDIG3_TMDS_CTL2_3_GEN_CNTL 0x4587
+#define mmDIG3_TMDS_CTL_BITS 0x4583
+#define mmDIG3_TMDS_DCBALANCER_CONTROL 0x4584
+#define mmDIG3_TMDS_DEBUG 0x4582
+#define mmDIG3_TMDS_STEREOSYNC_CTL_SEL 0x457F
+#define mmDIG3_TMDS_SYNC_CHAR_PATTERN_0_1 0x4580
+#define mmDIG3_TMDS_SYNC_CHAR_PATTERN_2_3 0x4581
+#define mmDIG4_AFMT_60958_0 0x4841
+#define mmDIG4_AFMT_60958_1 0x4842
+#define mmDIG4_AFMT_60958_2 0x4848
+#define mmDIG4_AFMT_AUDIO_CRC_CONTROL 0x4843
+#define mmDIG4_AFMT_AUDIO_CRC_RESULT 0x4849
+#define mmDIG4_AFMT_AUDIO_DBG_DTO_CNTL 0x4852
+#define mmDIG4_AFMT_AUDIO_INFO0 0x483F
+#define mmDIG4_AFMT_AUDIO_INFO1 0x4840
+#define mmDIG4_AFMT_AUDIO_PACKET_CONTROL 0x484B
+#define mmDIG4_AFMT_AUDIO_PACKET_CONTROL2 0x4817
+#define mmDIG4_AFMT_AUDIO_SRC_CONTROL 0x484F
+#define mmDIG4_AFMT_AVI_INFO0 0x4821
+#define mmDIG4_AFMT_AVI_INFO1 0x4822
+#define mmDIG4_AFMT_AVI_INFO2 0x4823
+#define mmDIG4_AFMT_AVI_INFO3 0x4824
+#define mmDIG4_AFMT_GENERIC_0 0x4828
+#define mmDIG4_AFMT_GENERIC_1 0x4829
+#define mmDIG4_AFMT_GENERIC_2 0x482A
+#define mmDIG4_AFMT_GENERIC_3 0x482B
+#define mmDIG4_AFMT_GENERIC_4 0x482C
+#define mmDIG4_AFMT_GENERIC_5 0x482D
+#define mmDIG4_AFMT_GENERIC_6 0x482E
+#define mmDIG4_AFMT_GENERIC_7 0x482F
+#define mmDIG4_AFMT_GENERIC_HDR 0x4827
+#define mmDIG4_AFMT_INFOFRAME_CONTROL0 0x484D
+#define mmDIG4_AFMT_INTERRUPT_STATUS 0x4814
+#define mmDIG4_AFMT_ISRC1_0 0x4818
+#define mmDIG4_AFMT_ISRC1_1 0x4819
+#define mmDIG4_AFMT_ISRC1_2 0x481A
+#define mmDIG4_AFMT_ISRC1_3 0x481B
+#define mmDIG4_AFMT_ISRC1_4 0x481C
+#define mmDIG4_AFMT_ISRC2_0 0x481D
+#define mmDIG4_AFMT_ISRC2_1 0x481E
+#define mmDIG4_AFMT_ISRC2_2 0x481F
+#define mmDIG4_AFMT_ISRC2_3 0x4820
+#define mmDIG4_AFMT_MPEG_INFO0 0x4825
+#define mmDIG4_AFMT_MPEG_INFO1 0x4826
+#define mmDIG4_AFMT_RAMP_CONTROL0 0x4844
+#define mmDIG4_AFMT_RAMP_CONTROL1 0x4845
+#define mmDIG4_AFMT_RAMP_CONTROL2 0x4846
+#define mmDIG4_AFMT_RAMP_CONTROL3 0x4847
+#define mmDIG4_AFMT_STATUS 0x484A
+#define mmDIG4_AFMT_VBI_PACKET_CONTROL 0x484C
+#define mmDIG4_DIG_BE_CNTL 0x4850
+#define mmDIG4_DIG_BE_EN_CNTL 0x4851
+#define mmDIG4_DIG_CLOCK_PATTERN 0x4803
+#define mmDIG4_DIG_DISPCLK_SWITCH_CNTL 0x4808
+#define mmDIG4_DIG_DISPCLK_SWITCH_STATUS 0x4809
+#define mmDIG4_DIG_FE_CNTL 0x4800
+#define mmDIG4_DIG_FIFO_STATUS 0x480A
+#define mmDIG4_DIG_LANE_ENABLE 0x488D
+#define mmDIG4_DIG_OUTPUT_CRC_CNTL 0x4801
+#define mmDIG4_DIG_OUTPUT_CRC_RESULT 0x4802
+#define mmDIG4_DIG_RANDOM_PATTERN_SEED 0x4805
+#define mmDIG4_DIG_TEST_PATTERN 0x4804
+#define mmDIG4_HDMI_ACR_32_0 0x4837
+#define mmDIG4_HDMI_ACR_32_1 0x4838
+#define mmDIG4_HDMI_ACR_44_0 0x4839
+#define mmDIG4_HDMI_ACR_44_1 0x483A
+#define mmDIG4_HDMI_ACR_48_0 0x483B
+#define mmDIG4_HDMI_ACR_48_1 0x483C
+#define mmDIG4_HDMI_ACR_PACKET_CONTROL 0x480F
+#define mmDIG4_HDMI_ACR_STATUS_0 0x483D
+#define mmDIG4_HDMI_ACR_STATUS_1 0x483E
+#define mmDIG4_HDMI_AUDIO_PACKET_CONTROL 0x480E
+#define mmDIG4_HDMI_CONTROL 0x480C
+#define mmDIG4_HDMI_GC 0x4816
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL0 0x4813
+#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL1 0x4830
+#define mmDIG4_HDMI_INFOFRAME_CONTROL0 0x4811
+#define mmDIG4_HDMI_INFOFRAME_CONTROL1 0x4812
+#define mmDIG4_HDMI_STATUS 0x480D
+#define mmDIG4_HDMI_VBI_PACKET_CONTROL 0x4810
+#define mmDIG4_LVDS_DATA_CNTL 0x488C
+#define mmDIG4_TMDS_CNTL 0x487C
+#define mmDIG4_TMDS_CONTROL0_FEEDBACK 0x487E
+#define mmDIG4_TMDS_CONTROL_CHAR 0x487D
+#define mmDIG4_TMDS_CTL0_1_GEN_CNTL 0x4886
+#define mmDIG4_TMDS_CTL2_3_GEN_CNTL 0x4887
+#define mmDIG4_TMDS_CTL_BITS 0x4883
+#define mmDIG4_TMDS_DCBALANCER_CONTROL 0x4884
+#define mmDIG4_TMDS_DEBUG 0x4882
+#define mmDIG4_TMDS_STEREOSYNC_CTL_SEL 0x487F
+#define mmDIG4_TMDS_SYNC_CHAR_PATTERN_0_1 0x4880
+#define mmDIG4_TMDS_SYNC_CHAR_PATTERN_2_3 0x4881
+#define mmDIG5_AFMT_60958_0 0x4B41
+#define mmDIG5_AFMT_60958_1 0x4B42
+#define mmDIG5_AFMT_60958_2 0x4B48
+#define mmDIG5_AFMT_AUDIO_CRC_CONTROL 0x4B43
+#define mmDIG5_AFMT_AUDIO_CRC_RESULT 0x4B49
+#define mmDIG5_AFMT_AUDIO_DBG_DTO_CNTL 0x4B52
+#define mmDIG5_AFMT_AUDIO_INFO0 0x4B3F
+#define mmDIG5_AFMT_AUDIO_INFO1 0x4B40
+#define mmDIG5_AFMT_AUDIO_PACKET_CONTROL 0x4B4B
+#define mmDIG5_AFMT_AUDIO_PACKET_CONTROL2 0x4B17
+#define mmDIG5_AFMT_AUDIO_SRC_CONTROL 0x4B4F
+#define mmDIG5_AFMT_AVI_INFO0 0x4B21
+#define mmDIG5_AFMT_AVI_INFO1 0x4B22
+#define mmDIG5_AFMT_AVI_INFO2 0x4B23
+#define mmDIG5_AFMT_AVI_INFO3 0x4B24
+#define mmDIG5_AFMT_GENERIC_0 0x4B28
+#define mmDIG5_AFMT_GENERIC_1 0x4B29
+#define mmDIG5_AFMT_GENERIC_2 0x4B2A
+#define mmDIG5_AFMT_GENERIC_3 0x4B2B
+#define mmDIG5_AFMT_GENERIC_4 0x4B2C
+#define mmDIG5_AFMT_GENERIC_5 0x4B2D
+#define mmDIG5_AFMT_GENERIC_6 0x4B2E
+#define mmDIG5_AFMT_GENERIC_7 0x4B2F
+#define mmDIG5_AFMT_GENERIC_HDR 0x4B27
+#define mmDIG5_AFMT_INFOFRAME_CONTROL0 0x4B4D
+#define mmDIG5_AFMT_INTERRUPT_STATUS 0x4B14
+#define mmDIG5_AFMT_ISRC1_0 0x4B18
+#define mmDIG5_AFMT_ISRC1_1 0x4B19
+#define mmDIG5_AFMT_ISRC1_2 0x4B1A
+#define mmDIG5_AFMT_ISRC1_3 0x4B1B
+#define mmDIG5_AFMT_ISRC1_4 0x4B1C
+#define mmDIG5_AFMT_ISRC2_0 0x4B1D
+#define mmDIG5_AFMT_ISRC2_1 0x4B1E
+#define mmDIG5_AFMT_ISRC2_2 0x4B1F
+#define mmDIG5_AFMT_ISRC2_3 0x4B20
+#define mmDIG5_AFMT_MPEG_INFO0 0x4B25
+#define mmDIG5_AFMT_MPEG_INFO1 0x4B26
+#define mmDIG5_AFMT_RAMP_CONTROL0 0x4B44
+#define mmDIG5_AFMT_RAMP_CONTROL1 0x4B45
+#define mmDIG5_AFMT_RAMP_CONTROL2 0x4B46
+#define mmDIG5_AFMT_RAMP_CONTROL3 0x4B47
+#define mmDIG5_AFMT_STATUS 0x4B4A
+#define mmDIG5_AFMT_VBI_PACKET_CONTROL 0x4B4C
+#define mmDIG5_DIG_BE_CNTL 0x4B50
+#define mmDIG5_DIG_BE_EN_CNTL 0x4B51
+#define mmDIG5_DIG_CLOCK_PATTERN 0x4B03
+#define mmDIG5_DIG_DISPCLK_SWITCH_CNTL 0x4B08
+#define mmDIG5_DIG_DISPCLK_SWITCH_STATUS 0x4B09
+#define mmDIG5_DIG_FE_CNTL 0x4B00
+#define mmDIG5_DIG_FIFO_STATUS 0x4B0A
+#define mmDIG5_DIG_LANE_ENABLE 0x4B8D
+#define mmDIG5_DIG_OUTPUT_CRC_CNTL 0x4B01
+#define mmDIG5_DIG_OUTPUT_CRC_RESULT 0x4B02
+#define mmDIG5_DIG_RANDOM_PATTERN_SEED 0x4B05
+#define mmDIG5_DIG_TEST_PATTERN 0x4B04
+#define mmDIG5_HDMI_ACR_32_0 0x4B37
+#define mmDIG5_HDMI_ACR_32_1 0x4B38
+#define mmDIG5_HDMI_ACR_44_0 0x4B39
+#define mmDIG5_HDMI_ACR_44_1 0x4B3A
+#define mmDIG5_HDMI_ACR_48_0 0x4B3B
+#define mmDIG5_HDMI_ACR_48_1 0x4B3C
+#define mmDIG5_HDMI_ACR_PACKET_CONTROL 0x4B0F
+#define mmDIG5_HDMI_ACR_STATUS_0 0x4B3D
+#define mmDIG5_HDMI_ACR_STATUS_1 0x4B3E
+#define mmDIG5_HDMI_AUDIO_PACKET_CONTROL 0x4B0E
+#define mmDIG5_HDMI_CONTROL 0x4B0C
+#define mmDIG5_HDMI_GC 0x4B16
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL0 0x4B13
+#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL1 0x4B30
+#define mmDIG5_HDMI_INFOFRAME_CONTROL0 0x4B11
+#define mmDIG5_HDMI_INFOFRAME_CONTROL1 0x4B12
+#define mmDIG5_HDMI_STATUS 0x4B0D
+#define mmDIG5_HDMI_VBI_PACKET_CONTROL 0x4B10
+#define mmDIG5_LVDS_DATA_CNTL 0x4B8C
+#define mmDIG5_TMDS_CNTL 0x4B7C
+#define mmDIG5_TMDS_CONTROL0_FEEDBACK 0x4B7E
+#define mmDIG5_TMDS_CONTROL_CHAR 0x4B7D
+#define mmDIG5_TMDS_CTL0_1_GEN_CNTL 0x4B86
+#define mmDIG5_TMDS_CTL2_3_GEN_CNTL 0x4B87
+#define mmDIG5_TMDS_CTL_BITS 0x4B83
+#define mmDIG5_TMDS_DCBALANCER_CONTROL 0x4B84
+#define mmDIG5_TMDS_DEBUG 0x4B82
+#define mmDIG5_TMDS_STEREOSYNC_CTL_SEL 0x4B7F
+#define mmDIG5_TMDS_SYNC_CHAR_PATTERN_0_1 0x4B80
+#define mmDIG5_TMDS_SYNC_CHAR_PATTERN_2_3 0x4B81
+#define mmDIG_BE_CNTL 0x1C50
+#define mmDIG_BE_EN_CNTL 0x1C51
+#define mmDIG_CLOCK_PATTERN 0x1C03
+#define mmDIG_DISPCLK_SWITCH_CNTL 0x1C08
+#define mmDIG_DISPCLK_SWITCH_STATUS 0x1C09
+#define mmDIG_FE_CNTL 0x1C00
+#define mmDIG_FIFO_STATUS 0x1C0A
+#define mmDIG_LANE_ENABLE 0x1C8D
+#define mmDIG_OUTPUT_CRC_CNTL 0x1C01
+#define mmDIG_OUTPUT_CRC_RESULT 0x1C02
+#define mmDIG_RANDOM_PATTERN_SEED 0x1C05
+#define mmDIG_SOFT_RESET 0x013D
+#define mmDIG_TEST_PATTERN 0x1C04
+#define mmDISPCLK_CGTT_BLK_CTRL_REG 0x0135
+#define mmDISPCLK_FREQ_CHANGE_CNTL 0x0131
+#define mmDISP_INTERRUPT_STATUS 0x183D
+#define mmDISP_INTERRUPT_STATUS_CONTINUE 0x183E
+#define mmDISP_INTERRUPT_STATUS_CONTINUE2 0x183F
+#define mmDISP_INTERRUPT_STATUS_CONTINUE3 0x1840
+#define mmDISP_INTERRUPT_STATUS_CONTINUE4 0x1853
+#define mmDISP_INTERRUPT_STATUS_CONTINUE5 0x1854
+#define mmDISPOUT_STEREOSYNC_SEL 0x18BF
+#define mmDISPPLL_BG_CNTL 0x013C
+#define mmDISP_TIMER_CONTROL 0x1842
+#define mmDMCU_CTRL 0x1600
+#define mmDMCU_ERAM_RD_CTRL 0x160B
+#define mmDMCU_ERAM_RD_DATA 0x160C
+#define mmDMCU_ERAM_WR_CTRL 0x1609
+#define mmDMCU_ERAM_WR_DATA 0x160A
+#define mmDMCU_EVENT_TRIGGER 0x1611
+#define mmDMCU_FW_CHECKSUM_SMPL_BYTE_POS 0x161A
+#define mmDMCU_FW_CS_HI 0x1606
+#define mmDMCU_FW_CS_LO 0x1607
+#define mmDMCU_FW_END_ADDR 0x1604
+#define mmDMCU_FW_ISR_START_ADDR 0x1605
+#define mmDMCU_FW_START_ADDR 0x1603
+#define mmDMCU_INT_CNT 0x1619
+#define mmDMCU_INTERRUPT_STATUS 0x1614
+#define mmDMCU_INTERRUPT_TO_HOST_EN_MASK 0x1615
+#define mmDMCU_INTERRUPT_TO_UC_EN_MASK 0x1616
+#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL 0x1617
+#define mmDMCU_IRAM_RD_CTRL 0x160F
+#define mmDMCU_IRAM_RD_DATA 0x1610
+#define mmDMCU_IRAM_WR_CTRL 0x160D
+#define mmDMCU_IRAM_WR_DATA 0x160E
+#define mmDMCU_PC_START_ADDR 0x1602
+#define mmDMCU_RAM_ACCESS_CTRL 0x1608
+#define mmDMCU_STATUS 0x1601
+#define mmDMCU_TEST_DEBUG_DATA 0x1627
+#define mmDMCU_TEST_DEBUG_INDEX 0x1626
+#define mmDMCU_UC_CLK_GATING_CNTL 0x161B
+#define mmDMCU_UC_INTERNAL_INT_STATUS 0x1612
+#define mmDMIF_ADDR_CALC 0x0300
+#define mmDMIF_ADDR_CONFIG 0x02F5
+#define mmDMIF_ARBITRATION_CONTROL 0x02F9
+#define mmDMIF_CONTROL 0x02F6
+#define mmDMIF_HW_DEBUG 0x02F8
+#define mmDMIF_PG0_DPG_PIPE_ARBITRATION_CONTROL1 0x1B30
+#define mmDMIF_PG0_DPG_PIPE_ARBITRATION_CONTROL2 0x1B31
+#define mmDMIF_PG0_DPG_PIPE_DPM_CONTROL 0x1B34
+#define mmDMIF_PG0_DPG_PIPE_NB_PSTATE_CHANGE_CONTROL 0x1B36
+#define mmDMIF_PG0_DPG_PIPE_STUTTER_CONTROL 0x1B35
+#define mmDMIF_PG0_DPG_PIPE_STUTTER_CONTROL_NONLPTCH 0x1B37
+#define mmDMIF_PG0_DPG_PIPE_URGENCY_CONTROL 0x1B33
+#define mmDMIF_PG0_DPG_TEST_DEBUG_DATA 0x1B39
+#define mmDMIF_PG0_DPG_TEST_DEBUG_INDEX 0x1B38
+#define mmDMIF_PG1_DPG_PIPE_ARBITRATION_CONTROL1 0x1E30
+#define mmDMIF_PG1_DPG_PIPE_ARBITRATION_CONTROL2 0x1E31
+#define mmDMIF_PG1_DPG_PIPE_DPM_CONTROL 0x1E34
+#define mmDMIF_PG1_DPG_PIPE_NB_PSTATE_CHANGE_CONTROL 0x1E36
+#define mmDMIF_PG1_DPG_PIPE_STUTTER_CONTROL 0x1E35
+#define mmDMIF_PG1_DPG_PIPE_STUTTER_CONTROL_NONLPTCH 0x1E37
+#define mmDMIF_PG1_DPG_PIPE_URGENCY_CONTROL 0x1E33
+#define mmDMIF_PG1_DPG_TEST_DEBUG_DATA 0x1E39
+#define mmDMIF_PG1_DPG_TEST_DEBUG_INDEX 0x1E38
+#define mmDMIF_PG2_DPG_PIPE_ARBITRATION_CONTROL1 0x4130
+#define mmDMIF_PG2_DPG_PIPE_ARBITRATION_CONTROL2 0x4131
+#define mmDMIF_PG2_DPG_PIPE_DPM_CONTROL 0x4134
+#define mmDMIF_PG2_DPG_PIPE_NB_PSTATE_CHANGE_CONTROL 0x4136
+#define mmDMIF_PG2_DPG_PIPE_STUTTER_CONTROL 0x4135
+#define mmDMIF_PG2_DPG_PIPE_STUTTER_CONTROL_NONLPTCH 0x4137
+#define mmDMIF_PG2_DPG_PIPE_URGENCY_CONTROL 0x4133
+#define mmDMIF_PG2_DPG_TEST_DEBUG_DATA 0x4139
+#define mmDMIF_PG2_DPG_TEST_DEBUG_INDEX 0x4138
+#define mmDMIF_PG3_DPG_PIPE_ARBITRATION_CONTROL1 0x4430
+#define mmDMIF_PG3_DPG_PIPE_ARBITRATION_CONTROL2 0x4431
+#define mmDMIF_PG3_DPG_PIPE_DPM_CONTROL 0x4434
+#define mmDMIF_PG3_DPG_PIPE_NB_PSTATE_CHANGE_CONTROL 0x4436
+#define mmDMIF_PG3_DPG_PIPE_STUTTER_CONTROL 0x4435
+#define mmDMIF_PG3_DPG_PIPE_STUTTER_CONTROL_NONLPTCH 0x4437
+#define mmDMIF_PG3_DPG_PIPE_URGENCY_CONTROL 0x4433
+#define mmDMIF_PG3_DPG_TEST_DEBUG_DATA 0x4439
+#define mmDMIF_PG3_DPG_TEST_DEBUG_INDEX 0x4438
+#define mmDMIF_PG4_DPG_PIPE_ARBITRATION_CONTROL1 0x4730
+#define mmDMIF_PG4_DPG_PIPE_ARBITRATION_CONTROL2 0x4731
+#define mmDMIF_PG4_DPG_PIPE_DPM_CONTROL 0x4734
+#define mmDMIF_PG4_DPG_PIPE_NB_PSTATE_CHANGE_CONTROL 0x4736
+#define mmDMIF_PG4_DPG_PIPE_STUTTER_CONTROL 0x4735
+#define mmDMIF_PG4_DPG_PIPE_STUTTER_CONTROL_NONLPTCH 0x4737
+#define mmDMIF_PG4_DPG_PIPE_URGENCY_CONTROL 0x4733
+#define mmDMIF_PG4_DPG_TEST_DEBUG_DATA 0x4739
+#define mmDMIF_PG4_DPG_TEST_DEBUG_INDEX 0x4738
+#define mmDMIF_PG5_DPG_PIPE_ARBITRATION_CONTROL1 0x4A30
+#define mmDMIF_PG5_DPG_PIPE_ARBITRATION_CONTROL2 0x4A31
+#define mmDMIF_PG5_DPG_PIPE_DPM_CONTROL 0x4A34
+#define mmDMIF_PG5_DPG_PIPE_NB_PSTATE_CHANGE_CONTROL 0x4A36
+#define mmDMIF_PG5_DPG_PIPE_STUTTER_CONTROL 0x4A35
+#define mmDMIF_PG5_DPG_PIPE_STUTTER_CONTROL_NONLPTCH 0x4A37
+#define mmDMIF_PG5_DPG_PIPE_URGENCY_CONTROL 0x4A33
+#define mmDMIF_PG5_DPG_TEST_DEBUG_DATA 0x4A39
+#define mmDMIF_PG5_DPG_TEST_DEBUG_INDEX 0x4A38
+#define mmDMIF_STATUS 0x02F7
+#define mmDMIF_STATUS2 0x0301
+#define mmDMIF_TEST_DEBUG_DATA 0x0313
+#define mmDMIF_TEST_DEBUG_INDEX 0x0312
+#define mmDOUT_DCE_VCE_CONTROL 0x18FF
+#define mmDOUT_POWER_MANAGEMENT_CNTL 0x1841
+#define mmDOUT_SCRATCH0 0x1844
+#define mmDOUT_SCRATCH1 0x1845
+#define mmDOUT_SCRATCH2 0x1846
+#define mmDOUT_SCRATCH3 0x1847
+#define mmDOUT_SCRATCH4 0x1848
+#define mmDOUT_SCRATCH5 0x1849
+#define mmDOUT_SCRATCH6 0x184A
+#define mmDOUT_SCRATCH7 0x184B
+#define mmDOUT_TEST_DEBUG_DATA 0x184E
+#define mmDOUT_TEST_DEBUG_INDEX 0x184D
+#define mmDP0_DP_CONFIG 0x1CC2
+#define mmDP0_DP_DPHY_8B10B_CNTL 0x1CD3
+#define mmDP0_DP_DPHY_CNTL 0x1CD0
+#define mmDP0_DP_DPHY_CRC_CNTL 0x1CD7
+#define mmDP0_DP_DPHY_CRC_EN 0x1CD6
+#define mmDP0_DP_DPHY_CRC_MST_CNTL 0x1CC6
+#define mmDP0_DP_DPHY_CRC_MST_STATUS 0x1CC7
+#define mmDP0_DP_DPHY_CRC_RESULT 0x1CD8
+#define mmDP0_DP_DPHY_FAST_TRAINING 0x1CCE
+#define mmDP0_DP_DPHY_FAST_TRAINING_STATUS 0x1CE9
+#define mmDP0_DP_DPHY_PRBS_CNTL 0x1CD4
+#define mmDP0_DP_DPHY_SYM0 0x1CD2
+#define mmDP0_DP_DPHY_SYM1 0x1CE0
+#define mmDP0_DP_DPHY_SYM2 0x1CDF
+#define mmDP0_DP_DPHY_TRAINING_PATTERN_SEL 0x1CD1
+#define mmDP0_DP_HBR2_EYE_PATTERN 0x1CC8
+#define mmDP0_DP_LINK_CNTL 0x1CC0
+#define mmDP0_DP_LINK_FRAMING_CNTL 0x1CCC
+#define mmDP0_DP_MSA_COLORIMETRY 0x1CDA
+#define mmDP0_DP_MSA_MISC 0x1CC5
+#define mmDP0_DP_MSA_V_TIMING_OVERRIDE1 0x1CEA
+#define mmDP0_DP_MSA_V_TIMING_OVERRIDE2 0x1CEB
+#define mmDP0_DP_MSE_LINK_TIMING 0x1CE8
+#define mmDP0_DP_MSE_MISC_CNTL 0x1CDB
+#define mmDP0_DP_MSE_RATE_CNTL 0x1CE1
+#define mmDP0_DP_MSE_RATE_UPDATE 0x1CE3
+#define mmDP0_DP_MSE_SAT0 0x1CE4
+#define mmDP0_DP_MSE_SAT1 0x1CE5
+#define mmDP0_DP_MSE_SAT2 0x1CE6
+#define mmDP0_DP_MSE_SAT_UPDATE 0x1CE7
+#define mmDP0_DP_PIXEL_FORMAT 0x1CC1
+#define mmDP0_DP_SEC_AUD_M 0x1CA7
+#define mmDP0_DP_SEC_AUD_M_READBACK 0x1CA8
+#define mmDP0_DP_SEC_AUD_N 0x1CA5
+#define mmDP0_DP_SEC_AUD_N_READBACK 0x1CA6
+#define mmDP0_DP_SEC_CNTL 0x1CA0
+#define mmDP0_DP_SEC_CNTL1 0x1CAB
+#define mmDP0_DP_SEC_FRAMING1 0x1CA1
+#define mmDP0_DP_SEC_FRAMING2 0x1CA2
+#define mmDP0_DP_SEC_FRAMING3 0x1CA3
+#define mmDP0_DP_SEC_FRAMING4 0x1CA4
+#define mmDP0_DP_SEC_PACKET_CNTL 0x1CAA
+#define mmDP0_DP_SEC_TIMESTAMP 0x1CA9
+#define mmDP0_DP_STEER_FIFO 0x1CC4
+#define mmDP0_DP_TEST_DEBUG_DATA 0x1CFD
+#define mmDP0_DP_TEST_DEBUG_INDEX 0x1CFC
+#define mmDP0_DP_VID_INTERRUPT_CNTL 0x1CCF
+#define mmDP0_DP_VID_M 0x1CCB
+#define mmDP0_DP_VID_MSA_VBID 0x1CCD
+#define mmDP0_DP_VID_N 0x1CCA
+#define mmDP0_DP_VID_STREAM_CNTL 0x1CC3
+#define mmDP0_DP_VID_TIMING 0x1CC9
+#define mmDP1_DP_CONFIG 0x1FC2
+#define mmDP1_DP_DPHY_8B10B_CNTL 0x1FD3
+#define mmDP1_DP_DPHY_CNTL 0x1FD0
+#define mmDP1_DP_DPHY_CRC_CNTL 0x1FD7
+#define mmDP1_DP_DPHY_CRC_EN 0x1FD6
+#define mmDP1_DP_DPHY_CRC_MST_CNTL 0x1FC6
+#define mmDP1_DP_DPHY_CRC_MST_STATUS 0x1FC7
+#define mmDP1_DP_DPHY_CRC_RESULT 0x1FD8
+#define mmDP1_DP_DPHY_FAST_TRAINING 0x1FCE
+#define mmDP1_DP_DPHY_FAST_TRAINING_STATUS 0x1FE9
+#define mmDP1_DP_DPHY_PRBS_CNTL 0x1FD4
+#define mmDP1_DP_DPHY_SYM0 0x1FD2
+#define mmDP1_DP_DPHY_SYM1 0x1FE0
+#define mmDP1_DP_DPHY_SYM2 0x1FDF
+#define mmDP1_DP_DPHY_TRAINING_PATTERN_SEL 0x1FD1
+#define mmDP1_DP_HBR2_EYE_PATTERN 0x1FC8
+#define mmDP1_DP_LINK_CNTL 0x1FC0
+#define mmDP1_DP_LINK_FRAMING_CNTL 0x1FCC
+#define mmDP1_DP_MSA_COLORIMETRY 0x1FDA
+#define mmDP1_DP_MSA_MISC 0x1FC5
+#define mmDP1_DP_MSA_V_TIMING_OVERRIDE1 0x1FEA
+#define mmDP1_DP_MSA_V_TIMING_OVERRIDE2 0x1FEB
+#define mmDP1_DP_MSE_LINK_TIMING 0x1FE8
+#define mmDP1_DP_MSE_MISC_CNTL 0x1FDB
+#define mmDP1_DP_MSE_RATE_CNTL 0x1FE1
+#define mmDP1_DP_MSE_RATE_UPDATE 0x1FE3
+#define mmDP1_DP_MSE_SAT0 0x1FE4
+#define mmDP1_DP_MSE_SAT1 0x1FE5
+#define mmDP1_DP_MSE_SAT2 0x1FE6
+#define mmDP1_DP_MSE_SAT_UPDATE 0x1FE7
+#define mmDP1_DP_PIXEL_FORMAT 0x1FC1
+#define mmDP1_DP_SEC_AUD_M 0x1FA7
+#define mmDP1_DP_SEC_AUD_M_READBACK 0x1FA8
+#define mmDP1_DP_SEC_AUD_N 0x1FA5
+#define mmDP1_DP_SEC_AUD_N_READBACK 0x1FA6
+#define mmDP1_DP_SEC_CNTL 0x1FA0
+#define mmDP1_DP_SEC_CNTL1 0x1FAB
+#define mmDP1_DP_SEC_FRAMING1 0x1FA1
+#define mmDP1_DP_SEC_FRAMING2 0x1FA2
+#define mmDP1_DP_SEC_FRAMING3 0x1FA3
+#define mmDP1_DP_SEC_FRAMING4 0x1FA4
+#define mmDP1_DP_SEC_PACKET_CNTL 0x1FAA
+#define mmDP1_DP_SEC_TIMESTAMP 0x1FA9
+#define mmDP1_DP_STEER_FIFO 0x1FC4
+#define mmDP1_DP_TEST_DEBUG_DATA 0x1FFD
+#define mmDP1_DP_TEST_DEBUG_INDEX 0x1FFC
+#define mmDP1_DP_VID_INTERRUPT_CNTL 0x1FCF
+#define mmDP1_DP_VID_M 0x1FCB
+#define mmDP1_DP_VID_MSA_VBID 0x1FCD
+#define mmDP1_DP_VID_N 0x1FCA
+#define mmDP1_DP_VID_STREAM_CNTL 0x1FC3
+#define mmDP1_DP_VID_TIMING 0x1FC9
+#define mmDP2_DP_CONFIG 0x42C2
+#define mmDP2_DP_DPHY_8B10B_CNTL 0x42D3
+#define mmDP2_DP_DPHY_CNTL 0x42D0
+#define mmDP2_DP_DPHY_CRC_CNTL 0x42D7
+#define mmDP2_DP_DPHY_CRC_EN 0x42D6
+#define mmDP2_DP_DPHY_CRC_MST_CNTL 0x42C6
+#define mmDP2_DP_DPHY_CRC_MST_STATUS 0x42C7
+#define mmDP2_DP_DPHY_CRC_RESULT 0x42D8
+#define mmDP2_DP_DPHY_FAST_TRAINING 0x42CE
+#define mmDP2_DP_DPHY_FAST_TRAINING_STATUS 0x42E9
+#define mmDP2_DP_DPHY_PRBS_CNTL 0x42D4
+#define mmDP2_DP_DPHY_SYM0 0x42D2
+#define mmDP2_DP_DPHY_SYM1 0x42E0
+#define mmDP2_DP_DPHY_SYM2 0x42DF
+#define mmDP2_DP_DPHY_TRAINING_PATTERN_SEL 0x42D1
+#define mmDP2_DP_HBR2_EYE_PATTERN 0x42C8
+#define mmDP2_DP_LINK_CNTL 0x42C0
+#define mmDP2_DP_LINK_FRAMING_CNTL 0x42CC
+#define mmDP2_DP_MSA_COLORIMETRY 0x42DA
+#define mmDP2_DP_MSA_MISC 0x42C5
+#define mmDP2_DP_MSA_V_TIMING_OVERRIDE1 0x42EA
+#define mmDP2_DP_MSA_V_TIMING_OVERRIDE2 0x42EB
+#define mmDP2_DP_MSE_LINK_TIMING 0x42E8
+#define mmDP2_DP_MSE_MISC_CNTL 0x42DB
+#define mmDP2_DP_MSE_RATE_CNTL 0x42E1
+#define mmDP2_DP_MSE_RATE_UPDATE 0x42E3
+#define mmDP2_DP_MSE_SAT0 0x42E4
+#define mmDP2_DP_MSE_SAT1 0x42E5
+#define mmDP2_DP_MSE_SAT2 0x42E6
+#define mmDP2_DP_MSE_SAT_UPDATE 0x42E7
+#define mmDP2_DP_PIXEL_FORMAT 0x42C1
+#define mmDP2_DP_SEC_AUD_M 0x42A7
+#define mmDP2_DP_SEC_AUD_M_READBACK 0x42A8
+#define mmDP2_DP_SEC_AUD_N 0x42A5
+#define mmDP2_DP_SEC_AUD_N_READBACK 0x42A6
+#define mmDP2_DP_SEC_CNTL 0x42A0
+#define mmDP2_DP_SEC_CNTL1 0x42AB
+#define mmDP2_DP_SEC_FRAMING1 0x42A1
+#define mmDP2_DP_SEC_FRAMING2 0x42A2
+#define mmDP2_DP_SEC_FRAMING3 0x42A3
+#define mmDP2_DP_SEC_FRAMING4 0x42A4
+#define mmDP2_DP_SEC_PACKET_CNTL 0x42AA
+#define mmDP2_DP_SEC_TIMESTAMP 0x42A9
+#define mmDP2_DP_STEER_FIFO 0x42C4
+#define mmDP2_DP_TEST_DEBUG_DATA 0x42FD
+#define mmDP2_DP_TEST_DEBUG_INDEX 0x42FC
+#define mmDP2_DP_VID_INTERRUPT_CNTL 0x42CF
+#define mmDP2_DP_VID_M 0x42CB
+#define mmDP2_DP_VID_MSA_VBID 0x42CD
+#define mmDP2_DP_VID_N 0x42CA
+#define mmDP2_DP_VID_STREAM_CNTL 0x42C3
+#define mmDP2_DP_VID_TIMING 0x42C9
+#define mmDP3_DP_CONFIG 0x45C2
+#define mmDP3_DP_DPHY_8B10B_CNTL 0x45D3
+#define mmDP3_DP_DPHY_CNTL 0x45D0
+#define mmDP3_DP_DPHY_CRC_CNTL 0x45D7
+#define mmDP3_DP_DPHY_CRC_EN 0x45D6
+#define mmDP3_DP_DPHY_CRC_MST_CNTL 0x45C6
+#define mmDP3_DP_DPHY_CRC_MST_STATUS 0x45C7
+#define mmDP3_DP_DPHY_CRC_RESULT 0x45D8
+#define mmDP3_DP_DPHY_FAST_TRAINING 0x45CE
+#define mmDP3_DP_DPHY_FAST_TRAINING_STATUS 0x45E9
+#define mmDP3_DP_DPHY_PRBS_CNTL 0x45D4
+#define mmDP3_DP_DPHY_SYM0 0x45D2
+#define mmDP3_DP_DPHY_SYM1 0x45E0
+#define mmDP3_DP_DPHY_SYM2 0x45DF
+#define mmDP3_DP_DPHY_TRAINING_PATTERN_SEL 0x45D1
+#define mmDP3_DP_HBR2_EYE_PATTERN 0x45C8
+#define mmDP3_DP_LINK_CNTL 0x45C0
+#define mmDP3_DP_LINK_FRAMING_CNTL 0x45CC
+#define mmDP3_DP_MSA_COLORIMETRY 0x45DA
+#define mmDP3_DP_MSA_MISC 0x45C5
+#define mmDP3_DP_MSA_V_TIMING_OVERRIDE1 0x45EA
+#define mmDP3_DP_MSA_V_TIMING_OVERRIDE2 0x45EB
+#define mmDP3_DP_MSE_LINK_TIMING 0x45E8
+#define mmDP3_DP_MSE_MISC_CNTL 0x45DB
+#define mmDP3_DP_MSE_RATE_CNTL 0x45E1
+#define mmDP3_DP_MSE_RATE_UPDATE 0x45E3
+#define mmDP3_DP_MSE_SAT0 0x45E4
+#define mmDP3_DP_MSE_SAT1 0x45E5
+#define mmDP3_DP_MSE_SAT2 0x45E6
+#define mmDP3_DP_MSE_SAT_UPDATE 0x45E7
+#define mmDP3_DP_PIXEL_FORMAT 0x45C1
+#define mmDP3_DP_SEC_AUD_M 0x45A7
+#define mmDP3_DP_SEC_AUD_M_READBACK 0x45A8
+#define mmDP3_DP_SEC_AUD_N 0x45A5
+#define mmDP3_DP_SEC_AUD_N_READBACK 0x45A6
+#define mmDP3_DP_SEC_CNTL 0x45A0
+#define mmDP3_DP_SEC_CNTL1 0x45AB
+#define mmDP3_DP_SEC_FRAMING1 0x45A1
+#define mmDP3_DP_SEC_FRAMING2 0x45A2
+#define mmDP3_DP_SEC_FRAMING3 0x45A3
+#define mmDP3_DP_SEC_FRAMING4 0x45A4
+#define mmDP3_DP_SEC_PACKET_CNTL 0x45AA
+#define mmDP3_DP_SEC_TIMESTAMP 0x45A9
+#define mmDP3_DP_STEER_FIFO 0x45C4
+#define mmDP3_DP_TEST_DEBUG_DATA 0x45FD
+#define mmDP3_DP_TEST_DEBUG_INDEX 0x45FC
+#define mmDP3_DP_VID_INTERRUPT_CNTL 0x45CF
+#define mmDP3_DP_VID_M 0x45CB
+#define mmDP3_DP_VID_MSA_VBID 0x45CD
+#define mmDP3_DP_VID_N 0x45CA
+#define mmDP3_DP_VID_STREAM_CNTL 0x45C3
+#define mmDP3_DP_VID_TIMING 0x45C9
+#define mmDP4_DP_CONFIG 0x48C2
+#define mmDP4_DP_DPHY_8B10B_CNTL 0x48D3
+#define mmDP4_DP_DPHY_CNTL 0x48D0
+#define mmDP4_DP_DPHY_CRC_CNTL 0x48D7
+#define mmDP4_DP_DPHY_CRC_EN 0x48D6
+#define mmDP4_DP_DPHY_CRC_MST_CNTL 0x48C6
+#define mmDP4_DP_DPHY_CRC_MST_STATUS 0x48C7
+#define mmDP4_DP_DPHY_CRC_RESULT 0x48D8
+#define mmDP4_DP_DPHY_FAST_TRAINING 0x48CE
+#define mmDP4_DP_DPHY_FAST_TRAINING_STATUS 0x48E9
+#define mmDP4_DP_DPHY_PRBS_CNTL 0x48D4
+#define mmDP4_DP_DPHY_SYM0 0x48D2
+#define mmDP4_DP_DPHY_SYM1 0x48E0
+#define mmDP4_DP_DPHY_SYM2 0x48DF
+#define mmDP4_DP_DPHY_TRAINING_PATTERN_SEL 0x48D1
+#define mmDP4_DP_HBR2_EYE_PATTERN 0x48C8
+#define mmDP4_DP_LINK_CNTL 0x48C0
+#define mmDP4_DP_LINK_FRAMING_CNTL 0x48CC
+#define mmDP4_DP_MSA_COLORIMETRY 0x48DA
+#define mmDP4_DP_MSA_MISC 0x48C5
+#define mmDP4_DP_MSA_V_TIMING_OVERRIDE1 0x48EA
+#define mmDP4_DP_MSA_V_TIMING_OVERRIDE2 0x48EB
+#define mmDP4_DP_MSE_LINK_TIMING 0x48E8
+#define mmDP4_DP_MSE_MISC_CNTL 0x48DB
+#define mmDP4_DP_MSE_RATE_CNTL 0x48E1
+#define mmDP4_DP_MSE_RATE_UPDATE 0x48E3
+#define mmDP4_DP_MSE_SAT0 0x48E4
+#define mmDP4_DP_MSE_SAT1 0x48E5
+#define mmDP4_DP_MSE_SAT2 0x48E6
+#define mmDP4_DP_MSE_SAT_UPDATE 0x48E7
+#define mmDP4_DP_PIXEL_FORMAT 0x48C1
+#define mmDP4_DP_SEC_AUD_M 0x48A7
+#define mmDP4_DP_SEC_AUD_M_READBACK 0x48A8
+#define mmDP4_DP_SEC_AUD_N 0x48A5
+#define mmDP4_DP_SEC_AUD_N_READBACK 0x48A6
+#define mmDP4_DP_SEC_CNTL 0x48A0
+#define mmDP4_DP_SEC_CNTL1 0x48AB
+#define mmDP4_DP_SEC_FRAMING1 0x48A1
+#define mmDP4_DP_SEC_FRAMING2 0x48A2
+#define mmDP4_DP_SEC_FRAMING3 0x48A3
+#define mmDP4_DP_SEC_FRAMING4 0x48A4
+#define mmDP4_DP_SEC_PACKET_CNTL 0x48AA
+#define mmDP4_DP_SEC_TIMESTAMP 0x48A9
+#define mmDP4_DP_STEER_FIFO 0x48C4
+#define mmDP4_DP_TEST_DEBUG_DATA 0x48FD
+#define mmDP4_DP_TEST_DEBUG_INDEX 0x48FC
+#define mmDP4_DP_VID_INTERRUPT_CNTL 0x48CF
+#define mmDP4_DP_VID_M 0x48CB
+#define mmDP4_DP_VID_MSA_VBID 0x48CD
+#define mmDP4_DP_VID_N 0x48CA
+#define mmDP4_DP_VID_STREAM_CNTL 0x48C3
+#define mmDP4_DP_VID_TIMING 0x48C9
+#define mmDP5_DP_CONFIG 0x4BC2
+#define mmDP5_DP_DPHY_8B10B_CNTL 0x4BD3
+#define mmDP5_DP_DPHY_CNTL 0x4BD0
+#define mmDP5_DP_DPHY_CRC_CNTL 0x4BD7
+#define mmDP5_DP_DPHY_CRC_EN 0x4BD6
+#define mmDP5_DP_DPHY_CRC_MST_CNTL 0x4BC6
+#define mmDP5_DP_DPHY_CRC_MST_STATUS 0x4BC7
+#define mmDP5_DP_DPHY_CRC_RESULT 0x4BD8
+#define mmDP5_DP_DPHY_FAST_TRAINING 0x4BCE
+#define mmDP5_DP_DPHY_FAST_TRAINING_STATUS 0x4BE9
+#define mmDP5_DP_DPHY_PRBS_CNTL 0x4BD4
+#define mmDP5_DP_DPHY_SYM0 0x4BD2
+#define mmDP5_DP_DPHY_SYM1 0x4BE0
+#define mmDP5_DP_DPHY_SYM2 0x4BDF
+#define mmDP5_DP_DPHY_TRAINING_PATTERN_SEL 0x4BD1
+#define mmDP5_DP_HBR2_EYE_PATTERN 0x4BC8
+#define mmDP5_DP_LINK_CNTL 0x4BC0
+#define mmDP5_DP_LINK_FRAMING_CNTL 0x4BCC
+#define mmDP5_DP_MSA_COLORIMETRY 0x4BDA
+#define mmDP5_DP_MSA_MISC 0x4BC5
+#define mmDP5_DP_MSA_V_TIMING_OVERRIDE1 0x4BEA
+#define mmDP5_DP_MSA_V_TIMING_OVERRIDE2 0x4BEB
+#define mmDP5_DP_MSE_LINK_TIMING 0x4BE8
+#define mmDP5_DP_MSE_MISC_CNTL 0x4BDB
+#define mmDP5_DP_MSE_RATE_CNTL 0x4BE1
+#define mmDP5_DP_MSE_RATE_UPDATE 0x4BE3
+#define mmDP5_DP_MSE_SAT0 0x4BE4
+#define mmDP5_DP_MSE_SAT1 0x4BE5
+#define mmDP5_DP_MSE_SAT2 0x4BE6
+#define mmDP5_DP_MSE_SAT_UPDATE 0x4BE7
+#define mmDP5_DP_PIXEL_FORMAT 0x4BC1
+#define mmDP5_DP_SEC_AUD_M 0x4BA7
+#define mmDP5_DP_SEC_AUD_M_READBACK 0x4BA8
+#define mmDP5_DP_SEC_AUD_N 0x4BA5
+#define mmDP5_DP_SEC_AUD_N_READBACK 0x4BA6
+#define mmDP5_DP_SEC_CNTL 0x4BA0
+#define mmDP5_DP_SEC_CNTL1 0x4BAB
+#define mmDP5_DP_SEC_FRAMING1 0x4BA1
+#define mmDP5_DP_SEC_FRAMING2 0x4BA2
+#define mmDP5_DP_SEC_FRAMING3 0x4BA3
+#define mmDP5_DP_SEC_FRAMING4 0x4BA4
+#define mmDP5_DP_SEC_PACKET_CNTL 0x4BAA
+#define mmDP5_DP_SEC_TIMESTAMP 0x4BA9
+#define mmDP5_DP_STEER_FIFO 0x4BC4
+#define mmDP5_DP_TEST_DEBUG_DATA 0x4BFD
+#define mmDP5_DP_TEST_DEBUG_INDEX 0x4BFC
+#define mmDP5_DP_VID_INTERRUPT_CNTL 0x4BCF
+#define mmDP5_DP_VID_M 0x4BCB
+#define mmDP5_DP_VID_MSA_VBID 0x4BCD
+#define mmDP5_DP_VID_N 0x4BCA
+#define mmDP5_DP_VID_STREAM_CNTL 0x4BC3
+#define mmDP5_DP_VID_TIMING 0x4BC9
+#define mmDP_AUX0_AUX_ARB_CONTROL 0x1882
+#define mmDP_AUX0_AUX_CONTROL 0x1880
+#define mmDP_AUX0_AUX_DPHY_RX_CONTROL0 0x188A
+#define mmDP_AUX0_AUX_DPHY_RX_CONTROL1 0x188B
+#define mmDP_AUX0_AUX_DPHY_RX_STATUS 0x188D
+#define mmDP_AUX0_AUX_DPHY_TX_CONTROL 0x1889
+#define mmDP_AUX0_AUX_DPHY_TX_REF_CONTROL 0x1888
+#define mmDP_AUX0_AUX_DPHY_TX_STATUS 0x188C
+#define mmDP_AUX0_AUX_GTC_SYNC_CONTROL 0x188E
+#define mmDP_AUX0_AUX_GTC_SYNC_DATA 0x1890
+#define mmDP_AUX0_AUX_INTERRUPT_CONTROL 0x1883
+#define mmDP_AUX0_AUX_LS_DATA 0x1887
+#define mmDP_AUX0_AUX_LS_STATUS 0x1885
+#define mmDP_AUX0_AUX_SW_CONTROL 0x1881
+#define mmDP_AUX0_AUX_SW_DATA 0x1886
+#define mmDP_AUX0_AUX_SW_STATUS 0x1884
+#define mmDP_AUX1_AUX_ARB_CONTROL 0x1896
+#define mmDP_AUX1_AUX_CONTROL 0x1894
+#define mmDP_AUX1_AUX_DPHY_RX_CONTROL0 0x189E
+#define mmDP_AUX1_AUX_DPHY_RX_CONTROL1 0x189F
+#define mmDP_AUX1_AUX_DPHY_RX_STATUS 0x18A1
+#define mmDP_AUX1_AUX_DPHY_TX_CONTROL 0x189D
+#define mmDP_AUX1_AUX_DPHY_TX_REF_CONTROL 0x189C
+#define mmDP_AUX1_AUX_DPHY_TX_STATUS 0x18A0
+#define mmDP_AUX1_AUX_GTC_SYNC_CONTROL 0x18A2
+#define mmDP_AUX1_AUX_GTC_SYNC_DATA 0x18A4
+#define mmDP_AUX1_AUX_INTERRUPT_CONTROL 0x1897
+#define mmDP_AUX1_AUX_LS_DATA 0x189B
+#define mmDP_AUX1_AUX_LS_STATUS 0x1899
+#define mmDP_AUX1_AUX_SW_CONTROL 0x1895
+#define mmDP_AUX1_AUX_SW_DATA 0x189A
+#define mmDP_AUX1_AUX_SW_STATUS 0x1898
+#define mmDP_AUX2_AUX_ARB_CONTROL 0x18AA
+#define mmDP_AUX2_AUX_CONTROL 0x18A8
+#define mmDP_AUX2_AUX_DPHY_RX_CONTROL0 0x18B2
+#define mmDP_AUX2_AUX_DPHY_RX_CONTROL1 0x18B3
+#define mmDP_AUX2_AUX_DPHY_RX_STATUS 0x18B5
+#define mmDP_AUX2_AUX_DPHY_TX_CONTROL 0x18B1
+#define mmDP_AUX2_AUX_DPHY_TX_REF_CONTROL 0x18B0
+#define mmDP_AUX2_AUX_DPHY_TX_STATUS 0x18B4
+#define mmDP_AUX2_AUX_GTC_SYNC_CONTROL 0x18B6
+#define mmDP_AUX2_AUX_GTC_SYNC_DATA 0x18B8
+#define mmDP_AUX2_AUX_INTERRUPT_CONTROL 0x18AB
+#define mmDP_AUX2_AUX_LS_DATA 0x18AF
+#define mmDP_AUX2_AUX_LS_STATUS 0x18AD
+#define mmDP_AUX2_AUX_SW_CONTROL 0x18A9
+#define mmDP_AUX2_AUX_SW_DATA 0x18AE
+#define mmDP_AUX2_AUX_SW_STATUS 0x18AC
+#define mmDP_AUX3_AUX_ARB_CONTROL 0x18C2
+#define mmDP_AUX3_AUX_CONTROL 0x18C0
+#define mmDP_AUX3_AUX_DPHY_RX_CONTROL0 0x18CA
+#define mmDP_AUX3_AUX_DPHY_RX_CONTROL1 0x18CB
+#define mmDP_AUX3_AUX_DPHY_RX_STATUS 0x18CD
+#define mmDP_AUX3_AUX_DPHY_TX_CONTROL 0x18C9
+#define mmDP_AUX3_AUX_DPHY_TX_REF_CONTROL 0x18C8
+#define mmDP_AUX3_AUX_DPHY_TX_STATUS 0x18CC
+#define mmDP_AUX3_AUX_GTC_SYNC_CONTROL 0x18CE
+#define mmDP_AUX3_AUX_GTC_SYNC_DATA 0x18D0
+#define mmDP_AUX3_AUX_INTERRUPT_CONTROL 0x18C3
+#define mmDP_AUX3_AUX_LS_DATA 0x18C7
+#define mmDP_AUX3_AUX_LS_STATUS 0x18C5
+#define mmDP_AUX3_AUX_SW_CONTROL 0x18C1
+#define mmDP_AUX3_AUX_SW_DATA 0x18C6
+#define mmDP_AUX3_AUX_SW_STATUS 0x18C4
+#define mmDP_AUX4_AUX_ARB_CONTROL 0x18D6
+#define mmDP_AUX4_AUX_CONTROL 0x18D4
+#define mmDP_AUX4_AUX_DPHY_RX_CONTROL0 0x18DE
+#define mmDP_AUX4_AUX_DPHY_RX_CONTROL1 0x18DF
+#define mmDP_AUX4_AUX_DPHY_RX_STATUS 0x18E1
+#define mmDP_AUX4_AUX_DPHY_TX_CONTROL 0x18DD
+#define mmDP_AUX4_AUX_DPHY_TX_REF_CONTROL 0x18DC
+#define mmDP_AUX4_AUX_DPHY_TX_STATUS 0x18E0
+#define mmDP_AUX4_AUX_GTC_SYNC_CONTROL 0x18E2
+#define mmDP_AUX4_AUX_GTC_SYNC_DATA 0x18E4
+#define mmDP_AUX4_AUX_INTERRUPT_CONTROL 0x18D7
+#define mmDP_AUX4_AUX_LS_DATA 0x18DB
+#define mmDP_AUX4_AUX_LS_STATUS 0x18D9
+#define mmDP_AUX4_AUX_SW_CONTROL 0x18D5
+#define mmDP_AUX4_AUX_SW_DATA 0x18DA
+#define mmDP_AUX4_AUX_SW_STATUS 0x18D8
+#define mmDP_AUX5_AUX_ARB_CONTROL 0x18EA
+#define mmDP_AUX5_AUX_CONTROL 0x18E8
+#define mmDP_AUX5_AUX_DPHY_RX_CONTROL0 0x18F2
+#define mmDP_AUX5_AUX_DPHY_RX_CONTROL1 0x18F3
+#define mmDP_AUX5_AUX_DPHY_RX_STATUS 0x18F5
+#define mmDP_AUX5_AUX_DPHY_TX_CONTROL 0x18F1
+#define mmDP_AUX5_AUX_DPHY_TX_REF_CONTROL 0x18F0
+#define mmDP_AUX5_AUX_DPHY_TX_STATUS 0x18F4
+#define mmDP_AUX5_AUX_GTC_SYNC_CONTROL 0x18F6
+#define mmDP_AUX5_AUX_GTC_SYNC_DATA 0x18F8
+#define mmDP_AUX5_AUX_INTERRUPT_CONTROL 0x18EB
+#define mmDP_AUX5_AUX_LS_DATA 0x18EF
+#define mmDP_AUX5_AUX_LS_STATUS 0x18ED
+#define mmDP_AUX5_AUX_SW_CONTROL 0x18E9
+#define mmDP_AUX5_AUX_SW_DATA 0x18EE
+#define mmDP_AUX5_AUX_SW_STATUS 0x18EC
+#define mmDP_CONFIG 0x1CC2
+#define mmDP_DPHY_8B10B_CNTL 0x1CD3
+#define mmDP_DPHY_CNTL 0x1CD0
+#define mmDP_DPHY_CRC_CNTL 0x1CD7
+#define mmDP_DPHY_CRC_EN 0x1CD6
+#define mmDP_DPHY_CRC_MST_CNTL 0x1CC6
+#define mmDP_DPHY_CRC_MST_STATUS 0x1CC7
+#define mmDP_DPHY_CRC_RESULT 0x1CD8
+#define mmDP_DPHY_FAST_TRAINING 0x1CCE
+#define mmDP_DPHY_FAST_TRAINING_STATUS 0x1CE9
+#define mmDP_DPHY_PRBS_CNTL 0x1CD4
+#define mmDP_DPHY_SYM0 0x1CD2
+#define mmDP_DPHY_SYM1 0x1CE0
+#define mmDP_DPHY_SYM2 0x1CDF
+#define mmDP_DPHY_TRAINING_PATTERN_SEL 0x1CD1
+#define mmDP_DTO0_MODULO 0x0142
+#define mmDP_DTO0_PHASE 0x0141
+#define mmDP_DTO1_MODULO 0x0146
+#define mmDP_DTO1_PHASE 0x0145
+#define mmDP_DTO2_MODULO 0x014A
+#define mmDP_DTO2_PHASE 0x0149
+#define mmDP_DTO3_MODULO 0x014E
+#define mmDP_DTO3_PHASE 0x014D
+#define mmDP_DTO4_MODULO 0x0152
+#define mmDP_DTO4_PHASE 0x0151
+#define mmDP_DTO5_MODULO 0x0156
+#define mmDP_DTO5_PHASE 0x0155
+#define mmDPG_PIPE_ARBITRATION_CONTROL1 0x1B30
+#define mmDPG_PIPE_ARBITRATION_CONTROL2 0x1B31
+#define mmDPG_PIPE_DPM_CONTROL 0x1B34
+#define mmDPG_PIPE_NB_PSTATE_CHANGE_CONTROL 0x1B36
+#define mmDPG_PIPE_STUTTER_CONTROL 0x1B35
+#define mmDPG_PIPE_STUTTER_CONTROL_NONLPTCH 0x1B37
+#define mmDPG_PIPE_URGENCY_CONTROL 0x1B33
+#define mmDPG_TEST_DEBUG_DATA 0x1B39
+#define mmDPG_TEST_DEBUG_INDEX 0x1B38
+#define mmDP_HBR2_EYE_PATTERN 0x1CC8
+#define mmDP_LINK_CNTL 0x1CC0
+#define mmDP_LINK_FRAMING_CNTL 0x1CCC
+#define mmDP_MSA_COLORIMETRY 0x1CDA
+#define mmDP_MSA_MISC 0x1CC5
+#define mmDP_MSA_V_TIMING_OVERRIDE1 0x1CEA
+#define mmDP_MSA_V_TIMING_OVERRIDE2 0x1CEB
+#define mmDP_MSE_LINK_TIMING 0x1CE8
+#define mmDP_MSE_MISC_CNTL 0x1CDB
+#define mmDP_MSE_RATE_CNTL 0x1CE1
+#define mmDP_MSE_RATE_UPDATE 0x1CE3
+#define mmDP_MSE_SAT0 0x1CE4
+#define mmDP_MSE_SAT1 0x1CE5
+#define mmDP_MSE_SAT2 0x1CE6
+#define mmDP_MSE_SAT_UPDATE 0x1CE7
+#define mmDP_PIXEL_FORMAT 0x1CC1
+#define mmDP_SEC_AUD_M 0x1CA7
+#define mmDP_SEC_AUD_M_READBACK 0x1CA8
+#define mmDP_SEC_AUD_N 0x1CA5
+#define mmDP_SEC_AUD_N_READBACK 0x1CA6
+#define mmDP_SEC_CNTL 0x1CA0
+#define mmDP_SEC_CNTL1 0x1CAB
+#define mmDP_SEC_FRAMING1 0x1CA1
+#define mmDP_SEC_FRAMING2 0x1CA2
+#define mmDP_SEC_FRAMING3 0x1CA3
+#define mmDP_SEC_FRAMING4 0x1CA4
+#define mmDP_SEC_PACKET_CNTL 0x1CAA
+#define mmDP_SEC_TIMESTAMP 0x1CA9
+#define mmDP_STEER_FIFO 0x1CC4
+#define mmDP_TEST_DEBUG_DATA 0x1CFD
+#define mmDP_TEST_DEBUG_INDEX 0x1CFC
+#define mmDP_VID_INTERRUPT_CNTL 0x1CCF
+#define mmDP_VID_M 0x1CCB
+#define mmDP_VID_MSA_VBID 0x1CCD
+#define mmDP_VID_N 0x1CCA
+#define mmDP_VID_STREAM_CNTL 0x1CC3
+#define mmDP_VID_TIMING 0x1CC9
+#define mmDVOACLKC_CNTL 0x016A
+#define mmDVOACLKC_MVP_CNTL 0x0169
+#define mmDVOACLKD_CNTL 0x0168
+#define mmDVO_CLK_ENABLE 0x0129
+#define mmDVO_CONTROL 0x185B
+#define mmDVO_CRC2_SIG_MASK 0x185D
+#define mmDVO_CRC2_SIG_RESULT 0x185E
+#define mmDVO_CRC_EN 0x185C
+#define mmDVO_ENABLE 0x1858
+#define mmDVO_FIFO_ERROR_STATUS 0x185F
+#define mmDVO_OUTPUT 0x185A
+#define mmDVO_SKEW_ADJUST 0x197D
+#define mmDVO_SOURCE_SELECT 0x1859
+#define mmDVO_STRENGTH_CONTROL 0x197B
+#define mmDVO_VREF_CONTROL 0x197C
+#define mmEXT_OVERSCAN_LEFT_RIGHT 0x1B5E
+#define mmEXT_OVERSCAN_TOP_BOTTOM 0x1B5F
+#define mmFBC_CLIENT_REGION_MASK 0x16EB
+#define mmFBC_CNTL 0x16D0
+#define mmFBC_COMP_CNTL 0x16D4
+#define mmFBC_COMP_MODE 0x16D5
+#define mmFBC_CSM_REGION_OFFSET_01 0x16E9
+#define mmFBC_CSM_REGION_OFFSET_23 0x16EA
+#define mmFBC_DEBUG0 0x16D6
+#define mmFBC_DEBUG1 0x16D7
+#define mmFBC_DEBUG2 0x16D8
+#define mmFBC_DEBUG_COMP 0x16EC
+#define mmFBC_DEBUG_CSR 0x16ED
+#define mmFBC_DEBUG_CSR_RDATA 0x16EE
+#define mmFBC_DEBUG_CSR_RDATA_HI 0x16F6
+#define mmFBC_DEBUG_CSR_WDATA 0x16EF
+#define mmFBC_DEBUG_CSR_WDATA_HI 0x16F7
+#define mmFBC_IDLE_FORCE_CLEAR_MASK 0x16D2
+#define mmFBC_IDLE_MASK 0x16D1
+#define mmFBC_IND_LUT0 0x16D9
+#define mmFBC_IND_LUT10 0x16E3
+#define mmFBC_IND_LUT1 0x16DA
+#define mmFBC_IND_LUT11 0x16E4
+#define mmFBC_IND_LUT12 0x16E5
+#define mmFBC_IND_LUT13 0x16E6
+#define mmFBC_IND_LUT14 0x16E7
+#define mmFBC_IND_LUT15 0x16E8
+#define mmFBC_IND_LUT2 0x16DB
+#define mmFBC_IND_LUT3 0x16DC
+#define mmFBC_IND_LUT4 0x16DD
+#define mmFBC_IND_LUT5 0x16DE
+#define mmFBC_IND_LUT6 0x16DF
+#define mmFBC_IND_LUT7 0x16E0
+#define mmFBC_IND_LUT8 0x16E1
+#define mmFBC_IND_LUT9 0x16E2
+#define mmFBC_MISC 0x16F0
+#define mmFBC_START_STOP_DELAY 0x16D3
+#define mmFBC_STATUS 0x16F1
+#define mmFBC_TEST_DEBUG_DATA 0x16F5
+#define mmFBC_TEST_DEBUG_INDEX 0x16F4
+#define mmFMT0_FMT_BIT_DEPTH_CONTROL 0x1BF2
+#define mmFMT0_FMT_CLAMP_CNTL 0x1BF9
+#define mmFMT0_FMT_CONTROL 0x1BEE
+#define mmFMT0_FMT_CRC_CNTL 0x1BFA
+#define mmFMT0_FMT_CRC_SIG_BLUE_CONTROL 0x1BFE
+#define mmFMT0_FMT_CRC_SIG_BLUE_CONTROL_MASK 0x1BFC
+#define mmFMT0_FMT_CRC_SIG_RED_GREEN 0x1BFD
+#define mmFMT0_FMT_CRC_SIG_RED_GREEN_MASK 0x1BFB
+#define mmFMT0_FMT_DEBUG_CNTL 0x1BFF
+#define mmFMT0_FMT_DITHER_RAND_B_SEED 0x1BF5
+#define mmFMT0_FMT_DITHER_RAND_G_SEED 0x1BF4
+#define mmFMT0_FMT_DITHER_RAND_R_SEED 0x1BF3
+#define mmFMT0_FMT_DYNAMIC_EXP_CNTL 0x1BED
+#define mmFMT0_FMT_FORCE_DATA_0_1 0x1BF0
+#define mmFMT0_FMT_FORCE_DATA_2_3 0x1BF1
+#define mmFMT0_FMT_FORCE_OUTPUT_CNTL 0x1BEF
+#define mmFMT0_FMT_TEMPORAL_DITHER_PATTERN_CONTROL 0x1BF6
+#define mmFMT0_FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_S_MATRIX 0x1BF7
+#define mmFMT0_FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_T_MATRIX 0x1BF8
+#define mmFMT0_FMT_TEST_DEBUG_DATA 0x1BEC
+#define mmFMT0_FMT_TEST_DEBUG_INDEX 0x1BEB
+#define mmFMT1_FMT_BIT_DEPTH_CONTROL 0x1EF2
+#define mmFMT1_FMT_CLAMP_CNTL 0x1EF9
+#define mmFMT1_FMT_CONTROL 0x1EEE
+#define mmFMT1_FMT_CRC_CNTL 0x1EFA
+#define mmFMT1_FMT_CRC_SIG_BLUE_CONTROL 0x1EFE
+#define mmFMT1_FMT_CRC_SIG_BLUE_CONTROL_MASK 0x1EFC
+#define mmFMT1_FMT_CRC_SIG_RED_GREEN 0x1EFD
+#define mmFMT1_FMT_CRC_SIG_RED_GREEN_MASK 0x1EFB
+#define mmFMT1_FMT_DEBUG_CNTL 0x1EFF
+#define mmFMT1_FMT_DITHER_RAND_B_SEED 0x1EF5
+#define mmFMT1_FMT_DITHER_RAND_G_SEED 0x1EF4
+#define mmFMT1_FMT_DITHER_RAND_R_SEED 0x1EF3
+#define mmFMT1_FMT_DYNAMIC_EXP_CNTL 0x1EED
+#define mmFMT1_FMT_FORCE_DATA_0_1 0x1EF0
+#define mmFMT1_FMT_FORCE_DATA_2_3 0x1EF1
+#define mmFMT1_FMT_FORCE_OUTPUT_CNTL 0x1EEF
+#define mmFMT1_FMT_TEMPORAL_DITHER_PATTERN_CONTROL 0x1EF6
+#define mmFMT1_FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_S_MATRIX 0x1EF7
+#define mmFMT1_FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_T_MATRIX 0x1EF8
+#define mmFMT1_FMT_TEST_DEBUG_DATA 0x1EEC
+#define mmFMT1_FMT_TEST_DEBUG_INDEX 0x1EEB
+#define mmFMT2_FMT_BIT_DEPTH_CONTROL 0x41F2
+#define mmFMT2_FMT_CLAMP_CNTL 0x41F9
+#define mmFMT2_FMT_CONTROL 0x41EE
+#define mmFMT2_FMT_CRC_CNTL 0x41FA
+#define mmFMT2_FMT_CRC_SIG_BLUE_CONTROL 0x41FE
+#define mmFMT2_FMT_CRC_SIG_BLUE_CONTROL_MASK 0x41FC
+#define mmFMT2_FMT_CRC_SIG_RED_GREEN 0x41FD
+#define mmFMT2_FMT_CRC_SIG_RED_GREEN_MASK 0x41FB
+#define mmFMT2_FMT_DEBUG_CNTL 0x41FF
+#define mmFMT2_FMT_DITHER_RAND_B_SEED 0x41F5
+#define mmFMT2_FMT_DITHER_RAND_G_SEED 0x41F4
+#define mmFMT2_FMT_DITHER_RAND_R_SEED 0x41F3
+#define mmFMT2_FMT_DYNAMIC_EXP_CNTL 0x41ED
+#define mmFMT2_FMT_FORCE_DATA_0_1 0x41F0
+#define mmFMT2_FMT_FORCE_DATA_2_3 0x41F1
+#define mmFMT2_FMT_FORCE_OUTPUT_CNTL 0x41EF
+#define mmFMT2_FMT_TEMPORAL_DITHER_PATTERN_CONTROL 0x41F6
+#define mmFMT2_FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_S_MATRIX 0x41F7
+#define mmFMT2_FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_T_MATRIX 0x41F8
+#define mmFMT2_FMT_TEST_DEBUG_DATA 0x41EC
+#define mmFMT2_FMT_TEST_DEBUG_INDEX 0x41EB
+#define mmFMT3_FMT_BIT_DEPTH_CONTROL 0x44F2
+#define mmFMT3_FMT_CLAMP_CNTL 0x44F9
+#define mmFMT3_FMT_CONTROL 0x44EE
+#define mmFMT3_FMT_CRC_CNTL 0x44FA
+#define mmFMT3_FMT_CRC_SIG_BLUE_CONTROL 0x44FE
+#define mmFMT3_FMT_CRC_SIG_BLUE_CONTROL_MASK 0x44FC
+#define mmFMT3_FMT_CRC_SIG_RED_GREEN 0x44FD
+#define mmFMT3_FMT_CRC_SIG_RED_GREEN_MASK 0x44FB
+#define mmFMT3_FMT_DEBUG_CNTL 0x44FF
+#define mmFMT3_FMT_DITHER_RAND_B_SEED 0x44F5
+#define mmFMT3_FMT_DITHER_RAND_G_SEED 0x44F4
+#define mmFMT3_FMT_DITHER_RAND_R_SEED 0x44F3
+#define mmFMT3_FMT_DYNAMIC_EXP_CNTL 0x44ED
+#define mmFMT3_FMT_FORCE_DATA_0_1 0x44F0
+#define mmFMT3_FMT_FORCE_DATA_2_3 0x44F1
+#define mmFMT3_FMT_FORCE_OUTPUT_CNTL 0x44EF
+#define mmFMT3_FMT_TEMPORAL_DITHER_PATTERN_CONTROL 0x44F6
+#define mmFMT3_FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_S_MATRIX 0x44F7
+#define mmFMT3_FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_T_MATRIX 0x44F8
+#define mmFMT3_FMT_TEST_DEBUG_DATA 0x44EC
+#define mmFMT3_FMT_TEST_DEBUG_INDEX 0x44EB
+#define mmFMT4_FMT_BIT_DEPTH_CONTROL 0x47F2
+#define mmFMT4_FMT_CLAMP_CNTL 0x47F9
+#define mmFMT4_FMT_CONTROL 0x47EE
+#define mmFMT4_FMT_CRC_CNTL 0x47FA
+#define mmFMT4_FMT_CRC_SIG_BLUE_CONTROL 0x47FE
+#define mmFMT4_FMT_CRC_SIG_BLUE_CONTROL_MASK 0x47FC
+#define mmFMT4_FMT_CRC_SIG_RED_GREEN 0x47FD
+#define mmFMT4_FMT_CRC_SIG_RED_GREEN_MASK 0x47FB
+#define mmFMT4_FMT_DEBUG_CNTL 0x47FF
+#define mmFMT4_FMT_DITHER_RAND_B_SEED 0x47F5
+#define mmFMT4_FMT_DITHER_RAND_G_SEED 0x47F4
+#define mmFMT4_FMT_DITHER_RAND_R_SEED 0x47F3
+#define mmFMT4_FMT_DYNAMIC_EXP_CNTL 0x47ED
+#define mmFMT4_FMT_FORCE_DATA_0_1 0x47F0
+#define mmFMT4_FMT_FORCE_DATA_2_3 0x47F1
+#define mmFMT4_FMT_FORCE_OUTPUT_CNTL 0x47EF
+#define mmFMT4_FMT_TEMPORAL_DITHER_PATTERN_CONTROL 0x47F6
+#define mmFMT4_FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_S_MATRIX 0x47F7
+#define mmFMT4_FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_T_MATRIX 0x47F8
+#define mmFMT4_FMT_TEST_DEBUG_DATA 0x47EC
+#define mmFMT4_FMT_TEST_DEBUG_INDEX 0x47EB
+#define mmFMT5_FMT_BIT_DEPTH_CONTROL 0x4AF2
+#define mmFMT5_FMT_CLAMP_CNTL 0x4AF9
+#define mmFMT5_FMT_CONTROL 0x4AEE
+#define mmFMT5_FMT_CRC_CNTL 0x4AFA
+#define mmFMT5_FMT_CRC_SIG_BLUE_CONTROL 0x4AFE
+#define mmFMT5_FMT_CRC_SIG_BLUE_CONTROL_MASK 0x4AFC
+#define mmFMT5_FMT_CRC_SIG_RED_GREEN 0x4AFD
+#define mmFMT5_FMT_CRC_SIG_RED_GREEN_MASK 0x4AFB
+#define mmFMT5_FMT_DEBUG_CNTL 0x4AFF
+#define mmFMT5_FMT_DITHER_RAND_B_SEED 0x4AF5
+#define mmFMT5_FMT_DITHER_RAND_G_SEED 0x4AF4
+#define mmFMT5_FMT_DITHER_RAND_R_SEED 0x4AF3
+#define mmFMT5_FMT_DYNAMIC_EXP_CNTL 0x4AED
+#define mmFMT5_FMT_FORCE_DATA_0_1 0x4AF0
+#define mmFMT5_FMT_FORCE_DATA_2_3 0x4AF1
+#define mmFMT5_FMT_FORCE_OUTPUT_CNTL 0x4AEF
+#define mmFMT5_FMT_TEMPORAL_DITHER_PATTERN_CONTROL 0x4AF6
+#define mmFMT5_FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_S_MATRIX 0x4AF7
+#define mmFMT5_FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_T_MATRIX 0x4AF8
+#define mmFMT5_FMT_TEST_DEBUG_DATA 0x4AEC
+#define mmFMT5_FMT_TEST_DEBUG_INDEX 0x4AEB
+#define mmFMT_BIT_DEPTH_CONTROL 0x1BF2
+#define mmFMT_CLAMP_CNTL 0x1BF9
+#define mmFMT_CONTROL 0x1BEE
+#define mmFMT_CRC_CNTL 0x1BFA
+#define mmFMT_CRC_SIG_BLUE_CONTROL 0x1BFE
+#define mmFMT_CRC_SIG_BLUE_CONTROL_MASK 0x1BFC
+#define mmFMT_CRC_SIG_RED_GREEN 0x1BFD
+#define mmFMT_CRC_SIG_RED_GREEN_MASK 0x1BFB
+#define mmFMT_DEBUG_CNTL 0x1BFF
+#define mmFMT_DITHER_RAND_B_SEED 0x1BF5
+#define mmFMT_DITHER_RAND_G_SEED 0x1BF4
+#define mmFMT_DITHER_RAND_R_SEED 0x1BF3
+#define mmFMT_DYNAMIC_EXP_CNTL 0x1BED
+#define mmFMT_FORCE_DATA_0_1 0x1BF0
+#define mmFMT_FORCE_DATA_2_3 0x1BF1
+#define mmFMT_FORCE_OUTPUT_CNTL 0x1BEF
+#define mmFMT_TEMPORAL_DITHER_PATTERN_CONTROL 0x1BF6
+#define mmFMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_S_MATRIX 0x1BF7
+#define mmFMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_T_MATRIX 0x1BF8
+#define mmFMT_TEST_DEBUG_DATA 0x1BEC
+#define mmFMT_TEST_DEBUG_INDEX 0x1BEB
+#define mmGAMUT_REMAP_C11_C12 0x1A5A
+#define mmGAMUT_REMAP_C13_C14 0x1A5B
+#define mmGAMUT_REMAP_C21_C22 0x1A5C
+#define mmGAMUT_REMAP_C23_C24 0x1A5D
+#define mmGAMUT_REMAP_C31_C32 0x1A5E
+#define mmGAMUT_REMAP_C33_C34 0x1A5F
+#define mmGAMUT_REMAP_CONTROL 0x1A59
+#define mmGENENB 0x00F0
+#define mmGENERIC_I2C_CONTROL 0x1834
+#define mmGENERIC_I2C_DATA 0x183A
+#define mmGENERIC_I2C_INTERRUPT_CONTROL 0x1835
+#define mmGENERIC_I2C_PIN_DEBUG 0x183C
+#define mmGENERIC_I2C_PIN_SELECTION 0x183B
+#define mmGENERIC_I2C_SETUP 0x1838
+#define mmGENERIC_I2C_SPEED 0x1837
+#define mmGENERIC_I2C_STATUS 0x1836
+#define mmGENERIC_I2C_TRANSACTION 0x1839
+#define mmGENFC_RD 0x00F2
+#define mmGENFC_WT 0x00EE
+#define mmGENMO_RD 0x00F3
+#define mmGENMO_WT 0x00F0
+#define mmGENS0 0x00F0
+#define mmGENS1 0x00EE
+#define mmGRPH8_DATA 0x00F3
+#define mmGRPH8_IDX 0x00F3
+#define mmGRPH_COMPRESS_PITCH 0x1A1A
+#define mmGRPH_COMPRESS_SURFACE_ADDRESS 0x1A19
+#define mmGRPH_COMPRESS_SURFACE_ADDRESS_HIGH 0x1A1B
+#define mmGRPH_CONTROL 0x1A01
+#define mmGRPH_DFQ_CONTROL 0x1A14
+#define mmGRPH_DFQ_STATUS 0x1A15
+#define mmGRPH_ENABLE 0x1A00
+#define mmGRPH_FLIP_CONTROL 0x1A12
+#define mmGRPH_INTERRUPT_CONTROL 0x1A17
+#define mmGRPH_INTERRUPT_STATUS 0x1A16
+#define mmGRPH_LUT_10BIT_BYPASS 0x1A02
+#define mmGRPH_PITCH 0x1A06
+#define mmGRPH_PRIMARY_SURFACE_ADDRESS 0x1A04
+#define mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH 0x1A07
+#define mmGRPH_SECONDARY_SURFACE_ADDRESS 0x1A05
+#define mmGRPH_SECONDARY_SURFACE_ADDRESS_HIGH 0x1A08
+#define mmGRPH_STEREOSYNC_FLIP 0x1A97
+#define mmGRPH_SURFACE_ADDRESS_HIGH_INUSE 0x1A18
+#define mmGRPH_SURFACE_ADDRESS_INUSE 0x1A13
+#define mmGRPH_SURFACE_OFFSET_X 0x1A09
+#define mmGRPH_SURFACE_OFFSET_Y 0x1A0A
+#define mmGRPH_SWAP_CNTL 0x1A03
+#define mmGRPH_UPDATE 0x1A11
+#define mmGRPH_X_END 0x1A0D
+#define mmGRPH_X_START 0x1A0B
+#define mmGRPH_Y_END 0x1A0E
+#define mmGRPH_Y_START 0x1A0C
+#define mmHDMI_ACR_32_0 0x1C37
+#define mmHDMI_ACR_32_1 0x1C38
+#define mmHDMI_ACR_44_0 0x1C39
+#define mmHDMI_ACR_44_1 0x1C3A
+#define mmHDMI_ACR_48_0 0x1C3B
+#define mmHDMI_ACR_48_1 0x1C3C
+#define mmHDMI_ACR_PACKET_CONTROL 0x1C0F
+#define mmHDMI_ACR_STATUS_0 0x1C3D
+#define mmHDMI_ACR_STATUS_1 0x1C3E
+#define mmHDMI_AUDIO_PACKET_CONTROL 0x1C0E
+#define mmHDMI_CONTROL 0x1C0C
+#define mmHDMI_GC 0x1C16
+#define mmHDMI_GENERIC_PACKET_CONTROL0 0x1C13
+#define mmHDMI_GENERIC_PACKET_CONTROL1 0x1C30
+#define mmHDMI_INFOFRAME_CONTROL0 0x1C11
+#define mmHDMI_INFOFRAME_CONTROL1 0x1C12
+#define mmHDMI_STATUS 0x1C0D
+#define mmHDMI_VBI_PACKET_CONTROL 0x1C10
+#define mmINPUT_CSC_C11_C12 0x1A36
+#define mmINPUT_CSC_C13_C14 0x1A37
+#define mmINPUT_CSC_C21_C22 0x1A38
+#define mmINPUT_CSC_C23_C24 0x1A39
+#define mmINPUT_CSC_C31_C32 0x1A3A
+#define mmINPUT_CSC_C33_C34 0x1A3B
+#define mmINPUT_CSC_CONTROL 0x1A35
+#define mmINPUT_GAMMA_CONTROL 0x1A10
+#define mmKEY_CONTROL 0x1A53
+#define mmKEY_RANGE_ALPHA 0x1A54
+#define mmKEY_RANGE_BLUE 0x1A57
+#define mmKEY_RANGE_GREEN 0x1A56
+#define mmKEY_RANGE_RED 0x1A55
+#define mmLB0_DC_MVP_LB_CONTROL 0x1ADB
+#define mmLB0_LB_DEBUG 0x1AFC
+#define mmLB0_LB_DEBUG2 0x1AC9
+#define mmLB0_LB_NO_OUTSTANDING_REQ_STATUS 0x1AC8
+#define mmLB0_LB_SYNC_RESET_SEL 0x1ACA
+#define mmLB0_LB_TEST_DEBUG_DATA 0x1AFF
+#define mmLB0_LB_TEST_DEBUG_INDEX 0x1AFE
+#define mmLB0_MVP_AFR_FLIP_FIFO_CNTL 0x1AD9
+#define mmLB0_MVP_AFR_FLIP_MODE 0x1AD8
+#define mmLB0_MVP_FLIP_LINE_NUM_INSERT 0x1ADA
+#define mmLB1_DC_MVP_LB_CONTROL 0x1DDB
+#define mmLB1_LB_DEBUG 0x1DFC
+#define mmLB1_LB_DEBUG2 0x1DC9
+#define mmLB1_LB_NO_OUTSTANDING_REQ_STATUS 0x1DC8
+#define mmLB1_LB_SYNC_RESET_SEL 0x1DCA
+#define mmLB1_LB_TEST_DEBUG_DATA 0x1DFF
+#define mmLB1_LB_TEST_DEBUG_INDEX 0x1DFE
+#define mmLB1_MVP_AFR_FLIP_FIFO_CNTL 0x1DD9
+#define mmLB1_MVP_AFR_FLIP_MODE 0x1DD8
+#define mmLB1_MVP_FLIP_LINE_NUM_INSERT 0x1DDA
+#define mmLB2_DC_MVP_LB_CONTROL 0x40DB
+#define mmLB2_LB_DEBUG 0x40FC
+#define mmLB2_LB_DEBUG2 0x40C9
+#define mmLB2_LB_NO_OUTSTANDING_REQ_STATUS 0x40C8
+#define mmLB2_LB_SYNC_RESET_SEL 0x40CA
+#define mmLB2_LB_TEST_DEBUG_DATA 0x40FF
+#define mmLB2_LB_TEST_DEBUG_INDEX 0x40FE
+#define mmLB2_MVP_AFR_FLIP_FIFO_CNTL 0x40D9
+#define mmLB2_MVP_AFR_FLIP_MODE 0x40D8
+#define mmLB2_MVP_FLIP_LINE_NUM_INSERT 0x40DA
+#define mmLB3_DC_MVP_LB_CONTROL 0x43DB
+#define mmLB3_LB_DEBUG 0x43FC
+#define mmLB3_LB_DEBUG2 0x43C9
+#define mmLB3_LB_NO_OUTSTANDING_REQ_STATUS 0x43C8
+#define mmLB3_LB_SYNC_RESET_SEL 0x43CA
+#define mmLB3_LB_TEST_DEBUG_DATA 0x43FF
+#define mmLB3_LB_TEST_DEBUG_INDEX 0x43FE
+#define mmLB3_MVP_AFR_FLIP_FIFO_CNTL 0x43D9
+#define mmLB3_MVP_AFR_FLIP_MODE 0x43D8
+#define mmLB3_MVP_FLIP_LINE_NUM_INSERT 0x43DA
+#define mmLB4_DC_MVP_LB_CONTROL 0x46DB
+#define mmLB4_LB_DEBUG 0x46FC
+#define mmLB4_LB_DEBUG2 0x46C9
+#define mmLB4_LB_NO_OUTSTANDING_REQ_STATUS 0x46C8
+#define mmLB4_LB_SYNC_RESET_SEL 0x46CA
+#define mmLB4_LB_TEST_DEBUG_DATA 0x46FF
+#define mmLB4_LB_TEST_DEBUG_INDEX 0x46FE
+#define mmLB4_MVP_AFR_FLIP_FIFO_CNTL 0x46D9
+#define mmLB4_MVP_AFR_FLIP_MODE 0x46D8
+#define mmLB4_MVP_FLIP_LINE_NUM_INSERT 0x46DA
+#define mmLB5_DC_MVP_LB_CONTROL 0x49DB
+#define mmLB5_LB_DEBUG 0x49FC
+#define mmLB5_LB_DEBUG2 0x49C9
+#define mmLB5_LB_NO_OUTSTANDING_REQ_STATUS 0x49C8
+#define mmLB5_LB_SYNC_RESET_SEL 0x49CA
+#define mmLB5_LB_TEST_DEBUG_DATA 0x49FF
+#define mmLB5_LB_TEST_DEBUG_INDEX 0x49FE
+#define mmLB5_MVP_AFR_FLIP_FIFO_CNTL 0x49D9
+#define mmLB5_MVP_AFR_FLIP_MODE 0x49D8
+#define mmLB5_MVP_FLIP_LINE_NUM_INSERT 0x49DA
+#define mmLB_DEBUG 0x1AFC
+#define mmLB_DEBUG2 0x1AC9
+#define mmLB_NO_OUTSTANDING_REQ_STATUS 0x1AC8
+#define mmLB_SYNC_RESET_SEL 0x1ACA
+#define mmLB_TEST_DEBUG_DATA 0x1AFF
+#define mmLB_TEST_DEBUG_INDEX 0x1AFE
+#define mmLIGHT_SLEEP_CNTL 0x0132
+#define mmLOW_POWER_TILING_CONTROL 0x0325
+#define mmLVDS_DATA_CNTL 0x1C8C
+#define mmLVTMA_PWRSEQ_CNTL 0x1919
+#define mmLVTMA_PWRSEQ_DELAY1 0x191C
+#define mmLVTMA_PWRSEQ_DELAY2 0x191D
+#define mmLVTMA_PWRSEQ_REF_DIV 0x191B
+#define mmLVTMA_PWRSEQ_STATE 0x191A
+#define mmMASTER_COMM_CMD_REG 0x161F
+#define mmMASTER_COMM_CNTL_REG 0x1620
+#define mmMASTER_COMM_DATA_REG1 0x161C
+#define mmMASTER_COMM_DATA_REG2 0x161D
+#define mmMASTER_COMM_DATA_REG3 0x161E
+#define mmMASTER_UPDATE_LOCK 0x1BBD
+#define mmMASTER_UPDATE_MODE 0x1BBE
+#define mmMC_DC_INTERFACE_NACK_STATUS 0x031C
+#define mmMCIF_CONTROL 0x0314
+#define mmMCIF_MEM_CONTROL 0x0319
+#define mmMCIF_TEST_DEBUG_DATA 0x0317
+#define mmMCIF_TEST_DEBUG_INDEX 0x0316
+#define mmMCIF_VMID 0x0318
+#define mmMCIF_WRITE_COMBINE_CONTROL 0x0315
+#define mmMICROSECOND_TIME_BASE_DIV 0x013B
+#define mmMILLISECOND_TIME_BASE_DIV 0x0130
+#define mmMVP_AFR_FLIP_FIFO_CNTL 0x1AD9
+#define mmMVP_AFR_FLIP_MODE 0x1AD8
+#define mmMVP_BLACK_KEYER 0x1686
+#define mmMVP_CONTROL1 0x1680
+#define mmMVP_CONTROL2 0x1681
+#define mmMVP_CONTROL3 0x168A
+#define mmMVP_CRC_CNTL 0x1687
+#define mmMVP_CRC_RESULT_BLUE_GREEN 0x1688
+#define mmMVP_CRC_RESULT_RED 0x1689
+#define mmMVP_DEBUG 0x168F
+#define mmMVP_FIFO_CONTROL 0x1682
+#define mmMVP_FIFO_STATUS 0x1683
+#define mmMVP_FLIP_LINE_NUM_INSERT 0x1ADA
+#define mmMVP_INBAND_CNTL_CAP 0x1685
+#define mmMVP_RECEIVE_CNT_CNTL1 0x168B
+#define mmMVP_RECEIVE_CNT_CNTL2 0x168C
+#define mmMVP_SLAVE_STATUS 0x1684
+#define mmMVP_TEST_DEBUG_DATA 0x168E
+#define mmMVP_TEST_DEBUG_INDEX 0x168D
+#define mmOUTPUT_CSC_C11_C12 0x1A3D
+#define mmOUTPUT_CSC_C13_C14 0x1A3E
+#define mmOUTPUT_CSC_C21_C22 0x1A3F
+#define mmOUTPUT_CSC_C23_C24 0x1A40
+#define mmOUTPUT_CSC_C31_C32 0x1A41
+#define mmOUTPUT_CSC_C33_C34 0x1A42
+#define mmOUTPUT_CSC_CONTROL 0x1A3C
+#define mmOUT_ROUND_CONTROL 0x1A51
+#define mmOVL_CONTROL1 0x1A1D
+#define mmOVL_CONTROL2 0x1A1E
+#define mmOVL_DFQ_CONTROL 0x1A29
+#define mmOVL_DFQ_STATUS 0x1A2A
+#define mmOVL_ENABLE 0x1A1C
+#define mmOVL_END 0x1A26
+#define mmOVL_PITCH 0x1A21
+#define mmOVLSCL_EDGE_PIXEL_CNTL 0x1A2C
+#define mmOVL_SECONDARY_SURFACE_ADDRESS 0x1A92
+#define mmOVL_SECONDARY_SURFACE_ADDRESS_HIGH 0x1A94
+#define mmOVL_START 0x1A25
+#define mmOVL_STEREOSYNC_FLIP 0x1A93
+#define mmOVL_SURFACE_ADDRESS 0x1A20
+#define mmOVL_SURFACE_ADDRESS_HIGH 0x1A22
+#define mmOVL_SURFACE_ADDRESS_HIGH_INUSE 0x1A2B
+#define mmOVL_SURFACE_ADDRESS_INUSE 0x1A28
+#define mmOVL_SURFACE_OFFSET_X 0x1A23
+#define mmOVL_SURFACE_OFFSET_Y 0x1A24
+#define mmOVL_SWAP_CNTL 0x1A1F
+#define mmOVL_UPDATE 0x1A27
+#define mmPHY_AUX_CNTL 0x197F
+#define mmPIPE0_ARBITRATION_CONTROL3 0x02FA
+#define mmPIPE0_DMIF_BUFFER_CONTROL 0x0328
+#define mmPIPE0_MAX_REQUESTS 0x0302
+#define mmPIPE0_PG_CONFIG 0x1760
+#define mmPIPE0_PG_ENABLE 0x1761
+#define mmPIPE0_PG_STATUS 0x1762
+#define mmPIPE1_ARBITRATION_CONTROL3 0x02FB
+#define mmPIPE1_DMIF_BUFFER_CONTROL 0x0330
+#define mmPIPE1_MAX_REQUESTS 0x0303
+#define mmPIPE1_PG_CONFIG 0x1764
+#define mmPIPE1_PG_ENABLE 0x1765
+#define mmPIPE1_PG_STATUS 0x1766
+#define mmPIPE2_ARBITRATION_CONTROL3 0x02FC
+#define mmPIPE2_DMIF_BUFFER_CONTROL 0x0338
+#define mmPIPE2_MAX_REQUESTS 0x0304
+#define mmPIPE2_PG_CONFIG 0x1768
+#define mmPIPE2_PG_ENABLE 0x1769
+#define mmPIPE2_PG_STATUS 0x176A
+#define mmPIPE3_ARBITRATION_CONTROL3 0x02FD
+#define mmPIPE3_DMIF_BUFFER_CONTROL 0x0340
+#define mmPIPE3_MAX_REQUESTS 0x0305
+#define mmPIPE3_PG_CONFIG 0x176C
+#define mmPIPE3_PG_ENABLE 0x176D
+#define mmPIPE3_PG_STATUS 0x176E
+#define mmPIPE4_ARBITRATION_CONTROL3 0x02FE
+#define mmPIPE4_DMIF_BUFFER_CONTROL 0x0348
+#define mmPIPE4_MAX_REQUESTS 0x0306
+#define mmPIPE4_PG_CONFIG 0x1770
+#define mmPIPE4_PG_ENABLE 0x1771
+#define mmPIPE4_PG_STATUS 0x1772
+#define mmPIPE5_ARBITRATION_CONTROL3 0x02FF
+#define mmPIPE5_DMIF_BUFFER_CONTROL 0x0350
+#define mmPIPE5_MAX_REQUESTS 0x0307
+#define mmPIPE5_PG_CONFIG 0x1774
+#define mmPIPE5_PG_ENABLE 0x1775
+#define mmPIPE5_PG_STATUS 0x1776
+#define mmPIXCLK0_RESYNC_CNTL 0x013A
+#define mmPIXCLK1_RESYNC_CNTL 0x0138
+#define mmPIXCLK2_RESYNC_CNTL 0x0139
+#define mmPLL_ANALOG 0x1708
+#define mmPLL_CNTL 0x1707
+#define mmPLL_DEBUG_CNTL 0x170B
+#define mmPLL_DISPCLK_CURRENT_DTO_PHASE 0x170F
+#define mmPLL_DISPCLK_DTO_CNTL 0x170E
+#define mmPLL_DS_CNTL 0x1705
+#define mmPLL_FB_DIV 0x1701
+#define mmPLL_IDCLK_CNTL 0x1706
+#define mmPLL_POST_DIV 0x1702
+#define mmPLL_REF_DIV 0x1700
+#define mmPLL_SS_AMOUNT_DSFRAC 0x1703
+#define mmPLL_SS_CNTL 0x1704
+#define mmPLL_UNLOCK_DETECT_CNTL 0x170A
+#define mmPLL_UPDATE_CNTL 0x170D
+#define mmPLL_UPDATE_LOCK 0x170C
+#define mmPLL_VREG_CNTL 0x1709
+#define mmPRESCALE_GRPH_CONTROL 0x1A2D
+#define mmPRESCALE_OVL_CONTROL 0x1A31
+#define mmPRESCALE_VALUES_GRPH_B 0x1A30
+#define mmPRESCALE_VALUES_GRPH_G 0x1A2F
+#define mmPRESCALE_VALUES_GRPH_R 0x1A2E
+#define mmPRESCALE_VALUES_OVL_CB 0x1A32
+#define mmPRESCALE_VALUES_OVL_CR 0x1A34
+#define mmPRESCALE_VALUES_OVL_Y 0x1A33
+#define mmREGAMMA_CNTLA_END_CNTL1 0x1AA6
+#define mmREGAMMA_CNTLA_END_CNTL2 0x1AA7
+#define mmREGAMMA_CNTLA_REGION_0_1 0x1AA8
+#define mmREGAMMA_CNTLA_REGION_10_11 0x1AAD
+#define mmREGAMMA_CNTLA_REGION_12_13 0x1AAE
+#define mmREGAMMA_CNTLA_REGION_14_15 0x1AAF
+#define mmREGAMMA_CNTLA_REGION_2_3 0x1AA9
+#define mmREGAMMA_CNTLA_REGION_4_5 0x1AAA
+#define mmREGAMMA_CNTLA_REGION_6_7 0x1AAB
+#define mmREGAMMA_CNTLA_REGION_8_9 0x1AAC
+#define mmREGAMMA_CNTLA_SLOPE_CNTL 0x1AA5
+#define mmREGAMMA_CNTLA_START_CNTL 0x1AA4
+#define mmREGAMMA_CNTLB_END_CNTL1 0x1AB2
+#define mmREGAMMA_CNTLB_END_CNTL2 0x1AB3
+#define mmREGAMMA_CNTLB_REGION_0_1 0x1AB4
+#define mmREGAMMA_CNTLB_REGION_10_11 0x1AB9
+#define mmREGAMMA_CNTLB_REGION_12_13 0x1ABA
+#define mmREGAMMA_CNTLB_REGION_14_15 0x1ABB
+#define mmREGAMMA_CNTLB_REGION_2_3 0x1AB5
+#define mmREGAMMA_CNTLB_REGION_4_5 0x1AB6
+#define mmREGAMMA_CNTLB_REGION_6_7 0x1AB7
+#define mmREGAMMA_CNTLB_REGION_8_9 0x1AB8
+#define mmREGAMMA_CNTLB_SLOPE_CNTL 0x1AB1
+#define mmREGAMMA_CNTLB_START_CNTL 0x1AB0
+#define mmREGAMMA_CONTROL 0x1AA0
+#define mmREGAMMA_LUT_DATA 0x1AA2
+#define mmREGAMMA_LUT_INDEX 0x1AA1
+#define mmREGAMMA_LUT_WRITE_EN_MASK 0x1AA3
+#define mmSCL0_EXT_OVERSCAN_LEFT_RIGHT 0x1B5E
+#define mmSCL0_EXT_OVERSCAN_TOP_BOTTOM 0x1B5F
+#define mmSCL0_SCL_ALU_CONTROL 0x1B54
+#define mmSCL0_SCL_AUTOMATIC_MODE_CONTROL 0x1B47
+#define mmSCL0_SCL_BYPASS_CONTROL 0x1B45
+#define mmSCL0_SCL_COEF_RAM_CONFLICT_STATUS 0x1B55
+#define mmSCL0_SCL_COEF_RAM_SELECT 0x1B40
+#define mmSCL0_SCL_COEF_RAM_TAP_DATA 0x1B41
+#define mmSCL0_SCL_CONTROL 0x1B44
+#define mmSCL0_SCL_DEBUG 0x1B6A
+#define mmSCL0_SCL_DEBUG2 0x1B69
+#define mmSCL0_SCL_F_SHARP_CONTROL 0x1B53
+#define mmSCL0_SCL_HORZ_FILTER_CONTROL 0x1B4A
+#define mmSCL0_SCL_HORZ_FILTER_SCALE_RATIO 0x1B4B
+#define mmSCL0_SCL_MANUAL_REPLICATE_CONTROL 0x1B46
+#define mmSCL0_SCL_MODE_CHANGE_DET1 0x1B60
+#define mmSCL0_SCL_MODE_CHANGE_DET2 0x1B61
+#define mmSCL0_SCL_MODE_CHANGE_DET3 0x1B62
+#define mmSCL0_SCL_MODE_CHANGE_MASK 0x1B63
+#define mmSCL0_SCL_TAP_CONTROL 0x1B43
+#define mmSCL0_SCL_TEST_DEBUG_DATA 0x1B6C
+#define mmSCL0_SCL_TEST_DEBUG_INDEX 0x1B6B
+#define mmSCL0_SCL_UPDATE 0x1B51
+#define mmSCL0_SCL_VERT_FILTER_CONTROL 0x1B4E
+#define mmSCL0_SCL_VERT_FILTER_INIT 0x1B50
+#define mmSCL0_SCL_VERT_FILTER_INIT_BOT 0x1B57
+#define mmSCL0_SCL_VERT_FILTER_SCALE_RATIO 0x1B4F
+#define mmSCL0_VIEWPORT_SIZE 0x1B5D
+#define mmSCL0_VIEWPORT_START 0x1B5C
+#define mmSCL1_EXT_OVERSCAN_LEFT_RIGHT 0x1E5E
+#define mmSCL1_EXT_OVERSCAN_TOP_BOTTOM 0x1E5F
+#define mmSCL1_SCL_ALU_CONTROL 0x1E54
+#define mmSCL1_SCL_AUTOMATIC_MODE_CONTROL 0x1E47
+#define mmSCL1_SCL_BYPASS_CONTROL 0x1E45
+#define mmSCL1_SCL_COEF_RAM_CONFLICT_STATUS 0x1E55
+#define mmSCL1_SCL_COEF_RAM_SELECT 0x1E40
+#define mmSCL1_SCL_COEF_RAM_TAP_DATA 0x1E41
+#define mmSCL1_SCL_CONTROL 0x1E44
+#define mmSCL1_SCL_DEBUG 0x1E6A
+#define mmSCL1_SCL_DEBUG2 0x1E69
+#define mmSCL1_SCL_F_SHARP_CONTROL 0x1E53
+#define mmSCL1_SCL_HORZ_FILTER_CONTROL 0x1E4A
+#define mmSCL1_SCL_HORZ_FILTER_SCALE_RATIO 0x1E4B
+#define mmSCL1_SCL_MANUAL_REPLICATE_CONTROL 0x1E46
+#define mmSCL1_SCL_MODE_CHANGE_DET1 0x1E60
+#define mmSCL1_SCL_MODE_CHANGE_DET2 0x1E61
+#define mmSCL1_SCL_MODE_CHANGE_DET3 0x1E62
+#define mmSCL1_SCL_MODE_CHANGE_MASK 0x1E63
+#define mmSCL1_SCL_TAP_CONTROL 0x1E43
+#define mmSCL1_SCL_TEST_DEBUG_DATA 0x1E6C
+#define mmSCL1_SCL_TEST_DEBUG_INDEX 0x1E6B
+#define mmSCL1_SCL_UPDATE 0x1E51
+#define mmSCL1_SCL_VERT_FILTER_CONTROL 0x1E4E
+#define mmSCL1_SCL_VERT_FILTER_INIT 0x1E50
+#define mmSCL1_SCL_VERT_FILTER_INIT_BOT 0x1E57
+#define mmSCL1_SCL_VERT_FILTER_SCALE_RATIO 0x1E4F
+#define mmSCL1_VIEWPORT_SIZE 0x1E5D
+#define mmSCL1_VIEWPORT_START 0x1E5C
+#define mmSCL2_EXT_OVERSCAN_LEFT_RIGHT 0x415E
+#define mmSCL2_EXT_OVERSCAN_TOP_BOTTOM 0x415F
+#define mmSCL2_SCL_ALU_CONTROL 0x4154
+#define mmSCL2_SCL_AUTOMATIC_MODE_CONTROL 0x4147
+#define mmSCL2_SCL_BYPASS_CONTROL 0x4145
+#define mmSCL2_SCL_COEF_RAM_CONFLICT_STATUS 0x4155
+#define mmSCL2_SCL_COEF_RAM_SELECT 0x4140
+#define mmSCL2_SCL_COEF_RAM_TAP_DATA 0x4141
+#define mmSCL2_SCL_CONTROL 0x4144
+#define mmSCL2_SCL_DEBUG 0x416A
+#define mmSCL2_SCL_DEBUG2 0x4169
+#define mmSCL2_SCL_F_SHARP_CONTROL 0x4153
+#define mmSCL2_SCL_HORZ_FILTER_CONTROL 0x414A
+#define mmSCL2_SCL_HORZ_FILTER_SCALE_RATIO 0x414B
+#define mmSCL2_SCL_MANUAL_REPLICATE_CONTROL 0x4146
+#define mmSCL2_SCL_MODE_CHANGE_DET1 0x4160
+#define mmSCL2_SCL_MODE_CHANGE_DET2 0x4161
+#define mmSCL2_SCL_MODE_CHANGE_DET3 0x4162
+#define mmSCL2_SCL_MODE_CHANGE_MASK 0x4163
+#define mmSCL2_SCL_TAP_CONTROL 0x4143
+#define mmSCL2_SCL_TEST_DEBUG_DATA 0x416C
+#define mmSCL2_SCL_TEST_DEBUG_INDEX 0x416B
+#define mmSCL2_SCL_UPDATE 0x4151
+#define mmSCL2_SCL_VERT_FILTER_CONTROL 0x414E
+#define mmSCL2_SCL_VERT_FILTER_INIT 0x4150
+#define mmSCL2_SCL_VERT_FILTER_INIT_BOT 0x4157
+#define mmSCL2_SCL_VERT_FILTER_SCALE_RATIO 0x414F
+#define mmSCL2_VIEWPORT_SIZE 0x415D
+#define mmSCL2_VIEWPORT_START 0x415C
+#define mmSCL3_EXT_OVERSCAN_LEFT_RIGHT 0x445E
+#define mmSCL3_EXT_OVERSCAN_TOP_BOTTOM 0x445F
+#define mmSCL3_SCL_ALU_CONTROL 0x4454
+#define mmSCL3_SCL_AUTOMATIC_MODE_CONTROL 0x4447
+#define mmSCL3_SCL_BYPASS_CONTROL 0x4445
+#define mmSCL3_SCL_COEF_RAM_CONFLICT_STATUS 0x4455
+#define mmSCL3_SCL_COEF_RAM_SELECT 0x4440
+#define mmSCL3_SCL_COEF_RAM_TAP_DATA 0x4441
+#define mmSCL3_SCL_CONTROL 0x4444
+#define mmSCL3_SCL_DEBUG 0x446A
+#define mmSCL3_SCL_DEBUG2 0x4469
+#define mmSCL3_SCL_F_SHARP_CONTROL 0x4453
+#define mmSCL3_SCL_HORZ_FILTER_CONTROL 0x444A
+#define mmSCL3_SCL_HORZ_FILTER_SCALE_RATIO 0x444B
+#define mmSCL3_SCL_MANUAL_REPLICATE_CONTROL 0x4446
+#define mmSCL3_SCL_MODE_CHANGE_DET1 0x4460
+#define mmSCL3_SCL_MODE_CHANGE_DET2 0x4461
+#define mmSCL3_SCL_MODE_CHANGE_DET3 0x4462
+#define mmSCL3_SCL_MODE_CHANGE_MASK 0x4463
+#define mmSCL3_SCL_TAP_CONTROL 0x4443
+#define mmSCL3_SCL_TEST_DEBUG_DATA 0x446C
+#define mmSCL3_SCL_TEST_DEBUG_INDEX 0x446B
+#define mmSCL3_SCL_UPDATE 0x4451
+#define mmSCL3_SCL_VERT_FILTER_CONTROL 0x444E
+#define mmSCL3_SCL_VERT_FILTER_INIT 0x4450
+#define mmSCL3_SCL_VERT_FILTER_INIT_BOT 0x4457
+#define mmSCL3_SCL_VERT_FILTER_SCALE_RATIO 0x444F
+#define mmSCL3_VIEWPORT_SIZE 0x445D
+#define mmSCL3_VIEWPORT_START 0x445C
+#define mmSCL4_EXT_OVERSCAN_LEFT_RIGHT 0x475E
+#define mmSCL4_EXT_OVERSCAN_TOP_BOTTOM 0x475F
+#define mmSCL4_SCL_ALU_CONTROL 0x4754
+#define mmSCL4_SCL_AUTOMATIC_MODE_CONTROL 0x4747
+#define mmSCL4_SCL_BYPASS_CONTROL 0x4745
+#define mmSCL4_SCL_COEF_RAM_CONFLICT_STATUS 0x4755
+#define mmSCL4_SCL_COEF_RAM_SELECT 0x4740
+#define mmSCL4_SCL_COEF_RAM_TAP_DATA 0x4741
+#define mmSCL4_SCL_CONTROL 0x4744
+#define mmSCL4_SCL_DEBUG 0x476A
+#define mmSCL4_SCL_DEBUG2 0x4769
+#define mmSCL4_SCL_F_SHARP_CONTROL 0x4753
+#define mmSCL4_SCL_HORZ_FILTER_CONTROL 0x474A
+#define mmSCL4_SCL_HORZ_FILTER_SCALE_RATIO 0x474B
+#define mmSCL4_SCL_MANUAL_REPLICATE_CONTROL 0x4746
+#define mmSCL4_SCL_MODE_CHANGE_DET1 0x4760
+#define mmSCL4_SCL_MODE_CHANGE_DET2 0x4761
+#define mmSCL4_SCL_MODE_CHANGE_DET3 0x4762
+#define mmSCL4_SCL_MODE_CHANGE_MASK 0x4763
+#define mmSCL4_SCL_TAP_CONTROL 0x4743
+#define mmSCL4_SCL_TEST_DEBUG_DATA 0x476C
+#define mmSCL4_SCL_TEST_DEBUG_INDEX 0x476B
+#define mmSCL4_SCL_UPDATE 0x4751
+#define mmSCL4_SCL_VERT_FILTER_CONTROL 0x474E
+#define mmSCL4_SCL_VERT_FILTER_INIT 0x4750
+#define mmSCL4_SCL_VERT_FILTER_INIT_BOT 0x4757
+#define mmSCL4_SCL_VERT_FILTER_SCALE_RATIO 0x474F
+#define mmSCL4_VIEWPORT_SIZE 0x475D
+#define mmSCL4_VIEWPORT_START 0x475C
+#define mmSCL5_EXT_OVERSCAN_LEFT_RIGHT 0x4A5E
+#define mmSCL5_EXT_OVERSCAN_TOP_BOTTOM 0x4A5F
+#define mmSCL5_SCL_ALU_CONTROL 0x4A54
+#define mmSCL5_SCL_AUTOMATIC_MODE_CONTROL 0x4A47
+#define mmSCL5_SCL_BYPASS_CONTROL 0x4A45
+#define mmSCL5_SCL_COEF_RAM_CONFLICT_STATUS 0x4A55
+#define mmSCL5_SCL_COEF_RAM_SELECT 0x4A40
+#define mmSCL5_SCL_COEF_RAM_TAP_DATA 0x4A41
+#define mmSCL5_SCL_CONTROL 0x4A44
+#define mmSCL5_SCL_DEBUG 0x4A6A
+#define mmSCL5_SCL_DEBUG2 0x4A69
+#define mmSCL5_SCL_F_SHARP_CONTROL 0x4A53
+#define mmSCL5_SCL_HORZ_FILTER_CONTROL 0x4A4A
+#define mmSCL5_SCL_HORZ_FILTER_SCALE_RATIO 0x4A4B
+#define mmSCL5_SCL_MANUAL_REPLICATE_CONTROL 0x4A46
+#define mmSCL5_SCL_MODE_CHANGE_DET1 0x4A60
+#define mmSCL5_SCL_MODE_CHANGE_DET2 0x4A61
+#define mmSCL5_SCL_MODE_CHANGE_DET3 0x4A62
+#define mmSCL5_SCL_MODE_CHANGE_MASK 0x4A63
+#define mmSCL5_SCL_TAP_CONTROL 0x4A43
+#define mmSCL5_SCL_TEST_DEBUG_DATA 0x4A6C
+#define mmSCL5_SCL_TEST_DEBUG_INDEX 0x4A6B
+#define mmSCL5_SCL_UPDATE 0x4A51
+#define mmSCL5_SCL_VERT_FILTER_CONTROL 0x4A4E
+#define mmSCL5_SCL_VERT_FILTER_INIT 0x4A50
+#define mmSCL5_SCL_VERT_FILTER_INIT_BOT 0x4A57
+#define mmSCL5_SCL_VERT_FILTER_SCALE_RATIO 0x4A4F
+#define mmSCL5_VIEWPORT_SIZE 0x4A5D
+#define mmSCL5_VIEWPORT_START 0x4A5C
+#define mmSCL_ALU_CONTROL 0x1B54
+#define mmSCL_AUTOMATIC_MODE_CONTROL 0x1B47
+#define mmSCL_BYPASS_CONTROL 0x1B45
+#define mmSCL_COEF_RAM_CONFLICT_STATUS 0x1B55
+#define mmSCL_COEF_RAM_SELECT 0x1B40
+#define mmSCL_COEF_RAM_TAP_DATA 0x1B41
+#define mmSCL_CONTROL 0x1B44
+#define mmSCL_DEBUG 0x1B6A
+#define mmSCL_DEBUG2 0x1B69
+#define mmSCL_F_SHARP_CONTROL 0x1B53
+#define mmSCL_HORZ_FILTER_CONTROL 0x1B4A
+#define mmSCL_HORZ_FILTER_SCALE_RATIO 0x1B4B
+#define mmSCLK_CGTT_BLK_CTRL_REG 0x0136
+#define mmSCL_MANUAL_REPLICATE_CONTROL 0x1B46
+#define mmSCL_MODE_CHANGE_DET1 0x1B60
+#define mmSCL_MODE_CHANGE_DET2 0x1B61
+#define mmSCL_MODE_CHANGE_DET3 0x1B62
+#define mmSCL_MODE_CHANGE_MASK 0x1B63
+#define mmSCL_TAP_CONTROL 0x1B43
+#define mmSCL_TEST_DEBUG_DATA 0x1B6C
+#define mmSCL_TEST_DEBUG_INDEX 0x1B6B
+#define mmSCL_UPDATE 0x1B51
+#define mmSCL_VERT_FILTER_CONTROL 0x1B4E
+#define mmSCL_VERT_FILTER_INIT 0x1B50
+#define mmSCL_VERT_FILTER_INIT_BOT 0x1B57
+#define mmSCL_VERT_FILTER_SCALE_RATIO 0x1B4F
+#define mmSEQ8_DATA 0x00F1
+#define mmSEQ8_IDX 0x00F1
+#define mmSLAVE_COMM_CMD_REG 0x1624
+#define mmSLAVE_COMM_CNTL_REG 0x1625
+#define mmSLAVE_COMM_DATA_REG1 0x1621
+#define mmSLAVE_COMM_DATA_REG2 0x1622
+#define mmSLAVE_COMM_DATA_REG3 0x1623
+#define mmSYMCLKA_CLOCK_ENABLE 0x0160
+#define mmSYMCLKB_CLOCK_ENABLE 0x0161
+#define mmSYMCLKC_CLOCK_ENABLE 0x0162
+#define mmSYMCLKD_CLOCK_ENABLE 0x0163
+#define mmSYMCLKE_CLOCK_ENABLE 0x0164
+#define mmSYMCLKF_CLOCK_ENABLE 0x0165
+#define mmTMDS_CNTL 0x1C7C
+#define mmTMDS_CONTROL0_FEEDBACK 0x1C7E
+#define mmTMDS_CONTROL_CHAR 0x1C7D
+#define mmTMDS_CTL0_1_GEN_CNTL 0x1C86
+#define mmTMDS_CTL2_3_GEN_CNTL 0x1C87
+#define mmTMDS_CTL_BITS 0x1C83
+#define mmTMDS_DCBALANCER_CONTROL 0x1C84
+#define mmTMDS_DEBUG 0x1C82
+#define mmTMDS_STEREOSYNC_CTL_SEL 0x1C7F
+#define mmTMDS_SYNC_CHAR_PATTERN_0_1 0x1C80
+#define mmTMDS_SYNC_CHAR_PATTERN_2_3 0x1C81
+#define mmUNIPHYAB_TPG_CONTROL 0x1931
+#define mmUNIPHYAB_TPG_SEED 0x1932
+#define mmUNIPHY_ANG_BIST_CNTL 0x198C
+#define mmUNIPHYCD_TPG_CONTROL 0x1933
+#define mmUNIPHYCD_TPG_SEED 0x1934
+#define mmUNIPHY_CHANNEL_XBAR_CNTL 0x198E
+#define mmUNIPHY_DATA_SYNCHRONIZATION 0x198A
+#define mmUNIPHYEF_TPG_CONTROL 0x1935
+#define mmUNIPHYEF_TPG_SEED 0x1936
+#define mmUNIPHY_IMPCAL_LINKA 0x1908
+#define mmUNIPHY_IMPCAL_LINKB 0x1909
+#define mmUNIPHY_IMPCAL_LINKC 0x190F
+#define mmUNIPHY_IMPCAL_LINKD 0x1910
+#define mmUNIPHY_IMPCAL_LINKE 0x1913
+#define mmUNIPHY_IMPCAL_LINKF 0x1914
+#define mmUNIPHY_IMPCAL_PERIOD 0x190A
+#define mmUNIPHY_IMPCAL_PSW_AB 0x190E
+#define mmUNIPHY_IMPCAL_PSW_CD 0x1912
+#define mmUNIPHY_IMPCAL_PSW_EF 0x1916
+#define mmUNIPHY_LINK_CNTL 0x198D
+#define mmUNIPHY_PLL_CONTROL1 0x1986
+#define mmUNIPHY_PLL_CONTROL2 0x1987
+#define mmUNIPHY_PLL_FBDIV 0x1985
+#define mmUNIPHY_PLL_SS_CNTL 0x1989
+#define mmUNIPHY_PLL_SS_STEP_SIZE 0x1988
+#define mmUNIPHY_POWER_CONTROL 0x1984
+#define mmUNIPHY_REG_TEST_OUTPUT 0x198B
+#define mmUNIPHY_SOFT_RESET 0x0166
+#define mmUNIPHY_TX_CONTROL1 0x1980
+#define mmUNIPHY_TX_CONTROL2 0x1981
+#define mmUNIPHY_TX_CONTROL3 0x1982
+#define mmUNIPHY_TX_CONTROL4 0x1983
+#define mmVGA25_PPLL_ANALOG 0x00E4
+#define mmVGA25_PPLL_FB_DIV 0x00DC
+#define mmVGA25_PPLL_POST_DIV 0x00E0
+#define mmVGA25_PPLL_REF_DIV 0x00D8
+#define mmVGA28_PPLL_ANALOG 0x00E5
+#define mmVGA28_PPLL_FB_DIV 0x00DD
+#define mmVGA28_PPLL_POST_DIV 0x00E1
+#define mmVGA28_PPLL_REF_DIV 0x00D9
+#define mmVGA41_PPLL_ANALOG 0x00E6
+#define mmVGA41_PPLL_FB_DIV 0x00DE
+#define mmVGA41_PPLL_POST_DIV 0x00E2
+#define mmVGA41_PPLL_REF_DIV 0x00DA
+#define mmVGA_CACHE_CONTROL 0x00CB
+#define mmVGA_DEBUG_READBACK_DATA 0x00D7
+#define mmVGA_DEBUG_READBACK_INDEX 0x00D6
+#define mmVGA_DISPBUF1_SURFACE_ADDR 0x00C6
+#define mmVGA_DISPBUF2_SURFACE_ADDR 0x00C8
+#define mmVGA_HDP_CONTROL 0x00CA
+#define mmVGA_HW_DEBUG 0x00CF
+#define mmVGA_INTERRUPT_CONTROL 0x00D1
+#define mmVGA_INTERRUPT_STATUS 0x00D3
+#define mmVGA_MAIN_CONTROL 0x00D4
+#define mmVGA_MEMORY_BASE_ADDRESS 0x00C4
+#define mmVGA_MEMORY_BASE_ADDRESS_HIGH 0x00C9
+#define mmVGA_MEM_READ_PAGE_ADDR 0x0013
+#define mmVGA_MEM_WRITE_PAGE_ADDR 0x0012
+#define mmVGA_MODE_CONTROL 0x00C2
+#define mmVGA_RENDER_CONTROL 0x00C0
+#define mmVGA_SEQUENCER_RESET_CONTROL 0x00C1
+#define mmVGA_SOURCE_SELECT 0x00FC
+#define mmVGA_STATUS 0x00D0
+#define mmVGA_STATUS_CLEAR 0x00D2
+#define mmVGA_SURFACE_PITCH_SELECT 0x00C3
+#define mmVGA_TEST_CONTROL 0x00D5
+#define mmVGA_TEST_DEBUG_DATA 0x00C7
+#define mmVGA_TEST_DEBUG_INDEX 0x00C5
+#define mmVIEWPORT_SIZE 0x1B5D
+#define mmVIEWPORT_START 0x1B5C
+#define mmXDMA_CLOCK_GATING_CNTL 0x0409
+#define mmXDMA_IF_BIF_STATUS 0x0418
+#define mmXDMA_INTERRUPT 0x0406
+#define mmXDMA_LOCAL_SURFACE_TILING1 0x03F4
+#define mmXDMA_LOCAL_SURFACE_TILING2 0x03F5
+#define mmXDMA_MC_PCIE_CLIENT_CONFIG 0x03E9
+#define mmXDMA_MEM_POWER_CNTL 0x040B
+#define mmXDMA_MSTR_CMD_URGENT_CNTL 0x03F6
+#define mmXDMA_MSTR_CNTL 0x03E0
+#define mmXDMA_MSTR_HEIGHT 0x03E3
+#define mmXDMA_MSTR_LOCAL_SURFACE_BASE_ADDR 0x03F1
+#define mmXDMA_MSTR_LOCAL_SURFACE_BASE_ADDR_HIGH 0x03F2
+#define mmXDMA_MSTR_LOCAL_SURFACE_PITCH 0x03F3
+#define mmXDMA_MSTR_MEM_CLIENT_CONFIG 0x03EA
+#define mmXDMA_MSTR_MEM_NACK_STATUS 0x040D
+#define mmXDMA_MSTR_MEM_URGENT_CNTL 0x03F7
+#define mmXDMA_MSTR_PCIE_NACK_STATUS 0x040C
+#define mmXDMA_MSTR_READ_COMMAND 0x03E1
+#define mmXDMA_MSTR_REMOTE_GPU_ADDRESS 0x03E6
+#define mmXDMA_MSTR_REMOTE_GPU_ADDRESS_HIGH 0x03E7
+#define mmXDMA_MSTR_REMOTE_SURFACE_BASE 0x03E4
+#define mmXDMA_MSTR_REMOTE_SURFACE_BASE_HIGH 0x03E5
+#define mmXDMA_MSTR_STATUS 0x03E8
+#define mmXDMA_RBBMIF_RDWR_CNTL 0x040A
+#define mmXDMA_SLV_CNTL 0x03FB
+#define mmXDMA_SLV_FLIP_PENDING 0x0407
+#define mmXDMA_SLV_MEM_CLIENT_CONFIG 0x03FD
+#define mmXDMA_SLV_MEM_NACK_STATUS 0x040F
+#define mmXDMA_SLV_PCIE_NACK_STATUS 0x040E
+#define mmXDMA_SLV_READ_LATENCY_AVE 0x0405
+#define mmXDMA_SLV_READ_LATENCY_MINMAX 0x0404
+#define mmXDMA_SLV_READ_LATENCY_TIMER 0x0412
+#define mmXDMA_SLV_READ_URGENT_CNTL 0x03FF
+#define mmXDMA_SLV_REMOTE_GPU_ADDRESS 0x0402
+#define mmXDMA_SLV_REMOTE_GPU_ADDRESS_HIGH 0x0403
+#define mmXDMA_SLV_SLS_PITCH 0x03FE
+#define mmXDMA_SLV_WB_RATE_CNTL 0x0401
+#define mmXDMA_SLV_WRITE_URGENT_CNTL 0x0400
+#define mmXDMA_TEST_DEBUG_DATA 0x041D
+#define mmXDMA_TEST_DEBUG_INDEX 0x041C
+
+/* Registers that spilled out of sid.h */
+#define mmDATA_FORMAT                              0x1AC0
+#define mmDESKTOP_HEIGHT                           0x1AC1
+#define mmDC_LB_MEMORY_SPLIT                       0x1AC3
+#define mmPRIORITY_A_CNT                           0x1AC6
+#define mmPRIORITY_B_CNT                           0x1AC7
+#define mmDPG_PIPE_ARBITRATION_CONTROL3            0x1B32
+#define mmINT_MASK                                 0x1AD0
+#define mmVLINE_STATUS                             0x1AEE
+#define mmVBLANK_STATUS                            0x1AEF
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_sh_mask.h
new file mode 100644
index 000000000000..9a4d4c299d5b
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_6_0_sh_mask.h
@@ -0,0 +1,9836 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef DCE_6_0_SH_MASK_H
+#define DCE_6_0_SH_MASK_H
+
+#define ABM_TEST_DEBUG_DATA__ABM_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define ABM_TEST_DEBUG_DATA__ABM_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define ABM_TEST_DEBUG_INDEX__ABM_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define ABM_TEST_DEBUG_INDEX__ABM_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define ABM_TEST_DEBUG_INDEX__ABM_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define ABM_TEST_DEBUG_INDEX__ABM_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define AFMT_60958_0__AFMT_60958_CS_A_MASK 0x00000001L
+#define AFMT_60958_0__AFMT_60958_CS_A__SHIFT 0x00000000
+#define AFMT_60958_0__AFMT_60958_CS_B_MASK 0x00000002L
+#define AFMT_60958_0__AFMT_60958_CS_B__SHIFT 0x00000001
+#define AFMT_60958_0__AFMT_60958_CS_CATEGORY_CODE_MASK 0x0000ff00L
+#define AFMT_60958_0__AFMT_60958_CS_CATEGORY_CODE__SHIFT 0x00000008
+#define AFMT_60958_0__AFMT_60958_CS_CHANNEL_NUMBER_L_MASK 0x00f00000L
+#define AFMT_60958_0__AFMT_60958_CS_CHANNEL_NUMBER_L__SHIFT 0x00000014
+#define AFMT_60958_0__AFMT_60958_CS_CLOCK_ACCURACY_MASK 0x30000000L
+#define AFMT_60958_0__AFMT_60958_CS_CLOCK_ACCURACY__SHIFT 0x0000001c
+#define AFMT_60958_0__AFMT_60958_CS_C_MASK 0x00000004L
+#define AFMT_60958_0__AFMT_60958_CS_C__SHIFT 0x00000002
+#define AFMT_60958_0__AFMT_60958_CS_D_MASK 0x00000038L
+#define AFMT_60958_0__AFMT_60958_CS_D__SHIFT 0x00000003
+#define AFMT_60958_0__AFMT_60958_CS_MODE_MASK 0x000000c0L
+#define AFMT_60958_0__AFMT_60958_CS_MODE__SHIFT 0x00000006
+#define AFMT_60958_0__AFMT_60958_CS_SAMPLING_FREQUENCY_MASK 0x0f000000L
+#define AFMT_60958_0__AFMT_60958_CS_SAMPLING_FREQUENCY__SHIFT 0x00000018
+#define AFMT_60958_0__AFMT_60958_CS_SOURCE_NUMBER_MASK 0x000f0000L
+#define AFMT_60958_0__AFMT_60958_CS_SOURCE_NUMBER__SHIFT 0x00000010
+#define AFMT_60958_1__AFMT_60958_CS_CHANNEL_NUMBER_R_MASK 0x00f00000L
+#define AFMT_60958_1__AFMT_60958_CS_CHANNEL_NUMBER_R__SHIFT 0x00000014
+#define AFMT_60958_1__AFMT_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK 0x000000f0L
+#define AFMT_60958_1__AFMT_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT 0x00000004
+#define AFMT_60958_1__AFMT_60958_CS_WORD_LENGTH_MASK 0x0000000fL
+#define AFMT_60958_1__AFMT_60958_CS_WORD_LENGTH__SHIFT 0x00000000
+#define AFMT_60958_1__AFMT_60958_VALID_L_MASK 0x00010000L
+#define AFMT_60958_1__AFMT_60958_VALID_L__SHIFT 0x00000010
+#define AFMT_60958_1__AFMT_60958_VALID_R_MASK 0x00040000L
+#define AFMT_60958_1__AFMT_60958_VALID_R__SHIFT 0x00000012
+#define AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_2_MASK 0x0000000fL
+#define AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_2__SHIFT 0x00000000
+#define AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_3_MASK 0x000000f0L
+#define AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_3__SHIFT 0x00000004
+#define AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_4_MASK 0x00000f00L
+#define AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_4__SHIFT 0x00000008
+#define AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_5_MASK 0x0000f000L
+#define AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_5__SHIFT 0x0000000c
+#define AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_6_MASK 0x000f0000L
+#define AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_6__SHIFT 0x00000010
+#define AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_7_MASK 0x00f00000L
+#define AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_7__SHIFT 0x00000014
+#define AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CH_SEL_MASK 0x0000f000L
+#define AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CH_SEL__SHIFT 0x0000000c
+#define AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CONT_MASK 0x00000010L
+#define AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CONT__SHIFT 0x00000004
+#define AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_COUNT_MASK 0xffff0000L
+#define AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_COUNT__SHIFT 0x00000010
+#define AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_EN_MASK 0x00000001L
+#define AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_EN__SHIFT 0x00000000
+#define AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_SOURCE_MASK 0x00000100L
+#define AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_SOURCE__SHIFT 0x00000008
+#define AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC_DONE_MASK 0x00000001L
+#define AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC_DONE__SHIFT 0x00000000
+#define AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC_MASK 0xffffff00L
+#define AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC__SHIFT 0x00000008
+#define AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_BASE_MASK 0x00000100L
+#define AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_BASE__SHIFT 0x00000008
+#define AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_DIV_MASK 0x00070000L
+#define AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_DIV__SHIFT 0x00000010
+#define AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_MULTI_MASK 0x00007000L
+#define AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_MULTI__SHIFT 0x0000000c
+#define AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_FS_DIV_SEL_MASK 0x00000007L
+#define AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_FS_DIV_SEL__SHIFT 0x00000000
+#define AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CC_MASK 0x00000700L
+#define AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CC__SHIFT 0x00000008
+#define AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM_MASK 0x000000ffL
+#define AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM_OFFSET_MASK 0x00ff0000L
+#define AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM_OFFSET__SHIFT 0x00000010
+#define AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM__SHIFT 0x00000000
+#define AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CT_MASK 0x00007800L
+#define AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CT__SHIFT 0x0000000b
+#define AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CXT_MASK 0x1f000000L
+#define AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CXT__SHIFT 0x00000018
+#define AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_CA_MASK 0x000000ffL
+#define AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_CA__SHIFT 0x00000000
+#define AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_DM_INH_MASK 0x00008000L
+#define AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_DM_INH__SHIFT 0x0000000f
+#define AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LFEPBL_MASK 0x00030000L
+#define AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LFEPBL__SHIFT 0x00000010
+#define AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LSV_MASK 0x00007800L
+#define AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LSV__SHIFT 0x0000000b
+#define AFMT_AUDIO_PACKET_CONTROL2__AFMT_60958_OSF_OVRD_MASK 0x10000000L
+#define AFMT_AUDIO_PACKET_CONTROL2__AFMT_60958_OSF_OVRD__SHIFT 0x0000001c
+#define AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_CHANNEL_ENABLE_MASK 0x0000ff00L
+#define AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_CHANNEL_ENABLE__SHIFT 0x00000008
+#define AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_OVRD_MASK 0x00000001L
+#define AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_OVRD__SHIFT 0x00000000
+#define AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_SELECT_MASK 0x00000002L
+#define AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_SELECT__SHIFT 0x00000001
+#define AFMT_AUDIO_PACKET_CONTROL2__AFMT_DP_AUDIO_STREAM_ID_MASK 0x00ff0000L
+#define AFMT_AUDIO_PACKET_CONTROL2__AFMT_DP_AUDIO_STREAM_ID__SHIFT 0x00000010
+#define AFMT_AUDIO_PACKET_CONTROL2__AFMT_HBR_ENABLE_OVRD_MASK 0x01000000L
+#define AFMT_AUDIO_PACKET_CONTROL2__AFMT_HBR_ENABLE_OVRD__SHIFT 0x00000018
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_60958_CS_UPDATE_MASK 0x04000000L
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_60958_CS_UPDATE__SHIFT 0x0000001a
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_CHANNEL_SWAP_MASK 0x01000000L
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_CHANNEL_SWAP__SHIFT 0x00000018
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_FIFO_OVERFLOW_ACK_MASK 0x00800000L
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_FIFO_OVERFLOW_ACK__SHIFT 0x00000017
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND_MASK 0x00000001L
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND__SHIFT 0x00000000
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_EN_MASK 0x00001000L
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_EN__SHIFT 0x0000000c
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_MODE_MASK 0x00004000L
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_MODE__SHIFT 0x0000000e
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_AZ_AUDIO_ENABLE_CHG_ACK_MASK 0x40000000L
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_AZ_AUDIO_ENABLE_CHG_ACK__SHIFT 0x0000001e
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_BLANK_TEST_DATA_ON_ENC_ENB_MASK 0x80000000L
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_BLANK_TEST_DATA_ON_ENC_ENB__SHIFT 0x0000001f
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_RESET_FIFO_WHEN_AUDIO_DIS_MASK 0x00000800L
+#define AFMT_AUDIO_PACKET_CONTROL__AFMT_RESET_FIFO_WHEN_AUDIO_DIS__SHIFT 0x0000000b
+#define AFMT_AUDIO_SRC_CONTROL__AFMT_AUDIO_SRC_SELECT_MASK 0x00000007L
+#define AFMT_AUDIO_SRC_CONTROL__AFMT_AUDIO_SRC_SELECT__SHIFT 0x00000000
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_A_MASK 0x00001000L
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_A__SHIFT 0x0000000c
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_B_MASK 0x00000c00L
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_B__SHIFT 0x0000000a
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_CHECKSUM_MASK 0x000000ffL
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_CHECKSUM__SHIFT 0x00000000
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_C_MASK 0x00c00000L
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_C__SHIFT 0x00000016
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_EC_MASK 0x70000000L
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_EC__SHIFT 0x0000001c
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_ITC_MASK 0x80000000L
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_ITC__SHIFT 0x0000001f
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_M_MASK 0x00300000L
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_M__SHIFT 0x00000014
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_PB1_RSVD_MASK 0x00008000L
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_PB1_RSVD__SHIFT 0x0000000f
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_Q_MASK 0x0c000000L
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_Q__SHIFT 0x0000001a
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_R_MASK 0x000f0000L
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_R__SHIFT 0x00000010
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_SC_MASK 0x03000000L
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_SC__SHIFT 0x00000018
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_S_MASK 0x00000300L
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_S__SHIFT 0x00000008
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_Y_MASK 0x00006000L
+#define AFMT_AVI_INFO0__AFMT_AVI_INFO_Y__SHIFT 0x0000000d
+#define AFMT_AVI_INFO1__AFMT_AVI_INFO_CN_MASK 0x00003000L
+#define AFMT_AVI_INFO1__AFMT_AVI_INFO_CN__SHIFT 0x0000000c
+#define AFMT_AVI_INFO1__AFMT_AVI_INFO_PB4_RSVD_MASK 0x00000080L
+#define AFMT_AVI_INFO1__AFMT_AVI_INFO_PB4_RSVD__SHIFT 0x00000007
+#define AFMT_AVI_INFO1__AFMT_AVI_INFO_PR_MASK 0x00000f00L
+#define AFMT_AVI_INFO1__AFMT_AVI_INFO_PR__SHIFT 0x00000008
+#define AFMT_AVI_INFO1__AFMT_AVI_INFO_TOP_MASK 0xffff0000L
+#define AFMT_AVI_INFO1__AFMT_AVI_INFO_TOP__SHIFT 0x00000010
+#define AFMT_AVI_INFO1__AFMT_AVI_INFO_VIC_MASK 0x0000007fL
+#define AFMT_AVI_INFO1__AFMT_AVI_INFO_VIC__SHIFT 0x00000000
+#define AFMT_AVI_INFO1__AFMT_AVI_INFO_YQ_MASK 0x0000c000L
+#define AFMT_AVI_INFO1__AFMT_AVI_INFO_YQ__SHIFT 0x0000000e
+#define AFMT_AVI_INFO2__AFMT_AVI_INFO_BOTTOM_MASK 0x0000ffffL
+#define AFMT_AVI_INFO2__AFMT_AVI_INFO_BOTTOM__SHIFT 0x00000000
+#define AFMT_AVI_INFO2__AFMT_AVI_INFO_LEFT_MASK 0xffff0000L
+#define AFMT_AVI_INFO2__AFMT_AVI_INFO_LEFT__SHIFT 0x00000010
+#define AFMT_AVI_INFO3__AFMT_AVI_INFO_RIGHT_MASK 0x0000ffffL
+#define AFMT_AVI_INFO3__AFMT_AVI_INFO_RIGHT__SHIFT 0x00000000
+#define AFMT_AVI_INFO3__AFMT_AVI_INFO_VERSION_MASK 0xff000000L
+#define AFMT_AVI_INFO3__AFMT_AVI_INFO_VERSION__SHIFT 0x00000018
+#define AFMT_GENERIC_0__AFMT_GENERIC_BYTE0_MASK 0x000000ffL
+#define AFMT_GENERIC_0__AFMT_GENERIC_BYTE0__SHIFT 0x00000000
+#define AFMT_GENERIC_0__AFMT_GENERIC_BYTE1_MASK 0x0000ff00L
+#define AFMT_GENERIC_0__AFMT_GENERIC_BYTE1__SHIFT 0x00000008
+#define AFMT_GENERIC_0__AFMT_GENERIC_BYTE2_MASK 0x00ff0000L
+#define AFMT_GENERIC_0__AFMT_GENERIC_BYTE2__SHIFT 0x00000010
+#define AFMT_GENERIC_0__AFMT_GENERIC_BYTE3_MASK 0xff000000L
+#define AFMT_GENERIC_0__AFMT_GENERIC_BYTE3__SHIFT 0x00000018
+#define AFMT_GENERIC_1__AFMT_GENERIC_BYTE4_MASK 0x000000ffL
+#define AFMT_GENERIC_1__AFMT_GENERIC_BYTE4__SHIFT 0x00000000
+#define AFMT_GENERIC_1__AFMT_GENERIC_BYTE5_MASK 0x0000ff00L
+#define AFMT_GENERIC_1__AFMT_GENERIC_BYTE5__SHIFT 0x00000008
+#define AFMT_GENERIC_1__AFMT_GENERIC_BYTE6_MASK 0x00ff0000L
+#define AFMT_GENERIC_1__AFMT_GENERIC_BYTE6__SHIFT 0x00000010
+#define AFMT_GENERIC_1__AFMT_GENERIC_BYTE7_MASK 0xff000000L
+#define AFMT_GENERIC_1__AFMT_GENERIC_BYTE7__SHIFT 0x00000018
+#define AFMT_GENERIC_2__AFMT_GENERIC_BYTE10_MASK 0x00ff0000L
+#define AFMT_GENERIC_2__AFMT_GENERIC_BYTE10__SHIFT 0x00000010
+#define AFMT_GENERIC_2__AFMT_GENERIC_BYTE11_MASK 0xff000000L
+#define AFMT_GENERIC_2__AFMT_GENERIC_BYTE11__SHIFT 0x00000018
+#define AFMT_GENERIC_2__AFMT_GENERIC_BYTE8_MASK 0x000000ffL
+#define AFMT_GENERIC_2__AFMT_GENERIC_BYTE8__SHIFT 0x00000000
+#define AFMT_GENERIC_2__AFMT_GENERIC_BYTE9_MASK 0x0000ff00L
+#define AFMT_GENERIC_2__AFMT_GENERIC_BYTE9__SHIFT 0x00000008
+#define AFMT_GENERIC_3__AFMT_GENERIC_BYTE12_MASK 0x000000ffL
+#define AFMT_GENERIC_3__AFMT_GENERIC_BYTE12__SHIFT 0x00000000
+#define AFMT_GENERIC_3__AFMT_GENERIC_BYTE13_MASK 0x0000ff00L
+#define AFMT_GENERIC_3__AFMT_GENERIC_BYTE13__SHIFT 0x00000008
+#define AFMT_GENERIC_3__AFMT_GENERIC_BYTE14_MASK 0x00ff0000L
+#define AFMT_GENERIC_3__AFMT_GENERIC_BYTE14__SHIFT 0x00000010
+#define AFMT_GENERIC_3__AFMT_GENERIC_BYTE15_MASK 0xff000000L
+#define AFMT_GENERIC_3__AFMT_GENERIC_BYTE15__SHIFT 0x00000018
+#define AFMT_GENERIC_4__AFMT_GENERIC_BYTE16_MASK 0x000000ffL
+#define AFMT_GENERIC_4__AFMT_GENERIC_BYTE16__SHIFT 0x00000000
+#define AFMT_GENERIC_4__AFMT_GENERIC_BYTE17_MASK 0x0000ff00L
+#define AFMT_GENERIC_4__AFMT_GENERIC_BYTE17__SHIFT 0x00000008
+#define AFMT_GENERIC_4__AFMT_GENERIC_BYTE18_MASK 0x00ff0000L
+#define AFMT_GENERIC_4__AFMT_GENERIC_BYTE18__SHIFT 0x00000010
+#define AFMT_GENERIC_4__AFMT_GENERIC_BYTE19_MASK 0xff000000L
+#define AFMT_GENERIC_4__AFMT_GENERIC_BYTE19__SHIFT 0x00000018
+#define AFMT_GENERIC_5__AFMT_GENERIC_BYTE20_MASK 0x000000ffL
+#define AFMT_GENERIC_5__AFMT_GENERIC_BYTE20__SHIFT 0x00000000
+#define AFMT_GENERIC_5__AFMT_GENERIC_BYTE21_MASK 0x0000ff00L
+#define AFMT_GENERIC_5__AFMT_GENERIC_BYTE21__SHIFT 0x00000008
+#define AFMT_GENERIC_5__AFMT_GENERIC_BYTE22_MASK 0x00ff0000L
+#define AFMT_GENERIC_5__AFMT_GENERIC_BYTE22__SHIFT 0x00000010
+#define AFMT_GENERIC_5__AFMT_GENERIC_BYTE23_MASK 0xff000000L
+#define AFMT_GENERIC_5__AFMT_GENERIC_BYTE23__SHIFT 0x00000018
+#define AFMT_GENERIC_6__AFMT_GENERIC_BYTE24_MASK 0x000000ffL
+#define AFMT_GENERIC_6__AFMT_GENERIC_BYTE24__SHIFT 0x00000000
+#define AFMT_GENERIC_6__AFMT_GENERIC_BYTE25_MASK 0x0000ff00L
+#define AFMT_GENERIC_6__AFMT_GENERIC_BYTE25__SHIFT 0x00000008
+#define AFMT_GENERIC_6__AFMT_GENERIC_BYTE26_MASK 0x00ff0000L
+#define AFMT_GENERIC_6__AFMT_GENERIC_BYTE26__SHIFT 0x00000010
+#define AFMT_GENERIC_6__AFMT_GENERIC_BYTE27_MASK 0xff000000L
+#define AFMT_GENERIC_6__AFMT_GENERIC_BYTE27__SHIFT 0x00000018
+#define AFMT_GENERIC_7__AFMT_GENERIC_BYTE28_MASK 0x000000ffL
+#define AFMT_GENERIC_7__AFMT_GENERIC_BYTE28__SHIFT 0x00000000
+#define AFMT_GENERIC_7__AFMT_GENERIC_BYTE29_MASK 0x0000ff00L
+#define AFMT_GENERIC_7__AFMT_GENERIC_BYTE29__SHIFT 0x00000008
+#define AFMT_GENERIC_7__AFMT_GENERIC_BYTE30_MASK 0x00ff0000L
+#define AFMT_GENERIC_7__AFMT_GENERIC_BYTE30__SHIFT 0x00000010
+#define AFMT_GENERIC_7__AFMT_GENERIC_BYTE31_MASK 0xff000000L
+#define AFMT_GENERIC_7__AFMT_GENERIC_BYTE31__SHIFT 0x00000018
+#define AFMT_GENERIC_HDR__AFMT_GENERIC_HB0_MASK 0x000000ffL
+#define AFMT_GENERIC_HDR__AFMT_GENERIC_HB0__SHIFT 0x00000000
+#define AFMT_GENERIC_HDR__AFMT_GENERIC_HB1_MASK 0x0000ff00L
+#define AFMT_GENERIC_HDR__AFMT_GENERIC_HB1__SHIFT 0x00000008
+#define AFMT_GENERIC_HDR__AFMT_GENERIC_HB2_MASK 0x00ff0000L
+#define AFMT_GENERIC_HDR__AFMT_GENERIC_HB2__SHIFT 0x00000010
+#define AFMT_GENERIC_HDR__AFMT_GENERIC_HB3_MASK 0xff000000L
+#define AFMT_GENERIC_HDR__AFMT_GENERIC_HB3__SHIFT 0x00000018
+#define AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_SOURCE_MASK 0x00000040L
+#define AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_SOURCE__SHIFT 0x00000006
+#define AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_UPDATE_MASK 0x00000080L
+#define AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_UPDATE__SHIFT 0x00000007
+#define AFMT_INFOFRAME_CONTROL0__AFMT_MPEG_INFO_UPDATE_MASK 0x00000400L
+#define AFMT_INFOFRAME_CONTROL0__AFMT_MPEG_INFO_UPDATE__SHIFT 0x0000000a
+#define AFMT_ISRC1_0__AFMT_ISRC_CONTINUE_MASK 0x00000040L
+#define AFMT_ISRC1_0__AFMT_ISRC_CONTINUE__SHIFT 0x00000006
+#define AFMT_ISRC1_0__AFMT_ISRC_STATUS_MASK 0x00000007L
+#define AFMT_ISRC1_0__AFMT_ISRC_STATUS__SHIFT 0x00000000
+#define AFMT_ISRC1_0__AFMT_ISRC_VALID_MASK 0x00000080L
+#define AFMT_ISRC1_0__AFMT_ISRC_VALID__SHIFT 0x00000007
+#define AFMT_ISRC1_1__AFMT_UPC_EAN_ISRC0_MASK 0x000000ffL
+#define AFMT_ISRC1_1__AFMT_UPC_EAN_ISRC0__SHIFT 0x00000000
+#define AFMT_ISRC1_1__AFMT_UPC_EAN_ISRC1_MASK 0x0000ff00L
+#define AFMT_ISRC1_1__AFMT_UPC_EAN_ISRC1__SHIFT 0x00000008
+#define AFMT_ISRC1_1__AFMT_UPC_EAN_ISRC2_MASK 0x00ff0000L
+#define AFMT_ISRC1_1__AFMT_UPC_EAN_ISRC2__SHIFT 0x00000010
+#define AFMT_ISRC1_1__AFMT_UPC_EAN_ISRC3_MASK 0xff000000L
+#define AFMT_ISRC1_1__AFMT_UPC_EAN_ISRC3__SHIFT 0x00000018
+#define AFMT_ISRC1_2__AFMT_UPC_EAN_ISRC4_MASK 0x000000ffL
+#define AFMT_ISRC1_2__AFMT_UPC_EAN_ISRC4__SHIFT 0x00000000
+#define AFMT_ISRC1_2__AFMT_UPC_EAN_ISRC5_MASK 0x0000ff00L
+#define AFMT_ISRC1_2__AFMT_UPC_EAN_ISRC5__SHIFT 0x00000008
+#define AFMT_ISRC1_2__AFMT_UPC_EAN_ISRC6_MASK 0x00ff0000L
+#define AFMT_ISRC1_2__AFMT_UPC_EAN_ISRC6__SHIFT 0x00000010
+#define AFMT_ISRC1_2__AFMT_UPC_EAN_ISRC7_MASK 0xff000000L
+#define AFMT_ISRC1_2__AFMT_UPC_EAN_ISRC7__SHIFT 0x00000018
+#define AFMT_ISRC1_3__AFMT_UPC_EAN_ISRC10_MASK 0x00ff0000L
+#define AFMT_ISRC1_3__AFMT_UPC_EAN_ISRC10__SHIFT 0x00000010
+#define AFMT_ISRC1_3__AFMT_UPC_EAN_ISRC11_MASK 0xff000000L
+#define AFMT_ISRC1_3__AFMT_UPC_EAN_ISRC11__SHIFT 0x00000018
+#define AFMT_ISRC1_3__AFMT_UPC_EAN_ISRC8_MASK 0x000000ffL
+#define AFMT_ISRC1_3__AFMT_UPC_EAN_ISRC8__SHIFT 0x00000000
+#define AFMT_ISRC1_3__AFMT_UPC_EAN_ISRC9_MASK 0x0000ff00L
+#define AFMT_ISRC1_3__AFMT_UPC_EAN_ISRC9__SHIFT 0x00000008
+#define AFMT_ISRC1_4__AFMT_UPC_EAN_ISRC12_MASK 0x000000ffL
+#define AFMT_ISRC1_4__AFMT_UPC_EAN_ISRC12__SHIFT 0x00000000
+#define AFMT_ISRC1_4__AFMT_UPC_EAN_ISRC13_MASK 0x0000ff00L
+#define AFMT_ISRC1_4__AFMT_UPC_EAN_ISRC13__SHIFT 0x00000008
+#define AFMT_ISRC1_4__AFMT_UPC_EAN_ISRC14_MASK 0x00ff0000L
+#define AFMT_ISRC1_4__AFMT_UPC_EAN_ISRC14__SHIFT 0x00000010
+#define AFMT_ISRC1_4__AFMT_UPC_EAN_ISRC15_MASK 0xff000000L
+#define AFMT_ISRC1_4__AFMT_UPC_EAN_ISRC15__SHIFT 0x00000018
+#define AFMT_ISRC2_0__AFMT_UPC_EAN_ISRC16_MASK 0x000000ffL
+#define AFMT_ISRC2_0__AFMT_UPC_EAN_ISRC16__SHIFT 0x00000000
+#define AFMT_ISRC2_0__AFMT_UPC_EAN_ISRC17_MASK 0x0000ff00L
+#define AFMT_ISRC2_0__AFMT_UPC_EAN_ISRC17__SHIFT 0x00000008
+#define AFMT_ISRC2_0__AFMT_UPC_EAN_ISRC18_MASK 0x00ff0000L
+#define AFMT_ISRC2_0__AFMT_UPC_EAN_ISRC18__SHIFT 0x00000010
+#define AFMT_ISRC2_0__AFMT_UPC_EAN_ISRC19_MASK 0xff000000L
+#define AFMT_ISRC2_0__AFMT_UPC_EAN_ISRC19__SHIFT 0x00000018
+#define AFMT_ISRC2_1__AFMT_UPC_EAN_ISRC20_MASK 0x000000ffL
+#define AFMT_ISRC2_1__AFMT_UPC_EAN_ISRC20__SHIFT 0x00000000
+#define AFMT_ISRC2_1__AFMT_UPC_EAN_ISRC21_MASK 0x0000ff00L
+#define AFMT_ISRC2_1__AFMT_UPC_EAN_ISRC21__SHIFT 0x00000008
+#define AFMT_ISRC2_1__AFMT_UPC_EAN_ISRC22_MASK 0x00ff0000L
+#define AFMT_ISRC2_1__AFMT_UPC_EAN_ISRC22__SHIFT 0x00000010
+#define AFMT_ISRC2_1__AFMT_UPC_EAN_ISRC23_MASK 0xff000000L
+#define AFMT_ISRC2_1__AFMT_UPC_EAN_ISRC23__SHIFT 0x00000018
+#define AFMT_ISRC2_2__AFMT_UPC_EAN_ISRC24_MASK 0x000000ffL
+#define AFMT_ISRC2_2__AFMT_UPC_EAN_ISRC24__SHIFT 0x00000000
+#define AFMT_ISRC2_2__AFMT_UPC_EAN_ISRC25_MASK 0x0000ff00L
+#define AFMT_ISRC2_2__AFMT_UPC_EAN_ISRC25__SHIFT 0x00000008
+#define AFMT_ISRC2_2__AFMT_UPC_EAN_ISRC26_MASK 0x00ff0000L
+#define AFMT_ISRC2_2__AFMT_UPC_EAN_ISRC26__SHIFT 0x00000010
+#define AFMT_ISRC2_2__AFMT_UPC_EAN_ISRC27_MASK 0xff000000L
+#define AFMT_ISRC2_2__AFMT_UPC_EAN_ISRC27__SHIFT 0x00000018
+#define AFMT_ISRC2_3__AFMT_UPC_EAN_ISRC28_MASK 0x000000ffL
+#define AFMT_ISRC2_3__AFMT_UPC_EAN_ISRC28__SHIFT 0x00000000
+#define AFMT_ISRC2_3__AFMT_UPC_EAN_ISRC29_MASK 0x0000ff00L
+#define AFMT_ISRC2_3__AFMT_UPC_EAN_ISRC29__SHIFT 0x00000008
+#define AFMT_ISRC2_3__AFMT_UPC_EAN_ISRC30_MASK 0x00ff0000L
+#define AFMT_ISRC2_3__AFMT_UPC_EAN_ISRC30__SHIFT 0x00000010
+#define AFMT_ISRC2_3__AFMT_UPC_EAN_ISRC31_MASK 0xff000000L
+#define AFMT_ISRC2_3__AFMT_UPC_EAN_ISRC31__SHIFT 0x00000018
+#define AFMT_MPEG_INFO0__AFMT_MPEG_INFO_CHECKSUM_MASK 0x000000ffL
+#define AFMT_MPEG_INFO0__AFMT_MPEG_INFO_CHECKSUM__SHIFT 0x00000000
+#define AFMT_MPEG_INFO0__AFMT_MPEG_INFO_MB0_MASK 0x0000ff00L
+#define AFMT_MPEG_INFO0__AFMT_MPEG_INFO_MB0__SHIFT 0x00000008
+#define AFMT_MPEG_INFO0__AFMT_MPEG_INFO_MB1_MASK 0x00ff0000L
+#define AFMT_MPEG_INFO0__AFMT_MPEG_INFO_MB1__SHIFT 0x00000010
+#define AFMT_MPEG_INFO0__AFMT_MPEG_INFO_MB2_MASK 0xff000000L
+#define AFMT_MPEG_INFO0__AFMT_MPEG_INFO_MB2__SHIFT 0x00000018
+#define AFMT_MPEG_INFO1__AFMT_MPEG_INFO_FR_MASK 0x00001000L
+#define AFMT_MPEG_INFO1__AFMT_MPEG_INFO_FR__SHIFT 0x0000000c
+#define AFMT_MPEG_INFO1__AFMT_MPEG_INFO_MB3_MASK 0x000000ffL
+#define AFMT_MPEG_INFO1__AFMT_MPEG_INFO_MB3__SHIFT 0x00000000
+#define AFMT_MPEG_INFO1__AFMT_MPEG_INFO_MF_MASK 0x00000300L
+#define AFMT_MPEG_INFO1__AFMT_MPEG_INFO_MF__SHIFT 0x00000008
+#define AFMT_RAMP_CONTROL0__AFMT_RAMP_DATA_SIGN_MASK 0x80000000L
+#define AFMT_RAMP_CONTROL0__AFMT_RAMP_DATA_SIGN__SHIFT 0x0000001f
+#define AFMT_RAMP_CONTROL0__AFMT_RAMP_MAX_COUNT_MASK 0x00ffffffL
+#define AFMT_RAMP_CONTROL0__AFMT_RAMP_MAX_COUNT__SHIFT 0x00000000
+#define AFMT_RAMP_CONTROL1__AFMT_AUDIO_TEST_CH_DISABLE_MASK 0xff000000L
+#define AFMT_RAMP_CONTROL1__AFMT_AUDIO_TEST_CH_DISABLE__SHIFT 0x00000018
+#define AFMT_RAMP_CONTROL1__AFMT_RAMP_MIN_COUNT_MASK 0x00ffffffL
+#define AFMT_RAMP_CONTROL1__AFMT_RAMP_MIN_COUNT__SHIFT 0x00000000
+#define AFMT_RAMP_CONTROL2__AFMT_RAMP_INC_COUNT_MASK 0x00ffffffL
+#define AFMT_RAMP_CONTROL2__AFMT_RAMP_INC_COUNT__SHIFT 0x00000000
+#define AFMT_RAMP_CONTROL3__AFMT_RAMP_DEC_COUNT_MASK 0x00ffffffL
+#define AFMT_RAMP_CONTROL3__AFMT_RAMP_DEC_COUNT__SHIFT 0x00000000
+#define AFMT_STATUS__AFMT_AUDIO_ENABLE_MASK 0x00000010L
+#define AFMT_STATUS__AFMT_AUDIO_ENABLE__SHIFT 0x00000004
+#define AFMT_STATUS__AFMT_AUDIO_FIFO_OVERFLOW_MASK 0x01000000L
+#define AFMT_STATUS__AFMT_AUDIO_FIFO_OVERFLOW__SHIFT 0x00000018
+#define AFMT_STATUS__AFMT_AZ_AUDIO_ENABLE_CHG_MASK 0x40000000L
+#define AFMT_STATUS__AFMT_AZ_AUDIO_ENABLE_CHG__SHIFT 0x0000001e
+#define AFMT_STATUS__AFMT_AZ_HBR_ENABLE_MASK 0x00000100L
+#define AFMT_STATUS__AFMT_AZ_HBR_ENABLE__SHIFT 0x00000008
+#define AFMT_VBI_PACKET_CONTROL__AFMT_GENERIC0_UPDATE_MASK 0x00000004L
+#define AFMT_VBI_PACKET_CONTROL__AFMT_GENERIC0_UPDATE__SHIFT 0x00000002
+#define AFMT_VBI_PACKET_CONTROL__AFMT_GENERIC2_UPDATE_MASK 0x00000008L
+#define AFMT_VBI_PACKET_CONTROL__AFMT_GENERIC2_UPDATE__SHIFT 0x00000003
+#define AFMT_VBI_PACKET_CONTROL__AFMT_GENERIC_INDEX_MASK 0xc0000000L
+#define AFMT_VBI_PACKET_CONTROL__AFMT_GENERIC_INDEX__SHIFT 0x0000001e
+#define ATTR00__ATTR_PAL_MASK 0x0000003fL
+#define ATTR00__ATTR_PAL__SHIFT 0x00000000
+#define ATTR01__ATTR_PAL_MASK 0x0000003fL
+#define ATTR01__ATTR_PAL__SHIFT 0x00000000
+#define ATTR02__ATTR_PAL_MASK 0x0000003fL
+#define ATTR02__ATTR_PAL__SHIFT 0x00000000
+#define ATTR03__ATTR_PAL_MASK 0x0000003fL
+#define ATTR03__ATTR_PAL__SHIFT 0x00000000
+#define ATTR04__ATTR_PAL_MASK 0x0000003fL
+#define ATTR04__ATTR_PAL__SHIFT 0x00000000
+#define ATTR05__ATTR_PAL_MASK 0x0000003fL
+#define ATTR05__ATTR_PAL__SHIFT 0x00000000
+#define ATTR06__ATTR_PAL_MASK 0x0000003fL
+#define ATTR06__ATTR_PAL__SHIFT 0x00000000
+#define ATTR07__ATTR_PAL_MASK 0x0000003fL
+#define ATTR07__ATTR_PAL__SHIFT 0x00000000
+#define ATTR08__ATTR_PAL_MASK 0x0000003fL
+#define ATTR08__ATTR_PAL__SHIFT 0x00000000
+#define ATTR09__ATTR_PAL_MASK 0x0000003fL
+#define ATTR09__ATTR_PAL__SHIFT 0x00000000
+#define ATTR0A__ATTR_PAL_MASK 0x0000003fL
+#define ATTR0A__ATTR_PAL__SHIFT 0x00000000
+#define ATTR0B__ATTR_PAL_MASK 0x0000003fL
+#define ATTR0B__ATTR_PAL__SHIFT 0x00000000
+#define ATTR0C__ATTR_PAL_MASK 0x0000003fL
+#define ATTR0C__ATTR_PAL__SHIFT 0x00000000
+#define ATTR0D__ATTR_PAL_MASK 0x0000003fL
+#define ATTR0D__ATTR_PAL__SHIFT 0x00000000
+#define ATTR0E__ATTR_PAL_MASK 0x0000003fL
+#define ATTR0E__ATTR_PAL__SHIFT 0x00000000
+#define ATTR0F__ATTR_PAL_MASK 0x0000003fL
+#define ATTR0F__ATTR_PAL__SHIFT 0x00000000
+#define ATTR10__ATTR_BLINK_EN_MASK 0x00000008L
+#define ATTR10__ATTR_BLINK_EN__SHIFT 0x00000003
+#define ATTR10__ATTR_CSEL_EN_MASK 0x00000080L
+#define ATTR10__ATTR_CSEL_EN__SHIFT 0x00000007
+#define ATTR10__ATTR_GRPH_MODE_MASK 0x00000001L
+#define ATTR10__ATTR_GRPH_MODE__SHIFT 0x00000000
+#define ATTR10__ATTR_LGRPH_EN_MASK 0x00000004L
+#define ATTR10__ATTR_LGRPH_EN__SHIFT 0x00000002
+#define ATTR10__ATTR_MONO_EN_MASK 0x00000002L
+#define ATTR10__ATTR_MONO_EN__SHIFT 0x00000001
+#define ATTR10__ATTR_PANTOPONLY_MASK 0x00000020L
+#define ATTR10__ATTR_PANTOPONLY__SHIFT 0x00000005
+#define ATTR10__ATTR_PCLKBY2_MASK 0x00000040L
+#define ATTR10__ATTR_PCLKBY2__SHIFT 0x00000006
+#define ATTR11__ATTR_OVSC_MASK 0x000000ffL
+#define ATTR11__ATTR_OVSC__SHIFT 0x00000000
+#define ATTR12__ATTR_MAP_EN_MASK 0x0000000fL
+#define ATTR12__ATTR_MAP_EN__SHIFT 0x00000000
+#define ATTR12__ATTR_VSMUX_MASK 0x00000030L
+#define ATTR12__ATTR_VSMUX__SHIFT 0x00000004
+#define ATTR13__ATTR_PPAN_MASK 0x0000000fL
+#define ATTR13__ATTR_PPAN__SHIFT 0x00000000
+#define ATTR14__ATTR_CSEL1_MASK 0x00000003L
+#define ATTR14__ATTR_CSEL1__SHIFT 0x00000000
+#define ATTR14__ATTR_CSEL2_MASK 0x0000000cL
+#define ATTR14__ATTR_CSEL2__SHIFT 0x00000002
+#define ATTRDR__ATTR_DATA_MASK 0x000000ffL
+#define ATTRDR__ATTR_DATA__SHIFT 0x00000000
+#define ATTRDW__ATTR_DATA_MASK 0x000000ffL
+#define ATTRDW__ATTR_DATA__SHIFT 0x00000000
+#define ATTRX__ATTR_IDX_MASK 0x0000001fL
+#define ATTRX__ATTR_IDX__SHIFT 0x00000000
+#define ATTRX__ATTR_PAL_RW_ENB_MASK 0x00000020L
+#define ATTRX__ATTR_PAL_RW_ENB__SHIFT 0x00000005
+#define AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR0__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR10__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR10__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR11__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR11__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR12__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR12__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR13__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR13__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR1__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR1__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR2__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR2__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR3__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR3__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR4__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR4__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR5__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR5__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR6__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR6__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR7__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR7__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR8__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR8__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AUDIO_DESCRIPTOR9__MAX_CHANNELS_MASK 0x00000007L
+#define AUDIO_DESCRIPTOR9__MAX_CHANNELS__SHIFT 0x00000000
+#define AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AUX_ARB_CONTROL__AUX_ARB_PRIORITY_MASK 0x00000003L
+#define AUX_ARB_CONTROL__AUX_ARB_PRIORITY__SHIFT 0x00000000
+#define AUX_ARB_CONTROL__AUX_DMCU_DONE_USING_AUX_REG_MASK 0x02000000L
+#define AUX_ARB_CONTROL__AUX_DMCU_DONE_USING_AUX_REG__SHIFT 0x00000019
+#define AUX_ARB_CONTROL__AUX_DMCU_PENDING_USE_AUX_REG_REQ_MASK 0x01000000L
+#define AUX_ARB_CONTROL__AUX_DMCU_PENDING_USE_AUX_REG_REQ__SHIFT 0x00000018
+#define AUX_ARB_CONTROL__AUX_DMCU_USE_AUX_REG_REQ_MASK 0x01000000L
+#define AUX_ARB_CONTROL__AUX_DMCU_USE_AUX_REG_REQ__SHIFT 0x00000018
+#define AUX_ARB_CONTROL__AUX_NO_QUEUED_LS_GO_MASK 0x00000400L
+#define AUX_ARB_CONTROL__AUX_NO_QUEUED_LS_GO__SHIFT 0x0000000a
+#define AUX_ARB_CONTROL__AUX_NO_QUEUED_SW_GO_MASK 0x00000100L
+#define AUX_ARB_CONTROL__AUX_NO_QUEUED_SW_GO__SHIFT 0x00000008
+#define AUX_ARB_CONTROL__AUX_REG_RW_CNTL_STATUS_MASK 0x0000000cL
+#define AUX_ARB_CONTROL__AUX_REG_RW_CNTL_STATUS__SHIFT 0x00000002
+#define AUX_ARB_CONTROL__AUX_SW_DONE_USING_AUX_REG_MASK 0x00020000L
+#define AUX_ARB_CONTROL__AUX_SW_DONE_USING_AUX_REG__SHIFT 0x00000011
+#define AUX_ARB_CONTROL__AUX_SW_PENDING_USE_AUX_REG_REQ_MASK 0x00010000L
+#define AUX_ARB_CONTROL__AUX_SW_PENDING_USE_AUX_REG_REQ__SHIFT 0x00000010
+#define AUX_ARB_CONTROL__AUX_SW_USE_AUX_REG_REQ_MASK 0x00010000L
+#define AUX_ARB_CONTROL__AUX_SW_USE_AUX_REG_REQ__SHIFT 0x00000010
+#define AUX_CONTROL__AUX_DEGLITCH_EN_MASK 0x20000000L
+#define AUX_CONTROL__AUX_DEGLITCH_EN__SHIFT 0x0000001d
+#define AUX_CONTROL__AUX_EN_MASK 0x00000001L
+#define AUX_CONTROL__AUX_EN__SHIFT 0x00000000
+#define AUX_CONTROL__AUX_HPD_SEL_MASK 0x00700000L
+#define AUX_CONTROL__AUX_HPD_SEL__SHIFT 0x00000014
+#define AUX_CONTROL__AUX_IGNORE_HPD_DISCON_MASK 0x00010000L
+#define AUX_CONTROL__AUX_IGNORE_HPD_DISCON__SHIFT 0x00000010
+#define AUX_CONTROL__AUX_IMPCAL_REQ_EN_MASK 0x01000000L
+#define AUX_CONTROL__AUX_IMPCAL_REQ_EN__SHIFT 0x00000018
+#define AUX_CONTROL__AUX_LS_READ_EN_MASK 0x00000100L
+#define AUX_CONTROL__AUX_LS_READ_EN__SHIFT 0x00000008
+#define AUX_CONTROL__AUX_LS_UPDATE_DISABLE_MASK 0x00001000L
+#define AUX_CONTROL__AUX_LS_UPDATE_DISABLE__SHIFT 0x0000000c
+#define AUX_CONTROL__AUX_MODE_DET_EN_MASK 0x00040000L
+#define AUX_CONTROL__AUX_MODE_DET_EN__SHIFT 0x00000012
+#define AUX_CONTROL__AUX_TEST_MODE_MASK 0x10000000L
+#define AUX_CONTROL__AUX_TEST_MODE__SHIFT 0x0000001c
+#define AUX_CONTROL__SPARE_0_MASK 0x40000000L
+#define AUX_CONTROL__SPARE_0__SHIFT 0x0000001e
+#define AUX_CONTROL__SPARE_1_MASK 0x80000000L
+#define AUX_CONTROL__SPARE_1__SHIFT 0x0000001f
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT_MASK 0x00020000L
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_PHASE_DETECT__SHIFT 0x00000011
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_START_MASK 0x00040000L
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_START__SHIFT 0x00000012
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_STOP_MASK 0x00080000L
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_ALLOW_BELOW_THRESHOLD_STOP__SHIFT 0x00000013
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_DETECTION_THRESHOLD_MASK 0x70000000L
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_DETECTION_THRESHOLD__SHIFT 0x0000001c
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_HALF_SYM_DETECT_LEN_MASK 0x00003000L
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_HALF_SYM_DETECT_LEN__SHIFT 0x0000000c
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_PHASE_DETECT_LEN_MASK 0x00300000L
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_PHASE_DETECT_LEN__SHIFT 0x00000014
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_RECEIVE_WINDOW_MASK 0x00000700L
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_RECEIVE_WINDOW__SHIFT 0x00000008
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_START_WINDOW_MASK 0x00000070L
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_START_WINDOW__SHIFT 0x00000004
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_TIMEOUT_LEN_MASK 0x07000000L
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_TIMEOUT_LEN__SHIFT 0x00000018
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_TRANSITION_FILTER_EN_MASK 0x00010000L
+#define AUX_DPHY_RX_CONTROL0__AUX_RX_TRANSITION_FILTER_EN__SHIFT 0x00000010
+#define AUX_DPHY_RX_CONTROL1__AUX_RX_PRECHARGE_SKIP_MASK 0x000000ffL
+#define AUX_DPHY_RX_CONTROL1__AUX_RX_PRECHARGE_SKIP__SHIFT 0x00000000
+#define AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_FRACT_MASK 0x001f0000L
+#define AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_FRACT__SHIFT 0x00000010
+#define AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD_MASK 0x3fe00000L
+#define AUX_DPHY_RX_STATUS__AUX_RX_HALF_SYM_PERIOD__SHIFT 0x00000015
+#define AUX_DPHY_RX_STATUS__AUX_RX_STATE_MASK 0x00000007L
+#define AUX_DPHY_RX_STATUS__AUX_RX_STATE__SHIFT 0x00000000
+#define AUX_DPHY_RX_STATUS__AUX_RX_SYNC_VALID_COUNT_MASK 0x00001f00L
+#define AUX_DPHY_RX_STATUS__AUX_RX_SYNC_VALID_COUNT__SHIFT 0x00000008
+#define AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN_MASK 0x00000007L
+#define AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_LEN__SHIFT 0x00000000
+#define AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_SYMBOLS_MASK 0x00003f00L
+#define AUX_DPHY_TX_CONTROL__AUX_TX_PRECHARGE_SYMBOLS__SHIFT 0x00000008
+#define AUX_DPHY_TX_REF_CONTROL__AUX_TX_RATE_MASK 0x00000030L
+#define AUX_DPHY_TX_REF_CONTROL__AUX_TX_RATE__SHIFT 0x00000004
+#define AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_DIV_MASK 0x01ff0000L
+#define AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_DIV__SHIFT 0x00000010
+#define AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_SEL_MASK 0x00000001L
+#define AUX_DPHY_TX_REF_CONTROL__AUX_TX_REF_SEL__SHIFT 0x00000000
+#define AUX_DPHY_TX_STATUS__AUX_TX_ACTIVE_MASK 0x00000001L
+#define AUX_DPHY_TX_STATUS__AUX_TX_ACTIVE__SHIFT 0x00000000
+#define AUX_DPHY_TX_STATUS__AUX_TX_HALF_SYM_PERIOD_MASK 0x01ff0000L
+#define AUX_DPHY_TX_STATUS__AUX_TX_HALF_SYM_PERIOD__SHIFT 0x00000010
+#define AUX_DPHY_TX_STATUS__AUX_TX_STATE_MASK 0x00000070L
+#define AUX_DPHY_TX_STATUS__AUX_TX_STATE__SHIFT 0x00000004
+#define AUX_GTC_SYNC_CONTROL__AUX_GTC_SYNC_EN_MASK 0x00000001L
+#define AUX_GTC_SYNC_CONTROL__AUX_GTC_SYNC_EN__SHIFT 0x00000000
+#define AUX_INTERRUPT_CONTROL__AUX_LS_DONE_ACK_MASK 0x00000020L
+#define AUX_INTERRUPT_CONTROL__AUX_LS_DONE_ACK__SHIFT 0x00000005
+#define AUX_INTERRUPT_CONTROL__AUX_LS_DONE_INT_MASK 0x00000010L
+#define AUX_INTERRUPT_CONTROL__AUX_LS_DONE_INT__SHIFT 0x00000004
+#define AUX_INTERRUPT_CONTROL__AUX_LS_DONE_MASK_MASK 0x00000040L
+#define AUX_INTERRUPT_CONTROL__AUX_LS_DONE_MASK__SHIFT 0x00000006
+#define AUX_INTERRUPT_CONTROL__AUX_SW_DONE_ACK_MASK 0x00000002L
+#define AUX_INTERRUPT_CONTROL__AUX_SW_DONE_ACK__SHIFT 0x00000001
+#define AUX_INTERRUPT_CONTROL__AUX_SW_DONE_INT_MASK 0x00000001L
+#define AUX_INTERRUPT_CONTROL__AUX_SW_DONE_INT__SHIFT 0x00000000
+#define AUX_INTERRUPT_CONTROL__AUX_SW_DONE_MASK_MASK 0x00000004L
+#define AUX_INTERRUPT_CONTROL__AUX_SW_DONE_MASK__SHIFT 0x00000002
+#define AUX_LS_DATA__AUX_LS_DATA_MASK 0x0000ff00L
+#define AUX_LS_DATA__AUX_LS_DATA__SHIFT 0x00000008
+#define AUX_LS_DATA__AUX_LS_INDEX_MASK 0x001f0000L
+#define AUX_LS_DATA__AUX_LS_INDEX__SHIFT 0x00000010
+#define AUX_LS_STATUS__AUX_LS_CP_IRQ_MASK 0x20000000L
+#define AUX_LS_STATUS__AUX_LS_CP_IRQ__SHIFT 0x0000001d
+#define AUX_LS_STATUS__AUX_LS_DONE_MASK 0x00000001L
+#define AUX_LS_STATUS__AUX_LS_DONE__SHIFT 0x00000000
+#define AUX_LS_STATUS__AUX_LS_HPD_DISCON_MASK 0x00000200L
+#define AUX_LS_STATUS__AUX_LS_HPD_DISCON__SHIFT 0x00000009
+#define AUX_LS_STATUS__AUX_LS_NON_AUX_MODE_MASK 0x00000800L
+#define AUX_LS_STATUS__AUX_LS_NON_AUX_MODE__SHIFT 0x0000000b
+#define AUX_LS_STATUS__AUX_LS_REPLY_BYTE_COUNT_MASK 0x1f000000L
+#define AUX_LS_STATUS__AUX_LS_REPLY_BYTE_COUNT__SHIFT 0x00000018
+#define AUX_LS_STATUS__AUX_LS_REQ_MASK 0x00000002L
+#define AUX_LS_STATUS__AUX_LS_REQ__SHIFT 0x00000001
+#define AUX_LS_STATUS__AUX_LS_RX_INVALID_START_MASK 0x00080000L
+#define AUX_LS_STATUS__AUX_LS_RX_INVALID_START__SHIFT 0x00000013
+#define AUX_LS_STATUS__AUX_LS_RX_INVALID_STOP_MASK 0x00004000L
+#define AUX_LS_STATUS__AUX_LS_RX_INVALID_STOP__SHIFT 0x0000000e
+#define AUX_LS_STATUS__AUX_LS_RX_MIN_COUNT_VIOL_MASK 0x00001000L
+#define AUX_LS_STATUS__AUX_LS_RX_MIN_COUNT_VIOL__SHIFT 0x0000000c
+#define AUX_LS_STATUS__AUX_LS_RX_OVERFLOW_MASK 0x00000100L
+#define AUX_LS_STATUS__AUX_LS_RX_OVERFLOW__SHIFT 0x00000008
+#define AUX_LS_STATUS__AUX_LS_RX_PARTIAL_BYTE_MASK 0x00000400L
+#define AUX_LS_STATUS__AUX_LS_RX_PARTIAL_BYTE__SHIFT 0x0000000a
+#define AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_H_MASK 0x00400000L
+#define AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_H__SHIFT 0x00000016
+#define AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_L_MASK 0x00800000L
+#define AUX_LS_STATUS__AUX_LS_RX_RECV_INVALID_L__SHIFT 0x00000017
+#define AUX_LS_STATUS__AUX_LS_RX_RECV_NO_DET_MASK 0x00100000L
+#define AUX_LS_STATUS__AUX_LS_RX_RECV_NO_DET__SHIFT 0x00000014
+#define AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_H_MASK 0x00040000L
+#define AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_H__SHIFT 0x00000012
+#define AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_L_MASK 0x00020000L
+#define AUX_LS_STATUS__AUX_LS_RX_SYNC_INVALID_L__SHIFT 0x00000011
+#define AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_MASK 0x00000080L
+#define AUX_LS_STATUS__AUX_LS_RX_TIMEOUT__SHIFT 0x00000007
+#define AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_STATE_MASK 0x00000070L
+#define AUX_LS_STATUS__AUX_LS_RX_TIMEOUT_STATE__SHIFT 0x00000004
+#define AUX_LS_STATUS__AUX_LS_UPDATED_ACK_MASK 0x80000000L
+#define AUX_LS_STATUS__AUX_LS_UPDATED_ACK__SHIFT 0x0000001f
+#define AUX_LS_STATUS__AUX_LS_UPDATED_MASK 0x40000000L
+#define AUX_LS_STATUS__AUX_LS_UPDATED__SHIFT 0x0000001e
+#define AUXN_IMPCAL__AUXN_CALOUT_ERROR_AK_MASK 0x00000400L
+#define AUXN_IMPCAL__AUXN_CALOUT_ERROR_AK__SHIFT 0x0000000a
+#define AUXN_IMPCAL__AUXN_CALOUT_ERROR_MASK 0x00000200L
+#define AUXN_IMPCAL__AUXN_CALOUT_ERROR__SHIFT 0x00000009
+#define AUXN_IMPCAL__AUXN_IMPCAL_CALOUT_MASK 0x00000100L
+#define AUXN_IMPCAL__AUXN_IMPCAL_CALOUT__SHIFT 0x00000008
+#define AUXN_IMPCAL__AUXN_IMPCAL_ENABLE_MASK 0x00000001L
+#define AUXN_IMPCAL__AUXN_IMPCAL_ENABLE__SHIFT 0x00000000
+#define AUXN_IMPCAL__AUXN_IMPCAL_OVERRIDE_ENABLE_MASK 0x10000000L
+#define AUXN_IMPCAL__AUXN_IMPCAL_OVERRIDE_ENABLE__SHIFT 0x0000001c
+#define AUXN_IMPCAL__AUXN_IMPCAL_OVERRIDE_MASK 0x0f000000L
+#define AUXN_IMPCAL__AUXN_IMPCAL_OVERRIDE__SHIFT 0x00000018
+#define AUXN_IMPCAL__AUXN_IMPCAL_STEP_DELAY_MASK 0x00f00000L
+#define AUXN_IMPCAL__AUXN_IMPCAL_STEP_DELAY__SHIFT 0x00000014
+#define AUXN_IMPCAL__AUXN_IMPCAL_VALUE_MASK 0x000f0000L
+#define AUXN_IMPCAL__AUXN_IMPCAL_VALUE__SHIFT 0x00000010
+#define AUXP_IMPCAL__AUXP_CALOUT_ERROR_AK_MASK 0x00000400L
+#define AUXP_IMPCAL__AUXP_CALOUT_ERROR_AK__SHIFT 0x0000000a
+#define AUXP_IMPCAL__AUXP_CALOUT_ERROR_MASK 0x00000200L
+#define AUXP_IMPCAL__AUXP_CALOUT_ERROR__SHIFT 0x00000009
+#define AUXP_IMPCAL__AUXP_IMPCAL_CALOUT_MASK 0x00000100L
+#define AUXP_IMPCAL__AUXP_IMPCAL_CALOUT__SHIFT 0x00000008
+#define AUXP_IMPCAL__AUXP_IMPCAL_ENABLE_MASK 0x00000001L
+#define AUXP_IMPCAL__AUXP_IMPCAL_ENABLE__SHIFT 0x00000000
+#define AUXP_IMPCAL__AUXP_IMPCAL_OVERRIDE_ENABLE_MASK 0x10000000L
+#define AUXP_IMPCAL__AUXP_IMPCAL_OVERRIDE_ENABLE__SHIFT 0x0000001c
+#define AUXP_IMPCAL__AUXP_IMPCAL_OVERRIDE_MASK 0x0f000000L
+#define AUXP_IMPCAL__AUXP_IMPCAL_OVERRIDE__SHIFT 0x00000018
+#define AUXP_IMPCAL__AUXP_IMPCAL_STEP_DELAY_MASK 0x00f00000L
+#define AUXP_IMPCAL__AUXP_IMPCAL_STEP_DELAY__SHIFT 0x00000014
+#define AUXP_IMPCAL__AUXP_IMPCAL_VALUE_MASK 0x000f0000L
+#define AUXP_IMPCAL__AUXP_IMPCAL_VALUE__SHIFT 0x00000010
+#define AUX_SW_CONTROL__AUX_LS_READ_TRIG_MASK 0x00000004L
+#define AUX_SW_CONTROL__AUX_LS_READ_TRIG__SHIFT 0x00000002
+#define AUX_SW_CONTROL__AUX_SW_GO_MASK 0x00000001L
+#define AUX_SW_CONTROL__AUX_SW_GO__SHIFT 0x00000000
+#define AUX_SW_CONTROL__AUX_SW_START_DELAY_MASK 0x000000f0L
+#define AUX_SW_CONTROL__AUX_SW_START_DELAY__SHIFT 0x00000004
+#define AUX_SW_CONTROL__AUX_SW_WR_BYTES_MASK 0x001f0000L
+#define AUX_SW_CONTROL__AUX_SW_WR_BYTES__SHIFT 0x00000010
+#define AUX_SW_DATA__AUX_SW_AUTOINCREMENT_DISABLE_MASK 0x80000000L
+#define AUX_SW_DATA__AUX_SW_AUTOINCREMENT_DISABLE__SHIFT 0x0000001f
+#define AUX_SW_DATA__AUX_SW_DATA_MASK 0x0000ff00L
+#define AUX_SW_DATA__AUX_SW_DATA_RW_MASK 0x00000001L
+#define AUX_SW_DATA__AUX_SW_DATA_RW__SHIFT 0x00000000
+#define AUX_SW_DATA__AUX_SW_DATA__SHIFT 0x00000008
+#define AUX_SW_DATA__AUX_SW_INDEX_MASK 0x001f0000L
+#define AUX_SW_DATA__AUX_SW_INDEX__SHIFT 0x00000010
+#define AUX_SW_STATUS__AUX_ARB_STATUS_MASK 0xc0000000L
+#define AUX_SW_STATUS__AUX_ARB_STATUS__SHIFT 0x0000001e
+#define AUX_SW_STATUS__AUX_SW_DONE_MASK 0x00000001L
+#define AUX_SW_STATUS__AUX_SW_DONE__SHIFT 0x00000000
+#define AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK 0x00000200L
+#define AUX_SW_STATUS__AUX_SW_HPD_DISCON__SHIFT 0x00000009
+#define AUX_SW_STATUS__AUX_SW_NON_AUX_MODE_MASK 0x00000800L
+#define AUX_SW_STATUS__AUX_SW_NON_AUX_MODE__SHIFT 0x0000000b
+#define AUX_SW_STATUS__AUX_SW_REPLY_BYTE_COUNT_MASK 0x1f000000L
+#define AUX_SW_STATUS__AUX_SW_REPLY_BYTE_COUNT__SHIFT 0x00000018
+#define AUX_SW_STATUS__AUX_SW_REQ_MASK 0x00000002L
+#define AUX_SW_STATUS__AUX_SW_REQ__SHIFT 0x00000001
+#define AUX_SW_STATUS__AUX_SW_RX_INVALID_START_MASK 0x00080000L
+#define AUX_SW_STATUS__AUX_SW_RX_INVALID_START__SHIFT 0x00000013
+#define AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP_MASK 0x00004000L
+#define AUX_SW_STATUS__AUX_SW_RX_INVALID_STOP__SHIFT 0x0000000e
+#define AUX_SW_STATUS__AUX_SW_RX_MIN_COUNT_VIOL_MASK 0x00001000L
+#define AUX_SW_STATUS__AUX_SW_RX_MIN_COUNT_VIOL__SHIFT 0x0000000c
+#define AUX_SW_STATUS__AUX_SW_RX_OVERFLOW_MASK 0x00000100L
+#define AUX_SW_STATUS__AUX_SW_RX_OVERFLOW__SHIFT 0x00000008
+#define AUX_SW_STATUS__AUX_SW_RX_PARTIAL_BYTE_MASK 0x00000400L
+#define AUX_SW_STATUS__AUX_SW_RX_PARTIAL_BYTE__SHIFT 0x0000000a
+#define AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H_MASK 0x00400000L
+#define AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_H__SHIFT 0x00000016
+#define AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L_MASK 0x00800000L
+#define AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L__SHIFT 0x00000017
+#define AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET_MASK 0x00100000L
+#define AUX_SW_STATUS__AUX_SW_RX_RECV_NO_DET__SHIFT 0x00000014
+#define AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_H_MASK 0x00040000L
+#define AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_H__SHIFT 0x00000012
+#define AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_L_MASK 0x00020000L
+#define AUX_SW_STATUS__AUX_SW_RX_SYNC_INVALID_L__SHIFT 0x00000011
+#define AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_MASK 0x00000080L
+#define AUX_SW_STATUS__AUX_SW_RX_TIMEOUT__SHIFT 0x00000007
+#define AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE_MASK 0x00000070L
+#define AUX_SW_STATUS__AUX_SW_RX_TIMEOUT_STATE__SHIFT 0x00000004
+#define AZALIA_APPLICATION_POSITION_IN_CYCLIC_BUFFER__APPLICATION_POSITION_IN_CYCLIC_BUFFER_MASK 0xffffffffL
+#define AZALIA_APPLICATION_POSITION_IN_CYCLIC_BUFFER__APPLICATION_POSITION_IN_CYCLIC_BUFFER__SHIFT 0x00000000
+#define AZALIA_AUDIO_DTO__AZALIA_AUDIO_DTO_MODULE_MASK 0xffff0000L
+#define AZALIA_AUDIO_DTO__AZALIA_AUDIO_DTO_MODULE__SHIFT 0x00000010
+#define AZALIA_AUDIO_DTO__AZALIA_AUDIO_DTO_PHASE_MASK 0x0000ffffL
+#define AZALIA_AUDIO_DTO__AZALIA_AUDIO_DTO_PHASE__SHIFT 0x00000000
+#define AZALIA_AUDIO_DTO_CONTROL__AZALIA_AUDIO_FORCE_DTO_MASK 0x00000300L
+#define AZALIA_AUDIO_DTO_CONTROL__AZALIA_AUDIO_FORCE_DTO__SHIFT 0x00000008
+#define AZALIA_BDL_DMA_CONTROL__BDL_DMA_ISOCHRONOUS_MASK 0x00000030L
+#define AZALIA_BDL_DMA_CONTROL__BDL_DMA_ISOCHRONOUS__SHIFT 0x00000004
+#define AZALIA_BDL_DMA_CONTROL__BDL_DMA_NON_SNOOP_MASK 0x00000003L
+#define AZALIA_BDL_DMA_CONTROL__BDL_DMA_NON_SNOOP__SHIFT 0x00000000
+#define AZALIA_CONTROLLER_DEBUG__CONTROLLER_DEBUG_MASK 0xffffffffL
+#define AZALIA_CONTROLLER_DEBUG__CONTROLLER_DEBUG__SHIFT 0x00000000
+#define AZALIA_CORB_DMA_CONTROL__CORB_DMA_ISOCHRONOUS_MASK 0x00000010L
+#define AZALIA_CORB_DMA_CONTROL__CORB_DMA_ISOCHRONOUS__SHIFT 0x00000004
+#define AZALIA_CORB_DMA_CONTROL__CORB_DMA_NON_SNOOP_MASK 0x00000001L
+#define AZALIA_CORB_DMA_CONTROL__CORB_DMA_NON_SNOOP__SHIFT 0x00000000
+#define AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT_MASK 0xffffffffL
+#define AZALIA_CUMULATIVE_LATENCY_COUNT__AZALIA_CUMULATIVE_LATENCY_COUNT__SHIFT 0x00000000
+#define AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT_MASK 0xffffffffL
+#define AZALIA_CUMULATIVE_REQUEST_COUNT__AZALIA_CUMULATIVE_REQUEST_COUNT__SHIFT 0x00000000
+#define AZALIA_CYCLIC_BUFFER_SYNC__CYCLIC_BUFFER_SYNC_ENABLE_MASK 0x00000001L
+#define AZALIA_CYCLIC_BUFFER_SYNC__CYCLIC_BUFFER_SYNC_ENABLE__SHIFT 0x00000000
+#define AZALIA_DATA_DMA_CONTROL__AZALIA_IOC_GENERATION_METHOD_MASK 0x00010000L
+#define AZALIA_DATA_DMA_CONTROL__AZALIA_IOC_GENERATION_METHOD__SHIFT 0x00000010
+#define AZALIA_DATA_DMA_CONTROL__AZALIA_UNDERFLOW_CONTROL_MASK 0x00020000L
+#define AZALIA_DATA_DMA_CONTROL__AZALIA_UNDERFLOW_CONTROL__SHIFT 0x00000011
+#define AZALIA_DATA_DMA_CONTROL__DATA_DMA_ISOCHRONOUS_MASK 0x00000030L
+#define AZALIA_DATA_DMA_CONTROL__DATA_DMA_ISOCHRONOUS__SHIFT 0x00000004
+#define AZALIA_DATA_DMA_CONTROL__DATA_DMA_NON_SNOOP_MASK 0x00000003L
+#define AZALIA_DATA_DMA_CONTROL__DATA_DMA_NON_SNOOP__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL__COMPRESSED_CHANNEL_COUNT_MASK 0x00000070L
+#define AZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL__COMPRESSED_CHANNEL_COUNT__SHIFT 0x00000004
+#define AZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL__HBR_CHANNEL_COUNT_MASK 0x00000007L
+#define AZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL__HBR_CHANNEL_COUNT__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK 0x0000000fL
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK 0x000000f0L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT 0x00000004
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK 0x00000070L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT 0x00000004
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK 0x0000000fL
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK 0x00000700L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK 0x00003800L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT 0x0000000b
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK 0x00004000L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT 0x0000000e
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK 0x00008000L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT 0x0000000f
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK 0x00007f00L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK 0x00000010L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT 0x00000004
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK 0x00000001L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK 0x00800000L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT 0x00000017
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK 0x00000080L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT 0x00000007
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK 0x00000020L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT 0x00000005
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK 0x00000008L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT 0x00000003
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK 0x00000040L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT 0x00000006
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK 0x00000004L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT 0x00000002
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK 0x00000002L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT 0x00000001
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING__PRESENTATION_TIME_EMBEDDING_ENABLE_MASK 0x00000001L
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING__PRESENTATION_TIME_EMBEDDING_ENABLE__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE_MASK 0x000000ffL
+#define AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK 0x00000008L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT 0x00000003
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK 0x00000001L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK 0x000f0000L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK 0x00000100L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK 0x00000200L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT 0x00000009
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK 0x00000010L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT 0x00000004
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK 0x00000002L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT 0x00000001
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK 0x00000800L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT 0x0000000b
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK 0x00000004L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT 0x00000002
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK 0x00000400L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT 0x0000000a
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK 0x00000040L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT 0x00000006
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK 0x00000020L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT 0x00000005
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK 0x00f00000L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT 0x00000014
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK 0x00000080L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT 0x00000007
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK 0xffffffffL
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK 0x001f0000L
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK 0x00000fffL
+#define AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_CONVERTER_PIN_DEBUG__AZALIA_DEBUG__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY_MASK 0x00700000L
+#define AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY__SHIFT 0x00000014
+#define AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL_MASK 0x00000003L
+#define AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_DEBUG__CODEC_DEBUG_MASK 0xffffffffL
+#define AZALIA_F0_CODEC_DEBUG__CODEC_DEBUG__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA_MASK 0xffffffffL
+#define AZALIA_F0_CODEC_ENDPOINT_DATA__AZALIA_ENDPOINT_REG_DATA__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX_MASK 0x000000ffL
+#define AZALIA_F0_CODEC_ENDPOINT_INDEX__AZALIA_ENDPOINT_REG_INDEX__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION__CONVERTER_SYNCHRONIZATION_MASK 0x0000003fL
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION__CONVERTER_SYNCHRONIZATION__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__CLKSTOPOK_MASK 0x00000200L
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__CLKSTOPOK__SHIFT 0x00000009
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_ACT_MASK 0x000000f0L
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_ACT__SHIFT 0x00000004
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SET_MASK 0x0000000fL
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SET__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SETTINGS_RESET_MASK 0x00000400L
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SETTINGS_RESET__SHIFT 0x0000000a
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESET__CODEC_RESET_MASK 0x00000001L
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESET__CODEC_RESET__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE0_MASK 0x000000ffL
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE0__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE1_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE1__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE3_MASK 0xff000000L
+#define AZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE3__SHIFT 0x00000018
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_GROUP_TYPE__AZALIA_CODEC_FUNCTION_PARAMETER_GROUP_TYPE_MASK 0xffffffffL
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_GROUP_TYPE__AZALIA_CODEC_FUNCTION_PARAMETER_GROUP_TYPE__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES__AZALIA_CODEC_FUNCTION_PARAMETER_POWER_STATES_MASK 0x3fffffffL
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES__AZALIA_CODEC_FUNCTION_PARAMETER_POWER_STATES__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES__CLKSTOP_MASK 0x40000000L
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES__CLKSTOP__SHIFT 0x0000001e
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES__EPSS_MASK 0x80000000L
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES__EPSS__SHIFT 0x0000001f
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS__AZALIA_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS_MASK 0xffffffffL
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS__AZALIA_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK 0x001f0000L
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK 0x00000fffL
+#define AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO_MASK 0xffffffffL
+#define AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__CHANNEL_ALLOCATION__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT_MASK 0x80000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DOWN_MIX_INHIBIT__SHIFT 0x0000001f
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION_MASK 0x00020000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__DP_CONNECTION__SHIFT 0x00000011
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO_MASK 0x00fc0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__EXTRA_CONNECTION_INFO__SHIFT 0x00000012
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION_MASK 0x00010000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__HDMI_CONNECTION__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT_MASK 0x78000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__LEVEL_SHIFT__SHIFT 0x0000001b
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION_MASK 0x0000007fL
+#define AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER__SPEAKER_ALLOCATION__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED_MASK 0x80000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL__AUDIO_ENABLED__SHIFT 0x0000001f
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID_MASK 0x000000f0L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_CHANNEL_ID__SHIFT 0x00000004
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE_MASK 0x00000001L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_ENABLE__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE_MASK 0x00000002L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL1_MUTE__SHIFT 0x00000001
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID_MASK 0x0000f000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_CHANNEL_ID__SHIFT 0x0000000c
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE_MASK 0x00000100L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_ENABLE__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE_MASK 0x00000200L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL3_MUTE__SHIFT 0x00000009
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID_MASK 0x00f00000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_CHANNEL_ID__SHIFT 0x00000014
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE_MASK 0x00010000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_ENABLE__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE_MASK 0x00020000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL5_MUTE__SHIFT 0x00000011
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID_MASK 0xf0000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_CHANNEL_ID__SHIFT 0x0000001c
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE_MASK 0x01000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_ENABLE__SHIFT 0x00000018
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE_MASK 0x02000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2__MULTICHANNEL7_MUTE__SHIFT 0x00000019
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID_MASK 0x000000f0L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_CHANNEL_ID__SHIFT 0x00000004
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE_MASK 0x00000001L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_ENABLE__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE_MASK 0x00000002L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL01_MUTE__SHIFT 0x00000001
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID_MASK 0x0000f000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_CHANNEL_ID__SHIFT 0x0000000c
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE_MASK 0x00000100L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_ENABLE__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE_MASK 0x00000200L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL23_MUTE__SHIFT 0x00000009
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID_MASK 0x00f00000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_CHANNEL_ID__SHIFT 0x00000014
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE_MASK 0x00010000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_ENABLE__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE_MASK 0x00020000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL45_MUTE__SHIFT 0x00000011
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID_MASK 0xf0000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_CHANNEL_ID__SHIFT 0x0000001c
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE_MASK 0x01000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_ENABLE__SHIFT 0x00000018
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE_MASK 0x02000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE__MULTICHANNEL67_MUTE__SHIFT 0x00000019
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE_MASK 0x00000001L
+#define AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK 0x0000f000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT 0x0000000c
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK 0x000f0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK 0x000000f0L
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT 0x00000004
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK 0x00f00000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT 0x00000014
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK 0x3f000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT 0x00000018
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK 0x00000f00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK 0xc0000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT 0x0000001e
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK 0x0000000fL
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE_MASK 0x00000001L
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_CAPABLE__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE_MASK 0x00000010L
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR__HBR_ENABLE__SHIFT 0x00000004
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__AUDIO_LIPSYNC__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC_MASK 0x000000ffL
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC__VIDEO_LIPSYNC__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE_MASK 0x7fffffffL
+#define AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID_MASK 0x0000ffffL
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__MANUFACTURER_ID__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID_MASK 0xffff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0__PRODUCT_ID__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN_MASK 0x000000ffL
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1__SINK_DESCRIPTION_LEN__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0_MASK 0xffffffffL
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2__PORT_ID0__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1_MASK 0xffffffffL
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3__PORT_ID1__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0_MASK 0x000000ffL
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION0__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION1__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION2__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3_MASK 0xff000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4__DESCRIPTION3__SHIFT 0x00000018
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4_MASK 0x000000ffL
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION4__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION5__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION6__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7_MASK 0xff000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5__DESCRIPTION7__SHIFT 0x00000018
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION10__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11_MASK 0xff000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION11__SHIFT 0x00000018
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8_MASK 0x000000ffL
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION8__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6__DESCRIPTION9__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12_MASK 0x000000ffL
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION12__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION13__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14_MASK 0x00ff0000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION14__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15_MASK 0xff000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7__DESCRIPTION15__SHIFT 0x00000018
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16_MASK 0x000000ffL
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION16__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8__DESCRIPTION17__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK 0x00000080L
+#define AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT 0x00000007
+#define AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE_MASK 0x10000000L
+#define AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_FORCE__SHIFT 0x0000001c
+#define AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD_MASK 0x03ffffffL
+#define AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE__UNSOLICITED_RESPONSE_PAYLOAD__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK 0x0000003fL
+#define AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE_MASK 0x00000040L
+#define AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE__SHIFT 0x00000006
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK 0x00000008L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT 0x00000003
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK 0x00000001L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK 0x000f0000L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK 0x00000100L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK 0x00000200L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT 0x00000009
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK 0x00000002L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT 0x00000001
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK 0x00000800L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT 0x0000000b
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK 0x00000004L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT 0x00000002
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK 0x00000400L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT 0x0000000a
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK 0x00000040L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT 0x00000006
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK 0x00000020L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT 0x00000005
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK 0x00f00000L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT 0x00000014
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK 0x00000080L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT 0x00000007
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK 0x00000040L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT 0x00000006
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP_MASK 0x01000000L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__DP__SHIFT 0x00000018
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK 0x00010000L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT 0x00000010
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI_MASK 0x00000080L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT 0x00000007
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK 0x00000008L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT 0x00000003
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK 0x00000001L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK 0x00000020L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT 0x00000005
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK 0x00000004L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT 0x00000002
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK 0x00000010L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT 0x00000004
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK 0x00000002L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT 0x00000001
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK 0x0000ff00L
+#define AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT 0x00000008
+#define AZALIA_F0_CODEC_RESYNC_FIFO_CONTROL__RESYNC_FIFO_STARTUP_KEEPOUT_WINDOW_MASK 0x0000003fL
+#define AZALIA_F0_CODEC_RESYNC_FIFO_CONTROL__RESYNC_FIFO_STARTUP_KEEPOUT_WINDOW__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_ROOT_PARAMETER_REVISION_ID__AZALIA_CODEC_ROOT_PARAMETER_REVISION_ID_MASK 0xffffffffL
+#define AZALIA_F0_CODEC_ROOT_PARAMETER_REVISION_ID__AZALIA_CODEC_ROOT_PARAMETER_REVISION_ID__SHIFT 0x00000000
+#define AZALIA_F0_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID__AZALIA_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID_MASK 0xffffffffL
+#define AZALIA_F0_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID__AZALIA_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID__SHIFT 0x00000000
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE_MASK 0x00000003L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE__SHIFT 0x00000000
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER_MASK 0x0000003cL
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER__SHIFT 0x00000002
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_MASK 0x00000003L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN_MASK 0x00000004L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN__SHIFT 0x00000002
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY__SHIFT 0x00000000
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_MASK 0x00000078L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN_MASK 0x00000080L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN__SHIFT 0x00000007
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH__SHIFT 0x00000003
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_MASK 0x0000003fL
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN_MASK 0x00000040L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT 0x00000006
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY__SHIFT 0x00000000
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK 0x0000000fL
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN_MASK 0x00000010L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT 0x00000004
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT 0x00000000
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_MASK 0x00000060L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A__SHIFT 0x00000005
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID_MASK 0x00000080L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID__SHIFT 0x00000007
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO_MASK 0x00000010L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO__SHIFT 0x00000004
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF_MASK 0x0000000fL
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF__SHIFT 0x00000000
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L_MASK 0x0000000fL
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L__SHIFT 0x00000000
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R_MASK 0x000000f0L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R__SHIFT 0x00000004
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2_MASK 0x0000000fL
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2__SHIFT 0x00000000
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3_MASK 0x000000f0L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3__SHIFT 0x00000004
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4_MASK 0x0000000fL
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4__SHIFT 0x00000000
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5_MASK 0x000000f0L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5__SHIFT 0x00000004
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6_MASK 0x0000000fL
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6__SHIFT 0x00000000
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7_MASK 0x000000f0L
+#define AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID_MASK 0x0000000fL
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__CHANNEL_ID__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID__STREAM_ID__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE_MASK 0x00000070L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__BITS_PER_SAMPLE__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS_MASK 0x0000000fL
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__NUMBER_OF_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR_MASK 0x00000700L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_DIVISOR__SHIFT 0x00000008
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE_MASK 0x00003800L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_MULTIPLE__SHIFT 0x0000000b
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE_MASK 0x00004000L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__SAMPLE_BASE_RATE__SHIFT 0x0000000e
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_MASK 0x00008000L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_R_MASK 0x00008000L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE_R__SHIFT 0x0000000f
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT__STREAM_TYPE__SHIFT 0x0000000f
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_2__CC_MASK 0x0000007fL
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_2__CC__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_3__KEEPALIVE_MASK 0x00000080L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_3__KEEPALIVE__SHIFT 0x00000007
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC_MASK 0x00007f00L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__CC__SHIFT 0x00000008
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY_MASK 0x00000010L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__COPY__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN_MASK 0x00000001L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__DIGEN__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE_MASK 0x00800000L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__KEEPALIVE__SHIFT 0x00000017
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L_MASK 0x00000080L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__L__SHIFT 0x00000007
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO_MASK 0x00000020L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__NON_AUDIO__SHIFT 0x00000005
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE_MASK 0x00000008L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRE__SHIFT 0x00000003
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO_MASK 0x00000040L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__PRO__SHIFT 0x00000006
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG_MASK 0x00000004L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__VCFG__SHIFT 0x00000002
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V_MASK 0x00000002L
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER__V__SHIFT 0x00000001
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE_MASK 0x000000ffL
+#define AZALIA_F2_CODEC_CONVERTER_CONTROL_RAMP_RATE__RAMP_RATE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK 0x00000008L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT 0x00000003
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK 0x00000001L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK 0x000f0000L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT 0x00000010
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK 0x00000100L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT 0x00000008
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK 0x00000200L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT 0x00000009
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE_MASK 0x00000010L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__FORMAT_OVERRIDE__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK 0x00000002L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT 0x00000001
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK 0x00000800L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT 0x0000000b
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK 0x00000004L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT 0x00000002
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK 0x00000400L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT 0x0000000a
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK 0x00000040L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT 0x00000006
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK 0x00000020L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT 0x00000005
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK 0x00f00000L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT 0x00000014
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK 0x00000080L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT 0x00000007
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS__STREAM_FORMATS__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK 0x001f0000L
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT 0x00000010
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK 0x00000fffL
+#define AZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY_MASK 0x00700000L
+#define AZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CAPABILITY__SHIFT 0x00000014
+#define AZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL_MASK 0x00000003L
+#define AZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL__STRIPE_CONTROL__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION__CONVERTER_SYNCHRONIZATION_MASK 0x0000003fL
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION__CONVERTER_SYNCHRONIZATION__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__CLKSTOPOK_MASK 0x00000200L
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__CLKSTOPOK__SHIFT 0x00000009
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_ACT_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_ACT__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SET_MASK 0x0000000fL
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SET__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SETTINGS_RESET_MASK 0x00000400L
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE__POWER_STATE_SETTINGS_RESET__SHIFT 0x0000000a
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESET__CODEC_RESET_MASK 0x00000001L
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESET__CODEC_RESET__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_2__SUBSYSTEM_ID_BYTE1_MASK 0x000000ffL
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_2__SUBSYSTEM_ID_BYTE1__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_3__SUBSYSTEM_ID_BYTE2_MASK 0x000000ffL
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_3__SUBSYSTEM_ID_BYTE2__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_4__SUBSYSTEM_ID_BYTE3_MASK 0x000000ffL
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_4__SUBSYSTEM_ID_BYTE3__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE0_MASK 0x000000ffL
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE0__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE1_MASK 0x0000ff00L
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE1__SHIFT 0x00000008
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE2_MASK 0x00ff0000L
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE2__SHIFT 0x00000010
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE3_MASK 0xff000000L
+#define AZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID__SUBSYSTEM_ID_BYTE3__SHIFT 0x00000018
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_GROUP_TYPE__AZALIA_CODEC_FUNCTION_PARAMETER_GROUP_TYPE_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_GROUP_TYPE__AZALIA_CODEC_FUNCTION_PARAMETER_GROUP_TYPE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES__AZALIA_CODEC_FUNCTION_PARAMETER_POWER_STATES_MASK 0x3fffffffL
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES__AZALIA_CODEC_FUNCTION_PARAMETER_POWER_STATES__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES__CLKSTOP_MASK 0x40000000L
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES__CLKSTOP__SHIFT 0x0000001e
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES__EPSS_MASK 0x80000000L
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES__EPSS__SHIFT 0x0000001f
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS__AZALIA_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS__AZALIA_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT__AZALIA_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT__AZALIA_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES_MASK 0x001f0000L
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_BIT_CAPABILITIES__SHIFT 0x00000010
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES_MASK 0x00000fffL
+#define AZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES__AUDIO_RATE_CAPABILITIES__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_PIN_ASSOCIATION_INFO__ASSOCIATION_INFO__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR_DATA__DESCRIPTOR_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR_DATA__DESCRIPTOR__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__DESCRIPTOR_BYTE_2_MASK 0x00ff0000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__DESCRIPTOR_BYTE_2__SHIFT 0x00000010
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__FORMAT_CODE_MASK 0x00000078L
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__FORMAT_CODE__SHIFT 0x00000003
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__MAX_CHANNELS_MASK 0x00000007L
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__MAX_CHANNELS__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__SUPPORTED_FREQUENCIES_MASK 0x0000ff00L
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__SUPPORTED_FREQUENCIES__SHIFT 0x00000008
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__SUPPORTED_FREQUENCIES_STEREO_MASK 0xff000000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR__SUPPORTED_FREQUENCIES_STEREO__SHIFT 0x00000018
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_DATA__SINK_DATA_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_DATA__SINK_DATA__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_INDEX__SINK_INFO_INDEX_MASK 0x000000ffL
+#define AZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_INDEX__SINK_INFO_INDEX__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION_MASK 0x000000ffL
+#define AZALIA_F2_CODEC_PIN_CONTROL_CHANNEL_ALLOCATION__CHANNEL_ALLOCATION__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO__DOWN_MIX_INHIBIT_MASK 0x00000080L
+#define AZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO__DOWN_MIX_INHIBIT__SHIFT 0x00000007
+#define AZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO__LEVEL_SHIFT_MASK 0x00000078L
+#define AZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO__LEVEL_SHIFT__SHIFT 0x00000003
+#define AZALIA_F2_CODEC_PIN_CONTROL_HBR__HBR_CAPABLE_MASK 0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_HBR__HBR_CAPABLE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_HBR__HBR_ENABLE_MASK 0x00000010L
+#define AZALIA_F2_CODEC_PIN_CONTROL_HBR__HBR_ENABLE__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC__AUDIO_LIPSYNC_MASK 0x0000ff00L
+#define AZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC__AUDIO_LIPSYNC__SHIFT 0x00000008
+#define AZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC__VIDEO_LIPSYNC_MASK 0x000000ffL
+#define AZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC__VIDEO_LIPSYNC__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_MANUFACTURER_ID__MANUFACTURER_ID_MASK 0x0000ffffL
+#define AZALIA_F2_CODEC_PIN_CONTROL_MANUFACTURER_ID__MANUFACTURER_ID__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE__MULTICHANNEL01_CHANNEL_ID_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE__MULTICHANNEL01_CHANNEL_ID__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE__MULTICHANNEL01_ENABLE_MASK 0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE__MULTICHANNEL01_ENABLE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE__MULTICHANNEL01_MUTE_MASK 0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE__MULTICHANNEL01_MUTE__SHIFT 0x00000001
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_CHANNEL_ID_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_CHANNEL_ID__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_ENABLE_MASK 0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_ENABLE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_MUTE_MASK 0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE__MULTICHANNEL1_MUTE__SHIFT 0x00000001
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE__MULTICHANNEL23_CHANNEL_ID_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE__MULTICHANNEL23_CHANNEL_ID__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE__MULTICHANNEL23_ENABLE_MASK 0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE__MULTICHANNEL23_ENABLE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE__MULTICHANNEL23_MUTE_MASK 0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE__MULTICHANNEL23_MUTE__SHIFT 0x00000001
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_CHANNEL_ID_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_CHANNEL_ID__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_ENABLE_MASK 0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_ENABLE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_MUTE_MASK 0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE__MULTICHANNEL3_MUTE__SHIFT 0x00000001
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE__MULTICHANNEL45_CHANNEL_ID_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE__MULTICHANNEL45_CHANNEL_ID__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE__MULTICHANNEL45_ENABLE_MASK 0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE__MULTICHANNEL45_ENABLE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE__MULTICHANNEL45_MUTE_MASK 0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE__MULTICHANNEL45_MUTE__SHIFT 0x00000001
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_CHANNEL_ID_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_CHANNEL_ID__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_ENABLE_MASK 0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_ENABLE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_MUTE_MASK 0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE__MULTICHANNEL5_MUTE__SHIFT 0x00000001
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE__MULTICHANNEL67_CHANNEL_ID_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE__MULTICHANNEL67_CHANNEL_ID__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE__MULTICHANNEL67_ENABLE_MASK 0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE__MULTICHANNEL67_ENABLE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE__MULTICHANNEL67_MUTE_MASK 0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE__MULTICHANNEL67_MUTE__SHIFT 0x00000001
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_CHANNEL_ID_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_CHANNEL_ID__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_ENABLE_MASK 0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_ENABLE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_MUTE_MASK 0x00000002L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE__MULTICHANNEL7_MUTE__SHIFT 0x00000001
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE_MASK 0x00000001L
+#define AZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL_MODE__MULTICHANNEL_MODE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_PORTID0__PORTID_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_PIN_CONTROL_PORTID0__PORTID__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_PORTID1__PORTID_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_PIN_CONTROL_PORTID1__PORTID__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_PRODUCT_ID__PRODUCT_ID_MASK 0x0000ffffL
+#define AZALIA_F2_CODEC_PIN_CONTROL_PRODUCT_ID__PRODUCT_ID__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2__COLOR_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2__COLOR__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2__MISC_MASK 0x0000000fL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2__MISC__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3__CONNECTION_TYPE_MASK 0x0000000fL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3__CONNECTION_TYPE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3__DEFAULT_DEVICE_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3__DEFAULT_DEVICE__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4__LOCATION_MASK 0x0000003fL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4__LOCATION__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4__PORT_CONNECTIVITY_MASK 0x000000c0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4__PORT_CONNECTIVITY__SHIFT 0x00000006
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR_MASK 0x0000f000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__COLOR__SHIFT 0x0000000c
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE_MASK 0x000f0000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__CONNECTION_TYPE__SHIFT 0x00000010
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION_MASK 0x000000f0L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_ASSOCIATION__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE_MASK 0x00f00000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__DEFAULT_DEVICE__SHIFT 0x00000014
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION_MASK 0x3f000000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__LOCATION__SHIFT 0x00000018
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC_MASK 0x00000f00L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__MISC__SHIFT 0x00000008
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY_MASK 0xc0000000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__PORT_CONNECTIVITY__SHIFT 0x0000001e
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE_MASK 0x0000000fL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT__SEQUENCE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONNECTION_LIST_ENTRY__CONNECTION_LIST_ENTRY_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONNECTION_LIST_ENTRY__CONNECTION_LIST_ENTRY__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE_MASK 0x7fffffffL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__IMPEDANCE_SENSE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__PRESENCE_DETECT_MASK 0x80000000L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE__PRESENCE_DETECT__SHIFT 0x0000001f
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__DP_CONNECTION_MASK 0x00000200L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__DP_CONNECTION__SHIFT 0x00000009
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__EXTRA_CONNECTION_INFO_MASK 0x0000fc00L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__EXTRA_CONNECTION_INFO__SHIFT 0x0000000a
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__HDMI_CONNECTION_MASK 0x00000100L
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__HDMI_CONNECTION__SHIFT 0x00000008
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__SPEAKER_ALLOCATION_MASK 0x0000007fL
+#define AZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION__SPEAKER_ALLOCATION__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_SINK_DESCRIPTION_LEN__SINK_DESCRIPTION_LEN_MASK 0x000000ffL
+#define AZALIA_F2_CODEC_PIN_CONTROL_SINK_DESCRIPTION_LEN__SINK_DESCRIPTION_LEN__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE_MASK 0x00000080L
+#define AZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__ENABLE__SHIFT 0x00000007
+#define AZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG_MASK 0x0000003fL
+#define AZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE__TAG__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE_MASK 0x00000040L
+#define AZALIA_F2_CODEC_PIN_CONTROL_WIDGET_CONTROL__OUT_ENABLE__SHIFT 0x00000006
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE_MASK 0x00000008L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AMPLIFIER_PARAMETER_OVERRIDE__SHIFT 0x00000003
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES_MASK 0x00000001L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_CHANNEL_CAPABILITIES__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY_MASK 0x000f0000L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__AUDIO_WIDGET_CAPABILITIES_DELAY__SHIFT 0x00000010
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST_MASK 0x00000100L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__CONNECTION_LIST__SHIFT 0x00000008
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL_MASK 0x00000200L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__DIGITAL__SHIFT 0x00000009
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT_MASK 0x00000002L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__INPUT_AMPLIFIER_PRESENT__SHIFT 0x00000001
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP_MASK 0x00000800L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__LR_SWAP__SHIFT 0x0000000b
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT_MASK 0x00000004L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__OUTPUT_AMPLIFIER_PRESENT__SHIFT 0x00000002
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL_MASK 0x00000400L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__POWER_CONTROL__SHIFT 0x0000000a
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET_MASK 0x00000040L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__PROCESSING_WIDGET__SHIFT 0x00000006
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE_MASK 0x00000020L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__STRIPE__SHIFT 0x00000005
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE_MASK 0x00f00000L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__TYPE__SHIFT 0x00000014
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY_MASK 0x00000080L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES__UNSOLICITED_RESPONSE_CAPABILITY__SHIFT 0x00000007
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS_MASK 0x00000040L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__BALANCED_I_O_PINS__SHIFT 0x00000006
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__DP_MASK 0x01000000L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__DP__SHIFT 0x00000018
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE_MASK 0x00010000L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__EAPD_CAPABLE__SHIFT 0x00000010
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI_MASK 0x00000080L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__HDMI__SHIFT 0x00000007
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE_MASK 0x00000008L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__HEADPHONE_DRIVE_CAPABLE__SHIFT 0x00000003
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE_MASK 0x00000001L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__IMPEDANCE_SENSE_CAPABLE__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE_MASK 0x00000020L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__INPUT_CAPABLE__SHIFT 0x00000005
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY_MASK 0x00000004L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__JACK_DETECTION_CAPABILITY__SHIFT 0x00000002
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE_MASK 0x00000010L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__OUTPUT_CAPABLE__SHIFT 0x00000004
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED_MASK 0x00000002L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__TRIGGER_REQUIRED__SHIFT 0x00000001
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL_MASK 0x0000ff00L
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES__VREF_CONTROL__SHIFT 0x00000008
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CONNECTION_LIST_LENGTH__CONNECTION_LIST_LENGTH_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_PIN_PARAMETER_CONNECTION_LIST_LENGTH__CONNECTION_LIST_LENGTH__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_ROOT_PARAMETER_REVISION_ID__AZALIA_CODEC_ROOT_PARAMETER_REVISION_ID_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_ROOT_PARAMETER_REVISION_ID__AZALIA_CODEC_ROOT_PARAMETER_REVISION_ID__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT__AZALIA_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT__AZALIA_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT__SHIFT 0x00000000
+#define AZALIA_F2_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID__AZALIA_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID_MASK 0xffffffffL
+#define AZALIA_F2_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID__AZALIA_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID__SHIFT 0x00000000
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE_MASK 0x00000003L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_MODE__SHIFT 0x00000000
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER_MASK 0x0000003cL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0__IEC_60958_CS_SOURCE_NUMBER__SHIFT 0x00000002
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_MASK 0x00000003L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN_MASK 0x00000004L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY_OVRRD_EN__SHIFT 0x00000002
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_CLOCK_ACCURACY__SHIFT 0x00000000
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_MASK 0x00000078L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN_MASK 0x00000080L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH_OVRRD_EN__SHIFT 0x00000007
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1__IEC_60958_CS_WORD_LENGTH__SHIFT 0x00000003
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_MASK 0x0000003fL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN_MASK 0x00000040L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT 0x00000006
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2__IEC_60958_CS_SAMPLING_FREQUENCY__SHIFT 0x00000000
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK 0x0000000fL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN_MASK 0x00000010L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_OVRRD_EN__SHIFT 0x00000004
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3__IEC_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT 0x00000000
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_MASK 0x00000060L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A__SHIFT 0x00000005
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID_MASK 0x00000080L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_CGMS_A_VALID__SHIFT 0x00000007
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO_MASK 0x00000010L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_MPEG_SURROUND_INFO__SHIFT 0x00000004
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF_MASK 0x0000000fL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4__IEC_60958_CS_SAMPLING_FREQUENCY_COEFF__SHIFT 0x00000000
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L_MASK 0x0000000fL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_L__SHIFT 0x00000000
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R_MASK 0x000000f0L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5__IEC_60958_CS_CHANNEL_NUMBER_R__SHIFT 0x00000004
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2_MASK 0x0000000fL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_2__SHIFT 0x00000000
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3_MASK 0x000000f0L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6__IEC_60958_CS_CHANNEL_NUMBER_3__SHIFT 0x00000004
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4_MASK 0x0000000fL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_4__SHIFT 0x00000000
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5_MASK 0x000000f0L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7__IEC_60958_CS_CHANNEL_NUMBER_5__SHIFT 0x00000004
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6_MASK 0x0000000fL
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_6__SHIFT 0x00000000
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7_MASK 0x000000f0L
+#define AZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8__IEC_60958_CS_CHANNEL_NUMBER_7__SHIFT 0x00000004
+#define AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE_MASK 0x00007f00L
+#define AZALIA_FIFO_SIZE_CONTROL__MAX_FIFO_SIZE__SHIFT 0x00000008
+#define AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT_MASK 0x00ff0000L
+#define AZALIA_FIFO_SIZE_CONTROL__MAX_LATENCY_SUPPORT__SHIFT 0x00000010
+#define AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE_MASK 0x0000007fL
+#define AZALIA_FIFO_SIZE_CONTROL__MIN_FIFO_SIZE__SHIFT 0x00000000
+#define AZALIA_GLOBAL_CAPABILITIES__NUMBER_OF_SERIAL_DATA_OUTPUT_SIGNALS_MASK 0x00000006L
+#define AZALIA_GLOBAL_CAPABILITIES__NUMBER_OF_SERIAL_DATA_OUTPUT_SIGNALS__SHIFT 0x00000001
+#define AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET_MASK 0x00000001L
+#define AZALIA_LATENCY_COUNTER_CONTROL__AZALIA_LATENCY_COUNTER_RESET__SHIFT 0x00000000
+#define AZALIA_OUTPUT_PAYLOAD_CAPABILITY__OUTPUT_PAYLOAD_CAPABILITY_MASK 0x0000ffffL
+#define AZALIA_OUTPUT_PAYLOAD_CAPABILITY__OUTPUT_PAYLOAD_CAPABILITY__SHIFT 0x00000000
+#define AZALIA_OUTPUT_PAYLOAD_CAPABILITY__OUTSTRMPAY_MASK 0xffff0000L
+#define AZALIA_OUTPUT_PAYLOAD_CAPABILITY__OUTSTRMPAY__SHIFT 0x00000010
+#define AZALIA_OUTPUT_STREAM_ARBITER_CONTROL__LATENCY_HIDING_LEVEL_MASK 0x000000ffL
+#define AZALIA_OUTPUT_STREAM_ARBITER_CONTROL__LATENCY_HIDING_LEVEL__SHIFT 0x00000000
+#define AZALIA_OUTPUT_STREAM_ARBITER_CONTROL__SYS_MEM_ACTIVE_ENABLE_MASK 0x00000100L
+#define AZALIA_OUTPUT_STREAM_ARBITER_CONTROL__SYS_MEM_ACTIVE_ENABLE__SHIFT 0x00000008
+#define AZALIA_RIRB_AND_DP_CONTROL__DP_DMA_NON_SNOOP_MASK 0x00000010L
+#define AZALIA_RIRB_AND_DP_CONTROL__DP_DMA_NON_SNOOP__SHIFT 0x00000004
+#define AZALIA_RIRB_AND_DP_CONTROL__RIRB_NON_SNOOP_MASK 0x00000001L
+#define AZALIA_RIRB_AND_DP_CONTROL__RIRB_NON_SNOOP__SHIFT 0x00000000
+#define AZALIA_SCLK_CONTROL__AUDIO_SCLK_CONTROL_MASK 0x00000030L
+#define AZALIA_SCLK_CONTROL__AUDIO_SCLK_CONTROL__SHIFT 0x00000004
+#define AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA_MASK 0xffffffffL
+#define AZALIA_STREAM_DATA__AZALIA_STREAM_REG_DATA__SHIFT 0x00000000
+#define AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA_MASK 0xffffffffL
+#define AZALIA_STREAM_DEBUG__STREAM_DEBUG_DATA__SHIFT 0x00000000
+#define AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX_MASK 0x000000ffL
+#define AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_INDEX__SHIFT 0x00000000
+#define AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN_MASK 0x00000100L
+#define AZALIA_STREAM_INDEX__AZALIA_STREAM_REG_WRITE_EN__SHIFT 0x00000008
+#define AZALIA_UNDERFLOW_FILLER_SAMPLE__AZALIA_UNDERFLOW_FILLER_SAMPLE_MASK 0xffffffffL
+#define AZALIA_UNDERFLOW_FILLER_SAMPLE__AZALIA_UNDERFLOW_FILLER_SAMPLE__SHIFT 0x00000000
+#define AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT_MASK 0xffffffffL
+#define AZALIA_WORSTCASE_LATENCY_COUNT__AZALIA_WORSTCASE_LATENCY_COUNT__SHIFT 0x00000000
+#define AZ_TEST_DEBUG_DATA__AZ_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define AZ_TEST_DEBUG_DATA__AZ_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define AZ_TEST_DEBUG_INDEX__AZ_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define AZ_TEST_DEBUG_INDEX__AZ_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define AZ_TEST_DEBUG_INDEX__AZ_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define AZ_TEST_DEBUG_INDEX__AZ_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_CALC_FINAL_DUTY_CYCLE_EN_MASK 0x00000008L
+#define BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_CALC_FINAL_DUTY_CYCLE_EN__SHIFT 0x00000003
+#define BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_LEVEL_EN_MASK 0x00000004L
+#define BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_LEVEL_EN__SHIFT 0x00000002
+#define BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_STEP_SIZE_MASK 0xffff0000L
+#define BL1_PWM_ABM_CNTL__BL1_PWM_AUTO_UPDATE_CURRENT_ABM_STEP_SIZE__SHIFT 0x00000010
+#define BL1_PWM_ABM_CNTL__BL1_PWM_USE_ABM_EN_MASK 0x00000001L
+#define BL1_PWM_ABM_CNTL__BL1_PWM_USE_ABM_EN__SHIFT 0x00000000
+#define BL1_PWM_ABM_CNTL__BL1_PWM_USE_AMBIENT_LEVEL_EN_MASK 0x00000002L
+#define BL1_PWM_ABM_CNTL__BL1_PWM_USE_AMBIENT_LEVEL_EN__SHIFT 0x00000001
+#define BL1_PWM_AMBIENT_LIGHT_LEVEL__BL1_PWM_AMBIENT_LIGHT_LEVEL_MASK 0x0001ffffL
+#define BL1_PWM_AMBIENT_LIGHT_LEVEL__BL1_PWM_AMBIENT_LIGHT_LEVEL__SHIFT 0x00000000
+#define BL1_PWM_BL_UPDATE_SAMPLE_RATE__ABM1_HGLS_REG_LOCK_MASK 0x80000000L
+#define BL1_PWM_BL_UPDATE_SAMPLE_RATE__ABM1_HGLS_REG_LOCK__SHIFT 0x0000001f
+#define BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_INITIAL_SAMPLE_RATE_COUNT_VALUE_WHEN_RESET_MASK 0x00ff0000L
+#define BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_INITIAL_SAMPLE_RATE_COUNT_VALUE_WHEN_RESET__SHIFT 0x00000010
+#define BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_RESET_SAMPLE_RATE_FRAME_COUNTER_MASK 0x00000002L
+#define BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_RESET_SAMPLE_RATE_FRAME_COUNTER__SHIFT 0x00000001
+#define BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_COUNT_EN_MASK 0x00000001L
+#define BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_COUNT_EN__SHIFT 0x00000000
+#define BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_FRAME_COUNT_MASK 0x0000ff00L
+#define BL1_PWM_BL_UPDATE_SAMPLE_RATE__BL1_PWM_BL_UPDATE_SAMPLE_RATE_FRAME_COUNT__SHIFT 0x00000008
+#define BL1_PWM_CURRENT_ABM_LEVEL__BL1_PWM_CURRENT_ABM_LEVEL_MASK 0x0001ffffL
+#define BL1_PWM_CURRENT_ABM_LEVEL__BL1_PWM_CURRENT_ABM_LEVEL__SHIFT 0x00000000
+#define BL1_PWM_FINAL_DUTY_CYCLE__BL1_PWM_FINAL_DUTY_CYCLE_MASK 0x0001ffffL
+#define BL1_PWM_FINAL_DUTY_CYCLE__BL1_PWM_FINAL_DUTY_CYCLE__SHIFT 0x00000000
+#define BL1_PWM_GRP2_REG_LOCK__BL1_PWM_GRP2_FRAME_START_DISP_SEL_MASK 0x000e0000L
+#define BL1_PWM_GRP2_REG_LOCK__BL1_PWM_GRP2_FRAME_START_DISP_SEL__SHIFT 0x00000011
+#define BL1_PWM_GRP2_REG_LOCK__BL1_PWM_GRP2_IGNORE_MASTER_LOCK_EN_MASK 0x80000000L
+#define BL1_PWM_GRP2_REG_LOCK__BL1_PWM_GRP2_IGNORE_MASTER_LOCK_EN__SHIFT 0x0000001f
+#define BL1_PWM_GRP2_REG_LOCK__BL1_PWM_GRP2_READBACK_DB_REG_VALUE_EN_MASK 0x01000000L
+#define BL1_PWM_GRP2_REG_LOCK__BL1_PWM_GRP2_READBACK_DB_REG_VALUE_EN__SHIFT 0x00000018
+#define BL1_PWM_GRP2_REG_LOCK__BL1_PWM_GRP2_REG_LOCK_MASK 0x00000001L
+#define BL1_PWM_GRP2_REG_LOCK__BL1_PWM_GRP2_REG_LOCK__SHIFT 0x00000000
+#define BL1_PWM_GRP2_REG_LOCK__BL1_PWM_GRP2_REG_UPDATE_PENDING_MASK 0x00000100L
+#define BL1_PWM_GRP2_REG_LOCK__BL1_PWM_GRP2_REG_UPDATE_PENDING__SHIFT 0x00000008
+#define BL1_PWM_GRP2_REG_LOCK__BL1_PWM_GRP2_UPDATE_AT_FRAME_START_MASK 0x00010000L
+#define BL1_PWM_GRP2_REG_LOCK__BL1_PWM_GRP2_UPDATE_AT_FRAME_START__SHIFT 0x00000010
+#define BL1_PWM_MINIMUM_DUTY_CYCLE__BL1_PWM_MINIMUM_DUTY_CYCLE_MASK 0x0001ffffL
+#define BL1_PWM_MINIMUM_DUTY_CYCLE__BL1_PWM_MINIMUM_DUTY_CYCLE__SHIFT 0x00000000
+#define BL1_PWM_TARGET_ABM_LEVEL__BL1_PWM_TARGET_ABM_LEVEL_MASK 0x0001ffffL
+#define BL1_PWM_TARGET_ABM_LEVEL__BL1_PWM_TARGET_ABM_LEVEL__SHIFT 0x00000000
+#define BL1_PWM_USER_LEVEL__BL1_PWM_USER_LEVEL_MASK 0x0001ffffL
+#define BL1_PWM_USER_LEVEL__BL1_PWM_USER_LEVEL__SHIFT 0x00000000
+#define BL_PWM_CNTL2__BL_PWM_OVERRIDE_BL_OUT_ENABLE_MASK 0x40000000L
+#define BL_PWM_CNTL2__BL_PWM_OVERRIDE_BL_OUT_ENABLE__SHIFT 0x0000001e
+#define BL_PWM_CNTL2__BL_PWM_OVERRIDE_LVTMA_PWRSEQ_EN_MASK 0x80000000L
+#define BL_PWM_CNTL2__BL_PWM_OVERRIDE_LVTMA_PWRSEQ_EN__SHIFT 0x0000001f
+#define BL_PWM_CNTL2__BL_PWM_POST_FRAME_START_DELAY_BEFORE_UPDATE_MASK 0x0000ffffL
+#define BL_PWM_CNTL2__BL_PWM_POST_FRAME_START_DELAY_BEFORE_UPDATE__SHIFT 0x00000000
+#define BL_PWM_CNTL2__DBG_BL_PWM_INPUT_REFCLK_SELECT_MASK 0x30000000L
+#define BL_PWM_CNTL2__DBG_BL_PWM_INPUT_REFCLK_SELECT__SHIFT 0x0000001c
+#define BL_PWM_CNTL__BL_ACTIVE_INT_FRAC_CNT_MASK 0x0000ffffL
+#define BL_PWM_CNTL__BL_ACTIVE_INT_FRAC_CNT__SHIFT 0x00000000
+#define BL_PWM_CNTL__BL_PWM_EN_MASK 0x80000000L
+#define BL_PWM_CNTL__BL_PWM_EN__SHIFT 0x0000001f
+#define BL_PWM_CNTL__BL_PWM_FRACTIONAL_EN_MASK 0x40000000L
+#define BL_PWM_CNTL__BL_PWM_FRACTIONAL_EN__SHIFT 0x0000001e
+#define BL_PWM_GRP1_REG_LOCK__BL_PWM_GRP1_FRAME_START_DISP_SEL_MASK 0x000e0000L
+#define BL_PWM_GRP1_REG_LOCK__BL_PWM_GRP1_FRAME_START_DISP_SEL__SHIFT 0x00000011
+#define BL_PWM_GRP1_REG_LOCK__BL_PWM_GRP1_IGNORE_MASTER_LOCK_EN_MASK 0x80000000L
+#define BL_PWM_GRP1_REG_LOCK__BL_PWM_GRP1_IGNORE_MASTER_LOCK_EN__SHIFT 0x0000001f
+#define BL_PWM_GRP1_REG_LOCK__BL_PWM_GRP1_READBACK_DB_REG_VALUE_EN_MASK 0x01000000L
+#define BL_PWM_GRP1_REG_LOCK__BL_PWM_GRP1_READBACK_DB_REG_VALUE_EN__SHIFT 0x00000018
+#define BL_PWM_GRP1_REG_LOCK__BL_PWM_GRP1_REG_LOCK_MASK 0x00000001L
+#define BL_PWM_GRP1_REG_LOCK__BL_PWM_GRP1_REG_LOCK__SHIFT 0x00000000
+#define BL_PWM_GRP1_REG_LOCK__BL_PWM_GRP1_REG_UPDATE_PENDING_MASK 0x00000100L
+#define BL_PWM_GRP1_REG_LOCK__BL_PWM_GRP1_REG_UPDATE_PENDING__SHIFT 0x00000008
+#define BL_PWM_GRP1_REG_LOCK__BL_PWM_GRP1_UPDATE_AT_FRAME_START_MASK 0x00010000L
+#define BL_PWM_GRP1_REG_LOCK__BL_PWM_GRP1_UPDATE_AT_FRAME_START__SHIFT 0x00000010
+#define BL_PWM_PERIOD_CNTL__BL_PWM_PERIOD_BITCNT_MASK 0x000f0000L
+#define BL_PWM_PERIOD_CNTL__BL_PWM_PERIOD_BITCNT__SHIFT 0x00000010
+#define BL_PWM_PERIOD_CNTL__BL_PWM_PERIOD_MASK 0x0000ffffL
+#define BL_PWM_PERIOD_CNTL__BL_PWM_PERIOD__SHIFT 0x00000000
+#define BPHYC_DAC_AUTO_CALIB_CONTROL__BPHYC_DAC_CAL_COMPLETE_MASK 0x10000000L
+#define BPHYC_DAC_AUTO_CALIB_CONTROL__BPHYC_DAC_CAL_COMPLETE__SHIFT 0x0000001c
+#define BPHYC_DAC_AUTO_CALIB_CONTROL__BPHYC_DAC_CAL_DACADJ_EN_MASK 0x00000004L
+#define BPHYC_DAC_AUTO_CALIB_CONTROL__BPHYC_DAC_CAL_DACADJ_EN__SHIFT 0x00000002
+#define BPHYC_DAC_AUTO_CALIB_CONTROL__BPHYC_DAC_CAL_EN_MASK 0x00000002L
+#define BPHYC_DAC_AUTO_CALIB_CONTROL__BPHYC_DAC_CAL_EN__SHIFT 0x00000001
+#define BPHYC_DAC_AUTO_CALIB_CONTROL__BPHYC_DAC_CAL_INITB_MASK 0x00000001L
+#define BPHYC_DAC_AUTO_CALIB_CONTROL__BPHYC_DAC_CAL_INITB__SHIFT 0x00000000
+#define BPHYC_DAC_AUTO_CALIB_CONTROL__BPHYC_DAC_CAL_MASK_MASK 0x00700000L
+#define BPHYC_DAC_AUTO_CALIB_CONTROL__BPHYC_DAC_CAL_MASK__SHIFT 0x00000014
+#define BPHYC_DAC_AUTO_CALIB_CONTROL__BPHYC_DAC_CAL_WAIT_ADJUST_MASK 0x00003ff0L
+#define BPHYC_DAC_AUTO_CALIB_CONTROL__BPHYC_DAC_CAL_WAIT_ADJUST__SHIFT 0x00000004
+#define BPHYC_DAC_MACRO_CNTL__BPHYC_DAC_ANALOG_MONITOR_MASK 0x0f000000L
+#define BPHYC_DAC_MACRO_CNTL__BPHYC_DAC_ANALOG_MONITOR__SHIFT 0x00000018
+#define BPHYC_DAC_MACRO_CNTL__BPHYC_DAC_BANDGAP_ADJUSTMENT_MASK 0x003f0000L
+#define BPHYC_DAC_MACRO_CNTL__BPHYC_DAC_BANDGAP_ADJUSTMENT__SHIFT 0x00000010
+#define BPHYC_DAC_MACRO_CNTL__BPHYC_DAC_COREMON_MASK 0x10000000L
+#define BPHYC_DAC_MACRO_CNTL__BPHYC_DAC_COREMON__SHIFT 0x0000001c
+#define BPHYC_DAC_MACRO_CNTL__BPHYC_DAC_WHITE_FINE_CONTROL_MASK 0x00003f00L
+#define BPHYC_DAC_MACRO_CNTL__BPHYC_DAC_WHITE_FINE_CONTROL__SHIFT 0x00000008
+#define BPHYC_DAC_MACRO_CNTL__BPHYC_DAC_WHITE_LEVEL_MASK 0x00000003L
+#define BPHYC_DAC_MACRO_CNTL__BPHYC_DAC_WHITE_LEVEL__SHIFT 0x00000000
+#define CC_DC_PIPE_DIS__DC_PIPE_DIS_MASK 0x0000007eL
+#define CC_DC_PIPE_DIS__DC_PIPE_DIS__SHIFT 0x00000001
+#define CC_RCU_DC_AUDIO_PORT_CONNECTIVITY__PORT_CONNECTIVITY_MASK 0x00000007L
+#define CC_RCU_DC_AUDIO_PORT_CONNECTIVITY__PORT_CONNECTIVITY_OVERRIDE_ENABLE_MASK 0x00000010L
+#define CC_RCU_DC_AUDIO_PORT_CONNECTIVITY__PORT_CONNECTIVITY_OVERRIDE_ENABLE__SHIFT 0x00000004
+#define CC_RCU_DC_AUDIO_PORT_CONNECTIVITY__PORT_CONNECTIVITY__SHIFT 0x00000000
+#define COMM_MATRIXA_TRANS_C11_C12__COMM_MATRIXA_TRANS_C11_MASK 0x0000ffffL
+#define COMM_MATRIXA_TRANS_C11_C12__COMM_MATRIXA_TRANS_C11__SHIFT 0x00000000
+#define COMM_MATRIXA_TRANS_C11_C12__COMM_MATRIXA_TRANS_C12_MASK 0xffff0000L
+#define COMM_MATRIXA_TRANS_C11_C12__COMM_MATRIXA_TRANS_C12__SHIFT 0x00000010
+#define COMM_MATRIXA_TRANS_C13_C14__COMM_MATRIXA_TRANS_C13_MASK 0x0000ffffL
+#define COMM_MATRIXA_TRANS_C13_C14__COMM_MATRIXA_TRANS_C13__SHIFT 0x00000000
+#define COMM_MATRIXA_TRANS_C13_C14__COMM_MATRIXA_TRANS_C14_MASK 0xffff0000L
+#define COMM_MATRIXA_TRANS_C13_C14__COMM_MATRIXA_TRANS_C14__SHIFT 0x00000010
+#define COMM_MATRIXA_TRANS_C21_C22__COMM_MATRIXA_TRANS_C21_MASK 0x0000ffffL
+#define COMM_MATRIXA_TRANS_C21_C22__COMM_MATRIXA_TRANS_C21__SHIFT 0x00000000
+#define COMM_MATRIXA_TRANS_C21_C22__COMM_MATRIXA_TRANS_C22_MASK 0xffff0000L
+#define COMM_MATRIXA_TRANS_C21_C22__COMM_MATRIXA_TRANS_C22__SHIFT 0x00000010
+#define COMM_MATRIXA_TRANS_C23_C24__COMM_MATRIXA_TRANS_C23_MASK 0x0000ffffL
+#define COMM_MATRIXA_TRANS_C23_C24__COMM_MATRIXA_TRANS_C23__SHIFT 0x00000000
+#define COMM_MATRIXA_TRANS_C23_C24__COMM_MATRIXA_TRANS_C24_MASK 0xffff0000L
+#define COMM_MATRIXA_TRANS_C23_C24__COMM_MATRIXA_TRANS_C24__SHIFT 0x00000010
+#define COMM_MATRIXA_TRANS_C31_C32__COMM_MATRIXA_TRANS_C31_MASK 0x0000ffffL
+#define COMM_MATRIXA_TRANS_C31_C32__COMM_MATRIXA_TRANS_C31__SHIFT 0x00000000
+#define COMM_MATRIXA_TRANS_C31_C32__COMM_MATRIXA_TRANS_C32_MASK 0xffff0000L
+#define COMM_MATRIXA_TRANS_C31_C32__COMM_MATRIXA_TRANS_C32__SHIFT 0x00000010
+#define COMM_MATRIXA_TRANS_C33_C34__COMM_MATRIXA_TRANS_C33_MASK 0x0000ffffL
+#define COMM_MATRIXA_TRANS_C33_C34__COMM_MATRIXA_TRANS_C33__SHIFT 0x00000000
+#define COMM_MATRIXA_TRANS_C33_C34__COMM_MATRIXA_TRANS_C34_MASK 0xffff0000L
+#define COMM_MATRIXA_TRANS_C33_C34__COMM_MATRIXA_TRANS_C34__SHIFT 0x00000010
+#define COMM_MATRIXB_TRANS_C11_C12__COMM_MATRIXB_TRANS_C11_MASK 0x0000ffffL
+#define COMM_MATRIXB_TRANS_C11_C12__COMM_MATRIXB_TRANS_C11__SHIFT 0x00000000
+#define COMM_MATRIXB_TRANS_C11_C12__COMM_MATRIXB_TRANS_C12_MASK 0xffff0000L
+#define COMM_MATRIXB_TRANS_C11_C12__COMM_MATRIXB_TRANS_C12__SHIFT 0x00000010
+#define COMM_MATRIXB_TRANS_C13_C14__COMM_MATRIXB_TRANS_C13_MASK 0x0000ffffL
+#define COMM_MATRIXB_TRANS_C13_C14__COMM_MATRIXB_TRANS_C13__SHIFT 0x00000000
+#define COMM_MATRIXB_TRANS_C13_C14__COMM_MATRIXB_TRANS_C14_MASK 0xffff0000L
+#define COMM_MATRIXB_TRANS_C13_C14__COMM_MATRIXB_TRANS_C14__SHIFT 0x00000010
+#define COMM_MATRIXB_TRANS_C21_C22__COMM_MATRIXB_TRANS_C21_MASK 0x0000ffffL
+#define COMM_MATRIXB_TRANS_C21_C22__COMM_MATRIXB_TRANS_C21__SHIFT 0x00000000
+#define COMM_MATRIXB_TRANS_C21_C22__COMM_MATRIXB_TRANS_C22_MASK 0xffff0000L
+#define COMM_MATRIXB_TRANS_C21_C22__COMM_MATRIXB_TRANS_C22__SHIFT 0x00000010
+#define COMM_MATRIXB_TRANS_C23_C24__COMM_MATRIXB_TRANS_C23_MASK 0x0000ffffL
+#define COMM_MATRIXB_TRANS_C23_C24__COMM_MATRIXB_TRANS_C23__SHIFT 0x00000000
+#define COMM_MATRIXB_TRANS_C23_C24__COMM_MATRIXB_TRANS_C24_MASK 0xffff0000L
+#define COMM_MATRIXB_TRANS_C23_C24__COMM_MATRIXB_TRANS_C24__SHIFT 0x00000010
+#define COMM_MATRIXB_TRANS_C31_C32__COMM_MATRIXB_TRANS_C31_MASK 0x0000ffffL
+#define COMM_MATRIXB_TRANS_C31_C32__COMM_MATRIXB_TRANS_C31__SHIFT 0x00000000
+#define COMM_MATRIXB_TRANS_C31_C32__COMM_MATRIXB_TRANS_C32_MASK 0xffff0000L
+#define COMM_MATRIXB_TRANS_C31_C32__COMM_MATRIXB_TRANS_C32__SHIFT 0x00000010
+#define COMM_MATRIXB_TRANS_C33_C34__COMM_MATRIXB_TRANS_C33_MASK 0x0000ffffL
+#define COMM_MATRIXB_TRANS_C33_C34__COMM_MATRIXB_TRANS_C33__SHIFT 0x00000000
+#define COMM_MATRIXB_TRANS_C33_C34__COMM_MATRIXB_TRANS_C34_MASK 0xffff0000L
+#define COMM_MATRIXB_TRANS_C33_C34__COMM_MATRIXB_TRANS_C34__SHIFT 0x00000010
+#define CRT00__H_TOTAL_MASK 0x000000ffL
+#define CRT00__H_TOTAL__SHIFT 0x00000000
+#define CRT01__H_DISP_END_MASK 0x000000ffL
+#define CRT01__H_DISP_END__SHIFT 0x00000000
+#define CRT02__H_BLANK_START_MASK 0x000000ffL
+#define CRT02__H_BLANK_START__SHIFT 0x00000000
+#define CRT03__CR10CR11_R_DIS_B_MASK 0x00000080L
+#define CRT03__CR10CR11_R_DIS_B__SHIFT 0x00000007
+#define CRT03__H_BLANK_END_MASK 0x0000001fL
+#define CRT03__H_BLANK_END__SHIFT 0x00000000
+#define CRT03__H_DE_SKEW_MASK 0x00000060L
+#define CRT03__H_DE_SKEW__SHIFT 0x00000005
+#define CRT04__H_SYNC_START_MASK 0x000000ffL
+#define CRT04__H_SYNC_START__SHIFT 0x00000000
+#define CRT05__H_BLANK_END_B5_MASK 0x00000080L
+#define CRT05__H_BLANK_END_B5__SHIFT 0x00000007
+#define CRT05__H_SYNC_END_MASK 0x0000001fL
+#define CRT05__H_SYNC_END__SHIFT 0x00000000
+#define CRT05__H_SYNC_SKEW_MASK 0x00000060L
+#define CRT05__H_SYNC_SKEW__SHIFT 0x00000005
+#define CRT06__V_TOTAL_MASK 0x000000ffL
+#define CRT06__V_TOTAL__SHIFT 0x00000000
+#define CRT07__LINE_CMP_B8_MASK 0x00000010L
+#define CRT07__LINE_CMP_B8__SHIFT 0x00000004
+#define CRT07__V_BLANK_START_B8_MASK 0x00000008L
+#define CRT07__V_BLANK_START_B8__SHIFT 0x00000003
+#define CRT07__V_DISP_END_B8_MASK 0x00000002L
+#define CRT07__V_DISP_END_B8__SHIFT 0x00000001
+#define CRT07__V_DISP_END_B9_MASK 0x00000040L
+#define CRT07__V_DISP_END_B9__SHIFT 0x00000006
+#define CRT07__V_SYNC_START_B8_MASK 0x00000004L
+#define CRT07__V_SYNC_START_B8__SHIFT 0x00000002
+#define CRT07__V_SYNC_START_B9_MASK 0x00000080L
+#define CRT07__V_SYNC_START_B9__SHIFT 0x00000007
+#define CRT07__V_TOTAL_B8_MASK 0x00000001L
+#define CRT07__V_TOTAL_B8__SHIFT 0x00000000
+#define CRT07__V_TOTAL_B9_MASK 0x00000020L
+#define CRT07__V_TOTAL_B9__SHIFT 0x00000005
+#define CRT08__BYTE_PAN_MASK 0x00000060L
+#define CRT08__BYTE_PAN__SHIFT 0x00000005
+#define CRT08__ROW_SCAN_START_MASK 0x0000001fL
+#define CRT08__ROW_SCAN_START__SHIFT 0x00000000
+#define CRT09__DOUBLE_CHAR_HEIGHT_MASK 0x00000080L
+#define CRT09__DOUBLE_CHAR_HEIGHT__SHIFT 0x00000007
+#define CRT09__LINE_CMP_B9_MASK 0x00000040L
+#define CRT09__LINE_CMP_B9__SHIFT 0x00000006
+#define CRT09__MAX_ROW_SCAN_MASK 0x0000001fL
+#define CRT09__MAX_ROW_SCAN__SHIFT 0x00000000
+#define CRT09__V_BLANK_START_B9_MASK 0x00000020L
+#define CRT09__V_BLANK_START_B9__SHIFT 0x00000005
+#define CRT0A__CURSOR_DISABLE_MASK 0x00000020L
+#define CRT0A__CURSOR_DISABLE__SHIFT 0x00000005
+#define CRT0A__CURSOR_START_MASK 0x0000001fL
+#define CRT0A__CURSOR_START__SHIFT 0x00000000
+#define CRT0B__CURSOR_END_MASK 0x0000001fL
+#define CRT0B__CURSOR_END__SHIFT 0x00000000
+#define CRT0B__CURSOR_SKEW_MASK 0x00000060L
+#define CRT0B__CURSOR_SKEW__SHIFT 0x00000005
+#define CRT0C__DISP_START_MASK 0x000000ffL
+#define CRT0C__DISP_START__SHIFT 0x00000000
+#define CRT0D__DISP_START_MASK 0x000000ffL
+#define CRT0D__DISP_START__SHIFT 0x00000000
+#define CRT0E__CURSOR_LOC_HI_MASK 0x000000ffL
+#define CRT0E__CURSOR_LOC_HI__SHIFT 0x00000000
+#define CRT0F__CURSOR_LOC_LO_MASK 0x000000ffL
+#define CRT0F__CURSOR_LOC_LO__SHIFT 0x00000000
+#define CRT10__V_SYNC_START_MASK 0x000000ffL
+#define CRT10__V_SYNC_START__SHIFT 0x00000000
+#define CRT11__C0T7_WR_ONLY_MASK 0x00000080L
+#define CRT11__C0T7_WR_ONLY__SHIFT 0x00000007
+#define CRT11__SEL5_REFRESH_CYC_MASK 0x00000040L
+#define CRT11__SEL5_REFRESH_CYC__SHIFT 0x00000006
+#define CRT11__V_INTR_CLR_MASK 0x00000010L
+#define CRT11__V_INTR_CLR__SHIFT 0x00000004
+#define CRT11__V_INTR_EN_MASK 0x00000020L
+#define CRT11__V_INTR_EN__SHIFT 0x00000005
+#define CRT11__V_SYNC_END_MASK 0x0000000fL
+#define CRT11__V_SYNC_END__SHIFT 0x00000000
+#define CRT12__V_DISP_END_MASK 0x000000ffL
+#define CRT12__V_DISP_END__SHIFT 0x00000000
+#define CRT13__DISP_PITCH_MASK 0x000000ffL
+#define CRT13__DISP_PITCH__SHIFT 0x00000000
+#define CRT14__ADDR_CNT_BY4_MASK 0x00000020L
+#define CRT14__ADDR_CNT_BY4__SHIFT 0x00000005
+#define CRT14__DOUBLE_WORD_MASK 0x00000040L
+#define CRT14__DOUBLE_WORD__SHIFT 0x00000006
+#define CRT14__UNDRLN_LOC_MASK 0x0000001fL
+#define CRT14__UNDRLN_LOC__SHIFT 0x00000000
+#define CRT15__V_BLANK_START_MASK 0x000000ffL
+#define CRT15__V_BLANK_START__SHIFT 0x00000000
+#define CRT16__V_BLANK_END_MASK 0x000000ffL
+#define CRT16__V_BLANK_END__SHIFT 0x00000000
+#define CRT17__ADDR_CNT_BY2_MASK 0x00000008L
+#define CRT17__ADDR_CNT_BY2__SHIFT 0x00000003
+#define CRT17__BYTE_MODE_MASK 0x00000040L
+#define CRT17__BYTE_MODE__SHIFT 0x00000006
+#define CRT17__CRTC_SYNC_EN_MASK 0x00000080L
+#define CRT17__CRTC_SYNC_EN__SHIFT 0x00000007
+#define CRT17__RA0_AS_A13B_MASK 0x00000001L
+#define CRT17__RA0_AS_A13B__SHIFT 0x00000000
+#define CRT17__RA1_AS_A14B_MASK 0x00000002L
+#define CRT17__RA1_AS_A14B__SHIFT 0x00000001
+#define CRT17__VCOUNT_BY2_MASK 0x00000004L
+#define CRT17__VCOUNT_BY2__SHIFT 0x00000002
+#define CRT17__WRAP_A15TOA0_MASK 0x00000020L
+#define CRT17__WRAP_A15TOA0__SHIFT 0x00000005
+#define CRT18__LINE_CMP_MASK 0x000000ffL
+#define CRT18__LINE_CMP__SHIFT 0x00000000
+#define CRT1E__GRPH_DEC_RD1_MASK 0x00000002L
+#define CRT1E__GRPH_DEC_RD1__SHIFT 0x00000001
+#define CRT1F__GRPH_DEC_RD0_MASK 0x000000ffL
+#define CRT1F__GRPH_DEC_RD0__SHIFT 0x00000000
+#define CRT22__GRPH_LATCH_DATA_MASK 0x000000ffL
+#define CRT22__GRPH_LATCH_DATA__SHIFT 0x00000000
+#define CRTC0_PIXEL_RATE_CNTL__CRTC0_ADD_PIXEL_MASK 0x00000100L
+#define CRTC0_PIXEL_RATE_CNTL__CRTC0_ADD_PIXEL__SHIFT 0x00000008
+#define CRTC0_PIXEL_RATE_CNTL__CRTC0_DISPOUT_ERROR_COUNT_MASK 0x0fff0000L
+#define CRTC0_PIXEL_RATE_CNTL__CRTC0_DISPOUT_ERROR_COUNT__SHIFT 0x00000010
+#define CRTC0_PIXEL_RATE_CNTL__CRTC0_DISPOUT_FIFO_ERROR_MASK 0x0000c000L
+#define CRTC0_PIXEL_RATE_CNTL__CRTC0_DISPOUT_FIFO_ERROR__SHIFT 0x0000000e
+#define CRTC0_PIXEL_RATE_CNTL__CRTC0_DROP_PIXEL_MASK 0x00000200L
+#define CRTC0_PIXEL_RATE_CNTL__CRTC0_DROP_PIXEL__SHIFT 0x00000009
+#define CRTC0_PIXEL_RATE_CNTL__CRTC0_PIXEL_RATE_SOURCE_MASK 0x00000003L
+#define CRTC0_PIXEL_RATE_CNTL__CRTC0_PIXEL_RATE_SOURCE__SHIFT 0x00000000
+#define CRTC0_PIXEL_RATE_CNTL__DP_DTO0_ENABLE_MASK 0x00000010L
+#define CRTC0_PIXEL_RATE_CNTL__DP_DTO0_ENABLE__SHIFT 0x00000004
+#define CRTC1_PIXEL_RATE_CNTL__CRTC1_ADD_PIXEL_MASK 0x00000100L
+#define CRTC1_PIXEL_RATE_CNTL__CRTC1_ADD_PIXEL__SHIFT 0x00000008
+#define CRTC1_PIXEL_RATE_CNTL__CRTC1_DISPOUT_ERROR_COUNT_MASK 0x0fff0000L
+#define CRTC1_PIXEL_RATE_CNTL__CRTC1_DISPOUT_ERROR_COUNT__SHIFT 0x00000010
+#define CRTC1_PIXEL_RATE_CNTL__CRTC1_DISPOUT_FIFO_ERROR_MASK 0x0000c000L
+#define CRTC1_PIXEL_RATE_CNTL__CRTC1_DISPOUT_FIFO_ERROR__SHIFT 0x0000000e
+#define CRTC1_PIXEL_RATE_CNTL__CRTC1_DROP_PIXEL_MASK 0x00000200L
+#define CRTC1_PIXEL_RATE_CNTL__CRTC1_DROP_PIXEL__SHIFT 0x00000009
+#define CRTC1_PIXEL_RATE_CNTL__CRTC1_PIXEL_RATE_SOURCE_MASK 0x00000003L
+#define CRTC1_PIXEL_RATE_CNTL__CRTC1_PIXEL_RATE_SOURCE__SHIFT 0x00000000
+#define CRTC1_PIXEL_RATE_CNTL__DP_DTO1_ENABLE_MASK 0x00000010L
+#define CRTC1_PIXEL_RATE_CNTL__DP_DTO1_ENABLE__SHIFT 0x00000004
+#define CRTC2_PIXEL_RATE_CNTL__CRTC2_ADD_PIXEL_MASK 0x00000100L
+#define CRTC2_PIXEL_RATE_CNTL__CRTC2_ADD_PIXEL__SHIFT 0x00000008
+#define CRTC2_PIXEL_RATE_CNTL__CRTC2_DISPOUT_ERROR_COUNT_MASK 0x0fff0000L
+#define CRTC2_PIXEL_RATE_CNTL__CRTC2_DISPOUT_ERROR_COUNT__SHIFT 0x00000010
+#define CRTC2_PIXEL_RATE_CNTL__CRTC2_DISPOUT_FIFO_ERROR_MASK 0x0000c000L
+#define CRTC2_PIXEL_RATE_CNTL__CRTC2_DISPOUT_FIFO_ERROR__SHIFT 0x0000000e
+#define CRTC2_PIXEL_RATE_CNTL__CRTC2_DROP_PIXEL_MASK 0x00000200L
+#define CRTC2_PIXEL_RATE_CNTL__CRTC2_DROP_PIXEL__SHIFT 0x00000009
+#define CRTC2_PIXEL_RATE_CNTL__CRTC2_PIXEL_RATE_SOURCE_MASK 0x00000003L
+#define CRTC2_PIXEL_RATE_CNTL__CRTC2_PIXEL_RATE_SOURCE__SHIFT 0x00000000
+#define CRTC2_PIXEL_RATE_CNTL__DP_DTO2_ENABLE_MASK 0x00000010L
+#define CRTC2_PIXEL_RATE_CNTL__DP_DTO2_ENABLE__SHIFT 0x00000004
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_EN_DB_MASK 0x00000010L
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_EN_DB__SHIFT 0x00000004
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_EN_MASK 0x00000001L
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_EN__SHIFT 0x00000000
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_F_COUNT_MASK 0x000c0000L
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_F_COUNT_RESET_MASK 0x00010000L
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_F_COUNT_RESET_PENDING_MASK 0x00020000L
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_F_COUNT_RESET_PENDING__SHIFT 0x00000011
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_F_COUNT_RESET__SHIFT 0x00000010
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_F_COUNT__SHIFT 0x00000012
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_STEREO_SEL_OVR_MASK 0x00001000L
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_STEREO_SEL_OVR__SHIFT 0x0000000c
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_V_UPDATE_MODE_MASK 0x00000300L
+#define CRTC_3D_STRUCTURE_CONTROL__CRTC_3D_STRUCTURE_V_UPDATE_MODE__SHIFT 0x00000008
+#define CRTC3_PIXEL_RATE_CNTL__CRTC3_ADD_PIXEL_MASK 0x00000100L
+#define CRTC3_PIXEL_RATE_CNTL__CRTC3_ADD_PIXEL__SHIFT 0x00000008
+#define CRTC3_PIXEL_RATE_CNTL__CRTC3_DISPOUT_ERROR_COUNT_MASK 0x0fff0000L
+#define CRTC3_PIXEL_RATE_CNTL__CRTC3_DISPOUT_ERROR_COUNT__SHIFT 0x00000010
+#define CRTC3_PIXEL_RATE_CNTL__CRTC3_DISPOUT_FIFO_ERROR_MASK 0x0000c000L
+#define CRTC3_PIXEL_RATE_CNTL__CRTC3_DISPOUT_FIFO_ERROR__SHIFT 0x0000000e
+#define CRTC3_PIXEL_RATE_CNTL__CRTC3_DROP_PIXEL_MASK 0x00000200L
+#define CRTC3_PIXEL_RATE_CNTL__CRTC3_DROP_PIXEL__SHIFT 0x00000009
+#define CRTC3_PIXEL_RATE_CNTL__CRTC3_PIXEL_RATE_SOURCE_MASK 0x00000003L
+#define CRTC3_PIXEL_RATE_CNTL__CRTC3_PIXEL_RATE_SOURCE__SHIFT 0x00000000
+#define CRTC3_PIXEL_RATE_CNTL__DP_DTO3_ENABLE_MASK 0x00000010L
+#define CRTC3_PIXEL_RATE_CNTL__DP_DTO3_ENABLE__SHIFT 0x00000004
+#define CRTC4_PIXEL_RATE_CNTL__CRTC4_ADD_PIXEL_MASK 0x00000100L
+#define CRTC4_PIXEL_RATE_CNTL__CRTC4_ADD_PIXEL__SHIFT 0x00000008
+#define CRTC4_PIXEL_RATE_CNTL__CRTC4_DISPOUT_ERROR_COUNT_MASK 0x0fff0000L
+#define CRTC4_PIXEL_RATE_CNTL__CRTC4_DISPOUT_ERROR_COUNT__SHIFT 0x00000010
+#define CRTC4_PIXEL_RATE_CNTL__CRTC4_DISPOUT_FIFO_ERROR_MASK 0x0000c000L
+#define CRTC4_PIXEL_RATE_CNTL__CRTC4_DISPOUT_FIFO_ERROR__SHIFT 0x0000000e
+#define CRTC4_PIXEL_RATE_CNTL__CRTC4_DROP_PIXEL_MASK 0x00000200L
+#define CRTC4_PIXEL_RATE_CNTL__CRTC4_DROP_PIXEL__SHIFT 0x00000009
+#define CRTC4_PIXEL_RATE_CNTL__CRTC4_PIXEL_RATE_SOURCE_MASK 0x00000003L
+#define CRTC4_PIXEL_RATE_CNTL__CRTC4_PIXEL_RATE_SOURCE__SHIFT 0x00000000
+#define CRTC4_PIXEL_RATE_CNTL__DP_DTO4_ENABLE_MASK 0x00000010L
+#define CRTC4_PIXEL_RATE_CNTL__DP_DTO4_ENABLE__SHIFT 0x00000004
+#define CRTC5_PIXEL_RATE_CNTL__CRTC5_ADD_PIXEL_MASK 0x00000100L
+#define CRTC5_PIXEL_RATE_CNTL__CRTC5_ADD_PIXEL__SHIFT 0x00000008
+#define CRTC5_PIXEL_RATE_CNTL__CRTC5_DISPOUT_ERROR_COUNT_MASK 0x0fff0000L
+#define CRTC5_PIXEL_RATE_CNTL__CRTC5_DISPOUT_ERROR_COUNT__SHIFT 0x00000010
+#define CRTC5_PIXEL_RATE_CNTL__CRTC5_DISPOUT_FIFO_ERROR_MASK 0x0000c000L
+#define CRTC5_PIXEL_RATE_CNTL__CRTC5_DISPOUT_FIFO_ERROR__SHIFT 0x0000000e
+#define CRTC5_PIXEL_RATE_CNTL__CRTC5_DROP_PIXEL_MASK 0x00000200L
+#define CRTC5_PIXEL_RATE_CNTL__CRTC5_DROP_PIXEL__SHIFT 0x00000009
+#define CRTC5_PIXEL_RATE_CNTL__CRTC5_PIXEL_RATE_SOURCE_MASK 0x00000003L
+#define CRTC5_PIXEL_RATE_CNTL__CRTC5_PIXEL_RATE_SOURCE__SHIFT 0x00000000
+#define CRTC5_PIXEL_RATE_CNTL__DP_DTO5_ENABLE_MASK 0x00000010L
+#define CRTC5_PIXEL_RATE_CNTL__DP_DTO5_ENABLE__SHIFT 0x00000004
+#define CRTC8_DATA__VCRTC_DATA_MASK 0x000000ffL
+#define CRTC8_DATA__VCRTC_DATA__SHIFT 0x00000000
+#define CRTC8_IDX__VCRTC_IDX_MASK 0x0000003fL
+#define CRTC8_IDX__VCRTC_IDX__SHIFT 0x00000000
+#define CRTC_ALLOW_STOP_OFF_V_CNT__CRTC_ALLOW_STOP_OFF_V_CNT_MASK 0x000000ffL
+#define CRTC_ALLOW_STOP_OFF_V_CNT__CRTC_ALLOW_STOP_OFF_V_CNT__SHIFT 0x00000000
+#define CRTC_ALLOW_STOP_OFF_V_CNT__CRTC_DISABLE_ALLOW_STOP_OFF_V_CNT_MASK 0x00010000L
+#define CRTC_ALLOW_STOP_OFF_V_CNT__CRTC_DISABLE_ALLOW_STOP_OFF_V_CNT__SHIFT 0x00000010
+#define CRTC_BLACK_COLOR__CRTC_BLACK_COLOR_B_CB_MASK 0x000003ffL
+#define CRTC_BLACK_COLOR__CRTC_BLACK_COLOR_B_CB__SHIFT 0x00000000
+#define CRTC_BLACK_COLOR__CRTC_BLACK_COLOR_G_Y_MASK 0x000ffc00L
+#define CRTC_BLACK_COLOR__CRTC_BLACK_COLOR_G_Y__SHIFT 0x0000000a
+#define CRTC_BLACK_COLOR__CRTC_BLACK_COLOR_R_CR_MASK 0x3ff00000L
+#define CRTC_BLACK_COLOR__CRTC_BLACK_COLOR_R_CR__SHIFT 0x00000014
+#define CRTC_BLANK_CONTROL__CRTC_BLANK_DATA_EN_MASK 0x00000100L
+#define CRTC_BLANK_CONTROL__CRTC_BLANK_DATA_EN__SHIFT 0x00000008
+#define CRTC_BLANK_CONTROL__CRTC_BLANK_DE_MODE_MASK 0x00010000L
+#define CRTC_BLANK_CONTROL__CRTC_BLANK_DE_MODE__SHIFT 0x00000010
+#define CRTC_BLANK_CONTROL__CRTC_CURRENT_BLANK_STATE_MASK 0x00000001L
+#define CRTC_BLANK_CONTROL__CRTC_CURRENT_BLANK_STATE__SHIFT 0x00000000
+#define CRTC_BLANK_DATA_COLOR__CRTC_BLANK_DATA_COLOR_BLUE_CB_MASK 0x000003ffL
+#define CRTC_BLANK_DATA_COLOR__CRTC_BLANK_DATA_COLOR_BLUE_CB__SHIFT 0x00000000
+#define CRTC_BLANK_DATA_COLOR__CRTC_BLANK_DATA_COLOR_GREEN_Y_MASK 0x000ffc00L
+#define CRTC_BLANK_DATA_COLOR__CRTC_BLANK_DATA_COLOR_GREEN_Y__SHIFT 0x0000000a
+#define CRTC_BLANK_DATA_COLOR__CRTC_BLANK_DATA_COLOR_RED_CR_MASK 0x3ff00000L
+#define CRTC_BLANK_DATA_COLOR__CRTC_BLANK_DATA_COLOR_RED_CR__SHIFT 0x00000014
+#define CRTC_CONTROL__CRTC_CURRENT_MASTER_EN_STATE_MASK 0x00010000L
+#define CRTC_CONTROL__CRTC_CURRENT_MASTER_EN_STATE__SHIFT 0x00000010
+#define CRTC_CONTROL__CRTC_DISABLE_POINT_CNTL_MASK 0x00000300L
+#define CRTC_CONTROL__CRTC_DISABLE_POINT_CNTL__SHIFT 0x00000008
+#define CRTC_CONTROL__CRTC_DISP_READ_REQUEST_DISABLE_MASK 0x01000000L
+#define CRTC_CONTROL__CRTC_DISP_READ_REQUEST_DISABLE__SHIFT 0x00000018
+#define CRTC_CONTROL__CRTC_FIELD_NUMBER_CNTL_MASK 0x00002000L
+#define CRTC_CONTROL__CRTC_FIELD_NUMBER_CNTL__SHIFT 0x0000000d
+#define CRTC_CONTROL__CRTC_HBLANK_EARLY_CONTROL_MASK 0x00700000L
+#define CRTC_CONTROL__CRTC_HBLANK_EARLY_CONTROL__SHIFT 0x00000014
+#define CRTC_CONTROL__CRTC_MASTER_EN_MASK 0x00000001L
+#define CRTC_CONTROL__CRTC_MASTER_EN__SHIFT 0x00000000
+#define CRTC_CONTROL__CRTC_SOF_PULL_EN_MASK 0x20000000L
+#define CRTC_CONTROL__CRTC_SOF_PULL_EN__SHIFT 0x0000001d
+#define CRTC_CONTROL__CRTC_START_POINT_CNTL_MASK 0x00001000L
+#define CRTC_CONTROL__CRTC_START_POINT_CNTL__SHIFT 0x0000000c
+#define CRTC_CONTROL__CRTC_SYNC_RESET_SEL_MASK 0x00000010L
+#define CRTC_CONTROL__CRTC_SYNC_RESET_SEL__SHIFT 0x00000004
+#define CRTC_COUNT_CONTROL__CRTC_HORZ_COUNT_BY2_EN_MASK 0x00000001L
+#define CRTC_COUNT_CONTROL__CRTC_HORZ_COUNT_BY2_EN__SHIFT 0x00000000
+#define CRTC_COUNT_CONTROL__CRTC_HORZ_REPETITION_COUNT_MASK 0x0000001eL
+#define CRTC_COUNT_CONTROL__CRTC_HORZ_REPETITION_COUNT__SHIFT 0x00000001
+#define CRTC_COUNT_RESET__CRTC_RESET_FRAME_COUNT_MASK 0x00000001L
+#define CRTC_COUNT_RESET__CRTC_RESET_FRAME_COUNT__SHIFT 0x00000000
+#define CRTC_DCFE_CLOCK_CONTROL__CRTC_DCFE_CLOCK_ENABLE_MASK 0x80000000L
+#define CRTC_DCFE_CLOCK_CONTROL__CRTC_DCFE_CLOCK_ENABLE__SHIFT 0x0000001f
+#define CRTC_DCFE_CLOCK_CONTROL__CRTC_DCFE_TEST_CLK_SEL_MASK 0x1f000000L
+#define CRTC_DCFE_CLOCK_CONTROL__CRTC_DCFE_TEST_CLK_SEL__SHIFT 0x00000018
+#define CRTC_DCFE_CLOCK_CONTROL__CRTC_DISPCLK_G_DCP_GATE_DISABLE_MASK 0x00000100L
+#define CRTC_DCFE_CLOCK_CONTROL__CRTC_DISPCLK_G_DCP_GATE_DISABLE__SHIFT 0x00000008
+#define CRTC_DCFE_CLOCK_CONTROL__CRTC_DISPCLK_G_SCL_GATE_DISABLE_MASK 0x00001000L
+#define CRTC_DCFE_CLOCK_CONTROL__CRTC_DISPCLK_G_SCL_GATE_DISABLE__SHIFT 0x0000000c
+#define CRTC_DCFE_CLOCK_CONTROL__CRTC_DISPCLK_R_DCFE_GATE_DISABLE_MASK 0x00000010L
+#define CRTC_DCFE_CLOCK_CONTROL__CRTC_DISPCLK_R_DCFE_GATE_DISABLE__SHIFT 0x00000004
+#define CRTC_DOUBLE_BUFFER_CONTROL__CRTC_BLANK_DATA_DOUBLE_BUFFER_EN_MASK 0x00010000L
+#define CRTC_DOUBLE_BUFFER_CONTROL__CRTC_BLANK_DATA_DOUBLE_BUFFER_EN__SHIFT 0x00000010
+#define CRTC_DOUBLE_BUFFER_CONTROL__CRTC_UPDATE_INSTANTLY_MASK 0x00000100L
+#define CRTC_DOUBLE_BUFFER_CONTROL__CRTC_UPDATE_INSTANTLY__SHIFT 0x00000008
+#define CRTC_DOUBLE_BUFFER_CONTROL__CRTC_UPDATE_PENDING_MASK 0x00000001L
+#define CRTC_DOUBLE_BUFFER_CONTROL__CRTC_UPDATE_PENDING__SHIFT 0x00000000
+#define CRTC_DTMTEST_CNTL__CRTC_DTMTEST_CLK_DIV_MASK 0x0000001eL
+#define CRTC_DTMTEST_CNTL__CRTC_DTMTEST_CLK_DIV__SHIFT 0x00000001
+#define CRTC_DTMTEST_CNTL__CRTC_DTMTEST_CRTC_EN_MASK 0x00000001L
+#define CRTC_DTMTEST_CNTL__CRTC_DTMTEST_CRTC_EN__SHIFT 0x00000000
+#define CRTC_DTMTEST_STATUS_POSITION__CRTC_DTMTEST_HORZ_COUNT_MASK 0x1fff0000L
+#define CRTC_DTMTEST_STATUS_POSITION__CRTC_DTMTEST_HORZ_COUNT__SHIFT 0x00000010
+#define CRTC_DTMTEST_STATUS_POSITION__CRTC_DTMTEST_VERT_COUNT_MASK 0x00001fffL
+#define CRTC_DTMTEST_STATUS_POSITION__CRTC_DTMTEST_VERT_COUNT__SHIFT 0x00000000
+#define CRTC_FLOW_CONTROL__CRTC_FLOW_CONTROL_GRANULARITY_MASK 0x00010000L
+#define CRTC_FLOW_CONTROL__CRTC_FLOW_CONTROL_GRANULARITY__SHIFT 0x00000010
+#define CRTC_FLOW_CONTROL__CRTC_FLOW_CONTROL_INPUT_STATUS_MASK 0x01000000L
+#define CRTC_FLOW_CONTROL__CRTC_FLOW_CONTROL_INPUT_STATUS__SHIFT 0x00000018
+#define CRTC_FLOW_CONTROL__CRTC_FLOW_CONTROL_POLARITY_MASK 0x00000100L
+#define CRTC_FLOW_CONTROL__CRTC_FLOW_CONTROL_POLARITY__SHIFT 0x00000008
+#define CRTC_FLOW_CONTROL__CRTC_FLOW_CONTROL_SOURCE_SELECT_MASK 0x0000001fL
+#define CRTC_FLOW_CONTROL__CRTC_FLOW_CONTROL_SOURCE_SELECT__SHIFT 0x00000000
+#define CRTC_FORCE_COUNT_NOW_CNTL__CRTC_FORCE_COUNT_NOW_CHECK_MASK 0x00000010L
+#define CRTC_FORCE_COUNT_NOW_CNTL__CRTC_FORCE_COUNT_NOW_CHECK__SHIFT 0x00000004
+#define CRTC_FORCE_COUNT_NOW_CNTL__CRTC_FORCE_COUNT_NOW_CLEAR_MASK 0x01000000L
+#define CRTC_FORCE_COUNT_NOW_CNTL__CRTC_FORCE_COUNT_NOW_CLEAR__SHIFT 0x00000018
+#define CRTC_FORCE_COUNT_NOW_CNTL__CRTC_FORCE_COUNT_NOW_MODE_MASK 0x00000003L
+#define CRTC_FORCE_COUNT_NOW_CNTL__CRTC_FORCE_COUNT_NOW_MODE__SHIFT 0x00000000
+#define CRTC_FORCE_COUNT_NOW_CNTL__CRTC_FORCE_COUNT_NOW_OCCURRED_MASK 0x00010000L
+#define CRTC_FORCE_COUNT_NOW_CNTL__CRTC_FORCE_COUNT_NOW_OCCURRED__SHIFT 0x00000010
+#define CRTC_FORCE_COUNT_NOW_CNTL__CRTC_FORCE_COUNT_NOW_TRIG_SEL_MASK 0x00000100L
+#define CRTC_FORCE_COUNT_NOW_CNTL__CRTC_FORCE_COUNT_NOW_TRIG_SEL__SHIFT 0x00000008
+#define CRTC_GSL_CONTROL__CRTC_GSL_CHECK_ALL_FIELDS_MASK 0x10000000L
+#define CRTC_GSL_CONTROL__CRTC_GSL_CHECK_ALL_FIELDS__SHIFT 0x0000001c
+#define CRTC_GSL_CONTROL__CRTC_GSL_CHECK_LINE_NUM_MASK 0x00001fffL
+#define CRTC_GSL_CONTROL__CRTC_GSL_CHECK_LINE_NUM__SHIFT 0x00000000
+#define CRTC_GSL_CONTROL__CRTC_GSL_FORCE_DELAY_MASK 0x001f0000L
+#define CRTC_GSL_CONTROL__CRTC_GSL_FORCE_DELAY__SHIFT 0x00000010
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_CLEAR_MASK 0x00080000L
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_CLEAR__SHIFT 0x00000013
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_DELAY_MASK 0x0000ff00L
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_DELAY__SHIFT 0x00000008
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_LIMIT_MASK 0x000000ffL
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_LIMIT__SHIFT 0x00000000
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_MASK 0xff000000L
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_MASTER_FASTER_MASK 0x00800000L
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_MASTER_FASTER__SHIFT 0x00000017
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_MODE_MASK 0x00060000L
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_MODE__SHIFT 0x00000011
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_OCCURRED_MASK 0x00100000L
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_OCCURRED__SHIFT 0x00000014
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP__SHIFT 0x00000018
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_SOURCE_SEL_MASK 0x00010000L
+#define CRTC_GSL_VSYNC_GAP__CRTC_GSL_VSYNC_GAP_SOURCE_SEL__SHIFT 0x00000010
+#define CRTC_GSL_WINDOW__CRTC_GSL_WINDOW_END_MASK 0x1fff0000L
+#define CRTC_GSL_WINDOW__CRTC_GSL_WINDOW_END__SHIFT 0x00000010
+#define CRTC_GSL_WINDOW__CRTC_GSL_WINDOW_START_MASK 0x00001fffL
+#define CRTC_GSL_WINDOW__CRTC_GSL_WINDOW_START__SHIFT 0x00000000
+#define CRTC_H_BLANK_EARLY_NUM__CRTC_H_BLANK_EARLY_NUM_DIS_MASK 0x00010000L
+#define CRTC_H_BLANK_EARLY_NUM__CRTC_H_BLANK_EARLY_NUM_DIS__SHIFT 0x00000010
+#define CRTC_H_BLANK_EARLY_NUM__CRTC_H_BLANK_EARLY_NUM_MASK 0x000003ffL
+#define CRTC_H_BLANK_EARLY_NUM__CRTC_H_BLANK_EARLY_NUM__SHIFT 0x00000000
+#define CRTC_H_BLANK_START_END__CRTC_H_BLANK_END_MASK 0x1fff0000L
+#define CRTC_H_BLANK_START_END__CRTC_H_BLANK_END__SHIFT 0x00000010
+#define CRTC_H_BLANK_START_END__CRTC_H_BLANK_START_MASK 0x00001fffL
+#define CRTC_H_BLANK_START_END__CRTC_H_BLANK_START__SHIFT 0x00000000
+#define CRTC_H_SYNC_A_CNTL__CRTC_COMP_SYNC_A_EN_MASK 0x00010000L
+#define CRTC_H_SYNC_A_CNTL__CRTC_COMP_SYNC_A_EN__SHIFT 0x00000010
+#define CRTC_H_SYNC_A_CNTL__CRTC_H_SYNC_A_CUTOFF_MASK 0x00020000L
+#define CRTC_H_SYNC_A_CNTL__CRTC_H_SYNC_A_CUTOFF__SHIFT 0x00000011
+#define CRTC_H_SYNC_A_CNTL__CRTC_H_SYNC_A_POL_MASK 0x00000001L
+#define CRTC_H_SYNC_A_CNTL__CRTC_H_SYNC_A_POL__SHIFT 0x00000000
+#define CRTC_H_SYNC_A__CRTC_H_SYNC_A_END_MASK 0x1fff0000L
+#define CRTC_H_SYNC_A__CRTC_H_SYNC_A_END__SHIFT 0x00000010
+#define CRTC_H_SYNC_A__CRTC_H_SYNC_A_START_MASK 0x00001fffL
+#define CRTC_H_SYNC_A__CRTC_H_SYNC_A_START__SHIFT 0x00000000
+#define CRTC_H_SYNC_B_CNTL__CRTC_COMP_SYNC_B_EN_MASK 0x00010000L
+#define CRTC_H_SYNC_B_CNTL__CRTC_COMP_SYNC_B_EN__SHIFT 0x00000010
+#define CRTC_H_SYNC_B_CNTL__CRTC_H_SYNC_B_CUTOFF_MASK 0x00020000L
+#define CRTC_H_SYNC_B_CNTL__CRTC_H_SYNC_B_CUTOFF__SHIFT 0x00000011
+#define CRTC_H_SYNC_B_CNTL__CRTC_H_SYNC_B_POL_MASK 0x00000001L
+#define CRTC_H_SYNC_B_CNTL__CRTC_H_SYNC_B_POL__SHIFT 0x00000000
+#define CRTC_H_SYNC_B__CRTC_H_SYNC_B_END_MASK 0x1fff0000L
+#define CRTC_H_SYNC_B__CRTC_H_SYNC_B_END__SHIFT 0x00000010
+#define CRTC_H_SYNC_B__CRTC_H_SYNC_B_START_MASK 0x00001fffL
+#define CRTC_H_SYNC_B__CRTC_H_SYNC_B_START__SHIFT 0x00000000
+#define CRTC_H_TOTAL__CRTC_H_TOTAL_MASK 0x00001fffL
+#define CRTC_H_TOTAL__CRTC_H_TOTAL__SHIFT 0x00000000
+#define CRTC_INTERLACE_CONTROL__CRTC_INTERLACE_ENABLE_MASK 0x00000001L
+#define CRTC_INTERLACE_CONTROL__CRTC_INTERLACE_ENABLE__SHIFT 0x00000000
+#define CRTC_INTERLACE_CONTROL__CRTC_INTERLACE_FORCE_NEXT_FIELD_MASK 0x00030000L
+#define CRTC_INTERLACE_CONTROL__CRTC_INTERLACE_FORCE_NEXT_FIELD__SHIFT 0x00000010
+#define CRTC_INTERLACE_STATUS__CRTC_INTERLACE_CURRENT_FIELD_MASK 0x00000001L
+#define CRTC_INTERLACE_STATUS__CRTC_INTERLACE_CURRENT_FIELD__SHIFT 0x00000000
+#define CRTC_INTERLACE_STATUS__CRTC_INTERLACE_NEXT_FIELD_MASK 0x00000002L
+#define CRTC_INTERLACE_STATUS__CRTC_INTERLACE_NEXT_FIELD__SHIFT 0x00000001
+#define CRTC_INTERRUPT_CONTROL__CRTC_FORCE_COUNT_NOW_INT_MSK_MASK 0x00000100L
+#define CRTC_INTERRUPT_CONTROL__CRTC_FORCE_COUNT_NOW_INT_MSK__SHIFT 0x00000008
+#define CRTC_INTERRUPT_CONTROL__CRTC_FORCE_COUNT_NOW_INT_TYPE_MASK 0x00000200L
+#define CRTC_INTERRUPT_CONTROL__CRTC_FORCE_COUNT_NOW_INT_TYPE__SHIFT 0x00000009
+#define CRTC_INTERRUPT_CONTROL__CRTC_FORCE_VSYNC_NEXT_LINE_INT_MSK_MASK 0x00010000L
+#define CRTC_INTERRUPT_CONTROL__CRTC_FORCE_VSYNC_NEXT_LINE_INT_MSK__SHIFT 0x00000010
+#define CRTC_INTERRUPT_CONTROL__CRTC_FORCE_VSYNC_NEXT_LINE_INT_TYPE_MASK 0x00020000L
+#define CRTC_INTERRUPT_CONTROL__CRTC_FORCE_VSYNC_NEXT_LINE_INT_TYPE__SHIFT 0x00000011
+#define CRTC_INTERRUPT_CONTROL__CRTC_GSL_VSYNC_GAP_INT_MSK_MASK 0x40000000L
+#define CRTC_INTERRUPT_CONTROL__CRTC_GSL_VSYNC_GAP_INT_MSK__SHIFT 0x0000001e
+#define CRTC_INTERRUPT_CONTROL__CRTC_GSL_VSYNC_GAP_INT_TYPE_MASK 0x80000000L
+#define CRTC_INTERRUPT_CONTROL__CRTC_GSL_VSYNC_GAP_INT_TYPE__SHIFT 0x0000001f
+#define CRTC_INTERRUPT_CONTROL__CRTC_SNAPSHOT_INT_MSK_MASK 0x00000001L
+#define CRTC_INTERRUPT_CONTROL__CRTC_SNAPSHOT_INT_MSK__SHIFT 0x00000000
+#define CRTC_INTERRUPT_CONTROL__CRTC_SNAPSHOT_INT_TYPE_MASK 0x00000002L
+#define CRTC_INTERRUPT_CONTROL__CRTC_SNAPSHOT_INT_TYPE__SHIFT 0x00000001
+#define CRTC_INTERRUPT_CONTROL__CRTC_TRIGA_INT_MSK_MASK 0x01000000L
+#define CRTC_INTERRUPT_CONTROL__CRTC_TRIGA_INT_MSK__SHIFT 0x00000018
+#define CRTC_INTERRUPT_CONTROL__CRTC_TRIGA_INT_TYPE_MASK 0x04000000L
+#define CRTC_INTERRUPT_CONTROL__CRTC_TRIGA_INT_TYPE__SHIFT 0x0000001a
+#define CRTC_INTERRUPT_CONTROL__CRTC_TRIGB_INT_MSK_MASK 0x02000000L
+#define CRTC_INTERRUPT_CONTROL__CRTC_TRIGB_INT_MSK__SHIFT 0x00000019
+#define CRTC_INTERRUPT_CONTROL__CRTC_TRIGB_INT_TYPE_MASK 0x08000000L
+#define CRTC_INTERRUPT_CONTROL__CRTC_TRIGB_INT_TYPE__SHIFT 0x0000001b
+#define CRTC_INTERRUPT_CONTROL__CRTC_VSYNC_NOM_INT_MSK_MASK 0x10000000L
+#define CRTC_INTERRUPT_CONTROL__CRTC_VSYNC_NOM_INT_MSK__SHIFT 0x0000001c
+#define CRTC_INTERRUPT_CONTROL__CRTC_VSYNC_NOM_INT_TYPE_MASK 0x20000000L
+#define CRTC_INTERRUPT_CONTROL__CRTC_VSYNC_NOM_INT_TYPE__SHIFT 0x0000001d
+#define CRTC_INTERRUPT_CONTROL__CRTC_V_UPDATE_INT_MSK_MASK 0x00000010L
+#define CRTC_INTERRUPT_CONTROL__CRTC_V_UPDATE_INT_MSK__SHIFT 0x00000004
+#define CRTC_INTERRUPT_CONTROL__CRTC_V_UPDATE_INT_TYPE_MASK 0x00000020L
+#define CRTC_INTERRUPT_CONTROL__CRTC_V_UPDATE_INT_TYPE__SHIFT 0x00000005
+#define CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE__CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE_MASK 0x00000001L
+#define CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE__CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE__SHIFT 0x00000000
+#define CRTC_MASTER_EN__CRTC_MASTER_EN_MASK 0x00000001L
+#define CRTC_MASTER_EN__CRTC_MASTER_EN__SHIFT 0x00000000
+#define CRTC_MVP_INBAND_CNTL_INSERT__CRTC_MVP_INBAND_CNTL_CHAR_INSERT_MASK 0xffffff00L
+#define CRTC_MVP_INBAND_CNTL_INSERT__CRTC_MVP_INBAND_CNTL_CHAR_INSERT__SHIFT 0x00000008
+#define CRTC_MVP_INBAND_CNTL_INSERT__CRTC_MVP_INBAND_OUT_MODE_MASK 0x00000003L
+#define CRTC_MVP_INBAND_CNTL_INSERT__CRTC_MVP_INBAND_OUT_MODE__SHIFT 0x00000000
+#define CRTC_MVP_INBAND_CNTL_INSERT_TIMER__CRTC_MVP_INBAND_CNTL_CHAR_INSERT_TIMER_MASK 0x000000ffL
+#define CRTC_MVP_INBAND_CNTL_INSERT_TIMER__CRTC_MVP_INBAND_CNTL_CHAR_INSERT_TIMER__SHIFT 0x00000000
+#define CRTC_MVP_STATUS__CRTC_AFR_HSYNC_SWITCH_DONE_CLEAR_MASK 0x00100000L
+#define CRTC_MVP_STATUS__CRTC_AFR_HSYNC_SWITCH_DONE_CLEAR__SHIFT 0x00000014
+#define CRTC_MVP_STATUS__CRTC_AFR_HSYNC_SWITCH_DONE_OCCURRED_MASK 0x00000010L
+#define CRTC_MVP_STATUS__CRTC_AFR_HSYNC_SWITCH_DONE_OCCURRED__SHIFT 0x00000004
+#define CRTC_MVP_STATUS__CRTC_FLIP_NOW_CLEAR_MASK 0x00010000L
+#define CRTC_MVP_STATUS__CRTC_FLIP_NOW_CLEAR__SHIFT 0x00000010
+#define CRTC_MVP_STATUS__CRTC_FLIP_NOW_OCCURRED_MASK 0x00000001L
+#define CRTC_MVP_STATUS__CRTC_FLIP_NOW_OCCURRED__SHIFT 0x00000000
+#define CRTC_NOM_VERT_POSITION__CRTC_VERT_COUNT_NOM_MASK 0x00001fffL
+#define CRTC_NOM_VERT_POSITION__CRTC_VERT_COUNT_NOM__SHIFT 0x00000000
+#define CRTC_OVERSCAN_COLOR__CRTC_OVERSCAN_COLOR_BLUE_MASK 0x000003ffL
+#define CRTC_OVERSCAN_COLOR__CRTC_OVERSCAN_COLOR_BLUE__SHIFT 0x00000000
+#define CRTC_OVERSCAN_COLOR__CRTC_OVERSCAN_COLOR_GREEN_MASK 0x000ffc00L
+#define CRTC_OVERSCAN_COLOR__CRTC_OVERSCAN_COLOR_GREEN__SHIFT 0x0000000a
+#define CRTC_OVERSCAN_COLOR__CRTC_OVERSCAN_COLOR_RED_MASK 0x3ff00000L
+#define CRTC_OVERSCAN_COLOR__CRTC_OVERSCAN_COLOR_RED__SHIFT 0x00000014
+#define CRTC_SNAPSHOT_CONTROL__CRTC_AUTO_SNAPSHOT_TRIG_SEL_MASK 0x00000003L
+#define CRTC_SNAPSHOT_CONTROL__CRTC_AUTO_SNAPSHOT_TRIG_SEL__SHIFT 0x00000000
+#define CRTC_SNAPSHOT_FRAME__CRTC_SNAPSHOT_FRAME_COUNT_MASK 0x00ffffffL
+#define CRTC_SNAPSHOT_FRAME__CRTC_SNAPSHOT_FRAME_COUNT__SHIFT 0x00000000
+#define CRTC_SNAPSHOT_POSITION__CRTC_SNAPSHOT_HORZ_COUNT_MASK 0x1fff0000L
+#define CRTC_SNAPSHOT_POSITION__CRTC_SNAPSHOT_HORZ_COUNT__SHIFT 0x00000010
+#define CRTC_SNAPSHOT_POSITION__CRTC_SNAPSHOT_VERT_COUNT_MASK 0x00001fffL
+#define CRTC_SNAPSHOT_POSITION__CRTC_SNAPSHOT_VERT_COUNT__SHIFT 0x00000000
+#define CRTC_SNAPSHOT_STATUS__CRTC_SNAPSHOT_CLEAR_MASK 0x00000002L
+#define CRTC_SNAPSHOT_STATUS__CRTC_SNAPSHOT_CLEAR__SHIFT 0x00000001
+#define CRTC_SNAPSHOT_STATUS__CRTC_SNAPSHOT_MANUAL_TRIGGER_MASK 0x00000004L
+#define CRTC_SNAPSHOT_STATUS__CRTC_SNAPSHOT_MANUAL_TRIGGER__SHIFT 0x00000002
+#define CRTC_SNAPSHOT_STATUS__CRTC_SNAPSHOT_OCCURRED_MASK 0x00000001L
+#define CRTC_SNAPSHOT_STATUS__CRTC_SNAPSHOT_OCCURRED__SHIFT 0x00000000
+#define CRTC_START_LINE_CONTROL__CRTC_ADVANCED_START_LINE_POSITION_MASK 0x000f0000L
+#define CRTC_START_LINE_CONTROL__CRTC_ADVANCED_START_LINE_POSITION__SHIFT 0x00000010
+#define CRTC_START_LINE_CONTROL__CRTC_INTERLACE_START_LINE_EARLY_MASK 0x00000100L
+#define CRTC_START_LINE_CONTROL__CRTC_INTERLACE_START_LINE_EARLY__SHIFT 0x00000008
+#define CRTC_START_LINE_CONTROL__CRTC_PROGRESSIVE_START_LINE_EARLY_MASK 0x00000001L
+#define CRTC_START_LINE_CONTROL__CRTC_PROGRESSIVE_START_LINE_EARLY__SHIFT 0x00000000
+#define CRTC_STATUS__CRTC_H_ACTIVE_DISP_MASK 0x00020000L
+#define CRTC_STATUS__CRTC_H_ACTIVE_DISP__SHIFT 0x00000011
+#define CRTC_STATUS__CRTC_H_BLANK_MASK 0x00010000L
+#define CRTC_STATUS__CRTC_H_BLANK__SHIFT 0x00000010
+#define CRTC_STATUS__CRTC_H_SYNC_A_MASK 0x00040000L
+#define CRTC_STATUS__CRTC_H_SYNC_A__SHIFT 0x00000012
+#define CRTC_STATUS__CRTC_V_ACTIVE_DISP_MASK 0x00000002L
+#define CRTC_STATUS__CRTC_V_ACTIVE_DISP__SHIFT 0x00000001
+#define CRTC_STATUS__CRTC_V_BLANK_3D_STRUCTURE_MASK 0x00000020L
+#define CRTC_STATUS__CRTC_V_BLANK_3D_STRUCTURE__SHIFT 0x00000005
+#define CRTC_STATUS__CRTC_V_BLANK_MASK 0x00000001L
+#define CRTC_STATUS__CRTC_V_BLANK__SHIFT 0x00000000
+#define CRTC_STATUS__CRTC_V_START_LINE_MASK 0x00000010L
+#define CRTC_STATUS__CRTC_V_START_LINE__SHIFT 0x00000004
+#define CRTC_STATUS__CRTC_V_SYNC_A_MASK 0x00000004L
+#define CRTC_STATUS__CRTC_V_SYNC_A__SHIFT 0x00000002
+#define CRTC_STATUS__CRTC_V_UPDATE_MASK 0x00000008L
+#define CRTC_STATUS__CRTC_V_UPDATE__SHIFT 0x00000003
+#define CRTC_STATUS_FRAME_COUNT__CRTC_FRAME_COUNT_MASK 0x00ffffffL
+#define CRTC_STATUS_FRAME_COUNT__CRTC_FRAME_COUNT__SHIFT 0x00000000
+#define CRTC_STATUS_HV_COUNT__CRTC_HV_COUNT_MASK 0x1fffffffL
+#define CRTC_STATUS_HV_COUNT__CRTC_HV_COUNT__SHIFT 0x00000000
+#define CRTC_STATUS_POSITION__CRTC_HORZ_COUNT_MASK 0x1fff0000L
+#define CRTC_STATUS_POSITION__CRTC_HORZ_COUNT__SHIFT 0x00000010
+#define CRTC_STATUS_POSITION__CRTC_VERT_COUNT_MASK 0x00001fffL
+#define CRTC_STATUS_POSITION__CRTC_VERT_COUNT__SHIFT 0x00000000
+#define CRTC_STATUS_VF_COUNT__CRTC_VF_COUNT_MASK 0x1fffffffL
+#define CRTC_STATUS_VF_COUNT__CRTC_VF_COUNT__SHIFT 0x00000000
+#define CRTC_STEREO_CONTROL__CRTC_STEREO_EN_MASK 0x01000000L
+#define CRTC_STEREO_CONTROL__CRTC_STEREO_EN__SHIFT 0x00000018
+#define CRTC_STEREO_CONTROL__CRTC_STEREO_SYNC_OUTPUT_LINE_NUM_MASK 0x00001fffL
+#define CRTC_STEREO_CONTROL__CRTC_STEREO_SYNC_OUTPUT_LINE_NUM__SHIFT 0x00000000
+#define CRTC_STEREO_CONTROL__CRTC_STEREO_SYNC_OUTPUT_POLARITY_MASK 0x00008000L
+#define CRTC_STEREO_CONTROL__CRTC_STEREO_SYNC_OUTPUT_POLARITY__SHIFT 0x0000000f
+#define CRTC_STEREO_CONTROL__CRTC_STEREO_SYNC_SELECT_POLARITY_MASK 0x00010000L
+#define CRTC_STEREO_CONTROL__CRTC_STEREO_SYNC_SELECT_POLARITY__SHIFT 0x00000010
+#define CRTC_STEREO_FORCE_NEXT_EYE__CRTC_STEREO_FORCE_NEXT_EYE_MASK 0x00000003L
+#define CRTC_STEREO_FORCE_NEXT_EYE__CRTC_STEREO_FORCE_NEXT_EYE__SHIFT 0x00000000
+#define CRTC_STEREO_STATUS__CRTC_STEREO_CURRENT_EYE_MASK 0x00000001L
+#define CRTC_STEREO_STATUS__CRTC_STEREO_CURRENT_EYE__SHIFT 0x00000000
+#define CRTC_STEREO_STATUS__CRTC_STEREO_FORCE_NEXT_EYE_PENDING_MASK 0x03000000L
+#define CRTC_STEREO_STATUS__CRTC_STEREO_FORCE_NEXT_EYE_PENDING__SHIFT 0x00000018
+#define CRTC_STEREO_STATUS__CRTC_STEREO_SYNC_OUTPUT_MASK 0x00000100L
+#define CRTC_STEREO_STATUS__CRTC_STEREO_SYNC_OUTPUT__SHIFT 0x00000008
+#define CRTC_STEREO_STATUS__CRTC_STEREO_SYNC_SELECT_MASK 0x00010000L
+#define CRTC_STEREO_STATUS__CRTC_STEREO_SYNC_SELECT__SHIFT 0x00000010
+#define CRTC_TEST_DEBUG_DATA__CRTC_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define CRTC_TEST_DEBUG_DATA__CRTC_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define CRTC_TEST_DEBUG_INDEX__CRTC_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define CRTC_TEST_DEBUG_INDEX__CRTC_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define CRTC_TEST_DEBUG_INDEX__CRTC_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define CRTC_TEST_DEBUG_INDEX__CRTC_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define CRTC_TEST_PATTERN_COLOR__CRTC_TEST_PATTERN_DATA_MASK 0x0000ffffL
+#define CRTC_TEST_PATTERN_COLOR__CRTC_TEST_PATTERN_DATA__SHIFT 0x00000000
+#define CRTC_TEST_PATTERN_COLOR__CRTC_TEST_PATTERN_MASK_MASK 0x003f0000L
+#define CRTC_TEST_PATTERN_COLOR__CRTC_TEST_PATTERN_MASK__SHIFT 0x00000010
+#define CRTC_TEST_PATTERN_CONTROL__CRTC_TEST_PATTERN_COLOR_FORMAT_MASK 0xff000000L
+#define CRTC_TEST_PATTERN_CONTROL__CRTC_TEST_PATTERN_COLOR_FORMAT__SHIFT 0x00000018
+#define CRTC_TEST_PATTERN_CONTROL__CRTC_TEST_PATTERN_DYNAMIC_RANGE_MASK 0x00010000L
+#define CRTC_TEST_PATTERN_CONTROL__CRTC_TEST_PATTERN_DYNAMIC_RANGE__SHIFT 0x00000010
+#define CRTC_TEST_PATTERN_CONTROL__CRTC_TEST_PATTERN_EN_MASK 0x00000001L
+#define CRTC_TEST_PATTERN_CONTROL__CRTC_TEST_PATTERN_EN__SHIFT 0x00000000
+#define CRTC_TEST_PATTERN_CONTROL__CRTC_TEST_PATTERN_MODE_MASK 0x00000700L
+#define CRTC_TEST_PATTERN_CONTROL__CRTC_TEST_PATTERN_MODE__SHIFT 0x00000008
+#define CRTC_TEST_PATTERN_PARAMETERS__CRTC_TEST_PATTERN_HRES_MASK 0x0000f000L
+#define CRTC_TEST_PATTERN_PARAMETERS__CRTC_TEST_PATTERN_HRES__SHIFT 0x0000000c
+#define CRTC_TEST_PATTERN_PARAMETERS__CRTC_TEST_PATTERN_INC0_MASK 0x0000000fL
+#define CRTC_TEST_PATTERN_PARAMETERS__CRTC_TEST_PATTERN_INC0__SHIFT 0x00000000
+#define CRTC_TEST_PATTERN_PARAMETERS__CRTC_TEST_PATTERN_INC1_MASK 0x000000f0L
+#define CRTC_TEST_PATTERN_PARAMETERS__CRTC_TEST_PATTERN_INC1__SHIFT 0x00000004
+#define CRTC_TEST_PATTERN_PARAMETERS__CRTC_TEST_PATTERN_RAMP0_OFFSET_MASK 0xffff0000L
+#define CRTC_TEST_PATTERN_PARAMETERS__CRTC_TEST_PATTERN_RAMP0_OFFSET__SHIFT 0x00000010
+#define CRTC_TEST_PATTERN_PARAMETERS__CRTC_TEST_PATTERN_VRES_MASK 0x00000f00L
+#define CRTC_TEST_PATTERN_PARAMETERS__CRTC_TEST_PATTERN_VRES__SHIFT 0x00000008
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_CLEAR_MASK 0x80000000L
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_CLEAR__SHIFT 0x0000001f
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_DELAY_MASK 0x1f000000L
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_DELAY__SHIFT 0x00000018
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_FALLING_EDGE_DETECT_CNTL_MASK 0x00030000L
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_FALLING_EDGE_DETECT_CNTL__SHIFT 0x00000010
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_FREQUENCY_SELECT_MASK 0x00300000L
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_FREQUENCY_SELECT__SHIFT 0x00000014
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_INPUT_STATUS_MASK 0x00000200L
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_INPUT_STATUS__SHIFT 0x00000009
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_OCCURRED_MASK 0x00000800L
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_OCCURRED__SHIFT 0x0000000b
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_POLARITY_SELECT_MASK 0x000000e0L
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_POLARITY_SELECT__SHIFT 0x00000005
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_POLARITY_STATUS_MASK 0x00000400L
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_POLARITY_STATUS__SHIFT 0x0000000a
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_RESYNC_BYPASS_EN_MASK 0x00000100L
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_RESYNC_BYPASS_EN__SHIFT 0x00000008
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_RISING_EDGE_DETECT_CNTL_MASK 0x00003000L
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_RISING_EDGE_DETECT_CNTL__SHIFT 0x0000000c
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_SOURCE_SELECT_MASK 0x0000001fL
+#define CRTC_TRIGA_CNTL__CRTC_TRIGA_SOURCE_SELECT__SHIFT 0x00000000
+#define CRTC_TRIGA_MANUAL_TRIG__CRTC_TRIGA_MANUAL_TRIG_MASK 0x00000001L
+#define CRTC_TRIGA_MANUAL_TRIG__CRTC_TRIGA_MANUAL_TRIG__SHIFT 0x00000000
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_CLEAR_MASK 0x80000000L
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_CLEAR__SHIFT 0x0000001f
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_DELAY_MASK 0x1f000000L
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_DELAY__SHIFT 0x00000018
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_FALLING_EDGE_DETECT_CNTL_MASK 0x00030000L
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_FALLING_EDGE_DETECT_CNTL__SHIFT 0x00000010
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_FREQUENCY_SELECT_MASK 0x00300000L
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_FREQUENCY_SELECT__SHIFT 0x00000014
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_INPUT_STATUS_MASK 0x00000200L
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_INPUT_STATUS__SHIFT 0x00000009
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_OCCURRED_MASK 0x00000800L
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_OCCURRED__SHIFT 0x0000000b
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_POLARITY_SELECT_MASK 0x000000e0L
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_POLARITY_SELECT__SHIFT 0x00000005
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_POLARITY_STATUS_MASK 0x00000400L
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_POLARITY_STATUS__SHIFT 0x0000000a
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_RESYNC_BYPASS_EN_MASK 0x00000100L
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_RESYNC_BYPASS_EN__SHIFT 0x00000008
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_RISING_EDGE_DETECT_CNTL_MASK 0x00003000L
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_RISING_EDGE_DETECT_CNTL__SHIFT 0x0000000c
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_SOURCE_SELECT_MASK 0x0000001fL
+#define CRTC_TRIGB_CNTL__CRTC_TRIGB_SOURCE_SELECT__SHIFT 0x00000000
+#define CRTC_TRIGB_MANUAL_TRIG__CRTC_TRIGB_MANUAL_TRIG_MASK 0x00000001L
+#define CRTC_TRIGB_MANUAL_TRIG__CRTC_TRIGB_MANUAL_TRIG__SHIFT 0x00000000
+#define CRTC_UPDATE_LOCK__CRTC_UPDATE_LOCK_MASK 0x00000001L
+#define CRTC_UPDATE_LOCK__CRTC_UPDATE_LOCK__SHIFT 0x00000000
+#define CRTC_VBI_END__CRTC_VBI_H_END_MASK 0x1fff0000L
+#define CRTC_VBI_END__CRTC_VBI_H_END__SHIFT 0x00000010
+#define CRTC_VBI_END__CRTC_VBI_V_END_MASK 0x00001fffL
+#define CRTC_VBI_END__CRTC_VBI_V_END__SHIFT 0x00000000
+#define CRTC_V_BLANK_START_END__CRTC_V_BLANK_END_MASK 0x1fff0000L
+#define CRTC_V_BLANK_START_END__CRTC_V_BLANK_END__SHIFT 0x00000010
+#define CRTC_V_BLANK_START_END__CRTC_V_BLANK_START_MASK 0x00001fffL
+#define CRTC_V_BLANK_START_END__CRTC_V_BLANK_START__SHIFT 0x00000000
+#define CRTC_VERT_SYNC_CONTROL__CRTC_AUTO_FORCE_VSYNC_MODE_MASK 0x00030000L
+#define CRTC_VERT_SYNC_CONTROL__CRTC_AUTO_FORCE_VSYNC_MODE__SHIFT 0x00000010
+#define CRTC_VERT_SYNC_CONTROL__CRTC_FORCE_VSYNC_NEXT_LINE_CLEAR_MASK 0x00000100L
+#define CRTC_VERT_SYNC_CONTROL__CRTC_FORCE_VSYNC_NEXT_LINE_CLEAR__SHIFT 0x00000008
+#define CRTC_VERT_SYNC_CONTROL__CRTC_FORCE_VSYNC_NEXT_LINE_OCCURRED_MASK 0x00000001L
+#define CRTC_VERT_SYNC_CONTROL__CRTC_FORCE_VSYNC_NEXT_LINE_OCCURRED__SHIFT 0x00000000
+#define CRTC_VGA_PARAMETER_CAPTURE_MODE__CRTC_VGA_PARAMETER_CAPTURE_MODE_MASK 0x00000001L
+#define CRTC_VGA_PARAMETER_CAPTURE_MODE__CRTC_VGA_PARAMETER_CAPTURE_MODE__SHIFT 0x00000000
+#define CRTC_V_SYNC_A_CNTL__CRTC_V_SYNC_A_POL_MASK 0x00000001L
+#define CRTC_V_SYNC_A_CNTL__CRTC_V_SYNC_A_POL__SHIFT 0x00000000
+#define CRTC_V_SYNC_A__CRTC_V_SYNC_A_END_MASK 0x1fff0000L
+#define CRTC_V_SYNC_A__CRTC_V_SYNC_A_END__SHIFT 0x00000010
+#define CRTC_V_SYNC_A__CRTC_V_SYNC_A_START_MASK 0x00001fffL
+#define CRTC_V_SYNC_A__CRTC_V_SYNC_A_START__SHIFT 0x00000000
+#define CRTC_V_SYNC_B_CNTL__CRTC_V_SYNC_B_POL_MASK 0x00000001L
+#define CRTC_V_SYNC_B_CNTL__CRTC_V_SYNC_B_POL__SHIFT 0x00000000
+#define CRTC_V_SYNC_B__CRTC_V_SYNC_B_END_MASK 0x1fff0000L
+#define CRTC_V_SYNC_B__CRTC_V_SYNC_B_END__SHIFT 0x00000010
+#define CRTC_V_SYNC_B__CRTC_V_SYNC_B_START_MASK 0x00001fffL
+#define CRTC_V_SYNC_B__CRTC_V_SYNC_B_START__SHIFT 0x00000000
+#define CRTC_VSYNC_NOM_INT_STATUS__CRTC_VSYNC_NOM_INT_CLEAR_MASK 0x00000010L
+#define CRTC_VSYNC_NOM_INT_STATUS__CRTC_VSYNC_NOM_INT_CLEAR__SHIFT 0x00000004
+#define CRTC_VSYNC_NOM_INT_STATUS__CRTC_VSYNC_NOM_MASK 0x00000001L
+#define CRTC_VSYNC_NOM_INT_STATUS__CRTC_VSYNC_NOM__SHIFT 0x00000000
+#define CRTC_V_TOTAL_CONTROL__CRTC_FORCE_LOCK_ON_EVENT_MASK 0x00000100L
+#define CRTC_V_TOTAL_CONTROL__CRTC_FORCE_LOCK_ON_EVENT__SHIFT 0x00000008
+#define CRTC_V_TOTAL_CONTROL__CRTC_FORCE_LOCK_TO_MASTER_VSYNC_MASK 0x00001000L
+#define CRTC_V_TOTAL_CONTROL__CRTC_FORCE_LOCK_TO_MASTER_VSYNC__SHIFT 0x0000000c
+#define CRTC_V_TOTAL_CONTROL__CRTC_SET_V_TOTAL_MIN_MASK_MASK 0xffff0000L
+#define CRTC_V_TOTAL_CONTROL__CRTC_SET_V_TOTAL_MIN_MASK__SHIFT 0x00000010
+#define CRTC_V_TOTAL_CONTROL__CRTC_V_TOTAL_MAX_SEL_MASK 0x00000010L
+#define CRTC_V_TOTAL_CONTROL__CRTC_V_TOTAL_MAX_SEL__SHIFT 0x00000004
+#define CRTC_V_TOTAL_CONTROL__CRTC_V_TOTAL_MIN_SEL_MASK 0x00000001L
+#define CRTC_V_TOTAL_CONTROL__CRTC_V_TOTAL_MIN_SEL__SHIFT 0x00000000
+#define CRTC_V_TOTAL__CRTC_V_TOTAL_MASK 0x00001fffL
+#define CRTC_V_TOTAL__CRTC_V_TOTAL__SHIFT 0x00000000
+#define CRTC_V_TOTAL_INT_STATUS__CRTC_SET_V_TOTAL_MIN_EVENT_OCCURED_ACK_MASK 0x00000100L
+#define CRTC_V_TOTAL_INT_STATUS__CRTC_SET_V_TOTAL_MIN_EVENT_OCCURED_ACK__SHIFT 0x00000008
+#define CRTC_V_TOTAL_INT_STATUS__CRTC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK 0x00000010L
+#define CRTC_V_TOTAL_INT_STATUS__CRTC_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT 0x00000004
+#define CRTC_V_TOTAL_INT_STATUS__CRTC_SET_V_TOTAL_MIN_EVENT_OCCURED_MASK 0x00000001L
+#define CRTC_V_TOTAL_INT_STATUS__CRTC_SET_V_TOTAL_MIN_EVENT_OCCURED_MSK_MASK 0x00001000L
+#define CRTC_V_TOTAL_INT_STATUS__CRTC_SET_V_TOTAL_MIN_EVENT_OCCURED_MSK__SHIFT 0x0000000c
+#define CRTC_V_TOTAL_INT_STATUS__CRTC_SET_V_TOTAL_MIN_EVENT_OCCURED__SHIFT 0x00000000
+#define CRTC_V_TOTAL_MAX__CRTC_ALLOW_VBLANK_EXTENSION_FOR_MC_TRAINING_MASK 0x00010000L
+#define CRTC_V_TOTAL_MAX__CRTC_ALLOW_VBLANK_EXTENSION_FOR_MC_TRAINING__SHIFT 0x00000010
+#define CRTC_V_TOTAL_MAX__CRTC_V_TOTAL_MAX_MASK 0x00001fffL
+#define CRTC_V_TOTAL_MAX__CRTC_V_TOTAL_MAX__SHIFT 0x00000000
+#define CRTC_V_TOTAL_MIN__CRTC_V_TOTAL_MIN_MASK 0x00001fffL
+#define CRTC_V_TOTAL_MIN__CRTC_V_TOTAL_MIN__SHIFT 0x00000000
+#define CRTC_V_UPDATE_INT_STATUS__CRTC_V_UPDATE_INT_CLEAR_MASK 0x00000100L
+#define CRTC_V_UPDATE_INT_STATUS__CRTC_V_UPDATE_INT_CLEAR__SHIFT 0x00000008
+#define CRTC_V_UPDATE_INT_STATUS__CRTC_V_UPDATE_INT_OCCURRED_MASK 0x00000001L
+#define CRTC_V_UPDATE_INT_STATUS__CRTC_V_UPDATE_INT_OCCURRED__SHIFT 0x00000000
+#define CUR_COLOR1__CUR_COLOR1_BLUE_MASK 0x000000ffL
+#define CUR_COLOR1__CUR_COLOR1_BLUE__SHIFT 0x00000000
+#define CUR_COLOR1__CUR_COLOR1_GREEN_MASK 0x0000ff00L
+#define CUR_COLOR1__CUR_COLOR1_GREEN__SHIFT 0x00000008
+#define CUR_COLOR1__CUR_COLOR1_RED_MASK 0x00ff0000L
+#define CUR_COLOR1__CUR_COLOR1_RED__SHIFT 0x00000010
+#define CUR_COLOR2__CUR_COLOR2_BLUE_MASK 0x000000ffL
+#define CUR_COLOR2__CUR_COLOR2_BLUE__SHIFT 0x00000000
+#define CUR_COLOR2__CUR_COLOR2_GREEN_MASK 0x0000ff00L
+#define CUR_COLOR2__CUR_COLOR2_GREEN__SHIFT 0x00000008
+#define CUR_COLOR2__CUR_COLOR2_RED_MASK 0x00ff0000L
+#define CUR_COLOR2__CUR_COLOR2_RED__SHIFT 0x00000010
+#define CUR_CONTROL__CUR_INV_TRANS_CLAMP_MASK 0x00000010L
+#define CUR_CONTROL__CUR_INV_TRANS_CLAMP__SHIFT 0x00000004
+#define CUR_CONTROL__CURSOR_2X_MAGNIFY_MASK 0x00010000L
+#define CUR_CONTROL__CURSOR_2X_MAGNIFY__SHIFT 0x00000010
+#define CUR_CONTROL__CURSOR_EN_MASK 0x00000001L
+#define CUR_CONTROL__CURSOR_EN__SHIFT 0x00000000
+#define CUR_CONTROL__CURSOR_FORCE_MC_ON_MASK 0x00100000L
+#define CUR_CONTROL__CURSOR_FORCE_MC_ON__SHIFT 0x00000014
+#define CUR_CONTROL__CURSOR_MODE_MASK 0x00000300L
+#define CUR_CONTROL__CURSOR_MODE__SHIFT 0x00000008
+#define CUR_CONTROL__CURSOR_URGENT_CONTROL_MASK 0x07000000L
+#define CUR_CONTROL__CURSOR_URGENT_CONTROL__SHIFT 0x00000018
+#define CUR_HOT_SPOT__CURSOR_HOT_SPOT_X_MASK 0x003f0000L
+#define CUR_HOT_SPOT__CURSOR_HOT_SPOT_X__SHIFT 0x00000010
+#define CUR_HOT_SPOT__CURSOR_HOT_SPOT_Y_MASK 0x0000003fL
+#define CUR_HOT_SPOT__CURSOR_HOT_SPOT_Y__SHIFT 0x00000000
+#define CUR_POSITION__CURSOR_X_POSITION_MASK 0x3fff0000L
+#define CUR_POSITION__CURSOR_X_POSITION__SHIFT 0x00000010
+#define CUR_POSITION__CURSOR_Y_POSITION_MASK 0x00003fffL
+#define CUR_POSITION__CURSOR_Y_POSITION__SHIFT 0x00000000
+#define CUR_REQUEST_FILTER_CNTL__CUR_REQUEST_FILTER_DIS_MASK 0x00000001L
+#define CUR_REQUEST_FILTER_CNTL__CUR_REQUEST_FILTER_DIS__SHIFT 0x00000000
+#define CUR_SIZE__CURSOR_HEIGHT_MASK 0x0000003fL
+#define CUR_SIZE__CURSOR_HEIGHT__SHIFT 0x00000000
+#define CUR_SIZE__CURSOR_WIDTH_MASK 0x003f0000L
+#define CUR_SIZE__CURSOR_WIDTH__SHIFT 0x00000010
+#define CUR_SURFACE_ADDRESS__CURSOR_SURFACE_ADDRESS_MASK 0xffffffffL
+#define CUR_SURFACE_ADDRESS__CURSOR_SURFACE_ADDRESS__SHIFT 0x00000000
+#define CUR_SURFACE_ADDRESS_HIGH__CURSOR_SURFACE_ADDRESS_HIGH_MASK 0x000000ffL
+#define CUR_SURFACE_ADDRESS_HIGH__CURSOR_SURFACE_ADDRESS_HIGH__SHIFT 0x00000000
+#define CUR_UPDATE__CURSOR_DISABLE_MULTIPLE_UPDATE_MASK 0x01000000L
+#define CUR_UPDATE__CURSOR_DISABLE_MULTIPLE_UPDATE__SHIFT 0x00000018
+#define CUR_UPDATE__CURSOR_UPDATE_LOCK_MASK 0x00010000L
+#define CUR_UPDATE__CURSOR_UPDATE_LOCK__SHIFT 0x00000010
+#define CUR_UPDATE__CURSOR_UPDATE_PENDING_MASK 0x00000001L
+#define CUR_UPDATE__CURSOR_UPDATE_PENDING__SHIFT 0x00000000
+#define CUR_UPDATE__CURSOR_UPDATE_TAKEN_MASK 0x00000002L
+#define CUR_UPDATE__CURSOR_UPDATE_TAKEN__SHIFT 0x00000001
+#define D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK 0x00000001L
+#define D1VGA_CONTROL__D1VGA_MODE_ENABLE__SHIFT 0x00000000
+#define D1VGA_CONTROL__D1VGA_OVERSCAN_COLOR_EN_MASK 0x00010000L
+#define D1VGA_CONTROL__D1VGA_OVERSCAN_COLOR_EN__SHIFT 0x00000010
+#define D1VGA_CONTROL__D1VGA_ROTATE_MASK 0x03000000L
+#define D1VGA_CONTROL__D1VGA_ROTATE__SHIFT 0x00000018
+#define D1VGA_CONTROL__D1VGA_SYNC_POLARITY_SELECT_MASK 0x00000200L
+#define D1VGA_CONTROL__D1VGA_SYNC_POLARITY_SELECT__SHIFT 0x00000009
+#define D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK 0x00000100L
+#define D1VGA_CONTROL__D1VGA_TIMING_SELECT__SHIFT 0x00000008
+#define D2VGA_CONTROL__D2VGA_MODE_ENABLE_MASK 0x00000001L
+#define D2VGA_CONTROL__D2VGA_MODE_ENABLE__SHIFT 0x00000000
+#define D2VGA_CONTROL__D2VGA_OVERSCAN_COLOR_EN_MASK 0x00010000L
+#define D2VGA_CONTROL__D2VGA_OVERSCAN_COLOR_EN__SHIFT 0x00000010
+#define D2VGA_CONTROL__D2VGA_ROTATE_MASK 0x03000000L
+#define D2VGA_CONTROL__D2VGA_ROTATE__SHIFT 0x00000018
+#define D2VGA_CONTROL__D2VGA_SYNC_POLARITY_SELECT_MASK 0x00000200L
+#define D2VGA_CONTROL__D2VGA_SYNC_POLARITY_SELECT__SHIFT 0x00000009
+#define D2VGA_CONTROL__D2VGA_TIMING_SELECT_MASK 0x00000100L
+#define D2VGA_CONTROL__D2VGA_TIMING_SELECT__SHIFT 0x00000008
+#define D3VGA_CONTROL__D3VGA_MODE_ENABLE_MASK 0x00000001L
+#define D3VGA_CONTROL__D3VGA_MODE_ENABLE__SHIFT 0x00000000
+#define D3VGA_CONTROL__D3VGA_OVERSCAN_COLOR_EN_MASK 0x00010000L
+#define D3VGA_CONTROL__D3VGA_OVERSCAN_COLOR_EN__SHIFT 0x00000010
+#define D3VGA_CONTROL__D3VGA_ROTATE_MASK 0x03000000L
+#define D3VGA_CONTROL__D3VGA_ROTATE__SHIFT 0x00000018
+#define D3VGA_CONTROL__D3VGA_SYNC_POLARITY_SELECT_MASK 0x00000200L
+#define D3VGA_CONTROL__D3VGA_SYNC_POLARITY_SELECT__SHIFT 0x00000009
+#define D3VGA_CONTROL__D3VGA_TIMING_SELECT_MASK 0x00000100L
+#define D3VGA_CONTROL__D3VGA_TIMING_SELECT__SHIFT 0x00000008
+#define D4VGA_CONTROL__D4VGA_MODE_ENABLE_MASK 0x00000001L
+#define D4VGA_CONTROL__D4VGA_MODE_ENABLE__SHIFT 0x00000000
+#define D4VGA_CONTROL__D4VGA_OVERSCAN_COLOR_EN_MASK 0x00010000L
+#define D4VGA_CONTROL__D4VGA_OVERSCAN_COLOR_EN__SHIFT 0x00000010
+#define D4VGA_CONTROL__D4VGA_ROTATE_MASK 0x03000000L
+#define D4VGA_CONTROL__D4VGA_ROTATE__SHIFT 0x00000018
+#define D4VGA_CONTROL__D4VGA_SYNC_POLARITY_SELECT_MASK 0x00000200L
+#define D4VGA_CONTROL__D4VGA_SYNC_POLARITY_SELECT__SHIFT 0x00000009
+#define D4VGA_CONTROL__D4VGA_TIMING_SELECT_MASK 0x00000100L
+#define D4VGA_CONTROL__D4VGA_TIMING_SELECT__SHIFT 0x00000008
+#define D5VGA_CONTROL__D5VGA_MODE_ENABLE_MASK 0x00000001L
+#define D5VGA_CONTROL__D5VGA_MODE_ENABLE__SHIFT 0x00000000
+#define D5VGA_CONTROL__D5VGA_OVERSCAN_COLOR_EN_MASK 0x00010000L
+#define D5VGA_CONTROL__D5VGA_OVERSCAN_COLOR_EN__SHIFT 0x00000010
+#define D5VGA_CONTROL__D5VGA_ROTATE_MASK 0x03000000L
+#define D5VGA_CONTROL__D5VGA_ROTATE__SHIFT 0x00000018
+#define D5VGA_CONTROL__D5VGA_SYNC_POLARITY_SELECT_MASK 0x00000200L
+#define D5VGA_CONTROL__D5VGA_SYNC_POLARITY_SELECT__SHIFT 0x00000009
+#define D5VGA_CONTROL__D5VGA_TIMING_SELECT_MASK 0x00000100L
+#define D5VGA_CONTROL__D5VGA_TIMING_SELECT__SHIFT 0x00000008
+#define D6VGA_CONTROL__D6VGA_MODE_ENABLE_MASK 0x00000001L
+#define D6VGA_CONTROL__D6VGA_MODE_ENABLE__SHIFT 0x00000000
+#define D6VGA_CONTROL__D6VGA_OVERSCAN_COLOR_EN_MASK 0x00010000L
+#define D6VGA_CONTROL__D6VGA_OVERSCAN_COLOR_EN__SHIFT 0x00000010
+#define D6VGA_CONTROL__D6VGA_ROTATE_MASK 0x03000000L
+#define D6VGA_CONTROL__D6VGA_ROTATE__SHIFT 0x00000018
+#define D6VGA_CONTROL__D6VGA_SYNC_POLARITY_SELECT_MASK 0x00000200L
+#define D6VGA_CONTROL__D6VGA_SYNC_POLARITY_SELECT__SHIFT 0x00000009
+#define D6VGA_CONTROL__D6VGA_TIMING_SELECT_MASK 0x00000100L
+#define D6VGA_CONTROL__D6VGA_TIMING_SELECT__SHIFT 0x00000008
+#define DAC_AUTODETECT_CONTROL2__DAC_AUTODETECT_POWERUP_COUNTER_MASK 0x000000ffL
+#define DAC_AUTODETECT_CONTROL2__DAC_AUTODETECT_POWERUP_COUNTER__SHIFT 0x00000000
+#define DAC_AUTODETECT_CONTROL2__DAC_AUTODETECT_TESTMODE_MASK 0x00000100L
+#define DAC_AUTODETECT_CONTROL2__DAC_AUTODETECT_TESTMODE__SHIFT 0x00000008
+#define DAC_AUTODETECT_CONTROL3__DAC_AUTODET_COMPARATOR_IN_DELAY_MASK 0x000000ffL
+#define DAC_AUTODETECT_CONTROL3__DAC_AUTODET_COMPARATOR_IN_DELAY__SHIFT 0x00000000
+#define DAC_AUTODETECT_CONTROL3__DAC_AUTODET_COMPARATOR_OUT_DELAY_MASK 0x0000ff00L
+#define DAC_AUTODETECT_CONTROL3__DAC_AUTODET_COMPARATOR_OUT_DELAY__SHIFT 0x00000008
+#define DAC_AUTODETECT_CONTROL__DAC_AUTODETECT_CHECK_MASK_MASK 0x00070000L
+#define DAC_AUTODETECT_CONTROL__DAC_AUTODETECT_CHECK_MASK__SHIFT 0x00000010
+#define DAC_AUTODETECT_CONTROL__DAC_AUTODETECT_FRAME_TIME_COUNTER_MASK 0x0000ff00L
+#define DAC_AUTODETECT_CONTROL__DAC_AUTODETECT_FRAME_TIME_COUNTER__SHIFT 0x00000008
+#define DAC_AUTODETECT_CONTROL__DAC_AUTODETECT_MODE_MASK 0x00000003L
+#define DAC_AUTODETECT_CONTROL__DAC_AUTODETECT_MODE__SHIFT 0x00000000
+#define DAC_AUTODETECT_INT_CONTROL__DAC_AUTODETECT_ACK_MASK 0x00000001L
+#define DAC_AUTODETECT_INT_CONTROL__DAC_AUTODETECT_ACK__SHIFT 0x00000000
+#define DAC_AUTODETECT_INT_CONTROL__DAC_AUTODETECT_INT_ENABLE_MASK 0x00010000L
+#define DAC_AUTODETECT_INT_CONTROL__DAC_AUTODETECT_INT_ENABLE__SHIFT 0x00000010
+#define DAC_AUTODETECT_STATUS__DAC_AUTODETECT_BLUE_SENSE_MASK 0x03000000L
+#define DAC_AUTODETECT_STATUS__DAC_AUTODETECT_BLUE_SENSE__SHIFT 0x00000018
+#define DAC_AUTODETECT_STATUS__DAC_AUTODETECT_CONNECT_MASK 0x00000010L
+#define DAC_AUTODETECT_STATUS__DAC_AUTODETECT_CONNECT__SHIFT 0x00000004
+#define DAC_AUTODETECT_STATUS__DAC_AUTODETECT_GREEN_SENSE_MASK 0x00030000L
+#define DAC_AUTODETECT_STATUS__DAC_AUTODETECT_GREEN_SENSE__SHIFT 0x00000010
+#define DAC_AUTODETECT_STATUS__DAC_AUTODETECT_RED_SENSE_MASK 0x00000300L
+#define DAC_AUTODETECT_STATUS__DAC_AUTODETECT_RED_SENSE__SHIFT 0x00000008
+#define DAC_AUTODETECT_STATUS__DAC_AUTODETECT_STATUS_MASK 0x00000001L
+#define DAC_AUTODETECT_STATUS__DAC_AUTODETECT_STATUS__SHIFT 0x00000000
+#define DAC_CLK_ENABLE__DACA_CLK_ENABLE_MASK 0x00000001L
+#define DAC_CLK_ENABLE__DACA_CLK_ENABLE__SHIFT 0x00000000
+#define DAC_CLK_ENABLE__DACB_CLK_ENABLE_MASK 0x00000010L
+#define DAC_CLK_ENABLE__DACB_CLK_ENABLE__SHIFT 0x00000004
+#define DAC_COMPARATOR_ENABLE__DAC_B_ASYNC_ENABLE_MASK 0x00040000L
+#define DAC_COMPARATOR_ENABLE__DAC_B_ASYNC_ENABLE__SHIFT 0x00000012
+#define DAC_COMPARATOR_ENABLE__DAC_COMP_DDET_REF_EN_MASK 0x00000001L
+#define DAC_COMPARATOR_ENABLE__DAC_COMP_DDET_REF_EN__SHIFT 0x00000000
+#define DAC_COMPARATOR_ENABLE__DAC_COMP_SDET_REF_EN_MASK 0x00000100L
+#define DAC_COMPARATOR_ENABLE__DAC_COMP_SDET_REF_EN__SHIFT 0x00000008
+#define DAC_COMPARATOR_ENABLE__DAC_G_ASYNC_ENABLE_MASK 0x00020000L
+#define DAC_COMPARATOR_ENABLE__DAC_G_ASYNC_ENABLE__SHIFT 0x00000011
+#define DAC_COMPARATOR_ENABLE__DAC_R_ASYNC_ENABLE_MASK 0x00010000L
+#define DAC_COMPARATOR_ENABLE__DAC_R_ASYNC_ENABLE__SHIFT 0x00000010
+#define DAC_COMPARATOR_OUTPUT__DAC_COMPARATOR_OUTPUT_BLUE_MASK 0x00000002L
+#define DAC_COMPARATOR_OUTPUT__DAC_COMPARATOR_OUTPUT_BLUE__SHIFT 0x00000001
+#define DAC_COMPARATOR_OUTPUT__DAC_COMPARATOR_OUTPUT_GREEN_MASK 0x00000004L
+#define DAC_COMPARATOR_OUTPUT__DAC_COMPARATOR_OUTPUT_GREEN__SHIFT 0x00000002
+#define DAC_COMPARATOR_OUTPUT__DAC_COMPARATOR_OUTPUT_MASK 0x00000001L
+#define DAC_COMPARATOR_OUTPUT__DAC_COMPARATOR_OUTPUT_RED_MASK 0x00000008L
+#define DAC_COMPARATOR_OUTPUT__DAC_COMPARATOR_OUTPUT_RED__SHIFT 0x00000003
+#define DAC_COMPARATOR_OUTPUT__DAC_COMPARATOR_OUTPUT__SHIFT 0x00000000
+#define DAC_CONTROL__DAC_DFORCE_EN_MASK 0x00000001L
+#define DAC_CONTROL__DAC_DFORCE_EN__SHIFT 0x00000000
+#define DAC_CONTROL__DAC_TV_ENABLE_MASK 0x00000100L
+#define DAC_CONTROL__DAC_TV_ENABLE__SHIFT 0x00000008
+#define DAC_CONTROL__DAC_ZSCALE_SHIFT_MASK 0x00010000L
+#define DAC_CONTROL__DAC_ZSCALE_SHIFT__SHIFT 0x00000010
+#define DAC_CRC_CONTROL__DAC_CRC_FIELD_MASK 0x00000001L
+#define DAC_CRC_CONTROL__DAC_CRC_FIELD__SHIFT 0x00000000
+#define DAC_CRC_CONTROL__DAC_CRC_ONLY_BLANKb_MASK 0x00000100L
+#define DAC_CRC_CONTROL__DAC_CRC_ONLY_BLANKb__SHIFT 0x00000008
+#define DAC_CRC_EN__DAC_CRC_CONT_EN_MASK 0x00010000L
+#define DAC_CRC_EN__DAC_CRC_CONT_EN__SHIFT 0x00000010
+#define DAC_CRC_EN__DAC_CRC_EN_MASK 0x00000001L
+#define DAC_CRC_EN__DAC_CRC_EN__SHIFT 0x00000000
+#define DAC_CRC_SIG_CONTROL__DAC_CRC_SIG_CONTROL_MASK 0x0000003fL
+#define DAC_CRC_SIG_CONTROL__DAC_CRC_SIG_CONTROL__SHIFT 0x00000000
+#define DAC_CRC_SIG_CONTROL_MASK__DAC_CRC_SIG_CONTROL_MASK_MASK 0x0000003fL
+#define DAC_CRC_SIG_CONTROL_MASK__DAC_CRC_SIG_CONTROL_MASK__SHIFT 0x00000000
+#define DAC_CRC_SIG_RGB__DAC_CRC_SIG_BLUE_MASK 0x000003ffL
+#define DAC_CRC_SIG_RGB__DAC_CRC_SIG_BLUE__SHIFT 0x00000000
+#define DAC_CRC_SIG_RGB__DAC_CRC_SIG_GREEN_MASK 0x000ffc00L
+#define DAC_CRC_SIG_RGB__DAC_CRC_SIG_GREEN__SHIFT 0x0000000a
+#define DAC_CRC_SIG_RGB__DAC_CRC_SIG_RED_MASK 0x3ff00000L
+#define DAC_CRC_SIG_RGB__DAC_CRC_SIG_RED__SHIFT 0x00000014
+#define DAC_CRC_SIG_RGB_MASK__DAC_CRC_SIG_BLUE_MASK_MASK 0x000003ffL
+#define DAC_CRC_SIG_RGB_MASK__DAC_CRC_SIG_BLUE_MASK__SHIFT 0x00000000
+#define DAC_CRC_SIG_RGB_MASK__DAC_CRC_SIG_GREEN_MASK_MASK 0x000ffc00L
+#define DAC_CRC_SIG_RGB_MASK__DAC_CRC_SIG_GREEN_MASK__SHIFT 0x0000000a
+#define DAC_CRC_SIG_RGB_MASK__DAC_CRC_SIG_RED_MASK_MASK 0x3ff00000L
+#define DAC_CRC_SIG_RGB_MASK__DAC_CRC_SIG_RED_MASK__SHIFT 0x00000014
+#define DAC_DATA__DAC_DATA_MASK 0x0000003fL
+#define DAC_DATA__DAC_DATA__SHIFT 0x00000000
+#define DAC_DFT_CONFIG__DAC_DFT_CONFIG_MASK 0xffffffffL
+#define DAC_DFT_CONFIG__DAC_DFT_CONFIG__SHIFT 0x00000000
+#define DAC_ENABLE__DAC_ENABLE_MASK 0x00000001L
+#define DAC_ENABLE__DAC_ENABLE__SHIFT 0x00000000
+#define DAC_ENABLE__DAC_RESYNC_FIFO_ENABLE_MASK 0x00000002L
+#define DAC_ENABLE__DAC_RESYNC_FIFO_ENABLE__SHIFT 0x00000001
+#define DAC_ENABLE__DAC_RESYNC_FIFO_ERROR_ACK_MASK 0x00000020L
+#define DAC_ENABLE__DAC_RESYNC_FIFO_ERROR_ACK__SHIFT 0x00000005
+#define DAC_ENABLE__DAC_RESYNC_FIFO_ERROR_MASK 0x00000010L
+#define DAC_ENABLE__DAC_RESYNC_FIFO_ERROR__SHIFT 0x00000004
+#define DAC_ENABLE__DAC_RESYNC_FIFO_POINTER_SKEW_MASK 0x0000000cL
+#define DAC_ENABLE__DAC_RESYNC_FIFO_POINTER_SKEW__SHIFT 0x00000002
+#define DAC_ENABLE__DAC_RESYNC_FIFO_TVOUT_SIM_MASK 0x00000100L
+#define DAC_ENABLE__DAC_RESYNC_FIFO_TVOUT_SIM__SHIFT 0x00000008
+#define DAC_FIFO_STATUS__DAC_FIFO_CAL_AVERAGE_LEVEL_MASK 0x0000fc00L
+#define DAC_FIFO_STATUS__DAC_FIFO_CAL_AVERAGE_LEVEL__SHIFT 0x0000000a
+#define DAC_FIFO_STATUS__DAC_FIFO_CALIBRATED_MASK 0x20000000L
+#define DAC_FIFO_STATUS__DAC_FIFO_CALIBRATED__SHIFT 0x0000001d
+#define DAC_FIFO_STATUS__DAC_FIFO_FORCE_RECAL_AVERAGE_MASK 0x40000000L
+#define DAC_FIFO_STATUS__DAC_FIFO_FORCE_RECAL_AVERAGE__SHIFT 0x0000001e
+#define DAC_FIFO_STATUS__DAC_FIFO_FORCE_RECOMP_MINMAX_MASK 0x80000000L
+#define DAC_FIFO_STATUS__DAC_FIFO_FORCE_RECOMP_MINMAX__SHIFT 0x0000001f
+#define DAC_FIFO_STATUS__DAC_FIFO_MAXIMUM_LEVEL_MASK 0x000f0000L
+#define DAC_FIFO_STATUS__DAC_FIFO_MAXIMUM_LEVEL__SHIFT 0x00000010
+#define DAC_FIFO_STATUS__DAC_FIFO_MINIMUM_LEVEL_MASK 0x03c00000L
+#define DAC_FIFO_STATUS__DAC_FIFO_MINIMUM_LEVEL__SHIFT 0x00000016
+#define DAC_FIFO_STATUS__DAC_FIFO_OVERWRITE_LEVEL_MASK 0x000000fcL
+#define DAC_FIFO_STATUS__DAC_FIFO_OVERWRITE_LEVEL__SHIFT 0x00000002
+#define DAC_FIFO_STATUS__DAC_FIFO_USE_OVERWRITE_LEVEL_MASK 0x00000002L
+#define DAC_FIFO_STATUS__DAC_FIFO_USE_OVERWRITE_LEVEL__SHIFT 0x00000001
+#define DAC_FORCE_DATA__DAC_FORCE_DATA_MASK 0x000003ffL
+#define DAC_FORCE_DATA__DAC_FORCE_DATA__SHIFT 0x00000000
+#define DAC_FORCE_OUTPUT_CNTL__DAC_FORCE_DATA_EN_MASK 0x00000001L
+#define DAC_FORCE_OUTPUT_CNTL__DAC_FORCE_DATA_EN__SHIFT 0x00000000
+#define DAC_FORCE_OUTPUT_CNTL__DAC_FORCE_DATA_ON_BLANKb_ONLY_MASK 0x01000000L
+#define DAC_FORCE_OUTPUT_CNTL__DAC_FORCE_DATA_ON_BLANKb_ONLY__SHIFT 0x00000018
+#define DAC_FORCE_OUTPUT_CNTL__DAC_FORCE_DATA_SEL_MASK 0x00000700L
+#define DAC_FORCE_OUTPUT_CNTL__DAC_FORCE_DATA_SEL__SHIFT 0x00000008
+#define DAC_MACRO_CNTL_RESERVED0__DAC_MACRO_CNTL_RESERVED_MASK 0xffffffffL
+#define DAC_MACRO_CNTL_RESERVED0__DAC_MACRO_CNTL_RESERVED__SHIFT 0x00000000
+#define DAC_MACRO_CNTL_RESERVED1__DAC_MACRO_CNTL_RESERVED_MASK 0xffffffffL
+#define DAC_MACRO_CNTL_RESERVED1__DAC_MACRO_CNTL_RESERVED__SHIFT 0x00000000
+#define DAC_MACRO_CNTL_RESERVED2__DAC_MACRO_CNTL_RESERVED_MASK 0xffffffffL
+#define DAC_MACRO_CNTL_RESERVED2__DAC_MACRO_CNTL_RESERVED__SHIFT 0x00000000
+#define DAC_MACRO_CNTL_RESERVED3__DAC_MACRO_CNTL_RESERVED_MASK 0xffffffffL
+#define DAC_MACRO_CNTL_RESERVED3__DAC_MACRO_CNTL_RESERVED__SHIFT 0x00000000
+#define DAC_MASK__DAC_MASK_MASK 0x000000ffL
+#define DAC_MASK__DAC_MASK__SHIFT 0x00000000
+#define DAC_POWERDOWN__DAC_POWERDOWN_BLUE_MASK 0x00000100L
+#define DAC_POWERDOWN__DAC_POWERDOWN_BLUE__SHIFT 0x00000008
+#define DAC_POWERDOWN__DAC_POWERDOWN_GREEN_MASK 0x00010000L
+#define DAC_POWERDOWN__DAC_POWERDOWN_GREEN__SHIFT 0x00000010
+#define DAC_POWERDOWN__DAC_POWERDOWN_MASK 0x00000001L
+#define DAC_POWERDOWN__DAC_POWERDOWN_RED_MASK 0x01000000L
+#define DAC_POWERDOWN__DAC_POWERDOWN_RED__SHIFT 0x00000018
+#define DAC_POWERDOWN__DAC_POWERDOWN__SHIFT 0x00000000
+#define DAC_PWR_CNTL__DAC_BG_MODE_MASK 0x00000003L
+#define DAC_PWR_CNTL__DAC_BG_MODE__SHIFT 0x00000000
+#define DAC_PWR_CNTL__DAC_PWRCNTL_MASK 0x00030000L
+#define DAC_PWR_CNTL__DAC_PWRCNTL__SHIFT 0x00000010
+#define DAC_R_INDEX__DAC_R_INDEX_MASK 0x000000ffL
+#define DAC_R_INDEX__DAC_R_INDEX__SHIFT 0x00000000
+#define DAC_SOURCE_SELECT__DAC_SOURCE_SELECT_MASK 0x00000007L
+#define DAC_SOURCE_SELECT__DAC_SOURCE_SELECT__SHIFT 0x00000000
+#define DAC_SOURCE_SELECT__DAC_TV_SELECT_MASK 0x00000008L
+#define DAC_SOURCE_SELECT__DAC_TV_SELECT__SHIFT 0x00000003
+#define DAC_STEREOSYNC_SELECT__DAC_STEREOSYNC_SELECT_MASK 0x00000007L
+#define DAC_STEREOSYNC_SELECT__DAC_STEREOSYNC_SELECT__SHIFT 0x00000000
+#define DAC_SYNC_TRISTATE_CONTROL__DAC_HSYNCA_TRISTATE_MASK 0x00000001L
+#define DAC_SYNC_TRISTATE_CONTROL__DAC_HSYNCA_TRISTATE__SHIFT 0x00000000
+#define DAC_SYNC_TRISTATE_CONTROL__DAC_SYNCA_TRISTATE_MASK 0x00010000L
+#define DAC_SYNC_TRISTATE_CONTROL__DAC_SYNCA_TRISTATE__SHIFT 0x00000010
+#define DAC_SYNC_TRISTATE_CONTROL__DAC_VSYNCA_TRISTATE_MASK 0x00000100L
+#define DAC_SYNC_TRISTATE_CONTROL__DAC_VSYNCA_TRISTATE__SHIFT 0x00000008
+#define DAC_W_INDEX__DAC_W_INDEX_MASK 0x000000ffL
+#define DAC_W_INDEX__DAC_W_INDEX__SHIFT 0x00000000
+#define DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_CLEAR_MASK 0x00000100L
+#define DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_CLEAR__SHIFT 0x00000008
+#define DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME_MASK 0x00000001L
+#define DC_ABM1_ACE_CNTL_MISC__ABM1_ACE_REG_WR_MISSED_FRAME__SHIFT 0x00000000
+#define DC_ABM1_ACE_OFFSET_SLOPE_0__ABM1_ACE_LOCK_MASK 0x80000000L
+#define DC_ABM1_ACE_OFFSET_SLOPE_0__ABM1_ACE_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_ACE_OFFSET_SLOPE_0__ABM1_ACE_OFFSET_0_MASK 0x07ff0000L
+#define DC_ABM1_ACE_OFFSET_SLOPE_0__ABM1_ACE_OFFSET_0__SHIFT 0x00000010
+#define DC_ABM1_ACE_OFFSET_SLOPE_0__ABM1_ACE_SLOPE_0_MASK 0x00007fffL
+#define DC_ABM1_ACE_OFFSET_SLOPE_0__ABM1_ACE_SLOPE_0__SHIFT 0x00000000
+#define DC_ABM1_ACE_OFFSET_SLOPE_1__ABM1_ACE_LOCK_MASK 0x80000000L
+#define DC_ABM1_ACE_OFFSET_SLOPE_1__ABM1_ACE_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_ACE_OFFSET_SLOPE_1__ABM1_ACE_OFFSET_1_MASK 0x07ff0000L
+#define DC_ABM1_ACE_OFFSET_SLOPE_1__ABM1_ACE_OFFSET_1__SHIFT 0x00000010
+#define DC_ABM1_ACE_OFFSET_SLOPE_1__ABM1_ACE_SLOPE_1_MASK 0x00007fffL
+#define DC_ABM1_ACE_OFFSET_SLOPE_1__ABM1_ACE_SLOPE_1__SHIFT 0x00000000
+#define DC_ABM1_ACE_OFFSET_SLOPE_2__ABM1_ACE_LOCK_MASK 0x80000000L
+#define DC_ABM1_ACE_OFFSET_SLOPE_2__ABM1_ACE_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_ACE_OFFSET_SLOPE_2__ABM1_ACE_OFFSET_2_MASK 0x07ff0000L
+#define DC_ABM1_ACE_OFFSET_SLOPE_2__ABM1_ACE_OFFSET_2__SHIFT 0x00000010
+#define DC_ABM1_ACE_OFFSET_SLOPE_2__ABM1_ACE_SLOPE_2_MASK 0x00007fffL
+#define DC_ABM1_ACE_OFFSET_SLOPE_2__ABM1_ACE_SLOPE_2__SHIFT 0x00000000
+#define DC_ABM1_ACE_OFFSET_SLOPE_3__ABM1_ACE_LOCK_MASK 0x80000000L
+#define DC_ABM1_ACE_OFFSET_SLOPE_3__ABM1_ACE_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_ACE_OFFSET_SLOPE_3__ABM1_ACE_OFFSET_3_MASK 0x07ff0000L
+#define DC_ABM1_ACE_OFFSET_SLOPE_3__ABM1_ACE_OFFSET_3__SHIFT 0x00000010
+#define DC_ABM1_ACE_OFFSET_SLOPE_3__ABM1_ACE_SLOPE_3_MASK 0x00007fffL
+#define DC_ABM1_ACE_OFFSET_SLOPE_3__ABM1_ACE_SLOPE_3__SHIFT 0x00000000
+#define DC_ABM1_ACE_OFFSET_SLOPE_4__ABM1_ACE_LOCK_MASK 0x80000000L
+#define DC_ABM1_ACE_OFFSET_SLOPE_4__ABM1_ACE_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_ACE_OFFSET_SLOPE_4__ABM1_ACE_OFFSET_4_MASK 0x07ff0000L
+#define DC_ABM1_ACE_OFFSET_SLOPE_4__ABM1_ACE_OFFSET_4__SHIFT 0x00000010
+#define DC_ABM1_ACE_OFFSET_SLOPE_4__ABM1_ACE_SLOPE_4_MASK 0x00007fffL
+#define DC_ABM1_ACE_OFFSET_SLOPE_4__ABM1_ACE_SLOPE_4__SHIFT 0x00000000
+#define DC_ABM1_ACE_THRES_12__ABM1_ACE_LOCK_MASK 0x80000000L
+#define DC_ABM1_ACE_THRES_12__ABM1_ACE_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_ACE_THRES_12__ABM1_ACE_THRES_1_MASK 0x000003ffL
+#define DC_ABM1_ACE_THRES_12__ABM1_ACE_THRES_1__SHIFT 0x00000000
+#define DC_ABM1_ACE_THRES_12__ABM1_ACE_THRES_2_MASK 0x03ff0000L
+#define DC_ABM1_ACE_THRES_12__ABM1_ACE_THRES_2__SHIFT 0x00000010
+#define DC_ABM1_ACE_THRES_34__ABM1_ACE_DBUF_REG_UPDATE_PENDING_MASK 0x40000000L
+#define DC_ABM1_ACE_THRES_34__ABM1_ACE_DBUF_REG_UPDATE_PENDING__SHIFT 0x0000001e
+#define DC_ABM1_ACE_THRES_34__ABM1_ACE_IGNORE_MASTER_LOCK_EN_MASK 0x10000000L
+#define DC_ABM1_ACE_THRES_34__ABM1_ACE_IGNORE_MASTER_LOCK_EN__SHIFT 0x0000001c
+#define DC_ABM1_ACE_THRES_34__ABM1_ACE_LOCK_MASK 0x80000000L
+#define DC_ABM1_ACE_THRES_34__ABM1_ACE_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_ACE_THRES_34__ABM1_ACE_READBACK_DB_REG_VALUE_EN_MASK 0x20000000L
+#define DC_ABM1_ACE_THRES_34__ABM1_ACE_READBACK_DB_REG_VALUE_EN__SHIFT 0x0000001d
+#define DC_ABM1_ACE_THRES_34__ABM1_ACE_THRES_3_MASK 0x000003ffL
+#define DC_ABM1_ACE_THRES_34__ABM1_ACE_THRES_3__SHIFT 0x00000000
+#define DC_ABM1_ACE_THRES_34__ABM1_ACE_THRES_4_MASK 0x03ff0000L
+#define DC_ABM1_ACE_THRES_34__ABM1_ACE_THRES_4__SHIFT 0x00000010
+#define DC_ABM1_BL_MASTER_LOCK__ABM1_BL_MASTER_LOCK_MASK 0x80000000L
+#define DC_ABM1_BL_MASTER_LOCK__ABM1_BL_MASTER_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_CNTL__ABM1_BLANK_MODE_SUPPORT_ENABLE_MASK 0x80000000L
+#define DC_ABM1_CNTL__ABM1_BLANK_MODE_SUPPORT_ENABLE__SHIFT 0x0000001f
+#define DC_ABM1_CNTL__ABM1_EN_MASK 0x00000001L
+#define DC_ABM1_CNTL__ABM1_EN__SHIFT 0x00000000
+#define DC_ABM1_CNTL__ABM1_SOURCE_SELECT_MASK 0x00000700L
+#define DC_ABM1_CNTL__ABM1_SOURCE_SELECT__SHIFT 0x00000008
+#define DC_ABM1_DEBUG_MISC__ABM1_BL_FORCE_INTERRUPT_MASK 0x00010000L
+#define DC_ABM1_DEBUG_MISC__ABM1_BL_FORCE_INTERRUPT__SHIFT 0x00000010
+#define DC_ABM1_DEBUG_MISC__ABM1_HG_FORCE_INTERRUPT_MASK 0x00000001L
+#define DC_ABM1_DEBUG_MISC__ABM1_HG_FORCE_INTERRUPT__SHIFT 0x00000000
+#define DC_ABM1_DEBUG_MISC__ABM1_LS_FORCE_INTERRUPT_MASK 0x00000100L
+#define DC_ABM1_DEBUG_MISC__ABM1_LS_FORCE_INTERRUPT__SHIFT 0x00000008
+#define DC_ABM1_HG_BIN_1_32_SHIFT_FLAG__ABM1_HG_BIN_1_32_SHIFT_FLAG_MASK 0xffffffffL
+#define DC_ABM1_HG_BIN_1_32_SHIFT_FLAG__ABM1_HG_BIN_1_32_SHIFT_FLAG__SHIFT 0x00000000
+#define DC_ABM1_HG_BIN_17_24_SHIFT_INDEX__ABM1_HG_BIN_17_24_SHIFT_INDEX_MASK 0xffffffffL
+#define DC_ABM1_HG_BIN_17_24_SHIFT_INDEX__ABM1_HG_BIN_17_24_SHIFT_INDEX__SHIFT 0x00000000
+#define DC_ABM1_HG_BIN_1_8_SHIFT_INDEX__ABM1_HG_BIN_1_8_SHIFT_INDEX_MASK 0xffffffffL
+#define DC_ABM1_HG_BIN_1_8_SHIFT_INDEX__ABM1_HG_BIN_1_8_SHIFT_INDEX__SHIFT 0x00000000
+#define DC_ABM1_HG_BIN_25_32_SHIFT_INDEX__ABM1_HG_BIN_25_32_SHIFT_INDEX_MASK 0xffffffffL
+#define DC_ABM1_HG_BIN_25_32_SHIFT_INDEX__ABM1_HG_BIN_25_32_SHIFT_INDEX__SHIFT 0x00000000
+#define DC_ABM1_HG_BIN_9_16_SHIFT_INDEX__ABM1_HG_BIN_9_16_SHIFT_INDEX_MASK 0xffffffffL
+#define DC_ABM1_HG_BIN_9_16_SHIFT_INDEX__ABM1_HG_BIN_9_16_SHIFT_INDEX__SHIFT 0x00000000
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_IN_PROGRESS_MASK 0x00000004L
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_IN_PROGRESS__SHIFT 0x00000002
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_CLEAR_MASK 0x80000000L
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_CLEAR__SHIFT 0x0000001f
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME_MASK 0x00000400L
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_BL_REG_READ_MISSED_FRAME__SHIFT 0x0000000a
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_IN_PROGRESS_MASK 0x00000001L
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_IN_PROGRESS__SHIFT 0x00000000
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_CLEAR_MASK 0x00010000L
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_CLEAR__SHIFT 0x00000010
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME_MASK 0x00000100L
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_HG_REG_READ_MISSED_FRAME__SHIFT 0x00000008
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_IN_PROGRESS_MASK 0x00000002L
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_IN_PROGRESS__SHIFT 0x00000001
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_CLEAR_MASK 0x01000000L
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_CLEAR__SHIFT 0x00000018
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME_MASK 0x00000200L
+#define DC_ABM1_HGLS_REG_READ_PROGRESS__ABM1_LS_REG_READ_MISSED_FRAME__SHIFT 0x00000009
+#define DC_ABM1_HG_MISC_CTRL__ABM1_DBUF_HGLS_READBACK_DB_REG_VALUE_EN_MASK 0x00800000L
+#define DC_ABM1_HG_MISC_CTRL__ABM1_DBUF_HGLS_READBACK_DB_REG_VALUE_EN__SHIFT 0x00000017
+#define DC_ABM1_HG_MISC_CTRL__ABM1_DBUF_HGLS_REG_FRAME_START_DISP_SEL_MASK 0x07000000L
+#define DC_ABM1_HG_MISC_CTRL__ABM1_DBUF_HGLS_REG_FRAME_START_DISP_SEL__SHIFT 0x00000018
+#define DC_ABM1_HG_MISC_CTRL__ABM1_DBUF_HGLS_REG_UPDATE_AT_FRAME_START_MASK 0x10000000L
+#define DC_ABM1_HG_MISC_CTRL__ABM1_DBUF_HGLS_REG_UPDATE_AT_FRAME_START__SHIFT 0x0000001c
+#define DC_ABM1_HG_MISC_CTRL__ABM1_DBUF_HGLS_REG_UPDATE_PENDING_MASK 0x40000000L
+#define DC_ABM1_HG_MISC_CTRL__ABM1_DBUF_HGLS_REG_UPDATE_PENDING__SHIFT 0x0000001e
+#define DC_ABM1_HG_MISC_CTRL__ABM1_HG_BIN_BITWIDTH_SIZE_SEL_MASK 0x00030000L
+#define DC_ABM1_HG_MISC_CTRL__ABM1_HG_BIN_BITWIDTH_SIZE_SEL__SHIFT 0x00000010
+#define DC_ABM1_HG_MISC_CTRL__ABM1_HG_FINE_MODE_BIN_SEL_MASK 0x00001000L
+#define DC_ABM1_HG_MISC_CTRL__ABM1_HG_FINE_MODE_BIN_SEL__SHIFT 0x0000000c
+#define DC_ABM1_HG_MISC_CTRL__ABM1_HGLS_IGNORE_MASTER_LOCK_EN_MASK 0x20000000L
+#define DC_ABM1_HG_MISC_CTRL__ABM1_HGLS_IGNORE_MASTER_LOCK_EN__SHIFT 0x0000001d
+#define DC_ABM1_HG_MISC_CTRL__ABM1_HGLS_REG_LOCK_MASK 0x80000000L
+#define DC_ABM1_HG_MISC_CTRL__ABM1_HGLS_REG_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_HG_MISC_CTRL__ABM1_HG_NUM_OF_BINS_SEL_MASK 0x00000003L
+#define DC_ABM1_HG_MISC_CTRL__ABM1_HG_NUM_OF_BINS_SEL__SHIFT 0x00000000
+#define DC_ABM1_HG_MISC_CTRL__ABM1_HG_VMAX_SEL_MASK 0x00000100L
+#define DC_ABM1_HG_MISC_CTRL__ABM1_HG_VMAX_SEL__SHIFT 0x00000008
+#define DC_ABM1_HG_MISC_CTRL__ABM1_OVR_SCAN_PIXEL_PROCESS_EN_MASK 0x00100000L
+#define DC_ABM1_HG_MISC_CTRL__ABM1_OVR_SCAN_PIXEL_PROCESS_EN__SHIFT 0x00000014
+#define DC_ABM1_HG_RESULT_10__ABM1_HG_RESULT_10_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_10__ABM1_HG_RESULT_10__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_11__ABM1_HG_RESULT_11_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_11__ABM1_HG_RESULT_11__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_12__ABM1_HG_RESULT_12_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_12__ABM1_HG_RESULT_12__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_13__ABM1_HG_RESULT_13_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_13__ABM1_HG_RESULT_13__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_14__ABM1_HG_RESULT_14_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_14__ABM1_HG_RESULT_14__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_15__ABM1_HG_RESULT_15_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_15__ABM1_HG_RESULT_15__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_16__ABM1_HG_RESULT_16_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_16__ABM1_HG_RESULT_16__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_17__ABM1_HG_RESULT_17_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_17__ABM1_HG_RESULT_17__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_18__ABM1_HG_RESULT_18_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_18__ABM1_HG_RESULT_18__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_19__ABM1_HG_RESULT_19_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_19__ABM1_HG_RESULT_19__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_1__ABM1_HG_RESULT_1_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_1__ABM1_HG_RESULT_1__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_20__ABM1_HG_RESULT_20_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_20__ABM1_HG_RESULT_20__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_21__ABM1_HG_RESULT_21_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_21__ABM1_HG_RESULT_21__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_22__ABM1_HG_RESULT_22_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_22__ABM1_HG_RESULT_22__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_23__ABM1_HG_RESULT_23_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_23__ABM1_HG_RESULT_23__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_24__ABM1_HG_RESULT_24_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_24__ABM1_HG_RESULT_24__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_2__ABM1_HG_RESULT_2_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_2__ABM1_HG_RESULT_2__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_3__ABM1_HG_RESULT_3_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_3__ABM1_HG_RESULT_3__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_4__ABM1_HG_RESULT_4_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_4__ABM1_HG_RESULT_4__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_5__ABM1_HG_RESULT_5_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_5__ABM1_HG_RESULT_5__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_6__ABM1_HG_RESULT_6_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_6__ABM1_HG_RESULT_6__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_7__ABM1_HG_RESULT_7_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_7__ABM1_HG_RESULT_7__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_8__ABM1_HG_RESULT_8_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_8__ABM1_HG_RESULT_8__SHIFT 0x00000000
+#define DC_ABM1_HG_RESULT_9__ABM1_HG_RESULT_9_MASK 0xffffffffL
+#define DC_ABM1_HG_RESULT_9__ABM1_HG_RESULT_9__SHIFT 0x00000000
+#define DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_INITIAL_SAMPLE_RATE_COUNT_VALUE_WHEN_RESET_MASK 0x00ff0000L
+#define DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_INITIAL_SAMPLE_RATE_COUNT_VALUE_WHEN_RESET__SHIFT 0x00000010
+#define DC_ABM1_HG_SAMPLE_RATE__ABM1_HGLS_REG_LOCK_MASK 0x80000000L
+#define DC_ABM1_HG_SAMPLE_RATE__ABM1_HGLS_REG_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_RESET_SAMPLE_RATE_FRAME_COUNTER_MASK 0x00000002L
+#define DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_RESET_SAMPLE_RATE_FRAME_COUNTER__SHIFT 0x00000001
+#define DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_COUNT_EN_MASK 0x00000001L
+#define DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_COUNT_EN__SHIFT 0x00000000
+#define DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_FRAME_COUNT_MASK 0x0000ff00L
+#define DC_ABM1_HG_SAMPLE_RATE__ABM1_HG_SAMPLE_RATE_FRAME_COUNT__SHIFT 0x00000008
+#define DC_ABM1_IPCSC_COEFF_SEL__ABM1_HGLS_REG_LOCK_MASK 0x80000000L
+#define DC_ABM1_IPCSC_COEFF_SEL__ABM1_HGLS_REG_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_B_MASK 0x0000000fL
+#define DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_B__SHIFT 0x00000000
+#define DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_G_MASK 0x00000f00L
+#define DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_G__SHIFT 0x00000008
+#define DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_R_MASK 0x000f0000L
+#define DC_ABM1_IPCSC_COEFF_SEL__ABM1_IPCSC_COEFF_SEL_R__SHIFT 0x00000010
+#define DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MAX_LUMA_MASK 0x03ff0000L
+#define DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MAX_LUMA__SHIFT 0x00000010
+#define DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MIN_LUMA_MASK 0x000003ffL
+#define DC_ABM1_LS_FILTERED_MIN_MAX_LUMA__ABM1_LS_FILTERED_MIN_LUMA__SHIFT 0x00000000
+#define DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT__ABM1_LS_MAX_PIXEL_VALUE_COUNT_MASK 0x00ffffffL
+#define DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT__ABM1_LS_MAX_PIXEL_VALUE_COUNT__SHIFT 0x00000000
+#define DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MAX_LUMA_MASK 0x03ff0000L
+#define DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MAX_LUMA__SHIFT 0x00000010
+#define DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MIN_LUMA_MASK 0x000003ffL
+#define DC_ABM1_LS_MIN_MAX_LUMA__ABM1_LS_MIN_LUMA__SHIFT 0x00000000
+#define DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_HGLS_REG_LOCK_MASK 0x80000000L
+#define DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_HGLS_REG_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MAX_PIXEL_VALUE_THRES_MASK 0x03ff0000L
+#define DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MAX_PIXEL_VALUE_THRES__SHIFT 0x00000010
+#define DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MIN_PIXEL_VALUE_THRES_MASK 0x000003ffL
+#define DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES__ABM1_LS_MIN_PIXEL_VALUE_THRES__SHIFT 0x00000000
+#define DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT__ABM1_LS_MIN_PIXEL_VALUE_COUNT_MASK 0x00ffffffL
+#define DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT__ABM1_LS_MIN_PIXEL_VALUE_COUNT__SHIFT 0x00000000
+#define DC_ABM1_LS_OVR_SCAN_BIN__ABM1_LS_OVR_SCAN_BIN_MASK 0x00ffffffL
+#define DC_ABM1_LS_OVR_SCAN_BIN__ABM1_LS_OVR_SCAN_BIN__SHIFT 0x00000000
+#define DC_ABM1_LS_PIXEL_COUNT__ABM1_LS_PIXEL_COUNT_MASK 0x00ffffffL
+#define DC_ABM1_LS_PIXEL_COUNT__ABM1_LS_PIXEL_COUNT__SHIFT 0x00000000
+#define DC_ABM1_LS_SAMPLE_RATE__ABM1_HGLS_REG_LOCK_MASK 0x80000000L
+#define DC_ABM1_LS_SAMPLE_RATE__ABM1_HGLS_REG_LOCK__SHIFT 0x0000001f
+#define DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_INITIAL_SAMPLE_RATE_COUNT_VALUE_WHEN_RESET_MASK 0x00ff0000L
+#define DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_INITIAL_SAMPLE_RATE_COUNT_VALUE_WHEN_RESET__SHIFT 0x00000010
+#define DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_RESET_SAMPLE_RATE_FRAME_COUNTER_MASK 0x00000002L
+#define DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_RESET_SAMPLE_RATE_FRAME_COUNTER__SHIFT 0x00000001
+#define DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_COUNT_EN_MASK 0x00000001L
+#define DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_COUNT_EN__SHIFT 0x00000000
+#define DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_FRAME_COUNT_MASK 0x0000ff00L
+#define DC_ABM1_LS_SAMPLE_RATE__ABM1_LS_SAMPLE_RATE_FRAME_COUNT__SHIFT 0x00000008
+#define DC_ABM1_LS_SUM_OF_LUMA__ABM1_LS_SUM_OF_LUMA_MASK 0xffffffffL
+#define DC_ABM1_LS_SUM_OF_LUMA__ABM1_LS_SUM_OF_LUMA__SHIFT 0x00000000
+#define DC_ABM1_OVERSCAN_PIXEL_VALUE__ABM1_OVERSCAN_B_PIXEL_VALUE_MASK 0x3ff00000L
+#define DC_ABM1_OVERSCAN_PIXEL_VALUE__ABM1_OVERSCAN_B_PIXEL_VALUE__SHIFT 0x00000014
+#define DC_ABM1_OVERSCAN_PIXEL_VALUE__ABM1_OVERSCAN_G_PIXEL_VALUE_MASK 0x000ffc00L
+#define DC_ABM1_OVERSCAN_PIXEL_VALUE__ABM1_OVERSCAN_G_PIXEL_VALUE__SHIFT 0x0000000a
+#define DC_ABM1_OVERSCAN_PIXEL_VALUE__ABM1_OVERSCAN_R_PIXEL_VALUE_MASK 0x000003ffL
+#define DC_ABM1_OVERSCAN_PIXEL_VALUE__ABM1_OVERSCAN_R_PIXEL_VALUE__SHIFT 0x00000000
+#define DCCG_AUDIO_DTO0_MODULE__DCCG_AUDIO_DTO0_MODULE_MASK 0xffffffffL
+#define DCCG_AUDIO_DTO0_MODULE__DCCG_AUDIO_DTO0_MODULE__SHIFT 0x00000000
+#define DCCG_AUDIO_DTO0_PHASE__DCCG_AUDIO_DTO0_PHASE_MASK 0xffffffffL
+#define DCCG_AUDIO_DTO0_PHASE__DCCG_AUDIO_DTO0_PHASE__SHIFT 0x00000000
+#define DCCG_AUDIO_DTO1_MODULE__DCCG_AUDIO_DTO1_MODULE_MASK 0xffffffffL
+#define DCCG_AUDIO_DTO1_MODULE__DCCG_AUDIO_DTO1_MODULE__SHIFT 0x00000000
+#define DCCG_AUDIO_DTO1_PHASE__DCCG_AUDIO_DTO1_PHASE_MASK 0xffffffffL
+#define DCCG_AUDIO_DTO1_PHASE__DCCG_AUDIO_DTO1_PHASE__SHIFT 0x00000000
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO0_SOURCE_SEL_MASK 0x00000007L
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO0_SOURCE_SEL__SHIFT 0x00000000
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO_SEL_MASK 0x00000010L
+#define DCCG_AUDIO_DTO_SOURCE__DCCG_AUDIO_DTO_SEL__SHIFT 0x00000004
+#define DCCG_CAC_STATUS__CAC_STATUS_RDDATA_MASK 0xffffffffL
+#define DCCG_CAC_STATUS__CAC_STATUS_RDDATA__SHIFT 0x00000000
+#define DCCG_GATE_DISABLE_CNTL__DACACLK_GATE_DISABLE_MASK 0x00000010L
+#define DCCG_GATE_DISABLE_CNTL__DACACLK_GATE_DISABLE__SHIFT 0x00000004
+#define DCCG_GATE_DISABLE_CNTL__DACBCLK_GATE_DISABLE_MASK 0x00000020L
+#define DCCG_GATE_DISABLE_CNTL__DACBCLK_GATE_DISABLE__SHIFT 0x00000005
+#define DCCG_GATE_DISABLE_CNTL__DISPCLK_DCCG_GATE_DISABLE_MASK 0x00000001L
+#define DCCG_GATE_DISABLE_CNTL__DISPCLK_DCCG_GATE_DISABLE__SHIFT 0x00000000
+#define DCCG_GATE_DISABLE_CNTL__DISPCLK_RAMP_DIV_ID_MASK 0x07000000L
+#define DCCG_GATE_DISABLE_CNTL__DISPCLK_RAMP_DIV_ID__SHIFT 0x00000018
+#define DCCG_GATE_DISABLE_CNTL__DISPCLK_R_DCCG_GATE_DISABLE_MASK 0x00000002L
+#define DCCG_GATE_DISABLE_CNTL__DISPCLK_R_DCCG_GATE_DISABLE__SHIFT 0x00000001
+#define DCCG_GATE_DISABLE_CNTL__DISPCLK_R_DCCG_RAMP_DISABLE_MASK 0x00100000L
+#define DCCG_GATE_DISABLE_CNTL__DISPCLK_R_DCCG_RAMP_DISABLE__SHIFT 0x00000014
+#define DCCG_GATE_DISABLE_CNTL__DVOACLK_GATE_DISABLE_MASK 0x00000040L
+#define DCCG_GATE_DISABLE_CNTL__DVOACLK_GATE_DISABLE__SHIFT 0x00000006
+#define DCCG_GATE_DISABLE_CNTL__PCLK_TV_GATE_DISABLE_MASK 0x00010000L
+#define DCCG_GATE_DISABLE_CNTL__PCLK_TV_GATE_DISABLE__SHIFT 0x00000010
+#define DCCG_GATE_DISABLE_CNTL__SCLK_GATE_DISABLE_MASK 0x00000004L
+#define DCCG_GATE_DISABLE_CNTL__SCLK_GATE_DISABLE__SHIFT 0x00000002
+#define DCCG_GATE_DISABLE_CNTL__SCLK_RAMP_DIV_ID_MASK 0x70000000L
+#define DCCG_GATE_DISABLE_CNTL__SCLK_RAMP_DIV_ID__SHIFT 0x0000001c
+#define DCCG_GATE_DISABLE_CNTL__SYMCLKA_GATE_DISABLE_MASK 0x00000100L
+#define DCCG_GATE_DISABLE_CNTL__SYMCLKA_GATE_DISABLE__SHIFT 0x00000008
+#define DCCG_GATE_DISABLE_CNTL__SYMCLKB_GATE_DISABLE_MASK 0x00000200L
+#define DCCG_GATE_DISABLE_CNTL__SYMCLKB_GATE_DISABLE__SHIFT 0x00000009
+#define DCCG_GATE_DISABLE_CNTL__SYMCLKC_GATE_DISABLE_MASK 0x00000400L
+#define DCCG_GATE_DISABLE_CNTL__SYMCLKC_GATE_DISABLE__SHIFT 0x0000000a
+#define DCCG_GATE_DISABLE_CNTL__SYMCLKD_GATE_DISABLE_MASK 0x00000800L
+#define DCCG_GATE_DISABLE_CNTL__SYMCLKD_GATE_DISABLE__SHIFT 0x0000000b
+#define DCCG_GATE_DISABLE_CNTL__SYMCLKE_GATE_DISABLE_MASK 0x00001000L
+#define DCCG_GATE_DISABLE_CNTL__SYMCLKE_GATE_DISABLE__SHIFT 0x0000000c
+#define DCCG_GATE_DISABLE_CNTL__SYMCLKF_GATE_DISABLE_MASK 0x00002000L
+#define DCCG_GATE_DISABLE_CNTL__SYMCLKF_GATE_DISABLE__SHIFT 0x0000000d
+#define DCCG_GTC_CNTL__DCCG_GTC_ENABLE_MASK 0x00000001L
+#define DCCG_GTC_CNTL__DCCG_GTC_ENABLE__SHIFT 0x00000000
+#define DCCG_GTC_CURRENT__DCCG_GTC_CURRENT_MASK 0xffffffffL
+#define DCCG_GTC_CURRENT__DCCG_GTC_CURRENT__SHIFT 0x00000000
+#define DCCG_GTC_DTO_MODULO__DCCG_GTC_DTO_MODULO_MASK 0xffffffffL
+#define DCCG_GTC_DTO_MODULO__DCCG_GTC_DTO_MODULO__SHIFT 0x00000000
+#define DCCG_PERFMON_CNTL__DCCG_PERF_CRTC_SEL_MASK 0x00000700L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_CRTC_SEL__SHIFT 0x00000008
+#define DCCG_PERFMON_CNTL__DCCG_PERF_DISPCLK_ENABLE_MASK 0x00000001L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_DISPCLK_ENABLE__SHIFT 0x00000000
+#define DCCG_PERFMON_CNTL__DCCG_PERF_DPREFCLK_ENABLE_MASK 0x00000002L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_DPREFCLK_ENABLE__SHIFT 0x00000001
+#define DCCG_PERFMON_CNTL__DCCG_PERF_MODE_HSYNC_MASK 0x00000080L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_MODE_HSYNC__SHIFT 0x00000007
+#define DCCG_PERFMON_CNTL__DCCG_PERF_MODE_VSYNC_MASK 0x00000040L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_MODE_VSYNC__SHIFT 0x00000006
+#define DCCG_PERFMON_CNTL__DCCG_PERF_PIXCLK0_ENABLE_MASK 0x00000010L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_PIXCLK0_ENABLE__SHIFT 0x00000004
+#define DCCG_PERFMON_CNTL__DCCG_PERF_PIXCLK1_ENABLE_MASK 0x00000004L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_PIXCLK1_ENABLE__SHIFT 0x00000002
+#define DCCG_PERFMON_CNTL__DCCG_PERF_PIXCLK2_ENABLE_MASK 0x00000008L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_PIXCLK2_ENABLE__SHIFT 0x00000003
+#define DCCG_PERFMON_CNTL__DCCG_PERF_RUN_MASK 0x00000020L
+#define DCCG_PERFMON_CNTL__DCCG_PERF_RUN__SHIFT 0x00000005
+#define DCCG_SOFT_RESET__REFCLK_SOFT_RESET_MASK 0x00000001L
+#define DCCG_SOFT_RESET__REFCLK_SOFT_RESET__SHIFT 0x00000000
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICA_INV_MASK 0x00010000L
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICA_INV__SHIFT 0x00000010
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICA_SEL_MASK 0x000000ffL
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICA_SEL__SHIFT 0x00000000
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICB_INV_MASK 0x01000000L
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICB_INV__SHIFT 0x00000018
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICB_SEL_MASK 0x0000ff00L
+#define DCCG_TEST_CLK_SEL__DCCG_TEST_CLK_GENERICB_SEL__SHIFT 0x00000008
+#define DCCG_TEST_DEBUG_DATA__DCCG_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define DCCG_TEST_DEBUG_DATA__DCCG_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define DCCG_TEST_DEBUG_INDEX__DCCG_DBG_SEL_MASK 0x00001000L
+#define DCCG_TEST_DEBUG_INDEX__DCCG_DBG_SEL__SHIFT 0x0000000c
+#define DCCG_TEST_DEBUG_INDEX__DCCG_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define DCCG_TEST_DEBUG_INDEX__DCCG_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define DCCG_TEST_DEBUG_INDEX__DCCG_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define DCCG_TEST_DEBUG_INDEX__DCCG_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define DCCG_VPCLK_CNTL__AZ_LIGHT_SLEEP_DIS_MASK 0x00000004L
+#define DCCG_VPCLK_CNTL__AZ_LIGHT_SLEEP_DIS__SHIFT 0x00000002
+#define DCCG_VPCLK_CNTL__AZ_MEM_SHUTDOWN_DIS_MASK 0x04000000L
+#define DCCG_VPCLK_CNTL__AZ_MEM_SHUTDOWN_DIS__SHIFT 0x0000001a
+#define DCCG_VPCLK_CNTL__DCCG_VPCLK_POL_MASK 0x00000001L
+#define DCCG_VPCLK_CNTL__DCCG_VPCLK_POL__SHIFT 0x00000000
+#define DCCG_VPCLK_CNTL__DMCU_LIGHT_SLEEP_DIS_MASK 0x00000008L
+#define DCCG_VPCLK_CNTL__DMCU_LIGHT_SLEEP_DIS__SHIFT 0x00000003
+#define DCCG_VPCLK_CNTL__DMCU_MEM_SHUTDOWN_DIS_MASK 0x00010000L
+#define DCCG_VPCLK_CNTL__DMCU_MEM_SHUTDOWN_DIS__SHIFT 0x00000010
+#define DCCG_VPCLK_CNTL__DMIF0_LIGHT_SLEEP_DIS_MASK 0x00000100L
+#define DCCG_VPCLK_CNTL__DMIF0_LIGHT_SLEEP_DIS__SHIFT 0x00000008
+#define DCCG_VPCLK_CNTL__DMIF0_MEM_SHUTDOWN_DIS_MASK 0x00100000L
+#define DCCG_VPCLK_CNTL__DMIF0_MEM_SHUTDOWN_DIS__SHIFT 0x00000014
+#define DCCG_VPCLK_CNTL__DMIF1_LIGHT_SLEEP_DIS_MASK 0x00000200L
+#define DCCG_VPCLK_CNTL__DMIF1_LIGHT_SLEEP_DIS__SHIFT 0x00000009
+#define DCCG_VPCLK_CNTL__DMIF1_MEM_SHUTDOWN_DIS_MASK 0x00200000L
+#define DCCG_VPCLK_CNTL__DMIF1_MEM_SHUTDOWN_DIS__SHIFT 0x00000015
+#define DCCG_VPCLK_CNTL__DMIF2_LIGHT_SLEEP_DIS_MASK 0x00000400L
+#define DCCG_VPCLK_CNTL__DMIF2_LIGHT_SLEEP_DIS__SHIFT 0x0000000a
+#define DCCG_VPCLK_CNTL__DMIF2_MEM_SHUTDOWN_DIS_MASK 0x00400000L
+#define DCCG_VPCLK_CNTL__DMIF2_MEM_SHUTDOWN_DIS__SHIFT 0x00000016
+#define DCCG_VPCLK_CNTL__DMIF3_LIGHT_SLEEP_DIS_MASK 0x00000800L
+#define DCCG_VPCLK_CNTL__DMIF3_LIGHT_SLEEP_DIS__SHIFT 0x0000000b
+#define DCCG_VPCLK_CNTL__DMIF3_MEM_SHUTDOWN_DIS_MASK 0x00800000L
+#define DCCG_VPCLK_CNTL__DMIF3_MEM_SHUTDOWN_DIS__SHIFT 0x00000017
+#define DCCG_VPCLK_CNTL__DMIF4_LIGHT_SLEEP_DIS_MASK 0x00001000L
+#define DCCG_VPCLK_CNTL__DMIF4_LIGHT_SLEEP_DIS__SHIFT 0x0000000c
+#define DCCG_VPCLK_CNTL__DMIF4_MEM_SHUTDOWN_DIS_MASK 0x01000000L
+#define DCCG_VPCLK_CNTL__DMIF4_MEM_SHUTDOWN_DIS__SHIFT 0x00000018
+#define DCCG_VPCLK_CNTL__DMIF5_LIGHT_SLEEP_DIS_MASK 0x00002000L
+#define DCCG_VPCLK_CNTL__DMIF5_LIGHT_SLEEP_DIS__SHIFT 0x0000000d
+#define DCCG_VPCLK_CNTL__DMIF5_MEM_SHUTDOWN_DIS_MASK 0x02000000L
+#define DCCG_VPCLK_CNTL__DMIF5_MEM_SHUTDOWN_DIS__SHIFT 0x00000019
+#define DCCG_VPCLK_CNTL__DMIF_XLR_LIGHT_SLEEP_MODE_FORCE_MASK 0x00000020L
+#define DCCG_VPCLK_CNTL__DMIF_XLR_LIGHT_SLEEP_MODE_FORCE__SHIFT 0x00000005
+#define DCCG_VPCLK_CNTL__DMIF_XLR_MEM_SHUTDOWN_MODE_FORCE_MASK 0x00040000L
+#define DCCG_VPCLK_CNTL__DMIF_XLR_MEM_SHUTDOWN_MODE_FORCE__SHIFT 0x00000012
+#define DCCG_VPCLK_CNTL__FBC_LIGHT_SLEEP_DIS_MASK 0x00004000L
+#define DCCG_VPCLK_CNTL__FBC_LIGHT_SLEEP_DIS__SHIFT 0x0000000e
+#define DCCG_VPCLK_CNTL__FBC_MEM_SHUTDOWN_DIS_MASK 0x00080000L
+#define DCCG_VPCLK_CNTL__FBC_MEM_SHUTDOWN_DIS__SHIFT 0x00000013
+#define DCCG_VPCLK_CNTL__MCIF_LIGHT_SLEEP_MODE_FORCE_MASK 0x00000010L
+#define DCCG_VPCLK_CNTL__MCIF_LIGHT_SLEEP_MODE_FORCE__SHIFT 0x00000004
+#define DCCG_VPCLK_CNTL__MCIF_MEM_SHUTDOWN_MODE_FORCE_MASK 0x00020000L
+#define DCCG_VPCLK_CNTL__MCIF_MEM_SHUTDOWN_MODE_FORCE__SHIFT 0x00000011
+#define DCCG_VPCLK_CNTL__VGA_LIGHT_SLEEP_MODE_FORCE_MASK 0x00000002L
+#define DCCG_VPCLK_CNTL__VGA_LIGHT_SLEEP_MODE_FORCE__SHIFT 0x00000001
+#define DCCG_VPCLK_CNTL__VIP_LIGHT_SLEEP_DIS_MASK 0x00008000L
+#define DCCG_VPCLK_CNTL__VIP_LIGHT_SLEEP_DIS__SHIFT 0x0000000f
+#define DCDEBUG_BUS_CLK1_SEL__DCDEBUG_BUS_CLK1_SEL_MASK 0xffffffffL
+#define DCDEBUG_BUS_CLK1_SEL__DCDEBUG_BUS_CLK1_SEL__SHIFT 0x00000000
+#define DCDEBUG_BUS_CLK2_SEL__DCDEBUG_BUS_CLK2_SEL_MASK 0xffffffffL
+#define DCDEBUG_BUS_CLK2_SEL__DCDEBUG_BUS_CLK2_SEL__SHIFT 0x00000000
+#define DCDEBUG_BUS_CLK3_SEL__DCDEBUG_BUS_CLK3_SEL_MASK 0xffffffffL
+#define DCDEBUG_BUS_CLK3_SEL__DCDEBUG_BUS_CLK3_SEL__SHIFT 0x00000000
+#define DCDEBUG_BUS_CLK4_SEL__DCDEBUG_BUS_CLK4_SEL_MASK 0xffffffffL
+#define DCDEBUG_BUS_CLK4_SEL__DCDEBUG_BUS_CLK4_SEL__SHIFT 0x00000000
+#define DCDEBUG_OUT_CNTL__DCDEBUG_BLOCK_SEL_MASK 0x0000001fL
+#define DCDEBUG_OUT_CNTL__DCDEBUG_BLOCK_SEL__SHIFT 0x00000000
+#define DCDEBUG_OUT_CNTL__DCDEBUG_OUT_EN_MASK 0x00000020L
+#define DCDEBUG_OUT_CNTL__DCDEBUG_OUT_EN__SHIFT 0x00000005
+#define DCDEBUG_OUT_CNTL__DCDEBUG_OUT_PIN_SEL_MASK 0x00000040L
+#define DCDEBUG_OUT_CNTL__DCDEBUG_OUT_PIN_SEL__SHIFT 0x00000006
+#define DCDEBUG_OUT_CNTL__DCDEBUG_OUT_SEL_MASK 0x00300000L
+#define DCDEBUG_OUT_CNTL__DCDEBUG_OUT_SEL__SHIFT 0x00000014
+#define DCDEBUG_OUT_CNTL__DCDEBUG_OUT_TEST_DATA_EN_MASK 0x00000080L
+#define DCDEBUG_OUT_CNTL__DCDEBUG_OUT_TEST_DATA_EN__SHIFT 0x00000007
+#define DCDEBUG_OUT_CNTL__DCDEBUG_OUT_TEST_DATA_MASK 0x000fff00L
+#define DCDEBUG_OUT_CNTL__DCDEBUG_OUT_TEST_DATA__SHIFT 0x00000008
+#define DCDEBUG_OUT_DATA__DCDEBUG_OUT_DATA_MASK 0xffffffffL
+#define DCDEBUG_OUT_DATA__DCDEBUG_OUT_DATA__SHIFT 0x00000000
+#define DCDEBUG_OUT_PIN_OVERRIDE__DCDEBUG_OUT_OVERRIDE1_EN_MASK 0x00001000L
+#define DCDEBUG_OUT_PIN_OVERRIDE__DCDEBUG_OUT_OVERRIDE1_EN__SHIFT 0x0000000c
+#define DCDEBUG_OUT_PIN_OVERRIDE__DCDEBUG_OUT_OVERRIDE1_PIN_SEL_MASK 0x0000000fL
+#define DCDEBUG_OUT_PIN_OVERRIDE__DCDEBUG_OUT_OVERRIDE1_PIN_SEL__SHIFT 0x00000000
+#define DCDEBUG_OUT_PIN_OVERRIDE__DCDEBUG_OUT_OVERRIDE1_REGBIT_SEL_MASK 0x000001f0L
+#define DCDEBUG_OUT_PIN_OVERRIDE__DCDEBUG_OUT_OVERRIDE1_REGBIT_SEL__SHIFT 0x00000004
+#define DCDEBUG_OUT_PIN_OVERRIDE__DCDEBUG_OUT_OVERRIDE2_EN_MASK 0x10000000L
+#define DCDEBUG_OUT_PIN_OVERRIDE__DCDEBUG_OUT_OVERRIDE2_EN__SHIFT 0x0000001c
+#define DCDEBUG_OUT_PIN_OVERRIDE__DCDEBUG_OUT_OVERRIDE2_PIN_SEL_MASK 0x000f0000L
+#define DCDEBUG_OUT_PIN_OVERRIDE__DCDEBUG_OUT_OVERRIDE2_PIN_SEL__SHIFT 0x00000010
+#define DCDEBUG_OUT_PIN_OVERRIDE__DCDEBUG_OUT_OVERRIDE2_REGBIT_SEL_MASK 0x01f00000L
+#define DCDEBUG_OUT_PIN_OVERRIDE__DCDEBUG_OUT_OVERRIDE2_REGBIT_SEL__SHIFT 0x00000014
+#define DC_DMCU_SCRATCH__DMCU_SCRATCH_MASK 0xffffffffL
+#define DC_DMCU_SCRATCH__DMCU_SCRATCH__SHIFT 0x00000000
+#define DC_DVODATA_CONFIG__DVO_ALTER_MAPPING_EN_MASK 0x00200000L
+#define DC_DVODATA_CONFIG__DVO_ALTER_MAPPING_EN__SHIFT 0x00000015
+#define DC_DVODATA_CONFIG__VIP_ALTER_MAPPING_EN_MASK 0x00100000L
+#define DC_DVODATA_CONFIG__VIP_ALTER_MAPPING_EN__SHIFT 0x00000014
+#define DC_DVODATA_CONFIG__VIP_MUX_EN_MASK 0x00080000L
+#define DC_DVODATA_CONFIG__VIP_MUX_EN__SHIFT 0x00000013
+#define DCFE0_SOFT_RESET__CRTC0_SOFT_RESET_MASK 0x00000010L
+#define DCFE0_SOFT_RESET__CRTC0_SOFT_RESET__SHIFT 0x00000004
+#define DCFE0_SOFT_RESET__DCP0_PIXPIPE_SOFT_RESET_MASK 0x00000001L
+#define DCFE0_SOFT_RESET__DCP0_PIXPIPE_SOFT_RESET__SHIFT 0x00000000
+#define DCFE0_SOFT_RESET__DCP0_REQ_SOFT_RESET_MASK 0x00000002L
+#define DCFE0_SOFT_RESET__DCP0_REQ_SOFT_RESET__SHIFT 0x00000001
+#define DCFE0_SOFT_RESET__SCL0_ALU_SOFT_RESET_MASK 0x00000004L
+#define DCFE0_SOFT_RESET__SCL0_ALU_SOFT_RESET__SHIFT 0x00000002
+#define DCFE0_SOFT_RESET__SCL0_SOFT_RESET_MASK 0x00000008L
+#define DCFE0_SOFT_RESET__SCL0_SOFT_RESET__SHIFT 0x00000003
+#define DCFE1_SOFT_RESET__CRTC1_SOFT_RESET_MASK 0x00000010L
+#define DCFE1_SOFT_RESET__CRTC1_SOFT_RESET__SHIFT 0x00000004
+#define DCFE1_SOFT_RESET__DCP1_PIXPIPE_SOFT_RESET_MASK 0x00000001L
+#define DCFE1_SOFT_RESET__DCP1_PIXPIPE_SOFT_RESET__SHIFT 0x00000000
+#define DCFE1_SOFT_RESET__DCP1_REQ_SOFT_RESET_MASK 0x00000002L
+#define DCFE1_SOFT_RESET__DCP1_REQ_SOFT_RESET__SHIFT 0x00000001
+#define DCFE1_SOFT_RESET__SCL1_ALU_SOFT_RESET_MASK 0x00000004L
+#define DCFE1_SOFT_RESET__SCL1_ALU_SOFT_RESET__SHIFT 0x00000002
+#define DCFE1_SOFT_RESET__SCL1_SOFT_RESET_MASK 0x00000008L
+#define DCFE1_SOFT_RESET__SCL1_SOFT_RESET__SHIFT 0x00000003
+#define DCFE2_SOFT_RESET__CRTC2_SOFT_RESET_MASK 0x00000010L
+#define DCFE2_SOFT_RESET__CRTC2_SOFT_RESET__SHIFT 0x00000004
+#define DCFE2_SOFT_RESET__DCP2_PIXPIPE_SOFT_RESET_MASK 0x00000001L
+#define DCFE2_SOFT_RESET__DCP2_PIXPIPE_SOFT_RESET__SHIFT 0x00000000
+#define DCFE2_SOFT_RESET__DCP2_REQ_SOFT_RESET_MASK 0x00000002L
+#define DCFE2_SOFT_RESET__DCP2_REQ_SOFT_RESET__SHIFT 0x00000001
+#define DCFE2_SOFT_RESET__SCL2_ALU_SOFT_RESET_MASK 0x00000004L
+#define DCFE2_SOFT_RESET__SCL2_ALU_SOFT_RESET__SHIFT 0x00000002
+#define DCFE2_SOFT_RESET__SCL2_SOFT_RESET_MASK 0x00000008L
+#define DCFE2_SOFT_RESET__SCL2_SOFT_RESET__SHIFT 0x00000003
+#define DCFE3_SOFT_RESET__CRTC3_SOFT_RESET_MASK 0x00000010L
+#define DCFE3_SOFT_RESET__CRTC3_SOFT_RESET__SHIFT 0x00000004
+#define DCFE3_SOFT_RESET__DCP3_PIXPIPE_SOFT_RESET_MASK 0x00000001L
+#define DCFE3_SOFT_RESET__DCP3_PIXPIPE_SOFT_RESET__SHIFT 0x00000000
+#define DCFE3_SOFT_RESET__DCP3_REQ_SOFT_RESET_MASK 0x00000002L
+#define DCFE3_SOFT_RESET__DCP3_REQ_SOFT_RESET__SHIFT 0x00000001
+#define DCFE3_SOFT_RESET__SCL3_ALU_SOFT_RESET_MASK 0x00000004L
+#define DCFE3_SOFT_RESET__SCL3_ALU_SOFT_RESET__SHIFT 0x00000002
+#define DCFE3_SOFT_RESET__SCL3_SOFT_RESET_MASK 0x00000008L
+#define DCFE3_SOFT_RESET__SCL3_SOFT_RESET__SHIFT 0x00000003
+#define DCFE4_SOFT_RESET__CRTC4_SOFT_RESET_MASK 0x00000010L
+#define DCFE4_SOFT_RESET__CRTC4_SOFT_RESET__SHIFT 0x00000004
+#define DCFE4_SOFT_RESET__DCP4_PIXPIPE_SOFT_RESET_MASK 0x00000001L
+#define DCFE4_SOFT_RESET__DCP4_PIXPIPE_SOFT_RESET__SHIFT 0x00000000
+#define DCFE4_SOFT_RESET__DCP4_REQ_SOFT_RESET_MASK 0x00000002L
+#define DCFE4_SOFT_RESET__DCP4_REQ_SOFT_RESET__SHIFT 0x00000001
+#define DCFE4_SOFT_RESET__SCL4_ALU_SOFT_RESET_MASK 0x00000004L
+#define DCFE4_SOFT_RESET__SCL4_ALU_SOFT_RESET__SHIFT 0x00000002
+#define DCFE4_SOFT_RESET__SCL4_SOFT_RESET_MASK 0x00000008L
+#define DCFE4_SOFT_RESET__SCL4_SOFT_RESET__SHIFT 0x00000003
+#define DCFE5_SOFT_RESET__CRTC5_SOFT_RESET_MASK 0x00000010L
+#define DCFE5_SOFT_RESET__CRTC5_SOFT_RESET__SHIFT 0x00000004
+#define DCFE5_SOFT_RESET__DCP5_PIXPIPE_SOFT_RESET_MASK 0x00000001L
+#define DCFE5_SOFT_RESET__DCP5_PIXPIPE_SOFT_RESET__SHIFT 0x00000000
+#define DCFE5_SOFT_RESET__DCP5_REQ_SOFT_RESET_MASK 0x00000002L
+#define DCFE5_SOFT_RESET__DCP5_REQ_SOFT_RESET__SHIFT 0x00000001
+#define DCFE5_SOFT_RESET__SCL5_ALU_SOFT_RESET_MASK 0x00000004L
+#define DCFE5_SOFT_RESET__SCL5_ALU_SOFT_RESET__SHIFT 0x00000002
+#define DCFE5_SOFT_RESET__SCL5_SOFT_RESET_MASK 0x00000008L
+#define DCFE5_SOFT_RESET__SCL5_SOFT_RESET__SHIFT 0x00000003
+#define DCFE_DBG_SEL__DCFE_DBG_SEL_MASK 0x0000000fL
+#define DCFE_DBG_SEL__DCFE_DBG_SEL__SHIFT 0x00000000
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__DCP_CURSOR_LIGHT_SLEEP_DIS_MASK 0x00000001L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__DCP_CURSOR_LIGHT_SLEEP_DIS__SHIFT 0x00000000
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__DCP_CURSOR_MEM_PWR_STATE_MASK 0x00000300L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__DCP_CURSOR_MEM_PWR_STATE__SHIFT 0x00000008
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__DCP_LUT_LIGHT_SLEEP_DIS_MASK 0x00000004L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__DCP_LUT_LIGHT_SLEEP_DIS__SHIFT 0x00000002
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__DCP_LUT_MEM_PWR_STATE_MASK 0x00003000L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__DCP_LUT_MEM_PWR_STATE__SHIFT 0x0000000c
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__LB1_MEM_SHUTDOWN_DIS_MASK 0x20000000L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__LB1_MEM_SHUTDOWN_DIS__SHIFT 0x0000001d
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__LB2_MEM_SHUTDOWN_DIS_MASK 0x40000000L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__LB2_MEM_SHUTDOWN_DIS__SHIFT 0x0000001e
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__LB_LIGHT_SLEEP_DIS_MASK 0x00000010L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__LB_LIGHT_SLEEP_DIS__SHIFT 0x00000004
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__LB_MEM_PWR_STATE_0_MASK 0x00030000L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__LB_MEM_PWR_STATE_0__SHIFT 0x00000010
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__LB_MEM_PWR_STATE_1_MASK 0x00c00000L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__LB_MEM_PWR_STATE_1__SHIFT 0x00000016
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__LB_MEM_PWR_STATE_2_MASK 0x03000000L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__LB_MEM_PWR_STATE_2__SHIFT 0x00000018
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__OVLSCL_LIGHT_SLEEP_DIS_MASK 0x00000008L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__OVLSCL_LIGHT_SLEEP_DIS__SHIFT 0x00000003
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__OVLSCL_MEM_PWR_STATE_MASK 0x0000c000L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__OVLSCL_MEM_PWR_STATE__SHIFT 0x0000000e
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__PIPE_MEM_SHUTDOWN_DIS_MASK 0x10000000L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__PIPE_MEM_SHUTDOWN_DIS__SHIFT 0x0000001c
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__REGAMMA_LUT_LIGHT_SLEEP_DIS_MASK 0x00000040L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__REGAMMA_LUT_LIGHT_SLEEP_DIS__SHIFT 0x00000006
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__REGAMMA_LUT_MEM_PWR_STATE_MASK 0x00300000L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__REGAMMA_LUT_MEM_PWR_STATE__SHIFT 0x00000014
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__SCL_LIGHT_SLEEP_DIS_MASK 0x00000020L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__SCL_LIGHT_SLEEP_DIS__SHIFT 0x00000005
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__SCL_MEM_PWR_STATE_MASK 0x000c0000L
+#define DCFE_MEM_LIGHT_SLEEP_CNTL__SCL_MEM_PWR_STATE__SHIFT 0x00000012
+#define DC_GENERICA__GENERICA_EN_MASK 0x00000001L
+#define DC_GENERICA__GENERICA_EN__SHIFT 0x00000000
+#define DC_GENERICA__GENERICA_SEL_MASK 0x00000f00L
+#define DC_GENERICA__GENERICA_SEL__SHIFT 0x00000008
+#define DC_GENERICA__GENERICA_UNIPHY_FBDIV_CLK_DIV2_SEL_MASK 0x07000000L
+#define DC_GENERICA__GENERICA_UNIPHY_FBDIV_CLK_DIV2_SEL__SHIFT 0x00000018
+#define DC_GENERICA__GENERICA_UNIPHY_FBDIV_CLK_SEL_MASK 0x00070000L
+#define DC_GENERICA__GENERICA_UNIPHY_FBDIV_CLK_SEL__SHIFT 0x00000010
+#define DC_GENERICA__GENERICA_UNIPHY_FBDIV_SSC_CLK_SEL_MASK 0x00700000L
+#define DC_GENERICA__GENERICA_UNIPHY_FBDIV_SSC_CLK_SEL__SHIFT 0x00000014
+#define DC_GENERICA__GENERICA_UNIPHY_REFDIV_CLK_SEL_MASK 0x00007000L
+#define DC_GENERICA__GENERICA_UNIPHY_REFDIV_CLK_SEL__SHIFT 0x0000000c
+#define DC_GENERICB__GENERICB_EN_MASK 0x00000001L
+#define DC_GENERICB__GENERICB_EN__SHIFT 0x00000000
+#define DC_GENERICB__GENERICB_SEL_MASK 0x00000f00L
+#define DC_GENERICB__GENERICB_SEL__SHIFT 0x00000008
+#define DC_GENERICB__GENERICB_UNIPHY_FBDIV_CLK_DIV2_SEL_MASK 0x07000000L
+#define DC_GENERICB__GENERICB_UNIPHY_FBDIV_CLK_DIV2_SEL__SHIFT 0x00000018
+#define DC_GENERICB__GENERICB_UNIPHY_FBDIV_CLK_SEL_MASK 0x00070000L
+#define DC_GENERICB__GENERICB_UNIPHY_FBDIV_CLK_SEL__SHIFT 0x00000010
+#define DC_GENERICB__GENERICB_UNIPHY_FBDIV_SSC_CLK_SEL_MASK 0x00700000L
+#define DC_GENERICB__GENERICB_UNIPHY_FBDIV_SSC_CLK_SEL__SHIFT 0x00000014
+#define DC_GENERICB__GENERICB_UNIPHY_REFDIV_CLK_SEL_MASK 0x00007000L
+#define DC_GENERICB__GENERICB_UNIPHY_REFDIV_CLK_SEL__SHIFT 0x0000000c
+#define DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK 0x00000001L
+#define DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A__SHIFT 0x00000000
+#define DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK 0x00000100L
+#define DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A__SHIFT 0x00000008
+#define DC_GPIO_DDC1_EN__DC_GPIO_DDC1CLK_EN_MASK 0x00000001L
+#define DC_GPIO_DDC1_EN__DC_GPIO_DDC1CLK_EN__SHIFT 0x00000000
+#define DC_GPIO_DDC1_EN__DC_GPIO_DDC1DATA_EN_MASK 0x00000100L
+#define DC_GPIO_DDC1_EN__DC_GPIO_DDC1DATA_EN__SHIFT 0x00000008
+#define DC_GPIO_DDC1_MASK__ALLOW_HW_DDC1_PD_EN_MASK 0x00400000L
+#define DC_GPIO_DDC1_MASK__ALLOW_HW_DDC1_PD_EN__SHIFT 0x00000016
+#define DC_GPIO_DDC1_MASK__AUX1_POL_MASK 0x00100000L
+#define DC_GPIO_DDC1_MASK__AUX1_POL__SHIFT 0x00000014
+#define DC_GPIO_DDC1_MASK__AUX_PAD1_MODE_MASK 0x00010000L
+#define DC_GPIO_DDC1_MASK__AUX_PAD1_MODE__SHIFT 0x00000010
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_MASK_MASK 0x00000001L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_MASK__SHIFT 0x00000000
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_PD_EN_MASK 0x00000010L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_PD_EN__SHIFT 0x00000004
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_RECV_MASK 0x00000040L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_RECV__SHIFT 0x00000006
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_STR_MASK 0x0f000000L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1CLK_STR__SHIFT 0x00000018
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_MASK_MASK 0x00000100L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_MASK__SHIFT 0x00000008
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_PD_EN_MASK 0x00001000L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_PD_EN__SHIFT 0x0000000c
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_RECV_MASK 0x00004000L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_RECV__SHIFT 0x0000000e
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_STR_MASK 0xf0000000L
+#define DC_GPIO_DDC1_MASK__DC_GPIO_DDC1DATA_STR__SHIFT 0x0000001c
+#define DC_GPIO_DDC1_Y__DC_GPIO_DDC1CLK_Y_MASK 0x00000001L
+#define DC_GPIO_DDC1_Y__DC_GPIO_DDC1CLK_Y__SHIFT 0x00000000
+#define DC_GPIO_DDC1_Y__DC_GPIO_DDC1DATA_Y_MASK 0x00000100L
+#define DC_GPIO_DDC1_Y__DC_GPIO_DDC1DATA_Y__SHIFT 0x00000008
+#define DC_GPIO_DDC2_A__DC_GPIO_DDC2CLK_A_MASK 0x00000001L
+#define DC_GPIO_DDC2_A__DC_GPIO_DDC2CLK_A__SHIFT 0x00000000
+#define DC_GPIO_DDC2_A__DC_GPIO_DDC2DATA_A_MASK 0x00000100L
+#define DC_GPIO_DDC2_A__DC_GPIO_DDC2DATA_A__SHIFT 0x00000008
+#define DC_GPIO_DDC2_EN__DC_GPIO_DDC2CLK_EN_MASK 0x00000001L
+#define DC_GPIO_DDC2_EN__DC_GPIO_DDC2CLK_EN__SHIFT 0x00000000
+#define DC_GPIO_DDC2_EN__DC_GPIO_DDC2DATA_EN_MASK 0x00000100L
+#define DC_GPIO_DDC2_EN__DC_GPIO_DDC2DATA_EN__SHIFT 0x00000008
+#define DC_GPIO_DDC2_MASK__ALLOW_HW_DDC2_PD_EN_MASK 0x00400000L
+#define DC_GPIO_DDC2_MASK__ALLOW_HW_DDC2_PD_EN__SHIFT 0x00000016
+#define DC_GPIO_DDC2_MASK__AUX2_POL_MASK 0x00100000L
+#define DC_GPIO_DDC2_MASK__AUX2_POL__SHIFT 0x00000014
+#define DC_GPIO_DDC2_MASK__AUX_PAD2_MODE_MASK 0x00010000L
+#define DC_GPIO_DDC2_MASK__AUX_PAD2_MODE__SHIFT 0x00000010
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_MASK_MASK 0x00000001L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_MASK__SHIFT 0x00000000
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_PD_EN_MASK 0x00000010L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_PD_EN__SHIFT 0x00000004
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_RECV_MASK 0x00000040L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_RECV__SHIFT 0x00000006
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_STR_MASK 0x0f000000L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2CLK_STR__SHIFT 0x00000018
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_MASK_MASK 0x00000100L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_MASK__SHIFT 0x00000008
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_PD_EN_MASK 0x00001000L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_PD_EN__SHIFT 0x0000000c
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_RECV_MASK 0x00004000L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_RECV__SHIFT 0x0000000e
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_STR_MASK 0xf0000000L
+#define DC_GPIO_DDC2_MASK__DC_GPIO_DDC2DATA_STR__SHIFT 0x0000001c
+#define DC_GPIO_DDC2_Y__DC_GPIO_DDC2CLK_Y_MASK 0x00000001L
+#define DC_GPIO_DDC2_Y__DC_GPIO_DDC2CLK_Y__SHIFT 0x00000000
+#define DC_GPIO_DDC2_Y__DC_GPIO_DDC2DATA_Y_MASK 0x00000100L
+#define DC_GPIO_DDC2_Y__DC_GPIO_DDC2DATA_Y__SHIFT 0x00000008
+#define DC_GPIO_DDC3_A__DC_GPIO_DDC3CLK_A_MASK 0x00000001L
+#define DC_GPIO_DDC3_A__DC_GPIO_DDC3CLK_A__SHIFT 0x00000000
+#define DC_GPIO_DDC3_A__DC_GPIO_DDC3DATA_A_MASK 0x00000100L
+#define DC_GPIO_DDC3_A__DC_GPIO_DDC3DATA_A__SHIFT 0x00000008
+#define DC_GPIO_DDC3_EN__DC_GPIO_DDC3CLK_EN_MASK 0x00000001L
+#define DC_GPIO_DDC3_EN__DC_GPIO_DDC3CLK_EN__SHIFT 0x00000000
+#define DC_GPIO_DDC3_EN__DC_GPIO_DDC3DATA_EN_MASK 0x00000100L
+#define DC_GPIO_DDC3_EN__DC_GPIO_DDC3DATA_EN__SHIFT 0x00000008
+#define DC_GPIO_DDC3_MASK__ALLOW_HW_DDC3_PD_EN_MASK 0x00400000L
+#define DC_GPIO_DDC3_MASK__ALLOW_HW_DDC3_PD_EN__SHIFT 0x00000016
+#define DC_GPIO_DDC3_MASK__AUX3_POL_MASK 0x00100000L
+#define DC_GPIO_DDC3_MASK__AUX3_POL__SHIFT 0x00000014
+#define DC_GPIO_DDC3_MASK__AUX_PAD3_MODE_MASK 0x00010000L
+#define DC_GPIO_DDC3_MASK__AUX_PAD3_MODE__SHIFT 0x00000010
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_MASK_MASK 0x00000001L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_MASK__SHIFT 0x00000000
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_PD_EN_MASK 0x00000010L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_PD_EN__SHIFT 0x00000004
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_RECV_MASK 0x00000040L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_RECV__SHIFT 0x00000006
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_STR_MASK 0x0f000000L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3CLK_STR__SHIFT 0x00000018
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_MASK_MASK 0x00000100L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_MASK__SHIFT 0x00000008
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_PD_EN_MASK 0x00001000L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_PD_EN__SHIFT 0x0000000c
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_RECV_MASK 0x00004000L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_RECV__SHIFT 0x0000000e
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_STR_MASK 0xf0000000L
+#define DC_GPIO_DDC3_MASK__DC_GPIO_DDC3DATA_STR__SHIFT 0x0000001c
+#define DC_GPIO_DDC3_Y__DC_GPIO_DDC3CLK_Y_MASK 0x00000001L
+#define DC_GPIO_DDC3_Y__DC_GPIO_DDC3CLK_Y__SHIFT 0x00000000
+#define DC_GPIO_DDC3_Y__DC_GPIO_DDC3DATA_Y_MASK 0x00000100L
+#define DC_GPIO_DDC3_Y__DC_GPIO_DDC3DATA_Y__SHIFT 0x00000008
+#define DC_GPIO_DDC4_A__DC_GPIO_DDC4CLK_A_MASK 0x00000001L
+#define DC_GPIO_DDC4_A__DC_GPIO_DDC4CLK_A__SHIFT 0x00000000
+#define DC_GPIO_DDC4_A__DC_GPIO_DDC4DATA_A_MASK 0x00000100L
+#define DC_GPIO_DDC4_A__DC_GPIO_DDC4DATA_A__SHIFT 0x00000008
+#define DC_GPIO_DDC4_EN__DC_GPIO_DDC4CLK_EN_MASK 0x00000001L
+#define DC_GPIO_DDC4_EN__DC_GPIO_DDC4CLK_EN__SHIFT 0x00000000
+#define DC_GPIO_DDC4_EN__DC_GPIO_DDC4DATA_EN_MASK 0x00000100L
+#define DC_GPIO_DDC4_EN__DC_GPIO_DDC4DATA_EN__SHIFT 0x00000008
+#define DC_GPIO_DDC4_MASK__ALLOW_HW_DDC4_PD_EN_MASK 0x00400000L
+#define DC_GPIO_DDC4_MASK__ALLOW_HW_DDC4_PD_EN__SHIFT 0x00000016
+#define DC_GPIO_DDC4_MASK__AUX4_POL_MASK 0x00100000L
+#define DC_GPIO_DDC4_MASK__AUX4_POL__SHIFT 0x00000014
+#define DC_GPIO_DDC4_MASK__AUX_PAD4_MODE_MASK 0x00010000L
+#define DC_GPIO_DDC4_MASK__AUX_PAD4_MODE__SHIFT 0x00000010
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_MASK_MASK 0x00000001L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_MASK__SHIFT 0x00000000
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_PD_EN_MASK 0x00000010L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_PD_EN__SHIFT 0x00000004
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_RECV_MASK 0x00000040L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_RECV__SHIFT 0x00000006
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_STR_MASK 0x0f000000L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4CLK_STR__SHIFT 0x00000018
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_MASK_MASK 0x00000100L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_MASK__SHIFT 0x00000008
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_PD_EN_MASK 0x00001000L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_PD_EN__SHIFT 0x0000000c
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_RECV_MASK 0x00004000L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_RECV__SHIFT 0x0000000e
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_STR_MASK 0xf0000000L
+#define DC_GPIO_DDC4_MASK__DC_GPIO_DDC4DATA_STR__SHIFT 0x0000001c
+#define DC_GPIO_DDC4_Y__DC_GPIO_DDC4CLK_Y_MASK 0x00000001L
+#define DC_GPIO_DDC4_Y__DC_GPIO_DDC4CLK_Y__SHIFT 0x00000000
+#define DC_GPIO_DDC4_Y__DC_GPIO_DDC4DATA_Y_MASK 0x00000100L
+#define DC_GPIO_DDC4_Y__DC_GPIO_DDC4DATA_Y__SHIFT 0x00000008
+#define DC_GPIO_DDC5_A__DC_GPIO_DDC5CLK_A_MASK 0x00000001L
+#define DC_GPIO_DDC5_A__DC_GPIO_DDC5CLK_A__SHIFT 0x00000000
+#define DC_GPIO_DDC5_A__DC_GPIO_DDC5DATA_A_MASK 0x00000100L
+#define DC_GPIO_DDC5_A__DC_GPIO_DDC5DATA_A__SHIFT 0x00000008
+#define DC_GPIO_DDC5_EN__DC_GPIO_DDC5CLK_EN_MASK 0x00000001L
+#define DC_GPIO_DDC5_EN__DC_GPIO_DDC5CLK_EN__SHIFT 0x00000000
+#define DC_GPIO_DDC5_EN__DC_GPIO_DDC5DATA_EN_MASK 0x00000100L
+#define DC_GPIO_DDC5_EN__DC_GPIO_DDC5DATA_EN__SHIFT 0x00000008
+#define DC_GPIO_DDC5_MASK__ALLOW_HW_DDC5_PD_EN_MASK 0x00400000L
+#define DC_GPIO_DDC5_MASK__ALLOW_HW_DDC5_PD_EN__SHIFT 0x00000016
+#define DC_GPIO_DDC5_MASK__AUX5_POL_MASK 0x00100000L
+#define DC_GPIO_DDC5_MASK__AUX5_POL__SHIFT 0x00000014
+#define DC_GPIO_DDC5_MASK__AUX_PAD5_MODE_MASK 0x00010000L
+#define DC_GPIO_DDC5_MASK__AUX_PAD5_MODE__SHIFT 0x00000010
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_MASK_MASK 0x00000001L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_MASK__SHIFT 0x00000000
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_PD_EN_MASK 0x00000010L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_PD_EN__SHIFT 0x00000004
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_RECV_MASK 0x00000040L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_RECV__SHIFT 0x00000006
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_STR_MASK 0x0f000000L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5CLK_STR__SHIFT 0x00000018
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_MASK_MASK 0x00000100L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_MASK__SHIFT 0x00000008
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_PD_EN_MASK 0x00001000L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_PD_EN__SHIFT 0x0000000c
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_RECV_MASK 0x00004000L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_RECV__SHIFT 0x0000000e
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_STR_MASK 0xf0000000L
+#define DC_GPIO_DDC5_MASK__DC_GPIO_DDC5DATA_STR__SHIFT 0x0000001c
+#define DC_GPIO_DDC5_Y__DC_GPIO_DDC5CLK_Y_MASK 0x00000001L
+#define DC_GPIO_DDC5_Y__DC_GPIO_DDC5CLK_Y__SHIFT 0x00000000
+#define DC_GPIO_DDC5_Y__DC_GPIO_DDC5DATA_Y_MASK 0x00000100L
+#define DC_GPIO_DDC5_Y__DC_GPIO_DDC5DATA_Y__SHIFT 0x00000008
+#define DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK 0x00000001L
+#define DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A__SHIFT 0x00000000
+#define DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK 0x00000100L
+#define DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A__SHIFT 0x00000008
+#define DC_GPIO_DDC6_EN__DC_GPIO_DDC6CLK_EN_MASK 0x00000001L
+#define DC_GPIO_DDC6_EN__DC_GPIO_DDC6CLK_EN__SHIFT 0x00000000
+#define DC_GPIO_DDC6_EN__DC_GPIO_DDC6DATA_EN_MASK 0x00000100L
+#define DC_GPIO_DDC6_EN__DC_GPIO_DDC6DATA_EN__SHIFT 0x00000008
+#define DC_GPIO_DDC6_MASK__ALLOW_HW_DDC6_PD_EN_MASK 0x00400000L
+#define DC_GPIO_DDC6_MASK__ALLOW_HW_DDC6_PD_EN__SHIFT 0x00000016
+#define DC_GPIO_DDC6_MASK__AUX6_POL_MASK 0x00100000L
+#define DC_GPIO_DDC6_MASK__AUX6_POL__SHIFT 0x00000014
+#define DC_GPIO_DDC6_MASK__AUX_PAD6_MODE_MASK 0x00010000L
+#define DC_GPIO_DDC6_MASK__AUX_PAD6_MODE__SHIFT 0x00000010
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6CLK_MASK_MASK 0x00000001L
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6CLK_MASK__SHIFT 0x00000000
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6CLK_PD_EN_MASK 0x00000010L
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6CLK_PD_EN__SHIFT 0x00000004
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6CLK_RECV_MASK 0x00000040L
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6CLK_RECV__SHIFT 0x00000006
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6CLK_STR_MASK 0x0f000000L
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6CLK_STR__SHIFT 0x00000018
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6DATA_MASK_MASK 0x00000100L
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6DATA_MASK__SHIFT 0x00000008
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6DATA_PD_EN_MASK 0x00001000L
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6DATA_PD_EN__SHIFT 0x0000000c
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6DATA_RECV_MASK 0x00004000L
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6DATA_RECV__SHIFT 0x0000000e
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6DATA_STR_MASK 0xf0000000L
+#define DC_GPIO_DDC6_MASK__DC_GPIO_DDC6DATA_STR__SHIFT 0x0000001c
+#define DC_GPIO_DDC6_Y__DC_GPIO_DDC6CLK_Y_MASK 0x00000001L
+#define DC_GPIO_DDC6_Y__DC_GPIO_DDC6CLK_Y__SHIFT 0x00000000
+#define DC_GPIO_DDC6_Y__DC_GPIO_DDC6DATA_Y_MASK 0x00000100L
+#define DC_GPIO_DDC6_Y__DC_GPIO_DDC6DATA_Y__SHIFT 0x00000008
+#define DC_GPIO_DDCVGA_A__DC_GPIO_DDCVGACLK_A_MASK 0x00000001L
+#define DC_GPIO_DDCVGA_A__DC_GPIO_DDCVGACLK_A__SHIFT 0x00000000
+#define DC_GPIO_DDCVGA_A__DC_GPIO_DDCVGADATA_A_MASK 0x00000100L
+#define DC_GPIO_DDCVGA_A__DC_GPIO_DDCVGADATA_A__SHIFT 0x00000008
+#define DC_GPIO_DDCVGA_EN__DC_GPIO_DDCVGACLK_EN_MASK 0x00000001L
+#define DC_GPIO_DDCVGA_EN__DC_GPIO_DDCVGACLK_EN__SHIFT 0x00000000
+#define DC_GPIO_DDCVGA_EN__DC_GPIO_DDCVGADATA_EN_MASK 0x00000100L
+#define DC_GPIO_DDCVGA_EN__DC_GPIO_DDCVGADATA_EN__SHIFT 0x00000008
+#define DC_GPIO_DDCVGA_MASK__ALLOW_HW_DDCVGA_PD_EN_MASK 0x00400000L
+#define DC_GPIO_DDCVGA_MASK__ALLOW_HW_DDCVGA_PD_EN__SHIFT 0x00000016
+#define DC_GPIO_DDCVGA_MASK__AUX_PADVGA_MODE_MASK 0x00010000L
+#define DC_GPIO_DDCVGA_MASK__AUX_PADVGA_MODE__SHIFT 0x00000010
+#define DC_GPIO_DDCVGA_MASK__AUXVGA_POL_MASK 0x00100000L
+#define DC_GPIO_DDCVGA_MASK__AUXVGA_POL__SHIFT 0x00000014
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGACLK_MASK_MASK 0x00000001L
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGACLK_MASK__SHIFT 0x00000000
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGACLK_RECV_MASK 0x00000040L
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGACLK_RECV__SHIFT 0x00000006
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGACLK_STR_MASK 0x0f000000L
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGACLK_STR__SHIFT 0x00000018
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_MASK_MASK 0x00000100L
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_MASK__SHIFT 0x00000008
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_PD_EN_MASK 0x00001000L
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_PD_EN__SHIFT 0x0000000c
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_RECV_MASK 0x00004000L
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_RECV__SHIFT 0x0000000e
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_STR_MASK 0xf0000000L
+#define DC_GPIO_DDCVGA_MASK__DC_GPIO_DDCVGADATA_STR__SHIFT 0x0000001c
+#define DC_GPIO_DDCVGA_Y__DC_GPIO_DDCVGACLK_Y_MASK 0x00000001L
+#define DC_GPIO_DDCVGA_Y__DC_GPIO_DDCVGACLK_Y__SHIFT 0x00000000
+#define DC_GPIO_DDCVGA_Y__DC_GPIO_DDCVGADATA_Y_MASK 0x00000100L
+#define DC_GPIO_DDCVGA_Y__DC_GPIO_DDCVGADATA_Y__SHIFT 0x00000008
+#define DC_GPIO_DEBUG__DC_GPIO_CHIP_DEBUG_OUT_PIN_SEL_MASK 0x00010000L
+#define DC_GPIO_DEBUG__DC_GPIO_CHIP_DEBUG_OUT_PIN_SEL__SHIFT 0x00000010
+#define DC_GPIO_DEBUG__DC_GPIO_MACRO_DEBUG_MASK 0x00000300L
+#define DC_GPIO_DEBUG__DC_GPIO_MACRO_DEBUG__SHIFT 0x00000008
+#define DC_GPIO_DEBUG__DC_GPIO_VIP_DEBUG_MASK 0x00000001L
+#define DC_GPIO_DEBUG__DC_GPIO_VIP_DEBUG__SHIFT 0x00000000
+#define DC_GPIO_DVODATA_A__DC_GPIO_DVOCLK_A_MASK 0x10000000L
+#define DC_GPIO_DVODATA_A__DC_GPIO_DVOCLK_A__SHIFT 0x0000001c
+#define DC_GPIO_DVODATA_A__DC_GPIO_DVOCNTL_A_MASK 0x07000000L
+#define DC_GPIO_DVODATA_A__DC_GPIO_DVOCNTL_A__SHIFT 0x00000018
+#define DC_GPIO_DVODATA_A__DC_GPIO_DVODATA_A_MASK 0x00ffffffL
+#define DC_GPIO_DVODATA_A__DC_GPIO_DVODATA_A__SHIFT 0x00000000
+#define DC_GPIO_DVODATA_A__DC_GPIO_MVP_DVOCNTL_A_MASK 0xc0000000L
+#define DC_GPIO_DVODATA_A__DC_GPIO_MVP_DVOCNTL_A__SHIFT 0x0000001e
+#define DC_GPIO_DVODATA_EN__DC_GPIO_DVOCLK_EN_MASK 0x10000000L
+#define DC_GPIO_DVODATA_EN__DC_GPIO_DVOCLK_EN__SHIFT 0x0000001c
+#define DC_GPIO_DVODATA_EN__DC_GPIO_DVOCNTL_EN_MASK 0x07000000L
+#define DC_GPIO_DVODATA_EN__DC_GPIO_DVOCNTL_EN__SHIFT 0x00000018
+#define DC_GPIO_DVODATA_EN__DC_GPIO_DVODATA_EN_MASK 0x00ffffffL
+#define DC_GPIO_DVODATA_EN__DC_GPIO_DVODATA_EN__SHIFT 0x00000000
+#define DC_GPIO_DVODATA_EN__DC_GPIO_MVP_DVOCNTL_EN_MASK 0xc0000000L
+#define DC_GPIO_DVODATA_EN__DC_GPIO_MVP_DVOCNTL_EN__SHIFT 0x0000001e
+#define DC_GPIO_DVODATA_MASK__DC_GPIO_DVOCLK_MASK_MASK 0x10000000L
+#define DC_GPIO_DVODATA_MASK__DC_GPIO_DVOCLK_MASK__SHIFT 0x0000001c
+#define DC_GPIO_DVODATA_MASK__DC_GPIO_DVOCNTL_MASK_MASK 0x07000000L
+#define DC_GPIO_DVODATA_MASK__DC_GPIO_DVOCNTL_MASK__SHIFT 0x00000018
+#define DC_GPIO_DVODATA_MASK__DC_GPIO_DVODATA_MASK_MASK 0x00ffffffL
+#define DC_GPIO_DVODATA_MASK__DC_GPIO_DVODATA_MASK__SHIFT 0x00000000
+#define DC_GPIO_DVODATA_MASK__DC_GPIO_MVP_DVOCNTL_MASK_MASK 0xc0000000L
+#define DC_GPIO_DVODATA_MASK__DC_GPIO_MVP_DVOCNTL_MASK__SHIFT 0x0000001e
+#define DC_GPIO_DVODATA_Y__DC_GPIO_DVOCLK_Y_MASK 0x10000000L
+#define DC_GPIO_DVODATA_Y__DC_GPIO_DVOCLK_Y__SHIFT 0x0000001c
+#define DC_GPIO_DVODATA_Y__DC_GPIO_DVOCNTL_Y_MASK 0x07000000L
+#define DC_GPIO_DVODATA_Y__DC_GPIO_DVOCNTL_Y__SHIFT 0x00000018
+#define DC_GPIO_DVODATA_Y__DC_GPIO_DVODATA_Y_MASK 0x00ffffffL
+#define DC_GPIO_DVODATA_Y__DC_GPIO_DVODATA_Y__SHIFT 0x00000000
+#define DC_GPIO_DVODATA_Y__DC_GPIO_MVP_DVOCNTL_Y_MASK 0xc0000000L
+#define DC_GPIO_DVODATA_Y__DC_GPIO_MVP_DVOCNTL_Y__SHIFT 0x0000001e
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK 0x00000001L
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A__SHIFT 0x00000000
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK 0x00000100L
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A__SHIFT 0x00000008
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK 0x00010000L
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A__SHIFT 0x00000010
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK 0x00100000L
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A__SHIFT 0x00000014
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK 0x00200000L
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A__SHIFT 0x00000015
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK 0x00400000L
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A__SHIFT 0x00000016
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK 0x00800000L
+#define DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A__SHIFT 0x00000017
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICA_EN_MASK 0x00000001L
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICA_EN__SHIFT 0x00000000
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICB_EN_MASK 0x00000100L
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICB_EN__SHIFT 0x00000008
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICC_EN_MASK 0x00010000L
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICC_EN__SHIFT 0x00000010
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICD_EN_MASK 0x00100000L
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICD_EN__SHIFT 0x00000014
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICE_EN_MASK 0x00200000L
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICE_EN__SHIFT 0x00000015
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICF_EN_MASK 0x00400000L
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICF_EN__SHIFT 0x00000016
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICG_EN_MASK 0x00800000L
+#define DC_GPIO_GENERIC_EN__DC_GPIO_GENERICG_EN__SHIFT 0x00000017
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICA_MASK_MASK 0x00000001L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICA_MASK__SHIFT 0x00000000
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICA_PD_DIS_MASK 0x00000002L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICA_PD_DIS__SHIFT 0x00000001
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICA_RECV_MASK 0x00000004L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICA_RECV__SHIFT 0x00000002
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICB_MASK_MASK 0x00000010L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICB_MASK__SHIFT 0x00000004
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICB_PD_DIS_MASK 0x00000020L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICB_PD_DIS__SHIFT 0x00000005
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICB_RECV_MASK 0x00000040L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICB_RECV__SHIFT 0x00000006
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICC_MASK_MASK 0x00000100L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICC_MASK__SHIFT 0x00000008
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICC_PD_DIS_MASK 0x00000200L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICC_PD_DIS__SHIFT 0x00000009
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICC_RECV_MASK 0x00000400L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICC_RECV__SHIFT 0x0000000a
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICD_MASK_MASK 0x00001000L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICD_MASK__SHIFT 0x0000000c
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICD_PD_DIS_MASK 0x00002000L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICD_PD_DIS__SHIFT 0x0000000d
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICD_RECV_MASK 0x00004000L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICD_RECV__SHIFT 0x0000000e
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICE_MASK_MASK 0x00010000L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICE_MASK__SHIFT 0x00000010
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICE_PD_DIS_MASK 0x00020000L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICE_PD_DIS__SHIFT 0x00000011
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICE_RECV_MASK 0x00040000L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICE_RECV__SHIFT 0x00000012
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICF_MASK_MASK 0x00100000L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICF_MASK__SHIFT 0x00000014
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICF_PD_DIS_MASK 0x00200000L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICF_PD_DIS__SHIFT 0x00000015
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICF_RECV_MASK 0x00400000L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICF_RECV__SHIFT 0x00000016
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICG_MASK_MASK 0x01000000L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICG_MASK__SHIFT 0x00000018
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICG_PD_DIS_MASK 0x02000000L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICG_PD_DIS__SHIFT 0x00000019
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICG_RECV_MASK 0x04000000L
+#define DC_GPIO_GENERIC_MASK__DC_GPIO_GENERICG_RECV__SHIFT 0x0000001a
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICA_Y_MASK 0x00000001L
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICA_Y__SHIFT 0x00000000
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICB_Y_MASK 0x00000100L
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICB_Y__SHIFT 0x00000008
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICC_Y_MASK 0x00010000L
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICC_Y__SHIFT 0x00000010
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICD_Y_MASK 0x00100000L
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICD_Y__SHIFT 0x00000014
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICE_Y_MASK 0x00200000L
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICE_Y__SHIFT 0x00000015
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICF_Y_MASK 0x00400000L
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICF_Y__SHIFT 0x00000016
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICG_Y_MASK 0x00800000L
+#define DC_GPIO_GENERIC_Y__DC_GPIO_GENERICG_Y__SHIFT 0x00000017
+#define DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK 0x00000001L
+#define DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A__SHIFT 0x00000000
+#define DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK 0x00000100L
+#define DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A__SHIFT 0x00000008
+#define DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK 0x00010000L
+#define DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A__SHIFT 0x00000010
+#define DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK 0x01000000L
+#define DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A__SHIFT 0x00000018
+#define DC_GPIO_GENLK_EN__DC_GPIO_GENLK_CLK_EN_MASK 0x00000001L
+#define DC_GPIO_GENLK_EN__DC_GPIO_GENLK_CLK_EN__SHIFT 0x00000000
+#define DC_GPIO_GENLK_EN__DC_GPIO_GENLK_VSYNC_EN_MASK 0x00000100L
+#define DC_GPIO_GENLK_EN__DC_GPIO_GENLK_VSYNC_EN__SHIFT 0x00000008
+#define DC_GPIO_GENLK_EN__DC_GPIO_SWAPLOCK_A_EN_MASK 0x00010000L
+#define DC_GPIO_GENLK_EN__DC_GPIO_SWAPLOCK_A_EN__SHIFT 0x00000010
+#define DC_GPIO_GENLK_EN__DC_GPIO_SWAPLOCK_B_EN_MASK 0x01000000L
+#define DC_GPIO_GENLK_EN__DC_GPIO_SWAPLOCK_B_EN__SHIFT 0x00000018
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_CLK_MASK_MASK 0x00000001L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_CLK_MASK__SHIFT 0x00000000
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_CLK_PD_DIS_MASK 0x00000002L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_CLK_PD_DIS__SHIFT 0x00000001
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_CLK_PU_EN_MASK 0x00000008L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_CLK_PU_EN__SHIFT 0x00000003
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_CLK_RECV_MASK 0x00000004L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_CLK_RECV__SHIFT 0x00000002
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_VSYNC_MASK_MASK 0x00000100L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_VSYNC_MASK__SHIFT 0x00000008
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_VSYNC_PD_DIS_MASK 0x00000200L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_VSYNC_PD_DIS__SHIFT 0x00000009
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_VSYNC_PU_EN_MASK 0x00000800L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_VSYNC_PU_EN__SHIFT 0x0000000b
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_VSYNC_RECV_MASK 0x00000400L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_GENLK_VSYNC_RECV__SHIFT 0x0000000a
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_A_MASK_MASK 0x00010000L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_A_MASK__SHIFT 0x00000010
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_A_PD_DIS_MASK 0x00020000L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_A_PD_DIS__SHIFT 0x00000011
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_A_PU_EN_MASK 0x00080000L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_A_PU_EN__SHIFT 0x00000013
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_A_RECV_MASK 0x00040000L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_A_RECV__SHIFT 0x00000012
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_B_MASK_MASK 0x01000000L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_B_MASK__SHIFT 0x00000018
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_B_PD_DIS_MASK 0x02000000L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_B_PD_DIS__SHIFT 0x00000019
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_B_PU_EN_MASK 0x08000000L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_B_PU_EN__SHIFT 0x0000001b
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_B_RECV_MASK 0x04000000L
+#define DC_GPIO_GENLK_MASK__DC_GPIO_SWAPLOCK_B_RECV__SHIFT 0x0000001a
+#define DC_GPIO_GENLK_Y__DC_GPIO_GENLK_CLK_Y_MASK 0x00000001L
+#define DC_GPIO_GENLK_Y__DC_GPIO_GENLK_CLK_Y__SHIFT 0x00000000
+#define DC_GPIO_GENLK_Y__DC_GPIO_GENLK_VSYNC_Y_MASK 0x00000100L
+#define DC_GPIO_GENLK_Y__DC_GPIO_GENLK_VSYNC_Y__SHIFT 0x00000008
+#define DC_GPIO_GENLK_Y__DC_GPIO_SWAPLOCK_A_Y_MASK 0x00010000L
+#define DC_GPIO_GENLK_Y__DC_GPIO_SWAPLOCK_A_Y__SHIFT 0x00000010
+#define DC_GPIO_GENLK_Y__DC_GPIO_SWAPLOCK_B_Y_MASK 0x01000000L
+#define DC_GPIO_GENLK_Y__DC_GPIO_SWAPLOCK_B_Y__SHIFT 0x00000018
+#define DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK 0x00000001L
+#define DC_GPIO_HPD_A__DC_GPIO_HPD1_A__SHIFT 0x00000000
+#define DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK 0x00000100L
+#define DC_GPIO_HPD_A__DC_GPIO_HPD2_A__SHIFT 0x00000008
+#define DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK 0x00010000L
+#define DC_GPIO_HPD_A__DC_GPIO_HPD3_A__SHIFT 0x00000010
+#define DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK 0x01000000L
+#define DC_GPIO_HPD_A__DC_GPIO_HPD4_A__SHIFT 0x00000018
+#define DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK 0x04000000L
+#define DC_GPIO_HPD_A__DC_GPIO_HPD5_A__SHIFT 0x0000001a
+#define DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK 0x10000000L
+#define DC_GPIO_HPD_A__DC_GPIO_HPD6_A__SHIFT 0x0000001c
+#define DC_GPIO_HPD_EN__DC_GPIO_HPD1_EN_MASK 0x00000001L
+#define DC_GPIO_HPD_EN__DC_GPIO_HPD1_EN__SHIFT 0x00000000
+#define DC_GPIO_HPD_EN__DC_GPIO_HPD2_EN_MASK 0x00000100L
+#define DC_GPIO_HPD_EN__DC_GPIO_HPD2_EN__SHIFT 0x00000008
+#define DC_GPIO_HPD_EN__DC_GPIO_HPD3_EN_MASK 0x00010000L
+#define DC_GPIO_HPD_EN__DC_GPIO_HPD3_EN__SHIFT 0x00000010
+#define DC_GPIO_HPD_EN__DC_GPIO_HPD4_EN_MASK 0x01000000L
+#define DC_GPIO_HPD_EN__DC_GPIO_HPD4_EN__SHIFT 0x00000018
+#define DC_GPIO_HPD_EN__DC_GPIO_HPD5_EN_MASK 0x04000000L
+#define DC_GPIO_HPD_EN__DC_GPIO_HPD5_EN__SHIFT 0x0000001a
+#define DC_GPIO_HPD_EN__DC_GPIO_HPD6_EN_MASK 0x10000000L
+#define DC_GPIO_HPD_EN__DC_GPIO_HPD6_EN__SHIFT 0x0000001c
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD1_MASK_MASK 0x00000001L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD1_MASK__SHIFT 0x00000000
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD1_PD_DIS_MASK 0x00000010L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD1_PD_DIS__SHIFT 0x00000004
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD1_RECV_MASK 0x00000040L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD1_RECV__SHIFT 0x00000006
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD2_MASK_MASK 0x00000100L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD2_MASK__SHIFT 0x00000008
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD2_PD_DIS_MASK 0x00000200L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD2_PD_DIS__SHIFT 0x00000009
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD2_RECV_MASK 0x00000400L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD2_RECV__SHIFT 0x0000000a
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD3_MASK_MASK 0x00010000L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD3_MASK__SHIFT 0x00000010
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD3_PD_DIS_MASK 0x00020000L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD3_PD_DIS__SHIFT 0x00000011
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD3_RECV_MASK 0x00040000L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD3_RECV__SHIFT 0x00000012
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD4_MASK_MASK 0x00100000L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD4_MASK__SHIFT 0x00000014
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD4_PD_DIS_MASK 0x00200000L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD4_PD_DIS__SHIFT 0x00000015
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD4_RECV_MASK 0x00400000L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD4_RECV__SHIFT 0x00000016
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD5_MASK_MASK 0x01000000L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD5_MASK__SHIFT 0x00000018
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD5_PD_DIS_MASK 0x02000000L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD5_PD_DIS__SHIFT 0x00000019
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD5_RECV_MASK 0x04000000L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD5_RECV__SHIFT 0x0000001a
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD6_MASK_MASK 0x10000000L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD6_MASK__SHIFT 0x0000001c
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD6_PD_DIS_MASK 0x20000000L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD6_PD_DIS__SHIFT 0x0000001d
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD6_RECV_MASK 0x40000000L
+#define DC_GPIO_HPD_MASK__DC_GPIO_HPD6_RECV__SHIFT 0x0000001e
+#define DC_GPIO_HPD_Y__DC_GPIO_HPD1_Y_MASK 0x00000001L
+#define DC_GPIO_HPD_Y__DC_GPIO_HPD1_Y__SHIFT 0x00000000
+#define DC_GPIO_HPD_Y__DC_GPIO_HPD2_Y_MASK 0x00000100L
+#define DC_GPIO_HPD_Y__DC_GPIO_HPD2_Y__SHIFT 0x00000008
+#define DC_GPIO_HPD_Y__DC_GPIO_HPD3_Y_MASK 0x00010000L
+#define DC_GPIO_HPD_Y__DC_GPIO_HPD3_Y__SHIFT 0x00000010
+#define DC_GPIO_HPD_Y__DC_GPIO_HPD4_Y_MASK 0x01000000L
+#define DC_GPIO_HPD_Y__DC_GPIO_HPD4_Y__SHIFT 0x00000018
+#define DC_GPIO_HPD_Y__DC_GPIO_HPD5_Y_MASK 0x04000000L
+#define DC_GPIO_HPD_Y__DC_GPIO_HPD5_Y__SHIFT 0x0000001a
+#define DC_GPIO_HPD_Y__DC_GPIO_HPD6_Y_MASK 0x10000000L
+#define DC_GPIO_HPD_Y__DC_GPIO_HPD6_Y__SHIFT 0x0000001c
+#define DC_GPIO_I2CPAD_A__DC_GPIO_SCL_A_MASK 0x00000001L
+#define DC_GPIO_I2CPAD_A__DC_GPIO_SCL_A__SHIFT 0x00000000
+#define DC_GPIO_I2CPAD_A__DC_GPIO_SDA_A_MASK 0x00000002L
+#define DC_GPIO_I2CPAD_A__DC_GPIO_SDA_A__SHIFT 0x00000001
+#define DC_GPIO_I2CPAD_EN__DC_GPIO_SCL_EN_MASK 0x00000001L
+#define DC_GPIO_I2CPAD_EN__DC_GPIO_SCL_EN__SHIFT 0x00000000
+#define DC_GPIO_I2CPAD_EN__DC_GPIO_SDA_EN_MASK 0x00000002L
+#define DC_GPIO_I2CPAD_EN__DC_GPIO_SDA_EN__SHIFT 0x00000001
+#define DC_GPIO_I2CPAD_MASK__DC_GPIO_SCL_MASK_MASK 0x00000001L
+#define DC_GPIO_I2CPAD_MASK__DC_GPIO_SCL_MASK__SHIFT 0x00000000
+#define DC_GPIO_I2CPAD_MASK__DC_GPIO_SCL_PD_DIS_MASK 0x00000002L
+#define DC_GPIO_I2CPAD_MASK__DC_GPIO_SCL_PD_DIS__SHIFT 0x00000001
+#define DC_GPIO_I2CPAD_MASK__DC_GPIO_SCL_RECV_MASK 0x00000004L
+#define DC_GPIO_I2CPAD_MASK__DC_GPIO_SCL_RECV__SHIFT 0x00000002
+#define DC_GPIO_I2CPAD_MASK__DC_GPIO_SDA_MASK_MASK 0x00000010L
+#define DC_GPIO_I2CPAD_MASK__DC_GPIO_SDA_MASK__SHIFT 0x00000004
+#define DC_GPIO_I2CPAD_MASK__DC_GPIO_SDA_PD_DIS_MASK 0x00000020L
+#define DC_GPIO_I2CPAD_MASK__DC_GPIO_SDA_PD_DIS__SHIFT 0x00000005
+#define DC_GPIO_I2CPAD_MASK__DC_GPIO_SDA_RECV_MASK 0x00000040L
+#define DC_GPIO_I2CPAD_MASK__DC_GPIO_SDA_RECV__SHIFT 0x00000006
+#define DC_GPIO_I2CPAD_STRENGTH__I2C_STRENGTH_SN_MASK 0x0000000fL
+#define DC_GPIO_I2CPAD_STRENGTH__I2C_STRENGTH_SN__SHIFT 0x00000000
+#define DC_GPIO_I2CPAD_STRENGTH__I2C_STRENGTH_SP_MASK 0x000000f0L
+#define DC_GPIO_I2CPAD_STRENGTH__I2C_STRENGTH_SP__SHIFT 0x00000004
+#define DC_GPIO_I2CPAD_Y__DC_GPIO_SCL_Y_MASK 0x00000001L
+#define DC_GPIO_I2CPAD_Y__DC_GPIO_SCL_Y__SHIFT 0x00000000
+#define DC_GPIO_I2CPAD_Y__DC_GPIO_SDA_Y_MASK 0x00000002L
+#define DC_GPIO_I2CPAD_Y__DC_GPIO_SDA_Y__SHIFT 0x00000001
+#define DC_GPIO_PAD_STRENGTH_1__GENLK_STRENGTH_SN_MASK 0x0000000fL
+#define DC_GPIO_PAD_STRENGTH_1__GENLK_STRENGTH_SN__SHIFT 0x00000000
+#define DC_GPIO_PAD_STRENGTH_1__GENLK_STRENGTH_SP_MASK 0x000000f0L
+#define DC_GPIO_PAD_STRENGTH_1__GENLK_STRENGTH_SP__SHIFT 0x00000004
+#define DC_GPIO_PAD_STRENGTH_1__SYNC_STRENGTH_SN_MASK 0x0f000000L
+#define DC_GPIO_PAD_STRENGTH_1__SYNC_STRENGTH_SN__SHIFT 0x00000018
+#define DC_GPIO_PAD_STRENGTH_1__SYNC_STRENGTH_SP_MASK 0xf0000000L
+#define DC_GPIO_PAD_STRENGTH_1__SYNC_STRENGTH_SP__SHIFT 0x0000001c
+#define DC_GPIO_PAD_STRENGTH_2__PWRSEQ_STRENGTH_SN_MASK 0x000f0000L
+#define DC_GPIO_PAD_STRENGTH_2__PWRSEQ_STRENGTH_SN__SHIFT 0x00000010
+#define DC_GPIO_PAD_STRENGTH_2__PWRSEQ_STRENGTH_SP_MASK 0x00f00000L
+#define DC_GPIO_PAD_STRENGTH_2__PWRSEQ_STRENGTH_SP__SHIFT 0x00000014
+#define DC_GPIO_PAD_STRENGTH_2__STRENGTH_SN_MASK 0x0000000fL
+#define DC_GPIO_PAD_STRENGTH_2__STRENGTH_SN__SHIFT 0x00000000
+#define DC_GPIO_PAD_STRENGTH_2__STRENGTH_SP_MASK 0x000000f0L
+#define DC_GPIO_PAD_STRENGTH_2__STRENGTH_SP__SHIFT 0x00000004
+#define DC_GPIO_PWRSEQ_A__DC_GPIO_BLON_A_MASK 0x00000001L
+#define DC_GPIO_PWRSEQ_A__DC_GPIO_BLON_A__SHIFT 0x00000000
+#define DC_GPIO_PWRSEQ_A__DC_GPIO_DIGON_A_MASK 0x00000100L
+#define DC_GPIO_PWRSEQ_A__DC_GPIO_DIGON_A__SHIFT 0x00000008
+#define DC_GPIO_PWRSEQ_A__DC_GPIO_ENA_BL_A_MASK 0x00010000L
+#define DC_GPIO_PWRSEQ_A__DC_GPIO_ENA_BL_A__SHIFT 0x00000010
+#define DC_GPIO_PWRSEQ_EN__DC_GPIO_BLON_EN_MASK 0x00000001L
+#define DC_GPIO_PWRSEQ_EN__DC_GPIO_BLON_EN__SHIFT 0x00000000
+#define DC_GPIO_PWRSEQ_EN__DC_GPIO_DIGON_EN_MASK 0x00000100L
+#define DC_GPIO_PWRSEQ_EN__DC_GPIO_DIGON_EN__SHIFT 0x00000008
+#define DC_GPIO_PWRSEQ_EN__DC_GPIO_ENA_BL_EN_MASK 0x00010000L
+#define DC_GPIO_PWRSEQ_EN__DC_GPIO_ENA_BL_EN__SHIFT 0x00000010
+#define DC_GPIO_PWRSEQ_EN__DC_GPIO_VARY_BL_GENERICA_EN_MASK 0x00000002L
+#define DC_GPIO_PWRSEQ_EN__DC_GPIO_VARY_BL_GENERICA_EN__SHIFT 0x00000001
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_BLON_MASK_MASK 0x00000001L
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_BLON_MASK__SHIFT 0x00000000
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_BLON_PD_DIS_MASK 0x00000010L
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_BLON_PD_DIS__SHIFT 0x00000004
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_BLON_RECV_MASK 0x00000040L
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_BLON_RECV__SHIFT 0x00000006
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_DIGON_MASK_MASK 0x00000100L
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_DIGON_MASK__SHIFT 0x00000008
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_DIGON_PD_DIS_MASK 0x00001000L
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_DIGON_PD_DIS__SHIFT 0x0000000c
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_DIGON_RECV_MASK 0x00004000L
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_DIGON_RECV__SHIFT 0x0000000e
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_ENA_BL_MASK_MASK 0x00010000L
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_ENA_BL_MASK__SHIFT 0x00000010
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_ENA_BL_PD_DIS_MASK 0x00100000L
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_ENA_BL_PD_DIS__SHIFT 0x00000014
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_ENA_BL_RECV_MASK 0x00400000L
+#define DC_GPIO_PWRSEQ_MASK__DC_GPIO_ENA_BL_RECV__SHIFT 0x00000016
+#define DC_GPIO_PWRSEQ_Y__DC_GPIO_BLON_Y_MASK 0x00000001L
+#define DC_GPIO_PWRSEQ_Y__DC_GPIO_BLON_Y__SHIFT 0x00000000
+#define DC_GPIO_PWRSEQ_Y__DC_GPIO_DIGON_Y_MASK 0x00000100L
+#define DC_GPIO_PWRSEQ_Y__DC_GPIO_DIGON_Y__SHIFT 0x00000008
+#define DC_GPIO_PWRSEQ_Y__DC_GPIO_ENA_BL_Y_MASK 0x00010000L
+#define DC_GPIO_PWRSEQ_Y__DC_GPIO_ENA_BL_Y__SHIFT 0x00000010
+#define DC_GPIO_SYNCA_A__DC_GPIO_HSYNCA_A_MASK 0x00000001L
+#define DC_GPIO_SYNCA_A__DC_GPIO_HSYNCA_A__SHIFT 0x00000000
+#define DC_GPIO_SYNCA_A__DC_GPIO_VSYNCA_A_MASK 0x00000100L
+#define DC_GPIO_SYNCA_A__DC_GPIO_VSYNCA_A__SHIFT 0x00000008
+#define DC_GPIO_SYNCA_EN__DC_GPIO_HSYNCA_EN_MASK 0x00000001L
+#define DC_GPIO_SYNCA_EN__DC_GPIO_HSYNCA_EN__SHIFT 0x00000000
+#define DC_GPIO_SYNCA_EN__DC_GPIO_VSYNCA_EN_MASK 0x00000100L
+#define DC_GPIO_SYNCA_EN__DC_GPIO_VSYNCA_EN__SHIFT 0x00000008
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_HSYNCA_CRTC_HSYNC_MASK_MASK 0x07000000L
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_HSYNCA_CRTC_HSYNC_MASK__SHIFT 0x00000018
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_HSYNCA_MASK_MASK 0x00000001L
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_HSYNCA_MASK__SHIFT 0x00000000
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_HSYNCA_PD_DIS_MASK 0x00000010L
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_HSYNCA_PD_DIS__SHIFT 0x00000004
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_HSYNCA_RECV_MASK 0x00000040L
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_HSYNCA_RECV__SHIFT 0x00000006
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_VSYNCA_CRTC_VSYNC_MASK_MASK 0x70000000L
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_VSYNCA_CRTC_VSYNC_MASK__SHIFT 0x0000001c
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_VSYNCA_MASK_MASK 0x00000100L
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_VSYNCA_MASK__SHIFT 0x00000008
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_VSYNCA_PD_DIS_MASK 0x00001000L
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_VSYNCA_PD_DIS__SHIFT 0x0000000c
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_VSYNCA_RECV_MASK 0x00004000L
+#define DC_GPIO_SYNCA_MASK__DC_GPIO_VSYNCA_RECV__SHIFT 0x0000000e
+#define DC_GPIO_SYNCA_Y__DC_GPIO_HSYNCA_Y_MASK 0x00000001L
+#define DC_GPIO_SYNCA_Y__DC_GPIO_HSYNCA_Y__SHIFT 0x00000000
+#define DC_GPIO_SYNCA_Y__DC_GPIO_VSYNCA_Y_MASK 0x00000100L
+#define DC_GPIO_SYNCA_Y__DC_GPIO_VSYNCA_Y__SHIFT 0x00000008
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_READ_SELECT_MASK 0x0000003fL
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_READ_SELECT__SHIFT 0x00000000
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D1_VSYNC_NOM_MASK 0x00000700L
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D1_VSYNC_NOM__SHIFT 0x00000008
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D2_VSYNC_NOM_MASK 0x00003800L
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D2_VSYNC_NOM__SHIFT 0x0000000b
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D3_VSYNC_NOM_MASK 0x0001c000L
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D3_VSYNC_NOM__SHIFT 0x0000000e
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D4_VSYNC_NOM_MASK 0x000e0000L
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D4_VSYNC_NOM__SHIFT 0x00000011
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D5_VSYNC_NOM_MASK 0x00700000L
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D5_VSYNC_NOM__SHIFT 0x00000014
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D6_VSYNC_NOM_MASK 0x03800000L
+#define DC_GPU_TIMER_READ_CNTL__DC_GPU_TIMER_START_POSITION_D6_VSYNC_NOM__SHIFT 0x00000017
+#define DC_GPU_TIMER_READ__DC_GPU_TIMER_READ_MASK 0xffffffffL
+#define DC_GPU_TIMER_READ__DC_GPU_TIMER_READ__SHIFT 0x00000000
+#define DC_GPU_TIMER_START_POSITION_P_FLIP__DC_GPU_TIMER_START_POSITION_D1_P_FLIP_MASK 0x00000007L
+#define DC_GPU_TIMER_START_POSITION_P_FLIP__DC_GPU_TIMER_START_POSITION_D1_P_FLIP__SHIFT 0x00000000
+#define DC_GPU_TIMER_START_POSITION_P_FLIP__DC_GPU_TIMER_START_POSITION_D2_P_FLIP_MASK 0x00000070L
+#define DC_GPU_TIMER_START_POSITION_P_FLIP__DC_GPU_TIMER_START_POSITION_D2_P_FLIP__SHIFT 0x00000004
+#define DC_GPU_TIMER_START_POSITION_P_FLIP__DC_GPU_TIMER_START_POSITION_D3_P_FLIP_MASK 0x00000700L
+#define DC_GPU_TIMER_START_POSITION_P_FLIP__DC_GPU_TIMER_START_POSITION_D3_P_FLIP__SHIFT 0x00000008
+#define DC_GPU_TIMER_START_POSITION_P_FLIP__DC_GPU_TIMER_START_POSITION_D4_P_FLIP_MASK 0x00007000L
+#define DC_GPU_TIMER_START_POSITION_P_FLIP__DC_GPU_TIMER_START_POSITION_D4_P_FLIP__SHIFT 0x0000000c
+#define DC_GPU_TIMER_START_POSITION_P_FLIP__DC_GPU_TIMER_START_POSITION_D5_P_FLIP_MASK 0x00070000L
+#define DC_GPU_TIMER_START_POSITION_P_FLIP__DC_GPU_TIMER_START_POSITION_D5_P_FLIP__SHIFT 0x00000010
+#define DC_GPU_TIMER_START_POSITION_P_FLIP__DC_GPU_TIMER_START_POSITION_D6_P_FLIP_MASK 0x00700000L
+#define DC_GPU_TIMER_START_POSITION_P_FLIP__DC_GPU_TIMER_START_POSITION_D6_P_FLIP__SHIFT 0x00000014
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D1_V_UPDATE_MASK 0x00000007L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D1_V_UPDATE__SHIFT 0x00000000
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D2_V_UPDATE_MASK 0x00000070L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D2_V_UPDATE__SHIFT 0x00000004
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D3_V_UPDATE_MASK 0x00000700L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D3_V_UPDATE__SHIFT 0x00000008
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D4_V_UPDATE_MASK 0x00007000L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D4_V_UPDATE__SHIFT 0x0000000c
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D5_V_UPDATE_MASK 0x00070000L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D5_V_UPDATE__SHIFT 0x00000010
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D6_V_UPDATE_MASK 0x00700000L
+#define DC_GPU_TIMER_START_POSITION_V_UPDATE__DC_GPU_TIMER_START_POSITION_D6_V_UPDATE__SHIFT 0x00000014
+#define DC_HPD1_CONTROL__DC_HPD1_CONNECTION_TIMER_MASK 0x00001fffL
+#define DC_HPD1_CONTROL__DC_HPD1_CONNECTION_TIMER__SHIFT 0x00000000
+#define DC_HPD1_CONTROL__DC_HPD1_EN_MASK 0x10000000L
+#define DC_HPD1_CONTROL__DC_HPD1_EN__SHIFT 0x0000001c
+#define DC_HPD1_CONTROL__DC_HPD1_RX_INT_TIMER_MASK 0x03ff0000L
+#define DC_HPD1_CONTROL__DC_HPD1_RX_INT_TIMER__SHIFT 0x00000010
+#define DC_HPD1_FAST_TRAIN_CNTL__DC_HPD1_CONNECT_AUX_TX_DELAY_MASK 0x000000ffL
+#define DC_HPD1_FAST_TRAIN_CNTL__DC_HPD1_CONNECT_AUX_TX_DELAY__SHIFT 0x00000000
+#define DC_HPD1_FAST_TRAIN_CNTL__DC_HPD1_CONNECT_AUX_TX_EN_MASK 0x01000000L
+#define DC_HPD1_FAST_TRAIN_CNTL__DC_HPD1_CONNECT_AUX_TX_EN__SHIFT 0x00000018
+#define DC_HPD1_FAST_TRAIN_CNTL__DC_HPD1_CONNECT_FAST_TRAIN_DELAY_MASK 0x000ff000L
+#define DC_HPD1_FAST_TRAIN_CNTL__DC_HPD1_CONNECT_FAST_TRAIN_DELAY__SHIFT 0x0000000c
+#define DC_HPD1_FAST_TRAIN_CNTL__DC_HPD1_CONNECT_FAST_TRAIN_EN_MASK 0x10000000L
+#define DC_HPD1_FAST_TRAIN_CNTL__DC_HPD1_CONNECT_FAST_TRAIN_EN__SHIFT 0x0000001c
+#define DC_HPD1_INT_CONTROL__DC_HPD1_INT_ACK_MASK 0x00000001L
+#define DC_HPD1_INT_CONTROL__DC_HPD1_INT_ACK__SHIFT 0x00000000
+#define DC_HPD1_INT_CONTROL__DC_HPD1_INT_EN_MASK 0x00010000L
+#define DC_HPD1_INT_CONTROL__DC_HPD1_INT_EN__SHIFT 0x00000010
+#define DC_HPD1_INT_CONTROL__DC_HPD1_INT_POLARITY_MASK 0x00000100L
+#define DC_HPD1_INT_CONTROL__DC_HPD1_INT_POLARITY__SHIFT 0x00000008
+#define DC_HPD1_INT_CONTROL__DC_HPD1_RX_INT_ACK_MASK 0x00100000L
+#define DC_HPD1_INT_CONTROL__DC_HPD1_RX_INT_ACK__SHIFT 0x00000014
+#define DC_HPD1_INT_CONTROL__DC_HPD1_RX_INT_EN_MASK 0x01000000L
+#define DC_HPD1_INT_CONTROL__DC_HPD1_RX_INT_EN__SHIFT 0x00000018
+#define DC_HPD1_INT_STATUS__DC_HPD1_INT_STATUS_MASK 0x00000001L
+#define DC_HPD1_INT_STATUS__DC_HPD1_INT_STATUS__SHIFT 0x00000000
+#define DC_HPD1_INT_STATUS__DC_HPD1_RX_INT_STATUS_MASK 0x00000100L
+#define DC_HPD1_INT_STATUS__DC_HPD1_RX_INT_STATUS__SHIFT 0x00000008
+#define DC_HPD1_INT_STATUS__DC_HPD1_SENSE_DELAYED_MASK 0x00000010L
+#define DC_HPD1_INT_STATUS__DC_HPD1_SENSE_DELAYED__SHIFT 0x00000004
+#define DC_HPD1_INT_STATUS__DC_HPD1_SENSE_MASK 0x00000002L
+#define DC_HPD1_INT_STATUS__DC_HPD1_SENSE__SHIFT 0x00000001
+#define DC_HPD1_INT_STATUS__DC_HPD1_TOGGLE_FILT_CON_TIMER_VAL_MASK 0x000ff000L
+#define DC_HPD1_INT_STATUS__DC_HPD1_TOGGLE_FILT_CON_TIMER_VAL__SHIFT 0x0000000c
+#define DC_HPD1_INT_STATUS__DC_HPD1_TOGGLE_FILT_DISCON_TIMER_VAL_MASK 0xff000000L
+#define DC_HPD1_INT_STATUS__DC_HPD1_TOGGLE_FILT_DISCON_TIMER_VAL__SHIFT 0x00000018
+#define DC_HPD1_TOGGLE_FILT_CNTL__DC_HPD1_CONNECT_INT_DELAY_MASK 0x000000ffL
+#define DC_HPD1_TOGGLE_FILT_CNTL__DC_HPD1_CONNECT_INT_DELAY__SHIFT 0x00000000
+#define DC_HPD1_TOGGLE_FILT_CNTL__DC_HPD1_DISCONNECT_INT_DELAY_MASK 0x0ff00000L
+#define DC_HPD1_TOGGLE_FILT_CNTL__DC_HPD1_DISCONNECT_INT_DELAY__SHIFT 0x00000014
+#define DC_HPD2_CONTROL__DC_HPD2_CONNECTION_TIMER_MASK 0x00001fffL
+#define DC_HPD2_CONTROL__DC_HPD2_CONNECTION_TIMER__SHIFT 0x00000000
+#define DC_HPD2_CONTROL__DC_HPD2_EN_MASK 0x10000000L
+#define DC_HPD2_CONTROL__DC_HPD2_EN__SHIFT 0x0000001c
+#define DC_HPD2_CONTROL__DC_HPD2_RX_INT_TIMER_MASK 0x03ff0000L
+#define DC_HPD2_CONTROL__DC_HPD2_RX_INT_TIMER__SHIFT 0x00000010
+#define DC_HPD2_FAST_TRAIN_CNTL__DC_HPD2_CONNECT_AUX_TX_DELAY_MASK 0x000000ffL
+#define DC_HPD2_FAST_TRAIN_CNTL__DC_HPD2_CONNECT_AUX_TX_DELAY__SHIFT 0x00000000
+#define DC_HPD2_FAST_TRAIN_CNTL__DC_HPD2_CONNECT_AUX_TX_EN_MASK 0x01000000L
+#define DC_HPD2_FAST_TRAIN_CNTL__DC_HPD2_CONNECT_AUX_TX_EN__SHIFT 0x00000018
+#define DC_HPD2_FAST_TRAIN_CNTL__DC_HPD2_CONNECT_FAST_TRAIN_DELAY_MASK 0x000ff000L
+#define DC_HPD2_FAST_TRAIN_CNTL__DC_HPD2_CONNECT_FAST_TRAIN_DELAY__SHIFT 0x0000000c
+#define DC_HPD2_FAST_TRAIN_CNTL__DC_HPD2_CONNECT_FAST_TRAIN_EN_MASK 0x10000000L
+#define DC_HPD2_FAST_TRAIN_CNTL__DC_HPD2_CONNECT_FAST_TRAIN_EN__SHIFT 0x0000001c
+#define DC_HPD2_INT_CONTROL__DC_HPD2_INT_ACK_MASK 0x00000001L
+#define DC_HPD2_INT_CONTROL__DC_HPD2_INT_ACK__SHIFT 0x00000000
+#define DC_HPD2_INT_CONTROL__DC_HPD2_INT_EN_MASK 0x00010000L
+#define DC_HPD2_INT_CONTROL__DC_HPD2_INT_EN__SHIFT 0x00000010
+#define DC_HPD2_INT_CONTROL__DC_HPD2_INT_POLARITY_MASK 0x00000100L
+#define DC_HPD2_INT_CONTROL__DC_HPD2_INT_POLARITY__SHIFT 0x00000008
+#define DC_HPD2_INT_CONTROL__DC_HPD2_RX_INT_ACK_MASK 0x00100000L
+#define DC_HPD2_INT_CONTROL__DC_HPD2_RX_INT_ACK__SHIFT 0x00000014
+#define DC_HPD2_INT_CONTROL__DC_HPD2_RX_INT_EN_MASK 0x01000000L
+#define DC_HPD2_INT_CONTROL__DC_HPD2_RX_INT_EN__SHIFT 0x00000018
+#define DC_HPD2_INT_STATUS__DC_HPD2_INT_STATUS_MASK 0x00000001L
+#define DC_HPD2_INT_STATUS__DC_HPD2_INT_STATUS__SHIFT 0x00000000
+#define DC_HPD2_INT_STATUS__DC_HPD2_RX_INT_STATUS_MASK 0x00000100L
+#define DC_HPD2_INT_STATUS__DC_HPD2_RX_INT_STATUS__SHIFT 0x00000008
+#define DC_HPD2_INT_STATUS__DC_HPD2_SENSE_DELAYED_MASK 0x00000010L
+#define DC_HPD2_INT_STATUS__DC_HPD2_SENSE_DELAYED__SHIFT 0x00000004
+#define DC_HPD2_INT_STATUS__DC_HPD2_SENSE_MASK 0x00000002L
+#define DC_HPD2_INT_STATUS__DC_HPD2_SENSE__SHIFT 0x00000001
+#define DC_HPD2_INT_STATUS__DC_HPD2_TOGGLE_FILT_CON_TIMER_VAL_MASK 0x000ff000L
+#define DC_HPD2_INT_STATUS__DC_HPD2_TOGGLE_FILT_CON_TIMER_VAL__SHIFT 0x0000000c
+#define DC_HPD2_INT_STATUS__DC_HPD2_TOGGLE_FILT_DISCON_TIMER_VAL_MASK 0xff000000L
+#define DC_HPD2_INT_STATUS__DC_HPD2_TOGGLE_FILT_DISCON_TIMER_VAL__SHIFT 0x00000018
+#define DC_HPD2_TOGGLE_FILT_CNTL__DC_HPD2_CONNECT_INT_DELAY_MASK 0x000000ffL
+#define DC_HPD2_TOGGLE_FILT_CNTL__DC_HPD2_CONNECT_INT_DELAY__SHIFT 0x00000000
+#define DC_HPD2_TOGGLE_FILT_CNTL__DC_HPD2_DISCONNECT_INT_DELAY_MASK 0x0ff00000L
+#define DC_HPD2_TOGGLE_FILT_CNTL__DC_HPD2_DISCONNECT_INT_DELAY__SHIFT 0x00000014
+#define DC_HPD3_CONTROL__DC_HPD3_CONNECTION_TIMER_MASK 0x00001fffL
+#define DC_HPD3_CONTROL__DC_HPD3_CONNECTION_TIMER__SHIFT 0x00000000
+#define DC_HPD3_CONTROL__DC_HPD3_EN_MASK 0x10000000L
+#define DC_HPD3_CONTROL__DC_HPD3_EN__SHIFT 0x0000001c
+#define DC_HPD3_CONTROL__DC_HPD3_RX_INT_TIMER_MASK 0x03ff0000L
+#define DC_HPD3_CONTROL__DC_HPD3_RX_INT_TIMER__SHIFT 0x00000010
+#define DC_HPD3_FAST_TRAIN_CNTL__DC_HPD3_CONNECT_AUX_TX_DELAY_MASK 0x000000ffL
+#define DC_HPD3_FAST_TRAIN_CNTL__DC_HPD3_CONNECT_AUX_TX_DELAY__SHIFT 0x00000000
+#define DC_HPD3_FAST_TRAIN_CNTL__DC_HPD3_CONNECT_AUX_TX_EN_MASK 0x01000000L
+#define DC_HPD3_FAST_TRAIN_CNTL__DC_HPD3_CONNECT_AUX_TX_EN__SHIFT 0x00000018
+#define DC_HPD3_FAST_TRAIN_CNTL__DC_HPD3_CONNECT_FAST_TRAIN_DELAY_MASK 0x000ff000L
+#define DC_HPD3_FAST_TRAIN_CNTL__DC_HPD3_CONNECT_FAST_TRAIN_DELAY__SHIFT 0x0000000c
+#define DC_HPD3_FAST_TRAIN_CNTL__DC_HPD3_CONNECT_FAST_TRAIN_EN_MASK 0x10000000L
+#define DC_HPD3_FAST_TRAIN_CNTL__DC_HPD3_CONNECT_FAST_TRAIN_EN__SHIFT 0x0000001c
+#define DC_HPD3_INT_CONTROL__DC_HPD3_INT_ACK_MASK 0x00000001L
+#define DC_HPD3_INT_CONTROL__DC_HPD3_INT_ACK__SHIFT 0x00000000
+#define DC_HPD3_INT_CONTROL__DC_HPD3_INT_EN_MASK 0x00010000L
+#define DC_HPD3_INT_CONTROL__DC_HPD3_INT_EN__SHIFT 0x00000010
+#define DC_HPD3_INT_CONTROL__DC_HPD3_INT_POLARITY_MASK 0x00000100L
+#define DC_HPD3_INT_CONTROL__DC_HPD3_INT_POLARITY__SHIFT 0x00000008
+#define DC_HPD3_INT_CONTROL__DC_HPD3_RX_INT_ACK_MASK 0x00100000L
+#define DC_HPD3_INT_CONTROL__DC_HPD3_RX_INT_ACK__SHIFT 0x00000014
+#define DC_HPD3_INT_CONTROL__DC_HPD3_RX_INT_EN_MASK 0x01000000L
+#define DC_HPD3_INT_CONTROL__DC_HPD3_RX_INT_EN__SHIFT 0x00000018
+#define DC_HPD3_INT_STATUS__DC_HPD3_INT_STATUS_MASK 0x00000001L
+#define DC_HPD3_INT_STATUS__DC_HPD3_INT_STATUS__SHIFT 0x00000000
+#define DC_HPD3_INT_STATUS__DC_HPD3_RX_INT_STATUS_MASK 0x00000100L
+#define DC_HPD3_INT_STATUS__DC_HPD3_RX_INT_STATUS__SHIFT 0x00000008
+#define DC_HPD3_INT_STATUS__DC_HPD3_SENSE_DELAYED_MASK 0x00000010L
+#define DC_HPD3_INT_STATUS__DC_HPD3_SENSE_DELAYED__SHIFT 0x00000004
+#define DC_HPD3_INT_STATUS__DC_HPD3_SENSE_MASK 0x00000002L
+#define DC_HPD3_INT_STATUS__DC_HPD3_SENSE__SHIFT 0x00000001
+#define DC_HPD3_INT_STATUS__DC_HPD3_TOGGLE_FILT_CON_TIMER_VAL_MASK 0x000ff000L
+#define DC_HPD3_INT_STATUS__DC_HPD3_TOGGLE_FILT_CON_TIMER_VAL__SHIFT 0x0000000c
+#define DC_HPD3_INT_STATUS__DC_HPD3_TOGGLE_FILT_DISCON_TIMER_VAL_MASK 0xff000000L
+#define DC_HPD3_INT_STATUS__DC_HPD3_TOGGLE_FILT_DISCON_TIMER_VAL__SHIFT 0x00000018
+#define DC_HPD3_TOGGLE_FILT_CNTL__DC_HPD3_CONNECT_INT_DELAY_MASK 0x000000ffL
+#define DC_HPD3_TOGGLE_FILT_CNTL__DC_HPD3_CONNECT_INT_DELAY__SHIFT 0x00000000
+#define DC_HPD3_TOGGLE_FILT_CNTL__DC_HPD3_DISCONNECT_INT_DELAY_MASK 0x0ff00000L
+#define DC_HPD3_TOGGLE_FILT_CNTL__DC_HPD3_DISCONNECT_INT_DELAY__SHIFT 0x00000014
+#define DC_HPD4_CONTROL__DC_HPD4_CONNECTION_TIMER_MASK 0x00001fffL
+#define DC_HPD4_CONTROL__DC_HPD4_CONNECTION_TIMER__SHIFT 0x00000000
+#define DC_HPD4_CONTROL__DC_HPD4_EN_MASK 0x10000000L
+#define DC_HPD4_CONTROL__DC_HPD4_EN__SHIFT 0x0000001c
+#define DC_HPD4_CONTROL__DC_HPD4_RX_INT_TIMER_MASK 0x03ff0000L
+#define DC_HPD4_CONTROL__DC_HPD4_RX_INT_TIMER__SHIFT 0x00000010
+#define DC_HPD4_FAST_TRAIN_CNTL__DC_HPD4_CONNECT_AUX_TX_DELAY_MASK 0x000000ffL
+#define DC_HPD4_FAST_TRAIN_CNTL__DC_HPD4_CONNECT_AUX_TX_DELAY__SHIFT 0x00000000
+#define DC_HPD4_FAST_TRAIN_CNTL__DC_HPD4_CONNECT_AUX_TX_EN_MASK 0x01000000L
+#define DC_HPD4_FAST_TRAIN_CNTL__DC_HPD4_CONNECT_AUX_TX_EN__SHIFT 0x00000018
+#define DC_HPD4_FAST_TRAIN_CNTL__DC_HPD4_CONNECT_FAST_TRAIN_DELAY_MASK 0x000ff000L
+#define DC_HPD4_FAST_TRAIN_CNTL__DC_HPD4_CONNECT_FAST_TRAIN_DELAY__SHIFT 0x0000000c
+#define DC_HPD4_FAST_TRAIN_CNTL__DC_HPD4_CONNECT_FAST_TRAIN_EN_MASK 0x10000000L
+#define DC_HPD4_FAST_TRAIN_CNTL__DC_HPD4_CONNECT_FAST_TRAIN_EN__SHIFT 0x0000001c
+#define DC_HPD4_INT_CONTROL__DC_HPD4_INT_ACK_MASK 0x00000001L
+#define DC_HPD4_INT_CONTROL__DC_HPD4_INT_ACK__SHIFT 0x00000000
+#define DC_HPD4_INT_CONTROL__DC_HPD4_INT_EN_MASK 0x00010000L
+#define DC_HPD4_INT_CONTROL__DC_HPD4_INT_EN__SHIFT 0x00000010
+#define DC_HPD4_INT_CONTROL__DC_HPD4_INT_POLARITY_MASK 0x00000100L
+#define DC_HPD4_INT_CONTROL__DC_HPD4_INT_POLARITY__SHIFT 0x00000008
+#define DC_HPD4_INT_CONTROL__DC_HPD4_RX_INT_ACK_MASK 0x00100000L
+#define DC_HPD4_INT_CONTROL__DC_HPD4_RX_INT_ACK__SHIFT 0x00000014
+#define DC_HPD4_INT_CONTROL__DC_HPD4_RX_INT_EN_MASK 0x01000000L
+#define DC_HPD4_INT_CONTROL__DC_HPD4_RX_INT_EN__SHIFT 0x00000018
+#define DC_HPD4_INT_STATUS__DC_HPD4_INT_STATUS_MASK 0x00000001L
+#define DC_HPD4_INT_STATUS__DC_HPD4_INT_STATUS__SHIFT 0x00000000
+#define DC_HPD4_INT_STATUS__DC_HPD4_RX_INT_STATUS_MASK 0x00000100L
+#define DC_HPD4_INT_STATUS__DC_HPD4_RX_INT_STATUS__SHIFT 0x00000008
+#define DC_HPD4_INT_STATUS__DC_HPD4_SENSE_DELAYED_MASK 0x00000010L
+#define DC_HPD4_INT_STATUS__DC_HPD4_SENSE_DELAYED__SHIFT 0x00000004
+#define DC_HPD4_INT_STATUS__DC_HPD4_SENSE_MASK 0x00000002L
+#define DC_HPD4_INT_STATUS__DC_HPD4_SENSE__SHIFT 0x00000001
+#define DC_HPD4_INT_STATUS__DC_HPD4_TOGGLE_FILT_CON_TIMER_VAL_MASK 0x000ff000L
+#define DC_HPD4_INT_STATUS__DC_HPD4_TOGGLE_FILT_CON_TIMER_VAL__SHIFT 0x0000000c
+#define DC_HPD4_INT_STATUS__DC_HPD4_TOGGLE_FILT_DISCON_TIMER_VAL_MASK 0xff000000L
+#define DC_HPD4_INT_STATUS__DC_HPD4_TOGGLE_FILT_DISCON_TIMER_VAL__SHIFT 0x00000018
+#define DC_HPD4_TOGGLE_FILT_CNTL__DC_HPD4_CONNECT_INT_DELAY_MASK 0x000000ffL
+#define DC_HPD4_TOGGLE_FILT_CNTL__DC_HPD4_CONNECT_INT_DELAY__SHIFT 0x00000000
+#define DC_HPD4_TOGGLE_FILT_CNTL__DC_HPD4_DISCONNECT_INT_DELAY_MASK 0x0ff00000L
+#define DC_HPD4_TOGGLE_FILT_CNTL__DC_HPD4_DISCONNECT_INT_DELAY__SHIFT 0x00000014
+#define DC_HPD5_CONTROL__DC_HPD5_CONNECTION_TIMER_MASK 0x00001fffL
+#define DC_HPD5_CONTROL__DC_HPD5_CONNECTION_TIMER__SHIFT 0x00000000
+#define DC_HPD5_CONTROL__DC_HPD5_EN_MASK 0x10000000L
+#define DC_HPD5_CONTROL__DC_HPD5_EN__SHIFT 0x0000001c
+#define DC_HPD5_CONTROL__DC_HPD5_RX_INT_TIMER_MASK 0x03ff0000L
+#define DC_HPD5_CONTROL__DC_HPD5_RX_INT_TIMER__SHIFT 0x00000010
+#define DC_HPD5_FAST_TRAIN_CNTL__DC_HPD5_CONNECT_AUX_TX_DELAY_MASK 0x000000ffL
+#define DC_HPD5_FAST_TRAIN_CNTL__DC_HPD5_CONNECT_AUX_TX_DELAY__SHIFT 0x00000000
+#define DC_HPD5_FAST_TRAIN_CNTL__DC_HPD5_CONNECT_AUX_TX_EN_MASK 0x01000000L
+#define DC_HPD5_FAST_TRAIN_CNTL__DC_HPD5_CONNECT_AUX_TX_EN__SHIFT 0x00000018
+#define DC_HPD5_FAST_TRAIN_CNTL__DC_HPD5_CONNECT_FAST_TRAIN_DELAY_MASK 0x000ff000L
+#define DC_HPD5_FAST_TRAIN_CNTL__DC_HPD5_CONNECT_FAST_TRAIN_DELAY__SHIFT 0x0000000c
+#define DC_HPD5_FAST_TRAIN_CNTL__DC_HPD5_CONNECT_FAST_TRAIN_EN_MASK 0x10000000L
+#define DC_HPD5_FAST_TRAIN_CNTL__DC_HPD5_CONNECT_FAST_TRAIN_EN__SHIFT 0x0000001c
+#define DC_HPD5_INT_CONTROL__DC_HPD5_INT_ACK_MASK 0x00000001L
+#define DC_HPD5_INT_CONTROL__DC_HPD5_INT_ACK__SHIFT 0x00000000
+#define DC_HPD5_INT_CONTROL__DC_HPD5_INT_EN_MASK 0x00010000L
+#define DC_HPD5_INT_CONTROL__DC_HPD5_INT_EN__SHIFT 0x00000010
+#define DC_HPD5_INT_CONTROL__DC_HPD5_INT_POLARITY_MASK 0x00000100L
+#define DC_HPD5_INT_CONTROL__DC_HPD5_INT_POLARITY__SHIFT 0x00000008
+#define DC_HPD5_INT_CONTROL__DC_HPD5_RX_INT_ACK_MASK 0x00100000L
+#define DC_HPD5_INT_CONTROL__DC_HPD5_RX_INT_ACK__SHIFT 0x00000014
+#define DC_HPD5_INT_CONTROL__DC_HPD5_RX_INT_EN_MASK 0x01000000L
+#define DC_HPD5_INT_CONTROL__DC_HPD5_RX_INT_EN__SHIFT 0x00000018
+#define DC_HPD5_INT_STATUS__DC_HPD5_INT_STATUS_MASK 0x00000001L
+#define DC_HPD5_INT_STATUS__DC_HPD5_INT_STATUS__SHIFT 0x00000000
+#define DC_HPD5_INT_STATUS__DC_HPD5_RX_INT_STATUS_MASK 0x00000100L
+#define DC_HPD5_INT_STATUS__DC_HPD5_RX_INT_STATUS__SHIFT 0x00000008
+#define DC_HPD5_INT_STATUS__DC_HPD5_SENSE_DELAYED_MASK 0x00000010L
+#define DC_HPD5_INT_STATUS__DC_HPD5_SENSE_DELAYED__SHIFT 0x00000004
+#define DC_HPD5_INT_STATUS__DC_HPD5_SENSE_MASK 0x00000002L
+#define DC_HPD5_INT_STATUS__DC_HPD5_SENSE__SHIFT 0x00000001
+#define DC_HPD5_INT_STATUS__DC_HPD5_TOGGLE_FILT_CON_TIMER_VAL_MASK 0x000ff000L
+#define DC_HPD5_INT_STATUS__DC_HPD5_TOGGLE_FILT_CON_TIMER_VAL__SHIFT 0x0000000c
+#define DC_HPD5_INT_STATUS__DC_HPD5_TOGGLE_FILT_DISCON_TIMER_VAL_MASK 0xff000000L
+#define DC_HPD5_INT_STATUS__DC_HPD5_TOGGLE_FILT_DISCON_TIMER_VAL__SHIFT 0x00000018
+#define DC_HPD5_TOGGLE_FILT_CNTL__DC_HPD5_CONNECT_INT_DELAY_MASK 0x000000ffL
+#define DC_HPD5_TOGGLE_FILT_CNTL__DC_HPD5_CONNECT_INT_DELAY__SHIFT 0x00000000
+#define DC_HPD5_TOGGLE_FILT_CNTL__DC_HPD5_DISCONNECT_INT_DELAY_MASK 0x0ff00000L
+#define DC_HPD5_TOGGLE_FILT_CNTL__DC_HPD5_DISCONNECT_INT_DELAY__SHIFT 0x00000014
+#define DC_HPD6_CONTROL__DC_HPD6_CONNECTION_TIMER_MASK 0x00001fffL
+#define DC_HPD6_CONTROL__DC_HPD6_CONNECTION_TIMER__SHIFT 0x00000000
+#define DC_HPD6_CONTROL__DC_HPD6_EN_MASK 0x10000000L
+#define DC_HPD6_CONTROL__DC_HPD6_EN__SHIFT 0x0000001c
+#define DC_HPD6_CONTROL__DC_HPD6_RX_INT_TIMER_MASK 0x03ff0000L
+#define DC_HPD6_CONTROL__DC_HPD6_RX_INT_TIMER__SHIFT 0x00000010
+#define DC_HPD6_FAST_TRAIN_CNTL__DC_HPD6_CONNECT_AUX_TX_DELAY_MASK 0x000000ffL
+#define DC_HPD6_FAST_TRAIN_CNTL__DC_HPD6_CONNECT_AUX_TX_DELAY__SHIFT 0x00000000
+#define DC_HPD6_FAST_TRAIN_CNTL__DC_HPD6_CONNECT_AUX_TX_EN_MASK 0x01000000L
+#define DC_HPD6_FAST_TRAIN_CNTL__DC_HPD6_CONNECT_AUX_TX_EN__SHIFT 0x00000018
+#define DC_HPD6_FAST_TRAIN_CNTL__DC_HPD6_CONNECT_FAST_TRAIN_DELAY_MASK 0x000ff000L
+#define DC_HPD6_FAST_TRAIN_CNTL__DC_HPD6_CONNECT_FAST_TRAIN_DELAY__SHIFT 0x0000000c
+#define DC_HPD6_FAST_TRAIN_CNTL__DC_HPD6_CONNECT_FAST_TRAIN_EN_MASK 0x10000000L
+#define DC_HPD6_FAST_TRAIN_CNTL__DC_HPD6_CONNECT_FAST_TRAIN_EN__SHIFT 0x0000001c
+#define DC_HPD6_INT_CONTROL__DC_HPD6_INT_ACK_MASK 0x00000001L
+#define DC_HPD6_INT_CONTROL__DC_HPD6_INT_ACK__SHIFT 0x00000000
+#define DC_HPD6_INT_CONTROL__DC_HPD6_INT_EN_MASK 0x00010000L
+#define DC_HPD6_INT_CONTROL__DC_HPD6_INT_EN__SHIFT 0x00000010
+#define DC_HPD6_INT_CONTROL__DC_HPD6_INT_POLARITY_MASK 0x00000100L
+#define DC_HPD6_INT_CONTROL__DC_HPD6_INT_POLARITY__SHIFT 0x00000008
+#define DC_HPD6_INT_CONTROL__DC_HPD6_RX_INT_ACK_MASK 0x00100000L
+#define DC_HPD6_INT_CONTROL__DC_HPD6_RX_INT_ACK__SHIFT 0x00000014
+#define DC_HPD6_INT_CONTROL__DC_HPD6_RX_INT_EN_MASK 0x01000000L
+#define DC_HPD6_INT_CONTROL__DC_HPD6_RX_INT_EN__SHIFT 0x00000018
+#define DC_HPD6_INT_STATUS__DC_HPD6_INT_STATUS_MASK 0x00000001L
+#define DC_HPD6_INT_STATUS__DC_HPD6_INT_STATUS__SHIFT 0x00000000
+#define DC_HPD6_INT_STATUS__DC_HPD6_RX_INT_STATUS_MASK 0x00000100L
+#define DC_HPD6_INT_STATUS__DC_HPD6_RX_INT_STATUS__SHIFT 0x00000008
+#define DC_HPD6_INT_STATUS__DC_HPD6_SENSE_DELAYED_MASK 0x00000010L
+#define DC_HPD6_INT_STATUS__DC_HPD6_SENSE_DELAYED__SHIFT 0x00000004
+#define DC_HPD6_INT_STATUS__DC_HPD6_SENSE_MASK 0x00000002L
+#define DC_HPD6_INT_STATUS__DC_HPD6_SENSE__SHIFT 0x00000001
+#define DC_HPD6_INT_STATUS__DC_HPD6_TOGGLE_FILT_CON_TIMER_VAL_MASK 0x000ff000L
+#define DC_HPD6_INT_STATUS__DC_HPD6_TOGGLE_FILT_CON_TIMER_VAL__SHIFT 0x0000000c
+#define DC_HPD6_INT_STATUS__DC_HPD6_TOGGLE_FILT_DISCON_TIMER_VAL_MASK 0xff000000L
+#define DC_HPD6_INT_STATUS__DC_HPD6_TOGGLE_FILT_DISCON_TIMER_VAL__SHIFT 0x00000018
+#define DC_HPD6_TOGGLE_FILT_CNTL__DC_HPD6_CONNECT_INT_DELAY_MASK 0x000000ffL
+#define DC_HPD6_TOGGLE_FILT_CNTL__DC_HPD6_CONNECT_INT_DELAY__SHIFT 0x00000000
+#define DC_HPD6_TOGGLE_FILT_CNTL__DC_HPD6_DISCONNECT_INT_DELAY_MASK 0x0ff00000L
+#define DC_HPD6_TOGGLE_FILT_CNTL__DC_HPD6_DISCONNECT_INT_DELAY__SHIFT 0x00000014
+#define DC_I2C_ARBITRATION__DC_I2C_ABORT_HW_XFER_MASK 0x00000100L
+#define DC_I2C_ARBITRATION__DC_I2C_ABORT_HW_XFER__SHIFT 0x00000008
+#define DC_I2C_ARBITRATION__DC_I2C_ABORT_SW_XFER_MASK 0x00001000L
+#define DC_I2C_ARBITRATION__DC_I2C_ABORT_SW_XFER__SHIFT 0x0000000c
+#define DC_I2C_ARBITRATION__DC_I2C_DMCU_DONE_USING_I2C_REG_MASK 0x02000000L
+#define DC_I2C_ARBITRATION__DC_I2C_DMCU_DONE_USING_I2C_REG__SHIFT 0x00000019
+#define DC_I2C_ARBITRATION__DC_I2C_DMCU_USE_I2C_REG_REQ_MASK 0x01000000L
+#define DC_I2C_ARBITRATION__DC_I2C_DMCU_USE_I2C_REG_REQ__SHIFT 0x00000018
+#define DC_I2C_ARBITRATION__DC_I2C_NO_QUEUED_SW_GO_MASK 0x00000010L
+#define DC_I2C_ARBITRATION__DC_I2C_NO_QUEUED_SW_GO__SHIFT 0x00000004
+#define DC_I2C_ARBITRATION__DC_I2C_REG_RW_CNTL_STATUS_MASK 0x0000000cL
+#define DC_I2C_ARBITRATION__DC_I2C_REG_RW_CNTL_STATUS__SHIFT 0x00000002
+#define DC_I2C_ARBITRATION__DC_I2C_SW_DONE_USING_I2C_REG_MASK 0x00200000L
+#define DC_I2C_ARBITRATION__DC_I2C_SW_DONE_USING_I2C_REG__SHIFT 0x00000015
+#define DC_I2C_ARBITRATION__DC_I2C_SW_PRIORITY_MASK 0x00000003L
+#define DC_I2C_ARBITRATION__DC_I2C_SW_PRIORITY__SHIFT 0x00000000
+#define DC_I2C_ARBITRATION__DC_I2C_SW_USE_I2C_REG_REQ_MASK 0x00100000L
+#define DC_I2C_ARBITRATION__DC_I2C_SW_USE_I2C_REG_REQ__SHIFT 0x00000014
+#define DC_I2C_CONTROL__DC_I2C_DBG_REF_SEL_MASK 0x80000000L
+#define DC_I2C_CONTROL__DC_I2C_DBG_REF_SEL__SHIFT 0x0000001f
+#define DC_I2C_CONTROL__DC_I2C_DDC_SELECT_MASK 0x00000700L
+#define DC_I2C_CONTROL__DC_I2C_DDC_SELECT__SHIFT 0x00000008
+#define DC_I2C_CONTROL__DC_I2C_GO_MASK 0x00000001L
+#define DC_I2C_CONTROL__DC_I2C_GO__SHIFT 0x00000000
+#define DC_I2C_CONTROL__DC_I2C_SEND_RESET_MASK 0x00000004L
+#define DC_I2C_CONTROL__DC_I2C_SEND_RESET__SHIFT 0x00000002
+#define DC_I2C_CONTROL__DC_I2C_SOFT_RESET_MASK 0x00000002L
+#define DC_I2C_CONTROL__DC_I2C_SOFT_RESET__SHIFT 0x00000001
+#define DC_I2C_CONTROL__DC_I2C_SW_STATUS_RESET_MASK 0x00000008L
+#define DC_I2C_CONTROL__DC_I2C_SW_STATUS_RESET__SHIFT 0x00000003
+#define DC_I2C_CONTROL__DC_I2C_TRANSACTION_COUNT_MASK 0x00300000L
+#define DC_I2C_CONTROL__DC_I2C_TRANSACTION_COUNT__SHIFT 0x00000014
+#define DC_I2C_DATA__DC_I2C_DATA_MASK 0x0000ff00L
+#define DC_I2C_DATA__DC_I2C_DATA_RW_MASK 0x00000001L
+#define DC_I2C_DATA__DC_I2C_DATA_RW__SHIFT 0x00000000
+#define DC_I2C_DATA__DC_I2C_DATA__SHIFT 0x00000008
+#define DC_I2C_DATA__DC_I2C_INDEX_MASK 0x00ff0000L
+#define DC_I2C_DATA__DC_I2C_INDEX__SHIFT 0x00000010
+#define DC_I2C_DATA__DC_I2C_INDEX_WRITE_MASK 0x80000000L
+#define DC_I2C_DATA__DC_I2C_INDEX_WRITE__SHIFT 0x0000001f
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_EDID_DETECT_NUM_VALID_TRIES_MASK 0x0f000000L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_EDID_DETECT_NUM_VALID_TRIES__SHIFT 0x00000018
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_EDID_DETECT_STATE_MASK 0x70000000L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_EDID_DETECT_STATE__SHIFT 0x0000001c
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_EDID_DETECT_STATUS_MASK 0x00100000L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_EDID_DETECT_STATUS__SHIFT 0x00000014
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_DONE_MASK 0x00000008L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_DONE__SHIFT 0x00000003
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_REQ_MASK 0x00010000L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_REQ__SHIFT 0x00000010
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_STATUS_MASK 0x00000003L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_STATUS__SHIFT 0x00000000
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_URG_MASK 0x00020000L
+#define DC_I2C_DDC1_HW_STATUS__DC_I2C_DDC1_HW_URG__SHIFT 0x00000011
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_CLK_DRIVE_EN_MASK 0x00000080L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_CLK_DRIVE_EN__SHIFT 0x00000007
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_DATA_DRIVE_EN_MASK 0x00000001L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_DATA_DRIVE_EN__SHIFT 0x00000000
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_DATA_DRIVE_SEL_MASK 0x00000002L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_DATA_DRIVE_SEL__SHIFT 0x00000001
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_EDID_DETECT_ENABLE_MASK 0x00000010L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_EDID_DETECT_ENABLE__SHIFT 0x00000004
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_EDID_DETECT_MODE_MASK 0x00000020L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_EDID_DETECT_MODE__SHIFT 0x00000005
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_ENABLE_MASK 0x00000040L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_ENABLE__SHIFT 0x00000006
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_INTRA_BYTE_DELAY_MASK 0x0000ff00L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_INTRA_BYTE_DELAY__SHIFT 0x00000008
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_INTRA_TRANSACTION_DELAY_MASK 0x00ff0000L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_INTRA_TRANSACTION_DELAY__SHIFT 0x00000010
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_TIME_LIMIT_MASK 0xff000000L
+#define DC_I2C_DDC1_SETUP__DC_I2C_DDC1_TIME_LIMIT__SHIFT 0x00000018
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_DISABLE_FILTER_DURING_STALL_MASK 0x00000010L
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_DISABLE_FILTER_DURING_STALL__SHIFT 0x00000004
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_PRESCALE_MASK 0xffff0000L
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_PRESCALE__SHIFT 0x00000010
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_THRESHOLD_MASK 0x00000003L
+#define DC_I2C_DDC1_SPEED__DC_I2C_DDC1_THRESHOLD__SHIFT 0x00000000
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_EDID_DETECT_NUM_VALID_TRIES_MASK 0x0f000000L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_EDID_DETECT_NUM_VALID_TRIES__SHIFT 0x00000018
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_EDID_DETECT_STATE_MASK 0x70000000L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_EDID_DETECT_STATE__SHIFT 0x0000001c
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_EDID_DETECT_STATUS_MASK 0x00100000L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_EDID_DETECT_STATUS__SHIFT 0x00000014
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_DONE_MASK 0x00000008L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_DONE__SHIFT 0x00000003
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_REQ_MASK 0x00010000L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_REQ__SHIFT 0x00000010
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_STATUS_MASK 0x00000003L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_STATUS__SHIFT 0x00000000
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_URG_MASK 0x00020000L
+#define DC_I2C_DDC2_HW_STATUS__DC_I2C_DDC2_HW_URG__SHIFT 0x00000011
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_CLK_DRIVE_EN_MASK 0x00000080L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_CLK_DRIVE_EN__SHIFT 0x00000007
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_DATA_DRIVE_EN_MASK 0x00000001L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_DATA_DRIVE_EN__SHIFT 0x00000000
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_DATA_DRIVE_SEL_MASK 0x00000002L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_DATA_DRIVE_SEL__SHIFT 0x00000001
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_EDID_DETECT_ENABLE_MASK 0x00000010L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_EDID_DETECT_ENABLE__SHIFT 0x00000004
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_EDID_DETECT_MODE_MASK 0x00000020L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_EDID_DETECT_MODE__SHIFT 0x00000005
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_ENABLE_MASK 0x00000040L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_ENABLE__SHIFT 0x00000006
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_INTRA_BYTE_DELAY_MASK 0x0000ff00L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_INTRA_BYTE_DELAY__SHIFT 0x00000008
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_INTRA_TRANSACTION_DELAY_MASK 0x00ff0000L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_INTRA_TRANSACTION_DELAY__SHIFT 0x00000010
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_TIME_LIMIT_MASK 0xff000000L
+#define DC_I2C_DDC2_SETUP__DC_I2C_DDC2_TIME_LIMIT__SHIFT 0x00000018
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_DISABLE_FILTER_DURING_STALL_MASK 0x00000010L
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_DISABLE_FILTER_DURING_STALL__SHIFT 0x00000004
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_PRESCALE_MASK 0xffff0000L
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_PRESCALE__SHIFT 0x00000010
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_THRESHOLD_MASK 0x00000003L
+#define DC_I2C_DDC2_SPEED__DC_I2C_DDC2_THRESHOLD__SHIFT 0x00000000
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_EDID_DETECT_NUM_VALID_TRIES_MASK 0x0f000000L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_EDID_DETECT_NUM_VALID_TRIES__SHIFT 0x00000018
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_EDID_DETECT_STATE_MASK 0x70000000L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_EDID_DETECT_STATE__SHIFT 0x0000001c
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_EDID_DETECT_STATUS_MASK 0x00100000L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_EDID_DETECT_STATUS__SHIFT 0x00000014
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_DONE_MASK 0x00000008L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_DONE__SHIFT 0x00000003
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_REQ_MASK 0x00010000L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_REQ__SHIFT 0x00000010
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_STATUS_MASK 0x00000003L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_STATUS__SHIFT 0x00000000
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_URG_MASK 0x00020000L
+#define DC_I2C_DDC3_HW_STATUS__DC_I2C_DDC3_HW_URG__SHIFT 0x00000011
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_CLK_DRIVE_EN_MASK 0x00000080L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_CLK_DRIVE_EN__SHIFT 0x00000007
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_DATA_DRIVE_EN_MASK 0x00000001L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_DATA_DRIVE_EN__SHIFT 0x00000000
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_DATA_DRIVE_SEL_MASK 0x00000002L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_DATA_DRIVE_SEL__SHIFT 0x00000001
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_EDID_DETECT_ENABLE_MASK 0x00000010L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_EDID_DETECT_ENABLE__SHIFT 0x00000004
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_EDID_DETECT_MODE_MASK 0x00000020L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_EDID_DETECT_MODE__SHIFT 0x00000005
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_ENABLE_MASK 0x00000040L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_ENABLE__SHIFT 0x00000006
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_INTRA_BYTE_DELAY_MASK 0x0000ff00L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_INTRA_BYTE_DELAY__SHIFT 0x00000008
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_INTRA_TRANSACTION_DELAY_MASK 0x00ff0000L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_INTRA_TRANSACTION_DELAY__SHIFT 0x00000010
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_TIME_LIMIT_MASK 0xff000000L
+#define DC_I2C_DDC3_SETUP__DC_I2C_DDC3_TIME_LIMIT__SHIFT 0x00000018
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_DISABLE_FILTER_DURING_STALL_MASK 0x00000010L
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_DISABLE_FILTER_DURING_STALL__SHIFT 0x00000004
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_PRESCALE_MASK 0xffff0000L
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_PRESCALE__SHIFT 0x00000010
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_THRESHOLD_MASK 0x00000003L
+#define DC_I2C_DDC3_SPEED__DC_I2C_DDC3_THRESHOLD__SHIFT 0x00000000
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_EDID_DETECT_NUM_VALID_TRIES_MASK 0x0f000000L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_EDID_DETECT_NUM_VALID_TRIES__SHIFT 0x00000018
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_EDID_DETECT_STATE_MASK 0x70000000L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_EDID_DETECT_STATE__SHIFT 0x0000001c
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_EDID_DETECT_STATUS_MASK 0x00100000L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_EDID_DETECT_STATUS__SHIFT 0x00000014
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_DONE_MASK 0x00000008L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_DONE__SHIFT 0x00000003
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_REQ_MASK 0x00010000L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_REQ__SHIFT 0x00000010
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_STATUS_MASK 0x00000003L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_STATUS__SHIFT 0x00000000
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_URG_MASK 0x00020000L
+#define DC_I2C_DDC4_HW_STATUS__DC_I2C_DDC4_HW_URG__SHIFT 0x00000011
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_CLK_DRIVE_EN_MASK 0x00000080L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_CLK_DRIVE_EN__SHIFT 0x00000007
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_DATA_DRIVE_EN_MASK 0x00000001L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_DATA_DRIVE_EN__SHIFT 0x00000000
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_DATA_DRIVE_SEL_MASK 0x00000002L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_DATA_DRIVE_SEL__SHIFT 0x00000001
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_EDID_DETECT_ENABLE_MASK 0x00000010L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_EDID_DETECT_ENABLE__SHIFT 0x00000004
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_EDID_DETECT_MODE_MASK 0x00000020L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_EDID_DETECT_MODE__SHIFT 0x00000005
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_ENABLE_MASK 0x00000040L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_ENABLE__SHIFT 0x00000006
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_INTRA_BYTE_DELAY_MASK 0x0000ff00L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_INTRA_BYTE_DELAY__SHIFT 0x00000008
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_INTRA_TRANSACTION_DELAY_MASK 0x00ff0000L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_INTRA_TRANSACTION_DELAY__SHIFT 0x00000010
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_TIME_LIMIT_MASK 0xff000000L
+#define DC_I2C_DDC4_SETUP__DC_I2C_DDC4_TIME_LIMIT__SHIFT 0x00000018
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_DISABLE_FILTER_DURING_STALL_MASK 0x00000010L
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_DISABLE_FILTER_DURING_STALL__SHIFT 0x00000004
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_PRESCALE_MASK 0xffff0000L
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_PRESCALE__SHIFT 0x00000010
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_THRESHOLD_MASK 0x00000003L
+#define DC_I2C_DDC4_SPEED__DC_I2C_DDC4_THRESHOLD__SHIFT 0x00000000
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_EDID_DETECT_NUM_VALID_TRIES_MASK 0x0f000000L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_EDID_DETECT_NUM_VALID_TRIES__SHIFT 0x00000018
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_EDID_DETECT_STATE_MASK 0x70000000L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_EDID_DETECT_STATE__SHIFT 0x0000001c
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_EDID_DETECT_STATUS_MASK 0x00100000L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_EDID_DETECT_STATUS__SHIFT 0x00000014
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_DONE_MASK 0x00000008L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_DONE__SHIFT 0x00000003
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_REQ_MASK 0x00010000L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_REQ__SHIFT 0x00000010
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_STATUS_MASK 0x00000003L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_STATUS__SHIFT 0x00000000
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_URG_MASK 0x00020000L
+#define DC_I2C_DDC5_HW_STATUS__DC_I2C_DDC5_HW_URG__SHIFT 0x00000011
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_CLK_DRIVE_EN_MASK 0x00000080L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_CLK_DRIVE_EN__SHIFT 0x00000007
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_DATA_DRIVE_EN_MASK 0x00000001L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_DATA_DRIVE_EN__SHIFT 0x00000000
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_DATA_DRIVE_SEL_MASK 0x00000002L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_DATA_DRIVE_SEL__SHIFT 0x00000001
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_EDID_DETECT_ENABLE_MASK 0x00000010L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_EDID_DETECT_ENABLE__SHIFT 0x00000004
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_EDID_DETECT_MODE_MASK 0x00000020L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_EDID_DETECT_MODE__SHIFT 0x00000005
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_ENABLE_MASK 0x00000040L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_ENABLE__SHIFT 0x00000006
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_INTRA_BYTE_DELAY_MASK 0x0000ff00L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_INTRA_BYTE_DELAY__SHIFT 0x00000008
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_INTRA_TRANSACTION_DELAY_MASK 0x00ff0000L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_INTRA_TRANSACTION_DELAY__SHIFT 0x00000010
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_TIME_LIMIT_MASK 0xff000000L
+#define DC_I2C_DDC5_SETUP__DC_I2C_DDC5_TIME_LIMIT__SHIFT 0x00000018
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_DISABLE_FILTER_DURING_STALL_MASK 0x00000010L
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_DISABLE_FILTER_DURING_STALL__SHIFT 0x00000004
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_PRESCALE_MASK 0xffff0000L
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_PRESCALE__SHIFT 0x00000010
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_THRESHOLD_MASK 0x00000003L
+#define DC_I2C_DDC5_SPEED__DC_I2C_DDC5_THRESHOLD__SHIFT 0x00000000
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_EDID_DETECT_NUM_VALID_TRIES_MASK 0x0f000000L
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_EDID_DETECT_NUM_VALID_TRIES__SHIFT 0x00000018
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_EDID_DETECT_STATE_MASK 0x70000000L
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_EDID_DETECT_STATE__SHIFT 0x0000001c
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_EDID_DETECT_STATUS_MASK 0x00100000L
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_EDID_DETECT_STATUS__SHIFT 0x00000014
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_HW_DONE_MASK 0x00000008L
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_HW_DONE__SHIFT 0x00000003
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_HW_REQ_MASK 0x00010000L
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_HW_REQ__SHIFT 0x00000010
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_HW_STATUS_MASK 0x00000003L
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_HW_STATUS__SHIFT 0x00000000
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_HW_URG_MASK 0x00020000L
+#define DC_I2C_DDC6_HW_STATUS__DC_I2C_DDC6_HW_URG__SHIFT 0x00000011
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_CLK_DRIVE_EN_MASK 0x00000080L
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_CLK_DRIVE_EN__SHIFT 0x00000007
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_DATA_DRIVE_EN_MASK 0x00000001L
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_DATA_DRIVE_EN__SHIFT 0x00000000
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_DATA_DRIVE_SEL_MASK 0x00000002L
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_DATA_DRIVE_SEL__SHIFT 0x00000001
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_EDID_DETECT_ENABLE_MASK 0x00000010L
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_EDID_DETECT_ENABLE__SHIFT 0x00000004
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_EDID_DETECT_MODE_MASK 0x00000020L
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_EDID_DETECT_MODE__SHIFT 0x00000005
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_ENABLE_MASK 0x00000040L
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_ENABLE__SHIFT 0x00000006
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_INTRA_BYTE_DELAY_MASK 0x0000ff00L
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_INTRA_BYTE_DELAY__SHIFT 0x00000008
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_INTRA_TRANSACTION_DELAY_MASK 0x00ff0000L
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_INTRA_TRANSACTION_DELAY__SHIFT 0x00000010
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_TIME_LIMIT_MASK 0xff000000L
+#define DC_I2C_DDC6_SETUP__DC_I2C_DDC6_TIME_LIMIT__SHIFT 0x00000018
+#define DC_I2C_DDC6_SPEED__DC_I2C_DDC6_DISABLE_FILTER_DURING_STALL_MASK 0x00000010L
+#define DC_I2C_DDC6_SPEED__DC_I2C_DDC6_DISABLE_FILTER_DURING_STALL__SHIFT 0x00000004
+#define DC_I2C_DDC6_SPEED__DC_I2C_DDC6_PRESCALE_MASK 0xffff0000L
+#define DC_I2C_DDC6_SPEED__DC_I2C_DDC6_PRESCALE__SHIFT 0x00000010
+#define DC_I2C_DDC6_SPEED__DC_I2C_DDC6_THRESHOLD_MASK 0x00000003L
+#define DC_I2C_DDC6_SPEED__DC_I2C_DDC6_THRESHOLD__SHIFT 0x00000000
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_EDID_DETECT_NUM_VALID_TRIES_MASK 0x0f000000L
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_EDID_DETECT_NUM_VALID_TRIES__SHIFT 0x00000018
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_EDID_DETECT_STATE_MASK 0x70000000L
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_EDID_DETECT_STATE__SHIFT 0x0000001c
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_EDID_DETECT_STATUS_MASK 0x00100000L
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_EDID_DETECT_STATUS__SHIFT 0x00000014
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_HW_DONE_MASK 0x00000008L
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_HW_DONE__SHIFT 0x00000003
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_HW_REQ_MASK 0x00010000L
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_HW_REQ__SHIFT 0x00000010
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_HW_STATUS_MASK 0x00000003L
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_HW_STATUS__SHIFT 0x00000000
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_HW_URG_MASK 0x00020000L
+#define DC_I2C_DDCVGA_HW_STATUS__DC_I2C_DDCVGA_HW_URG__SHIFT 0x00000011
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_CLK_DRIVE_EN_MASK 0x00000080L
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_CLK_DRIVE_EN__SHIFT 0x00000007
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_DATA_DRIVE_EN_MASK 0x00000001L
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_DATA_DRIVE_EN__SHIFT 0x00000000
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_DATA_DRIVE_SEL_MASK 0x00000002L
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_DATA_DRIVE_SEL__SHIFT 0x00000001
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_EDID_DETECT_ENABLE_MASK 0x00000010L
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_EDID_DETECT_ENABLE__SHIFT 0x00000004
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_EDID_DETECT_MODE_MASK 0x00000020L
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_EDID_DETECT_MODE__SHIFT 0x00000005
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_ENABLE_MASK 0x00000040L
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_ENABLE__SHIFT 0x00000006
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_INTRA_BYTE_DELAY_MASK 0x0000ff00L
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_INTRA_BYTE_DELAY__SHIFT 0x00000008
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_INTRA_TRANSACTION_DELAY_MASK 0x00ff0000L
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_INTRA_TRANSACTION_DELAY__SHIFT 0x00000010
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_TIME_LIMIT_MASK 0xff000000L
+#define DC_I2C_DDCVGA_SETUP__DC_I2C_DDCVGA_TIME_LIMIT__SHIFT 0x00000018
+#define DC_I2C_DDCVGA_SPEED__DC_I2C_DDCVGA_DISABLE_FILTER_DURING_STALL_MASK 0x00000010L
+#define DC_I2C_DDCVGA_SPEED__DC_I2C_DDCVGA_DISABLE_FILTER_DURING_STALL__SHIFT 0x00000004
+#define DC_I2C_DDCVGA_SPEED__DC_I2C_DDCVGA_PRESCALE_MASK 0xffff0000L
+#define DC_I2C_DDCVGA_SPEED__DC_I2C_DDCVGA_PRESCALE__SHIFT 0x00000010
+#define DC_I2C_DDCVGA_SPEED__DC_I2C_DDCVGA_THRESHOLD_MASK 0x00000003L
+#define DC_I2C_DDCVGA_SPEED__DC_I2C_DDCVGA_THRESHOLD__SHIFT 0x00000000
+#define DC_I2C_EDID_DETECT_CTRL__DC_I2C_EDID_DETECT_NUM_TRIES_UNTIL_VALID_MASK 0x00f00000L
+#define DC_I2C_EDID_DETECT_CTRL__DC_I2C_EDID_DETECT_NUM_TRIES_UNTIL_VALID__SHIFT 0x00000014
+#define DC_I2C_EDID_DETECT_CTRL__DC_I2C_EDID_DETECT_SEND_RESET_MASK 0x10000000L
+#define DC_I2C_EDID_DETECT_CTRL__DC_I2C_EDID_DETECT_SEND_RESET__SHIFT 0x0000001c
+#define DC_I2C_EDID_DETECT_CTRL__DC_I2C_EDID_DETECT_WAIT_TIME_MASK 0x0000ffffL
+#define DC_I2C_EDID_DETECT_CTRL__DC_I2C_EDID_DETECT_WAIT_TIME__SHIFT 0x00000000
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC1_HW_DONE_ACK_MASK 0x00000020L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC1_HW_DONE_ACK__SHIFT 0x00000005
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC1_HW_DONE_INT_MASK 0x00000010L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC1_HW_DONE_INT__SHIFT 0x00000004
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC1_HW_DONE_MASK_MASK 0x00000040L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC1_HW_DONE_MASK__SHIFT 0x00000006
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC2_HW_DONE_ACK_MASK 0x00000200L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC2_HW_DONE_ACK__SHIFT 0x00000009
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC2_HW_DONE_INT_MASK 0x00000100L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC2_HW_DONE_INT__SHIFT 0x00000008
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC2_HW_DONE_MASK_MASK 0x00000400L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC2_HW_DONE_MASK__SHIFT 0x0000000a
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC3_HW_DONE_ACK_MASK 0x00002000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC3_HW_DONE_ACK__SHIFT 0x0000000d
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC3_HW_DONE_INT_MASK 0x00001000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC3_HW_DONE_INT__SHIFT 0x0000000c
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC3_HW_DONE_MASK_MASK 0x00004000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC3_HW_DONE_MASK__SHIFT 0x0000000e
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC4_HW_DONE_ACK_MASK 0x00020000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC4_HW_DONE_ACK__SHIFT 0x00000011
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC4_HW_DONE_INT_MASK 0x00010000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC4_HW_DONE_INT__SHIFT 0x00000010
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC4_HW_DONE_MASK_MASK 0x00040000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC4_HW_DONE_MASK__SHIFT 0x00000012
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC5_HW_DONE_ACK_MASK 0x00200000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC5_HW_DONE_ACK__SHIFT 0x00000015
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC5_HW_DONE_INT_MASK 0x00100000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC5_HW_DONE_INT__SHIFT 0x00000014
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC5_HW_DONE_MASK_MASK 0x00400000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC5_HW_DONE_MASK__SHIFT 0x00000016
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC6_HW_DONE_ACK_MASK 0x02000000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC6_HW_DONE_ACK__SHIFT 0x00000019
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC6_HW_DONE_INT_MASK 0x01000000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC6_HW_DONE_INT__SHIFT 0x00000018
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC6_HW_DONE_MASK_MASK 0x04000000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDC6_HW_DONE_MASK__SHIFT 0x0000001a
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDCVGA_HW_DONE_ACK_MASK 0x10000000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDCVGA_HW_DONE_ACK__SHIFT 0x0000001c
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDCVGA_HW_DONE_INT_MASK 0x08000000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDCVGA_HW_DONE_INT__SHIFT 0x0000001b
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDCVGA_HW_DONE_MASK_MASK 0x20000000L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_DDCVGA_HW_DONE_MASK__SHIFT 0x0000001d
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_SW_DONE_ACK_MASK 0x00000002L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_SW_DONE_ACK__SHIFT 0x00000001
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_SW_DONE_INT_MASK 0x00000001L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_SW_DONE_INT__SHIFT 0x00000000
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_SW_DONE_MASK_MASK 0x00000004L
+#define DC_I2C_INTERRUPT_CONTROL__DC_I2C_SW_DONE_MASK__SHIFT 0x00000002
+#define DC_I2C_SW_STATUS__DC_I2C_SW_ABORTED_MASK 0x00000010L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_ABORTED__SHIFT 0x00000004
+#define DC_I2C_SW_STATUS__DC_I2C_SW_BUFFER_OVERFLOW_MASK 0x00000080L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_BUFFER_OVERFLOW__SHIFT 0x00000007
+#define DC_I2C_SW_STATUS__DC_I2C_SW_DONE_MASK 0x00000004L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_DONE__SHIFT 0x00000002
+#define DC_I2C_SW_STATUS__DC_I2C_SW_INTERRUPTED_MASK 0x00000040L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_INTERRUPTED__SHIFT 0x00000006
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK0_MASK 0x00001000L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK0__SHIFT 0x0000000c
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK1_MASK 0x00002000L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK1__SHIFT 0x0000000d
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK2_MASK 0x00004000L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK2__SHIFT 0x0000000e
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK3_MASK 0x00008000L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_NACK3__SHIFT 0x0000000f
+#define DC_I2C_SW_STATUS__DC_I2C_SW_REQ_MASK 0x00040000L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_REQ__SHIFT 0x00000012
+#define DC_I2C_SW_STATUS__DC_I2C_SW_STATUS_MASK 0x00000003L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_STATUS__SHIFT 0x00000000
+#define DC_I2C_SW_STATUS__DC_I2C_SW_STOPPED_ON_NACK_MASK 0x00000100L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_STOPPED_ON_NACK__SHIFT 0x00000008
+#define DC_I2C_SW_STATUS__DC_I2C_SW_TIMEOUT_MASK 0x00000020L
+#define DC_I2C_SW_STATUS__DC_I2C_SW_TIMEOUT__SHIFT 0x00000005
+#define DC_I2C_TRANSACTION0__DC_I2C_COUNT0_MASK 0x00ff0000L
+#define DC_I2C_TRANSACTION0__DC_I2C_COUNT0__SHIFT 0x00000010
+#define DC_I2C_TRANSACTION0__DC_I2C_RW0_MASK 0x00000001L
+#define DC_I2C_TRANSACTION0__DC_I2C_RW0__SHIFT 0x00000000
+#define DC_I2C_TRANSACTION0__DC_I2C_START0_MASK 0x00001000L
+#define DC_I2C_TRANSACTION0__DC_I2C_START0__SHIFT 0x0000000c
+#define DC_I2C_TRANSACTION0__DC_I2C_STOP0_MASK 0x00002000L
+#define DC_I2C_TRANSACTION0__DC_I2C_STOP0__SHIFT 0x0000000d
+#define DC_I2C_TRANSACTION0__DC_I2C_STOP_ON_NACK0_MASK 0x00000100L
+#define DC_I2C_TRANSACTION0__DC_I2C_STOP_ON_NACK0__SHIFT 0x00000008
+#define DC_I2C_TRANSACTION1__DC_I2C_COUNT1_MASK 0x00ff0000L
+#define DC_I2C_TRANSACTION1__DC_I2C_COUNT1__SHIFT 0x00000010
+#define DC_I2C_TRANSACTION1__DC_I2C_RW1_MASK 0x00000001L
+#define DC_I2C_TRANSACTION1__DC_I2C_RW1__SHIFT 0x00000000
+#define DC_I2C_TRANSACTION1__DC_I2C_START1_MASK 0x00001000L
+#define DC_I2C_TRANSACTION1__DC_I2C_START1__SHIFT 0x0000000c
+#define DC_I2C_TRANSACTION1__DC_I2C_STOP1_MASK 0x00002000L
+#define DC_I2C_TRANSACTION1__DC_I2C_STOP1__SHIFT 0x0000000d
+#define DC_I2C_TRANSACTION1__DC_I2C_STOP_ON_NACK1_MASK 0x00000100L
+#define DC_I2C_TRANSACTION1__DC_I2C_STOP_ON_NACK1__SHIFT 0x00000008
+#define DC_I2C_TRANSACTION2__DC_I2C_COUNT2_MASK 0x00ff0000L
+#define DC_I2C_TRANSACTION2__DC_I2C_COUNT2__SHIFT 0x00000010
+#define DC_I2C_TRANSACTION2__DC_I2C_RW2_MASK 0x00000001L
+#define DC_I2C_TRANSACTION2__DC_I2C_RW2__SHIFT 0x00000000
+#define DC_I2C_TRANSACTION2__DC_I2C_START2_MASK 0x00001000L
+#define DC_I2C_TRANSACTION2__DC_I2C_START2__SHIFT 0x0000000c
+#define DC_I2C_TRANSACTION2__DC_I2C_STOP2_MASK 0x00002000L
+#define DC_I2C_TRANSACTION2__DC_I2C_STOP2__SHIFT 0x0000000d
+#define DC_I2C_TRANSACTION2__DC_I2C_STOP_ON_NACK2_MASK 0x00000100L
+#define DC_I2C_TRANSACTION2__DC_I2C_STOP_ON_NACK2__SHIFT 0x00000008
+#define DC_I2C_TRANSACTION3__DC_I2C_COUNT3_MASK 0x00ff0000L
+#define DC_I2C_TRANSACTION3__DC_I2C_COUNT3__SHIFT 0x00000010
+#define DC_I2C_TRANSACTION3__DC_I2C_RW3_MASK 0x00000001L
+#define DC_I2C_TRANSACTION3__DC_I2C_RW3__SHIFT 0x00000000
+#define DC_I2C_TRANSACTION3__DC_I2C_START3_MASK 0x00001000L
+#define DC_I2C_TRANSACTION3__DC_I2C_START3__SHIFT 0x0000000c
+#define DC_I2C_TRANSACTION3__DC_I2C_STOP3_MASK 0x00002000L
+#define DC_I2C_TRANSACTION3__DC_I2C_STOP3__SHIFT 0x0000000d
+#define DC_I2C_TRANSACTION3__DC_I2C_STOP_ON_NACK3_MASK 0x00000100L
+#define DC_I2C_TRANSACTION3__DC_I2C_STOP_ON_NACK3__SHIFT 0x00000008
+#define DCI_CLK_CNTL__DCI_PG_TEST_CLK_SEL_MASK 0xf8000000L
+#define DCI_CLK_CNTL__DCI_PG_TEST_CLK_SEL__SHIFT 0x0000001b
+#define DCI_CLK_CNTL__DCI_TEST_CLK_SEL_MASK 0x0000001fL
+#define DCI_CLK_CNTL__DCI_TEST_CLK_SEL__SHIFT 0x00000000
+#define DCI_CLK_CNTL__DISPCLK_G_DMCU_GATE_DIS_MASK 0x00008000L
+#define DCI_CLK_CNTL__DISPCLK_G_DMCU_GATE_DIS__SHIFT 0x0000000f
+#define DCI_CLK_CNTL__DISPCLK_G_DMIF0_GATE_DIS_MASK 0x00010000L
+#define DCI_CLK_CNTL__DISPCLK_G_DMIF0_GATE_DIS__SHIFT 0x00000010
+#define DCI_CLK_CNTL__DISPCLK_G_DMIF1_GATE_DIS_MASK 0x00020000L
+#define DCI_CLK_CNTL__DISPCLK_G_DMIF1_GATE_DIS__SHIFT 0x00000011
+#define DCI_CLK_CNTL__DISPCLK_G_DMIF2_GATE_DIS_MASK 0x00040000L
+#define DCI_CLK_CNTL__DISPCLK_G_DMIF2_GATE_DIS__SHIFT 0x00000012
+#define DCI_CLK_CNTL__DISPCLK_G_DMIF3_GATE_DIS_MASK 0x00080000L
+#define DCI_CLK_CNTL__DISPCLK_G_DMIF3_GATE_DIS__SHIFT 0x00000013
+#define DCI_CLK_CNTL__DISPCLK_G_DMIF4_GATE_DIS_MASK 0x00100000L
+#define DCI_CLK_CNTL__DISPCLK_G_DMIF4_GATE_DIS__SHIFT 0x00000014
+#define DCI_CLK_CNTL__DISPCLK_G_DMIF5_GATE_DIS_MASK 0x00200000L
+#define DCI_CLK_CNTL__DISPCLK_G_DMIF5_GATE_DIS__SHIFT 0x00000015
+#define DCI_CLK_CNTL__DISPCLK_G_FBC_GATE_DIS_MASK 0x00000200L
+#define DCI_CLK_CNTL__DISPCLK_G_FBC_GATE_DIS__SHIFT 0x00000009
+#define DCI_CLK_CNTL__DISPCLK_G_VGA_GATE_DIS_MASK 0x00000800L
+#define DCI_CLK_CNTL__DISPCLK_G_VGA_GATE_DIS__SHIFT 0x0000000b
+#define DCI_CLK_CNTL__DISPCLK_G_VIP_GATE_DIS_MASK 0x00002000L
+#define DCI_CLK_CNTL__DISPCLK_G_VIP_GATE_DIS__SHIFT 0x0000000d
+#define DCI_CLK_CNTL__DISPCLK_M_GATE_DIS_MASK 0x00000040L
+#define DCI_CLK_CNTL__DISPCLK_M_GATE_DIS__SHIFT 0x00000006
+#define DCI_CLK_CNTL__DISPCLK_R_DCI_GATE_DIS_MASK 0x00000020L
+#define DCI_CLK_CNTL__DISPCLK_R_DCI_GATE_DIS__SHIFT 0x00000005
+#define DCI_CLK_CNTL__DISPCLK_R_DMCU_GATE_DIS_MASK 0x00004000L
+#define DCI_CLK_CNTL__DISPCLK_R_DMCU_GATE_DIS__SHIFT 0x0000000e
+#define DCI_CLK_CNTL__DISPCLK_R_VGA_GATE_DIS_MASK 0x00000400L
+#define DCI_CLK_CNTL__DISPCLK_R_VGA_GATE_DIS__SHIFT 0x0000000a
+#define DCI_CLK_CNTL__DISPCLK_R_VIP_GATE_DIS_MASK 0x00001000L
+#define DCI_CLK_CNTL__DISPCLK_R_VIP_GATE_DIS__SHIFT 0x0000000c
+#define DCI_CLK_CNTL__SCLK_G_DMIF_GATE_DIS_MASK 0x00400000L
+#define DCI_CLK_CNTL__SCLK_G_DMIF_GATE_DIS__SHIFT 0x00000016
+#define DCI_CLK_CNTL__SCLK_G_DMIFTRK_GATE_DIS_MASK 0x00800000L
+#define DCI_CLK_CNTL__SCLK_G_DMIFTRK_GATE_DIS__SHIFT 0x00000017
+#define DCI_CLK_CNTL__SCLK_R_AZ_GATE_DIS_MASK 0x00000100L
+#define DCI_CLK_CNTL__SCLK_R_AZ_GATE_DIS__SHIFT 0x00000008
+#define DCI_DEBUG_CONFIG__DCI_DBG_SEL_MASK 0x0000000fL
+#define DCI_DEBUG_CONFIG__DCI_DBG_SEL__SHIFT 0x00000000
+#define DCI_MEM_PWR_CNTL__DMIF0_ASYNC_LIGHT_SLEEP_DIS_MASK 0x00000001L
+#define DCI_MEM_PWR_CNTL__DMIF0_ASYNC_LIGHT_SLEEP_DIS__SHIFT 0x00000000
+#define DCI_MEM_PWR_CNTL__DMIF0_ASYNC_MEM_PWR_STATE_MASK 0x00003000L
+#define DCI_MEM_PWR_CNTL__DMIF0_ASYNC_MEM_PWR_STATE__SHIFT 0x0000000c
+#define DCI_MEM_PWR_CNTL__DMIF0_ASYNC_MEM_SHUTDOWN_DIS_MASK 0x00000040L
+#define DCI_MEM_PWR_CNTL__DMIF0_ASYNC_MEM_SHUTDOWN_DIS__SHIFT 0x00000006
+#define DCI_MEM_PWR_CNTL__DMIF1_ASYNC_LIGHT_SLEEP_DIS_MASK 0x00000002L
+#define DCI_MEM_PWR_CNTL__DMIF1_ASYNC_LIGHT_SLEEP_DIS__SHIFT 0x00000001
+#define DCI_MEM_PWR_CNTL__DMIF1_ASYNC_MEM_PWR_STATE_MASK 0x0000c000L
+#define DCI_MEM_PWR_CNTL__DMIF1_ASYNC_MEM_PWR_STATE__SHIFT 0x0000000e
+#define DCI_MEM_PWR_CNTL__DMIF1_ASYNC_MEM_SHUTDOWN_DIS_MASK 0x00000080L
+#define DCI_MEM_PWR_CNTL__DMIF1_ASYNC_MEM_SHUTDOWN_DIS__SHIFT 0x00000007
+#define DCI_MEM_PWR_CNTL__DMIF2_ASYNC_LIGHT_SLEEP_DIS_MASK 0x00000004L
+#define DCI_MEM_PWR_CNTL__DMIF2_ASYNC_LIGHT_SLEEP_DIS__SHIFT 0x00000002
+#define DCI_MEM_PWR_CNTL__DMIF2_ASYNC_MEM_PWR_STATE_MASK 0x00030000L
+#define DCI_MEM_PWR_CNTL__DMIF2_ASYNC_MEM_PWR_STATE__SHIFT 0x00000010
+#define DCI_MEM_PWR_CNTL__DMIF2_ASYNC_MEM_SHUTDOWN_DIS_MASK 0x00000100L
+#define DCI_MEM_PWR_CNTL__DMIF2_ASYNC_MEM_SHUTDOWN_DIS__SHIFT 0x00000008
+#define DCI_MEM_PWR_CNTL__DMIF3_ASYNC_LIGHT_SLEEP_DIS_MASK 0x00000008L
+#define DCI_MEM_PWR_CNTL__DMIF3_ASYNC_LIGHT_SLEEP_DIS__SHIFT 0x00000003
+#define DCI_MEM_PWR_CNTL__DMIF3_ASYNC_MEM_PWR_STATE_MASK 0x000c0000L
+#define DCI_MEM_PWR_CNTL__DMIF3_ASYNC_MEM_PWR_STATE__SHIFT 0x00000012
+#define DCI_MEM_PWR_CNTL__DMIF3_ASYNC_MEM_SHUTDOWN_DIS_MASK 0x00000200L
+#define DCI_MEM_PWR_CNTL__DMIF3_ASYNC_MEM_SHUTDOWN_DIS__SHIFT 0x00000009
+#define DCI_MEM_PWR_CNTL__DMIF4_ASYNC_LIGHT_SLEEP_DIS_MASK 0x00000010L
+#define DCI_MEM_PWR_CNTL__DMIF4_ASYNC_LIGHT_SLEEP_DIS__SHIFT 0x00000004
+#define DCI_MEM_PWR_CNTL__DMIF4_ASYNC_MEM_PWR_STATE_MASK 0x00300000L
+#define DCI_MEM_PWR_CNTL__DMIF4_ASYNC_MEM_PWR_STATE__SHIFT 0x00000014
+#define DCI_MEM_PWR_CNTL__DMIF4_ASYNC_MEM_SHUTDOWN_DIS_MASK 0x00000400L
+#define DCI_MEM_PWR_CNTL__DMIF4_ASYNC_MEM_SHUTDOWN_DIS__SHIFT 0x0000000a
+#define DCI_MEM_PWR_CNTL__DMIF5_ASYNC_LIGHT_SLEEP_DIS_MASK 0x00000020L
+#define DCI_MEM_PWR_CNTL__DMIF5_ASYNC_LIGHT_SLEEP_DIS__SHIFT 0x00000005
+#define DCI_MEM_PWR_CNTL__DMIF5_ASYNC_MEM_PWR_STATE_MASK 0x00c00000L
+#define DCI_MEM_PWR_CNTL__DMIF5_ASYNC_MEM_PWR_STATE__SHIFT 0x00000016
+#define DCI_MEM_PWR_CNTL__DMIF5_ASYNC_MEM_SHUTDOWN_DIS_MASK 0x00000800L
+#define DCI_MEM_PWR_CNTL__DMIF5_ASYNC_MEM_SHUTDOWN_DIS__SHIFT 0x0000000b
+#define DCI_MEM_PWR_STATE2__DMCU_ERAM1_PWR_STATE_MASK 0x00000003L
+#define DCI_MEM_PWR_STATE2__DMCU_ERAM1_PWR_STATE__SHIFT 0x00000000
+#define DCI_MEM_PWR_STATE2__DMCU_ERAM2_PWR_STATE_MASK 0x0000000cL
+#define DCI_MEM_PWR_STATE2__DMCU_ERAM2_PWR_STATE__SHIFT 0x00000002
+#define DCI_MEM_PWR_STATE2__DMCU_ERAM3_PWR_STATE_MASK 0x00000030L
+#define DCI_MEM_PWR_STATE2__DMCU_ERAM3_PWR_STATE__SHIFT 0x00000004
+#define DCI_MEM_PWR_STATE__AZ_MEM_PWR_STATE_MASK 0x00c00000L
+#define DCI_MEM_PWR_STATE__AZ_MEM_PWR_STATE__SHIFT 0x00000016
+#define DCI_MEM_PWR_STATE__DMCU_IRAM_PWR_STATE_MASK 0x30000000L
+#define DCI_MEM_PWR_STATE__DMCU_IRAM_PWR_STATE__SHIFT 0x0000001c
+#define DCI_MEM_PWR_STATE__DMCU_MEM_PWR_STATE_MASK 0x00000003L
+#define DCI_MEM_PWR_STATE__DMCU_MEM_PWR_STATE__SHIFT 0x00000000
+#define DCI_MEM_PWR_STATE__DMIF0_MEM_PWR_STATE_MASK 0x0000000cL
+#define DCI_MEM_PWR_STATE__DMIF0_MEM_PWR_STATE__SHIFT 0x00000002
+#define DCI_MEM_PWR_STATE__DMIF1_MEM_PWR_STATE_MASK 0x00000030L
+#define DCI_MEM_PWR_STATE__DMIF1_MEM_PWR_STATE__SHIFT 0x00000004
+#define DCI_MEM_PWR_STATE__DMIF2_MEM_PWR_STATE_MASK 0x000000c0L
+#define DCI_MEM_PWR_STATE__DMIF2_MEM_PWR_STATE__SHIFT 0x00000006
+#define DCI_MEM_PWR_STATE__DMIF3_MEM_PWR_STATE_MASK 0x00000300L
+#define DCI_MEM_PWR_STATE__DMIF3_MEM_PWR_STATE__SHIFT 0x00000008
+#define DCI_MEM_PWR_STATE__DMIF4_MEM_PWR_STATE_MASK 0x00000c00L
+#define DCI_MEM_PWR_STATE__DMIF4_MEM_PWR_STATE__SHIFT 0x0000000a
+#define DCI_MEM_PWR_STATE__DMIF5_MEM_PWR_STATE_MASK 0x00003000L
+#define DCI_MEM_PWR_STATE__DMIF5_MEM_PWR_STATE__SHIFT 0x0000000c
+#define DCI_MEM_PWR_STATE__DMIF_XLR_MEM1_PWR_STATE_MASK 0x0c000000L
+#define DCI_MEM_PWR_STATE__DMIF_XLR_MEM1_PWR_STATE__SHIFT 0x0000001a
+#define DCI_MEM_PWR_STATE__DMIF_XLR_MEM_PWR_STATE_MASK 0x03000000L
+#define DCI_MEM_PWR_STATE__DMIF_XLR_MEM_PWR_STATE__SHIFT 0x00000018
+#define DCI_MEM_PWR_STATE__FBC_MEM_PWR_STATE_MASK 0x00030000L
+#define DCI_MEM_PWR_STATE__FBC_MEM_PWR_STATE__SHIFT 0x00000010
+#define DCI_MEM_PWR_STATE__MCIF_MEM_PWR_STATE_MASK 0x000c0000L
+#define DCI_MEM_PWR_STATE__MCIF_MEM_PWR_STATE__SHIFT 0x00000012
+#define DCI_MEM_PWR_STATE__VGA_MEM_PWR_STATE_MASK 0x0000c000L
+#define DCI_MEM_PWR_STATE__VGA_MEM_PWR_STATE__SHIFT 0x0000000e
+#define DCI_MEM_PWR_STATE__VIP_MEM_PWR_STATE_MASK 0x00300000L
+#define DCI_MEM_PWR_STATE__VIP_MEM_PWR_STATE__SHIFT 0x00000014
+#define DCIO_DEBUG10__DCIO_DIGC_DEBUG_MASK 0xffffffffL
+#define DCIO_DEBUG10__DCIO_DIGC_DEBUG__SHIFT 0x00000000
+#define DCIO_DEBUG11__DCIO_DIGD_DEBUG_MASK 0xffffffffL
+#define DCIO_DEBUG11__DCIO_DIGD_DEBUG__SHIFT 0x00000000
+#define DCIO_DEBUG12__DCIO_DIGE_DEBUG_MASK 0xffffffffL
+#define DCIO_DEBUG12__DCIO_DIGE_DEBUG__SHIFT 0x00000000
+#define DCIO_DEBUG13__DCIO_DIGF_DEBUG_MASK 0xffffffffL
+#define DCIO_DEBUG13__DCIO_DIGF_DEBUG__SHIFT 0x00000000
+#define DCIO_DEBUG1__DOUT_DCIO_DVO_CLK_TRISTATE_MASK 0x00040000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVO_CLK_TRISTATE__SHIFT 0x00000012
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_A0_MASK 0x00008000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_A0_PREMUX_MASK 0x00004000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_A0_PREMUX__SHIFT 0x0000000e
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_A0_REG_MASK 0x00002000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_A0_REG__SHIFT 0x0000000d
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_A0__SHIFT 0x0000000f
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_EN_MASK 0x00100000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_EN_PREMUX_MASK 0x00080000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_EN_PREMUX__SHIFT 0x00000013
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_EN_REG_MASK 0x00010000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_EN_REG__SHIFT 0x00000010
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_EN__SHIFT 0x00000014
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_MASK_REG_MASK 0x00400000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_MASK_REG__SHIFT 0x00000016
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_MUX_MASK 0x00200000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_MUX__SHIFT 0x00000015
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_SEL0_MASK 0x08000000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_SEL0_PREMUX_MASK 0x04000000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_SEL0_PREMUX__SHIFT 0x0000001a
+#define DCIO_DEBUG1__DOUT_DCIO_DVOCNTL1_SEL0__SHIFT 0x0000001b
+#define DCIO_DEBUG1__DOUT_DCIO_DVO_ENABLE_MASK 0x00800000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVO_ENABLE__SHIFT 0x00000017
+#define DCIO_DEBUG1__DOUT_DCIO_DVO_HSYNC_TRISTATE_MASK 0x00020000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVO_HSYNC_TRISTATE__SHIFT 0x00000011
+#define DCIO_DEBUG1__DOUT_DCIO_DVO_RATE_SEL_MASK 0x02000000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVO_RATE_SEL__SHIFT 0x00000019
+#define DCIO_DEBUG1__DOUT_DCIO_DVO_VSYNC_TRISTATE_MASK 0x01000000L
+#define DCIO_DEBUG1__DOUT_DCIO_DVO_VSYNC_TRISTATE__SHIFT 0x00000018
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCLK_C_MASK 0x00001000L
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCLK_C__SHIFT 0x0000000c
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCNTL_A0_MASK 0x000000c0L
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCNTL_A0_REG_MASK 0x00000003L
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCNTL_A0_REG__SHIFT 0x00000000
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCNTL_A0__SHIFT 0x00000006
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCNTL_EN_MASK 0x00000c00L
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCNTL_EN_REG_MASK 0x00000030L
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCNTL_EN_REG__SHIFT 0x00000004
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCNTL_EN__SHIFT 0x0000000a
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCNTL_MASK_REG_MASK 0x0000000cL
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCNTL_MASK_REG__SHIFT 0x00000002
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCNTL_SEL0_MASK 0x00000300L
+#define DCIO_DEBUG1__DOUT_DCIO_MVP_DVOCNTL_SEL0__SHIFT 0x00000008
+#define DCIO_DEBUG2__DCIO_DEBUG2_MASK 0xffffffffL
+#define DCIO_DEBUG2__DCIO_DEBUG2__SHIFT 0x00000000
+#define DCIO_DEBUG3__DCIO_DEBUG3_MASK 0xffffffffL
+#define DCIO_DEBUG3__DCIO_DEBUG3__SHIFT 0x00000000
+#define DCIO_DEBUG4__DCIO_DEBUG4_MASK 0xffffffffL
+#define DCIO_DEBUG4__DCIO_DEBUG4__SHIFT 0x00000000
+#define DCIO_DEBUG5__DCIO_DEBUG5_MASK 0xffffffffL
+#define DCIO_DEBUG5__DCIO_DEBUG5__SHIFT 0x00000000
+#define DCIO_DEBUG6__DCIO_DEBUG6_MASK 0xffffffffL
+#define DCIO_DEBUG6__DCIO_DEBUG6__SHIFT 0x00000000
+#define DCIO_DEBUG7__DCIO_DEBUG7_MASK 0xffffffffL
+#define DCIO_DEBUG7__DCIO_DEBUG7__SHIFT 0x00000000
+#define DCIO_DEBUG8__DCIO_DEBUG8_MASK 0xffffffffL
+#define DCIO_DEBUG8__DCIO_DEBUG8__SHIFT 0x00000000
+#define DCIO_DEBUG9__DCIO_DEBUG9_MASK 0xffffffffL
+#define DCIO_DEBUG9__DCIO_DEBUG9__SHIFT 0x00000000
+#define DCIO_DEBUGA__DCIO_DEBUGA_MASK 0xffffffffL
+#define DCIO_DEBUGA__DCIO_DEBUGA__SHIFT 0x00000000
+#define DCIO_DEBUGB__DCIO_DEBUGB_MASK 0xffffffffL
+#define DCIO_DEBUGB__DCIO_DEBUGB__SHIFT 0x00000000
+#define DCIO_DEBUGC__DCIO_DEBUGC_MASK 0xffffffffL
+#define DCIO_DEBUGC__DCIO_DEBUGC__SHIFT 0x00000000
+#define DCIO_DEBUG__DCIO_DEBUG_MASK 0xffffffffL
+#define DCIO_DEBUG__DCIO_DEBUG__SHIFT 0x00000000
+#define DCIO_DEBUGD__DCIO_DEBUGD_MASK 0xffffffffL
+#define DCIO_DEBUGD__DCIO_DEBUGD__SHIFT 0x00000000
+#define DCIO_DEBUGE__DCIO_DIGA_DEBUG_MASK 0xffffffffL
+#define DCIO_DEBUGE__DCIO_DIGA_DEBUG__SHIFT 0x00000000
+#define DCIO_DEBUGF__DCIO_DIGB_DEBUG_MASK 0xffffffffL
+#define DCIO_DEBUGF__DCIO_DIGB_DEBUG__SHIFT 0x00000000
+#define DCIO_DEBUG_ID__DCIO_DEBUG_ID_MASK 0xffffffffL
+#define DCIO_DEBUG_ID__DCIO_DEBUG_ID__SHIFT 0x00000000
+#define DCIO_GSL0_CNTL__DCIO_GSL0_GLOBAL_UNLOCK_SEL_MASK 0x00070000L
+#define DCIO_GSL0_CNTL__DCIO_GSL0_GLOBAL_UNLOCK_SEL__SHIFT 0x00000010
+#define DCIO_GSL0_CNTL__DCIO_GSL0_TIMING_SYNC_SEL_MASK 0x00000700L
+#define DCIO_GSL0_CNTL__DCIO_GSL0_TIMING_SYNC_SEL__SHIFT 0x00000008
+#define DCIO_GSL0_CNTL__DCIO_GSL0_VSYNC_SEL_MASK 0x00000007L
+#define DCIO_GSL0_CNTL__DCIO_GSL0_VSYNC_SEL__SHIFT 0x00000000
+#define DCIO_GSL1_CNTL__DCIO_GSL1_GLOBAL_UNLOCK_SEL_MASK 0x00070000L
+#define DCIO_GSL1_CNTL__DCIO_GSL1_GLOBAL_UNLOCK_SEL__SHIFT 0x00000010
+#define DCIO_GSL1_CNTL__DCIO_GSL1_TIMING_SYNC_SEL_MASK 0x00000700L
+#define DCIO_GSL1_CNTL__DCIO_GSL1_TIMING_SYNC_SEL__SHIFT 0x00000008
+#define DCIO_GSL1_CNTL__DCIO_GSL1_VSYNC_SEL_MASK 0x00000007L
+#define DCIO_GSL1_CNTL__DCIO_GSL1_VSYNC_SEL__SHIFT 0x00000000
+#define DCIO_GSL2_CNTL__DCIO_GSL2_GLOBAL_UNLOCK_SEL_MASK 0x00070000L
+#define DCIO_GSL2_CNTL__DCIO_GSL2_GLOBAL_UNLOCK_SEL__SHIFT 0x00000010
+#define DCIO_GSL2_CNTL__DCIO_GSL2_TIMING_SYNC_SEL_MASK 0x00000700L
+#define DCIO_GSL2_CNTL__DCIO_GSL2_TIMING_SYNC_SEL__SHIFT 0x00000008
+#define DCIO_GSL2_CNTL__DCIO_GSL2_VSYNC_SEL_MASK 0x00000007L
+#define DCIO_GSL2_CNTL__DCIO_GSL2_VSYNC_SEL__SHIFT 0x00000000
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_CLK_GSL_FLIP_LOCK_SEL_MASK 0x00000030L
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_CLK_GSL_FLIP_LOCK_SEL__SHIFT 0x00000004
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_CLK_GSL_MASK_MASK 0x00000300L
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_CLK_GSL_MASK__SHIFT 0x00000008
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_CLK_GSL_TIMING_SYNC_SEL_MASK 0x00000003L
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_CLK_GSL_TIMING_SYNC_SEL__SHIFT 0x00000000
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_VSYNC_GSL_FLIP_LOCK_SEL_MASK 0x00300000L
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_VSYNC_GSL_FLIP_LOCK_SEL__SHIFT 0x00000014
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_VSYNC_GSL_MASK_MASK 0x03000000L
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_VSYNC_GSL_MASK__SHIFT 0x00000018
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_VSYNC_GSL_TIMING_SYNC_SEL_MASK 0x00030000L
+#define DCIO_GSL_GENLK_PAD_CNTL__DCIO_GENLK_VSYNC_GSL_TIMING_SYNC_SEL__SHIFT 0x00000010
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_A_GSL_FLIP_LOCK_SEL_MASK 0x00000030L
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_A_GSL_FLIP_LOCK_SEL__SHIFT 0x00000004
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_A_GSL_MASK_MASK 0x00000300L
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_A_GSL_MASK__SHIFT 0x00000008
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_A_GSL_TIMING_SYNC_SEL_MASK 0x00000003L
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_A_GSL_TIMING_SYNC_SEL__SHIFT 0x00000000
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_B_GSL_FLIP_LOCK_SEL_MASK 0x00300000L
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_B_GSL_FLIP_LOCK_SEL__SHIFT 0x00000014
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_B_GSL_MASK_MASK 0x03000000L
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_B_GSL_MASK__SHIFT 0x00000018
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_B_GSL_TIMING_SYNC_SEL_MASK 0x00030000L
+#define DCIO_GSL_SWAPLOCK_PAD_CNTL__DCIO_SWAPLOCK_B_GSL_TIMING_SYNC_SEL__SHIFT 0x00000010
+#define DCIO_IMPCAL_CNTL_AB__CALR_CNTL_OVERRIDE_MASK 0x0000000fL
+#define DCIO_IMPCAL_CNTL_AB__CALR_CNTL_OVERRIDE__SHIFT 0x00000000
+#define DCIO_IMPCAL_CNTL_AB__IMPCAL_ARB_STATE_MASK 0x00007000L
+#define DCIO_IMPCAL_CNTL_AB__IMPCAL_ARB_STATE__SHIFT 0x0000000c
+#define DCIO_IMPCAL_CNTL_AB__IMPCAL_SOFT_RESET_MASK 0x00000020L
+#define DCIO_IMPCAL_CNTL_AB__IMPCAL_SOFT_RESET__SHIFT 0x00000005
+#define DCIO_IMPCAL_CNTL_AB__IMPCAL_STATUS_MASK 0x00000300L
+#define DCIO_IMPCAL_CNTL_AB__IMPCAL_STATUS__SHIFT 0x00000008
+#define DCIO_IMPCAL_CNTL_CD__CALR_CNTL_OVERRIDE_MASK 0x0000000fL
+#define DCIO_IMPCAL_CNTL_CD__CALR_CNTL_OVERRIDE__SHIFT 0x00000000
+#define DCIO_IMPCAL_CNTL_CD__IMPCAL_ARB_STATE_MASK 0x00007000L
+#define DCIO_IMPCAL_CNTL_CD__IMPCAL_ARB_STATE__SHIFT 0x0000000c
+#define DCIO_IMPCAL_CNTL_CD__IMPCAL_SOFT_RESET_MASK 0x00000020L
+#define DCIO_IMPCAL_CNTL_CD__IMPCAL_SOFT_RESET__SHIFT 0x00000005
+#define DCIO_IMPCAL_CNTL_CD__IMPCAL_STATUS_MASK 0x00000300L
+#define DCIO_IMPCAL_CNTL_CD__IMPCAL_STATUS__SHIFT 0x00000008
+#define DCIO_IMPCAL_CNTL_EF__CALR_CNTL_OVERRIDE_MASK 0x0000000fL
+#define DCIO_IMPCAL_CNTL_EF__CALR_CNTL_OVERRIDE__SHIFT 0x00000000
+#define DCIO_IMPCAL_CNTL_EF__IMPCAL_ARB_STATE_MASK 0x00007000L
+#define DCIO_IMPCAL_CNTL_EF__IMPCAL_ARB_STATE__SHIFT 0x0000000c
+#define DCIO_IMPCAL_CNTL_EF__IMPCAL_SOFT_RESET_MASK 0x00000020L
+#define DCIO_IMPCAL_CNTL_EF__IMPCAL_SOFT_RESET__SHIFT 0x00000005
+#define DCIO_IMPCAL_CNTL_EF__IMPCAL_STATUS_MASK 0x00000300L
+#define DCIO_IMPCAL_CNTL_EF__IMPCAL_STATUS__SHIFT 0x00000008
+#define DCIO_TEST_DEBUG_DATA__DCIO_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define DCIO_TEST_DEBUG_DATA__DCIO_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define DCIO_TEST_DEBUG_INDEX__DCIO_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define DCIO_TEST_DEBUG_INDEX__DCIO_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define DCIO_TEST_DEBUG_INDEX__DCIO_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define DCIO_TEST_DEBUG_INDEX__DCIO_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define DCI_SOFT_RESET__DMIF0_SOFT_RESET_MASK 0x00000010L
+#define DCI_SOFT_RESET__DMIF0_SOFT_RESET__SHIFT 0x00000004
+#define DCI_SOFT_RESET__DMIF1_SOFT_RESET_MASK 0x00000020L
+#define DCI_SOFT_RESET__DMIF1_SOFT_RESET__SHIFT 0x00000005
+#define DCI_SOFT_RESET__DMIF2_SOFT_RESET_MASK 0x00000040L
+#define DCI_SOFT_RESET__DMIF2_SOFT_RESET__SHIFT 0x00000006
+#define DCI_SOFT_RESET__DMIF3_SOFT_RESET_MASK 0x00000080L
+#define DCI_SOFT_RESET__DMIF3_SOFT_RESET__SHIFT 0x00000007
+#define DCI_SOFT_RESET__DMIF4_SOFT_RESET_MASK 0x00000100L
+#define DCI_SOFT_RESET__DMIF4_SOFT_RESET__SHIFT 0x00000008
+#define DCI_SOFT_RESET__DMIF5_SOFT_RESET_MASK 0x00000200L
+#define DCI_SOFT_RESET__DMIF5_SOFT_RESET__SHIFT 0x00000009
+#define DCI_SOFT_RESET__DMIFARB_SOFT_RESET_MASK 0x00001000L
+#define DCI_SOFT_RESET__DMIFARB_SOFT_RESET__SHIFT 0x0000000c
+#define DCI_SOFT_RESET__FBC_SOFT_RESET_MASK 0x00000008L
+#define DCI_SOFT_RESET__FBC_SOFT_RESET__SHIFT 0x00000003
+#define DCI_SOFT_RESET__MCIF_SOFT_RESET_MASK 0x00000004L
+#define DCI_SOFT_RESET__MCIF_SOFT_RESET__SHIFT 0x00000002
+#define DCI_SOFT_RESET__VGA_SOFT_RESET_MASK 0x00000001L
+#define DCI_SOFT_RESET__VGA_SOFT_RESET__SHIFT 0x00000000
+#define DCI_SOFT_RESET__VIP_SOFT_RESET_MASK 0x00000002L
+#define DCI_SOFT_RESET__VIP_SOFT_RESET__SHIFT 0x00000001
+#define DCI_TEST_DEBUG_DATA__DCI_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define DCI_TEST_DEBUG_DATA__DCI_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define DCI_TEST_DEBUG_INDEX__DCI_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define DCI_TEST_DEBUG_INDEX__DCI_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define DCI_TEST_DEBUG_INDEX__DCI_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define DCI_TEST_DEBUG_INDEX__DCI_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define DC_LUT_30_COLOR__DC_LUT_COLOR_10_BLUE_MASK 0x000003ffL
+#define DC_LUT_30_COLOR__DC_LUT_COLOR_10_BLUE__SHIFT 0x00000000
+#define DC_LUT_30_COLOR__DC_LUT_COLOR_10_GREEN_MASK 0x000ffc00L
+#define DC_LUT_30_COLOR__DC_LUT_COLOR_10_GREEN__SHIFT 0x0000000a
+#define DC_LUT_30_COLOR__DC_LUT_COLOR_10_RED_MASK 0x3ff00000L
+#define DC_LUT_30_COLOR__DC_LUT_COLOR_10_RED__SHIFT 0x00000014
+#define DC_LUT_AUTOFILL__DC_LUT_AUTOFILL_DONE_MASK 0x00000002L
+#define DC_LUT_AUTOFILL__DC_LUT_AUTOFILL_DONE__SHIFT 0x00000001
+#define DC_LUT_AUTOFILL__DC_LUT_AUTOFILL_MASK 0x00000001L
+#define DC_LUT_AUTOFILL__DC_LUT_AUTOFILL__SHIFT 0x00000000
+#define DC_LUT_BLACK_OFFSET_BLUE__DC_LUT_BLACK_OFFSET_BLUE_MASK 0x0000ffffL
+#define DC_LUT_BLACK_OFFSET_BLUE__DC_LUT_BLACK_OFFSET_BLUE__SHIFT 0x00000000
+#define DC_LUT_BLACK_OFFSET_GREEN__DC_LUT_BLACK_OFFSET_GREEN_MASK 0x0000ffffL
+#define DC_LUT_BLACK_OFFSET_GREEN__DC_LUT_BLACK_OFFSET_GREEN__SHIFT 0x00000000
+#define DC_LUT_BLACK_OFFSET_RED__DC_LUT_BLACK_OFFSET_RED_MASK 0x0000ffffL
+#define DC_LUT_BLACK_OFFSET_RED__DC_LUT_BLACK_OFFSET_RED__SHIFT 0x00000000
+#define DC_LUT_CONTROL__DC_LUT_DATA_B_FLOAT_POINT_EN_MASK 0x00000020L
+#define DC_LUT_CONTROL__DC_LUT_DATA_B_FLOAT_POINT_EN__SHIFT 0x00000005
+#define DC_LUT_CONTROL__DC_LUT_DATA_B_FORMAT_MASK 0x000000c0L
+#define DC_LUT_CONTROL__DC_LUT_DATA_B_FORMAT__SHIFT 0x00000006
+#define DC_LUT_CONTROL__DC_LUT_DATA_B_SIGNED_EN_MASK 0x00000010L
+#define DC_LUT_CONTROL__DC_LUT_DATA_B_SIGNED_EN__SHIFT 0x00000004
+#define DC_LUT_CONTROL__DC_LUT_DATA_G_FLOAT_POINT_EN_MASK 0x00002000L
+#define DC_LUT_CONTROL__DC_LUT_DATA_G_FLOAT_POINT_EN__SHIFT 0x0000000d
+#define DC_LUT_CONTROL__DC_LUT_DATA_G_FORMAT_MASK 0x0000c000L
+#define DC_LUT_CONTROL__DC_LUT_DATA_G_FORMAT__SHIFT 0x0000000e
+#define DC_LUT_CONTROL__DC_LUT_DATA_G_SIGNED_EN_MASK 0x00001000L
+#define DC_LUT_CONTROL__DC_LUT_DATA_G_SIGNED_EN__SHIFT 0x0000000c
+#define DC_LUT_CONTROL__DC_LUT_DATA_R_FLOAT_POINT_EN_MASK 0x00200000L
+#define DC_LUT_CONTROL__DC_LUT_DATA_R_FLOAT_POINT_EN__SHIFT 0x00000015
+#define DC_LUT_CONTROL__DC_LUT_DATA_R_FORMAT_MASK 0x00c00000L
+#define DC_LUT_CONTROL__DC_LUT_DATA_R_FORMAT__SHIFT 0x00000016
+#define DC_LUT_CONTROL__DC_LUT_DATA_R_SIGNED_EN_MASK 0x00100000L
+#define DC_LUT_CONTROL__DC_LUT_DATA_R_SIGNED_EN__SHIFT 0x00000014
+#define DC_LUT_CONTROL__DC_LUT_INC_B_MASK 0x0000000fL
+#define DC_LUT_CONTROL__DC_LUT_INC_B__SHIFT 0x00000000
+#define DC_LUT_CONTROL__DC_LUT_INC_G_MASK 0x00000f00L
+#define DC_LUT_CONTROL__DC_LUT_INC_G__SHIFT 0x00000008
+#define DC_LUT_CONTROL__DC_LUT_INC_R_MASK 0x000f0000L
+#define DC_LUT_CONTROL__DC_LUT_INC_R__SHIFT 0x00000010
+#define DC_LUT_PWL_DATA__DC_LUT_BASE_MASK 0x0000ffffL
+#define DC_LUT_PWL_DATA__DC_LUT_BASE__SHIFT 0x00000000
+#define DC_LUT_PWL_DATA__DC_LUT_DELTA_MASK 0xffff0000L
+#define DC_LUT_PWL_DATA__DC_LUT_DELTA__SHIFT 0x00000010
+#define DC_LUT_RW_INDEX__DC_LUT_RW_INDEX_MASK 0x000000ffL
+#define DC_LUT_RW_INDEX__DC_LUT_RW_INDEX__SHIFT 0x00000000
+#define DC_LUT_RW_MODE__DC_LUT_RW_MODE_MASK 0x00000001L
+#define DC_LUT_RW_MODE__DC_LUT_RW_MODE__SHIFT 0x00000000
+#define DC_LUT_SEQ_COLOR__DC_LUT_SEQ_COLOR_MASK 0x0000ffffL
+#define DC_LUT_SEQ_COLOR__DC_LUT_SEQ_COLOR__SHIFT 0x00000000
+#define DC_LUT_VGA_ACCESS_ENABLE__DC_LUT_VGA_ACCESS_ENABLE_MASK 0x00000001L
+#define DC_LUT_VGA_ACCESS_ENABLE__DC_LUT_VGA_ACCESS_ENABLE__SHIFT 0x00000000
+#define DC_LUT_WHITE_OFFSET_BLUE__DC_LUT_WHITE_OFFSET_BLUE_MASK 0x0000ffffL
+#define DC_LUT_WHITE_OFFSET_BLUE__DC_LUT_WHITE_OFFSET_BLUE__SHIFT 0x00000000
+#define DC_LUT_WHITE_OFFSET_GREEN__DC_LUT_WHITE_OFFSET_GREEN_MASK 0x0000ffffL
+#define DC_LUT_WHITE_OFFSET_GREEN__DC_LUT_WHITE_OFFSET_GREEN__SHIFT 0x00000000
+#define DC_LUT_WHITE_OFFSET_RED__DC_LUT_WHITE_OFFSET_RED_MASK 0x0000ffffL
+#define DC_LUT_WHITE_OFFSET_RED__DC_LUT_WHITE_OFFSET_RED__SHIFT 0x00000000
+#define DC_LUT_WRITE_EN_MASK__DC_LUT_WRITE_EN_MASK_MASK 0x00000007L
+#define DC_LUT_WRITE_EN_MASK__DC_LUT_WRITE_EN_MASK__SHIFT 0x00000000
+#define DC_MVP_LB_CONTROL__DC_MVP_SPARE_FLOPS_MASK 0x80000000L
+#define DC_MVP_LB_CONTROL__DC_MVP_SPARE_FLOPS__SHIFT 0x0000001f
+#define DC_MVP_LB_CONTROL__DC_MVP_SWAP_LOCK_IN_CAP_MASK 0x10000000L
+#define DC_MVP_LB_CONTROL__DC_MVP_SWAP_LOCK_IN_CAP__SHIFT 0x0000001c
+#define DC_MVP_LB_CONTROL__DC_MVP_SWAP_LOCK_OUT_FORCE_ONE_MASK 0x00001000L
+#define DC_MVP_LB_CONTROL__DC_MVP_SWAP_LOCK_OUT_FORCE_ONE__SHIFT 0x0000000c
+#define DC_MVP_LB_CONTROL__DC_MVP_SWAP_LOCK_OUT_FORCE_ZERO_MASK 0x00010000L
+#define DC_MVP_LB_CONTROL__DC_MVP_SWAP_LOCK_OUT_FORCE_ZERO__SHIFT 0x00000010
+#define DC_MVP_LB_CONTROL__DC_MVP_SWAP_LOCK_OUT_SEL_MASK 0x00000100L
+#define DC_MVP_LB_CONTROL__DC_MVP_SWAP_LOCK_OUT_SEL__SHIFT 0x00000008
+#define DC_MVP_LB_CONTROL__DC_MVP_SWAP_LOCK_STATUS_MASK 0x00100000L
+#define DC_MVP_LB_CONTROL__DC_MVP_SWAP_LOCK_STATUS__SHIFT 0x00000014
+#define DC_MVP_LB_CONTROL__MVP_SWAP_LOCK_IN_MODE_MASK 0x00000003L
+#define DC_MVP_LB_CONTROL__MVP_SWAP_LOCK_IN_MODE__SHIFT 0x00000000
+#define DCO_CLK_CNTL__DCO_TEST_CLK_SEL_MASK 0x0000001fL
+#define DCO_CLK_CNTL__DCO_TEST_CLK_SEL__SHIFT 0x00000000
+#define DCO_CLK_CNTL__DISPCLK_G_ABM_GATE_DIS_MASK 0x00000040L
+#define DCO_CLK_CNTL__DISPCLK_G_ABM_GATE_DIS__SHIFT 0x00000006
+#define DCO_CLK_CNTL__DISPCLK_G_DACA_GATE_DIS_MASK 0x00000100L
+#define DCO_CLK_CNTL__DISPCLK_G_DACA_GATE_DIS__SHIFT 0x00000008
+#define DCO_CLK_CNTL__DISPCLK_G_DACB_GATE_DIS_MASK 0x00000200L
+#define DCO_CLK_CNTL__DISPCLK_G_DACB_GATE_DIS__SHIFT 0x00000009
+#define DCO_CLK_CNTL__DISPCLK_G_DIGA_GATE_DIS_MASK 0x01000000L
+#define DCO_CLK_CNTL__DISPCLK_G_DIGA_GATE_DIS__SHIFT 0x00000018
+#define DCO_CLK_CNTL__DISPCLK_G_DIGB_GATE_DIS_MASK 0x02000000L
+#define DCO_CLK_CNTL__DISPCLK_G_DIGB_GATE_DIS__SHIFT 0x00000019
+#define DCO_CLK_CNTL__DISPCLK_G_DIGC_GATE_DIS_MASK 0x04000000L
+#define DCO_CLK_CNTL__DISPCLK_G_DIGC_GATE_DIS__SHIFT 0x0000001a
+#define DCO_CLK_CNTL__DISPCLK_G_DIGD_GATE_DIS_MASK 0x08000000L
+#define DCO_CLK_CNTL__DISPCLK_G_DIGD_GATE_DIS__SHIFT 0x0000001b
+#define DCO_CLK_CNTL__DISPCLK_G_DIGE_GATE_DIS_MASK 0x10000000L
+#define DCO_CLK_CNTL__DISPCLK_G_DIGE_GATE_DIS__SHIFT 0x0000001c
+#define DCO_CLK_CNTL__DISPCLK_G_DIGF_GATE_DIS_MASK 0x20000000L
+#define DCO_CLK_CNTL__DISPCLK_G_DIGF_GATE_DIS__SHIFT 0x0000001d
+#define DCO_CLK_CNTL__DISPCLK_G_DVO_GATE_DIS_MASK 0x00000080L
+#define DCO_CLK_CNTL__DISPCLK_G_DVO_GATE_DIS__SHIFT 0x00000007
+#define DCO_CLK_CNTL__DISPCLK_G_FMT0_GATE_DIS_MASK 0x00010000L
+#define DCO_CLK_CNTL__DISPCLK_G_FMT0_GATE_DIS__SHIFT 0x00000010
+#define DCO_CLK_CNTL__DISPCLK_G_FMT1_GATE_DIS_MASK 0x00020000L
+#define DCO_CLK_CNTL__DISPCLK_G_FMT1_GATE_DIS__SHIFT 0x00000011
+#define DCO_CLK_CNTL__DISPCLK_G_FMT2_GATE_DIS_MASK 0x00040000L
+#define DCO_CLK_CNTL__DISPCLK_G_FMT2_GATE_DIS__SHIFT 0x00000012
+#define DCO_CLK_CNTL__DISPCLK_G_FMT3_GATE_DIS_MASK 0x00080000L
+#define DCO_CLK_CNTL__DISPCLK_G_FMT3_GATE_DIS__SHIFT 0x00000013
+#define DCO_CLK_CNTL__DISPCLK_G_FMT4_GATE_DIS_MASK 0x00100000L
+#define DCO_CLK_CNTL__DISPCLK_G_FMT4_GATE_DIS__SHIFT 0x00000014
+#define DCO_CLK_CNTL__DISPCLK_G_FMT5_GATE_DIS_MASK 0x00200000L
+#define DCO_CLK_CNTL__DISPCLK_G_FMT5_GATE_DIS__SHIFT 0x00000015
+#define DCO_CLK_CNTL__DISPCLK_R_ABM_GATE_DIS_MASK 0x00001000L
+#define DCO_CLK_CNTL__DISPCLK_R_ABM_GATE_DIS__SHIFT 0x0000000c
+#define DCO_CLK_CNTL__DISPCLK_R_DCO_GATE_DIS_MASK 0x00000020L
+#define DCO_CLK_CNTL__DISPCLK_R_DCO_GATE_DIS__SHIFT 0x00000005
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_ABM_RAMP_DIS_MASK 0x00000040L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_ABM_RAMP_DIS__SHIFT 0x00000006
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DACA_RAMP_DIS_MASK 0x00000100L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DACA_RAMP_DIS__SHIFT 0x00000008
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DACB_RAMP_DIS_MASK 0x00000200L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DACB_RAMP_DIS__SHIFT 0x00000009
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DIGA_RAMP_DIS_MASK 0x01000000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DIGA_RAMP_DIS__SHIFT 0x00000018
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DIGB_RAMP_DIS_MASK 0x02000000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DIGB_RAMP_DIS__SHIFT 0x00000019
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DIGC_RAMP_DIS_MASK 0x04000000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DIGC_RAMP_DIS__SHIFT 0x0000001a
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DIGD_RAMP_DIS_MASK 0x08000000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DIGD_RAMP_DIS__SHIFT 0x0000001b
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DIGE_RAMP_DIS_MASK 0x10000000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DIGE_RAMP_DIS__SHIFT 0x0000001c
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DIGF_RAMP_DIS_MASK 0x20000000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DIGF_RAMP_DIS__SHIFT 0x0000001d
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DVO_RAMP_DIS_MASK 0x00000080L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_DVO_RAMP_DIS__SHIFT 0x00000007
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_FMT0_RAMP_DIS_MASK 0x00010000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_FMT0_RAMP_DIS__SHIFT 0x00000010
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_FMT1_RAMP_DIS_MASK 0x00020000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_FMT1_RAMP_DIS__SHIFT 0x00000011
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_FMT2_RAMP_DIS_MASK 0x00040000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_FMT2_RAMP_DIS__SHIFT 0x00000012
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_FMT3_RAMP_DIS_MASK 0x00080000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_FMT3_RAMP_DIS__SHIFT 0x00000013
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_FMT4_RAMP_DIS_MASK 0x00100000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_FMT4_RAMP_DIS__SHIFT 0x00000014
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_FMT5_RAMP_DIS_MASK 0x00200000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_G_FMT5_RAMP_DIS__SHIFT 0x00000015
+#define DCO_CLK_RAMP_CNTL__DISPCLK_R_ABM_RAMP_DIS_MASK 0x00001000L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_R_ABM_RAMP_DIS__SHIFT 0x0000000c
+#define DCO_CLK_RAMP_CNTL__DISPCLK_R_DCO_RAMP_DIS_MASK 0x00000020L
+#define DCO_CLK_RAMP_CNTL__DISPCLK_R_DCO_RAMP_DIS__SHIFT 0x00000005
+#define DCO_LIGHT_SLEEP_DIS__DPA_LIGHT_SLEEP_DIS_MASK 0x00000008L
+#define DCO_LIGHT_SLEEP_DIS__DPA_LIGHT_SLEEP_DIS__SHIFT 0x00000003
+#define DCO_LIGHT_SLEEP_DIS__DPA_MEM_SHUTDOWN_DIS_MASK 0x00020000L
+#define DCO_LIGHT_SLEEP_DIS__DPA_MEM_SHUTDOWN_DIS__SHIFT 0x00000011
+#define DCO_LIGHT_SLEEP_DIS__DPB_LIGHT_SLEEP_DIS_MASK 0x00000010L
+#define DCO_LIGHT_SLEEP_DIS__DPB_LIGHT_SLEEP_DIS__SHIFT 0x00000004
+#define DCO_LIGHT_SLEEP_DIS__DPB_MEM_SHUTDOWN_DIS_MASK 0x00040000L
+#define DCO_LIGHT_SLEEP_DIS__DPB_MEM_SHUTDOWN_DIS__SHIFT 0x00000012
+#define DCO_LIGHT_SLEEP_DIS__DPC_LIGHT_SLEEP_DIS_MASK 0x00000020L
+#define DCO_LIGHT_SLEEP_DIS__DPC_LIGHT_SLEEP_DIS__SHIFT 0x00000005
+#define DCO_LIGHT_SLEEP_DIS__DPC_MEM_SHUTDOWN_DIS_MASK 0x00080000L
+#define DCO_LIGHT_SLEEP_DIS__DPC_MEM_SHUTDOWN_DIS__SHIFT 0x00000013
+#define DCO_LIGHT_SLEEP_DIS__DPD_LIGHT_SLEEP_DIS_MASK 0x00000040L
+#define DCO_LIGHT_SLEEP_DIS__DPD_LIGHT_SLEEP_DIS__SHIFT 0x00000006
+#define DCO_LIGHT_SLEEP_DIS__DPD_MEM_SHUTDOWN_DIS_MASK 0x00100000L
+#define DCO_LIGHT_SLEEP_DIS__DPD_MEM_SHUTDOWN_DIS__SHIFT 0x00000014
+#define DCO_LIGHT_SLEEP_DIS__DPE_LIGHT_SLEEP_DIS_MASK 0x00000080L
+#define DCO_LIGHT_SLEEP_DIS__DPE_LIGHT_SLEEP_DIS__SHIFT 0x00000007
+#define DCO_LIGHT_SLEEP_DIS__DPE_MEM_SHUTDOWN_DIS_MASK 0x00200000L
+#define DCO_LIGHT_SLEEP_DIS__DPE_MEM_SHUTDOWN_DIS__SHIFT 0x00000015
+#define DCO_LIGHT_SLEEP_DIS__DPF_LIGHT_SLEEP_DIS_MASK 0x00000100L
+#define DCO_LIGHT_SLEEP_DIS__DPF_LIGHT_SLEEP_DIS__SHIFT 0x00000008
+#define DCO_LIGHT_SLEEP_DIS__DPF_MEM_SHUTDOWN_DIS_MASK 0x00400000L
+#define DCO_LIGHT_SLEEP_DIS__DPF_MEM_SHUTDOWN_DIS__SHIFT 0x00000016
+#define DCO_LIGHT_SLEEP_DIS__HDMI0_LIGHT_SLEEP_DIS_MASK 0x00000200L
+#define DCO_LIGHT_SLEEP_DIS__HDMI0_LIGHT_SLEEP_DIS__SHIFT 0x00000009
+#define DCO_LIGHT_SLEEP_DIS__HDMI1_LIGHT_SLEEP_DIS_MASK 0x00000400L
+#define DCO_LIGHT_SLEEP_DIS__HDMI1_LIGHT_SLEEP_DIS__SHIFT 0x0000000a
+#define DCO_LIGHT_SLEEP_DIS__HDMI2_LIGHT_SLEEP_DIS_MASK 0x00000800L
+#define DCO_LIGHT_SLEEP_DIS__HDMI2_LIGHT_SLEEP_DIS__SHIFT 0x0000000b
+#define DCO_LIGHT_SLEEP_DIS__HDMI3_LIGHT_SLEEP_DIS_MASK 0x00001000L
+#define DCO_LIGHT_SLEEP_DIS__HDMI3_LIGHT_SLEEP_DIS__SHIFT 0x0000000c
+#define DCO_LIGHT_SLEEP_DIS__HDMI4_LIGHT_SLEEP_DIS_MASK 0x00002000L
+#define DCO_LIGHT_SLEEP_DIS__HDMI4_LIGHT_SLEEP_DIS__SHIFT 0x0000000d
+#define DCO_LIGHT_SLEEP_DIS__HDMI5_LIGHT_SLEEP_DIS_MASK 0x00004000L
+#define DCO_LIGHT_SLEEP_DIS__HDMI5_LIGHT_SLEEP_DIS__SHIFT 0x0000000e
+#define DCO_LIGHT_SLEEP_DIS__I2C_LIGHT_SLEEP_FORCE_MASK 0x00000002L
+#define DCO_LIGHT_SLEEP_DIS__I2C_LIGHT_SLEEP_FORCE__SHIFT 0x00000001
+#define DCO_LIGHT_SLEEP_DIS__MVP_LIGHT_SLEEP_DIS_MASK 0x00000004L
+#define DCO_LIGHT_SLEEP_DIS__MVP_LIGHT_SLEEP_DIS__SHIFT 0x00000002
+#define DCO_LIGHT_SLEEP_DIS__MVP_MEM_SHUTDOWN_DIS_MASK 0x00010000L
+#define DCO_LIGHT_SLEEP_DIS__MVP_MEM_SHUTDOWN_DIS__SHIFT 0x00000010
+#define DCO_LIGHT_SLEEP_DIS__TVOUT_LIGHT_SLEEP_DIS_MASK 0x00000001L
+#define DCO_LIGHT_SLEEP_DIS__TVOUT_LIGHT_SLEEP_DIS__SHIFT 0x00000000
+#define DCO_MEM_POWER_STATE__DPA_MEM_PWR_STATE_MASK 0x000000c0L
+#define DCO_MEM_POWER_STATE__DPA_MEM_PWR_STATE__SHIFT 0x00000006
+#define DCO_MEM_POWER_STATE__DPB_MEM_PWR_STATE_MASK 0x00000300L
+#define DCO_MEM_POWER_STATE__DPB_MEM_PWR_STATE__SHIFT 0x00000008
+#define DCO_MEM_POWER_STATE__DPC_MEM_PWR_STATE_MASK 0x00000c00L
+#define DCO_MEM_POWER_STATE__DPC_MEM_PWR_STATE__SHIFT 0x0000000a
+#define DCO_MEM_POWER_STATE__DPD_MEM_PWR_STATE_MASK 0x00003000L
+#define DCO_MEM_POWER_STATE__DPD_MEM_PWR_STATE__SHIFT 0x0000000c
+#define DCO_MEM_POWER_STATE__DPE_MEM_PWR_STATE_MASK 0x0000c000L
+#define DCO_MEM_POWER_STATE__DPE_MEM_PWR_STATE__SHIFT 0x0000000e
+#define DCO_MEM_POWER_STATE__DPF_MEM_PWR_STATE_MASK 0x00030000L
+#define DCO_MEM_POWER_STATE__DPF_MEM_PWR_STATE__SHIFT 0x00000010
+#define DCO_MEM_POWER_STATE__HDMI0_MEM_PWR_STATE_MASK 0x000c0000L
+#define DCO_MEM_POWER_STATE__HDMI0_MEM_PWR_STATE__SHIFT 0x00000012
+#define DCO_MEM_POWER_STATE__HDMI1_MEM_PWR_STATE_MASK 0x00300000L
+#define DCO_MEM_POWER_STATE__HDMI1_MEM_PWR_STATE__SHIFT 0x00000014
+#define DCO_MEM_POWER_STATE__HDMI2_MEM_PWR_STATE_MASK 0x00c00000L
+#define DCO_MEM_POWER_STATE__HDMI2_MEM_PWR_STATE__SHIFT 0x00000016
+#define DCO_MEM_POWER_STATE__HDMI3_MEM_PWR_STATE_MASK 0x03000000L
+#define DCO_MEM_POWER_STATE__HDMI3_MEM_PWR_STATE__SHIFT 0x00000018
+#define DCO_MEM_POWER_STATE__HDMI4_MEM_PWR_STATE_MASK 0x0c000000L
+#define DCO_MEM_POWER_STATE__HDMI4_MEM_PWR_STATE__SHIFT 0x0000001a
+#define DCO_MEM_POWER_STATE__HDMI5_MEM_PWR_STATE_MASK 0x30000000L
+#define DCO_MEM_POWER_STATE__HDMI5_MEM_PWR_STATE__SHIFT 0x0000001c
+#define DCO_MEM_POWER_STATE__I2C_MEM_PWR_STATE_MASK 0x0000000cL
+#define DCO_MEM_POWER_STATE__I2C_MEM_PWR_STATE__SHIFT 0x00000002
+#define DCO_MEM_POWER_STATE__MVP_MEM_PWR_STATE_MASK 0x00000030L
+#define DCO_MEM_POWER_STATE__MVP_MEM_PWR_STATE__SHIFT 0x00000004
+#define DCO_MEM_POWER_STATE__TVOUT_MEM_PWR_STATE_MASK 0x00000003L
+#define DCO_MEM_POWER_STATE__TVOUT_MEM_PWR_STATE__SHIFT 0x00000000
+#define DCO_SOFT_RESET__ABM_SOFT_RESET_MASK 0x02000000L
+#define DCO_SOFT_RESET__ABM_SOFT_RESET__SHIFT 0x00000019
+#define DCO_SOFT_RESET__DACA_CFG_IF_SOFT_RESET_MASK 0x20000000L
+#define DCO_SOFT_RESET__DACA_CFG_IF_SOFT_RESET__SHIFT 0x0000001d
+#define DCO_SOFT_RESET__DACA_SOFT_RESET_MASK 0x00000001L
+#define DCO_SOFT_RESET__DACA_SOFT_RESET__SHIFT 0x00000000
+#define DCO_SOFT_RESET__DACB_SOFT_RESET_MASK 0x00000002L
+#define DCO_SOFT_RESET__DACB_SOFT_RESET__SHIFT 0x00000001
+#define DCO_SOFT_RESET__DVO_ENABLE_RST_MASK 0x00000008L
+#define DCO_SOFT_RESET__DVO_ENABLE_RST__SHIFT 0x00000003
+#define DCO_SOFT_RESET__DVO_SOFT_RESET_MASK 0x08000000L
+#define DCO_SOFT_RESET__DVO_SOFT_RESET__SHIFT 0x0000001b
+#define DCO_SOFT_RESET__FMT0_SOFT_RESET_MASK 0x00010000L
+#define DCO_SOFT_RESET__FMT0_SOFT_RESET__SHIFT 0x00000010
+#define DCO_SOFT_RESET__FMT1_SOFT_RESET_MASK 0x00020000L
+#define DCO_SOFT_RESET__FMT1_SOFT_RESET__SHIFT 0x00000011
+#define DCO_SOFT_RESET__FMT2_SOFT_RESET_MASK 0x00040000L
+#define DCO_SOFT_RESET__FMT2_SOFT_RESET__SHIFT 0x00000012
+#define DCO_SOFT_RESET__FMT3_SOFT_RESET_MASK 0x00080000L
+#define DCO_SOFT_RESET__FMT3_SOFT_RESET__SHIFT 0x00000013
+#define DCO_SOFT_RESET__FMT4_SOFT_RESET_MASK 0x00100000L
+#define DCO_SOFT_RESET__FMT4_SOFT_RESET__SHIFT 0x00000014
+#define DCO_SOFT_RESET__FMT5_SOFT_RESET_MASK 0x00200000L
+#define DCO_SOFT_RESET__FMT5_SOFT_RESET__SHIFT 0x00000015
+#define DCO_SOFT_RESET__MVP_SOFT_RESET_MASK 0x01000000L
+#define DCO_SOFT_RESET__MVP_SOFT_RESET__SHIFT 0x00000018
+#define DCO_SOFT_RESET__SOFT_RESET_DVO_MASK 0x00000004L
+#define DCO_SOFT_RESET__SOFT_RESET_DVO__SHIFT 0x00000002
+#define DCO_SOFT_RESET__SRBM_SOFT_RESET_ENABLE_MASK 0x10000000L
+#define DCO_SOFT_RESET__SRBM_SOFT_RESET_ENABLE__SHIFT 0x0000001c
+#define DCO_SOFT_RESET__TVOUT_SOFT_RESET_MASK 0x04000000L
+#define DCO_SOFT_RESET__TVOUT_SOFT_RESET__SHIFT 0x0000001a
+#define DC_PAD_EXTERN_SIG__DC_PAD_EXTERN_SIG_SEL_MASK 0x0000000fL
+#define DC_PAD_EXTERN_SIG__DC_PAD_EXTERN_SIG_SEL__SHIFT 0x00000000
+#define DC_PAD_EXTERN_SIG__MVP_PIXEL_SRC_STATUS_MASK 0x00000030L
+#define DC_PAD_EXTERN_SIG__MVP_PIXEL_SRC_STATUS__SHIFT 0x00000004
+#define DCP_CRC_CONTROL__DCP_CRC_ENABLE_MASK 0x00000001L
+#define DCP_CRC_CONTROL__DCP_CRC_ENABLE__SHIFT 0x00000000
+#define DCP_CRC_CONTROL__DCP_CRC_LINE_SEL_MASK 0x00000300L
+#define DCP_CRC_CONTROL__DCP_CRC_LINE_SEL__SHIFT 0x00000008
+#define DCP_CRC_CONTROL__DCP_CRC_SOURCE_SEL_MASK 0x0000001cL
+#define DCP_CRC_CONTROL__DCP_CRC_SOURCE_SEL__SHIFT 0x00000002
+#define DCP_CRC_CURRENT__DCP_CRC_CURRENT_MASK 0xffffffffL
+#define DCP_CRC_CURRENT__DCP_CRC_CURRENT__SHIFT 0x00000000
+#define DCP_CRC_LAST__DCP_CRC_LAST_MASK 0xffffffffL
+#define DCP_CRC_LAST__DCP_CRC_LAST__SHIFT 0x00000000
+#define DCP_CRC_MASK__DCP_CRC_MASK_MASK 0xffffffffL
+#define DCP_CRC_MASK__DCP_CRC_MASK__SHIFT 0x00000000
+#define DCP_DEBUG2__DCP_DEBUG2_MASK 0xffffffffL
+#define DCP_DEBUG2__DCP_DEBUG2__SHIFT 0x00000000
+#define DCP_DEBUG__DCP_DEBUG_MASK 0xffffffffL
+#define DCP_DEBUG__DCP_DEBUG__SHIFT 0x00000000
+#define DCP_FP_CONVERTED_FIELD__DCP_FP_CONVERTED_FIELD_DATA_MASK 0x0003ffffL
+#define DCP_FP_CONVERTED_FIELD__DCP_FP_CONVERTED_FIELD_DATA__SHIFT 0x00000000
+#define DCP_FP_CONVERTED_FIELD__DCP_FP_CONVERTED_FIELD_INDEX_MASK 0x07f00000L
+#define DCP_FP_CONVERTED_FIELD__DCP_FP_CONVERTED_FIELD_INDEX__SHIFT 0x00000014
+#define DC_PGCNTL_STATUS_REG__DCPG_ECO_DEBUG_MASK 0xffff0000L
+#define DC_PGCNTL_STATUS_REG__DCPG_ECO_DEBUG__SHIFT 0x00000010
+#define DC_PGCNTL_STATUS_REG__IPREQ_IGNORE_STATUS_MASK 0x00000004L
+#define DC_PGCNTL_STATUS_REG__IPREQ_IGNORE_STATUS__SHIFT 0x00000002
+#define DC_PGCNTL_STATUS_REG__SWREQ_RWOP_BUSY_MASK 0x00000001L
+#define DC_PGCNTL_STATUS_REG__SWREQ_RWOP_BUSY__SHIFT 0x00000000
+#define DC_PGCNTL_STATUS_REG__SWREQ_RWOP_FORCE_MASK 0x00000002L
+#define DC_PGCNTL_STATUS_REG__SWREQ_RWOP_FORCE__SHIFT 0x00000001
+#define DC_PGFSM_CONFIG_REG__PGFSM_CONFIG_REG_MASK 0xffffffffL
+#define DC_PGFSM_CONFIG_REG__PGFSM_CONFIG_REG__SHIFT 0x00000000
+#define DC_PGFSM_WRITE_REG__PGFSM_WRITE_REG_MASK 0xffffffffL
+#define DC_PGFSM_WRITE_REG__PGFSM_WRITE_REG__SHIFT 0x00000000
+#define DCP_GSL_CONTROL__DCP_GSL0_EN_MASK 0x00000001L
+#define DCP_GSL_CONTROL__DCP_GSL0_EN__SHIFT 0x00000000
+#define DCP_GSL_CONTROL__DCP_GSL1_EN_MASK 0x00000002L
+#define DCP_GSL_CONTROL__DCP_GSL1_EN__SHIFT 0x00000001
+#define DCP_GSL_CONTROL__DCP_GSL2_EN_MASK 0x00000004L
+#define DCP_GSL_CONTROL__DCP_GSL2_EN__SHIFT 0x00000002
+#define DCP_GSL_CONTROL__DCP_GSL_DELAY_SURFACE_UPDATE_PENDING_MASK 0x08000000L
+#define DCP_GSL_CONTROL__DCP_GSL_DELAY_SURFACE_UPDATE_PENDING__SHIFT 0x0000001b
+#define DCP_GSL_CONTROL__DCP_GSL_HSYNC_FLIP_CHECK_DELAY_MASK 0xf0000000L
+#define DCP_GSL_CONTROL__DCP_GSL_HSYNC_FLIP_CHECK_DELAY__SHIFT 0x0000001c
+#define DCP_GSL_CONTROL__DCP_GSL_HSYNC_FLIP_FORCE_DELAY_MASK 0x0000f000L
+#define DCP_GSL_CONTROL__DCP_GSL_HSYNC_FLIP_FORCE_DELAY__SHIFT 0x0000000c
+#define DCP_GSL_CONTROL__DCP_GSL_MASTER_EN_MASK 0x00010000L
+#define DCP_GSL_CONTROL__DCP_GSL_MASTER_EN__SHIFT 0x00000010
+#define DCP_GSL_CONTROL__DCP_GSL_MODE_MASK 0x00000300L
+#define DCP_GSL_CONTROL__DCP_GSL_MODE__SHIFT 0x00000008
+#define DCP_GSL_CONTROL__DCP_GSL_SYNC_SOURCE_MASK 0x03000000L
+#define DCP_GSL_CONTROL__DCP_GSL_SYNC_SOURCE__SHIFT 0x00000018
+#define DCPG_TEST_DEBUG_DATA__DCPG_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define DCPG_TEST_DEBUG_DATA__DCPG_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define DCPG_TEST_DEBUG_INDEX__DCPG_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define DCPG_TEST_DEBUG_INDEX__DCPG_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define DCPG_TEST_DEBUG_INDEX__DCPG_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define DCPG_TEST_DEBUG_INDEX__DCPG_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define DC_PINSTRAPS__DC_PINSTRAPS_AUDIO_MASK 0x0000c000L
+#define DC_PINSTRAPS__DC_PINSTRAPS_AUDIO__SHIFT 0x0000000e
+#define DC_PINSTRAPS__DC_PINSTRAPS_BIF_CEC_DIS_MASK 0x00000400L
+#define DC_PINSTRAPS__DC_PINSTRAPS_BIF_CEC_DIS__SHIFT 0x0000000a
+#define DC_PINSTRAPS__DC_PINSTRAPS_CCBYPASS_MASK 0x00010000L
+#define DC_PINSTRAPS__DC_PINSTRAPS_CCBYPASS__SHIFT 0x00000010
+#define DC_PINSTRAPS__DC_PINSTRAPS_SMS_EN_HARD_MASK 0x00002000L
+#define DC_PINSTRAPS__DC_PINSTRAPS_SMS_EN_HARD__SHIFT 0x0000000d
+#define DC_PINSTRAPS__DC_PINSTRAPS_VIP_DEVICE_MASK 0x00000800L
+#define DC_PINSTRAPS__DC_PINSTRAPS_VIP_DEVICE__SHIFT 0x0000000b
+#define DCP_LB_DATA_GAP_BETWEEN_CHUNK__DCP_LB_GAP_BETWEEN_CHUNK_20BPP_MASK 0x0000000fL
+#define DCP_LB_DATA_GAP_BETWEEN_CHUNK__DCP_LB_GAP_BETWEEN_CHUNK_20BPP__SHIFT 0x00000000
+#define DCP_LB_DATA_GAP_BETWEEN_CHUNK__DCP_LB_GAP_BETWEEN_CHUNK_30BPP_MASK 0x000000f0L
+#define DCP_LB_DATA_GAP_BETWEEN_CHUNK__DCP_LB_GAP_BETWEEN_CHUNK_30BPP__SHIFT 0x00000004
+#define DCP_RANDOM_SEEDS__DCP_RAND_B_SEED_MASK 0x00ff0000L
+#define DCP_RANDOM_SEEDS__DCP_RAND_B_SEED__SHIFT 0x00000010
+#define DCP_RANDOM_SEEDS__DCP_RAND_G_SEED_MASK 0x0000ff00L
+#define DCP_RANDOM_SEEDS__DCP_RAND_G_SEED__SHIFT 0x00000008
+#define DCP_RANDOM_SEEDS__DCP_RAND_R_SEED_MASK 0x000000ffL
+#define DCP_RANDOM_SEEDS__DCP_RAND_R_SEED__SHIFT 0x00000000
+#define DCP_SPATIAL_DITHER_CNTL__DCP_FRAME_RANDOM_ENABLE_MASK 0x00000100L
+#define DCP_SPATIAL_DITHER_CNTL__DCP_FRAME_RANDOM_ENABLE__SHIFT 0x00000008
+#define DCP_SPATIAL_DITHER_CNTL__DCP_HIGHPASS_RANDOM_ENABLE_MASK 0x00000400L
+#define DCP_SPATIAL_DITHER_CNTL__DCP_HIGHPASS_RANDOM_ENABLE__SHIFT 0x0000000a
+#define DCP_SPATIAL_DITHER_CNTL__DCP_RGB_RANDOM_ENABLE_MASK 0x00000200L
+#define DCP_SPATIAL_DITHER_CNTL__DCP_RGB_RANDOM_ENABLE__SHIFT 0x00000009
+#define DCP_SPATIAL_DITHER_CNTL__DCP_SPATIAL_DITHER_DEPTH_MASK 0x00000040L
+#define DCP_SPATIAL_DITHER_CNTL__DCP_SPATIAL_DITHER_DEPTH__SHIFT 0x00000006
+#define DCP_SPATIAL_DITHER_CNTL__DCP_SPATIAL_DITHER_EN_MASK 0x00000001L
+#define DCP_SPATIAL_DITHER_CNTL__DCP_SPATIAL_DITHER_EN__SHIFT 0x00000000
+#define DCP_SPATIAL_DITHER_CNTL__DCP_SPATIAL_DITHER_MODE_MASK 0x00000030L
+#define DCP_SPATIAL_DITHER_CNTL__DCP_SPATIAL_DITHER_MODE__SHIFT 0x00000004
+#define DCP_TEST_DEBUG_DATA__DCP_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define DCP_TEST_DEBUG_DATA__DCP_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define DCP_TEST_DEBUG_INDEX__DCP_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define DCP_TEST_DEBUG_INDEX__DCP_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define DCP_TEST_DEBUG_INDEX__DCP_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define DCP_TEST_DEBUG_INDEX__DCP_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT0_RDWR_DELAY_MASK 0x00000007L
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT0_RDWR_DELAY__SHIFT 0x00000000
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT0_RDWR_TIMEOUT_DIS_MASK 0x00000008L
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT0_RDWR_TIMEOUT_DIS__SHIFT 0x00000003
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT1_RDWR_DELAY_MASK 0x00000070L
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT1_RDWR_DELAY__SHIFT 0x00000004
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT1_RDWR_TIMEOUT_DIS_MASK 0x00000080L
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT1_RDWR_TIMEOUT_DIS__SHIFT 0x00000007
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT2_RDWR_DELAY_MASK 0x00000700L
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT2_RDWR_DELAY__SHIFT 0x00000008
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT2_RDWR_TIMEOUT_DIS_MASK 0x00000800L
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT2_RDWR_TIMEOUT_DIS__SHIFT 0x0000000b
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT3_RDWR_DELAY_MASK 0x00007000L
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT3_RDWR_DELAY__SHIFT 0x0000000c
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT3_RDWR_TIMEOUT_DIS_MASK 0x00008000L
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT3_RDWR_TIMEOUT_DIS__SHIFT 0x0000000f
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT4_RDWR_DELAY_MASK 0x00070000L
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT4_RDWR_DELAY__SHIFT 0x00000010
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT4_RDWR_TIMEOUT_DIS_MASK 0x00080000L
+#define DC_RBBMIF_RDWR_CNTL1__DC_RBBMIF_CLIENT4_RDWR_TIMEOUT_DIS__SHIFT 0x00000013
+#define DC_RBBMIF_RDWR_CNTL2__DC_RBBMIF_CLIENT8_RDWR_DELAY_MASK 0x00000007L
+#define DC_RBBMIF_RDWR_CNTL2__DC_RBBMIF_CLIENT8_RDWR_DELAY__SHIFT 0x00000000
+#define DC_RBBMIF_RDWR_CNTL2__DC_RBBMIF_CLIENT8_RDWR_TIMEOUT_DIS_MASK 0x00000008L
+#define DC_RBBMIF_RDWR_CNTL2__DC_RBBMIF_CLIENT8_RDWR_TIMEOUT_DIS__SHIFT 0x00000003
+#define DC_RBBMIF_RDWR_CNTL2__DC_RBBMIF_CLIENT9_RDWR_DELAY_MASK 0x00000070L
+#define DC_RBBMIF_RDWR_CNTL2__DC_RBBMIF_CLIENT9_RDWR_DELAY__SHIFT 0x00000004
+#define DC_RBBMIF_RDWR_CNTL2__DC_RBBMIF_CLIENT9_RDWR_TIMEOUT_DIS_MASK 0x00000080L
+#define DC_RBBMIF_RDWR_CNTL2__DC_RBBMIF_CLIENT9_RDWR_TIMEOUT_DIS__SHIFT 0x00000007
+#define DC_REF_CLK_CNTL__GENLK_CLK_OUTPUT_SEL_MASK 0x00000300L
+#define DC_REF_CLK_CNTL__GENLK_CLK_OUTPUT_SEL__SHIFT 0x00000008
+#define DC_REF_CLK_CNTL__HSYNCA_OUTPUT_SEL_MASK 0x00000003L
+#define DC_REF_CLK_CNTL__HSYNCA_OUTPUT_SEL__SHIFT 0x00000000
+#define DC_XDMA_INTERFACE_CNTL__DC_FLIP_PENDING_TO_DCP_MASK 0x00400000L
+#define DC_XDMA_INTERFACE_CNTL__DC_FLIP_PENDING_TO_DCP__SHIFT 0x00000016
+#define DC_XDMA_INTERFACE_CNTL__DC_XDMA_FLIP_PENDING_MASK 0x00010000L
+#define DC_XDMA_INTERFACE_CNTL__DC_XDMA_FLIP_PENDING__SHIFT 0x00000010
+#define DC_XDMA_INTERFACE_CNTL__XDMA_M_FLIP_PENDING_TO_DCP_MASK 0x00100000L
+#define DC_XDMA_INTERFACE_CNTL__XDMA_M_FLIP_PENDING_TO_DCP__SHIFT 0x00000014
+#define DC_XDMA_INTERFACE_CNTL__XDMA_PIPE_ENABLE_MASK 0x0000003fL
+#define DC_XDMA_INTERFACE_CNTL__XDMA_PIPE_ENABLE__SHIFT 0x00000000
+#define DC_XDMA_INTERFACE_CNTL__XDMA_PIPE_SEL_MASK 0x00000700L
+#define DC_XDMA_INTERFACE_CNTL__XDMA_PIPE_SEL__SHIFT 0x00000008
+#define DC_XDMA_INTERFACE_CNTL__XDMA_S_FLIP_PENDING_TO_DCP_MASK 0x00200000L
+#define DC_XDMA_INTERFACE_CNTL__XDMA_S_FLIP_PENDING_TO_DCP__SHIFT 0x00000015
+#define DEGAMMA_CONTROL__CURSOR_DEGAMMA_MODE_MASK 0x00003000L
+#define DEGAMMA_CONTROL__CURSOR_DEGAMMA_MODE__SHIFT 0x0000000c
+#define DEGAMMA_CONTROL__GRPH_DEGAMMA_MODE_MASK 0x00000003L
+#define DEGAMMA_CONTROL__GRPH_DEGAMMA_MODE__SHIFT 0x00000000
+#define DEGAMMA_CONTROL__OVL_DEGAMMA_MODE_MASK 0x00000030L
+#define DEGAMMA_CONTROL__OVL_DEGAMMA_MODE__SHIFT 0x00000004
+#define DENORM_CONTROL__DENORM_MODE_MASK 0x00000007L
+#define DENORM_CONTROL__DENORM_MODE__SHIFT 0x00000000
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_CHG_DONE_MASK 0x00080000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_CHG_DONE__SHIFT 0x00000013
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_CHG_MODE_MASK 0x00018000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_CHG_MODE__SHIFT 0x0000000f
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_CHGTOG_MASK 0x00020000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_CHGTOG__SHIFT 0x00000011
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_DONETOG_MASK 0x00040000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_DONETOG__SHIFT 0x00000012
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_RDIVIDER_MASK 0x00007f00L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_RDIVIDER__SHIFT 0x00000008
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_WDIVIDER_MASK 0x0000007fL
+#define DENTIST_DISPCLK_CNTL__DENTIST_DISPCLK_WDIVIDER__SHIFT 0x00000000
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPREFCLK_CHG_DONE_MASK 0x00100000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPREFCLK_CHG_DONE__SHIFT 0x00000014
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPREFCLK_CHGTOG_MASK 0x00200000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPREFCLK_CHGTOG__SHIFT 0x00000015
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPREFCLK_DONETOG_MASK 0x00400000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPREFCLK_DONETOG__SHIFT 0x00000016
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPREFCLK_WDIVIDER_MASK 0x7f000000L
+#define DENTIST_DISPCLK_CNTL__DENTIST_DPREFCLK_WDIVIDER__SHIFT 0x00000018
+#define DIG_BE_CNTL__DIG_FE_SOURCE_SELECT_MASK 0x00003f00L
+#define DIG_BE_CNTL__DIG_FE_SOURCE_SELECT__SHIFT 0x00000008
+#define DIG_BE_CNTL__DIG_HPD_SELECT_MASK 0x70000000L
+#define DIG_BE_CNTL__DIG_HPD_SELECT__SHIFT 0x0000001c
+#define DIG_BE_CNTL__DIG_MODE_MASK 0x00070000L
+#define DIG_BE_CNTL__DIG_MODE__SHIFT 0x00000010
+#define DIG_BE_EN_CNTL__DIG_ENABLE_MASK 0x00000001L
+#define DIG_BE_EN_CNTL__DIG_ENABLE__SHIFT 0x00000000
+#define DIG_BE_EN_CNTL__DIG_SYMCLK_BE_ON_MASK 0x00000100L
+#define DIG_BE_EN_CNTL__DIG_SYMCLK_BE_ON__SHIFT 0x00000008
+#define DIG_CLOCK_PATTERN__DIG_CLOCK_PATTERN_MASK 0x000003ffL
+#define DIG_CLOCK_PATTERN__DIG_CLOCK_PATTERN__SHIFT 0x00000000
+#define DIG_DISPCLK_SWITCH_CNTL__DIG_DISPCLK_SWITCH_POINT_MASK 0x00000001L
+#define DIG_DISPCLK_SWITCH_CNTL__DIG_DISPCLK_SWITCH_POINT__SHIFT 0x00000000
+#define DIG_DISPCLK_SWITCH_STATUS__DIG_DISPCLK_SWITCH_ALLOWED_INT_ACK_MASK 0x00000100L
+#define DIG_DISPCLK_SWITCH_STATUS__DIG_DISPCLK_SWITCH_ALLOWED_INT_ACK__SHIFT 0x00000008
+#define DIG_DISPCLK_SWITCH_STATUS__DIG_DISPCLK_SWITCH_ALLOWED_INT_MASK 0x00000010L
+#define DIG_DISPCLK_SWITCH_STATUS__DIG_DISPCLK_SWITCH_ALLOWED_INT_MASK_MASK 0x00001000L
+#define DIG_DISPCLK_SWITCH_STATUS__DIG_DISPCLK_SWITCH_ALLOWED_INT_MASK__SHIFT 0x0000000c
+#define DIG_DISPCLK_SWITCH_STATUS__DIG_DISPCLK_SWITCH_ALLOWED_INT__SHIFT 0x00000004
+#define DIG_DISPCLK_SWITCH_STATUS__DIG_DISPCLK_SWITCH_ALLOWED_MASK 0x00000001L
+#define DIG_DISPCLK_SWITCH_STATUS__DIG_DISPCLK_SWITCH_ALLOWED__SHIFT 0x00000000
+#define DIG_FE_CNTL__DIG_DUAL_LINK_ENABLE_MASK 0x00010000L
+#define DIG_FE_CNTL__DIG_DUAL_LINK_ENABLE__SHIFT 0x00000010
+#define DIG_FE_CNTL__DIG_RB_SWITCH_EN_MASK 0x00100000L
+#define DIG_FE_CNTL__DIG_RB_SWITCH_EN__SHIFT 0x00000014
+#define DIG_FE_CNTL__DIG_SOURCE_SELECT_MASK 0x00000007L
+#define DIG_FE_CNTL__DIG_SOURCE_SELECT__SHIFT 0x00000000
+#define DIG_FE_CNTL__DIG_START_MASK 0x00000400L
+#define DIG_FE_CNTL__DIG_START__SHIFT 0x0000000a
+#define DIG_FE_CNTL__DIG_STEREOSYNC_GATE_EN_MASK 0x00000100L
+#define DIG_FE_CNTL__DIG_STEREOSYNC_GATE_EN__SHIFT 0x00000008
+#define DIG_FE_CNTL__DIG_STEREOSYNC_SELECT_MASK 0x00000070L
+#define DIG_FE_CNTL__DIG_STEREOSYNC_SELECT__SHIFT 0x00000004
+#define DIG_FE_CNTL__DIG_SWAP_MASK 0x00040000L
+#define DIG_FE_CNTL__DIG_SWAP__SHIFT 0x00000012
+#define DIG_FE_CNTL__DIG_SYMCLK_FE_ON_MASK 0x01000000L
+#define DIG_FE_CNTL__DIG_SYMCLK_FE_ON__SHIFT 0x00000018
+#define DIG_FIFO_STATUS__DIG_FIFO_CAL_AVERAGE_LEVEL_MASK 0x0000fc00L
+#define DIG_FIFO_STATUS__DIG_FIFO_CAL_AVERAGE_LEVEL__SHIFT 0x0000000a
+#define DIG_FIFO_STATUS__DIG_FIFO_CALIBRATED_MASK 0x20000000L
+#define DIG_FIFO_STATUS__DIG_FIFO_CALIBRATED__SHIFT 0x0000001d
+#define DIG_FIFO_STATUS__DIG_FIFO_ERROR_ACK_MASK 0x00000100L
+#define DIG_FIFO_STATUS__DIG_FIFO_ERROR_ACK__SHIFT 0x00000008
+#define DIG_FIFO_STATUS__DIG_FIFO_FORCE_RECAL_AVERAGE_MASK 0x40000000L
+#define DIG_FIFO_STATUS__DIG_FIFO_FORCE_RECAL_AVERAGE__SHIFT 0x0000001e
+#define DIG_FIFO_STATUS__DIG_FIFO_FORCE_RECOMP_MINMAX_MASK 0x80000000L
+#define DIG_FIFO_STATUS__DIG_FIFO_FORCE_RECOMP_MINMAX__SHIFT 0x0000001f
+#define DIG_FIFO_STATUS__DIG_FIFO_LEVEL_ERROR_MASK 0x00000001L
+#define DIG_FIFO_STATUS__DIG_FIFO_LEVEL_ERROR__SHIFT 0x00000000
+#define DIG_FIFO_STATUS__DIG_FIFO_MAXIMUM_LEVEL_MASK 0x001f0000L
+#define DIG_FIFO_STATUS__DIG_FIFO_MAXIMUM_LEVEL__SHIFT 0x00000010
+#define DIG_FIFO_STATUS__DIG_FIFO_MINIMUM_LEVEL_MASK 0x03c00000L
+#define DIG_FIFO_STATUS__DIG_FIFO_MINIMUM_LEVEL__SHIFT 0x00000016
+#define DIG_FIFO_STATUS__DIG_FIFO_OVERWRITE_LEVEL_MASK 0x000000fcL
+#define DIG_FIFO_STATUS__DIG_FIFO_OVERWRITE_LEVEL__SHIFT 0x00000002
+#define DIG_FIFO_STATUS__DIG_FIFO_USE_OVERWRITE_LEVEL_MASK 0x00000002L
+#define DIG_FIFO_STATUS__DIG_FIFO_USE_OVERWRITE_LEVEL__SHIFT 0x00000001
+#define DIG_LANE_ENABLE__DIG_CLK_EN_MASK 0x00000100L
+#define DIG_LANE_ENABLE__DIG_CLK_EN__SHIFT 0x00000008
+#define DIG_LANE_ENABLE__DIG_LANE0EN_MASK 0x00000001L
+#define DIG_LANE_ENABLE__DIG_LANE0EN__SHIFT 0x00000000
+#define DIG_LANE_ENABLE__DIG_LANE1EN_MASK 0x00000002L
+#define DIG_LANE_ENABLE__DIG_LANE1EN__SHIFT 0x00000001
+#define DIG_LANE_ENABLE__DIG_LANE2EN_MASK 0x00000004L
+#define DIG_LANE_ENABLE__DIG_LANE2EN__SHIFT 0x00000002
+#define DIG_LANE_ENABLE__DIG_LANE3EN_MASK 0x00000008L
+#define DIG_LANE_ENABLE__DIG_LANE3EN__SHIFT 0x00000003
+#define DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_DATA_SEL_MASK 0x00000300L
+#define DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_DATA_SEL__SHIFT 0x00000008
+#define DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_EN_MASK 0x00000001L
+#define DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_EN__SHIFT 0x00000000
+#define DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_LINK_SEL_MASK 0x00000010L
+#define DIG_OUTPUT_CRC_CNTL__DIG_OUTPUT_CRC_LINK_SEL__SHIFT 0x00000004
+#define DIG_OUTPUT_CRC_RESULT__DIG_OUTPUT_CRC_RESULT_MASK 0x3fffffffL
+#define DIG_OUTPUT_CRC_RESULT__DIG_OUTPUT_CRC_RESULT__SHIFT 0x00000000
+#define DIG_RANDOM_PATTERN_SEED__DIG_RANDOM_PATTERN_SEED_MASK 0x00ffffffL
+#define DIG_RANDOM_PATTERN_SEED__DIG_RANDOM_PATTERN_SEED__SHIFT 0x00000000
+#define DIG_RANDOM_PATTERN_SEED__DIG_RAN_PAT_DURING_DE_ONLY_MASK 0x01000000L
+#define DIG_RANDOM_PATTERN_SEED__DIG_RAN_PAT_DURING_DE_ONLY__SHIFT 0x00000018
+#define DIG_SOFT_RESET__DIGA_BE_SOFT_RESET_MASK 0x00000002L
+#define DIG_SOFT_RESET__DIGA_BE_SOFT_RESET__SHIFT 0x00000001
+#define DIG_SOFT_RESET__DIGA_FE_SOFT_RESET_MASK 0x00000001L
+#define DIG_SOFT_RESET__DIGA_FE_SOFT_RESET__SHIFT 0x00000000
+#define DIG_SOFT_RESET__DIGB_BE_SOFT_RESET_MASK 0x00000020L
+#define DIG_SOFT_RESET__DIGB_BE_SOFT_RESET__SHIFT 0x00000005
+#define DIG_SOFT_RESET__DIGB_FE_SOFT_RESET_MASK 0x00000010L
+#define DIG_SOFT_RESET__DIGB_FE_SOFT_RESET__SHIFT 0x00000004
+#define DIG_SOFT_RESET__DIGC_BE_SOFT_RESET_MASK 0x00000200L
+#define DIG_SOFT_RESET__DIGC_BE_SOFT_RESET__SHIFT 0x00000009
+#define DIG_SOFT_RESET__DIGC_FE_SOFT_RESET_MASK 0x00000100L
+#define DIG_SOFT_RESET__DIGC_FE_SOFT_RESET__SHIFT 0x00000008
+#define DIG_SOFT_RESET__DIGD_BE_SOFT_RESET_MASK 0x00002000L
+#define DIG_SOFT_RESET__DIGD_BE_SOFT_RESET__SHIFT 0x0000000d
+#define DIG_SOFT_RESET__DIGD_FE_SOFT_RESET_MASK 0x00001000L
+#define DIG_SOFT_RESET__DIGD_FE_SOFT_RESET__SHIFT 0x0000000c
+#define DIG_SOFT_RESET__DIGE_BE_SOFT_RESET_MASK 0x00020000L
+#define DIG_SOFT_RESET__DIGE_BE_SOFT_RESET__SHIFT 0x00000011
+#define DIG_SOFT_RESET__DIGE_FE_SOFT_RESET_MASK 0x00010000L
+#define DIG_SOFT_RESET__DIGE_FE_SOFT_RESET__SHIFT 0x00000010
+#define DIG_SOFT_RESET__DIGF_BE_SOFT_RESET_MASK 0x00200000L
+#define DIG_SOFT_RESET__DIGF_BE_SOFT_RESET__SHIFT 0x00000015
+#define DIG_SOFT_RESET__DIGF_FE_SOFT_RESET_MASK 0x00100000L
+#define DIG_SOFT_RESET__DIGF_FE_SOFT_RESET__SHIFT 0x00000014
+#define DIG_TEST_PATTERN__DIG_HALF_CLOCK_PATTERN_SEL_MASK 0x00000002L
+#define DIG_TEST_PATTERN__DIG_HALF_CLOCK_PATTERN_SEL__SHIFT 0x00000001
+#define DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_OUT_EN_MASK 0x00000010L
+#define DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_OUT_EN__SHIFT 0x00000004
+#define DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_RESET_MASK 0x00000020L
+#define DIG_TEST_PATTERN__DIG_RANDOM_PATTERN_RESET__SHIFT 0x00000005
+#define DIG_TEST_PATTERN__DIG_STATIC_TEST_PATTERN_MASK 0x03ff0000L
+#define DIG_TEST_PATTERN__DIG_STATIC_TEST_PATTERN__SHIFT 0x00000010
+#define DIG_TEST_PATTERN__DIG_TEST_PATTERN_EXTERNAL_RESET_EN_MASK 0x00000040L
+#define DIG_TEST_PATTERN__DIG_TEST_PATTERN_EXTERNAL_RESET_EN__SHIFT 0x00000006
+#define DIG_TEST_PATTERN__DIG_TEST_PATTERN_OUT_EN_MASK 0x00000001L
+#define DIG_TEST_PATTERN__DIG_TEST_PATTERN_OUT_EN__SHIFT 0x00000000
+#define DIG_TEST_PATTERN__LVDS_EYE_PATTERN_MASK 0x00000100L
+#define DIG_TEST_PATTERN__LVDS_EYE_PATTERN__SHIFT 0x00000008
+#define DIG_TEST_PATTERN__LVDS_TEST_CLOCK_DATA_MASK 0x00000004L
+#define DIG_TEST_PATTERN__LVDS_TEST_CLOCK_DATA__SHIFT 0x00000002
+#define DISPCLK_CGTT_BLK_CTRL_REG__DISPCLK_TURN_OFF_DELAY_MASK 0x00000ff0L
+#define DISPCLK_CGTT_BLK_CTRL_REG__DISPCLK_TURN_OFF_DELAY__SHIFT 0x00000004
+#define DISPCLK_CGTT_BLK_CTRL_REG__DISPCLK_TURN_ON_DELAY_MASK 0x0000000fL
+#define DISPCLK_CGTT_BLK_CTRL_REG__DISPCLK_TURN_ON_DELAY__SHIFT 0x00000000
+#define DISPCLK_FREQ_CHANGE_CNTL__DCCG_FIFO_ERRDET_OVR_EN_MASK 0x40000000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DCCG_FIFO_ERRDET_OVR_EN__SHIFT 0x0000001e
+#define DISPCLK_FREQ_CHANGE_CNTL__DCCG_FIFO_ERRDET_RESET_MASK 0x10000000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DCCG_FIFO_ERRDET_RESET__SHIFT 0x0000001c
+#define DISPCLK_FREQ_CHANGE_CNTL__DCCG_FIFO_ERRDET_STATE_MASK 0x20000000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DCCG_FIFO_ERRDET_STATE__SHIFT 0x0000001d
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_CHG_FWD_CORR_DISABLE_MASK 0x80000000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_CHG_FWD_CORR_DISABLE__SHIFT 0x0000001f
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_FREQ_RAMP_DONE_MASK 0x00100000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_FREQ_RAMP_DONE__SHIFT 0x00000014
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_MAX_ERRDET_CYCLES_MASK 0x0e000000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_MAX_ERRDET_CYCLES__SHIFT 0x00000019
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_STEP_DELAY_MASK 0x00003fffL
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_STEP_DELAY__SHIFT 0x00000000
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_STEP_SIZE_MASK 0x000f0000L
+#define DISPCLK_FREQ_CHANGE_CNTL__DISPCLK_STEP_SIZE__SHIFT 0x00000010
+#define DISP_INTERRUPT_STATUS__ABM1_BL_UPDATE_INT_MASK 0x40000000L
+#define DISP_INTERRUPT_STATUS__ABM1_BL_UPDATE_INT__SHIFT 0x0000001e
+#define DISP_INTERRUPT_STATUS__ABM1_HG_READY_INT_MASK 0x10000000L
+#define DISP_INTERRUPT_STATUS__ABM1_HG_READY_INT__SHIFT 0x0000001c
+#define DISP_INTERRUPT_STATUS__ABM1_LS_READY_INT_MASK 0x20000000L
+#define DISP_INTERRUPT_STATUS__ABM1_LS_READY_INT__SHIFT 0x0000001d
+#define DISP_INTERRUPT_STATUS__AUX1_LS_DONE_INTERRUPT_MASK 0x00100000L
+#define DISP_INTERRUPT_STATUS__AUX1_LS_DONE_INTERRUPT__SHIFT 0x00000014
+#define DISP_INTERRUPT_STATUS__AUX1_SW_DONE_INTERRUPT_MASK 0x00080000L
+#define DISP_INTERRUPT_STATUS__AUX1_SW_DONE_INTERRUPT__SHIFT 0x00000013
+#define DISP_INTERRUPT_STATUS_CONTINUE2__AUX3_LS_DONE_INTERRUPT_MASK 0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__AUX3_LS_DONE_INTERRUPT__SHIFT 0x00000014
+#define DISP_INTERRUPT_STATUS_CONTINUE2__AUX3_SW_DONE_INTERRUPT_MASK 0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__AUX3_SW_DONE_INTERRUPT__SHIFT 0x00000013
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_FORCE_COUNT_NOW_INTERRUPT_MASK 0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_FORCE_COUNT_NOW_INTERRUPT__SHIFT 0x00000006
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_FORCE_VSYNC_NEXT_LINE_INTERRUPT_MASK 0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_FORCE_VSYNC_NEXT_LINE_INTERRUPT__SHIFT 0x00000005
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK 0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT 0x0000000a
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_SNAPSHOT_INTERRUPT_MASK 0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_SNAPSHOT_INTERRUPT__SHIFT 0x00000004
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_TRIGA_INTERRUPT_MASK 0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_TRIGA_INTERRUPT__SHIFT 0x00000007
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_TRIGB_INTERRUPT_MASK 0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_TRIGB_INTERRUPT__SHIFT 0x00000008
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_VSYNC_NOM_INTERRUPT_MASK 0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__CRTC3_VSYNC_NOM_INTERRUPT__SHIFT 0x00000009
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DC_HPD3_INTERRUPT_MASK 0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DC_HPD3_INTERRUPT__SHIFT 0x00000011
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DC_HPD3_RX_INTERRUPT_MASK 0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DC_HPD3_RX_INTERRUPT__SHIFT 0x00000012
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DIGC_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK 0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DIGC_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT 0x0000000f
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DIGC_DP_VID_STREAM_DISABLE_INTERRUPT_MASK 0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DIGC_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT 0x00000010
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DISP_INTERRUPT_STATUS_CONTINUE3_MASK 0x80000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__DISP_INTERRUPT_STATUS_CONTINUE3__SHIFT 0x0000001f
+#define DISP_INTERRUPT_STATUS_CONTINUE2__LB_D3_VBLANK_INTERRUPT_MASK 0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__LB_D3_VBLANK_INTERRUPT__SHIFT 0x00000003
+#define DISP_INTERRUPT_STATUS_CONTINUE2__LB_D3_VLINE_INTERRUPT_MASK 0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__LB_D3_VLINE_INTERRUPT__SHIFT 0x00000002
+#define DISP_INTERRUPT_STATUS_CONTINUE2__SCL_DISP3_MODE_CHANGE_INTERRUPT_MASK 0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE2__SCL_DISP3_MODE_CHANGE_INTERRUPT__SHIFT 0x00000000
+#define DISP_INTERRUPT_STATUS_CONTINUE3__AUX4_LS_DONE_INTERRUPT_MASK 0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__AUX4_LS_DONE_INTERRUPT__SHIFT 0x00000014
+#define DISP_INTERRUPT_STATUS_CONTINUE3__AUX4_SW_DONE_INTERRUPT_MASK 0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__AUX4_SW_DONE_INTERRUPT__SHIFT 0x00000013
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_FORCE_COUNT_NOW_INTERRUPT_MASK 0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_FORCE_COUNT_NOW_INTERRUPT__SHIFT 0x00000006
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_FORCE_VSYNC_NEXT_LINE_INTERRUPT_MASK 0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_FORCE_VSYNC_NEXT_LINE_INTERRUPT__SHIFT 0x00000005
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK 0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT 0x0000000a
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_SNAPSHOT_INTERRUPT_MASK 0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_SNAPSHOT_INTERRUPT__SHIFT 0x00000004
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_TRIGA_INTERRUPT_MASK 0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_TRIGA_INTERRUPT__SHIFT 0x00000007
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_TRIGB_INTERRUPT_MASK 0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_TRIGB_INTERRUPT__SHIFT 0x00000008
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_VSYNC_NOM_INTERRUPT_MASK 0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__CRTC4_VSYNC_NOM_INTERRUPT__SHIFT 0x00000009
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DC_HPD4_INTERRUPT_MASK 0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DC_HPD4_INTERRUPT__SHIFT 0x00000011
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DC_HPD4_RX_INTERRUPT_MASK 0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DC_HPD4_RX_INTERRUPT__SHIFT 0x00000012
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DIGD_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK 0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DIGD_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT 0x0000000f
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DIGD_DP_VID_STREAM_DISABLE_INTERRUPT_MASK 0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DIGD_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT 0x00000010
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DISP_INTERRUPT_STATUS_CONTINUE4_MASK 0x80000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__DISP_INTERRUPT_STATUS_CONTINUE4__SHIFT 0x0000001f
+#define DISP_INTERRUPT_STATUS_CONTINUE3__LB_D4_VBLANK_INTERRUPT_MASK 0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__LB_D4_VBLANK_INTERRUPT__SHIFT 0x00000003
+#define DISP_INTERRUPT_STATUS_CONTINUE3__LB_D4_VLINE_INTERRUPT_MASK 0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__LB_D4_VLINE_INTERRUPT__SHIFT 0x00000002
+#define DISP_INTERRUPT_STATUS_CONTINUE3__SCL_DISP4_MODE_CHANGE_INTERRUPT_MASK 0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE3__SCL_DISP4_MODE_CHANGE_INTERRUPT__SHIFT 0x00000000
+#define DISP_INTERRUPT_STATUS_CONTINUE4__AUX5_LS_DONE_INTERRUPT_MASK 0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__AUX5_LS_DONE_INTERRUPT__SHIFT 0x00000014
+#define DISP_INTERRUPT_STATUS_CONTINUE4__AUX5_SW_DONE_INTERRUPT_MASK 0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__AUX5_SW_DONE_INTERRUPT__SHIFT 0x00000013
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_FORCE_COUNT_NOW_INTERRUPT_MASK 0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_FORCE_COUNT_NOW_INTERRUPT__SHIFT 0x00000006
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_FORCE_VSYNC_NEXT_LINE_INTERRUPT_MASK 0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_FORCE_VSYNC_NEXT_LINE_INTERRUPT__SHIFT 0x00000005
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK 0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT 0x0000000a
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_SNAPSHOT_INTERRUPT_MASK 0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_SNAPSHOT_INTERRUPT__SHIFT 0x00000004
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_TRIGA_INTERRUPT_MASK 0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_TRIGA_INTERRUPT__SHIFT 0x00000007
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_TRIGB_INTERRUPT_MASK 0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_TRIGB_INTERRUPT__SHIFT 0x00000008
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_VSYNC_NOM_INTERRUPT_MASK 0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__CRTC5_VSYNC_NOM_INTERRUPT__SHIFT 0x00000009
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DC_HPD5_INTERRUPT_MASK 0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DC_HPD5_INTERRUPT__SHIFT 0x00000011
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DC_HPD5_RX_INTERRUPT_MASK 0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DC_HPD5_RX_INTERRUPT__SHIFT 0x00000012
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DIGE_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK 0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DIGE_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT 0x0000000f
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DIGE_DP_VID_STREAM_DISABLE_INTERRUPT_MASK 0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DIGE_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT 0x00000010
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DISP_INTERRUPT_STATUS_CONTINUE5_MASK 0x80000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__DISP_INTERRUPT_STATUS_CONTINUE5__SHIFT 0x0000001f
+#define DISP_INTERRUPT_STATUS_CONTINUE4__LB_D5_VBLANK_INTERRUPT_MASK 0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__LB_D5_VBLANK_INTERRUPT__SHIFT 0x00000003
+#define DISP_INTERRUPT_STATUS_CONTINUE4__LB_D5_VLINE_INTERRUPT_MASK 0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__LB_D5_VLINE_INTERRUPT__SHIFT 0x00000002
+#define DISP_INTERRUPT_STATUS_CONTINUE4__SCL_DISP5_MODE_CHANGE_INTERRUPT_MASK 0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE4__SCL_DISP5_MODE_CHANGE_INTERRUPT__SHIFT 0x00000000
+#define DISP_INTERRUPT_STATUS_CONTINUE5__AUX6_LS_DONE_INTERRUPT_MASK 0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__AUX6_LS_DONE_INTERRUPT__SHIFT 0x00000014
+#define DISP_INTERRUPT_STATUS_CONTINUE5__AUX6_SW_DONE_INTERRUPT_MASK 0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__AUX6_SW_DONE_INTERRUPT__SHIFT 0x00000013
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_FORCE_COUNT_NOW_INTERRUPT_MASK 0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_FORCE_COUNT_NOW_INTERRUPT__SHIFT 0x00000006
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_FORCE_VSYNC_NEXT_LINE_INTERRUPT_MASK 0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_FORCE_VSYNC_NEXT_LINE_INTERRUPT__SHIFT 0x00000005
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK 0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT 0x0000000a
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_SNAPSHOT_INTERRUPT_MASK 0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_SNAPSHOT_INTERRUPT__SHIFT 0x00000004
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_TRIGA_INTERRUPT_MASK 0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_TRIGA_INTERRUPT__SHIFT 0x00000007
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_TRIGB_INTERRUPT_MASK 0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_TRIGB_INTERRUPT__SHIFT 0x00000008
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_VSYNC_NOM_INTERRUPT_MASK 0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__CRTC6_VSYNC_NOM_INTERRUPT__SHIFT 0x00000009
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DC_HPD6_INTERRUPT_MASK 0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DC_HPD6_INTERRUPT__SHIFT 0x00000011
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DC_HPD6_RX_INTERRUPT_MASK 0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DC_HPD6_RX_INTERRUPT__SHIFT 0x00000012
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DIGF_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK 0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DIGF_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT 0x0000000f
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DIGF_DP_VID_STREAM_DISABLE_INTERRUPT_MASK 0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__DIGF_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT 0x00000010
+#define DISP_INTERRUPT_STATUS_CONTINUE5__LB_D6_VBLANK_INTERRUPT_MASK 0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__LB_D6_VBLANK_INTERRUPT__SHIFT 0x00000003
+#define DISP_INTERRUPT_STATUS_CONTINUE5__LB_D6_VLINE_INTERRUPT_MASK 0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__LB_D6_VLINE_INTERRUPT__SHIFT 0x00000002
+#define DISP_INTERRUPT_STATUS_CONTINUE5__SCL_DISP6_MODE_CHANGE_INTERRUPT_MASK 0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE5__SCL_DISP6_MODE_CHANGE_INTERRUPT__SHIFT 0x00000000
+#define DISP_INTERRUPT_STATUS_CONTINUE__AUX2_LS_DONE_INTERRUPT_MASK 0x00100000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__AUX2_LS_DONE_INTERRUPT__SHIFT 0x00000014
+#define DISP_INTERRUPT_STATUS_CONTINUE__AUX2_SW_DONE_INTERRUPT_MASK 0x00080000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__AUX2_SW_DONE_INTERRUPT__SHIFT 0x00000013
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_FORCE_COUNT_NOW_INTERRUPT_MASK 0x00000040L
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_FORCE_COUNT_NOW_INTERRUPT__SHIFT 0x00000006
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_FORCE_VSYNC_NEXT_LINE_INTERRUPT_MASK 0x00000020L
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_FORCE_VSYNC_NEXT_LINE_INTERRUPT__SHIFT 0x00000005
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK 0x00000400L
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT 0x0000000a
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_SNAPSHOT_INTERRUPT_MASK 0x00000010L
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_SNAPSHOT_INTERRUPT__SHIFT 0x00000004
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_TRIGA_INTERRUPT_MASK 0x00000080L
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_TRIGA_INTERRUPT__SHIFT 0x00000007
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_TRIGB_INTERRUPT_MASK 0x00000100L
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_TRIGB_INTERRUPT__SHIFT 0x00000008
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_VSYNC_NOM_INTERRUPT_MASK 0x00000200L
+#define DISP_INTERRUPT_STATUS_CONTINUE__CRTC2_VSYNC_NOM_INTERRUPT__SHIFT 0x00000009
+#define DISP_INTERRUPT_STATUS_CONTINUE__DC_HPD2_INTERRUPT_MASK 0x00020000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__DC_HPD2_INTERRUPT__SHIFT 0x00000011
+#define DISP_INTERRUPT_STATUS_CONTINUE__DC_HPD2_RX_INTERRUPT_MASK 0x00040000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__DC_HPD2_RX_INTERRUPT__SHIFT 0x00000012
+#define DISP_INTERRUPT_STATUS_CONTINUE__DIGB_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK 0x00008000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__DIGB_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT 0x0000000f
+#define DISP_INTERRUPT_STATUS_CONTINUE__DIGB_DP_VID_STREAM_DISABLE_INTERRUPT_MASK 0x00010000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__DIGB_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT 0x00000010
+#define DISP_INTERRUPT_STATUS_CONTINUE__DISP_INTERRUPT_STATUS_CONTINUE2_MASK 0x80000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__DISP_INTERRUPT_STATUS_CONTINUE2__SHIFT 0x0000001f
+#define DISP_INTERRUPT_STATUS_CONTINUE__DISP_TIMER_INTERRUPT_MASK 0x01000000L
+#define DISP_INTERRUPT_STATUS_CONTINUE__DISP_TIMER_INTERRUPT__SHIFT 0x00000018
+#define DISP_INTERRUPT_STATUS_CONTINUE__LB_D2_VBLANK_INTERRUPT_MASK 0x00000008L
+#define DISP_INTERRUPT_STATUS_CONTINUE__LB_D2_VBLANK_INTERRUPT__SHIFT 0x00000003
+#define DISP_INTERRUPT_STATUS_CONTINUE__LB_D2_VLINE_INTERRUPT_MASK 0x00000004L
+#define DISP_INTERRUPT_STATUS_CONTINUE__LB_D2_VLINE_INTERRUPT__SHIFT 0x00000002
+#define DISP_INTERRUPT_STATUS_CONTINUE__SCL_DISP2_MODE_CHANGE_INTERRUPT_MASK 0x00000001L
+#define DISP_INTERRUPT_STATUS_CONTINUE__SCL_DISP2_MODE_CHANGE_INTERRUPT__SHIFT 0x00000000
+#define DISP_INTERRUPT_STATUS__CRTC1_FORCE_COUNT_NOW_INTERRUPT_MASK 0x00000040L
+#define DISP_INTERRUPT_STATUS__CRTC1_FORCE_COUNT_NOW_INTERRUPT__SHIFT 0x00000006
+#define DISP_INTERRUPT_STATUS__CRTC1_FORCE_VSYNC_NEXT_LINE_INTERRUPT_MASK 0x00000020L
+#define DISP_INTERRUPT_STATUS__CRTC1_FORCE_VSYNC_NEXT_LINE_INTERRUPT__SHIFT 0x00000005
+#define DISP_INTERRUPT_STATUS__CRTC1_SET_V_TOTAL_MIN_EVENT_OCCURED_INT_MASK 0x00000400L
+#define DISP_INTERRUPT_STATUS__CRTC1_SET_V_TOTAL_MIN_EVENT_OCCURED_INT__SHIFT 0x0000000a
+#define DISP_INTERRUPT_STATUS__CRTC1_SNAPSHOT_INTERRUPT_MASK 0x00000010L
+#define DISP_INTERRUPT_STATUS__CRTC1_SNAPSHOT_INTERRUPT__SHIFT 0x00000004
+#define DISP_INTERRUPT_STATUS__CRTC1_TRIGA_INTERRUPT_MASK 0x00000080L
+#define DISP_INTERRUPT_STATUS__CRTC1_TRIGA_INTERRUPT__SHIFT 0x00000007
+#define DISP_INTERRUPT_STATUS__CRTC1_TRIGB_INTERRUPT_MASK 0x00000100L
+#define DISP_INTERRUPT_STATUS__CRTC1_TRIGB_INTERRUPT__SHIFT 0x00000008
+#define DISP_INTERRUPT_STATUS__CRTC1_VSYNC_NOM_INTERRUPT_MASK 0x00000200L
+#define DISP_INTERRUPT_STATUS__CRTC1_VSYNC_NOM_INTERRUPT__SHIFT 0x00000009
+#define DISP_INTERRUPT_STATUS__DACA_AUTODETECT_INTERRUPT_MASK 0x00400000L
+#define DISP_INTERRUPT_STATUS__DACA_AUTODETECT_INTERRUPT__SHIFT 0x00000016
+#define DISP_INTERRUPT_STATUS__DACB_AUTODETECT_INTERRUPT_MASK 0x00800000L
+#define DISP_INTERRUPT_STATUS__DACB_AUTODETECT_INTERRUPT__SHIFT 0x00000017
+#define DISP_INTERRUPT_STATUS__DC_HPD1_INTERRUPT_MASK 0x00020000L
+#define DISP_INTERRUPT_STATUS__DC_HPD1_INTERRUPT__SHIFT 0x00000011
+#define DISP_INTERRUPT_STATUS__DC_HPD1_RX_INTERRUPT_MASK 0x00040000L
+#define DISP_INTERRUPT_STATUS__DC_HPD1_RX_INTERRUPT__SHIFT 0x00000012
+#define DISP_INTERRUPT_STATUS__DC_I2C_HW_DONE_INTERRUPT_MASK 0x02000000L
+#define DISP_INTERRUPT_STATUS__DC_I2C_HW_DONE_INTERRUPT__SHIFT 0x00000019
+#define DISP_INTERRUPT_STATUS__DC_I2C_SW_DONE_INTERRUPT_MASK 0x01000000L
+#define DISP_INTERRUPT_STATUS__DC_I2C_SW_DONE_INTERRUPT__SHIFT 0x00000018
+#define DISP_INTERRUPT_STATUS__DIGA_DISPCLK_SWITCH_ALLOWED_INTERRUPT_MASK 0x00200000L
+#define DISP_INTERRUPT_STATUS__DIGA_DISPCLK_SWITCH_ALLOWED_INTERRUPT__SHIFT 0x00000015
+#define DISP_INTERRUPT_STATUS__DIGA_DP_FAST_TRAINING_COMPLETE_INTERRUPT_MASK 0x00008000L
+#define DISP_INTERRUPT_STATUS__DIGA_DP_FAST_TRAINING_COMPLETE_INTERRUPT__SHIFT 0x0000000f
+#define DISP_INTERRUPT_STATUS__DIGA_DP_VID_STREAM_DISABLE_INTERRUPT_MASK 0x00010000L
+#define DISP_INTERRUPT_STATUS__DIGA_DP_VID_STREAM_DISABLE_INTERRUPT__SHIFT 0x00000010
+#define DISP_INTERRUPT_STATUS__DISP_INTERRUPT_STATUS_CONTINUE_MASK 0x80000000L
+#define DISP_INTERRUPT_STATUS__DISP_INTERRUPT_STATUS_CONTINUE__SHIFT 0x0000001f
+#define DISP_INTERRUPT_STATUS__DMCU_SCP_INT_MASK 0x08000000L
+#define DISP_INTERRUPT_STATUS__DMCU_SCP_INT__SHIFT 0x0000001b
+#define DISP_INTERRUPT_STATUS__DMCU_UC_INTERNAL_INT_MASK 0x04000000L
+#define DISP_INTERRUPT_STATUS__DMCU_UC_INTERNAL_INT__SHIFT 0x0000001a
+#define DISP_INTERRUPT_STATUS__LB_D1_VBLANK_INTERRUPT_MASK 0x00000008L
+#define DISP_INTERRUPT_STATUS__LB_D1_VBLANK_INTERRUPT__SHIFT 0x00000003
+#define DISP_INTERRUPT_STATUS__LB_D1_VLINE_INTERRUPT_MASK 0x00000004L
+#define DISP_INTERRUPT_STATUS__LB_D1_VLINE_INTERRUPT__SHIFT 0x00000002
+#define DISP_INTERRUPT_STATUS__SCL_DISP1_MODE_CHANGE_INTERRUPT_MASK 0x00000001L
+#define DISP_INTERRUPT_STATUS__SCL_DISP1_MODE_CHANGE_INTERRUPT__SHIFT 0x00000000
+#define DISPOUT_STEREOSYNC_SEL__GENERICA_STEREOSYNC_SEL_MASK 0x00000007L
+#define DISPOUT_STEREOSYNC_SEL__GENERICA_STEREOSYNC_SEL__SHIFT 0x00000000
+#define DISPOUT_STEREOSYNC_SEL__GENERICB_STEREOSYNC_SEL_MASK 0x00070000L
+#define DISPOUT_STEREOSYNC_SEL__GENERICB_STEREOSYNC_SEL__SHIFT 0x00000010
+#define DISPPLL_BG_CNTL__DISPPLL_BG_ADJ_MASK 0x000000f0L
+#define DISPPLL_BG_CNTL__DISPPLL_BG_ADJ__SHIFT 0x00000004
+#define DISPPLL_BG_CNTL__DISPPLL_BG_PDN_MASK 0x00000001L
+#define DISPPLL_BG_CNTL__DISPPLL_BG_PDN__SHIFT 0x00000000
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_COUNT_MASK 0x01ffffffL
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_COUNT__SHIFT 0x00000000
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_ENABLE_MASK 0x02000000L
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_ENABLE__SHIFT 0x00000019
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_MASK 0x40000000L
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_MSK_MASK 0x08000000L
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_MSK__SHIFT 0x0000001b
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_RUNNING_MASK 0x04000000L
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_RUNNING__SHIFT 0x0000001a
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT__SHIFT 0x0000001e
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_STAT_AK_MASK 0x20000000L
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_STAT_AK__SHIFT 0x0000001d
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_STAT_MASK 0x10000000L
+#define DISP_TIMER_CONTROL__DISP_TIMER_INT_STAT__SHIFT 0x0000001c
+#define DMCU_CTRL__DISABLE_IRQ_TO_UC_MASK 0x00000004L
+#define DMCU_CTRL__DISABLE_IRQ_TO_UC__SHIFT 0x00000002
+#define DMCU_CTRL__DISABLE_XIRQ_TO_UC_MASK 0x00000008L
+#define DMCU_CTRL__DISABLE_XIRQ_TO_UC__SHIFT 0x00000003
+#define DMCU_CTRL__DMCU_ENABLE_MASK 0x00000010L
+#define DMCU_CTRL__DMCU_ENABLE__SHIFT 0x00000004
+#define DMCU_CTRL__IGNORE_PWRMGT_MASK 0x00000002L
+#define DMCU_CTRL__IGNORE_PWRMGT__SHIFT 0x00000001
+#define DMCU_CTRL__RESET_UC_MASK 0x00000001L
+#define DMCU_CTRL__RESET_UC__SHIFT 0x00000000
+#define DMCU_CTRL__UC_REG_RD_TIMEOUT_MASK 0xffc00000L
+#define DMCU_CTRL__UC_REG_RD_TIMEOUT__SHIFT 0x00000016
+#define DMCU_ERAM_RD_CTRL__ERAM_RD_ADDR_MASK 0x0000ffffL
+#define DMCU_ERAM_RD_CTRL__ERAM_RD_ADDR__SHIFT 0x00000000
+#define DMCU_ERAM_RD_CTRL__ERAM_RD_BE_MASK 0x000f0000L
+#define DMCU_ERAM_RD_CTRL__ERAM_RD_BE__SHIFT 0x00000010
+#define DMCU_ERAM_RD_CTRL__ERAM_RD_BYTE_MODE_MASK 0x00100000L
+#define DMCU_ERAM_RD_CTRL__ERAM_RD_BYTE_MODE__SHIFT 0x00000014
+#define DMCU_ERAM_RD_DATA__ERAM_RD_DATA_MASK 0xffffffffL
+#define DMCU_ERAM_RD_DATA__ERAM_RD_DATA__SHIFT 0x00000000
+#define DMCU_ERAM_WR_CTRL__ERAM_WR_ADDR_MASK 0x0000ffffL
+#define DMCU_ERAM_WR_CTRL__ERAM_WR_ADDR__SHIFT 0x00000000
+#define DMCU_ERAM_WR_CTRL__ERAM_WR_BE_MASK 0x000f0000L
+#define DMCU_ERAM_WR_CTRL__ERAM_WR_BE__SHIFT 0x00000010
+#define DMCU_ERAM_WR_CTRL__ERAM_WR_BYTE_MODE_MASK 0x00100000L
+#define DMCU_ERAM_WR_CTRL__ERAM_WR_BYTE_MODE__SHIFT 0x00000014
+#define DMCU_ERAM_WR_DATA__ERAM_WR_DATA_MASK 0xffffffffL
+#define DMCU_ERAM_WR_DATA__ERAM_WR_DATA__SHIFT 0x00000000
+#define DMCU_EVENT_TRIGGER__GEN_SW_INT_TO_UC_MASK 0x00000001L
+#define DMCU_EVENT_TRIGGER__GEN_SW_INT_TO_UC__SHIFT 0x00000000
+#define DMCU_EVENT_TRIGGER__GEN_UC_INTERNAL_INT_TO_HOST_MASK 0x00800000L
+#define DMCU_EVENT_TRIGGER__GEN_UC_INTERNAL_INT_TO_HOST__SHIFT 0x00000017
+#define DMCU_EVENT_TRIGGER__UC_INTERNAL_INT_CODE_MASK 0x007f0000L
+#define DMCU_EVENT_TRIGGER__UC_INTERNAL_INT_CODE__SHIFT 0x00000010
+#define DMCU_FW_CHECKSUM_SMPL_BYTE_POS__DMCU_FW_CHECKSUM_HI_SMPL_BYTE_POS_MASK 0x0000000cL
+#define DMCU_FW_CHECKSUM_SMPL_BYTE_POS__DMCU_FW_CHECKSUM_HI_SMPL_BYTE_POS__SHIFT 0x00000002
+#define DMCU_FW_CHECKSUM_SMPL_BYTE_POS__DMCU_FW_CHECKSUM_LO_SMPL_BYTE_POS_MASK 0x00000003L
+#define DMCU_FW_CHECKSUM_SMPL_BYTE_POS__DMCU_FW_CHECKSUM_LO_SMPL_BYTE_POS__SHIFT 0x00000000
+#define DMCU_FW_CS_HI__FW_CHECKSUM_HI_MASK 0xffffffffL
+#define DMCU_FW_CS_HI__FW_CHECKSUM_HI__SHIFT 0x00000000
+#define DMCU_FW_CS_LO__FW_CHECKSUM_LO_MASK 0xffffffffL
+#define DMCU_FW_CS_LO__FW_CHECKSUM_LO__SHIFT 0x00000000
+#define DMCU_FW_END_ADDR__FW_END_ADDR_LSB_MASK 0x000000ffL
+#define DMCU_FW_END_ADDR__FW_END_ADDR_LSB__SHIFT 0x00000000
+#define DMCU_FW_END_ADDR__FW_END_ADDR_MSB_MASK 0x0000ff00L
+#define DMCU_FW_END_ADDR__FW_END_ADDR_MSB__SHIFT 0x00000008
+#define DMCU_FW_ISR_START_ADDR__FW_ISR_START_ADDR_LSB_MASK 0x000000ffL
+#define DMCU_FW_ISR_START_ADDR__FW_ISR_START_ADDR_LSB__SHIFT 0x00000000
+#define DMCU_FW_ISR_START_ADDR__FW_ISR_START_ADDR_MSB_MASK 0x0000ff00L
+#define DMCU_FW_ISR_START_ADDR__FW_ISR_START_ADDR_MSB__SHIFT 0x00000008
+#define DMCU_FW_START_ADDR__FW_START_ADDR_LSB_MASK 0x000000ffL
+#define DMCU_FW_START_ADDR__FW_START_ADDR_LSB__SHIFT 0x00000000
+#define DMCU_FW_START_ADDR__FW_START_ADDR_MSB_MASK 0x0000ff00L
+#define DMCU_FW_START_ADDR__FW_START_ADDR_MSB__SHIFT 0x00000008
+#define DMCU_INT_CNT__DMCU_ABM1_BL_UPDATE_INT_CNT_MASK 0x00ff0000L
+#define DMCU_INT_CNT__DMCU_ABM1_BL_UPDATE_INT_CNT__SHIFT 0x00000010
+#define DMCU_INT_CNT__DMCU_ABM1_HG_READY_INT_CNT_MASK 0x000000ffL
+#define DMCU_INT_CNT__DMCU_ABM1_HG_READY_INT_CNT__SHIFT 0x00000000
+#define DMCU_INT_CNT__DMCU_ABM1_LS_READY_INT_CNT_MASK 0x0000ff00L
+#define DMCU_INT_CNT__DMCU_ABM1_LS_READY_INT_CNT__SHIFT 0x00000008
+#define DMCU_INTERRUPT_STATUS__ABM1_BL_UPDATE_INT_CLEAR_MASK 0x00000004L
+#define DMCU_INTERRUPT_STATUS__ABM1_BL_UPDATE_INT_CLEAR__SHIFT 0x00000002
+#define DMCU_INTERRUPT_STATUS__ABM1_BL_UPDATE_INT_OCCURRED_MASK 0x00000004L
+#define DMCU_INTERRUPT_STATUS__ABM1_BL_UPDATE_INT_OCCURRED__SHIFT 0x00000002
+#define DMCU_INTERRUPT_STATUS__ABM1_HG_READY_INT_CLEAR_MASK 0x00000001L
+#define DMCU_INTERRUPT_STATUS__ABM1_HG_READY_INT_CLEAR__SHIFT 0x00000000
+#define DMCU_INTERRUPT_STATUS__ABM1_HG_READY_INT_OCCURRED_MASK 0x00000001L
+#define DMCU_INTERRUPT_STATUS__ABM1_HG_READY_INT_OCCURRED__SHIFT 0x00000000
+#define DMCU_INTERRUPT_STATUS__ABM1_LS_READY_INT_CLEAR_MASK 0x00000002L
+#define DMCU_INTERRUPT_STATUS__ABM1_LS_READY_INT_CLEAR__SHIFT 0x00000001
+#define DMCU_INTERRUPT_STATUS__ABM1_LS_READY_INT_OCCURRED_MASK 0x00000002L
+#define DMCU_INTERRUPT_STATUS__ABM1_LS_READY_INT_OCCURRED__SHIFT 0x00000001
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE0_POWER_DOWN_INT_CLEAR_MASK 0x00040000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE0_POWER_DOWN_INT_CLEAR__SHIFT 0x00000012
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE0_POWER_DOWN_INT_OCCURRED_MASK 0x00040000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE0_POWER_DOWN_INT_OCCURRED__SHIFT 0x00000012
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE0_POWER_UP_INT_CLEAR_MASK 0x00001000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE0_POWER_UP_INT_CLEAR__SHIFT 0x0000000c
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE0_POWER_UP_INT_OCCURRED_MASK 0x00001000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE0_POWER_UP_INT_OCCURRED__SHIFT 0x0000000c
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE1_POWER_DOWN_INT_CLEAR_MASK 0x00080000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE1_POWER_DOWN_INT_CLEAR__SHIFT 0x00000013
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE1_POWER_DOWN_INT_OCCURRED_MASK 0x00080000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE1_POWER_DOWN_INT_OCCURRED__SHIFT 0x00000013
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE1_POWER_UP_INT_CLEAR_MASK 0x00002000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE1_POWER_UP_INT_CLEAR__SHIFT 0x0000000d
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE1_POWER_UP_INT_OCCURRED_MASK 0x00002000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE1_POWER_UP_INT_OCCURRED__SHIFT 0x0000000d
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE2_POWER_DOWN_INT_CLEAR_MASK 0x00100000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE2_POWER_DOWN_INT_CLEAR__SHIFT 0x00000014
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE2_POWER_DOWN_INT_OCCURRED_MASK 0x00100000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE2_POWER_DOWN_INT_OCCURRED__SHIFT 0x00000014
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE2_POWER_UP_INT_CLEAR_MASK 0x00004000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE2_POWER_UP_INT_CLEAR__SHIFT 0x0000000e
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE2_POWER_UP_INT_OCCURRED_MASK 0x00004000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE2_POWER_UP_INT_OCCURRED__SHIFT 0x0000000e
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE3_POWER_DOWN_INT_CLEAR_MASK 0x00200000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE3_POWER_DOWN_INT_CLEAR__SHIFT 0x00000015
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE3_POWER_DOWN_INT_OCCURRED_MASK 0x00200000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE3_POWER_DOWN_INT_OCCURRED__SHIFT 0x00000015
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE3_POWER_UP_INT_CLEAR_MASK 0x00008000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE3_POWER_UP_INT_CLEAR__SHIFT 0x0000000f
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE3_POWER_UP_INT_OCCURRED_MASK 0x00008000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE3_POWER_UP_INT_OCCURRED__SHIFT 0x0000000f
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE4_POWER_DOWN_INT_CLEAR_MASK 0x00400000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE4_POWER_DOWN_INT_CLEAR__SHIFT 0x00000016
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE4_POWER_DOWN_INT_OCCURRED_MASK 0x00400000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE4_POWER_DOWN_INT_OCCURRED__SHIFT 0x00000016
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE4_POWER_UP_INT_CLEAR_MASK 0x00010000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE4_POWER_UP_INT_CLEAR__SHIFT 0x00000010
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE4_POWER_UP_INT_OCCURRED_MASK 0x00010000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE4_POWER_UP_INT_OCCURRED__SHIFT 0x00000010
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE5_POWER_DOWN_INT_CLEAR_MASK 0x00800000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE5_POWER_DOWN_INT_CLEAR__SHIFT 0x00000017
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE5_POWER_DOWN_INT_OCCURRED_MASK 0x00800000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE5_POWER_DOWN_INT_OCCURRED__SHIFT 0x00000017
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE5_POWER_UP_INT_CLEAR_MASK 0x00020000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE5_POWER_UP_INT_CLEAR__SHIFT 0x00000011
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE5_POWER_UP_INT_OCCURRED_MASK 0x00020000L
+#define DMCU_INTERRUPT_STATUS__DCPG_IHC_DCFE5_POWER_UP_INT_OCCURRED__SHIFT 0x00000011
+#define DMCU_INTERRUPT_STATUS__EXTERNAL_SW_INT_CLEAR_MASK 0x00000100L
+#define DMCU_INTERRUPT_STATUS__EXTERNAL_SW_INT_CLEAR__SHIFT 0x00000008
+#define DMCU_INTERRUPT_STATUS__EXTERNAL_SW_INT_OCCURRED_MASK 0x00000100L
+#define DMCU_INTERRUPT_STATUS__EXTERNAL_SW_INT_OCCURRED__SHIFT 0x00000008
+#define DMCU_INTERRUPT_STATUS__MCP_INT_OCCURRED_MASK 0x00000008L
+#define DMCU_INTERRUPT_STATUS__MCP_INT_OCCURRED__SHIFT 0x00000003
+#define DMCU_INTERRUPT_STATUS__SCP_INT_OCCURRED_MASK 0x00000200L
+#define DMCU_INTERRUPT_STATUS__SCP_INT_OCCURRED__SHIFT 0x00000009
+#define DMCU_INTERRUPT_STATUS__UC_INTERNAL_INT_CLEAR_MASK 0x00000400L
+#define DMCU_INTERRUPT_STATUS__UC_INTERNAL_INT_CLEAR__SHIFT 0x0000000a
+#define DMCU_INTERRUPT_STATUS__UC_INTERNAL_INT_OCCURRED_MASK 0x00000400L
+#define DMCU_INTERRUPT_STATUS__UC_INTERNAL_INT_OCCURRED__SHIFT 0x0000000a
+#define DMCU_INTERRUPT_STATUS__UC_REG_RD_TIMEOUT_INT_CLEAR_MASK 0x00000800L
+#define DMCU_INTERRUPT_STATUS__UC_REG_RD_TIMEOUT_INT_CLEAR__SHIFT 0x0000000b
+#define DMCU_INTERRUPT_STATUS__UC_REG_RD_TIMEOUT_INT_OCCURRED_MASK 0x00000800L
+#define DMCU_INTERRUPT_STATUS__UC_REG_RD_TIMEOUT_INT_OCCURRED__SHIFT 0x0000000b
+#define DMCU_INTERRUPT_STATUS__VBLANK1_INT_CLEAR_MASK 0x01000000L
+#define DMCU_INTERRUPT_STATUS__VBLANK1_INT_CLEAR__SHIFT 0x00000018
+#define DMCU_INTERRUPT_STATUS__VBLANK1_INT_OCCURRED_MASK 0x01000000L
+#define DMCU_INTERRUPT_STATUS__VBLANK1_INT_OCCURRED__SHIFT 0x00000018
+#define DMCU_INTERRUPT_STATUS__VBLANK2_INT_CLEAR_MASK 0x02000000L
+#define DMCU_INTERRUPT_STATUS__VBLANK2_INT_CLEAR__SHIFT 0x00000019
+#define DMCU_INTERRUPT_STATUS__VBLANK2_INT_OCCURRED_MASK 0x02000000L
+#define DMCU_INTERRUPT_STATUS__VBLANK2_INT_OCCURRED__SHIFT 0x00000019
+#define DMCU_INTERRUPT_STATUS__VBLANK3_INT_CLEAR_MASK 0x04000000L
+#define DMCU_INTERRUPT_STATUS__VBLANK3_INT_CLEAR__SHIFT 0x0000001a
+#define DMCU_INTERRUPT_STATUS__VBLANK3_INT_OCCURRED_MASK 0x04000000L
+#define DMCU_INTERRUPT_STATUS__VBLANK3_INT_OCCURRED__SHIFT 0x0000001a
+#define DMCU_INTERRUPT_STATUS__VBLANK4_INT_CLEAR_MASK 0x08000000L
+#define DMCU_INTERRUPT_STATUS__VBLANK4_INT_CLEAR__SHIFT 0x0000001b
+#define DMCU_INTERRUPT_STATUS__VBLANK4_INT_OCCURRED_MASK 0x08000000L
+#define DMCU_INTERRUPT_STATUS__VBLANK4_INT_OCCURRED__SHIFT 0x0000001b
+#define DMCU_INTERRUPT_STATUS__VBLANK5_INT_CLEAR_MASK 0x10000000L
+#define DMCU_INTERRUPT_STATUS__VBLANK5_INT_CLEAR__SHIFT 0x0000001c
+#define DMCU_INTERRUPT_STATUS__VBLANK5_INT_OCCURRED_MASK 0x10000000L
+#define DMCU_INTERRUPT_STATUS__VBLANK5_INT_OCCURRED__SHIFT 0x0000001c
+#define DMCU_INTERRUPT_STATUS__VBLANK6_INT_CLEAR_MASK 0x20000000L
+#define DMCU_INTERRUPT_STATUS__VBLANK6_INT_CLEAR__SHIFT 0x0000001d
+#define DMCU_INTERRUPT_STATUS__VBLANK6_INT_OCCURRED_MASK 0x20000000L
+#define DMCU_INTERRUPT_STATUS__VBLANK6_INT_OCCURRED__SHIFT 0x0000001d
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__ABM1_BL_UPDATE_INT_MASK_MASK 0x00000004L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__ABM1_BL_UPDATE_INT_MASK__SHIFT 0x00000002
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__ABM1_HG_READY_INT_MASK_MASK 0x00000001L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__ABM1_HG_READY_INT_MASK__SHIFT 0x00000000
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__ABM1_LS_READY_INT_MASK_MASK 0x00000002L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__ABM1_LS_READY_INT_MASK__SHIFT 0x00000001
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE0_POWER_DOWN_INT_MASK_MASK 0x00040000L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE0_POWER_DOWN_INT_MASK__SHIFT 0x00000012
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE0_POWER_UP_INT_MASK_MASK 0x00001000L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE0_POWER_UP_INT_MASK__SHIFT 0x0000000c
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE1_POWER_DOWN_INT_MASK_MASK 0x00080000L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE1_POWER_DOWN_INT_MASK__SHIFT 0x00000013
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE1_POWER_UP_INT_MASK_MASK 0x00002000L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE1_POWER_UP_INT_MASK__SHIFT 0x0000000d
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE2_POWER_DOWN_INT_MASK_MASK 0x00100000L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE2_POWER_DOWN_INT_MASK__SHIFT 0x00000014
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE2_POWER_UP_INT_MASK_MASK 0x00004000L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE2_POWER_UP_INT_MASK__SHIFT 0x0000000e
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE3_POWER_DOWN_INT_MASK_MASK 0x00200000L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE3_POWER_DOWN_INT_MASK__SHIFT 0x00000015
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE3_POWER_UP_INT_MASK_MASK 0x00008000L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE3_POWER_UP_INT_MASK__SHIFT 0x0000000f
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE4_POWER_DOWN_INT_MASK_MASK 0x00400000L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE4_POWER_DOWN_INT_MASK__SHIFT 0x00000016
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE4_POWER_UP_INT_MASK_MASK 0x00010000L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE4_POWER_UP_INT_MASK__SHIFT 0x00000010
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE5_POWER_DOWN_INT_MASK_MASK 0x00800000L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE5_POWER_DOWN_INT_MASK__SHIFT 0x00000017
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE5_POWER_UP_INT_MASK_MASK 0x00020000L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__DCPG_IHC_DCFE5_POWER_UP_INT_MASK__SHIFT 0x00000011
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__SCP_INT_MASK_MASK 0x00000200L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__SCP_INT_MASK__SHIFT 0x00000009
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__UC_INTERNAL_INT_MASK_MASK 0x00000400L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__UC_INTERNAL_INT_MASK__SHIFT 0x0000000a
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__UC_REG_RD_TIMEOUT_INT_MASK_MASK 0x00000800L
+#define DMCU_INTERRUPT_TO_HOST_EN_MASK__UC_REG_RD_TIMEOUT_INT_MASK__SHIFT 0x0000000b
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__ABM1_BL_UPDATE_INT_TO_UC_EN_MASK 0x00000004L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__ABM1_BL_UPDATE_INT_TO_UC_EN__SHIFT 0x00000002
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__ABM1_HG_READY_INT_TO_UC_EN_MASK 0x00000001L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__ABM1_HG_READY_INT_TO_UC_EN__SHIFT 0x00000000
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__ABM1_LS_READY_INT_TO_UC_EN_MASK 0x00000002L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__ABM1_LS_READY_INT_TO_UC_EN__SHIFT 0x00000001
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE0_POWER_DOWN_INT_TO_UC_EN_MASK 0x00040000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE0_POWER_DOWN_INT_TO_UC_EN__SHIFT 0x00000012
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE0_POWER_UP_INT_TO_UC_EN_MASK 0x00001000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE0_POWER_UP_INT_TO_UC_EN__SHIFT 0x0000000c
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE1_POWER_DOWN_INT_TO_UC_EN_MASK 0x00080000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE1_POWER_DOWN_INT_TO_UC_EN__SHIFT 0x00000013
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE1_POWER_UP_INT_TO_UC_EN_MASK 0x00002000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE1_POWER_UP_INT_TO_UC_EN__SHIFT 0x0000000d
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE2_POWER_DOWN_INT_TO_UC_EN_MASK 0x00100000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE2_POWER_DOWN_INT_TO_UC_EN__SHIFT 0x00000014
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE2_POWER_UP_INT_TO_UC_EN_MASK 0x00004000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE2_POWER_UP_INT_TO_UC_EN__SHIFT 0x0000000e
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE3_POWER_DOWN_INT_TO_UC_EN_MASK 0x00200000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE3_POWER_DOWN_INT_TO_UC_EN__SHIFT 0x00000015
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE3_POWER_UP_INT_TO_UC_EN_MASK 0x00008000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE3_POWER_UP_INT_TO_UC_EN__SHIFT 0x0000000f
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE4_POWER_DOWN_INT_TO_UC_EN_MASK 0x00400000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE4_POWER_DOWN_INT_TO_UC_EN__SHIFT 0x00000016
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE4_POWER_UP_INT_TO_UC_EN_MASK 0x00010000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE4_POWER_UP_INT_TO_UC_EN__SHIFT 0x00000010
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE5_POWER_DOWN_INT_TO_UC_EN_MASK 0x00800000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE5_POWER_DOWN_INT_TO_UC_EN__SHIFT 0x00000017
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE5_POWER_UP_INT_TO_UC_EN_MASK 0x00020000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__DCPG_IHC_DCFE5_POWER_UP_INT_TO_UC_EN__SHIFT 0x00000011
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__EXTERNAL_SW_INT_TO_UC_EN_MASK 0x00000100L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__EXTERNAL_SW_INT_TO_UC_EN__SHIFT 0x00000008
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__MCP_INT_TO_UC_EN_MASK 0x00000008L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__MCP_INT_TO_UC_EN__SHIFT 0x00000003
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__VBLANK1_INT_TO_UC_EN_MASK 0x01000000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__VBLANK1_INT_TO_UC_EN__SHIFT 0x00000018
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__VBLANK2_INT_TO_UC_EN_MASK 0x02000000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__VBLANK2_INT_TO_UC_EN__SHIFT 0x00000019
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__VBLANK3_INT_TO_UC_EN_MASK 0x04000000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__VBLANK3_INT_TO_UC_EN__SHIFT 0x0000001a
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__VBLANK4_INT_TO_UC_EN_MASK 0x08000000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__VBLANK4_INT_TO_UC_EN__SHIFT 0x0000001b
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__VBLANK5_INT_TO_UC_EN_MASK 0x10000000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__VBLANK5_INT_TO_UC_EN__SHIFT 0x0000001c
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__VBLANK6_INT_TO_UC_EN_MASK 0x20000000L
+#define DMCU_INTERRUPT_TO_UC_EN_MASK__VBLANK6_INT_TO_UC_EN__SHIFT 0x0000001d
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__ABM1_BL_UPDATE_INT_XIRQ_IRQ_SEL_MASK 0x00000004L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__ABM1_BL_UPDATE_INT_XIRQ_IRQ_SEL__SHIFT 0x00000002
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__ABM1_HG_READY_INT_XIRQ_IRQ_SEL_MASK 0x00000001L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__ABM1_HG_READY_INT_XIRQ_IRQ_SEL__SHIFT 0x00000000
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__ABM1_LS_READY_INT_XIRQ_IRQ_SEL_MASK 0x00000002L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__ABM1_LS_READY_INT_XIRQ_IRQ_SEL__SHIFT 0x00000001
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE0_POWER_DOWN_INT_XIRQ_IRQ_SEL_MASK 0x00040000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE0_POWER_DOWN_INT_XIRQ_IRQ_SEL__SHIFT 0x00000012
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE0_POWER_UP_INT_XIRQ_IRQ_SEL_MASK 0x00001000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE0_POWER_UP_INT_XIRQ_IRQ_SEL__SHIFT 0x0000000c
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE1_POWER_DOWN_INT_XIRQ_IRQ_SEL_MASK 0x00080000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE1_POWER_DOWN_INT_XIRQ_IRQ_SEL__SHIFT 0x00000013
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE1_POWER_UP_INT_XIRQ_IRQ_SEL_MASK 0x00002000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE1_POWER_UP_INT_XIRQ_IRQ_SEL__SHIFT 0x0000000d
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE2_POWER_DOWN_INT_XIRQ_IRQ_SEL_MASK 0x00100000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE2_POWER_DOWN_INT_XIRQ_IRQ_SEL__SHIFT 0x00000014
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE2_POWER_UP_INT_XIRQ_IRQ_SEL_MASK 0x00004000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE2_POWER_UP_INT_XIRQ_IRQ_SEL__SHIFT 0x0000000e
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE3_POWER_DOWN_INT_XIRQ_IRQ_SEL_MASK 0x00200000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE3_POWER_DOWN_INT_XIRQ_IRQ_SEL__SHIFT 0x00000015
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE3_POWER_UP_INT_XIRQ_IRQ_SEL_MASK 0x00008000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE3_POWER_UP_INT_XIRQ_IRQ_SEL__SHIFT 0x0000000f
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE4_POWER_DOWN_INT_XIRQ_IRQ_SEL_MASK 0x00400000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE4_POWER_DOWN_INT_XIRQ_IRQ_SEL__SHIFT 0x00000016
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE4_POWER_UP_INT_XIRQ_IRQ_SEL_MASK 0x00010000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE4_POWER_UP_INT_XIRQ_IRQ_SEL__SHIFT 0x00000010
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE5_POWER_DOWN_INT_XIRQ_IRQ_SEL_MASK 0x00800000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE5_POWER_DOWN_INT_XIRQ_IRQ_SEL__SHIFT 0x00000017
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE5_POWER_UP_INT_XIRQ_IRQ_SEL_MASK 0x00020000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__DCPG_IHC_DCFE5_POWER_UP_INT_XIRQ_IRQ_SEL__SHIFT 0x00000011
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__EXTERNAL_SW_INT_XIRQ_IRQ_SEL_MASK 0x00000100L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__EXTERNAL_SW_INT_XIRQ_IRQ_SEL__SHIFT 0x00000008
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__MCP_INT_XIRQ_IRQ_SEL_MASK 0x00000008L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__MCP_INT_XIRQ_IRQ_SEL__SHIFT 0x00000003
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__VBLANK1_INT_XIRQ_IRQ_SEL_MASK 0x01000000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__VBLANK1_INT_XIRQ_IRQ_SEL__SHIFT 0x00000018
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__VBLANK2_INT_XIRQ_IRQ_SEL_MASK 0x02000000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__VBLANK2_INT_XIRQ_IRQ_SEL__SHIFT 0x00000019
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__VBLANK3_INT_XIRQ_IRQ_SEL_MASK 0x04000000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__VBLANK3_INT_XIRQ_IRQ_SEL__SHIFT 0x0000001a
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__VBLANK4_INT_XIRQ_IRQ_SEL_MASK 0x08000000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__VBLANK4_INT_XIRQ_IRQ_SEL__SHIFT 0x0000001b
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__VBLANK5_INT_XIRQ_IRQ_SEL_MASK 0x10000000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__VBLANK5_INT_XIRQ_IRQ_SEL__SHIFT 0x0000001c
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__VBLANK6_INT_XIRQ_IRQ_SEL_MASK 0x20000000L
+#define DMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL__VBLANK6_INT_XIRQ_IRQ_SEL__SHIFT 0x0000001d
+#define DMCU_IRAM_RD_CTRL__IRAM_RD_ADDR_MASK 0x000003ffL
+#define DMCU_IRAM_RD_CTRL__IRAM_RD_ADDR__SHIFT 0x00000000
+#define DMCU_IRAM_RD_DATA__IRAM_RD_DATA_MASK 0x000000ffL
+#define DMCU_IRAM_RD_DATA__IRAM_RD_DATA__SHIFT 0x00000000
+#define DMCU_IRAM_WR_CTRL__IRAM_WR_ADDR_MASK 0x000003ffL
+#define DMCU_IRAM_WR_CTRL__IRAM_WR_ADDR__SHIFT 0x00000000
+#define DMCU_IRAM_WR_DATA__IRAM_WR_DATA_MASK 0x000000ffL
+#define DMCU_IRAM_WR_DATA__IRAM_WR_DATA__SHIFT 0x00000000
+#define DMCU_PC_START_ADDR__PC_START_ADDR_LSB_MASK 0x000000ffL
+#define DMCU_PC_START_ADDR__PC_START_ADDR_LSB__SHIFT 0x00000000
+#define DMCU_PC_START_ADDR__PC_START_ADDR_MSB_MASK 0x0000ff00L
+#define DMCU_PC_START_ADDR__PC_START_ADDR_MSB__SHIFT 0x00000008
+#define DMCU_RAM_ACCESS_CTRL__ERAM_HOST_ACCESS_EN_MASK 0x00000010L
+#define DMCU_RAM_ACCESS_CTRL__ERAM_HOST_ACCESS_EN__SHIFT 0x00000004
+#define DMCU_RAM_ACCESS_CTRL__ERAM_RD_ADDR_AUTO_INC_MASK 0x00000002L
+#define DMCU_RAM_ACCESS_CTRL__ERAM_RD_ADDR_AUTO_INC__SHIFT 0x00000001
+#define DMCU_RAM_ACCESS_CTRL__ERAM_WR_ADDR_AUTO_INC_MASK 0x00000001L
+#define DMCU_RAM_ACCESS_CTRL__ERAM_WR_ADDR_AUTO_INC__SHIFT 0x00000000
+#define DMCU_RAM_ACCESS_CTRL__IRAM_HOST_ACCESS_EN_MASK 0x00000020L
+#define DMCU_RAM_ACCESS_CTRL__IRAM_HOST_ACCESS_EN__SHIFT 0x00000005
+#define DMCU_RAM_ACCESS_CTRL__IRAM_RD_ADDR_AUTO_INC_MASK 0x00000008L
+#define DMCU_RAM_ACCESS_CTRL__IRAM_RD_ADDR_AUTO_INC__SHIFT 0x00000003
+#define DMCU_RAM_ACCESS_CTRL__IRAM_WR_ADDR_AUTO_INC_MASK 0x00000004L
+#define DMCU_RAM_ACCESS_CTRL__IRAM_WR_ADDR_AUTO_INC__SHIFT 0x00000002
+#define DMCU_RAM_ACCESS_CTRL__UC_RST_RELEASE_DELAY_CNT_MASK 0x0000ff00L
+#define DMCU_RAM_ACCESS_CTRL__UC_RST_RELEASE_DELAY_CNT__SHIFT 0x00000008
+#define DMCU_STATUS__UC_IN_RESET_MASK 0x00000001L
+#define DMCU_STATUS__UC_IN_RESET__SHIFT 0x00000000
+#define DMCU_STATUS__UC_IN_STOP_MODE_MASK 0x00000004L
+#define DMCU_STATUS__UC_IN_STOP_MODE__SHIFT 0x00000002
+#define DMCU_STATUS__UC_IN_WAIT_MODE_MASK 0x00000002L
+#define DMCU_STATUS__UC_IN_WAIT_MODE__SHIFT 0x00000001
+#define DMCU_TEST_DEBUG_DATA__DMCU_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define DMCU_TEST_DEBUG_DATA__DMCU_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define DMCU_TEST_DEBUG_INDEX__DMCU_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define DMCU_TEST_DEBUG_INDEX__DMCU_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define DMCU_TEST_DEBUG_INDEX__DMCU_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define DMCU_TEST_DEBUG_INDEX__DMCU_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define DMCU_UC_CLK_GATING_CNTL__UC_ERAM_RD_DELAY_MASK 0x00000700L
+#define DMCU_UC_CLK_GATING_CNTL__UC_ERAM_RD_DELAY__SHIFT 0x00000008
+#define DMCU_UC_CLK_GATING_CNTL__UC_IRAM_RD_DELAY_MASK 0x00000007L
+#define DMCU_UC_CLK_GATING_CNTL__UC_IRAM_RD_DELAY__SHIFT 0x00000000
+#define DMCU_UC_CLK_GATING_CNTL__UC_RBBM_RD_CLK_GATING_EN_MASK 0x00010000L
+#define DMCU_UC_CLK_GATING_CNTL__UC_RBBM_RD_CLK_GATING_EN__SHIFT 0x00000010
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_ILLEGAL_OPCODE_TRAP_MASK 0x00000008L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_ILLEGAL_OPCODE_TRAP__SHIFT 0x00000003
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_IRQ_N_PIN_MASK 0x00000001L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_IRQ_N_PIN__SHIFT 0x00000000
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_PULSE_ACCUMULATOR_INPUT_EDGE_MASK 0x00004000L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_PULSE_ACCUMULATOR_INPUT_EDGE__SHIFT 0x0000000e
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_PULSE_ACCUMULATOR_OVERFLOW_MASK 0x00008000L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_PULSE_ACCUMULATOR_OVERFLOW__SHIFT 0x0000000f
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_REAL_TIME_INTERRUPT_MASK 0x00000200L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_REAL_TIME_INTERRUPT__SHIFT 0x00000009
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_SOFTWARE_INTERRUPT_MASK 0x00000004L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_SOFTWARE_INTERRUPT__SHIFT 0x00000002
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_INPUT_CAPTURE_1_MASK 0x00002000L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_INPUT_CAPTURE_1__SHIFT 0x0000000d
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_INPUT_CAPTURE_2_MASK 0x00001000L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_INPUT_CAPTURE_2__SHIFT 0x0000000c
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_INPUT_CAPTURE_3_MASK 0x00000800L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_INPUT_CAPTURE_3__SHIFT 0x0000000b
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_INPUT_CAPTURE_4_OUTPUT_COMPARE_5_MASK 0x00000400L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_INPUT_CAPTURE_4_OUTPUT_COMPARE_5__SHIFT 0x0000000a
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_OUTPUT_COMPARE_1_MASK 0x00000080L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_OUTPUT_COMPARE_1__SHIFT 0x00000007
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_OUTPUT_COMPARE_2_MASK 0x00000040L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_OUTPUT_COMPARE_2__SHIFT 0x00000006
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_OUTPUT_COMPARE_3_MASK 0x00000020L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_OUTPUT_COMPARE_3__SHIFT 0x00000005
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_OUTPUT_COMPARE_4_MASK 0x00000010L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_OUTPUT_COMPARE_4__SHIFT 0x00000004
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_OVERFLOW_MASK 0x00000100L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_TIMER_OVERFLOW__SHIFT 0x00000008
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_XIRQ_N_PIN_MASK 0x00000002L
+#define DMCU_UC_INTERNAL_INT_STATUS__UC_INT_XIRQ_N_PIN__SHIFT 0x00000001
+#define DMIF_ADDR_CALC__ADDR_CONFIG_PIPE_INTERLEAVE_SIZE_MASK 0x00000070L
+#define DMIF_ADDR_CALC__ADDR_CONFIG_PIPE_INTERLEAVE_SIZE__SHIFT 0x00000004
+#define DMIF_ADDR_CALC__ADDR_CONFIG_ROW_SIZE_MASK 0x30000000L
+#define DMIF_ADDR_CALC__ADDR_CONFIG_ROW_SIZE__SHIFT 0x0000001c
+#define DMIF_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK 0x00000700L
+#define DMIF_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT 0x00000008
+#define DMIF_ADDR_CONFIG__NUM_LOWER_PIPES_MASK 0x40000000L
+#define DMIF_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT 0x0000001e
+#define DMIF_ADDR_CONFIG__NUM_PIPES_MASK 0x00000007L
+#define DMIF_ADDR_CONFIG__NUM_PIPES__SHIFT 0x00000000
+#define DMIF_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK 0x00003000L
+#define DMIF_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT 0x0000000c
+#define DMIF_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK 0x00000070L
+#define DMIF_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT 0x00000004
+#define DMIF_ADDR_CONFIG__ROW_SIZE_MASK 0x30000000L
+#define DMIF_ADDR_CONFIG__ROW_SIZE__SHIFT 0x0000001c
+#define DMIF_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK 0x00070000L
+#define DMIF_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT 0x00000010
+#define DMIF_ARBITRATION_CONTROL__DMIF_ARBITRATION_REFERENCE_CLOCK_PERIOD_MASK 0x0000ffffL
+#define DMIF_ARBITRATION_CONTROL__DMIF_ARBITRATION_REFERENCE_CLOCK_PERIOD__SHIFT 0x00000000
+#define DMIF_ARBITRATION_CONTROL__PIPE_SWITCH_EFFICIENCY_WEIGHT_MASK 0xffff0000L
+#define DMIF_ARBITRATION_CONTROL__PIPE_SWITCH_EFFICIENCY_WEIGHT__SHIFT 0x00000010
+#define DMIF_CONTROL__DMIF_BUFF_SIZE_MASK 0x00000003L
+#define DMIF_CONTROL__DMIF_BUFF_SIZE__SHIFT 0x00000000
+#define DMIF_CONTROL__DMIF_CHUNK_BUFF_MARGIN_MASK 0x60000000L
+#define DMIF_CONTROL__DMIF_CHUNK_BUFF_MARGIN__SHIFT 0x0000001d
+#define DMIF_CONTROL__DMIF_DELAY_ARBITRATION_MASK 0x1f000000L
+#define DMIF_CONTROL__DMIF_DELAY_ARBITRATION__SHIFT 0x00000018
+#define DMIF_CONTROL__DMIF_DISABLE_EARLY_RECEIVED_LEVEL_COUNT_MASK 0x00000010L
+#define DMIF_CONTROL__DMIF_DISABLE_EARLY_RECEIVED_LEVEL_COUNT__SHIFT 0x00000004
+#define DMIF_CONTROL__DMIF_FORCE_TOTAL_REQ_BURST_SIZE_MASK 0x0000f000L
+#define DMIF_CONTROL__DMIF_FORCE_TOTAL_REQ_BURST_SIZE__SHIFT 0x0000000c
+#define DMIF_CONTROL__DMIF_GROUP_REQUESTS_IN_CHUNK_MASK 0x00000004L
+#define DMIF_CONTROL__DMIF_GROUP_REQUESTS_IN_CHUNK__SHIFT 0x00000002
+#define DMIF_CONTROL__DMIF_MAX_TOTAL_OUTSTANDING_CHUNK_REQUESTS_MASK 0x003f0000L
+#define DMIF_CONTROL__DMIF_MAX_TOTAL_OUTSTANDING_CHUNK_REQUESTS__SHIFT 0x00000010
+#define DMIF_CONTROL__DMIF_REQ_BURST_SIZE_MASK 0x00000700L
+#define DMIF_CONTROL__DMIF_REQ_BURST_SIZE__SHIFT 0x00000008
+#define DMIF_DEBUG02_CORE0__DB_DATA_MASK 0x0000ffffL
+#define DMIF_DEBUG02_CORE0__DB_DATA__SHIFT 0x00000000
+#define DMIF_DEBUG02_CORE0__MC_RDRET_COUNT_EN_MASK 0x00010000L
+#define DMIF_DEBUG02_CORE0__MC_RDRET_COUNT_EN__SHIFT 0x00000010
+#define DMIF_DEBUG02_CORE0__MC_RDRET_COUNTER_MASK 0x0ffe0000L
+#define DMIF_DEBUG02_CORE0__MC_RDRET_COUNTER__SHIFT 0x00000011
+#define DMIF_DEBUG02_CORE1__DB_DATA_MASK 0x0000ffffL
+#define DMIF_DEBUG02_CORE1__DB_DATA__SHIFT 0x00000000
+#define DMIF_DEBUG02_CORE1__MC_RDRET_COUNT_EN_MASK 0x00010000L
+#define DMIF_DEBUG02_CORE1__MC_RDRET_COUNT_EN__SHIFT 0x00000010
+#define DMIF_DEBUG02_CORE1__MC_RDRET_COUNTER_MASK 0x0ffe0000L
+#define DMIF_DEBUG02_CORE1__MC_RDRET_COUNTER__SHIFT 0x00000011
+#define DMIF_HW_DEBUG__DMIF_HW_DEBUG_MASK 0xffffffffL
+#define DMIF_HW_DEBUG__DMIF_HW_DEBUG__SHIFT 0x00000000
+#define DMIF_STATUS2__DMIF_CHUNK_TRACKER_SCLK_STATUS_MASK 0x00000100L
+#define DMIF_STATUS2__DMIF_CHUNK_TRACKER_SCLK_STATUS__SHIFT 0x00000008
+#define DMIF_STATUS2__DMIF_FBC_TRACKER_SCLK_STATUS_MASK 0x00000200L
+#define DMIF_STATUS2__DMIF_FBC_TRACKER_SCLK_STATUS__SHIFT 0x00000009
+#define DMIF_STATUS2__DMIF_PIPE0_DISPCLK_STATUS_MASK 0x00000001L
+#define DMIF_STATUS2__DMIF_PIPE0_DISPCLK_STATUS__SHIFT 0x00000000
+#define DMIF_STATUS2__DMIF_PIPE1_DISPCLK_STATUS_MASK 0x00000002L
+#define DMIF_STATUS2__DMIF_PIPE1_DISPCLK_STATUS__SHIFT 0x00000001
+#define DMIF_STATUS2__DMIF_PIPE2_DISPCLK_STATUS_MASK 0x00000004L
+#define DMIF_STATUS2__DMIF_PIPE2_DISPCLK_STATUS__SHIFT 0x00000002
+#define DMIF_STATUS2__DMIF_PIPE3_DISPCLK_STATUS_MASK 0x00000008L
+#define DMIF_STATUS2__DMIF_PIPE3_DISPCLK_STATUS__SHIFT 0x00000003
+#define DMIF_STATUS2__DMIF_PIPE4_DISPCLK_STATUS_MASK 0x00000010L
+#define DMIF_STATUS2__DMIF_PIPE4_DISPCLK_STATUS__SHIFT 0x00000004
+#define DMIF_STATUS2__DMIF_PIPE5_DISPCLK_STATUS_MASK 0x00000020L
+#define DMIF_STATUS2__DMIF_PIPE5_DISPCLK_STATUS__SHIFT 0x00000005
+#define DMIF_STATUS__DMIF_CLEAR_MC_SEND_ON_IDLE_MASK 0x00003f00L
+#define DMIF_STATUS__DMIF_CLEAR_MC_SEND_ON_IDLE__SHIFT 0x00000008
+#define DMIF_STATUS__DMIF_MC_LATENCY_COUNTER_ENABLE_MASK 0x00010000L
+#define DMIF_STATUS__DMIF_MC_LATENCY_COUNTER_ENABLE__SHIFT 0x00000010
+#define DMIF_STATUS__DMIF_MC_LATENCY_COUNTER_SOURCE_SELECT_MASK 0x00700000L
+#define DMIF_STATUS__DMIF_MC_LATENCY_COUNTER_SOURCE_SELECT__SHIFT 0x00000014
+#define DMIF_STATUS__DMIF_MC_LATENCY_COUNTER_URGENT_ONLY_MASK 0x00020000L
+#define DMIF_STATUS__DMIF_MC_LATENCY_COUNTER_URGENT_ONLY__SHIFT 0x00000011
+#define DMIF_STATUS__DMIF_MC_SEND_ON_IDLE_MASK 0x0000003fL
+#define DMIF_STATUS__DMIF_MC_SEND_ON_IDLE__SHIFT 0x00000000
+#define DMIF_STATUS__DMIF_PERFORMANCE_COUNTER_SOURCE_SELECT_MASK 0x07000000L
+#define DMIF_STATUS__DMIF_PERFORMANCE_COUNTER_SOURCE_SELECT__SHIFT 0x00000018
+#define DMIF_STATUS__DMIF_UNDERFLOW_MASK 0x10000000L
+#define DMIF_STATUS__DMIF_UNDERFLOW__SHIFT 0x0000001c
+#define DMIF_TEST_DEBUG_DATA__DMIF_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define DMIF_TEST_DEBUG_DATA__DMIF_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define DMIF_TEST_DEBUG_INDEX__DMIF_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define DMIF_TEST_DEBUG_INDEX__DMIF_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define DMIF_TEST_DEBUG_INDEX__DMIF_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define DMIF_TEST_DEBUG_INDEX__DMIF_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define DOUT_DCE_VCE_CONTROL__DC_VCE_AUDIO_STREAM_SELECT_MASK 0x00000070L
+#define DOUT_DCE_VCE_CONTROL__DC_VCE_AUDIO_STREAM_SELECT__SHIFT 0x00000004
+#define DOUT_DCE_VCE_CONTROL__DC_VCE_VIDEO_PIPE_SELECT_MASK 0x00000007L
+#define DOUT_DCE_VCE_CONTROL__DC_VCE_VIDEO_PIPE_SELECT__SHIFT 0x00000000
+#define DOUT_POWER_MANAGEMENT_CNTL__PM_ALL_BUSY_OFF_MASK 0x00000100L
+#define DOUT_POWER_MANAGEMENT_CNTL__PM_ALL_BUSY_OFF__SHIFT 0x00000008
+#define DOUT_POWER_MANAGEMENT_CNTL__PM_ASSERT_RESET_MASK 0x00000001L
+#define DOUT_POWER_MANAGEMENT_CNTL__PM_ASSERT_RESET__SHIFT 0x00000000
+#define DOUT_SCRATCH0__DOUT_SCRATCH0_MASK 0xffffffffL
+#define DOUT_SCRATCH0__DOUT_SCRATCH0__SHIFT 0x00000000
+#define DOUT_SCRATCH1__DOUT_SCRATCH1_MASK 0xffffffffL
+#define DOUT_SCRATCH1__DOUT_SCRATCH1__SHIFT 0x00000000
+#define DOUT_SCRATCH2__DOUT_SCRATCH2_MASK 0xffffffffL
+#define DOUT_SCRATCH2__DOUT_SCRATCH2__SHIFT 0x00000000
+#define DOUT_SCRATCH3__DOUT_SCRATCH3_MASK 0xffffffffL
+#define DOUT_SCRATCH3__DOUT_SCRATCH3__SHIFT 0x00000000
+#define DOUT_SCRATCH4__DOUT_SCRATCH4_MASK 0xffffffffL
+#define DOUT_SCRATCH4__DOUT_SCRATCH4__SHIFT 0x00000000
+#define DOUT_SCRATCH5__DOUT_SCRATCH5_MASK 0xffffffffL
+#define DOUT_SCRATCH5__DOUT_SCRATCH5__SHIFT 0x00000000
+#define DOUT_SCRATCH6__DOUT_SCRATCH6_MASK 0xffffffffL
+#define DOUT_SCRATCH6__DOUT_SCRATCH6__SHIFT 0x00000000
+#define DOUT_SCRATCH7__DOUT_SCRATCH7_MASK 0xffffffffL
+#define DOUT_SCRATCH7__DOUT_SCRATCH7__SHIFT 0x00000000
+#define DOUT_TEST_DEBUG_DATA__DOUT_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define DOUT_TEST_DEBUG_DATA__DOUT_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define DOUT_TEST_DEBUG_INDEX__DOUT_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define DOUT_TEST_DEBUG_INDEX__DOUT_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define DOUT_TEST_DEBUG_INDEX__DOUT_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define DOUT_TEST_DEBUG_INDEX__DOUT_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define DP_AUX1_DEBUG_A__DP_AUX1_DEBUG_A_MASK 0xffffffffL
+#define DP_AUX1_DEBUG_A__DP_AUX1_DEBUG_A__SHIFT 0x00000000
+#define DP_AUX1_DEBUG_B__DP_AUX1_DEBUG_B_MASK 0xffffffffL
+#define DP_AUX1_DEBUG_B__DP_AUX1_DEBUG_B__SHIFT 0x00000000
+#define DP_AUX1_DEBUG_C__DP_AUX1_DEBUG_C_MASK 0xffffffffL
+#define DP_AUX1_DEBUG_C__DP_AUX1_DEBUG_C__SHIFT 0x00000000
+#define DP_AUX1_DEBUG_D__DP_AUX1_DEBUG_D_MASK 0xffffffffL
+#define DP_AUX1_DEBUG_D__DP_AUX1_DEBUG_D__SHIFT 0x00000000
+#define DP_AUX1_DEBUG_E__DP_AUX1_DEBUG_E_MASK 0xffffffffL
+#define DP_AUX1_DEBUG_E__DP_AUX1_DEBUG_E__SHIFT 0x00000000
+#define DP_AUX1_DEBUG_F__DP_AUX1_DEBUG_F_MASK 0xffffffffL
+#define DP_AUX1_DEBUG_F__DP_AUX1_DEBUG_F__SHIFT 0x00000000
+#define DP_AUX1_DEBUG_G__DP_AUX1_DEBUG_G_MASK 0xffffffffL
+#define DP_AUX1_DEBUG_G__DP_AUX1_DEBUG_G__SHIFT 0x00000000
+#define DP_AUX1_DEBUG_H__DP_AUX1_DEBUG_H_MASK 0xffffffffL
+#define DP_AUX1_DEBUG_H__DP_AUX1_DEBUG_H__SHIFT 0x00000000
+#define DP_AUX1_DEBUG_I__DP_AUX1_DEBUG_I_MASK 0xffffffffL
+#define DP_AUX1_DEBUG_I__DP_AUX1_DEBUG_I__SHIFT 0x00000000
+#define DP_AUX2_DEBUG_A__DP_AUX2_DEBUG_A_MASK 0xffffffffL
+#define DP_AUX2_DEBUG_A__DP_AUX2_DEBUG_A__SHIFT 0x00000000
+#define DP_AUX2_DEBUG_B__DP_AUX2_DEBUG_B_MASK 0xffffffffL
+#define DP_AUX2_DEBUG_B__DP_AUX2_DEBUG_B__SHIFT 0x00000000
+#define DP_AUX2_DEBUG_C__DP_AUX2_DEBUG_C_MASK 0xffffffffL
+#define DP_AUX2_DEBUG_C__DP_AUX2_DEBUG_C__SHIFT 0x00000000
+#define DP_AUX2_DEBUG_D__DP_AUX2_DEBUG_D_MASK 0xffffffffL
+#define DP_AUX2_DEBUG_D__DP_AUX2_DEBUG_D__SHIFT 0x00000000
+#define DP_AUX2_DEBUG_E__DP_AUX2_DEBUG_E_MASK 0xffffffffL
+#define DP_AUX2_DEBUG_E__DP_AUX2_DEBUG_E__SHIFT 0x00000000
+#define DP_AUX2_DEBUG_F__DP_AUX2_DEBUG_F_MASK 0xffffffffL
+#define DP_AUX2_DEBUG_F__DP_AUX2_DEBUG_F__SHIFT 0x00000000
+#define DP_AUX2_DEBUG_G__DP_AUX2_DEBUG_G_MASK 0xffffffffL
+#define DP_AUX2_DEBUG_G__DP_AUX2_DEBUG_G__SHIFT 0x00000000
+#define DP_AUX2_DEBUG_H__DP_AUX2_DEBUG_H_MASK 0xffffffffL
+#define DP_AUX2_DEBUG_H__DP_AUX2_DEBUG_H__SHIFT 0x00000000
+#define DP_AUX2_DEBUG_I__DP_AUX2_DEBUG_I_MASK 0xffffffffL
+#define DP_AUX2_DEBUG_I__DP_AUX2_DEBUG_I__SHIFT 0x00000000
+#define DP_AUX3_DEBUG_A__DP_AUX3_DEBUG_A_MASK 0xffffffffL
+#define DP_AUX3_DEBUG_A__DP_AUX3_DEBUG_A__SHIFT 0x00000000
+#define DP_AUX3_DEBUG_B__DP_AUX3_DEBUG_B_MASK 0xffffffffL
+#define DP_AUX3_DEBUG_B__DP_AUX3_DEBUG_B__SHIFT 0x00000000
+#define DP_AUX3_DEBUG_C__DP_AUX3_DEBUG_C_MASK 0xffffffffL
+#define DP_AUX3_DEBUG_C__DP_AUX3_DEBUG_C__SHIFT 0x00000000
+#define DP_AUX3_DEBUG_D__DP_AUX3_DEBUG_D_MASK 0xffffffffL
+#define DP_AUX3_DEBUG_D__DP_AUX3_DEBUG_D__SHIFT 0x00000000
+#define DP_AUX3_DEBUG_E__DP_AUX3_DEBUG_E_MASK 0xffffffffL
+#define DP_AUX3_DEBUG_E__DP_AUX3_DEBUG_E__SHIFT 0x00000000
+#define DP_AUX3_DEBUG_F__DP_AUX3_DEBUG_F_MASK 0xffffffffL
+#define DP_AUX3_DEBUG_F__DP_AUX3_DEBUG_F__SHIFT 0x00000000
+#define DP_AUX3_DEBUG_G__DP_AUX3_DEBUG_G_MASK 0xffffffffL
+#define DP_AUX3_DEBUG_G__DP_AUX3_DEBUG_G__SHIFT 0x00000000
+#define DP_AUX3_DEBUG_H__DP_AUX3_DEBUG_H_MASK 0xffffffffL
+#define DP_AUX3_DEBUG_H__DP_AUX3_DEBUG_H__SHIFT 0x00000000
+#define DP_AUX3_DEBUG_I__DP_AUX3_DEBUG_I_MASK 0xffffffffL
+#define DP_AUX3_DEBUG_I__DP_AUX3_DEBUG_I__SHIFT 0x00000000
+#define DP_AUX4_DEBUG_A__DP_AUX4_DEBUG_A_MASK 0xffffffffL
+#define DP_AUX4_DEBUG_A__DP_AUX4_DEBUG_A__SHIFT 0x00000000
+#define DP_AUX4_DEBUG_B__DP_AUX4_DEBUG_B_MASK 0xffffffffL
+#define DP_AUX4_DEBUG_B__DP_AUX4_DEBUG_B__SHIFT 0x00000000
+#define DP_AUX4_DEBUG_C__DP_AUX4_DEBUG_C_MASK 0xffffffffL
+#define DP_AUX4_DEBUG_C__DP_AUX4_DEBUG_C__SHIFT 0x00000000
+#define DP_AUX4_DEBUG_D__DP_AUX4_DEBUG_D_MASK 0xffffffffL
+#define DP_AUX4_DEBUG_D__DP_AUX4_DEBUG_D__SHIFT 0x00000000
+#define DP_AUX4_DEBUG_E__DP_AUX4_DEBUG_E_MASK 0xffffffffL
+#define DP_AUX4_DEBUG_E__DP_AUX4_DEBUG_E__SHIFT 0x00000000
+#define DP_AUX4_DEBUG_F__DP_AUX4_DEBUG_F_MASK 0xffffffffL
+#define DP_AUX4_DEBUG_F__DP_AUX4_DEBUG_F__SHIFT 0x00000000
+#define DP_AUX4_DEBUG_G__DP_AUX4_DEBUG_G_MASK 0xffffffffL
+#define DP_AUX4_DEBUG_G__DP_AUX4_DEBUG_G__SHIFT 0x00000000
+#define DP_AUX4_DEBUG_H__DP_AUX4_DEBUG_H_MASK 0xffffffffL
+#define DP_AUX4_DEBUG_H__DP_AUX4_DEBUG_H__SHIFT 0x00000000
+#define DP_AUX4_DEBUG_I__DP_AUX4_DEBUG_I_MASK 0xffffffffL
+#define DP_AUX4_DEBUG_I__DP_AUX4_DEBUG_I__SHIFT 0x00000000
+#define DP_AUX5_DEBUG_A__DP_AUX5_DEBUG_A_MASK 0xffffffffL
+#define DP_AUX5_DEBUG_A__DP_AUX5_DEBUG_A__SHIFT 0x00000000
+#define DP_AUX5_DEBUG_B__DP_AUX5_DEBUG_B_MASK 0xffffffffL
+#define DP_AUX5_DEBUG_B__DP_AUX5_DEBUG_B__SHIFT 0x00000000
+#define DP_AUX5_DEBUG_C__DP_AUX5_DEBUG_C_MASK 0xffffffffL
+#define DP_AUX5_DEBUG_C__DP_AUX5_DEBUG_C__SHIFT 0x00000000
+#define DP_AUX5_DEBUG_D__DP_AUX5_DEBUG_D_MASK 0xffffffffL
+#define DP_AUX5_DEBUG_D__DP_AUX5_DEBUG_D__SHIFT 0x00000000
+#define DP_AUX5_DEBUG_E__DP_AUX5_DEBUG_E_MASK 0xffffffffL
+#define DP_AUX5_DEBUG_E__DP_AUX5_DEBUG_E__SHIFT 0x00000000
+#define DP_AUX5_DEBUG_F__DP_AUX5_DEBUG_F_MASK 0xffffffffL
+#define DP_AUX5_DEBUG_F__DP_AUX5_DEBUG_F__SHIFT 0x00000000
+#define DP_AUX5_DEBUG_G__DP_AUX5_DEBUG_G_MASK 0xffffffffL
+#define DP_AUX5_DEBUG_G__DP_AUX5_DEBUG_G__SHIFT 0x00000000
+#define DP_AUX5_DEBUG_H__DP_AUX5_DEBUG_H_MASK 0xffffffffL
+#define DP_AUX5_DEBUG_H__DP_AUX5_DEBUG_H__SHIFT 0x00000000
+#define DP_AUX5_DEBUG_I__DP_AUX5_DEBUG_I_MASK 0xffffffffL
+#define DP_AUX5_DEBUG_I__DP_AUX5_DEBUG_I__SHIFT 0x00000000
+#define DP_AUX6_DEBUG_A__DP_AUX6_DEBUG_A_MASK 0xffffffffL
+#define DP_AUX6_DEBUG_A__DP_AUX6_DEBUG_A__SHIFT 0x00000000
+#define DP_AUX6_DEBUG_B__DP_AUX6_DEBUG_B_MASK 0xffffffffL
+#define DP_AUX6_DEBUG_B__DP_AUX6_DEBUG_B__SHIFT 0x00000000
+#define DP_AUX6_DEBUG_C__DP_AUX6_DEBUG_C_MASK 0xffffffffL
+#define DP_AUX6_DEBUG_C__DP_AUX6_DEBUG_C__SHIFT 0x00000000
+#define DP_AUX6_DEBUG_D__DP_AUX6_DEBUG_D_MASK 0xffffffffL
+#define DP_AUX6_DEBUG_D__DP_AUX6_DEBUG_D__SHIFT 0x00000000
+#define DP_AUX6_DEBUG_E__DP_AUX6_DEBUG_E_MASK 0xffffffffL
+#define DP_AUX6_DEBUG_E__DP_AUX6_DEBUG_E__SHIFT 0x00000000
+#define DP_AUX6_DEBUG_F__DP_AUX6_DEBUG_F_MASK 0xffffffffL
+#define DP_AUX6_DEBUG_F__DP_AUX6_DEBUG_F__SHIFT 0x00000000
+#define DP_AUX6_DEBUG_G__DP_AUX6_DEBUG_G_MASK 0xffffffffL
+#define DP_AUX6_DEBUG_G__DP_AUX6_DEBUG_G__SHIFT 0x00000000
+#define DP_AUX6_DEBUG_H__DP_AUX6_DEBUG_H_MASK 0xffffffffL
+#define DP_AUX6_DEBUG_H__DP_AUX6_DEBUG_H__SHIFT 0x00000000
+#define DP_AUX6_DEBUG_I__DP_AUX6_DEBUG_I_MASK 0xffffffffL
+#define DP_AUX6_DEBUG_I__DP_AUX6_DEBUG_I__SHIFT 0x00000000
+#define DP_CONFIG__DP_UDI_LANES_MASK 0x00000003L
+#define DP_CONFIG__DP_UDI_LANES__SHIFT 0x00000000
+#define DP_DPHY_8B10B_CNTL__DPHY_8B10B_CUR_DISP_MASK 0x01000000L
+#define DP_DPHY_8B10B_CNTL__DPHY_8B10B_CUR_DISP__SHIFT 0x00000018
+#define DP_DPHY_8B10B_CNTL__DPHY_8B10B_EXT_DISP_MASK 0x00010000L
+#define DP_DPHY_8B10B_CNTL__DPHY_8B10B_EXT_DISP__SHIFT 0x00000010
+#define DP_DPHY_8B10B_CNTL__DPHY_8B10B_RESET_MASK 0x00000100L
+#define DP_DPHY_8B10B_CNTL__DPHY_8B10B_RESET__SHIFT 0x00000008
+#define DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE0_MASK 0x00000001L
+#define DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE0__SHIFT 0x00000000
+#define DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE1_MASK 0x00000002L
+#define DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE1__SHIFT 0x00000001
+#define DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE2_MASK 0x00000004L
+#define DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE2__SHIFT 0x00000002
+#define DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE3_MASK 0x00000008L
+#define DP_DPHY_CNTL__DPHY_ATEST_SEL_LANE3__SHIFT 0x00000003
+#define DP_DPHY_CNTL__DPHY_BYPASS_MASK 0x00010000L
+#define DP_DPHY_CNTL__DPHY_BYPASS__SHIFT 0x00000010
+#define DP_DPHY_CNTL__DPHY_SKEW_BYPASS_MASK 0x01000000L
+#define DP_DPHY_CNTL__DPHY_SKEW_BYPASS__SHIFT 0x00000018
+#define DP_DPHY_CRC_CNTL__DPHY_CRC_FIELD_MASK 0x00000001L
+#define DP_DPHY_CRC_CNTL__DPHY_CRC_FIELD__SHIFT 0x00000000
+#define DP_DPHY_CRC_CNTL__DPHY_CRC_MASK_MASK 0x00ff0000L
+#define DP_DPHY_CRC_CNTL__DPHY_CRC_MASK__SHIFT 0x00000010
+#define DP_DPHY_CRC_CNTL__DPHY_CRC_SEL_MASK 0x00000030L
+#define DP_DPHY_CRC_CNTL__DPHY_CRC_SEL__SHIFT 0x00000004
+#define DP_DPHY_CRC_EN__DPHY_CRC_CONT_EN_MASK 0x00000010L
+#define DP_DPHY_CRC_EN__DPHY_CRC_CONT_EN__SHIFT 0x00000004
+#define DP_DPHY_CRC_EN__DPHY_CRC_EN_MASK 0x00000001L
+#define DP_DPHY_CRC_EN__DPHY_CRC_EN__SHIFT 0x00000000
+#define DP_DPHY_CRC_EN__DPHY_CRC_RESULT_VALID_MASK 0x00000100L
+#define DP_DPHY_CRC_EN__DPHY_CRC_RESULT_VALID__SHIFT 0x00000008
+#define DP_DPHY_CRC_MST_CNTL__DPHY_CRC_MST_FIRST_SLOT_MASK 0x0000003fL
+#define DP_DPHY_CRC_MST_CNTL__DPHY_CRC_MST_FIRST_SLOT__SHIFT 0x00000000
+#define DP_DPHY_CRC_MST_CNTL__DPHY_CRC_MST_LAST_SLOT_MASK 0x00003f00L
+#define DP_DPHY_CRC_MST_CNTL__DPHY_CRC_MST_LAST_SLOT__SHIFT 0x00000008
+#define DP_DPHY_CRC_MST_STATUS__DPHY_CRC_MST_PHASE_ERROR_ACK_MASK 0x00010000L
+#define DP_DPHY_CRC_MST_STATUS__DPHY_CRC_MST_PHASE_ERROR_ACK__SHIFT 0x00000010
+#define DP_DPHY_CRC_MST_STATUS__DPHY_CRC_MST_PHASE_ERROR_MASK 0x00000100L
+#define DP_DPHY_CRC_MST_STATUS__DPHY_CRC_MST_PHASE_ERROR__SHIFT 0x00000008
+#define DP_DPHY_CRC_MST_STATUS__DPHY_CRC_MST_PHASE_LOCK_MASK 0x00000001L
+#define DP_DPHY_CRC_MST_STATUS__DPHY_CRC_MST_PHASE_LOCK__SHIFT 0x00000000
+#define DP_DPHY_CRC_RESULT__DPHY_CRC_RESULT1_MASK 0x0000ff00L
+#define DP_DPHY_CRC_RESULT__DPHY_CRC_RESULT1__SHIFT 0x00000008
+#define DP_DPHY_CRC_RESULT__DPHY_CRC_RESULT2_MASK 0x00ff0000L
+#define DP_DPHY_CRC_RESULT__DPHY_CRC_RESULT2__SHIFT 0x00000010
+#define DP_DPHY_CRC_RESULT__DPHY_CRC_RESULT3_MASK 0xff000000L
+#define DP_DPHY_CRC_RESULT__DPHY_CRC_RESULT3__SHIFT 0x00000018
+#define DP_DPHY_CRC_RESULT__DPHY_CRC_RESULT_MASK 0x000000ffL
+#define DP_DPHY_CRC_RESULT__DPHY_CRC_RESULT__SHIFT 0x00000000
+#define DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP1_TIME_MASK 0x000fff00L
+#define DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP1_TIME__SHIFT 0x00000008
+#define DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP2_TIME_MASK 0xfff00000L
+#define DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_TP2_TIME__SHIFT 0x00000014
+#define DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_VBLANK_EDGE_DETECT_EN_MASK 0x00000004L
+#define DP_DPHY_FAST_TRAINING__DPHY_FAST_TRAINING_VBLANK_EDGE_DETECT_EN__SHIFT 0x00000002
+#define DP_DPHY_FAST_TRAINING__DPHY_RX_FAST_TRAINING_CAPABLE_MASK 0x00000001L
+#define DP_DPHY_FAST_TRAINING__DPHY_RX_FAST_TRAINING_CAPABLE__SHIFT 0x00000000
+#define DP_DPHY_FAST_TRAINING__DPHY_SW_FAST_TRAINING_START_MASK 0x00000002L
+#define DP_DPHY_FAST_TRAINING__DPHY_SW_FAST_TRAINING_START__SHIFT 0x00000001
+#define DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_ACK_MASK 0x00001000L
+#define DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_ACK__SHIFT 0x0000000c
+#define DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_MASK_MASK 0x00000100L
+#define DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_MASK__SHIFT 0x00000008
+#define DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_OCCURRED_MASK 0x00000010L
+#define DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_COMPLETE_OCCURRED__SHIFT 0x00000004
+#define DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_STATE_MASK 0x00000007L
+#define DP_DPHY_FAST_TRAINING_STATUS__DPHY_FAST_TRAINING_STATE__SHIFT 0x00000000
+#define DP_DPHY_PRBS_CNTL__DPHY_PRBS_EN_MASK 0x00000001L
+#define DP_DPHY_PRBS_CNTL__DPHY_PRBS_EN__SHIFT 0x00000000
+#define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED_MASK 0x7fffff00L
+#define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEED__SHIFT 0x00000008
+#define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL_MASK 0x00000030L
+#define DP_DPHY_PRBS_CNTL__DPHY_PRBS_SEL__SHIFT 0x00000004
+#define DP_DPHY_SYM0__DPHY_SYM1_MASK 0x000003ffL
+#define DP_DPHY_SYM0__DPHY_SYM1__SHIFT 0x00000000
+#define DP_DPHY_SYM0__DPHY_SYM2_MASK 0x000ffc00L
+#define DP_DPHY_SYM0__DPHY_SYM2__SHIFT 0x0000000a
+#define DP_DPHY_SYM0__DPHY_SYM3_MASK 0x3ff00000L
+#define DP_DPHY_SYM0__DPHY_SYM3__SHIFT 0x00000014
+#define DP_DPHY_SYM1__DPHY_SYM4_MASK 0x000003ffL
+#define DP_DPHY_SYM1__DPHY_SYM4__SHIFT 0x00000000
+#define DP_DPHY_SYM1__DPHY_SYM5_MASK 0x000ffc00L
+#define DP_DPHY_SYM1__DPHY_SYM5__SHIFT 0x0000000a
+#define DP_DPHY_SYM1__DPHY_SYM6_MASK 0x3ff00000L
+#define DP_DPHY_SYM1__DPHY_SYM6__SHIFT 0x00000014
+#define DP_DPHY_SYM2__DPHY_SYM7_MASK 0x000003ffL
+#define DP_DPHY_SYM2__DPHY_SYM7__SHIFT 0x00000000
+#define DP_DPHY_SYM2__DPHY_SYM8_MASK 0x000ffc00L
+#define DP_DPHY_SYM2__DPHY_SYM8__SHIFT 0x0000000a
+#define DP_DPHY_TRAINING_PATTERN_SEL__DPHY_TRAINING_PATTERN_SEL_MASK 0x00000003L
+#define DP_DPHY_TRAINING_PATTERN_SEL__DPHY_TRAINING_PATTERN_SEL__SHIFT 0x00000000
+#define DP_DTO0_MODULO__DP_DTO0_MODULO_MASK 0xffffffffL
+#define DP_DTO0_MODULO__DP_DTO0_MODULO__SHIFT 0x00000000
+#define DP_DTO0_PHASE__DP_DTO0_PHASE_MASK 0xffffffffL
+#define DP_DTO0_PHASE__DP_DTO0_PHASE__SHIFT 0x00000000
+#define DP_DTO1_MODULO__DP_DTO1_MODULO_MASK 0xffffffffL
+#define DP_DTO1_MODULO__DP_DTO1_MODULO__SHIFT 0x00000000
+#define DP_DTO1_PHASE__DP_DTO1_PHASE_MASK 0xffffffffL
+#define DP_DTO1_PHASE__DP_DTO1_PHASE__SHIFT 0x00000000
+#define DP_DTO2_MODULO__DP_DTO2_MODULO_MASK 0xffffffffL
+#define DP_DTO2_MODULO__DP_DTO2_MODULO__SHIFT 0x00000000
+#define DP_DTO2_PHASE__DP_DTO2_PHASE_MASK 0xffffffffL
+#define DP_DTO2_PHASE__DP_DTO2_PHASE__SHIFT 0x00000000
+#define DP_DTO3_MODULO__DP_DTO3_MODULO_MASK 0xffffffffL
+#define DP_DTO3_MODULO__DP_DTO3_MODULO__SHIFT 0x00000000
+#define DP_DTO3_PHASE__DP_DTO3_PHASE_MASK 0xffffffffL
+#define DP_DTO3_PHASE__DP_DTO3_PHASE__SHIFT 0x00000000
+#define DP_DTO4_MODULO__DP_DTO4_MODULO_MASK 0xffffffffL
+#define DP_DTO4_MODULO__DP_DTO4_MODULO__SHIFT 0x00000000
+#define DP_DTO4_PHASE__DP_DTO4_PHASE_MASK 0xffffffffL
+#define DP_DTO4_PHASE__DP_DTO4_PHASE__SHIFT 0x00000000
+#define DP_DTO5_MODULO__DP_DTO5_MODULO_MASK 0xffffffffL
+#define DP_DTO5_MODULO__DP_DTO5_MODULO__SHIFT 0x00000000
+#define DP_DTO5_PHASE__DP_DTO5_PHASE_MASK 0xffffffffL
+#define DP_DTO5_PHASE__DP_DTO5_PHASE__SHIFT 0x00000000
+#define DPG_PIPE_ARBITRATION_CONTROL1__BASE_WEIGHT_MASK 0xffff0000L
+#define DPG_PIPE_ARBITRATION_CONTROL1__BASE_WEIGHT__SHIFT 0x00000010
+#define DPG_PIPE_ARBITRATION_CONTROL1__PIXEL_DURATION_MASK 0x0000ffffL
+#define DPG_PIPE_ARBITRATION_CONTROL1__PIXEL_DURATION__SHIFT 0x00000000
+#define DPG_PIPE_ARBITRATION_CONTROL2__TIME_WEIGHT_MASK 0x0000ffffL
+#define DPG_PIPE_ARBITRATION_CONTROL2__TIME_WEIGHT__SHIFT 0x00000000
+#define DPG_PIPE_ARBITRATION_CONTROL2__URGENCY_WEIGHT_MASK 0xffff0000L
+#define DPG_PIPE_ARBITRATION_CONTROL2__URGENCY_WEIGHT__SHIFT 0x00000010
+#define DPG_PIPE_DPM_CONTROL__DPM_ENABLE_MASK 0x00000001L
+#define DPG_PIPE_DPM_CONTROL__DPM_ENABLE__SHIFT 0x00000000
+#define DPG_PIPE_DPM_CONTROL__MCLK_CHANGE_ENABLE_MASK 0x00000010L
+#define DPG_PIPE_DPM_CONTROL__MCLK_CHANGE_ENABLE__SHIFT 0x00000004
+#define DPG_PIPE_DPM_CONTROL__MCLK_CHANGE_FORCE_ON_MASK 0x00000100L
+#define DPG_PIPE_DPM_CONTROL__MCLK_CHANGE_FORCE_ON__SHIFT 0x00000008
+#define DPG_PIPE_DPM_CONTROL__MCLK_CHANGE_WATERMARK_MASK 0xffff0000L
+#define DPG_PIPE_DPM_CONTROL__MCLK_CHANGE_WATERMARK_MASK_MASK 0x00003000L
+#define DPG_PIPE_DPM_CONTROL__MCLK_CHANGE_WATERMARK_MASK__SHIFT 0x0000000c
+#define DPG_PIPE_DPM_CONTROL__MCLK_CHANGE_WATERMARK__SHIFT 0x00000010
+#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_ALLOW_FOR_URGENT_MASK 0x00000400L
+#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_ALLOW_FOR_URGENT__SHIFT 0x0000000a
+#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_ENABLE_MASK 0x00000001L
+#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_ENABLE__SHIFT 0x00000000
+#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_FORCE_ON_MASK 0x00000200L
+#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_FORCE_ON__SHIFT 0x00000009
+#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_NOT_SELF_REFRESH_DURING_REQUEST_MASK 0x00000100L
+#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_NOT_SELF_REFRESH_DURING_REQUEST__SHIFT 0x00000008
+#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_URGENT_DURING_REQUEST_MASK 0x00000010L
+#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_URGENT_DURING_REQUEST__SHIFT 0x00000004
+#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_WATERMARK_MASK 0xffff0000L
+#define DPG_PIPE_NB_PSTATE_CHANGE_CONTROL__NB_PSTATE_CHANGE_WATERMARK__SHIFT 0x00000010
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_ENABLE_NONLPTCH_MASK 0x00000001L
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_ENABLE_NONLPTCH__SHIFT 0x00000000
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_IGNORE_CURSOR_NONLPTCH_MASK 0x00000010L
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_IGNORE_CURSOR_NONLPTCH__SHIFT 0x00000004
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_IGNORE_FBC_NONLPTCH_MASK 0x00000080L
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_IGNORE_FBC_NONLPTCH__SHIFT 0x00000007
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_IGNORE_ICON_NONLPTCH_MASK 0x00000020L
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_IGNORE_ICON_NONLPTCH__SHIFT 0x00000005
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_IGNORE_VGA_NONLPTCH_MASK 0x00000040L
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_IGNORE_VGA_NONLPTCH__SHIFT 0x00000006
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_SELF_REFRESH_FORCE_ON_NONLPTCH_MASK 0x00000800L
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_SELF_REFRESH_FORCE_ON_NONLPTCH__SHIFT 0x0000000b
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_URGENT_IN_NOT_SELF_REFRESH_NONLPTCH_MASK 0x00000400L
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_URGENT_IN_NOT_SELF_REFRESH_NONLPTCH__SHIFT 0x0000000a
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_WM_HIGH_EXCLUDES_VBLANK_NONLPTCH_MASK 0x00000200L
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_WM_HIGH_EXCLUDES_VBLANK_NONLPTCH__SHIFT 0x00000009
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_WM_HIGH_FORCE_ON_NONLPTCH_MASK 0x00000100L
+#define DPG_PIPE_STUTTER_CONTROL_NONLPTCH__STUTTER_WM_HIGH_FORCE_ON_NONLPTCH__SHIFT 0x00000008
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_ENABLE_MASK 0x00000001L
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_ENABLE__SHIFT 0x00000000
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_EXIT_SELF_REFRESH_WATERMARK_MASK 0xffff0000L
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_EXIT_SELF_REFRESH_WATERMARK__SHIFT 0x00000010
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_IGNORE_CURSOR_MASK 0x00000010L
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_IGNORE_CURSOR__SHIFT 0x00000004
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_IGNORE_FBC_MASK 0x00000080L
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_IGNORE_FBC__SHIFT 0x00000007
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_IGNORE_ICON_MASK 0x00000020L
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_IGNORE_ICON__SHIFT 0x00000005
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_IGNORE_VGA_MASK 0x00000040L
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_IGNORE_VGA__SHIFT 0x00000006
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_SELF_REFRESH_FORCE_ON_MASK 0x00000800L
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_SELF_REFRESH_FORCE_ON__SHIFT 0x0000000b
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_URGENT_IN_NOT_SELF_REFRESH_MASK 0x00000400L
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_URGENT_IN_NOT_SELF_REFRESH__SHIFT 0x0000000a
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_WM_HIGH_EXCLUDES_VBLANK_MASK 0x00000200L
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_WM_HIGH_EXCLUDES_VBLANK__SHIFT 0x00000009
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_WM_HIGH_FORCE_ON_MASK 0x00000100L
+#define DPG_PIPE_STUTTER_CONTROL__STUTTER_WM_HIGH_FORCE_ON__SHIFT 0x00000008
+#define DPG_PIPE_URGENCY_CONTROL__URGENCY_HIGH_WATERMARK_MASK 0xffff0000L
+#define DPG_PIPE_URGENCY_CONTROL__URGENCY_HIGH_WATERMARK__SHIFT 0x00000010
+#define DPG_PIPE_URGENCY_CONTROL__URGENCY_LOW_WATERMARK_MASK 0x0000ffffL
+#define DPG_PIPE_URGENCY_CONTROL__URGENCY_LOW_WATERMARK__SHIFT 0x00000000
+#define DPG_TEST_DEBUG_DATA__DPG_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define DPG_TEST_DEBUG_DATA__DPG_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define DPG_TEST_DEBUG_INDEX__DPG_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define DPG_TEST_DEBUG_INDEX__DPG_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define DPG_TEST_DEBUG_INDEX__DPG_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define DPG_TEST_DEBUG_INDEX__DPG_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define DP_HBR2_EYE_PATTERN__DP_HBR2_EYE_PATTERN_ENABLE_MASK 0x00000001L
+#define DP_HBR2_EYE_PATTERN__DP_HBR2_EYE_PATTERN_ENABLE__SHIFT 0x00000000
+#define DP_LINK_CNTL__DP_EMBEDDED_PANEL_MODE_MASK 0x00020000L
+#define DP_LINK_CNTL__DP_EMBEDDED_PANEL_MODE__SHIFT 0x00000011
+#define DP_LINK_CNTL__DP_LINK_STATUS_MASK 0x00000100L
+#define DP_LINK_CNTL__DP_LINK_STATUS__SHIFT 0x00000008
+#define DP_LINK_CNTL__DP_LINK_TRAINING_COMPLETE_MASK 0x00000010L
+#define DP_LINK_CNTL__DP_LINK_TRAINING_COMPLETE__SHIFT 0x00000004
+#define DP_LINK_FRAMING_CNTL__DP_IDLE_BS_INTERVAL_MASK 0x0003ffffL
+#define DP_LINK_FRAMING_CNTL__DP_IDLE_BS_INTERVAL__SHIFT 0x00000000
+#define DP_LINK_FRAMING_CNTL__DP_VBID_DISABLE_MASK 0x01000000L
+#define DP_LINK_FRAMING_CNTL__DP_VBID_DISABLE__SHIFT 0x00000018
+#define DP_LINK_FRAMING_CNTL__DP_VID_ENHANCED_FRAME_MODE_MASK 0x10000000L
+#define DP_LINK_FRAMING_CNTL__DP_VID_ENHANCED_FRAME_MODE__SHIFT 0x0000001c
+#define DP_MSA_COLORIMETRY__DP_MSA_MISC0_OVERRIDE_ENABLE_MASK 0x00000100L
+#define DP_MSA_COLORIMETRY__DP_MSA_MISC0_OVERRIDE_ENABLE__SHIFT 0x00000008
+#define DP_MSA_COLORIMETRY__DP_MSA_MISC0_OVERRIDE_MASK 0x000000ffL
+#define DP_MSA_COLORIMETRY__DP_MSA_MISC0_OVERRIDE__SHIFT 0x00000000
+#define DP_MSA_MISC__DP_MSA_MISC1_MASK 0x000000f8L
+#define DP_MSA_MISC__DP_MSA_MISC1__SHIFT 0x00000003
+#define DP_MSA_MISC__DP_MSA_MISC2_MASK 0x0000ff00L
+#define DP_MSA_MISC__DP_MSA_MISC2__SHIFT 0x00000008
+#define DP_MSA_MISC__DP_MSA_MISC3_MASK 0x00ff0000L
+#define DP_MSA_MISC__DP_MSA_MISC3__SHIFT 0x00000010
+#define DP_MSA_MISC__DP_MSA_MISC4_MASK 0xff000000L
+#define DP_MSA_MISC__DP_MSA_MISC4__SHIFT 0x00000018
+#define DP_MSA_V_TIMING_OVERRIDE1__DP_MSA_V_TIMING_OVERRIDE_EN_MASK 0x00000001L
+#define DP_MSA_V_TIMING_OVERRIDE1__DP_MSA_V_TIMING_OVERRIDE_EN__SHIFT 0x00000000
+#define DP_MSA_V_TIMING_OVERRIDE1__DP_MSA_V_TOTAL_OVERRIDE_MASK 0x0001fff0L
+#define DP_MSA_V_TIMING_OVERRIDE1__DP_MSA_V_TOTAL_OVERRIDE__SHIFT 0x00000004
+#define DP_MSA_V_TIMING_OVERRIDE2__DP_MSA_V_BLANK_END_OVERRIDE_MASK 0x1fff0000L
+#define DP_MSA_V_TIMING_OVERRIDE2__DP_MSA_V_BLANK_END_OVERRIDE__SHIFT 0x00000010
+#define DP_MSA_V_TIMING_OVERRIDE2__DP_MSA_V_BLANK_START_OVERRIDE_MASK 0x00001fffL
+#define DP_MSA_V_TIMING_OVERRIDE2__DP_MSA_V_BLANK_START_OVERRIDE__SHIFT 0x00000000
+#define DP_MSE_LINK_TIMING__DP_MSE_LINK_FRAME_MASK 0x000003ffL
+#define DP_MSE_LINK_TIMING__DP_MSE_LINK_FRAME__SHIFT 0x00000000
+#define DP_MSE_LINK_TIMING__DP_MSE_LINK_LINE_MASK 0x00030000L
+#define DP_MSE_LINK_TIMING__DP_MSE_LINK_LINE__SHIFT 0x00000010
+#define DP_MSE_MISC_CNTL__DP_MSE_BLANK_CODE_MASK 0x00000001L
+#define DP_MSE_MISC_CNTL__DP_MSE_BLANK_CODE__SHIFT 0x00000000
+#define DP_MSE_MISC_CNTL__DP_MSE_TIMESTAMP_MODE_MASK 0x00000010L
+#define DP_MSE_MISC_CNTL__DP_MSE_TIMESTAMP_MODE__SHIFT 0x00000004
+#define DP_MSE_MISC_CNTL__DP_MSE_ZERO_ENCODER_MASK 0x00000100L
+#define DP_MSE_MISC_CNTL__DP_MSE_ZERO_ENCODER__SHIFT 0x00000008
+#define DP_MSE_RATE_CNTL__DP_MSE_RATE_X_MASK 0xfc000000L
+#define DP_MSE_RATE_CNTL__DP_MSE_RATE_X__SHIFT 0x0000001a
+#define DP_MSE_RATE_CNTL__DP_MSE_RATE_Y_MASK 0x03ffffffL
+#define DP_MSE_RATE_CNTL__DP_MSE_RATE_Y__SHIFT 0x00000000
+#define DP_MSE_RATE_UPDATE__DP_MSE_RATE_UPDATE_PENDING_MASK 0x00000001L
+#define DP_MSE_RATE_UPDATE__DP_MSE_RATE_UPDATE_PENDING__SHIFT 0x00000000
+#define DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT0_MASK 0x00003f00L
+#define DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT0__SHIFT 0x00000008
+#define DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT1_MASK 0x3f000000L
+#define DP_MSE_SAT0__DP_MSE_SAT_SLOT_COUNT1__SHIFT 0x00000018
+#define DP_MSE_SAT0__DP_MSE_SAT_SRC0_MASK 0x00000007L
+#define DP_MSE_SAT0__DP_MSE_SAT_SRC0__SHIFT 0x00000000
+#define DP_MSE_SAT0__DP_MSE_SAT_SRC1_MASK 0x00070000L
+#define DP_MSE_SAT0__DP_MSE_SAT_SRC1__SHIFT 0x00000010
+#define DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT2_MASK 0x00003f00L
+#define DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT2__SHIFT 0x00000008
+#define DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT3_MASK 0x3f000000L
+#define DP_MSE_SAT1__DP_MSE_SAT_SLOT_COUNT3__SHIFT 0x00000018
+#define DP_MSE_SAT1__DP_MSE_SAT_SRC2_MASK 0x00000007L
+#define DP_MSE_SAT1__DP_MSE_SAT_SRC2__SHIFT 0x00000000
+#define DP_MSE_SAT1__DP_MSE_SAT_SRC3_MASK 0x00070000L
+#define DP_MSE_SAT1__DP_MSE_SAT_SRC3__SHIFT 0x00000010
+#define DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT4_MASK 0x00003f00L
+#define DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT4__SHIFT 0x00000008
+#define DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT5_MASK 0x3f000000L
+#define DP_MSE_SAT2__DP_MSE_SAT_SLOT_COUNT5__SHIFT 0x00000018
+#define DP_MSE_SAT2__DP_MSE_SAT_SRC4_MASK 0x00000007L
+#define DP_MSE_SAT2__DP_MSE_SAT_SRC4__SHIFT 0x00000000
+#define DP_MSE_SAT2__DP_MSE_SAT_SRC5_MASK 0x00070000L
+#define DP_MSE_SAT2__DP_MSE_SAT_SRC5__SHIFT 0x00000010
+#define DP_MSE_SAT_UPDATE__DP_MSE_16_MTP_KEEPOUT_MASK 0x00000100L
+#define DP_MSE_SAT_UPDATE__DP_MSE_16_MTP_KEEPOUT__SHIFT 0x00000008
+#define DP_MSE_SAT_UPDATE__DP_MSE_SAT_UPDATE_MASK 0x00000003L
+#define DP_MSE_SAT_UPDATE__DP_MSE_SAT_UPDATE__SHIFT 0x00000000
+#define DP_PIXEL_FORMAT__DP_COMPONENT_DEPTH_MASK 0x07000000L
+#define DP_PIXEL_FORMAT__DP_COMPONENT_DEPTH__SHIFT 0x00000018
+#define DP_PIXEL_FORMAT__DP_DYN_RANGE_MASK 0x00000100L
+#define DP_PIXEL_FORMAT__DP_DYN_RANGE__SHIFT 0x00000008
+#define DP_PIXEL_FORMAT__DP_PIXEL_ENCODING_MASK 0x00000003L
+#define DP_PIXEL_FORMAT__DP_PIXEL_ENCODING__SHIFT 0x00000000
+#define DP_PIXEL_FORMAT__DP_YCBCR_RANGE_MASK 0x00010000L
+#define DP_PIXEL_FORMAT__DP_YCBCR_RANGE__SHIFT 0x00000010
+#define DP_SEC_AUD_M__DP_SEC_AUD_M_MASK 0x00ffffffL
+#define DP_SEC_AUD_M__DP_SEC_AUD_M__SHIFT 0x00000000
+#define DP_SEC_AUD_M_READBACK__DP_SEC_AUD_M_READBACK_MASK 0x00ffffffL
+#define DP_SEC_AUD_M_READBACK__DP_SEC_AUD_M_READBACK__SHIFT 0x00000000
+#define DP_SEC_AUD_N__DP_SEC_AUD_N_MASK 0x00ffffffL
+#define DP_SEC_AUD_N__DP_SEC_AUD_N__SHIFT 0x00000000
+#define DP_SEC_AUD_N_READBACK__DP_SEC_AUD_N_READBACK_MASK 0x00ffffffL
+#define DP_SEC_AUD_N_READBACK__DP_SEC_AUD_N_READBACK__SHIFT 0x00000000
+#define DP_SEC_CNTL1__DP_SEC_ISRC_ENABLE_MASK 0x00000001L
+#define DP_SEC_CNTL1__DP_SEC_ISRC_ENABLE__SHIFT 0x00000000
+#define DP_SEC_CNTL__DP_SEC_ACM_ENABLE_MASK 0x00010000L
+#define DP_SEC_CNTL__DP_SEC_ACM_ENABLE__SHIFT 0x00000010
+#define DP_SEC_CNTL__DP_SEC_AIP_ENABLE_MASK 0x00001000L
+#define DP_SEC_CNTL__DP_SEC_AIP_ENABLE__SHIFT 0x0000000c
+#define DP_SEC_CNTL__DP_SEC_ASP_ENABLE_MASK 0x00000010L
+#define DP_SEC_CNTL__DP_SEC_ASP_ENABLE__SHIFT 0x00000004
+#define DP_SEC_CNTL__DP_SEC_ATP_ENABLE_MASK 0x00000100L
+#define DP_SEC_CNTL__DP_SEC_ATP_ENABLE__SHIFT 0x00000008
+#define DP_SEC_CNTL__DP_SEC_AVI_ENABLE_MASK 0x01000000L
+#define DP_SEC_CNTL__DP_SEC_AVI_ENABLE__SHIFT 0x00000018
+#define DP_SEC_CNTL__DP_SEC_GSP0_ENABLE_MASK 0x00100000L
+#define DP_SEC_CNTL__DP_SEC_GSP0_ENABLE__SHIFT 0x00000014
+#define DP_SEC_CNTL__DP_SEC_GSP1_ENABLE_MASK 0x00200000L
+#define DP_SEC_CNTL__DP_SEC_GSP1_ENABLE__SHIFT 0x00000015
+#define DP_SEC_CNTL__DP_SEC_GSP2_ENABLE_MASK 0x00400000L
+#define DP_SEC_CNTL__DP_SEC_GSP2_ENABLE__SHIFT 0x00000016
+#define DP_SEC_CNTL__DP_SEC_GSP3_ENABLE_MASK 0x00800000L
+#define DP_SEC_CNTL__DP_SEC_GSP3_ENABLE__SHIFT 0x00000017
+#define DP_SEC_CNTL__DP_SEC_MPG_ENABLE_MASK 0x10000000L
+#define DP_SEC_CNTL__DP_SEC_MPG_ENABLE__SHIFT 0x0000001c
+#define DP_SEC_CNTL__DP_SEC_STREAM_ENABLE_MASK 0x00000001L
+#define DP_SEC_CNTL__DP_SEC_STREAM_ENABLE__SHIFT 0x00000000
+#define DP_SEC_FRAMING1__DP_SEC_FRAME_START_LOCATION_MASK 0x00000fffL
+#define DP_SEC_FRAMING1__DP_SEC_FRAME_START_LOCATION__SHIFT 0x00000000
+#define DP_SEC_FRAMING1__DP_SEC_VBLANK_TRANSMIT_WIDTH_MASK 0xffff0000L
+#define DP_SEC_FRAMING1__DP_SEC_VBLANK_TRANSMIT_WIDTH__SHIFT 0x00000010
+#define DP_SEC_FRAMING2__DP_SEC_HBLANK_TRANSMIT_WIDTH_MASK 0xffff0000L
+#define DP_SEC_FRAMING2__DP_SEC_HBLANK_TRANSMIT_WIDTH__SHIFT 0x00000010
+#define DP_SEC_FRAMING2__DP_SEC_START_POSITION_MASK 0x0000ffffL
+#define DP_SEC_FRAMING2__DP_SEC_START_POSITION__SHIFT 0x00000000
+#define DP_SEC_FRAMING3__DP_SEC_IDLE_FRAME_SIZE_MASK 0x00003fffL
+#define DP_SEC_FRAMING3__DP_SEC_IDLE_FRAME_SIZE__SHIFT 0x00000000
+#define DP_SEC_FRAMING3__DP_SEC_IDLE_TRANSMIT_WIDTH_MASK 0xffff0000L
+#define DP_SEC_FRAMING3__DP_SEC_IDLE_TRANSMIT_WIDTH__SHIFT 0x00000010
+#define DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_MASK 0x10000000L
+#define DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE__SHIFT 0x0000001c
+#define DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_STATUS_MASK 0x20000000L
+#define DP_SEC_FRAMING4__DP_SEC_AUDIO_MUTE_STATUS__SHIFT 0x0000001d
+#define DP_SEC_FRAMING4__DP_SEC_COLLISION_ACK_MASK 0x01000000L
+#define DP_SEC_FRAMING4__DP_SEC_COLLISION_ACK__SHIFT 0x00000018
+#define DP_SEC_FRAMING4__DP_SEC_COLLISION_STATUS_MASK 0x00100000L
+#define DP_SEC_FRAMING4__DP_SEC_COLLISION_STATUS__SHIFT 0x00000014
+#define DP_SEC_PACKET_CNTL__DP_SEC_ASP_CHANNEL_COUNT_OVERRIDE_MASK 0x00010000L
+#define DP_SEC_PACKET_CNTL__DP_SEC_ASP_CHANNEL_COUNT_OVERRIDE__SHIFT 0x00000010
+#define DP_SEC_PACKET_CNTL__DP_SEC_ASP_CODING_TYPE_MASK 0x0000000eL
+#define DP_SEC_PACKET_CNTL__DP_SEC_ASP_CODING_TYPE__SHIFT 0x00000001
+#define DP_SEC_PACKET_CNTL__DP_SEC_ASP_PRIORITY_MASK 0x00000010L
+#define DP_SEC_PACKET_CNTL__DP_SEC_ASP_PRIORITY__SHIFT 0x00000004
+#define DP_SEC_PACKET_CNTL__DP_SEC_VERSION_MASK 0x00003f00L
+#define DP_SEC_PACKET_CNTL__DP_SEC_VERSION__SHIFT 0x00000008
+#define DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE_MASK 0x00000001L
+#define DP_SEC_TIMESTAMP__DP_SEC_TIMESTAMP_MODE__SHIFT 0x00000000
+#define DP_STEER_FIFO__DP_STEER_FIFO_RESET_MASK 0x00000001L
+#define DP_STEER_FIFO__DP_STEER_FIFO_RESET__SHIFT 0x00000000
+#define DP_STEER_FIFO__DP_STEER_OVERFLOW_ACK_MASK 0x00000040L
+#define DP_STEER_FIFO__DP_STEER_OVERFLOW_ACK__SHIFT 0x00000006
+#define DP_STEER_FIFO__DP_STEER_OVERFLOW_FLAG_MASK 0x00000010L
+#define DP_STEER_FIFO__DP_STEER_OVERFLOW_FLAG__SHIFT 0x00000004
+#define DP_STEER_FIFO__DP_STEER_OVERFLOW_INT_MASK 0x00000020L
+#define DP_STEER_FIFO__DP_STEER_OVERFLOW_INT__SHIFT 0x00000005
+#define DP_STEER_FIFO__DP_STEER_OVERFLOW_MASK_MASK 0x00000080L
+#define DP_STEER_FIFO__DP_STEER_OVERFLOW_MASK__SHIFT 0x00000007
+#define DP_STEER_FIFO__DP_TU_OVERFLOW_ACK_MASK 0x00001000L
+#define DP_STEER_FIFO__DP_TU_OVERFLOW_ACK__SHIFT 0x0000000c
+#define DP_STEER_FIFO__DP_TU_OVERFLOW_FLAG_MASK 0x00000100L
+#define DP_STEER_FIFO__DP_TU_OVERFLOW_FLAG__SHIFT 0x00000008
+#define DP_TEST_DEBUG_DATA__DP_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define DP_TEST_DEBUG_DATA__DP_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define DP_TEST_DEBUG_INDEX__DP_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define DP_TEST_DEBUG_INDEX__DP_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define DP_TEST_DEBUG_INDEX__DP_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define DP_TEST_DEBUG_INDEX__DP_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_ACK_MASK 0x00000002L
+#define DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_ACK__SHIFT 0x00000001
+#define DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_INT_MASK 0x00000001L
+#define DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_INT__SHIFT 0x00000000
+#define DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_MASK_MASK 0x00000004L
+#define DP_VID_INTERRUPT_CNTL__DP_VID_STREAM_DISABLE_MASK__SHIFT 0x00000002
+#define DP_VID_M__DP_VID_M_MASK 0x00ffffffL
+#define DP_VID_M__DP_VID_M__SHIFT 0x00000000
+#define DP_VID_MSA_VBID__DP_VID_MSA_LOCATION_MASK 0x00000fffL
+#define DP_VID_MSA_VBID__DP_VID_MSA_LOCATION__SHIFT 0x00000000
+#define DP_VID_MSA_VBID__DP_VID_MSA_TOP_FIELD_MODE_MASK 0x00010000L
+#define DP_VID_MSA_VBID__DP_VID_MSA_TOP_FIELD_MODE__SHIFT 0x00000010
+#define DP_VID_MSA_VBID__DP_VID_VBID_FIELD_POL_MASK 0x01000000L
+#define DP_VID_MSA_VBID__DP_VID_VBID_FIELD_POL__SHIFT 0x00000018
+#define DP_VID_N__DP_VID_N_MASK 0x00ffffffL
+#define DP_VID_N__DP_VID_N__SHIFT 0x00000000
+#define DP_VID_STREAM_CNTL__DP_VID_STREAM_CHANGE_KEEPOUT_MASK 0x00100000L
+#define DP_VID_STREAM_CNTL__DP_VID_STREAM_CHANGE_KEEPOUT__SHIFT 0x00000014
+#define DP_VID_STREAM_CNTL__DP_VID_STREAM_DIS_DEFER_MASK 0x00000300L
+#define DP_VID_STREAM_CNTL__DP_VID_STREAM_DIS_DEFER__SHIFT 0x00000008
+#define DP_VID_STREAM_CNTL__DP_VID_STREAM_ENABLE_MASK 0x00000001L
+#define DP_VID_STREAM_CNTL__DP_VID_STREAM_ENABLE__SHIFT 0x00000000
+#define DP_VID_STREAM_CNTL__DP_VID_STREAM_STATUS_MASK 0x00010000L
+#define DP_VID_STREAM_CNTL__DP_VID_STREAM_STATUS__SHIFT 0x00000010
+#define DP_VID_TIMING__DP_VID_M_N_GEN_EN_MASK 0x00000100L
+#define DP_VID_TIMING__DP_VID_M_N_GEN_EN__SHIFT 0x00000008
+#define DP_VID_TIMING__DP_VID_N_DIV_MASK 0xff000000L
+#define DP_VID_TIMING__DP_VID_N_DIV__SHIFT 0x00000018
+#define DP_VID_TIMING__DP_VID_TIMING_MODE_MASK 0x00000001L
+#define DP_VID_TIMING__DP_VID_TIMING_MODE__SHIFT 0x00000000
+#define DVOACLKC_CNTL__DVOACLKC_COARSE_ADJUST_EN_MASK 0x00020000L
+#define DVOACLKC_CNTL__DVOACLKC_COARSE_ADJUST_EN__SHIFT 0x00000011
+#define DVOACLKC_CNTL__DVOACLKC_COARSE_SKEW_CNTL_MASK 0x00001f00L
+#define DVOACLKC_CNTL__DVOACLKC_COARSE_SKEW_CNTL__SHIFT 0x00000008
+#define DVOACLKC_CNTL__DVOACLKC_FINE_ADJUST_EN_MASK 0x00010000L
+#define DVOACLKC_CNTL__DVOACLKC_FINE_ADJUST_EN__SHIFT 0x00000010
+#define DVOACLKC_CNTL__DVOACLKC_FINE_SKEW_CNTL_MASK 0x00000007L
+#define DVOACLKC_CNTL__DVOACLKC_FINE_SKEW_CNTL__SHIFT 0x00000000
+#define DVOACLKC_CNTL__DVOACLKC_IN_PHASE_MASK 0x00040000L
+#define DVOACLKC_CNTL__DVOACLKC_IN_PHASE__SHIFT 0x00000012
+#define DVOACLKC_MVP_CNTL__DVOACLKC_MVP_COARSE_ADJUST_EN_MASK 0x00020000L
+#define DVOACLKC_MVP_CNTL__DVOACLKC_MVP_COARSE_ADJUST_EN__SHIFT 0x00000011
+#define DVOACLKC_MVP_CNTL__DVOACLKC_MVP_COARSE_SKEW_CNTL_MASK 0x00001f00L
+#define DVOACLKC_MVP_CNTL__DVOACLKC_MVP_COARSE_SKEW_CNTL__SHIFT 0x00000008
+#define DVOACLKC_MVP_CNTL__DVOACLKC_MVP_FINE_ADJUST_EN_MASK 0x00010000L
+#define DVOACLKC_MVP_CNTL__DVOACLKC_MVP_FINE_ADJUST_EN__SHIFT 0x00000010
+#define DVOACLKC_MVP_CNTL__DVOACLKC_MVP_FINE_SKEW_CNTL_MASK 0x00000007L
+#define DVOACLKC_MVP_CNTL__DVOACLKC_MVP_FINE_SKEW_CNTL__SHIFT 0x00000000
+#define DVOACLKC_MVP_CNTL__DVOACLKC_MVP_IN_PHASE_MASK 0x00040000L
+#define DVOACLKC_MVP_CNTL__DVOACLKC_MVP_IN_PHASE__SHIFT 0x00000012
+#define DVOACLKC_MVP_CNTL__DVOACLKC_MVP_SKEW_PHASE_OVERRIDE_MASK 0x00100000L
+#define DVOACLKC_MVP_CNTL__DVOACLKC_MVP_SKEW_PHASE_OVERRIDE__SHIFT 0x00000014
+#define DVOACLKC_MVP_CNTL__MVP_CLK_A_SRC_SEL_MASK 0x03000000L
+#define DVOACLKC_MVP_CNTL__MVP_CLK_A_SRC_SEL__SHIFT 0x00000018
+#define DVOACLKC_MVP_CNTL__MVP_CLK_B_SRC_SEL_MASK 0x30000000L
+#define DVOACLKC_MVP_CNTL__MVP_CLK_B_SRC_SEL__SHIFT 0x0000001c
+#define DVOACLKD_CNTL__DVOACLKD_COARSE_ADJUST_EN_MASK 0x00020000L
+#define DVOACLKD_CNTL__DVOACLKD_COARSE_ADJUST_EN__SHIFT 0x00000011
+#define DVOACLKD_CNTL__DVOACLKD_COARSE_SKEW_CNTL_MASK 0x00001f00L
+#define DVOACLKD_CNTL__DVOACLKD_COARSE_SKEW_CNTL__SHIFT 0x00000008
+#define DVOACLKD_CNTL__DVOACLKD_FINE_ADJUST_EN_MASK 0x00010000L
+#define DVOACLKD_CNTL__DVOACLKD_FINE_ADJUST_EN__SHIFT 0x00000010
+#define DVOACLKD_CNTL__DVOACLKD_FINE_SKEW_CNTL_MASK 0x00000007L
+#define DVOACLKD_CNTL__DVOACLKD_FINE_SKEW_CNTL__SHIFT 0x00000000
+#define DVOACLKD_CNTL__DVOACLKD_IN_PHASE_MASK 0x00040000L
+#define DVOACLKD_CNTL__DVOACLKD_IN_PHASE__SHIFT 0x00000012
+#define DVO_CLK_ENABLE__DVO_CLK_ENABLE_MASK 0x00000001L
+#define DVO_CLK_ENABLE__DVO_CLK_ENABLE__SHIFT 0x00000000
+#define DVO_CONTROL__DVO_COLOR_FORMAT_MASK 0x03000000L
+#define DVO_CONTROL__DVO_COLOR_FORMAT__SHIFT 0x00000018
+#define DVO_CONTROL__DVO_CTL3_MASK 0x80000000L
+#define DVO_CONTROL__DVO_CTL3__SHIFT 0x0000001f
+#define DVO_CONTROL__DVO_DUAL_CHANNEL_EN_MASK 0x00000100L
+#define DVO_CONTROL__DVO_DUAL_CHANNEL_EN__SHIFT 0x00000008
+#define DVO_CONTROL__DVO_INVERT_DVOCLK_MASK 0x00040000L
+#define DVO_CONTROL__DVO_INVERT_DVOCLK__SHIFT 0x00000012
+#define DVO_CONTROL__DVO_RATE_SELECT_MASK 0x00000001L
+#define DVO_CONTROL__DVO_RATE_SELECT__SHIFT 0x00000000
+#define DVO_CONTROL__DVO_RESET_FIFO_MASK 0x00010000L
+#define DVO_CONTROL__DVO_RESET_FIFO__SHIFT 0x00000010
+#define DVO_CONTROL__DVO_SDRCLK_SEL_MASK 0x00000002L
+#define DVO_CONTROL__DVO_SDRCLK_SEL__SHIFT 0x00000001
+#define DVO_CONTROL__DVO_SYNC_PHASE_MASK 0x00020000L
+#define DVO_CONTROL__DVO_SYNC_PHASE__SHIFT 0x00000011
+#define DVO_CRC2_SIG_MASK__DVO_CRC2_SIG_MASK_MASK 0x07ffffffL
+#define DVO_CRC2_SIG_MASK__DVO_CRC2_SIG_MASK__SHIFT 0x00000000
+#define DVO_CRC2_SIG_RESULT__DVO_CRC2_SIG_RESULT_MASK 0x07ffffffL
+#define DVO_CRC2_SIG_RESULT__DVO_CRC2_SIG_RESULT__SHIFT 0x00000000
+#define DVO_CRC_EN__DVO_CRC2_EN_MASK 0x00010000L
+#define DVO_CRC_EN__DVO_CRC2_EN__SHIFT 0x00000010
+#define DVO_ENABLE__DVO_ENABLE_MASK 0x00000001L
+#define DVO_ENABLE__DVO_ENABLE__SHIFT 0x00000000
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_CAL_AVERAGE_LEVEL_MASK 0x0000fc00L
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_CAL_AVERAGE_LEVEL__SHIFT 0x0000000a
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_CALIBRATED_MASK 0x20000000L
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_CALIBRATED__SHIFT 0x0000001d
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_ERROR_ACK_MASK 0x00000100L
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_ERROR_ACK__SHIFT 0x00000008
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_FORCE_RECAL_AVERAGE_MASK 0x40000000L
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_FORCE_RECAL_AVERAGE__SHIFT 0x0000001e
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_FORCE_RECOMP_MINMAX_MASK 0x80000000L
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_FORCE_RECOMP_MINMAX__SHIFT 0x0000001f
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_LEVEL_ERROR_MASK 0x00000001L
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_LEVEL_ERROR__SHIFT 0x00000000
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_MAXIMUM_LEVEL_MASK 0x000f0000L
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_MAXIMUM_LEVEL__SHIFT 0x00000010
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_MINIMUM_LEVEL_MASK 0x03c00000L
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_MINIMUM_LEVEL__SHIFT 0x00000016
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_OVERWRITE_LEVEL_MASK 0x000000fcL
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_OVERWRITE_LEVEL__SHIFT 0x00000002
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_USE_OVERWRITE_LEVEL_MASK 0x00000002L
+#define DVO_FIFO_ERROR_STATUS__DVO_FIFO_USE_OVERWRITE_LEVEL__SHIFT 0x00000001
+#define DVO_OUTPUT__DVO_CLOCK_MODE_MASK 0x00000100L
+#define DVO_OUTPUT__DVO_CLOCK_MODE__SHIFT 0x00000008
+#define DVO_OUTPUT__DVO_OUTPUT_ENABLE_MODE_MASK 0x00000003L
+#define DVO_OUTPUT__DVO_OUTPUT_ENABLE_MODE__SHIFT 0x00000000
+#define DVO_SKEW_ADJUST__DVO_SKEW_ADJUST_MASK 0xffffffffL
+#define DVO_SKEW_ADJUST__DVO_SKEW_ADJUST__SHIFT 0x00000000
+#define DVO_SOURCE_SELECT__DVO_SOURCE_SELECT_MASK 0x00000007L
+#define DVO_SOURCE_SELECT__DVO_SOURCE_SELECT__SHIFT 0x00000000
+#define DVO_SOURCE_SELECT__DVO_STEREOSYNC_SELECT_MASK 0x00070000L
+#define DVO_SOURCE_SELECT__DVO_STEREOSYNC_SELECT__SHIFT 0x00000010
+#define DVO_STRENGTH_CONTROL__DVOCLK_SN_MASK 0x0000f000L
+#define DVO_STRENGTH_CONTROL__DVOCLK_SN__SHIFT 0x0000000c
+#define DVO_STRENGTH_CONTROL__DVOCLK_SP_MASK 0x00000f00L
+#define DVO_STRENGTH_CONTROL__DVOCLK_SP__SHIFT 0x00000008
+#define DVO_STRENGTH_CONTROL__DVO_LSB_VMODE_MASK 0x10000000L
+#define DVO_STRENGTH_CONTROL__DVO_LSB_VMODE__SHIFT 0x0000001c
+#define DVO_STRENGTH_CONTROL__DVO_MSB_VMODE_MASK 0x20000000L
+#define DVO_STRENGTH_CONTROL__DVO_MSB_VMODE__SHIFT 0x0000001d
+#define DVO_STRENGTH_CONTROL__DVO_SN_MASK 0x000000f0L
+#define DVO_STRENGTH_CONTROL__DVO_SN__SHIFT 0x00000004
+#define DVO_STRENGTH_CONTROL__DVO_SP_MASK 0x0000000fL
+#define DVO_STRENGTH_CONTROL__DVO_SP__SHIFT 0x00000000
+#define DVO_VREF_CONTROL__DVO_VREFCAL_MASK 0x000000f0L
+#define DVO_VREF_CONTROL__DVO_VREFCAL__SHIFT 0x00000004
+#define DVO_VREF_CONTROL__DVO_VREFPON_MASK 0x00000001L
+#define DVO_VREF_CONTROL__DVO_VREFPON__SHIFT 0x00000000
+#define DVO_VREF_CONTROL__DVO_VREFSEL_MASK 0x00000002L
+#define DVO_VREF_CONTROL__DVO_VREFSEL__SHIFT 0x00000001
+#define EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_LEFT_MASK 0x0fff0000L
+#define EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_LEFT__SHIFT 0x00000010
+#define EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_RIGHT_MASK 0x00000fffL
+#define EXT_OVERSCAN_LEFT_RIGHT__EXT_OVERSCAN_RIGHT__SHIFT 0x00000000
+#define EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_BOTTOM_MASK 0x00000fffL
+#define EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_BOTTOM__SHIFT 0x00000000
+#define EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_TOP_MASK 0x0fff0000L
+#define EXT_OVERSCAN_TOP_BOTTOM__EXT_OVERSCAN_TOP__SHIFT 0x00000010
+#define FBC_CLIENT_REGION_MASK__FBC_MEMORY_REGION_MASK_MASK 0x000f0000L
+#define FBC_CLIENT_REGION_MASK__FBC_MEMORY_REGION_MASK__SHIFT 0x00000010
+#define FBC_CNTL__FBC_COHERENCY_MODE_MASK 0x00030000L
+#define FBC_CNTL__FBC_COHERENCY_MODE__SHIFT 0x00000010
+#define FBC_CNTL__FBC_EN_MASK 0x80000000L
+#define FBC_CNTL__FBC_EN__SHIFT 0x0000001f
+#define FBC_CNTL__FBC_GRPH_COMP_EN_MASK 0x00000001L
+#define FBC_CNTL__FBC_GRPH_COMP_EN__SHIFT 0x00000000
+#define FBC_CNTL__FBC_SOFT_COMPRESS_EN_MASK 0x02000000L
+#define FBC_CNTL__FBC_SOFT_COMPRESS_EN__SHIFT 0x00000019
+#define FBC_CNTL__FBC_SRC_SEL_MASK 0x0000000eL
+#define FBC_CNTL__FBC_SRC_SEL__SHIFT 0x00000001
+#define FBC_COMP_CNTL__FBC_DEPTH_MONO08_EN_MASK 0x00010000L
+#define FBC_COMP_CNTL__FBC_DEPTH_MONO08_EN__SHIFT 0x00000010
+#define FBC_COMP_CNTL__FBC_DEPTH_MONO16_EN_MASK 0x00020000L
+#define FBC_COMP_CNTL__FBC_DEPTH_MONO16_EN__SHIFT 0x00000011
+#define FBC_COMP_CNTL__FBC_DEPTH_RGB04_EN_MASK 0x00040000L
+#define FBC_COMP_CNTL__FBC_DEPTH_RGB04_EN__SHIFT 0x00000012
+#define FBC_COMP_CNTL__FBC_DEPTH_RGB08_EN_MASK 0x00080000L
+#define FBC_COMP_CNTL__FBC_DEPTH_RGB08_EN__SHIFT 0x00000013
+#define FBC_COMP_CNTL__FBC_DEPTH_RGB16_EN_MASK 0x00100000L
+#define FBC_COMP_CNTL__FBC_DEPTH_RGB16_EN__SHIFT 0x00000014
+#define FBC_COMP_CNTL__FBC_MIN_COMPRESSION_MASK 0x0000000fL
+#define FBC_COMP_CNTL__FBC_MIN_COMPRESSION__SHIFT 0x00000000
+#define FBC_COMP_MODE__FBC_DPCM4_RGB_EN_MASK 0x00000100L
+#define FBC_COMP_MODE__FBC_DPCM4_RGB_EN__SHIFT 0x00000008
+#define FBC_COMP_MODE__FBC_DPCM4_YUV_EN_MASK 0x00000400L
+#define FBC_COMP_MODE__FBC_DPCM4_YUV_EN__SHIFT 0x0000000a
+#define FBC_COMP_MODE__FBC_DPCM8_RGB_EN_MASK 0x00000200L
+#define FBC_COMP_MODE__FBC_DPCM8_RGB_EN__SHIFT 0x00000009
+#define FBC_COMP_MODE__FBC_DPCM8_YUV_EN_MASK 0x00000800L
+#define FBC_COMP_MODE__FBC_DPCM8_YUV_EN__SHIFT 0x0000000b
+#define FBC_COMP_MODE__FBC_IND_EN_MASK 0x00010000L
+#define FBC_COMP_MODE__FBC_IND_EN__SHIFT 0x00000010
+#define FBC_COMP_MODE__FBC_RLE_EN_MASK 0x00000001L
+#define FBC_COMP_MODE__FBC_RLE_EN__SHIFT 0x00000000
+#define FBC_CSM_REGION_OFFSET_01__FBC_CSM_REGION_OFFSET_0_MASK 0x000003ffL
+#define FBC_CSM_REGION_OFFSET_01__FBC_CSM_REGION_OFFSET_0__SHIFT 0x00000000
+#define FBC_CSM_REGION_OFFSET_01__FBC_CSM_REGION_OFFSET_1_MASK 0x03ff0000L
+#define FBC_CSM_REGION_OFFSET_01__FBC_CSM_REGION_OFFSET_1__SHIFT 0x00000010
+#define FBC_CSM_REGION_OFFSET_23__FBC_CSM_REGION_OFFSET_2_MASK 0x000003ffL
+#define FBC_CSM_REGION_OFFSET_23__FBC_CSM_REGION_OFFSET_2__SHIFT 0x00000000
+#define FBC_CSM_REGION_OFFSET_23__FBC_CSM_REGION_OFFSET_3_MASK 0x03ff0000L
+#define FBC_CSM_REGION_OFFSET_23__FBC_CSM_REGION_OFFSET_3__SHIFT 0x00000010
+#define FBC_DEBUG0__FBC_COMP_WAKE_DIS_MASK 0x00010000L
+#define FBC_DEBUG0__FBC_COMP_WAKE_DIS__SHIFT 0x00000010
+#define FBC_DEBUG0__FBC_DEBUG0_MASK 0x00fe0000L
+#define FBC_DEBUG0__FBC_DEBUG0__SHIFT 0x00000011
+#define FBC_DEBUG0__FBC_DEBUG_MUX_MASK 0xff000000L
+#define FBC_DEBUG0__FBC_DEBUG_MUX__SHIFT 0x00000018
+#define FBC_DEBUG0__FBC_PERF_MUX0_MASK 0x000000ffL
+#define FBC_DEBUG0__FBC_PERF_MUX0__SHIFT 0x00000000
+#define FBC_DEBUG0__FBC_PERF_MUX1_MASK 0x0000ff00L
+#define FBC_DEBUG0__FBC_PERF_MUX1__SHIFT 0x00000008
+#define FBC_DEBUG1__FBC_DEBUG1_MASK 0xffffffffL
+#define FBC_DEBUG1__FBC_DEBUG1__SHIFT 0x00000000
+#define FBC_DEBUG2__FBC_DEBUG2_MASK 0xffffffffL
+#define FBC_DEBUG2__FBC_DEBUG2__SHIFT 0x00000000
+#define FBC_DEBUG_COMP__FBC_COMP_ADDRESS_TRANSLATION_ENABLE_MASK 0x00000800L
+#define FBC_DEBUG_COMP__FBC_COMP_ADDRESS_TRANSLATION_ENABLE__SHIFT 0x0000000b
+#define FBC_DEBUG_COMP__FBC_COMP_BUSY_HYSTERESIS_MASK 0x000000f0L
+#define FBC_DEBUG_COMP__FBC_COMP_BUSY_HYSTERESIS__SHIFT 0x00000004
+#define FBC_DEBUG_COMP__FBC_COMP_CLK_CNTL_MASK 0x00000300L
+#define FBC_DEBUG_COMP__FBC_COMP_CLK_CNTL__SHIFT 0x00000008
+#define FBC_DEBUG_COMP__FBC_COMP_PRIVILEGED_ACCESS_ENABLE_MASK 0x00000400L
+#define FBC_DEBUG_COMP__FBC_COMP_PRIVILEGED_ACCESS_ENABLE__SHIFT 0x0000000a
+#define FBC_DEBUG_COMP__FBC_COMP_RSIZE_MASK 0x00000008L
+#define FBC_DEBUG_COMP__FBC_COMP_RSIZE__SHIFT 0x00000003
+#define FBC_DEBUG_COMP__FBC_COMP_SWAP_MASK 0x00000003L
+#define FBC_DEBUG_COMP__FBC_COMP_SWAP__SHIFT 0x00000000
+#define FBC_DEBUG_CSR__FBC_DEBUG_CSR_ADDR_MASK 0x000003ffL
+#define FBC_DEBUG_CSR__FBC_DEBUG_CSR_ADDR__SHIFT 0x00000000
+#define FBC_DEBUG_CSR__FBC_DEBUG_CSR_EN_MASK 0x80000000L
+#define FBC_DEBUG_CSR__FBC_DEBUG_CSR_EN__SHIFT 0x0000001f
+#define FBC_DEBUG_CSR__FBC_DEBUG_CSR_RD_DATA_MASK 0x00020000L
+#define FBC_DEBUG_CSR__FBC_DEBUG_CSR_RD_DATA__SHIFT 0x00000011
+#define FBC_DEBUG_CSR__FBC_DEBUG_CSR_WR_DATA_MASK 0x00010000L
+#define FBC_DEBUG_CSR__FBC_DEBUG_CSR_WR_DATA__SHIFT 0x00000010
+#define FBC_DEBUG_CSR_RDATA__FBC_DEBUG_CSR_RDATA_MASK 0xffffffffL
+#define FBC_DEBUG_CSR_RDATA__FBC_DEBUG_CSR_RDATA__SHIFT 0x00000000
+#define FBC_DEBUG_CSR_RDATA_HI__FBC_DEBUG_CSR_RDATA_HI_MASK 0x000000ffL
+#define FBC_DEBUG_CSR_RDATA_HI__FBC_DEBUG_CSR_RDATA_HI__SHIFT 0x00000000
+#define FBC_DEBUG_CSR_WDATA__FBC_DEBUG_CSR_WDATA_MASK 0xffffffffL
+#define FBC_DEBUG_CSR_WDATA__FBC_DEBUG_CSR_WDATA__SHIFT 0x00000000
+#define FBC_DEBUG_CSR_WDATA_HI__FBC_DEBUG_CSR_WDATA_HI_MASK 0x000000ffL
+#define FBC_DEBUG_CSR_WDATA_HI__FBC_DEBUG_CSR_WDATA_HI__SHIFT 0x00000000
+#define FBC_IDLE_FORCE_CLEAR_MASK__FBC_IDLE_FORCE_CLEAR_MASK_MASK 0xffffffffL
+#define FBC_IDLE_FORCE_CLEAR_MASK__FBC_IDLE_FORCE_CLEAR_MASK__SHIFT 0x00000000
+#define FBC_IDLE_MASK__FBC_IDLE_MASK_MASK 0xffffffffL
+#define FBC_IDLE_MASK__FBC_IDLE_MASK__SHIFT 0x00000000
+#define FBC_IND_LUT0__FBC_IND_LUT0_MASK 0x00ffffffL
+#define FBC_IND_LUT0__FBC_IND_LUT0__SHIFT 0x00000000
+#define FBC_IND_LUT10__FBC_IND_LUT10_MASK 0x00ffffffL
+#define FBC_IND_LUT10__FBC_IND_LUT10__SHIFT 0x00000000
+#define FBC_IND_LUT11__FBC_IND_LUT11_MASK 0x00ffffffL
+#define FBC_IND_LUT11__FBC_IND_LUT11__SHIFT 0x00000000
+#define FBC_IND_LUT12__FBC_IND_LUT12_MASK 0x00ffffffL
+#define FBC_IND_LUT12__FBC_IND_LUT12__SHIFT 0x00000000
+#define FBC_IND_LUT13__FBC_IND_LUT13_MASK 0x00ffffffL
+#define FBC_IND_LUT13__FBC_IND_LUT13__SHIFT 0x00000000
+#define FBC_IND_LUT14__FBC_IND_LUT14_MASK 0x00ffffffL
+#define FBC_IND_LUT14__FBC_IND_LUT14__SHIFT 0x00000000
+#define FBC_IND_LUT15__FBC_IND_LUT15_MASK 0x00ffffffL
+#define FBC_IND_LUT15__FBC_IND_LUT15__SHIFT 0x00000000
+#define FBC_IND_LUT1__FBC_IND_LUT1_MASK 0x00ffffffL
+#define FBC_IND_LUT1__FBC_IND_LUT1__SHIFT 0x00000000
+#define FBC_IND_LUT2__FBC_IND_LUT2_MASK 0x00ffffffL
+#define FBC_IND_LUT2__FBC_IND_LUT2__SHIFT 0x00000000
+#define FBC_IND_LUT3__FBC_IND_LUT3_MASK 0x00ffffffL
+#define FBC_IND_LUT3__FBC_IND_LUT3__SHIFT 0x00000000
+#define FBC_IND_LUT4__FBC_IND_LUT4_MASK 0x00ffffffL
+#define FBC_IND_LUT4__FBC_IND_LUT4__SHIFT 0x00000000
+#define FBC_IND_LUT5__FBC_IND_LUT5_MASK 0x00ffffffL
+#define FBC_IND_LUT5__FBC_IND_LUT5__SHIFT 0x00000000
+#define FBC_IND_LUT6__FBC_IND_LUT6_MASK 0x00ffffffL
+#define FBC_IND_LUT6__FBC_IND_LUT6__SHIFT 0x00000000
+#define FBC_IND_LUT7__FBC_IND_LUT7_MASK 0x00ffffffL
+#define FBC_IND_LUT7__FBC_IND_LUT7__SHIFT 0x00000000
+#define FBC_IND_LUT8__FBC_IND_LUT8_MASK 0x00ffffffL
+#define FBC_IND_LUT8__FBC_IND_LUT8__SHIFT 0x00000000
+#define FBC_IND_LUT9__FBC_IND_LUT9_MASK 0x00ffffffL
+#define FBC_IND_LUT9__FBC_IND_LUT9__SHIFT 0x00000000
+#define FBC_MISC__FBC_DECOMPRESS_ERROR_CLEAR_MASK 0x00010000L
+#define FBC_MISC__FBC_DECOMPRESS_ERROR_CLEAR__SHIFT 0x00000010
+#define FBC_MISC__FBC_DECOMPRESS_ERROR_MASK 0x00000003L
+#define FBC_MISC__FBC_DECOMPRESS_ERROR__SHIFT 0x00000000
+#define FBC_MISC__FBC_DIVIDE_X_MASK 0x00000300L
+#define FBC_MISC__FBC_DIVIDE_X__SHIFT 0x00000008
+#define FBC_MISC__FBC_DIVIDE_Y_MASK 0x00000400L
+#define FBC_MISC__FBC_DIVIDE_Y__SHIFT 0x0000000a
+#define FBC_MISC__FBC_ERROR_PIXEL_MASK 0x000000f0L
+#define FBC_MISC__FBC_ERROR_PIXEL__SHIFT 0x00000004
+#define FBC_MISC__FBC_INVALIDATE_ON_ERROR_MASK 0x00000008L
+#define FBC_MISC__FBC_INVALIDATE_ON_ERROR__SHIFT 0x00000003
+#define FBC_MISC__FBC_RESET_AT_DISABLE_MASK 0x00200000L
+#define FBC_MISC__FBC_RESET_AT_DISABLE__SHIFT 0x00000015
+#define FBC_MISC__FBC_RESET_AT_ENABLE_MASK 0x00100000L
+#define FBC_MISC__FBC_RESET_AT_ENABLE__SHIFT 0x00000014
+#define FBC_MISC__FBC_RSM_UNCOMP_DATA_IMMEDIATELY_MASK 0x00001000L
+#define FBC_MISC__FBC_RSM_UNCOMP_DATA_IMMEDIATELY__SHIFT 0x0000000c
+#define FBC_MISC__FBC_RSM_WRITE_VALUE_MASK 0x00000800L
+#define FBC_MISC__FBC_RSM_WRITE_VALUE__SHIFT 0x0000000b
+#define FBC_MISC__FBC_SLOW_REQ_INTERVAL_MASK 0xf0000000L
+#define FBC_MISC__FBC_SLOW_REQ_INTERVAL__SHIFT 0x0000001c
+#define FBC_MISC__FBC_STOP_ON_ERROR_MASK 0x00000004L
+#define FBC_MISC__FBC_STOP_ON_ERROR__SHIFT 0x00000002
+#define FBC_START_STOP_DELAY__FBC_COMP_START_DELAY_MASK 0x00001f00L
+#define FBC_START_STOP_DELAY__FBC_COMP_START_DELAY__SHIFT 0x00000008
+#define FBC_START_STOP_DELAY__FBC_DECOMP_START_DELAY_MASK 0x0000001fL
+#define FBC_START_STOP_DELAY__FBC_DECOMP_START_DELAY__SHIFT 0x00000000
+#define FBC_START_STOP_DELAY__FBC_DECOMP_STOP_DELAY_MASK 0x00000080L
+#define FBC_START_STOP_DELAY__FBC_DECOMP_STOP_DELAY__SHIFT 0x00000007
+#define FBC_STATUS__FBC_ENABLE_STATUS_MASK 0x00000001L
+#define FBC_STATUS__FBC_ENABLE_STATUS__SHIFT 0x00000000
+#define FBC_TEST_DEBUG_DATA__FBC_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define FBC_TEST_DEBUG_DATA__FBC_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define FBC_TEST_DEBUG_INDEX__FBC_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define FBC_TEST_DEBUG_INDEX__FBC_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define FBC_TEST_DEBUG_INDEX__FBC_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define FBC_TEST_DEBUG_INDEX__FBC_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define FMT_BIT_DEPTH_CONTROL__FMT_25FRC_SEL_MASK 0x0c000000L
+#define FMT_BIT_DEPTH_CONTROL__FMT_25FRC_SEL__SHIFT 0x0000001a
+#define FMT_BIT_DEPTH_CONTROL__FMT_50FRC_SEL_MASK 0x30000000L
+#define FMT_BIT_DEPTH_CONTROL__FMT_50FRC_SEL__SHIFT 0x0000001c
+#define FMT_BIT_DEPTH_CONTROL__FMT_75FRC_SEL_MASK 0xc0000000L
+#define FMT_BIT_DEPTH_CONTROL__FMT_75FRC_SEL__SHIFT 0x0000001e
+#define FMT_BIT_DEPTH_CONTROL__FMT_FRAME_RANDOM_ENABLE_MASK 0x00002000L
+#define FMT_BIT_DEPTH_CONTROL__FMT_FRAME_RANDOM_ENABLE__SHIFT 0x0000000d
+#define FMT_BIT_DEPTH_CONTROL__FMT_HIGHPASS_RANDOM_ENABLE_MASK 0x00008000L
+#define FMT_BIT_DEPTH_CONTROL__FMT_HIGHPASS_RANDOM_ENABLE__SHIFT 0x0000000f
+#define FMT_BIT_DEPTH_CONTROL__FMT_RGB_RANDOM_ENABLE_MASK 0x00004000L
+#define FMT_BIT_DEPTH_CONTROL__FMT_RGB_RANDOM_ENABLE__SHIFT 0x0000000e
+#define FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_DEPTH_MASK 0x00001000L
+#define FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_DEPTH__SHIFT 0x0000000c
+#define FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_EN_MASK 0x00000100L
+#define FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_EN__SHIFT 0x00000008
+#define FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_MODE_MASK 0x00000600L
+#define FMT_BIT_DEPTH_CONTROL__FMT_SPATIAL_DITHER_MODE__SHIFT 0x00000009
+#define FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_DEPTH_MASK 0x00100000L
+#define FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_DEPTH__SHIFT 0x00000014
+#define FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_EN_MASK 0x00010000L
+#define FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_EN__SHIFT 0x00000010
+#define FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_OFFSET_MASK 0x00600000L
+#define FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_OFFSET__SHIFT 0x00000015
+#define FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_RESET_MASK 0x02000000L
+#define FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_DITHER_RESET__SHIFT 0x00000019
+#define FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_LEVEL_MASK 0x01000000L
+#define FMT_BIT_DEPTH_CONTROL__FMT_TEMPORAL_LEVEL__SHIFT 0x00000018
+#define FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_DEPTH_MASK 0x00000010L
+#define FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_DEPTH__SHIFT 0x00000004
+#define FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_EN_MASK 0x00000001L
+#define FMT_BIT_DEPTH_CONTROL__FMT_TRUNCATE_EN__SHIFT 0x00000000
+#define FMT_CLAMP_CNTL__FMT_CLAMP_COLOR_FORMAT_MASK 0x00070000L
+#define FMT_CLAMP_CNTL__FMT_CLAMP_COLOR_FORMAT__SHIFT 0x00000010
+#define FMT_CLAMP_CNTL__FMT_CLAMP_DATA_EN_MASK 0x00000001L
+#define FMT_CLAMP_CNTL__FMT_CLAMP_DATA_EN__SHIFT 0x00000000
+#define FMT_CONTROL__FMT_PIXEL_ENCODING_MASK 0x00010000L
+#define FMT_CONTROL__FMT_PIXEL_ENCODING__SHIFT 0x00000010
+#define FMT_CONTROL__FMT_STEREOSYNC_OVERRIDE_MASK 0x00000001L
+#define FMT_CONTROL__FMT_STEREOSYNC_OVERRIDE__SHIFT 0x00000000
+#define FMT_CONTROL__FMT_STEREOSYNC_OVR_POL_MASK 0x00000010L
+#define FMT_CONTROL__FMT_STEREOSYNC_OVR_POL__SHIFT 0x00000004
+#define FMT_CRC_CNTL__FMT_CRC_CONT_EN_MASK 0x00000010L
+#define FMT_CRC_CNTL__FMT_CRC_CONT_EN__SHIFT 0x00000004
+#define FMT_CRC_CNTL__FMT_CRC_EN_MASK 0x00000001L
+#define FMT_CRC_CNTL__FMT_CRC_EN__SHIFT 0x00000000
+#define FMT_CRC_CNTL__FMT_CRC_EVEN_ODD_PIX_ENABLE_MASK 0x00100000L
+#define FMT_CRC_CNTL__FMT_CRC_EVEN_ODD_PIX_ENABLE__SHIFT 0x00000014
+#define FMT_CRC_CNTL__FMT_CRC_EVEN_ODD_PIX_SELECT_MASK 0x01000000L
+#define FMT_CRC_CNTL__FMT_CRC_EVEN_ODD_PIX_SELECT__SHIFT 0x00000018
+#define FMT_CRC_CNTL__FMT_CRC_INTERLACE_MODE_MASK 0x00003000L
+#define FMT_CRC_CNTL__FMT_CRC_INTERLACE_MODE__SHIFT 0x0000000c
+#define FMT_CRC_CNTL__FMT_CRC_ONLY_BLANKb_MASK 0x00000100L
+#define FMT_CRC_CNTL__FMT_CRC_ONLY_BLANKb__SHIFT 0x00000008
+#define FMT_CRC_CNTL__FMT_CRC_USE_NEW_AND_REPEATED_PIXELS_MASK 0x00010000L
+#define FMT_CRC_CNTL__FMT_CRC_USE_NEW_AND_REPEATED_PIXELS__SHIFT 0x00000010
+#define FMT_CRC_SIG_BLUE_CONTROL__FMT_CRC_SIG_BLUE_MASK 0x0000ffffL
+#define FMT_CRC_SIG_BLUE_CONTROL__FMT_CRC_SIG_BLUE__SHIFT 0x00000000
+#define FMT_CRC_SIG_BLUE_CONTROL__FMT_CRC_SIG_CONTROL_MASK 0xffff0000L
+#define FMT_CRC_SIG_BLUE_CONTROL__FMT_CRC_SIG_CONTROL__SHIFT 0x00000010
+#define FMT_CRC_SIG_BLUE_CONTROL_MASK__FMT_CRC_SIG_BLUE_MASK_MASK 0x0000ffffL
+#define FMT_CRC_SIG_BLUE_CONTROL_MASK__FMT_CRC_SIG_BLUE_MASK__SHIFT 0x00000000
+#define FMT_CRC_SIG_BLUE_CONTROL_MASK__FMT_CRC_SIG_CONTROL_MASK_MASK 0xffff0000L
+#define FMT_CRC_SIG_BLUE_CONTROL_MASK__FMT_CRC_SIG_CONTROL_MASK__SHIFT 0x00000010
+#define FMT_CRC_SIG_RED_GREEN__FMT_CRC_SIG_GREEN_MASK 0xffff0000L
+#define FMT_CRC_SIG_RED_GREEN__FMT_CRC_SIG_GREEN__SHIFT 0x00000010
+#define FMT_CRC_SIG_RED_GREEN__FMT_CRC_SIG_RED_MASK 0x0000ffffL
+#define FMT_CRC_SIG_RED_GREEN__FMT_CRC_SIG_RED__SHIFT 0x00000000
+#define FMT_CRC_SIG_RED_GREEN_MASK__FMT_CRC_SIG_GREEN_MASK_MASK 0xffff0000L
+#define FMT_CRC_SIG_RED_GREEN_MASK__FMT_CRC_SIG_GREEN_MASK__SHIFT 0x00000010
+#define FMT_CRC_SIG_RED_GREEN_MASK__FMT_CRC_SIG_RED_MASK_MASK 0x0000ffffL
+#define FMT_CRC_SIG_RED_GREEN_MASK__FMT_CRC_SIG_RED_MASK__SHIFT 0x00000000
+#define FMT_DEBUG0__FMT_DEBUG0_MASK 0xffffffffL
+#define FMT_DEBUG0__FMT_DEBUG0__SHIFT 0x00000000
+#define FMT_DEBUG1__FMT_DEBUG1_MASK 0xffffffffL
+#define FMT_DEBUG1__FMT_DEBUG1__SHIFT 0x00000000
+#define FMT_DEBUG2__FMT_DEBUG2_MASK 0xffffffffL
+#define FMT_DEBUG2__FMT_DEBUG2__SHIFT 0x00000000
+#define FMT_DEBUG_CNTL__FMT_DEBUG_COLOR_SELECT_MASK 0x00000003L
+#define FMT_DEBUG_CNTL__FMT_DEBUG_COLOR_SELECT__SHIFT 0x00000000
+#define FMT_DEBUG_ID__FMT_DEBUG_ID_MASK 0xffffffffL
+#define FMT_DEBUG_ID__FMT_DEBUG_ID__SHIFT 0x00000000
+#define FMT_DITHER_RAND_B_SEED__FMT_RAND_B_SEED_MASK 0x000000ffL
+#define FMT_DITHER_RAND_B_SEED__FMT_RAND_B_SEED__SHIFT 0x00000000
+#define FMT_DITHER_RAND_G_SEED__FMT_RAND_G_SEED_MASK 0x000000ffL
+#define FMT_DITHER_RAND_G_SEED__FMT_RAND_G_SEED__SHIFT 0x00000000
+#define FMT_DITHER_RAND_R_SEED__FMT_RAND_R_SEED_MASK 0x000000ffL
+#define FMT_DITHER_RAND_R_SEED__FMT_RAND_R_SEED__SHIFT 0x00000000
+#define FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_EN_MASK 0x00000001L
+#define FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_EN__SHIFT 0x00000000
+#define FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_MODE_MASK 0x00000010L
+#define FMT_DYNAMIC_EXP_CNTL__FMT_DYNAMIC_EXP_MODE__SHIFT 0x00000004
+#define FMT_FORCE_DATA_0_1__FMT_FORCE_DATA0_MASK 0x0000ffffL
+#define FMT_FORCE_DATA_0_1__FMT_FORCE_DATA0__SHIFT 0x00000000
+#define FMT_FORCE_DATA_0_1__FMT_FORCE_DATA1_MASK 0xffff0000L
+#define FMT_FORCE_DATA_0_1__FMT_FORCE_DATA1__SHIFT 0x00000010
+#define FMT_FORCE_DATA_2_3__FMT_FORCE_DATA2_MASK 0x0000ffffL
+#define FMT_FORCE_DATA_2_3__FMT_FORCE_DATA2__SHIFT 0x00000000
+#define FMT_FORCE_DATA_2_3__FMT_FORCE_DATA3_MASK 0xffff0000L
+#define FMT_FORCE_DATA_2_3__FMT_FORCE_DATA3__SHIFT 0x00000010
+#define FMT_FORCE_OUTPUT_CNTL__FMT_FORCE_DATA_EN_MASK 0x00000001L
+#define FMT_FORCE_OUTPUT_CNTL__FMT_FORCE_DATA_EN__SHIFT 0x00000000
+#define FMT_FORCE_OUTPUT_CNTL__FMT_FORCE_DATA_ON_BLANKb_ONLY_MASK 0x00010000L
+#define FMT_FORCE_OUTPUT_CNTL__FMT_FORCE_DATA_ON_BLANKb_ONLY__SHIFT 0x00000010
+#define FMT_FORCE_OUTPUT_CNTL__FMT_FORCE_DATA_SEL_COLOR_MASK 0x00000700L
+#define FMT_FORCE_OUTPUT_CNTL__FMT_FORCE_DATA_SEL_COLOR__SHIFT 0x00000008
+#define FMT_FORCE_OUTPUT_CNTL__FMT_FORCE_DATA_SEL_SLOT_MASK 0x0000f000L
+#define FMT_FORCE_OUTPUT_CNTL__FMT_FORCE_DATA_SEL_SLOT__SHIFT 0x0000000c
+#define FMT_TEMPORAL_DITHER_PATTERN_CONTROL__FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_RGB1_BGR0_MASK 0x00000010L
+#define FMT_TEMPORAL_DITHER_PATTERN_CONTROL__FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_RGB1_BGR0__SHIFT 0x00000004
+#define FMT_TEMPORAL_DITHER_PATTERN_CONTROL__FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_SELECT_MASK 0x00000001L
+#define FMT_TEMPORAL_DITHER_PATTERN_CONTROL__FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_SELECT__SHIFT 0x00000000
+#define FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_S_MATRIX__FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_S_MATRIX_MASK 0xffffffffL
+#define FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_S_MATRIX__FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_S_MATRIX__SHIFT 0x00000000
+#define FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_T_MATRIX__FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_T_MATRIX_MASK 0xffffffffL
+#define FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_T_MATRIX__FMT_TEMPORAL_DITHER_PROGRAMMABLE_PATTERN_T_MATRIX__SHIFT 0x00000000
+#define FMT_TEST_DEBUG_DATA__FMT_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define FMT_TEST_DEBUG_DATA__FMT_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define FMT_TEST_DEBUG_INDEX__FMT_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define FMT_TEST_DEBUG_INDEX__FMT_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define FMT_TEST_DEBUG_INDEX__FMT_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define FMT_TEST_DEBUG_INDEX__FMT_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define GAMUT_REMAP_C11_C12__GAMUT_REMAP_C11_MASK 0x0000ffffL
+#define GAMUT_REMAP_C11_C12__GAMUT_REMAP_C11__SHIFT 0x00000000
+#define GAMUT_REMAP_C11_C12__GAMUT_REMAP_C12_MASK 0xffff0000L
+#define GAMUT_REMAP_C11_C12__GAMUT_REMAP_C12__SHIFT 0x00000010
+#define GAMUT_REMAP_C13_C14__GAMUT_REMAP_C13_MASK 0x0000ffffL
+#define GAMUT_REMAP_C13_C14__GAMUT_REMAP_C13__SHIFT 0x00000000
+#define GAMUT_REMAP_C13_C14__GAMUT_REMAP_C14_MASK 0xffff0000L
+#define GAMUT_REMAP_C13_C14__GAMUT_REMAP_C14__SHIFT 0x00000010
+#define GAMUT_REMAP_C21_C22__GAMUT_REMAP_C21_MASK 0x0000ffffL
+#define GAMUT_REMAP_C21_C22__GAMUT_REMAP_C21__SHIFT 0x00000000
+#define GAMUT_REMAP_C21_C22__GAMUT_REMAP_C22_MASK 0xffff0000L
+#define GAMUT_REMAP_C21_C22__GAMUT_REMAP_C22__SHIFT 0x00000010
+#define GAMUT_REMAP_C23_C24__GAMUT_REMAP_C23_MASK 0x0000ffffL
+#define GAMUT_REMAP_C23_C24__GAMUT_REMAP_C23__SHIFT 0x00000000
+#define GAMUT_REMAP_C23_C24__GAMUT_REMAP_C24_MASK 0xffff0000L
+#define GAMUT_REMAP_C23_C24__GAMUT_REMAP_C24__SHIFT 0x00000010
+#define GAMUT_REMAP_C31_C32__GAMUT_REMAP_C31_MASK 0x0000ffffL
+#define GAMUT_REMAP_C31_C32__GAMUT_REMAP_C31__SHIFT 0x00000000
+#define GAMUT_REMAP_C31_C32__GAMUT_REMAP_C32_MASK 0xffff0000L
+#define GAMUT_REMAP_C31_C32__GAMUT_REMAP_C32__SHIFT 0x00000010
+#define GAMUT_REMAP_C33_C34__GAMUT_REMAP_C33_MASK 0x0000ffffL
+#define GAMUT_REMAP_C33_C34__GAMUT_REMAP_C33__SHIFT 0x00000000
+#define GAMUT_REMAP_C33_C34__GAMUT_REMAP_C34_MASK 0xffff0000L
+#define GAMUT_REMAP_C33_C34__GAMUT_REMAP_C34__SHIFT 0x00000010
+#define GAMUT_REMAP_CONTROL__GRPH_GAMUT_REMAP_MODE_MASK 0x00000003L
+#define GAMUT_REMAP_CONTROL__GRPH_GAMUT_REMAP_MODE__SHIFT 0x00000000
+#define GAMUT_REMAP_CONTROL__OVL_GAMUT_REMAP_MODE_MASK 0x00000030L
+#define GAMUT_REMAP_CONTROL__OVL_GAMUT_REMAP_MODE__SHIFT 0x00000004
+#define GENENB__BLK_IO_BASE_MASK 0x000000ffL
+#define GENENB__BLK_IO_BASE__SHIFT 0x00000000
+#define GENERIC_I2C_CONTROL__GENERIC_I2C_DBG_REF_SEL_MASK 0x80000000L
+#define GENERIC_I2C_CONTROL__GENERIC_I2C_DBG_REF_SEL__SHIFT 0x0000001f
+#define GENERIC_I2C_CONTROL__GENERIC_I2C_ENABLE_MASK 0x00000008L
+#define GENERIC_I2C_CONTROL__GENERIC_I2C_ENABLE__SHIFT 0x00000003
+#define GENERIC_I2C_CONTROL__GENERIC_I2C_GO_MASK 0x00000001L
+#define GENERIC_I2C_CONTROL__GENERIC_I2C_GO__SHIFT 0x00000000
+#define GENERIC_I2C_CONTROL__GENERIC_I2C_SEND_RESET_MASK 0x00000004L
+#define GENERIC_I2C_CONTROL__GENERIC_I2C_SEND_RESET__SHIFT 0x00000002
+#define GENERIC_I2C_CONTROL__GENERIC_I2C_SOFT_RESET_MASK 0x00000002L
+#define GENERIC_I2C_CONTROL__GENERIC_I2C_SOFT_RESET__SHIFT 0x00000001
+#define GENERIC_I2C_DATA__GENERIC_I2C_DATA_MASK 0x0000ff00L
+#define GENERIC_I2C_DATA__GENERIC_I2C_DATA_RW_MASK 0x00000001L
+#define GENERIC_I2C_DATA__GENERIC_I2C_DATA_RW__SHIFT 0x00000000
+#define GENERIC_I2C_DATA__GENERIC_I2C_DATA__SHIFT 0x00000008
+#define GENERIC_I2C_DATA__GENERIC_I2C_INDEX_MASK 0x000f0000L
+#define GENERIC_I2C_DATA__GENERIC_I2C_INDEX__SHIFT 0x00000010
+#define GENERIC_I2C_DATA__GENERIC_I2C_INDEX_WRITE_MASK 0x80000000L
+#define GENERIC_I2C_DATA__GENERIC_I2C_INDEX_WRITE__SHIFT 0x0000001f
+#define GENERIC_I2C_INTERRUPT_CONTROL__GENERIC_I2C_DONE_ACK_MASK 0x00000002L
+#define GENERIC_I2C_INTERRUPT_CONTROL__GENERIC_I2C_DONE_ACK__SHIFT 0x00000001
+#define GENERIC_I2C_INTERRUPT_CONTROL__GENERIC_I2C_DONE_INT_MASK 0x00000001L
+#define GENERIC_I2C_INTERRUPT_CONTROL__GENERIC_I2C_DONE_INT__SHIFT 0x00000000
+#define GENERIC_I2C_INTERRUPT_CONTROL__GENERIC_I2C_DONE_MASK_MASK 0x00000004L
+#define GENERIC_I2C_INTERRUPT_CONTROL__GENERIC_I2C_DONE_MASK__SHIFT 0x00000002
+#define GENERIC_I2C_PIN_DEBUG__GENERIC_I2C_SCL_EN_MASK 0x00000004L
+#define GENERIC_I2C_PIN_DEBUG__GENERIC_I2C_SCL_EN__SHIFT 0x00000002
+#define GENERIC_I2C_PIN_DEBUG__GENERIC_I2C_SCL_INPUT_MASK 0x00000002L
+#define GENERIC_I2C_PIN_DEBUG__GENERIC_I2C_SCL_INPUT__SHIFT 0x00000001
+#define GENERIC_I2C_PIN_DEBUG__GENERIC_I2C_SCL_OUTPUT_MASK 0x00000001L
+#define GENERIC_I2C_PIN_DEBUG__GENERIC_I2C_SCL_OUTPUT__SHIFT 0x00000000
+#define GENERIC_I2C_PIN_DEBUG__GENERIC_I2C_SDA_EN_MASK 0x00000040L
+#define GENERIC_I2C_PIN_DEBUG__GENERIC_I2C_SDA_EN__SHIFT 0x00000006
+#define GENERIC_I2C_PIN_DEBUG__GENERIC_I2C_SDA_INPUT_MASK 0x00000020L
+#define GENERIC_I2C_PIN_DEBUG__GENERIC_I2C_SDA_INPUT__SHIFT 0x00000005
+#define GENERIC_I2C_PIN_DEBUG__GENERIC_I2C_SDA_OUTPUT_MASK 0x00000010L
+#define GENERIC_I2C_PIN_DEBUG__GENERIC_I2C_SDA_OUTPUT__SHIFT 0x00000004
+#define GENERIC_I2C_PIN_SELECTION__GENERIC_I2C_SCL_PIN_SEL_MASK 0x0000007fL
+#define GENERIC_I2C_PIN_SELECTION__GENERIC_I2C_SCL_PIN_SEL__SHIFT 0x00000000
+#define GENERIC_I2C_PIN_SELECTION__GENERIC_I2C_SDA_PIN_SEL_MASK 0x00007f00L
+#define GENERIC_I2C_PIN_SELECTION__GENERIC_I2C_SDA_PIN_SEL__SHIFT 0x00000008
+#define GENERIC_I2C_SETUP__GENERIC_I2C_CLK_DRIVE_EN_MASK 0x00000080L
+#define GENERIC_I2C_SETUP__GENERIC_I2C_CLK_DRIVE_EN__SHIFT 0x00000007
+#define GENERIC_I2C_SETUP__GENERIC_I2C_DATA_DRIVE_EN_MASK 0x00000001L
+#define GENERIC_I2C_SETUP__GENERIC_I2C_DATA_DRIVE_EN__SHIFT 0x00000000
+#define GENERIC_I2C_SETUP__GENERIC_I2C_DATA_DRIVE_SEL_MASK 0x00000002L
+#define GENERIC_I2C_SETUP__GENERIC_I2C_DATA_DRIVE_SEL__SHIFT 0x00000001
+#define GENERIC_I2C_SETUP__GENERIC_I2C_INTRA_BYTE_DELAY_MASK 0x0000ff00L
+#define GENERIC_I2C_SETUP__GENERIC_I2C_INTRA_BYTE_DELAY__SHIFT 0x00000008
+#define GENERIC_I2C_SETUP__GENERIC_I2C_TIME_LIMIT_MASK 0xff000000L
+#define GENERIC_I2C_SETUP__GENERIC_I2C_TIME_LIMIT__SHIFT 0x00000018
+#define GENERIC_I2C_SPEED__GENERIC_I2C_DISABLE_FILTER_DURING_STALL_MASK 0x00000010L
+#define GENERIC_I2C_SPEED__GENERIC_I2C_DISABLE_FILTER_DURING_STALL__SHIFT 0x00000004
+#define GENERIC_I2C_SPEED__GENERIC_I2C_PRESCALE_MASK 0xffff0000L
+#define GENERIC_I2C_SPEED__GENERIC_I2C_PRESCALE__SHIFT 0x00000010
+#define GENERIC_I2C_SPEED__GENERIC_I2C_THRESHOLD_MASK 0x00000003L
+#define GENERIC_I2C_SPEED__GENERIC_I2C_THRESHOLD__SHIFT 0x00000000
+#define GENERIC_I2C_STATUS__GENERIC_I2C_ABORTED_MASK 0x00000020L
+#define GENERIC_I2C_STATUS__GENERIC_I2C_ABORTED__SHIFT 0x00000005
+#define GENERIC_I2C_STATUS__GENERIC_I2C_DONE_MASK 0x00000010L
+#define GENERIC_I2C_STATUS__GENERIC_I2C_DONE__SHIFT 0x00000004
+#define GENERIC_I2C_STATUS__GENERIC_I2C_NACK_MASK 0x00000400L
+#define GENERIC_I2C_STATUS__GENERIC_I2C_NACK__SHIFT 0x0000000a
+#define GENERIC_I2C_STATUS__GENERIC_I2C_STATUS_MASK 0x0000000fL
+#define GENERIC_I2C_STATUS__GENERIC_I2C_STATUS__SHIFT 0x00000000
+#define GENERIC_I2C_STATUS__GENERIC_I2C_STOPPED_ON_NACK_MASK 0x00000200L
+#define GENERIC_I2C_STATUS__GENERIC_I2C_STOPPED_ON_NACK__SHIFT 0x00000009
+#define GENERIC_I2C_STATUS__GENERIC_I2C_TIMEOUT_MASK 0x00000040L
+#define GENERIC_I2C_STATUS__GENERIC_I2C_TIMEOUT__SHIFT 0x00000006
+#define GENERIC_I2C_TRANSACTION__GENERIC_I2C_ACK_ON_READ_MASK 0x00000200L
+#define GENERIC_I2C_TRANSACTION__GENERIC_I2C_ACK_ON_READ__SHIFT 0x00000009
+#define GENERIC_I2C_TRANSACTION__GENERIC_I2C_COUNT_MASK 0x000f0000L
+#define GENERIC_I2C_TRANSACTION__GENERIC_I2C_COUNT__SHIFT 0x00000010
+#define GENERIC_I2C_TRANSACTION__GENERIC_I2C_RW_MASK 0x00000001L
+#define GENERIC_I2C_TRANSACTION__GENERIC_I2C_RW__SHIFT 0x00000000
+#define GENERIC_I2C_TRANSACTION__GENERIC_I2C_START_MASK 0x00001000L
+#define GENERIC_I2C_TRANSACTION__GENERIC_I2C_START__SHIFT 0x0000000c
+#define GENERIC_I2C_TRANSACTION__GENERIC_I2C_STOP_MASK 0x00002000L
+#define GENERIC_I2C_TRANSACTION__GENERIC_I2C_STOP_ON_NACK_MASK 0x00000100L
+#define GENERIC_I2C_TRANSACTION__GENERIC_I2C_STOP_ON_NACK__SHIFT 0x00000008
+#define GENERIC_I2C_TRANSACTION__GENERIC_I2C_STOP__SHIFT 0x0000000d
+#define GENFC_RD__VSYNC_SEL_R_MASK 0x00000008L
+#define GENFC_RD__VSYNC_SEL_R__SHIFT 0x00000003
+#define GENFC_WT__VSYNC_SEL_W_MASK 0x00000008L
+#define GENFC_WT__VSYNC_SEL_W__SHIFT 0x00000003
+#define GENMO_RD__GENMO_MONO_ADDRESS_B_MASK 0x00000001L
+#define GENMO_RD__GENMO_MONO_ADDRESS_B__SHIFT 0x00000000
+#define GENMO_RD__ODD_EVEN_MD_PGSEL_MASK 0x00000020L
+#define GENMO_RD__ODD_EVEN_MD_PGSEL__SHIFT 0x00000005
+#define GENMO_RD__VGA_CKSEL_MASK 0x0000000cL
+#define GENMO_RD__VGA_CKSEL__SHIFT 0x00000002
+#define GENMO_RD__VGA_HSYNC_POL_MASK 0x00000040L
+#define GENMO_RD__VGA_HSYNC_POL__SHIFT 0x00000006
+#define GENMO_RD__VGA_RAM_EN_MASK 0x00000002L
+#define GENMO_RD__VGA_RAM_EN__SHIFT 0x00000001
+#define GENMO_RD__VGA_VSYNC_POL_MASK 0x00000080L
+#define GENMO_RD__VGA_VSYNC_POL__SHIFT 0x00000007
+#define GENMO_WT__GENMO_MONO_ADDRESS_B_MASK 0x00000001L
+#define GENMO_WT__GENMO_MONO_ADDRESS_B__SHIFT 0x00000000
+#define GENMO_WT__ODD_EVEN_MD_PGSEL_MASK 0x00000020L
+#define GENMO_WT__ODD_EVEN_MD_PGSEL__SHIFT 0x00000005
+#define GENMO_WT__VGA_CKSEL_MASK 0x0000000cL
+#define GENMO_WT__VGA_CKSEL__SHIFT 0x00000002
+#define GENMO_WT__VGA_HSYNC_POL_MASK 0x00000040L
+#define GENMO_WT__VGA_HSYNC_POL__SHIFT 0x00000006
+#define GENMO_WT__VGA_RAM_EN_MASK 0x00000002L
+#define GENMO_WT__VGA_RAM_EN__SHIFT 0x00000001
+#define GENMO_WT__VGA_VSYNC_POL_MASK 0x00000080L
+#define GENMO_WT__VGA_VSYNC_POL__SHIFT 0x00000007
+#define GENS0__CRT_INTR_MASK 0x00000080L
+#define GENS0__CRT_INTR__SHIFT 0x00000007
+#define GENS0__SENSE_SWITCH_MASK 0x00000010L
+#define GENS0__SENSE_SWITCH__SHIFT 0x00000004
+#define GENS1__NO_DISPLAY_MASK 0x00000001L
+#define GENS1__NO_DISPLAY__SHIFT 0x00000000
+#define GENS1__PIXEL_READ_BACK_MASK 0x00000030L
+#define GENS1__PIXEL_READ_BACK__SHIFT 0x00000004
+#define GENS1__VGA_VSTATUS_MASK 0x00000008L
+#define GENS1__VGA_VSTATUS__SHIFT 0x00000003
+#define GRA00__GRPH_SET_RESET0_MASK 0x00000001L
+#define GRA00__GRPH_SET_RESET0__SHIFT 0x00000000
+#define GRA00__GRPH_SET_RESET1_MASK 0x00000002L
+#define GRA00__GRPH_SET_RESET1__SHIFT 0x00000001
+#define GRA00__GRPH_SET_RESET2_MASK 0x00000004L
+#define GRA00__GRPH_SET_RESET2__SHIFT 0x00000002
+#define GRA00__GRPH_SET_RESET3_MASK 0x00000008L
+#define GRA00__GRPH_SET_RESET3__SHIFT 0x00000003
+#define GRA01__GRPH_SET_RESET_ENA0_MASK 0x00000001L
+#define GRA01__GRPH_SET_RESET_ENA0__SHIFT 0x00000000
+#define GRA01__GRPH_SET_RESET_ENA1_MASK 0x00000002L
+#define GRA01__GRPH_SET_RESET_ENA1__SHIFT 0x00000001
+#define GRA01__GRPH_SET_RESET_ENA2_MASK 0x00000004L
+#define GRA01__GRPH_SET_RESET_ENA2__SHIFT 0x00000002
+#define GRA01__GRPH_SET_RESET_ENA3_MASK 0x00000008L
+#define GRA01__GRPH_SET_RESET_ENA3__SHIFT 0x00000003
+#define GRA02__GRPH_CCOMP_MASK 0x0000000fL
+#define GRA02__GRPH_CCOMP__SHIFT 0x00000000
+#define GRA03__GRPH_FN_SEL_MASK 0x00000018L
+#define GRA03__GRPH_FN_SEL__SHIFT 0x00000003
+#define GRA03__GRPH_ROTATE_MASK 0x00000007L
+#define GRA03__GRPH_ROTATE__SHIFT 0x00000000
+#define GRA04__GRPH_RMAP_MASK 0x00000003L
+#define GRA04__GRPH_RMAP__SHIFT 0x00000000
+#define GRA05__CGA_ODDEVEN_MASK 0x00000010L
+#define GRA05__CGA_ODDEVEN__SHIFT 0x00000004
+#define GRA05__GRPH_OES_MASK 0x00000020L
+#define GRA05__GRPH_OES__SHIFT 0x00000005
+#define GRA05__GRPH_PACK_MASK 0x00000040L
+#define GRA05__GRPH_PACK__SHIFT 0x00000006
+#define GRA05__GRPH_READ1_MASK 0x00000008L
+#define GRA05__GRPH_READ1__SHIFT 0x00000003
+#define GRA05__GRPH_WRITE_MODE_MASK 0x00000003L
+#define GRA05__GRPH_WRITE_MODE__SHIFT 0x00000000
+#define GRA06__GRPH_ADRSEL_MASK 0x0000000cL
+#define GRA06__GRPH_ADRSEL__SHIFT 0x00000002
+#define GRA06__GRPH_GRAPHICS_MASK 0x00000001L
+#define GRA06__GRPH_GRAPHICS__SHIFT 0x00000000
+#define GRA06__GRPH_ODDEVEN_MASK 0x00000002L
+#define GRA06__GRPH_ODDEVEN__SHIFT 0x00000001
+#define GRA07__GRPH_XCARE0_MASK 0x00000001L
+#define GRA07__GRPH_XCARE0__SHIFT 0x00000000
+#define GRA07__GRPH_XCARE1_MASK 0x00000002L
+#define GRA07__GRPH_XCARE1__SHIFT 0x00000001
+#define GRA07__GRPH_XCARE2_MASK 0x00000004L
+#define GRA07__GRPH_XCARE2__SHIFT 0x00000002
+#define GRA07__GRPH_XCARE3_MASK 0x00000008L
+#define GRA07__GRPH_XCARE3__SHIFT 0x00000003
+#define GRA08__GRPH_BMSK_MASK 0x000000ffL
+#define GRA08__GRPH_BMSK__SHIFT 0x00000000
+#define GRPH8_DATA__GRPH_DATA_MASK 0x000000ffL
+#define GRPH8_DATA__GRPH_DATA__SHIFT 0x00000000
+#define GRPH8_IDX__GRPH_IDX_MASK 0x0000000fL
+#define GRPH8_IDX__GRPH_IDX__SHIFT 0x00000000
+#define GRPH_COMPRESS_PITCH__GRPH_COMPRESS_PITCH_MASK 0x0001ffc0L
+#define GRPH_COMPRESS_PITCH__GRPH_COMPRESS_PITCH__SHIFT 0x00000006
+#define GRPH_COMPRESS_SURFACE_ADDRESS__GRPH_COMPRESS_SURFACE_ADDRESS_MASK 0xffffff00L
+#define GRPH_COMPRESS_SURFACE_ADDRESS__GRPH_COMPRESS_SURFACE_ADDRESS__SHIFT 0x00000008
+#define GRPH_COMPRESS_SURFACE_ADDRESS_HIGH__GRPH_COMPRESS_SURFACE_ADDRESS_HIGH_MASK 0x000000ffL
+#define GRPH_COMPRESS_SURFACE_ADDRESS_HIGH__GRPH_COMPRESS_SURFACE_ADDRESS_HIGH__SHIFT 0x00000000
+#define GRPH_CONTROL__GRPH_ADDRESS_TRANSLATION_ENABLE_MASK 0x00010000L
+#define GRPH_CONTROL__GRPH_ADDRESS_TRANSLATION_ENABLE__SHIFT 0x00000010
+#define GRPH_CONTROL__GRPH_ARRAY_MODE_MASK 0x00f00000L
+#define GRPH_CONTROL__GRPH_ARRAY_MODE__SHIFT 0x00000014
+#define GRPH_CONTROL__GRPH_BANK_HEIGHT_MASK 0x00001800L
+#define GRPH_CONTROL__GRPH_BANK_HEIGHT__SHIFT 0x0000000b
+#define GRPH_CONTROL__GRPH_BANK_WIDTH_MASK 0x000000c0L
+#define GRPH_CONTROL__GRPH_BANK_WIDTH__SHIFT 0x00000006
+#define GRPH_CONTROL__GRPH_COLOR_EXPANSION_MODE_MASK 0x80000000L
+#define GRPH_CONTROL__GRPH_COLOR_EXPANSION_MODE__SHIFT 0x0000001f
+#define GRPH_CONTROL__GRPH_DEPTH_MASK 0x00000003L
+#define GRPH_CONTROL__GRPH_DEPTH__SHIFT 0x00000000
+#define GRPH_CONTROL__GRPH_FORMAT_MASK 0x00000700L
+#define GRPH_CONTROL__GRPH_FORMAT__SHIFT 0x00000008
+#define GRPH_CONTROL__GRPH_MACRO_TILE_ASPECT_MASK 0x000c0000L
+#define GRPH_CONTROL__GRPH_MACRO_TILE_ASPECT__SHIFT 0x00000012
+#define GRPH_CONTROL__GRPH_NUM_BANKS_MASK 0x0000000cL
+#define GRPH_CONTROL__GRPH_NUM_BANKS__SHIFT 0x00000002
+#define GRPH_CONTROL__GRPH_PIPE_CONFIG_MASK 0x1f000000L
+#define GRPH_CONTROL__GRPH_PIPE_CONFIG__SHIFT 0x00000018
+#define GRPH_CONTROL__GRPH_PRIVILEGED_ACCESS_ENABLE_MASK 0x00020000L
+#define GRPH_CONTROL__GRPH_PRIVILEGED_ACCESS_ENABLE__SHIFT 0x00000011
+#define GRPH_CONTROL__GRPH_TILE_SPLIT_MASK 0x0000e000L
+#define GRPH_CONTROL__GRPH_TILE_SPLIT__SHIFT 0x0000000d
+#define GRPH_CONTROL__GRPH_Z_MASK 0x00000030L
+#define GRPH_CONTROL__GRPH_Z__SHIFT 0x00000004
+#define GRPH_DFQ_CONTROL__GRPH_DFQ_MIN_FREE_ENTRIES_MASK 0x00000700L
+#define GRPH_DFQ_CONTROL__GRPH_DFQ_MIN_FREE_ENTRIES__SHIFT 0x00000008
+#define GRPH_DFQ_CONTROL__GRPH_DFQ_RESET_MASK 0x00000001L
+#define GRPH_DFQ_CONTROL__GRPH_DFQ_RESET__SHIFT 0x00000000
+#define GRPH_DFQ_CONTROL__GRPH_DFQ_SIZE_MASK 0x00000070L
+#define GRPH_DFQ_CONTROL__GRPH_DFQ_SIZE__SHIFT 0x00000004
+#define GRPH_DFQ_STATUS__GRPH_DFQ_RESET_ACK_MASK 0x00000200L
+#define GRPH_DFQ_STATUS__GRPH_DFQ_RESET_ACK__SHIFT 0x00000009
+#define GRPH_DFQ_STATUS__GRPH_DFQ_RESET_FLAG_MASK 0x00000100L
+#define GRPH_DFQ_STATUS__GRPH_DFQ_RESET_FLAG__SHIFT 0x00000008
+#define GRPH_DFQ_STATUS__GRPH_PRIMARY_DFQ_NUM_ENTRIES_MASK 0x0000000fL
+#define GRPH_DFQ_STATUS__GRPH_PRIMARY_DFQ_NUM_ENTRIES__SHIFT 0x00000000
+#define GRPH_DFQ_STATUS__GRPH_SECONDARY_DFQ_NUM_ENTRIES_MASK 0x000000f0L
+#define GRPH_DFQ_STATUS__GRPH_SECONDARY_DFQ_NUM_ENTRIES__SHIFT 0x00000004
+#define GRPH_ENABLE__GRPH_ENABLE_MASK 0x00000001L
+#define GRPH_ENABLE__GRPH_ENABLE__SHIFT 0x00000000
+#define GRPH_FLIP_CONTROL__GRPH_SURFACE_UPDATE_H_RETRACE_EN_MASK 0x00000001L
+#define GRPH_FLIP_CONTROL__GRPH_SURFACE_UPDATE_H_RETRACE_EN__SHIFT 0x00000000
+#define GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK_MASK 0x00000001L
+#define GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_MASK__SHIFT 0x00000000
+#define GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_TYPE_MASK 0x00000100L
+#define GRPH_INTERRUPT_CONTROL__GRPH_PFLIP_INT_TYPE__SHIFT 0x00000008
+#define GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR_MASK 0x00000100L
+#define GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_CLEAR__SHIFT 0x00000008
+#define GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_OCCURRED_MASK 0x00000001L
+#define GRPH_INTERRUPT_STATUS__GRPH_PFLIP_INT_OCCURRED__SHIFT 0x00000000
+#define GRPH_LUT_10BIT_BYPASS__GRPH_LUT_10BIT_BYPASS_DBL_BUF_EN_MASK 0x00010000L
+#define GRPH_LUT_10BIT_BYPASS__GRPH_LUT_10BIT_BYPASS_DBL_BUF_EN__SHIFT 0x00000010
+#define GRPH_LUT_10BIT_BYPASS__GRPH_LUT_10BIT_BYPASS_EN_MASK 0x00000100L
+#define GRPH_LUT_10BIT_BYPASS__GRPH_LUT_10BIT_BYPASS_EN__SHIFT 0x00000008
+#define GRPH_PITCH__GRPH_PITCH_MASK 0x00007fffL
+#define GRPH_PITCH__GRPH_PITCH__SHIFT 0x00000000
+#define GRPH_PRIMARY_SURFACE_ADDRESS__GRPH_PRIMARY_DFQ_ENABLE_MASK 0x00000001L
+#define GRPH_PRIMARY_SURFACE_ADDRESS__GRPH_PRIMARY_DFQ_ENABLE__SHIFT 0x00000000
+#define GRPH_PRIMARY_SURFACE_ADDRESS__GRPH_PRIMARY_SURFACE_ADDRESS_MASK 0xffffff00L
+#define GRPH_PRIMARY_SURFACE_ADDRESS__GRPH_PRIMARY_SURFACE_ADDRESS__SHIFT 0x00000008
+#define GRPH_PRIMARY_SURFACE_ADDRESS_HIGH__GRPH_PRIMARY_SURFACE_ADDRESS_HIGH_MASK 0x000000ffL
+#define GRPH_PRIMARY_SURFACE_ADDRESS_HIGH__GRPH_PRIMARY_SURFACE_ADDRESS_HIGH__SHIFT 0x00000000
+#define GRPH_SECONDARY_SURFACE_ADDRESS__GRPH_SECONDARY_DFQ_ENABLE_MASK 0x00000001L
+#define GRPH_SECONDARY_SURFACE_ADDRESS__GRPH_SECONDARY_DFQ_ENABLE__SHIFT 0x00000000
+#define GRPH_SECONDARY_SURFACE_ADDRESS__GRPH_SECONDARY_SURFACE_ADDRESS_MASK 0xffffff00L
+#define GRPH_SECONDARY_SURFACE_ADDRESS__GRPH_SECONDARY_SURFACE_ADDRESS__SHIFT 0x00000008
+#define GRPH_SECONDARY_SURFACE_ADDRESS_HIGH__GRPH_SECONDARY_SURFACE_ADDRESS_HIGH_MASK 0x000000ffL
+#define GRPH_SECONDARY_SURFACE_ADDRESS_HIGH__GRPH_SECONDARY_SURFACE_ADDRESS_HIGH__SHIFT 0x00000000
+#define GRPH_STEREOSYNC_FLIP__GRPH_PRIMARY_SURFACE_PENDING_MASK 0x00010000L
+#define GRPH_STEREOSYNC_FLIP__GRPH_PRIMARY_SURFACE_PENDING__SHIFT 0x00000010
+#define GRPH_STEREOSYNC_FLIP__GRPH_SECONDARY_SURFACE_PENDING_MASK 0x00020000L
+#define GRPH_STEREOSYNC_FLIP__GRPH_SECONDARY_SURFACE_PENDING__SHIFT 0x00000011
+#define GRPH_STEREOSYNC_FLIP__GRPH_STEREOSYNC_FLIP_EN_MASK 0x00000001L
+#define GRPH_STEREOSYNC_FLIP__GRPH_STEREOSYNC_FLIP_EN__SHIFT 0x00000000
+#define GRPH_STEREOSYNC_FLIP__GRPH_STEREOSYNC_FLIP_MODE_MASK 0x00000300L
+#define GRPH_STEREOSYNC_FLIP__GRPH_STEREOSYNC_FLIP_MODE__SHIFT 0x00000008
+#define GRPH_STEREOSYNC_FLIP__GRPH_STEREOSYNC_SELECT_DISABLE_MASK 0x10000000L
+#define GRPH_STEREOSYNC_FLIP__GRPH_STEREOSYNC_SELECT_DISABLE__SHIFT 0x0000001c
+#define GRPH_SURFACE_ADDRESS_HIGH_INUSE__GRPH_SURFACE_ADDRESS_HIGH_INUSE_MASK 0x000000ffL
+#define GRPH_SURFACE_ADDRESS_HIGH_INUSE__GRPH_SURFACE_ADDRESS_HIGH_INUSE__SHIFT 0x00000000
+#define GRPH_SURFACE_ADDRESS_INUSE__GRPH_SURFACE_ADDRESS_INUSE_MASK 0xffffff00L
+#define GRPH_SURFACE_ADDRESS_INUSE__GRPH_SURFACE_ADDRESS_INUSE__SHIFT 0x00000008
+#define GRPH_SURFACE_OFFSET_X__GRPH_SURFACE_OFFSET_X_MASK 0x00003fffL
+#define GRPH_SURFACE_OFFSET_X__GRPH_SURFACE_OFFSET_X__SHIFT 0x00000000
+#define GRPH_SURFACE_OFFSET_Y__GRPH_SURFACE_OFFSET_Y_MASK 0x00003fffL
+#define GRPH_SURFACE_OFFSET_Y__GRPH_SURFACE_OFFSET_Y__SHIFT 0x00000000
+#define GRPH_SWAP_CNTL__GRPH_ALPHA_CROSSBAR_MASK 0x00000c00L
+#define GRPH_SWAP_CNTL__GRPH_ALPHA_CROSSBAR__SHIFT 0x0000000a
+#define GRPH_SWAP_CNTL__GRPH_BLUE_CROSSBAR_MASK 0x00000300L
+#define GRPH_SWAP_CNTL__GRPH_BLUE_CROSSBAR__SHIFT 0x00000008
+#define GRPH_SWAP_CNTL__GRPH_ENDIAN_SWAP_MASK 0x00000003L
+#define GRPH_SWAP_CNTL__GRPH_ENDIAN_SWAP__SHIFT 0x00000000
+#define GRPH_SWAP_CNTL__GRPH_GREEN_CROSSBAR_MASK 0x000000c0L
+#define GRPH_SWAP_CNTL__GRPH_GREEN_CROSSBAR__SHIFT 0x00000006
+#define GRPH_SWAP_CNTL__GRPH_RED_CROSSBAR_MASK 0x00000030L
+#define GRPH_SWAP_CNTL__GRPH_RED_CROSSBAR__SHIFT 0x00000004
+#define GRPH_UPDATE__GRPH_MODE_DISABLE_MULTIPLE_UPDATE_MASK 0x01000000L
+#define GRPH_UPDATE__GRPH_MODE_DISABLE_MULTIPLE_UPDATE__SHIFT 0x00000018
+#define GRPH_UPDATE__GRPH_MODE_UPDATE_PENDING_MASK 0x00000001L
+#define GRPH_UPDATE__GRPH_MODE_UPDATE_PENDING__SHIFT 0x00000000
+#define GRPH_UPDATE__GRPH_MODE_UPDATE_TAKEN_MASK 0x00000002L
+#define GRPH_UPDATE__GRPH_MODE_UPDATE_TAKEN__SHIFT 0x00000001
+#define GRPH_UPDATE__GRPH_SURFACE_DISABLE_MULTIPLE_UPDATE_MASK 0x10000000L
+#define GRPH_UPDATE__GRPH_SURFACE_DISABLE_MULTIPLE_UPDATE__SHIFT 0x0000001c
+#define GRPH_UPDATE__GRPH_SURFACE_UPDATE_PENDING_MASK 0x00000004L
+#define GRPH_UPDATE__GRPH_SURFACE_UPDATE_PENDING__SHIFT 0x00000002
+#define GRPH_UPDATE__GRPH_SURFACE_UPDATE_TAKEN_MASK 0x00000008L
+#define GRPH_UPDATE__GRPH_SURFACE_UPDATE_TAKEN__SHIFT 0x00000003
+#define GRPH_UPDATE__GRPH_SURFACE_XDMA_PENDING_ENABLE_MASK 0x00000100L
+#define GRPH_UPDATE__GRPH_SURFACE_XDMA_PENDING_ENABLE__SHIFT 0x00000008
+#define GRPH_UPDATE__GRPH_UPDATE_LOCK_MASK 0x00010000L
+#define GRPH_UPDATE__GRPH_UPDATE_LOCK__SHIFT 0x00000010
+#define GRPH_X_END__GRPH_X_END_MASK 0x00007fffL
+#define GRPH_X_END__GRPH_X_END__SHIFT 0x00000000
+#define GRPH_X_START__GRPH_X_START_MASK 0x00003fffL
+#define GRPH_X_START__GRPH_X_START__SHIFT 0x00000000
+#define GRPH_Y_END__GRPH_Y_END_MASK 0x00007fffL
+#define GRPH_Y_END__GRPH_Y_END__SHIFT 0x00000000
+#define GRPH_Y_START__GRPH_Y_START_MASK 0x00003fffL
+#define GRPH_Y_START__GRPH_Y_START__SHIFT 0x00000000
+#define HDMI_ACR_32_0__HDMI_ACR_CTS_32_MASK 0xfffff000L
+#define HDMI_ACR_32_0__HDMI_ACR_CTS_32__SHIFT 0x0000000c
+#define HDMI_ACR_32_1__HDMI_ACR_N_32_MASK 0x000fffffL
+#define HDMI_ACR_32_1__HDMI_ACR_N_32__SHIFT 0x00000000
+#define HDMI_ACR_44_0__HDMI_ACR_CTS_44_MASK 0xfffff000L
+#define HDMI_ACR_44_0__HDMI_ACR_CTS_44__SHIFT 0x0000000c
+#define HDMI_ACR_44_1__HDMI_ACR_N_44_MASK 0x000fffffL
+#define HDMI_ACR_44_1__HDMI_ACR_N_44__SHIFT 0x00000000
+#define HDMI_ACR_48_0__HDMI_ACR_CTS_48_MASK 0xfffff000L
+#define HDMI_ACR_48_0__HDMI_ACR_CTS_48__SHIFT 0x0000000c
+#define HDMI_ACR_48_1__HDMI_ACR_N_48_MASK 0x000fffffL
+#define HDMI_ACR_48_1__HDMI_ACR_N_48__SHIFT 0x00000000
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY_MASK 0x80000000L
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY__SHIFT 0x0000001f
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK 0x00001000L
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT 0x0000000c
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK 0x00000002L
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT 0x00000001
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK 0x00070000L
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT 0x00000010
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK 0x00000030L
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT 0x00000004
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK 0x00000001L
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT 0x00000000
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK 0x00000100L
+#define HDMI_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT 0x00000008
+#define HDMI_ACR_STATUS_0__HDMI_ACR_CTS_MASK 0xfffff000L
+#define HDMI_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT 0x0000000c
+#define HDMI_ACR_STATUS_1__HDMI_ACR_N_MASK 0x000fffffL
+#define HDMI_ACR_STATUS_1__HDMI_ACR_N__SHIFT 0x00000000
+#define HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_DELAY_EN_MASK 0x00000030L
+#define HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_DELAY_EN__SHIFT 0x00000004
+#define HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_PACKETS_PER_LINE_MASK 0x001f0000L
+#define HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_PACKETS_PER_LINE__SHIFT 0x00000010
+#define HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_SEND_MAX_PACKETS_MASK 0x00000100L
+#define HDMI_AUDIO_PACKET_CONTROL__HDMI_AUDIO_SEND_MAX_PACKETS__SHIFT 0x00000008
+#define HDMI_CONTROL__HDMI_DEEP_COLOR_DEPTH_MASK 0x30000000L
+#define HDMI_CONTROL__HDMI_DEEP_COLOR_DEPTH__SHIFT 0x0000001c
+#define HDMI_CONTROL__HDMI_DEEP_COLOR_ENABLE_MASK 0x01000000L
+#define HDMI_CONTROL__HDMI_DEEP_COLOR_ENABLE__SHIFT 0x00000018
+#define HDMI_CONTROL__HDMI_ERROR_ACK_MASK 0x00000100L
+#define HDMI_CONTROL__HDMI_ERROR_ACK__SHIFT 0x00000008
+#define HDMI_CONTROL__HDMI_ERROR_MASK_MASK 0x00000200L
+#define HDMI_CONTROL__HDMI_ERROR_MASK__SHIFT 0x00000009
+#define HDMI_CONTROL__HDMI_KEEPOUT_MODE_MASK 0x00000001L
+#define HDMI_CONTROL__HDMI_KEEPOUT_MODE__SHIFT 0x00000000
+#define HDMI_CONTROL__HDMI_PACKET_GEN_VERSION_MASK 0x00000010L
+#define HDMI_CONTROL__HDMI_PACKET_GEN_VERSION__SHIFT 0x00000004
+#define HDMI_GC__HDMI_DEFAULT_PHASE_MASK 0x00000010L
+#define HDMI_GC__HDMI_DEFAULT_PHASE__SHIFT 0x00000004
+#define HDMI_GC__HDMI_GC_AVMUTE_CONT_MASK 0x00000004L
+#define HDMI_GC__HDMI_GC_AVMUTE_CONT__SHIFT 0x00000002
+#define HDMI_GC__HDMI_GC_AVMUTE_MASK 0x00000001L
+#define HDMI_GC__HDMI_GC_AVMUTE__SHIFT 0x00000000
+#define HDMI_GC__HDMI_PACKING_PHASE_MASK 0x00000f00L
+#define HDMI_GC__HDMI_PACKING_PHASE_OVERRIDE_MASK 0x00001000L
+#define HDMI_GC__HDMI_PACKING_PHASE_OVERRIDE__SHIFT 0x0000000c
+#define HDMI_GC__HDMI_PACKING_PHASE__SHIFT 0x00000008
+#define HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK 0x00000002L
+#define HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT 0x00000001
+#define HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_MASK 0x003f0000L
+#define HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE__SHIFT 0x00000010
+#define HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK 0x00000001L
+#define HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT 0x00000000
+#define HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK 0x00000020L
+#define HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT 0x00000005
+#define HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_MASK 0x3f000000L
+#define HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE__SHIFT 0x00000018
+#define HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK 0x00000010L
+#define HDMI_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT 0x00000004
+#define HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC2_CONT_MASK 0x00000002L
+#define HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC2_CONT__SHIFT 0x00000001
+#define HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC2_LINE_MASK 0x003f0000L
+#define HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC2_LINE__SHIFT 0x00000010
+#define HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC2_SEND_MASK 0x00000001L
+#define HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC2_SEND__SHIFT 0x00000000
+#define HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC3_CONT_MASK 0x00000020L
+#define HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC3_CONT__SHIFT 0x00000005
+#define HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC3_LINE_MASK 0x3f000000L
+#define HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC3_LINE__SHIFT 0x00000018
+#define HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC3_SEND_MASK 0x00000010L
+#define HDMI_GENERIC_PACKET_CONTROL1__HDMI_GENERIC3_SEND__SHIFT 0x00000004
+#define HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_CONT_MASK 0x00000020L
+#define HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_CONT__SHIFT 0x00000005
+#define HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_SEND_MASK 0x00000010L
+#define HDMI_INFOFRAME_CONTROL0__HDMI_AUDIO_INFO_SEND__SHIFT 0x00000004
+#define HDMI_INFOFRAME_CONTROL0__HDMI_AVI_INFO_CONT_MASK 0x00000002L
+#define HDMI_INFOFRAME_CONTROL0__HDMI_AVI_INFO_CONT__SHIFT 0x00000001
+#define HDMI_INFOFRAME_CONTROL0__HDMI_AVI_INFO_SEND_MASK 0x00000001L
+#define HDMI_INFOFRAME_CONTROL0__HDMI_AVI_INFO_SEND__SHIFT 0x00000000
+#define HDMI_INFOFRAME_CONTROL0__HDMI_MPEG_INFO_CONT_MASK 0x00000200L
+#define HDMI_INFOFRAME_CONTROL0__HDMI_MPEG_INFO_CONT__SHIFT 0x00000009
+#define HDMI_INFOFRAME_CONTROL0__HDMI_MPEG_INFO_SEND_MASK 0x00000100L
+#define HDMI_INFOFRAME_CONTROL0__HDMI_MPEG_INFO_SEND__SHIFT 0x00000008
+#define HDMI_INFOFRAME_CONTROL1__HDMI_AUDIO_INFO_LINE_MASK 0x00003f00L
+#define HDMI_INFOFRAME_CONTROL1__HDMI_AUDIO_INFO_LINE__SHIFT 0x00000008
+#define HDMI_INFOFRAME_CONTROL1__HDMI_AVI_INFO_LINE_MASK 0x0000003fL
+#define HDMI_INFOFRAME_CONTROL1__HDMI_AVI_INFO_LINE__SHIFT 0x00000000
+#define HDMI_INFOFRAME_CONTROL1__HDMI_MPEG_INFO_LINE_MASK 0x003f0000L
+#define HDMI_INFOFRAME_CONTROL1__HDMI_MPEG_INFO_LINE__SHIFT 0x00000010
+#define HDMI_STATUS__HDMI_ACTIVE_AVMUTE_MASK 0x00000001L
+#define HDMI_STATUS__HDMI_ACTIVE_AVMUTE__SHIFT 0x00000000
+#define HDMI_STATUS__HDMI_AUDIO_PACKET_ERROR_MASK 0x00010000L
+#define HDMI_STATUS__HDMI_AUDIO_PACKET_ERROR__SHIFT 0x00000010
+#define HDMI_STATUS__HDMI_ERROR_INT_MASK 0x08000000L
+#define HDMI_STATUS__HDMI_ERROR_INT__SHIFT 0x0000001b
+#define HDMI_STATUS__HDMI_VBI_PACKET_ERROR_MASK 0x00100000L
+#define HDMI_STATUS__HDMI_VBI_PACKET_ERROR__SHIFT 0x00000014
+#define HDMI_VBI_PACKET_CONTROL__HDMI_GC_CONT_MASK 0x00000020L
+#define HDMI_VBI_PACKET_CONTROL__HDMI_GC_CONT__SHIFT 0x00000005
+#define HDMI_VBI_PACKET_CONTROL__HDMI_GC_SEND_MASK 0x00000010L
+#define HDMI_VBI_PACKET_CONTROL__HDMI_GC_SEND__SHIFT 0x00000004
+#define HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_CONT_MASK 0x00000200L
+#define HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_CONT__SHIFT 0x00000009
+#define HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_LINE_MASK 0x003f0000L
+#define HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_LINE__SHIFT 0x00000010
+#define HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_SEND_MASK 0x00000100L
+#define HDMI_VBI_PACKET_CONTROL__HDMI_ISRC_SEND__SHIFT 0x00000008
+#define HDMI_VBI_PACKET_CONTROL__HDMI_NULL_SEND_MASK 0x00000001L
+#define HDMI_VBI_PACKET_CONTROL__HDMI_NULL_SEND__SHIFT 0x00000000
+#define IDDCCIF02_DBG_DCCIF_C__DBG_DCCIF_C_MASK 0xffffffffL
+#define IDDCCIF02_DBG_DCCIF_C__DBG_DCCIF_C__SHIFT 0x00000000
+#define IDDCCIF04_DBG_DCCIF_E__DBG_DCCIF_E_MASK 0xffffffffL
+#define IDDCCIF04_DBG_DCCIF_E__DBG_DCCIF_E__SHIFT 0x00000000
+#define IDDCCIF05_DBG_DCCIF_F__DBG_DCCIF_F_MASK 0xffffffffL
+#define IDDCCIF05_DBG_DCCIF_F__DBG_DCCIF_F__SHIFT 0x00000000
+#define INPUT_CSC_C11_C12__INPUT_CSC_C11_MASK 0x0000ffffL
+#define INPUT_CSC_C11_C12__INPUT_CSC_C11__SHIFT 0x00000000
+#define INPUT_CSC_C11_C12__INPUT_CSC_C12_MASK 0xffff0000L
+#define INPUT_CSC_C11_C12__INPUT_CSC_C12__SHIFT 0x00000010
+#define INPUT_CSC_C13_C14__INPUT_CSC_C13_MASK 0x0000ffffL
+#define INPUT_CSC_C13_C14__INPUT_CSC_C13__SHIFT 0x00000000
+#define INPUT_CSC_C13_C14__INPUT_CSC_C14_MASK 0xffff0000L
+#define INPUT_CSC_C13_C14__INPUT_CSC_C14__SHIFT 0x00000010
+#define INPUT_CSC_C21_C22__INPUT_CSC_C21_MASK 0x0000ffffL
+#define INPUT_CSC_C21_C22__INPUT_CSC_C21__SHIFT 0x00000000
+#define INPUT_CSC_C21_C22__INPUT_CSC_C22_MASK 0xffff0000L
+#define INPUT_CSC_C21_C22__INPUT_CSC_C22__SHIFT 0x00000010
+#define INPUT_CSC_C23_C24__INPUT_CSC_C23_MASK 0x0000ffffL
+#define INPUT_CSC_C23_C24__INPUT_CSC_C23__SHIFT 0x00000000
+#define INPUT_CSC_C23_C24__INPUT_CSC_C24_MASK 0xffff0000L
+#define INPUT_CSC_C23_C24__INPUT_CSC_C24__SHIFT 0x00000010
+#define INPUT_CSC_C31_C32__INPUT_CSC_C31_MASK 0x0000ffffL
+#define INPUT_CSC_C31_C32__INPUT_CSC_C31__SHIFT 0x00000000
+#define INPUT_CSC_C31_C32__INPUT_CSC_C32_MASK 0xffff0000L
+#define INPUT_CSC_C31_C32__INPUT_CSC_C32__SHIFT 0x00000010
+#define INPUT_CSC_C33_C34__INPUT_CSC_C33_MASK 0x0000ffffL
+#define INPUT_CSC_C33_C34__INPUT_CSC_C33__SHIFT 0x00000000
+#define INPUT_CSC_C33_C34__INPUT_CSC_C34_MASK 0xffff0000L
+#define INPUT_CSC_C33_C34__INPUT_CSC_C34__SHIFT 0x00000010
+#define INPUT_CSC_CONTROL__INPUT_CSC_GRPH_MODE_MASK 0x00000003L
+#define INPUT_CSC_CONTROL__INPUT_CSC_GRPH_MODE__SHIFT 0x00000000
+#define INPUT_CSC_CONTROL__INPUT_CSC_OVL_MODE_MASK 0x00000030L
+#define INPUT_CSC_CONTROL__INPUT_CSC_OVL_MODE__SHIFT 0x00000004
+#define INPUT_GAMMA_CONTROL__GRPH_INPUT_GAMMA_MODE_MASK 0x00000003L
+#define INPUT_GAMMA_CONTROL__GRPH_INPUT_GAMMA_MODE__SHIFT 0x00000000
+#define INPUT_GAMMA_CONTROL__OVL_INPUT_GAMMA_MODE_MASK 0x00000030L
+#define INPUT_GAMMA_CONTROL__OVL_INPUT_GAMMA_MODE__SHIFT 0x00000004
+#define KEY_CONTROL__GRPH_OVL_HALF_BLEND_MASK 0x10000000L
+#define KEY_CONTROL__GRPH_OVL_HALF_BLEND__SHIFT 0x0000001c
+#define KEY_CONTROL__KEY_MODE_MASK 0x00000006L
+#define KEY_CONTROL__KEY_MODE__SHIFT 0x00000001
+#define KEY_CONTROL__KEY_SELECT_MASK 0x00000001L
+#define KEY_CONTROL__KEY_SELECT__SHIFT 0x00000000
+#define KEY_RANGE_ALPHA__KEY_ALPHA_HIGH_MASK 0xffff0000L
+#define KEY_RANGE_ALPHA__KEY_ALPHA_HIGH__SHIFT 0x00000010
+#define KEY_RANGE_ALPHA__KEY_ALPHA_LOW_MASK 0x0000ffffL
+#define KEY_RANGE_ALPHA__KEY_ALPHA_LOW__SHIFT 0x00000000
+#define KEY_RANGE_BLUE__KEY_BLUE_HIGH_MASK 0xffff0000L
+#define KEY_RANGE_BLUE__KEY_BLUE_HIGH__SHIFT 0x00000010
+#define KEY_RANGE_BLUE__KEY_BLUE_LOW_MASK 0x0000ffffL
+#define KEY_RANGE_BLUE__KEY_BLUE_LOW__SHIFT 0x00000000
+#define KEY_RANGE_GREEN__KEY_GREEN_HIGH_MASK 0xffff0000L
+#define KEY_RANGE_GREEN__KEY_GREEN_HIGH__SHIFT 0x00000010
+#define KEY_RANGE_GREEN__KEY_GREEN_LOW_MASK 0x0000ffffL
+#define KEY_RANGE_GREEN__KEY_GREEN_LOW__SHIFT 0x00000000
+#define KEY_RANGE_RED__KEY_RED_HIGH_MASK 0xffff0000L
+#define KEY_RANGE_RED__KEY_RED_HIGH__SHIFT 0x00000010
+#define KEY_RANGE_RED__KEY_RED_LOW_MASK 0x0000ffffL
+#define KEY_RANGE_RED__KEY_RED_LOW__SHIFT 0x00000000
+#define LB_DEBUG2__LB_DEBUG2_MASK 0xffffffffL
+#define LB_DEBUG2__LB_DEBUG2__SHIFT 0x00000000
+#define LB_DEBUG__LB_DEBUG_MASK 0xffffffffL
+#define LB_DEBUG__LB_DEBUG__SHIFT 0x00000000
+#define LB_NO_OUTSTANDING_REQ_STATUS__LB_NO_OUTSTANDING_REQ_STAT_MASK 0x00000001L
+#define LB_NO_OUTSTANDING_REQ_STATUS__LB_NO_OUTSTANDING_REQ_STAT__SHIFT 0x00000000
+#define LB_SYNC_RESET_SEL__LB_SYNC_RESET_SEL2_MASK 0x00000010L
+#define LB_SYNC_RESET_SEL__LB_SYNC_RESET_SEL2__SHIFT 0x00000004
+#define LB_SYNC_RESET_SEL__LB_SYNC_RESET_SEL_MASK 0x00000003L
+#define LB_SYNC_RESET_SEL__LB_SYNC_RESET_SEL__SHIFT 0x00000000
+#define LB_TEST_DEBUG_DATA__LB_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define LB_TEST_DEBUG_DATA__LB_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define LB_TEST_DEBUG_INDEX__LB_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define LB_TEST_DEBUG_INDEX__LB_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define LB_TEST_DEBUG_INDEX__LB_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define LB_TEST_DEBUG_INDEX__LB_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define LIGHT_SLEEP_CNTL__LIGHT_SLEEP_DIS_MASK 0x00000001L
+#define LIGHT_SLEEP_CNTL__LIGHT_SLEEP_DIS__SHIFT 0x00000000
+#define LIGHT_SLEEP_CNTL__MEM_SHUTDOWN_DIS_MASK 0x00000100L
+#define LIGHT_SLEEP_CNTL__MEM_SHUTDOWN_DIS__SHIFT 0x00000008
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_ENABLE_MASK 0x00000001L
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_ENABLE__SHIFT 0x00000000
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_MODE_MASK 0x00000018L
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_MODE__SHIFT 0x00000003
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_NUM_BANKS_MASK 0x00000700L
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_NUM_BANKS__SHIFT 0x00000008
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_NUM_PIPES_MASK 0x000000e0L
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_NUM_PIPES__SHIFT 0x00000005
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_PIPE_INTERLEAVE_SIZE_MASK 0x00000800L
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_PIPE_INTERLEAVE_SIZE__SHIFT 0x0000000b
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_ROW_SIZE_MASK 0x00007000L
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_ROW_SIZE__SHIFT 0x0000000c
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_ROWS_PER_CHAN_MASK 0x0fff0000L
+#define LOW_POWER_TILING_CONTROL__LOW_POWER_TILING_ROWS_PER_CHAN__SHIFT 0x00000010
+#define LVDS_DATA_CNTL__LVDS_24BIT_ENABLE_MASK 0x00000001L
+#define LVDS_DATA_CNTL__LVDS_24BIT_ENABLE__SHIFT 0x00000000
+#define LVDS_DATA_CNTL__LVDS_24BIT_FORMAT_MASK 0x00000010L
+#define LVDS_DATA_CNTL__LVDS_24BIT_FORMAT__SHIFT 0x00000004
+#define LVDS_DATA_CNTL__LVDS_2ND_CHAN_DE_MASK 0x00000100L
+#define LVDS_DATA_CNTL__LVDS_2ND_CHAN_DE__SHIFT 0x00000008
+#define LVDS_DATA_CNTL__LVDS_2ND_CHAN_HS_MASK 0x00000400L
+#define LVDS_DATA_CNTL__LVDS_2ND_CHAN_HS__SHIFT 0x0000000a
+#define LVDS_DATA_CNTL__LVDS_2ND_CHAN_VS_MASK 0x00000200L
+#define LVDS_DATA_CNTL__LVDS_2ND_CHAN_VS__SHIFT 0x00000009
+#define LVDS_DATA_CNTL__LVDS_2ND_LINK_CNTL_BITS_MASK 0x00007000L
+#define LVDS_DATA_CNTL__LVDS_2ND_LINK_CNTL_BITS__SHIFT 0x0000000c
+#define LVDS_DATA_CNTL__LVDS_DTMG_POL_MASK 0x00040000L
+#define LVDS_DATA_CNTL__LVDS_DTMG_POL__SHIFT 0x00000012
+#define LVDS_DATA_CNTL__LVDS_FP_POL_MASK 0x00010000L
+#define LVDS_DATA_CNTL__LVDS_FP_POL__SHIFT 0x00000010
+#define LVDS_DATA_CNTL__LVDS_LP_POL_MASK 0x00020000L
+#define LVDS_DATA_CNTL__LVDS_LP_POL__SHIFT 0x00000011
+#define LVTMA_PWRSEQ_CNTL__LVTMA_BLON_MASK 0x01000000L
+#define LVTMA_PWRSEQ_CNTL__LVTMA_BLON_OVRD_MASK 0x02000000L
+#define LVTMA_PWRSEQ_CNTL__LVTMA_BLON_OVRD__SHIFT 0x00000019
+#define LVTMA_PWRSEQ_CNTL__LVTMA_BLON_POL_MASK 0x04000000L
+#define LVTMA_PWRSEQ_CNTL__LVTMA_BLON_POL__SHIFT 0x0000001a
+#define LVTMA_PWRSEQ_CNTL__LVTMA_BLON__SHIFT 0x00000018
+#define LVTMA_PWRSEQ_CNTL__LVTMA_DIGON_MASK 0x00010000L
+#define LVTMA_PWRSEQ_CNTL__LVTMA_DIGON_OVRD_MASK 0x00020000L
+#define LVTMA_PWRSEQ_CNTL__LVTMA_DIGON_OVRD__SHIFT 0x00000011
+#define LVTMA_PWRSEQ_CNTL__LVTMA_DIGON_POL_MASK 0x00040000L
+#define LVTMA_PWRSEQ_CNTL__LVTMA_DIGON_POL__SHIFT 0x00000012
+#define LVTMA_PWRSEQ_CNTL__LVTMA_DIGON__SHIFT 0x00000010
+#define LVTMA_PWRSEQ_CNTL__LVTMA_PWRSEQ_DISABLE_SYNCEN_CONTROL_OF_TX_EN_MASK 0x00000002L
+#define LVTMA_PWRSEQ_CNTL__LVTMA_PWRSEQ_DISABLE_SYNCEN_CONTROL_OF_TX_EN__SHIFT 0x00000001
+#define LVTMA_PWRSEQ_CNTL__LVTMA_PWRSEQ_EN_MASK 0x00000001L
+#define LVTMA_PWRSEQ_CNTL__LVTMA_PWRSEQ_EN__SHIFT 0x00000000
+#define LVTMA_PWRSEQ_CNTL__LVTMA_PWRSEQ_TARGET_STATE_MASK 0x00000010L
+#define LVTMA_PWRSEQ_CNTL__LVTMA_PWRSEQ_TARGET_STATE__SHIFT 0x00000004
+#define LVTMA_PWRSEQ_CNTL__LVTMA_SYNCEN_MASK 0x00000100L
+#define LVTMA_PWRSEQ_CNTL__LVTMA_SYNCEN_OVRD_MASK 0x00000200L
+#define LVTMA_PWRSEQ_CNTL__LVTMA_SYNCEN_OVRD__SHIFT 0x00000009
+#define LVTMA_PWRSEQ_CNTL__LVTMA_SYNCEN_POL_MASK 0x00000400L
+#define LVTMA_PWRSEQ_CNTL__LVTMA_SYNCEN_POL__SHIFT 0x0000000a
+#define LVTMA_PWRSEQ_CNTL__LVTMA_SYNCEN__SHIFT 0x00000008
+#define LVTMA_PWRSEQ_DELAY1__LVTMA_PWRDN_DELAY1_MASK 0x00ff0000L
+#define LVTMA_PWRSEQ_DELAY1__LVTMA_PWRDN_DELAY1__SHIFT 0x00000010
+#define LVTMA_PWRSEQ_DELAY1__LVTMA_PWRDN_DELAY2_MASK 0xff000000L
+#define LVTMA_PWRSEQ_DELAY1__LVTMA_PWRDN_DELAY2__SHIFT 0x00000018
+#define LVTMA_PWRSEQ_DELAY1__LVTMA_PWRUP_DELAY1_MASK 0x000000ffL
+#define LVTMA_PWRSEQ_DELAY1__LVTMA_PWRUP_DELAY1__SHIFT 0x00000000
+#define LVTMA_PWRSEQ_DELAY1__LVTMA_PWRUP_DELAY2_MASK 0x0000ff00L
+#define LVTMA_PWRSEQ_DELAY1__LVTMA_PWRUP_DELAY2__SHIFT 0x00000008
+#define LVTMA_PWRSEQ_DELAY2__LVTMA_PWRDN_DELAY3_MASK 0x00ff0000L
+#define LVTMA_PWRSEQ_DELAY2__LVTMA_PWRDN_DELAY3__SHIFT 0x00000010
+#define LVTMA_PWRSEQ_DELAY2__LVTMA_PWRDN_MIN_LENGTH_MASK 0x000000ffL
+#define LVTMA_PWRSEQ_DELAY2__LVTMA_PWRDN_MIN_LENGTH__SHIFT 0x00000000
+#define LVTMA_PWRSEQ_DELAY2__LVTMA_PWRUP_DELAY3_MASK 0x0000ff00L
+#define LVTMA_PWRSEQ_DELAY2__LVTMA_PWRUP_DELAY3__SHIFT 0x00000008
+#define LVTMA_PWRSEQ_DELAY2__LVTMA_VARY_BL_OVERRIDE_EN_MASK 0x01000000L
+#define LVTMA_PWRSEQ_DELAY2__LVTMA_VARY_BL_OVERRIDE_EN__SHIFT 0x00000018
+#define LVTMA_PWRSEQ_REF_DIV__BL_PWM_REF_DIV_MASK 0xffff0000L
+#define LVTMA_PWRSEQ_REF_DIV__BL_PWM_REF_DIV__SHIFT 0x00000010
+#define LVTMA_PWRSEQ_REF_DIV__LVTMA_PWRSEQ_REF_DIV_MASK 0x00000fffL
+#define LVTMA_PWRSEQ_REF_DIV__LVTMA_PWRSEQ_REF_DIV__SHIFT 0x00000000
+#define LVTMA_PWRSEQ_STATE__LVTMA_PWRSEQ_BLON_MASK 0x00000008L
+#define LVTMA_PWRSEQ_STATE__LVTMA_PWRSEQ_BLON__SHIFT 0x00000003
+#define LVTMA_PWRSEQ_STATE__LVTMA_PWRSEQ_DIGON_MASK 0x00000002L
+#define LVTMA_PWRSEQ_STATE__LVTMA_PWRSEQ_DIGON__SHIFT 0x00000001
+#define LVTMA_PWRSEQ_STATE__LVTMA_PWRSEQ_DONE_MASK 0x00000010L
+#define LVTMA_PWRSEQ_STATE__LVTMA_PWRSEQ_DONE__SHIFT 0x00000004
+#define LVTMA_PWRSEQ_STATE__LVTMA_PWRSEQ_STATE_MASK 0x00000f00L
+#define LVTMA_PWRSEQ_STATE__LVTMA_PWRSEQ_STATE__SHIFT 0x00000008
+#define LVTMA_PWRSEQ_STATE__LVTMA_PWRSEQ_SYNCEN_MASK 0x00000004L
+#define LVTMA_PWRSEQ_STATE__LVTMA_PWRSEQ_SYNCEN__SHIFT 0x00000002
+#define LVTMA_PWRSEQ_STATE__LVTMA_PWRSEQ_TARGET_STATE_R_MASK 0x00000001L
+#define LVTMA_PWRSEQ_STATE__LVTMA_PWRSEQ_TARGET_STATE_R__SHIFT 0x00000000
+#define MASTER_COMM_CMD_REG__MASTER_COMM_CMD_REG_BYTE0_MASK 0x000000ffL
+#define MASTER_COMM_CMD_REG__MASTER_COMM_CMD_REG_BYTE0__SHIFT 0x00000000
+#define MASTER_COMM_CMD_REG__MASTER_COMM_CMD_REG_BYTE1_MASK 0x0000ff00L
+#define MASTER_COMM_CMD_REG__MASTER_COMM_CMD_REG_BYTE1__SHIFT 0x00000008
+#define MASTER_COMM_CMD_REG__MASTER_COMM_CMD_REG_BYTE2_MASK 0x00ff0000L
+#define MASTER_COMM_CMD_REG__MASTER_COMM_CMD_REG_BYTE2__SHIFT 0x00000010
+#define MASTER_COMM_CMD_REG__MASTER_COMM_CMD_REG_BYTE3_MASK 0xff000000L
+#define MASTER_COMM_CMD_REG__MASTER_COMM_CMD_REG_BYTE3__SHIFT 0x00000018
+#define MASTER_COMM_CNTL_REG__MASTER_COMM_INTERRUPT_MASK 0x00000001L
+#define MASTER_COMM_CNTL_REG__MASTER_COMM_INTERRUPT__SHIFT 0x00000000
+#define MASTER_COMM_DATA_REG1__MASTER_COMM_DATA_REG1_BYTE0_MASK 0x000000ffL
+#define MASTER_COMM_DATA_REG1__MASTER_COMM_DATA_REG1_BYTE0__SHIFT 0x00000000
+#define MASTER_COMM_DATA_REG1__MASTER_COMM_DATA_REG1_BYTE1_MASK 0x0000ff00L
+#define MASTER_COMM_DATA_REG1__MASTER_COMM_DATA_REG1_BYTE1__SHIFT 0x00000008
+#define MASTER_COMM_DATA_REG1__MASTER_COMM_DATA_REG1_BYTE2_MASK 0x00ff0000L
+#define MASTER_COMM_DATA_REG1__MASTER_COMM_DATA_REG1_BYTE2__SHIFT 0x00000010
+#define MASTER_COMM_DATA_REG1__MASTER_COMM_DATA_REG1_BYTE3_MASK 0xff000000L
+#define MASTER_COMM_DATA_REG1__MASTER_COMM_DATA_REG1_BYTE3__SHIFT 0x00000018
+#define MASTER_COMM_DATA_REG2__MASTER_COMM_DATA_REG2_BYTE0_MASK 0x000000ffL
+#define MASTER_COMM_DATA_REG2__MASTER_COMM_DATA_REG2_BYTE0__SHIFT 0x00000000
+#define MASTER_COMM_DATA_REG2__MASTER_COMM_DATA_REG2_BYTE1_MASK 0x0000ff00L
+#define MASTER_COMM_DATA_REG2__MASTER_COMM_DATA_REG2_BYTE1__SHIFT 0x00000008
+#define MASTER_COMM_DATA_REG2__MASTER_COMM_DATA_REG2_BYTE2_MASK 0x00ff0000L
+#define MASTER_COMM_DATA_REG2__MASTER_COMM_DATA_REG2_BYTE2__SHIFT 0x00000010
+#define MASTER_COMM_DATA_REG2__MASTER_COMM_DATA_REG2_BYTE3_MASK 0xff000000L
+#define MASTER_COMM_DATA_REG2__MASTER_COMM_DATA_REG2_BYTE3__SHIFT 0x00000018
+#define MASTER_COMM_DATA_REG3__MASTER_COMM_DATA_REG3_BYTE0_MASK 0x000000ffL
+#define MASTER_COMM_DATA_REG3__MASTER_COMM_DATA_REG3_BYTE0__SHIFT 0x00000000
+#define MASTER_COMM_DATA_REG3__MASTER_COMM_DATA_REG3_BYTE1_MASK 0x0000ff00L
+#define MASTER_COMM_DATA_REG3__MASTER_COMM_DATA_REG3_BYTE1__SHIFT 0x00000008
+#define MASTER_COMM_DATA_REG3__MASTER_COMM_DATA_REG3_BYTE2_MASK 0x00ff0000L
+#define MASTER_COMM_DATA_REG3__MASTER_COMM_DATA_REG3_BYTE2__SHIFT 0x00000010
+#define MASTER_COMM_DATA_REG3__MASTER_COMM_DATA_REG3_BYTE3_MASK 0xff000000L
+#define MASTER_COMM_DATA_REG3__MASTER_COMM_DATA_REG3_BYTE3__SHIFT 0x00000018
+#define MASTER_UPDATE_LOCK__GSL_CONTROL_MASTER_UPDATE_LOCK_MASK 0x00000100L
+#define MASTER_UPDATE_LOCK__GSL_CONTROL_MASTER_UPDATE_LOCK__SHIFT 0x00000008
+#define MASTER_UPDATE_LOCK__MASTER_UPDATE_LOCK_MASK 0x00000001L
+#define MASTER_UPDATE_LOCK__MASTER_UPDATE_LOCK__SHIFT 0x00000000
+#define MASTER_UPDATE_MODE__MASTER_UPDATE_INTERLACED_MODE_MASK 0x00030000L
+#define MASTER_UPDATE_MODE__MASTER_UPDATE_INTERLACED_MODE__SHIFT 0x00000010
+#define MASTER_UPDATE_MODE__MASTER_UPDATE_MODE_MASK 0x00000007L
+#define MASTER_UPDATE_MODE__MASTER_UPDATE_MODE__SHIFT 0x00000000
+#define MC_DC_INTERFACE_NACK_STATUS__DMIF_RDRET_NACK_CLEAR_MASK 0x00000010L
+#define MC_DC_INTERFACE_NACK_STATUS__DMIF_RDRET_NACK_CLEAR__SHIFT 0x00000004
+#define MC_DC_INTERFACE_NACK_STATUS__DMIF_RDRET_NACK_OCCURRED_MASK 0x00000001L
+#define MC_DC_INTERFACE_NACK_STATUS__DMIF_RDRET_NACK_OCCURRED__SHIFT 0x00000000
+#define MC_DC_INTERFACE_NACK_STATUS__MCIF_RDRET_NACK_CLEAR_MASK 0x00100000L
+#define MC_DC_INTERFACE_NACK_STATUS__MCIF_RDRET_NACK_CLEAR__SHIFT 0x00000014
+#define MC_DC_INTERFACE_NACK_STATUS__MCIF_RDRET_NACK_OCCURRED_MASK 0x00010000L
+#define MC_DC_INTERFACE_NACK_STATUS__MCIF_RDRET_NACK_OCCURRED__SHIFT 0x00000010
+#define MC_DC_INTERFACE_NACK_STATUS__MCIF_WRRET_NACK_CLEAR_MASK 0x10000000L
+#define MC_DC_INTERFACE_NACK_STATUS__MCIF_WRRET_NACK_CLEAR__SHIFT 0x0000001c
+#define MC_DC_INTERFACE_NACK_STATUS__MCIF_WRRET_NACK_OCCURRED_MASK 0x01000000L
+#define MC_DC_INTERFACE_NACK_STATUS__MCIF_WRRET_NACK_OCCURRED__SHIFT 0x00000018
+#define MC_DC_INTERFACE_NACK_STATUS__VIP_WRRET_NACK_CLEAR_MASK 0x00001000L
+#define MC_DC_INTERFACE_NACK_STATUS__VIP_WRRET_NACK_CLEAR__SHIFT 0x0000000c
+#define MC_DC_INTERFACE_NACK_STATUS__VIP_WRRET_NACK_OCCURRED_MASK 0x00000100L
+#define MC_DC_INTERFACE_NACK_STATUS__VIP_WRRET_NACK_OCCURRED__SHIFT 0x00000008
+#define MCIF_CONTROL__ADDRESS_TRANSLATION_ENABLE_MASK 0x00000010L
+#define MCIF_CONTROL__ADDRESS_TRANSLATION_ENABLE__SHIFT 0x00000004
+#define MCIF_CONTROL__LOW_READ_URG_LEVEL_MASK 0x00ff0000L
+#define MCIF_CONTROL__LOW_READ_URG_LEVEL__SHIFT 0x00000010
+#define MCIF_CONTROL__MC_CLEAN_DEASSERT_LATENCY_MASK 0x3f000000L
+#define MCIF_CONTROL__MC_CLEAN_DEASSERT_LATENCY__SHIFT 0x00000018
+#define MCIF_CONTROL__MCIF_BUFF_SIZE_MASK 0x00000003L
+#define MCIF_CONTROL__MCIF_BUFF_SIZE__SHIFT 0x00000000
+#define MCIF_CONTROL__MCIF_MC_LATENCY_COUNTER_ENABLE_MASK 0x40000000L
+#define MCIF_CONTROL__MCIF_MC_LATENCY_COUNTER_ENABLE__SHIFT 0x0000001e
+#define MCIF_CONTROL__MCIF_MC_LATENCY_COUNTER_URGENT_ONLY_MASK 0x80000000L
+#define MCIF_CONTROL__MCIF_MC_LATENCY_COUNTER_URGENT_ONLY__SHIFT 0x0000001f
+#define MCIF_CONTROL__MCIF_SLOW_REQ_INTERVAL_MASK 0x0000f000L
+#define MCIF_CONTROL__MCIF_SLOW_REQ_INTERVAL__SHIFT 0x0000000c
+#define MCIF_CONTROL__PRIVILEGED_ACCESS_ENABLE_MASK 0x00000100L
+#define MCIF_CONTROL__PRIVILEGED_ACCESS_ENABLE__SHIFT 0x00000008
+#define MCIF_MEM_CONTROL__MCIFMEM_CACHE_MODE_DIS_MASK 0x00000001L
+#define MCIF_MEM_CONTROL__MCIFMEM_CACHE_MODE_DIS__SHIFT 0x00000000
+#define MCIF_MEM_CONTROL__MCIFMEM_CACHE_MODE_MASK 0x00000030L
+#define MCIF_MEM_CONTROL__MCIFMEM_CACHE_MODE__SHIFT 0x00000004
+#define MCIF_MEM_CONTROL__MCIFMEM_CACHE_PIPE_MASK 0x00070000L
+#define MCIF_MEM_CONTROL__MCIFMEM_CACHE_PIPE__SHIFT 0x00000010
+#define MCIF_MEM_CONTROL__MCIFMEM_CACHE_SIZE_MASK 0x00007f00L
+#define MCIF_MEM_CONTROL__MCIFMEM_CACHE_SIZE__SHIFT 0x00000008
+#define MCIF_MEM_CONTROL__MCIFMEM_CACHE_TYPE_MASK 0x00180000L
+#define MCIF_MEM_CONTROL__MCIFMEM_CACHE_TYPE__SHIFT 0x00000013
+#define MCIF_TEST_DEBUG_DATA__MCIF_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define MCIF_TEST_DEBUG_DATA__MCIF_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define MCIF_TEST_DEBUG_INDEX__MCIF_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define MCIF_TEST_DEBUG_INDEX__MCIF_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define MCIF_TEST_DEBUG_INDEX__MCIF_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define MCIF_TEST_DEBUG_INDEX__MCIF_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define MCIF_VMID__MCIF_WR_VMID_MASK 0x0000000fL
+#define MCIF_VMID__MCIF_WR_VMID__SHIFT 0x00000000
+#define MCIF_VMID__VIP_WR_VMID_MASK 0x000000f0L
+#define MCIF_VMID__VIP_WR_VMID__SHIFT 0x00000004
+#define MCIF_WRITE_COMBINE_CONTROL__MCIF_WRITE_COMBINE_TIMEOUT_MASK 0x000000ffL
+#define MCIF_WRITE_COMBINE_CONTROL__MCIF_WRITE_COMBINE_TIMEOUT__SHIFT 0x00000000
+#define MCIF_WRITE_COMBINE_CONTROL__VIP_WRITE_COMBINE_TIMEOUT_MASK 0x0000ff00L
+#define MCIF_WRITE_COMBINE_CONTROL__VIP_WRITE_COMBINE_TIMEOUT__SHIFT 0x00000008
+#define MICROSECOND_TIME_BASE_DIV__MICROSECOND_TIME_BASE_CLOCK_SOURCE_SEL_MASK 0x00100000L
+#define MICROSECOND_TIME_BASE_DIV__MICROSECOND_TIME_BASE_CLOCK_SOURCE_SEL__SHIFT 0x00000014
+#define MICROSECOND_TIME_BASE_DIV__MICROSECOND_TIME_BASE_DIV_MASK 0x0000007fL
+#define MICROSECOND_TIME_BASE_DIV__MICROSECOND_TIME_BASE_DIV__SHIFT 0x00000000
+#define MICROSECOND_TIME_BASE_DIV__XTAL_REF_CLOCK_SOURCE_SEL_MASK 0x00020000L
+#define MICROSECOND_TIME_BASE_DIV__XTAL_REF_CLOCK_SOURCE_SEL__SHIFT 0x00000011
+#define MICROSECOND_TIME_BASE_DIV__XTAL_REF_DIV_MASK 0x00007f00L
+#define MICROSECOND_TIME_BASE_DIV__XTAL_REF_DIV__SHIFT 0x00000008
+#define MICROSECOND_TIME_BASE_DIV__XTAL_REF_SEL_MASK 0x00010000L
+#define MICROSECOND_TIME_BASE_DIV__XTAL_REF_SEL__SHIFT 0x00000010
+#define MILLISECOND_TIME_BASE_DIV__MILLISECOND_TIME_BASE_CLOCK_SOURCE_SEL_MASK 0x00100000L
+#define MILLISECOND_TIME_BASE_DIV__MILLISECOND_TIME_BASE_CLOCK_SOURCE_SEL__SHIFT 0x00000014
+#define MILLISECOND_TIME_BASE_DIV__MILLISECOND_TIME_BASE_DIV_MASK 0x0001ffffL
+#define MILLISECOND_TIME_BASE_DIV__MILLISECOND_TIME_BASE_DIV__SHIFT 0x00000000
+#define MVP_AFR_FLIP_FIFO_CNTL__MVP_AFR_FLIP_FIFO_NUM_ENTRIES_MASK 0x0000000fL
+#define MVP_AFR_FLIP_FIFO_CNTL__MVP_AFR_FLIP_FIFO_NUM_ENTRIES__SHIFT 0x00000000
+#define MVP_AFR_FLIP_FIFO_CNTL__MVP_AFR_FLIP_FIFO_RESET_ACK_MASK 0x00001000L
+#define MVP_AFR_FLIP_FIFO_CNTL__MVP_AFR_FLIP_FIFO_RESET_ACK__SHIFT 0x0000000c
+#define MVP_AFR_FLIP_FIFO_CNTL__MVP_AFR_FLIP_FIFO_RESET_FLAG_MASK 0x00000100L
+#define MVP_AFR_FLIP_FIFO_CNTL__MVP_AFR_FLIP_FIFO_RESET_FLAG__SHIFT 0x00000008
+#define MVP_AFR_FLIP_FIFO_CNTL__MVP_AFR_FLIP_FIFO_RESET_MASK 0x00000010L
+#define MVP_AFR_FLIP_FIFO_CNTL__MVP_AFR_FLIP_FIFO_RESET__SHIFT 0x00000004
+#define MVP_AFR_FLIP_MODE__MVP_AFR_FLIP_MODE_MASK 0x00000003L
+#define MVP_AFR_FLIP_MODE__MVP_AFR_FLIP_MODE__SHIFT 0x00000000
+#define MVP_BLACK_KEYER__MVP_BLACK_KEYER_B_MASK 0x3ff00000L
+#define MVP_BLACK_KEYER__MVP_BLACK_KEYER_B__SHIFT 0x00000014
+#define MVP_BLACK_KEYER__MVP_BLACK_KEYER_G_MASK 0x000ffc00L
+#define MVP_BLACK_KEYER__MVP_BLACK_KEYER_G__SHIFT 0x0000000a
+#define MVP_BLACK_KEYER__MVP_BLACK_KEYER_R_MASK 0x000003ffL
+#define MVP_BLACK_KEYER__MVP_BLACK_KEYER_R__SHIFT 0x00000000
+#define MVP_CONTROL1__MVP_30BPP_EN_MASK 0x10000000L
+#define MVP_CONTROL1__MVP_30BPP_EN__SHIFT 0x0000001c
+#define MVP_CONTROL1__MVP_ARBITRATION_MODE_FOR_AFR_MANUAL_SWITCH_MODE_MASK 0x00000400L
+#define MVP_CONTROL1__MVP_ARBITRATION_MODE_FOR_AFR_MANUAL_SWITCH_MODE__SHIFT 0x0000000a
+#define MVP_CONTROL1__MVP_CHANNEL_CONTROL_MASK 0x00010000L
+#define MVP_CONTROL1__MVP_CHANNEL_CONTROL__SHIFT 0x00000010
+#define MVP_CONTROL1__MVP_DISABLE_MSB_EXPAND_MASK 0x01000000L
+#define MVP_CONTROL1__MVP_DISABLE_MSB_EXPAND__SHIFT 0x00000018
+#define MVP_CONTROL1__MVP_EN_MASK 0x00000001L
+#define MVP_CONTROL1__MVP_EN__SHIFT 0x00000000
+#define MVP_CONTROL1__MVP_GPU_CHAIN_LOCATION_MASK 0x00300000L
+#define MVP_CONTROL1__MVP_GPU_CHAIN_LOCATION__SHIFT 0x00000014
+#define MVP_CONTROL1__MVP_MIXER_MODE_MASK 0x00000070L
+#define MVP_CONTROL1__MVP_MIXER_MODE__SHIFT 0x00000004
+#define MVP_CONTROL1__MVP_MIXER_SLAVE_SEL_DELAY_UNTIL_END_OF_BLANK_MASK 0x00000200L
+#define MVP_CONTROL1__MVP_MIXER_SLAVE_SEL_DELAY_UNTIL_END_OF_BLANK__SHIFT 0x00000009
+#define MVP_CONTROL1__MVP_MIXER_SLAVE_SEL_MASK 0x00000100L
+#define MVP_CONTROL1__MVP_MIXER_SLAVE_SEL__SHIFT 0x00000008
+#define MVP_CONTROL1__MVP_RATE_CONTROL_MASK 0x00001000L
+#define MVP_CONTROL1__MVP_RATE_CONTROL__SHIFT 0x0000000c
+#define MVP_CONTROL1__MVP_TERMINATION_CNTL_A_MASK 0x40000000L
+#define MVP_CONTROL1__MVP_TERMINATION_CNTL_A__SHIFT 0x0000001e
+#define MVP_CONTROL1__MVP_TERMINATION_CNTL_B_MASK 0x80000000L
+#define MVP_CONTROL1__MVP_TERMINATION_CNTL_B__SHIFT 0x0000001f
+#define MVP_CONTROL2__MVP_DVOCNTL_MUX_MASK 0x00010000L
+#define MVP_CONTROL2__MVP_DVOCNTL_MUX__SHIFT 0x00000010
+#define MVP_CONTROL2__MVP_FLOW_CONTROL_OUT_EN_MASK 0x00100000L
+#define MVP_CONTROL2__MVP_FLOW_CONTROL_OUT_EN__SHIFT 0x00000014
+#define MVP_CONTROL2__MVP_MUXA_CLK_SEL_MASK 0x00000100L
+#define MVP_CONTROL2__MVP_MUXA_CLK_SEL__SHIFT 0x00000008
+#define MVP_CONTROL2__MVP_MUXB_CLK_SEL_MASK 0x00001000L
+#define MVP_CONTROL2__MVP_MUXB_CLK_SEL__SHIFT 0x0000000c
+#define MVP_CONTROL2__MVP_MUX_DE_DVOCNTL0_SEL_MASK 0x00000001L
+#define MVP_CONTROL2__MVP_MUX_DE_DVOCNTL0_SEL__SHIFT 0x00000000
+#define MVP_CONTROL2__MVP_MUX_DE_DVOCNTL2_SEL_MASK 0x00000010L
+#define MVP_CONTROL2__MVP_MUX_DE_DVOCNTL2_SEL__SHIFT 0x00000004
+#define MVP_CONTROL2__MVP_SWAP_AB_IN_DC_DDR_MASK 0x10000000L
+#define MVP_CONTROL2__MVP_SWAP_AB_IN_DC_DDR__SHIFT 0x0000001c
+#define MVP_CONTROL2__MVP_SWAP_LOCK_OUT_EN_MASK 0x01000000L
+#define MVP_CONTROL2__MVP_SWAP_LOCK_OUT_EN__SHIFT 0x00000018
+#define MVP_CONTROL3__MVP_DDR_SC_AB_SEL_MASK 0x00000010L
+#define MVP_CONTROL3__MVP_DDR_SC_AB_SEL__SHIFT 0x00000004
+#define MVP_CONTROL3__MVP_DDR_SC_B_START_MODE_MASK 0x00000100L
+#define MVP_CONTROL3__MVP_DDR_SC_B_START_MODE__SHIFT 0x00000008
+#define MVP_CONTROL3__MVP_FLOW_CONTROL_CASCADE_EN_MASK 0x00100000L
+#define MVP_CONTROL3__MVP_FLOW_CONTROL_CASCADE_EN__SHIFT 0x00000014
+#define MVP_CONTROL3__MVP_FLOW_CONTROL_IN_CAP_MASK 0x10000000L
+#define MVP_CONTROL3__MVP_FLOW_CONTROL_IN_CAP__SHIFT 0x0000001c
+#define MVP_CONTROL3__MVP_FLOW_CONTROL_OUT_FORCE_ONE_MASK 0x00001000L
+#define MVP_CONTROL3__MVP_FLOW_CONTROL_OUT_FORCE_ONE__SHIFT 0x0000000c
+#define MVP_CONTROL3__MVP_FLOW_CONTROL_OUT_FORCE_ZERO_MASK 0x00010000L
+#define MVP_CONTROL3__MVP_FLOW_CONTROL_OUT_FORCE_ZERO__SHIFT 0x00000010
+#define MVP_CONTROL3__MVP_RESET_IN_BETWEEN_FRAMES_MASK 0x00000001L
+#define MVP_CONTROL3__MVP_RESET_IN_BETWEEN_FRAMES__SHIFT 0x00000000
+#define MVP_CONTROL3__MVP_SWAP_48BIT_EN_MASK 0x01000000L
+#define MVP_CONTROL3__MVP_SWAP_48BIT_EN__SHIFT 0x00000018
+#define MVP_CRC_CNTL__MVP_CRC_BLUE_MASK_MASK 0x000000ffL
+#define MVP_CRC_CNTL__MVP_CRC_BLUE_MASK__SHIFT 0x00000000
+#define MVP_CRC_CNTL__MVP_CRC_CONT_EN_MASK 0x20000000L
+#define MVP_CRC_CNTL__MVP_CRC_CONT_EN__SHIFT 0x0000001d
+#define MVP_CRC_CNTL__MVP_CRC_EN_MASK 0x10000000L
+#define MVP_CRC_CNTL__MVP_CRC_EN__SHIFT 0x0000001c
+#define MVP_CRC_CNTL__MVP_CRC_GREEN_MASK_MASK 0x0000ff00L
+#define MVP_CRC_CNTL__MVP_CRC_GREEN_MASK__SHIFT 0x00000008
+#define MVP_CRC_CNTL__MVP_CRC_RED_MASK_MASK 0x00ff0000L
+#define MVP_CRC_CNTL__MVP_CRC_RED_MASK__SHIFT 0x00000010
+#define MVP_CRC_CNTL__MVP_DC_DDR_CRC_EVEN_ODD_PIX_SEL_MASK 0x40000000L
+#define MVP_CRC_CNTL__MVP_DC_DDR_CRC_EVEN_ODD_PIX_SEL__SHIFT 0x0000001e
+#define MVP_CRC_RESULT_BLUE_GREEN__MVP_CRC_BLUE_RESULT_MASK 0x0000ffffL
+#define MVP_CRC_RESULT_BLUE_GREEN__MVP_CRC_BLUE_RESULT__SHIFT 0x00000000
+#define MVP_CRC_RESULT_BLUE_GREEN__MVP_CRC_GREEN_RESULT_MASK 0xffff0000L
+#define MVP_CRC_RESULT_BLUE_GREEN__MVP_CRC_GREEN_RESULT__SHIFT 0x00000010
+#define MVP_CRC_RESULT_RED__MVP_CRC_RED_RESULT_MASK 0x0000ffffL
+#define MVP_CRC_RESULT_RED__MVP_CRC_RED_RESULT__SHIFT 0x00000000
+#define MVP_DEBUG_05__IDE0_MVP_GPU_CHAIN_LOCATION_MASK 0x00000006L
+#define MVP_DEBUG_05__IDE0_MVP_GPU_CHAIN_LOCATION__SHIFT 0x00000001
+#define MVP_DEBUG_09__IDE4_CRTC2_MVP_GPU_CHAIN_LOCATION_MASK 0x00000006L
+#define MVP_DEBUG_09__IDE4_CRTC2_MVP_GPU_CHAIN_LOCATION__SHIFT 0x00000001
+#define MVP_DEBUG_12__IDEC_MVP_DATA_A_H_MASK 0x00000001L
+#define MVP_DEBUG_12__IDEC_MVP_DATA_A_H__SHIFT 0x00000000
+#define MVP_DEBUG_12__IDEC_MVP_DATA_A_MASK 0x01fffffeL
+#define MVP_DEBUG_12__IDEC_MVP_DATA_A__SHIFT 0x00000001
+#define MVP_DEBUG_13__IDED_MVP_DATA_B_H_MASK 0x00000001L
+#define MVP_DEBUG_13__IDED_MVP_DATA_B_H__SHIFT 0x00000000
+#define MVP_DEBUG_13__IDED_MVP_DATA_B_MASK 0x01fffffeL
+#define MVP_DEBUG_13__IDED_MVP_DATA_B__SHIFT 0x00000001
+#define MVP_DEBUG_13__IDED_READ_FIFO_ENTRY_DE_B_MASK 0x04000000L
+#define MVP_DEBUG_13__IDED_READ_FIFO_ENTRY_DE_B__SHIFT 0x0000001a
+#define MVP_DEBUG_13__IDED_START_READ_B_MASK 0x02000000L
+#define MVP_DEBUG_13__IDED_START_READ_B__SHIFT 0x00000019
+#define MVP_DEBUG_13__IDED_WRITE_ADD_B_MASK 0x38000000L
+#define MVP_DEBUG_13__IDED_WRITE_ADD_B__SHIFT 0x0000001b
+#define MVP_DEBUG_14__IDEE_CRC_PHASE_MASK 0x00100000L
+#define MVP_DEBUG_14__IDEE_CRC_PHASE__SHIFT 0x00000014
+#define MVP_DEBUG_14__IDEE_CRTC1_CNTL_CAPTURE_START_A_MASK 0x00080000L
+#define MVP_DEBUG_14__IDEE_CRTC1_CNTL_CAPTURE_START_A__SHIFT 0x00000013
+#define MVP_DEBUG_14__IDEE_READ_ADD_MASK 0x00000007L
+#define MVP_DEBUG_14__IDEE_READ_ADD__SHIFT 0x00000000
+#define MVP_DEBUG_14__IDEE_READ_FIFO_DE_B_MASK 0x00020000L
+#define MVP_DEBUG_14__IDEE_READ_FIFO_DE_B__SHIFT 0x00000011
+#define MVP_DEBUG_14__IDEE_READ_FIFO_DE_MASK 0x00010000L
+#define MVP_DEBUG_14__IDEE_READ_FIFO_DE__SHIFT 0x00000010
+#define MVP_DEBUG_14__IDEE_READ_FIFO_ENABLE_MASK 0x00040000L
+#define MVP_DEBUG_14__IDEE_READ_FIFO_ENABLE__SHIFT 0x00000012
+#define MVP_DEBUG_14__IDEE_READ_FIFO_ENTRY_DE_B_MASK 0x00008000L
+#define MVP_DEBUG_14__IDEE_READ_FIFO_ENTRY_DE_B__SHIFT 0x0000000f
+#define MVP_DEBUG_14__IDEE_READ_FIFO_ENTRY_DE_MASK 0x00004000L
+#define MVP_DEBUG_14__IDEE_READ_FIFO_ENTRY_DE__SHIFT 0x0000000e
+#define MVP_DEBUG_14__IDEE_START_INCR_WR_A_MASK 0x00000800L
+#define MVP_DEBUG_14__IDEE_START_INCR_WR_A__SHIFT 0x0000000b
+#define MVP_DEBUG_14__IDEE_START_INCR_WR_B_MASK 0x00001000L
+#define MVP_DEBUG_14__IDEE_START_INCR_WR_B__SHIFT 0x0000000c
+#define MVP_DEBUG_14__IDEE_START_READ_B_MASK 0x00000400L
+#define MVP_DEBUG_14__IDEE_START_READ_B__SHIFT 0x0000000a
+#define MVP_DEBUG_14__IDEE_START_READ_MASK 0x00000200L
+#define MVP_DEBUG_14__IDEE_START_READ__SHIFT 0x00000009
+#define MVP_DEBUG_14__IDEE_WRITE2FIFO_MASK 0x00002000L
+#define MVP_DEBUG_14__IDEE_WRITE2FIFO__SHIFT 0x0000000d
+#define MVP_DEBUG_14__IDEE_WRITE_ADD_A_MASK 0x00000038L
+#define MVP_DEBUG_14__IDEE_WRITE_ADD_A__SHIFT 0x00000003
+#define MVP_DEBUG_14__IDEE_WRITE_ADD_B_MASK 0x000001c0L
+#define MVP_DEBUG_14__IDEE_WRITE_ADD_B__SHIFT 0x00000006
+#define MVP_DEBUG_15__IDEF_MVP_ASYNC_FIFO_WDATA_MASK 0xfffffff0L
+#define MVP_DEBUG_15__IDEF_MVP_ASYNC_FIFO_WDATA__SHIFT 0x00000004
+#define MVP_DEBUG_15__IDEF_MVP_ASYNC_FIFO_WEN_MASK 0x00000001L
+#define MVP_DEBUG_15__IDEF_MVP_ASYNC_FIFO_WEN__SHIFT 0x00000000
+#define MVP_DEBUG_16__IDCC_FLOW_CONTROL_OUT_MASK 0x00000008L
+#define MVP_DEBUG_16__IDCC_FLOW_CONTROL_OUT__SHIFT 0x00000003
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_FIFO_EXCEED_PAUSE_LEVEL_MASK 0x00000004L
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_FIFO_EXCEED_PAUSE_LEVEL__SHIFT 0x00000002
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_FIFO_EXCEED_STOP_LEVEL_MASK 0x00000002L
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_FIFO_EXCEED_STOP_LEVEL__SHIFT 0x00000001
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_FIFO_NUM_ENTRIES_MASK 0x00000ff0L
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_FIFO_NUM_ENTRIES__SHIFT 0x00000004
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_FIFO_OVERFLOW_MASK 0x00001000L
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_FIFO_OVERFLOW__SHIFT 0x0000000c
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_FIFO_READ_MASK 0x00000001L
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_FIFO_READ__SHIFT 0x00000000
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_FIFO_UNDERFLOW_MASK 0x00002000L
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_FIFO_UNDERFLOW__SHIFT 0x0000000d
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_READ_ADDR_MASK 0x00ff0000L
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_READ_ADDR__SHIFT 0x00000010
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_WRITE_ADDR_MASK 0xff000000L
+#define MVP_DEBUG_16__IDCC_MVP_ASYNC_WRITE_ADDR__SHIFT 0x00000018
+#define MVP_DEBUG_17__IDCD_MVP_ASYNC_FIFO_PHASE_MASK 0x00000002L
+#define MVP_DEBUG_17__IDCD_MVP_ASYNC_FIFO_PHASE__SHIFT 0x00000001
+#define MVP_DEBUG_17__IDCD_MVP_ASYNC_FIFO_READ_DATA_MASK 0xfffffffcL
+#define MVP_DEBUG_17__IDCD_MVP_ASYNC_FIFO_READ_DATA__SHIFT 0x00000002
+#define MVP_DEBUG_17__IDCD_MVP_ASYNC_FIFO_READ_MASK 0x00000001L
+#define MVP_DEBUG_17__IDCD_MVP_ASYNC_FIFO_READ__SHIFT 0x00000000
+#define MVP_DEBUG__MVP_DEBUG_BITS_MASK 0xffffff00L
+#define MVP_DEBUG__MVP_DEBUG_BITS__SHIFT 0x00000008
+#define MVP_DEBUG__MVP_DIS_FIX_AFR_AUTO_VSYNC_FLIP_MASK 0x00000020L
+#define MVP_DEBUG__MVP_DIS_FIX_AFR_AUTO_VSYNC_FLIP__SHIFT 0x00000005
+#define MVP_DEBUG__MVP_DIS_FIX_AFR_MANUAL_HSYNC_FLIP_MASK 0x00000010L
+#define MVP_DEBUG__MVP_DIS_FIX_AFR_MANUAL_HSYNC_FLIP__SHIFT 0x00000004
+#define MVP_DEBUG__MVP_DIS_READ_POINTER_RESET_DELAY_MASK 0x00000080L
+#define MVP_DEBUG__MVP_DIS_READ_POINTER_RESET_DELAY__SHIFT 0x00000007
+#define MVP_DEBUG__MVP_EN_FIX_AFR_MANUAL_SWITCH_IN_SFR_MASK 0x00000040L
+#define MVP_DEBUG__MVP_EN_FIX_AFR_MANUAL_SWITCH_IN_SFR__SHIFT 0x00000006
+#define MVP_DEBUG__MVP_FLOW_CONTROL_IN_EN_MASK 0x00000002L
+#define MVP_DEBUG__MVP_FLOW_CONTROL_IN_EN__SHIFT 0x00000001
+#define MVP_DEBUG__MVP_FLOW_CONTROL_IN_SEL_MASK 0x00000008L
+#define MVP_DEBUG__MVP_FLOW_CONTROL_IN_SEL__SHIFT 0x00000003
+#define MVP_DEBUG__MVP_SWAP_LOCK_IN_EN_MASK 0x00000001L
+#define MVP_DEBUG__MVP_SWAP_LOCK_IN_EN__SHIFT 0x00000000
+#define MVP_DEBUG__MVP_SWAP_LOCK_IN_SEL_MASK 0x00000004L
+#define MVP_DEBUG__MVP_SWAP_LOCK_IN_SEL__SHIFT 0x00000002
+#define MVP_FIFO_CONTROL__MVP_PAUSE_SLAVE_CNT_MASK 0x00ff0000L
+#define MVP_FIFO_CONTROL__MVP_PAUSE_SLAVE_CNT__SHIFT 0x00000010
+#define MVP_FIFO_CONTROL__MVP_PAUSE_SLAVE_WM_MASK 0x0000ff00L
+#define MVP_FIFO_CONTROL__MVP_PAUSE_SLAVE_WM__SHIFT 0x00000008
+#define MVP_FIFO_CONTROL__MVP_STOP_SLAVE_WM_MASK 0x000000ffL
+#define MVP_FIFO_CONTROL__MVP_STOP_SLAVE_WM__SHIFT 0x00000000
+#define MVP_FIFO_STATUS__MVP_FIFO_ERROR_INT_STATUS_MASK 0x80000000L
+#define MVP_FIFO_STATUS__MVP_FIFO_ERROR_INT_STATUS__SHIFT 0x0000001f
+#define MVP_FIFO_STATUS__MVP_FIFO_ERROR_MASK_MASK 0x40000000L
+#define MVP_FIFO_STATUS__MVP_FIFO_ERROR_MASK__SHIFT 0x0000001e
+#define MVP_FIFO_STATUS__MVP_FIFO_LEVEL_MASK 0x000000ffL
+#define MVP_FIFO_STATUS__MVP_FIFO_LEVEL__SHIFT 0x00000000
+#define MVP_FIFO_STATUS__MVP_FIFO_OVERFLOW_ACK_MASK 0x00010000L
+#define MVP_FIFO_STATUS__MVP_FIFO_OVERFLOW_ACK__SHIFT 0x00000010
+#define MVP_FIFO_STATUS__MVP_FIFO_OVERFLOW_MASK 0x00000100L
+#define MVP_FIFO_STATUS__MVP_FIFO_OVERFLOW_OCCURRED_MASK 0x00001000L
+#define MVP_FIFO_STATUS__MVP_FIFO_OVERFLOW_OCCURRED__SHIFT 0x0000000c
+#define MVP_FIFO_STATUS__MVP_FIFO_OVERFLOW__SHIFT 0x00000008
+#define MVP_FIFO_STATUS__MVP_FIFO_UNDERFLOW_ACK_MASK 0x10000000L
+#define MVP_FIFO_STATUS__MVP_FIFO_UNDERFLOW_ACK__SHIFT 0x0000001c
+#define MVP_FIFO_STATUS__MVP_FIFO_UNDERFLOW_MASK 0x00100000L
+#define MVP_FIFO_STATUS__MVP_FIFO_UNDERFLOW_OCCURRED_MASK 0x01000000L
+#define MVP_FIFO_STATUS__MVP_FIFO_UNDERFLOW_OCCURRED__SHIFT 0x00000018
+#define MVP_FIFO_STATUS__MVP_FIFO_UNDERFLOW__SHIFT 0x00000014
+#define MVP_FLIP_LINE_NUM_INSERT__MVP_FLIP_AUTO_ENABLE_MASK 0x40000000L
+#define MVP_FLIP_LINE_NUM_INSERT__MVP_FLIP_AUTO_ENABLE__SHIFT 0x0000001e
+#define MVP_FLIP_LINE_NUM_INSERT__MVP_FLIP_LINE_NUM_INSERT_MASK 0x007fff00L
+#define MVP_FLIP_LINE_NUM_INSERT__MVP_FLIP_LINE_NUM_INSERT_MODE_MASK 0x00000003L
+#define MVP_FLIP_LINE_NUM_INSERT__MVP_FLIP_LINE_NUM_INSERT_MODE__SHIFT 0x00000000
+#define MVP_FLIP_LINE_NUM_INSERT__MVP_FLIP_LINE_NUM_INSERT__SHIFT 0x00000008
+#define MVP_FLIP_LINE_NUM_INSERT__MVP_FLIP_LINE_NUM_OFFSET_MASK 0x3f000000L
+#define MVP_FLIP_LINE_NUM_INSERT__MVP_FLIP_LINE_NUM_OFFSET__SHIFT 0x00000018
+#define MVP_INBAND_CNTL_CAP__MVP_IGNOR_INBAND_CNTL_MASK 0x00000001L
+#define MVP_INBAND_CNTL_CAP__MVP_IGNOR_INBAND_CNTL__SHIFT 0x00000000
+#define MVP_INBAND_CNTL_CAP__MVP_INBAND_CNTL_CHAR_CAP_MASK 0xffffff00L
+#define MVP_INBAND_CNTL_CAP__MVP_INBAND_CNTL_CHAR_CAP__SHIFT 0x00000008
+#define MVP_INBAND_CNTL_CAP__MVP_PASSING_INBAND_CNTL_EN_MASK 0x00000010L
+#define MVP_INBAND_CNTL_CAP__MVP_PASSING_INBAND_CNTL_EN__SHIFT 0x00000004
+#define MVP_RECEIVE_CNT_CNTL1__MVP_SLAVE_DATA_CHK_EN_MASK 0x80000000L
+#define MVP_RECEIVE_CNT_CNTL1__MVP_SLAVE_DATA_CHK_EN__SHIFT 0x0000001f
+#define MVP_RECEIVE_CNT_CNTL1__MVP_SLAVE_LINE_ERROR_CNT_MASK 0x1fff0000L
+#define MVP_RECEIVE_CNT_CNTL1__MVP_SLAVE_LINE_ERROR_CNT__SHIFT 0x00000010
+#define MVP_RECEIVE_CNT_CNTL1__MVP_SLAVE_PIXEL_ERROR_CNT_MASK 0x00001fffL
+#define MVP_RECEIVE_CNT_CNTL1__MVP_SLAVE_PIXEL_ERROR_CNT__SHIFT 0x00000000
+#define MVP_RECEIVE_CNT_CNTL2__MVP_SLAVE_FRAME_ERROR_CNT_MASK 0x00001fffL
+#define MVP_RECEIVE_CNT_CNTL2__MVP_SLAVE_FRAME_ERROR_CNT_RESET_MASK 0x80000000L
+#define MVP_RECEIVE_CNT_CNTL2__MVP_SLAVE_FRAME_ERROR_CNT_RESET__SHIFT 0x0000001f
+#define MVP_RECEIVE_CNT_CNTL2__MVP_SLAVE_FRAME_ERROR_CNT__SHIFT 0x00000000
+#define MVP_SLAVE_STATUS__MVP_SLAVE_LINES_PER_FRAME_RCVED_MASK 0x1fff0000L
+#define MVP_SLAVE_STATUS__MVP_SLAVE_LINES_PER_FRAME_RCVED__SHIFT 0x00000010
+#define MVP_SLAVE_STATUS__MVP_SLAVE_PIXELS_PER_LINE_RCVED_MASK 0x00001fffL
+#define MVP_SLAVE_STATUS__MVP_SLAVE_PIXELS_PER_LINE_RCVED__SHIFT 0x00000000
+#define MVP_TEST_DEBUG_DATA__MVP_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define MVP_TEST_DEBUG_DATA__MVP_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define MVP_TEST_DEBUG_INDEX__MVP_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define MVP_TEST_DEBUG_INDEX__MVP_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define MVP_TEST_DEBUG_INDEX__MVP_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define MVP_TEST_DEBUG_INDEX__MVP_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define OUTPUT_CSC_C11_C12__OUTPUT_CSC_C11_MASK 0x0000ffffL
+#define OUTPUT_CSC_C11_C12__OUTPUT_CSC_C11__SHIFT 0x00000000
+#define OUTPUT_CSC_C11_C12__OUTPUT_CSC_C12_MASK 0xffff0000L
+#define OUTPUT_CSC_C11_C12__OUTPUT_CSC_C12__SHIFT 0x00000010
+#define OUTPUT_CSC_C13_C14__OUTPUT_CSC_C13_MASK 0x0000ffffL
+#define OUTPUT_CSC_C13_C14__OUTPUT_CSC_C13__SHIFT 0x00000000
+#define OUTPUT_CSC_C13_C14__OUTPUT_CSC_C14_MASK 0xffff0000L
+#define OUTPUT_CSC_C13_C14__OUTPUT_CSC_C14__SHIFT 0x00000010
+#define OUTPUT_CSC_C21_C22__OUTPUT_CSC_C21_MASK 0x0000ffffL
+#define OUTPUT_CSC_C21_C22__OUTPUT_CSC_C21__SHIFT 0x00000000
+#define OUTPUT_CSC_C21_C22__OUTPUT_CSC_C22_MASK 0xffff0000L
+#define OUTPUT_CSC_C21_C22__OUTPUT_CSC_C22__SHIFT 0x00000010
+#define OUTPUT_CSC_C23_C24__OUTPUT_CSC_C23_MASK 0x0000ffffL
+#define OUTPUT_CSC_C23_C24__OUTPUT_CSC_C23__SHIFT 0x00000000
+#define OUTPUT_CSC_C23_C24__OUTPUT_CSC_C24_MASK 0xffff0000L
+#define OUTPUT_CSC_C23_C24__OUTPUT_CSC_C24__SHIFT 0x00000010
+#define OUTPUT_CSC_C31_C32__OUTPUT_CSC_C31_MASK 0x0000ffffL
+#define OUTPUT_CSC_C31_C32__OUTPUT_CSC_C31__SHIFT 0x00000000
+#define OUTPUT_CSC_C31_C32__OUTPUT_CSC_C32_MASK 0xffff0000L
+#define OUTPUT_CSC_C31_C32__OUTPUT_CSC_C32__SHIFT 0x00000010
+#define OUTPUT_CSC_C33_C34__OUTPUT_CSC_C33_MASK 0x0000ffffL
+#define OUTPUT_CSC_C33_C34__OUTPUT_CSC_C33__SHIFT 0x00000000
+#define OUTPUT_CSC_C33_C34__OUTPUT_CSC_C34_MASK 0xffff0000L
+#define OUTPUT_CSC_C33_C34__OUTPUT_CSC_C34__SHIFT 0x00000010
+#define OUTPUT_CSC_CONTROL__OUTPUT_CSC_GRPH_MODE_MASK 0x00000007L
+#define OUTPUT_CSC_CONTROL__OUTPUT_CSC_GRPH_MODE__SHIFT 0x00000000
+#define OUTPUT_CSC_CONTROL__OUTPUT_CSC_OVL_MODE_MASK 0x00000070L
+#define OUTPUT_CSC_CONTROL__OUTPUT_CSC_OVL_MODE__SHIFT 0x00000004
+#define OUT_ROUND_CONTROL__OUT_ROUND_TRUNC_MODE_MASK 0x0000000fL
+#define OUT_ROUND_CONTROL__OUT_ROUND_TRUNC_MODE__SHIFT 0x00000000
+#define OVL_CONTROL1__OVL_ADDRESS_TRANSLATION_ENABLE_MASK 0x00010000L
+#define OVL_CONTROL1__OVL_ADDRESS_TRANSLATION_ENABLE__SHIFT 0x00000010
+#define OVL_CONTROL1__OVL_ARRAY_MODE_MASK 0x00f00000L
+#define OVL_CONTROL1__OVL_ARRAY_MODE__SHIFT 0x00000014
+#define OVL_CONTROL1__OVL_BANK_HEIGHT_MASK 0x00001800L
+#define OVL_CONTROL1__OVL_BANK_HEIGHT__SHIFT 0x0000000b
+#define OVL_CONTROL1__OVL_BANK_WIDTH_MASK 0x000000c0L
+#define OVL_CONTROL1__OVL_BANK_WIDTH__SHIFT 0x00000006
+#define OVL_CONTROL1__OVL_COLOR_EXPANSION_MODE_MASK 0x01000000L
+#define OVL_CONTROL1__OVL_COLOR_EXPANSION_MODE__SHIFT 0x00000018
+#define OVL_CONTROL1__OVL_DEPTH_MASK 0x00000003L
+#define OVL_CONTROL1__OVL_DEPTH__SHIFT 0x00000000
+#define OVL_CONTROL1__OVL_FORMAT_MASK 0x00000700L
+#define OVL_CONTROL1__OVL_FORMAT__SHIFT 0x00000008
+#define OVL_CONTROL1__OVL_MACRO_TILE_ASPECT_MASK 0x000c0000L
+#define OVL_CONTROL1__OVL_MACRO_TILE_ASPECT__SHIFT 0x00000012
+#define OVL_CONTROL1__OVL_NUM_BANKS_MASK 0x0000000cL
+#define OVL_CONTROL1__OVL_NUM_BANKS__SHIFT 0x00000002
+#define OVL_CONTROL1__OVL_PIPE_CONFIG_MASK 0x3e000000L
+#define OVL_CONTROL1__OVL_PIPE_CONFIG__SHIFT 0x00000019
+#define OVL_CONTROL1__OVL_PRIVILEGED_ACCESS_ENABLE_MASK 0x00020000L
+#define OVL_CONTROL1__OVL_PRIVILEGED_ACCESS_ENABLE__SHIFT 0x00000011
+#define OVL_CONTROL1__OVL_TILE_SPLIT_MASK 0x0000e000L
+#define OVL_CONTROL1__OVL_TILE_SPLIT__SHIFT 0x0000000d
+#define OVL_CONTROL1__OVL_Z_MASK 0x00000030L
+#define OVL_CONTROL1__OVL_Z__SHIFT 0x00000004
+#define OVL_CONTROL2__OVL_HALF_RESOLUTION_ENABLE_MASK 0x00000001L
+#define OVL_CONTROL2__OVL_HALF_RESOLUTION_ENABLE__SHIFT 0x00000000
+#define OVL_DFQ_CONTROL__OVL_DFQ_MIN_FREE_ENTRIES_MASK 0x00000700L
+#define OVL_DFQ_CONTROL__OVL_DFQ_MIN_FREE_ENTRIES__SHIFT 0x00000008
+#define OVL_DFQ_CONTROL__OVL_DFQ_RESET_MASK 0x00000001L
+#define OVL_DFQ_CONTROL__OVL_DFQ_RESET__SHIFT 0x00000000
+#define OVL_DFQ_CONTROL__OVL_DFQ_SIZE_MASK 0x00000070L
+#define OVL_DFQ_CONTROL__OVL_DFQ_SIZE__SHIFT 0x00000004
+#define OVL_DFQ_STATUS__OVL_DFQ_NUM_ENTRIES_MASK 0x0000000fL
+#define OVL_DFQ_STATUS__OVL_DFQ_NUM_ENTRIES__SHIFT 0x00000000
+#define OVL_DFQ_STATUS__OVL_DFQ_RESET_ACK_MASK 0x00000200L
+#define OVL_DFQ_STATUS__OVL_DFQ_RESET_ACK__SHIFT 0x00000009
+#define OVL_DFQ_STATUS__OVL_DFQ_RESET_FLAG_MASK 0x00000100L
+#define OVL_DFQ_STATUS__OVL_DFQ_RESET_FLAG__SHIFT 0x00000008
+#define OVL_DFQ_STATUS__OVL_SECONDARY_DFQ_NUM_ENTRIES_MASK 0x000000f0L
+#define OVL_DFQ_STATUS__OVL_SECONDARY_DFQ_NUM_ENTRIES__SHIFT 0x00000004
+#define OVL_ENABLE__OVL_ENABLE_MASK 0x00000001L
+#define OVL_ENABLE__OVL_ENABLE__SHIFT 0x00000000
+#define OVL_ENABLE__OVLSCL_EN_MASK 0x00000100L
+#define OVL_ENABLE__OVLSCL_EN__SHIFT 0x00000008
+#define OVL_END__OVL_X_END_MASK 0x7fff0000L
+#define OVL_END__OVL_X_END__SHIFT 0x00000010
+#define OVL_END__OVL_Y_END_MASK 0x00007fffL
+#define OVL_END__OVL_Y_END__SHIFT 0x00000000
+#define OVL_PITCH__OVL_PITCH_MASK 0x00007fffL
+#define OVL_PITCH__OVL_PITCH__SHIFT 0x00000000
+#define OVLSCL_EDGE_PIXEL_CNTL__OVLSCL_BLACK_COLOR_BCB_MASK 0x000003ffL
+#define OVLSCL_EDGE_PIXEL_CNTL__OVLSCL_BLACK_COLOR_BCB__SHIFT 0x00000000
+#define OVLSCL_EDGE_PIXEL_CNTL__OVLSCL_BLACK_COLOR_GY_MASK 0x000ffc00L
+#define OVLSCL_EDGE_PIXEL_CNTL__OVLSCL_BLACK_COLOR_GY__SHIFT 0x0000000a
+#define OVLSCL_EDGE_PIXEL_CNTL__OVLSCL_BLACK_COLOR_RCR_MASK 0x3ff00000L
+#define OVLSCL_EDGE_PIXEL_CNTL__OVLSCL_BLACK_COLOR_RCR__SHIFT 0x00000014
+#define OVLSCL_EDGE_PIXEL_CNTL__OVLSCL_EDGE_PIXEL_SEL_MASK 0x80000000L
+#define OVLSCL_EDGE_PIXEL_CNTL__OVLSCL_EDGE_PIXEL_SEL__SHIFT 0x0000001f
+#define OVL_SECONDARY_SURFACE_ADDRESS_HIGH__OVL_SECONDARY_SURFACE_ADDRESS_HIGH_MASK 0x000000ffL
+#define OVL_SECONDARY_SURFACE_ADDRESS_HIGH__OVL_SECONDARY_SURFACE_ADDRESS_HIGH__SHIFT 0x00000000
+#define OVL_SECONDARY_SURFACE_ADDRESS__OVL_SECONDARY_DFQ_ENABLE_MASK 0x00000001L
+#define OVL_SECONDARY_SURFACE_ADDRESS__OVL_SECONDARY_DFQ_ENABLE__SHIFT 0x00000000
+#define OVL_SECONDARY_SURFACE_ADDRESS__OVL_SECONDARY_SURFACE_ADDRESS_MASK 0xffffff00L
+#define OVL_SECONDARY_SURFACE_ADDRESS__OVL_SECONDARY_SURFACE_ADDRESS__SHIFT 0x00000008
+#define OVL_START__OVL_X_START_MASK 0x3fff0000L
+#define OVL_START__OVL_X_START__SHIFT 0x00000010
+#define OVL_START__OVL_Y_START_MASK 0x00003fffL
+#define OVL_START__OVL_Y_START__SHIFT 0x00000000
+#define OVL_STEREOSYNC_FLIP__OVL_PRIMARY_SURFACE_PENDING_MASK 0x00010000L
+#define OVL_STEREOSYNC_FLIP__OVL_PRIMARY_SURFACE_PENDING__SHIFT 0x00000010
+#define OVL_STEREOSYNC_FLIP__OVL_SECONDARY_SURFACE_PENDING_MASK 0x00020000L
+#define OVL_STEREOSYNC_FLIP__OVL_SECONDARY_SURFACE_PENDING__SHIFT 0x00000011
+#define OVL_STEREOSYNC_FLIP__OVL_STEREOSYNC_FLIP_EN_MASK 0x00000001L
+#define OVL_STEREOSYNC_FLIP__OVL_STEREOSYNC_FLIP_EN__SHIFT 0x00000000
+#define OVL_STEREOSYNC_FLIP__OVL_STEREOSYNC_FLIP_MODE_MASK 0x00000300L
+#define OVL_STEREOSYNC_FLIP__OVL_STEREOSYNC_FLIP_MODE__SHIFT 0x00000008
+#define OVL_STEREOSYNC_FLIP__OVL_STEREOSYNC_SELECT_DISABLE_MASK 0x10000000L
+#define OVL_STEREOSYNC_FLIP__OVL_STEREOSYNC_SELECT_DISABLE__SHIFT 0x0000001c
+#define OVL_SURFACE_ADDRESS_HIGH_INUSE__OVL_SURFACE_ADDRESS_HIGH_INUSE_MASK 0x000000ffL
+#define OVL_SURFACE_ADDRESS_HIGH_INUSE__OVL_SURFACE_ADDRESS_HIGH_INUSE__SHIFT 0x00000000
+#define OVL_SURFACE_ADDRESS_HIGH__OVL_SURFACE_ADDRESS_HIGH_MASK 0x000000ffL
+#define OVL_SURFACE_ADDRESS_HIGH__OVL_SURFACE_ADDRESS_HIGH__SHIFT 0x00000000
+#define OVL_SURFACE_ADDRESS_INUSE__OVL_SURFACE_ADDRESS_INUSE_MASK 0xffffff00L
+#define OVL_SURFACE_ADDRESS_INUSE__OVL_SURFACE_ADDRESS_INUSE__SHIFT 0x00000008
+#define OVL_SURFACE_ADDRESS__OVL_DFQ_ENABLE_MASK 0x00000001L
+#define OVL_SURFACE_ADDRESS__OVL_DFQ_ENABLE__SHIFT 0x00000000
+#define OVL_SURFACE_ADDRESS__OVL_SURFACE_ADDRESS_MASK 0xffffff00L
+#define OVL_SURFACE_ADDRESS__OVL_SURFACE_ADDRESS__SHIFT 0x00000008
+#define OVL_SURFACE_OFFSET_X__OVL_SURFACE_OFFSET_X_MASK 0x00003fffL
+#define OVL_SURFACE_OFFSET_X__OVL_SURFACE_OFFSET_X__SHIFT 0x00000000
+#define OVL_SURFACE_OFFSET_Y__OVL_SURFACE_OFFSET_Y_MASK 0x00003fffL
+#define OVL_SURFACE_OFFSET_Y__OVL_SURFACE_OFFSET_Y__SHIFT 0x00000000
+#define OVL_SWAP_CNTL__OVL_ALPHA_CROSSBAR_MASK 0x00000c00L
+#define OVL_SWAP_CNTL__OVL_ALPHA_CROSSBAR__SHIFT 0x0000000a
+#define OVL_SWAP_CNTL__OVL_BLUE_CROSSBAR_MASK 0x00000300L
+#define OVL_SWAP_CNTL__OVL_BLUE_CROSSBAR__SHIFT 0x00000008
+#define OVL_SWAP_CNTL__OVL_ENDIAN_SWAP_MASK 0x00000003L
+#define OVL_SWAP_CNTL__OVL_ENDIAN_SWAP__SHIFT 0x00000000
+#define OVL_SWAP_CNTL__OVL_GREEN_CROSSBAR_MASK 0x000000c0L
+#define OVL_SWAP_CNTL__OVL_GREEN_CROSSBAR__SHIFT 0x00000006
+#define OVL_SWAP_CNTL__OVL_RED_CROSSBAR_MASK 0x00000030L
+#define OVL_SWAP_CNTL__OVL_RED_CROSSBAR__SHIFT 0x00000004
+#define OVL_UPDATE__OVL_DISABLE_MULTIPLE_UPDATE_MASK 0x01000000L
+#define OVL_UPDATE__OVL_DISABLE_MULTIPLE_UPDATE__SHIFT 0x00000018
+#define OVL_UPDATE__OVL_UPDATE_LOCK_MASK 0x00010000L
+#define OVL_UPDATE__OVL_UPDATE_LOCK__SHIFT 0x00000010
+#define OVL_UPDATE__OVL_UPDATE_PENDING_MASK 0x00000001L
+#define OVL_UPDATE__OVL_UPDATE_PENDING__SHIFT 0x00000000
+#define OVL_UPDATE__OVL_UPDATE_TAKEN_MASK 0x00000002L
+#define OVL_UPDATE__OVL_UPDATE_TAKEN__SHIFT 0x00000001
+#define PHY_AUX_CNTL__AUX_PAD_RXSEL_MASK 0x00010000L
+#define PHY_AUX_CNTL__AUX_PAD_RXSEL__SHIFT 0x00000010
+#define PHY_AUX_CNTL__AUX_PAD_SLEWN_MASK 0x00001000L
+#define PHY_AUX_CNTL__AUX_PAD_SLEWN__SHIFT 0x0000000c
+#define PHY_AUX_CNTL__AUX_PAD_WAKE_MASK 0x00004000L
+#define PHY_AUX_CNTL__AUX_PAD_WAKE__SHIFT 0x0000000e
+#define PIPE0_ARBITRATION_CONTROL3__EFFICIENCY_WEIGHT_MASK 0x0000ffffL
+#define PIPE0_ARBITRATION_CONTROL3__EFFICIENCY_WEIGHT__SHIFT 0x00000000
+#define PIPE0_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED_MASK 0x00000007L
+#define PIPE0_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED__SHIFT 0x00000000
+#define PIPE0_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED_MASK 0x00000010L
+#define PIPE0_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED__SHIFT 0x00000004
+#define PIPE0_MAX_REQUESTS__MAX_REQUESTS_MASK 0x000003ffL
+#define PIPE0_MAX_REQUESTS__MAX_REQUESTS__SHIFT 0x00000000
+#define PIPE0_PG_CONFIG__PIPE0_POWER_FORCEON_MASK 0x00000001L
+#define PIPE0_PG_CONFIG__PIPE0_POWER_FORCEON__SHIFT 0x00000000
+#define PIPE0_PG_ENABLE__PIPE0_POWER_GATE_MASK 0x00000001L
+#define PIPE0_PG_ENABLE__PIPE0_POWER_GATE__SHIFT 0x00000000
+#define PIPE0_PG_STATUS__PIPE0_DESIRED_PWR_STATE_MASK 0x10000000L
+#define PIPE0_PG_STATUS__PIPE0_DESIRED_PWR_STATE__SHIFT 0x0000001c
+#define PIPE0_PG_STATUS__PIPE0_PGFSM_PWR_STATUS_MASK 0xc0000000L
+#define PIPE0_PG_STATUS__PIPE0_PGFSM_PWR_STATUS__SHIFT 0x0000001e
+#define PIPE0_PG_STATUS__PIPE0_PGFSM_READ_DATA_MASK 0x00ffffffL
+#define PIPE0_PG_STATUS__PIPE0_PGFSM_READ_DATA__SHIFT 0x00000000
+#define PIPE0_PG_STATUS__PIPE0_REQUESTED_PWR_STATE_MASK 0x20000000L
+#define PIPE0_PG_STATUS__PIPE0_REQUESTED_PWR_STATE__SHIFT 0x0000001d
+#define PIPE1_ARBITRATION_CONTROL3__EFFICIENCY_WEIGHT_MASK 0x0000ffffL
+#define PIPE1_ARBITRATION_CONTROL3__EFFICIENCY_WEIGHT__SHIFT 0x00000000
+#define PIPE1_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED_MASK 0x00000007L
+#define PIPE1_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED__SHIFT 0x00000000
+#define PIPE1_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED_MASK 0x00000010L
+#define PIPE1_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED__SHIFT 0x00000004
+#define PIPE1_MAX_REQUESTS__MAX_REQUESTS_MASK 0x000003ffL
+#define PIPE1_MAX_REQUESTS__MAX_REQUESTS__SHIFT 0x00000000
+#define PIPE1_PG_CONFIG__PIPE1_POWER_FORCEON_MASK 0x00000001L
+#define PIPE1_PG_CONFIG__PIPE1_POWER_FORCEON__SHIFT 0x00000000
+#define PIPE1_PG_ENABLE__PIPE1_POWER_GATE_MASK 0x00000001L
+#define PIPE1_PG_ENABLE__PIPE1_POWER_GATE__SHIFT 0x00000000
+#define PIPE1_PG_STATUS__PIPE1_DESIRED_PWR_STATE_MASK 0x10000000L
+#define PIPE1_PG_STATUS__PIPE1_DESIRED_PWR_STATE__SHIFT 0x0000001c
+#define PIPE1_PG_STATUS__PIPE1_PGFSM_PWR_STATUS_MASK 0xc0000000L
+#define PIPE1_PG_STATUS__PIPE1_PGFSM_PWR_STATUS__SHIFT 0x0000001e
+#define PIPE1_PG_STATUS__PIPE1_PGFSM_READ_DATA_MASK 0x00ffffffL
+#define PIPE1_PG_STATUS__PIPE1_PGFSM_READ_DATA__SHIFT 0x00000000
+#define PIPE1_PG_STATUS__PIPE1_REQUESTED_PWR_STATE_MASK 0x20000000L
+#define PIPE1_PG_STATUS__PIPE1_REQUESTED_PWR_STATE__SHIFT 0x0000001d
+#define PIPE2_ARBITRATION_CONTROL3__EFFICIENCY_WEIGHT_MASK 0x0000ffffL
+#define PIPE2_ARBITRATION_CONTROL3__EFFICIENCY_WEIGHT__SHIFT 0x00000000
+#define PIPE2_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED_MASK 0x00000007L
+#define PIPE2_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED__SHIFT 0x00000000
+#define PIPE2_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED_MASK 0x00000010L
+#define PIPE2_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED__SHIFT 0x00000004
+#define PIPE2_MAX_REQUESTS__MAX_REQUESTS_MASK 0x000003ffL
+#define PIPE2_MAX_REQUESTS__MAX_REQUESTS__SHIFT 0x00000000
+#define PIPE2_PG_CONFIG__PIPE2_POWER_FORCEON_MASK 0x00000001L
+#define PIPE2_PG_CONFIG__PIPE2_POWER_FORCEON__SHIFT 0x00000000
+#define PIPE2_PG_ENABLE__PIPE2_POWER_GATE_MASK 0x00000001L
+#define PIPE2_PG_ENABLE__PIPE2_POWER_GATE__SHIFT 0x00000000
+#define PIPE2_PG_STATUS__PIPE2_DESIRED_PWR_STATE_MASK 0x10000000L
+#define PIPE2_PG_STATUS__PIPE2_DESIRED_PWR_STATE__SHIFT 0x0000001c
+#define PIPE2_PG_STATUS__PIPE2_PGFSM_PWR_STATUS_MASK 0xc0000000L
+#define PIPE2_PG_STATUS__PIPE2_PGFSM_PWR_STATUS__SHIFT 0x0000001e
+#define PIPE2_PG_STATUS__PIPE2_PGFSM_READ_DATA_MASK 0x00ffffffL
+#define PIPE2_PG_STATUS__PIPE2_PGFSM_READ_DATA__SHIFT 0x00000000
+#define PIPE2_PG_STATUS__PIPE2_REQUESTED_PWR_STATE_MASK 0x20000000L
+#define PIPE2_PG_STATUS__PIPE2_REQUESTED_PWR_STATE__SHIFT 0x0000001d
+#define PIPE3_ARBITRATION_CONTROL3__EFFICIENCY_WEIGHT_MASK 0x0000ffffL
+#define PIPE3_ARBITRATION_CONTROL3__EFFICIENCY_WEIGHT__SHIFT 0x00000000
+#define PIPE3_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED_MASK 0x00000007L
+#define PIPE3_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED__SHIFT 0x00000000
+#define PIPE3_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED_MASK 0x00000010L
+#define PIPE3_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED__SHIFT 0x00000004
+#define PIPE3_MAX_REQUESTS__MAX_REQUESTS_MASK 0x000003ffL
+#define PIPE3_MAX_REQUESTS__MAX_REQUESTS__SHIFT 0x00000000
+#define PIPE3_PG_CONFIG__PIPE3_POWER_FORCEON_MASK 0x00000001L
+#define PIPE3_PG_CONFIG__PIPE3_POWER_FORCEON__SHIFT 0x00000000
+#define PIPE3_PG_ENABLE__PIPE3_POWER_GATE_MASK 0x00000001L
+#define PIPE3_PG_ENABLE__PIPE3_POWER_GATE__SHIFT 0x00000000
+#define PIPE3_PG_STATUS__PIPE3_DESIRED_PWR_STATE_MASK 0x10000000L
+#define PIPE3_PG_STATUS__PIPE3_DESIRED_PWR_STATE__SHIFT 0x0000001c
+#define PIPE3_PG_STATUS__PIPE3_PGFSM_PWR_STATUS_MASK 0xc0000000L
+#define PIPE3_PG_STATUS__PIPE3_PGFSM_PWR_STATUS__SHIFT 0x0000001e
+#define PIPE3_PG_STATUS__PIPE3_PGFSM_READ_DATA_MASK 0x00ffffffL
+#define PIPE3_PG_STATUS__PIPE3_PGFSM_READ_DATA__SHIFT 0x00000000
+#define PIPE3_PG_STATUS__PIPE3_REQUESTED_PWR_STATE_MASK 0x20000000L
+#define PIPE3_PG_STATUS__PIPE3_REQUESTED_PWR_STATE__SHIFT 0x0000001d
+#define PIPE4_ARBITRATION_CONTROL3__EFFICIENCY_WEIGHT_MASK 0x0000ffffL
+#define PIPE4_ARBITRATION_CONTROL3__EFFICIENCY_WEIGHT__SHIFT 0x00000000
+#define PIPE4_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED_MASK 0x00000007L
+#define PIPE4_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED__SHIFT 0x00000000
+#define PIPE4_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED_MASK 0x00000010L
+#define PIPE4_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED__SHIFT 0x00000004
+#define PIPE4_MAX_REQUESTS__MAX_REQUESTS_MASK 0x000003ffL
+#define PIPE4_MAX_REQUESTS__MAX_REQUESTS__SHIFT 0x00000000
+#define PIPE4_PG_CONFIG__PIPE4_POWER_FORCEON_MASK 0x00000001L
+#define PIPE4_PG_CONFIG__PIPE4_POWER_FORCEON__SHIFT 0x00000000
+#define PIPE4_PG_ENABLE__PIPE4_POWER_GATE_MASK 0x00000001L
+#define PIPE4_PG_ENABLE__PIPE4_POWER_GATE__SHIFT 0x00000000
+#define PIPE4_PG_STATUS__PIPE4_DESIRED_PWR_STATE_MASK 0x10000000L
+#define PIPE4_PG_STATUS__PIPE4_DESIRED_PWR_STATE__SHIFT 0x0000001c
+#define PIPE4_PG_STATUS__PIPE4_PGFSM_PWR_STATUS_MASK 0xc0000000L
+#define PIPE4_PG_STATUS__PIPE4_PGFSM_PWR_STATUS__SHIFT 0x0000001e
+#define PIPE4_PG_STATUS__PIPE4_PGFSM_READ_DATA_MASK 0x00ffffffL
+#define PIPE4_PG_STATUS__PIPE4_PGFSM_READ_DATA__SHIFT 0x00000000
+#define PIPE4_PG_STATUS__PIPE4_REQUESTED_PWR_STATE_MASK 0x20000000L
+#define PIPE4_PG_STATUS__PIPE4_REQUESTED_PWR_STATE__SHIFT 0x0000001d
+#define PIPE5_ARBITRATION_CONTROL3__EFFICIENCY_WEIGHT_MASK 0x0000ffffL
+#define PIPE5_ARBITRATION_CONTROL3__EFFICIENCY_WEIGHT__SHIFT 0x00000000
+#define PIPE5_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED_MASK 0x00000007L
+#define PIPE5_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATED__SHIFT 0x00000000
+#define PIPE5_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED_MASK 0x00000010L
+#define PIPE5_DMIF_BUFFER_CONTROL__DMIF_BUFFERS_ALLOCATION_COMPLETED__SHIFT 0x00000004
+#define PIPE5_MAX_REQUESTS__MAX_REQUESTS_MASK 0x000003ffL
+#define PIPE5_MAX_REQUESTS__MAX_REQUESTS__SHIFT 0x00000000
+#define PIPE5_PG_CONFIG__PIPE5_POWER_FORCEON_MASK 0x00000001L
+#define PIPE5_PG_CONFIG__PIPE5_POWER_FORCEON__SHIFT 0x00000000
+#define PIPE5_PG_ENABLE__PIPE5_POWER_GATE_MASK 0x00000001L
+#define PIPE5_PG_ENABLE__PIPE5_POWER_GATE__SHIFT 0x00000000
+#define PIPE5_PG_STATUS__PIPE5_DESIRED_PWR_STATE_MASK 0x10000000L
+#define PIPE5_PG_STATUS__PIPE5_DESIRED_PWR_STATE__SHIFT 0x0000001c
+#define PIPE5_PG_STATUS__PIPE5_PGFSM_PWR_STATUS_MASK 0xc0000000L
+#define PIPE5_PG_STATUS__PIPE5_PGFSM_PWR_STATUS__SHIFT 0x0000001e
+#define PIPE5_PG_STATUS__PIPE5_PGFSM_READ_DATA_MASK 0x00ffffffL
+#define PIPE5_PG_STATUS__PIPE5_PGFSM_READ_DATA__SHIFT 0x00000000
+#define PIPE5_PG_STATUS__PIPE5_REQUESTED_PWR_STATE_MASK 0x20000000L
+#define PIPE5_PG_STATUS__PIPE5_REQUESTED_PWR_STATE__SHIFT 0x0000001d
+#define PIXCLK0_RESYNC_CNTL__DCCG_DEEP_COLOR_CNTL0_MASK 0x00000030L
+#define PIXCLK0_RESYNC_CNTL__DCCG_DEEP_COLOR_CNTL0__SHIFT 0x00000004
+#define PIXCLK0_RESYNC_CNTL__PIXCLK0_RESYNC_ENABLE_MASK 0x00000001L
+#define PIXCLK0_RESYNC_CNTL__PIXCLK0_RESYNC_ENABLE__SHIFT 0x00000000
+#define PIXCLK1_RESYNC_CNTL__DCCG_DEEP_COLOR_CNTL1_MASK 0x00000030L
+#define PIXCLK1_RESYNC_CNTL__DCCG_DEEP_COLOR_CNTL1__SHIFT 0x00000004
+#define PIXCLK1_RESYNC_CNTL__PIXCLK1_RESYNC_ENABLE_MASK 0x00000001L
+#define PIXCLK1_RESYNC_CNTL__PIXCLK1_RESYNC_ENABLE__SHIFT 0x00000000
+#define PIXCLK2_RESYNC_CNTL__DCCG_DEEP_COLOR_CNTL2_MASK 0x00000030L
+#define PIXCLK2_RESYNC_CNTL__DCCG_DEEP_COLOR_CNTL2__SHIFT 0x00000004
+#define PIXCLK2_RESYNC_CNTL__PIXCLK2_RESYNC_ENABLE_MASK 0x00000001L
+#define PIXCLK2_RESYNC_CNTL__PIXCLK2_RESYNC_ENABLE__SHIFT 0x00000000
+#define PLL_ANALOG__PLL_CAL_MODE_MASK 0x0000001fL
+#define PLL_ANALOG__PLL_CAL_MODE__SHIFT 0x00000000
+#define PLL_ANALOG__PLL_CP_MASK 0x00000f00L
+#define PLL_ANALOG__PLL_CP__SHIFT 0x00000008
+#define PLL_ANALOG__PLL_IBIAS_MASK 0xff000000L
+#define PLL_ANALOG__PLL_IBIAS__SHIFT 0x00000018
+#define PLL_ANALOG__PLL_LF_MODE_MASK 0x001ff000L
+#define PLL_ANALOG__PLL_LF_MODE__SHIFT 0x0000000c
+#define PLL_ANALOG__PLL_PFD_PULSE_SEL_MASK 0x00000060L
+#define PLL_ANALOG__PLL_PFD_PULSE_SEL__SHIFT 0x00000005
+#define PLL_CNTL__PLL_ANTIGLITCH_RESETB_MASK 0x00000080L
+#define PLL_CNTL__PLL_ANTIGLITCH_RESETB__SHIFT 0x00000007
+#define PLL_CNTL__PLL_ANTI_GLITCH_RESET_MASK 0x00002000L
+#define PLL_CNTL__PLL_ANTI_GLITCH_RESET__SHIFT 0x0000000d
+#define PLL_CNTL__PLL_BYPASS_CAL_MASK 0x00000004L
+#define PLL_CNTL__PLL_BYPASS_CAL__SHIFT 0x00000002
+#define PLL_CNTL__PLL_CAL_BYPASS_REFDIV_MASK 0x00000400L
+#define PLL_CNTL__PLL_CAL_BYPASS_REFDIV__SHIFT 0x0000000a
+#define PLL_CNTL__PLL_CALIB_DONE_MASK 0x00100000L
+#define PLL_CNTL__PLL_CALIB_DONE__SHIFT 0x00000014
+#define PLL_CNTL__PLL_CALREF_MASK 0x00000300L
+#define PLL_CNTL__PLL_CALREF__SHIFT 0x00000008
+#define PLL_CNTL__PLL_DIG_SPARE_MASK 0xfc000000L
+#define PLL_CNTL__PLL_DIG_SPARE__SHIFT 0x0000001a
+#define PLL_CNTL__PLL_LOCKED_MASK 0x00200000L
+#define PLL_CNTL__PLL_LOCKED__SHIFT 0x00000015
+#define PLL_CNTL__PLL_LOCK_FREQ_SEL_MASK 0x00080000L
+#define PLL_CNTL__PLL_LOCK_FREQ_SEL__SHIFT 0x00000013
+#define PLL_CNTL__PLL_PCIE_REFCLK_SEL_MASK 0x00000040L
+#define PLL_CNTL__PLL_PCIE_REFCLK_SEL__SHIFT 0x00000006
+#define PLL_CNTL__PLL_POST_DIV_SRC_MASK 0x00000008L
+#define PLL_CNTL__PLL_POST_DIV_SRC__SHIFT 0x00000003
+#define PLL_CNTL__PLL_POWER_DOWN_MASK 0x00000002L
+#define PLL_CNTL__PLL_POWER_DOWN__SHIFT 0x00000001
+#define PLL_CNTL__PLL_REFCLK_SEL_MASK 0x00001800L
+#define PLL_CNTL__PLL_REFCLK_SEL__SHIFT 0x0000000b
+#define PLL_CNTL__PLL_REF_DIV_SRC_MASK 0x00070000L
+#define PLL_CNTL__PLL_REF_DIV_SRC__SHIFT 0x00000010
+#define PLL_CNTL__PLL_RESET_MASK 0x00000001L
+#define PLL_CNTL__PLL_RESET__SHIFT 0x00000000
+#define PLL_CNTL__PLL_TIMING_MODE_STATUS_MASK 0x03000000L
+#define PLL_CNTL__PLL_TIMING_MODE_STATUS__SHIFT 0x00000018
+#define PLL_CNTL__PLL_VCOREF_MASK 0x00000030L
+#define PLL_CNTL__PLL_VCOREF__SHIFT 0x00000004
+#define PLL_DEBUG_CNTL__PLL_DEBUG_CLK_SEL_MASK 0x00000f00L
+#define PLL_DEBUG_CNTL__PLL_DEBUG_CLK_SEL__SHIFT 0x00000008
+#define PLL_DEBUG_CNTL__PLL_DEBUG_MUXOUT_SEL_MASK 0x000000f0L
+#define PLL_DEBUG_CNTL__PLL_DEBUG_MUXOUT_SEL__SHIFT 0x00000004
+#define PLL_DEBUG_CNTL__PLL_DEBUG_SIGNALS_ENABLE_MASK 0x00000001L
+#define PLL_DEBUG_CNTL__PLL_DEBUG_SIGNALS_ENABLE__SHIFT 0x00000000
+#define PLL_DISPCLK_CURRENT_DTO_PHASE__PLL_DISPCLK_CURRENT_DTO_PHASE_MASK 0x000001ffL
+#define PLL_DISPCLK_CURRENT_DTO_PHASE__PLL_DISPCLK_CURRENT_DTO_PHASE__SHIFT 0x00000000
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_COMPL_DELAY_MASK 0xff000000L
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_COMPL_DELAY__SHIFT 0x00000018
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_DIS_MASK 0x00010000L
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_DIS__SHIFT 0x00000010
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_PHASE_MASK 0x000001ffL
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_PHASE__SHIFT 0x00000000
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_UPDATE_ACK_MASK 0x00400000L
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_UPDATE_ACK__SHIFT 0x00000016
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_UPDATE_MODE_MASK 0x00060000L
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_UPDATE_MODE__SHIFT 0x00000011
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_UPDATE_PENDING_MASK 0x00100000L
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_UPDATE_PENDING__SHIFT 0x00000014
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_UPDATE_REQ_MASK 0x00200000L
+#define PLL_DISPCLK_DTO_CNTL__PLL_DISPCLK_DTO_UPDATE_REQ__SHIFT 0x00000015
+#define PLL_DS_CNTL__PLL_DS_FRAC_MASK 0x0000ffffL
+#define PLL_DS_CNTL__PLL_DS_FRAC__SHIFT 0x00000000
+#define PLL_DS_CNTL__PLL_DS_MODE_MASK 0x00040000L
+#define PLL_DS_CNTL__PLL_DS_MODE__SHIFT 0x00000012
+#define PLL_DS_CNTL__PLL_DS_ORDER_MASK 0x00030000L
+#define PLL_DS_CNTL__PLL_DS_ORDER__SHIFT 0x00000010
+#define PLL_DS_CNTL__PLL_DS_PRBS_EN_MASK 0x00080000L
+#define PLL_DS_CNTL__PLL_DS_PRBS_EN__SHIFT 0x00000013
+#define PLL_FB_DIV__PLL_FB_DIV_FRACTION_CNTL_MASK 0x00000030L
+#define PLL_FB_DIV__PLL_FB_DIV_FRACTION_CNTL__SHIFT 0x00000004
+#define PLL_FB_DIV__PLL_FB_DIV_FRACTION_MASK 0x0000000fL
+#define PLL_FB_DIV__PLL_FB_DIV_FRACTION__SHIFT 0x00000000
+#define PLL_FB_DIV__PLL_FB_DIV_MASK 0x0fff0000L
+#define PLL_FB_DIV__PLL_FB_DIV__SHIFT 0x00000010
+#define PLL_IDCLK_CNTL__PLL_DIFF_POST_DIV_MASK 0x000f0000L
+#define PLL_IDCLK_CNTL__PLL_DIFF_POST_DIV_RESET_MASK 0x00000100L
+#define PLL_IDCLK_CNTL__PLL_DIFF_POST_DIV_RESET__SHIFT 0x00000008
+#define PLL_IDCLK_CNTL__PLL_DIFF_POST_DIV_SELECT_MASK 0x00001000L
+#define PLL_IDCLK_CNTL__PLL_DIFF_POST_DIV_SELECT__SHIFT 0x0000000c
+#define PLL_IDCLK_CNTL__PLL_DIFF_POST_DIV__SHIFT 0x00000010
+#define PLL_IDCLK_CNTL__PLL_LTDP_IDCLK_DIFF_EN_MASK 0x00000002L
+#define PLL_IDCLK_CNTL__PLL_LTDP_IDCLK_DIFF_EN__SHIFT 0x00000001
+#define PLL_IDCLK_CNTL__PLL_LTDP_IDCLK_EN_MASK 0x00000001L
+#define PLL_IDCLK_CNTL__PLL_LTDP_IDCLK_EN__SHIFT 0x00000000
+#define PLL_IDCLK_CNTL__PLL_TMDP_IDCLK_DIFF_EN_MASK 0x00000008L
+#define PLL_IDCLK_CNTL__PLL_TMDP_IDCLK_DIFF_EN__SHIFT 0x00000003
+#define PLL_IDCLK_CNTL__PLL_TMDP_IDCLK_EN_MASK 0x00000004L
+#define PLL_IDCLK_CNTL__PLL_TMDP_IDCLK_EN__SHIFT 0x00000002
+#define PLL_IDCLK_CNTL__PLL_UNIPHY_IDCLK_DIFF_EN_MASK 0x00000010L
+#define PLL_IDCLK_CNTL__PLL_UNIPHY_IDCLK_DIFF_EN__SHIFT 0x00000004
+#define PLL_POST_DIV__PLL_POST_DIV1P5_DISPCLK_MASK 0x00000080L
+#define PLL_POST_DIV__PLL_POST_DIV1P5_DISPCLK__SHIFT 0x00000007
+#define PLL_POST_DIV__PLL_POST_DIV1P5_DPREFCLK_MASK 0x00008000L
+#define PLL_POST_DIV__PLL_POST_DIV1P5_DPREFCLK__SHIFT 0x0000000f
+#define PLL_POST_DIV__PLL_POST_DIV_DVOCLK_MASK 0x00007f00L
+#define PLL_POST_DIV__PLL_POST_DIV_DVOCLK__SHIFT 0x00000008
+#define PLL_POST_DIV__PLL_POST_DIV_IDCLK_MASK 0x007f0000L
+#define PLL_POST_DIV__PLL_POST_DIV_IDCLK__SHIFT 0x00000010
+#define PLL_POST_DIV__PLL_POST_DIV_PIXCLK_MASK 0x0000007fL
+#define PLL_POST_DIV__PLL_POST_DIV_PIXCLK__SHIFT 0x00000000
+#define PLL_REF_DIV__PLL_CALIBRATION_REF_DIV_MASK 0x0000f000L
+#define PLL_REF_DIV__PLL_CALIBRATION_REF_DIV__SHIFT 0x0000000c
+#define PLL_REF_DIV__PLL_REF_DIV_MASK 0x000003ffL
+#define PLL_REF_DIV__PLL_REF_DIV__SHIFT 0x00000000
+#define PLL_SS_AMOUNT_DSFRAC__PLL_SS_AMOUNT_DSFRAC_MASK 0x0000ffffL
+#define PLL_SS_AMOUNT_DSFRAC__PLL_SS_AMOUNT_DSFRAC__SHIFT 0x00000000
+#define PLL_SS_CNTL__PLL_SS_AMOUNT_FBDIV_MASK 0x000000ffL
+#define PLL_SS_CNTL__PLL_SS_AMOUNT_FBDIV__SHIFT 0x00000000
+#define PLL_SS_CNTL__PLL_SS_AMOUNT_NFRAC_SLIP_MASK 0x00000f00L
+#define PLL_SS_CNTL__PLL_SS_AMOUNT_NFRAC_SLIP__SHIFT 0x00000008
+#define PLL_SS_CNTL__PLL_SS_EN_MASK 0x00001000L
+#define PLL_SS_CNTL__PLL_SS_EN__SHIFT 0x0000000c
+#define PLL_SS_CNTL__PLL_SS_MODE_MASK 0x00002000L
+#define PLL_SS_CNTL__PLL_SS_MODE__SHIFT 0x0000000d
+#define PLL_SS_CNTL__PLL_SS_STEP_SIZE_DSFRAC_MASK 0xffff0000L
+#define PLL_SS_CNTL__PLL_SS_STEP_SIZE_DSFRAC__SHIFT 0x00000010
+#define PLL_UNLOCK_DETECT_CNTL__PLL_UNLOCK_DET_COUNT_MASK 0x00000070L
+#define PLL_UNLOCK_DETECT_CNTL__PLL_UNLOCK_DET_COUNT__SHIFT 0x00000004
+#define PLL_UNLOCK_DETECT_CNTL__PLL_UNLOCK_DETECT_ENABLE_MASK 0x00000001L
+#define PLL_UNLOCK_DETECT_CNTL__PLL_UNLOCK_DETECT_ENABLE__SHIFT 0x00000000
+#define PLL_UNLOCK_DETECT_CNTL__PLL_UNLOCK_DET_RES100_SELECT_MASK 0x00000002L
+#define PLL_UNLOCK_DETECT_CNTL__PLL_UNLOCK_DET_RES100_SELECT__SHIFT 0x00000001
+#define PLL_UNLOCK_DETECT_CNTL__PLL_UNLOCK_STICKY_CLEAR_MASK 0x00000008L
+#define PLL_UNLOCK_DETECT_CNTL__PLL_UNLOCK_STICKY_CLEAR__SHIFT 0x00000003
+#define PLL_UNLOCK_DETECT_CNTL__PLL_UNLOCK_STICKY_STATUS_MASK 0x00000004L
+#define PLL_UNLOCK_DETECT_CNTL__PLL_UNLOCK_STICKY_STATUS__SHIFT 0x00000002
+#define PLL_UPDATE_CNTL__PLL_AUTO_RESET_DISABLE_MASK 0x00010000L
+#define PLL_UPDATE_CNTL__PLL_AUTO_RESET_DISABLE__SHIFT 0x00000010
+#define PLL_UPDATE_CNTL__PLL_UPDATE_PENDING_MASK 0x00000001L
+#define PLL_UPDATE_CNTL__PLL_UPDATE_PENDING__SHIFT 0x00000000
+#define PLL_UPDATE_CNTL__PLL_UPDATE_POINT_MASK 0x00000100L
+#define PLL_UPDATE_CNTL__PLL_UPDATE_POINT__SHIFT 0x00000008
+#define PLL_UPDATE_LOCK__PLL_UPDATE_LOCK_MASK 0x00000001L
+#define PLL_UPDATE_LOCK__PLL_UPDATE_LOCK__SHIFT 0x00000000
+#define PLL_VREG_CNTL__PLL_VREF_SEL_MASK 0x04000000L
+#define PLL_VREG_CNTL__PLL_VREF_SEL__SHIFT 0x0000001a
+#define PLL_VREG_CNTL__PLL_VREG_BIAS_MASK 0xf0000000L
+#define PLL_VREG_CNTL__PLL_VREG_BIAS__SHIFT 0x0000001c
+#define PLL_VREG_CNTL__PLL_VREG_CNTL_MASK 0x000fffffL
+#define PLL_VREG_CNTL__PLL_VREG_CNTL__SHIFT 0x00000000
+#define PRESCALE_GRPH_CONTROL__GRPH_PRESCALE_B_SIGN_MASK 0x00000008L
+#define PRESCALE_GRPH_CONTROL__GRPH_PRESCALE_B_SIGN__SHIFT 0x00000003
+#define PRESCALE_GRPH_CONTROL__GRPH_PRESCALE_BYPASS_MASK 0x00000010L
+#define PRESCALE_GRPH_CONTROL__GRPH_PRESCALE_BYPASS__SHIFT 0x00000004
+#define PRESCALE_GRPH_CONTROL__GRPH_PRESCALE_G_SIGN_MASK 0x00000004L
+#define PRESCALE_GRPH_CONTROL__GRPH_PRESCALE_G_SIGN__SHIFT 0x00000002
+#define PRESCALE_GRPH_CONTROL__GRPH_PRESCALE_R_SIGN_MASK 0x00000002L
+#define PRESCALE_GRPH_CONTROL__GRPH_PRESCALE_R_SIGN__SHIFT 0x00000001
+#define PRESCALE_GRPH_CONTROL__GRPH_PRESCALE_SELECT_MASK 0x00000001L
+#define PRESCALE_GRPH_CONTROL__GRPH_PRESCALE_SELECT__SHIFT 0x00000000
+#define PRESCALE_OVL_CONTROL__OVL_PRESCALE_BYPASS_MASK 0x00000010L
+#define PRESCALE_OVL_CONTROL__OVL_PRESCALE_BYPASS__SHIFT 0x00000004
+#define PRESCALE_OVL_CONTROL__OVL_PRESCALE_CB_SIGN_MASK 0x00000002L
+#define PRESCALE_OVL_CONTROL__OVL_PRESCALE_CB_SIGN__SHIFT 0x00000001
+#define PRESCALE_OVL_CONTROL__OVL_PRESCALE_CR_SIGN_MASK 0x00000008L
+#define PRESCALE_OVL_CONTROL__OVL_PRESCALE_CR_SIGN__SHIFT 0x00000003
+#define PRESCALE_OVL_CONTROL__OVL_PRESCALE_SELECT_MASK 0x00000001L
+#define PRESCALE_OVL_CONTROL__OVL_PRESCALE_SELECT__SHIFT 0x00000000
+#define PRESCALE_OVL_CONTROL__OVL_PRESCALE_Y_SIGN_MASK 0x00000004L
+#define PRESCALE_OVL_CONTROL__OVL_PRESCALE_Y_SIGN__SHIFT 0x00000002
+#define PRESCALE_VALUES_GRPH_B__GRPH_PRESCALE_BIAS_B_MASK 0x0000ffffL
+#define PRESCALE_VALUES_GRPH_B__GRPH_PRESCALE_BIAS_B__SHIFT 0x00000000
+#define PRESCALE_VALUES_GRPH_B__GRPH_PRESCALE_SCALE_B_MASK 0xffff0000L
+#define PRESCALE_VALUES_GRPH_B__GRPH_PRESCALE_SCALE_B__SHIFT 0x00000010
+#define PRESCALE_VALUES_GRPH_G__GRPH_PRESCALE_BIAS_G_MASK 0x0000ffffL
+#define PRESCALE_VALUES_GRPH_G__GRPH_PRESCALE_BIAS_G__SHIFT 0x00000000
+#define PRESCALE_VALUES_GRPH_G__GRPH_PRESCALE_SCALE_G_MASK 0xffff0000L
+#define PRESCALE_VALUES_GRPH_G__GRPH_PRESCALE_SCALE_G__SHIFT 0x00000010
+#define PRESCALE_VALUES_GRPH_R__GRPH_PRESCALE_BIAS_R_MASK 0x0000ffffL
+#define PRESCALE_VALUES_GRPH_R__GRPH_PRESCALE_BIAS_R__SHIFT 0x00000000
+#define PRESCALE_VALUES_GRPH_R__GRPH_PRESCALE_SCALE_R_MASK 0xffff0000L
+#define PRESCALE_VALUES_GRPH_R__GRPH_PRESCALE_SCALE_R__SHIFT 0x00000010
+#define PRESCALE_VALUES_OVL_CB__OVL_PRESCALE_BIAS_CB_MASK 0x0000ffffL
+#define PRESCALE_VALUES_OVL_CB__OVL_PRESCALE_BIAS_CB__SHIFT 0x00000000
+#define PRESCALE_VALUES_OVL_CB__OVL_PRESCALE_SCALE_CB_MASK 0xffff0000L
+#define PRESCALE_VALUES_OVL_CB__OVL_PRESCALE_SCALE_CB__SHIFT 0x00000010
+#define PRESCALE_VALUES_OVL_CR__OVL_PRESCALE_BIAS_CR_MASK 0x0000ffffL
+#define PRESCALE_VALUES_OVL_CR__OVL_PRESCALE_BIAS_CR__SHIFT 0x00000000
+#define PRESCALE_VALUES_OVL_CR__OVL_PRESCALE_SCALE_CR_MASK 0xffff0000L
+#define PRESCALE_VALUES_OVL_CR__OVL_PRESCALE_SCALE_CR__SHIFT 0x00000010
+#define PRESCALE_VALUES_OVL_Y__OVL_PRESCALE_BIAS_Y_MASK 0x0000ffffL
+#define PRESCALE_VALUES_OVL_Y__OVL_PRESCALE_BIAS_Y__SHIFT 0x00000000
+#define PRESCALE_VALUES_OVL_Y__OVL_PRESCALE_SCALE_Y_MASK 0xffff0000L
+#define PRESCALE_VALUES_OVL_Y__OVL_PRESCALE_SCALE_Y__SHIFT 0x00000010
+#define REGAMMA_CNTLA_END_CNTL1__REGAMMA_CNTLA_EXP_REGION_END_MASK 0x0000ffffL
+#define REGAMMA_CNTLA_END_CNTL1__REGAMMA_CNTLA_EXP_REGION_END__SHIFT 0x00000000
+#define REGAMMA_CNTLA_END_CNTL2__REGAMMA_CNTLA_EXP_REGION_END_BASE_MASK 0xffff0000L
+#define REGAMMA_CNTLA_END_CNTL2__REGAMMA_CNTLA_EXP_REGION_END_BASE__SHIFT 0x00000010
+#define REGAMMA_CNTLA_END_CNTL2__REGAMMA_CNTLA_EXP_REGION_END_SLOPE_MASK 0x0000ffffL
+#define REGAMMA_CNTLA_END_CNTL2__REGAMMA_CNTLA_EXP_REGION_END_SLOPE__SHIFT 0x00000000
+#define REGAMMA_CNTLA_REGION_0_1__REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLA_REGION_0_1__REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLA_REGION_0_1__REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLA_REGION_0_1__REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLA_REGION_0_1__REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLA_REGION_0_1__REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLA_REGION_0_1__REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLA_REGION_0_1__REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLA_REGION_10_11__REGAMMA_CNTLA_EXP_REGION10_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLA_REGION_10_11__REGAMMA_CNTLA_EXP_REGION10_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLA_REGION_10_11__REGAMMA_CNTLA_EXP_REGION10_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLA_REGION_10_11__REGAMMA_CNTLA_EXP_REGION10_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLA_REGION_10_11__REGAMMA_CNTLA_EXP_REGION11_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLA_REGION_10_11__REGAMMA_CNTLA_EXP_REGION11_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLA_REGION_10_11__REGAMMA_CNTLA_EXP_REGION11_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLA_REGION_10_11__REGAMMA_CNTLA_EXP_REGION11_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLA_REGION_12_13__REGAMMA_CNTLA_EXP_REGION12_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLA_REGION_12_13__REGAMMA_CNTLA_EXP_REGION12_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLA_REGION_12_13__REGAMMA_CNTLA_EXP_REGION12_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLA_REGION_12_13__REGAMMA_CNTLA_EXP_REGION12_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLA_REGION_12_13__REGAMMA_CNTLA_EXP_REGION13_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLA_REGION_12_13__REGAMMA_CNTLA_EXP_REGION13_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLA_REGION_12_13__REGAMMA_CNTLA_EXP_REGION13_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLA_REGION_12_13__REGAMMA_CNTLA_EXP_REGION13_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLA_REGION_14_15__REGAMMA_CNTLA_EXP_REGION14_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLA_REGION_14_15__REGAMMA_CNTLA_EXP_REGION14_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLA_REGION_14_15__REGAMMA_CNTLA_EXP_REGION14_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLA_REGION_14_15__REGAMMA_CNTLA_EXP_REGION14_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLA_REGION_14_15__REGAMMA_CNTLA_EXP_REGION15_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLA_REGION_14_15__REGAMMA_CNTLA_EXP_REGION15_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLA_REGION_14_15__REGAMMA_CNTLA_EXP_REGION15_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLA_REGION_14_15__REGAMMA_CNTLA_EXP_REGION15_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLA_REGION_2_3__REGAMMA_CNTLA_EXP_REGION2_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLA_REGION_2_3__REGAMMA_CNTLA_EXP_REGION2_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLA_REGION_2_3__REGAMMA_CNTLA_EXP_REGION2_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLA_REGION_2_3__REGAMMA_CNTLA_EXP_REGION2_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLA_REGION_2_3__REGAMMA_CNTLA_EXP_REGION3_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLA_REGION_2_3__REGAMMA_CNTLA_EXP_REGION3_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLA_REGION_2_3__REGAMMA_CNTLA_EXP_REGION3_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLA_REGION_2_3__REGAMMA_CNTLA_EXP_REGION3_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLA_REGION_4_5__REGAMMA_CNTLA_EXP_REGION4_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLA_REGION_4_5__REGAMMA_CNTLA_EXP_REGION4_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLA_REGION_4_5__REGAMMA_CNTLA_EXP_REGION4_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLA_REGION_4_5__REGAMMA_CNTLA_EXP_REGION4_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLA_REGION_4_5__REGAMMA_CNTLA_EXP_REGION5_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLA_REGION_4_5__REGAMMA_CNTLA_EXP_REGION5_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLA_REGION_4_5__REGAMMA_CNTLA_EXP_REGION5_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLA_REGION_4_5__REGAMMA_CNTLA_EXP_REGION5_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLA_REGION_6_7__REGAMMA_CNTLA_EXP_REGION6_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLA_REGION_6_7__REGAMMA_CNTLA_EXP_REGION6_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLA_REGION_6_7__REGAMMA_CNTLA_EXP_REGION6_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLA_REGION_6_7__REGAMMA_CNTLA_EXP_REGION6_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLA_REGION_6_7__REGAMMA_CNTLA_EXP_REGION7_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLA_REGION_6_7__REGAMMA_CNTLA_EXP_REGION7_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLA_REGION_6_7__REGAMMA_CNTLA_EXP_REGION7_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLA_REGION_6_7__REGAMMA_CNTLA_EXP_REGION7_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLA_REGION_8_9__REGAMMA_CNTLA_EXP_REGION8_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLA_REGION_8_9__REGAMMA_CNTLA_EXP_REGION8_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLA_REGION_8_9__REGAMMA_CNTLA_EXP_REGION8_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLA_REGION_8_9__REGAMMA_CNTLA_EXP_REGION8_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLA_REGION_8_9__REGAMMA_CNTLA_EXP_REGION9_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLA_REGION_8_9__REGAMMA_CNTLA_EXP_REGION9_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLA_REGION_8_9__REGAMMA_CNTLA_EXP_REGION9_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLA_REGION_8_9__REGAMMA_CNTLA_EXP_REGION9_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLA_SLOPE_CNTL__REGAMMA_CNTLA_EXP_REGION_LINEAR_SLOPE_MASK 0x0003ffffL
+#define REGAMMA_CNTLA_SLOPE_CNTL__REGAMMA_CNTLA_EXP_REGION_LINEAR_SLOPE__SHIFT 0x00000000
+#define REGAMMA_CNTLA_START_CNTL__REGAMMA_CNTLA_EXP_REGION_START_MASK 0x0003ffffL
+#define REGAMMA_CNTLA_START_CNTL__REGAMMA_CNTLA_EXP_REGION_START_SEGMENT_MASK 0x07f00000L
+#define REGAMMA_CNTLA_START_CNTL__REGAMMA_CNTLA_EXP_REGION_START_SEGMENT__SHIFT 0x00000014
+#define REGAMMA_CNTLA_START_CNTL__REGAMMA_CNTLA_EXP_REGION_START__SHIFT 0x00000000
+#define REGAMMA_CNTLB_END_CNTL1__REGAMMA_CNTLB_EXP_REGION_END_MASK 0x0000ffffL
+#define REGAMMA_CNTLB_END_CNTL1__REGAMMA_CNTLB_EXP_REGION_END__SHIFT 0x00000000
+#define REGAMMA_CNTLB_END_CNTL2__REGAMMA_CNTLB_EXP_REGION_END_BASE_MASK 0xffff0000L
+#define REGAMMA_CNTLB_END_CNTL2__REGAMMA_CNTLB_EXP_REGION_END_BASE__SHIFT 0x00000010
+#define REGAMMA_CNTLB_END_CNTL2__REGAMMA_CNTLB_EXP_REGION_END_SLOPE_MASK 0x0000ffffL
+#define REGAMMA_CNTLB_END_CNTL2__REGAMMA_CNTLB_EXP_REGION_END_SLOPE__SHIFT 0x00000000
+#define REGAMMA_CNTLB_REGION_0_1__REGAMMA_CNTLB_EXP_REGION0_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLB_REGION_0_1__REGAMMA_CNTLB_EXP_REGION0_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLB_REGION_0_1__REGAMMA_CNTLB_EXP_REGION0_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLB_REGION_0_1__REGAMMA_CNTLB_EXP_REGION0_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLB_REGION_0_1__REGAMMA_CNTLB_EXP_REGION1_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLB_REGION_0_1__REGAMMA_CNTLB_EXP_REGION1_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLB_REGION_0_1__REGAMMA_CNTLB_EXP_REGION1_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLB_REGION_0_1__REGAMMA_CNTLB_EXP_REGION1_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLB_REGION_10_11__REGAMMA_CNTLB_EXP_REGION10_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLB_REGION_10_11__REGAMMA_CNTLB_EXP_REGION10_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLB_REGION_10_11__REGAMMA_CNTLB_EXP_REGION10_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLB_REGION_10_11__REGAMMA_CNTLB_EXP_REGION10_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLB_REGION_10_11__REGAMMA_CNTLB_EXP_REGION11_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLB_REGION_10_11__REGAMMA_CNTLB_EXP_REGION11_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLB_REGION_10_11__REGAMMA_CNTLB_EXP_REGION11_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLB_REGION_10_11__REGAMMA_CNTLB_EXP_REGION11_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLB_REGION_12_13__REGAMMA_CNTLB_EXP_REGION12_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLB_REGION_12_13__REGAMMA_CNTLB_EXP_REGION12_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLB_REGION_12_13__REGAMMA_CNTLB_EXP_REGION12_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLB_REGION_12_13__REGAMMA_CNTLB_EXP_REGION12_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLB_REGION_12_13__REGAMMA_CNTLB_EXP_REGION13_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLB_REGION_12_13__REGAMMA_CNTLB_EXP_REGION13_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLB_REGION_12_13__REGAMMA_CNTLB_EXP_REGION13_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLB_REGION_12_13__REGAMMA_CNTLB_EXP_REGION13_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLB_REGION_14_15__REGAMMA_CNTLB_EXP_REGION14_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLB_REGION_14_15__REGAMMA_CNTLB_EXP_REGION14_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLB_REGION_14_15__REGAMMA_CNTLB_EXP_REGION14_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLB_REGION_14_15__REGAMMA_CNTLB_EXP_REGION14_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLB_REGION_14_15__REGAMMA_CNTLB_EXP_REGION15_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLB_REGION_14_15__REGAMMA_CNTLB_EXP_REGION15_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLB_REGION_14_15__REGAMMA_CNTLB_EXP_REGION15_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLB_REGION_14_15__REGAMMA_CNTLB_EXP_REGION15_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLB_REGION_2_3__REGAMMA_CNTLB_EXP_REGION2_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLB_REGION_2_3__REGAMMA_CNTLB_EXP_REGION2_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLB_REGION_2_3__REGAMMA_CNTLB_EXP_REGION2_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLB_REGION_2_3__REGAMMA_CNTLB_EXP_REGION2_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLB_REGION_2_3__REGAMMA_CNTLB_EXP_REGION3_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLB_REGION_2_3__REGAMMA_CNTLB_EXP_REGION3_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLB_REGION_2_3__REGAMMA_CNTLB_EXP_REGION3_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLB_REGION_2_3__REGAMMA_CNTLB_EXP_REGION3_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLB_REGION_4_5__REGAMMA_CNTLB_EXP_REGION4_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLB_REGION_4_5__REGAMMA_CNTLB_EXP_REGION4_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLB_REGION_4_5__REGAMMA_CNTLB_EXP_REGION4_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLB_REGION_4_5__REGAMMA_CNTLB_EXP_REGION4_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLB_REGION_4_5__REGAMMA_CNTLB_EXP_REGION5_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLB_REGION_4_5__REGAMMA_CNTLB_EXP_REGION5_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLB_REGION_4_5__REGAMMA_CNTLB_EXP_REGION5_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLB_REGION_4_5__REGAMMA_CNTLB_EXP_REGION5_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLB_REGION_6_7__REGAMMA_CNTLB_EXP_REGION6_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLB_REGION_6_7__REGAMMA_CNTLB_EXP_REGION6_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLB_REGION_6_7__REGAMMA_CNTLB_EXP_REGION6_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLB_REGION_6_7__REGAMMA_CNTLB_EXP_REGION6_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLB_REGION_6_7__REGAMMA_CNTLB_EXP_REGION7_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLB_REGION_6_7__REGAMMA_CNTLB_EXP_REGION7_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLB_REGION_6_7__REGAMMA_CNTLB_EXP_REGION7_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLB_REGION_6_7__REGAMMA_CNTLB_EXP_REGION7_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLB_REGION_8_9__REGAMMA_CNTLB_EXP_REGION8_LUT_OFFSET_MASK 0x000001ffL
+#define REGAMMA_CNTLB_REGION_8_9__REGAMMA_CNTLB_EXP_REGION8_LUT_OFFSET__SHIFT 0x00000000
+#define REGAMMA_CNTLB_REGION_8_9__REGAMMA_CNTLB_EXP_REGION8_NUM_SEGMENTS_MASK 0x00007000L
+#define REGAMMA_CNTLB_REGION_8_9__REGAMMA_CNTLB_EXP_REGION8_NUM_SEGMENTS__SHIFT 0x0000000c
+#define REGAMMA_CNTLB_REGION_8_9__REGAMMA_CNTLB_EXP_REGION9_LUT_OFFSET_MASK 0x01ff0000L
+#define REGAMMA_CNTLB_REGION_8_9__REGAMMA_CNTLB_EXP_REGION9_LUT_OFFSET__SHIFT 0x00000010
+#define REGAMMA_CNTLB_REGION_8_9__REGAMMA_CNTLB_EXP_REGION9_NUM_SEGMENTS_MASK 0x70000000L
+#define REGAMMA_CNTLB_REGION_8_9__REGAMMA_CNTLB_EXP_REGION9_NUM_SEGMENTS__SHIFT 0x0000001c
+#define REGAMMA_CNTLB_SLOPE_CNTL__REGAMMA_CNTLB_EXP_REGION_LINEAR_SLOPE_MASK 0x0003ffffL
+#define REGAMMA_CNTLB_SLOPE_CNTL__REGAMMA_CNTLB_EXP_REGION_LINEAR_SLOPE__SHIFT 0x00000000
+#define REGAMMA_CNTLB_START_CNTL__REGAMMA_CNTLB_EXP_REGION_START_MASK 0x0003ffffL
+#define REGAMMA_CNTLB_START_CNTL__REGAMMA_CNTLB_EXP_REGION_START_SEGMENT_MASK 0x07f00000L
+#define REGAMMA_CNTLB_START_CNTL__REGAMMA_CNTLB_EXP_REGION_START_SEGMENT__SHIFT 0x00000014
+#define REGAMMA_CNTLB_START_CNTL__REGAMMA_CNTLB_EXP_REGION_START__SHIFT 0x00000000
+#define REGAMMA_CONTROL__GRPH_REGAMMA_MODE_MASK 0x00000007L
+#define REGAMMA_CONTROL__GRPH_REGAMMA_MODE__SHIFT 0x00000000
+#define REGAMMA_CONTROL__OVL_REGAMMA_MODE_MASK 0x00000070L
+#define REGAMMA_CONTROL__OVL_REGAMMA_MODE__SHIFT 0x00000004
+#define REGAMMA_LUT_DATA__REGAMMA_LUT_DATA_MASK 0x0007ffffL
+#define REGAMMA_LUT_DATA__REGAMMA_LUT_DATA__SHIFT 0x00000000
+#define REGAMMA_LUT_INDEX__REGAMMA_LUT_INDEX_MASK 0x000001ffL
+#define REGAMMA_LUT_INDEX__REGAMMA_LUT_INDEX__SHIFT 0x00000000
+#define REGAMMA_LUT_WRITE_EN_MASK__REGAMMA_LUT_WRITE_EN_MASK_MASK 0x00000007L
+#define REGAMMA_LUT_WRITE_EN_MASK__REGAMMA_LUT_WRITE_EN_MASK__SHIFT 0x00000000
+#define SCL_ALU_CONTROL__SCL_ALU_DISABLE_MASK 0x00000001L
+#define SCL_ALU_CONTROL__SCL_ALU_DISABLE__SHIFT 0x00000000
+#define SCL_BYPASS_CONTROL__SCL_BYPASS_MODE_MASK 0x00000003L
+#define SCL_BYPASS_CONTROL__SCL_BYPASS_MODE__SHIFT 0x00000000
+#define SCL_COEF_RAM_CONFLICT_STATUS__SCL_HOST_CONFLICT_ACK_MASK 0x00000100L
+#define SCL_COEF_RAM_CONFLICT_STATUS__SCL_HOST_CONFLICT_ACK__SHIFT 0x00000008
+#define SCL_COEF_RAM_CONFLICT_STATUS__SCL_HOST_CONFLICT_FLAG_MASK 0x00000001L
+#define SCL_COEF_RAM_CONFLICT_STATUS__SCL_HOST_CONFLICT_FLAG__SHIFT 0x00000000
+#define SCL_COEF_RAM_CONFLICT_STATUS__SCL_HOST_CONFLICT_INT_STATUS_MASK 0x00010000L
+#define SCL_COEF_RAM_CONFLICT_STATUS__SCL_HOST_CONFLICT_INT_STATUS__SHIFT 0x00000010
+#define SCL_COEF_RAM_CONFLICT_STATUS__SCL_HOST_CONFLICT_MASK_MASK 0x00001000L
+#define SCL_COEF_RAM_CONFLICT_STATUS__SCL_HOST_CONFLICT_MASK__SHIFT 0x0000000c
+#define SCL_COEF_RAM_SELECT__SCL_C_RAM_FILTER_TYPE_MASK 0x00030000L
+#define SCL_COEF_RAM_SELECT__SCL_C_RAM_FILTER_TYPE__SHIFT 0x00000010
+#define SCL_COEF_RAM_SELECT__SCL_C_RAM_PHASE_MASK 0x00000f00L
+#define SCL_COEF_RAM_SELECT__SCL_C_RAM_PHASE__SHIFT 0x00000008
+#define SCL_COEF_RAM_SELECT__SCL_C_RAM_TAP_PAIR_IDX_MASK 0x0000000fL
+#define SCL_COEF_RAM_SELECT__SCL_C_RAM_TAP_PAIR_IDX__SHIFT 0x00000000
+#define SCL_COEF_RAM_TAP_DATA__SCL_C_RAM_EVEN_TAP_COEF_EN_MASK 0x00008000L
+#define SCL_COEF_RAM_TAP_DATA__SCL_C_RAM_EVEN_TAP_COEF_EN__SHIFT 0x0000000f
+#define SCL_COEF_RAM_TAP_DATA__SCL_C_RAM_EVEN_TAP_COEF_MASK 0x00003fffL
+#define SCL_COEF_RAM_TAP_DATA__SCL_C_RAM_EVEN_TAP_COEF__SHIFT 0x00000000
+#define SCL_COEF_RAM_TAP_DATA__SCL_C_RAM_ODD_TAP_COEF_EN_MASK 0x80000000L
+#define SCL_COEF_RAM_TAP_DATA__SCL_C_RAM_ODD_TAP_COEF_EN__SHIFT 0x0000001f
+#define SCL_COEF_RAM_TAP_DATA__SCL_C_RAM_ODD_TAP_COEF_MASK 0x3fff0000L
+#define SCL_COEF_RAM_TAP_DATA__SCL_C_RAM_ODD_TAP_COEF__SHIFT 0x00000010
+#define SCL_DEBUG2__SCL_DEBUG2_MASK 0xffffffffL
+#define SCL_DEBUG2__SCL_DEBUG2__SHIFT 0x00000000
+#define SCL_DEBUG__SCL_DEBUG_MASK 0xffffffffL
+#define SCL_DEBUG__SCL_DEBUG__SHIFT 0x00000000
+#define SCL_F_SHARP_CONTROL__SCL_HF_SHARP_EN_MASK 0x00000010L
+#define SCL_F_SHARP_CONTROL__SCL_HF_SHARP_EN__SHIFT 0x00000004
+#define SCL_F_SHARP_CONTROL__SCL_HF_SHARP_SCALE_FACTOR_MASK 0x00000007L
+#define SCL_F_SHARP_CONTROL__SCL_HF_SHARP_SCALE_FACTOR__SHIFT 0x00000000
+#define SCL_F_SHARP_CONTROL__SCL_VF_SHARP_EN_MASK 0x00001000L
+#define SCL_F_SHARP_CONTROL__SCL_VF_SHARP_EN__SHIFT 0x0000000c
+#define SCL_F_SHARP_CONTROL__SCL_VF_SHARP_SCALE_FACTOR_MASK 0x00000700L
+#define SCL_F_SHARP_CONTROL__SCL_VF_SHARP_SCALE_FACTOR__SHIFT 0x00000008
+#define SCL_HORZ_FILTER_CONTROL__SCL_H_FILTER_PICK_NEAREST_MASK 0x00000001L
+#define SCL_HORZ_FILTER_CONTROL__SCL_H_FILTER_PICK_NEAREST__SHIFT 0x00000000
+#define SCL_HORZ_FILTER_SCALE_RATIO__SCL_H_SCALE_RATIO_MASK 0x03ffffffL
+#define SCL_HORZ_FILTER_SCALE_RATIO__SCL_H_SCALE_RATIO__SHIFT 0x00000000
+#define SCLK_CGTT_BLK_CTRL_REG__SCLK_TURN_OFF_DELAY_MASK 0x00000ff0L
+#define SCLK_CGTT_BLK_CTRL_REG__SCLK_TURN_OFF_DELAY__SHIFT 0x00000004
+#define SCLK_CGTT_BLK_CTRL_REG__SCLK_TURN_ON_DELAY_MASK 0x0000000fL
+#define SCLK_CGTT_BLK_CTRL_REG__SCLK_TURN_ON_DELAY__SHIFT 0x00000000
+#define SCL_MANUAL_REPLICATE_CONTROL__SCL_H_MANUAL_REPLICATE_FACTOR_MASK 0x00000f00L
+#define SCL_MANUAL_REPLICATE_CONTROL__SCL_H_MANUAL_REPLICATE_FACTOR__SHIFT 0x00000008
+#define SCL_MANUAL_REPLICATE_CONTROL__SCL_V_MANUAL_REPLICATE_FACTOR_MASK 0x0000000fL
+#define SCL_MANUAL_REPLICATE_CONTROL__SCL_V_MANUAL_REPLICATE_FACTOR__SHIFT 0x00000000
+#define SCL_MODE_CHANGE_DET1__SCL_ALU_H_SCALE_RATIO_MASK 0x0fffff80L
+#define SCL_MODE_CHANGE_DET1__SCL_ALU_H_SCALE_RATIO__SHIFT 0x00000007
+#define SCL_MODE_CHANGE_DET1__SCL_MODE_CHANGE_ACK_MASK 0x00000010L
+#define SCL_MODE_CHANGE_DET1__SCL_MODE_CHANGE_ACK__SHIFT 0x00000004
+#define SCL_MODE_CHANGE_DET1__SCL_MODE_CHANGE_MASK 0x00000001L
+#define SCL_MODE_CHANGE_DET1__SCL_MODE_CHANGE__SHIFT 0x00000000
+#define SCL_MODE_CHANGE_DET2__SCL_ALU_V_SCALE_RATIO_MASK 0x001fffffL
+#define SCL_MODE_CHANGE_DET2__SCL_ALU_V_SCALE_RATIO__SHIFT 0x00000000
+#define SCL_MODE_CHANGE_DET3__SCL_ALU_SOURCE_HEIGHT_MASK 0x00003fffL
+#define SCL_MODE_CHANGE_DET3__SCL_ALU_SOURCE_HEIGHT__SHIFT 0x00000000
+#define SCL_MODE_CHANGE_DET3__SCL_ALU_SOURCE_WIDTH_MASK 0x3fff0000L
+#define SCL_MODE_CHANGE_DET3__SCL_ALU_SOURCE_WIDTH__SHIFT 0x00000010
+#define SCL_MODE_CHANGE_MASK__SCL_MODE_CHANGE_MASK_MASK 0x00000001L
+#define SCL_MODE_CHANGE_MASK__SCL_MODE_CHANGE_MASK__SHIFT 0x00000000
+#define SCL_TEST_DEBUG_DATA__SCL_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define SCL_TEST_DEBUG_DATA__SCL_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define SCL_TEST_DEBUG_INDEX__SCL_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define SCL_TEST_DEBUG_INDEX__SCL_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define SCL_TEST_DEBUG_INDEX__SCL_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define SCL_TEST_DEBUG_INDEX__SCL_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define SCL_UPDATE__SCL_UPDATE_LOCK_MASK 0x00010000L
+#define SCL_UPDATE__SCL_UPDATE_LOCK__SHIFT 0x00000010
+#define SCL_UPDATE__SCL_UPDATE_PENDING_MASK 0x00000001L
+#define SCL_UPDATE__SCL_UPDATE_PENDING__SHIFT 0x00000000
+#define SCL_UPDATE__SCL_UPDATE_TAKEN_MASK 0x00000100L
+#define SCL_UPDATE__SCL_UPDATE_TAKEN__SHIFT 0x00000008
+#define SCL_VERT_FILTER_CONTROL__SCL_V_FILTER_PICK_NEAREST_MASK 0x00000001L
+#define SCL_VERT_FILTER_CONTROL__SCL_V_FILTER_PICK_NEAREST__SHIFT 0x00000000
+#define SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_FRAC_BOT_MASK 0x0000ffffL
+#define SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_FRAC_BOT__SHIFT 0x00000000
+#define SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_INT_BOT_MASK 0x00070000L
+#define SCL_VERT_FILTER_INIT_BOT__SCL_V_INIT_INT_BOT__SHIFT 0x00000010
+#define SCL_VERT_FILTER_INIT__SCL_V_INIT_FRAC_MASK 0x0000ffffL
+#define SCL_VERT_FILTER_INIT__SCL_V_INIT_FRAC__SHIFT 0x00000000
+#define SCL_VERT_FILTER_INIT__SCL_V_INIT_INT_MASK 0x00070000L
+#define SCL_VERT_FILTER_INIT__SCL_V_INIT_INT__SHIFT 0x00000010
+#define SCL_VERT_FILTER_SCALE_RATIO__SCL_V_SCALE_RATIO_MASK 0x03ffffffL
+#define SCL_VERT_FILTER_SCALE_RATIO__SCL_V_SCALE_RATIO__SHIFT 0x00000000
+#define SEQ00__SEQ_RST0B_MASK 0x00000001L
+#define SEQ00__SEQ_RST0B__SHIFT 0x00000000
+#define SEQ00__SEQ_RST1B_MASK 0x00000002L
+#define SEQ00__SEQ_RST1B__SHIFT 0x00000001
+#define SEQ01__SEQ_DOT8_MASK 0x00000001L
+#define SEQ01__SEQ_DOT8__SHIFT 0x00000000
+#define SEQ01__SEQ_MAXBW_MASK 0x00000020L
+#define SEQ01__SEQ_MAXBW__SHIFT 0x00000005
+#define SEQ01__SEQ_PCLKBY2_MASK 0x00000008L
+#define SEQ01__SEQ_PCLKBY2__SHIFT 0x00000003
+#define SEQ01__SEQ_SHIFT2_MASK 0x00000004L
+#define SEQ01__SEQ_SHIFT2__SHIFT 0x00000002
+#define SEQ01__SEQ_SHIFT4_MASK 0x00000010L
+#define SEQ01__SEQ_SHIFT4__SHIFT 0x00000004
+#define SEQ02__SEQ_MAP0_EN_MASK 0x00000001L
+#define SEQ02__SEQ_MAP0_EN__SHIFT 0x00000000
+#define SEQ02__SEQ_MAP1_EN_MASK 0x00000002L
+#define SEQ02__SEQ_MAP1_EN__SHIFT 0x00000001
+#define SEQ02__SEQ_MAP2_EN_MASK 0x00000004L
+#define SEQ02__SEQ_MAP2_EN__SHIFT 0x00000002
+#define SEQ02__SEQ_MAP3_EN_MASK 0x00000008L
+#define SEQ02__SEQ_MAP3_EN__SHIFT 0x00000003
+#define SEQ03__SEQ_FONT_A0_MASK 0x00000020L
+#define SEQ03__SEQ_FONT_A0__SHIFT 0x00000005
+#define SEQ03__SEQ_FONT_A1_MASK 0x00000004L
+#define SEQ03__SEQ_FONT_A1__SHIFT 0x00000002
+#define SEQ03__SEQ_FONT_A2_MASK 0x00000008L
+#define SEQ03__SEQ_FONT_A2__SHIFT 0x00000003
+#define SEQ03__SEQ_FONT_B0_MASK 0x00000010L
+#define SEQ03__SEQ_FONT_B0__SHIFT 0x00000004
+#define SEQ03__SEQ_FONT_B1_MASK 0x00000001L
+#define SEQ03__SEQ_FONT_B1__SHIFT 0x00000000
+#define SEQ03__SEQ_FONT_B2_MASK 0x00000002L
+#define SEQ03__SEQ_FONT_B2__SHIFT 0x00000001
+#define SEQ04__SEQ_256K_MASK 0x00000002L
+#define SEQ04__SEQ_256K__SHIFT 0x00000001
+#define SEQ04__SEQ_CHAIN_MASK 0x00000008L
+#define SEQ04__SEQ_CHAIN__SHIFT 0x00000003
+#define SEQ04__SEQ_ODDEVEN_MASK 0x00000004L
+#define SEQ04__SEQ_ODDEVEN__SHIFT 0x00000002
+#define SEQ8_DATA__SEQ_DATA_MASK 0x000000ffL
+#define SEQ8_DATA__SEQ_DATA__SHIFT 0x00000000
+#define SEQ8_IDX__SEQ_IDX_MASK 0x00000007L
+#define SEQ8_IDX__SEQ_IDX__SHIFT 0x00000000
+#define SINK_DESCRIPTION0__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION0__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION10__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION10__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION11__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION11__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION12__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION12__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION13__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION13__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION14__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION14__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION15__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION15__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION16__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION16__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION17__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION17__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION1__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION1__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION2__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION2__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION3__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION3__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION4__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION4__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION5__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION5__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION6__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION6__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION7__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION7__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION8__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION8__DESCRIPTION__SHIFT 0x00000000
+#define SINK_DESCRIPTION9__DESCRIPTION_MASK 0x000000ffL
+#define SINK_DESCRIPTION9__DESCRIPTION__SHIFT 0x00000000
+#define SLAVE_COMM_CMD_REG__SLAVE_COMM_CMD_REG_BYTE0_MASK 0x000000ffL
+#define SLAVE_COMM_CMD_REG__SLAVE_COMM_CMD_REG_BYTE0__SHIFT 0x00000000
+#define SLAVE_COMM_CMD_REG__SLAVE_COMM_CMD_REG_BYTE1_MASK 0x0000ff00L
+#define SLAVE_COMM_CMD_REG__SLAVE_COMM_CMD_REG_BYTE1__SHIFT 0x00000008
+#define SLAVE_COMM_CMD_REG__SLAVE_COMM_CMD_REG_BYTE2_MASK 0x00ff0000L
+#define SLAVE_COMM_CMD_REG__SLAVE_COMM_CMD_REG_BYTE2__SHIFT 0x00000010
+#define SLAVE_COMM_CMD_REG__SLAVE_COMM_CMD_REG_BYTE3_MASK 0xff000000L
+#define SLAVE_COMM_CMD_REG__SLAVE_COMM_CMD_REG_BYTE3__SHIFT 0x00000018
+#define SLAVE_COMM_CNTL_REG__COMM_PORT_MSG_TO_HOST_IN_PROGRESS_MASK 0x00000100L
+#define SLAVE_COMM_CNTL_REG__COMM_PORT_MSG_TO_HOST_IN_PROGRESS__SHIFT 0x00000008
+#define SLAVE_COMM_CNTL_REG__SLAVE_COMM_INTERRUPT_MASK 0x00000001L
+#define SLAVE_COMM_CNTL_REG__SLAVE_COMM_INTERRUPT__SHIFT 0x00000000
+#define SLAVE_COMM_DATA_REG1__SLAVE_COMM_DATA_REG1_BYTE0_MASK 0x000000ffL
+#define SLAVE_COMM_DATA_REG1__SLAVE_COMM_DATA_REG1_BYTE0__SHIFT 0x00000000
+#define SLAVE_COMM_DATA_REG1__SLAVE_COMM_DATA_REG1_BYTE1_MASK 0x0000ff00L
+#define SLAVE_COMM_DATA_REG1__SLAVE_COMM_DATA_REG1_BYTE1__SHIFT 0x00000008
+#define SLAVE_COMM_DATA_REG1__SLAVE_COMM_DATA_REG1_BYTE2_MASK 0x00ff0000L
+#define SLAVE_COMM_DATA_REG1__SLAVE_COMM_DATA_REG1_BYTE2__SHIFT 0x00000010
+#define SLAVE_COMM_DATA_REG1__SLAVE_COMM_DATA_REG1_BYTE3_MASK 0xff000000L
+#define SLAVE_COMM_DATA_REG1__SLAVE_COMM_DATA_REG1_BYTE3__SHIFT 0x00000018
+#define SLAVE_COMM_DATA_REG2__SLAVE_COMM_DATA_REG2_BYTE0_MASK 0x000000ffL
+#define SLAVE_COMM_DATA_REG2__SLAVE_COMM_DATA_REG2_BYTE0__SHIFT 0x00000000
+#define SLAVE_COMM_DATA_REG2__SLAVE_COMM_DATA_REG2_BYTE1_MASK 0x0000ff00L
+#define SLAVE_COMM_DATA_REG2__SLAVE_COMM_DATA_REG2_BYTE1__SHIFT 0x00000008
+#define SLAVE_COMM_DATA_REG2__SLAVE_COMM_DATA_REG2_BYTE2_MASK 0x00ff0000L
+#define SLAVE_COMM_DATA_REG2__SLAVE_COMM_DATA_REG2_BYTE2__SHIFT 0x00000010
+#define SLAVE_COMM_DATA_REG2__SLAVE_COMM_DATA_REG2_BYTE3_MASK 0xff000000L
+#define SLAVE_COMM_DATA_REG2__SLAVE_COMM_DATA_REG2_BYTE3__SHIFT 0x00000018
+#define SLAVE_COMM_DATA_REG3__SLAVE_COMM_DATA_REG3_BYTE0_MASK 0x000000ffL
+#define SLAVE_COMM_DATA_REG3__SLAVE_COMM_DATA_REG3_BYTE0__SHIFT 0x00000000
+#define SLAVE_COMM_DATA_REG3__SLAVE_COMM_DATA_REG3_BYTE1_MASK 0x0000ff00L
+#define SLAVE_COMM_DATA_REG3__SLAVE_COMM_DATA_REG3_BYTE1__SHIFT 0x00000008
+#define SLAVE_COMM_DATA_REG3__SLAVE_COMM_DATA_REG3_BYTE2_MASK 0x00ff0000L
+#define SLAVE_COMM_DATA_REG3__SLAVE_COMM_DATA_REG3_BYTE2__SHIFT 0x00000010
+#define SLAVE_COMM_DATA_REG3__SLAVE_COMM_DATA_REG3_BYTE3_MASK 0xff000000L
+#define SLAVE_COMM_DATA_REG3__SLAVE_COMM_DATA_REG3_BYTE3__SHIFT 0x00000018
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_CLOCK_ENABLE_MASK 0x00000001L
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_CLOCK_ENABLE__SHIFT 0x00000000
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_FE_FORCE_EN_MASK 0x00000010L
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_FE_FORCE_EN__SHIFT 0x00000004
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_FE_FORCE_SRC_MASK 0x00000700L
+#define SYMCLKA_CLOCK_ENABLE__SYMCLKA_FE_FORCE_SRC__SHIFT 0x00000008
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_CLOCK_ENABLE_MASK 0x00000001L
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_CLOCK_ENABLE__SHIFT 0x00000000
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_FE_FORCE_EN_MASK 0x00000010L
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_FE_FORCE_EN__SHIFT 0x00000004
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_FE_FORCE_SRC_MASK 0x00000700L
+#define SYMCLKB_CLOCK_ENABLE__SYMCLKB_FE_FORCE_SRC__SHIFT 0x00000008
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_CLOCK_ENABLE_MASK 0x00000001L
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_CLOCK_ENABLE__SHIFT 0x00000000
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_FE_FORCE_EN_MASK 0x00000010L
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_FE_FORCE_EN__SHIFT 0x00000004
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_FE_FORCE_SRC_MASK 0x00000700L
+#define SYMCLKC_CLOCK_ENABLE__SYMCLKC_FE_FORCE_SRC__SHIFT 0x00000008
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_CLOCK_ENABLE_MASK 0x00000001L
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_CLOCK_ENABLE__SHIFT 0x00000000
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_FE_FORCE_EN_MASK 0x00000010L
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_FE_FORCE_EN__SHIFT 0x00000004
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_FE_FORCE_SRC_MASK 0x00000700L
+#define SYMCLKD_CLOCK_ENABLE__SYMCLKD_FE_FORCE_SRC__SHIFT 0x00000008
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_CLOCK_ENABLE_MASK 0x00000001L
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_CLOCK_ENABLE__SHIFT 0x00000000
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_FE_FORCE_EN_MASK 0x00000010L
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_FE_FORCE_EN__SHIFT 0x00000004
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_FE_FORCE_SRC_MASK 0x00000700L
+#define SYMCLKE_CLOCK_ENABLE__SYMCLKE_FE_FORCE_SRC__SHIFT 0x00000008
+#define SYMCLKF_CLOCK_ENABLE__SYMCLKF_CLOCK_ENABLE_MASK 0x00000001L
+#define SYMCLKF_CLOCK_ENABLE__SYMCLKF_CLOCK_ENABLE__SHIFT 0x00000000
+#define SYMCLKF_CLOCK_ENABLE__SYMCLKF_FE_FORCE_EN_MASK 0x00000010L
+#define SYMCLKF_CLOCK_ENABLE__SYMCLKF_FE_FORCE_EN__SHIFT 0x00000004
+#define SYMCLKF_CLOCK_ENABLE__SYMCLKF_FE_FORCE_SRC_MASK 0x00000700L
+#define SYMCLKF_CLOCK_ENABLE__SYMCLKF_FE_FORCE_SRC__SHIFT 0x00000008
+#define TMDS_CNTL__TMDS_COLOR_FORMAT_MASK 0x00000300L
+#define TMDS_CNTL__TMDS_COLOR_FORMAT__SHIFT 0x00000008
+#define TMDS_CNTL__TMDS_PIXEL_ENCODING_MASK 0x00000010L
+#define TMDS_CNTL__TMDS_PIXEL_ENCODING__SHIFT 0x00000004
+#define TMDS_CNTL__TMDS_SYNC_PHASE_MASK 0x00000001L
+#define TMDS_CNTL__TMDS_SYNC_PHASE__SHIFT 0x00000000
+#define TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_DELAY_MASK 0x00000300L
+#define TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_DELAY__SHIFT 0x00000008
+#define TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_SELECT_MASK 0x00000003L
+#define TMDS_CONTROL0_FEEDBACK__TMDS_CONTROL0_FEEDBACK_SELECT__SHIFT 0x00000000
+#define TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR0_OUT_EN_MASK 0x00000001L
+#define TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR0_OUT_EN__SHIFT 0x00000000
+#define TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR1_OUT_EN_MASK 0x00000002L
+#define TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR1_OUT_EN__SHIFT 0x00000001
+#define TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR2_OUT_EN_MASK 0x00000004L
+#define TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR2_OUT_EN__SHIFT 0x00000002
+#define TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR3_OUT_EN_MASK 0x00000008L
+#define TMDS_CONTROL_CHAR__TMDS_CONTROL_CHAR3_OUT_EN__SHIFT 0x00000003
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_2BIT_COUNTER_EN_MASK 0x80000000L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_2BIT_COUNTER_EN__SHIFT 0x0000001f
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_DELAY_MASK 0x00000070L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_DELAY__SHIFT 0x00000004
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_INVERT_MASK 0x00000080L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_INVERT__SHIFT 0x00000007
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_MODULATION_MASK 0x00000300L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_MODULATION__SHIFT 0x00000008
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_SEL_MASK 0x0000000fL
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_DATA_SEL__SHIFT 0x00000000
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_FB_SYNC_CONT_MASK 0x00000800L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_FB_SYNC_CONT__SHIFT 0x0000000b
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_PATTERN_OUT_EN_MASK 0x00001000L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_PATTERN_OUT_EN__SHIFT 0x0000000c
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_USE_FEEDBACK_PATH_MASK 0x00000400L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL0_USE_FEEDBACK_PATH__SHIFT 0x0000000a
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_DELAY_MASK 0x00700000L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_DELAY__SHIFT 0x00000014
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_INVERT_MASK 0x00800000L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_INVERT__SHIFT 0x00000017
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_MODULATION_MASK 0x03000000L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_MODULATION__SHIFT 0x00000018
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_SEL_MASK 0x000f0000L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_DATA_SEL__SHIFT 0x00000010
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_FB_SYNC_CONT_MASK 0x08000000L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_FB_SYNC_CONT__SHIFT 0x0000001b
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_PATTERN_OUT_EN_MASK 0x10000000L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_PATTERN_OUT_EN__SHIFT 0x0000001c
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_USE_FEEDBACK_PATH_MASK 0x04000000L
+#define TMDS_CTL0_1_GEN_CNTL__TMDS_CTL1_USE_FEEDBACK_PATH__SHIFT 0x0000001a
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_DELAY_MASK 0x00000070L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_DELAY__SHIFT 0x00000004
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_INVERT_MASK 0x00000080L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_INVERT__SHIFT 0x00000007
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_MODULATION_MASK 0x00000300L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_MODULATION__SHIFT 0x00000008
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_SEL_MASK 0x0000000fL
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_DATA_SEL__SHIFT 0x00000000
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_FB_SYNC_CONT_MASK 0x00000800L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_FB_SYNC_CONT__SHIFT 0x0000000b
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_PATTERN_OUT_EN_MASK 0x00001000L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_PATTERN_OUT_EN__SHIFT 0x0000000c
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_USE_FEEDBACK_PATH_MASK 0x00000400L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL2_USE_FEEDBACK_PATH__SHIFT 0x0000000a
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_DELAY_MASK 0x00700000L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_DELAY__SHIFT 0x00000014
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_INVERT_MASK 0x00800000L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_INVERT__SHIFT 0x00000017
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_MODULATION_MASK 0x03000000L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_MODULATION__SHIFT 0x00000018
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_SEL_MASK 0x000f0000L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_DATA_SEL__SHIFT 0x00000010
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_FB_SYNC_CONT_MASK 0x08000000L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_FB_SYNC_CONT__SHIFT 0x0000001b
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_PATTERN_OUT_EN_MASK 0x10000000L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_PATTERN_OUT_EN__SHIFT 0x0000001c
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_USE_FEEDBACK_PATH_MASK 0x04000000L
+#define TMDS_CTL2_3_GEN_CNTL__TMDS_CTL3_USE_FEEDBACK_PATH__SHIFT 0x0000001a
+#define TMDS_CTL_BITS__TMDS_CTL0_MASK 0x00000001L
+#define TMDS_CTL_BITS__TMDS_CTL0__SHIFT 0x00000000
+#define TMDS_CTL_BITS__TMDS_CTL1_MASK 0x00000100L
+#define TMDS_CTL_BITS__TMDS_CTL1__SHIFT 0x00000008
+#define TMDS_CTL_BITS__TMDS_CTL2_MASK 0x00010000L
+#define TMDS_CTL_BITS__TMDS_CTL2__SHIFT 0x00000010
+#define TMDS_CTL_BITS__TMDS_CTL3_MASK 0x01000000L
+#define TMDS_CTL_BITS__TMDS_CTL3__SHIFT 0x00000018
+#define TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_EN_MASK 0x00000001L
+#define TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_EN__SHIFT 0x00000000
+#define TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_FORCE_MASK 0x01000000L
+#define TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_FORCE__SHIFT 0x00000018
+#define TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_EN_MASK 0x00000100L
+#define TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_EN__SHIFT 0x00000008
+#define TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_IN_MASK 0x000f0000L
+#define TMDS_DCBALANCER_CONTROL__TMDS_DCBALANCER_TEST_IN__SHIFT 0x00000010
+#define TMDS_DCBALANCER_CONTROL__TMDS_SYNC_DCBAL_EN_MASK 0x00000070L
+#define TMDS_DCBALANCER_CONTROL__TMDS_SYNC_DCBAL_EN__SHIFT 0x00000004
+#define TMDS_DEBUG__TMDS_DEBUG_DE_EN_MASK 0x02000000L
+#define TMDS_DEBUG__TMDS_DEBUG_DE_EN__SHIFT 0x00000019
+#define TMDS_DEBUG__TMDS_DEBUG_DE_MASK 0x01000000L
+#define TMDS_DEBUG__TMDS_DEBUG_DE__SHIFT 0x00000018
+#define TMDS_DEBUG__TMDS_DEBUG_EN_MASK 0x00000001L
+#define TMDS_DEBUG__TMDS_DEBUG_EN__SHIFT 0x00000000
+#define TMDS_DEBUG__TMDS_DEBUG_HSYNC_EN_MASK 0x00000200L
+#define TMDS_DEBUG__TMDS_DEBUG_HSYNC_EN__SHIFT 0x00000009
+#define TMDS_DEBUG__TMDS_DEBUG_HSYNC_MASK 0x00000100L
+#define TMDS_DEBUG__TMDS_DEBUG_HSYNC__SHIFT 0x00000008
+#define TMDS_DEBUG__TMDS_DEBUG_VSYNC_EN_MASK 0x00020000L
+#define TMDS_DEBUG__TMDS_DEBUG_VSYNC_EN__SHIFT 0x00000011
+#define TMDS_DEBUG__TMDS_DEBUG_VSYNC_MASK 0x00010000L
+#define TMDS_DEBUG__TMDS_DEBUG_VSYNC__SHIFT 0x00000010
+#define TMDS_STEREOSYNC_CTL_SEL__TMDS_STEREOSYNC_CTL_SEL_MASK 0x00000003L
+#define TMDS_STEREOSYNC_CTL_SEL__TMDS_STEREOSYNC_CTL_SEL__SHIFT 0x00000000
+#define TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN0_MASK 0x000003ffL
+#define TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN0__SHIFT 0x00000000
+#define TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN1_MASK 0x03ff0000L
+#define TMDS_SYNC_CHAR_PATTERN_0_1__TMDS_SYNC_CHAR_PATTERN1__SHIFT 0x00000010
+#define TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN2_MASK 0x000003ffL
+#define TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN2__SHIFT 0x00000000
+#define TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN3_MASK 0x03ff0000L
+#define TMDS_SYNC_CHAR_PATTERN_2_3__TMDS_SYNC_CHAR_PATTERN3__SHIFT 0x00000010
+#define UNIPHYAB_TPG_CONTROL__UNIPHYAB_STATIC_TEST_PATTERN_MASK 0x000003ffL
+#define UNIPHYAB_TPG_CONTROL__UNIPHYAB_STATIC_TEST_PATTERN__SHIFT 0x00000000
+#define UNIPHYAB_TPG_CONTROL__UNIPHYAB_TPG_EN_MASK 0x00010000L
+#define UNIPHYAB_TPG_CONTROL__UNIPHYAB_TPG_EN__SHIFT 0x00000010
+#define UNIPHYAB_TPG_CONTROL__UNIPHYAB_TPG_SEL_MASK 0x000e0000L
+#define UNIPHYAB_TPG_CONTROL__UNIPHYAB_TPG_SEL__SHIFT 0x00000011
+#define UNIPHYAB_TPG_SEED__UNIPHYAB_TPG_SEED_MASK 0x007fffffL
+#define UNIPHYAB_TPG_SEED__UNIPHYAB_TPG_SEED__SHIFT 0x00000000
+#define UNIPHY_ANG_BIST_CNTL__UNIPHY_ANG_BIST_ERROR_MASK 0x001f0000L
+#define UNIPHY_ANG_BIST_CNTL__UNIPHY_ANG_BIST_ERROR__SHIFT 0x00000010
+#define UNIPHY_ANG_BIST_CNTL__UNIPHY_ANG_BIST_RESET_MASK 0x00000002L
+#define UNIPHY_ANG_BIST_CNTL__UNIPHY_ANG_BIST_RESET__SHIFT 0x00000001
+#define UNIPHY_ANG_BIST_CNTL__UNIPHY_PRESETB_MASK 0x01000000L
+#define UNIPHY_ANG_BIST_CNTL__UNIPHY_PRESETB__SHIFT 0x00000018
+#define UNIPHY_ANG_BIST_CNTL__UNIPHY_RX_BIAS_MASK 0x00000f00L
+#define UNIPHY_ANG_BIST_CNTL__UNIPHY_RX_BIAS__SHIFT 0x00000008
+#define UNIPHY_ANG_BIST_CNTL__UNIPHY_TEST_RX_EN_MASK 0x00000001L
+#define UNIPHY_ANG_BIST_CNTL__UNIPHY_TEST_RX_EN__SHIFT 0x00000000
+#define UNIPHYCD_TPG_CONTROL__UNIPHYCD_STATIC_TEST_PATTERN_MASK 0x000003ffL
+#define UNIPHYCD_TPG_CONTROL__UNIPHYCD_STATIC_TEST_PATTERN__SHIFT 0x00000000
+#define UNIPHYCD_TPG_CONTROL__UNIPHYCD_TPG_EN_MASK 0x00010000L
+#define UNIPHYCD_TPG_CONTROL__UNIPHYCD_TPG_EN__SHIFT 0x00000010
+#define UNIPHYCD_TPG_CONTROL__UNIPHYCD_TPG_SEL_MASK 0x000e0000L
+#define UNIPHYCD_TPG_CONTROL__UNIPHYCD_TPG_SEL__SHIFT 0x00000011
+#define UNIPHYCD_TPG_SEED__UNIPHYCD_TPG_SEED_MASK 0x007fffffL
+#define UNIPHYCD_TPG_SEED__UNIPHYCD_TPG_SEED__SHIFT 0x00000000
+#define UNIPHY_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL0_XBAR_SOURCE_MASK 0x00000003L
+#define UNIPHY_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL0_XBAR_SOURCE__SHIFT 0x00000000
+#define UNIPHY_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL1_XBAR_SOURCE_MASK 0x00000300L
+#define UNIPHY_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL1_XBAR_SOURCE__SHIFT 0x00000008
+#define UNIPHY_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL2_XBAR_SOURCE_MASK 0x00030000L
+#define UNIPHY_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL2_XBAR_SOURCE__SHIFT 0x00000010
+#define UNIPHY_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL3_XBAR_SOURCE_MASK 0x03000000L
+#define UNIPHY_CHANNEL_XBAR_CNTL__UNIPHY_CHANNEL3_XBAR_SOURCE__SHIFT 0x00000018
+#define UNIPHY_DATA_SYNCHRONIZATION__UNIPHY_DSYN_ERROR_MASK 0x00000040L
+#define UNIPHY_DATA_SYNCHRONIZATION__UNIPHY_DSYN_ERROR__SHIFT 0x00000006
+#define UNIPHY_DATA_SYNCHRONIZATION__UNIPHY_DSYN_LEVEL_MASK 0x00000030L
+#define UNIPHY_DATA_SYNCHRONIZATION__UNIPHY_DSYN_LEVEL__SHIFT 0x00000004
+#define UNIPHY_DATA_SYNCHRONIZATION__UNIPHY_DSYNSEL_MASK 0x00000001L
+#define UNIPHY_DATA_SYNCHRONIZATION__UNIPHY_DSYNSEL__SHIFT 0x00000000
+#define UNIPHY_DATA_SYNCHRONIZATION__UNIPHY_DUAL_LINK_PHASE_MASK 0x00010000L
+#define UNIPHY_DATA_SYNCHRONIZATION__UNIPHY_DUAL_LINK_PHASE__SHIFT 0x00000010
+#define UNIPHY_DATA_SYNCHRONIZATION__UNIPHY_LINK_ENABLE_MASK 0x00001000L
+#define UNIPHY_DATA_SYNCHRONIZATION__UNIPHY_LINK_ENABLE__SHIFT 0x0000000c
+#define UNIPHY_DATA_SYNCHRONIZATION__UNIPHY_SOURCE_SELECT_MASK 0x00000100L
+#define UNIPHY_DATA_SYNCHRONIZATION__UNIPHY_SOURCE_SELECT__SHIFT 0x00000008
+#define UNIPHYEF_TPG_CONTROL__UNIPHYEF_STATIC_TEST_PATTERN_MASK 0x000003ffL
+#define UNIPHYEF_TPG_CONTROL__UNIPHYEF_STATIC_TEST_PATTERN__SHIFT 0x00000000
+#define UNIPHYEF_TPG_CONTROL__UNIPHYEF_TPG_EN_MASK 0x00010000L
+#define UNIPHYEF_TPG_CONTROL__UNIPHYEF_TPG_EN__SHIFT 0x00000010
+#define UNIPHYEF_TPG_CONTROL__UNIPHYEF_TPG_SEL_MASK 0x000e0000L
+#define UNIPHYEF_TPG_CONTROL__UNIPHYEF_TPG_SEL__SHIFT 0x00000011
+#define UNIPHYEF_TPG_SEED__UNIPHYEF_TPG_SEED_MASK 0x007fffffL
+#define UNIPHYEF_TPG_SEED__UNIPHYEF_TPG_SEED__SHIFT 0x00000000
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_CALOUT_ERROR_LINKA_AK_MASK 0x00000400L
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_CALOUT_ERROR_LINKA_AK__SHIFT 0x0000000a
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_CALOUT_ERROR_LINKA_MASK 0x00000200L
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_CALOUT_ERROR_LINKA__SHIFT 0x00000009
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_CALOUT_LINKA_MASK 0x00000100L
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_CALOUT_LINKA__SHIFT 0x00000008
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_ENABLE_LINKA_MASK 0x00000001L
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_ENABLE_LINKA__SHIFT 0x00000000
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_OVERRIDE_ENABLE_LINKA_MASK 0x10000000L
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_OVERRIDE_ENABLE_LINKA__SHIFT 0x0000001c
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_OVERRIDE_LINKA_MASK 0x0f000000L
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_OVERRIDE_LINKA__SHIFT 0x00000018
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_SEL_LINKA_MASK 0x40000000L
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_SEL_LINKA__SHIFT 0x0000001e
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_STEP_DELAY_LINKA_MASK 0x00f00000L
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_STEP_DELAY_LINKA__SHIFT 0x00000014
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_VALUE_LINKA_MASK 0x000f0000L
+#define UNIPHY_IMPCAL_LINKA__UNIPHY_IMPCAL_VALUE_LINKA__SHIFT 0x00000010
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_CALOUT_ERROR_LINKB_AK_MASK 0x00000400L
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_CALOUT_ERROR_LINKB_AK__SHIFT 0x0000000a
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_CALOUT_ERROR_LINKB_MASK 0x00000200L
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_CALOUT_ERROR_LINKB__SHIFT 0x00000009
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_CALOUT_LINKB_MASK 0x00000100L
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_CALOUT_LINKB__SHIFT 0x00000008
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_ENABLE_LINKB_MASK 0x00000001L
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_ENABLE_LINKB__SHIFT 0x00000000
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_OVERRIDE_ENABLE_LINKB_MASK 0x10000000L
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_OVERRIDE_ENABLE_LINKB__SHIFT 0x0000001c
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_OVERRIDE_LINKB_MASK 0x0f000000L
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_OVERRIDE_LINKB__SHIFT 0x00000018
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_SEL_LINKB_MASK 0x40000000L
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_SEL_LINKB__SHIFT 0x0000001e
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_STEP_DELAY_LINKB_MASK 0x00f00000L
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_STEP_DELAY_LINKB__SHIFT 0x00000014
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_VALUE_LINKB_MASK 0x000f0000L
+#define UNIPHY_IMPCAL_LINKB__UNIPHY_IMPCAL_VALUE_LINKB__SHIFT 0x00000010
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_CALOUT_ERROR_LINKC_AK_MASK 0x00000400L
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_CALOUT_ERROR_LINKC_AK__SHIFT 0x0000000a
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_CALOUT_ERROR_LINKC_MASK 0x00000200L
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_CALOUT_ERROR_LINKC__SHIFT 0x00000009
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_CALOUT_LINKC_MASK 0x00000100L
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_CALOUT_LINKC__SHIFT 0x00000008
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_ENABLE_LINKC_MASK 0x00000001L
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_ENABLE_LINKC__SHIFT 0x00000000
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_OVERRIDE_ENABLE_LINKC_MASK 0x10000000L
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_OVERRIDE_ENABLE_LINKC__SHIFT 0x0000001c
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_OVERRIDE_LINKC_MASK 0x0f000000L
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_OVERRIDE_LINKC__SHIFT 0x00000018
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_SEL_LINKC_MASK 0x40000000L
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_SEL_LINKC__SHIFT 0x0000001e
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_STEP_DELAY_LINKC_MASK 0x00f00000L
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_STEP_DELAY_LINKC__SHIFT 0x00000014
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_VALUE_LINKC_MASK 0x000f0000L
+#define UNIPHY_IMPCAL_LINKC__UNIPHY_IMPCAL_VALUE_LINKC__SHIFT 0x00000010
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_CALOUT_ERROR_LINKD_AK_MASK 0x00000400L
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_CALOUT_ERROR_LINKD_AK__SHIFT 0x0000000a
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_CALOUT_ERROR_LINKD_MASK 0x00000200L
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_CALOUT_ERROR_LINKD__SHIFT 0x00000009
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_CALOUT_LINKD_MASK 0x00000100L
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_CALOUT_LINKD__SHIFT 0x00000008
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_ENABLE_LINKD_MASK 0x00000001L
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_ENABLE_LINKD__SHIFT 0x00000000
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_OVERRIDE_ENABLE_LINKD_MASK 0x10000000L
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_OVERRIDE_ENABLE_LINKD__SHIFT 0x0000001c
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_OVERRIDE_LINKD_MASK 0x0f000000L
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_OVERRIDE_LINKD__SHIFT 0x00000018
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_SEL_LINKD_MASK 0x40000000L
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_SEL_LINKD__SHIFT 0x0000001e
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_STEP_DELAY_LINKD_MASK 0x00f00000L
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_STEP_DELAY_LINKD__SHIFT 0x00000014
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_VALUE_LINKD_MASK 0x000f0000L
+#define UNIPHY_IMPCAL_LINKD__UNIPHY_IMPCAL_VALUE_LINKD__SHIFT 0x00000010
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_CALOUT_ERROR_LINKE_AK_MASK 0x00000400L
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_CALOUT_ERROR_LINKE_AK__SHIFT 0x0000000a
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_CALOUT_ERROR_LINKE_MASK 0x00000200L
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_CALOUT_ERROR_LINKE__SHIFT 0x00000009
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_CALOUT_LINKE_MASK 0x00000100L
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_CALOUT_LINKE__SHIFT 0x00000008
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_ENABLE_LINKE_MASK 0x00000001L
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_ENABLE_LINKE__SHIFT 0x00000000
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_OVERRIDE_ENABLE_LINKE_MASK 0x10000000L
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_OVERRIDE_ENABLE_LINKE__SHIFT 0x0000001c
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_OVERRIDE_LINKE_MASK 0x0f000000L
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_OVERRIDE_LINKE__SHIFT 0x00000018
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_SEL_LINKE_MASK 0x40000000L
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_SEL_LINKE__SHIFT 0x0000001e
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_STEP_DELAY_LINKE_MASK 0x00f00000L
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_STEP_DELAY_LINKE__SHIFT 0x00000014
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_VALUE_LINKE_MASK 0x000f0000L
+#define UNIPHY_IMPCAL_LINKE__UNIPHY_IMPCAL_VALUE_LINKE__SHIFT 0x00000010
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_CALOUT_ERROR_LINKF_AK_MASK 0x00000400L
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_CALOUT_ERROR_LINKF_AK__SHIFT 0x0000000a
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_CALOUT_ERROR_LINKF_MASK 0x00000200L
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_CALOUT_ERROR_LINKF__SHIFT 0x00000009
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_CALOUT_LINKF_MASK 0x00000100L
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_CALOUT_LINKF__SHIFT 0x00000008
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_ENABLE_LINKF_MASK 0x00000001L
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_ENABLE_LINKF__SHIFT 0x00000000
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_OVERRIDE_ENABLE_LINKF_MASK 0x10000000L
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_OVERRIDE_ENABLE_LINKF__SHIFT 0x0000001c
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_OVERRIDE_LINKF_MASK 0x0f000000L
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_OVERRIDE_LINKF__SHIFT 0x00000018
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_SEL_LINKF_MASK 0x40000000L
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_SEL_LINKF__SHIFT 0x0000001e
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_STEP_DELAY_LINKF_MASK 0x00f00000L
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_STEP_DELAY_LINKF__SHIFT 0x00000014
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_VALUE_LINKF_MASK 0x000f0000L
+#define UNIPHY_IMPCAL_LINKF__UNIPHY_IMPCAL_VALUE_LINKF__SHIFT 0x00000010
+#define UNIPHY_IMPCAL_PERIOD__UNIPHY_IMPCAL_PERIOD_MASK 0xffffffffL
+#define UNIPHY_IMPCAL_PERIOD__UNIPHY_IMPCAL_PERIOD__SHIFT 0x00000000
+#define UNIPHY_IMPCAL_PSW_AB__UNIPHY_IMPCAL_PSW_LINKA_MASK 0x00007fffL
+#define UNIPHY_IMPCAL_PSW_AB__UNIPHY_IMPCAL_PSW_LINKA__SHIFT 0x00000000
+#define UNIPHY_IMPCAL_PSW_AB__UNIPHY_IMPCAL_PSW_LINKB_MASK 0x7fff0000L
+#define UNIPHY_IMPCAL_PSW_AB__UNIPHY_IMPCAL_PSW_LINKB__SHIFT 0x00000010
+#define UNIPHY_IMPCAL_PSW_CD__UNIPHY_IMPCAL_PSW_LINKC_MASK 0x00007fffL
+#define UNIPHY_IMPCAL_PSW_CD__UNIPHY_IMPCAL_PSW_LINKC__SHIFT 0x00000000
+#define UNIPHY_IMPCAL_PSW_CD__UNIPHY_IMPCAL_PSW_LINKD_MASK 0x7fff0000L
+#define UNIPHY_IMPCAL_PSW_CD__UNIPHY_IMPCAL_PSW_LINKD__SHIFT 0x00000010
+#define UNIPHY_IMPCAL_PSW_EF__UNIPHY_IMPCAL_PSW_LINKE_MASK 0x00007fffL
+#define UNIPHY_IMPCAL_PSW_EF__UNIPHY_IMPCAL_PSW_LINKE__SHIFT 0x00000000
+#define UNIPHY_IMPCAL_PSW_EF__UNIPHY_IMPCAL_PSW_LINKF_MASK 0x7fff0000L
+#define UNIPHY_IMPCAL_PSW_EF__UNIPHY_IMPCAL_PSW_LINKF__SHIFT 0x00000010
+#define UNIPHY_LINK_CNTL__UNIPHY_CHANNEL0_INVERT_MASK 0x00001000L
+#define UNIPHY_LINK_CNTL__UNIPHY_CHANNEL0_INVERT__SHIFT 0x0000000c
+#define UNIPHY_LINK_CNTL__UNIPHY_CHANNEL1_INVERT_MASK 0x00002000L
+#define UNIPHY_LINK_CNTL__UNIPHY_CHANNEL1_INVERT__SHIFT 0x0000000d
+#define UNIPHY_LINK_CNTL__UNIPHY_CHANNEL2_INVERT_MASK 0x00004000L
+#define UNIPHY_LINK_CNTL__UNIPHY_CHANNEL2_INVERT__SHIFT 0x0000000e
+#define UNIPHY_LINK_CNTL__UNIPHY_CHANNEL3_INVERT_MASK 0x00008000L
+#define UNIPHY_LINK_CNTL__UNIPHY_CHANNEL3_INVERT__SHIFT 0x0000000f
+#define UNIPHY_LINK_CNTL__UNIPHY_LANE_STAGGER_DELAY_MASK 0x00700000L
+#define UNIPHY_LINK_CNTL__UNIPHY_LANE_STAGGER_DELAY__SHIFT 0x00000014
+#define UNIPHY_LINK_CNTL__UNIPHY_LINK_ENABLE_HPD_MASK_MASK 0x00030000L
+#define UNIPHY_LINK_CNTL__UNIPHY_LINK_ENABLE_HPD_MASK__SHIFT 0x00000010
+#define UNIPHY_LINK_CNTL__UNIPHY_MINIMUM_PIXVLD_LOW_DURATION_MASK 0x00000700L
+#define UNIPHY_LINK_CNTL__UNIPHY_MINIMUM_PIXVLD_LOW_DURATION__SHIFT 0x00000008
+#define UNIPHY_LINK_CNTL__UNIPHY_PFREQCHG_MASK 0x00000001L
+#define UNIPHY_LINK_CNTL__UNIPHY_PFREQCHG__SHIFT 0x00000000
+#define UNIPHY_LINK_CNTL__UNIPHY_PIXVLD_RESET_MASK 0x00000010L
+#define UNIPHY_LINK_CNTL__UNIPHY_PIXVLD_RESET__SHIFT 0x00000004
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_BW_CNTL_MASK 0x00ff0000L
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_BW_CNTL__SHIFT 0x00000010
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_CLK_EN_MASK 0x00000008L
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_CLK_EN__SHIFT 0x00000003
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_CLKPH_EN_MASK 0x000000f0L
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_CLKPH_EN__SHIFT 0x00000004
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_ENABLE_MASK 0x00000001L
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_ENABLE__SHIFT 0x00000000
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_EXT_RESET_EN_MASK 0x00000004L
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_EXT_RESET_EN__SHIFT 0x00000002
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_LF_CNTL_MASK 0x00007f00L
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_LF_CNTL__SHIFT 0x00000008
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_RESET_MASK 0x00000002L
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_RESET__SHIFT 0x00000001
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_TEST_BYPCLK_EN_MASK 0x02000000L
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_TEST_BYPCLK_EN__SHIFT 0x00000019
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_TEST_BYPCLK_SRC_MASK 0x01000000L
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_TEST_BYPCLK_SRC__SHIFT 0x00000018
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_TEST_VCTL_ADC_EN_MASK 0x04000000L
+#define UNIPHY_PLL_CONTROL1__UNIPHY_PLL_TEST_VCTL_ADC_EN__SHIFT 0x0000001a
+#define UNIPHY_PLL_CONTROL1__UNIPHY_VCO_MODE_MASK 0x30000000L
+#define UNIPHY_PLL_CONTROL1__UNIPHY_VCO_MODE__SHIFT 0x0000001c
+#define UNIPHY_PLL_CONTROL2__UNIPHY_CLKINV_MASK 0x00002000L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_CLKINV__SHIFT 0x0000000d
+#define UNIPHY_PLL_CONTROL2__UNIPHY_DPLLSEL_MASK 0x0000000cL
+#define UNIPHY_PLL_CONTROL2__UNIPHY_DPLLSEL__SHIFT 0x00000002
+#define UNIPHY_PLL_CONTROL2__UNIPHY_IDCLK_EN_MASK 0x00001000L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_IDCLK_EN__SHIFT 0x0000000c
+#define UNIPHY_PLL_CONTROL2__UNIPHY_IDCLK_SEL_MASK 0x00000010L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_IDCLK_SEL__SHIFT 0x00000004
+#define UNIPHY_PLL_CONTROL2__UNIPHY_IPCIE_REFCLK_SEL_MASK 0x00000020L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_IPCIE_REFCLK_SEL__SHIFT 0x00000005
+#define UNIPHY_PLL_CONTROL2__UNIPHY_IXTALIN_SEL_MASK 0x00000040L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_IXTALIN_SEL__SHIFT 0x00000006
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PCIEREF_CLK_EN_MASK 0x00000800L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PCIEREF_CLK_EN__SHIFT 0x0000000b
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PDIVFRAC_SEL_MASK 0x00100000L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PDIVFRAC_SEL__SHIFT 0x00000014
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PDIV_SEL_MASK 0xe0000000L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PDIV_SEL__SHIFT 0x0000001d
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PLL_DISPCLK_MODE_MASK 0x00000003L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PLL_DISPCLK_MODE__SHIFT 0x00000000
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PLL_REFCLK_SRC_MASK 0x00000700L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PLL_REFCLK_SRC__SHIFT 0x00000008
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PLL_REFDIV_MASK 0x1f000000L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PLL_REFDIV__SHIFT 0x00000018
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PLL_TEST_FBDIV_FRAC_BYPASS_MASK 0x00080000L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PLL_TEST_FBDIV_FRAC_BYPASS__SHIFT 0x00000013
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PLL_VTOI_BIAS_CNTL_MASK 0x00010000L
+#define UNIPHY_PLL_CONTROL2__UNIPHY_PLL_VTOI_BIAS_CNTL__SHIFT 0x00000010
+#define UNIPHY_PLL_FBDIV__UNIPHY_PLL_FBDIV_FRACTION_MASK 0x0000fffcL
+#define UNIPHY_PLL_FBDIV__UNIPHY_PLL_FBDIV_FRACTION__SHIFT 0x00000002
+#define UNIPHY_PLL_FBDIV__UNIPHY_PLL_FBDIV_MASK 0x0fff0000L
+#define UNIPHY_PLL_FBDIV__UNIPHY_PLL_FBDIV__SHIFT 0x00000010
+#define UNIPHY_PLL_SS_CNTL__UNIPHY_PLL_DSMOD_EN_MASK 0x00001000L
+#define UNIPHY_PLL_SS_CNTL__UNIPHY_PLL_DSMOD_EN__SHIFT 0x0000000c
+#define UNIPHY_PLL_SS_CNTL__UNIPHY_PLL_SS_EN_MASK 0x00002000L
+#define UNIPHY_PLL_SS_CNTL__UNIPHY_PLL_SS_EN__SHIFT 0x0000000d
+#define UNIPHY_PLL_SS_CNTL__UNIPHY_PLL_SS_STEP_NUM_MASK 0x00000fffL
+#define UNIPHY_PLL_SS_CNTL__UNIPHY_PLL_SS_STEP_NUM__SHIFT 0x00000000
+#define UNIPHY_PLL_SS_STEP_SIZE__UNIPHY_PLL_SS_STEP_SIZE_MASK 0x03ffffffL
+#define UNIPHY_PLL_SS_STEP_SIZE__UNIPHY_PLL_SS_STEP_SIZE__SHIFT 0x00000000
+#define UNIPHY_POWER_CONTROL__UNIPHY_BGADJ0P45_MASK 0x000f0000L
+#define UNIPHY_POWER_CONTROL__UNIPHY_BGADJ0P45__SHIFT 0x00000010
+#define UNIPHY_POWER_CONTROL__UNIPHY_BGADJ1P00_MASK 0x00000f00L
+#define UNIPHY_POWER_CONTROL__UNIPHY_BGADJ1P00__SHIFT 0x00000008
+#define UNIPHY_POWER_CONTROL__UNIPHY_BGADJ1P25_MASK 0x0000f000L
+#define UNIPHY_POWER_CONTROL__UNIPHY_BGADJ1P25__SHIFT 0x0000000c
+#define UNIPHY_POWER_CONTROL__UNIPHY_BGPDN_MASK 0x00000001L
+#define UNIPHY_POWER_CONTROL__UNIPHY_BGPDN__SHIFT 0x00000000
+#define UNIPHY_POWER_CONTROL__UNIPHY_BIASREF_SEL_MASK 0x00000004L
+#define UNIPHY_POWER_CONTROL__UNIPHY_BIASREF_SEL__SHIFT 0x00000002
+#define UNIPHY_POWER_CONTROL__UNIPHY_RST_LOGIC_MASK 0x00000002L
+#define UNIPHY_POWER_CONTROL__UNIPHY_RST_LOGIC__SHIFT 0x00000001
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_DIG_BIST_ERROR_MASK 0x01f00000L
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_DIG_BIST_ERROR__SHIFT 0x00000014
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_DIG_BIST_RESET_MASK 0x00008000L
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_DIG_BIST_RESET__SHIFT 0x0000000f
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_DIG_BIST_SEL_MASK 0x00010000L
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_DIG_BIST_SEL__SHIFT 0x00000010
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_PLL_INTRESET_MASK 0x20000000L
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_PLL_INTRESET__SHIFT 0x0000001d
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_PLL_TEST_FREQ_LOCK_MASK 0x10000000L
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_PLL_TEST_FREQ_LOCK__SHIFT 0x0000001c
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_PLL_TEST_VCTL_ADC_MASK 0x0e000000L
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_PLL_TEST_VCTL_ADC__SHIFT 0x00000019
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_TEST_CNTL_MASK 0x0000001fL
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_TEST_CNTL__SHIFT 0x00000000
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_TEST_VCTL_EN_MASK 0x00020000L
+#define UNIPHY_REG_TEST_OUTPUT__UNIPHY_TEST_VCTL_EN__SHIFT 0x00000011
+#define UNIPHY_SOFT_RESET__DSYNCA_SOFT_RESET_MASK 0x00000001L
+#define UNIPHY_SOFT_RESET__DSYNCA_SOFT_RESET__SHIFT 0x00000000
+#define UNIPHY_SOFT_RESET__DSYNCB_SOFT_RESET_MASK 0x00000002L
+#define UNIPHY_SOFT_RESET__DSYNCB_SOFT_RESET__SHIFT 0x00000001
+#define UNIPHY_SOFT_RESET__DSYNCC_SOFT_RESET_MASK 0x00000004L
+#define UNIPHY_SOFT_RESET__DSYNCC_SOFT_RESET__SHIFT 0x00000002
+#define UNIPHY_SOFT_RESET__DSYNCD_SOFT_RESET_MASK 0x00000008L
+#define UNIPHY_SOFT_RESET__DSYNCD_SOFT_RESET__SHIFT 0x00000003
+#define UNIPHY_SOFT_RESET__DSYNCE_SOFT_RESET_MASK 0x00000010L
+#define UNIPHY_SOFT_RESET__DSYNCE_SOFT_RESET__SHIFT 0x00000004
+#define UNIPHY_SOFT_RESET__DSYNCF_SOFT_RESET_MASK 0x00000020L
+#define UNIPHY_SOFT_RESET__DSYNCF_SOFT_RESET__SHIFT 0x00000005
+#define UNIPHY_TX_CONTROL1__UNIPHY_PREMPH_STR0_MASK 0x00000007L
+#define UNIPHY_TX_CONTROL1__UNIPHY_PREMPH_STR0__SHIFT 0x00000000
+#define UNIPHY_TX_CONTROL1__UNIPHY_PREMPH_STR1_MASK 0x00000070L
+#define UNIPHY_TX_CONTROL1__UNIPHY_PREMPH_STR1__SHIFT 0x00000004
+#define UNIPHY_TX_CONTROL1__UNIPHY_PREMPH_STR2_MASK 0x00000700L
+#define UNIPHY_TX_CONTROL1__UNIPHY_PREMPH_STR2__SHIFT 0x00000008
+#define UNIPHY_TX_CONTROL1__UNIPHY_PREMPH_STR3_MASK 0x00007000L
+#define UNIPHY_TX_CONTROL1__UNIPHY_PREMPH_STR3__SHIFT 0x0000000c
+#define UNIPHY_TX_CONTROL1__UNIPHY_PREMPH_STR4_MASK 0x00070000L
+#define UNIPHY_TX_CONTROL1__UNIPHY_PREMPH_STR4__SHIFT 0x00000010
+#define UNIPHY_TX_CONTROL1__UNIPHY_TX_VS0_MASK 0x00300000L
+#define UNIPHY_TX_CONTROL1__UNIPHY_TX_VS0__SHIFT 0x00000014
+#define UNIPHY_TX_CONTROL1__UNIPHY_TX_VS1_MASK 0x00c00000L
+#define UNIPHY_TX_CONTROL1__UNIPHY_TX_VS1__SHIFT 0x00000016
+#define UNIPHY_TX_CONTROL1__UNIPHY_TX_VS2_MASK 0x03000000L
+#define UNIPHY_TX_CONTROL1__UNIPHY_TX_VS2__SHIFT 0x00000018
+#define UNIPHY_TX_CONTROL1__UNIPHY_TX_VS3_MASK 0x0c000000L
+#define UNIPHY_TX_CONTROL1__UNIPHY_TX_VS3__SHIFT 0x0000001a
+#define UNIPHY_TX_CONTROL1__UNIPHY_TX_VS4_MASK 0x30000000L
+#define UNIPHY_TX_CONTROL1__UNIPHY_TX_VS4__SHIFT 0x0000001c
+#define UNIPHY_TX_CONTROL2__UNIPHY_PREMPH0_PC_MASK 0x00000003L
+#define UNIPHY_TX_CONTROL2__UNIPHY_PREMPH0_PC__SHIFT 0x00000000
+#define UNIPHY_TX_CONTROL2__UNIPHY_PREMPH1_PC_MASK 0x00000030L
+#define UNIPHY_TX_CONTROL2__UNIPHY_PREMPH1_PC__SHIFT 0x00000004
+#define UNIPHY_TX_CONTROL2__UNIPHY_PREMPH2_PC_MASK 0x00000300L
+#define UNIPHY_TX_CONTROL2__UNIPHY_PREMPH2_PC__SHIFT 0x00000008
+#define UNIPHY_TX_CONTROL2__UNIPHY_PREMPH3_PC_MASK 0x00003000L
+#define UNIPHY_TX_CONTROL2__UNIPHY_PREMPH3_PC__SHIFT 0x0000000c
+#define UNIPHY_TX_CONTROL2__UNIPHY_PREMPH4_PC_MASK 0x00030000L
+#define UNIPHY_TX_CONTROL2__UNIPHY_PREMPH4_PC__SHIFT 0x00000010
+#define UNIPHY_TX_CONTROL2__UNIPHY_PREMPH_SEL_MASK 0x00100000L
+#define UNIPHY_TX_CONTROL2__UNIPHY_PREMPH_SEL__SHIFT 0x00000014
+#define UNIPHY_TX_CONTROL2__UNIPHY_RT0_CPSEL_MASK 0x00600000L
+#define UNIPHY_TX_CONTROL2__UNIPHY_RT0_CPSEL__SHIFT 0x00000015
+#define UNIPHY_TX_CONTROL2__UNIPHY_RT1_CPSEL_MASK 0x01800000L
+#define UNIPHY_TX_CONTROL2__UNIPHY_RT1_CPSEL__SHIFT 0x00000017
+#define UNIPHY_TX_CONTROL2__UNIPHY_RT2_CPSEL_MASK 0x06000000L
+#define UNIPHY_TX_CONTROL2__UNIPHY_RT2_CPSEL__SHIFT 0x00000019
+#define UNIPHY_TX_CONTROL2__UNIPHY_RT3_CPSEL_MASK 0x18000000L
+#define UNIPHY_TX_CONTROL2__UNIPHY_RT3_CPSEL__SHIFT 0x0000001b
+#define UNIPHY_TX_CONTROL2__UNIPHY_RT4_CPSEL_MASK 0x60000000L
+#define UNIPHY_TX_CONTROL2__UNIPHY_RT4_CPSEL__SHIFT 0x0000001d
+#define UNIPHY_TX_CONTROL3__UNIPHY_LVDS_PULLDWN_MASK 0x80000000L
+#define UNIPHY_TX_CONTROL3__UNIPHY_LVDS_PULLDWN__SHIFT 0x0000001f
+#define UNIPHY_TX_CONTROL3__UNIPHY_PESEL0_MASK 0x00100000L
+#define UNIPHY_TX_CONTROL3__UNIPHY_PESEL0__SHIFT 0x00000014
+#define UNIPHY_TX_CONTROL3__UNIPHY_PESEL1_MASK 0x00200000L
+#define UNIPHY_TX_CONTROL3__UNIPHY_PESEL1__SHIFT 0x00000015
+#define UNIPHY_TX_CONTROL3__UNIPHY_PESEL2_MASK 0x00400000L
+#define UNIPHY_TX_CONTROL3__UNIPHY_PESEL2__SHIFT 0x00000016
+#define UNIPHY_TX_CONTROL3__UNIPHY_PESEL3_MASK 0x00800000L
+#define UNIPHY_TX_CONTROL3__UNIPHY_PESEL3__SHIFT 0x00000017
+#define UNIPHY_TX_CONTROL3__UNIPHY_PREMPH_CS_CLK_MASK 0x000000f0L
+#define UNIPHY_TX_CONTROL3__UNIPHY_PREMPH_CS_CLK__SHIFT 0x00000004
+#define UNIPHY_TX_CONTROL3__UNIPHY_PREMPH_CS_DAT_MASK 0x00000f00L
+#define UNIPHY_TX_CONTROL3__UNIPHY_PREMPH_CS_DAT__SHIFT 0x00000008
+#define UNIPHY_TX_CONTROL3__UNIPHY_PREMPH_PW_CLK_MASK 0x00000003L
+#define UNIPHY_TX_CONTROL3__UNIPHY_PREMPH_PW_CLK__SHIFT 0x00000000
+#define UNIPHY_TX_CONTROL3__UNIPHY_PREMPH_PW_DAT_MASK 0x0000000cL
+#define UNIPHY_TX_CONTROL3__UNIPHY_PREMPH_PW_DAT__SHIFT 0x00000002
+#define UNIPHY_TX_CONTROL3__UNIPHY_PREMPH_STR_CLK_MASK 0x00007000L
+#define UNIPHY_TX_CONTROL3__UNIPHY_PREMPH_STR_CLK__SHIFT 0x0000000c
+#define UNIPHY_TX_CONTROL3__UNIPHY_PREMPH_STR_DAT_MASK 0x00070000L
+#define UNIPHY_TX_CONTROL3__UNIPHY_PREMPH_STR_DAT__SHIFT 0x00000010
+#define UNIPHY_TX_CONTROL3__UNIPHY_TX_VS_ADJ_MASK 0x1f000000L
+#define UNIPHY_TX_CONTROL3__UNIPHY_TX_VS_ADJ__SHIFT 0x00000018
+#define UNIPHY_TX_CONTROL4__UNIPHY_TX_NVS_CLK_MASK 0x0000001fL
+#define UNIPHY_TX_CONTROL4__UNIPHY_TX_NVS_CLK__SHIFT 0x00000000
+#define UNIPHY_TX_CONTROL4__UNIPHY_TX_NVS_DAT_MASK 0x000003e0L
+#define UNIPHY_TX_CONTROL4__UNIPHY_TX_NVS_DAT__SHIFT 0x00000005
+#define UNIPHY_TX_CONTROL4__UNIPHY_TX_OP_CLK_MASK 0x07000000L
+#define UNIPHY_TX_CONTROL4__UNIPHY_TX_OP_CLK__SHIFT 0x00000018
+#define UNIPHY_TX_CONTROL4__UNIPHY_TX_OP_DAT_MASK 0x70000000L
+#define UNIPHY_TX_CONTROL4__UNIPHY_TX_OP_DAT__SHIFT 0x0000001c
+#define UNIPHY_TX_CONTROL4__UNIPHY_TX_PVS_CLK_MASK 0x0001f000L
+#define UNIPHY_TX_CONTROL4__UNIPHY_TX_PVS_CLK__SHIFT 0x0000000c
+#define UNIPHY_TX_CONTROL4__UNIPHY_TX_PVS_DAT_MASK 0x003e0000L
+#define UNIPHY_TX_CONTROL4__UNIPHY_TX_PVS_DAT__SHIFT 0x00000011
+#define VGA25_PPLL_ANALOG__VGA25_CAL_MODE_MASK 0x0000001fL
+#define VGA25_PPLL_ANALOG__VGA25_CAL_MODE__SHIFT 0x00000000
+#define VGA25_PPLL_ANALOG__VGA25_PPLL_CP_MASK 0x00000f00L
+#define VGA25_PPLL_ANALOG__VGA25_PPLL_CP__SHIFT 0x00000008
+#define VGA25_PPLL_ANALOG__VGA25_PPLL_IBIAS_MASK 0xff000000L
+#define VGA25_PPLL_ANALOG__VGA25_PPLL_IBIAS__SHIFT 0x00000018
+#define VGA25_PPLL_ANALOG__VGA25_PPLL_LF_MODE_MASK 0x001ff000L
+#define VGA25_PPLL_ANALOG__VGA25_PPLL_LF_MODE__SHIFT 0x0000000c
+#define VGA25_PPLL_ANALOG__VGA25_PPLL_PFD_PULSE_SEL_MASK 0x00000060L
+#define VGA25_PPLL_ANALOG__VGA25_PPLL_PFD_PULSE_SEL__SHIFT 0x00000005
+#define VGA25_PPLL_FB_DIV__VGA25_PPLL_FB_DIV_FRACTION_CNTL_MASK 0x00000030L
+#define VGA25_PPLL_FB_DIV__VGA25_PPLL_FB_DIV_FRACTION_CNTL__SHIFT 0x00000004
+#define VGA25_PPLL_FB_DIV__VGA25_PPLL_FB_DIV_FRACTION_MASK 0x0000000fL
+#define VGA25_PPLL_FB_DIV__VGA25_PPLL_FB_DIV_FRACTION__SHIFT 0x00000000
+#define VGA25_PPLL_FB_DIV__VGA25_PPLL_FB_DIV_MASK 0x07ff0000L
+#define VGA25_PPLL_FB_DIV__VGA25_PPLL_FB_DIV__SHIFT 0x00000010
+#define VGA25_PPLL_POST_DIV__VGA25_PPLL_POST_DIV_DVOCLK_MASK 0x00007f00L
+#define VGA25_PPLL_POST_DIV__VGA25_PPLL_POST_DIV_DVOCLK__SHIFT 0x00000008
+#define VGA25_PPLL_POST_DIV__VGA25_PPLL_POST_DIV_IDCLK_MASK 0x007f0000L
+#define VGA25_PPLL_POST_DIV__VGA25_PPLL_POST_DIV_IDCLK__SHIFT 0x00000010
+#define VGA25_PPLL_POST_DIV__VGA25_PPLL_POST_DIV_PIXCLK_MASK 0x0000007fL
+#define VGA25_PPLL_POST_DIV__VGA25_PPLL_POST_DIV_PIXCLK__SHIFT 0x00000000
+#define VGA25_PPLL_REF_DIV__VGA25_PPLL_REF_DIV_MASK 0x000003ffL
+#define VGA25_PPLL_REF_DIV__VGA25_PPLL_REF_DIV__SHIFT 0x00000000
+#define VGA28_PPLL_ANALOG__VGA28_CAL_MODE_MASK 0x0000001fL
+#define VGA28_PPLL_ANALOG__VGA28_CAL_MODE__SHIFT 0x00000000
+#define VGA28_PPLL_ANALOG__VGA28_PPLL_CP_MASK 0x00000f00L
+#define VGA28_PPLL_ANALOG__VGA28_PPLL_CP__SHIFT 0x00000008
+#define VGA28_PPLL_ANALOG__VGA28_PPLL_IBIAS_MASK 0xff000000L
+#define VGA28_PPLL_ANALOG__VGA28_PPLL_IBIAS__SHIFT 0x00000018
+#define VGA28_PPLL_ANALOG__VGA28_PPLL_LF_MODE_MASK 0x001ff000L
+#define VGA28_PPLL_ANALOG__VGA28_PPLL_LF_MODE__SHIFT 0x0000000c
+#define VGA28_PPLL_ANALOG__VGA28_PPLL_PFD_PULSE_SEL_MASK 0x00000060L
+#define VGA28_PPLL_ANALOG__VGA28_PPLL_PFD_PULSE_SEL__SHIFT 0x00000005
+#define VGA28_PPLL_FB_DIV__VGA28_PPLL_FB_DIV_FRACTION_CNTL_MASK 0x00000030L
+#define VGA28_PPLL_FB_DIV__VGA28_PPLL_FB_DIV_FRACTION_CNTL__SHIFT 0x00000004
+#define VGA28_PPLL_FB_DIV__VGA28_PPLL_FB_DIV_FRACTION_MASK 0x0000000fL
+#define VGA28_PPLL_FB_DIV__VGA28_PPLL_FB_DIV_FRACTION__SHIFT 0x00000000
+#define VGA28_PPLL_FB_DIV__VGA28_PPLL_FB_DIV_MASK 0x07ff0000L
+#define VGA28_PPLL_FB_DIV__VGA28_PPLL_FB_DIV__SHIFT 0x00000010
+#define VGA28_PPLL_POST_DIV__VGA28_PPLL_POST_DIV_DVOCLK_MASK 0x00007f00L
+#define VGA28_PPLL_POST_DIV__VGA28_PPLL_POST_DIV_DVOCLK__SHIFT 0x00000008
+#define VGA28_PPLL_POST_DIV__VGA28_PPLL_POST_DIV_IDCLK_MASK 0x007f0000L
+#define VGA28_PPLL_POST_DIV__VGA28_PPLL_POST_DIV_IDCLK__SHIFT 0x00000010
+#define VGA28_PPLL_POST_DIV__VGA28_PPLL_POST_DIV_PIXCLK_MASK 0x0000007fL
+#define VGA28_PPLL_POST_DIV__VGA28_PPLL_POST_DIV_PIXCLK__SHIFT 0x00000000
+#define VGA28_PPLL_REF_DIV__VGA28_PPLL_REF_DIV_MASK 0x000003ffL
+#define VGA28_PPLL_REF_DIV__VGA28_PPLL_REF_DIV__SHIFT 0x00000000
+#define VGA41_PPLL_ANALOG__VGA41_CAL_MODE_MASK 0x0000001fL
+#define VGA41_PPLL_ANALOG__VGA41_CAL_MODE__SHIFT 0x00000000
+#define VGA41_PPLL_ANALOG__VGA41_PPLL_CP_MASK 0x00000f00L
+#define VGA41_PPLL_ANALOG__VGA41_PPLL_CP__SHIFT 0x00000008
+#define VGA41_PPLL_ANALOG__VGA41_PPLL_IBIAS_MASK 0xff000000L
+#define VGA41_PPLL_ANALOG__VGA41_PPLL_IBIAS__SHIFT 0x00000018
+#define VGA41_PPLL_ANALOG__VGA41_PPLL_LF_MODE_MASK 0x001ff000L
+#define VGA41_PPLL_ANALOG__VGA41_PPLL_LF_MODE__SHIFT 0x0000000c
+#define VGA41_PPLL_ANALOG__VGA41_PPLL_PFD_PULSE_SEL_MASK 0x00000060L
+#define VGA41_PPLL_ANALOG__VGA41_PPLL_PFD_PULSE_SEL__SHIFT 0x00000005
+#define VGA41_PPLL_FB_DIV__VGA41_PPLL_FB_DIV_FRACTION_CNTL_MASK 0x00000030L
+#define VGA41_PPLL_FB_DIV__VGA41_PPLL_FB_DIV_FRACTION_CNTL__SHIFT 0x00000004
+#define VGA41_PPLL_FB_DIV__VGA41_PPLL_FB_DIV_FRACTION_MASK 0x0000000fL
+#define VGA41_PPLL_FB_DIV__VGA41_PPLL_FB_DIV_FRACTION__SHIFT 0x00000000
+#define VGA41_PPLL_FB_DIV__VGA41_PPLL_FB_DIV_MASK 0x07ff0000L
+#define VGA41_PPLL_FB_DIV__VGA41_PPLL_FB_DIV__SHIFT 0x00000010
+#define VGA41_PPLL_POST_DIV__VGA41_PPLL_POST_DIV_DVOCLK_MASK 0x00007f00L
+#define VGA41_PPLL_POST_DIV__VGA41_PPLL_POST_DIV_DVOCLK__SHIFT 0x00000008
+#define VGA41_PPLL_POST_DIV__VGA41_PPLL_POST_DIV_IDCLK_MASK 0x007f0000L
+#define VGA41_PPLL_POST_DIV__VGA41_PPLL_POST_DIV_IDCLK__SHIFT 0x00000010
+#define VGA41_PPLL_POST_DIV__VGA41_PPLL_POST_DIV_PIXCLK_MASK 0x0000007fL
+#define VGA41_PPLL_POST_DIV__VGA41_PPLL_POST_DIV_PIXCLK__SHIFT 0x00000000
+#define VGA41_PPLL_REF_DIV__VGA41_PPLL_REF_DIV_MASK 0x000003ffL
+#define VGA41_PPLL_REF_DIV__VGA41_PPLL_REF_DIV__SHIFT 0x00000000
+#define VGA_CACHE_CONTROL__VGA_DCCIF_W256ONLY_MASK 0x00100000L
+#define VGA_CACHE_CONTROL__VGA_DCCIF_W256ONLY__SHIFT 0x00000014
+#define VGA_CACHE_CONTROL__VGA_DCCIF_WC_TIMEOUT_MASK 0x3f000000L
+#define VGA_CACHE_CONTROL__VGA_DCCIF_WC_TIMEOUT__SHIFT 0x00000018
+#define VGA_CACHE_CONTROL__VGA_READ_BUFFER_INVALIDATE_MASK 0x00010000L
+#define VGA_CACHE_CONTROL__VGA_READ_BUFFER_INVALIDATE__SHIFT 0x00000010
+#define VGA_CACHE_CONTROL__VGA_READ_CACHE_DISABLE_MASK 0x00000100L
+#define VGA_CACHE_CONTROL__VGA_READ_CACHE_DISABLE__SHIFT 0x00000008
+#define VGA_CACHE_CONTROL__VGA_WRITE_THROUGH_CACHE_DIS_MASK 0x00000001L
+#define VGA_CACHE_CONTROL__VGA_WRITE_THROUGH_CACHE_DIS__SHIFT 0x00000000
+#define VGADCC_DBG_DCCIF_C__DBG_DCCIF_C_MASK 0xffffffffL
+#define VGADCC_DBG_DCCIF_C__DBG_DCCIF_C__SHIFT 0x00000000
+#define VGA_DEBUG_READBACK_DATA__VGA_DEBUG_READBACK_DATA_MASK 0xffffffffL
+#define VGA_DEBUG_READBACK_DATA__VGA_DEBUG_READBACK_DATA__SHIFT 0x00000000
+#define VGA_DEBUG_READBACK_INDEX__VGA_DEBUG_READBACK_INDEX_MASK 0x000000ffL
+#define VGA_DEBUG_READBACK_INDEX__VGA_DEBUG_READBACK_INDEX__SHIFT 0x00000000
+#define VGA_DISPBUF1_SURFACE_ADDR__VGA_DISPBUF1_SURFACE_ADDR_MASK 0x01ffffffL
+#define VGA_DISPBUF1_SURFACE_ADDR__VGA_DISPBUF1_SURFACE_ADDR__SHIFT 0x00000000
+#define VGA_DISPBUF2_SURFACE_ADDR__VGA_DISPBUF2_SURFACE_ADDR_MASK 0x01ffffffL
+#define VGA_DISPBUF2_SURFACE_ADDR__VGA_DISPBUF2_SURFACE_ADDR__SHIFT 0x00000000
+#define VGA_HDP_CONTROL__VGA_MEMORY_DISABLE_MASK 0x00000010L
+#define VGA_HDP_CONTROL__VGA_MEMORY_DISABLE__SHIFT 0x00000004
+#define VGA_HDP_CONTROL__VGA_MEM_PAGE_SELECT_EN_MASK 0x00000001L
+#define VGA_HDP_CONTROL__VGA_MEM_PAGE_SELECT_EN__SHIFT 0x00000000
+#define VGA_HDP_CONTROL__VGA_RBBM_LOCK_DISABLE_MASK 0x00000100L
+#define VGA_HDP_CONTROL__VGA_RBBM_LOCK_DISABLE__SHIFT 0x00000008
+#define VGA_HDP_CONTROL__VGA_SOFT_RESET_MASK 0x00010000L
+#define VGA_HDP_CONTROL__VGA_SOFT_RESET__SHIFT 0x00000010
+#define VGA_HDP_CONTROL__VGA_TEST_RESET_CONTROL_MASK 0x01000000L
+#define VGA_HDP_CONTROL__VGA_TEST_RESET_CONTROL__SHIFT 0x00000018
+#define VGA_HW_DEBUG__VGA_HW_DEBUG_MASK 0xffffffffL
+#define VGA_HW_DEBUG__VGA_HW_DEBUG__SHIFT 0x00000000
+#define VGA_INTERRUPT_CONTROL__VGA_DISPLAY_SWITCH_INT_MASK_MASK 0x00010000L
+#define VGA_INTERRUPT_CONTROL__VGA_DISPLAY_SWITCH_INT_MASK__SHIFT 0x00000010
+#define VGA_INTERRUPT_CONTROL__VGA_MEM_ACCESS_INT_MASK_MASK 0x00000001L
+#define VGA_INTERRUPT_CONTROL__VGA_MEM_ACCESS_INT_MASK__SHIFT 0x00000000
+#define VGA_INTERRUPT_CONTROL__VGA_MODE_AUTO_TRIGGER_INT_MASK_MASK 0x01000000L
+#define VGA_INTERRUPT_CONTROL__VGA_MODE_AUTO_TRIGGER_INT_MASK__SHIFT 0x00000018
+#define VGA_INTERRUPT_CONTROL__VGA_REG_ACCESS_INT_MASK_MASK 0x00000100L
+#define VGA_INTERRUPT_CONTROL__VGA_REG_ACCESS_INT_MASK__SHIFT 0x00000008
+#define VGA_INTERRUPT_STATUS__VGA_DISPLAY_SWITCH_INT_STATUS_MASK 0x00000004L
+#define VGA_INTERRUPT_STATUS__VGA_DISPLAY_SWITCH_INT_STATUS__SHIFT 0x00000002
+#define VGA_INTERRUPT_STATUS__VGA_MEM_ACCESS_INT_STATUS_MASK 0x00000001L
+#define VGA_INTERRUPT_STATUS__VGA_MEM_ACCESS_INT_STATUS__SHIFT 0x00000000
+#define VGA_INTERRUPT_STATUS__VGA_MODE_AUTO_TRIGGER_INT_STATUS_MASK 0x00000008L
+#define VGA_INTERRUPT_STATUS__VGA_MODE_AUTO_TRIGGER_INT_STATUS__SHIFT 0x00000003
+#define VGA_INTERRUPT_STATUS__VGA_REG_ACCESS_INT_STATUS_MASK 0x00000002L
+#define VGA_INTERRUPT_STATUS__VGA_REG_ACCESS_INT_STATUS__SHIFT 0x00000001
+#define VGA_MAIN_CONTROL__VGA_CRTC_TIMEOUT_MASK 0x00000003L
+#define VGA_MAIN_CONTROL__VGA_CRTC_TIMEOUT__SHIFT 0x00000000
+#define VGA_MAIN_CONTROL__VGA_EXTERNAL_DAC_SENSE_MASK 0x20000000L
+#define VGA_MAIN_CONTROL__VGA_EXTERNAL_DAC_SENSE__SHIFT 0x0000001d
+#define VGA_MAIN_CONTROL__VGA_MAIN_TEST_VSTATUS_NO_DISPLAY_CRTC_TIMEOUT_MASK 0x80000000L
+#define VGA_MAIN_CONTROL__VGA_MAIN_TEST_VSTATUS_NO_DISPLAY_CRTC_TIMEOUT__SHIFT 0x0000001f
+#define VGA_MAIN_CONTROL__VGA_READBACK_CRT_INTR_SOURCE_SELECT_MASK 0x03000000L
+#define VGA_MAIN_CONTROL__VGA_READBACK_CRT_INTR_SOURCE_SELECT__SHIFT 0x00000018
+#define VGA_MAIN_CONTROL__VGA_READBACK_NO_DISPLAY_SOURCE_SELECT_MASK 0x00030000L
+#define VGA_MAIN_CONTROL__VGA_READBACK_NO_DISPLAY_SOURCE_SELECT__SHIFT 0x00000010
+#define VGA_MAIN_CONTROL__VGA_READBACK_SENSE_SWITCH_SELECT_MASK 0x04000000L
+#define VGA_MAIN_CONTROL__VGA_READBACK_SENSE_SWITCH_SELECT__SHIFT 0x0000001a
+#define VGA_MAIN_CONTROL__VGA_READBACK_VGA_VSTATUS_SOURCE_SELECT_MASK 0x00000300L
+#define VGA_MAIN_CONTROL__VGA_READBACK_VGA_VSTATUS_SOURCE_SELECT__SHIFT 0x00000008
+#define VGA_MAIN_CONTROL__VGA_READ_URGENT_ENABLE_MASK 0x08000000L
+#define VGA_MAIN_CONTROL__VGA_READ_URGENT_ENABLE__SHIFT 0x0000001b
+#define VGA_MAIN_CONTROL__VGA_RENDER_TIMEOUT_COUNT_MASK 0x00000018L
+#define VGA_MAIN_CONTROL__VGA_RENDER_TIMEOUT_COUNT__SHIFT 0x00000003
+#define VGA_MAIN_CONTROL__VGA_VIRTUAL_VERTICAL_RETRACE_DURATION_MASK 0x000000e0L
+#define VGA_MAIN_CONTROL__VGA_VIRTUAL_VERTICAL_RETRACE_DURATION__SHIFT 0x00000005
+#define VGA_MAIN_CONTROL__VGA_WRITES_URGENT_ENABLE_MASK 0x10000000L
+#define VGA_MAIN_CONTROL__VGA_WRITES_URGENT_ENABLE__SHIFT 0x0000001c
+#define VGA_MEMORY_BASE_ADDRESS_HIGH__VGA_MEMORY_BASE_ADDRESS_HIGH_MASK 0x000000ffL
+#define VGA_MEMORY_BASE_ADDRESS_HIGH__VGA_MEMORY_BASE_ADDRESS_HIGH__SHIFT 0x00000000
+#define VGA_MEMORY_BASE_ADDRESS__VGA_MEMORY_BASE_ADDRESS_MASK 0xffffffffL
+#define VGA_MEMORY_BASE_ADDRESS__VGA_MEMORY_BASE_ADDRESS__SHIFT 0x00000000
+#define VGA_MEM_READ_PAGE_ADDR__VGA_MEM_READ_PAGE0_ADDR_MASK 0x000003ffL
+#define VGA_MEM_READ_PAGE_ADDR__VGA_MEM_READ_PAGE0_ADDR__SHIFT 0x00000000
+#define VGA_MEM_READ_PAGE_ADDR__VGA_MEM_READ_PAGE1_ADDR_MASK 0x03ff0000L
+#define VGA_MEM_READ_PAGE_ADDR__VGA_MEM_READ_PAGE1_ADDR__SHIFT 0x00000010
+#define VGA_MEM_WRITE_PAGE_ADDR__VGA_MEM_WRITE_PAGE0_ADDR_MASK 0x000003ffL
+#define VGA_MEM_WRITE_PAGE_ADDR__VGA_MEM_WRITE_PAGE0_ADDR__SHIFT 0x00000000
+#define VGA_MEM_WRITE_PAGE_ADDR__VGA_MEM_WRITE_PAGE1_ADDR_MASK 0x03ff0000L
+#define VGA_MEM_WRITE_PAGE_ADDR__VGA_MEM_WRITE_PAGE1_ADDR__SHIFT 0x00000010
+#define VGA_MODE_CONTROL__VGA_128K_APERTURE_PAGING_MASK 0x00000100L
+#define VGA_MODE_CONTROL__VGA_128K_APERTURE_PAGING__SHIFT 0x00000008
+#define VGA_MODE_CONTROL__VGA_ATI_LINEAR_MASK 0x00000001L
+#define VGA_MODE_CONTROL__VGA_ATI_LINEAR__SHIFT 0x00000000
+#define VGA_MODE_CONTROL__VGA_LUT_PALETTE_UPDATE_MODE_MASK 0x00000030L
+#define VGA_MODE_CONTROL__VGA_LUT_PALETTE_UPDATE_MODE__SHIFT 0x00000004
+#define VGA_MODE_CONTROL__VGA_TEXT_132_COLUMNS_EN_MASK 0x00010000L
+#define VGA_MODE_CONTROL__VGA_TEXT_132_COLUMNS_EN__SHIFT 0x00000010
+#define VGA_RENDER_CONTROL__VGA_BLINK_MODE_MASK 0x00000060L
+#define VGA_RENDER_CONTROL__VGA_BLINK_MODE__SHIFT 0x00000005
+#define VGA_RENDER_CONTROL__VGA_BLINK_RATE_MASK 0x0000001fL
+#define VGA_RENDER_CONTROL__VGA_BLINK_RATE__SHIFT 0x00000000
+#define VGA_RENDER_CONTROL__VGA_CURSOR_BLINK_INVERT_MASK 0x00000080L
+#define VGA_RENDER_CONTROL__VGA_CURSOR_BLINK_INVERT__SHIFT 0x00000007
+#define VGA_RENDER_CONTROL__VGA_EXTD_ADDR_COUNT_ENABLE_MASK 0x00000100L
+#define VGA_RENDER_CONTROL__VGA_EXTD_ADDR_COUNT_ENABLE__SHIFT 0x00000008
+#define VGA_RENDER_CONTROL__VGA_LOCK_8DOT_MASK 0x01000000L
+#define VGA_RENDER_CONTROL__VGA_LOCK_8DOT__SHIFT 0x00000018
+#define VGA_RENDER_CONTROL__VGAREG_LINECMP_COMPATIBILITY_SEL_MASK 0x02000000L
+#define VGA_RENDER_CONTROL__VGAREG_LINECMP_COMPATIBILITY_SEL__SHIFT 0x00000019
+#define VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK 0x00030000L
+#define VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL__SHIFT 0x00000010
+#define VGA_SEQUENCER_RESET_CONTROL__D1_BLANK_DISPLAY_WHEN_SEQUENCER_RESET_MASK 0x00000001L
+#define VGA_SEQUENCER_RESET_CONTROL__D1_BLANK_DISPLAY_WHEN_SEQUENCER_RESET__SHIFT 0x00000000
+#define VGA_SEQUENCER_RESET_CONTROL__D1_DISABLE_SYNCS_AND_DE_WHEN_SEQUENCER_RESET_MASK 0x00000100L
+#define VGA_SEQUENCER_RESET_CONTROL__D1_DISABLE_SYNCS_AND_DE_WHEN_SEQUENCER_RESET__SHIFT 0x00000008
+#define VGA_SEQUENCER_RESET_CONTROL__D2_BLANK_DISPLAY_WHEN_SEQUENCER_RESET_MASK 0x00000002L
+#define VGA_SEQUENCER_RESET_CONTROL__D2_BLANK_DISPLAY_WHEN_SEQUENCER_RESET__SHIFT 0x00000001
+#define VGA_SEQUENCER_RESET_CONTROL__D2_DISABLE_SYNCS_AND_DE_WHEN_SEQUENCER_RESET_MASK 0x00000200L
+#define VGA_SEQUENCER_RESET_CONTROL__D2_DISABLE_SYNCS_AND_DE_WHEN_SEQUENCER_RESET__SHIFT 0x00000009
+#define VGA_SEQUENCER_RESET_CONTROL__D3_BLANK_DISPLAY_WHEN_SEQUENCER_RESET_MASK 0x00000004L
+#define VGA_SEQUENCER_RESET_CONTROL__D3_BLANK_DISPLAY_WHEN_SEQUENCER_RESET__SHIFT 0x00000002
+#define VGA_SEQUENCER_RESET_CONTROL__D3_DISABLE_SYNCS_AND_DE_WHEN_SEQUENCER_RESET_MASK 0x00000400L
+#define VGA_SEQUENCER_RESET_CONTROL__D3_DISABLE_SYNCS_AND_DE_WHEN_SEQUENCER_RESET__SHIFT 0x0000000a
+#define VGA_SEQUENCER_RESET_CONTROL__D4_BLANK_DISPLAY_WHEN_SEQUENCER_RESET_MASK 0x00000008L
+#define VGA_SEQUENCER_RESET_CONTROL__D4_BLANK_DISPLAY_WHEN_SEQUENCER_RESET__SHIFT 0x00000003
+#define VGA_SEQUENCER_RESET_CONTROL__D4_DISABLE_SYNCS_AND_DE_WHEN_SEQUENCER_RESET_MASK 0x00000800L
+#define VGA_SEQUENCER_RESET_CONTROL__D4_DISABLE_SYNCS_AND_DE_WHEN_SEQUENCER_RESET__SHIFT 0x0000000b
+#define VGA_SEQUENCER_RESET_CONTROL__D5_BLANK_DISPLAY_WHEN_SEQUENCER_RESET_MASK 0x00000010L
+#define VGA_SEQUENCER_RESET_CONTROL__D5_BLANK_DISPLAY_WHEN_SEQUENCER_RESET__SHIFT 0x00000004
+#define VGA_SEQUENCER_RESET_CONTROL__D5_DISABLE_SYNCS_AND_DE_WHEN_SEQUENCER_RESET_MASK 0x00001000L
+#define VGA_SEQUENCER_RESET_CONTROL__D5_DISABLE_SYNCS_AND_DE_WHEN_SEQUENCER_RESET__SHIFT 0x0000000c
+#define VGA_SEQUENCER_RESET_CONTROL__D6_BLANK_DISPLAY_WHEN_SEQUENCER_RESET_MASK 0x00000020L
+#define VGA_SEQUENCER_RESET_CONTROL__D6_BLANK_DISPLAY_WHEN_SEQUENCER_RESET__SHIFT 0x00000005
+#define VGA_SEQUENCER_RESET_CONTROL__D6_DISABLE_SYNCS_AND_DE_WHEN_SEQUENCER_RESET_MASK 0x00002000L
+#define VGA_SEQUENCER_RESET_CONTROL__D6_DISABLE_SYNCS_AND_DE_WHEN_SEQUENCER_RESET__SHIFT 0x0000000d
+#define VGA_SEQUENCER_RESET_CONTROL__VGA_MODE_AUTO_TRIGGER_ENABLE_MASK 0x00010000L
+#define VGA_SEQUENCER_RESET_CONTROL__VGA_MODE_AUTO_TRIGGER_ENABLE__SHIFT 0x00000010
+#define VGA_SEQUENCER_RESET_CONTROL__VGA_MODE_AUTO_TRIGGER_INDEX_SELECT_MASK 0x00fc0000L
+#define VGA_SEQUENCER_RESET_CONTROL__VGA_MODE_AUTO_TRIGGER_INDEX_SELECT__SHIFT 0x00000012
+#define VGA_SEQUENCER_RESET_CONTROL__VGA_MODE_AUTO_TRIGGER_REGISTER_SELECT_MASK 0x00020000L
+#define VGA_SEQUENCER_RESET_CONTROL__VGA_MODE_AUTO_TRIGGER_REGISTER_SELECT__SHIFT 0x00000011
+#define VGA_SOURCE_SELECT__VGA_SOURCE_SEL_A_MASK 0x00000007L
+#define VGA_SOURCE_SELECT__VGA_SOURCE_SEL_A__SHIFT 0x00000000
+#define VGA_SOURCE_SELECT__VGA_SOURCE_SEL_B_MASK 0x00000700L
+#define VGA_SOURCE_SELECT__VGA_SOURCE_SEL_B__SHIFT 0x00000008
+#define VGA_STATUS_CLEAR__VGA_DISPLAY_SWITCH_INT_CLEAR_MASK 0x00010000L
+#define VGA_STATUS_CLEAR__VGA_DISPLAY_SWITCH_INT_CLEAR__SHIFT 0x00000010
+#define VGA_STATUS_CLEAR__VGA_MEM_ACCESS_INT_CLEAR_MASK 0x00000001L
+#define VGA_STATUS_CLEAR__VGA_MEM_ACCESS_INT_CLEAR__SHIFT 0x00000000
+#define VGA_STATUS_CLEAR__VGA_MODE_AUTO_TRIGGER_INT_CLEAR_MASK 0x01000000L
+#define VGA_STATUS_CLEAR__VGA_MODE_AUTO_TRIGGER_INT_CLEAR__SHIFT 0x00000018
+#define VGA_STATUS_CLEAR__VGA_REG_ACCESS_INT_CLEAR_MASK 0x00000100L
+#define VGA_STATUS_CLEAR__VGA_REG_ACCESS_INT_CLEAR__SHIFT 0x00000008
+#define VGA_STATUS__VGA_DISPLAY_SWITCH_STATUS_MASK 0x00000004L
+#define VGA_STATUS__VGA_DISPLAY_SWITCH_STATUS__SHIFT 0x00000002
+#define VGA_STATUS__VGA_MEM_ACCESS_STATUS_MASK 0x00000001L
+#define VGA_STATUS__VGA_MEM_ACCESS_STATUS__SHIFT 0x00000000
+#define VGA_STATUS__VGA_MODE_AUTO_TRIGGER_STATUS_MASK 0x00000008L
+#define VGA_STATUS__VGA_MODE_AUTO_TRIGGER_STATUS__SHIFT 0x00000003
+#define VGA_STATUS__VGA_REG_ACCESS_STATUS_MASK 0x00000002L
+#define VGA_STATUS__VGA_REG_ACCESS_STATUS__SHIFT 0x00000001
+#define VGA_SURFACE_PITCH_SELECT__VGA_SURFACE_HEIGHT_SELECT_MASK 0x00000300L
+#define VGA_SURFACE_PITCH_SELECT__VGA_SURFACE_HEIGHT_SELECT__SHIFT 0x00000008
+#define VGA_SURFACE_PITCH_SELECT__VGA_SURFACE_PITCH_SELECT_MASK 0x00000003L
+#define VGA_SURFACE_PITCH_SELECT__VGA_SURFACE_PITCH_SELECT__SHIFT 0x00000000
+#define VGA_TEST_CONTROL__VGA_TEST_ENABLE_MASK 0x00000001L
+#define VGA_TEST_CONTROL__VGA_TEST_ENABLE__SHIFT 0x00000000
+#define VGA_TEST_CONTROL__VGA_TEST_RENDER_DISPBUF_SELECT_MASK 0x01000000L
+#define VGA_TEST_CONTROL__VGA_TEST_RENDER_DISPBUF_SELECT__SHIFT 0x00000018
+#define VGA_TEST_CONTROL__VGA_TEST_RENDER_DONE_MASK 0x00010000L
+#define VGA_TEST_CONTROL__VGA_TEST_RENDER_DONE__SHIFT 0x00000010
+#define VGA_TEST_CONTROL__VGA_TEST_RENDER_START_MASK 0x00000100L
+#define VGA_TEST_CONTROL__VGA_TEST_RENDER_START__SHIFT 0x00000008
+#define VGA_TEST_DEBUG_DATA__VGA_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define VGA_TEST_DEBUG_DATA__VGA_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define VGA_TEST_DEBUG_INDEX__VGA_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define VGA_TEST_DEBUG_INDEX__VGA_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define VGA_TEST_DEBUG_INDEX__VGA_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define VGA_TEST_DEBUG_INDEX__VGA_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define VIEWPORT_SIZE__VIEWPORT_HEIGHT_MASK 0x00003fffL
+#define VIEWPORT_SIZE__VIEWPORT_HEIGHT__SHIFT 0x00000000
+#define VIEWPORT_SIZE__VIEWPORT_WIDTH_MASK 0x3fff0000L
+#define VIEWPORT_SIZE__VIEWPORT_WIDTH__SHIFT 0x00000010
+#define VIEWPORT_START__VIEWPORT_X_START_MASK 0x3fff0000L
+#define VIEWPORT_START__VIEWPORT_X_START__SHIFT 0x00000010
+#define VIEWPORT_START__VIEWPORT_Y_START_MASK 0x00003fffL
+#define VIEWPORT_START__VIEWPORT_Y_START__SHIFT 0x00000000
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_GATE_DIS_MASK 0x00008000L
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_GATE_DIS__SHIFT 0x0000000f
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_G_MSTAT_GATE_DIS_MASK 0x00080000L
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_G_MSTAT_GATE_DIS__SHIFT 0x00000013
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_G_SDYN_GATE_DIS_MASK 0x00040000L
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_G_SDYN_GATE_DIS__SHIFT 0x00000012
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_G_SSTAT_GATE_DIS_MASK 0x00100000L
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_G_SSTAT_GATE_DIS__SHIFT 0x00000014
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_REG_GATE_DIS_MASK 0x00010000L
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_REG_GATE_DIS__SHIFT 0x00000010
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_TURN_OFF_DELAY_MASK 0x00000ff0L
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_TURN_OFF_DELAY__SHIFT 0x00000004
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_TURN_ON_DELAY_MASK 0x0000000fL
+#define XDMA_CLOCK_GATING_CNTL__XDMA_SCLK_TURN_ON_DELAY__SHIFT 0x00000000
+#define XDMA_IF_BIF_STATUS__XDMA_IF_BIF_ERROR_CLEAR_MASK 0x00000100L
+#define XDMA_IF_BIF_STATUS__XDMA_IF_BIF_ERROR_CLEAR__SHIFT 0x00000008
+#define XDMA_IF_BIF_STATUS__XDMA_IF_BIF_ERROR_STATUS_MASK 0x0000000fL
+#define XDMA_IF_BIF_STATUS__XDMA_IF_BIF_ERROR_STATUS__SHIFT 0x00000000
+#define XDMA_INTERRUPT__XDMA_MSTR_MEM_URGENT_ACK_MASK 0x00000400L
+#define XDMA_INTERRUPT__XDMA_MSTR_MEM_URGENT_ACK__SHIFT 0x0000000a
+#define XDMA_INTERRUPT__XDMA_MSTR_MEM_URGENT_MASK_MASK 0x00000200L
+#define XDMA_INTERRUPT__XDMA_MSTR_MEM_URGENT_MASK__SHIFT 0x00000009
+#define XDMA_INTERRUPT__XDMA_MSTR_MEM_URGENT_STAT_MASK 0x00000100L
+#define XDMA_INTERRUPT__XDMA_MSTR_MEM_URGENT_STAT__SHIFT 0x00000008
+#define XDMA_INTERRUPT__XDMA_MSTR_UNDERFLOW_ACK_MASK 0x00004000L
+#define XDMA_INTERRUPT__XDMA_MSTR_UNDERFLOW_ACK__SHIFT 0x0000000e
+#define XDMA_INTERRUPT__XDMA_MSTR_UNDERFLOW_MASK_MASK 0x00002000L
+#define XDMA_INTERRUPT__XDMA_MSTR_UNDERFLOW_MASK__SHIFT 0x0000000d
+#define XDMA_INTERRUPT__XDMA_MSTR_UNDERFLOW_STAT_MASK 0x00001000L
+#define XDMA_INTERRUPT__XDMA_MSTR_UNDERFLOW_STAT__SHIFT 0x0000000c
+#define XDMA_INTERRUPT__XDMA_SLV_READ_URGENT_ACK_MASK 0x00040000L
+#define XDMA_INTERRUPT__XDMA_SLV_READ_URGENT_ACK__SHIFT 0x00000012
+#define XDMA_INTERRUPT__XDMA_SLV_READ_URGENT_MASK_MASK 0x00020000L
+#define XDMA_INTERRUPT__XDMA_SLV_READ_URGENT_MASK__SHIFT 0x00000011
+#define XDMA_INTERRUPT__XDMA_SLV_READ_URGENT_STAT_MASK 0x00010000L
+#define XDMA_INTERRUPT__XDMA_SLV_READ_URGENT_STAT__SHIFT 0x00000010
+#define XDMA_LOCAL_SURFACE_TILING1__XDMA_LOCAL_ARRAY_MODE_MASK 0x0000000fL
+#define XDMA_LOCAL_SURFACE_TILING1__XDMA_LOCAL_ARRAY_MODE__SHIFT 0x00000000
+#define XDMA_LOCAL_SURFACE_TILING1__XDMA_LOCAL_BANK_HEIGHT_MASK 0x00000c00L
+#define XDMA_LOCAL_SURFACE_TILING1__XDMA_LOCAL_BANK_HEIGHT__SHIFT 0x0000000a
+#define XDMA_LOCAL_SURFACE_TILING1__XDMA_LOCAL_BANK_WIDTH_MASK 0x00000300L
+#define XDMA_LOCAL_SURFACE_TILING1__XDMA_LOCAL_BANK_WIDTH__SHIFT 0x00000008
+#define XDMA_LOCAL_SURFACE_TILING1__XDMA_LOCAL_MACRO_TILE_ASPECT_MASK 0x00003000L
+#define XDMA_LOCAL_SURFACE_TILING1__XDMA_LOCAL_MACRO_TILE_ASPECT__SHIFT 0x0000000c
+#define XDMA_LOCAL_SURFACE_TILING1__XDMA_LOCAL_NUM_BANKS_MASK 0x00300000L
+#define XDMA_LOCAL_SURFACE_TILING1__XDMA_LOCAL_NUM_BANKS__SHIFT 0x00000014
+#define XDMA_LOCAL_SURFACE_TILING1__XDMA_LOCAL_TILE_SPLIT_MASK 0x00000070L
+#define XDMA_LOCAL_SURFACE_TILING1__XDMA_LOCAL_TILE_SPLIT__SHIFT 0x00000004
+#define XDMA_LOCAL_SURFACE_TILING2__XDMA_LOCAL_MICRO_TILE_MODE_MASK 0x00c00000L
+#define XDMA_LOCAL_SURFACE_TILING2__XDMA_LOCAL_MICRO_TILE_MODE__SHIFT 0x00000016
+#define XDMA_LOCAL_SURFACE_TILING2__XDMA_LOCAL_PIPE_CONFIG_MASK 0xf8000000L
+#define XDMA_LOCAL_SURFACE_TILING2__XDMA_LOCAL_PIPE_CONFIG__SHIFT 0x0000001b
+#define XDMA_LOCAL_SURFACE_TILING2__XDMA_LOCAL_PIPE_INTERLEAVE_SIZE_MASK 0x00000007L
+#define XDMA_LOCAL_SURFACE_TILING2__XDMA_LOCAL_PIPE_INTERLEAVE_SIZE__SHIFT 0x00000000
+#define XDMA_MC_PCIE_CLIENT_CONFIG__XDMA_MC_PCIE_PRIV_MASK 0x00010000L
+#define XDMA_MC_PCIE_CLIENT_CONFIG__XDMA_MC_PCIE_PRIV__SHIFT 0x00000010
+#define XDMA_MC_PCIE_CLIENT_CONFIG__XDMA_MC_PCIE_SWAP_MASK 0x00000300L
+#define XDMA_MC_PCIE_CLIENT_CONFIG__XDMA_MC_PCIE_SWAP__SHIFT 0x00000008
+#define XDMA_MC_PCIE_CLIENT_CONFIG__XDMA_MC_PCIE_VMID_MASK 0x0000f000L
+#define XDMA_MC_PCIE_CLIENT_CONFIG__XDMA_MC_PCIE_VMID__SHIFT 0x0000000c
+#define XDMA_MEM_POWER_CNTL__XDMA_MEM_LIGHT_SLEEP_DIS_MASK 0x00000001L
+#define XDMA_MEM_POWER_CNTL__XDMA_MEM_LIGHT_SLEEP_DIS__SHIFT 0x00000000
+#define XDMA_MEM_POWER_CNTL__XDMA_MEM_LIGHT_SLEEP_MODE_FORCE_MASK 0x00010000L
+#define XDMA_MEM_POWER_CNTL__XDMA_MEM_LIGHT_SLEEP_MODE_FORCE__SHIFT 0x00000010
+#define XDMA_MEM_POWER_CNTL__XDMA_MEM_POWER_STATE_MASK 0xc0000000L
+#define XDMA_MEM_POWER_CNTL__XDMA_MEM_POWER_STATE__SHIFT 0x0000001e
+#define XDMA_MEM_POWER_CNTL__XDMA_MEM_SHUTDOWN_DIS_MASK 0x00000100L
+#define XDMA_MEM_POWER_CNTL__XDMA_MEM_SHUTDOWN_DIS__SHIFT 0x00000008
+#define XDMA_MEM_POWER_CNTL__XDMA_MEM_SHUTDOWN_MODE_FORCE_MASK 0x01000000L
+#define XDMA_MEM_POWER_CNTL__XDMA_MEM_SHUTDOWN_MODE_FORCE__SHIFT 0x00000018
+#define XDMA_MSTR_CMD_URGENT_CNTL__XDMA_MSTR_CMD_URGENT_LEVEL_MASK 0x00000f00L
+#define XDMA_MSTR_CMD_URGENT_CNTL__XDMA_MSTR_CMD_URGENT_LEVEL__SHIFT 0x00000008
+#define XDMA_MSTR_CNTL__XDMA_MSTR_DEBUG_MODE_MASK 0x00040000L
+#define XDMA_MSTR_CNTL__XDMA_MSTR_DEBUG_MODE__SHIFT 0x00000012
+#define XDMA_MSTR_CNTL__XDMA_MSTR_ENABLE_MASK 0x00010000L
+#define XDMA_MSTR_CNTL__XDMA_MSTR_ENABLE__SHIFT 0x00000010
+#define XDMA_MSTR_CNTL__XDMA_MSTR_MEM_READY_MASK 0x00000200L
+#define XDMA_MSTR_CNTL__XDMA_MSTR_MEM_READY__SHIFT 0x00000009
+#define XDMA_MSTR_CNTL__XDMA_MSTR_SOFT_RESET_MASK 0x00100000L
+#define XDMA_MSTR_CNTL__XDMA_MSTR_SOFT_RESET__SHIFT 0x00000014
+#define XDMA_MSTR_HEIGHT__XDMA_MSTR_ACTIVE_HEIGHT_MASK 0x00003fffL
+#define XDMA_MSTR_HEIGHT__XDMA_MSTR_ACTIVE_HEIGHT__SHIFT 0x00000000
+#define XDMA_MSTR_HEIGHT__XDMA_MSTR_FRAME_HEIGHT_MASK 0x3fff0000L
+#define XDMA_MSTR_HEIGHT__XDMA_MSTR_FRAME_HEIGHT__SHIFT 0x00000010
+#define XDMA_MSTR_LOCAL_SURFACE_BASE_ADDR_HIGH__XDMA_MSTR_LOCAL_SURFACE_BASE_ADDR_HIGH_MASK 0x000000ffL
+#define XDMA_MSTR_LOCAL_SURFACE_BASE_ADDR_HIGH__XDMA_MSTR_LOCAL_SURFACE_BASE_ADDR_HIGH__SHIFT 0x00000000
+#define XDMA_MSTR_LOCAL_SURFACE_BASE_ADDR__XDMA_MSTR_LOCAL_SURFACE_BASE_ADDR_MASK 0xffffffffL
+#define XDMA_MSTR_LOCAL_SURFACE_BASE_ADDR__XDMA_MSTR_LOCAL_SURFACE_BASE_ADDR__SHIFT 0x00000000
+#define XDMA_MSTR_LOCAL_SURFACE_PITCH__XDMA_MSTR_LOCAL_SURFACE_PITCH_MASK 0x00003fffL
+#define XDMA_MSTR_LOCAL_SURFACE_PITCH__XDMA_MSTR_LOCAL_SURFACE_PITCH__SHIFT 0x00000000
+#define XDMA_MSTR_MEM_CLIENT_CONFIG__XDMA_MSTR_MEM_CLIENT_PRIV_MASK 0x00010000L
+#define XDMA_MSTR_MEM_CLIENT_CONFIG__XDMA_MSTR_MEM_CLIENT_PRIV__SHIFT 0x00000010
+#define XDMA_MSTR_MEM_CLIENT_CONFIG__XDMA_MSTR_MEM_CLIENT_SWAP_MASK 0x00000300L
+#define XDMA_MSTR_MEM_CLIENT_CONFIG__XDMA_MSTR_MEM_CLIENT_SWAP__SHIFT 0x00000008
+#define XDMA_MSTR_MEM_CLIENT_CONFIG__XDMA_MSTR_MEM_CLIENT_VMID_MASK 0x0000f000L
+#define XDMA_MSTR_MEM_CLIENT_CONFIG__XDMA_MSTR_MEM_CLIENT_VMID__SHIFT 0x0000000c
+#define XDMA_MSTR_MEM_NACK_STATUS__XDMA_MSTR_MEM_NACK_CLR_MASK 0x00010000L
+#define XDMA_MSTR_MEM_NACK_STATUS__XDMA_MSTR_MEM_NACK_CLR__SHIFT 0x00000010
+#define XDMA_MSTR_MEM_NACK_STATUS__XDMA_MSTR_MEM_NACK_MASK 0x00003000L
+#define XDMA_MSTR_MEM_NACK_STATUS__XDMA_MSTR_MEM_NACK__SHIFT 0x0000000c
+#define XDMA_MSTR_MEM_NACK_STATUS__XDMA_MSTR_MEM_NACK_TAG_MASK 0x000003ffL
+#define XDMA_MSTR_MEM_NACK_STATUS__XDMA_MSTR_MEM_NACK_TAG__SHIFT 0x00000000
+#define XDMA_MSTR_MEM_URGENT_CNTL__XDMA_MSTR_MEM_CLIENT_STALL_MASK 0x00000001L
+#define XDMA_MSTR_MEM_URGENT_CNTL__XDMA_MSTR_MEM_CLIENT_STALL__SHIFT 0x00000000
+#define XDMA_MSTR_MEM_URGENT_CNTL__XDMA_MSTR_MEM_STALL_DELAY_MASK 0x0000f000L
+#define XDMA_MSTR_MEM_URGENT_CNTL__XDMA_MSTR_MEM_STALL_DELAY__SHIFT 0x0000000c
+#define XDMA_MSTR_MEM_URGENT_CNTL__XDMA_MSTR_MEM_URGENT_LEVEL_MASK 0x00000f00L
+#define XDMA_MSTR_MEM_URGENT_CNTL__XDMA_MSTR_MEM_URGENT_LEVEL__SHIFT 0x00000008
+#define XDMA_MSTR_MEM_URGENT_CNTL__XDMA_MSTR_MEM_URGENT_LIMIT_MASK 0x000000f0L
+#define XDMA_MSTR_MEM_URGENT_CNTL__XDMA_MSTR_MEM_URGENT_LIMIT__SHIFT 0x00000004
+#define XDMA_MSTR_MEM_URGENT_CNTL__XDMA_MSTR_MEM_URGENT_TIMER_MASK 0xffff0000L
+#define XDMA_MSTR_MEM_URGENT_CNTL__XDMA_MSTR_MEM_URGENT_TIMER__SHIFT 0x00000010
+#define XDMA_MSTR_PCIE_NACK_STATUS__XDMA_MSTR_PCIE_NACK_CLR_MASK 0x00010000L
+#define XDMA_MSTR_PCIE_NACK_STATUS__XDMA_MSTR_PCIE_NACK_CLR__SHIFT 0x00000010
+#define XDMA_MSTR_PCIE_NACK_STATUS__XDMA_MSTR_PCIE_NACK_MASK 0x00003000L
+#define XDMA_MSTR_PCIE_NACK_STATUS__XDMA_MSTR_PCIE_NACK__SHIFT 0x0000000c
+#define XDMA_MSTR_PCIE_NACK_STATUS__XDMA_MSTR_PCIE_NACK_TAG_MASK 0x000003ffL
+#define XDMA_MSTR_PCIE_NACK_STATUS__XDMA_MSTR_PCIE_NACK_TAG__SHIFT 0x00000000
+#define XDMA_MSTR_READ_COMMAND__XDMA_MSTR_REQUEST_PREFETCH_MASK 0x3fff0000L
+#define XDMA_MSTR_READ_COMMAND__XDMA_MSTR_REQUEST_PREFETCH__SHIFT 0x00000010
+#define XDMA_MSTR_READ_COMMAND__XDMA_MSTR_REQUEST_SIZE_MASK 0x00003fffL
+#define XDMA_MSTR_READ_COMMAND__XDMA_MSTR_REQUEST_SIZE__SHIFT 0x00000000
+#define XDMA_MSTR_REMOTE_GPU_ADDRESS_HIGH__XDMA_MSTR_REMOTE_GPU_ADDRESS_HIGH_MASK 0x000000ffL
+#define XDMA_MSTR_REMOTE_GPU_ADDRESS_HIGH__XDMA_MSTR_REMOTE_GPU_ADDRESS_HIGH__SHIFT 0x00000000
+#define XDMA_MSTR_REMOTE_GPU_ADDRESS__XDMA_MSTR_REMOTE_GPU_ADDRESS_MASK 0xffffffffL
+#define XDMA_MSTR_REMOTE_GPU_ADDRESS__XDMA_MSTR_REMOTE_GPU_ADDRESS__SHIFT 0x00000000
+#define XDMA_MSTR_REMOTE_SURFACE_BASE_HIGH__XDMA_MSTR_REMOTE_SURFACE_BASE_HIGH_MASK 0x000000ffL
+#define XDMA_MSTR_REMOTE_SURFACE_BASE_HIGH__XDMA_MSTR_REMOTE_SURFACE_BASE_HIGH__SHIFT 0x00000000
+#define XDMA_MSTR_REMOTE_SURFACE_BASE__XDMA_MSTR_REMOTE_SURFACE_BASE_MASK 0xffffffffL
+#define XDMA_MSTR_REMOTE_SURFACE_BASE__XDMA_MSTR_REMOTE_SURFACE_BASE__SHIFT 0x00000000
+#define XDMA_MSTR_STATUS__XDMA_MSTR_VCOUNT_CURRENT_MASK 0x00003fffL
+#define XDMA_MSTR_STATUS__XDMA_MSTR_VCOUNT_CURRENT__SHIFT 0x00000000
+#define XDMA_MSTR_STATUS__XDMA_MSTR_WRITE_LINE_CURRENT_MASK 0x3fff0000L
+#define XDMA_MSTR_STATUS__XDMA_MSTR_WRITE_LINE_CURRENT__SHIFT 0x00000010
+#define XDMA_RBBMIF_RDWR_CNTL__XDMA_RBBMIF_RDWR_DELAY_MASK 0x00000007L
+#define XDMA_RBBMIF_RDWR_CNTL__XDMA_RBBMIF_RDWR_DELAY__SHIFT 0x00000000
+#define XDMA_RBBMIF_RDWR_CNTL__XDMA_RBBMIF_RDWR_TIMEOUT_DIS_MASK 0x00000008L
+#define XDMA_RBBMIF_RDWR_CNTL__XDMA_RBBMIF_RDWR_TIMEOUT_DIS__SHIFT 0x00000003
+#define XDMA_RBBMIF_RDWR_CNTL__XDMA_RBBMIF_TIMEOUT_DELAY_MASK 0xffff8000L
+#define XDMA_RBBMIF_RDWR_CNTL__XDMA_RBBMIF_TIMEOUT_DELAY__SHIFT 0x0000000f
+#define XDMA_SLV_CNTL__XDMA_SLV_ACTIVE_MASK 0x00000100L
+#define XDMA_SLV_CNTL__XDMA_SLV_ACTIVE__SHIFT 0x00000008
+#define XDMA_SLV_CNTL__XDMA_SLV_ENABLE_MASK 0x00010000L
+#define XDMA_SLV_CNTL__XDMA_SLV_ENABLE__SHIFT 0x00000010
+#define XDMA_SLV_CNTL__XDMA_SLV_MEM_READY_MASK 0x00000200L
+#define XDMA_SLV_CNTL__XDMA_SLV_MEM_READY__SHIFT 0x00000009
+#define XDMA_SLV_CNTL__XDMA_SLV_READ_LAT_TEST_EN_MASK 0x00080000L
+#define XDMA_SLV_CNTL__XDMA_SLV_READ_LAT_TEST_EN__SHIFT 0x00000013
+#define XDMA_SLV_CNTL__XDMA_SLV_SOFT_RESET_MASK 0x00100000L
+#define XDMA_SLV_CNTL__XDMA_SLV_SOFT_RESET__SHIFT 0x00000014
+#define XDMA_SLV_FLIP_PENDING__XDMA_SLV_FLIP_PENDING_MASK 0x00000001L
+#define XDMA_SLV_FLIP_PENDING__XDMA_SLV_FLIP_PENDING__SHIFT 0x00000000
+#define XDMA_SLV_MEM_CLIENT_CONFIG__XDMA_SLV_MEM_CLIENT_PRIV_MASK 0x00010000L
+#define XDMA_SLV_MEM_CLIENT_CONFIG__XDMA_SLV_MEM_CLIENT_PRIV__SHIFT 0x00000010
+#define XDMA_SLV_MEM_CLIENT_CONFIG__XDMA_SLV_MEM_CLIENT_SWAP_MASK 0x00000300L
+#define XDMA_SLV_MEM_CLIENT_CONFIG__XDMA_SLV_MEM_CLIENT_SWAP__SHIFT 0x00000008
+#define XDMA_SLV_MEM_CLIENT_CONFIG__XDMA_SLV_MEM_CLIENT_VMID_MASK 0x0000f000L
+#define XDMA_SLV_MEM_CLIENT_CONFIG__XDMA_SLV_MEM_CLIENT_VMID__SHIFT 0x0000000c
+#define XDMA_SLV_MEM_NACK_STATUS__XDMA_SLV_MEM_NACK_CLR_MASK 0x80000000L
+#define XDMA_SLV_MEM_NACK_STATUS__XDMA_SLV_MEM_NACK_CLR__SHIFT 0x0000001f
+#define XDMA_SLV_MEM_NACK_STATUS__XDMA_SLV_MEM_NACK_MASK 0x00030000L
+#define XDMA_SLV_MEM_NACK_STATUS__XDMA_SLV_MEM_NACK__SHIFT 0x00000010
+#define XDMA_SLV_MEM_NACK_STATUS__XDMA_SLV_MEM_NACK_TAG_MASK 0x0000ffffL
+#define XDMA_SLV_MEM_NACK_STATUS__XDMA_SLV_MEM_NACK_TAG__SHIFT 0x00000000
+#define XDMA_SLV_PCIE_NACK_STATUS__XDMA_SLV_PCIE_NACK_CLR_MASK 0x00010000L
+#define XDMA_SLV_PCIE_NACK_STATUS__XDMA_SLV_PCIE_NACK_CLR__SHIFT 0x00000010
+#define XDMA_SLV_PCIE_NACK_STATUS__XDMA_SLV_PCIE_NACK_MASK 0x00003000L
+#define XDMA_SLV_PCIE_NACK_STATUS__XDMA_SLV_PCIE_NACK__SHIFT 0x0000000c
+#define XDMA_SLV_PCIE_NACK_STATUS__XDMA_SLV_PCIE_NACK_TAG_MASK 0x000003ffL
+#define XDMA_SLV_PCIE_NACK_STATUS__XDMA_SLV_PCIE_NACK_TAG__SHIFT 0x00000000
+#define XDMA_SLV_READ_LATENCY_AVE__XDMA_SLV_READ_LATENCY_ACC_MASK 0x000fffffL
+#define XDMA_SLV_READ_LATENCY_AVE__XDMA_SLV_READ_LATENCY_ACC__SHIFT 0x00000000
+#define XDMA_SLV_READ_LATENCY_AVE__XDMA_SLV_READ_LATENCY_COUNT_MASK 0xfff00000L
+#define XDMA_SLV_READ_LATENCY_AVE__XDMA_SLV_READ_LATENCY_COUNT__SHIFT 0x00000014
+#define XDMA_SLV_READ_LATENCY_MINMAX__XDMA_SLV_READ_LATENCY_MAX_MASK 0xffff0000L
+#define XDMA_SLV_READ_LATENCY_MINMAX__XDMA_SLV_READ_LATENCY_MAX__SHIFT 0x00000010
+#define XDMA_SLV_READ_LATENCY_MINMAX__XDMA_SLV_READ_LATENCY_MIN_MASK 0x0000ffffL
+#define XDMA_SLV_READ_LATENCY_MINMAX__XDMA_SLV_READ_LATENCY_MIN__SHIFT 0x00000000
+#define XDMA_SLV_READ_LATENCY_TIMER__XDMA_SLV_READ_LATENCY_TIMER_MASK 0x0000ffffL
+#define XDMA_SLV_READ_LATENCY_TIMER__XDMA_SLV_READ_LATENCY_TIMER__SHIFT 0x00000000
+#define XDMA_SLV_READ_URGENT_CNTL__XDMA_SLV_READ_CLIENT_STALL_MASK 0x00000001L
+#define XDMA_SLV_READ_URGENT_CNTL__XDMA_SLV_READ_CLIENT_STALL__SHIFT 0x00000000
+#define XDMA_SLV_READ_URGENT_CNTL__XDMA_SLV_READ_STALL_DELAY_MASK 0x0000f000L
+#define XDMA_SLV_READ_URGENT_CNTL__XDMA_SLV_READ_STALL_DELAY__SHIFT 0x0000000c
+#define XDMA_SLV_READ_URGENT_CNTL__XDMA_SLV_READ_URGENT_LEVEL_MASK 0x00000f00L
+#define XDMA_SLV_READ_URGENT_CNTL__XDMA_SLV_READ_URGENT_LEVEL__SHIFT 0x00000008
+#define XDMA_SLV_READ_URGENT_CNTL__XDMA_SLV_READ_URGENT_LIMIT_MASK 0x000000f0L
+#define XDMA_SLV_READ_URGENT_CNTL__XDMA_SLV_READ_URGENT_LIMIT__SHIFT 0x00000004
+#define XDMA_SLV_READ_URGENT_CNTL__XDMA_SLV_READ_URGENT_TIMER_MASK 0xffff0000L
+#define XDMA_SLV_READ_URGENT_CNTL__XDMA_SLV_READ_URGENT_TIMER__SHIFT 0x00000010
+#define XDMA_SLV_REMOTE_GPU_ADDRESS_HIGH__XDMA_SLV_REMOTE_GPU_ADDRESS_HIGH_MASK 0x000000ffL
+#define XDMA_SLV_REMOTE_GPU_ADDRESS_HIGH__XDMA_SLV_REMOTE_GPU_ADDRESS_HIGH__SHIFT 0x00000000
+#define XDMA_SLV_REMOTE_GPU_ADDRESS__XDMA_SLV_REMOTE_GPU_ADDRESS_MASK 0xffffffffL
+#define XDMA_SLV_REMOTE_GPU_ADDRESS__XDMA_SLV_REMOTE_GPU_ADDRESS__SHIFT 0x00000000
+#define XDMA_SLV_SLS_PITCH__XDMA_SLV_SLS_PITCH_MASK 0x00003fffL
+#define XDMA_SLV_SLS_PITCH__XDMA_SLV_SLS_PITCH__SHIFT 0x00000000
+#define XDMA_SLV_SLS_PITCH__XDMA_SLV_SLS_WIDTH_MASK 0x3fff0000L
+#define XDMA_SLV_SLS_PITCH__XDMA_SLV_SLS_WIDTH__SHIFT 0x00000010
+#define XDMA_SLV_WB_RATE_CNTL__XDMA_SLV_WB_BURST_PERIOD_MASK 0xffff0000L
+#define XDMA_SLV_WB_RATE_CNTL__XDMA_SLV_WB_BURST_PERIOD__SHIFT 0x00000010
+#define XDMA_SLV_WB_RATE_CNTL__XDMA_SLV_WB_BURST_SIZE_MASK 0x000001ffL
+#define XDMA_SLV_WB_RATE_CNTL__XDMA_SLV_WB_BURST_SIZE__SHIFT 0x00000000
+#define XDMA_SLV_WRITE_URGENT_CNTL__XDMA_SLV_WRITE_STALL_DELAY_MASK 0x0000f000L
+#define XDMA_SLV_WRITE_URGENT_CNTL__XDMA_SLV_WRITE_STALL_DELAY__SHIFT 0x0000000c
+#define XDMA_SLV_WRITE_URGENT_CNTL__XDMA_SLV_WRITE_STALL_MASK 0x00000001L
+#define XDMA_SLV_WRITE_URGENT_CNTL__XDMA_SLV_WRITE_STALL__SHIFT 0x00000000
+#define XDMA_SLV_WRITE_URGENT_CNTL__XDMA_SLV_WRITE_URGENT_LEVEL_MASK 0x00000f00L
+#define XDMA_SLV_WRITE_URGENT_CNTL__XDMA_SLV_WRITE_URGENT_LEVEL__SHIFT 0x00000008
+#define XDMA_TEST_DEBUG_DATA__XDMA_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define XDMA_TEST_DEBUG_DATA__XDMA_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define XDMA_TEST_DEBUG_INDEX__XDMA_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define XDMA_TEST_DEBUG_INDEX__XDMA_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define XDMA_TEST_DEBUG_INDEX__XDMA_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define XDMA_TEST_DEBUG_INDEX__XDMA_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_6_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_6_0_d.h
new file mode 100644
index 000000000000..c75aee25619e
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_6_0_d.h
@@ -0,0 +1,1784 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef GFX_6_0_D_H
+#define GFX_6_0_D_H
+
+#define ixCLIPPER_DEBUG_REG00 0x0000
+#define ixCLIPPER_DEBUG_REG01 0x0001
+#define ixCLIPPER_DEBUG_REG02 0x0002
+#define ixCLIPPER_DEBUG_REG03 0x0003
+#define ixCLIPPER_DEBUG_REG04 0x0004
+#define ixCLIPPER_DEBUG_REG05 0x0005
+#define ixCLIPPER_DEBUG_REG06 0x0006
+#define ixCLIPPER_DEBUG_REG07 0x0007
+#define ixCLIPPER_DEBUG_REG08 0x0008
+#define ixCLIPPER_DEBUG_REG09 0x0009
+#define ixCLIPPER_DEBUG_REG10 0x000A
+#define ixCLIPPER_DEBUG_REG11 0x000B
+#define ixCLIPPER_DEBUG_REG12 0x000C
+#define ixCLIPPER_DEBUG_REG13 0x000D
+#define ixCLIPPER_DEBUG_REG14 0x000E
+#define ixCLIPPER_DEBUG_REG15 0x000F
+#define ixCLIPPER_DEBUG_REG16 0x0010
+#define ixCLIPPER_DEBUG_REG17 0x0011
+#define ixCLIPPER_DEBUG_REG18 0x0012
+#define ixCLIPPER_DEBUG_REG19 0x0013
+#define ixGDS_DEBUG_REG0 0x0000
+#define ixGDS_DEBUG_REG1 0x0001
+#define ixGDS_DEBUG_REG2 0x0002
+#define ixGDS_DEBUG_REG3 0x0003
+#define ixGDS_DEBUG_REG4 0x0004
+#define ixGDS_DEBUG_REG5 0x0005
+#define ixGDS_DEBUG_REG6 0x0006
+#define ixIA_DEBUG_REG0 0x0000
+#define ixIA_DEBUG_REG1 0x0001
+#define ixIA_DEBUG_REG2 0x0002
+#define ixIA_DEBUG_REG3 0x0003
+#define ixIA_DEBUG_REG4 0x0004
+#define ixIA_DEBUG_REG5 0x0005
+#define ixIA_DEBUG_REG6 0x0006
+#define ixIA_DEBUG_REG7 0x0007
+#define ixIA_DEBUG_REG8 0x0008
+#define ixIA_DEBUG_REG9 0x0009
+#define ixPA_SC_DEBUG_REG0 0x0000
+#define ixPA_SC_DEBUG_REG1 0x0001
+#define ixSETUP_DEBUG_REG0 0x0018
+#define ixSETUP_DEBUG_REG1 0x0019
+#define ixSETUP_DEBUG_REG2 0x001A
+#define ixSETUP_DEBUG_REG3 0x001B
+#define ixSETUP_DEBUG_REG4 0x001C
+#define ixSETUP_DEBUG_REG5 0x001D
+#define ixSQ_DEBUG_CTRL_LOCAL 0x0009
+#define ixSQ_DEBUG_STS_LOCAL 0x0008
+#define ixSQ_INTERRUPT_WORD_AUTO 0x20C0
+#define ixSQ_INTERRUPT_WORD_CMN 0x20C0
+#define ixSQ_INTERRUPT_WORD_WAVE 0x20C0
+#define ixSQ_WAVE_EXEC_HI 0x027F
+#define ixSQ_WAVE_EXEC_LO 0x027E
+#define ixSQ_WAVE_GPR_ALLOC 0x0015
+#define ixSQ_WAVE_HW_ID 0x0014
+#define ixSQ_WAVE_IB_DBG0 0x001C
+#define ixSQ_WAVE_IB_STS 0x0017
+#define ixSQ_WAVE_INST_DW0 0x001A
+#define ixSQ_WAVE_INST_DW1 0x001B
+#define ixSQ_WAVE_LDS_ALLOC 0x0016
+#define ixSQ_WAVE_M0 0x027C
+#define ixSQ_WAVE_MODE 0x0011
+#define ixSQ_WAVE_PC_HI 0x0019
+#define ixSQ_WAVE_PC_LO 0x0018
+#define ixSQ_WAVE_STATUS 0x0012
+#define ixSQ_WAVE_TBA_HI 0x026D
+#define ixSQ_WAVE_TBA_LO 0x026C
+#define ixSQ_WAVE_TMA_HI 0x026F
+#define ixSQ_WAVE_TMA_LO 0x026E
+#define ixSQ_WAVE_TRAPSTS 0x0013
+#define ixSQ_WAVE_TTMP0 0x0270
+#define ixSQ_WAVE_TTMP10 0x027A
+#define ixSQ_WAVE_TTMP1 0x0271
+#define ixSQ_WAVE_TTMP11 0x027B
+#define ixSQ_WAVE_TTMP2 0x0272
+#define ixSQ_WAVE_TTMP3 0x0273
+#define ixSQ_WAVE_TTMP4 0x0274
+#define ixSQ_WAVE_TTMP5 0x0275
+#define ixSQ_WAVE_TTMP6 0x0276
+#define ixSQ_WAVE_TTMP7 0x0277
+#define ixSQ_WAVE_TTMP8 0x0278
+#define ixSQ_WAVE_TTMP9 0x0279
+#define ixSXIFCCG_DEBUG_REG0 0x0014
+#define ixSXIFCCG_DEBUG_REG1 0x0015
+#define ixSXIFCCG_DEBUG_REG2 0x0016
+#define ixSXIFCCG_DEBUG_REG3 0x0017
+#define ixVGT_DEBUG_REG0 0x0000
+#define ixVGT_DEBUG_REG10 0x000A
+#define ixVGT_DEBUG_REG1 0x0001
+#define ixVGT_DEBUG_REG11 0x000B
+#define ixVGT_DEBUG_REG12 0x000C
+#define ixVGT_DEBUG_REG13 0x000D
+#define ixVGT_DEBUG_REG14 0x000E
+#define ixVGT_DEBUG_REG15 0x000F
+#define ixVGT_DEBUG_REG16 0x0010
+#define ixVGT_DEBUG_REG17 0x0011
+#define ixVGT_DEBUG_REG18 0x0012
+#define ixVGT_DEBUG_REG19 0x0013
+#define ixVGT_DEBUG_REG20 0x0014
+#define ixVGT_DEBUG_REG2 0x0002
+#define ixVGT_DEBUG_REG21 0x0015
+#define ixVGT_DEBUG_REG22 0x0016
+#define ixVGT_DEBUG_REG23 0x0017
+#define ixVGT_DEBUG_REG24 0x0018
+#define ixVGT_DEBUG_REG25 0x0019
+#define ixVGT_DEBUG_REG26 0x001A
+#define ixVGT_DEBUG_REG27 0x001B
+#define ixVGT_DEBUG_REG28 0x001C
+#define ixVGT_DEBUG_REG29 0x001D
+#define ixVGT_DEBUG_REG30 0x001E
+#define ixVGT_DEBUG_REG3 0x0003
+#define ixVGT_DEBUG_REG31 0x001F
+#define ixVGT_DEBUG_REG32 0x0020
+#define ixVGT_DEBUG_REG33 0x0021
+#define ixVGT_DEBUG_REG34 0x0022
+#define ixVGT_DEBUG_REG35 0x0023
+#define ixVGT_DEBUG_REG36 0x0024
+#define ixVGT_DEBUG_REG4 0x0004
+#define ixVGT_DEBUG_REG5 0x0005
+#define ixVGT_DEBUG_REG6 0x0006
+#define ixVGT_DEBUG_REG7 0x0007
+#define ixVGT_DEBUG_REG8 0x0008
+#define ixVGT_DEBUG_REG9 0x0009
+#define mmBCI_DEBUG_READ 0x24E3
+#define mmCB_BLEND0_CONTROL 0xA1E0
+#define mmCB_BLEND1_CONTROL 0xA1E1
+#define mmCB_BLEND2_CONTROL 0xA1E2
+#define mmCB_BLEND3_CONTROL 0xA1E3
+#define mmCB_BLEND4_CONTROL 0xA1E4
+#define mmCB_BLEND5_CONTROL 0xA1E5
+#define mmCB_BLEND6_CONTROL 0xA1E6
+#define mmCB_BLEND7_CONTROL 0xA1E7
+#define mmCB_BLEND_ALPHA 0xA108
+#define mmCB_BLEND_BLUE 0xA107
+#define mmCB_BLEND_GREEN 0xA106
+#define mmCB_BLEND_RED 0xA105
+#define mmCB_CGTT_SCLK_CTRL 0x2698
+#define mmCB_COLOR0_ATTRIB 0xA31D
+#define mmCB_COLOR0_BASE 0xA318
+#define mmCB_COLOR0_CLEAR_WORD0 0xA323
+#define mmCB_COLOR0_CLEAR_WORD1 0xA324
+#define mmCB_COLOR0_CMASK 0xA31F
+#define mmCB_COLOR0_CMASK_SLICE 0xA320
+#define mmCB_COLOR0_FMASK 0xA321
+#define mmCB_COLOR0_FMASK_SLICE 0xA322
+#define mmCB_COLOR0_INFO 0xA31C
+#define mmCB_COLOR0_PITCH 0xA319
+#define mmCB_COLOR0_SLICE 0xA31A
+#define mmCB_COLOR0_VIEW 0xA31B
+#define mmCB_COLOR1_ATTRIB 0xA32C
+#define mmCB_COLOR1_BASE 0xA327
+#define mmCB_COLOR1_CLEAR_WORD0 0xA332
+#define mmCB_COLOR1_CLEAR_WORD1 0xA333
+#define mmCB_COLOR1_CMASK 0xA32E
+#define mmCB_COLOR1_CMASK_SLICE 0xA32F
+#define mmCB_COLOR1_FMASK 0xA330
+#define mmCB_COLOR1_FMASK_SLICE 0xA331
+#define mmCB_COLOR1_INFO 0xA32B
+#define mmCB_COLOR1_PITCH 0xA328
+#define mmCB_COLOR1_SLICE 0xA329
+#define mmCB_COLOR1_VIEW 0xA32A
+#define mmCB_COLOR2_ATTRIB 0xA33B
+#define mmCB_COLOR2_BASE 0xA336
+#define mmCB_COLOR2_CLEAR_WORD0 0xA341
+#define mmCB_COLOR2_CLEAR_WORD1 0xA342
+#define mmCB_COLOR2_CMASK 0xA33D
+#define mmCB_COLOR2_CMASK_SLICE 0xA33E
+#define mmCB_COLOR2_FMASK 0xA33F
+#define mmCB_COLOR2_FMASK_SLICE 0xA340
+#define mmCB_COLOR2_INFO 0xA33A
+#define mmCB_COLOR2_PITCH 0xA337
+#define mmCB_COLOR2_SLICE 0xA338
+#define mmCB_COLOR2_VIEW 0xA339
+#define mmCB_COLOR3_ATTRIB 0xA34A
+#define mmCB_COLOR3_BASE 0xA345
+#define mmCB_COLOR3_CLEAR_WORD0 0xA350
+#define mmCB_COLOR3_CLEAR_WORD1 0xA351
+#define mmCB_COLOR3_CMASK 0xA34C
+#define mmCB_COLOR3_CMASK_SLICE 0xA34D
+#define mmCB_COLOR3_FMASK 0xA34E
+#define mmCB_COLOR3_FMASK_SLICE 0xA34F
+#define mmCB_COLOR3_INFO 0xA349
+#define mmCB_COLOR3_PITCH 0xA346
+#define mmCB_COLOR3_SLICE 0xA347
+#define mmCB_COLOR3_VIEW 0xA348
+#define mmCB_COLOR4_ATTRIB 0xA359
+#define mmCB_COLOR4_BASE 0xA354
+#define mmCB_COLOR4_CLEAR_WORD0 0xA35F
+#define mmCB_COLOR4_CLEAR_WORD1 0xA360
+#define mmCB_COLOR4_CMASK 0xA35B
+#define mmCB_COLOR4_CMASK_SLICE 0xA35C
+#define mmCB_COLOR4_FMASK 0xA35D
+#define mmCB_COLOR4_FMASK_SLICE 0xA35E
+#define mmCB_COLOR4_INFO 0xA358
+#define mmCB_COLOR4_PITCH 0xA355
+#define mmCB_COLOR4_SLICE 0xA356
+#define mmCB_COLOR4_VIEW 0xA357
+#define mmCB_COLOR5_ATTRIB 0xA368
+#define mmCB_COLOR5_BASE 0xA363
+#define mmCB_COLOR5_CLEAR_WORD0 0xA36E
+#define mmCB_COLOR5_CLEAR_WORD1 0xA36F
+#define mmCB_COLOR5_CMASK 0xA36A
+#define mmCB_COLOR5_CMASK_SLICE 0xA36B
+#define mmCB_COLOR5_FMASK 0xA36C
+#define mmCB_COLOR5_FMASK_SLICE 0xA36D
+#define mmCB_COLOR5_INFO 0xA367
+#define mmCB_COLOR5_PITCH 0xA364
+#define mmCB_COLOR5_SLICE 0xA365
+#define mmCB_COLOR5_VIEW 0xA366
+#define mmCB_COLOR6_ATTRIB 0xA377
+#define mmCB_COLOR6_BASE 0xA372
+#define mmCB_COLOR6_CLEAR_WORD0 0xA37D
+#define mmCB_COLOR6_CLEAR_WORD1 0xA37E
+#define mmCB_COLOR6_CMASK 0xA379
+#define mmCB_COLOR6_CMASK_SLICE 0xA37A
+#define mmCB_COLOR6_FMASK 0xA37B
+#define mmCB_COLOR6_FMASK_SLICE 0xA37C
+#define mmCB_COLOR6_INFO 0xA376
+#define mmCB_COLOR6_PITCH 0xA373
+#define mmCB_COLOR6_SLICE 0xA374
+#define mmCB_COLOR6_VIEW 0xA375
+#define mmCB_COLOR7_ATTRIB 0xA386
+#define mmCB_COLOR7_BASE 0xA381
+#define mmCB_COLOR7_CLEAR_WORD0 0xA38C
+#define mmCB_COLOR7_CLEAR_WORD1 0xA38D
+#define mmCB_COLOR7_CMASK 0xA388
+#define mmCB_COLOR7_CMASK_SLICE 0xA389
+#define mmCB_COLOR7_FMASK 0xA38A
+#define mmCB_COLOR7_FMASK_SLICE 0xA38B
+#define mmCB_COLOR7_INFO 0xA385
+#define mmCB_COLOR7_PITCH 0xA382
+#define mmCB_COLOR7_SLICE 0xA383
+#define mmCB_COLOR7_VIEW 0xA384
+#define mmCB_COLOR_CONTROL 0xA202
+#define mmCB_DEBUG_BUS_10 0x26A2
+#define mmCB_DEBUG_BUS_1 0x2699
+#define mmCB_DEBUG_BUS_11 0x26A3
+#define mmCB_DEBUG_BUS_12 0x26A4
+#define mmCB_DEBUG_BUS_13 0x26A5
+#define mmCB_DEBUG_BUS_14 0x26A6
+#define mmCB_DEBUG_BUS_15 0x26A7
+#define mmCB_DEBUG_BUS_16 0x26A8
+#define mmCB_DEBUG_BUS_17 0x26A9
+#define mmCB_DEBUG_BUS_18 0x26AA
+#define mmCB_DEBUG_BUS_2 0x269A
+#define mmCB_DEBUG_BUS_3 0x269B
+#define mmCB_DEBUG_BUS_4 0x269C
+#define mmCB_DEBUG_BUS_5 0x269D
+#define mmCB_DEBUG_BUS_6 0x269E
+#define mmCB_DEBUG_BUS_7 0x269F
+#define mmCB_DEBUG_BUS_8 0x26A0
+#define mmCB_DEBUG_BUS_9 0x26A1
+#define mmCB_HW_CONTROL 0x2684
+#define mmCB_HW_CONTROL_1 0x2685
+#define mmCB_HW_CONTROL_2 0x2686
+#define mmCB_PERFCOUNTER0_HI 0x2691
+#define mmCB_PERFCOUNTER0_LO 0x2690
+#define mmCB_PERFCOUNTER0_SELECT1 0x2689
+#define mmCB_PERFCOUNTER1_HI 0x2693
+#define mmCB_PERFCOUNTER1_LO 0x2692
+#define mmCB_PERFCOUNTER2_HI 0x2695
+#define mmCB_PERFCOUNTER2_LO 0x2694
+#define mmCB_PERFCOUNTER3_HI 0x2697
+#define mmCB_PERFCOUNTER3_LO 0x2696
+#define mmCB_SHADER_MASK 0xA08F
+#define mmCB_TARGET_MASK 0xA08E
+#define mmCC_GC_SHADER_ARRAY_CONFIG 0x226F
+#define mmCC_RB_BACKEND_DISABLE 0x263D
+#define mmCC_RB_DAISY_CHAIN 0x2641
+#define mmCC_RB_REDUNDANCY 0x263C
+#define mmCC_SQC_BANK_DISABLE 0x2307
+#define mmCGTS_RD_CTRL_REG 0x2455
+#define mmCGTS_RD_REG 0x2456
+#define mmCGTS_SM_CTRL_REG 0x2454
+#define mmCGTS_TCC_DISABLE 0x2452
+#define mmCGTS_USER_TCC_DISABLE 0x2453
+#define mmCGTT_BCI_CLK_CTRL 0x24A9
+#define mmCGTT_CP_CLK_CTRL 0x3059
+#define mmCGTT_GDS_CLK_CTRL 0x25DD
+#define mmCGTT_IA_CLK_CTRL 0x2261
+#define mmCGTT_PA_CLK_CTRL 0x2286
+#define mmCGTT_PC_CLK_CTRL 0x24A8
+#define mmCGTT_RLC_CLK_CTRL 0x30E0
+#define mmCGTT_SC_CLK_CTRL 0x22CA
+#define mmCGTT_SPI_CLK_CTRL 0x2451
+#define mmCGTT_SQ_CLK_CTRL 0x2362
+#define mmCGTT_SQG_CLK_CTRL 0x2363
+#define mmCGTT_SX_CLK_CTRL0 0x240C
+#define mmCGTT_SX_CLK_CTRL1 0x240D
+#define mmCGTT_SX_CLK_CTRL2 0x240E
+#define mmCGTT_SX_CLK_CTRL3 0x240F
+#define mmCGTT_SX_CLK_CTRL4 0x2410
+#define mmCGTT_TCI_CLK_CTRL 0x2B60
+#define mmCGTT_TCP_CLK_CTRL 0x2B15
+#define mmCGTT_VGT_CLK_CTRL 0x225F
+#define mmCOHER_DEST_BASE_0 0xA092
+#define mmCOHER_DEST_BASE_1 0xA093
+#define mmCOHER_DEST_BASE_2 0xA07E
+#define mmCOHER_DEST_BASE_3 0xA07F
+#define mmCOMPUTE_DIM_X 0x2E01
+#define mmCOMPUTE_DIM_Y 0x2E02
+#define mmCOMPUTE_DIM_Z 0x2E03
+#define mmCOMPUTE_DISPATCH_INITIATOR 0x2E00
+#define mmCOMPUTE_NUM_THREAD_X 0x2E07
+#define mmCOMPUTE_NUM_THREAD_Y 0x2E08
+#define mmCOMPUTE_NUM_THREAD_Z 0x2E09
+#define mmCOMPUTE_PGM_HI 0x2E0D
+#define mmCOMPUTE_PGM_LO 0x2E0C
+#define mmCOMPUTE_PGM_RSRC1 0x2E12
+#define mmCOMPUTE_PGM_RSRC2 0x2E13
+#define mmCOMPUTE_RESOURCE_LIMITS 0x2E15
+#define mmCOMPUTE_START_X 0x2E04
+#define mmCOMPUTE_START_Y 0x2E05
+#define mmCOMPUTE_START_Z 0x2E06
+#define mmCOMPUTE_STATIC_THREAD_MGMT_SE0 0x2E16
+#define mmCOMPUTE_STATIC_THREAD_MGMT_SE1 0x2E17
+#define mmCOMPUTE_TBA_HI 0x2E0F
+#define mmCOMPUTE_TBA_LO 0x2E0E
+#define mmCOMPUTE_TMA_HI 0x2E11
+#define mmCOMPUTE_TMA_LO 0x2E10
+#define mmCOMPUTE_TMPRING_SIZE 0x2E18
+#define mmCOMPUTE_USER_DATA_0 0x2E40
+#define mmCOMPUTE_USER_DATA_10 0x2E4A
+#define mmCOMPUTE_USER_DATA_1 0x2E41
+#define mmCOMPUTE_USER_DATA_11 0x2E4B
+#define mmCOMPUTE_USER_DATA_12 0x2E4C
+#define mmCOMPUTE_USER_DATA_13 0x2E4D
+#define mmCOMPUTE_USER_DATA_14 0x2E4E
+#define mmCOMPUTE_USER_DATA_15 0x2E4F
+#define mmCOMPUTE_USER_DATA_2 0x2E42
+#define mmCOMPUTE_USER_DATA_3 0x2E43
+#define mmCOMPUTE_USER_DATA_4 0x2E44
+#define mmCOMPUTE_USER_DATA_5 0x2E45
+#define mmCOMPUTE_USER_DATA_6 0x2E46
+#define mmCOMPUTE_USER_DATA_7 0x2E47
+#define mmCOMPUTE_USER_DATA_8 0x2E48
+#define mmCOMPUTE_USER_DATA_9 0x2E49
+#define mmCOMPUTE_VMID 0x2E14
+#define mmCP_APPEND_ADDR_HI 0x2159
+#define mmCP_APPEND_ADDR_LO 0x2158
+#define mmCP_APPEND_DATA 0x215A
+#define mmCP_APPEND_LAST_CS_FENCE 0x215B
+#define mmCP_APPEND_LAST_PS_FENCE 0x215C
+#define mmCP_ATOMIC_PREOP_HI 0x215E
+#define mmCP_ATOMIC_PREOP_LO 0x215D
+#define mmCP_BUSY_STAT 0x219F
+#define mmCP_CE_HEADER_DUMP 0x21A4
+#define mmCP_CE_IB1_BASE_HI 0x21C7
+#define mmCP_CE_IB1_BASE_LO 0x21C6
+#define mmCP_CE_IB1_BUFSZ 0x21C8
+#define mmCP_CE_IB2_BASE_HI 0x21CA
+#define mmCP_CE_IB2_BASE_LO 0x21C9
+#define mmCP_CE_IB2_BUFSZ 0x21CB
+#define mmCP_CE_INIT_BASE_HI 0x21C4
+#define mmCP_CE_INIT_BASE_LO 0x21C3
+#define mmCP_CE_INIT_BUFSZ 0x21C5
+#define mmCP_CEQ1_AVAIL 0x21E6
+#define mmCP_CEQ2_AVAIL 0x21E7
+#define mmCP_CE_ROQ_IB1_STAT 0x21E9
+#define mmCP_CE_ROQ_IB2_STAT 0x21EA
+#define mmCP_CE_ROQ_RB_STAT 0x21E8
+#define mmCP_CE_UCODE_ADDR 0x305A
+#define mmCP_CE_UCODE_DATA 0x305B
+#define mmCP_CMD_DATA 0x21DF
+#define mmCP_CMD_INDEX 0x21DE
+#define mmCP_CNTX_STAT 0x21B8
+#define mmCP_COHER_BASE 0x217E
+#define mmCP_COHER_CNTL 0x217C
+#define mmCP_COHER_SIZE 0x217D
+#define mmCP_COHER_START_DELAY 0x217B
+#define mmCP_COHER_STATUS 0x217F
+#define mmCP_CSF_CNTL 0x21B5
+#define mmCP_CSF_STAT 0x21B4
+#define mmCP_DMA_CNTL 0x218A
+#define mmCP_DMA_ME_COMMAND 0x2184
+#define mmCP_DMA_ME_DST_ADDR 0x2182
+#define mmCP_DMA_ME_DST_ADDR_HI 0x2183
+#define mmCP_DMA_ME_SRC_ADDR 0x2180
+#define mmCP_DMA_ME_SRC_ADDR_HI 0x2181
+#define mmCP_DMA_PFP_COMMAND 0x2189
+#define mmCP_DMA_PFP_DST_ADDR 0x2187
+#define mmCP_DMA_PFP_DST_ADDR_HI 0x2188
+#define mmCP_DMA_PFP_SRC_ADDR 0x2185
+#define mmCP_DMA_PFP_SRC_ADDR_HI 0x2186
+#define mmCP_DMA_READ_TAGS 0x218B
+#define mmCP_ECC_FIRSTOCCURRENCE 0x307A
+#define mmCP_ECC_FIRSTOCCURRENCE_RING0 0x307B
+#define mmCP_ECC_FIRSTOCCURRENCE_RING1 0x307C
+#define mmCP_ECC_FIRSTOCCURRENCE_RING2 0x307D
+#define mmCP_EOP_DONE_ADDR_HI 0x2101
+#define mmCP_EOP_DONE_ADDR_LO 0x2100
+#define mmCP_EOP_DONE_DATA_HI 0x2103
+#define mmCP_EOP_DONE_DATA_LO 0x2102
+#define mmCP_EOP_LAST_FENCE_HI 0x2105
+#define mmCP_EOP_LAST_FENCE_LO 0x2104
+#define mmCP_GDS_ATOMIC0_PREOP_HI 0x2160
+#define mmCP_GDS_ATOMIC0_PREOP_LO 0x215F
+#define mmCP_GDS_ATOMIC1_PREOP_HI 0x2162
+#define mmCP_GDS_ATOMIC1_PREOP_LO 0x2161
+#define mmCP_GRBM_FREE_COUNT 0x21A3
+#define mmCP_IB1_BASE_HI 0x21CD
+#define mmCP_IB1_BASE_LO 0x21CC
+#define mmCP_IB1_BUFSZ 0x21CE
+#define mmCP_IB1_OFFSET 0x2192
+#define mmCP_IB1_PREAMBLE_BEGIN 0x2194
+#define mmCP_IB1_PREAMBLE_END 0x2195
+#define mmCP_IB2_BASE_HI 0x21D0
+#define mmCP_IB2_BASE_LO 0x21CF
+#define mmCP_IB2_BUFSZ 0x21D1
+#define mmCP_IB2_OFFSET 0x2193
+#define mmCP_IB2_PREAMBLE_BEGIN 0x2196
+#define mmCP_IB2_PREAMBLE_END 0x2197
+#define mmCP_INT_CNTL 0x3049
+#define mmCP_INT_CNTL_RING0 0x306A
+#define mmCP_INT_CNTL_RING1 0x306B
+#define mmCP_INT_CNTL_RING2 0x306C
+#define mmCP_INT_STAT_DEBUG 0x21F7
+#define mmCP_INT_STATUS 0x304A
+#define mmCP_INT_STATUS_RING0 0x306D
+#define mmCP_INT_STATUS_RING1 0x306E
+#define mmCP_INT_STATUS_RING2 0x306F
+#define mmCP_MC_PACK_DELAY_CNT 0x21A7
+#define mmCP_ME_CNTL 0x21B6
+#define mmCP_ME_HEADER_DUMP 0x21A1
+#define mmCP_ME_MC_RADDR_HI 0x216E
+#define mmCP_ME_MC_RADDR_LO 0x216D
+#define mmCP_ME_MC_WADDR_HI 0x216A
+#define mmCP_ME_MC_WADDR_LO 0x2169
+#define mmCP_ME_MC_WDATA_HI 0x216C
+#define mmCP_ME_MC_WDATA_LO 0x216B
+#define mmCP_MEM_SLP_CNTL 0x3079
+#define mmCP_ME_PREEMPTION 0x21B9
+#define mmCP_MEQ_AVAIL 0x21DD
+#define mmCP_MEQ_STAT 0x21E5
+#define mmCP_MEQ_THRESHOLDS 0x21D9
+#define mmCP_ME_RAM_DATA 0x3058
+#define mmCP_ME_RAM_RADDR 0x3056
+#define mmCP_ME_RAM_WADDR 0x3057
+#define mmCP_NUM_PRIM_NEEDED_COUNT0_HI 0x210B
+#define mmCP_NUM_PRIM_NEEDED_COUNT0_LO 0x210A
+#define mmCP_NUM_PRIM_NEEDED_COUNT1_HI 0x210F
+#define mmCP_NUM_PRIM_NEEDED_COUNT1_LO 0x210E
+#define mmCP_NUM_PRIM_NEEDED_COUNT2_HI 0x2113
+#define mmCP_NUM_PRIM_NEEDED_COUNT2_LO 0x2112
+#define mmCP_NUM_PRIM_NEEDED_COUNT3_HI 0x2117
+#define mmCP_NUM_PRIM_NEEDED_COUNT3_LO 0x2116
+#define mmCP_NUM_PRIM_WRITTEN_COUNT0_HI 0x2109
+#define mmCP_NUM_PRIM_WRITTEN_COUNT0_LO 0x2108
+#define mmCP_NUM_PRIM_WRITTEN_COUNT1_HI 0x210D
+#define mmCP_NUM_PRIM_WRITTEN_COUNT1_LO 0x210C
+#define mmCP_NUM_PRIM_WRITTEN_COUNT2_HI 0x2111
+#define mmCP_NUM_PRIM_WRITTEN_COUNT2_LO 0x2110
+#define mmCP_NUM_PRIM_WRITTEN_COUNT3_HI 0x2115
+#define mmCP_NUM_PRIM_WRITTEN_COUNT3_LO 0x2114
+#define mmCP_PA_CINVOC_COUNT_HI 0x2129
+#define mmCP_PA_CINVOC_COUNT_LO 0x2128
+#define mmCP_PA_CPRIM_COUNT_HI 0x212B
+#define mmCP_PA_CPRIM_COUNT_LO 0x212A
+#define mmCP_PERFMON_CNTL 0x21FF
+#define mmCP_PERFMON_CNTX_CNTL 0xA0D8
+#define mmCP_PFP_HEADER_DUMP 0x21A2
+#define mmCP_PFP_IB_CONTROL 0x218D
+#define mmCP_PFP_LOAD_CONTROL 0x218E
+#define mmCP_PFP_UCODE_ADDR 0x3054
+#define mmCP_PFP_UCODE_DATA 0x3055
+#define mmCP_PIPE_STATS_ADDR_HI 0x2119
+#define mmCP_PIPE_STATS_ADDR_LO 0x2118
+#define mmCP_PWR_CNTL 0x3078
+#define mmCP_QUEUE_THRESHOLDS 0x21D8
+#define mmCP_RB0_BASE 0x3040
+#define mmCP_RB0_CNTL 0x3041
+#define mmCP_RB0_RPTR 0x21C0
+#define mmCP_RB0_RPTR_ADDR 0x3043
+#define mmCP_RB0_RPTR_ADDR_HI 0x3044
+#define mmCP_RB0_WPTR 0x3045
+#define mmCP_RB1_BASE 0x3060
+#define mmCP_RB1_CNTL 0x3061
+#define mmCP_RB1_RPTR 0x21BF
+#define mmCP_RB1_RPTR_ADDR 0x3062
+#define mmCP_RB1_RPTR_ADDR_HI 0x3063
+#define mmCP_RB1_WPTR 0x3064
+#define mmCP_RB2_BASE 0x3065
+#define mmCP_RB2_CNTL 0x3066
+#define mmCP_RB2_RPTR 0x21BE
+#define mmCP_RB2_RPTR_ADDR 0x3067
+#define mmCP_RB2_RPTR_ADDR_HI 0x3068
+#define mmCP_RB2_WPTR 0x3069
+#define mmCP_RB_BASE 0x3040
+#define mmCP_RB_CNTL 0x3041
+#define mmCP_RB_OFFSET 0x2191
+#define mmCP_RB_RPTR 0x21C0
+#define mmCP_RB_RPTR_ADDR 0x3043
+#define mmCP_RB_RPTR_ADDR_HI 0x3044
+#define mmCP_RB_RPTR_WR 0x3042
+#define mmCP_RB_VMID 0x3051
+#define mmCP_RB_WPTR 0x3045
+#define mmCP_RB_WPTR_DELAY 0x21C1
+#define mmCP_RB_WPTR_POLL_ADDR_HI 0x3047
+#define mmCP_RB_WPTR_POLL_ADDR_LO 0x3046
+#define mmCP_RB_WPTR_POLL_CNTL 0x21C2
+#define mmCP_RING0_PRIORITY 0x304D
+#define mmCP_RING1_PRIORITY 0x304E
+#define mmCP_RING2_PRIORITY 0x304F
+#define mmCP_RINGID 0xA0D9
+#define mmCP_RING_PRIORITY_CNTS 0x304C
+#define mmCP_ROQ1_THRESHOLDS 0x21D5
+#define mmCP_ROQ2_AVAIL 0x21DC
+#define mmCP_ROQ2_THRESHOLDS 0x21D6
+#define mmCP_ROQ_AVAIL 0x21DA
+#define mmCP_ROQ_IB1_STAT 0x21E1
+#define mmCP_ROQ_IB2_STAT 0x21E2
+#define mmCP_ROQ_RB_STAT 0x21E0
+#define mmCP_SC_PSINVOC_COUNT0_HI 0x212D
+#define mmCP_SC_PSINVOC_COUNT0_LO 0x212C
+#define mmCP_SC_PSINVOC_COUNT1_HI 0x212F
+#define mmCP_SC_PSINVOC_COUNT1_LO 0x212E
+#define mmCP_SCRATCH_DATA 0x2190
+#define mmCP_SCRATCH_INDEX 0x218F
+#define mmCP_SEM_INCOMPLETE_TIMER_CNTL 0x2172
+#define mmCP_SEM_WAIT_TIMER 0x216F
+#define mmCP_SIG_SEM_ADDR_HI 0x2171
+#define mmCP_SIG_SEM_ADDR_LO 0x2170
+#define mmCP_STALLED_STAT1 0x219D
+#define mmCP_STALLED_STAT2 0x219E
+#define mmCP_STALLED_STAT3 0x219C
+#define mmCP_STAT 0x21A0
+#define mmCP_ST_BASE_HI 0x21D3
+#define mmCP_ST_BASE_LO 0x21D2
+#define mmCP_ST_BUFSZ 0x21D4
+#define mmCP_STQ_AVAIL 0x21DB
+#define mmCP_STQ_STAT 0x21E3
+#define mmCP_STQ_THRESHOLDS 0x21D7
+#define mmCP_STREAM_OUT_ADDR_HI 0x2107
+#define mmCP_STREAM_OUT_ADDR_LO 0x2106
+#define mmCP_STRMOUT_CNTL 0x213F
+#define mmCP_VGT_CSINVOC_COUNT_HI 0x2131
+#define mmCP_VGT_CSINVOC_COUNT_LO 0x2130
+#define mmCP_VGT_DSINVOC_COUNT_HI 0x2127
+#define mmCP_VGT_DSINVOC_COUNT_LO 0x2126
+#define mmCP_VGT_GSINVOC_COUNT_HI 0x2123
+#define mmCP_VGT_GSINVOC_COUNT_LO 0x2122
+#define mmCP_VGT_GSPRIM_COUNT_HI 0x211F
+#define mmCP_VGT_GSPRIM_COUNT_LO 0x211E
+#define mmCP_VGT_HSINVOC_COUNT_HI 0x2125
+#define mmCP_VGT_HSINVOC_COUNT_LO 0x2124
+#define mmCP_VGT_IAPRIM_COUNT_HI 0x211D
+#define mmCP_VGT_IAPRIM_COUNT_LO 0x211C
+#define mmCP_VGT_IAVERT_COUNT_HI 0x211B
+#define mmCP_VGT_IAVERT_COUNT_LO 0x211A
+#define mmCP_VGT_VSINVOC_COUNT_HI 0x2121
+#define mmCP_VGT_VSINVOC_COUNT_LO 0x2120
+#define mmCP_VMID 0xA0DA
+#define mmCP_WAIT_REG_MEM_TIMEOUT 0x2174
+#define mmCP_WAIT_SEM_ADDR_HI 0x2176
+#define mmCP_WAIT_SEM_ADDR_LO 0x2175
+#define mmCS_COPY_STATE 0xA1F3
+#define mmDB_ALPHA_TO_MASK 0xA2DC
+#define mmDB_CGTT_CLK_CTRL_0 0x261A
+#define mmDB_COUNT_CONTROL 0xA001
+#define mmDB_CREDIT_LIMIT 0x2614
+#define mmDB_DEBUG 0x260C
+#define mmDB_DEBUG2 0x260D
+#define mmDB_DEBUG3 0x260E
+#define mmDB_DEBUG4 0x260F
+#define mmDB_DEPTH_BOUNDS_MAX 0xA009
+#define mmDB_DEPTH_BOUNDS_MIN 0xA008
+#define mmDB_DEPTH_CLEAR 0xA00B
+#define mmDB_DEPTH_CONTROL 0xA200
+#define mmDB_DEPTH_INFO 0xA00F
+#define mmDB_DEPTH_SIZE 0xA016
+#define mmDB_DEPTH_SLICE 0xA017
+#define mmDB_DEPTH_VIEW 0xA002
+#define mmDB_EQAA 0xA201
+#define mmDB_FIFO_DEPTH1 0x2618
+#define mmDB_FIFO_DEPTH2 0x2619
+#define mmDB_FREE_CACHELINES 0x2617
+#define mmDB_HTILE_DATA_BASE 0xA005
+#define mmDB_HTILE_SURFACE 0xA2AF
+#define mmDB_PERFCOUNTER0_HI 0x2602
+#define mmDB_PERFCOUNTER0_LO 0x2601
+#define mmDB_PERFCOUNTER0_SELECT 0x2600
+#define mmDB_PERFCOUNTER1_HI 0x2605
+#define mmDB_PERFCOUNTER1_LO 0x2604
+#define mmDB_PERFCOUNTER1_SELECT 0x2603
+#define mmDB_PERFCOUNTER2_HI 0x2608
+#define mmDB_PERFCOUNTER2_LO 0x2607
+#define mmDB_PERFCOUNTER2_SELECT 0x2606
+#define mmDB_PERFCOUNTER3_HI 0x260B
+#define mmDB_PERFCOUNTER3_LO 0x260A
+#define mmDB_PERFCOUNTER3_SELECT 0x2609
+#define mmDB_PRELOAD_CONTROL 0xA2B2
+#define mmDB_READ_DEBUG_0 0x2620
+#define mmDB_READ_DEBUG_1 0x2621
+#define mmDB_READ_DEBUG_2 0x2622
+#define mmDB_READ_DEBUG_3 0x2623
+#define mmDB_READ_DEBUG_4 0x2624
+#define mmDB_READ_DEBUG_5 0x2625
+#define mmDB_READ_DEBUG_6 0x2626
+#define mmDB_READ_DEBUG_7 0x2627
+#define mmDB_READ_DEBUG_8 0x2628
+#define mmDB_READ_DEBUG_9 0x2629
+#define mmDB_READ_DEBUG_A 0x262A
+#define mmDB_READ_DEBUG_B 0x262B
+#define mmDB_READ_DEBUG_C 0x262C
+#define mmDB_READ_DEBUG_D 0x262D
+#define mmDB_READ_DEBUG_E 0x262E
+#define mmDB_READ_DEBUG_F 0x262F
+#define mmDB_RENDER_CONTROL 0xA000
+#define mmDB_RENDER_OVERRIDE 0xA003
+#define mmDB_RENDER_OVERRIDE2 0xA004
+#define mmDB_SHADER_CONTROL 0xA203
+#define mmDB_SRESULTS_COMPARE_STATE0 0xA2B0
+#define mmDB_SRESULTS_COMPARE_STATE1 0xA2B1
+#define mmDB_STENCIL_CLEAR 0xA00A
+#define mmDB_STENCIL_CONTROL 0xA10B
+#define mmDB_STENCIL_INFO 0xA011
+#define mmDB_STENCIL_READ_BASE 0xA013
+#define mmDB_STENCILREFMASK 0xA10C
+#define mmDB_STENCILREFMASK_BF 0xA10D
+#define mmDB_STENCIL_WRITE_BASE 0xA015
+#define mmDB_SUBTILE_CONTROL 0x2616
+#define mmDB_WATERMARKS 0x2615
+#define mmDB_Z_INFO 0xA010
+#define mmDB_ZPASS_COUNT_HI 0x261D
+#define mmDB_ZPASS_COUNT_LOW 0x261C
+#define mmDB_Z_READ_BASE 0xA012
+#define mmDB_Z_WRITE_BASE 0xA014
+#define mmDEBUG_DATA 0x203D
+#define mmDEBUG_INDEX 0x203C
+#define mmGB_ADDR_CONFIG 0x263E
+#define mmGB_BACKEND_MAP 0x263F
+#define mmGB_EDC_MODE 0x307E
+#define mmGB_GPU_ID 0x2640
+#define mmGB_TILE_MODE0 0x2644
+#define mmGB_TILE_MODE10 0x264E
+#define mmGB_TILE_MODE1 0x2645
+#define mmGB_TILE_MODE11 0x264F
+#define mmGB_TILE_MODE12 0x2650
+#define mmGB_TILE_MODE13 0x2651
+#define mmGB_TILE_MODE14 0x2652
+#define mmGB_TILE_MODE15 0x2653
+#define mmGB_TILE_MODE16 0x2654
+#define mmGB_TILE_MODE17 0x2655
+#define mmGB_TILE_MODE18 0x2656
+#define mmGB_TILE_MODE19 0x2657
+#define mmGB_TILE_MODE20 0x2658
+#define mmGB_TILE_MODE2 0x2646
+#define mmGB_TILE_MODE21 0x2659
+#define mmGB_TILE_MODE22 0x265A
+#define mmGB_TILE_MODE23 0x265B
+#define mmGB_TILE_MODE24 0x265C
+#define mmGB_TILE_MODE25 0x265D
+#define mmGB_TILE_MODE26 0x265E
+#define mmGB_TILE_MODE27 0x265F
+#define mmGB_TILE_MODE28 0x2660
+#define mmGB_TILE_MODE29 0x2661
+#define mmGB_TILE_MODE30 0x2662
+#define mmGB_TILE_MODE3 0x2647
+#define mmGB_TILE_MODE31 0x2663
+#define mmGB_TILE_MODE4 0x2648
+#define mmGB_TILE_MODE5 0x2649
+#define mmGB_TILE_MODE6 0x264A
+#define mmGB_TILE_MODE7 0x264B
+#define mmGB_TILE_MODE8 0x264C
+#define mmGB_TILE_MODE9 0x264D
+#define mmGC_PRIV_MODE 0x3048
+#define mmGC_USER_RB_BACKEND_DISABLE 0x26DF
+#define mmGC_USER_SHADER_ARRAY_CONFIG 0x2270
+#define mmGDS_ATOM_BASE 0x25CE
+#define mmGDS_ATOM_CNTL 0x25CC
+#define mmGDS_ATOM_COMPLETE 0x25CD
+#define mmGDS_ATOM_DST 0x25D2
+#define mmGDS_ATOM_OFFSET0 0x25D0
+#define mmGDS_ATOM_OFFSET1 0x25D1
+#define mmGDS_ATOM_OP 0x25D3
+#define mmGDS_ATOM_READ0 0x25D8
+#define mmGDS_ATOM_READ0_U 0x25D9
+#define mmGDS_ATOM_READ1 0x25DA
+#define mmGDS_ATOM_READ1_U 0x25DB
+#define mmGDS_ATOM_SIZE 0x25CF
+#define mmGDS_ATOM_SRC0 0x25D4
+#define mmGDS_ATOM_SRC0_U 0x25D5
+#define mmGDS_ATOM_SRC1 0x25D6
+#define mmGDS_ATOM_SRC1_U 0x25D7
+#define mmGDS_CNTL_STATUS 0x25C1
+#define mmGDS_CONFIG 0x25C0
+#define mmGDS_DEBUG_CNTL 0x25DE
+#define mmGDS_DEBUG_DATA 0x25DF
+#define mmGDS_ENHANCE 0x25DC
+#define mmGDS_GRBM_SECDED_CNT 0x25E3
+#define mmGDS_GWS_RESOURCE 0x25E1
+#define mmGDS_GWS_RESOURCE_CNTL 0x25E0
+#define mmGDS_OA_DED 0x25E4
+#define mmGDS_PERFCOUNTER0_HI 0x25E7
+#define mmGDS_PERFCOUNTER0_LO 0x25E6
+#define mmGDS_PERFCOUNTER0_SELECT 0x25E5
+#define mmGDS_PERFCOUNTER1_HI 0x25EA
+#define mmGDS_PERFCOUNTER1_LO 0x25E9
+#define mmGDS_PERFCOUNTER1_SELECT 0x25E8
+#define mmGDS_PERFCOUNTER2_HI 0x25ED
+#define mmGDS_PERFCOUNTER2_LO 0x25EC
+#define mmGDS_PERFCOUNTER2_SELECT 0x25EB
+#define mmGDS_PERFCOUNTER3_HI 0x25F0
+#define mmGDS_PERFCOUNTER3_LO 0x25EF
+#define mmGDS_PERFCOUNTER3_SELECT 0x25EE
+#define mmGDS_RD_ADDR 0x25C2
+#define mmGDS_RD_BURST_ADDR 0x25C4
+#define mmGDS_RD_BURST_COUNT 0x25C5
+#define mmGDS_RD_BURST_DATA 0x25C6
+#define mmGDS_RD_DATA 0x25C3
+#define mmGDS_SECDED_CNT 0x25E2
+#define mmGDS_WR_ADDR 0x25C7
+#define mmGDS_WR_BURST_ADDR 0x25C9
+#define mmGDS_WR_BURST_DATA 0x25CA
+#define mmGDS_WR_DATA 0x25C8
+#define mmGDS_WRITE_COMPLETE 0x25CB
+#define mmGFX_COPY_STATE 0xA1F4
+#define mmGRBM_CAM_DATA 0x3001
+#define mmGRBM_CAM_INDEX 0x3000
+#define mmGRBM_CNTL 0x2000
+#define mmGRBM_DEBUG 0x2014
+#define mmGRBM_DEBUG_CNTL 0x2009
+#define mmGRBM_DEBUG_DATA 0x200A
+#define mmGRBM_DEBUG_SNAPSHOT 0x2015
+#define mmGRBM_GFX_CLKEN_CNTL 0x200C
+#define mmGRBM_GFX_INDEX 0x200B
+#define mmGRBM_INT_CNTL 0x2018
+#define mmGRBM_NOWHERE 0x203F
+#define mmGRBM_PERFCOUNTER0_HI 0x201F
+#define mmGRBM_PERFCOUNTER0_LO 0x201E
+#define mmGRBM_PERFCOUNTER0_SELECT 0x201C
+#define mmGRBM_PERFCOUNTER1_HI 0x2021
+#define mmGRBM_PERFCOUNTER1_LO 0x2020
+#define mmGRBM_PERFCOUNTER1_SELECT 0x201D
+#define mmGRBM_PWR_CNTL 0x2003
+#define mmGRBM_READ_ERROR 0x2016
+#define mmGRBM_SCRATCH_REG0 0x2040
+#define mmGRBM_SCRATCH_REG1 0x2041
+#define mmGRBM_SCRATCH_REG2 0x2042
+#define mmGRBM_SCRATCH_REG3 0x2043
+#define mmGRBM_SCRATCH_REG4 0x2044
+#define mmGRBM_SCRATCH_REG5 0x2045
+#define mmGRBM_SCRATCH_REG6 0x2046
+#define mmGRBM_SCRATCH_REG7 0x2047
+#define mmGRBM_SE0_PERFCOUNTER_HI 0x202B
+#define mmGRBM_SE0_PERFCOUNTER_LO 0x202A
+#define mmGRBM_SE0_PERFCOUNTER_SELECT 0x2026
+#define mmGRBM_SE1_PERFCOUNTER_HI 0x202D
+#define mmGRBM_SE1_PERFCOUNTER_LO 0x202C
+#define mmGRBM_SE1_PERFCOUNTER_SELECT 0x2027
+#define mmGRBM_SKEW_CNTL 0x2001
+#define mmGRBM_SOFT_RESET 0x2008
+#define mmGRBM_STATUS 0x2004
+#define mmGRBM_STATUS2 0x2002
+#define mmGRBM_STATUS_SE0 0x2005
+#define mmGRBM_STATUS_SE1 0x2006
+#define mmGRBM_WAIT_IDLE_CLOCKS 0x200D
+#define mmIA_CNTL_STATUS 0x2237
+#define mmIA_DEBUG_CNTL 0x223A
+#define mmIA_DEBUG_DATA 0x223B
+#define mmIA_ENHANCE 0xA29C
+#define mmIA_MULTI_VGT_PARAM 0xA2AA
+#define mmIA_PERFCOUNTER0_HI 0x2225
+#define mmIA_PERFCOUNTER0_LO 0x2224
+#define mmIA_PERFCOUNTER0_SELECT 0x2220
+#define mmIA_PERFCOUNTER1_HI 0x2227
+#define mmIA_PERFCOUNTER1_LO 0x2226
+#define mmIA_PERFCOUNTER1_SELECT 0x2221
+#define mmIA_PERFCOUNTER2_HI 0x2229
+#define mmIA_PERFCOUNTER2_LO 0x2228
+#define mmIA_PERFCOUNTER2_SELECT 0x2222
+#define mmIA_PERFCOUNTER3_HI 0x222B
+#define mmIA_PERFCOUNTER3_LO 0x222A
+#define mmIA_PERFCOUNTER3_SELECT 0x2223
+#define mmIA_VMID_OVERRIDE 0x2260
+#define mmPA_CL_CLIP_CNTL 0xA204
+#define mmPA_CL_CNTL_STATUS 0x2284
+#define mmPA_CL_ENHANCE 0x2285
+#define mmPA_CL_GB_HORZ_CLIP_ADJ 0xA2FC
+#define mmPA_CL_GB_HORZ_DISC_ADJ 0xA2FD
+#define mmPA_CL_GB_VERT_CLIP_ADJ 0xA2FA
+#define mmPA_CL_GB_VERT_DISC_ADJ 0xA2FB
+#define mmPA_CL_NANINF_CNTL 0xA208
+#define mmPA_CL_POINT_CULL_RAD 0xA1F8
+#define mmPA_CL_POINT_SIZE 0xA1F7
+#define mmPA_CL_POINT_X_RAD 0xA1F5
+#define mmPA_CL_POINT_Y_RAD 0xA1F6
+#define mmPA_CL_UCP_0_W 0xA172
+#define mmPA_CL_UCP_0_X 0xA16F
+#define mmPA_CL_UCP_0_Y 0xA170
+#define mmPA_CL_UCP_0_Z 0xA171
+#define mmPA_CL_UCP_1_W 0xA176
+#define mmPA_CL_UCP_1_X 0xA173
+#define mmPA_CL_UCP_1_Y 0xA174
+#define mmPA_CL_UCP_1_Z 0xA175
+#define mmPA_CL_UCP_2_W 0xA17A
+#define mmPA_CL_UCP_2_X 0xA177
+#define mmPA_CL_UCP_2_Y 0xA178
+#define mmPA_CL_UCP_2_Z 0xA179
+#define mmPA_CL_UCP_3_W 0xA17E
+#define mmPA_CL_UCP_3_X 0xA17B
+#define mmPA_CL_UCP_3_Y 0xA17C
+#define mmPA_CL_UCP_3_Z 0xA17D
+#define mmPA_CL_UCP_4_W 0xA182
+#define mmPA_CL_UCP_4_X 0xA17F
+#define mmPA_CL_UCP_4_Y 0xA180
+#define mmPA_CL_UCP_4_Z 0xA181
+#define mmPA_CL_UCP_5_W 0xA186
+#define mmPA_CL_UCP_5_X 0xA183
+#define mmPA_CL_UCP_5_Y 0xA184
+#define mmPA_CL_UCP_5_Z 0xA185
+#define mmPA_CL_VPORT_XOFFSET 0xA110
+#define mmPA_CL_VPORT_XOFFSET_10 0xA14C
+#define mmPA_CL_VPORT_XOFFSET_1 0xA116
+#define mmPA_CL_VPORT_XOFFSET_11 0xA152
+#define mmPA_CL_VPORT_XOFFSET_12 0xA158
+#define mmPA_CL_VPORT_XOFFSET_13 0xA15E
+#define mmPA_CL_VPORT_XOFFSET_14 0xA164
+#define mmPA_CL_VPORT_XOFFSET_15 0xA16A
+#define mmPA_CL_VPORT_XOFFSET_2 0xA11C
+#define mmPA_CL_VPORT_XOFFSET_3 0xA122
+#define mmPA_CL_VPORT_XOFFSET_4 0xA128
+#define mmPA_CL_VPORT_XOFFSET_5 0xA12E
+#define mmPA_CL_VPORT_XOFFSET_6 0xA134
+#define mmPA_CL_VPORT_XOFFSET_7 0xA13A
+#define mmPA_CL_VPORT_XOFFSET_8 0xA140
+#define mmPA_CL_VPORT_XOFFSET_9 0xA146
+#define mmPA_CL_VPORT_XSCALE 0xA10F
+#define mmPA_CL_VPORT_XSCALE_10 0xA14B
+#define mmPA_CL_VPORT_XSCALE_1 0xA115
+#define mmPA_CL_VPORT_XSCALE_11 0xA151
+#define mmPA_CL_VPORT_XSCALE_12 0xA157
+#define mmPA_CL_VPORT_XSCALE_13 0xA15D
+#define mmPA_CL_VPORT_XSCALE_14 0xA163
+#define mmPA_CL_VPORT_XSCALE_15 0xA169
+#define mmPA_CL_VPORT_XSCALE_2 0xA11B
+#define mmPA_CL_VPORT_XSCALE_3 0xA121
+#define mmPA_CL_VPORT_XSCALE_4 0xA127
+#define mmPA_CL_VPORT_XSCALE_5 0xA12D
+#define mmPA_CL_VPORT_XSCALE_6 0xA133
+#define mmPA_CL_VPORT_XSCALE_7 0xA139
+#define mmPA_CL_VPORT_XSCALE_8 0xA13F
+#define mmPA_CL_VPORT_XSCALE_9 0xA145
+#define mmPA_CL_VPORT_YOFFSET 0xA112
+#define mmPA_CL_VPORT_YOFFSET_10 0xA14E
+#define mmPA_CL_VPORT_YOFFSET_1 0xA118
+#define mmPA_CL_VPORT_YOFFSET_11 0xA154
+#define mmPA_CL_VPORT_YOFFSET_12 0xA15A
+#define mmPA_CL_VPORT_YOFFSET_13 0xA160
+#define mmPA_CL_VPORT_YOFFSET_14 0xA166
+#define mmPA_CL_VPORT_YOFFSET_15 0xA16C
+#define mmPA_CL_VPORT_YOFFSET_2 0xA11E
+#define mmPA_CL_VPORT_YOFFSET_3 0xA124
+#define mmPA_CL_VPORT_YOFFSET_4 0xA12A
+#define mmPA_CL_VPORT_YOFFSET_5 0xA130
+#define mmPA_CL_VPORT_YOFFSET_6 0xA136
+#define mmPA_CL_VPORT_YOFFSET_7 0xA13C
+#define mmPA_CL_VPORT_YOFFSET_8 0xA142
+#define mmPA_CL_VPORT_YOFFSET_9 0xA148
+#define mmPA_CL_VPORT_YSCALE 0xA111
+#define mmPA_CL_VPORT_YSCALE_10 0xA14D
+#define mmPA_CL_VPORT_YSCALE_1 0xA117
+#define mmPA_CL_VPORT_YSCALE_11 0xA153
+#define mmPA_CL_VPORT_YSCALE_12 0xA159
+#define mmPA_CL_VPORT_YSCALE_13 0xA15F
+#define mmPA_CL_VPORT_YSCALE_14 0xA165
+#define mmPA_CL_VPORT_YSCALE_15 0xA16B
+#define mmPA_CL_VPORT_YSCALE_2 0xA11D
+#define mmPA_CL_VPORT_YSCALE_3 0xA123
+#define mmPA_CL_VPORT_YSCALE_4 0xA129
+#define mmPA_CL_VPORT_YSCALE_5 0xA12F
+#define mmPA_CL_VPORT_YSCALE_6 0xA135
+#define mmPA_CL_VPORT_YSCALE_7 0xA13B
+#define mmPA_CL_VPORT_YSCALE_8 0xA141
+#define mmPA_CL_VPORT_YSCALE_9 0xA147
+#define mmPA_CL_VPORT_ZOFFSET 0xA114
+#define mmPA_CL_VPORT_ZOFFSET_10 0xA150
+#define mmPA_CL_VPORT_ZOFFSET_1 0xA11A
+#define mmPA_CL_VPORT_ZOFFSET_11 0xA156
+#define mmPA_CL_VPORT_ZOFFSET_12 0xA15C
+#define mmPA_CL_VPORT_ZOFFSET_13 0xA162
+#define mmPA_CL_VPORT_ZOFFSET_14 0xA168
+#define mmPA_CL_VPORT_ZOFFSET_15 0xA16E
+#define mmPA_CL_VPORT_ZOFFSET_2 0xA120
+#define mmPA_CL_VPORT_ZOFFSET_3 0xA126
+#define mmPA_CL_VPORT_ZOFFSET_4 0xA12C
+#define mmPA_CL_VPORT_ZOFFSET_5 0xA132
+#define mmPA_CL_VPORT_ZOFFSET_6 0xA138
+#define mmPA_CL_VPORT_ZOFFSET_7 0xA13E
+#define mmPA_CL_VPORT_ZOFFSET_8 0xA144
+#define mmPA_CL_VPORT_ZOFFSET_9 0xA14A
+#define mmPA_CL_VPORT_ZSCALE 0xA113
+#define mmPA_CL_VPORT_ZSCALE_10 0xA14F
+#define mmPA_CL_VPORT_ZSCALE_1 0xA119
+#define mmPA_CL_VPORT_ZSCALE_11 0xA155
+#define mmPA_CL_VPORT_ZSCALE_12 0xA15B
+#define mmPA_CL_VPORT_ZSCALE_13 0xA161
+#define mmPA_CL_VPORT_ZSCALE_14 0xA167
+#define mmPA_CL_VPORT_ZSCALE_15 0xA16D
+#define mmPA_CL_VPORT_ZSCALE_2 0xA11F
+#define mmPA_CL_VPORT_ZSCALE_3 0xA125
+#define mmPA_CL_VPORT_ZSCALE_4 0xA12B
+#define mmPA_CL_VPORT_ZSCALE_5 0xA131
+#define mmPA_CL_VPORT_ZSCALE_6 0xA137
+#define mmPA_CL_VPORT_ZSCALE_7 0xA13D
+#define mmPA_CL_VPORT_ZSCALE_8 0xA143
+#define mmPA_CL_VPORT_ZSCALE_9 0xA149
+#define mmPA_CL_VS_OUT_CNTL 0xA207
+#define mmPA_CL_VTE_CNTL 0xA206
+#define mmPA_SC_AA_CONFIG 0xA2F8
+#define mmPA_SC_AA_MASK_X0Y0_X1Y0 0xA30E
+#define mmPA_SC_AA_MASK_X0Y1_X1Y1 0xA30F
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 0xA2FE
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1 0xA2FF
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2 0xA300
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3 0xA301
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0 0xA306
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1 0xA307
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2 0xA308
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3 0xA309
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0 0xA302
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1 0xA303
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2 0xA304
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3 0xA305
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0 0xA30A
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1 0xA30B
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2 0xA30C
+#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3 0xA30D
+#define mmPA_SC_CENTROID_PRIORITY_0 0xA2F5
+#define mmPA_SC_CENTROID_PRIORITY_1 0xA2F6
+#define mmPA_SC_CLIPRECT_0_BR 0xA085
+#define mmPA_SC_CLIPRECT_0_TL 0xA084
+#define mmPA_SC_CLIPRECT_1_BR 0xA087
+#define mmPA_SC_CLIPRECT_1_TL 0xA086
+#define mmPA_SC_CLIPRECT_2_BR 0xA089
+#define mmPA_SC_CLIPRECT_2_TL 0xA088
+#define mmPA_SC_CLIPRECT_3_BR 0xA08B
+#define mmPA_SC_CLIPRECT_3_TL 0xA08A
+#define mmPA_SC_CLIPRECT_RULE 0xA083
+#define mmPA_SC_DEBUG_CNTL 0x22F6
+#define mmPA_SC_DEBUG_DATA 0x22F7
+#define mmPA_SC_EDGERULE 0xA08C
+#define mmPA_SC_ENHANCE 0x22FC
+#define mmPA_SC_FIFO_DEPTH_CNTL 0x2295
+#define mmPA_SC_FIFO_SIZE 0x22F3
+#define mmPA_SC_FORCE_EOV_MAX_CNTS 0x22C9
+#define mmPA_SC_GENERIC_SCISSOR_BR 0xA091
+#define mmPA_SC_GENERIC_SCISSOR_TL 0xA090
+#define mmPA_SC_IF_FIFO_SIZE 0x22F5
+#define mmPA_SC_LINE_CNTL 0xA2F7
+#define mmPA_SC_LINE_STIPPLE 0xA283
+#define mmPA_SC_LINE_STIPPLE_STATE 0x22C4
+#define mmPA_SC_MODE_CNTL_0 0xA292
+#define mmPA_SC_MODE_CNTL_1 0xA293
+#define mmPA_SC_PERFCOUNTER0_HI 0x22A9
+#define mmPA_SC_PERFCOUNTER0_LO 0x22A8
+#define mmPA_SC_PERFCOUNTER0_SELECT 0x22A0
+#define mmPA_SC_PERFCOUNTER1_HI 0x22AB
+#define mmPA_SC_PERFCOUNTER1_LO 0x22AA
+#define mmPA_SC_PERFCOUNTER1_SELECT 0x22A1
+#define mmPA_SC_PERFCOUNTER2_HI 0x22AD
+#define mmPA_SC_PERFCOUNTER2_LO 0x22AC
+#define mmPA_SC_PERFCOUNTER2_SELECT 0x22A2
+#define mmPA_SC_PERFCOUNTER3_HI 0x22AF
+#define mmPA_SC_PERFCOUNTER3_LO 0x22AE
+#define mmPA_SC_PERFCOUNTER3_SELECT 0x22A3
+#define mmPA_SC_PERFCOUNTER4_HI 0x22B1
+#define mmPA_SC_PERFCOUNTER4_LO 0x22B0
+#define mmPA_SC_PERFCOUNTER4_SELECT 0x22A4
+#define mmPA_SC_PERFCOUNTER5_HI 0x22B3
+#define mmPA_SC_PERFCOUNTER5_LO 0x22B2
+#define mmPA_SC_PERFCOUNTER5_SELECT 0x22A5
+#define mmPA_SC_PERFCOUNTER6_HI 0x22B5
+#define mmPA_SC_PERFCOUNTER6_LO 0x22B4
+#define mmPA_SC_PERFCOUNTER6_SELECT 0x22A6
+#define mmPA_SC_PERFCOUNTER7_HI 0x22B7
+#define mmPA_SC_PERFCOUNTER7_LO 0x22B6
+#define mmPA_SC_PERFCOUNTER7_SELECT 0x22A7
+#define mmPA_SC_RASTER_CONFIG 0xA0D4
+#define mmPA_SC_SCREEN_SCISSOR_BR 0xA00D
+#define mmPA_SC_SCREEN_SCISSOR_TL 0xA00C
+#define mmPA_SC_VPORT_SCISSOR_0_BR 0xA095
+#define mmPA_SC_VPORT_SCISSOR_0_TL 0xA094
+#define mmPA_SC_VPORT_SCISSOR_10_BR 0xA0A9
+#define mmPA_SC_VPORT_SCISSOR_10_TL 0xA0A8
+#define mmPA_SC_VPORT_SCISSOR_11_BR 0xA0AB
+#define mmPA_SC_VPORT_SCISSOR_11_TL 0xA0AA
+#define mmPA_SC_VPORT_SCISSOR_12_BR 0xA0AD
+#define mmPA_SC_VPORT_SCISSOR_12_TL 0xA0AC
+#define mmPA_SC_VPORT_SCISSOR_13_BR 0xA0AF
+#define mmPA_SC_VPORT_SCISSOR_13_TL 0xA0AE
+#define mmPA_SC_VPORT_SCISSOR_14_BR 0xA0B1
+#define mmPA_SC_VPORT_SCISSOR_14_TL 0xA0B0
+#define mmPA_SC_VPORT_SCISSOR_15_BR 0xA0B3
+#define mmPA_SC_VPORT_SCISSOR_15_TL 0xA0B2
+#define mmPA_SC_VPORT_SCISSOR_1_BR 0xA097
+#define mmPA_SC_VPORT_SCISSOR_1_TL 0xA096
+#define mmPA_SC_VPORT_SCISSOR_2_BR 0xA099
+#define mmPA_SC_VPORT_SCISSOR_2_TL 0xA098
+#define mmPA_SC_VPORT_SCISSOR_3_BR 0xA09B
+#define mmPA_SC_VPORT_SCISSOR_3_TL 0xA09A
+#define mmPA_SC_VPORT_SCISSOR_4_BR 0xA09D
+#define mmPA_SC_VPORT_SCISSOR_4_TL 0xA09C
+#define mmPA_SC_VPORT_SCISSOR_5_BR 0xA09F
+#define mmPA_SC_VPORT_SCISSOR_5_TL 0xA09E
+#define mmPA_SC_VPORT_SCISSOR_6_BR 0xA0A1
+#define mmPA_SC_VPORT_SCISSOR_6_TL 0xA0A0
+#define mmPA_SC_VPORT_SCISSOR_7_BR 0xA0A3
+#define mmPA_SC_VPORT_SCISSOR_7_TL 0xA0A2
+#define mmPA_SC_VPORT_SCISSOR_8_BR 0xA0A5
+#define mmPA_SC_VPORT_SCISSOR_8_TL 0xA0A4
+#define mmPA_SC_VPORT_SCISSOR_9_BR 0xA0A7
+#define mmPA_SC_VPORT_SCISSOR_9_TL 0xA0A6
+#define mmPA_SC_VPORT_ZMAX_0 0xA0B5
+#define mmPA_SC_VPORT_ZMAX_10 0xA0C9
+#define mmPA_SC_VPORT_ZMAX_1 0xA0B7
+#define mmPA_SC_VPORT_ZMAX_11 0xA0CB
+#define mmPA_SC_VPORT_ZMAX_12 0xA0CD
+#define mmPA_SC_VPORT_ZMAX_13 0xA0CF
+#define mmPA_SC_VPORT_ZMAX_14 0xA0D1
+#define mmPA_SC_VPORT_ZMAX_15 0xA0D3
+#define mmPA_SC_VPORT_ZMAX_2 0xA0B9
+#define mmPA_SC_VPORT_ZMAX_3 0xA0BB
+#define mmPA_SC_VPORT_ZMAX_4 0xA0BD
+#define mmPA_SC_VPORT_ZMAX_5 0xA0BF
+#define mmPA_SC_VPORT_ZMAX_6 0xA0C1
+#define mmPA_SC_VPORT_ZMAX_7 0xA0C3
+#define mmPA_SC_VPORT_ZMAX_8 0xA0C5
+#define mmPA_SC_VPORT_ZMAX_9 0xA0C7
+#define mmPA_SC_VPORT_ZMIN_0 0xA0B4
+#define mmPA_SC_VPORT_ZMIN_10 0xA0C8
+#define mmPA_SC_VPORT_ZMIN_1 0xA0B6
+#define mmPA_SC_VPORT_ZMIN_11 0xA0CA
+#define mmPA_SC_VPORT_ZMIN_12 0xA0CC
+#define mmPA_SC_VPORT_ZMIN_13 0xA0CE
+#define mmPA_SC_VPORT_ZMIN_14 0xA0D0
+#define mmPA_SC_VPORT_ZMIN_15 0xA0D2
+#define mmPA_SC_VPORT_ZMIN_2 0xA0B8
+#define mmPA_SC_VPORT_ZMIN_3 0xA0BA
+#define mmPA_SC_VPORT_ZMIN_4 0xA0BC
+#define mmPA_SC_VPORT_ZMIN_5 0xA0BE
+#define mmPA_SC_VPORT_ZMIN_6 0xA0C0
+#define mmPA_SC_VPORT_ZMIN_7 0xA0C2
+#define mmPA_SC_VPORT_ZMIN_8 0xA0C4
+#define mmPA_SC_VPORT_ZMIN_9 0xA0C6
+#define mmPA_SC_WINDOW_OFFSET 0xA080
+#define mmPA_SC_WINDOW_SCISSOR_BR 0xA082
+#define mmPA_SC_WINDOW_SCISSOR_TL 0xA081
+#define mmPA_SU_CNTL_STATUS 0x2294
+#define mmPA_SU_DEBUG_CNTL 0x2280
+#define mmPA_SU_DEBUG_DATA 0x2281
+#define mmPA_SU_HARDWARE_SCREEN_OFFSET 0xA08D
+#define mmPA_SU_LINE_CNTL 0xA282
+#define mmPA_SU_LINE_STIPPLE_CNTL 0xA209
+#define mmPA_SU_LINE_STIPPLE_SCALE 0xA20A
+#define mmPA_SU_LINE_STIPPLE_VALUE 0x2298
+#define mmPA_SU_PERFCOUNTER0_HI 0x228D
+#define mmPA_SU_PERFCOUNTER0_LO 0x228C
+#define mmPA_SU_PERFCOUNTER0_SELECT 0x2288
+#define mmPA_SU_PERFCOUNTER1_HI 0x228F
+#define mmPA_SU_PERFCOUNTER1_LO 0x228E
+#define mmPA_SU_PERFCOUNTER1_SELECT 0x2289
+#define mmPA_SU_PERFCOUNTER2_HI 0x2291
+#define mmPA_SU_PERFCOUNTER2_LO 0x2290
+#define mmPA_SU_PERFCOUNTER2_SELECT 0x228A
+#define mmPA_SU_PERFCOUNTER3_HI 0x2293
+#define mmPA_SU_PERFCOUNTER3_LO 0x2292
+#define mmPA_SU_PERFCOUNTER3_SELECT 0x228B
+#define mmPA_SU_POINT_MINMAX 0xA281
+#define mmPA_SU_POINT_SIZE 0xA280
+#define mmPA_SU_POLY_OFFSET_BACK_OFFSET 0xA2E3
+#define mmPA_SU_POLY_OFFSET_BACK_SCALE 0xA2E2
+#define mmPA_SU_POLY_OFFSET_CLAMP 0xA2DF
+#define mmPA_SU_POLY_OFFSET_DB_FMT_CNTL 0xA2DE
+#define mmPA_SU_POLY_OFFSET_FRONT_OFFSET 0xA2E1
+#define mmPA_SU_POLY_OFFSET_FRONT_SCALE 0xA2E0
+#define mmPA_SU_PRIM_FILTER_CNTL 0xA20B
+#define mmPA_SU_SC_MODE_CNTL 0xA205
+#define mmPA_SU_VTX_CNTL 0xA2F9
+#define mmRAS_BCI_SIGNATURE0 0x339E
+#define mmRAS_BCI_SIGNATURE1 0x339F
+#define mmRAS_CB_SIGNATURE0 0x339D
+#define mmRAS_DB_SIGNATURE0 0x338B
+#define mmRAS_IA_SIGNATURE0 0x3397
+#define mmRAS_IA_SIGNATURE1 0x3398
+#define mmRAS_PA_SIGNATURE0 0x338C
+#define mmRAS_SC_SIGNATURE0 0x338F
+#define mmRAS_SC_SIGNATURE1 0x3390
+#define mmRAS_SC_SIGNATURE2 0x3391
+#define mmRAS_SC_SIGNATURE3 0x3392
+#define mmRAS_SC_SIGNATURE4 0x3393
+#define mmRAS_SC_SIGNATURE5 0x3394
+#define mmRAS_SC_SIGNATURE6 0x3395
+#define mmRAS_SC_SIGNATURE7 0x3396
+#define mmRAS_SIGNATURE_CONTROL 0x3380
+#define mmRAS_SIGNATURE_MASK 0x3381
+#define mmRAS_SPI_SIGNATURE0 0x3399
+#define mmRAS_SPI_SIGNATURE1 0x339A
+#define mmRAS_SQ_SIGNATURE0 0x338E
+#define mmRAS_SX_SIGNATURE0 0x3382
+#define mmRAS_SX_SIGNATURE1 0x3383
+#define mmRAS_SX_SIGNATURE2 0x3384
+#define mmRAS_SX_SIGNATURE3 0x3385
+#define mmRAS_TA_SIGNATURE0 0x339B
+#define mmRAS_TD_SIGNATURE0 0x339C
+#define mmRAS_VGT_SIGNATURE0 0x338D
+#define mmRLC_AUTO_PG_CTRL 0x310D
+#define mmRLC_CAPTURE_GPU_CLOCK_COUNT 0x30D0
+#define mmRLC_CGCG_CGLS_CTRL 0x3101
+#define mmRLC_CGCG_RAMP_CTRL 0x3102
+#define mmRLC_CGTT_MGCG_OVERRIDE 0x3100
+#define mmRLC_CNTL 0x30C0
+#define mmRLC_CU_STATUS 0x3106
+#define mmRLC_DEBUG 0x30CA
+#define mmRLC_DEBUG_SELECT 0x30C9
+#define mmRLC_DRIVER_CPDMA_STATUS 0x30C7
+#define mmRLC_DYN_PG_REQUEST 0x3104
+#define mmRLC_DYN_PG_STATUS 0x3103
+#define mmRLC_GPU_CLOCK_32 0x30D5
+#define mmRLC_GPU_CLOCK_32_RES_SEL 0x30D4
+#define mmRLC_GPU_CLOCK_COUNT_LSB 0x30CE
+#define mmRLC_GPU_CLOCK_COUNT_MSB 0x30CF
+#define mmRLC_LB_ALWAYS_ACTIVE_CU_MASK 0x3108
+#define mmRLC_LB_CNTL 0x30C3
+#define mmRLC_LB_CNTR_INIT 0x30C6
+#define mmRLC_LB_CNTR_MAX 0x30C5
+#define mmRLC_LB_INIT_CU_MASK 0x3107
+#define mmRLC_LB_PARAMS 0x3109
+#define mmRLC_LOAD_BALANCE_CNTR 0x30F6
+#define mmRLC_MAX_PG_CU 0x310C
+#define mmRLC_MC_CNTL 0x30D1
+#define mmRLC_MEM_SLP_CNTL 0x30D8
+#define mmRLC_PERFCOUNTER0_HI 0x30DC
+#define mmRLC_PERFCOUNTER0_LO 0x30DB
+#define mmRLC_PERFCOUNTER0_SELECT 0x30DA
+#define mmRLC_PERFCOUNTER1_HI 0x30DF
+#define mmRLC_PERFCOUNTER1_LO 0x30DE
+#define mmRLC_PERFCOUNTER1_SELECT 0x30DD
+#define mmRLC_PERFMON_CNTL 0x30D9
+#define mmRLC_PG_ALWAYS_ON_CU_MASK 0x310B
+#define mmRLC_PG_CNTL 0x30D7
+#define mmRLC_SAVE_AND_RESTORE_BASE 0x30C4
+#define mmRLC_SERDES_RD_DATA_0 0x3112
+#define mmRLC_SERDES_RD_DATA_1 0x3113
+#define mmRLC_SERDES_RD_DATA_2 0x3114
+#define mmRLC_SERDES_RD_MASTER_INDEX 0x3111
+#define mmRLC_SERDES_WR_CTRL 0x3117
+#define mmRLC_SERDES_WR_DATA 0x3118
+#define mmRLC_SMU_GRBM_REG_SAVE_CTRL 0x310E
+#define mmRLC_SMU_PG_CTRL 0x310F
+#define mmRLC_SMU_PG_WAKE_UP_CTRL 0x3110
+#define mmRLC_SOFT_RESET_GPU 0x30D6
+#define mmRLC_STAT 0x30D3
+#define mmRLC_THREAD1_DELAY 0x310A
+#define mmRLC_UCODE_CNTL 0x30D2
+#define mmSCRATCH_ADDR 0x2151
+#define mmSCRATCH_REG0 0x2140
+#define mmSCRATCH_REG1 0x2141
+#define mmSCRATCH_REG2 0x2142
+#define mmSCRATCH_REG3 0x2143
+#define mmSCRATCH_REG4 0x2144
+#define mmSCRATCH_REG5 0x2145
+#define mmSCRATCH_REG6 0x2146
+#define mmSCRATCH_REG7 0x2147
+#define mmSCRATCH_UMSK 0x2150
+#define mmSPI_ARB_CYCLES_0 0x243D
+#define mmSPI_ARB_CYCLES_1 0x243E
+#define mmSPI_ARB_PRIORITY 0x243C
+#define mmSPI_BARYC_CNTL 0xA1B8
+#define mmSPI_CONFIG_CNTL 0x2440
+#define mmSPI_CONFIG_CNTL_1 0x244F
+#define mmSPI_DEBUG_BUSY 0x2450
+#define mmSPI_DEBUG_CNTL 0x2441
+#define mmSPI_DEBUG_READ 0x2442
+#define mmSPI_GDS_CREDITS 0x24D8
+#define mmSPI_INTERP_CONTROL_0 0xA1B5
+#define mmSPI_LB_CTR_CTRL 0x24D4
+#define mmSPI_LB_CU_MASK 0x24D5
+#define mmSPI_LB_DATA_REG 0x24D6
+#define mmSPI_PERFCOUNTER0_HI 0x2447
+#define mmSPI_PERFCOUNTER0_LO 0x2448
+#define mmSPI_PERFCOUNTER0_SELECT 0x2443
+#define mmSPI_PERFCOUNTER1_HI 0x2449
+#define mmSPI_PERFCOUNTER1_LO 0x244A
+#define mmSPI_PERFCOUNTER1_SELECT 0x2444
+#define mmSPI_PERFCOUNTER2_HI 0x244B
+#define mmSPI_PERFCOUNTER2_LO 0x244C
+#define mmSPI_PERFCOUNTER2_SELECT 0x2445
+#define mmSPI_PERFCOUNTER3_HI 0x244D
+#define mmSPI_PERFCOUNTER3_LO 0x244E
+#define mmSPI_PERFCOUNTER3_SELECT 0x2446
+#define mmSPI_PERFCOUNTER_BINS 0x243F
+#define mmSPI_PG_ENABLE_STATIC_CU_MASK 0x24D7
+#define mmSPI_PS_IN_CONTROL 0xA1B6
+#define mmSPI_PS_INPUT_ADDR 0xA1B4
+#define mmSPI_PS_INPUT_CNTL_0 0xA191
+#define mmSPI_PS_INPUT_CNTL_10 0xA19B
+#define mmSPI_PS_INPUT_CNTL_1 0xA192
+#define mmSPI_PS_INPUT_CNTL_11 0xA19C
+#define mmSPI_PS_INPUT_CNTL_12 0xA19D
+#define mmSPI_PS_INPUT_CNTL_13 0xA19E
+#define mmSPI_PS_INPUT_CNTL_14 0xA19F
+#define mmSPI_PS_INPUT_CNTL_15 0xA1A0
+#define mmSPI_PS_INPUT_CNTL_16 0xA1A1
+#define mmSPI_PS_INPUT_CNTL_17 0xA1A2
+#define mmSPI_PS_INPUT_CNTL_18 0xA1A3
+#define mmSPI_PS_INPUT_CNTL_19 0xA1A4
+#define mmSPI_PS_INPUT_CNTL_20 0xA1A5
+#define mmSPI_PS_INPUT_CNTL_2 0xA193
+#define mmSPI_PS_INPUT_CNTL_21 0xA1A6
+#define mmSPI_PS_INPUT_CNTL_22 0xA1A7
+#define mmSPI_PS_INPUT_CNTL_23 0xA1A8
+#define mmSPI_PS_INPUT_CNTL_24 0xA1A9
+#define mmSPI_PS_INPUT_CNTL_25 0xA1AA
+#define mmSPI_PS_INPUT_CNTL_26 0xA1AB
+#define mmSPI_PS_INPUT_CNTL_27 0xA1AC
+#define mmSPI_PS_INPUT_CNTL_28 0xA1AD
+#define mmSPI_PS_INPUT_CNTL_29 0xA1AE
+#define mmSPI_PS_INPUT_CNTL_30 0xA1AF
+#define mmSPI_PS_INPUT_CNTL_3 0xA194
+#define mmSPI_PS_INPUT_CNTL_31 0xA1B0
+#define mmSPI_PS_INPUT_CNTL_4 0xA195
+#define mmSPI_PS_INPUT_CNTL_5 0xA196
+#define mmSPI_PS_INPUT_CNTL_6 0xA197
+#define mmSPI_PS_INPUT_CNTL_7 0xA198
+#define mmSPI_PS_INPUT_CNTL_8 0xA199
+#define mmSPI_PS_INPUT_CNTL_9 0xA19A
+#define mmSPI_PS_INPUT_ENA 0xA1B3
+#define mmSPI_PS_MAX_WAVE_ID 0x243B
+#define mmSPI_SHADER_COL_FORMAT 0xA1C5
+#define mmSPI_SHADER_PGM_HI_ES 0x2CC9
+#define mmSPI_SHADER_PGM_HI_GS 0x2C89
+#define mmSPI_SHADER_PGM_HI_HS 0x2D09
+#define mmSPI_SHADER_PGM_HI_LS 0x2D49
+#define mmSPI_SHADER_PGM_HI_PS 0x2C09
+#define mmSPI_SHADER_PGM_HI_VS 0x2C49
+#define mmSPI_SHADER_PGM_LO_ES 0x2CC8
+#define mmSPI_SHADER_PGM_LO_GS 0x2C88
+#define mmSPI_SHADER_PGM_LO_HS 0x2D08
+#define mmSPI_SHADER_PGM_LO_LS 0x2D48
+#define mmSPI_SHADER_PGM_LO_PS 0x2C08
+#define mmSPI_SHADER_PGM_LO_VS 0x2C48
+#define mmSPI_SHADER_PGM_RSRC1_ES 0x2CCA
+#define mmSPI_SHADER_PGM_RSRC1_GS 0x2C8A
+#define mmSPI_SHADER_PGM_RSRC1_HS 0x2D0A
+#define mmSPI_SHADER_PGM_RSRC1_LS 0x2D4A
+#define mmSPI_SHADER_PGM_RSRC1_PS 0x2C0A
+#define mmSPI_SHADER_PGM_RSRC1_VS 0x2C4A
+#define mmSPI_SHADER_PGM_RSRC2_ES 0x2CCB
+#define mmSPI_SHADER_PGM_RSRC2_GS 0x2C8B
+#define mmSPI_SHADER_PGM_RSRC2_HS 0x2D0B
+#define mmSPI_SHADER_PGM_RSRC2_LS 0x2D4B
+#define mmSPI_SHADER_PGM_RSRC2_PS 0x2C0B
+#define mmSPI_SHADER_PGM_RSRC2_VS 0x2C4B
+#define mmSPI_SHADER_POS_FORMAT 0xA1C3
+#define mmSPI_SHADER_TBA_HI_ES 0x2CC1
+#define mmSPI_SHADER_TBA_HI_GS 0x2C81
+#define mmSPI_SHADER_TBA_HI_HS 0x2D01
+#define mmSPI_SHADER_TBA_HI_LS 0x2D41
+#define mmSPI_SHADER_TBA_HI_PS 0x2C01
+#define mmSPI_SHADER_TBA_HI_VS 0x2C41
+#define mmSPI_SHADER_TBA_LO_ES 0x2CC0
+#define mmSPI_SHADER_TBA_LO_GS 0x2C80
+#define mmSPI_SHADER_TBA_LO_HS 0x2D00
+#define mmSPI_SHADER_TBA_LO_LS 0x2D40
+#define mmSPI_SHADER_TBA_LO_PS 0x2C00
+#define mmSPI_SHADER_TBA_LO_VS 0x2C40
+#define mmSPI_SHADER_TMA_HI_ES 0x2CC3
+#define mmSPI_SHADER_TMA_HI_GS 0x2C83
+#define mmSPI_SHADER_TMA_HI_HS 0x2D03
+#define mmSPI_SHADER_TMA_HI_LS 0x2D43
+#define mmSPI_SHADER_TMA_HI_PS 0x2C03
+#define mmSPI_SHADER_TMA_HI_VS 0x2C43
+#define mmSPI_SHADER_TMA_LO_ES 0x2CC2
+#define mmSPI_SHADER_TMA_LO_GS 0x2C82
+#define mmSPI_SHADER_TMA_LO_HS 0x2D02
+#define mmSPI_SHADER_TMA_LO_LS 0x2D42
+#define mmSPI_SHADER_TMA_LO_PS 0x2C02
+#define mmSPI_SHADER_TMA_LO_VS 0x2C42
+#define mmSPI_SHADER_USER_DATA_ES_0 0x2CCC
+#define mmSPI_SHADER_USER_DATA_ES_10 0x2CD6
+#define mmSPI_SHADER_USER_DATA_ES_1 0x2CCD
+#define mmSPI_SHADER_USER_DATA_ES_11 0x2CD7
+#define mmSPI_SHADER_USER_DATA_ES_12 0x2CD8
+#define mmSPI_SHADER_USER_DATA_ES_13 0x2CD9
+#define mmSPI_SHADER_USER_DATA_ES_14 0x2CDA
+#define mmSPI_SHADER_USER_DATA_ES_15 0x2CDB
+#define mmSPI_SHADER_USER_DATA_ES_2 0x2CCE
+#define mmSPI_SHADER_USER_DATA_ES_3 0x2CCF
+#define mmSPI_SHADER_USER_DATA_ES_4 0x2CD0
+#define mmSPI_SHADER_USER_DATA_ES_5 0x2CD1
+#define mmSPI_SHADER_USER_DATA_ES_6 0x2CD2
+#define mmSPI_SHADER_USER_DATA_ES_7 0x2CD3
+#define mmSPI_SHADER_USER_DATA_ES_8 0x2CD4
+#define mmSPI_SHADER_USER_DATA_ES_9 0x2CD5
+#define mmSPI_SHADER_USER_DATA_GS_0 0x2C8C
+#define mmSPI_SHADER_USER_DATA_GS_10 0x2C96
+#define mmSPI_SHADER_USER_DATA_GS_1 0x2C8D
+#define mmSPI_SHADER_USER_DATA_GS_11 0x2C97
+#define mmSPI_SHADER_USER_DATA_GS_12 0x2C98
+#define mmSPI_SHADER_USER_DATA_GS_13 0x2C99
+#define mmSPI_SHADER_USER_DATA_GS_14 0x2C9A
+#define mmSPI_SHADER_USER_DATA_GS_15 0x2C9B
+#define mmSPI_SHADER_USER_DATA_GS_2 0x2C8E
+#define mmSPI_SHADER_USER_DATA_GS_3 0x2C8F
+#define mmSPI_SHADER_USER_DATA_GS_4 0x2C90
+#define mmSPI_SHADER_USER_DATA_GS_5 0x2C91
+#define mmSPI_SHADER_USER_DATA_GS_6 0x2C92
+#define mmSPI_SHADER_USER_DATA_GS_7 0x2C93
+#define mmSPI_SHADER_USER_DATA_GS_8 0x2C94
+#define mmSPI_SHADER_USER_DATA_GS_9 0x2C95
+#define mmSPI_SHADER_USER_DATA_HS_0 0x2D0C
+#define mmSPI_SHADER_USER_DATA_HS_10 0x2D16
+#define mmSPI_SHADER_USER_DATA_HS_1 0x2D0D
+#define mmSPI_SHADER_USER_DATA_HS_11 0x2D17
+#define mmSPI_SHADER_USER_DATA_HS_12 0x2D18
+#define mmSPI_SHADER_USER_DATA_HS_13 0x2D19
+#define mmSPI_SHADER_USER_DATA_HS_14 0x2D1A
+#define mmSPI_SHADER_USER_DATA_HS_15 0x2D1B
+#define mmSPI_SHADER_USER_DATA_HS_2 0x2D0E
+#define mmSPI_SHADER_USER_DATA_HS_3 0x2D0F
+#define mmSPI_SHADER_USER_DATA_HS_4 0x2D10
+#define mmSPI_SHADER_USER_DATA_HS_5 0x2D11
+#define mmSPI_SHADER_USER_DATA_HS_6 0x2D12
+#define mmSPI_SHADER_USER_DATA_HS_7 0x2D13
+#define mmSPI_SHADER_USER_DATA_HS_8 0x2D14
+#define mmSPI_SHADER_USER_DATA_HS_9 0x2D15
+#define mmSPI_SHADER_USER_DATA_LS_0 0x2D4C
+#define mmSPI_SHADER_USER_DATA_LS_10 0x2D56
+#define mmSPI_SHADER_USER_DATA_LS_1 0x2D4D
+#define mmSPI_SHADER_USER_DATA_LS_11 0x2D57
+#define mmSPI_SHADER_USER_DATA_LS_12 0x2D58
+#define mmSPI_SHADER_USER_DATA_LS_13 0x2D59
+#define mmSPI_SHADER_USER_DATA_LS_14 0x2D5A
+#define mmSPI_SHADER_USER_DATA_LS_15 0x2D5B
+#define mmSPI_SHADER_USER_DATA_LS_2 0x2D4E
+#define mmSPI_SHADER_USER_DATA_LS_3 0x2D4F
+#define mmSPI_SHADER_USER_DATA_LS_4 0x2D50
+#define mmSPI_SHADER_USER_DATA_LS_5 0x2D51
+#define mmSPI_SHADER_USER_DATA_LS_6 0x2D52
+#define mmSPI_SHADER_USER_DATA_LS_7 0x2D53
+#define mmSPI_SHADER_USER_DATA_LS_8 0x2D54
+#define mmSPI_SHADER_USER_DATA_LS_9 0x2D55
+#define mmSPI_SHADER_USER_DATA_PS_0 0x2C0C
+#define mmSPI_SHADER_USER_DATA_PS_10 0x2C16
+#define mmSPI_SHADER_USER_DATA_PS_1 0x2C0D
+#define mmSPI_SHADER_USER_DATA_PS_11 0x2C17
+#define mmSPI_SHADER_USER_DATA_PS_12 0x2C18
+#define mmSPI_SHADER_USER_DATA_PS_13 0x2C19
+#define mmSPI_SHADER_USER_DATA_PS_14 0x2C1A
+#define mmSPI_SHADER_USER_DATA_PS_15 0x2C1B
+#define mmSPI_SHADER_USER_DATA_PS_2 0x2C0E
+#define mmSPI_SHADER_USER_DATA_PS_3 0x2C0F
+#define mmSPI_SHADER_USER_DATA_PS_4 0x2C10
+#define mmSPI_SHADER_USER_DATA_PS_5 0x2C11
+#define mmSPI_SHADER_USER_DATA_PS_6 0x2C12
+#define mmSPI_SHADER_USER_DATA_PS_7 0x2C13
+#define mmSPI_SHADER_USER_DATA_PS_8 0x2C14
+#define mmSPI_SHADER_USER_DATA_PS_9 0x2C15
+#define mmSPI_SHADER_USER_DATA_VS_0 0x2C4C
+#define mmSPI_SHADER_USER_DATA_VS_10 0x2C56
+#define mmSPI_SHADER_USER_DATA_VS_1 0x2C4D
+#define mmSPI_SHADER_USER_DATA_VS_11 0x2C57
+#define mmSPI_SHADER_USER_DATA_VS_12 0x2C58
+#define mmSPI_SHADER_USER_DATA_VS_13 0x2C59
+#define mmSPI_SHADER_USER_DATA_VS_14 0x2C5A
+#define mmSPI_SHADER_USER_DATA_VS_15 0x2C5B
+#define mmSPI_SHADER_USER_DATA_VS_2 0x2C4E
+#define mmSPI_SHADER_USER_DATA_VS_3 0x2C4F
+#define mmSPI_SHADER_USER_DATA_VS_4 0x2C50
+#define mmSPI_SHADER_USER_DATA_VS_5 0x2C51
+#define mmSPI_SHADER_USER_DATA_VS_6 0x2C52
+#define mmSPI_SHADER_USER_DATA_VS_7 0x2C53
+#define mmSPI_SHADER_USER_DATA_VS_8 0x2C54
+#define mmSPI_SHADER_USER_DATA_VS_9 0x2C55
+#define mmSPI_SHADER_Z_FORMAT 0xA1C4
+#define mmSPI_SLAVE_DEBUG_BUSY 0x24D3
+#define mmSPI_SX_EXPORT_BUFFER_SIZES 0x24D9
+#define mmSPI_SX_SCOREBOARD_BUFFER_SIZES 0x24DA
+#define mmSPI_TMPRING_SIZE 0xA1BA
+#define mmSPI_VS_OUT_CONFIG 0xA1B1
+#define mmSQ_ALU_CLK_CTRL 0x2360
+#define mmSQ_BUF_RSRC_WORD0 0x23C0
+#define mmSQ_BUF_RSRC_WORD1 0x23C1
+#define mmSQ_BUF_RSRC_WORD2 0x23C2
+#define mmSQ_BUF_RSRC_WORD3 0x23C3
+#define mmSQC_CACHES 0x2302
+#define mmSQC_CONFIG 0x2301
+#define mmSQ_CONFIG 0x2300
+#define mmSQC_SECDED_CNT 0x23A0
+#define mmSQ_DEBUG_STS_GLOBAL 0x2309
+#define mmSQ_DED_CNT 0x23A2
+#define mmSQ_DED_INFO 0x23A3
+#define mmSQ_DS_0 0x237F
+#define mmSQ_DS_1 0x237F
+#define mmSQ_EXP_0 0x237F
+#define mmSQ_EXP_1 0x237F
+#define mmSQ_FIFO_SIZES 0x2305
+#define mmSQ_IMG_RSRC_WORD0 0x23C4
+#define mmSQ_IMG_RSRC_WORD1 0x23C5
+#define mmSQ_IMG_RSRC_WORD2 0x23C6
+#define mmSQ_IMG_RSRC_WORD3 0x23C7
+#define mmSQ_IMG_RSRC_WORD4 0x23C8
+#define mmSQ_IMG_RSRC_WORD5 0x23C9
+#define mmSQ_IMG_RSRC_WORD6 0x23CA
+#define mmSQ_IMG_RSRC_WORD7 0x23CB
+#define mmSQ_IMG_SAMP_WORD0 0x23CC
+#define mmSQ_IMG_SAMP_WORD1 0x23CD
+#define mmSQ_IMG_SAMP_WORD2 0x23CE
+#define mmSQ_IMG_SAMP_WORD3 0x23CF
+#define mmSQ_IND_CMD 0x237A
+#define mmSQ_IND_DATA 0x2379
+#define mmSQ_IND_INDEX 0x2378
+#define mmSQ_INST 0x237F
+#define mmSQ_LB_CTR_CTRL 0x2398
+#define mmSQ_LB_DATA_ALU_CYCLES 0x2399
+#define mmSQ_LB_DATA_ALU_STALLS 0x239B
+#define mmSQ_LB_DATA_TEX_CYCLES 0x239A
+#define mmSQ_LB_DATA_TEX_STALLS 0x239C
+#define mmSQ_MIMG_0 0x237F
+#define mmSQ_MIMG_1 0x237F
+#define mmSQ_MTBUF_0 0x237F
+#define mmSQ_MTBUF_1 0x237F
+#define mmSQ_MUBUF_0 0x237F
+#define mmSQ_MUBUF_1 0x237F
+#define mmSQ_PERFCOUNTER0_HI 0x2321
+#define mmSQ_PERFCOUNTER0_LO 0x2320
+#define mmSQ_PERFCOUNTER0_SELECT 0x2340
+#define mmSQ_PERFCOUNTER10_HI 0x2335
+#define mmSQ_PERFCOUNTER10_LO 0x2334
+#define mmSQ_PERFCOUNTER10_SELECT 0x234A
+#define mmSQ_PERFCOUNTER11_HI 0x2337
+#define mmSQ_PERFCOUNTER11_LO 0x2336
+#define mmSQ_PERFCOUNTER11_SELECT 0x234B
+#define mmSQ_PERFCOUNTER12_HI 0x2339
+#define mmSQ_PERFCOUNTER12_LO 0x2338
+#define mmSQ_PERFCOUNTER12_SELECT 0x234C
+#define mmSQ_PERFCOUNTER13_HI 0x233B
+#define mmSQ_PERFCOUNTER13_LO 0x233A
+#define mmSQ_PERFCOUNTER13_SELECT 0x234D
+#define mmSQ_PERFCOUNTER14_HI 0x233D
+#define mmSQ_PERFCOUNTER14_LO 0x233C
+#define mmSQ_PERFCOUNTER14_SELECT 0x234E
+#define mmSQ_PERFCOUNTER15_HI 0x233F
+#define mmSQ_PERFCOUNTER15_LO 0x233E
+#define mmSQ_PERFCOUNTER15_SELECT 0x234F
+#define mmSQ_PERFCOUNTER1_HI 0x2323
+#define mmSQ_PERFCOUNTER1_LO 0x2322
+#define mmSQ_PERFCOUNTER1_SELECT 0x2341
+#define mmSQ_PERFCOUNTER2_HI 0x2325
+#define mmSQ_PERFCOUNTER2_LO 0x2324
+#define mmSQ_PERFCOUNTER2_SELECT 0x2342
+#define mmSQ_PERFCOUNTER3_HI 0x2327
+#define mmSQ_PERFCOUNTER3_LO 0x2326
+#define mmSQ_PERFCOUNTER3_SELECT 0x2343
+#define mmSQ_PERFCOUNTER4_HI 0x2329
+#define mmSQ_PERFCOUNTER4_LO 0x2328
+#define mmSQ_PERFCOUNTER4_SELECT 0x2344
+#define mmSQ_PERFCOUNTER5_HI 0x232B
+#define mmSQ_PERFCOUNTER5_LO 0x232A
+#define mmSQ_PERFCOUNTER5_SELECT 0x2345
+#define mmSQ_PERFCOUNTER6_HI 0x232D
+#define mmSQ_PERFCOUNTER6_LO 0x232C
+#define mmSQ_PERFCOUNTER6_SELECT 0x2346
+#define mmSQ_PERFCOUNTER7_HI 0x232F
+#define mmSQ_PERFCOUNTER7_LO 0x232E
+#define mmSQ_PERFCOUNTER7_SELECT 0x2347
+#define mmSQ_PERFCOUNTER8_HI 0x2331
+#define mmSQ_PERFCOUNTER8_LO 0x2330
+#define mmSQ_PERFCOUNTER8_SELECT 0x2348
+#define mmSQ_PERFCOUNTER9_HI 0x2333
+#define mmSQ_PERFCOUNTER9_LO 0x2332
+#define mmSQ_PERFCOUNTER9_SELECT 0x2349
+#define mmSQ_PERFCOUNTER_CTRL 0x2306
+#define mmSQ_POWER_THROTTLE 0x2396
+#define mmSQ_POWER_THROTTLE2 0x2397
+#define mmSQ_RANDOM_WAVE_PRI 0x2303
+#define mmSQ_REG_CREDITS 0x2304
+#define mmSQ_SEC_CNT 0x23A1
+#define mmSQ_SMRD 0x237F
+#define mmSQ_SOP1 0x237F
+#define mmSQ_SOP2 0x237F
+#define mmSQ_SOPC 0x237F
+#define mmSQ_SOPK 0x237F
+#define mmSQ_SOPP 0x237F
+#define mmSQ_TEX_CLK_CTRL 0x2361
+#define mmSQ_THREAD_TRACE_BASE 0x2380
+#define mmSQ_THREAD_TRACE_CNTR 0x2390
+#define mmSQ_THREAD_TRACE_CTRL 0x238F
+#define mmSQ_THREAD_TRACE_HIWATER 0x2392
+#define mmSQ_THREAD_TRACE_MASK 0x2382
+#define mmSQ_THREAD_TRACE_MODE 0x238E
+#define mmSQ_THREAD_TRACE_PERF_MASK 0x2384
+#define mmSQ_THREAD_TRACE_SIZE 0x2381
+#define mmSQ_THREAD_TRACE_STATUS 0x238D
+#define mmSQ_THREAD_TRACE_TOKEN_MASK 0x2383
+#define mmSQ_THREAD_TRACE_USERDATA_0 0x2388
+#define mmSQ_THREAD_TRACE_USERDATA_1 0x2389
+#define mmSQ_THREAD_TRACE_USERDATA_2 0x238A
+#define mmSQ_THREAD_TRACE_USERDATA_3 0x238B
+#define mmSQ_THREAD_TRACE_WORD_CMN 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_EVENT 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_INST 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_INST_PC_1_OF_2 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_INST_PC_2_OF_2 0x23B1
+#define mmSQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2 0x23B1
+#define mmSQ_THREAD_TRACE_WORD_ISSUE 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_MISC 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_PERF_1_OF_2 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_PERF_2_OF_2 0x23B1
+#define mmSQ_THREAD_TRACE_WORD_REG_1_OF_2 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_REG_2_OF_2 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_TIME 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2 0x23B1
+#define mmSQ_THREAD_TRACE_WORD_WAVE 0x23B0
+#define mmSQ_THREAD_TRACE_WORD_WAVE_START 0x23B0
+#define mmSQ_THREAD_TRACE_WPTR 0x238C
+#define mmSQ_TIME_HI 0x237C
+#define mmSQ_TIME_LO 0x237D
+#define mmSQ_VINTRP 0x237F
+#define mmSQ_VOP1 0x237F
+#define mmSQ_VOP2 0x237F
+#define mmSQ_VOP3_0 0x237F
+#define mmSQ_VOP3_0_SDST_ENC 0x237F
+#define mmSQ_VOP3_1 0x237F
+#define mmSQ_VOPC 0x237F
+#define mmSX_DEBUG_1 0x2418
+#define mmSX_DEBUG_BUSY 0x2414
+#define mmSX_DEBUG_BUSY_2 0x2415
+#define mmSX_DEBUG_BUSY_3 0x2416
+#define mmSX_DEBUG_BUSY_4 0x2417
+#define mmSX_PERFCOUNTER0_HI 0x2421
+#define mmSX_PERFCOUNTER0_LO 0x2420
+#define mmSX_PERFCOUNTER0_SELECT 0x241C
+#define mmSX_PERFCOUNTER1_HI 0x2423
+#define mmSX_PERFCOUNTER1_LO 0x2422
+#define mmSX_PERFCOUNTER1_SELECT 0x241D
+#define mmSX_PERFCOUNTER2_HI 0x2425
+#define mmSX_PERFCOUNTER2_LO 0x2424
+#define mmSX_PERFCOUNTER2_SELECT 0x241E
+#define mmSX_PERFCOUNTER3_HI 0x2427
+#define mmSX_PERFCOUNTER3_LO 0x2426
+#define mmSX_PERFCOUNTER3_SELECT 0x241F
+#define mmTA_BC_BASE_ADDR 0xA020
+#define mmTA_CGTT_CTRL 0x2544
+#define mmTA_CNTL 0x2541
+#define mmTA_CNTL_AUX 0x2542
+#define mmTA_CS_BC_BASE_ADDR 0x2543
+#define mmTA_DEBUG_DATA 0x254D
+#define mmTA_DEBUG_INDEX 0x254C
+#define mmTA_PERFCOUNTER0_HI 0x2556
+#define mmTA_PERFCOUNTER0_LO 0x2555
+#define mmTA_PERFCOUNTER0_SELECT 0x2554
+#define mmTA_PERFCOUNTER1_HI 0x2562
+#define mmTA_PERFCOUNTER1_LO 0x2561
+#define mmTA_PERFCOUNTER1_SELECT 0x2560
+#define mmTA_SCRATCH 0x2564
+#define mmTA_STATUS 0x2548
+#define mmTCA_CGTT_SCLK_CTRL 0x2BC1
+#define mmTCA_CTRL 0x2BC0
+#define mmTCA_PERFCOUNTER0_HI 0x2BD2
+#define mmTCA_PERFCOUNTER0_LO 0x2BD1
+#define mmTCA_PERFCOUNTER0_SELECT 0x2BD0
+#define mmTCA_PERFCOUNTER1_HI 0x2BD5
+#define mmTCA_PERFCOUNTER1_LO 0x2BD4
+#define mmTCA_PERFCOUNTER1_SELECT 0x2BD3
+#define mmTCA_PERFCOUNTER2_HI 0x2BD8
+#define mmTCA_PERFCOUNTER2_LO 0x2BD7
+#define mmTCA_PERFCOUNTER2_SELECT 0x2BD6
+#define mmTCA_PERFCOUNTER3_HI 0x2BDB
+#define mmTCA_PERFCOUNTER3_LO 0x2BDA
+#define mmTCA_PERFCOUNTER3_SELECT 0x2BD9
+#define mmTCC_CGTT_SCLK_CTRL 0x2B81
+#define mmTCC_CTRL 0x2B80
+#define mmTCC_EDC_COUNTER 0x2B82
+#define mmTCC_PERFCOUNTER0_HI 0x2B92
+#define mmTCC_PERFCOUNTER0_LO 0x2B91
+#define mmTCC_PERFCOUNTER0_SELECT 0x2B90
+#define mmTCC_PERFCOUNTER1_HI 0x2B95
+#define mmTCC_PERFCOUNTER1_LO 0x2B94
+#define mmTCC_PERFCOUNTER1_SELECT 0x2B93
+#define mmTCC_PERFCOUNTER2_HI 0x2B98
+#define mmTCC_PERFCOUNTER2_LO 0x2B97
+#define mmTCC_PERFCOUNTER2_SELECT 0x2B96
+#define mmTCC_PERFCOUNTER3_HI 0x2B9B
+#define mmTCC_PERFCOUNTER3_LO 0x2B9A
+#define mmTCC_PERFCOUNTER3_SELECT 0x2B99
+#define mmTCI_CNTL_1 0x2B62
+#define mmTCI_CNTL_2 0x2B63
+#define mmTCI_STATUS 0x2B61
+#define mmTCP_ADDR_CONFIG 0x2B05
+#define mmTCP_BUFFER_ADDR_HASH_CNTL 0x2B16
+#define mmTCP_CHAN_STEER_HI 0x2B04
+#define mmTCP_CHAN_STEER_LO 0x2B03
+#define mmTCP_CNTL 0x2B02
+#define mmTCP_CREDIT 0x2B06
+#define mmTCP_EDC_COUNTER 0x2B17
+#define mmTCP_INVALIDATE 0x2B00
+#define mmTCP_PERFCOUNTER0_HI 0x2B0A
+#define mmTCP_PERFCOUNTER0_LO 0x2B0B
+#define mmTCP_PERFCOUNTER0_SELECT 0x2B09
+#define mmTCP_PERFCOUNTER1_HI 0x2B0D
+#define mmTCP_PERFCOUNTER1_LO 0x2B0E
+#define mmTCP_PERFCOUNTER1_SELECT 0x2B0C
+#define mmTCP_PERFCOUNTER2_HI 0x2B10
+#define mmTCP_PERFCOUNTER2_LO 0x2B11
+#define mmTCP_PERFCOUNTER2_SELECT 0x2B0F
+#define mmTCP_PERFCOUNTER3_HI 0x2B13
+#define mmTCP_PERFCOUNTER3_LO 0x2B14
+#define mmTCP_PERFCOUNTER3_SELECT 0x2B12
+#define mmTCP_STATUS 0x2B01
+#define mmTD_CGTT_CTRL 0x2527
+#define mmTD_CNTL 0x2525
+#define mmTD_DEBUG_DATA 0x2529
+#define mmTD_DEBUG_INDEX 0x2528
+#define mmTD_PERFCOUNTER0_HI 0x252E
+#define mmTD_PERFCOUNTER0_LO 0x252D
+#define mmTD_PERFCOUNTER0_SELECT 0x252C
+#define mmTD_SCRATCH 0x2530
+#define mmTD_STATUS 0x2526
+#define mmUSER_SQC_BANK_DISABLE 0x2308
+#define mmVGT_CACHE_INVALIDATION 0x2231
+#define mmVGT_CNTL_STATUS 0x223C
+#define mmVGT_DEBUG_CNTL 0x2238
+#define mmVGT_DEBUG_DATA 0x2239
+#define mmVGT_DMA_BASE 0xA1FA
+#define mmVGT_DMA_BASE_HI 0xA1F9
+#define mmVGT_DMA_DATA_FIFO_DEPTH 0x222D
+#define mmVGT_DMA_INDEX_TYPE 0xA29F
+#define mmVGT_DMA_MAX_SIZE 0xA29E
+#define mmVGT_DMA_NUM_INSTANCES 0xA2A2
+#define mmVGT_DMA_REQ_FIFO_DEPTH 0x222E
+#define mmVGT_DMA_SIZE 0xA29D
+#define mmVGT_DRAW_INIT_FIFO_DEPTH 0x222F
+#define mmVGT_DRAW_INITIATOR 0xA1FC
+#define mmVGT_ENHANCE 0xA294
+#define mmVGT_ESGS_RING_ITEMSIZE 0xA2AB
+#define mmVGT_ESGS_RING_SIZE 0x2232
+#define mmVGT_ES_PER_GS 0xA296
+#define mmVGT_EVENT_ADDRESS_REG 0xA1FE
+#define mmVGT_EVENT_INITIATOR 0xA2A4
+#define mmVGT_FIFO_DEPTHS 0x2234
+#define mmVGT_GROUP_DECR 0xA28B
+#define mmVGT_GROUP_FIRST_DECR 0xA28A
+#define mmVGT_GROUP_PRIM_TYPE 0xA289
+#define mmVGT_GROUP_VECT_0_CNTL 0xA28C
+#define mmVGT_GROUP_VECT_0_FMT_CNTL 0xA28E
+#define mmVGT_GROUP_VECT_1_CNTL 0xA28D
+#define mmVGT_GROUP_VECT_1_FMT_CNTL 0xA28F
+#define mmVGT_GS_INSTANCE_CNT 0xA2E4
+#define mmVGT_GS_MAX_VERT_OUT 0xA2CE
+#define mmVGT_GS_MODE 0xA290
+#define mmVGT_GS_OUT_PRIM_TYPE 0xA29B
+#define mmVGT_GS_PER_ES 0xA295
+#define mmVGT_GS_PER_VS 0xA297
+#define mmVGT_GS_VERTEX_REUSE 0x2235
+#define mmVGT_GS_VERT_ITEMSIZE 0xA2D7
+#define mmVGT_GS_VERT_ITEMSIZE_1 0xA2D8
+#define mmVGT_GS_VERT_ITEMSIZE_2 0xA2D9
+#define mmVGT_GS_VERT_ITEMSIZE_3 0xA2DA
+#define mmVGT_GSVS_RING_ITEMSIZE 0xA2AC
+#define mmVGT_GSVS_RING_OFFSET_1 0xA298
+#define mmVGT_GSVS_RING_OFFSET_2 0xA299
+#define mmVGT_GSVS_RING_OFFSET_3 0xA29A
+#define mmVGT_GSVS_RING_SIZE 0x2233
+#define mmVGT_HOS_CNTL 0xA285
+#define mmVGT_HOS_MAX_TESS_LEVEL 0xA286
+#define mmVGT_HOS_MIN_TESS_LEVEL 0xA287
+#define mmVGT_HOS_REUSE_DEPTH 0xA288
+#define mmVGT_HS_OFFCHIP_PARAM 0x226C
+#define mmVGT_IMMED_DATA 0xA1FD
+#define mmVGT_INDEX_TYPE 0x2257
+#define mmVGT_INDX_OFFSET 0xA102
+#define mmVGT_INSTANCE_STEP_RATE_0 0xA2A8
+#define mmVGT_INSTANCE_STEP_RATE_1 0xA2A9
+#define mmVGT_LAST_COPY_STATE 0x2230
+#define mmVGT_LS_HS_CONFIG 0xA2D6
+#define mmVGT_MAX_VTX_INDX 0xA100
+#define mmVGT_MC_LAT_CNTL 0x2236
+#define mmVGT_MIN_VTX_INDX 0xA101
+#define mmVGT_MULTI_PRIM_IB_RESET_EN 0xA2A5
+#define mmVGT_MULTI_PRIM_IB_RESET_INDX 0xA103
+#define mmVGT_NUM_INDICES 0x225C
+#define mmVGT_NUM_INSTANCES 0x225D
+#define mmVGT_OUT_DEALLOC_CNTL 0xA317
+#define mmVGT_OUTPUT_PATH_CNTL 0xA284
+#define mmVGT_PERFCOUNTER0_HI 0x224D
+#define mmVGT_PERFCOUNTER0_LO 0x224C
+#define mmVGT_PERFCOUNTER0_SELECT 0x2248
+#define mmVGT_PERFCOUNTER1_HI 0x224F
+#define mmVGT_PERFCOUNTER1_LO 0x224E
+#define mmVGT_PERFCOUNTER1_SELECT 0x2249
+#define mmVGT_PERFCOUNTER2_HI 0x2251
+#define mmVGT_PERFCOUNTER2_LO 0x2250
+#define mmVGT_PERFCOUNTER2_SELECT 0x224A
+#define mmVGT_PERFCOUNTER3_HI 0x2253
+#define mmVGT_PERFCOUNTER3_LO 0x2252
+#define mmVGT_PERFCOUNTER3_SELECT 0x224B
+#define mmVGT_PERFCOUNTER_SEID_MASK 0x2247
+#define mmVGT_PRIMITIVEID_EN 0xA2A1
+#define mmVGT_PRIMITIVEID_RESET 0xA2A3
+#define mmVGT_PRIMITIVE_TYPE 0x2256
+#define mmVGT_REUSE_OFF 0xA2AD
+#define mmVGT_SHADER_STAGES_EN 0xA2D5
+#define mmVGT_STRMOUT_BUFFER_CONFIG 0xA2E6
+#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_0 0x2258
+#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_1 0x2259
+#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_2 0x225A
+#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_3 0x225B
+#define mmVGT_STRMOUT_BUFFER_OFFSET_0 0xA2B7
+#define mmVGT_STRMOUT_BUFFER_OFFSET_1 0xA2BB
+#define mmVGT_STRMOUT_BUFFER_OFFSET_2 0xA2BF
+#define mmVGT_STRMOUT_BUFFER_OFFSET_3 0xA2C3
+#define mmVGT_STRMOUT_BUFFER_SIZE_0 0xA2B4
+#define mmVGT_STRMOUT_BUFFER_SIZE_1 0xA2B8
+#define mmVGT_STRMOUT_BUFFER_SIZE_2 0xA2BC
+#define mmVGT_STRMOUT_BUFFER_SIZE_3 0xA2C0
+#define mmVGT_STRMOUT_CONFIG 0xA2E5
+#define mmVGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE 0xA2CB
+#define mmVGT_STRMOUT_DRAW_OPAQUE_OFFSET 0xA2CA
+#define mmVGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE 0xA2CC
+#define mmVGT_STRMOUT_VTX_STRIDE_0 0xA2B5
+#define mmVGT_STRMOUT_VTX_STRIDE_1 0xA2B9
+#define mmVGT_STRMOUT_VTX_STRIDE_2 0xA2BD
+#define mmVGT_STRMOUT_VTX_STRIDE_3 0xA2C1
+#define mmVGT_SYS_CONFIG 0x2263
+#define mmVGT_TF_MEMORY_BASE 0x226E
+#define mmVGT_TF_PARAM 0xA2DB
+#define mmVGT_TF_RING_SIZE 0x2262
+#define mmVGT_VERTEX_REUSE_BLOCK_CNTL 0xA316
+#define mmVGT_VTX_CNT_EN 0xA2AE
+#define mmVGT_VTX_VECT_EJECT_REG 0x222C
+
+/* manually added from old sid.h */
+#define mmCB_PERFCOUNTER0_SELECT0                       0x2688
+#define mmCB_PERFCOUNTER1_SELECT0                       0x268A
+#define mmCB_PERFCOUNTER1_SELECT1                       0x268B
+#define mmCB_PERFCOUNTER2_SELECT0                       0x268C
+#define mmCB_PERFCOUNTER2_SELECT1                       0x268D
+#define mmCB_PERFCOUNTER3_SELECT0                       0x268E
+#define mmCB_PERFCOUNTER3_SELECT1                       0x268F
+#define mmCP_COHER_CNTL2                                0x217A
+#define mmCP_DEBUG                                      0x307F
+#define mmRLC_SERDES_MASTER_BUSY_0                      0x3119
+#define mmRLC_SERDES_MASTER_BUSY_1                      0x311A
+#define mmRLC_RL_BASE                                   0x30C1
+#define mmRLC_RL_SIZE                                   0x30C2
+#define mmRLC_UCODE_ADDR                                0x30CB
+#define mmRLC_UCODE_DATA                                0x30CC
+#define mmRLC_GCPM_GENERAL_3                            0x311E
+#define mmRLC_SERDES_WR_MASTER_MASK_0                   0x3115
+#define mmRLC_SERDES_WR_MASTER_MASK_1                   0x3116
+#define mmRLC_TTOP_D                                    0x3105
+#define mmRLC_CLEAR_STATE_RESTORE_BASE                  0x30C8
+#define mmRLC_PG_AO_CU_MASK                             0x310B
+#define mmSPI_STATIC_THREAD_MGMT_3                      0x243A
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_6_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_6_0_sh_mask.h
new file mode 100644
index 000000000000..b5e634749665
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_6_0_sh_mask.h
@@ -0,0 +1,12821 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef GFX_6_0_SH_MASK_H
+#define GFX_6_0_SH_MASK_H
+
+#define BCI_DEBUG_READ__DATA_MASK 0x00ffffffL
+#define BCI_DEBUG_READ__DATA__SHIFT 0x00000000
+#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN_MASK 0x00e00000L
+#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN__SHIFT 0x00000015
+#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000L
+#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND__SHIFT 0x00000018
+#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND_MASK 0x001f0000L
+#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND__SHIFT 0x00000010
+#define CB_BLEND0_CONTROL__COLOR_COMB_FCN_MASK 0x000000e0L
+#define CB_BLEND0_CONTROL__COLOR_COMB_FCN__SHIFT 0x00000005
+#define CB_BLEND0_CONTROL__COLOR_DESTBLEND_MASK 0x00001f00L
+#define CB_BLEND0_CONTROL__COLOR_DESTBLEND__SHIFT 0x00000008
+#define CB_BLEND0_CONTROL__COLOR_SRCBLEND_MASK 0x0000001fL
+#define CB_BLEND0_CONTROL__COLOR_SRCBLEND__SHIFT 0x00000000
+#define CB_BLEND0_CONTROL__DISABLE_ROP3_MASK 0x80000000L
+#define CB_BLEND0_CONTROL__DISABLE_ROP3__SHIFT 0x0000001f
+#define CB_BLEND0_CONTROL__ENABLE_MASK 0x40000000L
+#define CB_BLEND0_CONTROL__ENABLE__SHIFT 0x0000001e
+#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000L
+#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x0000001d
+#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN_MASK 0x00e00000L
+#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN__SHIFT 0x00000015
+#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000L
+#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND__SHIFT 0x00000018
+#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND_MASK 0x001f0000L
+#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND__SHIFT 0x00000010
+#define CB_BLEND1_CONTROL__COLOR_COMB_FCN_MASK 0x000000e0L
+#define CB_BLEND1_CONTROL__COLOR_COMB_FCN__SHIFT 0x00000005
+#define CB_BLEND1_CONTROL__COLOR_DESTBLEND_MASK 0x00001f00L
+#define CB_BLEND1_CONTROL__COLOR_DESTBLEND__SHIFT 0x00000008
+#define CB_BLEND1_CONTROL__COLOR_SRCBLEND_MASK 0x0000001fL
+#define CB_BLEND1_CONTROL__COLOR_SRCBLEND__SHIFT 0x00000000
+#define CB_BLEND1_CONTROL__DISABLE_ROP3_MASK 0x80000000L
+#define CB_BLEND1_CONTROL__DISABLE_ROP3__SHIFT 0x0000001f
+#define CB_BLEND1_CONTROL__ENABLE_MASK 0x40000000L
+#define CB_BLEND1_CONTROL__ENABLE__SHIFT 0x0000001e
+#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000L
+#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x0000001d
+#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN_MASK 0x00e00000L
+#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN__SHIFT 0x00000015
+#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000L
+#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND__SHIFT 0x00000018
+#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND_MASK 0x001f0000L
+#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND__SHIFT 0x00000010
+#define CB_BLEND2_CONTROL__COLOR_COMB_FCN_MASK 0x000000e0L
+#define CB_BLEND2_CONTROL__COLOR_COMB_FCN__SHIFT 0x00000005
+#define CB_BLEND2_CONTROL__COLOR_DESTBLEND_MASK 0x00001f00L
+#define CB_BLEND2_CONTROL__COLOR_DESTBLEND__SHIFT 0x00000008
+#define CB_BLEND2_CONTROL__COLOR_SRCBLEND_MASK 0x0000001fL
+#define CB_BLEND2_CONTROL__COLOR_SRCBLEND__SHIFT 0x00000000
+#define CB_BLEND2_CONTROL__DISABLE_ROP3_MASK 0x80000000L
+#define CB_BLEND2_CONTROL__DISABLE_ROP3__SHIFT 0x0000001f
+#define CB_BLEND2_CONTROL__ENABLE_MASK 0x40000000L
+#define CB_BLEND2_CONTROL__ENABLE__SHIFT 0x0000001e
+#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000L
+#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x0000001d
+#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN_MASK 0x00e00000L
+#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN__SHIFT 0x00000015
+#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000L
+#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND__SHIFT 0x00000018
+#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND_MASK 0x001f0000L
+#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND__SHIFT 0x00000010
+#define CB_BLEND3_CONTROL__COLOR_COMB_FCN_MASK 0x000000e0L
+#define CB_BLEND3_CONTROL__COLOR_COMB_FCN__SHIFT 0x00000005
+#define CB_BLEND3_CONTROL__COLOR_DESTBLEND_MASK 0x00001f00L
+#define CB_BLEND3_CONTROL__COLOR_DESTBLEND__SHIFT 0x00000008
+#define CB_BLEND3_CONTROL__COLOR_SRCBLEND_MASK 0x0000001fL
+#define CB_BLEND3_CONTROL__COLOR_SRCBLEND__SHIFT 0x00000000
+#define CB_BLEND3_CONTROL__DISABLE_ROP3_MASK 0x80000000L
+#define CB_BLEND3_CONTROL__DISABLE_ROP3__SHIFT 0x0000001f
+#define CB_BLEND3_CONTROL__ENABLE_MASK 0x40000000L
+#define CB_BLEND3_CONTROL__ENABLE__SHIFT 0x0000001e
+#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000L
+#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x0000001d
+#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN_MASK 0x00e00000L
+#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN__SHIFT 0x00000015
+#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000L
+#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND__SHIFT 0x00000018
+#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND_MASK 0x001f0000L
+#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND__SHIFT 0x00000010
+#define CB_BLEND4_CONTROL__COLOR_COMB_FCN_MASK 0x000000e0L
+#define CB_BLEND4_CONTROL__COLOR_COMB_FCN__SHIFT 0x00000005
+#define CB_BLEND4_CONTROL__COLOR_DESTBLEND_MASK 0x00001f00L
+#define CB_BLEND4_CONTROL__COLOR_DESTBLEND__SHIFT 0x00000008
+#define CB_BLEND4_CONTROL__COLOR_SRCBLEND_MASK 0x0000001fL
+#define CB_BLEND4_CONTROL__COLOR_SRCBLEND__SHIFT 0x00000000
+#define CB_BLEND4_CONTROL__DISABLE_ROP3_MASK 0x80000000L
+#define CB_BLEND4_CONTROL__DISABLE_ROP3__SHIFT 0x0000001f
+#define CB_BLEND4_CONTROL__ENABLE_MASK 0x40000000L
+#define CB_BLEND4_CONTROL__ENABLE__SHIFT 0x0000001e
+#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000L
+#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x0000001d
+#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN_MASK 0x00e00000L
+#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN__SHIFT 0x00000015
+#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000L
+#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND__SHIFT 0x00000018
+#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND_MASK 0x001f0000L
+#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND__SHIFT 0x00000010
+#define CB_BLEND5_CONTROL__COLOR_COMB_FCN_MASK 0x000000e0L
+#define CB_BLEND5_CONTROL__COLOR_COMB_FCN__SHIFT 0x00000005
+#define CB_BLEND5_CONTROL__COLOR_DESTBLEND_MASK 0x00001f00L
+#define CB_BLEND5_CONTROL__COLOR_DESTBLEND__SHIFT 0x00000008
+#define CB_BLEND5_CONTROL__COLOR_SRCBLEND_MASK 0x0000001fL
+#define CB_BLEND5_CONTROL__COLOR_SRCBLEND__SHIFT 0x00000000
+#define CB_BLEND5_CONTROL__DISABLE_ROP3_MASK 0x80000000L
+#define CB_BLEND5_CONTROL__DISABLE_ROP3__SHIFT 0x0000001f
+#define CB_BLEND5_CONTROL__ENABLE_MASK 0x40000000L
+#define CB_BLEND5_CONTROL__ENABLE__SHIFT 0x0000001e
+#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000L
+#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x0000001d
+#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN_MASK 0x00e00000L
+#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN__SHIFT 0x00000015
+#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000L
+#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND__SHIFT 0x00000018
+#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND_MASK 0x001f0000L
+#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND__SHIFT 0x00000010
+#define CB_BLEND6_CONTROL__COLOR_COMB_FCN_MASK 0x000000e0L
+#define CB_BLEND6_CONTROL__COLOR_COMB_FCN__SHIFT 0x00000005
+#define CB_BLEND6_CONTROL__COLOR_DESTBLEND_MASK 0x00001f00L
+#define CB_BLEND6_CONTROL__COLOR_DESTBLEND__SHIFT 0x00000008
+#define CB_BLEND6_CONTROL__COLOR_SRCBLEND_MASK 0x0000001fL
+#define CB_BLEND6_CONTROL__COLOR_SRCBLEND__SHIFT 0x00000000
+#define CB_BLEND6_CONTROL__DISABLE_ROP3_MASK 0x80000000L
+#define CB_BLEND6_CONTROL__DISABLE_ROP3__SHIFT 0x0000001f
+#define CB_BLEND6_CONTROL__ENABLE_MASK 0x40000000L
+#define CB_BLEND6_CONTROL__ENABLE__SHIFT 0x0000001e
+#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000L
+#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x0000001d
+#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN_MASK 0x00e00000L
+#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN__SHIFT 0x00000015
+#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000L
+#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND__SHIFT 0x00000018
+#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND_MASK 0x001f0000L
+#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND__SHIFT 0x00000010
+#define CB_BLEND7_CONTROL__COLOR_COMB_FCN_MASK 0x000000e0L
+#define CB_BLEND7_CONTROL__COLOR_COMB_FCN__SHIFT 0x00000005
+#define CB_BLEND7_CONTROL__COLOR_DESTBLEND_MASK 0x00001f00L
+#define CB_BLEND7_CONTROL__COLOR_DESTBLEND__SHIFT 0x00000008
+#define CB_BLEND7_CONTROL__COLOR_SRCBLEND_MASK 0x0000001fL
+#define CB_BLEND7_CONTROL__COLOR_SRCBLEND__SHIFT 0x00000000
+#define CB_BLEND7_CONTROL__DISABLE_ROP3_MASK 0x80000000L
+#define CB_BLEND7_CONTROL__DISABLE_ROP3__SHIFT 0x0000001f
+#define CB_BLEND7_CONTROL__ENABLE_MASK 0x40000000L
+#define CB_BLEND7_CONTROL__ENABLE__SHIFT 0x0000001e
+#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000L
+#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x0000001d
+#define CB_BLEND_ALPHA__BLEND_ALPHA_MASK 0xffffffffL
+#define CB_BLEND_ALPHA__BLEND_ALPHA__SHIFT 0x00000000
+#define CB_BLEND_BLUE__BLEND_BLUE_MASK 0xffffffffL
+#define CB_BLEND_BLUE__BLEND_BLUE__SHIFT 0x00000000
+#define CB_BLEND_GREEN__BLEND_GREEN_MASK 0xffffffffL
+#define CB_BLEND_GREEN__BLEND_GREEN__SHIFT 0x00000000
+#define CB_BLEND_RED__BLEND_RED_MASK 0xffffffffL
+#define CB_BLEND_RED__BLEND_RED__SHIFT 0x00000000
+#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CB_CGTT_SCLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CB_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CB_COLOR0_ATTRIB__FMASK_BANK_HEIGHT_MASK 0x00000c00L
+#define CB_COLOR0_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0x0000000a
+#define CB_COLOR0_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x000003e0L
+#define CB_COLOR0_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x00000005
+#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x00020000L
+#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x00000011
+#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS_MASK 0x00018000L
+#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS__SHIFT 0x0000000f
+#define CB_COLOR0_ATTRIB__NUM_SAMPLES_MASK 0x00007000L
+#define CB_COLOR0_ATTRIB__NUM_SAMPLES__SHIFT 0x0000000c
+#define CB_COLOR0_ATTRIB__TILE_MODE_INDEX_MASK 0x0000001fL
+#define CB_COLOR0_ATTRIB__TILE_MODE_INDEX__SHIFT 0x00000000
+#define CB_COLOR0_BASE__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR0_BASE__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR0_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffffL
+#define CB_COLOR0_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x00000000
+#define CB_COLOR0_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffffL
+#define CB_COLOR0_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x00000000
+#define CB_COLOR0_CMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR0_CMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR0_CMASK_SLICE__TILE_MAX_MASK 0x00003fffL
+#define CB_COLOR0_CMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR0_FMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR0_FMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR0_FMASK_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR0_FMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR0_INFO__BLEND_BYPASS_MASK 0x00010000L
+#define CB_COLOR0_INFO__BLEND_BYPASS__SHIFT 0x00000010
+#define CB_COLOR0_INFO__BLEND_CLAMP_MASK 0x00008000L
+#define CB_COLOR0_INFO__BLEND_CLAMP__SHIFT 0x0000000f
+#define CB_COLOR0_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x03800000L
+#define CB_COLOR0_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x00000017
+#define CB_COLOR0_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x00700000L
+#define CB_COLOR0_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x00000014
+#define CB_COLOR0_INFO__CMASK_IS_LINEAR_MASK 0x00080000L
+#define CB_COLOR0_INFO__CMASK_IS_LINEAR__SHIFT 0x00000013
+#define CB_COLOR0_INFO__COMPRESSION_MASK 0x00004000L
+#define CB_COLOR0_INFO__COMPRESSION__SHIFT 0x0000000e
+#define CB_COLOR0_INFO__COMP_SWAP_MASK 0x00001800L
+#define CB_COLOR0_INFO__COMP_SWAP__SHIFT 0x0000000b
+#define CB_COLOR0_INFO__ENDIAN_MASK 0x00000003L
+#define CB_COLOR0_INFO__ENDIAN__SHIFT 0x00000000
+#define CB_COLOR0_INFO__FAST_CLEAR_MASK 0x00002000L
+#define CB_COLOR0_INFO__FAST_CLEAR__SHIFT 0x0000000d
+#define CB_COLOR0_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x04000000L
+#define CB_COLOR0_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x0000001a
+#define CB_COLOR0_INFO__FORMAT_MASK 0x0000007cL
+#define CB_COLOR0_INFO__FORMAT__SHIFT 0x00000002
+#define CB_COLOR0_INFO__LINEAR_GENERAL_MASK 0x00000080L
+#define CB_COLOR0_INFO__LINEAR_GENERAL__SHIFT 0x00000007
+#define CB_COLOR0_INFO__NUMBER_TYPE_MASK 0x00000700L
+#define CB_COLOR0_INFO__NUMBER_TYPE__SHIFT 0x00000008
+#define CB_COLOR0_INFO__ROUND_MODE_MASK 0x00040000L
+#define CB_COLOR0_INFO__ROUND_MODE__SHIFT 0x00000012
+#define CB_COLOR0_INFO__SIMPLE_FLOAT_MASK 0x00020000L
+#define CB_COLOR0_INFO__SIMPLE_FLOAT__SHIFT 0x00000011
+#define CB_COLOR0_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000L
+#define CB_COLOR0_PITCH__FMASK_TILE_MAX__SHIFT 0x00000014
+#define CB_COLOR0_PITCH__TILE_MAX_MASK 0x000007ffL
+#define CB_COLOR0_PITCH__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR0_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR0_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR0_VIEW__SLICE_MAX_MASK 0x00ffe000L
+#define CB_COLOR0_VIEW__SLICE_MAX__SHIFT 0x0000000d
+#define CB_COLOR0_VIEW__SLICE_START_MASK 0x000007ffL
+#define CB_COLOR0_VIEW__SLICE_START__SHIFT 0x00000000
+#define CB_COLOR1_ATTRIB__FMASK_BANK_HEIGHT_MASK 0x00000c00L
+#define CB_COLOR1_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0x0000000a
+#define CB_COLOR1_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x000003e0L
+#define CB_COLOR1_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x00000005
+#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x00020000L
+#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x00000011
+#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS_MASK 0x00018000L
+#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS__SHIFT 0x0000000f
+#define CB_COLOR1_ATTRIB__NUM_SAMPLES_MASK 0x00007000L
+#define CB_COLOR1_ATTRIB__NUM_SAMPLES__SHIFT 0x0000000c
+#define CB_COLOR1_ATTRIB__TILE_MODE_INDEX_MASK 0x0000001fL
+#define CB_COLOR1_ATTRIB__TILE_MODE_INDEX__SHIFT 0x00000000
+#define CB_COLOR1_BASE__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR1_BASE__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR1_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffffL
+#define CB_COLOR1_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x00000000
+#define CB_COLOR1_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffffL
+#define CB_COLOR1_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x00000000
+#define CB_COLOR1_CMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR1_CMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR1_CMASK_SLICE__TILE_MAX_MASK 0x00003fffL
+#define CB_COLOR1_CMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR1_FMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR1_FMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR1_FMASK_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR1_FMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR1_INFO__BLEND_BYPASS_MASK 0x00010000L
+#define CB_COLOR1_INFO__BLEND_BYPASS__SHIFT 0x00000010
+#define CB_COLOR1_INFO__BLEND_CLAMP_MASK 0x00008000L
+#define CB_COLOR1_INFO__BLEND_CLAMP__SHIFT 0x0000000f
+#define CB_COLOR1_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x03800000L
+#define CB_COLOR1_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x00000017
+#define CB_COLOR1_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x00700000L
+#define CB_COLOR1_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x00000014
+#define CB_COLOR1_INFO__CMASK_IS_LINEAR_MASK 0x00080000L
+#define CB_COLOR1_INFO__CMASK_IS_LINEAR__SHIFT 0x00000013
+#define CB_COLOR1_INFO__COMPRESSION_MASK 0x00004000L
+#define CB_COLOR1_INFO__COMPRESSION__SHIFT 0x0000000e
+#define CB_COLOR1_INFO__COMP_SWAP_MASK 0x00001800L
+#define CB_COLOR1_INFO__COMP_SWAP__SHIFT 0x0000000b
+#define CB_COLOR1_INFO__ENDIAN_MASK 0x00000003L
+#define CB_COLOR1_INFO__ENDIAN__SHIFT 0x00000000
+#define CB_COLOR1_INFO__FAST_CLEAR_MASK 0x00002000L
+#define CB_COLOR1_INFO__FAST_CLEAR__SHIFT 0x0000000d
+#define CB_COLOR1_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x04000000L
+#define CB_COLOR1_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x0000001a
+#define CB_COLOR1_INFO__FORMAT_MASK 0x0000007cL
+#define CB_COLOR1_INFO__FORMAT__SHIFT 0x00000002
+#define CB_COLOR1_INFO__LINEAR_GENERAL_MASK 0x00000080L
+#define CB_COLOR1_INFO__LINEAR_GENERAL__SHIFT 0x00000007
+#define CB_COLOR1_INFO__NUMBER_TYPE_MASK 0x00000700L
+#define CB_COLOR1_INFO__NUMBER_TYPE__SHIFT 0x00000008
+#define CB_COLOR1_INFO__ROUND_MODE_MASK 0x00040000L
+#define CB_COLOR1_INFO__ROUND_MODE__SHIFT 0x00000012
+#define CB_COLOR1_INFO__SIMPLE_FLOAT_MASK 0x00020000L
+#define CB_COLOR1_INFO__SIMPLE_FLOAT__SHIFT 0x00000011
+#define CB_COLOR1_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000L
+#define CB_COLOR1_PITCH__FMASK_TILE_MAX__SHIFT 0x00000014
+#define CB_COLOR1_PITCH__TILE_MAX_MASK 0x000007ffL
+#define CB_COLOR1_PITCH__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR1_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR1_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR1_VIEW__SLICE_MAX_MASK 0x00ffe000L
+#define CB_COLOR1_VIEW__SLICE_MAX__SHIFT 0x0000000d
+#define CB_COLOR1_VIEW__SLICE_START_MASK 0x000007ffL
+#define CB_COLOR1_VIEW__SLICE_START__SHIFT 0x00000000
+#define CB_COLOR2_ATTRIB__FMASK_BANK_HEIGHT_MASK 0x00000c00L
+#define CB_COLOR2_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0x0000000a
+#define CB_COLOR2_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x000003e0L
+#define CB_COLOR2_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x00000005
+#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x00020000L
+#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x00000011
+#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS_MASK 0x00018000L
+#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS__SHIFT 0x0000000f
+#define CB_COLOR2_ATTRIB__NUM_SAMPLES_MASK 0x00007000L
+#define CB_COLOR2_ATTRIB__NUM_SAMPLES__SHIFT 0x0000000c
+#define CB_COLOR2_ATTRIB__TILE_MODE_INDEX_MASK 0x0000001fL
+#define CB_COLOR2_ATTRIB__TILE_MODE_INDEX__SHIFT 0x00000000
+#define CB_COLOR2_BASE__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR2_BASE__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR2_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffffL
+#define CB_COLOR2_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x00000000
+#define CB_COLOR2_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffffL
+#define CB_COLOR2_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x00000000
+#define CB_COLOR2_CMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR2_CMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR2_CMASK_SLICE__TILE_MAX_MASK 0x00003fffL
+#define CB_COLOR2_CMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR2_FMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR2_FMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR2_FMASK_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR2_FMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR2_INFO__BLEND_BYPASS_MASK 0x00010000L
+#define CB_COLOR2_INFO__BLEND_BYPASS__SHIFT 0x00000010
+#define CB_COLOR2_INFO__BLEND_CLAMP_MASK 0x00008000L
+#define CB_COLOR2_INFO__BLEND_CLAMP__SHIFT 0x0000000f
+#define CB_COLOR2_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x03800000L
+#define CB_COLOR2_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x00000017
+#define CB_COLOR2_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x00700000L
+#define CB_COLOR2_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x00000014
+#define CB_COLOR2_INFO__CMASK_IS_LINEAR_MASK 0x00080000L
+#define CB_COLOR2_INFO__CMASK_IS_LINEAR__SHIFT 0x00000013
+#define CB_COLOR2_INFO__COMPRESSION_MASK 0x00004000L
+#define CB_COLOR2_INFO__COMPRESSION__SHIFT 0x0000000e
+#define CB_COLOR2_INFO__COMP_SWAP_MASK 0x00001800L
+#define CB_COLOR2_INFO__COMP_SWAP__SHIFT 0x0000000b
+#define CB_COLOR2_INFO__ENDIAN_MASK 0x00000003L
+#define CB_COLOR2_INFO__ENDIAN__SHIFT 0x00000000
+#define CB_COLOR2_INFO__FAST_CLEAR_MASK 0x00002000L
+#define CB_COLOR2_INFO__FAST_CLEAR__SHIFT 0x0000000d
+#define CB_COLOR2_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x04000000L
+#define CB_COLOR2_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x0000001a
+#define CB_COLOR2_INFO__FORMAT_MASK 0x0000007cL
+#define CB_COLOR2_INFO__FORMAT__SHIFT 0x00000002
+#define CB_COLOR2_INFO__LINEAR_GENERAL_MASK 0x00000080L
+#define CB_COLOR2_INFO__LINEAR_GENERAL__SHIFT 0x00000007
+#define CB_COLOR2_INFO__NUMBER_TYPE_MASK 0x00000700L
+#define CB_COLOR2_INFO__NUMBER_TYPE__SHIFT 0x00000008
+#define CB_COLOR2_INFO__ROUND_MODE_MASK 0x00040000L
+#define CB_COLOR2_INFO__ROUND_MODE__SHIFT 0x00000012
+#define CB_COLOR2_INFO__SIMPLE_FLOAT_MASK 0x00020000L
+#define CB_COLOR2_INFO__SIMPLE_FLOAT__SHIFT 0x00000011
+#define CB_COLOR2_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000L
+#define CB_COLOR2_PITCH__FMASK_TILE_MAX__SHIFT 0x00000014
+#define CB_COLOR2_PITCH__TILE_MAX_MASK 0x000007ffL
+#define CB_COLOR2_PITCH__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR2_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR2_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR2_VIEW__SLICE_MAX_MASK 0x00ffe000L
+#define CB_COLOR2_VIEW__SLICE_MAX__SHIFT 0x0000000d
+#define CB_COLOR2_VIEW__SLICE_START_MASK 0x000007ffL
+#define CB_COLOR2_VIEW__SLICE_START__SHIFT 0x00000000
+#define CB_COLOR3_ATTRIB__FMASK_BANK_HEIGHT_MASK 0x00000c00L
+#define CB_COLOR3_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0x0000000a
+#define CB_COLOR3_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x000003e0L
+#define CB_COLOR3_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x00000005
+#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x00020000L
+#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x00000011
+#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS_MASK 0x00018000L
+#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS__SHIFT 0x0000000f
+#define CB_COLOR3_ATTRIB__NUM_SAMPLES_MASK 0x00007000L
+#define CB_COLOR3_ATTRIB__NUM_SAMPLES__SHIFT 0x0000000c
+#define CB_COLOR3_ATTRIB__TILE_MODE_INDEX_MASK 0x0000001fL
+#define CB_COLOR3_ATTRIB__TILE_MODE_INDEX__SHIFT 0x00000000
+#define CB_COLOR3_BASE__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR3_BASE__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR3_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffffL
+#define CB_COLOR3_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x00000000
+#define CB_COLOR3_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffffL
+#define CB_COLOR3_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x00000000
+#define CB_COLOR3_CMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR3_CMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR3_CMASK_SLICE__TILE_MAX_MASK 0x00003fffL
+#define CB_COLOR3_CMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR3_FMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR3_FMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR3_FMASK_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR3_FMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR3_INFO__BLEND_BYPASS_MASK 0x00010000L
+#define CB_COLOR3_INFO__BLEND_BYPASS__SHIFT 0x00000010
+#define CB_COLOR3_INFO__BLEND_CLAMP_MASK 0x00008000L
+#define CB_COLOR3_INFO__BLEND_CLAMP__SHIFT 0x0000000f
+#define CB_COLOR3_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x03800000L
+#define CB_COLOR3_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x00000017
+#define CB_COLOR3_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x00700000L
+#define CB_COLOR3_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x00000014
+#define CB_COLOR3_INFO__CMASK_IS_LINEAR_MASK 0x00080000L
+#define CB_COLOR3_INFO__CMASK_IS_LINEAR__SHIFT 0x00000013
+#define CB_COLOR3_INFO__COMPRESSION_MASK 0x00004000L
+#define CB_COLOR3_INFO__COMPRESSION__SHIFT 0x0000000e
+#define CB_COLOR3_INFO__COMP_SWAP_MASK 0x00001800L
+#define CB_COLOR3_INFO__COMP_SWAP__SHIFT 0x0000000b
+#define CB_COLOR3_INFO__ENDIAN_MASK 0x00000003L
+#define CB_COLOR3_INFO__ENDIAN__SHIFT 0x00000000
+#define CB_COLOR3_INFO__FAST_CLEAR_MASK 0x00002000L
+#define CB_COLOR3_INFO__FAST_CLEAR__SHIFT 0x0000000d
+#define CB_COLOR3_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x04000000L
+#define CB_COLOR3_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x0000001a
+#define CB_COLOR3_INFO__FORMAT_MASK 0x0000007cL
+#define CB_COLOR3_INFO__FORMAT__SHIFT 0x00000002
+#define CB_COLOR3_INFO__LINEAR_GENERAL_MASK 0x00000080L
+#define CB_COLOR3_INFO__LINEAR_GENERAL__SHIFT 0x00000007
+#define CB_COLOR3_INFO__NUMBER_TYPE_MASK 0x00000700L
+#define CB_COLOR3_INFO__NUMBER_TYPE__SHIFT 0x00000008
+#define CB_COLOR3_INFO__ROUND_MODE_MASK 0x00040000L
+#define CB_COLOR3_INFO__ROUND_MODE__SHIFT 0x00000012
+#define CB_COLOR3_INFO__SIMPLE_FLOAT_MASK 0x00020000L
+#define CB_COLOR3_INFO__SIMPLE_FLOAT__SHIFT 0x00000011
+#define CB_COLOR3_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000L
+#define CB_COLOR3_PITCH__FMASK_TILE_MAX__SHIFT 0x00000014
+#define CB_COLOR3_PITCH__TILE_MAX_MASK 0x000007ffL
+#define CB_COLOR3_PITCH__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR3_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR3_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR3_VIEW__SLICE_MAX_MASK 0x00ffe000L
+#define CB_COLOR3_VIEW__SLICE_MAX__SHIFT 0x0000000d
+#define CB_COLOR3_VIEW__SLICE_START_MASK 0x000007ffL
+#define CB_COLOR3_VIEW__SLICE_START__SHIFT 0x00000000
+#define CB_COLOR4_ATTRIB__FMASK_BANK_HEIGHT_MASK 0x00000c00L
+#define CB_COLOR4_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0x0000000a
+#define CB_COLOR4_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x000003e0L
+#define CB_COLOR4_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x00000005
+#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x00020000L
+#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x00000011
+#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS_MASK 0x00018000L
+#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS__SHIFT 0x0000000f
+#define CB_COLOR4_ATTRIB__NUM_SAMPLES_MASK 0x00007000L
+#define CB_COLOR4_ATTRIB__NUM_SAMPLES__SHIFT 0x0000000c
+#define CB_COLOR4_ATTRIB__TILE_MODE_INDEX_MASK 0x0000001fL
+#define CB_COLOR4_ATTRIB__TILE_MODE_INDEX__SHIFT 0x00000000
+#define CB_COLOR4_BASE__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR4_BASE__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR4_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffffL
+#define CB_COLOR4_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x00000000
+#define CB_COLOR4_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffffL
+#define CB_COLOR4_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x00000000
+#define CB_COLOR4_CMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR4_CMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR4_CMASK_SLICE__TILE_MAX_MASK 0x00003fffL
+#define CB_COLOR4_CMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR4_FMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR4_FMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR4_FMASK_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR4_FMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR4_INFO__BLEND_BYPASS_MASK 0x00010000L
+#define CB_COLOR4_INFO__BLEND_BYPASS__SHIFT 0x00000010
+#define CB_COLOR4_INFO__BLEND_CLAMP_MASK 0x00008000L
+#define CB_COLOR4_INFO__BLEND_CLAMP__SHIFT 0x0000000f
+#define CB_COLOR4_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x03800000L
+#define CB_COLOR4_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x00000017
+#define CB_COLOR4_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x00700000L
+#define CB_COLOR4_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x00000014
+#define CB_COLOR4_INFO__CMASK_IS_LINEAR_MASK 0x00080000L
+#define CB_COLOR4_INFO__CMASK_IS_LINEAR__SHIFT 0x00000013
+#define CB_COLOR4_INFO__COMPRESSION_MASK 0x00004000L
+#define CB_COLOR4_INFO__COMPRESSION__SHIFT 0x0000000e
+#define CB_COLOR4_INFO__COMP_SWAP_MASK 0x00001800L
+#define CB_COLOR4_INFO__COMP_SWAP__SHIFT 0x0000000b
+#define CB_COLOR4_INFO__ENDIAN_MASK 0x00000003L
+#define CB_COLOR4_INFO__ENDIAN__SHIFT 0x00000000
+#define CB_COLOR4_INFO__FAST_CLEAR_MASK 0x00002000L
+#define CB_COLOR4_INFO__FAST_CLEAR__SHIFT 0x0000000d
+#define CB_COLOR4_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x04000000L
+#define CB_COLOR4_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x0000001a
+#define CB_COLOR4_INFO__FORMAT_MASK 0x0000007cL
+#define CB_COLOR4_INFO__FORMAT__SHIFT 0x00000002
+#define CB_COLOR4_INFO__LINEAR_GENERAL_MASK 0x00000080L
+#define CB_COLOR4_INFO__LINEAR_GENERAL__SHIFT 0x00000007
+#define CB_COLOR4_INFO__NUMBER_TYPE_MASK 0x00000700L
+#define CB_COLOR4_INFO__NUMBER_TYPE__SHIFT 0x00000008
+#define CB_COLOR4_INFO__ROUND_MODE_MASK 0x00040000L
+#define CB_COLOR4_INFO__ROUND_MODE__SHIFT 0x00000012
+#define CB_COLOR4_INFO__SIMPLE_FLOAT_MASK 0x00020000L
+#define CB_COLOR4_INFO__SIMPLE_FLOAT__SHIFT 0x00000011
+#define CB_COLOR4_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000L
+#define CB_COLOR4_PITCH__FMASK_TILE_MAX__SHIFT 0x00000014
+#define CB_COLOR4_PITCH__TILE_MAX_MASK 0x000007ffL
+#define CB_COLOR4_PITCH__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR4_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR4_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR4_VIEW__SLICE_MAX_MASK 0x00ffe000L
+#define CB_COLOR4_VIEW__SLICE_MAX__SHIFT 0x0000000d
+#define CB_COLOR4_VIEW__SLICE_START_MASK 0x000007ffL
+#define CB_COLOR4_VIEW__SLICE_START__SHIFT 0x00000000
+#define CB_COLOR5_ATTRIB__FMASK_BANK_HEIGHT_MASK 0x00000c00L
+#define CB_COLOR5_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0x0000000a
+#define CB_COLOR5_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x000003e0L
+#define CB_COLOR5_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x00000005
+#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x00020000L
+#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x00000011
+#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS_MASK 0x00018000L
+#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS__SHIFT 0x0000000f
+#define CB_COLOR5_ATTRIB__NUM_SAMPLES_MASK 0x00007000L
+#define CB_COLOR5_ATTRIB__NUM_SAMPLES__SHIFT 0x0000000c
+#define CB_COLOR5_ATTRIB__TILE_MODE_INDEX_MASK 0x0000001fL
+#define CB_COLOR5_ATTRIB__TILE_MODE_INDEX__SHIFT 0x00000000
+#define CB_COLOR5_BASE__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR5_BASE__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR5_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffffL
+#define CB_COLOR5_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x00000000
+#define CB_COLOR5_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffffL
+#define CB_COLOR5_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x00000000
+#define CB_COLOR5_CMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR5_CMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR5_CMASK_SLICE__TILE_MAX_MASK 0x00003fffL
+#define CB_COLOR5_CMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR5_FMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR5_FMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR5_FMASK_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR5_FMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR5_INFO__BLEND_BYPASS_MASK 0x00010000L
+#define CB_COLOR5_INFO__BLEND_BYPASS__SHIFT 0x00000010
+#define CB_COLOR5_INFO__BLEND_CLAMP_MASK 0x00008000L
+#define CB_COLOR5_INFO__BLEND_CLAMP__SHIFT 0x0000000f
+#define CB_COLOR5_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x03800000L
+#define CB_COLOR5_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x00000017
+#define CB_COLOR5_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x00700000L
+#define CB_COLOR5_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x00000014
+#define CB_COLOR5_INFO__CMASK_IS_LINEAR_MASK 0x00080000L
+#define CB_COLOR5_INFO__CMASK_IS_LINEAR__SHIFT 0x00000013
+#define CB_COLOR5_INFO__COMPRESSION_MASK 0x00004000L
+#define CB_COLOR5_INFO__COMPRESSION__SHIFT 0x0000000e
+#define CB_COLOR5_INFO__COMP_SWAP_MASK 0x00001800L
+#define CB_COLOR5_INFO__COMP_SWAP__SHIFT 0x0000000b
+#define CB_COLOR5_INFO__ENDIAN_MASK 0x00000003L
+#define CB_COLOR5_INFO__ENDIAN__SHIFT 0x00000000
+#define CB_COLOR5_INFO__FAST_CLEAR_MASK 0x00002000L
+#define CB_COLOR5_INFO__FAST_CLEAR__SHIFT 0x0000000d
+#define CB_COLOR5_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x04000000L
+#define CB_COLOR5_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x0000001a
+#define CB_COLOR5_INFO__FORMAT_MASK 0x0000007cL
+#define CB_COLOR5_INFO__FORMAT__SHIFT 0x00000002
+#define CB_COLOR5_INFO__LINEAR_GENERAL_MASK 0x00000080L
+#define CB_COLOR5_INFO__LINEAR_GENERAL__SHIFT 0x00000007
+#define CB_COLOR5_INFO__NUMBER_TYPE_MASK 0x00000700L
+#define CB_COLOR5_INFO__NUMBER_TYPE__SHIFT 0x00000008
+#define CB_COLOR5_INFO__ROUND_MODE_MASK 0x00040000L
+#define CB_COLOR5_INFO__ROUND_MODE__SHIFT 0x00000012
+#define CB_COLOR5_INFO__SIMPLE_FLOAT_MASK 0x00020000L
+#define CB_COLOR5_INFO__SIMPLE_FLOAT__SHIFT 0x00000011
+#define CB_COLOR5_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000L
+#define CB_COLOR5_PITCH__FMASK_TILE_MAX__SHIFT 0x00000014
+#define CB_COLOR5_PITCH__TILE_MAX_MASK 0x000007ffL
+#define CB_COLOR5_PITCH__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR5_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR5_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR5_VIEW__SLICE_MAX_MASK 0x00ffe000L
+#define CB_COLOR5_VIEW__SLICE_MAX__SHIFT 0x0000000d
+#define CB_COLOR5_VIEW__SLICE_START_MASK 0x000007ffL
+#define CB_COLOR5_VIEW__SLICE_START__SHIFT 0x00000000
+#define CB_COLOR6_ATTRIB__FMASK_BANK_HEIGHT_MASK 0x00000c00L
+#define CB_COLOR6_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0x0000000a
+#define CB_COLOR6_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x000003e0L
+#define CB_COLOR6_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x00000005
+#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x00020000L
+#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x00000011
+#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS_MASK 0x00018000L
+#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS__SHIFT 0x0000000f
+#define CB_COLOR6_ATTRIB__NUM_SAMPLES_MASK 0x00007000L
+#define CB_COLOR6_ATTRIB__NUM_SAMPLES__SHIFT 0x0000000c
+#define CB_COLOR6_ATTRIB__TILE_MODE_INDEX_MASK 0x0000001fL
+#define CB_COLOR6_ATTRIB__TILE_MODE_INDEX__SHIFT 0x00000000
+#define CB_COLOR6_BASE__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR6_BASE__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR6_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffffL
+#define CB_COLOR6_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x00000000
+#define CB_COLOR6_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffffL
+#define CB_COLOR6_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x00000000
+#define CB_COLOR6_CMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR6_CMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR6_CMASK_SLICE__TILE_MAX_MASK 0x00003fffL
+#define CB_COLOR6_CMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR6_FMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR6_FMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR6_FMASK_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR6_FMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR6_INFO__BLEND_BYPASS_MASK 0x00010000L
+#define CB_COLOR6_INFO__BLEND_BYPASS__SHIFT 0x00000010
+#define CB_COLOR6_INFO__BLEND_CLAMP_MASK 0x00008000L
+#define CB_COLOR6_INFO__BLEND_CLAMP__SHIFT 0x0000000f
+#define CB_COLOR6_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x03800000L
+#define CB_COLOR6_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x00000017
+#define CB_COLOR6_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x00700000L
+#define CB_COLOR6_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x00000014
+#define CB_COLOR6_INFO__CMASK_IS_LINEAR_MASK 0x00080000L
+#define CB_COLOR6_INFO__CMASK_IS_LINEAR__SHIFT 0x00000013
+#define CB_COLOR6_INFO__COMPRESSION_MASK 0x00004000L
+#define CB_COLOR6_INFO__COMPRESSION__SHIFT 0x0000000e
+#define CB_COLOR6_INFO__COMP_SWAP_MASK 0x00001800L
+#define CB_COLOR6_INFO__COMP_SWAP__SHIFT 0x0000000b
+#define CB_COLOR6_INFO__ENDIAN_MASK 0x00000003L
+#define CB_COLOR6_INFO__ENDIAN__SHIFT 0x00000000
+#define CB_COLOR6_INFO__FAST_CLEAR_MASK 0x00002000L
+#define CB_COLOR6_INFO__FAST_CLEAR__SHIFT 0x0000000d
+#define CB_COLOR6_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x04000000L
+#define CB_COLOR6_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x0000001a
+#define CB_COLOR6_INFO__FORMAT_MASK 0x0000007cL
+#define CB_COLOR6_INFO__FORMAT__SHIFT 0x00000002
+#define CB_COLOR6_INFO__LINEAR_GENERAL_MASK 0x00000080L
+#define CB_COLOR6_INFO__LINEAR_GENERAL__SHIFT 0x00000007
+#define CB_COLOR6_INFO__NUMBER_TYPE_MASK 0x00000700L
+#define CB_COLOR6_INFO__NUMBER_TYPE__SHIFT 0x00000008
+#define CB_COLOR6_INFO__ROUND_MODE_MASK 0x00040000L
+#define CB_COLOR6_INFO__ROUND_MODE__SHIFT 0x00000012
+#define CB_COLOR6_INFO__SIMPLE_FLOAT_MASK 0x00020000L
+#define CB_COLOR6_INFO__SIMPLE_FLOAT__SHIFT 0x00000011
+#define CB_COLOR6_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000L
+#define CB_COLOR6_PITCH__FMASK_TILE_MAX__SHIFT 0x00000014
+#define CB_COLOR6_PITCH__TILE_MAX_MASK 0x000007ffL
+#define CB_COLOR6_PITCH__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR6_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR6_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR6_VIEW__SLICE_MAX_MASK 0x00ffe000L
+#define CB_COLOR6_VIEW__SLICE_MAX__SHIFT 0x0000000d
+#define CB_COLOR6_VIEW__SLICE_START_MASK 0x000007ffL
+#define CB_COLOR6_VIEW__SLICE_START__SHIFT 0x00000000
+#define CB_COLOR7_ATTRIB__FMASK_BANK_HEIGHT_MASK 0x00000c00L
+#define CB_COLOR7_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0x0000000a
+#define CB_COLOR7_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x000003e0L
+#define CB_COLOR7_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x00000005
+#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x00020000L
+#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x00000011
+#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS_MASK 0x00018000L
+#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS__SHIFT 0x0000000f
+#define CB_COLOR7_ATTRIB__NUM_SAMPLES_MASK 0x00007000L
+#define CB_COLOR7_ATTRIB__NUM_SAMPLES__SHIFT 0x0000000c
+#define CB_COLOR7_ATTRIB__TILE_MODE_INDEX_MASK 0x0000001fL
+#define CB_COLOR7_ATTRIB__TILE_MODE_INDEX__SHIFT 0x00000000
+#define CB_COLOR7_BASE__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR7_BASE__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR7_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffffL
+#define CB_COLOR7_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x00000000
+#define CB_COLOR7_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffffL
+#define CB_COLOR7_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x00000000
+#define CB_COLOR7_CMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR7_CMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR7_CMASK_SLICE__TILE_MAX_MASK 0x00003fffL
+#define CB_COLOR7_CMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR7_FMASK__BASE_256B_MASK 0xffffffffL
+#define CB_COLOR7_FMASK__BASE_256B__SHIFT 0x00000000
+#define CB_COLOR7_FMASK_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR7_FMASK_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR7_INFO__BLEND_BYPASS_MASK 0x00010000L
+#define CB_COLOR7_INFO__BLEND_BYPASS__SHIFT 0x00000010
+#define CB_COLOR7_INFO__BLEND_CLAMP_MASK 0x00008000L
+#define CB_COLOR7_INFO__BLEND_CLAMP__SHIFT 0x0000000f
+#define CB_COLOR7_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x03800000L
+#define CB_COLOR7_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x00000017
+#define CB_COLOR7_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x00700000L
+#define CB_COLOR7_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x00000014
+#define CB_COLOR7_INFO__CMASK_IS_LINEAR_MASK 0x00080000L
+#define CB_COLOR7_INFO__CMASK_IS_LINEAR__SHIFT 0x00000013
+#define CB_COLOR7_INFO__COMPRESSION_MASK 0x00004000L
+#define CB_COLOR7_INFO__COMPRESSION__SHIFT 0x0000000e
+#define CB_COLOR7_INFO__COMP_SWAP_MASK 0x00001800L
+#define CB_COLOR7_INFO__COMP_SWAP__SHIFT 0x0000000b
+#define CB_COLOR7_INFO__ENDIAN_MASK 0x00000003L
+#define CB_COLOR7_INFO__ENDIAN__SHIFT 0x00000000
+#define CB_COLOR7_INFO__FAST_CLEAR_MASK 0x00002000L
+#define CB_COLOR7_INFO__FAST_CLEAR__SHIFT 0x0000000d
+#define CB_COLOR7_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x04000000L
+#define CB_COLOR7_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x0000001a
+#define CB_COLOR7_INFO__FORMAT_MASK 0x0000007cL
+#define CB_COLOR7_INFO__FORMAT__SHIFT 0x00000002
+#define CB_COLOR7_INFO__LINEAR_GENERAL_MASK 0x00000080L
+#define CB_COLOR7_INFO__LINEAR_GENERAL__SHIFT 0x00000007
+#define CB_COLOR7_INFO__NUMBER_TYPE_MASK 0x00000700L
+#define CB_COLOR7_INFO__NUMBER_TYPE__SHIFT 0x00000008
+#define CB_COLOR7_INFO__ROUND_MODE_MASK 0x00040000L
+#define CB_COLOR7_INFO__ROUND_MODE__SHIFT 0x00000012
+#define CB_COLOR7_INFO__SIMPLE_FLOAT_MASK 0x00020000L
+#define CB_COLOR7_INFO__SIMPLE_FLOAT__SHIFT 0x00000011
+#define CB_COLOR7_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000L
+#define CB_COLOR7_PITCH__FMASK_TILE_MAX__SHIFT 0x00000014
+#define CB_COLOR7_PITCH__TILE_MAX_MASK 0x000007ffL
+#define CB_COLOR7_PITCH__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR7_SLICE__TILE_MAX_MASK 0x003fffffL
+#define CB_COLOR7_SLICE__TILE_MAX__SHIFT 0x00000000
+#define CB_COLOR7_VIEW__SLICE_MAX_MASK 0x00ffe000L
+#define CB_COLOR7_VIEW__SLICE_MAX__SHIFT 0x0000000d
+#define CB_COLOR7_VIEW__SLICE_START_MASK 0x000007ffL
+#define CB_COLOR7_VIEW__SLICE_START__SHIFT 0x00000000
+#define CB_COLOR_CONTROL__DEGAMMA_ENABLE_MASK 0x00000008L
+#define CB_COLOR_CONTROL__DEGAMMA_ENABLE__SHIFT 0x00000003
+#define CB_COLOR_CONTROL__MODE_MASK 0x00000070L
+#define CB_COLOR_CONTROL__MODE__SHIFT 0x00000004
+#define CB_COLOR_CONTROL__ROP3_MASK 0x00ff0000L
+#define CB_COLOR_CONTROL__ROP3__SHIFT 0x00000010
+#define CB_DEBUG_BUS_13__AC_BUSY_MASK 0x00000008L
+#define CB_DEBUG_BUS_13__AC_BUSY__SHIFT 0x00000003
+#define CB_DEBUG_BUS_13__CACHE_CTRL_BUSY_MASK 0x00000020L
+#define CB_DEBUG_BUS_13__CACHE_CTRL_BUSY__SHIFT 0x00000005
+#define CB_DEBUG_BUS_13__CRW_BUSY_MASK 0x00000010L
+#define CB_DEBUG_BUS_13__CRW_BUSY__SHIFT 0x00000004
+#define CB_DEBUG_BUS_13__EVICT_PENDING_MASK 0x00000200L
+#define CB_DEBUG_BUS_13__EVICT_PENDING__SHIFT 0x00000009
+#define CB_DEBUG_BUS_13__FC_RD_PENDING_MASK 0x00000100L
+#define CB_DEBUG_BUS_13__FC_RD_PENDING__SHIFT 0x00000008
+#define CB_DEBUG_BUS_13__FC_WR_PENDING_MASK 0x00000080L
+#define CB_DEBUG_BUS_13__FC_WR_PENDING__SHIFT 0x00000007
+#define CB_DEBUG_BUS_13__LAST_RD_ARB_WINNER_MASK 0x00000400L
+#define CB_DEBUG_BUS_13__LAST_RD_ARB_WINNER__SHIFT 0x0000000a
+#define CB_DEBUG_BUS_13__MC_WR_PENDING_MASK 0x00000040L
+#define CB_DEBUG_BUS_13__MC_WR_PENDING__SHIFT 0x00000006
+#define CB_DEBUG_BUS_13__MU_BUSY_MASK 0x00000002L
+#define CB_DEBUG_BUS_13__MU_BUSY__SHIFT 0x00000001
+#define CB_DEBUG_BUS_13__MU_STATE_MASK 0x0007f800L
+#define CB_DEBUG_BUS_13__MU_STATE__SHIFT 0x0000000b
+#define CB_DEBUG_BUS_13__TILE_INTFC_BUSY_MASK 0x00000001L
+#define CB_DEBUG_BUS_13__TILE_INTFC_BUSY__SHIFT 0x00000000
+#define CB_DEBUG_BUS_13__TQ_BUSY_MASK 0x00000004L
+#define CB_DEBUG_BUS_13__TQ_BUSY__SHIFT 0x00000002
+#define CB_DEBUG_BUS_14__ADDR_BUSY_MASK 0x00000010L
+#define CB_DEBUG_BUS_14__ADDR_BUSY__SHIFT 0x00000004
+#define CB_DEBUG_BUS_14__CACHE_CTL_BUSY_MASK 0x00000008L
+#define CB_DEBUG_BUS_14__CACHE_CTL_BUSY__SHIFT 0x00000003
+#define CB_DEBUG_BUS_14__CLEAR_BUSY_MASK 0x00000100L
+#define CB_DEBUG_BUS_14__CLEAR_BUSY__SHIFT 0x00000008
+#define CB_DEBUG_BUS_14__FOP_BUSY_MASK 0x00000002L
+#define CB_DEBUG_BUS_14__FOP_BUSY__SHIFT 0x00000001
+#define CB_DEBUG_BUS_14__LAT_BUSY_MASK 0x00000004L
+#define CB_DEBUG_BUS_14__LAT_BUSY__SHIFT 0x00000002
+#define CB_DEBUG_BUS_14__MERGE_BUSY_MASK 0x00000020L
+#define CB_DEBUG_BUS_14__MERGE_BUSY__SHIFT 0x00000005
+#define CB_DEBUG_BUS_14__QUAD_BUSY_MASK 0x00000040L
+#define CB_DEBUG_BUS_14__QUAD_BUSY__SHIFT 0x00000006
+#define CB_DEBUG_BUS_14__TILE_BUSY_MASK 0x00000080L
+#define CB_DEBUG_BUS_14__TILE_BUSY__SHIFT 0x00000007
+#define CB_DEBUG_BUS_14__TILE_RETIREMENT_BUSY_MASK 0x00000001L
+#define CB_DEBUG_BUS_14__TILE_RETIREMENT_BUSY__SHIFT 0x00000000
+#define CB_DEBUG_BUS_15__CS_BUSY_MASK 0x00000010L
+#define CB_DEBUG_BUS_15__CS_BUSY__SHIFT 0x00000004
+#define CB_DEBUG_BUS_15__DS_BUSY_MASK 0x00000040L
+#define CB_DEBUG_BUS_15__DS_BUSY__SHIFT 0x00000006
+#define CB_DEBUG_BUS_15__IB_BUSY_MASK 0x00000100L
+#define CB_DEBUG_BUS_15__IB_BUSY__SHIFT 0x00000008
+#define CB_DEBUG_BUS_15__RB_BUSY_MASK 0x00000020L
+#define CB_DEBUG_BUS_15__RB_BUSY__SHIFT 0x00000005
+#define CB_DEBUG_BUS_15__SF_BUSY_MASK 0x00000008L
+#define CB_DEBUG_BUS_15__SF_BUSY__SHIFT 0x00000003
+#define CB_DEBUG_BUS_15__SURF_SYNC_START_MASK 0x00000004L
+#define CB_DEBUG_BUS_15__SURF_SYNC_START__SHIFT 0x00000002
+#define CB_DEBUG_BUS_15__SURF_SYNC_STATE_MASK 0x00000003L
+#define CB_DEBUG_BUS_15__SURF_SYNC_STATE__SHIFT 0x00000000
+#define CB_DEBUG_BUS_15__TB_BUSY_MASK 0x00000080L
+#define CB_DEBUG_BUS_15__TB_BUSY__SHIFT 0x00000007
+#define CB_DEBUG_BUS_16__CC_WRREQ_FIFO_EMPTY_MASK 0x00100000L
+#define CB_DEBUG_BUS_16__CC_WRREQ_FIFO_EMPTY__SHIFT 0x00000014
+#define CB_DEBUG_BUS_16__CM_WRREQ_FIFO_EMPTY_MASK 0x00400000L
+#define CB_DEBUG_BUS_16__CM_WRREQ_FIFO_EMPTY__SHIFT 0x00000016
+#define CB_DEBUG_BUS_16__FC_WRREQ_FIFO_EMPTY_MASK 0x00200000L
+#define CB_DEBUG_BUS_16__FC_WRREQ_FIFO_EMPTY__SHIFT 0x00000015
+#define CB_DEBUG_BUS_16__LAST_RD_GRANT_VEC_MASK 0x000003c0L
+#define CB_DEBUG_BUS_16__LAST_RD_GRANT_VEC__SHIFT 0x00000006
+#define CB_DEBUG_BUS_16__LAST_WR_GRANT_VEC_MASK 0x000f0000L
+#define CB_DEBUG_BUS_16__LAST_WR_GRANT_VEC__SHIFT 0x00000010
+#define CB_DEBUG_BUS_16__MC_RDREQ_CREDITS_MASK 0x0000003fL
+#define CB_DEBUG_BUS_16__MC_RDREQ_CREDITS__SHIFT 0x00000000
+#define CB_DEBUG_BUS_16__MC_WRREQ_CREDITS_MASK 0x0000fc00L
+#define CB_DEBUG_BUS_16__MC_WRREQ_CREDITS__SHIFT 0x0000000a
+#define CB_DEBUG_BUS_17__BB_BUSY_MASK 0x00000008L
+#define CB_DEBUG_BUS_17__BB_BUSY__SHIFT 0x00000003
+#define CB_DEBUG_BUS_17__CC_BUSY_MASK 0x00000004L
+#define CB_DEBUG_BUS_17__CC_BUSY__SHIFT 0x00000002
+#define CB_DEBUG_BUS_17__CM_BUSY_MASK 0x00000001L
+#define CB_DEBUG_BUS_17__CM_BUSY__SHIFT 0x00000000
+#define CB_DEBUG_BUS_17__CORE_SCLK_VLD_MASK 0x00000020L
+#define CB_DEBUG_BUS_17__CORE_SCLK_VLD__SHIFT 0x00000005
+#define CB_DEBUG_BUS_17__FC_BUSY_MASK 0x00000002L
+#define CB_DEBUG_BUS_17__FC_BUSY__SHIFT 0x00000001
+#define CB_DEBUG_BUS_17__MA_BUSY_MASK 0x00000010L
+#define CB_DEBUG_BUS_17__MA_BUSY__SHIFT 0x00000004
+#define CB_DEBUG_BUS_17__REG_SCLK0_VLD_MASK 0x00000080L
+#define CB_DEBUG_BUS_17__REG_SCLK0_VLD__SHIFT 0x00000007
+#define CB_DEBUG_BUS_17__REG_SCLK1_VLD_MASK 0x00000040L
+#define CB_DEBUG_BUS_17__REG_SCLK1_VLD__SHIFT 0x00000006
+#define CB_DEBUG_BUS_18__NOT_USED_MASK 0x00ffffffL
+#define CB_DEBUG_BUS_18__NOT_USED__SHIFT 0x00000000
+#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS_MASK 0x0001f800L
+#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS__SHIFT 0x0000000b
+#define CB_HW_CONTROL_1__CHICKEN_BITS_MASK 0xfc000000L
+#define CB_HW_CONTROL_1__CHICKEN_BITS__SHIFT 0x0000001a
+#define CB_HW_CONTROL_1__CM_CACHE_NUM_TAGS_MASK 0x0000001fL
+#define CB_HW_CONTROL_1__CM_CACHE_NUM_TAGS__SHIFT 0x00000000
+#define CB_HW_CONTROL_1__CM_TILE_FIFO_DEPTH_MASK 0x03fe0000L
+#define CB_HW_CONTROL_1__CM_TILE_FIFO_DEPTH__SHIFT 0x00000011
+#define CB_HW_CONTROL_1__FC_CACHE_NUM_TAGS_MASK 0x000007e0L
+#define CB_HW_CONTROL_1__FC_CACHE_NUM_TAGS__SHIFT 0x00000005
+#define CB_HW_CONTROL_2__CC_EVEN_ODD_FIFO_DEPTH_MASK 0x000000ffL
+#define CB_HW_CONTROL_2__CC_EVEN_ODD_FIFO_DEPTH__SHIFT 0x00000000
+#define CB_HW_CONTROL_2__CHICKEN_BITS_MASK 0xff800000L
+#define CB_HW_CONTROL_2__CHICKEN_BITS__SHIFT 0x00000017
+#define CB_HW_CONTROL_2__FC_RDLAT_QUAD_FIFO_DEPTH_MASK 0x007f8000L
+#define CB_HW_CONTROL_2__FC_RDLAT_QUAD_FIFO_DEPTH__SHIFT 0x0000000f
+#define CB_HW_CONTROL_2__FC_RDLAT_TILE_FIFO_DEPTH_MASK 0x00007f00L
+#define CB_HW_CONTROL_2__FC_RDLAT_TILE_FIFO_DEPTH__SHIFT 0x00000008
+#define CB_HW_CONTROL_3__DISABLE_SLOW_MODE_EMPTY_HALF_QUAD_KILL_MASK 0x00000001L
+#define CB_HW_CONTROL_3__DISABLE_SLOW_MODE_EMPTY_HALF_QUAD_KILL__SHIFT 0x00000000
+#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE_MASK 0x00010000L
+#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE__SHIFT 0x00000010
+#define CB_HW_CONTROL__CC_CACHE_EVICT_POINT_MASK 0x0000f000L
+#define CB_HW_CONTROL__CC_CACHE_EVICT_POINT__SHIFT 0x0000000c
+#define CB_HW_CONTROL__CM_CACHE_EVICT_POINT_MASK 0x0000000fL
+#define CB_HW_CONTROL__CM_CACHE_EVICT_POINT__SHIFT 0x00000000
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS_MASK 0x02000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS__SHIFT 0x00000019
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL_MASK 0x04000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL__SHIFT 0x0000001a
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST_MASK 0x01000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST__SHIFT 0x00000018
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST_MASK 0x00200000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST__SHIFT 0x00000015
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED_MASK 0x08000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED__SHIFT 0x0000001b
+#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT_MASK 0x40000000L
+#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT__SHIFT 0x0000001e
+#define CB_HW_CONTROL__DISABLE_FULL_WRITE_MASK_MASK 0x00400000L
+#define CB_HW_CONTROL__DISABLE_FULL_WRITE_MASK__SHIFT 0x00000016
+#define CB_HW_CONTROL__DISABLE_INTNORM_LE11BPC_CLAMPING_MASK 0x00040000L
+#define CB_HW_CONTROL__DISABLE_INTNORM_LE11BPC_CLAMPING__SHIFT 0x00000012
+#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE_MASK 0x80000000L
+#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE__SHIFT 0x0000001f
+#define CB_HW_CONTROL__DISABLE_RESOLVE_OPT_FOR_SINGLE_FRAG_MASK 0x00800000L
+#define CB_HW_CONTROL__DISABLE_RESOLVE_OPT_FOR_SINGLE_FRAG__SHIFT 0x00000017
+#define CB_HW_CONTROL__FC_CACHE_EVICT_POINT_MASK 0x000003c0L
+#define CB_HW_CONTROL__FC_CACHE_EVICT_POINT__SHIFT 0x00000006
+#define CB_HW_CONTROL__FORCE_ALWAYS_TOGGLE_MASK 0x00100000L
+#define CB_HW_CONTROL__FORCE_ALWAYS_TOGGLE__SHIFT 0x00000014
+#define CB_HW_CONTROL__FORCE_NEEDS_DST_MASK 0x00080000L
+#define CB_HW_CONTROL__FORCE_NEEDS_DST__SHIFT 0x00000013
+#define CB_HW_CONTROL__PRIORITIZE_FC_EVICT_OVER_FOP_RD_ON_BANK_CONFLICT_MASK 0x20000000L
+#define CB_HW_CONTROL__PRIORITIZE_FC_EVICT_OVER_FOP_RD_ON_BANK_CONFLICT__SHIFT 0x0000001d
+#define CB_HW_CONTROL__PRIORITIZE_FC_WR_OVER_FC_RD_ON_CMASK_CONFLICT_MASK 0x10000000L
+#define CB_HW_CONTROL__PRIORITIZE_FC_WR_OVER_FC_RD_ON_CMASK_CONFLICT__SHIFT 0x0000001c
+#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000L
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x0000001c
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0x0f000000L
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x00000018
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x000001ffL
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x0007fc00L
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define CB_SHADER_MASK__OUTPUT0_ENABLE_MASK 0x0000000fL
+#define CB_SHADER_MASK__OUTPUT0_ENABLE__SHIFT 0x00000000
+#define CB_SHADER_MASK__OUTPUT1_ENABLE_MASK 0x000000f0L
+#define CB_SHADER_MASK__OUTPUT1_ENABLE__SHIFT 0x00000004
+#define CB_SHADER_MASK__OUTPUT2_ENABLE_MASK 0x00000f00L
+#define CB_SHADER_MASK__OUTPUT2_ENABLE__SHIFT 0x00000008
+#define CB_SHADER_MASK__OUTPUT3_ENABLE_MASK 0x0000f000L
+#define CB_SHADER_MASK__OUTPUT3_ENABLE__SHIFT 0x0000000c
+#define CB_SHADER_MASK__OUTPUT4_ENABLE_MASK 0x000f0000L
+#define CB_SHADER_MASK__OUTPUT4_ENABLE__SHIFT 0x00000010
+#define CB_SHADER_MASK__OUTPUT5_ENABLE_MASK 0x00f00000L
+#define CB_SHADER_MASK__OUTPUT5_ENABLE__SHIFT 0x00000014
+#define CB_SHADER_MASK__OUTPUT6_ENABLE_MASK 0x0f000000L
+#define CB_SHADER_MASK__OUTPUT6_ENABLE__SHIFT 0x00000018
+#define CB_SHADER_MASK__OUTPUT7_ENABLE_MASK 0xf0000000L
+#define CB_SHADER_MASK__OUTPUT7_ENABLE__SHIFT 0x0000001c
+#define CB_TARGET_MASK__TARGET0_ENABLE_MASK 0x0000000fL
+#define CB_TARGET_MASK__TARGET0_ENABLE__SHIFT 0x00000000
+#define CB_TARGET_MASK__TARGET1_ENABLE_MASK 0x000000f0L
+#define CB_TARGET_MASK__TARGET1_ENABLE__SHIFT 0x00000004
+#define CB_TARGET_MASK__TARGET2_ENABLE_MASK 0x00000f00L
+#define CB_TARGET_MASK__TARGET2_ENABLE__SHIFT 0x00000008
+#define CB_TARGET_MASK__TARGET3_ENABLE_MASK 0x0000f000L
+#define CB_TARGET_MASK__TARGET3_ENABLE__SHIFT 0x0000000c
+#define CB_TARGET_MASK__TARGET4_ENABLE_MASK 0x000f0000L
+#define CB_TARGET_MASK__TARGET4_ENABLE__SHIFT 0x00000010
+#define CB_TARGET_MASK__TARGET5_ENABLE_MASK 0x00f00000L
+#define CB_TARGET_MASK__TARGET5_ENABLE__SHIFT 0x00000014
+#define CB_TARGET_MASK__TARGET6_ENABLE_MASK 0x0f000000L
+#define CB_TARGET_MASK__TARGET6_ENABLE__SHIFT 0x00000018
+#define CB_TARGET_MASK__TARGET7_ENABLE_MASK 0xf0000000L
+#define CB_TARGET_MASK__TARGET7_ENABLE__SHIFT 0x0000001c
+#define CC_GC_SHADER_ARRAY_CONFIG__DPFP_RATE_MASK 0x00000006L
+#define CC_GC_SHADER_ARRAY_CONFIG__DPFP_RATE__SHIFT 0x00000001
+#define CC_GC_SHADER_ARRAY_CONFIG__HALF_LDS_MASK 0x00000010L
+#define CC_GC_SHADER_ARRAY_CONFIG__HALF_LDS__SHIFT 0x00000004
+#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK 0xffff0000L
+#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT 0x00000010
+#define CC_GC_SHADER_ARRAY_CONFIG__SQC_BALANCE_DISABLE_MASK 0x00000008L
+#define CC_GC_SHADER_ARRAY_CONFIG__SQC_BALANCE_DISABLE__SHIFT 0x00000003
+#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK 0x00ff0000L
+#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT 0x00000010
+#define CC_RB_DAISY_CHAIN__RB_0_MASK 0x0000000fL
+#define CC_RB_DAISY_CHAIN__RB_0__SHIFT 0x00000000
+#define CC_RB_DAISY_CHAIN__RB_1_MASK 0x000000f0L
+#define CC_RB_DAISY_CHAIN__RB_1__SHIFT 0x00000004
+#define CC_RB_DAISY_CHAIN__RB_2_MASK 0x00000f00L
+#define CC_RB_DAISY_CHAIN__RB_2__SHIFT 0x00000008
+#define CC_RB_DAISY_CHAIN__RB_3_MASK 0x0000f000L
+#define CC_RB_DAISY_CHAIN__RB_3__SHIFT 0x0000000c
+#define CC_RB_DAISY_CHAIN__RB_4_MASK 0x000f0000L
+#define CC_RB_DAISY_CHAIN__RB_4__SHIFT 0x00000010
+#define CC_RB_DAISY_CHAIN__RB_5_MASK 0x00f00000L
+#define CC_RB_DAISY_CHAIN__RB_5__SHIFT 0x00000014
+#define CC_RB_DAISY_CHAIN__RB_6_MASK 0x0f000000L
+#define CC_RB_DAISY_CHAIN__RB_6__SHIFT 0x00000018
+#define CC_RB_DAISY_CHAIN__RB_7_MASK 0xf0000000L
+#define CC_RB_DAISY_CHAIN__RB_7__SHIFT 0x0000001c
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY0_MASK 0x00001000L
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY0__SHIFT 0x0000000c
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY1_MASK 0x00100000L
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY1__SHIFT 0x00000014
+#define CC_RB_REDUNDANCY__FAILED_RB0_MASK 0x00000f00L
+#define CC_RB_REDUNDANCY__FAILED_RB0__SHIFT 0x00000008
+#define CC_RB_REDUNDANCY__FAILED_RB1_MASK 0x000f0000L
+#define CC_RB_REDUNDANCY__FAILED_RB1__SHIFT 0x00000010
+#define CC_SQC_BANK_DISABLE__SQC0_BANK_DISABLE_MASK 0x000f0000L
+#define CC_SQC_BANK_DISABLE__SQC0_BANK_DISABLE__SHIFT 0x00000010
+#define CC_SQC_BANK_DISABLE__SQC1_BANK_DISABLE_MASK 0x00f00000L
+#define CC_SQC_BANK_DISABLE__SQC1_BANK_DISABLE__SHIFT 0x00000014
+#define CC_SQC_BANK_DISABLE__SQC2_BANK_DISABLE_MASK 0x0f000000L
+#define CC_SQC_BANK_DISABLE__SQC2_BANK_DISABLE__SHIFT 0x00000018
+#define CC_SQC_BANK_DISABLE__SQC3_BANK_DISABLE_MASK 0xf0000000L
+#define CC_SQC_BANK_DISABLE__SQC3_BANK_DISABLE__SHIFT 0x0000001c
+#define CGTS_RD_CTRL_REG__REG_MUX_SEL_MASK 0x00001f00L
+#define CGTS_RD_CTRL_REG__REG_MUX_SEL__SHIFT 0x00000008
+#define CGTS_RD_CTRL_REG__ROW_MUX_SEL_MASK 0x0000001fL
+#define CGTS_RD_CTRL_REG__ROW_MUX_SEL__SHIFT 0x00000000
+#define CGTS_RD_REG__READ_DATA_MASK 0x00003fffL
+#define CGTS_RD_REG__READ_DATA__SHIFT 0x00000000
+#define CGTS_SM_CTRL_REG__BASE_MODE_MASK 0x00010000L
+#define CGTS_SM_CTRL_REG__BASE_MODE__SHIFT 0x00000010
+#define CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK 0x00400000L
+#define CGTS_SM_CTRL_REG__LS_OVERRIDE__SHIFT 0x00000016
+#define CGTS_SM_CTRL_REG__MGCG_ENABLED_MASK 0x00001000L
+#define CGTS_SM_CTRL_REG__MGCG_ENABLED__SHIFT 0x0000000c
+#define CGTS_SM_CTRL_REG__OFF_SEQ_DELAY_MASK 0x00000ff0L
+#define CGTS_SM_CTRL_REG__OFF_SEQ_DELAY__SHIFT 0x00000004
+#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK 0x00800000L
+#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN__SHIFT 0x00000017
+#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_MASK 0xff000000L
+#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT 0x00000018
+#define CGTS_SM_CTRL_REG__ON_SEQ_DELAY_MASK 0x0000000fL
+#define CGTS_SM_CTRL_REG__ON_SEQ_DELAY__SHIFT 0x00000000
+#define CGTS_SM_CTRL_REG__OVERRIDE_MASK 0x00200000L
+#define CGTS_SM_CTRL_REG__OVERRIDE__SHIFT 0x00000015
+#define CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK 0x00100000L
+#define CGTS_SM_CTRL_REG__SM_MODE_ENABLE__SHIFT 0x00000014
+#define CGTS_SM_CTRL_REG__SM_MODE_MASK 0x000e0000L
+#define CGTS_SM_CTRL_REG__SM_MODE__SHIFT 0x00000011
+#define CGTS_TCC_DISABLE__TCC_DISABLE_MASK 0xffff0000L
+#define CGTS_TCC_DISABLE__TCC_DISABLE__SHIFT 0x00000010
+#define CGTS_USER_TCC_DISABLE__TCC_DISABLE_MASK 0xffff0000L
+#define CGTS_USER_TCC_DISABLE__TCC_DISABLE__SHIFT 0x00000010
+#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE_MASK 0x40000000L
+#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE__SHIFT 0x0000001e
+#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE_MASK 0x20000000L
+#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE__SHIFT 0x0000001d
+#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE_MASK 0x10000000L
+#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE__SHIFT 0x0000001c
+#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE_MASK 0x08000000L
+#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE__SHIFT 0x0000001b
+#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE_MASK 0x04000000L
+#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE__SHIFT 0x0000001a
+#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE_MASK 0x02000000L
+#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE__SHIFT 0x00000019
+#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE_MASK 0x01000000L
+#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE__SHIFT 0x00000018
+#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_BCI_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_BCI_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L
+#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE__SHIFT 0x0000001f
+#define CGTT_BCI_CLK_CTRL__RESERVED_MASK 0x00fff000L
+#define CGTT_BCI_CLK_CTRL__RESERVED__SHIFT 0x0000000c
+#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_CP_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_CP_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x0000001e
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x0000001f
+#define CGTT_GDS_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_GDS_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_GDS_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_GDS_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE_MASK 0x20000000L
+#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x0000001d
+#define CGTT_IA_CLK_CTRL__DBG_ENABLE_MASK 0x04000000L
+#define CGTT_IA_CLK_CTRL__DBG_ENABLE__SHIFT 0x0000001a
+#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_IA_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_IA_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_IA_CLK_CTRL__PERF_ENABLE_MASK 0x02000000L
+#define CGTT_IA_CLK_CTRL__PERF_ENABLE__SHIFT 0x00000019
+#define CGTT_IA_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L
+#define CGTT_IA_CLK_CTRL__REG_OVERRIDE__SHIFT 0x0000001f
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE_MASK 0x40000000L
+#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE__SHIFT 0x0000001e
+#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_PA_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_PA_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE_MASK 0x80000000L
+#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE__SHIFT 0x0000001f
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE_MASK 0x20000000L
+#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE__SHIFT 0x0000001d
+#define CGTT_PC_CLK_CTRL__BACK_CLK_ON_OVERRIDE_MASK 0x02000000L
+#define CGTT_PC_CLK_CTRL__BACK_CLK_ON_OVERRIDE__SHIFT 0x00000019
+#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE_MASK 0x40000000L
+#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE__SHIFT 0x0000001e
+#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE_MASK 0x20000000L
+#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE__SHIFT 0x0000001d
+#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE_MASK 0x10000000L
+#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE__SHIFT 0x0000001c
+#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE_MASK 0x08000000L
+#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE__SHIFT 0x0000001b
+#define CGTT_PC_CLK_CTRL__FRONT_CLK_ON_OVERRIDE_MASK 0x04000000L
+#define CGTT_PC_CLK_CTRL__FRONT_CLK_ON_OVERRIDE__SHIFT 0x0000001a
+#define CGTT_PC_CLK_CTRL__GRP5_CG_OFF_HYST_MASK 0x00fc0000L
+#define CGTT_PC_CLK_CTRL__GRP5_CG_OFF_HYST__SHIFT 0x00000012
+#define CGTT_PC_CLK_CTRL__GRP5_CG_OVERRIDE_MASK 0x01000000L
+#define CGTT_PC_CLK_CTRL__GRP5_CG_OVERRIDE__SHIFT 0x00000018
+#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_PC_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_PC_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_PC_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L
+#define CGTT_PC_CLK_CTRL__REG_OVERRIDE__SHIFT 0x0000001f
+#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_RLC_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_RLC_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000L
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x0000001e
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000L
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x0000001f
+#define CGTT_SC_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_SC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_SC_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_SC_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CGTT_SPI_CLK_CTRL__ALL_CLK_ON_OVERRIDE_MASK 0x04000000L
+#define CGTT_SPI_CLK_CTRL__ALL_CLK_ON_OVERRIDE__SHIFT 0x0000001a
+#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE_MASK 0x40000000L
+#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE__SHIFT 0x0000001e
+#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE_MASK 0x20000000L
+#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE__SHIFT 0x0000001d
+#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE_MASK 0x10000000L
+#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE__SHIFT 0x0000001c
+#define CGTT_SPI_CLK_CTRL__GRP3_OVERRIDE_MASK 0x08000000L
+#define CGTT_SPI_CLK_CTRL__GRP3_OVERRIDE__SHIFT 0x0000001b
+#define CGTT_SPI_CLK_CTRL__GRP5_CG_OFF_HYST_MASK 0x00fc0000L
+#define CGTT_SPI_CLK_CTRL__GRP5_CG_OFF_HYST__SHIFT 0x00000012
+#define CGTT_SPI_CLK_CTRL__GRP5_CG_OVERRIDE_MASK 0x01000000L
+#define CGTT_SPI_CLK_CTRL__GRP5_CG_OVERRIDE__SHIFT 0x00000018
+#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_SPI_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_SPI_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L
+#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE__SHIFT 0x0000001f
+#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000L
+#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x0000001e
+#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_SQ_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_SQ_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L
+#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE__SHIFT 0x0000001f
+#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000L
+#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x0000001e
+#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_SQG_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_SQG_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L
+#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE__SHIFT 0x0000001f
+#define CGTT_SX_CLK_CTRL0__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_SX_CLK_CTRL0__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_SX_CLK_CTRL0__ON_DELAY_MASK 0x0000000fL
+#define CGTT_SX_CLK_CTRL0__ON_DELAY__SHIFT 0x00000000
+#define CGTT_SX_CLK_CTRL0__RESERVED_MASK 0x00fff000L
+#define CGTT_SX_CLK_CTRL0__RESERVED__SHIFT 0x0000000c
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE0_MASK 0x80000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE1_MASK 0x40000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE2_MASK 0x20000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE5_MASK 0x04000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE6_MASK 0x02000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CGTT_SX_CLK_CTRL1__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_SX_CLK_CTRL1__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_SX_CLK_CTRL1__ON_DELAY_MASK 0x0000000fL
+#define CGTT_SX_CLK_CTRL1__ON_DELAY__SHIFT 0x00000000
+#define CGTT_SX_CLK_CTRL1__RESERVED_MASK 0x00fff000L
+#define CGTT_SX_CLK_CTRL1__RESERVED__SHIFT 0x0000000c
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE0_MASK 0x80000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE1_MASK 0x40000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE2_MASK 0x20000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE5_MASK 0x04000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE6_MASK 0x02000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CGTT_SX_CLK_CTRL2__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_SX_CLK_CTRL2__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_SX_CLK_CTRL2__ON_DELAY_MASK 0x0000000fL
+#define CGTT_SX_CLK_CTRL2__ON_DELAY__SHIFT 0x00000000
+#define CGTT_SX_CLK_CTRL2__RESERVED_MASK 0x00fff000L
+#define CGTT_SX_CLK_CTRL2__RESERVED__SHIFT 0x0000000c
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE0_MASK 0x80000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE1_MASK 0x40000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE2_MASK 0x20000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE5_MASK 0x04000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE6_MASK 0x02000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CGTT_SX_CLK_CTRL3__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_SX_CLK_CTRL3__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_SX_CLK_CTRL3__ON_DELAY_MASK 0x0000000fL
+#define CGTT_SX_CLK_CTRL3__ON_DELAY__SHIFT 0x00000000
+#define CGTT_SX_CLK_CTRL3__RESERVED_MASK 0x00fff000L
+#define CGTT_SX_CLK_CTRL3__RESERVED__SHIFT 0x0000000c
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE0_MASK 0x80000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE1_MASK 0x40000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE2_MASK 0x20000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE5_MASK 0x04000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE6_MASK 0x02000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CGTT_SX_CLK_CTRL4__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_SX_CLK_CTRL4__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_SX_CLK_CTRL4__ON_DELAY_MASK 0x0000000fL
+#define CGTT_SX_CLK_CTRL4__ON_DELAY__SHIFT 0x00000000
+#define CGTT_SX_CLK_CTRL4__RESERVED_MASK 0x00fff000L
+#define CGTT_SX_CLK_CTRL4__RESERVED__SHIFT 0x0000000c
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE0_MASK 0x80000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE1_MASK 0x40000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE2_MASK 0x20000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE5_MASK 0x04000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE6_MASK 0x02000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CGTT_TCI_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_TCI_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_TCI_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_TCI_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CGTT_TCP_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_TCP_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_TCP_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_TCP_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000L
+#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x0000001e
+#define CGTT_VGT_CLK_CTRL__DBG_ENABLE_MASK 0x04000000L
+#define CGTT_VGT_CLK_CTRL__DBG_ENABLE__SHIFT 0x0000001a
+#define CGTT_VGT_CLK_CTRL__GS_OVERRIDE_MASK 0x20000000L
+#define CGTT_VGT_CLK_CTRL__GS_OVERRIDE__SHIFT 0x0000001d
+#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define CGTT_VGT_CLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define CGTT_VGT_CLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define CGTT_VGT_CLK_CTRL__PERF_ENABLE_MASK 0x02000000L
+#define CGTT_VGT_CLK_CTRL__PERF_ENABLE__SHIFT 0x00000019
+#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L
+#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE__SHIFT 0x0000001f
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L
+#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define CLIPPER_DEBUG_REG00__ALWAYS_ZERO_MASK 0x000000ffL
+#define CLIPPER_DEBUG_REG00__ALWAYS_ZERO__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG00__ccgen_to_clipcc_fifo_empty_MASK 0x08000000L
+#define CLIPPER_DEBUG_REG00__ccgen_to_clipcc_fifo_empty__SHIFT 0x0000001b
+#define CLIPPER_DEBUG_REG00__ccgen_to_clipcc_fifo_full_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG00__ccgen_to_clipcc_fifo_full__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG00__clipcode_fifo_fifo_empty_MASK 0x00200000L
+#define CLIPPER_DEBUG_REG00__clipcode_fifo_fifo_empty__SHIFT 0x00000015
+#define CLIPPER_DEBUG_REG00__clipcode_fifo_full_MASK 0x00400000L
+#define CLIPPER_DEBUG_REG00__clipcode_fifo_full__SHIFT 0x00000016
+#define CLIPPER_DEBUG_REG00__clip_ga_bc_fifo_write_MASK 0x00000100L
+#define CLIPPER_DEBUG_REG00__clip_ga_bc_fifo_write__SHIFT 0x00000008
+#define CLIPPER_DEBUG_REG00__clip_to_ga_fifo_full_MASK 0x00001000L
+#define CLIPPER_DEBUG_REG00__clip_to_ga_fifo_full__SHIFT 0x0000000c
+#define CLIPPER_DEBUG_REG00__clip_to_ga_fifo_write_MASK 0x00000800L
+#define CLIPPER_DEBUG_REG00__clip_to_ga_fifo_write__SHIFT 0x0000000b
+#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_empty_MASK 0x00008000L
+#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_empty__SHIFT 0x0000000f
+#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_full_MASK 0x00010000L
+#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_full__SHIFT 0x00000010
+#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_write_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_write__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG00__primic_to_clprim_fifo_empty_MASK 0x00002000L
+#define CLIPPER_DEBUG_REG00__primic_to_clprim_fifo_empty__SHIFT 0x0000000d
+#define CLIPPER_DEBUG_REG00__primic_to_clprim_fifo_full_MASK 0x00004000L
+#define CLIPPER_DEBUG_REG00__primic_to_clprim_fifo_full__SHIFT 0x0000000e
+#define CLIPPER_DEBUG_REG00__su_clip_baryc_free_MASK 0x00000600L
+#define CLIPPER_DEBUG_REG00__su_clip_baryc_free__SHIFT 0x00000009
+#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_empty_MASK 0x00020000L
+#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_empty__SHIFT 0x00000011
+#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_full_MASK 0x00040000L
+#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_full__SHIFT 0x00000012
+#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_write_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_write__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG00__vgt_to_clips_fifo_empty_MASK 0x00080000L
+#define CLIPPER_DEBUG_REG00__vgt_to_clips_fifo_empty__SHIFT 0x00000013
+#define CLIPPER_DEBUG_REG00__vgt_to_clips_fifo_full_MASK 0x00100000L
+#define CLIPPER_DEBUG_REG00__vgt_to_clips_fifo_full__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG00__vte_out_clip_fifo_fifo_empty_MASK 0x00800000L
+#define CLIPPER_DEBUG_REG00__vte_out_clip_fifo_fifo_empty__SHIFT 0x00000017
+#define CLIPPER_DEBUG_REG00__vte_out_clip_fifo_fifo_full_MASK 0x01000000L
+#define CLIPPER_DEBUG_REG00__vte_out_clip_fifo_fifo_full__SHIFT 0x00000018
+#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_empty_MASK 0x02000000L
+#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_empty__SHIFT 0x00000019
+#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_full_MASK 0x04000000L
+#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_full__SHIFT 0x0000001a
+#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_write_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_write__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG01__ALWAYS_ZERO_MASK 0x000000ffL
+#define CLIPPER_DEBUG_REG01__ALWAYS_ZERO__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG01__clip_extra_bc_valid_MASK 0x00000700L
+#define CLIPPER_DEBUG_REG01__clip_extra_bc_valid__SHIFT 0x00000008
+#define CLIPPER_DEBUG_REG01__clip_ga_bc_fifo_write_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG01__clip_ga_bc_fifo_write__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG01__clip_to_ga_fifo_write_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG01__clip_to_ga_fifo_write__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG01__clip_to_outsm_deallocate_slot_MASK 0x000e0000L
+#define CLIPPER_DEBUG_REG01__clip_to_outsm_deallocate_slot__SHIFT 0x00000011
+#define CLIPPER_DEBUG_REG01__clip_to_outsm_null_primitive_MASK 0x00100000L
+#define CLIPPER_DEBUG_REG01__clip_to_outsm_null_primitive__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG01__clip_to_outsm_vertex_deallocate_MASK 0x0001c000L
+#define CLIPPER_DEBUG_REG01__clip_to_outsm_vertex_deallocate__SHIFT 0x0000000e
+#define CLIPPER_DEBUG_REG01__clip_vert_vte_valid_MASK 0x00003800L
+#define CLIPPER_DEBUG_REG01__clip_vert_vte_valid__SHIFT 0x0000000b
+#define CLIPPER_DEBUG_REG01__vte_out_clip_fifo_fifo_advanceread_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG01__vte_out_clip_fifo_fifo_advanceread__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG01__vte_out_clip_fifo_fifo_empty_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG01__vte_out_clip_fifo_fifo_empty__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_extra_bc_valid_MASK 0x01000000L
+#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_extra_bc_valid__SHIFT 0x00000018
+#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_vertex_store_indx_MASK 0x0c000000L
+#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_vertex_store_indx__SHIFT 0x0000001a
+#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_vte_naninf_kill_MASK 0x02000000L
+#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_vte_naninf_kill__SHIFT 0x00000019
+#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_0_MASK 0x00800000L
+#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_0__SHIFT 0x00000017
+#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_1_MASK 0x00400000L
+#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_1__SHIFT 0x00000016
+#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_2_MASK 0x00200000L
+#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_2__SHIFT 0x00000015
+#define CLIPPER_DEBUG_REG02__clip_extra_bc_valid_MASK 0x00000007L
+#define CLIPPER_DEBUG_REG02__clip_extra_bc_valid__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG02__clip_ga_bc_fifo_full_MASK 0x04000000L
+#define CLIPPER_DEBUG_REG02__clip_ga_bc_fifo_full__SHIFT 0x0000001a
+#define CLIPPER_DEBUG_REG02__clip_ga_bc_fifo_write_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG02__clip_ga_bc_fifo_write__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG02__clip_to_clipga_extra_bc_coords_MASK 0x00100000L
+#define CLIPPER_DEBUG_REG02__clip_to_clipga_extra_bc_coords__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG02__clip_to_clipga_vte_naninf_kill_MASK 0x00200000L
+#define CLIPPER_DEBUG_REG02__clip_to_clipga_vte_naninf_kill__SHIFT 0x00000015
+#define CLIPPER_DEBUG_REG02__clip_to_ga_fifo_full_MASK 0x08000000L
+#define CLIPPER_DEBUG_REG02__clip_to_ga_fifo_full__SHIFT 0x0000001b
+#define CLIPPER_DEBUG_REG02__clip_to_ga_fifo_write_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG02__clip_to_ga_fifo_write__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_clipped_prim_MASK 0x01000000L
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_clipped_prim__SHIFT 0x00000018
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_clip_seq_indx_MASK 0x000000c0L
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_clip_seq_indx__SHIFT 0x00000006
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_end_of_packet_MASK 0x00400000L
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_end_of_packet__SHIFT 0x00000016
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_fifo_advanceread_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_fifo_advanceread__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_fifo_empty_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_fifo_empty__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_first_prim_of_slot_MASK 0x00800000L
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_first_prim_of_slot__SHIFT 0x00000017
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_null_primitive_MASK 0x02000000L
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_null_primitive__SHIFT 0x00000019
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_0_MASK 0x000f0000L
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_0__SHIFT 0x00000010
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_1_MASK 0x0000f000L
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_1__SHIFT 0x0000000c
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_2_MASK 0x00000f00L
+#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_2__SHIFT 0x00000008
+#define CLIPPER_DEBUG_REG02__clip_vert_vte_valid_MASK 0x00000038L
+#define CLIPPER_DEBUG_REG02__clip_vert_vte_valid__SHIFT 0x00000003
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_clip_code_or_MASK 0x00003fffL
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_clip_code_or__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_clip_primitive_MASK 0x00800000L
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_clip_primitive__SHIFT 0x00000017
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_deallocate_slot_MASK 0x07000000L
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_deallocate_slot__SHIFT 0x00000018
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_end_of_packet_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_end_of_packet__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_event_id_MASK 0x000fc000L
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_event_id__SHIFT 0x0000000e
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_event_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_event__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_first_prim_of_slot_MASK 0x08000000L
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_first_prim_of_slot__SHIFT 0x0000001b
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_null_primitive_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_null_primitive__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_state_var_indx_MASK 0x00700000L
+#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_state_var_indx__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_event_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_event__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_null_primitive_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_null_primitive__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_param_cache_indx_0_MASK 0x000007feL
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_param_cache_indx_0__SHIFT 0x00000001
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_0_MASK 0x1f800000L
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_0__SHIFT 0x00000017
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_1_MASK 0x007e0000L
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_1__SHIFT 0x00000011
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_2_MASK 0x0001f800L
+#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_2__SHIFT 0x0000000b
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_clip_code_or_MASK 0x00003fffL
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_clip_code_or__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_clip_primitive_MASK 0x00800000L
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_clip_primitive__SHIFT 0x00000017
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_deallocate_slot_MASK 0x07000000L
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_deallocate_slot__SHIFT 0x00000018
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_end_of_packet_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_end_of_packet__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_event_id_MASK 0x000fc000L
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_event_id__SHIFT 0x0000000e
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_event_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_event__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_first_prim_of_slot_MASK 0x08000000L
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_first_prim_of_slot__SHIFT 0x0000001b
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_null_primitive_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_null_primitive__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_state_var_indx_MASK 0x00700000L
+#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_state_var_indx__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_event_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_event__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_null_primitive_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_null_primitive__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_param_cache_indx_0_MASK 0x000007feL
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_param_cache_indx_0__SHIFT 0x00000001
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_0_MASK 0x1f800000L
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_0__SHIFT 0x00000017
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_1_MASK 0x007e0000L
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_1__SHIFT 0x00000011
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_2_MASK 0x0001f800L
+#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_2__SHIFT 0x0000000b
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_clip_code_or_MASK 0x00003fffL
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_clip_code_or__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_clip_primitive_MASK 0x00800000L
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_clip_primitive__SHIFT 0x00000017
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_deallocate_slot_MASK 0x07000000L
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_deallocate_slot__SHIFT 0x00000018
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_end_of_packet_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_end_of_packet__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_event_id_MASK 0x000fc000L
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_event_id__SHIFT 0x0000000e
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_event_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_event__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_first_prim_of_slot_MASK 0x08000000L
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_first_prim_of_slot__SHIFT 0x0000001b
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_null_primitive_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_null_primitive__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_state_var_indx_MASK 0x00700000L
+#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_state_var_indx__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_event_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_event__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_null_primitive_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_null_primitive__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_param_cache_indx_0_MASK 0x000007feL
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_param_cache_indx_0__SHIFT 0x00000001
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_0_MASK 0x1f800000L
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_0__SHIFT 0x00000017
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_1_MASK 0x007e0000L
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_1__SHIFT 0x00000011
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_2_MASK 0x0001f800L
+#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_2__SHIFT 0x0000000b
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_clip_code_or_MASK 0x00003fffL
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_clip_code_or__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_clip_primitive_MASK 0x00800000L
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_clip_primitive__SHIFT 0x00000017
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_deallocate_slot_MASK 0x07000000L
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_deallocate_slot__SHIFT 0x00000018
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_end_of_packet_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_end_of_packet__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_event_id_MASK 0x000fc000L
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_event_id__SHIFT 0x0000000e
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_event_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_event__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_first_prim_of_slot_MASK 0x08000000L
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_first_prim_of_slot__SHIFT 0x0000001b
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_null_primitive_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_null_primitive__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_state_var_indx_MASK 0x00700000L
+#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_state_var_indx__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_event_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_event__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_null_primitive_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_null_primitive__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_param_cache_indx_0_MASK 0x000007feL
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_param_cache_indx_0__SHIFT 0x00000001
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_0_MASK 0x1f800000L
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_0__SHIFT 0x00000017
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_1_MASK 0x007e0000L
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_1__SHIFT 0x00000011
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_2_MASK 0x0001f800L
+#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_2__SHIFT 0x0000000b
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_clip_primitive_MASK 0x00000080L
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_clip_primitive__SHIFT 0x00000007
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_clip_to_outsm_cnt_MASK 0x00f00000L
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_event_MASK 0x00000008L
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_event__SHIFT 0x00000003
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_prim_valid_MASK 0x08000000L
+#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_prim_valid__SHIFT 0x0000001b
+#define CLIPPER_DEBUG_REG11__clipsm0_inc_clip_to_clipga_clip_to_outsm_cnt_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG11__clipsm0_inc_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_clip_primitive_MASK 0x00000040L
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_clip_primitive__SHIFT 0x00000006
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_clip_to_outsm_cnt_MASK 0x000f0000L
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x00000010
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_event_MASK 0x00000004L
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_event__SHIFT 0x00000002
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_prim_valid_MASK 0x04000000L
+#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_prim_valid__SHIFT 0x0000001a
+#define CLIPPER_DEBUG_REG11__clipsm1_inc_clip_to_clipga_clip_to_outsm_cnt_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG11__clipsm1_inc_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_clip_primitive_MASK 0x00000020L
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_clip_primitive__SHIFT 0x00000005
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_clip_to_outsm_cnt_MASK 0x0000f000L
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x0000000c
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_event_MASK 0x00000002L
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_event__SHIFT 0x00000001
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_prim_valid_MASK 0x02000000L
+#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_prim_valid__SHIFT 0x00000019
+#define CLIPPER_DEBUG_REG11__clipsm2_inc_clip_to_clipga_clip_to_outsm_cnt_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG11__clipsm2_inc_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_clip_primitive_MASK 0x00000010L
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_clip_primitive__SHIFT 0x00000004
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_clip_to_outsm_cnt_MASK 0x00000f00L
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x00000008
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_event_MASK 0x00000001L
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_event__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_prim_valid_MASK 0x01000000L
+#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_prim_valid__SHIFT 0x00000018
+#define CLIPPER_DEBUG_REG11__clipsm3_inc_clip_to_clipga_clip_to_outsm_cnt_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG11__clipsm3_inc_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG12__ALWAYS_ZERO_MASK 0x000000ffL
+#define CLIPPER_DEBUG_REG12__ALWAYS_ZERO__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG12__clip_priority_available_clip_verts_MASK 0x0003e000L
+#define CLIPPER_DEBUG_REG12__clip_priority_available_clip_verts__SHIFT 0x0000000d
+#define CLIPPER_DEBUG_REG12__clip_priority_available_vte_out_clip_MASK 0x00001f00L
+#define CLIPPER_DEBUG_REG12__clip_priority_available_vte_out_clip__SHIFT 0x00000008
+#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_load_MASK 0x00c00000L
+#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_load__SHIFT 0x00000016
+#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_out_MASK 0x000c0000L
+#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_out__SHIFT 0x00000012
+#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_vert_MASK 0x00300000L
+#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_vert__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG12__clipsm0_clprim_to_clip_clip_primitive_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG12__clipsm0_clprim_to_clip_clip_primitive__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG12__clipsm0_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG12__clipsm0_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG12__clipsm1_clprim_to_clip_clip_primitive_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG12__clipsm1_clprim_to_clip_clip_primitive__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG12__clipsm1_clprim_to_clip_prim_valid_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG12__clipsm1_clprim_to_clip_prim_valid__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG12__clipsm2_clprim_to_clip_clip_primitive_MASK 0x04000000L
+#define CLIPPER_DEBUG_REG12__clipsm2_clprim_to_clip_clip_primitive__SHIFT 0x0000001a
+#define CLIPPER_DEBUG_REG12__clipsm2_clprim_to_clip_prim_valid_MASK 0x08000000L
+#define CLIPPER_DEBUG_REG12__clipsm2_clprim_to_clip_prim_valid__SHIFT 0x0000001b
+#define CLIPPER_DEBUG_REG12__clipsm3_clprim_to_clip_clip_primitive_MASK 0x01000000L
+#define CLIPPER_DEBUG_REG12__clipsm3_clprim_to_clip_clip_primitive__SHIFT 0x00000018
+#define CLIPPER_DEBUG_REG12__clipsm3_clprim_to_clip_prim_valid_MASK 0x02000000L
+#define CLIPPER_DEBUG_REG12__clipsm3_clprim_to_clip_prim_valid__SHIFT 0x00000019
+#define CLIPPER_DEBUG_REG13__ccgen_to_clipcc_fifo_empty_MASK 0x00010000L
+#define CLIPPER_DEBUG_REG13__ccgen_to_clipcc_fifo_empty__SHIFT 0x00000010
+#define CLIPPER_DEBUG_REG13__clipcc_vertex_store_indx_MASK 0x00003000L
+#define CLIPPER_DEBUG_REG13__clipcc_vertex_store_indx__SHIFT 0x0000000c
+#define CLIPPER_DEBUG_REG13__clipcode_fifo_fifo_empty_MASK 0x00008000L
+#define CLIPPER_DEBUG_REG13__clipcode_fifo_fifo_empty__SHIFT 0x0000000f
+#define CLIPPER_DEBUG_REG13__clip_priority_seq_indx_out_cnt_MASK 0x001e0000L
+#define CLIPPER_DEBUG_REG13__clip_priority_seq_indx_out_cnt__SHIFT 0x00000011
+#define CLIPPER_DEBUG_REG13__clprim_clip_primitive_MASK 0x00000020L
+#define CLIPPER_DEBUG_REG13__clprim_clip_primitive__SHIFT 0x00000005
+#define CLIPPER_DEBUG_REG13__clprim_cull_primitive_MASK 0x00000040L
+#define CLIPPER_DEBUG_REG13__clprim_cull_primitive__SHIFT 0x00000006
+#define CLIPPER_DEBUG_REG13__clprim_in_back_state_var_indx_MASK 0x00000007L
+#define CLIPPER_DEBUG_REG13__clprim_in_back_state_var_indx__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_advanceread_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_advanceread__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_contents_MASK 0x1f000000L
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_contents__SHIFT 0x00000018
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_full_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_full__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_write_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_write__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG13__outsm_clr_rd_clipsm_wait_MASK 0x00800000L
+#define CLIPPER_DEBUG_REG13__outsm_clr_rd_clipsm_wait__SHIFT 0x00000017
+#define CLIPPER_DEBUG_REG13__outsm_clr_rd_orig_vertices_MASK 0x00600000L
+#define CLIPPER_DEBUG_REG13__outsm_clr_rd_orig_vertices__SHIFT 0x00000015
+#define CLIPPER_DEBUG_REG13__point_clip_candidate_MASK 0x00000008L
+#define CLIPPER_DEBUG_REG13__point_clip_candidate__SHIFT 0x00000003
+#define CLIPPER_DEBUG_REG13__prim_back_valid_MASK 0x00000080L
+#define CLIPPER_DEBUG_REG13__prim_back_valid__SHIFT 0x00000007
+#define CLIPPER_DEBUG_REG13__prim_nan_kill_MASK 0x00000010L
+#define CLIPPER_DEBUG_REG13__prim_nan_kill__SHIFT 0x00000004
+#define CLIPPER_DEBUG_REG13__vertval_bits_vertex_cc_next_valid_MASK 0x00000f00L
+#define CLIPPER_DEBUG_REG13__vertval_bits_vertex_cc_next_valid__SHIFT 0x00000008
+#define CLIPPER_DEBUG_REG13__vte_out_orig_fifo_fifo_empty_MASK 0x00004000L
+#define CLIPPER_DEBUG_REG13__vte_out_orig_fifo_fifo_empty__SHIFT 0x0000000e
+#define CLIPPER_DEBUG_REG14__clprim_in_back_deallocate_slot_MASK 0x00e00000L
+#define CLIPPER_DEBUG_REG14__clprim_in_back_deallocate_slot__SHIFT 0x00000015
+#define CLIPPER_DEBUG_REG14__clprim_in_back_end_of_packet_MASK 0x00080000L
+#define CLIPPER_DEBUG_REG14__clprim_in_back_end_of_packet__SHIFT 0x00000013
+#define CLIPPER_DEBUG_REG14__clprim_in_back_event_id_MASK 0x3f000000L
+#define CLIPPER_DEBUG_REG14__clprim_in_back_event_id__SHIFT 0x00000018
+#define CLIPPER_DEBUG_REG14__clprim_in_back_event_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG14__clprim_in_back_event__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG14__clprim_in_back_first_prim_of_slot_MASK 0x00100000L
+#define CLIPPER_DEBUG_REG14__clprim_in_back_first_prim_of_slot__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_0_MASK 0x0003f000L
+#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_0__SHIFT 0x0000000c
+#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_1_MASK 0x00000fc0L
+#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_1__SHIFT 0x00000006
+#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_2_MASK 0x0000003fL
+#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_2__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG14__outputclprimtoclip_null_primitive_MASK 0x00040000L
+#define CLIPPER_DEBUG_REG14__outputclprimtoclip_null_primitive__SHIFT 0x00000012
+#define CLIPPER_DEBUG_REG14__prim_back_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG14__prim_back_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_0_MASK 0x7c000000L
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_0__SHIFT 0x0000001a
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_1_MASK 0x03e00000L
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_1__SHIFT 0x00000015
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_2_MASK 0x001f0000L
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_2__SHIFT 0x00000010
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG15__primic_to_clprim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG15__vertval_bits_vertex_vertex_store_msb_MASK 0x0000ffffL
+#define CLIPPER_DEBUG_REG15__vertval_bits_vertex_vertex_store_msb__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG16__sm0_clip_to_clipga_clip_to_outsm_cnt_eq0_MASK 0x08000000L
+#define CLIPPER_DEBUG_REG16__sm0_clip_to_clipga_clip_to_outsm_cnt_eq0__SHIFT 0x0000001b
+#define CLIPPER_DEBUG_REG16__sm0_clip_to_outsm_fifo_full_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG16__sm0_clip_to_outsm_fifo_full__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG16__sm0_clip_vert_cnt_MASK 0x00001f00L
+#define CLIPPER_DEBUG_REG16__sm0_clip_vert_cnt__SHIFT 0x00000008
+#define CLIPPER_DEBUG_REG16__sm0_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG16__sm0_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG16__sm0_current_state_MASK 0x07f00000L
+#define CLIPPER_DEBUG_REG16__sm0_current_state__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG16__sm0_highest_priority_seq_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG16__sm0_highest_priority_seq__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG16__sm0_inv_to_clip_data_valid_0_MASK 0x00080000L
+#define CLIPPER_DEBUG_REG16__sm0_inv_to_clip_data_valid_0__SHIFT 0x00000013
+#define CLIPPER_DEBUG_REG16__sm0_inv_to_clip_data_valid_1_MASK 0x00040000L
+#define CLIPPER_DEBUG_REG16__sm0_inv_to_clip_data_valid_1__SHIFT 0x00000012
+#define CLIPPER_DEBUG_REG16__sm0_outputcliptoclipga_0_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG16__sm0_outputcliptoclipga_0__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG16__sm0_prim_end_state_MASK 0x0000007fL
+#define CLIPPER_DEBUG_REG16__sm0_prim_end_state__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG16__sm0_ps_expand_MASK 0x00000080L
+#define CLIPPER_DEBUG_REG16__sm0_ps_expand__SHIFT 0x00000007
+#define CLIPPER_DEBUG_REG16__sm0_vertex_clip_cnt_MASK 0x0003e000L
+#define CLIPPER_DEBUG_REG16__sm0_vertex_clip_cnt__SHIFT 0x0000000d
+#define CLIPPER_DEBUG_REG17__sm1_clip_to_clipga_clip_to_outsm_cnt_eq0_MASK 0x08000000L
+#define CLIPPER_DEBUG_REG17__sm1_clip_to_clipga_clip_to_outsm_cnt_eq0__SHIFT 0x0000001b
+#define CLIPPER_DEBUG_REG17__sm1_clip_to_outsm_fifo_full_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG17__sm1_clip_to_outsm_fifo_full__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG17__sm1_clip_vert_cnt_MASK 0x00001f00L
+#define CLIPPER_DEBUG_REG17__sm1_clip_vert_cnt__SHIFT 0x00000008
+#define CLIPPER_DEBUG_REG17__sm1_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG17__sm1_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG17__sm1_current_state_MASK 0x07f00000L
+#define CLIPPER_DEBUG_REG17__sm1_current_state__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG17__sm1_highest_priority_seq_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG17__sm1_highest_priority_seq__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG17__sm1_inv_to_clip_data_valid_0_MASK 0x00080000L
+#define CLIPPER_DEBUG_REG17__sm1_inv_to_clip_data_valid_0__SHIFT 0x00000013
+#define CLIPPER_DEBUG_REG17__sm1_inv_to_clip_data_valid_1_MASK 0x00040000L
+#define CLIPPER_DEBUG_REG17__sm1_inv_to_clip_data_valid_1__SHIFT 0x00000012
+#define CLIPPER_DEBUG_REG17__sm1_outputcliptoclipga_0_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG17__sm1_outputcliptoclipga_0__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG17__sm1_prim_end_state_MASK 0x0000007fL
+#define CLIPPER_DEBUG_REG17__sm1_prim_end_state__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG17__sm1_ps_expand_MASK 0x00000080L
+#define CLIPPER_DEBUG_REG17__sm1_ps_expand__SHIFT 0x00000007
+#define CLIPPER_DEBUG_REG17__sm1_vertex_clip_cnt_MASK 0x0003e000L
+#define CLIPPER_DEBUG_REG17__sm1_vertex_clip_cnt__SHIFT 0x0000000d
+#define CLIPPER_DEBUG_REG18__sm2_clip_to_clipga_clip_to_outsm_cnt_eq0_MASK 0x08000000L
+#define CLIPPER_DEBUG_REG18__sm2_clip_to_clipga_clip_to_outsm_cnt_eq0__SHIFT 0x0000001b
+#define CLIPPER_DEBUG_REG18__sm2_clip_to_outsm_fifo_full_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG18__sm2_clip_to_outsm_fifo_full__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG18__sm2_clip_vert_cnt_MASK 0x00001f00L
+#define CLIPPER_DEBUG_REG18__sm2_clip_vert_cnt__SHIFT 0x00000008
+#define CLIPPER_DEBUG_REG18__sm2_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG18__sm2_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG18__sm2_current_state_MASK 0x07f00000L
+#define CLIPPER_DEBUG_REG18__sm2_current_state__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG18__sm2_highest_priority_seq_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG18__sm2_highest_priority_seq__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG18__sm2_inv_to_clip_data_valid_0_MASK 0x00080000L
+#define CLIPPER_DEBUG_REG18__sm2_inv_to_clip_data_valid_0__SHIFT 0x00000013
+#define CLIPPER_DEBUG_REG18__sm2_inv_to_clip_data_valid_1_MASK 0x00040000L
+#define CLIPPER_DEBUG_REG18__sm2_inv_to_clip_data_valid_1__SHIFT 0x00000012
+#define CLIPPER_DEBUG_REG18__sm2_outputcliptoclipga_0_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG18__sm2_outputcliptoclipga_0__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG18__sm2_prim_end_state_MASK 0x0000007fL
+#define CLIPPER_DEBUG_REG18__sm2_prim_end_state__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG18__sm2_ps_expand_MASK 0x00000080L
+#define CLIPPER_DEBUG_REG18__sm2_ps_expand__SHIFT 0x00000007
+#define CLIPPER_DEBUG_REG18__sm2_vertex_clip_cnt_MASK 0x0003e000L
+#define CLIPPER_DEBUG_REG18__sm2_vertex_clip_cnt__SHIFT 0x0000000d
+#define CLIPPER_DEBUG_REG19__sm3_clip_to_clipga_clip_to_outsm_cnt_eq0_MASK 0x08000000L
+#define CLIPPER_DEBUG_REG19__sm3_clip_to_clipga_clip_to_outsm_cnt_eq0__SHIFT 0x0000001b
+#define CLIPPER_DEBUG_REG19__sm3_clip_to_outsm_fifo_full_MASK 0x10000000L
+#define CLIPPER_DEBUG_REG19__sm3_clip_to_outsm_fifo_full__SHIFT 0x0000001c
+#define CLIPPER_DEBUG_REG19__sm3_clip_vert_cnt_MASK 0x00001f00L
+#define CLIPPER_DEBUG_REG19__sm3_clip_vert_cnt__SHIFT 0x00000008
+#define CLIPPER_DEBUG_REG19__sm3_clprim_to_clip_prim_valid_MASK 0x80000000L
+#define CLIPPER_DEBUG_REG19__sm3_clprim_to_clip_prim_valid__SHIFT 0x0000001f
+#define CLIPPER_DEBUG_REG19__sm3_current_state_MASK 0x07f00000L
+#define CLIPPER_DEBUG_REG19__sm3_current_state__SHIFT 0x00000014
+#define CLIPPER_DEBUG_REG19__sm3_highest_priority_seq_MASK 0x20000000L
+#define CLIPPER_DEBUG_REG19__sm3_highest_priority_seq__SHIFT 0x0000001d
+#define CLIPPER_DEBUG_REG19__sm3_inv_to_clip_data_valid_0_MASK 0x00080000L
+#define CLIPPER_DEBUG_REG19__sm3_inv_to_clip_data_valid_0__SHIFT 0x00000013
+#define CLIPPER_DEBUG_REG19__sm3_inv_to_clip_data_valid_1_MASK 0x00040000L
+#define CLIPPER_DEBUG_REG19__sm3_inv_to_clip_data_valid_1__SHIFT 0x00000012
+#define CLIPPER_DEBUG_REG19__sm3_outputcliptoclipga_0_MASK 0x40000000L
+#define CLIPPER_DEBUG_REG19__sm3_outputcliptoclipga_0__SHIFT 0x0000001e
+#define CLIPPER_DEBUG_REG19__sm3_prim_end_state_MASK 0x0000007fL
+#define CLIPPER_DEBUG_REG19__sm3_prim_end_state__SHIFT 0x00000000
+#define CLIPPER_DEBUG_REG19__sm3_ps_expand_MASK 0x00000080L
+#define CLIPPER_DEBUG_REG19__sm3_ps_expand__SHIFT 0x00000007
+#define CLIPPER_DEBUG_REG19__sm3_vertex_clip_cnt_MASK 0x0003e000L
+#define CLIPPER_DEBUG_REG19__sm3_vertex_clip_cnt__SHIFT 0x0000000d
+#define COHER_DEST_BASE_0__DEST_BASE_256B_MASK 0xffffffffL
+#define COHER_DEST_BASE_0__DEST_BASE_256B__SHIFT 0x00000000
+#define COHER_DEST_BASE_1__DEST_BASE_256B_MASK 0xffffffffL
+#define COHER_DEST_BASE_1__DEST_BASE_256B__SHIFT 0x00000000
+#define COHER_DEST_BASE_2__DEST_BASE_256B_MASK 0xffffffffL
+#define COHER_DEST_BASE_2__DEST_BASE_256B__SHIFT 0x00000000
+#define COHER_DEST_BASE_3__DEST_BASE_256B_MASK 0xffffffffL
+#define COHER_DEST_BASE_3__DEST_BASE_256B__SHIFT 0x00000000
+#define COMPUTE_DIM_X__SIZE_MASK 0xffffffffL
+#define COMPUTE_DIM_X__SIZE__SHIFT 0x00000000
+#define COMPUTE_DIM_Y__SIZE_MASK 0xffffffffL
+#define COMPUTE_DIM_Y__SIZE__SHIFT 0x00000000
+#define COMPUTE_DIM_Z__SIZE_MASK 0xffffffffL
+#define COMPUTE_DIM_Z__SIZE__SHIFT 0x00000000
+#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN_MASK 0x00000001L
+#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN__SHIFT 0x00000000
+#define COMPUTE_DISPATCH_INITIATOR__DATA_ATC_MASK 0x00001000L
+#define COMPUTE_DISPATCH_INITIATOR__DATA_ATC__SHIFT 0x0000000c
+#define COMPUTE_DISPATCH_INITIATOR__DISPATCH_CACHE_CNTL_MASK 0x00000380L
+#define COMPUTE_DISPATCH_INITIATOR__DISPATCH_CACHE_CNTL__SHIFT 0x00000007
+#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000_MASK 0x00000004L
+#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000__SHIFT 0x00000002
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL_MASK 0x00000008L
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL__SHIFT 0x00000003
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE_MASK 0x00000010L
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE__SHIFT 0x00000004
+#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE_MASK 0x00000040L
+#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE__SHIFT 0x00000006
+#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN_MASK 0x00000002L
+#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN__SHIFT 0x00000001
+#define COMPUTE_DISPATCH_INITIATOR__RESTORE_MASK 0x00004000L
+#define COMPUTE_DISPATCH_INITIATOR__RESTORE__SHIFT 0x0000000e
+#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL_MASK 0x00000400L
+#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL__SHIFT 0x0000000a
+#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS_MASK 0x00000020L
+#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS__SHIFT 0x00000005
+#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL_MASK 0x00000800L
+#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL__SHIFT 0x0000000b
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL_MASK 0x0000ffffL
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL__SHIFT 0x00000000
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL_MASK 0xffff0000L
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL__SHIFT 0x00000010
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL_MASK 0x0000ffffL
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL__SHIFT 0x00000000
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL_MASK 0xffff0000L
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL__SHIFT 0x00000010
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL_MASK 0x0000ffffL
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL__SHIFT 0x00000000
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL_MASK 0xffff0000L
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL__SHIFT 0x00000010
+#define COMPUTE_PGM_HI__DATA_MASK 0x000000ffL
+#define COMPUTE_PGM_HI__DATA__SHIFT 0x00000000
+#define COMPUTE_PGM_HI__INST_ATC_MASK 0x00000100L
+#define COMPUTE_PGM_HI__INST_ATC__SHIFT 0x00000008
+#define COMPUTE_PGM_LO__DATA_MASK 0xffffffffL
+#define COMPUTE_PGM_LO__DATA__SHIFT 0x00000000
+#define COMPUTE_PGM_RSRC1__BULKY_MASK 0x01000000L
+#define COMPUTE_PGM_RSRC1__BULKY__SHIFT 0x00000018
+#define COMPUTE_PGM_RSRC1__CDBG_USER_MASK 0x02000000L
+#define COMPUTE_PGM_RSRC1__CDBG_USER__SHIFT 0x00000019
+#define COMPUTE_PGM_RSRC1__DEBUG_MODE_MASK 0x00400000L
+#define COMPUTE_PGM_RSRC1__DEBUG_MODE__SHIFT 0x00000016
+#define COMPUTE_PGM_RSRC1__DX10_CLAMP_MASK 0x00200000L
+#define COMPUTE_PGM_RSRC1__DX10_CLAMP__SHIFT 0x00000015
+#define COMPUTE_PGM_RSRC1__FLOAT_MODE_MASK 0x000ff000L
+#define COMPUTE_PGM_RSRC1__FLOAT_MODE__SHIFT 0x0000000c
+#define COMPUTE_PGM_RSRC1__IEEE_MODE_MASK 0x00800000L
+#define COMPUTE_PGM_RSRC1__IEEE_MODE__SHIFT 0x00000017
+#define COMPUTE_PGM_RSRC1__PRIORITY_MASK 0x00000c00L
+#define COMPUTE_PGM_RSRC1__PRIORITY__SHIFT 0x0000000a
+#define COMPUTE_PGM_RSRC1__PRIV_MASK 0x00100000L
+#define COMPUTE_PGM_RSRC1__PRIV__SHIFT 0x00000014
+#define COMPUTE_PGM_RSRC1__SGPRS_MASK 0x000003c0L
+#define COMPUTE_PGM_RSRC1__SGPRS__SHIFT 0x00000006
+#define COMPUTE_PGM_RSRC1__VGPRS_MASK 0x0000003fL
+#define COMPUTE_PGM_RSRC1__VGPRS__SHIFT 0x00000000
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MASK 0x7f000000L
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB_MASK 0x00006000L
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB__SHIFT 0x0000000d
+#define COMPUTE_PGM_RSRC2__EXCP_EN__SHIFT 0x00000018
+#define COMPUTE_PGM_RSRC2__LDS_SIZE_MASK 0x00ff8000L
+#define COMPUTE_PGM_RSRC2__LDS_SIZE__SHIFT 0x0000000f
+#define COMPUTE_PGM_RSRC2__SCRATCH_EN_MASK 0x00000001L
+#define COMPUTE_PGM_RSRC2__SCRATCH_EN__SHIFT 0x00000000
+#define COMPUTE_PGM_RSRC2__TGID_X_EN_MASK 0x00000080L
+#define COMPUTE_PGM_RSRC2__TGID_X_EN__SHIFT 0x00000007
+#define COMPUTE_PGM_RSRC2__TGID_Y_EN_MASK 0x00000100L
+#define COMPUTE_PGM_RSRC2__TGID_Y_EN__SHIFT 0x00000008
+#define COMPUTE_PGM_RSRC2__TGID_Z_EN_MASK 0x00000200L
+#define COMPUTE_PGM_RSRC2__TGID_Z_EN__SHIFT 0x00000009
+#define COMPUTE_PGM_RSRC2__TG_SIZE_EN_MASK 0x00000400L
+#define COMPUTE_PGM_RSRC2__TG_SIZE_EN__SHIFT 0x0000000a
+#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT_MASK 0x00001800L
+#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT__SHIFT 0x0000000b
+#define COMPUTE_PGM_RSRC2__TRAP_PRESENT_MASK 0x00000040L
+#define COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT 0x00000006
+#define COMPUTE_PGM_RSRC2__USER_SGPR_MASK 0x0000003eL
+#define COMPUTE_PGM_RSRC2__USER_SGPR__SHIFT 0x00000001
+#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT_MASK 0x07000000L
+#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT__SHIFT 0x00000018
+#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST_MASK 0x00800000L
+#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST__SHIFT 0x00000017
+#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD_MASK 0x003f0000L
+#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD__SHIFT 0x00000010
+#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL_MASK 0x00400000L
+#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL__SHIFT 0x00000016
+#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU_MASK 0x0000f000L
+#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU__SHIFT 0x0000000c
+#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH_MASK 0x0000003fL
+#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH__SHIFT 0x00000000
+#define COMPUTE_START_X__START_MASK 0xffffffffL
+#define COMPUTE_START_X__START__SHIFT 0x00000000
+#define COMPUTE_START_Y__START_MASK 0xffffffffL
+#define COMPUTE_START_Y__START__SHIFT 0x00000000
+#define COMPUTE_START_Z__START_MASK 0xffffffffL
+#define COMPUTE_START_Z__START__SHIFT 0x00000000
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH0_CU_EN_MASK 0x0000ffffL
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH0_CU_EN__SHIFT 0x00000000
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH1_CU_EN_MASK 0xffff0000L
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH1_CU_EN__SHIFT 0x00000010
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH0_CU_EN_MASK 0x0000ffffL
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH0_CU_EN__SHIFT 0x00000000
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH1_CU_EN_MASK 0xffff0000L
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH1_CU_EN__SHIFT 0x00000010
+#define COMPUTE_TBA_HI__DATA_MASK 0x000000ffL
+#define COMPUTE_TBA_HI__DATA__SHIFT 0x00000000
+#define COMPUTE_TBA_LO__DATA_MASK 0xffffffffL
+#define COMPUTE_TBA_LO__DATA__SHIFT 0x00000000
+#define COMPUTE_TMA_HI__DATA_MASK 0x000000ffL
+#define COMPUTE_TMA_HI__DATA__SHIFT 0x00000000
+#define COMPUTE_TMA_LO__DATA_MASK 0xffffffffL
+#define COMPUTE_TMA_LO__DATA__SHIFT 0x00000000
+#define COMPUTE_TMPRING_SIZE__WAVESIZE_MASK 0x01fff000L
+#define COMPUTE_TMPRING_SIZE__WAVESIZE__SHIFT 0x0000000c
+#define COMPUTE_TMPRING_SIZE__WAVES_MASK 0x00000fffL
+#define COMPUTE_TMPRING_SIZE__WAVES__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_0__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_0__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_10__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_10__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_11__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_11__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_12__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_12__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_13__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_13__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_14__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_14__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_15__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_15__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_1__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_1__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_2__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_2__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_3__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_3__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_4__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_4__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_5__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_5__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_6__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_6__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_7__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_7__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_8__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_8__DATA__SHIFT 0x00000000
+#define COMPUTE_USER_DATA_9__DATA_MASK 0xffffffffL
+#define COMPUTE_USER_DATA_9__DATA__SHIFT 0x00000000
+#define COMPUTE_VMID__DATA_MASK 0x0000000fL
+#define COMPUTE_VMID__DATA__SHIFT 0x00000000
+#define CP_APPEND_ADDR_HI__COMMAND_MASK 0xe0000000L
+#define CP_APPEND_ADDR_HI__COMMAND__SHIFT 0x0000001d
+#define CP_APPEND_ADDR_HI__CS_PS_SEL_MASK 0x00030000L
+#define CP_APPEND_ADDR_HI__CS_PS_SEL__SHIFT 0x00000010
+#define CP_APPEND_ADDR_HI__MEM_ADDR_HI_MASK 0x000000ffL
+#define CP_APPEND_ADDR_HI__MEM_ADDR_HI__SHIFT 0x00000000
+#define CP_APPEND_ADDR_LO__MEM_ADDR_LO_MASK 0xfffffffcL
+#define CP_APPEND_ADDR_LO__MEM_ADDR_LO__SHIFT 0x00000002
+#define CP_APPEND_DATA__DATA_MASK 0xffffffffL
+#define CP_APPEND_DATA__DATA__SHIFT 0x00000000
+#define CP_APPEND_LAST_CS_FENCE__LAST_FENCE_MASK 0xffffffffL
+#define CP_APPEND_LAST_CS_FENCE__LAST_FENCE__SHIFT 0x00000000
+#define CP_APPEND_LAST_PS_FENCE__LAST_FENCE_MASK 0xffffffffL
+#define CP_APPEND_LAST_PS_FENCE__LAST_FENCE__SHIFT 0x00000000
+#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK 0xffffffffL
+#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT 0x00000000
+#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK 0xffffffffL
+#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT 0x00000000
+#define CP_BUSY_STAT__CE_PARSING_PACKETS_MASK 0x00400000L
+#define CP_BUSY_STAT__CE_PARSING_PACKETS__SHIFT 0x00000016
+#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO_MASK 0x00000040L
+#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO__SHIFT 0x00000006
+#define CP_BUSY_STAT__EOP_DONE_BUSY_MASK 0x00040000L
+#define CP_BUSY_STAT__EOP_DONE_BUSY__SHIFT 0x00000012
+#define CP_BUSY_STAT__GFX_CONTEXT_BUSY_MASK 0x00008000L
+#define CP_BUSY_STAT__GFX_CONTEXT_BUSY__SHIFT 0x0000000f
+#define CP_BUSY_STAT__ME_PARSER_BUSY_MASK 0x00020000L
+#define CP_BUSY_STAT__ME_PARSER_BUSY__SHIFT 0x00000011
+#define CP_BUSY_STAT__ME_PARSING_PACKETS_MASK 0x00000100L
+#define CP_BUSY_STAT__ME_PARSING_PACKETS__SHIFT 0x00000008
+#define CP_BUSY_STAT__PFP_PARSING_PACKETS_MASK 0x00000080L
+#define CP_BUSY_STAT__PFP_PARSING_PACKETS__SHIFT 0x00000007
+#define CP_BUSY_STAT__PIPE_STATS_BUSY_MASK 0x00100000L
+#define CP_BUSY_STAT__PIPE_STATS_BUSY__SHIFT 0x00000014
+#define CP_BUSY_STAT__RCIU_CE_BUSY_MASK 0x00200000L
+#define CP_BUSY_STAT__RCIU_CE_BUSY__SHIFT 0x00000015
+#define CP_BUSY_STAT__RCIU_ME_BUSY_MASK 0x00000400L
+#define CP_BUSY_STAT__RCIU_ME_BUSY__SHIFT 0x0000000a
+#define CP_BUSY_STAT__RCIU_PFP_BUSY_MASK 0x00000200L
+#define CP_BUSY_STAT__RCIU_PFP_BUSY__SHIFT 0x00000009
+#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK 0x00000001L
+#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT 0x00000000
+#define CP_BUSY_STAT__SEM_CMDFIFO_NOT_EMPTY_MASK 0x00001000L
+#define CP_BUSY_STAT__SEM_CMDFIFO_NOT_EMPTY__SHIFT 0x0000000c
+#define CP_BUSY_STAT__SEM_FAILED_AND_HOLDING_MASK 0x00002000L
+#define CP_BUSY_STAT__SEM_FAILED_AND_HOLDING__SHIFT 0x0000000d
+#define CP_BUSY_STAT__SEM_POLLING_FOR_PASS_MASK 0x00004000L
+#define CP_BUSY_STAT__SEM_POLLING_FOR_PASS__SHIFT 0x0000000e
+#define CP_BUSY_STAT__STRM_OUT_BUSY_MASK 0x00080000L
+#define CP_BUSY_STAT__STRM_OUT_BUSY__SHIFT 0x00000013
+#define CP_CE_HEADER_DUMP__CE_HEADER_DUMP_MASK 0xffffffffL
+#define CP_CE_HEADER_DUMP__CE_HEADER_DUMP__SHIFT 0x00000000
+#define CP_CE_IB1_BASE_HI__IB1_BASE_HI_MASK 0x000000ffL
+#define CP_CE_IB1_BASE_HI__IB1_BASE_HI__SHIFT 0x00000000
+#define CP_CE_IB1_BASE_LO__IB1_BASE_LO_MASK 0xfffffffcL
+#define CP_CE_IB1_BASE_LO__IB1_BASE_LO__SHIFT 0x00000002
+#define CP_CE_IB1_BUFSZ__IB1_BUFSZ_MASK 0x000fffffL
+#define CP_CE_IB1_BUFSZ__IB1_BUFSZ__SHIFT 0x00000000
+#define CP_CE_IB2_BASE_HI__IB2_BASE_HI_MASK 0x000000ffL
+#define CP_CE_IB2_BASE_HI__IB2_BASE_HI__SHIFT 0x00000000
+#define CP_CE_IB2_BASE_LO__IB2_BASE_LO_MASK 0xfffffffcL
+#define CP_CE_IB2_BASE_LO__IB2_BASE_LO__SHIFT 0x00000002
+#define CP_CE_IB2_BUFSZ__IB2_BUFSZ_MASK 0x000fffffL
+#define CP_CE_IB2_BUFSZ__IB2_BUFSZ__SHIFT 0x00000000
+#define CP_CE_INIT_BASE_HI__INIT_BASE_HI_MASK 0x000000ffL
+#define CP_CE_INIT_BASE_HI__INIT_BASE_HI__SHIFT 0x00000000
+#define CP_CE_INIT_BASE_LO__INIT_BASE_LO_MASK 0xffffffe0L
+#define CP_CE_INIT_BASE_LO__INIT_BASE_LO__SHIFT 0x00000005
+#define CP_CE_INIT_BUFSZ__INIT_BUFSZ_MASK 0x00000fffL
+#define CP_CE_INIT_BUFSZ__INIT_BUFSZ__SHIFT 0x00000000
+#define CP_CEQ1_AVAIL__CEQ_CNT_IB1_MASK 0x07ff0000L
+#define CP_CEQ1_AVAIL__CEQ_CNT_IB1__SHIFT 0x00000010
+#define CP_CEQ1_AVAIL__CEQ_CNT_RING_MASK 0x000007ffL
+#define CP_CEQ1_AVAIL__CEQ_CNT_RING__SHIFT 0x00000000
+#define CP_CEQ2_AVAIL__CEQ_CNT_IB2_MASK 0x000007ffL
+#define CP_CEQ2_AVAIL__CEQ_CNT_IB2__SHIFT 0x00000000
+#define CP_CE_ROQ_IB1_STAT__CEQ_RPTR_INDIRECT1_MASK 0x000003ffL
+#define CP_CE_ROQ_IB1_STAT__CEQ_RPTR_INDIRECT1__SHIFT 0x00000000
+#define CP_CE_ROQ_IB1_STAT__CEQ_WPTR_INDIRECT1_MASK 0x03ff0000L
+#define CP_CE_ROQ_IB1_STAT__CEQ_WPTR_INDIRECT1__SHIFT 0x00000010
+#define CP_CE_ROQ_IB2_STAT__CEQ_RPTR_INDIRECT2_MASK 0x000003ffL
+#define CP_CE_ROQ_IB2_STAT__CEQ_RPTR_INDIRECT2__SHIFT 0x00000000
+#define CP_CE_ROQ_IB2_STAT__CEQ_WPTR_INDIRECT2_MASK 0x03ff0000L
+#define CP_CE_ROQ_IB2_STAT__CEQ_WPTR_INDIRECT2__SHIFT 0x00000010
+#define CP_CE_ROQ_RB_STAT__CEQ_RPTR_PRIMARY_MASK 0x000003ffL
+#define CP_CE_ROQ_RB_STAT__CEQ_RPTR_PRIMARY__SHIFT 0x00000000
+#define CP_CE_ROQ_RB_STAT__CEQ_WPTR_PRIMARY_MASK 0x03ff0000L
+#define CP_CE_ROQ_RB_STAT__CEQ_WPTR_PRIMARY__SHIFT 0x00000010
+#define CP_CE_UCODE_ADDR__UCODE_ADDR_MASK 0x00000fffL
+#define CP_CE_UCODE_ADDR__UCODE_ADDR__SHIFT 0x00000000
+#define CP_CE_UCODE_DATA__UCODE_DATA_MASK 0xffffffffL
+#define CP_CE_UCODE_DATA__UCODE_DATA__SHIFT 0x00000000
+#define CP_CMD_DATA__CMD_DATA_MASK 0xffffffffL
+#define CP_CMD_DATA__CMD_DATA__SHIFT 0x00000000
+#define CP_CMD_INDEX__CMD_INDEX_MASK 0x000007ffL
+#define CP_CMD_INDEX__CMD_INDEX__SHIFT 0x00000000
+#define CP_CMD_INDEX__CMD_ME_SEL_MASK 0x00003000L
+#define CP_CMD_INDEX__CMD_ME_SEL__SHIFT 0x0000000c
+#define CP_CMD_INDEX__CMD_QUEUE_SEL_MASK 0x00030000L
+#define CP_CMD_INDEX__CMD_QUEUE_SEL__SHIFT 0x00000010
+#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS_MASK 0x0ff00000L
+#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS__SHIFT 0x00000014
+#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS_MASK 0x000000ffL
+#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS__SHIFT 0x00000000
+#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT_MASK 0x70000000L
+#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT__SHIFT 0x0000001c
+#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT_MASK 0x00000700L
+#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT__SHIFT 0x00000008
+#define CP_COHER_BASE__COHER_BASE_256B_MASK 0xffffffffL
+#define CP_COHER_BASE__COHER_BASE_256B__SHIFT 0x00000000
+#define CP_COHER_BASE_HI__COHER_BASE_HI_256B_MASK 0x000000ffL
+#define CP_COHER_BASE_HI__COHER_BASE_HI_256B__SHIFT 0x00000000
+#define CP_COHER_CNTL__CB0_DEST_BASE_ENA_MASK 0x00000040L
+#define CP_COHER_CNTL__CB0_DEST_BASE_ENA__SHIFT 0x00000006
+#define CP_COHER_CNTL__CB1_DEST_BASE_ENA_MASK 0x00000080L
+#define CP_COHER_CNTL__CB1_DEST_BASE_ENA__SHIFT 0x00000007
+#define CP_COHER_CNTL__CB2_DEST_BASE_ENA_MASK 0x00000100L
+#define CP_COHER_CNTL__CB2_DEST_BASE_ENA__SHIFT 0x00000008
+#define CP_COHER_CNTL__CB3_DEST_BASE_ENA_MASK 0x00000200L
+#define CP_COHER_CNTL__CB3_DEST_BASE_ENA__SHIFT 0x00000009
+#define CP_COHER_CNTL__CB4_DEST_BASE_ENA_MASK 0x00000400L
+#define CP_COHER_CNTL__CB4_DEST_BASE_ENA__SHIFT 0x0000000a
+#define CP_COHER_CNTL__CB5_DEST_BASE_ENA_MASK 0x00000800L
+#define CP_COHER_CNTL__CB5_DEST_BASE_ENA__SHIFT 0x0000000b
+#define CP_COHER_CNTL__CB6_DEST_BASE_ENA_MASK 0x00001000L
+#define CP_COHER_CNTL__CB6_DEST_BASE_ENA__SHIFT 0x0000000c
+#define CP_COHER_CNTL__CB7_DEST_BASE_ENA_MASK 0x00002000L
+#define CP_COHER_CNTL__CB7_DEST_BASE_ENA__SHIFT 0x0000000d
+#define CP_COHER_CNTL__CB_ACTION_ENA_MASK 0x02000000L
+#define CP_COHER_CNTL__CB_ACTION_ENA__SHIFT 0x00000019
+#define CP_COHER_CNTL__DB_ACTION_ENA_MASK 0x04000000L
+#define CP_COHER_CNTL__DB_ACTION_ENA__SHIFT 0x0000001a
+#define CP_COHER_CNTL__DB_DEST_BASE_ENA_MASK 0x00004000L
+#define CP_COHER_CNTL__DB_DEST_BASE_ENA__SHIFT 0x0000000e
+#define CP_COHER_CNTL__DEST_BASE_0_ENA_MASK 0x00000001L
+#define CP_COHER_CNTL__DEST_BASE_0_ENA__SHIFT 0x00000000
+#define CP_COHER_CNTL__DEST_BASE_1_ENA_MASK 0x00000002L
+#define CP_COHER_CNTL__DEST_BASE_1_ENA__SHIFT 0x00000001
+#define CP_COHER_CNTL__DEST_BASE_2_ENA_MASK 0x00080000L
+#define CP_COHER_CNTL__DEST_BASE_2_ENA__SHIFT 0x00000013
+#define CP_COHER_CNTL__DEST_BASE_3_ENA_MASK 0x00200000L
+#define CP_COHER_CNTL__DEST_BASE_3_ENA__SHIFT 0x00000015
+#define CP_COHER_CNTL__SH_ICACHE_ACTION_ENA_MASK 0x20000000L
+#define CP_COHER_CNTL__SH_ICACHE_ACTION_ENA__SHIFT 0x0000001d
+#define CP_COHER_CNTL__SH_KCACHE_ACTION_ENA_MASK 0x08000000L
+#define CP_COHER_CNTL__SH_KCACHE_ACTION_ENA__SHIFT 0x0000001b
+#define CP_COHER_CNTL__SH_KCACHE_VOL_ACTION_ENA_MASK 0x10000000L
+#define CP_COHER_CNTL__SH_KCACHE_VOL_ACTION_ENA__SHIFT 0x0000001c
+#define CP_COHER_CNTL__TC_ACTION_ENA_MASK 0x00800000L
+#define CP_COHER_CNTL__TC_ACTION_ENA__SHIFT 0x00000017
+#define CP_COHER_CNTL__TCL1_ACTION_ENA_MASK 0x00400000L
+#define CP_COHER_CNTL__TCL1_ACTION_ENA__SHIFT 0x00000016
+#define CP_COHER_CNTL__TCL1_VOL_ACTION_ENA_MASK 0x00008000L
+#define CP_COHER_CNTL__TCL1_VOL_ACTION_ENA__SHIFT 0x0000000f
+#define CP_COHER_CNTL__TC_VOL_ACTION_ENA_MASK 0x00010000L
+#define CP_COHER_CNTL__TC_VOL_ACTION_ENA__SHIFT 0x00000010
+#define CP_COHER_CNTL__TC_WB_ACTION_ENA_MASK 0x00040000L
+#define CP_COHER_CNTL__TC_WB_ACTION_ENA__SHIFT 0x00000012
+#define CP_COHER_SIZE__COHER_SIZE_256B_MASK 0xffffffffL
+#define CP_COHER_SIZE__COHER_SIZE_256B__SHIFT 0x00000000
+#define CP_COHER_SIZE_HI__COHER_SIZE_HI_256B_MASK 0x000000ffL
+#define CP_COHER_SIZE_HI__COHER_SIZE_HI_256B__SHIFT 0x00000000
+#define CP_COHER_START_DELAY__START_DELAY_COUNT_MASK 0x0000003fL
+#define CP_COHER_START_DELAY__START_DELAY_COUNT__SHIFT 0x00000000
+#define CP_COHER_STATUS__MATCHING_GFX_CNTX_MASK 0x000000ffL
+#define CP_COHER_STATUS__MATCHING_GFX_CNTX__SHIFT 0x00000000
+#define CP_COHER_STATUS__MEID_MASK 0x03000000L
+#define CP_COHER_STATUS__MEID__SHIFT 0x00000018
+#define CP_COHER_STATUS__PHASE1_STATUS_MASK 0x40000000L
+#define CP_COHER_STATUS__PHASE1_STATUS__SHIFT 0x0000001e
+#define CP_COHER_STATUS__STATUS_MASK 0x80000000L
+#define CP_COHER_STATUS__STATUS__SHIFT 0x0000001f
+#define CP_CSF_CNTL__FETCH_BUFFER_DEPTH_MASK 0x0000000fL
+#define CP_CSF_CNTL__FETCH_BUFFER_DEPTH__SHIFT 0x00000000
+#define CP_CSF_STAT__BUFFER_REQUEST_COUNT_MASK 0x00003f00L
+#define CP_CSF_STAT__BUFFER_REQUEST_COUNT__SHIFT 0x00000008
+#define CP_CSF_STAT__BUFFER_SLOTS_ALLOCATED_MASK 0x0000000fL
+#define CP_CSF_STAT__BUFFER_SLOTS_ALLOCATED__SHIFT 0x00000000
+#define CP_DMA_CNTL__BUFFER_DEPTH_MASK 0x000f0000L
+#define CP_DMA_CNTL__BUFFER_DEPTH__SHIFT 0x00000010
+#define CP_DMA_CNTL__MIN_AVAILSZ_MASK 0x00000030L
+#define CP_DMA_CNTL__MIN_AVAILSZ__SHIFT 0x00000004
+#define CP_DMA_CNTL__PIO_COUNT_MASK 0xc0000000L
+#define CP_DMA_CNTL__PIO_COUNT__SHIFT 0x0000001e
+#define CP_DMA_CNTL__PIO_FIFO_EMPTY_MASK 0x10000000L
+#define CP_DMA_CNTL__PIO_FIFO_EMPTY__SHIFT 0x0000001c
+#define CP_DMA_CNTL__PIO_FIFO_FULL_MASK 0x20000000L
+#define CP_DMA_CNTL__PIO_FIFO_FULL__SHIFT 0x0000001d
+#define CP_DMA_ME_COMMAND__BYTE_COUNT_MASK 0x001fffffL
+#define CP_DMA_ME_COMMAND__BYTE_COUNT__SHIFT 0x00000000
+#define CP_DMA_ME_COMMAND__DAIC_MASK 0x20000000L
+#define CP_DMA_ME_COMMAND__DAIC__SHIFT 0x0000001d
+#define CP_DMA_ME_COMMAND__DAS_MASK 0x08000000L
+#define CP_DMA_ME_COMMAND__DAS__SHIFT 0x0000001b
+#define CP_DMA_ME_COMMAND__DIS_WC_MASK 0x00200000L
+#define CP_DMA_ME_COMMAND__DIS_WC__SHIFT 0x00000015
+#define CP_DMA_ME_COMMAND__DST_SWAP_MASK 0x03000000L
+#define CP_DMA_ME_COMMAND__DST_SWAP__SHIFT 0x00000018
+#define CP_DMA_ME_COMMAND__RAW_WAIT_MASK 0x40000000L
+#define CP_DMA_ME_COMMAND__RAW_WAIT__SHIFT 0x0000001e
+#define CP_DMA_ME_COMMAND__SAIC_MASK 0x10000000L
+#define CP_DMA_ME_COMMAND__SAIC__SHIFT 0x0000001c
+#define CP_DMA_ME_COMMAND__SAS_MASK 0x04000000L
+#define CP_DMA_ME_COMMAND__SAS__SHIFT 0x0000001a
+#define CP_DMA_ME_COMMAND__SRC_SWAP_MASK 0x00c00000L
+#define CP_DMA_ME_COMMAND__SRC_SWAP__SHIFT 0x00000016
+#define CP_DMA_ME_DST_ADDR__DST_ADDR_MASK 0xffffffffL
+#define CP_DMA_ME_DST_ADDR__DST_ADDR__SHIFT 0x00000000
+#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI_MASK 0x000000ffL
+#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI__SHIFT 0x00000000
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI_MASK 0x000000ffL
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT 0x00000000
+#define CP_DMA_ME_SRC_ADDR__SRC_ADDR_MASK 0xffffffffL
+#define CP_DMA_ME_SRC_ADDR__SRC_ADDR__SHIFT 0x00000000
+#define CP_DMA_PFP_COMMAND__BYTE_COUNT_MASK 0x001fffffL
+#define CP_DMA_PFP_COMMAND__BYTE_COUNT__SHIFT 0x00000000
+#define CP_DMA_PFP_COMMAND__DAIC_MASK 0x20000000L
+#define CP_DMA_PFP_COMMAND__DAIC__SHIFT 0x0000001d
+#define CP_DMA_PFP_COMMAND__DAS_MASK 0x08000000L
+#define CP_DMA_PFP_COMMAND__DAS__SHIFT 0x0000001b
+#define CP_DMA_PFP_COMMAND__DIS_WC_MASK 0x00200000L
+#define CP_DMA_PFP_COMMAND__DIS_WC__SHIFT 0x00000015
+#define CP_DMA_PFP_COMMAND__DST_SWAP_MASK 0x03000000L
+#define CP_DMA_PFP_COMMAND__DST_SWAP__SHIFT 0x00000018
+#define CP_DMA_PFP_COMMAND__RAW_WAIT_MASK 0x40000000L
+#define CP_DMA_PFP_COMMAND__RAW_WAIT__SHIFT 0x0000001e
+#define CP_DMA_PFP_COMMAND__SAIC_MASK 0x10000000L
+#define CP_DMA_PFP_COMMAND__SAIC__SHIFT 0x0000001c
+#define CP_DMA_PFP_COMMAND__SAS_MASK 0x04000000L
+#define CP_DMA_PFP_COMMAND__SAS__SHIFT 0x0000001a
+#define CP_DMA_PFP_COMMAND__SRC_SWAP_MASK 0x00c00000L
+#define CP_DMA_PFP_COMMAND__SRC_SWAP__SHIFT 0x00000016
+#define CP_DMA_PFP_DST_ADDR__DST_ADDR_MASK 0xffffffffL
+#define CP_DMA_PFP_DST_ADDR__DST_ADDR__SHIFT 0x00000000
+#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI_MASK 0x000000ffL
+#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI__SHIFT 0x00000000
+#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI_MASK 0x000000ffL
+#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT 0x00000000
+#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR_MASK 0xffffffffL
+#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR__SHIFT 0x00000000
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_MASK 0x03ffffffL
+#define CP_DMA_READ_TAGS__DMA_READ_TAG__SHIFT 0x00000000
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID_MASK 0x10000000L
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID__SHIFT 0x0000001c
+#define CP_ECC_FIRSTOCCURRENCE__INTERFACE_MASK 0x00000003L
+#define CP_ECC_FIRSTOCCURRENCE__INTERFACE__SHIFT 0x00000000
+#define CP_ECC_FIRSTOCCURRENCE__REQUEST_CLIENT_MASK 0x000000f0L
+#define CP_ECC_FIRSTOCCURRENCE__REQUEST_CLIENT__SHIFT 0x00000004
+#define CP_ECC_FIRSTOCCURRENCE_RING0__INTERFACE_MASK 0x00000003L
+#define CP_ECC_FIRSTOCCURRENCE_RING0__INTERFACE__SHIFT 0x00000000
+#define CP_ECC_FIRSTOCCURRENCE_RING0__REQUEST_CLIENT_MASK 0x000000f0L
+#define CP_ECC_FIRSTOCCURRENCE_RING0__REQUEST_CLIENT__SHIFT 0x00000004
+#define CP_ECC_FIRSTOCCURRENCE_RING0__RING_ID_MASK 0x00003c00L
+#define CP_ECC_FIRSTOCCURRENCE_RING0__RING_ID__SHIFT 0x0000000a
+#define CP_ECC_FIRSTOCCURRENCE_RING0__VMID_MASK 0x000f0000L
+#define CP_ECC_FIRSTOCCURRENCE_RING0__VMID__SHIFT 0x00000010
+#define CP_ECC_FIRSTOCCURRENCE_RING1__INTERFACE_MASK 0x00000003L
+#define CP_ECC_FIRSTOCCURRENCE_RING1__INTERFACE__SHIFT 0x00000000
+#define CP_ECC_FIRSTOCCURRENCE_RING1__REQUEST_CLIENT_MASK 0x000000f0L
+#define CP_ECC_FIRSTOCCURRENCE_RING1__REQUEST_CLIENT__SHIFT 0x00000004
+#define CP_ECC_FIRSTOCCURRENCE_RING1__RING_ID_MASK 0x00003c00L
+#define CP_ECC_FIRSTOCCURRENCE_RING1__RING_ID__SHIFT 0x0000000a
+#define CP_ECC_FIRSTOCCURRENCE_RING1__VMID_MASK 0x000f0000L
+#define CP_ECC_FIRSTOCCURRENCE_RING1__VMID__SHIFT 0x00000010
+#define CP_ECC_FIRSTOCCURRENCE_RING2__INTERFACE_MASK 0x00000003L
+#define CP_ECC_FIRSTOCCURRENCE_RING2__INTERFACE__SHIFT 0x00000000
+#define CP_ECC_FIRSTOCCURRENCE_RING2__REQUEST_CLIENT_MASK 0x000000f0L
+#define CP_ECC_FIRSTOCCURRENCE_RING2__REQUEST_CLIENT__SHIFT 0x00000004
+#define CP_ECC_FIRSTOCCURRENCE_RING2__RING_ID_MASK 0x00003c00L
+#define CP_ECC_FIRSTOCCURRENCE_RING2__RING_ID__SHIFT 0x0000000a
+#define CP_ECC_FIRSTOCCURRENCE_RING2__VMID_MASK 0x000f0000L
+#define CP_ECC_FIRSTOCCURRENCE_RING2__VMID__SHIFT 0x00000010
+#define CP_ECC_FIRSTOCCURRENCE__RING_ID_MASK 0x00003c00L
+#define CP_ECC_FIRSTOCCURRENCE__RING_ID__SHIFT 0x0000000a
+#define CP_ECC_FIRSTOCCURRENCE__VMID_MASK 0x000f0000L
+#define CP_ECC_FIRSTOCCURRENCE__VMID__SHIFT 0x00000010
+#define CP_EOP_DONE_ADDR_HI__ADDR_HI_MASK 0x0000ffffL
+#define CP_EOP_DONE_ADDR_HI__ADDR_HI__SHIFT 0x00000000
+#define CP_EOP_DONE_ADDR_LO__ADDR_LO_MASK 0xfffffffcL
+#define CP_EOP_DONE_ADDR_LO__ADDR_LO__SHIFT 0x00000002
+#define CP_EOP_DONE_ADDR_LO__ADDR_SWAP_MASK 0x00000003L
+#define CP_EOP_DONE_ADDR_LO__ADDR_SWAP__SHIFT 0x00000000
+#define CP_EOP_DONE_DATA_CNTL__CNTX_ID_MASK 0x0000ffffL
+#define CP_EOP_DONE_DATA_CNTL__CNTX_ID__SHIFT 0x00000000
+#define CP_EOP_DONE_DATA_CNTL__DATA_SEL_MASK 0xe0000000L
+#define CP_EOP_DONE_DATA_CNTL__DATA_SEL__SHIFT 0x0000001d
+#define CP_EOP_DONE_DATA_CNTL__DST_SEL_MASK 0x00030000L
+#define CP_EOP_DONE_DATA_CNTL__DST_SEL__SHIFT 0x00000010
+#define CP_EOP_DONE_DATA_CNTL__INT_SEL_MASK 0x07000000L
+#define CP_EOP_DONE_DATA_CNTL__INT_SEL__SHIFT 0x00000018
+#define CP_EOP_DONE_DATA_HI__DATA_HI_MASK 0xffffffffL
+#define CP_EOP_DONE_DATA_HI__DATA_HI__SHIFT 0x00000000
+#define CP_EOP_DONE_DATA_LO__DATA_LO_MASK 0xffffffffL
+#define CP_EOP_DONE_DATA_LO__DATA_LO__SHIFT 0x00000000
+#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI_MASK 0xffffffffL
+#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI__SHIFT 0x00000000
+#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO_MASK 0xffffffffL
+#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO__SHIFT 0x00000000
+#define CP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK 0xffffffffL
+#define CP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT 0x00000000
+#define CP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK 0xffffffffL
+#define CP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT 0x00000000
+#define CP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK 0xffffffffL
+#define CP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT 0x00000000
+#define CP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK 0xffffffffL
+#define CP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT 0x00000000
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_GDS_MASK 0x00003f00L
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_GDS__SHIFT 0x00000008
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_MASK 0x0000003fL
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP_MASK 0x003f0000L
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP__SHIFT 0x00000010
+#define CP_GRBM_FREE_COUNT__FREE_COUNT__SHIFT 0x00000000
+#define CP_IB1_BASE_HI__IB1_BASE_HI_MASK 0x000000ffL
+#define CP_IB1_BASE_HI__IB1_BASE_HI__SHIFT 0x00000000
+#define CP_IB1_BASE_LO__IB1_BASE_LO_MASK 0xfffffffcL
+#define CP_IB1_BASE_LO__IB1_BASE_LO__SHIFT 0x00000002
+#define CP_IB1_BUFSZ__IB1_BUFSZ_MASK 0x000fffffL
+#define CP_IB1_BUFSZ__IB1_BUFSZ__SHIFT 0x00000000
+#define CP_IB1_OFFSET__IB1_OFFSET_MASK 0x000fffffL
+#define CP_IB1_OFFSET__IB1_OFFSET__SHIFT 0x00000000
+#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN_MASK 0x000fffffL
+#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN__SHIFT 0x00000000
+#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END_MASK 0x000fffffL
+#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END__SHIFT 0x00000000
+#define CP_IB2_BASE_HI__IB2_BASE_HI_MASK 0x000000ffL
+#define CP_IB2_BASE_HI__IB2_BASE_HI__SHIFT 0x00000000
+#define CP_IB2_BASE_LO__IB2_BASE_LO_MASK 0xfffffffcL
+#define CP_IB2_BASE_LO__IB2_BASE_LO__SHIFT 0x00000002
+#define CP_IB2_BUFSZ__IB2_BUFSZ_MASK 0x000fffffL
+#define CP_IB2_BUFSZ__IB2_BUFSZ__SHIFT 0x00000000
+#define CP_IB2_OFFSET__IB2_OFFSET_MASK 0x000fffffL
+#define CP_IB2_OFFSET__IB2_OFFSET__SHIFT 0x00000000
+#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN_MASK 0x000fffffL
+#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN__SHIFT 0x00000000
+#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END_MASK 0x000fffffL
+#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END__SHIFT 0x00000000
+#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE_MASK 0x00080000L
+#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE__SHIFT 0x00000013
+#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE_MASK 0x00100000L
+#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE__SHIFT 0x00000014
+#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x00004000L
+#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0x0000000e
+#define CP_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000L
+#define CP_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x0000001f
+#define CP_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000L
+#define CP_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x0000001e
+#define CP_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000L
+#define CP_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x0000001d
+#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x01000000L
+#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x00000018
+#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE_MASK 0x00400000L
+#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE__SHIFT 0x00000016
+#define CP_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x00800000L
+#define CP_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x00000017
+#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x08000000L
+#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x0000001b
+#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE_MASK 0x00080000L
+#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE__SHIFT 0x00000013
+#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE_MASK 0x00100000L
+#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE__SHIFT 0x00000014
+#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE_MASK 0x00004000L
+#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE__SHIFT 0x0000000e
+#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE_MASK 0x80000000L
+#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE__SHIFT 0x0000001f
+#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE_MASK 0x40000000L
+#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE__SHIFT 0x0000001e
+#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE_MASK 0x20000000L
+#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE__SHIFT 0x0000001d
+#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK 0x01000000L
+#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE__SHIFT 0x00000018
+#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK 0x00400000L
+#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE__SHIFT 0x00000016
+#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK 0x00800000L
+#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE__SHIFT 0x00000017
+#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x08000000L
+#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x0000001b
+#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK 0x04000000L
+#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE__SHIFT 0x0000001a
+#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x00020000L
+#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x00000011
+#define CP_INT_CNTL_RING1__CNTX_BUSY_INT_ENABLE_MASK 0x00080000L
+#define CP_INT_CNTL_RING1__CNTX_BUSY_INT_ENABLE__SHIFT 0x00000013
+#define CP_INT_CNTL_RING1__CNTX_EMPTY_INT_ENABLE_MASK 0x00100000L
+#define CP_INT_CNTL_RING1__CNTX_EMPTY_INT_ENABLE__SHIFT 0x00000014
+#define CP_INT_CNTL_RING1__CP_ECC_ERROR_INT_ENABLE_MASK 0x00004000L
+#define CP_INT_CNTL_RING1__CP_ECC_ERROR_INT_ENABLE__SHIFT 0x0000000e
+#define CP_INT_CNTL_RING1__GENERIC0_INT_ENABLE_MASK 0x80000000L
+#define CP_INT_CNTL_RING1__GENERIC0_INT_ENABLE__SHIFT 0x0000001f
+#define CP_INT_CNTL_RING1__GENERIC1_INT_ENABLE_MASK 0x40000000L
+#define CP_INT_CNTL_RING1__GENERIC1_INT_ENABLE__SHIFT 0x0000001e
+#define CP_INT_CNTL_RING1__GENERIC2_INT_ENABLE_MASK 0x20000000L
+#define CP_INT_CNTL_RING1__GENERIC2_INT_ENABLE__SHIFT 0x0000001d
+#define CP_INT_CNTL_RING1__OPCODE_ERROR_INT_ENABLE_MASK 0x01000000L
+#define CP_INT_CNTL_RING1__OPCODE_ERROR_INT_ENABLE__SHIFT 0x00000018
+#define CP_INT_CNTL_RING1__PRIV_INSTR_INT_ENABLE_MASK 0x00400000L
+#define CP_INT_CNTL_RING1__PRIV_INSTR_INT_ENABLE__SHIFT 0x00000016
+#define CP_INT_CNTL_RING1__PRIV_REG_INT_ENABLE_MASK 0x00800000L
+#define CP_INT_CNTL_RING1__PRIV_REG_INT_ENABLE__SHIFT 0x00000017
+#define CP_INT_CNTL_RING1__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x08000000L
+#define CP_INT_CNTL_RING1__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x0000001b
+#define CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE_MASK 0x04000000L
+#define CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE__SHIFT 0x0000001a
+#define CP_INT_CNTL_RING1__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x00020000L
+#define CP_INT_CNTL_RING1__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x00000011
+#define CP_INT_CNTL_RING2__CNTX_BUSY_INT_ENABLE_MASK 0x00080000L
+#define CP_INT_CNTL_RING2__CNTX_BUSY_INT_ENABLE__SHIFT 0x00000013
+#define CP_INT_CNTL_RING2__CNTX_EMPTY_INT_ENABLE_MASK 0x00100000L
+#define CP_INT_CNTL_RING2__CNTX_EMPTY_INT_ENABLE__SHIFT 0x00000014
+#define CP_INT_CNTL_RING2__CP_ECC_ERROR_INT_ENABLE_MASK 0x00004000L
+#define CP_INT_CNTL_RING2__CP_ECC_ERROR_INT_ENABLE__SHIFT 0x0000000e
+#define CP_INT_CNTL_RING2__GENERIC0_INT_ENABLE_MASK 0x80000000L
+#define CP_INT_CNTL_RING2__GENERIC0_INT_ENABLE__SHIFT 0x0000001f
+#define CP_INT_CNTL_RING2__GENERIC1_INT_ENABLE_MASK 0x40000000L
+#define CP_INT_CNTL_RING2__GENERIC1_INT_ENABLE__SHIFT 0x0000001e
+#define CP_INT_CNTL_RING2__GENERIC2_INT_ENABLE_MASK 0x20000000L
+#define CP_INT_CNTL_RING2__GENERIC2_INT_ENABLE__SHIFT 0x0000001d
+#define CP_INT_CNTL_RING2__OPCODE_ERROR_INT_ENABLE_MASK 0x01000000L
+#define CP_INT_CNTL_RING2__OPCODE_ERROR_INT_ENABLE__SHIFT 0x00000018
+#define CP_INT_CNTL_RING2__PRIV_INSTR_INT_ENABLE_MASK 0x00400000L
+#define CP_INT_CNTL_RING2__PRIV_INSTR_INT_ENABLE__SHIFT 0x00000016
+#define CP_INT_CNTL_RING2__PRIV_REG_INT_ENABLE_MASK 0x00800000L
+#define CP_INT_CNTL_RING2__PRIV_REG_INT_ENABLE__SHIFT 0x00000017
+#define CP_INT_CNTL_RING2__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x08000000L
+#define CP_INT_CNTL_RING2__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x0000001b
+#define CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE_MASK 0x04000000L
+#define CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE__SHIFT 0x0000001a
+#define CP_INT_CNTL_RING2__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x00020000L
+#define CP_INT_CNTL_RING2__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x00000011
+#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x04000000L
+#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x0000001a
+#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x00020000L
+#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x00000011
+#define CP_INT_STAT_DEBUG__CNTX_BUSY_INT_ASSERTED_MASK 0x00080000L
+#define CP_INT_STAT_DEBUG__CNTX_BUSY_INT_ASSERTED__SHIFT 0x00000013
+#define CP_INT_STAT_DEBUG__CNTX_EMPTY_INT_ASSERTED_MASK 0x00100000L
+#define CP_INT_STAT_DEBUG__CNTX_EMPTY_INT_ASSERTED__SHIFT 0x00000014
+#define CP_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK 0x00004000L
+#define CP_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT 0x0000000e
+#define CP_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK 0x80000000L
+#define CP_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT 0x0000001f
+#define CP_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK 0x40000000L
+#define CP_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT 0x0000001e
+#define CP_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK 0x20000000L
+#define CP_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT 0x0000001d
+#define CP_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK 0x01000000L
+#define CP_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT 0x00000018
+#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED_MASK 0x00400000L
+#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED__SHIFT 0x00000016
+#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK 0x00800000L
+#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT 0x00000017
+#define CP_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK 0x08000000L
+#define CP_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT 0x0000001b
+#define CP_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK 0x04000000L
+#define CP_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT 0x0000001a
+#define CP_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK 0x00020000L
+#define CP_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT 0x00000011
+#define CP_INT_STATUS__CNTX_BUSY_INT_STAT_MASK 0x00080000L
+#define CP_INT_STATUS__CNTX_BUSY_INT_STAT__SHIFT 0x00000013
+#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT_MASK 0x00100000L
+#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT__SHIFT 0x00000014
+#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT_MASK 0x00004000L
+#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT__SHIFT 0x0000000e
+#define CP_INT_STATUS__GENERIC0_INT_STAT_MASK 0x80000000L
+#define CP_INT_STATUS__GENERIC0_INT_STAT__SHIFT 0x0000001f
+#define CP_INT_STATUS__GENERIC1_INT_STAT_MASK 0x40000000L
+#define CP_INT_STATUS__GENERIC1_INT_STAT__SHIFT 0x0000001e
+#define CP_INT_STATUS__GENERIC2_INT_STAT_MASK 0x20000000L
+#define CP_INT_STATUS__GENERIC2_INT_STAT__SHIFT 0x0000001d
+#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT_MASK 0x01000000L
+#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT__SHIFT 0x00000018
+#define CP_INT_STATUS__PRIV_INSTR_INT_STAT_MASK 0x00400000L
+#define CP_INT_STATUS__PRIV_INSTR_INT_STAT__SHIFT 0x00000016
+#define CP_INT_STATUS__PRIV_REG_INT_STAT_MASK 0x00800000L
+#define CP_INT_STATUS__PRIV_REG_INT_STAT__SHIFT 0x00000017
+#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT_MASK 0x08000000L
+#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT__SHIFT 0x0000001b
+#define CP_INT_STATUS_RING0__CNTX_BUSY_INT_STAT_MASK 0x00080000L
+#define CP_INT_STATUS_RING0__CNTX_BUSY_INT_STAT__SHIFT 0x00000013
+#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT_MASK 0x00100000L
+#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT__SHIFT 0x00000014
+#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT_MASK 0x00004000L
+#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT__SHIFT 0x0000000e
+#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT_MASK 0x80000000L
+#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT__SHIFT 0x0000001f
+#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT_MASK 0x40000000L
+#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT__SHIFT 0x0000001e
+#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT_MASK 0x20000000L
+#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT__SHIFT 0x0000001d
+#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT_MASK 0x01000000L
+#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT__SHIFT 0x00000018
+#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT_MASK 0x00400000L
+#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT__SHIFT 0x00000016
+#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT_MASK 0x00800000L
+#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT__SHIFT 0x00000017
+#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT_MASK 0x08000000L
+#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT__SHIFT 0x0000001b
+#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT_MASK 0x04000000L
+#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT__SHIFT 0x0000001a
+#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT_MASK 0x00020000L
+#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT__SHIFT 0x00000011
+#define CP_INT_STATUS_RING1__CNTX_BUSY_INT_STAT_MASK 0x00080000L
+#define CP_INT_STATUS_RING1__CNTX_BUSY_INT_STAT__SHIFT 0x00000013
+#define CP_INT_STATUS_RING1__CNTX_EMPTY_INT_STAT_MASK 0x00100000L
+#define CP_INT_STATUS_RING1__CNTX_EMPTY_INT_STAT__SHIFT 0x00000014
+#define CP_INT_STATUS_RING1__CP_ECC_ERROR_INT_STAT_MASK 0x00004000L
+#define CP_INT_STATUS_RING1__CP_ECC_ERROR_INT_STAT__SHIFT 0x0000000e
+#define CP_INT_STATUS_RING1__GENERIC0_INT_STAT_MASK 0x80000000L
+#define CP_INT_STATUS_RING1__GENERIC0_INT_STAT__SHIFT 0x0000001f
+#define CP_INT_STATUS_RING1__GENERIC1_INT_STAT_MASK 0x40000000L
+#define CP_INT_STATUS_RING1__GENERIC1_INT_STAT__SHIFT 0x0000001e
+#define CP_INT_STATUS_RING1__GENERIC2_INT_STAT_MASK 0x20000000L
+#define CP_INT_STATUS_RING1__GENERIC2_INT_STAT__SHIFT 0x0000001d
+#define CP_INT_STATUS_RING1__OPCODE_ERROR_INT_STAT_MASK 0x01000000L
+#define CP_INT_STATUS_RING1__OPCODE_ERROR_INT_STAT__SHIFT 0x00000018
+#define CP_INT_STATUS_RING1__PRIV_INSTR_INT_STAT_MASK 0x00400000L
+#define CP_INT_STATUS_RING1__PRIV_INSTR_INT_STAT__SHIFT 0x00000016
+#define CP_INT_STATUS_RING1__PRIV_REG_INT_STAT_MASK 0x00800000L
+#define CP_INT_STATUS_RING1__PRIV_REG_INT_STAT__SHIFT 0x00000017
+#define CP_INT_STATUS_RING1__RESERVED_BIT_ERROR_INT_STAT_MASK 0x08000000L
+#define CP_INT_STATUS_RING1__RESERVED_BIT_ERROR_INT_STAT__SHIFT 0x0000001b
+#define CP_INT_STATUS_RING1__TIME_STAMP_INT_STAT_MASK 0x04000000L
+#define CP_INT_STATUS_RING1__TIME_STAMP_INT_STAT__SHIFT 0x0000001a
+#define CP_INT_STATUS_RING1__WRM_POLL_TIMEOUT_INT_STAT_MASK 0x00020000L
+#define CP_INT_STATUS_RING1__WRM_POLL_TIMEOUT_INT_STAT__SHIFT 0x00000011
+#define CP_INT_STATUS_RING2__CNTX_BUSY_INT_STAT_MASK 0x00080000L
+#define CP_INT_STATUS_RING2__CNTX_BUSY_INT_STAT__SHIFT 0x00000013
+#define CP_INT_STATUS_RING2__CNTX_EMPTY_INT_STAT_MASK 0x00100000L
+#define CP_INT_STATUS_RING2__CNTX_EMPTY_INT_STAT__SHIFT 0x00000014
+#define CP_INT_STATUS_RING2__CP_ECC_ERROR_INT_STAT_MASK 0x00004000L
+#define CP_INT_STATUS_RING2__CP_ECC_ERROR_INT_STAT__SHIFT 0x0000000e
+#define CP_INT_STATUS_RING2__GENERIC0_INT_STAT_MASK 0x80000000L
+#define CP_INT_STATUS_RING2__GENERIC0_INT_STAT__SHIFT 0x0000001f
+#define CP_INT_STATUS_RING2__GENERIC1_INT_STAT_MASK 0x40000000L
+#define CP_INT_STATUS_RING2__GENERIC1_INT_STAT__SHIFT 0x0000001e
+#define CP_INT_STATUS_RING2__GENERIC2_INT_STAT_MASK 0x20000000L
+#define CP_INT_STATUS_RING2__GENERIC2_INT_STAT__SHIFT 0x0000001d
+#define CP_INT_STATUS_RING2__OPCODE_ERROR_INT_STAT_MASK 0x01000000L
+#define CP_INT_STATUS_RING2__OPCODE_ERROR_INT_STAT__SHIFT 0x00000018
+#define CP_INT_STATUS_RING2__PRIV_INSTR_INT_STAT_MASK 0x00400000L
+#define CP_INT_STATUS_RING2__PRIV_INSTR_INT_STAT__SHIFT 0x00000016
+#define CP_INT_STATUS_RING2__PRIV_REG_INT_STAT_MASK 0x00800000L
+#define CP_INT_STATUS_RING2__PRIV_REG_INT_STAT__SHIFT 0x00000017
+#define CP_INT_STATUS_RING2__RESERVED_BIT_ERROR_INT_STAT_MASK 0x08000000L
+#define CP_INT_STATUS_RING2__RESERVED_BIT_ERROR_INT_STAT__SHIFT 0x0000001b
+#define CP_INT_STATUS_RING2__TIME_STAMP_INT_STAT_MASK 0x04000000L
+#define CP_INT_STATUS_RING2__TIME_STAMP_INT_STAT__SHIFT 0x0000001a
+#define CP_INT_STATUS_RING2__WRM_POLL_TIMEOUT_INT_STAT_MASK 0x00020000L
+#define CP_INT_STATUS_RING2__WRM_POLL_TIMEOUT_INT_STAT__SHIFT 0x00000011
+#define CP_INT_STATUS__TIME_STAMP_INT_STAT_MASK 0x04000000L
+#define CP_INT_STATUS__TIME_STAMP_INT_STAT__SHIFT 0x0000001a
+#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT_MASK 0x00020000L
+#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT__SHIFT 0x00000011
+#define CP_MC_PACK_DELAY_CNT__PACK_DELAY_CNT_MASK 0x0000001fL
+#define CP_MC_PACK_DELAY_CNT__PACK_DELAY_CNT__SHIFT 0x00000000
+#define CP_ME_CNTL__CE_HALT_MASK 0x01000000L
+#define CP_ME_CNTL__CE_HALT__SHIFT 0x00000018
+#define CP_ME_CNTL__CE_INVALIDATE_ICACHE_MASK 0x00000010L
+#define CP_ME_CNTL__CE_INVALIDATE_ICACHE__SHIFT 0x00000004
+#define CP_ME_CNTL__CE_STEP_MASK 0x02000000L
+#define CP_ME_CNTL__CE_STEP__SHIFT 0x00000019
+#define CP_ME_CNTL__ME_HALT_MASK 0x10000000L
+#define CP_ME_CNTL__ME_HALT__SHIFT 0x0000001c
+#define CP_ME_CNTL__ME_INVALIDATE_ICACHE_MASK 0x00000100L
+#define CP_ME_CNTL__ME_INVALIDATE_ICACHE__SHIFT 0x00000008
+#define CP_ME_CNTL__ME_STEP_MASK 0x20000000L
+#define CP_ME_CNTL__ME_STEP__SHIFT 0x0000001d
+#define CP_ME_CNTL__PFP_HALT_MASK 0x04000000L
+#define CP_ME_CNTL__PFP_HALT__SHIFT 0x0000001a
+#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE_MASK 0x00000040L
+#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE__SHIFT 0x00000006
+#define CP_ME_CNTL__PFP_STEP_MASK 0x08000000L
+#define CP_ME_CNTL__PFP_STEP__SHIFT 0x0000001b
+#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP_MASK 0xffffffffL
+#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP__SHIFT 0x00000000
+#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI_MASK 0x000000ffL
+#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI__SHIFT 0x00000000
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO_MASK 0xfffffffcL
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO__SHIFT 0x00000002
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_SWAP_MASK 0x00000003L
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_SWAP__SHIFT 0x00000000
+#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI_MASK 0x000000ffL
+#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI__SHIFT 0x00000000
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO_MASK 0xfffffffcL
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO__SHIFT 0x00000002
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_SWAP_MASK 0x00000003L
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_SWAP__SHIFT 0x00000000
+#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI_MASK 0xffffffffL
+#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI__SHIFT 0x00000000
+#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO_MASK 0xffffffffL
+#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO__SHIFT 0x00000000
+#define CP_MEM_SLP_CNTL__CP_MEM_DS_EN_MASK 0x00000002L
+#define CP_MEM_SLP_CNTL__CP_MEM_DS_EN__SHIFT 0x00000001
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK 0x00000001L
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_EN__SHIFT 0x00000000
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_OFF_DELAY_MASK 0x00ff0000L
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_OFF_DELAY__SHIFT 0x00000010
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_ON_DELAY_MASK 0x0000ff00L
+#define CP_MEM_SLP_CNTL__CP_MEM_LS_ON_DELAY__SHIFT 0x00000008
+#define CP_MEM_SLP_CNTL__RESERVED1_MASK 0xff000000L
+#define CP_MEM_SLP_CNTL__RESERVED1__SHIFT 0x00000018
+#define CP_MEM_SLP_CNTL__RESERVED_MASK 0x000000fcL
+#define CP_MEM_SLP_CNTL__RESERVED__SHIFT 0x00000002
+#define CP_ME_PREEMPTION__ME_CNTXSW_PREEMPTION_MASK 0x00000001L
+#define CP_ME_PREEMPTION__ME_CNTXSW_PREEMPTION__SHIFT 0x00000000
+#define CP_MEQ_AVAIL__MEQ_CNT_MASK 0x000003ffL
+#define CP_MEQ_AVAIL__MEQ_CNT__SHIFT 0x00000000
+#define CP_MEQ_STAT__MEQ_RPTR_MASK 0x000003ffL
+#define CP_MEQ_STAT__MEQ_RPTR__SHIFT 0x00000000
+#define CP_MEQ_STAT__MEQ_WPTR_MASK 0x03ff0000L
+#define CP_MEQ_STAT__MEQ_WPTR__SHIFT 0x00000010
+#define CP_MEQ_THRESHOLDS__MEQ1_START_MASK 0x000000ffL
+#define CP_MEQ_THRESHOLDS__MEQ1_START__SHIFT 0x00000000
+#define CP_MEQ_THRESHOLDS__MEQ2_START_MASK 0x0000ff00L
+#define CP_MEQ_THRESHOLDS__MEQ2_START__SHIFT 0x00000008
+#define CP_ME_RAM_DATA__ME_RAM_DATA_MASK 0xffffffffL
+#define CP_ME_RAM_DATA__ME_RAM_DATA__SHIFT 0x00000000
+#define CP_ME_RAM_RADDR__ME_RAM_RADDR_MASK 0x00000fffL
+#define CP_ME_RAM_RADDR__ME_RAM_RADDR__SHIFT 0x00000000
+#define CP_ME_RAM_WADDR__ME_RAM_WADDR_MASK 0x00000fffL
+#define CP_ME_RAM_WADDR__ME_RAM_WADDR__SHIFT 0x00000000
+#define CP_NUM_PRIM_NEEDED_COUNT0_HI__NUM_PRIM_NEEDED_CNT0_HI_MASK 0xffffffffL
+#define CP_NUM_PRIM_NEEDED_COUNT0_HI__NUM_PRIM_NEEDED_CNT0_HI__SHIFT 0x00000000
+#define CP_NUM_PRIM_NEEDED_COUNT0_LO__NUM_PRIM_NEEDED_CNT0_LO_MASK 0xffffffffL
+#define CP_NUM_PRIM_NEEDED_COUNT0_LO__NUM_PRIM_NEEDED_CNT0_LO__SHIFT 0x00000000
+#define CP_NUM_PRIM_NEEDED_COUNT1_HI__NUM_PRIM_NEEDED_CNT1_HI_MASK 0xffffffffL
+#define CP_NUM_PRIM_NEEDED_COUNT1_HI__NUM_PRIM_NEEDED_CNT1_HI__SHIFT 0x00000000
+#define CP_NUM_PRIM_NEEDED_COUNT1_LO__NUM_PRIM_NEEDED_CNT1_LO_MASK 0xffffffffL
+#define CP_NUM_PRIM_NEEDED_COUNT1_LO__NUM_PRIM_NEEDED_CNT1_LO__SHIFT 0x00000000
+#define CP_NUM_PRIM_NEEDED_COUNT2_HI__NUM_PRIM_NEEDED_CNT2_HI_MASK 0xffffffffL
+#define CP_NUM_PRIM_NEEDED_COUNT2_HI__NUM_PRIM_NEEDED_CNT2_HI__SHIFT 0x00000000
+#define CP_NUM_PRIM_NEEDED_COUNT2_LO__NUM_PRIM_NEEDED_CNT2_LO_MASK 0xffffffffL
+#define CP_NUM_PRIM_NEEDED_COUNT2_LO__NUM_PRIM_NEEDED_CNT2_LO__SHIFT 0x00000000
+#define CP_NUM_PRIM_NEEDED_COUNT3_HI__NUM_PRIM_NEEDED_CNT3_HI_MASK 0xffffffffL
+#define CP_NUM_PRIM_NEEDED_COUNT3_HI__NUM_PRIM_NEEDED_CNT3_HI__SHIFT 0x00000000
+#define CP_NUM_PRIM_NEEDED_COUNT3_LO__NUM_PRIM_NEEDED_CNT3_LO_MASK 0xffffffffL
+#define CP_NUM_PRIM_NEEDED_COUNT3_LO__NUM_PRIM_NEEDED_CNT3_LO__SHIFT 0x00000000
+#define CP_NUM_PRIM_WRITTEN_COUNT0_HI__NUM_PRIM_WRITTEN_CNT0_HI_MASK 0xffffffffL
+#define CP_NUM_PRIM_WRITTEN_COUNT0_HI__NUM_PRIM_WRITTEN_CNT0_HI__SHIFT 0x00000000
+#define CP_NUM_PRIM_WRITTEN_COUNT0_LO__NUM_PRIM_WRITTEN_CNT0_LO_MASK 0xffffffffL
+#define CP_NUM_PRIM_WRITTEN_COUNT0_LO__NUM_PRIM_WRITTEN_CNT0_LO__SHIFT 0x00000000
+#define CP_NUM_PRIM_WRITTEN_COUNT1_HI__NUM_PRIM_WRITTEN_CNT1_HI_MASK 0xffffffffL
+#define CP_NUM_PRIM_WRITTEN_COUNT1_HI__NUM_PRIM_WRITTEN_CNT1_HI__SHIFT 0x00000000
+#define CP_NUM_PRIM_WRITTEN_COUNT1_LO__NUM_PRIM_WRITTEN_CNT1_LO_MASK 0xffffffffL
+#define CP_NUM_PRIM_WRITTEN_COUNT1_LO__NUM_PRIM_WRITTEN_CNT1_LO__SHIFT 0x00000000
+#define CP_NUM_PRIM_WRITTEN_COUNT2_HI__NUM_PRIM_WRITTEN_CNT2_HI_MASK 0xffffffffL
+#define CP_NUM_PRIM_WRITTEN_COUNT2_HI__NUM_PRIM_WRITTEN_CNT2_HI__SHIFT 0x00000000
+#define CP_NUM_PRIM_WRITTEN_COUNT2_LO__NUM_PRIM_WRITTEN_CNT2_LO_MASK 0xffffffffL
+#define CP_NUM_PRIM_WRITTEN_COUNT2_LO__NUM_PRIM_WRITTEN_CNT2_LO__SHIFT 0x00000000
+#define CP_NUM_PRIM_WRITTEN_COUNT3_HI__NUM_PRIM_WRITTEN_CNT3_HI_MASK 0xffffffffL
+#define CP_NUM_PRIM_WRITTEN_COUNT3_HI__NUM_PRIM_WRITTEN_CNT3_HI__SHIFT 0x00000000
+#define CP_NUM_PRIM_WRITTEN_COUNT3_LO__NUM_PRIM_WRITTEN_CNT3_LO_MASK 0xffffffffL
+#define CP_NUM_PRIM_WRITTEN_COUNT3_LO__NUM_PRIM_WRITTEN_CNT3_LO__SHIFT 0x00000000
+#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI_MASK 0xffffffffL
+#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI__SHIFT 0x00000000
+#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO_MASK 0xffffffffL
+#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO__SHIFT 0x00000000
+#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI_MASK 0xffffffffL
+#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI__SHIFT 0x00000000
+#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO_MASK 0xffffffffL
+#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO__SHIFT 0x00000000
+#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE_MASK 0x00000300L
+#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE__SHIFT 0x00000008
+#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK 0x00000400L
+#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT 0x0000000a
+#define CP_PERFMON_CNTL__PERFMON_STATE_MASK 0x0000000fL
+#define CP_PERFMON_CNTL__PERFMON_STATE__SHIFT 0x00000000
+#define CP_PERFMON_CNTL__SPM_PERFMON_STATE_MASK 0x000000f0L
+#define CP_PERFMON_CNTL__SPM_PERFMON_STATE__SHIFT 0x00000004
+#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE_MASK 0x80000000L
+#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE__SHIFT 0x0000001f
+#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP_MASK 0xffffffffL
+#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP__SHIFT 0x00000000
+#define CP_PFP_IB_CONTROL__IB_EN_MASK 0x00000001L
+#define CP_PFP_IB_CONTROL__IB_EN__SHIFT 0x00000000
+#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN_MASK 0x00000002L
+#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN__SHIFT 0x00000001
+#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN_MASK 0x00000001L
+#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN__SHIFT 0x00000000
+#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN_MASK 0x01000000L
+#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN__SHIFT 0x00000018
+#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN_MASK 0x00010000L
+#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN__SHIFT 0x00000010
+#define CP_PFP_LOAD_CONTROL__UCONFIG_REG_EN_MASK 0x00008000L
+#define CP_PFP_LOAD_CONTROL__UCONFIG_REG_EN__SHIFT 0x0000000f
+#define CP_PFP_UCODE_ADDR__UCODE_ADDR_MASK 0x00000fffL
+#define CP_PFP_UCODE_ADDR__UCODE_ADDR__SHIFT 0x00000000
+#define CP_PFP_UCODE_DATA__UCODE_DATA_MASK 0xffffffffL
+#define CP_PFP_UCODE_DATA__UCODE_DATA__SHIFT 0x00000000
+#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI_MASK 0xffffffffL
+#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI__SHIFT 0x00000000
+#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO_MASK 0xfffffffcL
+#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO__SHIFT 0x00000002
+#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_SWAP_MASK 0x00000003L
+#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_SWAP__SHIFT 0x00000000
+#define CP_PWR_CNTL__GFX_CLK_HALT_MASK 0x00000001L
+#define CP_PWR_CNTL__GFX_CLK_HALT__SHIFT 0x00000000
+#define CP_QUEUE_THRESHOLDS__ROQ_IB1_START_MASK 0x0000003fL
+#define CP_QUEUE_THRESHOLDS__ROQ_IB1_START__SHIFT 0x00000000
+#define CP_QUEUE_THRESHOLDS__ROQ_IB2_START_MASK 0x00003f00L
+#define CP_QUEUE_THRESHOLDS__ROQ_IB2_START__SHIFT 0x00000008
+#define CP_RB0_BASE_HI__RB_BASE_HI_MASK 0x000000ffL
+#define CP_RB0_BASE_HI__RB_BASE_HI__SHIFT 0x00000000
+#define CP_RB0_BASE__RB_BASE_MASK 0xffffffffL
+#define CP_RB0_BASE__RB_BASE__SHIFT 0x00000000
+#define CP_RB0_CNTL__BUF_SWAP_MASK 0x00030000L
+#define CP_RB0_CNTL__BUF_SWAP__SHIFT 0x00000010
+#define CP_RB0_CNTL__CACHE_POLICY_MASK 0x03000000L
+#define CP_RB0_CNTL__CACHE_POLICY__SHIFT 0x00000018
+#define CP_RB0_CNTL__MIN_AVAILSZ_MASK 0x00300000L
+#define CP_RB0_CNTL__MIN_AVAILSZ__SHIFT 0x00000014
+#define CP_RB0_CNTL__MIN_IB_AVAILSZ_MASK 0x00c00000L
+#define CP_RB0_CNTL__MIN_IB_AVAILSZ__SHIFT 0x00000016
+#define CP_RB0_CNTL__RB_BLKSZ_MASK 0x00003f00L
+#define CP_RB0_CNTL__RB_BLKSZ__SHIFT 0x00000008
+#define CP_RB0_CNTL__RB_BUFSZ_MASK 0x0000003fL
+#define CP_RB0_CNTL__RB_BUFSZ__SHIFT 0x00000000
+#define CP_RB0_CNTL__RB_NO_UPDATE_MASK 0x08000000L
+#define CP_RB0_CNTL__RB_NO_UPDATE__SHIFT 0x0000001b
+#define CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK 0x80000000L
+#define CP_RB0_CNTL__RB_RPTR_WR_ENA__SHIFT 0x0000001f
+#define CP_RB0_CNTL__RB_VOLATILE_MASK 0x04000000L
+#define CP_RB0_CNTL__RB_VOLATILE__SHIFT 0x0000001a
+#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK 0x000000ffL
+#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT 0x00000000
+#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xfffffffcL
+#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x00000002
+#define CP_RB0_RPTR_ADDR__RB_RPTR_SWAP_MASK 0x00000003L
+#define CP_RB0_RPTR_ADDR__RB_RPTR_SWAP__SHIFT 0x00000000
+#define CP_RB0_RPTR__RB_RPTR_MASK 0x000fffffL
+#define CP_RB0_RPTR__RB_RPTR__SHIFT 0x00000000
+#define CP_RB0_WPTR__RB_WPTR_MASK 0x000fffffL
+#define CP_RB0_WPTR__RB_WPTR__SHIFT 0x00000000
+#define CP_RB1_BASE_HI__RB_BASE_HI_MASK 0x000000ffL
+#define CP_RB1_BASE_HI__RB_BASE_HI__SHIFT 0x00000000
+#define CP_RB1_BASE__RB_BASE_MASK 0xffffffffL
+#define CP_RB1_BASE__RB_BASE__SHIFT 0x00000000
+#define CP_RB1_CNTL__CACHE_POLICY_MASK 0x03000000L
+#define CP_RB1_CNTL__CACHE_POLICY__SHIFT 0x00000018
+#define CP_RB1_CNTL__MIN_AVAILSZ_MASK 0x00300000L
+#define CP_RB1_CNTL__MIN_AVAILSZ__SHIFT 0x00000014
+#define CP_RB1_CNTL__MIN_IB_AVAILSZ_MASK 0x00c00000L
+#define CP_RB1_CNTL__MIN_IB_AVAILSZ__SHIFT 0x00000016
+#define CP_RB1_CNTL__RB_BLKSZ_MASK 0x00003f00L
+#define CP_RB1_CNTL__RB_BLKSZ__SHIFT 0x00000008
+#define CP_RB1_CNTL__RB_BUFSZ_MASK 0x0000003fL
+#define CP_RB1_CNTL__RB_BUFSZ__SHIFT 0x00000000
+#define CP_RB1_CNTL__RB_NO_UPDATE_MASK 0x08000000L
+#define CP_RB1_CNTL__RB_NO_UPDATE__SHIFT 0x0000001b
+#define CP_RB1_CNTL__RB_RPTR_WR_ENA_MASK 0x80000000L
+#define CP_RB1_CNTL__RB_RPTR_WR_ENA__SHIFT 0x0000001f
+#define CP_RB1_CNTL__RB_VOLATILE_MASK 0x04000000L
+#define CP_RB1_CNTL__RB_VOLATILE__SHIFT 0x0000001a
+#define CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK 0x000000ffL
+#define CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT 0x00000000
+#define CP_RB1_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xfffffffcL
+#define CP_RB1_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x00000002
+#define CP_RB1_RPTR_ADDR__RB_RPTR_SWAP_MASK 0x00000003L
+#define CP_RB1_RPTR_ADDR__RB_RPTR_SWAP__SHIFT 0x00000000
+#define CP_RB1_RPTR__RB_RPTR_MASK 0x000fffffL
+#define CP_RB1_RPTR__RB_RPTR__SHIFT 0x00000000
+#define CP_RB1_WPTR__RB_WPTR_MASK 0x000fffffL
+#define CP_RB1_WPTR__RB_WPTR__SHIFT 0x00000000
+#define CP_RB2_BASE__RB_BASE_MASK 0xffffffffL
+#define CP_RB2_BASE__RB_BASE__SHIFT 0x00000000
+#define CP_RB2_CNTL__CACHE_POLICY_MASK 0x03000000L
+#define CP_RB2_CNTL__CACHE_POLICY__SHIFT 0x00000018
+#define CP_RB2_CNTL__MIN_AVAILSZ_MASK 0x00300000L
+#define CP_RB2_CNTL__MIN_AVAILSZ__SHIFT 0x00000014
+#define CP_RB2_CNTL__MIN_IB_AVAILSZ_MASK 0x00c00000L
+#define CP_RB2_CNTL__MIN_IB_AVAILSZ__SHIFT 0x00000016
+#define CP_RB2_CNTL__RB_BLKSZ_MASK 0x00003f00L
+#define CP_RB2_CNTL__RB_BLKSZ__SHIFT 0x00000008
+#define CP_RB2_CNTL__RB_BUFSZ_MASK 0x0000003fL
+#define CP_RB2_CNTL__RB_BUFSZ__SHIFT 0x00000000
+#define CP_RB2_CNTL__RB_NO_UPDATE_MASK 0x08000000L
+#define CP_RB2_CNTL__RB_NO_UPDATE__SHIFT 0x0000001b
+#define CP_RB2_CNTL__RB_RPTR_WR_ENA_MASK 0x80000000L
+#define CP_RB2_CNTL__RB_RPTR_WR_ENA__SHIFT 0x0000001f
+#define CP_RB2_CNTL__RB_VOLATILE_MASK 0x04000000L
+#define CP_RB2_CNTL__RB_VOLATILE__SHIFT 0x0000001a
+#define CP_RB2_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK 0x000000ffL
+#define CP_RB2_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT 0x00000000
+#define CP_RB2_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xfffffffcL
+#define CP_RB2_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x00000002
+#define CP_RB2_RPTR_ADDR__RB_RPTR_SWAP_MASK 0x00000003L
+#define CP_RB2_RPTR_ADDR__RB_RPTR_SWAP__SHIFT 0x00000000
+#define CP_RB2_RPTR__RB_RPTR_MASK 0x000fffffL
+#define CP_RB2_RPTR__RB_RPTR__SHIFT 0x00000000
+#define CP_RB2_WPTR__RB_WPTR_MASK 0x000fffffL
+#define CP_RB2_WPTR__RB_WPTR__SHIFT 0x00000000
+#define CP_RB_BASE__RB_BASE_MASK 0xffffffffL
+#define CP_RB_BASE__RB_BASE__SHIFT 0x00000000
+#define CP_RB_CNTL__BUF_SWAP_MASK 0x00030000L
+#define CP_RB_CNTL__BUF_SWAP__SHIFT 0x00000010
+#define CP_RB_CNTL__CACHE_POLICY_MASK 0x03000000L
+#define CP_RB_CNTL__CACHE_POLICY__SHIFT 0x00000018
+#define CP_RB_CNTL__MIN_AVAILSZ_MASK 0x00300000L
+#define CP_RB_CNTL__MIN_AVAILSZ__SHIFT 0x00000014
+#define CP_RB_CNTL__MIN_IB_AVAILSZ_MASK 0x00c00000L
+#define CP_RB_CNTL__MIN_IB_AVAILSZ__SHIFT 0x00000016
+#define CP_RB_CNTL__RB_BLKSZ_MASK 0x00003f00L
+#define CP_RB_CNTL__RB_BLKSZ__SHIFT 0x00000008
+#define CP_RB_CNTL__RB_BUFSZ_MASK 0x0000003fL
+#define CP_RB_CNTL__RB_BUFSZ__SHIFT 0x00000000
+#define CP_RB_CNTL__RB_NO_UPDATE_MASK 0x08000000L
+#define CP_RB_CNTL__RB_NO_UPDATE__SHIFT 0x0000001b
+#define CP_RB_CNTL__RB_RPTR_WR_ENA_MASK 0x80000000L
+#define CP_RB_CNTL__RB_RPTR_WR_ENA__SHIFT 0x0000001f
+#define CP_RB_CNTL__RB_VOLATILE_MASK 0x04000000L
+#define CP_RB_CNTL__RB_VOLATILE__SHIFT 0x0000001a
+#define CP_RB_OFFSET__RB_OFFSET_MASK 0x000fffffL
+#define CP_RB_OFFSET__RB_OFFSET__SHIFT 0x00000000
+#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK 0x000000ffL
+#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT 0x00000000
+#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xfffffffcL
+#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x00000002
+#define CP_RB_RPTR_ADDR__RB_RPTR_SWAP_MASK 0x00000003L
+#define CP_RB_RPTR_ADDR__RB_RPTR_SWAP__SHIFT 0x00000000
+#define CP_RB_RPTR__RB_RPTR_MASK 0x000fffffL
+#define CP_RB_RPTR__RB_RPTR__SHIFT 0x00000000
+#define CP_RB_RPTR_WR__RB_RPTR_WR_MASK 0x000fffffL
+#define CP_RB_RPTR_WR__RB_RPTR_WR__SHIFT 0x00000000
+#define CP_RB_VMID__RB0_VMID_MASK 0x0000000fL
+#define CP_RB_VMID__RB0_VMID__SHIFT 0x00000000
+#define CP_RB_VMID__RB1_VMID_MASK 0x00000f00L
+#define CP_RB_VMID__RB1_VMID__SHIFT 0x00000008
+#define CP_RB_VMID__RB2_VMID_MASK 0x000f0000L
+#define CP_RB_VMID__RB2_VMID__SHIFT 0x00000010
+#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT_MASK 0xf0000000L
+#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT__SHIFT 0x0000001c
+#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER_MASK 0x0fffffffL
+#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER__SHIFT 0x00000000
+#define CP_RB_WPTR_POLL_ADDR_HI__OBSOLETE_MASK 0x000000ffL
+#define CP_RB_WPTR_POLL_ADDR_HI__OBSOLETE__SHIFT 0x00000000
+#define CP_RB_WPTR_POLL_ADDR_HI__RB_WPTR_POLL_ADDR_HI_MASK 0x000000ffL
+#define CP_RB_WPTR_POLL_ADDR_HI__RB_WPTR_POLL_ADDR_HI__SHIFT 0x00000000
+#define CP_RB_WPTR_POLL_ADDR_LO__OBSOLETE_MASK 0xfffffffcL
+#define CP_RB_WPTR_POLL_ADDR_LO__OBSOLETE__SHIFT 0x00000002
+#define CP_RB_WPTR_POLL_ADDR_LO__RB_WPTR_POLL_ADDR_LO_MASK 0xfffffffcL
+#define CP_RB_WPTR_POLL_ADDR_LO__RB_WPTR_POLL_ADDR_LO__SHIFT 0x00000002
+#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK 0xffff0000L
+#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT 0x00000010
+#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK 0x0000ffffL
+#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT 0x00000000
+#define CP_RB_WPTR__RB_WPTR_MASK 0x000fffffL
+#define CP_RB_WPTR__RB_WPTR__SHIFT 0x00000000
+#define CP_RING0_PRIORITY__PRIORITY_MASK 0x00000003L
+#define CP_RING0_PRIORITY__PRIORITY__SHIFT 0x00000000
+#define CP_RING1_PRIORITY__PRIORITY_MASK 0x00000003L
+#define CP_RING1_PRIORITY__PRIORITY__SHIFT 0x00000000
+#define CP_RING2_PRIORITY__PRIORITY_MASK 0x00000003L
+#define CP_RING2_PRIORITY__PRIORITY__SHIFT 0x00000000
+#define CP_RINGID__RINGID_MASK 0x00000003L
+#define CP_RINGID__RINGID__SHIFT 0x00000000
+#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT_MASK 0x000000ffL
+#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT 0x00000000
+#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT_MASK 0x0000ff00L
+#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT 0x00000008
+#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT_MASK 0x00ff0000L
+#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT 0x00000010
+#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT_MASK 0xff000000L
+#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT 0x00000018
+#define CP_ROQ1_THRESHOLDS__R0_IB1_START_MASK 0x00ff0000L
+#define CP_ROQ1_THRESHOLDS__R0_IB1_START__SHIFT 0x00000010
+#define CP_ROQ1_THRESHOLDS__R1_IB1_START_MASK 0xff000000L
+#define CP_ROQ1_THRESHOLDS__R1_IB1_START__SHIFT 0x00000018
+#define CP_ROQ1_THRESHOLDS__RB1_START_MASK 0x000000ffL
+#define CP_ROQ1_THRESHOLDS__RB1_START__SHIFT 0x00000000
+#define CP_ROQ1_THRESHOLDS__RB2_START_MASK 0x0000ff00L
+#define CP_ROQ1_THRESHOLDS__RB2_START__SHIFT 0x00000008
+#define CP_ROQ2_AVAIL__ROQ_CNT_IB2_MASK 0x000007ffL
+#define CP_ROQ2_AVAIL__ROQ_CNT_IB2__SHIFT 0x00000000
+#define CP_ROQ2_THRESHOLDS__R0_IB2_START_MASK 0x0000ff00L
+#define CP_ROQ2_THRESHOLDS__R0_IB2_START__SHIFT 0x00000008
+#define CP_ROQ2_THRESHOLDS__R1_IB2_START_MASK 0x00ff0000L
+#define CP_ROQ2_THRESHOLDS__R1_IB2_START__SHIFT 0x00000010
+#define CP_ROQ2_THRESHOLDS__R2_IB1_START_MASK 0x000000ffL
+#define CP_ROQ2_THRESHOLDS__R2_IB1_START__SHIFT 0x00000000
+#define CP_ROQ2_THRESHOLDS__R2_IB2_START_MASK 0xff000000L
+#define CP_ROQ2_THRESHOLDS__R2_IB2_START__SHIFT 0x00000018
+#define CP_ROQ_AVAIL__ROQ_CNT_IB1_MASK 0x07ff0000L
+#define CP_ROQ_AVAIL__ROQ_CNT_IB1__SHIFT 0x00000010
+#define CP_ROQ_AVAIL__ROQ_CNT_RING_MASK 0x000007ffL
+#define CP_ROQ_AVAIL__ROQ_CNT_RING__SHIFT 0x00000000
+#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1_MASK 0x000003ffL
+#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1__SHIFT 0x00000000
+#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1_MASK 0x03ff0000L
+#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1__SHIFT 0x00000010
+#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2_MASK 0x000003ffL
+#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2__SHIFT 0x00000000
+#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2_MASK 0x03ff0000L
+#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2__SHIFT 0x00000010
+#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY_MASK 0x000003ffL
+#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY__SHIFT 0x00000000
+#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY_MASK 0x03ff0000L
+#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY__SHIFT 0x00000010
+#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI_MASK 0xffffffffL
+#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI__SHIFT 0x00000000
+#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO_MASK 0xffffffffL
+#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO__SHIFT 0x00000000
+#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE_MASK 0xffffffffL
+#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE__SHIFT 0x00000000
+#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE_MASK 0xffffffffL
+#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE__SHIFT 0x00000000
+#define CP_SCRATCH_DATA__SCRATCH_DATA_MASK 0xffffffffL
+#define CP_SCRATCH_DATA__SCRATCH_DATA__SHIFT 0x00000000
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX_MASK 0x000000ffL
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT 0x00000000
+#define CP_SEM_WAIT_TIMER__SEM_WAIT_TIMER_MASK 0xffffffffL
+#define CP_SEM_WAIT_TIMER__SEM_WAIT_TIMER__SHIFT 0x00000000
+#define CP_SIG_SEM_ADDR_HI__SEM_ADDR_HI_MASK 0x000000ffL
+#define CP_SIG_SEM_ADDR_HI__SEM_ADDR_HI__SHIFT 0x00000000
+#define CP_SIG_SEM_ADDR_HI__SEM_CLIENT_CODE_MASK 0x03000000L
+#define CP_SIG_SEM_ADDR_HI__SEM_CLIENT_CODE__SHIFT 0x00000018
+#define CP_SIG_SEM_ADDR_HI__SEM_SELECT_MASK 0xe0000000L
+#define CP_SIG_SEM_ADDR_HI__SEM_SELECT__SHIFT 0x0000001d
+#define CP_SIG_SEM_ADDR_HI__SEM_SIGNAL_TYPE_MASK 0x00100000L
+#define CP_SIG_SEM_ADDR_HI__SEM_SIGNAL_TYPE__SHIFT 0x00000014
+#define CP_SIG_SEM_ADDR_HI__SEM_USE_MAILBOX_MASK 0x00010000L
+#define CP_SIG_SEM_ADDR_HI__SEM_USE_MAILBOX__SHIFT 0x00000010
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_LO_MASK 0xfffffff8L
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_LO__SHIFT 0x00000003
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_SWAP_MASK 0x00000003L
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_SWAP__SHIFT 0x00000000
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG_MASK 0x00000400L
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG__SHIFT 0x0000000a
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG_MASK 0x00000800L
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG__SHIFT 0x0000000b
+#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA_MASK 0x00002000L
+#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA__SHIFT 0x0000000d
+#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM_MASK 0x00001000L
+#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM__SHIFT 0x0000000c
+#define CP_STALLED_STAT1__ME_WAITING_ON_MC_READ_DATA_MASK 0x00004000L
+#define CP_STALLED_STAT1__ME_WAITING_ON_MC_READ_DATA__SHIFT 0x0000000e
+#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA_MASK 0x00008000L
+#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA__SHIFT 0x0000000f
+#define CP_STALLED_STAT1__MIU_WAITING_ON_RDREQ_FREE_MASK 0x00010000L
+#define CP_STALLED_STAT1__MIU_WAITING_ON_RDREQ_FREE__SHIFT 0x00000010
+#define CP_STALLED_STAT1__MIU_WAITING_ON_WRREQ_FREE_MASK 0x00020000L
+#define CP_STALLED_STAT1__MIU_WAITING_ON_WRREQ_FREE__SHIFT 0x00000011
+#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV_MASK 0x00000001L
+#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV__SHIFT 0x00000000
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_MASK 0x00000010L
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV__SHIFT 0x00000004
+#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV_MASK 0x00000004L
+#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV__SHIFT 0x00000002
+#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION_MASK 0x10000000L
+#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION__SHIFT 0x0000001c
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ_MASK 0x10000000L
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ__SHIFT 0x0000001c
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ_MASK 0x08000000L
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ__SHIFT 0x0000001b
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ_MASK 0x04000000L
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ__SHIFT 0x0000001a
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE_MASK 0x00800000L
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE__SHIFT 0x00000017
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE_MASK 0x01000000L
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE__SHIFT 0x00000018
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE_MASK 0x02000000L
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE__SHIFT 0x00000019
+#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION_MASK 0x10000000L
+#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION__SHIFT 0x0000001c
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE_MASK 0x02000000L
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE__SHIFT 0x00000019
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE_MASK 0x04000000L
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE__SHIFT 0x0000001a
+#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE_MASK 0x20000000L
+#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE__SHIFT 0x0000001d
+#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM_MASK 0x08000000L
+#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM__SHIFT 0x0000001b
+#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_SC_EOP_DONE_MASK 0x00200000L
+#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_SC_EOP_DONE__SHIFT 0x00000015
+#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_WR_CONFIRM_MASK 0x00400000L
+#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_WR_CONFIRM__SHIFT 0x00000016
+#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME_MASK 0x00000800L
+#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME__SHIFT 0x0000000b
+#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV_MASK 0x00010000L
+#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV__SHIFT 0x00000010
+#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV_MASK 0x00001000L
+#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV__SHIFT 0x0000000c
+#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV_MASK 0x00002000L
+#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV__SHIFT 0x0000000d
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP_MASK 0x00004000L
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP__SHIFT 0x0000000e
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ_MASK 0x00040000L
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ__SHIFT 0x00000012
+#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH_MASK 0x00008000L
+#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH__SHIFT 0x0000000f
+#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER_MASK 0x00000400L
+#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER__SHIFT 0x0000000a
+#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER_MASK 0x00000200L
+#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER__SHIFT 0x00000009
+#define CP_STALLED_STAT2__PFP_MIU_READ_PENDING_MASK 0x00000040L
+#define CP_STALLED_STAT2__PFP_MIU_READ_PENDING__SHIFT 0x00000006
+#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING_MASK 0x00000020L
+#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING__SHIFT 0x00000005
+#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA_MASK 0x00100000L
+#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA__SHIFT 0x00000014
+#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM_MASK 0x00080000L
+#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM__SHIFT 0x00000013
+#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV_MASK 0x00000001L
+#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV__SHIFT 0x00000000
+#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV_MASK 0x00000002L
+#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV__SHIFT 0x00000001
+#define CP_STALLED_STAT2__PFP_TO_MIU_WRITE_NOT_RDY_TO_RCV_MASK 0x00000080L
+#define CP_STALLED_STAT2__PFP_TO_MIU_WRITE_NOT_RDY_TO_RCV__SHIFT 0x00000007
+#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV_MASK 0x00000004L
+#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV__SHIFT 0x00000002
+#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING_MASK 0x00000010L
+#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING__SHIFT 0x00000004
+#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA_MASK 0x00000100L
+#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA__SHIFT 0x00000008
+#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING_MASK 0x01000000L
+#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING__SHIFT 0x00000018
+#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV_MASK 0x00020000L
+#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV__SHIFT 0x00000011
+#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING_MASK 0x00800000L
+#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING__SHIFT 0x00000017
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN_MASK 0x80000000L
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN__SHIFT 0x0000001f
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS_MASK 0x40000000L
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS__SHIFT 0x0000001e
+#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV_MASK 0x00000001L
+#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV__SHIFT 0x00000000
+#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV_MASK 0x00000040L
+#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV__SHIFT 0x00000006
+#define CP_STALLED_STAT3__CE_TO_MIU_WRITE_NOT_RDY_TO_RCV_MASK 0x00000100L
+#define CP_STALLED_STAT3__CE_TO_MIU_WRITE_NOT_RDY_TO_RCV__SHIFT 0x00000008
+#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY_MASK 0x00000010L
+#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY__SHIFT 0x00000004
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV_MASK 0x00000002L
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV__SHIFT 0x00000001
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY_MASK 0x00000008L
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY__SHIFT 0x00000003
+#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY_MASK 0x00000020L
+#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY__SHIFT 0x00000005
+#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV_MASK 0x00000080L
+#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV__SHIFT 0x00000007
+#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA_MASK 0x00000400L
+#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA__SHIFT 0x0000000a
+#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG_MASK 0x00000800L
+#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG__SHIFT 0x0000000b
+#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER_MASK 0x00000004L
+#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER__SHIFT 0x00000002
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_MASK 0x00001000L
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER__SHIFT 0x0000000c
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW_MASK 0x00002000L
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW__SHIFT 0x0000000d
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE_MASK 0x00004000L
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE__SHIFT 0x0000000e
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS_MASK 0x00008000L
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS__SHIFT 0x0000000f
+#define CP_STAT__CE_BUSY_MASK 0x04000000L
+#define CP_STAT__CE_BUSY__SHIFT 0x0000001a
+#define CP_STAT__CP_BUSY_MASK 0x80000000L
+#define CP_STAT__CP_BUSY__SHIFT 0x0000001f
+#define CP_STAT__CPC_CPG_BUSY_MASK 0x02000000L
+#define CP_STAT__CPC_CPG_BUSY__SHIFT 0x00000019
+#define CP_STAT__DC_BUSY_MASK 0x00002000L
+#define CP_STAT__DC_BUSY__SHIFT 0x0000000d
+#define CP_STAT__DMA_BUSY_MASK 0x00400000L
+#define CP_STAT__DMA_BUSY__SHIFT 0x00000016
+#define CP_STAT__INTERRUPT_BUSY_MASK 0x00100000L
+#define CP_STAT__INTERRUPT_BUSY__SHIFT 0x00000014
+#define CP_STAT__ME_BUSY_MASK 0x00020000L
+#define CP_STAT__ME_BUSY__SHIFT 0x00000011
+#define CP_STAT__MEQ_BUSY_MASK 0x00010000L
+#define CP_STAT__MEQ_BUSY__SHIFT 0x00000010
+#define CP_STAT__MIU_RDREQ_BUSY_MASK 0x00000080L
+#define CP_STAT__MIU_RDREQ_BUSY__SHIFT 0x00000007
+#define CP_STAT__MIU_WRREQ_BUSY_MASK 0x00000100L
+#define CP_STAT__MIU_WRREQ_BUSY__SHIFT 0x00000008
+#define CP_STAT__PFP_BUSY_MASK 0x00008000L
+#define CP_STAT__PFP_BUSY__SHIFT 0x0000000f
+#define CP_STAT__QUERY_BUSY_MASK 0x00040000L
+#define CP_STAT__QUERY_BUSY__SHIFT 0x00000012
+#define CP_STAT__RCIU_BUSY_MASK 0x00800000L
+#define CP_STAT__RCIU_BUSY__SHIFT 0x00000017
+#define CP_STAT__ROQ_CE_INDIRECT1_BUSY_MASK 0x20000000L
+#define CP_STAT__ROQ_CE_INDIRECT1_BUSY__SHIFT 0x0000001d
+#define CP_STAT__ROQ_CE_INDIRECT2_BUSY_MASK 0x40000000L
+#define CP_STAT__ROQ_CE_INDIRECT2_BUSY__SHIFT 0x0000001e
+#define CP_STAT__ROQ_CE_RING_BUSY_MASK 0x10000000L
+#define CP_STAT__ROQ_CE_RING_BUSY__SHIFT 0x0000001c
+#define CP_STAT__ROQ_INDIRECT1_BUSY_MASK 0x00000400L
+#define CP_STAT__ROQ_INDIRECT1_BUSY__SHIFT 0x0000000a
+#define CP_STAT__ROQ_INDIRECT2_BUSY_MASK 0x00000800L
+#define CP_STAT__ROQ_INDIRECT2_BUSY__SHIFT 0x0000000b
+#define CP_STAT__ROQ_RING_BUSY_MASK 0x00000200L
+#define CP_STAT__ROQ_RING_BUSY__SHIFT 0x00000009
+#define CP_STAT__ROQ_STATE_BUSY_MASK 0x00001000L
+#define CP_STAT__ROQ_STATE_BUSY__SHIFT 0x0000000c
+#define CP_STAT__SCRATCH_RAM_BUSY_MASK 0x01000000L
+#define CP_STAT__SCRATCH_RAM_BUSY__SHIFT 0x00000018
+#define CP_STAT__SEMAPHORE_BUSY_MASK 0x00080000L
+#define CP_STAT__SEMAPHORE_BUSY__SHIFT 0x00000013
+#define CP_STAT__SURFACE_SYNC_BUSY_MASK 0x00200000L
+#define CP_STAT__SURFACE_SYNC_BUSY__SHIFT 0x00000015
+#define CP_STAT__TCIU_BUSY_MASK 0x08000000L
+#define CP_STAT__TCIU_BUSY__SHIFT 0x0000001b
+#define CP_ST_BASE_HI__ST_BASE_HI_MASK 0x000000ffL
+#define CP_ST_BASE_HI__ST_BASE_HI__SHIFT 0x00000000
+#define CP_ST_BASE_LO__ST_BASE_LO_MASK 0xfffffffcL
+#define CP_ST_BASE_LO__ST_BASE_LO__SHIFT 0x00000002
+#define CP_ST_BUFSZ__ST_BUFSZ_MASK 0x000fffffL
+#define CP_ST_BUFSZ__ST_BUFSZ__SHIFT 0x00000000
+#define CP_STQ_AVAIL__STQ_CNT_MASK 0x000001ffL
+#define CP_STQ_AVAIL__STQ_CNT__SHIFT 0x00000000
+#define CP_STQ_STAT__STQ_RPTR_MASK 0x000003ffL
+#define CP_STQ_STAT__STQ_RPTR__SHIFT 0x00000000
+#define CP_STQ_THRESHOLDS__STQ0_START_MASK 0x000000ffL
+#define CP_STQ_THRESHOLDS__STQ0_START__SHIFT 0x00000000
+#define CP_STQ_THRESHOLDS__STQ1_START_MASK 0x0000ff00L
+#define CP_STQ_THRESHOLDS__STQ1_START__SHIFT 0x00000008
+#define CP_STQ_THRESHOLDS__STQ2_START_MASK 0x00ff0000L
+#define CP_STQ_THRESHOLDS__STQ2_START__SHIFT 0x00000010
+#define CP_STREAM_OUT_ADDR_HI__STREAM_OUT_ADDR_HI_MASK 0xffffffffL
+#define CP_STREAM_OUT_ADDR_HI__STREAM_OUT_ADDR_HI__SHIFT 0x00000000
+#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_LO_MASK 0xfffffffcL
+#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_LO__SHIFT 0x00000002
+#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_SWAP_MASK 0x00000003L
+#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_SWAP__SHIFT 0x00000000
+#define CP_STRMOUT_CNTL__OFFSET_UPDATE_DONE_MASK 0x00000001L
+#define CP_STRMOUT_CNTL__OFFSET_UPDATE_DONE__SHIFT 0x00000000
+#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI_MASK 0xffffffffL
+#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI__SHIFT 0x00000000
+#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO_MASK 0xffffffffL
+#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO__SHIFT 0x00000000
+#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI_MASK 0xffffffffL
+#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI__SHIFT 0x00000000
+#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO_MASK 0xffffffffL
+#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO__SHIFT 0x00000000
+#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI_MASK 0xffffffffL
+#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI__SHIFT 0x00000000
+#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO_MASK 0xffffffffL
+#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO__SHIFT 0x00000000
+#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI_MASK 0xffffffffL
+#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI__SHIFT 0x00000000
+#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO_MASK 0xffffffffL
+#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO__SHIFT 0x00000000
+#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI_MASK 0xffffffffL
+#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI__SHIFT 0x00000000
+#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO_MASK 0xffffffffL
+#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO__SHIFT 0x00000000
+#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI_MASK 0xffffffffL
+#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI__SHIFT 0x00000000
+#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO_MASK 0xffffffffL
+#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO__SHIFT 0x00000000
+#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI_MASK 0xffffffffL
+#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI__SHIFT 0x00000000
+#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO_MASK 0xffffffffL
+#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO__SHIFT 0x00000000
+#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI_MASK 0xffffffffL
+#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI__SHIFT 0x00000000
+#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO_MASK 0xffffffffL
+#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO__SHIFT 0x00000000
+#define CP_VMID_PREEMPT__PREEMPT_REQUEST_MASK 0x0000ffffL
+#define CP_VMID_PREEMPT__PREEMPT_REQUEST__SHIFT 0x00000000
+#define CP_VMID_PREEMPT__PREEMPT_STATUS_MASK 0xffff0000L
+#define CP_VMID_PREEMPT__PREEMPT_STATUS__SHIFT 0x00000010
+#define CP_VMID_RESET__RESET_REQUEST_MASK 0x0000ffffL
+#define CP_VMID_RESET__RESET_REQUEST__SHIFT 0x00000000
+#define CP_VMID_RESET__RESET_STATUS_MASK 0xffff0000L
+#define CP_VMID_RESET__RESET_STATUS__SHIFT 0x00000010
+#define CP_VMID__VMID_MASK 0x0000000fL
+#define CP_VMID__VMID__SHIFT 0x00000000
+#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT_MASK 0xffffffffL
+#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT__SHIFT 0x00000000
+#define CP_WAIT_SEM_ADDR_HI__SEM_ADDR_HI_MASK 0x000000ffL
+#define CP_WAIT_SEM_ADDR_HI__SEM_ADDR_HI__SHIFT 0x00000000
+#define CP_WAIT_SEM_ADDR_HI__SEM_CLIENT_CODE_MASK 0x03000000L
+#define CP_WAIT_SEM_ADDR_HI__SEM_CLIENT_CODE__SHIFT 0x00000018
+#define CP_WAIT_SEM_ADDR_HI__SEM_SELECT_MASK 0xe0000000L
+#define CP_WAIT_SEM_ADDR_HI__SEM_SELECT__SHIFT 0x0000001d
+#define CP_WAIT_SEM_ADDR_HI__SEM_SIGNAL_TYPE_MASK 0x00100000L
+#define CP_WAIT_SEM_ADDR_HI__SEM_SIGNAL_TYPE__SHIFT 0x00000014
+#define CP_WAIT_SEM_ADDR_HI__SEM_USE_MAILBOX_MASK 0x00010000L
+#define CP_WAIT_SEM_ADDR_HI__SEM_USE_MAILBOX__SHIFT 0x00000010
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_LO_MASK 0xfffffff8L
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_LO__SHIFT 0x00000003
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_SWAP_MASK 0x00000003L
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_SWAP__SHIFT 0x00000000
+#define CS_COPY_STATE__SRC_STATE_ID_MASK 0x00000007L
+#define CS_COPY_STATE__SRC_STATE_ID__SHIFT 0x00000000
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE_MASK 0x00000001L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE__SHIFT 0x00000000
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0_MASK 0x00000300L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0__SHIFT 0x00000008
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1_MASK 0x00000c00L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1__SHIFT 0x0000000a
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2_MASK 0x00003000L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2__SHIFT 0x0000000c
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3_MASK 0x0000c000L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3__SHIFT 0x0000000e
+#define DB_ALPHA_TO_MASK__OFFSET_ROUND_MASK 0x00010000L
+#define DB_ALPHA_TO_MASK__OFFSET_ROUND__SHIFT 0x00000010
+#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS__SHIFT 0x00000004
+#define DB_CGTT_CLK_CTRL_0__ON_DELAY_MASK 0x0000000fL
+#define DB_CGTT_CLK_CTRL_0__ON_DELAY__SHIFT 0x00000000
+#define DB_CGTT_CLK_CTRL_0__RESERVED_MASK 0x00fff000L
+#define DB_CGTT_CLK_CTRL_0__RESERVED__SHIFT 0x0000000c
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0_MASK 0x80000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1_MASK 0x40000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2_MASK 0x20000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3_MASK 0x10000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4_MASK 0x08000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5_MASK 0x04000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6_MASK 0x02000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7_MASK 0x01000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define DB_COUNT_CONTROL__DBFAIL_ENABLE_MASK 0x00f00000L
+#define DB_COUNT_CONTROL__DBFAIL_ENABLE__SHIFT 0x00000014
+#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS_MASK 0x00000002L
+#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS__SHIFT 0x00000001
+#define DB_COUNT_CONTROL__SAMPLE_RATE_MASK 0x00000070L
+#define DB_COUNT_CONTROL__SAMPLE_RATE__SHIFT 0x00000004
+#define DB_COUNT_CONTROL__SFAIL_ENABLE_MASK 0x000f0000L
+#define DB_COUNT_CONTROL__SFAIL_ENABLE__SHIFT 0x00000010
+#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE_MASK 0x0f000000L
+#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE__SHIFT 0x00000018
+#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE_MASK 0xf0000000L
+#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE__SHIFT 0x0000001c
+#define DB_COUNT_CONTROL__ZFAIL_ENABLE_MASK 0x0000f000L
+#define DB_COUNT_CONTROL__ZFAIL_ENABLE__SHIFT 0x0000000c
+#define DB_COUNT_CONTROL__ZPASS_ENABLE_MASK 0x00000f00L
+#define DB_COUNT_CONTROL__ZPASS_ENABLE__SHIFT 0x00000008
+#define DB_COUNT_CONTROL__ZPASS_INCREMENT_DISABLE_MASK 0x00000001L
+#define DB_COUNT_CONTROL__ZPASS_INCREMENT_DISABLE__SHIFT 0x00000000
+#define DB_CREDIT_LIMIT__DB_CB_LQUAD_CREDITS_MASK 0x00001c00L
+#define DB_CREDIT_LIMIT__DB_CB_LQUAD_CREDITS__SHIFT 0x0000000a
+#define DB_CREDIT_LIMIT__DB_CB_TILE_CREDITS_MASK 0x7f000000L
+#define DB_CREDIT_LIMIT__DB_CB_TILE_CREDITS__SHIFT 0x00000018
+#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS_MASK 0x000003e0L
+#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS__SHIFT 0x00000005
+#define DB_CREDIT_LIMIT__DB_SC_TILE_CREDITS_MASK 0x0000001fL
+#define DB_CREDIT_LIMIT__DB_SC_TILE_CREDITS__SHIFT 0x00000000
+#define DB_DEBUG2__ALLOW_COMPZ_BYTE_MASKING_MASK 0x00000001L
+#define DB_DEBUG2__ALLOW_COMPZ_BYTE_MASKING__SHIFT 0x00000000
+#define DB_DEBUG2__CLK_OFF_DELAY_MASK 0x00003e00L
+#define DB_DEBUG2__CLK_OFF_DELAY__SHIFT 0x00000009
+#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING_MASK 0x00040000L
+#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING__SHIFT 0x00000012
+#define DB_DEBUG2__DISABLE_HTILE_PAIRED_PIPES_MASK 0x00010000L
+#define DB_DEBUG2__DISABLE_HTILE_PAIRED_PIPES__SHIFT 0x00000010
+#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING_MASK 0x00020000L
+#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING__SHIFT 0x00000011
+#define DB_DEBUG2__DISABLE_PREZL_CB_STALL_REZ_MASK 0x00000100L
+#define DB_DEBUG2__DISABLE_PREZL_CB_STALL_REZ__SHIFT 0x00000008
+#define DB_DEBUG2__DISABLE_PREZL_LPF_STALL_MASK 0x00000020L
+#define DB_DEBUG2__DISABLE_PREZL_LPF_STALL_REZ_MASK 0x00000080L
+#define DB_DEBUG2__DISABLE_PREZL_LPF_STALL_REZ__SHIFT 0x00000007
+#define DB_DEBUG2__DISABLE_PREZL_LPF_STALL__SHIFT 0x00000005
+#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL_MASK 0x20000000L
+#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL__SHIFT 0x0000001d
+#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL_MASK 0x00080000L
+#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL__SHIFT 0x00000013
+#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM_MASK 0x80000000L
+#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM__SHIFT 0x0000001f
+#define DB_DEBUG2__DISABLE_TC_MASK_L0_CACHE_MASK 0x00000004L
+#define DB_DEBUG2__DISABLE_TC_MASK_L0_CACHE__SHIFT 0x00000002
+#define DB_DEBUG2__DISABLE_TC_ZRANGE_L0_CACHE_MASK 0x00000002L
+#define DB_DEBUG2__DISABLE_TC_ZRANGE_L0_CACHE__SHIFT 0x00000001
+#define DB_DEBUG2__DISABLE_TILE_COVERED_FOR_PS_ITER_MASK 0x00004000L
+#define DB_DEBUG2__DISABLE_TILE_COVERED_FOR_PS_ITER__SHIFT 0x0000000e
+#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT_MASK 0x80000000L
+#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT__SHIFT 0x0000001f
+#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM_MASK 0x00000010L
+#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM__SHIFT 0x00000004
+#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB_MASK 0x00000008L
+#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB__SHIFT 0x00000003
+#define DB_DEBUG2__ENABLE_PREZL_CB_STALL_MASK 0x00000040L
+#define DB_DEBUG2__ENABLE_PREZL_CB_STALL__SHIFT 0x00000006
+#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM_MASK 0x10000000L
+#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM__SHIFT 0x0000001c
+#define DB_DEBUG2__ENABLE_SUBTILE_GROUPING_MASK 0x00008000L
+#define DB_DEBUG2__ENABLE_SUBTILE_GROUPING__SHIFT 0x0000000f
+#define DB_DEBUG3__ALLOW_RF2P_RW_COLLISION_MASK 0x00020000L
+#define DB_DEBUG3__ALLOW_RF2P_RW_COLLISION__SHIFT 0x00000011
+#define DB_DEBUG3__DB_EXTRA_DEBUG3_MASK 0xfc000000L
+#define DB_DEBUG3__DB_EXTRA_DEBUG3__SHIFT 0x0000001a
+#define DB_DEBUG3__DISABLE_DI_DT_STALL_MASK 0x02000000L
+#define DB_DEBUG3__DISABLE_DI_DT_STALL__SHIFT 0x00000019
+#define DB_DEBUG3__DISABLE_EQAA_A2M_PERF_OPT_MASK 0x01000000L
+#define DB_DEBUG3__DISABLE_EQAA_A2M_PERF_OPT__SHIFT 0x00000018
+#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP_MASK 0x00000010L
+#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP__SHIFT 0x00000004
+#define DB_DEBUG3__DISABLE_HZ_TC_WRITE_COMBINE_MASK 0x00200000L
+#define DB_DEBUG3__DISABLE_HZ_TC_WRITE_COMBINE__SHIFT 0x00000015
+#define DB_DEBUG3__DISABLE_OP_DF_BYPASS_MASK 0x00004000L
+#define DB_DEBUG3__DISABLE_OP_DF_BYPASS__SHIFT 0x0000000e
+#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK_MASK 0x00010000L
+#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK__SHIFT 0x00000010
+#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE_MASK 0x00008000L
+#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE__SHIFT 0x0000000f
+#define DB_DEBUG3__DISABLE_OP_S_DATA_FORWARDING_MASK 0x00080000L
+#define DB_DEBUG3__DISABLE_OP_S_DATA_FORWARDING__SHIFT 0x00000013
+#define DB_DEBUG3__DISABLE_OP_Z_DATA_FORWARDING_MASK 0x00002000L
+#define DB_DEBUG3__DISABLE_OP_Z_DATA_FORWARDING__SHIFT 0x0000000d
+#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX_MASK 0x08000000L
+#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX__SHIFT 0x0000001b
+#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD_MASK 0x00800000L
+#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD__SHIFT 0x00000017
+#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP_MASK 0x00000800L
+#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP__SHIFT 0x0000000b
+#define DB_DEBUG3__DISABLE_REDUNDANT_PLANE_FLUSHES_OPT_MASK 0x00000400L
+#define DB_DEBUG3__DISABLE_REDUNDANT_PLANE_FLUSHES_OPT__SHIFT 0x0000000a
+#define DB_DEBUG3__DISABLE_TCP_CAM_BYPASS_MASK 0x00000080L
+#define DB_DEBUG3__DISABLE_TCP_CAM_BYPASS__SHIFT 0x00000007
+#define DB_DEBUG3__DISABLE_TC_UPDATE_WRITE_COMBINE_MASK 0x00100000L
+#define DB_DEBUG3__DISABLE_TC_UPDATE_WRITE_COMBINE__SHIFT 0x00000014
+#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION_MASK 0x00000008L
+#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION__SHIFT 0x00000003
+#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION_MASK 0x00000100L
+#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION__SHIFT 0x00000008
+#define DB_DEBUG3__DONT_DELETE_CONTEXT_SUSPEND_MASK 0x20000000L
+#define DB_DEBUG3__DONT_DELETE_CONTEXT_SUSPEND__SHIFT 0x0000001d
+#define DB_DEBUG3__DONT_INSERT_CONTEXT_SUSPEND_MASK 0x10000000L
+#define DB_DEBUG3__DONT_INSERT_CONTEXT_SUSPEND__SHIFT 0x0000001c
+#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET_MASK 0x04000000L
+#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET__SHIFT 0x0000001a
+#define DB_DEBUG3__ENABLE_INCOHERENT_EQAA_READS_MASK 0x00001000L
+#define DB_DEBUG3__ENABLE_INCOHERENT_EQAA_READS__SHIFT 0x0000000c
+#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT_MASK 0x00400000L
+#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT__SHIFT 0x00000016
+#define DB_DEBUG3__ENABLE_TC_MA_ROUND_ROBIN_ARB_MASK 0x00800000L
+#define DB_DEBUG3__ENABLE_TC_MA_ROUND_ROBIN_ARB__SHIFT 0x00000017
+#define DB_DEBUG3__EQAA_INTERPOLATE_COMP_Z_MASK 0x00000020L
+#define DB_DEBUG3__EQAA_INTERPOLATE_COMP_Z__SHIFT 0x00000005
+#define DB_DEBUG3__EQAA_INTERPOLATE_SRC_Z_MASK 0x00000040L
+#define DB_DEBUG3__EQAA_INTERPOLATE_SRC_Z__SHIFT 0x00000006
+#define DB_DEBUG3__FORCE_DB_IS_GOOD_MASK 0x00000004L
+#define DB_DEBUG3__FORCE_DB_IS_GOOD__SHIFT 0x00000002
+#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE_MASK 0x00040000L
+#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE__SHIFT 0x00000012
+#define DB_DEBUG4__DB_EXTRA_DEBUG4_MASK 0xffffffc0L
+#define DB_DEBUG4__DB_EXTRA_DEBUG4__SHIFT 0x00000006
+#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL_MASK 0x00000020L
+#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL__SHIFT 0x00000005
+#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION_MASK 0x00000002L
+#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION__SHIFT 0x00000001
+#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION_MASK 0x00000001L
+#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION__SHIFT 0x00000000
+#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL_MASK 0x00000004L
+#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL__SHIFT 0x00000002
+#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE_MASK 0x00000002L
+#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE__SHIFT 0x00000001
+#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE_MASK 0x00008000L
+#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE__SHIFT 0x0000000f
+#define DB_DEBUG__DEBUG_FAST_Z_DISABLE_MASK 0x00004000L
+#define DB_DEBUG__DEBUG_FAST_Z_DISABLE__SHIFT 0x0000000e
+#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ_MASK 0x00000040L
+#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ__SHIFT 0x00000006
+#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE_MASK 0x00180000L
+#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE__SHIFT 0x00000013
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0_MASK 0x00000c00L
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0__SHIFT 0x0000000a
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1_MASK 0x00003000L
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1__SHIFT 0x0000000c
+#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE_MASK 0x00000300L
+#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE__SHIFT 0x00000008
+#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ_MASK 0x00000080L
+#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ__SHIFT 0x00000007
+#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE_MASK 0x00010000L
+#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE__SHIFT 0x00000010
+#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE_MASK 0x00000001L
+#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE__SHIFT 0x00000000
+#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES_MASK 0x0f000000L
+#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES__SHIFT 0x00000018
+#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS_MASK 0x00040000L
+#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS__SHIFT 0x00000012
+#define DB_DEBUG__DISABLE_DEPTH_SURFACE_SYNC_MASK 0x40000000L
+#define DB_DEBUG__DISABLE_DEPTH_SURFACE_SYNC__SHIFT 0x0000001e
+#define DB_DEBUG__DISABLE_HTILE_SURFACE_SYNC_MASK 0x80000000L
+#define DB_DEBUG__DISABLE_HTILE_SURFACE_SYNC__SHIFT 0x0000001f
+#define DB_DEBUG__DISABLE_SUMM_SQUADS_MASK 0x00020000L
+#define DB_DEBUG__DISABLE_SUMM_SQUADS__SHIFT 0x00000011
+#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION_MASK 0x00800000L
+#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION__SHIFT 0x00000017
+#define DB_DEBUG__FETCH_FULL_STENCIL_TILE_MASK 0x00000008L
+#define DB_DEBUG__FETCH_FULL_STENCIL_TILE__SHIFT 0x00000003
+#define DB_DEBUG__FETCH_FULL_Z_TILE_MASK 0x00000004L
+#define DB_DEBUG__FETCH_FULL_Z_TILE__SHIFT 0x00000002
+#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT_MASK 0x20000000L
+#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT__SHIFT 0x0000001d
+#define DB_DEBUG__FORCE_Z_MODE_MASK 0x00000030L
+#define DB_DEBUG__FORCE_Z_MODE__SHIFT 0x00000004
+#define DB_DEBUG__NEVER_FREE_Z_ONLY_MASK 0x00200000L
+#define DB_DEBUG__NEVER_FREE_Z_ONLY__SHIFT 0x00000015
+#define DB_DEBUG__ONE_FREE_IN_FLIGHT_MASK 0x10000000L
+#define DB_DEBUG__ONE_FREE_IN_FLIGHT__SHIFT 0x0000001c
+#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS_MASK 0x00400000L
+#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS__SHIFT 0x00000016
+#define DB_DEPTH_BOUNDS_MAX__MAX_MASK 0xffffffffL
+#define DB_DEPTH_BOUNDS_MAX__MAX__SHIFT 0x00000000
+#define DB_DEPTH_BOUNDS_MIN__MIN_MASK 0xffffffffL
+#define DB_DEPTH_BOUNDS_MIN__MIN__SHIFT 0x00000000
+#define DB_DEPTH_CLEAR__DEPTH_CLEAR_MASK 0xffffffffL
+#define DB_DEPTH_CLEAR__DEPTH_CLEAR__SHIFT 0x00000000
+#define DB_DEPTH_CONTROL__BACKFACE_ENABLE_MASK 0x00000080L
+#define DB_DEPTH_CONTROL__BACKFACE_ENABLE__SHIFT 0x00000007
+#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE_MASK 0x00000008L
+#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE__SHIFT 0x00000003
+#define DB_DEPTH_CONTROL__DISABLE_COLOR_WRITES_ON_DEPTH_PASS_MASK 0x80000000L
+#define DB_DEPTH_CONTROL__DISABLE_COLOR_WRITES_ON_DEPTH_PASS__SHIFT 0x0000001f
+#define DB_DEPTH_CONTROL__ENABLE_COLOR_WRITES_ON_DEPTH_FAIL_MASK 0x40000000L
+#define DB_DEPTH_CONTROL__ENABLE_COLOR_WRITES_ON_DEPTH_FAIL__SHIFT 0x0000001e
+#define DB_DEPTH_CONTROL__STENCIL_ENABLE_MASK 0x00000001L
+#define DB_DEPTH_CONTROL__STENCIL_ENABLE__SHIFT 0x00000000
+#define DB_DEPTH_CONTROL__STENCILFUNC_BF_MASK 0x00700000L
+#define DB_DEPTH_CONTROL__STENCILFUNC_BF__SHIFT 0x00000014
+#define DB_DEPTH_CONTROL__STENCILFUNC_MASK 0x00000700L
+#define DB_DEPTH_CONTROL__STENCILFUNC__SHIFT 0x00000008
+#define DB_DEPTH_CONTROL__Z_ENABLE_MASK 0x00000002L
+#define DB_DEPTH_CONTROL__Z_ENABLE__SHIFT 0x00000001
+#define DB_DEPTH_CONTROL__ZFUNC_MASK 0x00000070L
+#define DB_DEPTH_CONTROL__ZFUNC__SHIFT 0x00000004
+#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE_MASK 0x00000004L
+#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE__SHIFT 0x00000002
+#define DB_DEPTH_INFO__ADDR5_SWIZZLE_MASK_MASK 0x0000000fL
+#define DB_DEPTH_INFO__ADDR5_SWIZZLE_MASK__SHIFT 0x00000000
+#define DB_DEPTH_INFO__ARRAY_MODE_MASK 0x000000f0L
+#define DB_DEPTH_INFO__ARRAY_MODE__SHIFT 0x00000004
+#define DB_DEPTH_INFO__BANK_HEIGHT_MASK 0x00018000L
+#define DB_DEPTH_INFO__BANK_HEIGHT__SHIFT 0x0000000f
+#define DB_DEPTH_INFO__BANK_WIDTH_MASK 0x00006000L
+#define DB_DEPTH_INFO__BANK_WIDTH__SHIFT 0x0000000d
+#define DB_DEPTH_INFO__MACRO_TILE_ASPECT_MASK 0x00060000L
+#define DB_DEPTH_INFO__MACRO_TILE_ASPECT__SHIFT 0x00000011
+#define DB_DEPTH_INFO__NUM_BANKS_MASK 0x00180000L
+#define DB_DEPTH_INFO__NUM_BANKS__SHIFT 0x00000013
+#define DB_DEPTH_INFO__PIPE_CONFIG_MASK 0x00001f00L
+#define DB_DEPTH_INFO__PIPE_CONFIG__SHIFT 0x00000008
+#define DB_DEPTH_SIZE__HEIGHT_TILE_MAX_MASK 0x003ff800L
+#define DB_DEPTH_SIZE__HEIGHT_TILE_MAX__SHIFT 0x0000000b
+#define DB_DEPTH_SIZE__PITCH_TILE_MAX_MASK 0x000007ffL
+#define DB_DEPTH_SIZE__PITCH_TILE_MAX__SHIFT 0x00000000
+#define DB_DEPTH_SLICE__SLICE_TILE_MAX_MASK 0x003fffffL
+#define DB_DEPTH_SLICE__SLICE_TILE_MAX__SHIFT 0x00000000
+#define DB_DEPTH_VIEW__SLICE_MAX_MASK 0x00ffe000L
+#define DB_DEPTH_VIEW__SLICE_MAX__SHIFT 0x0000000d
+#define DB_DEPTH_VIEW__SLICE_START_MASK 0x000007ffL
+#define DB_DEPTH_VIEW__SLICE_START__SHIFT 0x00000000
+#define DB_DEPTH_VIEW__STENCIL_READ_ONLY_MASK 0x02000000L
+#define DB_DEPTH_VIEW__STENCIL_READ_ONLY__SHIFT 0x00000019
+#define DB_DEPTH_VIEW__Z_READ_ONLY_MASK 0x01000000L
+#define DB_DEPTH_VIEW__Z_READ_ONLY__SHIFT 0x00000018
+#define DB_EQAA__ALPHA_TO_MASK_EQAA_DISABLE_MASK 0x00200000L
+#define DB_EQAA__ALPHA_TO_MASK_EQAA_DISABLE__SHIFT 0x00000015
+#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES_MASK 0x00007000L
+#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES__SHIFT 0x0000000c
+#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION_MASK 0x08000000L
+#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION__SHIFT 0x0000001b
+#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS_MASK 0x00010000L
+#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS__SHIFT 0x00000010
+#define DB_EQAA__INCOHERENT_EQAA_READS_MASK 0x00020000L
+#define DB_EQAA__INCOHERENT_EQAA_READS__SHIFT 0x00000011
+#define DB_EQAA__INTERPOLATE_COMP_Z_MASK 0x00040000L
+#define DB_EQAA__INTERPOLATE_COMP_Z__SHIFT 0x00000012
+#define DB_EQAA__INTERPOLATE_SRC_Z_MASK 0x00080000L
+#define DB_EQAA__INTERPOLATE_SRC_Z__SHIFT 0x00000013
+#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES_MASK 0x00000700L
+#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES__SHIFT 0x00000008
+#define DB_EQAA__MAX_ANCHOR_SAMPLES_MASK 0x00000007L
+#define DB_EQAA__MAX_ANCHOR_SAMPLES__SHIFT 0x00000000
+#define DB_EQAA__OVERRASTERIZATION_AMOUNT_MASK 0x07000000L
+#define DB_EQAA__OVERRASTERIZATION_AMOUNT__SHIFT 0x00000018
+#define DB_EQAA__PS_ITER_SAMPLES_MASK 0x00000070L
+#define DB_EQAA__PS_ITER_SAMPLES__SHIFT 0x00000004
+#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS_MASK 0x00100000L
+#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS__SHIFT 0x00000014
+#define DB_FIFO_DEPTH1__LTILE_PROBE_FIFO_DEPTH_MASK 0x1fe00000L
+#define DB_FIFO_DEPTH1__LTILE_PROBE_FIFO_DEPTH__SHIFT 0x00000015
+#define DB_FIFO_DEPTH1__MCC_DEPTH_MASK 0x0000fc00L
+#define DB_FIFO_DEPTH1__MCC_DEPTH__SHIFT 0x0000000a
+#define DB_FIFO_DEPTH1__MI_RDREQ_FIFO_DEPTH_MASK 0x0000001fL
+#define DB_FIFO_DEPTH1__MI_RDREQ_FIFO_DEPTH__SHIFT 0x00000000
+#define DB_FIFO_DEPTH1__MI_WRREQ_FIFO_DEPTH_MASK 0x000003e0L
+#define DB_FIFO_DEPTH1__MI_WRREQ_FIFO_DEPTH__SHIFT 0x00000005
+#define DB_FIFO_DEPTH1__QC_DEPTH_MASK 0x001f0000L
+#define DB_FIFO_DEPTH1__QC_DEPTH__SHIFT 0x00000010
+#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH_MASK 0x000000ffL
+#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH__SHIFT 0x00000000
+#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH_MASK 0x00007f00L
+#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH__SHIFT 0x00000008
+#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH_MASK 0x01ff8000L
+#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH__SHIFT 0x0000000f
+#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH_MASK 0xfe000000L
+#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH__SHIFT 0x00000019
+#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH_MASK 0x0000007fL
+#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH__SHIFT 0x00000000
+#define DB_FREE_CACHELINES__FREE_HTILE_DEPTH_MASK 0x01e00000L
+#define DB_FREE_CACHELINES__FREE_HTILE_DEPTH__SHIFT 0x00000015
+#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH_MASK 0x00003f80L
+#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH__SHIFT 0x00000007
+#define DB_FREE_CACHELINES__FREE_Z_DEPTH_MASK 0x001fc000L
+#define DB_FREE_CACHELINES__FREE_Z_DEPTH__SHIFT 0x0000000e
+#define DB_FREE_CACHELINES__QUAD_READ_REQS_MASK 0xfe000000L
+#define DB_FREE_CACHELINES__QUAD_READ_REQS__SHIFT 0x00000019
+#define DB_HTILE_DATA_BASE__BASE_256B_MASK 0xffffffffL
+#define DB_HTILE_DATA_BASE__BASE_256B__SHIFT 0x00000000
+#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE_MASK 0x00010000L
+#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE__SHIFT 0x00000010
+#define DB_HTILE_SURFACE__FULL_CACHE_MASK 0x00000002L
+#define DB_HTILE_SURFACE__FULL_CACHE__SHIFT 0x00000001
+#define DB_HTILE_SURFACE__HTILE_USES_PRELOAD_WIN_MASK 0x00000004L
+#define DB_HTILE_SURFACE__HTILE_USES_PRELOAD_WIN__SHIFT 0x00000002
+#define DB_HTILE_SURFACE__LINEAR_MASK 0x00000001L
+#define DB_HTILE_SURFACE__LINEAR__SHIFT 0x00000000
+#define DB_HTILE_SURFACE__PREFETCH_HEIGHT_MASK 0x0000fc00L
+#define DB_HTILE_SURFACE__PREFETCH_HEIGHT__SHIFT 0x0000000a
+#define DB_HTILE_SURFACE__PREFETCH_WIDTH_MASK 0x000003f0L
+#define DB_HTILE_SURFACE__PREFETCH_WIDTH__SHIFT 0x00000004
+#define DB_HTILE_SURFACE__PRELOAD_MASK 0x00000008L
+#define DB_HTILE_SURFACE__PRELOAD__SHIFT 0x00000003
+#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000L
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x0000001c
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0x0f000000L
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x00000018
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define DB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define DB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000L
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x000000ffL
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf0000000L
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x0000001c
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0x0f000000L
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x00000018
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define DB_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define DB_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000L
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x000000ffL
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define DB_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define DB_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000L
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x000000ffL
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x00000000
+#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define DB_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define DB_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000L
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x000000ffL
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x00000000
+#define DB_PRELOAD_CONTROL__MAX_X_MASK 0x00ff0000L
+#define DB_PRELOAD_CONTROL__MAX_X__SHIFT 0x00000010
+#define DB_PRELOAD_CONTROL__MAX_Y_MASK 0xff000000L
+#define DB_PRELOAD_CONTROL__MAX_Y__SHIFT 0x00000018
+#define DB_PRELOAD_CONTROL__START_X_MASK 0x000000ffL
+#define DB_PRELOAD_CONTROL__START_X__SHIFT 0x00000000
+#define DB_PRELOAD_CONTROL__START_Y_MASK 0x0000ff00L
+#define DB_PRELOAD_CONTROL__START_Y__SHIFT 0x00000008
+#define DB_READ_DEBUG_0__BUSY_DATA0_MASK 0xffffffffL
+#define DB_READ_DEBUG_0__BUSY_DATA0__SHIFT 0x00000000
+#define DB_READ_DEBUG_1__BUSY_DATA1_MASK 0xffffffffL
+#define DB_READ_DEBUG_1__BUSY_DATA1__SHIFT 0x00000000
+#define DB_READ_DEBUG_2__BUSY_DATA2_MASK 0xffffffffL
+#define DB_READ_DEBUG_2__BUSY_DATA2__SHIFT 0x00000000
+#define DB_READ_DEBUG_3__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_3__DEBUG_DATA__SHIFT 0x00000000
+#define DB_READ_DEBUG_4__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_4__DEBUG_DATA__SHIFT 0x00000000
+#define DB_READ_DEBUG_5__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_5__DEBUG_DATA__SHIFT 0x00000000
+#define DB_READ_DEBUG_6__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_6__DEBUG_DATA__SHIFT 0x00000000
+#define DB_READ_DEBUG_7__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_7__DEBUG_DATA__SHIFT 0x00000000
+#define DB_READ_DEBUG_8__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_8__DEBUG_DATA__SHIFT 0x00000000
+#define DB_READ_DEBUG_9__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_9__DEBUG_DATA__SHIFT 0x00000000
+#define DB_READ_DEBUG_A__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_A__DEBUG_DATA__SHIFT 0x00000000
+#define DB_READ_DEBUG_B__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_B__DEBUG_DATA__SHIFT 0x00000000
+#define DB_READ_DEBUG_C__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_C__DEBUG_DATA__SHIFT 0x00000000
+#define DB_READ_DEBUG_D__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_D__DEBUG_DATA__SHIFT 0x00000000
+#define DB_READ_DEBUG_E__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_E__DEBUG_DATA__SHIFT 0x00000000
+#define DB_READ_DEBUG_F__DEBUG_DATA_MASK 0xffffffffL
+#define DB_READ_DEBUG_F__DEBUG_DATA__SHIFT 0x00000000
+#define DB_RENDER_CONTROL__COPY_CENTROID_MASK 0x00000080L
+#define DB_RENDER_CONTROL__COPY_CENTROID__SHIFT 0x00000007
+#define DB_RENDER_CONTROL__COPY_SAMPLE_MASK 0x00000f00L
+#define DB_RENDER_CONTROL__COPY_SAMPLE__SHIFT 0x00000008
+#define DB_RENDER_CONTROL__DEPTH_CLEAR_ENABLE_MASK 0x00000001L
+#define DB_RENDER_CONTROL__DEPTH_CLEAR_ENABLE__SHIFT 0x00000000
+#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE_MASK 0x00000040L
+#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE__SHIFT 0x00000006
+#define DB_RENDER_CONTROL__DEPTH_COPY_MASK 0x00000004L
+#define DB_RENDER_CONTROL__DEPTH_COPY__SHIFT 0x00000002
+#define DB_RENDER_CONTROL__RESUMMARIZE_ENABLE_MASK 0x00000010L
+#define DB_RENDER_CONTROL__RESUMMARIZE_ENABLE__SHIFT 0x00000004
+#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE_MASK 0x00000002L
+#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE__SHIFT 0x00000001
+#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE_MASK 0x00000020L
+#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE__SHIFT 0x00000005
+#define DB_RENDER_CONTROL__STENCIL_COPY_MASK 0x00000008L
+#define DB_RENDER_CONTROL__STENCIL_COPY__SHIFT 0x00000003
+#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH_MASK 0x00000100L
+#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH__SHIFT 0x00000008
+#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE_MASK 0x00000400L
+#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE__SHIFT 0x0000000a
+#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION_MASK 0x00000080L
+#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION__SHIFT 0x00000007
+#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS_MASK 0x00800000L
+#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS__SHIFT 0x00000017
+#define DB_RENDER_OVERRIDE2__DISABLE_REG_SNOOP_MASK 0x00000200L
+#define DB_RENDER_OVERRIDE2__DISABLE_REG_SNOOP__SHIFT 0x00000009
+#define DB_RENDER_OVERRIDE2__DISABLE_SMEM_EXPCLEAR_OPTIMIZATION_MASK 0x00000040L
+#define DB_RENDER_OVERRIDE2__DISABLE_SMEM_EXPCLEAR_OPTIMIZATION__SHIFT 0x00000006
+#define DB_RENDER_OVERRIDE2__DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION_MASK 0x00000020L
+#define DB_RENDER_OVERRIDE2__DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION__SHIFT 0x00000005
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_BF_MASK 0x001c0000L
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_BF__SHIFT 0x00000012
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_FF_MASK 0x00038000L
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_FF__SHIFT 0x0000000f
+#define DB_RENDER_OVERRIDE2__HIZ_ZFUNC_MASK 0x00007000L
+#define DB_RENDER_OVERRIDE2__HIZ_ZFUNC__SHIFT 0x0000000c
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL_MASK 0x00000003L
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL__SHIFT 0x00000000
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN_MASK 0x0000001cL
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN__SHIFT 0x00000002
+#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS_MASK 0x00400000L
+#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS__SHIFT 0x00000016
+#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE_MASK 0x00200000L
+#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE__SHIFT 0x00000015
+#define DB_RENDER_OVERRIDE2__SEPARATE_HIZS_FUNC_ENABLE_MASK 0x00000800L
+#define DB_RENDER_OVERRIDE2__SEPARATE_HIZS_FUNC_ENABLE__SHIFT 0x0000000b
+#define DB_RENDER_OVERRIDE__DISABLE_FULLY_COVERED_MASK 0x00040000L
+#define DB_RENDER_OVERRIDE__DISABLE_FULLY_COVERED__SHIFT 0x00000012
+#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES_MASK 0x04000000L
+#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES__SHIFT 0x0000001a
+#define DB_RENDER_OVERRIDE__DISABLE_VIEWPORT_CLAMP_MASK 0x00010000L
+#define DB_RENDER_OVERRIDE__DISABLE_VIEWPORT_CLAMP__SHIFT 0x00000010
+#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE_MASK 0x00000100L
+#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE__SHIFT 0x00000008
+#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE_MASK 0x00000080L
+#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE__SHIFT 0x00000007
+#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL_MASK 0x00000400L
+#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL__SHIFT 0x0000000a
+#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE_MASK 0x00006000L
+#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE__SHIFT 0x0000000d
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0_MASK 0x0000000cL
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0__SHIFT 0x00000002
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1_MASK 0x00000030L
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1__SHIFT 0x00000004
+#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE_MASK 0x00000003L
+#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE__SHIFT 0x00000000
+#define DB_RENDER_OVERRIDE__FORCE_QC_SMASK_CONFLICT_MASK 0x00008000L
+#define DB_RENDER_OVERRIDE__FORCE_QC_SMASK_CONFLICT__SHIFT 0x0000000f
+#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER_MASK 0x00000040L
+#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER__SHIFT 0x00000006
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY_MASK 0x10000000L
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY__SHIFT 0x0000001c
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ_MASK 0x00001000L
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ__SHIFT 0x0000000c
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID_MASK 0x40000000L
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID__SHIFT 0x0000001e
+#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY_MASK 0x08000000L
+#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY__SHIFT 0x0000001b
+#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM_MASK 0x00180000L
+#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM__SHIFT 0x00000013
+#define DB_RENDER_OVERRIDE__FORCE_Z_READ_MASK 0x00000800L
+#define DB_RENDER_OVERRIDE__FORCE_Z_READ__SHIFT 0x0000000b
+#define DB_RENDER_OVERRIDE__FORCE_Z_VALID_MASK 0x20000000L
+#define DB_RENDER_OVERRIDE__FORCE_Z_VALID__SHIFT 0x0000001d
+#define DB_RENDER_OVERRIDE__IGNORE_SC_ZRANGE_MASK 0x00020000L
+#define DB_RENDER_OVERRIDE__IGNORE_SC_ZRANGE__SHIFT 0x00000011
+#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT_MASK 0x03e00000L
+#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT__SHIFT 0x00000015
+#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE_MASK 0x00000200L
+#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE__SHIFT 0x00000009
+#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION_MASK 0x80000000L
+#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION__SHIFT 0x0000001f
+#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE_MASK 0x00000800L
+#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE__SHIFT 0x0000000b
+#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT_MASK 0x00006000L
+#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT__SHIFT 0x0000000d
+#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE_MASK 0x00000080L
+#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE__SHIFT 0x00000007
+#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER_MASK 0x00001000L
+#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER__SHIFT 0x0000000c
+#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL_MASK 0x00000200L
+#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL__SHIFT 0x00000009
+#define DB_SHADER_CONTROL__EXEC_ON_NOOP_MASK 0x00000400L
+#define DB_SHADER_CONTROL__EXEC_ON_NOOP__SHIFT 0x0000000a
+#define DB_SHADER_CONTROL__KILL_ENABLE_MASK 0x00000040L
+#define DB_SHADER_CONTROL__KILL_ENABLE__SHIFT 0x00000006
+#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE_MASK 0x00000100L
+#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE__SHIFT 0x00000008
+#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE_MASK 0x00000004L
+#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE__SHIFT 0x00000002
+#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE_MASK 0x00000002L
+#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE__SHIFT 0x00000001
+#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE_MASK 0x00000001L
+#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE__SHIFT 0x00000000
+#define DB_SHADER_CONTROL__Z_ORDER_MASK 0x00000030L
+#define DB_SHADER_CONTROL__Z_ORDER__SHIFT 0x00000004
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0_MASK 0x00000007L
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0__SHIFT 0x00000000
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0_MASK 0x000ff000L
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0__SHIFT 0x0000000c
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0_MASK 0x00000ff0L
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0__SHIFT 0x00000004
+#define DB_SRESULTS_COMPARE_STATE0__ENABLE0_MASK 0x01000000L
+#define DB_SRESULTS_COMPARE_STATE0__ENABLE0__SHIFT 0x00000018
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1_MASK 0x00000007L
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1__SHIFT 0x00000000
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1_MASK 0x000ff000L
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1__SHIFT 0x0000000c
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1_MASK 0x00000ff0L
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1__SHIFT 0x00000004
+#define DB_SRESULTS_COMPARE_STATE1__ENABLE1_MASK 0x01000000L
+#define DB_SRESULTS_COMPARE_STATE1__ENABLE1__SHIFT 0x00000018
+#define DB_STENCIL_CLEAR__CLEAR_MASK 0x000000ffL
+#define DB_STENCIL_CLEAR__CLEAR__SHIFT 0x00000000
+#define DB_STENCIL_CONTROL__STENCILFAIL_BF_MASK 0x0000f000L
+#define DB_STENCIL_CONTROL__STENCILFAIL_BF__SHIFT 0x0000000c
+#define DB_STENCIL_CONTROL__STENCILFAIL_MASK 0x0000000fL
+#define DB_STENCIL_CONTROL__STENCILFAIL__SHIFT 0x00000000
+#define DB_STENCIL_CONTROL__STENCILZFAIL_BF_MASK 0x00f00000L
+#define DB_STENCIL_CONTROL__STENCILZFAIL_BF__SHIFT 0x00000014
+#define DB_STENCIL_CONTROL__STENCILZFAIL_MASK 0x00000f00L
+#define DB_STENCIL_CONTROL__STENCILZFAIL__SHIFT 0x00000008
+#define DB_STENCIL_CONTROL__STENCILZPASS_BF_MASK 0x000f0000L
+#define DB_STENCIL_CONTROL__STENCILZPASS_BF__SHIFT 0x00000010
+#define DB_STENCIL_CONTROL__STENCILZPASS_MASK 0x000000f0L
+#define DB_STENCIL_CONTROL__STENCILZPASS__SHIFT 0x00000004
+#define DB_STENCIL_INFO__ALLOW_EXPCLEAR_MASK 0x08000000L
+#define DB_STENCIL_INFO__ALLOW_EXPCLEAR__SHIFT 0x0000001b
+#define DB_STENCIL_INFO__FORMAT_MASK 0x00000001L
+#define DB_STENCIL_INFO__FORMAT__SHIFT 0x00000000
+#define DB_STENCIL_INFO__TILE_MODE_INDEX_MASK 0x00700000L
+#define DB_STENCIL_INFO__TILE_MODE_INDEX__SHIFT 0x00000014
+#define DB_STENCIL_INFO__TILE_SPLIT_MASK 0x0000e000L
+#define DB_STENCIL_INFO__TILE_SPLIT__SHIFT 0x0000000d
+#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE_MASK 0x20000000L
+#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE__SHIFT 0x0000001d
+#define DB_STENCIL_READ_BASE__BASE_256B_MASK 0xffffffffL
+#define DB_STENCIL_READ_BASE__BASE_256B__SHIFT 0x00000000
+#define DB_STENCILREFMASK_BF__STENCILMASK_BF_MASK 0x0000ff00L
+#define DB_STENCILREFMASK_BF__STENCILMASK_BF__SHIFT 0x00000008
+#define DB_STENCILREFMASK_BF__STENCILOPVAL_BF_MASK 0xff000000L
+#define DB_STENCILREFMASK_BF__STENCILOPVAL_BF__SHIFT 0x00000018
+#define DB_STENCILREFMASK_BF__STENCILTESTVAL_BF_MASK 0x000000ffL
+#define DB_STENCILREFMASK_BF__STENCILTESTVAL_BF__SHIFT 0x00000000
+#define DB_STENCILREFMASK_BF__STENCILWRITEMASK_BF_MASK 0x00ff0000L
+#define DB_STENCILREFMASK_BF__STENCILWRITEMASK_BF__SHIFT 0x00000010
+#define DB_STENCILREFMASK__STENCILMASK_MASK 0x0000ff00L
+#define DB_STENCILREFMASK__STENCILMASK__SHIFT 0x00000008
+#define DB_STENCILREFMASK__STENCILOPVAL_MASK 0xff000000L
+#define DB_STENCILREFMASK__STENCILOPVAL__SHIFT 0x00000018
+#define DB_STENCILREFMASK__STENCILTESTVAL_MASK 0x000000ffL
+#define DB_STENCILREFMASK__STENCILTESTVAL__SHIFT 0x00000000
+#define DB_STENCILREFMASK__STENCILWRITEMASK_MASK 0x00ff0000L
+#define DB_STENCILREFMASK__STENCILWRITEMASK__SHIFT 0x00000010
+#define DB_STENCIL_WRITE_BASE__BASE_256B_MASK 0xffffffffL
+#define DB_STENCIL_WRITE_BASE__BASE_256B__SHIFT 0x00000000
+#define DB_SUBTILE_CONTROL__MSAA16_X_MASK 0x00030000L
+#define DB_SUBTILE_CONTROL__MSAA16_X__SHIFT 0x00000010
+#define DB_SUBTILE_CONTROL__MSAA16_Y_MASK 0x000c0000L
+#define DB_SUBTILE_CONTROL__MSAA16_Y__SHIFT 0x00000012
+#define DB_SUBTILE_CONTROL__MSAA1_X_MASK 0x00000003L
+#define DB_SUBTILE_CONTROL__MSAA1_X__SHIFT 0x00000000
+#define DB_SUBTILE_CONTROL__MSAA1_Y_MASK 0x0000000cL
+#define DB_SUBTILE_CONTROL__MSAA1_Y__SHIFT 0x00000002
+#define DB_SUBTILE_CONTROL__MSAA2_X_MASK 0x00000030L
+#define DB_SUBTILE_CONTROL__MSAA2_X__SHIFT 0x00000004
+#define DB_SUBTILE_CONTROL__MSAA2_Y_MASK 0x000000c0L
+#define DB_SUBTILE_CONTROL__MSAA2_Y__SHIFT 0x00000006
+#define DB_SUBTILE_CONTROL__MSAA4_X_MASK 0x00000300L
+#define DB_SUBTILE_CONTROL__MSAA4_X__SHIFT 0x00000008
+#define DB_SUBTILE_CONTROL__MSAA4_Y_MASK 0x00000c00L
+#define DB_SUBTILE_CONTROL__MSAA4_Y__SHIFT 0x0000000a
+#define DB_SUBTILE_CONTROL__MSAA8_X_MASK 0x00003000L
+#define DB_SUBTILE_CONTROL__MSAA8_X__SHIFT 0x0000000c
+#define DB_SUBTILE_CONTROL__MSAA8_Y_MASK 0x0000c000L
+#define DB_SUBTILE_CONTROL__MSAA8_Y__SHIFT 0x0000000e
+#define DB_WATERMARKS__AUTO_FLUSH_HTILE_MASK 0x40000000L
+#define DB_WATERMARKS__AUTO_FLUSH_HTILE__SHIFT 0x0000001e
+#define DB_WATERMARKS__AUTO_FLUSH_QUAD_MASK 0x80000000L
+#define DB_WATERMARKS__AUTO_FLUSH_QUAD__SHIFT 0x0000001f
+#define DB_WATERMARKS__DEPTH_CACHELINE_FREE_MASK 0x07f00000L
+#define DB_WATERMARKS__DEPTH_CACHELINE_FREE__SHIFT 0x00000014
+#define DB_WATERMARKS__DEPTH_FLUSH_MASK 0x000007e0L
+#define DB_WATERMARKS__DEPTH_FLUSH__SHIFT 0x00000005
+#define DB_WATERMARKS__DEPTH_FREE_MASK 0x0000001fL
+#define DB_WATERMARKS__DEPTH_FREE__SHIFT 0x00000000
+#define DB_WATERMARKS__DEPTH_PENDING_FREE_MASK 0x000f8000L
+#define DB_WATERMARKS__DEPTH_PENDING_FREE__SHIFT 0x0000000f
+#define DB_WATERMARKS__EARLY_Z_PANIC_DISABLE_MASK 0x08000000L
+#define DB_WATERMARKS__EARLY_Z_PANIC_DISABLE__SHIFT 0x0000001b
+#define DB_WATERMARKS__FORCE_SUMMARIZE_MASK 0x00007800L
+#define DB_WATERMARKS__FORCE_SUMMARIZE__SHIFT 0x0000000b
+#define DB_WATERMARKS__LATE_Z_PANIC_DISABLE_MASK 0x10000000L
+#define DB_WATERMARKS__LATE_Z_PANIC_DISABLE__SHIFT 0x0000001c
+#define DB_WATERMARKS__RE_Z_PANIC_DISABLE_MASK 0x20000000L
+#define DB_WATERMARKS__RE_Z_PANIC_DISABLE__SHIFT 0x0000001d
+#define DB_Z_INFO__ALLOW_EXPCLEAR_MASK 0x08000000L
+#define DB_Z_INFO__ALLOW_EXPCLEAR__SHIFT 0x0000001b
+#define DB_Z_INFO__FORMAT_MASK 0x00000003L
+#define DB_Z_INFO__FORMAT__SHIFT 0x00000000
+#define DB_Z_INFO__NUM_SAMPLES_MASK 0x0000000cL
+#define DB_Z_INFO__NUM_SAMPLES__SHIFT 0x00000002
+#define DB_Z_INFO__READ_SIZE_MASK 0x10000000L
+#define DB_Z_INFO__READ_SIZE__SHIFT 0x0000001c
+#define DB_Z_INFO__TILE_MODE_INDEX_MASK 0x00700000L
+#define DB_Z_INFO__TILE_MODE_INDEX__SHIFT 0x00000014
+#define DB_Z_INFO__TILE_SPLIT_MASK 0x0000e000L
+#define DB_Z_INFO__TILE_SPLIT__SHIFT 0x0000000d
+#define DB_Z_INFO__TILE_SURFACE_ENABLE_MASK 0x20000000L
+#define DB_Z_INFO__TILE_SURFACE_ENABLE__SHIFT 0x0000001d
+#define DB_Z_INFO__ZRANGE_PRECISION_MASK 0x80000000L
+#define DB_Z_INFO__ZRANGE_PRECISION__SHIFT 0x0000001f
+#define DB_ZPASS_COUNT_HI__COUNT_HI_MASK 0x7fffffffL
+#define DB_ZPASS_COUNT_HI__COUNT_HI__SHIFT 0x00000000
+#define DB_ZPASS_COUNT_LOW__COUNT_LOW_MASK 0xffffffffL
+#define DB_ZPASS_COUNT_LOW__COUNT_LOW__SHIFT 0x00000000
+#define DB_Z_READ_BASE__BASE_256B_MASK 0xffffffffL
+#define DB_Z_READ_BASE__BASE_256B__SHIFT 0x00000000
+#define DB_Z_WRITE_BASE__BASE_256B_MASK 0xffffffffL
+#define DB_Z_WRITE_BASE__BASE_256B__SHIFT 0x00000000
+#define DEBUG_DATA__DEBUG_DATA_MASK 0xffffffffL
+#define DEBUG_DATA__DEBUG_DATA__SHIFT 0x00000000
+#define DEBUG_INDEX__DEBUG_INDEX_MASK 0x0003ffffL
+#define DEBUG_INDEX__DEBUG_INDEX__SHIFT 0x00000000
+#define GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK 0x00000700L
+#define GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT 0x00000008
+#define GB_ADDR_CONFIG__MULTI_GPU_TILE_SIZE_MASK 0x03000000L
+#define GB_ADDR_CONFIG__MULTI_GPU_TILE_SIZE__SHIFT 0x00000018
+#define GB_ADDR_CONFIG__NUM_GPUS_MASK 0x00700000L
+#define GB_ADDR_CONFIG__NUM_GPUS__SHIFT 0x00000014
+#define GB_ADDR_CONFIG__NUM_LOWER_PIPES_MASK 0x40000000L
+#define GB_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT 0x0000001e
+#define GB_ADDR_CONFIG__NUM_PIPES_MASK 0x00000007L
+#define GB_ADDR_CONFIG__NUM_PIPES__SHIFT 0x00000000
+#define GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK 0x00003000L
+#define GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT 0x0000000c
+#define GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK 0x00000070L
+#define GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT 0x00000004
+#define GB_ADDR_CONFIG__ROW_SIZE_MASK 0x30000000L
+#define GB_ADDR_CONFIG__ROW_SIZE__SHIFT 0x0000001c
+#define GB_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK 0x00070000L
+#define GB_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT 0x00000010
+#define GB_BACKEND_MAP__BACKEND_MAP_MASK 0xffffffffL
+#define GB_BACKEND_MAP__BACKEND_MAP__SHIFT 0x00000000
+#define GB_EDC_MODE__BYPASS_MASK 0x80000000L
+#define GB_EDC_MODE__BYPASS__SHIFT 0x0000001f
+#define GB_EDC_MODE__DED_MODE_MASK 0x00300000L
+#define GB_EDC_MODE__DED_MODE__SHIFT 0x00000014
+#define GB_EDC_MODE__FORCE_SEC_ON_DED_MASK 0x00010000L
+#define GB_EDC_MODE__FORCE_SEC_ON_DED__SHIFT 0x00000010
+#define GB_EDC_MODE__PROP_FED_MASK 0x20000000L
+#define GB_EDC_MODE__PROP_FED__SHIFT 0x0000001d
+#define GB_GPU_ID__GPU_ID_MASK 0x0000000fL
+#define GB_GPU_ID__GPU_ID__SHIFT 0x00000000
+#define GB_TILE_MODE0__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE0__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE0__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE0__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE0__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE0__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE0__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE0__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE10__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE10__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE10__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE10__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE10__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE10__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE10__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE10__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE10__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE10__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE11__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE11__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE11__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE11__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE11__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE11__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE11__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE11__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE11__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE11__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE12__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE12__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE12__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE12__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE12__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE12__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE12__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE12__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE12__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE12__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE13__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE13__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE13__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE13__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE13__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE13__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE13__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE13__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE13__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE13__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE14__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE14__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE14__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE14__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE14__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE14__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE14__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE14__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE14__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE14__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE15__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE15__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE15__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE15__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE15__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE15__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE15__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE15__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE15__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE15__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE16__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE16__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE16__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE16__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE16__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE16__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE16__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE16__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE16__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE16__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE17__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE17__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE17__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE17__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE17__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE17__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE17__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE17__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE17__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE17__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE18__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE18__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE18__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE18__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE18__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE18__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE18__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE18__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE18__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE18__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE19__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE19__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE19__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE19__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE19__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE19__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE19__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE19__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE19__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE19__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE1__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE1__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE1__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE1__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE1__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE1__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE1__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE1__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE1__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE1__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE20__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE20__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE20__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE20__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE20__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE20__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE20__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE20__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE20__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE20__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE21__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE21__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE21__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE21__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE21__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE21__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE21__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE21__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE21__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE21__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE22__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE22__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE22__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE22__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE22__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE22__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE22__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE22__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE22__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE22__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE23__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE23__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE23__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE23__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE23__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE23__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE23__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE23__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE23__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE23__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE24__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE24__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE24__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE24__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE24__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE24__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE24__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE24__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE24__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE24__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE25__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE25__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE25__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE25__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE25__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE25__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE25__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE25__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE25__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE25__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE26__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE26__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE26__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE26__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE26__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE26__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE26__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE26__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE26__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE26__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE27__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE27__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE27__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE27__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE27__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE27__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE27__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE27__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE27__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE27__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE28__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE28__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE28__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE28__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE28__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE28__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE28__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE28__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE28__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE28__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE29__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE29__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE29__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE29__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE29__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE29__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE29__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE29__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE29__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE29__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE2__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE2__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE2__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE2__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE2__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE2__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE2__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE2__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE2__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE2__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE30__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE30__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE30__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE30__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE30__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE30__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE30__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE30__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE30__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE30__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE31__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE31__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE31__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE31__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE31__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE31__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE31__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE31__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE31__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE31__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE3__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE3__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE3__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE3__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE3__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE3__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE3__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE3__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE3__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE3__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE4__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE4__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE4__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE4__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE4__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE4__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE4__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE4__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE4__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE4__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE5__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE5__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE5__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE5__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE5__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE5__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE5__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE5__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE5__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE5__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE6__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE6__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE6__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE6__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE6__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE6__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE6__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE6__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE6__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE6__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE7__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE7__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE7__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE7__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE7__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE7__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE7__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE7__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE7__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE7__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE8__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE8__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE8__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE8__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE8__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE8__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE8__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE8__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE8__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE8__TILE_SPLIT__SHIFT 0x0000000b
+#define GB_TILE_MODE9__ARRAY_MODE_MASK 0x0000003cL
+#define GB_TILE_MODE9__ARRAY_MODE__SHIFT 0x00000002
+#define GB_TILE_MODE9__MICRO_TILE_MODE_NEW_MASK 0x01c00000L
+#define GB_TILE_MODE9__MICRO_TILE_MODE_NEW__SHIFT 0x00000016
+#define GB_TILE_MODE9__PIPE_CONFIG_MASK 0x000007c0L
+#define GB_TILE_MODE9__PIPE_CONFIG__SHIFT 0x00000006
+#define GB_TILE_MODE9__SAMPLE_SPLIT_MASK 0x06000000L
+#define GB_TILE_MODE9__SAMPLE_SPLIT__SHIFT 0x00000019
+#define GB_TILE_MODE9__TILE_SPLIT_MASK 0x00003800L
+#define GB_TILE_MODE9__TILE_SPLIT__SHIFT 0x0000000b
+#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK 0x00ff0000L
+#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT 0x00000010
+#define GC_USER_SHADER_ARRAY_CONFIG__DPFP_RATE_MASK 0x00000006L
+#define GC_USER_SHADER_ARRAY_CONFIG__DPFP_RATE__SHIFT 0x00000001
+#define GC_USER_SHADER_ARRAY_CONFIG__HALF_LDS_MASK 0x00000010L
+#define GC_USER_SHADER_ARRAY_CONFIG__HALF_LDS__SHIFT 0x00000004
+#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK 0xffff0000L
+#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT 0x00000010
+#define GC_USER_SHADER_ARRAY_CONFIG__SQC_BALANCE_DISABLE_MASK 0x00000008L
+#define GC_USER_SHADER_ARRAY_CONFIG__SQC_BALANCE_DISABLE__SHIFT 0x00000003
+#define GDS_ATOM_BASE__BASE_MASK 0x0000ffffL
+#define GDS_ATOM_BASE__BASE__SHIFT 0x00000000
+#define GDS_ATOM_BASE__UNUSED_MASK 0xffff0000L
+#define GDS_ATOM_BASE__UNUSED__SHIFT 0x00000010
+#define GDS_ATOM_CNTL__AINC_MASK 0x0000003fL
+#define GDS_ATOM_CNTL__AINC__SHIFT 0x00000000
+#define GDS_ATOM_CNTL__DMODE_MASK 0x00000100L
+#define GDS_ATOM_CNTL__DMODE__SHIFT 0x00000008
+#define GDS_ATOM_CNTL__UNUSED1_MASK 0x000000c0L
+#define GDS_ATOM_CNTL__UNUSED1__SHIFT 0x00000006
+#define GDS_ATOM_CNTL__UNUSED2_MASK 0xfffffe00L
+#define GDS_ATOM_CNTL__UNUSED2__SHIFT 0x00000009
+#define GDS_ATOM_COMPLETE__COMPLETE_MASK 0x00000001L
+#define GDS_ATOM_COMPLETE__COMPLETE__SHIFT 0x00000000
+#define GDS_ATOM_COMPLETE__UNUSED_MASK 0xfffffffeL
+#define GDS_ATOM_COMPLETE__UNUSED__SHIFT 0x00000001
+#define GDS_ATOM_DST__DST_MASK 0xffffffffL
+#define GDS_ATOM_DST__DST__SHIFT 0x00000000
+#define GDS_ATOM_OFFSET0__OFFSET0_MASK 0x000000ffL
+#define GDS_ATOM_OFFSET0__OFFSET0__SHIFT 0x00000000
+#define GDS_ATOM_OFFSET0__UNUSED_MASK 0xffffff00L
+#define GDS_ATOM_OFFSET0__UNUSED__SHIFT 0x00000008
+#define GDS_ATOM_OFFSET1__OFFSET1_MASK 0x000000ffL
+#define GDS_ATOM_OFFSET1__OFFSET1__SHIFT 0x00000000
+#define GDS_ATOM_OFFSET1__UNUSED_MASK 0xffffff00L
+#define GDS_ATOM_OFFSET1__UNUSED__SHIFT 0x00000008
+#define GDS_ATOM_OP__OP_MASK 0x000000ffL
+#define GDS_ATOM_OP__OP__SHIFT 0x00000000
+#define GDS_ATOM_OP__UNUSED_MASK 0xffffff00L
+#define GDS_ATOM_OP__UNUSED__SHIFT 0x00000008
+#define GDS_ATOM_READ0__DATA_MASK 0xffffffffL
+#define GDS_ATOM_READ0__DATA__SHIFT 0x00000000
+#define GDS_ATOM_READ0_U__DATA_MASK 0xffffffffL
+#define GDS_ATOM_READ0_U__DATA__SHIFT 0x00000000
+#define GDS_ATOM_READ1__DATA_MASK 0xffffffffL
+#define GDS_ATOM_READ1__DATA__SHIFT 0x00000000
+#define GDS_ATOM_READ1_U__DATA_MASK 0xffffffffL
+#define GDS_ATOM_READ1_U__DATA__SHIFT 0x00000000
+#define GDS_ATOM_SIZE__SIZE_MASK 0x0000ffffL
+#define GDS_ATOM_SIZE__SIZE__SHIFT 0x00000000
+#define GDS_ATOM_SIZE__UNUSED_MASK 0xffff0000L
+#define GDS_ATOM_SIZE__UNUSED__SHIFT 0x00000010
+#define GDS_ATOM_SRC0__DATA_MASK 0xffffffffL
+#define GDS_ATOM_SRC0__DATA__SHIFT 0x00000000
+#define GDS_ATOM_SRC0_U__DATA_MASK 0xffffffffL
+#define GDS_ATOM_SRC0_U__DATA__SHIFT 0x00000000
+#define GDS_ATOM_SRC1__DATA_MASK 0xffffffffL
+#define GDS_ATOM_SRC1__DATA__SHIFT 0x00000000
+#define GDS_ATOM_SRC1_U__DATA_MASK 0xffffffffL
+#define GDS_ATOM_SRC1_U__DATA__SHIFT 0x00000000
+#define GDS_CNTL_STATUS__DS_ADDR_CONFLICT_MASK 0x00000010L
+#define GDS_CNTL_STATUS__DS_ADDR_CONFLICT__SHIFT 0x00000004
+#define GDS_CNTL_STATUS__DS_BANK_CONFLICT_MASK 0x00000008L
+#define GDS_CNTL_STATUS__DS_BANK_CONFLICT__SHIFT 0x00000003
+#define GDS_CNTL_STATUS__DS_RD_CLAMP_MASK 0x00000040L
+#define GDS_CNTL_STATUS__DS_RD_CLAMP__SHIFT 0x00000006
+#define GDS_CNTL_STATUS__DS_WR_CLAMP_MASK 0x00000020L
+#define GDS_CNTL_STATUS__DS_WR_CLAMP__SHIFT 0x00000005
+#define GDS_CNTL_STATUS__GDS_BUSY_MASK 0x00000001L
+#define GDS_CNTL_STATUS__GDS_BUSY__SHIFT 0x00000000
+#define GDS_CNTL_STATUS__GRBM_WBUF_BUSY_MASK 0x00000002L
+#define GDS_CNTL_STATUS__GRBM_WBUF_BUSY__SHIFT 0x00000001
+#define GDS_CNTL_STATUS__ORD_APP_BUSY_MASK 0x00000004L
+#define GDS_CNTL_STATUS__ORD_APP_BUSY__SHIFT 0x00000002
+#define GDS_CONFIG__SH0_GPR_PHASE_SEL_MASK 0x00000006L
+#define GDS_CONFIG__SH0_GPR_PHASE_SEL__SHIFT 0x00000001
+#define GDS_CONFIG__SH1_GPR_PHASE_SEL_MASK 0x00000018L
+#define GDS_CONFIG__SH1_GPR_PHASE_SEL__SHIFT 0x00000003
+#define GDS_CONFIG__SH2_GPR_PHASE_SEL_MASK 0x00000060L
+#define GDS_CONFIG__SH2_GPR_PHASE_SEL__SHIFT 0x00000005
+#define GDS_CONFIG__SH3_GPR_PHASE_SEL_MASK 0x00000180L
+#define GDS_CONFIG__SH3_GPR_PHASE_SEL__SHIFT 0x00000007
+#define GDS_DEBUG_CNTL__GDS_DEBUG_INDX_MASK 0x0000001fL
+#define GDS_DEBUG_CNTL__GDS_DEBUG_INDX__SHIFT 0x00000000
+#define GDS_DEBUG_CNTL__UNUSED_MASK 0xffffffe0L
+#define GDS_DEBUG_CNTL__UNUSED__SHIFT 0x00000005
+#define GDS_DEBUG_DATA__DATA_MASK 0xffffffffL
+#define GDS_DEBUG_DATA__DATA__SHIFT 0x00000000
+#define GDS_DEBUG_REG0__buff_write_MASK 0x00020000L
+#define GDS_DEBUG_REG0__buff_write__SHIFT 0x00000011
+#define GDS_DEBUG_REG0__cstate_MASK 0x0001e000L
+#define GDS_DEBUG_REG0__cstate__SHIFT 0x0000000d
+#define GDS_DEBUG_REG0__flush_request_MASK 0x00040000L
+#define GDS_DEBUG_REG0__flush_request__SHIFT 0x00000012
+#define GDS_DEBUG_REG0__last_pixel_ptr_MASK 0x00001000L
+#define GDS_DEBUG_REG0__last_pixel_ptr__SHIFT 0x0000000c
+#define GDS_DEBUG_REG0__spare1_MASK 0x00000001L
+#define GDS_DEBUG_REG0__spare1__SHIFT 0x00000000
+#define GDS_DEBUG_REG0__spare_MASK 0xff000000L
+#define GDS_DEBUG_REG0__spare__SHIFT 0x00000018
+#define GDS_DEBUG_REG0__wbuf_fifo_empty_MASK 0x00100000L
+#define GDS_DEBUG_REG0__wbuf_fifo_empty__SHIFT 0x00000014
+#define GDS_DEBUG_REG0__wbuf_fifo_full_MASK 0x00200000L
+#define GDS_DEBUG_REG0__wbuf_fifo_full__SHIFT 0x00000015
+#define GDS_DEBUG_REG0__wr_buffer_wr_complete_MASK 0x00080000L
+#define GDS_DEBUG_REG0__wr_buffer_wr_complete__SHIFT 0x00000013
+#define GDS_DEBUG_REG0__write_buff_valid_MASK 0x00000040L
+#define GDS_DEBUG_REG0__write_buff_valid__SHIFT 0x00000006
+#define GDS_DEBUG_REG0__wr_pixel_nxt_ptr_MASK 0x00000f80L
+#define GDS_DEBUG_REG0__wr_pixel_nxt_ptr__SHIFT 0x00000007
+#define GDS_DEBUG_REG1__addr_fifo_empty_MASK 0x00200000L
+#define GDS_DEBUG_REG1__addr_fifo_empty__SHIFT 0x00000015
+#define GDS_DEBUG_REG1__addr_fifo_full_MASK 0x00100000L
+#define GDS_DEBUG_REG1__addr_fifo_full__SHIFT 0x00000014
+#define GDS_DEBUG_REG1__awaiting_data_MASK 0x00080000L
+#define GDS_DEBUG_REG1__awaiting_data__SHIFT 0x00000013
+#define GDS_DEBUG_REG1__buffer_invalid_MASK 0x00800000L
+#define GDS_DEBUG_REG1__buffer_invalid__SHIFT 0x00000017
+#define GDS_DEBUG_REG1__buffer_loaded_MASK 0x00400000L
+#define GDS_DEBUG_REG1__buffer_loaded__SHIFT 0x00000016
+#define GDS_DEBUG_REG1__data_ready_MASK 0x00040000L
+#define GDS_DEBUG_REG1__data_ready__SHIFT 0x00000012
+#define GDS_DEBUG_REG1__pixel_addr_MASK 0x0001fffcL
+#define GDS_DEBUG_REG1__pixel_addr__SHIFT 0x00000002
+#define GDS_DEBUG_REG1__pixel_vld_MASK 0x00020000L
+#define GDS_DEBUG_REG1__pixel_vld__SHIFT 0x00000011
+#define GDS_DEBUG_REG1__spare_MASK 0xff000000L
+#define GDS_DEBUG_REG1__spare__SHIFT 0x00000018
+#define GDS_DEBUG_REG1__tag_hit_MASK 0x00000001L
+#define GDS_DEBUG_REG1__tag_hit__SHIFT 0x00000000
+#define GDS_DEBUG_REG1__tag_miss_MASK 0x00000002L
+#define GDS_DEBUG_REG1__tag_miss__SHIFT 0x00000001
+#define GDS_DEBUG_REG2__app_sel_MASK 0x000000f0L
+#define GDS_DEBUG_REG2__app_sel__SHIFT 0x00000004
+#define GDS_DEBUG_REG2__cmd_write_MASK 0x00000008L
+#define GDS_DEBUG_REG2__cmd_write__SHIFT 0x00000003
+#define GDS_DEBUG_REG2__ds_credit_avail_MASK 0x00000002L
+#define GDS_DEBUG_REG2__ds_credit_avail__SHIFT 0x00000001
+#define GDS_DEBUG_REG2__ds_full_MASK 0x00000001L
+#define GDS_DEBUG_REG2__ds_full__SHIFT 0x00000000
+#define GDS_DEBUG_REG2__ord_idx_free_MASK 0x00000004L
+#define GDS_DEBUG_REG2__ord_idx_free__SHIFT 0x00000002
+#define GDS_DEBUG_REG2__req_MASK 0x007fff00L
+#define GDS_DEBUG_REG2__req__SHIFT 0x00000008
+#define GDS_DEBUG_REG2__spare_MASK 0xff000000L
+#define GDS_DEBUG_REG2__spare__SHIFT 0x00000018
+#define GDS_DEBUG_REG3__pipe0_busy_num_MASK 0x00007800L
+#define GDS_DEBUG_REG3__pipe0_busy_num__SHIFT 0x0000000b
+#define GDS_DEBUG_REG3__pipe_num_busy_MASK 0x000007ffL
+#define GDS_DEBUG_REG3__pipe_num_busy__SHIFT 0x00000000
+#define GDS_DEBUG_REG3__spare_MASK 0xff000000L
+#define GDS_DEBUG_REG3__spare__SHIFT 0x00000018
+#define GDS_DEBUG_REG4__cmd_write_MASK 0x00020000L
+#define GDS_DEBUG_REG4__cmd_write__SHIFT 0x00000011
+#define GDS_DEBUG_REG4__credit_cnt_gt0_MASK 0x00010000L
+#define GDS_DEBUG_REG4__credit_cnt_gt0__SHIFT 0x00000010
+#define GDS_DEBUG_REG4__cur_reso_barrier_MASK 0x00002000L
+#define GDS_DEBUG_REG4__cur_reso_barrier__SHIFT 0x0000000d
+#define GDS_DEBUG_REG4__cur_reso_cnt_gt0_MASK 0x00008000L
+#define GDS_DEBUG_REG4__cur_reso_cnt_gt0__SHIFT 0x0000000f
+#define GDS_DEBUG_REG4__cur_reso_fed_MASK 0x00001000L
+#define GDS_DEBUG_REG4__cur_reso_fed__SHIFT 0x0000000c
+#define GDS_DEBUG_REG4__cur_reso_flag_MASK 0x00004000L
+#define GDS_DEBUG_REG4__cur_reso_flag__SHIFT 0x0000000e
+#define GDS_DEBUG_REG4__cur_reso_head_dirty_MASK 0x00000400L
+#define GDS_DEBUG_REG4__cur_reso_head_dirty__SHIFT 0x0000000a
+#define GDS_DEBUG_REG4__cur_reso_head_flag_MASK 0x00000800L
+#define GDS_DEBUG_REG4__cur_reso_head_flag__SHIFT 0x0000000b
+#define GDS_DEBUG_REG4__cur_reso_head_valid_MASK 0x00000200L
+#define GDS_DEBUG_REG4__cur_reso_head_valid__SHIFT 0x00000009
+#define GDS_DEBUG_REG4__cur_reso_MASK 0x000001f8L
+#define GDS_DEBUG_REG4__cur_reso__SHIFT 0x00000003
+#define GDS_DEBUG_REG4__grbm_gws_reso_rd_MASK 0x00080000L
+#define GDS_DEBUG_REG4__grbm_gws_reso_rd__SHIFT 0x00000013
+#define GDS_DEBUG_REG4__grbm_gws_reso_wr_MASK 0x00040000L
+#define GDS_DEBUG_REG4__grbm_gws_reso_wr__SHIFT 0x00000012
+#define GDS_DEBUG_REG4__gws_bulkfree_MASK 0x00200000L
+#define GDS_DEBUG_REG4__gws_bulkfree__SHIFT 0x00000015
+#define GDS_DEBUG_REG4__gws_busy_MASK 0x00000001L
+#define GDS_DEBUG_REG4__gws_busy__SHIFT 0x00000000
+#define GDS_DEBUG_REG4__gws_out_stall_MASK 0x00000004L
+#define GDS_DEBUG_REG4__gws_out_stall__SHIFT 0x00000002
+#define GDS_DEBUG_REG4__gws_req_MASK 0x00000002L
+#define GDS_DEBUG_REG4__gws_req__SHIFT 0x00000001
+#define GDS_DEBUG_REG4__ram_gws_re_MASK 0x00400000L
+#define GDS_DEBUG_REG4__ram_gws_re__SHIFT 0x00000016
+#define GDS_DEBUG_REG4__ram_gws_we_MASK 0x00800000L
+#define GDS_DEBUG_REG4__ram_gws_we__SHIFT 0x00000017
+#define GDS_DEBUG_REG4__ram_read_busy_MASK 0x00100000L
+#define GDS_DEBUG_REG4__ram_read_busy__SHIFT 0x00000014
+#define GDS_DEBUG_REG4__spare_MASK 0xff000000L
+#define GDS_DEBUG_REG4__spare__SHIFT 0x00000018
+#define GDS_DEBUG_REG5__alloc_opco_error_MASK 0x00000004L
+#define GDS_DEBUG_REG5__alloc_opco_error__SHIFT 0x00000002
+#define GDS_DEBUG_REG5__dealloc_opco_error_MASK 0x00000008L
+#define GDS_DEBUG_REG5__dealloc_opco_error__SHIFT 0x00000003
+#define GDS_DEBUG_REG5__dec_error_MASK 0x00000002L
+#define GDS_DEBUG_REG5__dec_error__SHIFT 0x00000001
+#define GDS_DEBUG_REG5__error_ds_address_MASK 0x003fff00L
+#define GDS_DEBUG_REG5__error_ds_address__SHIFT 0x00000008
+#define GDS_DEBUG_REG5__spare1_MASK 0xffc00000L
+#define GDS_DEBUG_REG5__spare1__SHIFT 0x00000016
+#define GDS_DEBUG_REG5__spare_MASK 0xff000000L
+#define GDS_DEBUG_REG5__spare__SHIFT 0x00000018
+#define GDS_DEBUG_REG5__wrap_opco_error_MASK 0x00000010L
+#define GDS_DEBUG_REG5__wrap_opco_error__SHIFT 0x00000004
+#define GDS_DEBUG_REG5__write_dis_MASK 0x00000001L
+#define GDS_DEBUG_REG5__write_dis__SHIFT 0x00000000
+#define GDS_DEBUG_REG6__counters_busy_MASK 0x001fffe0L
+#define GDS_DEBUG_REG6__counters_busy__SHIFT 0x00000005
+#define GDS_DEBUG_REG6__counters_enabled_MASK 0x0000001eL
+#define GDS_DEBUG_REG6__counters_enabled__SHIFT 0x00000001
+#define GDS_DEBUG_REG6__oa_busy_MASK 0x00000001L
+#define GDS_DEBUG_REG6__oa_busy__SHIFT 0x00000000
+#define GDS_DEBUG_REG6__spare_MASK 0xff000000L
+#define GDS_DEBUG_REG6__spare__SHIFT 0x00000018
+#define GDS_ENHANCE__AUTO_INC_INDEX_MASK 0x00010000L
+#define GDS_ENHANCE__AUTO_INC_INDEX__SHIFT 0x00000010
+#define GDS_ENHANCE__MISC_MASK 0x0000ffffL
+#define GDS_ENHANCE__MISC__SHIFT 0x00000000
+#define GDS_ENHANCE__UNUSED_MASK 0xffff0000L
+#define GDS_ENHANCE__UNUSED__SHIFT 0x00000010
+#define GDS_GRBM_SECDED_CNT__DED_MASK 0xffff0000L
+#define GDS_GRBM_SECDED_CNT__DED__SHIFT 0x00000010
+#define GDS_GRBM_SECDED_CNT__SEC_MASK 0x0000ffffL
+#define GDS_GRBM_SECDED_CNT__SEC__SHIFT 0x00000000
+#define GDS_GWS_RESOURCE_CNTL__INDEX_MASK 0x0000003fL
+#define GDS_GWS_RESOURCE_CNTL__INDEX__SHIFT 0x00000000
+#define GDS_GWS_RESOURCE_CNTL__UNUSED_MASK 0xffffffc0L
+#define GDS_GWS_RESOURCE_CNTL__UNUSED__SHIFT 0x00000006
+#define GDS_GWS_RESOURCE_CNT__RESOURCE_CNT_MASK 0x0000ffffL
+#define GDS_GWS_RESOURCE_CNT__RESOURCE_CNT__SHIFT 0x00000000
+#define GDS_GWS_RESOURCE_CNT__UNUSED_MASK 0xffff0000L
+#define GDS_GWS_RESOURCE_CNT__UNUSED__SHIFT 0x00000010
+#define GDS_GWS_RESOURCE__COUNTER_MASK 0x00001ffeL
+#define GDS_GWS_RESOURCE__COUNTER__SHIFT 0x00000001
+#define GDS_GWS_RESOURCE__DED_MASK 0x00004000L
+#define GDS_GWS_RESOURCE__DED__SHIFT 0x0000000e
+#define GDS_GWS_RESOURCE__FLAG_MASK 0x00000001L
+#define GDS_GWS_RESOURCE__FLAG__SHIFT 0x00000000
+#define GDS_GWS_RESOURCE__HEAD_FLAG_MASK 0x10000000L
+#define GDS_GWS_RESOURCE__HEAD_FLAG__SHIFT 0x0000001c
+#define GDS_GWS_RESOURCE__HEAD_QUEUE_MASK 0x07ff0000L
+#define GDS_GWS_RESOURCE__HEAD_QUEUE__SHIFT 0x00000010
+#define GDS_GWS_RESOURCE__HEAD_VALID_MASK 0x08000000L
+#define GDS_GWS_RESOURCE__HEAD_VALID__SHIFT 0x0000001b
+#define GDS_GWS_RESOURCE__RELEASE_ALL_MASK 0x00008000L
+#define GDS_GWS_RESOURCE__RELEASE_ALL__SHIFT 0x0000000f
+#define GDS_GWS_RESOURCE_RESET__RESET_MASK 0x00000001L
+#define GDS_GWS_RESOURCE_RESET__RESET__SHIFT 0x00000000
+#define GDS_GWS_RESOURCE_RESET__RESOURCE_ID_MASK 0x0000ff00L
+#define GDS_GWS_RESOURCE_RESET__RESOURCE_ID__SHIFT 0x00000008
+#define GDS_GWS_RESOURCE__TYPE_MASK 0x00002000L
+#define GDS_GWS_RESOURCE__TYPE__SHIFT 0x0000000d
+#define GDS_GWS_RESOURCE__UNUSED1_MASK 0xe0000000L
+#define GDS_GWS_RESOURCE__UNUSED1__SHIFT 0x0000001d
+#define GDS_OA_DED__ME0_CS_DED_MASK 0x00000004L
+#define GDS_OA_DED__ME0_CS_DED__SHIFT 0x00000002
+#define GDS_OA_DED__ME0_GFXHP3D_PIX_DED_MASK 0x00000001L
+#define GDS_OA_DED__ME0_GFXHP3D_PIX_DED__SHIFT 0x00000000
+#define GDS_OA_DED__ME0_GFXHP3D_VTX_DED_MASK 0x00000002L
+#define GDS_OA_DED__ME0_GFXHP3D_VTX_DED__SHIFT 0x00000001
+#define GDS_OA_DED__ME1_PIPE0_DED_MASK 0x00000010L
+#define GDS_OA_DED__ME1_PIPE0_DED__SHIFT 0x00000004
+#define GDS_OA_DED__ME1_PIPE1_DED_MASK 0x00000020L
+#define GDS_OA_DED__ME1_PIPE1_DED__SHIFT 0x00000005
+#define GDS_OA_DED__ME1_PIPE2_DED_MASK 0x00000040L
+#define GDS_OA_DED__ME1_PIPE2_DED__SHIFT 0x00000006
+#define GDS_OA_DED__ME1_PIPE3_DED_MASK 0x00000080L
+#define GDS_OA_DED__ME1_PIPE3_DED__SHIFT 0x00000007
+#define GDS_OA_DED__ME2_PIPE0_DED_MASK 0x00000100L
+#define GDS_OA_DED__ME2_PIPE0_DED__SHIFT 0x00000008
+#define GDS_OA_DED__ME2_PIPE1_DED_MASK 0x00000200L
+#define GDS_OA_DED__ME2_PIPE1_DED__SHIFT 0x00000009
+#define GDS_OA_DED__ME2_PIPE2_DED_MASK 0x00000400L
+#define GDS_OA_DED__ME2_PIPE2_DED__SHIFT 0x0000000a
+#define GDS_OA_DED__ME2_PIPE3_DED_MASK 0x00000800L
+#define GDS_OA_DED__ME2_PIPE3_DED__SHIFT 0x0000000b
+#define GDS_OA_DED__UNUSED0_MASK 0x00000008L
+#define GDS_OA_DED__UNUSED0__SHIFT 0x00000003
+#define GDS_OA_DED__UNUSED1_MASK 0xfffff000L
+#define GDS_OA_DED__UNUSED1__SHIFT 0x0000000c
+#define GDS_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define GDS_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define GDS_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define GDS_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2_MASK 0x000003ffL
+#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2__SHIFT 0x00000000
+#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3_MASK 0x000ffc00L
+#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3__SHIFT 0x0000000a
+#define GDS_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define GDS_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1_MASK 0x000ffc00L
+#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1__SHIFT 0x0000000a
+#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK 0x000000ffL
+#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT 0x00000000
+#define GDS_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define GDS_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define GDS_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define GDS_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define GDS_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define GDS_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1_MASK 0x000ffc00L
+#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1__SHIFT 0x0000000a
+#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK 0x000000ffL
+#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT 0x00000000
+#define GDS_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define GDS_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define GDS_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define GDS_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define GDS_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define GDS_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1_MASK 0x000ffc00L
+#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1__SHIFT 0x0000000a
+#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT_MASK 0x000000ffL
+#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT__SHIFT 0x00000000
+#define GDS_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define GDS_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define GDS_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define GDS_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define GDS_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define GDS_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1_MASK 0x000ffc00L
+#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1__SHIFT 0x0000000a
+#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT_MASK 0x000000ffL
+#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT__SHIFT 0x00000000
+#define GDS_RD_ADDR__READ_ADDR_MASK 0xffffffffL
+#define GDS_RD_ADDR__READ_ADDR__SHIFT 0x00000000
+#define GDS_RD_BURST_ADDR__BURST_ADDR_MASK 0xffffffffL
+#define GDS_RD_BURST_ADDR__BURST_ADDR__SHIFT 0x00000000
+#define GDS_RD_BURST_COUNT__BURST_COUNT_MASK 0xffffffffL
+#define GDS_RD_BURST_COUNT__BURST_COUNT__SHIFT 0x00000000
+#define GDS_RD_BURST_DATA__BURST_DATA_MASK 0xffffffffL
+#define GDS_RD_BURST_DATA__BURST_DATA__SHIFT 0x00000000
+#define GDS_RD_DATA__READ_DATA_MASK 0xffffffffL
+#define GDS_RD_DATA__READ_DATA__SHIFT 0x00000000
+#define GDS_SECDED_CNT__DED_MASK 0xffff0000L
+#define GDS_SECDED_CNT__DED__SHIFT 0x00000010
+#define GDS_SECDED_CNT__SEC_MASK 0x0000ffffL
+#define GDS_SECDED_CNT__SEC__SHIFT 0x00000000
+#define GDS_WR_ADDR__WRITE_ADDR_MASK 0xffffffffL
+#define GDS_WR_ADDR__WRITE_ADDR__SHIFT 0x00000000
+#define GDS_WR_BURST_ADDR__WRITE_ADDR_MASK 0xffffffffL
+#define GDS_WR_BURST_ADDR__WRITE_ADDR__SHIFT 0x00000000
+#define GDS_WR_BURST_DATA__WRITE_DATA_MASK 0xffffffffL
+#define GDS_WR_BURST_DATA__WRITE_DATA__SHIFT 0x00000000
+#define GDS_WR_DATA__WRITE_DATA_MASK 0xffffffffL
+#define GDS_WR_DATA__WRITE_DATA__SHIFT 0x00000000
+#define GDS_WRITE_COMPLETE__WRITE_COMPLETE_MASK 0xffffffffL
+#define GDS_WRITE_COMPLETE__WRITE_COMPLETE__SHIFT 0x00000000
+#define GFX_COPY_STATE__SRC_STATE_ID_MASK 0x00000007L
+#define GFX_COPY_STATE__SRC_STATE_ID__SHIFT 0x00000000
+#define GRBM_CAM_DATA__CAM_ADDR_MASK 0x0000ffffL
+#define GRBM_CAM_DATA__CAM_ADDR__SHIFT 0x00000000
+#define GRBM_CAM_DATA__CAM_REMAPADDR_MASK 0xffff0000L
+#define GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT 0x00000010
+#define GRBM_CAM_INDEX__CAM_INDEX_MASK 0x00000007L
+#define GRBM_CAM_INDEX__CAM_INDEX__SHIFT 0x00000000
+#define GRBM_CNTL__READ_TIMEOUT_MASK 0x000000ffL
+#define GRBM_CNTL__READ_TIMEOUT__SHIFT 0x00000000
+#define GRBM_DEBUG_CNTL__GRBM_DEBUG_INDEX_MASK 0x0000003fL
+#define GRBM_DEBUG_CNTL__GRBM_DEBUG_INDEX__SHIFT 0x00000000
+#define GRBM_DEBUG_DATA__DATA_MASK 0xffffffffL
+#define GRBM_DEBUG_DATA__DATA__SHIFT 0x00000000
+#define GRBM_DEBUG__DISABLE_READ_TIMEOUT_MASK 0x00000040L
+#define GRBM_DEBUG__DISABLE_READ_TIMEOUT__SHIFT 0x00000006
+#define GRBM_DEBUG__GFX_CLOCK_DOMAIN_OVERRIDE_MASK 0x00001000L
+#define GRBM_DEBUG__GFX_CLOCK_DOMAIN_OVERRIDE__SHIFT 0x0000000c
+#define GRBM_DEBUG__HYSTERESIS_GUI_ACTIVE_MASK 0x00000f00L
+#define GRBM_DEBUG__HYSTERESIS_GUI_ACTIVE__SHIFT 0x00000008
+#define GRBM_DEBUG__IGNORE_FAO_MASK 0x00000020L
+#define GRBM_DEBUG__IGNORE_FAO__SHIFT 0x00000005
+#define GRBM_DEBUG__IGNORE_RDY_MASK 0x00000002L
+#define GRBM_DEBUG__IGNORE_RDY__SHIFT 0x00000001
+#define GRBM_DEBUG_SNAPSHOT__CPF_RDY_MASK 0x00000001L
+#define GRBM_DEBUG_SNAPSHOT__CPF_RDY__SHIFT 0x00000000
+#define GRBM_DEBUG_SNAPSHOT__CPG_RDY_MASK 0x00000002L
+#define GRBM_DEBUG_SNAPSHOT__CPG_RDY__SHIFT 0x00000001
+#define GRBM_DEBUG__SNAPSHOT_FREE_CNTRS_MASK 0x00000080L
+#define GRBM_DEBUG__SNAPSHOT_FREE_CNTRS__SHIFT 0x00000007
+#define GRBM_DEBUG_SNAPSHOT__GDS_RDY_MASK 0x00000200L
+#define GRBM_DEBUG_SNAPSHOT__GDS_RDY__SHIFT 0x00000009
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE0_RDY0_MASK 0x00000040L
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE0_RDY0__SHIFT 0x00000006
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE0_RDY1_MASK 0x00004000L
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE0_RDY1__SHIFT 0x0000000e
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE1_RDY0_MASK 0x00000080L
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE1_RDY0__SHIFT 0x00000007
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE1_RDY1_MASK 0x00008000L
+#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE1_RDY1__SHIFT 0x0000000f
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE0_RDY0_MASK 0x00000100L
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE0_RDY0__SHIFT 0x00000008
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE0_RDY1_MASK 0x00010000L
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE0_RDY1__SHIFT 0x00000010
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE1_RDY0_MASK 0x00000200L
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE1_RDY0__SHIFT 0x00000009
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE1_RDY1_MASK 0x00020000L
+#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE1_RDY1__SHIFT 0x00000011
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE0_RDY0_MASK 0x00000400L
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE0_RDY0__SHIFT 0x0000000a
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE0_RDY1_MASK 0x00040000L
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE0_RDY1__SHIFT 0x00000012
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE1_RDY0_MASK 0x00000800L
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE1_RDY0__SHIFT 0x0000000b
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE1_RDY1_MASK 0x00080000L
+#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE1_RDY1__SHIFT 0x00000013
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE0_RDY0_MASK 0x00001000L
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE0_RDY0__SHIFT 0x0000000c
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE0_RDY1_MASK 0x00100000L
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE0_RDY1__SHIFT 0x00000014
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE1_RDY0_MASK 0x00002000L
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE1_RDY0__SHIFT 0x0000000d
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE1_RDY1_MASK 0x00200000L
+#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE1_RDY1__SHIFT 0x00000015
+#define GRBM_DEBUG_SNAPSHOT__SRBM_RDY_MASK 0x00000002L
+#define GRBM_DEBUG_SNAPSHOT__SRBM_RDY__SHIFT 0x00000001
+#define GRBM_DEBUG_SNAPSHOT__WD_ME0PIPE0_RDY_MASK 0x00000008L
+#define GRBM_DEBUG_SNAPSHOT__WD_ME0PIPE0_RDY__SHIFT 0x00000003
+#define GRBM_DEBUG_SNAPSHOT__WD_ME0PIPE1_RDY_MASK 0x00000010L
+#define GRBM_DEBUG_SNAPSHOT__WD_ME0PIPE1_RDY__SHIFT 0x00000004
+#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT_MASK 0x00001f00L
+#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT__SHIFT 0x00000008
+#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT_MASK 0x0000000fL
+#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT__SHIFT 0x00000000
+#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK 0x40000000L
+#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES__SHIFT 0x0000001e
+#define GRBM_GFX_INDEX__INSTANCE_INDEX_MASK 0x000000ffL
+#define GRBM_GFX_INDEX__INSTANCE_INDEX__SHIFT 0x00000000
+#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK 0x80000000L
+#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT 0x0000001f
+#define GRBM_GFX_INDEX__SE_INDEX_MASK 0x00ff0000L
+#define GRBM_GFX_INDEX__SE_INDEX__SHIFT 0x00000010
+#define GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK 0x20000000L
+#define GRBM_GFX_INDEX__SH_BROADCAST_WRITES__SHIFT 0x0000001d
+#define GRBM_GFX_INDEX__SH_INDEX_MASK 0x0000ff00L
+#define GRBM_GFX_INDEX__SH_INDEX__SHIFT 0x00000008
+#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE_MASK 0x00080000L
+#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE__SHIFT 0x00000013
+#define GRBM_INT_CNTL__RDERR_INT_ENABLE_MASK 0x00000001L
+#define GRBM_INT_CNTL__RDERR_INT_ENABLE__SHIFT 0x00000000
+#define GRBM_NOWHERE__DATA_MASK 0xffffffffL
+#define GRBM_NOWHERE__DATA__SHIFT 0x00000000
+#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x02000000L
+#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x00000019
+#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x00200000L
+#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x00000015
+#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x00000800L
+#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0x0000000b
+#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK 0x00400000L
+#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT 0x00000016
+#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x00100000L
+#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x00000014
+#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x00000400L
+#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0x0000000a
+#define GRBM_PERFCOUNTER0_SELECT__GDS_BUSY_USER_DEFINED_MASK_MASK 0x01000000L
+#define GRBM_PERFCOUNTER0_SELECT__GDS_BUSY_USER_DEFINED_MASK__SHIFT 0x00000018
+#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK 0x00080000L
+#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT 0x00000013
+#define GRBM_PERFCOUNTER0_SELECT__IA_BUSY_USER_DEFINED_MASK_MASK 0x00800000L
+#define GRBM_PERFCOUNTER0_SELECT__IA_BUSY_USER_DEFINED_MASK__SHIFT 0x00000017
+#define GRBM_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x00040000L
+#define GRBM_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x00000012
+#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x0000003fL
+#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK 0x04000000L
+#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT 0x0000001a
+#define GRBM_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x00020000L
+#define GRBM_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x00000011
+#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x00010000L
+#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0x00000010
+#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x00004000L
+#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0x0000000e
+#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x00002000L
+#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0x0000000d
+#define GRBM_PERFCOUNTER0_SELECT__TC_BUSY_USER_DEFINED_MASK_MASK 0x08000000L
+#define GRBM_PERFCOUNTER0_SELECT__TC_BUSY_USER_DEFINED_MASK__SHIFT 0x0000001b
+#define GRBM_PERFCOUNTER0_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x00001000L
+#define GRBM_PERFCOUNTER0_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0x0000000c
+#define GRBM_PERFCOUNTER0_SELECT__WD_BUSY_USER_DEFINED_MASK_MASK 0x10000000L
+#define GRBM_PERFCOUNTER0_SELECT__WD_BUSY_USER_DEFINED_MASK__SHIFT 0x0000001c
+#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x02000000L
+#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x00000019
+#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x00200000L
+#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x00000015
+#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x00000800L
+#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0x0000000b
+#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK 0x00400000L
+#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT 0x00000016
+#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x00100000L
+#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x00000014
+#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x00000400L
+#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0x0000000a
+#define GRBM_PERFCOUNTER1_SELECT__GDS_BUSY_USER_DEFINED_MASK_MASK 0x01000000L
+#define GRBM_PERFCOUNTER1_SELECT__GDS_BUSY_USER_DEFINED_MASK__SHIFT 0x00000018
+#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK 0x00080000L
+#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT 0x00000013
+#define GRBM_PERFCOUNTER1_SELECT__IA_BUSY_USER_DEFINED_MASK_MASK 0x00800000L
+#define GRBM_PERFCOUNTER1_SELECT__IA_BUSY_USER_DEFINED_MASK__SHIFT 0x00000017
+#define GRBM_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x00040000L
+#define GRBM_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x00000012
+#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x0000003fL
+#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK 0x04000000L
+#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT 0x0000001a
+#define GRBM_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x00020000L
+#define GRBM_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x00000011
+#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x00010000L
+#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0x00000010
+#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x00004000L
+#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0x0000000e
+#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x00002000L
+#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0x0000000d
+#define GRBM_PERFCOUNTER1_SELECT__TC_BUSY_USER_DEFINED_MASK_MASK 0x08000000L
+#define GRBM_PERFCOUNTER1_SELECT__TC_BUSY_USER_DEFINED_MASK__SHIFT 0x0000001b
+#define GRBM_PERFCOUNTER1_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x00001000L
+#define GRBM_PERFCOUNTER1_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0x0000000c
+#define GRBM_PERFCOUNTER1_SELECT__WD_BUSY_USER_DEFINED_MASK_MASK 0x10000000L
+#define GRBM_PERFCOUNTER1_SELECT__WD_BUSY_USER_DEFINED_MASK__SHIFT 0x0000001c
+#define GRBM_PWR_CNTL__REQ_TYPE_MASK 0x0000000fL
+#define GRBM_PWR_CNTL__REQ_TYPE__SHIFT 0x00000000
+#define GRBM_PWR_CNTL__RSP_TYPE_MASK 0x000000f0L
+#define GRBM_PWR_CNTL__RSP_TYPE__SHIFT 0x00000004
+#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA_MASK 0x00080000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA__SHIFT 0x00000013
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF_MASK 0x00100000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF__SHIFT 0x00000014
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF_MASK 0x00200000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF__SHIFT 0x00000015
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF_MASK 0x00400000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF__SHIFT 0x00000016
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF_MASK 0x00800000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF__SHIFT 0x00000017
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0_MASK 0x01000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0__SHIFT 0x00000018
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1_MASK 0x02000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1__SHIFT 0x00000019
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2_MASK 0x04000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2__SHIFT 0x0000001a
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3_MASK 0x08000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3__SHIFT 0x0000001b
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0_MASK 0x10000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0__SHIFT 0x0000001c
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1_MASK 0x20000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1__SHIFT 0x0000001d
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2_MASK 0x40000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2__SHIFT 0x0000001e
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3_MASK 0x80000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3__SHIFT 0x0000001f
+#define GRBM_READ_ERROR2__READ_REQUESTER_RLC_MASK 0x00040000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_RLC__SHIFT 0x00000012
+#define GRBM_READ_ERROR2__READ_REQUESTER_SRBM_MASK 0x00020000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_SRBM__SHIFT 0x00000011
+#define GRBM_READ_ERROR__READ_ADDRESS_MASK 0x0003fffcL
+#define GRBM_READ_ERROR__READ_ADDRESS__SHIFT 0x00000002
+#define GRBM_READ_ERROR__READ_ERROR_MASK 0x80000000L
+#define GRBM_READ_ERROR__READ_ERROR__SHIFT 0x0000001f
+#define GRBM_READ_ERROR__READ_MEID_MASK 0x00c00000L
+#define GRBM_READ_ERROR__READ_MEID__SHIFT 0x00000016
+#define GRBM_READ_ERROR__READ_PIPEID_MASK 0x00300000L
+#define GRBM_READ_ERROR__READ_PIPEID__SHIFT 0x00000014
+#define GRBM_SCRATCH_REG0__SCRATCH_REG0_MASK 0xffffffffL
+#define GRBM_SCRATCH_REG0__SCRATCH_REG0__SHIFT 0x00000000
+#define GRBM_SCRATCH_REG1__SCRATCH_REG1_MASK 0xffffffffL
+#define GRBM_SCRATCH_REG1__SCRATCH_REG1__SHIFT 0x00000000
+#define GRBM_SCRATCH_REG2__SCRATCH_REG2_MASK 0xffffffffL
+#define GRBM_SCRATCH_REG2__SCRATCH_REG2__SHIFT 0x00000000
+#define GRBM_SCRATCH_REG3__SCRATCH_REG3_MASK 0xffffffffL
+#define GRBM_SCRATCH_REG3__SCRATCH_REG3__SHIFT 0x00000000
+#define GRBM_SCRATCH_REG4__SCRATCH_REG4_MASK 0xffffffffL
+#define GRBM_SCRATCH_REG4__SCRATCH_REG4__SHIFT 0x00000000
+#define GRBM_SCRATCH_REG5__SCRATCH_REG5_MASK 0xffffffffL
+#define GRBM_SCRATCH_REG5__SCRATCH_REG5__SHIFT 0x00000000
+#define GRBM_SCRATCH_REG6__SCRATCH_REG6_MASK 0xffffffffL
+#define GRBM_SCRATCH_REG6__SCRATCH_REG6__SHIFT 0x00000000
+#define GRBM_SCRATCH_REG7__SCRATCH_REG7_MASK 0xffffffffL
+#define GRBM_SCRATCH_REG7__SCRATCH_REG7__SHIFT 0x00000000
+#define GRBM_SE0_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define GRBM_SE0_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define GRBM_SE0_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define GRBM_SE0_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define GRBM_SE0_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x00200000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x00000015
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x00040000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x00000012
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x00000800L
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0x0000000b
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x00020000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x00000011
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x00000400L
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0x0000000a
+#define GRBM_SE0_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x00100000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x00000014
+#define GRBM_SE0_PERFCOUNTER_SELECT__PERF_SEL_MASK 0x0000003fL
+#define GRBM_SE0_PERFCOUNTER_SELECT__PERF_SEL__SHIFT 0x00000000
+#define GRBM_SE0_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x00010000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x00000010
+#define GRBM_SE0_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x00008000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0x0000000f
+#define GRBM_SE0_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x00002000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0x0000000d
+#define GRBM_SE0_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x00001000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0x0000000c
+#define GRBM_SE0_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x00080000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0x00000013
+#define GRBM_SE1_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define GRBM_SE1_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define GRBM_SE1_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define GRBM_SE1_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define GRBM_SE1_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x00200000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x00000015
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x00040000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x00000012
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x00000800L
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0x0000000b
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x00020000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x00000011
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x00000400L
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0x0000000a
+#define GRBM_SE1_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x00100000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x00000014
+#define GRBM_SE1_PERFCOUNTER_SELECT__PERF_SEL_MASK 0x0000003fL
+#define GRBM_SE1_PERFCOUNTER_SELECT__PERF_SEL__SHIFT 0x00000000
+#define GRBM_SE1_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x00010000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x00000010
+#define GRBM_SE1_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x00008000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0x0000000f
+#define GRBM_SE1_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x00002000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0x0000000d
+#define GRBM_SE1_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x00001000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0x0000000c
+#define GRBM_SE1_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x00080000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0x00000013
+#define GRBM_SKEW_CNTL__SKEW_COUNT_MASK 0x00000fc0L
+#define GRBM_SKEW_CNTL__SKEW_COUNT__SHIFT 0x00000006
+#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD_MASK 0x0000003fL
+#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD__SHIFT 0x00000000
+#define GRBM_SOFT_RESET__SOFT_RESET_CPC_MASK 0x00040000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPC__SHIFT 0x00000012
+#define GRBM_SOFT_RESET__SOFT_RESET_CPF_MASK 0x00020000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPF__SHIFT 0x00000011
+#define GRBM_SOFT_RESET__SOFT_RESET_CPG_MASK 0x00080000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPG__SHIFT 0x00000013
+#define GRBM_SOFT_RESET__SOFT_RESET_CP_MASK 0x00000001L
+#define GRBM_SOFT_RESET__SOFT_RESET_CP__SHIFT 0x00000000
+#define GRBM_SOFT_RESET__SOFT_RESET_GFX_MASK 0x00010000L
+#define GRBM_SOFT_RESET__SOFT_RESET_GFX__SHIFT 0x00000010
+#define GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK 0x00000004L
+#define GRBM_SOFT_RESET__SOFT_RESET_RLC__SHIFT 0x00000002
+#define GRBM_STATUS2__CPC_BUSY_MASK 0x20000000L
+#define GRBM_STATUS2__CPC_BUSY__SHIFT 0x0000001d
+#define GRBM_STATUS2__CPF_BUSY_MASK 0x10000000L
+#define GRBM_STATUS2__CPF_BUSY__SHIFT 0x0000001c
+#define GRBM_STATUS2__CPG_BUSY_MASK 0x40000000L
+#define GRBM_STATUS2__CPG_BUSY__SHIFT 0x0000001e
+#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING_MASK 0x00000010L
+#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING__SHIFT 0x00000004
+#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL_MASK 0x0000000fL
+#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL__SHIFT 0x00000000
+#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING_MASK 0x00000020L
+#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING__SHIFT 0x00000005
+#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING_MASK 0x00000040L
+#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING__SHIFT 0x00000006
+#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING_MASK 0x00000080L
+#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING__SHIFT 0x00000007
+#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING_MASK 0x00000100L
+#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING__SHIFT 0x00000008
+#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING_MASK 0x00000200L
+#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING__SHIFT 0x00000009
+#define GRBM_STATUS2__ME2PIPE0_RQ_PENDING_MASK 0x00000400L
+#define GRBM_STATUS2__ME2PIPE0_RQ_PENDING__SHIFT 0x0000000a
+#define GRBM_STATUS2__ME2PIPE1_RQ_PENDING_MASK 0x00000800L
+#define GRBM_STATUS2__ME2PIPE1_RQ_PENDING__SHIFT 0x0000000b
+#define GRBM_STATUS2__ME2PIPE2_RQ_PENDING_MASK 0x00001000L
+#define GRBM_STATUS2__ME2PIPE2_RQ_PENDING__SHIFT 0x0000000c
+#define GRBM_STATUS2__ME2PIPE3_RQ_PENDING_MASK 0x00002000L
+#define GRBM_STATUS2__ME2PIPE3_RQ_PENDING__SHIFT 0x0000000d
+#define GRBM_STATUS2__RLC_BUSY_MASK 0x00000100L
+#define GRBM_STATUS2__RLC_BUSY__SHIFT 0x00000008
+#define GRBM_STATUS2__RLC_RQ_PENDING_MASK 0x00000001L
+#define GRBM_STATUS2__RLC_RQ_PENDING__SHIFT 0x00000000
+#define GRBM_STATUS2__TC_BUSY_MASK 0x00000200L
+#define GRBM_STATUS2__TC_BUSY__SHIFT 0x00000009
+#define GRBM_STATUS__BCI_BUSY_MASK 0x00800000L
+#define GRBM_STATUS__BCI_BUSY__SHIFT 0x00000017
+#define GRBM_STATUS__CB_BUSY_MASK 0x40000000L
+#define GRBM_STATUS__CB_BUSY__SHIFT 0x0000001e
+#define GRBM_STATUS__CB_CLEAN_MASK 0x00002000L
+#define GRBM_STATUS__CB_CLEAN__SHIFT 0x0000000d
+#define GRBM_STATUS__CP_BUSY_MASK 0x20000000L
+#define GRBM_STATUS__CP_BUSY__SHIFT 0x0000001d
+#define GRBM_STATUS__CP_COHERENCY_BUSY_MASK 0x10000000L
+#define GRBM_STATUS__CP_COHERENCY_BUSY__SHIFT 0x0000001c
+#define GRBM_STATUS__DB_BUSY_MASK 0x04000000L
+#define GRBM_STATUS__DB_BUSY__SHIFT 0x0000001a
+#define GRBM_STATUS__DB_CLEAN_MASK 0x00001000L
+#define GRBM_STATUS__DB_CLEAN__SHIFT 0x0000000c
+#define GRBM_STATUS__GDS_BUSY_MASK 0x00008000L
+#define GRBM_STATUS__GDS_BUSY__SHIFT 0x0000000f
+#define GRBM_STATUS__GDS_DMA_RQ_PENDING_MASK 0x00000200L
+#define GRBM_STATUS__GDS_DMA_RQ_PENDING__SHIFT 0x00000009
+#define GRBM_STATUS__GUI_ACTIVE_MASK 0x80000000L
+#define GRBM_STATUS__GUI_ACTIVE__SHIFT 0x0000001f
+#define GRBM_STATUS__IA_BUSY_MASK 0x00080000L
+#define GRBM_STATUS__IA_BUSY_NO_DMA_MASK 0x00040000L
+#define GRBM_STATUS__IA_BUSY_NO_DMA__SHIFT 0x00000012
+#define GRBM_STATUS__IA_BUSY__SHIFT 0x00000013
+#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING_MASK 0x00000080L
+#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING__SHIFT 0x00000007
+#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL_MASK 0x0000000fL
+#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL__SHIFT 0x00000000
+#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING_MASK 0x00000100L
+#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING__SHIFT 0x00000008
+#define GRBM_STATUS__PA_BUSY_MASK 0x02000000L
+#define GRBM_STATUS__PA_BUSY__SHIFT 0x00000019
+#define GRBM_STATUS__SC_BUSY_MASK 0x01000000L
+#define GRBM_STATUS__SC_BUSY__SHIFT 0x00000018
+#define GRBM_STATUS_SE0__BCI_BUSY_MASK 0x00400000L
+#define GRBM_STATUS_SE0__BCI_BUSY__SHIFT 0x00000016
+#define GRBM_STATUS_SE0__CB_BUSY_MASK 0x80000000L
+#define GRBM_STATUS_SE0__CB_BUSY__SHIFT 0x0000001f
+#define GRBM_STATUS_SE0__CB_CLEAN_MASK 0x00000004L
+#define GRBM_STATUS_SE0__CB_CLEAN__SHIFT 0x00000002
+#define GRBM_STATUS_SE0__DB_BUSY_MASK 0x40000000L
+#define GRBM_STATUS_SE0__DB_BUSY__SHIFT 0x0000001e
+#define GRBM_STATUS_SE0__DB_CLEAN_MASK 0x00000002L
+#define GRBM_STATUS_SE0__DB_CLEAN__SHIFT 0x00000001
+#define GRBM_STATUS_SE0__PA_BUSY_MASK 0x01000000L
+#define GRBM_STATUS_SE0__PA_BUSY__SHIFT 0x00000018
+#define GRBM_STATUS_SE0__SC_BUSY_MASK 0x20000000L
+#define GRBM_STATUS_SE0__SC_BUSY__SHIFT 0x0000001d
+#define GRBM_STATUS_SE0__SPI_BUSY_MASK 0x08000000L
+#define GRBM_STATUS_SE0__SPI_BUSY__SHIFT 0x0000001b
+#define GRBM_STATUS_SE0__SX_BUSY_MASK 0x04000000L
+#define GRBM_STATUS_SE0__SX_BUSY__SHIFT 0x0000001a
+#define GRBM_STATUS_SE0__TA_BUSY_MASK 0x02000000L
+#define GRBM_STATUS_SE0__TA_BUSY__SHIFT 0x00000019
+#define GRBM_STATUS_SE0__VGT_BUSY_MASK 0x00800000L
+#define GRBM_STATUS_SE0__VGT_BUSY__SHIFT 0x00000017
+#define GRBM_STATUS_SE1__BCI_BUSY_MASK 0x00400000L
+#define GRBM_STATUS_SE1__BCI_BUSY__SHIFT 0x00000016
+#define GRBM_STATUS_SE1__CB_BUSY_MASK 0x80000000L
+#define GRBM_STATUS_SE1__CB_BUSY__SHIFT 0x0000001f
+#define GRBM_STATUS_SE1__CB_CLEAN_MASK 0x00000004L
+#define GRBM_STATUS_SE1__CB_CLEAN__SHIFT 0x00000002
+#define GRBM_STATUS_SE1__DB_BUSY_MASK 0x40000000L
+#define GRBM_STATUS_SE1__DB_BUSY__SHIFT 0x0000001e
+#define GRBM_STATUS_SE1__DB_CLEAN_MASK 0x00000002L
+#define GRBM_STATUS_SE1__DB_CLEAN__SHIFT 0x00000001
+#define GRBM_STATUS_SE1__PA_BUSY_MASK 0x01000000L
+#define GRBM_STATUS_SE1__PA_BUSY__SHIFT 0x00000018
+#define GRBM_STATUS_SE1__SC_BUSY_MASK 0x20000000L
+#define GRBM_STATUS_SE1__SC_BUSY__SHIFT 0x0000001d
+#define GRBM_STATUS_SE1__SPI_BUSY_MASK 0x08000000L
+#define GRBM_STATUS_SE1__SPI_BUSY__SHIFT 0x0000001b
+#define GRBM_STATUS_SE1__SX_BUSY_MASK 0x04000000L
+#define GRBM_STATUS_SE1__SX_BUSY__SHIFT 0x0000001a
+#define GRBM_STATUS_SE1__TA_BUSY_MASK 0x02000000L
+#define GRBM_STATUS_SE1__TA_BUSY__SHIFT 0x00000019
+#define GRBM_STATUS_SE1__VGT_BUSY_MASK 0x00800000L
+#define GRBM_STATUS_SE1__VGT_BUSY__SHIFT 0x00000017
+#define GRBM_STATUS_SE2__BCI_BUSY_MASK 0x00400000L
+#define GRBM_STATUS_SE2__BCI_BUSY__SHIFT 0x00000016
+#define GRBM_STATUS_SE2__CB_BUSY_MASK 0x80000000L
+#define GRBM_STATUS_SE2__CB_BUSY__SHIFT 0x0000001f
+#define GRBM_STATUS_SE2__CB_CLEAN_MASK 0x00000004L
+#define GRBM_STATUS_SE2__CB_CLEAN__SHIFT 0x00000002
+#define GRBM_STATUS_SE2__DB_BUSY_MASK 0x40000000L
+#define GRBM_STATUS_SE2__DB_BUSY__SHIFT 0x0000001e
+#define GRBM_STATUS_SE2__DB_CLEAN_MASK 0x00000002L
+#define GRBM_STATUS_SE2__DB_CLEAN__SHIFT 0x00000001
+#define GRBM_STATUS_SE2__PA_BUSY_MASK 0x01000000L
+#define GRBM_STATUS_SE2__PA_BUSY__SHIFT 0x00000018
+#define GRBM_STATUS_SE2__SC_BUSY_MASK 0x20000000L
+#define GRBM_STATUS_SE2__SC_BUSY__SHIFT 0x0000001d
+#define GRBM_STATUS_SE2__SPI_BUSY_MASK 0x08000000L
+#define GRBM_STATUS_SE2__SPI_BUSY__SHIFT 0x0000001b
+#define GRBM_STATUS_SE2__SX_BUSY_MASK 0x04000000L
+#define GRBM_STATUS_SE2__SX_BUSY__SHIFT 0x0000001a
+#define GRBM_STATUS_SE2__TA_BUSY_MASK 0x02000000L
+#define GRBM_STATUS_SE2__TA_BUSY__SHIFT 0x00000019
+#define GRBM_STATUS_SE2__VGT_BUSY_MASK 0x00800000L
+#define GRBM_STATUS_SE2__VGT_BUSY__SHIFT 0x00000017
+#define GRBM_STATUS_SE3__BCI_BUSY_MASK 0x00400000L
+#define GRBM_STATUS_SE3__BCI_BUSY__SHIFT 0x00000016
+#define GRBM_STATUS_SE3__CB_BUSY_MASK 0x80000000L
+#define GRBM_STATUS_SE3__CB_BUSY__SHIFT 0x0000001f
+#define GRBM_STATUS_SE3__CB_CLEAN_MASK 0x00000004L
+#define GRBM_STATUS_SE3__CB_CLEAN__SHIFT 0x00000002
+#define GRBM_STATUS_SE3__DB_BUSY_MASK 0x40000000L
+#define GRBM_STATUS_SE3__DB_BUSY__SHIFT 0x0000001e
+#define GRBM_STATUS_SE3__DB_CLEAN_MASK 0x00000002L
+#define GRBM_STATUS_SE3__DB_CLEAN__SHIFT 0x00000001
+#define GRBM_STATUS_SE3__PA_BUSY_MASK 0x01000000L
+#define GRBM_STATUS_SE3__PA_BUSY__SHIFT 0x00000018
+#define GRBM_STATUS_SE3__SC_BUSY_MASK 0x20000000L
+#define GRBM_STATUS_SE3__SC_BUSY__SHIFT 0x0000001d
+#define GRBM_STATUS_SE3__SPI_BUSY_MASK 0x08000000L
+#define GRBM_STATUS_SE3__SPI_BUSY__SHIFT 0x0000001b
+#define GRBM_STATUS_SE3__SX_BUSY_MASK 0x04000000L
+#define GRBM_STATUS_SE3__SX_BUSY__SHIFT 0x0000001a
+#define GRBM_STATUS_SE3__TA_BUSY_MASK 0x02000000L
+#define GRBM_STATUS_SE3__TA_BUSY__SHIFT 0x00000019
+#define GRBM_STATUS_SE3__VGT_BUSY_MASK 0x00800000L
+#define GRBM_STATUS_SE3__VGT_BUSY__SHIFT 0x00000017
+#define GRBM_STATUS__SPI_BUSY_MASK 0x00400000L
+#define GRBM_STATUS__SPI_BUSY__SHIFT 0x00000016
+#define GRBM_STATUS__SRBM_RQ_PENDING_MASK 0x00000020L
+#define GRBM_STATUS__SRBM_RQ_PENDING__SHIFT 0x00000005
+#define GRBM_STATUS__SX_BUSY_MASK 0x00100000L
+#define GRBM_STATUS__SX_BUSY__SHIFT 0x00000014
+#define GRBM_STATUS__TA_BUSY_MASK 0x00004000L
+#define GRBM_STATUS__TA_BUSY__SHIFT 0x0000000e
+#define GRBM_STATUS__VGT_BUSY_MASK 0x00020000L
+#define GRBM_STATUS__VGT_BUSY__SHIFT 0x00000011
+#define GRBM_STATUS__WD_BUSY_MASK 0x00200000L
+#define GRBM_STATUS__WD_BUSY_NO_DMA_MASK 0x00010000L
+#define GRBM_STATUS__WD_BUSY_NO_DMA__SHIFT 0x00000010
+#define GRBM_STATUS__WD_BUSY__SHIFT 0x00000015
+#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS_MASK 0x000000ffL
+#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS__SHIFT 0x00000000
+#define IA_CNTL_STATUS__IA_ADC_BUSY_MASK 0x00000010L
+#define IA_CNTL_STATUS__IA_ADC_BUSY__SHIFT 0x00000004
+#define IA_CNTL_STATUS__IA_BUSY_MASK 0x00000001L
+#define IA_CNTL_STATUS__IA_BUSY__SHIFT 0x00000000
+#define IA_CNTL_STATUS__IA_DMA_BUSY_MASK 0x00000002L
+#define IA_CNTL_STATUS__IA_DMA_BUSY__SHIFT 0x00000001
+#define IA_CNTL_STATUS__IA_DMA_REQ_BUSY_MASK 0x00000004L
+#define IA_CNTL_STATUS__IA_DMA_REQ_BUSY__SHIFT 0x00000002
+#define IA_CNTL_STATUS__IA_GRP_BUSY_MASK 0x00000008L
+#define IA_CNTL_STATUS__IA_GRP_BUSY__SHIFT 0x00000003
+#define IA_DEBUG_CNTL__IA_DEBUG_INDX_MASK 0x0000003fL
+#define IA_DEBUG_CNTL__IA_DEBUG_INDX__SHIFT 0x00000000
+#define IA_DEBUG_CNTL__IA_DEBUG_SEL_BUS_B_MASK 0x00000040L
+#define IA_DEBUG_CNTL__IA_DEBUG_SEL_BUS_B__SHIFT 0x00000006
+#define IA_DEBUG_DATA__DATA_MASK 0xffffffffL
+#define IA_DEBUG_DATA__DATA__SHIFT 0x00000000
+#define IA_DEBUG_REG0__core_clk_busy_MASK 0x04000000L
+#define IA_DEBUG_REG0__core_clk_busy__SHIFT 0x0000001a
+#define IA_DEBUG_REG0__dma_busy_MASK 0x00000040L
+#define IA_DEBUG_REG0__dma_busy__SHIFT 0x00000006
+#define IA_DEBUG_REG0__dma_grp_hp_valid_MASK 0x00001000L
+#define IA_DEBUG_REG0__dma_grp_hp_valid__SHIFT 0x0000000c
+#define IA_DEBUG_REG0__dma_grp_valid_MASK 0x00000400L
+#define IA_DEBUG_REG0__dma_grp_valid__SHIFT 0x0000000a
+#define IA_DEBUG_REG0__dma_req_busy_MASK 0x00000020L
+#define IA_DEBUG_REG0__dma_req_busy__SHIFT 0x00000005
+#define IA_DEBUG_REG0__grp_busy_MASK 0x00000100L
+#define IA_DEBUG_REG0__grp_busy__SHIFT 0x00000008
+#define IA_DEBUG_REG0__grp_dma_hp_read_MASK 0x00002000L
+#define IA_DEBUG_REG0__grp_dma_hp_read__SHIFT 0x0000000d
+#define IA_DEBUG_REG0__grp_dma_read_MASK 0x00000800L
+#define IA_DEBUG_REG0__grp_dma_read__SHIFT 0x0000000b
+#define IA_DEBUG_REG0__ia_busy_extended_MASK 0x00000001L
+#define IA_DEBUG_REG0__ia_busy_extended__SHIFT 0x00000000
+#define IA_DEBUG_REG0__ia_busy_MASK 0x00000004L
+#define IA_DEBUG_REG0__ia_busy__SHIFT 0x00000002
+#define IA_DEBUG_REG0__ia_nodma_busy_extended_MASK 0x00000002L
+#define IA_DEBUG_REG0__ia_nodma_busy_extended__SHIFT 0x00000001
+#define IA_DEBUG_REG0__ia_nodma_busy_MASK 0x00000008L
+#define IA_DEBUG_REG0__ia_nodma_busy__SHIFT 0x00000003
+#define IA_DEBUG_REG0__mc_xl8r_busy_MASK 0x00000080L
+#define IA_DEBUG_REG0__mc_xl8r_busy__SHIFT 0x00000007
+#define IA_DEBUG_REG0__reg_clk_busy_MASK 0x01000000L
+#define IA_DEBUG_REG0__reg_clk_busy__SHIFT 0x00000018
+#define IA_DEBUG_REG0__sclk_core_vld_MASK 0x20000000L
+#define IA_DEBUG_REG0__sclk_core_vld__SHIFT 0x0000001d
+#define IA_DEBUG_REG0__sclk_reg_vld_MASK 0x10000000L
+#define IA_DEBUG_REG0__sclk_reg_vld__SHIFT 0x0000001c
+#define IA_DEBUG_REG0__SPARE0_MASK 0x00000010L
+#define IA_DEBUG_REG0__SPARE0__SHIFT 0x00000004
+#define IA_DEBUG_REG0__SPARE1_MASK 0x00000200L
+#define IA_DEBUG_REG0__SPARE1__SHIFT 0x00000009
+#define IA_DEBUG_REG0__SPARE2_MASK 0x00ffc000L
+#define IA_DEBUG_REG0__SPARE2__SHIFT 0x0000000e
+#define IA_DEBUG_REG0__SPARE3_MASK 0x00100000L
+#define IA_DEBUG_REG0__SPARE3__SHIFT 0x00000014
+#define IA_DEBUG_REG0__SPARE4_MASK 0x08000000L
+#define IA_DEBUG_REG0__SPARE4__SHIFT 0x0000001b
+#define IA_DEBUG_REG0__SPARE5_MASK 0x40000000L
+#define IA_DEBUG_REG0__SPARE5__SHIFT 0x0000001e
+#define IA_DEBUG_REG0__SPARE6_MASK 0x80000000L
+#define IA_DEBUG_REG0__SPARE6__SHIFT 0x0000001f
+#define IA_DEBUG_REG1__current_data_valid_MASK 0x10000000L
+#define IA_DEBUG_REG1__current_data_valid__SHIFT 0x0000001c
+#define IA_DEBUG_REG1__discard_1st_chunk_MASK 0x00000100L
+#define IA_DEBUG_REG1__discard_1st_chunk__SHIFT 0x00000008
+#define IA_DEBUG_REG1__discard_2nd_chunk_MASK 0x00000200L
+#define IA_DEBUG_REG1__discard_2nd_chunk__SHIFT 0x00000009
+#define IA_DEBUG_REG1__dma_buf_type_q_MASK 0x00000060L
+#define IA_DEBUG_REG1__dma_buf_type_q__SHIFT 0x00000005
+#define IA_DEBUG_REG1__dma_data_fifo_empty_q_MASK 0x00004000L
+#define IA_DEBUG_REG1__dma_data_fifo_empty_q__SHIFT 0x0000000e
+#define IA_DEBUG_REG1__dma_data_fifo_full_MASK 0x00008000L
+#define IA_DEBUG_REG1__dma_data_fifo_full__SHIFT 0x0000000f
+#define IA_DEBUG_REG1__dma_grp_valid_MASK 0x04000000L
+#define IA_DEBUG_REG1__dma_grp_valid__SHIFT 0x0000001a
+#define IA_DEBUG_REG1__dma_input_fifo_empty_MASK 0x00000001L
+#define IA_DEBUG_REG1__dma_input_fifo_empty__SHIFT 0x00000000
+#define IA_DEBUG_REG1__dma_input_fifo_full_MASK 0x00000002L
+#define IA_DEBUG_REG1__dma_input_fifo_full__SHIFT 0x00000001
+#define IA_DEBUG_REG1__dma_mask_fifo_empty_MASK 0x00002000L
+#define IA_DEBUG_REG1__dma_mask_fifo_empty__SHIFT 0x0000000d
+#define IA_DEBUG_REG1__dma_mask_fifo_we_MASK 0x40000000L
+#define IA_DEBUG_REG1__dma_mask_fifo_we__SHIFT 0x0000001e
+#define IA_DEBUG_REG1__dma_rdreq_dr_q_MASK 0x00000008L
+#define IA_DEBUG_REG1__dma_rdreq_dr_q__SHIFT 0x00000003
+#define IA_DEBUG_REG1__dma_req_fifo_empty_MASK 0x00010000L
+#define IA_DEBUG_REG1__dma_req_fifo_empty__SHIFT 0x00000010
+#define IA_DEBUG_REG1__dma_req_fifo_full_MASK 0x00020000L
+#define IA_DEBUG_REG1__dma_req_fifo_full__SHIFT 0x00000011
+#define IA_DEBUG_REG1__dma_req_path_q_MASK 0x00000080L
+#define IA_DEBUG_REG1__dma_req_path_q__SHIFT 0x00000007
+#define IA_DEBUG_REG1__dma_ret_data_we_q_MASK 0x80000000L
+#define IA_DEBUG_REG1__dma_ret_data_we_q__SHIFT 0x0000001f
+#define IA_DEBUG_REG1__dma_skid_fifo_empty_MASK 0x01000000L
+#define IA_DEBUG_REG1__dma_skid_fifo_empty__SHIFT 0x00000018
+#define IA_DEBUG_REG1__dma_skid_fifo_full_MASK 0x02000000L
+#define IA_DEBUG_REG1__dma_skid_fifo_full__SHIFT 0x00000019
+#define IA_DEBUG_REG1__dma_tc_ret_sel_q_MASK 0x00000800L
+#define IA_DEBUG_REG1__dma_tc_ret_sel_q__SHIFT 0x0000000b
+#define IA_DEBUG_REG1__dma_zero_indices_q_MASK 0x00000010L
+#define IA_DEBUG_REG1__dma_zero_indices_q__SHIFT 0x00000004
+#define IA_DEBUG_REG1__grp_dma_read_MASK 0x08000000L
+#define IA_DEBUG_REG1__grp_dma_read__SHIFT 0x0000001b
+#define IA_DEBUG_REG1__last_rdreq_in_dma_op_MASK 0x00001000L
+#define IA_DEBUG_REG1__last_rdreq_in_dma_op__SHIFT 0x0000000c
+#define IA_DEBUG_REG1__out_of_range_r2_q_MASK 0x20000000L
+#define IA_DEBUG_REG1__out_of_range_r2_q__SHIFT 0x0000001d
+#define IA_DEBUG_REG1__second_tc_ret_data_q_MASK 0x00000400L
+#define IA_DEBUG_REG1__second_tc_ret_data_q__SHIFT 0x0000000a
+#define IA_DEBUG_REG1__stage2_dr_MASK 0x00040000L
+#define IA_DEBUG_REG1__stage2_dr__SHIFT 0x00000012
+#define IA_DEBUG_REG1__stage2_rtr_MASK 0x00080000L
+#define IA_DEBUG_REG1__stage2_rtr__SHIFT 0x00000013
+#define IA_DEBUG_REG1__stage3_dr_MASK 0x00100000L
+#define IA_DEBUG_REG1__stage3_dr__SHIFT 0x00000014
+#define IA_DEBUG_REG1__stage3_rtr_MASK 0x00200000L
+#define IA_DEBUG_REG1__stage3_rtr__SHIFT 0x00000015
+#define IA_DEBUG_REG1__stage4_dr_MASK 0x00400000L
+#define IA_DEBUG_REG1__stage4_dr__SHIFT 0x00000016
+#define IA_DEBUG_REG1__stage4_rtr_MASK 0x00800000L
+#define IA_DEBUG_REG1__stage4_rtr__SHIFT 0x00000017
+#define IA_DEBUG_REG1__start_new_packet_MASK 0x00000004L
+#define IA_DEBUG_REG1__start_new_packet__SHIFT 0x00000002
+#define IA_DEBUG_REG2__hp_current_data_valid_MASK 0x10000000L
+#define IA_DEBUG_REG2__hp_current_data_valid__SHIFT 0x0000001c
+#define IA_DEBUG_REG2__hp_discard_1st_chunk_MASK 0x00000100L
+#define IA_DEBUG_REG2__hp_discard_1st_chunk__SHIFT 0x00000008
+#define IA_DEBUG_REG2__hp_discard_2nd_chunk_MASK 0x00000200L
+#define IA_DEBUG_REG2__hp_discard_2nd_chunk__SHIFT 0x00000009
+#define IA_DEBUG_REG2__hp_dma_buf_type_q_MASK 0x00000060L
+#define IA_DEBUG_REG2__hp_dma_buf_type_q__SHIFT 0x00000005
+#define IA_DEBUG_REG2__hp_dma_data_fifo_empty_q_MASK 0x00004000L
+#define IA_DEBUG_REG2__hp_dma_data_fifo_empty_q__SHIFT 0x0000000e
+#define IA_DEBUG_REG2__hp_dma_data_fifo_full_MASK 0x00008000L
+#define IA_DEBUG_REG2__hp_dma_data_fifo_full__SHIFT 0x0000000f
+#define IA_DEBUG_REG2__hp_dma_grp_valid_MASK 0x04000000L
+#define IA_DEBUG_REG2__hp_dma_grp_valid__SHIFT 0x0000001a
+#define IA_DEBUG_REG2__hp_dma_input_fifo_empty_MASK 0x00000001L
+#define IA_DEBUG_REG2__hp_dma_input_fifo_empty__SHIFT 0x00000000
+#define IA_DEBUG_REG2__hp_dma_input_fifo_full_MASK 0x00000002L
+#define IA_DEBUG_REG2__hp_dma_input_fifo_full__SHIFT 0x00000001
+#define IA_DEBUG_REG2__hp_dma_mask_fifo_empty_MASK 0x00002000L
+#define IA_DEBUG_REG2__hp_dma_mask_fifo_empty__SHIFT 0x0000000d
+#define IA_DEBUG_REG2__hp_dma_mask_fifo_we_MASK 0x40000000L
+#define IA_DEBUG_REG2__hp_dma_mask_fifo_we__SHIFT 0x0000001e
+#define IA_DEBUG_REG2__hp_dma_rdreq_dr_q_MASK 0x00000008L
+#define IA_DEBUG_REG2__hp_dma_rdreq_dr_q__SHIFT 0x00000003
+#define IA_DEBUG_REG2__hp_dma_req_fifo_empty_MASK 0x00010000L
+#define IA_DEBUG_REG2__hp_dma_req_fifo_empty__SHIFT 0x00000010
+#define IA_DEBUG_REG2__hp_dma_req_fifo_full_MASK 0x00020000L
+#define IA_DEBUG_REG2__hp_dma_req_fifo_full__SHIFT 0x00000011
+#define IA_DEBUG_REG2__hp_dma_req_path_q_MASK 0x00000080L
+#define IA_DEBUG_REG2__hp_dma_req_path_q__SHIFT 0x00000007
+#define IA_DEBUG_REG2__hp_dma_ret_data_we_q_MASK 0x80000000L
+#define IA_DEBUG_REG2__hp_dma_ret_data_we_q__SHIFT 0x0000001f
+#define IA_DEBUG_REG2__hp_dma_skid_fifo_empty_MASK 0x01000000L
+#define IA_DEBUG_REG2__hp_dma_skid_fifo_empty__SHIFT 0x00000018
+#define IA_DEBUG_REG2__hp_dma_skid_fifo_full_MASK 0x02000000L
+#define IA_DEBUG_REG2__hp_dma_skid_fifo_full__SHIFT 0x00000019
+#define IA_DEBUG_REG2__hp_dma_tc_ret_sel_q_MASK 0x00000800L
+#define IA_DEBUG_REG2__hp_dma_tc_ret_sel_q__SHIFT 0x0000000b
+#define IA_DEBUG_REG2__hp_dma_zero_indices_q_MASK 0x00000010L
+#define IA_DEBUG_REG2__hp_dma_zero_indices_q__SHIFT 0x00000004
+#define IA_DEBUG_REG2__hp_grp_dma_read_MASK 0x08000000L
+#define IA_DEBUG_REG2__hp_grp_dma_read__SHIFT 0x0000001b
+#define IA_DEBUG_REG2__hp_last_rdreq_in_dma_op_MASK 0x00001000L
+#define IA_DEBUG_REG2__hp_last_rdreq_in_dma_op__SHIFT 0x0000000c
+#define IA_DEBUG_REG2__hp_out_of_range_r2_q_MASK 0x20000000L
+#define IA_DEBUG_REG2__hp_out_of_range_r2_q__SHIFT 0x0000001d
+#define IA_DEBUG_REG2__hp_second_tc_ret_data_q_MASK 0x00000400L
+#define IA_DEBUG_REG2__hp_second_tc_ret_data_q__SHIFT 0x0000000a
+#define IA_DEBUG_REG2__hp_stage2_dr_MASK 0x00040000L
+#define IA_DEBUG_REG2__hp_stage2_dr__SHIFT 0x00000012
+#define IA_DEBUG_REG2__hp_stage2_rtr_MASK 0x00080000L
+#define IA_DEBUG_REG2__hp_stage2_rtr__SHIFT 0x00000013
+#define IA_DEBUG_REG2__hp_stage3_dr_MASK 0x00100000L
+#define IA_DEBUG_REG2__hp_stage3_dr__SHIFT 0x00000014
+#define IA_DEBUG_REG2__hp_stage3_rtr_MASK 0x00200000L
+#define IA_DEBUG_REG2__hp_stage3_rtr__SHIFT 0x00000015
+#define IA_DEBUG_REG2__hp_stage4_dr_MASK 0x00400000L
+#define IA_DEBUG_REG2__hp_stage4_dr__SHIFT 0x00000016
+#define IA_DEBUG_REG2__hp_stage4_rtr_MASK 0x00800000L
+#define IA_DEBUG_REG2__hp_stage4_rtr__SHIFT 0x00000017
+#define IA_DEBUG_REG2__hp_start_new_packet_MASK 0x00000004L
+#define IA_DEBUG_REG2__hp_start_new_packet__SHIFT 0x00000002
+#define IA_DEBUG_REG3__discard_1st_chunk_MASK 0x04000000L
+#define IA_DEBUG_REG3__discard_1st_chunk__SHIFT 0x0000001a
+#define IA_DEBUG_REG3__discard_2nd_chunk_MASK 0x08000000L
+#define IA_DEBUG_REG3__discard_2nd_chunk__SHIFT 0x0000001b
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_eop_out_MASK 0x00000008L
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_eop_out__SHIFT 0x00000003
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_null_out_MASK 0x00000004L
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_null_out__SHIFT 0x00000002
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_read_MASK 0x00000002L
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_read__SHIFT 0x00000001
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_use_tc_out_MASK 0x00000010L
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_use_tc_out__SHIFT 0x00000004
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_valid_MASK 0x00000001L
+#define IA_DEBUG_REG3__dma_pipe0_rdreq_valid__SHIFT 0x00000000
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_eop_out_MASK 0x00000800L
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_eop_out__SHIFT 0x0000000b
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_null_out_MASK 0x00000400L
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_null_out__SHIFT 0x0000000a
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_read_MASK 0x00000200L
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_read__SHIFT 0x00000009
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_use_tc_out_MASK 0x00001000L
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_use_tc_out__SHIFT 0x0000000c
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_valid_MASK 0x00000100L
+#define IA_DEBUG_REG3__dma_pipe1_rdreq_valid__SHIFT 0x00000008
+#define IA_DEBUG_REG3__dma_rdreq_send_out_MASK 0x00008000L
+#define IA_DEBUG_REG3__dma_rdreq_send_out__SHIFT 0x0000000f
+#define IA_DEBUG_REG3__grp_dma_draw_is_pipe0_MASK 0x00000020L
+#define IA_DEBUG_REG3__grp_dma_draw_is_pipe0__SHIFT 0x00000005
+#define IA_DEBUG_REG3__ia_mc_rdreq_rtr_q_MASK 0x00002000L
+#define IA_DEBUG_REG3__ia_mc_rdreq_rtr_q__SHIFT 0x0000000d
+#define IA_DEBUG_REG3__ia_tc_rdreq_rtr_q_MASK 0x00040000L
+#define IA_DEBUG_REG3__ia_tc_rdreq_rtr_q__SHIFT 0x00000012
+#define IA_DEBUG_REG3__IA_TC_rdreq_send_out_MASK 0x20000000L
+#define IA_DEBUG_REG3__IA_TC_rdreq_send_out__SHIFT 0x0000001d
+#define IA_DEBUG_REG3__last_tc_req_p1_MASK 0x10000000L
+#define IA_DEBUG_REG3__last_tc_req_p1__SHIFT 0x0000001c
+#define IA_DEBUG_REG3__mc_out_rtr_MASK 0x00004000L
+#define IA_DEBUG_REG3__mc_out_rtr__SHIFT 0x0000000e
+#define IA_DEBUG_REG3__must_service_pipe0_req_MASK 0x00000040L
+#define IA_DEBUG_REG3__must_service_pipe0_req__SHIFT 0x00000006
+#define IA_DEBUG_REG3__pair0_valid_p1_MASK 0x00100000L
+#define IA_DEBUG_REG3__pair0_valid_p1__SHIFT 0x00000014
+#define IA_DEBUG_REG3__pair1_valid_p1_MASK 0x00200000L
+#define IA_DEBUG_REG3__pair1_valid_p1__SHIFT 0x00000015
+#define IA_DEBUG_REG3__pair2_valid_p1_MASK 0x00400000L
+#define IA_DEBUG_REG3__pair2_valid_p1__SHIFT 0x00000016
+#define IA_DEBUG_REG3__pair3_valid_p1_MASK 0x00800000L
+#define IA_DEBUG_REG3__pair3_valid_p1__SHIFT 0x00000017
+#define IA_DEBUG_REG3__pipe0_dr_MASK 0x00010000L
+#define IA_DEBUG_REG3__pipe0_dr__SHIFT 0x00000010
+#define IA_DEBUG_REG3__pipe0_rtr_MASK 0x00020000L
+#define IA_DEBUG_REG3__pipe0_rtr__SHIFT 0x00000011
+#define IA_DEBUG_REG3__send_pipe1_req_MASK 0x00000080L
+#define IA_DEBUG_REG3__send_pipe1_req__SHIFT 0x00000007
+#define IA_DEBUG_REG3__TAP_IA_rdret_vld_in_MASK 0x80000000L
+#define IA_DEBUG_REG3__TAP_IA_rdret_vld_in__SHIFT 0x0000001f
+#define IA_DEBUG_REG3__TC_IA_rdret_valid_in_MASK 0x40000000L
+#define IA_DEBUG_REG3__TC_IA_rdret_valid_in__SHIFT 0x0000001e
+#define IA_DEBUG_REG3__tc_out_rtr_MASK 0x00080000L
+#define IA_DEBUG_REG3__tc_out_rtr__SHIFT 0x00000013
+#define IA_DEBUG_REG3__tc_req_count_q_MASK 0x03000000L
+#define IA_DEBUG_REG3__tc_req_count_q__SHIFT 0x00000018
+#define IA_DEBUG_REG4__current_shift_is_vect1_q_MASK 0x80000000L
+#define IA_DEBUG_REG4__current_shift_is_vect1_q__SHIFT 0x0000001f
+#define IA_DEBUG_REG4__di_event_flag_p1_q_MASK 0x00100000L
+#define IA_DEBUG_REG4__di_event_flag_p1_q__SHIFT 0x00000014
+#define IA_DEBUG_REG4__di_first_group_of_draw_q_MASK 0x20000000L
+#define IA_DEBUG_REG4__di_first_group_of_draw_q__SHIFT 0x0000001d
+#define IA_DEBUG_REG4__di_major_mode_p1_q_MASK 0x00010000L
+#define IA_DEBUG_REG4__di_major_mode_p1_q__SHIFT 0x00000010
+#define IA_DEBUG_REG4__di_source_select_p1_q_MASK 0x0c000000L
+#define IA_DEBUG_REG4__di_source_select_p1_q__SHIFT 0x0000001a
+#define IA_DEBUG_REG4__di_state_sel_p1_q_MASK 0x00e00000L
+#define IA_DEBUG_REG4__di_state_sel_p1_q__SHIFT 0x00000015
+#define IA_DEBUG_REG4__draw_opaq_active_q_MASK 0x02000000L
+#define IA_DEBUG_REG4__draw_opaq_active_q__SHIFT 0x00000019
+#define IA_DEBUG_REG4__draw_opaq_en_p1_q_MASK 0x01000000L
+#define IA_DEBUG_REG4__draw_opaq_en_p1_q__SHIFT 0x00000018
+#define IA_DEBUG_REG4__grp_se0_fifo_empty_MASK 0x00000040L
+#define IA_DEBUG_REG4__grp_se0_fifo_empty__SHIFT 0x00000006
+#define IA_DEBUG_REG4__grp_se0_fifo_full_MASK 0x00000080L
+#define IA_DEBUG_REG4__grp_se0_fifo_full__SHIFT 0x00000007
+#define IA_DEBUG_REG4__gs_mode_p1_q_MASK 0x000e0000L
+#define IA_DEBUG_REG4__gs_mode_p1_q__SHIFT 0x00000011
+#define IA_DEBUG_REG4__ia_se1vgt_prim_rtr_q_MASK 0x00008000L
+#define IA_DEBUG_REG4__ia_se1vgt_prim_rtr_q__SHIFT 0x0000000f
+#define IA_DEBUG_REG4__ia_vgt_prim_rtr_q_MASK 0x00004000L
+#define IA_DEBUG_REG4__ia_vgt_prim_rtr_q__SHIFT 0x0000000e
+#define IA_DEBUG_REG4__last_shift_of_draw_MASK 0x40000000L
+#define IA_DEBUG_REG4__last_shift_of_draw__SHIFT 0x0000001e
+#define IA_DEBUG_REG4__pipe0_dr_MASK 0x00000001L
+#define IA_DEBUG_REG4__pipe0_dr__SHIFT 0x00000000
+#define IA_DEBUG_REG4__pipe0_rtr_MASK 0x00000100L
+#define IA_DEBUG_REG4__pipe0_rtr__SHIFT 0x00000008
+#define IA_DEBUG_REG4__pipe1_dr_MASK 0x00000002L
+#define IA_DEBUG_REG4__pipe1_dr__SHIFT 0x00000001
+#define IA_DEBUG_REG4__pipe1_rtr_MASK 0x00000200L
+#define IA_DEBUG_REG4__pipe1_rtr__SHIFT 0x00000009
+#define IA_DEBUG_REG4__pipe2_dr_MASK 0x00000004L
+#define IA_DEBUG_REG4__pipe2_dr__SHIFT 0x00000002
+#define IA_DEBUG_REG4__pipe2_rtr_MASK 0x00000400L
+#define IA_DEBUG_REG4__pipe2_rtr__SHIFT 0x0000000a
+#define IA_DEBUG_REG4__pipe3_dr_MASK 0x00000008L
+#define IA_DEBUG_REG4__pipe3_dr__SHIFT 0x00000003
+#define IA_DEBUG_REG4__pipe3_rtr_MASK 0x00000800L
+#define IA_DEBUG_REG4__pipe3_rtr__SHIFT 0x0000000b
+#define IA_DEBUG_REG4__pipe4_dr_MASK 0x00000010L
+#define IA_DEBUG_REG4__pipe4_dr__SHIFT 0x00000004
+#define IA_DEBUG_REG4__pipe4_rtr_MASK 0x00001000L
+#define IA_DEBUG_REG4__pipe4_rtr__SHIFT 0x0000000c
+#define IA_DEBUG_REG4__pipe5_dr_MASK 0x00000020L
+#define IA_DEBUG_REG4__pipe5_dr__SHIFT 0x00000005
+#define IA_DEBUG_REG4__pipe5_rtr_MASK 0x00002000L
+#define IA_DEBUG_REG4__pipe5_rtr__SHIFT 0x0000000d
+#define IA_DEBUG_REG4__ready_to_read_di_MASK 0x10000000L
+#define IA_DEBUG_REG4__ready_to_read_di__SHIFT 0x0000001c
+#define IA_DEBUG_REG5__di_index_counter_q_15_0_MASK 0x0000ffffL
+#define IA_DEBUG_REG5__di_index_counter_q_15_0__SHIFT 0x00000000
+#define IA_DEBUG_REG5__draw_input_fifo_empty_MASK 0x80000000L
+#define IA_DEBUG_REG5__draw_input_fifo_empty__SHIFT 0x0000001f
+#define IA_DEBUG_REG5__draw_input_fifo_full_MASK 0x40000000L
+#define IA_DEBUG_REG5__draw_input_fifo_full__SHIFT 0x0000001e
+#define IA_DEBUG_REG5__instanceid_13_0_MASK 0x3fff0000L
+#define IA_DEBUG_REG5__instanceid_13_0__SHIFT 0x00000010
+#define IA_DEBUG_REG6__after_group_partial_MASK 0x00400000L
+#define IA_DEBUG_REG6__after_group_partial__SHIFT 0x00000016
+#define IA_DEBUG_REG6__current_shift_q_MASK 0x0000000fL
+#define IA_DEBUG_REG6__current_shift_q__SHIFT 0x00000000
+#define IA_DEBUG_REG6__current_stride_pre_MASK 0x000000f0L
+#define IA_DEBUG_REG6__current_stride_pre__SHIFT 0x00000004
+#define IA_DEBUG_REG6__current_stride_q_MASK 0x00001f00L
+#define IA_DEBUG_REG6__current_stride_q__SHIFT 0x00000008
+#define IA_DEBUG_REG6__curr_prim_partial_MASK 0x00008000L
+#define IA_DEBUG_REG6__curr_prim_partial__SHIFT 0x0000000f
+#define IA_DEBUG_REG6__extract_group_MASK 0x00800000L
+#define IA_DEBUG_REG6__extract_group__SHIFT 0x00000017
+#define IA_DEBUG_REG6__first_group_partial_MASK 0x00002000L
+#define IA_DEBUG_REG6__first_group_partial__SHIFT 0x0000000d
+#define IA_DEBUG_REG6__grp_shift_debug_data_MASK 0xff000000L
+#define IA_DEBUG_REG6__grp_shift_debug_data__SHIFT 0x00000018
+#define IA_DEBUG_REG6__next_group_partial_MASK 0x00200000L
+#define IA_DEBUG_REG6__next_group_partial__SHIFT 0x00000015
+#define IA_DEBUG_REG6__next_stride_q_MASK 0x001f0000L
+#define IA_DEBUG_REG6__next_stride_q__SHIFT 0x00000010
+#define IA_DEBUG_REG6__second_group_partial_MASK 0x00004000L
+#define IA_DEBUG_REG6__second_group_partial__SHIFT 0x0000000e
+#define IA_DEBUG_REG7__indx_shift_is_one_p2_q_MASK 0x02000000L
+#define IA_DEBUG_REG7__indx_shift_is_one_p2_q__SHIFT 0x00000019
+#define IA_DEBUG_REG7__indx_shift_is_two_p2_q_MASK 0x04000000L
+#define IA_DEBUG_REG7__indx_shift_is_two_p2_q__SHIFT 0x0000001a
+#define IA_DEBUG_REG7__indx_stride_is_four_p2_q_MASK 0x08000000L
+#define IA_DEBUG_REG7__indx_stride_is_four_p2_q__SHIFT 0x0000001b
+#define IA_DEBUG_REG7__last_group_of_draw_p2_q_MASK 0x00800000L
+#define IA_DEBUG_REG7__last_group_of_draw_p2_q__SHIFT 0x00000017
+#define IA_DEBUG_REG7__num_indx_in_group_p2_q_MASK 0x00700000L
+#define IA_DEBUG_REG7__num_indx_in_group_p2_q__SHIFT 0x00000014
+#define IA_DEBUG_REG7__reset_indx_state_q_MASK 0x0000000fL
+#define IA_DEBUG_REG7__reset_indx_state_q__SHIFT 0x00000000
+#define IA_DEBUG_REG7__shift_event_flag_p2_q_MASK 0x01000000L
+#define IA_DEBUG_REG7__shift_event_flag_p2_q__SHIFT 0x00000018
+#define IA_DEBUG_REG7__shift_prim0_partial_p3_q_MASK 0x80000000L
+#define IA_DEBUG_REG7__shift_prim0_partial_p3_q__SHIFT 0x0000001f
+#define IA_DEBUG_REG7__shift_prim0_reset_p3_q_MASK 0x40000000L
+#define IA_DEBUG_REG7__shift_prim0_reset_p3_q__SHIFT 0x0000001e
+#define IA_DEBUG_REG7__shift_prim1_partial_p3_q_MASK 0x20000000L
+#define IA_DEBUG_REG7__shift_prim1_partial_p3_q__SHIFT 0x0000001d
+#define IA_DEBUG_REG7__shift_prim1_reset_p3_q_MASK 0x10000000L
+#define IA_DEBUG_REG7__shift_prim1_reset_p3_q__SHIFT 0x0000001c
+#define IA_DEBUG_REG7__shift_vect0_reset_match_p2_q_MASK 0x0000f000L
+#define IA_DEBUG_REG7__shift_vect0_reset_match_p2_q__SHIFT 0x0000000c
+#define IA_DEBUG_REG7__shift_vect1_reset_match_p2_q_MASK 0x000f0000L
+#define IA_DEBUG_REG7__shift_vect1_reset_match_p2_q__SHIFT 0x00000010
+#define IA_DEBUG_REG7__shift_vect1_valid_p2_q_MASK 0x00000f00L
+#define IA_DEBUG_REG7__shift_vect1_valid_p2_q__SHIFT 0x00000008
+#define IA_DEBUG_REG7__shift_vect_valid_p2_q_MASK 0x000000f0L
+#define IA_DEBUG_REG7__shift_vect_valid_p2_q__SHIFT 0x00000004
+#define IA_DEBUG_REG8__di_prim_type_p1_q_MASK 0x0000001fL
+#define IA_DEBUG_REG8__di_prim_type_p1_q__SHIFT 0x00000000
+#define IA_DEBUG_REG8__grp_components_valid_MASK 0xf0000000L
+#define IA_DEBUG_REG8__grp_components_valid__SHIFT 0x0000001c
+#define IA_DEBUG_REG8__grp_continued_MASK 0x00000800L
+#define IA_DEBUG_REG8__grp_continued__SHIFT 0x0000000b
+#define IA_DEBUG_REG8__grp_eopg_MASK 0x04000000L
+#define IA_DEBUG_REG8__grp_eopg__SHIFT 0x0000001a
+#define IA_DEBUG_REG8__grp_eop_MASK 0x02000000L
+#define IA_DEBUG_REG8__grp_eop__SHIFT 0x00000019
+#define IA_DEBUG_REG8__grp_event_flag_MASK 0x08000000L
+#define IA_DEBUG_REG8__grp_event_flag__SHIFT 0x0000001b
+#define IA_DEBUG_REG8__grp_null_primitive_MASK 0x01000000L
+#define IA_DEBUG_REG8__grp_null_primitive__SHIFT 0x00000018
+#define IA_DEBUG_REG8__grp_output_path_MASK 0x00e00000L
+#define IA_DEBUG_REG8__grp_output_path__SHIFT 0x00000015
+#define IA_DEBUG_REG8__grp_state_sel_MASK 0x00007000L
+#define IA_DEBUG_REG8__grp_state_sel__SHIFT 0x0000000c
+#define IA_DEBUG_REG8__grp_sub_prim_type_MASK 0x001f8000L
+#define IA_DEBUG_REG8__grp_sub_prim_type__SHIFT 0x0000000f
+#define IA_DEBUG_REG8__last_group_of_inst_p5_q_MASK 0x00000100L
+#define IA_DEBUG_REG8__last_group_of_inst_p5_q__SHIFT 0x00000008
+#define IA_DEBUG_REG8__shift_prim0_null_flag_p5_q_MASK 0x00000400L
+#define IA_DEBUG_REG8__shift_prim0_null_flag_p5_q__SHIFT 0x0000000a
+#define IA_DEBUG_REG8__shift_prim1_null_flag_p5_q_MASK 0x00000200L
+#define IA_DEBUG_REG8__shift_prim1_null_flag_p5_q__SHIFT 0x00000009
+#define IA_DEBUG_REG8__shift_vect_end_of_packet_p5_q_MASK 0x00000080L
+#define IA_DEBUG_REG8__shift_vect_end_of_packet_p5_q__SHIFT 0x00000007
+#define IA_DEBUG_REG8__two_cycle_xfer_p1_q_MASK 0x00000020L
+#define IA_DEBUG_REG8__two_cycle_xfer_p1_q__SHIFT 0x00000005
+#define IA_DEBUG_REG8__two_prim_input_p1_q_MASK 0x00000040L
+#define IA_DEBUG_REG8__two_prim_input_p1_q__SHIFT 0x00000006
+#define IA_DEBUG_REG9__eopg_between_prims_p6_MASK 0x00000400L
+#define IA_DEBUG_REG9__eopg_between_prims_p6__SHIFT 0x0000000a
+#define IA_DEBUG_REG9__eopg_on_last_prim_p6_MASK 0x00000200L
+#define IA_DEBUG_REG9__eopg_on_last_prim_p6__SHIFT 0x00000009
+#define IA_DEBUG_REG9__gfx_se_switch_p6_MASK 0x00000002L
+#define IA_DEBUG_REG9__gfx_se_switch_p6__SHIFT 0x00000001
+#define IA_DEBUG_REG9__grp_se1_fifo_empty_MASK 0x00040000L
+#define IA_DEBUG_REG9__grp_se1_fifo_empty__SHIFT 0x00000012
+#define IA_DEBUG_REG9__grp_se1_fifo_full_MASK 0x00080000L
+#define IA_DEBUG_REG9__grp_se1_fifo_full__SHIFT 0x00000013
+#define IA_DEBUG_REG9__null_eoi_xfer_prim0_p6_MASK 0x00000008L
+#define IA_DEBUG_REG9__null_eoi_xfer_prim0_p6__SHIFT 0x00000003
+#define IA_DEBUG_REG9__null_eoi_xfer_prim1_p6_MASK 0x00000004L
+#define IA_DEBUG_REG9__null_eoi_xfer_prim1_p6__SHIFT 0x00000002
+#define IA_DEBUG_REG9__prim0_eoi_p6_MASK 0x00000020L
+#define IA_DEBUG_REG9__prim0_eoi_p6__SHIFT 0x00000005
+#define IA_DEBUG_REG9__prim0_valid_eopg_p6_MASK 0x00000080L
+#define IA_DEBUG_REG9__prim0_valid_eopg_p6__SHIFT 0x00000007
+#define IA_DEBUG_REG9__prim1_eoi_p6_MASK 0x00000010L
+#define IA_DEBUG_REG9__prim1_eoi_p6__SHIFT 0x00000004
+#define IA_DEBUG_REG9__prim1_to_other_se_p6_MASK 0x00000100L
+#define IA_DEBUG_REG9__prim1_to_other_se_p6__SHIFT 0x00000008
+#define IA_DEBUG_REG9__prim1_valid_eopg_p6_MASK 0x00000040L
+#define IA_DEBUG_REG9__prim1_valid_eopg_p6__SHIFT 0x00000006
+#define IA_DEBUG_REG9__prim1_xfer_p6_MASK 0x00020000L
+#define IA_DEBUG_REG9__prim1_xfer_p6__SHIFT 0x00000011
+#define IA_DEBUG_REG9__prim_count_eq_group_size_p6_MASK 0x00000800L
+#define IA_DEBUG_REG9__prim_count_eq_group_size_p6__SHIFT 0x0000000b
+#define IA_DEBUG_REG9__prim_counter_q_MASK 0xfffc0000L
+#define IA_DEBUG_REG9__prim_counter_q__SHIFT 0x00000012
+#define IA_DEBUG_REG9__prim_count_gt_group_size_p6_MASK 0x00001000L
+#define IA_DEBUG_REG9__prim_count_gt_group_size_p6__SHIFT 0x0000000c
+#define IA_DEBUG_REG9__send_to_se1_p6_MASK 0x00000001L
+#define IA_DEBUG_REG9__send_to_se1_p6__SHIFT 0x00000000
+#define IA_DEBUG_REG9__shift_vect_end_of_packet_p5_q_MASK 0x00010000L
+#define IA_DEBUG_REG9__shift_vect_end_of_packet_p5_q__SHIFT 0x00000010
+#define IA_DEBUG_REG9__SPARE0_MASK 0x00004000L
+#define IA_DEBUG_REG9__SPARE0__SHIFT 0x0000000e
+#define IA_DEBUG_REG9__SPARE1_MASK 0x00008000L
+#define IA_DEBUG_REG9__SPARE1__SHIFT 0x0000000f
+#define IA_DEBUG_REG9__two_prim_output_p5_q_MASK 0x00002000L
+#define IA_DEBUG_REG9__two_prim_output_p5_q__SHIFT 0x0000000d
+#define IA_ENHANCE__MISC_MASK 0xffffffffL
+#define IA_ENHANCE__MISC__SHIFT 0x00000000
+#define IA_MULTI_VGT_PARAM__PARTIAL_ES_WAVE_ON_MASK 0x00040000L
+#define IA_MULTI_VGT_PARAM__PARTIAL_ES_WAVE_ON__SHIFT 0x00000012
+#define IA_MULTI_VGT_PARAM__PARTIAL_VS_WAVE_ON_MASK 0x00010000L
+#define IA_MULTI_VGT_PARAM__PARTIAL_VS_WAVE_ON__SHIFT 0x00000010
+#define IA_MULTI_VGT_PARAM__PRIMGROUP_SIZE_MASK 0x0000ffffL
+#define IA_MULTI_VGT_PARAM__PRIMGROUP_SIZE__SHIFT 0x00000000
+#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOI_MASK 0x00080000L
+#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOI__SHIFT 0x00000013
+#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOP_MASK 0x00020000L
+#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOP__SHIFT 0x00000011
+#define IA_MULTI_VGT_PARAM__WD_SWITCH_ON_EOP_MASK 0x00100000L
+#define IA_MULTI_VGT_PARAM__WD_SWITCH_ON_EOP__SHIFT 0x00000014
+#define IA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define IA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define IA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define IA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define IA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000L
+#define IA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x0000001c
+#define IA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0x0f000000L
+#define IA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x00000018
+#define IA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define IA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define IA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define IA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define IA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define IA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define IA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define IA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define IA_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000L
+#define IA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define IA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define IA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define IA_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x000000ffL
+#define IA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define IA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define IA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define IA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define IA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define IA_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000L
+#define IA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define IA_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x000000ffL
+#define IA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define IA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define IA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define IA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define IA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define IA_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000L
+#define IA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define IA_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x000000ffL
+#define IA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x00000000
+#define IA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define IA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define IA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define IA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define IA_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000L
+#define IA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define IA_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x000000ffL
+#define IA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x00000000
+#define IA_VMID_OVERRIDE__ENABLE_MASK 0x00000001L
+#define IA_VMID_OVERRIDE__ENABLE__SHIFT 0x00000000
+#define IA_VMID_OVERRIDE__VMID_MASK 0x0000001eL
+#define IA_VMID_OVERRIDE__VMID__SHIFT 0x00000001
+#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA_MASK 0x00040000L
+#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA__SHIFT 0x00000012
+#define PA_CL_CLIP_CNTL__CLIP_DISABLE_MASK 0x00010000L
+#define PA_CL_CLIP_CNTL__CLIP_DISABLE__SHIFT 0x00000010
+#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT_MASK 0x00100000L
+#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT__SHIFT 0x00000014
+#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF_MASK 0x00080000L
+#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF__SHIFT 0x00000013
+#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA_MASK 0x01000000L
+#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA__SHIFT 0x00000018
+#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL_MASK 0x00400000L
+#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL__SHIFT 0x00000016
+#define PA_CL_CLIP_CNTL__PS_UCP_MODE_MASK 0x0000c000L
+#define PA_CL_CLIP_CNTL__PS_UCP_MODE__SHIFT 0x0000000e
+#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG_MASK 0x00002000L
+#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG__SHIFT 0x0000000d
+#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA_MASK 0x00020000L
+#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA__SHIFT 0x00000011
+#define PA_CL_CLIP_CNTL__UCP_ENA_0_MASK 0x00000001L
+#define PA_CL_CLIP_CNTL__UCP_ENA_0__SHIFT 0x00000000
+#define PA_CL_CLIP_CNTL__UCP_ENA_1_MASK 0x00000002L
+#define PA_CL_CLIP_CNTL__UCP_ENA_1__SHIFT 0x00000001
+#define PA_CL_CLIP_CNTL__UCP_ENA_2_MASK 0x00000004L
+#define PA_CL_CLIP_CNTL__UCP_ENA_2__SHIFT 0x00000002
+#define PA_CL_CLIP_CNTL__UCP_ENA_3_MASK 0x00000008L
+#define PA_CL_CLIP_CNTL__UCP_ENA_3__SHIFT 0x00000003
+#define PA_CL_CLIP_CNTL__UCP_ENA_4_MASK 0x00000010L
+#define PA_CL_CLIP_CNTL__UCP_ENA_4__SHIFT 0x00000004
+#define PA_CL_CLIP_CNTL__UCP_ENA_5_MASK 0x00000020L
+#define PA_CL_CLIP_CNTL__UCP_ENA_5__SHIFT 0x00000005
+#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE_MASK 0x02000000L
+#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE__SHIFT 0x00000019
+#define PA_CL_CLIP_CNTL__VTX_KILL_OR_MASK 0x00200000L
+#define PA_CL_CLIP_CNTL__VTX_KILL_OR__SHIFT 0x00000015
+#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE_MASK 0x08000000L
+#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE__SHIFT 0x0000001b
+#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE_MASK 0x04000000L
+#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE__SHIFT 0x0000001a
+#define PA_CL_CNTL_STATUS__CL_BUSY_MASK 0x80000000L
+#define PA_CL_CNTL_STATUS__CL_BUSY__SHIFT 0x0000001f
+#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL_MASK 0x00000008L
+#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL__SHIFT 0x00000003
+#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA_MASK 0x00000001L
+#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA__SHIFT 0x00000000
+#define PA_CL_ENHANCE__ECO_SPARE0_MASK 0x80000000L
+#define PA_CL_ENHANCE__ECO_SPARE0__SHIFT 0x0000001f
+#define PA_CL_ENHANCE__ECO_SPARE1_MASK 0x40000000L
+#define PA_CL_ENHANCE__ECO_SPARE1__SHIFT 0x0000001e
+#define PA_CL_ENHANCE__ECO_SPARE2_MASK 0x20000000L
+#define PA_CL_ENHANCE__ECO_SPARE2__SHIFT 0x0000001d
+#define PA_CL_ENHANCE__ECO_SPARE3_MASK 0x10000000L
+#define PA_CL_ENHANCE__ECO_SPARE3__SHIFT 0x0000001c
+#define PA_CL_ENHANCE__NUM_CLIP_SEQ_MASK 0x00000006L
+#define PA_CL_ENHANCE__NUM_CLIP_SEQ__SHIFT 0x00000001
+#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE_MASK 0x00000010L
+#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE__SHIFT 0x00000004
+#define PA_CL_ENHANCE__XTRA_DEBUG_REG_SEL_MASK 0x00000020L
+#define PA_CL_ENHANCE__XTRA_DEBUG_REG_SEL__SHIFT 0x00000005
+#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD_MASK 0x00004000L
+#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD__SHIFT 0x0000000e
+#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN_MASK 0x00002000L
+#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN__SHIFT 0x0000000d
+#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF_MASK 0x00001000L
+#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF__SHIFT 0x0000000c
+#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN_MASK 0x00000200L
+#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN__SHIFT 0x00000009
+#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF_MASK 0x00000100L
+#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF__SHIFT 0x00000008
+#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN_MASK 0x00000800L
+#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN__SHIFT 0x0000000b
+#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF_MASK 0x00000400L
+#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF__SHIFT 0x0000000a
+#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0_MASK 0x00000008L
+#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0__SHIFT 0x00000003
+#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0_MASK 0x00100000L
+#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0__SHIFT 0x00000014
+#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD_MASK 0x00000004L
+#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD__SHIFT 0x00000002
+#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN_MASK 0x00000040L
+#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN__SHIFT 0x00000006
+#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0_MASK 0x00000080L
+#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0__SHIFT 0x00000007
+#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD_MASK 0x00000001L
+#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD__SHIFT 0x00000000
+#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN_MASK 0x00000010L
+#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN__SHIFT 0x00000004
+#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD_MASK 0x00000002L
+#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD__SHIFT 0x00000001
+#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN_MASK 0x00000020L
+#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN__SHIFT 0x00000005
+#define PA_CL_POINT_CULL_RAD__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_POINT_CULL_RAD__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_POINT_SIZE__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_POINT_SIZE__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_POINT_X_RAD__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_POINT_X_RAD__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_POINT_Y_RAD__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_POINT_Y_RAD__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_0_W__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_0_W__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_0_X__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_0_X__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_0_Y__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_0_Y__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_0_Z__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_0_Z__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_1_W__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_1_W__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_1_X__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_1_X__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_1_Y__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_1_Y__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_1_Z__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_1_Z__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_2_W__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_2_W__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_2_X__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_2_X__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_2_Y__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_2_Y__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_2_Z__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_2_Z__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_3_W__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_3_W__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_3_X__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_3_X__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_3_Y__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_3_Y__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_3_Z__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_3_Z__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_4_W__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_4_W__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_4_X__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_4_X__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_4_Y__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_4_Y__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_4_Z__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_4_Z__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_5_W__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_5_W__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_5_X__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_5_X__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_5_Y__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_5_Y__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_UCP_5_Z__DATA_REGISTER_MASK 0xffffffffL
+#define PA_CL_UCP_5_Z__DATA_REGISTER__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_XSCALE__VPORT_XSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_XSCALE__VPORT_XSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_YSCALE__VPORT_YSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_YSCALE__VPORT_YSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET_MASK 0xffffffffL
+#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE_MASK 0xffffffffL
+#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE__SHIFT 0x00000000
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0_MASK 0x00000001L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0__SHIFT 0x00000000
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1_MASK 0x00000002L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1__SHIFT 0x00000001
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2_MASK 0x00000004L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2__SHIFT 0x00000002
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3_MASK 0x00000008L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3__SHIFT 0x00000003
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4_MASK 0x00000010L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4__SHIFT 0x00000004
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5_MASK 0x00000020L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5__SHIFT 0x00000005
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6_MASK 0x00000040L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6__SHIFT 0x00000006
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7_MASK 0x00000080L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7__SHIFT 0x00000007
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0_MASK 0x00000100L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0__SHIFT 0x00000008
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1_MASK 0x00000200L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1__SHIFT 0x00000009
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2_MASK 0x00000400L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2__SHIFT 0x0000000a
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3_MASK 0x00000800L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3__SHIFT 0x0000000b
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4_MASK 0x00001000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4__SHIFT 0x0000000c
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5_MASK 0x00002000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5__SHIFT 0x0000000d
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6_MASK 0x00004000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6__SHIFT 0x0000000e
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7_MASK 0x00008000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7__SHIFT 0x0000000f
+#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG_MASK 0x00020000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG__SHIFT 0x00000011
+#define PA_CL_VS_OUT_CNTL__USE_VTX_GS_CUT_FLAG_MASK 0x02000000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_GS_CUT_FLAG__SHIFT 0x00000019
+#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG_MASK 0x00100000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG__SHIFT 0x00000014
+#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE_MASK 0x00010000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE__SHIFT 0x00000010
+#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX_MASK 0x00040000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX__SHIFT 0x00000012
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX_MASK 0x00080000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX__SHIFT 0x00000013
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA_MASK 0x00400000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA__SHIFT 0x00000016
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA_MASK 0x00800000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA__SHIFT 0x00000017
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA_MASK 0x01000000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA__SHIFT 0x00000018
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA_MASK 0x00200000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA__SHIFT 0x00000015
+#define PA_CL_VTE_CNTL__PERFCOUNTER_REF_MASK 0x00000800L
+#define PA_CL_VTE_CNTL__PERFCOUNTER_REF__SHIFT 0x0000000b
+#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA_MASK 0x00000002L
+#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA__SHIFT 0x00000001
+#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA_MASK 0x00000001L
+#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA__SHIFT 0x00000000
+#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA_MASK 0x00000008L
+#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA__SHIFT 0x00000003
+#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA_MASK 0x00000004L
+#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA__SHIFT 0x00000002
+#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA_MASK 0x00000020L
+#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA__SHIFT 0x00000005
+#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA_MASK 0x00000010L
+#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA__SHIFT 0x00000004
+#define PA_CL_VTE_CNTL__VTX_W0_FMT_MASK 0x00000400L
+#define PA_CL_VTE_CNTL__VTX_W0_FMT__SHIFT 0x0000000a
+#define PA_CL_VTE_CNTL__VTX_XY_FMT_MASK 0x00000100L
+#define PA_CL_VTE_CNTL__VTX_XY_FMT__SHIFT 0x00000008
+#define PA_CL_VTE_CNTL__VTX_Z_FMT_MASK 0x00000200L
+#define PA_CL_VTE_CNTL__VTX_Z_FMT__SHIFT 0x00000009
+#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN_MASK 0x00000010L
+#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN__SHIFT 0x00000004
+#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE_MASK 0x03000000L
+#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE__SHIFT 0x00000018
+#define PA_SC_AA_CONFIG__MAX_SAMPLE_DIST_MASK 0x0001e000L
+#define PA_SC_AA_CONFIG__MAX_SAMPLE_DIST__SHIFT 0x0000000d
+#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES_MASK 0x00700000L
+#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES__SHIFT 0x00000014
+#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES_MASK 0x00000007L
+#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES__SHIFT 0x00000000
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0_MASK 0x0000ffffL
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0__SHIFT 0x00000000
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0_MASK 0xffff0000L
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0__SHIFT 0x00000010
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1_MASK 0x0000ffffL
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1__SHIFT 0x00000000
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1_MASK 0xffff0000L
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y__SHIFT 0x0000001c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X_MASK 0x0000000fL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X__SHIFT 0x00000000
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y_MASK 0x000000f0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y__SHIFT 0x00000004
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X_MASK 0x00000f00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X__SHIFT 0x00000008
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y_MASK 0x0000f000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y__SHIFT 0x0000000c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X_MASK 0x000f0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X__SHIFT 0x00000010
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y_MASK 0x00f00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y__SHIFT 0x00000014
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X_MASK 0x0f000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X__SHIFT 0x00000018
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y_MASK 0xf0000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y__SHIFT 0x0000001c
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0_MASK 0x0000000fL
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0__SHIFT 0x00000000
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1_MASK 0x000000f0L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1__SHIFT 0x00000004
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2_MASK 0x00000f00L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2__SHIFT 0x00000008
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3_MASK 0x0000f000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3__SHIFT 0x0000000c
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4_MASK 0x000f0000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4__SHIFT 0x00000010
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5_MASK 0x00f00000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5__SHIFT 0x00000014
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6_MASK 0x0f000000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6__SHIFT 0x00000018
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7_MASK 0xf0000000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7__SHIFT 0x0000001c
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10_MASK 0x00000f00L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10__SHIFT 0x00000008
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11_MASK 0x0000f000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11__SHIFT 0x0000000c
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12_MASK 0x000f0000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12__SHIFT 0x00000010
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13_MASK 0x00f00000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13__SHIFT 0x00000014
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14_MASK 0x0f000000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14__SHIFT 0x00000018
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15_MASK 0xf0000000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15__SHIFT 0x0000001c
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8_MASK 0x0000000fL
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8__SHIFT 0x00000000
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9_MASK 0x000000f0L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9__SHIFT 0x00000004
+#define PA_SC_CLIPRECT_0_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_CLIPRECT_0_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_CLIPRECT_0_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_CLIPRECT_0_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_CLIPRECT_0_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_CLIPRECT_0_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_CLIPRECT_0_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_CLIPRECT_0_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_CLIPRECT_1_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_CLIPRECT_1_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_CLIPRECT_1_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_CLIPRECT_1_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_CLIPRECT_1_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_CLIPRECT_1_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_CLIPRECT_1_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_CLIPRECT_1_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_CLIPRECT_2_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_CLIPRECT_2_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_CLIPRECT_2_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_CLIPRECT_2_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_CLIPRECT_2_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_CLIPRECT_2_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_CLIPRECT_2_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_CLIPRECT_2_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_CLIPRECT_3_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_CLIPRECT_3_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_CLIPRECT_3_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_CLIPRECT_3_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_CLIPRECT_3_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_CLIPRECT_3_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_CLIPRECT_3_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_CLIPRECT_3_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_CLIPRECT_RULE__CLIP_RULE_MASK 0x0000ffffL
+#define PA_SC_CLIPRECT_RULE__CLIP_RULE__SHIFT 0x00000000
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_INDX_MASK 0x0000003fL
+#define PA_SC_DEBUG_CNTL__SC_DEBUG_INDX__SHIFT 0x00000000
+#define PA_SC_DEBUG_DATA__DATA_MASK 0xffffffffL
+#define PA_SC_DEBUG_DATA__DATA__SHIFT 0x00000000
+#define PA_SC_DEBUG_REG0__REG0_FIELD0_MASK 0x00000003L
+#define PA_SC_DEBUG_REG0__REG0_FIELD0__SHIFT 0x00000000
+#define PA_SC_DEBUG_REG0__REG0_FIELD1_MASK 0x0000000cL
+#define PA_SC_DEBUG_REG0__REG0_FIELD1__SHIFT 0x00000002
+#define PA_SC_DEBUG_REG1__REG1_FIELD0_MASK 0x00000003L
+#define PA_SC_DEBUG_REG1__REG1_FIELD0__SHIFT 0x00000000
+#define PA_SC_DEBUG_REG1__REG1_FIELD1_MASK 0x0000000cL
+#define PA_SC_DEBUG_REG1__REG1_FIELD1__SHIFT 0x00000002
+#define PA_SC_EDGERULE__ER_LINE_BT_MASK 0xf0000000L
+#define PA_SC_EDGERULE__ER_LINE_BT__SHIFT 0x0000001c
+#define PA_SC_EDGERULE__ER_LINE_LR_MASK 0x0003f000L
+#define PA_SC_EDGERULE__ER_LINE_LR__SHIFT 0x0000000c
+#define PA_SC_EDGERULE__ER_LINE_RL_MASK 0x00fc0000L
+#define PA_SC_EDGERULE__ER_LINE_RL__SHIFT 0x00000012
+#define PA_SC_EDGERULE__ER_LINE_TB_MASK 0x0f000000L
+#define PA_SC_EDGERULE__ER_LINE_TB__SHIFT 0x00000018
+#define PA_SC_EDGERULE__ER_POINT_MASK 0x000000f0L
+#define PA_SC_EDGERULE__ER_POINT__SHIFT 0x00000004
+#define PA_SC_EDGERULE__ER_RECT_MASK 0x00000f00L
+#define PA_SC_EDGERULE__ER_RECT__SHIFT 0x00000008
+#define PA_SC_EDGERULE__ER_TRI_MASK 0x0000000fL
+#define PA_SC_EDGERULE__ER_TRI__SHIFT 0x00000000
+#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX_MASK 0x00000004L
+#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX__SHIFT 0x00000002
+#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION_MASK 0x00000200L
+#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION__SHIFT 0x00000009
+#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS_MASK 0x00004000L
+#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS__SHIFT 0x0000000e
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING_MASK 0x00200000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING__SHIFT 0x00000015
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS_MASK 0x00800000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS__SHIFT 0x00000017
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST_MASK 0x00040000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST__SHIFT 0x00000012
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE_MASK 0x00010000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE__SHIFT 0x00000010
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING_MASK 0x00400000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING__SHIFT 0x00000016
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING_MASK 0x00080000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING__SHIFT 0x00000013
+#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE_MASK 0x00002000L
+#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE__SHIFT 0x0000000d
+#define PA_SC_ENHANCE__DISABLE_PW_BUBBLE_COLLAPSE_MASK 0x000000c0L
+#define PA_SC_ENHANCE__DISABLE_PW_BUBBLE_COLLAPSE__SHIFT 0x00000006
+#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX_MASK 0x00000002L
+#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX__SHIFT 0x00000001
+#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX_MASK 0x00000020L
+#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX__SHIFT 0x00000005
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM_MASK 0x00000400L
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM__SHIFT 0x0000000a
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE_MASK 0x00000800L
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE__SHIFT 0x0000000b
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE_MASK 0x00001000L
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE__SHIFT 0x0000000c
+#define PA_SC_ENHANCE__ECO_SPARE0_MASK 0x80000000L
+#define PA_SC_ENHANCE__ECO_SPARE0__SHIFT 0x0000001f
+#define PA_SC_ENHANCE__ECO_SPARE1_MASK 0x40000000L
+#define PA_SC_ENHANCE__ECO_SPARE1__SHIFT 0x0000001e
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS_MASK 0x00000008L
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS__SHIFT 0x00000003
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID_MASK 0x00000010L
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID__SHIFT 0x00000004
+#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE_MASK 0x00008000L
+#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE__SHIFT 0x0000000f
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID_MASK 0x01000000L
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID__SHIFT 0x00000018
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE_MASK 0x00020000L
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE__SHIFT 0x00000011
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY_MASK 0x00100000L
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY__SHIFT 0x00000014
+#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER_MASK 0x00000001L
+#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER__SHIFT 0x00000000
+#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER_MASK 0x00000100L
+#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER__SHIFT 0x00000008
+#define PA_SC_FIFO_DEPTH_CNTL__DEPTH_MASK 0x000000ffL
+#define PA_SC_FIFO_DEPTH_CNTL__DEPTH__SHIFT 0x00000000
+#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE_MASK 0x00007fc0L
+#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT 0x00000006
+#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE_MASK 0xff800000L
+#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT 0x00000017
+#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE_MASK 0x0000003fL
+#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT 0x00000000
+#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE_MASK 0x001f8000L
+#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT 0x0000000f
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT_MASK 0x0000ffffL
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT__SHIFT 0x00000000
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT_MASK 0xffff0000L
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT__SHIFT 0x00000010
+#define PA_SC_GENERIC_SCISSOR_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_GENERIC_SCISSOR_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_GENERIC_SCISSOR_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_GENERIC_SCISSOR_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_GENERIC_SCISSOR_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_GENERIC_SCISSOR_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_GENERIC_SCISSOR_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_GENERIC_SCISSOR_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_GENERIC_SCISSOR_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_GENERIC_SCISSOR_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE_MASK 0x00fc0000L
+#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE__SHIFT 0x00000012
+#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE_MASK 0x00000fc0L
+#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE__SHIFT 0x00000006
+#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE_MASK 0x0000003fL
+#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE__SHIFT 0x00000000
+#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE_MASK 0x0003f000L
+#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE__SHIFT 0x0000000c
+#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA_MASK 0x00001000L
+#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA__SHIFT 0x0000000c
+#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH_MASK 0x00000200L
+#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH__SHIFT 0x00000009
+#define PA_SC_LINE_CNTL__LAST_PIXEL_MASK 0x00000400L
+#define PA_SC_LINE_CNTL__LAST_PIXEL__SHIFT 0x0000000a
+#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA_MASK 0x00000800L
+#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA__SHIFT 0x0000000b
+#define PA_SC_LINE_STIPPLE__AUTO_RESET_CNTL_MASK 0x60000000L
+#define PA_SC_LINE_STIPPLE__AUTO_RESET_CNTL__SHIFT 0x0000001d
+#define PA_SC_LINE_STIPPLE__LINE_PATTERN_MASK 0x0000ffffL
+#define PA_SC_LINE_STIPPLE__LINE_PATTERN__SHIFT 0x00000000
+#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER_MASK 0x10000000L
+#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER__SHIFT 0x0000001c
+#define PA_SC_LINE_STIPPLE__REPEAT_COUNT_MASK 0x00ff0000L
+#define PA_SC_LINE_STIPPLE__REPEAT_COUNT__SHIFT 0x00000010
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT_MASK 0x0000ff00L
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT__SHIFT 0x00000008
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR_MASK 0x0000000fL
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR__SHIFT 0x00000000
+#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE_MASK 0x00000004L
+#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE__SHIFT 0x00000002
+#define PA_SC_MODE_CNTL_0__MSAA_ENABLE_MASK 0x00000001L
+#define PA_SC_MODE_CNTL_0__MSAA_ENABLE__SHIFT 0x00000000
+#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR_MASK 0x00000008L
+#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR__SHIFT 0x00000003
+#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE_MASK 0x00000002L
+#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE__SHIFT 0x00000001
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE_MASK 0x02000000L
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE__SHIFT 0x00000019
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE_MASK 0x04000000L
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE__SHIFT 0x0000001a
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE_MASK 0x00080000L
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE__SHIFT 0x00000013
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_MASK 0x00f00000L
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE__SHIFT 0x00000014
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK_MASK 0x00008000L
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK__SHIFT 0x0000000f
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z_MASK 0x00004000L
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z__SHIFT 0x0000000e
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE_MASK 0x01000000L
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE__SHIFT 0x00000018
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE_MASK 0x00040000L
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE__SHIFT 0x00000012
+#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE_MASK 0x00020000L
+#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE__SHIFT 0x00000011
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE_MASK 0x08000000L
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE__SHIFT 0x0000001b
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK_MASK 0x70000000L
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK__SHIFT 0x0000001c
+#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE_MASK 0x00010000L
+#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE__SHIFT 0x00000010
+#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE_MASK 0x00000080L
+#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE__SHIFT 0x00000007
+#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE_MASK 0x00000200L
+#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE__SHIFT 0x00000009
+#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR_MASK 0x00000400L
+#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR__SHIFT 0x0000000a
+#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE_MASK 0x00000100L
+#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE__SHIFT 0x00000008
+#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST_MASK 0x00000004L
+#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST__SHIFT 0x00000002
+#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT_MASK 0x00000002L
+#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT__SHIFT 0x00000001
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE_MASK 0x00000008L
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE__SHIFT 0x00000003
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE_MASK 0x00000070L
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE__SHIFT 0x00000004
+#define PA_SC_MODE_CNTL_1__WALK_SIZE_MASK 0x00000001L
+#define PA_SC_MODE_CNTL_1__WALK_SIZE__SHIFT 0x00000000
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT_MASK 0x00000800L
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT__SHIFT 0x0000000b
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET_MASK 0x00001000L
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET__SHIFT 0x0000000c
+#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT_MASK 0x00002000L
+#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT__SHIFT 0x0000000d
+#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x000001ffL
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x000001ffL
+#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x000001ffL
+#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x000001ffL
+#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL_MASK 0x000001ffL
+#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL_MASK 0x000001ffL
+#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL_MASK 0x000001ffL
+#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL_MASK 0x000001ffL
+#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT 0x00000000
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP_MASK 0x00000003L
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP__SHIFT 0x00000000
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL_MASK 0x0000000cL
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL__SHIFT 0x00000002
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL_MASK 0x00000030L
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL__SHIFT 0x00000004
+#define PA_SC_RASTER_CONFIG__PKR_MAP_MASK 0x00000300L
+#define PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT 0x00000008
+#define PA_SC_RASTER_CONFIG__PKR_XSEL2_MASK 0x0000c000L
+#define PA_SC_RASTER_CONFIG__PKR_XSEL2__SHIFT 0x0000000e
+#define PA_SC_RASTER_CONFIG__PKR_XSEL_MASK 0x00000c00L
+#define PA_SC_RASTER_CONFIG__PKR_XSEL__SHIFT 0x0000000a
+#define PA_SC_RASTER_CONFIG__PKR_YSEL_MASK 0x00003000L
+#define PA_SC_RASTER_CONFIG__PKR_YSEL__SHIFT 0x0000000c
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0_MASK 0x00000003L
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0__SHIFT 0x00000000
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1_MASK 0x0000000cL
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1__SHIFT 0x00000002
+#define PA_SC_RASTER_CONFIG__RB_XSEL2_MASK 0x00000030L
+#define PA_SC_RASTER_CONFIG__RB_XSEL2__SHIFT 0x00000004
+#define PA_SC_RASTER_CONFIG__RB_XSEL_MASK 0x00000040L
+#define PA_SC_RASTER_CONFIG__RB_XSEL__SHIFT 0x00000006
+#define PA_SC_RASTER_CONFIG__RB_YSEL_MASK 0x00000080L
+#define PA_SC_RASTER_CONFIG__RB_YSEL__SHIFT 0x00000007
+#define PA_SC_RASTER_CONFIG__SC_MAP_MASK 0x00030000L
+#define PA_SC_RASTER_CONFIG__SC_MAP__SHIFT 0x00000010
+#define PA_SC_RASTER_CONFIG__SC_XSEL_MASK 0x000c0000L
+#define PA_SC_RASTER_CONFIG__SC_XSEL__SHIFT 0x00000012
+#define PA_SC_RASTER_CONFIG__SC_YSEL_MASK 0x00300000L
+#define PA_SC_RASTER_CONFIG__SC_YSEL__SHIFT 0x00000014
+#define PA_SC_RASTER_CONFIG__SE_MAP_MASK 0x03000000L
+#define PA_SC_RASTER_CONFIG__SE_MAP__SHIFT 0x00000018
+#define PA_SC_RASTER_CONFIG__SE_XSEL_MASK 0x0c000000L
+#define PA_SC_RASTER_CONFIG__SE_XSEL__SHIFT 0x0000001a
+#define PA_SC_RASTER_CONFIG__SE_YSEL_MASK 0x30000000L
+#define PA_SC_RASTER_CONFIG__SE_YSEL__SHIFT 0x0000001c
+#define PA_SC_SCREEN_SCISSOR_BR__BR_X_MASK 0x0000ffffL
+#define PA_SC_SCREEN_SCISSOR_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_SCREEN_SCISSOR_BR__BR_Y_MASK 0xffff0000L
+#define PA_SC_SCREEN_SCISSOR_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_SCREEN_SCISSOR_TL__TL_X_MASK 0x0000ffffL
+#define PA_SC_SCREEN_SCISSOR_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_SCREEN_SCISSOR_TL__TL_Y_MASK 0xffff0000L
+#define PA_SC_SCREEN_SCISSOR_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_0_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_0_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_10_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_10_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_11_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_11_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_12_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_12_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_13_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_13_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_14_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_14_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_15_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_15_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_1_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_1_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_2_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_2_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_3_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_3_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_4_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_4_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_5_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_5_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_6_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_6_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_7_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_7_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_8_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_8_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_VPORT_SCISSOR_9_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_VPORT_SCISSOR_9_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN_MASK 0xffffffffL
+#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN__SHIFT 0x00000000
+#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET_MASK 0x0000ffffL
+#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET__SHIFT 0x00000000
+#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET_MASK 0xffff0000L
+#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET__SHIFT 0x00000010
+#define PA_SC_WINDOW_SCISSOR_BR__BR_X_MASK 0x00007fffL
+#define PA_SC_WINDOW_SCISSOR_BR__BR_X__SHIFT 0x00000000
+#define PA_SC_WINDOW_SCISSOR_BR__BR_Y_MASK 0x7fff0000L
+#define PA_SC_WINDOW_SCISSOR_BR__BR_Y__SHIFT 0x00000010
+#define PA_SC_WINDOW_SCISSOR_TL__TL_X_MASK 0x00007fffL
+#define PA_SC_WINDOW_SCISSOR_TL__TL_X__SHIFT 0x00000000
+#define PA_SC_WINDOW_SCISSOR_TL__TL_Y_MASK 0x7fff0000L
+#define PA_SC_WINDOW_SCISSOR_TL__TL_Y__SHIFT 0x00000010
+#define PA_SC_WINDOW_SCISSOR_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000L
+#define PA_SC_WINDOW_SCISSOR_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x0000001f
+#define PA_SU_CNTL_STATUS__SU_BUSY_MASK 0x80000000L
+#define PA_SU_CNTL_STATUS__SU_BUSY__SHIFT 0x0000001f
+#define PA_SU_DEBUG_CNTL__SU_DEBUG_INDX_MASK 0x0000001fL
+#define PA_SU_DEBUG_CNTL__SU_DEBUG_INDX__SHIFT 0x00000000
+#define PA_SU_DEBUG_DATA__DATA_MASK 0xffffffffL
+#define PA_SU_DEBUG_DATA__DATA__SHIFT 0x00000000
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X_MASK 0x000001ffL
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X__SHIFT 0x00000000
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y_MASK 0x01ff0000L
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y__SHIFT 0x00000010
+#define PA_SU_LINE_CNTL__WIDTH_MASK 0x0000ffffL
+#define PA_SU_LINE_CNTL__WIDTH__SHIFT 0x00000000
+#define PA_SU_LINE_STIPPLE_CNTL__DIAMOND_ADJUST_MASK 0x00000010L
+#define PA_SU_LINE_STIPPLE_CNTL__DIAMOND_ADJUST__SHIFT 0x00000004
+#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH_MASK 0x00000004L
+#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH__SHIFT 0x00000002
+#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM_MASK 0x00000008L
+#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM__SHIFT 0x00000003
+#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET_MASK 0x00000003L
+#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET__SHIFT 0x00000000
+#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE_MASK 0xffffffffL
+#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE__SHIFT 0x00000000
+#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE_MASK 0x00ffffffL
+#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0x0000ffffL
+#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x000000ffL
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0x0000ffffL
+#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x000000ffL
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0x0000ffffL
+#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x000000ffL
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0x0000ffffL
+#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x000000ffL
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x00000000
+#define PA_SU_POINT_MINMAX__MAX_SIZE_MASK 0xffff0000L
+#define PA_SU_POINT_MINMAX__MAX_SIZE__SHIFT 0x00000010
+#define PA_SU_POINT_MINMAX__MIN_SIZE_MASK 0x0000ffffL
+#define PA_SU_POINT_MINMAX__MIN_SIZE__SHIFT 0x00000000
+#define PA_SU_POINT_SIZE__HEIGHT_MASK 0x0000ffffL
+#define PA_SU_POINT_SIZE__HEIGHT__SHIFT 0x00000000
+#define PA_SU_POINT_SIZE__WIDTH_MASK 0xffff0000L
+#define PA_SU_POINT_SIZE__WIDTH__SHIFT 0x00000010
+#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET_MASK 0xffffffffL
+#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET__SHIFT 0x00000000
+#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE_MASK 0xffffffffL
+#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE__SHIFT 0x00000000
+#define PA_SU_POLY_OFFSET_CLAMP__CLAMP_MASK 0xffffffffL
+#define PA_SU_POLY_OFFSET_CLAMP__CLAMP__SHIFT 0x00000000
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT_MASK 0x00000100L
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT__SHIFT 0x00000008
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS_MASK 0x000000ffL
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS__SHIFT 0x00000000
+#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET_MASK 0xffffffffL
+#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET__SHIFT 0x00000000
+#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE_MASK 0xffffffffL
+#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE__SHIFT 0x00000000
+#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA_MASK 0x00000020L
+#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA__SHIFT 0x00000005
+#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE_MASK 0x00000002L
+#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE__SHIFT 0x00000001
+#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA_MASK 0x00000040L
+#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA__SHIFT 0x00000006
+#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE_MASK 0x00000004L
+#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE__SHIFT 0x00000002
+#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT_MASK 0x0000ff00L
+#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT__SHIFT 0x00000008
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA_MASK 0x00000080L
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA__SHIFT 0x00000007
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE_MASK 0x00000008L
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE__SHIFT 0x00000003
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA_MASK 0x00000010L
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA__SHIFT 0x00000004
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE_MASK 0x00000001L
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE__SHIFT 0x00000000
+#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION_MASK 0x40000000L
+#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION__SHIFT 0x0000001e
+#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION_MASK 0x80000000L
+#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION__SHIFT 0x0000001f
+#define PA_SU_SC_MODE_CNTL__CULL_BACK_MASK 0x00000002L
+#define PA_SU_SC_MODE_CNTL__CULL_BACK__SHIFT 0x00000001
+#define PA_SU_SC_MODE_CNTL__CULL_FRONT_MASK 0x00000001L
+#define PA_SU_SC_MODE_CNTL__CULL_FRONT__SHIFT 0x00000000
+#define PA_SU_SC_MODE_CNTL__FACE_MASK 0x00000004L
+#define PA_SU_SC_MODE_CNTL__FACE__SHIFT 0x00000002
+#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA_MASK 0x00200000L
+#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA__SHIFT 0x00000015
+#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS_MASK 0x00100000L
+#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS__SHIFT 0x00000014
+#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE_MASK 0x00000700L
+#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE__SHIFT 0x00000008
+#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE_MASK 0x000000e0L
+#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE__SHIFT 0x00000005
+#define PA_SU_SC_MODE_CNTL__POLY_MODE_MASK 0x00000018L
+#define PA_SU_SC_MODE_CNTL__POLY_MODE__SHIFT 0x00000003
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE_MASK 0x00001000L
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE__SHIFT 0x0000000c
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE_MASK 0x00000800L
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE__SHIFT 0x0000000b
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE_MASK 0x00002000L
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE__SHIFT 0x0000000d
+#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST_MASK 0x00080000L
+#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST__SHIFT 0x00000013
+#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE_MASK 0x00010000L
+#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE__SHIFT 0x00000010
+#define PA_SU_VTX_CNTL__PIX_CENTER_MASK 0x00000001L
+#define PA_SU_VTX_CNTL__PIX_CENTER__SHIFT 0x00000000
+#define PA_SU_VTX_CNTL__QUANT_MODE_MASK 0x00000038L
+#define PA_SU_VTX_CNTL__QUANT_MODE__SHIFT 0x00000003
+#define PA_SU_VTX_CNTL__ROUND_MODE_MASK 0x00000006L
+#define PA_SU_VTX_CNTL__ROUND_MODE__SHIFT 0x00000001
+#define RAS_BCI_SIGNATURE0__SIGNATURE_MASK 0xffffffffL
+#define RAS_BCI_SIGNATURE0__SIGNATURE__SHIFT 0x00000000
+#define RAS_BCI_SIGNATURE1__SIGNATURE_MASK 0xffffffffL
+#define RAS_BCI_SIGNATURE1__SIGNATURE__SHIFT 0x00000000
+#define RAS_CB_SIGNATURE0__SIGNATURE_MASK 0xffffffffL
+#define RAS_CB_SIGNATURE0__SIGNATURE__SHIFT 0x00000000
+#define RAS_DB_SIGNATURE0__SIGNATURE_MASK 0xffffffffL
+#define RAS_DB_SIGNATURE0__SIGNATURE__SHIFT 0x00000000
+#define RAS_IA_SIGNATURE0__SIGNATURE_MASK 0xffffffffL
+#define RAS_IA_SIGNATURE0__SIGNATURE__SHIFT 0x00000000
+#define RAS_IA_SIGNATURE1__SIGNATURE_MASK 0xffffffffL
+#define RAS_IA_SIGNATURE1__SIGNATURE__SHIFT 0x00000000
+#define RAS_PA_SIGNATURE0__SIGNATURE_MASK 0xffffffffL
+#define RAS_PA_SIGNATURE0__SIGNATURE__SHIFT 0x00000000
+#define RAS_SC_SIGNATURE0__SIGNATURE_MASK 0xffffffffL
+#define RAS_SC_SIGNATURE0__SIGNATURE__SHIFT 0x00000000
+#define RAS_SC_SIGNATURE1__SIGNATURE_MASK 0xffffffffL
+#define RAS_SC_SIGNATURE1__SIGNATURE__SHIFT 0x00000000
+#define RAS_SC_SIGNATURE2__SIGNATURE_MASK 0xffffffffL
+#define RAS_SC_SIGNATURE2__SIGNATURE__SHIFT 0x00000000
+#define RAS_SC_SIGNATURE3__SIGNATURE_MASK 0xffffffffL
+#define RAS_SC_SIGNATURE3__SIGNATURE__SHIFT 0x00000000
+#define RAS_SC_SIGNATURE4__SIGNATURE_MASK 0xffffffffL
+#define RAS_SC_SIGNATURE4__SIGNATURE__SHIFT 0x00000000
+#define RAS_SC_SIGNATURE5__SIGNATURE_MASK 0xffffffffL
+#define RAS_SC_SIGNATURE5__SIGNATURE__SHIFT 0x00000000
+#define RAS_SC_SIGNATURE6__SIGNATURE_MASK 0xffffffffL
+#define RAS_SC_SIGNATURE6__SIGNATURE__SHIFT 0x00000000
+#define RAS_SC_SIGNATURE7__SIGNATURE_MASK 0xffffffffL
+#define RAS_SC_SIGNATURE7__SIGNATURE__SHIFT 0x00000000
+#define RAS_SIGNATURE_CONTROL__ENABLE_MASK 0x00000001L
+#define RAS_SIGNATURE_CONTROL__ENABLE__SHIFT 0x00000000
+#define RAS_SIGNATURE_MASK__INPUT_BUS_MASK_MASK 0xffffffffL
+#define RAS_SIGNATURE_MASK__INPUT_BUS_MASK__SHIFT 0x00000000
+#define RAS_SPI_SIGNATURE0__SIGNATURE_MASK 0xffffffffL
+#define RAS_SPI_SIGNATURE0__SIGNATURE__SHIFT 0x00000000
+#define RAS_SPI_SIGNATURE1__SIGNATURE_MASK 0xffffffffL
+#define RAS_SPI_SIGNATURE1__SIGNATURE__SHIFT 0x00000000
+#define RAS_SQ_SIGNATURE0__SIGNATURE_MASK 0xffffffffL
+#define RAS_SQ_SIGNATURE0__SIGNATURE__SHIFT 0x00000000
+#define RAS_SX_SIGNATURE0__SIGNATURE_MASK 0xffffffffL
+#define RAS_SX_SIGNATURE0__SIGNATURE__SHIFT 0x00000000
+#define RAS_SX_SIGNATURE1__SIGNATURE_MASK 0xffffffffL
+#define RAS_SX_SIGNATURE1__SIGNATURE__SHIFT 0x00000000
+#define RAS_SX_SIGNATURE2__SIGNATURE_MASK 0xffffffffL
+#define RAS_SX_SIGNATURE2__SIGNATURE__SHIFT 0x00000000
+#define RAS_SX_SIGNATURE3__SIGNATURE_MASK 0xffffffffL
+#define RAS_SX_SIGNATURE3__SIGNATURE__SHIFT 0x00000000
+#define RAS_TA_SIGNATURE0__SIGNATURE_MASK 0xffffffffL
+#define RAS_TA_SIGNATURE0__SIGNATURE__SHIFT 0x00000000
+#define RAS_TD_SIGNATURE0__SIGNATURE_MASK 0xffffffffL
+#define RAS_TD_SIGNATURE0__SIGNATURE__SHIFT 0x00000000
+#define RAS_VGT_SIGNATURE0__SIGNATURE_MASK 0xffffffffL
+#define RAS_VGT_SIGNATURE0__SIGNATURE__SHIFT 0x00000000
+#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN_MASK 0x00000002L
+#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN__SHIFT 0x00000001
+#define RLC_AUTO_PG_CTRL__AUTO_PG_EN_MASK 0x00000001L
+#define RLC_AUTO_PG_CTRL__AUTO_PG_EN__SHIFT 0x00000000
+#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN_MASK 0x00000004L
+#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN__SHIFT 0x00000002
+#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK 0x0007fff8L
+#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT 0x00000003
+#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD_MASK 0xfff80000L
+#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD__SHIFT 0x00000013
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE_MASK 0x00000001L
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE__SHIFT 0x00000000
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED_MASK 0xfffffffeL
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED__SHIFT 0x00000001
+#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER_MASK 0x08000000L
+#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER__SHIFT 0x0000001b
+#define RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK 0x00000001L
+#define RLC_CGCG_CGLS_CTRL__CGCG_EN__SHIFT 0x00000000
+#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK 0x0000ff00L
+#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT 0x00000008
+#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL_MASK 0x10000000L
+#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL__SHIFT 0x0000001c
+#define RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK 0x00000002L
+#define RLC_CGCG_CGLS_CTRL__CGLS_EN__SHIFT 0x00000001
+#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK 0x000000fcL
+#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT 0x00000002
+#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE_MASK 0x60000000L
+#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE__SHIFT 0x0000001d
+#define RLC_CGCG_CGLS_CTRL__SPARE_MASK 0x80000000L
+#define RLC_CGCG_CGLS_CTRL__SPARE__SHIFT 0x0000001f
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT_MASK 0x0000000fL
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT__SHIFT 0x00000000
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT_MASK 0x000000f0L
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT__SHIFT 0x00000004
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT_MASK 0x0fff0000L
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT__SHIFT 0x00000010
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT_MASK 0xf0000000L
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT__SHIFT 0x0000001c
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT_MASK 0x00000f00L
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT__SHIFT 0x00000008
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT_MASK 0x0000f000L
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT__SHIFT 0x0000000c
+#define RLC_CGTT_MGCG_OVERRIDE__OVERRIDE_MASK 0xffffffffL
+#define RLC_CGTT_MGCG_OVERRIDE__OVERRIDE__SHIFT 0x00000000
+#define RLC_CNTL__FORCE_RETRY_MASK 0x00000002L
+#define RLC_CNTL__FORCE_RETRY__SHIFT 0x00000001
+#define RLC_CNTL__READ_CACHE_DISABLE_MASK 0x00000004L
+#define RLC_CNTL__READ_CACHE_DISABLE__SHIFT 0x00000002
+#define RLC_CNTL__RESERVED_MASK 0xffffff00L
+#define RLC_CNTL__RESERVED__SHIFT 0x00000008
+#define RLC_CNTL__RLC_ENABLE_F32_MASK 0x00000001L
+#define RLC_CNTL__RLC_ENABLE_F32__SHIFT 0x00000000
+#define RLC_CNTL__RLC_STEP_F32_MASK 0x00000008L
+#define RLC_CNTL__RLC_STEP_F32__SHIFT 0x00000003
+#define RLC_CNTL__SOFT_RESET_DEBUG_MODE_MASK 0x00000010L
+#define RLC_CNTL__SOFT_RESET_DEBUG_MODE__SHIFT 0x00000004
+#define RLC_CU_STATUS__WORK_PENDING_MASK 0xffffffffL
+#define RLC_CU_STATUS__WORK_PENDING__SHIFT 0x00000000
+#define RLC_DEBUG__DATA_MASK 0xffffffffL
+#define RLC_DEBUG__DATA__SHIFT 0x00000000
+#define RLC_DEBUG_SELECT__RESERVED_MASK 0xffff8000L
+#define RLC_DEBUG_SELECT__RESERVED__SHIFT 0x0000000f
+#define RLC_DEBUG_SELECT__SELECT_MASK 0x000000ffL
+#define RLC_DEBUG_SELECT__SELECT__SHIFT 0x00000000
+#define RLC_DRIVER_CPDMA_STATUS__DRIVER_ACK_MASK 0x00000010L
+#define RLC_DRIVER_CPDMA_STATUS__DRIVER_ACK__SHIFT 0x00000004
+#define RLC_DRIVER_CPDMA_STATUS__DRIVER_REQUEST_MASK 0x00000001L
+#define RLC_DRIVER_CPDMA_STATUS__DRIVER_REQUEST__SHIFT 0x00000000
+#define RLC_DRIVER_CPDMA_STATUS__RESERVED1_MASK 0x0000000eL
+#define RLC_DRIVER_CPDMA_STATUS__RESERVED1__SHIFT 0x00000001
+#define RLC_DRIVER_CPDMA_STATUS__RESERVED_MASK 0xffffffe0L
+#define RLC_DRIVER_CPDMA_STATUS__RESERVED__SHIFT 0x00000005
+#define RLC_DYN_PG_REQUEST__PG_REQUEST_CU_MASK_MASK 0xffffffffL
+#define RLC_DYN_PG_REQUEST__PG_REQUEST_CU_MASK__SHIFT 0x00000000
+#define RLC_DYN_PG_STATUS__PG_STATUS_CU_MASK_MASK 0xffffffffL
+#define RLC_DYN_PG_STATUS__PG_STATUS_CU_MASK__SHIFT 0x00000000
+#define RLC_GPM_SCRATCH_ADDR__ADDR_MASK 0x000001ffL
+#define RLC_GPM_SCRATCH_ADDR__ADDR__SHIFT 0x00000000
+#define RLC_GPM_SCRATCH_ADDR__RESERVED_MASK 0xfffffe00L
+#define RLC_GPM_SCRATCH_ADDR__RESERVED__SHIFT 0x00000009
+#define RLC_GPU_CLOCK_32__GPU_CLOCK_32_MASK 0xffffffffL
+#define RLC_GPU_CLOCK_32__GPU_CLOCK_32__SHIFT 0x00000000
+#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED_MASK 0xffffffc0L
+#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED__SHIFT 0x00000006
+#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL_MASK 0x0000003fL
+#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL__SHIFT 0x00000000
+#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB_MASK 0xffffffffL
+#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB__SHIFT 0x00000000
+#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB_MASK 0xffffffffL
+#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB__SHIFT 0x00000000
+#define RLC_LB_ALWAYS_ACTIVE_CU_MASK__ALWAYS_ACTIVE_CU_MASK_MASK 0xffffffffL
+#define RLC_LB_ALWAYS_ACTIVE_CU_MASK__ALWAYS_ACTIVE_CU_MASK__SHIFT 0x00000000
+#define RLC_LB_CNTL__CU_MASK_USED_OFF_HYST_MASK 0x00000ff0L
+#define RLC_LB_CNTL__CU_MASK_USED_OFF_HYST__SHIFT 0x00000004
+#define RLC_LB_CNTL__LB_CNT_CP_BUSY_MASK 0x00000002L
+#define RLC_LB_CNTL__LB_CNT_CP_BUSY__SHIFT 0x00000001
+#define RLC_LB_CNTL__LB_CNT_REG_INC_MASK 0x00000008L
+#define RLC_LB_CNTL__LB_CNT_REG_INC__SHIFT 0x00000003
+#define RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK 0x00000004L
+#define RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE__SHIFT 0x00000002
+#define RLC_LB_CNTL__LOAD_BALANCE_ENABLE_MASK 0x00000001L
+#define RLC_LB_CNTL__LOAD_BALANCE_ENABLE__SHIFT 0x00000000
+#define RLC_LB_CNTL__RESERVED_MASK 0xfffffff0L
+#define RLC_LB_CNTL__RESERVED__SHIFT 0x00000004
+#define RLC_LB_CNTR_INIT__LB_CNTR_INIT_MASK 0xffffffffL
+#define RLC_LB_CNTR_INIT__LB_CNTR_INIT__SHIFT 0x00000000
+#define RLC_LB_CNTR_MAX__LB_CNTR_MAX_MASK 0xffffffffL
+#define RLC_LB_CNTR_MAX__LB_CNTR_MAX__SHIFT 0x00000000
+#define RLC_LB_INIT_CU_MASK__INIT_CU_MASK_MASK 0xffffffffL
+#define RLC_LB_INIT_CU_MASK__INIT_CU_MASK__SHIFT 0x00000000
+#define RLC_LB_PARAMS__FIFO_SAMPLES_MASK 0x000000feL
+#define RLC_LB_PARAMS__FIFO_SAMPLES__SHIFT 0x00000001
+#define RLC_LB_PARAMS__PG_IDLE_SAMPLE_INTERVAL_MASK 0xffff0000L
+#define RLC_LB_PARAMS__PG_IDLE_SAMPLE_INTERVAL__SHIFT 0x00000010
+#define RLC_LB_PARAMS__PG_IDLE_SAMPLES_MASK 0x0000ff00L
+#define RLC_LB_PARAMS__PG_IDLE_SAMPLES__SHIFT 0x00000008
+#define RLC_LB_PARAMS__SKIP_L2_CHECK_MASK 0x00000001L
+#define RLC_LB_PARAMS__SKIP_L2_CHECK__SHIFT 0x00000000
+#define RLC_LOAD_BALANCE_CNTR__RLC_LOAD_BALANCE_CNTR_MASK 0xffffffffL
+#define RLC_LOAD_BALANCE_CNTR__RLC_LOAD_BALANCE_CNTR__SHIFT 0x00000000
+#define RLC_MAX_PG_CU__MAX_POWERED_UP_CU_MASK 0x000000ffL
+#define RLC_MAX_PG_CU__MAX_POWERED_UP_CU__SHIFT 0x00000000
+#define RLC_MAX_PG_CU__SPARE_MASK 0xffffff00L
+#define RLC_MAX_PG_CU__SPARE__SHIFT 0x00000008
+#define RLC_MC_CNTL__RDNFO_STALL_MASK 0x10000000L
+#define RLC_MC_CNTL__RDNFO_STALL__SHIFT 0x0000001c
+#define RLC_MC_CNTL__RDNFO_URG_MASK 0x00f00000L
+#define RLC_MC_CNTL__RDNFO_URG__SHIFT 0x00000014
+#define RLC_MC_CNTL__RDREQ_PRIV_MASK 0x08000000L
+#define RLC_MC_CNTL__RDREQ_PRIV__SHIFT 0x0000001b
+#define RLC_MC_CNTL__RDREQ_SWAP_MASK 0x03000000L
+#define RLC_MC_CNTL__RDREQ_SWAP__SHIFT 0x00000018
+#define RLC_MC_CNTL__RDREQ_TRAN_MASK 0x04000000L
+#define RLC_MC_CNTL__RDREQ_TRAN__SHIFT 0x0000001a
+#define RLC_MC_CNTL__RESERVED_B_MASK 0x000fe000L
+#define RLC_MC_CNTL__RESERVED_B__SHIFT 0x0000000d
+#define RLC_MC_CNTL__RESERVED_MASK 0xe0000000L
+#define RLC_MC_CNTL__RESERVED__SHIFT 0x0000001d
+#define RLC_MC_CNTL__WRNFO_STALL_MASK 0x00000010L
+#define RLC_MC_CNTL__WRNFO_STALL__SHIFT 0x00000004
+#define RLC_MC_CNTL__WRNFO_URG_MASK 0x000001e0L
+#define RLC_MC_CNTL__WRNFO_URG__SHIFT 0x00000005
+#define RLC_MC_CNTL__WRREQ_DW_IMASK_MASK 0x00001e00L
+#define RLC_MC_CNTL__WRREQ_DW_IMASK__SHIFT 0x00000009
+#define RLC_MC_CNTL__WRREQ_PRIV_MASK 0x00000008L
+#define RLC_MC_CNTL__WRREQ_PRIV__SHIFT 0x00000003
+#define RLC_MC_CNTL__WRREQ_SWAP_MASK 0x00000003L
+#define RLC_MC_CNTL__WRREQ_SWAP__SHIFT 0x00000000
+#define RLC_MC_CNTL__WRREQ_TRAN_MASK 0x00000004L
+#define RLC_MC_CNTL__WRREQ_TRAN__SHIFT 0x00000002
+#define RLC_MEM_SLP_CNTL__RESERVED1_MASK 0xff000000L
+#define RLC_MEM_SLP_CNTL__RESERVED1__SHIFT 0x00000018
+#define RLC_MEM_SLP_CNTL__RESERVED_MASK 0x000000fcL
+#define RLC_MEM_SLP_CNTL__RESERVED__SHIFT 0x00000002
+#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN_MASK 0x00000002L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN__SHIFT 0x00000001
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK 0x00000001L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN__SHIFT 0x00000000
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY_MASK 0x00ff0000L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY__SHIFT 0x00000010
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY_MASK 0x0000ff00L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY__SHIFT 0x00000008
+#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK 0x000000ffL
+#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT 0x00000000
+#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK 0x000000ffL
+#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT 0x00000000
+#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK 0x00000400L
+#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT 0x0000000a
+#define RLC_PERFMON_CNTL__PERFMON_STATE_MASK 0x00000007L
+#define RLC_PERFMON_CNTL__PERFMON_STATE__SHIFT 0x00000000
+#define RLC_PG_ALWAYS_ON_CU_MASK__AON_CU_MASK_MASK 0xffffffffL
+#define RLC_PG_ALWAYS_ON_CU_MASK__AON_CU_MASK__SHIFT 0x00000000
+#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE_MASK 0x00010000L
+#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE__SHIFT 0x00000010
+#define RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK 0x00000004L
+#define RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE__SHIFT 0x00000002
+#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK 0x00000001L
+#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE__SHIFT 0x00000000
+#define RLC_PG_CNTL__GFX_POWER_GATING_SRC_MASK 0x00000002L
+#define RLC_PG_CNTL__GFX_POWER_GATING_SRC__SHIFT 0x00000001
+#define RLC_PG_CNTL__PG_ERROR_STATUS_MASK 0xff000000L
+#define RLC_PG_CNTL__PG_ERROR_STATUS__SHIFT 0x00000018
+#define RLC_PG_CNTL__RESERVED1_MASK 0x00f80000L
+#define RLC_PG_CNTL__RESERVED1__SHIFT 0x00000013
+#define RLC_PG_CNTL__RESERVED_MASK 0xfffffff0L
+#define RLC_PG_CNTL__RESERVED__SHIFT 0x00000004
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK 0x00040000L
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE__SHIFT 0x00000012
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK 0x00020000L
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE__SHIFT 0x00000011
+#define RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK 0x00000008L
+#define RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE__SHIFT 0x00000003
+#define RLC_SAVE_AND_RESTORE_BASE__BASE_MASK 0xffffffffL
+#define RLC_SAVE_AND_RESTORE_BASE__BASE__SHIFT 0x00000000
+#define RLC_SERDES_RD_DATA_0__DATA_MASK 0xffffffffL
+#define RLC_SERDES_RD_DATA_0__DATA__SHIFT 0x00000000
+#define RLC_SERDES_RD_DATA_1__DATA_MASK 0xffffffffL
+#define RLC_SERDES_RD_DATA_1__DATA__SHIFT 0x00000000
+#define RLC_SERDES_RD_DATA_2__DATA_MASK 0xffffffffL
+#define RLC_SERDES_RD_DATA_2__DATA__SHIFT 0x00000000
+#define RLC_SERDES_RD_MASTER_INDEX__CU_ID_MASK 0x0000000fL
+#define RLC_SERDES_RD_MASTER_INDEX__CU_ID__SHIFT 0x00000000
+#define RLC_SERDES_RD_MASTER_INDEX__DATA_REG_ID_MASK 0x0000c000L
+#define RLC_SERDES_RD_MASTER_INDEX__DATA_REG_ID__SHIFT 0x0000000e
+#define RLC_SERDES_RD_MASTER_INDEX__NON_SE_MASK 0x00003800L
+#define RLC_SERDES_RD_MASTER_INDEX__NON_SE__SHIFT 0x0000000b
+#define RLC_SERDES_RD_MASTER_INDEX__SE_ID_MASK 0x000001c0L
+#define RLC_SERDES_RD_MASTER_INDEX__SE_ID__SHIFT 0x00000006
+#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_ID_MASK 0x00000200L
+#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_ID__SHIFT 0x00000009
+#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_MASK 0x00000400L
+#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU__SHIFT 0x0000000a
+#define RLC_SERDES_RD_MASTER_INDEX__SH_ID_MASK 0x00000030L
+#define RLC_SERDES_RD_MASTER_INDEX__SH_ID__SHIFT 0x00000004
+#define RLC_SERDES_RD_MASTER_INDEX__SPARE_MASK 0xffffc000L
+#define RLC_SERDES_RD_MASTER_INDEX__SPARE__SHIFT 0x0000000e
+#define RLC_SERDES_WR_CTRL__BPM_ADDR_MASK 0x000000ffL
+#define RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT 0x00000000
+#define RLC_SERDES_WR_CTRL__CGCG_OVERRIDE_0_MASK 0x00100000L
+#define RLC_SERDES_WR_CTRL__CGCG_OVERRIDE_0__SHIFT 0x00000014
+#define RLC_SERDES_WR_CTRL__CGCG_OVERRIDE_1_MASK 0x00200000L
+#define RLC_SERDES_WR_CTRL__CGCG_OVERRIDE_1__SHIFT 0x00000015
+#define RLC_SERDES_WR_CTRL__CGLS_DISABLE_MASK 0x00020000L
+#define RLC_SERDES_WR_CTRL__CGLS_DISABLE__SHIFT 0x00000011
+#define RLC_SERDES_WR_CTRL__CGLS_ENABLE_MASK 0x00010000L
+#define RLC_SERDES_WR_CTRL__CGLS_ENABLE__SHIFT 0x00000010
+#define RLC_SERDES_WR_CTRL__CGLS_OFF_MASK 0x00080000L
+#define RLC_SERDES_WR_CTRL__CGLS_OFF__SHIFT 0x00000013
+#define RLC_SERDES_WR_CTRL__CGLS_ON_MASK 0x00040000L
+#define RLC_SERDES_WR_CTRL__CGLS_ON__SHIFT 0x00000012
+#define RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_0_MASK 0x00400000L
+#define RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_0__SHIFT 0x00000016
+#define RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_1_MASK 0x00800000L
+#define RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_1__SHIFT 0x00000017
+#define RLC_SERDES_WR_CTRL__P1_SELECT_MASK 0x00000400L
+#define RLC_SERDES_WR_CTRL__P1_SELECT__SHIFT 0x0000000a
+#define RLC_SERDES_WR_CTRL__P2_SELECT_MASK 0x00000800L
+#define RLC_SERDES_WR_CTRL__P2_SELECT__SHIFT 0x0000000b
+#define RLC_SERDES_WR_CTRL__POWER_DOWN_MASK 0x00000100L
+#define RLC_SERDES_WR_CTRL__POWER_DOWN__SHIFT 0x00000008
+#define RLC_SERDES_WR_CTRL__POWER_UP_MASK 0x00000200L
+#define RLC_SERDES_WR_CTRL__POWER_UP__SHIFT 0x00000009
+#define RLC_SERDES_WR_CTRL__READ_COMMAND_MASK 0x00002000L
+#define RLC_SERDES_WR_CTRL__READ_COMMAND__SHIFT 0x0000000d
+#define RLC_SERDES_WR_CTRL__REG_ADDR_MASK 0xf0000000L
+#define RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT 0x0000001c
+#define RLC_SERDES_WR_CTRL__RESERVED_1_MASK 0x0000c000L
+#define RLC_SERDES_WR_CTRL__RESERVED_1__SHIFT 0x0000000e
+#define RLC_SERDES_WR_CTRL__RESERVED_2_MASK 0x0f000000L
+#define RLC_SERDES_WR_CTRL__RESERVED_2__SHIFT 0x00000018
+#define RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK 0x00001000L
+#define RLC_SERDES_WR_CTRL__WRITE_COMMAND__SHIFT 0x0000000c
+#define RLC_SERDES_WR_DATA__DATA_MASK 0xffffffffL
+#define RLC_SERDES_WR_DATA__DATA__SHIFT 0x00000000
+#define RLC_SMU_GRBM_REG_SAVE_CTRL__SPARE_MASK 0xfffffffeL
+#define RLC_SMU_GRBM_REG_SAVE_CTRL__SPARE__SHIFT 0x00000001
+#define RLC_SMU_GRBM_REG_SAVE_CTRL__START_GRBM_REG_SAVE_MASK 0x00000001L
+#define RLC_SMU_GRBM_REG_SAVE_CTRL__START_GRBM_REG_SAVE__SHIFT 0x00000000
+#define RLC_SMU_PG_CTRL__SPARE_MASK 0xfffffffeL
+#define RLC_SMU_PG_CTRL__SPARE__SHIFT 0x00000001
+#define RLC_SMU_PG_CTRL__START_PG_MASK 0x00000001L
+#define RLC_SMU_PG_CTRL__START_PG__SHIFT 0x00000000
+#define RLC_SMU_PG_WAKE_UP_CTRL__SPARE_MASK 0xfffffffeL
+#define RLC_SMU_PG_WAKE_UP_CTRL__SPARE__SHIFT 0x00000001
+#define RLC_SMU_PG_WAKE_UP_CTRL__START_PG_WAKE_UP_MASK 0x00000001L
+#define RLC_SMU_PG_WAKE_UP_CTRL__START_PG_WAKE_UP__SHIFT 0x00000000
+#define RLC_SOFT_RESET_GPU__RESERVED_MASK 0xfffffffeL
+#define RLC_SOFT_RESET_GPU__RESERVED__SHIFT 0x00000001
+#define RLC_SOFT_RESET_GPU__SOFT_RESET_GPU_MASK 0x00000001L
+#define RLC_SOFT_RESET_GPU__SOFT_RESET_GPU__SHIFT 0x00000000
+#define RLC_STATIC_PG_STATUS__PG_STATUS_CU_MASK_MASK 0xffffffffL
+#define RLC_STATIC_PG_STATUS__PG_STATUS_CU_MASK__SHIFT 0x00000000
+#define RLC_STAT__RESERVED_MASK 0xfffffff0L
+#define RLC_STAT__RESERVED__SHIFT 0x00000004
+#define RLC_STAT__RLC_BUSY_MASK 0x00000001L
+#define RLC_STAT__RLC_BUSY__SHIFT 0x00000000
+#define RLC_STAT__RLC_GPM_BUSY_MASK 0x00000002L
+#define RLC_STAT__RLC_GPM_BUSY__SHIFT 0x00000001
+#define RLC_STAT__RLC_SPM_BUSY_MASK 0x00000004L
+#define RLC_STAT__RLC_SPM_BUSY__SHIFT 0x00000002
+#define RLC_THREAD1_DELAY__CU_IDEL_DELAY_MASK 0x000000ffL
+#define RLC_THREAD1_DELAY__CU_IDEL_DELAY__SHIFT 0x00000000
+#define RLC_THREAD1_DELAY__LBPW_INNER_LOOP_DELAY_MASK 0x0000ff00L
+#define RLC_THREAD1_DELAY__LBPW_INNER_LOOP_DELAY__SHIFT 0x00000008
+#define RLC_THREAD1_DELAY__LBPW_OUTER_LOOP_DELAY_MASK 0x00ff0000L
+#define RLC_THREAD1_DELAY__LBPW_OUTER_LOOP_DELAY__SHIFT 0x00000010
+#define RLC_THREAD1_DELAY__SPARE_MASK 0xff000000L
+#define RLC_THREAD1_DELAY__SPARE__SHIFT 0x00000018
+#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS_MASK 0xffffffffL
+#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS__SHIFT 0x00000000
+#define SCRATCH_ADDR__OBSOLETE_ADDR_MASK 0xffffffffL
+#define SCRATCH_ADDR__OBSOLETE_ADDR__SHIFT 0x00000000
+#define SCRATCH_REG0__SCRATCH_REG0_MASK 0xffffffffL
+#define SCRATCH_REG0__SCRATCH_REG0__SHIFT 0x00000000
+#define SCRATCH_REG1__SCRATCH_REG1_MASK 0xffffffffL
+#define SCRATCH_REG1__SCRATCH_REG1__SHIFT 0x00000000
+#define SCRATCH_REG2__SCRATCH_REG2_MASK 0xffffffffL
+#define SCRATCH_REG2__SCRATCH_REG2__SHIFT 0x00000000
+#define SCRATCH_REG3__SCRATCH_REG3_MASK 0xffffffffL
+#define SCRATCH_REG3__SCRATCH_REG3__SHIFT 0x00000000
+#define SCRATCH_REG4__SCRATCH_REG4_MASK 0xffffffffL
+#define SCRATCH_REG4__SCRATCH_REG4__SHIFT 0x00000000
+#define SCRATCH_REG5__SCRATCH_REG5_MASK 0xffffffffL
+#define SCRATCH_REG5__SCRATCH_REG5__SHIFT 0x00000000
+#define SCRATCH_REG6__SCRATCH_REG6_MASK 0xffffffffL
+#define SCRATCH_REG6__SCRATCH_REG6__SHIFT 0x00000000
+#define SCRATCH_REG7__SCRATCH_REG7_MASK 0xffffffffL
+#define SCRATCH_REG7__SCRATCH_REG7__SHIFT 0x00000000
+#define SCRATCH_UMSK__OBSOLETE_SWAP_MASK 0x00030000L
+#define SCRATCH_UMSK__OBSOLETE_SWAP__SHIFT 0x00000010
+#define SCRATCH_UMSK__OBSOLETE_UMSK_MASK 0x000000ffL
+#define SCRATCH_UMSK__OBSOLETE_UMSK__SHIFT 0x00000000
+#define SETUP_DEBUG_REG0__cl_dyn_sclk_vld_MASK 0x80000000L
+#define SETUP_DEBUG_REG0__cl_dyn_sclk_vld__SHIFT 0x0000001f
+#define SETUP_DEBUG_REG0__event_gated_MASK 0x10000000L
+#define SETUP_DEBUG_REG0__event_gated__SHIFT 0x0000001c
+#define SETUP_DEBUG_REG0__event_id_gated_MASK 0x0fc00000L
+#define SETUP_DEBUG_REG0__event_id_gated__SHIFT 0x00000016
+#define SETUP_DEBUG_REG0__geom_busy_MASK 0x00200000L
+#define SETUP_DEBUG_REG0__geom_busy__SHIFT 0x00000015
+#define SETUP_DEBUG_REG0__geom_enable_MASK 0x00008000L
+#define SETUP_DEBUG_REG0__geom_enable__SHIFT 0x0000000f
+#define SETUP_DEBUG_REG0__ge_stallb_MASK 0x00004000L
+#define SETUP_DEBUG_REG0__ge_stallb__SHIFT 0x0000000e
+#define SETUP_DEBUG_REG0__pfifo_busy_MASK 0x00080000L
+#define SETUP_DEBUG_REG0__pfifo_busy__SHIFT 0x00000013
+#define SETUP_DEBUG_REG0__pmode_prim_gated_MASK 0x20000000L
+#define SETUP_DEBUG_REG0__pmode_prim_gated__SHIFT 0x0000001d
+#define SETUP_DEBUG_REG0__pmode_state_MASK 0x00003f00L
+#define SETUP_DEBUG_REG0__pmode_state__SHIFT 0x00000008
+#define SETUP_DEBUG_REG0__su_baryc_cntl_state_MASK 0x00000003L
+#define SETUP_DEBUG_REG0__su_baryc_cntl_state__SHIFT 0x00000000
+#define SETUP_DEBUG_REG0__su_clip_baryc_free_MASK 0x00030000L
+#define SETUP_DEBUG_REG0__su_clip_baryc_free__SHIFT 0x00000010
+#define SETUP_DEBUG_REG0__su_clip_rtr_MASK 0x00040000L
+#define SETUP_DEBUG_REG0__su_clip_rtr__SHIFT 0x00000012
+#define SETUP_DEBUG_REG0__su_cntl_busy_MASK 0x00100000L
+#define SETUP_DEBUG_REG0__su_cntl_busy__SHIFT 0x00000014
+#define SETUP_DEBUG_REG0__su_cntl_state_MASK 0x0000003cL
+#define SETUP_DEBUG_REG0__su_cntl_state__SHIFT 0x00000002
+#define SETUP_DEBUG_REG0__su_dyn_sclk_vld_MASK 0x40000000L
+#define SETUP_DEBUG_REG0__su_dyn_sclk_vld__SHIFT 0x0000001e
+#define SETUP_DEBUG_REG1__x_sort0_gated_23_8_MASK 0xffff0000L
+#define SETUP_DEBUG_REG1__x_sort0_gated_23_8__SHIFT 0x00000010
+#define SETUP_DEBUG_REG1__y_sort0_gated_23_8_MASK 0x0000ffffL
+#define SETUP_DEBUG_REG1__y_sort0_gated_23_8__SHIFT 0x00000000
+#define SETUP_DEBUG_REG2__x_sort1_gated_23_8_MASK 0xffff0000L
+#define SETUP_DEBUG_REG2__x_sort1_gated_23_8__SHIFT 0x00000010
+#define SETUP_DEBUG_REG2__y_sort1_gated_23_8_MASK 0x0000ffffL
+#define SETUP_DEBUG_REG2__y_sort1_gated_23_8__SHIFT 0x00000000
+#define SETUP_DEBUG_REG3__x_sort2_gated_23_8_MASK 0xffff0000L
+#define SETUP_DEBUG_REG3__x_sort2_gated_23_8__SHIFT 0x00000010
+#define SETUP_DEBUG_REG3__y_sort2_gated_23_8_MASK 0x0000ffffL
+#define SETUP_DEBUG_REG3__y_sort2_gated_23_8__SHIFT 0x00000000
+#define SETUP_DEBUG_REG4__attr_indx_sort0_gated_MASK 0x00003fffL
+#define SETUP_DEBUG_REG4__attr_indx_sort0_gated__SHIFT 0x00000000
+#define SETUP_DEBUG_REG4__backfacing_gated_MASK 0x00008000L
+#define SETUP_DEBUG_REG4__backfacing_gated__SHIFT 0x0000000f
+#define SETUP_DEBUG_REG4__clipped_gated_MASK 0x00080000L
+#define SETUP_DEBUG_REG4__clipped_gated__SHIFT 0x00000013
+#define SETUP_DEBUG_REG4__dealloc_slot_gated_MASK 0x00700000L
+#define SETUP_DEBUG_REG4__dealloc_slot_gated__SHIFT 0x00000014
+#define SETUP_DEBUG_REG4__diamond_rule_gated_MASK 0x03000000L
+#define SETUP_DEBUG_REG4__diamond_rule_gated__SHIFT 0x00000018
+#define SETUP_DEBUG_REG4__eop_gated_MASK 0x80000000L
+#define SETUP_DEBUG_REG4__eop_gated__SHIFT 0x0000001f
+#define SETUP_DEBUG_REG4__fpov_gated_MASK 0x60000000L
+#define SETUP_DEBUG_REG4__fpov_gated__SHIFT 0x0000001d
+#define SETUP_DEBUG_REG4__null_prim_gated_MASK 0x00004000L
+#define SETUP_DEBUG_REG4__null_prim_gated__SHIFT 0x0000000e
+#define SETUP_DEBUG_REG4__st_indx_gated_MASK 0x00070000L
+#define SETUP_DEBUG_REG4__st_indx_gated__SHIFT 0x00000010
+#define SETUP_DEBUG_REG4__type_gated_MASK 0x1c000000L
+#define SETUP_DEBUG_REG4__type_gated__SHIFT 0x0000001a
+#define SETUP_DEBUG_REG4__xmajor_gated_MASK 0x00800000L
+#define SETUP_DEBUG_REG4__xmajor_gated__SHIFT 0x00000017
+#define SETUP_DEBUG_REG5__attr_indx_sort1_gated_MASK 0x0fffc000L
+#define SETUP_DEBUG_REG5__attr_indx_sort1_gated__SHIFT 0x0000000e
+#define SETUP_DEBUG_REG5__attr_indx_sort2_gated_MASK 0x00003fffL
+#define SETUP_DEBUG_REG5__attr_indx_sort2_gated__SHIFT 0x00000000
+#define SETUP_DEBUG_REG5__pa_reg_sclk_vld_MASK 0x80000000L
+#define SETUP_DEBUG_REG5__pa_reg_sclk_vld__SHIFT 0x0000001f
+#define SETUP_DEBUG_REG5__provoking_vtx_gated_MASK 0x30000000L
+#define SETUP_DEBUG_REG5__provoking_vtx_gated__SHIFT 0x0000001c
+#define SETUP_DEBUG_REG5__valid_prim_gated_MASK 0x40000000L
+#define SETUP_DEBUG_REG5__valid_prim_gated__SHIFT 0x0000001e
+#define SPI_ARB_CYCLES_0__TS0_DURATION_MASK 0x0000ffffL
+#define SPI_ARB_CYCLES_0__TS0_DURATION__SHIFT 0x00000000
+#define SPI_ARB_CYCLES_0__TS1_DURATION_MASK 0xffff0000L
+#define SPI_ARB_CYCLES_0__TS1_DURATION__SHIFT 0x00000010
+#define SPI_ARB_CYCLES_1__TS2_DURATION_MASK 0x0000ffffL
+#define SPI_ARB_CYCLES_1__TS2_DURATION__SHIFT 0x00000000
+#define SPI_ARB_CYCLES_1__TS3_DURATION_MASK 0xffff0000L
+#define SPI_ARB_CYCLES_1__TS3_DURATION__SHIFT 0x00000010
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0_MASK 0x00000007L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0__SHIFT 0x00000000
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1_MASK 0x00000038L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1__SHIFT 0x00000003
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2_MASK 0x000001c0L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2__SHIFT 0x00000006
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3_MASK 0x00000e00L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3__SHIFT 0x00000009
+#define SPI_ARB_PRIORITY__TS0_DUR_MULT_MASK 0x00003000L
+#define SPI_ARB_PRIORITY__TS0_DUR_MULT__SHIFT 0x0000000c
+#define SPI_ARB_PRIORITY__TS1_DUR_MULT_MASK 0x0000c000L
+#define SPI_ARB_PRIORITY__TS1_DUR_MULT__SHIFT 0x0000000e
+#define SPI_ARB_PRIORITY__TS2_DUR_MULT_MASK 0x00030000L
+#define SPI_ARB_PRIORITY__TS2_DUR_MULT__SHIFT 0x00000010
+#define SPI_ARB_PRIORITY__TS3_DUR_MULT_MASK 0x000c0000L
+#define SPI_ARB_PRIORITY__TS3_DUR_MULT__SHIFT 0x00000012
+#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS_MASK 0x01000000L
+#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS__SHIFT 0x00000018
+#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL_MASK 0x00000100L
+#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL__SHIFT 0x00000008
+#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL_MASK 0x00001000L
+#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL__SHIFT 0x0000000c
+#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL_MASK 0x00000001L
+#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL__SHIFT 0x00000000
+#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL_MASK 0x00000010L
+#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL__SHIFT 0x00000004
+#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION_MASK 0x00030000L
+#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION__SHIFT 0x00000010
+#define SPI_BARYC_CNTL__POS_FLOAT_ULC_MASK 0x00100000L
+#define SPI_BARYC_CNTL__POS_FLOAT_ULC__SHIFT 0x00000014
+#define SPI_CONFIG_CNTL_1__CRC_SIMD_ID_WADDR_DISABLE_MASK 0x00000100L
+#define SPI_CONFIG_CNTL_1__CRC_SIMD_ID_WADDR_DISABLE__SHIFT 0x00000008
+#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW_MASK 0x00000010L
+#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW__SHIFT 0x00000004
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT_MASK 0x00003c00L
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT__SHIFT 0x0000000a
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_MODE_MASK 0x00000200L
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_MODE__SHIFT 0x00000009
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE_MASK 0x00000040L
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE__SHIFT 0x00000006
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_SIZE_MASK 0xffff0000L
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_SIZE__SHIFT 0x00000010
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT_MASK 0x00000080L
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT__SHIFT 0x00000007
+#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY_MASK 0x0000000fL
+#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY__SHIFT 0x00000000
+#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS_MASK 0x02000000L
+#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS__SHIFT 0x00000019
+#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS_MASK 0x01000000L
+#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS__SHIFT 0x00000018
+#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER_MASK 0x00e00000L
+#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER__SHIFT 0x00000015
+#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY_MASK 0x001fffffL
+#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY__SHIFT 0x00000000
+#define SPI_CONFIG_CNTL__RSRC_MGMT_RESET_MASK 0x04000000L
+#define SPI_CONFIG_CNTL__RSRC_MGMT_RESET__SHIFT 0x0000001a
+#define SPI_CONFIG_CNTL__TTRACE_STALL_ALL_MASK 0x08000000L
+#define SPI_CONFIG_CNTL__TTRACE_STALL_ALL__SHIFT 0x0000001b
+#define SPI_DEBUG_BUSY__CS0_BUSY_MASK 0x00000080L
+#define SPI_DEBUG_BUSY__CS0_BUSY__SHIFT 0x00000007
+#define SPI_DEBUG_BUSY__CS1_BUSY_MASK 0x00000100L
+#define SPI_DEBUG_BUSY__CS1_BUSY__SHIFT 0x00000008
+#define SPI_DEBUG_BUSY__CS2_BUSY_MASK 0x00000200L
+#define SPI_DEBUG_BUSY__CS2_BUSY__SHIFT 0x00000009
+#define SPI_DEBUG_BUSY__CS3_BUSY_MASK 0x00000800L
+#define SPI_DEBUG_BUSY__CS3_BUSY__SHIFT 0x0000000b
+#define SPI_DEBUG_BUSY__CS4_BUSY_MASK 0x00001000L
+#define SPI_DEBUG_BUSY__CS4_BUSY__SHIFT 0x0000000c
+#define SPI_DEBUG_BUSY__CS5_BUSY_MASK 0x00002000L
+#define SPI_DEBUG_BUSY__CS5_BUSY__SHIFT 0x0000000d
+#define SPI_DEBUG_BUSY__CS6_BUSY_MASK 0x00004000L
+#define SPI_DEBUG_BUSY__CS6_BUSY__SHIFT 0x0000000e
+#define SPI_DEBUG_BUSY__CS7_BUSY_MASK 0x00008000L
+#define SPI_DEBUG_BUSY__CS7_BUSY__SHIFT 0x0000000f
+#define SPI_DEBUG_BUSY__CSG_BUSY_MASK 0x00000080L
+#define SPI_DEBUG_BUSY__CSG_BUSY__SHIFT 0x00000007
+#define SPI_DEBUG_BUSY__ES_BUSY_MASK 0x00000004L
+#define SPI_DEBUG_BUSY__ES_BUSY__SHIFT 0x00000002
+#define SPI_DEBUG_BUSY__EVENT_CLCTR_BUSY_MASK 0x00008000L
+#define SPI_DEBUG_BUSY__EVENT_CLCTR_BUSY__SHIFT 0x0000000f
+#define SPI_DEBUG_BUSY__GRBM_BUSY_MASK 0x00010000L
+#define SPI_DEBUG_BUSY__GRBM_BUSY__SHIFT 0x00000010
+#define SPI_DEBUG_BUSY__GS_BUSY_MASK 0x00000008L
+#define SPI_DEBUG_BUSY__GS_BUSY__SHIFT 0x00000003
+#define SPI_DEBUG_BUSY__HS_BUSY_MASK 0x00000002L
+#define SPI_DEBUG_BUSY__HS_BUSY__SHIFT 0x00000001
+#define SPI_DEBUG_BUSY__LDS_WR_CTL0_BUSY_MASK 0x00000400L
+#define SPI_DEBUG_BUSY__LDS_WR_CTL0_BUSY__SHIFT 0x0000000a
+#define SPI_DEBUG_BUSY__LDS_WR_CTL1_BUSY_MASK 0x00000800L
+#define SPI_DEBUG_BUSY__LDS_WR_CTL1_BUSY__SHIFT 0x0000000b
+#define SPI_DEBUG_BUSY__LS_BUSY_MASK 0x00000001L
+#define SPI_DEBUG_BUSY__LS_BUSY__SHIFT 0x00000000
+#define SPI_DEBUG_BUSY__PC_DEALLOC_BUSY_MASK 0x00100000L
+#define SPI_DEBUG_BUSY__PC_DEALLOC_BUSY__SHIFT 0x00000014
+#define SPI_DEBUG_BUSY__PS0_BUSY_MASK 0x00000020L
+#define SPI_DEBUG_BUSY__PS0_BUSY__SHIFT 0x00000005
+#define SPI_DEBUG_BUSY__PS1_BUSY_MASK 0x00000040L
+#define SPI_DEBUG_BUSY__PS1_BUSY__SHIFT 0x00000006
+#define SPI_DEBUG_BUSY__RSRC_ALLOC0_BUSY_MASK 0x00001000L
+#define SPI_DEBUG_BUSY__RSRC_ALLOC0_BUSY__SHIFT 0x0000000c
+#define SPI_DEBUG_BUSY__RSRC_ALLOC1_BUSY_MASK 0x00002000L
+#define SPI_DEBUG_BUSY__RSRC_ALLOC1_BUSY__SHIFT 0x0000000d
+#define SPI_DEBUG_BUSY__SPIS_BUSY_MASK 0x00020000L
+#define SPI_DEBUG_BUSY__SPIS_BUSY__SHIFT 0x00000011
+#define SPI_DEBUG_BUSY__VS_BUSY_MASK 0x00000010L
+#define SPI_DEBUG_BUSY__VS_BUSY__SHIFT 0x00000004
+#define SPI_DEBUG_CNTL__DEBUG_GRBM_OVERRIDE_MASK 0x00000001L
+#define SPI_DEBUG_CNTL__DEBUG_GRBM_OVERRIDE__SHIFT 0x00000000
+#define SPI_DEBUG_CNTL__DEBUG_GROUP_SEL_MASK 0x000003e0L
+#define SPI_DEBUG_CNTL__DEBUG_GROUP_SEL__SHIFT 0x00000005
+#define SPI_DEBUG_CNTL__DEBUG_PIPE_SEL_MASK 0x0e000000L
+#define SPI_DEBUG_CNTL__DEBUG_PIPE_SEL__SHIFT 0x00000019
+#define SPI_DEBUG_CNTL__DEBUG_REG_EN_MASK 0x80000000L
+#define SPI_DEBUG_CNTL__DEBUG_REG_EN__SHIFT 0x0000001f
+#define SPI_DEBUG_CNTL__DEBUG_SH_SEL_MASK 0x00010000L
+#define SPI_DEBUG_CNTL__DEBUG_SH_SEL__SHIFT 0x00000010
+#define SPI_DEBUG_CNTL__DEBUG_SIMD_SEL_MASK 0x0000fc00L
+#define SPI_DEBUG_CNTL__DEBUG_SIMD_SEL__SHIFT 0x0000000a
+#define SPI_DEBUG_CNTL__DEBUG_THREAD_TYPE_SEL_MASK 0x0000001eL
+#define SPI_DEBUG_CNTL__DEBUG_THREAD_TYPE_SEL__SHIFT 0x00000001
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_0_MASK 0x00020000L
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_0__SHIFT 0x00000011
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_1_MASK 0x00040000L
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_1__SHIFT 0x00000012
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_2_MASK 0x00080000L
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_2__SHIFT 0x00000013
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_3_MASK 0x00100000L
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_3__SHIFT 0x00000014
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_4_MASK 0x00200000L
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_4__SHIFT 0x00000015
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_5_MASK 0x00400000L
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_5__SHIFT 0x00000016
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_6_MASK 0x00800000L
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_6__SHIFT 0x00000017
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_7_MASK 0x01000000L
+#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_7__SHIFT 0x00000018
+#define SPI_DEBUG_READ__DATA_MASK 0x00ffffffL
+#define SPI_DEBUG_READ__DATA__SHIFT 0x00000000
+#define SPI_GDS_CREDITS__DS_CMD_CREDITS_MASK 0x0000ff00L
+#define SPI_GDS_CREDITS__DS_CMD_CREDITS__SHIFT 0x00000008
+#define SPI_GDS_CREDITS__DS_DATA_CREDITS_MASK 0x000000ffL
+#define SPI_GDS_CREDITS__DS_DATA_CREDITS__SHIFT 0x00000000
+#define SPI_GDS_CREDITS__UNUSED_MASK 0xffff0000L
+#define SPI_GDS_CREDITS__UNUSED__SHIFT 0x00000010
+#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA_MASK 0x00000001L
+#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA__SHIFT 0x00000000
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA_MASK 0x00000002L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA__SHIFT 0x00000001
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W_MASK 0x00003800L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W__SHIFT 0x0000000b
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X_MASK 0x0000001cL
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X__SHIFT 0x00000002
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y_MASK 0x000000e0L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y__SHIFT 0x00000005
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z_MASK 0x00000700L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z__SHIFT 0x00000008
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1_MASK 0x00004000L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1__SHIFT 0x0000000e
+#define SPI_LB_CTR_CTRL__LOAD_MASK 0x00000001L
+#define SPI_LB_CTR_CTRL__LOAD__SHIFT 0x00000000
+#define SPI_LB_CU_MASK__CU_MASK_MASK 0x0000ffffL
+#define SPI_LB_CU_MASK__CU_MASK__SHIFT 0x00000000
+#define SPI_LB_DATA_REG__CNT_DATA_MASK 0xffffffffL
+#define SPI_LB_DATA_REG__CNT_DATA__SHIFT 0x00000000
+#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x000000ffL
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x000000ffL
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x000000ffL
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x000000ffL
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SPI_PERFCOUNTER_BINS__BIN0_MAX_MASK 0x000000f0L
+#define SPI_PERFCOUNTER_BINS__BIN0_MAX__SHIFT 0x00000004
+#define SPI_PERFCOUNTER_BINS__BIN0_MIN_MASK 0x0000000fL
+#define SPI_PERFCOUNTER_BINS__BIN0_MIN__SHIFT 0x00000000
+#define SPI_PERFCOUNTER_BINS__BIN1_MAX_MASK 0x0000f000L
+#define SPI_PERFCOUNTER_BINS__BIN1_MAX__SHIFT 0x0000000c
+#define SPI_PERFCOUNTER_BINS__BIN1_MIN_MASK 0x00000f00L
+#define SPI_PERFCOUNTER_BINS__BIN1_MIN__SHIFT 0x00000008
+#define SPI_PERFCOUNTER_BINS__BIN2_MAX_MASK 0x00f00000L
+#define SPI_PERFCOUNTER_BINS__BIN2_MAX__SHIFT 0x00000014
+#define SPI_PERFCOUNTER_BINS__BIN2_MIN_MASK 0x000f0000L
+#define SPI_PERFCOUNTER_BINS__BIN2_MIN__SHIFT 0x00000010
+#define SPI_PERFCOUNTER_BINS__BIN3_MAX_MASK 0xf0000000L
+#define SPI_PERFCOUNTER_BINS__BIN3_MAX__SHIFT 0x0000001c
+#define SPI_PERFCOUNTER_BINS__BIN3_MIN_MASK 0x0f000000L
+#define SPI_PERFCOUNTER_BINS__BIN3_MIN__SHIFT 0x00000018
+#define SPI_PG_ENABLE_STATIC_CU_MASK__CU_MASK_MASK 0x0000ffffL
+#define SPI_PG_ENABLE_STATIC_CU_MASK__CU_MASK__SHIFT 0x00000000
+#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE_MASK 0x00004000L
+#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE__SHIFT 0x0000000e
+#define SPI_PS_IN_CONTROL__NUM_INTERP_MASK 0x0000003fL
+#define SPI_PS_IN_CONTROL__NUM_INTERP__SHIFT 0x00000000
+#define SPI_PS_IN_CONTROL__PARAM_GEN_MASK 0x00000040L
+#define SPI_PS_IN_CONTROL__PARAM_GEN__SHIFT 0x00000006
+#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA_MASK 0x00002000L
+#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA__SHIFT 0x0000000d
+#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA_MASK 0x00001000L
+#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA__SHIFT 0x0000000c
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA_MASK 0x00000020L
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA__SHIFT 0x00000005
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA_MASK 0x00000040L
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA__SHIFT 0x00000006
+#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA_MASK 0x00000010L
+#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA__SHIFT 0x00000004
+#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA_MASK 0x00000080L
+#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA__SHIFT 0x00000007
+#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA_MASK 0x00000002L
+#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA__SHIFT 0x00000001
+#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA_MASK 0x00000004L
+#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA__SHIFT 0x00000002
+#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA_MASK 0x00000008L
+#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA__SHIFT 0x00000003
+#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA_MASK 0x00000001L
+#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA__SHIFT 0x00000000
+#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA_MASK 0x00008000L
+#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA__SHIFT 0x0000000f
+#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA_MASK 0x00000800L
+#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA__SHIFT 0x0000000b
+#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA_MASK 0x00000100L
+#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA__SHIFT 0x00000008
+#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA_MASK 0x00000200L
+#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA__SHIFT 0x00000009
+#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA_MASK 0x00000400L
+#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA__SHIFT 0x0000000a
+#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA_MASK 0x00004000L
+#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA__SHIFT 0x0000000e
+#define SPI_PS_INPUT_CNTL_0__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_0__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_0__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_0__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_0__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_0__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_10__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_10__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_10__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_10__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_10__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_10__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_11__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_11__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_11__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_11__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_11__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_11__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_12__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_12__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_12__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_12__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_12__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_12__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_13__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_13__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_13__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_13__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_13__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_13__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_14__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_14__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_14__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_14__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_14__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_14__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_15__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_15__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_15__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_15__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_15__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_15__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_16__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_16__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_16__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_16__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_16__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_16__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_17__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_17__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_17__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_17__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_17__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_17__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_18__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_18__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_18__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_18__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_18__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_18__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_19__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_19__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_19__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_19__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_19__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_19__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_1__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_1__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_1__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_1__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_1__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_1__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_20__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_20__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_20__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_20__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_21__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_21__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_21__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_21__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_22__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_22__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_22__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_22__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_23__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_23__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_23__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_23__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_24__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_24__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_24__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_24__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_25__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_25__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_25__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_25__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_26__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_26__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_26__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_26__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_27__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_27__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_27__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_27__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_28__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_28__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_28__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_28__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_29__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_29__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_29__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_29__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_2__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_2__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_2__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_2__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_2__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_2__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_30__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_30__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_30__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_30__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_31__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_31__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_31__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_31__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_3__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_3__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_3__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_3__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_3__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_3__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_4__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_4__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_4__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_4__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_4__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_4__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_5__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_5__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_5__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_5__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_5__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_5__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_6__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_6__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_6__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_6__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_6__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_6__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_7__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_7__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_7__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_7__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_7__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_7__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_8__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_8__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_8__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_8__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_8__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_8__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_CNTL_9__CYL_WRAP_MASK 0x0001e000L
+#define SPI_PS_INPUT_CNTL_9__CYL_WRAP__SHIFT 0x0000000d
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_MASK 0x00000300L
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL__SHIFT 0x00000008
+#define SPI_PS_INPUT_CNTL_9__DUP_MASK 0x00040000L
+#define SPI_PS_INPUT_CNTL_9__DUP__SHIFT 0x00000012
+#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE_MASK 0x00000400L
+#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE__SHIFT 0x0000000a
+#define SPI_PS_INPUT_CNTL_9__OFFSET_MASK 0x0000003fL
+#define SPI_PS_INPUT_CNTL_9__OFFSET__SHIFT 0x00000000
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_MASK 0x00020000L
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX__SHIFT 0x00000011
+#define SPI_PS_INPUT_ENA__ANCILLARY_ENA_MASK 0x00002000L
+#define SPI_PS_INPUT_ENA__ANCILLARY_ENA__SHIFT 0x0000000d
+#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA_MASK 0x00001000L
+#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA__SHIFT 0x0000000c
+#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA_MASK 0x00000020L
+#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA__SHIFT 0x00000005
+#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA_MASK 0x00000040L
+#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA__SHIFT 0x00000006
+#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA_MASK 0x00000010L
+#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA__SHIFT 0x00000004
+#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA_MASK 0x00000080L
+#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA__SHIFT 0x00000007
+#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA_MASK 0x00000002L
+#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA__SHIFT 0x00000001
+#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA_MASK 0x00000004L
+#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA__SHIFT 0x00000002
+#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA_MASK 0x00000008L
+#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA__SHIFT 0x00000003
+#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA_MASK 0x00000001L
+#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA__SHIFT 0x00000000
+#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA_MASK 0x00008000L
+#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA__SHIFT 0x0000000f
+#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA_MASK 0x00000800L
+#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA__SHIFT 0x0000000b
+#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA_MASK 0x00000100L
+#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA__SHIFT 0x00000008
+#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA_MASK 0x00000200L
+#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA__SHIFT 0x00000009
+#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA_MASK 0x00000400L
+#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA__SHIFT 0x0000000a
+#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA_MASK 0x00004000L
+#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA__SHIFT 0x0000000e
+#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID_MASK 0x00000fffL
+#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT 0x00000000
+#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT_MASK 0x0000000fL
+#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT__SHIFT 0x00000000
+#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT_MASK 0x000000f0L
+#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT__SHIFT 0x00000004
+#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT_MASK 0x00000f00L
+#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT__SHIFT 0x00000008
+#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT_MASK 0x0000f000L
+#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT__SHIFT 0x0000000c
+#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT_MASK 0x000f0000L
+#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT__SHIFT 0x00000010
+#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT_MASK 0x00f00000L
+#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT__SHIFT 0x00000014
+#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT_MASK 0x0f000000L
+#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT__SHIFT 0x00000018
+#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT_MASK 0xf0000000L
+#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT__SHIFT 0x0000001c
+#define SPI_SHADER_PGM_HI_ES__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_PGM_HI_ES__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_PGM_HI_GS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_PGM_HI_GS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_PGM_HI_HS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_PGM_HI_HS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_PGM_HI_LS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_PGM_HI_LS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_PGM_HI_PS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_PGM_HI_PS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_PGM_HI_VS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_PGM_HI_VS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_PGM_LO_ES__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_PGM_LO_ES__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_PGM_LO_GS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_PGM_LO_GS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_PGM_LO_HS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_PGM_LO_HS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_PGM_LO_LS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_PGM_LO_LS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_PGM_LO_PS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_PGM_LO_PS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_PGM_LO_VS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_PGM_LO_VS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC1_ES__CACHE_CTL_MASK 0x38000000L
+#define SPI_SHADER_PGM_RSRC1_ES__CACHE_CTL__SHIFT 0x0000001b
+#define SPI_SHADER_PGM_RSRC1_ES__CDBG_USER_MASK 0x40000000L
+#define SPI_SHADER_PGM_RSRC1_ES__CDBG_USER__SHIFT 0x0000001e
+#define SPI_SHADER_PGM_RSRC1_ES__CU_GROUP_ENABLE_MASK 0x04000000L
+#define SPI_SHADER_PGM_RSRC1_ES__CU_GROUP_ENABLE__SHIFT 0x0000001a
+#define SPI_SHADER_PGM_RSRC1_ES__DEBUG_MODE_MASK 0x00400000L
+#define SPI_SHADER_PGM_RSRC1_ES__DEBUG_MODE__SHIFT 0x00000016
+#define SPI_SHADER_PGM_RSRC1_ES__DX10_CLAMP_MASK 0x00200000L
+#define SPI_SHADER_PGM_RSRC1_ES__DX10_CLAMP__SHIFT 0x00000015
+#define SPI_SHADER_PGM_RSRC1_ES__FLOAT_MODE_MASK 0x000ff000L
+#define SPI_SHADER_PGM_RSRC1_ES__FLOAT_MODE__SHIFT 0x0000000c
+#define SPI_SHADER_PGM_RSRC1_ES__IEEE_MODE_MASK 0x00800000L
+#define SPI_SHADER_PGM_RSRC1_ES__IEEE_MODE__SHIFT 0x00000017
+#define SPI_SHADER_PGM_RSRC1_ES__PRIORITY_MASK 0x00000c00L
+#define SPI_SHADER_PGM_RSRC1_ES__PRIORITY__SHIFT 0x0000000a
+#define SPI_SHADER_PGM_RSRC1_ES__PRIV_MASK 0x00100000L
+#define SPI_SHADER_PGM_RSRC1_ES__PRIV__SHIFT 0x00000014
+#define SPI_SHADER_PGM_RSRC1_ES__SGPRS_MASK 0x000003c0L
+#define SPI_SHADER_PGM_RSRC1_ES__SGPRS__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC1_ES__VGPR_COMP_CNT_MASK 0x03000000L
+#define SPI_SHADER_PGM_RSRC1_ES__VGPR_COMP_CNT__SHIFT 0x00000018
+#define SPI_SHADER_PGM_RSRC1_ES__VGPRS_MASK 0x0000003fL
+#define SPI_SHADER_PGM_RSRC1_ES__VGPRS__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC1_GS__CACHE_CTL_MASK 0x0e000000L
+#define SPI_SHADER_PGM_RSRC1_GS__CACHE_CTL__SHIFT 0x00000019
+#define SPI_SHADER_PGM_RSRC1_GS__CDBG_USER_MASK 0x10000000L
+#define SPI_SHADER_PGM_RSRC1_GS__CDBG_USER__SHIFT 0x0000001c
+#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE_MASK 0x01000000L
+#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE__SHIFT 0x00000018
+#define SPI_SHADER_PGM_RSRC1_GS__DEBUG_MODE_MASK 0x00400000L
+#define SPI_SHADER_PGM_RSRC1_GS__DEBUG_MODE__SHIFT 0x00000016
+#define SPI_SHADER_PGM_RSRC1_GS__DX10_CLAMP_MASK 0x00200000L
+#define SPI_SHADER_PGM_RSRC1_GS__DX10_CLAMP__SHIFT 0x00000015
+#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE_MASK 0x000ff000L
+#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE__SHIFT 0x0000000c
+#define SPI_SHADER_PGM_RSRC1_GS__IEEE_MODE_MASK 0x00800000L
+#define SPI_SHADER_PGM_RSRC1_GS__IEEE_MODE__SHIFT 0x00000017
+#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY_MASK 0x00000c00L
+#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY__SHIFT 0x0000000a
+#define SPI_SHADER_PGM_RSRC1_GS__PRIV_MASK 0x00100000L
+#define SPI_SHADER_PGM_RSRC1_GS__PRIV__SHIFT 0x00000014
+#define SPI_SHADER_PGM_RSRC1_GS__SGPRS_MASK 0x000003c0L
+#define SPI_SHADER_PGM_RSRC1_GS__SGPRS__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC1_GS__VGPRS_MASK 0x0000003fL
+#define SPI_SHADER_PGM_RSRC1_GS__VGPRS__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC1_HS__CACHE_CTL_MASK 0x07000000L
+#define SPI_SHADER_PGM_RSRC1_HS__CACHE_CTL__SHIFT 0x00000018
+#define SPI_SHADER_PGM_RSRC1_HS__CDBG_USER_MASK 0x08000000L
+#define SPI_SHADER_PGM_RSRC1_HS__CDBG_USER__SHIFT 0x0000001b
+#define SPI_SHADER_PGM_RSRC1_HS__DEBUG_MODE_MASK 0x00400000L
+#define SPI_SHADER_PGM_RSRC1_HS__DEBUG_MODE__SHIFT 0x00000016
+#define SPI_SHADER_PGM_RSRC1_HS__DX10_CLAMP_MASK 0x00200000L
+#define SPI_SHADER_PGM_RSRC1_HS__DX10_CLAMP__SHIFT 0x00000015
+#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE_MASK 0x000ff000L
+#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE__SHIFT 0x0000000c
+#define SPI_SHADER_PGM_RSRC1_HS__IEEE_MODE_MASK 0x00800000L
+#define SPI_SHADER_PGM_RSRC1_HS__IEEE_MODE__SHIFT 0x00000017
+#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY_MASK 0x00000c00L
+#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY__SHIFT 0x0000000a
+#define SPI_SHADER_PGM_RSRC1_HS__PRIV_MASK 0x00100000L
+#define SPI_SHADER_PGM_RSRC1_HS__PRIV__SHIFT 0x00000014
+#define SPI_SHADER_PGM_RSRC1_HS__SGPRS_MASK 0x000003c0L
+#define SPI_SHADER_PGM_RSRC1_HS__SGPRS__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC1_HS__VGPRS_MASK 0x0000003fL
+#define SPI_SHADER_PGM_RSRC1_HS__VGPRS__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC1_LS__CACHE_CTL_MASK 0x1c000000L
+#define SPI_SHADER_PGM_RSRC1_LS__CACHE_CTL__SHIFT 0x0000001a
+#define SPI_SHADER_PGM_RSRC1_LS__CDBG_USER_MASK 0x20000000L
+#define SPI_SHADER_PGM_RSRC1_LS__CDBG_USER__SHIFT 0x0000001d
+#define SPI_SHADER_PGM_RSRC1_LS__DEBUG_MODE_MASK 0x00400000L
+#define SPI_SHADER_PGM_RSRC1_LS__DEBUG_MODE__SHIFT 0x00000016
+#define SPI_SHADER_PGM_RSRC1_LS__DX10_CLAMP_MASK 0x00200000L
+#define SPI_SHADER_PGM_RSRC1_LS__DX10_CLAMP__SHIFT 0x00000015
+#define SPI_SHADER_PGM_RSRC1_LS__FLOAT_MODE_MASK 0x000ff000L
+#define SPI_SHADER_PGM_RSRC1_LS__FLOAT_MODE__SHIFT 0x0000000c
+#define SPI_SHADER_PGM_RSRC1_LS__IEEE_MODE_MASK 0x00800000L
+#define SPI_SHADER_PGM_RSRC1_LS__IEEE_MODE__SHIFT 0x00000017
+#define SPI_SHADER_PGM_RSRC1_LS__PRIORITY_MASK 0x00000c00L
+#define SPI_SHADER_PGM_RSRC1_LS__PRIORITY__SHIFT 0x0000000a
+#define SPI_SHADER_PGM_RSRC1_LS__PRIV_MASK 0x00100000L
+#define SPI_SHADER_PGM_RSRC1_LS__PRIV__SHIFT 0x00000014
+#define SPI_SHADER_PGM_RSRC1_LS__SGPRS_MASK 0x000003c0L
+#define SPI_SHADER_PGM_RSRC1_LS__SGPRS__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC1_LS__VGPR_COMP_CNT_MASK 0x03000000L
+#define SPI_SHADER_PGM_RSRC1_LS__VGPR_COMP_CNT__SHIFT 0x00000018
+#define SPI_SHADER_PGM_RSRC1_LS__VGPRS_MASK 0x0000003fL
+#define SPI_SHADER_PGM_RSRC1_LS__VGPRS__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC1_PS__CACHE_CTL_MASK 0x0e000000L
+#define SPI_SHADER_PGM_RSRC1_PS__CACHE_CTL__SHIFT 0x00000019
+#define SPI_SHADER_PGM_RSRC1_PS__CDBG_USER_MASK 0x10000000L
+#define SPI_SHADER_PGM_RSRC1_PS__CDBG_USER__SHIFT 0x0000001c
+#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE_MASK 0x01000000L
+#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE__SHIFT 0x00000018
+#define SPI_SHADER_PGM_RSRC1_PS__DEBUG_MODE_MASK 0x00400000L
+#define SPI_SHADER_PGM_RSRC1_PS__DEBUG_MODE__SHIFT 0x00000016
+#define SPI_SHADER_PGM_RSRC1_PS__DX10_CLAMP_MASK 0x00200000L
+#define SPI_SHADER_PGM_RSRC1_PS__DX10_CLAMP__SHIFT 0x00000015
+#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE_MASK 0x000ff000L
+#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE__SHIFT 0x0000000c
+#define SPI_SHADER_PGM_RSRC1_PS__IEEE_MODE_MASK 0x00800000L
+#define SPI_SHADER_PGM_RSRC1_PS__IEEE_MODE__SHIFT 0x00000017
+#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY_MASK 0x00000c00L
+#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY__SHIFT 0x0000000a
+#define SPI_SHADER_PGM_RSRC1_PS__PRIV_MASK 0x00100000L
+#define SPI_SHADER_PGM_RSRC1_PS__PRIV__SHIFT 0x00000014
+#define SPI_SHADER_PGM_RSRC1_PS__SGPRS_MASK 0x000003c0L
+#define SPI_SHADER_PGM_RSRC1_PS__SGPRS__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC1_PS__VGPRS_MASK 0x0000003fL
+#define SPI_SHADER_PGM_RSRC1_PS__VGPRS__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC1_VS__CACHE_CTL_MASK 0x38000000L
+#define SPI_SHADER_PGM_RSRC1_VS__CACHE_CTL__SHIFT 0x0000001b
+#define SPI_SHADER_PGM_RSRC1_VS__CDBG_USER_MASK 0x40000000L
+#define SPI_SHADER_PGM_RSRC1_VS__CDBG_USER__SHIFT 0x0000001e
+#define SPI_SHADER_PGM_RSRC1_VS__CU_GROUP_ENABLE_MASK 0x04000000L
+#define SPI_SHADER_PGM_RSRC1_VS__CU_GROUP_ENABLE__SHIFT 0x0000001a
+#define SPI_SHADER_PGM_RSRC1_VS__DEBUG_MODE_MASK 0x00400000L
+#define SPI_SHADER_PGM_RSRC1_VS__DEBUG_MODE__SHIFT 0x00000016
+#define SPI_SHADER_PGM_RSRC1_VS__DX10_CLAMP_MASK 0x00200000L
+#define SPI_SHADER_PGM_RSRC1_VS__DX10_CLAMP__SHIFT 0x00000015
+#define SPI_SHADER_PGM_RSRC1_VS__FLOAT_MODE_MASK 0x000ff000L
+#define SPI_SHADER_PGM_RSRC1_VS__FLOAT_MODE__SHIFT 0x0000000c
+#define SPI_SHADER_PGM_RSRC1_VS__IEEE_MODE_MASK 0x00800000L
+#define SPI_SHADER_PGM_RSRC1_VS__IEEE_MODE__SHIFT 0x00000017
+#define SPI_SHADER_PGM_RSRC1_VS__PRIORITY_MASK 0x00000c00L
+#define SPI_SHADER_PGM_RSRC1_VS__PRIORITY__SHIFT 0x0000000a
+#define SPI_SHADER_PGM_RSRC1_VS__PRIV_MASK 0x00100000L
+#define SPI_SHADER_PGM_RSRC1_VS__PRIV__SHIFT 0x00000014
+#define SPI_SHADER_PGM_RSRC1_VS__SGPRS_MASK 0x000003c0L
+#define SPI_SHADER_PGM_RSRC1_VS__SGPRS__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC1_VS__VGPR_COMP_CNT_MASK 0x03000000L
+#define SPI_SHADER_PGM_RSRC1_VS__VGPR_COMP_CNT__SHIFT 0x00000018
+#define SPI_SHADER_PGM_RSRC1_VS__VGPRS_MASK 0x0000003fL
+#define SPI_SHADER_PGM_RSRC1_VS__VGPRS__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC2_ES__EXCP_EN_MASK 0x00007f00L
+#define SPI_SHADER_PGM_RSRC2_ES__EXCP_EN__SHIFT 0x00000008
+#define SPI_SHADER_PGM_RSRC2_ES_GS__EXCP_EN_MASK 0x0001ff00L
+#define SPI_SHADER_PGM_RSRC2_ES_GS__EXCP_EN__SHIFT 0x00000008
+#define SPI_SHADER_PGM_RSRC2_ES_GS__LDS_SIZE_MASK 0x1ff00000L
+#define SPI_SHADER_PGM_RSRC2_ES_GS__LDS_SIZE__SHIFT 0x00000014
+#define SPI_SHADER_PGM_RSRC2_ES_GS__OC_LDS_EN_MASK 0x00000080L
+#define SPI_SHADER_PGM_RSRC2_ES_GS__OC_LDS_EN__SHIFT 0x00000007
+#define SPI_SHADER_PGM_RSRC2_ES_GS__SCRATCH_EN_MASK 0x00000001L
+#define SPI_SHADER_PGM_RSRC2_ES_GS__SCRATCH_EN__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC2_ES_GS__TRAP_PRESENT_MASK 0x00000040L
+#define SPI_SHADER_PGM_RSRC2_ES_GS__TRAP_PRESENT__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC2_ES_GS__USER_SGPR_MASK 0x0000003eL
+#define SPI_SHADER_PGM_RSRC2_ES_GS__USER_SGPR__SHIFT 0x00000001
+#define SPI_SHADER_PGM_RSRC2_ES__LDS_SIZE_MASK 0x1ff00000L
+#define SPI_SHADER_PGM_RSRC2_ES__LDS_SIZE__SHIFT 0x00000014
+#define SPI_SHADER_PGM_RSRC2_ES__OC_LDS_EN_MASK 0x00000080L
+#define SPI_SHADER_PGM_RSRC2_ES__OC_LDS_EN__SHIFT 0x00000007
+#define SPI_SHADER_PGM_RSRC2_ES__SCRATCH_EN_MASK 0x00000001L
+#define SPI_SHADER_PGM_RSRC2_ES__SCRATCH_EN__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC2_ES__TRAP_PRESENT_MASK 0x00000040L
+#define SPI_SHADER_PGM_RSRC2_ES__TRAP_PRESENT__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC2_ES__USER_SGPR_MASK 0x0000003eL
+#define SPI_SHADER_PGM_RSRC2_ES__USER_SGPR__SHIFT 0x00000001
+#define SPI_SHADER_PGM_RSRC2_ES_VS__EXCP_EN_MASK 0x0001ff00L
+#define SPI_SHADER_PGM_RSRC2_ES_VS__EXCP_EN__SHIFT 0x00000008
+#define SPI_SHADER_PGM_RSRC2_ES_VS__LDS_SIZE_MASK 0x1ff00000L
+#define SPI_SHADER_PGM_RSRC2_ES_VS__LDS_SIZE__SHIFT 0x00000014
+#define SPI_SHADER_PGM_RSRC2_ES_VS__OC_LDS_EN_MASK 0x00000080L
+#define SPI_SHADER_PGM_RSRC2_ES_VS__OC_LDS_EN__SHIFT 0x00000007
+#define SPI_SHADER_PGM_RSRC2_ES_VS__SCRATCH_EN_MASK 0x00000001L
+#define SPI_SHADER_PGM_RSRC2_ES_VS__SCRATCH_EN__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC2_ES_VS__TRAP_PRESENT_MASK 0x00000040L
+#define SPI_SHADER_PGM_RSRC2_ES_VS__TRAP_PRESENT__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC2_ES_VS__USER_SGPR_MASK 0x0000003eL
+#define SPI_SHADER_PGM_RSRC2_ES_VS__USER_SGPR__SHIFT 0x00000001
+#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN_MASK 0x00003f80L
+#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN__SHIFT 0x00000007
+#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN_MASK 0x00000001L
+#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT_MASK 0x00000040L
+#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MASK 0x0000003eL
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR__SHIFT 0x00000001
+#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN_MASK 0x0000fe00L
+#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN__SHIFT 0x00000009
+#define SPI_SHADER_PGM_RSRC2_HS__OC_LDS_EN_MASK 0x00000080L
+#define SPI_SHADER_PGM_RSRC2_HS__OC_LDS_EN__SHIFT 0x00000007
+#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN_MASK 0x00000001L
+#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC2_HS__TG_SIZE_EN_MASK 0x00000100L
+#define SPI_SHADER_PGM_RSRC2_HS__TG_SIZE_EN__SHIFT 0x00000008
+#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT_MASK 0x00000040L
+#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MASK 0x0000003eL
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR__SHIFT 0x00000001
+#define SPI_SHADER_PGM_RSRC2_LS_ES__EXCP_EN_MASK 0x01ff0000L
+#define SPI_SHADER_PGM_RSRC2_LS_ES__EXCP_EN__SHIFT 0x00000010
+#define SPI_SHADER_PGM_RSRC2_LS_ES__LDS_SIZE_MASK 0x0000ff80L
+#define SPI_SHADER_PGM_RSRC2_LS_ES__LDS_SIZE__SHIFT 0x00000007
+#define SPI_SHADER_PGM_RSRC2_LS_ES__SCRATCH_EN_MASK 0x00000001L
+#define SPI_SHADER_PGM_RSRC2_LS_ES__SCRATCH_EN__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC2_LS_ES__TRAP_PRESENT_MASK 0x00000040L
+#define SPI_SHADER_PGM_RSRC2_LS_ES__TRAP_PRESENT__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC2_LS_ES__USER_SGPR_MASK 0x0000003eL
+#define SPI_SHADER_PGM_RSRC2_LS_ES__USER_SGPR__SHIFT 0x00000001
+#define SPI_SHADER_PGM_RSRC2_LS__EXCP_EN_MASK 0x007f0000L
+#define SPI_SHADER_PGM_RSRC2_LS__EXCP_EN__SHIFT 0x00000010
+#define SPI_SHADER_PGM_RSRC2_LS_HS__EXCP_EN_MASK 0x01ff0000L
+#define SPI_SHADER_PGM_RSRC2_LS_HS__EXCP_EN__SHIFT 0x00000010
+#define SPI_SHADER_PGM_RSRC2_LS_HS__LDS_SIZE_MASK 0x0000ff80L
+#define SPI_SHADER_PGM_RSRC2_LS_HS__LDS_SIZE__SHIFT 0x00000007
+#define SPI_SHADER_PGM_RSRC2_LS_HS__SCRATCH_EN_MASK 0x00000001L
+#define SPI_SHADER_PGM_RSRC2_LS_HS__SCRATCH_EN__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC2_LS_HS__TRAP_PRESENT_MASK 0x00000040L
+#define SPI_SHADER_PGM_RSRC2_LS_HS__TRAP_PRESENT__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC2_LS_HS__USER_SGPR_MASK 0x0000003eL
+#define SPI_SHADER_PGM_RSRC2_LS_HS__USER_SGPR__SHIFT 0x00000001
+#define SPI_SHADER_PGM_RSRC2_LS__LDS_SIZE_MASK 0x0000ff80L
+#define SPI_SHADER_PGM_RSRC2_LS__LDS_SIZE__SHIFT 0x00000007
+#define SPI_SHADER_PGM_RSRC2_LS__SCRATCH_EN_MASK 0x00000001L
+#define SPI_SHADER_PGM_RSRC2_LS__SCRATCH_EN__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC2_LS__TRAP_PRESENT_MASK 0x00000040L
+#define SPI_SHADER_PGM_RSRC2_LS__TRAP_PRESENT__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC2_LS__USER_SGPR_MASK 0x0000003eL
+#define SPI_SHADER_PGM_RSRC2_LS__USER_SGPR__SHIFT 0x00000001
+#define SPI_SHADER_PGM_RSRC2_LS_VS__EXCP_EN_MASK 0x01ff0000L
+#define SPI_SHADER_PGM_RSRC2_LS_VS__EXCP_EN__SHIFT 0x00000010
+#define SPI_SHADER_PGM_RSRC2_LS_VS__LDS_SIZE_MASK 0x0000ff80L
+#define SPI_SHADER_PGM_RSRC2_LS_VS__LDS_SIZE__SHIFT 0x00000007
+#define SPI_SHADER_PGM_RSRC2_LS_VS__SCRATCH_EN_MASK 0x00000001L
+#define SPI_SHADER_PGM_RSRC2_LS_VS__SCRATCH_EN__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC2_LS_VS__TRAP_PRESENT_MASK 0x00000040L
+#define SPI_SHADER_PGM_RSRC2_LS_VS__TRAP_PRESENT__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC2_LS_VS__USER_SGPR_MASK 0x0000003eL
+#define SPI_SHADER_PGM_RSRC2_LS_VS__USER_SGPR__SHIFT 0x00000001
+#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN_MASK 0x007f0000L
+#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN__SHIFT 0x00000010
+#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE_MASK 0x0000ff00L
+#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE__SHIFT 0x00000008
+#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN_MASK 0x00000001L
+#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT_MASK 0x00000040L
+#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MASK 0x0000003eL
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR__SHIFT 0x00000001
+#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN_MASK 0x00000080L
+#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN__SHIFT 0x00000007
+#define SPI_SHADER_PGM_RSRC2_VS__EXCP_EN_MASK 0x000fe000L
+#define SPI_SHADER_PGM_RSRC2_VS__EXCP_EN__SHIFT 0x0000000d
+#define SPI_SHADER_PGM_RSRC2_VS__OC_LDS_EN_MASK 0x00000080L
+#define SPI_SHADER_PGM_RSRC2_VS__OC_LDS_EN__SHIFT 0x00000007
+#define SPI_SHADER_PGM_RSRC2_VS__SCRATCH_EN_MASK 0x00000001L
+#define SPI_SHADER_PGM_RSRC2_VS__SCRATCH_EN__SHIFT 0x00000000
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE0_EN_MASK 0x00000100L
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE0_EN__SHIFT 0x00000008
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE1_EN_MASK 0x00000200L
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE1_EN__SHIFT 0x00000009
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE2_EN_MASK 0x00000400L
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE2_EN__SHIFT 0x0000000a
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE3_EN_MASK 0x00000800L
+#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE3_EN__SHIFT 0x0000000b
+#define SPI_SHADER_PGM_RSRC2_VS__SO_EN_MASK 0x00001000L
+#define SPI_SHADER_PGM_RSRC2_VS__SO_EN__SHIFT 0x0000000c
+#define SPI_SHADER_PGM_RSRC2_VS__TRAP_PRESENT_MASK 0x00000040L
+#define SPI_SHADER_PGM_RSRC2_VS__TRAP_PRESENT__SHIFT 0x00000006
+#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR_MASK 0x0000003eL
+#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR__SHIFT 0x00000001
+#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT_MASK 0x0000000fL
+#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT__SHIFT 0x00000000
+#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT_MASK 0x000000f0L
+#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT__SHIFT 0x00000004
+#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT_MASK 0x00000f00L
+#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT__SHIFT 0x00000008
+#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT_MASK 0x0000f000L
+#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT__SHIFT 0x0000000c
+#define SPI_SHADER_TBA_HI_ES__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_TBA_HI_ES__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TBA_HI_GS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_TBA_HI_GS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TBA_HI_HS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_TBA_HI_HS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TBA_HI_LS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_TBA_HI_LS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TBA_HI_PS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_TBA_HI_PS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TBA_HI_VS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_TBA_HI_VS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TBA_LO_ES__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_TBA_LO_ES__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TBA_LO_GS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_TBA_LO_GS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TBA_LO_HS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_TBA_LO_HS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TBA_LO_LS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_TBA_LO_LS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TBA_LO_PS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_TBA_LO_PS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TBA_LO_VS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_TBA_LO_VS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TMA_HI_ES__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_TMA_HI_ES__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TMA_HI_GS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_TMA_HI_GS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TMA_HI_HS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_TMA_HI_HS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TMA_HI_LS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_TMA_HI_LS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TMA_HI_PS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_TMA_HI_PS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TMA_HI_VS__MEM_BASE_MASK 0x000000ffL
+#define SPI_SHADER_TMA_HI_VS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TMA_LO_ES__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_TMA_LO_ES__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TMA_LO_GS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_TMA_LO_GS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TMA_LO_HS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_TMA_LO_HS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TMA_LO_LS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_TMA_LO_LS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TMA_LO_PS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_TMA_LO_PS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_TMA_LO_VS__MEM_BASE_MASK 0xffffffffL
+#define SPI_SHADER_TMA_LO_VS__MEM_BASE__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_0__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_0__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_10__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_10__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_11__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_11__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_12__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_12__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_13__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_13__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_14__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_14__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_15__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_15__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_1__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_1__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_2__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_2__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_3__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_3__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_4__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_4__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_5__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_5__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_6__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_6__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_7__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_7__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_8__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_8__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_ES_9__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_ES_9__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_0__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_0__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_10__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_10__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_11__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_11__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_12__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_12__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_13__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_13__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_14__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_14__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_15__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_15__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_1__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_1__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_2__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_2__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_3__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_3__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_4__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_4__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_5__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_5__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_6__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_6__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_7__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_7__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_8__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_8__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_GS_9__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_GS_9__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_0__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_0__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_10__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_10__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_11__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_11__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_12__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_12__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_13__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_13__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_14__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_14__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_15__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_15__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_1__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_1__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_2__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_2__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_3__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_3__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_4__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_4__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_5__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_5__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_6__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_6__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_7__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_7__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_8__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_8__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_HS_9__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_HS_9__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_0__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_0__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_10__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_10__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_11__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_11__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_12__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_12__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_13__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_13__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_14__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_14__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_15__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_15__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_1__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_1__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_2__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_2__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_3__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_3__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_4__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_4__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_5__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_5__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_6__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_6__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_7__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_7__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_8__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_8__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_LS_9__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_LS_9__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_0__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_0__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_10__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_10__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_11__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_11__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_12__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_12__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_13__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_13__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_14__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_14__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_15__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_15__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_1__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_1__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_2__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_2__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_3__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_3__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_4__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_4__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_5__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_5__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_6__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_6__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_7__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_7__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_8__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_8__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_PS_9__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_PS_9__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_0__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_0__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_10__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_10__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_11__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_11__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_12__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_12__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_13__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_13__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_14__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_14__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_15__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_15__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_1__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_1__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_2__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_2__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_3__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_3__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_4__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_4__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_5__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_5__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_6__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_6__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_7__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_7__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_8__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_8__DATA__SHIFT 0x00000000
+#define SPI_SHADER_USER_DATA_VS_9__DATA_MASK 0xffffffffL
+#define SPI_SHADER_USER_DATA_VS_9__DATA__SHIFT 0x00000000
+#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT_MASK 0x0000000fL
+#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT__SHIFT 0x00000000
+#define SPI_SLAVE_DEBUG_BUSY__ES_VTX_BUSY_MASK 0x00000004L
+#define SPI_SLAVE_DEBUG_BUSY__ES_VTX_BUSY__SHIFT 0x00000002
+#define SPI_SLAVE_DEBUG_BUSY__EVENT_CNTL_BUSY_MASK 0x00200000L
+#define SPI_SLAVE_DEBUG_BUSY__EVENT_CNTL_BUSY__SHIFT 0x00000015
+#define SPI_SLAVE_DEBUG_BUSY__GS_VTX_BUSY_MASK 0x00000008L
+#define SPI_SLAVE_DEBUG_BUSY__GS_VTX_BUSY__SHIFT 0x00000003
+#define SPI_SLAVE_DEBUG_BUSY__HS_VTX_BUSY_MASK 0x00000002L
+#define SPI_SLAVE_DEBUG_BUSY__HS_VTX_BUSY__SHIFT 0x00000001
+#define SPI_SLAVE_DEBUG_BUSY__LS_VTX_BUSY_MASK 0x00000001L
+#define SPI_SLAVE_DEBUG_BUSY__LS_VTX_BUSY__SHIFT 0x00000000
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC00_BUSY_MASK 0x00000200L
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC00_BUSY__SHIFT 0x00000009
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC01_BUSY_MASK 0x00000400L
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC01_BUSY__SHIFT 0x0000000a
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC02_BUSY_MASK 0x00000800L
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC02_BUSY__SHIFT 0x0000000b
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC03_BUSY_MASK 0x00001000L
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC03_BUSY__SHIFT 0x0000000c
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC10_BUSY_MASK 0x00002000L
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC10_BUSY__SHIFT 0x0000000d
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC11_BUSY_MASK 0x00004000L
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC11_BUSY__SHIFT 0x0000000e
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC12_BUSY_MASK 0x00008000L
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC12_BUSY__SHIFT 0x0000000f
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC13_BUSY_MASK 0x00010000L
+#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC13_BUSY__SHIFT 0x00000010
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC00_BUSY_MASK 0x00000020L
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC00_BUSY__SHIFT 0x00000005
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC01_BUSY_MASK 0x00000040L
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC01_BUSY__SHIFT 0x00000006
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC10_BUSY_MASK 0x00000080L
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC10_BUSY__SHIFT 0x00000007
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC11_BUSY_MASK 0x00000100L
+#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC11_BUSY__SHIFT 0x00000008
+#define SPI_SLAVE_DEBUG_BUSY__VS_VTX_BUSY_MASK 0x00000010L
+#define SPI_SLAVE_DEBUG_BUSY__VS_VTX_BUSY__SHIFT 0x00000004
+#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER0_BUSY_MASK 0x00020000L
+#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER0_BUSY__SHIFT 0x00000011
+#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER1_BUSY_MASK 0x00040000L
+#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER1_BUSY__SHIFT 0x00000012
+#define SPI_SLAVE_DEBUG_BUSY__WAVE_WC0_BUSY_MASK 0x00080000L
+#define SPI_SLAVE_DEBUG_BUSY__WAVE_WC0_BUSY__SHIFT 0x00000013
+#define SPI_SLAVE_DEBUG_BUSY__WAVE_WC1_BUSY_MASK 0x00100000L
+#define SPI_SLAVE_DEBUG_BUSY__WAVE_WC1_BUSY__SHIFT 0x00000014
+#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE_MASK 0x0000ffffL
+#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE__SHIFT 0x00000000
+#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE_MASK 0xffff0000L
+#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE__SHIFT 0x00000010
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE_MASK 0x0000ffffL
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE__SHIFT 0x00000000
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE_MASK 0xffff0000L
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE__SHIFT 0x00000010
+#define SPI_TMPRING_SIZE__WAVESIZE_MASK 0x01fff000L
+#define SPI_TMPRING_SIZE__WAVESIZE__SHIFT 0x0000000c
+#define SPI_TMPRING_SIZE__WAVES_MASK 0x00000fffL
+#define SPI_TMPRING_SIZE__WAVES__SHIFT 0x00000000
+#define SPI_VS_OUT_CONFIG__VS_EXPORT_COUNT_MASK 0x0000003eL
+#define SPI_VS_OUT_CONFIG__VS_EXPORT_COUNT__SHIFT 0x00000001
+#define SPI_VS_OUT_CONFIG__VS_HALF_PACK_MASK 0x00000040L
+#define SPI_VS_OUT_CONFIG__VS_HALF_PACK__SHIFT 0x00000006
+#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH0_MASK 0x0000ffffL
+#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT 0x00000000
+#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH1_MASK 0xffff0000L
+#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT 0x00000010
+#define SQ_BUF_RSRC_WORD0__BASE_ADDRESS_MASK 0xffffffffL
+#define SQ_BUF_RSRC_WORD0__BASE_ADDRESS__SHIFT 0x00000000
+#define SQ_BUF_RSRC_WORD1__BASE_ADDRESS_HI_MASK 0x0000ffffL
+#define SQ_BUF_RSRC_WORD1__BASE_ADDRESS_HI__SHIFT 0x00000000
+#define SQ_BUF_RSRC_WORD1__CACHE_SWIZZLE_MASK 0x40000000L
+#define SQ_BUF_RSRC_WORD1__CACHE_SWIZZLE__SHIFT 0x0000001e
+#define SQ_BUF_RSRC_WORD1__STRIDE_MASK 0x3fff0000L
+#define SQ_BUF_RSRC_WORD1__STRIDE__SHIFT 0x00000010
+#define SQ_BUF_RSRC_WORD1__SWIZZLE_ENABLE_MASK 0x80000000L
+#define SQ_BUF_RSRC_WORD1__SWIZZLE_ENABLE__SHIFT 0x0000001f
+#define SQ_BUF_RSRC_WORD2__NUM_RECORDS_MASK 0xffffffffL
+#define SQ_BUF_RSRC_WORD2__NUM_RECORDS__SHIFT 0x00000000
+#define SQ_BUF_RSRC_WORD3__ADD_TID_ENABLE_MASK 0x00800000L
+#define SQ_BUF_RSRC_WORD3__ADD_TID_ENABLE__SHIFT 0x00000017
+#define SQ_BUF_RSRC_WORD3__ATC_MASK 0x01000000L
+#define SQ_BUF_RSRC_WORD3__ATC__SHIFT 0x00000018
+#define SQ_BUF_RSRC_WORD3__DATA_FORMAT_MASK 0x00078000L
+#define SQ_BUF_RSRC_WORD3__DATA_FORMAT__SHIFT 0x0000000f
+#define SQ_BUF_RSRC_WORD3__DST_SEL_W_MASK 0x00000e00L
+#define SQ_BUF_RSRC_WORD3__DST_SEL_W__SHIFT 0x00000009
+#define SQ_BUF_RSRC_WORD3__DST_SEL_X_MASK 0x00000007L
+#define SQ_BUF_RSRC_WORD3__DST_SEL_X__SHIFT 0x00000000
+#define SQ_BUF_RSRC_WORD3__DST_SEL_Y_MASK 0x00000038L
+#define SQ_BUF_RSRC_WORD3__DST_SEL_Y__SHIFT 0x00000003
+#define SQ_BUF_RSRC_WORD3__DST_SEL_Z_MASK 0x000001c0L
+#define SQ_BUF_RSRC_WORD3__DST_SEL_Z__SHIFT 0x00000006
+#define SQ_BUF_RSRC_WORD3__ELEMENT_SIZE_MASK 0x00180000L
+#define SQ_BUF_RSRC_WORD3__ELEMENT_SIZE__SHIFT 0x00000013
+#define SQ_BUF_RSRC_WORD3__HASH_ENABLE_MASK 0x02000000L
+#define SQ_BUF_RSRC_WORD3__HASH_ENABLE__SHIFT 0x00000019
+#define SQ_BUF_RSRC_WORD3__HEAP_MASK 0x04000000L
+#define SQ_BUF_RSRC_WORD3__HEAP__SHIFT 0x0000001a
+#define SQ_BUF_RSRC_WORD3__INDEX_STRIDE_MASK 0x00600000L
+#define SQ_BUF_RSRC_WORD3__INDEX_STRIDE__SHIFT 0x00000015
+#define SQ_BUF_RSRC_WORD3__MTYPE_MASK 0x38000000L
+#define SQ_BUF_RSRC_WORD3__MTYPE__SHIFT 0x0000001b
+#define SQ_BUF_RSRC_WORD3__NUM_FORMAT_MASK 0x00007000L
+#define SQ_BUF_RSRC_WORD3__NUM_FORMAT__SHIFT 0x0000000c
+#define SQ_BUF_RSRC_WORD3__TYPE_MASK 0xc0000000L
+#define SQ_BUF_RSRC_WORD3__TYPE__SHIFT 0x0000001e
+#define SQC_CACHES__DATA_INVALIDATE_MASK 0x00000002L
+#define SQC_CACHES__DATA_INVALIDATE__SHIFT 0x00000001
+#define SQC_CACHES__INST_INVALIDATE_MASK 0x00000001L
+#define SQC_CACHES__INST_INVALIDATE__SHIFT 0x00000000
+#define SQC_CACHES__INVALIDATE_VOLATILE_MASK 0x00000004L
+#define SQC_CACHES__INVALIDATE_VOLATILE__SHIFT 0x00000002
+#define SQC_CONFIG__DATA_CACHE_SIZE_MASK 0x0000000cL
+#define SQC_CONFIG__DATA_CACHE_SIZE__SHIFT 0x00000002
+#define SQC_CONFIG__FORCE_ALWAYS_MISS_MASK 0x00000080L
+#define SQC_CONFIG__FORCE_ALWAYS_MISS__SHIFT 0x00000007
+#define SQC_CONFIG__FORCE_IN_ORDER_MASK 0x00000100L
+#define SQC_CONFIG__FORCE_IN_ORDER__SHIFT 0x00000008
+#define SQC_CONFIG__HIT_FIFO_DEPTH_MASK 0x00000040L
+#define SQC_CONFIG__HIT_FIFO_DEPTH__SHIFT 0x00000006
+#define SQC_CONFIG__IDENTITY_HASH_BANK_MASK 0x00000200L
+#define SQC_CONFIG__IDENTITY_HASH_BANK__SHIFT 0x00000009
+#define SQC_CONFIG__IDENTITY_HASH_SET_MASK 0x00000400L
+#define SQC_CONFIG__IDENTITY_HASH_SET__SHIFT 0x0000000a
+#define SQC_CONFIG__INST_CACHE_SIZE_MASK 0x00000003L
+#define SQC_CONFIG__INST_CACHE_SIZE__SHIFT 0x00000000
+#define SQC_CONFIG__MISS_FIFO_DEPTH_MASK 0x00000030L
+#define SQC_CONFIG__MISS_FIFO_DEPTH__SHIFT 0x00000004
+#define SQC_CONFIG__PER_VMID_INV_DISABLE_MASK 0x00000800L
+#define SQC_CONFIG__PER_VMID_INV_DISABLE__SHIFT 0x0000000b
+#define SQ_CONFIG__DEBUG_EN_MASK 0x00000100L
+#define SQ_CONFIG__DEBUG_EN__SHIFT 0x00000008
+#define SQ_CONFIG__DISABLE_IB_DEP_CHECK_MASK 0x00000400L
+#define SQ_CONFIG__DISABLE_IB_DEP_CHECK__SHIFT 0x0000000a
+#define SQ_CONFIG__DISABLE_SCA_BYPASS_MASK 0x00000200L
+#define SQ_CONFIG__DISABLE_SCA_BYPASS__SHIFT 0x00000009
+#define SQ_CONFIG__DUA_FLAT_LDS_PINGPONG_DISABLE_MASK 0x00008000L
+#define SQ_CONFIG__DUA_FLAT_LDS_PINGPONG_DISABLE__SHIFT 0x0000000f
+#define SQ_CONFIG__DUA_FLAT_LOCK_ENABLE_MASK 0x00002000L
+#define SQ_CONFIG__DUA_FLAT_LOCK_ENABLE__SHIFT 0x0000000d
+#define SQ_CONFIG__DUA_LDS_BYPASS_DISABLE_MASK 0x00004000L
+#define SQ_CONFIG__DUA_LDS_BYPASS_DISABLE__SHIFT 0x0000000e
+#define SQ_CONFIG__EARLY_TA_DONE_DISABLE_MASK 0x00001000L
+#define SQ_CONFIG__EARLY_TA_DONE_DISABLE__SHIFT 0x0000000c
+#define SQ_CONFIG__ENABLE_SOFT_CLAUSE_MASK 0x00000800L
+#define SQ_CONFIG__ENABLE_SOFT_CLAUSE__SHIFT 0x0000000b
+#define SQ_CONFIG__UNUSED_MASK 0x000000ffL
+#define SQ_CONFIG__UNUSED__SHIFT 0x00000000
+#define SQC_SECDED_CNT__DATA_DED_MASK 0xff000000L
+#define SQC_SECDED_CNT__DATA_DED__SHIFT 0x00000018
+#define SQC_SECDED_CNT__DATA_SEC_MASK 0x00ff0000L
+#define SQC_SECDED_CNT__DATA_SEC__SHIFT 0x00000010
+#define SQC_SECDED_CNT__INST_DED_MASK 0x0000ff00L
+#define SQC_SECDED_CNT__INST_DED__SHIFT 0x00000008
+#define SQC_SECDED_CNT__INST_SEC_MASK 0x000000ffL
+#define SQC_SECDED_CNT__INST_SEC__SHIFT 0x00000000
+#define SQ_DEBUG_CTRL_LOCAL__UNUSED_MASK 0x000000ffL
+#define SQ_DEBUG_CTRL_LOCAL__UNUSED__SHIFT 0x00000000
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX0_MASK 0x000000ffL
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX0__SHIFT 0x00000000
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX1_MASK 0x0000ff00L
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX1__SHIFT 0x00000008
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_HOST_MASK 0xff000000L
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_HOST__SHIFT 0x00000018
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_IMMED_MASK 0x00ff0000L
+#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_IMMED__SHIFT 0x00000010
+#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_CMD_MASK 0x0000000fL
+#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_CMD__SHIFT 0x00000000
+#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_REG_MASK 0x000000f0L
+#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_REG__SHIFT 0x00000004
+#define SQ_DEBUG_STS_GLOBAL__BUSY_MASK 0x00000001L
+#define SQ_DEBUG_STS_GLOBAL__BUSY__SHIFT 0x00000000
+#define SQ_DEBUG_STS_GLOBAL__INTERRUPT_MSG_BUSY_MASK 0x00000002L
+#define SQ_DEBUG_STS_GLOBAL__INTERRUPT_MSG_BUSY__SHIFT 0x00000001
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH0_MASK 0x0000fff0L
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH0__SHIFT 0x00000004
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH1_MASK 0x0fff0000L
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH1__SHIFT 0x00000010
+#define SQ_DEBUG_STS_LOCAL__BUSY_MASK 0x00000001L
+#define SQ_DEBUG_STS_LOCAL__BUSY__SHIFT 0x00000000
+#define SQ_DEBUG_STS_LOCAL__WAVE_LEVEL_MASK 0x000003f0L
+#define SQ_DEBUG_STS_LOCAL__WAVE_LEVEL__SHIFT 0x00000004
+#define SQ_DED_CNT__LDS_DED_MASK 0x0000003fL
+#define SQ_DED_CNT__LDS_DED__SHIFT 0x00000000
+#define SQ_DED_CNT__SGPR_DED_MASK 0x00001f00L
+#define SQ_DED_CNT__SGPR_DED__SHIFT 0x00000008
+#define SQ_DED_CNT__VGPR_DED_MASK 0x01ff0000L
+#define SQ_DED_CNT__VGPR_DED__SHIFT 0x00000010
+#define SQ_DED_INFO__SIMD_ID_MASK 0x00000030L
+#define SQ_DED_INFO__SIMD_ID__SHIFT 0x00000004
+#define SQ_DED_INFO__SOURCE_MASK 0x000001c0L
+#define SQ_DED_INFO__SOURCE__SHIFT 0x00000006
+#define SQ_DED_INFO__VM_ID_MASK 0x00001e00L
+#define SQ_DED_INFO__VM_ID__SHIFT 0x00000009
+#define SQ_DED_INFO__WAVE_ID_MASK 0x0000000fL
+#define SQ_DED_INFO__WAVE_ID__SHIFT 0x00000000
+#define SQ_DS_0__ENCODING_MASK 0xfc000000L
+#define SQ_DS_0__ENCODING__SHIFT 0x0000001a
+#define SQ_DS_0__GDS_MASK 0x00020000L
+#define SQ_DS_0__GDS__SHIFT 0x00000011
+#define SQ_DS_0__OFFSET0_MASK 0x000000ffL
+#define SQ_DS_0__OFFSET0__SHIFT 0x00000000
+#define SQ_DS_0__OFFSET1_MASK 0x0000ff00L
+#define SQ_DS_0__OFFSET1__SHIFT 0x00000008
+#define SQ_DS_0__OP_MASK 0x03fc0000L
+#define SQ_DS_0__OP__SHIFT 0x00000012
+#define SQ_DS_1__ADDR_MASK 0x000000ffL
+#define SQ_DS_1__ADDR__SHIFT 0x00000000
+#define SQ_DS_1__DATA0_MASK 0x0000ff00L
+#define SQ_DS_1__DATA0__SHIFT 0x00000008
+#define SQ_DS_1__DATA1_MASK 0x00ff0000L
+#define SQ_DS_1__DATA1__SHIFT 0x00000010
+#define SQ_DS_1__VDST_MASK 0xff000000L
+#define SQ_DS_1__VDST__SHIFT 0x00000018
+#define SQ_EXP_0__COMPR_MASK 0x00000400L
+#define SQ_EXP_0__COMPR__SHIFT 0x0000000a
+#define SQ_EXP_0__DONE_MASK 0x00000800L
+#define SQ_EXP_0__DONE__SHIFT 0x0000000b
+#define SQ_EXP_0__ENCODING_MASK 0xfc000000L
+#define SQ_EXP_0__ENCODING__SHIFT 0x0000001a
+#define SQ_EXP_0__EN_MASK 0x0000000fL
+#define SQ_EXP_0__EN__SHIFT 0x00000000
+#define SQ_EXP_0__TGT_MASK 0x000003f0L
+#define SQ_EXP_0__TGT__SHIFT 0x00000004
+#define SQ_EXP_0__VM_MASK 0x00001000L
+#define SQ_EXP_0__VM__SHIFT 0x0000000c
+#define SQ_EXP_1__VSRC0_MASK 0x000000ffL
+#define SQ_EXP_1__VSRC0__SHIFT 0x00000000
+#define SQ_EXP_1__VSRC1_MASK 0x0000ff00L
+#define SQ_EXP_1__VSRC1__SHIFT 0x00000008
+#define SQ_EXP_1__VSRC2_MASK 0x00ff0000L
+#define SQ_EXP_1__VSRC2__SHIFT 0x00000010
+#define SQ_EXP_1__VSRC3_MASK 0xff000000L
+#define SQ_EXP_1__VSRC3__SHIFT 0x00000018
+#define SQ_FIFO_SIZES__EXPORT_BUF_SIZE_MASK 0x00030000L
+#define SQ_FIFO_SIZES__EXPORT_BUF_SIZE__SHIFT 0x00000010
+#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE_MASK 0x0000000fL
+#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE__SHIFT 0x00000000
+#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE_MASK 0x00000f00L
+#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE__SHIFT 0x00000008
+#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE_MASK 0x000c0000L
+#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE__SHIFT 0x00000012
+#define SQ_IMG_RSRC_WORD0__BASE_ADDRESS_MASK 0xffffffffL
+#define SQ_IMG_RSRC_WORD0__BASE_ADDRESS__SHIFT 0x00000000
+#define SQ_IMG_RSRC_WORD1__BASE_ADDRESS_HI_MASK 0x000000ffL
+#define SQ_IMG_RSRC_WORD1__BASE_ADDRESS_HI__SHIFT 0x00000000
+#define SQ_IMG_RSRC_WORD1__DATA_FORMAT_MASK 0x03f00000L
+#define SQ_IMG_RSRC_WORD1__DATA_FORMAT__SHIFT 0x00000014
+#define SQ_IMG_RSRC_WORD1__MIN_LOD_MASK 0x000fff00L
+#define SQ_IMG_RSRC_WORD1__MIN_LOD__SHIFT 0x00000008
+#define SQ_IMG_RSRC_WORD1__MTYPE_MASK 0xc0000000L
+#define SQ_IMG_RSRC_WORD1__MTYPE__SHIFT 0x0000001e
+#define SQ_IMG_RSRC_WORD1__NUM_FORMAT_MASK 0x3c000000L
+#define SQ_IMG_RSRC_WORD1__NUM_FORMAT__SHIFT 0x0000001a
+#define SQ_IMG_RSRC_WORD2__HEIGHT_MASK 0x0fffc000L
+#define SQ_IMG_RSRC_WORD2__HEIGHT__SHIFT 0x0000000e
+#define SQ_IMG_RSRC_WORD2__INTERLACED_MASK 0x80000000L
+#define SQ_IMG_RSRC_WORD2__INTERLACED__SHIFT 0x0000001f
+#define SQ_IMG_RSRC_WORD2__PERF_MOD_MASK 0x70000000L
+#define SQ_IMG_RSRC_WORD2__PERF_MOD__SHIFT 0x0000001c
+#define SQ_IMG_RSRC_WORD2__WIDTH_MASK 0x00003fffL
+#define SQ_IMG_RSRC_WORD2__WIDTH__SHIFT 0x00000000
+#define SQ_IMG_RSRC_WORD3__ATC_MASK 0x08000000L
+#define SQ_IMG_RSRC_WORD3__ATC__SHIFT 0x0000001b
+#define SQ_IMG_RSRC_WORD3__BASE_LEVEL_MASK 0x0000f000L
+#define SQ_IMG_RSRC_WORD3__BASE_LEVEL__SHIFT 0x0000000c
+#define SQ_IMG_RSRC_WORD3__DST_SEL_W_MASK 0x00000e00L
+#define SQ_IMG_RSRC_WORD3__DST_SEL_W__SHIFT 0x00000009
+#define SQ_IMG_RSRC_WORD3__DST_SEL_X_MASK 0x00000007L
+#define SQ_IMG_RSRC_WORD3__DST_SEL_X__SHIFT 0x00000000
+#define SQ_IMG_RSRC_WORD3__DST_SEL_Y_MASK 0x00000038L
+#define SQ_IMG_RSRC_WORD3__DST_SEL_Y__SHIFT 0x00000003
+#define SQ_IMG_RSRC_WORD3__DST_SEL_Z_MASK 0x000001c0L
+#define SQ_IMG_RSRC_WORD3__DST_SEL_Z__SHIFT 0x00000006
+#define SQ_IMG_RSRC_WORD3__LAST_LEVEL_MASK 0x000f0000L
+#define SQ_IMG_RSRC_WORD3__LAST_LEVEL__SHIFT 0x00000010
+#define SQ_IMG_RSRC_WORD3__MTYPE_MASK 0x04000000L
+#define SQ_IMG_RSRC_WORD3__MTYPE__SHIFT 0x0000001a
+#define SQ_IMG_RSRC_WORD3__POW2_PAD_MASK 0x02000000L
+#define SQ_IMG_RSRC_WORD3__POW2_PAD__SHIFT 0x00000019
+#define SQ_IMG_RSRC_WORD3__TILING_INDEX_MASK 0x01f00000L
+#define SQ_IMG_RSRC_WORD3__TILING_INDEX__SHIFT 0x00000014
+#define SQ_IMG_RSRC_WORD3__TYPE_MASK 0xf0000000L
+#define SQ_IMG_RSRC_WORD3__TYPE__SHIFT 0x0000001c
+#define SQ_IMG_RSRC_WORD4__DEPTH_MASK 0x00001fffL
+#define SQ_IMG_RSRC_WORD4__DEPTH__SHIFT 0x00000000
+#define SQ_IMG_RSRC_WORD4__PITCH_MASK 0x07ffe000L
+#define SQ_IMG_RSRC_WORD4__PITCH__SHIFT 0x0000000d
+#define SQ_IMG_RSRC_WORD5__BASE_ARRAY_MASK 0x00001fffL
+#define SQ_IMG_RSRC_WORD5__BASE_ARRAY__SHIFT 0x00000000
+#define SQ_IMG_RSRC_WORD5__LAST_ARRAY_MASK 0x03ffe000L
+#define SQ_IMG_RSRC_WORD5__LAST_ARRAY__SHIFT 0x0000000d
+#define SQ_IMG_RSRC_WORD6__COUNTER_BANK_ID_MASK 0x000ff000L
+#define SQ_IMG_RSRC_WORD6__COUNTER_BANK_ID__SHIFT 0x0000000c
+#define SQ_IMG_RSRC_WORD6__LOD_HDW_CNT_EN_MASK 0x00100000L
+#define SQ_IMG_RSRC_WORD6__LOD_HDW_CNT_EN__SHIFT 0x00000014
+#define SQ_IMG_RSRC_WORD6__MIN_LOD_WARN_MASK 0x00000fffL
+#define SQ_IMG_RSRC_WORD6__MIN_LOD_WARN__SHIFT 0x00000000
+#define SQ_IMG_RSRC_WORD6__UNUNSED_MASK 0xffe00000L
+#define SQ_IMG_RSRC_WORD6__UNUNSED__SHIFT 0x00000015
+#define SQ_IMG_RSRC_WORD7__UNUNSED_MASK 0xffffffffL
+#define SQ_IMG_RSRC_WORD7__UNUNSED__SHIFT 0x00000000
+#define SQ_IMG_SAMP_WORD0__ANISO_BIAS_MASK 0x07e00000L
+#define SQ_IMG_SAMP_WORD0__ANISO_BIAS__SHIFT 0x00000015
+#define SQ_IMG_SAMP_WORD0__ANISO_THRESHOLD_MASK 0x00070000L
+#define SQ_IMG_SAMP_WORD0__ANISO_THRESHOLD__SHIFT 0x00000010
+#define SQ_IMG_SAMP_WORD0__CLAMP_X_MASK 0x00000007L
+#define SQ_IMG_SAMP_WORD0__CLAMP_X__SHIFT 0x00000000
+#define SQ_IMG_SAMP_WORD0__CLAMP_Y_MASK 0x00000038L
+#define SQ_IMG_SAMP_WORD0__CLAMP_Y__SHIFT 0x00000003
+#define SQ_IMG_SAMP_WORD0__CLAMP_Z_MASK 0x000001c0L
+#define SQ_IMG_SAMP_WORD0__CLAMP_Z__SHIFT 0x00000006
+#define SQ_IMG_SAMP_WORD0__DEPTH_COMPARE_FUNC_MASK 0x00007000L
+#define SQ_IMG_SAMP_WORD0__DEPTH_COMPARE_FUNC__SHIFT 0x0000000c
+#define SQ_IMG_SAMP_WORD0__DISABLE_CUBE_WRAP_MASK 0x10000000L
+#define SQ_IMG_SAMP_WORD0__DISABLE_CUBE_WRAP__SHIFT 0x0000001c
+#define SQ_IMG_SAMP_WORD0__FILTER_MODE_MASK 0x60000000L
+#define SQ_IMG_SAMP_WORD0__FILTER_MODE__SHIFT 0x0000001d
+#define SQ_IMG_SAMP_WORD0__FORCE_DEGAMMA_MASK 0x00100000L
+#define SQ_IMG_SAMP_WORD0__FORCE_DEGAMMA__SHIFT 0x00000014
+#define SQ_IMG_SAMP_WORD0__FORCE_UNNORMALIZED_MASK 0x00008000L
+#define SQ_IMG_SAMP_WORD0__FORCE_UNNORMALIZED__SHIFT 0x0000000f
+#define SQ_IMG_SAMP_WORD0__MAX_ANISO_RATIO_MASK 0x00000e00L
+#define SQ_IMG_SAMP_WORD0__MAX_ANISO_RATIO__SHIFT 0x00000009
+#define SQ_IMG_SAMP_WORD0__MC_COORD_TRUNC_MASK 0x00080000L
+#define SQ_IMG_SAMP_WORD0__MC_COORD_TRUNC__SHIFT 0x00000013
+#define SQ_IMG_SAMP_WORD0__TRUNC_COORD_MASK 0x08000000L
+#define SQ_IMG_SAMP_WORD0__TRUNC_COORD__SHIFT 0x0000001b
+#define SQ_IMG_SAMP_WORD1__MAX_LOD_MASK 0x00fff000L
+#define SQ_IMG_SAMP_WORD1__MAX_LOD__SHIFT 0x0000000c
+#define SQ_IMG_SAMP_WORD1__MIN_LOD_MASK 0x00000fffL
+#define SQ_IMG_SAMP_WORD1__MIN_LOD__SHIFT 0x00000000
+#define SQ_IMG_SAMP_WORD1__PERF_MIP_MASK 0x0f000000L
+#define SQ_IMG_SAMP_WORD1__PERF_MIP__SHIFT 0x00000018
+#define SQ_IMG_SAMP_WORD1__PERF_Z_MASK 0xf0000000L
+#define SQ_IMG_SAMP_WORD1__PERF_Z__SHIFT 0x0000001c
+#define SQ_IMG_SAMP_WORD2__DISABLE_LSB_CEIL_MASK 0x20000000L
+#define SQ_IMG_SAMP_WORD2__DISABLE_LSB_CEIL__SHIFT 0x0000001d
+#define SQ_IMG_SAMP_WORD2__FILTER_PREC_FIX_MASK 0x40000000L
+#define SQ_IMG_SAMP_WORD2__FILTER_PREC_FIX__SHIFT 0x0000001e
+#define SQ_IMG_SAMP_WORD2__LOD_BIAS_MASK 0x00003fffL
+#define SQ_IMG_SAMP_WORD2__LOD_BIAS_SEC_MASK 0x000fc000L
+#define SQ_IMG_SAMP_WORD2__LOD_BIAS_SEC__SHIFT 0x0000000e
+#define SQ_IMG_SAMP_WORD2__LOD_BIAS__SHIFT 0x00000000
+#define SQ_IMG_SAMP_WORD2__MIP_FILTER_MASK 0x0c000000L
+#define SQ_IMG_SAMP_WORD2__MIP_FILTER__SHIFT 0x0000001a
+#define SQ_IMG_SAMP_WORD2__MIP_POINT_PRECLAMP_MASK 0x10000000L
+#define SQ_IMG_SAMP_WORD2__MIP_POINT_PRECLAMP__SHIFT 0x0000001c
+#define SQ_IMG_SAMP_WORD2__XY_MAG_FILTER_MASK 0x00300000L
+#define SQ_IMG_SAMP_WORD2__XY_MAG_FILTER__SHIFT 0x00000014
+#define SQ_IMG_SAMP_WORD2__XY_MIN_FILTER_MASK 0x00c00000L
+#define SQ_IMG_SAMP_WORD2__XY_MIN_FILTER__SHIFT 0x00000016
+#define SQ_IMG_SAMP_WORD2__Z_FILTER_MASK 0x03000000L
+#define SQ_IMG_SAMP_WORD2__Z_FILTER__SHIFT 0x00000018
+#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_PTR_MASK 0x00000fffL
+#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_PTR__SHIFT 0x00000000
+#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_TYPE_MASK 0xc0000000L
+#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_TYPE__SHIFT 0x0000001e
+#define SQ_IND_DATA__DATA_MASK 0xffffffffL
+#define SQ_IND_DATA__DATA__SHIFT 0x00000000
+#define SQ_IND_INDEX__AUTO_INCR_MASK 0x00001000L
+#define SQ_IND_INDEX__AUTO_INCR__SHIFT 0x0000000c
+#define SQ_IND_INDEX__FORCE_READ_MASK 0x00002000L
+#define SQ_IND_INDEX__FORCE_READ__SHIFT 0x0000000d
+#define SQ_IND_INDEX__INDEX_MASK 0xffff0000L
+#define SQ_IND_INDEX__INDEX__SHIFT 0x00000010
+#define SQ_IND_INDEX__READ_TIMEOUT_MASK 0x00004000L
+#define SQ_IND_INDEX__READ_TIMEOUT__SHIFT 0x0000000e
+#define SQ_IND_INDEX__SIMD_ID_MASK 0x00000030L
+#define SQ_IND_INDEX__SIMD_ID__SHIFT 0x00000004
+#define SQ_IND_INDEX__THREAD_ID_MASK 0x00000fc0L
+#define SQ_IND_INDEX__THREAD_ID__SHIFT 0x00000006
+#define SQ_IND_INDEX__UNINDEXED_MASK 0x00008000L
+#define SQ_IND_INDEX__UNINDEXED__SHIFT 0x0000000f
+#define SQ_IND_INDEX__WAVE_ID_MASK 0x0000000fL
+#define SQ_IND_INDEX__WAVE_ID__SHIFT 0x00000000
+#define SQ_INST__ENCODING_MASK 0xffffffffL
+#define SQ_INST__ENCODING__SHIFT 0x00000000
+#define SQ_INTERRUPT_WORD_AUTO__CMD_TIMESTAMP_MASK 0x00000010L
+#define SQ_INTERRUPT_WORD_AUTO__CMD_TIMESTAMP__SHIFT 0x00000004
+#define SQ_INTERRUPT_WORD_AUTO__ENCODING_MASK 0x0c000000L
+#define SQ_INTERRUPT_WORD_AUTO__ENCODING__SHIFT 0x0000001a
+#define SQ_INTERRUPT_WORD_AUTO__HOST_CMD_OVERFLOW_MASK 0x00000020L
+#define SQ_INTERRUPT_WORD_AUTO__HOST_CMD_OVERFLOW__SHIFT 0x00000005
+#define SQ_INTERRUPT_WORD_AUTO__HOST_REG_OVERFLOW_MASK 0x00000040L
+#define SQ_INTERRUPT_WORD_AUTO__HOST_REG_OVERFLOW__SHIFT 0x00000006
+#define SQ_INTERRUPT_WORD_AUTO__IMMED_OVERFLOW_MASK 0x00000080L
+#define SQ_INTERRUPT_WORD_AUTO__IMMED_OVERFLOW__SHIFT 0x00000007
+#define SQ_INTERRUPT_WORD_AUTO__REG_TIMESTAMP_MASK 0x00000008L
+#define SQ_INTERRUPT_WORD_AUTO__REG_TIMESTAMP__SHIFT 0x00000003
+#define SQ_INTERRUPT_WORD_AUTO__SE_ID_MASK 0x02000000L
+#define SQ_INTERRUPT_WORD_AUTO__SE_ID__SHIFT 0x00000019
+#define SQ_INTERRUPT_WORD_AUTO__THREAD_TRACE_BUF_FULL_MASK 0x00000004L
+#define SQ_INTERRUPT_WORD_AUTO__THREAD_TRACE_BUF_FULL__SHIFT 0x00000002
+#define SQ_INTERRUPT_WORD_AUTO__THREAD_TRACE_MASK 0x00000001L
+#define SQ_INTERRUPT_WORD_AUTO__THREAD_TRACE__SHIFT 0x00000000
+#define SQ_INTERRUPT_WORD_AUTO__WLT_MASK 0x00000002L
+#define SQ_INTERRUPT_WORD_AUTO__WLT__SHIFT 0x00000001
+#define SQ_INTERRUPT_WORD_CMN__ENCODING_MASK 0x0c000000L
+#define SQ_INTERRUPT_WORD_CMN__ENCODING__SHIFT 0x0000001a
+#define SQ_INTERRUPT_WORD_CMN__SE_ID_MASK 0x02000000L
+#define SQ_INTERRUPT_WORD_CMN__SE_ID__SHIFT 0x00000019
+#define SQ_INTERRUPT_WORD_WAVE__CU_ID_MASK 0x00f00000L
+#define SQ_INTERRUPT_WORD_WAVE__CU_ID__SHIFT 0x00000014
+#define SQ_INTERRUPT_WORD_WAVE__DATA_MASK 0x000000ffL
+#define SQ_INTERRUPT_WORD_WAVE__DATA__SHIFT 0x00000000
+#define SQ_INTERRUPT_WORD_WAVE__ENCODING_MASK 0x0c000000L
+#define SQ_INTERRUPT_WORD_WAVE__ENCODING__SHIFT 0x0000001a
+#define SQ_INTERRUPT_WORD_WAVE__PRIV_MASK 0x00000200L
+#define SQ_INTERRUPT_WORD_WAVE__PRIV__SHIFT 0x00000009
+#define SQ_INTERRUPT_WORD_WAVE__SE_ID_MASK 0x02000000L
+#define SQ_INTERRUPT_WORD_WAVE__SE_ID__SHIFT 0x00000019
+#define SQ_INTERRUPT_WORD_WAVE__SH_ID_MASK 0x01000000L
+#define SQ_INTERRUPT_WORD_WAVE__SH_ID__SHIFT 0x00000018
+#define SQ_INTERRUPT_WORD_WAVE__SIMD_ID_MASK 0x000c0000L
+#define SQ_INTERRUPT_WORD_WAVE__SIMD_ID__SHIFT 0x00000012
+#define SQ_INTERRUPT_WORD_WAVE__VM_ID_MASK 0x00003c00L
+#define SQ_INTERRUPT_WORD_WAVE__VM_ID__SHIFT 0x0000000a
+#define SQ_INTERRUPT_WORD_WAVE__WAVE_ID_MASK 0x0003c000L
+#define SQ_INTERRUPT_WORD_WAVE__WAVE_ID__SHIFT 0x0000000e
+#define SQ_LB_CTR_CTRL__CLEAR_MASK 0x00000004L
+#define SQ_LB_CTR_CTRL__CLEAR__SHIFT 0x00000002
+#define SQ_LB_CTR_CTRL__LOAD_MASK 0x00000002L
+#define SQ_LB_CTR_CTRL__LOAD__SHIFT 0x00000001
+#define SQ_LB_CTR_CTRL__START_MASK 0x00000001L
+#define SQ_LB_CTR_CTRL__START__SHIFT 0x00000000
+#define SQ_LB_DATA_ALU_CYCLES__DATA_MASK 0xffffffffL
+#define SQ_LB_DATA_ALU_CYCLES__DATA__SHIFT 0x00000000
+#define SQ_LB_DATA_ALU_STALLS__DATA_MASK 0xffffffffL
+#define SQ_LB_DATA_ALU_STALLS__DATA__SHIFT 0x00000000
+#define SQ_LB_DATA_TEX_CYCLES__DATA_MASK 0xffffffffL
+#define SQ_LB_DATA_TEX_CYCLES__DATA__SHIFT 0x00000000
+#define SQ_LB_DATA_TEX_STALLS__DATA_MASK 0xffffffffL
+#define SQ_LB_DATA_TEX_STALLS__DATA__SHIFT 0x00000000
+#define SQ_MIMG_0__DA_MASK 0x00004000L
+#define SQ_MIMG_0__DA__SHIFT 0x0000000e
+#define SQ_MIMG_0__DMASK_MASK 0x00000f00L
+#define SQ_MIMG_0__DMASK__SHIFT 0x00000008
+#define SQ_MIMG_0__ENCODING_MASK 0xfc000000L
+#define SQ_MIMG_0__ENCODING__SHIFT 0x0000001a
+#define SQ_MIMG_0__GLC_MASK 0x00002000L
+#define SQ_MIMG_0__GLC__SHIFT 0x0000000d
+#define SQ_MIMG_0__LWE_MASK 0x00020000L
+#define SQ_MIMG_0__LWE__SHIFT 0x00000011
+#define SQ_MIMG_0__OP_MASK 0x01fc0000L
+#define SQ_MIMG_0__OP__SHIFT 0x00000012
+#define SQ_MIMG_0__R128_MASK 0x00008000L
+#define SQ_MIMG_0__R128__SHIFT 0x0000000f
+#define SQ_MIMG_0__SLC_MASK 0x02000000L
+#define SQ_MIMG_0__SLC__SHIFT 0x00000019
+#define SQ_MIMG_0__TFE_MASK 0x00010000L
+#define SQ_MIMG_0__TFE__SHIFT 0x00000010
+#define SQ_MIMG_0__UNORM_MASK 0x00001000L
+#define SQ_MIMG_0__UNORM__SHIFT 0x0000000c
+#define SQ_MIMG_1__SRSRC_MASK 0x001f0000L
+#define SQ_MIMG_1__SRSRC__SHIFT 0x00000010
+#define SQ_MIMG_1__SSAMP_MASK 0x03e00000L
+#define SQ_MIMG_1__SSAMP__SHIFT 0x00000015
+#define SQ_MIMG_1__VADDR_MASK 0x000000ffL
+#define SQ_MIMG_1__VADDR__SHIFT 0x00000000
+#define SQ_MIMG_1__VDATA_MASK 0x0000ff00L
+#define SQ_MIMG_1__VDATA__SHIFT 0x00000008
+#define SQ_MTBUF_0__ADDR64_MASK 0x00008000L
+#define SQ_MTBUF_0__ADDR64__SHIFT 0x0000000f
+#define SQ_MTBUF_0__DFMT_MASK 0x00780000L
+#define SQ_MTBUF_0__DFMT__SHIFT 0x00000013
+#define SQ_MTBUF_0__ENCODING_MASK 0xfc000000L
+#define SQ_MTBUF_0__ENCODING__SHIFT 0x0000001a
+#define SQ_MTBUF_0__GLC_MASK 0x00004000L
+#define SQ_MTBUF_0__GLC__SHIFT 0x0000000e
+#define SQ_MTBUF_0__IDXEN_MASK 0x00002000L
+#define SQ_MTBUF_0__IDXEN__SHIFT 0x0000000d
+#define SQ_MTBUF_0__NFMT_MASK 0x03800000L
+#define SQ_MTBUF_0__NFMT__SHIFT 0x00000017
+#define SQ_MTBUF_0__OFFEN_MASK 0x00001000L
+#define SQ_MTBUF_0__OFFEN__SHIFT 0x0000000c
+#define SQ_MTBUF_0__OFFSET_MASK 0x00000fffL
+#define SQ_MTBUF_0__OFFSET__SHIFT 0x00000000
+#define SQ_MTBUF_0__OP_MASK 0x00070000L
+#define SQ_MTBUF_0__OP__SHIFT 0x00000010
+#define SQ_MTBUF_1__SLC_MASK 0x00400000L
+#define SQ_MTBUF_1__SLC__SHIFT 0x00000016
+#define SQ_MTBUF_1__SOFFSET_MASK 0xff000000L
+#define SQ_MTBUF_1__SOFFSET__SHIFT 0x00000018
+#define SQ_MTBUF_1__SRSRC_MASK 0x001f0000L
+#define SQ_MTBUF_1__SRSRC__SHIFT 0x00000010
+#define SQ_MTBUF_1__TFE_MASK 0x00800000L
+#define SQ_MTBUF_1__TFE__SHIFT 0x00000017
+#define SQ_MTBUF_1__VADDR_MASK 0x000000ffL
+#define SQ_MTBUF_1__VADDR__SHIFT 0x00000000
+#define SQ_MTBUF_1__VDATA_MASK 0x0000ff00L
+#define SQ_MTBUF_1__VDATA__SHIFT 0x00000008
+#define SQ_MUBUF_0__ADDR64_MASK 0x00008000L
+#define SQ_MUBUF_0__ADDR64__SHIFT 0x0000000f
+#define SQ_MUBUF_0__ENCODING_MASK 0xfc000000L
+#define SQ_MUBUF_0__ENCODING__SHIFT 0x0000001a
+#define SQ_MUBUF_0__GLC_MASK 0x00004000L
+#define SQ_MUBUF_0__GLC__SHIFT 0x0000000e
+#define SQ_MUBUF_0__IDXEN_MASK 0x00002000L
+#define SQ_MUBUF_0__IDXEN__SHIFT 0x0000000d
+#define SQ_MUBUF_0__LDS_MASK 0x00010000L
+#define SQ_MUBUF_0__LDS__SHIFT 0x00000010
+#define SQ_MUBUF_0__OFFEN_MASK 0x00001000L
+#define SQ_MUBUF_0__OFFEN__SHIFT 0x0000000c
+#define SQ_MUBUF_0__OFFSET_MASK 0x00000fffL
+#define SQ_MUBUF_0__OFFSET__SHIFT 0x00000000
+#define SQ_MUBUF_0__OP_MASK 0x01fc0000L
+#define SQ_MUBUF_0__OP__SHIFT 0x00000012
+#define SQ_MUBUF_1__SLC_MASK 0x00400000L
+#define SQ_MUBUF_1__SLC__SHIFT 0x00000016
+#define SQ_MUBUF_1__SOFFSET_MASK 0xff000000L
+#define SQ_MUBUF_1__SOFFSET__SHIFT 0x00000018
+#define SQ_MUBUF_1__SRSRC_MASK 0x001f0000L
+#define SQ_MUBUF_1__SRSRC__SHIFT 0x00000010
+#define SQ_MUBUF_1__TFE_MASK 0x00800000L
+#define SQ_MUBUF_1__TFE__SHIFT 0x00000017
+#define SQ_MUBUF_1__VADDR_MASK 0x000000ffL
+#define SQ_MUBUF_1__VADDR__SHIFT 0x00000000
+#define SQ_MUBUF_1__VDATA_MASK 0x0000ff00L
+#define SQ_MUBUF_1__VDATA__SHIFT 0x00000008
+#define SQ_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER0_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER0_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER0_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER0_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER0_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER0_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER0_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER10_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER10_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER10_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER10_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER10_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER10_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER10_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER10_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER10_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER10_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER10_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER10_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER10_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER10_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER10_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER10_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER11_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER11_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER11_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER11_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER11_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER11_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER11_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER11_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER11_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER11_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER11_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER11_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER11_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER11_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER11_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER11_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER12_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER12_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER12_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER12_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER12_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER12_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER12_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER12_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER12_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER12_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER12_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER12_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER12_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER12_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER12_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER12_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER13_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER13_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER13_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER13_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER13_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER13_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER13_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER13_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER13_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER13_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER13_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER13_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER13_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER13_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER13_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER13_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER14_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER14_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER14_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER14_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER14_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER14_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER14_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER14_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER14_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER14_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER14_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER14_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER14_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER14_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER14_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER14_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER15_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER15_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER15_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER15_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER15_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER15_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER15_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER15_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER15_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER15_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER15_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER15_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER15_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER15_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER15_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER15_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER1_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER1_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER1_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER1_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER1_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER1_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER1_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER2_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER2_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER2_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER2_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER2_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER2_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER2_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER2_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER3_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER3_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER3_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER3_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER3_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER3_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER3_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER3_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER4_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER4_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER4_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER4_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER4_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER4_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER4_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER4_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER4_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER4_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER4_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER5_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER5_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER5_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER5_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER5_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER5_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER5_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER5_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER5_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER5_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER5_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER6_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER6_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER6_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER6_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER6_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER6_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER6_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER6_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER6_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER6_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER6_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER7_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER7_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER7_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER7_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER7_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER7_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER7_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER7_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER7_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER7_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER7_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER8_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER8_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER8_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER8_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER8_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER8_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER8_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER8_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER8_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER8_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER8_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER8_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER8_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER8_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER8_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER8_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER9_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SQ_PERFCOUNTER9_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SQ_PERFCOUNTER9_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SQ_PERFCOUNTER9_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SQ_PERFCOUNTER9_SELECT__PERF_MODE_MASK 0xf0000000L
+#define SQ_PERFCOUNTER9_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define SQ_PERFCOUNTER9_SELECT__PERF_SEL_MASK 0x000001ffL
+#define SQ_PERFCOUNTER9_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SQ_PERFCOUNTER9_SELECT__SIMD_MASK_MASK 0x0f000000L
+#define SQ_PERFCOUNTER9_SELECT__SIMD_MASK__SHIFT 0x00000018
+#define SQ_PERFCOUNTER9_SELECT__SPM_MODE_MASK 0x00f00000L
+#define SQ_PERFCOUNTER9_SELECT__SPM_MODE__SHIFT 0x00000014
+#define SQ_PERFCOUNTER9_SELECT__SQC_BANK_MASK_MASK 0x0000f000L
+#define SQ_PERFCOUNTER9_SELECT__SQC_BANK_MASK__SHIFT 0x0000000c
+#define SQ_PERFCOUNTER9_SELECT__SQC_CLIENT_MASK_MASK 0x000f0000L
+#define SQ_PERFCOUNTER9_SELECT__SQC_CLIENT_MASK__SHIFT 0x00000010
+#define SQ_PERFCOUNTER_CTRL2__FORCE_EN_MASK 0x00000001L
+#define SQ_PERFCOUNTER_CTRL2__FORCE_EN__SHIFT 0x00000000
+#define SQ_PERFCOUNTER_CTRL__CNTR_RATE_MASK 0x00001f00L
+#define SQ_PERFCOUNTER_CTRL__CNTR_RATE__SHIFT 0x00000008
+#define SQ_PERFCOUNTER_CTRL__CS_EN_MASK 0x00000040L
+#define SQ_PERFCOUNTER_CTRL__CS_EN__SHIFT 0x00000006
+#define SQ_PERFCOUNTER_CTRL__DISABLE_FLUSH_MASK 0x00002000L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_FLUSH__SHIFT 0x0000000d
+#define SQ_PERFCOUNTER_CTRL__ES_EN_MASK 0x00000008L
+#define SQ_PERFCOUNTER_CTRL__ES_EN__SHIFT 0x00000003
+#define SQ_PERFCOUNTER_CTRL__GS_EN_MASK 0x00000004L
+#define SQ_PERFCOUNTER_CTRL__GS_EN__SHIFT 0x00000002
+#define SQ_PERFCOUNTER_CTRL__HS_EN_MASK 0x00000010L
+#define SQ_PERFCOUNTER_CTRL__HS_EN__SHIFT 0x00000004
+#define SQ_PERFCOUNTER_CTRL__LS_EN_MASK 0x00000020L
+#define SQ_PERFCOUNTER_CTRL__LS_EN__SHIFT 0x00000005
+#define SQ_PERFCOUNTER_CTRL__PS_EN_MASK 0x00000001L
+#define SQ_PERFCOUNTER_CTRL__PS_EN__SHIFT 0x00000000
+#define SQ_PERFCOUNTER_CTRL__VS_EN_MASK 0x00000002L
+#define SQ_PERFCOUNTER_CTRL__VS_EN__SHIFT 0x00000001
+#define SQ_POWER_THROTTLE2__LONG_TERM_INTERVAL_RATIO_MASK 0x78000000L
+#define SQ_POWER_THROTTLE2__LONG_TERM_INTERVAL_RATIO__SHIFT 0x0000001b
+#define SQ_POWER_THROTTLE2__MAX_POWER_DELTA_MASK 0x00003fffL
+#define SQ_POWER_THROTTLE2__MAX_POWER_DELTA__SHIFT 0x00000000
+#define SQ_POWER_THROTTLE2__SHORT_TERM_INTERVAL_SIZE_MASK 0x03ff0000L
+#define SQ_POWER_THROTTLE2__SHORT_TERM_INTERVAL_SIZE__SHIFT 0x00000010
+#define SQ_POWER_THROTTLE2__USE_REF_CLOCK_MASK 0x80000000L
+#define SQ_POWER_THROTTLE2__USE_REF_CLOCK__SHIFT 0x0000001f
+#define SQ_POWER_THROTTLE__MAX_POWER_MASK 0x3fff0000L
+#define SQ_POWER_THROTTLE__MAX_POWER__SHIFT 0x00000010
+#define SQ_POWER_THROTTLE__MIN_POWER_MASK 0x00003fffL
+#define SQ_POWER_THROTTLE__MIN_POWER__SHIFT 0x00000000
+#define SQ_POWER_THROTTLE__PHASE_OFFSET_MASK 0xc0000000L
+#define SQ_POWER_THROTTLE__PHASE_OFFSET__SHIFT 0x0000001e
+#define SQ_RANDOM_WAVE_PRI__RET_MASK 0x0000007fL
+#define SQ_RANDOM_WAVE_PRI__RET__SHIFT 0x00000000
+#define SQ_RANDOM_WAVE_PRI__RNG_MASK 0x001ffc00L
+#define SQ_RANDOM_WAVE_PRI__RNG__SHIFT 0x0000000a
+#define SQ_RANDOM_WAVE_PRI__RUI_MASK 0x00000380L
+#define SQ_RANDOM_WAVE_PRI__RUI__SHIFT 0x00000007
+#define SQ_REG_CREDITS__CMD_CREDITS_MASK 0x00000f00L
+#define SQ_REG_CREDITS__CMD_CREDITS__SHIFT 0x00000008
+#define SQ_REG_CREDITS__CMD_OVERFLOW_MASK 0x80000000L
+#define SQ_REG_CREDITS__CMD_OVERFLOW__SHIFT 0x0000001f
+#define SQ_REG_CREDITS__IMMED_OVERFLOW_MASK 0x40000000L
+#define SQ_REG_CREDITS__IMMED_OVERFLOW__SHIFT 0x0000001e
+#define SQ_REG_CREDITS__REG_BUSY_MASK 0x10000000L
+#define SQ_REG_CREDITS__REG_BUSY__SHIFT 0x0000001c
+#define SQ_REG_CREDITS__SRBM_CREDITS_MASK 0x0000003fL
+#define SQ_REG_CREDITS__SRBM_CREDITS__SHIFT 0x00000000
+#define SQ_REG_CREDITS__SRBM_OVERFLOW_MASK 0x20000000L
+#define SQ_REG_CREDITS__SRBM_OVERFLOW__SHIFT 0x0000001d
+#define SQ_SEC_CNT__LDS_SEC_MASK 0x0000003fL
+#define SQ_SEC_CNT__LDS_SEC__SHIFT 0x00000000
+#define SQ_SEC_CNT__SGPR_SEC_MASK 0x00001f00L
+#define SQ_SEC_CNT__SGPR_SEC__SHIFT 0x00000008
+#define SQ_SEC_CNT__VGPR_SEC_MASK 0x01ff0000L
+#define SQ_SEC_CNT__VGPR_SEC__SHIFT 0x00000010
+#define SQ_SMRD__ENCODING_MASK 0xf8000000L
+#define SQ_SMRD__ENCODING__SHIFT 0x0000001b
+#define SQ_SMRD__IMM_MASK 0x00000100L
+#define SQ_SMRD__IMM__SHIFT 0x00000008
+#define SQ_SMRD__OFFSET_MASK 0x000000ffL
+#define SQ_SMRD__OFFSET__SHIFT 0x00000000
+#define SQ_SMRD__OP_MASK 0x07c00000L
+#define SQ_SMRD__OP__SHIFT 0x00000016
+#define SQ_SMRD__SBASE_MASK 0x00007e00L
+#define SQ_SMRD__SBASE__SHIFT 0x00000009
+#define SQ_SMRD__SDST_MASK 0x003f8000L
+#define SQ_SMRD__SDST__SHIFT 0x0000000f
+#define SQ_SOP1__ENCODING_MASK 0xff800000L
+#define SQ_SOP1__ENCODING__SHIFT 0x00000017
+#define SQ_SOP1__OP_MASK 0x0000ff00L
+#define SQ_SOP1__OP__SHIFT 0x00000008
+#define SQ_SOP1__SDST_MASK 0x007f0000L
+#define SQ_SOP1__SDST__SHIFT 0x00000010
+#define SQ_SOP1__SSRC0_MASK 0x000000ffL
+#define SQ_SOP1__SSRC0__SHIFT 0x00000000
+#define SQ_SOP2__ENCODING_MASK 0xc0000000L
+#define SQ_SOP2__ENCODING__SHIFT 0x0000001e
+#define SQ_SOP2__OP_MASK 0x3f800000L
+#define SQ_SOP2__OP__SHIFT 0x00000017
+#define SQ_SOP2__SDST_MASK 0x007f0000L
+#define SQ_SOP2__SDST__SHIFT 0x00000010
+#define SQ_SOP2__SSRC0_MASK 0x000000ffL
+#define SQ_SOP2__SSRC0__SHIFT 0x00000000
+#define SQ_SOP2__SSRC1_MASK 0x0000ff00L
+#define SQ_SOP2__SSRC1__SHIFT 0x00000008
+#define SQ_SOPC__ENCODING_MASK 0xff800000L
+#define SQ_SOPC__ENCODING__SHIFT 0x00000017
+#define SQ_SOPC__OP_MASK 0x007f0000L
+#define SQ_SOPC__OP__SHIFT 0x00000010
+#define SQ_SOPC__SSRC0_MASK 0x000000ffL
+#define SQ_SOPC__SSRC0__SHIFT 0x00000000
+#define SQ_SOPC__SSRC1_MASK 0x0000ff00L
+#define SQ_SOPC__SSRC1__SHIFT 0x00000008
+#define SQ_SOPK__ENCODING_MASK 0xf0000000L
+#define SQ_SOPK__ENCODING__SHIFT 0x0000001c
+#define SQ_SOPK__OP_MASK 0x0f800000L
+#define SQ_SOPK__OP__SHIFT 0x00000017
+#define SQ_SOPK__SDST_MASK 0x007f0000L
+#define SQ_SOPK__SDST__SHIFT 0x00000010
+#define SQ_SOPK__SIMM16_MASK 0x0000ffffL
+#define SQ_SOPK__SIMM16__SHIFT 0x00000000
+#define SQ_SOPP__ENCODING_MASK 0xff800000L
+#define SQ_SOPP__ENCODING__SHIFT 0x00000017
+#define SQ_SOPP__OP_MASK 0x007f0000L
+#define SQ_SOPP__OP__SHIFT 0x00000010
+#define SQ_SOPP__SIMM16_MASK 0x0000ffffL
+#define SQ_SOPP__SIMM16__SHIFT 0x00000000
+#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH0_MASK 0x0000ffffL
+#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT 0x00000000
+#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH1_MASK 0xffff0000L
+#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT 0x00000010
+#define SQ_THREAD_TRACE_BASE2__ADDR_HI_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_BASE2__ADDR_HI__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_BASE2__ATC_MASK 0x00000010L
+#define SQ_THREAD_TRACE_BASE2__ATC__SHIFT 0x00000004
+#define SQ_THREAD_TRACE_BASE__ADDR_MASK 0xffffffffL
+#define SQ_THREAD_TRACE_BASE__ADDR__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_CNTR__CNTR_MASK 0xffffffffL
+#define SQ_THREAD_TRACE_CNTR__CNTR__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_CTRL__RESET_BUFFER_MASK 0x80000000L
+#define SQ_THREAD_TRACE_CTRL__RESET_BUFFER__SHIFT 0x0000001f
+#define SQ_THREAD_TRACE_HIWATER__HIWATER_MASK 0x00000007L
+#define SQ_THREAD_TRACE_HIWATER__HIWATER__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_MASK__CU_SEL_MASK 0x0000001fL
+#define SQ_THREAD_TRACE_MASK__CU_SEL__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_MASK__RANDOM_SEED_MASK 0xffff0000L
+#define SQ_THREAD_TRACE_MASK__RANDOM_SEED__SHIFT 0x00000010
+#define SQ_THREAD_TRACE_MASK__REG_STALL_EN_MASK 0x00000080L
+#define SQ_THREAD_TRACE_MASK__REG_STALL_EN__SHIFT 0x00000007
+#define SQ_THREAD_TRACE_MASK__SH_SEL_MASK 0x00000020L
+#define SQ_THREAD_TRACE_MASK__SH_SEL__SHIFT 0x00000005
+#define SQ_THREAD_TRACE_MASK__SPI_STALL_EN_MASK 0x00004000L
+#define SQ_THREAD_TRACE_MASK__SPI_STALL_EN__SHIFT 0x0000000e
+#define SQ_THREAD_TRACE_MASK__SQ_STALL_EN_MASK 0x00008000L
+#define SQ_THREAD_TRACE_MASK__SQ_STALL_EN__SHIFT 0x0000000f
+#define SQ_THREAD_TRACE_MASK__VM_ID_MASK_MASK 0x00003000L
+#define SQ_THREAD_TRACE_MASK__VM_ID_MASK__SHIFT 0x0000000c
+#define SQ_THREAD_TRACE_MODE__AUTOFLUSH_EN_MASK 0x02000000L
+#define SQ_THREAD_TRACE_MODE__AUTOFLUSH_EN__SHIFT 0x00000019
+#define SQ_THREAD_TRACE_MODE__CAPTURE_MODE_MASK 0x01800000L
+#define SQ_THREAD_TRACE_MODE__CAPTURE_MODE__SHIFT 0x00000017
+#define SQ_THREAD_TRACE_MODE__INTERRUPT_EN_MASK 0x40000000L
+#define SQ_THREAD_TRACE_MODE__INTERRUPT_EN__SHIFT 0x0000001e
+#define SQ_THREAD_TRACE_MODE__ISSUE_MASK_MASK 0x18000000L
+#define SQ_THREAD_TRACE_MODE__ISSUE_MASK__SHIFT 0x0000001b
+#define SQ_THREAD_TRACE_MODE__MASK_CS_MASK 0x001c0000L
+#define SQ_THREAD_TRACE_MODE__MASK_CS__SHIFT 0x00000012
+#define SQ_THREAD_TRACE_MODE__MASK_ES_MASK 0x00000e00L
+#define SQ_THREAD_TRACE_MODE__MASK_ES__SHIFT 0x00000009
+#define SQ_THREAD_TRACE_MODE__MASK_GS_MASK 0x000001c0L
+#define SQ_THREAD_TRACE_MODE__MASK_GS__SHIFT 0x00000006
+#define SQ_THREAD_TRACE_MODE__MASK_HS_MASK 0x00007000L
+#define SQ_THREAD_TRACE_MODE__MASK_HS__SHIFT 0x0000000c
+#define SQ_THREAD_TRACE_MODE__MASK_LS_MASK 0x00038000L
+#define SQ_THREAD_TRACE_MODE__MASK_LS__SHIFT 0x0000000f
+#define SQ_THREAD_TRACE_MODE__MASK_PS_MASK 0x00000007L
+#define SQ_THREAD_TRACE_MODE__MASK_PS__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_MODE__MASK_VS_MASK 0x00000038L
+#define SQ_THREAD_TRACE_MODE__MASK_VS__SHIFT 0x00000003
+#define SQ_THREAD_TRACE_MODE__MODE_MASK 0x00600000L
+#define SQ_THREAD_TRACE_MODE__MODE__SHIFT 0x00000015
+#define SQ_THREAD_TRACE_MODE__PRIV_MASK 0x04000000L
+#define SQ_THREAD_TRACE_MODE__PRIV__SHIFT 0x0000001a
+#define SQ_THREAD_TRACE_MODE__TEST_MODE_MASK 0x20000000L
+#define SQ_THREAD_TRACE_MODE__TEST_MODE__SHIFT 0x0000001d
+#define SQ_THREAD_TRACE_MODE__WRAP_MASK 0x80000000L
+#define SQ_THREAD_TRACE_MODE__WRAP__SHIFT 0x0000001f
+#define SQ_THREAD_TRACE_PERF_MASK__SH0_MASK_MASK 0x0000ffffL
+#define SQ_THREAD_TRACE_PERF_MASK__SH0_MASK__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_PERF_MASK__SH1_MASK_MASK 0xffff0000L
+#define SQ_THREAD_TRACE_PERF_MASK__SH1_MASK__SHIFT 0x00000010
+#define SQ_THREAD_TRACE_SIZE__SIZE_MASK 0x003fffffL
+#define SQ_THREAD_TRACE_SIZE__SIZE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_STATUS__BUSY_MASK 0x40000000L
+#define SQ_THREAD_TRACE_STATUS__BUSY__SHIFT 0x0000001e
+#define SQ_THREAD_TRACE_STATUS__FINISH_DONE_MASK 0x00070000L
+#define SQ_THREAD_TRACE_STATUS__FINISH_DONE__SHIFT 0x00000010
+#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING_MASK 0x00000007L
+#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_STATUS__FULL_MASK 0x80000000L
+#define SQ_THREAD_TRACE_STATUS__FULL__SHIFT 0x0000001f
+#define SQ_THREAD_TRACE_STATUS__NEW_BUF_MASK 0x20000000L
+#define SQ_THREAD_TRACE_STATUS__NEW_BUF__SHIFT 0x0000001d
+#define SQ_THREAD_TRACE_TOKEN_MASK2__INST_MASK_MASK 0x0000ffffL
+#define SQ_THREAD_TRACE_TOKEN_MASK2__INST_MASK__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DROP_ON_STALL_MASK 0x01000000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DROP_ON_STALL__SHIFT 0x00000018
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_MASK_MASK 0x00ff0000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_MASK__SHIFT 0x00000010
+#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_MASK_MASK 0x0000ffffL
+#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_MASK__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_USERDATA_0__DATA_MASK 0xffffffffL
+#define SQ_THREAD_TRACE_USERDATA_0__DATA__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_USERDATA_1__DATA_MASK 0xffffffffL
+#define SQ_THREAD_TRACE_USERDATA_1__DATA__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_USERDATA_2__DATA_MASK 0xffffffffL
+#define SQ_THREAD_TRACE_USERDATA_2__DATA__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_USERDATA_3__DATA_MASK 0xffffffffL
+#define SQ_THREAD_TRACE_USERDATA_3__DATA__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_CMN__TIME_DELTA_MASK 0x00000010L
+#define SQ_THREAD_TRACE_WORD_CMN__TIME_DELTA__SHIFT 0x00000004
+#define SQ_THREAD_TRACE_WORD_CMN__TOKEN_TYPE_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_WORD_CMN__TOKEN_TYPE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_EVENT__EVENT_TYPE_MASK 0x0000fc00L
+#define SQ_THREAD_TRACE_WORD_EVENT__EVENT_TYPE__SHIFT 0x0000000a
+#define SQ_THREAD_TRACE_WORD_EVENT__SH_ID_MASK 0x00000020L
+#define SQ_THREAD_TRACE_WORD_EVENT__SH_ID__SHIFT 0x00000005
+#define SQ_THREAD_TRACE_WORD_EVENT__STAGE_MASK 0x000001c0L
+#define SQ_THREAD_TRACE_WORD_EVENT__STAGE__SHIFT 0x00000006
+#define SQ_THREAD_TRACE_WORD_EVENT__TIME_DELTA_MASK 0x00000010L
+#define SQ_THREAD_TRACE_WORD_EVENT__TIME_DELTA__SHIFT 0x00000004
+#define SQ_THREAD_TRACE_WORD_EVENT__TOKEN_TYPE_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_WORD_EVENT__TOKEN_TYPE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_INST__INST_TYPE_MASK 0x0000f000L
+#define SQ_THREAD_TRACE_WORD_INST__INST_TYPE__SHIFT 0x0000000c
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__PC_LO_MASK 0xffff0000L
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__PC_LO__SHIFT 0x00000010
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__SIMD_ID_MASK 0x00000600L
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__SIMD_ID__SHIFT 0x00000009
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TIME_DELTA_MASK 0x00000010L
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TIME_DELTA__SHIFT 0x00000004
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TOKEN_TYPE_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TOKEN_TYPE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__WAVE_ID_MASK 0x000001e0L
+#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__WAVE_ID__SHIFT 0x00000005
+#define SQ_THREAD_TRACE_WORD_INST_PC_2_OF_2__PC_HI_MASK 0x00ffffffL
+#define SQ_THREAD_TRACE_WORD_INST_PC_2_OF_2__PC_HI__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_INST__SIMD_ID_MASK 0x00000600L
+#define SQ_THREAD_TRACE_WORD_INST__SIMD_ID__SHIFT 0x00000009
+#define SQ_THREAD_TRACE_WORD_INST__SIZE_MASK 0x00000800L
+#define SQ_THREAD_TRACE_WORD_INST__SIZE__SHIFT 0x0000000b
+#define SQ_THREAD_TRACE_WORD_INST__TIME_DELTA_MASK 0x00000010L
+#define SQ_THREAD_TRACE_WORD_INST__TIME_DELTA__SHIFT 0x00000004
+#define SQ_THREAD_TRACE_WORD_INST__TOKEN_TYPE_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_WORD_INST__TOKEN_TYPE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__CU_ID_MASK 0x000003c0L
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__CU_ID__SHIFT 0x00000006
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__DATA_LO_MASK 0xffff0000L
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__DATA_LO__SHIFT 0x00000010
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SH_ID_MASK 0x00000020L
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SH_ID__SHIFT 0x00000005
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SIMD_ID_MASK 0x0000c000L
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SIMD_ID__SHIFT 0x0000000e
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TIME_DELTA_MASK 0x00000010L
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TIME_DELTA__SHIFT 0x00000004
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TOKEN_TYPE_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TOKEN_TYPE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__WAVE_ID_MASK 0x00003c00L
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__WAVE_ID__SHIFT 0x0000000a
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2__DATA_HI_MASK 0x0000ffffL
+#define SQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2__DATA_HI__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_INST__WAVE_ID_MASK 0x000001e0L
+#define SQ_THREAD_TRACE_WORD_INST__WAVE_ID__SHIFT 0x00000005
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST0_MASK 0x00000300L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST0__SHIFT 0x00000008
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST1_MASK 0x00000c00L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST1__SHIFT 0x0000000a
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST2_MASK 0x00003000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST2__SHIFT 0x0000000c
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST3_MASK 0x0000c000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST3__SHIFT 0x0000000e
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST4_MASK 0x00030000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST4__SHIFT 0x00000010
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST5_MASK 0x000c0000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST5__SHIFT 0x00000012
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST6_MASK 0x00300000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST6__SHIFT 0x00000014
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST7_MASK 0x00c00000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST7__SHIFT 0x00000016
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST8_MASK 0x03000000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST8__SHIFT 0x00000018
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST9_MASK 0x0c000000L
+#define SQ_THREAD_TRACE_WORD_ISSUE__INST9__SHIFT 0x0000001a
+#define SQ_THREAD_TRACE_WORD_ISSUE__SIMD_ID_MASK 0x00000060L
+#define SQ_THREAD_TRACE_WORD_ISSUE__SIMD_ID__SHIFT 0x00000005
+#define SQ_THREAD_TRACE_WORD_ISSUE__TIME_DELTA_MASK 0x00000010L
+#define SQ_THREAD_TRACE_WORD_ISSUE__TIME_DELTA__SHIFT 0x00000004
+#define SQ_THREAD_TRACE_WORD_ISSUE__TOKEN_TYPE_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_WORD_ISSUE__TOKEN_TYPE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_MISC__MISC_TOKEN_TYPE_MASK 0x000000c0L
+#define SQ_THREAD_TRACE_WORD_MISC__MISC_TOKEN_TYPE__SHIFT 0x00000006
+#define SQ_THREAD_TRACE_WORD_MISC__SH_ID_MASK 0x00000020L
+#define SQ_THREAD_TRACE_WORD_MISC__SH_ID__SHIFT 0x00000005
+#define SQ_THREAD_TRACE_WORD_MISC__TIME_DELTA_MASK 0x00000010L
+#define SQ_THREAD_TRACE_WORD_MISC__TIME_DELTA__SHIFT 0x00000004
+#define SQ_THREAD_TRACE_WORD_MISC__TOKEN_TYPE_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_WORD_MISC__TOKEN_TYPE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR0_MASK 0x01fff000L
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR0__SHIFT 0x0000000c
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR1_LO_MASK 0xfe000000L
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR1_LO__SHIFT 0x00000019
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR_BANK_MASK 0x00000c00L
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR_BANK__SHIFT 0x0000000a
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CU_ID_MASK 0x000003c0L
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CU_ID__SHIFT 0x00000006
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__SH_ID_MASK 0x00000020L
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__SH_ID__SHIFT 0x00000005
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TIME_DELTA_MASK 0x00000010L
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TIME_DELTA__SHIFT 0x00000004
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TOKEN_TYPE_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TOKEN_TYPE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR1_HI_MASK 0x0000003fL
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR1_HI__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR2_MASK 0x0007ffc0L
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR2__SHIFT 0x00000006
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR3_MASK 0xfff80000L
+#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR3__SHIFT 0x00000013
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__ME_ID_MASK 0x00000180L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__ME_ID__SHIFT 0x00000007
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__PIPE_ID_MASK 0x00000060L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__PIPE_ID__SHIFT 0x00000005
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_ADDR_MASK 0xffff0000L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_ADDR__SHIFT 0x00000010
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_DROPPED_PREV_MASK 0x00000200L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_DROPPED_PREV__SHIFT 0x00000009
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_OP_MASK 0x00008000L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_OP__SHIFT 0x0000000f
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_PRIV_MASK 0x00004000L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_PRIV__SHIFT 0x0000000e
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_TYPE_MASK 0x00001c00L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_TYPE__SHIFT 0x0000000a
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TIME_DELTA_MASK 0x00000010L
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TIME_DELTA__SHIFT 0x00000004
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TOKEN_TYPE_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TOKEN_TYPE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_REG_2_OF_2__DATA_MASK 0xffffffffL
+#define SQ_THREAD_TRACE_WORD_REG_2_OF_2__DATA__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TIME_LO_MASK 0xffff0000L
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TIME_LO__SHIFT 0x00000010
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TOKEN_TYPE_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TOKEN_TYPE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2__TIME_HI_MASK 0xffffffffL
+#define SQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2__TIME_HI__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_WAVE__CU_ID_MASK 0x000003c0L
+#define SQ_THREAD_TRACE_WORD_WAVE__CU_ID__SHIFT 0x00000006
+#define SQ_THREAD_TRACE_WORD_WAVE__SH_ID_MASK 0x00000020L
+#define SQ_THREAD_TRACE_WORD_WAVE__SH_ID__SHIFT 0x00000005
+#define SQ_THREAD_TRACE_WORD_WAVE__SIMD_ID_MASK 0x0000c000L
+#define SQ_THREAD_TRACE_WORD_WAVE__SIMD_ID__SHIFT 0x0000000e
+#define SQ_THREAD_TRACE_WORD_WAVE_START__COUNT_MASK 0x1fc00000L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__COUNT__SHIFT 0x00000016
+#define SQ_THREAD_TRACE_WORD_WAVE_START__CU_ID_MASK 0x000003c0L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__CU_ID__SHIFT 0x00000006
+#define SQ_THREAD_TRACE_WORD_WAVE_START__DISPATCHER_MASK 0x001f0000L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__DISPATCHER__SHIFT 0x00000010
+#define SQ_THREAD_TRACE_WORD_WAVE_START__SH_ID_MASK 0x00000020L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__SH_ID__SHIFT 0x00000005
+#define SQ_THREAD_TRACE_WORD_WAVE_START__SIMD_ID_MASK 0x0000c000L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__SIMD_ID__SHIFT 0x0000000e
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TG_ID_MASK 0xe0000000L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TG_ID__SHIFT 0x0000001d
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TIME_DELTA_MASK 0x00000010L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TIME_DELTA__SHIFT 0x00000004
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TOKEN_TYPE_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_WORD_WAVE_START__TOKEN_TYPE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_WAVE_START__VS_NO_ALLOC_OR_GROUPED_MASK 0x00200000L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__VS_NO_ALLOC_OR_GROUPED__SHIFT 0x00000015
+#define SQ_THREAD_TRACE_WORD_WAVE_START__WAVE_ID_MASK 0x00003c00L
+#define SQ_THREAD_TRACE_WORD_WAVE_START__WAVE_ID__SHIFT 0x0000000a
+#define SQ_THREAD_TRACE_WORD_WAVE__TIME_DELTA_MASK 0x00000010L
+#define SQ_THREAD_TRACE_WORD_WAVE__TIME_DELTA__SHIFT 0x00000004
+#define SQ_THREAD_TRACE_WORD_WAVE__TOKEN_TYPE_MASK 0x0000000fL
+#define SQ_THREAD_TRACE_WORD_WAVE__TOKEN_TYPE__SHIFT 0x00000000
+#define SQ_THREAD_TRACE_WORD_WAVE__WAVE_ID_MASK 0x00003c00L
+#define SQ_THREAD_TRACE_WORD_WAVE__WAVE_ID__SHIFT 0x0000000a
+#define SQ_THREAD_TRACE_WPTR__READ_OFFSET_MASK 0xc0000000L
+#define SQ_THREAD_TRACE_WPTR__READ_OFFSET__SHIFT 0x0000001e
+#define SQ_THREAD_TRACE_WPTR__WPTR_MASK 0x3fffffffL
+#define SQ_THREAD_TRACE_WPTR__WPTR__SHIFT 0x00000000
+#define SQ_TIME_HI__TIME_MASK 0xffffffffL
+#define SQ_TIME_HI__TIME__SHIFT 0x00000000
+#define SQ_TIME_LO__TIME_MASK 0xffffffffL
+#define SQ_TIME_LO__TIME__SHIFT 0x00000000
+#define SQ_VINTRP__ATTRCHAN_MASK 0x00000300L
+#define SQ_VINTRP__ATTRCHAN__SHIFT 0x00000008
+#define SQ_VINTRP__ATTR_MASK 0x0000fc00L
+#define SQ_VINTRP__ATTR__SHIFT 0x0000000a
+#define SQ_VINTRP__ENCODING_MASK 0xfc000000L
+#define SQ_VINTRP__ENCODING__SHIFT 0x0000001a
+#define SQ_VINTRP__OP_MASK 0x00030000L
+#define SQ_VINTRP__OP__SHIFT 0x00000010
+#define SQ_VINTRP__VDST_MASK 0x03fc0000L
+#define SQ_VINTRP__VDST__SHIFT 0x00000012
+#define SQ_VINTRP__VSRC_MASK 0x000000ffL
+#define SQ_VINTRP__VSRC__SHIFT 0x00000000
+#define SQ_VOP1__ENCODING_MASK 0xfe000000L
+#define SQ_VOP1__ENCODING__SHIFT 0x00000019
+#define SQ_VOP1__OP_MASK 0x0001fe00L
+#define SQ_VOP1__OP__SHIFT 0x00000009
+#define SQ_VOP1__SRC0_MASK 0x000001ffL
+#define SQ_VOP1__SRC0__SHIFT 0x00000000
+#define SQ_VOP1__VDST_MASK 0x01fe0000L
+#define SQ_VOP1__VDST__SHIFT 0x00000011
+#define SQ_VOP2__ENCODING_MASK 0x80000000L
+#define SQ_VOP2__ENCODING__SHIFT 0x0000001f
+#define SQ_VOP2__OP_MASK 0x7e000000L
+#define SQ_VOP2__OP__SHIFT 0x00000019
+#define SQ_VOP2__SRC0_MASK 0x000001ffL
+#define SQ_VOP2__SRC0__SHIFT 0x00000000
+#define SQ_VOP2__VDST_MASK 0x01fe0000L
+#define SQ_VOP2__VDST__SHIFT 0x00000011
+#define SQ_VOP2__VSRC1_MASK 0x0001fe00L
+#define SQ_VOP2__VSRC1__SHIFT 0x00000009
+#define SQ_VOP3_0__ABS_MASK 0x00000700L
+#define SQ_VOP3_0__ABS__SHIFT 0x00000008
+#define SQ_VOP3_0__CLAMP_MASK 0x00000800L
+#define SQ_VOP3_0__CLAMP__SHIFT 0x0000000b
+#define SQ_VOP3_0__ENCODING_MASK 0xfc000000L
+#define SQ_VOP3_0__ENCODING__SHIFT 0x0000001a
+#define SQ_VOP3_0__OP_MASK 0x03fe0000L
+#define SQ_VOP3_0__OP__SHIFT 0x00000011
+#define SQ_VOP3_0_SDST_ENC__ENCODING_MASK 0xfc000000L
+#define SQ_VOP3_0_SDST_ENC__ENCODING__SHIFT 0x0000001a
+#define SQ_VOP3_0_SDST_ENC__OP_MASK 0x03fe0000L
+#define SQ_VOP3_0_SDST_ENC__OP__SHIFT 0x00000011
+#define SQ_VOP3_0_SDST_ENC__SDST_MASK 0x00007f00L
+#define SQ_VOP3_0_SDST_ENC__SDST__SHIFT 0x00000008
+#define SQ_VOP3_0_SDST_ENC__VDST_MASK 0x000000ffL
+#define SQ_VOP3_0_SDST_ENC__VDST__SHIFT 0x00000000
+#define SQ_VOP3_0__VDST_MASK 0x000000ffL
+#define SQ_VOP3_0__VDST__SHIFT 0x00000000
+#define SQ_VOP3_1__NEG_MASK 0xe0000000L
+#define SQ_VOP3_1__NEG__SHIFT 0x0000001d
+#define SQ_VOP3_1__OMOD_MASK 0x18000000L
+#define SQ_VOP3_1__OMOD__SHIFT 0x0000001b
+#define SQ_VOP3_1__SRC0_MASK 0x000001ffL
+#define SQ_VOP3_1__SRC0__SHIFT 0x00000000
+#define SQ_VOP3_1__SRC1_MASK 0x0003fe00L
+#define SQ_VOP3_1__SRC1__SHIFT 0x00000009
+#define SQ_VOP3_1__SRC2_MASK 0x07fc0000L
+#define SQ_VOP3_1__SRC2__SHIFT 0x00000012
+#define SQ_VOPC__ENCODING_MASK 0xfe000000L
+#define SQ_VOPC__ENCODING__SHIFT 0x00000019
+#define SQ_VOPC__OP_MASK 0x01fe0000L
+#define SQ_VOPC__OP__SHIFT 0x00000011
+#define SQ_VOPC__SRC0_MASK 0x000001ffL
+#define SQ_VOPC__SRC0__SHIFT 0x00000000
+#define SQ_VOPC__VSRC1_MASK 0x0001fe00L
+#define SQ_VOPC__VSRC1__SHIFT 0x00000009
+#define SQ_WAVE_EXEC_HI__EXEC_HI_MASK 0xffffffffL
+#define SQ_WAVE_EXEC_HI__EXEC_HI__SHIFT 0x00000000
+#define SQ_WAVE_EXEC_LO__EXEC_LO_MASK 0xffffffffL
+#define SQ_WAVE_EXEC_LO__EXEC_LO__SHIFT 0x00000000
+#define SQ_WAVE_GPR_ALLOC__SGPR_BASE_MASK 0x003f0000L
+#define SQ_WAVE_GPR_ALLOC__SGPR_BASE__SHIFT 0x00000010
+#define SQ_WAVE_GPR_ALLOC__SGPR_SIZE_MASK 0x0f000000L
+#define SQ_WAVE_GPR_ALLOC__SGPR_SIZE__SHIFT 0x00000018
+#define SQ_WAVE_GPR_ALLOC__VGPR_BASE_MASK 0x0000003fL
+#define SQ_WAVE_GPR_ALLOC__VGPR_BASE__SHIFT 0x00000000
+#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE_MASK 0x00003f00L
+#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE__SHIFT 0x00000008
+#define SQ_WAVE_HW_ID__CU_ID_MASK 0x00000f00L
+#define SQ_WAVE_HW_ID__CU_ID__SHIFT 0x00000008
+#define SQ_WAVE_HW_ID__ME_ID_MASK 0xc0000000L
+#define SQ_WAVE_HW_ID__ME_ID__SHIFT 0x0000001e
+#define SQ_WAVE_HW_ID__PIPE_ID_MASK 0x000000c0L
+#define SQ_WAVE_HW_ID__PIPE_ID__SHIFT 0x00000006
+#define SQ_WAVE_HW_ID__QUEUE_ID_MASK 0x07000000L
+#define SQ_WAVE_HW_ID__QUEUE_ID__SHIFT 0x00000018
+#define SQ_WAVE_HW_ID__SE_ID_MASK 0x00002000L
+#define SQ_WAVE_HW_ID__SE_ID__SHIFT 0x0000000d
+#define SQ_WAVE_HW_ID__SH_ID_MASK 0x00001000L
+#define SQ_WAVE_HW_ID__SH_ID__SHIFT 0x0000000c
+#define SQ_WAVE_HW_ID__SIMD_ID_MASK 0x00000030L
+#define SQ_WAVE_HW_ID__SIMD_ID__SHIFT 0x00000004
+#define SQ_WAVE_HW_ID__STATE_ID_MASK 0x38000000L
+#define SQ_WAVE_HW_ID__STATE_ID__SHIFT 0x0000001b
+#define SQ_WAVE_HW_ID__TG_ID_MASK 0x000f0000L
+#define SQ_WAVE_HW_ID__TG_ID__SHIFT 0x00000010
+#define SQ_WAVE_HW_ID__VM_ID_MASK 0x00f00000L
+#define SQ_WAVE_HW_ID__VM_ID__SHIFT 0x00000014
+#define SQ_WAVE_HW_ID__WAVE_ID_MASK 0x0000000fL
+#define SQ_WAVE_HW_ID__WAVE_ID__SHIFT 0x00000000
+#define SQ_WAVE_IB_DBG0__ECC_ST_MASK 0x00c00000L
+#define SQ_WAVE_IB_DBG0__ECC_ST__SHIFT 0x00000016
+#define SQ_WAVE_IB_DBG0__HYB_CNT_MASK 0x06000000L
+#define SQ_WAVE_IB_DBG0__HYB_CNT__SHIFT 0x00000019
+#define SQ_WAVE_IB_DBG0__IBUF_RPTR_MASK 0x00000300L
+#define SQ_WAVE_IB_DBG0__IBUF_RPTR__SHIFT 0x00000008
+#define SQ_WAVE_IB_DBG0__IBUF_ST_MASK 0x00000007L
+#define SQ_WAVE_IB_DBG0__IBUF_ST__SHIFT 0x00000000
+#define SQ_WAVE_IB_DBG0__IBUF_WPTR_MASK 0x00000c00L
+#define SQ_WAVE_IB_DBG0__IBUF_WPTR__SHIFT 0x0000000a
+#define SQ_WAVE_IB_DBG0__INST_STR_ST_MASK 0x00070000L
+#define SQ_WAVE_IB_DBG0__INST_STR_ST__SHIFT 0x00000010
+#define SQ_WAVE_IB_DBG0__IS_HYB_MASK 0x01000000L
+#define SQ_WAVE_IB_DBG0__IS_HYB__SHIFT 0x00000018
+#define SQ_WAVE_IB_DBG0__KILL_MASK 0x08000000L
+#define SQ_WAVE_IB_DBG0__KILL__SHIFT 0x0000001b
+#define SQ_WAVE_IB_DBG0__MISC_CNT_MASK 0x00380000L
+#define SQ_WAVE_IB_DBG0__MISC_CNT__SHIFT 0x00000013
+#define SQ_WAVE_IB_DBG0__NEED_KILL_IFETCH_MASK 0x10000000L
+#define SQ_WAVE_IB_DBG0__NEED_KILL_IFETCH__SHIFT 0x0000001c
+#define SQ_WAVE_IB_DBG0__NEED_NEXT_DW_MASK 0x00000010L
+#define SQ_WAVE_IB_DBG0__NEED_NEXT_DW__SHIFT 0x00000004
+#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT_MASK 0x000000e0L
+#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT__SHIFT 0x00000005
+#define SQ_WAVE_IB_DBG0__PC_INVALID_MASK 0x00000008L
+#define SQ_WAVE_IB_DBG0__PC_INVALID__SHIFT 0x00000003
+#define SQ_WAVE_IB_STS__EXP_CNT_MASK 0x00000070L
+#define SQ_WAVE_IB_STS__EXP_CNT__SHIFT 0x00000004
+#define SQ_WAVE_IB_STS__LGKM_CNT_MASK 0x00001f00L
+#define SQ_WAVE_IB_STS__LGKM_CNT__SHIFT 0x00000008
+#define SQ_WAVE_IB_STS__VALU_CNT_MASK 0x0000e000L
+#define SQ_WAVE_IB_STS__VALU_CNT__SHIFT 0x0000000d
+#define SQ_WAVE_IB_STS__VM_CNT_MASK 0x0000000fL
+#define SQ_WAVE_IB_STS__VM_CNT__SHIFT 0x00000000
+#define SQ_WAVE_INST_DW0__INST_DW0_MASK 0xffffffffL
+#define SQ_WAVE_INST_DW0__INST_DW0__SHIFT 0x00000000
+#define SQ_WAVE_INST_DW1__INST_DW1_MASK 0xffffffffL
+#define SQ_WAVE_INST_DW1__INST_DW1__SHIFT 0x00000000
+#define SQ_WAVE_LDS_ALLOC__LDS_BASE_MASK 0x000000ffL
+#define SQ_WAVE_LDS_ALLOC__LDS_BASE__SHIFT 0x00000000
+#define SQ_WAVE_LDS_ALLOC__LDS_SIZE_MASK 0x001ff000L
+#define SQ_WAVE_LDS_ALLOC__LDS_SIZE__SHIFT 0x0000000c
+#define SQ_WAVE_M0__M0_MASK 0xffffffffL
+#define SQ_WAVE_M0__M0__SHIFT 0x00000000
+#define SQ_WAVE_MODE__CSP_MASK 0xe0000000L
+#define SQ_WAVE_MODE__CSP__SHIFT 0x0000001d
+#define SQ_WAVE_MODE__DEBUG_EN_MASK 0x00000800L
+#define SQ_WAVE_MODE__DEBUG_EN__SHIFT 0x0000000b
+#define SQ_WAVE_MODE__DX10_CLAMP_MASK 0x00000100L
+#define SQ_WAVE_MODE__DX10_CLAMP__SHIFT 0x00000008
+#define SQ_WAVE_MODE__EXCP_EN_MASK 0x0007f000L
+#define SQ_WAVE_MODE__EXCP_EN__SHIFT 0x0000000c
+#define SQ_WAVE_MODE__FP_DENORM_MASK 0x000000f0L
+#define SQ_WAVE_MODE__FP_DENORM__SHIFT 0x00000004
+#define SQ_WAVE_MODE__FP_ROUND_MASK 0x0000000fL
+#define SQ_WAVE_MODE__FP_ROUND__SHIFT 0x00000000
+#define SQ_WAVE_MODE__IEEE_MASK 0x00000200L
+#define SQ_WAVE_MODE__IEEE__SHIFT 0x00000009
+#define SQ_WAVE_MODE__LOD_CLAMPED_MASK 0x00000400L
+#define SQ_WAVE_MODE__LOD_CLAMPED__SHIFT 0x0000000a
+#define SQ_WAVE_MODE__VSKIP_MASK 0x10000000L
+#define SQ_WAVE_MODE__VSKIP__SHIFT 0x0000001c
+#define SQ_WAVE_PC_HI__PC_HI_MASK 0x000000ffL
+#define SQ_WAVE_PC_HI__PC_HI__SHIFT 0x00000000
+#define SQ_WAVE_PC_LO__PC_LO_MASK 0xffffffffL
+#define SQ_WAVE_PC_LO__PC_LO__SHIFT 0x00000000
+#define SQ_WAVE_STATUS__COND_DBG_SYS_MASK 0x00200000L
+#define SQ_WAVE_STATUS__COND_DBG_SYS__SHIFT 0x00000015
+#define SQ_WAVE_STATUS__COND_DBG_USER_MASK 0x00100000L
+#define SQ_WAVE_STATUS__COND_DBG_USER__SHIFT 0x00000014
+#define SQ_WAVE_STATUS__DATA_ATC_MASK 0x00400000L
+#define SQ_WAVE_STATUS__DATA_ATC__SHIFT 0x00000016
+#define SQ_WAVE_STATUS__DISPATCH_CACHE_CTRL_MASK 0x07000000L
+#define SQ_WAVE_STATUS__DISPATCH_CACHE_CTRL__SHIFT 0x00000018
+#define SQ_WAVE_STATUS__ECC_ERR_MASK 0x00020000L
+#define SQ_WAVE_STATUS__ECC_ERR__SHIFT 0x00000011
+#define SQ_WAVE_STATUS__EXECZ_MASK 0x00000200L
+#define SQ_WAVE_STATUS__EXECZ__SHIFT 0x00000009
+#define SQ_WAVE_STATUS__EXPORT_RDY_MASK 0x00000100L
+#define SQ_WAVE_STATUS__EXPORT_RDY__SHIFT 0x00000008
+#define SQ_WAVE_STATUS__HALT_MASK 0x00002000L
+#define SQ_WAVE_STATUS__HALT__SHIFT 0x0000000d
+#define SQ_WAVE_STATUS__IN_BARRIER_MASK 0x00001000L
+#define SQ_WAVE_STATUS__IN_BARRIER__SHIFT 0x0000000c
+#define SQ_WAVE_STATUS__INST_ATC_MASK 0x00800000L
+#define SQ_WAVE_STATUS__INST_ATC__SHIFT 0x00000017
+#define SQ_WAVE_STATUS__IN_TG_MASK 0x00000800L
+#define SQ_WAVE_STATUS__IN_TG__SHIFT 0x0000000b
+#define SQ_WAVE_STATUS__MUST_EXPORT_MASK 0x08000000L
+#define SQ_WAVE_STATUS__MUST_EXPORT__SHIFT 0x0000001b
+#define SQ_WAVE_STATUS__PERF_EN_MASK 0x00080000L
+#define SQ_WAVE_STATUS__PERF_EN__SHIFT 0x00000013
+#define SQ_WAVE_STATUS__PRIV_MASK 0x00000020L
+#define SQ_WAVE_STATUS__PRIV__SHIFT 0x00000005
+#define SQ_WAVE_STATUS__SCC_MASK 0x00000001L
+#define SQ_WAVE_STATUS__SCC__SHIFT 0x00000000
+#define SQ_WAVE_STATUS__SKIP_EXPORT_MASK 0x00040000L
+#define SQ_WAVE_STATUS__SKIP_EXPORT__SHIFT 0x00000012
+#define SQ_WAVE_STATUS__SPI_PRIO_MASK 0x00000006L
+#define SQ_WAVE_STATUS__SPI_PRIO__SHIFT 0x00000001
+#define SQ_WAVE_STATUS__TRAP_EN_MASK 0x00000040L
+#define SQ_WAVE_STATUS__TRAP_EN__SHIFT 0x00000006
+#define SQ_WAVE_STATUS__TRAP_MASK 0x00004000L
+#define SQ_WAVE_STATUS__TRAP__SHIFT 0x0000000e
+#define SQ_WAVE_STATUS__TTRACE_CU_EN_MASK 0x00008000L
+#define SQ_WAVE_STATUS__TTRACE_CU_EN__SHIFT 0x0000000f
+#define SQ_WAVE_STATUS__TTRACE_EN_MASK 0x00000080L
+#define SQ_WAVE_STATUS__TTRACE_EN__SHIFT 0x00000007
+#define SQ_WAVE_STATUS__VALID_MASK 0x00010000L
+#define SQ_WAVE_STATUS__VALID__SHIFT 0x00000010
+#define SQ_WAVE_STATUS__VCCZ_MASK 0x00000400L
+#define SQ_WAVE_STATUS__VCCZ__SHIFT 0x0000000a
+#define SQ_WAVE_STATUS__WAVE_PRIO_MASK 0x00000018L
+#define SQ_WAVE_STATUS__WAVE_PRIO__SHIFT 0x00000003
+#define SQ_WAVE_TBA_HI__ADDR_HI_MASK 0x000000ffL
+#define SQ_WAVE_TBA_HI__ADDR_HI__SHIFT 0x00000000
+#define SQ_WAVE_TBA_LO__ADDR_LO_MASK 0xffffffffL
+#define SQ_WAVE_TBA_LO__ADDR_LO__SHIFT 0x00000000
+#define SQ_WAVE_TMA_HI__ADDR_HI_MASK 0x000000ffL
+#define SQ_WAVE_TMA_HI__ADDR_HI__SHIFT 0x00000000
+#define SQ_WAVE_TMA_LO__ADDR_LO_MASK 0xffffffffL
+#define SQ_WAVE_TMA_LO__ADDR_LO__SHIFT 0x00000000
+#define SQ_WAVE_TRAPSTS__DP_RATE_MASK 0xe0000000L
+#define SQ_WAVE_TRAPSTS__DP_RATE__SHIFT 0x0000001d
+#define SQ_WAVE_TRAPSTS__EXCP_CYCLE_MASK 0x003f0000L
+#define SQ_WAVE_TRAPSTS__EXCP_CYCLE__SHIFT 0x00000010
+#define SQ_WAVE_TRAPSTS__EXCP_MASK 0x0000007fL
+#define SQ_WAVE_TRAPSTS__EXCP__SHIFT 0x00000000
+#define SQ_WAVE_TTMP0__DATA_MASK 0xffffffffL
+#define SQ_WAVE_TTMP0__DATA__SHIFT 0x00000000
+#define SQ_WAVE_TTMP10__DATA_MASK 0xffffffffL
+#define SQ_WAVE_TTMP10__DATA__SHIFT 0x00000000
+#define SQ_WAVE_TTMP11__DATA_MASK 0xffffffffL
+#define SQ_WAVE_TTMP11__DATA__SHIFT 0x00000000
+#define SQ_WAVE_TTMP1__DATA_MASK 0xffffffffL
+#define SQ_WAVE_TTMP1__DATA__SHIFT 0x00000000
+#define SQ_WAVE_TTMP2__DATA_MASK 0xffffffffL
+#define SQ_WAVE_TTMP2__DATA__SHIFT 0x00000000
+#define SQ_WAVE_TTMP3__DATA_MASK 0xffffffffL
+#define SQ_WAVE_TTMP3__DATA__SHIFT 0x00000000
+#define SQ_WAVE_TTMP4__DATA_MASK 0xffffffffL
+#define SQ_WAVE_TTMP4__DATA__SHIFT 0x00000000
+#define SQ_WAVE_TTMP5__DATA_MASK 0xffffffffL
+#define SQ_WAVE_TTMP5__DATA__SHIFT 0x00000000
+#define SQ_WAVE_TTMP6__DATA_MASK 0xffffffffL
+#define SQ_WAVE_TTMP6__DATA__SHIFT 0x00000000
+#define SQ_WAVE_TTMP7__DATA_MASK 0xffffffffL
+#define SQ_WAVE_TTMP7__DATA__SHIFT 0x00000000
+#define SQ_WAVE_TTMP8__DATA_MASK 0xffffffffL
+#define SQ_WAVE_TTMP8__DATA__SHIFT 0x00000000
+#define SQ_WAVE_TTMP9__DATA_MASK 0xffffffffL
+#define SQ_WAVE_TTMP9__DATA__SHIFT 0x00000000
+#define SX_DEBUG_1__DEBUG_DATA_MASK 0xffffff80L
+#define SX_DEBUG_1__DEBUG_DATA__SHIFT 0x00000007
+#define SX_DEBUG_1__SX_DB_QUAD_CREDIT_MASK 0x0000007fL
+#define SX_DEBUG_1__SX_DB_QUAD_CREDIT__SHIFT 0x00000000
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL1_BUSY_MASK 0x80000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL1_BUSY__SHIFT 0x0000001f
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL2_BUSY_MASK 0x40000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL2_BUSY__SHIFT 0x0000001e
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL3_BUSY_MASK 0x20000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL3_BUSY__SHIFT 0x0000001d
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL0_BUSY_MASK 0x10000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL0_BUSY__SHIFT 0x0000001c
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL1_BUSY_MASK 0x08000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL1_BUSY__SHIFT 0x0000001b
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL2_BUSY_MASK 0x04000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL2_BUSY__SHIFT 0x0000001a
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL3_BUSY_MASK 0x02000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL3_BUSY__SHIFT 0x00000019
+#define SX_DEBUG_BUSY_2__COL_DBIF0_FIFO_BUSY_MASK 0x00800000L
+#define SX_DEBUG_BUSY_2__COL_DBIF0_FIFO_BUSY__SHIFT 0x00000017
+#define SX_DEBUG_BUSY_2__COL_DBIF0_READ_VALID_MASK 0x01000000L
+#define SX_DEBUG_BUSY_2__COL_DBIF0_READ_VALID__SHIFT 0x00000018
+#define SX_DEBUG_BUSY_2__COL_DBIF0_SENDFREE_BUSY_MASK 0x00400000L
+#define SX_DEBUG_BUSY_2__COL_DBIF0_SENDFREE_BUSY__SHIFT 0x00000016
+#define SX_DEBUG_BUSY_2__COL_DBIF1_FIFO_BUSY_MASK 0x00100000L
+#define SX_DEBUG_BUSY_2__COL_DBIF1_FIFO_BUSY__SHIFT 0x00000014
+#define SX_DEBUG_BUSY_2__COL_DBIF1_READ_VALID_MASK 0x00200000L
+#define SX_DEBUG_BUSY_2__COL_DBIF1_READ_VALID__SHIFT 0x00000015
+#define SX_DEBUG_BUSY_2__COL_DBIF1_SENDFREE_BUSY_MASK 0x00080000L
+#define SX_DEBUG_BUSY_2__COL_DBIF1_SENDFREE_BUSY__SHIFT 0x00000013
+#define SX_DEBUG_BUSY_2__COL_DBIF2_FIFO_BUSY_MASK 0x00020000L
+#define SX_DEBUG_BUSY_2__COL_DBIF2_FIFO_BUSY__SHIFT 0x00000011
+#define SX_DEBUG_BUSY_2__COL_DBIF2_READ_VALID_MASK 0x00040000L
+#define SX_DEBUG_BUSY_2__COL_DBIF2_READ_VALID__SHIFT 0x00000012
+#define SX_DEBUG_BUSY_2__COL_DBIF2_SENDFREE_BUSY_MASK 0x00010000L
+#define SX_DEBUG_BUSY_2__COL_DBIF2_SENDFREE_BUSY__SHIFT 0x00000010
+#define SX_DEBUG_BUSY_2__COL_DBIF3_FIFO_BUSY_MASK 0x00004000L
+#define SX_DEBUG_BUSY_2__COL_DBIF3_FIFO_BUSY__SHIFT 0x0000000e
+#define SX_DEBUG_BUSY_2__COL_DBIF3_READ_VALID_MASK 0x00008000L
+#define SX_DEBUG_BUSY_2__COL_DBIF3_READ_VALID__SHIFT 0x0000000f
+#define SX_DEBUG_BUSY_2__COL_DBIF3_SENDFREE_BUSY_MASK 0x00002000L
+#define SX_DEBUG_BUSY_2__COL_DBIF3_SENDFREE_BUSY__SHIFT 0x0000000d
+#define SX_DEBUG_BUSY_2__COL_REQ0_BUSY_MASK 0x00001000L
+#define SX_DEBUG_BUSY_2__COL_REQ0_BUSY__SHIFT 0x0000000c
+#define SX_DEBUG_BUSY_2__COL_REQ0_FREECNT_NE0_MASK 0x00000400L
+#define SX_DEBUG_BUSY_2__COL_REQ0_FREECNT_NE0__SHIFT 0x0000000a
+#define SX_DEBUG_BUSY_2__COL_REQ0_IDLE_MASK 0x00000800L
+#define SX_DEBUG_BUSY_2__COL_REQ0_IDLE__SHIFT 0x0000000b
+#define SX_DEBUG_BUSY_2__COL_REQ1_BUSY_MASK 0x00000200L
+#define SX_DEBUG_BUSY_2__COL_REQ1_BUSY__SHIFT 0x00000009
+#define SX_DEBUG_BUSY_2__COL_REQ1_FREECNT_NE0_MASK 0x00000080L
+#define SX_DEBUG_BUSY_2__COL_REQ1_FREECNT_NE0__SHIFT 0x00000007
+#define SX_DEBUG_BUSY_2__COL_REQ1_IDLE_MASK 0x00000100L
+#define SX_DEBUG_BUSY_2__COL_REQ1_IDLE__SHIFT 0x00000008
+#define SX_DEBUG_BUSY_2__COL_REQ2_BUSY_MASK 0x00000040L
+#define SX_DEBUG_BUSY_2__COL_REQ2_BUSY__SHIFT 0x00000006
+#define SX_DEBUG_BUSY_2__COL_REQ2_FREECNT_NE0_MASK 0x00000010L
+#define SX_DEBUG_BUSY_2__COL_REQ2_FREECNT_NE0__SHIFT 0x00000004
+#define SX_DEBUG_BUSY_2__COL_REQ2_IDLE_MASK 0x00000020L
+#define SX_DEBUG_BUSY_2__COL_REQ2_IDLE__SHIFT 0x00000005
+#define SX_DEBUG_BUSY_2__COL_REQ3_BUSY_MASK 0x00000008L
+#define SX_DEBUG_BUSY_2__COL_REQ3_BUSY__SHIFT 0x00000003
+#define SX_DEBUG_BUSY_2__COL_REQ3_FREECNT_NE0_MASK 0x00000002L
+#define SX_DEBUG_BUSY_2__COL_REQ3_FREECNT_NE0__SHIFT 0x00000001
+#define SX_DEBUG_BUSY_2__COL_REQ3_IDLE_MASK 0x00000004L
+#define SX_DEBUG_BUSY_2__COL_REQ3_IDLE__SHIFT 0x00000002
+#define SX_DEBUG_BUSY_2__COL_SCBD_BUSY_MASK 0x00000001L
+#define SX_DEBUG_BUSY_2__COL_SCBD_BUSY__SHIFT 0x00000000
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL1_BUSY_MASK 0x80000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL1_BUSY__SHIFT 0x0000001f
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL2_BUSY_MASK 0x40000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL2_BUSY__SHIFT 0x0000001e
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL3_BUSY_MASK 0x20000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL3_BUSY__SHIFT 0x0000001d
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL0_BUSY_MASK 0x10000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL0_BUSY__SHIFT 0x0000001c
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL1_BUSY_MASK 0x08000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL1_BUSY__SHIFT 0x0000001b
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL2_BUSY_MASK 0x04000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL2_BUSY__SHIFT 0x0000001a
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL3_BUSY_MASK 0x02000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL3_BUSY__SHIFT 0x00000019
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL0_BUSY_MASK 0x01000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL0_BUSY__SHIFT 0x00000018
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL1_BUSY_MASK 0x00800000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL1_BUSY__SHIFT 0x00000017
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL2_BUSY_MASK 0x00400000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL2_BUSY__SHIFT 0x00000016
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL3_BUSY_MASK 0x00200000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL3_BUSY__SHIFT 0x00000015
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL0_BUSY_MASK 0x00100000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL0_BUSY__SHIFT 0x00000014
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL1_BUSY_MASK 0x00080000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL1_BUSY__SHIFT 0x00000013
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL2_BUSY_MASK 0x00040000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL2_BUSY__SHIFT 0x00000012
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL3_BUSY_MASK 0x00020000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL3_BUSY__SHIFT 0x00000011
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL0_BUSY_MASK 0x00010000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL0_BUSY__SHIFT 0x00000010
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL1_BUSY_MASK 0x00008000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL1_BUSY__SHIFT 0x0000000f
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL2_BUSY_MASK 0x00004000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL2_BUSY__SHIFT 0x0000000e
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL3_BUSY_MASK 0x00002000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL3_BUSY__SHIFT 0x0000000d
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL0_BUSY_MASK 0x00001000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL0_BUSY__SHIFT 0x0000000c
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL1_BUSY_MASK 0x00000800L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL1_BUSY__SHIFT 0x0000000b
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL2_BUSY_MASK 0x00000400L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL2_BUSY__SHIFT 0x0000000a
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL3_BUSY_MASK 0x00000200L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL3_BUSY__SHIFT 0x00000009
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL0_BUSY_MASK 0x00000100L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL0_BUSY__SHIFT 0x00000008
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL1_BUSY_MASK 0x00000080L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL1_BUSY__SHIFT 0x00000007
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL2_BUSY_MASK 0x00000040L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL2_BUSY__SHIFT 0x00000006
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL3_BUSY_MASK 0x00000020L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL3_BUSY__SHIFT 0x00000005
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL0_BUSY_MASK 0x00000010L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL0_BUSY__SHIFT 0x00000004
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL1_BUSY_MASK 0x00000008L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL1_BUSY__SHIFT 0x00000003
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL2_BUSY_MASK 0x00000004L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL2_BUSY__SHIFT 0x00000002
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL3_BUSY_MASK 0x00000002L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL3_BUSY__SHIFT 0x00000001
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK2_VAL0_BUSY_MASK 0x00000001L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK2_VAL0_BUSY__SHIFT 0x00000000
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL0_BUSY_MASK 0x01000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL0_BUSY__SHIFT 0x00000018
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL1_BUSY_MASK 0x00800000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL1_BUSY__SHIFT 0x00000017
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL2_BUSY_MASK 0x00400000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL2_BUSY__SHIFT 0x00000016
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL3_BUSY_MASK 0x00200000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL3_BUSY__SHIFT 0x00000015
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL0_BUSY_MASK 0x00100000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL0_BUSY__SHIFT 0x00000014
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL1_BUSY_MASK 0x00080000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL1_BUSY__SHIFT 0x00000013
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL2_BUSY_MASK 0x00040000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL2_BUSY__SHIFT 0x00000012
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL3_BUSY_MASK 0x00020000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL3_BUSY__SHIFT 0x00000011
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL0_BUSY_MASK 0x00010000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL0_BUSY__SHIFT 0x00000010
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL1_BUSY_MASK 0x00008000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL1_BUSY__SHIFT 0x0000000f
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL2_BUSY_MASK 0x00004000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL2_BUSY__SHIFT 0x0000000e
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL3_BUSY_MASK 0x00002000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL3_BUSY__SHIFT 0x0000000d
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL0_BUSY_MASK 0x00001000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL0_BUSY__SHIFT 0x0000000c
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL1_BUSY_MASK 0x00000800L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL1_BUSY__SHIFT 0x0000000b
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL2_BUSY_MASK 0x00000400L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL2_BUSY__SHIFT 0x0000000a
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL3_BUSY_MASK 0x00000200L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL3_BUSY__SHIFT 0x00000009
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL0_BUSY_MASK 0x00000100L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL0_BUSY__SHIFT 0x00000008
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL1_BUSY_MASK 0x00000080L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL1_BUSY__SHIFT 0x00000007
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL2_BUSY_MASK 0x00000040L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL2_BUSY__SHIFT 0x00000006
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL3_BUSY_MASK 0x00000020L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL3_BUSY__SHIFT 0x00000005
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL0_BUSY_MASK 0x00000010L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL0_BUSY__SHIFT 0x00000004
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL1_BUSY_MASK 0x00000008L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL1_BUSY__SHIFT 0x00000003
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL2_BUSY_MASK 0x00000004L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL2_BUSY__SHIFT 0x00000002
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL3_BUSY_MASK 0x00000002L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL3_BUSY__SHIFT 0x00000001
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK2_VAL0_BUSY_MASK 0x00000001L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK2_VAL0_BUSY__SHIFT 0x00000000
+#define SX_DEBUG_BUSY_4__RESERVED_MASK 0xfe000000L
+#define SX_DEBUG_BUSY_4__RESERVED__SHIFT 0x00000019
+#define SX_DEBUG_BUSY__ADDR_BUSYORVAL_MASK 0x80000000L
+#define SX_DEBUG_BUSY__ADDR_BUSYORVAL__SHIFT 0x0000001f
+#define SX_DEBUG_BUSY__CMD_BUSYORVAL_MASK 0x40000000L
+#define SX_DEBUG_BUSY__CMD_BUSYORVAL__SHIFT 0x0000001e
+#define SX_DEBUG_BUSY__PA_SX_BUSY_MASK 0x00000004L
+#define SX_DEBUG_BUSY__PA_SX_BUSY__SHIFT 0x00000002
+#define SX_DEBUG_BUSY__PCCMD_VALID_MASK 0x08000000L
+#define SX_DEBUG_BUSY__PCCMD_VALID__SHIFT 0x0000001b
+#define SX_DEBUG_BUSY__POS_BANK0VAL0_BUSY_MASK 0x00080000L
+#define SX_DEBUG_BUSY__POS_BANK0VAL0_BUSY__SHIFT 0x00000013
+#define SX_DEBUG_BUSY__POS_BANK0VAL1_BUSY_MASK 0x00040000L
+#define SX_DEBUG_BUSY__POS_BANK0VAL1_BUSY__SHIFT 0x00000012
+#define SX_DEBUG_BUSY__POS_BANK0VAL2_BUSY_MASK 0x00020000L
+#define SX_DEBUG_BUSY__POS_BANK0VAL2_BUSY__SHIFT 0x00000011
+#define SX_DEBUG_BUSY__POS_BANK0VAL3_BUSY_MASK 0x00010000L
+#define SX_DEBUG_BUSY__POS_BANK0VAL3_BUSY__SHIFT 0x00000010
+#define SX_DEBUG_BUSY__POS_BANK1VAL0_BUSY_MASK 0x00008000L
+#define SX_DEBUG_BUSY__POS_BANK1VAL0_BUSY__SHIFT 0x0000000f
+#define SX_DEBUG_BUSY__POS_BANK1VAL1_BUSY_MASK 0x00004000L
+#define SX_DEBUG_BUSY__POS_BANK1VAL1_BUSY__SHIFT 0x0000000e
+#define SX_DEBUG_BUSY__POS_BANK1VAL2_BUSY_MASK 0x00002000L
+#define SX_DEBUG_BUSY__POS_BANK1VAL2_BUSY__SHIFT 0x0000000d
+#define SX_DEBUG_BUSY__POS_BANK1VAL3_BUSY_MASK 0x00001000L
+#define SX_DEBUG_BUSY__POS_BANK1VAL3_BUSY__SHIFT 0x0000000c
+#define SX_DEBUG_BUSY__POS_BANK2VAL0_BUSY_MASK 0x00000800L
+#define SX_DEBUG_BUSY__POS_BANK2VAL0_BUSY__SHIFT 0x0000000b
+#define SX_DEBUG_BUSY__POS_BANK2VAL1_BUSY_MASK 0x00000400L
+#define SX_DEBUG_BUSY__POS_BANK2VAL1_BUSY__SHIFT 0x0000000a
+#define SX_DEBUG_BUSY__POS_BANK2VAL2_BUSY_MASK 0x00000200L
+#define SX_DEBUG_BUSY__POS_BANK2VAL2_BUSY__SHIFT 0x00000009
+#define SX_DEBUG_BUSY__POS_BANK2VAL3_BUSY_MASK 0x00000100L
+#define SX_DEBUG_BUSY__POS_BANK2VAL3_BUSY__SHIFT 0x00000008
+#define SX_DEBUG_BUSY__POS_BANK3VAL0_BUSY_MASK 0x00000080L
+#define SX_DEBUG_BUSY__POS_BANK3VAL0_BUSY__SHIFT 0x00000007
+#define SX_DEBUG_BUSY__POS_BANK3VAL1_BUSY_MASK 0x00000040L
+#define SX_DEBUG_BUSY__POS_BANK3VAL1_BUSY__SHIFT 0x00000006
+#define SX_DEBUG_BUSY__POS_BANK3VAL2_BUSY_MASK 0x00000020L
+#define SX_DEBUG_BUSY__POS_BANK3VAL2_BUSY__SHIFT 0x00000005
+#define SX_DEBUG_BUSY__POS_BANK3VAL3_BUSY_MASK 0x00000010L
+#define SX_DEBUG_BUSY__POS_BANK3VAL3_BUSY__SHIFT 0x00000004
+#define SX_DEBUG_BUSY__POS_FREE_OR_VALIDS_MASK 0x00000001L
+#define SX_DEBUG_BUSY__POS_FREE_OR_VALIDS__SHIFT 0x00000000
+#define SX_DEBUG_BUSY__POS_INMUX_VALID_MASK 0x00100000L
+#define SX_DEBUG_BUSY__POS_INMUX_VALID__SHIFT 0x00000014
+#define SX_DEBUG_BUSY__POS_REQUESTER_BUSY_MASK 0x00000002L
+#define SX_DEBUG_BUSY__POS_REQUESTER_BUSY__SHIFT 0x00000001
+#define SX_DEBUG_BUSY__POS_SCBD_BUSY_MASK 0x00000008L
+#define SX_DEBUG_BUSY__POS_SCBD_BUSY__SHIFT 0x00000003
+#define SX_DEBUG_BUSY__VDATA0_VALID_MASK 0x20000000L
+#define SX_DEBUG_BUSY__VDATA0_VALID__SHIFT 0x0000001d
+#define SX_DEBUG_BUSY__VDATA1_VALID_MASK 0x10000000L
+#define SX_DEBUG_BUSY__VDATA1_VALID__SHIFT 0x0000001c
+#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ1_MASK 0x04000000L
+#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ1__SHIFT 0x0000001a
+#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ2_MASK 0x02000000L
+#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ2__SHIFT 0x00000019
+#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ3_MASK 0x01000000L
+#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ3__SHIFT 0x00000018
+#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ1_MASK 0x00800000L
+#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ1__SHIFT 0x00000017
+#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ2_MASK 0x00400000L
+#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ2__SHIFT 0x00000016
+#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ3_MASK 0x00200000L
+#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ3__SHIFT 0x00000015
+#define SXIFCCG_DEBUG_REG0__point_address_MASK 0x000001c0L
+#define SXIFCCG_DEBUG_REG0__point_address__SHIFT 0x00000006
+#define SXIFCCG_DEBUG_REG0__position_address_MASK 0x0000003fL
+#define SXIFCCG_DEBUG_REG0__position_address__SHIFT 0x00000000
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_advance_MASK 0x80000000L
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_advance__SHIFT 0x0000001f
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_aux_inc_MASK 0x40000000L
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_aux_inc__SHIFT 0x0000001e
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_aux_sel_MASK 0x0c000000L
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_aux_sel__SHIFT 0x0000001a
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_pci_MASK 0x03ff0000L
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_pci__SHIFT 0x00000010
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_req_mask_MASK 0x0000f000L
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_req_mask__SHIFT 0x0000000c
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_sp_id_MASK 0x30000000L
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_sp_id__SHIFT 0x0000001c
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_state_var_indx_MASK 0x00000e00L
+#define SXIFCCG_DEBUG_REG0__sx_pending_rd_state_var_indx__SHIFT 0x00000009
+#define SXIFCCG_DEBUG_REG1__aux_sel_MASK 0x00300000L
+#define SXIFCCG_DEBUG_REG1__aux_sel__SHIFT 0x00000014
+#define SXIFCCG_DEBUG_REG1__available_positions_MASK 0x0000007fL
+#define SXIFCCG_DEBUG_REG1__available_positions__SHIFT 0x00000000
+#define SXIFCCG_DEBUG_REG1__pasx_req_cnt_0_MASK 0xf0000000L
+#define SXIFCCG_DEBUG_REG1__pasx_req_cnt_0__SHIFT 0x0000001c
+#define SXIFCCG_DEBUG_REG1__pasx_req_cnt_1_MASK 0x0f000000L
+#define SXIFCCG_DEBUG_REG1__pasx_req_cnt_1__SHIFT 0x00000018
+#define SXIFCCG_DEBUG_REG1__statevar_bits_disable_sp_MASK 0x000f0000L
+#define SXIFCCG_DEBUG_REG1__statevar_bits_disable_sp__SHIFT 0x00000010
+#define SXIFCCG_DEBUG_REG1__statevar_bits_vs_out_misc_vec_ena_MASK 0x00008000L
+#define SXIFCCG_DEBUG_REG1__statevar_bits_vs_out_misc_vec_ena__SHIFT 0x0000000f
+#define SXIFCCG_DEBUG_REG1__sx_pending_fifo_contents_MASK 0x00007c00L
+#define SXIFCCG_DEBUG_REG1__sx_pending_fifo_contents__SHIFT 0x0000000a
+#define SXIFCCG_DEBUG_REG1__sx_receive_indx_MASK 0x00000380L
+#define SXIFCCG_DEBUG_REG1__sx_receive_indx__SHIFT 0x00000007
+#define SXIFCCG_DEBUG_REG1__sx_to_pa_empty_0_MASK 0x00800000L
+#define SXIFCCG_DEBUG_REG1__sx_to_pa_empty_0__SHIFT 0x00000017
+#define SXIFCCG_DEBUG_REG1__sx_to_pa_empty_1_MASK 0x00400000L
+#define SXIFCCG_DEBUG_REG1__sx_to_pa_empty_1__SHIFT 0x00000016
+#define SXIFCCG_DEBUG_REG2__param_cache_base_MASK 0x0000007fL
+#define SXIFCCG_DEBUG_REG2__param_cache_base__SHIFT 0x00000000
+#define SXIFCCG_DEBUG_REG2__req_active_verts_loaded_MASK 0x00008000L
+#define SXIFCCG_DEBUG_REG2__req_active_verts_loaded__SHIFT 0x0000000f
+#define SXIFCCG_DEBUG_REG2__req_active_verts_MASK 0x007f0000L
+#define SXIFCCG_DEBUG_REG2__req_active_verts__SHIFT 0x00000010
+#define SXIFCCG_DEBUG_REG2__sx_aux_MASK 0x00000180L
+#define SXIFCCG_DEBUG_REG2__sx_aux__SHIFT 0x00000007
+#define SXIFCCG_DEBUG_REG2__sx_request_indx_MASK 0x00007e00L
+#define SXIFCCG_DEBUG_REG2__sx_request_indx__SHIFT 0x00000009
+#define SXIFCCG_DEBUG_REG2__vgt_to_ccgen_active_verts_MASK 0xfc000000L
+#define SXIFCCG_DEBUG_REG2__vgt_to_ccgen_active_verts__SHIFT 0x0000001a
+#define SXIFCCG_DEBUG_REG2__vgt_to_ccgen_state_var_indx_MASK 0x03800000L
+#define SXIFCCG_DEBUG_REG2__vgt_to_ccgen_state_var_indx__SHIFT 0x00000017
+#define SXIFCCG_DEBUG_REG3__ALWAYS_ZERO_MASK 0x000000ffL
+#define SXIFCCG_DEBUG_REG3__ALWAYS_ZERO__SHIFT 0x00000000
+#define SXIFCCG_DEBUG_REG3__available_positions_MASK 0x001fc000L
+#define SXIFCCG_DEBUG_REG3__available_positions__SHIFT 0x0000000e
+#define SXIFCCG_DEBUG_REG3__ccgen_to_clipcc_fifo_full_MASK 0x20000000L
+#define SXIFCCG_DEBUG_REG3__ccgen_to_clipcc_fifo_full__SHIFT 0x0000001d
+#define SXIFCCG_DEBUG_REG3__ccgen_to_clipcc_write_MASK 0x80000000L
+#define SXIFCCG_DEBUG_REG3__ccgen_to_clipcc_write__SHIFT 0x0000001f
+#define SXIFCCG_DEBUG_REG3__current_state_MASK 0x00600000L
+#define SXIFCCG_DEBUG_REG3__current_state__SHIFT 0x00000015
+#define SXIFCCG_DEBUG_REG3__statevar_bits_vs_out_ccdist0_vec_ena_MASK 0x00002000L
+#define SXIFCCG_DEBUG_REG3__statevar_bits_vs_out_ccdist0_vec_ena__SHIFT 0x0000000d
+#define SXIFCCG_DEBUG_REG3__statevar_bits_vs_out_ccdist1_vec_ena_MASK 0x00001000L
+#define SXIFCCG_DEBUG_REG3__statevar_bits_vs_out_ccdist1_vec_ena__SHIFT 0x0000000c
+#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_empty_MASK 0x02000000L
+#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_empty__SHIFT 0x00000019
+#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_full_MASK 0x04000000L
+#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_full__SHIFT 0x0000001a
+#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_write_MASK 0x40000000L
+#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_write__SHIFT 0x0000001e
+#define SXIFCCG_DEBUG_REG3__vertex_fifo_empty_MASK 0x00800000L
+#define SXIFCCG_DEBUG_REG3__vertex_fifo_empty__SHIFT 0x00000017
+#define SXIFCCG_DEBUG_REG3__vertex_fifo_entriesavailable_MASK 0x00000f00L
+#define SXIFCCG_DEBUG_REG3__vertex_fifo_entriesavailable__SHIFT 0x00000008
+#define SXIFCCG_DEBUG_REG3__vertex_fifo_full_MASK 0x01000000L
+#define SXIFCCG_DEBUG_REG3__vertex_fifo_full__SHIFT 0x00000018
+#define SXIFCCG_DEBUG_REG3__vgt_to_ccgen_fifo_empty_MASK 0x08000000L
+#define SXIFCCG_DEBUG_REG3__vgt_to_ccgen_fifo_empty__SHIFT 0x0000001b
+#define SXIFCCG_DEBUG_REG3__vgt_to_ccgen_fifo_full_MASK 0x10000000L
+#define SXIFCCG_DEBUG_REG3__vgt_to_ccgen_fifo_full__SHIFT 0x0000001c
+#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2_MASK 0x000003ffL
+#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2__SHIFT 0x00000000
+#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3_MASK 0x000ffc00L
+#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3__SHIFT 0x0000000a
+#define SX_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define SX_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1_MASK 0x000ffc00L
+#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1__SHIFT 0x0000000a
+#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK 0x000000ffL
+#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT 0x00000000
+#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT2_MASK 0x000003ffL
+#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT2__SHIFT 0x00000000
+#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT3_MASK 0x000ffc00L
+#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT3__SHIFT 0x0000000a
+#define SX_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define SX_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1_MASK 0x000ffc00L
+#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1__SHIFT 0x0000000a
+#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK 0x000000ffL
+#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT 0x00000000
+#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SX_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define SX_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1_MASK 0x000ffc00L
+#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1__SHIFT 0x0000000a
+#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT_MASK 0x000000ffL
+#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT__SHIFT 0x00000000
+#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define SX_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define SX_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1_MASK 0x000ffc00L
+#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1__SHIFT 0x0000000a
+#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT_MASK 0x000000ffL
+#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT__SHIFT 0x00000000
+#define TA_BC_BASE_ADDR__ADDRESS_MASK 0xffffffffL
+#define TA_BC_BASE_ADDR__ADDRESS__SHIFT 0x00000000
+#define TA_BC_BASE_ADDR_HI__ADDRESS_MASK 0x000000ffL
+#define TA_BC_BASE_ADDR_HI__ADDRESS__SHIFT 0x00000000
+#define TA_CGTT_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define TA_CGTT_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define TA_CGTT_CTRL__ON_DELAY_MASK 0x0000000fL
+#define TA_CGTT_CTRL__ON_DELAY__SHIFT 0x00000000
+#define TA_CGTT_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define TA_CGTT_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define TA_CGTT_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define TA_CGTT_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define TA_CGTT_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define TA_CGTT_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define TA_CGTT_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define TA_CGTT_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define TA_CNTL__ALIGNER_CREDIT_MASK 0x001f0000L
+#define TA_CNTL__ALIGNER_CREDIT__SHIFT 0x00000010
+#define TA_CNTL_AUX__ANISO_WEIGHT_MODE_MASK 0x00010000L
+#define TA_CNTL_AUX__ANISO_WEIGHT_MODE__SHIFT 0x00000010
+#define TA_CNTL__TC_DATA_CREDIT_MASK 0x0000e000L
+#define TA_CNTL__TC_DATA_CREDIT__SHIFT 0x0000000d
+#define TA_CNTL__TD_FIFO_CREDIT_MASK 0xffc00000L
+#define TA_CNTL__TD_FIFO_CREDIT__SHIFT 0x00000016
+#define TA_CS_BC_BASE_ADDR__ADDRESS_MASK 0xffffffffL
+#define TA_CS_BC_BASE_ADDR__ADDRESS__SHIFT 0x00000000
+#define TA_CS_BC_BASE_ADDR_HI__ADDRESS_MASK 0x000000ffL
+#define TA_CS_BC_BASE_ADDR_HI__ADDRESS__SHIFT 0x00000000
+#define TA_DEBUG_DATA__DATA_MASK 0xffffffffL
+#define TA_DEBUG_DATA__DATA__SHIFT 0x00000000
+#define TA_DEBUG_INDEX__INDEX_MASK 0x0000001fL
+#define TA_DEBUG_INDEX__INDEX__SHIFT 0x00000000
+#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000L
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x0000001c
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0x0f000000L
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x00000018
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x000000ffL
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x0003fc00L
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define TA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x0003fc00L
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0x0003fc00L
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TA_SCRATCH__SCRATCH_MASK 0xffffffffL
+#define TA_SCRATCH__SCRATCH__SHIFT 0x00000000
+#define TA_STATUS__AL_BUSY_MASK 0x40000000L
+#define TA_STATUS__AL_BUSY__SHIFT 0x0000001e
+#define TA_STATUS__BUSY_MASK 0x80000000L
+#define TA_STATUS__BUSY__SHIFT 0x0000001f
+#define TA_STATUS__FA_BUSY_MASK 0x20000000L
+#define TA_STATUS__FA_BUSY__SHIFT 0x0000001d
+#define TA_STATUS__FA_LFIFO_EMPTYB_MASK 0x00200000L
+#define TA_STATUS__FA_LFIFO_EMPTYB__SHIFT 0x00000015
+#define TA_STATUS__FA_PFIFO_EMPTYB_MASK 0x00100000L
+#define TA_STATUS__FA_PFIFO_EMPTYB__SHIFT 0x00000014
+#define TA_STATUS__FA_SFIFO_EMPTYB_MASK 0x00400000L
+#define TA_STATUS__FA_SFIFO_EMPTYB__SHIFT 0x00000016
+#define TA_STATUS__FG_BUSY_MASK 0x02000000L
+#define TA_STATUS__FG_BUSY__SHIFT 0x00000019
+#define TA_STATUS__FG_LFIFO_EMPTYB_MASK 0x00002000L
+#define TA_STATUS__FG_LFIFO_EMPTYB__SHIFT 0x0000000d
+#define TA_STATUS__FG_PFIFO_EMPTYB_MASK 0x00001000L
+#define TA_STATUS__FG_PFIFO_EMPTYB__SHIFT 0x0000000c
+#define TA_STATUS__FG_SFIFO_EMPTYB_MASK 0x00004000L
+#define TA_STATUS__FG_SFIFO_EMPTYB__SHIFT 0x0000000e
+#define TA_STATUS__FL_BUSY_MASK 0x08000000L
+#define TA_STATUS__FL_BUSY__SHIFT 0x0000001b
+#define TA_STATUS__FL_LFIFO_EMPTYB_MASK 0x00020000L
+#define TA_STATUS__FL_LFIFO_EMPTYB__SHIFT 0x00000011
+#define TA_STATUS__FL_PFIFO_EMPTYB_MASK 0x00010000L
+#define TA_STATUS__FL_PFIFO_EMPTYB__SHIFT 0x00000010
+#define TA_STATUS__FL_SFIFO_EMPTYB_MASK 0x00040000L
+#define TA_STATUS__FL_SFIFO_EMPTYB__SHIFT 0x00000012
+#define TA_STATUS__IN_BUSY_MASK 0x01000000L
+#define TA_STATUS__IN_BUSY__SHIFT 0x00000018
+#define TA_STATUS__LA_BUSY_MASK 0x04000000L
+#define TA_STATUS__LA_BUSY__SHIFT 0x0000001a
+#define TA_STATUS__TA_BUSY_MASK 0x10000000L
+#define TA_STATUS__TA_BUSY__SHIFT 0x0000001c
+#define TCA_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define TCA_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define TCA_CGTT_SCLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define TCA_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L
+#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define TCA_CTRL__HOLE_TIMEOUT_MASK 0x0000000fL
+#define TCA_CTRL__HOLE_TIMEOUT__SHIFT 0x00000000
+#define TCA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TCA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TCA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TCA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0x0f000000L
+#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x00000018
+#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf0000000L
+#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x0000001c
+#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define TCA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TCA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TCA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define TCA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define TCA_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TCA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TCA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define TCA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define TCA_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TCA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TCA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TCA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TCA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TCA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0x0f000000L
+#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x00000018
+#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0xf0000000L
+#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x0000001c
+#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define TCA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TCA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TCA_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define TCA_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define TCA_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TCA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TCA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define TCA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define TCA_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TCA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TCA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TCA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TCA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TCA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TCA_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TCA_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TCA_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TCA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TCA_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TCA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TCA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TCA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TCA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TCA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TCA_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TCA_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TCA_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TCA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TCA_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TCA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TCC_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define TCC_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define TCC_CGTT_SCLK_CTRL__ON_DELAY_MASK 0x0000000fL
+#define TCC_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x00000000
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L
+#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define TCC_CTRL__CACHE_SIZE_MASK 0x00000003L
+#define TCC_CTRL__CACHE_SIZE__SHIFT 0x00000000
+#define TCC_CTRL__LATENCY_FIFO_SIZE_MASK 0x000f0000L
+#define TCC_CTRL__LATENCY_FIFO_SIZE__SHIFT 0x00000010
+#define TCC_CTRL__RATE_MASK 0x0000000cL
+#define TCC_CTRL__RATE__SHIFT 0x00000002
+#define TCC_CTRL__SRC_FIFO_SIZE_MASK 0x0000f000L
+#define TCC_CTRL__SRC_FIFO_SIZE__SHIFT 0x0000000c
+#define TCC_CTRL__WB_OR_INV_ALL_VMIDS_MASK 0x00100000L
+#define TCC_CTRL__WB_OR_INV_ALL_VMIDS__SHIFT 0x00000014
+#define TCC_CTRL__WRITEBACK_MARGIN_MASK 0x000000f0L
+#define TCC_CTRL__WRITEBACK_MARGIN__SHIFT 0x00000004
+#define TCC_EDC_COUNTER__DED_COUNT_MASK 0x000f0000L
+#define TCC_EDC_COUNTER__DED_COUNT__SHIFT 0x00000010
+#define TCC_EDC_COUNTER__SEC_COUNT_MASK 0x0000000fL
+#define TCC_EDC_COUNTER__SEC_COUNT__SHIFT 0x00000000
+#define TCC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TCC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TCC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TCC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0x0f000000L
+#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x00000018
+#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf0000000L
+#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x0000001c
+#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define TCC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TCC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TCC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define TCC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define TCC_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TCC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TCC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define TCC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define TCC_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TCC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TCC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TCC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TCC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TCC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0x0f000000L
+#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x00000018
+#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0xf0000000L
+#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x0000001c
+#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define TCC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TCC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TCC_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define TCC_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define TCC_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TCC_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TCC_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define TCC_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define TCC_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TCC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TCC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TCC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TCC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TCC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TCC_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TCC_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TCC_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TCC_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TCC_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TCC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TCC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TCC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TCC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TCC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TCC_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TCC_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TCC_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TCC_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TCC_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TCC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TCI_CNTL_1__REQ_FIFO_DEPTH_MASK 0x00ff0000L
+#define TCI_CNTL_1__REQ_FIFO_DEPTH__SHIFT 0x00000010
+#define TCI_CNTL_1__WBINVL1_NUM_CYCLES_MASK 0x0000ffffL
+#define TCI_CNTL_1__WBINVL1_NUM_CYCLES__SHIFT 0x00000000
+#define TCI_CNTL_1__WDATA_RAM_DEPTH_MASK 0xff000000L
+#define TCI_CNTL_1__WDATA_RAM_DEPTH__SHIFT 0x00000018
+#define TCI_CNTL_2__L1_INVAL_ON_WBINVL2_MASK 0x00000001L
+#define TCI_CNTL_2__L1_INVAL_ON_WBINVL2__SHIFT 0x00000000
+#define TCI_CNTL_2__TCA_MAX_CREDIT_MASK 0x000001feL
+#define TCI_CNTL_2__TCA_MAX_CREDIT__SHIFT 0x00000001
+#define TCI_STATUS__TCI_BUSY_MASK 0x00000001L
+#define TCI_STATUS__TCI_BUSY__SHIFT 0x00000000
+#define TCP_ADDR_CONFIG__COLHI_WIDTH_MASK 0x000001c0L
+#define TCP_ADDR_CONFIG__COLHI_WIDTH__SHIFT 0x00000006
+#define TCP_ADDR_CONFIG__NUM_BANKS_MASK 0x00000030L
+#define TCP_ADDR_CONFIG__NUM_BANKS__SHIFT 0x00000004
+#define TCP_ADDR_CONFIG__NUM_TCC_BANKS_MASK 0x0000000fL
+#define TCP_ADDR_CONFIG__NUM_TCC_BANKS__SHIFT 0x00000000
+#define TCP_ADDR_CONFIG__RB_SPLIT_COLHI_MASK 0x00000200L
+#define TCP_ADDR_CONFIG__RB_SPLIT_COLHI__SHIFT 0x00000009
+#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_BITS_MASK 0x00000700L
+#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_BITS__SHIFT 0x00000008
+#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_XOR_COUNT_MASK 0x07000000L
+#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_XOR_COUNT__SHIFT 0x00000018
+#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_BITS_MASK 0x00000007L
+#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_BITS__SHIFT 0x00000000
+#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_XOR_COUNT_MASK 0x00070000L
+#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_XOR_COUNT__SHIFT 0x00000010
+#define TCP_CHAN_STEER_HI__CHAN8_MASK 0x0000000fL
+#define TCP_CHAN_STEER_HI__CHAN8__SHIFT 0x00000000
+#define TCP_CHAN_STEER_HI__CHAN9_MASK 0x000000f0L
+#define TCP_CHAN_STEER_HI__CHAN9__SHIFT 0x00000004
+#define TCP_CHAN_STEER_HI__CHANA_MASK 0x00000f00L
+#define TCP_CHAN_STEER_HI__CHANA__SHIFT 0x00000008
+#define TCP_CHAN_STEER_HI__CHANB_MASK 0x0000f000L
+#define TCP_CHAN_STEER_HI__CHANB__SHIFT 0x0000000c
+#define TCP_CHAN_STEER_HI__CHANC_MASK 0x000f0000L
+#define TCP_CHAN_STEER_HI__CHANC__SHIFT 0x00000010
+#define TCP_CHAN_STEER_HI__CHAND_MASK 0x00f00000L
+#define TCP_CHAN_STEER_HI__CHAND__SHIFT 0x00000014
+#define TCP_CHAN_STEER_HI__CHANE_MASK 0x0f000000L
+#define TCP_CHAN_STEER_HI__CHANE__SHIFT 0x00000018
+#define TCP_CHAN_STEER_HI__CHANF_MASK 0xf0000000L
+#define TCP_CHAN_STEER_HI__CHANF__SHIFT 0x0000001c
+#define TCP_CHAN_STEER_LO__CHAN0_MASK 0x0000000fL
+#define TCP_CHAN_STEER_LO__CHAN0__SHIFT 0x00000000
+#define TCP_CHAN_STEER_LO__CHAN1_MASK 0x000000f0L
+#define TCP_CHAN_STEER_LO__CHAN1__SHIFT 0x00000004
+#define TCP_CHAN_STEER_LO__CHAN2_MASK 0x00000f00L
+#define TCP_CHAN_STEER_LO__CHAN2__SHIFT 0x00000008
+#define TCP_CHAN_STEER_LO__CHAN3_MASK 0x0000f000L
+#define TCP_CHAN_STEER_LO__CHAN3__SHIFT 0x0000000c
+#define TCP_CHAN_STEER_LO__CHAN4_MASK 0x000f0000L
+#define TCP_CHAN_STEER_LO__CHAN4__SHIFT 0x00000010
+#define TCP_CHAN_STEER_LO__CHAN5_MASK 0x00f00000L
+#define TCP_CHAN_STEER_LO__CHAN5__SHIFT 0x00000014
+#define TCP_CHAN_STEER_LO__CHAN6_MASK 0x0f000000L
+#define TCP_CHAN_STEER_LO__CHAN6__SHIFT 0x00000018
+#define TCP_CHAN_STEER_LO__CHAN7_MASK 0xf0000000L
+#define TCP_CHAN_STEER_LO__CHAN7__SHIFT 0x0000001c
+#define TCP_CNTL__DISABLE_Z_MAP_MASK 0x10000000L
+#define TCP_CNTL__DISABLE_Z_MAP__SHIFT 0x0000001c
+#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE_MASK 0x00000020L
+#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE__SHIFT 0x00000005
+#define TCP_CNTL__FLAT_BUF_HASH_ENABLE_MASK 0x00000010L
+#define TCP_CNTL__FLAT_BUF_HASH_ENABLE__SHIFT 0x00000004
+#define TCP_CNTL__FORCE_EOW_TAGRAM_CNT_MASK 0x0fc00000L
+#define TCP_CNTL__FORCE_EOW_TAGRAM_CNT__SHIFT 0x00000016
+#define TCP_CNTL__FORCE_EOW_TOTAL_CNT_MASK 0x001f8000L
+#define TCP_CNTL__FORCE_EOW_TOTAL_CNT__SHIFT 0x0000000f
+#define TCP_CNTL__FORCE_HIT_MASK 0x00000001L
+#define TCP_CNTL__FORCE_HIT__SHIFT 0x00000000
+#define TCP_CNTL__FORCE_MISS_MASK 0x00000002L
+#define TCP_CNTL__FORCE_MISS__SHIFT 0x00000001
+#define TCP_CNTL__INV_ALL_VMIDS_MASK 0x20000000L
+#define TCP_CNTL__INV_ALL_VMIDS__SHIFT 0x0000001d
+#define TCP_CNTL__L1_SIZE_MASK 0x0000000cL
+#define TCP_CNTL__L1_SIZE__SHIFT 0x00000002
+#define TCP_CREDIT__LFIFO_CREDIT_MASK 0x000003ffL
+#define TCP_CREDIT__LFIFO_CREDIT__SHIFT 0x00000000
+#define TCP_CREDIT__REQ_FIFO_CREDIT_MASK 0x007f0000L
+#define TCP_CREDIT__REQ_FIFO_CREDIT__SHIFT 0x00000010
+#define TCP_CREDIT__TD_CREDIT_MASK 0xe0000000L
+#define TCP_CREDIT__TD_CREDIT__SHIFT 0x0000001d
+#define TCP_EDC_COUNTER__DED_COUNT_MASK 0x000f0000L
+#define TCP_EDC_COUNTER__DED_COUNT__SHIFT 0x00000010
+#define TCP_EDC_COUNTER__SEC_COUNT_MASK 0x0000000fL
+#define TCP_EDC_COUNTER__SEC_COUNT__SHIFT 0x00000000
+#define TCP_INVALIDATE__START_MASK 0x00000001L
+#define TCP_INVALIDATE__START__SHIFT 0x00000000
+#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000L
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x0000001c
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0x0f000000L
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x00000018
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf0000000L
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x0000001c
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0x0f000000L
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x00000018
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TCP_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TCP_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TCP_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TCP_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TCP_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TCP_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TCP_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TCP_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TCP_STATUS__TCP_BUSY_MASK 0x00000001L
+#define TCP_STATUS__TCP_BUSY__SHIFT 0x00000000
+#define TD_CGTT_CTRL__OFF_HYSTERESIS_MASK 0x00000ff0L
+#define TD_CGTT_CTRL__OFF_HYSTERESIS__SHIFT 0x00000004
+#define TD_CGTT_CTRL__ON_DELAY_MASK 0x0000000fL
+#define TD_CGTT_CTRL__ON_DELAY__SHIFT 0x00000000
+#define TD_CGTT_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT 0x0000001f
+#define TD_CGTT_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT 0x0000001e
+#define TD_CGTT_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT 0x0000001d
+#define TD_CGTT_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT 0x0000001c
+#define TD_CGTT_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT 0x0000001b
+#define TD_CGTT_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT 0x0000001a
+#define TD_CGTT_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT 0x00000019
+#define TD_CGTT_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT 0x00000018
+#define TD_CNTL__DISABLE_POWER_THROTTLE_MASK 0x00100000L
+#define TD_CNTL__DISABLE_POWER_THROTTLE__SHIFT 0x00000014
+#define TD_CNTL__EXTEND_LDS_STALL_MASK 0x00000600L
+#define TD_CNTL__EXTEND_LDS_STALL__SHIFT 0x00000009
+#define TD_CNTL__GATHER4_DX9_MODE_MASK 0x00080000L
+#define TD_CNTL__GATHER4_DX9_MODE__SHIFT 0x00000013
+#define TD_CNTL__GATHER4_FLOAT_MODE_MASK 0x00010000L
+#define TD_CNTL__GATHER4_FLOAT_MODE__SHIFT 0x00000010
+#define TD_CNTL__LD_FLOAT_MODE_MASK 0x00040000L
+#define TD_CNTL__LD_FLOAT_MODE__SHIFT 0x00000012
+#define TD_CNTL__LDS_STALL_PHASE_ADJUST_MASK 0x00001800L
+#define TD_CNTL__LDS_STALL_PHASE_ADJUST__SHIFT 0x0000000b
+#define TD_CNTL__PAD_STALL_EN_MASK 0x00000100L
+#define TD_CNTL__PAD_STALL_EN__SHIFT 0x00000008
+#define TD_CNTL__PRECISION_COMPATIBILITY_MASK 0x00008000L
+#define TD_CNTL__PRECISION_COMPATIBILITY__SHIFT 0x0000000f
+#define TD_CNTL__SYNC_PHASE_SH_MASK 0x00000003L
+#define TD_CNTL__SYNC_PHASE_SH__SHIFT 0x00000000
+#define TD_CNTL__SYNC_PHASE_VC_SMX_MASK 0x00000030L
+#define TD_CNTL__SYNC_PHASE_VC_SMX__SHIFT 0x00000004
+#define TD_DEBUG_DATA__DATA_MASK 0x00ffffffL
+#define TD_DEBUG_DATA__DATA__SHIFT 0x00000000
+#define TD_DEBUG_INDEX__INDEX_MASK 0x0000001fL
+#define TD_DEBUG_INDEX__INDEX__SHIFT 0x00000000
+#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000L
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x0000001c
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0x0f000000L
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x00000018
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x000000ffL
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x0003fc00L
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define TD_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define TD_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000L
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x0003fc00L
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x000000ffL
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define TD_SCRATCH__SCRATCH_MASK 0xffffffffL
+#define TD_SCRATCH__SCRATCH__SHIFT 0x00000000
+#define TD_STATUS__BUSY_MASK 0x80000000L
+#define TD_STATUS__BUSY__SHIFT 0x0000001f
+#define USER_SQC_BANK_DISABLE__SQC0_BANK_DISABLE_MASK 0x000f0000L
+#define USER_SQC_BANK_DISABLE__SQC0_BANK_DISABLE__SHIFT 0x00000010
+#define USER_SQC_BANK_DISABLE__SQC1_BANK_DISABLE_MASK 0x00f00000L
+#define USER_SQC_BANK_DISABLE__SQC1_BANK_DISABLE__SHIFT 0x00000014
+#define USER_SQC_BANK_DISABLE__SQC2_BANK_DISABLE_MASK 0x0f000000L
+#define USER_SQC_BANK_DISABLE__SQC2_BANK_DISABLE__SHIFT 0x00000018
+#define USER_SQC_BANK_DISABLE__SQC3_BANK_DISABLE_MASK 0xf0000000L
+#define USER_SQC_BANK_DISABLE__SQC3_BANK_DISABLE__SHIFT 0x0000001c
+#define VGT_CACHE_INVALIDATION__AUTO_INVLD_EN_MASK 0x000000c0L
+#define VGT_CACHE_INVALIDATION__AUTO_INVLD_EN__SHIFT 0x00000006
+#define VGT_CACHE_INVALIDATION__CACHE_INVALIDATION_MASK 0x00000003L
+#define VGT_CACHE_INVALIDATION__CACHE_INVALIDATION__SHIFT 0x00000000
+#define VGT_CACHE_INVALIDATION__DIS_RANGE_FULL_INVLD_MASK 0x00000800L
+#define VGT_CACHE_INVALIDATION__DIS_RANGE_FULL_INVLD__SHIFT 0x0000000b
+#define VGT_CACHE_INVALIDATION__ES_LIMIT_MASK 0x001f0000L
+#define VGT_CACHE_INVALIDATION__ES_LIMIT__SHIFT 0x00000010
+#define VGT_CACHE_INVALIDATION__GS_LATE_ALLOC_EN_MASK 0x00001000L
+#define VGT_CACHE_INVALIDATION__GS_LATE_ALLOC_EN__SHIFT 0x0000000c
+#define VGT_CACHE_INVALIDATION__STREAMOUT_FULL_FLUSH_MASK 0x00002000L
+#define VGT_CACHE_INVALIDATION__STREAMOUT_FULL_FLUSH__SHIFT 0x0000000d
+#define VGT_CACHE_INVALIDATION__USE_GS_DONE_MASK 0x00000200L
+#define VGT_CACHE_INVALIDATION__USE_GS_DONE__SHIFT 0x00000009
+#define VGT_CACHE_INVALIDATION__VS_NO_EXTRA_BUFFER_MASK 0x00000020L
+#define VGT_CACHE_INVALIDATION__VS_NO_EXTRA_BUFFER__SHIFT 0x00000005
+#define VGT_CNTL_STATUS__VGT_BUSY_MASK 0x00000001L
+#define VGT_CNTL_STATUS__VGT_BUSY__SHIFT 0x00000000
+#define VGT_CNTL_STATUS__VGT_GS_BUSY_MASK 0x00000080L
+#define VGT_CNTL_STATUS__VGT_GS_BUSY__SHIFT 0x00000007
+#define VGT_CNTL_STATUS__VGT_HS_BUSY_MASK 0x00000100L
+#define VGT_CNTL_STATUS__VGT_HS_BUSY__SHIFT 0x00000008
+#define VGT_CNTL_STATUS__VGT_OUT_BUSY_MASK 0x00000004L
+#define VGT_CNTL_STATUS__VGT_OUT_BUSY__SHIFT 0x00000002
+#define VGT_CNTL_STATUS__VGT_OUT_INDX_BUSY_MASK 0x00000002L
+#define VGT_CNTL_STATUS__VGT_OUT_INDX_BUSY__SHIFT 0x00000001
+#define VGT_CNTL_STATUS__VGT_PI_BUSY_MASK 0x00000040L
+#define VGT_CNTL_STATUS__VGT_PI_BUSY__SHIFT 0x00000006
+#define VGT_CNTL_STATUS__VGT_PT_BUSY_MASK 0x00000008L
+#define VGT_CNTL_STATUS__VGT_PT_BUSY__SHIFT 0x00000003
+#define VGT_CNTL_STATUS__VGT_TE11_BUSY_MASK 0x00000200L
+#define VGT_CNTL_STATUS__VGT_TE11_BUSY__SHIFT 0x00000009
+#define VGT_CNTL_STATUS__VGT_TE_BUSY_MASK 0x00000010L
+#define VGT_CNTL_STATUS__VGT_TE_BUSY__SHIFT 0x00000004
+#define VGT_CNTL_STATUS__VGT_VR_BUSY_MASK 0x00000020L
+#define VGT_CNTL_STATUS__VGT_VR_BUSY__SHIFT 0x00000005
+#define VGT_DEBUG_CNTL__VGT_DEBUG_INDX_MASK 0x0000003fL
+#define VGT_DEBUG_CNTL__VGT_DEBUG_INDX__SHIFT 0x00000000
+#define VGT_DEBUG_CNTL__VGT_DEBUG_SEL_BUS_B_MASK 0x00000040L
+#define VGT_DEBUG_CNTL__VGT_DEBUG_SEL_BUS_B__SHIFT 0x00000006
+#define VGT_DEBUG_DATA__DATA_MASK 0xffffffffL
+#define VGT_DEBUG_DATA__DATA__SHIFT 0x00000000
+#define VGT_DEBUG_REG0__cm_busy_MASK 0x00008000L
+#define VGT_DEBUG_REG0__cm_busy__SHIFT 0x0000000f
+#define VGT_DEBUG_REG0__combined_out_busy_MASK 0x00200000L
+#define VGT_DEBUG_REG0__combined_out_busy__SHIFT 0x00000015
+#define VGT_DEBUG_REG0__core_clk_busy_MASK 0x04000000L
+#define VGT_DEBUG_REG0__core_clk_busy__SHIFT 0x0000001a
+#define VGT_DEBUG_REG0__frmt_busy_MASK 0x00020000L
+#define VGT_DEBUG_REG0__frmt_busy__SHIFT 0x00000011
+#define VGT_DEBUG_REG0__gog_busy_MASK 0x00010000L
+#define VGT_DEBUG_REG0__gog_busy__SHIFT 0x00000010
+#define VGT_DEBUG_REG0__gs_busy_MASK 0x00001000L
+#define VGT_DEBUG_REG0__gs_busy__SHIFT 0x0000000c
+#define VGT_DEBUG_REG0__gs_clk_busy_MASK 0x08000000L
+#define VGT_DEBUG_REG0__gs_clk_busy__SHIFT 0x0000001b
+#define VGT_DEBUG_REG0__pa_interfaces_busy_MASK 0x00800000L
+#define VGT_DEBUG_REG0__pa_interfaces_busy__SHIFT 0x00000017
+#define VGT_DEBUG_REG0__pi_busy_MASK 0x00000100L
+#define VGT_DEBUG_REG0__pi_busy__SHIFT 0x00000008
+#define VGT_DEBUG_REG0__pt_pi_busy_MASK 0x00000400L
+#define VGT_DEBUG_REG0__pt_pi_busy__SHIFT 0x0000000a
+#define VGT_DEBUG_REG0__rcm_busy_MASK 0x00002000L
+#define VGT_DEBUG_REG0__rcm_busy__SHIFT 0x0000000d
+#define VGT_DEBUG_REG0__reg_clk_busy_MASK 0x01000000L
+#define VGT_DEBUG_REG0__reg_clk_busy__SHIFT 0x00000018
+#define VGT_DEBUG_REG0__sclk_core_vld_MASK 0x20000000L
+#define VGT_DEBUG_REG0__sclk_core_vld__SHIFT 0x0000001d
+#define VGT_DEBUG_REG0__sclk_gs_vld_MASK 0x40000000L
+#define VGT_DEBUG_REG0__sclk_gs_vld__SHIFT 0x0000001e
+#define VGT_DEBUG_REG0__SPARE0_MASK 0x80000000L
+#define VGT_DEBUG_REG0__SPARE0__SHIFT 0x0000001f
+#define VGT_DEBUG_REG0__SPARE10_MASK 0x00040000L
+#define VGT_DEBUG_REG0__SPARE10__SHIFT 0x00000012
+#define VGT_DEBUG_REG0__SPARE1_MASK 0x10000000L
+#define VGT_DEBUG_REG0__SPARE1__SHIFT 0x0000001c
+#define VGT_DEBUG_REG0__SPARE2_MASK 0x02000000L
+#define VGT_DEBUG_REG0__SPARE2__SHIFT 0x00000019
+#define VGT_DEBUG_REG0__SPARE3_MASK 0x00100000L
+#define VGT_DEBUG_REG0__SPARE3__SHIFT 0x00000014
+#define VGT_DEBUG_REG0__SPARE4_MASK 0x00000080L
+#define VGT_DEBUG_REG0__SPARE4__SHIFT 0x00000007
+#define VGT_DEBUG_REG0__SPARE5_MASK 0x00000040L
+#define VGT_DEBUG_REG0__SPARE5__SHIFT 0x00000006
+#define VGT_DEBUG_REG0__SPARE6_MASK 0x00000020L
+#define VGT_DEBUG_REG0__SPARE6__SHIFT 0x00000005
+#define VGT_DEBUG_REG0__SPARE7_MASK 0x00000010L
+#define VGT_DEBUG_REG0__SPARE7__SHIFT 0x00000004
+#define VGT_DEBUG_REG0__SPARE8_MASK 0x00000008L
+#define VGT_DEBUG_REG0__SPARE8__SHIFT 0x00000003
+#define VGT_DEBUG_REG0__SPARE9_MASK 0x00000002L
+#define VGT_DEBUG_REG0__SPARE9__SHIFT 0x00000001
+#define VGT_DEBUG_REG0__spi_vs_interfaces_busy_MASK 0x00400000L
+#define VGT_DEBUG_REG0__spi_vs_interfaces_busy__SHIFT 0x00000016
+#define VGT_DEBUG_REG0__te11_pi_busy_MASK 0x00080000L
+#define VGT_DEBUG_REG0__te11_pi_busy__SHIFT 0x00000013
+#define VGT_DEBUG_REG0__te_pi_busy_MASK 0x00000800L
+#define VGT_DEBUG_REG0__te_pi_busy__SHIFT 0x0000000b
+#define VGT_DEBUG_REG0__tm_busy_MASK 0x00004000L
+#define VGT_DEBUG_REG0__tm_busy__SHIFT 0x0000000e
+#define VGT_DEBUG_REG0__vgt_busy_extended_MASK 0x00000001L
+#define VGT_DEBUG_REG0__vgt_busy_extended__SHIFT 0x00000000
+#define VGT_DEBUG_REG0__vgt_busy_MASK 0x00000004L
+#define VGT_DEBUG_REG0__vgt_busy__SHIFT 0x00000002
+#define VGT_DEBUG_REG0__vr_pi_busy_MASK 0x00000200L
+#define VGT_DEBUG_REG0__vr_pi_busy__SHIFT 0x00000009
+#define VGT_DEBUG_REG10__eopg_r2_q_MASK 0x00000020L
+#define VGT_DEBUG_REG10__eopg_r2_q__SHIFT 0x00000005
+#define VGT_DEBUG_REG10__eotg_r2_q_MASK 0x00000040L
+#define VGT_DEBUG_REG10__eotg_r2_q__SHIFT 0x00000006
+#define VGT_DEBUG_REG10__es_rb_space_avail_r2_q_8_0_MASK 0xff800000L
+#define VGT_DEBUG_REG10__es_rb_space_avail_r2_q_8_0__SHIFT 0x00000017
+#define VGT_DEBUG_REG10__gs_rb_space_avail_r3_q_9_0_MASK 0x007fe000L
+#define VGT_DEBUG_REG10__gs_rb_space_avail_r3_q_9_0__SHIFT 0x0000000d
+#define VGT_DEBUG_REG10__index_buffer_depth_r1_q_MASK 0x0000001fL
+#define VGT_DEBUG_REG10__index_buffer_depth_r1_q__SHIFT 0x00000000
+#define VGT_DEBUG_REG10__onchip_gs_en_r0_q_MASK 0x00000180L
+#define VGT_DEBUG_REG10__onchip_gs_en_r0_q__SHIFT 0x00000007
+#define VGT_DEBUG_REG10__rcm_mem_gsprim_re_q_MASK 0x00001000L
+#define VGT_DEBUG_REG10__rcm_mem_gsprim_re_qq_MASK 0x00000800L
+#define VGT_DEBUG_REG10__rcm_mem_gsprim_re_qq__SHIFT 0x0000000b
+#define VGT_DEBUG_REG10__rcm_mem_gsprim_re_q__SHIFT 0x0000000c
+#define VGT_DEBUG_REG10__SPARE2_MASK 0x00000600L
+#define VGT_DEBUG_REG10__SPARE2__SHIFT 0x00000009
+#define VGT_DEBUG_REG11__counters_available_r0_MASK 0x00001000L
+#define VGT_DEBUG_REG11__counters_available_r0__SHIFT 0x0000000c
+#define VGT_DEBUG_REG11__counters_avail_r0_MASK 0x00000800L
+#define VGT_DEBUG_REG11__counters_avail_r0__SHIFT 0x0000000b
+#define VGT_DEBUG_REG11__counters_busy_r0_MASK 0x00000400L
+#define VGT_DEBUG_REG11__counters_busy_r0__SHIFT 0x0000000a
+#define VGT_DEBUG_REG11__es_r0_rtr_MASK 0x00100000L
+#define VGT_DEBUG_REG11__es_r0_rtr__SHIFT 0x00000014
+#define VGT_DEBUG_REG11__es_rb_dealloc_fifo_busy_MASK 0x00000008L
+#define VGT_DEBUG_REG11__es_rb_dealloc_fifo_busy__SHIFT 0x00000003
+#define VGT_DEBUG_REG11__es_rb_dealloc_fifo_full_MASK 0x08000000L
+#define VGT_DEBUG_REG11__es_rb_dealloc_fifo_full__SHIFT 0x0000001b
+#define VGT_DEBUG_REG11__es_rb_roll_over_r3_MASK 0x00000200L
+#define VGT_DEBUG_REG11__es_rb_roll_over_r3__SHIFT 0x00000009
+#define VGT_DEBUG_REG11__es_tbl_empty_MASK 0x40000000L
+#define VGT_DEBUG_REG11__es_tbl_empty__SHIFT 0x0000001e
+#define VGT_DEBUG_REG11__gog_tm_vs_event_rtr_MASK 0x00200000L
+#define VGT_DEBUG_REG11__gog_tm_vs_event_rtr__SHIFT 0x00000015
+#define VGT_DEBUG_REG11__gs_issue_rtr_MASK 0x00010000L
+#define VGT_DEBUG_REG11__gs_issue_rtr__SHIFT 0x00000010
+#define VGT_DEBUG_REG11__gs_r0_rtr_MASK 0x00080000L
+#define VGT_DEBUG_REG11__gs_r0_rtr__SHIFT 0x00000013
+#define VGT_DEBUG_REG11__hold_eswave_MASK 0x00000100L
+#define VGT_DEBUG_REG11__hold_eswave__SHIFT 0x00000008
+#define VGT_DEBUG_REG11__no_active_states_r0_MASK 0x80000000L
+#define VGT_DEBUG_REG11__no_active_states_r0__SHIFT 0x0000001f
+#define VGT_DEBUG_REG11__send_event_q_MASK 0x20000000L
+#define VGT_DEBUG_REG11__send_event_q__SHIFT 0x0000001d
+#define VGT_DEBUG_REG11__SPARE0_MASK 0x00040000L
+#define VGT_DEBUG_REG11__SPARE0__SHIFT 0x00000012
+#define VGT_DEBUG_REG11__SPARE1_MASK 0x00000020L
+#define VGT_DEBUG_REG11__SPARE1__SHIFT 0x00000005
+#define VGT_DEBUG_REG11__spi_esthread_fifo_busy_MASK 0x00000080L
+#define VGT_DEBUG_REG11__spi_esthread_fifo_busy__SHIFT 0x00000007
+#define VGT_DEBUG_REG11__spi_gsthread_fifo_busy_MASK 0x00000040L
+#define VGT_DEBUG_REG11__spi_gsthread_fifo_busy__SHIFT 0x00000006
+#define VGT_DEBUG_REG11__tm_busy_q_MASK 0x00000001L
+#define VGT_DEBUG_REG11__tm_busy_q__SHIFT 0x00000000
+#define VGT_DEBUG_REG11__tm_noif_busy_q_MASK 0x00000002L
+#define VGT_DEBUG_REG11__tm_noif_busy_q__SHIFT 0x00000001
+#define VGT_DEBUG_REG11__tm_out_busy_q_MASK 0x00000004L
+#define VGT_DEBUG_REG11__tm_out_busy_q__SHIFT 0x00000002
+#define VGT_DEBUG_REG11__tm_pt_event_rtr_MASK 0x00020000L
+#define VGT_DEBUG_REG11__tm_pt_event_rtr__SHIFT 0x00000011
+#define VGT_DEBUG_REG11__tm_rcm_es_tbl_rtr_MASK 0x01000000L
+#define VGT_DEBUG_REG11__tm_rcm_es_tbl_rtr__SHIFT 0x00000018
+#define VGT_DEBUG_REG11__tm_rcm_gs_event_rtr_MASK 0x00400000L
+#define VGT_DEBUG_REG11__tm_rcm_gs_event_rtr__SHIFT 0x00000016
+#define VGT_DEBUG_REG11__tm_rcm_gs_tbl_rtr_MASK 0x00800000L
+#define VGT_DEBUG_REG11__tm_rcm_gs_tbl_rtr__SHIFT 0x00000017
+#define VGT_DEBUG_REG11__VGT_SPI_esthread_rtr_q_MASK 0x00008000L
+#define VGT_DEBUG_REG11__VGT_SPI_esthread_rtr_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG11__VGT_SPI_gsthread_rtr_q_MASK 0x00004000L
+#define VGT_DEBUG_REG11__VGT_SPI_gsthread_rtr_q__SHIFT 0x0000000e
+#define VGT_DEBUG_REG11__vs_dealloc_tbl_busy_MASK 0x00000010L
+#define VGT_DEBUG_REG11__vs_dealloc_tbl_busy__SHIFT 0x00000004
+#define VGT_DEBUG_REG11__vs_dealloc_tbl_full_MASK 0x10000000L
+#define VGT_DEBUG_REG11__vs_dealloc_tbl_full__SHIFT 0x0000001c
+#define VGT_DEBUG_REG11__vs_event_fifo_empty_MASK 0x02000000L
+#define VGT_DEBUG_REG11__vs_event_fifo_empty__SHIFT 0x00000019
+#define VGT_DEBUG_REG11__vs_event_fifo_full_MASK 0x04000000L
+#define VGT_DEBUG_REG11__vs_event_fifo_full__SHIFT 0x0000001a
+#define VGT_DEBUG_REG11__vs_event_fifo_rtr_MASK 0x00002000L
+#define VGT_DEBUG_REG11__vs_event_fifo_rtr__SHIFT 0x0000000d
+#define VGT_DEBUG_REG12__gs_state0_r0_q_MASK 0x00000007L
+#define VGT_DEBUG_REG12__gs_state0_r0_q__SHIFT 0x00000000
+#define VGT_DEBUG_REG12__gs_state1_r0_q_MASK 0x00000038L
+#define VGT_DEBUG_REG12__gs_state1_r0_q__SHIFT 0x00000003
+#define VGT_DEBUG_REG12__gs_state2_r0_q_MASK 0x000001c0L
+#define VGT_DEBUG_REG12__gs_state2_r0_q__SHIFT 0x00000006
+#define VGT_DEBUG_REG12__gs_state3_r0_q_MASK 0x00000e00L
+#define VGT_DEBUG_REG12__gs_state3_r0_q__SHIFT 0x00000009
+#define VGT_DEBUG_REG12__gs_state4_r0_q_MASK 0x00007000L
+#define VGT_DEBUG_REG12__gs_state4_r0_q__SHIFT 0x0000000c
+#define VGT_DEBUG_REG12__gs_state5_r0_q_MASK 0x00038000L
+#define VGT_DEBUG_REG12__gs_state5_r0_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG12__gs_state6_r0_q_MASK 0x001c0000L
+#define VGT_DEBUG_REG12__gs_state6_r0_q__SHIFT 0x00000012
+#define VGT_DEBUG_REG12__gs_state7_r0_q_MASK 0x00e00000L
+#define VGT_DEBUG_REG12__gs_state7_r0_q__SHIFT 0x00000015
+#define VGT_DEBUG_REG12__gs_state8_r0_q_MASK 0x07000000L
+#define VGT_DEBUG_REG12__gs_state8_r0_q__SHIFT 0x00000018
+#define VGT_DEBUG_REG12__gs_state9_r0_q_MASK 0x38000000L
+#define VGT_DEBUG_REG12__gs_state9_r0_q__SHIFT 0x0000001b
+#define VGT_DEBUG_REG12__hold_eswave_eop_MASK 0x40000000L
+#define VGT_DEBUG_REG12__hold_eswave_eop__SHIFT 0x0000001e
+#define VGT_DEBUG_REG12__SPARE0_MASK 0x80000000L
+#define VGT_DEBUG_REG12__SPARE0__SHIFT 0x0000001f
+#define VGT_DEBUG_REG13__active_cm_sm_r0_q_MASK 0xf8000000L
+#define VGT_DEBUG_REG13__active_cm_sm_r0_q__SHIFT 0x0000001b
+#define VGT_DEBUG_REG13__es_tbl_full_MASK 0x01000000L
+#define VGT_DEBUG_REG13__es_tbl_full__SHIFT 0x00000018
+#define VGT_DEBUG_REG13__gsfetch_done_cnt_q_not_0_MASK 0x00800000L
+#define VGT_DEBUG_REG13__gsfetch_done_cnt_q_not_0__SHIFT 0x00000017
+#define VGT_DEBUG_REG13__gsfetch_done_fifo_cnt_q_not_0_MASK 0x00400000L
+#define VGT_DEBUG_REG13__gsfetch_done_fifo_cnt_q_not_0__SHIFT 0x00000016
+#define VGT_DEBUG_REG13__gs_state10_r0_q_MASK 0x00000007L
+#define VGT_DEBUG_REG13__gs_state10_r0_q__SHIFT 0x00000000
+#define VGT_DEBUG_REG13__gs_state11_r0_q_MASK 0x00000038L
+#define VGT_DEBUG_REG13__gs_state11_r0_q__SHIFT 0x00000003
+#define VGT_DEBUG_REG13__gs_state12_r0_q_MASK 0x000001c0L
+#define VGT_DEBUG_REG13__gs_state12_r0_q__SHIFT 0x00000006
+#define VGT_DEBUG_REG13__gs_state13_r0_q_MASK 0x00000e00L
+#define VGT_DEBUG_REG13__gs_state13_r0_q__SHIFT 0x00000009
+#define VGT_DEBUG_REG13__gs_state14_r0_q_MASK 0x00007000L
+#define VGT_DEBUG_REG13__gs_state14_r0_q__SHIFT 0x0000000c
+#define VGT_DEBUG_REG13__gs_state15_r0_q_MASK 0x00038000L
+#define VGT_DEBUG_REG13__gs_state15_r0_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG13__gs_tbl_wrptr_r0_q_3_0_MASK 0x003c0000L
+#define VGT_DEBUG_REG13__gs_tbl_wrptr_r0_q_3_0__SHIFT 0x00000012
+#define VGT_DEBUG_REG13__SPARE0_MASK 0x04000000L
+#define VGT_DEBUG_REG13__SPARE0__SHIFT 0x0000001a
+#define VGT_DEBUG_REG13__SPARE1_MASK 0x02000000L
+#define VGT_DEBUG_REG13__SPARE1__SHIFT 0x00000019
+#define VGT_DEBUG_REG14__es_flush_cnt_busy_q_MASK 0x00000400L
+#define VGT_DEBUG_REG14__es_flush_cnt_busy_q__SHIFT 0x0000000a
+#define VGT_DEBUG_REG14__gsfetch_done_fifo_full_MASK 0x00000010L
+#define VGT_DEBUG_REG14__gsfetch_done_fifo_full__SHIFT 0x00000004
+#define VGT_DEBUG_REG14__gsfetch_done_se1_cnt_q_not_0_MASK 0x20000000L
+#define VGT_DEBUG_REG14__gsfetch_done_se1_cnt_q_not_0__SHIFT 0x0000001d
+#define VGT_DEBUG_REG14__gs_rb_space_avail_r0_MASK 0x00000020L
+#define VGT_DEBUG_REG14__gs_rb_space_avail_r0__SHIFT 0x00000005
+#define VGT_DEBUG_REG14__gs_tbl_full_r0_MASK 0x00000800L
+#define VGT_DEBUG_REG14__gs_tbl_full_r0__SHIFT 0x0000000b
+#define VGT_DEBUG_REG14__se1spi_esthread_fifo_busy_MASK 0x08000000L
+#define VGT_DEBUG_REG14__se1spi_esthread_fifo_busy__SHIFT 0x0000001b
+#define VGT_DEBUG_REG14__se1spi_gsthread_fifo_busy_MASK 0x00200000L
+#define VGT_DEBUG_REG14__se1spi_gsthread_fifo_busy__SHIFT 0x00000015
+#define VGT_DEBUG_REG14__smx1_es_done_cnt_r0_q_not_0_MASK 0x04000000L
+#define VGT_DEBUG_REG14__smx1_es_done_cnt_r0_q_not_0__SHIFT 0x0000001a
+#define VGT_DEBUG_REG14__smx_es_done_cnt_r0_q_not_0_MASK 0x00000040L
+#define VGT_DEBUG_REG14__smx_es_done_cnt_r0_q_not_0__SHIFT 0x00000006
+#define VGT_DEBUG_REG14__SPARE0_MASK 0x40000000L
+#define VGT_DEBUG_REG14__SPARE0__SHIFT 0x0000001e
+#define VGT_DEBUG_REG14__SPARE1_MASK 0x10000000L
+#define VGT_DEBUG_REG14__SPARE1__SHIFT 0x0000001c
+#define VGT_DEBUG_REG14__SPARE2_MASK 0x001ff000L
+#define VGT_DEBUG_REG14__SPARE2__SHIFT 0x0000000c
+#define VGT_DEBUG_REG14__SPARE3_MASK 0x0000000fL
+#define VGT_DEBUG_REG14__SPARE3__SHIFT 0x00000000
+#define VGT_DEBUG_REG14__SPARE8_MASK 0x00000180L
+#define VGT_DEBUG_REG14__SPARE8__SHIFT 0x00000007
+#define VGT_DEBUG_REG14__SPARE_MASK 0x01c00000L
+#define VGT_DEBUG_REG14__SPARE__SHIFT 0x00000016
+#define VGT_DEBUG_REG14__VGT_SE1SPI_esthread_rtr_q_MASK 0x80000000L
+#define VGT_DEBUG_REG14__VGT_SE1SPI_esthread_rtr_q__SHIFT 0x0000001f
+#define VGT_DEBUG_REG14__VGT_SE1SPI_gsthread_rtr_q_MASK 0x02000000L
+#define VGT_DEBUG_REG14__VGT_SE1SPI_gsthread_rtr_q__SHIFT 0x00000019
+#define VGT_DEBUG_REG14__vs_done_cnt_q_not_0_MASK 0x00000200L
+#define VGT_DEBUG_REG14__vs_done_cnt_q_not_0__SHIFT 0x00000009
+#define VGT_DEBUG_REG15__active_sm_q_MASK 0x000003e0L
+#define VGT_DEBUG_REG15__active_sm_q__SHIFT 0x00000005
+#define VGT_DEBUG_REG15__cm_busy_q_MASK 0x00000001L
+#define VGT_DEBUG_REG15__cm_busy_q__SHIFT 0x00000000
+#define VGT_DEBUG_REG15__cntr_tbl_wrptr_q_MASK 0x000f8000L
+#define VGT_DEBUG_REG15__cntr_tbl_wrptr_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG15__counters_busy_q_MASK 0x00000002L
+#define VGT_DEBUG_REG15__counters_busy_q__SHIFT 0x00000001
+#define VGT_DEBUG_REG15__counters_full_MASK 0x00000010L
+#define VGT_DEBUG_REG15__counters_full__SHIFT 0x00000004
+#define VGT_DEBUG_REG15__entry_rdptr_q_MASK 0x00007c00L
+#define VGT_DEBUG_REG15__entry_rdptr_q__SHIFT 0x0000000a
+#define VGT_DEBUG_REG15__gs_done_array_q_not_0_MASK 0x10000000L
+#define VGT_DEBUG_REG15__gs_done_array_q_not_0__SHIFT 0x0000001c
+#define VGT_DEBUG_REG15__output_fifo_empty_MASK 0x00000004L
+#define VGT_DEBUG_REG15__output_fifo_empty__SHIFT 0x00000002
+#define VGT_DEBUG_REG15__output_fifo_full_MASK 0x00000008L
+#define VGT_DEBUG_REG15__output_fifo_full__SHIFT 0x00000003
+#define VGT_DEBUG_REG15__SPARE25_MASK 0x03f00000L
+#define VGT_DEBUG_REG15__SPARE25__SHIFT 0x00000014
+#define VGT_DEBUG_REG15__SPARE31_MASK 0xe0000000L
+#define VGT_DEBUG_REG15__SPARE31__SHIFT 0x0000001d
+#define VGT_DEBUG_REG15__st_cut_mode_q_MASK 0x0c000000L
+#define VGT_DEBUG_REG15__st_cut_mode_q__SHIFT 0x0000001a
+#define VGT_DEBUG_REG16__gog_busy_MASK 0x00000001L
+#define VGT_DEBUG_REG16__gog_busy__SHIFT 0x00000000
+#define VGT_DEBUG_REG16__gog_out_prim_state_sel_MASK 0x0e000000L
+#define VGT_DEBUG_REG16__gog_out_prim_state_sel__SHIFT 0x00000019
+#define VGT_DEBUG_REG16__gog_state_q_MASK 0x0000000eL
+#define VGT_DEBUG_REG16__gog_state_q__SHIFT 0x00000001
+#define VGT_DEBUG_REG16__gog_tm_vs_event_rtr_MASK 0x00000800L
+#define VGT_DEBUG_REG16__gog_tm_vs_event_rtr__SHIFT 0x0000000b
+#define VGT_DEBUG_REG16__indx_valid_r0_q_MASK 0x00080000L
+#define VGT_DEBUG_REG16__indx_valid_r0_q__SHIFT 0x00000013
+#define VGT_DEBUG_REG16__indx_valid_r1_q_MASK 0x00020000L
+#define VGT_DEBUG_REG16__indx_valid_r1_q__SHIFT 0x00000011
+#define VGT_DEBUG_REG16__indx_valid_r2_q_MASK 0x00002000L
+#define VGT_DEBUG_REG16__indx_valid_r2_q__SHIFT 0x0000000d
+#define VGT_DEBUG_REG16__multiple_streams_en_r1_q_MASK 0x10000000L
+#define VGT_DEBUG_REG16__multiple_streams_en_r1_q__SHIFT 0x0000001c
+#define VGT_DEBUG_REG16__new_vs_thread_r2_MASK 0x80000000L
+#define VGT_DEBUG_REG16__new_vs_thread_r2__SHIFT 0x0000001f
+#define VGT_DEBUG_REG16__num_gs_r2_q_not_0_MASK 0x40000000L
+#define VGT_DEBUG_REG16__num_gs_r2_q_not_0__SHIFT 0x0000001e
+#define VGT_DEBUG_REG16__prim_valid_r0_q_MASK 0x00100000L
+#define VGT_DEBUG_REG16__prim_valid_r0_q__SHIFT 0x00000014
+#define VGT_DEBUG_REG16__prim_valid_r1_q_MASK 0x00010000L
+#define VGT_DEBUG_REG16__prim_valid_r1_q__SHIFT 0x00000010
+#define VGT_DEBUG_REG16__prim_valid_r2_q_MASK 0x00004000L
+#define VGT_DEBUG_REG16__prim_valid_r2_q__SHIFT 0x0000000e
+#define VGT_DEBUG_REG16__r0_rtr_MASK 0x00000010L
+#define VGT_DEBUG_REG16__r0_rtr__SHIFT 0x00000004
+#define VGT_DEBUG_REG16__r1_rtr_MASK 0x00000020L
+#define VGT_DEBUG_REG16__r1_rtr__SHIFT 0x00000005
+#define VGT_DEBUG_REG16__r1_upstream_rtr_MASK 0x00000040L
+#define VGT_DEBUG_REG16__r1_upstream_rtr__SHIFT 0x00000006
+#define VGT_DEBUG_REG16__r2_indx_rtr_MASK 0x00000200L
+#define VGT_DEBUG_REG16__r2_indx_rtr__SHIFT 0x00000009
+#define VGT_DEBUG_REG16__r2_prim_rtr_MASK 0x00000100L
+#define VGT_DEBUG_REG16__r2_prim_rtr__SHIFT 0x00000008
+#define VGT_DEBUG_REG16__r2_rtr_MASK 0x00000400L
+#define VGT_DEBUG_REG16__r2_rtr__SHIFT 0x0000000a
+#define VGT_DEBUG_REG16__r2_vs_tbl_rtr_MASK 0x00000080L
+#define VGT_DEBUG_REG16__r2_vs_tbl_rtr__SHIFT 0x00000007
+#define VGT_DEBUG_REG16__r3_force_vs_tbl_we_rtr_MASK 0x00001000L
+#define VGT_DEBUG_REG16__r3_force_vs_tbl_we_rtr__SHIFT 0x0000000c
+#define VGT_DEBUG_REG16__send_event_q_MASK 0x00400000L
+#define VGT_DEBUG_REG16__send_event_q__SHIFT 0x00000016
+#define VGT_DEBUG_REG16__SPARE24_MASK 0x01800000L
+#define VGT_DEBUG_REG16__SPARE24__SHIFT 0x00000017
+#define VGT_DEBUG_REG16__valid_r0_q_MASK 0x00200000L
+#define VGT_DEBUG_REG16__valid_r0_q__SHIFT 0x00000015
+#define VGT_DEBUG_REG16__valid_r1_q_MASK 0x00040000L
+#define VGT_DEBUG_REG16__valid_r1_q__SHIFT 0x00000012
+#define VGT_DEBUG_REG16__valid_r2_q_MASK 0x00008000L
+#define VGT_DEBUG_REG16__valid_r2_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG16__vert_seen_since_sopg_r2_q_MASK 0x01000000L
+#define VGT_DEBUG_REG16__vert_seen_since_sopg_r2_q__SHIFT 0x00000018
+#define VGT_DEBUG_REG16__vs_vert_count_r2_q_not_0_MASK 0x20000000L
+#define VGT_DEBUG_REG16__vs_vert_count_r2_q_not_0__SHIFT 0x0000001d
+#define VGT_DEBUG_REG17__gog_out_indx_13_0_MASK 0xfffc0000L
+#define VGT_DEBUG_REG17__gog_out_indx_13_0__SHIFT 0x00000012
+#define VGT_DEBUG_REG17__gog_out_prim_rel_indx0_5_0_MASK 0x0003f000L
+#define VGT_DEBUG_REG17__gog_out_prim_rel_indx0_5_0__SHIFT 0x0000000c
+#define VGT_DEBUG_REG17__gog_out_prim_rel_indx1_5_0_MASK 0x00000fc0L
+#define VGT_DEBUG_REG17__gog_out_prim_rel_indx1_5_0__SHIFT 0x00000006
+#define VGT_DEBUG_REG17__gog_out_prim_rel_indx2_5_0_MASK 0x0000003fL
+#define VGT_DEBUG_REG17__gog_out_prim_rel_indx2_5_0__SHIFT 0x00000000
+#define VGT_DEBUG_REG18__components_valid_r0_q_MASK 0xe0000000L
+#define VGT_DEBUG_REG18__components_valid_r0_q__SHIFT 0x0000001d
+#define VGT_DEBUG_REG18__eject_vtx_vect_r1_d_MASK 0x00800000L
+#define VGT_DEBUG_REG18__eject_vtx_vect_r1_d__SHIFT 0x00000017
+#define VGT_DEBUG_REG18__eop_r0_q_MASK 0x00400000L
+#define VGT_DEBUG_REG18__eop_r0_q__SHIFT 0x00000016
+#define VGT_DEBUG_REG18__grp_vr_valid_MASK 0x00000001L
+#define VGT_DEBUG_REG18__grp_vr_valid__SHIFT 0x00000000
+#define VGT_DEBUG_REG18__gs_scenario_a_r0_q_MASK 0x08000000L
+#define VGT_DEBUG_REG18__gs_scenario_a_r0_q__SHIFT 0x0000001b
+#define VGT_DEBUG_REG18__gs_scenario_b_r0_q_MASK 0x10000000L
+#define VGT_DEBUG_REG18__gs_scenario_b_r0_q__SHIFT 0x0000001c
+#define VGT_DEBUG_REG18__indices_to_send_q_MASK 0x00000700L
+#define VGT_DEBUG_REG18__indices_to_send_q__SHIFT 0x00000008
+#define VGT_DEBUG_REG18__indx0_hit_d_MASK 0x00040000L
+#define VGT_DEBUG_REG18__indx0_hit_d__SHIFT 0x00000012
+#define VGT_DEBUG_REG18__indx0_new_d_MASK 0x00002000L
+#define VGT_DEBUG_REG18__indx0_new_d__SHIFT 0x0000000d
+#define VGT_DEBUG_REG18__indx1_hit_d_MASK 0x00020000L
+#define VGT_DEBUG_REG18__indx1_hit_d__SHIFT 0x00000011
+#define VGT_DEBUG_REG18__indx1_new_d_MASK 0x00004000L
+#define VGT_DEBUG_REG18__indx1_new_d__SHIFT 0x0000000e
+#define VGT_DEBUG_REG18__indx2_hit_d_MASK 0x00010000L
+#define VGT_DEBUG_REG18__indx2_hit_d__SHIFT 0x00000010
+#define VGT_DEBUG_REG18__indx2_new_d_MASK 0x00008000L
+#define VGT_DEBUG_REG18__indx2_new_d__SHIFT 0x0000000f
+#define VGT_DEBUG_REG18__last_group_of_instance_r0_q_MASK 0x00100000L
+#define VGT_DEBUG_REG18__last_group_of_instance_r0_q__SHIFT 0x00000014
+#define VGT_DEBUG_REG18__last_indx_of_prim_MASK 0x00001000L
+#define VGT_DEBUG_REG18__last_indx_of_prim__SHIFT 0x0000000c
+#define VGT_DEBUG_REG18__null_primitive_r0_q_MASK 0x00200000L
+#define VGT_DEBUG_REG18__null_primitive_r0_q__SHIFT 0x00000015
+#define VGT_DEBUG_REG18__out_vr_indx_read_MASK 0x00000040L
+#define VGT_DEBUG_REG18__out_vr_indx_read__SHIFT 0x00000006
+#define VGT_DEBUG_REG18__out_vr_prim_read_MASK 0x00000080L
+#define VGT_DEBUG_REG18__out_vr_prim_read__SHIFT 0x00000007
+#define VGT_DEBUG_REG18__pipe0_dr_MASK 0x00000002L
+#define VGT_DEBUG_REG18__pipe0_dr__SHIFT 0x00000001
+#define VGT_DEBUG_REG18__pipe0_rtr_MASK 0x00000010L
+#define VGT_DEBUG_REG18__pipe0_rtr__SHIFT 0x00000004
+#define VGT_DEBUG_REG18__pipe1_dr_MASK 0x00000004L
+#define VGT_DEBUG_REG18__pipe1_dr__SHIFT 0x00000002
+#define VGT_DEBUG_REG18__pipe1_rtr_MASK 0x00000020L
+#define VGT_DEBUG_REG18__pipe1_rtr__SHIFT 0x00000005
+#define VGT_DEBUG_REG18__st_vertex_reuse_off_r0_q_MASK 0x00080000L
+#define VGT_DEBUG_REG18__st_vertex_reuse_off_r0_q__SHIFT 0x00000013
+#define VGT_DEBUG_REG18__sub_prim_type_r0_q_MASK 0x07000000L
+#define VGT_DEBUG_REG18__sub_prim_type_r0_q__SHIFT 0x00000018
+#define VGT_DEBUG_REG18__valid_indices_MASK 0x00000800L
+#define VGT_DEBUG_REG18__valid_indices__SHIFT 0x0000000b
+#define VGT_DEBUG_REG18__vr_grp_read_MASK 0x00000008L
+#define VGT_DEBUG_REG18__vr_grp_read__SHIFT 0x00000003
+#define VGT_DEBUG_REG19__buffered_prim_eject_vtx_vect_MASK 0x00080000L
+#define VGT_DEBUG_REG19__buffered_prim_eject_vtx_vect__SHIFT 0x00000013
+#define VGT_DEBUG_REG19__buffered_prim_eop_MASK 0x00040000L
+#define VGT_DEBUG_REG19__buffered_prim_eop__SHIFT 0x00000012
+#define VGT_DEBUG_REG19__buffered_prim_event_MASK 0x00010000L
+#define VGT_DEBUG_REG19__buffered_prim_event__SHIFT 0x00000010
+#define VGT_DEBUG_REG19__buffered_prim_null_primitive_MASK 0x00020000L
+#define VGT_DEBUG_REG19__buffered_prim_null_primitive__SHIFT 0x00000011
+#define VGT_DEBUG_REG19__buffered_prim_type_event_MASK 0x03f00000L
+#define VGT_DEBUG_REG19__buffered_prim_type_event__SHIFT 0x00000014
+#define VGT_DEBUG_REG19__filter_event_MASK 0x80000000L
+#define VGT_DEBUG_REG19__filter_event__SHIFT 0x0000001f
+#define VGT_DEBUG_REG19__hold_prim_MASK 0x00000800L
+#define VGT_DEBUG_REG19__hold_prim__SHIFT 0x0000000b
+#define VGT_DEBUG_REG19__new_packet_q_MASK 0x00008000L
+#define VGT_DEBUG_REG19__new_packet_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG19__null_terminate_vtx_vector_MASK 0x40000000L
+#define VGT_DEBUG_REG19__null_terminate_vtx_vector__SHIFT 0x0000001e
+#define VGT_DEBUG_REG19__num_new_unique_rel_indx_MASK 0x30000000L
+#define VGT_DEBUG_REG19__num_new_unique_rel_indx__SHIFT 0x0000001c
+#define VGT_DEBUG_REG19__pa_clipp_fifo_busy_q_MASK 0x00000020L
+#define VGT_DEBUG_REG19__pa_clipp_fifo_busy_q__SHIFT 0x00000005
+#define VGT_DEBUG_REG19__pa_clips_fifo_busy_q_MASK 0x00000010L
+#define VGT_DEBUG_REG19__pa_clips_fifo_busy_q__SHIFT 0x00000004
+#define VGT_DEBUG_REG19__pa_clipv_fifo_busy_q_MASK 0x00000400L
+#define VGT_DEBUG_REG19__pa_clipv_fifo_busy_q__SHIFT 0x0000000a
+#define VGT_DEBUG_REG19__prim_buffer_empty_MASK 0x00000004L
+#define VGT_DEBUG_REG19__prim_buffer_empty__SHIFT 0x00000002
+#define VGT_DEBUG_REG19__prim_buffer_full_MASK 0x00000008L
+#define VGT_DEBUG_REG19__prim_buffer_full__SHIFT 0x00000003
+#define VGT_DEBUG_REG19__separate_out_busy_q_MASK 0x00000001L
+#define VGT_DEBUG_REG19__separate_out_busy_q__SHIFT 0x00000000
+#define VGT_DEBUG_REG19__separate_out_indx_busy_q_MASK 0x00000002L
+#define VGT_DEBUG_REG19__separate_out_indx_busy_q__SHIFT 0x00000001
+#define VGT_DEBUG_REG19__spi_vsthread_fifo_busy_q_MASK 0x00000100L
+#define VGT_DEBUG_REG19__spi_vsthread_fifo_busy_q__SHIFT 0x00000008
+#define VGT_DEBUG_REG19__spi_vsvert_fifo_busy_q_MASK 0x00000200L
+#define VGT_DEBUG_REG19__spi_vsvert_fifo_busy_q__SHIFT 0x00000009
+#define VGT_DEBUG_REG19__VGT_PA_clipp_rtr_q_MASK 0x00000080L
+#define VGT_DEBUG_REG19__VGT_PA_clipp_rtr_q__SHIFT 0x00000007
+#define VGT_DEBUG_REG19__VGT_PA_clips_rtr_q_MASK 0x00000040L
+#define VGT_DEBUG_REG19__VGT_PA_clips_rtr_q__SHIFT 0x00000006
+#define VGT_DEBUG_REG19__VGT_PA_clipv_rtr_q_MASK 0x00004000L
+#define VGT_DEBUG_REG19__VGT_PA_clipv_rtr_q__SHIFT 0x0000000e
+#define VGT_DEBUG_REG19__VGT_SE1SPI_vsvert_rtr_q_MASK 0x08000000L
+#define VGT_DEBUG_REG19__VGT_SE1SPI_vsvert_rtr_q__SHIFT 0x0000001b
+#define VGT_DEBUG_REG19__VGT_SE1SPI_vswave_rtr_q_MASK 0x04000000L
+#define VGT_DEBUG_REG19__VGT_SE1SPI_vswave_rtr_q__SHIFT 0x0000001a
+#define VGT_DEBUG_REG19__VGT_SPI_vsthread_rtr_q_MASK 0x00001000L
+#define VGT_DEBUG_REG19__VGT_SPI_vsthread_rtr_q__SHIFT 0x0000000c
+#define VGT_DEBUG_REG19__VGT_SPI_vsvert_rtr_q_MASK 0x00002000L
+#define VGT_DEBUG_REG19__VGT_SPI_vsvert_rtr_q__SHIFT 0x0000000d
+#define VGT_DEBUG_REG1__gog_out_indx_valid_MASK 0x10000000L
+#define VGT_DEBUG_REG1__gog_out_indx_valid__SHIFT 0x0000001c
+#define VGT_DEBUG_REG1__gog_out_prim_valid_MASK 0x40000000L
+#define VGT_DEBUG_REG1__gog_out_prim_valid__SHIFT 0x0000001e
+#define VGT_DEBUG_REG1__gs_pi_read_MASK 0x08000000L
+#define VGT_DEBUG_REG1__gs_pi_read__SHIFT 0x0000001b
+#define VGT_DEBUG_REG1__out_indx_read_MASK 0x20000000L
+#define VGT_DEBUG_REG1__out_indx_read__SHIFT 0x0000001d
+#define VGT_DEBUG_REG1__out_prim_read_MASK 0x80000000L
+#define VGT_DEBUG_REG1__out_prim_read__SHIFT 0x0000001f
+#define VGT_DEBUG_REG1__pi_gs_valid_MASK 0x04000000L
+#define VGT_DEBUG_REG1__pi_gs_valid__SHIFT 0x0000001a
+#define VGT_DEBUG_REG1__pi_pt_valid_MASK 0x00001000L
+#define VGT_DEBUG_REG1__pi_pt_valid__SHIFT 0x0000000c
+#define VGT_DEBUG_REG1__pi_te_valid_MASK 0x00004000L
+#define VGT_DEBUG_REG1__pi_te_valid__SHIFT 0x0000000e
+#define VGT_DEBUG_REG1__pi_vr_valid_MASK 0x00000400L
+#define VGT_DEBUG_REG1__pi_vr_valid__SHIFT 0x0000000a
+#define VGT_DEBUG_REG1__pt_out_indx_valid_MASK 0x00100000L
+#define VGT_DEBUG_REG1__pt_out_indx_valid__SHIFT 0x00000014
+#define VGT_DEBUG_REG1__pt_out_prim_valid_MASK 0x00400000L
+#define VGT_DEBUG_REG1__pt_out_prim_valid__SHIFT 0x00000016
+#define VGT_DEBUG_REG1__pt_pi_read_MASK 0x00002000L
+#define VGT_DEBUG_REG1__pt_pi_read__SHIFT 0x0000000d
+#define VGT_DEBUG_REG1__SPARE0_MASK 0x00000200L
+#define VGT_DEBUG_REG1__SPARE0__SHIFT 0x00000009
+#define VGT_DEBUG_REG1__SPARE10_MASK 0x00200000L
+#define VGT_DEBUG_REG1__SPARE10__SHIFT 0x00000015
+#define VGT_DEBUG_REG1__SPARE11_MASK 0x00080000L
+#define VGT_DEBUG_REG1__SPARE11__SHIFT 0x00000013
+#define VGT_DEBUG_REG1__SPARE12_MASK 0x00020000L
+#define VGT_DEBUG_REG1__SPARE12__SHIFT 0x00000011
+#define VGT_DEBUG_REG1__SPARE1_MASK 0x00000100L
+#define VGT_DEBUG_REG1__SPARE1__SHIFT 0x00000008
+#define VGT_DEBUG_REG1__SPARE23_MASK 0x00800000L
+#define VGT_DEBUG_REG1__SPARE23__SHIFT 0x00000017
+#define VGT_DEBUG_REG1__SPARE25_MASK 0x02000000L
+#define VGT_DEBUG_REG1__SPARE25__SHIFT 0x00000019
+#define VGT_DEBUG_REG1__SPARE2_MASK 0x00000080L
+#define VGT_DEBUG_REG1__SPARE2__SHIFT 0x00000007
+#define VGT_DEBUG_REG1__SPARE3_MASK 0x00000040L
+#define VGT_DEBUG_REG1__SPARE3__SHIFT 0x00000006
+#define VGT_DEBUG_REG1__SPARE4_MASK 0x00000020L
+#define VGT_DEBUG_REG1__SPARE4__SHIFT 0x00000005
+#define VGT_DEBUG_REG1__SPARE5_MASK 0x00000010L
+#define VGT_DEBUG_REG1__SPARE5__SHIFT 0x00000004
+#define VGT_DEBUG_REG1__SPARE6_MASK 0x00000008L
+#define VGT_DEBUG_REG1__SPARE6__SHIFT 0x00000003
+#define VGT_DEBUG_REG1__SPARE7_MASK 0x00000004L
+#define VGT_DEBUG_REG1__SPARE7__SHIFT 0x00000002
+#define VGT_DEBUG_REG1__SPARE8_MASK 0x00000002L
+#define VGT_DEBUG_REG1__SPARE8__SHIFT 0x00000001
+#define VGT_DEBUG_REG1__SPARE9_MASK 0x00000001L
+#define VGT_DEBUG_REG1__SPARE9__SHIFT 0x00000000
+#define VGT_DEBUG_REG1__te_grp_read_MASK 0x00008000L
+#define VGT_DEBUG_REG1__te_grp_read__SHIFT 0x0000000f
+#define VGT_DEBUG_REG1__te_out_data_valid_MASK 0x01000000L
+#define VGT_DEBUG_REG1__te_out_data_valid__SHIFT 0x00000018
+#define VGT_DEBUG_REG1__vr_out_indx_valid_MASK 0x00010000L
+#define VGT_DEBUG_REG1__vr_out_indx_valid__SHIFT 0x00000010
+#define VGT_DEBUG_REG1__vr_out_prim_valid_MASK 0x00040000L
+#define VGT_DEBUG_REG1__vr_out_prim_valid__SHIFT 0x00000012
+#define VGT_DEBUG_REG1__vr_pi_read_MASK 0x00000800L
+#define VGT_DEBUG_REG1__vr_pi_read__SHIFT 0x0000000b
+#define VGT_DEBUG_REG20__alloc_counter_q_MASK 0x003c0000L
+#define VGT_DEBUG_REG20__alloc_counter_q__SHIFT 0x00000012
+#define VGT_DEBUG_REG20__curr_dealloc_distance_q_MASK 0x1fc00000L
+#define VGT_DEBUG_REG20__curr_dealloc_distance_q__SHIFT 0x00000016
+#define VGT_DEBUG_REG20__curr_slot_in_vtx_vect_q_not_0_MASK 0x40000000L
+#define VGT_DEBUG_REG20__curr_slot_in_vtx_vect_q_not_0__SHIFT 0x0000001e
+#define VGT_DEBUG_REG20__dbg_VGT_SPI_vsthread_sovertexcount_not_0_MASK 0x00010000L
+#define VGT_DEBUG_REG20__dbg_VGT_SPI_vsthread_sovertexcount_not_0__SHIFT 0x00000010
+#define VGT_DEBUG_REG20__dbg_VGT_SPI_vsthread_sovertexindex_MASK 0x0000ffffL
+#define VGT_DEBUG_REG20__dbg_VGT_SPI_vsthread_sovertexindex__SHIFT 0x00000000
+#define VGT_DEBUG_REG20__int_vtx_counter_q_not_0_MASK 0x80000000L
+#define VGT_DEBUG_REG20__int_vtx_counter_q_not_0__SHIFT 0x0000001f
+#define VGT_DEBUG_REG20__new_allocate_q_MASK 0x20000000L
+#define VGT_DEBUG_REG20__new_allocate_q__SHIFT 0x0000001d
+#define VGT_DEBUG_REG20__SPARE17_MASK 0x00020000L
+#define VGT_DEBUG_REG20__SPARE17__SHIFT 0x00000011
+#define VGT_DEBUG_REG21__buff_full_p1_MASK 0x01000000L
+#define VGT_DEBUG_REG21__buff_full_p1__SHIFT 0x00000018
+#define VGT_DEBUG_REG21__eopg_p0_q_MASK 0x40000000L
+#define VGT_DEBUG_REG21__eopg_p0_q__SHIFT 0x0000001e
+#define VGT_DEBUG_REG21__eotg_r2_q_MASK 0x04000000L
+#define VGT_DEBUG_REG21__eotg_r2_q__SHIFT 0x0000001a
+#define VGT_DEBUG_REG21__full_state_p1_q_MASK 0x00008000L
+#define VGT_DEBUG_REG21__full_state_p1_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG21__indx_count_q_not_0_MASK 0x00002000L
+#define VGT_DEBUG_REG21__indx_count_q_not_0__SHIFT 0x0000000d
+#define VGT_DEBUG_REG21__indx_side_fifo_empty_MASK 0x00000002L
+#define VGT_DEBUG_REG21__indx_side_fifo_empty__SHIFT 0x00000001
+#define VGT_DEBUG_REG21__indx_side_fifo_full_MASK 0x00000080L
+#define VGT_DEBUG_REG21__indx_side_fifo_full__SHIFT 0x00000007
+#define VGT_DEBUG_REG21__indx_side_indx_valid_MASK 0x00010000L
+#define VGT_DEBUG_REG21__indx_side_indx_valid__SHIFT 0x00000010
+#define VGT_DEBUG_REG21__interfaces_rtr_MASK 0x00001000L
+#define VGT_DEBUG_REG21__interfaces_rtr__SHIFT 0x0000000c
+#define VGT_DEBUG_REG21__is_event_p0_q_MASK 0x00100000L
+#define VGT_DEBUG_REG21__is_event_p0_q__SHIFT 0x00000014
+#define VGT_DEBUG_REG21__lshs_dealloc_p1_MASK 0x00200000L
+#define VGT_DEBUG_REG21__lshs_dealloc_p1__SHIFT 0x00000015
+#define VGT_DEBUG_REG21__null_r2_q_MASK 0x08000000L
+#define VGT_DEBUG_REG21__null_r2_q__SHIFT 0x0000001b
+#define VGT_DEBUG_REG21__out_indx_fifo_empty_MASK 0x00000001L
+#define VGT_DEBUG_REG21__out_indx_fifo_empty__SHIFT 0x00000000
+#define VGT_DEBUG_REG21__out_indx_fifo_full_MASK 0x00000040L
+#define VGT_DEBUG_REG21__out_indx_fifo_full__SHIFT 0x00000006
+#define VGT_DEBUG_REG21__p0_dr_MASK 0x10000000L
+#define VGT_DEBUG_REG21__p0_dr__SHIFT 0x0000001c
+#define VGT_DEBUG_REG21__p0_nobp_MASK 0x80000000L
+#define VGT_DEBUG_REG21__p0_nobp__SHIFT 0x0000001f
+#define VGT_DEBUG_REG21__p0_rtr_MASK 0x20000000L
+#define VGT_DEBUG_REG21__p0_rtr__SHIFT 0x0000001d
+#define VGT_DEBUG_REG21__pipe0_dr_MASK 0x00000004L
+#define VGT_DEBUG_REG21__pipe0_dr__SHIFT 0x00000002
+#define VGT_DEBUG_REG21__pipe0_rtr_MASK 0x00000100L
+#define VGT_DEBUG_REG21__pipe0_rtr__SHIFT 0x00000008
+#define VGT_DEBUG_REG21__pipe1_dr_MASK 0x00000008L
+#define VGT_DEBUG_REG21__pipe1_dr__SHIFT 0x00000003
+#define VGT_DEBUG_REG21__pipe1_rtr_MASK 0x00000200L
+#define VGT_DEBUG_REG21__pipe1_rtr__SHIFT 0x00000009
+#define VGT_DEBUG_REG21__pipe2_dr_MASK 0x00000010L
+#define VGT_DEBUG_REG21__pipe2_dr__SHIFT 0x00000004
+#define VGT_DEBUG_REG21__pipe2_rtr_MASK 0x00000400L
+#define VGT_DEBUG_REG21__pipe2_rtr__SHIFT 0x0000000a
+#define VGT_DEBUG_REG21__stateid_p0_q_MASK 0x000e0000L
+#define VGT_DEBUG_REG21__stateid_p0_q__SHIFT 0x00000011
+#define VGT_DEBUG_REG21__stream_id_r2_q_MASK 0x00400000L
+#define VGT_DEBUG_REG21__stream_id_r2_q__SHIFT 0x00000016
+#define VGT_DEBUG_REG21__strmout_valid_p1_MASK 0x02000000L
+#define VGT_DEBUG_REG21__strmout_valid_p1__SHIFT 0x00000019
+#define VGT_DEBUG_REG21__vsthread_buff_empty_MASK 0x00000020L
+#define VGT_DEBUG_REG21__vsthread_buff_empty__SHIFT 0x00000005
+#define VGT_DEBUG_REG21__vsthread_buff_full_MASK 0x00000800L
+#define VGT_DEBUG_REG21__vsthread_buff_full__SHIFT 0x0000000b
+#define VGT_DEBUG_REG21__vtx_vect_counter_q_not_0_MASK 0x00800000L
+#define VGT_DEBUG_REG21__vtx_vect_counter_q_not_0__SHIFT 0x00000017
+#define VGT_DEBUG_REG21__wait_for_external_eopg_q_MASK 0x00004000L
+#define VGT_DEBUG_REG21__wait_for_external_eopg_q__SHIFT 0x0000000e
+#define VGT_DEBUG_REG22__cm_state16_MASK 0x00000003L
+#define VGT_DEBUG_REG22__cm_state16__SHIFT 0x00000000
+#define VGT_DEBUG_REG22__cm_state17_MASK 0x0000000cL
+#define VGT_DEBUG_REG22__cm_state17__SHIFT 0x00000002
+#define VGT_DEBUG_REG22__cm_state18_MASK 0x00000030L
+#define VGT_DEBUG_REG22__cm_state18__SHIFT 0x00000004
+#define VGT_DEBUG_REG22__cm_state19_MASK 0x000000c0L
+#define VGT_DEBUG_REG22__cm_state19__SHIFT 0x00000006
+#define VGT_DEBUG_REG22__cm_state20_MASK 0x00000300L
+#define VGT_DEBUG_REG22__cm_state20__SHIFT 0x00000008
+#define VGT_DEBUG_REG22__cm_state21_MASK 0x00000c00L
+#define VGT_DEBUG_REG22__cm_state21__SHIFT 0x0000000a
+#define VGT_DEBUG_REG22__cm_state22_MASK 0x00003000L
+#define VGT_DEBUG_REG22__cm_state22__SHIFT 0x0000000c
+#define VGT_DEBUG_REG22__cm_state23_MASK 0x0000c000L
+#define VGT_DEBUG_REG22__cm_state23__SHIFT 0x0000000e
+#define VGT_DEBUG_REG22__cm_state24_MASK 0x00030000L
+#define VGT_DEBUG_REG22__cm_state24__SHIFT 0x00000010
+#define VGT_DEBUG_REG22__cm_state25_MASK 0x000c0000L
+#define VGT_DEBUG_REG22__cm_state25__SHIFT 0x00000012
+#define VGT_DEBUG_REG22__cm_state26_MASK 0x00300000L
+#define VGT_DEBUG_REG22__cm_state26__SHIFT 0x00000014
+#define VGT_DEBUG_REG22__cm_state27_MASK 0x00c00000L
+#define VGT_DEBUG_REG22__cm_state27__SHIFT 0x00000016
+#define VGT_DEBUG_REG22__cm_state28_MASK 0x03000000L
+#define VGT_DEBUG_REG22__cm_state28__SHIFT 0x00000018
+#define VGT_DEBUG_REG22__cm_state29_MASK 0x0c000000L
+#define VGT_DEBUG_REG22__cm_state29__SHIFT 0x0000001a
+#define VGT_DEBUG_REG22__cm_state30_MASK 0x30000000L
+#define VGT_DEBUG_REG22__cm_state30__SHIFT 0x0000001c
+#define VGT_DEBUG_REG22__cm_state31_MASK 0xc0000000L
+#define VGT_DEBUG_REG22__cm_state31__SHIFT 0x0000001e
+#define VGT_DEBUG_REG23__frmt_busy_MASK 0x00000001L
+#define VGT_DEBUG_REG23__frmt_busy__SHIFT 0x00000000
+#define VGT_DEBUG_REG23__new_verts_r2_q_MASK 0x00018000L
+#define VGT_DEBUG_REG23__new_verts_r2_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG23__prim_dr_r2_q_MASK 0x00001000L
+#define VGT_DEBUG_REG23__prim_dr_r2_q__SHIFT 0x0000000c
+#define VGT_DEBUG_REG23__prim_fifo_empty_MASK 0x00000200L
+#define VGT_DEBUG_REG23__prim_fifo_empty__SHIFT 0x00000009
+#define VGT_DEBUG_REG23__prim_fifo_full_MASK 0x00000400L
+#define VGT_DEBUG_REG23__prim_fifo_full__SHIFT 0x0000000a
+#define VGT_DEBUG_REG23__prim_r2_rtr_MASK 0x00000010L
+#define VGT_DEBUG_REG23__prim_r2_rtr__SHIFT 0x00000004
+#define VGT_DEBUG_REG23__prim_r3_rtr_MASK 0x00000008L
+#define VGT_DEBUG_REG23__prim_r3_rtr__SHIFT 0x00000003
+#define VGT_DEBUG_REG23__prim_state_sel_r2_q_MASK 0x00e00000L
+#define VGT_DEBUG_REG23__prim_state_sel_r2_q__SHIFT 0x00000015
+#define VGT_DEBUG_REG23__rcm_frmt_prim_rtr_MASK 0x00000004L
+#define VGT_DEBUG_REG23__rcm_frmt_prim_rtr__SHIFT 0x00000002
+#define VGT_DEBUG_REG23__rcm_frmt_vert_rtr_MASK 0x00000002L
+#define VGT_DEBUG_REG23__rcm_frmt_vert_rtr__SHIFT 0x00000001
+#define VGT_DEBUG_REG23__SPARE_MASK 0xff000000L
+#define VGT_DEBUG_REG23__SPARE__SHIFT 0x00000018
+#define VGT_DEBUG_REG23__vert_dr_r0_q_MASK 0x00004000L
+#define VGT_DEBUG_REG23__vert_dr_r0_q__SHIFT 0x0000000e
+#define VGT_DEBUG_REG23__vert_dr_r1_q_MASK 0x00002000L
+#define VGT_DEBUG_REG23__vert_dr_r1_q__SHIFT 0x0000000d
+#define VGT_DEBUG_REG23__vert_dr_r2_q_MASK 0x00000800L
+#define VGT_DEBUG_REG23__vert_dr_r2_q__SHIFT 0x0000000b
+#define VGT_DEBUG_REG23__vert_r0_rtr_MASK 0x00000100L
+#define VGT_DEBUG_REG23__vert_r0_rtr__SHIFT 0x00000008
+#define VGT_DEBUG_REG23__vert_r1_rtr_MASK 0x00000080L
+#define VGT_DEBUG_REG23__vert_r1_rtr__SHIFT 0x00000007
+#define VGT_DEBUG_REG23__vert_r2_rtr_MASK 0x00000040L
+#define VGT_DEBUG_REG23__vert_r2_rtr__SHIFT 0x00000006
+#define VGT_DEBUG_REG23__vert_r3_rtr_MASK 0x00000020L
+#define VGT_DEBUG_REG23__vert_r3_rtr__SHIFT 0x00000005
+#define VGT_DEBUG_REG23__verts_sent_r2_q_MASK 0x001e0000L
+#define VGT_DEBUG_REG23__verts_sent_r2_q__SHIFT 0x00000011
+#define VGT_DEBUG_REG24__avail_es_rb_space_r0_q_23_0_MASK 0x00ffffffL
+#define VGT_DEBUG_REG24__avail_es_rb_space_r0_q_23_0__SHIFT 0x00000000
+#define VGT_DEBUG_REG24__dependent_st_cut_mode_q_MASK 0x03000000L
+#define VGT_DEBUG_REG24__dependent_st_cut_mode_q__SHIFT 0x00000018
+#define VGT_DEBUG_REG24__SPARE31_MASK 0xfc000000L
+#define VGT_DEBUG_REG24__SPARE31__SHIFT 0x0000001a
+#define VGT_DEBUG_REG25__active_sm_r0_q_MASK 0x3c000000L
+#define VGT_DEBUG_REG25__active_sm_r0_q__SHIFT 0x0000001a
+#define VGT_DEBUG_REG25__add_gs_rb_space_r0_q_MASK 0x80000000L
+#define VGT_DEBUG_REG25__add_gs_rb_space_r0_q__SHIFT 0x0000001f
+#define VGT_DEBUG_REG25__add_gs_rb_space_r1_q_MASK 0x40000000L
+#define VGT_DEBUG_REG25__add_gs_rb_space_r1_q__SHIFT 0x0000001e
+#define VGT_DEBUG_REG25__avail_gs_rb_space_r0_q_25_0_MASK 0x03ffffffL
+#define VGT_DEBUG_REG25__avail_gs_rb_space_r0_q_25_0__SHIFT 0x00000000
+#define VGT_DEBUG_REG26__cm_state0_MASK 0x00000003L
+#define VGT_DEBUG_REG26__cm_state0__SHIFT 0x00000000
+#define VGT_DEBUG_REG26__cm_state10_MASK 0x00300000L
+#define VGT_DEBUG_REG26__cm_state10__SHIFT 0x00000014
+#define VGT_DEBUG_REG26__cm_state11_MASK 0x00c00000L
+#define VGT_DEBUG_REG26__cm_state11__SHIFT 0x00000016
+#define VGT_DEBUG_REG26__cm_state12_MASK 0x03000000L
+#define VGT_DEBUG_REG26__cm_state12__SHIFT 0x00000018
+#define VGT_DEBUG_REG26__cm_state13_MASK 0x0c000000L
+#define VGT_DEBUG_REG26__cm_state13__SHIFT 0x0000001a
+#define VGT_DEBUG_REG26__cm_state14_MASK 0x30000000L
+#define VGT_DEBUG_REG26__cm_state14__SHIFT 0x0000001c
+#define VGT_DEBUG_REG26__cm_state15_MASK 0xc0000000L
+#define VGT_DEBUG_REG26__cm_state15__SHIFT 0x0000001e
+#define VGT_DEBUG_REG26__cm_state1_MASK 0x0000000cL
+#define VGT_DEBUG_REG26__cm_state1__SHIFT 0x00000002
+#define VGT_DEBUG_REG26__cm_state2_MASK 0x00000030L
+#define VGT_DEBUG_REG26__cm_state2__SHIFT 0x00000004
+#define VGT_DEBUG_REG26__cm_state3_MASK 0x000000c0L
+#define VGT_DEBUG_REG26__cm_state3__SHIFT 0x00000006
+#define VGT_DEBUG_REG26__cm_state4_MASK 0x00000300L
+#define VGT_DEBUG_REG26__cm_state4__SHIFT 0x00000008
+#define VGT_DEBUG_REG26__cm_state5_MASK 0x00000c00L
+#define VGT_DEBUG_REG26__cm_state5__SHIFT 0x0000000a
+#define VGT_DEBUG_REG26__cm_state6_MASK 0x00003000L
+#define VGT_DEBUG_REG26__cm_state6__SHIFT 0x0000000c
+#define VGT_DEBUG_REG26__cm_state7_MASK 0x0000c000L
+#define VGT_DEBUG_REG26__cm_state7__SHIFT 0x0000000e
+#define VGT_DEBUG_REG26__cm_state8_MASK 0x00030000L
+#define VGT_DEBUG_REG26__cm_state8__SHIFT 0x00000010
+#define VGT_DEBUG_REG26__cm_state9_MASK 0x000c0000L
+#define VGT_DEBUG_REG26__cm_state9__SHIFT 0x00000012
+#define VGT_DEBUG_REG27__eop_p1_q_MASK 0x00000800L
+#define VGT_DEBUG_REG27__eop_p1_q__SHIFT 0x0000000b
+#define VGT_DEBUG_REG27__event_flag_p1_q_MASK 0x00000400L
+#define VGT_DEBUG_REG27__event_flag_p1_q__SHIFT 0x0000000a
+#define VGT_DEBUG_REG27__first_vsprim_of_gsprim_p0_q_MASK 0x00080000L
+#define VGT_DEBUG_REG27__first_vsprim_of_gsprim_p0_q__SHIFT 0x00000013
+#define VGT_DEBUG_REG27__gsc0_dr_MASK 0x00000002L
+#define VGT_DEBUG_REG27__gsc0_dr__SHIFT 0x00000001
+#define VGT_DEBUG_REG27__gsc0_rtr_MASK 0x00000020L
+#define VGT_DEBUG_REG27__gsc0_rtr__SHIFT 0x00000005
+#define VGT_DEBUG_REG27__gsc_2cycle_output_MASK 0x00010000L
+#define VGT_DEBUG_REG27__gsc_2cycle_output__SHIFT 0x00000010
+#define VGT_DEBUG_REG27__gsc_2nd_cycle_p0_q_MASK 0x00020000L
+#define VGT_DEBUG_REG27__gsc_2nd_cycle_p0_q__SHIFT 0x00000011
+#define VGT_DEBUG_REG27__gsc_eop_p0_q_MASK 0x00008000L
+#define VGT_DEBUG_REG27__gsc_eop_p0_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG27__gsc_indx_count_p0_q_MASK 0x7ff00000L
+#define VGT_DEBUG_REG27__gsc_indx_count_p0_q__SHIFT 0x00000014
+#define VGT_DEBUG_REG27__gsc_null_primitive_p0_q_MASK 0x00004000L
+#define VGT_DEBUG_REG27__gsc_null_primitive_p0_q__SHIFT 0x0000000e
+#define VGT_DEBUG_REG27__gs_out_prim_type_p0_q_MASK 0x00003000L
+#define VGT_DEBUG_REG27__gs_out_prim_type_p0_q__SHIFT 0x0000000c
+#define VGT_DEBUG_REG27__indices_to_send_p0_q_MASK 0x00000300L
+#define VGT_DEBUG_REG27__indices_to_send_p0_q__SHIFT 0x00000008
+#define VGT_DEBUG_REG27__last_indx_of_prim_p1_q_MASK 0x00000080L
+#define VGT_DEBUG_REG27__last_indx_of_prim_p1_q__SHIFT 0x00000007
+#define VGT_DEBUG_REG27__last_indx_of_vsprim_MASK 0x00040000L
+#define VGT_DEBUG_REG27__last_indx_of_vsprim__SHIFT 0x00000012
+#define VGT_DEBUG_REG27__last_vsprim_of_gsprim_MASK 0x80000000L
+#define VGT_DEBUG_REG27__last_vsprim_of_gsprim__SHIFT 0x0000001f
+#define VGT_DEBUG_REG27__pipe0_dr_MASK 0x00000001L
+#define VGT_DEBUG_REG27__pipe0_dr__SHIFT 0x00000000
+#define VGT_DEBUG_REG27__pipe0_rtr_MASK 0x00000010L
+#define VGT_DEBUG_REG27__pipe0_rtr__SHIFT 0x00000004
+#define VGT_DEBUG_REG27__pipe1_dr_MASK 0x00000004L
+#define VGT_DEBUG_REG27__pipe1_dr__SHIFT 0x00000002
+#define VGT_DEBUG_REG27__pipe1_rtr_MASK 0x00000040L
+#define VGT_DEBUG_REG27__pipe1_rtr__SHIFT 0x00000006
+#define VGT_DEBUG_REG27__tm_pt_event_rtr_MASK 0x00000008L
+#define VGT_DEBUG_REG27__tm_pt_event_rtr__SHIFT 0x00000003
+#define VGT_DEBUG_REG28__advance_inner_point_p1_MASK 0x00800000L
+#define VGT_DEBUG_REG28__advance_inner_point_p1__SHIFT 0x00000017
+#define VGT_DEBUG_REG28__advance_outer_point_p1_MASK 0x00400000L
+#define VGT_DEBUG_REG28__advance_outer_point_p1__SHIFT 0x00000016
+#define VGT_DEBUG_REG28__con_state_q_MASK 0x0000000fL
+#define VGT_DEBUG_REG28__con_state_q__SHIFT 0x00000000
+#define VGT_DEBUG_REG28__first_ring_of_patch_p0_q_MASK 0x00010000L
+#define VGT_DEBUG_REG28__first_ring_of_patch_p0_q__SHIFT 0x00000010
+#define VGT_DEBUG_REG28__last_edge_of_outer_ring_p0_q_MASK 0x00040000L
+#define VGT_DEBUG_REG28__last_edge_of_outer_ring_p0_q__SHIFT 0x00000012
+#define VGT_DEBUG_REG28__last_point_of_inner_ring_p1_MASK 0x00100000L
+#define VGT_DEBUG_REG28__last_point_of_inner_ring_p1__SHIFT 0x00000014
+#define VGT_DEBUG_REG28__last_point_of_outer_ring_p1_MASK 0x00080000L
+#define VGT_DEBUG_REG28__last_point_of_outer_ring_p1__SHIFT 0x00000013
+#define VGT_DEBUG_REG28__last_ring_of_patch_p0_q_MASK 0x00020000L
+#define VGT_DEBUG_REG28__last_ring_of_patch_p0_q__SHIFT 0x00000011
+#define VGT_DEBUG_REG28__next_ring_is_rect_p0_q_MASK 0x01000000L
+#define VGT_DEBUG_REG28__next_ring_is_rect_p0_q__SHIFT 0x00000018
+#define VGT_DEBUG_REG28__outer_edge_tf_eq_one_p0_q_MASK 0x00200000L
+#define VGT_DEBUG_REG28__outer_edge_tf_eq_one_p0_q__SHIFT 0x00000015
+#define VGT_DEBUG_REG28__outer_parity_p0_q_MASK 0x00004000L
+#define VGT_DEBUG_REG28__outer_parity_p0_q__SHIFT 0x0000000e
+#define VGT_DEBUG_REG28__parallel_parity_p0_q_MASK 0x00008000L
+#define VGT_DEBUG_REG28__parallel_parity_p0_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG28__pipe0_edge_dr_MASK 0x00000200L
+#define VGT_DEBUG_REG28__pipe0_edge_dr__SHIFT 0x00000009
+#define VGT_DEBUG_REG28__pipe0_edge_rtr_MASK 0x00001000L
+#define VGT_DEBUG_REG28__pipe0_edge_rtr__SHIFT 0x0000000c
+#define VGT_DEBUG_REG28__pipe0_patch_dr_MASK 0x00000100L
+#define VGT_DEBUG_REG28__pipe0_patch_dr__SHIFT 0x00000008
+#define VGT_DEBUG_REG28__pipe0_patch_rtr_MASK 0x00000800L
+#define VGT_DEBUG_REG28__pipe0_patch_rtr__SHIFT 0x0000000b
+#define VGT_DEBUG_REG28__pipe1_dr_MASK 0x00000400L
+#define VGT_DEBUG_REG28__pipe1_dr__SHIFT 0x0000000a
+#define VGT_DEBUG_REG28__pipe1_edge_rtr_MASK 0x40000000L
+#define VGT_DEBUG_REG28__pipe1_edge_rtr__SHIFT 0x0000001e
+#define VGT_DEBUG_REG28__pipe1_inner1_rtr_MASK 0x08000000L
+#define VGT_DEBUG_REG28__pipe1_inner1_rtr__SHIFT 0x0000001b
+#define VGT_DEBUG_REG28__pipe1_inner2_rtr_MASK 0x10000000L
+#define VGT_DEBUG_REG28__pipe1_inner2_rtr__SHIFT 0x0000001c
+#define VGT_DEBUG_REG28__pipe1_outer1_rtr_MASK 0x02000000L
+#define VGT_DEBUG_REG28__pipe1_outer1_rtr__SHIFT 0x00000019
+#define VGT_DEBUG_REG28__pipe1_outer2_rtr_MASK 0x04000000L
+#define VGT_DEBUG_REG28__pipe1_outer2_rtr__SHIFT 0x0000001a
+#define VGT_DEBUG_REG28__pipe1_patch_rtr_MASK 0x20000000L
+#define VGT_DEBUG_REG28__pipe1_patch_rtr__SHIFT 0x0000001d
+#define VGT_DEBUG_REG28__pipe1_rtr_MASK 0x00002000L
+#define VGT_DEBUG_REG28__pipe1_rtr__SHIFT 0x0000000d
+#define VGT_DEBUG_REG28__process_tri_1st_2nd_half_p0_q_MASK 0x00000040L
+#define VGT_DEBUG_REG28__process_tri_1st_2nd_half_p0_q__SHIFT 0x00000006
+#define VGT_DEBUG_REG28__process_tri_center_poly_p0_q_MASK 0x00000080L
+#define VGT_DEBUG_REG28__process_tri_center_poly_p0_q__SHIFT 0x00000007
+#define VGT_DEBUG_REG28__process_tri_middle_p0_q_MASK 0x00000020L
+#define VGT_DEBUG_REG28__process_tri_middle_p0_q__SHIFT 0x00000005
+#define VGT_DEBUG_REG28__second_cycle_q_MASK 0x00000010L
+#define VGT_DEBUG_REG28__second_cycle_q__SHIFT 0x00000004
+#define VGT_DEBUG_REG28__use_stored_inner_q_ring2_MASK 0x80000000L
+#define VGT_DEBUG_REG28__use_stored_inner_q_ring2__SHIFT 0x0000001f
+#define VGT_DEBUG_REG29__advance_inner_point_p1_MASK 0x00800000L
+#define VGT_DEBUG_REG29__advance_inner_point_p1__SHIFT 0x00000017
+#define VGT_DEBUG_REG29__advance_outer_point_p1_MASK 0x00400000L
+#define VGT_DEBUG_REG29__advance_outer_point_p1__SHIFT 0x00000016
+#define VGT_DEBUG_REG29__con_state_q_MASK 0x0000000fL
+#define VGT_DEBUG_REG29__con_state_q__SHIFT 0x00000000
+#define VGT_DEBUG_REG29__first_ring_of_patch_p0_q_MASK 0x00010000L
+#define VGT_DEBUG_REG29__first_ring_of_patch_p0_q__SHIFT 0x00000010
+#define VGT_DEBUG_REG29__last_edge_of_outer_ring_p0_q_MASK 0x00040000L
+#define VGT_DEBUG_REG29__last_edge_of_outer_ring_p0_q__SHIFT 0x00000012
+#define VGT_DEBUG_REG29__last_point_of_inner_ring_p1_MASK 0x00100000L
+#define VGT_DEBUG_REG29__last_point_of_inner_ring_p1__SHIFT 0x00000014
+#define VGT_DEBUG_REG29__last_point_of_outer_ring_p1_MASK 0x00080000L
+#define VGT_DEBUG_REG29__last_point_of_outer_ring_p1__SHIFT 0x00000013
+#define VGT_DEBUG_REG29__last_ring_of_patch_p0_q_MASK 0x00020000L
+#define VGT_DEBUG_REG29__last_ring_of_patch_p0_q__SHIFT 0x00000011
+#define VGT_DEBUG_REG29__next_ring_is_rect_p0_q_MASK 0x01000000L
+#define VGT_DEBUG_REG29__next_ring_is_rect_p0_q__SHIFT 0x00000018
+#define VGT_DEBUG_REG29__outer_edge_tf_eq_one_p0_q_MASK 0x00200000L
+#define VGT_DEBUG_REG29__outer_edge_tf_eq_one_p0_q__SHIFT 0x00000015
+#define VGT_DEBUG_REG29__outer_parity_p0_q_MASK 0x00004000L
+#define VGT_DEBUG_REG29__outer_parity_p0_q__SHIFT 0x0000000e
+#define VGT_DEBUG_REG29__parallel_parity_p0_q_MASK 0x00008000L
+#define VGT_DEBUG_REG29__parallel_parity_p0_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG29__pipe0_edge_dr_MASK 0x00000200L
+#define VGT_DEBUG_REG29__pipe0_edge_dr__SHIFT 0x00000009
+#define VGT_DEBUG_REG29__pipe0_edge_rtr_MASK 0x00001000L
+#define VGT_DEBUG_REG29__pipe0_edge_rtr__SHIFT 0x0000000c
+#define VGT_DEBUG_REG29__pipe0_patch_dr_MASK 0x00000100L
+#define VGT_DEBUG_REG29__pipe0_patch_dr__SHIFT 0x00000008
+#define VGT_DEBUG_REG29__pipe0_patch_rtr_MASK 0x00000800L
+#define VGT_DEBUG_REG29__pipe0_patch_rtr__SHIFT 0x0000000b
+#define VGT_DEBUG_REG29__pipe1_dr_MASK 0x00000400L
+#define VGT_DEBUG_REG29__pipe1_dr__SHIFT 0x0000000a
+#define VGT_DEBUG_REG29__pipe1_edge_rtr_MASK 0x40000000L
+#define VGT_DEBUG_REG29__pipe1_edge_rtr__SHIFT 0x0000001e
+#define VGT_DEBUG_REG29__pipe1_inner1_rtr_MASK 0x08000000L
+#define VGT_DEBUG_REG29__pipe1_inner1_rtr__SHIFT 0x0000001b
+#define VGT_DEBUG_REG29__pipe1_inner2_rtr_MASK 0x10000000L
+#define VGT_DEBUG_REG29__pipe1_inner2_rtr__SHIFT 0x0000001c
+#define VGT_DEBUG_REG29__pipe1_outer1_rtr_MASK 0x02000000L
+#define VGT_DEBUG_REG29__pipe1_outer1_rtr__SHIFT 0x00000019
+#define VGT_DEBUG_REG29__pipe1_outer2_rtr_MASK 0x04000000L
+#define VGT_DEBUG_REG29__pipe1_outer2_rtr__SHIFT 0x0000001a
+#define VGT_DEBUG_REG29__pipe1_patch_rtr_MASK 0x20000000L
+#define VGT_DEBUG_REG29__pipe1_patch_rtr__SHIFT 0x0000001d
+#define VGT_DEBUG_REG29__pipe1_rtr_MASK 0x00002000L
+#define VGT_DEBUG_REG29__pipe1_rtr__SHIFT 0x0000000d
+#define VGT_DEBUG_REG29__process_tri_1st_2nd_half_p0_q_MASK 0x00000040L
+#define VGT_DEBUG_REG29__process_tri_1st_2nd_half_p0_q__SHIFT 0x00000006
+#define VGT_DEBUG_REG29__process_tri_center_poly_p0_q_MASK 0x00000080L
+#define VGT_DEBUG_REG29__process_tri_center_poly_p0_q__SHIFT 0x00000007
+#define VGT_DEBUG_REG29__process_tri_middle_p0_q_MASK 0x00000020L
+#define VGT_DEBUG_REG29__process_tri_middle_p0_q__SHIFT 0x00000005
+#define VGT_DEBUG_REG29__second_cycle_q_MASK 0x00000010L
+#define VGT_DEBUG_REG29__second_cycle_q__SHIFT 0x00000004
+#define VGT_DEBUG_REG29__use_stored_inner_q_ring3_MASK 0x80000000L
+#define VGT_DEBUG_REG29__use_stored_inner_q_ring3__SHIFT 0x0000001f
+#define VGT_DEBUG_REG2__grpModBusy_MASK 0x00000080L
+#define VGT_DEBUG_REG2__grpModBusy__SHIFT 0x00000007
+#define VGT_DEBUG_REG2__hs_grp_busy_MASK 0x00000001L
+#define VGT_DEBUG_REG2__hs_grp_busy__SHIFT 0x00000000
+#define VGT_DEBUG_REG2__hsInputFifoEmpty_MASK 0x00001000L
+#define VGT_DEBUG_REG2__hsInputFifoEmpty__SHIFT 0x0000000c
+#define VGT_DEBUG_REG2__hsInputFifoFull_MASK 0x00040000L
+#define VGT_DEBUG_REG2__hsInputFifoFull__SHIFT 0x00000012
+#define VGT_DEBUG_REG2__hs_noif_busy_MASK 0x00000002L
+#define VGT_DEBUG_REG2__hs_noif_busy__SHIFT 0x00000001
+#define VGT_DEBUG_REG2__hs_te11_tess_input_rts_MASK 0x00000040L
+#define VGT_DEBUG_REG2__hs_te11_tess_input_rts__SHIFT 0x00000006
+#define VGT_DEBUG_REG2__hsTifFifoEmpty_MASK 0x00002000L
+#define VGT_DEBUG_REG2__hsTifFifoEmpty__SHIFT 0x0000000d
+#define VGT_DEBUG_REG2__hsTifFifoFull_MASK 0x00080000L
+#define VGT_DEBUG_REG2__hsTifFifoFull__SHIFT 0x00000013
+#define VGT_DEBUG_REG2__hsVertFifoEmpty_MASK 0x00000400L
+#define VGT_DEBUG_REG2__hsVertFifoEmpty__SHIFT 0x0000000a
+#define VGT_DEBUG_REG2__hsVertFifoFull_MASK 0x00010000L
+#define VGT_DEBUG_REG2__hsVertFifoFull__SHIFT 0x00000010
+#define VGT_DEBUG_REG2__hsWaveFifoEmpty_MASK 0x00000800L
+#define VGT_DEBUG_REG2__hsWaveFifoEmpty__SHIFT 0x0000000b
+#define VGT_DEBUG_REG2__hsWaveFifoFull_MASK 0x00020000L
+#define VGT_DEBUG_REG2__hsWaveFifoFull__SHIFT 0x00000011
+#define VGT_DEBUG_REG2__lsFwaveFlag_MASK 0x08000000L
+#define VGT_DEBUG_REG2__lsFwaveFlag__SHIFT 0x0000001b
+#define VGT_DEBUG_REG2__ls_sh_id_MASK 0x04000000L
+#define VGT_DEBUG_REG2__ls_sh_id__SHIFT 0x0000001a
+#define VGT_DEBUG_REG2__lsVertFifoEmpty_MASK 0x00000100L
+#define VGT_DEBUG_REG2__lsVertFifoEmpty__SHIFT 0x00000008
+#define VGT_DEBUG_REG2__lsVertFifoFull_MASK 0x00004000L
+#define VGT_DEBUG_REG2__lsVertFifoFull__SHIFT 0x0000000e
+#define VGT_DEBUG_REG2__lsVertIfBusy_0_MASK 0x00000008L
+#define VGT_DEBUG_REG2__lsVertIfBusy_0__SHIFT 0x00000003
+#define VGT_DEBUG_REG2__lsWaveFifoEmpty_MASK 0x00000200L
+#define VGT_DEBUG_REG2__lsWaveFifoEmpty__SHIFT 0x00000009
+#define VGT_DEBUG_REG2__lsWaveFifoFull_MASK 0x00008000L
+#define VGT_DEBUG_REG2__lsWaveFifoFull__SHIFT 0x0000000f
+#define VGT_DEBUG_REG2__lsWaveIfBusy_0_MASK 0x00000020L
+#define VGT_DEBUG_REG2__lsWaveIfBusy_0__SHIFT 0x00000005
+#define VGT_DEBUG_REG2__lsWaveSendFlush_MASK 0x10000000L
+#define VGT_DEBUG_REG2__lsWaveSendFlush__SHIFT 0x0000001c
+#define VGT_DEBUG_REG2__p0_dr_MASK 0x00400000L
+#define VGT_DEBUG_REG2__p0_dr__SHIFT 0x00000016
+#define VGT_DEBUG_REG2__p0_rtr_MASK 0x00100000L
+#define VGT_DEBUG_REG2__p0_rtr__SHIFT 0x00000014
+#define VGT_DEBUG_REG2__p0_rts_MASK 0x01000000L
+#define VGT_DEBUG_REG2__p0_rts__SHIFT 0x00000018
+#define VGT_DEBUG_REG2__p1_dr_MASK 0x00800000L
+#define VGT_DEBUG_REG2__p1_dr__SHIFT 0x00000017
+#define VGT_DEBUG_REG2__p1_rtr_MASK 0x00200000L
+#define VGT_DEBUG_REG2__p1_rtr__SHIFT 0x00000015
+#define VGT_DEBUG_REG2__p1_rts_MASK 0x02000000L
+#define VGT_DEBUG_REG2__p1_rts__SHIFT 0x00000019
+#define VGT_DEBUG_REG2__SPARE_MASK 0xffffffffL
+#define VGT_DEBUG_REG2__SPARE__SHIFT 0x00000000
+#define VGT_DEBUG_REG2__te11_hs_tess_input_rtr_MASK 0x00000010L
+#define VGT_DEBUG_REG2__te11_hs_tess_input_rtr__SHIFT 0x00000004
+#define VGT_DEBUG_REG2__tfmmIsBusy_MASK 0x00000004L
+#define VGT_DEBUG_REG2__tfmmIsBusy__SHIFT 0x00000002
+#define VGT_DEBUG_REG30__dynamic_hs_p0_q_MASK 0x01000000L
+#define VGT_DEBUG_REG30__dynamic_hs_p0_q__SHIFT 0x00000018
+#define VGT_DEBUG_REG30__event_or_null_p0_q_MASK 0x00000008L
+#define VGT_DEBUG_REG30__event_or_null_p0_q__SHIFT 0x00000003
+#define VGT_DEBUG_REG30__first_data_chunk_invalid_p0_q_MASK 0x08000000L
+#define VGT_DEBUG_REG30__first_data_chunk_invalid_p0_q__SHIFT 0x0000001b
+#define VGT_DEBUG_REG30__first_data_ret_of_req_p0_q_MASK 0x04000000L
+#define VGT_DEBUG_REG30__first_data_ret_of_req_p0_q__SHIFT 0x0000001a
+#define VGT_DEBUG_REG30__first_fetch_of_tg_p0_q_MASK 0x02000000L
+#define VGT_DEBUG_REG30__first_fetch_of_tg_p0_q__SHIFT 0x00000019
+#define VGT_DEBUG_REG30__last_tf_of_tg_MASK 0x00080000L
+#define VGT_DEBUG_REG30__last_tf_of_tg__SHIFT 0x00000013
+#define VGT_DEBUG_REG30__pipe0_dr_MASK 0x00000001L
+#define VGT_DEBUG_REG30__pipe0_dr__SHIFT 0x00000000
+#define VGT_DEBUG_REG30__pipe0_rtr_MASK 0x00000010L
+#define VGT_DEBUG_REG30__pipe0_rtr__SHIFT 0x00000004
+#define VGT_DEBUG_REG30__pipe0_tf_dr_MASK 0x00000002L
+#define VGT_DEBUG_REG30__pipe0_tf_dr__SHIFT 0x00000001
+#define VGT_DEBUG_REG30__pipe1_rtr_MASK 0x00000020L
+#define VGT_DEBUG_REG30__pipe1_rtr__SHIFT 0x00000005
+#define VGT_DEBUG_REG30__pipe1_tf_rtr_MASK 0x00000040L
+#define VGT_DEBUG_REG30__pipe1_tf_rtr__SHIFT 0x00000006
+#define VGT_DEBUG_REG30__pipe2_dr_MASK 0x00000004L
+#define VGT_DEBUG_REG30__pipe2_dr__SHIFT 0x00000002
+#define VGT_DEBUG_REG30__pipe2_rtr_MASK 0x00000080L
+#define VGT_DEBUG_REG30__pipe2_rtr__SHIFT 0x00000007
+#define VGT_DEBUG_REG30__pipe4_dr_MASK 0x40000000L
+#define VGT_DEBUG_REG30__pipe4_dr__SHIFT 0x0000001e
+#define VGT_DEBUG_REG30__pipe4_rtr_MASK 0x80000000L
+#define VGT_DEBUG_REG30__pipe4_rtr__SHIFT 0x0000001f
+#define VGT_DEBUG_REG30__tf_fetch_state_q_MASK 0x00070000L
+#define VGT_DEBUG_REG30__tf_fetch_state_q__SHIFT 0x00000010
+#define VGT_DEBUG_REG30__tf_pointer_p0_q_MASK 0x00f00000L
+#define VGT_DEBUG_REG30__tf_pointer_p0_q__SHIFT 0x00000014
+#define VGT_DEBUG_REG30__tf_xfer_count_p2_q_MASK 0x30000000L
+#define VGT_DEBUG_REG30__tf_xfer_count_p2_q__SHIFT 0x0000001c
+#define VGT_DEBUG_REG30__ttp_patch_fifo_empty_MASK 0x00000200L
+#define VGT_DEBUG_REG30__ttp_patch_fifo_empty__SHIFT 0x00000009
+#define VGT_DEBUG_REG30__ttp_patch_fifo_full_MASK 0x00000100L
+#define VGT_DEBUG_REG30__ttp_patch_fifo_full__SHIFT 0x00000008
+#define VGT_DEBUG_REG30__ttp_tf0_fifo_empty_MASK 0x00000400L
+#define VGT_DEBUG_REG30__ttp_tf0_fifo_empty__SHIFT 0x0000000a
+#define VGT_DEBUG_REG30__ttp_tf1_fifo_empty_MASK 0x00000800L
+#define VGT_DEBUG_REG30__ttp_tf1_fifo_empty__SHIFT 0x0000000b
+#define VGT_DEBUG_REG30__ttp_tf2_fifo_empty_MASK 0x00001000L
+#define VGT_DEBUG_REG30__ttp_tf2_fifo_empty__SHIFT 0x0000000c
+#define VGT_DEBUG_REG30__ttp_tf3_fifo_empty_MASK 0x00002000L
+#define VGT_DEBUG_REG30__ttp_tf3_fifo_empty__SHIFT 0x0000000d
+#define VGT_DEBUG_REG30__ttp_tf4_fifo_empty_MASK 0x00004000L
+#define VGT_DEBUG_REG30__ttp_tf4_fifo_empty__SHIFT 0x0000000e
+#define VGT_DEBUG_REG30__ttp_tf5_fifo_empty_MASK 0x00008000L
+#define VGT_DEBUG_REG30__ttp_tf5_fifo_empty__SHIFT 0x0000000f
+#define VGT_DEBUG_REG31__inner_ring_done_q_MASK 0x80000000L
+#define VGT_DEBUG_REG31__inner_ring_done_q__SHIFT 0x0000001f
+#define VGT_DEBUG_REG31__outer_ring_done_q_MASK 0x40000000L
+#define VGT_DEBUG_REG31__outer_ring_done_q__SHIFT 0x0000001e
+#define VGT_DEBUG_REG31__pg_con_inner_point1_rts_MASK 0x00400000L
+#define VGT_DEBUG_REG31__pg_con_inner_point1_rts__SHIFT 0x00000016
+#define VGT_DEBUG_REG31__pg_con_inner_point2_rts_MASK 0x00800000L
+#define VGT_DEBUG_REG31__pg_con_inner_point2_rts__SHIFT 0x00000017
+#define VGT_DEBUG_REG31__pg_con_outer_point1_rts_MASK 0x00100000L
+#define VGT_DEBUG_REG31__pg_con_outer_point1_rts__SHIFT 0x00000014
+#define VGT_DEBUG_REG31__pg_con_outer_point2_rts_MASK 0x00200000L
+#define VGT_DEBUG_REG31__pg_con_outer_point2_rts__SHIFT 0x00000015
+#define VGT_DEBUG_REG31__pg_edge_fifo_empty_MASK 0x02000000L
+#define VGT_DEBUG_REG31__pg_edge_fifo_empty__SHIFT 0x00000019
+#define VGT_DEBUG_REG31__pg_edge_fifo_full_MASK 0x10000000L
+#define VGT_DEBUG_REG31__pg_edge_fifo_full__SHIFT 0x0000001c
+#define VGT_DEBUG_REG31__pg_inner3_perp_fifo_empty_MASK 0x04000000L
+#define VGT_DEBUG_REG31__pg_inner3_perp_fifo_empty__SHIFT 0x0000001a
+#define VGT_DEBUG_REG31__pg_inner_perp_fifo_full_MASK 0x20000000L
+#define VGT_DEBUG_REG31__pg_inner_perp_fifo_full__SHIFT 0x0000001d
+#define VGT_DEBUG_REG31__pg_patch_fifo_empty_MASK 0x01000000L
+#define VGT_DEBUG_REG31__pg_patch_fifo_empty__SHIFT 0x00000018
+#define VGT_DEBUG_REG31__pg_patch_fifo_full_MASK 0x08000000L
+#define VGT_DEBUG_REG31__pg_patch_fifo_full__SHIFT 0x0000001b
+#define VGT_DEBUG_REG31__pipe0_dr_MASK 0x00000001L
+#define VGT_DEBUG_REG31__pipe0_dr__SHIFT 0x00000000
+#define VGT_DEBUG_REG31__pipe0_rtr_MASK 0x00000002L
+#define VGT_DEBUG_REG31__pipe0_rtr__SHIFT 0x00000001
+#define VGT_DEBUG_REG31__pipe1_inner_dr_MASK 0x00000008L
+#define VGT_DEBUG_REG31__pipe1_inner_dr__SHIFT 0x00000003
+#define VGT_DEBUG_REG31__pipe1_outer_dr_MASK 0x00000004L
+#define VGT_DEBUG_REG31__pipe1_outer_dr__SHIFT 0x00000002
+#define VGT_DEBUG_REG31__pipe2_inner_dr_MASK 0x00000020L
+#define VGT_DEBUG_REG31__pipe2_inner_dr__SHIFT 0x00000005
+#define VGT_DEBUG_REG31__pipe2_inner_rtr_MASK 0x00002000L
+#define VGT_DEBUG_REG31__pipe2_inner_rtr__SHIFT 0x0000000d
+#define VGT_DEBUG_REG31__pipe2_outer_dr_MASK 0x00000010L
+#define VGT_DEBUG_REG31__pipe2_outer_dr__SHIFT 0x00000004
+#define VGT_DEBUG_REG31__pipe2_outer_rtr_MASK 0x00001000L
+#define VGT_DEBUG_REG31__pipe2_outer_rtr__SHIFT 0x0000000c
+#define VGT_DEBUG_REG31__pipe3_inner_dr_MASK 0x00000080L
+#define VGT_DEBUG_REG31__pipe3_inner_dr__SHIFT 0x00000007
+#define VGT_DEBUG_REG31__pipe3_inner_rtr_MASK 0x00008000L
+#define VGT_DEBUG_REG31__pipe3_inner_rtr__SHIFT 0x0000000f
+#define VGT_DEBUG_REG31__pipe3_outer_dr_MASK 0x00000040L
+#define VGT_DEBUG_REG31__pipe3_outer_dr__SHIFT 0x00000006
+#define VGT_DEBUG_REG31__pipe3_outer_rtr_MASK 0x00004000L
+#define VGT_DEBUG_REG31__pipe3_outer_rtr__SHIFT 0x0000000e
+#define VGT_DEBUG_REG31__pipe4_inner_dr_MASK 0x00000200L
+#define VGT_DEBUG_REG31__pipe4_inner_dr__SHIFT 0x00000009
+#define VGT_DEBUG_REG31__pipe4_inner_rtr_MASK 0x00020000L
+#define VGT_DEBUG_REG31__pipe4_inner_rtr__SHIFT 0x00000011
+#define VGT_DEBUG_REG31__pipe4_outer_dr_MASK 0x00000100L
+#define VGT_DEBUG_REG31__pipe4_outer_dr__SHIFT 0x00000008
+#define VGT_DEBUG_REG31__pipe4_outer_rtr_MASK 0x00010000L
+#define VGT_DEBUG_REG31__pipe4_outer_rtr__SHIFT 0x00000010
+#define VGT_DEBUG_REG31__pipe5_inner_dr_MASK 0x00000800L
+#define VGT_DEBUG_REG31__pipe5_inner_dr__SHIFT 0x0000000b
+#define VGT_DEBUG_REG31__pipe5_inner_rtr_MASK 0x00080000L
+#define VGT_DEBUG_REG31__pipe5_inner_rtr__SHIFT 0x00000013
+#define VGT_DEBUG_REG31__pipe5_outer_dr_MASK 0x00000400L
+#define VGT_DEBUG_REG31__pipe5_outer_dr__SHIFT 0x0000000a
+#define VGT_DEBUG_REG31__pipe5_outer_rtr_MASK 0x00040000L
+#define VGT_DEBUG_REG31__pipe5_outer_rtr__SHIFT 0x00000012
+#define VGT_DEBUG_REG32__event_flag_p5_q_MASK 0x00000100L
+#define VGT_DEBUG_REG32__event_flag_p5_q__SHIFT 0x00000008
+#define VGT_DEBUG_REG32__event_null_special_p0_q_MASK 0x00000080L
+#define VGT_DEBUG_REG32__event_null_special_p0_q__SHIFT 0x00000007
+#define VGT_DEBUG_REG32__fifos_rtr_MASK 0x08000000L
+#define VGT_DEBUG_REG32__fifos_rtr__SHIFT 0x0000001b
+#define VGT_DEBUG_REG32__first_point_of_edge_p5_q_MASK 0x00000400L
+#define VGT_DEBUG_REG32__first_point_of_edge_p5_q__SHIFT 0x0000000a
+#define VGT_DEBUG_REG32__first_point_of_patch_p5_q_MASK 0x00000200L
+#define VGT_DEBUG_REG32__first_point_of_patch_p5_q__SHIFT 0x00000009
+#define VGT_DEBUG_REG32__first_ring_of_patch_MASK 0x00000001L
+#define VGT_DEBUG_REG32__first_ring_of_patch__SHIFT 0x00000000
+#define VGT_DEBUG_REG32__inner2_fifos_rtr_MASK 0x01000000L
+#define VGT_DEBUG_REG32__inner2_fifos_rtr__SHIFT 0x00000018
+#define VGT_DEBUG_REG32__inner_fifos_rtr_MASK 0x02000000L
+#define VGT_DEBUG_REG32__inner_fifos_rtr__SHIFT 0x00000019
+#define VGT_DEBUG_REG32__last_edge_of_inner_ring_MASK 0x00000010L
+#define VGT_DEBUG_REG32__last_edge_of_inner_ring__SHIFT 0x00000004
+#define VGT_DEBUG_REG32__last_edge_of_outer_ring_MASK 0x00000004L
+#define VGT_DEBUG_REG32__last_edge_of_outer_ring__SHIFT 0x00000002
+#define VGT_DEBUG_REG32__last_patch_of_tg_p0_q_MASK 0x00000040L
+#define VGT_DEBUG_REG32__last_patch_of_tg_p0_q__SHIFT 0x00000006
+#define VGT_DEBUG_REG32__last_patch_of_tg_p5_q_MASK 0x00000800L
+#define VGT_DEBUG_REG32__last_patch_of_tg_p5_q__SHIFT 0x0000000b
+#define VGT_DEBUG_REG32__last_point_of_inner_edge_MASK 0x00000020L
+#define VGT_DEBUG_REG32__last_point_of_inner_edge__SHIFT 0x00000005
+#define VGT_DEBUG_REG32__last_point_of_outer_edge_MASK 0x00000008L
+#define VGT_DEBUG_REG32__last_point_of_outer_edge__SHIFT 0x00000003
+#define VGT_DEBUG_REG32__last_ring_of_patch_MASK 0x00000002L
+#define VGT_DEBUG_REG32__last_ring_of_patch__SHIFT 0x00000001
+#define VGT_DEBUG_REG32__outer_fifos_rtr_MASK 0x04000000L
+#define VGT_DEBUG_REG32__outer_fifos_rtr__SHIFT 0x0000001a
+#define VGT_DEBUG_REG32__pg_edge_fifo2_full_MASK 0x00020000L
+#define VGT_DEBUG_REG32__pg_edge_fifo2_full__SHIFT 0x00000011
+#define VGT_DEBUG_REG32__pg_edge_fifo3_full_MASK 0x00010000L
+#define VGT_DEBUG_REG32__pg_edge_fifo3_full__SHIFT 0x00000010
+#define VGT_DEBUG_REG32__pg_inner2_point_fifo_full_MASK 0x00100000L
+#define VGT_DEBUG_REG32__pg_inner2_point_fifo_full__SHIFT 0x00000014
+#define VGT_DEBUG_REG32__pg_inner3_point_fifo_full_MASK 0x00040000L
+#define VGT_DEBUG_REG32__pg_inner3_point_fifo_full__SHIFT 0x00000012
+#define VGT_DEBUG_REG32__pg_inner_point_fifo_full_MASK 0x00400000L
+#define VGT_DEBUG_REG32__pg_inner_point_fifo_full__SHIFT 0x00000016
+#define VGT_DEBUG_REG32__pg_outer2_point_fifo_full_MASK 0x00200000L
+#define VGT_DEBUG_REG32__pg_outer2_point_fifo_full__SHIFT 0x00000015
+#define VGT_DEBUG_REG32__pg_outer3_point_fifo_full_MASK 0x00080000L
+#define VGT_DEBUG_REG32__pg_outer3_point_fifo_full__SHIFT 0x00000013
+#define VGT_DEBUG_REG32__pg_outer_point_fifo_full_MASK 0x00800000L
+#define VGT_DEBUG_REG32__pg_outer_point_fifo_full__SHIFT 0x00000017
+#define VGT_DEBUG_REG32__pipe5_inner2_rtr_MASK 0x00008000L
+#define VGT_DEBUG_REG32__pipe5_inner2_rtr__SHIFT 0x0000000f
+#define VGT_DEBUG_REG32__pipe5_inner3_rtr_MASK 0x00004000L
+#define VGT_DEBUG_REG32__pipe5_inner3_rtr__SHIFT 0x0000000e
+#define VGT_DEBUG_REG32__SPARE_MASK 0x80000000L
+#define VGT_DEBUG_REG32__SPARE__SHIFT 0x0000001f
+#define VGT_DEBUG_REG32__tess_topology_p5_q_MASK 0x00003000L
+#define VGT_DEBUG_REG32__tess_topology_p5_q__SHIFT 0x0000000c
+#define VGT_DEBUG_REG33__con_prim_fifo_empty_MASK 0x00040000L
+#define VGT_DEBUG_REG33__con_prim_fifo_empty__SHIFT 0x00000012
+#define VGT_DEBUG_REG33__con_prim_fifo_full_MASK 0x00010000L
+#define VGT_DEBUG_REG33__con_prim_fifo_full__SHIFT 0x00000010
+#define VGT_DEBUG_REG33__con_ring1_busy_MASK 0x80000000L
+#define VGT_DEBUG_REG33__con_ring1_busy__SHIFT 0x0000001f
+#define VGT_DEBUG_REG33__con_ring2_busy_MASK 0x40000000L
+#define VGT_DEBUG_REG33__con_ring2_busy__SHIFT 0x0000001e
+#define VGT_DEBUG_REG33__con_ring3_busy_MASK 0x20000000L
+#define VGT_DEBUG_REG33__con_ring3_busy__SHIFT 0x0000001d
+#define VGT_DEBUG_REG33__con_vert_fifo_empty_MASK 0x00080000L
+#define VGT_DEBUG_REG33__con_vert_fifo_empty__SHIFT 0x00000013
+#define VGT_DEBUG_REG33__con_vert_fifo_full_MASK 0x00020000L
+#define VGT_DEBUG_REG33__con_vert_fifo_full__SHIFT 0x00000011
+#define VGT_DEBUG_REG33__first_prim_of_patch_q_MASK 0x00008000L
+#define VGT_DEBUG_REG33__first_prim_of_patch_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG33__last_patch_of_tg_p0_q_MASK 0x00100000L
+#define VGT_DEBUG_REG33__last_patch_of_tg_p0_q__SHIFT 0x00000014
+#define VGT_DEBUG_REG33__pipe0_patch_dr_MASK 0x00000001L
+#define VGT_DEBUG_REG33__pipe0_patch_dr__SHIFT 0x00000000
+#define VGT_DEBUG_REG33__pipe0_patch_rtr_MASK 0x00000010L
+#define VGT_DEBUG_REG33__pipe0_patch_rtr__SHIFT 0x00000004
+#define VGT_DEBUG_REG33__pipe1_dr_MASK 0x00000004L
+#define VGT_DEBUG_REG33__pipe1_dr__SHIFT 0x00000002
+#define VGT_DEBUG_REG33__pipe1_patch_rtr_MASK 0x00001000L
+#define VGT_DEBUG_REG33__pipe1_patch_rtr__SHIFT 0x0000000c
+#define VGT_DEBUG_REG33__pipe2_dr_MASK 0x00000008L
+#define VGT_DEBUG_REG33__pipe2_dr__SHIFT 0x00000003
+#define VGT_DEBUG_REG33__pipe2_rtr_MASK 0x00000080L
+#define VGT_DEBUG_REG33__pipe2_rtr__SHIFT 0x00000007
+#define VGT_DEBUG_REG33__pipe3_dr_MASK 0x00000100L
+#define VGT_DEBUG_REG33__pipe3_dr__SHIFT 0x00000008
+#define VGT_DEBUG_REG33__pipe3_rtr_MASK 0x00000200L
+#define VGT_DEBUG_REG33__pipe3_rtr__SHIFT 0x00000009
+#define VGT_DEBUG_REG33__ring1_in_sync_q_MASK 0x00000800L
+#define VGT_DEBUG_REG33__ring1_in_sync_q__SHIFT 0x0000000b
+#define VGT_DEBUG_REG33__ring1_pipe1_dr_MASK 0x00000040L
+#define VGT_DEBUG_REG33__ring1_pipe1_dr__SHIFT 0x00000006
+#define VGT_DEBUG_REG33__ring1_valid_p2_MASK 0x00800000L
+#define VGT_DEBUG_REG33__ring1_valid_p2__SHIFT 0x00000017
+#define VGT_DEBUG_REG33__ring2_in_sync_q_MASK 0x00000400L
+#define VGT_DEBUG_REG33__ring2_in_sync_q__SHIFT 0x0000000a
+#define VGT_DEBUG_REG33__ring2_pipe1_dr_MASK 0x00000020L
+#define VGT_DEBUG_REG33__ring2_pipe1_dr__SHIFT 0x00000005
+#define VGT_DEBUG_REG33__ring2_valid_p2_MASK 0x00400000L
+#define VGT_DEBUG_REG33__ring2_valid_p2__SHIFT 0x00000016
+#define VGT_DEBUG_REG33__ring3_in_sync_q_MASK 0x00002000L
+#define VGT_DEBUG_REG33__ring3_in_sync_q__SHIFT 0x0000000d
+#define VGT_DEBUG_REG33__ring3_pipe1_dr_MASK 0x00000002L
+#define VGT_DEBUG_REG33__ring3_pipe1_dr__SHIFT 0x00000001
+#define VGT_DEBUG_REG33__ring3_valid_p2_MASK 0x00200000L
+#define VGT_DEBUG_REG33__ring3_valid_p2__SHIFT 0x00000015
+#define VGT_DEBUG_REG33__te11_out_vert_gs_en_MASK 0x10000000L
+#define VGT_DEBUG_REG33__te11_out_vert_gs_en__SHIFT 0x0000001c
+#define VGT_DEBUG_REG33__tess_topology_p0_q_MASK 0x0c000000L
+#define VGT_DEBUG_REG33__tess_topology_p0_q__SHIFT 0x0000001a
+#define VGT_DEBUG_REG33__tess_type_p0_q_MASK 0x03000000L
+#define VGT_DEBUG_REG33__tess_type_p0_q__SHIFT 0x00000018
+#define VGT_DEBUG_REG33__tm_te11_event_rtr_MASK 0x00004000L
+#define VGT_DEBUG_REG33__tm_te11_event_rtr__SHIFT 0x0000000e
+#define VGT_DEBUG_REG34__advance_inner_point_p1_MASK 0x00800000L
+#define VGT_DEBUG_REG34__advance_inner_point_p1__SHIFT 0x00000017
+#define VGT_DEBUG_REG34__advance_outer_point_p1_MASK 0x00400000L
+#define VGT_DEBUG_REG34__advance_outer_point_p1__SHIFT 0x00000016
+#define VGT_DEBUG_REG34__con_state_q_MASK 0x0000000fL
+#define VGT_DEBUG_REG34__con_state_q__SHIFT 0x00000000
+#define VGT_DEBUG_REG34__first_ring_of_patch_p0_q_MASK 0x00010000L
+#define VGT_DEBUG_REG34__first_ring_of_patch_p0_q__SHIFT 0x00000010
+#define VGT_DEBUG_REG34__last_edge_of_outer_ring_p0_q_MASK 0x00040000L
+#define VGT_DEBUG_REG34__last_edge_of_outer_ring_p0_q__SHIFT 0x00000012
+#define VGT_DEBUG_REG34__last_point_of_inner_ring_p1_MASK 0x00100000L
+#define VGT_DEBUG_REG34__last_point_of_inner_ring_p1__SHIFT 0x00000014
+#define VGT_DEBUG_REG34__last_point_of_outer_ring_p1_MASK 0x00080000L
+#define VGT_DEBUG_REG34__last_point_of_outer_ring_p1__SHIFT 0x00000013
+#define VGT_DEBUG_REG34__last_ring_of_patch_p0_q_MASK 0x00020000L
+#define VGT_DEBUG_REG34__last_ring_of_patch_p0_q__SHIFT 0x00000011
+#define VGT_DEBUG_REG34__next_ring_is_rect_p0_q_MASK 0x01000000L
+#define VGT_DEBUG_REG34__next_ring_is_rect_p0_q__SHIFT 0x00000018
+#define VGT_DEBUG_REG34__outer_edge_tf_eq_one_p0_q_MASK 0x00200000L
+#define VGT_DEBUG_REG34__outer_edge_tf_eq_one_p0_q__SHIFT 0x00000015
+#define VGT_DEBUG_REG34__outer_parity_p0_q_MASK 0x00004000L
+#define VGT_DEBUG_REG34__outer_parity_p0_q__SHIFT 0x0000000e
+#define VGT_DEBUG_REG34__parallel_parity_p0_q_MASK 0x00008000L
+#define VGT_DEBUG_REG34__parallel_parity_p0_q__SHIFT 0x0000000f
+#define VGT_DEBUG_REG34__pipe0_edge_dr_MASK 0x00000200L
+#define VGT_DEBUG_REG34__pipe0_edge_dr__SHIFT 0x00000009
+#define VGT_DEBUG_REG34__pipe0_edge_rtr_MASK 0x00001000L
+#define VGT_DEBUG_REG34__pipe0_edge_rtr__SHIFT 0x0000000c
+#define VGT_DEBUG_REG34__pipe0_patch_dr_MASK 0x00000100L
+#define VGT_DEBUG_REG34__pipe0_patch_dr__SHIFT 0x00000008
+#define VGT_DEBUG_REG34__pipe0_patch_rtr_MASK 0x00000800L
+#define VGT_DEBUG_REG34__pipe0_patch_rtr__SHIFT 0x0000000b
+#define VGT_DEBUG_REG34__pipe1_dr_MASK 0x00000400L
+#define VGT_DEBUG_REG34__pipe1_dr__SHIFT 0x0000000a
+#define VGT_DEBUG_REG34__pipe1_edge_rtr_MASK 0x40000000L
+#define VGT_DEBUG_REG34__pipe1_edge_rtr__SHIFT 0x0000001e
+#define VGT_DEBUG_REG34__pipe1_inner1_rtr_MASK 0x08000000L
+#define VGT_DEBUG_REG34__pipe1_inner1_rtr__SHIFT 0x0000001b
+#define VGT_DEBUG_REG34__pipe1_inner2_rtr_MASK 0x10000000L
+#define VGT_DEBUG_REG34__pipe1_inner2_rtr__SHIFT 0x0000001c
+#define VGT_DEBUG_REG34__pipe1_outer1_rtr_MASK 0x02000000L
+#define VGT_DEBUG_REG34__pipe1_outer1_rtr__SHIFT 0x00000019
+#define VGT_DEBUG_REG34__pipe1_outer2_rtr_MASK 0x04000000L
+#define VGT_DEBUG_REG34__pipe1_outer2_rtr__SHIFT 0x0000001a
+#define VGT_DEBUG_REG34__pipe1_patch_rtr_MASK 0x20000000L
+#define VGT_DEBUG_REG34__pipe1_patch_rtr__SHIFT 0x0000001d
+#define VGT_DEBUG_REG34__pipe1_rtr_MASK 0x00002000L
+#define VGT_DEBUG_REG34__pipe1_rtr__SHIFT 0x0000000d
+#define VGT_DEBUG_REG34__process_tri_1st_2nd_half_p0_q_MASK 0x00000040L
+#define VGT_DEBUG_REG34__process_tri_1st_2nd_half_p0_q__SHIFT 0x00000006
+#define VGT_DEBUG_REG34__process_tri_center_poly_p0_q_MASK 0x00000080L
+#define VGT_DEBUG_REG34__process_tri_center_poly_p0_q__SHIFT 0x00000007
+#define VGT_DEBUG_REG34__process_tri_middle_p0_q_MASK 0x00000020L
+#define VGT_DEBUG_REG34__process_tri_middle_p0_q__SHIFT 0x00000005
+#define VGT_DEBUG_REG34__second_cycle_q_MASK 0x00000010L
+#define VGT_DEBUG_REG34__second_cycle_q__SHIFT 0x00000004
+#define VGT_DEBUG_REG34__use_stored_inner_q_ring1_MASK 0x80000000L
+#define VGT_DEBUG_REG34__use_stored_inner_q_ring1__SHIFT 0x0000001f
+#define VGT_DEBUG_REG35__event_flag_p1_q_MASK 0x00040000L
+#define VGT_DEBUG_REG35__event_flag_p1_q__SHIFT 0x00000012
+#define VGT_DEBUG_REG35__first_req_of_tg_p1_q_MASK 0x10000000L
+#define VGT_DEBUG_REG35__first_req_of_tg_p1_q__SHIFT 0x0000001c
+#define VGT_DEBUG_REG35__last_req_of_tg_p2_MASK 0x00000800L
+#define VGT_DEBUG_REG35__last_req_of_tg_p2__SHIFT 0x0000000b
+#define VGT_DEBUG_REG35__null_flag_p1_q_MASK 0x00080000L
+#define VGT_DEBUG_REG35__null_flag_p1_q__SHIFT 0x00000013
+#define VGT_DEBUG_REG35__pipe0_dr_MASK 0x00000001L
+#define VGT_DEBUG_REG35__pipe0_dr__SHIFT 0x00000000
+#define VGT_DEBUG_REG35__pipe0_rtr_MASK 0x00000004L
+#define VGT_DEBUG_REG35__pipe0_rtr__SHIFT 0x00000002
+#define VGT_DEBUG_REG35__pipe1_dr_MASK 0x00000002L
+#define VGT_DEBUG_REG35__pipe1_dr__SHIFT 0x00000001
+#define VGT_DEBUG_REG35__pipe1_rtr_MASK 0x00000008L
+#define VGT_DEBUG_REG35__pipe1_rtr__SHIFT 0x00000003
+#define VGT_DEBUG_REG35__second_tf_ret_data_q_MASK 0x08000000L
+#define VGT_DEBUG_REG35__second_tf_ret_data_q__SHIFT 0x0000001b
+#define VGT_DEBUG_REG35__spi_vgt_hs_done_cnt_q_MASK 0x0003f000L
+#define VGT_DEBUG_REG35__spi_vgt_hs_done_cnt_q__SHIFT 0x0000000c
+#define VGT_DEBUG_REG35__TC_VGT_rdret_data_in_MASK 0x80000000L
+#define VGT_DEBUG_REG35__TC_VGT_rdret_data_in__SHIFT 0x0000001f
+#define VGT_DEBUG_REG35__tf_data_fifo_busy_q_MASK 0x00000040L
+#define VGT_DEBUG_REG35__tf_data_fifo_busy_q__SHIFT 0x00000006
+#define VGT_DEBUG_REG35__tf_data_fifo_cnt_q_MASK 0x07f00000L
+#define VGT_DEBUG_REG35__tf_data_fifo_cnt_q__SHIFT 0x00000014
+#define VGT_DEBUG_REG35__tf_data_fifo_rtr_q_MASK 0x00000080L
+#define VGT_DEBUG_REG35__tf_data_fifo_rtr_q__SHIFT 0x00000007
+#define VGT_DEBUG_REG35__tfreq_tg_fifo_empty_MASK 0x00000010L
+#define VGT_DEBUG_REG35__tfreq_tg_fifo_empty__SHIFT 0x00000004
+#define VGT_DEBUG_REG35__tfreq_tg_fifo_full_MASK 0x00000020L
+#define VGT_DEBUG_REG35__tfreq_tg_fifo_full__SHIFT 0x00000005
+#define VGT_DEBUG_REG35__tf_skid_fifo_empty_MASK 0x00000100L
+#define VGT_DEBUG_REG35__tf_skid_fifo_empty__SHIFT 0x00000008
+#define VGT_DEBUG_REG35__tf_skid_fifo_full_MASK 0x00000200L
+#define VGT_DEBUG_REG35__tf_skid_fifo_full__SHIFT 0x00000009
+#define VGT_DEBUG_REG35__VGT_TC_rdnfo_stall_out_MASK 0x40000000L
+#define VGT_DEBUG_REG35__VGT_TC_rdnfo_stall_out__SHIFT 0x0000001e
+#define VGT_DEBUG_REG35__vgt_tc_rdreq_rtr_q_MASK 0x00000400L
+#define VGT_DEBUG_REG35__vgt_tc_rdreq_rtr_q__SHIFT 0x0000000a
+#define VGT_DEBUG_REG35__VGT_TC_rdreq_send_out_MASK 0x20000000L
+#define VGT_DEBUG_REG35__VGT_TC_rdreq_send_out__SHIFT 0x0000001d
+#define VGT_DEBUG_REG3__hsWaveRelInd_MASK 0xfc000000L
+#define VGT_DEBUG_REG3__hsWaveRelInd__SHIFT 0x0000001a
+#define VGT_DEBUG_REG3__lsPatchCnt_MASK 0x03fc0000L
+#define VGT_DEBUG_REG3__lsPatchCnt__SHIFT 0x00000012
+#define VGT_DEBUG_REG3__lsTgRelInd_MASK 0x00000fffL
+#define VGT_DEBUG_REG3__lsTgRelInd__SHIFT 0x00000000
+#define VGT_DEBUG_REG3__lsWaveRelInd_MASK 0x0003f000L
+#define VGT_DEBUG_REG3__lsWaveRelInd__SHIFT 0x0000000c
+#define VGT_DEBUG_REG4__hsCpCnt_MASK 0x1f000000L
+#define VGT_DEBUG_REG4__hsCpCnt__SHIFT 0x00000018
+#define VGT_DEBUG_REG4__hsFwaveFlag_MASK 0x40000000L
+#define VGT_DEBUG_REG4__hsFwaveFlag__SHIFT 0x0000001e
+#define VGT_DEBUG_REG4__hsPatchCnt_MASK 0x000000ffL
+#define VGT_DEBUG_REG4__hsPatchCnt__SHIFT 0x00000000
+#define VGT_DEBUG_REG4__hsPrimId_15_0_MASK 0x00ffff00L
+#define VGT_DEBUG_REG4__hsPrimId_15_0__SHIFT 0x00000008
+#define VGT_DEBUG_REG4__hsWaveSendFlush_MASK 0x20000000L
+#define VGT_DEBUG_REG4__hsWaveSendFlush__SHIFT 0x0000001d
+#define VGT_DEBUG_REG4__SPARE_MASK 0xffffffffL
+#define VGT_DEBUG_REG4__SPARE__SHIFT 0x00000000
+#define VGT_DEBUG_REG5__hsVertCreditCnt_0_MASK 0x0000f800L
+#define VGT_DEBUG_REG5__hsVertCreditCnt_0__SHIFT 0x0000000b
+#define VGT_DEBUG_REG5__hsWaveCreditCnt_0_MASK 0x000000f8L
+#define VGT_DEBUG_REG5__hsWaveCreditCnt_0__SHIFT 0x00000003
+#define VGT_DEBUG_REG5__lsVertCreditCnt_0_MASK 0xf8000000L
+#define VGT_DEBUG_REG5__lsVertCreditCnt_0__SHIFT 0x0000001b
+#define VGT_DEBUG_REG5__lsWaveCreditCnt_0_MASK 0x00f80000L
+#define VGT_DEBUG_REG5__lsWaveCreditCnt_0__SHIFT 0x00000013
+#define VGT_DEBUG_REG5__SPARE1_MASK 0x07000000L
+#define VGT_DEBUG_REG5__SPARE1__SHIFT 0x00000018
+#define VGT_DEBUG_REG5__SPARE2_MASK 0x00070000L
+#define VGT_DEBUG_REG5__SPARE2__SHIFT 0x00000010
+#define VGT_DEBUG_REG5__SPARE3_MASK 0x00000700L
+#define VGT_DEBUG_REG5__SPARE3__SHIFT 0x00000008
+#define VGT_DEBUG_REG5__SPARE4_MASK 0x00000007L
+#define VGT_DEBUG_REG5__SPARE4__SHIFT 0x00000000
+#define VGT_DEBUG_REG6__debug_BASE_MASK 0x0000ffffL
+#define VGT_DEBUG_REG6__debug_BASE__SHIFT 0x00000000
+#define VGT_DEBUG_REG6__debug_SIZE_MASK 0xffff0000L
+#define VGT_DEBUG_REG6__debug_SIZE__SHIFT 0x00000010
+#define VGT_DEBUG_REG7__debug_tfmmFifoEmpty_MASK 0x00000001L
+#define VGT_DEBUG_REG7__debug_tfmmFifoEmpty__SHIFT 0x00000000
+#define VGT_DEBUG_REG7__debug_tfmmFifoFull_MASK 0x00000002L
+#define VGT_DEBUG_REG7__debug_tfmmFifoFull__SHIFT 0x00000001
+#define VGT_DEBUG_REG7__hs_pipe0_dr_MASK 0x00000004L
+#define VGT_DEBUG_REG7__hs_pipe0_dr__SHIFT 0x00000002
+#define VGT_DEBUG_REG7__hs_pipe0_rtr_MASK 0x00000008L
+#define VGT_DEBUG_REG7__hs_pipe0_rtr__SHIFT 0x00000003
+#define VGT_DEBUG_REG7__hs_pipe1_rtr_MASK 0x00000010L
+#define VGT_DEBUG_REG7__hs_pipe1_rtr__SHIFT 0x00000004
+#define VGT_DEBUG_REG7__SPARE_MASK 0x0000ffe0L
+#define VGT_DEBUG_REG7__SPARE__SHIFT 0x00000005
+#define VGT_DEBUG_REG7__TF_addr_MASK 0xffff0000L
+#define VGT_DEBUG_REG7__TF_addr__SHIFT 0x00000010
+#define VGT_DEBUG_REG8__es_gs_rtr_MASK 0x00004000L
+#define VGT_DEBUG_REG8__es_gs_rtr__SHIFT 0x0000000e
+#define VGT_DEBUG_REG8__gs_event_fifo_empty_MASK 0x02000000L
+#define VGT_DEBUG_REG8__gs_event_fifo_empty__SHIFT 0x00000019
+#define VGT_DEBUG_REG8__gs_event_fifo_rtr_MASK 0x00008000L
+#define VGT_DEBUG_REG8__gs_event_fifo_rtr__SHIFT 0x0000000f
+#define VGT_DEBUG_REG8__gsprim_buff_empty_q_MASK 0x04000000L
+#define VGT_DEBUG_REG8__gsprim_buff_empty_q__SHIFT 0x0000001a
+#define VGT_DEBUG_REG8__gsprim_buff_full_q_MASK 0x08000000L
+#define VGT_DEBUG_REG8__gsprim_buff_full_q__SHIFT 0x0000001b
+#define VGT_DEBUG_REG8__gs_tbl_r3_rtr_MASK 0x00020000L
+#define VGT_DEBUG_REG8__gs_tbl_r3_rtr__SHIFT 0x00000011
+#define VGT_DEBUG_REG8__gs_tbl_valid_r3_q_MASK 0x00000020L
+#define VGT_DEBUG_REG8__gs_tbl_valid_r3_q__SHIFT 0x00000005
+#define VGT_DEBUG_REG8__hold_for_es_flush_MASK 0x01000000L
+#define VGT_DEBUG_REG8__hold_for_es_flush__SHIFT 0x00000018
+#define VGT_DEBUG_REG8__prim_skid_fifo_empty_MASK 0x00040000L
+#define VGT_DEBUG_REG8__prim_skid_fifo_empty__SHIFT 0x00000012
+#define VGT_DEBUG_REG8__r0_rtr_MASK 0x00000400L
+#define VGT_DEBUG_REG8__r0_rtr__SHIFT 0x0000000a
+#define VGT_DEBUG_REG8__r1_inst_rtr_MASK 0x00000004L
+#define VGT_DEBUG_REG8__r1_inst_rtr__SHIFT 0x00000002
+#define VGT_DEBUG_REG8__r1_rtr_MASK 0x00000800L
+#define VGT_DEBUG_REG8__r1_rtr__SHIFT 0x0000000b
+#define VGT_DEBUG_REG8__r2_indx_rtr_MASK 0x00001000L
+#define VGT_DEBUG_REG8__r2_indx_rtr__SHIFT 0x0000000c
+#define VGT_DEBUG_REG8__r2_no_bp_rtr_MASK 0x00800000L
+#define VGT_DEBUG_REG8__r2_no_bp_rtr__SHIFT 0x00000017
+#define VGT_DEBUG_REG8__r2_rtr_MASK 0x00002000L
+#define VGT_DEBUG_REG8__r2_rtr__SHIFT 0x0000000d
+#define VGT_DEBUG_REG8__rcm_busy_q_MASK 0x00000001L
+#define VGT_DEBUG_REG8__rcm_busy_q__SHIFT 0x00000000
+#define VGT_DEBUG_REG8__rcm_noif_busy_q_MASK 0x00000002L
+#define VGT_DEBUG_REG8__rcm_noif_busy_q__SHIFT 0x00000001
+#define VGT_DEBUG_REG8__spi_esvert_fifo_busy_q_MASK 0x00000010L
+#define VGT_DEBUG_REG8__spi_esvert_fifo_busy_q__SHIFT 0x00000004
+#define VGT_DEBUG_REG8__spi_gsprim_fifo_busy_q_MASK 0x00000008L
+#define VGT_DEBUG_REG8__spi_gsprim_fifo_busy_q__SHIFT 0x00000003
+#define VGT_DEBUG_REG8__te_prim_fifo_empty_MASK 0x10000000L
+#define VGT_DEBUG_REG8__te_prim_fifo_empty__SHIFT 0x0000001c
+#define VGT_DEBUG_REG8__te_prim_fifo_full_MASK 0x20000000L
+#define VGT_DEBUG_REG8__te_prim_fifo_full__SHIFT 0x0000001d
+#define VGT_DEBUG_REG8__te_vert_fifo_empty_MASK 0x40000000L
+#define VGT_DEBUG_REG8__te_vert_fifo_empty__SHIFT 0x0000001e
+#define VGT_DEBUG_REG8__te_vert_fifo_full_MASK 0x80000000L
+#define VGT_DEBUG_REG8__te_vert_fifo_full__SHIFT 0x0000001f
+#define VGT_DEBUG_REG8__tm_rcm_es_tbl_rtr_MASK 0x00200000L
+#define VGT_DEBUG_REG8__tm_rcm_es_tbl_rtr__SHIFT 0x00000015
+#define VGT_DEBUG_REG8__tm_rcm_gs_event_rtr_MASK 0x00010000L
+#define VGT_DEBUG_REG8__tm_rcm_gs_event_rtr__SHIFT 0x00000010
+#define VGT_DEBUG_REG8__tm_rcm_gs_tbl_rtr_MASK 0x00100000L
+#define VGT_DEBUG_REG8__tm_rcm_gs_tbl_rtr__SHIFT 0x00000014
+#define VGT_DEBUG_REG8__valid_r0_q_MASK 0x00000040L
+#define VGT_DEBUG_REG8__valid_r0_q__SHIFT 0x00000006
+#define VGT_DEBUG_REG8__valid_r1_q_MASK 0x00000080L
+#define VGT_DEBUG_REG8__valid_r1_q__SHIFT 0x00000007
+#define VGT_DEBUG_REG8__valid_r2_MASK 0x00000100L
+#define VGT_DEBUG_REG8__valid_r2_q_MASK 0x00000200L
+#define VGT_DEBUG_REG8__valid_r2_q__SHIFT 0x00000009
+#define VGT_DEBUG_REG8__valid_r2__SHIFT 0x00000008
+#define VGT_DEBUG_REG8__VGT_SPI_esvert_rtr_q_MASK 0x00400000L
+#define VGT_DEBUG_REG8__VGT_SPI_esvert_rtr_q__SHIFT 0x00000016
+#define VGT_DEBUG_REG8__VGT_SPI_gsprim_rtr_q_MASK 0x00080000L
+#define VGT_DEBUG_REG8__VGT_SPI_gsprim_rtr_q__SHIFT 0x00000013
+#define VGT_DEBUG_REG9__eop_indx_r3_MASK 0x00000010L
+#define VGT_DEBUG_REG9__eop_indx_r3__SHIFT 0x00000004
+#define VGT_DEBUG_REG9__eop_prim_r3_MASK 0x00000020L
+#define VGT_DEBUG_REG9__eop_prim_r3__SHIFT 0x00000005
+#define VGT_DEBUG_REG9__es_eov_r3_MASK 0x00000040L
+#define VGT_DEBUG_REG9__es_eov_r3__SHIFT 0x00000006
+#define VGT_DEBUG_REG9__es_per_gs_vert_cnt_r3_q_not_0_MASK 0x02000000L
+#define VGT_DEBUG_REG9__es_per_gs_vert_cnt_r3_q_not_0__SHIFT 0x00000019
+#define VGT_DEBUG_REG9__es_tbl_state_r3_q_0_MASK 0x00000080L
+#define VGT_DEBUG_REG9__es_tbl_state_r3_q_0__SHIFT 0x00000007
+#define VGT_DEBUG_REG9__gs_eov_r3_MASK 0x00000008L
+#define VGT_DEBUG_REG9__gs_eov_r3__SHIFT 0x00000003
+#define VGT_DEBUG_REG9__gs_instancing_state_q_MASK 0x01000000L
+#define VGT_DEBUG_REG9__gs_instancing_state_q__SHIFT 0x00000018
+#define VGT_DEBUG_REG9__gs_pending_state_r3_q_MASK 0x00400000L
+#define VGT_DEBUG_REG9__gs_pending_state_r3_q__SHIFT 0x00000016
+#define VGT_DEBUG_REG9__gs_prim_per_es_ctr_r3_q_not_0_MASK 0x04000000L
+#define VGT_DEBUG_REG9__gs_prim_per_es_ctr_r3_q_not_0__SHIFT 0x0000001a
+#define VGT_DEBUG_REG9__gs_tbl_eop_r3_q_MASK 0x00040000L
+#define VGT_DEBUG_REG9__gs_tbl_eop_r3_q__SHIFT 0x00000012
+#define VGT_DEBUG_REG9__gs_tbl_num_es_per_gs_r3_q_not_0_MASK 0x00000400L
+#define VGT_DEBUG_REG9__gs_tbl_num_es_per_gs_r3_q_not_0__SHIFT 0x0000000a
+#define VGT_DEBUG_REG9__gs_tbl_prim_cnt_r3_q_MASK 0x0003f800L
+#define VGT_DEBUG_REG9__gs_tbl_prim_cnt_r3_q__SHIFT 0x0000000b
+#define VGT_DEBUG_REG9__gs_tbl_state_r3_q_MASK 0x00380000L
+#define VGT_DEBUG_REG9__gs_tbl_state_r3_q__SHIFT 0x00000013
+#define VGT_DEBUG_REG9__indices_to_send_r2_q_MASK 0x00000003L
+#define VGT_DEBUG_REG9__indices_to_send_r2_q__SHIFT 0x00000000
+#define VGT_DEBUG_REG9__invalidate_rb_roll_over_q_MASK 0x00800000L
+#define VGT_DEBUG_REG9__invalidate_rb_roll_over_q__SHIFT 0x00000017
+#define VGT_DEBUG_REG9__off_chip_hs_r2_q_MASK 0x80000000L
+#define VGT_DEBUG_REG9__off_chip_hs_r2_q__SHIFT 0x0000001f
+#define VGT_DEBUG_REG9__pending_es_flush_r3_MASK 0x00000200L
+#define VGT_DEBUG_REG9__pending_es_flush_r3__SHIFT 0x00000009
+#define VGT_DEBUG_REG9__pending_es_send_r3_q_MASK 0x00000100L
+#define VGT_DEBUG_REG9__pending_es_send_r3_q__SHIFT 0x00000008
+#define VGT_DEBUG_REG9__pre_r0_rtr_MASK 0x08000000L
+#define VGT_DEBUG_REG9__pre_r0_rtr__SHIFT 0x0000001b
+#define VGT_DEBUG_REG9__SPARE0_MASK 0x40000000L
+#define VGT_DEBUG_REG9__SPARE0__SHIFT 0x0000001e
+#define VGT_DEBUG_REG9__valid_indices_r3_MASK 0x00000004L
+#define VGT_DEBUG_REG9__valid_indices_r3__SHIFT 0x00000002
+#define VGT_DEBUG_REG9__valid_pre_r0_q_MASK 0x20000000L
+#define VGT_DEBUG_REG9__valid_pre_r0_q__SHIFT 0x0000001d
+#define VGT_DEBUG_REG9__valid_r3_q_MASK 0x10000000L
+#define VGT_DEBUG_REG9__valid_r3_q__SHIFT 0x0000001c
+#define VGT_DMA_BASE__BASE_ADDR_MASK 0xffffffffL
+#define VGT_DMA_BASE__BASE_ADDR__SHIFT 0x00000000
+#define VGT_DMA_BASE_HI__BASE_ADDR_MASK 0x000000ffL
+#define VGT_DMA_BASE_HI__BASE_ADDR__SHIFT 0x00000000
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH_MASK 0x000001ffL
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH__SHIFT 0x00000000
+#define VGT_DMA_INDEX_TYPE__ATC_MASK 0x00000100L
+#define VGT_DMA_INDEX_TYPE__ATC__SHIFT 0x00000008
+#define VGT_DMA_INDEX_TYPE__BUF_TYPE_MASK 0x00000030L
+#define VGT_DMA_INDEX_TYPE__BUF_TYPE__SHIFT 0x00000004
+#define VGT_DMA_INDEX_TYPE__INDEX_TYPE_MASK 0x00000003L
+#define VGT_DMA_INDEX_TYPE__INDEX_TYPE__SHIFT 0x00000000
+#define VGT_DMA_INDEX_TYPE__NOT_EOP_MASK 0x00000200L
+#define VGT_DMA_INDEX_TYPE__NOT_EOP__SHIFT 0x00000009
+#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY_MASK 0x000000c0L
+#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY__SHIFT 0x00000006
+#define VGT_DMA_INDEX_TYPE__REQ_PATH_MASK 0x00000400L
+#define VGT_DMA_INDEX_TYPE__REQ_PATH__SHIFT 0x0000000a
+#define VGT_DMA_INDEX_TYPE__SWAP_MODE_MASK 0x0000000cL
+#define VGT_DMA_INDEX_TYPE__SWAP_MODE__SHIFT 0x00000002
+#define VGT_DMA_MAX_SIZE__MAX_SIZE_MASK 0xffffffffL
+#define VGT_DMA_MAX_SIZE__MAX_SIZE__SHIFT 0x00000000
+#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES_MASK 0xffffffffL
+#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES__SHIFT 0x00000000
+#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH_MASK 0x0000003fL
+#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH__SHIFT 0x00000000
+#define VGT_DMA_SIZE__NUM_INDICES_MASK 0xffffffffL
+#define VGT_DMA_SIZE__NUM_INDICES__SHIFT 0x00000000
+#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH_MASK 0x0000003fL
+#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH__SHIFT 0x00000000
+#define VGT_DRAW_INITIATOR__MAJOR_MODE_MASK 0x0000000cL
+#define VGT_DRAW_INITIATOR__MAJOR_MODE__SHIFT 0x00000002
+#define VGT_DRAW_INITIATOR__NOT_EOP_MASK 0x00000020L
+#define VGT_DRAW_INITIATOR__NOT_EOP__SHIFT 0x00000005
+#define VGT_DRAW_INITIATOR__SOURCE_SELECT_MASK 0x00000003L
+#define VGT_DRAW_INITIATOR__SOURCE_SELECT__SHIFT 0x00000000
+#define VGT_DRAW_INITIATOR__SPRITE_EN_R6XX_MASK 0x00000010L
+#define VGT_DRAW_INITIATOR__SPRITE_EN_R6XX__SHIFT 0x00000004
+#define VGT_DRAW_INITIATOR__USE_OPAQUE_MASK 0x00000040L
+#define VGT_DRAW_INITIATOR__USE_OPAQUE__SHIFT 0x00000006
+#define VGT_ENHANCE__MISC_MASK 0xffffffffL
+#define VGT_ENHANCE__MISC__SHIFT 0x00000000
+#define VGT_ESGS_RING_ITEMSIZE__ITEMSIZE_MASK 0x00007fffL
+#define VGT_ESGS_RING_ITEMSIZE__ITEMSIZE__SHIFT 0x00000000
+#define VGT_ESGS_RING_SIZE__MEM_SIZE_MASK 0xffffffffL
+#define VGT_ESGS_RING_SIZE__MEM_SIZE__SHIFT 0x00000000
+#define VGT_ES_PER_GS__ES_PER_GS_MASK 0x000007ffL
+#define VGT_ES_PER_GS__ES_PER_GS__SHIFT 0x00000000
+#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW_MASK 0x0fffffffL
+#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW__SHIFT 0x00000000
+#define VGT_EVENT_INITIATOR__ADDRESS_HI_MASK 0x07fc0000L
+#define VGT_EVENT_INITIATOR__ADDRESS_HI__SHIFT 0x00000012
+#define VGT_EVENT_INITIATOR__EVENT_TYPE_MASK 0x0000003fL
+#define VGT_EVENT_INITIATOR__EVENT_TYPE__SHIFT 0x00000000
+#define VGT_EVENT_INITIATOR__EXTENDED_EVENT_MASK 0x08000000L
+#define VGT_EVENT_INITIATOR__EXTENDED_EVENT__SHIFT 0x0000001b
+#define VGT_FIFO_DEPTHS__CLIPP_FIFO_DEPTH_MASK 0x003fff00L
+#define VGT_FIFO_DEPTHS__CLIPP_FIFO_DEPTH__SHIFT 0x00000008
+#define VGT_FIFO_DEPTHS__RESERVED_0_MASK 0x00000080L
+#define VGT_FIFO_DEPTHS__RESERVED_0__SHIFT 0x00000007
+#define VGT_FIFO_DEPTHS__RESERVED_1_MASK 0xffc00000L
+#define VGT_FIFO_DEPTHS__RESERVED_1__SHIFT 0x00000016
+#define VGT_FIFO_DEPTHS__VS_DEALLOC_TBL_DEPTH_MASK 0x0000007fL
+#define VGT_FIFO_DEPTHS__VS_DEALLOC_TBL_DEPTH__SHIFT 0x00000000
+#define VGT_GROUP_DECR__DECR_MASK 0x0000000fL
+#define VGT_GROUP_DECR__DECR__SHIFT 0x00000000
+#define VGT_GROUP_FIRST_DECR__FIRST_DECR_MASK 0x0000000fL
+#define VGT_GROUP_FIRST_DECR__FIRST_DECR__SHIFT 0x00000000
+#define VGT_GROUP_PRIM_TYPE__PRIM_ORDER_MASK 0x00070000L
+#define VGT_GROUP_PRIM_TYPE__PRIM_ORDER__SHIFT 0x00000010
+#define VGT_GROUP_PRIM_TYPE__PRIM_TYPE_MASK 0x0000001fL
+#define VGT_GROUP_PRIM_TYPE__PRIM_TYPE__SHIFT 0x00000000
+#define VGT_GROUP_PRIM_TYPE__RETAIN_ORDER_MASK 0x00004000L
+#define VGT_GROUP_PRIM_TYPE__RETAIN_ORDER__SHIFT 0x0000000e
+#define VGT_GROUP_PRIM_TYPE__RETAIN_QUADS_MASK 0x00008000L
+#define VGT_GROUP_PRIM_TYPE__RETAIN_QUADS__SHIFT 0x0000000f
+#define VGT_GROUP_VECT_0_CNTL__COMP_W_EN_MASK 0x00000008L
+#define VGT_GROUP_VECT_0_CNTL__COMP_W_EN__SHIFT 0x00000003
+#define VGT_GROUP_VECT_0_CNTL__COMP_X_EN_MASK 0x00000001L
+#define VGT_GROUP_VECT_0_CNTL__COMP_X_EN__SHIFT 0x00000000
+#define VGT_GROUP_VECT_0_CNTL__COMP_Y_EN_MASK 0x00000002L
+#define VGT_GROUP_VECT_0_CNTL__COMP_Y_EN__SHIFT 0x00000001
+#define VGT_GROUP_VECT_0_CNTL__COMP_Z_EN_MASK 0x00000004L
+#define VGT_GROUP_VECT_0_CNTL__COMP_Z_EN__SHIFT 0x00000002
+#define VGT_GROUP_VECT_0_CNTL__SHIFT_MASK 0x00ff0000L
+#define VGT_GROUP_VECT_0_CNTL__SHIFT__SHIFT 0x00000010
+#define VGT_GROUP_VECT_0_CNTL__STRIDE_MASK 0x0000ff00L
+#define VGT_GROUP_VECT_0_CNTL__STRIDE__SHIFT 0x00000008
+#define VGT_GROUP_VECT_0_FMT_CNTL__W_CONV_MASK 0x0f000000L
+#define VGT_GROUP_VECT_0_FMT_CNTL__W_CONV__SHIFT 0x00000018
+#define VGT_GROUP_VECT_0_FMT_CNTL__W_OFFSET_MASK 0xf0000000L
+#define VGT_GROUP_VECT_0_FMT_CNTL__W_OFFSET__SHIFT 0x0000001c
+#define VGT_GROUP_VECT_0_FMT_CNTL__X_CONV_MASK 0x0000000fL
+#define VGT_GROUP_VECT_0_FMT_CNTL__X_CONV__SHIFT 0x00000000
+#define VGT_GROUP_VECT_0_FMT_CNTL__X_OFFSET_MASK 0x000000f0L
+#define VGT_GROUP_VECT_0_FMT_CNTL__X_OFFSET__SHIFT 0x00000004
+#define VGT_GROUP_VECT_0_FMT_CNTL__Y_CONV_MASK 0x00000f00L
+#define VGT_GROUP_VECT_0_FMT_CNTL__Y_CONV__SHIFT 0x00000008
+#define VGT_GROUP_VECT_0_FMT_CNTL__Y_OFFSET_MASK 0x0000f000L
+#define VGT_GROUP_VECT_0_FMT_CNTL__Y_OFFSET__SHIFT 0x0000000c
+#define VGT_GROUP_VECT_0_FMT_CNTL__Z_CONV_MASK 0x000f0000L
+#define VGT_GROUP_VECT_0_FMT_CNTL__Z_CONV__SHIFT 0x00000010
+#define VGT_GROUP_VECT_0_FMT_CNTL__Z_OFFSET_MASK 0x00f00000L
+#define VGT_GROUP_VECT_0_FMT_CNTL__Z_OFFSET__SHIFT 0x00000014
+#define VGT_GROUP_VECT_1_CNTL__COMP_W_EN_MASK 0x00000008L
+#define VGT_GROUP_VECT_1_CNTL__COMP_W_EN__SHIFT 0x00000003
+#define VGT_GROUP_VECT_1_CNTL__COMP_X_EN_MASK 0x00000001L
+#define VGT_GROUP_VECT_1_CNTL__COMP_X_EN__SHIFT 0x00000000
+#define VGT_GROUP_VECT_1_CNTL__COMP_Y_EN_MASK 0x00000002L
+#define VGT_GROUP_VECT_1_CNTL__COMP_Y_EN__SHIFT 0x00000001
+#define VGT_GROUP_VECT_1_CNTL__COMP_Z_EN_MASK 0x00000004L
+#define VGT_GROUP_VECT_1_CNTL__COMP_Z_EN__SHIFT 0x00000002
+#define VGT_GROUP_VECT_1_CNTL__SHIFT_MASK 0x00ff0000L
+#define VGT_GROUP_VECT_1_CNTL__SHIFT__SHIFT 0x00000010
+#define VGT_GROUP_VECT_1_CNTL__STRIDE_MASK 0x0000ff00L
+#define VGT_GROUP_VECT_1_CNTL__STRIDE__SHIFT 0x00000008
+#define VGT_GROUP_VECT_1_FMT_CNTL__W_CONV_MASK 0x0f000000L
+#define VGT_GROUP_VECT_1_FMT_CNTL__W_CONV__SHIFT 0x00000018
+#define VGT_GROUP_VECT_1_FMT_CNTL__W_OFFSET_MASK 0xf0000000L
+#define VGT_GROUP_VECT_1_FMT_CNTL__W_OFFSET__SHIFT 0x0000001c
+#define VGT_GROUP_VECT_1_FMT_CNTL__X_CONV_MASK 0x0000000fL
+#define VGT_GROUP_VECT_1_FMT_CNTL__X_CONV__SHIFT 0x00000000
+#define VGT_GROUP_VECT_1_FMT_CNTL__X_OFFSET_MASK 0x000000f0L
+#define VGT_GROUP_VECT_1_FMT_CNTL__X_OFFSET__SHIFT 0x00000004
+#define VGT_GROUP_VECT_1_FMT_CNTL__Y_CONV_MASK 0x00000f00L
+#define VGT_GROUP_VECT_1_FMT_CNTL__Y_CONV__SHIFT 0x00000008
+#define VGT_GROUP_VECT_1_FMT_CNTL__Y_OFFSET_MASK 0x0000f000L
+#define VGT_GROUP_VECT_1_FMT_CNTL__Y_OFFSET__SHIFT 0x0000000c
+#define VGT_GROUP_VECT_1_FMT_CNTL__Z_CONV_MASK 0x000f0000L
+#define VGT_GROUP_VECT_1_FMT_CNTL__Z_CONV__SHIFT 0x00000010
+#define VGT_GROUP_VECT_1_FMT_CNTL__Z_OFFSET_MASK 0x00f00000L
+#define VGT_GROUP_VECT_1_FMT_CNTL__Z_OFFSET__SHIFT 0x00000014
+#define VGT_GS_INSTANCE_CNT__CNT_MASK 0x000001fcL
+#define VGT_GS_INSTANCE_CNT__CNT__SHIFT 0x00000002
+#define VGT_GS_INSTANCE_CNT__ENABLE_MASK 0x00000001L
+#define VGT_GS_INSTANCE_CNT__ENABLE__SHIFT 0x00000000
+#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT_MASK 0x000007ffL
+#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT__SHIFT 0x00000000
+#define VGT_GS_MODE__COMPUTE_MODE_MASK 0x00004000L
+#define VGT_GS_MODE__COMPUTE_MODE__SHIFT 0x0000000e
+#define VGT_GS_MODE__CUT_MODE_MASK 0x00000030L
+#define VGT_GS_MODE__CUT_MODE__SHIFT 0x00000004
+#define VGT_GS_MODE__ELEMENT_INFO_EN_MASK 0x00010000L
+#define VGT_GS_MODE__ELEMENT_INFO_EN__SHIFT 0x00000010
+#define VGT_GS_MODE__ES_PASSTHRU_MASK 0x00002000L
+#define VGT_GS_MODE__ES_PASSTHRU__SHIFT 0x0000000d
+#define VGT_GS_MODE__ES_WRITE_OPTIMIZE_MASK 0x00080000L
+#define VGT_GS_MODE__ES_WRITE_OPTIMIZE__SHIFT 0x00000013
+#define VGT_GS_MODE__FAST_COMPUTE_MODE_MASK 0x00008000L
+#define VGT_GS_MODE__FAST_COMPUTE_MODE__SHIFT 0x0000000f
+#define VGT_GS_MODE__GS_C_PACK_EN_MASK 0x00000800L
+#define VGT_GS_MODE__GS_C_PACK_EN__SHIFT 0x0000000b
+#define VGT_GS_MODE__GS_WRITE_OPTIMIZE_MASK 0x00100000L
+#define VGT_GS_MODE__GS_WRITE_OPTIMIZE__SHIFT 0x00000014
+#define VGT_GS_MODE__MODE_MASK 0x00000007L
+#define VGT_GS_MODE__MODE__SHIFT 0x00000000
+#define VGT_GS_MODE__ONCHIP_MASK 0x00600000L
+#define VGT_GS_MODE__ONCHIP__SHIFT 0x00000015
+#define VGT_GS_MODE__PARTIAL_THD_AT_EOI_MASK 0x00020000L
+#define VGT_GS_MODE__PARTIAL_THD_AT_EOI__SHIFT 0x00000011
+#define VGT_GS_MODE__RESERVED_0_MASK 0x00000008L
+#define VGT_GS_MODE__RESERVED_0__SHIFT 0x00000003
+#define VGT_GS_MODE__RESERVED_1_MASK 0x000007c0L
+#define VGT_GS_MODE__RESERVED_1__SHIFT 0x00000006
+#define VGT_GS_MODE__RESERVED_2_MASK 0x00001000L
+#define VGT_GS_MODE__RESERVED_2__SHIFT 0x0000000c
+#define VGT_GS_MODE__SUPPRESS_CUTS_MASK 0x00040000L
+#define VGT_GS_MODE__SUPPRESS_CUTS__SHIFT 0x00000012
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_1_MASK 0x00003f00L
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_1__SHIFT 0x00000008
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_2_MASK 0x003f0000L
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_2__SHIFT 0x00000010
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_3_MASK 0x0fc00000L
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_3__SHIFT 0x00000016
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_MASK 0x0000003fL
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE__SHIFT 0x00000000
+#define VGT_GS_OUT_PRIM_TYPE__UNIQUE_TYPE_PER_STREAM_MASK 0x80000000L
+#define VGT_GS_OUT_PRIM_TYPE__UNIQUE_TYPE_PER_STREAM__SHIFT 0x0000001f
+#define VGT_GS_PER_ES__GS_PER_ES_MASK 0x000007ffL
+#define VGT_GS_PER_ES__GS_PER_ES__SHIFT 0x00000000
+#define VGT_GS_PER_VS__GS_PER_VS_MASK 0x0000000fL
+#define VGT_GS_PER_VS__GS_PER_VS__SHIFT 0x00000000
+#define VGT_GS_VERTEX_REUSE__VERT_REUSE_MASK 0x0000001fL
+#define VGT_GS_VERTEX_REUSE__VERT_REUSE__SHIFT 0x00000000
+#define VGT_GS_VERT_ITEMSIZE_1__ITEMSIZE_MASK 0x00007fffL
+#define VGT_GS_VERT_ITEMSIZE_1__ITEMSIZE__SHIFT 0x00000000
+#define VGT_GS_VERT_ITEMSIZE_2__ITEMSIZE_MASK 0x00007fffL
+#define VGT_GS_VERT_ITEMSIZE_2__ITEMSIZE__SHIFT 0x00000000
+#define VGT_GS_VERT_ITEMSIZE_3__ITEMSIZE_MASK 0x00007fffL
+#define VGT_GS_VERT_ITEMSIZE_3__ITEMSIZE__SHIFT 0x00000000
+#define VGT_GS_VERT_ITEMSIZE__ITEMSIZE_MASK 0x00007fffL
+#define VGT_GS_VERT_ITEMSIZE__ITEMSIZE__SHIFT 0x00000000
+#define VGT_GSVS_RING_ITEMSIZE__ITEMSIZE_MASK 0x00007fffL
+#define VGT_GSVS_RING_ITEMSIZE__ITEMSIZE__SHIFT 0x00000000
+#define VGT_GSVS_RING_OFFSET_1__OFFSET_MASK 0x00007fffL
+#define VGT_GSVS_RING_OFFSET_1__OFFSET__SHIFT 0x00000000
+#define VGT_GSVS_RING_OFFSET_2__OFFSET_MASK 0x00007fffL
+#define VGT_GSVS_RING_OFFSET_2__OFFSET__SHIFT 0x00000000
+#define VGT_GSVS_RING_OFFSET_3__OFFSET_MASK 0x00007fffL
+#define VGT_GSVS_RING_OFFSET_3__OFFSET__SHIFT 0x00000000
+#define VGT_GSVS_RING_SIZE__MEM_SIZE_MASK 0xffffffffL
+#define VGT_GSVS_RING_SIZE__MEM_SIZE__SHIFT 0x00000000
+#define VGT_HOS_CNTL__TESS_MODE_MASK 0x00000003L
+#define VGT_HOS_CNTL__TESS_MODE__SHIFT 0x00000000
+#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS_MASK 0xffffffffL
+#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS__SHIFT 0x00000000
+#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS_MASK 0xffffffffL
+#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS__SHIFT 0x00000000
+#define VGT_HOS_REUSE_DEPTH__REUSE_DEPTH_MASK 0x000000ffL
+#define VGT_HOS_REUSE_DEPTH__REUSE_DEPTH__SHIFT 0x00000000
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING_MASK 0x0000007fL
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING__SHIFT 0x00000000
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY_MASK 0x00000600L
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY__SHIFT 0x00000009
+#define VGT_IMMED_DATA__DATA_MASK 0xffffffffL
+#define VGT_IMMED_DATA__DATA__SHIFT 0x00000000
+#define VGT_INDEX_TYPE__INDEX_TYPE_MASK 0x00000003L
+#define VGT_INDEX_TYPE__INDEX_TYPE__SHIFT 0x00000000
+#define VGT_INDX_OFFSET__INDX_OFFSET_MASK 0xffffffffL
+#define VGT_INDX_OFFSET__INDX_OFFSET__SHIFT 0x00000000
+#define VGT_INSTANCE_STEP_RATE_0__STEP_RATE_MASK 0xffffffffL
+#define VGT_INSTANCE_STEP_RATE_0__STEP_RATE__SHIFT 0x00000000
+#define VGT_INSTANCE_STEP_RATE_1__STEP_RATE_MASK 0xffffffffL
+#define VGT_INSTANCE_STEP_RATE_1__STEP_RATE__SHIFT 0x00000000
+#define VGT_LAST_COPY_STATE__DST_STATE_ID_MASK 0x00070000L
+#define VGT_LAST_COPY_STATE__DST_STATE_ID__SHIFT 0x00000010
+#define VGT_LAST_COPY_STATE__SRC_STATE_ID_MASK 0x00000007L
+#define VGT_LAST_COPY_STATE__SRC_STATE_ID__SHIFT 0x00000000
+#define VGT_LS_HS_CONFIG__HS_NUM_INPUT_CP_MASK 0x00003f00L
+#define VGT_LS_HS_CONFIG__HS_NUM_INPUT_CP__SHIFT 0x00000008
+#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP_MASK 0x000fc000L
+#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP__SHIFT 0x0000000e
+#define VGT_LS_HS_CONFIG__NUM_PATCHES_MASK 0x000000ffL
+#define VGT_LS_HS_CONFIG__NUM_PATCHES__SHIFT 0x00000000
+#define VGT_MAX_VTX_INDX__MAX_INDX_MASK 0xffffffffL
+#define VGT_MAX_VTX_INDX__MAX_INDX__SHIFT 0x00000000
+#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES_MASK 0x00000003L
+#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES__SHIFT 0x00000000
+#define VGT_MIN_VTX_INDX__MIN_INDX_MASK 0xffffffffL
+#define VGT_MIN_VTX_INDX__MIN_INDX__SHIFT 0x00000000
+#define VGT_MULTI_PRIM_IB_RESET_EN__RESET_EN_MASK 0x00000001L
+#define VGT_MULTI_PRIM_IB_RESET_EN__RESET_EN__SHIFT 0x00000000
+#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX_MASK 0xffffffffL
+#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX__SHIFT 0x00000000
+#define VGT_NUM_INDICES__NUM_INDICES_MASK 0xffffffffL
+#define VGT_NUM_INDICES__NUM_INDICES__SHIFT 0x00000000
+#define VGT_NUM_INSTANCES__NUM_INSTANCES_MASK 0xffffffffL
+#define VGT_NUM_INSTANCES__NUM_INSTANCES__SHIFT 0x00000000
+#define VGT_OUT_DEALLOC_CNTL__DEALLOC_DIST_MASK 0x0000007fL
+#define VGT_OUT_DEALLOC_CNTL__DEALLOC_DIST__SHIFT 0x00000000
+#define VGT_OUTPUT_PATH_CNTL__PATH_SELECT_MASK 0x00000007L
+#define VGT_OUTPUT_PATH_CNTL__PATH_SELECT__SHIFT 0x00000000
+#define VGT_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define VGT_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define VGT_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define VGT_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000L
+#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x0000001c
+#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0x0f000000L
+#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x00000018
+#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define VGT_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define VGT_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define VGT_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define VGT_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define VGT_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000L
+#define VGT_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define VGT_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define VGT_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define VGT_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x000000ffL
+#define VGT_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define VGT_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define VGT_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define VGT_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define VGT_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf0000000L
+#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x0000001c
+#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0x0f000000L
+#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x00000018
+#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x000003ffL
+#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x00000000
+#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0x000ffc00L
+#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0x0000000a
+#define VGT_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0x00f00000L
+#define VGT_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x00000014
+#define VGT_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0x0f000000L
+#define VGT_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x00000018
+#define VGT_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000L
+#define VGT_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define VGT_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0x000ffc00L
+#define VGT_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0x0000000a
+#define VGT_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x000000ffL
+#define VGT_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define VGT_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define VGT_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define VGT_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define VGT_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define VGT_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000L
+#define VGT_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define VGT_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x000000ffL
+#define VGT_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x00000000
+#define VGT_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffffL
+#define VGT_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x00000000
+#define VGT_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffffL
+#define VGT_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x00000000
+#define VGT_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000L
+#define VGT_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x0000001c
+#define VGT_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x000000ffL
+#define VGT_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x00000000
+#define VGT_PERFCOUNTER_SEID_MASK__PERF_SEID_IGNORE_MASK_MASK 0x000000ffL
+#define VGT_PERFCOUNTER_SEID_MASK__PERF_SEID_IGNORE_MASK__SHIFT 0x00000000
+#define VGT_PRIMITIVEID_EN__DISABLE_RESET_ON_EOI_MASK 0x00000002L
+#define VGT_PRIMITIVEID_EN__DISABLE_RESET_ON_EOI__SHIFT 0x00000001
+#define VGT_PRIMITIVEID_EN__PRIMITIVEID_EN_MASK 0x00000001L
+#define VGT_PRIMITIVEID_EN__PRIMITIVEID_EN__SHIFT 0x00000000
+#define VGT_PRIMITIVEID_RESET__VALUE_MASK 0xffffffffL
+#define VGT_PRIMITIVEID_RESET__VALUE__SHIFT 0x00000000
+#define VGT_PRIMITIVE_TYPE__PRIM_TYPE_MASK 0x0000003fL
+#define VGT_PRIMITIVE_TYPE__PRIM_TYPE__SHIFT 0x00000000
+#define VGT_REUSE_OFF__REUSE_OFF_MASK 0x00000001L
+#define VGT_REUSE_OFF__REUSE_OFF__SHIFT 0x00000000
+#define VGT_SHADER_STAGES_EN__DYNAMIC_HS_MASK 0x00000100L
+#define VGT_SHADER_STAGES_EN__DYNAMIC_HS__SHIFT 0x00000008
+#define VGT_SHADER_STAGES_EN__ES_EN_MASK 0x00000018L
+#define VGT_SHADER_STAGES_EN__ES_EN__SHIFT 0x00000003
+#define VGT_SHADER_STAGES_EN__GS_EN_MASK 0x00000020L
+#define VGT_SHADER_STAGES_EN__GS_EN__SHIFT 0x00000005
+#define VGT_SHADER_STAGES_EN__HS_EN_MASK 0x00000004L
+#define VGT_SHADER_STAGES_EN__HS_EN__SHIFT 0x00000002
+#define VGT_SHADER_STAGES_EN__LS_EN_MASK 0x00000003L
+#define VGT_SHADER_STAGES_EN__LS_EN__SHIFT 0x00000000
+#define VGT_SHADER_STAGES_EN__VS_EN_MASK 0x000000c0L
+#define VGT_SHADER_STAGES_EN__VS_EN__SHIFT 0x00000006
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_0_BUFFER_EN_MASK 0x0000000fL
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_0_BUFFER_EN__SHIFT 0x00000000
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_1_BUFFER_EN_MASK 0x000000f0L
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_1_BUFFER_EN__SHIFT 0x00000004
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_2_BUFFER_EN_MASK 0x00000f00L
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_2_BUFFER_EN__SHIFT 0x00000008
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_3_BUFFER_EN_MASK 0x0000f000L
+#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_3_BUFFER_EN__SHIFT 0x0000000c
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_0__SIZE_MASK 0xffffffffL
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_0__SIZE__SHIFT 0x00000000
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_1__SIZE_MASK 0xffffffffL
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_1__SIZE__SHIFT 0x00000000
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_2__SIZE_MASK 0xffffffffL
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_2__SIZE__SHIFT 0x00000000
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_3__SIZE_MASK 0xffffffffL
+#define VGT_STRMOUT_BUFFER_FILLED_SIZE_3__SIZE__SHIFT 0x00000000
+#define VGT_STRMOUT_BUFFER_OFFSET_0__OFFSET_MASK 0xffffffffL
+#define VGT_STRMOUT_BUFFER_OFFSET_0__OFFSET__SHIFT 0x00000000
+#define VGT_STRMOUT_BUFFER_OFFSET_1__OFFSET_MASK 0xffffffffL
+#define VGT_STRMOUT_BUFFER_OFFSET_1__OFFSET__SHIFT 0x00000000
+#define VGT_STRMOUT_BUFFER_OFFSET_2__OFFSET_MASK 0xffffffffL
+#define VGT_STRMOUT_BUFFER_OFFSET_2__OFFSET__SHIFT 0x00000000
+#define VGT_STRMOUT_BUFFER_OFFSET_3__OFFSET_MASK 0xffffffffL
+#define VGT_STRMOUT_BUFFER_OFFSET_3__OFFSET__SHIFT 0x00000000
+#define VGT_STRMOUT_BUFFER_SIZE_0__SIZE_MASK 0xffffffffL
+#define VGT_STRMOUT_BUFFER_SIZE_0__SIZE__SHIFT 0x00000000
+#define VGT_STRMOUT_BUFFER_SIZE_1__SIZE_MASK 0xffffffffL
+#define VGT_STRMOUT_BUFFER_SIZE_1__SIZE__SHIFT 0x00000000
+#define VGT_STRMOUT_BUFFER_SIZE_2__SIZE_MASK 0xffffffffL
+#define VGT_STRMOUT_BUFFER_SIZE_2__SIZE__SHIFT 0x00000000
+#define VGT_STRMOUT_BUFFER_SIZE_3__SIZE_MASK 0xffffffffL
+#define VGT_STRMOUT_BUFFER_SIZE_3__SIZE__SHIFT 0x00000000
+#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK 0x00000070L
+#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK_MASK 0x00000f00L
+#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK__SHIFT 0x00000008
+#define VGT_STRMOUT_CONFIG__RAST_STREAM__SHIFT 0x00000004
+#define VGT_STRMOUT_CONFIG__STREAMOUT_0_EN_MASK 0x00000001L
+#define VGT_STRMOUT_CONFIG__STREAMOUT_0_EN__SHIFT 0x00000000
+#define VGT_STRMOUT_CONFIG__STREAMOUT_1_EN_MASK 0x00000002L
+#define VGT_STRMOUT_CONFIG__STREAMOUT_1_EN__SHIFT 0x00000001
+#define VGT_STRMOUT_CONFIG__STREAMOUT_2_EN_MASK 0x00000004L
+#define VGT_STRMOUT_CONFIG__STREAMOUT_2_EN__SHIFT 0x00000002
+#define VGT_STRMOUT_CONFIG__STREAMOUT_3_EN_MASK 0x00000008L
+#define VGT_STRMOUT_CONFIG__STREAMOUT_3_EN__SHIFT 0x00000003
+#define VGT_STRMOUT_CONFIG__USE_RAST_STREAM_MASK_MASK 0x80000000L
+#define VGT_STRMOUT_CONFIG__USE_RAST_STREAM_MASK__SHIFT 0x0000001f
+#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE_MASK 0xffffffffL
+#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE__SHIFT 0x00000000
+#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET_MASK 0xffffffffL
+#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET__SHIFT 0x00000000
+#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE_MASK 0x000001ffL
+#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE__SHIFT 0x00000000
+#define VGT_STRMOUT_VTX_STRIDE_0__STRIDE_MASK 0x000003ffL
+#define VGT_STRMOUT_VTX_STRIDE_0__STRIDE__SHIFT 0x00000000
+#define VGT_STRMOUT_VTX_STRIDE_1__STRIDE_MASK 0x000003ffL
+#define VGT_STRMOUT_VTX_STRIDE_1__STRIDE__SHIFT 0x00000000
+#define VGT_STRMOUT_VTX_STRIDE_2__STRIDE_MASK 0x000003ffL
+#define VGT_STRMOUT_VTX_STRIDE_2__STRIDE__SHIFT 0x00000000
+#define VGT_STRMOUT_VTX_STRIDE_3__STRIDE_MASK 0x000003ffL
+#define VGT_STRMOUT_VTX_STRIDE_3__STRIDE__SHIFT 0x00000000
+#define VGT_SYS_CONFIG__ADC_EVENT_FILTER_DISABLE_MASK 0x00000080L
+#define VGT_SYS_CONFIG__ADC_EVENT_FILTER_DISABLE__SHIFT 0x00000007
+#define VGT_SYS_CONFIG__DUAL_CORE_EN_MASK 0x00000001L
+#define VGT_SYS_CONFIG__DUAL_CORE_EN__SHIFT 0x00000000
+#define VGT_SYS_CONFIG__MAX_LS_HS_THDGRP_MASK 0x0000007eL
+#define VGT_SYS_CONFIG__MAX_LS_HS_THDGRP__SHIFT 0x00000001
+#define VGT_TF_MEMORY_BASE__BASE_MASK 0xffffffffL
+#define VGT_TF_MEMORY_BASE__BASE__SHIFT 0x00000000
+#define VGT_TF_PARAM__DEPRECATED_MASK 0x00000200L
+#define VGT_TF_PARAM__DEPRECATED__SHIFT 0x00000009
+#define VGT_TF_PARAM__DISABLE_DONUTS_MASK 0x00004000L
+#define VGT_TF_PARAM__DISABLE_DONUTS__SHIFT 0x0000000e
+#define VGT_TF_PARAM__NUM_DS_WAVES_PER_SIMD_MASK 0x00003c00L
+#define VGT_TF_PARAM__NUM_DS_WAVES_PER_SIMD__SHIFT 0x0000000a
+#define VGT_TF_PARAM__PARTITIONING_MASK 0x0000001cL
+#define VGT_TF_PARAM__PARTITIONING__SHIFT 0x00000002
+#define VGT_TF_PARAM__RDREQ_POLICY_MASK 0x00018000L
+#define VGT_TF_PARAM__RDREQ_POLICY__SHIFT 0x0000000f
+#define VGT_TF_PARAM__RESERVED_REDUC_AXIS_MASK 0x00000100L
+#define VGT_TF_PARAM__RESERVED_REDUC_AXIS__SHIFT 0x00000008
+#define VGT_TF_PARAM__TOPOLOGY_MASK 0x000000e0L
+#define VGT_TF_PARAM__TOPOLOGY__SHIFT 0x00000005
+#define VGT_TF_PARAM__TYPE_MASK 0x00000003L
+#define VGT_TF_PARAM__TYPE__SHIFT 0x00000000
+#define VGT_TF_RING_SIZE__SIZE_MASK 0x0000ffffL
+#define VGT_TF_RING_SIZE__SIZE__SHIFT 0x00000000
+#define VGT_VERTEX_REUSE_BLOCK_CNTL__VTX_REUSE_DEPTH_MASK 0x000000ffL
+#define VGT_VERTEX_REUSE_BLOCK_CNTL__VTX_REUSE_DEPTH__SHIFT 0x00000000
+#define VGT_VTX_CNT_EN__VTX_CNT_EN_MASK 0x00000001L
+#define VGT_VTX_CNT_EN__VTX_CNT_EN__SHIFT 0x00000000
+#define VGT_VTX_VECT_EJECT_REG__PRIM_COUNT_MASK 0x000003ffL
+#define VGT_VTX_VECT_EJECT_REG__PRIM_COUNT__SHIFT 0x00000000
+#define WD_DEBUG_DATA__DATA_MASK 0xffffffffL
+#define WD_DEBUG_DATA__DATA__SHIFT 0x00000000
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gmc/gmc_6_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/gmc/gmc_6_0_d.h
new file mode 100644
index 000000000000..dc4e5b93801d
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/gmc/gmc_6_0_d.h
@@ -0,0 +1,1274 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef GMC_6_0_D_H
+#define GMC_6_0_D_H
+
+#define ixMC_IO_DEBUG_ACMD_CLKSEL_D0 0x00CE
+#define ixMC_IO_DEBUG_ACMD_CLKSEL_D1 0x00DE
+#define ixMC_IO_DEBUG_ACMD_MISC_D0 0x00AE
+#define ixMC_IO_DEBUG_ACMD_MISC_D1 0x00BE
+#define ixMC_IO_DEBUG_ACMD_OFSCAL_D0 0x00EE
+#define ixMC_IO_DEBUG_ACMD_OFSCAL_D1 0x00FE
+#define ixMC_IO_DEBUG_ACMD_RXPHASE_D0 0x010E
+#define ixMC_IO_DEBUG_ACMD_RXPHASE_D1 0x011E
+#define ixMC_IO_DEBUG_ACMD_TXBST_PD_D0 0x018E
+#define ixMC_IO_DEBUG_ACMD_TXBST_PD_D1 0x019E
+#define ixMC_IO_DEBUG_ACMD_TXBST_PU_D0 0x01AE
+#define ixMC_IO_DEBUG_ACMD_TXBST_PU_D1 0x01BE
+#define ixMC_IO_DEBUG_ACMD_TXPHASE_D0 0x012E
+#define ixMC_IO_DEBUG_ACMD_TXPHASE_D1 0x013E
+#define ixMC_IO_DEBUG_ACMD_TXSLF_D0 0x016E
+#define ixMC_IO_DEBUG_ACMD_TXSLF_D1 0x017E
+#define ixMC_IO_DEBUG_ADDRH_CLKSEL_D0 0x00CD
+#define ixMC_IO_DEBUG_ADDRH_CLKSEL_D1 0x00DD
+#define ixMC_IO_DEBUG_ADDRH_MISC_D0 0x00AD
+#define ixMC_IO_DEBUG_ADDRH_MISC_D1 0x00BD
+#define ixMC_IO_DEBUG_ADDRH_RXPHASE_D0 0x010D
+#define ixMC_IO_DEBUG_ADDRH_RXPHASE_D1 0x011D
+#define ixMC_IO_DEBUG_ADDRH_TXBST_PD_D0 0x018D
+#define ixMC_IO_DEBUG_ADDRH_TXBST_PD_D1 0x019D
+#define ixMC_IO_DEBUG_ADDRH_TXBST_PU_D0 0x01AD
+#define ixMC_IO_DEBUG_ADDRH_TXBST_PU_D1 0x01BD
+#define ixMC_IO_DEBUG_ADDRH_TXPHASE_D0 0x012D
+#define ixMC_IO_DEBUG_ADDRH_TXPHASE_D1 0x013D
+#define ixMC_IO_DEBUG_ADDRH_TXSLF_D0 0x016D
+#define ixMC_IO_DEBUG_ADDRH_TXSLF_D1 0x017D
+#define ixMC_IO_DEBUG_ADDRL_CLKSEL_D0 0x00CC
+#define ixMC_IO_DEBUG_ADDRL_CLKSEL_D1 0x00DC
+#define ixMC_IO_DEBUG_ADDRL_MISC_D0 0x00AC
+#define ixMC_IO_DEBUG_ADDRL_MISC_D1 0x00BC
+#define ixMC_IO_DEBUG_ADDRL_RXPHASE_D0 0x010C
+#define ixMC_IO_DEBUG_ADDRL_RXPHASE_D1 0x011C
+#define ixMC_IO_DEBUG_ADDRL_TXBST_PD_D0 0x018C
+#define ixMC_IO_DEBUG_ADDRL_TXBST_PD_D1 0x019C
+#define ixMC_IO_DEBUG_ADDRL_TXBST_PU_D0 0x01AC
+#define ixMC_IO_DEBUG_ADDRL_TXBST_PU_D1 0x01BC
+#define ixMC_IO_DEBUG_ADDRL_TXPHASE_D0 0x012C
+#define ixMC_IO_DEBUG_ADDRL_TXPHASE_D1 0x013C
+#define ixMC_IO_DEBUG_ADDRL_TXSLF_D0 0x016C
+#define ixMC_IO_DEBUG_ADDRL_TXSLF_D1 0x017C
+#define ixMC_IO_DEBUG_CK_CLKSEL_D0 0x00CB
+#define ixMC_IO_DEBUG_CK_CLKSEL_D1 0x00DB
+#define ixMC_IO_DEBUG_CK_MISC_D0 0x00AB
+#define ixMC_IO_DEBUG_CK_MISC_D1 0x00BB
+#define ixMC_IO_DEBUG_CK_RXPHASE_D0 0x010B
+#define ixMC_IO_DEBUG_CK_RXPHASE_D1 0x011B
+#define ixMC_IO_DEBUG_CK_TXBST_PD_D0 0x018B
+#define ixMC_IO_DEBUG_CK_TXBST_PD_D1 0x019B
+#define ixMC_IO_DEBUG_CK_TXBST_PU_D0 0x01AB
+#define ixMC_IO_DEBUG_CK_TXBST_PU_D1 0x01BB
+#define ixMC_IO_DEBUG_CK_TXPHASE_D0 0x012B
+#define ixMC_IO_DEBUG_CK_TXPHASE_D1 0x013B
+#define ixMC_IO_DEBUG_CK_TXSLF_D0 0x016B
+#define ixMC_IO_DEBUG_CK_TXSLF_D1 0x017B
+#define ixMC_IO_DEBUG_CMD_CLKSEL_D0 0x00CF
+#define ixMC_IO_DEBUG_CMD_CLKSEL_D1 0x00DF
+#define ixMC_IO_DEBUG_CMD_MISC_D0 0x00AF
+#define ixMC_IO_DEBUG_CMD_MISC_D1 0x00BF
+#define ixMC_IO_DEBUG_CMD_OFSCAL_D0 0x00EF
+#define ixMC_IO_DEBUG_CMD_OFSCAL_D1 0x00FF
+#define ixMC_IO_DEBUG_CMD_RX_EQ_D0 0x01CF
+#define ixMC_IO_DEBUG_CMD_RX_EQ_D1 0x01DF
+#define ixMC_IO_DEBUG_CMD_RXPHASE_D0 0x010F
+#define ixMC_IO_DEBUG_CMD_RXPHASE_D1 0x011F
+#define ixMC_IO_DEBUG_CMD_TXBST_PD_D0 0x018F
+#define ixMC_IO_DEBUG_CMD_TXBST_PD_D1 0x019F
+#define ixMC_IO_DEBUG_CMD_TXBST_PU_D0 0x01AF
+#define ixMC_IO_DEBUG_CMD_TXBST_PU_D1 0x01BF
+#define ixMC_IO_DEBUG_CMD_TXPHASE_D0 0x012F
+#define ixMC_IO_DEBUG_CMD_TXPHASE_D1 0x013F
+#define ixMC_IO_DEBUG_CMD_TXSLF_D0 0x016F
+#define ixMC_IO_DEBUG_CMD_TXSLF_D1 0x017F
+#define ixMC_IO_DEBUG_DBI_CDR_PHSIZE_D0 0x014F
+#define ixMC_IO_DEBUG_DBI_CDR_PHSIZE_D1 0x015F
+#define ixMC_IO_DEBUG_DBI_CLKSEL_D0 0x00C8
+#define ixMC_IO_DEBUG_DBI_CLKSEL_D1 0x00D8
+#define ixMC_IO_DEBUG_DBI_MISC_D0 0x00A8
+#define ixMC_IO_DEBUG_DBI_MISC_D1 0x00B8
+#define ixMC_IO_DEBUG_DBI_OFSCAL_D0 0x00E8
+#define ixMC_IO_DEBUG_DBI_OFSCAL_D1 0x00F8
+#define ixMC_IO_DEBUG_DBI_RX_EQ_D0 0x01C8
+#define ixMC_IO_DEBUG_DBI_RX_EQ_D1 0x01D8
+#define ixMC_IO_DEBUG_DBI_RXPHASE_D0 0x0108
+#define ixMC_IO_DEBUG_DBI_RXPHASE_D1 0x0118
+#define ixMC_IO_DEBUG_DBI_RX_VREF_CAL_D0 0x0148
+#define ixMC_IO_DEBUG_DBI_RX_VREF_CAL_D1 0x0158
+#define ixMC_IO_DEBUG_DBI_TXBST_PD_D0 0x0188
+#define ixMC_IO_DEBUG_DBI_TXBST_PD_D1 0x0198
+#define ixMC_IO_DEBUG_DBI_TXBST_PU_D0 0x01A8
+#define ixMC_IO_DEBUG_DBI_TXBST_PU_D1 0x01B8
+#define ixMC_IO_DEBUG_DBI_TXPHASE_D0 0x0128
+#define ixMC_IO_DEBUG_DBI_TXPHASE_D1 0x0138
+#define ixMC_IO_DEBUG_DBI_TXSLF_D0 0x0168
+#define ixMC_IO_DEBUG_DBI_TXSLF_D1 0x0178
+#define ixMC_IO_DEBUG_DQ0_RX_DYN_PM_D0 0x01CD
+#define ixMC_IO_DEBUG_DQ0_RX_DYN_PM_D1 0x01DD
+#define ixMC_IO_DEBUG_DQ0_RX_EQ_PM_D0 0x01CB
+#define ixMC_IO_DEBUG_DQ0_RX_EQ_PM_D1 0x01DB
+#define ixMC_IO_DEBUG_DQ1_RX_DYN_PM_D0 0x01CE
+#define ixMC_IO_DEBUG_DQ1_RX_DYN_PM_D1 0x01DE
+#define ixMC_IO_DEBUG_DQ1_RX_EQ_PM_D0 0x01CC
+#define ixMC_IO_DEBUG_DQ1_RX_EQ_PM_D1 0x01DC
+#define ixMC_IO_DEBUG_DQB0_CDR_PHSIZE_D0 0x014B
+#define ixMC_IO_DEBUG_DQB0_CDR_PHSIZE_D1 0x015B
+#define ixMC_IO_DEBUG_DQB0H_CLKSEL_D0 0x00C1
+#define ixMC_IO_DEBUG_DQB0H_CLKSEL_D1 0x00D1
+#define ixMC_IO_DEBUG_DQB0H_MISC_D0 0x00A1
+#define ixMC_IO_DEBUG_DQB0H_MISC_D1 0x00B1
+#define ixMC_IO_DEBUG_DQB0H_OFSCAL_D0 0x00E1
+#define ixMC_IO_DEBUG_DQB0H_OFSCAL_D1 0x00F1
+#define ixMC_IO_DEBUG_DQB0H_RX_EQ_D0 0x01C1
+#define ixMC_IO_DEBUG_DQB0H_RX_EQ_D1 0x01D1
+#define ixMC_IO_DEBUG_DQB0H_RXPHASE_D0 0x0101
+#define ixMC_IO_DEBUG_DQB0H_RXPHASE_D1 0x0111
+#define ixMC_IO_DEBUG_DQB0H_RX_VREF_CAL_D0 0x0141
+#define ixMC_IO_DEBUG_DQB0H_RX_VREF_CAL_D1 0x0151
+#define ixMC_IO_DEBUG_DQB0H_TXBST_PD_D0 0x0181
+#define ixMC_IO_DEBUG_DQB0H_TXBST_PD_D1 0x0191
+#define ixMC_IO_DEBUG_DQB0H_TXBST_PU_D0 0x01A1
+#define ixMC_IO_DEBUG_DQB0H_TXBST_PU_D1 0x01B1
+#define ixMC_IO_DEBUG_DQB0H_TXPHASE_D0 0x0121
+#define ixMC_IO_DEBUG_DQB0H_TXPHASE_D1 0x0131
+#define ixMC_IO_DEBUG_DQB0H_TXSLF_D0 0x0161
+#define ixMC_IO_DEBUG_DQB0H_TXSLF_D1 0x0171
+#define ixMC_IO_DEBUG_DQB0L_CLKSEL_D0 0x00C0
+#define ixMC_IO_DEBUG_DQB0L_CLKSEL_D1 0x00D0
+#define ixMC_IO_DEBUG_DQB0L_MISC_D0 0x00A0
+#define ixMC_IO_DEBUG_DQB0L_MISC_D1 0x00B0
+#define ixMC_IO_DEBUG_DQB0L_OFSCAL_D0 0x00E0
+#define ixMC_IO_DEBUG_DQB0L_OFSCAL_D1 0x00F0
+#define ixMC_IO_DEBUG_DQB0L_RX_EQ_D0 0x01C0
+#define ixMC_IO_DEBUG_DQB0L_RX_EQ_D1 0x01D0
+#define ixMC_IO_DEBUG_DQB0L_RXPHASE_D0 0x0100
+#define ixMC_IO_DEBUG_DQB0L_RXPHASE_D1 0x0110
+#define ixMC_IO_DEBUG_DQB0L_RX_VREF_CAL_D0 0x0140
+#define ixMC_IO_DEBUG_DQB0L_RX_VREF_CAL_D1 0x0150
+#define ixMC_IO_DEBUG_DQB0L_TXBST_PD_D0 0x0180
+#define ixMC_IO_DEBUG_DQB0L_TXBST_PD_D1 0x0190
+#define ixMC_IO_DEBUG_DQB0L_TXBST_PU_D0 0x01A0
+#define ixMC_IO_DEBUG_DQB0L_TXBST_PU_D1 0x01B0
+#define ixMC_IO_DEBUG_DQB0L_TXPHASE_D0 0x0120
+#define ixMC_IO_DEBUG_DQB0L_TXPHASE_D1 0x0130
+#define ixMC_IO_DEBUG_DQB0L_TXSLF_D0 0x0160
+#define ixMC_IO_DEBUG_DQB0L_TXSLF_D1 0x0170
+#define ixMC_IO_DEBUG_DQB1_CDR_PHSIZE_D0 0x014C
+#define ixMC_IO_DEBUG_DQB1_CDR_PHSIZE_D1 0x015C
+#define ixMC_IO_DEBUG_DQB1H_CLKSEL_D0 0x00C3
+#define ixMC_IO_DEBUG_DQB1H_CLKSEL_D1 0x00D3
+#define ixMC_IO_DEBUG_DQB1H_MISC_D0 0x00A3
+#define ixMC_IO_DEBUG_DQB1H_MISC_D1 0x00B3
+#define ixMC_IO_DEBUG_DQB1H_OFSCAL_D0 0x00E3
+#define ixMC_IO_DEBUG_DQB1H_OFSCAL_D1 0x00F3
+#define ixMC_IO_DEBUG_DQB1H_RX_EQ_D0 0x01C3
+#define ixMC_IO_DEBUG_DQB1H_RX_EQ_D1 0x01D3
+#define ixMC_IO_DEBUG_DQB1H_RXPHASE_D0 0x0103
+#define ixMC_IO_DEBUG_DQB1H_RXPHASE_D1 0x0113
+#define ixMC_IO_DEBUG_DQB1H_RX_VREF_CAL_D0 0x0143
+#define ixMC_IO_DEBUG_DQB1H_RX_VREF_CAL_D1 0x0153
+#define ixMC_IO_DEBUG_DQB1H_TXBST_PD_D0 0x0183
+#define ixMC_IO_DEBUG_DQB1H_TXBST_PD_D1 0x0193
+#define ixMC_IO_DEBUG_DQB1H_TXBST_PU_D0 0x01A3
+#define ixMC_IO_DEBUG_DQB1H_TXBST_PU_D1 0x01B3
+#define ixMC_IO_DEBUG_DQB1H_TXPHASE_D0 0x0123
+#define ixMC_IO_DEBUG_DQB1H_TXPHASE_D1 0x0133
+#define ixMC_IO_DEBUG_DQB1H_TXSLF_D0 0x0163
+#define ixMC_IO_DEBUG_DQB1H_TXSLF_D1 0x0173
+#define ixMC_IO_DEBUG_DQB1L_CLKSEL_D0 0x00C2
+#define ixMC_IO_DEBUG_DQB1L_CLKSEL_D1 0x00D2
+#define ixMC_IO_DEBUG_DQB1L_MISC_D0 0x00A2
+#define ixMC_IO_DEBUG_DQB1L_MISC_D1 0x00B2
+#define ixMC_IO_DEBUG_DQB1L_OFSCAL_D0 0x00E2
+#define ixMC_IO_DEBUG_DQB1L_OFSCAL_D1 0x00F2
+#define ixMC_IO_DEBUG_DQB1L_RX_EQ_D0 0x01C2
+#define ixMC_IO_DEBUG_DQB1L_RX_EQ_D1 0x01D2
+#define ixMC_IO_DEBUG_DQB1L_RXPHASE_D0 0x0102
+#define ixMC_IO_DEBUG_DQB1L_RXPHASE_D1 0x0112
+#define ixMC_IO_DEBUG_DQB1L_RX_VREF_CAL_D0 0x0142
+#define ixMC_IO_DEBUG_DQB1L_RX_VREF_CAL_D1 0x0152
+#define ixMC_IO_DEBUG_DQB1L_TXBST_PD_D0 0x0182
+#define ixMC_IO_DEBUG_DQB1L_TXBST_PD_D1 0x0192
+#define ixMC_IO_DEBUG_DQB1L_TXBST_PU_D0 0x01A2
+#define ixMC_IO_DEBUG_DQB1L_TXBST_PU_D1 0x01B2
+#define ixMC_IO_DEBUG_DQB1L_TXPHASE_D0 0x0122
+#define ixMC_IO_DEBUG_DQB1L_TXPHASE_D1 0x0132
+#define ixMC_IO_DEBUG_DQB1L_TXSLF_D0 0x0162
+#define ixMC_IO_DEBUG_DQB1L_TXSLF_D1 0x0172
+#define ixMC_IO_DEBUG_DQB2_CDR_PHSIZE_D0 0x014D
+#define ixMC_IO_DEBUG_DQB2_CDR_PHSIZE_D1 0x015D
+#define ixMC_IO_DEBUG_DQB2H_CLKSEL_D0 0x00C5
+#define ixMC_IO_DEBUG_DQB2H_CLKSEL_D1 0x00D5
+#define ixMC_IO_DEBUG_DQB2H_MISC_D0 0x00A5
+#define ixMC_IO_DEBUG_DQB2H_MISC_D1 0x00B5
+#define ixMC_IO_DEBUG_DQB2H_OFSCAL_D0 0x00E5
+#define ixMC_IO_DEBUG_DQB2H_OFSCAL_D1 0x00F5
+#define ixMC_IO_DEBUG_DQB2H_RX_EQ_D0 0x01C5
+#define ixMC_IO_DEBUG_DQB2H_RX_EQ_D1 0x01D5
+#define ixMC_IO_DEBUG_DQB2H_RXPHASE_D0 0x0105
+#define ixMC_IO_DEBUG_DQB2H_RXPHASE_D1 0x0115
+#define ixMC_IO_DEBUG_DQB2H_RX_VREF_CAL_D0 0x0145
+#define ixMC_IO_DEBUG_DQB2H_RX_VREF_CAL_D1 0x0155
+#define ixMC_IO_DEBUG_DQB2H_TXBST_PD_D0 0x0185
+#define ixMC_IO_DEBUG_DQB2H_TXBST_PD_D1 0x0195
+#define ixMC_IO_DEBUG_DQB2H_TXBST_PU_D0 0x01A5
+#define ixMC_IO_DEBUG_DQB2H_TXBST_PU_D1 0x01B5
+#define ixMC_IO_DEBUG_DQB2H_TXPHASE_D0 0x0125
+#define ixMC_IO_DEBUG_DQB2H_TXPHASE_D1 0x0135
+#define ixMC_IO_DEBUG_DQB2H_TXSLF_D0 0x0165
+#define ixMC_IO_DEBUG_DQB2H_TXSLF_D1 0x0175
+#define ixMC_IO_DEBUG_DQB2L_CLKSEL_D0 0x00C4
+#define ixMC_IO_DEBUG_DQB2L_CLKSEL_D1 0x00D4
+#define ixMC_IO_DEBUG_DQB2L_MISC_D0 0x00A4
+#define ixMC_IO_DEBUG_DQB2L_MISC_D1 0x00B4
+#define ixMC_IO_DEBUG_DQB2L_OFSCAL_D0 0x00E4
+#define ixMC_IO_DEBUG_DQB2L_OFSCAL_D1 0x00F4
+#define ixMC_IO_DEBUG_DQB2L_RX_EQ_D0 0x01C4
+#define ixMC_IO_DEBUG_DQB2L_RX_EQ_D1 0x01D4
+#define ixMC_IO_DEBUG_DQB2L_RXPHASE_D0 0x0104
+#define ixMC_IO_DEBUG_DQB2L_RXPHASE_D1 0x0114
+#define ixMC_IO_DEBUG_DQB2L_RX_VREF_CAL_D0 0x0144
+#define ixMC_IO_DEBUG_DQB2L_RX_VREF_CAL_D1 0x0154
+#define ixMC_IO_DEBUG_DQB2L_TXBST_PD_D0 0x0184
+#define ixMC_IO_DEBUG_DQB2L_TXBST_PD_D1 0x0194
+#define ixMC_IO_DEBUG_DQB2L_TXBST_PU_D0 0x01A4
+#define ixMC_IO_DEBUG_DQB2L_TXBST_PU_D1 0x01B4
+#define ixMC_IO_DEBUG_DQB2L_TXPHASE_D0 0x0124
+#define ixMC_IO_DEBUG_DQB2L_TXPHASE_D1 0x0134
+#define ixMC_IO_DEBUG_DQB2L_TXSLF_D0 0x0164
+#define ixMC_IO_DEBUG_DQB2L_TXSLF_D1 0x0174
+#define ixMC_IO_DEBUG_DQB3_CDR_PHSIZE_D0 0x014E
+#define ixMC_IO_DEBUG_DQB3_CDR_PHSIZE_D1 0x015E
+#define ixMC_IO_DEBUG_DQB3H_CLKSEL_D0 0x00C7
+#define ixMC_IO_DEBUG_DQB3H_CLKSEL_D1 0x00D7
+#define ixMC_IO_DEBUG_DQB3H_MISC_D0 0x00A7
+#define ixMC_IO_DEBUG_DQB3H_MISC_D1 0x00B7
+#define ixMC_IO_DEBUG_DQB3H_OFSCAL_D0 0x00E7
+#define ixMC_IO_DEBUG_DQB3H_OFSCAL_D1 0x00F7
+#define ixMC_IO_DEBUG_DQB3H_RX_EQ_D0 0x01C7
+#define ixMC_IO_DEBUG_DQB3H_RX_EQ_D1 0x01D7
+#define ixMC_IO_DEBUG_DQB3H_RXPHASE_D0 0x0107
+#define ixMC_IO_DEBUG_DQB3H_RXPHASE_D1 0x0117
+#define ixMC_IO_DEBUG_DQB3H_RX_VREF_CAL_D0 0x0147
+#define ixMC_IO_DEBUG_DQB3H_RX_VREF_CAL_D1 0x0157
+#define ixMC_IO_DEBUG_DQB3H_TXBST_PD_D0 0x0187
+#define ixMC_IO_DEBUG_DQB3H_TXBST_PD_D1 0x0197
+#define ixMC_IO_DEBUG_DQB3H_TXBST_PU_D0 0x01A7
+#define ixMC_IO_DEBUG_DQB3H_TXBST_PU_D1 0x01B7
+#define ixMC_IO_DEBUG_DQB3H_TXPHASE_D0 0x0127
+#define ixMC_IO_DEBUG_DQB3H_TXPHASE_D1 0x0137
+#define ixMC_IO_DEBUG_DQB3H_TXSLF_D0 0x0167
+#define ixMC_IO_DEBUG_DQB3H_TXSLF_D1 0x0177
+#define ixMC_IO_DEBUG_DQB3L_CLKSEL_D0 0x00C6
+#define ixMC_IO_DEBUG_DQB3L_CLKSEL_D1 0x00D6
+#define ixMC_IO_DEBUG_DQB3L_MISC_D0 0x00A6
+#define ixMC_IO_DEBUG_DQB3L_MISC_D1 0x00B6
+#define ixMC_IO_DEBUG_DQB3L_OFSCAL_D0 0x00E6
+#define ixMC_IO_DEBUG_DQB3L_OFSCAL_D1 0x00F6
+#define ixMC_IO_DEBUG_DQB3L_RX_EQ_D0 0x01C6
+#define ixMC_IO_DEBUG_DQB3L_RX_EQ_D1 0x01D6
+#define ixMC_IO_DEBUG_DQB3L_RXPHASE_D0 0x0106
+#define ixMC_IO_DEBUG_DQB3L_RXPHASE_D1 0x0116
+#define ixMC_IO_DEBUG_DQB3L_RX_VREF_CAL_D0 0x0146
+#define ixMC_IO_DEBUG_DQB3L_RX_VREF_CAL_D1 0x0156
+#define ixMC_IO_DEBUG_DQB3L_TXBST_PD_D0 0x0186
+#define ixMC_IO_DEBUG_DQB3L_TXBST_PD_D1 0x0196
+#define ixMC_IO_DEBUG_DQB3L_TXBST_PU_D0 0x01A6
+#define ixMC_IO_DEBUG_DQB3L_TXBST_PU_D1 0x01B6
+#define ixMC_IO_DEBUG_DQB3L_TXPHASE_D0 0x0126
+#define ixMC_IO_DEBUG_DQB3L_TXPHASE_D1 0x0136
+#define ixMC_IO_DEBUG_DQB3L_TXSLF_D0 0x0166
+#define ixMC_IO_DEBUG_DQB3L_TXSLF_D1 0x0176
+#define ixMC_IO_DEBUG_EDC_CDR_PHSIZE_D0 0x00ED
+#define ixMC_IO_DEBUG_EDC_CDR_PHSIZE_D1 0x00FD
+#define ixMC_IO_DEBUG_EDC_CLKSEL_D0 0x00C9
+#define ixMC_IO_DEBUG_EDC_CLKSEL_D1 0x00D9
+#define ixMC_IO_DEBUG_EDC_MISC_D0 0x00A9
+#define ixMC_IO_DEBUG_EDC_MISC_D1 0x00B9
+#define ixMC_IO_DEBUG_EDC_OFSCAL_D0 0x00E9
+#define ixMC_IO_DEBUG_EDC_OFSCAL_D1 0x00F9
+#define ixMC_IO_DEBUG_EDC_RX_DYN_PM_D0 0x00EC
+#define ixMC_IO_DEBUG_EDC_RX_DYN_PM_D1 0x00FC
+#define ixMC_IO_DEBUG_EDC_RX_EQ_D0 0x01C9
+#define ixMC_IO_DEBUG_EDC_RX_EQ_D1 0x01D9
+#define ixMC_IO_DEBUG_EDC_RX_EQ_PM_D0 0x00EB
+#define ixMC_IO_DEBUG_EDC_RX_EQ_PM_D1 0x00FB
+#define ixMC_IO_DEBUG_EDC_RXPHASE_D0 0x0109
+#define ixMC_IO_DEBUG_EDC_RXPHASE_D1 0x0119
+#define ixMC_IO_DEBUG_EDC_RX_VREF_CAL_D0 0x0149
+#define ixMC_IO_DEBUG_EDC_RX_VREF_CAL_D1 0x0159
+#define ixMC_IO_DEBUG_EDC_TXBST_PD_D0 0x0189
+#define ixMC_IO_DEBUG_EDC_TXBST_PD_D1 0x0199
+#define ixMC_IO_DEBUG_EDC_TXBST_PU_D0 0x01A9
+#define ixMC_IO_DEBUG_EDC_TXBST_PU_D1 0x01B9
+#define ixMC_IO_DEBUG_EDC_TXPHASE_D0 0x0129
+#define ixMC_IO_DEBUG_EDC_TXPHASE_D1 0x0139
+#define ixMC_IO_DEBUG_EDC_TXSLF_D0 0x0169
+#define ixMC_IO_DEBUG_EDC_TXSLF_D1 0x0179
+#define ixMC_IO_DEBUG_UP_0 0x0000
+#define ixMC_IO_DEBUG_UP_100 0x0064
+#define ixMC_IO_DEBUG_UP_10 0x000A
+#define ixMC_IO_DEBUG_UP_101 0x0065
+#define ixMC_IO_DEBUG_UP_102 0x0066
+#define ixMC_IO_DEBUG_UP_103 0x0067
+#define ixMC_IO_DEBUG_UP_104 0x0068
+#define ixMC_IO_DEBUG_UP_105 0x0069
+#define ixMC_IO_DEBUG_UP_106 0x006A
+#define ixMC_IO_DEBUG_UP_107 0x006B
+#define ixMC_IO_DEBUG_UP_108 0x006C
+#define ixMC_IO_DEBUG_UP_109 0x006D
+#define ixMC_IO_DEBUG_UP_1 0x0001
+#define ixMC_IO_DEBUG_UP_110 0x006E
+#define ixMC_IO_DEBUG_UP_11 0x000B
+#define ixMC_IO_DEBUG_UP_111 0x006F
+#define ixMC_IO_DEBUG_UP_112 0x0070
+#define ixMC_IO_DEBUG_UP_113 0x0071
+#define ixMC_IO_DEBUG_UP_114 0x0072
+#define ixMC_IO_DEBUG_UP_115 0x0073
+#define ixMC_IO_DEBUG_UP_116 0x0074
+#define ixMC_IO_DEBUG_UP_117 0x0075
+#define ixMC_IO_DEBUG_UP_118 0x0076
+#define ixMC_IO_DEBUG_UP_119 0x0077
+#define ixMC_IO_DEBUG_UP_120 0x0078
+#define ixMC_IO_DEBUG_UP_12 0x000C
+#define ixMC_IO_DEBUG_UP_121 0x0079
+#define ixMC_IO_DEBUG_UP_122 0x007A
+#define ixMC_IO_DEBUG_UP_123 0x007B
+#define ixMC_IO_DEBUG_UP_124 0x007C
+#define ixMC_IO_DEBUG_UP_125 0x007D
+#define ixMC_IO_DEBUG_UP_126 0x007E
+#define ixMC_IO_DEBUG_UP_127 0x007F
+#define ixMC_IO_DEBUG_UP_128 0x0080
+#define ixMC_IO_DEBUG_UP_129 0x0081
+#define ixMC_IO_DEBUG_UP_130 0x0082
+#define ixMC_IO_DEBUG_UP_13 0x000D
+#define ixMC_IO_DEBUG_UP_131 0x0083
+#define ixMC_IO_DEBUG_UP_132 0x0084
+#define ixMC_IO_DEBUG_UP_133 0x0085
+#define ixMC_IO_DEBUG_UP_134 0x0086
+#define ixMC_IO_DEBUG_UP_135 0x0087
+#define ixMC_IO_DEBUG_UP_136 0x0088
+#define ixMC_IO_DEBUG_UP_137 0x0089
+#define ixMC_IO_DEBUG_UP_138 0x008A
+#define ixMC_IO_DEBUG_UP_139 0x008B
+#define ixMC_IO_DEBUG_UP_140 0x008C
+#define ixMC_IO_DEBUG_UP_14 0x000E
+#define ixMC_IO_DEBUG_UP_141 0x008D
+#define ixMC_IO_DEBUG_UP_142 0x008E
+#define ixMC_IO_DEBUG_UP_143 0x008F
+#define ixMC_IO_DEBUG_UP_144 0x0090
+#define ixMC_IO_DEBUG_UP_145 0x0091
+#define ixMC_IO_DEBUG_UP_146 0x0092
+#define ixMC_IO_DEBUG_UP_147 0x0093
+#define ixMC_IO_DEBUG_UP_148 0x0094
+#define ixMC_IO_DEBUG_UP_149 0x0095
+#define ixMC_IO_DEBUG_UP_150 0x0096
+#define ixMC_IO_DEBUG_UP_15 0x000F
+#define ixMC_IO_DEBUG_UP_151 0x0097
+#define ixMC_IO_DEBUG_UP_152 0x0098
+#define ixMC_IO_DEBUG_UP_153 0x0099
+#define ixMC_IO_DEBUG_UP_154 0x009A
+#define ixMC_IO_DEBUG_UP_155 0x009B
+#define ixMC_IO_DEBUG_UP_156 0x009C
+#define ixMC_IO_DEBUG_UP_157 0x009D
+#define ixMC_IO_DEBUG_UP_158 0x009E
+#define ixMC_IO_DEBUG_UP_159 0x009F
+#define ixMC_IO_DEBUG_UP_16 0x0010
+#define ixMC_IO_DEBUG_UP_17 0x0011
+#define ixMC_IO_DEBUG_UP_18 0x0012
+#define ixMC_IO_DEBUG_UP_19 0x0013
+#define ixMC_IO_DEBUG_UP_20 0x0014
+#define ixMC_IO_DEBUG_UP_2 0x0002
+#define ixMC_IO_DEBUG_UP_21 0x0015
+#define ixMC_IO_DEBUG_UP_22 0x0016
+#define ixMC_IO_DEBUG_UP_23 0x0017
+#define ixMC_IO_DEBUG_UP_24 0x0018
+#define ixMC_IO_DEBUG_UP_25 0x0019
+#define ixMC_IO_DEBUG_UP_26 0x001A
+#define ixMC_IO_DEBUG_UP_27 0x001B
+#define ixMC_IO_DEBUG_UP_28 0x001C
+#define ixMC_IO_DEBUG_UP_29 0x001D
+#define ixMC_IO_DEBUG_UP_30 0x001E
+#define ixMC_IO_DEBUG_UP_3 0x0003
+#define ixMC_IO_DEBUG_UP_31 0x001F
+#define ixMC_IO_DEBUG_UP_32 0x0020
+#define ixMC_IO_DEBUG_UP_33 0x0021
+#define ixMC_IO_DEBUG_UP_34 0x0022
+#define ixMC_IO_DEBUG_UP_35 0x0023
+#define ixMC_IO_DEBUG_UP_36 0x0024
+#define ixMC_IO_DEBUG_UP_37 0x0025
+#define ixMC_IO_DEBUG_UP_38 0x0026
+#define ixMC_IO_DEBUG_UP_39 0x0027
+#define ixMC_IO_DEBUG_UP_40 0x0028
+#define ixMC_IO_DEBUG_UP_4 0x0004
+#define ixMC_IO_DEBUG_UP_41 0x0029
+#define ixMC_IO_DEBUG_UP_42 0x002A
+#define ixMC_IO_DEBUG_UP_43 0x002B
+#define ixMC_IO_DEBUG_UP_44 0x002C
+#define ixMC_IO_DEBUG_UP_45 0x002D
+#define ixMC_IO_DEBUG_UP_46 0x002E
+#define ixMC_IO_DEBUG_UP_47 0x002F
+#define ixMC_IO_DEBUG_UP_48 0x0030
+#define ixMC_IO_DEBUG_UP_49 0x0031
+#define ixMC_IO_DEBUG_UP_50 0x0032
+#define ixMC_IO_DEBUG_UP_5 0x0005
+#define ixMC_IO_DEBUG_UP_51 0x0033
+#define ixMC_IO_DEBUG_UP_52 0x0034
+#define ixMC_IO_DEBUG_UP_53 0x0035
+#define ixMC_IO_DEBUG_UP_54 0x0036
+#define ixMC_IO_DEBUG_UP_55 0x0037
+#define ixMC_IO_DEBUG_UP_56 0x0038
+#define ixMC_IO_DEBUG_UP_57 0x0039
+#define ixMC_IO_DEBUG_UP_58 0x003A
+#define ixMC_IO_DEBUG_UP_59 0x003B
+#define ixMC_IO_DEBUG_UP_60 0x003C
+#define ixMC_IO_DEBUG_UP_6 0x0006
+#define ixMC_IO_DEBUG_UP_61 0x003D
+#define ixMC_IO_DEBUG_UP_62 0x003E
+#define ixMC_IO_DEBUG_UP_63 0x003F
+#define ixMC_IO_DEBUG_UP_64 0x0040
+#define ixMC_IO_DEBUG_UP_65 0x0041
+#define ixMC_IO_DEBUG_UP_66 0x0042
+#define ixMC_IO_DEBUG_UP_67 0x0043
+#define ixMC_IO_DEBUG_UP_68 0x0044
+#define ixMC_IO_DEBUG_UP_69 0x0045
+#define ixMC_IO_DEBUG_UP_70 0x0046
+#define ixMC_IO_DEBUG_UP_7 0x0007
+#define ixMC_IO_DEBUG_UP_71 0x0047
+#define ixMC_IO_DEBUG_UP_72 0x0048
+#define ixMC_IO_DEBUG_UP_73 0x0049
+#define ixMC_IO_DEBUG_UP_74 0x004A
+#define ixMC_IO_DEBUG_UP_75 0x004B
+#define ixMC_IO_DEBUG_UP_76 0x004C
+#define ixMC_IO_DEBUG_UP_77 0x004D
+#define ixMC_IO_DEBUG_UP_78 0x004E
+#define ixMC_IO_DEBUG_UP_79 0x004F
+#define ixMC_IO_DEBUG_UP_80 0x0050
+#define ixMC_IO_DEBUG_UP_8 0x0008
+#define ixMC_IO_DEBUG_UP_81 0x0051
+#define ixMC_IO_DEBUG_UP_82 0x0052
+#define ixMC_IO_DEBUG_UP_83 0x0053
+#define ixMC_IO_DEBUG_UP_84 0x0054
+#define ixMC_IO_DEBUG_UP_85 0x0055
+#define ixMC_IO_DEBUG_UP_86 0x0056
+#define ixMC_IO_DEBUG_UP_87 0x0057
+#define ixMC_IO_DEBUG_UP_88 0x0058
+#define ixMC_IO_DEBUG_UP_89 0x0059
+#define ixMC_IO_DEBUG_UP_90 0x005A
+#define ixMC_IO_DEBUG_UP_9 0x0009
+#define ixMC_IO_DEBUG_UP_91 0x005B
+#define ixMC_IO_DEBUG_UP_92 0x005C
+#define ixMC_IO_DEBUG_UP_93 0x005D
+#define ixMC_IO_DEBUG_UP_94 0x005E
+#define ixMC_IO_DEBUG_UP_95 0x005F
+#define ixMC_IO_DEBUG_UP_96 0x0060
+#define ixMC_IO_DEBUG_UP_97 0x0061
+#define ixMC_IO_DEBUG_UP_98 0x0062
+#define ixMC_IO_DEBUG_UP_99 0x0063
+#define ixMC_IO_DEBUG_WCDR_CDR_PHSIZE_D0 0x01EA
+#define ixMC_IO_DEBUG_WCDR_CDR_PHSIZE_D1 0x01FA
+#define ixMC_IO_DEBUG_WCDR_CLKSEL_D0 0x01E1
+#define ixMC_IO_DEBUG_WCDR_CLKSEL_D1 0x01F1
+#define ixMC_IO_DEBUG_WCDR_MISC_D0 0x01E0
+#define ixMC_IO_DEBUG_WCDR_MISC_D1 0x01F0
+#define ixMC_IO_DEBUG_WCDR_OFSCAL_D0 0x01E2
+#define ixMC_IO_DEBUG_WCDR_OFSCAL_D1 0x01F2
+#define ixMC_IO_DEBUG_WCDR_RX_DYN_PM_D0 0x01EC
+#define ixMC_IO_DEBUG_WCDR_RX_DYN_PM_D1 0x01FC
+#define ixMC_IO_DEBUG_WCDR_RX_EQ_D0 0x01E9
+#define ixMC_IO_DEBUG_WCDR_RX_EQ_D1 0x01F9
+#define ixMC_IO_DEBUG_WCDR_RX_EQ_PM_D0 0x01EB
+#define ixMC_IO_DEBUG_WCDR_RX_EQ_PM_D1 0x01FB
+#define ixMC_IO_DEBUG_WCDR_RXPHASE_D0 0x01E3
+#define ixMC_IO_DEBUG_WCDR_RXPHASE_D1 0x01F3
+#define ixMC_IO_DEBUG_WCDR_RX_VREF_CAL_D0 0x01E5
+#define ixMC_IO_DEBUG_WCDR_RX_VREF_CAL_D1 0x01F5
+#define ixMC_IO_DEBUG_WCDR_TXBST_PD_D0 0x01E7
+#define ixMC_IO_DEBUG_WCDR_TXBST_PD_D1 0x01F7
+#define ixMC_IO_DEBUG_WCDR_TXBST_PU_D0 0x01E8
+#define ixMC_IO_DEBUG_WCDR_TXBST_PU_D1 0x01F8
+#define ixMC_IO_DEBUG_WCDR_TXPHASE_D0 0x01E4
+#define ixMC_IO_DEBUG_WCDR_TXPHASE_D1 0x01F4
+#define ixMC_IO_DEBUG_WCDR_TXSLF_D0 0x01E6
+#define ixMC_IO_DEBUG_WCDR_TXSLF_D1 0x01F6
+#define ixMC_IO_DEBUG_WCK_CLKSEL_D0 0x00CA
+#define ixMC_IO_DEBUG_WCK_CLKSEL_D1 0x00DA
+#define ixMC_IO_DEBUG_WCK_MISC_D0 0x00AA
+#define ixMC_IO_DEBUG_WCK_MISC_D1 0x00BA
+#define ixMC_IO_DEBUG_WCK_OFSCAL_D0 0x00EA
+#define ixMC_IO_DEBUG_WCK_OFSCAL_D1 0x00FA
+#define ixMC_IO_DEBUG_WCK_RX_EQ_D0 0x01CA
+#define ixMC_IO_DEBUG_WCK_RX_EQ_D1 0x01DA
+#define ixMC_IO_DEBUG_WCK_RXPHASE_D0 0x010A
+#define ixMC_IO_DEBUG_WCK_RXPHASE_D1 0x011A
+#define ixMC_IO_DEBUG_WCK_RX_VREF_CAL_D0 0x014A
+#define ixMC_IO_DEBUG_WCK_RX_VREF_CAL_D1 0x015A
+#define ixMC_IO_DEBUG_WCK_TXBST_PD_D0 0x018A
+#define ixMC_IO_DEBUG_WCK_TXBST_PD_D1 0x019A
+#define ixMC_IO_DEBUG_WCK_TXBST_PU_D0 0x01AA
+#define ixMC_IO_DEBUG_WCK_TXBST_PU_D1 0x01BA
+#define ixMC_IO_DEBUG_WCK_TXPHASE_D0 0x012A
+#define ixMC_IO_DEBUG_WCK_TXPHASE_D1 0x013A
+#define ixMC_IO_DEBUG_WCK_TXSLF_D0 0x016A
+#define ixMC_IO_DEBUG_WCK_TXSLF_D1 0x017A
+#define ixMC_TSM_DEBUG_BCNT0 0x0003
+#define ixMC_TSM_DEBUG_BCNT10 0x000D
+#define ixMC_TSM_DEBUG_BCNT1 0x0004
+#define ixMC_TSM_DEBUG_BCNT2 0x0005
+#define ixMC_TSM_DEBUG_BCNT3 0x0006
+#define ixMC_TSM_DEBUG_BCNT4 0x0007
+#define ixMC_TSM_DEBUG_BCNT5 0x0008
+#define ixMC_TSM_DEBUG_BCNT6 0x0009
+#define ixMC_TSM_DEBUG_BCNT7 0x000A
+#define ixMC_TSM_DEBUG_BCNT8 0x000B
+#define ixMC_TSM_DEBUG_BCNT9 0x000C
+#define ixMC_TSM_DEBUG_BKPT 0x0013
+#define ixMC_TSM_DEBUG_FLAG 0x0001
+#define ixMC_TSM_DEBUG_GCNT 0x0000
+#define ixMC_TSM_DEBUG_MISC 0x0002
+#define ixMC_TSM_DEBUG_ST01 0x0010
+#define ixMC_TSM_DEBUG_ST23 0x0011
+#define ixMC_TSM_DEBUG_ST45 0x0012
+#define mmATC_ATS_CNTL 0x0CC9
+#define mmATC_ATS_DEBUG 0x0CCA
+#define mmATC_ATS_DEFAULT_PAGE_CNTL 0x0CD1
+#define mmATC_ATS_DEFAULT_PAGE_LOW 0x0CD0
+#define mmATC_ATS_FAULT_CNTL 0x0CCD
+#define mmATC_ATS_FAULT_DEBUG 0x0CCB
+#define mmATC_ATS_FAULT_STATUS_ADDR 0x0CCF
+#define mmATC_ATS_FAULT_STATUS_INFO 0x0CCE
+#define mmATC_ATS_STATUS 0x0CCC
+#define mmATC_L1_ADDRESS_OFFSET 0x0CDD
+#define mmATC_L1_CNTL 0x0CDC
+#define mmATC_L1RD_DEBUG_TLB 0x0CDE
+#define mmATC_L1RD_STATUS 0x0CE0
+#define mmATC_L1WR_DEBUG_TLB 0x0CDF
+#define mmATC_L1WR_STATUS 0x0CE1
+#define mmATC_L2_CNTL 0x0CD5
+#define mmATC_L2_DEBUG 0x0CD7
+#define mmATC_MISC_CG 0x0CD4
+#define mmATC_VM_APERTURE0_CNTL 0x0CC4
+#define mmATC_VM_APERTURE0_CNTL2 0x0CC6
+#define mmATC_VM_APERTURE0_HIGH_ADDR 0x0CC2
+#define mmATC_VM_APERTURE0_LOW_ADDR 0x0CC0
+#define mmATC_VM_APERTURE1_CNTL 0x0CC5
+#define mmATC_VM_APERTURE1_CNTL2 0x0CC7
+#define mmATC_VM_APERTURE1_HIGH_ADDR 0x0CC3
+#define mmATC_VM_APERTURE1_LOW_ADDR 0x0CC1
+#define mmATC_VMID0_PASID_MAPPING 0x0CE7
+#define mmATC_VMID10_PASID_MAPPING 0x0CF1
+#define mmATC_VMID11_PASID_MAPPING 0x0CF2
+#define mmATC_VMID12_PASID_MAPPING 0x0CF3
+#define mmATC_VMID13_PASID_MAPPING 0x0CF4
+#define mmATC_VMID14_PASID_MAPPING 0x0CF5
+#define mmATC_VMID15_PASID_MAPPING 0x0CF6
+#define mmATC_VMID1_PASID_MAPPING 0x0CE8
+#define mmATC_VMID2_PASID_MAPPING 0x0CE9
+#define mmATC_VMID3_PASID_MAPPING 0x0CEA
+#define mmATC_VMID4_PASID_MAPPING 0x0CEB
+#define mmATC_VMID5_PASID_MAPPING 0x0CEC
+#define mmATC_VMID6_PASID_MAPPING 0x0CED
+#define mmATC_VMID7_PASID_MAPPING 0x0CEE
+#define mmATC_VMID8_PASID_MAPPING 0x0CEF
+#define mmATC_VMID9_PASID_MAPPING 0x0CF0
+#define mmATC_VMID_PASID_MAPPING_UPDATE_STATUS 0x0CE6
+#define mmCC_MC_MAX_CHANNEL 0x096E
+#define mmDLL_CNTL 0x0AE9
+#define mmGMCON_DEBUG 0x0D5F
+#define mmGMCON_MISC 0x0D43
+#define mmGMCON_MISC2 0x0D44
+#define mmGMCON_MISC3 0x0D51
+#define mmGMCON_PERF_MON_CNTL0 0x0D4A
+#define mmGMCON_PERF_MON_CNTL1 0x0D4B
+#define mmGMCON_PERF_MON_RSLT0 0x0D4C
+#define mmGMCON_PERF_MON_RSLT1 0x0D4D
+#define mmGMCON_PGFSM_CONFIG 0x0D4E
+#define mmGMCON_PGFSM_READ 0x0D50
+#define mmGMCON_PGFSM_WRITE 0x0D4F
+#define mmGMCON_RENG_EXECUTE 0x0D42
+#define mmGMCON_RENG_RAM_DATA 0x0D41
+#define mmGMCON_RENG_RAM_INDEX 0x0D40
+#define mmGMCON_STCTRL_REGISTER_SAVE_EXCL_SET0 0x0D48
+#define mmGMCON_STCTRL_REGISTER_SAVE_EXCL_SET1 0x0D49
+#define mmGMCON_STCTRL_REGISTER_SAVE_RANGE0 0x0D45
+#define mmGMCON_STCTRL_REGISTER_SAVE_RANGE1 0x0D46
+#define mmGMCON_STCTRL_REGISTER_SAVE_RANGE2 0x0D47
+#define mmMC_ARB_ADDR_HASH 0x09DC
+#define mmMC_ARB_AGE_RD 0x09E9
+#define mmMC_ARB_AGE_WR 0x09EA
+#define mmMC_ARB_BANKMAP 0x09D7
+#define mmMC_ARB_BURST_TIME 0x0A02
+#define mmMC_ARB_CAC_CNTL 0x09D4
+#define mmMC_ARB_CG 0x09FA
+#define mmMC_ARB_DRAM_TIMING 0x09DD
+#define mmMC_ARB_DRAM_TIMING_1 0x09FC
+#define mmMC_ARB_DRAM_TIMING2 0x09DE
+#define mmMC_ARB_DRAM_TIMING2_1 0x09FF
+#define mmMC_ARB_FED_CNTL 0x09C1
+#define mmMC_ARB_GDEC_RD_CNTL 0x09EE
+#define mmMC_ARB_GDEC_WR_CNTL 0x09EF
+#define mmMC_ARB_GECC2 0x09C9
+#define mmMC_ARB_GECC2_CLI 0x09CA
+#define mmMC_ARB_GECC2_DEBUG 0x09C4
+#define mmMC_ARB_GECC2_DEBUG2 0x09C5
+#define mmMC_ARB_GECC2_MISC 0x09C3
+#define mmMC_ARB_GECC2_STATUS 0x09C2
+#define mmMC_ARB_LAZY0_RD 0x09E5
+#define mmMC_ARB_LAZY0_WR 0x09E6
+#define mmMC_ARB_LAZY1_RD 0x09E7
+#define mmMC_ARB_LAZY1_WR 0x09E8
+#define mmMC_ARB_LM_RD 0x09F0
+#define mmMC_ARB_LM_WR 0x09F1
+#define mmMC_ARB_MINCLKS 0x09DA
+#define mmMC_ARB_MISC 0x09D6
+#define mmMC_ARB_MISC2 0x09D5
+#define mmMC_ARB_PM_CNTL 0x09ED
+#define mmMC_ARB_POP 0x09D9
+#define mmMC_ARB_RAMCFG 0x09D8
+#define mmMC_ARB_REMREQ 0x09F2
+#define mmMC_ARB_REPLAY 0x09F3
+#define mmMC_ARB_RET_CREDITS_RD 0x09F4
+#define mmMC_ARB_RET_CREDITS_WR 0x09F5
+#define mmMC_ARB_RFSH_CNTL 0x09EB
+#define mmMC_ARB_RFSH_RATE 0x09EC
+#define mmMC_ARB_RTT_CNTL0 0x09D0
+#define mmMC_ARB_RTT_CNTL1 0x09D1
+#define mmMC_ARB_RTT_CNTL2 0x09D2
+#define mmMC_ARB_RTT_DATA 0x09CF
+#define mmMC_ARB_RTT_DEBUG 0x09D3
+#define mmMC_ARB_SQM_CNTL 0x09DB
+#define mmMC_ARB_TM_CNTL_RD 0x09E3
+#define mmMC_ARB_TM_CNTL_WR 0x09E4
+#define mmMC_ARB_WCDR 0x09FB
+#define mmMC_ARB_WCDR_2 0x09CE
+#define mmMC_ARB_WTM_CNTL_RD 0x09DF
+#define mmMC_ARB_WTM_CNTL_WR 0x09E0
+#define mmMC_ARB_WTM_GRPWT_RD 0x09E1
+#define mmMC_ARB_WTM_GRPWT_WR 0x09E2
+#define mmMC_BIST_AUTO_CNTL 0x0A06
+#define mmMC_BIST_CMD_CNTL 0x0A8E
+#define mmMC_BIST_CMP_CNTL 0x0A8D
+#define mmMC_BIST_CMP_CNTL_2 0x0AB6
+#define mmMC_BIST_CNTL 0x0A05
+#define mmMC_BIST_DATA_MASK 0x0A12
+#define mmMC_BIST_DATA_WORD0 0x0A0A
+#define mmMC_BIST_DATA_WORD1 0x0A0B
+#define mmMC_BIST_DATA_WORD2 0x0A0C
+#define mmMC_BIST_DATA_WORD3 0x0A0D
+#define mmMC_BIST_DATA_WORD4 0x0A0E
+#define mmMC_BIST_DATA_WORD5 0x0A0F
+#define mmMC_BIST_DATA_WORD6 0x0A10
+#define mmMC_BIST_DATA_WORD7 0x0A11
+#define mmMC_BIST_DIR_CNTL 0x0A07
+#define mmMC_BIST_EADDR 0x0A09
+#define mmMC_BIST_MISMATCH_ADDR 0x0A13
+#define mmMC_BIST_RDATA_EDC 0x0A1D
+#define mmMC_BIST_RDATA_MASK 0x0A1C
+#define mmMC_BIST_RDATA_WORD0 0x0A14
+#define mmMC_BIST_RDATA_WORD1 0x0A15
+#define mmMC_BIST_RDATA_WORD2 0x0A16
+#define mmMC_BIST_RDATA_WORD3 0x0A17
+#define mmMC_BIST_RDATA_WORD4 0x0A18
+#define mmMC_BIST_RDATA_WORD5 0x0A19
+#define mmMC_BIST_RDATA_WORD6 0x0A1A
+#define mmMC_BIST_RDATA_WORD7 0x0A1B
+#define mmMC_BIST_SADDR 0x0A08
+#define mmMC_CG_CONFIG 0x096F
+#define mmMC_CG_CONFIG_MCD 0x0829
+#define mmMC_CG_DATAPORT 0x0A21
+#define mmMC_CITF_CNTL 0x0970
+#define mmMC_CITF_CREDITS_ARB_RD 0x0972
+#define mmMC_CITF_CREDITS_ARB_WR 0x0973
+#define mmMC_CITF_CREDITS_VM 0x0971
+#define mmMC_CITF_CREDITS_XBAR 0x0989
+#define mmMC_CITF_DAGB_CNTL 0x0974
+#define mmMC_CITF_DAGB_DLY 0x0977
+#define mmMC_CITF_INT_CREDITS 0x0975
+#define mmMC_CITF_INT_CREDITS_WR 0x097D
+#define mmMC_CITF_MISC_RD_CG 0x0992
+#define mmMC_CITF_MISC_VM_CG 0x0994
+#define mmMC_CITF_MISC_WR_CG 0x0993
+#define mmMC_CITF_PERF_MON_CNTL2 0x098E
+#define mmMC_CITF_PERF_MON_RSLT2 0x0991
+#define mmMC_CITF_REMREQ 0x097A
+#define mmMC_CITF_RET_MODE 0x0976
+#define mmMC_CITF_WTM_RD_CNTL 0x097F
+#define mmMC_CITF_WTM_WR_CNTL 0x0980
+#define mmMC_CITF_XTRA_ENABLE 0x096D
+#define mmMC_CONFIG 0x0800
+#define mmMC_CONFIG_MCD 0x0828
+#define mmMC_HUB_MISC_DBG 0x0831
+#define mmMC_HUB_MISC_FRAMING 0x0834
+#define mmMC_HUB_MISC_HUB_CG 0x082E
+#define mmMC_HUB_MISC_IDLE_STATUS 0x0847
+#define mmMC_HUB_MISC_OVERRIDE 0x0833
+#define mmMC_HUB_MISC_POWER 0x082D
+#define mmMC_HUB_MISC_SIP_CG 0x0830
+#define mmMC_HUB_MISC_STATUS 0x0832
+#define mmMC_HUB_MISC_VM_CG 0x082F
+#define mmMC_HUB_RDREQ_CNTL 0x083B
+#define mmMC_HUB_RDREQ_CREDITS 0x0844
+#define mmMC_HUB_RDREQ_CREDITS2 0x0845
+#define mmMC_HUB_RDREQ_DMIF 0x0863
+#define mmMC_HUB_RDREQ_DMIF_LIMIT 0x0848
+#define mmMC_HUB_RDREQ_GBL0 0x0856
+#define mmMC_HUB_RDREQ_GBL1 0x0857
+#define mmMC_HUB_RDREQ_HDP 0x085B
+#define mmMC_HUB_RDREQ_MCDW 0x0851
+#define mmMC_HUB_RDREQ_MCDX 0x0852
+#define mmMC_HUB_RDREQ_MCDY 0x0853
+#define mmMC_HUB_RDREQ_MCDZ 0x0854
+#define mmMC_HUB_RDREQ_MCIF 0x0864
+#define mmMC_HUB_RDREQ_RLC 0x085D
+#define mmMC_HUB_RDREQ_SEM 0x085E
+#define mmMC_HUB_RDREQ_SIP 0x0855
+#define mmMC_HUB_RDREQ_SMU 0x0858
+#define mmMC_HUB_RDREQ_STATUS 0x0839
+#define mmMC_HUB_RDREQ_UMC 0x0860
+#define mmMC_HUB_RDREQ_UVD 0x0861
+#define mmMC_HUB_RDREQ_VCE 0x085F
+#define mmMC_HUB_RDREQ_VCEU 0x0866
+#define mmMC_HUB_RDREQ_VMC 0x0865
+#define mmMC_HUB_RDREQ_WTM_CNTL 0x083D
+#define mmMC_HUB_RDREQ_XDMAM 0x0882
+#define mmMC_HUB_SHARED_DAGB_DLY 0x0846
+#define mmMC_HUB_WDP_BP 0x0837
+#define mmMC_HUB_WDP_CNTL 0x0835
+#define mmMC_HUB_WDP_CREDITS 0x083F
+#define mmMC_HUB_WDP_ERR 0x0836
+#define mmMC_HUB_WDP_GBL0 0x0841
+#define mmMC_HUB_WDP_GBL1 0x0842
+#define mmMC_HUB_WDP_HDP 0x0879
+#define mmMC_HUB_WDP_IH 0x0872
+#define mmMC_HUB_WDP_MCDW 0x0867
+#define mmMC_HUB_WDP_MCDX 0x0868
+#define mmMC_HUB_WDP_MCDY 0x0869
+#define mmMC_HUB_WDP_MCDZ 0x086A
+#define mmMC_HUB_WDP_MCIF 0x086F
+#define mmMC_HUB_WDP_MGPU 0x0843
+#define mmMC_HUB_WDP_MGPU2 0x0840
+#define mmMC_HUB_WDP_RLC 0x0873
+#define mmMC_HUB_WDP_SEM 0x0874
+#define mmMC_HUB_WDP_SH0 0x086E
+#define mmMC_HUB_WDP_SH1 0x0876
+#define mmMC_HUB_WDP_SIP 0x086B
+#define mmMC_HUB_WDP_SMU 0x0875
+#define mmMC_HUB_WDP_STATUS 0x0838
+#define mmMC_HUB_WDP_UMC 0x0877
+#define mmMC_HUB_WDP_UVD 0x0878
+#define mmMC_HUB_WDP_VCE 0x0870
+#define mmMC_HUB_WDP_VCEU 0x087F
+#define mmMC_HUB_WDP_WTM_CNTL 0x083E
+#define mmMC_HUB_WDP_XDMA 0x0881
+#define mmMC_HUB_WDP_XDMAM 0x0880
+#define mmMC_HUB_WDP_XDP 0x0871
+#define mmMC_HUB_WRRET_CNTL 0x083C
+#define mmMC_HUB_WRRET_MCDW 0x087B
+#define mmMC_HUB_WRRET_MCDX 0x087C
+#define mmMC_HUB_WRRET_MCDY 0x087D
+#define mmMC_HUB_WRRET_MCDZ 0x087E
+#define mmMC_HUB_WRRET_STATUS 0x083A
+#define mmMC_IMP_CNTL 0x0A36
+#define mmMC_IMP_DEBUG 0x0A37
+#define mmMC_IMP_DQ_STATUS 0x0ABC
+#define mmMC_IMP_STATUS 0x0A38
+#define mmMC_IO_APHY_STR_CNTL_D0 0x0A97
+#define mmMC_IO_APHY_STR_CNTL_D1 0x0A98
+#define mmMC_IO_CDRCNTL1_D0 0x0ADD
+#define mmMC_IO_CDRCNTL1_D1 0x0ADE
+#define mmMC_IO_CDRCNTL2_D0 0x0AE4
+#define mmMC_IO_CDRCNTL2_D1 0x0AE5
+#define mmMC_IO_CDRCNTL_D0 0x0A55
+#define mmMC_IO_CDRCNTL_D1 0x0A56
+#define mmMC_IO_DPHY_STR_CNTL_D0 0x0A4E
+#define mmMC_IO_DPHY_STR_CNTL_D1 0x0A54
+#define mmMC_IO_PAD_CNTL 0x0A73
+#define mmMC_IO_PAD_CNTL_D0 0x0A74
+#define mmMC_IO_PAD_CNTL_D1 0x0A75
+#define mmMC_IO_RXCNTL1_DPHY0_D0 0x0ADF
+#define mmMC_IO_RXCNTL1_DPHY0_D1 0x0AE1
+#define mmMC_IO_RXCNTL1_DPHY1_D0 0x0AE0
+#define mmMC_IO_RXCNTL1_DPHY1_D1 0x0AE2
+#define mmMC_IO_RXCNTL_DPHY0_D0 0x0A4C
+#define mmMC_IO_RXCNTL_DPHY0_D1 0x0A52
+#define mmMC_IO_RXCNTL_DPHY1_D0 0x0A4D
+#define mmMC_IO_RXCNTL_DPHY1_D1 0x0A53
+#define mmMC_IO_TXCNTL_APHY_D0 0x0A4B
+#define mmMC_IO_TXCNTL_APHY_D1 0x0A51
+#define mmMC_IO_TXCNTL_DPHY0_D0 0x0A49
+#define mmMC_IO_TXCNTL_DPHY0_D1 0x0A4F
+#define mmMC_IO_TXCNTL_DPHY1_D0 0x0A4A
+#define mmMC_IO_TXCNTL_DPHY1_D1 0x0A50
+#define mmMCLK_PWRMGT_CNTL 0x0AE8
+#define mmMC_MEM_POWER_LS 0x082A
+#define mmMC_NPL_STATUS 0x0A76
+#define mmMC_PHY_TIMING_2 0x0ACE
+#define mmMC_PHY_TIMING_D0 0x0ACC
+#define mmMC_PHY_TIMING_D1 0x0ACD
+#define mmMC_PMG_AUTO_CFG 0x0A35
+#define mmMC_PMG_AUTO_CMD 0x0A34
+#define mmMC_PMG_CFG 0x0A84
+#define mmMC_PMG_CMD_EMRS 0x0A83
+#define mmMC_PMG_CMD_MRS 0x0AAB
+#define mmMC_PMG_CMD_MRS1 0x0AD1
+#define mmMC_PMG_CMD_MRS2 0x0AD7
+#define mmMC_RD_CB 0x0981
+#define mmMC_RD_DB 0x0982
+#define mmMC_RD_GRP_EXT 0x0978
+#define mmMC_RD_GRP_GFX 0x0803
+#define mmMC_RD_GRP_LCL 0x098A
+#define mmMC_RD_GRP_OTH 0x0807
+#define mmMC_RD_GRP_SYS 0x0805
+#define mmMC_RD_HUB 0x0985
+#define mmMC_RD_TC0 0x0983
+#define mmMC_RD_TC1 0x0984
+#define mmMC_RPB_ARB_CNTL 0x0951
+#define mmMC_RPB_BIF_CNTL 0x0952
+#define mmMC_RPB_CID_QUEUE_EX 0x095A
+#define mmMC_RPB_CID_QUEUE_EX_DATA 0x095B
+#define mmMC_RPB_CID_QUEUE_RD 0x0957
+#define mmMC_RPB_CID_QUEUE_WR 0x0956
+#define mmMC_RPB_CONF 0x094D
+#define mmMC_RPB_DBG1 0x094F
+#define mmMC_RPB_EFF_CNTL 0x0950
+#define mmMC_RPB_IF_CONF 0x094E
+#define mmMC_RPB_PERF_COUNTER_CNTL 0x0958
+#define mmMC_RPB_PERF_COUNTER_STATUS 0x0959
+#define mmMC_RPB_RD_SWITCH_CNTL 0x0955
+#define mmMC_RPB_WR_COMBINE_CNTL 0x0954
+#define mmMC_RPB_WR_SWITCH_CNTL 0x0953
+#define mmMC_SEQ_BIT_REMAP_B0_D0 0x0AA3
+#define mmMC_SEQ_BIT_REMAP_B0_D1 0x0AA7
+#define mmMC_SEQ_BIT_REMAP_B1_D0 0x0AA4
+#define mmMC_SEQ_BIT_REMAP_B1_D1 0x0AA8
+#define mmMC_SEQ_BIT_REMAP_B2_D0 0x0AA5
+#define mmMC_SEQ_BIT_REMAP_B2_D1 0x0AA9
+#define mmMC_SEQ_BIT_REMAP_B3_D0 0x0AA6
+#define mmMC_SEQ_BIT_REMAP_B3_D1 0x0AAA
+#define mmMC_SEQ_BYTE_REMAP_D0 0x0A93
+#define mmMC_SEQ_BYTE_REMAP_D1 0x0A94
+#define mmMC_SEQ_CAS_TIMING 0x0A29
+#define mmMC_SEQ_CAS_TIMING_LP 0x0A9C
+#define mmMC_SEQ_CG 0x0A9A
+#define mmMC_SEQ_CMD 0x0A31
+#define mmMC_SEQ_CNTL 0x0A25
+#define mmMC_SEQ_CNTL_2 0x0AD4
+#define mmMC_SEQ_DRAM 0x0A26
+#define mmMC_SEQ_DRAM_2 0x0A27
+#define mmMC_SEQ_DRAM_ERROR_INSERTION 0x0ACB
+#define mmMC_SEQ_FIFO_CTL 0x0A57
+#define mmMC_SEQ_IO_DEBUG_DATA 0x0A92
+#define mmMC_SEQ_IO_DEBUG_INDEX 0x0A91
+#define mmMC_SEQ_IO_RDBI 0x0AB4
+#define mmMC_SEQ_IO_REDC 0x0AB5
+#define mmMC_SEQ_IO_RESERVE_D0 0x0AB7
+#define mmMC_SEQ_IO_RESERVE_D1 0x0AB8
+#define mmMC_SEQ_IO_RWORD0 0x0AAC
+#define mmMC_SEQ_IO_RWORD1 0x0AAD
+#define mmMC_SEQ_IO_RWORD2 0x0AAE
+#define mmMC_SEQ_IO_RWORD3 0x0AAF
+#define mmMC_SEQ_IO_RWORD4 0x0AB0
+#define mmMC_SEQ_IO_RWORD5 0x0AB1
+#define mmMC_SEQ_IO_RWORD6 0x0AB2
+#define mmMC_SEQ_IO_RWORD7 0x0AB3
+#define mmMC_SEQ_MISC0 0x0A80
+#define mmMC_SEQ_MISC1 0x0A81
+#define mmMC_SEQ_MISC3 0x0A8B
+#define mmMC_SEQ_MISC4 0x0A8C
+#define mmMC_SEQ_MISC5 0x0A95
+#define mmMC_SEQ_MISC6 0x0A96
+#define mmMC_SEQ_MISC7 0x0A99
+#define mmMC_SEQ_MISC8 0x0A5F
+#define mmMC_SEQ_MISC9 0x0AE7
+#define mmMC_SEQ_MISC_TIMING 0x0A2A
+#define mmMC_SEQ_MISC_TIMING2 0x0A2B
+#define mmMC_SEQ_MISC_TIMING2_LP 0x0A9E
+#define mmMC_SEQ_MISC_TIMING_LP 0x0A9D
+#define mmMC_SEQ_MPLL_OVERRIDE 0x0A22
+#define mmMC_SEQ_PERF_CNTL 0x0A77
+#define mmMC_SEQ_PERF_CNTL_1 0x0AFD
+#define mmMC_SEQ_PERF_SEQ_CNT_A_I0 0x0A79
+#define mmMC_SEQ_PERF_SEQ_CNT_A_I1 0x0A7A
+#define mmMC_SEQ_PERF_SEQ_CNT_B_I0 0x0A7B
+#define mmMC_SEQ_PERF_SEQ_CNT_B_I1 0x0A7C
+#define mmMC_SEQ_PERF_SEQ_CNT_C_I0 0x0AD9
+#define mmMC_SEQ_PERF_SEQ_CNT_C_I1 0x0ADA
+#define mmMC_SEQ_PERF_SEQ_CNT_D_I0 0x0ADB
+#define mmMC_SEQ_PERF_SEQ_CNT_D_I1 0x0ADC
+#define mmMC_SEQ_PERF_SEQ_CTL 0x0A78
+#define mmMC_SEQ_PMG_CMD_EMRS_LP 0x0AA1
+#define mmMC_SEQ_PMG_CMD_MRS1_LP 0x0AD2
+#define mmMC_SEQ_PMG_CMD_MRS2_LP 0x0AD8
+#define mmMC_SEQ_PMG_CMD_MRS_LP 0x0AA2
+#define mmMC_SEQ_PMG_PG_HWCNTL 0x0AB9
+#define mmMC_SEQ_PMG_PG_SWCNTL_0 0x0ABA
+#define mmMC_SEQ_PMG_PG_SWCNTL_1 0x0ABB
+#define mmMC_SEQ_PMG_TIMING 0x0A2C
+#define mmMC_SEQ_PMG_TIMING_LP 0x0AD3
+#define mmMC_SEQ_RAS_TIMING 0x0A28
+#define mmMC_SEQ_RAS_TIMING_LP 0x0A9B
+#define mmMC_SEQ_RD_CTL_D0 0x0A2D
+#define mmMC_SEQ_RD_CTL_D0_LP 0x0AC7
+#define mmMC_SEQ_RD_CTL_D1 0x0A2E
+#define mmMC_SEQ_RD_CTL_D1_LP 0x0AC8
+#define mmMC_SEQ_RESERVE_0_S 0x0A1E
+#define mmMC_SEQ_RESERVE_1_S 0x0A1F
+#define mmMC_SEQ_RESERVE_M 0x0A82
+#define mmMC_SEQ_RXFRAMING_BYTE0_D0 0x0A67
+#define mmMC_SEQ_RXFRAMING_BYTE0_D1 0x0A6D
+#define mmMC_SEQ_RXFRAMING_BYTE1_D0 0x0A68
+#define mmMC_SEQ_RXFRAMING_BYTE1_D1 0x0A6E
+#define mmMC_SEQ_RXFRAMING_BYTE2_D0 0x0A69
+#define mmMC_SEQ_RXFRAMING_BYTE2_D1 0x0A6F
+#define mmMC_SEQ_RXFRAMING_BYTE3_D0 0x0A6A
+#define mmMC_SEQ_RXFRAMING_BYTE3_D1 0x0A70
+#define mmMC_SEQ_RXFRAMING_DBI_D0 0x0A6B
+#define mmMC_SEQ_RXFRAMING_DBI_D1 0x0A71
+#define mmMC_SEQ_RXFRAMING_EDC_D0 0x0A6C
+#define mmMC_SEQ_RXFRAMING_EDC_D1 0x0A72
+#define mmMC_SEQ_STATUS_M 0x0A7D
+#define mmMC_SEQ_STATUS_S 0x0A20
+#define mmMC_SEQ_SUP_CNTL 0x0A32
+#define mmMC_SEQ_SUP_DEC_STAT 0x0A88
+#define mmMC_SEQ_SUP_GP0_STAT 0x0A8F
+#define mmMC_SEQ_SUP_GP1_STAT 0x0A90
+#define mmMC_SEQ_SUP_GP2_STAT 0x0A85
+#define mmMC_SEQ_SUP_GP3_STAT 0x0A86
+#define mmMC_SEQ_SUP_IR_STAT 0x0A87
+#define mmMC_SEQ_SUP_PGM 0x0A33
+#define mmMC_SEQ_SUP_PGM_STAT 0x0A89
+#define mmMC_SEQ_SUP_R_PGM 0x0A8A
+#define mmMC_SEQ_TCG_CNTL 0x0ABD
+#define mmMC_SEQ_TIMER_RD 0x0ACA
+#define mmMC_SEQ_TIMER_WR 0x0AC9
+#define mmMC_SEQ_TRAIN_CAPTURE 0x0A3E
+#define mmMC_SEQ_TRAIN_EDC_THRESHOLD 0x0A3B
+#define mmMC_SEQ_TRAIN_EDC_THRESHOLD2 0x0AFE
+#define mmMC_SEQ_TRAIN_EDC_THRESHOLD3 0x0AFF
+#define mmMC_SEQ_TRAIN_TIMING 0x0A40
+#define mmMC_SEQ_TRAIN_WAKEUP_CLEAR 0x0A3F
+#define mmMC_SEQ_TRAIN_WAKEUP_CNTL 0x0A3A
+#define mmMC_SEQ_TRAIN_WAKEUP_EDGE 0x0A3C
+#define mmMC_SEQ_TRAIN_WAKEUP_MASK 0x0A3D
+#define mmMC_SEQ_TSM_BCNT 0x0AC2
+#define mmMC_SEQ_TSM_CTRL 0x0ABE
+#define mmMC_SEQ_TSM_DBI 0x0AC6
+#define mmMC_SEQ_TSM_DEBUG_DATA 0x0AD0
+#define mmMC_SEQ_TSM_DEBUG_INDEX 0x0ACF
+#define mmMC_SEQ_TSM_EDC 0x0AC5
+#define mmMC_SEQ_TSM_FLAG 0x0AC3
+#define mmMC_SEQ_TSM_GCNT 0x0ABF
+#define mmMC_SEQ_TSM_MISC 0x0AE6
+#define mmMC_SEQ_TSM_NCNT 0x0AC1
+#define mmMC_SEQ_TSM_OCNT 0x0AC0
+#define mmMC_SEQ_TSM_UPDATE 0x0AC4
+#define mmMC_SEQ_TSM_WCDR 0x0AE3
+#define mmMC_SEQ_TXFRAMING_BYTE0_D0 0x0A58
+#define mmMC_SEQ_TXFRAMING_BYTE0_D1 0x0A60
+#define mmMC_SEQ_TXFRAMING_BYTE1_D0 0x0A59
+#define mmMC_SEQ_TXFRAMING_BYTE1_D1 0x0A61
+#define mmMC_SEQ_TXFRAMING_BYTE2_D0 0x0A5A
+#define mmMC_SEQ_TXFRAMING_BYTE2_D1 0x0A62
+#define mmMC_SEQ_TXFRAMING_BYTE3_D0 0x0A5B
+#define mmMC_SEQ_TXFRAMING_BYTE3_D1 0x0A63
+#define mmMC_SEQ_TXFRAMING_DBI_D0 0x0A5C
+#define mmMC_SEQ_TXFRAMING_DBI_D1 0x0A64
+#define mmMC_SEQ_TXFRAMING_EDC_D0 0x0A5D
+#define mmMC_SEQ_TXFRAMING_EDC_D1 0x0A65
+#define mmMC_SEQ_TXFRAMING_FCK_D0 0x0A5E
+#define mmMC_SEQ_TXFRAMING_FCK_D1 0x0A66
+#define mmMC_SEQ_VENDOR_ID_I0 0x0A7E
+#define mmMC_SEQ_VENDOR_ID_I1 0x0A7F
+#define mmMC_SEQ_WCDR_CTRL 0x0A39
+#define mmMC_SEQ_WR_CTL_2 0x0AD5
+#define mmMC_SEQ_WR_CTL_2_LP 0x0AD6
+#define mmMC_SEQ_WR_CTL_D0 0x0A2F
+#define mmMC_SEQ_WR_CTL_D0_LP 0x0A9F
+#define mmMC_SEQ_WR_CTL_D1 0x0A30
+#define mmMC_SEQ_WR_CTL_D1_LP 0x0AA0
+#define mmMC_SHARED_BLACKOUT_CNTL 0x082B
+#define mmMC_SHARED_CHMAP 0x0801
+#define mmMC_SHARED_CHREMAP 0x0802
+#define mmMC_TRAIN_EDCCDR_R_D0 0x0A41
+#define mmMC_TRAIN_EDCCDR_R_D1 0x0A42
+#define mmMC_TRAIN_EDC_STATUS_D0 0x0A45
+#define mmMC_TRAIN_EDC_STATUS_D1 0x0A48
+#define mmMC_TRAIN_PRBSERR_0_D0 0x0A43
+#define mmMC_TRAIN_PRBSERR_0_D1 0x0A46
+#define mmMC_TRAIN_PRBSERR_1_D0 0x0A44
+#define mmMC_TRAIN_PRBSERR_1_D1 0x0A47
+#define mmMC_TRAIN_PRBSERR_2_D0 0x0AFB
+#define mmMC_TRAIN_PRBSERR_2_D1 0x0AFC
+#define mmMC_VM_AGP_BASE 0x080C
+#define mmMC_VM_AGP_BOT 0x080B
+#define mmMC_VM_AGP_TOP 0x080A
+#define mmMC_VM_DC_WRITE_CNTL 0x0810
+#define mmMC_VM_DC_WRITE_HIT_REGION_0_HIGH_ADDR 0x0815
+#define mmMC_VM_DC_WRITE_HIT_REGION_0_LOW_ADDR 0x0811
+#define mmMC_VM_DC_WRITE_HIT_REGION_1_HIGH_ADDR 0x0816
+#define mmMC_VM_DC_WRITE_HIT_REGION_1_LOW_ADDR 0x0812
+#define mmMC_VM_DC_WRITE_HIT_REGION_2_HIGH_ADDR 0x0817
+#define mmMC_VM_DC_WRITE_HIT_REGION_2_LOW_ADDR 0x0813
+#define mmMC_VM_DC_WRITE_HIT_REGION_3_HIGH_ADDR 0x0818
+#define mmMC_VM_DC_WRITE_HIT_REGION_3_LOW_ADDR 0x0814
+#define mmMC_VM_FB_LOCATION 0x0809
+#define mmMC_VM_FB_OFFSET 0x081A
+#define mmMC_VM_MB_L1_TLB0_DEBUG 0x0891
+#define mmMC_VM_MB_L1_TLB0_STATUS 0x0895
+#define mmMC_VM_MB_L1_TLB1_STATUS 0x0896
+#define mmMC_VM_MB_L1_TLB2_DEBUG 0x0893
+#define mmMC_VM_MB_L1_TLB2_STATUS 0x0897
+#define mmMC_VM_MB_L1_TLB3_DEBUG 0x08A5
+#define mmMC_VM_MB_L1_TLB3_STATUS 0x08A6
+#define mmMC_VM_MB_L2ARBITER_L2_CREDITS 0x08A1
+#define mmMC_VM_MD_L1_TLB0_DEBUG 0x0998
+#define mmMC_VM_MD_L1_TLB0_STATUS 0x099B
+#define mmMC_VM_MD_L1_TLB1_DEBUG 0x0999
+#define mmMC_VM_MD_L1_TLB1_STATUS 0x099C
+#define mmMC_VM_MD_L1_TLB2_DEBUG 0x099A
+#define mmMC_VM_MD_L1_TLB2_STATUS 0x099D
+#define mmMC_VM_MD_L1_TLB3_DEBUG 0x09A7
+#define mmMC_VM_MD_L1_TLB3_STATUS 0x09A8
+#define mmMC_VM_MD_L2ARBITER_L2_CREDITS 0x09A4
+#define mmMC_VM_MX_L1_TLB_CNTL 0x0819
+#define mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR 0x080F
+#define mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR 0x080E
+#define mmMC_VM_SYSTEM_APERTURE_LOW_ADDR 0x080D
+#define mmMC_WR_CB 0x0986
+#define mmMC_WR_DB 0x0987
+#define mmMC_WR_GRP_EXT 0x0979
+#define mmMC_WR_GRP_GFX 0x0804
+#define mmMC_WR_GRP_LCL 0x098B
+#define mmMC_WR_GRP_OTH 0x0808
+#define mmMC_WR_GRP_SYS 0x0806
+#define mmMC_WR_HUB 0x0988
+#define mmMC_WR_TC0 0x097B
+#define mmMC_WR_TC1 0x097C
+#define mmMC_XBAR_ADDR_DEC 0x0C80
+#define mmMC_XBAR_ARB 0x0C8D
+#define mmMC_XBAR_ARB_MAX_BURST 0x0C8E
+#define mmMC_XBAR_CHTRIREMAP 0x0C8B
+#define mmMC_XBAR_PERF_MON_CNTL0 0x0C8F
+#define mmMC_XBAR_PERF_MON_CNTL1 0x0C90
+#define mmMC_XBAR_PERF_MON_CNTL2 0x0C91
+#define mmMC_XBAR_PERF_MON_MAX_THSH 0x0C96
+#define mmMC_XBAR_PERF_MON_RSLT0 0x0C92
+#define mmMC_XBAR_PERF_MON_RSLT1 0x0C93
+#define mmMC_XBAR_PERF_MON_RSLT2 0x0C94
+#define mmMC_XBAR_PERF_MON_RSLT3 0x0C95
+#define mmMC_XBAR_RDREQ_CREDIT 0x0C83
+#define mmMC_XBAR_RDREQ_PRI_CREDIT 0x0C84
+#define mmMC_XBAR_RDRET_CREDIT1 0x0C87
+#define mmMC_XBAR_RDRET_CREDIT2 0x0C88
+#define mmMC_XBAR_RDRET_PRI_CREDIT1 0x0C89
+#define mmMC_XBAR_RDRET_PRI_CREDIT2 0x0C8A
+#define mmMC_XBAR_REMOTE 0x0C81
+#define mmMC_XBAR_SPARE0 0x0C97
+#define mmMC_XBAR_SPARE1 0x0C98
+#define mmMC_XBAR_TWOCHAN 0x0C8C
+#define mmMC_XBAR_WRREQ_CREDIT 0x0C82
+#define mmMC_XBAR_WRRET_CREDIT1 0x0C85
+#define mmMC_XBAR_WRRET_CREDIT2 0x0C86
+#define mmMC_XPB_CLG_CFG0 0x08E9
+#define mmMC_XPB_CLG_CFG10 0x08F3
+#define mmMC_XPB_CLG_CFG1 0x08EA
+#define mmMC_XPB_CLG_CFG11 0x08F4
+#define mmMC_XPB_CLG_CFG12 0x08F5
+#define mmMC_XPB_CLG_CFG13 0x08F6
+#define mmMC_XPB_CLG_CFG14 0x08F7
+#define mmMC_XPB_CLG_CFG15 0x08F8
+#define mmMC_XPB_CLG_CFG16 0x08F9
+#define mmMC_XPB_CLG_CFG17 0x08FA
+#define mmMC_XPB_CLG_CFG18 0x08FB
+#define mmMC_XPB_CLG_CFG19 0x08FC
+#define mmMC_XPB_CLG_CFG20 0x0928
+#define mmMC_XPB_CLG_CFG2 0x08EB
+#define mmMC_XPB_CLG_CFG21 0x0929
+#define mmMC_XPB_CLG_CFG22 0x092A
+#define mmMC_XPB_CLG_CFG23 0x092B
+#define mmMC_XPB_CLG_CFG24 0x092C
+#define mmMC_XPB_CLG_CFG25 0x092D
+#define mmMC_XPB_CLG_CFG26 0x092E
+#define mmMC_XPB_CLG_CFG27 0x092F
+#define mmMC_XPB_CLG_CFG28 0x0930
+#define mmMC_XPB_CLG_CFG29 0x0931
+#define mmMC_XPB_CLG_CFG30 0x0932
+#define mmMC_XPB_CLG_CFG3 0x08EC
+#define mmMC_XPB_CLG_CFG31 0x0933
+#define mmMC_XPB_CLG_CFG32 0x0936
+#define mmMC_XPB_CLG_CFG33 0x0937
+#define mmMC_XPB_CLG_CFG34 0x0938
+#define mmMC_XPB_CLG_CFG35 0x0939
+#define mmMC_XPB_CLG_CFG36 0x093A
+#define mmMC_XPB_CLG_CFG4 0x08ED
+#define mmMC_XPB_CLG_CFG5 0x08EE
+#define mmMC_XPB_CLG_CFG6 0x08EF
+#define mmMC_XPB_CLG_CFG7 0x08F0
+#define mmMC_XPB_CLG_CFG8 0x08F1
+#define mmMC_XPB_CLG_CFG9 0x08F2
+#define mmMC_XPB_CLG_EXTRA 0x08FD
+#define mmMC_XPB_CLG_EXTRA_RD 0x0935
+#define mmMC_XPB_CLK_GAT 0x091E
+#define mmMC_XPB_INTF_CFG 0x091F
+#define mmMC_XPB_INTF_CFG2 0x0934
+#define mmMC_XPB_INTF_STS 0x0920
+#define mmMC_XPB_LB_ADDR 0x08FE
+#define mmMC_XPB_MAP_INVERT_FLUSH_NUM_LSB 0x0923
+#define mmMC_XPB_MISC_CFG 0x0927
+#define mmMC_XPB_P2P_BAR0 0x0904
+#define mmMC_XPB_P2P_BAR1 0x0905
+#define mmMC_XPB_P2P_BAR2 0x0906
+#define mmMC_XPB_P2P_BAR3 0x0907
+#define mmMC_XPB_P2P_BAR4 0x0908
+#define mmMC_XPB_P2P_BAR5 0x0909
+#define mmMC_XPB_P2P_BAR6 0x090A
+#define mmMC_XPB_P2P_BAR7 0x090B
+#define mmMC_XPB_P2P_BAR_CFG 0x0903
+#define mmMC_XPB_P2P_BAR_DEBUG 0x090D
+#define mmMC_XPB_P2P_BAR_DELTA_ABOVE 0x090E
+#define mmMC_XPB_P2P_BAR_DELTA_BELOW 0x090F
+#define mmMC_XPB_P2P_BAR_SETUP 0x090C
+#define mmMC_XPB_PEER_SYS_BAR0 0x0910
+#define mmMC_XPB_PEER_SYS_BAR1 0x0911
+#define mmMC_XPB_PEER_SYS_BAR2 0x0912
+#define mmMC_XPB_PEER_SYS_BAR3 0x0913
+#define mmMC_XPB_PEER_SYS_BAR4 0x0914
+#define mmMC_XPB_PEER_SYS_BAR5 0x0915
+#define mmMC_XPB_PEER_SYS_BAR6 0x0916
+#define mmMC_XPB_PEER_SYS_BAR7 0x0917
+#define mmMC_XPB_PEER_SYS_BAR8 0x0918
+#define mmMC_XPB_PEER_SYS_BAR9 0x0919
+#define mmMC_XPB_PERF_KNOBS 0x0924
+#define mmMC_XPB_PIPE_STS 0x0921
+#define mmMC_XPB_RTR_DEST_MAP0 0x08DB
+#define mmMC_XPB_RTR_DEST_MAP1 0x08DC
+#define mmMC_XPB_RTR_DEST_MAP2 0x08DD
+#define mmMC_XPB_RTR_DEST_MAP3 0x08DE
+#define mmMC_XPB_RTR_DEST_MAP4 0x08DF
+#define mmMC_XPB_RTR_DEST_MAP5 0x08E0
+#define mmMC_XPB_RTR_DEST_MAP6 0x08E1
+#define mmMC_XPB_RTR_DEST_MAP7 0x08E2
+#define mmMC_XPB_RTR_DEST_MAP8 0x08E3
+#define mmMC_XPB_RTR_DEST_MAP9 0x08E4
+#define mmMC_XPB_RTR_SRC_APRTR0 0x08CD
+#define mmMC_XPB_RTR_SRC_APRTR1 0x08CE
+#define mmMC_XPB_RTR_SRC_APRTR2 0x08CF
+#define mmMC_XPB_RTR_SRC_APRTR3 0x08D0
+#define mmMC_XPB_RTR_SRC_APRTR4 0x08D1
+#define mmMC_XPB_RTR_SRC_APRTR5 0x08D2
+#define mmMC_XPB_RTR_SRC_APRTR6 0x08D3
+#define mmMC_XPB_RTR_SRC_APRTR7 0x08D4
+#define mmMC_XPB_RTR_SRC_APRTR8 0x08D5
+#define mmMC_XPB_RTR_SRC_APRTR9 0x08D6
+#define mmMC_XPB_STICKY 0x0925
+#define mmMC_XPB_STICKY_W1C 0x0926
+#define mmMC_XPB_SUB_CTRL 0x0922
+#define mmMC_XPB_UNC_THRESH_HST 0x08FF
+#define mmMC_XPB_UNC_THRESH_SID 0x0900
+#define mmMC_XPB_WCB_CFG 0x0902
+#define mmMC_XPB_WCB_STS 0x0901
+#define mmMC_XPB_XDMA_PEER_SYS_BAR0 0x091A
+#define mmMC_XPB_XDMA_PEER_SYS_BAR1 0x091B
+#define mmMC_XPB_XDMA_PEER_SYS_BAR2 0x091C
+#define mmMC_XPB_XDMA_PEER_SYS_BAR3 0x091D
+#define mmMC_XPB_XDMA_RTR_DEST_MAP0 0x08E5
+#define mmMC_XPB_XDMA_RTR_DEST_MAP1 0x08E6
+#define mmMC_XPB_XDMA_RTR_DEST_MAP2 0x08E7
+#define mmMC_XPB_XDMA_RTR_DEST_MAP3 0x08E8
+#define mmMC_XPB_XDMA_RTR_SRC_APRTR0 0x08D7
+#define mmMC_XPB_XDMA_RTR_SRC_APRTR1 0x08D8
+#define mmMC_XPB_XDMA_RTR_SRC_APRTR2 0x08D9
+#define mmMC_XPB_XDMA_RTR_SRC_APRTR3 0x08DA
+#define mmMPLL_AD_FUNC_CNTL 0x0AF0
+#define mmMPLL_AD_STATUS 0x0AF6
+#define mmMPLL_CNTL_MODE 0x0AEC
+#define mmMPLL_CONTROL 0x0AF5
+#define mmMPLL_DQ_0_0_STATUS 0x0AF7
+#define mmMPLL_DQ_0_1_STATUS 0x0AF8
+#define mmMPLL_DQ_1_0_STATUS 0x0AF9
+#define mmMPLL_DQ_1_1_STATUS 0x0AFA
+#define mmMPLL_DQ_FUNC_CNTL 0x0AF1
+#define mmMPLL_FUNC_CNTL 0x0AED
+#define mmMPLL_FUNC_CNTL_1 0x0AEE
+#define mmMPLL_FUNC_CNTL_2 0x0AEF
+#define mmMPLL_SEQ_UCODE_1 0x0AEA
+#define mmMPLL_SEQ_UCODE_2 0x0AEB
+#define mmMPLL_SS1 0x0AF3
+#define mmMPLL_SS2 0x0AF4
+#define mmMPLL_TIME 0x0AF2
+#define mmVM_CONTEXT0_CNTL 0x0504
+#define mmVM_CONTEXT0_CNTL2 0x050C
+#define mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR 0x054F
+#define mmVM_CONTEXT0_PAGE_TABLE_END_ADDR 0x055F
+#define mmVM_CONTEXT0_PAGE_TABLE_START_ADDR 0x0557
+#define mmVM_CONTEXT0_PROTECTION_FAULT_ADDR 0x053E
+#define mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR 0x0546
+#define mmVM_CONTEXT0_PROTECTION_FAULT_STATUS 0x0536
+#define mmVM_CONTEXT10_PAGE_TABLE_BASE_ADDR 0x0510
+#define mmVM_CONTEXT11_PAGE_TABLE_BASE_ADDR 0x0511
+#define mmVM_CONTEXT12_PAGE_TABLE_BASE_ADDR 0x0512
+#define mmVM_CONTEXT13_PAGE_TABLE_BASE_ADDR 0x0513
+#define mmVM_CONTEXT14_PAGE_TABLE_BASE_ADDR 0x0514
+#define mmVM_CONTEXT15_PAGE_TABLE_BASE_ADDR 0x0515
+#define mmVM_CONTEXT1_CNTL 0x0505
+#define mmVM_CONTEXT1_CNTL2 0x050D
+#define mmVM_CONTEXT1_PAGE_TABLE_BASE_ADDR 0x0550
+#define mmVM_CONTEXT1_PAGE_TABLE_END_ADDR 0x0560
+#define mmVM_CONTEXT1_PAGE_TABLE_START_ADDR 0x0558
+#define mmVM_CONTEXT1_PROTECTION_FAULT_ADDR 0x053F
+#define mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR 0x0547
+#define mmVM_CONTEXT1_PROTECTION_FAULT_STATUS 0x0537
+#define mmVM_CONTEXT2_PAGE_TABLE_BASE_ADDR 0x0551
+#define mmVM_CONTEXT3_PAGE_TABLE_BASE_ADDR 0x0552
+#define mmVM_CONTEXT4_PAGE_TABLE_BASE_ADDR 0x0553
+#define mmVM_CONTEXT5_PAGE_TABLE_BASE_ADDR 0x0554
+#define mmVM_CONTEXT6_PAGE_TABLE_BASE_ADDR 0x0555
+#define mmVM_CONTEXT7_PAGE_TABLE_BASE_ADDR 0x0556
+#define mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR 0x050E
+#define mmVM_CONTEXT9_PAGE_TABLE_BASE_ADDR 0x050F
+#define mmVM_CONTEXTS_DISABLE 0x0535
+#define mmVM_DEBUG 0x056F
+#define mmVM_DUMMY_PAGE_FAULT_ADDR 0x0507
+#define mmVM_DUMMY_PAGE_FAULT_CNTL 0x0506
+#define mmVM_FAULT_CLIENT_ID 0x054E
+#define mmVM_INVALIDATE_REQUEST 0x051E
+#define mmVM_INVALIDATE_RESPONSE 0x051F
+#define mmVM_L2_BANK_SELECT_MASKA 0x0572
+#define mmVM_L2_BANK_SELECT_MASKB 0x0573
+#define mmVM_L2_CG 0x0570
+#define mmVM_L2_CNTL 0x0500
+#define mmVM_L2_CNTL2 0x0501
+#define mmVM_L2_CNTL3 0x0502
+#define mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR 0x0576
+#define mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR 0x0575
+#define mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET 0x0577
+#define mmVM_L2_STATUS 0x0503
+#define mmVM_PRT_APERTURE0_HIGH_ADDR 0x0530
+#define mmVM_PRT_APERTURE0_LOW_ADDR 0x052C
+#define mmVM_PRT_APERTURE1_HIGH_ADDR 0x0531
+#define mmVM_PRT_APERTURE1_LOW_ADDR 0x052D
+#define mmVM_PRT_APERTURE2_HIGH_ADDR 0x0532
+#define mmVM_PRT_APERTURE2_LOW_ADDR 0x052E
+#define mmVM_PRT_APERTURE3_HIGH_ADDR 0x0533
+#define mmVM_PRT_APERTURE3_LOW_ADDR 0x052F
+#define mmVM_PRT_CNTL 0x0534
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gmc/gmc_6_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gmc/gmc_6_0_sh_mask.h
new file mode 100644
index 000000000000..0f6c6c8d089b
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/gmc/gmc_6_0_sh_mask.h
@@ -0,0 +1,11895 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef GMC_6_0_SH_MASK_H
+#define GMC_6_0_SH_MASK_H
+
+#define ATC_ATS_CNTL__CREDITS_ATS_RPB_MASK 0x00003f00L
+#define ATC_ATS_CNTL__CREDITS_ATS_RPB__SHIFT 0x00000008
+#define ATC_ATS_CNTL__DEBUG_ECO_MASK 0x000f0000L
+#define ATC_ATS_CNTL__DEBUG_ECO__SHIFT 0x00000010
+#define ATC_ATS_CNTL__DISABLE_ATC_MASK 0x00000001L
+#define ATC_ATS_CNTL__DISABLE_ATC__SHIFT 0x00000000
+#define ATC_ATS_CNTL__DISABLE_PASID_MASK 0x00000004L
+#define ATC_ATS_CNTL__DISABLE_PASID__SHIFT 0x00000002
+#define ATC_ATS_CNTL__DISABLE_PRI_MASK 0x00000002L
+#define ATC_ATS_CNTL__DISABLE_PRI__SHIFT 0x00000001
+#define ATC_ATS_DEBUG__ADDRESS_TRANSLATION_REQUEST_WRITE_PERMS_MASK 0x00000004L
+#define ATC_ATS_DEBUG__ADDRESS_TRANSLATION_REQUEST_WRITE_PERMS__SHIFT 0x00000002
+#define ATC_ATS_DEBUG__DISALLOW_ERR_TO_DONE_MASK 0x00004000L
+#define ATC_ATS_DEBUG__DISALLOW_ERR_TO_DONE__SHIFT 0x0000000e
+#define ATC_ATS_DEBUG__EXE_BIT_MASK 0x00000080L
+#define ATC_ATS_DEBUG__EXE_BIT__SHIFT 0x00000007
+#define ATC_ATS_DEBUG__IDENT_RETURN_MASK 0x00000002L
+#define ATC_ATS_DEBUG__IDENT_RETURN__SHIFT 0x00000001
+#define ATC_ATS_DEBUG__IGNORE_FED_MASK 0x00008000L
+#define ATC_ATS_DEBUG__IGNORE_FED__SHIFT 0x0000000f
+#define ATC_ATS_DEBUG__INVALIDATE_ALL_MASK 0x00000001L
+#define ATC_ATS_DEBUG__INVALIDATE_ALL__SHIFT 0x00000000
+#define ATC_ATS_DEBUG__INVALIDATION_REQUESTS_DISALLOWED_WHEN_ATC_IS_DISABLED_MASK 0x00010000L
+#define ATC_ATS_DEBUG__INVALIDATION_REQUESTS_DISALLOWED_WHEN_ATC_IS_DISABLED__SHIFT 0x00000010
+#define ATC_ATS_DEBUG__NUM_REQUESTS_AT_ERR_MASK 0x00003c00L
+#define ATC_ATS_DEBUG__NUM_REQUESTS_AT_ERR__SHIFT 0x0000000a
+#define ATC_ATS_DEBUG__PAGE_REQUEST_PERMS_MASK 0x00000100L
+#define ATC_ATS_DEBUG__PAGE_REQUEST_PERMS__SHIFT 0x00000008
+#define ATC_ATS_DEBUG__PAGE_REQUESTS_USE_RELAXED_ORDERING_MASK 0x00000020L
+#define ATC_ATS_DEBUG__PAGE_REQUESTS_USE_RELAXED_ORDERING__SHIFT 0x00000005
+#define ATC_ATS_DEBUG__PRIV_BIT_MASK 0x00000040L
+#define ATC_ATS_DEBUG__PRIV_BIT__SHIFT 0x00000006
+#define ATC_ATS_DEBUG__UNTRANSLATED_ONLY_REQUESTS_CARRY_SIZE_MASK 0x00000200L
+#define ATC_ATS_DEBUG__UNTRANSLATED_ONLY_REQUESTS_CARRY_SIZE__SHIFT 0x00000009
+#define ATC_ATS_DEFAULT_PAGE_CNTL__DEFAULT_PAGE_HIGH_MASK 0x0000003cL
+#define ATC_ATS_DEFAULT_PAGE_CNTL__DEFAULT_PAGE_HIGH__SHIFT 0x00000002
+#define ATC_ATS_DEFAULT_PAGE_CNTL__SEND_DEFAULT_PAGE_MASK 0x00000001L
+#define ATC_ATS_DEFAULT_PAGE_CNTL__SEND_DEFAULT_PAGE__SHIFT 0x00000000
+#define ATC_ATS_DEFAULT_PAGE_LOW__DEFAULT_PAGE_MASK 0xffffffffL
+#define ATC_ATS_DEFAULT_PAGE_LOW__DEFAULT_PAGE__SHIFT 0x00000000
+#define ATC_ATS_FAULT_CNTL__FAULT_CRASH_TABLE_MASK 0x03f00000L
+#define ATC_ATS_FAULT_CNTL__FAULT_CRASH_TABLE__SHIFT 0x00000014
+#define ATC_ATS_FAULT_CNTL__FAULT_INTERRUPT_TABLE_MASK 0x0000fc00L
+#define ATC_ATS_FAULT_CNTL__FAULT_INTERRUPT_TABLE__SHIFT 0x0000000a
+#define ATC_ATS_FAULT_CNTL__FAULT_REGISTER_LOG_MASK 0x0000003fL
+#define ATC_ATS_FAULT_CNTL__FAULT_REGISTER_LOG__SHIFT 0x00000000
+#define ATC_ATS_FAULT_DEBUG__ALLOW_SUBSEQUENT_FAULT_STATUS_ADDR_UPDATES_MASK 0x00000100L
+#define ATC_ATS_FAULT_DEBUG__ALLOW_SUBSEQUENT_FAULT_STATUS_ADDR_UPDATES__SHIFT 0x00000008
+#define ATC_ATS_FAULT_DEBUG__CLEAR_FAULT_STATUS_ADDR_MASK 0x00010000L
+#define ATC_ATS_FAULT_DEBUG__CLEAR_FAULT_STATUS_ADDR__SHIFT 0x00000010
+#define ATC_ATS_FAULT_DEBUG__CREDITS_ATS_IH_MASK 0x0000001fL
+#define ATC_ATS_FAULT_DEBUG__CREDITS_ATS_IH__SHIFT 0x00000000
+#define ATC_ATS_FAULT_STATUS_ADDR__PAGE_ADDR_MASK 0xffffffffL
+#define ATC_ATS_FAULT_STATUS_ADDR__PAGE_ADDR__SHIFT 0x00000000
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO2_MASK 0x00010000L
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO2__SHIFT 0x00000010
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO_MASK 0x00008000L
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO__SHIFT 0x0000000f
+#define ATC_ATS_FAULT_STATUS_INFO__FAULT_TYPE_MASK 0x0000003fL
+#define ATC_ATS_FAULT_STATUS_INFO__FAULT_TYPE__SHIFT 0x00000000
+#define ATC_ATS_FAULT_STATUS_INFO__INVALIDATION_MASK 0x00020000L
+#define ATC_ATS_FAULT_STATUS_INFO__INVALIDATION__SHIFT 0x00000011
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_ADDR_HIGH_MASK 0x0f000000L
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_ADDR_HIGH__SHIFT 0x00000018
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_REQUEST_MASK 0x00040000L
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_REQUEST__SHIFT 0x00000012
+#define ATC_ATS_FAULT_STATUS_INFO__STATUS_MASK 0x00f80000L
+#define ATC_ATS_FAULT_STATUS_INFO__STATUS__SHIFT 0x00000013
+#define ATC_ATS_FAULT_STATUS_INFO__VMID_MASK 0x00007c00L
+#define ATC_ATS_FAULT_STATUS_INFO__VMID__SHIFT 0x0000000a
+#define ATC_ATS_STATUS__BUSY_MASK 0x00000001L
+#define ATC_ATS_STATUS__BUSY__SHIFT 0x00000000
+#define ATC_ATS_STATUS__CRASHED_MASK 0x00000002L
+#define ATC_ATS_STATUS__CRASHED__SHIFT 0x00000001
+#define ATC_ATS_STATUS__DEADLOCK_DETECTION_MASK 0x00000004L
+#define ATC_ATS_STATUS__DEADLOCK_DETECTION__SHIFT 0x00000002
+#define ATC_L1_ADDRESS_OFFSET__LOGICAL_ADDRESS_MASK 0xffffffffL
+#define ATC_L1_ADDRESS_OFFSET__LOGICAL_ADDRESS__SHIFT 0x00000000
+#define ATC_L1_CNTL__DONT_NEED_ATS_BEHAVIOR_MASK 0x00000003L
+#define ATC_L1_CNTL__DONT_NEED_ATS_BEHAVIOR__SHIFT 0x00000000
+#define ATC_L1_CNTL__NEED_ATS_BEHAVIOR_MASK 0x00000004L
+#define ATC_L1_CNTL__NEED_ATS_BEHAVIOR__SHIFT 0x00000002
+#define ATC_L1_CNTL__NEED_ATS_SNOOP_DEFAULT_MASK 0x00000010L
+#define ATC_L1_CNTL__NEED_ATS_SNOOP_DEFAULT__SHIFT 0x00000004
+#define ATC_L1RD_DEBUG_TLB__CREDITS_L1_L2_MASK 0x0003f000L
+#define ATC_L1RD_DEBUG_TLB__CREDITS_L1_L2__SHIFT 0x0000000c
+#define ATC_L1RD_DEBUG_TLB__CREDITS_L1_RPB_MASK 0x0ff00000L
+#define ATC_L1RD_DEBUG_TLB__CREDITS_L1_RPB__SHIFT 0x00000014
+#define ATC_L1RD_DEBUG_TLB__DEBUG_ECO_MASK 0x30000000L
+#define ATC_L1RD_DEBUG_TLB__DEBUG_ECO__SHIFT 0x0000001c
+#define ATC_L1RD_DEBUG_TLB__DISABLE_FRAGMENTS_MASK 0x00000001L
+#define ATC_L1RD_DEBUG_TLB__DISABLE_FRAGMENTS__SHIFT 0x00000000
+#define ATC_L1RD_DEBUG_TLB__EFFECTIVE_CAM_SIZE_MASK 0x000000f0L
+#define ATC_L1RD_DEBUG_TLB__EFFECTIVE_CAM_SIZE__SHIFT 0x00000004
+#define ATC_L1RD_DEBUG_TLB__EFFECTIVE_WORK_QUEUE_SIZE_MASK 0x00000700L
+#define ATC_L1RD_DEBUG_TLB__EFFECTIVE_WORK_QUEUE_SIZE__SHIFT 0x00000008
+#define ATC_L1RD_DEBUG_TLB__INVALIDATE_ALL_MASK 0x40000000L
+#define ATC_L1RD_DEBUG_TLB__INVALIDATE_ALL__SHIFT 0x0000001e
+#define ATC_L1RD_STATUS__BAD_NEED_ATS_MASK 0x00000100L
+#define ATC_L1RD_STATUS__BAD_NEED_ATS__SHIFT 0x00000008
+#define ATC_L1RD_STATUS__BUSY_MASK 0x00000001L
+#define ATC_L1RD_STATUS__BUSY__SHIFT 0x00000000
+#define ATC_L1RD_STATUS__DEADLOCK_DETECTION_MASK 0x00000002L
+#define ATC_L1RD_STATUS__DEADLOCK_DETECTION__SHIFT 0x00000001
+#define ATC_L1WR_DEBUG_TLB__CREDITS_L1_L2_MASK 0x0003f000L
+#define ATC_L1WR_DEBUG_TLB__CREDITS_L1_L2__SHIFT 0x0000000c
+#define ATC_L1WR_DEBUG_TLB__CREDITS_L1_RPB_MASK 0x0ff00000L
+#define ATC_L1WR_DEBUG_TLB__CREDITS_L1_RPB__SHIFT 0x00000014
+#define ATC_L1WR_DEBUG_TLB__DEBUG_ECO_MASK 0x30000000L
+#define ATC_L1WR_DEBUG_TLB__DEBUG_ECO__SHIFT 0x0000001c
+#define ATC_L1WR_DEBUG_TLB__DISABLE_FRAGMENTS_MASK 0x00000001L
+#define ATC_L1WR_DEBUG_TLB__DISABLE_FRAGMENTS__SHIFT 0x00000000
+#define ATC_L1WR_DEBUG_TLB__EFFECTIVE_CAM_SIZE_MASK 0x000000f0L
+#define ATC_L1WR_DEBUG_TLB__EFFECTIVE_CAM_SIZE__SHIFT 0x00000004
+#define ATC_L1WR_DEBUG_TLB__EFFECTIVE_WORK_QUEUE_SIZE_MASK 0x00000700L
+#define ATC_L1WR_DEBUG_TLB__EFFECTIVE_WORK_QUEUE_SIZE__SHIFT 0x00000008
+#define ATC_L1WR_DEBUG_TLB__INVALIDATE_ALL_MASK 0x40000000L
+#define ATC_L1WR_DEBUG_TLB__INVALIDATE_ALL__SHIFT 0x0000001e
+#define ATC_L1WR_STATUS__BAD_NEED_ATS_MASK 0x00000100L
+#define ATC_L1WR_STATUS__BAD_NEED_ATS__SHIFT 0x00000008
+#define ATC_L1WR_STATUS__BUSY_MASK 0x00000001L
+#define ATC_L1WR_STATUS__BUSY__SHIFT 0x00000000
+#define ATC_L1WR_STATUS__DEADLOCK_DETECTION_MASK 0x00000002L
+#define ATC_L1WR_STATUS__DEADLOCK_DETECTION__SHIFT 0x00000001
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS_MASK 0x00000003L
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS__SHIFT 0x00000000
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD_MASK 0x00000400L
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD__SHIFT 0x0000000a
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS_MASK 0x00000030L
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS__SHIFT 0x00000004
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD_MASK 0x00000800L
+#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD__SHIFT 0x0000000b
+#define ATC_L2_DEBUG__CREDITS_L2_ATS_MASK 0x0000003fL
+#define ATC_L2_DEBUG__CREDITS_L2_ATS__SHIFT 0x00000000
+#define ATC_MISC_CG__ENABLE_MASK 0x00040000L
+#define ATC_MISC_CG__ENABLE__SHIFT 0x00000012
+#define ATC_MISC_CG__MEM_LS_ENABLE_MASK 0x00080000L
+#define ATC_MISC_CG__MEM_LS_ENABLE__SHIFT 0x00000013
+#define ATC_MISC_CG__OFFDLY_MASK 0x00000fc0L
+#define ATC_MISC_CG__OFFDLY__SHIFT 0x00000006
+#define ATC_VM_APERTURE0_CNTL2__VMIDS_USING_RANGE_MASK 0x0000ffffL
+#define ATC_VM_APERTURE0_CNTL2__VMIDS_USING_RANGE__SHIFT 0x00000000
+#define ATC_VM_APERTURE0_CNTL__ATS_ACCESS_MODE_MASK 0x00000003L
+#define ATC_VM_APERTURE0_CNTL__ATS_ACCESS_MODE__SHIFT 0x00000000
+#define ATC_VM_APERTURE0_HIGH_ADDR__VIRTUAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define ATC_VM_APERTURE0_HIGH_ADDR__VIRTUAL_PAGE_NUMBER__SHIFT 0x00000000
+#define ATC_VM_APERTURE0_LOW_ADDR__VIRTUAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define ATC_VM_APERTURE0_LOW_ADDR__VIRTUAL_PAGE_NUMBER__SHIFT 0x00000000
+#define ATC_VM_APERTURE1_CNTL2__VMIDS_USING_RANGE_MASK 0x0000ffffL
+#define ATC_VM_APERTURE1_CNTL2__VMIDS_USING_RANGE__SHIFT 0x00000000
+#define ATC_VM_APERTURE1_CNTL__ATS_ACCESS_MODE_MASK 0x00000003L
+#define ATC_VM_APERTURE1_CNTL__ATS_ACCESS_MODE__SHIFT 0x00000000
+#define ATC_VM_APERTURE1_HIGH_ADDR__VIRTUAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define ATC_VM_APERTURE1_HIGH_ADDR__VIRTUAL_PAGE_NUMBER__SHIFT 0x00000000
+#define ATC_VM_APERTURE1_LOW_ADDR__VIRTUAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define ATC_VM_APERTURE1_LOW_ADDR__VIRTUAL_PAGE_NUMBER__SHIFT 0x00000000
+#define ATC_VMID0_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID0_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID0_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID0_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID10_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID10_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID10_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID10_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID11_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID11_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID11_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID11_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID12_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID12_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID12_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID12_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID13_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID13_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID13_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID13_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID14_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID14_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID14_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID14_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID15_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID15_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID15_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID15_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID1_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID1_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID1_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID1_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID2_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID2_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID2_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID2_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID3_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID3_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID3_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID3_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID4_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID4_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID4_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID4_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID5_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID5_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID5_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID5_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID6_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID6_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID6_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID6_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID7_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID7_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID7_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID7_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID8_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID8_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID8_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID8_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID9_PASID_MAPPING__PASID_MASK 0x0000ffffL
+#define ATC_VMID9_PASID_MAPPING__PASID__SHIFT 0x00000000
+#define ATC_VMID9_PASID_MAPPING__VALID_MASK 0x80000000L
+#define ATC_VMID9_PASID_MAPPING__VALID__SHIFT 0x0000001f
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID0_REMAPPING_FINISHED_MASK 0x00000001L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID0_REMAPPING_FINISHED__SHIFT 0x00000000
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID10_REMAPPING_FINISHED_MASK 0x00000400L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID10_REMAPPING_FINISHED__SHIFT 0x0000000a
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID11_REMAPPING_FINISHED_MASK 0x00000800L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID11_REMAPPING_FINISHED__SHIFT 0x0000000b
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID12_REMAPPING_FINISHED_MASK 0x00001000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID12_REMAPPING_FINISHED__SHIFT 0x0000000c
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID13_REMAPPING_FINISHED_MASK 0x00002000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID13_REMAPPING_FINISHED__SHIFT 0x0000000d
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID14_REMAPPING_FINISHED_MASK 0x00004000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID14_REMAPPING_FINISHED__SHIFT 0x0000000e
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID15_REMAPPING_FINISHED_MASK 0x00008000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID15_REMAPPING_FINISHED__SHIFT 0x0000000f
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID1_REMAPPING_FINISHED_MASK 0x00000002L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID1_REMAPPING_FINISHED__SHIFT 0x00000001
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID2_REMAPPING_FINISHED_MASK 0x00000004L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID2_REMAPPING_FINISHED__SHIFT 0x00000002
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID3_REMAPPING_FINISHED_MASK 0x00000008L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID3_REMAPPING_FINISHED__SHIFT 0x00000003
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID4_REMAPPING_FINISHED_MASK 0x00000010L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID4_REMAPPING_FINISHED__SHIFT 0x00000004
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID5_REMAPPING_FINISHED_MASK 0x00000020L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID5_REMAPPING_FINISHED__SHIFT 0x00000005
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID6_REMAPPING_FINISHED_MASK 0x00000040L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID6_REMAPPING_FINISHED__SHIFT 0x00000006
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID7_REMAPPING_FINISHED_MASK 0x00000080L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID7_REMAPPING_FINISHED__SHIFT 0x00000007
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID8_REMAPPING_FINISHED_MASK 0x00000100L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID8_REMAPPING_FINISHED__SHIFT 0x00000008
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID9_REMAPPING_FINISHED_MASK 0x00000200L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID9_REMAPPING_FINISHED__SHIFT 0x00000009
+#define CC_MC_MAX_CHANNEL__NOOFCHAN_MASK 0x0000001eL
+#define CC_MC_MAX_CHANNEL__NOOFCHAN__SHIFT 0x00000001
+#define DLL_CNTL__DLL_LOCK_TIME_MASK 0x003ff000L
+#define DLL_CNTL__DLL_LOCK_TIME__SHIFT 0x0000000c
+#define DLL_CNTL__DLL_RESET_TIME_MASK 0x000003ffL
+#define DLL_CNTL__DLL_RESET_TIME__SHIFT 0x00000000
+#define DLL_CNTL__MRDCK0_BYPASS_MASK 0x01000000L
+#define DLL_CNTL__MRDCK0_BYPASS__SHIFT 0x00000018
+#define DLL_CNTL__MRDCK1_BYPASS_MASK 0x02000000L
+#define DLL_CNTL__MRDCK1_BYPASS__SHIFT 0x00000019
+#define DLL_CNTL__PWR2_MODE_MASK 0x04000000L
+#define DLL_CNTL__PWR2_MODE__SHIFT 0x0000001a
+#define GMCON_DEBUG__GFX_CLEAR_MASK 0x00000002L
+#define GMCON_DEBUG__GFX_CLEAR__SHIFT 0x00000001
+#define GMCON_DEBUG__GFX_STALL_MASK 0x00000001L
+#define GMCON_DEBUG__GFX_STALL__SHIFT 0x00000000
+#define GMCON_DEBUG__MISC_FLAGS_MASK 0x3ffffffcL
+#define GMCON_DEBUG__MISC_FLAGS__SHIFT 0x00000002
+#define GMCON_MISC2__RENG_MEM_POWER_CTRL_OVERRIDE0_MASK 0x00000007L
+#define GMCON_MISC2__RENG_MEM_POWER_CTRL_OVERRIDE0__SHIFT 0x00000000
+#define GMCON_MISC2__RENG_MEM_POWER_CTRL_OVERRIDE1_MASK 0x00000038L
+#define GMCON_MISC2__RENG_MEM_POWER_CTRL_OVERRIDE1__SHIFT 0x00000003
+#define GMCON_MISC2__RENG_SR_HOLD_THRESHOLD_MASK 0x0000fc00L
+#define GMCON_MISC2__RENG_SR_HOLD_THRESHOLD__SHIFT 0x0000000a
+#define GMCON_MISC2__STCTRL_EXTEND_GMC_OFFLINE_MASK 0x20000000L
+#define GMCON_MISC2__STCTRL_EXTEND_GMC_OFFLINE__SHIFT 0x0000001d
+#define GMCON_MISC2__STCTRL_IGNORE_ARB_BUSY_MASK 0x10000000L
+#define GMCON_MISC2__STCTRL_IGNORE_ARB_BUSY__SHIFT 0x0000001c
+#define GMCON_MISC2__STCTRL_LPT_TARGET_MASK 0x0fff0000L
+#define GMCON_MISC2__STCTRL_LPT_TARGET__SHIFT 0x00000010
+#define GMCON_MISC3__RENG_DISABLE_MCC_MASK 0x0000003fL
+#define GMCON_MISC3__RENG_DISABLE_MCC__SHIFT 0x00000000
+#define GMCON_MISC3__RENG_DISABLE_MCD_MASK 0x00000fc0L
+#define GMCON_MISC3__RENG_DISABLE_MCD__SHIFT 0x00000006
+#define GMCON_MISC3__STCTRL_FORCE_PGFSM_CMD_DONE_MASK 0x00fff000L
+#define GMCON_MISC3__STCTRL_FORCE_PGFSM_CMD_DONE__SHIFT 0x0000000c
+#define GMCON_MISC__ALLOW_DEEP_SLEEP_MODE_MASK 0x30000000L
+#define GMCON_MISC__ALLOW_DEEP_SLEEP_MODE__SHIFT 0x0000001c
+#define GMCON_MISC__CRITICAL_REGS_LOCK_MASK 0x08000000L
+#define GMCON_MISC__CRITICAL_REGS_LOCK__SHIFT 0x0000001b
+#define GMCON_MISC__RENG_EXECUTE_NOW_MODE_MASK 0x00000400L
+#define GMCON_MISC__RENG_EXECUTE_NOW_MODE__SHIFT 0x0000000a
+#define GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK 0x00000800L
+#define GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE__SHIFT 0x0000000b
+#define GMCON_MISC__RENG_SRBM_CREDITS_MCD_MASK 0x0000f000L
+#define GMCON_MISC__RENG_SRBM_CREDITS_MCD__SHIFT 0x0000000c
+#define GMCON_MISC__STCTRL_DISABLE_ALLOW_SR_MASK 0x02000000L
+#define GMCON_MISC__STCTRL_DISABLE_ALLOW_SR__SHIFT 0x00000019
+#define GMCON_MISC__STCTRL_DISABLE_GMC_OFFLINE_MASK 0x04000000L
+#define GMCON_MISC__STCTRL_DISABLE_GMC_OFFLINE__SHIFT 0x0000001a
+#define GMCON_MISC__STCTRL_FORCE_ALLOW_SR_MASK 0x40000000L
+#define GMCON_MISC__STCTRL_FORCE_ALLOW_SR__SHIFT 0x0000001e
+#define GMCON_MISC__STCTRL_GMC_IDLE_THRESHOLD_MASK 0x00060000L
+#define GMCON_MISC__STCTRL_GMC_IDLE_THRESHOLD__SHIFT 0x00000011
+#define GMCON_MISC__STCTRL_IGNORE_ALLOW_STOP_MASK 0x00400000L
+#define GMCON_MISC__STCTRL_IGNORE_ALLOW_STOP__SHIFT 0x00000016
+#define GMCON_MISC__STCTRL_IGNORE_PRE_SR_MASK 0x00200000L
+#define GMCON_MISC__STCTRL_IGNORE_PRE_SR__SHIFT 0x00000015
+#define GMCON_MISC__STCTRL_IGNORE_PROTECTION_FAULT_MASK 0x01000000L
+#define GMCON_MISC__STCTRL_IGNORE_PROTECTION_FAULT__SHIFT 0x00000018
+#define GMCON_MISC__STCTRL_IGNORE_SR_COMMIT_MASK 0x00800000L
+#define GMCON_MISC__STCTRL_IGNORE_SR_COMMIT__SHIFT 0x00000017
+#define GMCON_MISC__STCTRL_SRBM_IDLE_THRESHOLD_MASK 0x00180000L
+#define GMCON_MISC__STCTRL_SRBM_IDLE_THRESHOLD__SHIFT 0x00000013
+#define GMCON_MISC__STCTRL_STUTTER_EN_MASK 0x00010000L
+#define GMCON_MISC__STCTRL_STUTTER_EN__SHIFT 0x00000010
+#define GMCON_PERF_MON_CNTL0__ALLOW_WRAP_MASK 0x10000000L
+#define GMCON_PERF_MON_CNTL0__ALLOW_WRAP__SHIFT 0x0000001c
+#define GMCON_PERF_MON_CNTL0__START_MODE_MASK 0x03000000L
+#define GMCON_PERF_MON_CNTL0__START_MODE__SHIFT 0x00000018
+#define GMCON_PERF_MON_CNTL0__START_THRESH_MASK 0x00000fffL
+#define GMCON_PERF_MON_CNTL0__START_THRESH__SHIFT 0x00000000
+#define GMCON_PERF_MON_CNTL0__STOP_MODE_MASK 0x0c000000L
+#define GMCON_PERF_MON_CNTL0__STOP_MODE__SHIFT 0x0000001a
+#define GMCON_PERF_MON_CNTL0__STOP_THRESH_MASK 0x00fff000L
+#define GMCON_PERF_MON_CNTL0__STOP_THRESH__SHIFT 0x0000000c
+#define GMCON_PERF_MON_CNTL1__MON0_ID_MASK 0x00fc0000L
+#define GMCON_PERF_MON_CNTL1__MON0_ID__SHIFT 0x00000012
+#define GMCON_PERF_MON_CNTL1__MON1_ID_MASK 0x3f000000L
+#define GMCON_PERF_MON_CNTL1__MON1_ID__SHIFT 0x00000018
+#define GMCON_PERF_MON_CNTL1__START_TRIG_ID_MASK 0x00000fc0L
+#define GMCON_PERF_MON_CNTL1__START_TRIG_ID__SHIFT 0x00000006
+#define GMCON_PERF_MON_CNTL1__STOP_TRIG_ID_MASK 0x0003f000L
+#define GMCON_PERF_MON_CNTL1__STOP_TRIG_ID__SHIFT 0x0000000c
+#define GMCON_PERF_MON_CNTL1__THRESH_CNTR_ID_MASK 0x0000003fL
+#define GMCON_PERF_MON_CNTL1__THRESH_CNTR_ID__SHIFT 0x00000000
+#define GMCON_PERF_MON_RSLT0__COUNT_MASK 0xffffffffL
+#define GMCON_PERF_MON_RSLT0__COUNT__SHIFT 0x00000000
+#define GMCON_PERF_MON_RSLT1__COUNT_MASK 0xffffffffL
+#define GMCON_PERF_MON_RSLT1__COUNT__SHIFT 0x00000000
+#define GMCON_PGFSM_CONFIG__FSM_ADDR_MASK 0x000000ffL
+#define GMCON_PGFSM_CONFIG__FSM_ADDR__SHIFT 0x00000000
+#define GMCON_PGFSM_CONFIG__P1_SELECT_MASK 0x00000400L
+#define GMCON_PGFSM_CONFIG__P1_SELECT__SHIFT 0x0000000a
+#define GMCON_PGFSM_CONFIG__P2_SELECT_MASK 0x00000800L
+#define GMCON_PGFSM_CONFIG__P2_SELECT__SHIFT 0x0000000b
+#define GMCON_PGFSM_CONFIG__POWER_DOWN_MASK 0x00000100L
+#define GMCON_PGFSM_CONFIG__POWER_DOWN__SHIFT 0x00000008
+#define GMCON_PGFSM_CONFIG__POWER_UP_MASK 0x00000200L
+#define GMCON_PGFSM_CONFIG__POWER_UP__SHIFT 0x00000009
+#define GMCON_PGFSM_CONFIG__READ_MASK 0x00002000L
+#define GMCON_PGFSM_CONFIG__READ__SHIFT 0x0000000d
+#define GMCON_PGFSM_CONFIG__REG_ADDR_MASK 0xf0000000L
+#define GMCON_PGFSM_CONFIG__REG_ADDR__SHIFT 0x0000001c
+#define GMCON_PGFSM_CONFIG__RSRVD_MASK 0x07ffc000L
+#define GMCON_PGFSM_CONFIG__RSRVD__SHIFT 0x0000000e
+#define GMCON_PGFSM_CONFIG__SRBM_OVERRIDE_MASK 0x08000000L
+#define GMCON_PGFSM_CONFIG__SRBM_OVERRIDE__SHIFT 0x0000001b
+#define GMCON_PGFSM_CONFIG__WRITE_MASK 0x00001000L
+#define GMCON_PGFSM_CONFIG__WRITE__SHIFT 0x0000000c
+#define GMCON_PGFSM_READ__PGFSM_SELECT_MASK 0x0f000000L
+#define GMCON_PGFSM_READ__PGFSM_SELECT__SHIFT 0x00000018
+#define GMCON_PGFSM_READ__READ_VALUE_MASK 0x00ffffffL
+#define GMCON_PGFSM_READ__READ_VALUE__SHIFT 0x00000000
+#define GMCON_PGFSM_READ__SERDES_MASTER_BUSY_MASK 0x10000000L
+#define GMCON_PGFSM_READ__SERDES_MASTER_BUSY__SHIFT 0x0000001c
+#define GMCON_PGFSM_WRITE__WRITE_VALUE_MASK 0xffffffffL
+#define GMCON_PGFSM_WRITE__WRITE_VALUE__SHIFT 0x00000000
+#define GMCON_RENG_EXECUTE__RENG_EXECUTE_DSP_END_PTR_MASK 0x003ff000L
+#define GMCON_RENG_EXECUTE__RENG_EXECUTE_DSP_END_PTR__SHIFT 0x0000000c
+#define GMCON_RENG_EXECUTE__RENG_EXECUTE_END_PTR_MASK 0xffc00000L
+#define GMCON_RENG_EXECUTE__RENG_EXECUTE_END_PTR__SHIFT 0x00000016
+#define GMCON_RENG_EXECUTE__RENG_EXECUTE_NOW_MASK 0x00000002L
+#define GMCON_RENG_EXECUTE__RENG_EXECUTE_NOW__SHIFT 0x00000001
+#define GMCON_RENG_EXECUTE__RENG_EXECUTE_NOW_START_PTR_MASK 0x00000ffcL
+#define GMCON_RENG_EXECUTE__RENG_EXECUTE_NOW_START_PTR__SHIFT 0x00000002
+#define GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK 0x00000001L
+#define GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP__SHIFT 0x00000000
+#define GMCON_RENG_RAM_DATA__RENG_RAM_DATA_MASK 0xffffffffL
+#define GMCON_RENG_RAM_DATA__RENG_RAM_DATA__SHIFT 0x00000000
+#define GMCON_RENG_RAM_INDEX__RENG_RAM_INDEX_MASK 0x000003ffL
+#define GMCON_RENG_RAM_INDEX__RENG_RAM_INDEX__SHIFT 0x00000000
+#define GMCON_STCTRL_REGISTER_SAVE_EXCL_SET0__STCTRL_REGISTER_SAVE_EXCL0_MASK 0x0000ffffL
+#define GMCON_STCTRL_REGISTER_SAVE_EXCL_SET0__STCTRL_REGISTER_SAVE_EXCL0__SHIFT 0x00000000
+#define GMCON_STCTRL_REGISTER_SAVE_EXCL_SET0__STCTRL_REGISTER_SAVE_EXCL1_MASK 0xffff0000L
+#define GMCON_STCTRL_REGISTER_SAVE_EXCL_SET0__STCTRL_REGISTER_SAVE_EXCL1__SHIFT 0x00000010
+#define GMCON_STCTRL_REGISTER_SAVE_EXCL_SET1__STCTRL_REGISTER_SAVE_EXCL2_MASK 0x0000ffffL
+#define GMCON_STCTRL_REGISTER_SAVE_EXCL_SET1__STCTRL_REGISTER_SAVE_EXCL2__SHIFT 0x00000000
+#define GMCON_STCTRL_REGISTER_SAVE_EXCL_SET1__STCTRL_REGISTER_SAVE_EXCL3_MASK 0xffff0000L
+#define GMCON_STCTRL_REGISTER_SAVE_EXCL_SET1__STCTRL_REGISTER_SAVE_EXCL3__SHIFT 0x00000010
+#define GMCON_STCTRL_REGISTER_SAVE_RANGE0__STCTRL_REGISTER_SAVE_BASE0_MASK 0x0000ffffL
+#define GMCON_STCTRL_REGISTER_SAVE_RANGE0__STCTRL_REGISTER_SAVE_BASE0__SHIFT 0x00000000
+#define GMCON_STCTRL_REGISTER_SAVE_RANGE0__STCTRL_REGISTER_SAVE_LIMIT0_MASK 0xffff0000L
+#define GMCON_STCTRL_REGISTER_SAVE_RANGE0__STCTRL_REGISTER_SAVE_LIMIT0__SHIFT 0x00000010
+#define GMCON_STCTRL_REGISTER_SAVE_RANGE1__STCTRL_REGISTER_SAVE_BASE1_MASK 0x0000ffffL
+#define GMCON_STCTRL_REGISTER_SAVE_RANGE1__STCTRL_REGISTER_SAVE_BASE1__SHIFT 0x00000000
+#define GMCON_STCTRL_REGISTER_SAVE_RANGE1__STCTRL_REGISTER_SAVE_LIMIT1_MASK 0xffff0000L
+#define GMCON_STCTRL_REGISTER_SAVE_RANGE1__STCTRL_REGISTER_SAVE_LIMIT1__SHIFT 0x00000010
+#define GMCON_STCTRL_REGISTER_SAVE_RANGE2__STCTRL_REGISTER_SAVE_BASE2_MASK 0x0000ffffL
+#define GMCON_STCTRL_REGISTER_SAVE_RANGE2__STCTRL_REGISTER_SAVE_BASE2__SHIFT 0x00000000
+#define GMCON_STCTRL_REGISTER_SAVE_RANGE2__STCTRL_REGISTER_SAVE_LIMIT2_MASK 0xffff0000L
+#define GMCON_STCTRL_REGISTER_SAVE_RANGE2__STCTRL_REGISTER_SAVE_LIMIT2__SHIFT 0x00000010
+#define MC_ARB_ADDR_HASH__BANK_XOR_ENABLE_MASK 0x0000000fL
+#define MC_ARB_ADDR_HASH__BANK_XOR_ENABLE__SHIFT 0x00000000
+#define MC_ARB_ADDR_HASH__COL_XOR_MASK 0x00000ff0L
+#define MC_ARB_ADDR_HASH__COL_XOR__SHIFT 0x00000004
+#define MC_ARB_ADDR_HASH__ROW_XOR_MASK 0x0ffff000L
+#define MC_ARB_ADDR_HASH__ROW_XOR__SHIFT 0x0000000c
+#define MC_ARB_AGE_RD__DIVIDE_GROUP0_MASK 0x01000000L
+#define MC_ARB_AGE_RD__DIVIDE_GROUP0__SHIFT 0x00000018
+#define MC_ARB_AGE_RD__DIVIDE_GROUP1_MASK 0x02000000L
+#define MC_ARB_AGE_RD__DIVIDE_GROUP1__SHIFT 0x00000019
+#define MC_ARB_AGE_RD__DIVIDE_GROUP2_MASK 0x04000000L
+#define MC_ARB_AGE_RD__DIVIDE_GROUP2__SHIFT 0x0000001a
+#define MC_ARB_AGE_RD__DIVIDE_GROUP3_MASK 0x08000000L
+#define MC_ARB_AGE_RD__DIVIDE_GROUP3__SHIFT 0x0000001b
+#define MC_ARB_AGE_RD__DIVIDE_GROUP4_MASK 0x10000000L
+#define MC_ARB_AGE_RD__DIVIDE_GROUP4__SHIFT 0x0000001c
+#define MC_ARB_AGE_RD__DIVIDE_GROUP5_MASK 0x20000000L
+#define MC_ARB_AGE_RD__DIVIDE_GROUP5__SHIFT 0x0000001d
+#define MC_ARB_AGE_RD__DIVIDE_GROUP6_MASK 0x40000000L
+#define MC_ARB_AGE_RD__DIVIDE_GROUP6__SHIFT 0x0000001e
+#define MC_ARB_AGE_RD__DIVIDE_GROUP7_MASK 0x80000000L
+#define MC_ARB_AGE_RD__DIVIDE_GROUP7__SHIFT 0x0000001f
+#define MC_ARB_AGE_RD__ENABLE_GROUP0_MASK 0x00010000L
+#define MC_ARB_AGE_RD__ENABLE_GROUP0__SHIFT 0x00000010
+#define MC_ARB_AGE_RD__ENABLE_GROUP1_MASK 0x00020000L
+#define MC_ARB_AGE_RD__ENABLE_GROUP1__SHIFT 0x00000011
+#define MC_ARB_AGE_RD__ENABLE_GROUP2_MASK 0x00040000L
+#define MC_ARB_AGE_RD__ENABLE_GROUP2__SHIFT 0x00000012
+#define MC_ARB_AGE_RD__ENABLE_GROUP3_MASK 0x00080000L
+#define MC_ARB_AGE_RD__ENABLE_GROUP3__SHIFT 0x00000013
+#define MC_ARB_AGE_RD__ENABLE_GROUP4_MASK 0x00100000L
+#define MC_ARB_AGE_RD__ENABLE_GROUP4__SHIFT 0x00000014
+#define MC_ARB_AGE_RD__ENABLE_GROUP5_MASK 0x00200000L
+#define MC_ARB_AGE_RD__ENABLE_GROUP5__SHIFT 0x00000015
+#define MC_ARB_AGE_RD__ENABLE_GROUP6_MASK 0x00400000L
+#define MC_ARB_AGE_RD__ENABLE_GROUP6__SHIFT 0x00000016
+#define MC_ARB_AGE_RD__ENABLE_GROUP7_MASK 0x00800000L
+#define MC_ARB_AGE_RD__ENABLE_GROUP7__SHIFT 0x00000017
+#define MC_ARB_AGE_RD__RATE_GROUP0_MASK 0x00000003L
+#define MC_ARB_AGE_RD__RATE_GROUP0__SHIFT 0x00000000
+#define MC_ARB_AGE_RD__RATE_GROUP1_MASK 0x0000000cL
+#define MC_ARB_AGE_RD__RATE_GROUP1__SHIFT 0x00000002
+#define MC_ARB_AGE_RD__RATE_GROUP2_MASK 0x00000030L
+#define MC_ARB_AGE_RD__RATE_GROUP2__SHIFT 0x00000004
+#define MC_ARB_AGE_RD__RATE_GROUP3_MASK 0x000000c0L
+#define MC_ARB_AGE_RD__RATE_GROUP3__SHIFT 0x00000006
+#define MC_ARB_AGE_RD__RATE_GROUP4_MASK 0x00000300L
+#define MC_ARB_AGE_RD__RATE_GROUP4__SHIFT 0x00000008
+#define MC_ARB_AGE_RD__RATE_GROUP5_MASK 0x00000c00L
+#define MC_ARB_AGE_RD__RATE_GROUP5__SHIFT 0x0000000a
+#define MC_ARB_AGE_RD__RATE_GROUP6_MASK 0x00003000L
+#define MC_ARB_AGE_RD__RATE_GROUP6__SHIFT 0x0000000c
+#define MC_ARB_AGE_RD__RATE_GROUP7_MASK 0x0000c000L
+#define MC_ARB_AGE_RD__RATE_GROUP7__SHIFT 0x0000000e
+#define MC_ARB_AGE_WR__DIVIDE_GROUP0_MASK 0x01000000L
+#define MC_ARB_AGE_WR__DIVIDE_GROUP0__SHIFT 0x00000018
+#define MC_ARB_AGE_WR__DIVIDE_GROUP1_MASK 0x02000000L
+#define MC_ARB_AGE_WR__DIVIDE_GROUP1__SHIFT 0x00000019
+#define MC_ARB_AGE_WR__DIVIDE_GROUP2_MASK 0x04000000L
+#define MC_ARB_AGE_WR__DIVIDE_GROUP2__SHIFT 0x0000001a
+#define MC_ARB_AGE_WR__DIVIDE_GROUP3_MASK 0x08000000L
+#define MC_ARB_AGE_WR__DIVIDE_GROUP3__SHIFT 0x0000001b
+#define MC_ARB_AGE_WR__DIVIDE_GROUP4_MASK 0x10000000L
+#define MC_ARB_AGE_WR__DIVIDE_GROUP4__SHIFT 0x0000001c
+#define MC_ARB_AGE_WR__DIVIDE_GROUP5_MASK 0x20000000L
+#define MC_ARB_AGE_WR__DIVIDE_GROUP5__SHIFT 0x0000001d
+#define MC_ARB_AGE_WR__DIVIDE_GROUP6_MASK 0x40000000L
+#define MC_ARB_AGE_WR__DIVIDE_GROUP6__SHIFT 0x0000001e
+#define MC_ARB_AGE_WR__DIVIDE_GROUP7_MASK 0x80000000L
+#define MC_ARB_AGE_WR__DIVIDE_GROUP7__SHIFT 0x0000001f
+#define MC_ARB_AGE_WR__ENABLE_GROUP0_MASK 0x00010000L
+#define MC_ARB_AGE_WR__ENABLE_GROUP0__SHIFT 0x00000010
+#define MC_ARB_AGE_WR__ENABLE_GROUP1_MASK 0x00020000L
+#define MC_ARB_AGE_WR__ENABLE_GROUP1__SHIFT 0x00000011
+#define MC_ARB_AGE_WR__ENABLE_GROUP2_MASK 0x00040000L
+#define MC_ARB_AGE_WR__ENABLE_GROUP2__SHIFT 0x00000012
+#define MC_ARB_AGE_WR__ENABLE_GROUP3_MASK 0x00080000L
+#define MC_ARB_AGE_WR__ENABLE_GROUP3__SHIFT 0x00000013
+#define MC_ARB_AGE_WR__ENABLE_GROUP4_MASK 0x00100000L
+#define MC_ARB_AGE_WR__ENABLE_GROUP4__SHIFT 0x00000014
+#define MC_ARB_AGE_WR__ENABLE_GROUP5_MASK 0x00200000L
+#define MC_ARB_AGE_WR__ENABLE_GROUP5__SHIFT 0x00000015
+#define MC_ARB_AGE_WR__ENABLE_GROUP6_MASK 0x00400000L
+#define MC_ARB_AGE_WR__ENABLE_GROUP6__SHIFT 0x00000016
+#define MC_ARB_AGE_WR__ENABLE_GROUP7_MASK 0x00800000L
+#define MC_ARB_AGE_WR__ENABLE_GROUP7__SHIFT 0x00000017
+#define MC_ARB_AGE_WR__RATE_GROUP0_MASK 0x00000003L
+#define MC_ARB_AGE_WR__RATE_GROUP0__SHIFT 0x00000000
+#define MC_ARB_AGE_WR__RATE_GROUP1_MASK 0x0000000cL
+#define MC_ARB_AGE_WR__RATE_GROUP1__SHIFT 0x00000002
+#define MC_ARB_AGE_WR__RATE_GROUP2_MASK 0x00000030L
+#define MC_ARB_AGE_WR__RATE_GROUP2__SHIFT 0x00000004
+#define MC_ARB_AGE_WR__RATE_GROUP3_MASK 0x000000c0L
+#define MC_ARB_AGE_WR__RATE_GROUP3__SHIFT 0x00000006
+#define MC_ARB_AGE_WR__RATE_GROUP4_MASK 0x00000300L
+#define MC_ARB_AGE_WR__RATE_GROUP4__SHIFT 0x00000008
+#define MC_ARB_AGE_WR__RATE_GROUP5_MASK 0x00000c00L
+#define MC_ARB_AGE_WR__RATE_GROUP5__SHIFT 0x0000000a
+#define MC_ARB_AGE_WR__RATE_GROUP6_MASK 0x00003000L
+#define MC_ARB_AGE_WR__RATE_GROUP6__SHIFT 0x0000000c
+#define MC_ARB_AGE_WR__RATE_GROUP7_MASK 0x0000c000L
+#define MC_ARB_AGE_WR__RATE_GROUP7__SHIFT 0x0000000e
+#define MC_ARB_BANKMAP__BANK0_MASK 0x0000000fL
+#define MC_ARB_BANKMAP__BANK0__SHIFT 0x00000000
+#define MC_ARB_BANKMAP__BANK1_MASK 0x000000f0L
+#define MC_ARB_BANKMAP__BANK1__SHIFT 0x00000004
+#define MC_ARB_BANKMAP__BANK2_MASK 0x00000f00L
+#define MC_ARB_BANKMAP__BANK2__SHIFT 0x00000008
+#define MC_ARB_BANKMAP__BANK3_MASK 0x0000f000L
+#define MC_ARB_BANKMAP__BANK3__SHIFT 0x0000000c
+#define MC_ARB_BANKMAP__RANK_MASK 0x000f0000L
+#define MC_ARB_BANKMAP__RANK__SHIFT 0x00000010
+#define MC_ARB_BURST_TIME__STATE0_MASK 0x0000001fL
+#define MC_ARB_BURST_TIME__STATE0__SHIFT 0x00000000
+#define MC_ARB_BURST_TIME__STATE1_MASK 0x000003e0L
+#define MC_ARB_BURST_TIME__STATE1__SHIFT 0x00000005
+#define MC_ARB_BURST_TIME__STATE2_MASK 0x00007c00L
+#define MC_ARB_BURST_TIME__STATE2__SHIFT 0x0000000a
+#define MC_ARB_BURST_TIME__STATE3_MASK 0x000f8000L
+#define MC_ARB_BURST_TIME__STATE3__SHIFT 0x0000000f
+#define MC_ARB_CAC_CNTL__ALLOW_OVERFLOW_MASK 0x00002000L
+#define MC_ARB_CAC_CNTL__ALLOW_OVERFLOW__SHIFT 0x0000000d
+#define MC_ARB_CAC_CNTL__ENABLE_MASK 0x00000001L
+#define MC_ARB_CAC_CNTL__ENABLE__SHIFT 0x00000000
+#define MC_ARB_CAC_CNTL__READ_WEIGHT_MASK 0x0000007eL
+#define MC_ARB_CAC_CNTL__READ_WEIGHT__SHIFT 0x00000001
+#define MC_ARB_CAC_CNTL__WRITE_WEIGHT_MASK 0x00001f80L
+#define MC_ARB_CAC_CNTL__WRITE_WEIGHT__SHIFT 0x00000007
+#define MC_ARB_CG__CG_ARB_REQ_MASK 0x000000ffL
+#define MC_ARB_CG__CG_ARB_REQ__SHIFT 0x00000000
+#define MC_ARB_CG__CG_ARB_RESP_MASK 0x0000ff00L
+#define MC_ARB_CG__CG_ARB_RESP__SHIFT 0x00000008
+#define MC_ARB_DRAM_TIMING_1__ACTRD_MASK 0x000000ffL
+#define MC_ARB_DRAM_TIMING_1__ACTRD__SHIFT 0x00000000
+#define MC_ARB_DRAM_TIMING_1__ACTWR_MASK 0x0000ff00L
+#define MC_ARB_DRAM_TIMING_1__ACTWR__SHIFT 0x00000008
+#define MC_ARB_DRAM_TIMING_1__RASMACTRD_MASK 0x00ff0000L
+#define MC_ARB_DRAM_TIMING_1__RASMACTRD__SHIFT 0x00000010
+#define MC_ARB_DRAM_TIMING_1__RASMACTWR_MASK 0xff000000L
+#define MC_ARB_DRAM_TIMING_1__RASMACTWR__SHIFT 0x00000018
+#define MC_ARB_DRAM_TIMING2_1__BUS_TURN_MASK 0x1f000000L
+#define MC_ARB_DRAM_TIMING2_1__BUS_TURN__SHIFT 0x00000018
+#define MC_ARB_DRAM_TIMING2_1__RAS2RAS_MASK 0x000000ffL
+#define MC_ARB_DRAM_TIMING2_1__RAS2RAS__SHIFT 0x00000000
+#define MC_ARB_DRAM_TIMING2_1__RP_MASK 0x0000ff00L
+#define MC_ARB_DRAM_TIMING2_1__RP__SHIFT 0x00000008
+#define MC_ARB_DRAM_TIMING2_1__WRPLUSRP_MASK 0x00ff0000L
+#define MC_ARB_DRAM_TIMING2_1__WRPLUSRP__SHIFT 0x00000010
+#define MC_ARB_DRAM_TIMING2__BUS_TURN_MASK 0x1f000000L
+#define MC_ARB_DRAM_TIMING2__BUS_TURN__SHIFT 0x00000018
+#define MC_ARB_DRAM_TIMING2__RAS2RAS_MASK 0x000000ffL
+#define MC_ARB_DRAM_TIMING2__RAS2RAS__SHIFT 0x00000000
+#define MC_ARB_DRAM_TIMING2__RP_MASK 0x0000ff00L
+#define MC_ARB_DRAM_TIMING2__RP__SHIFT 0x00000008
+#define MC_ARB_DRAM_TIMING2__WRPLUSRP_MASK 0x00ff0000L
+#define MC_ARB_DRAM_TIMING2__WRPLUSRP__SHIFT 0x00000010
+#define MC_ARB_DRAM_TIMING__ACTRD_MASK 0x000000ffL
+#define MC_ARB_DRAM_TIMING__ACTRD__SHIFT 0x00000000
+#define MC_ARB_DRAM_TIMING__ACTWR_MASK 0x0000ff00L
+#define MC_ARB_DRAM_TIMING__ACTWR__SHIFT 0x00000008
+#define MC_ARB_DRAM_TIMING__RASMACTRD_MASK 0x00ff0000L
+#define MC_ARB_DRAM_TIMING__RASMACTRD__SHIFT 0x00000010
+#define MC_ARB_DRAM_TIMING__RASMACTWR_MASK 0xff000000L
+#define MC_ARB_DRAM_TIMING__RASMACTWR__SHIFT 0x00000018
+#define MC_ARB_FED_CNTL__KEEP_POISON_IN_PAGE_MASK 0x00000010L
+#define MC_ARB_FED_CNTL__KEEP_POISON_IN_PAGE__SHIFT 0x00000004
+#define MC_ARB_FED_CNTL__MODE_MASK 0x00000003L
+#define MC_ARB_FED_CNTL__MODE__SHIFT 0x00000000
+#define MC_ARB_FED_CNTL__WR_ERR_MASK 0x0000000cL
+#define MC_ARB_FED_CNTL__WR_ERR__SHIFT 0x00000002
+#define MC_ARB_GDEC_RD_CNTL__PAGEBIT0_MASK 0x0000000fL
+#define MC_ARB_GDEC_RD_CNTL__PAGEBIT0__SHIFT 0x00000000
+#define MC_ARB_GDEC_RD_CNTL__PAGEBIT1_MASK 0x000000f0L
+#define MC_ARB_GDEC_RD_CNTL__PAGEBIT1__SHIFT 0x00000004
+#define MC_ARB_GDEC_RD_CNTL__REM_DEFAULT_GRP_MASK 0x00003c00L
+#define MC_ARB_GDEC_RD_CNTL__REM_DEFAULT_GRP__SHIFT 0x0000000a
+#define MC_ARB_GDEC_RD_CNTL__USE_RANK_MASK 0x00000100L
+#define MC_ARB_GDEC_RD_CNTL__USE_RANK__SHIFT 0x00000008
+#define MC_ARB_GDEC_RD_CNTL__USE_RSNO_MASK 0x00000200L
+#define MC_ARB_GDEC_RD_CNTL__USE_RSNO__SHIFT 0x00000009
+#define MC_ARB_GDEC_WR_CNTL__PAGEBIT0_MASK 0x0000000fL
+#define MC_ARB_GDEC_WR_CNTL__PAGEBIT0__SHIFT 0x00000000
+#define MC_ARB_GDEC_WR_CNTL__PAGEBIT1_MASK 0x000000f0L
+#define MC_ARB_GDEC_WR_CNTL__PAGEBIT1__SHIFT 0x00000004
+#define MC_ARB_GDEC_WR_CNTL__REM_DEFAULT_GRP_MASK 0x00003c00L
+#define MC_ARB_GDEC_WR_CNTL__REM_DEFAULT_GRP__SHIFT 0x0000000a
+#define MC_ARB_GDEC_WR_CNTL__USE_RANK_MASK 0x00000100L
+#define MC_ARB_GDEC_WR_CNTL__USE_RANK__SHIFT 0x00000008
+#define MC_ARB_GDEC_WR_CNTL__USE_RSNO_MASK 0x00000200L
+#define MC_ARB_GDEC_WR_CNTL__USE_RSNO__SHIFT 0x00000009
+#define MC_ARB_GECC2_CLI__NO_GECC_CLI0_MASK 0x000000ffL
+#define MC_ARB_GECC2_CLI__NO_GECC_CLI0__SHIFT 0x00000000
+#define MC_ARB_GECC2_CLI__NO_GECC_CLI1_MASK 0x0000ff00L
+#define MC_ARB_GECC2_CLI__NO_GECC_CLI1__SHIFT 0x00000008
+#define MC_ARB_GECC2_CLI__NO_GECC_CLI2_MASK 0x00ff0000L
+#define MC_ARB_GECC2_CLI__NO_GECC_CLI2__SHIFT 0x00000010
+#define MC_ARB_GECC2_CLI__NO_GECC_CLI3_MASK 0xff000000L
+#define MC_ARB_GECC2_CLI__NO_GECC_CLI3__SHIFT 0x00000018
+#define MC_ARB_GECC2__CLOSE_BANK_RMW_MASK 0x00004000L
+#define MC_ARB_GECC2__CLOSE_BANK_RMW__SHIFT 0x0000000e
+#define MC_ARB_GECC2__COLFIFO_WATER_MASK 0x001f8000L
+#define MC_ARB_GECC2__COLFIFO_WATER__SHIFT 0x0000000f
+#define MC_ARB_GECC2_DEBUG2__ERR0_START_MASK 0x0000ff00L
+#define MC_ARB_GECC2_DEBUG2__ERR0_START__SHIFT 0x00000008
+#define MC_ARB_GECC2_DEBUG2__ERR1_START_MASK 0x00ff0000L
+#define MC_ARB_GECC2_DEBUG2__ERR1_START__SHIFT 0x00000010
+#define MC_ARB_GECC2_DEBUG2__ERR2_START_MASK 0xff000000L
+#define MC_ARB_GECC2_DEBUG2__ERR2_START__SHIFT 0x00000018
+#define MC_ARB_GECC2_DEBUG2__PERIOD_MASK 0x000000ffL
+#define MC_ARB_GECC2_DEBUG2__PERIOD__SHIFT 0x00000000
+#define MC_ARB_GECC2_DEBUG__DATA_FIELD_MASK 0x00000018L
+#define MC_ARB_GECC2_DEBUG__DATA_FIELD__SHIFT 0x00000003
+#define MC_ARB_GECC2_DEBUG__DIRECTION_MASK 0x00000004L
+#define MC_ARB_GECC2_DEBUG__DIRECTION__SHIFT 0x00000002
+#define MC_ARB_GECC2_DEBUG__NUM_ERR_BITS_MASK 0x00000003L
+#define MC_ARB_GECC2_DEBUG__NUM_ERR_BITS__SHIFT 0x00000000
+#define MC_ARB_GECC2_DEBUG__SW_INJECTION_MASK 0x00000020L
+#define MC_ARB_GECC2_DEBUG__SW_INJECTION__SHIFT 0x00000005
+#define MC_ARB_GECC2__ECC_MODE_MASK 0x00000006L
+#define MC_ARB_GECC2__ECC_MODE__SHIFT 0x00000001
+#define MC_ARB_GECC2__ENABLE_MASK 0x00000001L
+#define MC_ARB_GECC2__ENABLE__SHIFT 0x00000000
+#define MC_ARB_GECC2__EXOR_BANK_SEL_MASK 0x00000060L
+#define MC_ARB_GECC2__EXOR_BANK_SEL__SHIFT 0x00000005
+#define MC_ARB_GECC2_MISC__STREAK_BREAK_MASK 0x0000000fL
+#define MC_ARB_GECC2_MISC__STREAK_BREAK__SHIFT 0x00000000
+#define MC_ARB_GECC2__NO_GECC_CLI_MASK 0x00000780L
+#define MC_ARB_GECC2__NO_GECC_CLI__SHIFT 0x00000007
+#define MC_ARB_GECC2__PAGE_BIT0_MASK 0x00000018L
+#define MC_ARB_GECC2__PAGE_BIT0__SHIFT 0x00000003
+#define MC_ARB_GECC2__READ_ERR_MASK 0x00003800L
+#define MC_ARB_GECC2__READ_ERR__SHIFT 0x0000000b
+#define MC_ARB_GECC2_STATUS__CORR_CLEAR0_MASK 0x00000100L
+#define MC_ARB_GECC2_STATUS__CORR_CLEAR0__SHIFT 0x00000008
+#define MC_ARB_GECC2_STATUS__CORR_CLEAR1_MASK 0x00001000L
+#define MC_ARB_GECC2_STATUS__CORR_CLEAR1__SHIFT 0x0000000c
+#define MC_ARB_GECC2_STATUS__CORR_STS0_MASK 0x00000001L
+#define MC_ARB_GECC2_STATUS__CORR_STS0__SHIFT 0x00000000
+#define MC_ARB_GECC2_STATUS__CORR_STS1_MASK 0x00000010L
+#define MC_ARB_GECC2_STATUS__CORR_STS1__SHIFT 0x00000004
+#define MC_ARB_GECC2_STATUS__FED_CLEAR0_MASK 0x00000400L
+#define MC_ARB_GECC2_STATUS__FED_CLEAR0__SHIFT 0x0000000a
+#define MC_ARB_GECC2_STATUS__FED_CLEAR1_MASK 0x00004000L
+#define MC_ARB_GECC2_STATUS__FED_CLEAR1__SHIFT 0x0000000e
+#define MC_ARB_GECC2_STATUS__FED_STS0_MASK 0x00000004L
+#define MC_ARB_GECC2_STATUS__FED_STS0__SHIFT 0x00000002
+#define MC_ARB_GECC2_STATUS__FED_STS1_MASK 0x00000040L
+#define MC_ARB_GECC2_STATUS__FED_STS1__SHIFT 0x00000006
+#define MC_ARB_GECC2_STATUS__RSVD0_MASK 0x00000008L
+#define MC_ARB_GECC2_STATUS__RSVD0__SHIFT 0x00000003
+#define MC_ARB_GECC2_STATUS__RSVD1_MASK 0x00000080L
+#define MC_ARB_GECC2_STATUS__RSVD1__SHIFT 0x00000007
+#define MC_ARB_GECC2_STATUS__RSVD2_MASK 0x00000800L
+#define MC_ARB_GECC2_STATUS__RSVD2__SHIFT 0x0000000b
+#define MC_ARB_GECC2_STATUS__UNCORR_CLEAR0_MASK 0x00000200L
+#define MC_ARB_GECC2_STATUS__UNCORR_CLEAR0__SHIFT 0x00000009
+#define MC_ARB_GECC2_STATUS__UNCORR_CLEAR1_MASK 0x00002000L
+#define MC_ARB_GECC2_STATUS__UNCORR_CLEAR1__SHIFT 0x0000000d
+#define MC_ARB_GECC2_STATUS__UNCORR_STS0_MASK 0x00000002L
+#define MC_ARB_GECC2_STATUS__UNCORR_STS0__SHIFT 0x00000001
+#define MC_ARB_GECC2_STATUS__UNCORR_STS1_MASK 0x00000020L
+#define MC_ARB_GECC2_STATUS__UNCORR_STS1__SHIFT 0x00000005
+#define MC_ARB_LAZY0_RD__GROUP0_MASK 0x000000ffL
+#define MC_ARB_LAZY0_RD__GROUP0__SHIFT 0x00000000
+#define MC_ARB_LAZY0_RD__GROUP1_MASK 0x0000ff00L
+#define MC_ARB_LAZY0_RD__GROUP1__SHIFT 0x00000008
+#define MC_ARB_LAZY0_RD__GROUP2_MASK 0x00ff0000L
+#define MC_ARB_LAZY0_RD__GROUP2__SHIFT 0x00000010
+#define MC_ARB_LAZY0_RD__GROUP3_MASK 0xff000000L
+#define MC_ARB_LAZY0_RD__GROUP3__SHIFT 0x00000018
+#define MC_ARB_LAZY0_WR__GROUP0_MASK 0x000000ffL
+#define MC_ARB_LAZY0_WR__GROUP0__SHIFT 0x00000000
+#define MC_ARB_LAZY0_WR__GROUP1_MASK 0x0000ff00L
+#define MC_ARB_LAZY0_WR__GROUP1__SHIFT 0x00000008
+#define MC_ARB_LAZY0_WR__GROUP2_MASK 0x00ff0000L
+#define MC_ARB_LAZY0_WR__GROUP2__SHIFT 0x00000010
+#define MC_ARB_LAZY0_WR__GROUP3_MASK 0xff000000L
+#define MC_ARB_LAZY0_WR__GROUP3__SHIFT 0x00000018
+#define MC_ARB_LAZY1_RD__GROUP4_MASK 0x000000ffL
+#define MC_ARB_LAZY1_RD__GROUP4__SHIFT 0x00000000
+#define MC_ARB_LAZY1_RD__GROUP5_MASK 0x0000ff00L
+#define MC_ARB_LAZY1_RD__GROUP5__SHIFT 0x00000008
+#define MC_ARB_LAZY1_RD__GROUP6_MASK 0x00ff0000L
+#define MC_ARB_LAZY1_RD__GROUP6__SHIFT 0x00000010
+#define MC_ARB_LAZY1_RD__GROUP7_MASK 0xff000000L
+#define MC_ARB_LAZY1_RD__GROUP7__SHIFT 0x00000018
+#define MC_ARB_LAZY1_WR__GROUP4_MASK 0x000000ffL
+#define MC_ARB_LAZY1_WR__GROUP4__SHIFT 0x00000000
+#define MC_ARB_LAZY1_WR__GROUP5_MASK 0x0000ff00L
+#define MC_ARB_LAZY1_WR__GROUP5__SHIFT 0x00000008
+#define MC_ARB_LAZY1_WR__GROUP6_MASK 0x00ff0000L
+#define MC_ARB_LAZY1_WR__GROUP6__SHIFT 0x00000010
+#define MC_ARB_LAZY1_WR__GROUP7_MASK 0xff000000L
+#define MC_ARB_LAZY1_WR__GROUP7__SHIFT 0x00000018
+#define MC_ARB_LM_RD__BANKGROUP_CONFIG_MASK 0x00e00000L
+#define MC_ARB_LM_RD__BANKGROUP_CONFIG__SHIFT 0x00000015
+#define MC_ARB_LM_RD__ENABLE_TWO_LIST_MASK 0x00040000L
+#define MC_ARB_LM_RD__ENABLE_TWO_LIST__SHIFT 0x00000012
+#define MC_ARB_LM_RD__POPIDLE_RST_TWOLIST_MASK 0x00080000L
+#define MC_ARB_LM_RD__POPIDLE_RST_TWOLIST__SHIFT 0x00000013
+#define MC_ARB_LM_RD__SKID1_RST_TWOLIST_MASK 0x00100000L
+#define MC_ARB_LM_RD__SKID1_RST_TWOLIST__SHIFT 0x00000014
+#define MC_ARB_LM_RD__STREAK_BREAK_MASK 0x00010000L
+#define MC_ARB_LM_RD__STREAK_BREAK__SHIFT 0x00000010
+#define MC_ARB_LM_RD__STREAK_LIMIT_MASK 0x000000ffL
+#define MC_ARB_LM_RD__STREAK_LIMIT__SHIFT 0x00000000
+#define MC_ARB_LM_RD__STREAK_LIMIT_UBER_MASK 0x0000ff00L
+#define MC_ARB_LM_RD__STREAK_LIMIT_UBER__SHIFT 0x00000008
+#define MC_ARB_LM_RD__STREAK_UBER_MASK 0x00020000L
+#define MC_ARB_LM_RD__STREAK_UBER__SHIFT 0x00000011
+#define MC_ARB_LM_WR__BANKGROUP_CONFIG_MASK 0x00e00000L
+#define MC_ARB_LM_WR__BANKGROUP_CONFIG__SHIFT 0x00000015
+#define MC_ARB_LM_WR__ENABLE_TWO_LIST_MASK 0x00040000L
+#define MC_ARB_LM_WR__ENABLE_TWO_LIST__SHIFT 0x00000012
+#define MC_ARB_LM_WR__POPIDLE_RST_TWOLIST_MASK 0x00080000L
+#define MC_ARB_LM_WR__POPIDLE_RST_TWOLIST__SHIFT 0x00000013
+#define MC_ARB_LM_WR__SKID1_RST_TWOLIST_MASK 0x00100000L
+#define MC_ARB_LM_WR__SKID1_RST_TWOLIST__SHIFT 0x00000014
+#define MC_ARB_LM_WR__STREAK_BREAK_MASK 0x00010000L
+#define MC_ARB_LM_WR__STREAK_BREAK__SHIFT 0x00000010
+#define MC_ARB_LM_WR__STREAK_LIMIT_MASK 0x000000ffL
+#define MC_ARB_LM_WR__STREAK_LIMIT__SHIFT 0x00000000
+#define MC_ARB_LM_WR__STREAK_LIMIT_UBER_MASK 0x0000ff00L
+#define MC_ARB_LM_WR__STREAK_LIMIT_UBER__SHIFT 0x00000008
+#define MC_ARB_LM_WR__STREAK_UBER_MASK 0x00020000L
+#define MC_ARB_LM_WR__STREAK_UBER__SHIFT 0x00000011
+#define MC_ARB_MINCLKS__ARB_RW_SWITCH_MASK 0x00010000L
+#define MC_ARB_MINCLKS__ARB_RW_SWITCH__SHIFT 0x00000010
+#define MC_ARB_MINCLKS__READ_CLKS_MASK 0x000000ffL
+#define MC_ARB_MINCLKS__READ_CLKS__SHIFT 0x00000000
+#define MC_ARB_MINCLKS__WRITE_CLKS_MASK 0x0000ff00L
+#define MC_ARB_MINCLKS__WRITE_CLKS__SHIFT 0x00000008
+#define MC_ARB_MISC2__ARB_DEBUG29_MASK 0x20000000L
+#define MC_ARB_MISC2__ARB_DEBUG29__SHIFT 0x0000001d
+#define MC_ARB_MISC2__GECC_MASK 0x00040000L
+#define MC_ARB_MISC2__GECC_RST_MASK 0x00080000L
+#define MC_ARB_MISC2__GECC_RST__SHIFT 0x00000013
+#define MC_ARB_MISC2__GECC__SHIFT 0x00000012
+#define MC_ARB_MISC2__GECC_STATUS_MASK 0x00100000L
+#define MC_ARB_MISC2__GECC_STATUS__SHIFT 0x00000014
+#define MC_ARB_MISC2__POP_IDLE_REPLAY_MASK 0x00000800L
+#define MC_ARB_MISC2__POP_IDLE_REPLAY__SHIFT 0x0000000b
+#define MC_ARB_MISC2__RDRET_NO_BP_MASK 0x00002000L
+#define MC_ARB_MISC2__RDRET_NO_BP__SHIFT 0x0000000d
+#define MC_ARB_MISC2__RDRET_NO_REORDERING_MASK 0x00001000L
+#define MC_ARB_MISC2__RDRET_NO_REORDERING__SHIFT 0x0000000c
+#define MC_ARB_MISC2__RDRET_SEQ_SKID_MASK 0x0003c000L
+#define MC_ARB_MISC2__RDRET_SEQ_SKID__SHIFT 0x0000000e
+#define MC_ARB_MISC2__REPLAY_DEBUG_MASK 0x10000000L
+#define MC_ARB_MISC2__REPLAY_DEBUG__SHIFT 0x0000001c
+#define MC_ARB_MISC2__SEQ_RDY_POP_IDLE_MASK 0x40000000L
+#define MC_ARB_MISC2__SEQ_RDY_POP_IDLE__SHIFT 0x0000001e
+#define MC_ARB_MISC2__TAGFIFO_THRESHOLD_MASK 0x01e00000L
+#define MC_ARB_MISC2__TAGFIFO_THRESHOLD__SHIFT 0x00000015
+#define MC_ARB_MISC2__TCCDL4_BANKBIT3_XOR_COLBIT4_MASK 0x00000040L
+#define MC_ARB_MISC2__TCCDL4_BANKBIT3_XOR_COLBIT4__SHIFT 0x00000006
+#define MC_ARB_MISC2__TCCDL4_BANKBIT3_XOR_COLBIT5_MASK 0x00000080L
+#define MC_ARB_MISC2__TCCDL4_BANKBIT3_XOR_COLBIT5__SHIFT 0x00000007
+#define MC_ARB_MISC2__TCCDL4_BANKBIT3_XOR_COLBIT6_MASK 0x00000100L
+#define MC_ARB_MISC2__TCCDL4_BANKBIT3_XOR_COLBIT6__SHIFT 0x00000008
+#define MC_ARB_MISC2__TCCDL4_BANKBIT3_XOR_COLBIT7_MASK 0x00000200L
+#define MC_ARB_MISC2__TCCDL4_BANKBIT3_XOR_COLBIT7__SHIFT 0x00000009
+#define MC_ARB_MISC2__TCCDL4_BANKBIT3_XOR_COLBIT8_MASK 0x00000400L
+#define MC_ARB_MISC2__TCCDL4_BANKBIT3_XOR_COLBIT8__SHIFT 0x0000000a
+#define MC_ARB_MISC2__TCCDL4_BANKBIT3_XOR_ENABLE_MASK 0x00000020L
+#define MC_ARB_MISC2__TCCDL4_BANKBIT3_XOR_ENABLE__SHIFT 0x00000005
+#define MC_ARB_MISC2__TCCDL4_REPLAY_EOB_MASK 0x80000000L
+#define MC_ARB_MISC2__TCCDL4_REPLAY_EOB__SHIFT 0x0000001f
+#define MC_ARB_MISC2__WCDR_REPLAY_MASKCNT_MASK 0x0e000000L
+#define MC_ARB_MISC2__WCDR_REPLAY_MASKCNT__SHIFT 0x00000019
+#define MC_ARB_MISC__CALI_ENABLE_MASK 0x00100000L
+#define MC_ARB_MISC__CALI_ENABLE__SHIFT 0x00000014
+#define MC_ARB_MISC__CALI_RATES_MASK 0x00600000L
+#define MC_ARB_MISC__CALI_RATES__SHIFT 0x00000015
+#define MC_ARB_MISC__CHAN_COUPLE_MASK 0x000007f8L
+#define MC_ARB_MISC__CHAN_COUPLE__SHIFT 0x00000003
+#define MC_ARB_MISC__DISPURG_NOSW2WR_MASK 0x01000000L
+#define MC_ARB_MISC__DISPURG_NOSW2WR__SHIFT 0x00000018
+#define MC_ARB_MISC__DISPURG_STALL_MASK 0x02000000L
+#define MC_ARB_MISC__DISPURG_STALL__SHIFT 0x00000019
+#define MC_ARB_MISC__DISPURG_THROTTLE_MASK 0x3c000000L
+#define MC_ARB_MISC__DISPURG_THROTTLE__SHIFT 0x0000001a
+#define MC_ARB_MISC__DISPURGVLD_NOWRT_MASK 0x00800000L
+#define MC_ARB_MISC__DISPURGVLD_NOWRT__SHIFT 0x00000017
+#define MC_ARB_MISC__HARSHNESS_MASK 0x0007f800L
+#define MC_ARB_MISC__HARSHNESS__SHIFT 0x0000000b
+#define MC_ARB_MISC__IDLE_RFSH_MASK 0x00000002L
+#define MC_ARB_MISC__IDLE_RFSH__SHIFT 0x00000001
+#define MC_ARB_MISC__SMART_RDWR_SW_MASK 0x00080000L
+#define MC_ARB_MISC__SMART_RDWR_SW__SHIFT 0x00000013
+#define MC_ARB_MISC__STICKY_RFSH_MASK 0x00000001L
+#define MC_ARB_MISC__STICKY_RFSH__SHIFT 0x00000000
+#define MC_ARB_MISC__STUTTER_RFSH_MASK 0x00000004L
+#define MC_ARB_MISC__STUTTER_RFSH__SHIFT 0x00000002
+#define MC_ARB_PM_CNTL__BLKOUT_ON_D1_MASK 0x00000020L
+#define MC_ARB_PM_CNTL__BLKOUT_ON_D1__SHIFT 0x00000005
+#define MC_ARB_PM_CNTL__IDLE_CNT_MASK 0x00f00000L
+#define MC_ARB_PM_CNTL__IDLE_CNT__SHIFT 0x00000014
+#define MC_ARB_PM_CNTL__IDLE_ON_D1_MASK 0x00000040L
+#define MC_ARB_PM_CNTL__IDLE_ON_D1__SHIFT 0x00000006
+#define MC_ARB_PM_CNTL__IDLE_ON_D2_MASK 0x00040000L
+#define MC_ARB_PM_CNTL__IDLE_ON_D2__SHIFT 0x00000012
+#define MC_ARB_PM_CNTL__IDLE_ON_D3_MASK 0x00080000L
+#define MC_ARB_PM_CNTL__IDLE_ON_D3__SHIFT 0x00000013
+#define MC_ARB_PM_CNTL__OVERRIDE_CGSTATE_MASK 0x00000003L
+#define MC_ARB_PM_CNTL__OVERRIDE_CGSTATE__SHIFT 0x00000000
+#define MC_ARB_PM_CNTL__OVRR_CGRFSH_MASK 0x00000004L
+#define MC_ARB_PM_CNTL__OVRR_CGRFSH__SHIFT 0x00000002
+#define MC_ARB_PM_CNTL__OVRR_CGSQM_MASK 0x00000008L
+#define MC_ARB_PM_CNTL__OVRR_CGSQM__SHIFT 0x00000003
+#define MC_ARB_PM_CNTL__OVRR_PM_MASK 0x00000080L
+#define MC_ARB_PM_CNTL__OVRR_PM__SHIFT 0x00000007
+#define MC_ARB_PM_CNTL__OVRR_PM_STATE_MASK 0x00000300L
+#define MC_ARB_PM_CNTL__OVRR_PM_STATE__SHIFT 0x00000008
+#define MC_ARB_PM_CNTL__OVRR_RD_MASK 0x00000400L
+#define MC_ARB_PM_CNTL__OVRR_RD__SHIFT 0x0000000a
+#define MC_ARB_PM_CNTL__OVRR_RD_STATE_MASK 0x00000800L
+#define MC_ARB_PM_CNTL__OVRR_RD_STATE__SHIFT 0x0000000b
+#define MC_ARB_PM_CNTL__OVRR_RFSH_MASK 0x00004000L
+#define MC_ARB_PM_CNTL__OVRR_RFSH__SHIFT 0x0000000e
+#define MC_ARB_PM_CNTL__OVRR_RFSH_STATE_MASK 0x00008000L
+#define MC_ARB_PM_CNTL__OVRR_RFSH_STATE__SHIFT 0x0000000f
+#define MC_ARB_PM_CNTL__OVRR_WR_MASK 0x00001000L
+#define MC_ARB_PM_CNTL__OVRR_WR__SHIFT 0x0000000c
+#define MC_ARB_PM_CNTL__OVRR_WR_STATE_MASK 0x00002000L
+#define MC_ARB_PM_CNTL__OVRR_WR_STATE__SHIFT 0x0000000d
+#define MC_ARB_PM_CNTL__SRFSH_ON_D1_MASK 0x00000010L
+#define MC_ARB_PM_CNTL__SRFSH_ON_D1__SHIFT 0x00000004
+#define MC_ARB_POP__ALLOW_EOB_BY_WRRET_STALL_MASK 0x00080000L
+#define MC_ARB_POP__ALLOW_EOB_BY_WRRET_STALL__SHIFT 0x00000013
+#define MC_ARB_POP__ENABLE_ARB_MASK 0x00000001L
+#define MC_ARB_POP__ENABLE_ARB__SHIFT 0x00000000
+#define MC_ARB_POP__ENABLE_TWO_PAGE_MASK 0x00040000L
+#define MC_ARB_POP__ENABLE_TWO_PAGE__SHIFT 0x00000012
+#define MC_ARB_POP__POP_DEPTH_MASK 0x0000003cL
+#define MC_ARB_POP__POP_DEPTH__SHIFT 0x00000002
+#define MC_ARB_POP__QUICK_STOP_MASK 0x00020000L
+#define MC_ARB_POP__QUICK_STOP__SHIFT 0x00000011
+#define MC_ARB_POP__SKID_DEPTH_MASK 0x00007000L
+#define MC_ARB_POP__SKID_DEPTH__SHIFT 0x0000000c
+#define MC_ARB_POP__SPEC_OPEN_MASK 0x00000002L
+#define MC_ARB_POP__SPEC_OPEN__SHIFT 0x00000001
+#define MC_ARB_POP__WAIT_AFTER_RFSH_MASK 0x00018000L
+#define MC_ARB_POP__WAIT_AFTER_RFSH__SHIFT 0x0000000f
+#define MC_ARB_POP__WRDATAINDEX_DEPTH_MASK 0x00000fc0L
+#define MC_ARB_POP__WRDATAINDEX_DEPTH__SHIFT 0x00000006
+#define MC_ARB_RAMCFG__CHANSIZE_MASK 0x00000100L
+#define MC_ARB_RAMCFG__CHANSIZE__SHIFT 0x00000008
+#define MC_ARB_RAMCFG__NOOFBANK_MASK 0x00000003L
+#define MC_ARB_RAMCFG__NOOFBANK__SHIFT 0x00000000
+#define MC_ARB_RAMCFG__NOOFCOLS_MASK 0x000000c0L
+#define MC_ARB_RAMCFG__NOOFCOLS__SHIFT 0x00000006
+#define MC_ARB_RAMCFG__NOOFGROUPS_MASK 0x00001000L
+#define MC_ARB_RAMCFG__NOOFGROUPS__SHIFT 0x0000000c
+#define MC_ARB_RAMCFG__NOOFRANKS_MASK 0x00000004L
+#define MC_ARB_RAMCFG__NOOFRANKS__SHIFT 0x00000002
+#define MC_ARB_RAMCFG__NOOFROWS_MASK 0x00000038L
+#define MC_ARB_RAMCFG__NOOFROWS__SHIFT 0x00000003
+#define MC_ARB_REMREQ__RD_WATER_MASK 0x000000ffL
+#define MC_ARB_REMREQ__RD_WATER__SHIFT 0x00000000
+#define MC_ARB_REMREQ__WR_LAZY_TIMER_MASK 0x00f00000L
+#define MC_ARB_REMREQ__WR_LAZY_TIMER__SHIFT 0x00000014
+#define MC_ARB_REMREQ__WR_MAXBURST_SIZE_MASK 0x000f0000L
+#define MC_ARB_REMREQ__WR_MAXBURST_SIZE__SHIFT 0x00000010
+#define MC_ARB_REMREQ__WR_WATER_MASK 0x0000ff00L
+#define MC_ARB_REMREQ__WR_WATER__SHIFT 0x00000008
+#define MC_ARB_REPLAY__BOS_ENABLE_WAIT_CYC_MASK 0x00000080L
+#define MC_ARB_REPLAY__BOS_ENABLE_WAIT_CYC__SHIFT 0x00000007
+#define MC_ARB_REPLAY__BOS_WAIT_CYC_MASK 0x00007f00L
+#define MC_ARB_REPLAY__BOS_WAIT_CYC__SHIFT 0x00000008
+#define MC_ARB_REPLAY__BREAK_ON_STALL_MASK 0x00000040L
+#define MC_ARB_REPLAY__BREAK_ON_STALL__SHIFT 0x00000006
+#define MC_ARB_REPLAY__ENABLE_RD_MASK 0x00000001L
+#define MC_ARB_REPLAY__ENABLE_RD__SHIFT 0x00000000
+#define MC_ARB_REPLAY__ENABLE_WR_MASK 0x00000002L
+#define MC_ARB_REPLAY__ENABLE_WR__SHIFT 0x00000001
+#define MC_ARB_REPLAY__IGNORE_WR_CDC_MASK 0x00000020L
+#define MC_ARB_REPLAY__IGNORE_WR_CDC__SHIFT 0x00000005
+#define MC_ARB_REPLAY__RAW_ENABLE_MASK 0x00000010L
+#define MC_ARB_REPLAY__RAW_ENABLE__SHIFT 0x00000004
+#define MC_ARB_REPLAY__WAW_ENABLE_MASK 0x00000008L
+#define MC_ARB_REPLAY__WAW_ENABLE__SHIFT 0x00000003
+#define MC_ARB_REPLAY__WRACK_MODE_MASK 0x00000004L
+#define MC_ARB_REPLAY__WRACK_MODE__SHIFT 0x00000002
+#define MC_ARB_RET_CREDITS_RD__DISP_MASK 0x00ff0000L
+#define MC_ARB_RET_CREDITS_RD__DISP__SHIFT 0x00000010
+#define MC_ARB_RET_CREDITS_RD__HUB_MASK 0x0000ff00L
+#define MC_ARB_RET_CREDITS_RD__HUB__SHIFT 0x00000008
+#define MC_ARB_RET_CREDITS_RD__LCL_MASK 0x000000ffL
+#define MC_ARB_RET_CREDITS_RD__LCL__SHIFT 0x00000000
+#define MC_ARB_RET_CREDITS_RD__RETURN_CREDIT_MASK 0xff000000L
+#define MC_ARB_RET_CREDITS_RD__RETURN_CREDIT__SHIFT 0x00000018
+#define MC_ARB_RET_CREDITS_WR__HUB_MASK 0x0000ff00L
+#define MC_ARB_RET_CREDITS_WR__HUB__SHIFT 0x00000008
+#define MC_ARB_RET_CREDITS_WR__LCL_MASK 0x000000ffL
+#define MC_ARB_RET_CREDITS_WR__LCL__SHIFT 0x00000000
+#define MC_ARB_RET_CREDITS_WR__RETURN_CREDIT_MASK 0x00ff0000L
+#define MC_ARB_RET_CREDITS_WR__RETURN_CREDIT__SHIFT 0x00000010
+#define MC_ARB_RET_CREDITS_WR__WRRET_SEQ_SKID_MASK 0x0f000000L
+#define MC_ARB_RET_CREDITS_WR__WRRET_SEQ_SKID__SHIFT 0x00000018
+#define MC_ARB_RFSH_CNTL__ACCUM_MASK 0x00000800L
+#define MC_ARB_RFSH_CNTL__ACCUM__SHIFT 0x0000000b
+#define MC_ARB_RFSH_CNTL__ENABLE_MASK 0x00000001L
+#define MC_ARB_RFSH_CNTL__ENABLE__SHIFT 0x00000000
+#define MC_ARB_RFSH_CNTL__URG0_MASK 0x0000003eL
+#define MC_ARB_RFSH_CNTL__URG0__SHIFT 0x00000001
+#define MC_ARB_RFSH_CNTL__URG1_MASK 0x000007c0L
+#define MC_ARB_RFSH_CNTL__URG1__SHIFT 0x00000006
+#define MC_ARB_RFSH_RATE__POWERMODE0_MASK 0x000000ffL
+#define MC_ARB_RFSH_RATE__POWERMODE0__SHIFT 0x00000000
+#define MC_ARB_RTT_CNTL0__BREAK_ON_HARSH_MASK 0x00000100L
+#define MC_ARB_RTT_CNTL0__BREAK_ON_HARSH__SHIFT 0x00000008
+#define MC_ARB_RTT_CNTL0__BREAK_ON_URGENTRD_MASK 0x00000200L
+#define MC_ARB_RTT_CNTL0__BREAK_ON_URGENTRD__SHIFT 0x00000009
+#define MC_ARB_RTT_CNTL0__BREAK_ON_URGENTWR_MASK 0x00000400L
+#define MC_ARB_RTT_CNTL0__BREAK_ON_URGENTWR__SHIFT 0x0000000a
+#define MC_ARB_RTT_CNTL0__DATA_CNTL_MASK 0x01000000L
+#define MC_ARB_RTT_CNTL0__DATA_CNTL__SHIFT 0x00000018
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_0_MASK 0x00008000L
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_0__SHIFT 0x0000000f
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_1_MASK 0x00010000L
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_1__SHIFT 0x00000010
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_2_MASK 0x00020000L
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_2__SHIFT 0x00000011
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_3_MASK 0x00040000L
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_3__SHIFT 0x00000012
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_4_MASK 0x00080000L
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_4__SHIFT 0x00000013
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_5_MASK 0x00100000L
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_5__SHIFT 0x00000014
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_6_MASK 0x00200000L
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_6__SHIFT 0x00000015
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_7_MASK 0x00400000L
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_7__SHIFT 0x00000016
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_8_MASK 0x00800000L
+#define MC_ARB_RTT_CNTL0__DEBUG_RSV_8__SHIFT 0x00000017
+#define MC_ARB_RTT_CNTL0__ENABLE_MASK 0x00000001L
+#define MC_ARB_RTT_CNTL0__ENABLE__SHIFT 0x00000000
+#define MC_ARB_RTT_CNTL0__FLUSH_ON_ENTER_MASK 0x00000010L
+#define MC_ARB_RTT_CNTL0__FLUSH_ON_ENTER__SHIFT 0x00000004
+#define MC_ARB_RTT_CNTL0__HARSH_START_MASK 0x00000020L
+#define MC_ARB_RTT_CNTL0__HARSH_START__SHIFT 0x00000005
+#define MC_ARB_RTT_CNTL0__NEIGHBOR_BIT_MASK 0x02000000L
+#define MC_ARB_RTT_CNTL0__NEIGHBOR_BIT__SHIFT 0x00000019
+#define MC_ARB_RTT_CNTL0__START_IDLE_MASK 0x00000002L
+#define MC_ARB_RTT_CNTL0__START_IDLE__SHIFT 0x00000001
+#define MC_ARB_RTT_CNTL0__START_R2W_MASK 0x0000000cL
+#define MC_ARB_RTT_CNTL0__START_R2W_RFSH_MASK 0x00004000L
+#define MC_ARB_RTT_CNTL0__START_R2W_RFSH__SHIFT 0x0000000e
+#define MC_ARB_RTT_CNTL0__START_R2W__SHIFT 0x00000002
+#define MC_ARB_RTT_CNTL0__TPS_HARSH_PRIORITY_MASK 0x00000040L
+#define MC_ARB_RTT_CNTL0__TPS_HARSH_PRIORITY__SHIFT 0x00000006
+#define MC_ARB_RTT_CNTL0__TRAIN_PERIOD_MASK 0x00003800L
+#define MC_ARB_RTT_CNTL0__TRAIN_PERIOD__SHIFT 0x0000000b
+#define MC_ARB_RTT_CNTL0__TWRT_HARSH_PRIORITY_MASK 0x00000080L
+#define MC_ARB_RTT_CNTL0__TWRT_HARSH_PRIORITY__SHIFT 0x00000007
+#define MC_ARB_RTT_CNTL1__WINDOW_DEC_THRESHOLD_MASK 0x000fe000L
+#define MC_ARB_RTT_CNTL1__WINDOW_DEC_THRESHOLD__SHIFT 0x0000000d
+#define MC_ARB_RTT_CNTL1__WINDOW_INC_THRESHOLD_MASK 0x00001fc0L
+#define MC_ARB_RTT_CNTL1__WINDOW_INC_THRESHOLD__SHIFT 0x00000006
+#define MC_ARB_RTT_CNTL1__WINDOW_SIZE_MASK 0x0000001fL
+#define MC_ARB_RTT_CNTL1__WINDOW_SIZE_MAX_MASK 0x01f00000L
+#define MC_ARB_RTT_CNTL1__WINDOW_SIZE_MAX__SHIFT 0x00000014
+#define MC_ARB_RTT_CNTL1__WINDOW_SIZE_MIN_MASK 0x3e000000L
+#define MC_ARB_RTT_CNTL1__WINDOW_SIZE_MIN__SHIFT 0x00000019
+#define MC_ARB_RTT_CNTL1__WINDOW_SIZE__SHIFT 0x00000000
+#define MC_ARB_RTT_CNTL1__WINDOW_UPDATE_COUNT_MASK 0xc0000000L
+#define MC_ARB_RTT_CNTL1__WINDOW_UPDATE_COUNT__SHIFT 0x0000001e
+#define MC_ARB_RTT_CNTL1__WINDOW_UPDATE_MASK 0x00000020L
+#define MC_ARB_RTT_CNTL1__WINDOW_UPDATE__SHIFT 0x00000005
+#define MC_ARB_RTT_CNTL2__FILTER_CNTL_MASK 0x00002000L
+#define MC_ARB_RTT_CNTL2__FILTER_CNTL__SHIFT 0x0000000d
+#define MC_ARB_RTT_CNTL2__PHASE_ADJUST_SIZE_MASK 0x00001000L
+#define MC_ARB_RTT_CNTL2__PHASE_ADJUST_SIZE__SHIFT 0x0000000c
+#define MC_ARB_RTT_CNTL2__PHASE_ADJUST_THRESHOLD_MASK 0x00000fc0L
+#define MC_ARB_RTT_CNTL2__PHASE_ADJUST_THRESHOLD__SHIFT 0x00000006
+#define MC_ARB_RTT_CNTL2__SAMPLE_CNT_MASK 0x0000003fL
+#define MC_ARB_RTT_CNTL2__SAMPLE_CNT__SHIFT 0x00000000
+#define MC_ARB_RTT_DATA__PATTERN_MASK 0x000000ffL
+#define MC_ARB_RTT_DATA__PATTERN__SHIFT 0x00000000
+#define MC_ARB_RTT_DEBUG__DEBUG_BYTE_CH0_MASK 0x00000003L
+#define MC_ARB_RTT_DEBUG__DEBUG_BYTE_CH0__SHIFT 0x00000000
+#define MC_ARB_RTT_DEBUG__DEBUG_BYTE_CH1_MASK 0x0000000cL
+#define MC_ARB_RTT_DEBUG__DEBUG_BYTE_CH1__SHIFT 0x00000002
+#define MC_ARB_RTT_DEBUG__SHIFTED_PHASE_CH0_MASK 0x00000ff0L
+#define MC_ARB_RTT_DEBUG__SHIFTED_PHASE_CH0__SHIFT 0x00000004
+#define MC_ARB_RTT_DEBUG__SHIFTED_PHASE_CH1_MASK 0x01fe0000L
+#define MC_ARB_RTT_DEBUG__SHIFTED_PHASE_CH1__SHIFT 0x00000011
+#define MC_ARB_RTT_DEBUG__WINDOW_SIZE_CH0_MASK 0x0001f000L
+#define MC_ARB_RTT_DEBUG__WINDOW_SIZE_CH0__SHIFT 0x0000000c
+#define MC_ARB_RTT_DEBUG__WINDOW_SIZE_CH1_MASK 0x3e000000L
+#define MC_ARB_RTT_DEBUG__WINDOW_SIZE_CH1__SHIFT 0x00000019
+#define MC_ARB_SQM_CNTL__DYN_SQM_ENABLE_MASK 0x00000100L
+#define MC_ARB_SQM_CNTL__DYN_SQM_ENABLE__SHIFT 0x00000008
+#define MC_ARB_SQM_CNTL__MIN_PENAL_MASK 0x000000ffL
+#define MC_ARB_SQM_CNTL__MIN_PENAL__SHIFT 0x00000000
+#define MC_ARB_SQM_CNTL__RATIO_DEBUG_MASK 0xff000000L
+#define MC_ARB_SQM_CNTL__RATIO_DEBUG__SHIFT 0x00000018
+#define MC_ARB_SQM_CNTL__RATIO_MASK 0x00ff0000L
+#define MC_ARB_SQM_CNTL__RATIO__SHIFT 0x00000010
+#define MC_ARB_SQM_CNTL__SQM_RESERVE_MASK 0x0000fe00L
+#define MC_ARB_SQM_CNTL__SQM_RESERVE__SHIFT 0x00000009
+#define MC_ARB_TM_CNTL_RD__BANK_SELECT_MASK 0x00000006L
+#define MC_ARB_TM_CNTL_RD__BANK_SELECT__SHIFT 0x00000001
+#define MC_ARB_TM_CNTL_RD__GROUPBY_RANK_MASK 0x00000001L
+#define MC_ARB_TM_CNTL_RD__GROUPBY_RANK__SHIFT 0x00000000
+#define MC_ARB_TM_CNTL_RD__MATCH_BANK_MASK 0x00000010L
+#define MC_ARB_TM_CNTL_RD__MATCH_BANK__SHIFT 0x00000004
+#define MC_ARB_TM_CNTL_RD__MATCH_RANK_MASK 0x00000008L
+#define MC_ARB_TM_CNTL_RD__MATCH_RANK__SHIFT 0x00000003
+#define MC_ARB_TM_CNTL_WR__BANK_SELECT_MASK 0x00000006L
+#define MC_ARB_TM_CNTL_WR__BANK_SELECT__SHIFT 0x00000001
+#define MC_ARB_TM_CNTL_WR__GROUPBY_RANK_MASK 0x00000001L
+#define MC_ARB_TM_CNTL_WR__GROUPBY_RANK__SHIFT 0x00000000
+#define MC_ARB_TM_CNTL_WR__MATCH_BANK_MASK 0x00000010L
+#define MC_ARB_TM_CNTL_WR__MATCH_BANK__SHIFT 0x00000004
+#define MC_ARB_TM_CNTL_WR__MATCH_RANK_MASK 0x00000008L
+#define MC_ARB_TM_CNTL_WR__MATCH_RANK__SHIFT 0x00000003
+#define MC_ARB_WCDR_2__DEBUG_0_MASK 0x00000200L
+#define MC_ARB_WCDR_2__DEBUG_0__SHIFT 0x00000009
+#define MC_ARB_WCDR_2__DEBUG_1_MASK 0x00000400L
+#define MC_ARB_WCDR_2__DEBUG_1__SHIFT 0x0000000a
+#define MC_ARB_WCDR_2__DEBUG_2_MASK 0x00000800L
+#define MC_ARB_WCDR_2__DEBUG_2__SHIFT 0x0000000b
+#define MC_ARB_WCDR_2__DEBUG_3_MASK 0x00001000L
+#define MC_ARB_WCDR_2__DEBUG_3__SHIFT 0x0000000c
+#define MC_ARB_WCDR_2__DEBUG_4_MASK 0x00002000L
+#define MC_ARB_WCDR_2__DEBUG_4__SHIFT 0x0000000d
+#define MC_ARB_WCDR_2__DEBUG_5_MASK 0x00004000L
+#define MC_ARB_WCDR_2__DEBUG_5__SHIFT 0x0000000e
+#define MC_ARB_WCDR_2__WPRE_INC_STEP_MASK 0x0000000fL
+#define MC_ARB_WCDR_2__WPRE_INC_STEP__SHIFT 0x00000000
+#define MC_ARB_WCDR_2__WPRE_MIN_THRESHOLD_MASK 0x000001f0L
+#define MC_ARB_WCDR_2__WPRE_MIN_THRESHOLD__SHIFT 0x00000004
+#define MC_ARB_WCDR__IDLE_BURST_MASK 0x00001f80L
+#define MC_ARB_WCDR__IDLE_BURST_MODE_MASK 0x00002000L
+#define MC_ARB_WCDR__IDLE_BURST_MODE__SHIFT 0x0000000d
+#define MC_ARB_WCDR__IDLE_BURST__SHIFT 0x00000007
+#define MC_ARB_WCDR__IDLE_DEGLITCH_ENABLE_MASK 0x00010000L
+#define MC_ARB_WCDR__IDLE_DEGLITCH_ENABLE__SHIFT 0x00000010
+#define MC_ARB_WCDR__IDLE_ENABLE_MASK 0x00000001L
+#define MC_ARB_WCDR__IDLE_ENABLE__SHIFT 0x00000000
+#define MC_ARB_WCDR__IDLE_PERIOD_MASK 0x0000007cL
+#define MC_ARB_WCDR__IDLE_PERIOD__SHIFT 0x00000002
+#define MC_ARB_WCDR__IDLE_WAKEUP_MASK 0x0000c000L
+#define MC_ARB_WCDR__IDLE_WAKEUP__SHIFT 0x0000000e
+#define MC_ARB_WCDR__SEQ_IDLE_MASK 0x00000002L
+#define MC_ARB_WCDR__SEQ_IDLE__SHIFT 0x00000001
+#define MC_ARB_WCDR__WPRE_ENABLE_MASK 0x00020000L
+#define MC_ARB_WCDR__WPRE_ENABLE__SHIFT 0x00000011
+#define MC_ARB_WCDR__WPRE_INC_READ_MASK 0x02000000L
+#define MC_ARB_WCDR__WPRE_INC_READ__SHIFT 0x00000019
+#define MC_ARB_WCDR__WPRE_INC_SEQIDLE_MASK 0x08000000L
+#define MC_ARB_WCDR__WPRE_INC_SEQIDLE__SHIFT 0x0000001b
+#define MC_ARB_WCDR__WPRE_INC_SKIDIDLE_MASK 0x04000000L
+#define MC_ARB_WCDR__WPRE_INC_SKIDIDLE__SHIFT 0x0000001a
+#define MC_ARB_WCDR__WPRE_MAX_BURST_MASK 0x01c00000L
+#define MC_ARB_WCDR__WPRE_MAX_BURST__SHIFT 0x00000016
+#define MC_ARB_WCDR__WPRE_THRESHOLD_MASK 0x003c0000L
+#define MC_ARB_WCDR__WPRE_THRESHOLD__SHIFT 0x00000012
+#define MC_ARB_WCDR__WPRE_TWOPAGE_MASK 0x10000000L
+#define MC_ARB_WCDR__WPRE_TWOPAGE__SHIFT 0x0000001c
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP0_MASK 0x00000008L
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP0__SHIFT 0x00000003
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP1_MASK 0x00000010L
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP1__SHIFT 0x00000004
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP2_MASK 0x00000020L
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP2__SHIFT 0x00000005
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP3_MASK 0x00000040L
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP3__SHIFT 0x00000006
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP4_MASK 0x00000080L
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP4__SHIFT 0x00000007
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP5_MASK 0x00000100L
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP5__SHIFT 0x00000008
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP6_MASK 0x00000200L
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP6__SHIFT 0x00000009
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP7_MASK 0x00000400L
+#define MC_ARB_WTM_CNTL_RD__ALLOW_STUTTER_GRP7__SHIFT 0x0000000a
+#define MC_ARB_WTM_CNTL_RD__HARSH_PRI_MASK 0x00000004L
+#define MC_ARB_WTM_CNTL_RD__HARSH_PRI__SHIFT 0x00000002
+#define MC_ARB_WTM_CNTL_RD__WTMODE_MASK 0x00000003L
+#define MC_ARB_WTM_CNTL_RD__WTMODE__SHIFT 0x00000000
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP0_MASK 0x00000008L
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP0__SHIFT 0x00000003
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP1_MASK 0x00000010L
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP1__SHIFT 0x00000004
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP2_MASK 0x00000020L
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP2__SHIFT 0x00000005
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP3_MASK 0x00000040L
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP3__SHIFT 0x00000006
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP4_MASK 0x00000080L
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP4__SHIFT 0x00000007
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP5_MASK 0x00000100L
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP5__SHIFT 0x00000008
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP6_MASK 0x00000200L
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP6__SHIFT 0x00000009
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP7_MASK 0x00000400L
+#define MC_ARB_WTM_CNTL_WR__ALLOW_STUTTER_GRP7__SHIFT 0x0000000a
+#define MC_ARB_WTM_CNTL_WR__HARSH_PRI_MASK 0x00000004L
+#define MC_ARB_WTM_CNTL_WR__HARSH_PRI__SHIFT 0x00000002
+#define MC_ARB_WTM_CNTL_WR__WTMODE_MASK 0x00000003L
+#define MC_ARB_WTM_CNTL_WR__WTMODE__SHIFT 0x00000000
+#define MC_ARB_WTM_GRPWT_RD__GRP0_MASK 0x00000003L
+#define MC_ARB_WTM_GRPWT_RD__GRP0__SHIFT 0x00000000
+#define MC_ARB_WTM_GRPWT_RD__GRP1_MASK 0x0000000cL
+#define MC_ARB_WTM_GRPWT_RD__GRP1__SHIFT 0x00000002
+#define MC_ARB_WTM_GRPWT_RD__GRP2_MASK 0x00000030L
+#define MC_ARB_WTM_GRPWT_RD__GRP2__SHIFT 0x00000004
+#define MC_ARB_WTM_GRPWT_RD__GRP3_MASK 0x000000c0L
+#define MC_ARB_WTM_GRPWT_RD__GRP3__SHIFT 0x00000006
+#define MC_ARB_WTM_GRPWT_RD__GRP4_MASK 0x00000300L
+#define MC_ARB_WTM_GRPWT_RD__GRP4__SHIFT 0x00000008
+#define MC_ARB_WTM_GRPWT_RD__GRP5_MASK 0x00000c00L
+#define MC_ARB_WTM_GRPWT_RD__GRP5__SHIFT 0x0000000a
+#define MC_ARB_WTM_GRPWT_RD__GRP6_MASK 0x00003000L
+#define MC_ARB_WTM_GRPWT_RD__GRP6__SHIFT 0x0000000c
+#define MC_ARB_WTM_GRPWT_RD__GRP7_MASK 0x0000c000L
+#define MC_ARB_WTM_GRPWT_RD__GRP7__SHIFT 0x0000000e
+#define MC_ARB_WTM_GRPWT_RD__GRP_EXT_MASK 0x00ff0000L
+#define MC_ARB_WTM_GRPWT_RD__GRP_EXT__SHIFT 0x00000010
+#define MC_ARB_WTM_GRPWT_WR__GRP0_MASK 0x00000003L
+#define MC_ARB_WTM_GRPWT_WR__GRP0__SHIFT 0x00000000
+#define MC_ARB_WTM_GRPWT_WR__GRP1_MASK 0x0000000cL
+#define MC_ARB_WTM_GRPWT_WR__GRP1__SHIFT 0x00000002
+#define MC_ARB_WTM_GRPWT_WR__GRP2_MASK 0x00000030L
+#define MC_ARB_WTM_GRPWT_WR__GRP2__SHIFT 0x00000004
+#define MC_ARB_WTM_GRPWT_WR__GRP3_MASK 0x000000c0L
+#define MC_ARB_WTM_GRPWT_WR__GRP3__SHIFT 0x00000006
+#define MC_ARB_WTM_GRPWT_WR__GRP4_MASK 0x00000300L
+#define MC_ARB_WTM_GRPWT_WR__GRP4__SHIFT 0x00000008
+#define MC_ARB_WTM_GRPWT_WR__GRP5_MASK 0x00000c00L
+#define MC_ARB_WTM_GRPWT_WR__GRP5__SHIFT 0x0000000a
+#define MC_ARB_WTM_GRPWT_WR__GRP6_MASK 0x00003000L
+#define MC_ARB_WTM_GRPWT_WR__GRP6__SHIFT 0x0000000c
+#define MC_ARB_WTM_GRPWT_WR__GRP7_MASK 0x0000c000L
+#define MC_ARB_WTM_GRPWT_WR__GRP7__SHIFT 0x0000000e
+#define MC_ARB_WTM_GRPWT_WR__GRP_EXT_MASK 0x00ff0000L
+#define MC_ARB_WTM_GRPWT_WR__GRP_EXT__SHIFT 0x00000010
+#define MC_BIST_AUTO_CNTL__ADR_GEN_MASK 0x000000f0L
+#define MC_BIST_AUTO_CNTL__ADR_GEN__SHIFT 0x00000004
+#define MC_BIST_AUTO_CNTL__ADR_RESET_MASK 0x02000000L
+#define MC_BIST_AUTO_CNTL__ADR_RESET__SHIFT 0x00000019
+#define MC_BIST_AUTO_CNTL__LFSR_KEY_MASK 0x00ffff00L
+#define MC_BIST_AUTO_CNTL__LFSR_KEY__SHIFT 0x00000008
+#define MC_BIST_AUTO_CNTL__LFSR_RESET_MASK 0x01000000L
+#define MC_BIST_AUTO_CNTL__LFSR_RESET__SHIFT 0x00000018
+#define MC_BIST_AUTO_CNTL__MOP_MASK 0x00000003L
+#define MC_BIST_AUTO_CNTL__MOP__SHIFT 0x00000000
+#define MC_BIST_CMD_CNTL__CMD_ISSUE_LOOP_MASK 0x00000004L
+#define MC_BIST_CMD_CNTL__CMD_ISSUE_LOOP__SHIFT 0x00000002
+#define MC_BIST_CMD_CNTL__CMD_ISSUE_MODE_MASK 0x00000002L
+#define MC_BIST_CMD_CNTL__CMD_ISSUE_MODE__SHIFT 0x00000001
+#define MC_BIST_CMD_CNTL__CMD_ISSUE_MODE_U_MASK 0x00010000L
+#define MC_BIST_CMD_CNTL__CMD_ISSUE_MODE_U__SHIFT 0x00000010
+#define MC_BIST_CMD_CNTL__CMD_ISSUE_RUN_MASK 0x00020000L
+#define MC_BIST_CMD_CNTL__CMD_ISSUE_RUN__SHIFT 0x00000011
+#define MC_BIST_CMD_CNTL__DONE_MASK 0x80000000L
+#define MC_BIST_CMD_CNTL__DONE__SHIFT 0x0000001f
+#define MC_BIST_CMD_CNTL__ENABLE_D0_MASK 0x10000000L
+#define MC_BIST_CMD_CNTL__ENABLE_D0__SHIFT 0x0000001c
+#define MC_BIST_CMD_CNTL__ENABLE_D1_MASK 0x20000000L
+#define MC_BIST_CMD_CNTL__ENABLE_D1__SHIFT 0x0000001d
+#define MC_BIST_CMD_CNTL__LOOP_CNT_MAX_MASK 0x0000fff0L
+#define MC_BIST_CMD_CNTL__LOOP_CNT_MAX__SHIFT 0x00000004
+#define MC_BIST_CMD_CNTL__LOOP_CNT_RD_MASK 0x0ffc0000L
+#define MC_BIST_CMD_CNTL__LOOP_CNT_RD__SHIFT 0x00000012
+#define MC_BIST_CMD_CNTL__LOOP_END_CONDITION_MASK 0x00000008L
+#define MC_BIST_CMD_CNTL__LOOP_END_CONDITION__SHIFT 0x00000003
+#define MC_BIST_CMD_CNTL__RESET_MASK 0x00000001L
+#define MC_BIST_CMD_CNTL__RESET__SHIFT 0x00000000
+#define MC_BIST_CMD_CNTL__STATUS_CH_MASK 0x40000000L
+#define MC_BIST_CMD_CNTL__STATUS_CH__SHIFT 0x0000001e
+#define MC_BIST_CMP_CNTL_2__DATA_STORE_CNT_MASK 0x0000001fL
+#define MC_BIST_CMP_CNTL_2__DATA_STORE_CNT_RST_MASK 0x00000100L
+#define MC_BIST_CMP_CNTL_2__DATA_STORE_CNT_RST__SHIFT 0x00000008
+#define MC_BIST_CMP_CNTL_2__DATA_STORE_CNT__SHIFT 0x00000000
+#define MC_BIST_CMP_CNTL_2__EDC_STORE_CNT_MASK 0x0001f000L
+#define MC_BIST_CMP_CNTL_2__EDC_STORE_CNT_RST_MASK 0x00100000L
+#define MC_BIST_CMP_CNTL_2__EDC_STORE_CNT_RST__SHIFT 0x00000014
+#define MC_BIST_CMP_CNTL_2__EDC_STORE_CNT__SHIFT 0x0000000c
+#define MC_BIST_CMP_CNTL__CMP_MASK 0x00030000L
+#define MC_BIST_CMP_CNTL__CMP_MASK_BIT_MASK 0x00000ff0L
+#define MC_BIST_CMP_CNTL__CMP_MASK_BIT__SHIFT 0x00000004
+#define MC_BIST_CMP_CNTL__CMP_MASK_BYTE_MASK 0x0000000fL
+#define MC_BIST_CMP_CNTL__CMP_MASK_BYTE__SHIFT 0x00000000
+#define MC_BIST_CMP_CNTL__CMP__SHIFT 0x00000010
+#define MC_BIST_CMP_CNTL__DATA_STORE_MODE_MASK 0x00300000L
+#define MC_BIST_CMP_CNTL__DATA_STORE_MODE__SHIFT 0x00000014
+#define MC_BIST_CMP_CNTL__DATA_STORE_SEL_MASK 0x00002000L
+#define MC_BIST_CMP_CNTL__DATA_STORE_SEL__SHIFT 0x0000000d
+#define MC_BIST_CMP_CNTL__DAT_MODE_MASK 0x00040000L
+#define MC_BIST_CMP_CNTL__DAT_MODE__SHIFT 0x00000012
+#define MC_BIST_CMP_CNTL__EDC_STORE_MODE_MASK 0x00080000L
+#define MC_BIST_CMP_CNTL__EDC_STORE_MODE__SHIFT 0x00000013
+#define MC_BIST_CMP_CNTL__EDC_STORE_SEL_MASK 0x00004000L
+#define MC_BIST_CMP_CNTL__EDC_STORE_SEL__SHIFT 0x0000000e
+#define MC_BIST_CMP_CNTL__ENABLE_CMD_FIFO_MASK 0x00008000L
+#define MC_BIST_CMP_CNTL__ENABLE_CMD_FIFO__SHIFT 0x0000000f
+#define MC_BIST_CMP_CNTL__LOAD_RTEDC_MASK 0x00001000L
+#define MC_BIST_CMP_CNTL__LOAD_RTEDC__SHIFT 0x0000000c
+#define MC_BIST_CMP_CNTL__MISMATCH_CNT_MASK 0xffc00000L
+#define MC_BIST_CMP_CNTL__MISMATCH_CNT__SHIFT 0x00000016
+#define MC_BIST_CNTL__ADR_MODE_MASK 0x00000020L
+#define MC_BIST_CNTL__ADR_MODE__SHIFT 0x00000005
+#define MC_BIST_CNTL__DAT_MODE_MASK 0x00000040L
+#define MC_BIST_CNTL__DAT_MODE__SHIFT 0x00000006
+#define MC_BIST_CNTL__DONE_MASK 0x40000000L
+#define MC_BIST_CNTL__DONE__SHIFT 0x0000001e
+#define MC_BIST_CNTL__ENABLE_D0_MASK 0x00001000L
+#define MC_BIST_CNTL__ENABLE_D0__SHIFT 0x0000000c
+#define MC_BIST_CNTL__ENABLE_D1_MASK 0x00002000L
+#define MC_BIST_CNTL__ENABLE_D1__SHIFT 0x0000000d
+#define MC_BIST_CNTL__LOAD_RTDATA_CH_MASK 0x00004000L
+#define MC_BIST_CNTL__LOAD_RTDATA_CH__SHIFT 0x0000000e
+#define MC_BIST_CNTL__LOAD_RTDATA_MASK 0x80000000L
+#define MC_BIST_CNTL__LOAD_RTDATA__SHIFT 0x0000001f
+#define MC_BIST_CNTL__LOOP_CNT_MASK 0x0fff0000L
+#define MC_BIST_CNTL__LOOP_CNT__SHIFT 0x00000010
+#define MC_BIST_CNTL__LOOP_MASK 0x00000c00L
+#define MC_BIST_CNTL__LOOP__SHIFT 0x0000000a
+#define MC_BIST_CNTL__MOP_MODE_MASK 0x00000010L
+#define MC_BIST_CNTL__MOP_MODE__SHIFT 0x00000004
+#define MC_BIST_CNTL__PTR_RST_D0_MASK 0x00000004L
+#define MC_BIST_CNTL__PTR_RST_D0__SHIFT 0x00000002
+#define MC_BIST_CNTL__PTR_RST_D1_MASK 0x00000008L
+#define MC_BIST_CNTL__PTR_RST_D1__SHIFT 0x00000003
+#define MC_BIST_CNTL__RESET_MASK 0x00000001L
+#define MC_BIST_CNTL__RESET__SHIFT 0x00000000
+#define MC_BIST_CNTL__RUN_MASK 0x00000002L
+#define MC_BIST_CNTL__RUN__SHIFT 0x00000001
+#define MC_BIST_DATA_MASK__MASK_MASK 0xffffffffL
+#define MC_BIST_DATA_MASK__MASK__SHIFT 0x00000000
+#define MC_BIST_DATA_WORD0__DATA_MASK 0xffffffffL
+#define MC_BIST_DATA_WORD0__DATA__SHIFT 0x00000000
+#define MC_BIST_DATA_WORD1__DATA_MASK 0xffffffffL
+#define MC_BIST_DATA_WORD1__DATA__SHIFT 0x00000000
+#define MC_BIST_DATA_WORD2__DATA_MASK 0xffffffffL
+#define MC_BIST_DATA_WORD2__DATA__SHIFT 0x00000000
+#define MC_BIST_DATA_WORD3__DATA_MASK 0xffffffffL
+#define MC_BIST_DATA_WORD3__DATA__SHIFT 0x00000000
+#define MC_BIST_DATA_WORD4__DATA_MASK 0xffffffffL
+#define MC_BIST_DATA_WORD4__DATA__SHIFT 0x00000000
+#define MC_BIST_DATA_WORD5__DATA_MASK 0xffffffffL
+#define MC_BIST_DATA_WORD5__DATA__SHIFT 0x00000000
+#define MC_BIST_DATA_WORD6__DATA_MASK 0xffffffffL
+#define MC_BIST_DATA_WORD6__DATA__SHIFT 0x00000000
+#define MC_BIST_DATA_WORD7__DATA_MASK 0xffffffffL
+#define MC_BIST_DATA_WORD7__DATA__SHIFT 0x00000000
+#define MC_BIST_DIR_CNTL__CMD_RTR_D0_MASK 0x00000040L
+#define MC_BIST_DIR_CNTL__CMD_RTR_D0__SHIFT 0x00000006
+#define MC_BIST_DIR_CNTL__CMD_RTR_D1_MASK 0x00000100L
+#define MC_BIST_DIR_CNTL__CMD_RTR_D1__SHIFT 0x00000008
+#define MC_BIST_DIR_CNTL__DATA_LOAD_MASK 0x00000020L
+#define MC_BIST_DIR_CNTL__DATA_LOAD__SHIFT 0x00000005
+#define MC_BIST_DIR_CNTL__DAT_RTR_D0_MASK 0x00000080L
+#define MC_BIST_DIR_CNTL__DAT_RTR_D0__SHIFT 0x00000007
+#define MC_BIST_DIR_CNTL__DAT_RTR_D1_MASK 0x00000200L
+#define MC_BIST_DIR_CNTL__DAT_RTR_D1__SHIFT 0x00000009
+#define MC_BIST_DIR_CNTL__EOB_MASK 0x00000008L
+#define MC_BIST_DIR_CNTL__EOB__SHIFT 0x00000003
+#define MC_BIST_DIR_CNTL__MOP3_MASK 0x00000400L
+#define MC_BIST_DIR_CNTL__MOP3__SHIFT 0x0000000a
+#define MC_BIST_DIR_CNTL__MOP_LOAD_MASK 0x00000010L
+#define MC_BIST_DIR_CNTL__MOP_LOAD__SHIFT 0x00000004
+#define MC_BIST_DIR_CNTL__MOP_MASK 0x00000007L
+#define MC_BIST_DIR_CNTL__MOP__SHIFT 0x00000000
+#define MC_BIST_EADDR__BANK_MASK 0x0f000000L
+#define MC_BIST_EADDR__BANK__SHIFT 0x00000018
+#define MC_BIST_EADDR__COLH_MASK 0x20000000L
+#define MC_BIST_EADDR__COLH__SHIFT 0x0000001d
+#define MC_BIST_EADDR__COL_MASK 0x000003ffL
+#define MC_BIST_EADDR__COL__SHIFT 0x00000000
+#define MC_BIST_EADDR__RANK_MASK 0x10000000L
+#define MC_BIST_EADDR__RANK__SHIFT 0x0000001c
+#define MC_BIST_EADDR__ROWH_MASK 0xc0000000L
+#define MC_BIST_EADDR__ROWH__SHIFT 0x0000001e
+#define MC_BIST_EADDR__ROW_MASK 0x00fffc00L
+#define MC_BIST_EADDR__ROW__SHIFT 0x0000000a
+#define MC_BIST_MISMATCH_ADDR__BANK_MASK 0x0f000000L
+#define MC_BIST_MISMATCH_ADDR__BANK__SHIFT 0x00000018
+#define MC_BIST_MISMATCH_ADDR__COLH_MASK 0x20000000L
+#define MC_BIST_MISMATCH_ADDR__COLH__SHIFT 0x0000001d
+#define MC_BIST_MISMATCH_ADDR__COL_MASK 0x000003ffL
+#define MC_BIST_MISMATCH_ADDR__COL__SHIFT 0x00000000
+#define MC_BIST_MISMATCH_ADDR__RANK_MASK 0x10000000L
+#define MC_BIST_MISMATCH_ADDR__RANK__SHIFT 0x0000001c
+#define MC_BIST_MISMATCH_ADDR__ROWH_MASK 0xc0000000L
+#define MC_BIST_MISMATCH_ADDR__ROWH__SHIFT 0x0000001e
+#define MC_BIST_MISMATCH_ADDR__ROW_MASK 0x00fffc00L
+#define MC_BIST_MISMATCH_ADDR__ROW__SHIFT 0x0000000a
+#define MC_BIST_RDATA_EDC__EDC_MASK 0xffffffffL
+#define MC_BIST_RDATA_EDC__EDC__SHIFT 0x00000000
+#define MC_BIST_RDATA_MASK__MASK_MASK 0xffffffffL
+#define MC_BIST_RDATA_MASK__MASK__SHIFT 0x00000000
+#define MC_BIST_RDATA_WORD0__RDATA_MASK 0xffffffffL
+#define MC_BIST_RDATA_WORD0__RDATA__SHIFT 0x00000000
+#define MC_BIST_RDATA_WORD1__RDATA_MASK 0xffffffffL
+#define MC_BIST_RDATA_WORD1__RDATA__SHIFT 0x00000000
+#define MC_BIST_RDATA_WORD2__RDATA_MASK 0xffffffffL
+#define MC_BIST_RDATA_WORD2__RDATA__SHIFT 0x00000000
+#define MC_BIST_RDATA_WORD3__RDATA_MASK 0xffffffffL
+#define MC_BIST_RDATA_WORD3__RDATA__SHIFT 0x00000000
+#define MC_BIST_RDATA_WORD4__RDATA_MASK 0xffffffffL
+#define MC_BIST_RDATA_WORD4__RDATA__SHIFT 0x00000000
+#define MC_BIST_RDATA_WORD5__RDATA_MASK 0xffffffffL
+#define MC_BIST_RDATA_WORD5__RDATA__SHIFT 0x00000000
+#define MC_BIST_RDATA_WORD6__RDATA_MASK 0xffffffffL
+#define MC_BIST_RDATA_WORD6__RDATA__SHIFT 0x00000000
+#define MC_BIST_RDATA_WORD7__RDATA_MASK 0xffffffffL
+#define MC_BIST_RDATA_WORD7__RDATA__SHIFT 0x00000000
+#define MC_BIST_SADDR__BANK_MASK 0x0f000000L
+#define MC_BIST_SADDR__BANK__SHIFT 0x00000018
+#define MC_BIST_SADDR__COLH_MASK 0x20000000L
+#define MC_BIST_SADDR__COLH__SHIFT 0x0000001d
+#define MC_BIST_SADDR__COL_MASK 0x000003ffL
+#define MC_BIST_SADDR__COL__SHIFT 0x00000000
+#define MC_BIST_SADDR__RANK_MASK 0x10000000L
+#define MC_BIST_SADDR__RANK__SHIFT 0x0000001c
+#define MC_BIST_SADDR__ROWH_MASK 0xc0000000L
+#define MC_BIST_SADDR__ROWH__SHIFT 0x0000001e
+#define MC_BIST_SADDR__ROW_MASK 0x00fffc00L
+#define MC_BIST_SADDR__ROW__SHIFT 0x0000000a
+#define MC_CG_CONFIG__INDEX_MASK 0x003fffc0L
+#define MC_CG_CONFIG__INDEX__SHIFT 0x00000006
+#define MC_CG_CONFIG_MCD__INDEX_MASK 0x1fffe000L
+#define MC_CG_CONFIG_MCD__INDEX__SHIFT 0x0000000d
+#define MC_CG_CONFIG_MCD__MCD0_WR_ENABLE_MASK 0x00000001L
+#define MC_CG_CONFIG_MCD__MCD0_WR_ENABLE__SHIFT 0x00000000
+#define MC_CG_CONFIG_MCD__MCD1_WR_ENABLE_MASK 0x00000002L
+#define MC_CG_CONFIG_MCD__MCD1_WR_ENABLE__SHIFT 0x00000001
+#define MC_CG_CONFIG_MCD__MCD2_WR_ENABLE_MASK 0x00000004L
+#define MC_CG_CONFIG_MCD__MCD2_WR_ENABLE__SHIFT 0x00000002
+#define MC_CG_CONFIG_MCD__MCD3_WR_ENABLE_MASK 0x00000008L
+#define MC_CG_CONFIG_MCD__MCD3_WR_ENABLE__SHIFT 0x00000003
+#define MC_CG_CONFIG_MCD__MCD4_WR_ENABLE_MASK 0x00000010L
+#define MC_CG_CONFIG_MCD__MCD4_WR_ENABLE__SHIFT 0x00000004
+#define MC_CG_CONFIG_MCD__MCD5_WR_ENABLE_MASK 0x00000020L
+#define MC_CG_CONFIG_MCD__MCD5_WR_ENABLE__SHIFT 0x00000005
+#define MC_CG_CONFIG_MCD__MC_RD_ENABLE_MASK 0x00000700L
+#define MC_CG_CONFIG_MCD__MC_RD_ENABLE__SHIFT 0x00000008
+#define MC_CG_CONFIG__MCDW_WR_ENABLE_MASK 0x00000001L
+#define MC_CG_CONFIG__MCDW_WR_ENABLE__SHIFT 0x00000000
+#define MC_CG_CONFIG__MCDX_WR_ENABLE_MASK 0x00000002L
+#define MC_CG_CONFIG__MCDX_WR_ENABLE__SHIFT 0x00000001
+#define MC_CG_CONFIG__MCDY_WR_ENABLE_MASK 0x00000004L
+#define MC_CG_CONFIG__MCDY_WR_ENABLE__SHIFT 0x00000002
+#define MC_CG_CONFIG__MCDZ_WR_ENABLE_MASK 0x00000008L
+#define MC_CG_CONFIG__MCDZ_WR_ENABLE__SHIFT 0x00000003
+#define MC_CG_CONFIG__MC_RD_ENABLE_MASK 0x00000030L
+#define MC_CG_CONFIG__MC_RD_ENABLE__SHIFT 0x00000004
+#define MC_CG_DATAPORT__DATA_FIELD_MASK 0xffffffffL
+#define MC_CG_DATAPORT__DATA_FIELD__SHIFT 0x00000000
+#define MC_CITF_CNTL__EXEMPTPM_MASK 0x00000008L
+#define MC_CITF_CNTL__EXEMPTPM__SHIFT 0x00000003
+#define MC_CITF_CNTL__GFX_IDLE_OVERRIDE_MASK 0x00000030L
+#define MC_CITF_CNTL__GFX_IDLE_OVERRIDE__SHIFT 0x00000004
+#define MC_CITF_CNTL__IGNOREPM_MASK 0x00000004L
+#define MC_CITF_CNTL__IGNOREPM__SHIFT 0x00000002
+#define MC_CITF_CNTL__MCD_SRBM_MASK_ENABLE_MASK 0x00000040L
+#define MC_CITF_CNTL__MCD_SRBM_MASK_ENABLE__SHIFT 0x00000006
+#define MC_CITF_CREDITS_ARB_RD__HUB_PRI_MASK 0x02000000L
+#define MC_CITF_CREDITS_ARB_RD__HUB_PRI__SHIFT 0x00000019
+#define MC_CITF_CREDITS_ARB_RD__LCL_PRI_MASK 0x01000000L
+#define MC_CITF_CREDITS_ARB_RD__LCL_PRI__SHIFT 0x00000018
+#define MC_CITF_CREDITS_ARB_RD__READ_HUB_MASK 0x0000ff00L
+#define MC_CITF_CREDITS_ARB_RD__READ_HUB__SHIFT 0x00000008
+#define MC_CITF_CREDITS_ARB_RD__READ_LCL_MASK 0x000000ffL
+#define MC_CITF_CREDITS_ARB_RD__READ_LCL__SHIFT 0x00000000
+#define MC_CITF_CREDITS_ARB_RD__READ_PRI_MASK 0x00ff0000L
+#define MC_CITF_CREDITS_ARB_RD__READ_PRI__SHIFT 0x00000010
+#define MC_CITF_CREDITS_ARB_WR__HUB_PRI_MASK 0x00010000L
+#define MC_CITF_CREDITS_ARB_WR__HUB_PRI__SHIFT 0x00000010
+#define MC_CITF_CREDITS_ARB_WR__LCL_PRI_MASK 0x00020000L
+#define MC_CITF_CREDITS_ARB_WR__LCL_PRI__SHIFT 0x00000011
+#define MC_CITF_CREDITS_ARB_WR__WRITE_HUB_MASK 0x0000ff00L
+#define MC_CITF_CREDITS_ARB_WR__WRITE_HUB__SHIFT 0x00000008
+#define MC_CITF_CREDITS_ARB_WR__WRITE_LCL_MASK 0x000000ffL
+#define MC_CITF_CREDITS_ARB_WR__WRITE_LCL__SHIFT 0x00000000
+#define MC_CITF_CREDITS_VM__READ_ALL_MASK 0x0000003fL
+#define MC_CITF_CREDITS_VM__READ_ALL__SHIFT 0x00000000
+#define MC_CITF_CREDITS_VM__WRITE_ALL_MASK 0x00000fc0L
+#define MC_CITF_CREDITS_VM__WRITE_ALL__SHIFT 0x00000006
+#define MC_CITF_CREDITS_XBAR__READ_LCL_MASK 0x000000ffL
+#define MC_CITF_CREDITS_XBAR__READ_LCL__SHIFT 0x00000000
+#define MC_CITF_CREDITS_XBAR__WRITE_LCL_MASK 0x0000ff00L
+#define MC_CITF_CREDITS_XBAR__WRITE_LCL__SHIFT 0x00000008
+#define MC_CITF_DAGB_CNTL__CENTER_RD_MAX_BURST_MASK 0x0000001eL
+#define MC_CITF_DAGB_CNTL__CENTER_RD_MAX_BURST__SHIFT 0x00000001
+#define MC_CITF_DAGB_CNTL__CENTER_WR_MAX_BURST_MASK 0x000003c0L
+#define MC_CITF_DAGB_CNTL__CENTER_WR_MAX_BURST__SHIFT 0x00000006
+#define MC_CITF_DAGB_CNTL__DISABLE_SELF_INIT_MASK 0x00000020L
+#define MC_CITF_DAGB_CNTL__DISABLE_SELF_INIT__SHIFT 0x00000005
+#define MC_CITF_DAGB_CNTL__JUMP_AHEAD_MASK 0x00000001L
+#define MC_CITF_DAGB_CNTL__JUMP_AHEAD__SHIFT 0x00000000
+#define MC_CITF_DAGB_DLY__CLI_MASK 0x001f0000L
+#define MC_CITF_DAGB_DLY__CLI__SHIFT 0x00000010
+#define MC_CITF_DAGB_DLY__DLY_MASK 0x0000001fL
+#define MC_CITF_DAGB_DLY__DLY__SHIFT 0x00000000
+#define MC_CITF_DAGB_DLY__POS_MASK 0x1f000000L
+#define MC_CITF_DAGB_DLY__POS__SHIFT 0x00000018
+#define MC_CITF_INT_CREDITS__CNTR_RD_HUB_HP_MASK 0x00fc0000L
+#define MC_CITF_INT_CREDITS__CNTR_RD_HUB_HP__SHIFT 0x00000012
+#define MC_CITF_INT_CREDITS__CNTR_RD_HUB_LP_MASK 0x0003f000L
+#define MC_CITF_INT_CREDITS__CNTR_RD_HUB_LP__SHIFT 0x0000000c
+#define MC_CITF_INT_CREDITS__CNTR_RD_LCL_MASK 0x3f000000L
+#define MC_CITF_INT_CREDITS__CNTR_RD_LCL__SHIFT 0x00000018
+#define MC_CITF_INT_CREDITS__REMRDRET_MASK 0x0000003fL
+#define MC_CITF_INT_CREDITS__REMRDRET__SHIFT 0x00000000
+#define MC_CITF_INT_CREDITS_WR__CNTR_WR_HUB_MASK 0x0000003fL
+#define MC_CITF_INT_CREDITS_WR__CNTR_WR_HUB__SHIFT 0x00000000
+#define MC_CITF_INT_CREDITS_WR__CNTR_WR_LCL_MASK 0x00000fc0L
+#define MC_CITF_INT_CREDITS_WR__CNTR_WR_LCL__SHIFT 0x00000006
+#define MC_CITF_MISC_RD_CG__ENABLE_MASK 0x00040000L
+#define MC_CITF_MISC_RD_CG__ENABLE__SHIFT 0x00000012
+#define MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK 0x00080000L
+#define MC_CITF_MISC_RD_CG__MEM_LS_ENABLE__SHIFT 0x00000013
+#define MC_CITF_MISC_RD_CG__OFFDLY_MASK 0x00000fc0L
+#define MC_CITF_MISC_RD_CG__OFFDLY__SHIFT 0x00000006
+#define MC_CITF_MISC_RD_CG__ONDLY_MASK 0x0000003fL
+#define MC_CITF_MISC_RD_CG__ONDLY__SHIFT 0x00000000
+#define MC_CITF_MISC_RD_CG__RDYDLY_MASK 0x0003f000L
+#define MC_CITF_MISC_RD_CG__RDYDLY__SHIFT 0x0000000c
+#define MC_CITF_MISC_VM_CG__ENABLE_MASK 0x00040000L
+#define MC_CITF_MISC_VM_CG__ENABLE__SHIFT 0x00000012
+#define MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK 0x00080000L
+#define MC_CITF_MISC_VM_CG__MEM_LS_ENABLE__SHIFT 0x00000013
+#define MC_CITF_MISC_VM_CG__OFFDLY_MASK 0x00000fc0L
+#define MC_CITF_MISC_VM_CG__OFFDLY__SHIFT 0x00000006
+#define MC_CITF_MISC_VM_CG__ONDLY_MASK 0x0000003fL
+#define MC_CITF_MISC_VM_CG__ONDLY__SHIFT 0x00000000
+#define MC_CITF_MISC_VM_CG__RDYDLY_MASK 0x0003f000L
+#define MC_CITF_MISC_VM_CG__RDYDLY__SHIFT 0x0000000c
+#define MC_CITF_MISC_WR_CG__ENABLE_MASK 0x00040000L
+#define MC_CITF_MISC_WR_CG__ENABLE__SHIFT 0x00000012
+#define MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK 0x00080000L
+#define MC_CITF_MISC_WR_CG__MEM_LS_ENABLE__SHIFT 0x00000013
+#define MC_CITF_MISC_WR_CG__OFFDLY_MASK 0x00000fc0L
+#define MC_CITF_MISC_WR_CG__OFFDLY__SHIFT 0x00000006
+#define MC_CITF_MISC_WR_CG__ONDLY_MASK 0x0000003fL
+#define MC_CITF_MISC_WR_CG__ONDLY__SHIFT 0x00000000
+#define MC_CITF_MISC_WR_CG__RDYDLY_MASK 0x0003f000L
+#define MC_CITF_MISC_WR_CG__RDYDLY__SHIFT 0x0000000c
+#define MC_CITF_PERF_MON_CNTL2__CID_MASK 0x000001ffL
+#define MC_CITF_PERF_MON_CNTL2__CID__SHIFT 0x00000000
+#define MC_CITF_PERF_MON_RSLT2__CB_RD_BUSY_MASK 0x00000040L
+#define MC_CITF_PERF_MON_RSLT2__CB_RD_BUSY__SHIFT 0x00000006
+#define MC_CITF_PERF_MON_RSLT2__CB_WR_BUSY_MASK 0x00001000L
+#define MC_CITF_PERF_MON_RSLT2__CB_WR_BUSY__SHIFT 0x0000000c
+#define MC_CITF_PERF_MON_RSLT2__DB_RD_BUSY_MASK 0x00000080L
+#define MC_CITF_PERF_MON_RSLT2__DB_RD_BUSY__SHIFT 0x00000007
+#define MC_CITF_PERF_MON_RSLT2__DB_WR_BUSY_MASK 0x00002000L
+#define MC_CITF_PERF_MON_RSLT2__DB_WR_BUSY__SHIFT 0x0000000d
+#define MC_CITF_PERF_MON_RSLT2__SX_WR_BUSY_MASK 0x00004000L
+#define MC_CITF_PERF_MON_RSLT2__SX_WR_BUSY__SHIFT 0x0000000e
+#define MC_CITF_PERF_MON_RSLT2__TC0_RD_BUSY_MASK 0x00000100L
+#define MC_CITF_PERF_MON_RSLT2__TC0_RD_BUSY__SHIFT 0x00000008
+#define MC_CITF_PERF_MON_RSLT2__TC0_WR_BUSY_MASK 0x00010000L
+#define MC_CITF_PERF_MON_RSLT2__TC0_WR_BUSY__SHIFT 0x00000010
+#define MC_CITF_PERF_MON_RSLT2__TC1_RD_BUSY_MASK 0x00000400L
+#define MC_CITF_PERF_MON_RSLT2__TC1_RD_BUSY__SHIFT 0x0000000a
+#define MC_CITF_PERF_MON_RSLT2__TC1_WR_BUSY_MASK 0x00020000L
+#define MC_CITF_PERF_MON_RSLT2__TC1_WR_BUSY__SHIFT 0x00000011
+#define MC_CITF_PERF_MON_RSLT2__TC2_RD_BUSY_MASK 0x00008000L
+#define MC_CITF_PERF_MON_RSLT2__TC2_RD_BUSY__SHIFT 0x0000000f
+#define MC_CITF_PERF_MON_RSLT2__TC2_WR_BUSY_MASK 0x00040000L
+#define MC_CITF_PERF_MON_RSLT2__TC2_WR_BUSY__SHIFT 0x00000012
+#define MC_CITF_PERF_MON_RSLT2__VC0_RD_BUSY_MASK 0x00000200L
+#define MC_CITF_PERF_MON_RSLT2__VC0_RD_BUSY__SHIFT 0x00000009
+#define MC_CITF_PERF_MON_RSLT2__VC1_RD_BUSY_MASK 0x00000800L
+#define MC_CITF_PERF_MON_RSLT2__VC1_RD_BUSY__SHIFT 0x0000000b
+#define MC_CITF_REMREQ__CREDITS_ENABLE_MASK 0x00004000L
+#define MC_CITF_REMREQ__CREDITS_ENABLE__SHIFT 0x0000000e
+#define MC_CITF_REMREQ__READ_CREDITS_MASK 0x0000007fL
+#define MC_CITF_REMREQ__READ_CREDITS__SHIFT 0x00000000
+#define MC_CITF_REMREQ__WRITE_CREDITS_MASK 0x00003f80L
+#define MC_CITF_REMREQ__WRITE_CREDITS__SHIFT 0x00000007
+#define MC_CITF_RET_MODE__INORDER_RD_MASK 0x00000001L
+#define MC_CITF_RET_MODE__INORDER_RD__SHIFT 0x00000000
+#define MC_CITF_RET_MODE__INORDER_WR_MASK 0x00000002L
+#define MC_CITF_RET_MODE__INORDER_WR__SHIFT 0x00000001
+#define MC_CITF_RET_MODE__LCLPRI_RD_MASK 0x00000010L
+#define MC_CITF_RET_MODE__LCLPRI_RD__SHIFT 0x00000004
+#define MC_CITF_RET_MODE__LCLPRI_WR_MASK 0x00000020L
+#define MC_CITF_RET_MODE__LCLPRI_WR__SHIFT 0x00000005
+#define MC_CITF_RET_MODE__REMPRI_RD_MASK 0x00000004L
+#define MC_CITF_RET_MODE__REMPRI_RD__SHIFT 0x00000002
+#define MC_CITF_RET_MODE__REMPRI_WR_MASK 0x00000008L
+#define MC_CITF_RET_MODE__REMPRI_WR__SHIFT 0x00000003
+#define MC_CITF_WTM_RD_CNTL__DISABLE_REMOTE_MASK 0x01000000L
+#define MC_CITF_WTM_RD_CNTL__DISABLE_REMOTE__SHIFT 0x00000018
+#define MC_CITF_WTM_RD_CNTL__GROUP0_DECREMENT_MASK 0x00000007L
+#define MC_CITF_WTM_RD_CNTL__GROUP0_DECREMENT__SHIFT 0x00000000
+#define MC_CITF_WTM_RD_CNTL__GROUP1_DECREMENT_MASK 0x00000038L
+#define MC_CITF_WTM_RD_CNTL__GROUP1_DECREMENT__SHIFT 0x00000003
+#define MC_CITF_WTM_RD_CNTL__GROUP2_DECREMENT_MASK 0x000001c0L
+#define MC_CITF_WTM_RD_CNTL__GROUP2_DECREMENT__SHIFT 0x00000006
+#define MC_CITF_WTM_RD_CNTL__GROUP3_DECREMENT_MASK 0x00000e00L
+#define MC_CITF_WTM_RD_CNTL__GROUP3_DECREMENT__SHIFT 0x00000009
+#define MC_CITF_WTM_RD_CNTL__GROUP4_DECREMENT_MASK 0x00007000L
+#define MC_CITF_WTM_RD_CNTL__GROUP4_DECREMENT__SHIFT 0x0000000c
+#define MC_CITF_WTM_RD_CNTL__GROUP5_DECREMENT_MASK 0x00038000L
+#define MC_CITF_WTM_RD_CNTL__GROUP5_DECREMENT__SHIFT 0x0000000f
+#define MC_CITF_WTM_RD_CNTL__GROUP6_DECREMENT_MASK 0x001c0000L
+#define MC_CITF_WTM_RD_CNTL__GROUP6_DECREMENT__SHIFT 0x00000012
+#define MC_CITF_WTM_RD_CNTL__GROUP7_DECREMENT_MASK 0x00e00000L
+#define MC_CITF_WTM_RD_CNTL__GROUP7_DECREMENT__SHIFT 0x00000015
+#define MC_CITF_WTM_WR_CNTL__DISABLE_REMOTE_MASK 0x01000000L
+#define MC_CITF_WTM_WR_CNTL__DISABLE_REMOTE__SHIFT 0x00000018
+#define MC_CITF_WTM_WR_CNTL__GROUP0_DECREMENT_MASK 0x00000007L
+#define MC_CITF_WTM_WR_CNTL__GROUP0_DECREMENT__SHIFT 0x00000000
+#define MC_CITF_WTM_WR_CNTL__GROUP1_DECREMENT_MASK 0x00000038L
+#define MC_CITF_WTM_WR_CNTL__GROUP1_DECREMENT__SHIFT 0x00000003
+#define MC_CITF_WTM_WR_CNTL__GROUP2_DECREMENT_MASK 0x000001c0L
+#define MC_CITF_WTM_WR_CNTL__GROUP2_DECREMENT__SHIFT 0x00000006
+#define MC_CITF_WTM_WR_CNTL__GROUP3_DECREMENT_MASK 0x00000e00L
+#define MC_CITF_WTM_WR_CNTL__GROUP3_DECREMENT__SHIFT 0x00000009
+#define MC_CITF_WTM_WR_CNTL__GROUP4_DECREMENT_MASK 0x00007000L
+#define MC_CITF_WTM_WR_CNTL__GROUP4_DECREMENT__SHIFT 0x0000000c
+#define MC_CITF_WTM_WR_CNTL__GROUP5_DECREMENT_MASK 0x00038000L
+#define MC_CITF_WTM_WR_CNTL__GROUP5_DECREMENT__SHIFT 0x0000000f
+#define MC_CITF_WTM_WR_CNTL__GROUP6_DECREMENT_MASK 0x001c0000L
+#define MC_CITF_WTM_WR_CNTL__GROUP6_DECREMENT__SHIFT 0x00000012
+#define MC_CITF_WTM_WR_CNTL__GROUP7_DECREMENT_MASK 0x00e00000L
+#define MC_CITF_WTM_WR_CNTL__GROUP7_DECREMENT__SHIFT 0x00000015
+#define MC_CITF_XTRA_ENABLE__ARB_DBG_MASK 0x00000f00L
+#define MC_CITF_XTRA_ENABLE__ARB_DBG__SHIFT 0x00000008
+#define MC_CITF_XTRA_ENABLE__CB1_RD_MASK 0x00000001L
+#define MC_CITF_XTRA_ENABLE__CB1_RD__SHIFT 0x00000000
+#define MC_CITF_XTRA_ENABLE__CB1_WR_MASK 0x00000002L
+#define MC_CITF_XTRA_ENABLE__CB1_WR__SHIFT 0x00000001
+#define MC_CITF_XTRA_ENABLE__DB1_RD_MASK 0x00000004L
+#define MC_CITF_XTRA_ENABLE__DB1_RD__SHIFT 0x00000002
+#define MC_CITF_XTRA_ENABLE__DB1_WR_MASK 0x00000008L
+#define MC_CITF_XTRA_ENABLE__DB1_WR__SHIFT 0x00000003
+#define MC_CITF_XTRA_ENABLE__TC2_RD_MASK 0x00000010L
+#define MC_CITF_XTRA_ENABLE__TC2_RD__SHIFT 0x00000004
+#define MC_CITF_XTRA_ENABLE__TC2_WR_MASK 0x00001000L
+#define MC_CITF_XTRA_ENABLE__TC2_WR__SHIFT 0x0000000c
+#define MC_CONFIG__MCC_INDEX_MODE_ENABLE_MASK 0x80000000L
+#define MC_CONFIG__MCC_INDEX_MODE_ENABLE__SHIFT 0x0000001f
+#define MC_CONFIG_MCD__MCD0_WR_ENABLE_MASK 0x00000001L
+#define MC_CONFIG_MCD__MCD0_WR_ENABLE__SHIFT 0x00000000
+#define MC_CONFIG_MCD__MCD1_WR_ENABLE_MASK 0x00000002L
+#define MC_CONFIG_MCD__MCD1_WR_ENABLE__SHIFT 0x00000001
+#define MC_CONFIG_MCD__MCD2_WR_ENABLE_MASK 0x00000004L
+#define MC_CONFIG_MCD__MCD2_WR_ENABLE__SHIFT 0x00000002
+#define MC_CONFIG_MCD__MCD3_WR_ENABLE_MASK 0x00000008L
+#define MC_CONFIG_MCD__MCD3_WR_ENABLE__SHIFT 0x00000003
+#define MC_CONFIG_MCD__MCD4_WR_ENABLE_MASK 0x00000010L
+#define MC_CONFIG_MCD__MCD4_WR_ENABLE__SHIFT 0x00000004
+#define MC_CONFIG_MCD__MCD5_WR_ENABLE_MASK 0x00000020L
+#define MC_CONFIG_MCD__MCD5_WR_ENABLE__SHIFT 0x00000005
+#define MC_CONFIG_MCD__MCD_INDEX_MODE_ENABLE_MASK 0x80000000L
+#define MC_CONFIG_MCD__MCD_INDEX_MODE_ENABLE__SHIFT 0x0000001f
+#define MC_CONFIG_MCD__MC_RD_ENABLE_MASK 0x00000700L
+#define MC_CONFIG_MCD__MC_RD_ENABLE__SHIFT 0x00000008
+#define MC_CONFIG__MCDW_WR_ENABLE_MASK 0x00000001L
+#define MC_CONFIG__MCDW_WR_ENABLE__SHIFT 0x00000000
+#define MC_CONFIG__MCDX_WR_ENABLE_MASK 0x00000002L
+#define MC_CONFIG__MCDX_WR_ENABLE__SHIFT 0x00000001
+#define MC_CONFIG__MCDY_WR_ENABLE_MASK 0x00000004L
+#define MC_CONFIG__MCDY_WR_ENABLE__SHIFT 0x00000002
+#define MC_CONFIG__MCDZ_WR_ENABLE_MASK 0x00000008L
+#define MC_CONFIG__MCDZ_WR_ENABLE__SHIFT 0x00000003
+#define MC_CONFIG__MC_RD_ENABLE_MASK 0x00000030L
+#define MC_CONFIG__MC_RD_ENABLE__SHIFT 0x00000004
+#define MC_HUB_MISC_DBG__SELECT0_MASK 0x0000000fL
+#define MC_HUB_MISC_DBG__SELECT0__SHIFT 0x00000000
+#define MC_HUB_MISC_DBG__SELECT1_MASK 0x000000f0L
+#define MC_HUB_MISC_DBG__SELECT1__SHIFT 0x00000004
+#define MC_HUB_MISC_FRAMING__BITS_MASK 0xffffffffL
+#define MC_HUB_MISC_FRAMING__BITS__SHIFT 0x00000000
+#define MC_HUB_MISC_HUB_CG__ENABLE_MASK 0x00040000L
+#define MC_HUB_MISC_HUB_CG__ENABLE__SHIFT 0x00000012
+#define MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK 0x00080000L
+#define MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE__SHIFT 0x00000013
+#define MC_HUB_MISC_HUB_CG__OFFDLY_MASK 0x00000fc0L
+#define MC_HUB_MISC_HUB_CG__OFFDLY__SHIFT 0x00000006
+#define MC_HUB_MISC_HUB_CG__ONDLY_MASK 0x0000003fL
+#define MC_HUB_MISC_HUB_CG__ONDLY__SHIFT 0x00000000
+#define MC_HUB_MISC_HUB_CG__RDYDLY_MASK 0x0003f000L
+#define MC_HUB_MISC_HUB_CG__RDYDLY__SHIFT 0x0000000c
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_CP_READ_MASK 0x00000001L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_CP_READ__SHIFT 0x00000000
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_CP_WRITE_MASK 0x00000002L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_CP_WRITE__SHIFT 0x00000001
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_DISP_READ_MASK 0x00000400L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_DISP_READ__SHIFT 0x0000000a
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_DISP_WRITE_MASK 0x00000800L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_DISP_WRITE__SHIFT 0x0000000b
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_GFX_READ_MASK 0x00000004L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_GFX_READ__SHIFT 0x00000002
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_GFX_WRITE_MASK 0x00000008L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_GFX_WRITE__SHIFT 0x00000003
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_HDP_READ_MASK 0x00010000L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_HDP_READ__SHIFT 0x00000010
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_HDP_WRITE_MASK 0x00020000L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_HDP_WRITE__SHIFT 0x00000011
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_OTH_READ_MASK 0x00040000L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_OTH_READ__SHIFT 0x00000012
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_OTH_WRITE_MASK 0x00080000L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_OTH_WRITE__SHIFT 0x00000013
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_RLC_READ_MASK 0x00000040L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_RLC_READ__SHIFT 0x00000006
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_RLC_WRITE_MASK 0x00000080L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_RLC_WRITE__SHIFT 0x00000007
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_SMU_READ_MASK 0x00004000L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_SMU_READ__SHIFT 0x0000000e
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_SMU_WRITE_MASK 0x00008000L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_SMU_WRITE__SHIFT 0x0000000f
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_UVD_READ_MASK 0x00001000L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_UVD_READ__SHIFT 0x0000000c
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_UVD_WRITE_MASK 0x00002000L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_UVD_WRITE__SHIFT 0x0000000d
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_VCE_READ_MASK 0x01000000L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_VCE_READ__SHIFT 0x00000018
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_VCE_WRITE_MASK 0x02000000L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_VCE_WRITE__SHIFT 0x00000019
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_VMC_READ_MASK 0x00100000L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_VMC_READ__SHIFT 0x00000014
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_VMC_WRITE_MASK 0x00200000L
+#define MC_HUB_MISC_IDLE_STATUS__OUTSTANDING_VMC_WRITE__SHIFT 0x00000015
+#define MC_HUB_MISC_OVERRIDE__IDLE_MASK 0x00000003L
+#define MC_HUB_MISC_OVERRIDE__IDLE__SHIFT 0x00000000
+#define MC_HUB_MISC_POWER__PM_BLACKOUT_CNTL_MASK 0x00000018L
+#define MC_HUB_MISC_POWER__PM_BLACKOUT_CNTL__SHIFT 0x00000003
+#define MC_HUB_MISC_POWER__SRBM_GATE_OVERRIDE_MASK 0x00000004L
+#define MC_HUB_MISC_POWER__SRBM_GATE_OVERRIDE__SHIFT 0x00000002
+#define MC_HUB_MISC_SIP_CG__ENABLE_MASK 0x00040000L
+#define MC_HUB_MISC_SIP_CG__ENABLE__SHIFT 0x00000012
+#define MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK 0x00080000L
+#define MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE__SHIFT 0x00000013
+#define MC_HUB_MISC_SIP_CG__OFFDLY_MASK 0x00000fc0L
+#define MC_HUB_MISC_SIP_CG__OFFDLY__SHIFT 0x00000006
+#define MC_HUB_MISC_SIP_CG__ONDLY_MASK 0x0000003fL
+#define MC_HUB_MISC_SIP_CG__ONDLY__SHIFT 0x00000000
+#define MC_HUB_MISC_SIP_CG__RDYDLY_MASK 0x0003f000L
+#define MC_HUB_MISC_SIP_CG__RDYDLY__SHIFT 0x0000000c
+#define MC_HUB_MISC_STATUS__GFX_BUSY_MASK 0x00002000L
+#define MC_HUB_MISC_STATUS__GFX_BUSY__SHIFT 0x0000000d
+#define MC_HUB_MISC_STATUS__OUTSTANDING_HUB_RDREQ_MASK 0x00000004L
+#define MC_HUB_MISC_STATUS__OUTSTANDING_HUB_RDREQ__SHIFT 0x00000002
+#define MC_HUB_MISC_STATUS__OUTSTANDING_HUB_RDRET_MASK 0x00000008L
+#define MC_HUB_MISC_STATUS__OUTSTANDING_HUB_RDRET__SHIFT 0x00000003
+#define MC_HUB_MISC_STATUS__OUTSTANDING_HUB_WRREQ_MASK 0x00000010L
+#define MC_HUB_MISC_STATUS__OUTSTANDING_HUB_WRREQ__SHIFT 0x00000004
+#define MC_HUB_MISC_STATUS__OUTSTANDING_HUB_WRRET_MASK 0x00000020L
+#define MC_HUB_MISC_STATUS__OUTSTANDING_HUB_WRRET__SHIFT 0x00000005
+#define MC_HUB_MISC_STATUS__OUTSTANDING_MCD_READ_MASK 0x00000100L
+#define MC_HUB_MISC_STATUS__OUTSTANDING_MCD_READ__SHIFT 0x00000008
+#define MC_HUB_MISC_STATUS__OUTSTANDING_MCD_WRITE_MASK 0x00000200L
+#define MC_HUB_MISC_STATUS__OUTSTANDING_MCD_WRITE__SHIFT 0x00000009
+#define MC_HUB_MISC_STATUS__OUTSTANDING_READ_MASK 0x00000001L
+#define MC_HUB_MISC_STATUS__OUTSTANDING_READ__SHIFT 0x00000000
+#define MC_HUB_MISC_STATUS__OUTSTANDING_RPB_READ_MASK 0x00000040L
+#define MC_HUB_MISC_STATUS__OUTSTANDING_RPB_READ__SHIFT 0x00000006
+#define MC_HUB_MISC_STATUS__OUTSTANDING_RPB_WRITE_MASK 0x00000080L
+#define MC_HUB_MISC_STATUS__OUTSTANDING_RPB_WRITE__SHIFT 0x00000007
+#define MC_HUB_MISC_STATUS__OUTSTANDING_WRITE_MASK 0x00000002L
+#define MC_HUB_MISC_STATUS__OUTSTANDING_WRITE__SHIFT 0x00000001
+#define MC_HUB_MISC_STATUS__READ_DEADLOCK_WARNING_MASK 0x00001000L
+#define MC_HUB_MISC_STATUS__READ_DEADLOCK_WARNING__SHIFT 0x0000000c
+#define MC_HUB_MISC_STATUS__RPB_BUSY_MASK 0x00000400L
+#define MC_HUB_MISC_STATUS__RPB_BUSY__SHIFT 0x0000000a
+#define MC_HUB_MISC_STATUS__WRITE_DEADLOCK_WARNING_MASK 0x00000800L
+#define MC_HUB_MISC_STATUS__WRITE_DEADLOCK_WARNING__SHIFT 0x0000000b
+#define MC_HUB_MISC_VM_CG__ENABLE_MASK 0x00040000L
+#define MC_HUB_MISC_VM_CG__ENABLE__SHIFT 0x00000012
+#define MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK 0x00080000L
+#define MC_HUB_MISC_VM_CG__MEM_LS_ENABLE__SHIFT 0x00000013
+#define MC_HUB_MISC_VM_CG__OFFDLY_MASK 0x00000fc0L
+#define MC_HUB_MISC_VM_CG__OFFDLY__SHIFT 0x00000006
+#define MC_HUB_MISC_VM_CG__ONDLY_MASK 0x0000003fL
+#define MC_HUB_MISC_VM_CG__ONDLY__SHIFT 0x00000000
+#define MC_HUB_MISC_VM_CG__RDYDLY_MASK 0x0003f000L
+#define MC_HUB_MISC_VM_CG__RDYDLY__SHIFT 0x0000000c
+#define MC_HUB_RDREQ_CNTL__BREAK_HDP_DEADLOCK_MASK 0x00000200L
+#define MC_HUB_RDREQ_CNTL__BREAK_HDP_DEADLOCK__SHIFT 0x00000009
+#define MC_HUB_RDREQ_CNTL__DEBUG_REG_MASK 0x0001fc00L
+#define MC_HUB_RDREQ_CNTL__DEBUG_REG__SHIFT 0x0000000a
+#define MC_HUB_RDREQ_CNTL__DISABLE_SELF_INIT_GBL0_MASK 0x00020000L
+#define MC_HUB_RDREQ_CNTL__DISABLE_SELF_INIT_GBL0__SHIFT 0x00000011
+#define MC_HUB_RDREQ_CNTL__DISABLE_SELF_INIT_GBL1_MASK 0x00040000L
+#define MC_HUB_RDREQ_CNTL__DISABLE_SELF_INIT_GBL1__SHIFT 0x00000012
+#define MC_HUB_RDREQ_CNTL__JUMPAHEAD_GBL0_MASK 0x00000004L
+#define MC_HUB_RDREQ_CNTL__JUMPAHEAD_GBL0__SHIFT 0x00000002
+#define MC_HUB_RDREQ_CNTL__JUMPAHEAD_GBL1_MASK 0x00000008L
+#define MC_HUB_RDREQ_CNTL__JUMPAHEAD_GBL1__SHIFT 0x00000003
+#define MC_HUB_RDREQ_CNTL__MCDW_STALL_MODE_MASK 0x00000020L
+#define MC_HUB_RDREQ_CNTL__MCDW_STALL_MODE__SHIFT 0x00000005
+#define MC_HUB_RDREQ_CNTL__MCDX_STALL_MODE_MASK 0x00000040L
+#define MC_HUB_RDREQ_CNTL__MCDX_STALL_MODE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_CNTL__MCDY_STALL_MODE_MASK 0x00000080L
+#define MC_HUB_RDREQ_CNTL__MCDY_STALL_MODE__SHIFT 0x00000007
+#define MC_HUB_RDREQ_CNTL__MCDZ_STALL_MODE_MASK 0x00000100L
+#define MC_HUB_RDREQ_CNTL__MCDZ_STALL_MODE__SHIFT 0x00000008
+#define MC_HUB_RDREQ_CNTL__OVERRIDE_STALL_ENABLE_MASK 0x00000010L
+#define MC_HUB_RDREQ_CNTL__OVERRIDE_STALL_ENABLE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_CNTL__PWRXPRESS_MODE_MASK 0x00080000L
+#define MC_HUB_RDREQ_CNTL__PWRXPRESS_MODE__SHIFT 0x00000013
+#define MC_HUB_RDREQ_CNTL__REMOTE_BLACKOUT_MASK 0x00000001L
+#define MC_HUB_RDREQ_CNTL__REMOTE_BLACKOUT__SHIFT 0x00000000
+#define MC_HUB_RDREQ_CREDITS2__STOR1_PRI_MASK 0x000000ffL
+#define MC_HUB_RDREQ_CREDITS2__STOR1_PRI__SHIFT 0x00000000
+#define MC_HUB_RDREQ_CREDITS__STOR0_MASK 0x00ff0000L
+#define MC_HUB_RDREQ_CREDITS__STOR0__SHIFT 0x00000010
+#define MC_HUB_RDREQ_CREDITS__STOR1_MASK 0xff000000L
+#define MC_HUB_RDREQ_CREDITS__STOR1__SHIFT 0x00000018
+#define MC_HUB_RDREQ_CREDITS__VM0_MASK 0x000000ffL
+#define MC_HUB_RDREQ_CREDITS__VM0__SHIFT 0x00000000
+#define MC_HUB_RDREQ_CREDITS__VM1_MASK 0x0000ff00L
+#define MC_HUB_RDREQ_CREDITS__VM1__SHIFT 0x00000008
+#define MC_HUB_RDREQ_DMIF__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_RDREQ_DMIF__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_DMIF__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_DMIF__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_DMIF__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_RDREQ_DMIF__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_DMIF_LIMIT__ENABLE_MASK 0x00000003L
+#define MC_HUB_RDREQ_DMIF_LIMIT__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_DMIF_LIMIT__LIMIT_COUNT_MASK 0x0000007cL
+#define MC_HUB_RDREQ_DMIF_LIMIT__LIMIT_COUNT__SHIFT 0x00000002
+#define MC_HUB_RDREQ_DMIF__MAXBURST_MASK 0x00000780L
+#define MC_HUB_RDREQ_DMIF__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_RDREQ_DMIF__PRESCALE_MASK 0x00000006L
+#define MC_HUB_RDREQ_DMIF__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_RDREQ_DMIF__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_RDREQ_DMIF__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_DMIF__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_RDREQ_DMIF__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_DMIF__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_RDREQ_DMIF__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_RDREQ_GBL0__STALL_THRESHOLD_MASK 0x000000ffL
+#define MC_HUB_RDREQ_GBL0__STALL_THRESHOLD__SHIFT 0x00000000
+#define MC_HUB_RDREQ_GBL1__STALL_THRESHOLD_MASK 0x000000ffL
+#define MC_HUB_RDREQ_GBL1__STALL_THRESHOLD__SHIFT 0x00000000
+#define MC_HUB_RDREQ_HDP__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_RDREQ_HDP__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_HDP__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_HDP__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_HDP__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_RDREQ_HDP__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_HDP__MAXBURST_MASK 0x00000780L
+#define MC_HUB_RDREQ_HDP__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_RDREQ_HDP__PRESCALE_MASK 0x00000006L
+#define MC_HUB_RDREQ_HDP__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_RDREQ_HDP__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_RDREQ_HDP__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_HDP__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_RDREQ_HDP__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_HDP__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_RDREQ_HDP__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_RDREQ_MCDW__ASK_CREDITS_MASK 0x0003f800L
+#define MC_HUB_RDREQ_MCDW__ASK_CREDITS__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_MCDW__BLACKOUT_EXEMPT_MASK 0x00000002L
+#define MC_HUB_RDREQ_MCDW__BLACKOUT_EXEMPT__SHIFT 0x00000001
+#define MC_HUB_RDREQ_MCDW__BUS_MASK 0x00000004L
+#define MC_HUB_RDREQ_MCDW__BUS__SHIFT 0x00000002
+#define MC_HUB_RDREQ_MCDW__DISPLAY_CREDITS_MASK 0x01fc0000L
+#define MC_HUB_RDREQ_MCDW__DISPLAY_CREDITS__SHIFT 0x00000012
+#define MC_HUB_RDREQ_MCDW__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_MCDW__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_MCDW__LAZY_TIMER_MASK 0x00000780L
+#define MC_HUB_RDREQ_MCDW__LAZY_TIMER__SHIFT 0x00000007
+#define MC_HUB_RDREQ_MCDW__MAXBURST_MASK 0x00000078L
+#define MC_HUB_RDREQ_MCDW__MAXBURST__SHIFT 0x00000003
+#define MC_HUB_RDREQ_MCDW__STALL_THRESHOLD_MASK 0xfe000000L
+#define MC_HUB_RDREQ_MCDW__STALL_THRESHOLD__SHIFT 0x00000019
+#define MC_HUB_RDREQ_MCDX__ASK_CREDITS_MASK 0x0003f800L
+#define MC_HUB_RDREQ_MCDX__ASK_CREDITS__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_MCDX__BLACKOUT_EXEMPT_MASK 0x00000002L
+#define MC_HUB_RDREQ_MCDX__BLACKOUT_EXEMPT__SHIFT 0x00000001
+#define MC_HUB_RDREQ_MCDX__BUS_MASK 0x00000004L
+#define MC_HUB_RDREQ_MCDX__BUS__SHIFT 0x00000002
+#define MC_HUB_RDREQ_MCDX__DISPLAY_CREDITS_MASK 0x01fc0000L
+#define MC_HUB_RDREQ_MCDX__DISPLAY_CREDITS__SHIFT 0x00000012
+#define MC_HUB_RDREQ_MCDX__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_MCDX__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_MCDX__LAZY_TIMER_MASK 0x00000780L
+#define MC_HUB_RDREQ_MCDX__LAZY_TIMER__SHIFT 0x00000007
+#define MC_HUB_RDREQ_MCDX__MAXBURST_MASK 0x00000078L
+#define MC_HUB_RDREQ_MCDX__MAXBURST__SHIFT 0x00000003
+#define MC_HUB_RDREQ_MCDX__STALL_THRESHOLD_MASK 0xfe000000L
+#define MC_HUB_RDREQ_MCDX__STALL_THRESHOLD__SHIFT 0x00000019
+#define MC_HUB_RDREQ_MCDY__ASK_CREDITS_MASK 0x0003f800L
+#define MC_HUB_RDREQ_MCDY__ASK_CREDITS__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_MCDY__BLACKOUT_EXEMPT_MASK 0x00000002L
+#define MC_HUB_RDREQ_MCDY__BLACKOUT_EXEMPT__SHIFT 0x00000001
+#define MC_HUB_RDREQ_MCDY__BUS_MASK 0x00000004L
+#define MC_HUB_RDREQ_MCDY__BUS__SHIFT 0x00000002
+#define MC_HUB_RDREQ_MCDY__DISPLAY_CREDITS_MASK 0x01fc0000L
+#define MC_HUB_RDREQ_MCDY__DISPLAY_CREDITS__SHIFT 0x00000012
+#define MC_HUB_RDREQ_MCDY__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_MCDY__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_MCDY__LAZY_TIMER_MASK 0x00000780L
+#define MC_HUB_RDREQ_MCDY__LAZY_TIMER__SHIFT 0x00000007
+#define MC_HUB_RDREQ_MCDY__MAXBURST_MASK 0x00000078L
+#define MC_HUB_RDREQ_MCDY__MAXBURST__SHIFT 0x00000003
+#define MC_HUB_RDREQ_MCDY__STALL_THRESHOLD_MASK 0xfe000000L
+#define MC_HUB_RDREQ_MCDY__STALL_THRESHOLD__SHIFT 0x00000019
+#define MC_HUB_RDREQ_MCDZ__ASK_CREDITS_MASK 0x0003f800L
+#define MC_HUB_RDREQ_MCDZ__ASK_CREDITS__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_MCDZ__BLACKOUT_EXEMPT_MASK 0x00000002L
+#define MC_HUB_RDREQ_MCDZ__BLACKOUT_EXEMPT__SHIFT 0x00000001
+#define MC_HUB_RDREQ_MCDZ__BUS_MASK 0x00000004L
+#define MC_HUB_RDREQ_MCDZ__BUS__SHIFT 0x00000002
+#define MC_HUB_RDREQ_MCDZ__DISPLAY_CREDITS_MASK 0x01fc0000L
+#define MC_HUB_RDREQ_MCDZ__DISPLAY_CREDITS__SHIFT 0x00000012
+#define MC_HUB_RDREQ_MCDZ__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_MCDZ__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_MCDZ__LAZY_TIMER_MASK 0x00000780L
+#define MC_HUB_RDREQ_MCDZ__LAZY_TIMER__SHIFT 0x00000007
+#define MC_HUB_RDREQ_MCDZ__MAXBURST_MASK 0x00000078L
+#define MC_HUB_RDREQ_MCDZ__MAXBURST__SHIFT 0x00000003
+#define MC_HUB_RDREQ_MCDZ__STALL_THRESHOLD_MASK 0xfe000000L
+#define MC_HUB_RDREQ_MCDZ__STALL_THRESHOLD__SHIFT 0x00000019
+#define MC_HUB_RDREQ_MCIF__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_RDREQ_MCIF__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_MCIF__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_MCIF__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_MCIF__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_RDREQ_MCIF__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_MCIF__MAXBURST_MASK 0x00000780L
+#define MC_HUB_RDREQ_MCIF__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_RDREQ_MCIF__PRESCALE_MASK 0x00000006L
+#define MC_HUB_RDREQ_MCIF__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_RDREQ_MCIF__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_RDREQ_MCIF__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_MCIF__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_RDREQ_MCIF__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_MCIF__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_RDREQ_MCIF__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_RDREQ_RLC__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_RDREQ_RLC__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_RLC__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_RLC__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_RLC__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_RDREQ_RLC__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_RLC__MAXBURST_MASK 0x00000780L
+#define MC_HUB_RDREQ_RLC__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_RDREQ_RLC__PRESCALE_MASK 0x00000006L
+#define MC_HUB_RDREQ_RLC__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_RDREQ_RLC__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_RDREQ_RLC__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_RLC__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_RDREQ_RLC__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_RLC__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_RDREQ_RLC__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_RDREQ_SEM__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_RDREQ_SEM__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_SEM__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_SEM__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_SEM__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_RDREQ_SEM__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_SEM__MAXBURST_MASK 0x00000780L
+#define MC_HUB_RDREQ_SEM__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_RDREQ_SEM__PRESCALE_MASK 0x00000006L
+#define MC_HUB_RDREQ_SEM__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_RDREQ_SEM__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_RDREQ_SEM__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_SEM__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_RDREQ_SEM__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_SEM__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_RDREQ_SEM__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_RDREQ_SIP__ASK_CREDITS_MASK 0x0000007fL
+#define MC_HUB_RDREQ_SIP__ASK_CREDITS__SHIFT 0x00000000
+#define MC_HUB_RDREQ_SIP__DISPLAY_CREDITS_MASK 0x00007f00L
+#define MC_HUB_RDREQ_SIP__DISPLAY_CREDITS__SHIFT 0x00000008
+#define MC_HUB_RDREQ_SIP__DUMMY_MASK 0x00000080L
+#define MC_HUB_RDREQ_SIP__DUMMY__SHIFT 0x00000007
+#define MC_HUB_RDREQ_SMU__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_RDREQ_SMU__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_SMU__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_SMU__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_SMU__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_RDREQ_SMU__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_SMU__MAXBURST_MASK 0x00000780L
+#define MC_HUB_RDREQ_SMU__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_RDREQ_SMU__PRESCALE_MASK 0x00000006L
+#define MC_HUB_RDREQ_SMU__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_RDREQ_SMU__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_RDREQ_SMU__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_SMU__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_RDREQ_SMU__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_SMU__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_RDREQ_SMU__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_RDREQ_STATUS__GBL0_BYPASS_STOR_FULL_MASK 0x00000080L
+#define MC_HUB_RDREQ_STATUS__GBL0_BYPASS_STOR_FULL__SHIFT 0x00000007
+#define MC_HUB_RDREQ_STATUS__GBL0_STOR_FULL_MASK 0x00000040L
+#define MC_HUB_RDREQ_STATUS__GBL0_STOR_FULL__SHIFT 0x00000006
+#define MC_HUB_RDREQ_STATUS__GBL0_VM_FULL_MASK 0x00000020L
+#define MC_HUB_RDREQ_STATUS__GBL0_VM_FULL__SHIFT 0x00000005
+#define MC_HUB_RDREQ_STATUS__GBL1_BYPASS_STOR_FULL_MASK 0x00000400L
+#define MC_HUB_RDREQ_STATUS__GBL1_BYPASS_STOR_FULL__SHIFT 0x0000000a
+#define MC_HUB_RDREQ_STATUS__GBL1_STOR_FULL_MASK 0x00000200L
+#define MC_HUB_RDREQ_STATUS__GBL1_STOR_FULL__SHIFT 0x00000009
+#define MC_HUB_RDREQ_STATUS__GBL1_VM_FULL_MASK 0x00000100L
+#define MC_HUB_RDREQ_STATUS__GBL1_VM_FULL__SHIFT 0x00000008
+#define MC_HUB_RDREQ_STATUS__MCDW_RD_AVAIL_MASK 0x00000002L
+#define MC_HUB_RDREQ_STATUS__MCDW_RD_AVAIL__SHIFT 0x00000001
+#define MC_HUB_RDREQ_STATUS__MCDX_RD_AVAIL_MASK 0x00000004L
+#define MC_HUB_RDREQ_STATUS__MCDX_RD_AVAIL__SHIFT 0x00000002
+#define MC_HUB_RDREQ_STATUS__MCDY_RD_AVAIL_MASK 0x00000008L
+#define MC_HUB_RDREQ_STATUS__MCDY_RD_AVAIL__SHIFT 0x00000003
+#define MC_HUB_RDREQ_STATUS__MCDZ_RD_AVAIL_MASK 0x00000010L
+#define MC_HUB_RDREQ_STATUS__MCDZ_RD_AVAIL__SHIFT 0x00000004
+#define MC_HUB_RDREQ_STATUS__PWRXPRESS_ERR_MASK 0x00000800L
+#define MC_HUB_RDREQ_STATUS__PWRXPRESS_ERR__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_STATUS__SIP_AVAIL_MASK 0x00000001L
+#define MC_HUB_RDREQ_STATUS__SIP_AVAIL__SHIFT 0x00000000
+#define MC_HUB_RDREQ_UMC__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_RDREQ_UMC__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_UMC__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_UMC__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_UMC__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_RDREQ_UMC__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_UMC__MAXBURST_MASK 0x00000780L
+#define MC_HUB_RDREQ_UMC__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_RDREQ_UMC__PRESCALE_MASK 0x00000006L
+#define MC_HUB_RDREQ_UMC__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_RDREQ_UMC__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_RDREQ_UMC__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_UMC__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_RDREQ_UMC__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_UMC__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_RDREQ_UMC__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_RDREQ_UVD__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_RDREQ_UVD__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_UVD__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_UVD__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_UVD__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_RDREQ_UVD__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_UVD__MAXBURST_MASK 0x00000780L
+#define MC_HUB_RDREQ_UVD__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_RDREQ_UVD__PRESCALE_MASK 0x00000006L
+#define MC_HUB_RDREQ_UVD__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_RDREQ_UVD__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_RDREQ_UVD__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_UVD__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_RDREQ_UVD__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_UVD__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_RDREQ_UVD__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_RDREQ_UVD__VM_BYPASS_MASK 0x00010000L
+#define MC_HUB_RDREQ_UVD__VM_BYPASS__SHIFT 0x00000010
+#define MC_HUB_RDREQ_VCE__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_RDREQ_VCE__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_VCE__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_VCE__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_VCE__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_RDREQ_VCE__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_VCE__MAXBURST_MASK 0x00000780L
+#define MC_HUB_RDREQ_VCE__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_RDREQ_VCE__PRESCALE_MASK 0x00000006L
+#define MC_HUB_RDREQ_VCE__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_RDREQ_VCE__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_RDREQ_VCE__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_VCE__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_RDREQ_VCE__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_VCE__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_RDREQ_VCE__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_RDREQ_VCEU__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_RDREQ_VCEU__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_VCEU__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_VCEU__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_VCEU__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_RDREQ_VCEU__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_VCEU__MAXBURST_MASK 0x00000780L
+#define MC_HUB_RDREQ_VCEU__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_RDREQ_VCEU__PRESCALE_MASK 0x00000006L
+#define MC_HUB_RDREQ_VCEU__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_RDREQ_VCEU__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_RDREQ_VCEU__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_VCEU__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_RDREQ_VCEU__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_VCEU__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_RDREQ_VCEU__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_RDREQ_VMC__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_RDREQ_VMC__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_VMC__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_VMC__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_VMC__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_RDREQ_VMC__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_VMC__MAXBURST_MASK 0x00000780L
+#define MC_HUB_RDREQ_VMC__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_RDREQ_VMC__PRESCALE_MASK 0x00000006L
+#define MC_HUB_RDREQ_VMC__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_RDREQ_VMC__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_RDREQ_VMC__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_VMC__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_RDREQ_VMC__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_VMC__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_RDREQ_VMC__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP0_DECREMENT_MASK 0x00000007L
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP0_DECREMENT__SHIFT 0x00000000
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP1_DECREMENT_MASK 0x00000038L
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP1_DECREMENT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP2_DECREMENT_MASK 0x000001c0L
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP2_DECREMENT__SHIFT 0x00000006
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP3_DECREMENT_MASK 0x00000e00L
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP3_DECREMENT__SHIFT 0x00000009
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP4_DECREMENT_MASK 0x00007000L
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP4_DECREMENT__SHIFT 0x0000000c
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP5_DECREMENT_MASK 0x00038000L
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP5_DECREMENT__SHIFT 0x0000000f
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP6_DECREMENT_MASK 0x001c0000L
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP6_DECREMENT__SHIFT 0x00000012
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP7_DECREMENT_MASK 0x00e00000L
+#define MC_HUB_RDREQ_WTM_CNTL__GROUP7_DECREMENT__SHIFT 0x00000015
+#define MC_HUB_RDREQ_XDMAM__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_RDREQ_XDMAM__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_RDREQ_XDMAM__ENABLE_MASK 0x00000001L
+#define MC_HUB_RDREQ_XDMAM__ENABLE__SHIFT 0x00000000
+#define MC_HUB_RDREQ_XDMAM__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_RDREQ_XDMAM__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_RDREQ_XDMAM__MAXBURST_MASK 0x00000780L
+#define MC_HUB_RDREQ_XDMAM__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_RDREQ_XDMAM__PRESCALE_MASK 0x00000006L
+#define MC_HUB_RDREQ_XDMAM__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_RDREQ_XDMAM__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_RDREQ_XDMAM__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_RDREQ_XDMAM__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_RDREQ_XDMAM__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_RDREQ_XDMAM__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_RDREQ_XDMAM__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_SHARED_DAGB_DLY__CLI_MASK 0x001f0000L
+#define MC_HUB_SHARED_DAGB_DLY__CLI__SHIFT 0x00000010
+#define MC_HUB_SHARED_DAGB_DLY__DLY_MASK 0x0000003fL
+#define MC_HUB_SHARED_DAGB_DLY__DLY__SHIFT 0x00000000
+#define MC_HUB_SHARED_DAGB_DLY__POS_MASK 0x1f000000L
+#define MC_HUB_SHARED_DAGB_DLY__POS__SHIFT 0x00000018
+#define MC_HUB_WDP_BP__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_BP__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_BP__RDRET_MASK 0x0003fffeL
+#define MC_HUB_WDP_BP__RDRET__SHIFT 0x00000001
+#define MC_HUB_WDP_BP__WRREQ_MASK 0x3ffc0000L
+#define MC_HUB_WDP_BP__WRREQ__SHIFT 0x00000012
+#define MC_HUB_WDP_CNTL__DEBUG_REG_MASK 0x00001fe0L
+#define MC_HUB_WDP_CNTL__DEBUG_REG__SHIFT 0x00000005
+#define MC_HUB_WDP_CNTL__DISABLE_SELF_INIT_GBL0_MASK 0x00002000L
+#define MC_HUB_WDP_CNTL__DISABLE_SELF_INIT_GBL0__SHIFT 0x0000000d
+#define MC_HUB_WDP_CNTL__DISABLE_SELF_INIT_GBL1_MASK 0x00004000L
+#define MC_HUB_WDP_CNTL__DISABLE_SELF_INIT_GBL1__SHIFT 0x0000000e
+#define MC_HUB_WDP_CNTL__DISABLE_SELF_INIT_INTERNAL_MASK 0x00008000L
+#define MC_HUB_WDP_CNTL__DISABLE_SELF_INIT_INTERNAL__SHIFT 0x0000000f
+#define MC_HUB_WDP_CNTL__DISP_WAIT_EOP_MASK 0x00040000L
+#define MC_HUB_WDP_CNTL__DISP_WAIT_EOP__SHIFT 0x00000012
+#define MC_HUB_WDP_CNTL__FAIR_CH_SW_MASK 0x00010000L
+#define MC_HUB_WDP_CNTL__FAIR_CH_SW__SHIFT 0x00000010
+#define MC_HUB_WDP_CNTL__JUMPAHEAD_GBL0_MASK 0x00000002L
+#define MC_HUB_WDP_CNTL__JUMPAHEAD_GBL0__SHIFT 0x00000001
+#define MC_HUB_WDP_CNTL__JUMPAHEAD_GBL1_MASK 0x00000004L
+#define MC_HUB_WDP_CNTL__JUMPAHEAD_GBL1__SHIFT 0x00000002
+#define MC_HUB_WDP_CNTL__JUMPAHEAD_INTERNAL_MASK 0x00000008L
+#define MC_HUB_WDP_CNTL__JUMPAHEAD_INTERNAL__SHIFT 0x00000003
+#define MC_HUB_WDP_CNTL__LCLWRREQ_BYPASS_MASK 0x00020000L
+#define MC_HUB_WDP_CNTL__LCLWRREQ_BYPASS__SHIFT 0x00000011
+#define MC_HUB_WDP_CNTL__MCD_WAIT_EOP_MASK 0x00080000L
+#define MC_HUB_WDP_CNTL__MCD_WAIT_EOP__SHIFT 0x00000013
+#define MC_HUB_WDP_CNTL__OVERRIDE_STALL_ENABLE_MASK 0x00000010L
+#define MC_HUB_WDP_CNTL__OVERRIDE_STALL_ENABLE__SHIFT 0x00000004
+#define MC_HUB_WDP_CNTL__SIP_WAIT_EOP_MASK 0x00100000L
+#define MC_HUB_WDP_CNTL__SIP_WAIT_EOP__SHIFT 0x00000014
+#define MC_HUB_WDP_CREDITS__STOR0_MASK 0x00ff0000L
+#define MC_HUB_WDP_CREDITS__STOR0__SHIFT 0x00000010
+#define MC_HUB_WDP_CREDITS__STOR1_MASK 0xff000000L
+#define MC_HUB_WDP_CREDITS__STOR1__SHIFT 0x00000018
+#define MC_HUB_WDP_CREDITS__VM0_MASK 0x000000ffL
+#define MC_HUB_WDP_CREDITS__VM0__SHIFT 0x00000000
+#define MC_HUB_WDP_CREDITS__VM1_MASK 0x0000ff00L
+#define MC_HUB_WDP_CREDITS__VM1__SHIFT 0x00000008
+#define MC_HUB_WDP_ERR__MGPU1_TARG_SYS_MASK 0x00000001L
+#define MC_HUB_WDP_ERR__MGPU1_TARG_SYS__SHIFT 0x00000000
+#define MC_HUB_WDP_ERR__MGPU2_TARG_SYS_MASK 0x00000002L
+#define MC_HUB_WDP_ERR__MGPU2_TARG_SYS__SHIFT 0x00000001
+#define MC_HUB_WDP_GBL0__LAZY_TIMER_MASK 0x000000f0L
+#define MC_HUB_WDP_GBL0__LAZY_TIMER__SHIFT 0x00000004
+#define MC_HUB_WDP_GBL0__MAXBURST_MASK 0x0000000fL
+#define MC_HUB_WDP_GBL0__MAXBURST__SHIFT 0x00000000
+#define MC_HUB_WDP_GBL0__STALL_MODE_MASK 0x00010000L
+#define MC_HUB_WDP_GBL0__STALL_MODE__SHIFT 0x00000010
+#define MC_HUB_WDP_GBL0__STALL_THRESHOLD_MASK 0x0000ff00L
+#define MC_HUB_WDP_GBL0__STALL_THRESHOLD__SHIFT 0x00000008
+#define MC_HUB_WDP_GBL1__LAZY_TIMER_MASK 0x000000f0L
+#define MC_HUB_WDP_GBL1__LAZY_TIMER__SHIFT 0x00000004
+#define MC_HUB_WDP_GBL1__MAXBURST_MASK 0x0000000fL
+#define MC_HUB_WDP_GBL1__MAXBURST__SHIFT 0x00000000
+#define MC_HUB_WDP_GBL1__STALL_MODE_MASK 0x00010000L
+#define MC_HUB_WDP_GBL1__STALL_MODE__SHIFT 0x00000010
+#define MC_HUB_WDP_GBL1__STALL_THRESHOLD_MASK 0x0000ff00L
+#define MC_HUB_WDP_GBL1__STALL_THRESHOLD__SHIFT 0x00000008
+#define MC_HUB_WDP_HDP__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_HDP__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_HDP__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_HDP__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_HDP__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_HDP__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_HDP__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_HDP__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_HDP__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_HDP__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_HDP__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_HDP__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_HDP__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_HDP__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_HDP__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_HDP__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_IH__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_IH__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_IH__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_IH__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_IH__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_IH__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_IH__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_IH__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_IH__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_IH__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_IH__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_IH__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_IH__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_IH__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_IH__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_IH__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_MCDW__ASK_CREDITS_MASK 0x00001f80L
+#define MC_HUB_WDP_MCDW__ASK_CREDITS__SHIFT 0x00000007
+#define MC_HUB_WDP_MCDW__ASK_CREDITS_W_MASK 0x7f000000L
+#define MC_HUB_WDP_MCDW__ASK_CREDITS_W__SHIFT 0x00000018
+#define MC_HUB_WDP_MCDW__BLACKOUT_EXEMPT_MASK 0x00000002L
+#define MC_HUB_WDP_MCDW__BLACKOUT_EXEMPT__SHIFT 0x00000001
+#define MC_HUB_WDP_MCDW__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_MCDW__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_MCDW__LAZY_TIMER_MASK 0x0001e000L
+#define MC_HUB_WDP_MCDW__LAZY_TIMER__SHIFT 0x0000000d
+#define MC_HUB_WDP_MCDW__MAXBURST_MASK 0x00000078L
+#define MC_HUB_WDP_MCDW__MAXBURST__SHIFT 0x00000003
+#define MC_HUB_WDP_MCDW__STALL_MODE_MASK 0x00000004L
+#define MC_HUB_WDP_MCDW__STALL_MODE__SHIFT 0x00000002
+#define MC_HUB_WDP_MCDW__STALL_THRESHOLD_MASK 0x00fe0000L
+#define MC_HUB_WDP_MCDW__STALL_THRESHOLD__SHIFT 0x00000011
+#define MC_HUB_WDP_MCDX__ASK_CREDITS_MASK 0x00001f80L
+#define MC_HUB_WDP_MCDX__ASK_CREDITS__SHIFT 0x00000007
+#define MC_HUB_WDP_MCDX__ASK_CREDITS_W_MASK 0x7f000000L
+#define MC_HUB_WDP_MCDX__ASK_CREDITS_W__SHIFT 0x00000018
+#define MC_HUB_WDP_MCDX__BLACKOUT_EXEMPT_MASK 0x00000002L
+#define MC_HUB_WDP_MCDX__BLACKOUT_EXEMPT__SHIFT 0x00000001
+#define MC_HUB_WDP_MCDX__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_MCDX__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_MCDX__LAZY_TIMER_MASK 0x0001e000L
+#define MC_HUB_WDP_MCDX__LAZY_TIMER__SHIFT 0x0000000d
+#define MC_HUB_WDP_MCDX__MAXBURST_MASK 0x00000078L
+#define MC_HUB_WDP_MCDX__MAXBURST__SHIFT 0x00000003
+#define MC_HUB_WDP_MCDX__STALL_MODE_MASK 0x00000004L
+#define MC_HUB_WDP_MCDX__STALL_MODE__SHIFT 0x00000002
+#define MC_HUB_WDP_MCDX__STALL_THRESHOLD_MASK 0x00fe0000L
+#define MC_HUB_WDP_MCDX__STALL_THRESHOLD__SHIFT 0x00000011
+#define MC_HUB_WDP_MCDY__ASK_CREDITS_MASK 0x00001f80L
+#define MC_HUB_WDP_MCDY__ASK_CREDITS__SHIFT 0x00000007
+#define MC_HUB_WDP_MCDY__ASK_CREDITS_W_MASK 0x7f000000L
+#define MC_HUB_WDP_MCDY__ASK_CREDITS_W__SHIFT 0x00000018
+#define MC_HUB_WDP_MCDY__BLACKOUT_EXEMPT_MASK 0x00000002L
+#define MC_HUB_WDP_MCDY__BLACKOUT_EXEMPT__SHIFT 0x00000001
+#define MC_HUB_WDP_MCDY__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_MCDY__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_MCDY__LAZY_TIMER_MASK 0x0001e000L
+#define MC_HUB_WDP_MCDY__LAZY_TIMER__SHIFT 0x0000000d
+#define MC_HUB_WDP_MCDY__MAXBURST_MASK 0x00000078L
+#define MC_HUB_WDP_MCDY__MAXBURST__SHIFT 0x00000003
+#define MC_HUB_WDP_MCDY__STALL_MODE_MASK 0x00000004L
+#define MC_HUB_WDP_MCDY__STALL_MODE__SHIFT 0x00000002
+#define MC_HUB_WDP_MCDY__STALL_THRESHOLD_MASK 0x00fe0000L
+#define MC_HUB_WDP_MCDY__STALL_THRESHOLD__SHIFT 0x00000011
+#define MC_HUB_WDP_MCDZ__ASK_CREDITS_MASK 0x00001f80L
+#define MC_HUB_WDP_MCDZ__ASK_CREDITS__SHIFT 0x00000007
+#define MC_HUB_WDP_MCDZ__ASK_CREDITS_W_MASK 0x7f000000L
+#define MC_HUB_WDP_MCDZ__ASK_CREDITS_W__SHIFT 0x00000018
+#define MC_HUB_WDP_MCDZ__BLACKOUT_EXEMPT_MASK 0x00000002L
+#define MC_HUB_WDP_MCDZ__BLACKOUT_EXEMPT__SHIFT 0x00000001
+#define MC_HUB_WDP_MCDZ__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_MCDZ__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_MCDZ__LAZY_TIMER_MASK 0x0001e000L
+#define MC_HUB_WDP_MCDZ__LAZY_TIMER__SHIFT 0x0000000d
+#define MC_HUB_WDP_MCDZ__MAXBURST_MASK 0x00000078L
+#define MC_HUB_WDP_MCDZ__MAXBURST__SHIFT 0x00000003
+#define MC_HUB_WDP_MCDZ__STALL_MODE_MASK 0x00000004L
+#define MC_HUB_WDP_MCDZ__STALL_MODE__SHIFT 0x00000002
+#define MC_HUB_WDP_MCDZ__STALL_THRESHOLD_MASK 0x00fe0000L
+#define MC_HUB_WDP_MCDZ__STALL_THRESHOLD__SHIFT 0x00000011
+#define MC_HUB_WDP_MCIF__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_MCIF__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_MCIF__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_MCIF__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_MCIF__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_MCIF__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_MCIF__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_MCIF__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_MCIF__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_MCIF__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_MCIF__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_MCIF__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_MCIF__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_MCIF__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_MCIF__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_MCIF__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_MGPU2__CID2_MASK 0x000000ffL
+#define MC_HUB_WDP_MGPU2__CID2__SHIFT 0x00000000
+#define MC_HUB_WDP_MGPU__CID_MASK 0x0000ff00L
+#define MC_HUB_WDP_MGPU__CID__SHIFT 0x00000008
+#define MC_HUB_WDP_MGPU__ENABLE_MASK 0x00800000L
+#define MC_HUB_WDP_MGPU__ENABLE__SHIFT 0x00000017
+#define MC_HUB_WDP_MGPU__MGPU_PRIORITY_TIME_MASK 0x007f0000L
+#define MC_HUB_WDP_MGPU__MGPU_PRIORITY_TIME__SHIFT 0x00000010
+#define MC_HUB_WDP_MGPU__OTH_PRIORITY_TIME_MASK 0x7f000000L
+#define MC_HUB_WDP_MGPU__OTH_PRIORITY_TIME__SHIFT 0x00000018
+#define MC_HUB_WDP_MGPU__STOR_MASK 0x000000ffL
+#define MC_HUB_WDP_MGPU__STOR__SHIFT 0x00000000
+#define MC_HUB_WDP_RLC__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_RLC__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_RLC__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_RLC__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_RLC__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_RLC__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_RLC__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_RLC__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_RLC__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_RLC__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_RLC__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_RLC__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_RLC__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_RLC__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_RLC__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_RLC__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_SEM__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_SEM__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_SEM__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_SEM__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_SEM__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_SEM__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_SEM__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_SEM__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_SEM__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_SEM__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_SEM__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_SEM__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_SEM__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_SEM__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_SEM__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_SEM__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_SH0__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_SH0__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_SH0__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_SH0__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_SH0__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_SH0__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_SH0__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_SH0__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_SH0__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_SH0__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_SH0__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_SH0__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_SH0__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_SH0__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_SH0__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_SH0__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_SH1__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_SH1__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_SH1__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_SH1__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_SH1__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_SH1__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_SH1__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_SH1__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_SH1__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_SH1__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_SH1__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_SH1__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_SH1__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_SH1__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_SH1__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_SH1__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_SIP__ASK_CREDITS_MASK 0x000001fcL
+#define MC_HUB_WDP_SIP__ASK_CREDITS__SHIFT 0x00000002
+#define MC_HUB_WDP_SIP__STALL_MODE_MASK 0x00000003L
+#define MC_HUB_WDP_SIP__STALL_MODE__SHIFT 0x00000000
+#define MC_HUB_WDP_SMU__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_SMU__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_SMU__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_SMU__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_SMU__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_SMU__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_SMU__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_SMU__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_SMU__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_SMU__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_SMU__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_SMU__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_SMU__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_SMU__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_SMU__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_SMU__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_STATUS__GBL0_BYPASS_STOR_FULL_MASK 0x00000080L
+#define MC_HUB_WDP_STATUS__GBL0_BYPASS_STOR_FULL__SHIFT 0x00000007
+#define MC_HUB_WDP_STATUS__GBL0_STOR_FULL_MASK 0x00000040L
+#define MC_HUB_WDP_STATUS__GBL0_STOR_FULL__SHIFT 0x00000006
+#define MC_HUB_WDP_STATUS__GBL0_VM_FULL_MASK 0x00000020L
+#define MC_HUB_WDP_STATUS__GBL0_VM_FULL__SHIFT 0x00000005
+#define MC_HUB_WDP_STATUS__GBL1_BYPASS_STOR_FULL_MASK 0x00000400L
+#define MC_HUB_WDP_STATUS__GBL1_BYPASS_STOR_FULL__SHIFT 0x0000000a
+#define MC_HUB_WDP_STATUS__GBL1_STOR_FULL_MASK 0x00000200L
+#define MC_HUB_WDP_STATUS__GBL1_STOR_FULL__SHIFT 0x00000009
+#define MC_HUB_WDP_STATUS__GBL1_VM_FULL_MASK 0x00000100L
+#define MC_HUB_WDP_STATUS__GBL1_VM_FULL__SHIFT 0x00000008
+#define MC_HUB_WDP_STATUS__MCDW_RD_AVAIL_MASK 0x00000002L
+#define MC_HUB_WDP_STATUS__MCDW_RD_AVAIL__SHIFT 0x00000001
+#define MC_HUB_WDP_STATUS__MCDW_WR_AVAIL_MASK 0x00000800L
+#define MC_HUB_WDP_STATUS__MCDW_WR_AVAIL__SHIFT 0x0000000b
+#define MC_HUB_WDP_STATUS__MCDX_RD_AVAIL_MASK 0x00000004L
+#define MC_HUB_WDP_STATUS__MCDX_RD_AVAIL__SHIFT 0x00000002
+#define MC_HUB_WDP_STATUS__MCDX_WR_AVAIL_MASK 0x00001000L
+#define MC_HUB_WDP_STATUS__MCDX_WR_AVAIL__SHIFT 0x0000000c
+#define MC_HUB_WDP_STATUS__MCDY_RD_AVAIL_MASK 0x00000008L
+#define MC_HUB_WDP_STATUS__MCDY_RD_AVAIL__SHIFT 0x00000003
+#define MC_HUB_WDP_STATUS__MCDY_WR_AVAIL_MASK 0x00002000L
+#define MC_HUB_WDP_STATUS__MCDY_WR_AVAIL__SHIFT 0x0000000d
+#define MC_HUB_WDP_STATUS__MCDZ_RD_AVAIL_MASK 0x00000010L
+#define MC_HUB_WDP_STATUS__MCDZ_RD_AVAIL__SHIFT 0x00000004
+#define MC_HUB_WDP_STATUS__MCDZ_WR_AVAIL_MASK 0x00004000L
+#define MC_HUB_WDP_STATUS__MCDZ_WR_AVAIL__SHIFT 0x0000000e
+#define MC_HUB_WDP_STATUS__SIP_AVAIL_MASK 0x00000001L
+#define MC_HUB_WDP_STATUS__SIP_AVAIL__SHIFT 0x00000000
+#define MC_HUB_WDP_UMC__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_UMC__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_UMC__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_UMC__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_UMC__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_UMC__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_UMC__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_UMC__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_UMC__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_UMC__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_UMC__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_UMC__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_UMC__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_UMC__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_UMC__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_UMC__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_UVD__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_UVD__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_UVD__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_UVD__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_UVD__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_UVD__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_UVD__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_UVD__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_UVD__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_UVD__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_UVD__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_UVD__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_UVD__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_UVD__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_UVD__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_UVD__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_UVD__VM_BYPASS_MASK 0x00010000L
+#define MC_HUB_WDP_UVD__VM_BYPASS__SHIFT 0x00000010
+#define MC_HUB_WDP_VCE__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_VCE__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_VCE__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_VCE__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_VCE__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_VCE__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_VCE__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_VCE__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_VCE__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_VCE__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_VCE__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_VCE__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_VCE__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_VCE__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_VCE__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_VCE__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_VCEU__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_VCEU__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_VCEU__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_VCEU__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_VCEU__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_VCEU__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_VCEU__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_VCEU__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_VCEU__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_VCEU__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_VCEU__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_VCEU__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_VCEU__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_VCEU__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_VCEU__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_VCEU__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_WTM_CNTL__GROUP0_DECREMENT_MASK 0x00000007L
+#define MC_HUB_WDP_WTM_CNTL__GROUP0_DECREMENT__SHIFT 0x00000000
+#define MC_HUB_WDP_WTM_CNTL__GROUP1_DECREMENT_MASK 0x00000038L
+#define MC_HUB_WDP_WTM_CNTL__GROUP1_DECREMENT__SHIFT 0x00000003
+#define MC_HUB_WDP_WTM_CNTL__GROUP2_DECREMENT_MASK 0x000001c0L
+#define MC_HUB_WDP_WTM_CNTL__GROUP2_DECREMENT__SHIFT 0x00000006
+#define MC_HUB_WDP_WTM_CNTL__GROUP3_DECREMENT_MASK 0x00000e00L
+#define MC_HUB_WDP_WTM_CNTL__GROUP3_DECREMENT__SHIFT 0x00000009
+#define MC_HUB_WDP_WTM_CNTL__GROUP4_DECREMENT_MASK 0x00007000L
+#define MC_HUB_WDP_WTM_CNTL__GROUP4_DECREMENT__SHIFT 0x0000000c
+#define MC_HUB_WDP_WTM_CNTL__GROUP5_DECREMENT_MASK 0x00038000L
+#define MC_HUB_WDP_WTM_CNTL__GROUP5_DECREMENT__SHIFT 0x0000000f
+#define MC_HUB_WDP_WTM_CNTL__GROUP6_DECREMENT_MASK 0x001c0000L
+#define MC_HUB_WDP_WTM_CNTL__GROUP6_DECREMENT__SHIFT 0x00000012
+#define MC_HUB_WDP_WTM_CNTL__GROUP7_DECREMENT_MASK 0x00e00000L
+#define MC_HUB_WDP_WTM_CNTL__GROUP7_DECREMENT__SHIFT 0x00000015
+#define MC_HUB_WDP_XDMA__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_XDMA__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_XDMA__BYPASS_AVAIL_OVERRIDE_MASK 0x00010000L
+#define MC_HUB_WDP_XDMA__BYPASS_AVAIL_OVERRIDE__SHIFT 0x00000010
+#define MC_HUB_WDP_XDMA__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_XDMA__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_XDMA__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_XDMA__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_XDMA__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_XDMA__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_XDMAM__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_XDMAM__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_XDMAM__BYPASS_AVAIL_OVERRIDE_MASK 0x00010000L
+#define MC_HUB_WDP_XDMAM__BYPASS_AVAIL_OVERRIDE__SHIFT 0x00000010
+#define MC_HUB_WDP_XDMAM__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_XDMAM__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_XDMAM__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_XDMAM__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_XDMAM__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_XDMAM__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_XDMAM__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_XDMAM__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_XDMAM__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_XDMAM__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_XDMAM__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_XDMAM__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_XDMAM__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_XDMAM__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_XDMA__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_XDMA__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_XDMA__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_XDMA__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_XDMA__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_XDMA__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_XDMA__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_XDMA__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WDP_XDP__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_HUB_WDP_XDP__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_HUB_WDP_XDP__ENABLE_MASK 0x00000001L
+#define MC_HUB_WDP_XDP__ENABLE__SHIFT 0x00000000
+#define MC_HUB_WDP_XDP__LAZY_TIMER_MASK 0x00007800L
+#define MC_HUB_WDP_XDP__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_HUB_WDP_XDP__MAXBURST_MASK 0x00000780L
+#define MC_HUB_WDP_XDP__MAXBURST__SHIFT 0x00000007
+#define MC_HUB_WDP_XDP__PRESCALE_MASK 0x00000006L
+#define MC_HUB_WDP_XDP__PRESCALE__SHIFT 0x00000001
+#define MC_HUB_WDP_XDP__STALL_MODE_MASK 0x00000030L
+#define MC_HUB_WDP_XDP__STALL_MODE__SHIFT 0x00000004
+#define MC_HUB_WDP_XDP__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_HUB_WDP_XDP__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_HUB_WDP_XDP__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_HUB_WDP_XDP__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_HUB_WRRET_CNTL__BP_ENABLE_MASK 0x00200000L
+#define MC_HUB_WRRET_CNTL__BP_ENABLE__SHIFT 0x00000015
+#define MC_HUB_WRRET_CNTL__BP_MASK 0x001ffffeL
+#define MC_HUB_WRRET_CNTL__BP__SHIFT 0x00000001
+#define MC_HUB_WRRET_CNTL__DEBUG_REG_MASK 0x3fc00000L
+#define MC_HUB_WRRET_CNTL__DEBUG_REG__SHIFT 0x00000016
+#define MC_HUB_WRRET_CNTL__DISABLE_SELF_INIT_MASK 0x40000000L
+#define MC_HUB_WRRET_CNTL__DISABLE_SELF_INIT__SHIFT 0x0000001e
+#define MC_HUB_WRRET_CNTL__FAIR_CH_SW_MASK 0x80000000L
+#define MC_HUB_WRRET_CNTL__FAIR_CH_SW__SHIFT 0x0000001f
+#define MC_HUB_WRRET_CNTL__JUMPAHEAD_MASK 0x00000001L
+#define MC_HUB_WRRET_CNTL__JUMPAHEAD__SHIFT 0x00000000
+#define MC_HUB_WRRET_MCDW__CREDIT_COUNT_MASK 0x000000feL
+#define MC_HUB_WRRET_MCDW__CREDIT_COUNT__SHIFT 0x00000001
+#define MC_HUB_WRRET_MCDW__STALL_MODE_MASK 0x00000001L
+#define MC_HUB_WRRET_MCDW__STALL_MODE__SHIFT 0x00000000
+#define MC_HUB_WRRET_MCDX__CREDIT_COUNT_MASK 0x000000feL
+#define MC_HUB_WRRET_MCDX__CREDIT_COUNT__SHIFT 0x00000001
+#define MC_HUB_WRRET_MCDX__STALL_MODE_MASK 0x00000001L
+#define MC_HUB_WRRET_MCDX__STALL_MODE__SHIFT 0x00000000
+#define MC_HUB_WRRET_MCDY__CREDIT_COUNT_MASK 0x000000feL
+#define MC_HUB_WRRET_MCDY__CREDIT_COUNT__SHIFT 0x00000001
+#define MC_HUB_WRRET_MCDY__STALL_MODE_MASK 0x00000001L
+#define MC_HUB_WRRET_MCDY__STALL_MODE__SHIFT 0x00000000
+#define MC_HUB_WRRET_MCDZ__CREDIT_COUNT_MASK 0x000000feL
+#define MC_HUB_WRRET_MCDZ__CREDIT_COUNT__SHIFT 0x00000001
+#define MC_HUB_WRRET_MCDZ__STALL_MODE_MASK 0x00000001L
+#define MC_HUB_WRRET_MCDZ__STALL_MODE__SHIFT 0x00000000
+#define MC_HUB_WRRET_STATUS__MCDW_AVAIL_MASK 0x00000001L
+#define MC_HUB_WRRET_STATUS__MCDW_AVAIL__SHIFT 0x00000000
+#define MC_HUB_WRRET_STATUS__MCDX_AVAIL_MASK 0x00000002L
+#define MC_HUB_WRRET_STATUS__MCDX_AVAIL__SHIFT 0x00000001
+#define MC_HUB_WRRET_STATUS__MCDY_AVAIL_MASK 0x00000004L
+#define MC_HUB_WRRET_STATUS__MCDY_AVAIL__SHIFT 0x00000002
+#define MC_HUB_WRRET_STATUS__MCDZ_AVAIL_MASK 0x00000008L
+#define MC_HUB_WRRET_STATUS__MCDZ_AVAIL__SHIFT 0x00000003
+#define MC_IMP_CNTL__CAL_PWRON_MASK 0x80000000L
+#define MC_IMP_CNTL__CAL_PWRON__SHIFT 0x0000001f
+#define MC_IMP_CNTL__CAL_VREF_MASK 0x007f0000L
+#define MC_IMP_CNTL__CAL_VREFMODE_MASK 0x00000040L
+#define MC_IMP_CNTL__CAL_VREFMODE__SHIFT 0x00000006
+#define MC_IMP_CNTL__CAL_VREF_SEL_MASK 0x00000020L
+#define MC_IMP_CNTL__CAL_VREF_SEL__SHIFT 0x00000005
+#define MC_IMP_CNTL__CAL_VREF__SHIFT 0x00000010
+#define MC_IMP_CNTL__CAL_WHEN_IDLE_MASK 0x20000000L
+#define MC_IMP_CNTL__CAL_WHEN_IDLE__SHIFT 0x0000001d
+#define MC_IMP_CNTL__CAL_WHEN_REFRESH_MASK 0x40000000L
+#define MC_IMP_CNTL__CAL_WHEN_REFRESH__SHIFT 0x0000001e
+#define MC_IMP_CNTL__CLEAR_TIMEOUT_ERR_MASK 0x00000200L
+#define MC_IMP_CNTL__CLEAR_TIMEOUT_ERR__SHIFT 0x00000009
+#define MC_IMP_CNTL__MEM_IO_SAMPLE_CNT_MASK 0x0000e000L
+#define MC_IMP_CNTL__MEM_IO_SAMPLE_CNT__SHIFT 0x0000000d
+#define MC_IMP_CNTL__MEM_IO_UPDATE_RATE_MASK 0x0000001fL
+#define MC_IMP_CNTL__MEM_IO_UPDATE_RATE__SHIFT 0x00000000
+#define MC_IMP_CNTL__TIMEOUT_ERR_MASK 0x00000100L
+#define MC_IMP_CNTL__TIMEOUT_ERR__SHIFT 0x00000008
+#define MC_IMP_DEBUG__DEBUG_CAL_DONE_MASK 0x80000000L
+#define MC_IMP_DEBUG__DEBUG_CAL_DONE__SHIFT 0x0000001f
+#define MC_IMP_DEBUG__DEBUG_CAL_EN_MASK 0x10000000L
+#define MC_IMP_DEBUG__DEBUG_CAL_EN__SHIFT 0x0000001c
+#define MC_IMP_DEBUG__DEBUG_CAL_INTR_MASK 0x40000000L
+#define MC_IMP_DEBUG__DEBUG_CAL_INTR__SHIFT 0x0000001e
+#define MC_IMP_DEBUG__DEBUG_CAL_START_MASK 0x20000000L
+#define MC_IMP_DEBUG__DEBUG_CAL_START__SHIFT 0x0000001d
+#define MC_IMP_DEBUG__PMVCAL_RESERVED_MASK 0x0fff0000L
+#define MC_IMP_DEBUG__PMVCAL_RESERVED__SHIFT 0x00000010
+#define MC_IMP_DEBUG__TIMEOUT_CNTR_MASK 0x0000ff00L
+#define MC_IMP_DEBUG__TIMEOUT_CNTR__SHIFT 0x00000008
+#define MC_IMP_DEBUG__TSTARTUP_CNTR_MASK 0x000000ffL
+#define MC_IMP_DEBUG__TSTARTUP_CNTR__SHIFT 0x00000000
+#define MC_IMP_DQ_STATUS__CH0_DQ_NSTR_MASK 0x0000ff00L
+#define MC_IMP_DQ_STATUS__CH0_DQ_NSTR__SHIFT 0x00000008
+#define MC_IMP_DQ_STATUS__CH0_DQ_PSTR_MASK 0x000000ffL
+#define MC_IMP_DQ_STATUS__CH0_DQ_PSTR__SHIFT 0x00000000
+#define MC_IMP_DQ_STATUS__CH1_DQ_NSTR_MASK 0xff000000L
+#define MC_IMP_DQ_STATUS__CH1_DQ_NSTR__SHIFT 0x00000018
+#define MC_IMP_DQ_STATUS__CH1_DQ_PSTR_MASK 0x00ff0000L
+#define MC_IMP_DQ_STATUS__CH1_DQ_PSTR__SHIFT 0x00000010
+#define MC_IMP_STATUS__NSTR_ACCUM_VAL_MASK 0xff000000L
+#define MC_IMP_STATUS__NSTR_ACCUM_VAL__SHIFT 0x00000018
+#define MC_IMP_STATUS__NSTR_CAL_MASK 0x00ff0000L
+#define MC_IMP_STATUS__NSTR_CAL__SHIFT 0x00000010
+#define MC_IMP_STATUS__PSTR_ACCUM_VAL_MASK 0x0000ff00L
+#define MC_IMP_STATUS__PSTR_ACCUM_VAL__SHIFT 0x00000008
+#define MC_IMP_STATUS__PSTR_CAL_MASK 0x000000ffL
+#define MC_IMP_STATUS__PSTR_CAL__SHIFT 0x00000000
+#define MC_IO_APHY_STR_CNTL_D0__CAL_SEL_MASK 0x0c000000L
+#define MC_IO_APHY_STR_CNTL_D0__CAL_SEL__SHIFT 0x0000001a
+#define MC_IO_APHY_STR_CNTL_D0__LOAD_A_STR_MASK 0x10000000L
+#define MC_IO_APHY_STR_CNTL_D0__LOAD_A_STR__SHIFT 0x0000001c
+#define MC_IO_APHY_STR_CNTL_D0__LOAD_D_RD_STR_MASK 0x20000000L
+#define MC_IO_APHY_STR_CNTL_D0__LOAD_D_RD_STR__SHIFT 0x0000001d
+#define MC_IO_APHY_STR_CNTL_D0__NSTR_OFF_A_MASK 0x00000fc0L
+#define MC_IO_APHY_STR_CNTL_D0__NSTR_OFF_A__SHIFT 0x00000006
+#define MC_IO_APHY_STR_CNTL_D0__PSTR_OFF_A_MASK 0x0000003fL
+#define MC_IO_APHY_STR_CNTL_D0__PSTR_OFF_A__SHIFT 0x00000000
+#define MC_IO_APHY_STR_CNTL_D0__PSTR_OFF_D_RD_MASK 0x0003f000L
+#define MC_IO_APHY_STR_CNTL_D0__PSTR_OFF_D_RD__SHIFT 0x0000000c
+#define MC_IO_APHY_STR_CNTL_D0__USE_A_CAL_MASK 0x01000000L
+#define MC_IO_APHY_STR_CNTL_D0__USE_A_CAL__SHIFT 0x00000018
+#define MC_IO_APHY_STR_CNTL_D0__USE_D_RD_CAL_MASK 0x02000000L
+#define MC_IO_APHY_STR_CNTL_D0__USE_D_RD_CAL__SHIFT 0x00000019
+#define MC_IO_APHY_STR_CNTL_D1__CAL_SEL_MASK 0x0c000000L
+#define MC_IO_APHY_STR_CNTL_D1__CAL_SEL__SHIFT 0x0000001a
+#define MC_IO_APHY_STR_CNTL_D1__LOAD_A_STR_MASK 0x10000000L
+#define MC_IO_APHY_STR_CNTL_D1__LOAD_A_STR__SHIFT 0x0000001c
+#define MC_IO_APHY_STR_CNTL_D1__LOAD_D_RD_STR_MASK 0x20000000L
+#define MC_IO_APHY_STR_CNTL_D1__LOAD_D_RD_STR__SHIFT 0x0000001d
+#define MC_IO_APHY_STR_CNTL_D1__NSTR_OFF_A_MASK 0x00000fc0L
+#define MC_IO_APHY_STR_CNTL_D1__NSTR_OFF_A__SHIFT 0x00000006
+#define MC_IO_APHY_STR_CNTL_D1__PSTR_OFF_A_MASK 0x0000003fL
+#define MC_IO_APHY_STR_CNTL_D1__PSTR_OFF_A__SHIFT 0x00000000
+#define MC_IO_APHY_STR_CNTL_D1__PSTR_OFF_D_RD_MASK 0x0003f000L
+#define MC_IO_APHY_STR_CNTL_D1__PSTR_OFF_D_RD__SHIFT 0x0000000c
+#define MC_IO_APHY_STR_CNTL_D1__USE_A_CAL_MASK 0x01000000L
+#define MC_IO_APHY_STR_CNTL_D1__USE_A_CAL__SHIFT 0x00000018
+#define MC_IO_APHY_STR_CNTL_D1__USE_D_RD_CAL_MASK 0x02000000L
+#define MC_IO_APHY_STR_CNTL_D1__USE_D_RD_CAL__SHIFT 0x00000019
+#define MC_IO_CDRCNTL1_D0__DQ_RXPHASE_B0_MASK 0x000000ffL
+#define MC_IO_CDRCNTL1_D0__DQ_RXPHASE_B0__SHIFT 0x00000000
+#define MC_IO_CDRCNTL1_D0__DQ_RXPHASE_B1_MASK 0x0000ff00L
+#define MC_IO_CDRCNTL1_D0__DQ_RXPHASE_B1__SHIFT 0x00000008
+#define MC_IO_CDRCNTL1_D0__WCDR_TXPHASE_B0_MASK 0x00ff0000L
+#define MC_IO_CDRCNTL1_D0__WCDR_TXPHASE_B0__SHIFT 0x00000010
+#define MC_IO_CDRCNTL1_D0__WCDR_TXPHASE_B1_MASK 0xff000000L
+#define MC_IO_CDRCNTL1_D0__WCDR_TXPHASE_B1__SHIFT 0x00000018
+#define MC_IO_CDRCNTL1_D1__DQ_RXPHASE_B0_MASK 0x000000ffL
+#define MC_IO_CDRCNTL1_D1__DQ_RXPHASE_B0__SHIFT 0x00000000
+#define MC_IO_CDRCNTL1_D1__DQ_RXPHASE_B1_MASK 0x0000ff00L
+#define MC_IO_CDRCNTL1_D1__DQ_RXPHASE_B1__SHIFT 0x00000008
+#define MC_IO_CDRCNTL1_D1__WCDR_TXPHASE_B0_MASK 0x00ff0000L
+#define MC_IO_CDRCNTL1_D1__WCDR_TXPHASE_B0__SHIFT 0x00000010
+#define MC_IO_CDRCNTL1_D1__WCDR_TXPHASE_B1_MASK 0xff000000L
+#define MC_IO_CDRCNTL1_D1__WCDR_TXPHASE_B1__SHIFT 0x00000018
+#define MC_IO_CDRCNTL2_D0__CDR_FB_SEL0_MASK 0x00000001L
+#define MC_IO_CDRCNTL2_D0__CDR_FB_SEL0__SHIFT 0x00000000
+#define MC_IO_CDRCNTL2_D0__CDR_FB_SEL1_MASK 0x00000002L
+#define MC_IO_CDRCNTL2_D0__CDR_FB_SEL1__SHIFT 0x00000001
+#define MC_IO_CDRCNTL2_D0__EDC_RXEN_OVR0_MASK 0x00000004L
+#define MC_IO_CDRCNTL2_D0__EDC_RXEN_OVR0__SHIFT 0x00000002
+#define MC_IO_CDRCNTL2_D0__EDC_RXEN_OVR1_MASK 0x00000008L
+#define MC_IO_CDRCNTL2_D0__EDC_RXEN_OVR1__SHIFT 0x00000003
+#define MC_IO_CDRCNTL2_D0__TXCDRBYPASS0_MASK 0x00000010L
+#define MC_IO_CDRCNTL2_D0__TXCDRBYPASS0__SHIFT 0x00000004
+#define MC_IO_CDRCNTL2_D0__TXCDRBYPASS1_MASK 0x00000020L
+#define MC_IO_CDRCNTL2_D0__TXCDRBYPASS1__SHIFT 0x00000005
+#define MC_IO_CDRCNTL2_D0__WCK_RXEN_OVR0_MASK 0x00000040L
+#define MC_IO_CDRCNTL2_D0__WCK_RXEN_OVR0__SHIFT 0x00000006
+#define MC_IO_CDRCNTL2_D0__WCK_RXEN_OVR1_MASK 0x00000080L
+#define MC_IO_CDRCNTL2_D0__WCK_RXEN_OVR1__SHIFT 0x00000007
+#define MC_IO_CDRCNTL2_D1__CDR_FB_SEL0_MASK 0x00000001L
+#define MC_IO_CDRCNTL2_D1__CDR_FB_SEL0__SHIFT 0x00000000
+#define MC_IO_CDRCNTL2_D1__CDR_FB_SEL1_MASK 0x00000002L
+#define MC_IO_CDRCNTL2_D1__CDR_FB_SEL1__SHIFT 0x00000001
+#define MC_IO_CDRCNTL2_D1__EDC_RXEN_OVR0_MASK 0x00000004L
+#define MC_IO_CDRCNTL2_D1__EDC_RXEN_OVR0__SHIFT 0x00000002
+#define MC_IO_CDRCNTL2_D1__EDC_RXEN_OVR1_MASK 0x00000008L
+#define MC_IO_CDRCNTL2_D1__EDC_RXEN_OVR1__SHIFT 0x00000003
+#define MC_IO_CDRCNTL2_D1__TXCDRBYPASS0_MASK 0x00000010L
+#define MC_IO_CDRCNTL2_D1__TXCDRBYPASS0__SHIFT 0x00000004
+#define MC_IO_CDRCNTL2_D1__TXCDRBYPASS1_MASK 0x00000020L
+#define MC_IO_CDRCNTL2_D1__TXCDRBYPASS1__SHIFT 0x00000005
+#define MC_IO_CDRCNTL2_D1__WCK_RXEN_OVR0_MASK 0x00000040L
+#define MC_IO_CDRCNTL2_D1__WCK_RXEN_OVR0__SHIFT 0x00000006
+#define MC_IO_CDRCNTL2_D1__WCK_RXEN_OVR1_MASK 0x00000080L
+#define MC_IO_CDRCNTL2_D1__WCK_RXEN_OVR1__SHIFT 0x00000007
+#define MC_IO_CDRCNTL_D0__DQRXCDREN_B0_MASK 0x00400000L
+#define MC_IO_CDRCNTL_D0__DQRXCDREN_B0__SHIFT 0x00000016
+#define MC_IO_CDRCNTL_D0__DQRXCDREN_B1_MASK 0x00800000L
+#define MC_IO_CDRCNTL_D0__DQRXCDREN_B1__SHIFT 0x00000017
+#define MC_IO_CDRCNTL_D0__DQRXSEL_B0_MASK 0x10000000L
+#define MC_IO_CDRCNTL_D0__DQRXSEL_B0__SHIFT 0x0000001c
+#define MC_IO_CDRCNTL_D0__DQRXSEL_B1_MASK 0x20000000L
+#define MC_IO_CDRCNTL_D0__DQRXSEL_B1__SHIFT 0x0000001d
+#define MC_IO_CDRCNTL_D0__DQTXCDREN_B0_MASK 0x00100000L
+#define MC_IO_CDRCNTL_D0__DQTXCDREN_B0__SHIFT 0x00000014
+#define MC_IO_CDRCNTL_D0__DQTXCDREN_B1_MASK 0x00200000L
+#define MC_IO_CDRCNTL_D0__DQTXCDREN_B1__SHIFT 0x00000015
+#define MC_IO_CDRCNTL_D0__DQTXSEL_B0_MASK 0x40000000L
+#define MC_IO_CDRCNTL_D0__DQTXSEL_B0__SHIFT 0x0000001e
+#define MC_IO_CDRCNTL_D0__DQTXSEL_B1_MASK 0x80000000L
+#define MC_IO_CDRCNTL_D0__DQTXSEL_B1__SHIFT 0x0000001f
+#define MC_IO_CDRCNTL_D0__RXCDRBYPASS_B01_MASK 0x00000400L
+#define MC_IO_CDRCNTL_D0__RXCDRBYPASS_B01__SHIFT 0x0000000a
+#define MC_IO_CDRCNTL_D0__RXCDRBYPASS_B23_MASK 0x00000800L
+#define MC_IO_CDRCNTL_D0__RXCDRBYPASS_B23__SHIFT 0x0000000b
+#define MC_IO_CDRCNTL_D0__RXCDREN_B01_MASK 0x00000100L
+#define MC_IO_CDRCNTL_D0__RXCDREN_B01__SHIFT 0x00000008
+#define MC_IO_CDRCNTL_D0__RXCDREN_B23_MASK 0x00000200L
+#define MC_IO_CDRCNTL_D0__RXCDREN_B23__SHIFT 0x00000009
+#define MC_IO_CDRCNTL_D0__RXPHASE1_B01_MASK 0x0000f000L
+#define MC_IO_CDRCNTL_D0__RXPHASE1_B01__SHIFT 0x0000000c
+#define MC_IO_CDRCNTL_D0__RXPHASE1_B23_MASK 0x000f0000L
+#define MC_IO_CDRCNTL_D0__RXPHASE1_B23__SHIFT 0x00000010
+#define MC_IO_CDRCNTL_D0__RXPHASE_B01_MASK 0x0000000fL
+#define MC_IO_CDRCNTL_D0__RXPHASE_B01__SHIFT 0x00000000
+#define MC_IO_CDRCNTL_D0__RXPHASE_B23_MASK 0x000000f0L
+#define MC_IO_CDRCNTL_D0__RXPHASE_B23__SHIFT 0x00000004
+#define MC_IO_CDRCNTL_D0__WCDREDC_B0_MASK 0x04000000L
+#define MC_IO_CDRCNTL_D0__WCDREDC_B0__SHIFT 0x0000001a
+#define MC_IO_CDRCNTL_D0__WCDREDC_B1_MASK 0x08000000L
+#define MC_IO_CDRCNTL_D0__WCDREDC_B1__SHIFT 0x0000001b
+#define MC_IO_CDRCNTL_D0__WCDRRXCDREN_B0_MASK 0x01000000L
+#define MC_IO_CDRCNTL_D0__WCDRRXCDREN_B0__SHIFT 0x00000018
+#define MC_IO_CDRCNTL_D0__WCDRRXCDREN_B1_MASK 0x02000000L
+#define MC_IO_CDRCNTL_D0__WCDRRXCDREN_B1__SHIFT 0x00000019
+#define MC_IO_CDRCNTL_D1__DQRXCDREN_B0_MASK 0x00400000L
+#define MC_IO_CDRCNTL_D1__DQRXCDREN_B0__SHIFT 0x00000016
+#define MC_IO_CDRCNTL_D1__DQRXCDREN_B1_MASK 0x00800000L
+#define MC_IO_CDRCNTL_D1__DQRXCDREN_B1__SHIFT 0x00000017
+#define MC_IO_CDRCNTL_D1__DQRXSEL_B0_MASK 0x10000000L
+#define MC_IO_CDRCNTL_D1__DQRXSEL_B0__SHIFT 0x0000001c
+#define MC_IO_CDRCNTL_D1__DQRXSEL_B1_MASK 0x20000000L
+#define MC_IO_CDRCNTL_D1__DQRXSEL_B1__SHIFT 0x0000001d
+#define MC_IO_CDRCNTL_D1__DQTXCDREN_B0_MASK 0x00100000L
+#define MC_IO_CDRCNTL_D1__DQTXCDREN_B0__SHIFT 0x00000014
+#define MC_IO_CDRCNTL_D1__DQTXCDREN_B1_MASK 0x00200000L
+#define MC_IO_CDRCNTL_D1__DQTXCDREN_B1__SHIFT 0x00000015
+#define MC_IO_CDRCNTL_D1__DQTXSEL_B0_MASK 0x40000000L
+#define MC_IO_CDRCNTL_D1__DQTXSEL_B0__SHIFT 0x0000001e
+#define MC_IO_CDRCNTL_D1__DQTXSEL_B1_MASK 0x80000000L
+#define MC_IO_CDRCNTL_D1__DQTXSEL_B1__SHIFT 0x0000001f
+#define MC_IO_CDRCNTL_D1__RXCDRBYPASS_B01_MASK 0x00000400L
+#define MC_IO_CDRCNTL_D1__RXCDRBYPASS_B01__SHIFT 0x0000000a
+#define MC_IO_CDRCNTL_D1__RXCDRBYPASS_B23_MASK 0x00000800L
+#define MC_IO_CDRCNTL_D1__RXCDRBYPASS_B23__SHIFT 0x0000000b
+#define MC_IO_CDRCNTL_D1__RXCDREN_B01_MASK 0x00000100L
+#define MC_IO_CDRCNTL_D1__RXCDREN_B01__SHIFT 0x00000008
+#define MC_IO_CDRCNTL_D1__RXCDREN_B23_MASK 0x00000200L
+#define MC_IO_CDRCNTL_D1__RXCDREN_B23__SHIFT 0x00000009
+#define MC_IO_CDRCNTL_D1__RXPHASE1_B01_MASK 0x0000f000L
+#define MC_IO_CDRCNTL_D1__RXPHASE1_B01__SHIFT 0x0000000c
+#define MC_IO_CDRCNTL_D1__RXPHASE1_B23_MASK 0x000f0000L
+#define MC_IO_CDRCNTL_D1__RXPHASE1_B23__SHIFT 0x00000010
+#define MC_IO_CDRCNTL_D1__RXPHASE_B01_MASK 0x0000000fL
+#define MC_IO_CDRCNTL_D1__RXPHASE_B01__SHIFT 0x00000000
+#define MC_IO_CDRCNTL_D1__RXPHASE_B23_MASK 0x000000f0L
+#define MC_IO_CDRCNTL_D1__RXPHASE_B23__SHIFT 0x00000004
+#define MC_IO_CDRCNTL_D1__WCDREDC_B0_MASK 0x04000000L
+#define MC_IO_CDRCNTL_D1__WCDREDC_B0__SHIFT 0x0000001a
+#define MC_IO_CDRCNTL_D1__WCDREDC_B1_MASK 0x08000000L
+#define MC_IO_CDRCNTL_D1__WCDREDC_B1__SHIFT 0x0000001b
+#define MC_IO_CDRCNTL_D1__WCDRRXCDREN_B0_MASK 0x01000000L
+#define MC_IO_CDRCNTL_D1__WCDRRXCDREN_B0__SHIFT 0x00000018
+#define MC_IO_CDRCNTL_D1__WCDRRXCDREN_B1_MASK 0x02000000L
+#define MC_IO_CDRCNTL_D1__WCDRRXCDREN_B1__SHIFT 0x00000019
+#define MC_IO_DEBUG_ACMD_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ACMD_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ACMD_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ACMD_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ACMD_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ACMD_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ACMD_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ACMD_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ACMD_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRH_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRH_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRH_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRH_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRH_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRH_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRH_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRH_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_ADDRL_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_ADDRL_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_ADDRL_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_ADDRL_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_ADDRL_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_ADDRL_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_ADDRL_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_ADDRL_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CK_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CK_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CK_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CK_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CK_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CK_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CK_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CK_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_CMD_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_CMD_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_CMD_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_CMD_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_CMD_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_CMD_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_CMD_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_CMD_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_CDR_PHSIZE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_RX_VREF_CAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DBI_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DBI_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DBI_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DBI_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DBI_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DBI_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DBI_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DBI_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQ0_RX_DYN_PM_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQ0_RX_EQ_PM_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQ1_RX_DYN_PM_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQ1_RX_EQ_PM_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0_CDR_PHSIZE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_RX_VREF_CAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0H_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0H_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0H_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0H_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0H_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0H_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0H_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0H_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_RX_VREF_CAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB0L_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB0L_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB0L_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB0L_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB0L_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB0L_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB0L_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB0L_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1_CDR_PHSIZE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_RX_VREF_CAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1H_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1H_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1H_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1H_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1H_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1H_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1H_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1H_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_RX_VREF_CAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB1L_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB1L_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB1L_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB1L_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB1L_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB1L_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB1L_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB1L_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2_CDR_PHSIZE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_RX_VREF_CAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2H_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2H_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2H_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2H_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2H_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2H_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2H_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2H_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_RX_VREF_CAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB2L_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB2L_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB2L_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB2L_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB2L_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB2L_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB2L_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB2L_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3_CDR_PHSIZE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_RX_VREF_CAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3H_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3H_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3H_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3H_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3H_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3H_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3H_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3H_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_RX_VREF_CAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_DQB3L_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_DQB3L_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_DQB3L_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_DQB3L_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_DQB3L_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_DQB3L_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_DQB3L_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_DQB3L_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_CDR_PHSIZE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_RX_DYN_PM_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_RX_EQ_PM_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_RX_VREF_CAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_EDC_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_EDC_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_EDC_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_EDC_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_EDC_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_EDC_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_EDC_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_EDC_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_100__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_100__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_100__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_100__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_100__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_100__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_100__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_100__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_101__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_101__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_101__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_101__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_101__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_101__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_101__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_101__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_102__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_102__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_102__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_102__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_102__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_102__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_102__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_102__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_103__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_103__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_103__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_103__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_103__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_103__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_103__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_103__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_104__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_104__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_104__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_104__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_104__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_104__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_104__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_104__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_105__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_105__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_105__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_105__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_105__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_105__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_105__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_105__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_106__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_106__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_106__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_106__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_106__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_106__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_106__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_106__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_107__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_107__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_107__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_107__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_107__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_107__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_107__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_107__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_108__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_108__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_108__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_108__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_108__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_108__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_108__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_108__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_109__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_109__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_109__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_109__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_109__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_109__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_109__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_109__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_10__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_10__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_10__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_10__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_10__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_10__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_10__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_10__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_110__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_110__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_110__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_110__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_110__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_110__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_110__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_110__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_111__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_111__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_111__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_111__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_111__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_111__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_111__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_111__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_112__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_112__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_112__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_112__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_112__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_112__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_112__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_112__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_113__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_113__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_113__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_113__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_113__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_113__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_113__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_113__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_114__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_114__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_114__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_114__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_114__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_114__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_114__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_114__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_115__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_115__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_115__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_115__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_115__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_115__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_115__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_115__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_116__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_116__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_116__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_116__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_116__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_116__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_116__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_116__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_117__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_117__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_117__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_117__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_117__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_117__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_117__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_117__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_118__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_118__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_118__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_118__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_118__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_118__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_118__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_118__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_119__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_119__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_119__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_119__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_119__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_119__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_119__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_119__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_11__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_11__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_11__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_11__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_11__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_11__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_11__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_11__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_120__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_120__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_120__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_120__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_120__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_120__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_120__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_120__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_121__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_121__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_121__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_121__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_121__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_121__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_121__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_121__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_122__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_122__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_122__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_122__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_122__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_122__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_122__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_122__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_123__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_123__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_123__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_123__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_123__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_123__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_123__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_123__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_124__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_124__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_124__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_124__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_124__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_124__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_124__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_124__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_125__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_125__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_125__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_125__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_125__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_125__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_125__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_125__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_126__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_126__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_126__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_126__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_126__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_126__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_126__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_126__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_127__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_127__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_127__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_127__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_127__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_127__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_127__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_127__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_128__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_128__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_128__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_128__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_128__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_128__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_128__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_128__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_129__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_129__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_129__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_129__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_129__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_129__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_129__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_129__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_12__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_12__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_12__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_12__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_12__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_12__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_12__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_12__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_130__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_130__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_130__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_130__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_130__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_130__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_130__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_130__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_131__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_131__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_131__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_131__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_131__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_131__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_131__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_131__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_132__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_132__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_132__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_132__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_132__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_132__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_132__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_132__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_133__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_133__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_133__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_133__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_133__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_133__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_133__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_133__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_134__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_134__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_134__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_134__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_134__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_134__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_134__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_134__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_135__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_135__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_135__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_135__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_135__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_135__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_135__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_135__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_136__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_136__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_136__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_136__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_136__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_136__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_136__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_136__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_137__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_137__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_137__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_137__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_137__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_137__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_137__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_137__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_138__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_138__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_138__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_138__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_138__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_138__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_138__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_138__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_139__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_139__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_139__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_139__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_139__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_139__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_139__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_139__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_13__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_13__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_13__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_13__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_13__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_13__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_13__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_13__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_140__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_140__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_140__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_140__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_140__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_140__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_140__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_140__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_141__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_141__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_141__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_141__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_141__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_141__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_141__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_141__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_142__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_142__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_142__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_142__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_142__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_142__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_142__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_142__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_143__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_143__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_143__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_143__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_143__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_143__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_143__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_143__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_144__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_144__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_144__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_144__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_144__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_144__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_144__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_144__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_145__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_145__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_145__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_145__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_145__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_145__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_145__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_145__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_146__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_146__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_146__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_146__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_146__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_146__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_146__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_146__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_147__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_147__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_147__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_147__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_147__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_147__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_147__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_147__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_148__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_148__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_148__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_148__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_148__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_148__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_148__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_148__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_149__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_149__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_149__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_149__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_149__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_149__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_149__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_149__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_14__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_14__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_14__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_14__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_14__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_14__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_14__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_14__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_150__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_150__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_150__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_150__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_150__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_150__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_150__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_150__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_151__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_151__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_151__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_151__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_151__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_151__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_151__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_151__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_152__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_152__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_152__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_152__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_152__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_152__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_152__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_152__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_153__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_153__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_153__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_153__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_153__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_153__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_153__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_153__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_154__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_154__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_154__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_154__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_154__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_154__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_154__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_154__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_155__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_155__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_155__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_155__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_155__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_155__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_155__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_155__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_156__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_156__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_156__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_156__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_156__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_156__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_156__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_156__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_157__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_157__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_157__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_157__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_157__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_157__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_157__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_157__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_158__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_158__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_158__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_158__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_158__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_158__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_158__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_158__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_159__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_159__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_159__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_159__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_159__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_159__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_159__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_159__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_15__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_15__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_15__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_15__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_15__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_15__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_15__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_15__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_16__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_16__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_16__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_16__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_16__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_16__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_16__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_16__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_17__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_17__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_17__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_17__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_17__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_17__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_17__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_17__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_18__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_18__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_18__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_18__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_18__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_18__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_18__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_18__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_19__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_19__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_19__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_19__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_19__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_19__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_19__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_19__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_20__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_20__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_20__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_20__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_20__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_20__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_20__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_20__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_21__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_21__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_21__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_21__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_21__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_21__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_21__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_21__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_22__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_22__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_22__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_22__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_22__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_22__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_22__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_22__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_23__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_23__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_23__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_23__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_23__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_23__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_23__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_23__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_24__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_24__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_24__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_24__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_24__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_24__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_24__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_24__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_25__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_25__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_25__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_25__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_25__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_25__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_25__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_25__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_26__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_26__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_26__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_26__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_26__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_26__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_26__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_26__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_27__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_27__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_27__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_27__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_27__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_27__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_27__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_27__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_28__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_28__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_28__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_28__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_28__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_28__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_28__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_28__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_29__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_29__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_29__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_29__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_29__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_29__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_29__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_29__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_2__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_2__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_2__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_2__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_2__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_2__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_2__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_2__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_30__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_30__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_30__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_30__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_30__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_30__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_30__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_30__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_31__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_31__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_31__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_31__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_31__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_31__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_31__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_31__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_32__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_32__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_32__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_32__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_32__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_32__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_32__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_32__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_33__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_33__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_33__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_33__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_33__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_33__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_33__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_33__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_34__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_34__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_34__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_34__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_34__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_34__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_34__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_34__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_35__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_35__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_35__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_35__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_35__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_35__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_35__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_35__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_36__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_36__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_36__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_36__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_36__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_36__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_36__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_36__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_37__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_37__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_37__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_37__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_37__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_37__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_37__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_37__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_38__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_38__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_38__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_38__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_38__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_38__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_38__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_38__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_39__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_39__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_39__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_39__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_39__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_39__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_39__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_39__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_3__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_3__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_3__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_3__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_3__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_3__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_3__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_3__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_40__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_40__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_40__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_40__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_40__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_40__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_40__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_40__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_41__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_41__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_41__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_41__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_41__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_41__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_41__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_41__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_42__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_42__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_42__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_42__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_42__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_42__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_42__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_42__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_43__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_43__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_43__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_43__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_43__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_43__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_43__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_43__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_44__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_44__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_44__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_44__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_44__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_44__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_44__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_44__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_45__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_45__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_45__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_45__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_45__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_45__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_45__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_45__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_46__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_46__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_46__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_46__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_46__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_46__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_46__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_46__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_47__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_47__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_47__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_47__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_47__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_47__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_47__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_47__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_48__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_48__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_48__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_48__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_48__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_48__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_48__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_48__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_49__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_49__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_49__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_49__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_49__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_49__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_49__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_49__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_4__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_4__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_4__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_4__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_4__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_4__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_4__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_4__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_50__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_50__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_50__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_50__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_50__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_50__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_50__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_50__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_51__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_51__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_51__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_51__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_51__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_51__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_51__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_51__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_52__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_52__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_52__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_52__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_52__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_52__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_52__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_52__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_53__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_53__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_53__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_53__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_53__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_53__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_53__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_53__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_54__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_54__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_54__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_54__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_54__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_54__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_54__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_54__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_55__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_55__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_55__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_55__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_55__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_55__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_55__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_55__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_56__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_56__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_56__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_56__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_56__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_56__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_56__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_56__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_57__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_57__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_57__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_57__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_57__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_57__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_57__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_57__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_58__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_58__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_58__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_58__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_58__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_58__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_58__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_58__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_59__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_59__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_59__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_59__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_59__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_59__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_59__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_59__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_5__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_5__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_5__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_5__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_5__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_5__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_5__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_5__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_60__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_60__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_60__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_60__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_60__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_60__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_60__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_60__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_61__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_61__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_61__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_61__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_61__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_61__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_61__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_61__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_62__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_62__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_62__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_62__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_62__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_62__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_62__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_62__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_63__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_63__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_63__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_63__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_63__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_63__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_63__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_63__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_64__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_64__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_64__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_64__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_64__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_64__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_64__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_64__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_65__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_65__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_65__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_65__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_65__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_65__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_65__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_65__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_66__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_66__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_66__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_66__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_66__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_66__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_66__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_66__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_67__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_67__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_67__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_67__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_67__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_67__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_67__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_67__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_68__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_68__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_68__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_68__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_68__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_68__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_68__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_68__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_69__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_69__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_69__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_69__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_69__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_69__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_69__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_69__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_6__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_6__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_6__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_6__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_6__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_6__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_6__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_6__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_70__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_70__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_70__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_70__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_70__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_70__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_70__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_70__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_71__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_71__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_71__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_71__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_71__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_71__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_71__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_71__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_72__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_72__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_72__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_72__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_72__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_72__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_72__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_72__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_73__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_73__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_73__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_73__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_73__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_73__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_73__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_73__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_74__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_74__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_74__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_74__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_74__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_74__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_74__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_74__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_75__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_75__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_75__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_75__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_75__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_75__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_75__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_75__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_76__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_76__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_76__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_76__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_76__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_76__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_76__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_76__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_77__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_77__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_77__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_77__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_77__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_77__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_77__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_77__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_78__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_78__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_78__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_78__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_78__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_78__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_78__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_78__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_79__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_79__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_79__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_79__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_79__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_79__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_79__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_79__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_7__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_7__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_7__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_7__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_7__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_7__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_7__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_7__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_80__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_80__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_80__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_80__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_80__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_80__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_80__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_80__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_81__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_81__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_81__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_81__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_81__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_81__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_81__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_81__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_82__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_82__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_82__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_82__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_82__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_82__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_82__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_82__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_83__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_83__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_83__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_83__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_83__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_83__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_83__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_83__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_84__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_84__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_84__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_84__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_84__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_84__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_84__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_84__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_85__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_85__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_85__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_85__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_85__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_85__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_85__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_85__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_86__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_86__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_86__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_86__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_86__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_86__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_86__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_86__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_87__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_87__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_87__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_87__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_87__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_87__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_87__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_87__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_88__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_88__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_88__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_88__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_88__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_88__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_88__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_88__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_89__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_89__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_89__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_89__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_89__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_89__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_89__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_89__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_8__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_8__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_8__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_8__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_8__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_8__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_8__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_8__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_90__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_90__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_90__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_90__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_90__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_90__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_90__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_90__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_91__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_91__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_91__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_91__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_91__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_91__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_91__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_91__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_92__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_92__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_92__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_92__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_92__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_92__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_92__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_92__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_93__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_93__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_93__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_93__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_93__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_93__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_93__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_93__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_94__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_94__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_94__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_94__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_94__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_94__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_94__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_94__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_95__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_95__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_95__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_95__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_95__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_95__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_95__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_95__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_96__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_96__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_96__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_96__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_96__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_96__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_96__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_96__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_97__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_97__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_97__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_97__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_97__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_97__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_97__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_97__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_98__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_98__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_98__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_98__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_98__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_98__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_98__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_98__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_99__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_99__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_99__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_99__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_99__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_99__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_99__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_99__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_UP_9__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_UP_9__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_UP_9__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_UP_9__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_UP_9__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_UP_9__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_UP_9__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_UP_9__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_CDR_PHSIZE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_RX_DYN_PM_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_RX_EQ_PM_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_RX_VREF_CAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCDR_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCDR_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCDR_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCDR_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCDR_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCDR_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCDR_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCDR_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_CLKSEL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_CLKSEL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_CLKSEL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_CLKSEL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_CLKSEL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_CLKSEL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_CLKSEL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_CLKSEL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_CLKSEL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_CLKSEL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_CLKSEL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_CLKSEL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_CLKSEL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_CLKSEL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_CLKSEL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_CLKSEL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_MISC_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_MISC_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_MISC_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_MISC_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_MISC_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_MISC_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_MISC_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_MISC_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_MISC_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_MISC_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_MISC_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_MISC_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_MISC_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_MISC_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_MISC_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_MISC_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_OFSCAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_OFSCAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_OFSCAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_OFSCAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_OFSCAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_OFSCAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_OFSCAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_OFSCAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_OFSCAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_OFSCAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_OFSCAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_OFSCAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_OFSCAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_OFSCAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_OFSCAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_OFSCAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_RX_EQ_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_RX_EQ_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_RX_EQ_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_RX_EQ_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_RX_EQ_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_RX_EQ_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_RX_EQ_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_RX_EQ_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_RX_EQ_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_RX_EQ_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_RX_EQ_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_RX_EQ_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_RX_EQ_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_RX_EQ_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_RX_EQ_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_RX_EQ_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_RXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_RXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_RXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_RXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_RXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_RXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_RXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_RXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_RXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_RXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_RXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_RXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_RXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_RXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_RXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_RXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_RX_VREF_CAL_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_TXBST_PD_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_TXBST_PD_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_TXBST_PD_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_TXBST_PD_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_TXBST_PD_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_TXBST_PD_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_TXBST_PD_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_TXBST_PD_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_TXBST_PD_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_TXBST_PD_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_TXBST_PD_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_TXBST_PD_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_TXBST_PD_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_TXBST_PD_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_TXBST_PD_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_TXBST_PD_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_TXBST_PU_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_TXBST_PU_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_TXBST_PU_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_TXBST_PU_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_TXBST_PU_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_TXBST_PU_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_TXBST_PU_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_TXBST_PU_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_TXBST_PU_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_TXBST_PU_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_TXBST_PU_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_TXBST_PU_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_TXBST_PU_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_TXBST_PU_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_TXBST_PU_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_TXBST_PU_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_TXPHASE_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_TXPHASE_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_TXPHASE_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_TXPHASE_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_TXPHASE_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_TXPHASE_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_TXPHASE_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_TXPHASE_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_TXPHASE_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_TXPHASE_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_TXPHASE_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_TXPHASE_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_TXPHASE_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_TXPHASE_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_TXPHASE_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_TXPHASE_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_TXSLF_D0__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_TXSLF_D0__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_TXSLF_D0__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_TXSLF_D0__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_TXSLF_D0__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_TXSLF_D0__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_TXSLF_D0__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_TXSLF_D0__VALUE3__SHIFT 0x00000018
+#define MC_IO_DEBUG_WCK_TXSLF_D1__VALUE0_MASK 0x000000ffL
+#define MC_IO_DEBUG_WCK_TXSLF_D1__VALUE0__SHIFT 0x00000000
+#define MC_IO_DEBUG_WCK_TXSLF_D1__VALUE1_MASK 0x0000ff00L
+#define MC_IO_DEBUG_WCK_TXSLF_D1__VALUE1__SHIFT 0x00000008
+#define MC_IO_DEBUG_WCK_TXSLF_D1__VALUE2_MASK 0x00ff0000L
+#define MC_IO_DEBUG_WCK_TXSLF_D1__VALUE2__SHIFT 0x00000010
+#define MC_IO_DEBUG_WCK_TXSLF_D1__VALUE3_MASK 0xff000000L
+#define MC_IO_DEBUG_WCK_TXSLF_D1__VALUE3__SHIFT 0x00000018
+#define MC_IO_DPHY_STR_CNTL_D0__CAL_SEL_MASK 0x0c000000L
+#define MC_IO_DPHY_STR_CNTL_D0__CAL_SEL__SHIFT 0x0000001a
+#define MC_IO_DPHY_STR_CNTL_D0__LOAD_D_STR_MASK 0x10000000L
+#define MC_IO_DPHY_STR_CNTL_D0__LOAD_D_STR__SHIFT 0x0000001c
+#define MC_IO_DPHY_STR_CNTL_D0__LOAD_S_STR_MASK 0x20000000L
+#define MC_IO_DPHY_STR_CNTL_D0__LOAD_S_STR__SHIFT 0x0000001d
+#define MC_IO_DPHY_STR_CNTL_D0__NSTR_OFF_D_MASK 0x00000fc0L
+#define MC_IO_DPHY_STR_CNTL_D0__NSTR_OFF_D__SHIFT 0x00000006
+#define MC_IO_DPHY_STR_CNTL_D0__NSTR_OFF_S_MASK 0x00fc0000L
+#define MC_IO_DPHY_STR_CNTL_D0__NSTR_OFF_S__SHIFT 0x00000012
+#define MC_IO_DPHY_STR_CNTL_D0__PSTR_OFF_D_MASK 0x0000003fL
+#define MC_IO_DPHY_STR_CNTL_D0__PSTR_OFF_D__SHIFT 0x00000000
+#define MC_IO_DPHY_STR_CNTL_D0__PSTR_OFF_S_MASK 0x0003f000L
+#define MC_IO_DPHY_STR_CNTL_D0__PSTR_OFF_S__SHIFT 0x0000000c
+#define MC_IO_DPHY_STR_CNTL_D0__USE_D_CAL_MASK 0x01000000L
+#define MC_IO_DPHY_STR_CNTL_D0__USE_D_CAL__SHIFT 0x00000018
+#define MC_IO_DPHY_STR_CNTL_D0__USE_S_CAL_MASK 0x02000000L
+#define MC_IO_DPHY_STR_CNTL_D0__USE_S_CAL__SHIFT 0x00000019
+#define MC_IO_DPHY_STR_CNTL_D1__CAL_SEL_MASK 0x0c000000L
+#define MC_IO_DPHY_STR_CNTL_D1__CAL_SEL__SHIFT 0x0000001a
+#define MC_IO_DPHY_STR_CNTL_D1__LOAD_D_STR_MASK 0x10000000L
+#define MC_IO_DPHY_STR_CNTL_D1__LOAD_D_STR__SHIFT 0x0000001c
+#define MC_IO_DPHY_STR_CNTL_D1__LOAD_S_STR_MASK 0x20000000L
+#define MC_IO_DPHY_STR_CNTL_D1__LOAD_S_STR__SHIFT 0x0000001d
+#define MC_IO_DPHY_STR_CNTL_D1__NSTR_OFF_D_MASK 0x00000fc0L
+#define MC_IO_DPHY_STR_CNTL_D1__NSTR_OFF_D__SHIFT 0x00000006
+#define MC_IO_DPHY_STR_CNTL_D1__NSTR_OFF_S_MASK 0x00fc0000L
+#define MC_IO_DPHY_STR_CNTL_D1__NSTR_OFF_S__SHIFT 0x00000012
+#define MC_IO_DPHY_STR_CNTL_D1__PSTR_OFF_D_MASK 0x0000003fL
+#define MC_IO_DPHY_STR_CNTL_D1__PSTR_OFF_D__SHIFT 0x00000000
+#define MC_IO_DPHY_STR_CNTL_D1__PSTR_OFF_S_MASK 0x0003f000L
+#define MC_IO_DPHY_STR_CNTL_D1__PSTR_OFF_S__SHIFT 0x0000000c
+#define MC_IO_DPHY_STR_CNTL_D1__USE_D_CAL_MASK 0x01000000L
+#define MC_IO_DPHY_STR_CNTL_D1__USE_D_CAL__SHIFT 0x00000018
+#define MC_IO_DPHY_STR_CNTL_D1__USE_S_CAL_MASK 0x02000000L
+#define MC_IO_DPHY_STR_CNTL_D1__USE_S_CAL__SHIFT 0x00000019
+#define MC_IO_PAD_CNTL__ATBEN_MASK 0x3f000000L
+#define MC_IO_PAD_CNTL__ATBEN__SHIFT 0x00000018
+#define MC_IO_PAD_CNTL__ATBSEL_D0_MASK 0x80000000L
+#define MC_IO_PAD_CNTL__ATBSEL_D0__SHIFT 0x0000001f
+#define MC_IO_PAD_CNTL__ATBSEL_D1_MASK 0x40000000L
+#define MC_IO_PAD_CNTL__ATBSEL_D1__SHIFT 0x0000001e
+#define MC_IO_PAD_CNTL__ATBSEL_MASK 0x00f00000L
+#define MC_IO_PAD_CNTL__ATBSEL__SHIFT 0x00000014
+#define MC_IO_PAD_CNTL_D0__CK_AUTO_EN_MASK 0x00100000L
+#define MC_IO_PAD_CNTL_D0__CK_AUTO_EN__SHIFT 0x00000014
+#define MC_IO_PAD_CNTL_D0__CK_DELAY_N_MASK 0x00c00000L
+#define MC_IO_PAD_CNTL_D0__CK_DELAY_N__SHIFT 0x00000016
+#define MC_IO_PAD_CNTL_D0__CK_DELAY_P_MASK 0x03000000L
+#define MC_IO_PAD_CNTL_D0__CK_DELAY_P__SHIFT 0x00000018
+#define MC_IO_PAD_CNTL_D0__CK_DELAY_SEL_MASK 0x00200000L
+#define MC_IO_PAD_CNTL_D0__CK_DELAY_SEL__SHIFT 0x00000015
+#define MC_IO_PAD_CNTL_D0__DELAY_ADR_SYNC_MASK 0x00000010L
+#define MC_IO_PAD_CNTL_D0__DELAY_ADR_SYNC__SHIFT 0x00000004
+#define MC_IO_PAD_CNTL_D0__DELAY_CLK_SYNC_MASK 0x00000004L
+#define MC_IO_PAD_CNTL_D0__DELAY_CLK_SYNC__SHIFT 0x00000002
+#define MC_IO_PAD_CNTL_D0__DELAY_CMD_SYNC_MASK 0x00000008L
+#define MC_IO_PAD_CNTL_D0__DELAY_CMD_SYNC__SHIFT 0x00000003
+#define MC_IO_PAD_CNTL_D0__DIFF_STR_MASK 0x20000000L
+#define MC_IO_PAD_CNTL_D0__DIFF_STR__SHIFT 0x0000001d
+#define MC_IO_PAD_CNTL_D0__DISABLE_ADR_MASK 0x00002000L
+#define MC_IO_PAD_CNTL_D0__DISABLE_ADR__SHIFT 0x0000000d
+#define MC_IO_PAD_CNTL_D0__DISABLE_CMD_MASK 0x00001000L
+#define MC_IO_PAD_CNTL_D0__DISABLE_CMD__SHIFT 0x0000000c
+#define MC_IO_PAD_CNTL_D0__EN_RD_STR_DLY_MASK 0x00000800L
+#define MC_IO_PAD_CNTL_D0__EN_RD_STR_DLY__SHIFT 0x0000000b
+#define MC_IO_PAD_CNTL_D0__FORCE_EN_RD_STR_MASK 0x00000400L
+#define MC_IO_PAD_CNTL_D0__FORCE_EN_RD_STR__SHIFT 0x0000000a
+#define MC_IO_PAD_CNTL_D0__GDDR_PWRON_MASK 0x40000000L
+#define MC_IO_PAD_CNTL_D0__GDDR_PWRON__SHIFT 0x0000001e
+#define MC_IO_PAD_CNTL_D0__MEM_FALL_OUT_ADR_MASK 0x00000200L
+#define MC_IO_PAD_CNTL_D0__MEM_FALL_OUT_ADR__SHIFT 0x00000009
+#define MC_IO_PAD_CNTL_D0__MEM_FALL_OUT_CLK_MASK 0x00000080L
+#define MC_IO_PAD_CNTL_D0__MEM_FALL_OUT_CLK__SHIFT 0x00000007
+#define MC_IO_PAD_CNTL_D0__MEM_FALL_OUT_CMD_MASK 0x00000100L
+#define MC_IO_PAD_CNTL_D0__MEM_FALL_OUT_CMD__SHIFT 0x00000008
+#define MC_IO_PAD_CNTL_D0__TXPWROFF_CKE_MASK 0x08000000L
+#define MC_IO_PAD_CNTL_D0__TXPWROFF_CKE__SHIFT 0x0000001b
+#define MC_IO_PAD_CNTL_D0__TXPWROFF_CLK_MASK 0x80000000L
+#define MC_IO_PAD_CNTL_D0__TXPWROFF_CLK__SHIFT 0x0000001f
+#define MC_IO_PAD_CNTL_D0__UNI_STR_MASK 0x10000000L
+#define MC_IO_PAD_CNTL_D0__UNI_STR__SHIFT 0x0000001c
+#define MC_IO_PAD_CNTL_D0__VREFI_EN_MASK 0x00004000L
+#define MC_IO_PAD_CNTL_D0__VREFI_EN__SHIFT 0x0000000e
+#define MC_IO_PAD_CNTL_D0__VREFI_SEL_MASK 0x000f8000L
+#define MC_IO_PAD_CNTL_D0__VREFI_SEL__SHIFT 0x0000000f
+#define MC_IO_PAD_CNTL_D1__CK_AUTO_EN_MASK 0x00100000L
+#define MC_IO_PAD_CNTL_D1__CK_AUTO_EN__SHIFT 0x00000014
+#define MC_IO_PAD_CNTL_D1__CK_DELAY_N_MASK 0x00c00000L
+#define MC_IO_PAD_CNTL_D1__CK_DELAY_N__SHIFT 0x00000016
+#define MC_IO_PAD_CNTL_D1__CK_DELAY_P_MASK 0x03000000L
+#define MC_IO_PAD_CNTL_D1__CK_DELAY_P__SHIFT 0x00000018
+#define MC_IO_PAD_CNTL_D1__CK_DELAY_SEL_MASK 0x00200000L
+#define MC_IO_PAD_CNTL_D1__CK_DELAY_SEL__SHIFT 0x00000015
+#define MC_IO_PAD_CNTL_D1__DELAY_ADR_SYNC_MASK 0x00000010L
+#define MC_IO_PAD_CNTL_D1__DELAY_ADR_SYNC__SHIFT 0x00000004
+#define MC_IO_PAD_CNTL_D1__DELAY_CLK_SYNC_MASK 0x00000004L
+#define MC_IO_PAD_CNTL_D1__DELAY_CLK_SYNC__SHIFT 0x00000002
+#define MC_IO_PAD_CNTL_D1__DELAY_CMD_SYNC_MASK 0x00000008L
+#define MC_IO_PAD_CNTL_D1__DELAY_CMD_SYNC__SHIFT 0x00000003
+#define MC_IO_PAD_CNTL_D1__DELAY_DATA_SYNC_MASK 0x00000001L
+#define MC_IO_PAD_CNTL_D1__DELAY_DATA_SYNC__SHIFT 0x00000000
+#define MC_IO_PAD_CNTL_D1__DELAY_STR_SYNC_MASK 0x00000002L
+#define MC_IO_PAD_CNTL_D1__DELAY_STR_SYNC__SHIFT 0x00000001
+#define MC_IO_PAD_CNTL_D1__DIFF_STR_MASK 0x20000000L
+#define MC_IO_PAD_CNTL_D1__DIFF_STR__SHIFT 0x0000001d
+#define MC_IO_PAD_CNTL_D1__DISABLE_ADR_MASK 0x00002000L
+#define MC_IO_PAD_CNTL_D1__DISABLE_ADR__SHIFT 0x0000000d
+#define MC_IO_PAD_CNTL_D1__DISABLE_CMD_MASK 0x00001000L
+#define MC_IO_PAD_CNTL_D1__DISABLE_CMD__SHIFT 0x0000000c
+#define MC_IO_PAD_CNTL_D1__EN_RD_STR_DLY_MASK 0x00000800L
+#define MC_IO_PAD_CNTL_D1__EN_RD_STR_DLY__SHIFT 0x0000000b
+#define MC_IO_PAD_CNTL_D1__FORCE_EN_RD_STR_MASK 0x00000400L
+#define MC_IO_PAD_CNTL_D1__FORCE_EN_RD_STR__SHIFT 0x0000000a
+#define MC_IO_PAD_CNTL_D1__GDDR_PWRON_MASK 0x40000000L
+#define MC_IO_PAD_CNTL_D1__GDDR_PWRON__SHIFT 0x0000001e
+#define MC_IO_PAD_CNTL_D1__MEM_FALL_OUT_ADR_MASK 0x00000200L
+#define MC_IO_PAD_CNTL_D1__MEM_FALL_OUT_ADR__SHIFT 0x00000009
+#define MC_IO_PAD_CNTL_D1__MEM_FALL_OUT_CLK_MASK 0x00000080L
+#define MC_IO_PAD_CNTL_D1__MEM_FALL_OUT_CLK__SHIFT 0x00000007
+#define MC_IO_PAD_CNTL_D1__MEM_FALL_OUT_CMD_MASK 0x00000100L
+#define MC_IO_PAD_CNTL_D1__MEM_FALL_OUT_CMD__SHIFT 0x00000008
+#define MC_IO_PAD_CNTL_D1__MEM_FALL_OUT_DATA_MASK 0x00000020L
+#define MC_IO_PAD_CNTL_D1__MEM_FALL_OUT_DATA__SHIFT 0x00000005
+#define MC_IO_PAD_CNTL_D1__MEM_FALL_OUT_STR_MASK 0x00000040L
+#define MC_IO_PAD_CNTL_D1__MEM_FALL_OUT_STR__SHIFT 0x00000006
+#define MC_IO_PAD_CNTL_D1__TXPWROFF_CKE_MASK 0x08000000L
+#define MC_IO_PAD_CNTL_D1__TXPWROFF_CKE__SHIFT 0x0000001b
+#define MC_IO_PAD_CNTL_D1__TXPWROFF_CLK_MASK 0x80000000L
+#define MC_IO_PAD_CNTL_D1__TXPWROFF_CLK__SHIFT 0x0000001f
+#define MC_IO_PAD_CNTL_D1__UNI_STR_MASK 0x10000000L
+#define MC_IO_PAD_CNTL_D1__UNI_STR__SHIFT 0x0000001c
+#define MC_IO_PAD_CNTL_D1__VREFI_EN_MASK 0x00004000L
+#define MC_IO_PAD_CNTL_D1__VREFI_EN__SHIFT 0x0000000e
+#define MC_IO_PAD_CNTL_D1__VREFI_SEL_MASK 0x000f8000L
+#define MC_IO_PAD_CNTL_D1__VREFI_SEL__SHIFT 0x0000000f
+#define MC_IO_PAD_CNTL__MEM_IO_IMP_MAX_MASK 0x0000ff00L
+#define MC_IO_PAD_CNTL__MEM_IO_IMP_MAX__SHIFT 0x00000008
+#define MC_IO_PAD_CNTL__MEM_IO_IMP_MIN_MASK 0x000000ffL
+#define MC_IO_PAD_CNTL__MEM_IO_IMP_MIN__SHIFT 0x00000000
+#define MC_IO_PAD_CNTL__OVL_YCLKON_D0_MASK 0x00040000L
+#define MC_IO_PAD_CNTL__OVL_YCLKON_D0__SHIFT 0x00000012
+#define MC_IO_PAD_CNTL__OVL_YCLKON_D1_MASK 0x00080000L
+#define MC_IO_PAD_CNTL__OVL_YCLKON_D1__SHIFT 0x00000013
+#define MC_IO_PAD_CNTL__RXPHASE_GRAY_MASK 0x00020000L
+#define MC_IO_PAD_CNTL__RXPHASE_GRAY__SHIFT 0x00000011
+#define MC_IO_PAD_CNTL__TXPHASE_GRAY_MASK 0x00010000L
+#define MC_IO_PAD_CNTL__TXPHASE_GRAY__SHIFT 0x00000010
+#define MC_IO_RXCNTL1_DPHY0_D0__DLL_RSV_MASK 0xf0000000L
+#define MC_IO_RXCNTL1_DPHY0_D0__DLL_RSV__SHIFT 0x0000001c
+#define MC_IO_RXCNTL1_DPHY0_D0__PMD_LOOPBACK_MASK 0x0e000000L
+#define MC_IO_RXCNTL1_DPHY0_D0__PMD_LOOPBACK__SHIFT 0x00000019
+#define MC_IO_RXCNTL1_DPHY0_D0__VREFCAL1_MSB_MASK 0x0000000fL
+#define MC_IO_RXCNTL1_DPHY0_D0__VREFCAL1_MSB__SHIFT 0x00000000
+#define MC_IO_RXCNTL1_DPHY0_D0__VREFCAL2_MSB_MASK 0x000000f0L
+#define MC_IO_RXCNTL1_DPHY0_D0__VREFCAL2_MSB__SHIFT 0x00000004
+#define MC_IO_RXCNTL1_DPHY0_D0__VREFCAL3_MASK 0x0000ff00L
+#define MC_IO_RXCNTL1_DPHY0_D0__VREFCAL3__SHIFT 0x00000008
+#define MC_IO_RXCNTL1_DPHY0_D0__VREFPDNB_1_MASK 0x00040000L
+#define MC_IO_RXCNTL1_DPHY0_D0__VREFPDNB_1__SHIFT 0x00000012
+#define MC_IO_RXCNTL1_DPHY0_D0__VREFSEL2_MASK 0x00010000L
+#define MC_IO_RXCNTL1_DPHY0_D0__VREFSEL2__SHIFT 0x00000010
+#define MC_IO_RXCNTL1_DPHY0_D0__VREFSEL3_MASK 0x00020000L
+#define MC_IO_RXCNTL1_DPHY0_D0__VREFSEL3__SHIFT 0x00000011
+#define MC_IO_RXCNTL1_DPHY0_D1__DLL_RSV_MASK 0xf0000000L
+#define MC_IO_RXCNTL1_DPHY0_D1__DLL_RSV__SHIFT 0x0000001c
+#define MC_IO_RXCNTL1_DPHY0_D1__PMD_LOOPBACK_MASK 0x0e000000L
+#define MC_IO_RXCNTL1_DPHY0_D1__PMD_LOOPBACK__SHIFT 0x00000019
+#define MC_IO_RXCNTL1_DPHY0_D1__VREFCAL1_MSB_MASK 0x0000000fL
+#define MC_IO_RXCNTL1_DPHY0_D1__VREFCAL1_MSB__SHIFT 0x00000000
+#define MC_IO_RXCNTL1_DPHY0_D1__VREFCAL2_MSB_MASK 0x000000f0L
+#define MC_IO_RXCNTL1_DPHY0_D1__VREFCAL2_MSB__SHIFT 0x00000004
+#define MC_IO_RXCNTL1_DPHY0_D1__VREFCAL3_MASK 0x0000ff00L
+#define MC_IO_RXCNTL1_DPHY0_D1__VREFCAL3__SHIFT 0x00000008
+#define MC_IO_RXCNTL1_DPHY0_D1__VREFPDNB_1_MASK 0x00040000L
+#define MC_IO_RXCNTL1_DPHY0_D1__VREFPDNB_1__SHIFT 0x00000012
+#define MC_IO_RXCNTL1_DPHY0_D1__VREFSEL2_MASK 0x00010000L
+#define MC_IO_RXCNTL1_DPHY0_D1__VREFSEL2__SHIFT 0x00000010
+#define MC_IO_RXCNTL1_DPHY0_D1__VREFSEL3_MASK 0x00020000L
+#define MC_IO_RXCNTL1_DPHY0_D1__VREFSEL3__SHIFT 0x00000011
+#define MC_IO_RXCNTL1_DPHY1_D0__DLL_RSV_MASK 0xf0000000L
+#define MC_IO_RXCNTL1_DPHY1_D0__DLL_RSV__SHIFT 0x0000001c
+#define MC_IO_RXCNTL1_DPHY1_D0__PMD_LOOPBACK_MASK 0x0e000000L
+#define MC_IO_RXCNTL1_DPHY1_D0__PMD_LOOPBACK__SHIFT 0x00000019
+#define MC_IO_RXCNTL1_DPHY1_D0__VREFCAL1_MSB_MASK 0x0000000fL
+#define MC_IO_RXCNTL1_DPHY1_D0__VREFCAL1_MSB__SHIFT 0x00000000
+#define MC_IO_RXCNTL1_DPHY1_D0__VREFCAL2_MSB_MASK 0x000000f0L
+#define MC_IO_RXCNTL1_DPHY1_D0__VREFCAL2_MSB__SHIFT 0x00000004
+#define MC_IO_RXCNTL1_DPHY1_D0__VREFCAL3_MASK 0x0000ff00L
+#define MC_IO_RXCNTL1_DPHY1_D0__VREFCAL3__SHIFT 0x00000008
+#define MC_IO_RXCNTL1_DPHY1_D0__VREFPDNB_1_MASK 0x00040000L
+#define MC_IO_RXCNTL1_DPHY1_D0__VREFPDNB_1__SHIFT 0x00000012
+#define MC_IO_RXCNTL1_DPHY1_D0__VREFSEL2_MASK 0x00010000L
+#define MC_IO_RXCNTL1_DPHY1_D0__VREFSEL2__SHIFT 0x00000010
+#define MC_IO_RXCNTL1_DPHY1_D0__VREFSEL3_MASK 0x00020000L
+#define MC_IO_RXCNTL1_DPHY1_D0__VREFSEL3__SHIFT 0x00000011
+#define MC_IO_RXCNTL1_DPHY1_D1__DLL_RSV_MASK 0xf0000000L
+#define MC_IO_RXCNTL1_DPHY1_D1__DLL_RSV__SHIFT 0x0000001c
+#define MC_IO_RXCNTL1_DPHY1_D1__PMD_LOOPBACK_MASK 0x0e000000L
+#define MC_IO_RXCNTL1_DPHY1_D1__PMD_LOOPBACK__SHIFT 0x00000019
+#define MC_IO_RXCNTL1_DPHY1_D1__VREFCAL1_MSB_MASK 0x0000000fL
+#define MC_IO_RXCNTL1_DPHY1_D1__VREFCAL1_MSB__SHIFT 0x00000000
+#define MC_IO_RXCNTL1_DPHY1_D1__VREFCAL2_MSB_MASK 0x000000f0L
+#define MC_IO_RXCNTL1_DPHY1_D1__VREFCAL2_MSB__SHIFT 0x00000004
+#define MC_IO_RXCNTL1_DPHY1_D1__VREFCAL3_MASK 0x0000ff00L
+#define MC_IO_RXCNTL1_DPHY1_D1__VREFCAL3__SHIFT 0x00000008
+#define MC_IO_RXCNTL1_DPHY1_D1__VREFPDNB_1_MASK 0x00040000L
+#define MC_IO_RXCNTL1_DPHY1_D1__VREFPDNB_1__SHIFT 0x00000012
+#define MC_IO_RXCNTL1_DPHY1_D1__VREFSEL2_MASK 0x00010000L
+#define MC_IO_RXCNTL1_DPHY1_D1__VREFSEL2__SHIFT 0x00000010
+#define MC_IO_RXCNTL1_DPHY1_D1__VREFSEL3_MASK 0x00020000L
+#define MC_IO_RXCNTL1_DPHY1_D1__VREFSEL3__SHIFT 0x00000011
+#define MC_IO_RXCNTL_DPHY0_D0__DLL_ADJ_B0_MASK 0x00700000L
+#define MC_IO_RXCNTL_DPHY0_D0__DLL_ADJ_B0__SHIFT 0x00000014
+#define MC_IO_RXCNTL_DPHY0_D0__DLL_ADJ_B1_MASK 0x07000000L
+#define MC_IO_RXCNTL_DPHY0_D0__DLL_ADJ_B1__SHIFT 0x00000018
+#define MC_IO_RXCNTL_DPHY0_D0__DLL_ADJ_M_MASK 0x10000000L
+#define MC_IO_RXCNTL_DPHY0_D0__DLL_ADJ_M__SHIFT 0x0000001c
+#define MC_IO_RXCNTL_DPHY0_D0__DLL_BW_CTRL_MASK 0xc0000000L
+#define MC_IO_RXCNTL_DPHY0_D0__DLL_BW_CTRL__SHIFT 0x0000001e
+#define MC_IO_RXCNTL_DPHY0_D0__RCVSEL_MASK 0x00000004L
+#define MC_IO_RXCNTL_DPHY0_D0__RCVSEL__SHIFT 0x00000002
+#define MC_IO_RXCNTL_DPHY0_D0__REFCLK_PWRON_MASK 0x20000000L
+#define MC_IO_RXCNTL_DPHY0_D0__REFCLK_PWRON__SHIFT 0x0000001d
+#define MC_IO_RXCNTL_DPHY0_D0__RXBIASSEL_MASK 0x00000003L
+#define MC_IO_RXCNTL_DPHY0_D0__RXBIASSEL__SHIFT 0x00000000
+#define MC_IO_RXCNTL_DPHY0_D0__RXDPWRON_DLY_MASK 0x00000030L
+#define MC_IO_RXCNTL_DPHY0_D0__RXDPWRON_DLY__SHIFT 0x00000004
+#define MC_IO_RXCNTL_DPHY0_D0__RXLP_MASK 0x00000080L
+#define MC_IO_RXCNTL_DPHY0_D0__RXLP__SHIFT 0x00000007
+#define MC_IO_RXCNTL_DPHY0_D0__RXPDNB_MASK 0x00000040L
+#define MC_IO_RXCNTL_DPHY0_D0__RXPDNB__SHIFT 0x00000006
+#define MC_IO_RXCNTL_DPHY0_D0__RX_PEAKSEL_MASK 0x000c0000L
+#define MC_IO_RXCNTL_DPHY0_D0__RX_PEAKSEL__SHIFT 0x00000012
+#define MC_IO_RXCNTL_DPHY0_D0__VREFCAL_MASK 0x00000f00L
+#define MC_IO_RXCNTL_DPHY0_D0__VREFCAL__SHIFT 0x00000008
+#define MC_IO_RXCNTL_DPHY0_D0__VREFCAL_STR_MASK 0x0000f000L
+#define MC_IO_RXCNTL_DPHY0_D0__VREFCAL_STR__SHIFT 0x0000000c
+#define MC_IO_RXCNTL_DPHY0_D0__VREFPDNB_MASK 0x00000008L
+#define MC_IO_RXCNTL_DPHY0_D0__VREFPDNB__SHIFT 0x00000003
+#define MC_IO_RXCNTL_DPHY0_D0__VREFSEL_MASK 0x00010000L
+#define MC_IO_RXCNTL_DPHY0_D0__VREFSEL__SHIFT 0x00000010
+#define MC_IO_RXCNTL_DPHY0_D1__DLL_ADJ_B0_MASK 0x00700000L
+#define MC_IO_RXCNTL_DPHY0_D1__DLL_ADJ_B0__SHIFT 0x00000014
+#define MC_IO_RXCNTL_DPHY0_D1__DLL_ADJ_B1_MASK 0x07000000L
+#define MC_IO_RXCNTL_DPHY0_D1__DLL_ADJ_B1__SHIFT 0x00000018
+#define MC_IO_RXCNTL_DPHY0_D1__DLL_ADJ_M_MASK 0x10000000L
+#define MC_IO_RXCNTL_DPHY0_D1__DLL_ADJ_M__SHIFT 0x0000001c
+#define MC_IO_RXCNTL_DPHY0_D1__DLL_BW_CTRL_MASK 0xc0000000L
+#define MC_IO_RXCNTL_DPHY0_D1__DLL_BW_CTRL__SHIFT 0x0000001e
+#define MC_IO_RXCNTL_DPHY0_D1__RCVSEL_MASK 0x00000004L
+#define MC_IO_RXCNTL_DPHY0_D1__RCVSEL__SHIFT 0x00000002
+#define MC_IO_RXCNTL_DPHY0_D1__REFCLK_PWRON_MASK 0x20000000L
+#define MC_IO_RXCNTL_DPHY0_D1__REFCLK_PWRON__SHIFT 0x0000001d
+#define MC_IO_RXCNTL_DPHY0_D1__RXBIASSEL_MASK 0x00000003L
+#define MC_IO_RXCNTL_DPHY0_D1__RXBIASSEL__SHIFT 0x00000000
+#define MC_IO_RXCNTL_DPHY0_D1__RXDPWRON_DLY_MASK 0x00000030L
+#define MC_IO_RXCNTL_DPHY0_D1__RXDPWRON_DLY__SHIFT 0x00000004
+#define MC_IO_RXCNTL_DPHY0_D1__RXLP_MASK 0x00000080L
+#define MC_IO_RXCNTL_DPHY0_D1__RXLP__SHIFT 0x00000007
+#define MC_IO_RXCNTL_DPHY0_D1__RXPDNB_MASK 0x00000040L
+#define MC_IO_RXCNTL_DPHY0_D1__RXPDNB__SHIFT 0x00000006
+#define MC_IO_RXCNTL_DPHY0_D1__RX_PEAKSEL_MASK 0x000c0000L
+#define MC_IO_RXCNTL_DPHY0_D1__RX_PEAKSEL__SHIFT 0x00000012
+#define MC_IO_RXCNTL_DPHY0_D1__VREFCAL_MASK 0x00000f00L
+#define MC_IO_RXCNTL_DPHY0_D1__VREFCAL__SHIFT 0x00000008
+#define MC_IO_RXCNTL_DPHY0_D1__VREFCAL_STR_MASK 0x0000f000L
+#define MC_IO_RXCNTL_DPHY0_D1__VREFCAL_STR__SHIFT 0x0000000c
+#define MC_IO_RXCNTL_DPHY0_D1__VREFPDNB_MASK 0x00000008L
+#define MC_IO_RXCNTL_DPHY0_D1__VREFPDNB__SHIFT 0x00000003
+#define MC_IO_RXCNTL_DPHY0_D1__VREFSEL_MASK 0x00010000L
+#define MC_IO_RXCNTL_DPHY0_D1__VREFSEL__SHIFT 0x00000010
+#define MC_IO_RXCNTL_DPHY1_D0__DLL_ADJ_B0_MASK 0x00700000L
+#define MC_IO_RXCNTL_DPHY1_D0__DLL_ADJ_B0__SHIFT 0x00000014
+#define MC_IO_RXCNTL_DPHY1_D0__DLL_ADJ_B1_MASK 0x07000000L
+#define MC_IO_RXCNTL_DPHY1_D0__DLL_ADJ_B1__SHIFT 0x00000018
+#define MC_IO_RXCNTL_DPHY1_D0__DLL_ADJ_M_MASK 0x10000000L
+#define MC_IO_RXCNTL_DPHY1_D0__DLL_ADJ_M__SHIFT 0x0000001c
+#define MC_IO_RXCNTL_DPHY1_D0__DLL_BW_CTRL_MASK 0xc0000000L
+#define MC_IO_RXCNTL_DPHY1_D0__DLL_BW_CTRL__SHIFT 0x0000001e
+#define MC_IO_RXCNTL_DPHY1_D0__RCVSEL_MASK 0x00000004L
+#define MC_IO_RXCNTL_DPHY1_D0__RCVSEL__SHIFT 0x00000002
+#define MC_IO_RXCNTL_DPHY1_D0__REFCLK_PWRON_MASK 0x20000000L
+#define MC_IO_RXCNTL_DPHY1_D0__REFCLK_PWRON__SHIFT 0x0000001d
+#define MC_IO_RXCNTL_DPHY1_D0__RXBIASSEL_MASK 0x00000003L
+#define MC_IO_RXCNTL_DPHY1_D0__RXBIASSEL__SHIFT 0x00000000
+#define MC_IO_RXCNTL_DPHY1_D0__RXDPWRON_DLY_MASK 0x00000030L
+#define MC_IO_RXCNTL_DPHY1_D0__RXDPWRON_DLY__SHIFT 0x00000004
+#define MC_IO_RXCNTL_DPHY1_D0__RXLP_MASK 0x00000080L
+#define MC_IO_RXCNTL_DPHY1_D0__RXLP__SHIFT 0x00000007
+#define MC_IO_RXCNTL_DPHY1_D0__RXPDNB_MASK 0x00000040L
+#define MC_IO_RXCNTL_DPHY1_D0__RXPDNB__SHIFT 0x00000006
+#define MC_IO_RXCNTL_DPHY1_D0__RX_PEAKSEL_MASK 0x000c0000L
+#define MC_IO_RXCNTL_DPHY1_D0__RX_PEAKSEL__SHIFT 0x00000012
+#define MC_IO_RXCNTL_DPHY1_D0__VREFCAL_MASK 0x00000f00L
+#define MC_IO_RXCNTL_DPHY1_D0__VREFCAL__SHIFT 0x00000008
+#define MC_IO_RXCNTL_DPHY1_D0__VREFCAL_STR_MASK 0x0000f000L
+#define MC_IO_RXCNTL_DPHY1_D0__VREFCAL_STR__SHIFT 0x0000000c
+#define MC_IO_RXCNTL_DPHY1_D0__VREFPDNB_MASK 0x00000008L
+#define MC_IO_RXCNTL_DPHY1_D0__VREFPDNB__SHIFT 0x00000003
+#define MC_IO_RXCNTL_DPHY1_D0__VREFSEL_MASK 0x00010000L
+#define MC_IO_RXCNTL_DPHY1_D0__VREFSEL__SHIFT 0x00000010
+#define MC_IO_RXCNTL_DPHY1_D1__DLL_ADJ_B0_MASK 0x00700000L
+#define MC_IO_RXCNTL_DPHY1_D1__DLL_ADJ_B0__SHIFT 0x00000014
+#define MC_IO_RXCNTL_DPHY1_D1__DLL_ADJ_B1_MASK 0x07000000L
+#define MC_IO_RXCNTL_DPHY1_D1__DLL_ADJ_B1__SHIFT 0x00000018
+#define MC_IO_RXCNTL_DPHY1_D1__DLL_ADJ_M_MASK 0x10000000L
+#define MC_IO_RXCNTL_DPHY1_D1__DLL_ADJ_M__SHIFT 0x0000001c
+#define MC_IO_RXCNTL_DPHY1_D1__DLL_BW_CTRL_MASK 0xc0000000L
+#define MC_IO_RXCNTL_DPHY1_D1__DLL_BW_CTRL__SHIFT 0x0000001e
+#define MC_IO_RXCNTL_DPHY1_D1__RCVSEL_MASK 0x00000004L
+#define MC_IO_RXCNTL_DPHY1_D1__RCVSEL__SHIFT 0x00000002
+#define MC_IO_RXCNTL_DPHY1_D1__REFCLK_PWRON_MASK 0x20000000L
+#define MC_IO_RXCNTL_DPHY1_D1__REFCLK_PWRON__SHIFT 0x0000001d
+#define MC_IO_RXCNTL_DPHY1_D1__RXBIASSEL_MASK 0x00000003L
+#define MC_IO_RXCNTL_DPHY1_D1__RXBIASSEL__SHIFT 0x00000000
+#define MC_IO_RXCNTL_DPHY1_D1__RXDPWRON_DLY_MASK 0x00000030L
+#define MC_IO_RXCNTL_DPHY1_D1__RXDPWRON_DLY__SHIFT 0x00000004
+#define MC_IO_RXCNTL_DPHY1_D1__RXLP_MASK 0x00000080L
+#define MC_IO_RXCNTL_DPHY1_D1__RXLP__SHIFT 0x00000007
+#define MC_IO_RXCNTL_DPHY1_D1__RXPDNB_MASK 0x00000040L
+#define MC_IO_RXCNTL_DPHY1_D1__RXPDNB__SHIFT 0x00000006
+#define MC_IO_RXCNTL_DPHY1_D1__RX_PEAKSEL_MASK 0x000c0000L
+#define MC_IO_RXCNTL_DPHY1_D1__RX_PEAKSEL__SHIFT 0x00000012
+#define MC_IO_RXCNTL_DPHY1_D1__VREFCAL_MASK 0x00000f00L
+#define MC_IO_RXCNTL_DPHY1_D1__VREFCAL__SHIFT 0x00000008
+#define MC_IO_RXCNTL_DPHY1_D1__VREFCAL_STR_MASK 0x0000f000L
+#define MC_IO_RXCNTL_DPHY1_D1__VREFCAL_STR__SHIFT 0x0000000c
+#define MC_IO_RXCNTL_DPHY1_D1__VREFPDNB_MASK 0x00000008L
+#define MC_IO_RXCNTL_DPHY1_D1__VREFPDNB__SHIFT 0x00000003
+#define MC_IO_RXCNTL_DPHY1_D1__VREFSEL_MASK 0x00010000L
+#define MC_IO_RXCNTL_DPHY1_D1__VREFSEL__SHIFT 0x00000010
+#define MC_IO_TXCNTL_APHY_D0__BIASSEL_MASK 0x00000003L
+#define MC_IO_TXCNTL_APHY_D0__BIASSEL__SHIFT 0x00000000
+#define MC_IO_TXCNTL_APHY_D0__CKE_BIT_MASK 0x40000000L
+#define MC_IO_TXCNTL_APHY_D0__CKE_BIT__SHIFT 0x0000001e
+#define MC_IO_TXCNTL_APHY_D0__CKE_SEL_MASK 0x80000000L
+#define MC_IO_TXCNTL_APHY_D0__CKE_SEL__SHIFT 0x0000001f
+#define MC_IO_TXCNTL_APHY_D0__DRVDUTY_MASK 0x0000000cL
+#define MC_IO_TXCNTL_APHY_D0__DRVDUTY__SHIFT 0x00000002
+#define MC_IO_TXCNTL_APHY_D0__EMPH_MASK 0x00000040L
+#define MC_IO_TXCNTL_APHY_D0__EMPH__SHIFT 0x00000006
+#define MC_IO_TXCNTL_APHY_D0__LOWCMEN_MASK 0x00000010L
+#define MC_IO_TXCNTL_APHY_D0__LOWCMEN__SHIFT 0x00000004
+#define MC_IO_TXCNTL_APHY_D0__NDRV_MASK 0x00700000L
+#define MC_IO_TXCNTL_APHY_D0__NDRV__SHIFT 0x00000014
+#define MC_IO_TXCNTL_APHY_D0__PDRV_MASK 0x000f0000L
+#define MC_IO_TXCNTL_APHY_D0__PDRV__SHIFT 0x00000010
+#define MC_IO_TXCNTL_APHY_D0__PMA_LOOPBACK_MASK 0x0000e000L
+#define MC_IO_TXCNTL_APHY_D0__PMA_LOOPBACK__SHIFT 0x0000000d
+#define MC_IO_TXCNTL_APHY_D0__PTERM_MASK 0x00000f00L
+#define MC_IO_TXCNTL_APHY_D0__PTERM__SHIFT 0x00000008
+#define MC_IO_TXCNTL_APHY_D0__QDR_MASK 0x00000020L
+#define MC_IO_TXCNTL_APHY_D0__QDR__SHIFT 0x00000005
+#define MC_IO_TXCNTL_APHY_D0__TSTEN_MASK 0x01000000L
+#define MC_IO_TXCNTL_APHY_D0__TSTEN__SHIFT 0x00000018
+#define MC_IO_TXCNTL_APHY_D0__TXBPASS_SEL_MASK 0x00001000L
+#define MC_IO_TXCNTL_APHY_D0__TXBPASS_SEL__SHIFT 0x0000000c
+#define MC_IO_TXCNTL_APHY_D0__TXBYPASS_DATA_MASK 0x38000000L
+#define MC_IO_TXCNTL_APHY_D0__TXBYPASS_DATA__SHIFT 0x0000001b
+#define MC_IO_TXCNTL_APHY_D0__TXBYPASS_MASK 0x04000000L
+#define MC_IO_TXCNTL_APHY_D0__TXBYPASS__SHIFT 0x0000001a
+#define MC_IO_TXCNTL_APHY_D0__TXPD_MASK 0x00000080L
+#define MC_IO_TXCNTL_APHY_D0__TXPD__SHIFT 0x00000007
+#define MC_IO_TXCNTL_APHY_D0__TXRESET_MASK 0x02000000L
+#define MC_IO_TXCNTL_APHY_D0__TXRESET__SHIFT 0x00000019
+#define MC_IO_TXCNTL_APHY_D0__YCLKON_MASK 0x00800000L
+#define MC_IO_TXCNTL_APHY_D0__YCLKON__SHIFT 0x00000017
+#define MC_IO_TXCNTL_APHY_D1__BIASSEL_MASK 0x00000003L
+#define MC_IO_TXCNTL_APHY_D1__BIASSEL__SHIFT 0x00000000
+#define MC_IO_TXCNTL_APHY_D1__CKE_BIT_MASK 0x40000000L
+#define MC_IO_TXCNTL_APHY_D1__CKE_BIT__SHIFT 0x0000001e
+#define MC_IO_TXCNTL_APHY_D1__CKE_SEL_MASK 0x80000000L
+#define MC_IO_TXCNTL_APHY_D1__CKE_SEL__SHIFT 0x0000001f
+#define MC_IO_TXCNTL_APHY_D1__DRVDUTY_MASK 0x0000000cL
+#define MC_IO_TXCNTL_APHY_D1__DRVDUTY__SHIFT 0x00000002
+#define MC_IO_TXCNTL_APHY_D1__EMPH_MASK 0x00000040L
+#define MC_IO_TXCNTL_APHY_D1__EMPH__SHIFT 0x00000006
+#define MC_IO_TXCNTL_APHY_D1__LOWCMEN_MASK 0x00000010L
+#define MC_IO_TXCNTL_APHY_D1__LOWCMEN__SHIFT 0x00000004
+#define MC_IO_TXCNTL_APHY_D1__NDRV_MASK 0x00700000L
+#define MC_IO_TXCNTL_APHY_D1__NDRV__SHIFT 0x00000014
+#define MC_IO_TXCNTL_APHY_D1__PDRV_MASK 0x000f0000L
+#define MC_IO_TXCNTL_APHY_D1__PDRV__SHIFT 0x00000010
+#define MC_IO_TXCNTL_APHY_D1__PMA_LOOPBACK_MASK 0x0000e000L
+#define MC_IO_TXCNTL_APHY_D1__PMA_LOOPBACK__SHIFT 0x0000000d
+#define MC_IO_TXCNTL_APHY_D1__PTERM_MASK 0x00000f00L
+#define MC_IO_TXCNTL_APHY_D1__PTERM__SHIFT 0x00000008
+#define MC_IO_TXCNTL_APHY_D1__QDR_MASK 0x00000020L
+#define MC_IO_TXCNTL_APHY_D1__QDR__SHIFT 0x00000005
+#define MC_IO_TXCNTL_APHY_D1__TSTEN_MASK 0x01000000L
+#define MC_IO_TXCNTL_APHY_D1__TSTEN__SHIFT 0x00000018
+#define MC_IO_TXCNTL_APHY_D1__TXBPASS_SEL_MASK 0x00001000L
+#define MC_IO_TXCNTL_APHY_D1__TXBPASS_SEL__SHIFT 0x0000000c
+#define MC_IO_TXCNTL_APHY_D1__TXBYPASS_DATA_MASK 0x38000000L
+#define MC_IO_TXCNTL_APHY_D1__TXBYPASS_DATA__SHIFT 0x0000001b
+#define MC_IO_TXCNTL_APHY_D1__TXBYPASS_MASK 0x04000000L
+#define MC_IO_TXCNTL_APHY_D1__TXBYPASS__SHIFT 0x0000001a
+#define MC_IO_TXCNTL_APHY_D1__TXPD_MASK 0x00000080L
+#define MC_IO_TXCNTL_APHY_D1__TXPD__SHIFT 0x00000007
+#define MC_IO_TXCNTL_APHY_D1__TXRESET_MASK 0x02000000L
+#define MC_IO_TXCNTL_APHY_D1__TXRESET__SHIFT 0x00000019
+#define MC_IO_TXCNTL_APHY_D1__YCLKON_MASK 0x00800000L
+#define MC_IO_TXCNTL_APHY_D1__YCLKON__SHIFT 0x00000017
+#define MC_IO_TXCNTL_DPHY0_D0__BIASSEL_MASK 0x00000003L
+#define MC_IO_TXCNTL_DPHY0_D0__BIASSEL__SHIFT 0x00000000
+#define MC_IO_TXCNTL_DPHY0_D0__DRVDUTY_MASK 0x0000000cL
+#define MC_IO_TXCNTL_DPHY0_D0__DRVDUTY__SHIFT 0x00000002
+#define MC_IO_TXCNTL_DPHY0_D0__EDCTX_CLKGATE_EN_MASK 0x02000000L
+#define MC_IO_TXCNTL_DPHY0_D0__EDCTX_CLKGATE_EN__SHIFT 0x00000019
+#define MC_IO_TXCNTL_DPHY0_D0__EMPH_MASK 0x00000040L
+#define MC_IO_TXCNTL_DPHY0_D0__EMPH__SHIFT 0x00000006
+#define MC_IO_TXCNTL_DPHY0_D0__LOWCMEN_MASK 0x00000010L
+#define MC_IO_TXCNTL_DPHY0_D0__LOWCMEN__SHIFT 0x00000004
+#define MC_IO_TXCNTL_DPHY0_D0__NDRV_MASK 0x00f00000L
+#define MC_IO_TXCNTL_DPHY0_D0__NDRV__SHIFT 0x00000014
+#define MC_IO_TXCNTL_DPHY0_D0__NTERM_MASK 0x0000f000L
+#define MC_IO_TXCNTL_DPHY0_D0__NTERM__SHIFT 0x0000000c
+#define MC_IO_TXCNTL_DPHY0_D0__PDRV_MASK 0x000f0000L
+#define MC_IO_TXCNTL_DPHY0_D0__PDRV__SHIFT 0x00000010
+#define MC_IO_TXCNTL_DPHY0_D0__PLL_LOOPBCK_MASK 0x08000000L
+#define MC_IO_TXCNTL_DPHY0_D0__PLL_LOOPBCK__SHIFT 0x0000001b
+#define MC_IO_TXCNTL_DPHY0_D0__PTERM_MASK 0x00000f00L
+#define MC_IO_TXCNTL_DPHY0_D0__PTERM__SHIFT 0x00000008
+#define MC_IO_TXCNTL_DPHY0_D0__QDR_MASK 0x00000020L
+#define MC_IO_TXCNTL_DPHY0_D0__QDR__SHIFT 0x00000005
+#define MC_IO_TXCNTL_DPHY0_D0__TSTEN_MASK 0x01000000L
+#define MC_IO_TXCNTL_DPHY0_D0__TSTEN__SHIFT 0x00000018
+#define MC_IO_TXCNTL_DPHY0_D0__TXBYPASS_DATA_MASK 0xf0000000L
+#define MC_IO_TXCNTL_DPHY0_D0__TXBYPASS_DATA__SHIFT 0x0000001c
+#define MC_IO_TXCNTL_DPHY0_D0__TXBYPASS_MASK 0x04000000L
+#define MC_IO_TXCNTL_DPHY0_D0__TXBYPASS__SHIFT 0x0000001a
+#define MC_IO_TXCNTL_DPHY0_D0__TXPD_MASK 0x00000080L
+#define MC_IO_TXCNTL_DPHY0_D0__TXPD__SHIFT 0x00000007
+#define MC_IO_TXCNTL_DPHY0_D1__BIASSEL_MASK 0x00000003L
+#define MC_IO_TXCNTL_DPHY0_D1__BIASSEL__SHIFT 0x00000000
+#define MC_IO_TXCNTL_DPHY0_D1__DRVDUTY_MASK 0x0000000cL
+#define MC_IO_TXCNTL_DPHY0_D1__DRVDUTY__SHIFT 0x00000002
+#define MC_IO_TXCNTL_DPHY0_D1__EDCTX_CLKGATE_EN_MASK 0x02000000L
+#define MC_IO_TXCNTL_DPHY0_D1__EDCTX_CLKGATE_EN__SHIFT 0x00000019
+#define MC_IO_TXCNTL_DPHY0_D1__EMPH_MASK 0x00000040L
+#define MC_IO_TXCNTL_DPHY0_D1__EMPH__SHIFT 0x00000006
+#define MC_IO_TXCNTL_DPHY0_D1__LOWCMEN_MASK 0x00000010L
+#define MC_IO_TXCNTL_DPHY0_D1__LOWCMEN__SHIFT 0x00000004
+#define MC_IO_TXCNTL_DPHY0_D1__NDRV_MASK 0x00f00000L
+#define MC_IO_TXCNTL_DPHY0_D1__NDRV__SHIFT 0x00000014
+#define MC_IO_TXCNTL_DPHY0_D1__NTERM_MASK 0x0000f000L
+#define MC_IO_TXCNTL_DPHY0_D1__NTERM__SHIFT 0x0000000c
+#define MC_IO_TXCNTL_DPHY0_D1__PDRV_MASK 0x000f0000L
+#define MC_IO_TXCNTL_DPHY0_D1__PDRV__SHIFT 0x00000010
+#define MC_IO_TXCNTL_DPHY0_D1__PLL_LOOPBCK_MASK 0x08000000L
+#define MC_IO_TXCNTL_DPHY0_D1__PLL_LOOPBCK__SHIFT 0x0000001b
+#define MC_IO_TXCNTL_DPHY0_D1__PTERM_MASK 0x00000f00L
+#define MC_IO_TXCNTL_DPHY0_D1__PTERM__SHIFT 0x00000008
+#define MC_IO_TXCNTL_DPHY0_D1__QDR_MASK 0x00000020L
+#define MC_IO_TXCNTL_DPHY0_D1__QDR__SHIFT 0x00000005
+#define MC_IO_TXCNTL_DPHY0_D1__TSTEN_MASK 0x01000000L
+#define MC_IO_TXCNTL_DPHY0_D1__TSTEN__SHIFT 0x00000018
+#define MC_IO_TXCNTL_DPHY0_D1__TXBYPASS_DATA_MASK 0xf0000000L
+#define MC_IO_TXCNTL_DPHY0_D1__TXBYPASS_DATA__SHIFT 0x0000001c
+#define MC_IO_TXCNTL_DPHY0_D1__TXBYPASS_MASK 0x04000000L
+#define MC_IO_TXCNTL_DPHY0_D1__TXBYPASS__SHIFT 0x0000001a
+#define MC_IO_TXCNTL_DPHY0_D1__TXPD_MASK 0x00000080L
+#define MC_IO_TXCNTL_DPHY0_D1__TXPD__SHIFT 0x00000007
+#define MC_IO_TXCNTL_DPHY1_D0__BIASSEL_MASK 0x00000003L
+#define MC_IO_TXCNTL_DPHY1_D0__BIASSEL__SHIFT 0x00000000
+#define MC_IO_TXCNTL_DPHY1_D0__DRVDUTY_MASK 0x0000000cL
+#define MC_IO_TXCNTL_DPHY1_D0__DRVDUTY__SHIFT 0x00000002
+#define MC_IO_TXCNTL_DPHY1_D0__EDCTX_CLKGATE_EN_MASK 0x02000000L
+#define MC_IO_TXCNTL_DPHY1_D0__EDCTX_CLKGATE_EN__SHIFT 0x00000019
+#define MC_IO_TXCNTL_DPHY1_D0__EMPH_MASK 0x00000040L
+#define MC_IO_TXCNTL_DPHY1_D0__EMPH__SHIFT 0x00000006
+#define MC_IO_TXCNTL_DPHY1_D0__LOWCMEN_MASK 0x00000010L
+#define MC_IO_TXCNTL_DPHY1_D0__LOWCMEN__SHIFT 0x00000004
+#define MC_IO_TXCNTL_DPHY1_D0__NDRV_MASK 0x00f00000L
+#define MC_IO_TXCNTL_DPHY1_D0__NDRV__SHIFT 0x00000014
+#define MC_IO_TXCNTL_DPHY1_D0__NTERM_MASK 0x0000f000L
+#define MC_IO_TXCNTL_DPHY1_D0__NTERM__SHIFT 0x0000000c
+#define MC_IO_TXCNTL_DPHY1_D0__PDRV_MASK 0x000f0000L
+#define MC_IO_TXCNTL_DPHY1_D0__PDRV__SHIFT 0x00000010
+#define MC_IO_TXCNTL_DPHY1_D0__PLL_LOOPBCK_MASK 0x08000000L
+#define MC_IO_TXCNTL_DPHY1_D0__PLL_LOOPBCK__SHIFT 0x0000001b
+#define MC_IO_TXCNTL_DPHY1_D0__PTERM_MASK 0x00000f00L
+#define MC_IO_TXCNTL_DPHY1_D0__PTERM__SHIFT 0x00000008
+#define MC_IO_TXCNTL_DPHY1_D0__QDR_MASK 0x00000020L
+#define MC_IO_TXCNTL_DPHY1_D0__QDR__SHIFT 0x00000005
+#define MC_IO_TXCNTL_DPHY1_D0__TSTEN_MASK 0x01000000L
+#define MC_IO_TXCNTL_DPHY1_D0__TSTEN__SHIFT 0x00000018
+#define MC_IO_TXCNTL_DPHY1_D0__TXBYPASS_DATA_MASK 0xf0000000L
+#define MC_IO_TXCNTL_DPHY1_D0__TXBYPASS_DATA__SHIFT 0x0000001c
+#define MC_IO_TXCNTL_DPHY1_D0__TXBYPASS_MASK 0x04000000L
+#define MC_IO_TXCNTL_DPHY1_D0__TXBYPASS__SHIFT 0x0000001a
+#define MC_IO_TXCNTL_DPHY1_D0__TXPD_MASK 0x00000080L
+#define MC_IO_TXCNTL_DPHY1_D0__TXPD__SHIFT 0x00000007
+#define MC_IO_TXCNTL_DPHY1_D1__BIASSEL_MASK 0x00000003L
+#define MC_IO_TXCNTL_DPHY1_D1__BIASSEL__SHIFT 0x00000000
+#define MC_IO_TXCNTL_DPHY1_D1__DRVDUTY_MASK 0x0000000cL
+#define MC_IO_TXCNTL_DPHY1_D1__DRVDUTY__SHIFT 0x00000002
+#define MC_IO_TXCNTL_DPHY1_D1__EDCTX_CLKGATE_EN_MASK 0x02000000L
+#define MC_IO_TXCNTL_DPHY1_D1__EDCTX_CLKGATE_EN__SHIFT 0x00000019
+#define MC_IO_TXCNTL_DPHY1_D1__EMPH_MASK 0x00000040L
+#define MC_IO_TXCNTL_DPHY1_D1__EMPH__SHIFT 0x00000006
+#define MC_IO_TXCNTL_DPHY1_D1__LOWCMEN_MASK 0x00000010L
+#define MC_IO_TXCNTL_DPHY1_D1__LOWCMEN__SHIFT 0x00000004
+#define MC_IO_TXCNTL_DPHY1_D1__NDRV_MASK 0x00f00000L
+#define MC_IO_TXCNTL_DPHY1_D1__NDRV__SHIFT 0x00000014
+#define MC_IO_TXCNTL_DPHY1_D1__NTERM_MASK 0x0000f000L
+#define MC_IO_TXCNTL_DPHY1_D1__NTERM__SHIFT 0x0000000c
+#define MC_IO_TXCNTL_DPHY1_D1__PDRV_MASK 0x000f0000L
+#define MC_IO_TXCNTL_DPHY1_D1__PDRV__SHIFT 0x00000010
+#define MC_IO_TXCNTL_DPHY1_D1__PLL_LOOPBCK_MASK 0x08000000L
+#define MC_IO_TXCNTL_DPHY1_D1__PLL_LOOPBCK__SHIFT 0x0000001b
+#define MC_IO_TXCNTL_DPHY1_D1__PTERM_MASK 0x00000f00L
+#define MC_IO_TXCNTL_DPHY1_D1__PTERM__SHIFT 0x00000008
+#define MC_IO_TXCNTL_DPHY1_D1__QDR_MASK 0x00000020L
+#define MC_IO_TXCNTL_DPHY1_D1__QDR__SHIFT 0x00000005
+#define MC_IO_TXCNTL_DPHY1_D1__TSTEN_MASK 0x01000000L
+#define MC_IO_TXCNTL_DPHY1_D1__TSTEN__SHIFT 0x00000018
+#define MC_IO_TXCNTL_DPHY1_D1__TXBYPASS_DATA_MASK 0xf0000000L
+#define MC_IO_TXCNTL_DPHY1_D1__TXBYPASS_DATA__SHIFT 0x0000001c
+#define MC_IO_TXCNTL_DPHY1_D1__TXBYPASS_MASK 0x04000000L
+#define MC_IO_TXCNTL_DPHY1_D1__TXBYPASS__SHIFT 0x0000001a
+#define MC_IO_TXCNTL_DPHY1_D1__TXPD_MASK 0x00000080L
+#define MC_IO_TXCNTL_DPHY1_D1__TXPD__SHIFT 0x00000007
+#define MCLK_PWRMGT_CNTL__DLL_READY_MASK 0x00000040L
+#define MCLK_PWRMGT_CNTL__DLL_READY_READ_MASK 0x01000000L
+#define MCLK_PWRMGT_CNTL__DLL_READY_READ__SHIFT 0x00000018
+#define MCLK_PWRMGT_CNTL__DLL_READY__SHIFT 0x00000006
+#define MCLK_PWRMGT_CNTL__DLL_SPEED_MASK 0x0000001fL
+#define MCLK_PWRMGT_CNTL__DLL_SPEED__SHIFT 0x00000000
+#define MCLK_PWRMGT_CNTL__MC_INT_CNTL_MASK 0x00000080L
+#define MCLK_PWRMGT_CNTL__MC_INT_CNTL__SHIFT 0x00000007
+#define MCLK_PWRMGT_CNTL__MRDCK0_PDNB_MASK 0x00000100L
+#define MCLK_PWRMGT_CNTL__MRDCK0_PDNB__SHIFT 0x00000008
+#define MCLK_PWRMGT_CNTL__MRDCK0_RESET_MASK 0x00010000L
+#define MCLK_PWRMGT_CNTL__MRDCK0_RESET__SHIFT 0x00000010
+#define MCLK_PWRMGT_CNTL__MRDCK1_PDNB_MASK 0x00000200L
+#define MCLK_PWRMGT_CNTL__MRDCK1_PDNB__SHIFT 0x00000009
+#define MCLK_PWRMGT_CNTL__MRDCK1_RESET_MASK 0x00020000L
+#define MCLK_PWRMGT_CNTL__MRDCK1_RESET__SHIFT 0x00000011
+#define MC_MEM_POWER_LS__LS_HOLD_MASK 0x00000fc0L
+#define MC_MEM_POWER_LS__LS_HOLD__SHIFT 0x00000006
+#define MC_MEM_POWER_LS__LS_SETUP_MASK 0x0000003fL
+#define MC_MEM_POWER_LS__LS_SETUP__SHIFT 0x00000000
+#define MC_NPL_STATUS__D0_NDELAY_MASK 0x0000000cL
+#define MC_NPL_STATUS__D0_NDELAY__SHIFT 0x00000002
+#define MC_NPL_STATUS__D0_NEARLY_MASK 0x00000020L
+#define MC_NPL_STATUS__D0_NEARLY__SHIFT 0x00000005
+#define MC_NPL_STATUS__D0_PDELAY_MASK 0x00000003L
+#define MC_NPL_STATUS__D0_PDELAY__SHIFT 0x00000000
+#define MC_NPL_STATUS__D0_PEARLY_MASK 0x00000010L
+#define MC_NPL_STATUS__D0_PEARLY__SHIFT 0x00000004
+#define MC_NPL_STATUS__D1_NDELAY_MASK 0x00000300L
+#define MC_NPL_STATUS__D1_NDELAY__SHIFT 0x00000008
+#define MC_NPL_STATUS__D1_NEARLY_MASK 0x00000800L
+#define MC_NPL_STATUS__D1_NEARLY__SHIFT 0x0000000b
+#define MC_NPL_STATUS__D1_PDELAY_MASK 0x000000c0L
+#define MC_NPL_STATUS__D1_PDELAY__SHIFT 0x00000006
+#define MC_NPL_STATUS__D1_PEARLY_MASK 0x00000400L
+#define MC_NPL_STATUS__D1_PEARLY__SHIFT 0x0000000a
+#define MC_PHY_TIMING_2__ADR_CLKEN_D0_MASK 0x00040000L
+#define MC_PHY_TIMING_2__ADR_CLKEN_D0__SHIFT 0x00000012
+#define MC_PHY_TIMING_2__ADR_CLKEN_D1_MASK 0x00080000L
+#define MC_PHY_TIMING_2__ADR_CLKEN_D1__SHIFT 0x00000013
+#define MC_PHY_TIMING_2__IND_LD_CNT_MASK 0x0000007fL
+#define MC_PHY_TIMING_2__IND_LD_CNT__SHIFT 0x00000000
+#define MC_PHY_TIMING_2__RXC0_FRC_MASK 0x00001000L
+#define MC_PHY_TIMING_2__RXC0_FRC__SHIFT 0x0000000c
+#define MC_PHY_TIMING_2__RXC0_INV_MASK 0x00000100L
+#define MC_PHY_TIMING_2__RXC0_INV__SHIFT 0x00000008
+#define MC_PHY_TIMING_2__RXC1_FRC_MASK 0x00002000L
+#define MC_PHY_TIMING_2__RXC1_FRC__SHIFT 0x0000000d
+#define MC_PHY_TIMING_2__RXC1_INV_MASK 0x00000200L
+#define MC_PHY_TIMING_2__RXC1_INV__SHIFT 0x00000009
+#define MC_PHY_TIMING_2__TXC0_FRC_MASK 0x00004000L
+#define MC_PHY_TIMING_2__TXC0_FRC__SHIFT 0x0000000e
+#define MC_PHY_TIMING_2__TXC0_INV_MASK 0x00000400L
+#define MC_PHY_TIMING_2__TXC0_INV__SHIFT 0x0000000a
+#define MC_PHY_TIMING_2__TXC1_FRC_MASK 0x00008000L
+#define MC_PHY_TIMING_2__TXC1_FRC__SHIFT 0x0000000f
+#define MC_PHY_TIMING_2__TXC1_INV_MASK 0x00000800L
+#define MC_PHY_TIMING_2__TXC1_INV__SHIFT 0x0000000b
+#define MC_PHY_TIMING_2__TX_CDREN_D0_MASK 0x00010000L
+#define MC_PHY_TIMING_2__TX_CDREN_D0__SHIFT 0x00000010
+#define MC_PHY_TIMING_2__TX_CDREN_D1_MASK 0x00020000L
+#define MC_PHY_TIMING_2__TX_CDREN_D1__SHIFT 0x00000011
+#define MC_PHY_TIMING_2__WR_DLY_MASK 0x00f00000L
+#define MC_PHY_TIMING_2__WR_DLY__SHIFT 0x00000014
+#define MC_PHY_TIMING_D0__RXC0_DLY_MASK 0x0000000fL
+#define MC_PHY_TIMING_D0__RXC0_DLY__SHIFT 0x00000000
+#define MC_PHY_TIMING_D0__RXC0_EXT_MASK 0x000000f0L
+#define MC_PHY_TIMING_D0__RXC0_EXT__SHIFT 0x00000004
+#define MC_PHY_TIMING_D0__RXC1_DLY_MASK 0x00000f00L
+#define MC_PHY_TIMING_D0__RXC1_DLY__SHIFT 0x00000008
+#define MC_PHY_TIMING_D0__RXC1_EXT_MASK 0x0000f000L
+#define MC_PHY_TIMING_D0__RXC1_EXT__SHIFT 0x0000000c
+#define MC_PHY_TIMING_D0__TXC0_DLY_MASK 0x00070000L
+#define MC_PHY_TIMING_D0__TXC0_DLY__SHIFT 0x00000010
+#define MC_PHY_TIMING_D0__TXC0_EXT_MASK 0x00f00000L
+#define MC_PHY_TIMING_D0__TXC0_EXT__SHIFT 0x00000014
+#define MC_PHY_TIMING_D0__TXC1_DLY_MASK 0x07000000L
+#define MC_PHY_TIMING_D0__TXC1_DLY__SHIFT 0x00000018
+#define MC_PHY_TIMING_D0__TXC1_EXT_MASK 0xf0000000L
+#define MC_PHY_TIMING_D0__TXC1_EXT__SHIFT 0x0000001c
+#define MC_PHY_TIMING_D1__RXC0_DLY_MASK 0x0000000fL
+#define MC_PHY_TIMING_D1__RXC0_DLY__SHIFT 0x00000000
+#define MC_PHY_TIMING_D1__RXC0_EXT_MASK 0x000000f0L
+#define MC_PHY_TIMING_D1__RXC0_EXT__SHIFT 0x00000004
+#define MC_PHY_TIMING_D1__RXC1_DLY_MASK 0x00000f00L
+#define MC_PHY_TIMING_D1__RXC1_DLY__SHIFT 0x00000008
+#define MC_PHY_TIMING_D1__RXC1_EXT_MASK 0x0000f000L
+#define MC_PHY_TIMING_D1__RXC1_EXT__SHIFT 0x0000000c
+#define MC_PHY_TIMING_D1__TXC0_DLY_MASK 0x00070000L
+#define MC_PHY_TIMING_D1__TXC0_DLY__SHIFT 0x00000010
+#define MC_PHY_TIMING_D1__TXC0_EXT_MASK 0x00f00000L
+#define MC_PHY_TIMING_D1__TXC0_EXT__SHIFT 0x00000014
+#define MC_PHY_TIMING_D1__TXC1_DLY_MASK 0x07000000L
+#define MC_PHY_TIMING_D1__TXC1_DLY__SHIFT 0x00000018
+#define MC_PHY_TIMING_D1__TXC1_EXT_MASK 0xf0000000L
+#define MC_PHY_TIMING_D1__TXC1_EXT__SHIFT 0x0000001c
+#define MC_PMG_AUTO_CFG__DLL_CNT_MASK 0xff000000L
+#define MC_PMG_AUTO_CFG__DLL_CNT__SHIFT 0x00000018
+#define MC_PMG_AUTO_CFG__EXIT_ALLOW_STOP_MASK 0x00000800L
+#define MC_PMG_AUTO_CFG__EXIT_ALLOW_STOP__SHIFT 0x0000000b
+#define MC_PMG_AUTO_CFG__MRS_WAIT_CNT_MASK 0x000f0000L
+#define MC_PMG_AUTO_CFG__MRS_WAIT_CNT__SHIFT 0x00000010
+#define MC_PMG_AUTO_CFG__PREA_SRX_MASK 0x00002000L
+#define MC_PMG_AUTO_CFG__PREA_SRX__SHIFT 0x0000000d
+#define MC_PMG_AUTO_CFG__RFS_SRX_MASK 0x00001000L
+#define MC_PMG_AUTO_CFG__RFS_SRX__SHIFT 0x0000000c
+#define MC_PMG_AUTO_CFG__RST_MRS_MASK 0x00000002L
+#define MC_PMG_AUTO_CFG__RST_MRS__SHIFT 0x00000001
+#define MC_PMG_AUTO_CFG__RXPDNB_MASK 0x00400000L
+#define MC_PMG_AUTO_CFG__RXPDNB__SHIFT 0x00000016
+#define MC_PMG_AUTO_CFG__SCDS_MODE_MASK 0x00000400L
+#define MC_PMG_AUTO_CFG__SCDS_MODE__SHIFT 0x0000000a
+#define MC_PMG_AUTO_CFG__SELFREFR_COMMIT_0_MASK 0x00008000L
+#define MC_PMG_AUTO_CFG__SELFREFR_COMMIT_0__SHIFT 0x0000000f
+#define MC_PMG_AUTO_CFG__SELFREFR_COMMIT_1_MASK 0x00800000L
+#define MC_PMG_AUTO_CFG__SELFREFR_COMMIT_1__SHIFT 0x00000017
+#define MC_PMG_AUTO_CFG__SS_ALWAYS_SLF_MASK 0x00000100L
+#define MC_PMG_AUTO_CFG__SS_ALWAYS_SLF__SHIFT 0x00000008
+#define MC_PMG_AUTO_CFG__SS_S_SLF_MASK 0x00000200L
+#define MC_PMG_AUTO_CFG__SS_S_SLF__SHIFT 0x00000009
+#define MC_PMG_AUTO_CFG__STUTTER_EN_MASK 0x00004000L
+#define MC_PMG_AUTO_CFG__STUTTER_EN__SHIFT 0x0000000e
+#define MC_PMG_AUTO_CFG__SYC_CLK_MASK 0x00000001L
+#define MC_PMG_AUTO_CFG__SYC_CLK__SHIFT 0x00000000
+#define MC_PMG_AUTO_CFG__TRI_MIO_MASK 0x00000004L
+#define MC_PMG_AUTO_CFG__TRI_MIO__SHIFT 0x00000002
+#define MC_PMG_AUTO_CFG__WRITE_DURING_DLOCK_MASK 0x00100000L
+#define MC_PMG_AUTO_CFG__WRITE_DURING_DLOCK__SHIFT 0x00000014
+#define MC_PMG_AUTO_CFG__XSR_TMR_MASK 0x000000f0L
+#define MC_PMG_AUTO_CFG__XSR_TMR__SHIFT 0x00000004
+#define MC_PMG_AUTO_CFG__YCLK_ON_MASK 0x00200000L
+#define MC_PMG_AUTO_CFG__YCLK_ON__SHIFT 0x00000015
+#define MC_PMG_AUTO_CMD__ADR_MASK 0x0001ffffL
+#define MC_PMG_AUTO_CMD__ADR_MSB0_MASK 0x20000000L
+#define MC_PMG_AUTO_CMD__ADR_MSB0__SHIFT 0x0000001d
+#define MC_PMG_AUTO_CMD__ADR_MSB1_MASK 0x10000000L
+#define MC_PMG_AUTO_CMD__ADR_MSB1__SHIFT 0x0000001c
+#define MC_PMG_AUTO_CMD__ADR__SHIFT 0x00000000
+#define MC_PMG_CFG__DPM_WAKE_MASK 0x00000400L
+#define MC_PMG_CFG__DPM_WAKE__SHIFT 0x0000000a
+#define MC_PMG_CFG__EARLY_ACK_ACPI_MASK 0x00400000L
+#define MC_PMG_CFG__EARLY_ACK_ACPI__SHIFT 0x00000016
+#define MC_PMG_CFG__MRS_WAIT_CNT_MASK 0x000f0000L
+#define MC_PMG_CFG__MRS_WAIT_CNT__SHIFT 0x00000010
+#define MC_PMG_CFG__PREA_SRX_MASK 0x00002000L
+#define MC_PMG_CFG__PREA_SRX__SHIFT 0x0000000d
+#define MC_PMG_CFG__RFS_SRX_MASK 0x00001000L
+#define MC_PMG_CFG__RFS_SRX__SHIFT 0x0000000c
+#define MC_PMG_CFG__RST_EMRS_MASK 0x00000004L
+#define MC_PMG_CFG__RST_EMRS__SHIFT 0x00000002
+#define MC_PMG_CFG__RST_MRS1_MASK 0x00000100L
+#define MC_PMG_CFG__RST_MRS1__SHIFT 0x00000008
+#define MC_PMG_CFG__RST_MRS2_MASK 0x00000200L
+#define MC_PMG_CFG__RST_MRS2__SHIFT 0x00000009
+#define MC_PMG_CFG__RST_MRS_MASK 0x00000002L
+#define MC_PMG_CFG__RST_MRS__SHIFT 0x00000001
+#define MC_PMG_CFG__RXPDNB_MASK 0x02000000L
+#define MC_PMG_CFG__RXPDNB__SHIFT 0x00000019
+#define MC_PMG_CFG__SYC_CLK_MASK 0x00000001L
+#define MC_PMG_CFG__SYC_CLK__SHIFT 0x00000000
+#define MC_PMG_CFG__TRI_MIO_MASK 0x00000008L
+#define MC_PMG_CFG__TRI_MIO__SHIFT 0x00000003
+#define MC_PMG_CFG__WRITE_DURING_DLOCK_MASK 0x00100000L
+#define MC_PMG_CFG__WRITE_DURING_DLOCK__SHIFT 0x00000014
+#define MC_PMG_CFG__XSR_TMR_MASK 0x000000f0L
+#define MC_PMG_CFG__XSR_TMR__SHIFT 0x00000004
+#define MC_PMG_CFG__YCLK_ON_MASK 0x00200000L
+#define MC_PMG_CFG__YCLK_ON__SHIFT 0x00000015
+#define MC_PMG_CFG__ZQCL_SEND_MASK 0x0c000000L
+#define MC_PMG_CFG__ZQCL_SEND__SHIFT 0x0000001a
+#define MC_PMG_CMD_EMRS__ADR_MASK 0x0000ffffL
+#define MC_PMG_CMD_EMRS__ADR_MSB0_MASK 0x20000000L
+#define MC_PMG_CMD_EMRS__ADR_MSB0__SHIFT 0x0000001d
+#define MC_PMG_CMD_EMRS__ADR_MSB1_MASK 0x10000000L
+#define MC_PMG_CMD_EMRS__ADR_MSB1__SHIFT 0x0000001c
+#define MC_PMG_CMD_EMRS__ADR__SHIFT 0x00000000
+#define MC_PMG_CMD_EMRS__BNK_MSB_MASK 0x00080000L
+#define MC_PMG_CMD_EMRS__BNK_MSB__SHIFT 0x00000013
+#define MC_PMG_CMD_EMRS__CSB_MASK 0x00600000L
+#define MC_PMG_CMD_EMRS__CSB__SHIFT 0x00000015
+#define MC_PMG_CMD_EMRS__END_MASK 0x00100000L
+#define MC_PMG_CMD_EMRS__END__SHIFT 0x00000014
+#define MC_PMG_CMD_EMRS__MOP_MASK 0x00070000L
+#define MC_PMG_CMD_EMRS__MOP__SHIFT 0x00000010
+#define MC_PMG_CMD_MRS1__ADR_MASK 0x0000ffffL
+#define MC_PMG_CMD_MRS1__ADR_MSB0_MASK 0x20000000L
+#define MC_PMG_CMD_MRS1__ADR_MSB0__SHIFT 0x0000001d
+#define MC_PMG_CMD_MRS1__ADR_MSB1_MASK 0x10000000L
+#define MC_PMG_CMD_MRS1__ADR_MSB1__SHIFT 0x0000001c
+#define MC_PMG_CMD_MRS1__ADR__SHIFT 0x00000000
+#define MC_PMG_CMD_MRS1__BNK_MSB_MASK 0x00080000L
+#define MC_PMG_CMD_MRS1__BNK_MSB__SHIFT 0x00000013
+#define MC_PMG_CMD_MRS1__CSB_MASK 0x00600000L
+#define MC_PMG_CMD_MRS1__CSB__SHIFT 0x00000015
+#define MC_PMG_CMD_MRS1__END_MASK 0x00100000L
+#define MC_PMG_CMD_MRS1__END__SHIFT 0x00000014
+#define MC_PMG_CMD_MRS1__MOP_MASK 0x00070000L
+#define MC_PMG_CMD_MRS1__MOP__SHIFT 0x00000010
+#define MC_PMG_CMD_MRS2__ADR_MASK 0x0000ffffL
+#define MC_PMG_CMD_MRS2__ADR_MSB0_MASK 0x20000000L
+#define MC_PMG_CMD_MRS2__ADR_MSB0__SHIFT 0x0000001d
+#define MC_PMG_CMD_MRS2__ADR_MSB1_MASK 0x10000000L
+#define MC_PMG_CMD_MRS2__ADR_MSB1__SHIFT 0x0000001c
+#define MC_PMG_CMD_MRS2__ADR__SHIFT 0x00000000
+#define MC_PMG_CMD_MRS2__BNK_MSB_MASK 0x00080000L
+#define MC_PMG_CMD_MRS2__BNK_MSB__SHIFT 0x00000013
+#define MC_PMG_CMD_MRS2__CSB_MASK 0x00600000L
+#define MC_PMG_CMD_MRS2__CSB__SHIFT 0x00000015
+#define MC_PMG_CMD_MRS2__END_MASK 0x00100000L
+#define MC_PMG_CMD_MRS2__END__SHIFT 0x00000014
+#define MC_PMG_CMD_MRS2__MOP_MASK 0x00070000L
+#define MC_PMG_CMD_MRS2__MOP__SHIFT 0x00000010
+#define MC_PMG_CMD_MRS__ADR_MASK 0x0000ffffL
+#define MC_PMG_CMD_MRS__ADR_MSB0_MASK 0x20000000L
+#define MC_PMG_CMD_MRS__ADR_MSB0__SHIFT 0x0000001d
+#define MC_PMG_CMD_MRS__ADR_MSB1_MASK 0x10000000L
+#define MC_PMG_CMD_MRS__ADR_MSB1__SHIFT 0x0000001c
+#define MC_PMG_CMD_MRS__ADR__SHIFT 0x00000000
+#define MC_PMG_CMD_MRS__BNK_MSB_MASK 0x00080000L
+#define MC_PMG_CMD_MRS__BNK_MSB__SHIFT 0x00000013
+#define MC_PMG_CMD_MRS__CSB_MASK 0x00600000L
+#define MC_PMG_CMD_MRS__CSB__SHIFT 0x00000015
+#define MC_PMG_CMD_MRS__END_MASK 0x00100000L
+#define MC_PMG_CMD_MRS__END__SHIFT 0x00000014
+#define MC_PMG_CMD_MRS__MOP_MASK 0x00070000L
+#define MC_PMG_CMD_MRS__MOP__SHIFT 0x00000010
+#define MC_RD_CB__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_RD_CB__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_RD_CB__ENABLE_MASK 0x00000001L
+#define MC_RD_CB__ENABLE__SHIFT 0x00000000
+#define MC_RD_CB__LAZY_TIMER_MASK 0x00007800L
+#define MC_RD_CB__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_RD_CB__MAX_BURST_MASK 0x00000780L
+#define MC_RD_CB__MAX_BURST__SHIFT 0x00000007
+#define MC_RD_CB__PRESCALE_MASK 0x00000006L
+#define MC_RD_CB__PRESCALE__SHIFT 0x00000001
+#define MC_RD_CB__STALL_MODE_MASK 0x00000030L
+#define MC_RD_CB__STALL_MODE__SHIFT 0x00000004
+#define MC_RD_CB__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_RD_CB__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_RD_CB__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_RD_CB__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_RD_DB__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_RD_DB__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_RD_DB__ENABLE_MASK 0x00000001L
+#define MC_RD_DB__ENABLE__SHIFT 0x00000000
+#define MC_RD_DB__LAZY_TIMER_MASK 0x00007800L
+#define MC_RD_DB__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_RD_DB__MAX_BURST_MASK 0x00000780L
+#define MC_RD_DB__MAX_BURST__SHIFT 0x00000007
+#define MC_RD_DB__PRESCALE_MASK 0x00000006L
+#define MC_RD_DB__PRESCALE__SHIFT 0x00000001
+#define MC_RD_DB__STALL_MODE_MASK 0x00000030L
+#define MC_RD_DB__STALL_MODE__SHIFT 0x00000004
+#define MC_RD_DB__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_RD_DB__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_RD_DB__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_RD_DB__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_RD_GRP_EXT__DBSTEN0_MASK 0x0000000fL
+#define MC_RD_GRP_EXT__DBSTEN0__SHIFT 0x00000000
+#define MC_RD_GRP_EXT__TC0_MASK 0x000000f0L
+#define MC_RD_GRP_EXT__TC0__SHIFT 0x00000004
+#define MC_RD_GRP_GFX__CP_MASK 0x0000000fL
+#define MC_RD_GRP_GFX__CP__SHIFT 0x00000000
+#define MC_RD_GRP_GFX__XDMAM_MASK 0x000f0000L
+#define MC_RD_GRP_GFX__XDMAM__SHIFT 0x00000010
+#define MC_RD_GRP_LCL__CB0_MASK 0x0000f000L
+#define MC_RD_GRP_LCL__CB0__SHIFT 0x0000000c
+#define MC_RD_GRP_LCL__CBCMASK0_MASK 0x000f0000L
+#define MC_RD_GRP_LCL__CBCMASK0__SHIFT 0x00000010
+#define MC_RD_GRP_LCL__CBFMASK0_MASK 0x00f00000L
+#define MC_RD_GRP_LCL__CBFMASK0__SHIFT 0x00000014
+#define MC_RD_GRP_LCL__DB0_MASK 0x0f000000L
+#define MC_RD_GRP_LCL__DB0__SHIFT 0x00000018
+#define MC_RD_GRP_LCL__DBHTILE0_MASK 0xf0000000L
+#define MC_RD_GRP_LCL__DBHTILE0__SHIFT 0x0000001c
+#define MC_RD_GRP_OTH__HDP_MASK 0x00000f00L
+#define MC_RD_GRP_OTH__HDP__SHIFT 0x00000008
+#define MC_RD_GRP_OTH__SEM_MASK 0x0000f000L
+#define MC_RD_GRP_OTH__SEM__SHIFT 0x0000000c
+#define MC_RD_GRP_OTH__UMC_MASK 0x000f0000L
+#define MC_RD_GRP_OTH__UMC__SHIFT 0x00000010
+#define MC_RD_GRP_OTH__UVD_EXT0_MASK 0x0000000fL
+#define MC_RD_GRP_OTH__UVD_EXT0__SHIFT 0x00000000
+#define MC_RD_GRP_OTH__UVD_EXT1_MASK 0x0f000000L
+#define MC_RD_GRP_OTH__UVD_EXT1__SHIFT 0x00000018
+#define MC_RD_GRP_OTH__UVD_MASK 0x00f00000L
+#define MC_RD_GRP_OTH__UVD__SHIFT 0x00000014
+#define MC_RD_GRP_SYS__DMIF_MASK 0x0000f000L
+#define MC_RD_GRP_SYS__DMIF__SHIFT 0x0000000c
+#define MC_RD_GRP_SYS__MCIF_MASK 0x000f0000L
+#define MC_RD_GRP_SYS__MCIF__SHIFT 0x00000010
+#define MC_RD_GRP_SYS__RLC_MASK 0x0000000fL
+#define MC_RD_GRP_SYS__RLC__SHIFT 0x00000000
+#define MC_RD_GRP_SYS__SMU_MASK 0x00f00000L
+#define MC_RD_GRP_SYS__SMU__SHIFT 0x00000014
+#define MC_RD_GRP_SYS__VCE_MASK 0x0f000000L
+#define MC_RD_GRP_SYS__VCE__SHIFT 0x00000018
+#define MC_RD_GRP_SYS__VCEU_MASK 0xf0000000L
+#define MC_RD_GRP_SYS__VCEU__SHIFT 0x0000001c
+#define MC_RD_GRP_SYS__VMC_MASK 0x000000f0L
+#define MC_RD_GRP_SYS__VMC__SHIFT 0x00000004
+#define MC_RD_HUB__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_RD_HUB__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_RD_HUB__ENABLE_MASK 0x00000001L
+#define MC_RD_HUB__ENABLE__SHIFT 0x00000000
+#define MC_RD_HUB__LAZY_TIMER_MASK 0x00007800L
+#define MC_RD_HUB__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_RD_HUB__MAX_BURST_MASK 0x00000780L
+#define MC_RD_HUB__MAX_BURST__SHIFT 0x00000007
+#define MC_RD_HUB__PRESCALE_MASK 0x00000006L
+#define MC_RD_HUB__PRESCALE__SHIFT 0x00000001
+#define MC_RD_HUB__STALL_MODE_MASK 0x00000030L
+#define MC_RD_HUB__STALL_MODE__SHIFT 0x00000004
+#define MC_RD_HUB__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_RD_HUB__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_RD_HUB__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_RD_HUB__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_RD_TC0__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_RD_TC0__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_RD_TC0__ENABLE_MASK 0x00000001L
+#define MC_RD_TC0__ENABLE__SHIFT 0x00000000
+#define MC_RD_TC0__LAZY_TIMER_MASK 0x00007800L
+#define MC_RD_TC0__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_RD_TC0__MAX_BURST_MASK 0x00000780L
+#define MC_RD_TC0__MAX_BURST__SHIFT 0x00000007
+#define MC_RD_TC0__PRESCALE_MASK 0x00000006L
+#define MC_RD_TC0__PRESCALE__SHIFT 0x00000001
+#define MC_RD_TC0__STALL_MODE_MASK 0x00000030L
+#define MC_RD_TC0__STALL_MODE__SHIFT 0x00000004
+#define MC_RD_TC0__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_RD_TC0__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_RD_TC0__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_RD_TC0__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_RD_TC1__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_RD_TC1__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_RD_TC1__ENABLE_MASK 0x00000001L
+#define MC_RD_TC1__ENABLE__SHIFT 0x00000000
+#define MC_RD_TC1__LAZY_TIMER_MASK 0x00007800L
+#define MC_RD_TC1__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_RD_TC1__MAX_BURST_MASK 0x00000780L
+#define MC_RD_TC1__MAX_BURST__SHIFT 0x00000007
+#define MC_RD_TC1__PRESCALE_MASK 0x00000006L
+#define MC_RD_TC1__PRESCALE__SHIFT 0x00000001
+#define MC_RD_TC1__STALL_MODE_MASK 0x00000030L
+#define MC_RD_TC1__STALL_MODE__SHIFT 0x00000004
+#define MC_RD_TC1__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_RD_TC1__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_RD_TC1__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_RD_TC1__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_RPB_ARB_CNTL__ATC_SWITCH_NUM_MASK 0x00ff0000L
+#define MC_RPB_ARB_CNTL__ATC_SWITCH_NUM__SHIFT 0x00000010
+#define MC_RPB_ARB_CNTL__RD_SWITCH_NUM_MASK 0x0000ff00L
+#define MC_RPB_ARB_CNTL__RD_SWITCH_NUM__SHIFT 0x00000008
+#define MC_RPB_ARB_CNTL__WR_SWITCH_NUM_MASK 0x000000ffL
+#define MC_RPB_ARB_CNTL__WR_SWITCH_NUM__SHIFT 0x00000000
+#define MC_RPB_BIF_CNTL__ARB_SWITCH_NUM_MASK 0x000000ffL
+#define MC_RPB_BIF_CNTL__ARB_SWITCH_NUM__SHIFT 0x00000000
+#define MC_RPB_BIF_CNTL__XPB_SWITCH_NUM_MASK 0x0000ff00L
+#define MC_RPB_BIF_CNTL__XPB_SWITCH_NUM__SHIFT 0x00000008
+#define MC_RPB_CID_QUEUE_EX_DATA__READ_ENTRIES_MASK 0xffff0000L
+#define MC_RPB_CID_QUEUE_EX_DATA__READ_ENTRIES__SHIFT 0x00000010
+#define MC_RPB_CID_QUEUE_EX_DATA__WRITE_ENTRIES_MASK 0x0000ffffL
+#define MC_RPB_CID_QUEUE_EX_DATA__WRITE_ENTRIES__SHIFT 0x00000000
+#define MC_RPB_CID_QUEUE_EX__OFFSET_MASK 0x0000003eL
+#define MC_RPB_CID_QUEUE_EX__OFFSET__SHIFT 0x00000001
+#define MC_RPB_CID_QUEUE_EX__START_MASK 0x00000001L
+#define MC_RPB_CID_QUEUE_EX__START__SHIFT 0x00000000
+#define MC_RPB_CID_QUEUE_RD__CLIENT_ID_MASK 0x000000ffL
+#define MC_RPB_CID_QUEUE_RD__CLIENT_ID__SHIFT 0x00000000
+#define MC_RPB_CID_QUEUE_RD__READ_QUEUE_MASK 0x00000c00L
+#define MC_RPB_CID_QUEUE_RD__READ_QUEUE__SHIFT 0x0000000a
+#define MC_RPB_CID_QUEUE_RD__WRITE_QUEUE_MASK 0x00000300L
+#define MC_RPB_CID_QUEUE_RD__WRITE_QUEUE__SHIFT 0x00000008
+#define MC_RPB_CID_QUEUE_WR__CLIENT_ID_MASK 0x000000ffL
+#define MC_RPB_CID_QUEUE_WR__CLIENT_ID__SHIFT 0x00000000
+#define MC_RPB_CID_QUEUE_WR__READ_QUEUE_MASK 0x00001800L
+#define MC_RPB_CID_QUEUE_WR__READ_QUEUE__SHIFT 0x0000000b
+#define MC_RPB_CID_QUEUE_WR__UPDATE_MASK 0x00002000L
+#define MC_RPB_CID_QUEUE_WR__UPDATE_MODE_MASK 0x00000100L
+#define MC_RPB_CID_QUEUE_WR__UPDATE_MODE__SHIFT 0x00000008
+#define MC_RPB_CID_QUEUE_WR__UPDATE__SHIFT 0x0000000d
+#define MC_RPB_CID_QUEUE_WR__WRITE_QUEUE_MASK 0x00000600L
+#define MC_RPB_CID_QUEUE_WR__WRITE_QUEUE__SHIFT 0x00000009
+#define MC_RPB_CONF__RPB_RD_PCIE_ORDER_MASK 0x00010000L
+#define MC_RPB_CONF__RPB_RD_PCIE_ORDER__SHIFT 0x00000010
+#define MC_RPB_CONF__RPB_WR_PCIE_ORDER_MASK 0x00020000L
+#define MC_RPB_CONF__RPB_WR_PCIE_ORDER__SHIFT 0x00000011
+#define MC_RPB_CONF__XPB_PCIE_ORDER_MASK 0x00008000L
+#define MC_RPB_CONF__XPB_PCIE_ORDER__SHIFT 0x0000000f
+#define MC_RPB_DBG1__DEBUG_BITS_MASK 0xfff00000L
+#define MC_RPB_DBG1__DEBUG_BITS__SHIFT 0x00000014
+#define MC_RPB_DBG1__RPB_BIF_OUTSTANDING_RD_32B_MASK 0x000fff00L
+#define MC_RPB_DBG1__RPB_BIF_OUTSTANDING_RD_32B__SHIFT 0x00000008
+#define MC_RPB_DBG1__RPB_BIF_OUTSTANDING_RD_MASK 0x000000ffL
+#define MC_RPB_DBG1__RPB_BIF_OUTSTANDING_RD__SHIFT 0x00000000
+#define MC_RPB_EFF_CNTL__RD_LAZY_TIMER_MASK 0x0000ff00L
+#define MC_RPB_EFF_CNTL__RD_LAZY_TIMER__SHIFT 0x00000008
+#define MC_RPB_EFF_CNTL__WR_LAZY_TIMER_MASK 0x000000ffL
+#define MC_RPB_EFF_CNTL__WR_LAZY_TIMER__SHIFT 0x00000000
+#define MC_RPB_IF_CONF__OUTSTANDING_WRRET_ASK_MASK 0x0000ff00L
+#define MC_RPB_IF_CONF__OUTSTANDING_WRRET_ASK__SHIFT 0x00000008
+#define MC_RPB_IF_CONF__RPB_BIF_CREDITS_MASK 0x000000ffL
+#define MC_RPB_IF_CONF__RPB_BIF_CREDITS__SHIFT 0x00000000
+#define MC_RPB_PERF_COUNTER_CNTL__CLEAR_ALL_PERF_COUNTERS_MASK 0x00000008L
+#define MC_RPB_PERF_COUNTER_CNTL__CLEAR_ALL_PERF_COUNTERS__SHIFT 0x00000003
+#define MC_RPB_PERF_COUNTER_CNTL__CLEAR_SELECTED_PERF_COUNTER_MASK 0x00000004L
+#define MC_RPB_PERF_COUNTER_CNTL__CLEAR_SELECTED_PERF_COUNTER__SHIFT 0x00000002
+#define MC_RPB_PERF_COUNTER_CNTL__ENABLE_PERF_COUNTERS_MASK 0x000001e0L
+#define MC_RPB_PERF_COUNTER_CNTL__ENABLE_PERF_COUNTERS__SHIFT 0x00000005
+#define MC_RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_0_MASK 0x00003e00L
+#define MC_RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_0__SHIFT 0x00000009
+#define MC_RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_1_MASK 0x0007c000L
+#define MC_RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_1__SHIFT 0x0000000e
+#define MC_RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_2_MASK 0x00f80000L
+#define MC_RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_2__SHIFT 0x00000013
+#define MC_RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_3_MASK 0x1f000000L
+#define MC_RPB_PERF_COUNTER_CNTL__PERF_COUNTER_ASSIGN_3__SHIFT 0x00000018
+#define MC_RPB_PERF_COUNTER_CNTL__PERF_COUNTER_SELECT_MASK 0x00000003L
+#define MC_RPB_PERF_COUNTER_CNTL__PERF_COUNTER_SELECT__SHIFT 0x00000000
+#define MC_RPB_PERF_COUNTER_CNTL__STOP_ON_COUNTER_SATURATION_MASK 0x00000010L
+#define MC_RPB_PERF_COUNTER_CNTL__STOP_ON_COUNTER_SATURATION__SHIFT 0x00000004
+#define MC_RPB_PERF_COUNTER_STATUS__PERFORMANCE_COUNTER_VALUE_MASK 0xffffffffL
+#define MC_RPB_PERF_COUNTER_STATUS__PERFORMANCE_COUNTER_VALUE__SHIFT 0x00000000
+#define MC_RPB_RD_SWITCH_CNTL__QUEUE0_SWITCH_NUM_MASK 0x000000ffL
+#define MC_RPB_RD_SWITCH_CNTL__QUEUE0_SWITCH_NUM__SHIFT 0x00000000
+#define MC_RPB_RD_SWITCH_CNTL__QUEUE1_SWITCH_NUM_MASK 0x0000ff00L
+#define MC_RPB_RD_SWITCH_CNTL__QUEUE1_SWITCH_NUM__SHIFT 0x00000008
+#define MC_RPB_RD_SWITCH_CNTL__QUEUE2_SWITCH_NUM_MASK 0x00ff0000L
+#define MC_RPB_RD_SWITCH_CNTL__QUEUE2_SWITCH_NUM__SHIFT 0x00000010
+#define MC_RPB_RD_SWITCH_CNTL__QUEUE3_SWITCH_NUM_MASK 0xff000000L
+#define MC_RPB_RD_SWITCH_CNTL__QUEUE3_SWITCH_NUM__SHIFT 0x00000018
+#define MC_RPB_WR_COMBINE_CNTL__WC_ALIGN_MASK 0x00000080L
+#define MC_RPB_WR_COMBINE_CNTL__WC_ALIGN__SHIFT 0x00000007
+#define MC_RPB_WR_COMBINE_CNTL__WC_ENABLE_MASK 0x00000001L
+#define MC_RPB_WR_COMBINE_CNTL__WC_ENABLE__SHIFT 0x00000000
+#define MC_RPB_WR_COMBINE_CNTL__WC_FLUSH_TIMER_MASK 0x00000078L
+#define MC_RPB_WR_COMBINE_CNTL__WC_FLUSH_TIMER__SHIFT 0x00000003
+#define MC_RPB_WR_COMBINE_CNTL__WC_MAX_PACKET_SIZE_MASK 0x00000006L
+#define MC_RPB_WR_COMBINE_CNTL__WC_MAX_PACKET_SIZE__SHIFT 0x00000001
+#define MC_RPB_WR_SWITCH_CNTL__QUEUE0_SWITCH_NUM_MASK 0x000000ffL
+#define MC_RPB_WR_SWITCH_CNTL__QUEUE0_SWITCH_NUM__SHIFT 0x00000000
+#define MC_RPB_WR_SWITCH_CNTL__QUEUE1_SWITCH_NUM_MASK 0x0000ff00L
+#define MC_RPB_WR_SWITCH_CNTL__QUEUE1_SWITCH_NUM__SHIFT 0x00000008
+#define MC_RPB_WR_SWITCH_CNTL__QUEUE2_SWITCH_NUM_MASK 0x00ff0000L
+#define MC_RPB_WR_SWITCH_CNTL__QUEUE2_SWITCH_NUM__SHIFT 0x00000010
+#define MC_RPB_WR_SWITCH_CNTL__QUEUE3_SWITCH_NUM_MASK 0xff000000L
+#define MC_RPB_WR_SWITCH_CNTL__QUEUE3_SWITCH_NUM__SHIFT 0x00000018
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT0_MASK 0x00000007L
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT0__SHIFT 0x00000000
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT1_MASK 0x00000038L
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT1__SHIFT 0x00000003
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT2_MASK 0x000001c0L
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT2__SHIFT 0x00000006
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT3_MASK 0x00000e00L
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT3__SHIFT 0x00000009
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT4_MASK 0x00007000L
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT4__SHIFT 0x0000000c
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT5_MASK 0x00038000L
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT5__SHIFT 0x0000000f
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT6_MASK 0x001c0000L
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT6__SHIFT 0x00000012
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT7_MASK 0x00e00000L
+#define MC_SEQ_BIT_REMAP_B0_D0__BIT7__SHIFT 0x00000015
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT0_MASK 0x00000007L
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT0__SHIFT 0x00000000
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT1_MASK 0x00000038L
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT1__SHIFT 0x00000003
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT2_MASK 0x000001c0L
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT2__SHIFT 0x00000006
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT3_MASK 0x00000e00L
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT3__SHIFT 0x00000009
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT4_MASK 0x00007000L
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT4__SHIFT 0x0000000c
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT5_MASK 0x00038000L
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT5__SHIFT 0x0000000f
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT6_MASK 0x001c0000L
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT6__SHIFT 0x00000012
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT7_MASK 0x00e00000L
+#define MC_SEQ_BIT_REMAP_B0_D1__BIT7__SHIFT 0x00000015
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT0_MASK 0x00000007L
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT0__SHIFT 0x00000000
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT1_MASK 0x00000038L
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT1__SHIFT 0x00000003
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT2_MASK 0x000001c0L
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT2__SHIFT 0x00000006
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT3_MASK 0x00000e00L
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT3__SHIFT 0x00000009
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT4_MASK 0x00007000L
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT4__SHIFT 0x0000000c
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT5_MASK 0x00038000L
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT5__SHIFT 0x0000000f
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT6_MASK 0x001c0000L
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT6__SHIFT 0x00000012
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT7_MASK 0x00e00000L
+#define MC_SEQ_BIT_REMAP_B1_D0__BIT7__SHIFT 0x00000015
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT0_MASK 0x00000007L
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT0__SHIFT 0x00000000
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT1_MASK 0x00000038L
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT1__SHIFT 0x00000003
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT2_MASK 0x000001c0L
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT2__SHIFT 0x00000006
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT3_MASK 0x00000e00L
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT3__SHIFT 0x00000009
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT4_MASK 0x00007000L
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT4__SHIFT 0x0000000c
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT5_MASK 0x00038000L
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT5__SHIFT 0x0000000f
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT6_MASK 0x001c0000L
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT6__SHIFT 0x00000012
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT7_MASK 0x00e00000L
+#define MC_SEQ_BIT_REMAP_B1_D1__BIT7__SHIFT 0x00000015
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT0_MASK 0x00000007L
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT0__SHIFT 0x00000000
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT1_MASK 0x00000038L
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT1__SHIFT 0x00000003
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT2_MASK 0x000001c0L
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT2__SHIFT 0x00000006
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT3_MASK 0x00000e00L
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT3__SHIFT 0x00000009
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT4_MASK 0x00007000L
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT4__SHIFT 0x0000000c
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT5_MASK 0x00038000L
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT5__SHIFT 0x0000000f
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT6_MASK 0x001c0000L
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT6__SHIFT 0x00000012
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT7_MASK 0x00e00000L
+#define MC_SEQ_BIT_REMAP_B2_D0__BIT7__SHIFT 0x00000015
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT0_MASK 0x00000007L
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT0__SHIFT 0x00000000
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT1_MASK 0x00000038L
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT1__SHIFT 0x00000003
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT2_MASK 0x000001c0L
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT2__SHIFT 0x00000006
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT3_MASK 0x00000e00L
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT3__SHIFT 0x00000009
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT4_MASK 0x00007000L
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT4__SHIFT 0x0000000c
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT5_MASK 0x00038000L
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT5__SHIFT 0x0000000f
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT6_MASK 0x001c0000L
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT6__SHIFT 0x00000012
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT7_MASK 0x00e00000L
+#define MC_SEQ_BIT_REMAP_B2_D1__BIT7__SHIFT 0x00000015
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT0_MASK 0x00000007L
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT0__SHIFT 0x00000000
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT1_MASK 0x00000038L
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT1__SHIFT 0x00000003
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT2_MASK 0x000001c0L
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT2__SHIFT 0x00000006
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT3_MASK 0x00000e00L
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT3__SHIFT 0x00000009
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT4_MASK 0x00007000L
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT4__SHIFT 0x0000000c
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT5_MASK 0x00038000L
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT5__SHIFT 0x0000000f
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT6_MASK 0x001c0000L
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT6__SHIFT 0x00000012
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT7_MASK 0x00e00000L
+#define MC_SEQ_BIT_REMAP_B3_D0__BIT7__SHIFT 0x00000015
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT0_MASK 0x00000007L
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT0__SHIFT 0x00000000
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT1_MASK 0x00000038L
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT1__SHIFT 0x00000003
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT2_MASK 0x000001c0L
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT2__SHIFT 0x00000006
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT3_MASK 0x00000e00L
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT3__SHIFT 0x00000009
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT4_MASK 0x00007000L
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT4__SHIFT 0x0000000c
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT5_MASK 0x00038000L
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT5__SHIFT 0x0000000f
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT6_MASK 0x001c0000L
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT6__SHIFT 0x00000012
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT7_MASK 0x00e00000L
+#define MC_SEQ_BIT_REMAP_B3_D1__BIT7__SHIFT 0x00000015
+#define MC_SEQ_BYTE_REMAP_D0__BYTE0_MASK 0x00000003L
+#define MC_SEQ_BYTE_REMAP_D0__BYTE0__SHIFT 0x00000000
+#define MC_SEQ_BYTE_REMAP_D0__BYTE1_MASK 0x0000000cL
+#define MC_SEQ_BYTE_REMAP_D0__BYTE1__SHIFT 0x00000002
+#define MC_SEQ_BYTE_REMAP_D0__BYTE2_MASK 0x00000030L
+#define MC_SEQ_BYTE_REMAP_D0__BYTE2__SHIFT 0x00000004
+#define MC_SEQ_BYTE_REMAP_D0__BYTE3_MASK 0x000000c0L
+#define MC_SEQ_BYTE_REMAP_D0__BYTE3__SHIFT 0x00000006
+#define MC_SEQ_BYTE_REMAP_D1__BYTE0_MASK 0x00000003L
+#define MC_SEQ_BYTE_REMAP_D1__BYTE0__SHIFT 0x00000000
+#define MC_SEQ_BYTE_REMAP_D1__BYTE1_MASK 0x0000000cL
+#define MC_SEQ_BYTE_REMAP_D1__BYTE1__SHIFT 0x00000002
+#define MC_SEQ_BYTE_REMAP_D1__BYTE2_MASK 0x00000030L
+#define MC_SEQ_BYTE_REMAP_D1__BYTE2__SHIFT 0x00000004
+#define MC_SEQ_BYTE_REMAP_D1__BYTE3_MASK 0x000000c0L
+#define MC_SEQ_BYTE_REMAP_D1__BYTE3__SHIFT 0x00000006
+#define MC_SEQ_CAS_TIMING_LP__TCCDL_MASK 0x00000e00L
+#define MC_SEQ_CAS_TIMING_LP__TCCDL__SHIFT 0x00000009
+#define MC_SEQ_CAS_TIMING_LP__TCL_MASK 0x1f000000L
+#define MC_SEQ_CAS_TIMING_LP__TCL__SHIFT 0x00000018
+#define MC_SEQ_CAS_TIMING_LP__TNOPR_MASK 0x0000000cL
+#define MC_SEQ_CAS_TIMING_LP__TNOPR__SHIFT 0x00000002
+#define MC_SEQ_CAS_TIMING_LP__TNOPW_MASK 0x00000003L
+#define MC_SEQ_CAS_TIMING_LP__TNOPW__SHIFT 0x00000000
+#define MC_SEQ_CAS_TIMING_LP__TR2R_MASK 0x0000f000L
+#define MC_SEQ_CAS_TIMING_LP__TR2R__SHIFT 0x0000000c
+#define MC_SEQ_CAS_TIMING_LP__TR2W_MASK 0x000001f0L
+#define MC_SEQ_CAS_TIMING_LP__TR2W__SHIFT 0x00000004
+#define MC_SEQ_CAS_TIMING_LP__TW2R_MASK 0x001f0000L
+#define MC_SEQ_CAS_TIMING_LP__TW2R__SHIFT 0x00000010
+#define MC_SEQ_CAS_TIMING__TCCDL_MASK 0x00000e00L
+#define MC_SEQ_CAS_TIMING__TCCDL__SHIFT 0x00000009
+#define MC_SEQ_CAS_TIMING__TCL_MASK 0x1f000000L
+#define MC_SEQ_CAS_TIMING__TCL__SHIFT 0x00000018
+#define MC_SEQ_CAS_TIMING__TNOPR_MASK 0x0000000cL
+#define MC_SEQ_CAS_TIMING__TNOPR__SHIFT 0x00000002
+#define MC_SEQ_CAS_TIMING__TNOPW_MASK 0x00000003L
+#define MC_SEQ_CAS_TIMING__TNOPW__SHIFT 0x00000000
+#define MC_SEQ_CAS_TIMING__TR2R_MASK 0x0000f000L
+#define MC_SEQ_CAS_TIMING__TR2R__SHIFT 0x0000000c
+#define MC_SEQ_CAS_TIMING__TR2W_MASK 0x000001f0L
+#define MC_SEQ_CAS_TIMING__TR2W__SHIFT 0x00000004
+#define MC_SEQ_CAS_TIMING__TW2R_MASK 0x001f0000L
+#define MC_SEQ_CAS_TIMING__TW2R__SHIFT 0x00000010
+#define MC_SEQ_CG__CG_SEQ_REQ_MASK 0x000000ffL
+#define MC_SEQ_CG__CG_SEQ_REQ__SHIFT 0x00000000
+#define MC_SEQ_CG__CG_SEQ_RESP_MASK 0x0000ff00L
+#define MC_SEQ_CG__CG_SEQ_RESP__SHIFT 0x00000008
+#define MC_SEQ_CG__SEQ_CG_REQ_MASK 0x00ff0000L
+#define MC_SEQ_CG__SEQ_CG_REQ__SHIFT 0x00000010
+#define MC_SEQ_CG__SEQ_CG_RESP_MASK 0xff000000L
+#define MC_SEQ_CG__SEQ_CG_RESP__SHIFT 0x00000018
+#define MC_SEQ_CMD__ADR_MASK 0x0000ffffL
+#define MC_SEQ_CMD__ADR_MSB0_MASK 0x20000000L
+#define MC_SEQ_CMD__ADR_MSB0__SHIFT 0x0000001d
+#define MC_SEQ_CMD__ADR_MSB1_MASK 0x10000000L
+#define MC_SEQ_CMD__ADR_MSB1__SHIFT 0x0000001c
+#define MC_SEQ_CMD__ADR__SHIFT 0x00000000
+#define MC_SEQ_CMD__CHAN0_MASK 0x01000000L
+#define MC_SEQ_CMD__CHAN0__SHIFT 0x00000018
+#define MC_SEQ_CMD__CHAN1_MASK 0x02000000L
+#define MC_SEQ_CMD__CHAN1__SHIFT 0x00000019
+#define MC_SEQ_CMD__CSB_MASK 0x00600000L
+#define MC_SEQ_CMD__CSB__SHIFT 0x00000015
+#define MC_SEQ_CMD__END_MASK 0x00100000L
+#define MC_SEQ_CMD__END__SHIFT 0x00000014
+#define MC_SEQ_CMD__MOP_MASK 0x000f0000L
+#define MC_SEQ_CMD__MOP__SHIFT 0x00000010
+#define MC_SEQ_CNTL_2__ARB_RTDAT_WMK_MSB_MASK 0x00000300L
+#define MC_SEQ_CNTL_2__ARB_RTDAT_WMK_MSB__SHIFT 0x00000008
+#define MC_SEQ_CNTL_2__DRST_NSTR_MASK 0x0000fc00L
+#define MC_SEQ_CNTL_2__DRST_NSTR__SHIFT 0x0000000a
+#define MC_SEQ_CNTL_2__DRST_PSTR_MASK 0x003f0000L
+#define MC_SEQ_CNTL_2__DRST_PSTR__SHIFT 0x00000010
+#define MC_SEQ_CNTL_2__PLL_RX_PWRON_D0_MASK 0x0f000000L
+#define MC_SEQ_CNTL_2__PLL_RX_PWRON_D0__SHIFT 0x00000018
+#define MC_SEQ_CNTL_2__PLL_RX_PWRON_D1_MASK 0xf0000000L
+#define MC_SEQ_CNTL_2__PLL_RX_PWRON_D1__SHIFT 0x0000001c
+#define MC_SEQ_CNTL_2__PLL_TX_PWRON_D0_MASK 0x00400000L
+#define MC_SEQ_CNTL_2__PLL_TX_PWRON_D0__SHIFT 0x00000016
+#define MC_SEQ_CNTL_2__PLL_TX_PWRON_D1_MASK 0x00800000L
+#define MC_SEQ_CNTL_2__PLL_TX_PWRON_D1__SHIFT 0x00000017
+#define MC_SEQ_CNTL__ARB_REQCMD_WMK_MASK 0x00f00000L
+#define MC_SEQ_CNTL__ARB_REQCMD_WMK__SHIFT 0x00000014
+#define MC_SEQ_CNTL__ARB_REQDAT_WMK_MASK 0x0f000000L
+#define MC_SEQ_CNTL__ARB_REQDAT_WMK__SHIFT 0x00000018
+#define MC_SEQ_CNTL__ARB_RTDAT_WMK_MASK 0xf0000000L
+#define MC_SEQ_CNTL__ARB_RTDAT_WMK__SHIFT 0x0000001c
+#define MC_SEQ_CNTL__BANKGROUP_ENB_MASK 0x00040000L
+#define MC_SEQ_CNTL__BANKGROUP_ENB__SHIFT 0x00000012
+#define MC_SEQ_CNTL__BANKGROUP_SIZE_MASK 0x00020000L
+#define MC_SEQ_CNTL__BANKGROUP_SIZE__SHIFT 0x00000011
+#define MC_SEQ_CNTL__CHANNEL_DISABLE_MASK 0x00000300L
+#define MC_SEQ_CNTL__CHANNEL_DISABLE__SHIFT 0x00000008
+#define MC_SEQ_CNTL__DAT_INV_MASK 0x00000040L
+#define MC_SEQ_CNTL__DAT_INV__SHIFT 0x00000006
+#define MC_SEQ_CNTL__MEM_ADDR_MAP_BANK_MASK 0x0000000cL
+#define MC_SEQ_CNTL__MEM_ADDR_MAP_BANK__SHIFT 0x00000002
+#define MC_SEQ_CNTL__MEM_ADDR_MAP_COLS_MASK 0x00000003L
+#define MC_SEQ_CNTL__MEM_ADDR_MAP_COLS__SHIFT 0x00000000
+#define MC_SEQ_CNTL__MSK_DF1_MASK 0x00000080L
+#define MC_SEQ_CNTL__MSK_DF1__SHIFT 0x00000007
+#define MC_SEQ_CNTL__MSKOFF_DAT_TH_MASK 0x00008000L
+#define MC_SEQ_CNTL__MSKOFF_DAT_TH__SHIFT 0x0000000f
+#define MC_SEQ_CNTL__MSKOFF_DAT_TL_MASK 0x00004000L
+#define MC_SEQ_CNTL__MSKOFF_DAT_TL__SHIFT 0x0000000e
+#define MC_SEQ_CNTL__RET_HOLD_EOP_MASK 0x00010000L
+#define MC_SEQ_CNTL__RET_HOLD_EOP__SHIFT 0x00000010
+#define MC_SEQ_CNTL__RTR_OVERRIDE_MASK 0x00080000L
+#define MC_SEQ_CNTL__RTR_OVERRIDE__SHIFT 0x00000013
+#define MC_SEQ_CNTL__SAFE_MODE_MASK 0x00000030L
+#define MC_SEQ_CNTL__SAFE_MODE__SHIFT 0x00000004
+#define MC_SEQ_DRAM_2__ADBI_ACT_MASK 0x04000000L
+#define MC_SEQ_DRAM_2__ADBI_ACT__SHIFT 0x0000001a
+#define MC_SEQ_DRAM_2__ADBI_DF1_MASK 0x02000000L
+#define MC_SEQ_DRAM_2__ADBI_DF1__SHIFT 0x00000019
+#define MC_SEQ_DRAM_2__ADR_DBI_ACM_MASK 0x00000004L
+#define MC_SEQ_DRAM_2__ADR_DBI_ACM__SHIFT 0x00000002
+#define MC_SEQ_DRAM_2__ADR_DBI_MASK 0x00000002L
+#define MC_SEQ_DRAM_2__ADR_DBI__SHIFT 0x00000001
+#define MC_SEQ_DRAM_2__ADR_DDR_MASK 0x00000001L
+#define MC_SEQ_DRAM_2__ADR_DDR__SHIFT 0x00000000
+#define MC_SEQ_DRAM_2__BNK_MRS_MASK 0x00002000L
+#define MC_SEQ_DRAM_2__BNK_MRS__SHIFT 0x0000000d
+#define MC_SEQ_DRAM_2__CMD_QDR_MASK 0x00000008L
+#define MC_SEQ_DRAM_2__CMD_QDR__SHIFT 0x00000003
+#define MC_SEQ_DRAM_2__CS_BY16_MASK 0x80000000L
+#define MC_SEQ_DRAM_2__CS_BY16__SHIFT 0x0000001f
+#define MC_SEQ_DRAM_2__DAT_QDR_MASK 0x00000010L
+#define MC_SEQ_DRAM_2__DAT_QDR__SHIFT 0x00000004
+#define MC_SEQ_DRAM_2__DBI_ACT_MASK 0x10000000L
+#define MC_SEQ_DRAM_2__DBI_ACT__SHIFT 0x0000001c
+#define MC_SEQ_DRAM_2__DBI_DF1_MASK 0x08000000L
+#define MC_SEQ_DRAM_2__DBI_DF1__SHIFT 0x0000001b
+#define MC_SEQ_DRAM_2__DBI_EDC_DF1_MASK 0x20000000L
+#define MC_SEQ_DRAM_2__DBI_EDC_DF1__SHIFT 0x0000001d
+#define MC_SEQ_DRAM_2__DBI_OVR_MASK 0x00004000L
+#define MC_SEQ_DRAM_2__DBI_OVR__SHIFT 0x0000000e
+#define MC_SEQ_DRAM_2__DLL_EST_MASK 0x00001000L
+#define MC_SEQ_DRAM_2__DLL_EST__SHIFT 0x0000000c
+#define MC_SEQ_DRAM_2__DQM_EST_MASK 0x00000080L
+#define MC_SEQ_DRAM_2__DQM_EST__SHIFT 0x00000007
+#define MC_SEQ_DRAM_2__PCH_BNK_MASK 0x01000000L
+#define MC_SEQ_DRAM_2__PCH_BNK__SHIFT 0x00000018
+#define MC_SEQ_DRAM_2__PLL_CLR_MASK 0x00000800L
+#define MC_SEQ_DRAM_2__PLL_CLR__SHIFT 0x0000000b
+#define MC_SEQ_DRAM_2__PLL_CNT_MASK 0x00ff0000L
+#define MC_SEQ_DRAM_2__PLL_CNT__SHIFT 0x00000010
+#define MC_SEQ_DRAM_2__PLL_EST_MASK 0x00000400L
+#define MC_SEQ_DRAM_2__PLL_EST__SHIFT 0x0000000a
+#define MC_SEQ_DRAM_2__RDAT_EDC_MASK 0x00000040L
+#define MC_SEQ_DRAM_2__RDAT_EDC__SHIFT 0x00000006
+#define MC_SEQ_DRAM_2__RD_DQS_MASK 0x00000100L
+#define MC_SEQ_DRAM_2__RD_DQS__SHIFT 0x00000008
+#define MC_SEQ_DRAM_2__TESTCHIP_EN_MASK 0x40000000L
+#define MC_SEQ_DRAM_2__TESTCHIP_EN__SHIFT 0x0000001e
+#define MC_SEQ_DRAM_2__TRI_CLK_MASK 0x00008000L
+#define MC_SEQ_DRAM_2__TRI_CLK__SHIFT 0x0000000f
+#define MC_SEQ_DRAM_2__WDAT_EDC_MASK 0x00000020L
+#define MC_SEQ_DRAM_2__WDAT_EDC__SHIFT 0x00000005
+#define MC_SEQ_DRAM_2__WR_DQS_MASK 0x00000200L
+#define MC_SEQ_DRAM_2__WR_DQS__SHIFT 0x00000009
+#define MC_SEQ_DRAM__ADR_2CK_MASK 0x00000001L
+#define MC_SEQ_DRAM__ADR_2CK__SHIFT 0x00000000
+#define MC_SEQ_DRAM__ADR_DF1_MASK 0x00000004L
+#define MC_SEQ_DRAM__ADR_DF1__SHIFT 0x00000002
+#define MC_SEQ_DRAM__ADR_MUX_MASK 0x00000002L
+#define MC_SEQ_DRAM__ADR_MUX__SHIFT 0x00000001
+#define MC_SEQ_DRAM__AP8_MASK 0x00000008L
+#define MC_SEQ_DRAM__AP8__SHIFT 0x00000003
+#define MC_SEQ_DRAM__BO4_MASK 0x00004000L
+#define MC_SEQ_DRAM__BO4__SHIFT 0x0000000e
+#define MC_SEQ_DRAM__CKE_ACT_MASK 0x00002000L
+#define MC_SEQ_DRAM__CKE_ACT__SHIFT 0x0000000d
+#define MC_SEQ_DRAM__CKE_DYN_MASK 0x00001000L
+#define MC_SEQ_DRAM__CKE_DYN__SHIFT 0x0000000c
+#define MC_SEQ_DRAM__DAT_DF1_MASK 0x00000010L
+#define MC_SEQ_DRAM__DAT_DF1__SHIFT 0x00000004
+#define MC_SEQ_DRAM__DAT_INV_MASK 0x01000000L
+#define MC_SEQ_DRAM__DAT_INV__SHIFT 0x00000018
+#define MC_SEQ_DRAM__DLL_CLR_MASK 0x00008000L
+#define MC_SEQ_DRAM__DLL_CLR__SHIFT 0x0000000f
+#define MC_SEQ_DRAM__DLL_CNT_MASK 0x00ff0000L
+#define MC_SEQ_DRAM__DLL_CNT__SHIFT 0x00000010
+#define MC_SEQ_DRAM__DQM_ACT_MASK 0x00000080L
+#define MC_SEQ_DRAM__DQM_ACT__SHIFT 0x00000007
+#define MC_SEQ_DRAM__DQM_DF1_MASK 0x00000040L
+#define MC_SEQ_DRAM__DQM_DF1__SHIFT 0x00000006
+#define MC_SEQ_DRAM__DQS_DF1_MASK 0x00000020L
+#define MC_SEQ_DRAM__DQS_DF1__SHIFT 0x00000005
+#define MC_SEQ_DRAM_ERROR_INSERTION__RX_MASK 0xffff0000L
+#define MC_SEQ_DRAM_ERROR_INSERTION__RX__SHIFT 0x00000010
+#define MC_SEQ_DRAM_ERROR_INSERTION__TX_MASK 0x0000ffffL
+#define MC_SEQ_DRAM_ERROR_INSERTION__TX__SHIFT 0x00000000
+#define MC_SEQ_DRAM__INV_ACM_MASK 0x02000000L
+#define MC_SEQ_DRAM__INV_ACM__SHIFT 0x00000019
+#define MC_SEQ_DRAM__ODT_ACT_MASK 0x08000000L
+#define MC_SEQ_DRAM__ODT_ACT__SHIFT 0x0000001b
+#define MC_SEQ_DRAM__ODT_ENB_MASK 0x04000000L
+#define MC_SEQ_DRAM__ODT_ENB__SHIFT 0x0000001a
+#define MC_SEQ_DRAM__RST_CTL_MASK 0x10000000L
+#define MC_SEQ_DRAM__RST_CTL__SHIFT 0x0000001c
+#define MC_SEQ_DRAM__STB_CNT_MASK 0x00000f00L
+#define MC_SEQ_DRAM__STB_CNT__SHIFT 0x00000008
+#define MC_SEQ_DRAM__TRI_CKE_MASK 0x40000000L
+#define MC_SEQ_DRAM__TRI_CKE__SHIFT 0x0000001e
+#define MC_SEQ_DRAM__TRI_MIO_DYN_MASK 0x20000000L
+#define MC_SEQ_DRAM__TRI_MIO_DYN__SHIFT 0x0000001d
+#define MC_SEQ_FIFO_CTL__CG_DIS_D0_MASK 0x00000100L
+#define MC_SEQ_FIFO_CTL__CG_DIS_D0__SHIFT 0x00000008
+#define MC_SEQ_FIFO_CTL__CG_DIS_D1_MASK 0x00000200L
+#define MC_SEQ_FIFO_CTL__CG_DIS_D1__SHIFT 0x00000009
+#define MC_SEQ_FIFO_CTL__R_LD_INIT_MASK 0x00000030L
+#define MC_SEQ_FIFO_CTL__R_LD_INIT__SHIFT 0x00000004
+#define MC_SEQ_FIFO_CTL__R_SYC_SEL_MASK 0x000000c0L
+#define MC_SEQ_FIFO_CTL__R_SYC_SEL__SHIFT 0x00000006
+#define MC_SEQ_FIFO_CTL__SYC_DLY_MASK 0x00007000L
+#define MC_SEQ_FIFO_CTL__SYC_DLY__SHIFT 0x0000000c
+#define MC_SEQ_FIFO_CTL__W_ASYC_EXT_MASK 0x00030000L
+#define MC_SEQ_FIFO_CTL__W_ASYC_EXT__SHIFT 0x00000010
+#define MC_SEQ_FIFO_CTL__W_DSYC_EXT_MASK 0x000c0000L
+#define MC_SEQ_FIFO_CTL__W_DSYC_EXT__SHIFT 0x00000012
+#define MC_SEQ_FIFO_CTL__W_LD_INIT_D0_MASK 0x00000003L
+#define MC_SEQ_FIFO_CTL__W_LD_INIT_D0__SHIFT 0x00000000
+#define MC_SEQ_FIFO_CTL__W_LD_INIT_D1_MASK 0x00000c00L
+#define MC_SEQ_FIFO_CTL__W_LD_INIT_D1__SHIFT 0x0000000a
+#define MC_SEQ_FIFO_CTL__W_SYC_SEL_MASK 0x0000000cL
+#define MC_SEQ_FIFO_CTL__W_SYC_SEL__SHIFT 0x00000002
+#define MC_SEQ_IO_DEBUG_DATA__IO_DEBUG_DATA_MASK 0xffffffffL
+#define MC_SEQ_IO_DEBUG_DATA__IO_DEBUG_DATA__SHIFT 0x00000000
+#define MC_SEQ_IO_DEBUG_INDEX__IO_DEBUG_INDEX_MASK 0x000001ffL
+#define MC_SEQ_IO_DEBUG_INDEX__IO_DEBUG_INDEX__SHIFT 0x00000000
+#define MC_SEQ_IO_RDBI__MASK_MASK 0xffffffffL
+#define MC_SEQ_IO_RDBI__MASK__SHIFT 0x00000000
+#define MC_SEQ_IO_REDC__EDC_MASK 0xffffffffL
+#define MC_SEQ_IO_REDC__EDC__SHIFT 0x00000000
+#define MC_SEQ_IO_RESERVE_D0__APHY_RSV_MASK 0xff000000L
+#define MC_SEQ_IO_RESERVE_D0__APHY_RSV__SHIFT 0x00000018
+#define MC_SEQ_IO_RESERVE_D0__DPHY0_RSV_MASK 0x00000fffL
+#define MC_SEQ_IO_RESERVE_D0__DPHY0_RSV__SHIFT 0x00000000
+#define MC_SEQ_IO_RESERVE_D0__DPHY1_RSV_MASK 0x00fff000L
+#define MC_SEQ_IO_RESERVE_D0__DPHY1_RSV__SHIFT 0x0000000c
+#define MC_SEQ_IO_RESERVE_D1__APHY_RSV_MASK 0xff000000L
+#define MC_SEQ_IO_RESERVE_D1__APHY_RSV__SHIFT 0x00000018
+#define MC_SEQ_IO_RESERVE_D1__DPHY0_RSV_MASK 0x00000fffL
+#define MC_SEQ_IO_RESERVE_D1__DPHY0_RSV__SHIFT 0x00000000
+#define MC_SEQ_IO_RESERVE_D1__DPHY1_RSV_MASK 0x00fff000L
+#define MC_SEQ_IO_RESERVE_D1__DPHY1_RSV__SHIFT 0x0000000c
+#define MC_SEQ_IO_RWORD0__RDATA_MASK 0xffffffffL
+#define MC_SEQ_IO_RWORD0__RDATA__SHIFT 0x00000000
+#define MC_SEQ_IO_RWORD1__RDATA_MASK 0xffffffffL
+#define MC_SEQ_IO_RWORD1__RDATA__SHIFT 0x00000000
+#define MC_SEQ_IO_RWORD2__RDATA_MASK 0xffffffffL
+#define MC_SEQ_IO_RWORD2__RDATA__SHIFT 0x00000000
+#define MC_SEQ_IO_RWORD3__RDATA_MASK 0xffffffffL
+#define MC_SEQ_IO_RWORD3__RDATA__SHIFT 0x00000000
+#define MC_SEQ_IO_RWORD4__RDATA_MASK 0xffffffffL
+#define MC_SEQ_IO_RWORD4__RDATA__SHIFT 0x00000000
+#define MC_SEQ_IO_RWORD5__RDATA_MASK 0xffffffffL
+#define MC_SEQ_IO_RWORD5__RDATA__SHIFT 0x00000000
+#define MC_SEQ_IO_RWORD6__RDATA_MASK 0xffffffffL
+#define MC_SEQ_IO_RWORD6__RDATA__SHIFT 0x00000000
+#define MC_SEQ_IO_RWORD7__RDATA_MASK 0xffffffffL
+#define MC_SEQ_IO_RWORD7__RDATA__SHIFT 0x00000000
+#define MC_SEQ_MISC0__VALUE_MASK 0xffffffffL
+#define MC_SEQ_MISC0__VALUE__SHIFT 0x00000000
+#define MC_SEQ_MISC1__VALUE_MASK 0xffffffffL
+#define MC_SEQ_MISC1__VALUE__SHIFT 0x00000000
+#define MC_SEQ_MISC3__VALUE_MASK 0xffffffffL
+#define MC_SEQ_MISC3__VALUE__SHIFT 0x00000000
+#define MC_SEQ_MISC4__VALUE_MASK 0xffffffffL
+#define MC_SEQ_MISC4__VALUE__SHIFT 0x00000000
+#define MC_SEQ_MISC5__VALUE_MASK 0xffffffffL
+#define MC_SEQ_MISC5__VALUE__SHIFT 0x00000000
+#define MC_SEQ_MISC6__VALUE_MASK 0xffffffffL
+#define MC_SEQ_MISC6__VALUE__SHIFT 0x00000000
+#define MC_SEQ_MISC7__VALUE_MASK 0xffffffffL
+#define MC_SEQ_MISC7__VALUE__SHIFT 0x00000000
+#define MC_SEQ_MISC8__VALUE_MASK 0xffffffffL
+#define MC_SEQ_MISC8__VALUE__SHIFT 0x00000000
+#define MC_SEQ_MISC9__VALUE_MASK 0xffffffffL
+#define MC_SEQ_MISC9__VALUE__SHIFT 0x00000000
+#define MC_SEQ_MISC_TIMING2__FAW_MASK 0x00001f00L
+#define MC_SEQ_MISC_TIMING2__FAW__SHIFT 0x00000008
+#define MC_SEQ_MISC_TIMING2_LP__FAW_MASK 0x00001f00L
+#define MC_SEQ_MISC_TIMING2_LP__FAW__SHIFT 0x00000008
+#define MC_SEQ_MISC_TIMING2_LP__PA2RDATA_MASK 0x00000007L
+#define MC_SEQ_MISC_TIMING2_LP__PA2RDATA__SHIFT 0x00000000
+#define MC_SEQ_MISC_TIMING2_LP__PA2WDATA_MASK 0x00000070L
+#define MC_SEQ_MISC_TIMING2_LP__PA2WDATA__SHIFT 0x00000004
+#define MC_SEQ_MISC_TIMING2_LP__TADR_MASK 0x00e00000L
+#define MC_SEQ_MISC_TIMING2_LP__TADR__SHIFT 0x00000015
+#define MC_SEQ_MISC_TIMING2_LP__TFCKTR_MASK 0x0f000000L
+#define MC_SEQ_MISC_TIMING2_LP__TFCKTR__SHIFT 0x00000018
+#define MC_SEQ_MISC_TIMING2_LP__TREDC_MASK 0x0000e000L
+#define MC_SEQ_MISC_TIMING2_LP__TREDC__SHIFT 0x0000000d
+#define MC_SEQ_MISC_TIMING2_LP__TWDATATR_MASK 0xf0000000L
+#define MC_SEQ_MISC_TIMING2_LP__TWDATATR__SHIFT 0x0000001c
+#define MC_SEQ_MISC_TIMING2_LP__TWEDC_MASK 0x001f0000L
+#define MC_SEQ_MISC_TIMING2_LP__TWEDC__SHIFT 0x00000010
+#define MC_SEQ_MISC_TIMING2__PA2RDATA_MASK 0x00000007L
+#define MC_SEQ_MISC_TIMING2__PA2RDATA__SHIFT 0x00000000
+#define MC_SEQ_MISC_TIMING2__PA2WDATA_MASK 0x00000070L
+#define MC_SEQ_MISC_TIMING2__PA2WDATA__SHIFT 0x00000004
+#define MC_SEQ_MISC_TIMING2__T32AW_MASK 0x01e00000L
+#define MC_SEQ_MISC_TIMING2__T32AW__SHIFT 0x00000015
+#define MC_SEQ_MISC_TIMING2__TREDC_MASK 0x0000e000L
+#define MC_SEQ_MISC_TIMING2__TREDC__SHIFT 0x0000000d
+#define MC_SEQ_MISC_TIMING2__TWDATATR_MASK 0xf0000000L
+#define MC_SEQ_MISC_TIMING2__TWDATATR__SHIFT 0x0000001c
+#define MC_SEQ_MISC_TIMING2__TWEDC_MASK 0x001f0000L
+#define MC_SEQ_MISC_TIMING2__TWEDC__SHIFT 0x00000010
+#define MC_SEQ_MISC_TIMING_LP__TRFC_MASK 0x1ff00000L
+#define MC_SEQ_MISC_TIMING_LP__TRFC__SHIFT 0x00000014
+#define MC_SEQ_MISC_TIMING_LP__TRP_MASK 0x000f8000L
+#define MC_SEQ_MISC_TIMING_LP__TRP_RDA_MASK 0x00003f00L
+#define MC_SEQ_MISC_TIMING_LP__TRP_RDA__SHIFT 0x00000008
+#define MC_SEQ_MISC_TIMING_LP__TRP__SHIFT 0x0000000f
+#define MC_SEQ_MISC_TIMING_LP__TRP_WRA_MASK 0x0000003fL
+#define MC_SEQ_MISC_TIMING_LP__TRP_WRA__SHIFT 0x00000000
+#define MC_SEQ_MISC_TIMING__TRFC_MASK 0x1ff00000L
+#define MC_SEQ_MISC_TIMING__TRFC__SHIFT 0x00000014
+#define MC_SEQ_MISC_TIMING__TRP_MASK 0x000f8000L
+#define MC_SEQ_MISC_TIMING__TRP_RDA_MASK 0x00003f00L
+#define MC_SEQ_MISC_TIMING__TRP_RDA__SHIFT 0x00000008
+#define MC_SEQ_MISC_TIMING__TRP__SHIFT 0x0000000f
+#define MC_SEQ_MISC_TIMING__TRP_WRA_MASK 0x0000003fL
+#define MC_SEQ_MISC_TIMING__TRP_WRA__SHIFT 0x00000000
+#define MC_SEQ_MPLL_OVERRIDE__AD_PLL_RESET_OVERRIDE_MASK 0x00000001L
+#define MC_SEQ_MPLL_OVERRIDE__AD_PLL_RESET_OVERRIDE__SHIFT 0x00000000
+#define MC_SEQ_MPLL_OVERRIDE__ATGM_CLK_SEL_OVERRIDE_MASK 0x00000020L
+#define MC_SEQ_MPLL_OVERRIDE__ATGM_CLK_SEL_OVERRIDE__SHIFT 0x00000005
+#define MC_SEQ_MPLL_OVERRIDE__DQ_0_0_PLL_RESET_OVERRIDE_MASK 0x00000002L
+#define MC_SEQ_MPLL_OVERRIDE__DQ_0_0_PLL_RESET_OVERRIDE__SHIFT 0x00000001
+#define MC_SEQ_MPLL_OVERRIDE__DQ_0_1_PLL_RESET_OVERRIDE_MASK 0x00000004L
+#define MC_SEQ_MPLL_OVERRIDE__DQ_0_1_PLL_RESET_OVERRIDE__SHIFT 0x00000002
+#define MC_SEQ_MPLL_OVERRIDE__DQ_1_0_PLL_RESET_OVERRIDE_MASK 0x00000008L
+#define MC_SEQ_MPLL_OVERRIDE__DQ_1_0_PLL_RESET_OVERRIDE__SHIFT 0x00000003
+#define MC_SEQ_MPLL_OVERRIDE__DQ_1_1_PLL_RESET_OVERRIDE_MASK 0x00000010L
+#define MC_SEQ_MPLL_OVERRIDE__DQ_1_1_PLL_RESET_OVERRIDE__SHIFT 0x00000004
+#define MC_SEQ_MPLL_OVERRIDE__TEST_BYPASS_CLK_EN_OVERRIDE_MASK 0x00000040L
+#define MC_SEQ_MPLL_OVERRIDE__TEST_BYPASS_CLK_EN_OVERRIDE__SHIFT 0x00000006
+#define MC_SEQ_MPLL_OVERRIDE__TEST_BYPASS_CLK_SEL_OVERRIDE_MASK 0x00000080L
+#define MC_SEQ_MPLL_OVERRIDE__TEST_BYPASS_CLK_SEL_OVERRIDE__SHIFT 0x00000007
+#define MC_SEQ_PERF_CNTL_1__PAUSE_MASK 0x00000001L
+#define MC_SEQ_PERF_CNTL_1__PAUSE__SHIFT 0x00000000
+#define MC_SEQ_PERF_CNTL_1__SEL_A_MSB_MASK 0x00000100L
+#define MC_SEQ_PERF_CNTL_1__SEL_A_MSB__SHIFT 0x00000008
+#define MC_SEQ_PERF_CNTL_1__SEL_B_MSB_MASK 0x00000200L
+#define MC_SEQ_PERF_CNTL_1__SEL_B_MSB__SHIFT 0x00000009
+#define MC_SEQ_PERF_CNTL_1__SEL_CH0_C_MSB_MASK 0x00000400L
+#define MC_SEQ_PERF_CNTL_1__SEL_CH0_C_MSB__SHIFT 0x0000000a
+#define MC_SEQ_PERF_CNTL_1__SEL_CH0_D_MSB_MASK 0x00000800L
+#define MC_SEQ_PERF_CNTL_1__SEL_CH0_D_MSB__SHIFT 0x0000000b
+#define MC_SEQ_PERF_CNTL_1__SEL_CH1_A_MSB_MASK 0x00001000L
+#define MC_SEQ_PERF_CNTL_1__SEL_CH1_A_MSB__SHIFT 0x0000000c
+#define MC_SEQ_PERF_CNTL_1__SEL_CH1_B_MSB_MASK 0x00002000L
+#define MC_SEQ_PERF_CNTL_1__SEL_CH1_B_MSB__SHIFT 0x0000000d
+#define MC_SEQ_PERF_CNTL_1__SEL_CH1_C_MSB_MASK 0x00004000L
+#define MC_SEQ_PERF_CNTL_1__SEL_CH1_C_MSB__SHIFT 0x0000000e
+#define MC_SEQ_PERF_CNTL_1__SEL_CH1_D_MSB_MASK 0x00008000L
+#define MC_SEQ_PERF_CNTL_1__SEL_CH1_D_MSB__SHIFT 0x0000000f
+#define MC_SEQ_PERF_CNTL__CNTL_MASK 0xc0000000L
+#define MC_SEQ_PERF_CNTL__CNTL__SHIFT 0x0000001e
+#define MC_SEQ_PERF_CNTL__MONITOR_PERIOD_MASK 0x3fffffffL
+#define MC_SEQ_PERF_CNTL__MONITOR_PERIOD__SHIFT 0x00000000
+#define MC_SEQ_PERF_SEQ_CNT_A_I0__VALUE_MASK 0xffffffffL
+#define MC_SEQ_PERF_SEQ_CNT_A_I0__VALUE__SHIFT 0x00000000
+#define MC_SEQ_PERF_SEQ_CNT_A_I1__VALUE_MASK 0xffffffffL
+#define MC_SEQ_PERF_SEQ_CNT_A_I1__VALUE__SHIFT 0x00000000
+#define MC_SEQ_PERF_SEQ_CNT_B_I0__VALUE_MASK 0xffffffffL
+#define MC_SEQ_PERF_SEQ_CNT_B_I0__VALUE__SHIFT 0x00000000
+#define MC_SEQ_PERF_SEQ_CNT_B_I1__VALUE_MASK 0xffffffffL
+#define MC_SEQ_PERF_SEQ_CNT_B_I1__VALUE__SHIFT 0x00000000
+#define MC_SEQ_PERF_SEQ_CNT_C_I0__VALUE_MASK 0xffffffffL
+#define MC_SEQ_PERF_SEQ_CNT_C_I0__VALUE__SHIFT 0x00000000
+#define MC_SEQ_PERF_SEQ_CNT_C_I1__VALUE_MASK 0xffffffffL
+#define MC_SEQ_PERF_SEQ_CNT_C_I1__VALUE__SHIFT 0x00000000
+#define MC_SEQ_PERF_SEQ_CNT_D_I0__VALUE_MASK 0xffffffffL
+#define MC_SEQ_PERF_SEQ_CNT_D_I0__VALUE__SHIFT 0x00000000
+#define MC_SEQ_PERF_SEQ_CNT_D_I1__VALUE_MASK 0xffffffffL
+#define MC_SEQ_PERF_SEQ_CNT_D_I1__VALUE__SHIFT 0x00000000
+#define MC_SEQ_PERF_SEQ_CTL__SEL_A_MASK 0x0000000fL
+#define MC_SEQ_PERF_SEQ_CTL__SEL_A__SHIFT 0x00000000
+#define MC_SEQ_PERF_SEQ_CTL__SEL_B_MASK 0x000000f0L
+#define MC_SEQ_PERF_SEQ_CTL__SEL_B__SHIFT 0x00000004
+#define MC_SEQ_PERF_SEQ_CTL__SEL_CH0_C_MASK 0x00000f00L
+#define MC_SEQ_PERF_SEQ_CTL__SEL_CH0_C__SHIFT 0x00000008
+#define MC_SEQ_PERF_SEQ_CTL__SEL_CH0_D_MASK 0x0000f000L
+#define MC_SEQ_PERF_SEQ_CTL__SEL_CH0_D__SHIFT 0x0000000c
+#define MC_SEQ_PERF_SEQ_CTL__SEL_CH1_A_MASK 0x000f0000L
+#define MC_SEQ_PERF_SEQ_CTL__SEL_CH1_A__SHIFT 0x00000010
+#define MC_SEQ_PERF_SEQ_CTL__SEL_CH1_B_MASK 0x00f00000L
+#define MC_SEQ_PERF_SEQ_CTL__SEL_CH1_B__SHIFT 0x00000014
+#define MC_SEQ_PERF_SEQ_CTL__SEL_CH1_C_MASK 0x0f000000L
+#define MC_SEQ_PERF_SEQ_CTL__SEL_CH1_C__SHIFT 0x00000018
+#define MC_SEQ_PERF_SEQ_CTL__SEL_CH1_D_MASK 0xf0000000L
+#define MC_SEQ_PERF_SEQ_CTL__SEL_CH1_D__SHIFT 0x0000001c
+#define MC_SEQ_PMG_CMD_EMRS_LP__ADR_MASK 0x0000ffffL
+#define MC_SEQ_PMG_CMD_EMRS_LP__ADR_MSB0_MASK 0x20000000L
+#define MC_SEQ_PMG_CMD_EMRS_LP__ADR_MSB0__SHIFT 0x0000001d
+#define MC_SEQ_PMG_CMD_EMRS_LP__ADR_MSB1_MASK 0x10000000L
+#define MC_SEQ_PMG_CMD_EMRS_LP__ADR_MSB1__SHIFT 0x0000001c
+#define MC_SEQ_PMG_CMD_EMRS_LP__ADR__SHIFT 0x00000000
+#define MC_SEQ_PMG_CMD_EMRS_LP__BNK_MSB_MASK 0x00080000L
+#define MC_SEQ_PMG_CMD_EMRS_LP__BNK_MSB__SHIFT 0x00000013
+#define MC_SEQ_PMG_CMD_EMRS_LP__CSB_MASK 0x00600000L
+#define MC_SEQ_PMG_CMD_EMRS_LP__CSB__SHIFT 0x00000015
+#define MC_SEQ_PMG_CMD_EMRS_LP__END_MASK 0x00100000L
+#define MC_SEQ_PMG_CMD_EMRS_LP__END__SHIFT 0x00000014
+#define MC_SEQ_PMG_CMD_EMRS_LP__MOP_MASK 0x00070000L
+#define MC_SEQ_PMG_CMD_EMRS_LP__MOP__SHIFT 0x00000010
+#define MC_SEQ_PMG_CMD_MRS1_LP__ADR_MASK 0x0000ffffL
+#define MC_SEQ_PMG_CMD_MRS1_LP__ADR_MSB0_MASK 0x20000000L
+#define MC_SEQ_PMG_CMD_MRS1_LP__ADR_MSB0__SHIFT 0x0000001d
+#define MC_SEQ_PMG_CMD_MRS1_LP__ADR_MSB1_MASK 0x10000000L
+#define MC_SEQ_PMG_CMD_MRS1_LP__ADR_MSB1__SHIFT 0x0000001c
+#define MC_SEQ_PMG_CMD_MRS1_LP__ADR__SHIFT 0x00000000
+#define MC_SEQ_PMG_CMD_MRS1_LP__BNK_MSB_MASK 0x00080000L
+#define MC_SEQ_PMG_CMD_MRS1_LP__BNK_MSB__SHIFT 0x00000013
+#define MC_SEQ_PMG_CMD_MRS1_LP__CSB_MASK 0x00600000L
+#define MC_SEQ_PMG_CMD_MRS1_LP__CSB__SHIFT 0x00000015
+#define MC_SEQ_PMG_CMD_MRS1_LP__END_MASK 0x00100000L
+#define MC_SEQ_PMG_CMD_MRS1_LP__END__SHIFT 0x00000014
+#define MC_SEQ_PMG_CMD_MRS1_LP__MOP_MASK 0x00070000L
+#define MC_SEQ_PMG_CMD_MRS1_LP__MOP__SHIFT 0x00000010
+#define MC_SEQ_PMG_CMD_MRS2_LP__ADR_MASK 0x0000ffffL
+#define MC_SEQ_PMG_CMD_MRS2_LP__ADR_MSB0_MASK 0x20000000L
+#define MC_SEQ_PMG_CMD_MRS2_LP__ADR_MSB0__SHIFT 0x0000001d
+#define MC_SEQ_PMG_CMD_MRS2_LP__ADR_MSB1_MASK 0x10000000L
+#define MC_SEQ_PMG_CMD_MRS2_LP__ADR_MSB1__SHIFT 0x0000001c
+#define MC_SEQ_PMG_CMD_MRS2_LP__ADR__SHIFT 0x00000000
+#define MC_SEQ_PMG_CMD_MRS2_LP__BNK_MSB_MASK 0x00080000L
+#define MC_SEQ_PMG_CMD_MRS2_LP__BNK_MSB__SHIFT 0x00000013
+#define MC_SEQ_PMG_CMD_MRS2_LP__CSB_MASK 0x00600000L
+#define MC_SEQ_PMG_CMD_MRS2_LP__CSB__SHIFT 0x00000015
+#define MC_SEQ_PMG_CMD_MRS2_LP__END_MASK 0x00100000L
+#define MC_SEQ_PMG_CMD_MRS2_LP__END__SHIFT 0x00000014
+#define MC_SEQ_PMG_CMD_MRS2_LP__MOP_MASK 0x00070000L
+#define MC_SEQ_PMG_CMD_MRS2_LP__MOP__SHIFT 0x00000010
+#define MC_SEQ_PMG_CMD_MRS_LP__ADR_MASK 0x0000ffffL
+#define MC_SEQ_PMG_CMD_MRS_LP__ADR_MSB0_MASK 0x20000000L
+#define MC_SEQ_PMG_CMD_MRS_LP__ADR_MSB0__SHIFT 0x0000001d
+#define MC_SEQ_PMG_CMD_MRS_LP__ADR_MSB1_MASK 0x10000000L
+#define MC_SEQ_PMG_CMD_MRS_LP__ADR_MSB1__SHIFT 0x0000001c
+#define MC_SEQ_PMG_CMD_MRS_LP__ADR__SHIFT 0x00000000
+#define MC_SEQ_PMG_CMD_MRS_LP__BNK_MSB_MASK 0x00080000L
+#define MC_SEQ_PMG_CMD_MRS_LP__BNK_MSB__SHIFT 0x00000013
+#define MC_SEQ_PMG_CMD_MRS_LP__CSB_MASK 0x00600000L
+#define MC_SEQ_PMG_CMD_MRS_LP__CSB__SHIFT 0x00000015
+#define MC_SEQ_PMG_CMD_MRS_LP__END_MASK 0x00100000L
+#define MC_SEQ_PMG_CMD_MRS_LP__END__SHIFT 0x00000014
+#define MC_SEQ_PMG_CMD_MRS_LP__MOP_MASK 0x00070000L
+#define MC_SEQ_PMG_CMD_MRS_LP__MOP__SHIFT 0x00000010
+#define MC_SEQ_PMG_PG_HWCNTL__ACAO_MASK 0x00040000L
+#define MC_SEQ_PMG_PG_HWCNTL__ACAO__SHIFT 0x00000012
+#define MC_SEQ_PMG_PG_HWCNTL__AC_DLY_MASK 0x00000300L
+#define MC_SEQ_PMG_PG_HWCNTL__AC_DLY__SHIFT 0x00000008
+#define MC_SEQ_PMG_PG_HWCNTL__D_DLY_MASK 0x000000c0L
+#define MC_SEQ_PMG_PG_HWCNTL__D_DLY__SHIFT 0x00000006
+#define MC_SEQ_PMG_PG_HWCNTL__G_DLY_MASK 0x00003c00L
+#define MC_SEQ_PMG_PG_HWCNTL__G_DLY__SHIFT 0x0000000a
+#define MC_SEQ_PMG_PG_HWCNTL__PWRGATE_EN_MASK 0x00000001L
+#define MC_SEQ_PMG_PG_HWCNTL__PWRGATE_EN__SHIFT 0x00000000
+#define MC_SEQ_PMG_PG_HWCNTL__RXAO_MASK 0x00020000L
+#define MC_SEQ_PMG_PG_HWCNTL__RXAO__SHIFT 0x00000011
+#define MC_SEQ_PMG_PG_HWCNTL__STAGGER_EN_MASK 0x00000002L
+#define MC_SEQ_PMG_PG_HWCNTL__STAGGER_EN__SHIFT 0x00000001
+#define MC_SEQ_PMG_PG_HWCNTL__TPGCG_MASK 0x0000003cL
+#define MC_SEQ_PMG_PG_HWCNTL__TPGCG__SHIFT 0x00000002
+#define MC_SEQ_PMG_PG_HWCNTL__TXAO_MASK 0x00010000L
+#define MC_SEQ_PMG_PG_HWCNTL__TXAO__SHIFT 0x00000010
+#define MC_SEQ_PMG_PG_SWCNTL_0__GMCON_SR_COMMIT_MASK 0x80000000L
+#define MC_SEQ_PMG_PG_SWCNTL_0__GMCON_SR_COMMIT__SHIFT 0x0000001f
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMA0_AC_ENB_MASK 0x00010000L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMA0_AC_ENB__SHIFT 0x00000010
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_DBI_RX_ENB_MASK 0x00000020L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_DBI_RX_ENB__SHIFT 0x00000005
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_DBI_TX_ENB_MASK 0x00000002L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_DBI_TX_ENB__SHIFT 0x00000001
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_DQ_RX_ENB_MASK 0x00000010L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_DQ_RX_ENB__SHIFT 0x00000004
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_DQ_TX_ENB_MASK 0x00000001L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_DQ_TX_ENB__SHIFT 0x00000000
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_EDC_RX_ENB_MASK 0x00000040L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_EDC_RX_ENB__SHIFT 0x00000006
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_EDC_TX_ENB_MASK 0x00000004L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_EDC_TX_ENB__SHIFT 0x00000002
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_WCLKX_RX_ENB_MASK 0x00000080L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_WCLKX_RX_ENB__SHIFT 0x00000007
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_WCLKX_TX_ENB_MASK 0x00000008L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD0_WCLKX_TX_ENB__SHIFT 0x00000003
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_DBI_RX_ENB_MASK 0x00002000L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_DBI_RX_ENB__SHIFT 0x0000000d
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_DBI_TX_ENB_MASK 0x00000200L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_DBI_TX_ENB__SHIFT 0x00000009
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_DQ_RX_ENB_MASK 0x00001000L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_DQ_RX_ENB__SHIFT 0x0000000c
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_DQ_TX_ENB_MASK 0x00000100L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_DQ_TX_ENB__SHIFT 0x00000008
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_EDC_RX_ENB_MASK 0x00004000L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_EDC_RX_ENB__SHIFT 0x0000000e
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_EDC_TX_ENB_MASK 0x00000400L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_EDC_TX_ENB__SHIFT 0x0000000a
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_WCLKX_RX_ENB_MASK 0x00008000L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_WCLKX_RX_ENB__SHIFT 0x0000000f
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_WCLKX_TX_ENB_MASK 0x00000800L
+#define MC_SEQ_PMG_PG_SWCNTL_0__PMD1_WCLKX_TX_ENB__SHIFT 0x0000000b
+#define MC_SEQ_PMG_PG_SWCNTL_1__GMCON_SR_COMMIT_MASK 0x80000000L
+#define MC_SEQ_PMG_PG_SWCNTL_1__GMCON_SR_COMMIT__SHIFT 0x0000001f
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMA1_AC_ENB_MASK 0x00010000L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMA1_AC_ENB__SHIFT 0x00000010
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_DBI_RX_ENB_MASK 0x00000020L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_DBI_RX_ENB__SHIFT 0x00000005
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_DBI_TX_ENB_MASK 0x00000002L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_DBI_TX_ENB__SHIFT 0x00000001
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_DQ_RX_ENB_MASK 0x00000010L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_DQ_RX_ENB__SHIFT 0x00000004
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_DQ_TX_ENB_MASK 0x00000001L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_DQ_TX_ENB__SHIFT 0x00000000
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_EDC_RX_ENB_MASK 0x00000040L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_EDC_RX_ENB__SHIFT 0x00000006
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_EDC_TX_ENB_MASK 0x00000004L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_EDC_TX_ENB__SHIFT 0x00000002
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_WCLKX_RX_ENB_MASK 0x00000080L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_WCLKX_RX_ENB__SHIFT 0x00000007
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_WCLKX_TX_ENB_MASK 0x00000008L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD2_WCLKX_TX_ENB__SHIFT 0x00000003
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_DBI_RX_ENB_MASK 0x00002000L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_DBI_RX_ENB__SHIFT 0x0000000d
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_DBI_TX_ENB_MASK 0x00000200L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_DBI_TX_ENB__SHIFT 0x00000009
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_DQ_RX_ENB_MASK 0x00001000L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_DQ_RX_ENB__SHIFT 0x0000000c
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_DQ_TX_ENB_MASK 0x00000100L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_DQ_TX_ENB__SHIFT 0x00000008
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_EDC_RX_ENB_MASK 0x00004000L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_EDC_RX_ENB__SHIFT 0x0000000e
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_EDC_TX_ENB_MASK 0x00000400L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_EDC_TX_ENB__SHIFT 0x0000000a
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_WCLKX_RX_ENB_MASK 0x00008000L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_WCLKX_RX_ENB__SHIFT 0x0000000f
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_WCLKX_TX_ENB_MASK 0x00000800L
+#define MC_SEQ_PMG_PG_SWCNTL_1__PMD3_WCLKX_TX_ENB__SHIFT 0x0000000b
+#define MC_SEQ_PMG_TIMING_LP__SEQ_IDLE_MASK 0x001c0000L
+#define MC_SEQ_PMG_TIMING_LP__SEQ_IDLE__SHIFT 0x00000012
+#define MC_SEQ_PMG_TIMING_LP__SEQ_IDLE_SS_MASK 0xff000000L
+#define MC_SEQ_PMG_TIMING_LP__SEQ_IDLE_SS__SHIFT 0x00000018
+#define MC_SEQ_PMG_TIMING_LP__TCKE_MASK 0x0003f000L
+#define MC_SEQ_PMG_TIMING_LP__TCKE_PULSE_MASK 0x00000f00L
+#define MC_SEQ_PMG_TIMING_LP__TCKE_PULSE_MSB_MASK 0x00800000L
+#define MC_SEQ_PMG_TIMING_LP__TCKE_PULSE_MSB__SHIFT 0x00000017
+#define MC_SEQ_PMG_TIMING_LP__TCKE_PULSE__SHIFT 0x00000008
+#define MC_SEQ_PMG_TIMING_LP__TCKE__SHIFT 0x0000000c
+#define MC_SEQ_PMG_TIMING_LP__TCKSRE_MASK 0x00000007L
+#define MC_SEQ_PMG_TIMING_LP__TCKSRE__SHIFT 0x00000000
+#define MC_SEQ_PMG_TIMING_LP__TCKSRX_MASK 0x00000070L
+#define MC_SEQ_PMG_TIMING_LP__TCKSRX__SHIFT 0x00000004
+#define MC_SEQ_PMG_TIMING__SEQ_IDLE_MASK 0x001c0000L
+#define MC_SEQ_PMG_TIMING__SEQ_IDLE__SHIFT 0x00000012
+#define MC_SEQ_PMG_TIMING__SEQ_IDLE_SS_MASK 0xff000000L
+#define MC_SEQ_PMG_TIMING__SEQ_IDLE_SS__SHIFT 0x00000018
+#define MC_SEQ_PMG_TIMING__TCKE_MASK 0x0003f000L
+#define MC_SEQ_PMG_TIMING__TCKE_PULSE_MASK 0x00000f00L
+#define MC_SEQ_PMG_TIMING__TCKE_PULSE_MSB_MASK 0x00800000L
+#define MC_SEQ_PMG_TIMING__TCKE_PULSE_MSB__SHIFT 0x00000017
+#define MC_SEQ_PMG_TIMING__TCKE_PULSE__SHIFT 0x00000008
+#define MC_SEQ_PMG_TIMING__TCKE__SHIFT 0x0000000c
+#define MC_SEQ_PMG_TIMING__TCKSRE_MASK 0x00000007L
+#define MC_SEQ_PMG_TIMING__TCKSRE__SHIFT 0x00000000
+#define MC_SEQ_PMG_TIMING__TCKSRX_MASK 0x00000070L
+#define MC_SEQ_PMG_TIMING__TCKSRX__SHIFT 0x00000004
+#define MC_SEQ_RAS_TIMING_LP__TRCDRA_MASK 0x000f8000L
+#define MC_SEQ_RAS_TIMING_LP__TRCDRA__SHIFT 0x0000000f
+#define MC_SEQ_RAS_TIMING_LP__TRCDR_MASK 0x00007c00L
+#define MC_SEQ_RAS_TIMING_LP__TRCDR__SHIFT 0x0000000a
+#define MC_SEQ_RAS_TIMING_LP__TRCDWA_MASK 0x000003e0L
+#define MC_SEQ_RAS_TIMING_LP__TRCDWA__SHIFT 0x00000005
+#define MC_SEQ_RAS_TIMING_LP__TRCDW_MASK 0x0000001fL
+#define MC_SEQ_RAS_TIMING_LP__TRCDW__SHIFT 0x00000000
+#define MC_SEQ_RAS_TIMING_LP__TRC_MASK 0x7f000000L
+#define MC_SEQ_RAS_TIMING_LP__TRC__SHIFT 0x00000018
+#define MC_SEQ_RAS_TIMING_LP__TRRD_MASK 0x00f00000L
+#define MC_SEQ_RAS_TIMING_LP__TRRD__SHIFT 0x00000014
+#define MC_SEQ_RAS_TIMING__TRCDRA_MASK 0x000f8000L
+#define MC_SEQ_RAS_TIMING__TRCDRA__SHIFT 0x0000000f
+#define MC_SEQ_RAS_TIMING__TRCDR_MASK 0x00007c00L
+#define MC_SEQ_RAS_TIMING__TRCDR__SHIFT 0x0000000a
+#define MC_SEQ_RAS_TIMING__TRCDWA_MASK 0x000003e0L
+#define MC_SEQ_RAS_TIMING__TRCDWA__SHIFT 0x00000005
+#define MC_SEQ_RAS_TIMING__TRCDW_MASK 0x0000001fL
+#define MC_SEQ_RAS_TIMING__TRCDW__SHIFT 0x00000000
+#define MC_SEQ_RAS_TIMING__TRC_MASK 0x7f000000L
+#define MC_SEQ_RAS_TIMING__TRC__SHIFT 0x00000018
+#define MC_SEQ_RAS_TIMING__TRRD_MASK 0x00f00000L
+#define MC_SEQ_RAS_TIMING__TRRD__SHIFT 0x00000014
+#define MC_SEQ_RD_CTL_D0_LP__RBS_DLY_MASK 0x01f00000L
+#define MC_SEQ_RD_CTL_D0_LP__RBS_DLY__SHIFT 0x00000014
+#define MC_SEQ_RD_CTL_D0_LP__RBS_WEDC_DLY_MASK 0x3e000000L
+#define MC_SEQ_RD_CTL_D0_LP__RBS_WEDC_DLY__SHIFT 0x00000019
+#define MC_SEQ_RD_CTL_D0_LP__RCV_DLY_MASK 0x00000007L
+#define MC_SEQ_RD_CTL_D0_LP__RCV_DLY__SHIFT 0x00000000
+#define MC_SEQ_RD_CTL_D0_LP__RCV_EXT_MASK 0x000000f8L
+#define MC_SEQ_RD_CTL_D0_LP__RCV_EXT__SHIFT 0x00000003
+#define MC_SEQ_RD_CTL_D0_LP__RST_HLD_MASK 0x0000f000L
+#define MC_SEQ_RD_CTL_D0_LP__RST_HLD__SHIFT 0x0000000c
+#define MC_SEQ_RD_CTL_D0_LP__RST_SEL_MASK 0x00000300L
+#define MC_SEQ_RD_CTL_D0_LP__RST_SEL__SHIFT 0x00000008
+#define MC_SEQ_RD_CTL_D0_LP__RXDPWRON_DLY_MASK 0x00000c00L
+#define MC_SEQ_RD_CTL_D0_LP__RXDPWRON_DLY__SHIFT 0x0000000a
+#define MC_SEQ_RD_CTL_D0_LP__STR_PRE_MASK 0x00010000L
+#define MC_SEQ_RD_CTL_D0_LP__STR_PRE__SHIFT 0x00000010
+#define MC_SEQ_RD_CTL_D0_LP__STR_PST_MASK 0x00020000L
+#define MC_SEQ_RD_CTL_D0_LP__STR_PST__SHIFT 0x00000011
+#define MC_SEQ_RD_CTL_D0__RBS_DLY_MASK 0x01f00000L
+#define MC_SEQ_RD_CTL_D0__RBS_DLY__SHIFT 0x00000014
+#define MC_SEQ_RD_CTL_D0__RBS_WEDC_DLY_MASK 0x3e000000L
+#define MC_SEQ_RD_CTL_D0__RBS_WEDC_DLY__SHIFT 0x00000019
+#define MC_SEQ_RD_CTL_D0__RCV_DLY_MASK 0x00000007L
+#define MC_SEQ_RD_CTL_D0__RCV_DLY__SHIFT 0x00000000
+#define MC_SEQ_RD_CTL_D0__RCV_EXT_MASK 0x000000f8L
+#define MC_SEQ_RD_CTL_D0__RCV_EXT__SHIFT 0x00000003
+#define MC_SEQ_RD_CTL_D0__RST_HLD_MASK 0x0000f000L
+#define MC_SEQ_RD_CTL_D0__RST_HLD__SHIFT 0x0000000c
+#define MC_SEQ_RD_CTL_D0__RST_SEL_MASK 0x00000300L
+#define MC_SEQ_RD_CTL_D0__RST_SEL__SHIFT 0x00000008
+#define MC_SEQ_RD_CTL_D0__RXDPWRON_DLY_MASK 0x00000c00L
+#define MC_SEQ_RD_CTL_D0__RXDPWRON_DLY__SHIFT 0x0000000a
+#define MC_SEQ_RD_CTL_D0__STR_PRE_MASK 0x00010000L
+#define MC_SEQ_RD_CTL_D0__STR_PRE__SHIFT 0x00000010
+#define MC_SEQ_RD_CTL_D0__STR_PST_MASK 0x00020000L
+#define MC_SEQ_RD_CTL_D0__STR_PST__SHIFT 0x00000011
+#define MC_SEQ_RD_CTL_D1_LP__RBS_DLY_MASK 0x01f00000L
+#define MC_SEQ_RD_CTL_D1_LP__RBS_DLY__SHIFT 0x00000014
+#define MC_SEQ_RD_CTL_D1_LP__RBS_WEDC_DLY_MASK 0x3e000000L
+#define MC_SEQ_RD_CTL_D1_LP__RBS_WEDC_DLY__SHIFT 0x00000019
+#define MC_SEQ_RD_CTL_D1_LP__RCV_DLY_MASK 0x00000007L
+#define MC_SEQ_RD_CTL_D1_LP__RCV_DLY__SHIFT 0x00000000
+#define MC_SEQ_RD_CTL_D1_LP__RCV_EXT_MASK 0x000000f8L
+#define MC_SEQ_RD_CTL_D1_LP__RCV_EXT__SHIFT 0x00000003
+#define MC_SEQ_RD_CTL_D1_LP__RST_HLD_MASK 0x0000f000L
+#define MC_SEQ_RD_CTL_D1_LP__RST_HLD__SHIFT 0x0000000c
+#define MC_SEQ_RD_CTL_D1_LP__RST_SEL_MASK 0x00000300L
+#define MC_SEQ_RD_CTL_D1_LP__RST_SEL__SHIFT 0x00000008
+#define MC_SEQ_RD_CTL_D1_LP__RXDPWRON_DLY_MASK 0x00000c00L
+#define MC_SEQ_RD_CTL_D1_LP__RXDPWRON_DLY__SHIFT 0x0000000a
+#define MC_SEQ_RD_CTL_D1_LP__STR_PRE_MASK 0x00010000L
+#define MC_SEQ_RD_CTL_D1_LP__STR_PRE__SHIFT 0x00000010
+#define MC_SEQ_RD_CTL_D1_LP__STR_PST_MASK 0x00020000L
+#define MC_SEQ_RD_CTL_D1_LP__STR_PST__SHIFT 0x00000011
+#define MC_SEQ_RD_CTL_D1__RBS_DLY_MASK 0x01f00000L
+#define MC_SEQ_RD_CTL_D1__RBS_DLY__SHIFT 0x00000014
+#define MC_SEQ_RD_CTL_D1__RBS_WEDC_DLY_MASK 0x3e000000L
+#define MC_SEQ_RD_CTL_D1__RBS_WEDC_DLY__SHIFT 0x00000019
+#define MC_SEQ_RD_CTL_D1__RCV_DLY_MASK 0x00000007L
+#define MC_SEQ_RD_CTL_D1__RCV_DLY__SHIFT 0x00000000
+#define MC_SEQ_RD_CTL_D1__RCV_EXT_MASK 0x000000f8L
+#define MC_SEQ_RD_CTL_D1__RCV_EXT__SHIFT 0x00000003
+#define MC_SEQ_RD_CTL_D1__RST_HLD_MASK 0x0000f000L
+#define MC_SEQ_RD_CTL_D1__RST_HLD__SHIFT 0x0000000c
+#define MC_SEQ_RD_CTL_D1__RST_SEL_MASK 0x00000300L
+#define MC_SEQ_RD_CTL_D1__RST_SEL__SHIFT 0x00000008
+#define MC_SEQ_RD_CTL_D1__RXDPWRON_DLY_MASK 0x00000c00L
+#define MC_SEQ_RD_CTL_D1__RXDPWRON_DLY__SHIFT 0x0000000a
+#define MC_SEQ_RD_CTL_D1__STR_PRE_MASK 0x00010000L
+#define MC_SEQ_RD_CTL_D1__STR_PRE__SHIFT 0x00000010
+#define MC_SEQ_RD_CTL_D1__STR_PST_MASK 0x00020000L
+#define MC_SEQ_RD_CTL_D1__STR_PST__SHIFT 0x00000011
+#define MC_SEQ_RESERVE_0_S__SCLK_FIELD_MASK 0xffffffffL
+#define MC_SEQ_RESERVE_0_S__SCLK_FIELD__SHIFT 0x00000000
+#define MC_SEQ_RESERVE_1_S__SCLK_FIELD_MASK 0xffffffffL
+#define MC_SEQ_RESERVE_1_S__SCLK_FIELD__SHIFT 0x00000000
+#define MC_SEQ_RESERVE_M__MCLK_FIELD_MASK 0xffffffffL
+#define MC_SEQ_RESERVE_M__MCLK_FIELD__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ0_MASK 0x0000000fL
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ0__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ1_MASK 0x000000f0L
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ1__SHIFT 0x00000004
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ2_MASK 0x00000f00L
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ2__SHIFT 0x00000008
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ3_MASK 0x0000f000L
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ4_MASK 0x000f0000L
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ4__SHIFT 0x00000010
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ5_MASK 0x00f00000L
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ5__SHIFT 0x00000014
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ6_MASK 0x0f000000L
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ6__SHIFT 0x00000018
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ7_MASK 0xf0000000L
+#define MC_SEQ_RXFRAMING_BYTE0_D0__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ0_MASK 0x0000000fL
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ0__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ1_MASK 0x000000f0L
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ1__SHIFT 0x00000004
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ2_MASK 0x00000f00L
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ2__SHIFT 0x00000008
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ3_MASK 0x0000f000L
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ4_MASK 0x000f0000L
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ4__SHIFT 0x00000010
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ5_MASK 0x00f00000L
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ5__SHIFT 0x00000014
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ6_MASK 0x0f000000L
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ6__SHIFT 0x00000018
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ7_MASK 0xf0000000L
+#define MC_SEQ_RXFRAMING_BYTE0_D1__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ0_MASK 0x0000000fL
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ0__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ1_MASK 0x000000f0L
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ1__SHIFT 0x00000004
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ2_MASK 0x00000f00L
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ2__SHIFT 0x00000008
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ3_MASK 0x0000f000L
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ4_MASK 0x000f0000L
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ4__SHIFT 0x00000010
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ5_MASK 0x00f00000L
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ5__SHIFT 0x00000014
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ6_MASK 0x0f000000L
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ6__SHIFT 0x00000018
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ7_MASK 0xf0000000L
+#define MC_SEQ_RXFRAMING_BYTE1_D0__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ0_MASK 0x0000000fL
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ0__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ1_MASK 0x000000f0L
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ1__SHIFT 0x00000004
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ2_MASK 0x00000f00L
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ2__SHIFT 0x00000008
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ3_MASK 0x0000f000L
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ4_MASK 0x000f0000L
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ4__SHIFT 0x00000010
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ5_MASK 0x00f00000L
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ5__SHIFT 0x00000014
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ6_MASK 0x0f000000L
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ6__SHIFT 0x00000018
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ7_MASK 0xf0000000L
+#define MC_SEQ_RXFRAMING_BYTE1_D1__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ0_MASK 0x0000000fL
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ0__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ1_MASK 0x000000f0L
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ1__SHIFT 0x00000004
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ2_MASK 0x00000f00L
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ2__SHIFT 0x00000008
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ3_MASK 0x0000f000L
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ4_MASK 0x000f0000L
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ4__SHIFT 0x00000010
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ5_MASK 0x00f00000L
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ5__SHIFT 0x00000014
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ6_MASK 0x0f000000L
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ6__SHIFT 0x00000018
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ7_MASK 0xf0000000L
+#define MC_SEQ_RXFRAMING_BYTE2_D0__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ0_MASK 0x0000000fL
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ0__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ1_MASK 0x000000f0L
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ1__SHIFT 0x00000004
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ2_MASK 0x00000f00L
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ2__SHIFT 0x00000008
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ3_MASK 0x0000f000L
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ4_MASK 0x000f0000L
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ4__SHIFT 0x00000010
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ5_MASK 0x00f00000L
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ5__SHIFT 0x00000014
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ6_MASK 0x0f000000L
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ6__SHIFT 0x00000018
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ7_MASK 0xf0000000L
+#define MC_SEQ_RXFRAMING_BYTE2_D1__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ0_MASK 0x0000000fL
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ0__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ1_MASK 0x000000f0L
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ1__SHIFT 0x00000004
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ2_MASK 0x00000f00L
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ2__SHIFT 0x00000008
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ3_MASK 0x0000f000L
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ4_MASK 0x000f0000L
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ4__SHIFT 0x00000010
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ5_MASK 0x00f00000L
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ5__SHIFT 0x00000014
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ6_MASK 0x0f000000L
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ6__SHIFT 0x00000018
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ7_MASK 0xf0000000L
+#define MC_SEQ_RXFRAMING_BYTE3_D0__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ0_MASK 0x0000000fL
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ0__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ1_MASK 0x000000f0L
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ1__SHIFT 0x00000004
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ2_MASK 0x00000f00L
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ2__SHIFT 0x00000008
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ3_MASK 0x0000f000L
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ4_MASK 0x000f0000L
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ4__SHIFT 0x00000010
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ5_MASK 0x00f00000L
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ5__SHIFT 0x00000014
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ6_MASK 0x0f000000L
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ6__SHIFT 0x00000018
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ7_MASK 0xf0000000L
+#define MC_SEQ_RXFRAMING_BYTE3_D1__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_RXFRAMING_DBI_D0__DBI0_MASK 0x0000000fL
+#define MC_SEQ_RXFRAMING_DBI_D0__DBI0__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_DBI_D0__DBI1_MASK 0x000000f0L
+#define MC_SEQ_RXFRAMING_DBI_D0__DBI1__SHIFT 0x00000004
+#define MC_SEQ_RXFRAMING_DBI_D0__DBI2_MASK 0x00000f00L
+#define MC_SEQ_RXFRAMING_DBI_D0__DBI2__SHIFT 0x00000008
+#define MC_SEQ_RXFRAMING_DBI_D0__DBI3_MASK 0x0000f000L
+#define MC_SEQ_RXFRAMING_DBI_D0__DBI3__SHIFT 0x0000000c
+#define MC_SEQ_RXFRAMING_DBI_D1__DBI0_MASK 0x0000000fL
+#define MC_SEQ_RXFRAMING_DBI_D1__DBI0__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_DBI_D1__DBI1_MASK 0x000000f0L
+#define MC_SEQ_RXFRAMING_DBI_D1__DBI1__SHIFT 0x00000004
+#define MC_SEQ_RXFRAMING_DBI_D1__DBI2_MASK 0x00000f00L
+#define MC_SEQ_RXFRAMING_DBI_D1__DBI2__SHIFT 0x00000008
+#define MC_SEQ_RXFRAMING_DBI_D1__DBI3_MASK 0x0000f000L
+#define MC_SEQ_RXFRAMING_DBI_D1__DBI3__SHIFT 0x0000000c
+#define MC_SEQ_RXFRAMING_EDC_D0__EDC0_MASK 0x0000000fL
+#define MC_SEQ_RXFRAMING_EDC_D0__EDC0__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_EDC_D0__EDC1_MASK 0x000000f0L
+#define MC_SEQ_RXFRAMING_EDC_D0__EDC1__SHIFT 0x00000004
+#define MC_SEQ_RXFRAMING_EDC_D0__EDC2_MASK 0x00000f00L
+#define MC_SEQ_RXFRAMING_EDC_D0__EDC2__SHIFT 0x00000008
+#define MC_SEQ_RXFRAMING_EDC_D0__EDC3_MASK 0x0000f000L
+#define MC_SEQ_RXFRAMING_EDC_D0__EDC3__SHIFT 0x0000000c
+#define MC_SEQ_RXFRAMING_EDC_D0__WCDR0_MASK 0x000f0000L
+#define MC_SEQ_RXFRAMING_EDC_D0__WCDR0__SHIFT 0x00000010
+#define MC_SEQ_RXFRAMING_EDC_D0__WCDR1_MASK 0x00f00000L
+#define MC_SEQ_RXFRAMING_EDC_D0__WCDR1__SHIFT 0x00000014
+#define MC_SEQ_RXFRAMING_EDC_D0__WCDR2_MASK 0x0f000000L
+#define MC_SEQ_RXFRAMING_EDC_D0__WCDR2__SHIFT 0x00000018
+#define MC_SEQ_RXFRAMING_EDC_D0__WCDR3_MASK 0xf0000000L
+#define MC_SEQ_RXFRAMING_EDC_D0__WCDR3__SHIFT 0x0000001c
+#define MC_SEQ_RXFRAMING_EDC_D1__EDC0_MASK 0x0000000fL
+#define MC_SEQ_RXFRAMING_EDC_D1__EDC0__SHIFT 0x00000000
+#define MC_SEQ_RXFRAMING_EDC_D1__EDC1_MASK 0x000000f0L
+#define MC_SEQ_RXFRAMING_EDC_D1__EDC1__SHIFT 0x00000004
+#define MC_SEQ_RXFRAMING_EDC_D1__EDC2_MASK 0x00000f00L
+#define MC_SEQ_RXFRAMING_EDC_D1__EDC2__SHIFT 0x00000008
+#define MC_SEQ_RXFRAMING_EDC_D1__EDC3_MASK 0x0000f000L
+#define MC_SEQ_RXFRAMING_EDC_D1__EDC3__SHIFT 0x0000000c
+#define MC_SEQ_RXFRAMING_EDC_D1__WCDR0_MASK 0x000f0000L
+#define MC_SEQ_RXFRAMING_EDC_D1__WCDR0__SHIFT 0x00000010
+#define MC_SEQ_RXFRAMING_EDC_D1__WCDR1_MASK 0x00f00000L
+#define MC_SEQ_RXFRAMING_EDC_D1__WCDR1__SHIFT 0x00000014
+#define MC_SEQ_RXFRAMING_EDC_D1__WCDR2_MASK 0x0f000000L
+#define MC_SEQ_RXFRAMING_EDC_D1__WCDR2__SHIFT 0x00000018
+#define MC_SEQ_RXFRAMING_EDC_D1__WCDR3_MASK 0xf0000000L
+#define MC_SEQ_RXFRAMING_EDC_D1__WCDR3__SHIFT 0x0000001c
+#define MC_SEQ_STATUS_M__CMD_RDY_D0_MASK 0x00000004L
+#define MC_SEQ_STATUS_M__CMD_RDY_D0__SHIFT 0x00000002
+#define MC_SEQ_STATUS_M__CMD_RDY_D1_MASK 0x00000008L
+#define MC_SEQ_STATUS_M__CMD_RDY_D1__SHIFT 0x00000003
+#define MC_SEQ_STATUS_M__PMG_FSMSTATE_MASK 0x01f00000L
+#define MC_SEQ_STATUS_M__PMG_FSMSTATE__SHIFT 0x00000014
+#define MC_SEQ_STATUS_M__PMG_PWRSTATE_MASK 0x00010000L
+#define MC_SEQ_STATUS_M__PMG_PWRSTATE__SHIFT 0x00000010
+#define MC_SEQ_STATUS_M__PWRUP_COMPL_D0_MASK 0x00000001L
+#define MC_SEQ_STATUS_M__PWRUP_COMPL_D0__SHIFT 0x00000000
+#define MC_SEQ_STATUS_M__PWRUP_COMPL_D1_MASK 0x00000002L
+#define MC_SEQ_STATUS_M__PWRUP_COMPL_D1__SHIFT 0x00000001
+#define MC_SEQ_STATUS_M__SEQ0_ARB_CMD_FIFO_EMPTY_MASK 0x00000100L
+#define MC_SEQ_STATUS_M__SEQ0_ARB_CMD_FIFO_EMPTY__SHIFT 0x00000008
+#define MC_SEQ_STATUS_M__SEQ0_BUSY_HYS_MASK 0x02000000L
+#define MC_SEQ_STATUS_M__SEQ0_BUSY_HYS__SHIFT 0x00000019
+#define MC_SEQ_STATUS_M__SEQ0_BUSY_MASK 0x00004000L
+#define MC_SEQ_STATUS_M__SEQ0_BUSY__SHIFT 0x0000000e
+#define MC_SEQ_STATUS_M__SEQ0_RS_DATA_FIFO_FULL_MASK 0x00001000L
+#define MC_SEQ_STATUS_M__SEQ0_RS_DATA_FIFO_FULL__SHIFT 0x0000000c
+#define MC_SEQ_STATUS_M__SEQ1_ARB_CMD_FIFO_EMPTY_MASK 0x00000200L
+#define MC_SEQ_STATUS_M__SEQ1_ARB_CMD_FIFO_EMPTY__SHIFT 0x00000009
+#define MC_SEQ_STATUS_M__SEQ1_BUSY_HYS_MASK 0x04000000L
+#define MC_SEQ_STATUS_M__SEQ1_BUSY_HYS__SHIFT 0x0000001a
+#define MC_SEQ_STATUS_M__SEQ1_BUSY_MASK 0x00008000L
+#define MC_SEQ_STATUS_M__SEQ1_BUSY__SHIFT 0x0000000f
+#define MC_SEQ_STATUS_M__SEQ1_RS_DATA_FIFO_FULL_MASK 0x00002000L
+#define MC_SEQ_STATUS_M__SEQ1_RS_DATA_FIFO_FULL__SHIFT 0x0000000d
+#define MC_SEQ_STATUS_M__SLF_D0_MASK 0x00000010L
+#define MC_SEQ_STATUS_M__SLF_D0__SHIFT 0x00000004
+#define MC_SEQ_STATUS_M__SLF_D1_MASK 0x00000020L
+#define MC_SEQ_STATUS_M__SLF_D1__SHIFT 0x00000005
+#define MC_SEQ_STATUS_M__SS_SLF_D0_MASK 0x00000040L
+#define MC_SEQ_STATUS_M__SS_SLF_D0__SHIFT 0x00000006
+#define MC_SEQ_STATUS_M__SS_SLF_D1_MASK 0x00000080L
+#define MC_SEQ_STATUS_M__SS_SLF_D1__SHIFT 0x00000007
+#define MC_SEQ_STATUS_S__SEQ0_ARB_CMD_FIFO_FULL_MASK 0x00000010L
+#define MC_SEQ_STATUS_S__SEQ0_ARB_CMD_FIFO_FULL__SHIFT 0x00000004
+#define MC_SEQ_STATUS_S__SEQ0_ARB_DATA_FIFO_FULL_MASK 0x00000001L
+#define MC_SEQ_STATUS_S__SEQ0_ARB_DATA_FIFO_FULL__SHIFT 0x00000000
+#define MC_SEQ_STATUS_S__SEQ0_RS_DATA_FIFO_EMPTY_MASK 0x00000100L
+#define MC_SEQ_STATUS_S__SEQ0_RS_DATA_FIFO_EMPTY__SHIFT 0x00000008
+#define MC_SEQ_STATUS_S__SEQ1_ARB_CMD_FIFO_FULL_MASK 0x00000020L
+#define MC_SEQ_STATUS_S__SEQ1_ARB_CMD_FIFO_FULL__SHIFT 0x00000005
+#define MC_SEQ_STATUS_S__SEQ1_ARB_DATA_FIFO_FULL_MASK 0x00000002L
+#define MC_SEQ_STATUS_S__SEQ1_ARB_DATA_FIFO_FULL__SHIFT 0x00000001
+#define MC_SEQ_STATUS_S__SEQ1_RS_DATA_FIFO_EMPTY_MASK 0x00000200L
+#define MC_SEQ_STATUS_S__SEQ1_RS_DATA_FIFO_EMPTY__SHIFT 0x00000009
+#define MC_SEQ_SUP_CNTL__BKPT_CLEAR_MASK 0x00000080L
+#define MC_SEQ_SUP_CNTL__BKPT_CLEAR__SHIFT 0x00000007
+#define MC_SEQ_SUP_CNTL__FAST_WRITE_MASK 0x00000040L
+#define MC_SEQ_SUP_CNTL__FAST_WRITE__SHIFT 0x00000006
+#define MC_SEQ_SUP_CNTL__PGM_CHKSUM_MASK 0xff800000L
+#define MC_SEQ_SUP_CNTL__PGM_CHKSUM__SHIFT 0x00000017
+#define MC_SEQ_SUP_CNTL__PGM_READ_MASK 0x00000020L
+#define MC_SEQ_SUP_CNTL__PGM_READ__SHIFT 0x00000005
+#define MC_SEQ_SUP_CNTL__PGM_WRITE_MASK 0x00000010L
+#define MC_SEQ_SUP_CNTL__PGM_WRITE__SHIFT 0x00000004
+#define MC_SEQ_SUP_CNTL__RESET_PC_MASK 0x00000008L
+#define MC_SEQ_SUP_CNTL__RESET_PC__SHIFT 0x00000003
+#define MC_SEQ_SUP_CNTL__RUN_MASK 0x00000001L
+#define MC_SEQ_SUP_CNTL__RUN__SHIFT 0x00000000
+#define MC_SEQ_SUP_CNTL__SINGLE_STEP_MASK 0x00000002L
+#define MC_SEQ_SUP_CNTL__SINGLE_STEP__SHIFT 0x00000001
+#define MC_SEQ_SUP_CNTL__SW_WAKE_MASK 0x00000004L
+#define MC_SEQ_SUP_CNTL__SW_WAKE__SHIFT 0x00000002
+#define MC_SEQ_SUP_DEC_STAT__STATUS_MASK 0xffffffffL
+#define MC_SEQ_SUP_DEC_STAT__STATUS__SHIFT 0x00000000
+#define MC_SEQ_SUP_GP0_STAT__STATUS_MASK 0xffffffffL
+#define MC_SEQ_SUP_GP0_STAT__STATUS__SHIFT 0x00000000
+#define MC_SEQ_SUP_GP1_STAT__STATUS_MASK 0xffffffffL
+#define MC_SEQ_SUP_GP1_STAT__STATUS__SHIFT 0x00000000
+#define MC_SEQ_SUP_GP2_STAT__STATUS_MASK 0xffffffffL
+#define MC_SEQ_SUP_GP2_STAT__STATUS__SHIFT 0x00000000
+#define MC_SEQ_SUP_GP3_STAT__STATUS_MASK 0xffffffffL
+#define MC_SEQ_SUP_GP3_STAT__STATUS__SHIFT 0x00000000
+#define MC_SEQ_SUP_IR_STAT__STATUS_MASK 0xffffffffL
+#define MC_SEQ_SUP_IR_STAT__STATUS__SHIFT 0x00000000
+#define MC_SEQ_SUP_PGM__CNTL_MASK 0xffffffffL
+#define MC_SEQ_SUP_PGM__CNTL__SHIFT 0x00000000
+#define MC_SEQ_SUP_PGM_STAT__STATUS_MASK 0xffffffffL
+#define MC_SEQ_SUP_PGM_STAT__STATUS__SHIFT 0x00000000
+#define MC_SEQ_SUP_R_PGM__PGM_MASK 0xffffffffL
+#define MC_SEQ_SUP_R_PGM__PGM__SHIFT 0x00000000
+#define MC_SEQ_TCG_CNTL__AREF_BOTH_MASK 0x04000000L
+#define MC_SEQ_TCG_CNTL__AREF_BOTH__SHIFT 0x0000001a
+#define MC_SEQ_TCG_CNTL__AREF_LAST_MASK 0x02000000L
+#define MC_SEQ_TCG_CNTL__AREF_LAST__SHIFT 0x00000019
+#define MC_SEQ_TCG_CNTL__BURST_NUM_MASK 0x00380000L
+#define MC_SEQ_TCG_CNTL__BURST_NUM__SHIFT 0x00000013
+#define MC_SEQ_TCG_CNTL__DATA_CNT_MASK 0x0000f000L
+#define MC_SEQ_TCG_CNTL__DATA_CNT__SHIFT 0x0000000c
+#define MC_SEQ_TCG_CNTL__DONE_MASK 0x80000000L
+#define MC_SEQ_TCG_CNTL__DONE__SHIFT 0x0000001f
+#define MC_SEQ_TCG_CNTL__ENABLE_D0_MASK 0x00000002L
+#define MC_SEQ_TCG_CNTL__ENABLE_D0__SHIFT 0x00000001
+#define MC_SEQ_TCG_CNTL__ENABLE_D1_MASK 0x00000004L
+#define MC_SEQ_TCG_CNTL__ENABLE_D1__SHIFT 0x00000002
+#define MC_SEQ_TCG_CNTL__FRAME_TRAIN_MASK 0x00040000L
+#define MC_SEQ_TCG_CNTL__FRAME_TRAIN__SHIFT 0x00000012
+#define MC_SEQ_TCG_CNTL__INFINITE_CMD_MASK 0x00000080L
+#define MC_SEQ_TCG_CNTL__INFINITE_CMD__SHIFT 0x00000007
+#define MC_SEQ_TCG_CNTL__ISSUE_AREF_MASK 0x00400000L
+#define MC_SEQ_TCG_CNTL__ISSUE_AREF__SHIFT 0x00000016
+#define MC_SEQ_TCG_CNTL__LOAD_FIFO_MASK 0x00010000L
+#define MC_SEQ_TCG_CNTL__LOAD_FIFO__SHIFT 0x00000010
+#define MC_SEQ_TCG_CNTL__MOP_MASK 0x00000f00L
+#define MC_SEQ_TCG_CNTL__MOP__SHIFT 0x00000008
+#define MC_SEQ_TCG_CNTL__NFIFO_MASK 0x00000070L
+#define MC_SEQ_TCG_CNTL__NFIFO__SHIFT 0x00000004
+#define MC_SEQ_TCG_CNTL__RESET_MASK 0x00000001L
+#define MC_SEQ_TCG_CNTL__RESET__SHIFT 0x00000000
+#define MC_SEQ_TCG_CNTL__SHORT_LDFF_MASK 0x00020000L
+#define MC_SEQ_TCG_CNTL__SHORT_LDFF__SHIFT 0x00000011
+#define MC_SEQ_TCG_CNTL__START_MASK 0x00000008L
+#define MC_SEQ_TCG_CNTL__START__SHIFT 0x00000003
+#define MC_SEQ_TCG_CNTL__TXDBI_CNTL_MASK 0x00800000L
+#define MC_SEQ_TCG_CNTL__TXDBI_CNTL__SHIFT 0x00000017
+#define MC_SEQ_TCG_CNTL__VPTR_MASK_MASK 0x01000000L
+#define MC_SEQ_TCG_CNTL__VPTR_MASK__SHIFT 0x00000018
+#define MC_SEQ_TIMER_RD__COUNTER_MASK 0xffffffffL
+#define MC_SEQ_TIMER_RD__COUNTER__SHIFT 0x00000000
+#define MC_SEQ_TIMER_WR__COUNTER_MASK 0xffffffffL
+#define MC_SEQ_TIMER_WR__COUNTER__SHIFT 0x00000000
+#define MC_SEQ_TRAIN_CAPTURE__ALLOWSTOP0_WAKEUP_MASK 0x00040000L
+#define MC_SEQ_TRAIN_CAPTURE__ALLOWSTOP0_WAKEUP__SHIFT 0x00000012
+#define MC_SEQ_TRAIN_CAPTURE__ALLOWSTOP1_WAKEUP_MASK 0x00080000L
+#define MC_SEQ_TRAIN_CAPTURE__ALLOWSTOP1_WAKEUP__SHIFT 0x00000013
+#define MC_SEQ_TRAIN_CAPTURE__ALLOWSTOPB0_WAKEUP_MASK 0x00200000L
+#define MC_SEQ_TRAIN_CAPTURE__ALLOWSTOPB0_WAKEUP__SHIFT 0x00000015
+#define MC_SEQ_TRAIN_CAPTURE__ALLOWSTOPB1_WAKEUP_MASK 0x00400000L
+#define MC_SEQ_TRAIN_CAPTURE__ALLOWSTOPB1_WAKEUP__SHIFT 0x00000016
+#define MC_SEQ_TRAIN_CAPTURE__D0_ARF_WAKEUP_MASK 0x00000001L
+#define MC_SEQ_TRAIN_CAPTURE__D0_ARF_WAKEUP__SHIFT 0x00000000
+#define MC_SEQ_TRAIN_CAPTURE__D0_CMD_FIFO_READY_WAKEUP_MASK 0x00000100L
+#define MC_SEQ_TRAIN_CAPTURE__D0_CMD_FIFO_READY_WAKEUP__SHIFT 0x00000008
+#define MC_SEQ_TRAIN_CAPTURE__D0_DATA_FIFO_READY_WAKEUP_MASK 0x00000400L
+#define MC_SEQ_TRAIN_CAPTURE__D0_DATA_FIFO_READY_WAKEUP__SHIFT 0x0000000a
+#define MC_SEQ_TRAIN_CAPTURE__D0_IDLEH_WAKEUP_MASK 0x01000000L
+#define MC_SEQ_TRAIN_CAPTURE__D0_IDLEH_WAKEUP__SHIFT 0x00000018
+#define MC_SEQ_TRAIN_CAPTURE__D0_REDC_WAKEUP_MASK 0x00000004L
+#define MC_SEQ_TRAIN_CAPTURE__D0_REDC_WAKEUP__SHIFT 0x00000002
+#define MC_SEQ_TRAIN_CAPTURE__D0_WEDC_WAKEUP_MASK 0x00000010L
+#define MC_SEQ_TRAIN_CAPTURE__D0_WEDC_WAKEUP__SHIFT 0x00000004
+#define MC_SEQ_TRAIN_CAPTURE__D1_ARF_WAKEUP_MASK 0x00000002L
+#define MC_SEQ_TRAIN_CAPTURE__D1_ARF_WAKEUP__SHIFT 0x00000001
+#define MC_SEQ_TRAIN_CAPTURE__D1_CMD_FIFO_READY_WAKEUP_MASK 0x00000200L
+#define MC_SEQ_TRAIN_CAPTURE__D1_CMD_FIFO_READY_WAKEUP__SHIFT 0x00000009
+#define MC_SEQ_TRAIN_CAPTURE__D1_DATA_FIFO_READY_WAKEUP_MASK 0x00000800L
+#define MC_SEQ_TRAIN_CAPTURE__D1_DATA_FIFO_READY_WAKEUP__SHIFT 0x0000000b
+#define MC_SEQ_TRAIN_CAPTURE__D1_IDLEH_WAKEUP_MASK 0x02000000L
+#define MC_SEQ_TRAIN_CAPTURE__D1_IDLEH_WAKEUP__SHIFT 0x00000019
+#define MC_SEQ_TRAIN_CAPTURE__D1_REDC_WAKEUP_MASK 0x00000008L
+#define MC_SEQ_TRAIN_CAPTURE__D1_REDC_WAKEUP__SHIFT 0x00000003
+#define MC_SEQ_TRAIN_CAPTURE__D1_WEDC_WAKEUP_MASK 0x00000020L
+#define MC_SEQ_TRAIN_CAPTURE__D1_WEDC_WAKEUP__SHIFT 0x00000005
+#define MC_SEQ_TRAIN_CAPTURE__DPM_LPT_WAKEUP_MASK 0x00800000L
+#define MC_SEQ_TRAIN_CAPTURE__DPM_LPT_WAKEUP__SHIFT 0x00000017
+#define MC_SEQ_TRAIN_CAPTURE__DPM_WAKEUP_MASK 0x00100000L
+#define MC_SEQ_TRAIN_CAPTURE__DPM_WAKEUP__SHIFT 0x00000014
+#define MC_SEQ_TRAIN_CAPTURE__MCLK_FREQ_CHANGE_WAKEUP_MASK 0x00000040L
+#define MC_SEQ_TRAIN_CAPTURE__MCLK_FREQ_CHANGE_WAKEUP__SHIFT 0x00000006
+#define MC_SEQ_TRAIN_CAPTURE__PHY_PG_WAKEUP_MASK 0x04000000L
+#define MC_SEQ_TRAIN_CAPTURE__PHY_PG_WAKEUP__SHIFT 0x0000001a
+#define MC_SEQ_TRAIN_CAPTURE__RESERVE0_WAKEUP_MASK 0x00002000L
+#define MC_SEQ_TRAIN_CAPTURE__RESERVE0_WAKEUP__SHIFT 0x0000000d
+#define MC_SEQ_TRAIN_CAPTURE__SCLK_SRBM_READY_WAKEUP_MASK 0x00000080L
+#define MC_SEQ_TRAIN_CAPTURE__SCLK_SRBM_READY_WAKEUP__SHIFT 0x00000007
+#define MC_SEQ_TRAIN_CAPTURE__SOFTWARE_WAKEUP_WAKEUP_MASK 0x00001000L
+#define MC_SEQ_TRAIN_CAPTURE__SOFTWARE_WAKEUP_WAKEUP__SHIFT 0x0000000c
+#define MC_SEQ_TRAIN_CAPTURE__TCG_DONE_WAKEUP_MASK 0x00020000L
+#define MC_SEQ_TRAIN_CAPTURE__TCG_DONE_WAKEUP__SHIFT 0x00000011
+#define MC_SEQ_TRAIN_CAPTURE__TIMER_DONE_WAKEUP_MASK 0x00008000L
+#define MC_SEQ_TRAIN_CAPTURE__TIMER_DONE_WAKEUP__SHIFT 0x0000000f
+#define MC_SEQ_TRAIN_CAPTURE__TSM_DONE_WAKEUP_MASK 0x00004000L
+#define MC_SEQ_TRAIN_CAPTURE__TSM_DONE_WAKEUP__SHIFT 0x0000000e
+#define MC_SEQ_TRAIN_EDC_THRESHOLD2__THRESHOLD_PERIOD_MASK 0xffffffffL
+#define MC_SEQ_TRAIN_EDC_THRESHOLD2__THRESHOLD_PERIOD__SHIFT 0x00000000
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__CH0_LINK_RETRAIN_IN_PROGRESS_MASK 0x00000100L
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__CH0_LINK_RETRAIN_IN_PROGRESS__SHIFT 0x00000008
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__CH0_LINK_RETRAIN_STATUS_MASK 0x00000001L
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__CH0_LINK_RETRAIN_STATUS__SHIFT 0x00000000
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__CH1_LINK_RETRAIN_IN_PROGRESS_MASK 0x00000200L
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__CH1_LINK_RETRAIN_IN_PROGRESS__SHIFT 0x00000009
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__CH1_LINK_RETRAIN_STATUS_MASK 0x00000002L
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__CH1_LINK_RETRAIN_STATUS__SHIFT 0x00000001
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__CLEAR_RETRAIN_STATUS_MASK 0x00000004L
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__CLEAR_RETRAIN_STATUS__SHIFT 0x00000002
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__RETRAIN_MONITOR_MASK 0x00000030L
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__RETRAIN_MONITOR__SHIFT 0x00000004
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__RETRAIN_VBI_MASK 0x00000008L
+#define MC_SEQ_TRAIN_EDC_THRESHOLD3__RETRAIN_VBI__SHIFT 0x00000003
+#define MC_SEQ_TRAIN_EDC_THRESHOLD__READ_EDC_THRESHOLD_MASK 0xffff0000L
+#define MC_SEQ_TRAIN_EDC_THRESHOLD__READ_EDC_THRESHOLD__SHIFT 0x00000010
+#define MC_SEQ_TRAIN_EDC_THRESHOLD__WRITE_EDC_THRESHOLD_MASK 0x0000ffffL
+#define MC_SEQ_TRAIN_EDC_THRESHOLD__WRITE_EDC_THRESHOLD__SHIFT 0x00000000
+#define MC_SEQ_TRAIN_TIMING__TARF2T_MASK 0x000003e0L
+#define MC_SEQ_TRAIN_TIMING__TARF2T__SHIFT 0x00000005
+#define MC_SEQ_TRAIN_TIMING__TLD2LD_MASK 0x000f8000L
+#define MC_SEQ_TRAIN_TIMING__TLD2LD__SHIFT 0x0000000f
+#define MC_SEQ_TRAIN_TIMING__TT2ROW_MASK 0x00007c00L
+#define MC_SEQ_TRAIN_TIMING__TT2ROW__SHIFT 0x0000000a
+#define MC_SEQ_TRAIN_TIMING__TWT2RT_MASK 0x0000001fL
+#define MC_SEQ_TRAIN_TIMING__TWT2RT__SHIFT 0x00000000
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__ALLOWSTOP0_WAKEUP_MASK 0x00040000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__ALLOWSTOP0_WAKEUP__SHIFT 0x00000012
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__ALLOWSTOP1_WAKEUP_MASK 0x00080000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__ALLOWSTOP1_WAKEUP__SHIFT 0x00000013
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__ALLOWSTOPB0_WAKEUP_MASK 0x00200000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__ALLOWSTOPB0_WAKEUP__SHIFT 0x00000015
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__ALLOWSTOPB1_WAKEUP_MASK 0x00400000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__ALLOWSTOPB1_WAKEUP__SHIFT 0x00000016
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__CLEARALL_MASK 0x00010000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__CLEARALL__SHIFT 0x00000010
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D0_ARF_WAKEUP_MASK 0x00000001L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D0_ARF_WAKEUP__SHIFT 0x00000000
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D0_CMD_FIFO_READY_WAKEUP_MASK 0x00000100L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D0_CMD_FIFO_READY_WAKEUP__SHIFT 0x00000008
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D0_DATA_FIFO_READY_WAKEUP_MASK 0x00000400L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D0_DATA_FIFO_READY_WAKEUP__SHIFT 0x0000000a
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D0_IDLEH_WAKEUP_MASK 0x01000000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D0_IDLEH_WAKEUP__SHIFT 0x00000018
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D0_REDC_WAKEUP_MASK 0x00000004L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D0_REDC_WAKEUP__SHIFT 0x00000002
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D0_WEDC_WAKEUP_MASK 0x00000010L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D0_WEDC_WAKEUP__SHIFT 0x00000004
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D1_ARF_WAKEUP_MASK 0x00000002L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D1_ARF_WAKEUP__SHIFT 0x00000001
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D1_CMD_FIFO_READY_WAKEUP_MASK 0x00000200L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D1_CMD_FIFO_READY_WAKEUP__SHIFT 0x00000009
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D1_DATA_FIFO_READY_WAKEUP_MASK 0x00000800L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D1_DATA_FIFO_READY_WAKEUP__SHIFT 0x0000000b
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D1_IDLEH_WAKEUP_MASK 0x02000000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D1_IDLEH_WAKEUP__SHIFT 0x00000019
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D1_REDC_WAKEUP_MASK 0x00000008L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D1_REDC_WAKEUP__SHIFT 0x00000003
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D1_WEDC_WAKEUP_MASK 0x00000020L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__D1_WEDC_WAKEUP__SHIFT 0x00000005
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__DPM_LPT_WAKEUP_MASK 0x00800000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__DPM_LPT_WAKEUP__SHIFT 0x00000017
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__DPM_WAKEUP_MASK 0x00100000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__DPM_WAKEUP__SHIFT 0x00000014
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__MCLK_FREQ_CHANGE_WAKEUP_MASK 0x00000040L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__MCLK_FREQ_CHANGE_WAKEUP__SHIFT 0x00000006
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__PHY_PG_WAKEUP_MASK 0x04000000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__PHY_PG_WAKEUP__SHIFT 0x0000001a
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__RESERVE0_WAKEUP_MASK 0x00002000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__RESERVE0_WAKEUP__SHIFT 0x0000000d
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__SCLK_SRBM_READY_WAKEUP_MASK 0x00000080L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__SCLK_SRBM_READY_WAKEUP__SHIFT 0x00000007
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__SOFTWARE_WAKEUP_WAKEUP_MASK 0x00001000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__SOFTWARE_WAKEUP_WAKEUP__SHIFT 0x0000000c
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__TCG_DONE_WAKEUP_MASK 0x00020000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__TCG_DONE_WAKEUP__SHIFT 0x00000011
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__TIMER_DONE_WAKEUP_MASK 0x00008000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__TIMER_DONE_WAKEUP__SHIFT 0x0000000f
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__TSM_DONE_WAKEUP_MASK 0x00004000L
+#define MC_SEQ_TRAIN_WAKEUP_CLEAR__TSM_DONE_WAKEUP__SHIFT 0x0000000e
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__AUTO_REFRESH_ADDR_TRAIN_MASK 0x00000100L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__AUTO_REFRESH_ADDR_TRAIN__SHIFT 0x00000008
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__AUTO_REFRESH_READ_TRAIN_MASK 0x00000400L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__AUTO_REFRESH_READ_TRAIN__SHIFT 0x0000000a
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__AUTO_REFRESH_WAKEUP_EARLY_MASK 0x00100000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__AUTO_REFRESH_WAKEUP_EARLY__SHIFT 0x00000014
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__AUTO_REFRESH_WCK_TRAIN_MASK 0x00000200L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__AUTO_REFRESH_WCK_TRAIN__SHIFT 0x00000009
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__AUTO_REFRESH_WRITE_TRAIN_MASK 0x00000800L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__AUTO_REFRESH_WRITE_TRAIN__SHIFT 0x0000000b
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BLOCK_ARB_RD_D0_MASK 0x01000000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BLOCK_ARB_RD_D0__SHIFT 0x00000018
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BLOCK_ARB_RD_D1_MASK 0x04000000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BLOCK_ARB_RD_D1__SHIFT 0x0000001a
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BLOCK_ARB_WR_D0_MASK 0x02000000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BLOCK_ARB_WR_D0__SHIFT 0x00000019
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BLOCK_ARB_WR_D1_MASK 0x08000000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BLOCK_ARB_WR_D1__SHIFT 0x0000001b
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BOOT_UP_ADDR_TRAIN_MASK 0x00000001L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BOOT_UP_ADDR_TRAIN__SHIFT 0x00000000
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BOOT_UP_READ_TRAIN_MASK 0x00000004L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BOOT_UP_READ_TRAIN__SHIFT 0x00000002
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BOOT_UP_WCK_TRAIN_MASK 0x00000002L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BOOT_UP_WCK_TRAIN__SHIFT 0x00000001
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BOOT_UP_WRITE_TRAIN_MASK 0x00000008L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__BOOT_UP_WRITE_TRAIN__SHIFT 0x00000003
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__DISP_ASTOP_WAKEUP_MASK 0x20000000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__DISP_ASTOP_WAKEUP__SHIFT 0x0000001d
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__READ_ECC_ADDR_TRAIN_MASK 0x00010000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__READ_ECC_ADDR_TRAIN__SHIFT 0x00000010
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__READ_ECC_READ_TRAIN_MASK 0x00040000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__READ_ECC_READ_TRAIN__SHIFT 0x00000012
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__READ_ECC_WCK_TRAIN_MASK 0x00020000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__READ_ECC_WCK_TRAIN__SHIFT 0x00000011
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__READ_ECC_WRITE_TRAIN_MASK 0x00080000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__READ_ECC_WRITE_TRAIN__SHIFT 0x00000013
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__SELF_REFRESH_ADDR_TRAIN_MASK 0x00000010L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__SELF_REFRESH_ADDR_TRAIN__SHIFT 0x00000004
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__SELF_REFRESH_READ_TRAIN_MASK 0x00000040L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__SELF_REFRESH_READ_TRAIN__SHIFT 0x00000006
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__SELF_REFRESH_WCK_TRAIN_MASK 0x00000020L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__SELF_REFRESH_WCK_TRAIN__SHIFT 0x00000005
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__SELF_REFRESH_WRITE_TRAIN_MASK 0x00000080L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__SELF_REFRESH_WRITE_TRAIN__SHIFT 0x00000007
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__STOP_WCK_D0_MASK 0x00200000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__STOP_WCK_D0__SHIFT 0x00000015
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__STOP_WCK_D1_MASK 0x00400000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__STOP_WCK_D1__SHIFT 0x00000016
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__SW_WAKEUP_MASK 0x10000000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__SW_WAKEUP__SHIFT 0x0000001c
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D0_MASK 0x40000000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D0__SHIFT 0x0000001e
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D1_MASK 0x80000000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D1__SHIFT 0x0000001f
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__WRITE_ECC_ADDR_TRAIN_MASK 0x00001000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__WRITE_ECC_ADDR_TRAIN__SHIFT 0x0000000c
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__WRITE_ECC_READ_TRAIN_MASK 0x00004000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__WRITE_ECC_READ_TRAIN__SHIFT 0x0000000e
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__WRITE_ECC_WCK_TRAIN_MASK 0x00002000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__WRITE_ECC_WCK_TRAIN__SHIFT 0x0000000d
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__WRITE_ECC_WRITE_TRAIN_MASK 0x00008000L
+#define MC_SEQ_TRAIN_WAKEUP_CNTL__WRITE_ECC_WRITE_TRAIN__SHIFT 0x0000000f
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__ALLOWSTOP0_WAKEUP_MASK 0x00040000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__ALLOWSTOP0_WAKEUP__SHIFT 0x00000012
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__ALLOWSTOP1_WAKEUP_MASK 0x00080000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__ALLOWSTOP1_WAKEUP__SHIFT 0x00000013
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__ALLOWSTOPB0_WAKEUP_MASK 0x00200000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__ALLOWSTOPB0_WAKEUP__SHIFT 0x00000015
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__ALLOWSTOPB1_WAKEUP_MASK 0x00400000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__ALLOWSTOPB1_WAKEUP__SHIFT 0x00000016
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D0_ARF_WAKEUP_MASK 0x00000001L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D0_ARF_WAKEUP__SHIFT 0x00000000
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D0_CMD_FIFO_READY_WAKEUP_MASK 0x00000100L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D0_CMD_FIFO_READY_WAKEUP__SHIFT 0x00000008
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D0_DATA_FIFO_READY_WAKEUP_MASK 0x00000400L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D0_DATA_FIFO_READY_WAKEUP__SHIFT 0x0000000a
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D0_IDLEH_WAKEUP_MASK 0x01000000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D0_IDLEH_WAKEUP__SHIFT 0x00000018
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D0_REDC_WAKEUP_MASK 0x00000004L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D0_REDC_WAKEUP__SHIFT 0x00000002
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D0_WEDC_WAKEUP_MASK 0x00000010L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D0_WEDC_WAKEUP__SHIFT 0x00000004
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D1_ARF_WAKEUP_MASK 0x00000002L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D1_ARF_WAKEUP__SHIFT 0x00000001
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D1_CMD_FIFO_READY_WAKEUP_MASK 0x00000200L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D1_CMD_FIFO_READY_WAKEUP__SHIFT 0x00000009
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D1_DATA_FIFO_READY_WAKEUP_MASK 0x00000800L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D1_DATA_FIFO_READY_WAKEUP__SHIFT 0x0000000b
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D1_IDLEH_WAKEUP_MASK 0x02000000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D1_IDLEH_WAKEUP__SHIFT 0x00000019
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D1_REDC_WAKEUP_MASK 0x00000008L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D1_REDC_WAKEUP__SHIFT 0x00000003
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D1_WEDC_WAKEUP_MASK 0x00000020L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__D1_WEDC_WAKEUP__SHIFT 0x00000005
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__DPM_LPT_WAKEUP_MASK 0x00800000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__DPM_LPT_WAKEUP__SHIFT 0x00000017
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__DPM_WAKEUP_MASK 0x00100000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__DPM_WAKEUP__SHIFT 0x00000014
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__MCLK_FREQ_CHANGE_WAKEUP_MASK 0x00000040L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__MCLK_FREQ_CHANGE_WAKEUP__SHIFT 0x00000006
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__PHY_PG_WAKEUP_MASK 0x04000000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__PHY_PG_WAKEUP__SHIFT 0x0000001a
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__RESERVE0_WAKEUP_MASK 0x00002000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__RESERVE0_WAKEUP__SHIFT 0x0000000d
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__SCLK_SRBM_READY_WAKEUP_MASK 0x00000080L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__SCLK_SRBM_READY_WAKEUP__SHIFT 0x00000007
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__SOFTWARE_WAKEUP_WAKEUP_MASK 0x00001000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__SOFTWARE_WAKEUP_WAKEUP__SHIFT 0x0000000c
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__TCG_DONE_WAKEUP_MASK 0x00020000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__TCG_DONE_WAKEUP__SHIFT 0x00000011
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__TIMER_DONE_WAKEUP_MASK 0x00008000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__TIMER_DONE_WAKEUP__SHIFT 0x0000000f
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__TSM_DONE_WAKEUP_MASK 0x00004000L
+#define MC_SEQ_TRAIN_WAKEUP_EDGE__TSM_DONE_WAKEUP__SHIFT 0x0000000e
+#define MC_SEQ_TRAIN_WAKEUP_MASK__ALLOWSTOP0_WAKEUP_MASK 0x00040000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__ALLOWSTOP0_WAKEUP__SHIFT 0x00000012
+#define MC_SEQ_TRAIN_WAKEUP_MASK__ALLOWSTOP1_WAKEUP_MASK 0x00080000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__ALLOWSTOP1_WAKEUP__SHIFT 0x00000013
+#define MC_SEQ_TRAIN_WAKEUP_MASK__ALLOWSTOPB0_WAKEUP_MASK 0x00200000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__ALLOWSTOPB0_WAKEUP__SHIFT 0x00000015
+#define MC_SEQ_TRAIN_WAKEUP_MASK__ALLOWSTOPB1_WAKEUP_MASK 0x00400000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__ALLOWSTOPB1_WAKEUP__SHIFT 0x00000016
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D0_ARF_WAKEUP_MASK 0x00000001L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D0_ARF_WAKEUP__SHIFT 0x00000000
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D0_CMD_FIFO_READY_WAKEUP_MASK 0x00000100L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D0_CMD_FIFO_READY_WAKEUP__SHIFT 0x00000008
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D0_DATA_FIFO_READY_WAKEUP_MASK 0x00000400L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D0_DATA_FIFO_READY_WAKEUP__SHIFT 0x0000000a
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D0_IDLEH_WAKEUP_MASK 0x01000000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D0_IDLEH_WAKEUP__SHIFT 0x00000018
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D0_REDC_WAKEUP_MASK 0x00000004L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D0_REDC_WAKEUP__SHIFT 0x00000002
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D0_WEDC_WAKEUP_MASK 0x00000010L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D0_WEDC_WAKEUP__SHIFT 0x00000004
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D1_ARF_WAKEUP_MASK 0x00000002L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D1_ARF_WAKEUP__SHIFT 0x00000001
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D1_CMD_FIFO_READY_WAKEUP_MASK 0x00000200L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D1_CMD_FIFO_READY_WAKEUP__SHIFT 0x00000009
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D1_DATA_FIFO_READY_WAKEUP_MASK 0x00000800L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D1_DATA_FIFO_READY_WAKEUP__SHIFT 0x0000000b
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D1_IDLEH_WAKEUP_MASK 0x02000000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D1_IDLEH_WAKEUP__SHIFT 0x00000019
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D1_REDC_WAKEUP_MASK 0x00000008L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D1_REDC_WAKEUP__SHIFT 0x00000003
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D1_WEDC_WAKEUP_MASK 0x00000020L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__D1_WEDC_WAKEUP__SHIFT 0x00000005
+#define MC_SEQ_TRAIN_WAKEUP_MASK__DPM_LPT_WAKEUP_MASK 0x00800000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__DPM_LPT_WAKEUP__SHIFT 0x00000017
+#define MC_SEQ_TRAIN_WAKEUP_MASK__DPM_WAKEUP_MASK 0x00100000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__DPM_WAKEUP__SHIFT 0x00000014
+#define MC_SEQ_TRAIN_WAKEUP_MASK__MCLK_FREQ_CHANGE_WAKEUP_MASK 0x00000040L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__MCLK_FREQ_CHANGE_WAKEUP__SHIFT 0x00000006
+#define MC_SEQ_TRAIN_WAKEUP_MASK__PHY_PG_WAKEUP_MASK 0x04000000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__PHY_PG_WAKEUP__SHIFT 0x0000001a
+#define MC_SEQ_TRAIN_WAKEUP_MASK__RESERVE0_WAKEUP_MASK 0x00002000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__RESERVE0_WAKEUP__SHIFT 0x0000000d
+#define MC_SEQ_TRAIN_WAKEUP_MASK__SCLK_SRBM_READY_WAKEUP_MASK 0x00000080L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__SCLK_SRBM_READY_WAKEUP__SHIFT 0x00000007
+#define MC_SEQ_TRAIN_WAKEUP_MASK__SOFTWARE_WAKEUP_WAKEUP_MASK 0x00001000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__SOFTWARE_WAKEUP_WAKEUP__SHIFT 0x0000000c
+#define MC_SEQ_TRAIN_WAKEUP_MASK__TCG_DONE_WAKEUP_MASK 0x00020000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__TCG_DONE_WAKEUP__SHIFT 0x00000011
+#define MC_SEQ_TRAIN_WAKEUP_MASK__TIMER_DONE_WAKEUP_MASK 0x00008000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__TIMER_DONE_WAKEUP__SHIFT 0x0000000f
+#define MC_SEQ_TRAIN_WAKEUP_MASK__TSM_DONE_WAKEUP_MASK 0x00004000L
+#define MC_SEQ_TRAIN_WAKEUP_MASK__TSM_DONE_WAKEUP__SHIFT 0x0000000e
+#define MC_SEQ_TSM_BCNT__BCNT_TESTS_MASK 0x0000ff00L
+#define MC_SEQ_TSM_BCNT__BCNT_TESTS__SHIFT 0x00000008
+#define MC_SEQ_TSM_BCNT__COMP_VALUE_MASK 0x00ff0000L
+#define MC_SEQ_TSM_BCNT__COMP_VALUE__SHIFT 0x00000010
+#define MC_SEQ_TSM_BCNT__DONE_TESTS_MASK 0xff000000L
+#define MC_SEQ_TSM_BCNT__DONE_TESTS__SHIFT 0x00000018
+#define MC_SEQ_TSM_BCNT__FALSE_ACT_MASK 0x000000f0L
+#define MC_SEQ_TSM_BCNT__FALSE_ACT__SHIFT 0x00000004
+#define MC_SEQ_TSM_BCNT__TRUE_ACT_MASK 0x0000000fL
+#define MC_SEQ_TSM_BCNT__TRUE_ACT__SHIFT 0x00000000
+#define MC_SEQ_TSM_CTRL__CAPTURE_START_MASK 0x00000002L
+#define MC_SEQ_TSM_CTRL__CAPTURE_START__SHIFT 0x00000001
+#define MC_SEQ_TSM_CTRL__DIRECTION_MASK 0x00000020L
+#define MC_SEQ_TSM_CTRL__DIRECTION__SHIFT 0x00000005
+#define MC_SEQ_TSM_CTRL__DONE_MASK 0x00000004L
+#define MC_SEQ_TSM_CTRL__DONE__SHIFT 0x00000002
+#define MC_SEQ_TSM_CTRL__ERR_MASK 0x00000008L
+#define MC_SEQ_TSM_CTRL__ERR__SHIFT 0x00000003
+#define MC_SEQ_TSM_CTRL__INVERT_MASK 0x00000040L
+#define MC_SEQ_TSM_CTRL__INVERT__SHIFT 0x00000006
+#define MC_SEQ_TSM_CTRL__MASK_BITS_MASK 0x00000080L
+#define MC_SEQ_TSM_CTRL__MASK_BITS__SHIFT 0x00000007
+#define MC_SEQ_TSM_CTRL__POINTER_MASK 0xffff0000L
+#define MC_SEQ_TSM_CTRL__POINTER__SHIFT 0x00000010
+#define MC_SEQ_TSM_CTRL__ROT_INV_MASK 0x00000400L
+#define MC_SEQ_TSM_CTRL__ROT_INV__SHIFT 0x0000000a
+#define MC_SEQ_TSM_CTRL__START_MASK 0x00000001L
+#define MC_SEQ_TSM_CTRL__START__SHIFT 0x00000000
+#define MC_SEQ_TSM_CTRL__STEP_MASK 0x00000010L
+#define MC_SEQ_TSM_CTRL__STEP__SHIFT 0x00000004
+#define MC_SEQ_TSM_CTRL__UPDATE_LOOP_MASK 0x00000300L
+#define MC_SEQ_TSM_CTRL__UPDATE_LOOP__SHIFT 0x00000008
+#define MC_SEQ_TSM_DBI__DBI_MASK 0xffffffffL
+#define MC_SEQ_TSM_DBI__DBI__SHIFT 0x00000000
+#define MC_SEQ_TSM_DEBUG_DATA__TSM_DEBUG_DATA_MASK 0xffffffffL
+#define MC_SEQ_TSM_DEBUG_DATA__TSM_DEBUG_DATA__SHIFT 0x00000000
+#define MC_SEQ_TSM_DEBUG_INDEX__TSM_DEBUG_INDEX_MASK 0x0000001fL
+#define MC_SEQ_TSM_DEBUG_INDEX__TSM_DEBUG_INDEX__SHIFT 0x00000000
+#define MC_SEQ_TSM_EDC__EDC_MASK 0xffffffffL
+#define MC_SEQ_TSM_EDC__EDC__SHIFT 0x00000000
+#define MC_SEQ_TSM_FLAG__ERROR_TESTS_MASK 0xff000000L
+#define MC_SEQ_TSM_FLAG__ERROR_TESTS__SHIFT 0x00000018
+#define MC_SEQ_TSM_FLAG__FALSE_ACT_MASK 0x000000f0L
+#define MC_SEQ_TSM_FLAG__FALSE_ACT__SHIFT 0x00000004
+#define MC_SEQ_TSM_FLAG__FLAG_TESTS_MASK 0x0000ff00L
+#define MC_SEQ_TSM_FLAG__FLAG_TESTS__SHIFT 0x00000008
+#define MC_SEQ_TSM_FLAG__NBBL_MASK_MASK 0x000f0000L
+#define MC_SEQ_TSM_FLAG__NBBL_MASK__SHIFT 0x00000010
+#define MC_SEQ_TSM_FLAG__TRUE_ACT_MASK 0x0000000fL
+#define MC_SEQ_TSM_FLAG__TRUE_ACT__SHIFT 0x00000000
+#define MC_SEQ_TSM_GCNT__COMP_VALUE_MASK 0xffff0000L
+#define MC_SEQ_TSM_GCNT__COMP_VALUE__SHIFT 0x00000010
+#define MC_SEQ_TSM_GCNT__FALSE_ACT_MASK 0x000000f0L
+#define MC_SEQ_TSM_GCNT__FALSE_ACT__SHIFT 0x00000004
+#define MC_SEQ_TSM_GCNT__TESTS_MASK 0x0000ff00L
+#define MC_SEQ_TSM_GCNT__TESTS__SHIFT 0x00000008
+#define MC_SEQ_TSM_GCNT__TRUE_ACT_MASK 0x0000000fL
+#define MC_SEQ_TSM_GCNT__TRUE_ACT__SHIFT 0x00000000
+#define MC_SEQ_TSM_MISC__WCDR_MASK_MASK 0x000f0000L
+#define MC_SEQ_TSM_MISC__WCDR_MASK__SHIFT 0x00000010
+#define MC_SEQ_TSM_MISC__WCDR_PTR_MASK 0x0000ffffL
+#define MC_SEQ_TSM_MISC__WCDR_PTR__SHIFT 0x00000000
+#define MC_SEQ_TSM_NCNT__FALSE_ACT_MASK 0x000000f0L
+#define MC_SEQ_TSM_NCNT__FALSE_ACT__SHIFT 0x00000004
+#define MC_SEQ_TSM_NCNT__NIBBLE_SKIP_MASK 0x0f000000L
+#define MC_SEQ_TSM_NCNT__NIBBLE_SKIP__SHIFT 0x00000018
+#define MC_SEQ_TSM_NCNT__RANGE_HIGH_MASK 0x00f00000L
+#define MC_SEQ_TSM_NCNT__RANGE_HIGH__SHIFT 0x00000014
+#define MC_SEQ_TSM_NCNT__RANGE_LOW_MASK 0x000f0000L
+#define MC_SEQ_TSM_NCNT__RANGE_LOW__SHIFT 0x00000010
+#define MC_SEQ_TSM_NCNT__TESTS_MASK 0x0000ff00L
+#define MC_SEQ_TSM_NCNT__TESTS__SHIFT 0x00000008
+#define MC_SEQ_TSM_NCNT__TRUE_ACT_MASK 0x0000000fL
+#define MC_SEQ_TSM_NCNT__TRUE_ACT__SHIFT 0x00000000
+#define MC_SEQ_TSM_OCNT__CMP_VALUE_MASK 0xffff0000L
+#define MC_SEQ_TSM_OCNT__CMP_VALUE__SHIFT 0x00000010
+#define MC_SEQ_TSM_OCNT__FALSE_ACT_MASK 0x000000f0L
+#define MC_SEQ_TSM_OCNT__FALSE_ACT__SHIFT 0x00000004
+#define MC_SEQ_TSM_OCNT__TESTS_MASK 0x0000ff00L
+#define MC_SEQ_TSM_OCNT__TESTS__SHIFT 0x00000008
+#define MC_SEQ_TSM_OCNT__TRUE_ACT_MASK 0x0000000fL
+#define MC_SEQ_TSM_OCNT__TRUE_ACT__SHIFT 0x00000000
+#define MC_SEQ_TSM_UPDATE__AREF_COUNT_MASK 0x00ff0000L
+#define MC_SEQ_TSM_UPDATE__AREF_COUNT__SHIFT 0x00000010
+#define MC_SEQ_TSM_UPDATE__CAPTR_TESTS_MASK 0xff000000L
+#define MC_SEQ_TSM_UPDATE__CAPTR_TESTS__SHIFT 0x00000018
+#define MC_SEQ_TSM_UPDATE__FALSE_ACT_MASK 0x000000f0L
+#define MC_SEQ_TSM_UPDATE__FALSE_ACT__SHIFT 0x00000004
+#define MC_SEQ_TSM_UPDATE__TRUE_ACT_MASK 0x0000000fL
+#define MC_SEQ_TSM_UPDATE__TRUE_ACT__SHIFT 0x00000000
+#define MC_SEQ_TSM_UPDATE__UPDT_TESTS_MASK 0x0000ff00L
+#define MC_SEQ_TSM_UPDATE__UPDT_TESTS__SHIFT 0x00000008
+#define MC_SEQ_TSM_WCDR__WCDR_MASK 0xffffffffL
+#define MC_SEQ_TSM_WCDR__WCDR__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ4_MASK 0x000f0000L
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ4__SHIFT 0x00000010
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ5_MASK 0x00f00000L
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ5__SHIFT 0x00000014
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ6_MASK 0x0f000000L
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ6__SHIFT 0x00000018
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ7_MASK 0xf0000000L
+#define MC_SEQ_TXFRAMING_BYTE0_D0__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ4_MASK 0x000f0000L
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ4__SHIFT 0x00000010
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ5_MASK 0x00f00000L
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ5__SHIFT 0x00000014
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ6_MASK 0x0f000000L
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ6__SHIFT 0x00000018
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ7_MASK 0xf0000000L
+#define MC_SEQ_TXFRAMING_BYTE0_D1__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ4_MASK 0x000f0000L
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ4__SHIFT 0x00000010
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ5_MASK 0x00f00000L
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ5__SHIFT 0x00000014
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ6_MASK 0x0f000000L
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ6__SHIFT 0x00000018
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ7_MASK 0xf0000000L
+#define MC_SEQ_TXFRAMING_BYTE1_D0__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ4_MASK 0x000f0000L
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ4__SHIFT 0x00000010
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ5_MASK 0x00f00000L
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ5__SHIFT 0x00000014
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ6_MASK 0x0f000000L
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ6__SHIFT 0x00000018
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ7_MASK 0xf0000000L
+#define MC_SEQ_TXFRAMING_BYTE1_D1__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ4_MASK 0x000f0000L
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ4__SHIFT 0x00000010
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ5_MASK 0x00f00000L
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ5__SHIFT 0x00000014
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ6_MASK 0x0f000000L
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ6__SHIFT 0x00000018
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ7_MASK 0xf0000000L
+#define MC_SEQ_TXFRAMING_BYTE2_D0__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ4_MASK 0x000f0000L
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ4__SHIFT 0x00000010
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ5_MASK 0x00f00000L
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ5__SHIFT 0x00000014
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ6_MASK 0x0f000000L
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ6__SHIFT 0x00000018
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ7_MASK 0xf0000000L
+#define MC_SEQ_TXFRAMING_BYTE2_D1__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ4_MASK 0x000f0000L
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ4__SHIFT 0x00000010
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ5_MASK 0x00f00000L
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ5__SHIFT 0x00000014
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ6_MASK 0x0f000000L
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ6__SHIFT 0x00000018
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ7_MASK 0xf0000000L
+#define MC_SEQ_TXFRAMING_BYTE3_D0__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ4_MASK 0x000f0000L
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ4__SHIFT 0x00000010
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ5_MASK 0x00f00000L
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ5__SHIFT 0x00000014
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ6_MASK 0x0f000000L
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ6__SHIFT 0x00000018
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ7_MASK 0xf0000000L
+#define MC_SEQ_TXFRAMING_BYTE3_D1__DQ7__SHIFT 0x0000001c
+#define MC_SEQ_TXFRAMING_DBI_D0__DBI0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_DBI_D0__DBI0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_DBI_D0__DBI1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_DBI_D0__DBI1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_DBI_D0__DBI2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_DBI_D0__DBI2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_DBI_D0__DBI3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_DBI_D0__DBI3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_DBI_D1__DBI0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_DBI_D1__DBI0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_DBI_D1__DBI1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_DBI_D1__DBI1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_DBI_D1__DBI2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_DBI_D1__DBI2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_DBI_D1__DBI3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_DBI_D1__DBI3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_EDC_D0__EDC0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_EDC_D0__EDC0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_EDC_D0__EDC1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_EDC_D0__EDC1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_EDC_D0__EDC2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_EDC_D0__EDC2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_EDC_D0__EDC3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_EDC_D0__EDC3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_EDC_D0__WCDR0_MASK 0x000f0000L
+#define MC_SEQ_TXFRAMING_EDC_D0__WCDR0__SHIFT 0x00000010
+#define MC_SEQ_TXFRAMING_EDC_D0__WCDR1_MASK 0x00f00000L
+#define MC_SEQ_TXFRAMING_EDC_D0__WCDR1__SHIFT 0x00000014
+#define MC_SEQ_TXFRAMING_EDC_D0__WCDR2_MASK 0x0f000000L
+#define MC_SEQ_TXFRAMING_EDC_D0__WCDR2__SHIFT 0x00000018
+#define MC_SEQ_TXFRAMING_EDC_D0__WCDR3_MASK 0xf0000000L
+#define MC_SEQ_TXFRAMING_EDC_D0__WCDR3__SHIFT 0x0000001c
+#define MC_SEQ_TXFRAMING_EDC_D1__EDC0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_EDC_D1__EDC0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_EDC_D1__EDC1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_EDC_D1__EDC1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_EDC_D1__EDC2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_EDC_D1__EDC2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_EDC_D1__EDC3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_EDC_D1__EDC3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_EDC_D1__WCDR0_MASK 0x000f0000L
+#define MC_SEQ_TXFRAMING_EDC_D1__WCDR0__SHIFT 0x00000010
+#define MC_SEQ_TXFRAMING_EDC_D1__WCDR1_MASK 0x00f00000L
+#define MC_SEQ_TXFRAMING_EDC_D1__WCDR1__SHIFT 0x00000014
+#define MC_SEQ_TXFRAMING_EDC_D1__WCDR2_MASK 0x0f000000L
+#define MC_SEQ_TXFRAMING_EDC_D1__WCDR2__SHIFT 0x00000018
+#define MC_SEQ_TXFRAMING_EDC_D1__WCDR3_MASK 0xf0000000L
+#define MC_SEQ_TXFRAMING_EDC_D1__WCDR3__SHIFT 0x0000001c
+#define MC_SEQ_TXFRAMING_FCK_D0__FCK0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_FCK_D0__FCK0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_FCK_D0__FCK1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_FCK_D0__FCK1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_FCK_D0__FCK2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_FCK_D0__FCK2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_FCK_D0__FCK3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_FCK_D0__FCK3__SHIFT 0x0000000c
+#define MC_SEQ_TXFRAMING_FCK_D1__FCK0_MASK 0x0000000fL
+#define MC_SEQ_TXFRAMING_FCK_D1__FCK0__SHIFT 0x00000000
+#define MC_SEQ_TXFRAMING_FCK_D1__FCK1_MASK 0x000000f0L
+#define MC_SEQ_TXFRAMING_FCK_D1__FCK1__SHIFT 0x00000004
+#define MC_SEQ_TXFRAMING_FCK_D1__FCK2_MASK 0x00000f00L
+#define MC_SEQ_TXFRAMING_FCK_D1__FCK2__SHIFT 0x00000008
+#define MC_SEQ_TXFRAMING_FCK_D1__FCK3_MASK 0x0000f000L
+#define MC_SEQ_TXFRAMING_FCK_D1__FCK3__SHIFT 0x0000000c
+#define MC_SEQ_VENDOR_ID_I0__VALUE_MASK 0xffffffffL
+#define MC_SEQ_VENDOR_ID_I0__VALUE__SHIFT 0x00000000
+#define MC_SEQ_VENDOR_ID_I1__VALUE_MASK 0xffffffffL
+#define MC_SEQ_VENDOR_ID_I1__VALUE__SHIFT 0x00000000
+#define MC_SEQ_WCDR_CTRL__AREF_EN_MASK 0x00004000L
+#define MC_SEQ_WCDR_CTRL__AREF_EN__SHIFT 0x0000000e
+#define MC_SEQ_WCDR_CTRL__PRBS_EN_MASK 0x00100000L
+#define MC_SEQ_WCDR_CTRL__PRBS_EN__SHIFT 0x00000014
+#define MC_SEQ_WCDR_CTRL__PRBS_RST_MASK 0x00200000L
+#define MC_SEQ_WCDR_CTRL__PRBS_RST__SHIFT 0x00000015
+#define MC_SEQ_WCDR_CTRL__PREAMBLE_MASK 0x0f000000L
+#define MC_SEQ_WCDR_CTRL__PREAMBLE__SHIFT 0x00000018
+#define MC_SEQ_WCDR_CTRL__PRE_MASK_MASK 0xf0000000L
+#define MC_SEQ_WCDR_CTRL__PRE_MASK__SHIFT 0x0000001c
+#define MC_SEQ_WCDR_CTRL__RD_EN_MASK 0x00002000L
+#define MC_SEQ_WCDR_CTRL__RD_EN__SHIFT 0x0000000d
+#define MC_SEQ_WCDR_CTRL__TRAIN_EN_MASK 0x00008000L
+#define MC_SEQ_WCDR_CTRL__TRAIN_EN__SHIFT 0x0000000f
+#define MC_SEQ_WCDR_CTRL__TWCDRL_MASK 0x000f0000L
+#define MC_SEQ_WCDR_CTRL__TWCDRL__SHIFT 0x00000010
+#define MC_SEQ_WCDR_CTRL__WCDR_PRE_MASK 0x000000ffL
+#define MC_SEQ_WCDR_CTRL__WCDR_PRE__SHIFT 0x00000000
+#define MC_SEQ_WCDR_CTRL__WCDR_TIM_MASK 0x00000f00L
+#define MC_SEQ_WCDR_CTRL__WCDR_TIM__SHIFT 0x00000008
+#define MC_SEQ_WCDR_CTRL__WR_EN_MASK 0x00001000L
+#define MC_SEQ_WCDR_CTRL__WR_EN__SHIFT 0x0000000c
+#define MC_SEQ_WR_CTL_2__DAT_DLY_H_D0_MASK 0x00000001L
+#define MC_SEQ_WR_CTL_2__DAT_DLY_H_D0__SHIFT 0x00000000
+#define MC_SEQ_WR_CTL_2__DAT_DLY_H_D1_MASK 0x00000008L
+#define MC_SEQ_WR_CTL_2__DAT_DLY_H_D1__SHIFT 0x00000003
+#define MC_SEQ_WR_CTL_2__DQS_DLY_H_D0_MASK 0x00000002L
+#define MC_SEQ_WR_CTL_2__DQS_DLY_H_D0__SHIFT 0x00000001
+#define MC_SEQ_WR_CTL_2__DQS_DLY_H_D1_MASK 0x00000010L
+#define MC_SEQ_WR_CTL_2__DQS_DLY_H_D1__SHIFT 0x00000004
+#define MC_SEQ_WR_CTL_2_LP__DAT_DLY_H_D0_MASK 0x00000001L
+#define MC_SEQ_WR_CTL_2_LP__DAT_DLY_H_D0__SHIFT 0x00000000
+#define MC_SEQ_WR_CTL_2_LP__DAT_DLY_H_D1_MASK 0x00000008L
+#define MC_SEQ_WR_CTL_2_LP__DAT_DLY_H_D1__SHIFT 0x00000003
+#define MC_SEQ_WR_CTL_2_LP__DQS_DLY_H_D0_MASK 0x00000002L
+#define MC_SEQ_WR_CTL_2_LP__DQS_DLY_H_D0__SHIFT 0x00000001
+#define MC_SEQ_WR_CTL_2_LP__DQS_DLY_H_D1_MASK 0x00000010L
+#define MC_SEQ_WR_CTL_2_LP__DQS_DLY_H_D1__SHIFT 0x00000004
+#define MC_SEQ_WR_CTL_2_LP__OEN_DLY_H_D0_MASK 0x00000004L
+#define MC_SEQ_WR_CTL_2_LP__OEN_DLY_H_D0__SHIFT 0x00000002
+#define MC_SEQ_WR_CTL_2_LP__OEN_DLY_H_D1_MASK 0x00000020L
+#define MC_SEQ_WR_CTL_2_LP__OEN_DLY_H_D1__SHIFT 0x00000005
+#define MC_SEQ_WR_CTL_2_LP__WCDR_EN_MASK 0x00000040L
+#define MC_SEQ_WR_CTL_2_LP__WCDR_EN__SHIFT 0x00000006
+#define MC_SEQ_WR_CTL_2__OEN_DLY_H_D0_MASK 0x00000004L
+#define MC_SEQ_WR_CTL_2__OEN_DLY_H_D0__SHIFT 0x00000002
+#define MC_SEQ_WR_CTL_2__OEN_DLY_H_D1_MASK 0x00000020L
+#define MC_SEQ_WR_CTL_2__OEN_DLY_H_D1__SHIFT 0x00000005
+#define MC_SEQ_WR_CTL_2__WCDR_EN_MASK 0x00000040L
+#define MC_SEQ_WR_CTL_2__WCDR_EN__SHIFT 0x00000006
+#define MC_SEQ_WR_CTL_D0__ADR_2Y_DLY_MASK 0x00000400L
+#define MC_SEQ_WR_CTL_D0__ADR_2Y_DLY__SHIFT 0x0000000a
+#define MC_SEQ_WR_CTL_D0__ADR_DLY_MASK 0x20000000L
+#define MC_SEQ_WR_CTL_D0__ADR_DLY__SHIFT 0x0000001d
+#define MC_SEQ_WR_CTL_D0__CMD_2Y_DLY_MASK 0x00000800L
+#define MC_SEQ_WR_CTL_D0__CMD_2Y_DLY__SHIFT 0x0000000b
+#define MC_SEQ_WR_CTL_D0__CMD_DLY_MASK 0x40000000L
+#define MC_SEQ_WR_CTL_D0__CMD_DLY__SHIFT 0x0000001e
+#define MC_SEQ_WR_CTL_D0__DAT_2Y_DLY_MASK 0x00000200L
+#define MC_SEQ_WR_CTL_D0__DAT_2Y_DLY__SHIFT 0x00000009
+#define MC_SEQ_WR_CTL_D0__DAT_DLY_MASK 0x0000000fL
+#define MC_SEQ_WR_CTL_D0__DAT_DLY__SHIFT 0x00000000
+#define MC_SEQ_WR_CTL_D0__DQS_DLY_MASK 0x000000f0L
+#define MC_SEQ_WR_CTL_D0__DQS_DLY__SHIFT 0x00000004
+#define MC_SEQ_WR_CTL_D0__DQS_XTR_MASK 0x00000100L
+#define MC_SEQ_WR_CTL_D0__DQS_XTR__SHIFT 0x00000008
+#define MC_SEQ_WR_CTL_D0_LP__ADR_2Y_DLY_MASK 0x00000400L
+#define MC_SEQ_WR_CTL_D0_LP__ADR_2Y_DLY__SHIFT 0x0000000a
+#define MC_SEQ_WR_CTL_D0_LP__ADR_DLY_MASK 0x20000000L
+#define MC_SEQ_WR_CTL_D0_LP__ADR_DLY__SHIFT 0x0000001d
+#define MC_SEQ_WR_CTL_D0_LP__CMD_2Y_DLY_MASK 0x00000800L
+#define MC_SEQ_WR_CTL_D0_LP__CMD_2Y_DLY__SHIFT 0x0000000b
+#define MC_SEQ_WR_CTL_D0_LP__CMD_DLY_MASK 0x40000000L
+#define MC_SEQ_WR_CTL_D0_LP__CMD_DLY__SHIFT 0x0000001e
+#define MC_SEQ_WR_CTL_D0_LP__DAT_2Y_DLY_MASK 0x00000200L
+#define MC_SEQ_WR_CTL_D0_LP__DAT_2Y_DLY__SHIFT 0x00000009
+#define MC_SEQ_WR_CTL_D0_LP__DAT_DLY_MASK 0x0000000fL
+#define MC_SEQ_WR_CTL_D0_LP__DAT_DLY__SHIFT 0x00000000
+#define MC_SEQ_WR_CTL_D0_LP__DQS_DLY_MASK 0x000000f0L
+#define MC_SEQ_WR_CTL_D0_LP__DQS_DLY__SHIFT 0x00000004
+#define MC_SEQ_WR_CTL_D0_LP__DQS_XTR_MASK 0x00000100L
+#define MC_SEQ_WR_CTL_D0_LP__DQS_XTR__SHIFT 0x00000008
+#define MC_SEQ_WR_CTL_D0_LP__ODT_DLY_MASK 0x0f000000L
+#define MC_SEQ_WR_CTL_D0_LP__ODT_DLY__SHIFT 0x00000018
+#define MC_SEQ_WR_CTL_D0_LP__ODT_EXT_MASK 0x10000000L
+#define MC_SEQ_WR_CTL_D0_LP__ODT_EXT__SHIFT 0x0000001c
+#define MC_SEQ_WR_CTL_D0_LP__OEN_DLY_MASK 0x0000f000L
+#define MC_SEQ_WR_CTL_D0_LP__OEN_DLY__SHIFT 0x0000000c
+#define MC_SEQ_WR_CTL_D0_LP__OEN_EXT_MASK 0x000f0000L
+#define MC_SEQ_WR_CTL_D0_LP__OEN_EXT__SHIFT 0x00000010
+#define MC_SEQ_WR_CTL_D0_LP__OEN_SEL_MASK 0x00300000L
+#define MC_SEQ_WR_CTL_D0_LP__OEN_SEL__SHIFT 0x00000014
+#define MC_SEQ_WR_CTL_D0__ODT_DLY_MASK 0x0f000000L
+#define MC_SEQ_WR_CTL_D0__ODT_DLY__SHIFT 0x00000018
+#define MC_SEQ_WR_CTL_D0__ODT_EXT_MASK 0x10000000L
+#define MC_SEQ_WR_CTL_D0__ODT_EXT__SHIFT 0x0000001c
+#define MC_SEQ_WR_CTL_D0__OEN_DLY_MASK 0x0000f000L
+#define MC_SEQ_WR_CTL_D0__OEN_DLY__SHIFT 0x0000000c
+#define MC_SEQ_WR_CTL_D0__OEN_EXT_MASK 0x000f0000L
+#define MC_SEQ_WR_CTL_D0__OEN_EXT__SHIFT 0x00000010
+#define MC_SEQ_WR_CTL_D0__OEN_SEL_MASK 0x00300000L
+#define MC_SEQ_WR_CTL_D0__OEN_SEL__SHIFT 0x00000014
+#define MC_SEQ_WR_CTL_D1__ADR_2Y_DLY_MASK 0x00000400L
+#define MC_SEQ_WR_CTL_D1__ADR_2Y_DLY__SHIFT 0x0000000a
+#define MC_SEQ_WR_CTL_D1__ADR_DLY_MASK 0x20000000L
+#define MC_SEQ_WR_CTL_D1__ADR_DLY__SHIFT 0x0000001d
+#define MC_SEQ_WR_CTL_D1__CMD_2Y_DLY_MASK 0x00000800L
+#define MC_SEQ_WR_CTL_D1__CMD_2Y_DLY__SHIFT 0x0000000b
+#define MC_SEQ_WR_CTL_D1__CMD_DLY_MASK 0x40000000L
+#define MC_SEQ_WR_CTL_D1__CMD_DLY__SHIFT 0x0000001e
+#define MC_SEQ_WR_CTL_D1__DAT_2Y_DLY_MASK 0x00000200L
+#define MC_SEQ_WR_CTL_D1__DAT_2Y_DLY__SHIFT 0x00000009
+#define MC_SEQ_WR_CTL_D1__DAT_DLY_MASK 0x0000000fL
+#define MC_SEQ_WR_CTL_D1__DAT_DLY__SHIFT 0x00000000
+#define MC_SEQ_WR_CTL_D1__DQS_DLY_MASK 0x000000f0L
+#define MC_SEQ_WR_CTL_D1__DQS_DLY__SHIFT 0x00000004
+#define MC_SEQ_WR_CTL_D1__DQS_XTR_MASK 0x00000100L
+#define MC_SEQ_WR_CTL_D1__DQS_XTR__SHIFT 0x00000008
+#define MC_SEQ_WR_CTL_D1_LP__ADR_2Y_DLY_MASK 0x00000400L
+#define MC_SEQ_WR_CTL_D1_LP__ADR_2Y_DLY__SHIFT 0x0000000a
+#define MC_SEQ_WR_CTL_D1_LP__ADR_DLY_MASK 0x20000000L
+#define MC_SEQ_WR_CTL_D1_LP__ADR_DLY__SHIFT 0x0000001d
+#define MC_SEQ_WR_CTL_D1_LP__CMD_2Y_DLY_MASK 0x00000800L
+#define MC_SEQ_WR_CTL_D1_LP__CMD_2Y_DLY__SHIFT 0x0000000b
+#define MC_SEQ_WR_CTL_D1_LP__CMD_DLY_MASK 0x40000000L
+#define MC_SEQ_WR_CTL_D1_LP__CMD_DLY__SHIFT 0x0000001e
+#define MC_SEQ_WR_CTL_D1_LP__DAT_2Y_DLY_MASK 0x00000200L
+#define MC_SEQ_WR_CTL_D1_LP__DAT_2Y_DLY__SHIFT 0x00000009
+#define MC_SEQ_WR_CTL_D1_LP__DAT_DLY_MASK 0x0000000fL
+#define MC_SEQ_WR_CTL_D1_LP__DAT_DLY__SHIFT 0x00000000
+#define MC_SEQ_WR_CTL_D1_LP__DQS_DLY_MASK 0x000000f0L
+#define MC_SEQ_WR_CTL_D1_LP__DQS_DLY__SHIFT 0x00000004
+#define MC_SEQ_WR_CTL_D1_LP__DQS_XTR_MASK 0x00000100L
+#define MC_SEQ_WR_CTL_D1_LP__DQS_XTR__SHIFT 0x00000008
+#define MC_SEQ_WR_CTL_D1_LP__ODT_DLY_MASK 0x0f000000L
+#define MC_SEQ_WR_CTL_D1_LP__ODT_DLY__SHIFT 0x00000018
+#define MC_SEQ_WR_CTL_D1_LP__ODT_EXT_MASK 0x10000000L
+#define MC_SEQ_WR_CTL_D1_LP__ODT_EXT__SHIFT 0x0000001c
+#define MC_SEQ_WR_CTL_D1_LP__OEN_DLY_MASK 0x0000f000L
+#define MC_SEQ_WR_CTL_D1_LP__OEN_DLY__SHIFT 0x0000000c
+#define MC_SEQ_WR_CTL_D1_LP__OEN_EXT_MASK 0x000f0000L
+#define MC_SEQ_WR_CTL_D1_LP__OEN_EXT__SHIFT 0x00000010
+#define MC_SEQ_WR_CTL_D1_LP__OEN_SEL_MASK 0x00300000L
+#define MC_SEQ_WR_CTL_D1_LP__OEN_SEL__SHIFT 0x00000014
+#define MC_SEQ_WR_CTL_D1__ODT_DLY_MASK 0x0f000000L
+#define MC_SEQ_WR_CTL_D1__ODT_DLY__SHIFT 0x00000018
+#define MC_SEQ_WR_CTL_D1__ODT_EXT_MASK 0x10000000L
+#define MC_SEQ_WR_CTL_D1__ODT_EXT__SHIFT 0x0000001c
+#define MC_SEQ_WR_CTL_D1__OEN_DLY_MASK 0x0000f000L
+#define MC_SEQ_WR_CTL_D1__OEN_DLY__SHIFT 0x0000000c
+#define MC_SEQ_WR_CTL_D1__OEN_EXT_MASK 0x000f0000L
+#define MC_SEQ_WR_CTL_D1__OEN_EXT__SHIFT 0x00000010
+#define MC_SEQ_WR_CTL_D1__OEN_SEL_MASK 0x00300000L
+#define MC_SEQ_WR_CTL_D1__OEN_SEL__SHIFT 0x00000014
+#define MC_SHARED_BLACKOUT_CNTL__BLACKOUT_MODE_MASK 0x00000007L
+#define MC_SHARED_BLACKOUT_CNTL__BLACKOUT_MODE__SHIFT 0x00000000
+#define MC_SHARED_CHMAP__CHAN0_MASK 0x0000000fL
+#define MC_SHARED_CHMAP__CHAN0__SHIFT 0x00000000
+#define MC_SHARED_CHMAP__CHAN1_MASK 0x000000f0L
+#define MC_SHARED_CHMAP__CHAN1__SHIFT 0x00000004
+#define MC_SHARED_CHMAP__CHAN2_MASK 0x00000f00L
+#define MC_SHARED_CHMAP__CHAN2__SHIFT 0x00000008
+#define MC_SHARED_CHMAP__NOOFCHAN_MASK 0x0000f000L
+#define MC_SHARED_CHMAP__NOOFCHAN__SHIFT 0x0000000c
+#define MC_SHARED_CHREMAP__CHAN0_MASK 0x00000007L
+#define MC_SHARED_CHREMAP__CHAN0__SHIFT 0x00000000
+#define MC_SHARED_CHREMAP__CHAN1_MASK 0x00000038L
+#define MC_SHARED_CHREMAP__CHAN1__SHIFT 0x00000003
+#define MC_SHARED_CHREMAP__CHAN2_MASK 0x000001c0L
+#define MC_SHARED_CHREMAP__CHAN2__SHIFT 0x00000006
+#define MC_SHARED_CHREMAP__CHAN3_MASK 0x00000e00L
+#define MC_SHARED_CHREMAP__CHAN3__SHIFT 0x00000009
+#define MC_SHARED_CHREMAP__CHAN4_MASK 0x00007000L
+#define MC_SHARED_CHREMAP__CHAN4__SHIFT 0x0000000c
+#define MC_SHARED_CHREMAP__CHAN5_MASK 0x00038000L
+#define MC_SHARED_CHREMAP__CHAN5__SHIFT 0x0000000f
+#define MC_SHARED_CHREMAP__CHAN6_MASK 0x001c0000L
+#define MC_SHARED_CHREMAP__CHAN6__SHIFT 0x00000012
+#define MC_SHARED_CHREMAP__CHAN7_MASK 0x00e00000L
+#define MC_SHARED_CHREMAP__CHAN7__SHIFT 0x00000015
+#define MC_TRAIN_EDCCDR_R_D0__EDC0_MASK 0x000000ffL
+#define MC_TRAIN_EDCCDR_R_D0__EDC0__SHIFT 0x00000000
+#define MC_TRAIN_EDCCDR_R_D0__EDC1_MASK 0x0000ff00L
+#define MC_TRAIN_EDCCDR_R_D0__EDC1__SHIFT 0x00000008
+#define MC_TRAIN_EDCCDR_R_D0__EDC2_MASK 0x00ff0000L
+#define MC_TRAIN_EDCCDR_R_D0__EDC2__SHIFT 0x00000010
+#define MC_TRAIN_EDCCDR_R_D0__EDC3_MASK 0xff000000L
+#define MC_TRAIN_EDCCDR_R_D0__EDC3__SHIFT 0x00000018
+#define MC_TRAIN_EDCCDR_R_D1__EDC0_MASK 0x000000ffL
+#define MC_TRAIN_EDCCDR_R_D1__EDC0__SHIFT 0x00000000
+#define MC_TRAIN_EDCCDR_R_D1__EDC1_MASK 0x0000ff00L
+#define MC_TRAIN_EDCCDR_R_D1__EDC1__SHIFT 0x00000008
+#define MC_TRAIN_EDCCDR_R_D1__EDC2_MASK 0x00ff0000L
+#define MC_TRAIN_EDCCDR_R_D1__EDC2__SHIFT 0x00000010
+#define MC_TRAIN_EDCCDR_R_D1__EDC3_MASK 0xff000000L
+#define MC_TRAIN_EDCCDR_R_D1__EDC3__SHIFT 0x00000018
+#define MC_TRAIN_EDC_STATUS_D0__REDC_CNT_MASK 0xffff0000L
+#define MC_TRAIN_EDC_STATUS_D0__REDC_CNT__SHIFT 0x00000010
+#define MC_TRAIN_EDC_STATUS_D0__WEDC_CNT_MASK 0x0000ffffL
+#define MC_TRAIN_EDC_STATUS_D0__WEDC_CNT__SHIFT 0x00000000
+#define MC_TRAIN_EDC_STATUS_D1__REDC_CNT_MASK 0xffff0000L
+#define MC_TRAIN_EDC_STATUS_D1__REDC_CNT__SHIFT 0x00000010
+#define MC_TRAIN_EDC_STATUS_D1__WEDC_CNT_MASK 0x0000ffffL
+#define MC_TRAIN_EDC_STATUS_D1__WEDC_CNT__SHIFT 0x00000000
+#define MC_TRAIN_PRBSERR_0_D0__DQ_STATUS_MASK 0xffffffffL
+#define MC_TRAIN_PRBSERR_0_D0__DQ_STATUS__SHIFT 0x00000000
+#define MC_TRAIN_PRBSERR_0_D1__DQ_STATUS_MASK 0xffffffffL
+#define MC_TRAIN_PRBSERR_0_D1__DQ_STATUS__SHIFT 0x00000000
+#define MC_TRAIN_PRBSERR_1_D0__DBI_STATUS_MASK 0x0000000fL
+#define MC_TRAIN_PRBSERR_1_D0__DBI_STATUS__SHIFT 0x00000000
+#define MC_TRAIN_PRBSERR_1_D0__EDC_STATUS_MASK 0x000000f0L
+#define MC_TRAIN_PRBSERR_1_D0__EDC_STATUS__SHIFT 0x00000004
+#define MC_TRAIN_PRBSERR_1_D0__PMA_PRBSCLR_MASK 0x10000000L
+#define MC_TRAIN_PRBSERR_1_D0__PMA_PRBSCLR__SHIFT 0x0000001c
+#define MC_TRAIN_PRBSERR_1_D0__PMD0_PRBSCLR_MASK 0x20000000L
+#define MC_TRAIN_PRBSERR_1_D0__PMD0_PRBSCLR__SHIFT 0x0000001d
+#define MC_TRAIN_PRBSERR_1_D0__PMD1_PRBSCLR_MASK 0x40000000L
+#define MC_TRAIN_PRBSERR_1_D0__PMD1_PRBSCLR__SHIFT 0x0000001e
+#define MC_TRAIN_PRBSERR_1_D0__WCDR_STATUS_MASK 0x0000f000L
+#define MC_TRAIN_PRBSERR_1_D0__WCDR_STATUS__SHIFT 0x0000000c
+#define MC_TRAIN_PRBSERR_1_D0__WCK_STATUS_MASK 0x00000f00L
+#define MC_TRAIN_PRBSERR_1_D0__WCK_STATUS__SHIFT 0x00000008
+#define MC_TRAIN_PRBSERR_1_D1__DBI_STATUS_MASK 0x0000000fL
+#define MC_TRAIN_PRBSERR_1_D1__DBI_STATUS__SHIFT 0x00000000
+#define MC_TRAIN_PRBSERR_1_D1__EDC_STATUS_MASK 0x000000f0L
+#define MC_TRAIN_PRBSERR_1_D1__EDC_STATUS__SHIFT 0x00000004
+#define MC_TRAIN_PRBSERR_1_D1__PMA_PRBSCLR_MASK 0x10000000L
+#define MC_TRAIN_PRBSERR_1_D1__PMA_PRBSCLR__SHIFT 0x0000001c
+#define MC_TRAIN_PRBSERR_1_D1__PMD0_PRBSCLR_MASK 0x20000000L
+#define MC_TRAIN_PRBSERR_1_D1__PMD0_PRBSCLR__SHIFT 0x0000001d
+#define MC_TRAIN_PRBSERR_1_D1__PMD1_PRBSCLR_MASK 0x40000000L
+#define MC_TRAIN_PRBSERR_1_D1__PMD1_PRBSCLR__SHIFT 0x0000001e
+#define MC_TRAIN_PRBSERR_1_D1__WCDR_STATUS_MASK 0x0000f000L
+#define MC_TRAIN_PRBSERR_1_D1__WCDR_STATUS__SHIFT 0x0000000c
+#define MC_TRAIN_PRBSERR_1_D1__WCK_STATUS_MASK 0x00000f00L
+#define MC_TRAIN_PRBSERR_1_D1__WCK_STATUS__SHIFT 0x00000008
+#define MC_TRAIN_PRBSERR_2_D0__ABI_STATUS_MASK 0x10000000L
+#define MC_TRAIN_PRBSERR_2_D0__ABI_STATUS__SHIFT 0x0000001c
+#define MC_TRAIN_PRBSERR_2_D0__ADDR_STATUS_MASK 0x03ff0000L
+#define MC_TRAIN_PRBSERR_2_D0__ADDR_STATUS__SHIFT 0x00000010
+#define MC_TRAIN_PRBSERR_2_D0__CAS_STATUS_MASK 0x00000400L
+#define MC_TRAIN_PRBSERR_2_D0__CAS_STATUS__SHIFT 0x0000000a
+#define MC_TRAIN_PRBSERR_2_D0__CKB_STATUS_MASK 0x00000002L
+#define MC_TRAIN_PRBSERR_2_D0__CKB_STATUS__SHIFT 0x00000001
+#define MC_TRAIN_PRBSERR_2_D0__CKE_STATUS_MASK 0x00000100L
+#define MC_TRAIN_PRBSERR_2_D0__CKE_STATUS__SHIFT 0x00000008
+#define MC_TRAIN_PRBSERR_2_D0__CK_STATUS_MASK 0x00000001L
+#define MC_TRAIN_PRBSERR_2_D0__CK_STATUS__SHIFT 0x00000000
+#define MC_TRAIN_PRBSERR_2_D0__CS_STATUS_MASK 0x00000030L
+#define MC_TRAIN_PRBSERR_2_D0__CS_STATUS__SHIFT 0x00000004
+#define MC_TRAIN_PRBSERR_2_D0__RAS_STATUS_MASK 0x00000200L
+#define MC_TRAIN_PRBSERR_2_D0__RAS_STATUS__SHIFT 0x00000009
+#define MC_TRAIN_PRBSERR_2_D0__WE_STATUS_MASK 0x00000800L
+#define MC_TRAIN_PRBSERR_2_D0__WE_STATUS__SHIFT 0x0000000b
+#define MC_TRAIN_PRBSERR_2_D1__ABI_STATUS_MASK 0x10000000L
+#define MC_TRAIN_PRBSERR_2_D1__ABI_STATUS__SHIFT 0x0000001c
+#define MC_TRAIN_PRBSERR_2_D1__ADDR_STATUS_MASK 0x03ff0000L
+#define MC_TRAIN_PRBSERR_2_D1__ADDR_STATUS__SHIFT 0x00000010
+#define MC_TRAIN_PRBSERR_2_D1__CAS_STATUS_MASK 0x00000400L
+#define MC_TRAIN_PRBSERR_2_D1__CAS_STATUS__SHIFT 0x0000000a
+#define MC_TRAIN_PRBSERR_2_D1__CKB_STATUS_MASK 0x00000002L
+#define MC_TRAIN_PRBSERR_2_D1__CKB_STATUS__SHIFT 0x00000001
+#define MC_TRAIN_PRBSERR_2_D1__CKE_STATUS_MASK 0x00000100L
+#define MC_TRAIN_PRBSERR_2_D1__CKE_STATUS__SHIFT 0x00000008
+#define MC_TRAIN_PRBSERR_2_D1__CK_STATUS_MASK 0x00000001L
+#define MC_TRAIN_PRBSERR_2_D1__CK_STATUS__SHIFT 0x00000000
+#define MC_TRAIN_PRBSERR_2_D1__CS_STATUS_MASK 0x00000030L
+#define MC_TRAIN_PRBSERR_2_D1__CS_STATUS__SHIFT 0x00000004
+#define MC_TRAIN_PRBSERR_2_D1__RAS_STATUS_MASK 0x00000200L
+#define MC_TRAIN_PRBSERR_2_D1__RAS_STATUS__SHIFT 0x00000009
+#define MC_TRAIN_PRBSERR_2_D1__WE_STATUS_MASK 0x00000800L
+#define MC_TRAIN_PRBSERR_2_D1__WE_STATUS__SHIFT 0x0000000b
+#define MC_TSM_DEBUG_BCNT0__BYTE0_MASK 0x000000ffL
+#define MC_TSM_DEBUG_BCNT0__BYTE0__SHIFT 0x00000000
+#define MC_TSM_DEBUG_BCNT0__BYTE1_MASK 0x0000ff00L
+#define MC_TSM_DEBUG_BCNT0__BYTE1__SHIFT 0x00000008
+#define MC_TSM_DEBUG_BCNT0__BYTE2_MASK 0x00ff0000L
+#define MC_TSM_DEBUG_BCNT0__BYTE2__SHIFT 0x00000010
+#define MC_TSM_DEBUG_BCNT0__BYTE3_MASK 0xff000000L
+#define MC_TSM_DEBUG_BCNT0__BYTE3__SHIFT 0x00000018
+#define MC_TSM_DEBUG_BCNT10__BYTE0_MASK 0x000000ffL
+#define MC_TSM_DEBUG_BCNT10__BYTE0__SHIFT 0x00000000
+#define MC_TSM_DEBUG_BCNT10__BYTE1_MASK 0x0000ff00L
+#define MC_TSM_DEBUG_BCNT10__BYTE1__SHIFT 0x00000008
+#define MC_TSM_DEBUG_BCNT10__BYTE2_MASK 0x00ff0000L
+#define MC_TSM_DEBUG_BCNT10__BYTE2__SHIFT 0x00000010
+#define MC_TSM_DEBUG_BCNT10__BYTE3_MASK 0xff000000L
+#define MC_TSM_DEBUG_BCNT10__BYTE3__SHIFT 0x00000018
+#define MC_TSM_DEBUG_BCNT1__BYTE0_MASK 0x000000ffL
+#define MC_TSM_DEBUG_BCNT1__BYTE0__SHIFT 0x00000000
+#define MC_TSM_DEBUG_BCNT1__BYTE1_MASK 0x0000ff00L
+#define MC_TSM_DEBUG_BCNT1__BYTE1__SHIFT 0x00000008
+#define MC_TSM_DEBUG_BCNT1__BYTE2_MASK 0x00ff0000L
+#define MC_TSM_DEBUG_BCNT1__BYTE2__SHIFT 0x00000010
+#define MC_TSM_DEBUG_BCNT1__BYTE3_MASK 0xff000000L
+#define MC_TSM_DEBUG_BCNT1__BYTE3__SHIFT 0x00000018
+#define MC_TSM_DEBUG_BCNT2__BYTE0_MASK 0x000000ffL
+#define MC_TSM_DEBUG_BCNT2__BYTE0__SHIFT 0x00000000
+#define MC_TSM_DEBUG_BCNT2__BYTE1_MASK 0x0000ff00L
+#define MC_TSM_DEBUG_BCNT2__BYTE1__SHIFT 0x00000008
+#define MC_TSM_DEBUG_BCNT2__BYTE2_MASK 0x00ff0000L
+#define MC_TSM_DEBUG_BCNT2__BYTE2__SHIFT 0x00000010
+#define MC_TSM_DEBUG_BCNT2__BYTE3_MASK 0xff000000L
+#define MC_TSM_DEBUG_BCNT2__BYTE3__SHIFT 0x00000018
+#define MC_TSM_DEBUG_BCNT3__BYTE0_MASK 0x000000ffL
+#define MC_TSM_DEBUG_BCNT3__BYTE0__SHIFT 0x00000000
+#define MC_TSM_DEBUG_BCNT3__BYTE1_MASK 0x0000ff00L
+#define MC_TSM_DEBUG_BCNT3__BYTE1__SHIFT 0x00000008
+#define MC_TSM_DEBUG_BCNT3__BYTE2_MASK 0x00ff0000L
+#define MC_TSM_DEBUG_BCNT3__BYTE2__SHIFT 0x00000010
+#define MC_TSM_DEBUG_BCNT3__BYTE3_MASK 0xff000000L
+#define MC_TSM_DEBUG_BCNT3__BYTE3__SHIFT 0x00000018
+#define MC_TSM_DEBUG_BCNT4__BYTE0_MASK 0x000000ffL
+#define MC_TSM_DEBUG_BCNT4__BYTE0__SHIFT 0x00000000
+#define MC_TSM_DEBUG_BCNT4__BYTE1_MASK 0x0000ff00L
+#define MC_TSM_DEBUG_BCNT4__BYTE1__SHIFT 0x00000008
+#define MC_TSM_DEBUG_BCNT4__BYTE2_MASK 0x00ff0000L
+#define MC_TSM_DEBUG_BCNT4__BYTE2__SHIFT 0x00000010
+#define MC_TSM_DEBUG_BCNT4__BYTE3_MASK 0xff000000L
+#define MC_TSM_DEBUG_BCNT4__BYTE3__SHIFT 0x00000018
+#define MC_TSM_DEBUG_BCNT5__BYTE0_MASK 0x000000ffL
+#define MC_TSM_DEBUG_BCNT5__BYTE0__SHIFT 0x00000000
+#define MC_TSM_DEBUG_BCNT5__BYTE1_MASK 0x0000ff00L
+#define MC_TSM_DEBUG_BCNT5__BYTE1__SHIFT 0x00000008
+#define MC_TSM_DEBUG_BCNT5__BYTE2_MASK 0x00ff0000L
+#define MC_TSM_DEBUG_BCNT5__BYTE2__SHIFT 0x00000010
+#define MC_TSM_DEBUG_BCNT5__BYTE3_MASK 0xff000000L
+#define MC_TSM_DEBUG_BCNT5__BYTE3__SHIFT 0x00000018
+#define MC_TSM_DEBUG_BCNT6__BYTE0_MASK 0x000000ffL
+#define MC_TSM_DEBUG_BCNT6__BYTE0__SHIFT 0x00000000
+#define MC_TSM_DEBUG_BCNT6__BYTE1_MASK 0x0000ff00L
+#define MC_TSM_DEBUG_BCNT6__BYTE1__SHIFT 0x00000008
+#define MC_TSM_DEBUG_BCNT6__BYTE2_MASK 0x00ff0000L
+#define MC_TSM_DEBUG_BCNT6__BYTE2__SHIFT 0x00000010
+#define MC_TSM_DEBUG_BCNT6__BYTE3_MASK 0xff000000L
+#define MC_TSM_DEBUG_BCNT6__BYTE3__SHIFT 0x00000018
+#define MC_TSM_DEBUG_BCNT7__BYTE0_MASK 0x000000ffL
+#define MC_TSM_DEBUG_BCNT7__BYTE0__SHIFT 0x00000000
+#define MC_TSM_DEBUG_BCNT7__BYTE1_MASK 0x0000ff00L
+#define MC_TSM_DEBUG_BCNT7__BYTE1__SHIFT 0x00000008
+#define MC_TSM_DEBUG_BCNT7__BYTE2_MASK 0x00ff0000L
+#define MC_TSM_DEBUG_BCNT7__BYTE2__SHIFT 0x00000010
+#define MC_TSM_DEBUG_BCNT7__BYTE3_MASK 0xff000000L
+#define MC_TSM_DEBUG_BCNT7__BYTE3__SHIFT 0x00000018
+#define MC_TSM_DEBUG_BCNT8__BYTE0_MASK 0x000000ffL
+#define MC_TSM_DEBUG_BCNT8__BYTE0__SHIFT 0x00000000
+#define MC_TSM_DEBUG_BCNT8__BYTE1_MASK 0x0000ff00L
+#define MC_TSM_DEBUG_BCNT8__BYTE1__SHIFT 0x00000008
+#define MC_TSM_DEBUG_BCNT8__BYTE2_MASK 0x00ff0000L
+#define MC_TSM_DEBUG_BCNT8__BYTE2__SHIFT 0x00000010
+#define MC_TSM_DEBUG_BCNT8__BYTE3_MASK 0xff000000L
+#define MC_TSM_DEBUG_BCNT8__BYTE3__SHIFT 0x00000018
+#define MC_TSM_DEBUG_BCNT9__BYTE0_MASK 0x000000ffL
+#define MC_TSM_DEBUG_BCNT9__BYTE0__SHIFT 0x00000000
+#define MC_TSM_DEBUG_BCNT9__BYTE1_MASK 0x0000ff00L
+#define MC_TSM_DEBUG_BCNT9__BYTE1__SHIFT 0x00000008
+#define MC_TSM_DEBUG_BCNT9__BYTE2_MASK 0x00ff0000L
+#define MC_TSM_DEBUG_BCNT9__BYTE2__SHIFT 0x00000010
+#define MC_TSM_DEBUG_BCNT9__BYTE3_MASK 0xff000000L
+#define MC_TSM_DEBUG_BCNT9__BYTE3__SHIFT 0x00000018
+#define MC_TSM_DEBUG_BKPT__DATA_MASK 0xffffffffL
+#define MC_TSM_DEBUG_BKPT__DATA__SHIFT 0x00000000
+#define MC_TSM_DEBUG_FLAG__DATA_MASK 0xffffffffL
+#define MC_TSM_DEBUG_FLAG__DATA__SHIFT 0x00000000
+#define MC_TSM_DEBUG_GCNT__DATA_MASK 0xffffffffL
+#define MC_TSM_DEBUG_GCNT__DATA__SHIFT 0x00000000
+#define MC_TSM_DEBUG_MISC__FLAG_MASK 0x000000ffL
+#define MC_TSM_DEBUG_MISC__FLAG__SHIFT 0x00000000
+#define MC_TSM_DEBUG_MISC__NCNT_RD_MASK 0x00000f00L
+#define MC_TSM_DEBUG_MISC__NCNT_RD__SHIFT 0x00000008
+#define MC_TSM_DEBUG_MISC__NCNT_WR_MASK 0x0000f000L
+#define MC_TSM_DEBUG_MISC__NCNT_WR__SHIFT 0x0000000c
+#define MC_TSM_DEBUG_ST01__DATA_MASK 0xffffffffL
+#define MC_TSM_DEBUG_ST01__DATA__SHIFT 0x00000000
+#define MC_TSM_DEBUG_ST23__DATA_MASK 0xffffffffL
+#define MC_TSM_DEBUG_ST23__DATA__SHIFT 0x00000000
+#define MC_TSM_DEBUG_ST45__DATA_MASK 0xffffffffL
+#define MC_TSM_DEBUG_ST45__DATA__SHIFT 0x00000000
+#define MC_VM_AGP_BASE__AGP_BASE_MASK 0x0003ffffL
+#define MC_VM_AGP_BASE__AGP_BASE__SHIFT 0x00000000
+#define MC_VM_AGP_BOT__AGP_BOT_MASK 0x0003ffffL
+#define MC_VM_AGP_BOT__AGP_BOT__SHIFT 0x00000000
+#define MC_VM_AGP_TOP__AGP_TOP_MASK 0x0003ffffL
+#define MC_VM_AGP_TOP__AGP_TOP__SHIFT 0x00000000
+#define MC_VM_DC_WRITE_CNTL__DC_MEMORY_WRITE_LOCAL_MASK 0x00000100L
+#define MC_VM_DC_WRITE_CNTL__DC_MEMORY_WRITE_LOCAL__SHIFT 0x00000008
+#define MC_VM_DC_WRITE_CNTL__DC_MEMORY_WRITE_SYSTEM_MASK 0x00000200L
+#define MC_VM_DC_WRITE_CNTL__DC_MEMORY_WRITE_SYSTEM__SHIFT 0x00000009
+#define MC_VM_DC_WRITE_CNTL__DC_WRITE_HIT_REGION_0_MODE_MASK 0x00000003L
+#define MC_VM_DC_WRITE_CNTL__DC_WRITE_HIT_REGION_0_MODE__SHIFT 0x00000000
+#define MC_VM_DC_WRITE_CNTL__DC_WRITE_HIT_REGION_1_MODE_MASK 0x0000000cL
+#define MC_VM_DC_WRITE_CNTL__DC_WRITE_HIT_REGION_1_MODE__SHIFT 0x00000002
+#define MC_VM_DC_WRITE_CNTL__DC_WRITE_HIT_REGION_2_MODE_MASK 0x00000030L
+#define MC_VM_DC_WRITE_CNTL__DC_WRITE_HIT_REGION_2_MODE__SHIFT 0x00000004
+#define MC_VM_DC_WRITE_CNTL__DC_WRITE_HIT_REGION_3_MODE_MASK 0x000000c0L
+#define MC_VM_DC_WRITE_CNTL__DC_WRITE_HIT_REGION_3_MODE__SHIFT 0x00000006
+#define MC_VM_DC_WRITE_HIT_REGION_0_HIGH_ADDR__PHYSICAL_ADDRESS_MASK 0x0fffffffL
+#define MC_VM_DC_WRITE_HIT_REGION_0_HIGH_ADDR__PHYSICAL_ADDRESS__SHIFT 0x00000000
+#define MC_VM_DC_WRITE_HIT_REGION_0_LOW_ADDR__PHYSICAL_ADDRESS_MASK 0x0fffffffL
+#define MC_VM_DC_WRITE_HIT_REGION_0_LOW_ADDR__PHYSICAL_ADDRESS__SHIFT 0x00000000
+#define MC_VM_DC_WRITE_HIT_REGION_1_HIGH_ADDR__PHYSICAL_ADDRESS_MASK 0x0fffffffL
+#define MC_VM_DC_WRITE_HIT_REGION_1_HIGH_ADDR__PHYSICAL_ADDRESS__SHIFT 0x00000000
+#define MC_VM_DC_WRITE_HIT_REGION_1_LOW_ADDR__PHYSICAL_ADDRESS_MASK 0x0fffffffL
+#define MC_VM_DC_WRITE_HIT_REGION_1_LOW_ADDR__PHYSICAL_ADDRESS__SHIFT 0x00000000
+#define MC_VM_DC_WRITE_HIT_REGION_2_HIGH_ADDR__PHYSICAL_ADDRESS_MASK 0x0fffffffL
+#define MC_VM_DC_WRITE_HIT_REGION_2_HIGH_ADDR__PHYSICAL_ADDRESS__SHIFT 0x00000000
+#define MC_VM_DC_WRITE_HIT_REGION_2_LOW_ADDR__PHYSICAL_ADDRESS_MASK 0x0fffffffL
+#define MC_VM_DC_WRITE_HIT_REGION_2_LOW_ADDR__PHYSICAL_ADDRESS__SHIFT 0x00000000
+#define MC_VM_DC_WRITE_HIT_REGION_3_HIGH_ADDR__PHYSICAL_ADDRESS_MASK 0x0fffffffL
+#define MC_VM_DC_WRITE_HIT_REGION_3_HIGH_ADDR__PHYSICAL_ADDRESS__SHIFT 0x00000000
+#define MC_VM_DC_WRITE_HIT_REGION_3_LOW_ADDR__PHYSICAL_ADDRESS_MASK 0x0fffffffL
+#define MC_VM_DC_WRITE_HIT_REGION_3_LOW_ADDR__PHYSICAL_ADDRESS__SHIFT 0x00000000
+#define MC_VM_FB_LOCATION__FB_BASE_MASK 0x0000ffffL
+#define MC_VM_FB_LOCATION__FB_BASE__SHIFT 0x00000000
+#define MC_VM_FB_LOCATION__FB_TOP_MASK 0xffff0000L
+#define MC_VM_FB_LOCATION__FB_TOP__SHIFT 0x00000010
+#define MC_VM_FB_OFFSET__FB_OFFSET_MASK 0x0003ffffL
+#define MC_VM_FB_OFFSET__FB_OFFSET__SHIFT 0x00000000
+#define MC_VM_MB_L1_TLB0_DEBUG__EFFECTIVE_L1_QUEUE_SIZE_MASK 0x00007000L
+#define MC_VM_MB_L1_TLB0_DEBUG__EFFECTIVE_L1_QUEUE_SIZE__SHIFT 0x0000000c
+#define MC_VM_MB_L1_TLB0_DEBUG__EFFECTIVE_L1_TLB_SIZE_MASK 0x00000e00L
+#define MC_VM_MB_L1_TLB0_DEBUG__EFFECTIVE_L1_TLB_SIZE__SHIFT 0x00000009
+#define MC_VM_MB_L1_TLB0_DEBUG__INVALIDATE_L1_TLB_MASK 0x00000001L
+#define MC_VM_MB_L1_TLB0_DEBUG__INVALIDATE_L1_TLB__SHIFT 0x00000000
+#define MC_VM_MB_L1_TLB0_DEBUG__L1_TLB_DEBUG_MASK 0x00078000L
+#define MC_VM_MB_L1_TLB0_DEBUG__L1_TLB_DEBUG__SHIFT 0x0000000f
+#define MC_VM_MB_L1_TLB0_DEBUG__SEND_FREE_AT_RTN_MASK 0x00000100L
+#define MC_VM_MB_L1_TLB0_DEBUG__SEND_FREE_AT_RTN__SHIFT 0x00000008
+#define MC_VM_MB_L1_TLB0_STATUS__BUSY_MASK 0x00000001L
+#define MC_VM_MB_L1_TLB0_STATUS__BUSY__SHIFT 0x00000000
+#define MC_VM_MB_L1_TLB1_STATUS__BUSY_MASK 0x00000001L
+#define MC_VM_MB_L1_TLB1_STATUS__BUSY__SHIFT 0x00000000
+#define MC_VM_MB_L1_TLB2_DEBUG__EFFECTIVE_L1_QUEUE_SIZE_MASK 0x00007000L
+#define MC_VM_MB_L1_TLB2_DEBUG__EFFECTIVE_L1_QUEUE_SIZE__SHIFT 0x0000000c
+#define MC_VM_MB_L1_TLB2_DEBUG__EFFECTIVE_L1_TLB_SIZE_MASK 0x00000e00L
+#define MC_VM_MB_L1_TLB2_DEBUG__EFFECTIVE_L1_TLB_SIZE__SHIFT 0x00000009
+#define MC_VM_MB_L1_TLB2_DEBUG__INVALIDATE_L1_TLB_MASK 0x00000001L
+#define MC_VM_MB_L1_TLB2_DEBUG__INVALIDATE_L1_TLB__SHIFT 0x00000000
+#define MC_VM_MB_L1_TLB2_DEBUG__L1_TLB_DEBUG_MASK 0x00078000L
+#define MC_VM_MB_L1_TLB2_DEBUG__L1_TLB_DEBUG__SHIFT 0x0000000f
+#define MC_VM_MB_L1_TLB2_DEBUG__SEND_FREE_AT_RTN_MASK 0x00000100L
+#define MC_VM_MB_L1_TLB2_DEBUG__SEND_FREE_AT_RTN__SHIFT 0x00000008
+#define MC_VM_MB_L1_TLB2_STATUS__BUSY_MASK 0x00000001L
+#define MC_VM_MB_L1_TLB2_STATUS__BUSY__SHIFT 0x00000000
+#define MC_VM_MB_L1_TLB3_DEBUG__EFFECTIVE_L1_QUEUE_SIZE_MASK 0x00007000L
+#define MC_VM_MB_L1_TLB3_DEBUG__EFFECTIVE_L1_QUEUE_SIZE__SHIFT 0x0000000c
+#define MC_VM_MB_L1_TLB3_DEBUG__EFFECTIVE_L1_TLB_SIZE_MASK 0x00000e00L
+#define MC_VM_MB_L1_TLB3_DEBUG__EFFECTIVE_L1_TLB_SIZE__SHIFT 0x00000009
+#define MC_VM_MB_L1_TLB3_DEBUG__INVALIDATE_L1_TLB_MASK 0x00000001L
+#define MC_VM_MB_L1_TLB3_DEBUG__INVALIDATE_L1_TLB__SHIFT 0x00000000
+#define MC_VM_MB_L1_TLB3_DEBUG__L1_TLB_DEBUG_MASK 0x00078000L
+#define MC_VM_MB_L1_TLB3_DEBUG__L1_TLB_DEBUG__SHIFT 0x0000000f
+#define MC_VM_MB_L1_TLB3_DEBUG__SEND_FREE_AT_RTN_MASK 0x00000100L
+#define MC_VM_MB_L1_TLB3_DEBUG__SEND_FREE_AT_RTN__SHIFT 0x00000008
+#define MC_VM_MB_L1_TLB3_STATUS__BUSY_MASK 0x00000001L
+#define MC_VM_MB_L1_TLB3_STATUS__BUSY__SHIFT 0x00000000
+#define MC_VM_MB_L2ARBITER_L2_CREDITS__L2_IF_CREDITS_MASK 0x0000003fL
+#define MC_VM_MB_L2ARBITER_L2_CREDITS__L2_IF_CREDITS__SHIFT 0x00000000
+#define MC_VM_MD_L1_TLB0_DEBUG__EFFECTIVE_L1_QUEUE_SIZE_MASK 0x00007000L
+#define MC_VM_MD_L1_TLB0_DEBUG__EFFECTIVE_L1_QUEUE_SIZE__SHIFT 0x0000000c
+#define MC_VM_MD_L1_TLB0_DEBUG__EFFECTIVE_L1_TLB_SIZE_MASK 0x00000e00L
+#define MC_VM_MD_L1_TLB0_DEBUG__EFFECTIVE_L1_TLB_SIZE__SHIFT 0x00000009
+#define MC_VM_MD_L1_TLB0_DEBUG__INVALIDATE_L1_TLB_MASK 0x00000001L
+#define MC_VM_MD_L1_TLB0_DEBUG__INVALIDATE_L1_TLB__SHIFT 0x00000000
+#define MC_VM_MD_L1_TLB0_DEBUG__L1_TLB_DEBUG_MASK 0x00078000L
+#define MC_VM_MD_L1_TLB0_DEBUG__L1_TLB_DEBUG__SHIFT 0x0000000f
+#define MC_VM_MD_L1_TLB0_DEBUG__SEND_FREE_AT_RTN_MASK 0x00000100L
+#define MC_VM_MD_L1_TLB0_DEBUG__SEND_FREE_AT_RTN__SHIFT 0x00000008
+#define MC_VM_MD_L1_TLB0_STATUS__BUSY_MASK 0x00000001L
+#define MC_VM_MD_L1_TLB0_STATUS__BUSY__SHIFT 0x00000000
+#define MC_VM_MD_L1_TLB1_DEBUG__EFFECTIVE_L1_QUEUE_SIZE_MASK 0x00007000L
+#define MC_VM_MD_L1_TLB1_DEBUG__EFFECTIVE_L1_QUEUE_SIZE__SHIFT 0x0000000c
+#define MC_VM_MD_L1_TLB1_DEBUG__EFFECTIVE_L1_TLB_SIZE_MASK 0x00000e00L
+#define MC_VM_MD_L1_TLB1_DEBUG__EFFECTIVE_L1_TLB_SIZE__SHIFT 0x00000009
+#define MC_VM_MD_L1_TLB1_DEBUG__INVALIDATE_L1_TLB_MASK 0x00000001L
+#define MC_VM_MD_L1_TLB1_DEBUG__INVALIDATE_L1_TLB__SHIFT 0x00000000
+#define MC_VM_MD_L1_TLB1_DEBUG__L1_TLB_DEBUG_MASK 0x00078000L
+#define MC_VM_MD_L1_TLB1_DEBUG__L1_TLB_DEBUG__SHIFT 0x0000000f
+#define MC_VM_MD_L1_TLB1_DEBUG__SEND_FREE_AT_RTN_MASK 0x00000100L
+#define MC_VM_MD_L1_TLB1_DEBUG__SEND_FREE_AT_RTN__SHIFT 0x00000008
+#define MC_VM_MD_L1_TLB1_STATUS__BUSY_MASK 0x00000001L
+#define MC_VM_MD_L1_TLB1_STATUS__BUSY__SHIFT 0x00000000
+#define MC_VM_MD_L1_TLB2_DEBUG__EFFECTIVE_L1_QUEUE_SIZE_MASK 0x00007000L
+#define MC_VM_MD_L1_TLB2_DEBUG__EFFECTIVE_L1_QUEUE_SIZE__SHIFT 0x0000000c
+#define MC_VM_MD_L1_TLB2_DEBUG__EFFECTIVE_L1_TLB_SIZE_MASK 0x00000e00L
+#define MC_VM_MD_L1_TLB2_DEBUG__EFFECTIVE_L1_TLB_SIZE__SHIFT 0x00000009
+#define MC_VM_MD_L1_TLB2_DEBUG__INVALIDATE_L1_TLB_MASK 0x00000001L
+#define MC_VM_MD_L1_TLB2_DEBUG__INVALIDATE_L1_TLB__SHIFT 0x00000000
+#define MC_VM_MD_L1_TLB2_DEBUG__L1_TLB_DEBUG_MASK 0x00078000L
+#define MC_VM_MD_L1_TLB2_DEBUG__L1_TLB_DEBUG__SHIFT 0x0000000f
+#define MC_VM_MD_L1_TLB2_DEBUG__SEND_FREE_AT_RTN_MASK 0x00000100L
+#define MC_VM_MD_L1_TLB2_DEBUG__SEND_FREE_AT_RTN__SHIFT 0x00000008
+#define MC_VM_MD_L1_TLB2_STATUS__BUSY_MASK 0x00000001L
+#define MC_VM_MD_L1_TLB2_STATUS__BUSY__SHIFT 0x00000000
+#define MC_VM_MD_L1_TLB3_DEBUG__EFFECTIVE_L1_QUEUE_SIZE_MASK 0x00007000L
+#define MC_VM_MD_L1_TLB3_DEBUG__EFFECTIVE_L1_QUEUE_SIZE__SHIFT 0x0000000c
+#define MC_VM_MD_L1_TLB3_DEBUG__EFFECTIVE_L1_TLB_SIZE_MASK 0x00000e00L
+#define MC_VM_MD_L1_TLB3_DEBUG__EFFECTIVE_L1_TLB_SIZE__SHIFT 0x00000009
+#define MC_VM_MD_L1_TLB3_DEBUG__INVALIDATE_L1_TLB_MASK 0x00000001L
+#define MC_VM_MD_L1_TLB3_DEBUG__INVALIDATE_L1_TLB__SHIFT 0x00000000
+#define MC_VM_MD_L1_TLB3_DEBUG__L1_TLB_DEBUG_MASK 0x00078000L
+#define MC_VM_MD_L1_TLB3_DEBUG__L1_TLB_DEBUG__SHIFT 0x0000000f
+#define MC_VM_MD_L1_TLB3_DEBUG__SEND_FREE_AT_RTN_MASK 0x00000100L
+#define MC_VM_MD_L1_TLB3_DEBUG__SEND_FREE_AT_RTN__SHIFT 0x00000008
+#define MC_VM_MD_L1_TLB3_STATUS__BUSY_MASK 0x00000001L
+#define MC_VM_MD_L1_TLB3_STATUS__BUSY__SHIFT 0x00000000
+#define MC_VM_MD_L2ARBITER_L2_CREDITS__L2_IF_CREDITS_MASK 0x0000003fL
+#define MC_VM_MD_L2ARBITER_L2_CREDITS__L2_IF_CREDITS__SHIFT 0x00000000
+#define MC_VM_MX_L1_TLB_CNTL__ECO_BITS_MASK 0x00000780L
+#define MC_VM_MX_L1_TLB_CNTL__ECO_BITS__SHIFT 0x00000007
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK 0x00000040L
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL__SHIFT 0x00000006
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_FRAGMENT_PROCESSING_MASK 0x00000002L
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_FRAGMENT_PROCESSING__SHIFT 0x00000001
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK 0x00000001L
+#define MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB__SHIFT 0x00000000
+#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK 0x00000018L
+#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE__SHIFT 0x00000003
+#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS_MASK 0x00000020L
+#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT 0x00000005
+#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define MC_VM_SYSTEM_APERTURE_HIGH_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define MC_VM_SYSTEM_APERTURE_HIGH_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define MC_VM_SYSTEM_APERTURE_LOW_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define MC_VM_SYSTEM_APERTURE_LOW_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define MC_WR_CB__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_WR_CB__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_WR_CB__ENABLE_MASK 0x00000001L
+#define MC_WR_CB__ENABLE__SHIFT 0x00000000
+#define MC_WR_CB__LAZY_TIMER_MASK 0x00007800L
+#define MC_WR_CB__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_WR_CB__MAX_BURST_MASK 0x00000780L
+#define MC_WR_CB__MAX_BURST__SHIFT 0x00000007
+#define MC_WR_CB__PRESCALE_MASK 0x00000006L
+#define MC_WR_CB__PRESCALE__SHIFT 0x00000001
+#define MC_WR_CB__STALL_MODE_MASK 0x00000030L
+#define MC_WR_CB__STALL_MODE__SHIFT 0x00000004
+#define MC_WR_CB__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_WR_CB__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_WR_CB__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_WR_CB__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_WR_DB__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_WR_DB__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_WR_DB__ENABLE_MASK 0x00000001L
+#define MC_WR_DB__ENABLE__SHIFT 0x00000000
+#define MC_WR_DB__LAZY_TIMER_MASK 0x00007800L
+#define MC_WR_DB__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_WR_DB__MAX_BURST_MASK 0x00000780L
+#define MC_WR_DB__MAX_BURST__SHIFT 0x00000007
+#define MC_WR_DB__PRESCALE_MASK 0x00000006L
+#define MC_WR_DB__PRESCALE__SHIFT 0x00000001
+#define MC_WR_DB__STALL_MODE_MASK 0x00000030L
+#define MC_WR_DB__STALL_MODE__SHIFT 0x00000004
+#define MC_WR_DB__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_WR_DB__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_WR_DB__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_WR_DB__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_WR_GRP_EXT__DBSTEN0_MASK 0x0000000fL
+#define MC_WR_GRP_EXT__DBSTEN0__SHIFT 0x00000000
+#define MC_WR_GRP_EXT__TC0_MASK 0x000000f0L
+#define MC_WR_GRP_EXT__TC0__SHIFT 0x00000004
+#define MC_WR_GRP_GFX__CP_MASK 0x0000000fL
+#define MC_WR_GRP_GFX__CP__SHIFT 0x00000000
+#define MC_WR_GRP_GFX__XDMA_MASK 0x0000f000L
+#define MC_WR_GRP_GFX__XDMAM_MASK 0x000f0000L
+#define MC_WR_GRP_GFX__XDMAM__SHIFT 0x00000010
+#define MC_WR_GRP_GFX__XDMA__SHIFT 0x0000000c
+#define MC_WR_GRP_LCL__CB0_MASK 0x0000000fL
+#define MC_WR_GRP_LCL__CB0__SHIFT 0x00000000
+#define MC_WR_GRP_LCL__CBCMASK0_MASK 0x000000f0L
+#define MC_WR_GRP_LCL__CBCMASK0__SHIFT 0x00000004
+#define MC_WR_GRP_LCL__CBFMASK0_MASK 0x00000f00L
+#define MC_WR_GRP_LCL__CBFMASK0__SHIFT 0x00000008
+#define MC_WR_GRP_LCL__CBIMMED0_MASK 0xf0000000L
+#define MC_WR_GRP_LCL__CBIMMED0__SHIFT 0x0000001c
+#define MC_WR_GRP_LCL__DB0_MASK 0x0000f000L
+#define MC_WR_GRP_LCL__DB0__SHIFT 0x0000000c
+#define MC_WR_GRP_LCL__DBHTILE0_MASK 0x000f0000L
+#define MC_WR_GRP_LCL__DBHTILE0__SHIFT 0x00000010
+#define MC_WR_GRP_LCL__SX0_MASK 0x00f00000L
+#define MC_WR_GRP_LCL__SX0__SHIFT 0x00000014
+#define MC_WR_GRP_OTH__HDP_MASK 0x00000f00L
+#define MC_WR_GRP_OTH__HDP__SHIFT 0x00000008
+#define MC_WR_GRP_OTH__SEM_MASK 0x0000f000L
+#define MC_WR_GRP_OTH__SEM__SHIFT 0x0000000c
+#define MC_WR_GRP_OTH__UMC_MASK 0x000f0000L
+#define MC_WR_GRP_OTH__UMC__SHIFT 0x00000010
+#define MC_WR_GRP_OTH__UVD_EXT0_MASK 0x0000000fL
+#define MC_WR_GRP_OTH__UVD_EXT0__SHIFT 0x00000000
+#define MC_WR_GRP_OTH__UVD_EXT1_MASK 0xf0000000L
+#define MC_WR_GRP_OTH__UVD_EXT1__SHIFT 0x0000001c
+#define MC_WR_GRP_OTH__UVD_MASK 0x00f00000L
+#define MC_WR_GRP_OTH__UVD__SHIFT 0x00000014
+#define MC_WR_GRP_OTH__XDP_MASK 0x0f000000L
+#define MC_WR_GRP_OTH__XDP__SHIFT 0x00000018
+#define MC_WR_GRP_SYS__IH_MASK 0x0000000fL
+#define MC_WR_GRP_SYS__IH__SHIFT 0x00000000
+#define MC_WR_GRP_SYS__MCIF_MASK 0x000000f0L
+#define MC_WR_GRP_SYS__MCIF__SHIFT 0x00000004
+#define MC_WR_GRP_SYS__RLC_MASK 0x00000f00L
+#define MC_WR_GRP_SYS__RLC__SHIFT 0x00000008
+#define MC_WR_GRP_SYS__SMU_MASK 0x00f00000L
+#define MC_WR_GRP_SYS__SMU__SHIFT 0x00000014
+#define MC_WR_GRP_SYS__VCE_MASK 0x0f000000L
+#define MC_WR_GRP_SYS__VCE__SHIFT 0x00000018
+#define MC_WR_GRP_SYS__VCEU_MASK 0xf0000000L
+#define MC_WR_GRP_SYS__VCEU__SHIFT 0x0000001c
+#define MC_WR_HUB__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_WR_HUB__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_WR_HUB__ENABLE_MASK 0x00000001L
+#define MC_WR_HUB__ENABLE__SHIFT 0x00000000
+#define MC_WR_HUB__LAZY_TIMER_MASK 0x00007800L
+#define MC_WR_HUB__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_WR_HUB__MAX_BURST_MASK 0x00000780L
+#define MC_WR_HUB__MAX_BURST__SHIFT 0x00000007
+#define MC_WR_HUB__PRESCALE_MASK 0x00000006L
+#define MC_WR_HUB__PRESCALE__SHIFT 0x00000001
+#define MC_WR_HUB__STALL_MODE_MASK 0x00000030L
+#define MC_WR_HUB__STALL_MODE__SHIFT 0x00000004
+#define MC_WR_HUB__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_WR_HUB__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_WR_HUB__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_WR_HUB__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_WR_TC0__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_WR_TC0__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_WR_TC0__ENABLE_MASK 0x00000001L
+#define MC_WR_TC0__ENABLE__SHIFT 0x00000000
+#define MC_WR_TC0__LAZY_TIMER_MASK 0x00007800L
+#define MC_WR_TC0__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_WR_TC0__MAX_BURST_MASK 0x00000780L
+#define MC_WR_TC0__MAX_BURST__SHIFT 0x00000007
+#define MC_WR_TC0__PRESCALE_MASK 0x00000006L
+#define MC_WR_TC0__PRESCALE__SHIFT 0x00000001
+#define MC_WR_TC0__STALL_MODE_MASK 0x00000030L
+#define MC_WR_TC0__STALL_MODE__SHIFT 0x00000004
+#define MC_WR_TC0__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_WR_TC0__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_WR_TC0__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_WR_TC0__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_WR_TC1__BLACKOUT_EXEMPT_MASK 0x00000008L
+#define MC_WR_TC1__BLACKOUT_EXEMPT__SHIFT 0x00000003
+#define MC_WR_TC1__ENABLE_MASK 0x00000001L
+#define MC_WR_TC1__ENABLE__SHIFT 0x00000000
+#define MC_WR_TC1__LAZY_TIMER_MASK 0x00007800L
+#define MC_WR_TC1__LAZY_TIMER__SHIFT 0x0000000b
+#define MC_WR_TC1__MAX_BURST_MASK 0x00000780L
+#define MC_WR_TC1__MAX_BURST__SHIFT 0x00000007
+#define MC_WR_TC1__PRESCALE_MASK 0x00000006L
+#define MC_WR_TC1__PRESCALE__SHIFT 0x00000001
+#define MC_WR_TC1__STALL_MODE_MASK 0x00000030L
+#define MC_WR_TC1__STALL_MODE__SHIFT 0x00000004
+#define MC_WR_TC1__STALL_OVERRIDE_MASK 0x00000040L
+#define MC_WR_TC1__STALL_OVERRIDE__SHIFT 0x00000006
+#define MC_WR_TC1__STALL_OVERRIDE_WTM_MASK 0x00008000L
+#define MC_WR_TC1__STALL_OVERRIDE_WTM__SHIFT 0x0000000f
+#define MC_XBAR_ADDR_DEC__GECC_MASK 0x00000002L
+#define MC_XBAR_ADDR_DEC__GECC__SHIFT 0x00000001
+#define MC_XBAR_ADDR_DEC__NO_DIV_BY_3_MASK 0x00000001L
+#define MC_XBAR_ADDR_DEC__NO_DIV_BY_3__SHIFT 0x00000000
+#define MC_XBAR_ADDR_DEC__RB_SPLIT_COLHI_MASK 0x00000008L
+#define MC_XBAR_ADDR_DEC__RB_SPLIT_COLHI__SHIFT 0x00000003
+#define MC_XBAR_ADDR_DEC__RB_SPLIT_MASK 0x00000004L
+#define MC_XBAR_ADDR_DEC__RB_SPLIT__SHIFT 0x00000002
+#define MC_XBAR_ARB__BREAK_BURST_CID_CHANGE_MASK 0x00000004L
+#define MC_XBAR_ARB__BREAK_BURST_CID_CHANGE__SHIFT 0x00000002
+#define MC_XBAR_ARB__DISABLE_HUB_STALL_HIGHEST_MASK 0x00000002L
+#define MC_XBAR_ARB__DISABLE_HUB_STALL_HIGHEST__SHIFT 0x00000001
+#define MC_XBAR_ARB__HUBRD_HIGHEST_MASK 0x00000001L
+#define MC_XBAR_ARB__HUBRD_HIGHEST__SHIFT 0x00000000
+#define MC_XBAR_ARB_MAX_BURST__RD_PORT0_MASK 0x0000000fL
+#define MC_XBAR_ARB_MAX_BURST__RD_PORT0__SHIFT 0x00000000
+#define MC_XBAR_ARB_MAX_BURST__RD_PORT1_MASK 0x000000f0L
+#define MC_XBAR_ARB_MAX_BURST__RD_PORT1__SHIFT 0x00000004
+#define MC_XBAR_ARB_MAX_BURST__RD_PORT2_MASK 0x00000f00L
+#define MC_XBAR_ARB_MAX_BURST__RD_PORT2__SHIFT 0x00000008
+#define MC_XBAR_ARB_MAX_BURST__RD_PORT3_MASK 0x0000f000L
+#define MC_XBAR_ARB_MAX_BURST__RD_PORT3__SHIFT 0x0000000c
+#define MC_XBAR_ARB_MAX_BURST__WR_PORT0_MASK 0x000f0000L
+#define MC_XBAR_ARB_MAX_BURST__WR_PORT0__SHIFT 0x00000010
+#define MC_XBAR_ARB_MAX_BURST__WR_PORT1_MASK 0x00f00000L
+#define MC_XBAR_ARB_MAX_BURST__WR_PORT1__SHIFT 0x00000014
+#define MC_XBAR_ARB_MAX_BURST__WR_PORT2_MASK 0x0f000000L
+#define MC_XBAR_ARB_MAX_BURST__WR_PORT2__SHIFT 0x00000018
+#define MC_XBAR_ARB_MAX_BURST__WR_PORT3_MASK 0xf0000000L
+#define MC_XBAR_ARB_MAX_BURST__WR_PORT3__SHIFT 0x0000001c
+#define MC_XBAR_CHTRIREMAP__CH0_MASK 0x00000003L
+#define MC_XBAR_CHTRIREMAP__CH0__SHIFT 0x00000000
+#define MC_XBAR_CHTRIREMAP__CH1_MASK 0x0000000cL
+#define MC_XBAR_CHTRIREMAP__CH1__SHIFT 0x00000002
+#define MC_XBAR_CHTRIREMAP__CH2_MASK 0x00000030L
+#define MC_XBAR_CHTRIREMAP__CH2__SHIFT 0x00000004
+#define MC_XBAR_PERF_MON_CNTL0__ALLOW_WRAP_MASK 0x10000000L
+#define MC_XBAR_PERF_MON_CNTL0__ALLOW_WRAP__SHIFT 0x0000001c
+#define MC_XBAR_PERF_MON_CNTL0__START_MODE_MASK 0x03000000L
+#define MC_XBAR_PERF_MON_CNTL0__START_MODE__SHIFT 0x00000018
+#define MC_XBAR_PERF_MON_CNTL0__START_THRESH_MASK 0x00000fffL
+#define MC_XBAR_PERF_MON_CNTL0__START_THRESH__SHIFT 0x00000000
+#define MC_XBAR_PERF_MON_CNTL0__STOP_MODE_MASK 0x0c000000L
+#define MC_XBAR_PERF_MON_CNTL0__STOP_MODE__SHIFT 0x0000001a
+#define MC_XBAR_PERF_MON_CNTL0__STOP_THRESH_MASK 0x00fff000L
+#define MC_XBAR_PERF_MON_CNTL0__STOP_THRESH__SHIFT 0x0000000c
+#define MC_XBAR_PERF_MON_CNTL1__START_TRIG_ID_MASK 0x0000ff00L
+#define MC_XBAR_PERF_MON_CNTL1__START_TRIG_ID__SHIFT 0x00000008
+#define MC_XBAR_PERF_MON_CNTL1__STOP_TRIG_ID_MASK 0x00ff0000L
+#define MC_XBAR_PERF_MON_CNTL1__STOP_TRIG_ID__SHIFT 0x00000010
+#define MC_XBAR_PERF_MON_CNTL1__THRESH_CNTR_ID_MASK 0x000000ffL
+#define MC_XBAR_PERF_MON_CNTL1__THRESH_CNTR_ID__SHIFT 0x00000000
+#define MC_XBAR_PERF_MON_CNTL2__MON0_ID_MASK 0x000000ffL
+#define MC_XBAR_PERF_MON_CNTL2__MON0_ID__SHIFT 0x00000000
+#define MC_XBAR_PERF_MON_CNTL2__MON1_ID_MASK 0x0000ff00L
+#define MC_XBAR_PERF_MON_CNTL2__MON1_ID__SHIFT 0x00000008
+#define MC_XBAR_PERF_MON_CNTL2__MON2_ID_MASK 0x00ff0000L
+#define MC_XBAR_PERF_MON_CNTL2__MON2_ID__SHIFT 0x00000010
+#define MC_XBAR_PERF_MON_CNTL2__MON3_ID_MASK 0xff000000L
+#define MC_XBAR_PERF_MON_CNTL2__MON3_ID__SHIFT 0x00000018
+#define MC_XBAR_PERF_MON_MAX_THSH__MON0_MASK 0x000000ffL
+#define MC_XBAR_PERF_MON_MAX_THSH__MON0__SHIFT 0x00000000
+#define MC_XBAR_PERF_MON_MAX_THSH__MON1_MASK 0x0000ff00L
+#define MC_XBAR_PERF_MON_MAX_THSH__MON1__SHIFT 0x00000008
+#define MC_XBAR_PERF_MON_MAX_THSH__MON2_MASK 0x00ff0000L
+#define MC_XBAR_PERF_MON_MAX_THSH__MON2__SHIFT 0x00000010
+#define MC_XBAR_PERF_MON_MAX_THSH__MON3_MASK 0xff000000L
+#define MC_XBAR_PERF_MON_MAX_THSH__MON3__SHIFT 0x00000018
+#define MC_XBAR_PERF_MON_RSLT0__COUNT_MASK 0xffffffffL
+#define MC_XBAR_PERF_MON_RSLT0__COUNT__SHIFT 0x00000000
+#define MC_XBAR_PERF_MON_RSLT1__COUNT_MASK 0xffffffffL
+#define MC_XBAR_PERF_MON_RSLT1__COUNT__SHIFT 0x00000000
+#define MC_XBAR_PERF_MON_RSLT2__COUNT_MASK 0xffffffffL
+#define MC_XBAR_PERF_MON_RSLT2__COUNT__SHIFT 0x00000000
+#define MC_XBAR_PERF_MON_RSLT3__COUNT_MASK 0xffffffffL
+#define MC_XBAR_PERF_MON_RSLT3__COUNT__SHIFT 0x00000000
+#define MC_XBAR_RDREQ_CREDIT__OUT0_MASK 0x000000ffL
+#define MC_XBAR_RDREQ_CREDIT__OUT0__SHIFT 0x00000000
+#define MC_XBAR_RDREQ_CREDIT__OUT1_MASK 0x0000ff00L
+#define MC_XBAR_RDREQ_CREDIT__OUT1__SHIFT 0x00000008
+#define MC_XBAR_RDREQ_CREDIT__OUT2_MASK 0x00ff0000L
+#define MC_XBAR_RDREQ_CREDIT__OUT2__SHIFT 0x00000010
+#define MC_XBAR_RDREQ_CREDIT__OUT3_MASK 0xff000000L
+#define MC_XBAR_RDREQ_CREDIT__OUT3__SHIFT 0x00000018
+#define MC_XBAR_RDREQ_PRI_CREDIT__OUT0_MASK 0x000000ffL
+#define MC_XBAR_RDREQ_PRI_CREDIT__OUT0__SHIFT 0x00000000
+#define MC_XBAR_RDREQ_PRI_CREDIT__OUT1_MASK 0x0000ff00L
+#define MC_XBAR_RDREQ_PRI_CREDIT__OUT1__SHIFT 0x00000008
+#define MC_XBAR_RDREQ_PRI_CREDIT__OUT2_MASK 0x00ff0000L
+#define MC_XBAR_RDREQ_PRI_CREDIT__OUT2__SHIFT 0x00000010
+#define MC_XBAR_RDREQ_PRI_CREDIT__OUT3_MASK 0xff000000L
+#define MC_XBAR_RDREQ_PRI_CREDIT__OUT3__SHIFT 0x00000018
+#define MC_XBAR_RDRET_CREDIT1__OUT0_MASK 0x000000ffL
+#define MC_XBAR_RDRET_CREDIT1__OUT0__SHIFT 0x00000000
+#define MC_XBAR_RDRET_CREDIT1__OUT1_MASK 0x0000ff00L
+#define MC_XBAR_RDRET_CREDIT1__OUT1__SHIFT 0x00000008
+#define MC_XBAR_RDRET_CREDIT1__OUT2_MASK 0x00ff0000L
+#define MC_XBAR_RDRET_CREDIT1__OUT2__SHIFT 0x00000010
+#define MC_XBAR_RDRET_CREDIT1__OUT3_MASK 0xff000000L
+#define MC_XBAR_RDRET_CREDIT1__OUT3__SHIFT 0x00000018
+#define MC_XBAR_RDRET_CREDIT2__HUB_LP_RDRET_SKID_MASK 0x00ff0000L
+#define MC_XBAR_RDRET_CREDIT2__HUB_LP_RDRET_SKID__SHIFT 0x00000010
+#define MC_XBAR_RDRET_CREDIT2__OUT4_MASK 0x000000ffL
+#define MC_XBAR_RDRET_CREDIT2__OUT4__SHIFT 0x00000000
+#define MC_XBAR_RDRET_CREDIT2__OUT5_MASK 0x0000ff00L
+#define MC_XBAR_RDRET_CREDIT2__OUT5__SHIFT 0x00000008
+#define MC_XBAR_RDRET_PRI_CREDIT1__OUT0_MASK 0x000000ffL
+#define MC_XBAR_RDRET_PRI_CREDIT1__OUT0__SHIFT 0x00000000
+#define MC_XBAR_RDRET_PRI_CREDIT1__OUT1_MASK 0x0000ff00L
+#define MC_XBAR_RDRET_PRI_CREDIT1__OUT1__SHIFT 0x00000008
+#define MC_XBAR_RDRET_PRI_CREDIT1__OUT2_MASK 0x00ff0000L
+#define MC_XBAR_RDRET_PRI_CREDIT1__OUT2__SHIFT 0x00000010
+#define MC_XBAR_RDRET_PRI_CREDIT1__OUT3_MASK 0xff000000L
+#define MC_XBAR_RDRET_PRI_CREDIT1__OUT3__SHIFT 0x00000018
+#define MC_XBAR_RDRET_PRI_CREDIT2__OUT4_MASK 0x000000ffL
+#define MC_XBAR_RDRET_PRI_CREDIT2__OUT4__SHIFT 0x00000000
+#define MC_XBAR_RDRET_PRI_CREDIT2__OUT5_MASK 0x0000ff00L
+#define MC_XBAR_RDRET_PRI_CREDIT2__OUT5__SHIFT 0x00000008
+#define MC_XBAR_REMOTE__RDREQ_EN_GOQ_MASK 0x00000002L
+#define MC_XBAR_REMOTE__RDREQ_EN_GOQ__SHIFT 0x00000001
+#define MC_XBAR_REMOTE__WRREQ_EN_GOQ_MASK 0x00000001L
+#define MC_XBAR_REMOTE__WRREQ_EN_GOQ__SHIFT 0x00000000
+#define MC_XBAR_SPARE0__BIT_MASK 0xffffffffL
+#define MC_XBAR_SPARE0__BIT__SHIFT 0x00000000
+#define MC_XBAR_SPARE1__BIT_MASK 0xffffffffL
+#define MC_XBAR_SPARE1__BIT__SHIFT 0x00000000
+#define MC_XBAR_TWOCHAN__CH0_MASK 0x00000006L
+#define MC_XBAR_TWOCHAN__CH0__SHIFT 0x00000001
+#define MC_XBAR_TWOCHAN__CH1_MASK 0x00000018L
+#define MC_XBAR_TWOCHAN__CH1__SHIFT 0x00000003
+#define MC_XBAR_TWOCHAN__DISABLE_ONEPORT_MASK 0x00000001L
+#define MC_XBAR_TWOCHAN__DISABLE_ONEPORT__SHIFT 0x00000000
+#define MC_XBAR_WRREQ_CREDIT__OUT0_MASK 0x000000ffL
+#define MC_XBAR_WRREQ_CREDIT__OUT0__SHIFT 0x00000000
+#define MC_XBAR_WRREQ_CREDIT__OUT1_MASK 0x0000ff00L
+#define MC_XBAR_WRREQ_CREDIT__OUT1__SHIFT 0x00000008
+#define MC_XBAR_WRREQ_CREDIT__OUT2_MASK 0x00ff0000L
+#define MC_XBAR_WRREQ_CREDIT__OUT2__SHIFT 0x00000010
+#define MC_XBAR_WRREQ_CREDIT__OUT3_MASK 0xff000000L
+#define MC_XBAR_WRREQ_CREDIT__OUT3__SHIFT 0x00000018
+#define MC_XBAR_WRRET_CREDIT1__OUT0_MASK 0x000000ffL
+#define MC_XBAR_WRRET_CREDIT1__OUT0__SHIFT 0x00000000
+#define MC_XBAR_WRRET_CREDIT1__OUT1_MASK 0x0000ff00L
+#define MC_XBAR_WRRET_CREDIT1__OUT1__SHIFT 0x00000008
+#define MC_XBAR_WRRET_CREDIT1__OUT2_MASK 0x00ff0000L
+#define MC_XBAR_WRRET_CREDIT1__OUT2__SHIFT 0x00000010
+#define MC_XBAR_WRRET_CREDIT1__OUT3_MASK 0xff000000L
+#define MC_XBAR_WRRET_CREDIT1__OUT3__SHIFT 0x00000018
+#define MC_XBAR_WRRET_CREDIT2__OUT4_MASK 0x000000ffL
+#define MC_XBAR_WRRET_CREDIT2__OUT4__SHIFT 0x00000000
+#define MC_XBAR_WRRET_CREDIT2__OUT5_MASK 0x0000ff00L
+#define MC_XBAR_WRRET_CREDIT2__OUT5__SHIFT 0x00000008
+#define MC_XPB_CLG_CFG0__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG0__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG0__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG0__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG0__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG0__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG0__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG0__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG0__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG0__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG10__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG10__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG10__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG10__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG10__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG10__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG10__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG10__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG10__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG10__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG11__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG11__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG11__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG11__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG11__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG11__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG11__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG11__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG11__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG11__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG12__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG12__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG12__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG12__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG12__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG12__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG12__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG12__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG12__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG12__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG13__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG13__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG13__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG13__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG13__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG13__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG13__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG13__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG13__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG13__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG14__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG14__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG14__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG14__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG14__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG14__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG14__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG14__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG14__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG14__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG15__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG15__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG15__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG15__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG15__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG15__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG15__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG15__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG15__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG15__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG16__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG16__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG16__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG16__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG16__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG16__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG16__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG16__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG16__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG16__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG17__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG17__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG17__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG17__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG17__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG17__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG17__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG17__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG17__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG17__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG18__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG18__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG18__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG18__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG18__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG18__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG18__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG18__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG18__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG18__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG19__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG19__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG19__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG19__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG19__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG19__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG19__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG19__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG19__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG19__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG1__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG1__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG1__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG1__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG1__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG1__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG1__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG1__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG1__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG1__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG20__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG20__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG20__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG20__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG20__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG20__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG20__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG20__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG20__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG20__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG21__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG21__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG21__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG21__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG21__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG21__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG21__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG21__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG21__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG21__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG22__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG22__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG22__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG22__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG22__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG22__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG22__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG22__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG22__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG22__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG23__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG23__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG23__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG23__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG23__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG23__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG23__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG23__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG23__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG23__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG24__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG24__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG24__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG24__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG24__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG24__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG24__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG24__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG24__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG24__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG25__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG25__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG25__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG25__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG25__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG25__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG25__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG25__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG25__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG25__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG26__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG26__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG26__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG26__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG26__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG26__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG26__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG26__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG26__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG26__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG27__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG27__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG27__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG27__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG27__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG27__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG27__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG27__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG27__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG27__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG28__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG28__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG28__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG28__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG28__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG28__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG28__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG28__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG28__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG28__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG29__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG29__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG29__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG29__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG29__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG29__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG29__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG29__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG29__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG29__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG2__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG2__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG2__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG2__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG2__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG2__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG2__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG2__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG2__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG2__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG30__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG30__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG30__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG30__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG30__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG30__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG30__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG30__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG30__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG30__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG31__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG31__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG31__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG31__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG31__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG31__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG31__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG31__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG31__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG31__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG32__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG32__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG32__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG32__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG32__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG32__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG32__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG32__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG32__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG32__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG33__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG33__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG33__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG33__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG33__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG33__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG33__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG33__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG33__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG33__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG34__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG34__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG34__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG34__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG34__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG34__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG34__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG34__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG34__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG34__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG35__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG35__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG35__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG35__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG35__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG35__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG35__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG35__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG35__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG35__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG36__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG36__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG36__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG36__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG36__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG36__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG36__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG36__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG36__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG36__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG3__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG3__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG3__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG3__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG3__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG3__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG3__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG3__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG3__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG3__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG4__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG4__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG4__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG4__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG4__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG4__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG4__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG4__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG4__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG4__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG5__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG5__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG5__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG5__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG5__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG5__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG5__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG5__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG5__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG5__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG6__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG6__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG6__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG6__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG6__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG6__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG6__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG6__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG6__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG6__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG7__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG7__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG7__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG7__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG7__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG7__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG7__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG7__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG7__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG7__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG8__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG8__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG8__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG8__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG8__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG8__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG8__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG8__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG8__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG8__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_CFG9__HOST_FLUSH_MASK 0x00003c00L
+#define MC_XPB_CLG_CFG9__HOST_FLUSH__SHIFT 0x0000000a
+#define MC_XPB_CLG_CFG9__LB_TYPE_MASK 0x00000070L
+#define MC_XPB_CLG_CFG9__LB_TYPE__SHIFT 0x00000004
+#define MC_XPB_CLG_CFG9__P2P_BAR_MASK 0x00000380L
+#define MC_XPB_CLG_CFG9__P2P_BAR__SHIFT 0x00000007
+#define MC_XPB_CLG_CFG9__SIDE_FLUSH_MASK 0x0003c000L
+#define MC_XPB_CLG_CFG9__SIDE_FLUSH__SHIFT 0x0000000e
+#define MC_XPB_CLG_CFG9__WCB_NUM_MASK 0x0000000fL
+#define MC_XPB_CLG_CFG9__WCB_NUM__SHIFT 0x00000000
+#define MC_XPB_CLG_EXTRA__CMP0_MASK 0x000000ffL
+#define MC_XPB_CLG_EXTRA__CMP0__SHIFT 0x00000000
+#define MC_XPB_CLG_EXTRA__CMP1_MASK 0x01fe0000L
+#define MC_XPB_CLG_EXTRA__CMP1__SHIFT 0x00000011
+#define MC_XPB_CLG_EXTRA__MSK0_MASK 0x0000ff00L
+#define MC_XPB_CLG_EXTRA__MSK0__SHIFT 0x00000008
+#define MC_XPB_CLG_EXTRA_RD__CMP0_MASK 0x000000ffL
+#define MC_XPB_CLG_EXTRA_RD__CMP0__SHIFT 0x00000000
+#define MC_XPB_CLG_EXTRA_RD__CMP1_MASK 0x01fe0000L
+#define MC_XPB_CLG_EXTRA_RD__CMP1__SHIFT 0x00000011
+#define MC_XPB_CLG_EXTRA_RD__MSK0_MASK 0x0000ff00L
+#define MC_XPB_CLG_EXTRA_RD__MSK0__SHIFT 0x00000008
+#define MC_XPB_CLG_EXTRA_RD__VLD0_MASK 0x00010000L
+#define MC_XPB_CLG_EXTRA_RD__VLD0__SHIFT 0x00000010
+#define MC_XPB_CLG_EXTRA_RD__VLD1_MASK 0x02000000L
+#define MC_XPB_CLG_EXTRA_RD__VLD1__SHIFT 0x00000019
+#define MC_XPB_CLG_EXTRA__VLD0_MASK 0x00010000L
+#define MC_XPB_CLG_EXTRA__VLD0__SHIFT 0x00000010
+#define MC_XPB_CLG_EXTRA__VLD1_MASK 0x02000000L
+#define MC_XPB_CLG_EXTRA__VLD1__SHIFT 0x00000019
+#define MC_XPB_CLK_GAT__ENABLE_MASK 0x00040000L
+#define MC_XPB_CLK_GAT__ENABLE__SHIFT 0x00000012
+#define MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK 0x00080000L
+#define MC_XPB_CLK_GAT__MEM_LS_ENABLE__SHIFT 0x00000013
+#define MC_XPB_CLK_GAT__OFFDLY_MASK 0x00000fc0L
+#define MC_XPB_CLK_GAT__OFFDLY__SHIFT 0x00000006
+#define MC_XPB_CLK_GAT__ONDLY_MASK 0x0000003fL
+#define MC_XPB_CLK_GAT__ONDLY__SHIFT 0x00000000
+#define MC_XPB_CLK_GAT__RDYDLY_MASK 0x0003f000L
+#define MC_XPB_CLK_GAT__RDYDLY__SHIFT 0x0000000c
+#define MC_XPB_INTF_CFG2__RPB_RDREQ_CRD_MASK 0x000000ffL
+#define MC_XPB_INTF_CFG2__RPB_RDREQ_CRD__SHIFT 0x00000000
+#define MC_XPB_INTF_CFG__BIF_MEM_SNOOP_SEL_MASK 0x02000000L
+#define MC_XPB_INTF_CFG__BIF_MEM_SNOOP_SEL__SHIFT 0x00000019
+#define MC_XPB_INTF_CFG__BIF_MEM_SNOOP_VAL_MASK 0x04000000L
+#define MC_XPB_INTF_CFG__BIF_MEM_SNOOP_VAL__SHIFT 0x0000001a
+#define MC_XPB_INTF_CFG__BIF_REG_SNOOP_SEL_MASK 0x00800000L
+#define MC_XPB_INTF_CFG__BIF_REG_SNOOP_SEL__SHIFT 0x00000017
+#define MC_XPB_INTF_CFG__BIF_REG_SNOOP_VAL_MASK 0x01000000L
+#define MC_XPB_INTF_CFG__BIF_REG_SNOOP_VAL__SHIFT 0x00000018
+#define MC_XPB_INTF_CFG__MC_WRRET_ASK_MASK 0x0000ff00L
+#define MC_XPB_INTF_CFG__MC_WRRET_ASK__SHIFT 0x00000008
+#define MC_XPB_INTF_CFG__RPB_WRREQ_CRD_MASK 0x000000ffL
+#define MC_XPB_INTF_CFG__RPB_WRREQ_CRD__SHIFT 0x00000000
+#define MC_XPB_INTF_CFG__XSP_ORDERING_SEL_MASK 0x40000000L
+#define MC_XPB_INTF_CFG__XSP_ORDERING_SEL__SHIFT 0x0000001e
+#define MC_XPB_INTF_CFG__XSP_ORDERING_VAL_MASK 0x80000000L
+#define MC_XPB_INTF_CFG__XSP_ORDERING_VAL__SHIFT 0x0000001f
+#define MC_XPB_INTF_CFG__XSP_REQ_CRD_MASK 0x007f0000L
+#define MC_XPB_INTF_CFG__XSP_REQ_CRD__SHIFT 0x00000010
+#define MC_XPB_INTF_CFG__XSP_SNOOP_SEL_MASK 0x18000000L
+#define MC_XPB_INTF_CFG__XSP_SNOOP_SEL__SHIFT 0x0000001b
+#define MC_XPB_INTF_CFG__XSP_SNOOP_VAL_MASK 0x20000000L
+#define MC_XPB_INTF_CFG__XSP_SNOOP_VAL__SHIFT 0x0000001d
+#define MC_XPB_INTF_STS__CNS_BUF_BUSY_MASK 0x00040000L
+#define MC_XPB_INTF_STS__CNS_BUF_BUSY__SHIFT 0x00000012
+#define MC_XPB_INTF_STS__CNS_BUF_FULL_MASK 0x00020000L
+#define MC_XPB_INTF_STS__CNS_BUF_FULL__SHIFT 0x00000011
+#define MC_XPB_INTF_STS__HOP_ATTR_BUF_FULL_MASK 0x00010000L
+#define MC_XPB_INTF_STS__HOP_ATTR_BUF_FULL__SHIFT 0x00000010
+#define MC_XPB_INTF_STS__HOP_DATA_BUF_FULL_MASK 0x00008000L
+#define MC_XPB_INTF_STS__HOP_DATA_BUF_FULL__SHIFT 0x0000000f
+#define MC_XPB_INTF_STS__RPB_RDREQ_CRD_MASK 0x07f80000L
+#define MC_XPB_INTF_STS__RPB_RDREQ_CRD__SHIFT 0x00000013
+#define MC_XPB_INTF_STS__RPB_WRREQ_CRD_MASK 0x000000ffL
+#define MC_XPB_INTF_STS__RPB_WRREQ_CRD__SHIFT 0x00000000
+#define MC_XPB_INTF_STS__XSP_REQ_CRD_MASK 0x00007f00L
+#define MC_XPB_INTF_STS__XSP_REQ_CRD__SHIFT 0x00000008
+#define MC_XPB_LB_ADDR__CMP0_MASK 0x000003ffL
+#define MC_XPB_LB_ADDR__CMP0__SHIFT 0x00000000
+#define MC_XPB_LB_ADDR__CMP1_MASK 0x03f00000L
+#define MC_XPB_LB_ADDR__CMP1__SHIFT 0x00000014
+#define MC_XPB_LB_ADDR__MASK0_MASK 0x000ffc00L
+#define MC_XPB_LB_ADDR__MASK0__SHIFT 0x0000000a
+#define MC_XPB_LB_ADDR__MASK1_MASK 0xfc000000L
+#define MC_XPB_LB_ADDR__MASK1__SHIFT 0x0000001a
+#define MC_XPB_MAP_INVERT_FLUSH_NUM_LSB__ALTER_FLUSH_NUM_MASK 0x0000ffffL
+#define MC_XPB_MAP_INVERT_FLUSH_NUM_LSB__ALTER_FLUSH_NUM__SHIFT 0x00000000
+#define MC_XPB_MISC_CFG__FIELDNAME0_MASK 0x000000ffL
+#define MC_XPB_MISC_CFG__FIELDNAME0__SHIFT 0x00000000
+#define MC_XPB_MISC_CFG__FIELDNAME1_MASK 0x0000ff00L
+#define MC_XPB_MISC_CFG__FIELDNAME1__SHIFT 0x00000008
+#define MC_XPB_MISC_CFG__FIELDNAME2_MASK 0x00ff0000L
+#define MC_XPB_MISC_CFG__FIELDNAME2__SHIFT 0x00000010
+#define MC_XPB_MISC_CFG__FIELDNAME3_MASK 0x7f000000L
+#define MC_XPB_MISC_CFG__FIELDNAME3__SHIFT 0x00000018
+#define MC_XPB_MISC_CFG__TRIGGERNAME_MASK 0x80000000L
+#define MC_XPB_MISC_CFG__TRIGGERNAME__SHIFT 0x0000001f
+#define MC_XPB_P2P_BAR0__ADDRESS_MASK 0xffff0000L
+#define MC_XPB_P2P_BAR0__ADDRESS__SHIFT 0x00000010
+#define MC_XPB_P2P_BAR0__COMPRESS_DIS_MASK 0x00004000L
+#define MC_XPB_P2P_BAR0__COMPRESS_DIS__SHIFT 0x0000000e
+#define MC_XPB_P2P_BAR0__HOST_FLUSH_MASK 0x0000000fL
+#define MC_XPB_P2P_BAR0__HOST_FLUSH__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR0__MEM_SYS_BAR_MASK 0x00000f00L
+#define MC_XPB_P2P_BAR0__MEM_SYS_BAR__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR0__REG_SYS_BAR_MASK 0x000000f0L
+#define MC_XPB_P2P_BAR0__REG_SYS_BAR__SHIFT 0x00000004
+#define MC_XPB_P2P_BAR0__RESERVED_MASK 0x00008000L
+#define MC_XPB_P2P_BAR0__RESERVED__SHIFT 0x0000000f
+#define MC_XPB_P2P_BAR0__SEND_DIS_MASK 0x00002000L
+#define MC_XPB_P2P_BAR0__SEND_DIS__SHIFT 0x0000000d
+#define MC_XPB_P2P_BAR0__VALID_MASK 0x00001000L
+#define MC_XPB_P2P_BAR0__VALID__SHIFT 0x0000000c
+#define MC_XPB_P2P_BAR1__ADDRESS_MASK 0xffff0000L
+#define MC_XPB_P2P_BAR1__ADDRESS__SHIFT 0x00000010
+#define MC_XPB_P2P_BAR1__COMPRESS_DIS_MASK 0x00004000L
+#define MC_XPB_P2P_BAR1__COMPRESS_DIS__SHIFT 0x0000000e
+#define MC_XPB_P2P_BAR1__HOST_FLUSH_MASK 0x0000000fL
+#define MC_XPB_P2P_BAR1__HOST_FLUSH__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR1__MEM_SYS_BAR_MASK 0x00000f00L
+#define MC_XPB_P2P_BAR1__MEM_SYS_BAR__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR1__REG_SYS_BAR_MASK 0x000000f0L
+#define MC_XPB_P2P_BAR1__REG_SYS_BAR__SHIFT 0x00000004
+#define MC_XPB_P2P_BAR1__RESERVED_MASK 0x00008000L
+#define MC_XPB_P2P_BAR1__RESERVED__SHIFT 0x0000000f
+#define MC_XPB_P2P_BAR1__SEND_DIS_MASK 0x00002000L
+#define MC_XPB_P2P_BAR1__SEND_DIS__SHIFT 0x0000000d
+#define MC_XPB_P2P_BAR1__VALID_MASK 0x00001000L
+#define MC_XPB_P2P_BAR1__VALID__SHIFT 0x0000000c
+#define MC_XPB_P2P_BAR2__ADDRESS_MASK 0xffff0000L
+#define MC_XPB_P2P_BAR2__ADDRESS__SHIFT 0x00000010
+#define MC_XPB_P2P_BAR2__COMPRESS_DIS_MASK 0x00004000L
+#define MC_XPB_P2P_BAR2__COMPRESS_DIS__SHIFT 0x0000000e
+#define MC_XPB_P2P_BAR2__HOST_FLUSH_MASK 0x0000000fL
+#define MC_XPB_P2P_BAR2__HOST_FLUSH__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR2__MEM_SYS_BAR_MASK 0x00000f00L
+#define MC_XPB_P2P_BAR2__MEM_SYS_BAR__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR2__REG_SYS_BAR_MASK 0x000000f0L
+#define MC_XPB_P2P_BAR2__REG_SYS_BAR__SHIFT 0x00000004
+#define MC_XPB_P2P_BAR2__RESERVED_MASK 0x00008000L
+#define MC_XPB_P2P_BAR2__RESERVED__SHIFT 0x0000000f
+#define MC_XPB_P2P_BAR2__SEND_DIS_MASK 0x00002000L
+#define MC_XPB_P2P_BAR2__SEND_DIS__SHIFT 0x0000000d
+#define MC_XPB_P2P_BAR2__VALID_MASK 0x00001000L
+#define MC_XPB_P2P_BAR2__VALID__SHIFT 0x0000000c
+#define MC_XPB_P2P_BAR3__ADDRESS_MASK 0xffff0000L
+#define MC_XPB_P2P_BAR3__ADDRESS__SHIFT 0x00000010
+#define MC_XPB_P2P_BAR3__COMPRESS_DIS_MASK 0x00004000L
+#define MC_XPB_P2P_BAR3__COMPRESS_DIS__SHIFT 0x0000000e
+#define MC_XPB_P2P_BAR3__HOST_FLUSH_MASK 0x0000000fL
+#define MC_XPB_P2P_BAR3__HOST_FLUSH__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR3__MEM_SYS_BAR_MASK 0x00000f00L
+#define MC_XPB_P2P_BAR3__MEM_SYS_BAR__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR3__REG_SYS_BAR_MASK 0x000000f0L
+#define MC_XPB_P2P_BAR3__REG_SYS_BAR__SHIFT 0x00000004
+#define MC_XPB_P2P_BAR3__RESERVED_MASK 0x00008000L
+#define MC_XPB_P2P_BAR3__RESERVED__SHIFT 0x0000000f
+#define MC_XPB_P2P_BAR3__SEND_DIS_MASK 0x00002000L
+#define MC_XPB_P2P_BAR3__SEND_DIS__SHIFT 0x0000000d
+#define MC_XPB_P2P_BAR3__VALID_MASK 0x00001000L
+#define MC_XPB_P2P_BAR3__VALID__SHIFT 0x0000000c
+#define MC_XPB_P2P_BAR4__ADDRESS_MASK 0xffff0000L
+#define MC_XPB_P2P_BAR4__ADDRESS__SHIFT 0x00000010
+#define MC_XPB_P2P_BAR4__COMPRESS_DIS_MASK 0x00004000L
+#define MC_XPB_P2P_BAR4__COMPRESS_DIS__SHIFT 0x0000000e
+#define MC_XPB_P2P_BAR4__HOST_FLUSH_MASK 0x0000000fL
+#define MC_XPB_P2P_BAR4__HOST_FLUSH__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR4__MEM_SYS_BAR_MASK 0x00000f00L
+#define MC_XPB_P2P_BAR4__MEM_SYS_BAR__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR4__REG_SYS_BAR_MASK 0x000000f0L
+#define MC_XPB_P2P_BAR4__REG_SYS_BAR__SHIFT 0x00000004
+#define MC_XPB_P2P_BAR4__RESERVED_MASK 0x00008000L
+#define MC_XPB_P2P_BAR4__RESERVED__SHIFT 0x0000000f
+#define MC_XPB_P2P_BAR4__SEND_DIS_MASK 0x00002000L
+#define MC_XPB_P2P_BAR4__SEND_DIS__SHIFT 0x0000000d
+#define MC_XPB_P2P_BAR4__VALID_MASK 0x00001000L
+#define MC_XPB_P2P_BAR4__VALID__SHIFT 0x0000000c
+#define MC_XPB_P2P_BAR5__ADDRESS_MASK 0xffff0000L
+#define MC_XPB_P2P_BAR5__ADDRESS__SHIFT 0x00000010
+#define MC_XPB_P2P_BAR5__COMPRESS_DIS_MASK 0x00004000L
+#define MC_XPB_P2P_BAR5__COMPRESS_DIS__SHIFT 0x0000000e
+#define MC_XPB_P2P_BAR5__HOST_FLUSH_MASK 0x0000000fL
+#define MC_XPB_P2P_BAR5__HOST_FLUSH__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR5__MEM_SYS_BAR_MASK 0x00000f00L
+#define MC_XPB_P2P_BAR5__MEM_SYS_BAR__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR5__REG_SYS_BAR_MASK 0x000000f0L
+#define MC_XPB_P2P_BAR5__REG_SYS_BAR__SHIFT 0x00000004
+#define MC_XPB_P2P_BAR5__RESERVED_MASK 0x00008000L
+#define MC_XPB_P2P_BAR5__RESERVED__SHIFT 0x0000000f
+#define MC_XPB_P2P_BAR5__SEND_DIS_MASK 0x00002000L
+#define MC_XPB_P2P_BAR5__SEND_DIS__SHIFT 0x0000000d
+#define MC_XPB_P2P_BAR5__VALID_MASK 0x00001000L
+#define MC_XPB_P2P_BAR5__VALID__SHIFT 0x0000000c
+#define MC_XPB_P2P_BAR6__ADDRESS_MASK 0xffff0000L
+#define MC_XPB_P2P_BAR6__ADDRESS__SHIFT 0x00000010
+#define MC_XPB_P2P_BAR6__COMPRESS_DIS_MASK 0x00004000L
+#define MC_XPB_P2P_BAR6__COMPRESS_DIS__SHIFT 0x0000000e
+#define MC_XPB_P2P_BAR6__HOST_FLUSH_MASK 0x0000000fL
+#define MC_XPB_P2P_BAR6__HOST_FLUSH__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR6__MEM_SYS_BAR_MASK 0x00000f00L
+#define MC_XPB_P2P_BAR6__MEM_SYS_BAR__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR6__REG_SYS_BAR_MASK 0x000000f0L
+#define MC_XPB_P2P_BAR6__REG_SYS_BAR__SHIFT 0x00000004
+#define MC_XPB_P2P_BAR6__RESERVED_MASK 0x00008000L
+#define MC_XPB_P2P_BAR6__RESERVED__SHIFT 0x0000000f
+#define MC_XPB_P2P_BAR6__SEND_DIS_MASK 0x00002000L
+#define MC_XPB_P2P_BAR6__SEND_DIS__SHIFT 0x0000000d
+#define MC_XPB_P2P_BAR6__VALID_MASK 0x00001000L
+#define MC_XPB_P2P_BAR6__VALID__SHIFT 0x0000000c
+#define MC_XPB_P2P_BAR7__ADDRESS_MASK 0xffff0000L
+#define MC_XPB_P2P_BAR7__ADDRESS__SHIFT 0x00000010
+#define MC_XPB_P2P_BAR7__COMPRESS_DIS_MASK 0x00004000L
+#define MC_XPB_P2P_BAR7__COMPRESS_DIS__SHIFT 0x0000000e
+#define MC_XPB_P2P_BAR7__HOST_FLUSH_MASK 0x0000000fL
+#define MC_XPB_P2P_BAR7__HOST_FLUSH__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR7__MEM_SYS_BAR_MASK 0x00000f00L
+#define MC_XPB_P2P_BAR7__MEM_SYS_BAR__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR7__REG_SYS_BAR_MASK 0x000000f0L
+#define MC_XPB_P2P_BAR7__REG_SYS_BAR__SHIFT 0x00000004
+#define MC_XPB_P2P_BAR7__RESERVED_MASK 0x00008000L
+#define MC_XPB_P2P_BAR7__RESERVED__SHIFT 0x0000000f
+#define MC_XPB_P2P_BAR7__SEND_DIS_MASK 0x00002000L
+#define MC_XPB_P2P_BAR7__SEND_DIS__SHIFT 0x0000000d
+#define MC_XPB_P2P_BAR7__VALID_MASK 0x00001000L
+#define MC_XPB_P2P_BAR7__VALID__SHIFT 0x0000000c
+#define MC_XPB_P2P_BAR_CFG__ADDR_SIZE_MASK 0x0000000fL
+#define MC_XPB_P2P_BAR_CFG__ADDR_SIZE__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR_CFG__ATC_TRANSLATED_MASK 0x00001000L
+#define MC_XPB_P2P_BAR_CFG__ATC_TRANSLATED__SHIFT 0x0000000c
+#define MC_XPB_P2P_BAR_CFG__COMPRESS_DIS_MASK 0x00000100L
+#define MC_XPB_P2P_BAR_CFG__COMPRESS_DIS__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR_CFG__RD_EN_MASK 0x00000800L
+#define MC_XPB_P2P_BAR_CFG__RD_EN__SHIFT 0x0000000b
+#define MC_XPB_P2P_BAR_CFG__REGBAR_FROM_SYSBAR_MASK 0x00000400L
+#define MC_XPB_P2P_BAR_CFG__REGBAR_FROM_SYSBAR__SHIFT 0x0000000a
+#define MC_XPB_P2P_BAR_CFG__SEND_BAR_MASK 0x00000030L
+#define MC_XPB_P2P_BAR_CFG__SEND_BAR__SHIFT 0x00000004
+#define MC_XPB_P2P_BAR_CFG__SEND_DIS_MASK 0x00000080L
+#define MC_XPB_P2P_BAR_CFG__SEND_DIS__SHIFT 0x00000007
+#define MC_XPB_P2P_BAR_CFG__SNOOP_MASK 0x00000040L
+#define MC_XPB_P2P_BAR_CFG__SNOOP__SHIFT 0x00000006
+#define MC_XPB_P2P_BAR_CFG__UPDATE_DIS_MASK 0x00000200L
+#define MC_XPB_P2P_BAR_CFG__UPDATE_DIS__SHIFT 0x00000009
+#define MC_XPB_P2P_BAR_DEBUG__HOST_FLUSH_MASK 0x00000f00L
+#define MC_XPB_P2P_BAR_DEBUG__HOST_FLUSH__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR_DEBUG__MEM_SYS_BAR_MASK 0x0000f000L
+#define MC_XPB_P2P_BAR_DEBUG__MEM_SYS_BAR__SHIFT 0x0000000c
+#define MC_XPB_P2P_BAR_DEBUG__SEL_MASK 0x000000ffL
+#define MC_XPB_P2P_BAR_DEBUG__SEL__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR_DELTA_ABOVE__DELTA_MASK 0x0fffff00L
+#define MC_XPB_P2P_BAR_DELTA_ABOVE__DELTA__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR_DELTA_ABOVE__EN_MASK 0x000000ffL
+#define MC_XPB_P2P_BAR_DELTA_ABOVE__EN__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR_DELTA_BELOW__DELTA_MASK 0x0fffff00L
+#define MC_XPB_P2P_BAR_DELTA_BELOW__DELTA__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR_DELTA_BELOW__EN_MASK 0x000000ffL
+#define MC_XPB_P2P_BAR_DELTA_BELOW__EN__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR_SETUP__ADDRESS_MASK 0xffff0000L
+#define MC_XPB_P2P_BAR_SETUP__ADDRESS__SHIFT 0x00000010
+#define MC_XPB_P2P_BAR_SETUP__COMPRESS_DIS_MASK 0x00004000L
+#define MC_XPB_P2P_BAR_SETUP__COMPRESS_DIS__SHIFT 0x0000000e
+#define MC_XPB_P2P_BAR_SETUP__REG_SYS_BAR_MASK 0x00000f00L
+#define MC_XPB_P2P_BAR_SETUP__REG_SYS_BAR__SHIFT 0x00000008
+#define MC_XPB_P2P_BAR_SETUP__RESERVED_MASK 0x00008000L
+#define MC_XPB_P2P_BAR_SETUP__RESERVED__SHIFT 0x0000000f
+#define MC_XPB_P2P_BAR_SETUP__SEL_MASK 0x000000ffL
+#define MC_XPB_P2P_BAR_SETUP__SEL__SHIFT 0x00000000
+#define MC_XPB_P2P_BAR_SETUP__SEND_DIS_MASK 0x00002000L
+#define MC_XPB_P2P_BAR_SETUP__SEND_DIS__SHIFT 0x0000000d
+#define MC_XPB_P2P_BAR_SETUP__VALID_MASK 0x00001000L
+#define MC_XPB_P2P_BAR_SETUP__VALID__SHIFT 0x0000000c
+#define MC_XPB_PEER_SYS_BAR0__ADDR_MASK 0x07fffffcL
+#define MC_XPB_PEER_SYS_BAR0__ADDR__SHIFT 0x00000002
+#define MC_XPB_PEER_SYS_BAR0__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_PEER_SYS_BAR0__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_PEER_SYS_BAR0__VALID_MASK 0x00000001L
+#define MC_XPB_PEER_SYS_BAR0__VALID__SHIFT 0x00000000
+#define MC_XPB_PEER_SYS_BAR1__ADDR_MASK 0x07fffffcL
+#define MC_XPB_PEER_SYS_BAR1__ADDR__SHIFT 0x00000002
+#define MC_XPB_PEER_SYS_BAR1__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_PEER_SYS_BAR1__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_PEER_SYS_BAR1__VALID_MASK 0x00000001L
+#define MC_XPB_PEER_SYS_BAR1__VALID__SHIFT 0x00000000
+#define MC_XPB_PEER_SYS_BAR2__ADDR_MASK 0x07fffffcL
+#define MC_XPB_PEER_SYS_BAR2__ADDR__SHIFT 0x00000002
+#define MC_XPB_PEER_SYS_BAR2__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_PEER_SYS_BAR2__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_PEER_SYS_BAR2__VALID_MASK 0x00000001L
+#define MC_XPB_PEER_SYS_BAR2__VALID__SHIFT 0x00000000
+#define MC_XPB_PEER_SYS_BAR3__ADDR_MASK 0x07fffffcL
+#define MC_XPB_PEER_SYS_BAR3__ADDR__SHIFT 0x00000002
+#define MC_XPB_PEER_SYS_BAR3__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_PEER_SYS_BAR3__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_PEER_SYS_BAR3__VALID_MASK 0x00000001L
+#define MC_XPB_PEER_SYS_BAR3__VALID__SHIFT 0x00000000
+#define MC_XPB_PEER_SYS_BAR4__ADDR_MASK 0x07fffffcL
+#define MC_XPB_PEER_SYS_BAR4__ADDR__SHIFT 0x00000002
+#define MC_XPB_PEER_SYS_BAR4__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_PEER_SYS_BAR4__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_PEER_SYS_BAR4__VALID_MASK 0x00000001L
+#define MC_XPB_PEER_SYS_BAR4__VALID__SHIFT 0x00000000
+#define MC_XPB_PEER_SYS_BAR5__ADDR_MASK 0x07fffffcL
+#define MC_XPB_PEER_SYS_BAR5__ADDR__SHIFT 0x00000002
+#define MC_XPB_PEER_SYS_BAR5__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_PEER_SYS_BAR5__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_PEER_SYS_BAR5__VALID_MASK 0x00000001L
+#define MC_XPB_PEER_SYS_BAR5__VALID__SHIFT 0x00000000
+#define MC_XPB_PEER_SYS_BAR6__ADDR_MASK 0x07fffffcL
+#define MC_XPB_PEER_SYS_BAR6__ADDR__SHIFT 0x00000002
+#define MC_XPB_PEER_SYS_BAR6__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_PEER_SYS_BAR6__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_PEER_SYS_BAR6__VALID_MASK 0x00000001L
+#define MC_XPB_PEER_SYS_BAR6__VALID__SHIFT 0x00000000
+#define MC_XPB_PEER_SYS_BAR7__ADDR_MASK 0x07fffffcL
+#define MC_XPB_PEER_SYS_BAR7__ADDR__SHIFT 0x00000002
+#define MC_XPB_PEER_SYS_BAR7__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_PEER_SYS_BAR7__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_PEER_SYS_BAR7__VALID_MASK 0x00000001L
+#define MC_XPB_PEER_SYS_BAR7__VALID__SHIFT 0x00000000
+#define MC_XPB_PEER_SYS_BAR8__ADDR_MASK 0x07fffffcL
+#define MC_XPB_PEER_SYS_BAR8__ADDR__SHIFT 0x00000002
+#define MC_XPB_PEER_SYS_BAR8__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_PEER_SYS_BAR8__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_PEER_SYS_BAR8__VALID_MASK 0x00000001L
+#define MC_XPB_PEER_SYS_BAR8__VALID__SHIFT 0x00000000
+#define MC_XPB_PEER_SYS_BAR9__ADDR_MASK 0x07fffffcL
+#define MC_XPB_PEER_SYS_BAR9__ADDR__SHIFT 0x00000002
+#define MC_XPB_PEER_SYS_BAR9__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_PEER_SYS_BAR9__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_PEER_SYS_BAR9__VALID_MASK 0x00000001L
+#define MC_XPB_PEER_SYS_BAR9__VALID__SHIFT 0x00000000
+#define MC_XPB_PERF_KNOBS__CNS_FIFO_DEPTH_MASK 0x0000003fL
+#define MC_XPB_PERF_KNOBS__CNS_FIFO_DEPTH__SHIFT 0x00000000
+#define MC_XPB_PERF_KNOBS__WCB_HST_FIFO_DEPTH_MASK 0x00000fc0L
+#define MC_XPB_PERF_KNOBS__WCB_HST_FIFO_DEPTH__SHIFT 0x00000006
+#define MC_XPB_PERF_KNOBS__WCB_SID_FIFO_DEPTH_MASK 0x0003f000L
+#define MC_XPB_PERF_KNOBS__WCB_SID_FIFO_DEPTH__SHIFT 0x0000000c
+#define MC_XPB_PIPE_STS__RET_BUF_FULL_MASK 0x00800000L
+#define MC_XPB_PIPE_STS__RET_BUF_FULL__SHIFT 0x00000017
+#define MC_XPB_PIPE_STS__WCB_ANY_PBUF_MASK 0x00000001L
+#define MC_XPB_PIPE_STS__WCB_ANY_PBUF__SHIFT 0x00000000
+#define MC_XPB_PIPE_STS__WCB_HST_DATA_BUF_CNT_MASK 0x000000feL
+#define MC_XPB_PIPE_STS__WCB_HST_DATA_BUF_CNT__SHIFT 0x00000001
+#define MC_XPB_PIPE_STS__WCB_HST_DATA_OBUF_FULL_MASK 0x00200000L
+#define MC_XPB_PIPE_STS__WCB_HST_DATA_OBUF_FULL__SHIFT 0x00000015
+#define MC_XPB_PIPE_STS__WCB_HST_RD_PTR_BUF_FULL_MASK 0x00008000L
+#define MC_XPB_PIPE_STS__WCB_HST_RD_PTR_BUF_FULL__SHIFT 0x0000000f
+#define MC_XPB_PIPE_STS__WCB_HST_REQ_FIFO_FULL_MASK 0x00020000L
+#define MC_XPB_PIPE_STS__WCB_HST_REQ_FIFO_FULL__SHIFT 0x00000011
+#define MC_XPB_PIPE_STS__WCB_HST_REQ_OBUF_FULL_MASK 0x00080000L
+#define MC_XPB_PIPE_STS__WCB_HST_REQ_OBUF_FULL__SHIFT 0x00000013
+#define MC_XPB_PIPE_STS__WCB_SID_DATA_BUF_CNT_MASK 0x00007f00L
+#define MC_XPB_PIPE_STS__WCB_SID_DATA_BUF_CNT__SHIFT 0x00000008
+#define MC_XPB_PIPE_STS__WCB_SID_DATA_OBUF_FULL_MASK 0x00400000L
+#define MC_XPB_PIPE_STS__WCB_SID_DATA_OBUF_FULL__SHIFT 0x00000016
+#define MC_XPB_PIPE_STS__WCB_SID_RD_PTR_BUF_FULL_MASK 0x00010000L
+#define MC_XPB_PIPE_STS__WCB_SID_RD_PTR_BUF_FULL__SHIFT 0x00000010
+#define MC_XPB_PIPE_STS__WCB_SID_REQ_FIFO_FULL_MASK 0x00040000L
+#define MC_XPB_PIPE_STS__WCB_SID_REQ_FIFO_FULL__SHIFT 0x00000012
+#define MC_XPB_PIPE_STS__WCB_SID_REQ_OBUF_FULL_MASK 0x00100000L
+#define MC_XPB_PIPE_STS__WCB_SID_REQ_OBUF_FULL__SHIFT 0x00000014
+#define MC_XPB_PIPE_STS__XPB_CLK_BUSY_BITS_MASK 0xff000000L
+#define MC_XPB_PIPE_STS__XPB_CLK_BUSY_BITS__SHIFT 0x00000018
+#define MC_XPB_RTR_DEST_MAP0__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_RTR_DEST_MAP0__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_RTR_DEST_MAP0__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_RTR_DEST_MAP0__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_RTR_DEST_MAP0__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_RTR_DEST_MAP0__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_RTR_DEST_MAP0__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_RTR_DEST_MAP0__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_RTR_DEST_MAP0__NMR_MASK 0x00000001L
+#define MC_XPB_RTR_DEST_MAP0__NMR__SHIFT 0x00000000
+#define MC_XPB_RTR_DEST_MAP0__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_RTR_DEST_MAP0__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_RTR_DEST_MAP1__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_RTR_DEST_MAP1__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_RTR_DEST_MAP1__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_RTR_DEST_MAP1__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_RTR_DEST_MAP1__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_RTR_DEST_MAP1__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_RTR_DEST_MAP1__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_RTR_DEST_MAP1__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_RTR_DEST_MAP1__NMR_MASK 0x00000001L
+#define MC_XPB_RTR_DEST_MAP1__NMR__SHIFT 0x00000000
+#define MC_XPB_RTR_DEST_MAP1__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_RTR_DEST_MAP1__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_RTR_DEST_MAP2__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_RTR_DEST_MAP2__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_RTR_DEST_MAP2__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_RTR_DEST_MAP2__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_RTR_DEST_MAP2__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_RTR_DEST_MAP2__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_RTR_DEST_MAP2__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_RTR_DEST_MAP2__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_RTR_DEST_MAP2__NMR_MASK 0x00000001L
+#define MC_XPB_RTR_DEST_MAP2__NMR__SHIFT 0x00000000
+#define MC_XPB_RTR_DEST_MAP2__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_RTR_DEST_MAP2__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_RTR_DEST_MAP3__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_RTR_DEST_MAP3__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_RTR_DEST_MAP3__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_RTR_DEST_MAP3__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_RTR_DEST_MAP3__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_RTR_DEST_MAP3__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_RTR_DEST_MAP3__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_RTR_DEST_MAP3__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_RTR_DEST_MAP3__NMR_MASK 0x00000001L
+#define MC_XPB_RTR_DEST_MAP3__NMR__SHIFT 0x00000000
+#define MC_XPB_RTR_DEST_MAP3__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_RTR_DEST_MAP3__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_RTR_DEST_MAP4__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_RTR_DEST_MAP4__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_RTR_DEST_MAP4__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_RTR_DEST_MAP4__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_RTR_DEST_MAP4__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_RTR_DEST_MAP4__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_RTR_DEST_MAP4__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_RTR_DEST_MAP4__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_RTR_DEST_MAP4__NMR_MASK 0x00000001L
+#define MC_XPB_RTR_DEST_MAP4__NMR__SHIFT 0x00000000
+#define MC_XPB_RTR_DEST_MAP4__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_RTR_DEST_MAP4__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_RTR_DEST_MAP5__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_RTR_DEST_MAP5__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_RTR_DEST_MAP5__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_RTR_DEST_MAP5__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_RTR_DEST_MAP5__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_RTR_DEST_MAP5__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_RTR_DEST_MAP5__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_RTR_DEST_MAP5__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_RTR_DEST_MAP5__NMR_MASK 0x00000001L
+#define MC_XPB_RTR_DEST_MAP5__NMR__SHIFT 0x00000000
+#define MC_XPB_RTR_DEST_MAP5__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_RTR_DEST_MAP5__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_RTR_DEST_MAP6__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_RTR_DEST_MAP6__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_RTR_DEST_MAP6__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_RTR_DEST_MAP6__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_RTR_DEST_MAP6__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_RTR_DEST_MAP6__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_RTR_DEST_MAP6__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_RTR_DEST_MAP6__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_RTR_DEST_MAP6__NMR_MASK 0x00000001L
+#define MC_XPB_RTR_DEST_MAP6__NMR__SHIFT 0x00000000
+#define MC_XPB_RTR_DEST_MAP6__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_RTR_DEST_MAP6__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_RTR_DEST_MAP7__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_RTR_DEST_MAP7__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_RTR_DEST_MAP7__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_RTR_DEST_MAP7__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_RTR_DEST_MAP7__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_RTR_DEST_MAP7__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_RTR_DEST_MAP7__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_RTR_DEST_MAP7__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_RTR_DEST_MAP7__NMR_MASK 0x00000001L
+#define MC_XPB_RTR_DEST_MAP7__NMR__SHIFT 0x00000000
+#define MC_XPB_RTR_DEST_MAP7__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_RTR_DEST_MAP7__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_RTR_DEST_MAP8__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_RTR_DEST_MAP8__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_RTR_DEST_MAP8__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_RTR_DEST_MAP8__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_RTR_DEST_MAP8__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_RTR_DEST_MAP8__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_RTR_DEST_MAP8__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_RTR_DEST_MAP8__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_RTR_DEST_MAP8__NMR_MASK 0x00000001L
+#define MC_XPB_RTR_DEST_MAP8__NMR__SHIFT 0x00000000
+#define MC_XPB_RTR_DEST_MAP8__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_RTR_DEST_MAP8__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_RTR_DEST_MAP9__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_RTR_DEST_MAP9__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_RTR_DEST_MAP9__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_RTR_DEST_MAP9__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_RTR_DEST_MAP9__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_RTR_DEST_MAP9__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_RTR_DEST_MAP9__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_RTR_DEST_MAP9__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_RTR_DEST_MAP9__NMR_MASK 0x00000001L
+#define MC_XPB_RTR_DEST_MAP9__NMR__SHIFT 0x00000000
+#define MC_XPB_RTR_DEST_MAP9__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_RTR_DEST_MAP9__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_RTR_SRC_APRTR0__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_RTR_SRC_APRTR0__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_RTR_SRC_APRTR1__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_RTR_SRC_APRTR1__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_RTR_SRC_APRTR2__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_RTR_SRC_APRTR2__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_RTR_SRC_APRTR3__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_RTR_SRC_APRTR3__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_RTR_SRC_APRTR4__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_RTR_SRC_APRTR4__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_RTR_SRC_APRTR5__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_RTR_SRC_APRTR5__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_RTR_SRC_APRTR6__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_RTR_SRC_APRTR6__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_RTR_SRC_APRTR7__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_RTR_SRC_APRTR7__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_RTR_SRC_APRTR8__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_RTR_SRC_APRTR8__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_RTR_SRC_APRTR9__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_RTR_SRC_APRTR9__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_STICKY__BITS_MASK 0xffffffffL
+#define MC_XPB_STICKY__BITS__SHIFT 0x00000000
+#define MC_XPB_STICKY_W1C__BITS_MASK 0xffffffffL
+#define MC_XPB_STICKY_W1C__BITS__SHIFT 0x00000000
+#define MC_XPB_SUB_CTRL__RESET_CGR_MASK 0x00080000L
+#define MC_XPB_SUB_CTRL__RESET_CGR__SHIFT 0x00000013
+#define MC_XPB_SUB_CTRL__RESET_CNS_MASK 0x00000400L
+#define MC_XPB_SUB_CTRL__RESET_CNS__SHIFT 0x0000000a
+#define MC_XPB_SUB_CTRL__RESET_HOP_MASK 0x00010000L
+#define MC_XPB_SUB_CTRL__RESET_HOP__SHIFT 0x00000010
+#define MC_XPB_SUB_CTRL__RESET_HST_MASK 0x00008000L
+#define MC_XPB_SUB_CTRL__RESET_HST__SHIFT 0x0000000f
+#define MC_XPB_SUB_CTRL__RESET_MAP_MASK 0x00002000L
+#define MC_XPB_SUB_CTRL__RESET_MAP__SHIFT 0x0000000d
+#define MC_XPB_SUB_CTRL__RESET_RET_MASK 0x00001000L
+#define MC_XPB_SUB_CTRL__RESET_RET__SHIFT 0x0000000c
+#define MC_XPB_SUB_CTRL__RESET_RTR_MASK 0x00000800L
+#define MC_XPB_SUB_CTRL__RESET_RTR__SHIFT 0x0000000b
+#define MC_XPB_SUB_CTRL__RESET_SID_MASK 0x00020000L
+#define MC_XPB_SUB_CTRL__RESET_SID__SHIFT 0x00000011
+#define MC_XPB_SUB_CTRL__RESET_SRB_MASK 0x00040000L
+#define MC_XPB_SUB_CTRL__RESET_SRB__SHIFT 0x00000012
+#define MC_XPB_SUB_CTRL__RESET_WCB_MASK 0x00004000L
+#define MC_XPB_SUB_CTRL__RESET_WCB__SHIFT 0x0000000e
+#define MC_XPB_SUB_CTRL__STALL_CNS_RTR_REQ_MASK 0x00000002L
+#define MC_XPB_SUB_CTRL__STALL_CNS_RTR_REQ__SHIFT 0x00000001
+#define MC_XPB_SUB_CTRL__STALL_HST_HOP_REQ_MASK 0x00000100L
+#define MC_XPB_SUB_CTRL__STALL_HST_HOP_REQ__SHIFT 0x00000008
+#define MC_XPB_SUB_CTRL__STALL_MAP_WCB_REQ_MASK 0x00000010L
+#define MC_XPB_SUB_CTRL__STALL_MAP_WCB_REQ__SHIFT 0x00000004
+#define MC_XPB_SUB_CTRL__STALL_MC_XSP_REQ_SEND_MASK 0x00000040L
+#define MC_XPB_SUB_CTRL__STALL_MC_XSP_REQ_SEND__SHIFT 0x00000006
+#define MC_XPB_SUB_CTRL__STALL_RTR_MAP_REQ_MASK 0x00000008L
+#define MC_XPB_SUB_CTRL__STALL_RTR_MAP_REQ__SHIFT 0x00000003
+#define MC_XPB_SUB_CTRL__STALL_RTR_RPB_WRREQ_MASK 0x00000004L
+#define MC_XPB_SUB_CTRL__STALL_RTR_RPB_WRREQ__SHIFT 0x00000002
+#define MC_XPB_SUB_CTRL__STALL_WCB_HST_REQ_MASK 0x00000080L
+#define MC_XPB_SUB_CTRL__STALL_WCB_HST_REQ__SHIFT 0x00000007
+#define MC_XPB_SUB_CTRL__STALL_WCB_SID_REQ_MASK 0x00000020L
+#define MC_XPB_SUB_CTRL__STALL_WCB_SID_REQ__SHIFT 0x00000005
+#define MC_XPB_SUB_CTRL__STALL_XPB_RPB_REQ_ATTR_MASK 0x00000200L
+#define MC_XPB_SUB_CTRL__STALL_XPB_RPB_REQ_ATTR__SHIFT 0x00000009
+#define MC_XPB_SUB_CTRL__WRREQ_BYPASS_XPB_MASK 0x00000001L
+#define MC_XPB_SUB_CTRL__WRREQ_BYPASS_XPB__SHIFT 0x00000000
+#define MC_XPB_UNC_THRESH_HST__CHANGE_PREF_MASK 0x0000003fL
+#define MC_XPB_UNC_THRESH_HST__CHANGE_PREF__SHIFT 0x00000000
+#define MC_XPB_UNC_THRESH_HST__STRONG_PREF_MASK 0x00000fc0L
+#define MC_XPB_UNC_THRESH_HST__STRONG_PREF__SHIFT 0x00000006
+#define MC_XPB_UNC_THRESH_HST__USE_UNFULL_MASK 0x0003f000L
+#define MC_XPB_UNC_THRESH_HST__USE_UNFULL__SHIFT 0x0000000c
+#define MC_XPB_UNC_THRESH_SID__CHANGE_PREF_MASK 0x0000003fL
+#define MC_XPB_UNC_THRESH_SID__CHANGE_PREF__SHIFT 0x00000000
+#define MC_XPB_UNC_THRESH_SID__STRONG_PREF_MASK 0x00000fc0L
+#define MC_XPB_UNC_THRESH_SID__STRONG_PREF__SHIFT 0x00000006
+#define MC_XPB_UNC_THRESH_SID__USE_UNFULL_MASK 0x0003f000L
+#define MC_XPB_UNC_THRESH_SID__USE_UNFULL__SHIFT 0x0000000c
+#define MC_XPB_WCB_CFG__HST_MAX_MASK 0x00030000L
+#define MC_XPB_WCB_CFG__HST_MAX__SHIFT 0x00000010
+#define MC_XPB_WCB_CFG__SID_MAX_MASK 0x000c0000L
+#define MC_XPB_WCB_CFG__SID_MAX__SHIFT 0x00000012
+#define MC_XPB_WCB_CFG__TIMEOUT_MASK 0x0000ffffL
+#define MC_XPB_WCB_CFG__TIMEOUT__SHIFT 0x00000000
+#define MC_XPB_WCB_STS__PBUF_VLD_MASK 0x0000ffffL
+#define MC_XPB_WCB_STS__PBUF_VLD__SHIFT 0x00000000
+#define MC_XPB_WCB_STS__WCB_HST_DATA_BUF_CNT_MASK 0x007f0000L
+#define MC_XPB_WCB_STS__WCB_HST_DATA_BUF_CNT__SHIFT 0x00000010
+#define MC_XPB_WCB_STS__WCB_SID_DATA_BUF_CNT_MASK 0x3f800000L
+#define MC_XPB_WCB_STS__WCB_SID_DATA_BUF_CNT__SHIFT 0x00000017
+#define MC_XPB_XDMA_PEER_SYS_BAR0__ADDR_MASK 0x07fffffcL
+#define MC_XPB_XDMA_PEER_SYS_BAR0__ADDR__SHIFT 0x00000002
+#define MC_XPB_XDMA_PEER_SYS_BAR0__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_XDMA_PEER_SYS_BAR0__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_XDMA_PEER_SYS_BAR0__VALID_MASK 0x00000001L
+#define MC_XPB_XDMA_PEER_SYS_BAR0__VALID__SHIFT 0x00000000
+#define MC_XPB_XDMA_PEER_SYS_BAR1__ADDR_MASK 0x07fffffcL
+#define MC_XPB_XDMA_PEER_SYS_BAR1__ADDR__SHIFT 0x00000002
+#define MC_XPB_XDMA_PEER_SYS_BAR1__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_XDMA_PEER_SYS_BAR1__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_XDMA_PEER_SYS_BAR1__VALID_MASK 0x00000001L
+#define MC_XPB_XDMA_PEER_SYS_BAR1__VALID__SHIFT 0x00000000
+#define MC_XPB_XDMA_PEER_SYS_BAR2__ADDR_MASK 0x07fffffcL
+#define MC_XPB_XDMA_PEER_SYS_BAR2__ADDR__SHIFT 0x00000002
+#define MC_XPB_XDMA_PEER_SYS_BAR2__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_XDMA_PEER_SYS_BAR2__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_XDMA_PEER_SYS_BAR2__VALID_MASK 0x00000001L
+#define MC_XPB_XDMA_PEER_SYS_BAR2__VALID__SHIFT 0x00000000
+#define MC_XPB_XDMA_PEER_SYS_BAR3__ADDR_MASK 0x07fffffcL
+#define MC_XPB_XDMA_PEER_SYS_BAR3__ADDR__SHIFT 0x00000002
+#define MC_XPB_XDMA_PEER_SYS_BAR3__SIDE_OK_MASK 0x00000002L
+#define MC_XPB_XDMA_PEER_SYS_BAR3__SIDE_OK__SHIFT 0x00000001
+#define MC_XPB_XDMA_PEER_SYS_BAR3__VALID_MASK 0x00000001L
+#define MC_XPB_XDMA_PEER_SYS_BAR3__VALID__SHIFT 0x00000000
+#define MC_XPB_XDMA_RTR_DEST_MAP0__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_XDMA_RTR_DEST_MAP0__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_XDMA_RTR_DEST_MAP0__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_XDMA_RTR_DEST_MAP0__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_XDMA_RTR_DEST_MAP0__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_XDMA_RTR_DEST_MAP0__NMR_MASK 0x00000001L
+#define MC_XPB_XDMA_RTR_DEST_MAP0__NMR__SHIFT 0x00000000
+#define MC_XPB_XDMA_RTR_DEST_MAP0__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_XDMA_RTR_DEST_MAP0__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_XDMA_RTR_DEST_MAP1__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_XDMA_RTR_DEST_MAP1__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_XDMA_RTR_DEST_MAP1__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_XDMA_RTR_DEST_MAP1__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_XDMA_RTR_DEST_MAP1__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_XDMA_RTR_DEST_MAP1__NMR_MASK 0x00000001L
+#define MC_XPB_XDMA_RTR_DEST_MAP1__NMR__SHIFT 0x00000000
+#define MC_XPB_XDMA_RTR_DEST_MAP1__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_XDMA_RTR_DEST_MAP1__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_XDMA_RTR_DEST_MAP2__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_XDMA_RTR_DEST_MAP2__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_XDMA_RTR_DEST_MAP2__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_XDMA_RTR_DEST_MAP2__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_XDMA_RTR_DEST_MAP2__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_XDMA_RTR_DEST_MAP2__NMR_MASK 0x00000001L
+#define MC_XPB_XDMA_RTR_DEST_MAP2__NMR__SHIFT 0x00000000
+#define MC_XPB_XDMA_RTR_DEST_MAP2__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_XDMA_RTR_DEST_MAP2__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_XDMA_RTR_DEST_MAP3__APRTR_SIZE_MASK 0x7c000000L
+#define MC_XPB_XDMA_RTR_DEST_MAP3__APRTR_SIZE__SHIFT 0x0000001a
+#define MC_XPB_XDMA_RTR_DEST_MAP3__DEST_OFFSET_MASK 0x000ffffeL
+#define MC_XPB_XDMA_RTR_DEST_MAP3__DEST_OFFSET__SHIFT 0x00000001
+#define MC_XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_MASK 0x00f00000L
+#define MC_XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_RPB_MASK 0x01000000L
+#define MC_XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_RPB__SHIFT 0x00000018
+#define MC_XPB_XDMA_RTR_DEST_MAP3__DEST_SEL__SHIFT 0x00000014
+#define MC_XPB_XDMA_RTR_DEST_MAP3__NMR_MASK 0x00000001L
+#define MC_XPB_XDMA_RTR_DEST_MAP3__NMR__SHIFT 0x00000000
+#define MC_XPB_XDMA_RTR_DEST_MAP3__SIDE_OK_MASK 0x02000000L
+#define MC_XPB_XDMA_RTR_DEST_MAP3__SIDE_OK__SHIFT 0x00000019
+#define MC_XPB_XDMA_RTR_SRC_APRTR0__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_XDMA_RTR_SRC_APRTR0__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_XDMA_RTR_SRC_APRTR1__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_XDMA_RTR_SRC_APRTR1__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_XDMA_RTR_SRC_APRTR2__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_XDMA_RTR_SRC_APRTR2__BASE_ADDR__SHIFT 0x00000000
+#define MC_XPB_XDMA_RTR_SRC_APRTR3__BASE_ADDR_MASK 0x01ffffffL
+#define MC_XPB_XDMA_RTR_SRC_APRTR3__BASE_ADDR__SHIFT 0x00000000
+#define MPLL_AD_FUNC_CNTL__SPARE_MASK 0xfffffff8L
+#define MPLL_AD_FUNC_CNTL__SPARE__SHIFT 0x00000003
+#define MPLL_AD_FUNC_CNTL__YCLK_POST_DIV_MASK 0x00000007L
+#define MPLL_AD_FUNC_CNTL__YCLK_POST_DIV__SHIFT 0x00000000
+#define MPLL_AD_STATUS__FREQ_LOCK_MASK 0x00040000L
+#define MPLL_AD_STATUS__FREQ_LOCK__SHIFT 0x00000012
+#define MPLL_AD_STATUS__FREQ_UNLOCK_STICKY_MASK 0x00080000L
+#define MPLL_AD_STATUS__FREQ_UNLOCK_STICKY__SHIFT 0x00000013
+#define MPLL_AD_STATUS__OINT_RESET_MASK 0x00020000L
+#define MPLL_AD_STATUS__OINT_RESET__SHIFT 0x00000011
+#define MPLL_AD_STATUS__TEST_FBDIV_FRAC_MASK 0x00000070L
+#define MPLL_AD_STATUS__TEST_FBDIV_FRAC__SHIFT 0x00000004
+#define MPLL_AD_STATUS__TEST_FBDIV_INT_MASK 0x0001ff80L
+#define MPLL_AD_STATUS__TEST_FBDIV_INT__SHIFT 0x00000007
+#define MPLL_AD_STATUS__VCTRLADC_MASK 0x00000007L
+#define MPLL_AD_STATUS__VCTRLADC__SHIFT 0x00000000
+#define MPLL_CNTL_MODE__FAST_LOCK_CNTRL_MASK 0x00600000L
+#define MPLL_CNTL_MODE__FAST_LOCK_CNTRL__SHIFT 0x00000015
+#define MPLL_CNTL_MODE__FAST_LOCK_EN_MASK 0x00100000L
+#define MPLL_CNTL_MODE__FAST_LOCK_EN__SHIFT 0x00000014
+#define MPLL_CNTL_MODE__FORCE_TESTMODE_MASK 0x00020000L
+#define MPLL_CNTL_MODE__FORCE_TESTMODE__SHIFT 0x00000011
+#define MPLL_CNTL_MODE__GLOBAL_MPLL_RESET_MASK 0x80000000L
+#define MPLL_CNTL_MODE__GLOBAL_MPLL_RESET__SHIFT 0x0000001f
+#define MPLL_CNTL_MODE__INSTR_DELAY_MASK 0x000000ffL
+#define MPLL_CNTL_MODE__INSTR_DELAY__SHIFT 0x00000000
+#define MPLL_CNTL_MODE__MPLL_CHG_STATUS_MASK 0x00010000L
+#define MPLL_CNTL_MODE__MPLL_CHG_STATUS__SHIFT 0x00000010
+#define MPLL_CNTL_MODE__MPLL_CTLREQ_MASK 0x00004000L
+#define MPLL_CNTL_MODE__MPLL_CTLREQ__SHIFT 0x0000000e
+#define MPLL_CNTL_MODE__MPLL_MCLK_SEL_MASK 0x00000800L
+#define MPLL_CNTL_MODE__MPLL_MCLK_SEL__SHIFT 0x0000000b
+#define MPLL_CNTL_MODE__MPLL_SW_DIR_CONTROL_MASK 0x00000100L
+#define MPLL_CNTL_MODE__MPLL_SW_DIR_CONTROL__SHIFT 0x00000008
+#define MPLL_CNTL_MODE__QDR_MASK 0x00002000L
+#define MPLL_CNTL_MODE__QDR__SHIFT 0x0000000d
+#define MPLL_CNTL_MODE__SPARE_1_MASK 0x00001000L
+#define MPLL_CNTL_MODE__SPARE_1__SHIFT 0x0000000c
+#define MPLL_CNTL_MODE__SPARE_2_MASK 0x00800000L
+#define MPLL_CNTL_MODE__SPARE_2__SHIFT 0x00000017
+#define MPLL_CNTL_MODE__SPARE_3_MASK 0x70000000L
+#define MPLL_CNTL_MODE__SPARE_3__SHIFT 0x0000001c
+#define MPLL_CNTL_MODE__SS_DSMODE_EN_MASK 0x04000000L
+#define MPLL_CNTL_MODE__SS_DSMODE_EN__SHIFT 0x0000001a
+#define MPLL_CNTL_MODE__SS_SSEN_MASK 0x03000000L
+#define MPLL_CNTL_MODE__SS_SSEN__SHIFT 0x00000018
+#define MPLL_CNTL_MODE__VTOI_BIAS_CNTRL_MASK 0x08000000L
+#define MPLL_CNTL_MODE__VTOI_BIAS_CNTRL__SHIFT 0x0000001b
+#define MPLL_CONTROL__AD_BG_PWRON_MASK 0x00001000L
+#define MPLL_CONTROL__AD_BG_PWRON__SHIFT 0x0000000c
+#define MPLL_CONTROL__AD_PLL_PWRON_MASK 0x00002000L
+#define MPLL_CONTROL__AD_PLL_PWRON__SHIFT 0x0000000d
+#define MPLL_CONTROL__AD_PLL_RESET_MASK 0x00004000L
+#define MPLL_CONTROL__AD_PLL_RESET__SHIFT 0x0000000e
+#define MPLL_CONTROL__DQ_0_0_BG_PWRON_MASK 0x00010000L
+#define MPLL_CONTROL__DQ_0_0_BG_PWRON__SHIFT 0x00000010
+#define MPLL_CONTROL__DQ_0_0_PLL_PWRON_MASK 0x00020000L
+#define MPLL_CONTROL__DQ_0_0_PLL_PWRON__SHIFT 0x00000011
+#define MPLL_CONTROL__DQ_0_0_PLL_RESET_MASK 0x00040000L
+#define MPLL_CONTROL__DQ_0_0_PLL_RESET__SHIFT 0x00000012
+#define MPLL_CONTROL__DQ_0_1_BG_PWRON_MASK 0x00100000L
+#define MPLL_CONTROL__DQ_0_1_BG_PWRON__SHIFT 0x00000014
+#define MPLL_CONTROL__DQ_0_1_PLL_PWRON_MASK 0x00200000L
+#define MPLL_CONTROL__DQ_0_1_PLL_PWRON__SHIFT 0x00000015
+#define MPLL_CONTROL__DQ_0_1_PLL_RESET_MASK 0x00400000L
+#define MPLL_CONTROL__DQ_0_1_PLL_RESET__SHIFT 0x00000016
+#define MPLL_CONTROL__DQ_1_0_BG_PWRON_MASK 0x01000000L
+#define MPLL_CONTROL__DQ_1_0_BG_PWRON__SHIFT 0x00000018
+#define MPLL_CONTROL__DQ_1_0_PLL_PWRON_MASK 0x02000000L
+#define MPLL_CONTROL__DQ_1_0_PLL_PWRON__SHIFT 0x00000019
+#define MPLL_CONTROL__DQ_1_0_PLL_RESET_MASK 0x04000000L
+#define MPLL_CONTROL__DQ_1_0_PLL_RESET__SHIFT 0x0000001a
+#define MPLL_CONTROL__DQ_1_1_BG_PWRON_MASK 0x10000000L
+#define MPLL_CONTROL__DQ_1_1_BG_PWRON__SHIFT 0x0000001c
+#define MPLL_CONTROL__DQ_1_1_PLL_PWRON_MASK 0x20000000L
+#define MPLL_CONTROL__DQ_1_1_PLL_PWRON__SHIFT 0x0000001d
+#define MPLL_CONTROL__DQ_1_1_PLL_RESET_MASK 0x40000000L
+#define MPLL_CONTROL__DQ_1_1_PLL_RESET__SHIFT 0x0000001e
+#define MPLL_CONTROL__GDDR_PWRON_MASK 0x00000001L
+#define MPLL_CONTROL__GDDR_PWRON__SHIFT 0x00000000
+#define MPLL_CONTROL__PLL_BUF_PWRON_TX_MASK 0x00000004L
+#define MPLL_CONTROL__PLL_BUF_PWRON_TX__SHIFT 0x00000002
+#define MPLL_CONTROL__REFCLK_PWRON_MASK 0x00000002L
+#define MPLL_CONTROL__REFCLK_PWRON__SHIFT 0x00000001
+#define MPLL_CONTROL__SPARE_AD_0_MASK 0x00008000L
+#define MPLL_CONTROL__SPARE_AD_0__SHIFT 0x0000000f
+#define MPLL_CONTROL__SPARE_DQ_0_0_MASK 0x00080000L
+#define MPLL_CONTROL__SPARE_DQ_0_0__SHIFT 0x00000013
+#define MPLL_CONTROL__SPARE_DQ_0_1_MASK 0x00800000L
+#define MPLL_CONTROL__SPARE_DQ_0_1__SHIFT 0x00000017
+#define MPLL_CONTROL__SPARE_DQ_1_0_MASK 0x08000000L
+#define MPLL_CONTROL__SPARE_DQ_1_0__SHIFT 0x0000001b
+#define MPLL_CONTROL__SPARE_DQ_1_1_MASK 0x80000000L
+#define MPLL_CONTROL__SPARE_DQ_1_1__SHIFT 0x0000001f
+#define MPLL_DQ_0_0_STATUS__FREQ_LOCK_MASK 0x00040000L
+#define MPLL_DQ_0_0_STATUS__FREQ_LOCK__SHIFT 0x00000012
+#define MPLL_DQ_0_0_STATUS__FREQ_UNLOCK_STICKY_MASK 0x00080000L
+#define MPLL_DQ_0_0_STATUS__FREQ_UNLOCK_STICKY__SHIFT 0x00000013
+#define MPLL_DQ_0_0_STATUS__OINT_RESET_MASK 0x00020000L
+#define MPLL_DQ_0_0_STATUS__OINT_RESET__SHIFT 0x00000011
+#define MPLL_DQ_0_0_STATUS__TEST_FBDIV_FRAC_MASK 0x00000070L
+#define MPLL_DQ_0_0_STATUS__TEST_FBDIV_FRAC__SHIFT 0x00000004
+#define MPLL_DQ_0_0_STATUS__TEST_FBDIV_INT_MASK 0x0001ff80L
+#define MPLL_DQ_0_0_STATUS__TEST_FBDIV_INT__SHIFT 0x00000007
+#define MPLL_DQ_0_0_STATUS__VCTRLADC_MASK 0x00000007L
+#define MPLL_DQ_0_0_STATUS__VCTRLADC__SHIFT 0x00000000
+#define MPLL_DQ_0_1_STATUS__FREQ_LOCK_MASK 0x00040000L
+#define MPLL_DQ_0_1_STATUS__FREQ_LOCK__SHIFT 0x00000012
+#define MPLL_DQ_0_1_STATUS__FREQ_UNLOCK_STICKY_MASK 0x00080000L
+#define MPLL_DQ_0_1_STATUS__FREQ_UNLOCK_STICKY__SHIFT 0x00000013
+#define MPLL_DQ_0_1_STATUS__OINT_RESET_MASK 0x00020000L
+#define MPLL_DQ_0_1_STATUS__OINT_RESET__SHIFT 0x00000011
+#define MPLL_DQ_0_1_STATUS__TEST_FBDIV_FRAC_MASK 0x00000070L
+#define MPLL_DQ_0_1_STATUS__TEST_FBDIV_FRAC__SHIFT 0x00000004
+#define MPLL_DQ_0_1_STATUS__TEST_FBDIV_INT_MASK 0x0001ff80L
+#define MPLL_DQ_0_1_STATUS__TEST_FBDIV_INT__SHIFT 0x00000007
+#define MPLL_DQ_0_1_STATUS__VCTRLADC_MASK 0x00000007L
+#define MPLL_DQ_0_1_STATUS__VCTRLADC__SHIFT 0x00000000
+#define MPLL_DQ_1_0_STATUS__FREQ_LOCK_MASK 0x00040000L
+#define MPLL_DQ_1_0_STATUS__FREQ_LOCK__SHIFT 0x00000012
+#define MPLL_DQ_1_0_STATUS__FREQ_UNLOCK_STICKY_MASK 0x00080000L
+#define MPLL_DQ_1_0_STATUS__FREQ_UNLOCK_STICKY__SHIFT 0x00000013
+#define MPLL_DQ_1_0_STATUS__OINT_RESET_MASK 0x00020000L
+#define MPLL_DQ_1_0_STATUS__OINT_RESET__SHIFT 0x00000011
+#define MPLL_DQ_1_0_STATUS__TEST_FBDIV_FRAC_MASK 0x00000070L
+#define MPLL_DQ_1_0_STATUS__TEST_FBDIV_FRAC__SHIFT 0x00000004
+#define MPLL_DQ_1_0_STATUS__TEST_FBDIV_INT_MASK 0x0001ff80L
+#define MPLL_DQ_1_0_STATUS__TEST_FBDIV_INT__SHIFT 0x00000007
+#define MPLL_DQ_1_0_STATUS__VCTRLADC_MASK 0x00000007L
+#define MPLL_DQ_1_0_STATUS__VCTRLADC__SHIFT 0x00000000
+#define MPLL_DQ_1_1_STATUS__FREQ_LOCK_MASK 0x00040000L
+#define MPLL_DQ_1_1_STATUS__FREQ_LOCK__SHIFT 0x00000012
+#define MPLL_DQ_1_1_STATUS__FREQ_UNLOCK_STICKY_MASK 0x00080000L
+#define MPLL_DQ_1_1_STATUS__FREQ_UNLOCK_STICKY__SHIFT 0x00000013
+#define MPLL_DQ_1_1_STATUS__OINT_RESET_MASK 0x00020000L
+#define MPLL_DQ_1_1_STATUS__OINT_RESET__SHIFT 0x00000011
+#define MPLL_DQ_1_1_STATUS__TEST_FBDIV_FRAC_MASK 0x00000070L
+#define MPLL_DQ_1_1_STATUS__TEST_FBDIV_FRAC__SHIFT 0x00000004
+#define MPLL_DQ_1_1_STATUS__TEST_FBDIV_INT_MASK 0x0001ff80L
+#define MPLL_DQ_1_1_STATUS__TEST_FBDIV_INT__SHIFT 0x00000007
+#define MPLL_DQ_1_1_STATUS__VCTRLADC_MASK 0x00000007L
+#define MPLL_DQ_1_1_STATUS__VCTRLADC__SHIFT 0x00000000
+#define MPLL_DQ_FUNC_CNTL__SPARE_0_MASK 0x00000008L
+#define MPLL_DQ_FUNC_CNTL__SPARE_0__SHIFT 0x00000003
+#define MPLL_DQ_FUNC_CNTL__SPARE_MASK 0xffffffe0L
+#define MPLL_DQ_FUNC_CNTL__SPARE__SHIFT 0x00000005
+#define MPLL_DQ_FUNC_CNTL__YCLK_POST_DIV_MASK 0x00000007L
+#define MPLL_DQ_FUNC_CNTL__YCLK_POST_DIV__SHIFT 0x00000000
+#define MPLL_DQ_FUNC_CNTL__YCLK_SEL_MASK 0x00000010L
+#define MPLL_DQ_FUNC_CNTL__YCLK_SEL__SHIFT 0x00000004
+#define MPLL_FUNC_CNTL_1__CLKF_MASK 0x0fff0000L
+#define MPLL_FUNC_CNTL_1__CLKFRAC_MASK 0x0000fff0L
+#define MPLL_FUNC_CNTL_1__CLKFRAC__SHIFT 0x00000004
+#define MPLL_FUNC_CNTL_1__CLKF__SHIFT 0x00000010
+#define MPLL_FUNC_CNTL_1__SPARE_0_MASK 0x0000000cL
+#define MPLL_FUNC_CNTL_1__SPARE_0__SHIFT 0x00000002
+#define MPLL_FUNC_CNTL_1__SPARE_1_MASK 0xf0000000L
+#define MPLL_FUNC_CNTL_1__SPARE_1__SHIFT 0x0000001c
+#define MPLL_FUNC_CNTL_1__VCO_MODE_MASK 0x00000003L
+#define MPLL_FUNC_CNTL_1__VCO_MODE__SHIFT 0x00000000
+#define MPLL_FUNC_CNTL_2__BACKUP_2_MASK 0x000e0000L
+#define MPLL_FUNC_CNTL_2__BACKUP_2__SHIFT 0x00000011
+#define MPLL_FUNC_CNTL_2__BACKUP_MASK 0xf8000000L
+#define MPLL_FUNC_CNTL_2__BACKUP__SHIFT 0x0000001b
+#define MPLL_FUNC_CNTL_2__LF_CNTRL_MASK 0x07f00000L
+#define MPLL_FUNC_CNTL_2__LF_CNTRL__SHIFT 0x00000014
+#define MPLL_FUNC_CNTL_2__MPLL_UNLOCK_CLEAR_MASK 0x00000080L
+#define MPLL_FUNC_CNTL_2__MPLL_UNLOCK_CLEAR__SHIFT 0x00000007
+#define MPLL_FUNC_CNTL_2__PFD_RESET_CNTRL_MASK 0x00003000L
+#define MPLL_FUNC_CNTL_2__PFD_RESET_CNTRL__SHIFT 0x0000000c
+#define MPLL_FUNC_CNTL_2__RESET_EN_MASK 0x00000004L
+#define MPLL_FUNC_CNTL_2__RESET_EN__SHIFT 0x00000002
+#define MPLL_FUNC_CNTL_2__RESET_TIMER_MASK 0x00000c00L
+#define MPLL_FUNC_CNTL_2__RESET_TIMER__SHIFT 0x0000000a
+#define MPLL_FUNC_CNTL_2__TEST_BYPCLK_EN_MASK 0x00000008L
+#define MPLL_FUNC_CNTL_2__TEST_BYPCLK_EN__SHIFT 0x00000003
+#define MPLL_FUNC_CNTL_2__TEST_BYPCLK_SRC_MASK 0x00000010L
+#define MPLL_FUNC_CNTL_2__TEST_BYPCLK_SRC__SHIFT 0x00000004
+#define MPLL_FUNC_CNTL_2__TEST_BYPMCLK_MASK 0x00000040L
+#define MPLL_FUNC_CNTL_2__TEST_BYPMCLK__SHIFT 0x00000006
+#define MPLL_FUNC_CNTL_2__TEST_FBDIV_FRAC_BYPASS_MASK 0x00000020L
+#define MPLL_FUNC_CNTL_2__TEST_FBDIV_FRAC_BYPASS__SHIFT 0x00000005
+#define MPLL_FUNC_CNTL_2__TEST_FBDIV_SSC_BYPASS_MASK 0x00000200L
+#define MPLL_FUNC_CNTL_2__TEST_FBDIV_SSC_BYPASS__SHIFT 0x00000009
+#define MPLL_FUNC_CNTL_2__TEST_VCTL_CNTRL_MASK 0x00000100L
+#define MPLL_FUNC_CNTL_2__TEST_VCTL_CNTRL__SHIFT 0x00000008
+#define MPLL_FUNC_CNTL_2__TEST_VCTL_EN_MASK 0x00000002L
+#define MPLL_FUNC_CNTL_2__TEST_VCTL_EN__SHIFT 0x00000001
+#define MPLL_FUNC_CNTL_2__VCTRLADC_EN_MASK 0x00000001L
+#define MPLL_FUNC_CNTL_2__VCTRLADC_EN__SHIFT 0x00000000
+#define MPLL_FUNC_CNTL__BG_100ADJ_MASK 0x00000f00L
+#define MPLL_FUNC_CNTL__BG_100ADJ__SHIFT 0x00000008
+#define MPLL_FUNC_CNTL__BG_135ADJ_MASK 0x000f0000L
+#define MPLL_FUNC_CNTL__BG_135ADJ__SHIFT 0x00000010
+#define MPLL_FUNC_CNTL__BWCTRL_MASK 0x0ff00000L
+#define MPLL_FUNC_CNTL__BWCTRL__SHIFT 0x00000014
+#define MPLL_FUNC_CNTL__REG_BIAS_MASK 0xc0000000L
+#define MPLL_FUNC_CNTL__REG_BIAS__SHIFT 0x0000001e
+#define MPLL_FUNC_CNTL__SPARE_0_MASK 0x00000020L
+#define MPLL_FUNC_CNTL__SPARE_0__SHIFT 0x00000005
+#define MPLL_SEQ_UCODE_1__INSTR0_MASK 0x0000000fL
+#define MPLL_SEQ_UCODE_1__INSTR0__SHIFT 0x00000000
+#define MPLL_SEQ_UCODE_1__INSTR1_MASK 0x000000f0L
+#define MPLL_SEQ_UCODE_1__INSTR1__SHIFT 0x00000004
+#define MPLL_SEQ_UCODE_1__INSTR2_MASK 0x00000f00L
+#define MPLL_SEQ_UCODE_1__INSTR2__SHIFT 0x00000008
+#define MPLL_SEQ_UCODE_1__INSTR3_MASK 0x0000f000L
+#define MPLL_SEQ_UCODE_1__INSTR3__SHIFT 0x0000000c
+#define MPLL_SEQ_UCODE_1__INSTR4_MASK 0x000f0000L
+#define MPLL_SEQ_UCODE_1__INSTR4__SHIFT 0x00000010
+#define MPLL_SEQ_UCODE_1__INSTR5_MASK 0x00f00000L
+#define MPLL_SEQ_UCODE_1__INSTR5__SHIFT 0x00000014
+#define MPLL_SEQ_UCODE_1__INSTR6_MASK 0x0f000000L
+#define MPLL_SEQ_UCODE_1__INSTR6__SHIFT 0x00000018
+#define MPLL_SEQ_UCODE_1__INSTR7_MASK 0xf0000000L
+#define MPLL_SEQ_UCODE_1__INSTR7__SHIFT 0x0000001c
+#define MPLL_SEQ_UCODE_2__INSTR10_MASK 0x00000f00L
+#define MPLL_SEQ_UCODE_2__INSTR10__SHIFT 0x00000008
+#define MPLL_SEQ_UCODE_2__INSTR11_MASK 0x0000f000L
+#define MPLL_SEQ_UCODE_2__INSTR11__SHIFT 0x0000000c
+#define MPLL_SEQ_UCODE_2__INSTR12_MASK 0x000f0000L
+#define MPLL_SEQ_UCODE_2__INSTR12__SHIFT 0x00000010
+#define MPLL_SEQ_UCODE_2__INSTR13_MASK 0x00f00000L
+#define MPLL_SEQ_UCODE_2__INSTR13__SHIFT 0x00000014
+#define MPLL_SEQ_UCODE_2__INSTR14_MASK 0x0f000000L
+#define MPLL_SEQ_UCODE_2__INSTR14__SHIFT 0x00000018
+#define MPLL_SEQ_UCODE_2__INSTR15_MASK 0xf0000000L
+#define MPLL_SEQ_UCODE_2__INSTR15__SHIFT 0x0000001c
+#define MPLL_SEQ_UCODE_2__INSTR8_MASK 0x0000000fL
+#define MPLL_SEQ_UCODE_2__INSTR8__SHIFT 0x00000000
+#define MPLL_SEQ_UCODE_2__INSTR9_MASK 0x000000f0L
+#define MPLL_SEQ_UCODE_2__INSTR9__SHIFT 0x00000004
+#define MPLL_SS1__CLKV_MASK 0x03ffffffL
+#define MPLL_SS1__CLKV__SHIFT 0x00000000
+#define MPLL_SS1__SPARE_MASK 0xfc000000L
+#define MPLL_SS1__SPARE__SHIFT 0x0000001a
+#define MPLL_SS2__CLKS_MASK 0x00000fffL
+#define MPLL_SS2__CLKS__SHIFT 0x00000000
+#define MPLL_SS2__SPARE_MASK 0xfffff000L
+#define MPLL_SS2__SPARE__SHIFT 0x0000000c
+#define MPLL_TIME__MPLL_LOCK_TIME_MASK 0x0000ffffL
+#define MPLL_TIME__MPLL_LOCK_TIME__SHIFT 0x00000000
+#define MPLL_TIME__MPLL_RESET_TIME_MASK 0xffff0000L
+#define MPLL_TIME__MPLL_RESET_TIME__SHIFT 0x00000010
+#define VM_CONTEXT0_CNTL2__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES_MASK 0x00000008L
+#define VM_CONTEXT0_CNTL2__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES__SHIFT 0x00000003
+#define VM_CONTEXT0_CNTL2__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK 0x00000001L
+#define VM_CONTEXT0_CNTL2__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT 0x00000000
+#define VM_CONTEXT0_CNTL2__ENABLE_CLEAR_PROTECTION_FAULT_STATUS_ADDR_WHEN_INVALIDATE_CONTEXT_MASK 0x00000002L
+#define VM_CONTEXT0_CNTL2__ENABLE_CLEAR_PROTECTION_FAULT_STATUS_ADDR_WHEN_INVALIDATE_CONTEXT__SHIFT 0x00000001
+#define VM_CONTEXT0_CNTL2__ENABLE_INTERRUPT_PROCESSING_FOR_SUBSEQUENT_FAULTS_PER_CONTEXT_MASK 0x00000004L
+#define VM_CONTEXT0_CNTL2__ENABLE_INTERRUPT_PROCESSING_FOR_SUBSEQUENT_FAULTS_PER_CONTEXT__SHIFT 0x00000002
+#define VM_CONTEXT0_CNTL2__WAIT_FOR_IDLE_WHEN_INVALIDATE_MASK 0x00000010L
+#define VM_CONTEXT0_CNTL2__WAIT_FOR_IDLE_WHEN_INVALIDATE__SHIFT 0x00000004
+#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00000080L
+#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x00000007
+#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00000040L
+#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x00000006
+#define VM_CONTEXT0_CNTL__ENABLE_CONTEXT_MASK 0x00000001L
+#define VM_CONTEXT0_CNTL__ENABLE_CONTEXT__SHIFT 0x00000000
+#define VM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK 0x0f000000L
+#define VM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT 0x00000018
+#define VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH_MASK 0x00000006L
+#define VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH__SHIFT 0x00000001
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00000400L
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x0000000a
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00000200L
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x00000009
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_SAVE_MASK 0x00000800L
+#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_SAVE__SHIFT 0x0000000b
+#define VM_CONTEXT0_CNTL__PRIVILEGED_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00400000L
+#define VM_CONTEXT0_CNTL__PRIVILEGED_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x00000016
+#define VM_CONTEXT0_CNTL__PRIVILEGED_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00200000L
+#define VM_CONTEXT0_CNTL__PRIVILEGED_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x00000015
+#define VM_CONTEXT0_CNTL__PRIVILEGED_PROTECTION_FAULT_ENABLE_SAVE_MASK 0x00800000L
+#define VM_CONTEXT0_CNTL__PRIVILEGED_PROTECTION_FAULT_ENABLE_SAVE__SHIFT 0x00000017
+#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00000010L
+#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x00000004
+#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00000008L
+#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x00000003
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00010000L
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x00000010
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00008000L
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x0000000f
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_SAVE_MASK 0x00020000L
+#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_SAVE__SHIFT 0x00000011
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00002000L
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x0000000d
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00001000L
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x0000000c
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_SAVE_MASK 0x00004000L
+#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_SAVE__SHIFT 0x0000000e
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00080000L
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x00000013
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00040000L
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x00000012
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_SAVE_MASK 0x00100000L
+#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_SAVE__SHIFT 0x00000014
+#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT0_PAGE_TABLE_END_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT0_PAGE_TABLE_END_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT0_PAGE_TABLE_START_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT0_PAGE_TABLE_START_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT0_PROTECTION_FAULT_ADDR__LOGICAL_PAGE_ADDR_MASK 0x0fffffffL
+#define VM_CONTEXT0_PROTECTION_FAULT_ADDR__LOGICAL_PAGE_ADDR__SHIFT 0x00000000
+#define VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR__PHYSICAL_PAGE_ADDR_MASK 0x0fffffffL
+#define VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR__PHYSICAL_PAGE_ADDR__SHIFT 0x00000000
+#define VM_CONTEXT0_PROTECTION_FAULT_STATUS__MEMORY_CLIENT_ID_MASK 0x000ff000L
+#define VM_CONTEXT0_PROTECTION_FAULT_STATUS__MEMORY_CLIENT_ID__SHIFT 0x0000000c
+#define VM_CONTEXT0_PROTECTION_FAULT_STATUS__MEMORY_CLIENT_RW_MASK 0x01000000L
+#define VM_CONTEXT0_PROTECTION_FAULT_STATUS__MEMORY_CLIENT_RW__SHIFT 0x00000018
+#define VM_CONTEXT0_PROTECTION_FAULT_STATUS__PROTECTIONS_MASK 0x000000ffL
+#define VM_CONTEXT0_PROTECTION_FAULT_STATUS__PROTECTIONS__SHIFT 0x00000000
+#define VM_CONTEXT0_PROTECTION_FAULT_STATUS__VMID_MASK 0x1e000000L
+#define VM_CONTEXT0_PROTECTION_FAULT_STATUS__VMID__SHIFT 0x00000019
+#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT1_CNTL2__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES_MASK 0x00000008L
+#define VM_CONTEXT1_CNTL2__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES__SHIFT 0x00000003
+#define VM_CONTEXT1_CNTL2__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK 0x00000001L
+#define VM_CONTEXT1_CNTL2__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT 0x00000000
+#define VM_CONTEXT1_CNTL2__ENABLE_CLEAR_PROTECTION_FAULT_STATUS_ADDR_WHEN_INVALIDATE_CONTEXT_MASK 0x00000002L
+#define VM_CONTEXT1_CNTL2__ENABLE_CLEAR_PROTECTION_FAULT_STATUS_ADDR_WHEN_INVALIDATE_CONTEXT__SHIFT 0x00000001
+#define VM_CONTEXT1_CNTL2__ENABLE_INTERRUPT_PROCESSING_FOR_SUBSEQUENT_FAULTS_PER_CONTEXT_MASK 0x00000004L
+#define VM_CONTEXT1_CNTL2__ENABLE_INTERRUPT_PROCESSING_FOR_SUBSEQUENT_FAULTS_PER_CONTEXT__SHIFT 0x00000002
+#define VM_CONTEXT1_CNTL2__WAIT_FOR_IDLE_WHEN_INVALIDATE_MASK 0x00000010L
+#define VM_CONTEXT1_CNTL2__WAIT_FOR_IDLE_WHEN_INVALIDATE__SHIFT 0x00000004
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00000080L
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x00000007
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00000040L
+#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x00000006
+#define VM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK 0x00000001L
+#define VM_CONTEXT1_CNTL__ENABLE_CONTEXT__SHIFT 0x00000000
+#define VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK 0x0f000000L
+#define VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT 0x00000018
+#define VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH_MASK 0x00000006L
+#define VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT 0x00000001
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00000400L
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x0000000a
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00000200L
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x00000009
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_SAVE_MASK 0x00000800L
+#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_SAVE__SHIFT 0x0000000b
+#define VM_CONTEXT1_CNTL__PRIVILEGED_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00400000L
+#define VM_CONTEXT1_CNTL__PRIVILEGED_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x00000016
+#define VM_CONTEXT1_CNTL__PRIVILEGED_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00200000L
+#define VM_CONTEXT1_CNTL__PRIVILEGED_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x00000015
+#define VM_CONTEXT1_CNTL__PRIVILEGED_PROTECTION_FAULT_ENABLE_SAVE_MASK 0x00800000L
+#define VM_CONTEXT1_CNTL__PRIVILEGED_PROTECTION_FAULT_ENABLE_SAVE__SHIFT 0x00000017
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00000010L
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x00000004
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00000008L
+#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x00000003
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00010000L
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x00000010
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00008000L
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x0000000f
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_SAVE_MASK 0x00020000L
+#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_SAVE__SHIFT 0x00000011
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00002000L
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x0000000d
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00001000L
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x0000000c
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_SAVE_MASK 0x00004000L
+#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_SAVE__SHIFT 0x0000000e
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK 0x00080000L
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT 0x00000013
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK 0x00040000L
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT 0x00000012
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_SAVE_MASK 0x00100000L
+#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_SAVE__SHIFT 0x00000014
+#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT1_PAGE_TABLE_END_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT1_PAGE_TABLE_END_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT1_PAGE_TABLE_START_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT1_PAGE_TABLE_START_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT1_PROTECTION_FAULT_ADDR__LOGICAL_PAGE_ADDR_MASK 0x0fffffffL
+#define VM_CONTEXT1_PROTECTION_FAULT_ADDR__LOGICAL_PAGE_ADDR__SHIFT 0x00000000
+#define VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR__PHYSICAL_PAGE_ADDR_MASK 0x0fffffffL
+#define VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR__PHYSICAL_PAGE_ADDR__SHIFT 0x00000000
+#define VM_CONTEXT1_PROTECTION_FAULT_STATUS__MEMORY_CLIENT_ID_MASK 0x000ff000L
+#define VM_CONTEXT1_PROTECTION_FAULT_STATUS__MEMORY_CLIENT_ID__SHIFT 0x0000000c
+#define VM_CONTEXT1_PROTECTION_FAULT_STATUS__MEMORY_CLIENT_RW_MASK 0x01000000L
+#define VM_CONTEXT1_PROTECTION_FAULT_STATUS__MEMORY_CLIENT_RW__SHIFT 0x00000018
+#define VM_CONTEXT1_PROTECTION_FAULT_STATUS__PROTECTIONS_MASK 0x000000ffL
+#define VM_CONTEXT1_PROTECTION_FAULT_STATUS__PROTECTIONS__SHIFT 0x00000000
+#define VM_CONTEXT1_PROTECTION_FAULT_STATUS__VMID_MASK 0x1e000000L
+#define VM_CONTEXT1_PROTECTION_FAULT_STATUS__VMID__SHIFT 0x00000019
+#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR__PHYSICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0_MASK 0x00000001L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0__SHIFT 0x00000000
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10_MASK 0x00000400L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10__SHIFT 0x0000000a
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11_MASK 0x00000800L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11__SHIFT 0x0000000b
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12_MASK 0x00001000L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12__SHIFT 0x0000000c
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13_MASK 0x00002000L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13__SHIFT 0x0000000d
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14_MASK 0x00004000L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14__SHIFT 0x0000000e
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15_MASK 0x00008000L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15__SHIFT 0x0000000f
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1_MASK 0x00000002L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1__SHIFT 0x00000001
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2_MASK 0x00000004L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2__SHIFT 0x00000002
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3_MASK 0x00000008L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3__SHIFT 0x00000003
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4_MASK 0x00000010L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4__SHIFT 0x00000004
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5_MASK 0x00000020L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5__SHIFT 0x00000005
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6_MASK 0x00000040L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6__SHIFT 0x00000006
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7_MASK 0x00000080L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7__SHIFT 0x00000007
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8_MASK 0x00000100L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8__SHIFT 0x00000008
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9_MASK 0x00000200L
+#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9__SHIFT 0x00000009
+#define VM_DEBUG__FLAGS_MASK 0xffffffffL
+#define VM_DEBUG__FLAGS__SHIFT 0x00000000
+#define VM_DUMMY_PAGE_FAULT_ADDR__DUMMY_PAGE_ADDR_MASK 0x0fffffffL
+#define VM_DUMMY_PAGE_FAULT_ADDR__DUMMY_PAGE_ADDR__SHIFT 0x00000000
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL_MASK 0x00000002L
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL__SHIFT 0x00000001
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MASK_MASK 0x0000000cL
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MASK__SHIFT 0x00000002
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE_MASK 0x00000001L
+#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE__SHIFT 0x00000000
+#define VM_FAULT_CLIENT_ID__MEMORY_CLIENT_MASK 0x000001ffL
+#define VM_FAULT_CLIENT_ID__MEMORY_CLIENT_MASK_MASK 0x0003fe00L
+#define VM_FAULT_CLIENT_ID__MEMORY_CLIENT_MASK__SHIFT 0x00000009
+#define VM_FAULT_CLIENT_ID__MEMORY_CLIENT__SHIFT 0x00000000
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_0_MASK 0x00000001L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_0__SHIFT 0x00000000
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_10_MASK 0x00000400L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_10__SHIFT 0x0000000a
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_11_MASK 0x00000800L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_11__SHIFT 0x0000000b
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_12_MASK 0x00001000L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_12__SHIFT 0x0000000c
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_13_MASK 0x00002000L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_13__SHIFT 0x0000000d
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_14_MASK 0x00004000L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_14__SHIFT 0x0000000e
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_15_MASK 0x00008000L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_15__SHIFT 0x0000000f
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_1_MASK 0x00000002L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_1__SHIFT 0x00000001
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_2_MASK 0x00000004L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_2__SHIFT 0x00000002
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_3_MASK 0x00000008L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_3__SHIFT 0x00000003
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_4_MASK 0x00000010L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_4__SHIFT 0x00000004
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_5_MASK 0x00000020L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_5__SHIFT 0x00000005
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_6_MASK 0x00000040L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_6__SHIFT 0x00000006
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_7_MASK 0x00000080L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_7__SHIFT 0x00000007
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_8_MASK 0x00000100L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_8__SHIFT 0x00000008
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_9_MASK 0x00000200L
+#define VM_INVALIDATE_REQUEST__INVALIDATE_DOMAIN_9__SHIFT 0x00000009
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_0_MASK 0x00000001L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_0__SHIFT 0x00000000
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_10_MASK 0x00000400L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_10__SHIFT 0x0000000a
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_11_MASK 0x00000800L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_11__SHIFT 0x0000000b
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_12_MASK 0x00001000L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_12__SHIFT 0x0000000c
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_13_MASK 0x00002000L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_13__SHIFT 0x0000000d
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_14_MASK 0x00004000L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_14__SHIFT 0x0000000e
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_15_MASK 0x00008000L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_15__SHIFT 0x0000000f
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_1_MASK 0x00000002L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_1__SHIFT 0x00000001
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_2_MASK 0x00000004L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_2__SHIFT 0x00000002
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_3_MASK 0x00000008L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_3__SHIFT 0x00000003
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_4_MASK 0x00000010L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_4__SHIFT 0x00000004
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_5_MASK 0x00000020L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_5__SHIFT 0x00000005
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_6_MASK 0x00000040L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_6__SHIFT 0x00000006
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_7_MASK 0x00000080L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_7__SHIFT 0x00000007
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_8_MASK 0x00000100L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_8__SHIFT 0x00000008
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_9_MASK 0x00000200L
+#define VM_INVALIDATE_RESPONSE__DOMAIN_INVALIDATED_9__SHIFT 0x00000009
+#define VM_L2_BANK_SELECT_MASKA__BANK_SELECT_MASK_MASK 0x0fffffffL
+#define VM_L2_BANK_SELECT_MASKA__BANK_SELECT_MASK__SHIFT 0x00000000
+#define VM_L2_BANK_SELECT_MASKB__BANK_SELECT_MASK_MASK 0x000000ffL
+#define VM_L2_BANK_SELECT_MASKB__BANK_SELECT_MASK__SHIFT 0x00000000
+#define VM_L2_CG__ENABLE_MASK 0x00040000L
+#define VM_L2_CG__ENABLE__SHIFT 0x00000012
+#define VM_L2_CG__MEM_LS_ENABLE_MASK 0x00080000L
+#define VM_L2_CG__MEM_LS_ENABLE__SHIFT 0x00000013
+#define VM_L2_CG__OFFDLY_MASK 0x00000fc0L
+#define VM_L2_CG__OFFDLY__SHIFT 0x00000006
+#define VM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION_MASK 0x00400000L
+#define VM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION__SHIFT 0x00000016
+#define VM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN_MASK 0x00200000L
+#define VM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN__SHIFT 0x00000015
+#define VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK 0x00000001L
+#define VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS__SHIFT 0x00000000
+#define VM_L2_CNTL2__INVALIDATE_CACHE_MODE_MASK 0x0c000000L
+#define VM_L2_CNTL2__INVALIDATE_CACHE_MODE__SHIFT 0x0000001a
+#define VM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK 0x00000002L
+#define VM_L2_CNTL2__INVALIDATE_L2_CACHE__SHIFT 0x00000001
+#define VM_L2_CNTL2__L2_CACHE_BIGK_VMID_MODE_MASK 0x03800000L
+#define VM_L2_CNTL2__L2_CACHE_BIGK_VMID_MODE__SHIFT 0x00000017
+#define VM_L2_CNTL3__BANK_SELECT_MASK 0x0000003fL
+#define VM_L2_CNTL3__BANK_SELECT__SHIFT 0x00000000
+#define VM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE_MASK 0x00e00000L
+#define VM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE__SHIFT 0x00000015
+#define VM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS_MASK 0x10000000L
+#define VM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS__SHIFT 0x0000001c
+#define VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK 0x00100000L
+#define VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY__SHIFT 0x00000014
+#define VM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE_MASK 0x0f000000L
+#define VM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE__SHIFT 0x00000018
+#define VM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS_MASK 0x20000000L
+#define VM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS__SHIFT 0x0000001d
+#define VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK 0x000f8000L
+#define VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT 0x0000000f
+#define VM_L2_CNTL3__L2_CACHE_UPDATE_MODE_MASK 0x000000c0L
+#define VM_L2_CNTL3__L2_CACHE_UPDATE_MODE__SHIFT 0x00000006
+#define VM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK 0x00001f00L
+#define VM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT 0x00000008
+#define VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE_MASK 0x00180000L
+#define VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT 0x00000013
+#define VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE_MASK 0x00038000L
+#define VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT 0x0000000f
+#define VM_L2_CNTL__ENABLE_L2_CACHE_MASK 0x00000001L
+#define VM_L2_CNTL__ENABLE_L2_CACHE__SHIFT 0x00000000
+#define VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING_MASK 0x00000002L
+#define VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING__SHIFT 0x00000001
+#define VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK 0x00000400L
+#define VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE__SHIFT 0x0000000a
+#define VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK 0x00000200L
+#define VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE__SHIFT 0x00000009
+#define VM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE_MASK 0x03e00000L
+#define VM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE__SHIFT 0x00000015
+#define VM_L2_CNTL__L2_CACHE_4K_SWAP_TAG_INDEX_LSBS_MASK 0x0c000000L
+#define VM_L2_CNTL__L2_CACHE_4K_SWAP_TAG_INDEX_LSBS__SHIFT 0x0000001a
+#define VM_L2_CNTL__L2_CACHE_BIGK_SWAP_TAG_INDEX_LSBS_MASK 0x70000000L
+#define VM_L2_CNTL__L2_CACHE_BIGK_SWAP_TAG_INDEX_LSBS__SHIFT 0x0000001c
+#define VM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE_MASK 0x00000030L
+#define VM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE__SHIFT 0x00000004
+#define VM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE_MASK 0x0000000cL
+#define VM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE__SHIFT 0x00000002
+#define VM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE_MASK 0x00007000L
+#define VM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE__SHIFT 0x0000000c
+#define VM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE_MASK 0x00000100L
+#define VM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE__SHIFT 0x00000008
+#define VM_L2_CNTL__PDE_FAULT_CLASSIFICATION_MASK 0x00040000L
+#define VM_L2_CNTL__PDE_FAULT_CLASSIFICATION__SHIFT 0x00000012
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET__PHYSICAL_PAGE_OFFSET_MASK 0x0fffffffL
+#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET__PHYSICAL_PAGE_OFFSET__SHIFT 0x00000000
+#define VM_L2_STATUS__CONTEXT_DOMAIN_BUSY_MASK 0x0001fffeL
+#define VM_L2_STATUS__CONTEXT_DOMAIN_BUSY__SHIFT 0x00000001
+#define VM_L2_STATUS__L2_BUSY_MASK 0x00000001L
+#define VM_L2_STATUS__L2_BUSY__SHIFT 0x00000000
+#define VM_PRT_APERTURE0_HIGH_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_PRT_APERTURE0_HIGH_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_PRT_APERTURE0_LOW_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_PRT_APERTURE0_LOW_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_PRT_APERTURE1_HIGH_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_PRT_APERTURE1_HIGH_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_PRT_APERTURE1_LOW_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_PRT_APERTURE1_LOW_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_PRT_APERTURE2_HIGH_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_PRT_APERTURE2_HIGH_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_PRT_APERTURE2_LOW_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_PRT_APERTURE2_LOW_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_PRT_APERTURE3_HIGH_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_PRT_APERTURE3_HIGH_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_PRT_APERTURE3_LOW_ADDR__LOGICAL_PAGE_NUMBER_MASK 0x0fffffffL
+#define VM_PRT_APERTURE3_LOW_ADDR__LOGICAL_PAGE_NUMBER__SHIFT 0x00000000
+#define VM_PRT_CNTL__L1_TLB_STORE_INVALID_ENTRIES_MASK 0x00000008L
+#define VM_PRT_CNTL__L1_TLB_STORE_INVALID_ENTRIES__SHIFT 0x00000003
+#define VM_PRT_CNTL__L2_CACHE_STORE_INVALID_ENTRIES_MASK 0x00000004L
+#define VM_PRT_CNTL__L2_CACHE_STORE_INVALID_ENTRIES__SHIFT 0x00000002
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/oss_1_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/oss/oss_1_0_d.h
new file mode 100644
index 000000000000..edc8a793a95d
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/oss/oss_1_0_d.h
@@ -0,0 +1,275 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef OSS_1_0_D_H
+#define OSS_1_0_D_H
+
+#define ixCLIENT0_BM 0x0220
+#define ixCLIENT0_CD0 0x0210
+#define ixCLIENT0_CD1 0x0214
+#define ixCLIENT0_CD2 0x0218
+#define ixCLIENT0_CD3 0x021C
+#define ixCLIENT0_CK0 0x0200
+#define ixCLIENT0_CK1 0x0204
+#define ixCLIENT0_CK2 0x0208
+#define ixCLIENT0_CK3 0x020C
+#define ixCLIENT0_K0 0x01F0
+#define ixCLIENT0_K1 0x01F4
+#define ixCLIENT0_K2 0x01F8
+#define ixCLIENT0_K3 0x01FC
+#define ixCLIENT0_OFFSET 0x0224
+#define ixCLIENT0_OFFSET_HI 0x0290
+#define ixCLIENT0_STATUS 0x0228
+#define ixCLIENT1_BM 0x025C
+#define ixCLIENT1_CD0 0x024C
+#define ixCLIENT1_CD1 0x0250
+#define ixCLIENT1_CD2 0x0254
+#define ixCLIENT1_CD3 0x0258
+#define ixCLIENT1_CK0 0x023C
+#define ixCLIENT1_CK1 0x0240
+#define ixCLIENT1_CK2 0x0244
+#define ixCLIENT1_CK3 0x0248
+#define ixCLIENT1_K0 0x022C
+#define ixCLIENT1_K1 0x0230
+#define ixCLIENT1_K2 0x0234
+#define ixCLIENT1_K3 0x0238
+#define ixCLIENT1_OFFSET 0x0260
+#define ixCLIENT1_OFFSET_HI 0x0294
+#define ixCLIENT1_PORT_STATUS 0x0264
+#define ixCLIENT2_BM 0x01E4
+#define ixCLIENT2_CD0 0x01D4
+#define ixCLIENT2_CD1 0x01D8
+#define ixCLIENT2_CD2 0x01DC
+#define ixCLIENT2_CD3 0x01E0
+#define ixCLIENT2_CK0 0x01C4
+#define ixCLIENT2_CK1 0x01C8
+#define ixCLIENT2_CK2 0x01CC
+#define ixCLIENT2_CK3 0x01D0
+#define ixCLIENT2_K0 0x01B4
+#define ixCLIENT2_K1 0x01B8
+#define ixCLIENT2_K2 0x01BC
+#define ixCLIENT2_K3 0x01C0
+#define ixCLIENT2_OFFSET 0x01E8
+#define ixCLIENT2_OFFSET_HI 0x0298
+#define ixCLIENT2_STATUS 0x01EC
+#define ixCLIENT3_BM 0x02D4
+#define ixCLIENT3_CD0 0x02C4
+#define ixCLIENT3_CD1 0x02C8
+#define ixCLIENT3_CD2 0x02CC
+#define ixCLIENT3_CD3 0x02D0
+#define ixCLIENT3_CK0 0x02B4
+#define ixCLIENT3_CK1 0x02B8
+#define ixCLIENT3_CK2 0x02BC
+#define ixCLIENT3_CK3 0x02C0
+#define ixCLIENT3_K0 0x02A4
+#define ixCLIENT3_K1 0x02A8
+#define ixCLIENT3_K2 0x02AC
+#define ixCLIENT3_K3 0x02B0
+#define ixCLIENT3_OFFSET 0x02D8
+#define ixCLIENT3_OFFSET_HI 0x02A0
+#define ixCLIENT3_STATUS 0x02DC
+#define ixDH_TEST 0x0000
+#define ixEXP0 0x0034
+#define ixEXP1 0x0038
+#define ixEXP2 0x003C
+#define ixEXP3 0x0040
+#define ixEXP4 0x0044
+#define ixEXP5 0x0048
+#define ixEXP6 0x004C
+#define ixEXP7 0x0050
+#define ixHFS_SEED0 0x0278
+#define ixHFS_SEED1 0x027C
+#define ixHFS_SEED2 0x0280
+#define ixHFS_SEED3 0x0284
+#define ixKEFUSE0 0x0268
+#define ixKEFUSE1 0x026C
+#define ixKEFUSE2 0x0270
+#define ixKEFUSE3 0x0274
+#define ixKHFS0 0x0004
+#define ixKHFS1 0x0008
+#define ixKHFS2 0x000C
+#define ixKHFS3 0x0010
+#define ixKSESSION0 0x0014
+#define ixKSESSION1 0x0018
+#define ixKSESSION2 0x001C
+#define ixKSESSION3 0x0020
+#define ixKSIG0 0x0024
+#define ixKSIG1 0x0028
+#define ixKSIG2 0x002C
+#define ixKSIG3 0x0030
+#define ixLX0 0x0054
+#define ixLX1 0x0058
+#define ixLX2 0x005C
+#define ixLX3 0x0060
+#define ixRINGOSC_MASK 0x0288
+#define ixSPU_PORT_STATUS 0x029C
+#define mmCC_DRM_ID_STRAPS 0x1559
+#define mmCC_SYS_RB_BACKEND_DISABLE 0x03A0
+#define mmCC_SYS_RB_REDUNDANCY 0x039F
+#define mmCGTT_DRM_CLK_CTRL0 0x1579
+#define mmCP_CONFIG 0x0F92
+#define mmDC_TEST_DEBUG_DATA 0x157D
+#define mmDC_TEST_DEBUG_INDEX 0x157C
+#define mmGC_USER_SYS_RB_BACKEND_DISABLE 0x03A1
+#define mmHDP_ADDR_CONFIG 0x0BD2
+#define mmHDP_DEBUG0 0x0BCC
+#define mmHDP_DEBUG1 0x0BCD
+#define mmHDP_HOST_PATH_CNTL 0x0B00
+#define mmHDP_LAST_SURFACE_HIT 0x0BCE
+#define mmHDP_MEMIO_ADDR 0x0BF7
+#define mmHDP_MEMIO_CNTL 0x0BF6
+#define mmHDP_MEMIO_RD_DATA 0x0BFA
+#define mmHDP_MEMIO_STATUS 0x0BF8
+#define mmHDP_MEMIO_WR_DATA 0x0BF9
+#define mmHDP_MEM_POWER_LS 0x0BD4
+#define mmHDP_MISC_CNTL 0x0BD3
+#define mmHDP_NONSURFACE_BASE 0x0B01
+#define mmHDP_NONSURFACE_INFO 0x0B02
+#define mmHDP_NONSURFACE_PREFETCH 0x0BD5
+#define mmHDP_NONSURFACE_SIZE 0x0B03
+#define mmHDP_NONSURF_FLAGS 0x0BC9
+#define mmHDP_NONSURF_FLAGS_CLR 0x0BCA
+#define mmHDP_OUTSTANDING_REQ 0x0BD1
+#define mmHDP_SC_MULTI_CHIP_CNTL 0x0BD0
+#define mmHDP_SW_SEMAPHORE 0x0BCB
+#define mmHDP_TILING_CONFIG 0x0BCF
+#define mmHDP_XDP_BARS_ADDR_39_36 0x0C44
+#define mmHDP_XDP_BUSY_STS 0x0C3E
+#define mmHDP_XDP_CGTT_BLK_CTRL 0x0C33
+#define mmHDP_XDP_CHKN 0x0C40
+#define mmHDP_XDP_D2H_BAR_UPDATE 0x0C02
+#define mmHDP_XDP_D2H_FLUSH 0x0C01
+#define mmHDP_XDP_D2H_RSVD_10 0x0C0A
+#define mmHDP_XDP_D2H_RSVD_11 0x0C0B
+#define mmHDP_XDP_D2H_RSVD_12 0x0C0C
+#define mmHDP_XDP_D2H_RSVD_13 0x0C0D
+#define mmHDP_XDP_D2H_RSVD_14 0x0C0E
+#define mmHDP_XDP_D2H_RSVD_15 0x0C0F
+#define mmHDP_XDP_D2H_RSVD_16 0x0C10
+#define mmHDP_XDP_D2H_RSVD_17 0x0C11
+#define mmHDP_XDP_D2H_RSVD_18 0x0C12
+#define mmHDP_XDP_D2H_RSVD_19 0x0C13
+#define mmHDP_XDP_D2H_RSVD_20 0x0C14
+#define mmHDP_XDP_D2H_RSVD_21 0x0C15
+#define mmHDP_XDP_D2H_RSVD_22 0x0C16
+#define mmHDP_XDP_D2H_RSVD_23 0x0C17
+#define mmHDP_XDP_D2H_RSVD_24 0x0C18
+#define mmHDP_XDP_D2H_RSVD_25 0x0C19
+#define mmHDP_XDP_D2H_RSVD_26 0x0C1A
+#define mmHDP_XDP_D2H_RSVD_27 0x0C1B
+#define mmHDP_XDP_D2H_RSVD_28 0x0C1C
+#define mmHDP_XDP_D2H_RSVD_29 0x0C1D
+#define mmHDP_XDP_D2H_RSVD_30 0x0C1E
+#define mmHDP_XDP_D2H_RSVD_3 0x0C03
+#define mmHDP_XDP_D2H_RSVD_31 0x0C1F
+#define mmHDP_XDP_D2H_RSVD_32 0x0C20
+#define mmHDP_XDP_D2H_RSVD_33 0x0C21
+#define mmHDP_XDP_D2H_RSVD_34 0x0C22
+#define mmHDP_XDP_D2H_RSVD_4 0x0C04
+#define mmHDP_XDP_D2H_RSVD_5 0x0C05
+#define mmHDP_XDP_D2H_RSVD_6 0x0C06
+#define mmHDP_XDP_D2H_RSVD_7 0x0C07
+#define mmHDP_XDP_D2H_RSVD_8 0x0C08
+#define mmHDP_XDP_D2H_RSVD_9 0x0C09
+#define mmHDP_XDP_DBG_ADDR 0x0C41
+#define mmHDP_XDP_DBG_DATA 0x0C42
+#define mmHDP_XDP_DBG_MASK 0x0C43
+#define mmHDP_XDP_DIRECT2HDP_FIRST 0x0C00
+#define mmHDP_XDP_DIRECT2HDP_LAST 0x0C23
+#define mmHDP_XDP_FLUSH_ARMED_STS 0x0C3C
+#define mmHDP_XDP_FLUSH_CNTR0_STS 0x0C3D
+#define mmHDP_XDP_HDP_IPH_CFG 0x0C31
+#define mmHDP_XDP_HDP_MBX_MC_CFG 0x0C2D
+#define mmHDP_XDP_HDP_MC_CFG 0x0C2E
+#define mmHDP_XDP_HST_CFG 0x0C2F
+#define mmHDP_XDP_P2P_BAR0 0x0C34
+#define mmHDP_XDP_P2P_BAR1 0x0C35
+#define mmHDP_XDP_P2P_BAR2 0x0C36
+#define mmHDP_XDP_P2P_BAR3 0x0C37
+#define mmHDP_XDP_P2P_BAR4 0x0C38
+#define mmHDP_XDP_P2P_BAR5 0x0C39
+#define mmHDP_XDP_P2P_BAR6 0x0C3A
+#define mmHDP_XDP_P2P_BAR7 0x0C3B
+#define mmHDP_XDP_P2P_BAR_CFG 0x0C24
+#define mmHDP_XDP_P2P_MBX_ADDR0 0x0C26
+#define mmHDP_XDP_P2P_MBX_ADDR1 0x0C27
+#define mmHDP_XDP_P2P_MBX_ADDR2 0x0C28
+#define mmHDP_XDP_P2P_MBX_ADDR3 0x0C29
+#define mmHDP_XDP_P2P_MBX_ADDR4 0x0C2A
+#define mmHDP_XDP_P2P_MBX_ADDR5 0x0C2B
+#define mmHDP_XDP_P2P_MBX_ADDR6 0x0C2C
+#define mmHDP_XDP_P2P_MBX_OFFSET 0x0C25
+#define mmHDP_XDP_SID_CFG 0x0C30
+#define mmHDP_XDP_SRBM_CFG 0x0C32
+#define mmHDP_XDP_STICKY 0x0C3F
+#define mmIH_ADVFAULT_CNTL 0x0F8C
+#define mmIH_CNTL 0x0F86
+#define mmIH_LEVEL_STATUS 0x0F87
+#define mmIH_PERFCOUNTER0_RESULT 0x0F8A
+#define mmIH_PERFCOUNTER1_RESULT 0x0F8B
+#define mmIH_PERFMON_CNTL 0x0F89
+#define mmIH_RB_BASE 0x0F81
+#define mmIH_RB_CNTL 0x0F80
+#define mmIH_RB_RPTR 0x0F82
+#define mmIH_RB_WPTR 0x0F83
+#define mmIH_RB_WPTR_ADDR_HI 0x0F84
+#define mmIH_RB_WPTR_ADDR_LO 0x0F85
+#define mmIH_STATUS 0x0F88
+#define mmSEM_MAILBOX 0x0F9B
+#define mmSEM_MAILBOX_CLIENTCONFIG 0x0F9A
+#define mmSEM_MAILBOX_CONTROL 0x0F9C
+#define mmSEM_MCIF_CONFIG 0x0F90
+#define mmSRBM_CAM_DATA 0x0397
+#define mmSRBM_CAM_INDEX 0x0396
+#define mmSRBM_CHIP_REVISION 0x039B
+#define mmSRBM_CNTL 0x0390
+#define mmSRBM_DEBUG 0x03A4
+#define mmSRBM_DEBUG_CNTL 0x0399
+#define mmSRBM_DEBUG_DATA 0x039A
+#define mmSRBM_DEBUG_SNAPSHOT 0x03A5
+#define mmSRBM_GFX_CNTL 0x0391
+#define mmSRBM_INT_ACK 0x03AA
+#define mmSRBM_INT_CNTL 0x03A8
+#define mmSRBM_INT_STATUS 0x03A9
+#define mmSRBM_MC_CLKEN_CNTL 0x03B3
+#define mmSRBM_PERFCOUNTER0_HI 0x0704
+#define mmSRBM_PERFCOUNTER0_LO 0x0703
+#define mmSRBM_PERFCOUNTER0_SELECT 0x0701
+#define mmSRBM_PERFCOUNTER1_HI 0x0706
+#define mmSRBM_PERFCOUNTER1_LO 0x0705
+#define mmSRBM_PERFCOUNTER1_SELECT 0x0702
+#define mmSRBM_PERFMON_CNTL 0x0700
+#define mmSRBM_READ_ERROR 0x03A6
+#define mmSRBM_SOFT_RESET 0x0398
+#define mmSRBM_STATUS 0x0394
+#define mmSRBM_STATUS2 0x0393
+#define mmSRBM_SYS_CLKEN_CNTL 0x03B4
+#define mmSRBM_UVD_CLKEN_CNTL 0x03B6
+#define mmSRBM_VCE_CLKEN_CNTL 0x03B5
+#define mmUVD_CONFIG 0x0F98
+#define mmVCE_CONFIG 0x0F94
+#define mmXDMA_MSTR_MEM_OVERFLOW_CNTL 0x03F8
+
+/* from the old sid.h */
+#define mmDMA_TILING_CONFIG 0x342E
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/oss_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/oss/oss_1_0_sh_mask.h
new file mode 100644
index 000000000000..1c540fe136cb
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/oss/oss_1_0_sh_mask.h
@@ -0,0 +1,1079 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef OSS_1_0_SH_MASK_H
+#define OSS_1_0_SH_MASK_H
+
+#define CC_DRM_ID_STRAPS__ATI_REV_ID_MASK 0xf0000000L
+#define CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT 0x0000001c
+#define CC_DRM_ID_STRAPS__DEVICE_ID_MASK 0x000ffff0L
+#define CC_DRM_ID_STRAPS__DEVICE_ID__SHIFT 0x00000004
+#define CC_DRM_ID_STRAPS__MAJOR_REV_ID_MASK 0x00f00000L
+#define CC_DRM_ID_STRAPS__MAJOR_REV_ID__SHIFT 0x00000014
+#define CC_DRM_ID_STRAPS__MINOR_REV_ID_MASK 0x0f000000L
+#define CC_DRM_ID_STRAPS__MINOR_REV_ID__SHIFT 0x00000018
+#define CC_SYS_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK 0x00ff0000L
+#define CC_SYS_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT 0x00000010
+#define CLIENT0_BM__RESERVED_MASK 0xffffffffL
+#define CLIENT0_BM__RESERVED__SHIFT 0x00000000
+#define CLIENT0_CD0__RESERVED_MASK 0xffffffffL
+#define CLIENT0_CD0__RESERVED__SHIFT 0x00000000
+#define CLIENT0_CD1__RESERVED_MASK 0xffffffffL
+#define CLIENT0_CD1__RESERVED__SHIFT 0x00000000
+#define CLIENT0_CD2__RESERVED_MASK 0xffffffffL
+#define CLIENT0_CD2__RESERVED__SHIFT 0x00000000
+#define CLIENT0_CD3__RESERVED_MASK 0xffffffffL
+#define CLIENT0_CD3__RESERVED__SHIFT 0x00000000
+#define CLIENT0_CK0__RESERVED_MASK 0xffffffffL
+#define CLIENT0_CK0__RESERVED__SHIFT 0x00000000
+#define CLIENT0_CK1__RESERVED_MASK 0xffffffffL
+#define CLIENT0_CK1__RESERVED__SHIFT 0x00000000
+#define CLIENT0_CK2__RESERVED_MASK 0xffffffffL
+#define CLIENT0_CK2__RESERVED__SHIFT 0x00000000
+#define CLIENT0_CK3__RESERVED_MASK 0xffffffffL
+#define CLIENT0_CK3__RESERVED__SHIFT 0x00000000
+#define CLIENT0_K0__RESERVED_MASK 0xffffffffL
+#define CLIENT0_K0__RESERVED__SHIFT 0x00000000
+#define CLIENT0_K1__RESERVED_MASK 0xffffffffL
+#define CLIENT0_K1__RESERVED__SHIFT 0x00000000
+#define CLIENT0_K2__RESERVED_MASK 0xffffffffL
+#define CLIENT0_K2__RESERVED__SHIFT 0x00000000
+#define CLIENT0_K3__RESERVED_MASK 0xffffffffL
+#define CLIENT0_K3__RESERVED__SHIFT 0x00000000
+#define CLIENT0_OFFSET_HI__RESERVED_MASK 0xffffffffL
+#define CLIENT0_OFFSET_HI__RESERVED__SHIFT 0x00000000
+#define CLIENT0_OFFSET__RESERVED_MASK 0xffffffffL
+#define CLIENT0_OFFSET__RESERVED__SHIFT 0x00000000
+#define CLIENT0_STATUS__RESERVED_MASK 0xffffffffL
+#define CLIENT0_STATUS__RESERVED__SHIFT 0x00000000
+#define CLIENT1_BM__RESERVED_MASK 0xffffffffL
+#define CLIENT1_BM__RESERVED__SHIFT 0x00000000
+#define CLIENT1_CD0__RESERVED_MASK 0xffffffffL
+#define CLIENT1_CD0__RESERVED__SHIFT 0x00000000
+#define CLIENT1_CD1__RESERVED_MASK 0xffffffffL
+#define CLIENT1_CD1__RESERVED__SHIFT 0x00000000
+#define CLIENT1_CD2__RESERVED_MASK 0xffffffffL
+#define CLIENT1_CD2__RESERVED__SHIFT 0x00000000
+#define CLIENT1_CD3__RESERVED_MASK 0xffffffffL
+#define CLIENT1_CD3__RESERVED__SHIFT 0x00000000
+#define CLIENT1_CK0__RESERVED_MASK 0xffffffffL
+#define CLIENT1_CK0__RESERVED__SHIFT 0x00000000
+#define CLIENT1_CK1__RESERVED_MASK 0xffffffffL
+#define CLIENT1_CK1__RESERVED__SHIFT 0x00000000
+#define CLIENT1_CK2__RESERVED_MASK 0xffffffffL
+#define CLIENT1_CK2__RESERVED__SHIFT 0x00000000
+#define CLIENT1_CK3__RESERVED_MASK 0xffffffffL
+#define CLIENT1_CK3__RESERVED__SHIFT 0x00000000
+#define CLIENT1_K0__RESERVED_MASK 0xffffffffL
+#define CLIENT1_K0__RESERVED__SHIFT 0x00000000
+#define CLIENT1_K1__RESERVED_MASK 0xffffffffL
+#define CLIENT1_K1__RESERVED__SHIFT 0x00000000
+#define CLIENT1_K2__RESERVED_MASK 0xffffffffL
+#define CLIENT1_K2__RESERVED__SHIFT 0x00000000
+#define CLIENT1_K3__RESERVED_MASK 0xffffffffL
+#define CLIENT1_K3__RESERVED__SHIFT 0x00000000
+#define CLIENT1_OFFSET_HI__RESERVED_MASK 0xffffffffL
+#define CLIENT1_OFFSET_HI__RESERVED__SHIFT 0x00000000
+#define CLIENT1_OFFSET__RESERVED_MASK 0xffffffffL
+#define CLIENT1_OFFSET__RESERVED__SHIFT 0x00000000
+#define CLIENT1_PORT_STATUS__RESERVED_MASK 0xffffffffL
+#define CLIENT1_PORT_STATUS__RESERVED__SHIFT 0x00000000
+#define CLIENT2_BM__RESERVED_MASK 0xffffffffL
+#define CLIENT2_BM__RESERVED__SHIFT 0x00000000
+#define CLIENT2_CD0__RESERVED_MASK 0xffffffffL
+#define CLIENT2_CD0__RESERVED__SHIFT 0x00000000
+#define CLIENT2_CD1__RESERVED_MASK 0xffffffffL
+#define CLIENT2_CD1__RESERVED__SHIFT 0x00000000
+#define CLIENT2_CD2__RESERVED_MASK 0xffffffffL
+#define CLIENT2_CD2__RESERVED__SHIFT 0x00000000
+#define CLIENT2_CD3__RESERVED_MASK 0xffffffffL
+#define CLIENT2_CD3__RESERVED__SHIFT 0x00000000
+#define CLIENT2_CK0__RESERVED_MASK 0xffffffffL
+#define CLIENT2_CK0__RESERVED__SHIFT 0x00000000
+#define CLIENT2_CK1__RESERVED_MASK 0xffffffffL
+#define CLIENT2_CK1__RESERVED__SHIFT 0x00000000
+#define CLIENT2_CK2__RESERVED_MASK 0xffffffffL
+#define CLIENT2_CK2__RESERVED__SHIFT 0x00000000
+#define CLIENT2_CK3__RESERVED_MASK 0xffffffffL
+#define CLIENT2_CK3__RESERVED__SHIFT 0x00000000
+#define CLIENT2_K0__RESERVED_MASK 0xffffffffL
+#define CLIENT2_K0__RESERVED__SHIFT 0x00000000
+#define CLIENT2_K1__RESERVED_MASK 0xffffffffL
+#define CLIENT2_K1__RESERVED__SHIFT 0x00000000
+#define CLIENT2_K2__RESERVED_MASK 0xffffffffL
+#define CLIENT2_K2__RESERVED__SHIFT 0x00000000
+#define CLIENT2_K3__RESERVED_MASK 0xffffffffL
+#define CLIENT2_K3__RESERVED__SHIFT 0x00000000
+#define CLIENT2_OFFSET_HI__RESERVED_MASK 0xffffffffL
+#define CLIENT2_OFFSET_HI__RESERVED__SHIFT 0x00000000
+#define CLIENT2_OFFSET__RESERVED_MASK 0xffffffffL
+#define CLIENT2_OFFSET__RESERVED__SHIFT 0x00000000
+#define CLIENT2_STATUS__RESERVED_MASK 0xffffffffL
+#define CLIENT2_STATUS__RESERVED__SHIFT 0x00000000
+#define CLIENT3_BM__RESERVED_MASK 0xffffffffL
+#define CLIENT3_BM__RESERVED__SHIFT 0x00000000
+#define CLIENT3_CD0__RESERVED_MASK 0xffffffffL
+#define CLIENT3_CD0__RESERVED__SHIFT 0x00000000
+#define CLIENT3_CD1__RESERVED_MASK 0xffffffffL
+#define CLIENT3_CD1__RESERVED__SHIFT 0x00000000
+#define CLIENT3_CD2__RESERVED_MASK 0xffffffffL
+#define CLIENT3_CD2__RESERVED__SHIFT 0x00000000
+#define CLIENT3_CD3__RESERVED_MASK 0xffffffffL
+#define CLIENT3_CD3__RESERVED__SHIFT 0x00000000
+#define CLIENT3_CK0__RESERVED_MASK 0xffffffffL
+#define CLIENT3_CK0__RESERVED__SHIFT 0x00000000
+#define CLIENT3_CK1__RESERVED_MASK 0xffffffffL
+#define CLIENT3_CK1__RESERVED__SHIFT 0x00000000
+#define CLIENT3_CK2__RESERVED_MASK 0xffffffffL
+#define CLIENT3_CK2__RESERVED__SHIFT 0x00000000
+#define CLIENT3_CK3__RESERVED_MASK 0xffffffffL
+#define CLIENT3_CK3__RESERVED__SHIFT 0x00000000
+#define CLIENT3_K0__RESERVED_MASK 0xffffffffL
+#define CLIENT3_K0__RESERVED__SHIFT 0x00000000
+#define CLIENT3_K1__RESERVED_MASK 0xffffffffL
+#define CLIENT3_K1__RESERVED__SHIFT 0x00000000
+#define CLIENT3_K2__RESERVED_MASK 0xffffffffL
+#define CLIENT3_K2__RESERVED__SHIFT 0x00000000
+#define CLIENT3_K3__RESERVED_MASK 0xffffffffL
+#define CLIENT3_K3__RESERVED__SHIFT 0x00000000
+#define CLIENT3_OFFSET_HI__RESERVED_MASK 0xffffffffL
+#define CLIENT3_OFFSET_HI__RESERVED__SHIFT 0x00000000
+#define CLIENT3_OFFSET__RESERVED_MASK 0xffffffffL
+#define CLIENT3_OFFSET__RESERVED__SHIFT 0x00000000
+#define CLIENT3_STATUS__RESERVED_MASK 0xffffffffL
+#define CLIENT3_STATUS__RESERVED__SHIFT 0x00000000
+#define CP_CONFIG__CP_RDREQ_URG_MASK 0x00000f00L
+#define CP_CONFIG__CP_RDREQ_URG__SHIFT 0x00000008
+#define CP_CONFIG__CP_REQ_TRAN_MASK 0x00010000L
+#define CP_CONFIG__CP_REQ_TRAN__SHIFT 0x00000010
+#define DC_TEST_DEBUG_DATA__DC_TEST_DEBUG_DATA_MASK 0xffffffffL
+#define DC_TEST_DEBUG_DATA__DC_TEST_DEBUG_DATA__SHIFT 0x00000000
+#define DC_TEST_DEBUG_INDEX__DC_TEST_DEBUG_INDEX_MASK 0x000000ffL
+#define DC_TEST_DEBUG_INDEX__DC_TEST_DEBUG_INDEX__SHIFT 0x00000000
+#define DC_TEST_DEBUG_INDEX__DC_TEST_DEBUG_WRITE_EN_MASK 0x00000100L
+#define DC_TEST_DEBUG_INDEX__DC_TEST_DEBUG_WRITE_EN__SHIFT 0x00000008
+#define DH_TEST__DH_TEST_MASK 0x00000001L
+#define DH_TEST__DH_TEST__SHIFT 0x00000000
+#define EXP0__RESERVED_MASK 0xffffffffL
+#define EXP0__RESERVED__SHIFT 0x00000000
+#define EXP1__RESERVED_MASK 0xffffffffL
+#define EXP1__RESERVED__SHIFT 0x00000000
+#define EXP2__RESERVED_MASK 0xffffffffL
+#define EXP2__RESERVED__SHIFT 0x00000000
+#define EXP3__RESERVED_MASK 0xffffffffL
+#define EXP3__RESERVED__SHIFT 0x00000000
+#define EXP4__RESERVED_MASK 0xffffffffL
+#define EXP4__RESERVED__SHIFT 0x00000000
+#define EXP5__RESERVED_MASK 0xffffffffL
+#define EXP5__RESERVED__SHIFT 0x00000000
+#define EXP6__RESERVED_MASK 0xffffffffL
+#define EXP6__RESERVED__SHIFT 0x00000000
+#define EXP7__RESERVED_MASK 0xffffffffL
+#define EXP7__RESERVED__SHIFT 0x00000000
+#define GC_USER_SYS_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK 0x00ff0000L
+#define GC_USER_SYS_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT 0x00000010
+#define HDP_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK 0x00000700L
+#define HDP_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT 0x00000008
+#define HDP_ADDR_CONFIG__MULTI_GPU_TILE_SIZE_MASK 0x03000000L
+#define HDP_ADDR_CONFIG__MULTI_GPU_TILE_SIZE__SHIFT 0x00000018
+#define HDP_ADDR_CONFIG__NUM_GPUS_MASK 0x00700000L
+#define HDP_ADDR_CONFIG__NUM_GPUS__SHIFT 0x00000014
+#define HDP_ADDR_CONFIG__NUM_LOWER_PIPES_MASK 0x40000000L
+#define HDP_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT 0x0000001e
+#define HDP_ADDR_CONFIG__NUM_PIPES_MASK 0x00000007L
+#define HDP_ADDR_CONFIG__NUM_PIPES__SHIFT 0x00000000
+#define HDP_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK 0x00003000L
+#define HDP_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT 0x0000000c
+#define HDP_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK 0x00000070L
+#define HDP_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT 0x00000004
+#define HDP_ADDR_CONFIG__ROW_SIZE_MASK 0x30000000L
+#define HDP_ADDR_CONFIG__ROW_SIZE__SHIFT 0x0000001c
+#define HDP_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK 0x00070000L
+#define HDP_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT 0x00000010
+#define HDP_DEBUG0__HDP_DEBUG__SHIFT 0x00000000
+#define HDP_DEBUG1__HDP_DEBUG__SHIFT 0x00000000
+#define HDP_HOST_PATH_CNTL__ALL_SURFACES_DIS_MASK 0x20000000L
+#define HDP_HOST_PATH_CNTL__ALL_SURFACES_DIS__SHIFT 0x0000001d
+#define HDP_HOST_PATH_CNTL__BIF_RDRET_CREDIT_MASK 0x00000007L
+#define HDP_HOST_PATH_CNTL__BIF_RDRET_CREDIT__SHIFT 0x00000000
+#define HDP_HOST_PATH_CNTL__CACHE_INVALIDATE_MASK 0x00400000L
+#define HDP_HOST_PATH_CNTL__CACHE_INVALIDATE__SHIFT 0x00000016
+#define HDP_HOST_PATH_CNTL__CLOCK_GATING_DIS_MASK 0x00800000L
+#define HDP_HOST_PATH_CNTL__CLOCK_GATING_DIS__SHIFT 0x00000017
+#define HDP_HOST_PATH_CNTL__LIN_RD_CACHE_DIS_MASK 0x80000000L
+#define HDP_HOST_PATH_CNTL__LIN_RD_CACHE_DIS__SHIFT 0x0000001f
+#define HDP_HOST_PATH_CNTL__MC_WRREQ_CREDIT_MASK 0x000001f8L
+#define HDP_HOST_PATH_CNTL__MC_WRREQ_CREDIT__SHIFT 0x00000003
+#define HDP_HOST_PATH_CNTL__RD_STALL_TIMER_MASK 0x00001800L
+#define HDP_HOST_PATH_CNTL__RD_STALL_TIMER__SHIFT 0x0000000b
+#define HDP_HOST_PATH_CNTL__REG_CLK_ENABLE_COUNT_MASK 0x0f000000L
+#define HDP_HOST_PATH_CNTL__REG_CLK_ENABLE_COUNT__SHIFT 0x00000018
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_EN_MASK 0x00200000L
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_EN__SHIFT 0x00000015
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER_MASK 0x00180000L
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER__SHIFT 0x00000013
+#define HDP_HOST_PATH_CNTL__WRITE_THROUGH_CACHE_DIS_MASK 0x40000000L
+#define HDP_HOST_PATH_CNTL__WRITE_THROUGH_CACHE_DIS__SHIFT 0x0000001e
+#define HDP_HOST_PATH_CNTL__WR_STALL_TIMER_MASK 0x00000600L
+#define HDP_HOST_PATH_CNTL__WR_STALL_TIMER__SHIFT 0x00000009
+#define HDP_LAST_SURFACE_HIT__LAST_SURFACE_HIT_MASK 0x0000003fL
+#define HDP_LAST_SURFACE_HIT__LAST_SURFACE_HIT__SHIFT 0x00000000
+#define HDP_MEMIO_ADDR__MEMIO_ADDR_LOWER_MASK 0xffffffffL
+#define HDP_MEMIO_ADDR__MEMIO_ADDR_LOWER__SHIFT 0x00000000
+#define HDP_MEMIO_CNTL__MEMIO_ADDR_UPPER_MASK 0x00003f00L
+#define HDP_MEMIO_CNTL__MEMIO_ADDR_UPPER__SHIFT 0x00000008
+#define HDP_MEMIO_CNTL__MEMIO_BE_MASK 0x0000003cL
+#define HDP_MEMIO_CNTL__MEMIO_BE__SHIFT 0x00000002
+#define HDP_MEMIO_CNTL__MEMIO_CLR_RD_ERROR_MASK 0x00008000L
+#define HDP_MEMIO_CNTL__MEMIO_CLR_RD_ERROR__SHIFT 0x0000000f
+#define HDP_MEMIO_CNTL__MEMIO_CLR_WR_ERROR_MASK 0x00004000L
+#define HDP_MEMIO_CNTL__MEMIO_CLR_WR_ERROR__SHIFT 0x0000000e
+#define HDP_MEMIO_CNTL__MEMIO_OP_MASK 0x00000002L
+#define HDP_MEMIO_CNTL__MEMIO_OP__SHIFT 0x00000001
+#define HDP_MEMIO_CNTL__MEMIO_RD_STROBE_MASK 0x00000080L
+#define HDP_MEMIO_CNTL__MEMIO_RD_STROBE__SHIFT 0x00000007
+#define HDP_MEMIO_CNTL__MEMIO_SEND_MASK 0x00000001L
+#define HDP_MEMIO_CNTL__MEMIO_SEND__SHIFT 0x00000000
+#define HDP_MEMIO_CNTL__MEMIO_WR_STROBE_MASK 0x00000040L
+#define HDP_MEMIO_CNTL__MEMIO_WR_STROBE__SHIFT 0x00000006
+#define HDP_MEMIO_RD_DATA__MEMIO_RD_DATA_MASK 0xffffffffL
+#define HDP_MEMIO_RD_DATA__MEMIO_RD_DATA__SHIFT 0x00000000
+#define HDP_MEMIO_STATUS__MEMIO_RD_ERROR_MASK 0x00000008L
+#define HDP_MEMIO_STATUS__MEMIO_RD_ERROR__SHIFT 0x00000003
+#define HDP_MEMIO_STATUS__MEMIO_RD_STATUS_MASK 0x00000002L
+#define HDP_MEMIO_STATUS__MEMIO_RD_STATUS__SHIFT 0x00000001
+#define HDP_MEMIO_STATUS__MEMIO_WR_ERROR_MASK 0x00000004L
+#define HDP_MEMIO_STATUS__MEMIO_WR_ERROR__SHIFT 0x00000002
+#define HDP_MEMIO_STATUS__MEMIO_WR_STATUS_MASK 0x00000001L
+#define HDP_MEMIO_STATUS__MEMIO_WR_STATUS__SHIFT 0x00000000
+#define HDP_MEMIO_WR_DATA__MEMIO_WR_DATA_MASK 0xffffffffL
+#define HDP_MEMIO_WR_DATA__MEMIO_WR_DATA__SHIFT 0x00000000
+#define HDP_MEM_POWER_LS__LS_ENABLE_MASK 0x00000001L
+#define HDP_MEM_POWER_LS__LS_ENABLE__SHIFT 0x00000000
+#define HDP_MEM_POWER_LS__LS_HOLD_MASK 0x00001f80L
+#define HDP_MEM_POWER_LS__LS_HOLD__SHIFT 0x00000007
+#define HDP_MEM_POWER_LS__LS_SETUP_MASK 0x0000007eL
+#define HDP_MEM_POWER_LS__LS_SETUP__SHIFT 0x00000001
+#define HDP_MISC_CNTL__ADDRLIB_LINEAR_BYPASS_MASK 0x00100000L
+#define HDP_MISC_CNTL__ADDRLIB_LINEAR_BYPASS__SHIFT 0x00000014
+#define HDP_MISC_CNTL__FED_ENABLE_MASK 0x00200000L
+#define HDP_MISC_CNTL__FED_ENABLE__SHIFT 0x00000015
+#define HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE_MASK 0x00000001L
+#define HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE__SHIFT 0x00000000
+#define HDP_MISC_CNTL__HDP_BIF_RDRET_CREDIT_MASK 0x00000780L
+#define HDP_MISC_CNTL__HDP_BIF_RDRET_CREDIT__SHIFT 0x00000007
+#define HDP_MISC_CNTL__MC_RDREQ_CREDIT_MASK 0x0007e000L
+#define HDP_MISC_CNTL__MC_RDREQ_CREDIT__SHIFT 0x0000000d
+#define HDP_MISC_CNTL__MULTIPLE_READS_MASK 0x00000040L
+#define HDP_MISC_CNTL__MULTIPLE_READS__SHIFT 0x00000006
+#define HDP_MISC_CNTL__NO_SPLIT_ARRAY_LINEAR_MASK 0x00001000L
+#define HDP_MISC_CNTL__NO_SPLIT_ARRAY_LINEAR__SHIFT 0x0000000c
+#define HDP_MISC_CNTL__OUTSTANDING_WRITE_COUNT_1024_MASK 0x00000020L
+#define HDP_MISC_CNTL__OUTSTANDING_WRITE_COUNT_1024__SHIFT 0x00000005
+#define HDP_MISC_CNTL__READ_CACHE_INVALIDATE_MASK 0x00080000L
+#define HDP_MISC_CNTL__READ_CACHE_INVALIDATE__SHIFT 0x00000013
+#define HDP_MISC_CNTL__SIMULTANEOUS_READS_WRITES_MASK 0x00000800L
+#define HDP_MISC_CNTL__SIMULTANEOUS_READS_WRITES__SHIFT 0x0000000b
+#define HDP_MISC_CNTL__VM_ID_MASK 0x0000001eL
+#define HDP_MISC_CNTL__VM_ID__SHIFT 0x00000001
+#define HDP_NONSURFACE_BASE__NONSURF_BASE_MASK 0xffffffffL
+#define HDP_NONSURFACE_BASE__NONSURF_BASE__SHIFT 0x00000000
+#define HDP_NONSURFACE_INFO__NONSURF_ADDR_TYPE_MASK 0x00000001L
+#define HDP_NONSURFACE_INFO__NONSURF_ADDR_TYPE__SHIFT 0x00000000
+#define HDP_NONSURFACE_INFO__NONSURF_ARRAY_MODE_MASK 0x0000001eL
+#define HDP_NONSURFACE_INFO__NONSURF_ARRAY_MODE__SHIFT 0x00000001
+#define HDP_NONSURFACE_INFO__NONSURF_BANK_HEIGHT_MASK 0x03000000L
+#define HDP_NONSURFACE_INFO__NONSURF_BANK_HEIGHT__SHIFT 0x00000018
+#define HDP_NONSURFACE_INFO__NONSURF_BANK_WIDTH_MASK 0x00c00000L
+#define HDP_NONSURFACE_INFO__NONSURF_BANK_WIDTH__SHIFT 0x00000016
+#define HDP_NONSURFACE_INFO__NONSURF_ENDIAN_MASK 0x00000060L
+#define HDP_NONSURFACE_INFO__NONSURF_ENDIAN__SHIFT 0x00000005
+#define HDP_NONSURFACE_INFO__NONSURF_MACRO_TILE_ASPECT_MASK 0x0c000000L
+#define HDP_NONSURFACE_INFO__NONSURF_MACRO_TILE_ASPECT__SHIFT 0x0000001a
+#define HDP_NONSURFACE_INFO__NONSURF_MICRO_TILE_MODE_MASK 0x30000000L
+#define HDP_NONSURFACE_INFO__NONSURF_MICRO_TILE_MODE__SHIFT 0x0000001c
+#define HDP_NONSURFACE_INFO__NONSURF_NUM_BANKS_MASK 0x00300000L
+#define HDP_NONSURFACE_INFO__NONSURF_NUM_BANKS__SHIFT 0x00000014
+#define HDP_NONSURFACE_INFO__NONSURF_PIXEL_SIZE_MASK 0x00000380L
+#define HDP_NONSURFACE_INFO__NONSURF_PIXEL_SIZE__SHIFT 0x00000007
+#define HDP_NONSURFACE_INFO__NONSURF_PRIV_MASK 0x00008000L
+#define HDP_NONSURFACE_INFO__NONSURF_PRIV__SHIFT 0x0000000f
+#define HDP_NONSURFACE_INFO__NONSURF_SAMPLE_NUM_MASK 0x00001c00L
+#define HDP_NONSURFACE_INFO__NONSURF_SAMPLE_NUM__SHIFT 0x0000000a
+#define HDP_NONSURFACE_INFO__NONSURF_SAMPLE_SIZE_MASK 0x00006000L
+#define HDP_NONSURFACE_INFO__NONSURF_SAMPLE_SIZE__SHIFT 0x0000000d
+#define HDP_NONSURFACE_INFO__NONSURF_SLICE_TILE_MAX_MSB_MASK 0x40000000L
+#define HDP_NONSURFACE_INFO__NONSURF_SLICE_TILE_MAX_MSB__SHIFT 0x0000001e
+#define HDP_NONSURFACE_INFO__NONSURF_TILE_COMPACT_MASK 0x00010000L
+#define HDP_NONSURFACE_INFO__NONSURF_TILE_COMPACT__SHIFT 0x00000010
+#define HDP_NONSURFACE_INFO__NONSURF_TILE_SPLIT_MASK 0x000e0000L
+#define HDP_NONSURFACE_INFO__NONSURF_TILE_SPLIT__SHIFT 0x00000011
+#define HDP_NONSURFACE_PREFETCH__NONSURF_PIPE_CONFIG_MASK 0xf8000000L
+#define HDP_NONSURFACE_PREFETCH__NONSURF_PIPE_CONFIG__SHIFT 0x0000001b
+#define HDP_NONSURFACE_PREFETCH__NONSURF_PREFETCH_DIR_MASK 0x00000038L
+#define HDP_NONSURFACE_PREFETCH__NONSURF_PREFETCH_DIR__SHIFT 0x00000003
+#define HDP_NONSURFACE_PREFETCH__NONSURF_PREFETCH_MAX_Z_MASK 0x000ffe00L
+#define HDP_NONSURFACE_PREFETCH__NONSURF_PREFETCH_MAX_Z__SHIFT 0x00000009
+#define HDP_NONSURFACE_PREFETCH__NONSURF_PREFETCH_NUM_MASK 0x000001c0L
+#define HDP_NONSURFACE_PREFETCH__NONSURF_PREFETCH_NUM__SHIFT 0x00000006
+#define HDP_NONSURFACE_PREFETCH__NONSURF_PREFETCH_PRI_MASK 0x00000007L
+#define HDP_NONSURFACE_PREFETCH__NONSURF_PREFETCH_PRI__SHIFT 0x00000000
+#define HDP_NONSURFACE_SIZE__NONSURF_PITCH_TILE_MAX_MASK 0x000007ffL
+#define HDP_NONSURFACE_SIZE__NONSURF_PITCH_TILE_MAX__SHIFT 0x00000000
+#define HDP_NONSURFACE_SIZE__NONSURF_SLICE_TILE_MAX_MASK 0xfffff800L
+#define HDP_NONSURFACE_SIZE__NONSURF_SLICE_TILE_MAX__SHIFT 0x0000000b
+#define HDP_NONSURF_FLAGS_CLR__NONSURF_READ_FLAG_CLR_MASK 0x00000002L
+#define HDP_NONSURF_FLAGS_CLR__NONSURF_READ_FLAG_CLR__SHIFT 0x00000001
+#define HDP_NONSURF_FLAGS_CLR__NONSURF_WRITE_FLAG_CLR_MASK 0x00000001L
+#define HDP_NONSURF_FLAGS_CLR__NONSURF_WRITE_FLAG_CLR__SHIFT 0x00000000
+#define HDP_NONSURF_FLAGS__NONSURF_READ_FLAG_MASK 0x00000002L
+#define HDP_NONSURF_FLAGS__NONSURF_READ_FLAG__SHIFT 0x00000001
+#define HDP_NONSURF_FLAGS__NONSURF_WRITE_FLAG_MASK 0x00000001L
+#define HDP_NONSURF_FLAGS__NONSURF_WRITE_FLAG__SHIFT 0x00000000
+#define HDP_OUTSTANDING_REQ__READ_REQ_MASK 0x0000ff00L
+#define HDP_OUTSTANDING_REQ__READ_REQ__SHIFT 0x00000008
+#define HDP_OUTSTANDING_REQ__WRITE_REQ_MASK 0x000000ffL
+#define HDP_OUTSTANDING_REQ__WRITE_REQ__SHIFT 0x00000000
+#define HDP_SC_MULTI_CHIP_CNTL__LOG2_NUM_CHIPS_MASK 0x00000007L
+#define HDP_SC_MULTI_CHIP_CNTL__LOG2_NUM_CHIPS__SHIFT 0x00000000
+#define HDP_SC_MULTI_CHIP_CNTL__MULTI_CHIP_TILE_SIZE_MASK 0x00000018L
+#define HDP_SC_MULTI_CHIP_CNTL__MULTI_CHIP_TILE_SIZE__SHIFT 0x00000003
+#define HDP_SW_SEMAPHORE__SW_SEMAPHORE_MASK 0xffffffffL
+#define HDP_SW_SEMAPHORE__SW_SEMAPHORE__SHIFT 0x00000000
+#define HDP_TILING_CONFIG__BANK_SWAPS_MASK 0x00003800L
+#define HDP_TILING_CONFIG__BANK_SWAPS__SHIFT 0x0000000b
+#define HDP_TILING_CONFIG__BANK_TILING_MASK 0x00000030L
+#define HDP_TILING_CONFIG__BANK_TILING__SHIFT 0x00000004
+#define HDP_TILING_CONFIG__GROUP_SIZE_MASK 0x000000c0L
+#define HDP_TILING_CONFIG__GROUP_SIZE__SHIFT 0x00000006
+#define HDP_TILING_CONFIG__PIPE_TILING_MASK 0x0000000eL
+#define HDP_TILING_CONFIG__PIPE_TILING__SHIFT 0x00000001
+#define HDP_TILING_CONFIG__ROW_TILING_MASK 0x00000700L
+#define HDP_TILING_CONFIG__ROW_TILING__SHIFT 0x00000008
+#define HDP_TILING_CONFIG__SAMPLE_SPLIT_MASK 0x0000c000L
+#define HDP_TILING_CONFIG__SAMPLE_SPLIT__SHIFT 0x0000000e
+#define HDP_XDP_BARS_ADDR_39_36__BAR0_ADDR_39_36_MASK 0x0000000fL
+#define HDP_XDP_BARS_ADDR_39_36__BAR0_ADDR_39_36__SHIFT 0x00000000
+#define HDP_XDP_BARS_ADDR_39_36__BAR1_ADDR_39_36_MASK 0x000000f0L
+#define HDP_XDP_BARS_ADDR_39_36__BAR1_ADDR_39_36__SHIFT 0x00000004
+#define HDP_XDP_BARS_ADDR_39_36__BAR2_ADDR_39_36_MASK 0x00000f00L
+#define HDP_XDP_BARS_ADDR_39_36__BAR2_ADDR_39_36__SHIFT 0x00000008
+#define HDP_XDP_BARS_ADDR_39_36__BAR3_ADDR_39_36_MASK 0x0000f000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR3_ADDR_39_36__SHIFT 0x0000000c
+#define HDP_XDP_BARS_ADDR_39_36__BAR4_ADDR_39_36_MASK 0x000f0000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR4_ADDR_39_36__SHIFT 0x00000010
+#define HDP_XDP_BARS_ADDR_39_36__BAR5_ADDR_39_36_MASK 0x00f00000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR5_ADDR_39_36__SHIFT 0x00000014
+#define HDP_XDP_BARS_ADDR_39_36__BAR6_ADDR_39_36_MASK 0x0f000000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR6_ADDR_39_36__SHIFT 0x00000018
+#define HDP_XDP_BARS_ADDR_39_36__BAR7_ADDR_39_36_MASK 0xf0000000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR7_ADDR_39_36__SHIFT 0x0000001c
+#define HDP_XDP_BUSY_STS__BUSY_BITS_MASK 0x0003ffffL
+#define HDP_XDP_BUSY_STS__BUSY_BITS__SHIFT 0x00000000
+#define HDP_XDP_CGTT_BLK_CTRL__CGTT_BLK_CTRL_0_ON_DELAY_MASK 0x0000000fL
+#define HDP_XDP_CGTT_BLK_CTRL__CGTT_BLK_CTRL_0_ON_DELAY__SHIFT 0x00000000
+#define HDP_XDP_CGTT_BLK_CTRL__CGTT_BLK_CTRL_1_OFF_DELAY_MASK 0x00000ff0L
+#define HDP_XDP_CGTT_BLK_CTRL__CGTT_BLK_CTRL_1_OFF_DELAY__SHIFT 0x00000004
+#define HDP_XDP_CGTT_BLK_CTRL__CGTT_BLK_CTRL_2_RSVD_MASK 0x3ffff000L
+#define HDP_XDP_CGTT_BLK_CTRL__CGTT_BLK_CTRL_2_RSVD__SHIFT 0x0000000c
+#define HDP_XDP_CGTT_BLK_CTRL__CGTT_BLK_CTRL_3_SOFT_CORE_OVERRIDE_MASK 0x40000000L
+#define HDP_XDP_CGTT_BLK_CTRL__CGTT_BLK_CTRL_3_SOFT_CORE_OVERRIDE__SHIFT 0x0000001e
+#define HDP_XDP_CGTT_BLK_CTRL__CGTT_BLK_CTRL_4_SOFT_REG_OVERRIDE_MASK 0x80000000L
+#define HDP_XDP_CGTT_BLK_CTRL__CGTT_BLK_CTRL_4_SOFT_REG_OVERRIDE__SHIFT 0x0000001f
+#define HDP_XDP_CHKN__CHKN_0_RSVD_MASK 0x000000ffL
+#define HDP_XDP_CHKN__CHKN_0_RSVD__SHIFT 0x00000000
+#define HDP_XDP_CHKN__CHKN_1_RSVD_MASK 0x0000ff00L
+#define HDP_XDP_CHKN__CHKN_1_RSVD__SHIFT 0x00000008
+#define HDP_XDP_CHKN__CHKN_2_RSVD_MASK 0x00ff0000L
+#define HDP_XDP_CHKN__CHKN_2_RSVD__SHIFT 0x00000010
+#define HDP_XDP_CHKN__CHKN_3_RSVD_MASK 0xff000000L
+#define HDP_XDP_CHKN__CHKN_3_RSVD__SHIFT 0x00000018
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_ADDR_MASK 0x0000ffffL
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_ADDR__SHIFT 0x00000000
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_BAR_NUM_MASK 0x00700000L
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_BAR_NUM__SHIFT 0x00000014
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_FLUSH_NUM_MASK 0x000f0000L
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_FLUSH_NUM__SHIFT 0x00000010
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_ALTER_FLUSH_NUM_MASK 0x00040000L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_ALTER_FLUSH_NUM__SHIFT 0x00000012
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_FLUSH_NUM_MASK 0x0000000fL
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_FLUSH_NUM__SHIFT 0x00000000
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ADDR_SEL_MASK 0x00000700L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ADDR_SEL__SHIFT 0x00000008
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ENC_DATA_MASK 0x000000f0L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ENC_DATA__SHIFT 0x00000004
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_0_MASK 0x00080000L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_0__SHIFT 0x00000013
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_1_MASK 0x00100000L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_1__SHIFT 0x00000014
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_SEND_HOST_MASK 0x00010000L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_SEND_HOST__SHIFT 0x00000010
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_SEND_SIDE_MASK 0x00020000L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_SEND_SIDE__SHIFT 0x00000011
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_XPB_CLG_MASK 0x0000f800L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_XPB_CLG__SHIFT 0x0000000b
+#define HDP_XDP_D2H_RSVD_10__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_10__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_11__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_11__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_12__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_12__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_13__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_13__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_14__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_14__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_15__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_15__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_16__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_16__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_17__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_17__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_18__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_18__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_19__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_19__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_20__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_20__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_21__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_21__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_22__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_22__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_23__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_23__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_24__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_24__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_25__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_25__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_26__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_26__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_27__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_27__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_28__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_28__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_29__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_29__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_30__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_30__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_31__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_31__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_32__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_32__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_33__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_33__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_34__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_34__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_3__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_3__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_4__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_4__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_5__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_5__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_6__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_6__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_7__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_7__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_8__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_8__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_D2H_RSVD_9__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_D2H_RSVD_9__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_DBG_ADDR__CTRL_MASK 0xffff0000L
+#define HDP_XDP_DBG_ADDR__CTRL__SHIFT 0x00000010
+#define HDP_XDP_DBG_ADDR__STS_MASK 0x0000ffffL
+#define HDP_XDP_DBG_ADDR__STS__SHIFT 0x00000000
+#define HDP_XDP_DBG_DATA__CTRL_MASK 0xffff0000L
+#define HDP_XDP_DBG_DATA__CTRL__SHIFT 0x00000010
+#define HDP_XDP_DBG_DATA__STS_MASK 0x0000ffffL
+#define HDP_XDP_DBG_DATA__STS__SHIFT 0x00000000
+#define HDP_XDP_DBG_MASK__CTRL_MASK 0xffff0000L
+#define HDP_XDP_DBG_MASK__CTRL__SHIFT 0x00000010
+#define HDP_XDP_DBG_MASK__STS_MASK 0x0000ffffL
+#define HDP_XDP_DBG_MASK__STS__SHIFT 0x00000000
+#define HDP_XDP_DIRECT2HDP_FIRST__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_DIRECT2HDP_FIRST__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_DIRECT2HDP_LAST__RESERVED_MASK 0xffffffffL
+#define HDP_XDP_DIRECT2HDP_LAST__RESERVED__SHIFT 0x00000000
+#define HDP_XDP_FLUSH_ARMED_STS__FLUSH_ARMED_STS_MASK 0xffffffffL
+#define HDP_XDP_FLUSH_ARMED_STS__FLUSH_ARMED_STS__SHIFT 0x00000000
+#define HDP_XDP_FLUSH_CNTR0_STS__FLUSH_CNTR0_STS_MASK 0x03ffffffL
+#define HDP_XDP_FLUSH_CNTR0_STS__FLUSH_CNTR0_STS__SHIFT 0x00000000
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_INVERSE_PEER_TAG_MATCHING_MASK 0x00001000L
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_INVERSE_PEER_TAG_MATCHING__SHIFT 0x0000000c
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_P2P_RD_EN_MASK 0x00002000L
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_P2P_RD_EN__SHIFT 0x0000000d
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_SYS_FIFO_DEPTH_OVERRIDE_MASK 0x0000003fL
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_SYS_FIFO_DEPTH_OVERRIDE__SHIFT 0x00000000
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_XDP_FIFO_DEPTH_OVERRIDE_MASK 0x00000fc0L
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_XDP_FIFO_DEPTH_OVERRIDE__SHIFT 0x00000006
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_PRIV_MASK 0x00000001L
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_PRIV__SHIFT 0x00000000
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SWAP_MASK 0x00000006L
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SWAP__SHIFT 0x00000001
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_TRAN_MASK 0x00000008L
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_TRAN__SHIFT 0x00000003
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_VMID_MASK 0x000000f0L
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_VMID__SHIFT 0x00000004
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_WRREQ_PRIV_MASK 0x00000001L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_WRREQ_PRIV__SHIFT 0x00000000
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_WRREQ_SWAP_MASK 0x00000006L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_WRREQ_SWAP__SHIFT 0x00000001
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_WRREQ_TRAN_MASK 0x00000008L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_WRREQ_TRAN__SHIFT 0x00000003
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_WRREQ_VMID_MASK 0x07800000L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_WRREQ_VMID__SHIFT 0x00000017
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_MC_STALL_ON_BUF_FULL_MASK_MASK 0x00700000L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_MC_STALL_ON_BUF_FULL_MASK__SHIFT 0x00000014
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_SID_TAP_WRREQ_PRIV_MASK 0x00000010L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_SID_TAP_WRREQ_PRIV__SHIFT 0x00000004
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_SID_TAP_WRREQ_SWAP_MASK 0x00000060L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_SID_TAP_WRREQ_SWAP__SHIFT 0x00000005
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_SID_TAP_WRREQ_TRAN_MASK 0x00000080L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_SID_TAP_WRREQ_TRAN__SHIFT 0x00000007
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_SID_TAP_WRREQ_VMID_MASK 0x78000000L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_SID_TAP_WRREQ_VMID__SHIFT 0x0000001b
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_XDP_HIGHER_PRI_THRESH_MASK 0x000fc000L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_XDP_HIGHER_PRI_THRESH__SHIFT 0x0000000e
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_XL8R_WRREQ_CRD_OVERRIDE_MASK 0x00003f00L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_XL8R_WRREQ_CRD_OVERRIDE__SHIFT 0x00000008
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_EN_MASK 0x00000001L
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_EN__SHIFT 0x00000000
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER_MASK 0x00000006L
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER__SHIFT 0x00000001
+#define HDP_XDP_P2P_BAR0__ADDR_MASK 0x0000ffffL
+#define HDP_XDP_P2P_BAR0__ADDR__SHIFT 0x00000000
+#define HDP_XDP_P2P_BAR0__FLUSH_MASK 0x000f0000L
+#define HDP_XDP_P2P_BAR0__FLUSH__SHIFT 0x00000010
+#define HDP_XDP_P2P_BAR0__VALID_MASK 0x00100000L
+#define HDP_XDP_P2P_BAR0__VALID__SHIFT 0x00000014
+#define HDP_XDP_P2P_BAR1__ADDR_MASK 0x0000ffffL
+#define HDP_XDP_P2P_BAR1__ADDR__SHIFT 0x00000000
+#define HDP_XDP_P2P_BAR1__FLUSH_MASK 0x000f0000L
+#define HDP_XDP_P2P_BAR1__FLUSH__SHIFT 0x00000010
+#define HDP_XDP_P2P_BAR1__VALID_MASK 0x00100000L
+#define HDP_XDP_P2P_BAR1__VALID__SHIFT 0x00000014
+#define HDP_XDP_P2P_BAR2__ADDR_MASK 0x0000ffffL
+#define HDP_XDP_P2P_BAR2__ADDR__SHIFT 0x00000000
+#define HDP_XDP_P2P_BAR2__FLUSH_MASK 0x000f0000L
+#define HDP_XDP_P2P_BAR2__FLUSH__SHIFT 0x00000010
+#define HDP_XDP_P2P_BAR2__VALID_MASK 0x00100000L
+#define HDP_XDP_P2P_BAR2__VALID__SHIFT 0x00000014
+#define HDP_XDP_P2P_BAR3__ADDR_MASK 0x0000ffffL
+#define HDP_XDP_P2P_BAR3__ADDR__SHIFT 0x00000000
+#define HDP_XDP_P2P_BAR3__FLUSH_MASK 0x000f0000L
+#define HDP_XDP_P2P_BAR3__FLUSH__SHIFT 0x00000010
+#define HDP_XDP_P2P_BAR3__VALID_MASK 0x00100000L
+#define HDP_XDP_P2P_BAR3__VALID__SHIFT 0x00000014
+#define HDP_XDP_P2P_BAR4__ADDR_MASK 0x0000ffffL
+#define HDP_XDP_P2P_BAR4__ADDR__SHIFT 0x00000000
+#define HDP_XDP_P2P_BAR4__FLUSH_MASK 0x000f0000L
+#define HDP_XDP_P2P_BAR4__FLUSH__SHIFT 0x00000010
+#define HDP_XDP_P2P_BAR4__VALID_MASK 0x00100000L
+#define HDP_XDP_P2P_BAR4__VALID__SHIFT 0x00000014
+#define HDP_XDP_P2P_BAR5__ADDR_MASK 0x0000ffffL
+#define HDP_XDP_P2P_BAR5__ADDR__SHIFT 0x00000000
+#define HDP_XDP_P2P_BAR5__FLUSH_MASK 0x000f0000L
+#define HDP_XDP_P2P_BAR5__FLUSH__SHIFT 0x00000010
+#define HDP_XDP_P2P_BAR5__VALID_MASK 0x00100000L
+#define HDP_XDP_P2P_BAR5__VALID__SHIFT 0x00000014
+#define HDP_XDP_P2P_BAR6__ADDR_MASK 0x0000ffffL
+#define HDP_XDP_P2P_BAR6__ADDR__SHIFT 0x00000000
+#define HDP_XDP_P2P_BAR6__FLUSH_MASK 0x000f0000L
+#define HDP_XDP_P2P_BAR6__FLUSH__SHIFT 0x00000010
+#define HDP_XDP_P2P_BAR6__VALID_MASK 0x00100000L
+#define HDP_XDP_P2P_BAR6__VALID__SHIFT 0x00000014
+#define HDP_XDP_P2P_BAR7__ADDR_MASK 0x0000ffffL
+#define HDP_XDP_P2P_BAR7__ADDR__SHIFT 0x00000000
+#define HDP_XDP_P2P_BAR7__FLUSH_MASK 0x000f0000L
+#define HDP_XDP_P2P_BAR7__FLUSH__SHIFT 0x00000010
+#define HDP_XDP_P2P_BAR7__VALID_MASK 0x00100000L
+#define HDP_XDP_P2P_BAR7__VALID__SHIFT 0x00000014
+#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_ADDR_SIZE_MASK 0x0000000fL
+#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_ADDR_SIZE__SHIFT 0x00000000
+#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_BAR_FROM_MASK 0x00000030L
+#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_BAR_FROM__SHIFT 0x00000004
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_39_36_MASK 0x01e00000L
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_39_36__SHIFT 0x00000015
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_MASK 0x001ffffeL
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR__SHIFT 0x00000001
+#define HDP_XDP_P2P_MBX_ADDR0__VALID_MASK 0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR0__VALID__SHIFT 0x00000000
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_39_36_MASK 0x01e00000L
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_39_36__SHIFT 0x00000015
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_MASK 0x001ffffeL
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR__SHIFT 0x00000001
+#define HDP_XDP_P2P_MBX_ADDR1__VALID_MASK 0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR1__VALID__SHIFT 0x00000000
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_39_36_MASK 0x01e00000L
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_39_36__SHIFT 0x00000015
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_MASK 0x001ffffeL
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR__SHIFT 0x00000001
+#define HDP_XDP_P2P_MBX_ADDR2__VALID_MASK 0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR2__VALID__SHIFT 0x00000000
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_39_36_MASK 0x01e00000L
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_39_36__SHIFT 0x00000015
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_MASK 0x001ffffeL
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR__SHIFT 0x00000001
+#define HDP_XDP_P2P_MBX_ADDR3__VALID_MASK 0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR3__VALID__SHIFT 0x00000000
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_39_36_MASK 0x01e00000L
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_39_36__SHIFT 0x00000015
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_MASK 0x001ffffeL
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR__SHIFT 0x00000001
+#define HDP_XDP_P2P_MBX_ADDR4__VALID_MASK 0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR4__VALID__SHIFT 0x00000000
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_39_36_MASK 0x01e00000L
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_39_36__SHIFT 0x00000015
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_MASK 0x001ffffeL
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR__SHIFT 0x00000001
+#define HDP_XDP_P2P_MBX_ADDR5__VALID_MASK 0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR5__VALID__SHIFT 0x00000000
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_39_36_MASK 0x01e00000L
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_39_36__SHIFT 0x00000015
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_MASK 0x001ffffeL
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR__SHIFT 0x00000001
+#define HDP_XDP_P2P_MBX_ADDR6__VALID_MASK 0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR6__VALID__SHIFT 0x00000000
+#define HDP_XDP_P2P_MBX_OFFSET__P2P_MBX_OFFSET_MASK 0x00003fffL
+#define HDP_XDP_P2P_MBX_OFFSET__P2P_MBX_OFFSET__SHIFT 0x00000000
+#define HDP_XDP_SID_CFG__SID_CFG_FLNUM_MSB_SEL_MASK 0x00000018L
+#define HDP_XDP_SID_CFG__SID_CFG_FLNUM_MSB_SEL__SHIFT 0x00000003
+#define HDP_XDP_SID_CFG__SID_CFG_WR_COMBINE_EN_MASK 0x00000001L
+#define HDP_XDP_SID_CFG__SID_CFG_WR_COMBINE_EN__SHIFT 0x00000000
+#define HDP_XDP_SID_CFG__SID_CFG_WR_COMBINE_TIMER_MASK 0x00000006L
+#define HDP_XDP_SID_CFG__SID_CFG_WR_COMBINE_TIMER__SHIFT 0x00000001
+#define HDP_XDP_SRBM_CFG__SRBM_CFG_REG_CLK_ENABLE_COUNT_MASK 0x0000003fL
+#define HDP_XDP_SRBM_CFG__SRBM_CFG_REG_CLK_ENABLE_COUNT__SHIFT 0x00000000
+#define HDP_XDP_SRBM_CFG__SRBM_CFG_REG_CLK_GATING_DIS_MASK 0x00000040L
+#define HDP_XDP_SRBM_CFG__SRBM_CFG_REG_CLK_GATING_DIS__SHIFT 0x00000006
+#define HDP_XDP_SRBM_CFG__SRBM_CFG_WAKE_DYN_CLK_MASK 0x00000080L
+#define HDP_XDP_SRBM_CFG__SRBM_CFG_WAKE_DYN_CLK__SHIFT 0x00000007
+#define HDP_XDP_STICKY__STICKY_STS_MASK 0x0000ffffL
+#define HDP_XDP_STICKY__STICKY_STS__SHIFT 0x00000000
+#define HDP_XDP_STICKY__STICKY_W1C_MASK 0xffff0000L
+#define HDP_XDP_STICKY__STICKY_W1C__SHIFT 0x00000010
+#define HFS_SEED0__RESERVED_MASK 0xffffffffL
+#define HFS_SEED0__RESERVED__SHIFT 0x00000000
+#define HFS_SEED1__RESERVED_MASK 0xffffffffL
+#define HFS_SEED1__RESERVED__SHIFT 0x00000000
+#define HFS_SEED2__RESERVED_MASK 0xffffffffL
+#define HFS_SEED2__RESERVED__SHIFT 0x00000000
+#define HFS_SEED3__RESERVED_MASK 0xffffffffL
+#define HFS_SEED3__RESERVED__SHIFT 0x00000000
+#define IH_ADVFAULT_CNTL__NUM_FAULTS_DROPPED_MASK 0x0000ff00L
+#define IH_ADVFAULT_CNTL__NUM_FAULTS_DROPPED__SHIFT 0x00000008
+#define IH_ADVFAULT_CNTL__WAIT_TIMER_MASK 0x3fff0000L
+#define IH_ADVFAULT_CNTL__WAIT_TIMER__SHIFT 0x00000010
+#define IH_ADVFAULT_CNTL__WATERMARK_ENABLE_MASK 0x00000008L
+#define IH_ADVFAULT_CNTL__WATERMARK_ENABLE__SHIFT 0x00000003
+#define IH_ADVFAULT_CNTL__WATERMARK_MASK 0x00000007L
+#define IH_ADVFAULT_CNTL__WATERMARK_REACHED_MASK 0x00000010L
+#define IH_ADVFAULT_CNTL__WATERMARK_REACHED__SHIFT 0x00000004
+#define IH_ADVFAULT_CNTL__WATERMARK__SHIFT 0x00000000
+#define IH_CNTL__CLIENT_FIFO_HIGHWATER_MASK 0x00000300L
+#define IH_CNTL__CLIENT_FIFO_HIGHWATER__SHIFT 0x00000008
+#define IH_CNTL__ENABLE_INTR_MASK 0x00000001L
+#define IH_CNTL__ENABLE_INTR__SHIFT 0x00000000
+#define IH_CNTL__MC_FIFO_HIGHWATER_MASK 0x00007c00L
+#define IH_CNTL__MC_FIFO_HIGHWATER__SHIFT 0x0000000a
+#define IH_CNTL__MC_SWAP_MASK 0x00000006L
+#define IH_CNTL__MC_SWAP__SHIFT 0x00000001
+#define IH_CNTL__MC_TRAN_MASK 0x00000008L
+#define IH_CNTL__MC_TRAN__SHIFT 0x00000003
+#define IH_CNTL__MC_VMID_MASK 0x1e000000L
+#define IH_CNTL__MC_VMID__SHIFT 0x00000019
+#define IH_CNTL__MC_WR_CLEAN_CNT_MASK 0x01f00000L
+#define IH_CNTL__MC_WR_CLEAN_CNT__SHIFT 0x00000014
+#define IH_CNTL__MC_WRREQ_CREDIT_MASK 0x000f8000L
+#define IH_CNTL__MC_WRREQ_CREDIT__SHIFT 0x0000000f
+#define IH_CNTL__RPTR_REARM_MASK 0x00000010L
+#define IH_CNTL__RPTR_REARM__SHIFT 0x00000004
+#define IH_LEVEL_STATUS__BIF_STATUS_MASK 0x00000010L
+#define IH_LEVEL_STATUS__BIF_STATUS__SHIFT 0x00000004
+#define IH_LEVEL_STATUS__DC_STATUS_MASK 0x00000001L
+#define IH_LEVEL_STATUS__DC_STATUS__SHIFT 0x00000000
+#define IH_LEVEL_STATUS__ROM_STATUS_MASK 0x00000004L
+#define IH_LEVEL_STATUS__ROM_STATUS__SHIFT 0x00000002
+#define IH_LEVEL_STATUS__SRBM_STATUS_MASK 0x00000008L
+#define IH_LEVEL_STATUS__SRBM_STATUS__SHIFT 0x00000003
+#define IH_LEVEL_STATUS__XDMA_STATUS_MASK 0x00000020L
+#define IH_LEVEL_STATUS__XDMA_STATUS__SHIFT 0x00000005
+#define IH_PERFCOUNTER0_RESULT__PERF_COUNT_MASK 0xffffffffL
+#define IH_PERFCOUNTER0_RESULT__PERF_COUNT__SHIFT 0x00000000
+#define IH_PERFCOUNTER1_RESULT__PERF_COUNT_MASK 0xffffffffL
+#define IH_PERFCOUNTER1_RESULT__PERF_COUNT__SHIFT 0x00000000
+#define IH_PERFMON_CNTL__CLEAR0_MASK 0x00000002L
+#define IH_PERFMON_CNTL__CLEAR0__SHIFT 0x00000001
+#define IH_PERFMON_CNTL__CLEAR1_MASK 0x00000200L
+#define IH_PERFMON_CNTL__CLEAR1__SHIFT 0x00000009
+#define IH_PERFMON_CNTL__ENABLE0_MASK 0x00000001L
+#define IH_PERFMON_CNTL__ENABLE0__SHIFT 0x00000000
+#define IH_PERFMON_CNTL__ENABLE1_MASK 0x00000100L
+#define IH_PERFMON_CNTL__ENABLE1__SHIFT 0x00000008
+#define IH_PERFMON_CNTL__PERF_SEL0_MASK 0x000000fcL
+#define IH_PERFMON_CNTL__PERF_SEL0__SHIFT 0x00000002
+#define IH_PERFMON_CNTL__PERF_SEL1_MASK 0x0000fc00L
+#define IH_PERFMON_CNTL__PERF_SEL1__SHIFT 0x0000000a
+#define IH_RB_BASE__ADDR_MASK 0xffffffffL
+#define IH_RB_BASE__ADDR__SHIFT 0x00000000
+#define IH_RB_CNTL__RB_ENABLE_MASK 0x00000001L
+#define IH_RB_CNTL__RB_ENABLE__SHIFT 0x00000000
+#define IH_RB_CNTL__RB_FULL_DRAIN_ENABLE_MASK 0x00000040L
+#define IH_RB_CNTL__RB_FULL_DRAIN_ENABLE__SHIFT 0x00000006
+#define IH_RB_CNTL__RB_GPU_TS_ENABLE_MASK 0x00000080L
+#define IH_RB_CNTL__RB_GPU_TS_ENABLE__SHIFT 0x00000007
+#define IH_RB_CNTL__RB_SIZE_MASK 0x0000003eL
+#define IH_RB_CNTL__RB_SIZE__SHIFT 0x00000001
+#define IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK 0x80000000L
+#define IH_RB_CNTL__WPTR_OVERFLOW_CLEAR__SHIFT 0x0000001f
+#define IH_RB_CNTL__WPTR_OVERFLOW_ENABLE_MASK 0x00010000L
+#define IH_RB_CNTL__WPTR_OVERFLOW_ENABLE__SHIFT 0x00000010
+#define IH_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK 0x00000100L
+#define IH_RB_CNTL__WPTR_WRITEBACK_ENABLE__SHIFT 0x00000008
+#define IH_RB_CNTL__WPTR_WRITEBACK_TIMER_MASK 0x00003e00L
+#define IH_RB_CNTL__WPTR_WRITEBACK_TIMER__SHIFT 0x00000009
+#define IH_RB_RPTR__OFFSET_MASK 0x0003fffcL
+#define IH_RB_RPTR__OFFSET__SHIFT 0x00000002
+#define IH_RB_WPTR_ADDR_HI__ADDR_MASK 0x000000ffL
+#define IH_RB_WPTR_ADDR_HI__ADDR__SHIFT 0x00000000
+#define IH_RB_WPTR_ADDR_LO__ADDR_MASK 0xfffffffcL
+#define IH_RB_WPTR_ADDR_LO__ADDR__SHIFT 0x00000002
+#define IH_RB_WPTR__OFFSET_MASK 0x0003fffcL
+#define IH_RB_WPTR__OFFSET__SHIFT 0x00000002
+#define IH_RB_WPTR__RB_OVERFLOW_MASK 0x00000001L
+#define IH_RB_WPTR__RB_OVERFLOW__SHIFT 0x00000000
+#define IH_STATUS__BIF_INTERRUPT_LINE_MASK 0x00000400L
+#define IH_STATUS__BIF_INTERRUPT_LINE__SHIFT 0x0000000a
+#define IH_STATUS__IDLE_MASK 0x00000001L
+#define IH_STATUS__IDLE__SHIFT 0x00000000
+#define IH_STATUS__INPUT_IDLE_MASK 0x00000002L
+#define IH_STATUS__INPUT_IDLE__SHIFT 0x00000001
+#define IH_STATUS__MC_WR_CLEAN_PENDING_MASK 0x00000100L
+#define IH_STATUS__MC_WR_CLEAN_PENDING__SHIFT 0x00000008
+#define IH_STATUS__MC_WR_CLEAN_STALL_MASK 0x00000200L
+#define IH_STATUS__MC_WR_CLEAN_STALL__SHIFT 0x00000009
+#define IH_STATUS__MC_WR_IDLE_MASK 0x00000040L
+#define IH_STATUS__MC_WR_IDLE__SHIFT 0x00000006
+#define IH_STATUS__MC_WR_STALL_MASK 0x00000080L
+#define IH_STATUS__MC_WR_STALL__SHIFT 0x00000007
+#define IH_STATUS__RB_FULL_DRAIN_MASK 0x00000010L
+#define IH_STATUS__RB_FULL_DRAIN__SHIFT 0x00000004
+#define IH_STATUS__RB_FULL_MASK 0x00000008L
+#define IH_STATUS__RB_FULL__SHIFT 0x00000003
+#define IH_STATUS__RB_IDLE_MASK 0x00000004L
+#define IH_STATUS__RB_IDLE__SHIFT 0x00000002
+#define IH_STATUS__RB_OVERFLOW_MASK 0x00000020L
+#define IH_STATUS__RB_OVERFLOW__SHIFT 0x00000005
+#define KEFUSE0__RESERVED_MASK 0xffffffffL
+#define KEFUSE0__RESERVED__SHIFT 0x00000000
+#define KEFUSE1__RESERVED_MASK 0xffffffffL
+#define KEFUSE1__RESERVED__SHIFT 0x00000000
+#define KEFUSE2__RESERVED_MASK 0xffffffffL
+#define KEFUSE2__RESERVED__SHIFT 0x00000000
+#define KEFUSE3__RESERVED_MASK 0xffffffffL
+#define KEFUSE3__RESERVED__SHIFT 0x00000000
+#define KHFS0__RESERVED_MASK 0xffffffffL
+#define KHFS0__RESERVED__SHIFT 0x00000000
+#define KHFS1__RESERVED_MASK 0xffffffffL
+#define KHFS1__RESERVED__SHIFT 0x00000000
+#define KHFS2__RESERVED_MASK 0xffffffffL
+#define KHFS2__RESERVED__SHIFT 0x00000000
+#define KHFS3__RESERVED_MASK 0xffffffffL
+#define KHFS3__RESERVED__SHIFT 0x00000000
+#define KSESSION0__RESERVED_MASK 0xffffffffL
+#define KSESSION0__RESERVED__SHIFT 0x00000000
+#define KSESSION1__RESERVED_MASK 0xffffffffL
+#define KSESSION1__RESERVED__SHIFT 0x00000000
+#define KSESSION2__RESERVED_MASK 0xffffffffL
+#define KSESSION2__RESERVED__SHIFT 0x00000000
+#define KSESSION3__RESERVED_MASK 0xffffffffL
+#define KSESSION3__RESERVED__SHIFT 0x00000000
+#define KSIG0__RESERVED_MASK 0xffffffffL
+#define KSIG0__RESERVED__SHIFT 0x00000000
+#define KSIG1__RESERVED_MASK 0xffffffffL
+#define KSIG1__RESERVED__SHIFT 0x00000000
+#define KSIG2__RESERVED_MASK 0xffffffffL
+#define KSIG2__RESERVED__SHIFT 0x00000000
+#define KSIG3__RESERVED_MASK 0xffffffffL
+#define KSIG3__RESERVED__SHIFT 0x00000000
+#define LX0__RESERVED_MASK 0xffffffffL
+#define LX0__RESERVED__SHIFT 0x00000000
+#define LX1__RESERVED_MASK 0xffffffffL
+#define LX1__RESERVED__SHIFT 0x00000000
+#define LX2__RESERVED_MASK 0xffffffffL
+#define LX2__RESERVED__SHIFT 0x00000000
+#define LX3__RESERVED_MASK 0xffffffffL
+#define LX3__RESERVED__SHIFT 0x00000000
+#define RINGOSC_MASK__MASK_MASK 0x0000ffffL
+#define RINGOSC_MASK__MASK__SHIFT 0x00000000
+#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT0_MASK 0x00000007L
+#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT0__SHIFT 0x00000000
+#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT1_MASK 0x00000038L
+#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT1__SHIFT 0x00000003
+#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT2_MASK 0x000001c0L
+#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT2__SHIFT 0x00000006
+#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT3_MASK 0x00000e00L
+#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT3__SHIFT 0x00000009
+#define SEM_MAILBOX_CLIENTCONFIG__UVD_CLIENT0_MASK 0x00038000L
+#define SEM_MAILBOX_CLIENTCONFIG__UVD_CLIENT0__SHIFT 0x0000000f
+#define SEM_MAILBOX_CLIENTCONFIG__VCE_CLIENT0_MASK 0x00e00000L
+#define SEM_MAILBOX_CLIENTCONFIG__VCE_CLIENT0__SHIFT 0x00000015
+#define SEM_MAILBOX_CONTROL__HOSTPORT_ENABLE_MASK 0x0000ff00L
+#define SEM_MAILBOX_CONTROL__HOSTPORT_ENABLE__SHIFT 0x00000008
+#define SEM_MAILBOX_CONTROL__SIDEPORT_ENABLE_MASK 0x000000ffL
+#define SEM_MAILBOX_CONTROL__SIDEPORT_ENABLE__SHIFT 0x00000000
+#define SEM_MAILBOX__HOSTPORT_MASK 0x0000ff00L
+#define SEM_MAILBOX__HOSTPORT__SHIFT 0x00000008
+#define SEM_MAILBOX__SIDEPORT_MASK 0x000000ffL
+#define SEM_MAILBOX__SIDEPORT__SHIFT 0x00000000
+#define SEM_MCIF_CONFIG__MC_REQ_SWAP_MASK 0x00000003L
+#define SEM_MCIF_CONFIG__MC_REQ_SWAP__SHIFT 0x00000000
+#define SPU_PORT_STATUS__RESERVED_MASK 0xffffffffL
+#define SPU_PORT_STATUS__RESERVED__SHIFT 0x00000000
+#define SRBM_CAM_DATA__CAM_ADDR_MASK 0x0000ffffL
+#define SRBM_CAM_DATA__CAM_ADDR__SHIFT 0x00000000
+#define SRBM_CAM_DATA__CAM_REMAPADDR_MASK 0xffff0000L
+#define SRBM_CAM_DATA__CAM_REMAPADDR__SHIFT 0x00000010
+#define SRBM_CAM_INDEX__CAM_INDEX_MASK 0x00000007L
+#define SRBM_CAM_INDEX__CAM_INDEX__SHIFT 0x00000000
+#define SRBM_CHIP_REVISION__CHIP_REVISION_MASK 0x000000ffL
+#define SRBM_CHIP_REVISION__CHIP_REVISION__SHIFT 0x00000000
+#define SRBM_CNTL__COMBINE_SYSTEM_MC_MASK 0x00020000L
+#define SRBM_CNTL__COMBINE_SYSTEM_MC__SHIFT 0x00000011
+#define SRBM_CNTL__PWR_REQUEST_HALT_MASK 0x00010000L
+#define SRBM_CNTL__PWR_REQUEST_HALT__SHIFT 0x00000010
+#define SRBM_CNTL__READ_TIMEOUT_MASK 0x000003ffL
+#define SRBM_CNTL__READ_TIMEOUT__SHIFT 0x00000000
+#define SRBM_DEBUG_CNTL__SRBM_DEBUG_INDEX_MASK 0x0000003fL
+#define SRBM_DEBUG_CNTL__SRBM_DEBUG_INDEX__SHIFT 0x00000000
+#define SRBM_DEBUG_DATA__DATA_MASK 0xffffffffL
+#define SRBM_DEBUG_DATA__DATA__SHIFT 0x00000000
+#define SRBM_DEBUG__DISABLE_READ_TIMEOUT_MASK 0x00000002L
+#define SRBM_DEBUG__DISABLE_READ_TIMEOUT__SHIFT 0x00000001
+#define SRBM_DEBUG__IGNORE_RDY_MASK 0x00000001L
+#define SRBM_DEBUG__IGNORE_RDY__SHIFT 0x00000000
+#define SRBM_DEBUG__MC_CLOCK_DOMAIN_OVERRIDE_MASK 0x00000100L
+#define SRBM_DEBUG__MC_CLOCK_DOMAIN_OVERRIDE__SHIFT 0x00000008
+#define SRBM_DEBUG_SNAPSHOT__BIF_RDY_MASK 0x00000080L
+#define SRBM_DEBUG_SNAPSHOT__BIF_RDY__SHIFT 0x00000007
+#define SRBM_DEBUG_SNAPSHOT__DC_RDY_MASK 0x00000040L
+#define SRBM_DEBUG_SNAPSHOT__DC_RDY__SHIFT 0x00000006
+#define SRBM_DEBUG__SNAPSHOT_FREE_CNTRS_MASK 0x00000004L
+#define SRBM_DEBUG__SNAPSHOT_FREE_CNTRS__SHIFT 0x00000002
+#define SRBM_DEBUG_SNAPSHOT__GRBM_RDY_MASK 0x00000020L
+#define SRBM_DEBUG_SNAPSHOT__GRBM_RDY__SHIFT 0x00000005
+#define SRBM_DEBUG_SNAPSHOT__MCB_RDY_MASK 0x00000001L
+#define SRBM_DEBUG_SNAPSHOT__MCB_RDY__SHIFT 0x00000000
+#define SRBM_DEBUG_SNAPSHOT__MCC0_RDY_MASK 0x10000000L
+#define SRBM_DEBUG_SNAPSHOT__MCC0_RDY__SHIFT 0x0000001c
+#define SRBM_DEBUG_SNAPSHOT__MCC1_RDY_MASK 0x08000000L
+#define SRBM_DEBUG_SNAPSHOT__MCC1_RDY__SHIFT 0x0000001b
+#define SRBM_DEBUG_SNAPSHOT__MCC2_RDY_MASK 0x04000000L
+#define SRBM_DEBUG_SNAPSHOT__MCC2_RDY__SHIFT 0x0000001a
+#define SRBM_DEBUG_SNAPSHOT__MCC3_RDY_MASK 0x02000000L
+#define SRBM_DEBUG_SNAPSHOT__MCC3_RDY__SHIFT 0x00000019
+#define SRBM_DEBUG_SNAPSHOT__MCC4_RDY_MASK 0x01000000L
+#define SRBM_DEBUG_SNAPSHOT__MCC4_RDY__SHIFT 0x00000018
+#define SRBM_DEBUG_SNAPSHOT__MCC5_RDY_MASK 0x00800000L
+#define SRBM_DEBUG_SNAPSHOT__MCC5_RDY__SHIFT 0x00000017
+#define SRBM_DEBUG_SNAPSHOT__MCC6_RDY_MASK 0x00400000L
+#define SRBM_DEBUG_SNAPSHOT__MCC6_RDY__SHIFT 0x00000016
+#define SRBM_DEBUG_SNAPSHOT__MCC7_RDY_MASK 0x00200000L
+#define SRBM_DEBUG_SNAPSHOT__MCC7_RDY__SHIFT 0x00000015
+#define SRBM_DEBUG_SNAPSHOT__MCD0_RDY_MASK 0x00100000L
+#define SRBM_DEBUG_SNAPSHOT__MCD0_RDY__SHIFT 0x00000014
+#define SRBM_DEBUG_SNAPSHOT__MCD1_RDY_MASK 0x00080000L
+#define SRBM_DEBUG_SNAPSHOT__MCD1_RDY__SHIFT 0x00000013
+#define SRBM_DEBUG_SNAPSHOT__MCD2_RDY_MASK 0x00040000L
+#define SRBM_DEBUG_SNAPSHOT__MCD2_RDY__SHIFT 0x00000012
+#define SRBM_DEBUG_SNAPSHOT__MCD3_RDY_MASK 0x00020000L
+#define SRBM_DEBUG_SNAPSHOT__MCD3_RDY__SHIFT 0x00000011
+#define SRBM_DEBUG_SNAPSHOT__MCD4_RDY_MASK 0x00010000L
+#define SRBM_DEBUG_SNAPSHOT__MCD4_RDY__SHIFT 0x00000010
+#define SRBM_DEBUG_SNAPSHOT__MCD5_RDY_MASK 0x00008000L
+#define SRBM_DEBUG_SNAPSHOT__MCD5_RDY__SHIFT 0x0000000f
+#define SRBM_DEBUG_SNAPSHOT__MCD6_RDY_MASK 0x00004000L
+#define SRBM_DEBUG_SNAPSHOT__MCD6_RDY__SHIFT 0x0000000e
+#define SRBM_DEBUG_SNAPSHOT__MCD7_RDY_MASK 0x00002000L
+#define SRBM_DEBUG_SNAPSHOT__MCD7_RDY__SHIFT 0x0000000d
+#define SRBM_DEBUG_SNAPSHOT__ORB_RDY_MASK 0x00001000L
+#define SRBM_DEBUG_SNAPSHOT__ORB_RDY__SHIFT 0x0000000c
+#define SRBM_DEBUG_SNAPSHOT__REGBB_RDY_MASK 0x00000800L
+#define SRBM_DEBUG_SNAPSHOT__REGBB_RDY__SHIFT 0x0000000b
+#define SRBM_DEBUG_SNAPSHOT__UVD_RDY_MASK 0x00000200L
+#define SRBM_DEBUG_SNAPSHOT__UVD_RDY__SHIFT 0x00000009
+#define SRBM_DEBUG_SNAPSHOT__VCE_RDY_MASK 0x20000000L
+#define SRBM_DEBUG_SNAPSHOT__VCE_RDY__SHIFT 0x0000001d
+#define SRBM_DEBUG_SNAPSHOT__XDMA_RDY_MASK 0x00000100L
+#define SRBM_DEBUG_SNAPSHOT__XDMA_RDY__SHIFT 0x00000008
+#define SRBM_DEBUG_SNAPSHOT__XSP_RDY_MASK 0x00000400L
+#define SRBM_DEBUG_SNAPSHOT__XSP_RDY__SHIFT 0x0000000a
+#define SRBM_DEBUG__SYS_CLOCK_DOMAIN_OVERRIDE_MASK 0x00000010L
+#define SRBM_DEBUG__SYS_CLOCK_DOMAIN_OVERRIDE__SHIFT 0x00000004
+#define SRBM_DEBUG__UVD_CLOCK_DOMAIN_OVERRIDE_MASK 0x00000040L
+#define SRBM_DEBUG__UVD_CLOCK_DOMAIN_OVERRIDE__SHIFT 0x00000006
+#define SRBM_DEBUG__VCE_CLOCK_DOMAIN_OVERRIDE_MASK 0x00000020L
+#define SRBM_DEBUG__VCE_CLOCK_DOMAIN_OVERRIDE__SHIFT 0x00000005
+#define SRBM_GFX_CNTL__VMID_MASK 0x000000f0L
+#define SRBM_GFX_CNTL__VMID__SHIFT 0x00000004
+#define SRBM_INT_ACK__RDERR_INT_ACK_MASK 0x00000001L
+#define SRBM_INT_ACK__RDERR_INT_ACK__SHIFT 0x00000000
+#define SRBM_INT_CNTL__RDERR_INT_MASK_MASK 0x00000001L
+#define SRBM_INT_CNTL__RDERR_INT_MASK__SHIFT 0x00000000
+#define SRBM_INT_STATUS__RDERR_INT_STAT_MASK 0x00000001L
+#define SRBM_INT_STATUS__RDERR_INT_STAT__SHIFT 0x00000000
+#define SRBM_MC_CLKEN_CNTL__POST_DELAY_CNT_MASK 0x00001f00L
+#define SRBM_MC_CLKEN_CNTL__POST_DELAY_CNT__SHIFT 0x00000008
+#define SRBM_MC_CLKEN_CNTL__PREFIX_DELAY_CNT_MASK 0x0000000fL
+#define SRBM_MC_CLKEN_CNTL__PREFIX_DELAY_CNT__SHIFT 0x00000000
+#define SRBM_PERFCOUNTER0_HI__PERF_COUNT0_HI_MASK 0xffffffffL
+#define SRBM_PERFCOUNTER0_HI__PERF_COUNT0_HI__SHIFT 0x00000000
+#define SRBM_PERFCOUNTER0_LO__PERF_COUNT0_LO_MASK 0xffffffffL
+#define SRBM_PERFCOUNTER0_LO__PERF_COUNT0_LO__SHIFT 0x00000000
+#define SRBM_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x0000003fL
+#define SRBM_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SRBM_PERFCOUNTER1_HI__PERF_COUNT1_HI_MASK 0xffffffffL
+#define SRBM_PERFCOUNTER1_HI__PERF_COUNT1_HI__SHIFT 0x00000000
+#define SRBM_PERFCOUNTER1_LO__PERF_COUNT1_LO_MASK 0xffffffffL
+#define SRBM_PERFCOUNTER1_LO__PERF_COUNT1_LO__SHIFT 0x00000000
+#define SRBM_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x0000003fL
+#define SRBM_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x00000000
+#define SRBM_PERFMON_CNTL__PERFMON_ENABLE_MODE_MASK 0x00000300L
+#define SRBM_PERFMON_CNTL__PERFMON_ENABLE_MODE__SHIFT 0x00000008
+#define SRBM_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK 0x00000400L
+#define SRBM_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT 0x0000000a
+#define SRBM_PERFMON_CNTL__PERFMON_STATE_MASK 0x0000000fL
+#define SRBM_PERFMON_CNTL__PERFMON_STATE__SHIFT 0x00000000
+#define SRBM_READ_ERROR__READ_ADDRESS_MASK 0x0003fffcL
+#define SRBM_READ_ERROR__READ_ADDRESS__SHIFT 0x00000002
+#define SRBM_READ_ERROR__READ_ERROR_MASK 0x80000000L
+#define SRBM_READ_ERROR__READ_ERROR__SHIFT 0x0000001f
+#define SRBM_READ_ERROR__READ_REQUESTER_GRBM_MASK 0x02000000L
+#define SRBM_READ_ERROR__READ_REQUESTER_GRBM__SHIFT 0x00000019
+#define SRBM_READ_ERROR__READ_REQUESTER_HI_MASK 0x01000000L
+#define SRBM_READ_ERROR__READ_REQUESTER_HI__SHIFT 0x00000018
+#define SRBM_READ_ERROR__READ_REQUESTER_SMU_MASK 0x04000000L
+#define SRBM_READ_ERROR__READ_REQUESTER_SMU__SHIFT 0x0000001a
+#define SRBM_READ_ERROR__READ_REQUESTER_TST_MASK 0x00400000L
+#define SRBM_READ_ERROR__READ_REQUESTER_TST__SHIFT 0x00000016
+#define SRBM_READ_ERROR__READ_REQUESTER_UVD_MASK 0x20000000L
+#define SRBM_READ_ERROR__READ_REQUESTER_UVD__SHIFT 0x0000001d
+#define SRBM_READ_ERROR__READ_REQUESTER_VCE_MASK 0x00100000L
+#define SRBM_READ_ERROR__READ_REQUESTER_VCE__SHIFT 0x00000014
+#define SRBM_SOFT_RESET__SOFT_RESET_BIF_MASK 0x00000002L
+#define SRBM_SOFT_RESET__SOFT_RESET_BIF__SHIFT 0x00000001
+#define SRBM_SOFT_RESET__SOFT_RESET_DC_MASK 0x00000020L
+#define SRBM_SOFT_RESET__SOFT_RESET_DC__SHIFT 0x00000005
+#define SRBM_SOFT_RESET__SOFT_RESET_GRBM_MASK 0x00000100L
+#define SRBM_SOFT_RESET__SOFT_RESET_GRBM__SHIFT 0x00000008
+#define SRBM_SOFT_RESET__SOFT_RESET_HDP_MASK 0x00000200L
+#define SRBM_SOFT_RESET__SOFT_RESET_HDP__SHIFT 0x00000009
+#define SRBM_SOFT_RESET__SOFT_RESET_IH_MASK 0x00000400L
+#define SRBM_SOFT_RESET__SOFT_RESET_IH__SHIFT 0x0000000a
+#define SRBM_SOFT_RESET__SOFT_RESET_MC_MASK 0x00000800L
+#define SRBM_SOFT_RESET__SOFT_RESET_MC__SHIFT 0x0000000b
+#define SRBM_SOFT_RESET__SOFT_RESET_ORB_MASK 0x00800000L
+#define SRBM_SOFT_RESET__SOFT_RESET_ORB__SHIFT 0x00000017
+#define SRBM_SOFT_RESET__SOFT_RESET_REGBB_MASK 0x00400000L
+#define SRBM_SOFT_RESET__SOFT_RESET_REGBB__SHIFT 0x00000016
+#define SRBM_SOFT_RESET__SOFT_RESET_ROM_MASK 0x00004000L
+#define SRBM_SOFT_RESET__SOFT_RESET_ROM__SHIFT 0x0000000e
+#define SRBM_SOFT_RESET__SOFT_RESET_SEM_MASK 0x00008000L
+#define SRBM_SOFT_RESET__SOFT_RESET_SEM__SHIFT 0x0000000f
+#define SRBM_SOFT_RESET__SOFT_RESET_TST_MASK 0x00200000L
+#define SRBM_SOFT_RESET__SOFT_RESET_TST__SHIFT 0x00000015
+#define SRBM_SOFT_RESET__SOFT_RESET_UVD_MASK 0x00040000L
+#define SRBM_SOFT_RESET__SOFT_RESET_UVD__SHIFT 0x00000012
+#define SRBM_SOFT_RESET__SOFT_RESET_VCE_MASK 0x01000000L
+#define SRBM_SOFT_RESET__SOFT_RESET_VCE__SHIFT 0x00000018
+#define SRBM_SOFT_RESET__SOFT_RESET_VMC_MASK 0x00020000L
+#define SRBM_SOFT_RESET__SOFT_RESET_VMC__SHIFT 0x00000011
+#define SRBM_SOFT_RESET__SOFT_RESET_XDMA_MASK 0x02000000L
+#define SRBM_SOFT_RESET__SOFT_RESET_XDMA__SHIFT 0x00000019
+#define SRBM_SOFT_RESET__SOFT_RESET_XSP_MASK 0x00080000L
+#define SRBM_SOFT_RESET__SOFT_RESET_XSP__SHIFT 0x00000013
+#define SRBM_STATUS2__TST_RQ_PENDING_MASK 0x00000002L
+#define SRBM_STATUS2__TST_RQ_PENDING__SHIFT 0x00000001
+#define SRBM_STATUS2__VCE_BUSY_MASK 0x00000080L
+#define SRBM_STATUS2__VCE_BUSY__SHIFT 0x00000007
+#define SRBM_STATUS2__VCE_RQ_PENDING_MASK 0x00000008L
+#define SRBM_STATUS2__VCE_RQ_PENDING__SHIFT 0x00000003
+#define SRBM_STATUS2__XDMA_BUSY_MASK 0x00000100L
+#define SRBM_STATUS2__XDMA_BUSY__SHIFT 0x00000008
+#define SRBM_STATUS2__XSP_BUSY_MASK 0x00000010L
+#define SRBM_STATUS2__XSP_BUSY__SHIFT 0x00000004
+#define SRBM_STATUS__BIF_BUSY_MASK 0x20000000L
+#define SRBM_STATUS__BIF_BUSY__SHIFT 0x0000001d
+#define SRBM_STATUS__GRBM_RQ_PENDING_MASK 0x00000020L
+#define SRBM_STATUS__GRBM_RQ_PENDING__SHIFT 0x00000005
+#define SRBM_STATUS__HI_RQ_PENDING_MASK 0x00000040L
+#define SRBM_STATUS__HI_RQ_PENDING__SHIFT 0x00000006
+#define SRBM_STATUS__IH_BUSY_MASK 0x00020000L
+#define SRBM_STATUS__IH_BUSY__SHIFT 0x00000011
+#define SRBM_STATUS__IO_EXTERN_SIGNAL_MASK 0x00000080L
+#define SRBM_STATUS__IO_EXTERN_SIGNAL__SHIFT 0x00000007
+#define SRBM_STATUS__MCB_BUSY_MASK 0x00000200L
+#define SRBM_STATUS__MCB_BUSY__SHIFT 0x00000009
+#define SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK 0x00000400L
+#define SRBM_STATUS__MCB_NON_DISPLAY_BUSY__SHIFT 0x0000000a
+#define SRBM_STATUS__MCC_BUSY_MASK 0x00000800L
+#define SRBM_STATUS__MCC_BUSY__SHIFT 0x0000000b
+#define SRBM_STATUS__MCD_BUSY_MASK 0x00001000L
+#define SRBM_STATUS__MCD_BUSY__SHIFT 0x0000000c
+#define SRBM_STATUS__SEM_BUSY_MASK 0x00004000L
+#define SRBM_STATUS__SEM_BUSY__SHIFT 0x0000000e
+#define SRBM_STATUS__SMU_RQ_PENDING_MASK 0x00000010L
+#define SRBM_STATUS__SMU_RQ_PENDING__SHIFT 0x00000004
+#define SRBM_STATUS__UVD_BUSY_MASK 0x00080000L
+#define SRBM_STATUS__UVD_BUSY__SHIFT 0x00000013
+#define SRBM_STATUS__UVD_RQ_PENDING_MASK 0x00000002L
+#define SRBM_STATUS__UVD_RQ_PENDING__SHIFT 0x00000001
+#define SRBM_STATUS__VMC_BUSY_MASK 0x00000100L
+#define SRBM_STATUS__VMC_BUSY__SHIFT 0x00000008
+#define SRBM_SYS_CLKEN_CNTL__POST_DELAY_CNT_MASK 0x00001f00L
+#define SRBM_SYS_CLKEN_CNTL__POST_DELAY_CNT__SHIFT 0x00000008
+#define SRBM_SYS_CLKEN_CNTL__PREFIX_DELAY_CNT_MASK 0x0000000fL
+#define SRBM_SYS_CLKEN_CNTL__PREFIX_DELAY_CNT__SHIFT 0x00000000
+#define SRBM_UVD_CLKEN_CNTL__POST_DELAY_CNT_MASK 0x00001f00L
+#define SRBM_UVD_CLKEN_CNTL__POST_DELAY_CNT__SHIFT 0x00000008
+#define SRBM_UVD_CLKEN_CNTL__PREFIX_DELAY_CNT_MASK 0x0000000fL
+#define SRBM_UVD_CLKEN_CNTL__PREFIX_DELAY_CNT__SHIFT 0x00000000
+#define SRBM_VCE_CLKEN_CNTL__POST_DELAY_CNT_MASK 0x00001f00L
+#define SRBM_VCE_CLKEN_CNTL__POST_DELAY_CNT__SHIFT 0x00000008
+#define SRBM_VCE_CLKEN_CNTL__PREFIX_DELAY_CNT_MASK 0x0000000fL
+#define SRBM_VCE_CLKEN_CNTL__PREFIX_DELAY_CNT__SHIFT 0x00000000
+#define UVD_CONFIG__UVD_RDREQ_URG_MASK 0x00000f00L
+#define UVD_CONFIG__UVD_RDREQ_URG__SHIFT 0x00000008
+#define UVD_CONFIG__UVD_REQ_TRAN_MASK 0x00010000L
+#define UVD_CONFIG__UVD_REQ_TRAN__SHIFT 0x00000010
+#define VCE_CONFIG__VCE_RDREQ_URG_MASK 0x00000f00L
+#define VCE_CONFIG__VCE_RDREQ_URG__SHIFT 0x00000008
+#define VCE_CONFIG__VCE_REQ_TRAN_MASK 0x00010000L
+#define VCE_CONFIG__VCE_REQ_TRAN__SHIFT 0x00000010
+#define XDMA_MSTR_CNTL__XDMA_MSTR_LAT_TEST_EN_MASK 0x00080000L
+#define XDMA_MSTR_CNTL__XDMA_MSTR_LAT_TEST_EN__SHIFT 0x00000013
+#define XDMA_MSTR_MEM_OVERFLOW_CNTL__XDMA_MSTR_OVERFLOW_COUNT_ENABLE_MASK 0x80000000L
+#define XDMA_MSTR_MEM_OVERFLOW_CNTL__XDMA_MSTR_OVERFLOW_COUNT_ENABLE__SHIFT 0x0000001f
+#define XDMA_MSTR_MEM_OVERFLOW_CNTL__XDMA_MSTR_OVERFLOW_COUNT_MASK 0x0000ffffL
+#define XDMA_MSTR_MEM_OVERFLOW_CNTL__XDMA_MSTR_OVERFLOW_COUNT__SHIFT 0x00000000
+#define XDMA_MSTR_MEM_OVERFLOW_CNTL__XDMA_MSTR_OVERFLOW_THRESHOLD_MASK 0x3fff0000L
+#define XDMA_MSTR_MEM_OVERFLOW_CNTL__XDMA_MSTR_OVERFLOW_THRESHOLD__SHIFT 0x00000010
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_6_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_6_0_d.h
new file mode 100644
index 000000000000..6b10be61efc3
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_6_0_d.h
@@ -0,0 +1,148 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SMU_6_0_D_H
+#define SMU_6_0_D_H
+
+#define ixLCAC_MC0_CNTL 0x011C
+#define ixLCAC_MC0_OVR_SEL 0x011D
+#define ixLCAC_MC0_OVR_VAL 0x011E
+#define ixLCAC_MC1_CNTL 0x011F
+#define ixLCAC_MC1_OVR_SEL 0x0120
+#define ixLCAC_MC1_OVR_VAL 0x0121
+#define ixLCAC_MC2_CNTL 0x0122
+#define ixLCAC_MC2_OVR_SEL 0x0123
+#define ixLCAC_MC2_OVR_VAL 0x0124
+#define ixLCAC_MC3_CNTL 0x0125
+#define ixLCAC_MC3_OVR_SEL 0x0126
+#define ixLCAC_MC3_OVR_VAL 0x0127
+#define ixLCAC_MC4_CNTL 0x0128
+#define ixLCAC_MC4_OVR_SEL 0x0129
+#define ixLCAC_MC4_OVR_VAL 0x012A
+#define ixLCAC_MC5_CNTL 0x012B
+#define ixLCAC_MC5_OVR_SEL 0x012C
+#define ixLCAC_MC5_OVR_VAL 0x012D
+#define ixSMC_PC_C 0x80000370
+#define ixTHM_TMON0_DEBUG 0x03F0
+#define ixTHM_TMON0_INT_DATA 0x0380
+#define ixTHM_TMON0_RDIL0_DATA 0x0300
+#define ixTHM_TMON0_RDIL10_DATA 0x030A
+#define ixTHM_TMON0_RDIL11_DATA 0x030B
+#define ixTHM_TMON0_RDIL12_DATA 0x030C
+#define ixTHM_TMON0_RDIL13_DATA 0x030D
+#define ixTHM_TMON0_RDIL14_DATA 0x030E
+#define ixTHM_TMON0_RDIL15_DATA 0x030F
+#define ixTHM_TMON0_RDIL1_DATA 0x0301
+#define ixTHM_TMON0_RDIL2_DATA 0x0302
+#define ixTHM_TMON0_RDIL3_DATA 0x0303
+#define ixTHM_TMON0_RDIL4_DATA 0x0304
+#define ixTHM_TMON0_RDIL5_DATA 0x0305
+#define ixTHM_TMON0_RDIL6_DATA 0x0306
+#define ixTHM_TMON0_RDIL7_DATA 0x0307
+#define ixTHM_TMON0_RDIL8_DATA 0x0308
+#define ixTHM_TMON0_RDIL9_DATA 0x0309
+#define ixTHM_TMON0_RDIR0_DATA 0x0310
+#define ixTHM_TMON0_RDIR10_DATA 0x031A
+#define ixTHM_TMON0_RDIR11_DATA 0x031B
+#define ixTHM_TMON0_RDIR12_DATA 0x031C
+#define ixTHM_TMON0_RDIR13_DATA 0x031D
+#define ixTHM_TMON0_RDIR14_DATA 0x031E
+#define ixTHM_TMON0_RDIR15_DATA 0x031F
+#define ixTHM_TMON0_RDIR1_DATA 0x0311
+#define ixTHM_TMON0_RDIR2_DATA 0x0312
+#define ixTHM_TMON0_RDIR3_DATA 0x0313
+#define ixTHM_TMON0_RDIR4_DATA 0x0314
+#define ixTHM_TMON0_RDIR5_DATA 0x0315
+#define ixTHM_TMON0_RDIR6_DATA 0x0316
+#define ixTHM_TMON0_RDIR7_DATA 0x0317
+#define ixTHM_TMON0_RDIR8_DATA 0x0318
+#define ixTHM_TMON0_RDIR9_DATA 0x0319
+#define ixTHM_TMON1_DEBUG 0x03F1
+#define ixTHM_TMON1_INT_DATA 0x0381
+#define ixTHM_TMON1_RDIL0_DATA 0x0320
+#define ixTHM_TMON1_RDIL10_DATA 0x032A
+#define ixTHM_TMON1_RDIL11_DATA 0x032B
+#define ixTHM_TMON1_RDIL12_DATA 0x032C
+#define ixTHM_TMON1_RDIL13_DATA 0x032D
+#define ixTHM_TMON1_RDIL14_DATA 0x032E
+#define ixTHM_TMON1_RDIL15_DATA 0x032F
+#define ixTHM_TMON1_RDIL1_DATA 0x0321
+#define ixTHM_TMON1_RDIL2_DATA 0x0322
+#define ixTHM_TMON1_RDIL3_DATA 0x0323
+#define ixTHM_TMON1_RDIL4_DATA 0x0324
+#define ixTHM_TMON1_RDIL5_DATA 0x0325
+#define ixTHM_TMON1_RDIL6_DATA 0x0326
+#define ixTHM_TMON1_RDIL7_DATA 0x0327
+#define ixTHM_TMON1_RDIL8_DATA 0x0328
+#define ixTHM_TMON1_RDIL9_DATA 0x0329
+#define ixTHM_TMON1_RDIR0_DATA 0x0330
+#define ixTHM_TMON1_RDIR10_DATA 0x033A
+#define ixTHM_TMON1_RDIR11_DATA 0x033B
+#define ixTHM_TMON1_RDIR12_DATA 0x033C
+#define ixTHM_TMON1_RDIR13_DATA 0x033D
+#define ixTHM_TMON1_RDIR14_DATA 0x033E
+#define ixTHM_TMON1_RDIR15_DATA 0x033F
+#define ixTHM_TMON1_RDIR1_DATA 0x0331
+#define ixTHM_TMON1_RDIR2_DATA 0x0332
+#define ixTHM_TMON1_RDIR3_DATA 0x0333
+#define ixTHM_TMON1_RDIR4_DATA 0x0334
+#define ixTHM_TMON1_RDIR5_DATA 0x0335
+#define ixTHM_TMON1_RDIR6_DATA 0x0336
+#define ixTHM_TMON1_RDIR7_DATA 0x0337
+#define ixTHM_TMON1_RDIR8_DATA 0x0338
+#define ixTHM_TMON1_RDIR9_DATA 0x0339
+#define mmGPIOPAD_A 0x05E7
+#define mmGPIOPAD_EN 0x05E8
+#define mmGPIOPAD_EXTERN_TRIG_CNTL 0x05F1
+#define mmGPIOPAD_INT_EN 0x05EE
+#define mmGPIOPAD_INT_POLARITY 0x05F0
+#define mmGPIOPAD_INT_STAT 0x05EC
+#define mmGPIOPAD_INT_STAT_AK 0x05ED
+#define mmGPIOPAD_INT_STAT_EN 0x05EB
+#define mmGPIOPAD_INT_TYPE 0x05EF
+#define mmGPIOPAD_MASK 0x05E6
+#define mmGPIOPAD_PD_EN 0x05F4
+#define mmGPIOPAD_PINSTRAPS 0x05EA
+#define mmGPIOPAD_PU_EN 0x05F3
+#define mmGPIOPAD_RCVR_SEL 0x05F2
+#define mmGPIOPAD_STRENGTH 0x05E5
+#define mmGPIOPAD_SW_INT_STAT 0x05E4
+#define mmGPIOPAD_Y 0x05E9
+#define mmSMC_IND_ACCESS_CNTL 0x008A
+#define mmSMC_IND_DATA_0 0x0081
+#define mmSMC_IND_DATA 0x0081
+#define mmSMC_IND_DATA_1 0x0083
+#define mmSMC_IND_DATA_2 0x0085
+#define mmSMC_IND_DATA_3 0x0087
+#define mmSMC_IND_INDEX_0 0x0080
+#define mmSMC_IND_INDEX 0x0080
+#define mmSMC_IND_INDEX_1 0x0082
+#define mmSMC_IND_INDEX_2 0x0084
+#define mmSMC_IND_INDEX_3 0x0086
+#define mmSMC_MESSAGE_0 0x008B
+#define mmSMC_MESSAGE_1 0x008D
+#define mmSMC_MESSAGE_2 0x008F
+#define mmSMC_RESP_0 0x008C
+#define mmSMC_RESP_1 0x008E
+#define mmSMC_RESP_2 0x0090
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_6_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_6_0_sh_mask.h
new file mode 100644
index 000000000000..7d3925b7266e
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_6_0_sh_mask.h
@@ -0,0 +1,715 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SMU_6_0_SH_MASK_H
+#define SMU_6_0_SH_MASK_H
+
+#define CG_SPLL_FUNC_CNTL_3__SPLL_FB_DIV_MASK 0x03ffffffL
+#define CG_SPLL_FUNC_CNTL_3__SPLL_FB_DIV__SHIFT 0x00000000
+#define CG_SPLL_FUNC_CNTL__SPLL_REF_DIV_MASK 0x000003f0L
+#define CG_SPLL_FUNC_CNTL__SPLL_REF_DIV__SHIFT 0x00000004
+#define GPIOPAD_A__GPIO_A_MASK 0x7fffffffL
+#define GPIOPAD_A__GPIO_A__SHIFT 0x00000000
+#define GPIOPAD_EN__GPIO_EN_MASK 0x7fffffffL
+#define GPIOPAD_EN__GPIO_EN__SHIFT 0x00000000
+#define GPIOPAD_EXTERN_TRIG_CNTL__EXTERN_TRIG_CLR_MASK 0x00000020L
+#define GPIOPAD_EXTERN_TRIG_CNTL__EXTERN_TRIG_CLR__SHIFT 0x00000005
+#define GPIOPAD_EXTERN_TRIG_CNTL__EXTERN_TRIG_READ_MASK 0x00000040L
+#define GPIOPAD_EXTERN_TRIG_CNTL__EXTERN_TRIG_READ__SHIFT 0x00000006
+#define GPIOPAD_EXTERN_TRIG_CNTL__EXTERN_TRIG_SEL_MASK 0x0000001fL
+#define GPIOPAD_EXTERN_TRIG_CNTL__EXTERN_TRIG_SEL__SHIFT 0x00000000
+#define GPIOPAD_INT_EN__GPIO_INT_EN_MASK 0x1fffffffL
+#define GPIOPAD_INT_EN__GPIO_INT_EN__SHIFT 0x00000000
+#define GPIOPAD_INT_EN__SW_INITIATED_INT_EN_MASK 0x80000000L
+#define GPIOPAD_INT_EN__SW_INITIATED_INT_EN__SHIFT 0x0000001f
+#define GPIOPAD_INT_POLARITY__GPIO_INT_POLARITY_MASK 0x1fffffffL
+#define GPIOPAD_INT_POLARITY__GPIO_INT_POLARITY__SHIFT 0x00000000
+#define GPIOPAD_INT_POLARITY__SW_INITIATED_INT_POLARITY_MASK 0x80000000L
+#define GPIOPAD_INT_POLARITY__SW_INITIATED_INT_POLARITY__SHIFT 0x0000001f
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_0_MASK 0x00000001L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_0__SHIFT 0x00000000
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_10_MASK 0x00000400L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_10__SHIFT 0x0000000a
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_11_MASK 0x00000800L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_11__SHIFT 0x0000000b
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_12_MASK 0x00001000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_12__SHIFT 0x0000000c
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_13_MASK 0x00002000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_13__SHIFT 0x0000000d
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_14_MASK 0x00004000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_14__SHIFT 0x0000000e
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_15_MASK 0x00008000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_15__SHIFT 0x0000000f
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_16_MASK 0x00010000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_16__SHIFT 0x00000010
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_17_MASK 0x00020000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_17__SHIFT 0x00000011
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_18_MASK 0x00040000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_18__SHIFT 0x00000012
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_19_MASK 0x00080000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_19__SHIFT 0x00000013
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_1_MASK 0x00000002L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_1__SHIFT 0x00000001
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_20_MASK 0x00100000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_20__SHIFT 0x00000014
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_21_MASK 0x00200000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_21__SHIFT 0x00000015
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_22_MASK 0x00400000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_22__SHIFT 0x00000016
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_23_MASK 0x00800000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_23__SHIFT 0x00000017
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_24_MASK 0x01000000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_24__SHIFT 0x00000018
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_25_MASK 0x02000000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_25__SHIFT 0x00000019
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_26_MASK 0x04000000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_26__SHIFT 0x0000001a
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_27_MASK 0x08000000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_27__SHIFT 0x0000001b
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_28_MASK 0x10000000L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_28__SHIFT 0x0000001c
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_2_MASK 0x00000004L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_2__SHIFT 0x00000002
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_3_MASK 0x00000008L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_3__SHIFT 0x00000003
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_4_MASK 0x00000010L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_4__SHIFT 0x00000004
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_5_MASK 0x00000020L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_5__SHIFT 0x00000005
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_6_MASK 0x00000040L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_6__SHIFT 0x00000006
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_7_MASK 0x00000080L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_7__SHIFT 0x00000007
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_8_MASK 0x00000100L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_8__SHIFT 0x00000008
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_9_MASK 0x00000200L
+#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_9__SHIFT 0x00000009
+#define GPIOPAD_INT_STAT_AK__SW_INITIATED_INT_STAT_AK_MASK 0x80000000L
+#define GPIOPAD_INT_STAT_AK__SW_INITIATED_INT_STAT_AK__SHIFT 0x0000001f
+#define GPIOPAD_INT_STAT_EN__GPIO_INT_STAT_EN_MASK 0x1fffffffL
+#define GPIOPAD_INT_STAT_EN__GPIO_INT_STAT_EN__SHIFT 0x00000000
+#define GPIOPAD_INT_STAT_EN__SW_INITIATED_INT_STAT_EN_MASK 0x80000000L
+#define GPIOPAD_INT_STAT_EN__SW_INITIATED_INT_STAT_EN__SHIFT 0x0000001f
+#define GPIOPAD_INT_STAT__GPIO_INT_STAT_MASK 0x1fffffffL
+#define GPIOPAD_INT_STAT__GPIO_INT_STAT__SHIFT 0x00000000
+#define GPIOPAD_INT_STAT__SW_INITIATED_INT_STAT_MASK 0x80000000L
+#define GPIOPAD_INT_STAT__SW_INITIATED_INT_STAT__SHIFT 0x0000001f
+#define GPIOPAD_INT_TYPE__GPIO_INT_TYPE_MASK 0x1fffffffL
+#define GPIOPAD_INT_TYPE__GPIO_INT_TYPE__SHIFT 0x00000000
+#define GPIOPAD_INT_TYPE__SW_INITIATED_INT_TYPE_MASK 0x80000000L
+#define GPIOPAD_INT_TYPE__SW_INITIATED_INT_TYPE__SHIFT 0x0000001f
+#define GPIOPAD_MASK__GPIO_MASK_MASK 0x7fffffffL
+#define GPIOPAD_MASK__GPIO_MASK__SHIFT 0x00000000
+#define GPIOPAD_PD_EN__GPIO_PD_EN_MASK 0x7fffffffL
+#define GPIOPAD_PD_EN__GPIO_PD_EN__SHIFT 0x00000000
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_0_MASK 0x00000001L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_0__SHIFT 0x00000000
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_10_MASK 0x00000400L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_10__SHIFT 0x0000000a
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_11_MASK 0x00000800L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_11__SHIFT 0x0000000b
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_12_MASK 0x00001000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_12__SHIFT 0x0000000c
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_13_MASK 0x00002000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_13__SHIFT 0x0000000d
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_14_MASK 0x00004000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_14__SHIFT 0x0000000e
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_15_MASK 0x00008000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_15__SHIFT 0x0000000f
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_16_MASK 0x00010000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_16__SHIFT 0x00000010
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_17_MASK 0x00020000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_17__SHIFT 0x00000011
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_18_MASK 0x00040000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_18__SHIFT 0x00000012
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_19_MASK 0x00080000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_19__SHIFT 0x00000013
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_1_MASK 0x00000002L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_1__SHIFT 0x00000001
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_20_MASK 0x00100000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_20__SHIFT 0x00000014
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_21_MASK 0x00200000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_21__SHIFT 0x00000015
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_22_MASK 0x00400000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_22__SHIFT 0x00000016
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_23_MASK 0x00800000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_23__SHIFT 0x00000017
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_24_MASK 0x01000000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_24__SHIFT 0x00000018
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_25_MASK 0x02000000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_25__SHIFT 0x00000019
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_26_MASK 0x04000000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_26__SHIFT 0x0000001a
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_27_MASK 0x08000000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_27__SHIFT 0x0000001b
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_28_MASK 0x10000000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_28__SHIFT 0x0000001c
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_29_MASK 0x20000000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_29__SHIFT 0x0000001d
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_2_MASK 0x00000004L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_2__SHIFT 0x00000002
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_30_MASK 0x40000000L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_30__SHIFT 0x0000001e
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_3_MASK 0x00000008L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_3__SHIFT 0x00000003
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_4_MASK 0x00000010L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_4__SHIFT 0x00000004
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_5_MASK 0x00000020L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_5__SHIFT 0x00000005
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_6_MASK 0x00000040L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_6__SHIFT 0x00000006
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_7_MASK 0x00000080L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_7__SHIFT 0x00000007
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_8_MASK 0x00000100L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_8__SHIFT 0x00000008
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_9_MASK 0x00000200L
+#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_9__SHIFT 0x00000009
+#define GPIOPAD_PU_EN__GPIO_PU_EN_MASK 0x7fffffffL
+#define GPIOPAD_PU_EN__GPIO_PU_EN__SHIFT 0x00000000
+#define GPIOPAD_RCVR_SEL__GPIO_RCVR_SEL_MASK 0x7fffffffL
+#define GPIOPAD_RCVR_SEL__GPIO_RCVR_SEL__SHIFT 0x00000000
+#define GPIOPAD_STRENGTH__GPIO_STRENGTH_SN_MASK 0x0000000fL
+#define GPIOPAD_STRENGTH__GPIO_STRENGTH_SN__SHIFT 0x00000000
+#define GPIOPAD_STRENGTH__GPIO_STRENGTH_SP_MASK 0x000000f0L
+#define GPIOPAD_STRENGTH__GPIO_STRENGTH_SP__SHIFT 0x00000004
+#define GPIOPAD_SW_INT_STAT__SW_INT_STAT_MASK 0x00000001L
+#define GPIOPAD_SW_INT_STAT__SW_INT_STAT__SHIFT 0x00000000
+#define GPIOPAD_Y__GPIO_Y_MASK 0x7fffffffL
+#define GPIOPAD_Y__GPIO_Y__SHIFT 0x00000000
+#define LCAC_MC0_CNTL__MC0_ENABLE_MASK 0x00000001L
+#define LCAC_MC0_CNTL__MC0_ENABLE__SHIFT 0x00000000
+#define LCAC_MC0_CNTL__MC0_THRESHOLD_MASK 0x0001fffeL
+#define LCAC_MC0_CNTL__MC0_THRESHOLD__SHIFT 0x00000001
+#define LCAC_MC0_OVR_SEL__MC0_OVR_SEL_MASK 0xffffffffL
+#define LCAC_MC0_OVR_SEL__MC0_OVR_SEL__SHIFT 0x00000000
+#define LCAC_MC0_OVR_VAL__MC0_OVR_VAL_MASK 0xffffffffL
+#define LCAC_MC0_OVR_VAL__MC0_OVR_VAL__SHIFT 0x00000000
+#define LCAC_MC1_CNTL__MC1_ENABLE_MASK 0x00000001L
+#define LCAC_MC1_CNTL__MC1_ENABLE__SHIFT 0x00000000
+#define LCAC_MC1_CNTL__MC1_THRESHOLD_MASK 0x0001fffeL
+#define LCAC_MC1_CNTL__MC1_THRESHOLD__SHIFT 0x00000001
+#define LCAC_MC1_OVR_SEL__MC1_OVR_SEL_MASK 0xffffffffL
+#define LCAC_MC1_OVR_SEL__MC1_OVR_SEL__SHIFT 0x00000000
+#define LCAC_MC1_OVR_VAL__MC1_OVR_VAL_MASK 0xffffffffL
+#define LCAC_MC1_OVR_VAL__MC1_OVR_VAL__SHIFT 0x00000000
+#define LCAC_MC2_CNTL__MC2_ENABLE_MASK 0x00000001L
+#define LCAC_MC2_CNTL__MC2_ENABLE__SHIFT 0x00000000
+#define LCAC_MC2_CNTL__MC2_THRESHOLD_MASK 0x0001fffeL
+#define LCAC_MC2_CNTL__MC2_THRESHOLD__SHIFT 0x00000001
+#define LCAC_MC2_OVR_SEL__MC2_OVR_SEL_MASK 0xffffffffL
+#define LCAC_MC2_OVR_SEL__MC2_OVR_SEL__SHIFT 0x00000000
+#define LCAC_MC2_OVR_VAL__MC2_OVR_VAL_MASK 0xffffffffL
+#define LCAC_MC2_OVR_VAL__MC2_OVR_VAL__SHIFT 0x00000000
+#define LCAC_MC3_CNTL__MC3_ENABLE_MASK 0x00000001L
+#define LCAC_MC3_CNTL__MC3_ENABLE__SHIFT 0x00000000
+#define LCAC_MC3_CNTL__MC3_THRESHOLD_MASK 0x0001fffeL
+#define LCAC_MC3_CNTL__MC3_THRESHOLD__SHIFT 0x00000001
+#define LCAC_MC3_OVR_SEL__MC3_OVR_SEL_MASK 0xffffffffL
+#define LCAC_MC3_OVR_SEL__MC3_OVR_SEL__SHIFT 0x00000000
+#define LCAC_MC3_OVR_VAL__MC3_OVR_VAL_MASK 0xffffffffL
+#define LCAC_MC3_OVR_VAL__MC3_OVR_VAL__SHIFT 0x00000000
+#define LCAC_MC4_CNTL__MC4_ENABLE_MASK 0x00000001L
+#define LCAC_MC4_CNTL__MC4_ENABLE__SHIFT 0x00000000
+#define LCAC_MC4_CNTL__MC4_THRESHOLD_MASK 0x0001fffeL
+#define LCAC_MC4_CNTL__MC4_THRESHOLD__SHIFT 0x00000001
+#define LCAC_MC4_OVR_SEL__MC4_OVR_SEL_MASK 0xffffffffL
+#define LCAC_MC4_OVR_SEL__MC4_OVR_SEL__SHIFT 0x00000000
+#define LCAC_MC4_OVR_VAL__MC4_OVR_VAL_MASK 0xffffffffL
+#define LCAC_MC4_OVR_VAL__MC4_OVR_VAL__SHIFT 0x00000000
+#define LCAC_MC5_CNTL__MC5_ENABLE_MASK 0x00000001L
+#define LCAC_MC5_CNTL__MC5_ENABLE__SHIFT 0x00000000
+#define LCAC_MC5_CNTL__MC5_THRESHOLD_MASK 0x0001fffeL
+#define LCAC_MC5_CNTL__MC5_THRESHOLD__SHIFT 0x00000001
+#define LCAC_MC5_OVR_SEL__MC5_OVR_SEL_MASK 0xffffffffL
+#define LCAC_MC5_OVR_SEL__MC5_OVR_SEL__SHIFT 0x00000000
+#define LCAC_MC5_OVR_VAL__MC5_OVR_VAL_MASK 0xffffffffL
+#define LCAC_MC5_OVR_VAL__MC5_OVR_VAL__SHIFT 0x00000000
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0_MASK 0x00000001L
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0__SHIFT 0x00000000
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_1_MASK 0x00000100L
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_1__SHIFT 0x00000008
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_2_MASK 0x00010000L
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_2__SHIFT 0x00000010
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_3_MASK 0x01000000L
+#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_3__SHIFT 0x00000018
+#define SMC_IND_DATA_0__SMC_IND_DATA_MASK 0xffffffffL
+#define SMC_IND_DATA_0__SMC_IND_DATA__SHIFT 0x00000000
+#define SMC_IND_DATA_1__SMC_IND_DATA_MASK 0xffffffffL
+#define SMC_IND_DATA_1__SMC_IND_DATA__SHIFT 0x00000000
+#define SMC_IND_DATA_2__SMC_IND_DATA_MASK 0xffffffffL
+#define SMC_IND_DATA_2__SMC_IND_DATA__SHIFT 0x00000000
+#define SMC_IND_DATA_3__SMC_IND_DATA_MASK 0xffffffffL
+#define SMC_IND_DATA_3__SMC_IND_DATA__SHIFT 0x00000000
+#define SMC_IND_DATA__SMC_IND_DATA_MASK 0xffffffffL
+#define SMC_IND_DATA__SMC_IND_DATA__SHIFT 0x00000000
+#define SMC_IND_INDEX_0__SMC_IND_ADDR_MASK 0xffffffffL
+#define SMC_IND_INDEX_0__SMC_IND_ADDR__SHIFT 0x00000000
+#define SMC_IND_INDEX_1__SMC_IND_ADDR_MASK 0xffffffffL
+#define SMC_IND_INDEX_1__SMC_IND_ADDR__SHIFT 0x00000000
+#define SMC_IND_INDEX_2__SMC_IND_ADDR_MASK 0xffffffffL
+#define SMC_IND_INDEX_2__SMC_IND_ADDR__SHIFT 0x00000000
+#define SMC_IND_INDEX_3__SMC_IND_ADDR_MASK 0xffffffffL
+#define SMC_IND_INDEX_3__SMC_IND_ADDR__SHIFT 0x00000000
+#define SMC_IND_INDEX__SMC_IND_ADDR_MASK 0xffffffffL
+#define SMC_IND_INDEX__SMC_IND_ADDR__SHIFT 0x00000000
+#define SMC_MESSAGE_0__SMC_MSG_MASK 0xffffffffL
+#define SMC_MESSAGE_0__SMC_MSG__SHIFT 0x00000000
+#define SMC_MESSAGE_1__SMC_MSG_MASK 0xffffffffL
+#define SMC_MESSAGE_1__SMC_MSG__SHIFT 0x00000000
+#define SMC_MESSAGE_2__SMC_MSG_MASK 0xffffffffL
+#define SMC_MESSAGE_2__SMC_MSG__SHIFT 0x00000000
+#define SMC_PC_C__smc_pc_c_MASK 0xffffffffL
+#define SMC_PC_C__smc_pc_c__SHIFT 0x00000000
+#define SMC_RESP_0__SMC_RESP_MASK 0xffffffffL
+#define SMC_RESP_0__SMC_RESP__SHIFT 0x00000000
+#define SMC_RESP_1__SMC_RESP_MASK 0xffffffffL
+#define SMC_RESP_1__SMC_RESP__SHIFT 0x00000000
+#define SMC_RESP_2__SMC_RESP_MASK 0xffffffffL
+#define SMC_RESP_2__SMC_RESP__SHIFT 0x00000000
+#define SPLL_CNTL_MODE__SPLL_CTLREQ_DLY_CNT_MASK 0x000ff000L
+#define SPLL_CNTL_MODE__SPLL_CTLREQ_DLY_CNT__SHIFT 0x0000000c
+#define SPLL_CNTL_MODE__SPLL_ENSAT_MASK 0x00000010L
+#define SPLL_CNTL_MODE__SPLL_ENSAT__SHIFT 0x00000004
+#define SPLL_CNTL_MODE__SPLL_FASTEN_MASK 0x00000008L
+#define SPLL_CNTL_MODE__SPLL_FASTEN__SHIFT 0x00000003
+#define SPLL_CNTL_MODE__SPLL_LEGACY_PDIV_MASK 0x00000002L
+#define SPLL_CNTL_MODE__SPLL_LEGACY_PDIV__SHIFT 0x00000001
+#define SPLL_CNTL_MODE__SPLL_RESET_EN_MASK 0x10000000L
+#define SPLL_CNTL_MODE__SPLL_RESET_EN__SHIFT 0x0000001c
+#define SPLL_CNTL_MODE__SPLL_SW_DIR_CONTROL_MASK 0x00000001L
+#define SPLL_CNTL_MODE__SPLL_SW_DIR_CONTROL__SHIFT 0x00000000
+#define SPLL_CNTL_MODE__SPLL_TEST_CLK_EXT_DIV_MASK 0x00000c00L
+#define SPLL_CNTL_MODE__SPLL_TEST_CLK_EXT_DIV__SHIFT 0x0000000a
+#define SPLL_CNTL_MODE__SPLL_TEST_MASK 0x00000004L
+#define SPLL_CNTL_MODE__SPLL_TEST__SHIFT 0x00000002
+#define SPLL_CNTL_MODE__SPLL_VCO_MODE_MASK 0x60000000L
+#define SPLL_CNTL_MODE__SPLL_VCO_MODE__SHIFT 0x0000001d
+#define TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX_MASK 0x0f000000L
+#define TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX__SHIFT 0x00000018
+#define TARGET_AND_CURRENT_PROFILE_INDEX_1__TARG_PCIE_INDEX_MASK 0xf0000000L
+#define TARGET_AND_CURRENT_PROFILE_INDEX_1__TARG_PCIE_INDEX__SHIFT 0x0000001c
+#define THM_TMON0_DEBUG__DEBUG_RDI_MASK 0x0000001fL
+#define THM_TMON0_DEBUG__DEBUG_RDI__SHIFT 0x00000000
+#define THM_TMON0_DEBUG__DEBUG_Z_MASK 0x0000ffe0L
+#define THM_TMON0_DEBUG__DEBUG_Z__SHIFT 0x00000005
+#define THM_TMON0_INT_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_INT_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_INT_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_INT_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_INT_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_INT_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL0_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL0_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL0_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL0_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL0_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL0_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL10_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL10_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL10_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL10_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL10_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL10_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL11_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL11_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL11_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL11_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL11_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL11_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL12_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL12_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL12_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL12_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL12_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL12_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL13_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL13_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL13_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL13_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL13_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL13_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL14_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL14_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL14_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL14_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL14_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL14_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL15_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL15_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL15_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL15_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL15_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL15_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL1_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL1_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL1_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL1_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL1_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL1_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL2_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL2_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL2_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL2_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL2_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL2_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL3_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL3_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL3_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL3_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL3_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL3_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL4_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL4_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL4_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL4_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL4_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL4_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL5_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL5_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL5_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL5_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL5_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL5_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL6_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL6_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL6_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL6_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL6_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL6_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL7_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL7_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL7_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL7_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL7_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL7_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL8_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL8_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL8_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL8_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL8_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL8_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIL9_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIL9_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIL9_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIL9_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIL9_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIL9_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR0_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR0_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR0_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR0_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR0_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR0_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR10_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR10_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR10_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR10_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR10_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR10_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR11_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR11_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR11_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR11_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR11_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR11_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR12_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR12_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR12_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR12_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR12_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR12_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR13_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR13_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR13_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR13_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR13_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR13_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR14_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR14_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR14_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR14_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR14_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR14_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR15_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR15_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR15_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR15_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR15_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR15_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR1_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR1_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR1_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR1_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR1_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR1_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR2_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR2_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR2_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR2_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR2_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR2_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR3_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR3_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR3_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR3_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR3_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR3_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR4_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR4_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR4_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR4_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR4_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR4_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR5_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR5_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR5_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR5_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR5_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR5_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR6_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR6_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR6_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR6_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR6_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR6_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR7_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR7_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR7_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR7_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR7_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR7_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR8_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR8_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR8_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR8_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR8_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR8_DATA__Z__SHIFT 0x00000000
+#define THM_TMON0_RDIR9_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON0_RDIR9_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON0_RDIR9_DATA__VALID_MASK 0x00000800L
+#define THM_TMON0_RDIR9_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON0_RDIR9_DATA__Z_MASK 0x000007ffL
+#define THM_TMON0_RDIR9_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_DEBUG__DEBUG_RDI_MASK 0x0000001fL
+#define THM_TMON1_DEBUG__DEBUG_RDI__SHIFT 0x00000000
+#define THM_TMON1_DEBUG__DEBUG_Z_MASK 0x0000ffe0L
+#define THM_TMON1_DEBUG__DEBUG_Z__SHIFT 0x00000005
+#define THM_TMON1_INT_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_INT_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_INT_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_INT_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_INT_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_INT_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL0_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL0_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL0_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL0_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL0_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL0_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL10_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL10_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL10_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL10_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL10_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL10_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL11_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL11_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL11_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL11_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL11_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL11_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL12_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL12_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL12_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL12_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL12_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL12_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL13_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL13_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL13_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL13_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL13_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL13_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL14_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL14_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL14_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL14_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL14_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL14_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL15_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL15_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL15_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL15_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL15_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL15_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL1_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL1_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL1_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL1_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL1_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL1_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL2_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL2_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL2_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL2_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL2_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL2_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL3_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL3_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL3_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL3_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL3_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL3_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL4_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL4_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL4_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL4_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL4_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL4_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL5_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL5_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL5_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL5_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL5_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL5_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL6_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL6_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL6_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL6_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL6_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL6_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL7_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL7_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL7_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL7_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL7_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL7_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL8_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL8_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL8_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL8_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL8_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL8_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIL9_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIL9_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIL9_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIL9_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIL9_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIL9_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR0_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR0_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR0_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR0_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR0_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR0_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR10_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR10_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR10_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR10_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR10_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR10_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR11_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR11_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR11_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR11_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR11_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR11_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR12_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR12_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR12_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR12_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR12_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR12_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR13_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR13_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR13_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR13_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR13_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR13_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR14_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR14_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR14_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR14_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR14_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR14_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR15_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR15_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR15_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR15_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR15_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR15_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR1_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR1_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR1_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR1_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR1_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR1_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR2_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR2_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR2_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR2_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR2_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR2_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR3_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR3_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR3_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR3_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR3_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR3_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR4_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR4_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR4_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR4_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR4_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR4_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR5_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR5_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR5_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR5_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR5_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR5_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR6_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR6_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR6_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR6_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR6_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR6_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR7_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR7_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR7_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR7_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR7_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR7_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR8_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR8_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR8_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR8_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR8_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR8_DATA__Z__SHIFT 0x00000000
+#define THM_TMON1_RDIR9_DATA__TEMP_MASK 0x00fff000L
+#define THM_TMON1_RDIR9_DATA__TEMP__SHIFT 0x0000000c
+#define THM_TMON1_RDIR9_DATA__VALID_MASK 0x00000800L
+#define THM_TMON1_RDIR9_DATA__VALID__SHIFT 0x0000000b
+#define THM_TMON1_RDIR9_DATA__Z_MASK 0x000007ffL
+#define THM_TMON1_RDIR9_DATA__Z__SHIFT 0x00000000
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_1_d.h b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_1_d.h
index 3014d4a58c43..a9ef1562f43b 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_1_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_1_d.h
@@ -176,6 +176,8 @@
 #define mmSMU1_SMU_SMC_IND_DATA                                                 0x83
 #define mmSMU2_SMU_SMC_IND_DATA                                                 0x85
 #define mmSMU3_SMU_SMC_IND_DATA                                                 0x87
+#define mmSMC_IND_INDEX_11														0x1AC
+#define mmSMC_IND_DATA_11														0x1AD
 #define ixRCU_UC_EVENTS                                                         0xc0000004
 #define ixRCU_MISC_CTRL                                                         0xc0000010
 #define ixCC_RCU_FUSES                                                          0xc00c0000
diff --git a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_2_d.h b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_2_d.h
index 933917479985..22dd4c2b7290 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_2_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_2_d.h
@@ -87,6 +87,8 @@
 #define mmSMC_IND_DATA_6                                                        0x8d
 #define mmSMC_IND_INDEX_7                                                       0x8e
 #define mmSMC_IND_DATA_7                                                        0x8f
+#define mmSMC_IND_INDEX_11														0x1AC
+#define mmSMC_IND_DATA_11														0x1AD
 #define mmSMC_IND_ACCESS_CNTL                                                   0x92
 #define mmSMC_MESSAGE_0                                                         0x94
 #define mmSMC_RESP_0                                                            0x95
diff --git a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_d.h b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_d.h
index 44b1855cb8df..eca2b851f25f 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_d.h
@@ -90,6 +90,8 @@
 #define mmSMC_IND_DATA_6                                                        0x8d
 #define mmSMC_IND_INDEX_7                                                       0x8e
 #define mmSMC_IND_DATA_7                                                        0x8f
+#define mmSMC_IND_INDEX_11														0x1AC
+#define mmSMC_IND_DATA_11														0x1AD
 #define mmSMC_IND_ACCESS_CNTL                                                   0x92
 #define mmSMC_MESSAGE_0                                                         0x94
 #define mmSMC_RESP_0                                                            0x95
diff --git a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_4_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_4_0_d.h
new file mode 100644
index 000000000000..5c0e3f3332e5
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_4_0_d.h
@@ -0,0 +1,96 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef UVD_4_0_D_H
+#define UVD_4_0_D_H
+
+#define ixUVD_CGC_CTRL2 0x00C1
+#define ixUVD_CGC_MEM_CTRL 0x00C0
+#define ixUVD_LMI_ADDR_EXT2 0x00AB
+#define ixUVD_LMI_CACHE_CTRL 0x009B
+#define ixUVD_LMI_SWAP_CNTL2 0x00AA
+#define ixUVD_MIF_CURR_ADDR_CONFIG 0x0048
+#define ixUVD_MIF_RECON1_ADDR_CONFIG 0x0114
+#define ixUVD_MIF_REF_ADDR_CONFIG 0x004C
+#define mmUVD_CGC_CTRL 0x3D2C
+#define mmUVD_CGC_GATE 0x3D2A
+#define mmUVD_CGC_STATUS 0x3D2B
+#define mmUVD_CGC_UDEC_STATUS 0x3D2D
+#define mmUVD_CONTEXT_ID 0x3DBD
+#define mmUVD_CTX_DATA 0x3D29
+#define mmUVD_CTX_INDEX 0x3D28
+#define mmUVD_ENGINE_CNTL 0x3BC6
+#define mmUVD_GPCOM_VCPU_CMD 0x3BC3
+#define mmUVD_GPCOM_VCPU_DATA0 0x3BC4
+#define mmUVD_GPCOM_VCPU_DATA1 0x3BC5
+#define mmUVD_GP_SCRATCH4 0x3D38
+#define mmUVD_LMI_ADDR_EXT 0x3D65
+#define mmUVD_LMI_CTRL 0x3D66
+#define mmUVD_LMI_CTRL2 0x3D3D
+#define mmUVD_LMI_EXT40_ADDR 0x3D26
+#define mmUVD_LMI_STATUS 0x3D67
+#define mmUVD_LMI_SWAP_CNTL 0x3D6D
+#define mmUVD_MASTINT_EN 0x3D40
+#define mmUVD_MPC_CNTL 0x3D77
+#define mmUVD_MPC_SET_ALU 0x3D7E
+#define mmUVD_MPC_SET_MUX 0x3D7D
+#define mmUVD_MPC_SET_MUXA0 0x3D79
+#define mmUVD_MPC_SET_MUXA1 0x3D7A
+#define mmUVD_MPC_SET_MUXB0 0x3D7B
+#define mmUVD_MPC_SET_MUXB1 0x3D7C
+#define mmUVD_MP_SWAP_CNTL 0x3D6F
+#define mmUVD_NO_OP 0x3BFF
+#define mmUVD_PGFSM_CONFIG 0x38F8
+#define mmUVD_PGFSM_READ_TILE1 0x38FA
+#define mmUVD_PGFSM_READ_TILE2 0x38FB
+#define mmUVD_POWER_STATUS 0x38FC
+#define mmUVD_RBC_IB_BASE 0x3DA1
+#define mmUVD_RBC_IB_SIZE 0x3DA2
+#define mmUVD_RBC_IB_SIZE_UPDATE 0x3DF1
+#define mmUVD_RBC_RB_BASE 0x3DA3
+#define mmUVD_RBC_RB_CNTL 0x3DA9
+#define mmUVD_RBC_RB_RPTR 0x3DA4
+#define mmUVD_RBC_RB_RPTR_ADDR 0x3DAA
+#define mmUVD_RBC_RB_WPTR 0x3DA5
+#define mmUVD_RBC_RB_WPTR_CNTL 0x3DA6
+#define mmUVD_SEMA_ADDR_HIGH 0x3BC1
+#define mmUVD_SEMA_ADDR_LOW 0x3BC0
+#define mmUVD_SEMA_CMD 0x3BC2
+#define mmUVD_SEMA_CNTL 0x3D00
+#define mmUVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL 0x3DB3
+#define mmUVD_SEMA_TIMEOUT_STATUS 0x3DB0
+#define mmUVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL 0x3DB2
+#define mmUVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL 0x3DB1
+#define mmUVD_SOFT_RESET 0x3DA0
+#define mmUVD_STATUS 0x3DAF
+#define mmUVD_UDEC_ADDR_CONFIG 0x3BD3
+#define mmUVD_UDEC_DB_ADDR_CONFIG 0x3BD4
+#define mmUVD_UDEC_DBW_ADDR_CONFIG 0x3BD5
+#define mmUVD_VCPU_CACHE_OFFSET0 0x3D36
+#define mmUVD_VCPU_CACHE_OFFSET1 0x3D38
+#define mmUVD_VCPU_CACHE_OFFSET2 0x3D3A
+#define mmUVD_VCPU_CACHE_SIZE0 0x3D37
+#define mmUVD_VCPU_CACHE_SIZE1 0x3D39
+#define mmUVD_VCPU_CACHE_SIZE2 0x3D3B
+#define mmUVD_VCPU_CNTL 0x3D98
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_4_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_4_0_sh_mask.h
new file mode 100644
index 000000000000..8ee3149df5b7
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_4_0_sh_mask.h
@@ -0,0 +1,795 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef UVD_4_0_SH_MASK_H
+#define UVD_4_0_SH_MASK_H
+
+#define UVD_CGC_CTRL2__DYN_OCLK_RAMP_EN_MASK 0x00000001L
+#define UVD_CGC_CTRL2__DYN_OCLK_RAMP_EN__SHIFT 0x00000000
+#define UVD_CGC_CTRL2__DYN_RCLK_RAMP_EN_MASK 0x00000002L
+#define UVD_CGC_CTRL2__DYN_RCLK_RAMP_EN__SHIFT 0x00000001
+#define UVD_CGC_CTRL2__GATER_DIV_ID_MASK 0x0000001cL
+#define UVD_CGC_CTRL2__GATER_DIV_ID__SHIFT 0x00000002
+#define UVD_CGC_CTRL__CLK_GATE_DLY_TIMER_MASK 0x0000003cL
+#define UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT 0x00000002
+#define UVD_CGC_CTRL__CLK_OFF_DELAY_MASK 0x000007c0L
+#define UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT 0x00000006
+#define UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK 0x00000001L
+#define UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT 0x00000000
+#define UVD_CGC_CTRL__IDCT_MODE_MASK 0x00800000L
+#define UVD_CGC_CTRL__IDCT_MODE__SHIFT 0x00000017
+#define UVD_CGC_CTRL__LBSI_MODE_MASK 0x04000000L
+#define UVD_CGC_CTRL__LBSI_MODE__SHIFT 0x0000001a
+#define UVD_CGC_CTRL__LMI_MC_MODE_MASK 0x00200000L
+#define UVD_CGC_CTRL__LMI_MC_MODE__SHIFT 0x00000015
+#define UVD_CGC_CTRL__LMI_UMC_MODE_MASK 0x00400000L
+#define UVD_CGC_CTRL__LMI_UMC_MODE__SHIFT 0x00000016
+#define UVD_CGC_CTRL__LRBBM_MODE_MASK 0x08000000L
+#define UVD_CGC_CTRL__LRBBM_MODE__SHIFT 0x0000001b
+#define UVD_CGC_CTRL__MPC_MODE_MASK 0x02000000L
+#define UVD_CGC_CTRL__MPC_MODE__SHIFT 0x00000019
+#define UVD_CGC_CTRL__MPEG2_MODE_MASK 0x00040000L
+#define UVD_CGC_CTRL__MPEG2_MODE__SHIFT 0x00000012
+#define UVD_CGC_CTRL__MPRD_MODE_MASK 0x01000000L
+#define UVD_CGC_CTRL__MPRD_MODE__SHIFT 0x00000018
+#define UVD_CGC_CTRL__RBC_MODE_MASK 0x00100000L
+#define UVD_CGC_CTRL__RBC_MODE__SHIFT 0x00000014
+#define UVD_CGC_CTRL__REGS_MODE_MASK 0x00080000L
+#define UVD_CGC_CTRL__REGS_MODE__SHIFT 0x00000013
+#define UVD_CGC_CTRL__SCPU_MODE_MASK 0x40000000L
+#define UVD_CGC_CTRL__SCPU_MODE__SHIFT 0x0000001e
+#define UVD_CGC_CTRL__SYS_MODE_MASK 0x00010000L
+#define UVD_CGC_CTRL__SYS_MODE__SHIFT 0x00000010
+#define UVD_CGC_CTRL__UDEC_CM_MODE_MASK 0x00001000L
+#define UVD_CGC_CTRL__UDEC_CM_MODE__SHIFT 0x0000000c
+#define UVD_CGC_CTRL__UDEC_DB_MODE_MASK 0x00004000L
+#define UVD_CGC_CTRL__UDEC_DB_MODE__SHIFT 0x0000000e
+#define UVD_CGC_CTRL__UDEC_IT_MODE_MASK 0x00002000L
+#define UVD_CGC_CTRL__UDEC_IT_MODE__SHIFT 0x0000000d
+#define UVD_CGC_CTRL__UDEC_MODE_MASK 0x00020000L
+#define UVD_CGC_CTRL__UDEC_MODE__SHIFT 0x00000011
+#define UVD_CGC_CTRL__UDEC_MP_MODE_MASK 0x00008000L
+#define UVD_CGC_CTRL__UDEC_MP_MODE__SHIFT 0x0000000f
+#define UVD_CGC_CTRL__UDEC_RE_MODE_MASK 0x00000800L
+#define UVD_CGC_CTRL__UDEC_RE_MODE__SHIFT 0x0000000b
+#define UVD_CGC_CTRL__VCPU_MODE_MASK 0x20000000L
+#define UVD_CGC_CTRL__VCPU_MODE__SHIFT 0x0000001d
+#define UVD_CGC_CTRL__WCB_MODE_MASK 0x10000000L
+#define UVD_CGC_CTRL__WCB_MODE__SHIFT 0x0000001c
+#define UVD_CGC_GATE__IDCT_MASK 0x00000080L
+#define UVD_CGC_GATE__IDCT__SHIFT 0x00000007
+#define UVD_CGC_GATE__LBSI_MASK 0x00000400L
+#define UVD_CGC_GATE__LBSI__SHIFT 0x0000000a
+#define UVD_CGC_GATE__LMI_MC_MASK 0x00000020L
+#define UVD_CGC_GATE__LMI_MC__SHIFT 0x00000005
+#define UVD_CGC_GATE__LMI_UMC_MASK 0x00000040L
+#define UVD_CGC_GATE__LMI_UMC__SHIFT 0x00000006
+#define UVD_CGC_GATE__LRBBM_MASK 0x00000800L
+#define UVD_CGC_GATE__LRBBM__SHIFT 0x0000000b
+#define UVD_CGC_GATE__MPC_MASK 0x00000200L
+#define UVD_CGC_GATE__MPC__SHIFT 0x00000009
+#define UVD_CGC_GATE__MPEG2_MASK 0x00000004L
+#define UVD_CGC_GATE__MPEG2__SHIFT 0x00000002
+#define UVD_CGC_GATE__MPRD_MASK 0x00000100L
+#define UVD_CGC_GATE__MPRD__SHIFT 0x00000008
+#define UVD_CGC_GATE__RBC_MASK 0x00000010L
+#define UVD_CGC_GATE__RBC__SHIFT 0x00000004
+#define UVD_CGC_GATE__REGS_MASK 0x00000008L
+#define UVD_CGC_GATE__REGS__SHIFT 0x00000003
+#define UVD_CGC_GATE__SCPU_MASK 0x00080000L
+#define UVD_CGC_GATE__SCPU__SHIFT 0x00000013
+#define UVD_CGC_GATE__SYS_MASK 0x00000001L
+#define UVD_CGC_GATE__SYS__SHIFT 0x00000000
+#define UVD_CGC_GATE__UDEC_CM_MASK 0x00002000L
+#define UVD_CGC_GATE__UDEC_CM__SHIFT 0x0000000d
+#define UVD_CGC_GATE__UDEC_DB_MASK 0x00008000L
+#define UVD_CGC_GATE__UDEC_DB__SHIFT 0x0000000f
+#define UVD_CGC_GATE__UDEC_IT_MASK 0x00004000L
+#define UVD_CGC_GATE__UDEC_IT__SHIFT 0x0000000e
+#define UVD_CGC_GATE__UDEC_MASK 0x00000002L
+#define UVD_CGC_GATE__UDEC_MP_MASK 0x00010000L
+#define UVD_CGC_GATE__UDEC_MP__SHIFT 0x00000010
+#define UVD_CGC_GATE__UDEC_RE_MASK 0x00001000L
+#define UVD_CGC_GATE__UDEC_RE__SHIFT 0x0000000c
+#define UVD_CGC_GATE__UDEC__SHIFT 0x00000001
+#define UVD_CGC_GATE__VCPU_MASK 0x00040000L
+#define UVD_CGC_GATE__VCPU__SHIFT 0x00000012
+#define UVD_CGC_GATE__WCB_MASK 0x00020000L
+#define UVD_CGC_GATE__WCB__SHIFT 0x00000011
+#define UVD_CGC_MEM_CTRL__LCM_LS_EN_MASK 0x00002000L
+#define UVD_CGC_MEM_CTRL__LCM_LS_EN__SHIFT 0x0000000d
+#define UVD_CGC_MEM_CTRL__LMI_MC_LS_EN_MASK 0x00000001L
+#define UVD_CGC_MEM_CTRL__LMI_MC_LS_EN__SHIFT 0x00000000
+#define UVD_CGC_MEM_CTRL__LS_CLEAR_DELAY_MASK 0x00f00000L
+#define UVD_CGC_MEM_CTRL__LS_CLEAR_DELAY__SHIFT 0x00000014
+#define UVD_CGC_MEM_CTRL__LS_SET_DELAY_MASK 0x000f0000L
+#define UVD_CGC_MEM_CTRL__LS_SET_DELAY__SHIFT 0x00000010
+#define UVD_CGC_MEM_CTRL__MIF_LS_EN_MASK 0x00001000L
+#define UVD_CGC_MEM_CTRL__MIF_LS_EN__SHIFT 0x0000000c
+#define UVD_CGC_MEM_CTRL__MPC_LS_EN_MASK 0x00000002L
+#define UVD_CGC_MEM_CTRL__MPC_LS_EN__SHIFT 0x00000001
+#define UVD_CGC_MEM_CTRL__MPRD_LS_EN_MASK 0x00000004L
+#define UVD_CGC_MEM_CTRL__MPRD_LS_EN__SHIFT 0x00000002
+#define UVD_CGC_MEM_CTRL__SCPU_LS_EN_MASK 0x00000800L
+#define UVD_CGC_MEM_CTRL__SCPU_LS_EN__SHIFT 0x0000000b
+#define UVD_CGC_MEM_CTRL__SYS_LS_EN_MASK 0x00000200L
+#define UVD_CGC_MEM_CTRL__SYS_LS_EN__SHIFT 0x00000009
+#define UVD_CGC_MEM_CTRL__UDEC_CM_LS_EN_MASK 0x00000020L
+#define UVD_CGC_MEM_CTRL__UDEC_CM_LS_EN__SHIFT 0x00000005
+#define UVD_CGC_MEM_CTRL__UDEC_DB_LS_EN_MASK 0x00000080L
+#define UVD_CGC_MEM_CTRL__UDEC_DB_LS_EN__SHIFT 0x00000007
+#define UVD_CGC_MEM_CTRL__UDEC_IT_LS_EN_MASK 0x00000040L
+#define UVD_CGC_MEM_CTRL__UDEC_IT_LS_EN__SHIFT 0x00000006
+#define UVD_CGC_MEM_CTRL__UDEC_MP_LS_EN_MASK 0x00000100L
+#define UVD_CGC_MEM_CTRL__UDEC_MP_LS_EN__SHIFT 0x00000008
+#define UVD_CGC_MEM_CTRL__UDEC_RE_LS_EN_MASK 0x00000010L
+#define UVD_CGC_MEM_CTRL__UDEC_RE_LS_EN__SHIFT 0x00000004
+#define UVD_CGC_MEM_CTRL__VCPU_LS_EN_MASK 0x00000400L
+#define UVD_CGC_MEM_CTRL__VCPU_LS_EN__SHIFT 0x0000000a
+#define UVD_CGC_MEM_CTRL__WCB_LS_EN_MASK 0x00000008L
+#define UVD_CGC_MEM_CTRL__WCB_LS_EN__SHIFT 0x00000003
+#define UVD_CGC_STATUS__IDCT_SCLK_MASK 0x00004000L
+#define UVD_CGC_STATUS__IDCT_SCLK__SHIFT 0x0000000e
+#define UVD_CGC_STATUS__IDCT_VCLK_MASK 0x00008000L
+#define UVD_CGC_STATUS__IDCT_VCLK__SHIFT 0x0000000f
+#define UVD_CGC_STATUS__LBSI_SCLK_MASK 0x00200000L
+#define UVD_CGC_STATUS__LBSI_SCLK__SHIFT 0x00000015
+#define UVD_CGC_STATUS__LBSI_VCLK_MASK 0x00400000L
+#define UVD_CGC_STATUS__LBSI_VCLK__SHIFT 0x00000016
+#define UVD_CGC_STATUS__LMI_MC_SCLK_MASK 0x00001000L
+#define UVD_CGC_STATUS__LMI_MC_SCLK__SHIFT 0x0000000c
+#define UVD_CGC_STATUS__LMI_UMC_SCLK_MASK 0x00002000L
+#define UVD_CGC_STATUS__LMI_UMC_SCLK__SHIFT 0x0000000d
+#define UVD_CGC_STATUS__LRBBM_SCLK_MASK 0x00800000L
+#define UVD_CGC_STATUS__LRBBM_SCLK__SHIFT 0x00000017
+#define UVD_CGC_STATUS__MPC_DCLK_MASK 0x00100000L
+#define UVD_CGC_STATUS__MPC_DCLK__SHIFT 0x00000014
+#define UVD_CGC_STATUS__MPC_SCLK_MASK 0x00080000L
+#define UVD_CGC_STATUS__MPC_SCLK__SHIFT 0x00000013
+#define UVD_CGC_STATUS__MPEG2_DCLK_MASK 0x00000080L
+#define UVD_CGC_STATUS__MPEG2_DCLK__SHIFT 0x00000007
+#define UVD_CGC_STATUS__MPEG2_SCLK_MASK 0x00000040L
+#define UVD_CGC_STATUS__MPEG2_SCLK__SHIFT 0x00000006
+#define UVD_CGC_STATUS__MPEG2_VCLK_MASK 0x00000100L
+#define UVD_CGC_STATUS__MPEG2_VCLK__SHIFT 0x00000008
+#define UVD_CGC_STATUS__MPRD_DCLK_MASK 0x00020000L
+#define UVD_CGC_STATUS__MPRD_DCLK__SHIFT 0x00000011
+#define UVD_CGC_STATUS__MPRD_SCLK_MASK 0x00010000L
+#define UVD_CGC_STATUS__MPRD_SCLK__SHIFT 0x00000010
+#define UVD_CGC_STATUS__MPRD_VCLK_MASK 0x00040000L
+#define UVD_CGC_STATUS__MPRD_VCLK__SHIFT 0x00000012
+#define UVD_CGC_STATUS__RBC_SCLK_MASK 0x00000800L
+#define UVD_CGC_STATUS__RBC_SCLK__SHIFT 0x0000000b
+#define UVD_CGC_STATUS__REGS_SCLK_MASK 0x00000200L
+#define UVD_CGC_STATUS__REGS_SCLK__SHIFT 0x00000009
+#define UVD_CGC_STATUS__REGS_VCLK_MASK 0x00000400L
+#define UVD_CGC_STATUS__REGS_VCLK__SHIFT 0x0000000a
+#define UVD_CGC_STATUS__SCPU_SCLK_MASK 0x08000000L
+#define UVD_CGC_STATUS__SCPU_SCLK__SHIFT 0x0000001b
+#define UVD_CGC_STATUS__SCPU_VCLK_MASK 0x10000000L
+#define UVD_CGC_STATUS__SCPU_VCLK__SHIFT 0x0000001c
+#define UVD_CGC_STATUS__SYS_DCLK_MASK 0x00000002L
+#define UVD_CGC_STATUS__SYS_DCLK__SHIFT 0x00000001
+#define UVD_CGC_STATUS__SYS_SCLK_MASK 0x00000001L
+#define UVD_CGC_STATUS__SYS_SCLK__SHIFT 0x00000000
+#define UVD_CGC_STATUS__SYS_VCLK_MASK 0x00000004L
+#define UVD_CGC_STATUS__SYS_VCLK__SHIFT 0x00000002
+#define UVD_CGC_STATUS__UDEC_DCLK_MASK 0x00000010L
+#define UVD_CGC_STATUS__UDEC_DCLK__SHIFT 0x00000004
+#define UVD_CGC_STATUS__UDEC_SCLK_MASK 0x00000008L
+#define UVD_CGC_STATUS__UDEC_SCLK__SHIFT 0x00000003
+#define UVD_CGC_STATUS__UDEC_VCLK_MASK 0x00000020L
+#define UVD_CGC_STATUS__UDEC_VCLK__SHIFT 0x00000005
+#define UVD_CGC_STATUS__VCPU_SCLK_MASK 0x02000000L
+#define UVD_CGC_STATUS__VCPU_SCLK__SHIFT 0x00000019
+#define UVD_CGC_STATUS__VCPU_VCLK_MASK 0x04000000L
+#define UVD_CGC_STATUS__VCPU_VCLK__SHIFT 0x0000001a
+#define UVD_CGC_STATUS__WCB_SCLK_MASK 0x01000000L
+#define UVD_CGC_STATUS__WCB_SCLK__SHIFT 0x00000018
+#define UVD_CGC_UDEC_STATUS__CM_DCLK_MASK 0x00000010L
+#define UVD_CGC_UDEC_STATUS__CM_DCLK__SHIFT 0x00000004
+#define UVD_CGC_UDEC_STATUS__CM_SCLK_MASK 0x00000008L
+#define UVD_CGC_UDEC_STATUS__CM_SCLK__SHIFT 0x00000003
+#define UVD_CGC_UDEC_STATUS__CM_VCLK_MASK 0x00000020L
+#define UVD_CGC_UDEC_STATUS__CM_VCLK__SHIFT 0x00000005
+#define UVD_CGC_UDEC_STATUS__DB_DCLK_MASK 0x00000400L
+#define UVD_CGC_UDEC_STATUS__DB_DCLK__SHIFT 0x0000000a
+#define UVD_CGC_UDEC_STATUS__DB_SCLK_MASK 0x00000200L
+#define UVD_CGC_UDEC_STATUS__DB_SCLK__SHIFT 0x00000009
+#define UVD_CGC_UDEC_STATUS__DB_VCLK_MASK 0x00000800L
+#define UVD_CGC_UDEC_STATUS__DB_VCLK__SHIFT 0x0000000b
+#define UVD_CGC_UDEC_STATUS__IT_DCLK_MASK 0x00000080L
+#define UVD_CGC_UDEC_STATUS__IT_DCLK__SHIFT 0x00000007
+#define UVD_CGC_UDEC_STATUS__IT_SCLK_MASK 0x00000040L
+#define UVD_CGC_UDEC_STATUS__IT_SCLK__SHIFT 0x00000006
+#define UVD_CGC_UDEC_STATUS__IT_VCLK_MASK 0x00000100L
+#define UVD_CGC_UDEC_STATUS__IT_VCLK__SHIFT 0x00000008
+#define UVD_CGC_UDEC_STATUS__MP_DCLK_MASK 0x00002000L
+#define UVD_CGC_UDEC_STATUS__MP_DCLK__SHIFT 0x0000000d
+#define UVD_CGC_UDEC_STATUS__MP_SCLK_MASK 0x00001000L
+#define UVD_CGC_UDEC_STATUS__MP_SCLK__SHIFT 0x0000000c
+#define UVD_CGC_UDEC_STATUS__MP_VCLK_MASK 0x00004000L
+#define UVD_CGC_UDEC_STATUS__MP_VCLK__SHIFT 0x0000000e
+#define UVD_CGC_UDEC_STATUS__RE_DCLK_MASK 0x00000002L
+#define UVD_CGC_UDEC_STATUS__RE_DCLK__SHIFT 0x00000001
+#define UVD_CGC_UDEC_STATUS__RE_SCLK_MASK 0x00000001L
+#define UVD_CGC_UDEC_STATUS__RE_SCLK__SHIFT 0x00000000
+#define UVD_CGC_UDEC_STATUS__RE_VCLK_MASK 0x00000004L
+#define UVD_CGC_UDEC_STATUS__RE_VCLK__SHIFT 0x00000002
+#define UVD_CONTEXT_ID__CONTEXT_ID_MASK 0xffffffffL
+#define UVD_CONTEXT_ID__CONTEXT_ID__SHIFT 0x00000000
+#define UVD_CTX_DATA__DATA_MASK 0xffffffffL
+#define UVD_CTX_DATA__DATA__SHIFT 0x00000000
+#define UVD_CTX_INDEX__INDEX_MASK 0x000001ffL
+#define UVD_CTX_INDEX__INDEX__SHIFT 0x00000000
+#define UVD_ENGINE_CNTL__ENGINE_START_MASK 0x00000001L
+#define UVD_ENGINE_CNTL__ENGINE_START_MODE_MASK 0x00000002L
+#define UVD_ENGINE_CNTL__ENGINE_START_MODE__SHIFT 0x00000001
+#define UVD_ENGINE_CNTL__ENGINE_START__SHIFT 0x00000000
+#define UVD_GPCOM_VCPU_CMD__CMD_MASK 0x7ffffffeL
+#define UVD_GPCOM_VCPU_CMD__CMD_SEND_MASK 0x00000001L
+#define UVD_GPCOM_VCPU_CMD__CMD_SEND__SHIFT 0x00000000
+#define UVD_GPCOM_VCPU_CMD__CMD__SHIFT 0x00000001
+#define UVD_GPCOM_VCPU_CMD__CMD_SOURCE_MASK 0x80000000L
+#define UVD_GPCOM_VCPU_CMD__CMD_SOURCE__SHIFT 0x0000001f
+#define UVD_GPCOM_VCPU_DATA0__DATA0_MASK 0xffffffffL
+#define UVD_GPCOM_VCPU_DATA0__DATA0__SHIFT 0x00000000
+#define UVD_GPCOM_VCPU_DATA1__DATA1_MASK 0xffffffffL
+#define UVD_GPCOM_VCPU_DATA1__DATA1__SHIFT 0x00000000
+#define UVD_LMI_ADDR_EXT2__SCPU_ADDR_EXT_MASK 0x0000000fL
+#define UVD_LMI_ADDR_EXT2__SCPU_ADDR_EXT__SHIFT 0x00000000
+#define UVD_LMI_ADDR_EXT2__SCPU_NC0_ADDR_EXT_MASK 0x00000f00L
+#define UVD_LMI_ADDR_EXT2__SCPU_NC0_ADDR_EXT__SHIFT 0x00000008
+#define UVD_LMI_ADDR_EXT2__SCPU_NC1_ADDR_EXT_MASK 0x0000f000L
+#define UVD_LMI_ADDR_EXT2__SCPU_NC1_ADDR_EXT__SHIFT 0x0000000c
+#define UVD_LMI_ADDR_EXT2__SCPU_VM_ADDR_EXT_MASK 0x000000f0L
+#define UVD_LMI_ADDR_EXT2__SCPU_VM_ADDR_EXT__SHIFT 0x00000004
+#define UVD_LMI_ADDR_EXT__CM_ADDR_EXT_MASK 0x000000f0L
+#define UVD_LMI_ADDR_EXT__CM_ADDR_EXT__SHIFT 0x00000004
+#define UVD_LMI_ADDR_EXT__IT_ADDR_EXT_MASK 0x00000f00L
+#define UVD_LMI_ADDR_EXT__IT_ADDR_EXT__SHIFT 0x00000008
+#define UVD_LMI_ADDR_EXT__MP_ADDR_EXT_MASK 0x00f00000L
+#define UVD_LMI_ADDR_EXT__MP_ADDR_EXT__SHIFT 0x00000014
+#define UVD_LMI_ADDR_EXT__RE_ADDR_EXT_MASK 0x000f0000L
+#define UVD_LMI_ADDR_EXT__RE_ADDR_EXT__SHIFT 0x00000010
+#define UVD_LMI_ADDR_EXT__VCPU_ADDR_EXT_MASK 0x0000000fL
+#define UVD_LMI_ADDR_EXT__VCPU_ADDR_EXT__SHIFT 0x00000000
+#define UVD_LMI_ADDR_EXT__VCPU_NC0_ADDR_EXT_MASK 0x0f000000L
+#define UVD_LMI_ADDR_EXT__VCPU_NC0_ADDR_EXT__SHIFT 0x00000018
+#define UVD_LMI_ADDR_EXT__VCPU_NC1_ADDR_EXT_MASK 0xf0000000L
+#define UVD_LMI_ADDR_EXT__VCPU_NC1_ADDR_EXT__SHIFT 0x0000001c
+#define UVD_LMI_ADDR_EXT__VCPU_VM_ADDR_EXT_MASK 0x0000f000L
+#define UVD_LMI_ADDR_EXT__VCPU_VM_ADDR_EXT__SHIFT 0x0000000c
+#define UVD_LMI_CACHE_CTRL__CM_EN_MASK 0x00000004L
+#define UVD_LMI_CACHE_CTRL__CM_EN__SHIFT 0x00000002
+#define UVD_LMI_CACHE_CTRL__CM_FLUSH_MASK 0x00000008L
+#define UVD_LMI_CACHE_CTRL__CM_FLUSH__SHIFT 0x00000003
+#define UVD_LMI_CACHE_CTRL__IT_EN_MASK 0x00000001L
+#define UVD_LMI_CACHE_CTRL__IT_EN__SHIFT 0x00000000
+#define UVD_LMI_CACHE_CTRL__IT_FLUSH_MASK 0x00000002L
+#define UVD_LMI_CACHE_CTRL__IT_FLUSH__SHIFT 0x00000001
+#define UVD_LMI_CACHE_CTRL__VCPU_EN_MASK 0x00000010L
+#define UVD_LMI_CACHE_CTRL__VCPU_EN__SHIFT 0x00000004
+#define UVD_LMI_CACHE_CTRL__VCPU_FLUSH_MASK 0x00000020L
+#define UVD_LMI_CACHE_CTRL__VCPU_FLUSH__SHIFT 0x00000005
+#define UVD_LMI_CTRL2__ASSERT_UMC_URGENT_MASK 0x00000004L
+#define UVD_LMI_CTRL2__ASSERT_UMC_URGENT__SHIFT 0x00000002
+#define UVD_LMI_CTRL2__DRCITF_BUBBLE_FIX_DIS_MASK 0x00000080L
+#define UVD_LMI_CTRL2__DRCITF_BUBBLE_FIX_DIS__SHIFT 0x00000007
+#define UVD_LMI_CTRL2__MASK_UMC_URGENT_MASK 0x00000008L
+#define UVD_LMI_CTRL2__MASK_UMC_URGENT__SHIFT 0x00000003
+#define UVD_LMI_CTRL2__MCIF_WR_WATERMARK_MASK 0x00000070L
+#define UVD_LMI_CTRL2__MCIF_WR_WATERMARK__SHIFT 0x00000004
+#define UVD_LMI_CTRL2__MC_READ_ID_SEL_MASK 0x00000600L
+#define UVD_LMI_CTRL2__MC_READ_ID_SEL__SHIFT 0x00000009
+#define UVD_LMI_CTRL2__MC_WRITE_ID_SEL_MASK 0x00001800L
+#define UVD_LMI_CTRL2__MC_WRITE_ID_SEL__SHIFT 0x0000000b
+#define UVD_LMI_CTRL2__SPH_DIS_MASK 0x00000001L
+#define UVD_LMI_CTRL2__SPH_DIS__SHIFT 0x00000000
+#define UVD_LMI_CTRL2__SPU_EXTRA_CID_EN_MASK 0x00008000L
+#define UVD_LMI_CTRL2__SPU_EXTRA_CID_EN__SHIFT 0x0000000f
+#define UVD_LMI_CTRL2__STALL_ARB_MASK 0x00000002L
+#define UVD_LMI_CTRL2__STALL_ARB__SHIFT 0x00000001
+#define UVD_LMI_CTRL2__STALL_ARB_UMC_MASK 0x00000100L
+#define UVD_LMI_CTRL2__STALL_ARB_UMC__SHIFT 0x00000008
+#define UVD_LMI_CTRL2__VCPU_NC0_EXT_EN_MASK 0x00002000L
+#define UVD_LMI_CTRL2__VCPU_NC0_EXT_EN__SHIFT 0x0000000d
+#define UVD_LMI_CTRL2__VCPU_NC1_EXT_EN_MASK 0x00004000L
+#define UVD_LMI_CTRL2__VCPU_NC1_EXT_EN__SHIFT 0x0000000e
+#define UVD_LMI_CTRL__ASSERT_MC_URGENT_MASK 0x00000800L
+#define UVD_LMI_CTRL__ASSERT_MC_URGENT__SHIFT 0x0000000b
+#define UVD_LMI_CTRL__CM_DATA_COHERENCY_EN_MASK 0x00400000L
+#define UVD_LMI_CTRL__CM_DATA_COHERENCY_EN__SHIFT 0x00000016
+#define UVD_LMI_CTRL__CRC_RESET_MASK 0x00004000L
+#define UVD_LMI_CTRL__CRC_RESET__SHIFT 0x0000000e
+#define UVD_LMI_CTRL__CRC_SEL_MASK 0x000f8000L
+#define UVD_LMI_CTRL__CRC_SEL__SHIFT 0x0000000f
+#define UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK 0x00002000L
+#define UVD_LMI_CTRL__DATA_COHERENCY_EN__SHIFT 0x0000000d
+#define UVD_LMI_CTRL__DB_DB_DATA_COHERENCY_EN_MASK 0x00800000L
+#define UVD_LMI_CTRL__DB_DB_DATA_COHERENCY_EN__SHIFT 0x00000017
+#define UVD_LMI_CTRL__DB_IT_DATA_COHERENCY_EN_MASK 0x01000000L
+#define UVD_LMI_CTRL__DB_IT_DATA_COHERENCY_EN__SHIFT 0x00000018
+#define UVD_LMI_CTRL__DISABLE_ON_FWV_FAIL_MASK 0x00100000L
+#define UVD_LMI_CTRL__DISABLE_ON_FWV_FAIL__SHIFT 0x00000014
+#define UVD_LMI_CTRL__IT_IT_DATA_COHERENCY_EN_MASK 0x02000000L
+#define UVD_LMI_CTRL__IT_IT_DATA_COHERENCY_EN__SHIFT 0x00000019
+#define UVD_LMI_CTRL__MASK_MC_URGENT_MASK 0x00001000L
+#define UVD_LMI_CTRL__MASK_MC_URGENT__SHIFT 0x0000000c
+#define UVD_LMI_CTRL__MIF_MIF_DATA_COHERENCY_EN_MASK 0x04000000L
+#define UVD_LMI_CTRL__MIF_MIF_DATA_COHERENCY_EN__SHIFT 0x0000001a
+#define UVD_LMI_CTRL__REQ_MODE_MASK 0x00000200L
+#define UVD_LMI_CTRL__REQ_MODE__SHIFT 0x00000009
+#define UVD_LMI_CTRL__RFU_MASK 0xf8000000L
+#define UVD_LMI_CTRL__RFU_MASK 0xfc000000L
+#define UVD_LMI_CTRL__RFU__SHIFT 0x0000001a
+#define UVD_LMI_CTRL__RFU__SHIFT 0x0000001b
+#define UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK 0x00200000L
+#define UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN__SHIFT 0x00000015
+#define UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK 0x00000100L
+#define UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN__SHIFT 0x00000008
+#define UVD_LMI_CTRL__WRITE_CLEAN_TIMER_MASK 0x000000ffL
+#define UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT 0x00000000
+#define UVD_LMI_EXT40_ADDR__ADDR_MASK 0x000000ffL
+#define UVD_LMI_EXT40_ADDR__ADDR__SHIFT 0x00000000
+#define UVD_LMI_EXT40_ADDR__INDEX_MASK 0x001f0000L
+#define UVD_LMI_EXT40_ADDR__INDEX__SHIFT 0x00000010
+#define UVD_LMI_EXT40_ADDR__WRITE_ADDR_MASK 0x80000000L
+#define UVD_LMI_EXT40_ADDR__WRITE_ADDR__SHIFT 0x0000001f
+#define UVD_LMI_STATUS__ADP_MC_READ_CLEAN_MASK 0x00001000L
+#define UVD_LMI_STATUS__ADP_MC_READ_CLEAN__SHIFT 0x0000000c
+#define UVD_LMI_STATUS__ADP_UMC_READ_CLEAN_MASK 0x00002000L
+#define UVD_LMI_STATUS__ADP_UMC_READ_CLEAN__SHIFT 0x0000000d
+#define UVD_LMI_STATUS__PENDING_UVD_MC_WRITE_MASK 0x00000080L
+#define UVD_LMI_STATUS__PENDING_UVD_MC_WRITE__SHIFT 0x00000007
+#define UVD_LMI_STATUS__READ_CLEAN_MASK 0x00000001L
+#define UVD_LMI_STATUS__READ_CLEAN_RAW_MASK 0x00000100L
+#define UVD_LMI_STATUS__READ_CLEAN_RAW__SHIFT 0x00000008
+#define UVD_LMI_STATUS__READ_CLEAN__SHIFT 0x00000000
+#define UVD_LMI_STATUS__UMC_AVP_IDLE_MASK 0x00000800L
+#define UVD_LMI_STATUS__UMC_AVP_IDLE__SHIFT 0x0000000b
+#define UVD_LMI_STATUS__UMC_READ_CLEAN_MASK 0x00000010L
+#define UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK 0x00000200L
+#define UVD_LMI_STATUS__UMC_READ_CLEAN_RAW__SHIFT 0x00000009
+#define UVD_LMI_STATUS__UMC_READ_CLEAN__SHIFT 0x00000004
+#define UVD_LMI_STATUS__UMC_UVD_IDLE_MASK 0x00000400L
+#define UVD_LMI_STATUS__UMC_UVD_IDLE__SHIFT 0x0000000a
+#define UVD_LMI_STATUS__UMC_WRITE_CLEAN_MASK 0x00000020L
+#define UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK 0x00000040L
+#define UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW__SHIFT 0x00000006
+#define UVD_LMI_STATUS__UMC_WRITE_CLEAN__SHIFT 0x00000005
+#define UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK 0x00000008L
+#define UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN__SHIFT 0x00000003
+#define UVD_LMI_STATUS__WRITE_CLEAN_MASK 0x00000002L
+#define UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK 0x00000004L
+#define UVD_LMI_STATUS__WRITE_CLEAN_RAW__SHIFT 0x00000002
+#define UVD_LMI_STATUS__WRITE_CLEAN__SHIFT 0x00000001
+#define UVD_LMI_SWAP_CNTL2__SCPU_R_MC_SWAP_MASK 0x00000003L
+#define UVD_LMI_SWAP_CNTL2__SCPU_R_MC_SWAP__SHIFT 0x00000000
+#define UVD_LMI_SWAP_CNTL2__SCPU_W_MC_SWAP_MASK 0x0000000cL
+#define UVD_LMI_SWAP_CNTL2__SCPU_W_MC_SWAP__SHIFT 0x00000002
+#define UVD_LMI_SWAP_CNTL__CM_MC_SWAP_MASK 0x00000c00L
+#define UVD_LMI_SWAP_CNTL__CM_MC_SWAP__SHIFT 0x0000000a
+#define UVD_LMI_SWAP_CNTL__CSM_MC_SWAP_MASK 0x000c0000L
+#define UVD_LMI_SWAP_CNTL__CSM_MC_SWAP__SHIFT 0x00000012
+#define UVD_LMI_SWAP_CNTL__DB_R_MC_SWAP_MASK 0x0000c000L
+#define UVD_LMI_SWAP_CNTL__DB_R_MC_SWAP__SHIFT 0x0000000e
+#define UVD_LMI_SWAP_CNTL__DB_W_MC_SWAP_MASK 0x00030000L
+#define UVD_LMI_SWAP_CNTL__DBW_MC_SWAP_MASK 0x03000000L
+#define UVD_LMI_SWAP_CNTL__DB_W_MC_SWAP__SHIFT 0x00000010
+#define UVD_LMI_SWAP_CNTL__DBW_MC_SWAP__SHIFT 0x00000018
+#define UVD_LMI_SWAP_CNTL__IB_MC_SWAP_MASK 0x0000000cL
+#define UVD_LMI_SWAP_CNTL__IB_MC_SWAP__SHIFT 0x00000002
+#define UVD_LMI_SWAP_CNTL__IT_MC_SWAP_MASK 0x00003000L
+#define UVD_LMI_SWAP_CNTL__IT_MC_SWAP__SHIFT 0x0000000c
+#define UVD_LMI_SWAP_CNTL__MP_MC_SWAP_MASK 0xc0000000L
+#define UVD_LMI_SWAP_CNTL__MP_MC_SWAP__SHIFT 0x0000001e
+#define UVD_LMI_SWAP_CNTL__MP_REF16_MC_SWAP_MASK 0x00c00000L
+#define UVD_LMI_SWAP_CNTL__MP_REF16_MC_SWAP__SHIFT 0x00000016
+#define UVD_LMI_SWAP_CNTL__RB_MC_SWAP_MASK 0x00000003L
+#define UVD_LMI_SWAP_CNTL__RB_MC_SWAP__SHIFT 0x00000000
+#define UVD_LMI_SWAP_CNTL__RB_RPTR_MC_SWAP_MASK 0x00000030L
+#define UVD_LMI_SWAP_CNTL__RB_RPTR_MC_SWAP__SHIFT 0x00000004
+#define UVD_LMI_SWAP_CNTL__RB_WR_MC_SWAP_MASK 0x0c000000L
+#define UVD_LMI_SWAP_CNTL__RB_WR_MC_SWAP__SHIFT 0x0000001a
+#define UVD_LMI_SWAP_CNTL__RE_MC_SWAP_MASK 0x30000000L
+#define UVD_LMI_SWAP_CNTL__RE_MC_SWAP__SHIFT 0x0000001c
+#define UVD_LMI_SWAP_CNTL__VCPU_R_MC_SWAP_MASK 0x000000c0L
+#define UVD_LMI_SWAP_CNTL__VCPU_R_MC_SWAP__SHIFT 0x00000006
+#define UVD_LMI_SWAP_CNTL__VCPU_W_MC_SWAP_MASK 0x00000300L
+#define UVD_LMI_SWAP_CNTL__VCPU_W_MC_SWAP__SHIFT 0x00000008
+#define UVD_MASTINT_EN__INT_OVERRUN_MASK 0x007ffff0L
+#define UVD_MASTINT_EN__INT_OVERRUN__SHIFT 0x00000004
+#define UVD_MASTINT_EN__OVERRUN_RST_MASK 0x00000001L
+#define UVD_MASTINT_EN__OVERRUN_RST__SHIFT 0x00000000
+#define UVD_MASTINT_EN__SYS_EN_MASK 0x00000004L
+#define UVD_MASTINT_EN__SYS_EN__SHIFT 0x00000002
+#define UVD_MASTINT_EN__VCPU_EN_MASK 0x00000002L
+#define UVD_MASTINT_EN__VCPU_EN__SHIFT 0x00000001
+#define UVD_MIF_CURR_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK 0x00000700L
+#define UVD_MIF_CURR_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT 0x00000008
+#define UVD_MIF_CURR_ADDR_CONFIG__MULTI_GPU_TILE_SIZE_MASK 0x03000000L
+#define UVD_MIF_CURR_ADDR_CONFIG__MULTI_GPU_TILE_SIZE__SHIFT 0x00000018
+#define UVD_MIF_CURR_ADDR_CONFIG__NUM_GPUS_MASK 0x00700000L
+#define UVD_MIF_CURR_ADDR_CONFIG__NUM_GPUS__SHIFT 0x00000014
+#define UVD_MIF_CURR_ADDR_CONFIG__NUM_LOWER_PIPES_MASK 0x40000000L
+#define UVD_MIF_CURR_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT 0x0000001e
+#define UVD_MIF_CURR_ADDR_CONFIG__NUM_PIPES_MASK 0x00000007L
+#define UVD_MIF_CURR_ADDR_CONFIG__NUM_PIPES__SHIFT 0x00000000
+#define UVD_MIF_CURR_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK 0x00003000L
+#define UVD_MIF_CURR_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT 0x0000000c
+#define UVD_MIF_CURR_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK 0x00000070L
+#define UVD_MIF_CURR_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT 0x00000004
+#define UVD_MIF_CURR_ADDR_CONFIG__ROW_SIZE_MASK 0x30000000L
+#define UVD_MIF_CURR_ADDR_CONFIG__ROW_SIZE__SHIFT 0x0000001c
+#define UVD_MIF_CURR_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK 0x00070000L
+#define UVD_MIF_CURR_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT 0x00000010
+#define UVD_MIF_RECON1_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK 0x00000700L
+#define UVD_MIF_RECON1_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT 0x00000008
+#define UVD_MIF_RECON1_ADDR_CONFIG__MULTI_GPU_TILE_SIZE_MASK 0x03000000L
+#define UVD_MIF_RECON1_ADDR_CONFIG__MULTI_GPU_TILE_SIZE__SHIFT 0x00000018
+#define UVD_MIF_RECON1_ADDR_CONFIG__NUM_GPUS_MASK 0x00700000L
+#define UVD_MIF_RECON1_ADDR_CONFIG__NUM_GPUS__SHIFT 0x00000014
+#define UVD_MIF_RECON1_ADDR_CONFIG__NUM_LOWER_PIPES_MASK 0x40000000L
+#define UVD_MIF_RECON1_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT 0x0000001e
+#define UVD_MIF_RECON1_ADDR_CONFIG__NUM_PIPES_MASK 0x00000007L
+#define UVD_MIF_RECON1_ADDR_CONFIG__NUM_PIPES__SHIFT 0x00000000
+#define UVD_MIF_RECON1_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK 0x00003000L
+#define UVD_MIF_RECON1_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT 0x0000000c
+#define UVD_MIF_RECON1_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK 0x00000070L
+#define UVD_MIF_RECON1_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT 0x00000004
+#define UVD_MIF_RECON1_ADDR_CONFIG__ROW_SIZE_MASK 0x30000000L
+#define UVD_MIF_RECON1_ADDR_CONFIG__ROW_SIZE__SHIFT 0x0000001c
+#define UVD_MIF_RECON1_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK 0x00070000L
+#define UVD_MIF_RECON1_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT 0x00000010
+#define UVD_MIF_REF_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK 0x00000700L
+#define UVD_MIF_REF_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT 0x00000008
+#define UVD_MIF_REF_ADDR_CONFIG__MULTI_GPU_TILE_SIZE_MASK 0x03000000L
+#define UVD_MIF_REF_ADDR_CONFIG__MULTI_GPU_TILE_SIZE__SHIFT 0x00000018
+#define UVD_MIF_REF_ADDR_CONFIG__NUM_GPUS_MASK 0x00700000L
+#define UVD_MIF_REF_ADDR_CONFIG__NUM_GPUS__SHIFT 0x00000014
+#define UVD_MIF_REF_ADDR_CONFIG__NUM_LOWER_PIPES_MASK 0x40000000L
+#define UVD_MIF_REF_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT 0x0000001e
+#define UVD_MIF_REF_ADDR_CONFIG__NUM_PIPES_MASK 0x00000007L
+#define UVD_MIF_REF_ADDR_CONFIG__NUM_PIPES__SHIFT 0x00000000
+#define UVD_MIF_REF_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK 0x00003000L
+#define UVD_MIF_REF_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT 0x0000000c
+#define UVD_MIF_REF_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK 0x00000070L
+#define UVD_MIF_REF_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT 0x00000004
+#define UVD_MIF_REF_ADDR_CONFIG__ROW_SIZE_MASK 0x30000000L
+#define UVD_MIF_REF_ADDR_CONFIG__ROW_SIZE__SHIFT 0x0000001c
+#define UVD_MIF_REF_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK 0x00070000L
+#define UVD_MIF_REF_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT 0x00000010
+#define UVD_MPC_CNTL__AVE_WEIGHT_MASK 0x00030000L
+#define UVD_MPC_CNTL__AVE_WEIGHT__SHIFT 0x00000010
+#define UVD_MPC_CNTL__DBG_MUX_MASK 0x00000700L
+#define UVD_MPC_CNTL__DBG_MUX__SHIFT 0x00000008
+#define UVD_MPC_CNTL__PERF_RST_MASK 0x00000040L
+#define UVD_MPC_CNTL__PERF_RST__SHIFT 0x00000006
+#define UVD_MPC_CNTL__REPLACEMENT_MODE_MASK 0x00000038L
+#define UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT 0x00000003
+#define UVD_MPC_CNTL__URGENT_EN_MASK 0x00040000L
+#define UVD_MPC_CNTL__URGENT_EN__SHIFT 0x00000012
+#define UVD_MPC_SET_ALU__FUNCT_MASK 0x00000007L
+#define UVD_MPC_SET_ALU__FUNCT__SHIFT 0x00000000
+#define UVD_MPC_SET_ALU__OPERAND_MASK 0x00000ff0L
+#define UVD_MPC_SET_ALU__OPERAND__SHIFT 0x00000004
+#define UVD_MPC_SET_MUXA0__VARA_0_MASK 0x0000003fL
+#define UVD_MPC_SET_MUXA0__VARA_0__SHIFT 0x00000000
+#define UVD_MPC_SET_MUXA0__VARA_1_MASK 0x00000fc0L
+#define UVD_MPC_SET_MUXA0__VARA_1__SHIFT 0x00000006
+#define UVD_MPC_SET_MUXA0__VARA_2_MASK 0x0003f000L
+#define UVD_MPC_SET_MUXA0__VARA_2__SHIFT 0x0000000c
+#define UVD_MPC_SET_MUXA0__VARA_3_MASK 0x00fc0000L
+#define UVD_MPC_SET_MUXA0__VARA_3__SHIFT 0x00000012
+#define UVD_MPC_SET_MUXA0__VARA_4_MASK 0x3f000000L
+#define UVD_MPC_SET_MUXA0__VARA_4__SHIFT 0x00000018
+#define UVD_MPC_SET_MUXA1__VARA_5_MASK 0x0000003fL
+#define UVD_MPC_SET_MUXA1__VARA_5__SHIFT 0x00000000
+#define UVD_MPC_SET_MUXA1__VARA_6_MASK 0x00000fc0L
+#define UVD_MPC_SET_MUXA1__VARA_6__SHIFT 0x00000006
+#define UVD_MPC_SET_MUXA1__VARA_7_MASK 0x0003f000L
+#define UVD_MPC_SET_MUXA1__VARA_7__SHIFT 0x0000000c
+#define UVD_MPC_SET_MUXB0__VARB_0_MASK 0x0000003fL
+#define UVD_MPC_SET_MUXB0__VARB_0__SHIFT 0x00000000
+#define UVD_MPC_SET_MUXB0__VARB_1_MASK 0x00000fc0L
+#define UVD_MPC_SET_MUXB0__VARB_1__SHIFT 0x00000006
+#define UVD_MPC_SET_MUXB0__VARB_2_MASK 0x0003f000L
+#define UVD_MPC_SET_MUXB0__VARB_2__SHIFT 0x0000000c
+#define UVD_MPC_SET_MUXB0__VARB_3_MASK 0x00fc0000L
+#define UVD_MPC_SET_MUXB0__VARB_3__SHIFT 0x00000012
+#define UVD_MPC_SET_MUXB0__VARB_4_MASK 0x3f000000L
+#define UVD_MPC_SET_MUXB0__VARB_4__SHIFT 0x00000018
+#define UVD_MPC_SET_MUXB1__VARB_5_MASK 0x0000003fL
+#define UVD_MPC_SET_MUXB1__VARB_5__SHIFT 0x00000000
+#define UVD_MPC_SET_MUXB1__VARB_6_MASK 0x00000fc0L
+#define UVD_MPC_SET_MUXB1__VARB_6__SHIFT 0x00000006
+#define UVD_MPC_SET_MUXB1__VARB_7_MASK 0x0003f000L
+#define UVD_MPC_SET_MUXB1__VARB_7__SHIFT 0x0000000c
+#define UVD_MPC_SET_MUX__SET_0_MASK 0x00000007L
+#define UVD_MPC_SET_MUX__SET_0__SHIFT 0x00000000
+#define UVD_MPC_SET_MUX__SET_1_MASK 0x00000038L
+#define UVD_MPC_SET_MUX__SET_1__SHIFT 0x00000003
+#define UVD_MPC_SET_MUX__SET_2_MASK 0x000001c0L
+#define UVD_MPC_SET_MUX__SET_2__SHIFT 0x00000006
+#define UVD_MP_SWAP_CNTL__MP_REF0_MC_SWAP_MASK 0x00000003L
+#define UVD_MP_SWAP_CNTL__MP_REF0_MC_SWAP__SHIFT 0x00000000
+#define UVD_MP_SWAP_CNTL__MP_REF10_MC_SWAP_MASK 0x00300000L
+#define UVD_MP_SWAP_CNTL__MP_REF10_MC_SWAP__SHIFT 0x00000014
+#define UVD_MP_SWAP_CNTL__MP_REF11_MC_SWAP_MASK 0x00c00000L
+#define UVD_MP_SWAP_CNTL__MP_REF11_MC_SWAP__SHIFT 0x00000016
+#define UVD_MP_SWAP_CNTL__MP_REF12_MC_SWAP_MASK 0x03000000L
+#define UVD_MP_SWAP_CNTL__MP_REF12_MC_SWAP__SHIFT 0x00000018
+#define UVD_MP_SWAP_CNTL__MP_REF13_MC_SWAP_MASK 0x0c000000L
+#define UVD_MP_SWAP_CNTL__MP_REF13_MC_SWAP__SHIFT 0x0000001a
+#define UVD_MP_SWAP_CNTL__MP_REF14_MC_SWAP_MASK 0x30000000L
+#define UVD_MP_SWAP_CNTL__MP_REF14_MC_SWAP__SHIFT 0x0000001c
+#define UVD_MP_SWAP_CNTL__MP_REF15_MC_SWAP_MASK 0xc0000000L
+#define UVD_MP_SWAP_CNTL__MP_REF15_MC_SWAP__SHIFT 0x0000001e
+#define UVD_MP_SWAP_CNTL__MP_REF1_MC_SWAP_MASK 0x0000000cL
+#define UVD_MP_SWAP_CNTL__MP_REF1_MC_SWAP__SHIFT 0x00000002
+#define UVD_MP_SWAP_CNTL__MP_REF2_MC_SWAP_MASK 0x00000030L
+#define UVD_MP_SWAP_CNTL__MP_REF2_MC_SWAP__SHIFT 0x00000004
+#define UVD_MP_SWAP_CNTL__MP_REF3_MC_SWAP_MASK 0x000000c0L
+#define UVD_MP_SWAP_CNTL__MP_REF3_MC_SWAP__SHIFT 0x00000006
+#define UVD_MP_SWAP_CNTL__MP_REF4_MC_SWAP_MASK 0x00000300L
+#define UVD_MP_SWAP_CNTL__MP_REF4_MC_SWAP__SHIFT 0x00000008
+#define UVD_MP_SWAP_CNTL__MP_REF5_MC_SWAP_MASK 0x00000c00L
+#define UVD_MP_SWAP_CNTL__MP_REF5_MC_SWAP__SHIFT 0x0000000a
+#define UVD_MP_SWAP_CNTL__MP_REF6_MC_SWAP_MASK 0x00003000L
+#define UVD_MP_SWAP_CNTL__MP_REF6_MC_SWAP__SHIFT 0x0000000c
+#define UVD_MP_SWAP_CNTL__MP_REF7_MC_SWAP_MASK 0x0000c000L
+#define UVD_MP_SWAP_CNTL__MP_REF7_MC_SWAP__SHIFT 0x0000000e
+#define UVD_MP_SWAP_CNTL__MP_REF8_MC_SWAP_MASK 0x00030000L
+#define UVD_MP_SWAP_CNTL__MP_REF8_MC_SWAP__SHIFT 0x00000010
+#define UVD_MP_SWAP_CNTL__MP_REF9_MC_SWAP_MASK 0x000c0000L
+#define UVD_MP_SWAP_CNTL__MP_REF9_MC_SWAP__SHIFT 0x00000012
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_FSM_ADDR_MASK 0x000000ffL
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_FSM_ADDR__SHIFT 0x00000000
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_P1_SELECT_MASK 0x00000400L
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_P1_SELECT__SHIFT 0x0000000a
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_P2_SELECT_MASK 0x00000800L
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_P2_SELECT__SHIFT 0x0000000b
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_POWER_DOWN_MASK 0x00000100L
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_POWER_DOWN__SHIFT 0x00000008
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_POWER_UP_MASK 0x00000200L
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_POWER_UP__SHIFT 0x00000009
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_READ_MASK 0x00002000L
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_READ__SHIFT 0x0000000d
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_REG_ADDR_MASK 0xf0000000L
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_REG_ADDR__SHIFT 0x0000001c
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_WRITE_MASK 0x00001000L
+#define UVD_PGFSM_CONFIG__UVD_PGFSM_WRITE__SHIFT 0x0000000c
+#define UVD_PGFSM_READ_TILE1__UVD_PGFSM_READ_TILE1_VALUE_MASK 0x00ffffffL
+#define UVD_PGFSM_READ_TILE1__UVD_PGFSM_READ_TILE1_VALUE__SHIFT 0x00000000
+#define UVD_PGFSM_READ_TILE2__UVD_PGFSM_READ_TILE2_VALUE_MASK 0x00ffffffL
+#define UVD_PGFSM_READ_TILE2__UVD_PGFSM_READ_TILE2_VALUE__SHIFT 0x00000000
+#define UVD_POWER_STATUS__UVD_POWER_STATUS_MASK 0x00000001L
+#define UVD_POWER_STATUS__UVD_POWER_STATUS__SHIFT 0x00000000
+#define UVD_RBC_IB_BASE__IB_BASE_MASK 0xffffffc0L
+#define UVD_RBC_IB_BASE__IB_BASE__SHIFT 0x00000006
+#define UVD_RBC_IB_SIZE__IB_SIZE_MASK 0x007ffff0L
+#define UVD_RBC_IB_SIZE__IB_SIZE__SHIFT 0x00000004
+#define UVD_RBC_RB_BASE__RB_BASE_MASK 0xffffffc0L
+#define UVD_RBC_RB_BASE__RB_BASE__SHIFT 0x00000006
+#define UVD_RBC_RB_CNTL__RB_BLKSZ_MASK 0x00001f00L
+#define UVD_RBC_RB_CNTL__RB_BLKSZ__SHIFT 0x00000008
+#define UVD_RBC_RB_CNTL__RB_BUFSZ_MASK 0x0000001fL
+#define UVD_RBC_RB_CNTL__RB_BUFSZ__SHIFT 0x00000000
+#define UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK 0x00010000L
+#define UVD_RBC_RB_CNTL__RB_NO_FETCH__SHIFT 0x00000010
+#define UVD_RBC_RB_CNTL__RB_NO_UPDATE_MASK 0x01000000L
+#define UVD_RBC_RB_CNTL__RB_NO_UPDATE__SHIFT 0x00000018
+#define UVD_RBC_RB_CNTL__RB_RPTR_WR_EN_MASK 0x10000000L
+#define UVD_RBC_RB_CNTL__RB_RPTR_WR_EN__SHIFT 0x0000001c
+#define UVD_RBC_RB_CNTL__RB_WPTR_POLL_EN_MASK 0x00100000L
+#define UVD_RBC_RB_CNTL__RB_WPTR_POLL_EN__SHIFT 0x00000014
+#define UVD_RBC_RB_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xffffffffL
+#define UVD_RBC_RB_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x00000000
+#define UVD_RBC_RB_RPTR__RB_RPTR_MASK 0x007ffff0L
+#define UVD_RBC_RB_RPTR__RB_RPTR__SHIFT 0x00000004
+#define UVD_RBC_RB_WPTR__RB_WPTR_MASK 0x007ffff0L
+#define UVD_RBC_RB_WPTR__RB_WPTR__SHIFT 0x00000004
+#define UVD_SEMA_ADDR_HIGH__ADDR_42_23_MASK 0x000fffffL
+#define UVD_SEMA_ADDR_HIGH__ADDR_42_23__SHIFT 0x00000000
+#define UVD_SEMA_ADDR_LOW__ADDR_22_3_MASK 0x000fffffL
+#define UVD_SEMA_ADDR_LOW__ADDR_22_3__SHIFT 0x00000000
+#define UVD_SEMA_CMD__MODE_MASK 0x00000040L
+#define UVD_SEMA_CMD__MODE__SHIFT 0x00000006
+#define UVD_SEMA_CMD__REQ_CMD_MASK 0x0000000fL
+#define UVD_SEMA_CMD__REQ_CMD__SHIFT 0x00000000
+#define UVD_SEMA_CMD__VMID_EN_MASK 0x00000080L
+#define UVD_SEMA_CMD__VMID_EN__SHIFT 0x00000007
+#define UVD_SEMA_CMD__VMID_MASK 0x00000f00L
+#define UVD_SEMA_CMD__VMID__SHIFT 0x00000008
+#define UVD_SEMA_CMD__WR_PHASE_MASK 0x00000030L
+#define UVD_SEMA_CMD__WR_PHASE__SHIFT 0x00000004
+#define UVD_SEMA_CNTL__ADVANCED_MODE_DIS_MASK 0x00000002L
+#define UVD_SEMA_CNTL__ADVANCED_MODE_DIS__SHIFT 0x00000001
+#define UVD_SEMA_CNTL__SEMAPHORE_EN_MASK 0x00000001L
+#define UVD_SEMA_CNTL__SEMAPHORE_EN__SHIFT 0x00000000
+#define UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL__RESEND_TIMER_MASK 0x07000000L
+#define UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL__RESEND_TIMER__SHIFT 0x00000018
+#define UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL__SIGNAL_INCOMPLETE_COUNT_MASK 0x001ffffeL
+#define UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL__SIGNAL_INCOMPLETE_COUNT__SHIFT 0x00000001
+#define UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL__SIGNAL_INCOMPLETE_EN_MASK 0x00000001L
+#define UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL__SIGNAL_INCOMPLETE_EN__SHIFT 0x00000000
+#define UVD_SEMA_TIMEOUT_STATUS__SEMAPHORE_SIGNAL_INCOMPLETE_TIMEOUT_STAT_MASK 0x00000004L
+#define UVD_SEMA_TIMEOUT_STATUS__SEMAPHORE_SIGNAL_INCOMPLETE_TIMEOUT_STAT__SHIFT 0x00000002
+#define UVD_SEMA_TIMEOUT_STATUS__SEMAPHORE_TIMEOUT_CLEAR_MASK 0x00000008L
+#define UVD_SEMA_TIMEOUT_STATUS__SEMAPHORE_TIMEOUT_CLEAR__SHIFT 0x00000003
+#define UVD_SEMA_TIMEOUT_STATUS__SEMAPHORE_WAIT_FAULT_TIMEOUT_STAT_MASK 0x00000002L
+#define UVD_SEMA_TIMEOUT_STATUS__SEMAPHORE_WAIT_FAULT_TIMEOUT_STAT__SHIFT 0x00000001
+#define UVD_SEMA_TIMEOUT_STATUS__SEMAPHORE_WAIT_INCOMPLETE_TIMEOUT_STAT_MASK 0x00000001L
+#define UVD_SEMA_TIMEOUT_STATUS__SEMAPHORE_WAIT_INCOMPLETE_TIMEOUT_STAT__SHIFT 0x00000000
+#define UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL__RESEND_TIMER_MASK 0x07000000L
+#define UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL__RESEND_TIMER__SHIFT 0x00000018
+#define UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL__WAIT_FAULT_COUNT_MASK 0x001ffffeL
+#define UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL__WAIT_FAULT_COUNT__SHIFT 0x00000001
+#define UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL__WAIT_FAULT_EN_MASK 0x00000001L
+#define UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL__WAIT_FAULT_EN__SHIFT 0x00000000
+#define UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL__RESEND_TIMER_MASK 0x07000000L
+#define UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL__RESEND_TIMER__SHIFT 0x00000018
+#define UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL__WAIT_INCOMPLETE_COUNT_MASK 0x001ffffeL
+#define UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL__WAIT_INCOMPLETE_COUNT__SHIFT 0x00000001
+#define UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL__WAIT_INCOMPLETE_EN_MASK 0x00000001L
+#define UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL__WAIT_INCOMPLETE_EN__SHIFT 0x00000000
+#define UVD_SOFT_RESET__CSM_SOFT_RESET_MASK 0x00000020L
+#define UVD_SOFT_RESET__CSM_SOFT_RESET__SHIFT 0x00000005
+#define UVD_SOFT_RESET__CXW_SOFT_RESET_MASK 0x00000040L
+#define UVD_SOFT_RESET__CXW_SOFT_RESET__SHIFT 0x00000006
+#define UVD_SOFT_RESET__FWV_SOFT_RESET_MASK 0x00000200L
+#define UVD_SOFT_RESET__FWV_SOFT_RESET__SHIFT 0x00000009
+#define UVD_SOFT_RESET__IDCT_SOFT_RESET_MASK 0x00001000L
+#define UVD_SOFT_RESET__IDCT_SOFT_RESET__SHIFT 0x0000000c
+#define UVD_SOFT_RESET__IH_SOFT_RESET_MASK 0x00000400L
+#define UVD_SOFT_RESET__IH_SOFT_RESET__SHIFT 0x0000000a
+#define UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK 0x00000002L
+#define UVD_SOFT_RESET__LBSI_SOFT_RESET__SHIFT 0x00000001
+#define UVD_SOFT_RESET__LCM_SOFT_RESET_MASK 0x00010000L
+#define UVD_SOFT_RESET__LCM_SOFT_RESET__SHIFT 0x00000010
+#define UVD_SOFT_RESET__LMI_SOFT_RESET_MASK 0x00000004L
+#define UVD_SOFT_RESET__LMI_SOFT_RESET__SHIFT 0x00000002
+#define UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK 0x00002000L
+#define UVD_SOFT_RESET__LMI_UMC_SOFT_RESET__SHIFT 0x0000000d
+#define UVD_SOFT_RESET__MIF_SOFT_RESET_MASK 0x00008000L
+#define UVD_SOFT_RESET__MIF_SOFT_RESET__SHIFT 0x0000000f
+#define UVD_SOFT_RESET__MPC_SOFT_RESET_MASK 0x00000100L
+#define UVD_SOFT_RESET__MPC_SOFT_RESET__SHIFT 0x00000008
+#define UVD_SOFT_RESET__MPRD_SOFT_RESET_MASK 0x00000800L
+#define UVD_SOFT_RESET__MPRD_SOFT_RESET__SHIFT 0x0000000b
+#define UVD_SOFT_RESET__RBC_SOFT_RESET_MASK 0x00000001L
+#define UVD_SOFT_RESET__RBC_SOFT_RESET__SHIFT 0x00000000
+#define UVD_SOFT_RESET__SPH_SOFT_RESET_MASK 0x00004000L
+#define UVD_SOFT_RESET__SPH_SOFT_RESET__SHIFT 0x0000000e
+#define UVD_SOFT_RESET__TAP_SOFT_RESET_MASK 0x00000080L
+#define UVD_SOFT_RESET__TAP_SOFT_RESET__SHIFT 0x00000007
+#define UVD_SOFT_RESET__UDEC_SOFT_RESET_MASK 0x00000010L
+#define UVD_SOFT_RESET__UDEC_SOFT_RESET__SHIFT 0x00000004
+#define UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK 0x00000008L
+#define UVD_SOFT_RESET__VCPU_SOFT_RESET__SHIFT 0x00000003
+#define UVD_STATUS__RBC_BUSY_MASK 0x00000001L
+#define UVD_STATUS__RBC_BUSY__SHIFT 0x00000000
+#define UVD_STATUS__VCPU_REPORT_MASK 0x000000feL
+#define UVD_STATUS__VCPU_REPORT__SHIFT 0x00000001
+#define UVD_UDEC_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK 0x00000700L
+#define UVD_UDEC_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT 0x00000008
+#define UVD_UDEC_ADDR_CONFIG__MULTI_GPU_TILE_SIZE_MASK 0x03000000L
+#define UVD_UDEC_ADDR_CONFIG__MULTI_GPU_TILE_SIZE__SHIFT 0x00000018
+#define UVD_UDEC_ADDR_CONFIG__NUM_GPUS_MASK 0x00700000L
+#define UVD_UDEC_ADDR_CONFIG__NUM_GPUS__SHIFT 0x00000014
+#define UVD_UDEC_ADDR_CONFIG__NUM_LOWER_PIPES_MASK 0x40000000L
+#define UVD_UDEC_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT 0x0000001e
+#define UVD_UDEC_ADDR_CONFIG__NUM_PIPES_MASK 0x00000007L
+#define UVD_UDEC_ADDR_CONFIG__NUM_PIPES__SHIFT 0x00000000
+#define UVD_UDEC_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK 0x00003000L
+#define UVD_UDEC_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT 0x0000000c
+#define UVD_UDEC_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK 0x00000070L
+#define UVD_UDEC_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT 0x00000004
+#define UVD_UDEC_ADDR_CONFIG__ROW_SIZE_MASK 0x30000000L
+#define UVD_UDEC_ADDR_CONFIG__ROW_SIZE__SHIFT 0x0000001c
+#define UVD_UDEC_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK 0x00070000L
+#define UVD_UDEC_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT 0x00000010
+#define UVD_UDEC_DB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK 0x00000700L
+#define UVD_UDEC_DB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT 0x00000008
+#define UVD_UDEC_DB_ADDR_CONFIG__MULTI_GPU_TILE_SIZE_MASK 0x03000000L
+#define UVD_UDEC_DB_ADDR_CONFIG__MULTI_GPU_TILE_SIZE__SHIFT 0x00000018
+#define UVD_UDEC_DB_ADDR_CONFIG__NUM_GPUS_MASK 0x00700000L
+#define UVD_UDEC_DB_ADDR_CONFIG__NUM_GPUS__SHIFT 0x00000014
+#define UVD_UDEC_DB_ADDR_CONFIG__NUM_LOWER_PIPES_MASK 0x40000000L
+#define UVD_UDEC_DB_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT 0x0000001e
+#define UVD_UDEC_DB_ADDR_CONFIG__NUM_PIPES_MASK 0x00000007L
+#define UVD_UDEC_DB_ADDR_CONFIG__NUM_PIPES__SHIFT 0x00000000
+#define UVD_UDEC_DB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK 0x00003000L
+#define UVD_UDEC_DB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT 0x0000000c
+#define UVD_UDEC_DB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK 0x00000070L
+#define UVD_UDEC_DB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT 0x00000004
+#define UVD_UDEC_DB_ADDR_CONFIG__ROW_SIZE_MASK 0x30000000L
+#define UVD_UDEC_DB_ADDR_CONFIG__ROW_SIZE__SHIFT 0x0000001c
+#define UVD_UDEC_DB_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK 0x00070000L
+#define UVD_UDEC_DB_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT 0x00000010
+#define UVD_UDEC_DBW_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK 0x00000700L
+#define UVD_UDEC_DBW_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT 0x00000008
+#define UVD_UDEC_DBW_ADDR_CONFIG__MULTI_GPU_TILE_SIZE_MASK 0x03000000L
+#define UVD_UDEC_DBW_ADDR_CONFIG__MULTI_GPU_TILE_SIZE__SHIFT 0x00000018
+#define UVD_UDEC_DBW_ADDR_CONFIG__NUM_GPUS_MASK 0x00700000L
+#define UVD_UDEC_DBW_ADDR_CONFIG__NUM_GPUS__SHIFT 0x00000014
+#define UVD_UDEC_DBW_ADDR_CONFIG__NUM_LOWER_PIPES_MASK 0x40000000L
+#define UVD_UDEC_DBW_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT 0x0000001e
+#define UVD_UDEC_DBW_ADDR_CONFIG__NUM_PIPES_MASK 0x00000007L
+#define UVD_UDEC_DBW_ADDR_CONFIG__NUM_PIPES__SHIFT 0x00000000
+#define UVD_UDEC_DBW_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK 0x00003000L
+#define UVD_UDEC_DBW_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT 0x0000000c
+#define UVD_UDEC_DBW_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK 0x00000070L
+#define UVD_UDEC_DBW_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT 0x00000004
+#define UVD_UDEC_DBW_ADDR_CONFIG__ROW_SIZE_MASK 0x30000000L
+#define UVD_UDEC_DBW_ADDR_CONFIG__ROW_SIZE__SHIFT 0x0000001c
+#define UVD_UDEC_DBW_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK 0x00070000L
+#define UVD_UDEC_DBW_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT 0x00000010
+#define UVD_VCPU_CACHE_OFFSET0__CACHE_OFFSET0_MASK 0x01ffffffL
+#define UVD_VCPU_CACHE_OFFSET0__CACHE_OFFSET0__SHIFT 0x00000000
+#define UVD_VCPU_CACHE_OFFSET1__CACHE_OFFSET1_MASK 0x01ffffffL
+#define UVD_VCPU_CACHE_OFFSET1__CACHE_OFFSET1__SHIFT 0x00000000
+#define UVD_VCPU_CACHE_OFFSET2__CACHE_OFFSET2_MASK 0x01ffffffL
+#define UVD_VCPU_CACHE_OFFSET2__CACHE_OFFSET2__SHIFT 0x00000000
+#define UVD_VCPU_CACHE_SIZE0__CACHE_SIZE0_MASK 0x001fffffL
+#define UVD_VCPU_CACHE_SIZE0__CACHE_SIZE0__SHIFT 0x00000000
+#define UVD_VCPU_CACHE_SIZE1__CACHE_SIZE1_MASK 0x001fffffL
+#define UVD_VCPU_CACHE_SIZE1__CACHE_SIZE1__SHIFT 0x00000000
+#define UVD_VCPU_CACHE_SIZE2__CACHE_SIZE2_MASK 0x001fffffL
+#define UVD_VCPU_CACHE_SIZE2__CACHE_SIZE2__SHIFT 0x00000000
+#define UVD_VCPU_CNTL__ABORT_REQ_MASK 0x00000100L
+#define UVD_VCPU_CNTL__ABORT_REQ__SHIFT 0x00000008
+#define UVD_VCPU_CNTL__AXI_MAX_BRST_SIZE_IS_4_MASK 0x00000010L
+#define UVD_VCPU_CNTL__AXI_MAX_BRST_SIZE_IS_4__SHIFT 0x00000004
+#define UVD_VCPU_CNTL__CABAC_MB_ACC_MASK 0x10000000L
+#define UVD_VCPU_CNTL__CABAC_MB_ACC__SHIFT 0x0000001c
+#define UVD_VCPU_CNTL__CLK_ACTIVE_MASK 0x00020000L
+#define UVD_VCPU_CNTL__CLK_ACTIVE__SHIFT 0x00000011
+#define UVD_VCPU_CNTL__CLK_EN_MASK 0x00000200L
+#define UVD_VCPU_CNTL__CLK_EN__SHIFT 0x00000009
+#define UVD_VCPU_CNTL__DBG_MUX_MASK 0x0000e000L
+#define UVD_VCPU_CNTL__DBG_MUX__SHIFT 0x0000000d
+#define UVD_VCPU_CNTL__ECPU_AM32_EN_MASK 0x20000000L
+#define UVD_VCPU_CNTL__ECPU_AM32_EN__SHIFT 0x0000001d
+#define UVD_VCPU_CNTL__IRQ_ERR_MASK 0x0000000fL
+#define UVD_VCPU_CNTL__IRQ_ERR__SHIFT 0x00000000
+#define UVD_VCPU_CNTL__JTAG_EN_MASK 0x00010000L
+#define UVD_VCPU_CNTL__JTAG_EN__SHIFT 0x00000010
+#define UVD_VCPU_CNTL__PMB_ED_ENABLE_MASK 0x00000020L
+#define UVD_VCPU_CNTL__PMB_ED_ENABLE__SHIFT 0x00000005
+#define UVD_VCPU_CNTL__PMB_SOFT_RESET_MASK 0x00000040L
+#define UVD_VCPU_CNTL__PMB_SOFT_RESET__SHIFT 0x00000006
+#define UVD_VCPU_CNTL__PRB_TIMEOUT_VAL_MASK 0x0ff00000L
+#define UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT 0x00000014
+#define UVD_VCPU_CNTL__RBBM_SOFT_RESET_MASK 0x00000080L
+#define UVD_VCPU_CNTL__RBBM_SOFT_RESET__SHIFT 0x00000007
+#define UVD_VCPU_CNTL__TIMEOUT_DIS_MASK 0x00040000L
+#define UVD_VCPU_CNTL__TIMEOUT_DIS__SHIFT 0x00000012
+#define UVD_VCPU_CNTL__TRCE_EN_MASK 0x00000400L
+#define UVD_VCPU_CNTL__TRCE_EN__SHIFT 0x0000000a
+#define UVD_VCPU_CNTL__TRCE_MUX_MASK 0x00001800L
+#define UVD_VCPU_CNTL__TRCE_MUX__SHIFT 0x0000000b
+#define UVD_VCPU_CNTL__WMV9_EN_MASK 0x40000000L
+#define UVD_VCPU_CNTL__WMV9_EN__SHIFT 0x0000001e
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vce/vce_1_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/vce/vce_1_0_d.h
new file mode 100644
index 000000000000..2176548e9203
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/vce/vce_1_0_d.h
@@ -0,0 +1,64 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef VCE_1_0_D_H
+#define VCE_1_0_D_H
+
+#define mmVCE_CLOCK_GATING_A 0x80BE
+#define mmVCE_CLOCK_GATING_B 0x80BF
+#define mmVCE_LMI_CACHE_CTRL 0x83BD
+#define mmVCE_LMI_CTRL 0x83A6
+#define mmVCE_LMI_CTRL2 0x839D
+#define mmVCE_LMI_MISC_CTRL 0x83B5
+#define mmVCE_LMI_STATUS 0x83A7
+#define mmVCE_LMI_SWAP_CNTL 0x83AD
+#define mmVCE_LMI_SWAP_CNTL1 0x83AE
+#define mmVCE_LMI_VCPU_CACHE_40BIT_BAR 0x8397
+#define mmVCE_LMI_VM_CTRL 0x83A8
+#define mmVCE_RB_ARB_CTRL 0x809F
+#define mmVCE_RB_BASE_HI 0x8061
+#define mmVCE_RB_BASE_HI2 0x805C
+#define mmVCE_RB_BASE_LO 0x8060
+#define mmVCE_RB_BASE_LO2 0x805B
+#define mmVCE_RB_RPTR 0x8063
+#define mmVCE_RB_RPTR2 0x805E
+#define mmVCE_RB_SIZE 0x8062
+#define mmVCE_RB_SIZE2 0x805D
+#define mmVCE_RB_WPTR 0x8064
+#define mmVCE_RB_WPTR2 0x805F
+#define mmVCE_SOFT_RESET 0x8048
+#define mmVCE_STATUS 0x8001
+#define mmVCE_SYS_INT_ACK 0x8341
+#define mmVCE_SYS_INT_EN 0x8340
+#define mmVCE_SYS_INT_STATUS 0x8341
+#define mmVCE_UENC_CLOCK_GATING 0x816F
+#define mmVCE_UENC_DMA_DCLK_CTRL 0x8250
+#define mmVCE_UENC_REG_CLOCK_GATING 0x8170
+#define mmVCE_VCPU_CACHE_OFFSET0 0x8009
+#define mmVCE_VCPU_CACHE_OFFSET1 0x800B
+#define mmVCE_VCPU_CACHE_OFFSET2 0x800D
+#define mmVCE_VCPU_CACHE_SIZE0 0x800A
+#define mmVCE_VCPU_CACHE_SIZE1 0x800C
+#define mmVCE_VCPU_CACHE_SIZE2 0x800E
+#define mmVCE_VCPU_CNTL 0x8005
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vce/vce_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/vce/vce_1_0_sh_mask.h
new file mode 100644
index 000000000000..ea5b26b11cb1
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/vce/vce_1_0_sh_mask.h
@@ -0,0 +1,99 @@
+/*
+ *
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef VCE_1_0_SH_MASK_H
+#define VCE_1_0_SH_MASK_H
+
+#define VCE_LMI_CACHE_CTRL__VCPU_EN_MASK 0x00000001L
+#define VCE_LMI_CACHE_CTRL__VCPU_EN__SHIFT 0x00000000
+#define VCE_LMI_CTRL2__STALL_ARB_UMC_MASK 0x00000100L
+#define VCE_LMI_CTRL2__STALL_ARB_UMC__SHIFT 0x00000008
+#define VCE_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK 0x00200000L
+#define VCE_LMI_CTRL__VCPU_DATA_COHERENCY_EN__SHIFT 0x00000015
+#define VCE_LMI_SWAP_CNTL1__RD_MC_CID_SWAP_MASK 0x00003ffcL
+#define VCE_LMI_SWAP_CNTL1__RD_MC_CID_SWAP__SHIFT 0x00000002
+#define VCE_LMI_SWAP_CNTL1__VCPU_R_MC_SWAP_MASK 0x00000003L
+#define VCE_LMI_SWAP_CNTL1__VCPU_R_MC_SWAP__SHIFT 0x00000000
+#define VCE_LMI_SWAP_CNTL__VCPU_W_MC_SWAP_MASK 0x00000003L
+#define VCE_LMI_SWAP_CNTL__VCPU_W_MC_SWAP__SHIFT 0x00000000
+#define VCE_LMI_SWAP_CNTL__WR_MC_CID_SWAP_MASK 0x00003ffcL
+#define VCE_LMI_SWAP_CNTL__WR_MC_CID_SWAP__SHIFT 0x00000002
+#define VCE_LMI_VCPU_CACHE_40BIT_BAR__BAR_MASK 0xffffffffL
+#define VCE_LMI_VCPU_CACHE_40BIT_BAR__BAR__SHIFT 0x00000000
+#define VCE_RB_BASE_HI2__RB_BASE_HI_MASK 0xffffffffL
+#define VCE_RB_BASE_HI2__RB_BASE_HI__SHIFT 0x00000000
+#define VCE_RB_BASE_HI__RB_BASE_HI_MASK 0xffffffffL
+#define VCE_RB_BASE_HI__RB_BASE_HI__SHIFT 0x00000000
+#define VCE_RB_BASE_LO2__RB_BASE_LO_MASK 0xffffffc0L
+#define VCE_RB_BASE_LO2__RB_BASE_LO__SHIFT 0x00000006
+#define VCE_RB_BASE_LO__RB_BASE_LO_MASK 0xffffffc0L
+#define VCE_RB_BASE_LO__RB_BASE_LO__SHIFT 0x00000006
+#define VCE_RB_RPTR2__RB_RPTR_MASK 0x007ffff0L
+#define VCE_RB_RPTR2__RB_RPTR__SHIFT 0x00000004
+#define VCE_RB_RPTR__RB_RPTR_MASK 0x007ffff0L
+#define VCE_RB_RPTR__RB_RPTR__SHIFT 0x00000004
+#define VCE_RB_SIZE2__RB_SIZE_MASK 0x007ffff0L
+#define VCE_RB_SIZE2__RB_SIZE__SHIFT 0x00000004
+#define VCE_RB_SIZE__RB_SIZE_MASK 0x007ffff0L
+#define VCE_RB_SIZE__RB_SIZE__SHIFT 0x00000004
+#define VCE_RB_WPTR2__RB_WPTR_MASK 0x007ffff0L
+#define VCE_RB_WPTR2__RB_WPTR__SHIFT 0x00000004
+#define VCE_RB_WPTR__RB_WPTR_MASK 0x007ffff0L
+#define VCE_RB_WPTR__RB_WPTR__SHIFT 0x00000004
+#define VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK 0x00000001L
+#define VCE_SOFT_RESET__ECPU_SOFT_RESET__SHIFT 0x00000000
+#define VCE_STATUS__JOB_BUSY_MASK 0x00000001L
+#define VCE_STATUS__JOB_BUSY__SHIFT 0x00000000
+#define VCE_STATUS__UENC_BUSY_MASK 0x00000100L
+#define VCE_STATUS__UENC_BUSY__SHIFT 0x00000008
+#define VCE_STATUS__VCPU_REPORT_MASK 0x000000feL
+#define VCE_STATUS__VCPU_REPORT__SHIFT 0x00000001
+#define VCE_SYS_INT_ACK__VCE_SYS_INT_TRAP_INTERRUPT_ACK_MASK 0x00000008L
+#define VCE_SYS_INT_ACK__VCE_SYS_INT_TRAP_INTERRUPT_ACK__SHIFT 0x00000003
+#define VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK 0x00000008L
+#define VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN__SHIFT 0x00000003
+#define VCE_SYS_INT_STATUS__VCE_SYS_INT_TRAP_INTERRUPT_INT_MASK 0x00000008L
+#define VCE_SYS_INT_STATUS__VCE_SYS_INT_TRAP_INTERRUPT_INT__SHIFT 0x00000003
+#define VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK 0x00000002L
+#define VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON__SHIFT 0x00000001
+#define VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK 0x00000004L
+#define VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON__SHIFT 0x00000002
+#define VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK 0x00000001L
+#define VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON__SHIFT 0x00000000
+#define VCE_VCPU_CACHE_OFFSET0__OFFSET_MASK 0x0fffffffL
+#define VCE_VCPU_CACHE_OFFSET0__OFFSET__SHIFT 0x00000000
+#define VCE_VCPU_CACHE_OFFSET1__OFFSET_MASK 0x0fffffffL
+#define VCE_VCPU_CACHE_OFFSET1__OFFSET__SHIFT 0x00000000
+#define VCE_VCPU_CACHE_OFFSET2__OFFSET_MASK 0x0fffffffL
+#define VCE_VCPU_CACHE_OFFSET2__OFFSET__SHIFT 0x00000000
+#define VCE_VCPU_CACHE_SIZE0__SIZE_MASK 0x00ffffffL
+#define VCE_VCPU_CACHE_SIZE0__SIZE__SHIFT 0x00000000
+#define VCE_VCPU_CACHE_SIZE1__SIZE_MASK 0x00ffffffL
+#define VCE_VCPU_CACHE_SIZE1__SIZE__SHIFT 0x00000000
+#define VCE_VCPU_CACHE_SIZE2__SIZE_MASK 0x00ffffffL
+#define VCE_VCPU_CACHE_SIZE2__SIZE__SHIFT 0x00000000
+#define VCE_VCPU_CNTL__CLK_EN_MASK 0x00000001L
+#define VCE_VCPU_CNTL__CLK_EN__SHIFT 0x00000000
+#define VCE_VCPU_CNTL__RBBM_SOFT_RESET_MASK 0x00040000L
+#define VCE_VCPU_CNTL__RBBM_SOFT_RESET__SHIFT 0x00000012
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/cgs_common.h b/drivers/gpu/drm/amd/include/cgs_common.h
index df7c18b6a02a..e4a1697ec1d3 100644
--- a/drivers/gpu/drm/amd/include/cgs_common.h
+++ b/drivers/gpu/drm/amd/include/cgs_common.h
@@ -106,6 +106,7 @@ enum cgs_ucode_id {
 	CGS_UCODE_ID_CP_MEC_JT2,
 	CGS_UCODE_ID_GMCON_RENG,
 	CGS_UCODE_ID_RLC_G,
+	CGS_UCODE_ID_STORAGE,
 	CGS_UCODE_ID_MAXIMUM,
 };
 
@@ -619,6 +620,8 @@ typedef int (*cgs_call_acpi_method)(struct cgs_device *cgs_device,
 typedef int (*cgs_query_system_info)(struct cgs_device *cgs_device,
 				struct cgs_system_info *sys_info);
 
+typedef int (*cgs_is_virtualization_enabled_t)(void *cgs_device);
+
 struct cgs_ops {
 	/* memory management calls (similar to KFD interface) */
 	cgs_gpu_mem_info_t gpu_mem_info;
@@ -670,6 +673,7 @@ struct cgs_ops {
 	cgs_call_acpi_method call_acpi_method;
 	/* get system info */
 	cgs_query_system_info query_system_info;
+	cgs_is_virtualization_enabled_t is_virtualization_enabled;
 };
 
 struct cgs_os_ops; /* To be define in OS-specific CGS header */
@@ -773,4 +777,6 @@ struct cgs_device
 	CGS_CALL(get_pci_resource, cgs_device, resource_type, size, offset, \
 	resource_base)
 
+#define cgs_is_virtualization_enabled(cgs_device) \
+		CGS_CALL(is_virtualization_enabled, cgs_device)
 #endif /* _CGS_COMMON_H */
diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
index 7174f7a68266..c81cf1412728 100644
--- a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
@@ -41,7 +41,7 @@
 #define PP_CHECK_HW(hwmgr)						\
 	do {								\
 		if ((hwmgr) == NULL || (hwmgr)->hwmgr_func == NULL)	\
-			return -EINVAL;					\
+			return 0;					\
 	} while (0)
 
 static int pp_early_init(void *handle)
@@ -115,6 +115,7 @@ static int pp_hw_init(void *handle)
 	struct pp_instance *pp_handle;
 	struct pp_smumgr *smumgr;
 	struct pp_eventmgr *eventmgr;
+	struct pp_hwmgr  *hwmgr;
 	int ret = 0;
 
 	if (handle == NULL)
@@ -122,6 +123,7 @@ static int pp_hw_init(void *handle)
 
 	pp_handle = (struct pp_instance *)handle;
 	smumgr = pp_handle->smu_mgr;
+	hwmgr = pp_handle->hwmgr;
 
 	if (smumgr == NULL || smumgr->smumgr_funcs == NULL ||
 		smumgr->smumgr_funcs->smu_init == NULL ||
@@ -141,9 +143,11 @@ static int pp_hw_init(void *handle)
 		return ret;
 	}
 
-	hw_init_power_state_table(pp_handle->hwmgr);
-	eventmgr = pp_handle->eventmgr;
+	PP_CHECK_HW(hwmgr);
+
+	hw_init_power_state_table(hwmgr);
 
+	eventmgr = pp_handle->eventmgr;
 	if (eventmgr == NULL || eventmgr->pp_eventmgr_init == NULL)
 		return -EINVAL;
 
@@ -243,7 +247,9 @@ static int pp_suspend(void *handle)
 
 	pp_handle = (struct pp_instance *)handle;
 	eventmgr = pp_handle->eventmgr;
-	pem_handle_event(eventmgr, AMD_PP_EVENT_SUSPEND, &event_data);
+
+	if (eventmgr != NULL)
+		pem_handle_event(eventmgr, AMD_PP_EVENT_SUSPEND, &event_data);
 	return 0;
 }
 
@@ -273,7 +279,8 @@ static int pp_resume(void *handle)
 	}
 
 	eventmgr = pp_handle->eventmgr;
-	pem_handle_event(eventmgr, AMD_PP_EVENT_RESUME, &event_data);
+	if (eventmgr != NULL)
+		pem_handle_event(eventmgr, AMD_PP_EVENT_RESUME, &event_data);
 
 	return 0;
 }
@@ -340,8 +347,7 @@ static enum amd_dpm_forced_level pp_dpm_get_performance_level(
 
 	hwmgr = ((struct pp_instance *)handle)->hwmgr;
 
-	if (hwmgr == NULL)
-		return -EINVAL;
+	PP_CHECK_HW(hwmgr);
 
 	return (((struct pp_instance *)handle)->hwmgr->dpm_level);
 }
@@ -436,7 +442,8 @@ static enum PP_StateUILabel power_state_convert(enum amd_pm_state_type  state)
 	}
 }
 
-int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_event event_id, void *input, void *output)
+static int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_event event_id,
+		void *input, void *output)
 {
 	int ret = 0;
 	struct pp_instance *pp_handle;
@@ -447,6 +454,9 @@ int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_event event_id, void *input,
 	if (pp_handle == NULL)
 		return -EINVAL;
 
+	if (pp_handle->eventmgr == NULL)
+		return 0;
+
 	switch (event_id) {
 	case AMD_PP_EVENT_DISPLAY_CONFIG_CHANGE:
 		ret = pem_handle_event(pp_handle->eventmgr, event_id, &data);
@@ -475,7 +485,7 @@ int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_event event_id, void *input,
 	return ret;
 }
 
-enum amd_pm_state_type pp_dpm_get_current_power_state(void *handle)
+static enum amd_pm_state_type pp_dpm_get_current_power_state(void *handle)
 {
 	struct pp_hwmgr *hwmgr;
 	struct pp_power_state *state;
@@ -581,6 +591,23 @@ static int pp_dpm_get_fan_speed_percent(void *handle, uint32_t *speed)
 	return hwmgr->hwmgr_func->get_fan_speed_percent(hwmgr, speed);
 }
 
+static int pp_dpm_get_fan_speed_rpm(void *handle, uint32_t *rpm)
+{
+	struct pp_hwmgr *hwmgr;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+	PP_CHECK_HW(hwmgr);
+
+	if (hwmgr->hwmgr_func->get_fan_speed_rpm == NULL)
+		return -EINVAL;
+
+	return hwmgr->hwmgr_func->get_fan_speed_rpm(hwmgr, rpm);
+}
+
 static int pp_dpm_get_temperature(void *handle)
 {
 	struct pp_hwmgr  *hwmgr;
@@ -820,6 +847,21 @@ static int pp_dpm_read_sensor(void *handle, int idx, int32_t *value)
 	return hwmgr->hwmgr_func->read_sensor(hwmgr, idx, value);
 }
 
+static struct amd_vce_state*
+pp_dpm_get_vce_clock_state(void *handle, unsigned idx)
+{
+	struct pp_hwmgr *hwmgr;
+
+	if (handle) {
+		hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+		if (hwmgr && idx < hwmgr->num_vce_state_tables)
+			return &hwmgr->vce_states[idx];
+	}
+
+	return NULL;
+}
+
 const struct amd_powerplay_funcs pp_dpm_funcs = {
 	.get_temperature = pp_dpm_get_temperature,
 	.load_firmware = pp_dpm_load_fw,
@@ -836,6 +878,7 @@ const struct amd_powerplay_funcs pp_dpm_funcs = {
 	.get_fan_control_mode = pp_dpm_get_fan_control_mode,
 	.set_fan_speed_percent = pp_dpm_set_fan_speed_percent,
 	.get_fan_speed_percent = pp_dpm_get_fan_speed_percent,
+	.get_fan_speed_rpm = pp_dpm_get_fan_speed_rpm,
 	.get_pp_num_states = pp_dpm_get_pp_num_states,
 	.get_pp_table = pp_dpm_get_pp_table,
 	.set_pp_table = pp_dpm_set_pp_table,
@@ -846,6 +889,7 @@ const struct amd_powerplay_funcs pp_dpm_funcs = {
 	.get_mclk_od = pp_dpm_get_mclk_od,
 	.set_mclk_od = pp_dpm_set_mclk_od,
 	.read_sensor = pp_dpm_read_sensor,
+	.get_vce_clock_state = pp_dpm_get_vce_clock_state,
 };
 
 static int amd_pp_instance_init(struct amd_pp_init *pp_init,
@@ -864,6 +908,13 @@ static int amd_pp_instance_init(struct amd_pp_init *pp_init,
 	if (ret)
 		goto fail_smum;
 
+
+	amd_pp->pp_handle = handle;
+
+	if ((amdgpu_dpm == 0)
+		|| cgs_is_virtualization_enabled(pp_init->device))
+		return 0;
+
 	ret = hwmgr_init(pp_init, handle);
 	if (ret)
 		goto fail_hwmgr;
@@ -872,7 +923,6 @@ static int amd_pp_instance_init(struct amd_pp_init *pp_init,
 	if (ret)
 		goto fail_eventmgr;
 
-	amd_pp->pp_handle = handle;
 	return 0;
 
 fail_eventmgr:
@@ -891,12 +941,13 @@ static int amd_pp_instance_fini(void *handle)
 	if (instance == NULL)
 		return -EINVAL;
 
-	eventmgr_fini(instance->eventmgr);
-
-	hwmgr_fini(instance->hwmgr);
+	if ((amdgpu_dpm != 0)
+		&& !cgs_is_virtualization_enabled(instance->smu_mgr->device)) {
+		eventmgr_fini(instance->eventmgr);
+		hwmgr_fini(instance->hwmgr);
+	}
 
 	smum_fini(instance->smu_mgr);
-
 	kfree(handle);
 	return 0;
 }
@@ -953,6 +1004,10 @@ int amd_powerplay_reset(void *handle)
 	if (ret)
 		return ret;
 
+	if ((amdgpu_dpm == 0)
+		|| cgs_is_virtualization_enabled(instance->smu_mgr->device))
+		return 0;
+
 	hw_init_power_state_table(instance->hwmgr);
 
 	if (eventmgr == NULL || eventmgr->pp_eventmgr_init == NULL)
@@ -976,6 +1031,8 @@ int amd_powerplay_display_configuration_change(void *handle,
 
 	hwmgr = ((struct pp_instance *)handle)->hwmgr;
 
+	PP_CHECK_HW(hwmgr);
+
 	phm_store_dal_configuration_data(hwmgr, display_config);
 
 	return 0;
@@ -993,6 +1050,8 @@ int amd_powerplay_get_display_power_level(void *handle,
 
 	hwmgr = ((struct pp_instance *)handle)->hwmgr;
 
+	PP_CHECK_HW(hwmgr);
+
 	return phm_get_dal_power_level(hwmgr, output);
 }
 
@@ -1010,6 +1069,8 @@ int amd_powerplay_get_current_clocks(void *handle,
 
 	hwmgr = ((struct pp_instance *)handle)->hwmgr;
 
+	PP_CHECK_HW(hwmgr);
+
 	phm_get_dal_power_level(hwmgr, &simple_clocks);
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PowerContainment)) {
@@ -1054,6 +1115,8 @@ int amd_powerplay_get_clock_by_type(void *handle, enum amd_pp_clock_type type, s
 
 	hwmgr = ((struct pp_instance *)handle)->hwmgr;
 
+	PP_CHECK_HW(hwmgr);
+
 	result = phm_get_clock_by_type(hwmgr, type, clocks);
 
 	return result;
@@ -1072,6 +1135,8 @@ int amd_powerplay_get_display_mode_validation_clocks(void *handle,
 
 	hwmgr = ((struct pp_instance *)handle)->hwmgr;
 
+	PP_CHECK_HW(hwmgr);
+
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DynamicPatchPowerState))
 		result = phm_get_max_high_clocks(hwmgr, clocks);
 
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c
index b6f45fd01fa6..ec36c0e28388 100644
--- a/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c
@@ -154,7 +154,7 @@ int pem_task_powerdown_vce_tasks(struct pp_eventmgr *eventmgr, struct pem_event_
 
 int pem_task_disable_clock_power_gatings_tasks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
 {
-	/* TODO */
+	phm_disable_clock_power_gatings(eventmgr->hwmgr);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c
index 2028980f1ed4..b0c63c5f54c9 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c
@@ -169,7 +169,7 @@ int cz_dpm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
 	if (bgate) {
 		cgs_set_clockgating_state(hwmgr->device,
 						AMD_IP_BLOCK_TYPE_UVD,
-						AMD_CG_STATE_UNGATE);
+						AMD_CG_STATE_GATE);
 		cgs_set_powergating_state(hwmgr->device,
 						AMD_IP_BLOCK_TYPE_UVD,
 						AMD_PG_STATE_GATE);
@@ -182,7 +182,7 @@ int cz_dpm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
 						AMD_CG_STATE_UNGATE);
 		cgs_set_clockgating_state(hwmgr->device,
 						AMD_IP_BLOCK_TYPE_UVD,
-						AMD_PG_STATE_GATE);
+						AMD_PG_STATE_UNGATE);
 		cz_dpm_update_uvd_dpm(hwmgr, false);
 	}
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
index 960424913496..4b14f259a147 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
@@ -66,7 +66,7 @@ static const struct cz_power_state *cast_const_PhwCzPowerState(
 	return (struct cz_power_state *)hw_ps;
 }
 
-uint32_t cz_get_eclk_level(struct pp_hwmgr *hwmgr,
+static uint32_t cz_get_eclk_level(struct pp_hwmgr *hwmgr,
 					uint32_t clock, uint32_t msg)
 {
 	int i = 0;
@@ -1017,7 +1017,7 @@ static int cz_tf_program_bootup_state(struct pp_hwmgr *hwmgr, void *input,
 	return 0;
 }
 
-int cz_tf_reset_acp_boot_level(struct pp_hwmgr *hwmgr, void *input,
+static int cz_tf_reset_acp_boot_level(struct pp_hwmgr *hwmgr, void *input,
 				void *output, void *storage, int result)
 {
 	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
@@ -1225,7 +1225,7 @@ static int cz_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int cz_phm_force_dpm_highest(struct pp_hwmgr *hwmgr)
+static int cz_phm_force_dpm_highest(struct pp_hwmgr *hwmgr)
 {
 	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
 
@@ -1239,7 +1239,7 @@ int cz_phm_force_dpm_highest(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int cz_phm_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
+static int cz_phm_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
 {
 	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
 	struct phm_clock_voltage_dependency_table *table =
@@ -1277,7 +1277,7 @@ int cz_phm_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int cz_phm_force_dpm_lowest(struct pp_hwmgr *hwmgr)
+static int cz_phm_force_dpm_lowest(struct pp_hwmgr *hwmgr)
 {
 	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
 
@@ -1533,7 +1533,7 @@ static int cz_dpm_get_pp_table_entry(struct pp_hwmgr *hwmgr,
 	return result;
 }
 
-int cz_get_power_state_size(struct pp_hwmgr *hwmgr)
+static int cz_get_power_state_size(struct pp_hwmgr *hwmgr)
 {
 	return sizeof(struct cz_power_state);
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
index a6abe81bc843..71822ae73a12 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
@@ -35,7 +35,7 @@ static int phm_run_table(struct pp_hwmgr *hwmgr,
 	phm_table_function *function;
 
 	if (rt_table->function_list == NULL) {
-		printk(KERN_INFO "[ powerplay ] this function not implement!\n");
+		pr_debug("[ powerplay ] this function not implement!\n");
 		return 0;
 	}
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
index 0723758ed065..c355a0f51663 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
@@ -209,6 +209,19 @@ int phm_enable_clock_power_gatings(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
+int phm_disable_clock_power_gatings(struct pp_hwmgr *hwmgr)
+{
+	PHM_FUNC_CHECK(hwmgr);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+		PHM_PlatformCaps_TablelessHardwareInterface)) {
+		if (NULL != hwmgr->hwmgr_func->disable_clock_power_gating)
+			return hwmgr->hwmgr_func->disable_clock_power_gating(hwmgr);
+	}
+	return 0;
+}
+
+
 int phm_display_configuration_changed(struct pp_hwmgr *hwmgr)
 {
 	PHM_FUNC_CHECK(hwmgr);
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
index e03dcb6ea9c1..dc6700aee18f 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
@@ -80,20 +80,17 @@ int hwmgr_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
 		switch (hwmgr->chip_id) {
 		case CHIP_TOPAZ:
 			topaz_set_asic_special_caps(hwmgr);
-			hwmgr->feature_mask &= ~(PP_SMC_VOLTAGE_CONTROL_MASK |
-						PP_VBI_TIME_SUPPORT_MASK |
+			hwmgr->feature_mask &= ~ (PP_VBI_TIME_SUPPORT_MASK |
 						PP_ENABLE_GFX_CG_THRU_SMU);
 			hwmgr->pp_table_version = PP_TABLE_V0;
 			break;
 		case CHIP_TONGA:
 			tonga_set_asic_special_caps(hwmgr);
-			hwmgr->feature_mask &= ~(PP_SMC_VOLTAGE_CONTROL_MASK |
-						PP_VBI_TIME_SUPPORT_MASK);
+			hwmgr->feature_mask &= ~PP_VBI_TIME_SUPPORT_MASK;
 			break;
 		case CHIP_FIJI:
 			fiji_set_asic_special_caps(hwmgr);
-			hwmgr->feature_mask &= ~(PP_SMC_VOLTAGE_CONTROL_MASK |
-						PP_VBI_TIME_SUPPORT_MASK |
+			hwmgr->feature_mask &= ~ (PP_VBI_TIME_SUPPORT_MASK |
 						PP_ENABLE_GFX_CG_THRU_SMU);
 			break;
 		case CHIP_POLARIS11:
@@ -685,20 +682,24 @@ void hwmgr_init_default_caps(struct pp_hwmgr *hwmgr)
 
 int hwmgr_set_user_specify_caps(struct pp_hwmgr *hwmgr)
 {
-	if (amdgpu_sclk_deep_sleep_en)
+	if (amdgpu_pp_feature_mask & PP_SCLK_DEEP_SLEEP_MASK)
 		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SclkDeepSleep);
 	else
 		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SclkDeepSleep);
 
-	if (amdgpu_powercontainment)
+	if (amdgpu_pp_feature_mask & PP_POWER_CONTAINMENT_MASK) {
 		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
 			    PHM_PlatformCaps_PowerContainment);
-	else
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_CAC);
+	} else {
 		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 			    PHM_PlatformCaps_PowerContainment);
-
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_CAC);
+	}
 	hwmgr->feature_mask = amdgpu_pp_feature_mask;
 
 	return 0;
@@ -736,9 +737,6 @@ int polaris_set_asic_special_caps(struct pp_hwmgr *hwmgr)
 						PHM_PlatformCaps_TCPRamping);
 
 	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-							PHM_PlatformCaps_CAC);
-
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
 						PHM_PlatformCaps_RegulatorHot);
 
 	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
@@ -767,8 +765,6 @@ int fiji_set_asic_special_caps(struct pp_hwmgr *hwmgr)
 	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_TablelessHardwareInterface);
 
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_CAC);
 	return 0;
 }
 
@@ -791,9 +787,6 @@ int tonga_set_asic_special_caps(struct pp_hwmgr *hwmgr)
 	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
 			 PHM_PlatformCaps_TablelessHardwareInterface);
 
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_CAC);
-
 	return 0;
 }
 
@@ -810,8 +803,6 @@ int topaz_set_asic_special_caps(struct pp_hwmgr *hwmgr)
 	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
 			 PHM_PlatformCaps_TablelessHardwareInterface);
 	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-			PHM_PlatformCaps_CAC);
-	phm_cap_set(hwmgr->platform_descriptor.platformCaps,
 		    PHM_PlatformCaps_EVV);
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c
index 1944d289f846..f5e8fda964f7 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c
@@ -25,6 +25,7 @@
 #include "linux/delay.h"
 #include "hwmgr.h"
 #include "amd_acpi.h"
+#include "pp_acpi.h"
 
 bool acpi_atcs_functions_supported(void *device, uint32_t index)
 {
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
index 4477c55a58e3..c45bd2560468 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
@@ -131,7 +131,7 @@ static int set_platform_caps(struct pp_hwmgr *hwmgr, uint32_t powerplay_caps)
 /**
  * Private Function to get the PowerPlay Table Address.
  */
-const void *get_powerplay_table(struct pp_hwmgr *hwmgr)
+static const void *get_powerplay_table(struct pp_hwmgr *hwmgr)
 {
 	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
 
@@ -1049,7 +1049,7 @@ static int check_powerplay_tables(
 	return 0;
 }
 
-int pp_tables_v1_0_initialize(struct pp_hwmgr *hwmgr)
+static int pp_tables_v1_0_initialize(struct pp_hwmgr *hwmgr)
 {
 	int result = 0;
 	const ATOM_Tonga_POWERPLAYTABLE *powerplay_table;
@@ -1100,7 +1100,7 @@ int pp_tables_v1_0_initialize(struct pp_hwmgr *hwmgr)
 	return result;
 }
 
-int pp_tables_v1_0_uninitialize(struct pp_hwmgr *hwmgr)
+static int pp_tables_v1_0_uninitialize(struct pp_hwmgr *hwmgr)
 {
 	struct phm_ppt_v1_information *pp_table_information =
 		(struct phm_ppt_v1_information *)(hwmgr->pptable);
@@ -1214,7 +1214,7 @@ static int ppt_get_num_of_vce_state_table_entries_v1_0(struct pp_hwmgr *hwmgr)
 }
 
 static int ppt_get_vce_state_table_entry_v1_0(struct pp_hwmgr *hwmgr, uint32_t i,
-		struct pp_vce_state *vce_state, void **clock_info, uint32_t *flag)
+		struct amd_vce_state *vce_state, void **clock_info, uint32_t *flag)
 {
 	const ATOM_Tonga_VCE_State_Record *vce_state_record;
 	ATOM_Tonga_SCLK_Dependency_Record *sclk_dep_record;
@@ -1318,7 +1318,7 @@ int get_powerplay_table_entry_v1_0(struct pp_hwmgr *hwmgr,
 
 	hwmgr->num_vce_state_tables = i = ppt_get_num_of_vce_state_table_entries_v1_0(hwmgr);
 
-	if ((i != 0) && (i <= PP_MAX_VCE_LEVELS)) {
+	if ((i != 0) && (i <= AMD_MAX_VCE_LEVELS)) {
 		for (j = 0; j < i; j++)
 			ppt_get_vce_state_table_entry_v1_0(hwmgr, j, &(hwmgr->vce_states[j]), NULL, &flags);
 	}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
index ccf7ebeaf892..a4e9cf429e62 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
@@ -1507,7 +1507,7 @@ static int init_phase_shedding_table(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
-int get_number_of_vce_state_table_entries(
+static int get_number_of_vce_state_table_entries(
 						  struct pp_hwmgr *hwmgr)
 {
 	const ATOM_PPLIB_POWERPLAYTABLE *table =
@@ -1521,9 +1521,9 @@ int get_number_of_vce_state_table_entries(
 	return 0;
 }
 
-int get_vce_state_table_entry(struct pp_hwmgr *hwmgr,
+static int get_vce_state_table_entry(struct pp_hwmgr *hwmgr,
 							unsigned long i,
-							struct pp_vce_state *vce_state,
+							struct amd_vce_state *vce_state,
 							void **clock_info,
 							unsigned long *flag)
 {
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c
index 6eb6db199250..a1fc4fcac1e0 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_clockpowergating.c
@@ -75,7 +75,7 @@ int smu7_powerdown_uvd(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smu7_powerup_uvd(struct pp_hwmgr *hwmgr)
+static int smu7_powerup_uvd(struct pp_hwmgr *hwmgr)
 {
 	if (phm_cf_want_uvd_power_gating(hwmgr)) {
 		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
@@ -91,7 +91,7 @@ int smu7_powerup_uvd(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smu7_powerdown_vce(struct pp_hwmgr *hwmgr)
+static int smu7_powerdown_vce(struct pp_hwmgr *hwmgr)
 {
 	if (phm_cf_want_vce_power_gating(hwmgr))
 		return smum_send_msg_to_smc(hwmgr->smumgr,
@@ -99,7 +99,7 @@ int smu7_powerdown_vce(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smu7_powerup_vce(struct pp_hwmgr *hwmgr)
+static int smu7_powerup_vce(struct pp_hwmgr *hwmgr)
 {
 	if (phm_cf_want_vce_power_gating(hwmgr))
 		return smum_send_msg_to_smc(hwmgr->smumgr,
@@ -107,7 +107,7 @@ int smu7_powerup_vce(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smu7_powerdown_samu(struct pp_hwmgr *hwmgr)
+static int smu7_powerdown_samu(struct pp_hwmgr *hwmgr)
 {
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SamuPowerGating))
@@ -116,7 +116,7 @@ int smu7_powerdown_samu(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smu7_powerup_samu(struct pp_hwmgr *hwmgr)
+static int smu7_powerup_samu(struct pp_hwmgr *hwmgr)
 {
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SamuPowerGating))
@@ -150,14 +150,20 @@ int smu7_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
 		cgs_set_clockgating_state(hwmgr->device,
 				AMD_IP_BLOCK_TYPE_UVD,
 				AMD_CG_STATE_GATE);
+		cgs_set_powergating_state(hwmgr->device,
+						AMD_IP_BLOCK_TYPE_UVD,
+						AMD_PG_STATE_GATE);
 		smu7_update_uvd_dpm(hwmgr, true);
 		smu7_powerdown_uvd(hwmgr);
 	} else {
 		smu7_powerup_uvd(hwmgr);
-		smu7_update_uvd_dpm(hwmgr, false);
+		cgs_set_powergating_state(hwmgr->device,
+						AMD_IP_BLOCK_TYPE_UVD,
+						AMD_CG_STATE_UNGATE);
 		cgs_set_clockgating_state(hwmgr->device,
 				AMD_IP_BLOCK_TYPE_UVD,
 				AMD_CG_STATE_UNGATE);
+		smu7_update_uvd_dpm(hwmgr, false);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
index 08cd0bd3ebe5..a74f60a575ae 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
@@ -89,7 +89,7 @@ enum DPM_EVENT_SRC {
 
 static const unsigned long PhwVIslands_Magic = (unsigned long)(PHM_VIslands_Magic);
 
-struct smu7_power_state *cast_phw_smu7_power_state(
+static struct smu7_power_state *cast_phw_smu7_power_state(
 				  struct pp_hw_power_state *hw_ps)
 {
 	PP_ASSERT_WITH_CODE((PhwVIslands_Magic == hw_ps->magic),
@@ -99,7 +99,7 @@ struct smu7_power_state *cast_phw_smu7_power_state(
 	return (struct smu7_power_state *)hw_ps;
 }
 
-const struct smu7_power_state *cast_const_phw_smu7_power_state(
+static const struct smu7_power_state *cast_const_phw_smu7_power_state(
 				 const struct pp_hw_power_state *hw_ps)
 {
 	PP_ASSERT_WITH_CODE((PhwVIslands_Magic == hw_ps->magic),
@@ -115,7 +115,7 @@ const struct smu7_power_state *cast_const_phw_smu7_power_state(
  * @param    hwmgr  the address of the powerplay hardware manager.
  * @return   always 0
  */
-int smu7_get_mc_microcode_version (struct pp_hwmgr *hwmgr)
+static int smu7_get_mc_microcode_version(struct pp_hwmgr *hwmgr)
 {
 	cgs_write_register(hwmgr->device, mmMC_SEQ_IO_DEBUG_INDEX, 0x9F);
 
@@ -124,7 +124,7 @@ int smu7_get_mc_microcode_version (struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-uint16_t smu7_get_current_pcie_speed(struct pp_hwmgr *hwmgr)
+static uint16_t smu7_get_current_pcie_speed(struct pp_hwmgr *hwmgr)
 {
 	uint32_t speedCntl = 0;
 
@@ -135,7 +135,7 @@ uint16_t smu7_get_current_pcie_speed(struct pp_hwmgr *hwmgr)
 			PCIE_LC_SPEED_CNTL, LC_CURRENT_DATA_RATE));
 }
 
-int smu7_get_current_pcie_lane_number(struct pp_hwmgr *hwmgr)
+static int smu7_get_current_pcie_lane_number(struct pp_hwmgr *hwmgr)
 {
 	uint32_t link_width;
 
@@ -155,7 +155,7 @@ int smu7_get_current_pcie_lane_number(struct pp_hwmgr *hwmgr)
 * @param    pHwMgr  the address of the powerplay hardware manager.
 * @return   always PP_Result_OK
 */
-int smu7_enable_smc_voltage_controller(struct pp_hwmgr *hwmgr)
+static int smu7_enable_smc_voltage_controller(struct pp_hwmgr *hwmgr)
 {
 	if (hwmgr->feature_mask & PP_SMC_VOLTAGE_CONTROL_MASK)
 		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_Voltage_Cntl_Enable);
@@ -802,7 +802,7 @@ static int smu7_setup_dpm_tables_v1(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smu7_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
+static int smu7_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
@@ -993,13 +993,6 @@ static int smu7_start_dpm(struct pp_hwmgr *hwmgr)
 	PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__PCIE,
 			SWRST_COMMAND_1, RESETLC, 0x0);
 
-	PP_ASSERT_WITH_CODE(
-			(0 == smum_send_msg_to_smc(hwmgr->smumgr,
-					PPSMC_MSG_Voltage_Cntl_Enable)),
-			"Failed to enable voltage DPM during DPM Start Function!",
-			return -EINVAL);
-
-
 	if (smu7_enable_sclk_mclk_dpm(hwmgr)) {
 		printk(KERN_ERR "Failed to enable Sclk DPM and Mclk DPM!");
 		return -EINVAL;
@@ -1153,7 +1146,7 @@ static int smu7_disable_thermal_auto_throttle(struct pp_hwmgr *hwmgr)
 	return smu7_disable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal);
 }
 
-int smu7_pcie_performance_request(struct pp_hwmgr *hwmgr)
+static int smu7_pcie_performance_request(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	data->pcie_performance_request = true;
@@ -1161,7 +1154,7 @@ int smu7_pcie_performance_request(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smu7_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
+static int smu7_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
 {
 	int tmp_result = 0;
 	int result = 0;
@@ -1352,6 +1345,8 @@ static void smu7_init_dpm_defaults(struct pp_hwmgr *hwmgr)
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	struct phm_ppt_v1_information *table_info =
 			(struct phm_ppt_v1_information *)(hwmgr->pptable);
+	struct cgs_system_info sys_info = {0};
+	int result;
 
 	data->dll_default_on = false;
 	data->mclk_dpm0_activity_target = 0xa;
@@ -1426,7 +1421,7 @@ static void smu7_init_dpm_defaults(struct pp_hwmgr *hwmgr)
 		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 				PHM_PlatformCaps_ControlVDDCI);
 
-	if ((hwmgr->pp_table_version != PP_TABLE_V0)
+	if ((hwmgr->pp_table_version != PP_TABLE_V0) && (hwmgr->feature_mask & PP_CLOCK_STRETCH_MASK)
 		&& (table_info->cac_dtp_table->usClockStretchAmount != 0))
 		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
 					PHM_PlatformCaps_ClockStretcher);
@@ -1439,6 +1434,18 @@ static void smu7_init_dpm_defaults(struct pp_hwmgr *hwmgr)
 	data->pcie_lane_performance.min = 16;
 	data->pcie_lane_power_saving.max = 0;
 	data->pcie_lane_power_saving.min = 16;
+
+	sys_info.size = sizeof(struct cgs_system_info);
+	sys_info.info_id = CGS_SYSTEM_INFO_PG_FLAGS;
+	result = cgs_query_system_info(hwmgr->device, &sys_info);
+	if (!result) {
+		if (sys_info.value & AMD_PG_SUPPORT_UVD)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				      PHM_PlatformCaps_UVDPowerGating);
+		if (sys_info.value & AMD_PG_SUPPORT_VCE)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+				      PHM_PlatformCaps_VCEPowerGating);
+	}
 }
 
 /**
@@ -1865,7 +1872,7 @@ static int smu7_set_private_data_based_on_pptable_v1(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smu7_patch_voltage_workaround(struct pp_hwmgr *hwmgr)
+static int smu7_patch_voltage_workaround(struct pp_hwmgr *hwmgr)
 {
 	struct phm_ppt_v1_information *table_info =
 		       (struct phm_ppt_v1_information *)(hwmgr->pptable);
@@ -1994,8 +2001,9 @@ static int smu7_thermal_parameter_init(struct pp_hwmgr *hwmgr)
 
 		hwmgr->dyn_state.cac_dtp_table->usTargetOperatingTemp =
 			       table_info->cac_dtp_table->usTargetOperatingTemp;
-		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-						PHM_PlatformCaps_ODFuzzyFanControlSupport);
+		if (hwmgr->feature_mask & PP_OD_FUZZY_FAN_CONTROL_MASK)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+					PHM_PlatformCaps_ODFuzzyFanControlSupport);
 	}
 
 	return 0;
@@ -2259,7 +2267,7 @@ static int smu7_set_private_data_based_on_pptable_v0(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smu7_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
+static int smu7_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data;
 	int result;
@@ -3678,14 +3686,16 @@ static int smu7_set_max_fan_pwm_output(struct pp_hwmgr *hwmgr, uint16_t us_max_f
 			PPSMC_MSG_SetFanPwmMax, us_max_fan_pwm);
 }
 
-int smu7_notify_smc_display_change(struct pp_hwmgr *hwmgr, bool has_display)
+static int
+smu7_notify_smc_display_change(struct pp_hwmgr *hwmgr, bool has_display)
 {
 	PPSMC_Msg msg = has_display ? (PPSMC_Msg)PPSMC_HasDisplay : (PPSMC_Msg)PPSMC_NoDisplay;
 
 	return (smum_send_msg_to_smc(hwmgr->smumgr, msg) == 0) ?  0 : -1;
 }
 
-int smu7_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr)
+static int
+smu7_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr)
 {
 	uint32_t num_active_displays = 0;
 	struct cgs_display_info info = {0};
@@ -3707,7 +3717,7 @@ int smu7_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwmgr)
 * @param    hwmgr  the address of the powerplay hardware manager.
 * @return   always OK
 */
-int smu7_program_display_gap(struct pp_hwmgr *hwmgr)
+static int smu7_program_display_gap(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	uint32_t num_active_displays = 0;
@@ -3757,7 +3767,7 @@ int smu7_program_display_gap(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smu7_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
+static int smu7_display_configuration_changed_task(struct pp_hwmgr *hwmgr)
 {
 	return smu7_program_display_gap(hwmgr);
 }
@@ -3781,13 +3791,14 @@ static int smu7_set_max_fan_rpm_output(struct pp_hwmgr *hwmgr, uint16_t us_max_f
 			PPSMC_MSG_SetFanRpmMax, us_max_fan_rpm);
 }
 
-int smu7_register_internal_thermal_interrupt(struct pp_hwmgr *hwmgr,
+static int smu7_register_internal_thermal_interrupt(struct pp_hwmgr *hwmgr,
 					const void *thermal_interrupt_info)
 {
 	return 0;
 }
 
-bool smu7_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmgr)
+static bool
+smu7_check_smc_update_required_for_display_configuration(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 	bool is_update_required = false;
@@ -3816,7 +3827,9 @@ static inline bool smu7_are_power_levels_equal(const struct smu7_performance_lev
 		  (pl1->pcie_lane == pl2->pcie_lane));
 }
 
-int smu7_check_states_equal(struct pp_hwmgr *hwmgr, const struct pp_hw_power_state *pstate1, const struct pp_hw_power_state *pstate2, bool *equal)
+static int smu7_check_states_equal(struct pp_hwmgr *hwmgr,
+		const struct pp_hw_power_state *pstate1,
+		const struct pp_hw_power_state *pstate2, bool *equal)
 {
 	const struct smu7_power_state *psa;
 	const struct smu7_power_state *psb;
@@ -3849,7 +3862,7 @@ int smu7_check_states_equal(struct pp_hwmgr *hwmgr, const struct pp_hw_power_sta
 	return 0;
 }
 
-int smu7_upload_mc_firmware(struct pp_hwmgr *hwmgr)
+static int smu7_upload_mc_firmware(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
@@ -3978,7 +3991,7 @@ static int smu7_init_sclk_threshold(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int smu7_setup_asic_task(struct pp_hwmgr *hwmgr)
+static int smu7_setup_asic_task(struct pp_hwmgr *hwmgr)
 {
 	int tmp_result, result = 0;
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
index 41b634ffa5b0..26477f0f09dc 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_powertune.c
@@ -603,9 +603,10 @@ int smu7_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n)
 	return 0;
 }
 
-static int smu7_set_overdriver_target_tdp(struct pp_hwmgr *pHwMgr, uint32_t target_tdp)
+static int smu7_set_overdriver_target_tdp(struct pp_hwmgr *hwmgr,
+						uint32_t target_tdp)
 {
-	return smum_send_msg_to_smc_with_parameter(pHwMgr->smumgr,
+	return smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
 			PPSMC_MSG_OverDriveSetTargetTdp, target_tdp);
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
index 3fb5e57a378b..3a883e6c601a 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
@@ -29,6 +29,8 @@
 #include "amd_shared.h"
 #include "cgs_common.h"
 
+extern int amdgpu_dpm;
+
 enum amd_pp_sensors {
 	AMDGPU_PP_SENSOR_GFX_SCLK = 0,
 	AMDGPU_PP_SENSOR_VDDNB,
@@ -349,6 +351,7 @@ struct amd_powerplay_funcs {
 	int (*get_fan_control_mode)(void *handle);
 	int (*set_fan_speed_percent)(void *handle, uint32_t percent);
 	int (*get_fan_speed_percent)(void *handle, uint32_t *speed);
+	int (*get_fan_speed_rpm)(void *handle, uint32_t *rpm);
 	int (*get_pp_num_states)(void *handle, struct pp_states_info *data);
 	int (*get_pp_table)(void *handle, char **table);
 	int (*set_pp_table)(void *handle, const char *buf, size_t size);
@@ -359,6 +362,7 @@ struct amd_powerplay_funcs {
 	int (*get_mclk_od)(void *handle);
 	int (*set_mclk_od)(void *handle, uint32_t value);
 	int (*read_sensor)(void *handle, int idx, int32_t *value);
+	struct amd_vce_state* (*get_vce_clock_state)(void *handle, unsigned idx);
 };
 
 struct amd_powerplay {
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
index d4495839c64c..26129972f686 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
@@ -334,6 +334,7 @@ struct phm_clocks {
 	uint32_t clock[MAX_NUM_CLOCKS];
 };
 
+extern int phm_disable_clock_power_gatings(struct pp_hwmgr *hwmgr);
 extern int phm_enable_clock_power_gatings(struct pp_hwmgr *hwmgr);
 extern int phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool gate);
 extern int phm_powergate_vce(struct pp_hwmgr *hwmgr, bool gate);
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
index 4f0fedd1e9d3..6cdb7cbf515e 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
@@ -38,8 +38,6 @@ struct pp_hwmgr;
 struct phm_fan_speed_info;
 struct pp_atomctrl_voltage_table;
 
-extern int amdgpu_powercontainment;
-extern int amdgpu_sclk_deep_sleep_en;
 extern unsigned amdgpu_pp_feature_mask;
 
 #define VOLTAGE_SCALE 4
@@ -85,7 +83,9 @@ enum PP_FEATURE_MASK {
 	PP_SMC_VOLTAGE_CONTROL_MASK = 0x40,
 	PP_VBI_TIME_SUPPORT_MASK = 0x80,
 	PP_ULV_MASK = 0x100,
-	PP_ENABLE_GFX_CG_THRU_SMU = 0x200
+	PP_ENABLE_GFX_CG_THRU_SMU = 0x200,
+	PP_CLOCK_STRETCH_MASK = 0x400,
+	PP_OD_FUZZY_FAN_CONTROL_MASK = 0x800
 };
 
 enum PHM_BackEnd_Magic {
@@ -367,7 +367,7 @@ struct pp_table_func {
 	int (*pptable_get_vce_state_table_entry)(
 						struct pp_hwmgr *hwmgr,
 						unsigned long i,
-						struct pp_vce_state *vce_state,
+						struct amd_vce_state *vce_state,
 						void **clock_info,
 						unsigned long *flag);
 };
@@ -586,18 +586,6 @@ struct phm_microcode_version_info {
 	uint32_t NB;
 };
 
-#define PP_MAX_VCE_LEVELS 6
-
-enum PP_VCE_LEVEL {
-	PP_VCE_LEVEL_AC_ALL = 0,     /* AC, All cases */
-	PP_VCE_LEVEL_DC_EE = 1,      /* DC, entropy encoding */
-	PP_VCE_LEVEL_DC_LL_LOW = 2,  /* DC, low latency queue, res <= 720 */
-	PP_VCE_LEVEL_DC_LL_HIGH = 3, /* DC, low latency queue, 1080 >= res > 720 */
-	PP_VCE_LEVEL_DC_GP_LOW = 4,  /* DC, general purpose queue, res <= 720 */
-	PP_VCE_LEVEL_DC_GP_HIGH = 5, /* DC, general purpose queue, 1080 >= res > 720 */
-};
-
-
 enum PP_TABLE_VERSION {
 	PP_TABLE_V0 = 0,
 	PP_TABLE_V1,
@@ -620,7 +608,7 @@ struct pp_hwmgr {
 	void *hardcode_pp_table;
 	bool need_pp_table_upload;
 
-	struct pp_vce_state vce_states[PP_MAX_VCE_LEVELS];
+	struct amd_vce_state vce_states[AMD_MAX_VCE_LEVELS];
 	uint32_t num_vce_state_tables;
 
 	enum amd_dpm_forced_level dpm_level;
diff --git a/drivers/gpu/drm/amd/powerplay/inc/power_state.h b/drivers/gpu/drm/amd/powerplay/inc/power_state.h
index 9ceaed9ac52a..827860fffe78 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/power_state.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/power_state.h
@@ -156,15 +156,6 @@ struct pp_power_state {
 	struct pp_hw_power_state  hardware;
 };
 
-
-/*Structure to hold a VCE state entry*/
-struct pp_vce_state {
-	uint32_t evclk;
-	uint32_t ecclk;
-	uint32_t sclk;
-	uint32_t mclk;
-};
-
 enum PP_MMProfilingState {
 	PP_MMProfilingState_NA = 0,
 	PP_MMProfilingState_Started,
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_acpi.h b/drivers/gpu/drm/amd/powerplay/inc/pp_acpi.h
index 3df5de2cdab0..8fe8ba9434ff 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/pp_acpi.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_acpi.h
@@ -21,9 +21,6 @@
  *
  */
 
-extern bool acpi_atcs_functions_supported(void *device,
-							uint32_t index);
-extern int acpi_pcie_perf_request(void *device,
-						uint8_t perf_req,
-						bool advertise);
-extern bool acpi_atcs_notify_pcie_device_ready(void *device);
+bool acpi_atcs_functions_supported(void *device, uint32_t index);
+int acpi_pcie_perf_request(void *device, uint8_t perf_req, bool advertise);
+bool acpi_atcs_notify_pcie_device_ready(void *device);
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c
index 76310ac7ef0d..6aeb1d20cc3b 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c
@@ -1958,6 +1958,12 @@ int fiji_thermal_setup_fan_table(struct pp_hwmgr *hwmgr)
 	int res;
 	uint64_t tmp64;
 
+	if (hwmgr->thermal_controller.fanInfo.bNoFan) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
 	if (smu_data->smu7_data.fan_table_start == 0) {
 		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 				PHM_PlatformCaps_MicrocodeFanControl);
@@ -2049,7 +2055,7 @@ int fiji_thermal_setup_fan_table(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-int fiji_program_mem_timing_parameters(struct pp_hwmgr *hwmgr)
+static int fiji_program_mem_timing_parameters(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
@@ -2125,7 +2131,7 @@ uint32_t fiji_get_offsetof(uint32_t type, uint32_t member)
 			return offsetof(SMU73_Discrete_DpmTable, LowSclkInterruptThreshold);
 		}
 	}
-	printk("cant't get the offset of type %x member %x \n", type, member);
+	printk(KERN_WARNING "can't get the offset of type %x member %x\n", type, member);
 	return 0;
 }
 
@@ -2150,7 +2156,7 @@ uint32_t fiji_get_mac_definition(uint32_t value)
 		return SMU73_MAX_LEVELS_MVDD;
 	}
 
-	printk("cant't get the mac of %x \n", value);
+	printk(KERN_WARNING "can't get the mac of %x\n", value);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
index 02fe1df855a9..26eff56b4a99 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
@@ -159,7 +159,7 @@ static int fiji_start_smu_in_non_protection_mode(struct pp_smumgr *smumgr)
 	return result;
 }
 
-int fiji_setup_pwr_virus(struct pp_smumgr *smumgr)
+static int fiji_setup_pwr_virus(struct pp_smumgr *smumgr)
 {
 	int i, result = -1;
 	uint32_t reg, data;
@@ -224,7 +224,7 @@ static int fiji_start_avfs_btc(struct pp_smumgr *smumgr)
 	return result;
 }
 
-int fiji_setup_pm_fuse_for_avfs(struct pp_smumgr *smumgr)
+static int fiji_setup_pm_fuse_for_avfs(struct pp_smumgr *smumgr)
 {
 	int result = 0;
 	uint32_t table_start;
@@ -260,7 +260,7 @@ int fiji_setup_pm_fuse_for_avfs(struct pp_smumgr *smumgr)
 	return result;
 }
 
-int fiji_setup_graphics_level_structure(struct pp_smumgr *smumgr)
+static int fiji_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 {
 	int32_t vr_config;
 	uint32_t table_start;
@@ -299,7 +299,7 @@ int fiji_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 }
 
 /* Work in Progress */
-int fiji_restore_vft_table(struct pp_smumgr *smumgr)
+static int fiji_restore_vft_table(struct pp_smumgr *smumgr)
 {
 	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
 
@@ -311,7 +311,7 @@ int fiji_restore_vft_table(struct pp_smumgr *smumgr)
 }
 
 /* Work in Progress */
-int fiji_save_vft_table(struct pp_smumgr *smumgr)
+static int fiji_save_vft_table(struct pp_smumgr *smumgr)
 {
 	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
 
@@ -322,7 +322,7 @@ int fiji_save_vft_table(struct pp_smumgr *smumgr)
 		return -EINVAL;
 }
 
-int fiji_avfs_event_mgr(struct pp_smumgr *smumgr, bool smu_started)
+static int fiji_avfs_event_mgr(struct pp_smumgr *smumgr, bool smu_started)
 {
 	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
 
@@ -396,7 +396,8 @@ static int fiji_start_smu(struct pp_smumgr *smumgr)
 	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
 
 	/* Only start SMC if SMC RAM is not running */
-	if (!smu7_is_smc_ram_running(smumgr)) {
+	if (!(smu7_is_smc_ram_running(smumgr)
+		|| cgs_is_virtualization_enabled(smumgr->device))) {
 		fiji_avfs_event_mgr(smumgr, false);
 
 		/* Check if SMU is running in protected mode */
@@ -443,6 +444,9 @@ static bool fiji_is_hw_avfs_present(struct pp_smumgr *smumgr)
 	uint32_t efuse = 0;
 	uint32_t mask = (1 << ((AVFS_EN_MSB - AVFS_EN_LSB) + 1)) - 1;
 
+	if (cgs_is_virtualization_enabled(smumgr->device))
+		return 0;
+
 	if (!atomctrl_read_efuse(smumgr->device, AVFS_EN_LSB, AVFS_EN_MSB,
 			mask, &efuse)) {
 		if (efuse)
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c
index 8c889caba420..a24971a33bfd 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smc.c
@@ -2006,6 +2006,12 @@ int iceland_thermal_setup_fan_table(struct pp_hwmgr *hwmgr)
 	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl))
 		return 0;
 
+	if (hwmgr->thermal_controller.fanInfo.bNoFan) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
 	if (0 == smu7_data->fan_table_start) {
 		phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_MicrocodeFanControl);
 		return 0;
@@ -2140,7 +2146,7 @@ uint32_t iceland_get_offsetof(uint32_t type, uint32_t member)
 			return offsetof(SMU71_Discrete_DpmTable, LowSclkInterruptThreshold);
 		}
 	}
-	printk("cant't get the offset of type %x member %x \n", type, member);
+	printk(KERN_WARNING "can't get the offset of type %x member %x\n", type, member);
 	return 0;
 }
 
@@ -2163,7 +2169,7 @@ uint32_t iceland_get_mac_definition(uint32_t value)
 		return SMU71_MAX_LEVELS_MVDD;
 	}
 
-	printk("cant't get the mac of %x \n", value);
+	printk(KERN_WARNING "can't get the mac of %x\n", value);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c
index 71bb2f8dc157..5190e821200c 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c
@@ -1885,6 +1885,12 @@ int polaris10_thermal_setup_fan_table(struct pp_hwmgr *hwmgr)
 	int res;
 	uint64_t tmp64;
 
+	if (hwmgr->thermal_controller.fanInfo.bNoFan) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
 	if (smu_data->smu7_data.fan_table_start == 0) {
 		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 				PHM_PlatformCaps_MicrocodeFanControl);
@@ -2174,7 +2180,7 @@ uint32_t polaris10_get_offsetof(uint32_t type, uint32_t member)
 			return offsetof(SMU74_Discrete_DpmTable, LowSclkInterruptThreshold);
 		}
 	}
-	printk("cant't get the offset of type %x member %x \n", type, member);
+	printk(KERN_WARNING "can't get the offset of type %x member %x\n", type, member);
 	return 0;
 }
 
@@ -2201,7 +2207,7 @@ uint32_t polaris10_get_mac_definition(uint32_t value)
 		return SMU7_UVD_MCLK_HANDSHAKE_DISABLE;
 	}
 
-	printk("cant't get the mac of %x \n", value);
+	printk(KERN_WARNING "can't get the mac of %x\n", value);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
index 5c3598ab7dae..f38a68747df0 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
@@ -118,7 +118,7 @@ static int polaris10_perform_btc(struct pp_smumgr *smumgr)
 }
 
 
-int polaris10_setup_graphics_level_structure(struct pp_smumgr *smumgr)
+static int polaris10_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 {
 	uint32_t vr_config;
 	uint32_t dpm_table_start;
@@ -172,7 +172,8 @@ int polaris10_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 	return 0;
 }
 
-int polaris10_avfs_event_mgr(struct pp_smumgr *smumgr, bool SMU_VFT_INTACT)
+static int
+polaris10_avfs_event_mgr(struct pp_smumgr *smumgr, bool SMU_VFT_INTACT)
 {
 	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
index 6af744f42ec9..f49b5487b951 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
@@ -278,6 +278,9 @@ enum cgs_ucode_id smu7_convert_fw_type_to_cgs(uint32_t fw_type)
 	case UCODE_ID_RLC_G:
 		result = CGS_UCODE_ID_RLC_G;
 		break;
+	case UCODE_ID_MEC_STORAGE:
+		result = CGS_UCODE_ID_STORAGE;
+		break;
 	default:
 		break;
 	}
@@ -363,12 +366,16 @@ static int smu7_populate_single_firmware_entry(struct pp_smumgr *smumgr,
 				&info);
 
 	if (!result) {
-		entry->version = info.version;
+		entry->version = info.fw_version;
 		entry->id = (uint16_t)fw_type;
 		entry->image_addr_high = smu_upper_32_bits(info.mc_addr);
 		entry->image_addr_low = smu_lower_32_bits(info.mc_addr);
 		entry->meta_data_addr_high = 0;
 		entry->meta_data_addr_low = 0;
+
+		/* digest need be excluded out */
+		if (cgs_is_virtualization_enabled(smumgr->device))
+			info.image_size -= 20;
 		entry->data_size_byte = info.image_size;
 		entry->num_register_entries = 0;
 	}
@@ -400,8 +407,14 @@ int smu7_request_smu_load_fw(struct pp_smumgr *smumgr)
 					0x0);
 
 	if (smumgr->chip_id > CHIP_TOPAZ) { /* add support for Topaz */
-		smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_SMU_DRAM_ADDR_HI, smu_data->smu_buffer.mc_addr_high);
-		smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_SMU_DRAM_ADDR_LO, smu_data->smu_buffer.mc_addr_low);
+		if (!cgs_is_virtualization_enabled(smumgr->device)) {
+			smu7_send_msg_to_smc_with_parameter(smumgr,
+						PPSMC_MSG_SMU_DRAM_ADDR_HI,
+						smu_data->smu_buffer.mc_addr_high);
+			smu7_send_msg_to_smc_with_parameter(smumgr,
+						PPSMC_MSG_SMU_DRAM_ADDR_LO,
+						smu_data->smu_buffer.mc_addr_low);
+		}
 		fw_to_load = UCODE_ID_RLC_G_MASK
 			   + UCODE_ID_SDMA0_MASK
 			   + UCODE_ID_SDMA1_MASK
@@ -452,6 +465,10 @@ int smu7_request_smu_load_fw(struct pp_smumgr *smumgr)
 	PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(smumgr,
 				UCODE_ID_SDMA1, &toc->entry[toc->num_entries++]),
 				"Failed to Get Firmware Entry.", return -EINVAL);
+	if (cgs_is_virtualization_enabled(smumgr->device))
+		PP_ASSERT_WITH_CODE(0 == smu7_populate_single_firmware_entry(smumgr,
+				UCODE_ID_MEC_STORAGE, &toc->entry[toc->num_entries++]),
+				"Failed to Get Firmware Entry.", return -EINVAL);
 
 	smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_DRV_DRAM_ADDR_HI, smu_data->header_buffer.mc_addr_high);
 	smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_DRV_DRAM_ADDR_LO, smu_data->header_buffer.mc_addr_low);
@@ -532,7 +549,6 @@ int smu7_init(struct pp_smumgr *smumgr)
 	smu_data = (struct smu7_smumgr *)(smumgr->backend);
 	smu_data->header_buffer.data_size =
 			((sizeof(struct SMU_DRAMData_TOC) / 4096) + 1) * 4096;
-	smu_data->smu_buffer.data_size = 200*4096;
 
 /* Allocate FW image data structure and header buffer and
  * send the header buffer address to SMU */
@@ -555,6 +571,10 @@ int smu7_init(struct pp_smumgr *smumgr)
 		(cgs_handle_t)smu_data->header_buffer.handle);
 		return -EINVAL);
 
+	if (cgs_is_virtualization_enabled(smumgr->device))
+		return 0;
+
+	smu_data->smu_buffer.data_size = 200*4096;
 	smu_allocate_memory(smumgr->device,
 		smu_data->smu_buffer.data_size,
 		CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB,
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
index 76352f2423ae..919be435b49c 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
@@ -28,8 +28,6 @@
 #include <pp_endian.h>
 
 #define SMC_RAM_END 0x40000
-#define mmSMC_IND_INDEX_11                              0x01AC
-#define mmSMC_IND_DATA_11                               0x01AD
 
 struct smu7_buffer_entry {
 	uint32_t data_size;
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c
index de2a24d85f48..2e1493ce1bb5 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smc.c
@@ -2496,6 +2496,12 @@ int tonga_thermal_setup_fan_table(struct pp_hwmgr *hwmgr)
 					PHM_PlatformCaps_MicrocodeFanControl))
 		return 0;
 
+	if (hwmgr->thermal_controller.fanInfo.bNoFan) {
+		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_MicrocodeFanControl);
+		return 0;
+	}
+
 	if (0 == smu_data->smu7_data.fan_table_start) {
 		phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 					PHM_PlatformCaps_MicrocodeFanControl);
@@ -2651,7 +2657,7 @@ uint32_t tonga_get_offsetof(uint32_t type, uint32_t member)
 			return offsetof(SMU72_Discrete_DpmTable, LowSclkInterruptThreshold);
 		}
 	}
-	printk("cant't get the offset of type %x member %x\n", type, member);
+	printk(KERN_WARNING "can't get the offset of type %x member %x\n", type, member);
 	return 0;
 }
 
@@ -2675,7 +2681,7 @@ uint32_t tonga_get_mac_definition(uint32_t value)
 	case SMU_MAX_LEVELS_MVDD:
 		return SMU72_MAX_LEVELS_MVDD;
 	}
-	printk("cant't get the mac value %x\n", value);
+	printk(KERN_WARNING "can't get the mac value %x\n", value);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
index 5f9124046b9b..eff9a232e72e 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
@@ -140,7 +140,8 @@ static int tonga_start_smu(struct pp_smumgr *smumgr)
 	int result;
 
 	/* Only start SMC if SMC RAM is not running */
-	if (!smu7_is_smc_ram_running(smumgr)) {
+	if (!(smu7_is_smc_ram_running(smumgr) ||
+		cgs_is_virtualization_enabled(smumgr->device))) {
 		/*Check if SMU is running in protected mode*/
 		if (0 == SMUM_READ_VFPF_INDIRECT_FIELD(smumgr->device, CGS_IND_REG__SMC,
 					SMU_FIRMWARE, SMU_MODE)) {
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h b/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h
index b961a1c6caf3..dbd4fd3a810b 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h
+++ b/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h
@@ -17,7 +17,7 @@ TRACE_EVENT(amd_sched_job,
 	    TP_STRUCT__entry(
 			     __field(struct amd_sched_entity *, entity)
 			     __field(struct amd_sched_job *, sched_job)
-			     __field(struct fence *, fence)
+			     __field(struct dma_fence *, fence)
 			     __field(const char *, name)
 			     __field(u32, job_count)
 			     __field(int, hw_job_count)
@@ -42,7 +42,7 @@ TRACE_EVENT(amd_sched_process_job,
 	    TP_PROTO(struct amd_sched_fence *fence),
 	    TP_ARGS(fence),
 	    TP_STRUCT__entry(
-		    __field(struct fence *, fence)
+		    __field(struct dma_fence *, fence)
 		    ),
 
 	    TP_fast_assign(
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
index ffe1f85ce300..1bf83ed113b3 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
+++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
@@ -32,7 +32,7 @@
 
 static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity);
 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
-static void amd_sched_process_job(struct fence *f, struct fence_cb *cb);
+static void amd_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
 
 /* Initialize a given run queue struct */
 static void amd_sched_rq_init(struct amd_sched_rq *rq)
@@ -138,7 +138,7 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
 		return r;
 
 	atomic_set(&entity->fence_seq, 0);
-	entity->fence_context = fence_context_alloc(2);
+	entity->fence_context = dma_fence_context_alloc(2);
 
 	return 0;
 }
@@ -218,32 +218,32 @@ void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
 	kfifo_free(&entity->job_queue);
 }
 
-static void amd_sched_entity_wakeup(struct fence *f, struct fence_cb *cb)
+static void amd_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
 {
 	struct amd_sched_entity *entity =
 		container_of(cb, struct amd_sched_entity, cb);
 	entity->dependency = NULL;
-	fence_put(f);
+	dma_fence_put(f);
 	amd_sched_wakeup(entity->sched);
 }
 
-static void amd_sched_entity_clear_dep(struct fence *f, struct fence_cb *cb)
+static void amd_sched_entity_clear_dep(struct dma_fence *f, struct dma_fence_cb *cb)
 {
 	struct amd_sched_entity *entity =
 		container_of(cb, struct amd_sched_entity, cb);
 	entity->dependency = NULL;
-	fence_put(f);
+	dma_fence_put(f);
 }
 
 static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
 {
 	struct amd_gpu_scheduler *sched = entity->sched;
-	struct fence * fence = entity->dependency;
+	struct dma_fence * fence = entity->dependency;
 	struct amd_sched_fence *s_fence;
 
 	if (fence->context == entity->fence_context) {
 		/* We can ignore fences from ourself */
-		fence_put(entity->dependency);
+		dma_fence_put(entity->dependency);
 		return false;
 	}
 
@@ -254,23 +254,23 @@ static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
 		 * Fence is from the same scheduler, only need to wait for
 		 * it to be scheduled
 		 */
-		fence = fence_get(&s_fence->scheduled);
-		fence_put(entity->dependency);
+		fence = dma_fence_get(&s_fence->scheduled);
+		dma_fence_put(entity->dependency);
 		entity->dependency = fence;
-		if (!fence_add_callback(fence, &entity->cb,
-					amd_sched_entity_clear_dep))
+		if (!dma_fence_add_callback(fence, &entity->cb,
+					    amd_sched_entity_clear_dep))
 			return true;
 
 		/* Ignore it when it is already scheduled */
-		fence_put(fence);
+		dma_fence_put(fence);
 		return false;
 	}
 
-	if (!fence_add_callback(entity->dependency, &entity->cb,
-				amd_sched_entity_wakeup))
+	if (!dma_fence_add_callback(entity->dependency, &entity->cb,
+				    amd_sched_entity_wakeup))
 		return true;
 
-	fence_put(entity->dependency);
+	dma_fence_put(entity->dependency);
 	return false;
 }
 
@@ -351,7 +351,8 @@ static void amd_sched_job_finish(struct work_struct *work)
 	sched->ops->free_job(s_job);
 }
 
-static void amd_sched_job_finish_cb(struct fence *f, struct fence_cb *cb)
+static void amd_sched_job_finish_cb(struct dma_fence *f,
+				    struct dma_fence_cb *cb)
 {
 	struct amd_sched_job *job = container_of(cb, struct amd_sched_job,
 						 finish_cb);
@@ -385,8 +386,8 @@ void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched)
 
 	spin_lock(&sched->job_list_lock);
 	list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) {
-		if (fence_remove_callback(s_job->s_fence->parent, &s_job->s_fence->cb)) {
-			fence_put(s_job->s_fence->parent);
+		if (dma_fence_remove_callback(s_job->s_fence->parent, &s_job->s_fence->cb)) {
+			dma_fence_put(s_job->s_fence->parent);
 			s_job->s_fence->parent = NULL;
 		}
 	}
@@ -407,21 +408,21 @@ void amd_sched_job_recovery(struct amd_gpu_scheduler *sched)
 
 	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
 		struct amd_sched_fence *s_fence = s_job->s_fence;
-		struct fence *fence;
+		struct dma_fence *fence;
 
 		spin_unlock(&sched->job_list_lock);
 		fence = sched->ops->run_job(s_job);
 		atomic_inc(&sched->hw_rq_count);
 		if (fence) {
-			s_fence->parent = fence_get(fence);
-			r = fence_add_callback(fence, &s_fence->cb,
-					       amd_sched_process_job);
+			s_fence->parent = dma_fence_get(fence);
+			r = dma_fence_add_callback(fence, &s_fence->cb,
+						   amd_sched_process_job);
 			if (r == -ENOENT)
 				amd_sched_process_job(fence, &s_fence->cb);
 			else if (r)
 				DRM_ERROR("fence add callback failed (%d)\n",
 					  r);
-			fence_put(fence);
+			dma_fence_put(fence);
 		} else {
 			DRM_ERROR("Failed to run job!\n");
 			amd_sched_process_job(NULL, &s_fence->cb);
@@ -443,8 +444,8 @@ void amd_sched_entity_push_job(struct amd_sched_job *sched_job)
 	struct amd_sched_entity *entity = sched_job->s_entity;
 
 	trace_amd_sched_job(sched_job);
-	fence_add_callback(&sched_job->s_fence->finished, &sched_job->finish_cb,
-			   amd_sched_job_finish_cb);
+	dma_fence_add_callback(&sched_job->s_fence->finished, &sched_job->finish_cb,
+			       amd_sched_job_finish_cb);
 	wait_event(entity->sched->job_scheduled,
 		   amd_sched_entity_in(sched_job));
 }
@@ -508,7 +509,7 @@ amd_sched_select_entity(struct amd_gpu_scheduler *sched)
 	return entity;
 }
 
-static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
+static void amd_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)
 {
 	struct amd_sched_fence *s_fence =
 		container_of(cb, struct amd_sched_fence, cb);
@@ -518,7 +519,7 @@ static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
 	amd_sched_fence_finished(s_fence);
 
 	trace_amd_sched_process_job(s_fence);
-	fence_put(&s_fence->finished);
+	dma_fence_put(&s_fence->finished);
 	wake_up_interruptible(&sched->wake_up_worker);
 }
 
@@ -544,7 +545,7 @@ static int amd_sched_main(void *param)
 		struct amd_sched_entity *entity = NULL;
 		struct amd_sched_fence *s_fence;
 		struct amd_sched_job *sched_job;
-		struct fence *fence;
+		struct dma_fence *fence;
 
 		wait_event_interruptible(sched->wake_up_worker,
 					 (!amd_sched_blocked(sched) &&
@@ -566,15 +567,15 @@ static int amd_sched_main(void *param)
 		fence = sched->ops->run_job(sched_job);
 		amd_sched_fence_scheduled(s_fence);
 		if (fence) {
-			s_fence->parent = fence_get(fence);
-			r = fence_add_callback(fence, &s_fence->cb,
-					       amd_sched_process_job);
+			s_fence->parent = dma_fence_get(fence);
+			r = dma_fence_add_callback(fence, &s_fence->cb,
+						   amd_sched_process_job);
 			if (r == -ENOENT)
 				amd_sched_process_job(fence, &s_fence->cb);
 			else if (r)
 				DRM_ERROR("fence add callback failed (%d)\n",
 					  r);
-			fence_put(fence);
+			dma_fence_put(fence);
 		} else {
 			DRM_ERROR("Failed to run job!\n");
 			amd_sched_process_job(NULL, &s_fence->cb);
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
index 51068e6c3d9a..d8dc681bcda6 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
+++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
@@ -25,7 +25,7 @@
 #define _GPU_SCHEDULER_H_
 
 #include <linux/kfifo.h>
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 
 struct amd_gpu_scheduler;
 struct amd_sched_rq;
@@ -47,8 +47,8 @@ struct amd_sched_entity {
 	atomic_t			fence_seq;
 	uint64_t                        fence_context;
 
-	struct fence			*dependency;
-	struct fence_cb			cb;
+	struct dma_fence		*dependency;
+	struct dma_fence_cb		cb;
 };
 
 /**
@@ -63,10 +63,10 @@ struct amd_sched_rq {
 };
 
 struct amd_sched_fence {
-	struct fence                    scheduled;
-	struct fence                    finished;
-	struct fence_cb                 cb;
-	struct fence                    *parent;
+	struct dma_fence                scheduled;
+	struct dma_fence                finished;
+	struct dma_fence_cb             cb;
+	struct dma_fence                *parent;
 	struct amd_gpu_scheduler	*sched;
 	spinlock_t			lock;
 	void                            *owner;
@@ -76,15 +76,15 @@ struct amd_sched_job {
 	struct amd_gpu_scheduler        *sched;
 	struct amd_sched_entity         *s_entity;
 	struct amd_sched_fence          *s_fence;
-	struct fence_cb			finish_cb;
+	struct dma_fence_cb		finish_cb;
 	struct work_struct		finish_work;
 	struct list_head		node;
 	struct delayed_work		work_tdr;
 };
 
-extern const struct fence_ops amd_sched_fence_ops_scheduled;
-extern const struct fence_ops amd_sched_fence_ops_finished;
-static inline struct amd_sched_fence *to_amd_sched_fence(struct fence *f)
+extern const struct dma_fence_ops amd_sched_fence_ops_scheduled;
+extern const struct dma_fence_ops amd_sched_fence_ops_finished;
+static inline struct amd_sched_fence *to_amd_sched_fence(struct dma_fence *f)
 {
 	if (f->ops == &amd_sched_fence_ops_scheduled)
 		return container_of(f, struct amd_sched_fence, scheduled);
@@ -100,8 +100,8 @@ static inline struct amd_sched_fence *to_amd_sched_fence(struct fence *f)
  * these functions should be implemented in driver side
 */
 struct amd_sched_backend_ops {
-	struct fence *(*dependency)(struct amd_sched_job *sched_job);
-	struct fence *(*run_job)(struct amd_sched_job *sched_job);
+	struct dma_fence *(*dependency)(struct amd_sched_job *sched_job);
+	struct dma_fence *(*run_job)(struct amd_sched_job *sched_job);
 	void (*timedout_job)(struct amd_sched_job *sched_job);
 	void (*free_job)(struct amd_sched_job *sched_job);
 };
diff --git a/drivers/gpu/drm/amd/scheduler/sched_fence.c b/drivers/gpu/drm/amd/scheduler/sched_fence.c
index 88fc2d662579..33f54d0a5c4f 100644
--- a/drivers/gpu/drm/amd/scheduler/sched_fence.c
+++ b/drivers/gpu/drm/amd/scheduler/sched_fence.c
@@ -61,46 +61,50 @@ struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *entity,
 	spin_lock_init(&fence->lock);
 
 	seq = atomic_inc_return(&entity->fence_seq);
-	fence_init(&fence->scheduled, &amd_sched_fence_ops_scheduled,
-		   &fence->lock, entity->fence_context, seq);
-	fence_init(&fence->finished, &amd_sched_fence_ops_finished,
-		   &fence->lock, entity->fence_context + 1, seq);
+	dma_fence_init(&fence->scheduled, &amd_sched_fence_ops_scheduled,
+		       &fence->lock, entity->fence_context, seq);
+	dma_fence_init(&fence->finished, &amd_sched_fence_ops_finished,
+		       &fence->lock, entity->fence_context + 1, seq);
 
 	return fence;
 }
 
 void amd_sched_fence_scheduled(struct amd_sched_fence *fence)
 {
-	int ret = fence_signal(&fence->scheduled);
+	int ret = dma_fence_signal(&fence->scheduled);
 
 	if (!ret)
-		FENCE_TRACE(&fence->scheduled, "signaled from irq context\n");
+		DMA_FENCE_TRACE(&fence->scheduled,
+				"signaled from irq context\n");
 	else
-		FENCE_TRACE(&fence->scheduled, "was already signaled\n");
+		DMA_FENCE_TRACE(&fence->scheduled,
+				"was already signaled\n");
 }
 
 void amd_sched_fence_finished(struct amd_sched_fence *fence)
 {
-	int ret = fence_signal(&fence->finished);
+	int ret = dma_fence_signal(&fence->finished);
 
 	if (!ret)
-		FENCE_TRACE(&fence->finished, "signaled from irq context\n");
+		DMA_FENCE_TRACE(&fence->finished,
+				"signaled from irq context\n");
 	else
-		FENCE_TRACE(&fence->finished, "was already signaled\n");
+		DMA_FENCE_TRACE(&fence->finished,
+				"was already signaled\n");
 }
 
-static const char *amd_sched_fence_get_driver_name(struct fence *fence)
+static const char *amd_sched_fence_get_driver_name(struct dma_fence *fence)
 {
 	return "amd_sched";
 }
 
-static const char *amd_sched_fence_get_timeline_name(struct fence *f)
+static const char *amd_sched_fence_get_timeline_name(struct dma_fence *f)
 {
 	struct amd_sched_fence *fence = to_amd_sched_fence(f);
 	return (const char *)fence->sched->name;
 }
 
-static bool amd_sched_fence_enable_signaling(struct fence *f)
+static bool amd_sched_fence_enable_signaling(struct dma_fence *f)
 {
 	return true;
 }
@@ -114,10 +118,10 @@ static bool amd_sched_fence_enable_signaling(struct fence *f)
  */
 static void amd_sched_fence_free(struct rcu_head *rcu)
 {
-	struct fence *f = container_of(rcu, struct fence, rcu);
+	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
 	struct amd_sched_fence *fence = to_amd_sched_fence(f);
 
-	fence_put(fence->parent);
+	dma_fence_put(fence->parent);
 	kmem_cache_free(sched_fence_slab, fence);
 }
 
@@ -129,7 +133,7 @@ static void amd_sched_fence_free(struct rcu_head *rcu)
  * This function is called when the reference count becomes zero.
  * It just RCU schedules freeing up the fence.
  */
-static void amd_sched_fence_release_scheduled(struct fence *f)
+static void amd_sched_fence_release_scheduled(struct dma_fence *f)
 {
 	struct amd_sched_fence *fence = to_amd_sched_fence(f);
 
@@ -143,27 +147,27 @@ static void amd_sched_fence_release_scheduled(struct fence *f)
  *
  * Drop the extra reference from the scheduled fence to the base fence.
  */
-static void amd_sched_fence_release_finished(struct fence *f)
+static void amd_sched_fence_release_finished(struct dma_fence *f)
 {
 	struct amd_sched_fence *fence = to_amd_sched_fence(f);
 
-	fence_put(&fence->scheduled);
+	dma_fence_put(&fence->scheduled);
 }
 
-const struct fence_ops amd_sched_fence_ops_scheduled = {
+const struct dma_fence_ops amd_sched_fence_ops_scheduled = {
 	.get_driver_name = amd_sched_fence_get_driver_name,
 	.get_timeline_name = amd_sched_fence_get_timeline_name,
 	.enable_signaling = amd_sched_fence_enable_signaling,
 	.signaled = NULL,
-	.wait = fence_default_wait,
+	.wait = dma_fence_default_wait,
 	.release = amd_sched_fence_release_scheduled,
 };
 
-const struct fence_ops amd_sched_fence_ops_finished = {
+const struct dma_fence_ops amd_sched_fence_ops_finished = {
 	.get_driver_name = amd_sched_fence_get_driver_name,
 	.get_timeline_name = amd_sched_fence_get_timeline_name,
 	.enable_signaling = amd_sched_fence_enable_signaling,
 	.signaled = NULL,
-	.wait = fence_default_wait,
+	.wait = dma_fence_default_wait,
 	.release = amd_sched_fence_release_finished,
 };
diff --git a/drivers/gpu/drm/arc/arcpgu_drv.c b/drivers/gpu/drm/arc/arcpgu_drv.c
index 28e6471257d0..0b6eaa49a1db 100644
--- a/drivers/gpu/drm/arc/arcpgu_drv.c
+++ b/drivers/gpu/drm/arc/arcpgu_drv.c
@@ -65,9 +65,7 @@ static const struct file_operations arcpgu_drm_ops = {
 	.open = drm_open,
 	.release = drm_release,
 	.unlocked_ioctl = drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.poll = drm_poll,
 	.read = drm_read,
 	.llseek = no_llseek,
diff --git a/drivers/gpu/drm/arc/arcpgu_sim.c b/drivers/gpu/drm/arc/arcpgu_sim.c
index 2bf06d71556a..bca3a678c955 100644
--- a/drivers/gpu/drm/arc/arcpgu_sim.c
+++ b/drivers/gpu/drm/arc/arcpgu_sim.c
@@ -41,12 +41,6 @@ static int arcpgu_drm_connector_get_modes(struct drm_connector *connector)
 	return count;
 }
 
-static enum drm_connector_status
-arcpgu_drm_connector_detect(struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static void arcpgu_drm_connector_destroy(struct drm_connector *connector)
 {
 	drm_connector_unregister(connector);
@@ -61,7 +55,6 @@ arcpgu_drm_connector_helper_funcs = {
 static const struct drm_connector_funcs arcpgu_drm_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.reset = drm_atomic_helper_connector_reset,
-	.detect = arcpgu_drm_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = arcpgu_drm_connector_destroy,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/arm/hdlcd_crtc.c b/drivers/gpu/drm/arm/hdlcd_crtc.c
index 28341b32067f..7d4e5aa77195 100644
--- a/drivers/gpu/drm/arm/hdlcd_crtc.c
+++ b/drivers/gpu/drm/arm/hdlcd_crtc.c
@@ -222,14 +222,12 @@ static void hdlcd_plane_atomic_update(struct drm_plane *plane,
 {
 	struct hdlcd_drm_private *hdlcd;
 	struct drm_gem_cma_object *gem;
-	unsigned int depth, bpp;
 	u32 src_w, src_h, dest_w, dest_h;
 	dma_addr_t scanout_start;
 
 	if (!plane->state->fb)
 		return;
 
-	drm_fb_get_bpp_depth(plane->state->fb->pixel_format, &depth, &bpp);
 	src_w = plane->state->src_w >> 16;
 	src_h = plane->state->src_h >> 16;
 	dest_w = plane->state->crtc_w;
@@ -237,7 +235,8 @@ static void hdlcd_plane_atomic_update(struct drm_plane *plane,
 	gem = drm_fb_cma_get_gem_obj(plane->state->fb, 0);
 	scanout_start = gem->paddr + plane->state->fb->offsets[0] +
 		plane->state->crtc_y * plane->state->fb->pitches[0] +
-		plane->state->crtc_x * bpp / 8;
+		plane->state->crtc_x *
+		drm_format_plane_cpp(plane->state->fb->pixel_format, 0);
 
 	hdlcd = plane->dev->dev_private;
 	hdlcd_write(hdlcd, HDLCD_REG_FB_LINE_LENGTH, plane->state->fb->pitches[0]);
diff --git a/drivers/gpu/drm/arm/hdlcd_drv.c b/drivers/gpu/drm/arm/hdlcd_drv.c
index e138fb51e8ce..e5f4f4a6546d 100644
--- a/drivers/gpu/drm/arm/hdlcd_drv.c
+++ b/drivers/gpu/drm/arm/hdlcd_drv.c
@@ -268,9 +268,7 @@ static const struct file_operations fops = {
 	.open		= drm_open,
 	.release	= drm_release,
 	.unlocked_ioctl	= drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl	= drm_compat_ioctl,
-#endif
 	.poll		= drm_poll,
 	.read		= drm_read,
 	.llseek		= noop_llseek,
@@ -337,14 +335,10 @@ static int hdlcd_drm_bind(struct device *dev)
 	if (ret)
 		goto err_free;
 
-	ret = drm_dev_register(drm, 0);
-	if (ret)
-		goto err_unload;
-
 	ret = component_bind_all(dev, drm);
 	if (ret) {
 		DRM_ERROR("Failed to bind all components\n");
-		goto err_unregister;
+		goto err_unload;
 	}
 
 	ret = pm_runtime_set_active(dev);
@@ -371,8 +365,17 @@ static int hdlcd_drm_bind(struct device *dev)
 		goto err_fbdev;
 	}
 
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		goto err_register;
+
 	return 0;
 
+err_register:
+	if (hdlcd->fbdev) {
+		drm_fbdev_cma_fini(hdlcd->fbdev);
+		hdlcd->fbdev = NULL;
+	}
 err_fbdev:
 	drm_kms_helper_poll_fini(drm);
 	drm_vblank_cleanup(drm);
@@ -380,8 +383,6 @@ err_vblank:
 	pm_runtime_disable(drm->dev);
 err_pm_active:
 	component_unbind_all(dev, drm);
-err_unregister:
-	drm_dev_unregister(drm);
 err_unload:
 	drm_irq_uninstall(drm);
 	of_reserved_mem_device_release(drm->dev);
@@ -398,6 +399,7 @@ static void hdlcd_drm_unbind(struct device *dev)
 	struct drm_device *drm = dev_get_drvdata(dev);
 	struct hdlcd_drm_private *hdlcd = drm->dev_private;
 
+	drm_dev_unregister(drm);
 	if (hdlcd->fbdev) {
 		drm_fbdev_cma_fini(hdlcd->fbdev);
 		hdlcd->fbdev = NULL;
@@ -411,7 +413,6 @@ static void hdlcd_drm_unbind(struct device *dev)
 	pm_runtime_disable(drm->dev);
 	of_reserved_mem_device_release(drm->dev);
 	drm_mode_config_cleanup(drm);
-	drm_dev_unregister(drm);
 	drm_dev_unref(drm);
 	drm->dev_private = NULL;
 	dev_set_drvdata(dev, NULL);
@@ -453,7 +454,8 @@ static int hdlcd_probe(struct platform_device *pdev)
 		return -EAGAIN;
 	}
 
-	component_match_add(&pdev->dev, &match, compare_dev, port);
+	drm_of_component_match_add(&pdev->dev, &match, compare_dev, port);
+	of_node_put(port);
 
 	return component_master_add_with_match(&pdev->dev, &hdlcd_master_ops,
 					       match);
diff --git a/drivers/gpu/drm/arm/malidp_drv.c b/drivers/gpu/drm/arm/malidp_drv.c
index 9280358b8f15..32f746e31379 100644
--- a/drivers/gpu/drm/arm/malidp_drv.c
+++ b/drivers/gpu/drm/arm/malidp_drv.c
@@ -42,6 +42,7 @@ static int malidp_set_and_wait_config_valid(struct drm_device *drm)
 	struct malidp_hw_device *hwdev = malidp->dev;
 	int ret;
 
+	atomic_set(&malidp->config_valid, 0);
 	hwdev->set_config_valid(hwdev);
 	/* don't wait for config_valid flag if we are in config mode */
 	if (hwdev->in_config_mode(hwdev))
@@ -91,8 +92,7 @@ static void malidp_atomic_commit_tail(struct drm_atomic_state *state)
 
 	drm_atomic_helper_commit_modeset_disables(drm, state);
 	drm_atomic_helper_commit_modeset_enables(drm, state);
-	drm_atomic_helper_commit_planes(drm, state,
-					DRM_PLANE_COMMIT_ACTIVE_ONLY);
+	drm_atomic_helper_commit_planes(drm, state, 0);
 
 	malidp_atomic_commit_hw_done(state);
 
@@ -155,6 +155,12 @@ static int malidp_init(struct drm_device *drm)
 	return 0;
 }
 
+static void malidp_fini(struct drm_device *drm)
+{
+	malidp_de_planes_destroy(drm);
+	drm_mode_config_cleanup(drm);
+}
+
 static int malidp_irq_init(struct platform_device *pdev)
 {
 	int irq_de, irq_se, ret = 0;
@@ -197,9 +203,7 @@ static const struct file_operations fops = {
 	.open = drm_open,
 	.release = drm_release,
 	.unlocked_ioctl = drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.poll = drm_poll,
 	.read = drm_read,
 	.llseek = noop_llseek,
@@ -355,10 +359,6 @@ static int malidp_bind(struct device *dev)
 	if (ret < 0)
 		goto init_fail;
 
-	ret = drm_dev_register(drm, 0);
-	if (ret)
-		goto register_fail;
-
 	/* Set the CRTC's port so that the encoder component can find it */
 	ep = of_graph_get_next_endpoint(dev->of_node, NULL);
 	if (!ep) {
@@ -377,6 +377,8 @@ static int malidp_bind(struct device *dev)
 	if (ret < 0)
 		goto irq_init_fail;
 
+	drm->irq_enabled = true;
+
 	ret = drm_vblank_init(drm, drm->mode_config.num_crtc);
 	if (ret < 0) {
 		DRM_ERROR("failed to initialise vblank\n");
@@ -395,23 +397,31 @@ static int malidp_bind(struct device *dev)
 	}
 
 	drm_kms_helper_poll_init(drm);
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		goto register_fail;
+
 	return 0;
 
+register_fail:
+	if (malidp->fbdev) {
+		drm_fbdev_cma_fini(malidp->fbdev);
+		malidp->fbdev = NULL;
+	}
 fbdev_fail:
 	drm_vblank_cleanup(drm);
 vblank_fail:
 	malidp_se_irq_fini(drm);
 	malidp_de_irq_fini(drm);
+	drm->irq_enabled = false;
 irq_init_fail:
 	component_unbind_all(dev, drm);
 bind_fail:
 	of_node_put(malidp->crtc.port);
 	malidp->crtc.port = NULL;
 port_fail:
-	drm_dev_unregister(drm);
-register_fail:
-	malidp_de_planes_destroy(drm);
-	drm_mode_config_cleanup(drm);
+	malidp_fini(drm);
 init_fail:
 	drm->dev_private = NULL;
 	dev_set_drvdata(dev, NULL);
@@ -432,6 +442,7 @@ static void malidp_unbind(struct device *dev)
 	struct malidp_drm *malidp = drm->dev_private;
 	struct malidp_hw_device *hwdev = malidp->dev;
 
+	drm_dev_unregister(drm);
 	if (malidp->fbdev) {
 		drm_fbdev_cma_fini(malidp->fbdev);
 		malidp->fbdev = NULL;
@@ -443,9 +454,7 @@ static void malidp_unbind(struct device *dev)
 	component_unbind_all(dev, drm);
 	of_node_put(malidp->crtc.port);
 	malidp->crtc.port = NULL;
-	drm_dev_unregister(drm);
-	malidp_de_planes_destroy(drm);
-	drm_mode_config_cleanup(drm);
+	malidp_fini(drm);
 	drm->dev_private = NULL;
 	dev_set_drvdata(dev, NULL);
 	clk_disable_unprepare(hwdev->mclk);
@@ -493,7 +502,9 @@ static int malidp_platform_probe(struct platform_device *pdev)
 		return -EAGAIN;
 	}
 
-	component_match_add(&pdev->dev, &match, malidp_compare_dev, port);
+	drm_of_component_match_add(&pdev->dev, &match, malidp_compare_dev,
+				   port);
+	of_node_put(port);
 	return component_master_add_with_match(&pdev->dev, &malidp_master_ops,
 					       match);
 }
diff --git a/drivers/gpu/drm/arm/malidp_drv.h b/drivers/gpu/drm/arm/malidp_drv.h
index 271d2fb9711c..9fc8a2e405e4 100644
--- a/drivers/gpu/drm/arm/malidp_drv.h
+++ b/drivers/gpu/drm/arm/malidp_drv.h
@@ -39,6 +39,9 @@ struct malidp_plane_state {
 
 	/* size of the required rotation memory if plane is rotated */
 	u32 rotmem_size;
+	/* internal format ID */
+	u8 format;
+	u8 n_planes;
 };
 
 #define to_malidp_plane(x) container_of(x, struct malidp_plane, base)
diff --git a/drivers/gpu/drm/arm/malidp_hw.c b/drivers/gpu/drm/arm/malidp_hw.c
index a6132f1d58c1..4bdf531f7844 100644
--- a/drivers/gpu/drm/arm/malidp_hw.c
+++ b/drivers/gpu/drm/arm/malidp_hw.c
@@ -125,6 +125,7 @@ static void malidp500_leave_config_mode(struct malidp_hw_device *hwdev)
 {
 	u32 status, count = 100;
 
+	malidp_hw_clearbits(hwdev, MALIDP_CFG_VALID, MALIDP500_CONFIG_VALID);
 	malidp_hw_clearbits(hwdev, MALIDP500_DC_CONFIG_REQ, MALIDP500_DC_CONTROL);
 	while (count) {
 		status = malidp_hw_read(hwdev, hwdev->map.dc_base + MALIDP_REG_STATUS);
@@ -198,9 +199,6 @@ static void malidp500_modeset(struct malidp_hw_device *hwdev, struct videomode *
 
 static int malidp500_rotmem_required(struct malidp_hw_device *hwdev, u16 w, u16 h, u32 fmt)
 {
-	unsigned int depth;
-	int bpp;
-
 	/* RGB888 or BGR888 can't be rotated */
 	if ((fmt == DRM_FORMAT_RGB888) || (fmt == DRM_FORMAT_BGR888))
 		return -EINVAL;
@@ -210,9 +208,7 @@ static int malidp500_rotmem_required(struct malidp_hw_device *hwdev, u16 w, u16
 	 * worth of pixel data. Required size is then:
 	 *    size = rotated_width * (bpp / 8) * 8;
 	 */
-	drm_fb_get_bpp_depth(fmt, &depth, &bpp);
-
-	return w * bpp;
+	return w * drm_format_plane_cpp(fmt, 0) * 8;
 }
 
 static int malidp550_query_hw(struct malidp_hw_device *hwdev)
@@ -271,6 +267,7 @@ static void malidp550_leave_config_mode(struct malidp_hw_device *hwdev)
 {
 	u32 status, count = 100;
 
+	malidp_hw_clearbits(hwdev, MALIDP_CFG_VALID, MALIDP550_CONFIG_VALID);
 	malidp_hw_clearbits(hwdev, MALIDP550_DC_CONFIG_REQ, MALIDP550_DC_CONTROL);
 	while (count) {
 		status = malidp_hw_read(hwdev, hwdev->map.dc_base + MALIDP_REG_STATUS);
@@ -441,6 +438,7 @@ const struct malidp_hw_device malidp_device[MALIDP_MAX_DEVICES] = {
 			},
 			.input_formats = malidp500_de_formats,
 			.n_input_formats = ARRAY_SIZE(malidp500_de_formats),
+			.bus_align_bytes = 8,
 		},
 		.query_hw = malidp500_query_hw,
 		.enter_config_mode = malidp500_enter_config_mode,
@@ -473,6 +471,7 @@ const struct malidp_hw_device malidp_device[MALIDP_MAX_DEVICES] = {
 			},
 			.input_formats = malidp550_de_formats,
 			.n_input_formats = ARRAY_SIZE(malidp550_de_formats),
+			.bus_align_bytes = 8,
 		},
 		.query_hw = malidp550_query_hw,
 		.enter_config_mode = malidp550_enter_config_mode,
@@ -506,6 +505,7 @@ const struct malidp_hw_device malidp_device[MALIDP_MAX_DEVICES] = {
 			},
 			.input_formats = malidp550_de_formats,
 			.n_input_formats = ARRAY_SIZE(malidp550_de_formats),
+			.bus_align_bytes = 16,
 		},
 		.query_hw = malidp650_query_hw,
 		.enter_config_mode = malidp550_enter_config_mode,
diff --git a/drivers/gpu/drm/arm/malidp_hw.h b/drivers/gpu/drm/arm/malidp_hw.h
index 141743e9f3a6..087e1202db3d 100644
--- a/drivers/gpu/drm/arm/malidp_hw.h
+++ b/drivers/gpu/drm/arm/malidp_hw.h
@@ -88,6 +88,9 @@ struct malidp_hw_regmap {
 	/* list of supported input formats for each layer */
 	const struct malidp_input_format *input_formats;
 	const u8 n_input_formats;
+
+	/* pitch alignment requirement in bytes */
+	const u8 bus_align_bytes;
 };
 
 struct malidp_hw_device {
@@ -229,6 +232,12 @@ void malidp_se_irq_fini(struct drm_device *drm);
 u8 malidp_hw_get_format_id(const struct malidp_hw_regmap *map,
 			   u8 layer_id, u32 format);
 
+static inline bool malidp_hw_pitch_valid(struct malidp_hw_device *hwdev,
+					 unsigned int pitch)
+{
+	return !(pitch & (hwdev->map.bus_align_bytes - 1));
+}
+
 /*
  * background color components are defined as 12bits values,
  * they will be shifted right when stored on hardware that
diff --git a/drivers/gpu/drm/arm/malidp_planes.c b/drivers/gpu/drm/arm/malidp_planes.c
index 82c193e5e0d6..63eec8f37cfc 100644
--- a/drivers/gpu/drm/arm/malidp_planes.c
+++ b/drivers/gpu/drm/arm/malidp_planes.c
@@ -27,6 +27,10 @@
 #define   LAYER_H_FLIP			(1 << 10)
 #define   LAYER_V_FLIP			(1 << 11)
 #define   LAYER_ROT_MASK		(0xf << 8)
+#define   LAYER_COMP_MASK		(0x3 << 12)
+#define   LAYER_COMP_PIXEL		(0x3 << 12)
+#define   LAYER_COMP_PLANE		(0x2 << 12)
+#define MALIDP_LAYER_COMPOSE		0x008
 #define MALIDP_LAYER_SIZE		0x00c
 #define   LAYER_H_VAL(x)		(((x) & 0x1fff) << 0)
 #define   LAYER_V_VAL(x)		(((x) & 0x1fff) << 16)
@@ -34,6 +38,14 @@
 #define MALIDP_LAYER_OFFSET		0x014
 #define MALIDP_LAYER_STRIDE		0x018
 
+/*
+ * This 4-entry look-up-table is used to determine the full 8-bit alpha value
+ * for formats with 1- or 2-bit alpha channels.
+ * We set it to give 100%/0% opacity for 1-bit formats and 100%/66%/33%/0%
+ * opacity for 2-bit formats.
+ */
+#define MALIDP_ALPHA_LUT 0xffaa5500
+
 static void malidp_de_plane_destroy(struct drm_plane *plane)
 {
 	struct malidp_plane *mp = to_malidp_plane(plane);
@@ -46,7 +58,8 @@ static void malidp_de_plane_destroy(struct drm_plane *plane)
 	devm_kfree(plane->dev->dev, mp);
 }
 
-struct drm_plane_state *malidp_duplicate_plane_state(struct drm_plane *plane)
+static struct
+drm_plane_state *malidp_duplicate_plane_state(struct drm_plane *plane)
 {
 	struct malidp_plane_state *state, *m_state;
 
@@ -58,13 +71,15 @@ struct drm_plane_state *malidp_duplicate_plane_state(struct drm_plane *plane)
 		m_state = to_malidp_plane_state(plane->state);
 		__drm_atomic_helper_plane_duplicate_state(plane, &state->base);
 		state->rotmem_size = m_state->rotmem_size;
+		state->format = m_state->format;
+		state->n_planes = m_state->n_planes;
 	}
 
 	return &state->base;
 }
 
-void malidp_destroy_plane_state(struct drm_plane *plane,
-				struct drm_plane_state *state)
+static void malidp_destroy_plane_state(struct drm_plane *plane,
+				       struct drm_plane_state *state)
 {
 	struct malidp_plane_state *m_state = to_malidp_plane_state(state);
 
@@ -75,6 +90,7 @@ void malidp_destroy_plane_state(struct drm_plane *plane,
 static const struct drm_plane_funcs malidp_de_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
+	.set_property = drm_atomic_helper_plane_set_property,
 	.destroy = malidp_de_plane_destroy,
 	.reset = drm_atomic_helper_plane_reset,
 	.atomic_duplicate_state = malidp_duplicate_plane_state,
@@ -86,17 +102,29 @@ static int malidp_de_plane_check(struct drm_plane *plane,
 {
 	struct malidp_plane *mp = to_malidp_plane(plane);
 	struct malidp_plane_state *ms = to_malidp_plane_state(state);
-	u8 format_id;
+	struct drm_framebuffer *fb;
+	int i;
 	u32 src_w, src_h;
 
 	if (!state->crtc || !state->fb)
 		return 0;
 
-	format_id = malidp_hw_get_format_id(&mp->hwdev->map, mp->layer->id,
-					    state->fb->pixel_format);
-	if (format_id == MALIDP_INVALID_FORMAT_ID)
+	fb = state->fb;
+
+	ms->format = malidp_hw_get_format_id(&mp->hwdev->map, mp->layer->id,
+					    fb->pixel_format);
+	if (ms->format == MALIDP_INVALID_FORMAT_ID)
 		return -EINVAL;
 
+	ms->n_planes = drm_format_num_planes(fb->pixel_format);
+	for (i = 0; i < ms->n_planes; i++) {
+		if (!malidp_hw_pitch_valid(mp->hwdev, fb->pitches[i])) {
+			DRM_DEBUG_KMS("Invalid pitch %u for plane %d\n",
+				      fb->pitches[i], i);
+			return -EINVAL;
+		}
+	}
+
 	src_w = state->src_w >> 16;
 	src_h = state->src_h >> 16;
 
@@ -135,17 +163,13 @@ static void malidp_de_plane_update(struct drm_plane *plane,
 	struct drm_gem_cma_object *obj;
 	struct malidp_plane *mp;
 	const struct malidp_hw_regmap *map;
-	u8 format_id;
+	struct malidp_plane_state *ms = to_malidp_plane_state(plane->state);
 	u16 ptr;
-	u32 format, src_w, src_h, dest_w, dest_h, val = 0;
-	int num_planes, i;
+	u32 src_w, src_h, dest_w, dest_h, val;
+	int i;
 
 	mp = to_malidp_plane(plane);
-
 	map = &mp->hwdev->map;
-	format = plane->state->fb->pixel_format;
-	format_id = malidp_hw_get_format_id(map, mp->layer->id, format);
-	num_planes = drm_format_num_planes(format);
 
 	/* convert src values from Q16 fixed point to integer */
 	src_w = plane->state->src_w >> 16;
@@ -158,9 +182,9 @@ static void malidp_de_plane_update(struct drm_plane *plane,
 		dest_h = plane->state->crtc_h;
 	}
 
-	malidp_hw_write(mp->hwdev, format_id, mp->layer->base);
+	malidp_hw_write(mp->hwdev, ms->format, mp->layer->base);
 
-	for (i = 0; i < num_planes; i++) {
+	for (i = 0; i < ms->n_planes; i++) {
 		/* calculate the offset for the layer's plane registers */
 		ptr = mp->layer->ptr + (i << 4);
 
@@ -181,9 +205,9 @@ static void malidp_de_plane_update(struct drm_plane *plane,
 			LAYER_V_VAL(plane->state->crtc_y),
 			mp->layer->base + MALIDP_LAYER_OFFSET);
 
-	/* first clear the rotation bits in the register */
-	malidp_hw_clearbits(mp->hwdev, LAYER_ROT_MASK,
-			    mp->layer->base + MALIDP_LAYER_CONTROL);
+	/* first clear the rotation bits */
+	val = malidp_hw_read(mp->hwdev, mp->layer->base + MALIDP_LAYER_CONTROL);
+	val &= ~LAYER_ROT_MASK;
 
 	/* setup the rotation and axis flip bits */
 	if (plane->state->rotation & DRM_ROTATE_MASK)
@@ -193,11 +217,18 @@ static void malidp_de_plane_update(struct drm_plane *plane,
 	if (plane->state->rotation & DRM_REFLECT_Y)
 		val |= LAYER_H_FLIP;
 
+	/*
+	 * always enable pixel alpha blending until we have a way to change
+	 * blend modes
+	 */
+	val &= ~LAYER_COMP_MASK;
+	val |= LAYER_COMP_PIXEL;
+
 	/* set the 'enable layer' bit */
 	val |= LAYER_ENABLE;
 
-	malidp_hw_setbits(mp->hwdev, val,
-			  mp->layer->base + MALIDP_LAYER_CONTROL);
+	malidp_hw_write(mp->hwdev, val,
+			mp->layer->base + MALIDP_LAYER_CONTROL);
 }
 
 static void malidp_de_plane_disable(struct drm_plane *plane,
@@ -222,6 +253,8 @@ int malidp_de_planes_init(struct drm_device *drm)
 	struct malidp_plane *plane = NULL;
 	enum drm_plane_type plane_type;
 	unsigned long crtcs = 1 << drm->mode_config.num_crtc;
+	unsigned long flags = DRM_ROTATE_0 | DRM_ROTATE_90 | DRM_ROTATE_180 |
+			      DRM_ROTATE_270 | DRM_REFLECT_X | DRM_REFLECT_Y;
 	u32 *formats;
 	int ret, i, j, n;
 
@@ -254,26 +287,18 @@ int malidp_de_planes_init(struct drm_device *drm)
 		if (ret < 0)
 			goto cleanup;
 
-		if (!drm->mode_config.rotation_property) {
-			unsigned long flags = DRM_ROTATE_0 |
-					      DRM_ROTATE_90 |
-					      DRM_ROTATE_180 |
-					      DRM_ROTATE_270 |
-					      DRM_REFLECT_X |
-					      DRM_REFLECT_Y;
-			drm->mode_config.rotation_property =
-				drm_mode_create_rotation_property(drm, flags);
-		}
-		/* SMART layer can't be rotated */
-		if (drm->mode_config.rotation_property && (id != DE_SMART))
-			drm_object_attach_property(&plane->base.base,
-						   drm->mode_config.rotation_property,
-						   DRM_ROTATE_0);
-
 		drm_plane_helper_add(&plane->base,
 				     &malidp_de_plane_helper_funcs);
 		plane->hwdev = malidp->dev;
 		plane->layer = &map->layers[i];
+
+		/* Skip the features which the SMART layer doesn't have */
+		if (id == DE_SMART)
+			continue;
+
+		drm_plane_create_rotation_property(&plane->base, DRM_ROTATE_0, flags);
+		malidp_hw_write(malidp->dev, MALIDP_ALPHA_LUT,
+				plane->layer->base + MALIDP_LAYER_COMPOSE);
 	}
 
 	kfree(formats);
diff --git a/drivers/gpu/drm/armada/Makefile b/drivers/gpu/drm/armada/Makefile
index ffd673615772..a18f156c8b66 100644
--- a/drivers/gpu/drm/armada/Makefile
+++ b/drivers/gpu/drm/armada/Makefile
@@ -1,5 +1,5 @@
 armada-y	:= armada_crtc.o armada_drv.o armada_fb.o armada_fbdev.o \
-		   armada_gem.o armada_overlay.o
+		   armada_gem.o armada_overlay.o armada_trace.o
 armada-y	+= armada_510.o
 armada-$(CONFIG_DEBUG_FS) += armada_debugfs.o
 
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
index a51f8cbcfe26..95cb3966b2ca 100644
--- a/drivers/gpu/drm/armada/armada_crtc.c
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -18,6 +18,7 @@
 #include "armada_fb.h"
 #include "armada_gem.h"
 #include "armada_hw.h"
+#include "armada_trace.h"
 
 struct armada_frame_work {
 	struct armada_plane_work work;
@@ -164,19 +165,37 @@ static void armada_drm_crtc_update(struct armada_crtc *dcrtc)
 	}
 }
 
+void armada_drm_plane_calc_addrs(u32 *addrs, struct drm_framebuffer *fb,
+	int x, int y)
+{
+	u32 addr = drm_fb_obj(fb)->dev_addr;
+	u32 pixel_format = fb->pixel_format;
+	int num_planes = drm_format_num_planes(pixel_format);
+	int i;
+
+	if (num_planes > 3)
+		num_planes = 3;
+
+	for (i = 0; i < num_planes; i++)
+		addrs[i] = addr + fb->offsets[i] + y * fb->pitches[i] +
+			     x * drm_format_plane_cpp(pixel_format, i);
+	for (; i < 3; i++)
+		addrs[i] = 0;
+}
+
 static unsigned armada_drm_crtc_calc_fb(struct drm_framebuffer *fb,
 	int x, int y, struct armada_regs *regs, bool interlaced)
 {
-	struct armada_gem_object *obj = drm_fb_obj(fb);
 	unsigned pitch = fb->pitches[0];
-	unsigned offset = y * pitch + x * fb->bits_per_pixel / 8;
-	uint32_t addr_odd, addr_even;
+	u32 addrs[3], addr_odd, addr_even;
 	unsigned i = 0;
 
 	DRM_DEBUG_DRIVER("pitch %u x %d y %d bpp %d\n",
 		pitch, x, y, fb->bits_per_pixel);
 
-	addr_odd = addr_even = obj->dev_addr + offset;
+	armada_drm_plane_calc_addrs(addrs, fb, x, y);
+
+	addr_odd = addr_even = addrs[0];
 
 	if (interlaced) {
 		addr_even += pitch;
@@ -192,17 +211,18 @@ static unsigned armada_drm_crtc_calc_fb(struct drm_framebuffer *fb,
 }
 
 static void armada_drm_plane_work_run(struct armada_crtc *dcrtc,
-	struct armada_plane *plane)
+	struct drm_plane *plane)
 {
-	struct armada_plane_work *work = xchg(&plane->work, NULL);
+	struct armada_plane *dplane = drm_to_armada_plane(plane);
+	struct armada_plane_work *work = xchg(&dplane->work, NULL);
 
 	/* Handle any pending frame work. */
 	if (work) {
-		work->fn(dcrtc, plane, work);
+		work->fn(dcrtc, dplane, work);
 		drm_crtc_vblank_put(&dcrtc->crtc);
 	}
 
-	wake_up(&plane->frame_wait);
+	wake_up(&dplane->frame_wait);
 }
 
 int armada_drm_plane_work_queue(struct armada_crtc *dcrtc,
@@ -307,14 +327,12 @@ static void armada_drm_crtc_finish_fb(struct armada_crtc *dcrtc,
 
 static void armada_drm_vblank_off(struct armada_crtc *dcrtc)
 {
-	struct armada_plane *plane = drm_to_armada_plane(dcrtc->crtc.primary);
-
 	/*
 	 * Tell the DRM core that vblank IRQs aren't going to happen for
 	 * a while.  This cleans up any pending vblank events for us.
 	 */
 	drm_crtc_vblank_off(&dcrtc->crtc);
-	armada_drm_plane_work_run(dcrtc, plane);
+	armada_drm_plane_work_run(dcrtc, dcrtc->crtc.primary);
 }
 
 void armada_drm_crtc_gamma_set(struct drm_crtc *crtc, u16 r, u16 g, u16 b,
@@ -416,10 +434,8 @@ static void armada_drm_crtc_irq(struct armada_crtc *dcrtc, u32 stat)
 
 	spin_lock(&dcrtc->irq_lock);
 	ovl_plane = dcrtc->plane;
-	if (ovl_plane) {
-		struct armada_plane *plane = drm_to_armada_plane(ovl_plane);
-		armada_drm_plane_work_run(dcrtc, plane);
-	}
+	if (ovl_plane)
+		armada_drm_plane_work_run(dcrtc, ovl_plane);
 
 	if (stat & GRA_FRAME_IRQ && dcrtc->interlaced) {
 		int i = stat & GRA_FRAME_IRQ0 ? 0 : 1;
@@ -449,10 +465,8 @@ static void armada_drm_crtc_irq(struct armada_crtc *dcrtc, u32 stat)
 
 	spin_unlock(&dcrtc->irq_lock);
 
-	if (stat & GRA_FRAME_IRQ) {
-		struct armada_plane *plane = drm_to_armada_plane(dcrtc->crtc.primary);
-		armada_drm_plane_work_run(dcrtc, plane);
-	}
+	if (stat & GRA_FRAME_IRQ)
+		armada_drm_plane_work_run(dcrtc, dcrtc->crtc.primary);
 }
 
 static irqreturn_t armada_drm_irq(int irq, void *arg)
@@ -466,6 +480,8 @@ static irqreturn_t armada_drm_irq(int irq, void *arg)
 	 */
 	writel_relaxed(0, dcrtc->base + LCD_SPU_IRQ_ISR);
 
+	trace_armada_drm_irq(&dcrtc->crtc, stat);
+
 	/* Mask out those interrupts we haven't enabled */
 	v = stat & dcrtc->irq_ena;
 
@@ -531,6 +547,35 @@ static uint32_t armada_drm_crtc_calculate_csc(struct armada_crtc *dcrtc)
 	return val;
 }
 
+static void armada_drm_primary_set(struct drm_crtc *crtc,
+	struct drm_plane *plane, int x, int y)
+{
+	struct armada_plane_state *state = &drm_to_armada_plane(plane)->state;
+	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+	struct armada_regs regs[8];
+	bool interlaced = dcrtc->interlaced;
+	unsigned i;
+	u32 ctrl0;
+
+	i = armada_drm_crtc_calc_fb(plane->fb, x, y, regs, interlaced);
+
+	armada_reg_queue_set(regs, i, state->dst_yx, LCD_SPU_GRA_OVSA_HPXL_VLN);
+	armada_reg_queue_set(regs, i, state->src_hw, LCD_SPU_GRA_HPXL_VLN);
+	armada_reg_queue_set(regs, i, state->dst_hw, LCD_SPU_GZM_HPXL_VLN);
+
+	ctrl0 = state->ctrl0;
+	if (interlaced)
+		ctrl0 |= CFG_GRA_FTOGGLE;
+
+	armada_reg_queue_mod(regs, i, ctrl0, CFG_GRAFORMAT |
+			     CFG_GRA_MOD(CFG_SWAPRB | CFG_SWAPUV |
+					 CFG_SWAPYU | CFG_YUV2RGB) |
+			     CFG_PALETTE_ENA | CFG_GRA_FTOGGLE,
+			     LCD_SPU_DMA_CTRL0);
+	armada_reg_queue_end(regs, i);
+	armada_drm_crtc_update_regs(dcrtc, regs);
+}
+
 /* The mode_config.mutex will be held for this call */
 static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
 	struct drm_display_mode *mode, struct drm_display_mode *adj,
@@ -547,9 +592,20 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
 
 	interlaced = !!(adj->flags & DRM_MODE_FLAG_INTERLACE);
 
-	i = armada_drm_crtc_calc_fb(dcrtc->crtc.primary->fb,
-				    x, y, regs, interlaced);
+	val = CFG_GRA_ENA | CFG_GRA_HSMOOTH;
+	val |= CFG_GRA_FMT(drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->fmt);
+	val |= CFG_GRA_MOD(drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->mod);
+
+	if (drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->fmt > CFG_420)
+		val |= CFG_PALETTE_ENA;
+
+	drm_to_armada_plane(crtc->primary)->state.ctrl0 = val;
+	drm_to_armada_plane(crtc->primary)->state.src_hw =
+	drm_to_armada_plane(crtc->primary)->state.dst_hw =
+		adj->crtc_vdisplay << 16 | adj->crtc_hdisplay;
+	drm_to_armada_plane(crtc->primary)->state.dst_yx = 0;
 
+	i = 0;
 	rm = adj->crtc_hsync_start - adj->crtc_hdisplay;
 	lm = adj->crtc_htotal - adj->crtc_hsync_end;
 	bm = adj->crtc_vsync_start - adj->crtc_vdisplay;
@@ -625,8 +681,6 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
 	val = adj->crtc_vdisplay << 16 | adj->crtc_hdisplay;
 
 	armada_reg_queue_set(regs, i, val, LCD_SPU_V_H_ACTIVE);
-	armada_reg_queue_set(regs, i, val, LCD_SPU_GRA_HPXL_VLN);
-	armada_reg_queue_set(regs, i, val, LCD_SPU_GZM_HPXL_VLN);
 	armada_reg_queue_set(regs, i, (lm << 16) | rm, LCD_SPU_H_PORCH);
 	armada_reg_queue_set(regs, i, dcrtc->v[0].spu_v_porch, LCD_SPU_V_PORCH);
 	armada_reg_queue_set(regs, i, dcrtc->v[0].spu_v_h_total,
@@ -638,22 +692,6 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
 				     ADV_VSYNCOFFEN, LCD_SPU_ADV_REG);
 	}
 
-	val = CFG_GRA_ENA | CFG_GRA_HSMOOTH;
-	val |= CFG_GRA_FMT(drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->fmt);
-	val |= CFG_GRA_MOD(drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->mod);
-
-	if (drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->fmt > CFG_420)
-		val |= CFG_PALETTE_ENA;
-
-	if (interlaced)
-		val |= CFG_GRA_FTOGGLE;
-
-	armada_reg_queue_mod(regs, i, val, CFG_GRAFORMAT |
-			     CFG_GRA_MOD(CFG_SWAPRB | CFG_SWAPUV |
-					 CFG_SWAPYU | CFG_YUV2RGB) |
-			     CFG_PALETTE_ENA | CFG_GRA_FTOGGLE,
-			     LCD_SPU_DMA_CTRL0);
-
 	val = adj->flags & DRM_MODE_FLAG_NVSYNC ? CFG_VSYNC_INV : 0;
 	armada_reg_queue_mod(regs, i, val, CFG_VSYNC_INV, LCD_SPU_DMA_CTRL1);
 
@@ -662,6 +700,8 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
 	armada_reg_queue_end(regs, i);
 
 	armada_drm_crtc_update_regs(dcrtc, regs);
+
+	armada_drm_primary_set(crtc, crtc->primary, x, y);
 	spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
 
 	armada_drm_crtc_update(dcrtc);
@@ -1038,7 +1078,7 @@ static int armada_drm_crtc_page_flip(struct drm_crtc *crtc,
 	 * interrupt, so complete it now.
 	 */
 	if (dpms_blanked(dcrtc->dpms))
-		armada_drm_plane_work_run(dcrtc, drm_to_armada_plane(dcrtc->crtc.primary));
+		armada_drm_plane_work_run(dcrtc, dcrtc->crtc.primary);
 
 	return 0;
 }
@@ -1172,7 +1212,6 @@ static int armada_drm_crtc_create(struct drm_device *drm, struct device *dev,
 		       CFG_PDWN32x32 | CFG_PDWN16x66 | CFG_PDWN32x66 |
 		       CFG_PDWN64x66, dcrtc->base + LCD_SPU_SRAM_PARA1);
 	writel_relaxed(0x2032ff81, dcrtc->base + LCD_SPU_DMA_CTRL1);
-	writel_relaxed(0x00000000, dcrtc->base + LCD_SPU_GRA_OVSA_HPXL_VLN);
 	writel_relaxed(dcrtc->irq_ena, dcrtc->base + LCD_SPU_IRQ_ENA);
 	writel_relaxed(0, dcrtc->base + LCD_SPU_IRQ_ISR);
 
diff --git a/drivers/gpu/drm/armada/armada_crtc.h b/drivers/gpu/drm/armada/armada_crtc.h
index 04fdd22d483b..b08043e8cc3b 100644
--- a/drivers/gpu/drm/armada/armada_crtc.h
+++ b/drivers/gpu/drm/armada/armada_crtc.h
@@ -41,10 +41,18 @@ struct armada_plane_work {
 				      struct armada_plane_work *);
 };
 
+struct armada_plane_state {
+	u32 src_hw;
+	u32 dst_hw;
+	u32 dst_yx;
+	u32 ctrl0;
+};
+
 struct armada_plane {
 	struct drm_plane	base;
 	wait_queue_head_t	frame_wait;
 	struct armada_plane_work *work;
+	struct armada_plane_state state;
 };
 #define drm_to_armada_plane(p) container_of(p, struct armada_plane, base)
 
@@ -54,6 +62,8 @@ int armada_drm_plane_work_queue(struct armada_crtc *dcrtc,
 int armada_drm_plane_work_wait(struct armada_plane *plane, long timeout);
 struct armada_plane_work *armada_drm_plane_work_cancel(
 	struct armada_crtc *dcrtc, struct armada_plane *plane);
+void armada_drm_plane_calc_addrs(u32 *addrs, struct drm_framebuffer *fb,
+	int x, int y);
 
 struct armada_crtc {
 	struct drm_crtc		crtc;
diff --git a/drivers/gpu/drm/armada/armada_debugfs.c b/drivers/gpu/drm/armada/armada_debugfs.c
index d4f7ab0a30d4..90222e60d2d6 100644
--- a/drivers/gpu/drm/armada/armada_debugfs.c
+++ b/drivers/gpu/drm/armada/armada_debugfs.c
@@ -113,7 +113,7 @@ static int drm_add_fake_info_node(struct drm_minor *minor, struct dentry *ent,
 	struct drm_info_node *node;
 
 	node = kmalloc(sizeof(struct drm_info_node), GFP_KERNEL);
-	if (node == NULL) {
+	if (!node) {
 		debugfs_remove(ent);
 		return -ENOMEM;
 	}
diff --git a/drivers/gpu/drm/armada/armada_drm.h b/drivers/gpu/drm/armada/armada_drm.h
index 3b2bb6128d40..77952d559a3c 100644
--- a/drivers/gpu/drm/armada/armada_drm.h
+++ b/drivers/gpu/drm/armada/armada_drm.h
@@ -53,6 +53,7 @@ struct armada_variant {
 extern const struct armada_variant armada510_ops;
 
 struct armada_private {
+	struct drm_device	drm;
 	struct work_struct	fb_unref_work;
 	DECLARE_KFIFO(fb_unref, struct drm_framebuffer *, 8);
 	struct drm_fb_helper	*fbdev;
diff --git a/drivers/gpu/drm/armada/armada_drv.c b/drivers/gpu/drm/armada/armada_drv.c
index 1e0e68f608e4..07086b427c22 100644
--- a/drivers/gpu/drm/armada/armada_drv.c
+++ b/drivers/gpu/drm/armada/armada_drv.c
@@ -49,106 +49,6 @@ void armada_drm_queue_unref_work(struct drm_device *dev,
 	spin_unlock_irqrestore(&dev->event_lock, flags);
 }
 
-static int armada_drm_load(struct drm_device *dev, unsigned long flags)
-{
-	struct armada_private *priv;
-	struct resource *mem = NULL;
-	int ret, n;
-
-	for (n = 0; ; n++) {
-		struct resource *r = platform_get_resource(dev->platformdev,
-							   IORESOURCE_MEM, n);
-		if (!r)
-			break;
-
-		/* Resources above 64K are graphics memory */
-		if (resource_size(r) > SZ_64K)
-			mem = r;
-		else
-			return -EINVAL;
-	}
-
-	if (!mem)
-		return -ENXIO;
-
-	if (!devm_request_mem_region(dev->dev, mem->start,
-			resource_size(mem), "armada-drm"))
-		return -EBUSY;
-
-	priv = devm_kzalloc(dev->dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv) {
-		DRM_ERROR("failed to allocate private\n");
-		return -ENOMEM;
-	}
-
-	platform_set_drvdata(dev->platformdev, dev);
-	dev->dev_private = priv;
-
-	INIT_WORK(&priv->fb_unref_work, armada_drm_unref_work);
-	INIT_KFIFO(priv->fb_unref);
-
-	/* Mode setting support */
-	drm_mode_config_init(dev);
-	dev->mode_config.min_width = 320;
-	dev->mode_config.min_height = 200;
-
-	/*
-	 * With vscale enabled, the maximum width is 1920 due to the
-	 * 1920 by 3 lines RAM
-	 */
-	dev->mode_config.max_width = 1920;
-	dev->mode_config.max_height = 2048;
-
-	dev->mode_config.preferred_depth = 24;
-	dev->mode_config.funcs = &armada_drm_mode_config_funcs;
-	drm_mm_init(&priv->linear, mem->start, resource_size(mem));
-	mutex_init(&priv->linear_lock);
-
-	ret = component_bind_all(dev->dev, dev);
-	if (ret)
-		goto err_kms;
-
-	ret = drm_vblank_init(dev, dev->mode_config.num_crtc);
-	if (ret)
-		goto err_comp;
-
-	dev->irq_enabled = true;
-
-	ret = armada_fbdev_init(dev);
-	if (ret)
-		goto err_comp;
-
-	drm_kms_helper_poll_init(dev);
-
-	return 0;
-
- err_comp:
-	component_unbind_all(dev->dev, dev);
- err_kms:
-	drm_mode_config_cleanup(dev);
-	drm_mm_takedown(&priv->linear);
-	flush_work(&priv->fb_unref_work);
-
-	return ret;
-}
-
-static int armada_drm_unload(struct drm_device *dev)
-{
-	struct armada_private *priv = dev->dev_private;
-
-	drm_kms_helper_poll_fini(dev);
-	armada_fbdev_fini(dev);
-
-	component_unbind_all(dev->dev, dev);
-
-	drm_mode_config_cleanup(dev);
-	drm_mm_takedown(&priv->linear);
-	flush_work(&priv->fb_unref_work);
-	dev->dev_private = NULL;
-
-	return 0;
-}
-
 /* These are called under the vbl_lock. */
 static int armada_drm_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
@@ -186,16 +86,10 @@ static const struct file_operations armada_drm_fops = {
 };
 
 static struct drm_driver armada_drm_driver = {
-	.load			= armada_drm_load,
 	.lastclose		= armada_drm_lastclose,
-	.unload			= armada_drm_unload,
 	.get_vblank_counter	= drm_vblank_no_hw_counter,
 	.enable_vblank		= armada_drm_enable_vblank,
 	.disable_vblank		= armada_drm_disable_vblank,
-#ifdef CONFIG_DEBUG_FS
-	.debugfs_init		= armada_drm_debugfs_init,
-	.debugfs_cleanup	= armada_drm_debugfs_cleanup,
-#endif
 	.gem_free_object_unlocked = armada_gem_free_object,
 	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
@@ -218,12 +112,138 @@ static struct drm_driver armada_drm_driver = {
 
 static int armada_drm_bind(struct device *dev)
 {
-	return drm_platform_init(&armada_drm_driver, to_platform_device(dev));
+	struct armada_private *priv;
+	struct resource *mem = NULL;
+	int ret, n;
+
+	for (n = 0; ; n++) {
+		struct resource *r = platform_get_resource(to_platform_device(dev),
+							   IORESOURCE_MEM, n);
+		if (!r)
+			break;
+
+		/* Resources above 64K are graphics memory */
+		if (resource_size(r) > SZ_64K)
+			mem = r;
+		else
+			return -EINVAL;
+	}
+
+	if (!mem)
+		return -ENXIO;
+
+	if (!devm_request_mem_region(dev, mem->start, resource_size(mem),
+				     "armada-drm"))
+		return -EBUSY;
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	/*
+	 * The drm_device structure must be at the start of
+	 * armada_private for drm_dev_unref() to work correctly.
+	 */
+	BUILD_BUG_ON(offsetof(struct armada_private, drm) != 0);
+
+	ret = drm_dev_init(&priv->drm, &armada_drm_driver, dev);
+	if (ret) {
+		dev_err(dev, "[" DRM_NAME ":%s] drm_dev_init failed: %d\n",
+			__func__, ret);
+		kfree(priv);
+		return ret;
+	}
+
+	priv->drm.platformdev = to_platform_device(dev);
+	priv->drm.dev_private = priv;
+
+	platform_set_drvdata(priv->drm.platformdev, &priv->drm);
+
+	INIT_WORK(&priv->fb_unref_work, armada_drm_unref_work);
+	INIT_KFIFO(priv->fb_unref);
+
+	/* Mode setting support */
+	drm_mode_config_init(&priv->drm);
+	priv->drm.mode_config.min_width = 320;
+	priv->drm.mode_config.min_height = 200;
+
+	/*
+	 * With vscale enabled, the maximum width is 1920 due to the
+	 * 1920 by 3 lines RAM
+	 */
+	priv->drm.mode_config.max_width = 1920;
+	priv->drm.mode_config.max_height = 2048;
+
+	priv->drm.mode_config.preferred_depth = 24;
+	priv->drm.mode_config.funcs = &armada_drm_mode_config_funcs;
+	drm_mm_init(&priv->linear, mem->start, resource_size(mem));
+	mutex_init(&priv->linear_lock);
+
+	ret = component_bind_all(dev, &priv->drm);
+	if (ret)
+		goto err_kms;
+
+	ret = drm_vblank_init(&priv->drm, priv->drm.mode_config.num_crtc);
+	if (ret)
+		goto err_comp;
+
+	priv->drm.irq_enabled = true;
+
+	ret = armada_fbdev_init(&priv->drm);
+	if (ret)
+		goto err_comp;
+
+	drm_kms_helper_poll_init(&priv->drm);
+
+	ret = drm_dev_register(&priv->drm, 0);
+	if (ret)
+		goto err_poll;
+
+#ifdef CONFIG_DEBUG_FS
+	armada_drm_debugfs_init(priv->drm.primary);
+#endif
+
+	DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n",
+		 armada_drm_driver.name, armada_drm_driver.major,
+		 armada_drm_driver.minor, armada_drm_driver.patchlevel,
+		 armada_drm_driver.date, dev_name(dev),
+		 priv->drm.primary->index);
+
+	return 0;
+
+ err_poll:
+	drm_kms_helper_poll_fini(&priv->drm);
+	armada_fbdev_fini(&priv->drm);
+ err_comp:
+	component_unbind_all(dev, &priv->drm);
+ err_kms:
+	drm_mode_config_cleanup(&priv->drm);
+	drm_mm_takedown(&priv->linear);
+	flush_work(&priv->fb_unref_work);
+	drm_dev_unref(&priv->drm);
+	return ret;
 }
 
 static void armada_drm_unbind(struct device *dev)
 {
-	drm_put_dev(dev_get_drvdata(dev));
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct armada_private *priv = drm->dev_private;
+
+	drm_kms_helper_poll_fini(&priv->drm);
+	armada_fbdev_fini(&priv->drm);
+
+#ifdef CONFIG_DEBUG_FS
+	armada_drm_debugfs_cleanup(priv->drm.primary);
+#endif
+	drm_dev_unregister(&priv->drm);
+
+	component_unbind_all(dev, &priv->drm);
+
+	drm_mode_config_cleanup(&priv->drm);
+	drm_mm_takedown(&priv->linear);
+	flush_work(&priv->fb_unref_work);
+
+	drm_dev_unref(&priv->drm);
 }
 
 static int compare_of(struct device *dev, void *data)
@@ -254,7 +274,7 @@ static void armada_add_endpoints(struct device *dev,
 			continue;
 		}
 
-		component_match_add(dev, match, compare_of, remote);
+		drm_of_component_match_add(dev, match, compare_of, remote);
 		of_node_put(remote);
 	}
 }
diff --git a/drivers/gpu/drm/armada/armada_fbdev.c b/drivers/gpu/drm/armada/armada_fbdev.c
index ca73ad8614fe..c5dc06a55883 100644
--- a/drivers/gpu/drm/armada/armada_fbdev.c
+++ b/drivers/gpu/drm/armada/armada_fbdev.c
@@ -19,16 +19,10 @@
 
 static /*const*/ struct fb_ops armada_fb_ops = {
 	.owner		= THIS_MODULE,
-	.fb_check_var	= drm_fb_helper_check_var,
-	.fb_set_par	= drm_fb_helper_set_par,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_fillrect	= drm_fb_helper_cfb_fillrect,
 	.fb_copyarea	= drm_fb_helper_cfb_copyarea,
 	.fb_imageblit	= drm_fb_helper_cfb_imageblit,
-	.fb_pan_display	= drm_fb_helper_pan_display,
-	.fb_blank	= drm_fb_helper_blank,
-	.fb_setcmap	= drm_fb_helper_setcmap,
-	.fb_debug_enter	= drm_fb_helper_debug_enter,
-	.fb_debug_leave	= drm_fb_helper_debug_leave,
 };
 
 static int armada_fb_create(struct drm_fb_helper *fbh,
diff --git a/drivers/gpu/drm/armada/armada_gem.c b/drivers/gpu/drm/armada/armada_gem.c
index 806791897304..a293c8be232c 100644
--- a/drivers/gpu/drm/armada/armada_gem.c
+++ b/drivers/gpu/drm/armada/armada_gem.c
@@ -17,12 +17,11 @@
 static int armada_gem_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct armada_gem_object *obj = drm_to_armada_gem(vma->vm_private_data);
-	unsigned long addr = (unsigned long)vmf->virtual_address;
 	unsigned long pfn = obj->phys_addr >> PAGE_SHIFT;
 	int ret;
 
-	pfn += (addr - vma->vm_start) >> PAGE_SHIFT;
-	ret = vm_insert_pfn(vma, addr, pfn);
+	pfn += (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+	ret = vm_insert_pfn(vma, vmf->address, pfn);
 
 	switch (ret) {
 	case 0:
@@ -212,7 +211,7 @@ armada_gem_alloc_private_object(struct drm_device *dev, size_t size)
 	return obj;
 }
 
-struct armada_gem_object *armada_gem_alloc_object(struct drm_device *dev,
+static struct armada_gem_object *armada_gem_alloc_object(struct drm_device *dev,
 	size_t size)
 {
 	struct armada_gem_object *obj;
@@ -419,7 +418,7 @@ int armada_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 }
 
 /* Prime support */
-struct sg_table *
+static struct sg_table *
 armada_gem_prime_map_dma_buf(struct dma_buf_attachment *attach,
 	enum dma_data_direction dir)
 {
@@ -594,11 +593,7 @@ int armada_gem_map_import(struct armada_gem_object *dobj)
 	int ret;
 
 	dobj->sgt = dma_buf_map_attachment(dobj->obj.import_attach,
-					  DMA_TO_DEVICE);
-	if (!dobj->sgt) {
-		DRM_ERROR("dma_buf_map_attachment() returned NULL\n");
-		return -EINVAL;
-	}
+					   DMA_TO_DEVICE);
 	if (IS_ERR(dobj->sgt)) {
 		ret = PTR_ERR(dobj->sgt);
 		dobj->sgt = NULL;
diff --git a/drivers/gpu/drm/armada/armada_overlay.c b/drivers/gpu/drm/armada/armada_overlay.c
index 152b4e716269..6743615232f5 100644
--- a/drivers/gpu/drm/armada/armada_overlay.c
+++ b/drivers/gpu/drm/armada/armada_overlay.c
@@ -15,6 +15,7 @@
 #include "armada_hw.h"
 #include <drm/armada_drm.h>
 #include "armada_ioctlP.h"
+#include "armada_trace.h"
 
 struct armada_ovl_plane_properties {
 	uint32_t colorkey_yr;
@@ -32,10 +33,6 @@ struct armada_ovl_plane_properties {
 struct armada_ovl_plane {
 	struct armada_plane base;
 	struct drm_framebuffer *old_fb;
-	uint32_t src_hw;
-	uint32_t dst_hw;
-	uint32_t dst_yx;
-	uint32_t ctrl0;
 	struct {
 		struct armada_plane_work work;
 		struct armada_regs regs[13];
@@ -87,6 +84,8 @@ static void armada_ovl_plane_work(struct armada_crtc *dcrtc,
 {
 	struct armada_ovl_plane *dplane = container_of(plane, struct armada_ovl_plane, base);
 
+	trace_armada_ovl_plane_work(&dcrtc->crtc, &plane->base);
+
 	armada_drm_crtc_update_regs(dcrtc, dplane->vbl.regs);
 	armada_ovl_retire_fb(dplane, NULL);
 }
@@ -120,6 +119,10 @@ armada_ovl_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
 	bool visible;
 	int ret;
 
+	trace_armada_ovl_plane_update(plane, crtc, fb,
+				 crtc_x, crtc_y, crtc_w, crtc_h,
+				 src_x, src_y, src_w, src_h);
+
 	ret = drm_plane_helper_check_update(plane, crtc, fb, &src, &dest, &clip,
 					    DRM_ROTATE_0,
 					    0, INT_MAX, true, false, &visible);
@@ -141,22 +144,22 @@ armada_ovl_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
 
 	/* FIXME: overlay on an interlaced display */
 	/* Just updating the position/size? */
-	if (plane->fb == fb && dplane->ctrl0 == ctrl0) {
+	if (plane->fb == fb && dplane->base.state.ctrl0 == ctrl0) {
 		val = (drm_rect_height(&src) & 0xffff0000) |
 		      drm_rect_width(&src) >> 16;
-		dplane->src_hw = val;
+		dplane->base.state.src_hw = val;
 		writel_relaxed(val, dcrtc->base + LCD_SPU_DMA_HPXL_VLN);
 
 		val = drm_rect_height(&dest) << 16 | drm_rect_width(&dest);
-		dplane->dst_hw = val;
+		dplane->base.state.dst_hw = val;
 		writel_relaxed(val, dcrtc->base + LCD_SPU_DZM_HPXL_VLN);
 
 		val = dest.y1 << 16 | dest.x1;
-		dplane->dst_yx = val;
+		dplane->base.state.dst_yx = val;
 		writel_relaxed(val, dcrtc->base + LCD_SPU_DMA_OVSA_HPXL_VLN);
 
 		return 0;
-	} else if (~dplane->ctrl0 & ctrl0 & CFG_DMA_ENA) {
+	} else if (~dplane->base.state.ctrl0 & ctrl0 & CFG_DMA_ENA) {
 		/* Power up the Y/U/V FIFOs on ENA 0->1 transitions */
 		armada_updatel(0, CFG_PDWN16x66 | CFG_PDWN32x66,
 			       dcrtc->base + LCD_SPU_SRAM_PARA1);
@@ -166,9 +169,8 @@ armada_ovl_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
 		armada_drm_plane_work_cancel(dcrtc, &dplane->base);
 
 	if (plane->fb != fb) {
-		struct armada_gem_object *obj = drm_fb_obj(fb);
-		uint32_t addr[3], pixel_format;
-		int i, num_planes, hsub;
+		u32 addrs[3], pixel_format;
+		int num_planes, hsub;
 
 		/*
 		 * Take a reference on the new framebuffer - we want to
@@ -182,6 +184,8 @@ armada_ovl_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
 		src_y = src.y1 >> 16;
 		src_x = src.x1 >> 16;
 
+		armada_drm_plane_calc_addrs(addrs, fb, src_x, src_y);
+
 		pixel_format = fb->pixel_format;
 		hsub = drm_format_horz_chroma_subsampling(pixel_format);
 		num_planes = drm_format_num_planes(pixel_format);
@@ -194,24 +198,17 @@ armada_ovl_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
 		if (src_x & (hsub - 1) && num_planes == 1)
 			ctrl0 ^= CFG_DMA_MOD(CFG_SWAPUV);
 
-		for (i = 0; i < num_planes; i++)
-			addr[i] = obj->dev_addr + fb->offsets[i] +
-				  src_y * fb->pitches[i] +
-				  src_x * drm_format_plane_cpp(pixel_format, i);
-		for (; i < ARRAY_SIZE(addr); i++)
-			addr[i] = 0;
-
-		armada_reg_queue_set(dplane->vbl.regs, idx, addr[0],
+		armada_reg_queue_set(dplane->vbl.regs, idx, addrs[0],
 				     LCD_SPU_DMA_START_ADDR_Y0);
-		armada_reg_queue_set(dplane->vbl.regs, idx, addr[1],
+		armada_reg_queue_set(dplane->vbl.regs, idx, addrs[1],
 				     LCD_SPU_DMA_START_ADDR_U0);
-		armada_reg_queue_set(dplane->vbl.regs, idx, addr[2],
+		armada_reg_queue_set(dplane->vbl.regs, idx, addrs[2],
 				     LCD_SPU_DMA_START_ADDR_V0);
-		armada_reg_queue_set(dplane->vbl.regs, idx, addr[0],
+		armada_reg_queue_set(dplane->vbl.regs, idx, addrs[0],
 				     LCD_SPU_DMA_START_ADDR_Y1);
-		armada_reg_queue_set(dplane->vbl.regs, idx, addr[1],
+		armada_reg_queue_set(dplane->vbl.regs, idx, addrs[1],
 				     LCD_SPU_DMA_START_ADDR_U1);
-		armada_reg_queue_set(dplane->vbl.regs, idx, addr[2],
+		armada_reg_queue_set(dplane->vbl.regs, idx, addrs[2],
 				     LCD_SPU_DMA_START_ADDR_V1);
 
 		val = fb->pitches[0] << 16 | fb->pitches[0];
@@ -223,28 +220,28 @@ armada_ovl_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
 	}
 
 	val = (drm_rect_height(&src) & 0xffff0000) | drm_rect_width(&src) >> 16;
-	if (dplane->src_hw != val) {
-		dplane->src_hw = val;
+	if (dplane->base.state.src_hw != val) {
+		dplane->base.state.src_hw = val;
 		armada_reg_queue_set(dplane->vbl.regs, idx, val,
 				     LCD_SPU_DMA_HPXL_VLN);
 	}
 
 	val = drm_rect_height(&dest) << 16 | drm_rect_width(&dest);
-	if (dplane->dst_hw != val) {
-		dplane->dst_hw = val;
+	if (dplane->base.state.dst_hw != val) {
+		dplane->base.state.dst_hw = val;
 		armada_reg_queue_set(dplane->vbl.regs, idx, val,
 				     LCD_SPU_DZM_HPXL_VLN);
 	}
 
 	val = dest.y1 << 16 | dest.x1;
-	if (dplane->dst_yx != val) {
-		dplane->dst_yx = val;
+	if (dplane->base.state.dst_yx != val) {
+		dplane->base.state.dst_yx = val;
 		armada_reg_queue_set(dplane->vbl.regs, idx, val,
 				     LCD_SPU_DMA_OVSA_HPXL_VLN);
 	}
 
-	if (dplane->ctrl0 != ctrl0) {
-		dplane->ctrl0 = ctrl0;
+	if (dplane->base.state.ctrl0 != ctrl0) {
+		dplane->base.state.ctrl0 = ctrl0;
 		armada_reg_queue_mod(dplane->vbl.regs, idx, ctrl0,
 			CFG_CBSH_ENA | CFG_DMAFORMAT | CFG_DMA_FTOGGLE |
 			CFG_DMA_HSMOOTH | CFG_DMA_TSTMODE |
@@ -275,7 +272,7 @@ static int armada_ovl_plane_disable(struct drm_plane *plane)
 	armada_drm_crtc_plane_disable(dcrtc, plane);
 
 	dcrtc->plane = NULL;
-	dplane->ctrl0 = 0;
+	dplane->base.state.ctrl0 = 0;
 
 	fb = xchg(&dplane->old_fb, NULL);
 	if (fb)
diff --git a/drivers/gpu/drm/armada/armada_trace.c b/drivers/gpu/drm/armada/armada_trace.c
new file mode 100644
index 000000000000..068b336ba75f
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_trace.c
@@ -0,0 +1,4 @@
+#ifndef __CHECKER__
+#define CREATE_TRACE_POINTS
+#include "armada_trace.h"
+#endif
diff --git a/drivers/gpu/drm/armada/armada_trace.h b/drivers/gpu/drm/armada/armada_trace.h
new file mode 100644
index 000000000000..dc0cba70fd1a
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_trace.h
@@ -0,0 +1,66 @@
+#if !defined(ARMADA_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define ARMADA_TRACE_H
+
+#include <linux/tracepoint.h>
+#include <drm/drmP.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM armada
+#define TRACE_INCLUDE_FILE armada_trace
+
+TRACE_EVENT(armada_drm_irq,
+	TP_PROTO(struct drm_crtc *crtc, u32 stat),
+	TP_ARGS(crtc, stat),
+	TP_STRUCT__entry(
+		__field(struct drm_crtc *, crtc)
+		__field(u32, stat)
+	),
+	TP_fast_assign(
+		__entry->crtc = crtc;
+		__entry->stat = stat;
+	),
+	TP_printk("crtc %p stat 0x%08x",
+		__entry->crtc, __entry->stat)
+);
+
+TRACE_EVENT(armada_ovl_plane_update,
+	TP_PROTO(struct drm_plane *plane, struct drm_crtc *crtc,
+		     struct drm_framebuffer *fb,
+		     int crtc_x, int crtc_y, unsigned crtc_w, unsigned crtc_h,
+		     uint32_t src_x, uint32_t src_y, uint32_t src_w, uint32_t src_h),
+	TP_ARGS(plane, crtc, fb, crtc_x, crtc_y, crtc_w, crtc_h, src_x, src_y, src_w, src_h),
+	TP_STRUCT__entry(
+		__field(struct drm_plane *, plane)
+		__field(struct drm_crtc *, crtc)
+		__field(struct drm_framebuffer *, fb)
+	),
+	TP_fast_assign(
+		__entry->plane = plane;
+		__entry->crtc = crtc;
+		__entry->fb = fb;
+	),
+	TP_printk("plane %p crtc %p fb %p",
+		__entry->plane, __entry->crtc, __entry->fb)
+);
+
+TRACE_EVENT(armada_ovl_plane_work,
+	TP_PROTO(struct drm_crtc *crtc, struct drm_plane *plane),
+	TP_ARGS(crtc, plane),
+	TP_STRUCT__entry(
+		__field(struct drm_plane *, plane)
+		__field(struct drm_crtc *, crtc)
+	),
+	TP_fast_assign(
+		__entry->plane = plane;
+		__entry->crtc = crtc;
+	),
+	TP_printk("plane %p crtc %p",
+		__entry->plane, __entry->crtc)
+);
+
+#endif
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c
index f54afd2113a9..fd7c9eec92e4 100644
--- a/drivers/gpu/drm/ast/ast_drv.c
+++ b/drivers/gpu/drm/ast/ast_drv.c
@@ -188,9 +188,7 @@ static const struct file_operations ast_fops = {
 	.unlocked_ioctl = drm_ioctl,
 	.mmap = ast_mmap,
 	.poll = drm_poll,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.read = drm_read,
 };
 
diff --git a/drivers/gpu/drm/ast/ast_fb.c b/drivers/gpu/drm/ast/ast_fb.c
index 7a86e24e2687..d6f5ec64c667 100644
--- a/drivers/gpu/drm/ast/ast_fb.c
+++ b/drivers/gpu/drm/ast/ast_fb.c
@@ -253,7 +253,7 @@ static int astfb_create(struct drm_fb_helper *helper,
 err_release_fbi:
 	drm_fb_helper_release_fbi(helper);
 err_free_vram:
-	vfree(afbdev->sysram);
+	vfree(sysram);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index 5957c3e659fe..e26c98f51eb4 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -839,12 +839,6 @@ static void ast_connector_destroy(struct drm_connector *connector)
 	kfree(connector);
 }
 
-static enum drm_connector_status
-ast_connector_detect(struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static const struct drm_connector_helper_funcs ast_connector_helper_funcs = {
 	.mode_valid = ast_mode_valid,
 	.get_modes = ast_get_modes,
@@ -853,7 +847,6 @@ static const struct drm_connector_helper_funcs ast_connector_helper_funcs = {
 
 static const struct drm_connector_funcs ast_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.detect = ast_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = ast_connector_destroy,
 };
diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c
index 0743e65cb240..2a1368fac1d1 100644
--- a/drivers/gpu/drm/ast/ast_ttm.c
+++ b/drivers/gpu/drm/ast/ast_ttm.c
@@ -230,6 +230,7 @@ struct ttm_bo_driver ast_bo_driver = {
 	.ttm_tt_populate = ast_ttm_tt_populate,
 	.ttm_tt_unpopulate = ast_ttm_tt_unpopulate,
 	.init_mem_type = ast_bo_init_mem_type,
+	.eviction_valuable = ttm_bo_eviction_valuable,
 	.evict_flags = ast_bo_evict_flags,
 	.move = NULL,
 	.verify_access = ast_bo_verify_access,
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
index 5f484310bee9..cbd0070265c9 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
@@ -464,7 +464,7 @@ atmel_hlcdc_dc_atomic_complete(struct atmel_hlcdc_dc_commit *commit)
 
 	drm_atomic_helper_cleanup_planes(dev, old_state);
 
-	drm_atomic_state_free(old_state);
+	drm_atomic_state_put(old_state);
 
 	/* Complete the commit, wake up any waiter. */
 	spin_lock(&dc->commit.wait.lock);
@@ -521,6 +521,7 @@ static int atmel_hlcdc_dc_atomic_commit(struct drm_device *dev,
 	/* Swap the state, this is the point of no return. */
 	drm_atomic_helper_swap_state(state, true);
 
+	drm_atomic_state_get(state);
 	if (async)
 		queue_work(dc->wq, &commit->work);
 	else
@@ -748,9 +749,7 @@ static const struct file_operations fops = {
 	.open               = drm_open,
 	.release            = drm_release,
 	.unlocked_ioctl     = drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl       = drm_compat_ioctl,
-#endif
 	.poll               = drm_poll,
 	.read               = drm_read,
 	.llseek             = no_llseek,
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
index 9d4c030672f0..246ed1e33d8a 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
@@ -393,7 +393,7 @@ static void atmel_hlcdc_plane_update_format(struct atmel_hlcdc_plane *plane,
 
 	if ((state->base.fb->pixel_format == DRM_FORMAT_YUV422 ||
 	     state->base.fb->pixel_format == DRM_FORMAT_NV61) &&
-	    (state->base.rotation & (DRM_ROTATE_90 | DRM_ROTATE_270)))
+	    drm_rotation_90_or_270(state->base.rotation))
 		cfg |= ATMEL_HLCDC_YUV422ROT;
 
 	atmel_hlcdc_layer_update_cfg(&plane->layer,
@@ -628,7 +628,7 @@ static int atmel_hlcdc_plane_atomic_check(struct drm_plane *p,
 	/*
 	 * Swap width and size in case of 90 or 270 degrees rotation
 	 */
-	if (state->base.rotation & (DRM_ROTATE_90 | DRM_ROTATE_270)) {
+	if (drm_rotation_90_or_270(state->base.rotation)) {
 		tmp = state->crtc_w;
 		state->crtc_w = state->crtc_h;
 		state->crtc_h = tmp;
@@ -883,9 +883,9 @@ static int atmel_hlcdc_plane_atomic_get_property(struct drm_plane *p,
 	return 0;
 }
 
-static void atmel_hlcdc_plane_init_properties(struct atmel_hlcdc_plane *plane,
-				const struct atmel_hlcdc_layer_desc *desc,
-				struct atmel_hlcdc_plane_properties *props)
+static int atmel_hlcdc_plane_init_properties(struct atmel_hlcdc_plane *plane,
+					     const struct atmel_hlcdc_layer_desc *desc,
+					     struct atmel_hlcdc_plane_properties *props)
 {
 	struct regmap *regmap = plane->layer.hlcdc->regmap;
 
@@ -902,10 +902,18 @@ static void atmel_hlcdc_plane_init_properties(struct atmel_hlcdc_plane *plane,
 				ATMEL_HLCDC_LAYER_GA_MASK);
 	}
 
-	if (desc->layout.xstride && desc->layout.pstride)
-		drm_object_attach_property(&plane->base.base,
-				plane->base.dev->mode_config.rotation_property,
-				DRM_ROTATE_0);
+	if (desc->layout.xstride && desc->layout.pstride) {
+		int ret;
+
+		ret = drm_plane_create_rotation_property(&plane->base,
+							 DRM_ROTATE_0,
+							 DRM_ROTATE_0 |
+							 DRM_ROTATE_90 |
+							 DRM_ROTATE_180 |
+							 DRM_ROTATE_270);
+		if (ret)
+			return ret;
+	}
 
 	if (desc->layout.csc) {
 		/*
@@ -925,6 +933,8 @@ static void atmel_hlcdc_plane_init_properties(struct atmel_hlcdc_plane *plane,
 			     ATMEL_HLCDC_LAYER_CSC_CFG(&plane->layer, 2),
 			     0x40040890);
 	}
+
+	return 0;
 }
 
 static struct drm_plane_helper_funcs atmel_hlcdc_layer_plane_helper_funcs = {
@@ -1036,7 +1046,9 @@ atmel_hlcdc_plane_create(struct drm_device *dev,
 			     &atmel_hlcdc_layer_plane_helper_funcs);
 
 	/* Set default property values*/
-	atmel_hlcdc_plane_init_properties(plane, desc, props);
+	ret = atmel_hlcdc_plane_init_properties(plane, desc, props);
+	if (ret)
+		return ERR_PTR(ret);
 
 	return plane;
 }
@@ -1054,15 +1066,6 @@ atmel_hlcdc_plane_create_properties(struct drm_device *dev)
 	if (!props->alpha)
 		return ERR_PTR(-ENOMEM);
 
-	dev->mode_config.rotation_property =
-			drm_mode_create_rotation_property(dev,
-							  DRM_ROTATE_0 |
-							  DRM_ROTATE_90 |
-							  DRM_ROTATE_180 |
-							  DRM_ROTATE_270);
-	if (!dev->mode_config.rotation_property)
-		return ERR_PTR(-ENOMEM);
-
 	return props;
 }
 
diff --git a/drivers/gpu/drm/bochs/bochs_drv.c b/drivers/gpu/drm/bochs/bochs_drv.c
index 534227df23f3..15a293e65b31 100644
--- a/drivers/gpu/drm/bochs/bochs_drv.c
+++ b/drivers/gpu/drm/bochs/bochs_drv.c
@@ -70,9 +70,7 @@ static const struct file_operations bochs_fops = {
 	.open		= drm_open,
 	.release	= drm_release,
 	.unlocked_ioctl	= drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl	= drm_compat_ioctl,
-#endif
 	.poll		= drm_poll,
 	.read		= drm_read,
 	.llseek		= no_llseek,
diff --git a/drivers/gpu/drm/bochs/bochs_fbdev.c b/drivers/gpu/drm/bochs/bochs_fbdev.c
index e1ec498a6b6e..da790a1c302a 100644
--- a/drivers/gpu/drm/bochs/bochs_fbdev.c
+++ b/drivers/gpu/drm/bochs/bochs_fbdev.c
@@ -22,14 +22,10 @@ static int bochsfb_mmap(struct fb_info *info,
 
 static struct fb_ops bochsfb_ops = {
 	.owner = THIS_MODULE,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_fillrect = drm_fb_helper_sys_fillrect,
 	.fb_copyarea = drm_fb_helper_sys_copyarea,
 	.fb_imageblit = drm_fb_helper_sys_imageblit,
-	.fb_pan_display = drm_fb_helper_pan_display,
-	.fb_blank = drm_fb_helper_blank,
-	.fb_setcmap = drm_fb_helper_setcmap,
 	.fb_mmap = bochsfb_mmap,
 };
 
diff --git a/drivers/gpu/drm/bochs/bochs_kms.c b/drivers/gpu/drm/bochs/bochs_kms.c
index 0b4e5d117043..d5e63eff357b 100644
--- a/drivers/gpu/drm/bochs/bochs_kms.c
+++ b/drivers/gpu/drm/bochs/bochs_kms.c
@@ -216,12 +216,6 @@ bochs_connector_best_encoder(struct drm_connector *connector)
 	return NULL;
 }
 
-static enum drm_connector_status bochs_connector_detect(struct drm_connector
-							*connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static const struct drm_connector_helper_funcs bochs_connector_connector_helper_funcs = {
 	.get_modes = bochs_connector_get_modes,
 	.mode_valid = bochs_connector_mode_valid,
@@ -230,7 +224,6 @@ static const struct drm_connector_helper_funcs bochs_connector_connector_helper_
 
 static const struct drm_connector_funcs bochs_connector_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.detect = bochs_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
 };
diff --git a/drivers/gpu/drm/bochs/bochs_mm.c b/drivers/gpu/drm/bochs/bochs_mm.c
index 269cfca9ca06..099a3c688c26 100644
--- a/drivers/gpu/drm/bochs/bochs_mm.c
+++ b/drivers/gpu/drm/bochs/bochs_mm.c
@@ -199,6 +199,7 @@ struct ttm_bo_driver bochs_bo_driver = {
 	.ttm_tt_populate = ttm_pool_populate,
 	.ttm_tt_unpopulate = ttm_pool_unpopulate,
 	.init_mem_type = bochs_bo_init_mem_type,
+	.eviction_valuable = ttm_bo_eviction_valuable,
 	.evict_flags = bochs_bo_evict_flags,
 	.move = NULL,
 	.verify_access = bochs_bo_verify_access,
diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
index 10e12e74fc9f..eb8688ec6f18 100644
--- a/drivers/gpu/drm/bridge/Kconfig
+++ b/drivers/gpu/drm/bridge/Kconfig
@@ -39,6 +39,15 @@ config DRM_DW_HDMI_AHB_AUDIO
 	  Designware HDMI block.  This is used in conjunction with
 	  the i.MX6 HDMI driver.
 
+config DRM_DW_HDMI_I2S_AUDIO
+	tristate "Synopsis Designware I2S Audio interface"
+	depends on SND_SOC
+	depends on DRM_DW_HDMI
+	select SND_SOC_HDMI_CODEC
+	help
+	  Support the I2S Audio interface which is part of the Synopsis
+	  Designware HDMI block.
+
 config DRM_NXP_PTN3460
 	tristate "NXP PTN3460 DP/LVDS bridge"
 	depends on OF
@@ -57,6 +66,13 @@ config DRM_PARADE_PS8622
 	---help---
 	  Parade eDP-LVDS bridge chip driver.
 
+config DRM_SIL_SII8620
+	tristate "Silicon Image SII8620 HDMI/MHL bridge"
+	depends on OF
+	select DRM_KMS_HELPER
+	help
+	  Silicon Image SII8620 HDMI/MHL bridge chip driver.
+
 config DRM_SII902X
 	tristate "Silicon Image sii902x RGB/HDMI bridge"
 	depends on OF
@@ -74,6 +90,13 @@ config DRM_TOSHIBA_TC358767
 	---help---
 	  Toshiba TC358767 eDP bridge chip driver.
 
+config DRM_TI_TFP410
+	tristate "TI TFP410 DVI/HDMI bridge"
+	depends on OF
+	select DRM_KMS_HELPER
+	---help---
+	  Texas Instruments TFP410 DVI/HDMI Transmitter driver
+
 source "drivers/gpu/drm/bridge/analogix/Kconfig"
 
 source "drivers/gpu/drm/bridge/adv7511/Kconfig"
diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile
index cdf3a3cf765d..2e83a7855399 100644
--- a/drivers/gpu/drm/bridge/Makefile
+++ b/drivers/gpu/drm/bridge/Makefile
@@ -4,9 +4,12 @@ obj-$(CONFIG_DRM_ANALOGIX_ANX78XX) += analogix-anx78xx.o
 obj-$(CONFIG_DRM_DUMB_VGA_DAC) += dumb-vga-dac.o
 obj-$(CONFIG_DRM_DW_HDMI) += dw-hdmi.o
 obj-$(CONFIG_DRM_DW_HDMI_AHB_AUDIO) += dw-hdmi-ahb-audio.o
+obj-$(CONFIG_DRM_DW_HDMI_I2S_AUDIO) += dw-hdmi-i2s-audio.o
 obj-$(CONFIG_DRM_NXP_PTN3460) += nxp-ptn3460.o
 obj-$(CONFIG_DRM_PARADE_PS8622) += parade-ps8622.o
+obj-$(CONFIG_DRM_SIL_SII8620) += sil-sii8620.o
 obj-$(CONFIG_DRM_SII902X) += sii902x.o
 obj-$(CONFIG_DRM_TOSHIBA_TC358767) += tc358767.o
 obj-$(CONFIG_DRM_ANALOGIX_DP) += analogix/
 obj-$(CONFIG_DRM_I2C_ADV7511) += adv7511/
+obj-$(CONFIG_DRM_TI_TFP410) += ti-tfp410.o
diff --git a/drivers/gpu/drm/bridge/adv7511/Kconfig b/drivers/gpu/drm/bridge/adv7511/Kconfig
index d2b0499ab7d7..2fed567f9943 100644
--- a/drivers/gpu/drm/bridge/adv7511/Kconfig
+++ b/drivers/gpu/drm/bridge/adv7511/Kconfig
@@ -6,6 +6,14 @@ config DRM_I2C_ADV7511
 	help
 	  Support for the Analog Device ADV7511(W) and ADV7513 HDMI encoders.
 
+config DRM_I2C_ADV7511_AUDIO
+	bool "ADV7511 HDMI Audio driver"
+	depends on DRM_I2C_ADV7511 && SND_SOC
+	select SND_SOC_HDMI_CODEC
+	help
+	  Support the ADV7511 HDMI Audio interface. This is used in
+	  conjunction with the AV7511  HDMI driver.
+
 config DRM_I2C_ADV7533
 	bool "ADV7533 encoder"
 	depends on DRM_I2C_ADV7511
diff --git a/drivers/gpu/drm/bridge/adv7511/Makefile b/drivers/gpu/drm/bridge/adv7511/Makefile
index 9019327fff4c..5ba675534f6e 100644
--- a/drivers/gpu/drm/bridge/adv7511/Makefile
+++ b/drivers/gpu/drm/bridge/adv7511/Makefile
@@ -1,3 +1,4 @@
 adv7511-y := adv7511_drv.o
+adv7511-$(CONFIG_DRM_I2C_ADV7511_AUDIO) += adv7511_audio.o
 adv7511-$(CONFIG_DRM_I2C_ADV7533) += adv7533.o
 obj-$(CONFIG_DRM_I2C_ADV7511) += adv7511.o
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511.h b/drivers/gpu/drm/bridge/adv7511/adv7511.h
index 161c923d6162..992d76ce02bb 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511.h
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511.h
@@ -309,6 +309,8 @@ struct adv7511 {
 	struct drm_display_mode curr_mode;
 
 	unsigned int f_tmds;
+	unsigned int f_audio;
+	unsigned int audio_source;
 
 	unsigned int current_edid_segment;
 	uint8_t edid_buf[256];
@@ -334,6 +336,7 @@ struct adv7511 {
 	bool use_timing_gen;
 
 	enum adv7511_type type;
+	struct platform_device *audio_pdev;
 };
 
 #ifdef CONFIG_DRM_I2C_ADV7533
@@ -389,4 +392,17 @@ static inline int adv7533_parse_dt(struct device_node *np, struct adv7511 *adv)
 }
 #endif
 
+#ifdef CONFIG_DRM_I2C_ADV7511_AUDIO
+int adv7511_audio_init(struct device *dev, struct adv7511 *adv7511);
+void adv7511_audio_exit(struct adv7511 *adv7511);
+#else /*CONFIG_DRM_I2C_ADV7511_AUDIO */
+static inline int adv7511_audio_init(struct device *dev, struct adv7511 *adv7511)
+{
+	return 0;
+}
+static inline void adv7511_audio_exit(struct adv7511 *adv7511)
+{
+}
+#endif /* CONFIG_DRM_I2C_ADV7511_AUDIO */
+
 #endif /* __DRM_I2C_ADV7511_H__ */
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_audio.c b/drivers/gpu/drm/bridge/adv7511/adv7511_audio.c
new file mode 100644
index 000000000000..cf92ebfe6ab7
--- /dev/null
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511_audio.c
@@ -0,0 +1,213 @@
+/*
+ * Analog Devices ADV7511 HDMI transmitter driver
+ *
+ * Copyright 2012 Analog Devices Inc.
+ * Copyright (c) 2016, Linaro Limited
+ *
+ * Licensed under the GPL-2.
+ */
+
+#include <sound/core.h>
+#include <sound/hdmi-codec.h>
+#include <sound/pcm.h>
+#include <sound/soc.h>
+
+#include "adv7511.h"
+
+static void adv7511_calc_cts_n(unsigned int f_tmds, unsigned int fs,
+			       unsigned int *cts, unsigned int *n)
+{
+	switch (fs) {
+	case 32000:
+		*n = 4096;
+		break;
+	case 44100:
+		*n = 6272;
+		break;
+	case 48000:
+		*n = 6144;
+		break;
+	}
+
+	*cts = ((f_tmds * *n) / (128 * fs)) * 1000;
+}
+
+static int adv7511_update_cts_n(struct adv7511 *adv7511)
+{
+	unsigned int cts = 0;
+	unsigned int n = 0;
+
+	adv7511_calc_cts_n(adv7511->f_tmds, adv7511->f_audio, &cts, &n);
+
+	regmap_write(adv7511->regmap, ADV7511_REG_N0, (n >> 16) & 0xf);
+	regmap_write(adv7511->regmap, ADV7511_REG_N1, (n >> 8) & 0xff);
+	regmap_write(adv7511->regmap, ADV7511_REG_N2, n & 0xff);
+
+	regmap_write(adv7511->regmap, ADV7511_REG_CTS_MANUAL0,
+		     (cts >> 16) & 0xf);
+	regmap_write(adv7511->regmap, ADV7511_REG_CTS_MANUAL1,
+		     (cts >> 8) & 0xff);
+	regmap_write(adv7511->regmap, ADV7511_REG_CTS_MANUAL2,
+		     cts & 0xff);
+
+	return 0;
+}
+
+int adv7511_hdmi_hw_params(struct device *dev, void *data,
+			   struct hdmi_codec_daifmt *fmt,
+			   struct hdmi_codec_params *hparms)
+{
+	struct adv7511 *adv7511 = dev_get_drvdata(dev);
+	unsigned int audio_source, i2s_format = 0;
+	unsigned int invert_clock;
+	unsigned int rate;
+	unsigned int len;
+
+	switch (hparms->sample_rate) {
+	case 32000:
+		rate = ADV7511_SAMPLE_FREQ_32000;
+		break;
+	case 44100:
+		rate = ADV7511_SAMPLE_FREQ_44100;
+		break;
+	case 48000:
+		rate = ADV7511_SAMPLE_FREQ_48000;
+		break;
+	case 88200:
+		rate = ADV7511_SAMPLE_FREQ_88200;
+		break;
+	case 96000:
+		rate = ADV7511_SAMPLE_FREQ_96000;
+		break;
+	case 176400:
+		rate = ADV7511_SAMPLE_FREQ_176400;
+		break;
+	case 192000:
+		rate = ADV7511_SAMPLE_FREQ_192000;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (hparms->sample_width) {
+	case 16:
+		len = ADV7511_I2S_SAMPLE_LEN_16;
+		break;
+	case 18:
+		len = ADV7511_I2S_SAMPLE_LEN_18;
+		break;
+	case 20:
+		len = ADV7511_I2S_SAMPLE_LEN_20;
+		break;
+	case 24:
+		len = ADV7511_I2S_SAMPLE_LEN_24;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (fmt->fmt) {
+	case HDMI_I2S:
+		audio_source = ADV7511_AUDIO_SOURCE_I2S;
+		i2s_format = ADV7511_I2S_FORMAT_I2S;
+		break;
+	case HDMI_RIGHT_J:
+		audio_source = ADV7511_AUDIO_SOURCE_I2S;
+		i2s_format = ADV7511_I2S_FORMAT_RIGHT_J;
+		break;
+	case HDMI_LEFT_J:
+		audio_source = ADV7511_AUDIO_SOURCE_I2S;
+		i2s_format = ADV7511_I2S_FORMAT_LEFT_J;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	invert_clock = fmt->bit_clk_inv;
+
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_AUDIO_SOURCE, 0x70,
+			   audio_source << 4);
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_AUDIO_CONFIG, BIT(6),
+			   invert_clock << 6);
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_I2S_CONFIG, 0x03,
+			   i2s_format);
+
+	adv7511->audio_source = audio_source;
+
+	adv7511->f_audio = hparms->sample_rate;
+
+	adv7511_update_cts_n(adv7511);
+
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_AUDIO_CFG3,
+			   ADV7511_AUDIO_CFG3_LEN_MASK, len);
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_I2C_FREQ_ID_CFG,
+			   ADV7511_I2C_FREQ_ID_CFG_RATE_MASK, rate << 4);
+	regmap_write(adv7511->regmap, 0x73, 0x1);
+
+	return 0;
+}
+
+static int audio_startup(struct device *dev, void *data)
+{
+	struct adv7511 *adv7511 = dev_get_drvdata(dev);
+
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_AUDIO_CONFIG,
+				BIT(7), 0);
+
+	/* hide Audio infoframe updates */
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_INFOFRAME_UPDATE,
+				BIT(5), BIT(5));
+	/* enable N/CTS, enable Audio sample packets */
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_PACKET_ENABLE1,
+				BIT(5), BIT(5));
+	/* enable N/CTS */
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_PACKET_ENABLE1,
+				BIT(6), BIT(6));
+	/* not copyrighted */
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_AUDIO_CFG1,
+				BIT(5), BIT(5));
+	/* enable audio infoframes */
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_PACKET_ENABLE1,
+				BIT(3), BIT(3));
+	/* AV mute disable */
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_GC(0),
+				BIT(7) | BIT(6), BIT(7));
+	/* use Audio infoframe updated info */
+	regmap_update_bits(adv7511->regmap, ADV7511_REG_GC(1),
+				BIT(5), 0);
+	return 0;
+}
+
+static void audio_shutdown(struct device *dev, void *data)
+{
+}
+
+static const struct hdmi_codec_ops adv7511_codec_ops = {
+	.hw_params	= adv7511_hdmi_hw_params,
+	.audio_shutdown = audio_shutdown,
+	.audio_startup	= audio_startup,
+};
+
+static struct hdmi_codec_pdata codec_data = {
+	.ops = &adv7511_codec_ops,
+	.max_i2s_channels = 2,
+	.i2s = 1,
+};
+
+int adv7511_audio_init(struct device *dev, struct adv7511 *adv7511)
+{
+	adv7511->audio_pdev = platform_device_register_data(dev,
+					HDMI_CODEC_DRV_NAME,
+					PLATFORM_DEVID_AUTO,
+					&codec_data,
+					sizeof(codec_data));
+	return PTR_ERR_OR_ZERO(adv7511->audio_pdev);
+}
+
+void adv7511_audio_exit(struct adv7511 *adv7511)
+{
+	if (adv7511->audio_pdev) {
+		platform_device_unregister(adv7511->audio_pdev);
+		adv7511->audio_pdev = NULL;
+	}
+}
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
index 8ed3906dd411..8dba729f6ef9 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
@@ -1037,6 +1037,8 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
 		goto err_unregister_cec;
 	}
 
+	adv7511_audio_init(dev, adv7511);
+
 	return 0;
 
 err_unregister_cec:
@@ -1058,6 +1060,8 @@ static int adv7511_remove(struct i2c_client *i2c)
 
 	drm_bridge_remove(&adv7511->bridge);
 
+	adv7511_audio_exit(adv7511);
+
 	i2c_unregister_device(adv7511->i2c_edid);
 
 	kfree(adv7511->edid);
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7533.c b/drivers/gpu/drm/bridge/adv7511/adv7533.c
index d7f7b7ce8ebe..8b210373cfa2 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7533.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7533.c
@@ -29,6 +29,7 @@ static const struct reg_sequence adv7533_cec_fixed_registers[] = {
 	{ 0x17, 0xd0 },
 	{ 0x24, 0x20 },
 	{ 0x57, 0x11 },
+	{ 0x05, 0xc8 },
 };
 
 static const struct regmap_config adv7533_cec_regmap_config = {
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
index 6e0447f329a2..eb9bf8786c24 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
@@ -112,7 +112,7 @@ int analogix_dp_enable_psr(struct device *dev)
 	struct edp_vsc_psr psr_vsc;
 
 	if (!dp->psr_support)
-		return -EINVAL;
+		return 0;
 
 	/* Prepare VSC packet as per EDP 1.4 spec, Table 6.9 */
 	memset(&psr_vsc, 0, sizeof(psr_vsc));
@@ -135,7 +135,7 @@ int analogix_dp_disable_psr(struct device *dev)
 	struct edp_vsc_psr psr_vsc;
 
 	if (!dp->psr_support)
-		return -EINVAL;
+		return 0;
 
 	/* Prepare VSC packet as per EDP 1.4 spec, Table 6.9 */
 	memset(&psr_vsc, 0, sizeof(psr_vsc));
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
index cd37ac058675..303083ad28e3 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
@@ -1162,5 +1162,5 @@ ssize_t analogix_dp_transfer(struct analogix_dp_device *dp,
 		 (msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_NATIVE_READ)
 		msg->reply = DP_AUX_NATIVE_REPLY_ACK;
 
-	return num_transferred;
+	return num_transferred > 0 ? num_transferred : -EBUSY;
 }
diff --git a/drivers/gpu/drm/bridge/dumb-vga-dac.c b/drivers/gpu/drm/bridge/dumb-vga-dac.c
index afec232185a7..e5706981c934 100644
--- a/drivers/gpu/drm/bridge/dumb-vga-dac.c
+++ b/drivers/gpu/drm/bridge/dumb-vga-dac.c
@@ -12,6 +12,7 @@
 
 #include <linux/module.h>
 #include <linux/of_graph.h>
+#include <linux/regulator/consumer.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_atomic_helper.h>
@@ -23,6 +24,7 @@ struct dumb_vga {
 	struct drm_connector	connector;
 
 	struct i2c_adapter	*ddc;
+	struct regulator	*vdd;
 };
 
 static inline struct dumb_vga *
@@ -124,8 +126,30 @@ static int dumb_vga_attach(struct drm_bridge *bridge)
 	return 0;
 }
 
+static void dumb_vga_enable(struct drm_bridge *bridge)
+{
+	struct dumb_vga *vga = drm_bridge_to_dumb_vga(bridge);
+	int ret = 0;
+
+	if (vga->vdd)
+		ret = regulator_enable(vga->vdd);
+
+	if (ret)
+		DRM_ERROR("Failed to enable vdd regulator: %d\n", ret);
+}
+
+static void dumb_vga_disable(struct drm_bridge *bridge)
+{
+	struct dumb_vga *vga = drm_bridge_to_dumb_vga(bridge);
+
+	if (vga->vdd)
+		regulator_disable(vga->vdd);
+}
+
 static const struct drm_bridge_funcs dumb_vga_bridge_funcs = {
 	.attach		= dumb_vga_attach,
+	.enable		= dumb_vga_enable,
+	.disable	= dumb_vga_disable,
 };
 
 static struct i2c_adapter *dumb_vga_retrieve_ddc(struct device *dev)
@@ -169,6 +193,15 @@ static int dumb_vga_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	platform_set_drvdata(pdev, vga);
 
+	vga->vdd = devm_regulator_get_optional(&pdev->dev, "vdd");
+	if (IS_ERR(vga->vdd)) {
+		ret = PTR_ERR(vga->vdd);
+		if (ret == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		vga->vdd = NULL;
+		dev_dbg(&pdev->dev, "No vdd regulator found: %d\n", ret);
+	}
+
 	vga->ddc = dumb_vga_retrieve_ddc(&pdev->dev);
 	if (IS_ERR(vga->ddc)) {
 		if (PTR_ERR(vga->ddc) == -ENODEV) {
diff --git a/drivers/gpu/drm/bridge/dw-hdmi-audio.h b/drivers/gpu/drm/bridge/dw-hdmi-audio.h
index 91f631beecc7..fd1f745c6073 100644
--- a/drivers/gpu/drm/bridge/dw-hdmi-audio.h
+++ b/drivers/gpu/drm/bridge/dw-hdmi-audio.h
@@ -11,4 +11,11 @@ struct dw_hdmi_audio_data {
 	u8 *eld;
 };
 
+struct dw_hdmi_i2s_audio_data {
+	struct dw_hdmi *hdmi;
+
+	void (*write)(struct dw_hdmi *hdmi, u8 val, int offset);
+	u8 (*read)(struct dw_hdmi *hdmi, int offset);
+};
+
 #endif
diff --git a/drivers/gpu/drm/bridge/dw-hdmi-i2s-audio.c b/drivers/gpu/drm/bridge/dw-hdmi-i2s-audio.c
new file mode 100644
index 000000000000..aaf287d2e91d
--- /dev/null
+++ b/drivers/gpu/drm/bridge/dw-hdmi-i2s-audio.c
@@ -0,0 +1,141 @@
+/*
+ * dw-hdmi-i2s-audio.c
+ *
+ * Copyright (c) 2016 Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <drm/bridge/dw_hdmi.h>
+
+#include <sound/hdmi-codec.h>
+
+#include "dw-hdmi.h"
+#include "dw-hdmi-audio.h"
+
+#define DRIVER_NAME "dw-hdmi-i2s-audio"
+
+static inline void hdmi_write(struct dw_hdmi_i2s_audio_data *audio,
+			      u8 val, int offset)
+{
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	audio->write(hdmi, val, offset);
+}
+
+static inline u8 hdmi_read(struct dw_hdmi_i2s_audio_data *audio, int offset)
+{
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	return audio->read(hdmi, offset);
+}
+
+static int dw_hdmi_i2s_hw_params(struct device *dev, void *data,
+				 struct hdmi_codec_daifmt *fmt,
+				 struct hdmi_codec_params *hparms)
+{
+	struct dw_hdmi_i2s_audio_data *audio = data;
+	struct dw_hdmi *hdmi = audio->hdmi;
+	u8 conf0 = 0;
+	u8 conf1 = 0;
+	u8 inputclkfs = 0;
+
+	/* it cares I2S only */
+	if ((fmt->fmt != HDMI_I2S) ||
+	    (fmt->bit_clk_master | fmt->frame_clk_master)) {
+		dev_err(dev, "unsupported format/settings\n");
+		return -EINVAL;
+	}
+
+	inputclkfs	= HDMI_AUD_INPUTCLKFS_64FS;
+	conf0		= HDMI_AUD_CONF0_I2S_ALL_ENABLE;
+
+	switch (hparms->sample_width) {
+	case 16:
+		conf1 = HDMI_AUD_CONF1_WIDTH_16;
+		break;
+	case 24:
+	case 32:
+		conf1 = HDMI_AUD_CONF1_WIDTH_24;
+		break;
+	}
+
+	dw_hdmi_set_sample_rate(hdmi, hparms->sample_rate);
+
+	hdmi_write(audio, inputclkfs, HDMI_AUD_INPUTCLKFS);
+	hdmi_write(audio, conf0, HDMI_AUD_CONF0);
+	hdmi_write(audio, conf1, HDMI_AUD_CONF1);
+
+	dw_hdmi_audio_enable(hdmi);
+
+	return 0;
+}
+
+static void dw_hdmi_i2s_audio_shutdown(struct device *dev, void *data)
+{
+	struct dw_hdmi_i2s_audio_data *audio = data;
+	struct dw_hdmi *hdmi = audio->hdmi;
+
+	dw_hdmi_audio_disable(hdmi);
+
+	hdmi_write(audio, HDMI_AUD_CONF0_SW_RESET, HDMI_AUD_CONF0);
+}
+
+static struct hdmi_codec_ops dw_hdmi_i2s_ops = {
+	.hw_params	= dw_hdmi_i2s_hw_params,
+	.audio_shutdown	= dw_hdmi_i2s_audio_shutdown,
+};
+
+static int snd_dw_hdmi_probe(struct platform_device *pdev)
+{
+	struct dw_hdmi_i2s_audio_data *audio = pdev->dev.platform_data;
+	struct platform_device_info pdevinfo;
+	struct hdmi_codec_pdata pdata;
+	struct platform_device *platform;
+
+	pdata.ops		= &dw_hdmi_i2s_ops;
+	pdata.i2s		= 1;
+	pdata.max_i2s_channels	= 6;
+	pdata.data		= audio;
+
+	memset(&pdevinfo, 0, sizeof(pdevinfo));
+	pdevinfo.parent		= pdev->dev.parent;
+	pdevinfo.id		= PLATFORM_DEVID_AUTO;
+	pdevinfo.name		= HDMI_CODEC_DRV_NAME;
+	pdevinfo.data		= &pdata;
+	pdevinfo.size_data	= sizeof(pdata);
+	pdevinfo.dma_mask	= DMA_BIT_MASK(32);
+
+	platform = platform_device_register_full(&pdevinfo);
+	if (IS_ERR(platform))
+		return PTR_ERR(platform);
+
+	dev_set_drvdata(&pdev->dev, platform);
+
+	return 0;
+}
+
+static int snd_dw_hdmi_remove(struct platform_device *pdev)
+{
+	struct platform_device *platform = dev_get_drvdata(&pdev->dev);
+
+	platform_device_unregister(platform);
+
+	return 0;
+}
+
+static struct platform_driver snd_dw_hdmi_driver = {
+	.probe	= snd_dw_hdmi_probe,
+	.remove	= snd_dw_hdmi_remove,
+	.driver	= {
+		.name = DRIVER_NAME,
+		.owner = THIS_MODULE,
+	},
+};
+module_platform_driver(snd_dw_hdmi_driver);
+
+MODULE_AUTHOR("Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>");
+MODULE_DESCRIPTION("Synopsis Designware HDMI I2S ALSA SoC interface");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:" DRIVER_NAME);
diff --git a/drivers/gpu/drm/bridge/dw-hdmi.c b/drivers/gpu/drm/bridge/dw-hdmi.c
index ab7023e5dfde..235ce7d1583d 100644
--- a/drivers/gpu/drm/bridge/dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/dw-hdmi.c
@@ -1,14 +1,15 @@
 /*
+ * DesignWare High-Definition Multimedia Interface (HDMI) driver
+ *
+ * Copyright (C) 2013-2015 Mentor Graphics Inc.
  * Copyright (C) 2011-2013 Freescale Semiconductor, Inc.
+ * Copyright (C) 2010, Guennadi Liakhovetski <g.liakhovetski@gmx.de>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
- * Designware High-Definition Multimedia Interface (HDMI) driver
- *
- * Copyright (C) 2010, Guennadi Liakhovetski <g.liakhovetski@gmx.de>
  */
 #include <linux/module.h>
 #include <linux/irq.h>
@@ -101,6 +102,17 @@ struct hdmi_data_info {
 	struct hdmi_vmode video_mode;
 };
 
+struct dw_hdmi_i2c {
+	struct i2c_adapter	adap;
+
+	struct mutex		lock;	/* used to serialize data transfers */
+	struct completion	cmp;
+	u8			stat;
+
+	u8			slave_reg;
+	bool			is_regaddr;
+};
+
 struct dw_hdmi {
 	struct drm_connector connector;
 	struct drm_encoder *encoder;
@@ -111,6 +123,7 @@ struct dw_hdmi {
 	struct device *dev;
 	struct clk *isfr_clk;
 	struct clk *iahb_clk;
+	struct dw_hdmi_i2c *i2c;
 
 	struct hdmi_data_info hdmi_data;
 	const struct dw_hdmi_plat_data *plat_data;
@@ -198,6 +211,201 @@ static void hdmi_mask_writeb(struct dw_hdmi *hdmi, u8 data, unsigned int reg,
 	hdmi_modb(hdmi, data << shift, mask, reg);
 }
 
+static void dw_hdmi_i2c_init(struct dw_hdmi *hdmi)
+{
+	/* Software reset */
+	hdmi_writeb(hdmi, 0x00, HDMI_I2CM_SOFTRSTZ);
+
+	/* Set Standard Mode speed (determined to be 100KHz on iMX6) */
+	hdmi_writeb(hdmi, 0x00, HDMI_I2CM_DIV);
+
+	/* Set done, not acknowledged and arbitration interrupt polarities */
+	hdmi_writeb(hdmi, HDMI_I2CM_INT_DONE_POL, HDMI_I2CM_INT);
+	hdmi_writeb(hdmi, HDMI_I2CM_CTLINT_NAC_POL | HDMI_I2CM_CTLINT_ARB_POL,
+		    HDMI_I2CM_CTLINT);
+
+	/* Clear DONE and ERROR interrupts */
+	hdmi_writeb(hdmi, HDMI_IH_I2CM_STAT0_ERROR | HDMI_IH_I2CM_STAT0_DONE,
+		    HDMI_IH_I2CM_STAT0);
+
+	/* Mute DONE and ERROR interrupts */
+	hdmi_writeb(hdmi, HDMI_IH_I2CM_STAT0_ERROR | HDMI_IH_I2CM_STAT0_DONE,
+		    HDMI_IH_MUTE_I2CM_STAT0);
+}
+
+static int dw_hdmi_i2c_read(struct dw_hdmi *hdmi,
+			    unsigned char *buf, unsigned int length)
+{
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	int stat;
+
+	if (!i2c->is_regaddr) {
+		dev_dbg(hdmi->dev, "set read register address to 0\n");
+		i2c->slave_reg = 0x00;
+		i2c->is_regaddr = true;
+	}
+
+	while (length--) {
+		reinit_completion(&i2c->cmp);
+
+		hdmi_writeb(hdmi, i2c->slave_reg++, HDMI_I2CM_ADDRESS);
+		hdmi_writeb(hdmi, HDMI_I2CM_OPERATION_READ,
+			    HDMI_I2CM_OPERATION);
+
+		stat = wait_for_completion_timeout(&i2c->cmp, HZ / 10);
+		if (!stat)
+			return -EAGAIN;
+
+		/* Check for error condition on the bus */
+		if (i2c->stat & HDMI_IH_I2CM_STAT0_ERROR)
+			return -EIO;
+
+		*buf++ = hdmi_readb(hdmi, HDMI_I2CM_DATAI);
+	}
+
+	return 0;
+}
+
+static int dw_hdmi_i2c_write(struct dw_hdmi *hdmi,
+			     unsigned char *buf, unsigned int length)
+{
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	int stat;
+
+	if (!i2c->is_regaddr) {
+		/* Use the first write byte as register address */
+		i2c->slave_reg = buf[0];
+		length--;
+		buf++;
+		i2c->is_regaddr = true;
+	}
+
+	while (length--) {
+		reinit_completion(&i2c->cmp);
+
+		hdmi_writeb(hdmi, *buf++, HDMI_I2CM_DATAO);
+		hdmi_writeb(hdmi, i2c->slave_reg++, HDMI_I2CM_ADDRESS);
+		hdmi_writeb(hdmi, HDMI_I2CM_OPERATION_WRITE,
+			    HDMI_I2CM_OPERATION);
+
+		stat = wait_for_completion_timeout(&i2c->cmp, HZ / 10);
+		if (!stat)
+			return -EAGAIN;
+
+		/* Check for error condition on the bus */
+		if (i2c->stat & HDMI_IH_I2CM_STAT0_ERROR)
+			return -EIO;
+	}
+
+	return 0;
+}
+
+static int dw_hdmi_i2c_xfer(struct i2c_adapter *adap,
+			    struct i2c_msg *msgs, int num)
+{
+	struct dw_hdmi *hdmi = i2c_get_adapdata(adap);
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	u8 addr = msgs[0].addr;
+	int i, ret = 0;
+
+	dev_dbg(hdmi->dev, "xfer: num: %d, addr: %#x\n", num, addr);
+
+	for (i = 0; i < num; i++) {
+		if (msgs[i].addr != addr) {
+			dev_warn(hdmi->dev,
+				 "unsupported transfer, changed slave address\n");
+			return -EOPNOTSUPP;
+		}
+
+		if (msgs[i].len == 0) {
+			dev_dbg(hdmi->dev,
+				"unsupported transfer %d/%d, no data\n",
+				i + 1, num);
+			return -EOPNOTSUPP;
+		}
+	}
+
+	mutex_lock(&i2c->lock);
+
+	/* Unmute DONE and ERROR interrupts */
+	hdmi_writeb(hdmi, 0x00, HDMI_IH_MUTE_I2CM_STAT0);
+
+	/* Set slave device address taken from the first I2C message */
+	hdmi_writeb(hdmi, addr, HDMI_I2CM_SLAVE);
+
+	/* Set slave device register address on transfer */
+	i2c->is_regaddr = false;
+
+	for (i = 0; i < num; i++) {
+		dev_dbg(hdmi->dev, "xfer: num: %d/%d, len: %d, flags: %#x\n",
+			i + 1, num, msgs[i].len, msgs[i].flags);
+
+		if (msgs[i].flags & I2C_M_RD)
+			ret = dw_hdmi_i2c_read(hdmi, msgs[i].buf, msgs[i].len);
+		else
+			ret = dw_hdmi_i2c_write(hdmi, msgs[i].buf, msgs[i].len);
+
+		if (ret < 0)
+			break;
+	}
+
+	if (!ret)
+		ret = num;
+
+	/* Mute DONE and ERROR interrupts */
+	hdmi_writeb(hdmi, HDMI_IH_I2CM_STAT0_ERROR | HDMI_IH_I2CM_STAT0_DONE,
+		    HDMI_IH_MUTE_I2CM_STAT0);
+
+	mutex_unlock(&i2c->lock);
+
+	return ret;
+}
+
+static u32 dw_hdmi_i2c_func(struct i2c_adapter *adapter)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm dw_hdmi_algorithm = {
+	.master_xfer	= dw_hdmi_i2c_xfer,
+	.functionality	= dw_hdmi_i2c_func,
+};
+
+static struct i2c_adapter *dw_hdmi_i2c_adapter(struct dw_hdmi *hdmi)
+{
+	struct i2c_adapter *adap;
+	struct dw_hdmi_i2c *i2c;
+	int ret;
+
+	i2c = devm_kzalloc(hdmi->dev, sizeof(*i2c), GFP_KERNEL);
+	if (!i2c)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_init(&i2c->lock);
+	init_completion(&i2c->cmp);
+
+	adap = &i2c->adap;
+	adap->class = I2C_CLASS_DDC;
+	adap->owner = THIS_MODULE;
+	adap->dev.parent = hdmi->dev;
+	adap->algo = &dw_hdmi_algorithm;
+	strlcpy(adap->name, "DesignWare HDMI", sizeof(adap->name));
+	i2c_set_adapdata(adap, hdmi);
+
+	ret = i2c_add_adapter(adap);
+	if (ret) {
+		dev_warn(hdmi->dev, "cannot add %s I2C adapter\n", adap->name);
+		devm_kfree(hdmi->dev, i2c);
+		return ERR_PTR(ret);
+	}
+
+	hdmi->i2c = i2c;
+
+	dev_info(hdmi->dev, "registered %s I2C bus driver\n", adap->name);
+
+	return adap;
+}
+
 static void hdmi_set_cts_n(struct dw_hdmi *hdmi, unsigned int cts,
 			   unsigned int n)
 {
@@ -1512,16 +1720,40 @@ static const struct drm_bridge_funcs dw_hdmi_bridge_funcs = {
 	.mode_set = dw_hdmi_bridge_mode_set,
 };
 
+static irqreturn_t dw_hdmi_i2c_irq(struct dw_hdmi *hdmi)
+{
+	struct dw_hdmi_i2c *i2c = hdmi->i2c;
+	unsigned int stat;
+
+	stat = hdmi_readb(hdmi, HDMI_IH_I2CM_STAT0);
+	if (!stat)
+		return IRQ_NONE;
+
+	hdmi_writeb(hdmi, stat, HDMI_IH_I2CM_STAT0);
+
+	i2c->stat = stat;
+
+	complete(&i2c->cmp);
+
+	return IRQ_HANDLED;
+}
+
 static irqreturn_t dw_hdmi_hardirq(int irq, void *dev_id)
 {
 	struct dw_hdmi *hdmi = dev_id;
 	u8 intr_stat;
+	irqreturn_t ret = IRQ_NONE;
+
+	if (hdmi->i2c)
+		ret = dw_hdmi_i2c_irq(hdmi);
 
 	intr_stat = hdmi_readb(hdmi, HDMI_IH_PHY_STAT0);
-	if (intr_stat)
+	if (intr_stat) {
 		hdmi_writeb(hdmi, ~0, HDMI_IH_MUTE_PHY_STAT0);
+		return IRQ_WAKE_THREAD;
+	}
 
-	return intr_stat ? IRQ_WAKE_THREAD : IRQ_NONE;
+	return ret;
 }
 
 static irqreturn_t dw_hdmi_irq(int irq, void *dev_id)
@@ -1639,10 +1871,11 @@ int dw_hdmi_bind(struct device *dev, struct device *master,
 	struct device_node *np = dev->of_node;
 	struct platform_device_info pdevinfo;
 	struct device_node *ddc_node;
-	struct dw_hdmi_audio_data audio;
 	struct dw_hdmi *hdmi;
 	int ret;
 	u32 val = 1;
+	u8 config0;
+	u8 config1;
 
 	hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL);
 	if (!hdmi)
@@ -1681,7 +1914,7 @@ int dw_hdmi_bind(struct device *dev, struct device *master,
 
 	ddc_node = of_parse_phandle(np, "ddc-i2c-bus", 0);
 	if (ddc_node) {
-		hdmi->ddc = of_find_i2c_adapter_by_node(ddc_node);
+		hdmi->ddc = of_get_i2c_adapter_by_node(ddc_node);
 		of_node_put(ddc_node);
 		if (!hdmi->ddc) {
 			dev_dbg(hdmi->dev, "failed to read ddc node\n");
@@ -1693,20 +1926,22 @@ int dw_hdmi_bind(struct device *dev, struct device *master,
 	}
 
 	hdmi->regs = devm_ioremap_resource(dev, iores);
-	if (IS_ERR(hdmi->regs))
-		return PTR_ERR(hdmi->regs);
+	if (IS_ERR(hdmi->regs)) {
+		ret = PTR_ERR(hdmi->regs);
+		goto err_res;
+	}
 
 	hdmi->isfr_clk = devm_clk_get(hdmi->dev, "isfr");
 	if (IS_ERR(hdmi->isfr_clk)) {
 		ret = PTR_ERR(hdmi->isfr_clk);
 		dev_err(hdmi->dev, "Unable to get HDMI isfr clk: %d\n", ret);
-		return ret;
+		goto err_res;
 	}
 
 	ret = clk_prepare_enable(hdmi->isfr_clk);
 	if (ret) {
 		dev_err(hdmi->dev, "Cannot enable HDMI isfr clock: %d\n", ret);
-		return ret;
+		goto err_res;
 	}
 
 	hdmi->iahb_clk = devm_clk_get(hdmi->dev, "iahb");
@@ -1744,6 +1979,13 @@ int dw_hdmi_bind(struct device *dev, struct device *master,
 	 */
 	hdmi_init_clk_regenerator(hdmi);
 
+	/* If DDC bus is not specified, try to register HDMI I2C bus */
+	if (!hdmi->ddc) {
+		hdmi->ddc = dw_hdmi_i2c_adapter(hdmi);
+		if (IS_ERR(hdmi->ddc))
+			hdmi->ddc = NULL;
+	}
+
 	/*
 	 * Configure registers related to HDMI interrupt
 	 * generation before registering IRQ.
@@ -1770,7 +2012,12 @@ int dw_hdmi_bind(struct device *dev, struct device *master,
 	pdevinfo.parent = dev;
 	pdevinfo.id = PLATFORM_DEVID_AUTO;
 
-	if (hdmi_readb(hdmi, HDMI_CONFIG1_ID) & HDMI_CONFIG1_AHB) {
+	config0 = hdmi_readb(hdmi, HDMI_CONFIG0_ID);
+	config1 = hdmi_readb(hdmi, HDMI_CONFIG1_ID);
+
+	if (config1 & HDMI_CONFIG1_AHB) {
+		struct dw_hdmi_audio_data audio;
+
 		audio.phys = iores->start;
 		audio.base = hdmi->regs;
 		audio.irq = irq;
@@ -1782,16 +2029,39 @@ int dw_hdmi_bind(struct device *dev, struct device *master,
 		pdevinfo.size_data = sizeof(audio);
 		pdevinfo.dma_mask = DMA_BIT_MASK(32);
 		hdmi->audio = platform_device_register_full(&pdevinfo);
+	} else if (config0 & HDMI_CONFIG0_I2S) {
+		struct dw_hdmi_i2s_audio_data audio;
+
+		audio.hdmi	= hdmi;
+		audio.write	= hdmi_writeb;
+		audio.read	= hdmi_readb;
+
+		pdevinfo.name = "dw-hdmi-i2s-audio";
+		pdevinfo.data = &audio;
+		pdevinfo.size_data = sizeof(audio);
+		pdevinfo.dma_mask = DMA_BIT_MASK(32);
+		hdmi->audio = platform_device_register_full(&pdevinfo);
 	}
 
+	/* Reset HDMI DDC I2C master controller and mute I2CM interrupts */
+	if (hdmi->i2c)
+		dw_hdmi_i2c_init(hdmi);
+
 	dev_set_drvdata(dev, hdmi);
 
 	return 0;
 
 err_iahb:
+	if (hdmi->i2c) {
+		i2c_del_adapter(&hdmi->i2c->adap);
+		hdmi->ddc = NULL;
+	}
+
 	clk_disable_unprepare(hdmi->iahb_clk);
 err_isfr:
 	clk_disable_unprepare(hdmi->isfr_clk);
+err_res:
+	i2c_put_adapter(hdmi->ddc);
 
 	return ret;
 }
@@ -1809,13 +2079,18 @@ void dw_hdmi_unbind(struct device *dev, struct device *master, void *data)
 
 	clk_disable_unprepare(hdmi->iahb_clk);
 	clk_disable_unprepare(hdmi->isfr_clk);
-	i2c_put_adapter(hdmi->ddc);
+
+	if (hdmi->i2c)
+		i2c_del_adapter(&hdmi->i2c->adap);
+	else
+		i2c_put_adapter(hdmi->ddc);
 }
 EXPORT_SYMBOL_GPL(dw_hdmi_unbind);
 
 MODULE_AUTHOR("Sascha Hauer <s.hauer@pengutronix.de>");
 MODULE_AUTHOR("Andy Yan <andy.yan@rock-chips.com>");
 MODULE_AUTHOR("Yakir Yang <ykk@rock-chips.com>");
+MODULE_AUTHOR("Vladimir Zapolskiy <vladimir_zapolskiy@mentor.com>");
 MODULE_DESCRIPTION("DW HDMI transmitter driver");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:dw-hdmi");
diff --git a/drivers/gpu/drm/bridge/dw-hdmi.h b/drivers/gpu/drm/bridge/dw-hdmi.h
index fc9a560429d6..55135bbd0c16 100644
--- a/drivers/gpu/drm/bridge/dw-hdmi.h
+++ b/drivers/gpu/drm/bridge/dw-hdmi.h
@@ -545,6 +545,9 @@
 #define HDMI_I2CM_FS_SCL_LCNT_0_ADDR            0x7E12
 
 enum {
+/* CONFIG0_ID field values */
+	HDMI_CONFIG0_I2S = 0x10,
+
 /* CONFIG1_ID field values */
 	HDMI_CONFIG1_AHB = 0x01,
 
@@ -566,6 +569,10 @@ enum {
 	HDMI_IH_PHY_STAT0_TX_PHY_LOCK = 0x2,
 	HDMI_IH_PHY_STAT0_HPD = 0x1,
 
+/* IH_I2CM_STAT0 and IH_MUTE_I2CM_STAT0 field values */
+	HDMI_IH_I2CM_STAT0_DONE = 0x2,
+	HDMI_IH_I2CM_STAT0_ERROR = 0x1,
+
 /* IH_MUTE_I2CMPHY_STAT0 field values */
 	HDMI_IH_MUTE_I2CMPHY_STAT0_I2CMPHYDONE = 0x2,
 	HDMI_IH_MUTE_I2CMPHY_STAT0_I2CMPHYERROR = 0x1,
@@ -887,6 +894,17 @@ enum {
 	HDMI_PHY_I2CM_CTLINT_ADDR_ARBITRATION_POL = 0x08,
 	HDMI_PHY_I2CM_CTLINT_ADDR_ARBITRATION_MASK = 0x04,
 
+/* AUD_CONF0 field values */
+	HDMI_AUD_CONF0_SW_RESET = 0x80,
+	HDMI_AUD_CONF0_I2S_ALL_ENABLE = 0x2F,
+
+/* AUD_CONF1 field values */
+	HDMI_AUD_CONF1_MODE_I2S = 0x00,
+	HDMI_AUD_CONF1_MODE_RIGHT_J = 0x02,
+	HDMI_AUD_CONF1_MODE_LEFT_J = 0x04,
+	HDMI_AUD_CONF1_WIDTH_16 = 0x10,
+	HDMI_AUD_CONF1_WIDTH_24 = 0x18,
+
 /* AUD_CTS3 field values */
 	HDMI_AUD_CTS3_N_SHIFT_OFFSET = 5,
 	HDMI_AUD_CTS3_N_SHIFT_MASK = 0xe0,
@@ -901,6 +919,12 @@ enum {
 	HDMI_AUD_CTS3_CTS_MANUAL = 0x10,
 	HDMI_AUD_CTS3_AUDCTS19_16_MASK = 0x0f,
 
+/* HDMI_AUD_INPUTCLKFS field values */
+	HDMI_AUD_INPUTCLKFS_128FS = 0,
+	HDMI_AUD_INPUTCLKFS_256FS = 1,
+	HDMI_AUD_INPUTCLKFS_512FS = 2,
+	HDMI_AUD_INPUTCLKFS_64FS = 4,
+
 /* AHB_DMA_CONF0 field values */
 	HDMI_AHB_DMA_CONF0_SW_FIFO_RST_OFFSET = 7,
 	HDMI_AHB_DMA_CONF0_SW_FIFO_RST_MASK = 0x80,
@@ -1032,6 +1056,21 @@ enum {
 	HDMI_A_VIDPOLCFG_HSYNCPOL_MASK = 0x2,
 	HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_HIGH = 0x2,
 	HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_LOW = 0x0,
+
+/* I2CM_OPERATION field values */
+	HDMI_I2CM_OPERATION_WRITE = 0x10,
+	HDMI_I2CM_OPERATION_READ_EXT = 0x2,
+	HDMI_I2CM_OPERATION_READ = 0x1,
+
+/* I2CM_INT field values */
+	HDMI_I2CM_INT_DONE_POL = 0x8,
+	HDMI_I2CM_INT_DONE_MASK = 0x4,
+
+/* I2CM_CTLINT field values */
+	HDMI_I2CM_CTLINT_NAC_POL = 0x80,
+	HDMI_I2CM_CTLINT_NAC_MASK = 0x40,
+	HDMI_I2CM_CTLINT_ARB_POL = 0x8,
+	HDMI_I2CM_CTLINT_ARB_MASK = 0x4,
 };
 
 #endif /* __DW_HDMI_H__ */
diff --git a/drivers/gpu/drm/bridge/nxp-ptn3460.c b/drivers/gpu/drm/bridge/nxp-ptn3460.c
index f1a99938e924..27f98c518dde 100644
--- a/drivers/gpu/drm/bridge/nxp-ptn3460.c
+++ b/drivers/gpu/drm/bridge/nxp-ptn3460.c
@@ -239,16 +239,9 @@ static const struct drm_connector_helper_funcs ptn3460_connector_helper_funcs =
 	.get_modes = ptn3460_get_modes,
 };
 
-static enum drm_connector_status ptn3460_detect(struct drm_connector *connector,
-		bool force)
-{
-	return connector_status_connected;
-}
-
 static const struct drm_connector_funcs ptn3460_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.detect = ptn3460_detect,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/bridge/parade-ps8622.c b/drivers/gpu/drm/bridge/parade-ps8622.c
index 6f7c2f9860d2..ac8cc5b50d9f 100644
--- a/drivers/gpu/drm/bridge/parade-ps8622.c
+++ b/drivers/gpu/drm/bridge/parade-ps8622.c
@@ -477,16 +477,9 @@ static const struct drm_connector_helper_funcs ps8622_connector_helper_funcs = {
 	.get_modes = ps8622_get_modes,
 };
 
-static enum drm_connector_status ps8622_detect(struct drm_connector *connector,
-								bool force)
-{
-	return connector_status_connected;
-}
-
 static const struct drm_connector_funcs ps8622_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.detect = ps8622_detect,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/bridge/sil-sii8620.c b/drivers/gpu/drm/bridge/sil-sii8620.c
new file mode 100644
index 000000000000..b2c267df7ee7
--- /dev/null
+++ b/drivers/gpu/drm/bridge/sil-sii8620.c
@@ -0,0 +1,1564 @@
+/*
+ * Silicon Image SiI8620 HDMI/MHL bridge driver
+ *
+ * Copyright (C) 2015, Samsung Electronics Co., Ltd.
+ * Andrzej Hajda <a.hajda@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <drm/bridge/mhl.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_edid.h>
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+
+#include "sil-sii8620.h"
+
+#define VAL_RX_HDMI_CTRL2_DEFVAL	VAL_RX_HDMI_CTRL2_IDLE_CNT(3)
+
+enum sii8620_mode {
+	CM_DISCONNECTED,
+	CM_DISCOVERY,
+	CM_MHL1,
+	CM_MHL3,
+	CM_ECBUS_S
+};
+
+enum sii8620_sink_type {
+	SINK_NONE,
+	SINK_HDMI,
+	SINK_DVI
+};
+
+enum sii8620_mt_state {
+	MT_STATE_READY,
+	MT_STATE_BUSY,
+	MT_STATE_DONE
+};
+
+struct sii8620 {
+	struct drm_bridge bridge;
+	struct device *dev;
+	struct clk *clk_xtal;
+	struct gpio_desc *gpio_reset;
+	struct gpio_desc *gpio_int;
+	struct regulator_bulk_data supplies[2];
+	struct mutex lock; /* context lock, protects fields below */
+	int error;
+	enum sii8620_mode mode;
+	enum sii8620_sink_type sink_type;
+	u8 cbus_status;
+	u8 stat[MHL_DST_SIZE];
+	u8 xstat[MHL_XDS_SIZE];
+	u8 devcap[MHL_DCAP_SIZE];
+	u8 xdevcap[MHL_XDC_SIZE];
+	u8 avif[19];
+	struct edid *edid;
+	unsigned int gen2_write_burst:1;
+	enum sii8620_mt_state mt_state;
+	struct list_head mt_queue;
+};
+
+struct sii8620_mt_msg;
+
+typedef void (*sii8620_mt_msg_cb)(struct sii8620 *ctx,
+				  struct sii8620_mt_msg *msg);
+
+struct sii8620_mt_msg {
+	struct list_head node;
+	u8 reg[4];
+	u8 ret;
+	sii8620_mt_msg_cb send;
+	sii8620_mt_msg_cb recv;
+};
+
+static const u8 sii8620_i2c_page[] = {
+	0x39, /* Main System */
+	0x3d, /* TDM and HSIC */
+	0x49, /* TMDS Receiver, MHL EDID */
+	0x4d, /* eMSC, HDCP, HSIC */
+	0x5d, /* MHL Spec */
+	0x64, /* MHL CBUS */
+	0x59, /* Hardware TPI (Transmitter Programming Interface) */
+	0x61, /* eCBUS-S, eCBUS-D */
+};
+
+static void sii8620_fetch_edid(struct sii8620 *ctx);
+static void sii8620_set_upstream_edid(struct sii8620 *ctx);
+static void sii8620_enable_hpd(struct sii8620 *ctx);
+static void sii8620_mhl_disconnected(struct sii8620 *ctx);
+
+static int sii8620_clear_error(struct sii8620 *ctx)
+{
+	int ret = ctx->error;
+
+	ctx->error = 0;
+	return ret;
+}
+
+static void sii8620_read_buf(struct sii8620 *ctx, u16 addr, u8 *buf, int len)
+{
+	struct device *dev = ctx->dev;
+	struct i2c_client *client = to_i2c_client(dev);
+	u8 data = addr;
+	struct i2c_msg msg[] = {
+		{
+			.addr = sii8620_i2c_page[addr >> 8],
+			.flags = client->flags,
+			.len = 1,
+			.buf = &data
+		},
+		{
+			.addr = sii8620_i2c_page[addr >> 8],
+			.flags = client->flags | I2C_M_RD,
+			.len = len,
+			.buf = buf
+		},
+	};
+	int ret;
+
+	if (ctx->error)
+		return;
+
+	ret = i2c_transfer(client->adapter, msg, 2);
+	dev_dbg(dev, "read at %04x: %*ph, %d\n", addr, len, buf, ret);
+
+	if (ret != 2) {
+		dev_err(dev, "Read at %#06x of %d bytes failed with code %d.\n",
+			addr, len, ret);
+		ctx->error = ret < 0 ? ret : -EIO;
+	}
+}
+
+static u8 sii8620_readb(struct sii8620 *ctx, u16 addr)
+{
+	u8 ret;
+
+	sii8620_read_buf(ctx, addr, &ret, 1);
+	return ret;
+}
+
+static void sii8620_write_buf(struct sii8620 *ctx, u16 addr, const u8 *buf,
+			      int len)
+{
+	struct device *dev = ctx->dev;
+	struct i2c_client *client = to_i2c_client(dev);
+	u8 data[2];
+	struct i2c_msg msg = {
+		.addr = sii8620_i2c_page[addr >> 8],
+		.flags = client->flags,
+		.len = len + 1,
+	};
+	int ret;
+
+	if (ctx->error)
+		return;
+
+	if (len > 1) {
+		msg.buf = kmalloc(len + 1, GFP_KERNEL);
+		if (!msg.buf) {
+			ctx->error = -ENOMEM;
+			return;
+		}
+		memcpy(msg.buf + 1, buf, len);
+	} else {
+		msg.buf = data;
+		msg.buf[1] = *buf;
+	}
+
+	msg.buf[0] = addr;
+
+	ret = i2c_transfer(client->adapter, &msg, 1);
+	dev_dbg(dev, "write at %04x: %*ph, %d\n", addr, len, buf, ret);
+
+	if (ret != 1) {
+		dev_err(dev, "Write at %#06x of %*ph failed with code %d.\n",
+			addr, len, buf, ret);
+		ctx->error = ret ?: -EIO;
+	}
+
+	if (len > 1)
+		kfree(msg.buf);
+}
+
+#define sii8620_write(ctx, addr, arr...) \
+({\
+	u8 d[] = { arr }; \
+	sii8620_write_buf(ctx, addr, d, ARRAY_SIZE(d)); \
+})
+
+static void __sii8620_write_seq(struct sii8620 *ctx, const u16 *seq, int len)
+{
+	int i;
+
+	for (i = 0; i < len; i += 2)
+		sii8620_write(ctx, seq[i], seq[i + 1]);
+}
+
+#define sii8620_write_seq(ctx, seq...) \
+({\
+	const u16 d[] = { seq }; \
+	__sii8620_write_seq(ctx, d, ARRAY_SIZE(d)); \
+})
+
+#define sii8620_write_seq_static(ctx, seq...) \
+({\
+	static const u16 d[] = { seq }; \
+	__sii8620_write_seq(ctx, d, ARRAY_SIZE(d)); \
+})
+
+static void sii8620_setbits(struct sii8620 *ctx, u16 addr, u8 mask, u8 val)
+{
+	val = (val & mask) | (sii8620_readb(ctx, addr) & ~mask);
+	sii8620_write(ctx, addr, val);
+}
+
+static void sii8620_mt_cleanup(struct sii8620 *ctx)
+{
+	struct sii8620_mt_msg *msg, *n;
+
+	list_for_each_entry_safe(msg, n, &ctx->mt_queue, node) {
+		list_del(&msg->node);
+		kfree(msg);
+	}
+	ctx->mt_state = MT_STATE_READY;
+}
+
+static void sii8620_mt_work(struct sii8620 *ctx)
+{
+	struct sii8620_mt_msg *msg;
+
+	if (ctx->error)
+		return;
+	if (ctx->mt_state == MT_STATE_BUSY || list_empty(&ctx->mt_queue))
+		return;
+
+	if (ctx->mt_state == MT_STATE_DONE) {
+		ctx->mt_state = MT_STATE_READY;
+		msg = list_first_entry(&ctx->mt_queue, struct sii8620_mt_msg,
+				       node);
+		if (msg->recv)
+			msg->recv(ctx, msg);
+		list_del(&msg->node);
+		kfree(msg);
+	}
+
+	if (ctx->mt_state != MT_STATE_READY || list_empty(&ctx->mt_queue))
+		return;
+
+	ctx->mt_state = MT_STATE_BUSY;
+	msg = list_first_entry(&ctx->mt_queue, struct sii8620_mt_msg, node);
+	if (msg->send)
+		msg->send(ctx, msg);
+}
+
+static void sii8620_mt_msc_cmd_send(struct sii8620 *ctx,
+				    struct sii8620_mt_msg *msg)
+{
+	switch (msg->reg[0]) {
+	case MHL_WRITE_STAT:
+	case MHL_SET_INT:
+		sii8620_write_buf(ctx, REG_MSC_CMD_OR_OFFSET, msg->reg + 1, 2);
+		sii8620_write(ctx, REG_MSC_COMMAND_START,
+			      BIT_MSC_COMMAND_START_WRITE_STAT);
+		break;
+	case MHL_MSC_MSG:
+		sii8620_write_buf(ctx, REG_MSC_CMD_OR_OFFSET, msg->reg, 3);
+		sii8620_write(ctx, REG_MSC_COMMAND_START,
+			      BIT_MSC_COMMAND_START_MSC_MSG);
+		break;
+	default:
+		dev_err(ctx->dev, "%s: command %#x not supported\n", __func__,
+			msg->reg[0]);
+	}
+}
+
+static struct sii8620_mt_msg *sii8620_mt_msg_new(struct sii8620 *ctx)
+{
+	struct sii8620_mt_msg *msg = kzalloc(sizeof(*msg), GFP_KERNEL);
+
+	if (!msg)
+		ctx->error = -ENOMEM;
+	else
+		list_add_tail(&msg->node, &ctx->mt_queue);
+
+	return msg;
+}
+
+static void sii8620_mt_msc_cmd(struct sii8620 *ctx, u8 cmd, u8 arg1, u8 arg2)
+{
+	struct sii8620_mt_msg *msg = sii8620_mt_msg_new(ctx);
+
+	if (!msg)
+		return;
+
+	msg->reg[0] = cmd;
+	msg->reg[1] = arg1;
+	msg->reg[2] = arg2;
+	msg->send = sii8620_mt_msc_cmd_send;
+}
+
+static void sii8620_mt_write_stat(struct sii8620 *ctx, u8 reg, u8 val)
+{
+	sii8620_mt_msc_cmd(ctx, MHL_WRITE_STAT, reg, val);
+}
+
+static inline void sii8620_mt_set_int(struct sii8620 *ctx, u8 irq, u8 mask)
+{
+	sii8620_mt_msc_cmd(ctx, MHL_SET_INT, irq, mask);
+}
+
+static void sii8620_mt_msc_msg(struct sii8620 *ctx, u8 cmd, u8 data)
+{
+	sii8620_mt_msc_cmd(ctx, MHL_MSC_MSG, cmd, data);
+}
+
+static void sii8620_mt_rap(struct sii8620 *ctx, u8 code)
+{
+	sii8620_mt_msc_msg(ctx, MHL_MSC_MSG_RAP, code);
+}
+
+static void sii8620_mt_read_devcap_send(struct sii8620 *ctx,
+					struct sii8620_mt_msg *msg)
+{
+	u8 ctrl = BIT_EDID_CTRL_DEVCAP_SELECT_DEVCAP
+			| BIT_EDID_CTRL_EDID_FIFO_ADDR_AUTO
+			| BIT_EDID_CTRL_EDID_MODE_EN;
+
+	if (msg->reg[0] == MHL_READ_XDEVCAP)
+		ctrl |= BIT_EDID_CTRL_XDEVCAP_EN;
+
+	sii8620_write_seq(ctx,
+		REG_INTR9_MASK, BIT_INTR9_DEVCAP_DONE,
+		REG_EDID_CTRL, ctrl,
+		REG_TPI_CBUS_START, BIT_TPI_CBUS_START_GET_DEVCAP_START
+	);
+}
+
+/* copy src to dst and set changed bits in src */
+static void sii8620_update_array(u8 *dst, u8 *src, int count)
+{
+	while (--count >= 0) {
+		*src ^= *dst;
+		*dst++ ^= *src++;
+	}
+}
+
+static void sii8620_mr_devcap(struct sii8620 *ctx)
+{
+	static const char * const sink_str[] = {
+		[SINK_NONE] = "NONE",
+		[SINK_HDMI] = "HDMI",
+		[SINK_DVI] = "DVI"
+	};
+
+	u8 dcap[MHL_DCAP_SIZE];
+	char sink_name[20];
+	struct device *dev = ctx->dev;
+
+	sii8620_read_buf(ctx, REG_EDID_FIFO_RD_DATA, dcap, MHL_DCAP_SIZE);
+	if (ctx->error < 0)
+		return;
+
+	dev_info(dev, "dcap: %*ph\n", MHL_DCAP_SIZE, dcap);
+	dev_info(dev, "detected dongle MHL %d.%d, ChipID %02x%02x:%02x%02x\n",
+		 dcap[MHL_DCAP_MHL_VERSION] / 16,
+		 dcap[MHL_DCAP_MHL_VERSION] % 16, dcap[MHL_DCAP_ADOPTER_ID_H],
+		 dcap[MHL_DCAP_ADOPTER_ID_L], dcap[MHL_DCAP_DEVICE_ID_H],
+		 dcap[MHL_DCAP_DEVICE_ID_L]);
+	sii8620_update_array(ctx->devcap, dcap, MHL_DCAP_SIZE);
+
+	if (!(dcap[MHL_DCAP_CAT] & MHL_DCAP_CAT_SINK))
+		return;
+
+	sii8620_fetch_edid(ctx);
+	if (!ctx->edid) {
+		dev_err(ctx->dev, "Cannot fetch EDID\n");
+		sii8620_mhl_disconnected(ctx);
+		return;
+	}
+
+	if (drm_detect_hdmi_monitor(ctx->edid))
+		ctx->sink_type = SINK_HDMI;
+	else
+		ctx->sink_type = SINK_DVI;
+
+	drm_edid_get_monitor_name(ctx->edid, sink_name, ARRAY_SIZE(sink_name));
+
+	dev_info(dev, "detected sink(type: %s): %s\n",
+		 sink_str[ctx->sink_type], sink_name);
+	sii8620_set_upstream_edid(ctx);
+	sii8620_enable_hpd(ctx);
+}
+
+static void sii8620_mr_xdevcap(struct sii8620 *ctx)
+{
+	sii8620_read_buf(ctx, REG_EDID_FIFO_RD_DATA, ctx->xdevcap,
+			 MHL_XDC_SIZE);
+
+	sii8620_mt_write_stat(ctx, MHL_XDS_REG(CURR_ECBUS_MODE),
+			      MHL_XDS_ECBUS_S | MHL_XDS_SLOT_MODE_8BIT);
+	sii8620_mt_rap(ctx, MHL_RAP_CBUS_MODE_UP);
+}
+
+static void sii8620_mt_read_devcap_recv(struct sii8620 *ctx,
+					struct sii8620_mt_msg *msg)
+{
+	u8 ctrl = BIT_EDID_CTRL_DEVCAP_SELECT_DEVCAP
+		| BIT_EDID_CTRL_EDID_FIFO_ADDR_AUTO
+		| BIT_EDID_CTRL_EDID_MODE_EN;
+
+	if (msg->reg[0] == MHL_READ_XDEVCAP)
+		ctrl |= BIT_EDID_CTRL_XDEVCAP_EN;
+
+	sii8620_write_seq(ctx,
+		REG_INTR9_MASK, BIT_INTR9_DEVCAP_DONE | BIT_INTR9_EDID_DONE
+			| BIT_INTR9_EDID_ERROR,
+		REG_EDID_CTRL, ctrl,
+		REG_EDID_FIFO_ADDR, 0
+	);
+
+	if (msg->reg[0] == MHL_READ_XDEVCAP)
+		sii8620_mr_xdevcap(ctx);
+	else
+		sii8620_mr_devcap(ctx);
+}
+
+static void sii8620_mt_read_devcap(struct sii8620 *ctx, bool xdevcap)
+{
+	struct sii8620_mt_msg *msg = sii8620_mt_msg_new(ctx);
+
+	if (!msg)
+		return;
+
+	msg->reg[0] = xdevcap ? MHL_READ_XDEVCAP : MHL_READ_DEVCAP;
+	msg->send = sii8620_mt_read_devcap_send;
+	msg->recv = sii8620_mt_read_devcap_recv;
+}
+
+static void sii8620_fetch_edid(struct sii8620 *ctx)
+{
+	u8 lm_ddc, ddc_cmd, int3, cbus;
+	int fetched, i;
+	int edid_len = EDID_LENGTH;
+	u8 *edid;
+
+	sii8620_readb(ctx, REG_CBUS_STATUS);
+	lm_ddc = sii8620_readb(ctx, REG_LM_DDC);
+	ddc_cmd = sii8620_readb(ctx, REG_DDC_CMD);
+
+	sii8620_write_seq(ctx,
+		REG_INTR9_MASK, 0,
+		REG_EDID_CTRL, BIT_EDID_CTRL_EDID_FIFO_ADDR_AUTO,
+		REG_HDCP2X_POLL_CS, 0x71,
+		REG_HDCP2X_CTRL_0, BIT_HDCP2X_CTRL_0_HDCP2X_HDCPTX,
+		REG_LM_DDC, lm_ddc | BIT_LM_DDC_SW_TPI_EN_DISABLED,
+	);
+
+	for (i = 0; i < 256; ++i) {
+		u8 ddc_stat = sii8620_readb(ctx, REG_DDC_STATUS);
+
+		if (!(ddc_stat & BIT_DDC_STATUS_DDC_I2C_IN_PROG))
+			break;
+		sii8620_write(ctx, REG_DDC_STATUS,
+			      BIT_DDC_STATUS_DDC_FIFO_EMPTY);
+	}
+
+	sii8620_write(ctx, REG_DDC_ADDR, 0x50 << 1);
+
+	edid = kmalloc(EDID_LENGTH, GFP_KERNEL);
+	if (!edid) {
+		ctx->error = -ENOMEM;
+		return;
+	}
+
+#define FETCH_SIZE 16
+	for (fetched = 0; fetched < edid_len; fetched += FETCH_SIZE) {
+		sii8620_readb(ctx, REG_DDC_STATUS);
+		sii8620_write_seq(ctx,
+			REG_DDC_CMD, ddc_cmd | VAL_DDC_CMD_DDC_CMD_ABORT,
+			REG_DDC_CMD, ddc_cmd | VAL_DDC_CMD_DDC_CMD_CLEAR_FIFO,
+			REG_DDC_STATUS, BIT_DDC_STATUS_DDC_FIFO_EMPTY
+		);
+		sii8620_write_seq(ctx,
+			REG_DDC_SEGM, fetched >> 8,
+			REG_DDC_OFFSET, fetched & 0xff,
+			REG_DDC_DIN_CNT1, FETCH_SIZE,
+			REG_DDC_DIN_CNT2, 0,
+			REG_DDC_CMD, ddc_cmd | VAL_DDC_CMD_ENH_DDC_READ_NO_ACK
+		);
+
+		do {
+			int3 = sii8620_readb(ctx, REG_INTR3);
+			cbus = sii8620_readb(ctx, REG_CBUS_STATUS);
+
+			if (int3 & BIT_DDC_CMD_DONE)
+				break;
+
+			if (!(cbus & BIT_CBUS_STATUS_CBUS_CONNECTED)) {
+				kfree(edid);
+				edid = NULL;
+				goto end;
+			}
+		} while (1);
+
+		sii8620_readb(ctx, REG_DDC_STATUS);
+		while (sii8620_readb(ctx, REG_DDC_DOUT_CNT) < FETCH_SIZE)
+			usleep_range(10, 20);
+
+		sii8620_read_buf(ctx, REG_DDC_DATA, edid + fetched, FETCH_SIZE);
+		if (fetched + FETCH_SIZE == EDID_LENGTH) {
+			u8 ext = ((struct edid *)edid)->extensions;
+
+			if (ext) {
+				u8 *new_edid;
+
+				edid_len += ext * EDID_LENGTH;
+				new_edid = krealloc(edid, edid_len, GFP_KERNEL);
+				if (!new_edid) {
+					kfree(edid);
+					ctx->error = -ENOMEM;
+					return;
+				}
+				edid = new_edid;
+			}
+		}
+
+		if (fetched + FETCH_SIZE == edid_len)
+			sii8620_write(ctx, REG_INTR3, int3);
+	}
+
+	sii8620_write(ctx, REG_LM_DDC, lm_ddc);
+
+end:
+	kfree(ctx->edid);
+	ctx->edid = (struct edid *)edid;
+}
+
+static void sii8620_set_upstream_edid(struct sii8620 *ctx)
+{
+	sii8620_setbits(ctx, REG_DPD, BIT_DPD_PDNRX12 | BIT_DPD_PDIDCK_N
+			| BIT_DPD_PD_MHL_CLK_N, 0xff);
+
+	sii8620_write_seq_static(ctx,
+		REG_RX_HDMI_CTRL3, 0x00,
+		REG_PKT_FILTER_0, 0xFF,
+		REG_PKT_FILTER_1, 0xFF,
+		REG_ALICE0_BW_I2C, 0x06
+	);
+
+	sii8620_setbits(ctx, REG_RX_HDMI_CLR_BUFFER,
+			BIT_RX_HDMI_CLR_BUFFER_VSI_CLR_EN, 0xff);
+
+	sii8620_write_seq_static(ctx,
+		REG_EDID_CTRL, BIT_EDID_CTRL_EDID_FIFO_ADDR_AUTO
+			| BIT_EDID_CTRL_EDID_MODE_EN,
+		REG_EDID_FIFO_ADDR, 0,
+	);
+
+	sii8620_write_buf(ctx, REG_EDID_FIFO_WR_DATA, (u8 *)ctx->edid,
+			  (ctx->edid->extensions + 1) * EDID_LENGTH);
+
+	sii8620_write_seq_static(ctx,
+		REG_EDID_CTRL, BIT_EDID_CTRL_EDID_PRIME_VALID
+			| BIT_EDID_CTRL_EDID_FIFO_ADDR_AUTO
+			| BIT_EDID_CTRL_EDID_MODE_EN,
+		REG_INTR5_MASK, BIT_INTR_SCDT_CHANGE,
+		REG_INTR9_MASK, 0
+	);
+}
+
+static void sii8620_xtal_set_rate(struct sii8620 *ctx)
+{
+	static const struct {
+		unsigned int rate;
+		u8 div;
+		u8 tp1;
+	} rates[] = {
+		{ 19200, 0x04, 0x53 },
+		{ 20000, 0x04, 0x62 },
+		{ 24000, 0x05, 0x75 },
+		{ 30000, 0x06, 0x92 },
+		{ 38400, 0x0c, 0xbc },
+	};
+	unsigned long rate = clk_get_rate(ctx->clk_xtal) / 1000;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(rates) - 1; ++i)
+		if (rate <= rates[i].rate)
+			break;
+
+	if (rate != rates[i].rate)
+		dev_err(ctx->dev, "xtal clock rate(%lukHz) not supported, setting MHL for %ukHz.\n",
+			rate, rates[i].rate);
+
+	sii8620_write(ctx, REG_DIV_CTL_MAIN, rates[i].div);
+	sii8620_write(ctx, REG_HDCP2X_TP1, rates[i].tp1);
+}
+
+static int sii8620_hw_on(struct sii8620 *ctx)
+{
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+	if (ret)
+		return ret;
+	usleep_range(10000, 20000);
+	return clk_prepare_enable(ctx->clk_xtal);
+}
+
+static int sii8620_hw_off(struct sii8620 *ctx)
+{
+	clk_disable_unprepare(ctx->clk_xtal);
+	gpiod_set_value(ctx->gpio_reset, 1);
+	return regulator_bulk_disable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+}
+
+static void sii8620_hw_reset(struct sii8620 *ctx)
+{
+	usleep_range(10000, 20000);
+	gpiod_set_value(ctx->gpio_reset, 0);
+	usleep_range(5000, 20000);
+	gpiod_set_value(ctx->gpio_reset, 1);
+	usleep_range(10000, 20000);
+	gpiod_set_value(ctx->gpio_reset, 0);
+	msleep(300);
+}
+
+static void sii8620_cbus_reset(struct sii8620 *ctx)
+{
+	sii8620_write_seq_static(ctx,
+		REG_PWD_SRST, BIT_PWD_SRST_CBUS_RST
+			| BIT_PWD_SRST_CBUS_RST_SW_EN,
+		REG_PWD_SRST, BIT_PWD_SRST_CBUS_RST_SW_EN
+	);
+}
+
+static void sii8620_set_auto_zone(struct sii8620 *ctx)
+{
+	if (ctx->mode != CM_MHL1) {
+		sii8620_write_seq_static(ctx,
+			REG_TX_ZONE_CTL1, 0x0,
+			REG_MHL_PLL_CTL0, VAL_MHL_PLL_CTL0_HDMI_CLK_RATIO_1X
+				| BIT_MHL_PLL_CTL0_CRYSTAL_CLK_SEL
+				| BIT_MHL_PLL_CTL0_ZONE_MASK_OE
+		);
+	} else {
+		sii8620_write_seq_static(ctx,
+			REG_TX_ZONE_CTL1, VAL_TX_ZONE_CTL1_TX_ZONE_CTRL_MODE,
+			REG_MHL_PLL_CTL0, VAL_MHL_PLL_CTL0_HDMI_CLK_RATIO_1X
+				| BIT_MHL_PLL_CTL0_ZONE_MASK_OE
+		);
+	}
+}
+
+static void sii8620_stop_video(struct sii8620 *ctx)
+{
+	u8 uninitialized_var(val);
+
+	sii8620_write_seq_static(ctx,
+		REG_TPI_INTR_EN, 0,
+		REG_HDCP2X_INTR0_MASK, 0,
+		REG_TPI_COPP_DATA2, 0,
+		REG_TPI_INTR_ST0, ~0,
+	);
+
+	switch (ctx->sink_type) {
+	case SINK_DVI:
+		val = BIT_TPI_SC_REG_TMDS_OE_POWER_DOWN
+			| BIT_TPI_SC_TPI_AV_MUTE;
+		break;
+	case SINK_HDMI:
+		val = BIT_TPI_SC_REG_TMDS_OE_POWER_DOWN
+			| BIT_TPI_SC_TPI_AV_MUTE
+			| BIT_TPI_SC_TPI_OUTPUT_MODE_0_HDMI;
+		break;
+	default:
+		return;
+	}
+
+	sii8620_write(ctx, REG_TPI_SC, val);
+}
+
+static void sii8620_start_hdmi(struct sii8620 *ctx)
+{
+	sii8620_write_seq_static(ctx,
+		REG_RX_HDMI_CTRL2, VAL_RX_HDMI_CTRL2_DEFVAL
+			| BIT_RX_HDMI_CTRL2_USE_AV_MUTE,
+		REG_VID_OVRRD, BIT_VID_OVRRD_PP_AUTO_DISABLE
+			| BIT_VID_OVRRD_M1080P_OVRRD,
+		REG_VID_MODE, 0,
+		REG_MHL_TOP_CTL, 0x1,
+		REG_MHLTX_CTL6, 0xa0,
+		REG_TPI_INPUT, VAL_TPI_FORMAT(RGB, FULL),
+		REG_TPI_OUTPUT, VAL_TPI_FORMAT(RGB, FULL),
+	);
+
+	sii8620_mt_write_stat(ctx, MHL_DST_REG(LINK_MODE),
+			      MHL_DST_LM_CLK_MODE_NORMAL |
+			      MHL_DST_LM_PATH_ENABLED);
+
+	sii8620_set_auto_zone(ctx);
+
+	sii8620_write(ctx, REG_TPI_SC, BIT_TPI_SC_TPI_OUTPUT_MODE_0_HDMI);
+
+	sii8620_write_buf(ctx, REG_TPI_AVI_CHSUM, ctx->avif,
+			  ARRAY_SIZE(ctx->avif));
+
+	sii8620_write(ctx, REG_PKT_FILTER_0, 0xa1, 0x2);
+}
+
+static void sii8620_start_video(struct sii8620 *ctx)
+{
+	if (ctx->mode < CM_MHL3)
+		sii8620_stop_video(ctx);
+
+	switch (ctx->sink_type) {
+	case SINK_HDMI:
+		sii8620_start_hdmi(ctx);
+		break;
+	case SINK_DVI:
+	default:
+		break;
+	}
+}
+
+static void sii8620_disable_hpd(struct sii8620 *ctx)
+{
+	sii8620_setbits(ctx, REG_EDID_CTRL, BIT_EDID_CTRL_EDID_PRIME_VALID, 0);
+	sii8620_write_seq_static(ctx,
+		REG_HPD_CTRL, BIT_HPD_CTRL_HPD_OUT_OVR_EN,
+		REG_INTR8_MASK, 0
+	);
+}
+
+static void sii8620_enable_hpd(struct sii8620 *ctx)
+{
+	sii8620_setbits(ctx, REG_TMDS_CSTAT_P3,
+			BIT_TMDS_CSTAT_P3_SCDT_CLR_AVI_DIS
+			| BIT_TMDS_CSTAT_P3_CLR_AVI, ~0);
+	sii8620_write_seq_static(ctx,
+		REG_HPD_CTRL, BIT_HPD_CTRL_HPD_OUT_OVR_EN
+			| BIT_HPD_CTRL_HPD_HIGH,
+	);
+}
+
+static void sii8620_enable_gen2_write_burst(struct sii8620 *ctx)
+{
+	if (ctx->gen2_write_burst)
+		return;
+
+	sii8620_write_seq_static(ctx,
+		REG_MDT_RCV_TIMEOUT, 100,
+		REG_MDT_RCV_CTRL, BIT_MDT_RCV_CTRL_MDT_RCV_EN
+	);
+	ctx->gen2_write_burst = 1;
+}
+
+static void sii8620_disable_gen2_write_burst(struct sii8620 *ctx)
+{
+	if (!ctx->gen2_write_burst)
+		return;
+
+	sii8620_write_seq_static(ctx,
+		REG_MDT_XMIT_CTRL, 0,
+		REG_MDT_RCV_CTRL, 0
+	);
+	ctx->gen2_write_burst = 0;
+}
+
+static void sii8620_start_gen2_write_burst(struct sii8620 *ctx)
+{
+	sii8620_write_seq_static(ctx,
+		REG_MDT_INT_1_MASK, BIT_MDT_RCV_TIMEOUT
+			| BIT_MDT_RCV_SM_ABORT_PKT_RCVD | BIT_MDT_RCV_SM_ERROR
+			| BIT_MDT_XMIT_TIMEOUT | BIT_MDT_XMIT_SM_ABORT_PKT_RCVD
+			| BIT_MDT_XMIT_SM_ERROR,
+		REG_MDT_INT_0_MASK, BIT_MDT_XFIFO_EMPTY
+			| BIT_MDT_IDLE_AFTER_HAWB_DISABLE
+			| BIT_MDT_RFIFO_DATA_RDY
+	);
+	sii8620_enable_gen2_write_burst(ctx);
+}
+
+static void sii8620_mhl_discover(struct sii8620 *ctx)
+{
+	sii8620_write_seq_static(ctx,
+		REG_DISC_CTRL9, BIT_DISC_CTRL9_WAKE_DRVFLT
+			| BIT_DISC_CTRL9_DISC_PULSE_PROCEED,
+		REG_DISC_CTRL4, VAL_DISC_CTRL4(VAL_PUP_5K, VAL_PUP_20K),
+		REG_CBUS_DISC_INTR0_MASK, BIT_MHL3_EST_INT
+			| BIT_MHL_EST_INT
+			| BIT_NOT_MHL_EST_INT
+			| BIT_CBUS_MHL3_DISCON_INT
+			| BIT_CBUS_MHL12_DISCON_INT
+			| BIT_RGND_READY_INT,
+		REG_MHL_PLL_CTL0, VAL_MHL_PLL_CTL0_HDMI_CLK_RATIO_1X
+			| BIT_MHL_PLL_CTL0_CRYSTAL_CLK_SEL
+			| BIT_MHL_PLL_CTL0_ZONE_MASK_OE,
+		REG_MHL_DP_CTL0, BIT_MHL_DP_CTL0_DP_OE
+			| BIT_MHL_DP_CTL0_TX_OE_OVR,
+		REG_M3_CTRL, VAL_M3_CTRL_MHL3_VALUE,
+		REG_MHL_DP_CTL1, 0xA2,
+		REG_MHL_DP_CTL2, 0x03,
+		REG_MHL_DP_CTL3, 0x35,
+		REG_MHL_DP_CTL5, 0x02,
+		REG_MHL_DP_CTL6, 0x02,
+		REG_MHL_DP_CTL7, 0x03,
+		REG_COC_CTLC, 0xFF,
+		REG_DPD, BIT_DPD_PWRON_PLL | BIT_DPD_PDNTX12
+			| BIT_DPD_OSC_EN | BIT_DPD_PWRON_HSIC,
+		REG_COC_INTR_MASK, BIT_COC_PLL_LOCK_STATUS_CHANGE
+			| BIT_COC_CALIBRATION_DONE,
+		REG_CBUS_INT_1_MASK, BIT_CBUS_MSC_ABORT_RCVD
+			| BIT_CBUS_CMD_ABORT,
+		REG_CBUS_INT_0_MASK, BIT_CBUS_MSC_MT_DONE
+			| BIT_CBUS_HPD_CHG
+			| BIT_CBUS_MSC_MR_WRITE_STAT
+			| BIT_CBUS_MSC_MR_MSC_MSG
+			| BIT_CBUS_MSC_MR_WRITE_BURST
+			| BIT_CBUS_MSC_MR_SET_INT
+			| BIT_CBUS_MSC_MT_DONE_NACK
+	);
+}
+
+static void sii8620_peer_specific_init(struct sii8620 *ctx)
+{
+	if (ctx->mode == CM_MHL3)
+		sii8620_write_seq_static(ctx,
+			REG_SYS_CTRL1, BIT_SYS_CTRL1_BLOCK_DDC_BY_HPD,
+			REG_EMSCINTRMASK1,
+				BIT_EMSCINTR1_EMSC_TRAINING_COMMA_ERR
+		);
+	else
+		sii8620_write_seq_static(ctx,
+			REG_HDCP2X_INTR0_MASK, 0x00,
+			REG_EMSCINTRMASK1, 0x00,
+			REG_HDCP2X_INTR0, 0xFF,
+			REG_INTR1, 0xFF,
+			REG_SYS_CTRL1, BIT_SYS_CTRL1_BLOCK_DDC_BY_HPD
+				| BIT_SYS_CTRL1_TX_CTRL_HDMI
+		);
+}
+
+#define SII8620_MHL_VERSION			0x32
+#define SII8620_SCRATCHPAD_SIZE			16
+#define SII8620_INT_STAT_SIZE			0x33
+
+static void sii8620_set_dev_cap(struct sii8620 *ctx)
+{
+	static const u8 devcap[MHL_DCAP_SIZE] = {
+		[MHL_DCAP_MHL_VERSION] = SII8620_MHL_VERSION,
+		[MHL_DCAP_CAT] = MHL_DCAP_CAT_SOURCE | MHL_DCAP_CAT_POWER,
+		[MHL_DCAP_ADOPTER_ID_H] = 0x01,
+		[MHL_DCAP_ADOPTER_ID_L] = 0x41,
+		[MHL_DCAP_VID_LINK_MODE] = MHL_DCAP_VID_LINK_RGB444
+			| MHL_DCAP_VID_LINK_PPIXEL
+			| MHL_DCAP_VID_LINK_16BPP,
+		[MHL_DCAP_AUD_LINK_MODE] = MHL_DCAP_AUD_LINK_2CH,
+		[MHL_DCAP_VIDEO_TYPE] = MHL_DCAP_VT_GRAPHICS,
+		[MHL_DCAP_LOG_DEV_MAP] = MHL_DCAP_LD_GUI,
+		[MHL_DCAP_BANDWIDTH] = 0x0f,
+		[MHL_DCAP_FEATURE_FLAG] = MHL_DCAP_FEATURE_RCP_SUPPORT
+			| MHL_DCAP_FEATURE_RAP_SUPPORT
+			| MHL_DCAP_FEATURE_SP_SUPPORT,
+		[MHL_DCAP_SCRATCHPAD_SIZE] = SII8620_SCRATCHPAD_SIZE,
+		[MHL_DCAP_INT_STAT_SIZE] = SII8620_INT_STAT_SIZE,
+	};
+	static const u8 xdcap[MHL_XDC_SIZE] = {
+		[MHL_XDC_ECBUS_SPEEDS] = MHL_XDC_ECBUS_S_075
+			| MHL_XDC_ECBUS_S_8BIT,
+		[MHL_XDC_TMDS_SPEEDS] = MHL_XDC_TMDS_150
+			| MHL_XDC_TMDS_300 | MHL_XDC_TMDS_600,
+		[MHL_XDC_ECBUS_ROLES] = MHL_XDC_DEV_HOST,
+		[MHL_XDC_LOG_DEV_MAPX] = MHL_XDC_LD_PHONE,
+	};
+
+	sii8620_write_buf(ctx, REG_MHL_DEVCAP_0, devcap, ARRAY_SIZE(devcap));
+	sii8620_write_buf(ctx, REG_MHL_EXTDEVCAP_0, xdcap, ARRAY_SIZE(xdcap));
+}
+
+static void sii8620_mhl_init(struct sii8620 *ctx)
+{
+	sii8620_write_seq_static(ctx,
+		REG_DISC_CTRL4, VAL_DISC_CTRL4(VAL_PUP_OFF, VAL_PUP_20K),
+		REG_CBUS_MSC_COMPAT_CTRL,
+			BIT_CBUS_MSC_COMPAT_CTRL_XDEVCAP_EN,
+	);
+
+	sii8620_peer_specific_init(ctx);
+
+	sii8620_disable_hpd(ctx);
+
+	sii8620_write_seq_static(ctx,
+		REG_EDID_CTRL, BIT_EDID_CTRL_EDID_FIFO_ADDR_AUTO,
+		REG_DISC_CTRL9, BIT_DISC_CTRL9_WAKE_DRVFLT
+			| BIT_DISC_CTRL9_WAKE_PULSE_BYPASS,
+		REG_TMDS0_CCTRL1, 0x90,
+		REG_TMDS_CLK_EN, 0x01,
+		REG_TMDS_CH_EN, 0x11,
+		REG_BGR_BIAS, 0x87,
+		REG_ALICE0_ZONE_CTRL, 0xE8,
+		REG_ALICE0_MODE_CTRL, 0x04,
+	);
+	sii8620_setbits(ctx, REG_LM_DDC, BIT_LM_DDC_SW_TPI_EN_DISABLED, 0);
+	sii8620_write_seq_static(ctx,
+		REG_TPI_HW_OPT3, 0x76,
+		REG_TMDS_CCTRL, BIT_TMDS_CCTRL_TMDS_OE,
+		REG_TPI_DTD_B2, 79,
+	);
+	sii8620_set_dev_cap(ctx);
+	sii8620_write_seq_static(ctx,
+		REG_MDT_XMIT_TIMEOUT, 100,
+		REG_MDT_XMIT_CTRL, 0x03,
+		REG_MDT_XFIFO_STAT, 0x00,
+		REG_MDT_RCV_TIMEOUT, 100,
+		REG_CBUS_LINK_CTRL_8, 0x1D,
+	);
+
+	sii8620_start_gen2_write_burst(ctx);
+	sii8620_write_seq_static(ctx,
+		REG_BIST_CTRL, 0x00,
+		REG_COC_CTL1, 0x10,
+		REG_COC_CTL2, 0x18,
+		REG_COC_CTLF, 0x07,
+		REG_COC_CTL11, 0xF8,
+		REG_COC_CTL17, 0x61,
+		REG_COC_CTL18, 0x46,
+		REG_COC_CTL19, 0x15,
+		REG_COC_CTL1A, 0x01,
+		REG_MHL_COC_CTL3, BIT_MHL_COC_CTL3_COC_AECHO_EN,
+		REG_MHL_COC_CTL4, 0x2D,
+		REG_MHL_COC_CTL5, 0xF9,
+		REG_MSC_HEARTBEAT_CTRL, 0x27,
+	);
+	sii8620_disable_gen2_write_burst(ctx);
+
+	/* currently MHL3 is not supported, so we force version to 0 */
+	sii8620_mt_write_stat(ctx, MHL_DST_REG(VERSION), 0);
+	sii8620_mt_write_stat(ctx, MHL_DST_REG(CONNECTED_RDY),
+			      MHL_DST_CONN_DCAP_RDY | MHL_DST_CONN_XDEVCAPP_SUPP
+			      | MHL_DST_CONN_POW_STAT);
+	sii8620_mt_set_int(ctx, MHL_INT_REG(RCHANGE), MHL_INT_RC_DCAP_CHG);
+}
+
+static void sii8620_set_mode(struct sii8620 *ctx, enum sii8620_mode mode)
+{
+	if (ctx->mode == mode)
+		return;
+
+	ctx->mode = mode;
+
+	switch (mode) {
+	case CM_MHL1:
+		sii8620_write_seq_static(ctx,
+			REG_CBUS_MSC_COMPAT_CTRL, 0x02,
+			REG_M3_CTRL, VAL_M3_CTRL_MHL1_2_VALUE,
+			REG_DPD, BIT_DPD_PWRON_PLL | BIT_DPD_PDNTX12
+				| BIT_DPD_OSC_EN,
+			REG_COC_INTR_MASK, 0
+		);
+		break;
+	case CM_MHL3:
+		sii8620_write_seq_static(ctx,
+			REG_M3_CTRL, VAL_M3_CTRL_MHL3_VALUE,
+			REG_COC_CTL0, 0x40,
+			REG_MHL_COC_CTL1, 0x07
+		);
+		break;
+	case CM_DISCONNECTED:
+		break;
+	default:
+		dev_err(ctx->dev, "%s mode %d not supported\n", __func__, mode);
+		break;
+	}
+
+	sii8620_set_auto_zone(ctx);
+
+	if (mode != CM_MHL1)
+		return;
+
+	sii8620_write_seq_static(ctx,
+		REG_MHL_DP_CTL0, 0xBC,
+		REG_MHL_DP_CTL1, 0xBB,
+		REG_MHL_DP_CTL3, 0x48,
+		REG_MHL_DP_CTL5, 0x39,
+		REG_MHL_DP_CTL2, 0x2A,
+		REG_MHL_DP_CTL6, 0x2A,
+		REG_MHL_DP_CTL7, 0x08
+	);
+}
+
+static void sii8620_disconnect(struct sii8620 *ctx)
+{
+	sii8620_disable_gen2_write_burst(ctx);
+	sii8620_stop_video(ctx);
+	msleep(50);
+	sii8620_cbus_reset(ctx);
+	sii8620_set_mode(ctx, CM_DISCONNECTED);
+	sii8620_write_seq_static(ctx,
+		REG_COC_CTL0, 0x40,
+		REG_CBUS3_CNVT, 0x84,
+		REG_COC_CTL14, 0x00,
+		REG_COC_CTL0, 0x40,
+		REG_HRXCTRL3, 0x07,
+		REG_MHL_PLL_CTL0, VAL_MHL_PLL_CTL0_HDMI_CLK_RATIO_1X
+			| BIT_MHL_PLL_CTL0_CRYSTAL_CLK_SEL
+			| BIT_MHL_PLL_CTL0_ZONE_MASK_OE,
+		REG_MHL_DP_CTL0, BIT_MHL_DP_CTL0_DP_OE
+			| BIT_MHL_DP_CTL0_TX_OE_OVR,
+		REG_MHL_DP_CTL1, 0xBB,
+		REG_MHL_DP_CTL3, 0x48,
+		REG_MHL_DP_CTL5, 0x3F,
+		REG_MHL_DP_CTL2, 0x2F,
+		REG_MHL_DP_CTL6, 0x2A,
+		REG_MHL_DP_CTL7, 0x03
+	);
+	sii8620_disable_hpd(ctx);
+	sii8620_write_seq_static(ctx,
+		REG_M3_CTRL, VAL_M3_CTRL_MHL3_VALUE,
+		REG_MHL_COC_CTL1, 0x07,
+		REG_DISC_CTRL4, VAL_DISC_CTRL4(VAL_PUP_OFF, VAL_PUP_20K),
+		REG_DISC_CTRL8, 0x00,
+		REG_DISC_CTRL9, BIT_DISC_CTRL9_WAKE_DRVFLT
+			| BIT_DISC_CTRL9_WAKE_PULSE_BYPASS,
+		REG_INT_CTRL, 0x00,
+		REG_MSC_HEARTBEAT_CTRL, 0x27,
+		REG_DISC_CTRL1, 0x25,
+		REG_CBUS_DISC_INTR0, (u8)~BIT_RGND_READY_INT,
+		REG_CBUS_DISC_INTR0_MASK, BIT_RGND_READY_INT,
+		REG_MDT_INT_1, 0xff,
+		REG_MDT_INT_1_MASK, 0x00,
+		REG_MDT_INT_0, 0xff,
+		REG_MDT_INT_0_MASK, 0x00,
+		REG_COC_INTR, 0xff,
+		REG_COC_INTR_MASK, 0x00,
+		REG_TRXINTH, 0xff,
+		REG_TRXINTMH, 0x00,
+		REG_CBUS_INT_0, 0xff,
+		REG_CBUS_INT_0_MASK, 0x00,
+		REG_CBUS_INT_1, 0xff,
+		REG_CBUS_INT_1_MASK, 0x00,
+		REG_EMSCINTR, 0xff,
+		REG_EMSCINTRMASK, 0x00,
+		REG_EMSCINTR1, 0xff,
+		REG_EMSCINTRMASK1, 0x00,
+		REG_INTR8, 0xff,
+		REG_INTR8_MASK, 0x00,
+		REG_TPI_INTR_ST0, 0xff,
+		REG_TPI_INTR_EN, 0x00,
+		REG_HDCP2X_INTR0, 0xff,
+		REG_HDCP2X_INTR0_MASK, 0x00,
+		REG_INTR9, 0xff,
+		REG_INTR9_MASK, 0x00,
+		REG_INTR3, 0xff,
+		REG_INTR3_MASK, 0x00,
+		REG_INTR5, 0xff,
+		REG_INTR5_MASK, 0x00,
+		REG_INTR2, 0xff,
+		REG_INTR2_MASK, 0x00,
+	);
+	memset(ctx->stat, 0, sizeof(ctx->stat));
+	memset(ctx->xstat, 0, sizeof(ctx->xstat));
+	memset(ctx->devcap, 0, sizeof(ctx->devcap));
+	memset(ctx->xdevcap, 0, sizeof(ctx->xdevcap));
+	ctx->cbus_status = 0;
+	ctx->sink_type = SINK_NONE;
+	kfree(ctx->edid);
+	ctx->edid = NULL;
+	sii8620_mt_cleanup(ctx);
+}
+
+static void sii8620_mhl_disconnected(struct sii8620 *ctx)
+{
+	sii8620_write_seq_static(ctx,
+		REG_DISC_CTRL4, VAL_DISC_CTRL4(VAL_PUP_OFF, VAL_PUP_20K),
+		REG_CBUS_MSC_COMPAT_CTRL,
+			BIT_CBUS_MSC_COMPAT_CTRL_XDEVCAP_EN
+	);
+	sii8620_disconnect(ctx);
+}
+
+static void sii8620_irq_disc(struct sii8620 *ctx)
+{
+	u8 stat = sii8620_readb(ctx, REG_CBUS_DISC_INTR0);
+
+	if (stat & VAL_CBUS_MHL_DISCON)
+		sii8620_mhl_disconnected(ctx);
+
+	if (stat & BIT_RGND_READY_INT) {
+		u8 stat2 = sii8620_readb(ctx, REG_DISC_STAT2);
+
+		if ((stat2 & MSK_DISC_STAT2_RGND) == VAL_RGND_1K) {
+			sii8620_mhl_discover(ctx);
+		} else {
+			sii8620_write_seq_static(ctx,
+				REG_DISC_CTRL9, BIT_DISC_CTRL9_WAKE_DRVFLT
+					| BIT_DISC_CTRL9_NOMHL_EST
+					| BIT_DISC_CTRL9_WAKE_PULSE_BYPASS,
+				REG_CBUS_DISC_INTR0_MASK, BIT_RGND_READY_INT
+					| BIT_CBUS_MHL3_DISCON_INT
+					| BIT_CBUS_MHL12_DISCON_INT
+					| BIT_NOT_MHL_EST_INT
+			);
+		}
+	}
+	if (stat & BIT_MHL_EST_INT)
+		sii8620_mhl_init(ctx);
+
+	sii8620_write(ctx, REG_CBUS_DISC_INTR0, stat);
+}
+
+static void sii8620_irq_g2wb(struct sii8620 *ctx)
+{
+	u8 stat = sii8620_readb(ctx, REG_MDT_INT_0);
+
+	if (stat & BIT_MDT_IDLE_AFTER_HAWB_DISABLE)
+		dev_dbg(ctx->dev, "HAWB idle\n");
+
+	sii8620_write(ctx, REG_MDT_INT_0, stat);
+}
+
+static void sii8620_status_changed_dcap(struct sii8620 *ctx)
+{
+	if (ctx->stat[MHL_DST_CONNECTED_RDY] & MHL_DST_CONN_DCAP_RDY) {
+		sii8620_set_mode(ctx, CM_MHL1);
+		sii8620_peer_specific_init(ctx);
+		sii8620_write(ctx, REG_INTR9_MASK, BIT_INTR9_DEVCAP_DONE
+			       | BIT_INTR9_EDID_DONE | BIT_INTR9_EDID_ERROR);
+	}
+}
+
+static void sii8620_status_changed_path(struct sii8620 *ctx)
+{
+	if (ctx->stat[MHL_DST_LINK_MODE] & MHL_DST_LM_PATH_ENABLED) {
+		sii8620_mt_write_stat(ctx, MHL_DST_REG(LINK_MODE),
+				      MHL_DST_LM_CLK_MODE_NORMAL
+				      | MHL_DST_LM_PATH_ENABLED);
+		sii8620_mt_read_devcap(ctx, false);
+	} else {
+		sii8620_mt_write_stat(ctx, MHL_DST_REG(LINK_MODE),
+				      MHL_DST_LM_CLK_MODE_NORMAL);
+	}
+}
+
+static void sii8620_msc_mr_write_stat(struct sii8620 *ctx)
+{
+	u8 st[MHL_DST_SIZE], xst[MHL_XDS_SIZE];
+
+	sii8620_read_buf(ctx, REG_MHL_STAT_0, st, MHL_DST_SIZE);
+	sii8620_read_buf(ctx, REG_MHL_EXTSTAT_0, xst, MHL_XDS_SIZE);
+
+	sii8620_update_array(ctx->stat, st, MHL_DST_SIZE);
+	sii8620_update_array(ctx->xstat, xst, MHL_XDS_SIZE);
+
+	if (st[MHL_DST_CONNECTED_RDY] & MHL_DST_CONN_DCAP_RDY)
+		sii8620_status_changed_dcap(ctx);
+
+	if (st[MHL_DST_LINK_MODE] & MHL_DST_LM_PATH_ENABLED)
+		sii8620_status_changed_path(ctx);
+}
+
+static void sii8620_msc_mr_set_int(struct sii8620 *ctx)
+{
+	u8 ints[MHL_INT_SIZE];
+
+	sii8620_read_buf(ctx, REG_MHL_INT_0, ints, MHL_INT_SIZE);
+	sii8620_write_buf(ctx, REG_MHL_INT_0, ints, MHL_INT_SIZE);
+}
+
+static struct sii8620_mt_msg *sii8620_msc_msg_first(struct sii8620 *ctx)
+{
+	struct device *dev = ctx->dev;
+
+	if (list_empty(&ctx->mt_queue)) {
+		dev_err(dev, "unexpected MSC MT response\n");
+		return NULL;
+	}
+
+	return list_first_entry(&ctx->mt_queue, struct sii8620_mt_msg, node);
+}
+
+static void sii8620_msc_mt_done(struct sii8620 *ctx)
+{
+	struct sii8620_mt_msg *msg = sii8620_msc_msg_first(ctx);
+
+	if (!msg)
+		return;
+
+	msg->ret = sii8620_readb(ctx, REG_MSC_MT_RCVD_DATA0);
+	ctx->mt_state = MT_STATE_DONE;
+}
+
+static void sii8620_msc_mr_msc_msg(struct sii8620 *ctx)
+{
+	struct sii8620_mt_msg *msg = sii8620_msc_msg_first(ctx);
+	u8 buf[2];
+
+	if (!msg)
+		return;
+
+	sii8620_read_buf(ctx, REG_MSC_MR_MSC_MSG_RCVD_1ST_DATA, buf, 2);
+
+	switch (buf[0]) {
+	case MHL_MSC_MSG_RAPK:
+		msg->ret = buf[1];
+		ctx->mt_state = MT_STATE_DONE;
+		break;
+	default:
+		dev_err(ctx->dev, "%s message type %d,%d not supported",
+			__func__, buf[0], buf[1]);
+	}
+}
+
+static void sii8620_irq_msc(struct sii8620 *ctx)
+{
+	u8 stat = sii8620_readb(ctx, REG_CBUS_INT_0);
+
+	if (stat & ~BIT_CBUS_HPD_CHG)
+		sii8620_write(ctx, REG_CBUS_INT_0, stat & ~BIT_CBUS_HPD_CHG);
+
+	if (stat & BIT_CBUS_HPD_CHG) {
+		u8 cbus_stat = sii8620_readb(ctx, REG_CBUS_STATUS);
+
+		if ((cbus_stat ^ ctx->cbus_status) & BIT_CBUS_STATUS_CBUS_HPD) {
+			sii8620_write(ctx, REG_CBUS_INT_0, BIT_CBUS_HPD_CHG);
+		} else {
+			stat ^= BIT_CBUS_STATUS_CBUS_HPD;
+			cbus_stat ^= BIT_CBUS_STATUS_CBUS_HPD;
+		}
+		ctx->cbus_status = cbus_stat;
+	}
+
+	if (stat & BIT_CBUS_MSC_MR_WRITE_STAT)
+		sii8620_msc_mr_write_stat(ctx);
+
+	if (stat & BIT_CBUS_MSC_MR_SET_INT)
+		sii8620_msc_mr_set_int(ctx);
+
+	if (stat & BIT_CBUS_MSC_MT_DONE)
+		sii8620_msc_mt_done(ctx);
+
+	if (stat & BIT_CBUS_MSC_MR_MSC_MSG)
+		sii8620_msc_mr_msc_msg(ctx);
+}
+
+static void sii8620_irq_coc(struct sii8620 *ctx)
+{
+	u8 stat = sii8620_readb(ctx, REG_COC_INTR);
+
+	sii8620_write(ctx, REG_COC_INTR, stat);
+}
+
+static void sii8620_irq_merr(struct sii8620 *ctx)
+{
+	u8 stat = sii8620_readb(ctx, REG_CBUS_INT_1);
+
+	sii8620_write(ctx, REG_CBUS_INT_1, stat);
+}
+
+static void sii8620_irq_edid(struct sii8620 *ctx)
+{
+	u8 stat = sii8620_readb(ctx, REG_INTR9);
+
+	sii8620_write(ctx, REG_INTR9, stat);
+
+	if (stat & BIT_INTR9_DEVCAP_DONE)
+		ctx->mt_state = MT_STATE_DONE;
+}
+
+static void sii8620_scdt_high(struct sii8620 *ctx)
+{
+	sii8620_write_seq_static(ctx,
+		REG_INTR8_MASK, BIT_CEA_NEW_AVI | BIT_CEA_NEW_VSI,
+		REG_TPI_SC, BIT_TPI_SC_TPI_OUTPUT_MODE_0_HDMI,
+	);
+}
+
+static void sii8620_scdt_low(struct sii8620 *ctx)
+{
+	sii8620_write(ctx, REG_TMDS_CSTAT_P3,
+		      BIT_TMDS_CSTAT_P3_SCDT_CLR_AVI_DIS |
+		      BIT_TMDS_CSTAT_P3_CLR_AVI);
+
+	sii8620_stop_video(ctx);
+
+	sii8620_write(ctx, REG_INTR8_MASK, 0);
+}
+
+static void sii8620_irq_scdt(struct sii8620 *ctx)
+{
+	u8 stat = sii8620_readb(ctx, REG_INTR5);
+
+	if (stat & BIT_INTR_SCDT_CHANGE) {
+		u8 cstat = sii8620_readb(ctx, REG_TMDS_CSTAT_P3);
+
+		if (cstat & BIT_TMDS_CSTAT_P3_SCDT)
+			sii8620_scdt_high(ctx);
+		else
+			sii8620_scdt_low(ctx);
+	}
+
+	sii8620_write(ctx, REG_INTR5, stat);
+}
+
+static void sii8620_new_vsi(struct sii8620 *ctx)
+{
+	u8 vsif[11];
+
+	sii8620_write(ctx, REG_RX_HDMI_CTRL2,
+		      VAL_RX_HDMI_CTRL2_DEFVAL |
+		      BIT_RX_HDMI_CTRL2_VSI_MON_SEL_VSI);
+	sii8620_read_buf(ctx, REG_RX_HDMI_MON_PKT_HEADER1, vsif,
+			 ARRAY_SIZE(vsif));
+}
+
+static void sii8620_new_avi(struct sii8620 *ctx)
+{
+	sii8620_write(ctx, REG_RX_HDMI_CTRL2, VAL_RX_HDMI_CTRL2_DEFVAL);
+	sii8620_read_buf(ctx, REG_RX_HDMI_MON_PKT_HEADER1, ctx->avif,
+			 ARRAY_SIZE(ctx->avif));
+}
+
+static void sii8620_irq_infr(struct sii8620 *ctx)
+{
+	u8 stat = sii8620_readb(ctx, REG_INTR8)
+		& (BIT_CEA_NEW_VSI | BIT_CEA_NEW_AVI);
+
+	sii8620_write(ctx, REG_INTR8, stat);
+
+	if (stat & BIT_CEA_NEW_VSI)
+		sii8620_new_vsi(ctx);
+
+	if (stat & BIT_CEA_NEW_AVI)
+		sii8620_new_avi(ctx);
+
+	if (stat & (BIT_CEA_NEW_VSI | BIT_CEA_NEW_AVI))
+		sii8620_start_video(ctx);
+}
+
+/* endian agnostic, non-volatile version of test_bit */
+static bool sii8620_test_bit(unsigned int nr, const u8 *addr)
+{
+	return 1 & (addr[nr / BITS_PER_BYTE] >> (nr % BITS_PER_BYTE));
+}
+
+static irqreturn_t sii8620_irq_thread(int irq, void *data)
+{
+	static const struct {
+		int bit;
+		void (*handler)(struct sii8620 *ctx);
+	} irq_vec[] = {
+		{ BIT_FAST_INTR_STAT_DISC, sii8620_irq_disc },
+		{ BIT_FAST_INTR_STAT_G2WB, sii8620_irq_g2wb },
+		{ BIT_FAST_INTR_STAT_COC, sii8620_irq_coc },
+		{ BIT_FAST_INTR_STAT_MSC, sii8620_irq_msc },
+		{ BIT_FAST_INTR_STAT_MERR, sii8620_irq_merr },
+		{ BIT_FAST_INTR_STAT_EDID, sii8620_irq_edid },
+		{ BIT_FAST_INTR_STAT_SCDT, sii8620_irq_scdt },
+		{ BIT_FAST_INTR_STAT_INFR, sii8620_irq_infr },
+	};
+	struct sii8620 *ctx = data;
+	u8 stats[LEN_FAST_INTR_STAT];
+	int i, ret;
+
+	mutex_lock(&ctx->lock);
+
+	sii8620_read_buf(ctx, REG_FAST_INTR_STAT, stats, ARRAY_SIZE(stats));
+	for (i = 0; i < ARRAY_SIZE(irq_vec); ++i)
+		if (sii8620_test_bit(irq_vec[i].bit, stats))
+			irq_vec[i].handler(ctx);
+
+	sii8620_mt_work(ctx);
+
+	ret = sii8620_clear_error(ctx);
+	if (ret) {
+		dev_err(ctx->dev, "Error during IRQ handling, %d.\n", ret);
+		sii8620_mhl_disconnected(ctx);
+	}
+	mutex_unlock(&ctx->lock);
+
+	return IRQ_HANDLED;
+}
+
+static void sii8620_cable_in(struct sii8620 *ctx)
+{
+	struct device *dev = ctx->dev;
+	u8 ver[5];
+	int ret;
+
+	ret = sii8620_hw_on(ctx);
+	if (ret) {
+		dev_err(dev, "Error powering on, %d.\n", ret);
+		return;
+	}
+	sii8620_hw_reset(ctx);
+
+	sii8620_read_buf(ctx, REG_VND_IDL, ver, ARRAY_SIZE(ver));
+	ret = sii8620_clear_error(ctx);
+	if (ret) {
+		dev_err(dev, "Error accessing I2C bus, %d.\n", ret);
+		return;
+	}
+
+	dev_info(dev, "ChipID %02x%02x:%02x%02x rev %02x.\n", ver[1], ver[0],
+		 ver[3], ver[2], ver[4]);
+
+	sii8620_write(ctx, REG_DPD,
+		      BIT_DPD_PWRON_PLL | BIT_DPD_PDNTX12 | BIT_DPD_OSC_EN);
+
+	sii8620_xtal_set_rate(ctx);
+	sii8620_disconnect(ctx);
+
+	sii8620_write_seq_static(ctx,
+		REG_MHL_CBUS_CTL0, VAL_MHL_CBUS_CTL0_CBUS_DRV_SEL_STRONG
+			| VAL_MHL_CBUS_CTL0_CBUS_RGND_VBIAS_734,
+		REG_MHL_CBUS_CTL1, VAL_MHL_CBUS_CTL1_1115_OHM,
+		REG_DPD, BIT_DPD_PWRON_PLL | BIT_DPD_PDNTX12 | BIT_DPD_OSC_EN,
+	);
+
+	ret = sii8620_clear_error(ctx);
+	if (ret) {
+		dev_err(dev, "Error accessing I2C bus, %d.\n", ret);
+		return;
+	}
+
+	enable_irq(to_i2c_client(ctx->dev)->irq);
+}
+
+static inline struct sii8620 *bridge_to_sii8620(struct drm_bridge *bridge)
+{
+	return container_of(bridge, struct sii8620, bridge);
+}
+
+static bool sii8620_mode_fixup(struct drm_bridge *bridge,
+			       const struct drm_display_mode *mode,
+			       struct drm_display_mode *adjusted_mode)
+{
+	struct sii8620 *ctx = bridge_to_sii8620(bridge);
+	bool ret = false;
+	int max_clock = 74250;
+
+	mutex_lock(&ctx->lock);
+
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+		goto out;
+
+	if (ctx->devcap[MHL_DCAP_VID_LINK_MODE] & MHL_DCAP_VID_LINK_PPIXEL)
+		max_clock = 300000;
+
+	ret = mode->clock <= max_clock;
+
+out:
+	mutex_unlock(&ctx->lock);
+
+	return ret;
+}
+
+static const struct drm_bridge_funcs sii8620_bridge_funcs = {
+	.mode_fixup = sii8620_mode_fixup,
+};
+
+static int sii8620_probe(struct i2c_client *client,
+			 const struct i2c_device_id *id)
+{
+	struct device *dev = &client->dev;
+	struct sii8620 *ctx;
+	int ret;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->dev = dev;
+	mutex_init(&ctx->lock);
+	INIT_LIST_HEAD(&ctx->mt_queue);
+
+	ctx->clk_xtal = devm_clk_get(dev, "xtal");
+	if (IS_ERR(ctx->clk_xtal)) {
+		dev_err(dev, "failed to get xtal clock from DT\n");
+		return PTR_ERR(ctx->clk_xtal);
+	}
+
+	if (!client->irq) {
+		dev_err(dev, "no irq provided\n");
+		return -EINVAL;
+	}
+	irq_set_status_flags(client->irq, IRQ_NOAUTOEN);
+	ret = devm_request_threaded_irq(dev, client->irq, NULL,
+					sii8620_irq_thread,
+					IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
+					"sii8620", ctx);
+
+	ctx->gpio_reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(ctx->gpio_reset)) {
+		dev_err(dev, "failed to get reset gpio from DT\n");
+		return PTR_ERR(ctx->gpio_reset);
+	}
+
+	ctx->supplies[0].supply = "cvcc10";
+	ctx->supplies[1].supply = "iovcc18";
+	ret = devm_regulator_bulk_get(dev, 2, ctx->supplies);
+	if (ret)
+		return ret;
+
+	i2c_set_clientdata(client, ctx);
+
+	ctx->bridge.funcs = &sii8620_bridge_funcs;
+	ctx->bridge.of_node = dev->of_node;
+	drm_bridge_add(&ctx->bridge);
+
+	sii8620_cable_in(ctx);
+
+	return 0;
+}
+
+static int sii8620_remove(struct i2c_client *client)
+{
+	struct sii8620 *ctx = i2c_get_clientdata(client);
+
+	disable_irq(to_i2c_client(ctx->dev)->irq);
+	drm_bridge_remove(&ctx->bridge);
+	sii8620_hw_off(ctx);
+
+	return 0;
+}
+
+static const struct of_device_id sii8620_dt_match[] = {
+	{ .compatible = "sil,sii8620" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, sii8620_dt_match);
+
+static const struct i2c_device_id sii8620_id[] = {
+	{ "sii8620", 0 },
+	{ },
+};
+
+MODULE_DEVICE_TABLE(i2c, sii8620_id);
+static struct i2c_driver sii8620_driver = {
+	.driver = {
+		.name	= "sii8620",
+		.of_match_table = of_match_ptr(sii8620_dt_match),
+	},
+	.probe		= sii8620_probe,
+	.remove		= sii8620_remove,
+	.id_table = sii8620_id,
+};
+
+module_i2c_driver(sii8620_driver);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/bridge/sil-sii8620.h b/drivers/gpu/drm/bridge/sil-sii8620.h
new file mode 100644
index 000000000000..6ff616a4f6ce
--- /dev/null
+++ b/drivers/gpu/drm/bridge/sil-sii8620.h
@@ -0,0 +1,1517 @@
+/*
+ * Registers of Silicon Image SiI8620 Mobile HD Transmitter
+ *
+ * Copyright (C) 2015, Samsung Electronics Co., Ltd.
+ * Andrzej Hajda <a.hajda@samsung.com>
+ *
+ * Based on MHL driver for Android devices.
+ * Copyright (C) 2013-2014 Silicon Image, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __SIL_SII8620_H__
+#define __SIL_SII8620_H__
+
+/* Vendor ID Low byte, default value: 0x01 */
+#define REG_VND_IDL				0x0000
+
+/* Vendor ID High byte, default value: 0x00 */
+#define REG_VND_IDH				0x0001
+
+/* Device ID Low byte, default value: 0x60 */
+#define REG_DEV_IDL				0x0002
+
+/* Device ID High byte, default value: 0x86 */
+#define REG_DEV_IDH				0x0003
+
+/* Device Revision, default value: 0x10 */
+#define REG_DEV_REV				0x0004
+
+/* OTP DBYTE510, default value: 0x00 */
+#define REG_OTP_DBYTE510			0x0006
+
+/* System Control #1, default value: 0x00 */
+#define REG_SYS_CTRL1				0x0008
+#define BIT_SYS_CTRL1_OTPVMUTEOVR_SET		BIT(7)
+#define BIT_SYS_CTRL1_VSYNCPIN			BIT(6)
+#define BIT_SYS_CTRL1_OTPADROPOVR_SET		BIT(5)
+#define BIT_SYS_CTRL1_BLOCK_DDC_BY_HPD		BIT(4)
+#define BIT_SYS_CTRL1_OTP2XVOVR_EN		BIT(3)
+#define BIT_SYS_CTRL1_OTP2XAOVR_EN		BIT(2)
+#define BIT_SYS_CTRL1_TX_CTRL_HDMI		BIT(1)
+#define BIT_SYS_CTRL1_OTPAMUTEOVR_SET		BIT(0)
+
+/* System Control DPD, default value: 0x90 */
+#define REG_DPD					0x000b
+#define BIT_DPD_PWRON_PLL			BIT(7)
+#define BIT_DPD_PDNTX12				BIT(6)
+#define BIT_DPD_PDNRX12				BIT(5)
+#define BIT_DPD_OSC_EN				BIT(4)
+#define BIT_DPD_PWRON_HSIC			BIT(3)
+#define BIT_DPD_PDIDCK_N			BIT(2)
+#define BIT_DPD_PD_MHL_CLK_N			BIT(1)
+
+/* Dual link Control, default value: 0x00 */
+#define REG_DCTL				0x000d
+#define BIT_DCTL_TDM_LCLK_PHASE			BIT(7)
+#define BIT_DCTL_HSIC_CLK_PHASE			BIT(6)
+#define BIT_DCTL_CTS_TCK_PHASE			BIT(5)
+#define BIT_DCTL_EXT_DDC_SEL			BIT(4)
+#define BIT_DCTL_TRANSCODE			BIT(3)
+#define BIT_DCTL_HSIC_RX_STROBE_PHASE		BIT(2)
+#define BIT_DCTL_HSIC_TX_BIST_START_SEL		BIT(1)
+#define BIT_DCTL_TCLKNX_PHASE			BIT(0)
+
+/* PWD Software Reset, default value: 0x20 */
+#define REG_PWD_SRST				0x000e
+#define BIT_PWD_SRST_COC_DOC_RST		BIT(7)
+#define BIT_PWD_SRST_CBUS_RST_SW		BIT(6)
+#define BIT_PWD_SRST_CBUS_RST_SW_EN		BIT(5)
+#define BIT_PWD_SRST_MHLFIFO_RST		BIT(4)
+#define BIT_PWD_SRST_CBUS_RST			BIT(3)
+#define BIT_PWD_SRST_SW_RST_AUTO		BIT(2)
+#define BIT_PWD_SRST_HDCP2X_SW_RST		BIT(1)
+#define BIT_PWD_SRST_SW_RST			BIT(0)
+
+/* AKSV_1, default value: 0x00 */
+#define REG_AKSV_1				0x001d
+
+/* Video H Resolution #1, default value: 0x00 */
+#define REG_H_RESL				0x003a
+
+/* Video Mode, default value: 0x00 */
+#define REG_VID_MODE				0x004a
+#define BIT_VID_MODE_M1080P			BIT(6)
+
+/* Video Input Mode, default value: 0xc0 */
+#define REG_VID_OVRRD				0x0051
+#define BIT_VID_OVRRD_PP_AUTO_DISABLE		BIT(7)
+#define BIT_VID_OVRRD_M1080P_OVRRD		BIT(6)
+#define BIT_VID_OVRRD_MINIVSYNC_ON		BIT(5)
+#define BIT_VID_OVRRD_3DCONV_EN_FRAME_PACK	BIT(4)
+#define BIT_VID_OVRRD_ENABLE_AUTO_PATH_EN	BIT(3)
+#define BIT_VID_OVRRD_ENRGB2YCBCR_OVRRD		BIT(2)
+#define BIT_VID_OVRRD_ENDOWNSAMPLE_OVRRD	BIT(0)
+
+/* I2C Address reassignment, default value: 0x00 */
+#define REG_PAGE_MHLSPEC_ADDR			0x0057
+#define REG_PAGE7_ADDR				0x0058
+#define REG_PAGE8_ADDR				0x005c
+
+/* Fast Interrupt Status, default value: 0x00 */
+#define REG_FAST_INTR_STAT			0x005f
+#define LEN_FAST_INTR_STAT			7
+#define BIT_FAST_INTR_STAT_TIMR			8
+#define BIT_FAST_INTR_STAT_INT2			9
+#define BIT_FAST_INTR_STAT_DDC			10
+#define BIT_FAST_INTR_STAT_SCDT			11
+#define BIT_FAST_INTR_STAT_INFR			13
+#define BIT_FAST_INTR_STAT_EDID			14
+#define BIT_FAST_INTR_STAT_HDCP			15
+#define BIT_FAST_INTR_STAT_MSC			16
+#define BIT_FAST_INTR_STAT_MERR			17
+#define BIT_FAST_INTR_STAT_G2WB			18
+#define BIT_FAST_INTR_STAT_G2WB_ERR		19
+#define BIT_FAST_INTR_STAT_DISC			28
+#define BIT_FAST_INTR_STAT_BLOCK		30
+#define BIT_FAST_INTR_STAT_LTRN			31
+#define BIT_FAST_INTR_STAT_HDCP2		32
+#define BIT_FAST_INTR_STAT_TDM			42
+#define BIT_FAST_INTR_STAT_COC			51
+
+/* GPIO Control, default value: 0x15 */
+#define REG_GPIO_CTRL1				0x006e
+#define BIT_CTRL1_GPIO_I_8			BIT(5)
+#define BIT_CTRL1_GPIO_OEN_8			BIT(4)
+#define BIT_CTRL1_GPIO_I_7			BIT(3)
+#define BIT_CTRL1_GPIO_OEN_7			BIT(2)
+#define BIT_CTRL1_GPIO_I_6			BIT(1)
+#define BIT_CTRL1_GPIO_OEN_6			BIT(0)
+
+/* Interrupt Control, default value: 0x06 */
+#define REG_INT_CTRL				0x006f
+#define BIT_INT_CTRL_SOFTWARE_WP		BIT(7)
+#define BIT_INT_CTRL_INTR_OD			BIT(2)
+#define BIT_INT_CTRL_INTR_POLARITY		BIT(1)
+
+/* Interrupt State, default value: 0x00 */
+#define REG_INTR_STATE				0x0070
+#define BIT_INTR_STATE_INTR_STATE		BIT(0)
+
+/* Interrupt Source #1, default value: 0x00 */
+#define REG_INTR1				0x0071
+
+/* Interrupt Source #2, default value: 0x00 */
+#define REG_INTR2				0x0072
+
+/* Interrupt Source #3, default value: 0x01 */
+#define REG_INTR3				0x0073
+#define BIT_DDC_CMD_DONE			BIT(3)
+
+/* Interrupt Source #5, default value: 0x00 */
+#define REG_INTR5				0x0074
+
+/* Interrupt #1 Mask, default value: 0x00 */
+#define REG_INTR1_MASK				0x0075
+
+/* Interrupt #2 Mask, default value: 0x00 */
+#define REG_INTR2_MASK				0x0076
+
+/* Interrupt #3 Mask, default value: 0x00 */
+#define REG_INTR3_MASK				0x0077
+
+/* Interrupt #5 Mask, default value: 0x00 */
+#define REG_INTR5_MASK				0x0078
+#define BIT_INTR_SCDT_CHANGE			BIT(0)
+
+/* Hot Plug Connection Control, default value: 0x45 */
+#define REG_HPD_CTRL				0x0079
+#define BIT_HPD_CTRL_HPD_DS_SIGNAL		BIT(7)
+#define BIT_HPD_CTRL_HPD_OUT_OD_EN		BIT(6)
+#define BIT_HPD_CTRL_HPD_HIGH			BIT(5)
+#define BIT_HPD_CTRL_HPD_OUT_OVR_EN		BIT(4)
+#define BIT_HPD_CTRL_GPIO_I_1			BIT(3)
+#define BIT_HPD_CTRL_GPIO_OEN_1			BIT(2)
+#define BIT_HPD_CTRL_GPIO_I_0			BIT(1)
+#define BIT_HPD_CTRL_GPIO_OEN_0			BIT(0)
+
+/* GPIO Control, default value: 0x55 */
+#define REG_GPIO_CTRL				0x007a
+#define BIT_CTRL_GPIO_I_5			BIT(7)
+#define BIT_CTRL_GPIO_OEN_5			BIT(6)
+#define BIT_CTRL_GPIO_I_4			BIT(5)
+#define BIT_CTRL_GPIO_OEN_4			BIT(4)
+#define BIT_CTRL_GPIO_I_3			BIT(3)
+#define BIT_CTRL_GPIO_OEN_3			BIT(2)
+#define BIT_CTRL_GPIO_I_2			BIT(1)
+#define BIT_CTRL_GPIO_OEN_2			BIT(0)
+
+/* Interrupt Source 7, default value: 0x00 */
+#define REG_INTR7				0x007b
+
+/* Interrupt Source 8, default value: 0x00 */
+#define REG_INTR8				0x007c
+
+/* Interrupt #7 Mask, default value: 0x00 */
+#define REG_INTR7_MASK				0x007d
+
+/* Interrupt #8 Mask, default value: 0x00 */
+#define REG_INTR8_MASK				0x007e
+#define BIT_CEA_NEW_VSI				BIT(2)
+#define BIT_CEA_NEW_AVI				BIT(1)
+
+/* IEEE, default value: 0x10 */
+#define REG_TMDS_CCTRL				0x0080
+#define BIT_TMDS_CCTRL_TMDS_OE			BIT(4)
+
+/* TMDS Control #4, default value: 0x02 */
+#define REG_TMDS_CTRL4				0x0085
+#define BIT_TMDS_CTRL4_SCDT_CKDT_SEL		BIT(1)
+#define BIT_TMDS_CTRL4_TX_EN_BY_SCDT		BIT(0)
+
+/* BIST CNTL, default value: 0x00 */
+#define REG_BIST_CTRL				0x00bb
+#define BIT_RXBIST_VGB_EN			BIT(7)
+#define BIT_TXBIST_VGB_EN			BIT(6)
+#define BIT_BIST_START_SEL			BIT(5)
+#define BIT_BIST_START_BIT			BIT(4)
+#define BIT_BIST_ALWAYS_ON			BIT(3)
+#define BIT_BIST_TRANS				BIT(2)
+#define BIT_BIST_RESET				BIT(1)
+#define BIT_BIST_EN				BIT(0)
+
+/* BIST DURATION0, default value: 0x00 */
+#define REG_BIST_TEST_SEL			0x00bd
+#define MSK_BIST_TEST_SEL_BIST_PATT_SEL		0x0f
+
+/* BIST VIDEO_MODE, default value: 0x00 */
+#define REG_BIST_VIDEO_MODE			0x00be
+#define MSK_BIST_VIDEO_MODE_BIST_VIDEO_MODE_3_0	0x0f
+
+/* BIST DURATION0, default value: 0x00 */
+#define REG_BIST_DURATION_0			0x00bf
+
+/* BIST DURATION1, default value: 0x00 */
+#define REG_BIST_DURATION_1			0x00c0
+
+/* BIST DURATION2, default value: 0x00 */
+#define REG_BIST_DURATION_2			0x00c1
+
+/* BIST 8BIT_PATTERN, default value: 0x00 */
+#define REG_BIST_8BIT_PATTERN			0x00c2
+
+/* LM DDC, default value: 0x80 */
+#define REG_LM_DDC				0x00c7
+#define BIT_LM_DDC_SW_TPI_EN_DISABLED		BIT(7)
+
+#define BIT_LM_DDC_VIDEO_MUTE_EN		BIT(5)
+#define BIT_LM_DDC_DDC_TPI_SW			BIT(2)
+#define BIT_LM_DDC_DDC_GRANT			BIT(1)
+#define BIT_LM_DDC_DDC_GPU_REQUEST		BIT(0)
+
+/* DDC I2C Manual, default value: 0x03 */
+#define REG_DDC_MANUAL				0x00ec
+#define BIT_DDC_MANUAL_MAN_DDC			BIT(7)
+#define BIT_DDC_MANUAL_VP_SEL			BIT(6)
+#define BIT_DDC_MANUAL_DSDA			BIT(5)
+#define BIT_DDC_MANUAL_DSCL			BIT(4)
+#define BIT_DDC_MANUAL_GCP_HW_CTL_EN		BIT(3)
+#define BIT_DDC_MANUAL_DDCM_ABORT_WP		BIT(2)
+#define BIT_DDC_MANUAL_IO_DSDA			BIT(1)
+#define BIT_DDC_MANUAL_IO_DSCL			BIT(0)
+
+/* DDC I2C Target Slave Address, default value: 0x00 */
+#define REG_DDC_ADDR				0x00ed
+#define MSK_DDC_ADDR_DDC_ADDR			0xfe
+
+/* DDC I2C Target Segment Address, default value: 0x00 */
+#define REG_DDC_SEGM				0x00ee
+
+/* DDC I2C Target Offset Address, default value: 0x00 */
+#define REG_DDC_OFFSET				0x00ef
+
+/* DDC I2C Data In count #1, default value: 0x00 */
+#define REG_DDC_DIN_CNT1			0x00f0
+
+/* DDC I2C Data In count #2, default value: 0x00 */
+#define REG_DDC_DIN_CNT2			0x00f1
+#define MSK_DDC_DIN_CNT2_DDC_DIN_CNT_9_8	0x03
+
+/* DDC I2C Status, default value: 0x04 */
+#define REG_DDC_STATUS				0x00f2
+#define BIT_DDC_STATUS_DDC_BUS_LOW		BIT(6)
+#define BIT_DDC_STATUS_DDC_NO_ACK		BIT(5)
+#define BIT_DDC_STATUS_DDC_I2C_IN_PROG		BIT(4)
+#define BIT_DDC_STATUS_DDC_FIFO_FULL		BIT(3)
+#define BIT_DDC_STATUS_DDC_FIFO_EMPTY		BIT(2)
+#define BIT_DDC_STATUS_DDC_FIFO_READ_IN_SUE	BIT(1)
+#define BIT_DDC_STATUS_DDC_FIFO_WRITE_IN_USE	BIT(0)
+
+/* DDC I2C Command, default value: 0x70 */
+#define REG_DDC_CMD				0x00f3
+#define BIT_DDC_CMD_HDCP_DDC_EN			BIT(6)
+#define BIT_DDC_CMD_SDA_DEL_EN			BIT(5)
+#define BIT_DDC_CMD_DDC_FLT_EN			BIT(4)
+
+#define MSK_DDC_CMD_DDC_CMD			0x0f
+#define VAL_DDC_CMD_ENH_DDC_READ_NO_ACK		0x04
+#define VAL_DDC_CMD_DDC_CMD_CLEAR_FIFO		0x09
+#define VAL_DDC_CMD_DDC_CMD_ABORT		0x0f
+
+/* DDC I2C FIFO Data In/Out, default value: 0x00 */
+#define REG_DDC_DATA				0x00f4
+
+/* DDC I2C Data Out Counter, default value: 0x00 */
+#define REG_DDC_DOUT_CNT			0x00f5
+#define BIT_DDC_DOUT_CNT_DDC_DELAY_CNT_8	BIT(7)
+#define MSK_DDC_DOUT_CNT_DDC_DATA_OUT_CNT	0x1f
+
+/* DDC I2C Delay Count, default value: 0x14 */
+#define REG_DDC_DELAY_CNT			0x00f6
+
+/* Test Control, default value: 0x80 */
+#define REG_TEST_TXCTRL				0x00f7
+#define BIT_TEST_TXCTRL_RCLK_REF_SEL		BIT(7)
+#define BIT_TEST_TXCTRL_PCLK_REF_SEL		BIT(6)
+#define MSK_TEST_TXCTRL_BYPASS_PLL_CLK		0x3c
+#define BIT_TEST_TXCTRL_HDMI_MODE		BIT(1)
+#define BIT_TEST_TXCTRL_TST_PLLCK		BIT(0)
+
+/* CBUS Address, default value: 0x00 */
+#define REG_PAGE_CBUS_ADDR			0x00f8
+
+/* I2C Device Address re-assignment */
+#define REG_PAGE1_ADDR				0x00fc
+#define REG_PAGE2_ADDR				0x00fd
+#define REG_PAGE3_ADDR				0x00fe
+#define REG_HW_TPI_ADDR				0x00ff
+
+/* USBT CTRL0, default value: 0x00 */
+#define REG_UTSRST				0x0100
+#define BIT_UTSRST_FC_SRST			BIT(5)
+#define BIT_UTSRST_KEEPER_SRST			BIT(4)
+#define BIT_UTSRST_HTX_SRST			BIT(3)
+#define BIT_UTSRST_TRX_SRST			BIT(2)
+#define BIT_UTSRST_TTX_SRST			BIT(1)
+#define BIT_UTSRST_HRX_SRST			BIT(0)
+
+/* HSIC RX Control3, default value: 0x07 */
+#define REG_HRXCTRL3				0x0104
+#define MSK_HRXCTRL3_HRX_AFFCTRL		0xf0
+#define BIT_HRXCTRL3_HRX_OUT_EN			BIT(2)
+#define BIT_HRXCTRL3_STATUS_EN			BIT(1)
+#define BIT_HRXCTRL3_HRX_STAY_RESET		BIT(0)
+
+/* HSIC RX INT Registers */
+#define REG_HRXINTL				0x0111
+#define REG_HRXINTH				0x0112
+
+/* TDM TX NUMBITS, default value: 0x0c */
+#define REG_TTXNUMB				0x0116
+#define MSK_TTXNUMB_TTX_AFFCTRL_3_0		0xf0
+#define BIT_TTXNUMB_TTX_COM1_AT_SYNC_WAIT	BIT(3)
+#define MSK_TTXNUMB_TTX_NUMBPS_2_0		0x07
+
+/* TDM TX NUMSPISYM, default value: 0x04 */
+#define REG_TTXSPINUMS				0x0117
+
+/* TDM TX NUMHSICSYM, default value: 0x14 */
+#define REG_TTXHSICNUMS				0x0118
+
+/* TDM TX NUMTOTSYM, default value: 0x18 */
+#define REG_TTXTOTNUMS				0x0119
+
+/* TDM TX INT Low, default value: 0x00 */
+#define REG_TTXINTL				0x0136
+#define BIT_TTXINTL_TTX_INTR7			BIT(7)
+#define BIT_TTXINTL_TTX_INTR6			BIT(6)
+#define BIT_TTXINTL_TTX_INTR5			BIT(5)
+#define BIT_TTXINTL_TTX_INTR4			BIT(4)
+#define BIT_TTXINTL_TTX_INTR3			BIT(3)
+#define BIT_TTXINTL_TTX_INTR2			BIT(2)
+#define BIT_TTXINTL_TTX_INTR1			BIT(1)
+#define BIT_TTXINTL_TTX_INTR0			BIT(0)
+
+/* TDM TX INT High, default value: 0x00 */
+#define REG_TTXINTH				0x0137
+#define BIT_TTXINTH_TTX_INTR15			BIT(7)
+#define BIT_TTXINTH_TTX_INTR14			BIT(6)
+#define BIT_TTXINTH_TTX_INTR13			BIT(5)
+#define BIT_TTXINTH_TTX_INTR12			BIT(4)
+#define BIT_TTXINTH_TTX_INTR11			BIT(3)
+#define BIT_TTXINTH_TTX_INTR10			BIT(2)
+#define BIT_TTXINTH_TTX_INTR9			BIT(1)
+#define BIT_TTXINTH_TTX_INTR8			BIT(0)
+
+/* TDM RX Control, default value: 0x1c */
+#define REG_TRXCTRL				0x013b
+#define BIT_TRXCTRL_TRX_CLR_WVALLOW		BIT(4)
+#define BIT_TRXCTRL_TRX_FROM_SE_COC		BIT(3)
+#define MSK_TRXCTRL_TRX_NUMBPS_2_0		0x07
+
+/* TDM RX NUMSPISYM, default value: 0x04 */
+#define REG_TRXSPINUMS				0x013c
+
+/* TDM RX NUMHSICSYM, default value: 0x14 */
+#define REG_TRXHSICNUMS				0x013d
+
+/* TDM RX NUMTOTSYM, default value: 0x18 */
+#define REG_TRXTOTNUMS				0x013e
+
+/* TDM RX Status 2nd, default value: 0x00 */
+#define REG_TRXSTA2				0x015c
+
+/* TDM RX INT Low, default value: 0x00 */
+#define REG_TRXINTL				0x0163
+
+/* TDM RX INT High, default value: 0x00 */
+#define REG_TRXINTH				0x0164
+
+/* TDM RX INTMASK High, default value: 0x00 */
+#define REG_TRXINTMH				0x0166
+
+/* HSIC TX CRTL, default value: 0x00 */
+#define REG_HTXCTRL				0x0169
+#define BIT_HTXCTRL_HTX_ALLSBE_SOP		BIT(4)
+#define BIT_HTXCTRL_HTX_RGDINV_USB		BIT(3)
+#define BIT_HTXCTRL_HTX_RSPTDM_BUSY		BIT(2)
+#define BIT_HTXCTRL_HTX_DRVCONN1		BIT(1)
+#define BIT_HTXCTRL_HTX_DRVRST1			BIT(0)
+
+/* HSIC TX INT Low, default value: 0x00 */
+#define REG_HTXINTL				0x017d
+
+/* HSIC TX INT High, default value: 0x00 */
+#define REG_HTXINTH				0x017e
+
+/* HSIC Keeper, default value: 0x00 */
+#define REG_KEEPER				0x0181
+#define MSK_KEEPER_KEEPER_MODE_1_0		0x03
+
+/* HSIC Flow Control General, default value: 0x02 */
+#define REG_FCGC				0x0183
+#define BIT_FCGC_HSIC_FC_HOSTMODE		BIT(1)
+#define BIT_FCGC_HSIC_FC_ENABLE			BIT(0)
+
+/* HSIC Flow Control CTR13, default value: 0xfc */
+#define REG_FCCTR13				0x0191
+
+/* HSIC Flow Control CTR14, default value: 0xff */
+#define REG_FCCTR14				0x0192
+
+/* HSIC Flow Control CTR15, default value: 0xff */
+#define REG_FCCTR15				0x0193
+
+/* HSIC Flow Control CTR50, default value: 0x03 */
+#define REG_FCCTR50				0x01b6
+
+/* HSIC Flow Control INTR0, default value: 0x00 */
+#define REG_FCINTR0				0x01ec
+#define REG_FCINTR1				0x01ed
+#define REG_FCINTR2				0x01ee
+#define REG_FCINTR3				0x01ef
+#define REG_FCINTR4				0x01f0
+#define REG_FCINTR5				0x01f1
+#define REG_FCINTR6				0x01f2
+#define REG_FCINTR7				0x01f3
+
+/* TDM Low Latency, default value: 0x20 */
+#define REG_TDMLLCTL				0x01fc
+#define MSK_TDMLLCTL_TRX_LL_SEL_MANUAL		0xc0
+#define MSK_TDMLLCTL_TRX_LL_SEL_MODE		0x30
+#define MSK_TDMLLCTL_TTX_LL_SEL_MANUAL		0x0c
+#define BIT_TDMLLCTL_TTX_LL_TIE_LOW		BIT(1)
+#define BIT_TDMLLCTL_TTX_LL_SEL_MODE		BIT(0)
+
+/* TMDS 0 Clock Control, default value: 0x10 */
+#define REG_TMDS0_CCTRL1			0x0210
+#define MSK_TMDS0_CCTRL1_TEST_SEL		0xc0
+#define MSK_TMDS0_CCTRL1_CLK1X_CTL		0x30
+
+/* TMDS Clock Enable, default value: 0x00 */
+#define REG_TMDS_CLK_EN				0x0211
+#define BIT_TMDS_CLK_EN_CLK_EN			BIT(0)
+
+/* TMDS Channel Enable, default value: 0x00 */
+#define REG_TMDS_CH_EN				0x0212
+#define BIT_TMDS_CH_EN_CH0_EN			BIT(4)
+#define BIT_TMDS_CH_EN_CH12_EN			BIT(0)
+
+/* BGR_BIAS, default value: 0x07 */
+#define REG_BGR_BIAS				0x0215
+#define BIT_BGR_BIAS_BGR_EN			BIT(7)
+#define MSK_BGR_BIAS_BIAS_BGR_D			0x0f
+
+/* TMDS 0 Digital I2C BW, default value: 0x0a */
+#define REG_ALICE0_BW_I2C			0x0231
+
+/* TMDS 0 Digital Zone Control, default value: 0xe0 */
+#define REG_ALICE0_ZONE_CTRL			0x024c
+#define BIT_ALICE0_ZONE_CTRL_ICRST_N		BIT(7)
+#define BIT_ALICE0_ZONE_CTRL_USE_INT_DIV20	BIT(6)
+#define MSK_ALICE0_ZONE_CTRL_SZONE_I2C		0x30
+#define MSK_ALICE0_ZONE_CTRL_ZONE_CTRL		0x0f
+
+/* TMDS 0 Digital PLL Mode Control, default value: 0x00 */
+#define REG_ALICE0_MODE_CTRL			0x024d
+#define MSK_ALICE0_MODE_CTRL_PLL_MODE_I2C	0x0c
+#define MSK_ALICE0_MODE_CTRL_DIV20_CTRL		0x03
+
+/* MHL Tx Control 6th, default value: 0xa0 */
+#define REG_MHLTX_CTL6				0x0285
+#define MSK_MHLTX_CTL6_EMI_SEL			0xe0
+#define MSK_MHLTX_CTL6_TX_CLK_SHAPE_9_8		0x03
+
+/* Packet Filter0, default value: 0x00 */
+#define REG_PKT_FILTER_0			0x0290
+#define BIT_PKT_FILTER_0_DROP_CEA_GAMUT_PKT	BIT(7)
+#define BIT_PKT_FILTER_0_DROP_CEA_CP_PKT	BIT(6)
+#define BIT_PKT_FILTER_0_DROP_MPEG_PKT		BIT(5)
+#define BIT_PKT_FILTER_0_DROP_SPIF_PKT		BIT(4)
+#define BIT_PKT_FILTER_0_DROP_AIF_PKT		BIT(3)
+#define BIT_PKT_FILTER_0_DROP_AVI_PKT		BIT(2)
+#define BIT_PKT_FILTER_0_DROP_CTS_PKT		BIT(1)
+#define BIT_PKT_FILTER_0_DROP_GCP_PKT		BIT(0)
+
+/* Packet Filter1, default value: 0x00 */
+#define REG_PKT_FILTER_1			0x0291
+#define BIT_PKT_FILTER_1_VSI_OVERRIDE_DIS	BIT(7)
+#define BIT_PKT_FILTER_1_AVI_OVERRIDE_DIS	BIT(6)
+#define BIT_PKT_FILTER_1_DROP_AUDIO_PKT		BIT(3)
+#define BIT_PKT_FILTER_1_DROP_GEN2_PKT		BIT(2)
+#define BIT_PKT_FILTER_1_DROP_GEN_PKT		BIT(1)
+#define BIT_PKT_FILTER_1_DROP_VSIF_PKT		BIT(0)
+
+/* TMDS Clock Status, default value: 0x10 */
+#define REG_TMDS_CSTAT_P3			0x02a0
+#define BIT_TMDS_CSTAT_P3_RX_HDMI_CP_CLR_MUTE	BIT(7)
+#define BIT_TMDS_CSTAT_P3_RX_HDMI_CP_SET_MUTE	BIT(6)
+#define BIT_TMDS_CSTAT_P3_RX_HDMI_CP_NEW_CP	BIT(5)
+#define BIT_TMDS_CSTAT_P3_CLR_AVI		BIT(3)
+#define BIT_TMDS_CSTAT_P3_SCDT_CLR_AVI_DIS	BIT(2)
+#define BIT_TMDS_CSTAT_P3_SCDT			BIT(1)
+#define BIT_TMDS_CSTAT_P3_CKDT			BIT(0)
+
+/* RX_HDMI Control, default value: 0x10 */
+#define REG_RX_HDMI_CTRL0			0x02a1
+#define BIT_RX_HDMI_CTRL0_BYP_DVIFILT_SYNC	BIT(5)
+#define BIT_RX_HDMI_CTRL0_HDMI_MODE_EN_ITSELF_CLR BIT(4)
+#define BIT_RX_HDMI_CTRL0_HDMI_MODE_SW_VALUE	BIT(3)
+#define BIT_RX_HDMI_CTRL0_HDMI_MODE_OVERWRITE	BIT(2)
+#define BIT_RX_HDMI_CTRL0_RX_HDMI_HDMI_MODE_EN	BIT(1)
+#define BIT_RX_HDMI_CTRL0_RX_HDMI_HDMI_MODE	BIT(0)
+
+/* RX_HDMI Control, default value: 0x38 */
+#define REG_RX_HDMI_CTRL2			0x02a3
+#define MSK_RX_HDMI_CTRL2_IDLE_CNT		0xf0
+#define VAL_RX_HDMI_CTRL2_IDLE_CNT(n)		((n) << 4)
+#define BIT_RX_HDMI_CTRL2_USE_AV_MUTE		BIT(3)
+#define BIT_RX_HDMI_CTRL2_VSI_MON_SEL_VSI	BIT(0)
+
+/* RX_HDMI Control, default value: 0x0f */
+#define REG_RX_HDMI_CTRL3			0x02a4
+#define MSK_RX_HDMI_CTRL3_PP_MODE_CLK_EN	0x0f
+
+/* rx_hdmi Clear Buffer, default value: 0x00 */
+#define REG_RX_HDMI_CLR_BUFFER			0x02ac
+#define MSK_RX_HDMI_CLR_BUFFER_AIF4VSI_CMP	0xc0
+#define BIT_RX_HDMI_CLR_BUFFER_USE_AIF4VSI	BIT(5)
+#define BIT_RX_HDMI_CLR_BUFFER_VSI_CLR_W_AVI	BIT(4)
+#define BIT_RX_HDMI_CLR_BUFFER_VSI_IEEE_ID_CHK_EN BIT(3)
+#define BIT_RX_HDMI_CLR_BUFFER_SWAP_VSI_IEEE_ID	BIT(2)
+#define BIT_RX_HDMI_CLR_BUFFER_AIF_CLR_EN	BIT(1)
+#define BIT_RX_HDMI_CLR_BUFFER_VSI_CLR_EN	BIT(0)
+
+/* RX_HDMI VSI Header1, default value: 0x00 */
+#define REG_RX_HDMI_MON_PKT_HEADER1		0x02b8
+
+/* RX_HDMI VSI MHL Monitor, default value: 0x3c */
+#define REG_RX_HDMI_VSIF_MHL_MON		0x02d7
+
+#define MSK_RX_HDMI_VSIF_MHL_MON_RX_HDMI_MHL_3D_FORMAT 0x3c
+#define MSK_RX_HDMI_VSIF_MHL_MON_RX_HDMI_MHL_VID_FORMAT 0x03
+
+/* Interrupt Source 9, default value: 0x00 */
+#define REG_INTR9				0x02e0
+#define BIT_INTR9_EDID_ERROR			BIT(6)
+#define BIT_INTR9_EDID_DONE			BIT(5)
+#define BIT_INTR9_DEVCAP_DONE			BIT(4)
+
+/* Interrupt 9 Mask, default value: 0x00 */
+#define REG_INTR9_MASK				0x02e1
+
+/* TPI CBUS Start, default value: 0x00 */
+#define REG_TPI_CBUS_START			0x02e2
+#define BIT_TPI_CBUS_START_RCP_REQ_START	BIT(7)
+#define BIT_TPI_CBUS_START_RCPK_REPLY_START	BIT(6)
+#define BIT_TPI_CBUS_START_RCPE_REPLY_START	BIT(5)
+#define BIT_TPI_CBUS_START_PUT_LINK_MODE_START	BIT(4)
+#define BIT_TPI_CBUS_START_PUT_DCAPCHG_START	BIT(3)
+#define BIT_TPI_CBUS_START_PUT_DCAPRDY_START	BIT(2)
+#define BIT_TPI_CBUS_START_GET_EDID_START_0	BIT(1)
+#define BIT_TPI_CBUS_START_GET_DEVCAP_START	BIT(0)
+
+/* EDID Control, default value: 0x10 */
+#define REG_EDID_CTRL				0x02e3
+#define BIT_EDID_CTRL_EDID_PRIME_VALID		BIT(7)
+#define BIT_EDID_CTRL_XDEVCAP_EN		BIT(6)
+#define BIT_EDID_CTRL_DEVCAP_SELECT_DEVCAP	BIT(5)
+#define BIT_EDID_CTRL_EDID_FIFO_ADDR_AUTO	BIT(4)
+#define BIT_EDID_CTRL_EDID_FIFO_ACCESS_ALWAYS_EN BIT(3)
+#define BIT_EDID_CTRL_EDID_FIFO_BLOCK_SEL	BIT(2)
+#define BIT_EDID_CTRL_INVALID_BKSV		BIT(1)
+#define BIT_EDID_CTRL_EDID_MODE_EN		BIT(0)
+
+/* EDID FIFO Addr, default value: 0x00 */
+#define REG_EDID_FIFO_ADDR			0x02e9
+
+/* EDID FIFO Write Data, default value: 0x00 */
+#define REG_EDID_FIFO_WR_DATA			0x02ea
+
+/* EDID/DEVCAP FIFO Internal Addr, default value: 0x00 */
+#define REG_EDID_FIFO_ADDR_MON			0x02eb
+
+/* EDID FIFO Read Data, default value: 0x00 */
+#define REG_EDID_FIFO_RD_DATA			0x02ec
+
+/* EDID DDC Segment Pointer, default value: 0x00 */
+#define REG_EDID_START_EXT			0x02ed
+
+/* TX IP BIST CNTL and Status, default value: 0x00 */
+#define REG_TX_IP_BIST_CNTLSTA			0x02f2
+#define BIT_TX_IP_BIST_CNTLSTA_TXBIST_QUARTER_CLK_SEL BIT(6)
+#define BIT_TX_IP_BIST_CNTLSTA_TXBIST_DONE	BIT(5)
+#define BIT_TX_IP_BIST_CNTLSTA_TXBIST_ON	BIT(4)
+#define BIT_TX_IP_BIST_CNTLSTA_TXBIST_RUN	BIT(3)
+#define BIT_TX_IP_BIST_CNTLSTA_TXCLK_HALF_SEL	BIT(2)
+#define BIT_TX_IP_BIST_CNTLSTA_TXBIST_EN	BIT(1)
+#define BIT_TX_IP_BIST_CNTLSTA_TXBIST_SEL	BIT(0)
+
+/* TX IP BIST INST LOW, default value: 0x00 */
+#define REG_TX_IP_BIST_INST_LOW			0x02f3
+#define REG_TX_IP_BIST_INST_HIGH		0x02f4
+
+/* TX IP BIST PATTERN LOW, default value: 0x00 */
+#define REG_TX_IP_BIST_PAT_LOW			0x02f5
+#define REG_TX_IP_BIST_PAT_HIGH			0x02f6
+
+/* TX IP BIST CONFIGURE LOW, default value: 0x00 */
+#define REG_TX_IP_BIST_CONF_LOW			0x02f7
+#define REG_TX_IP_BIST_CONF_HIGH		0x02f8
+
+/* E-MSC General Control, default value: 0x80 */
+#define REG_GENCTL				0x0300
+#define BIT_GENCTL_SPEC_TRANS_DIS		BIT(7)
+#define BIT_GENCTL_DIS_XMIT_ERR_STATE		BIT(6)
+#define BIT_GENCTL_SPI_MISO_EDGE		BIT(5)
+#define BIT_GENCTL_SPI_MOSI_EDGE		BIT(4)
+#define BIT_GENCTL_CLR_EMSC_RFIFO		BIT(3)
+#define BIT_GENCTL_CLR_EMSC_XFIFO		BIT(2)
+#define BIT_GENCTL_START_TRAIN_SEQ		BIT(1)
+#define BIT_GENCTL_EMSC_EN			BIT(0)
+
+/* E-MSC Comma ErrorCNT, default value: 0x03 */
+#define REG_COMMECNT				0x0305
+#define BIT_COMMECNT_I2C_TO_EMSC_EN		BIT(7)
+#define MSK_COMMECNT_COMMA_CHAR_ERR_CNT		0x0f
+
+/* E-MSC RFIFO ByteCnt, default value: 0x00 */
+#define REG_EMSCRFIFOBCNTL			0x031a
+#define REG_EMSCRFIFOBCNTH			0x031b
+
+/* SPI Burst Cnt Status, default value: 0x00 */
+#define REG_SPIBURSTCNT				0x031e
+
+/* SPI Burst Status and SWRST, default value: 0x00 */
+#define REG_SPIBURSTSTAT			0x0322
+#define BIT_SPIBURSTSTAT_SPI_HDCPRST		BIT(7)
+#define BIT_SPIBURSTSTAT_SPI_CBUSRST		BIT(6)
+#define BIT_SPIBURSTSTAT_SPI_SRST		BIT(5)
+#define BIT_SPIBURSTSTAT_EMSC_NORMAL_MODE	BIT(0)
+
+/* E-MSC 1st Interrupt, default value: 0x00 */
+#define REG_EMSCINTR				0x0323
+#define BIT_EMSCINTR_EMSC_XFIFO_EMPTY		BIT(7)
+#define BIT_EMSCINTR_EMSC_XMIT_ACK_TOUT		BIT(6)
+#define BIT_EMSCINTR_EMSC_RFIFO_READ_ERR	BIT(5)
+#define BIT_EMSCINTR_EMSC_XFIFO_WRITE_ERR	BIT(4)
+#define BIT_EMSCINTR_EMSC_COMMA_CHAR_ERR	BIT(3)
+#define BIT_EMSCINTR_EMSC_XMIT_DONE		BIT(2)
+#define BIT_EMSCINTR_EMSC_XMIT_GNT_TOUT		BIT(1)
+#define BIT_EMSCINTR_SPI_DVLD		BIT(0)
+
+/* E-MSC Interrupt Mask, default value: 0x00 */
+#define REG_EMSCINTRMASK			0x0324
+
+/* I2C E-MSC XMIT FIFO Write Port, default value: 0x00 */
+#define REG_EMSC_XMIT_WRITE_PORT		0x032a
+
+/* I2C E-MSC RCV FIFO Write Port, default value: 0x00 */
+#define REG_EMSC_RCV_READ_PORT			0x032b
+
+/* E-MSC 2nd Interrupt, default value: 0x00 */
+#define REG_EMSCINTR1				0x032c
+#define BIT_EMSCINTR1_EMSC_TRAINING_COMMA_ERR	BIT(0)
+
+/* E-MSC Interrupt Mask, default value: 0x00 */
+#define REG_EMSCINTRMASK1			0x032d
+#define BIT_EMSCINTRMASK1_EMSC_INTRMASK1_0	BIT(0)
+
+/* MHL Top Ctl, default value: 0x00 */
+#define REG_MHL_TOP_CTL				0x0330
+#define BIT_MHL_TOP_CTL_MHL3_DOC_SEL		BIT(7)
+#define BIT_MHL_TOP_CTL_MHL_PP_SEL		BIT(6)
+#define MSK_MHL_TOP_CTL_IF_TIMING_CTL		0x03
+
+/* MHL DataPath 1st Ctl, default value: 0xbc */
+#define REG_MHL_DP_CTL0				0x0331
+#define BIT_MHL_DP_CTL0_DP_OE			BIT(7)
+#define BIT_MHL_DP_CTL0_TX_OE_OVR		BIT(6)
+#define MSK_MHL_DP_CTL0_TX_OE			0x3f
+
+/* MHL DataPath 2nd Ctl, default value: 0xbb */
+#define REG_MHL_DP_CTL1				0x0332
+#define MSK_MHL_DP_CTL1_CK_SWING_CTL		0xf0
+#define MSK_MHL_DP_CTL1_DT_SWING_CTL		0x0f
+
+/* MHL DataPath 3rd Ctl, default value: 0x2f */
+#define REG_MHL_DP_CTL2				0x0333
+#define BIT_MHL_DP_CTL2_CLK_BYPASS_EN		BIT(7)
+#define MSK_MHL_DP_CTL2_DAMP_TERM_SEL		0x30
+#define MSK_MHL_DP_CTL2_CK_TERM_SEL		0x0c
+#define MSK_MHL_DP_CTL2_DT_TERM_SEL		0x03
+
+/* MHL DataPath 4th Ctl, default value: 0x48 */
+#define REG_MHL_DP_CTL3				0x0334
+#define MSK_MHL_DP_CTL3_DT_DRV_VNBC_CTL		0xf0
+#define MSK_MHL_DP_CTL3_DT_DRV_VNB_CTL		0x0f
+
+/* MHL DataPath 5th Ctl, default value: 0x48 */
+#define REG_MHL_DP_CTL4				0x0335
+#define MSK_MHL_DP_CTL4_CK_DRV_VNBC_CTL		0xf0
+#define MSK_MHL_DP_CTL4_CK_DRV_VNB_CTL		0x0f
+
+/* MHL DataPath 6th Ctl, default value: 0x3f */
+#define REG_MHL_DP_CTL5				0x0336
+#define BIT_MHL_DP_CTL5_RSEN_EN_OVR		BIT(7)
+#define BIT_MHL_DP_CTL5_RSEN_EN			BIT(6)
+#define MSK_MHL_DP_CTL5_DAMP_TERM_VGS_CTL	0x30
+#define MSK_MHL_DP_CTL5_CK_TERM_VGS_CTL		0x0c
+#define MSK_MHL_DP_CTL5_DT_TERM_VGS_CTL		0x03
+
+/* MHL PLL 1st Ctl, default value: 0x05 */
+#define REG_MHL_PLL_CTL0			0x0337
+#define BIT_MHL_PLL_CTL0_AUD_CLK_EN		BIT(7)
+
+#define MSK_MHL_PLL_CTL0_AUD_CLK_RATIO		0x70
+#define VAL_MHL_PLL_CTL0_AUD_CLK_RATIO_5_10	0x70
+#define VAL_MHL_PLL_CTL0_AUD_CLK_RATIO_5_6	0x60
+#define VAL_MHL_PLL_CTL0_AUD_CLK_RATIO_5_4	0x50
+#define VAL_MHL_PLL_CTL0_AUD_CLK_RATIO_5_2	0x40
+#define VAL_MHL_PLL_CTL0_AUD_CLK_RATIO_5_5	0x30
+#define VAL_MHL_PLL_CTL0_AUD_CLK_RATIO_5_3	0x20
+#define VAL_MHL_PLL_CTL0_AUD_CLK_RATIO_5_2_PRIME 0x10
+#define VAL_MHL_PLL_CTL0_AUD_CLK_RATIO_5_1	0x00
+
+#define MSK_MHL_PLL_CTL0_HDMI_CLK_RATIO		0x0c
+#define VAL_MHL_PLL_CTL0_HDMI_CLK_RATIO_4X	0x0c
+#define VAL_MHL_PLL_CTL0_HDMI_CLK_RATIO_2X	0x08
+#define VAL_MHL_PLL_CTL0_HDMI_CLK_RATIO_1X	0x04
+#define VAL_MHL_PLL_CTL0_HDMI_CLK_RATIO_HALF_X	0x00
+
+#define BIT_MHL_PLL_CTL0_CRYSTAL_CLK_SEL	BIT(1)
+#define BIT_MHL_PLL_CTL0_ZONE_MASK_OE		BIT(0)
+
+/* MHL PLL 3rd Ctl, default value: 0x80 */
+#define REG_MHL_PLL_CTL2			0x0339
+#define BIT_MHL_PLL_CTL2_CLKDETECT_EN		BIT(7)
+#define BIT_MHL_PLL_CTL2_MEAS_FVCO		BIT(3)
+#define BIT_MHL_PLL_CTL2_PLL_FAST_LOCK		BIT(2)
+#define MSK_MHL_PLL_CTL2_PLL_LF_SEL		0x03
+
+/* MHL CBUS 1st Ctl, default value: 0x12 */
+#define REG_MHL_CBUS_CTL0			0x0340
+#define BIT_MHL_CBUS_CTL0_CBUS_RGND_TEST_MODE	BIT(7)
+
+#define MSK_MHL_CBUS_CTL0_CBUS_RGND_VTH_CTL	0x30
+#define VAL_MHL_CBUS_CTL0_CBUS_RGND_VBIAS_734	0x00
+#define VAL_MHL_CBUS_CTL0_CBUS_RGND_VBIAS_747	0x10
+#define VAL_MHL_CBUS_CTL0_CBUS_RGND_VBIAS_740	0x20
+#define VAL_MHL_CBUS_CTL0_CBUS_RGND_VBIAS_754	0x30
+
+#define MSK_MHL_CBUS_CTL0_CBUS_RES_TEST_SEL	0x0c
+
+#define MSK_MHL_CBUS_CTL0_CBUS_DRV_SEL		0x03
+#define VAL_MHL_CBUS_CTL0_CBUS_DRV_SEL_WEAKEST	0x00
+#define VAL_MHL_CBUS_CTL0_CBUS_DRV_SEL_WEAK	0x01
+#define VAL_MHL_CBUS_CTL0_CBUS_DRV_SEL_STRONG	0x02
+#define VAL_MHL_CBUS_CTL0_CBUS_DRV_SEL_STRONGEST 0x03
+
+/* MHL CBUS 2nd Ctl, default value: 0x03 */
+#define REG_MHL_CBUS_CTL1			0x0341
+#define MSK_MHL_CBUS_CTL1_CBUS_RGND_RES_CTL	0x07
+#define VAL_MHL_CBUS_CTL1_0888_OHM		0x00
+#define VAL_MHL_CBUS_CTL1_1115_OHM		0x04
+#define VAL_MHL_CBUS_CTL1_1378_OHM		0x07
+
+/* MHL CoC 1st Ctl, default value: 0xc3 */
+#define REG_MHL_COC_CTL0			0x0342
+#define BIT_MHL_COC_CTL0_COC_BIAS_EN		BIT(7)
+#define MSK_MHL_COC_CTL0_COC_BIAS_CTL		0x70
+#define MSK_MHL_COC_CTL0_COC_TERM_CTL		0x07
+
+/* MHL CoC 2nd Ctl, default value: 0x87 */
+#define REG_MHL_COC_CTL1			0x0343
+#define BIT_MHL_COC_CTL1_COC_EN			BIT(7)
+#define MSK_MHL_COC_CTL1_COC_DRV_CTL		0x3f
+
+/* MHL CoC 4th Ctl, default value: 0x00 */
+#define REG_MHL_COC_CTL3			0x0345
+#define BIT_MHL_COC_CTL3_COC_AECHO_EN		BIT(0)
+
+/* MHL CoC 5th Ctl, default value: 0x28 */
+#define REG_MHL_COC_CTL4			0x0346
+#define MSK_MHL_COC_CTL4_COC_IF_CTL		0xf0
+#define MSK_MHL_COC_CTL4_COC_SLEW_CTL		0x0f
+
+/* MHL CoC 6th Ctl, default value: 0x0d */
+#define REG_MHL_COC_CTL5			0x0347
+
+/* MHL DoC 1st Ctl, default value: 0x18 */
+#define REG_MHL_DOC_CTL0			0x0349
+#define BIT_MHL_DOC_CTL0_DOC_RXDATA_EN		BIT(7)
+#define MSK_MHL_DOC_CTL0_DOC_DM_TERM		0x38
+#define MSK_MHL_DOC_CTL0_DOC_OPMODE		0x06
+#define BIT_MHL_DOC_CTL0_DOC_RXBIAS_EN		BIT(0)
+
+/* MHL DataPath 7th Ctl, default value: 0x2a */
+#define REG_MHL_DP_CTL6				0x0350
+#define BIT_MHL_DP_CTL6_DP_TAP2_SGN		BIT(5)
+#define BIT_MHL_DP_CTL6_DP_TAP2_EN		BIT(4)
+#define BIT_MHL_DP_CTL6_DP_TAP1_SGN		BIT(3)
+#define BIT_MHL_DP_CTL6_DP_TAP1_EN		BIT(2)
+#define BIT_MHL_DP_CTL6_DT_PREDRV_FEEDCAP_EN	BIT(1)
+#define BIT_MHL_DP_CTL6_DP_PRE_POST_SEL		BIT(0)
+
+/* MHL DataPath 8th Ctl, default value: 0x06 */
+#define REG_MHL_DP_CTL7				0x0351
+#define MSK_MHL_DP_CTL7_DT_DRV_VBIAS_CASCTL	0xf0
+#define MSK_MHL_DP_CTL7_DT_DRV_IREF_CTL		0x0f
+
+/* Tx Zone Ctl1, default value: 0x00 */
+#define REG_TX_ZONE_CTL1			0x0361
+#define VAL_TX_ZONE_CTL1_TX_ZONE_CTRL_MODE	0x08
+
+/* MHL3 Tx Zone Ctl, default value: 0x00 */
+#define REG_MHL3_TX_ZONE_CTL			0x0364
+#define BIT_MHL3_TX_ZONE_CTL_MHL2_INTPLT_ZONE_MANU_EN BIT(7)
+#define MSK_MHL3_TX_ZONE_CTL_MHL3_TX_ZONE	0x03
+
+#define MSK_TX_ZONE_CTL3_TX_ZONE		0x03
+#define VAL_TX_ZONE_CTL3_TX_ZONE_6GBPS		0x00
+#define VAL_TX_ZONE_CTL3_TX_ZONE_3GBPS		0x01
+#define VAL_TX_ZONE_CTL3_TX_ZONE_1_5GBPS	0x02
+
+/* HDCP Polling Control and Status, default value: 0x70 */
+#define REG_HDCP2X_POLL_CS			0x0391
+
+#define BIT_HDCP2X_POLL_CS_HDCP2X_MSG_SZ_CLR_OPTION BIT(6)
+#define BIT_HDCP2X_POLL_CS_HDCP2X_RPT_READY_CLR_OPTION BIT(5)
+#define BIT_HDCP2X_POLL_CS_HDCP2X_REAUTH_REQ_CLR_OPTION BIT(4)
+#define MSK_HDCP2X_POLL_CS_			0x0c
+#define BIT_HDCP2X_POLL_CS_HDCP2X_DIS_POLL_GNT	BIT(1)
+#define BIT_HDCP2X_POLL_CS_HDCP2X_DIS_POLL_EN	BIT(0)
+
+/* HDCP Interrupt 0, default value: 0x00 */
+#define REG_HDCP2X_INTR0			0x0398
+
+/* HDCP Interrupt 0 Mask, default value: 0x00 */
+#define REG_HDCP2X_INTR0_MASK			0x0399
+
+/* HDCP General Control 0, default value: 0x02 */
+#define REG_HDCP2X_CTRL_0			0x03a0
+#define BIT_HDCP2X_CTRL_0_HDCP2X_ENCRYPT_EN	BIT(7)
+#define BIT_HDCP2X_CTRL_0_HDCP2X_POLINT_SEL	BIT(6)
+#define BIT_HDCP2X_CTRL_0_HDCP2X_POLINT_OVR	BIT(5)
+#define BIT_HDCP2X_CTRL_0_HDCP2X_PRECOMPUTE	BIT(4)
+#define BIT_HDCP2X_CTRL_0_HDCP2X_HDMIMODE	BIT(3)
+#define BIT_HDCP2X_CTRL_0_HDCP2X_REPEATER	BIT(2)
+#define BIT_HDCP2X_CTRL_0_HDCP2X_HDCPTX		BIT(1)
+#define BIT_HDCP2X_CTRL_0_HDCP2X_EN		BIT(0)
+
+/* HDCP General Control 1, default value: 0x08 */
+#define REG_HDCP2X_CTRL_1			0x03a1
+#define MSK_HDCP2X_CTRL_1_HDCP2X_REAUTH_MSK_3_0	0xf0
+#define BIT_HDCP2X_CTRL_1_HDCP2X_HPD_SW		BIT(3)
+#define BIT_HDCP2X_CTRL_1_HDCP2X_HPD_OVR	BIT(2)
+#define BIT_HDCP2X_CTRL_1_HDCP2X_CTL3MSK	BIT(1)
+#define BIT_HDCP2X_CTRL_1_HDCP2X_REAUTH_SW	BIT(0)
+
+/* HDCP Misc Control, default value: 0x00 */
+#define REG_HDCP2X_MISC_CTRL			0x03a5
+#define BIT_HDCP2X_MISC_CTRL_HDCP2X_RPT_SMNG_XFER_START BIT(4)
+#define BIT_HDCP2X_MISC_CTRL_HDCP2X_RPT_SMNG_WR_START BIT(3)
+#define BIT_HDCP2X_MISC_CTRL_HDCP2X_RPT_SMNG_WR	BIT(2)
+#define BIT_HDCP2X_MISC_CTRL_HDCP2X_RPT_RCVID_RD_START BIT(1)
+#define BIT_HDCP2X_MISC_CTRL_HDCP2X_RPT_RCVID_RD	BIT(0)
+
+/* HDCP RPT SMNG K, default value: 0x00 */
+#define REG_HDCP2X_RPT_SMNG_K			0x03a6
+
+/* HDCP RPT SMNG In, default value: 0x00 */
+#define REG_HDCP2X_RPT_SMNG_IN			0x03a7
+
+/* HDCP Auth Status, default value: 0x00 */
+#define REG_HDCP2X_AUTH_STAT			0x03aa
+
+/* HDCP RPT RCVID Out, default value: 0x00 */
+#define REG_HDCP2X_RPT_RCVID_OUT		0x03ac
+
+/* HDCP TP1, default value: 0x62 */
+#define REG_HDCP2X_TP1				0x03b4
+
+/* HDCP GP Out 0, default value: 0x00 */
+#define REG_HDCP2X_GP_OUT0			0x03c7
+
+/* HDCP Repeater RCVR ID 0, default value: 0x00 */
+#define REG_HDCP2X_RPT_RCVR_ID0			0x03d1
+
+/* HDCP DDCM Status, default value: 0x00 */
+#define REG_HDCP2X_DDCM_STS			0x03d8
+#define MSK_HDCP2X_DDCM_STS_HDCP2X_DDCM_ERR_STS_3_0 0xf0
+#define MSK_HDCP2X_DDCM_STS_HDCP2X_DDCM_CTL_CS_3_0 0x0f
+
+/* HDMI2MHL3 Control, default value: 0x0a */
+#define REG_M3_CTRL				0x03e0
+#define BIT_M3_CTRL_H2M_SWRST			BIT(4)
+#define BIT_M3_CTRL_SW_MHL3_SEL			BIT(3)
+#define BIT_M3_CTRL_M3AV_EN			BIT(2)
+#define BIT_M3_CTRL_ENC_TMDS			BIT(1)
+#define BIT_M3_CTRL_MHL3_MASTER_EN		BIT(0)
+
+#define VAL_M3_CTRL_MHL1_2_VALUE (BIT_M3_CTRL_SW_MHL3_SEL \
+				  | BIT_M3_CTRL_ENC_TMDS)
+#define VAL_M3_CTRL_MHL3_VALUE (BIT_M3_CTRL_SW_MHL3_SEL \
+				| BIT_M3_CTRL_M3AV_EN \
+				| BIT_M3_CTRL_ENC_TMDS \
+				| BIT_M3_CTRL_MHL3_MASTER_EN)
+
+/* HDMI2MHL3 Port0 Control, default value: 0x04 */
+#define REG_M3_P0CTRL				0x03e1
+#define BIT_M3_P0CTRL_MHL3_P0_HDCP_ENC_EN	BIT(4)
+#define BIT_M3_P0CTRL_MHL3_P0_UNLIMIT_EN	BIT(3)
+#define BIT_M3_P0CTRL_MHL3_P0_HDCP_EN		BIT(2)
+#define BIT_M3_P0CTRL_MHL3_P0_PIXEL_MODE_PACKED	BIT(1)
+#define BIT_M3_P0CTRL_MHL3_P0_PORT_EN		BIT(0)
+
+#define REG_M3_POSTM				0x03e2
+#define MSK_M3_POSTM_RRP_DECODE			0xf8
+#define MSK_M3_POSTM_MHL3_P0_STM_ID		0x07
+
+/* HDMI2MHL3 Scramble Control, default value: 0x41 */
+#define REG_M3_SCTRL				0x03e6
+#define MSK_M3_SCTRL_MHL3_SR_LENGTH		0xf0
+#define BIT_M3_SCTRL_MHL3_SCRAMBLER_EN		BIT(0)
+
+/* HSIC Div Ctl, default value: 0x05 */
+#define REG_DIV_CTL_MAIN			0x03f2
+#define MSK_DIV_CTL_MAIN_PRE_DIV_CTL_MAIN	0x1c
+#define MSK_DIV_CTL_MAIN_FB_DIV_CTL_MAIN	0x03
+
+/* MHL Capability 1st Byte, default value: 0x00 */
+#define REG_MHL_DEVCAP_0			0x0400
+
+/* MHL Interrupt 1st Byte, default value: 0x00 */
+#define REG_MHL_INT_0				0x0420
+
+/* Device Status 1st byte, default value: 0x00 */
+#define REG_MHL_STAT_0				0x0430
+
+/* CBUS Scratch Pad 1st Byte, default value: 0x00 */
+#define REG_MHL_SCRPAD_0			0x0440
+
+/* MHL Extended Capability 1st Byte, default value: 0x00 */
+#define REG_MHL_EXTDEVCAP_0			0x0480
+
+/* Device Extended Status 1st byte, default value: 0x00 */
+#define REG_MHL_EXTSTAT_0			0x0490
+
+/* TPI DTD Byte2, default value: 0x00 */
+#define REG_TPI_DTD_B2				0x0602
+
+#define VAL_TPI_QUAN_RANGE_LIMITED		0x01
+#define VAL_TPI_QUAN_RANGE_FULL			0x02
+#define VAL_TPI_FORMAT_RGB			0x00
+#define VAL_TPI_FORMAT_YCBCR444			0x01
+#define VAL_TPI_FORMAT_YCBCR422			0x02
+#define VAL_TPI_FORMAT_INTERNAL_RGB		0x03
+#define VAL_TPI_FORMAT(_fmt, _qr) \
+		(VAL_TPI_FORMAT_##_fmt | (VAL_TPI_QUAN_RANGE_##_qr << 2))
+
+/* Input Format, default value: 0x00 */
+#define REG_TPI_INPUT				0x0609
+#define BIT_TPI_INPUT_EXTENDEDBITMODE		BIT(7)
+#define BIT_TPI_INPUT_ENDITHER			BIT(6)
+#define MSK_TPI_INPUT_INPUT_QUAN_RANGE		0x0c
+#define MSK_TPI_INPUT_INPUT_FORMAT		0x03
+
+/* Output Format, default value: 0x00 */
+#define REG_TPI_OUTPUT				0x060a
+#define BIT_TPI_OUTPUT_CSCMODE709		BIT(4)
+#define MSK_TPI_OUTPUT_OUTPUT_QUAN_RANGE	0x0c
+#define MSK_TPI_OUTPUT_OUTPUT_FORMAT		0x03
+
+/* TPI AVI Check Sum, default value: 0x00 */
+#define REG_TPI_AVI_CHSUM			0x060c
+
+/* TPI System Control, default value: 0x00 */
+#define REG_TPI_SC				0x061a
+#define BIT_TPI_SC_TPI_UPDATE_FLG		BIT(7)
+#define BIT_TPI_SC_TPI_REAUTH_CTL		BIT(6)
+#define BIT_TPI_SC_TPI_OUTPUT_MODE_1		BIT(5)
+#define BIT_TPI_SC_REG_TMDS_OE_POWER_DOWN	BIT(4)
+#define BIT_TPI_SC_TPI_AV_MUTE			BIT(3)
+#define BIT_TPI_SC_DDC_GPU_REQUEST		BIT(2)
+#define BIT_TPI_SC_DDC_TPI_SW			BIT(1)
+#define BIT_TPI_SC_TPI_OUTPUT_MODE_0_HDMI	BIT(0)
+
+/* TPI COPP Query Data, default value: 0x00 */
+#define REG_TPI_COPP_DATA1			0x0629
+#define BIT_TPI_COPP_DATA1_COPP_GPROT		BIT(7)
+#define BIT_TPI_COPP_DATA1_COPP_LPROT		BIT(6)
+#define MSK_TPI_COPP_DATA1_COPP_LINK_STATUS	0x30
+#define VAL_TPI_COPP_LINK_STATUS_NORMAL		0x00
+#define VAL_TPI_COPP_LINK_STATUS_LINK_LOST	0x10
+#define VAL_TPI_COPP_LINK_STATUS_RENEGOTIATION_REQ 0x20
+#define VAL_TPI_COPP_LINK_STATUS_LINK_SUSPENDED	0x30
+#define BIT_TPI_COPP_DATA1_COPP_HDCP_REP	BIT(3)
+#define BIT_TPI_COPP_DATA1_COPP_CONNTYPE_0	BIT(2)
+#define BIT_TPI_COPP_DATA1_COPP_PROTYPE		BIT(1)
+#define BIT_TPI_COPP_DATA1_COPP_CONNTYPE_1	BIT(0)
+
+/* TPI COPP Control Data, default value: 0x00 */
+#define REG_TPI_COPP_DATA2			0x062a
+#define BIT_TPI_COPP_DATA2_INTR_ENCRYPTION	BIT(5)
+#define BIT_TPI_COPP_DATA2_KSV_FORWARD		BIT(4)
+#define BIT_TPI_COPP_DATA2_INTERM_RI_CHECK_EN	BIT(3)
+#define BIT_TPI_COPP_DATA2_DOUBLE_RI_CHECK	BIT(2)
+#define BIT_TPI_COPP_DATA2_DDC_SHORT_RI_RD	BIT(1)
+#define BIT_TPI_COPP_DATA2_COPP_PROTLEVEL	BIT(0)
+
+/* TPI Interrupt Enable, default value: 0x00 */
+#define REG_TPI_INTR_EN				0x063c
+
+/* TPI Interrupt Status Low Byte, default value: 0x00 */
+#define REG_TPI_INTR_ST0			0x063d
+#define BIT_TPI_INTR_ST0_TPI_AUTH_CHNGE_STAT	BIT(7)
+#define BIT_TPI_INTR_ST0_TPI_V_RDY_STAT		BIT(6)
+#define BIT_TPI_INTR_ST0_TPI_COPP_CHNGE_STAT	BIT(5)
+#define BIT_TPI_INTR_ST0_KSV_FIFO_FIRST_STAT	BIT(3)
+#define BIT_TPI_INTR_ST0_READ_BKSV_BCAPS_DONE_STAT BIT(2)
+#define BIT_TPI_INTR_ST0_READ_BKSV_BCAPS_ERR_STAT BIT(1)
+#define BIT_TPI_INTR_ST0_READ_BKSV_ERR_STAT	BIT(0)
+
+/* TPI DS BCAPS Status, default value: 0x00 */
+#define REG_TPI_DS_BCAPS			0x0644
+
+/* TPI BStatus1, default value: 0x00 */
+#define REG_TPI_BSTATUS1			0x0645
+#define BIT_TPI_BSTATUS1_DS_DEV_EXCEED		BIT(7)
+#define MSK_TPI_BSTATUS1_DS_DEV_CNT		0x7f
+
+/* TPI BStatus2, default value: 0x10 */
+#define REG_TPI_BSTATUS2			0x0646
+#define MSK_TPI_BSTATUS2_DS_BSTATUS		0xe0
+#define BIT_TPI_BSTATUS2_DS_HDMI_MODE		BIT(4)
+#define BIT_TPI_BSTATUS2_DS_CASC_EXCEED		BIT(3)
+#define MSK_TPI_BSTATUS2_DS_DEPTH		0x07
+
+/* TPI HW Optimization Control #3, default value: 0x00 */
+#define REG_TPI_HW_OPT3				0x06bb
+#define BIT_TPI_HW_OPT3_DDC_DEBUG		BIT(7)
+#define BIT_TPI_HW_OPT3_RI_CHECK_SKIP		BIT(3)
+#define BIT_TPI_HW_OPT3_TPI_DDC_BURST_MODE	BIT(2)
+#define MSK_TPI_HW_OPT3_TPI_DDC_REQ_LEVEL	0x03
+
+/* TPI Info Frame Select, default value: 0x00 */
+#define REG_TPI_INFO_FSEL			0x06bf
+#define BIT_TPI_INFO_FSEL_TPI_INFO_EN		BIT(7)
+#define BIT_TPI_INFO_FSEL_TPI_INFO_RPT		BIT(6)
+#define BIT_TPI_INFO_FSEL_TPI_INFO_READ_FLAG	BIT(5)
+#define MSK_TPI_INFO_FSEL_TPI_INFO_SEL		0x07
+
+/* TPI Info Byte #0, default value: 0x00 */
+#define REG_TPI_INFO_B0				0x06c0
+
+/* CoC Status, default value: 0x00 */
+#define REG_COC_STAT_0				0x0700
+#define REG_COC_STAT_1				0x0701
+#define REG_COC_STAT_2				0x0702
+#define REG_COC_STAT_3				0x0703
+#define REG_COC_STAT_4				0x0704
+#define REG_COC_STAT_5				0x0705
+
+/* CoC 1st Ctl, default value: 0x40 */
+#define REG_COC_CTL0				0x0710
+
+/* CoC 2nd Ctl, default value: 0x0a */
+#define REG_COC_CTL1				0x0711
+#define MSK_COC_CTL1_COC_CTRL1_7_6		0xc0
+#define MSK_COC_CTL1_COC_CTRL1_5_0		0x3f
+
+/* CoC 3rd Ctl, default value: 0x14 */
+#define REG_COC_CTL2				0x0712
+#define MSK_COC_CTL2_COC_CTRL2_7_6		0xc0
+#define MSK_COC_CTL2_COC_CTRL2_5_0		0x3f
+
+/* CoC 4th Ctl, default value: 0x40 */
+#define REG_COC_CTL3				0x0713
+#define BIT_COC_CTL3_COC_CTRL3_7		BIT(7)
+#define MSK_COC_CTL3_COC_CTRL3_6_0		0x7f
+
+/* CoC 7th Ctl, default value: 0x00 */
+#define REG_COC_CTL6				0x0716
+#define BIT_COC_CTL6_COC_CTRL6_7		BIT(7)
+#define BIT_COC_CTL6_COC_CTRL6_6		BIT(6)
+#define MSK_COC_CTL6_COC_CTRL6_5_0		0x3f
+
+/* CoC 8th Ctl, default value: 0x06 */
+#define REG_COC_CTL7				0x0717
+#define BIT_COC_CTL7_COC_CTRL7_7		BIT(7)
+#define BIT_COC_CTL7_COC_CTRL7_6		BIT(6)
+#define BIT_COC_CTL7_COC_CTRL7_5		BIT(5)
+#define MSK_COC_CTL7_COC_CTRL7_4_3		0x18
+#define MSK_COC_CTL7_COC_CTRL7_2_0		0x07
+
+/* CoC 10th Ctl, default value: 0x00 */
+#define REG_COC_CTL9				0x0719
+
+/* CoC 11th Ctl, default value: 0x00 */
+#define REG_COC_CTLA				0x071a
+
+/* CoC 12th Ctl, default value: 0x00 */
+#define REG_COC_CTLB				0x071b
+
+/* CoC 13th Ctl, default value: 0x0f */
+#define REG_COC_CTLC				0x071c
+
+/* CoC 14th Ctl, default value: 0x0a */
+#define REG_COC_CTLD				0x071d
+#define BIT_COC_CTLD_COC_CTRLD_7		BIT(7)
+#define MSK_COC_CTLD_COC_CTRLD_6_0		0x7f
+
+/* CoC 15th Ctl, default value: 0x0a */
+#define REG_COC_CTLE				0x071e
+#define BIT_COC_CTLE_COC_CTRLE_7		BIT(7)
+#define MSK_COC_CTLE_COC_CTRLE_6_0		0x7f
+
+/* CoC 16th Ctl, default value: 0x00 */
+#define REG_COC_CTLF				0x071f
+#define MSK_COC_CTLF_COC_CTRLF_7_3		0xf8
+#define MSK_COC_CTLF_COC_CTRLF_2_0		0x07
+
+/* CoC 18th Ctl, default value: 0x32 */
+#define REG_COC_CTL11				0x0721
+#define MSK_COC_CTL11_COC_CTRL11_7_4		0xf0
+#define MSK_COC_CTL11_COC_CTRL11_3_0		0x0f
+
+/* CoC 21st Ctl, default value: 0x00 */
+#define REG_COC_CTL14				0x0724
+#define MSK_COC_CTL14_COC_CTRL14_7_4		0xf0
+#define MSK_COC_CTL14_COC_CTRL14_3_0		0x0f
+
+/* CoC 22nd Ctl, default value: 0x00 */
+#define REG_COC_CTL15				0x0725
+#define BIT_COC_CTL15_COC_CTRL15_7		BIT(7)
+#define MSK_COC_CTL15_COC_CTRL15_6_4		0x70
+#define MSK_COC_CTL15_COC_CTRL15_3_0		0x0f
+
+/* CoC Interrupt, default value: 0x00 */
+#define REG_COC_INTR				0x0726
+
+/* CoC Interrupt Mask, default value: 0x00 */
+#define REG_COC_INTR_MASK			0x0727
+#define BIT_COC_PLL_LOCK_STATUS_CHANGE		BIT(0)
+#define BIT_COC_CALIBRATION_DONE		BIT(1)
+
+/* CoC Misc Ctl, default value: 0x00 */
+#define REG_COC_MISC_CTL0			0x0728
+#define BIT_COC_MISC_CTL0_FSM_MON		BIT(7)
+
+/* CoC 24th Ctl, default value: 0x00 */
+#define REG_COC_CTL17				0x072a
+#define MSK_COC_CTL17_COC_CTRL17_7_4		0xf0
+#define MSK_COC_CTL17_COC_CTRL17_3_0		0x0f
+
+/* CoC 25th Ctl, default value: 0x00 */
+#define REG_COC_CTL18				0x072b
+#define MSK_COC_CTL18_COC_CTRL18_7_4		0xf0
+#define MSK_COC_CTL18_COC_CTRL18_3_0		0x0f
+
+/* CoC 26th Ctl, default value: 0x00 */
+#define REG_COC_CTL19				0x072c
+#define MSK_COC_CTL19_COC_CTRL19_7_4		0xf0
+#define MSK_COC_CTL19_COC_CTRL19_3_0		0x0f
+
+/* CoC 27th Ctl, default value: 0x00 */
+#define REG_COC_CTL1A				0x072d
+#define MSK_COC_CTL1A_COC_CTRL1A_7_2		0xfc
+#define MSK_COC_CTL1A_COC_CTRL1A_1_0		0x03
+
+/* DoC 9th Status, default value: 0x00 */
+#define REG_DOC_STAT_8				0x0740
+
+/* DoC 10th Status, default value: 0x00 */
+#define REG_DOC_STAT_9				0x0741
+
+/* DoC 5th CFG, default value: 0x00 */
+#define REG_DOC_CFG4				0x074e
+#define MSK_DOC_CFG4_DBG_STATE_DOC_FSM		0x0f
+
+/* DoC 1st Ctl, default value: 0x40 */
+#define REG_DOC_CTL0				0x0751
+
+/* DoC 7th Ctl, default value: 0x00 */
+#define REG_DOC_CTL6				0x0757
+#define BIT_DOC_CTL6_DOC_CTRL6_7		BIT(7)
+#define BIT_DOC_CTL6_DOC_CTRL6_6		BIT(6)
+#define MSK_DOC_CTL6_DOC_CTRL6_5_4		0x30
+#define MSK_DOC_CTL6_DOC_CTRL6_3_0		0x0f
+
+/* DoC 8th Ctl, default value: 0x00 */
+#define REG_DOC_CTL7				0x0758
+#define BIT_DOC_CTL7_DOC_CTRL7_7		BIT(7)
+#define BIT_DOC_CTL7_DOC_CTRL7_6		BIT(6)
+#define BIT_DOC_CTL7_DOC_CTRL7_5		BIT(5)
+#define MSK_DOC_CTL7_DOC_CTRL7_4_3		0x18
+#define MSK_DOC_CTL7_DOC_CTRL7_2_0		0x07
+
+/* DoC 9th Ctl, default value: 0x00 */
+#define REG_DOC_CTL8				0x076c
+#define BIT_DOC_CTL8_DOC_CTRL8_7		BIT(7)
+#define MSK_DOC_CTL8_DOC_CTRL8_6_4		0x70
+#define MSK_DOC_CTL8_DOC_CTRL8_3_2		0x0c
+#define MSK_DOC_CTL8_DOC_CTRL8_1_0		0x03
+
+/* DoC 10th Ctl, default value: 0x00 */
+#define REG_DOC_CTL9				0x076d
+
+/* DoC 11th Ctl, default value: 0x00 */
+#define REG_DOC_CTLA				0x076e
+
+/* DoC 15th Ctl, default value: 0x00 */
+#define REG_DOC_CTLE				0x0772
+#define BIT_DOC_CTLE_DOC_CTRLE_7		BIT(7)
+#define BIT_DOC_CTLE_DOC_CTRLE_6		BIT(6)
+#define MSK_DOC_CTLE_DOC_CTRLE_5_4		0x30
+#define MSK_DOC_CTLE_DOC_CTRLE_3_0		0x0f
+
+/* Interrupt Mask 1st, default value: 0x00 */
+#define REG_MHL_INT_0_MASK			0x0580
+
+/* Interrupt Mask 2nd, default value: 0x00 */
+#define REG_MHL_INT_1_MASK			0x0581
+
+/* Interrupt Mask 3rd, default value: 0x00 */
+#define REG_MHL_INT_2_MASK			0x0582
+
+/* Interrupt Mask 4th, default value: 0x00 */
+#define REG_MHL_INT_3_MASK			0x0583
+
+/* MDT Receive Time Out, default value: 0x00 */
+#define REG_MDT_RCV_TIMEOUT			0x0584
+
+/* MDT Transmit Time Out, default value: 0x00 */
+#define REG_MDT_XMIT_TIMEOUT			0x0585
+
+/* MDT Receive Control, default value: 0x00 */
+#define REG_MDT_RCV_CTRL			0x0586
+#define BIT_MDT_RCV_CTRL_MDT_RCV_EN		BIT(7)
+#define BIT_MDT_RCV_CTRL_MDT_DELAY_RCV_EN	BIT(6)
+#define BIT_MDT_RCV_CTRL_MDT_RFIFO_OVER_WR_EN	BIT(4)
+#define BIT_MDT_RCV_CTRL_MDT_XFIFO_OVER_WR_EN	BIT(3)
+#define BIT_MDT_RCV_CTRL_MDT_DISABLE		BIT(2)
+#define BIT_MDT_RCV_CTRL_MDT_RFIFO_CLR_ALL	BIT(1)
+#define BIT_MDT_RCV_CTRL_MDT_RFIFO_CLR_CUR	BIT(0)
+
+/* MDT Receive Read Port, default value: 0x00 */
+#define REG_MDT_RCV_READ_PORT			0x0587
+
+/* MDT Transmit Control, default value: 0x70 */
+#define REG_MDT_XMIT_CTRL			0x0588
+#define BIT_MDT_XMIT_CTRL_MDT_XMIT_EN		BIT(7)
+#define BIT_MDT_XMIT_CTRL_MDT_XMIT_CMD_MERGE_EN	BIT(6)
+#define BIT_MDT_XMIT_CTRL_MDT_XMIT_FIXED_BURST_LEN BIT(5)
+#define BIT_MDT_XMIT_CTRL_MDT_XMIT_FIXED_AID	BIT(4)
+#define BIT_MDT_XMIT_CTRL_MDT_XMIT_SINGLE_RUN_EN BIT(3)
+#define BIT_MDT_XMIT_CTRL_MDT_CLR_ABORT_WAIT	BIT(2)
+#define BIT_MDT_XMIT_CTRL_MDT_XFIFO_CLR_ALL	BIT(1)
+#define BIT_MDT_XMIT_CTRL_MDT_XFIFO_CLR_CUR	BIT(0)
+
+/* MDT Receive WRITE Port, default value: 0x00 */
+#define REG_MDT_XMIT_WRITE_PORT			0x0589
+
+/* MDT RFIFO Status, default value: 0x00 */
+#define REG_MDT_RFIFO_STAT			0x058a
+#define MSK_MDT_RFIFO_STAT_MDT_RFIFO_CNT	0xe0
+#define MSK_MDT_RFIFO_STAT_MDT_RFIFO_CUR_BYTE_CNT 0x1f
+
+/* MDT XFIFO Status, default value: 0x80 */
+#define REG_MDT_XFIFO_STAT			0x058b
+#define MSK_MDT_XFIFO_STAT_MDT_XFIFO_LEVEL_AVAIL 0xe0
+#define BIT_MDT_XFIFO_STAT_MDT_XMIT_PRE_HS_EN	BIT(4)
+#define MSK_MDT_XFIFO_STAT_MDT_WRITE_BURST_LEN	0x0f
+
+/* MDT Interrupt 0, default value: 0x0c */
+#define REG_MDT_INT_0				0x058c
+#define BIT_MDT_RFIFO_DATA_RDY			BIT(0)
+#define BIT_MDT_IDLE_AFTER_HAWB_DISABLE		BIT(2)
+#define BIT_MDT_XFIFO_EMPTY			BIT(3)
+
+/* MDT Interrupt 0 Mask, default value: 0x00 */
+#define REG_MDT_INT_0_MASK			0x058d
+
+/* MDT Interrupt 1, default value: 0x00 */
+#define REG_MDT_INT_1				0x058e
+#define BIT_MDT_RCV_TIMEOUT			BIT(0)
+#define BIT_MDT_RCV_SM_ABORT_PKT_RCVD		BIT(1)
+#define BIT_MDT_RCV_SM_ERROR			BIT(2)
+#define BIT_MDT_XMIT_TIMEOUT			BIT(5)
+#define BIT_MDT_XMIT_SM_ABORT_PKT_RCVD		BIT(6)
+#define BIT_MDT_XMIT_SM_ERROR			BIT(7)
+
+/* MDT Interrupt 1 Mask, default value: 0x00 */
+#define REG_MDT_INT_1_MASK			0x058f
+
+/* CBUS Vendor ID, default value: 0x01 */
+#define REG_CBUS_VENDOR_ID			0x0590
+
+/* CBUS Connection Status, default value: 0x00 */
+#define REG_CBUS_STATUS				0x0591
+#define BIT_CBUS_STATUS_MHL_CABLE_PRESENT	BIT(4)
+#define BIT_CBUS_STATUS_MSC_HB_SUCCESS		BIT(3)
+#define BIT_CBUS_STATUS_CBUS_HPD		BIT(2)
+#define BIT_CBUS_STATUS_MHL_MODE		BIT(1)
+#define BIT_CBUS_STATUS_CBUS_CONNECTED		BIT(0)
+
+/* CBUS Interrupt 1st, default value: 0x00 */
+#define REG_CBUS_INT_0				0x0592
+#define BIT_CBUS_MSC_MT_DONE_NACK		BIT(7)
+#define BIT_CBUS_MSC_MR_SET_INT			BIT(6)
+#define BIT_CBUS_MSC_MR_WRITE_BURST		BIT(5)
+#define BIT_CBUS_MSC_MR_MSC_MSG			BIT(4)
+#define BIT_CBUS_MSC_MR_WRITE_STAT		BIT(3)
+#define BIT_CBUS_HPD_CHG			BIT(2)
+#define BIT_CBUS_MSC_MT_DONE			BIT(1)
+#define BIT_CBUS_CNX_CHG			BIT(0)
+
+/* CBUS Interrupt Mask 1st, default value: 0x00 */
+#define REG_CBUS_INT_0_MASK			0x0593
+
+/* CBUS Interrupt 2nd, default value: 0x00 */
+#define REG_CBUS_INT_1				0x0594
+#define BIT_CBUS_CMD_ABORT			BIT(6)
+#define BIT_CBUS_MSC_ABORT_RCVD			BIT(3)
+#define BIT_CBUS_DDC_ABORT			BIT(2)
+#define BIT_CBUS_CEC_ABORT			BIT(1)
+
+/* CBUS Interrupt Mask 2nd, default value: 0x00 */
+#define REG_CBUS_INT_1_MASK			0x0595
+
+/* CBUS DDC Abort Interrupt, default value: 0x00 */
+#define REG_DDC_ABORT_INT			0x0598
+
+/* CBUS DDC Abort Interrupt Mask, default value: 0x00 */
+#define REG_DDC_ABORT_INT_MASK			0x0599
+
+/* CBUS MSC Requester Abort Interrupt, default value: 0x00 */
+#define REG_MSC_MT_ABORT_INT			0x059a
+
+/* CBUS MSC Requester Abort Interrupt Mask, default value: 0x00 */
+#define REG_MSC_MT_ABORT_INT_MASK		0x059b
+
+/* CBUS MSC Responder Abort Interrupt, default value: 0x00 */
+#define REG_MSC_MR_ABORT_INT			0x059c
+
+/* CBUS MSC Responder Abort Interrupt Mask, default value: 0x00 */
+#define REG_MSC_MR_ABORT_INT_MASK		0x059d
+
+/* CBUS RX DISCOVERY interrupt, default value: 0x00 */
+#define REG_CBUS_RX_DISC_INT0			0x059e
+
+/* CBUS RX DISCOVERY Interrupt Mask, default value: 0x00 */
+#define REG_CBUS_RX_DISC_INT0_MASK		0x059f
+
+/* CBUS_Link_Layer Control #8, default value: 0x00 */
+#define REG_CBUS_LINK_CTRL_8			0x05a7
+
+/* MDT State Machine Status, default value: 0x00 */
+#define REG_MDT_SM_STAT				0x05b5
+#define MSK_MDT_SM_STAT_MDT_RCV_STATE		0xf0
+#define MSK_MDT_SM_STAT_MDT_XMIT_STATE		0x0f
+
+/* CBUS MSC command trigger, default value: 0x00 */
+#define REG_MSC_COMMAND_START			0x05b8
+#define BIT_MSC_COMMAND_START_DEBUG		BIT(5)
+#define BIT_MSC_COMMAND_START_WRITE_BURST	BIT(4)
+#define BIT_MSC_COMMAND_START_WRITE_STAT	BIT(3)
+#define BIT_MSC_COMMAND_START_READ_DEVCAP	BIT(2)
+#define BIT_MSC_COMMAND_START_MSC_MSG		BIT(1)
+#define BIT_MSC_COMMAND_START_PEER		BIT(0)
+
+/* CBUS MSC Command/Offset, default value: 0x00 */
+#define REG_MSC_CMD_OR_OFFSET			0x05b9
+
+/* CBUS MSC Transmit Data */
+#define REG_MSC_1ST_TRANSMIT_DATA		0x05ba
+#define REG_MSC_2ND_TRANSMIT_DATA		0x05bb
+
+/* CBUS MSC Requester Received Data */
+#define REG_MSC_MT_RCVD_DATA0			0x05bc
+#define REG_MSC_MT_RCVD_DATA1			0x05bd
+
+/* CBUS MSC Responder MSC_MSG Received Data */
+#define REG_MSC_MR_MSC_MSG_RCVD_1ST_DATA	0x05bf
+#define REG_MSC_MR_MSC_MSG_RCVD_2ND_DATA	0x05c0
+
+/* CBUS MSC Heartbeat Control, default value: 0x27 */
+#define REG_MSC_HEARTBEAT_CTRL			0x05c4
+#define BIT_MSC_HEARTBEAT_CTRL_MSC_HB_EN	BIT(7)
+#define MSK_MSC_HEARTBEAT_CTRL_MSC_HB_FAIL_LIMIT 0x70
+#define MSK_MSC_HEARTBEAT_CTRL_MSC_HB_PERIOD_MSB 0x0f
+
+/* CBUS MSC Compatibility Control, default value: 0x02 */
+#define REG_CBUS_MSC_COMPAT_CTRL		0x05c7
+#define BIT_CBUS_MSC_COMPAT_CTRL_XDEVCAP_EN	BIT(7)
+#define BIT_CBUS_MSC_COMPAT_CTRL_DISABLE_MSC_ON_CBUS BIT(6)
+#define BIT_CBUS_MSC_COMPAT_CTRL_DISABLE_DDC_ON_CBUS BIT(5)
+#define BIT_CBUS_MSC_COMPAT_CTRL_DISABLE_GET_DDC_ERRORCODE BIT(3)
+#define BIT_CBUS_MSC_COMPAT_CTRL_DISABLE_GET_VS1_ERRORCODE BIT(2)
+
+/* CBUS3 Converter Control, default value: 0x24 */
+#define REG_CBUS3_CNVT				0x05dc
+#define MSK_CBUS3_CNVT_CBUS3_RETRYLMT		0xf0
+#define MSK_CBUS3_CNVT_CBUS3_PEERTOUT_SEL	0x0c
+#define BIT_CBUS3_CNVT_TEARCBUS_EN		BIT(1)
+#define BIT_CBUS3_CNVT_CBUS3CNVT_EN		BIT(0)
+
+/* Discovery Control1, default value: 0x24 */
+#define REG_DISC_CTRL1				0x05e0
+#define BIT_DISC_CTRL1_CBUS_INTR_EN		BIT(7)
+#define BIT_DISC_CTRL1_HB_ONLY			BIT(6)
+#define MSK_DISC_CTRL1_DISC_ATT			0x30
+#define MSK_DISC_CTRL1_DISC_CYC			0x0c
+#define BIT_DISC_CTRL1_DISC_EN			BIT(0)
+
+#define VAL_PUP_OFF				0
+#define VAL_PUP_20K				1
+#define VAL_PUP_5K				2
+
+/* Discovery Control4, default value: 0x80 */
+#define REG_DISC_CTRL4				0x05e3
+#define MSK_DISC_CTRL4_CBUSDISC_PUP_SEL		0xc0
+#define MSK_DISC_CTRL4_CBUSIDLE_PUP_SEL		0x30
+#define VAL_DISC_CTRL4(pup_disc, pup_idle) (((pup_disc) << 6) | (pup_idle << 4))
+
+/* Discovery Control5, default value: 0x03 */
+#define REG_DISC_CTRL5				0x05e4
+#define BIT_DISC_CTRL5_DSM_OVRIDE		BIT(3)
+#define MSK_DISC_CTRL5_CBUSMHL_PUP_SEL		0x03
+
+/* Discovery Control8, default value: 0x81 */
+#define REG_DISC_CTRL8				0x05e7
+#define BIT_DISC_CTRL8_NOMHLINT_CLR_BYPASS	BIT(7)
+#define BIT_DISC_CTRL8_DELAY_CBUS_INTR_EN	BIT(0)
+
+/* Discovery Control9, default value: 0x54 */
+#define REG_DISC_CTRL9			0x05e8
+#define BIT_DISC_CTRL9_MHL3_RSEN_BYP		BIT(7)
+#define BIT_DISC_CTRL9_MHL3DISC_EN		BIT(6)
+#define BIT_DISC_CTRL9_WAKE_DRVFLT		BIT(4)
+#define BIT_DISC_CTRL9_NOMHL_EST		BIT(3)
+#define BIT_DISC_CTRL9_DISC_PULSE_PROCEED	BIT(2)
+#define BIT_DISC_CTRL9_WAKE_PULSE_BYPASS	BIT(1)
+#define BIT_DISC_CTRL9_VBUS_OUTPUT_CAPABILITY_SRC BIT(0)
+
+/* Discovery Status1, default value: 0x00 */
+#define REG_DISC_STAT1				0x05eb
+#define BIT_DISC_STAT1_PSM_OVRIDE		BIT(5)
+#define MSK_DISC_STAT1_DISC_SM			0x0f
+
+/* Discovery Status2, default value: 0x00 */
+#define REG_DISC_STAT2				0x05ec
+#define BIT_DISC_STAT2_CBUS_OE_POL		BIT(6)
+#define BIT_DISC_STAT2_CBUS_SATUS		BIT(5)
+#define BIT_DISC_STAT2_RSEN			BIT(4)
+
+#define MSK_DISC_STAT2_MHL_VRSN			0x0c
+#define VAL_DISC_STAT2_DEFAULT			0x00
+#define VAL_DISC_STAT2_MHL1_2			0x04
+#define VAL_DISC_STAT2_MHL3			0x08
+#define VAL_DISC_STAT2_RESERVED			0x0c
+
+#define MSK_DISC_STAT2_RGND			0x03
+#define VAL_RGND_OPEN				0x00
+#define VAL_RGND_2K				0x01
+#define VAL_RGND_1K				0x02
+#define VAL_RGND_SHORT				0x03
+
+/* Interrupt CBUS_reg1 INTR0, default value: 0x00 */
+#define REG_CBUS_DISC_INTR0			0x05ed
+#define BIT_RGND_READY_INT			BIT(6)
+#define BIT_CBUS_MHL12_DISCON_INT		BIT(5)
+#define BIT_CBUS_MHL3_DISCON_INT		BIT(4)
+#define BIT_NOT_MHL_EST_INT			BIT(3)
+#define BIT_MHL_EST_INT				BIT(2)
+#define BIT_MHL3_EST_INT			BIT(1)
+#define VAL_CBUS_MHL_DISCON (BIT_CBUS_MHL12_DISCON_INT \
+			    | BIT_CBUS_MHL3_DISCON_INT \
+			    | BIT_NOT_MHL_EST_INT)
+
+/* Interrupt CBUS_reg1 INTR0 Mask, default value: 0x00 */
+#define REG_CBUS_DISC_INTR0_MASK		0x05ee
+
+#endif /* __SIL_SII8620_H__ */
diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
index 44d476ea6d2e..de9ffb49e9f6 100644
--- a/drivers/gpu/drm/bridge/tc358767.c
+++ b/drivers/gpu/drm/bridge/tc358767.c
@@ -908,7 +908,7 @@ static int tc_main_link_setup(struct tc_data *tc)
 			goto err_dpcd_read;
 
 		if (tmp[0] != tc->assr) {
-			dev_warn(dev, "Failed to switch display ASSR to %d, falling back to unscrambled mode\n",
+			dev_dbg(dev, "Failed to switch display ASSR to %d, falling back to unscrambled mode\n",
 				 tc->assr);
 			/* trying with disabled scrambler */
 			tc->link.scrambler_dis = 1;
@@ -1038,12 +1038,6 @@ err:
 	return ret;
 }
 
-static enum drm_connector_status
-tc_connector_detect(struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static void tc_bridge_pre_enable(struct drm_bridge *bridge)
 {
 	struct tc_data *tc = bridge_to_tc(bridge);
@@ -1168,7 +1162,6 @@ static const struct drm_connector_helper_funcs tc_connector_helper_funcs = {
 static const struct drm_connector_funcs tc_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.detect = tc_connector_detect,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/bridge/ti-tfp410.c b/drivers/gpu/drm/bridge/ti-tfp410.c
new file mode 100644
index 000000000000..b054ea349952
--- /dev/null
+++ b/drivers/gpu/drm/bridge/ti-tfp410.c
@@ -0,0 +1,317 @@
+/*
+ * Copyright (C) 2016 Texas Instruments
+ * Author: Jyri Sarha <jsarha@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+#include <linux/i2c.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+
+struct tfp410 {
+	struct drm_bridge	bridge;
+	struct drm_connector	connector;
+
+	struct i2c_adapter	*ddc;
+
+	struct device *dev;
+};
+
+static inline struct tfp410 *
+drm_bridge_to_tfp410(struct drm_bridge *bridge)
+{
+	return container_of(bridge, struct tfp410, bridge);
+}
+
+static inline struct tfp410 *
+drm_connector_to_tfp410(struct drm_connector *connector)
+{
+	return container_of(connector, struct tfp410, connector);
+}
+
+static int tfp410_get_modes(struct drm_connector *connector)
+{
+	struct tfp410 *dvi = drm_connector_to_tfp410(connector);
+	struct edid *edid;
+	int ret;
+
+	if (!dvi->ddc)
+		goto fallback;
+
+	edid = drm_get_edid(connector, dvi->ddc);
+	if (!edid) {
+		DRM_INFO("EDID read failed. Fallback to standard modes\n");
+		goto fallback;
+	}
+
+	drm_mode_connector_update_edid_property(connector, edid);
+
+	return drm_add_edid_modes(connector, edid);
+fallback:
+	/* No EDID, fallback on the XGA standard modes */
+	ret = drm_add_modes_noedid(connector, 1920, 1200);
+
+	/* And prefer a mode pretty much anything can handle */
+	drm_set_preferred_mode(connector, 1024, 768);
+
+	return ret;
+}
+
+static const struct drm_connector_helper_funcs tfp410_con_helper_funcs = {
+	.get_modes	= tfp410_get_modes,
+};
+
+static enum drm_connector_status
+tfp410_connector_detect(struct drm_connector *connector, bool force)
+{
+	struct tfp410 *dvi = drm_connector_to_tfp410(connector);
+
+	if (dvi->ddc) {
+		if (drm_probe_ddc(dvi->ddc))
+			return connector_status_connected;
+		else
+			return connector_status_disconnected;
+	}
+
+	return connector_status_unknown;
+}
+
+static const struct drm_connector_funcs tfp410_con_funcs = {
+	.dpms			= drm_atomic_helper_connector_dpms,
+	.detect			= tfp410_connector_detect,
+	.fill_modes		= drm_helper_probe_single_connector_modes,
+	.destroy		= drm_connector_cleanup,
+	.reset			= drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state	= drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
+};
+
+static int tfp410_attach(struct drm_bridge *bridge)
+{
+	struct tfp410 *dvi = drm_bridge_to_tfp410(bridge);
+	int ret;
+
+	if (!bridge->encoder) {
+		dev_err(dvi->dev, "Missing encoder\n");
+		return -ENODEV;
+	}
+
+	drm_connector_helper_add(&dvi->connector,
+				 &tfp410_con_helper_funcs);
+	ret = drm_connector_init(bridge->dev, &dvi->connector,
+				 &tfp410_con_funcs, DRM_MODE_CONNECTOR_HDMIA);
+	if (ret) {
+		dev_err(dvi->dev, "drm_connector_init() failed: %d\n", ret);
+		return ret;
+	}
+
+	drm_mode_connector_attach_encoder(&dvi->connector,
+					  bridge->encoder);
+
+	return 0;
+}
+
+static const struct drm_bridge_funcs tfp410_bridge_funcs = {
+	.attach		= tfp410_attach,
+};
+
+static int tfp410_get_connector_ddc(struct tfp410 *dvi)
+{
+	struct device_node *ep = NULL, *connector_node = NULL;
+	struct device_node *ddc_phandle = NULL;
+	int ret = 0;
+
+	/* port@1 is the connector node */
+	ep = of_graph_get_endpoint_by_regs(dvi->dev->of_node, 1, -1);
+	if (!ep)
+		goto fail;
+
+	connector_node = of_graph_get_remote_port_parent(ep);
+	if (!connector_node)
+		goto fail;
+
+	ddc_phandle = of_parse_phandle(connector_node, "ddc-i2c-bus", 0);
+	if (!ddc_phandle)
+		goto fail;
+
+	dvi->ddc = of_get_i2c_adapter_by_node(ddc_phandle);
+	if (dvi->ddc)
+		dev_info(dvi->dev, "Connector's ddc i2c bus found\n");
+	else
+		ret = -EPROBE_DEFER;
+
+fail:
+	of_node_put(ep);
+	of_node_put(connector_node);
+	of_node_put(ddc_phandle);
+	return ret;
+}
+
+static int tfp410_init(struct device *dev)
+{
+	struct tfp410 *dvi;
+	int ret;
+
+	if (!dev->of_node) {
+		dev_err(dev, "device-tree data is missing\n");
+		return -ENXIO;
+	}
+
+	dvi = devm_kzalloc(dev, sizeof(*dvi), GFP_KERNEL);
+	if (!dvi)
+		return -ENOMEM;
+	dev_set_drvdata(dev, dvi);
+
+	dvi->bridge.funcs = &tfp410_bridge_funcs;
+	dvi->bridge.of_node = dev->of_node;
+	dvi->dev = dev;
+
+	ret = tfp410_get_connector_ddc(dvi);
+	if (ret)
+		goto fail;
+
+	ret = drm_bridge_add(&dvi->bridge);
+	if (ret) {
+		dev_err(dev, "drm_bridge_add() failed: %d\n", ret);
+		goto fail;
+	}
+
+	return 0;
+fail:
+	i2c_put_adapter(dvi->ddc);
+	return ret;
+}
+
+static int tfp410_fini(struct device *dev)
+{
+	struct tfp410 *dvi = dev_get_drvdata(dev);
+
+	drm_bridge_remove(&dvi->bridge);
+
+	if (dvi->ddc)
+		i2c_put_adapter(dvi->ddc);
+
+	return 0;
+}
+
+static int tfp410_probe(struct platform_device *pdev)
+{
+	return tfp410_init(&pdev->dev);
+}
+
+static int tfp410_remove(struct platform_device *pdev)
+{
+	return tfp410_fini(&pdev->dev);
+}
+
+static const struct of_device_id tfp410_match[] = {
+	{ .compatible = "ti,tfp410" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, tfp410_match);
+
+struct platform_driver tfp410_platform_driver = {
+	.probe	= tfp410_probe,
+	.remove	= tfp410_remove,
+	.driver	= {
+		.name		= "tfp410-bridge",
+		.of_match_table	= tfp410_match,
+	},
+};
+
+#if IS_ENABLED(CONFIG_I2C)
+/* There is currently no i2c functionality. */
+static int tfp410_i2c_probe(struct i2c_client *client,
+			    const struct i2c_device_id *id)
+{
+	int reg;
+
+	if (!client->dev.of_node ||
+	    of_property_read_u32(client->dev.of_node, "reg", &reg)) {
+		dev_err(&client->dev,
+			"Can't get i2c reg property from device-tree\n");
+		return -ENXIO;
+	}
+
+	return tfp410_init(&client->dev);
+}
+
+static int tfp410_i2c_remove(struct i2c_client *client)
+{
+	return tfp410_fini(&client->dev);
+}
+
+static const struct i2c_device_id tfp410_i2c_ids[] = {
+	{ "tfp410", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, tfp410_i2c_ids);
+
+static struct i2c_driver tfp410_i2c_driver = {
+	.driver = {
+		.name	= "tfp410",
+		.of_match_table = of_match_ptr(tfp410_match),
+	},
+	.id_table	= tfp410_i2c_ids,
+	.probe		= tfp410_i2c_probe,
+	.remove		= tfp410_i2c_remove,
+};
+#endif /* IS_ENABLED(CONFIG_I2C) */
+
+static struct {
+	uint i2c:1;
+	uint platform:1;
+}  tfp410_registered_driver;
+
+static int __init tfp410_module_init(void)
+{
+	int ret;
+
+#if IS_ENABLED(CONFIG_I2C)
+	ret = i2c_add_driver(&tfp410_i2c_driver);
+	if (ret)
+		pr_err("%s: registering i2c driver failed: %d",
+		       __func__, ret);
+	else
+		tfp410_registered_driver.i2c = 1;
+#endif
+
+	ret = platform_driver_register(&tfp410_platform_driver);
+	if (ret)
+		pr_err("%s: registering platform driver failed: %d",
+		       __func__, ret);
+	else
+		tfp410_registered_driver.platform = 1;
+
+	if (tfp410_registered_driver.i2c ||
+	    tfp410_registered_driver.platform)
+		return 0;
+
+	return ret;
+}
+module_init(tfp410_module_init);
+
+static void __exit tfp410_module_exit(void)
+{
+#if IS_ENABLED(CONFIG_I2C)
+	if (tfp410_registered_driver.i2c)
+		i2c_del_driver(&tfp410_i2c_driver);
+#endif
+	if (tfp410_registered_driver.platform)
+		platform_driver_unregister(&tfp410_platform_driver);
+}
+module_exit(tfp410_module_exit);
+
+MODULE_AUTHOR("Jyri Sarha <jsarha@ti.com>");
+MODULE_DESCRIPTION("TI TFP410 DVI bridge driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/cirrus/cirrus_drv.c b/drivers/gpu/drm/cirrus/cirrus_drv.c
index 6c76d125995b..d893ea21a359 100644
--- a/drivers/gpu/drm/cirrus/cirrus_drv.c
+++ b/drivers/gpu/drm/cirrus/cirrus_drv.c
@@ -126,9 +126,7 @@ static const struct file_operations cirrus_driver_fops = {
 	.unlocked_ioctl = drm_ioctl,
 	.mmap = cirrus_mmap,
 	.poll = drm_poll,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 };
 static struct drm_driver driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM,
diff --git a/drivers/gpu/drm/cirrus/cirrus_fbdev.c b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
index daecf1ad76a4..3a6309d7d8e4 100644
--- a/drivers/gpu/drm/cirrus/cirrus_fbdev.c
+++ b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
@@ -138,12 +138,12 @@ static int cirrusfb_create_object(struct cirrus_fbdev *afbdev,
 {
 	struct drm_device *dev = afbdev->helper.dev;
 	struct cirrus_device *cdev = dev->dev_private;
-	u32 bpp, depth;
+	u32 bpp;
 	u32 size;
 	struct drm_gem_object *gobj;
-
 	int ret = 0;
-	drm_fb_get_bpp_depth(mode_cmd->pixel_format, &depth, &bpp);
+
+	bpp = drm_format_plane_cpp(mode_cmd->pixel_format, 0) * 8;
 
 	if (!cirrus_check_framebuffer(cdev, mode_cmd->width, mode_cmd->height,
 				      bpp, mode_cmd->pitches[0]))
diff --git a/drivers/gpu/drm/cirrus/cirrus_main.c b/drivers/gpu/drm/cirrus/cirrus_main.c
index 76bcb43e7c06..2c3c0d4072ce 100644
--- a/drivers/gpu/drm/cirrus/cirrus_main.c
+++ b/drivers/gpu/drm/cirrus/cirrus_main.c
@@ -52,10 +52,10 @@ cirrus_user_framebuffer_create(struct drm_device *dev,
 	struct cirrus_device *cdev = dev->dev_private;
 	struct drm_gem_object *obj;
 	struct cirrus_framebuffer *cirrus_fb;
+	u32 bpp;
 	int ret;
-	u32 bpp, depth;
 
-	drm_fb_get_bpp_depth(mode_cmd->pixel_format, &depth, &bpp);
+	bpp = drm_format_plane_cpp(mode_cmd->pixel_format, 0) * 8;
 
 	if (!cirrus_check_framebuffer(cdev, mode_cmd->width, mode_cmd->height,
 				      bpp, mode_cmd->pitches[0]))
diff --git a/drivers/gpu/drm/cirrus/cirrus_mode.c b/drivers/gpu/drm/cirrus/cirrus_mode.c
index 17c915d9a03e..9a4a27c1afd2 100644
--- a/drivers/gpu/drm/cirrus/cirrus_mode.c
+++ b/drivers/gpu/drm/cirrus/cirrus_mode.c
@@ -498,12 +498,6 @@ static struct drm_encoder *cirrus_connector_best_encoder(struct drm_connector
 	return NULL;
 }
 
-static enum drm_connector_status cirrus_vga_detect(struct drm_connector
-						   *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static void cirrus_connector_destroy(struct drm_connector *connector)
 {
 	drm_connector_cleanup(connector);
@@ -517,7 +511,6 @@ static const struct drm_connector_helper_funcs cirrus_vga_connector_helper_funcs
 
 static const struct drm_connector_funcs cirrus_vga_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.detect = cirrus_vga_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = cirrus_connector_destroy,
 };
diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c
index 5e7e63ce7bce..d6da848f7c6f 100644
--- a/drivers/gpu/drm/cirrus/cirrus_ttm.c
+++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c
@@ -230,6 +230,7 @@ struct ttm_bo_driver cirrus_bo_driver = {
 	.ttm_tt_populate = cirrus_ttm_tt_populate,
 	.ttm_tt_unpopulate = cirrus_ttm_tt_unpopulate,
 	.init_mem_type = cirrus_bo_init_mem_type,
+	.eviction_valuable = ttm_bo_eviction_valuable,
 	.evict_flags = cirrus_bo_evict_flags,
 	.move = NULL,
 	.verify_access = cirrus_bo_verify_access,
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index e6862a744210..60697482b94c 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -30,6 +30,8 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_mode.h>
 #include <drm/drm_plane_helper.h>
+#include <drm/drm_print.h>
+#include <linux/sync_file.h>
 
 #include "drm_crtc_internal.h"
 
@@ -74,6 +76,8 @@ EXPORT_SYMBOL(drm_atomic_state_default_release);
 int
 drm_atomic_state_init(struct drm_device *dev, struct drm_atomic_state *state)
 {
+	kref_init(&state->ref);
+
 	/* TODO legacy paths should maybe do a better job about
 	 * setting this appropriately?
 	 */
@@ -215,22 +219,16 @@ void drm_atomic_state_clear(struct drm_atomic_state *state)
 EXPORT_SYMBOL(drm_atomic_state_clear);
 
 /**
- * drm_atomic_state_free - free all memory for an atomic state
- * @state: atomic state to deallocate
+ * __drm_atomic_state_free - free all memory for an atomic state
+ * @ref: This atomic state to deallocate
  *
  * This frees all memory associated with an atomic state, including all the
  * per-object state for planes, crtcs and connectors.
  */
-void drm_atomic_state_free(struct drm_atomic_state *state)
+void __drm_atomic_state_free(struct kref *ref)
 {
-	struct drm_device *dev;
-	struct drm_mode_config *config;
-
-	if (!state)
-		return;
-
-	dev = state->dev;
-	config = &dev->mode_config;
+	struct drm_atomic_state *state = container_of(ref, typeof(*state), ref);
+	struct drm_mode_config *config = &state->dev->mode_config;
 
 	drm_atomic_state_clear(state);
 
@@ -243,7 +241,7 @@ void drm_atomic_state_free(struct drm_atomic_state *state)
 		kfree(state);
 	}
 }
-EXPORT_SYMBOL(drm_atomic_state_free);
+EXPORT_SYMBOL(__drm_atomic_state_free);
 
 /**
  * drm_atomic_get_crtc_state - get crtc state
@@ -292,6 +290,23 @@ drm_atomic_get_crtc_state(struct drm_atomic_state *state,
 }
 EXPORT_SYMBOL(drm_atomic_get_crtc_state);
 
+static void set_out_fence_for_crtc(struct drm_atomic_state *state,
+				   struct drm_crtc *crtc, s64 __user *fence_ptr)
+{
+	state->crtcs[drm_crtc_index(crtc)].out_fence_ptr = fence_ptr;
+}
+
+static s64 __user *get_out_fence_for_crtc(struct drm_atomic_state *state,
+					  struct drm_crtc *crtc)
+{
+	s64 __user *fence_ptr;
+
+	fence_ptr = state->crtcs[drm_crtc_index(crtc)].out_fence_ptr;
+	state->crtcs[drm_crtc_index(crtc)].out_fence_ptr = NULL;
+
+	return fence_ptr;
+}
+
 /**
  * drm_atomic_set_mode_for_crtc - set mode for CRTC
  * @state: the CRTC whose incoming state to update
@@ -496,6 +511,16 @@ int drm_atomic_crtc_set_property(struct drm_crtc *crtc,
 					&replaced);
 		state->color_mgmt_changed |= replaced;
 		return ret;
+	} else if (property == config->prop_out_fence_ptr) {
+		s64 __user *fence_ptr = u64_to_user_ptr(val);
+
+		if (!fence_ptr)
+			return 0;
+
+		if (put_user(-1, fence_ptr))
+			return -EFAULT;
+
+		set_out_fence_for_crtc(state->state, crtc, fence_ptr);
 	} else if (crtc->funcs->atomic_set_property)
 		return crtc->funcs->atomic_set_property(crtc, state, property, val);
 	else
@@ -538,6 +563,8 @@ drm_atomic_crtc_get_property(struct drm_crtc *crtc,
 		*val = (state->ctm) ? state->ctm->base.id : 0;
 	else if (property == config->gamma_lut_property)
 		*val = (state->gamma_lut) ? state->gamma_lut->base.id : 0;
+	else if (property == config->prop_out_fence_ptr)
+		*val = 0;
 	else if (crtc->funcs->atomic_get_property)
 		return crtc->funcs->atomic_get_property(crtc, state, property, val);
 	else
@@ -609,6 +636,28 @@ static int drm_atomic_crtc_check(struct drm_crtc *crtc,
 	return 0;
 }
 
+static void drm_atomic_crtc_print_state(struct drm_printer *p,
+		const struct drm_crtc_state *state)
+{
+	struct drm_crtc *crtc = state->crtc;
+
+	drm_printf(p, "crtc[%u]: %s\n", crtc->base.id, crtc->name);
+	drm_printf(p, "\tenable=%d\n", state->enable);
+	drm_printf(p, "\tactive=%d\n", state->active);
+	drm_printf(p, "\tplanes_changed=%d\n", state->planes_changed);
+	drm_printf(p, "\tmode_changed=%d\n", state->mode_changed);
+	drm_printf(p, "\tactive_changed=%d\n", state->active_changed);
+	drm_printf(p, "\tconnectors_changed=%d\n", state->connectors_changed);
+	drm_printf(p, "\tcolor_mgmt_changed=%d\n", state->color_mgmt_changed);
+	drm_printf(p, "\tplane_mask=%x\n", state->plane_mask);
+	drm_printf(p, "\tconnector_mask=%x\n", state->connector_mask);
+	drm_printf(p, "\tencoder_mask=%x\n", state->encoder_mask);
+	drm_printf(p, "\tmode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(&state->mode));
+
+	if (crtc->funcs->atomic_print_state)
+		crtc->funcs->atomic_print_state(p, state);
+}
+
 /**
  * drm_atomic_get_plane_state - get plane state
  * @state: global atomic state object
@@ -693,6 +742,17 @@ int drm_atomic_plane_set_property(struct drm_plane *plane,
 		drm_atomic_set_fb_for_plane(state, fb);
 		if (fb)
 			drm_framebuffer_unreference(fb);
+	} else if (property == config->prop_in_fence_fd) {
+		if (state->fence)
+			return -EINVAL;
+
+		if (U642I64(val) == -1)
+			return 0;
+
+		state->fence = sync_file_get_fence(val);
+		if (!state->fence)
+			return -EINVAL;
+
 	} else if (property == config->prop_crtc_id) {
 		struct drm_crtc *crtc = drm_crtc_find(dev, val);
 		return drm_atomic_set_crtc_for_plane(state, crtc);
@@ -712,7 +772,9 @@ int drm_atomic_plane_set_property(struct drm_plane *plane,
 		state->src_w = val;
 	} else if (property == config->prop_src_h) {
 		state->src_h = val;
-	} else if (property == config->rotation_property) {
+	} else if (property == plane->rotation_property) {
+		if (!is_power_of_2(val & DRM_ROTATE_MASK))
+			return -EINVAL;
 		state->rotation = val;
 	} else if (property == plane->zpos_property) {
 		state->zpos = val;
@@ -752,6 +814,8 @@ drm_atomic_plane_get_property(struct drm_plane *plane,
 
 	if (property == config->prop_fb_id) {
 		*val = (state->fb) ? state->fb->base.id : 0;
+	} else if (property == config->prop_in_fence_fd) {
+		*val = -1;
 	} else if (property == config->prop_crtc_id) {
 		*val = (state->crtc) ? state->crtc->base.id : 0;
 	} else if (property == config->prop_crtc_x) {
@@ -770,7 +834,7 @@ drm_atomic_plane_get_property(struct drm_plane *plane,
 		*val = state->src_w;
 	} else if (property == config->prop_src_h) {
 		*val = state->src_h;
-	} else if (property == config->rotation_property) {
+	} else if (property == plane->rotation_property) {
 		*val = state->rotation;
 	} else if (property == plane->zpos_property) {
 		*val = state->zpos;
@@ -840,9 +904,10 @@ static int drm_atomic_plane_check(struct drm_plane *plane,
 	/* Check whether this plane supports the fb pixel format. */
 	ret = drm_plane_check_pixel_format(plane, state->fb->pixel_format);
 	if (ret) {
-		char *format_name = drm_get_format_name(state->fb->pixel_format);
-		DRM_DEBUG_ATOMIC("Invalid pixel format %s\n", format_name);
-		kfree(format_name);
+		struct drm_format_name_buf format_name;
+		DRM_DEBUG_ATOMIC("Invalid pixel format %s\n",
+		                 drm_get_format_name(state->fb->pixel_format,
+		                                     &format_name));
 		return ret;
 	}
 
@@ -883,6 +948,39 @@ static int drm_atomic_plane_check(struct drm_plane *plane,
 	return 0;
 }
 
+static void drm_atomic_plane_print_state(struct drm_printer *p,
+		const struct drm_plane_state *state)
+{
+	struct drm_plane *plane = state->plane;
+	struct drm_rect src  = drm_plane_state_src(state);
+	struct drm_rect dest = drm_plane_state_dest(state);
+
+	drm_printf(p, "plane[%u]: %s\n", plane->base.id, plane->name);
+	drm_printf(p, "\tcrtc=%s\n", state->crtc ? state->crtc->name : "(null)");
+	drm_printf(p, "\tfb=%u\n", state->fb ? state->fb->base.id : 0);
+	if (state->fb) {
+		struct drm_framebuffer *fb = state->fb;
+		int i, n = drm_format_num_planes(fb->pixel_format);
+		struct drm_format_name_buf format_name;
+
+		drm_printf(p, "\t\tformat=%s\n",
+		              drm_get_format_name(fb->pixel_format, &format_name));
+		drm_printf(p, "\t\t\tmodifier=0x%llx\n", fb->modifier);
+		drm_printf(p, "\t\tsize=%dx%d\n", fb->width, fb->height);
+		drm_printf(p, "\t\tlayers:\n");
+		for (i = 0; i < n; i++) {
+			drm_printf(p, "\t\t\tpitch[%d]=%u\n", i, fb->pitches[i]);
+			drm_printf(p, "\t\t\toffset[%d]=%u\n", i, fb->offsets[i]);
+		}
+	}
+	drm_printf(p, "\tcrtc-pos=" DRM_RECT_FMT "\n", DRM_RECT_ARG(&dest));
+	drm_printf(p, "\tsrc-pos=" DRM_RECT_FP_FMT "\n", DRM_RECT_FP_ARG(&src));
+	drm_printf(p, "\trotation=%x\n", state->rotation);
+
+	if (plane->funcs->atomic_print_state)
+		plane->funcs->atomic_print_state(p, state);
+}
+
 /**
  * drm_atomic_get_connector_state - get connector state
  * @state: global atomic state object
@@ -989,15 +1087,53 @@ int drm_atomic_connector_set_property(struct drm_connector *connector,
 		 * now?) atomic writes to DPMS property:
 		 */
 		return -EINVAL;
+	} else if (property == config->tv_select_subconnector_property) {
+		state->tv.subconnector = val;
+	} else if (property == config->tv_left_margin_property) {
+		state->tv.margins.left = val;
+	} else if (property == config->tv_right_margin_property) {
+		state->tv.margins.right = val;
+	} else if (property == config->tv_top_margin_property) {
+		state->tv.margins.top = val;
+	} else if (property == config->tv_bottom_margin_property) {
+		state->tv.margins.bottom = val;
+	} else if (property == config->tv_mode_property) {
+		state->tv.mode = val;
+	} else if (property == config->tv_brightness_property) {
+		state->tv.brightness = val;
+	} else if (property == config->tv_contrast_property) {
+		state->tv.contrast = val;
+	} else if (property == config->tv_flicker_reduction_property) {
+		state->tv.flicker_reduction = val;
+	} else if (property == config->tv_overscan_property) {
+		state->tv.overscan = val;
+	} else if (property == config->tv_saturation_property) {
+		state->tv.saturation = val;
+	} else if (property == config->tv_hue_property) {
+		state->tv.hue = val;
 	} else if (connector->funcs->atomic_set_property) {
 		return connector->funcs->atomic_set_property(connector,
 				state, property, val);
 	} else {
 		return -EINVAL;
 	}
+
+	return 0;
 }
 EXPORT_SYMBOL(drm_atomic_connector_set_property);
 
+static void drm_atomic_connector_print_state(struct drm_printer *p,
+		const struct drm_connector_state *state)
+{
+	struct drm_connector *connector = state->connector;
+
+	drm_printf(p, "connector[%u]: %s\n", connector->base.id, connector->name);
+	drm_printf(p, "\tcrtc=%s\n", state->crtc ? state->crtc->name : "(null)");
+
+	if (connector->funcs->atomic_print_state)
+		connector->funcs->atomic_print_state(p, state);
+}
+
 /**
  * drm_atomic_connector_get_property - get property value from connector state
  * @connector: the drm connector to set a property on
@@ -1025,6 +1161,30 @@ drm_atomic_connector_get_property(struct drm_connector *connector,
 		*val = (state->crtc) ? state->crtc->base.id : 0;
 	} else if (property == config->dpms_property) {
 		*val = connector->dpms;
+	} else if (property == config->tv_select_subconnector_property) {
+		*val = state->tv.subconnector;
+	} else if (property == config->tv_left_margin_property) {
+		*val = state->tv.margins.left;
+	} else if (property == config->tv_right_margin_property) {
+		*val = state->tv.margins.right;
+	} else if (property == config->tv_top_margin_property) {
+		*val = state->tv.margins.top;
+	} else if (property == config->tv_bottom_margin_property) {
+		*val = state->tv.margins.bottom;
+	} else if (property == config->tv_mode_property) {
+		*val = state->tv.mode;
+	} else if (property == config->tv_brightness_property) {
+		*val = state->tv.brightness;
+	} else if (property == config->tv_contrast_property) {
+		*val = state->tv.contrast;
+	} else if (property == config->tv_flicker_reduction_property) {
+		*val = state->tv.flicker_reduction;
+	} else if (property == config->tv_overscan_property) {
+		*val = state->tv.overscan;
+	} else if (property == config->tv_saturation_property) {
+		*val = state->tv.saturation;
+	} else if (property == config->tv_hue_property) {
+		*val = state->tv.hue;
 	} else if (connector->funcs->atomic_get_property) {
 		return connector->funcs->atomic_get_property(connector,
 				state, property, val);
@@ -1136,22 +1296,48 @@ void
 drm_atomic_set_fb_for_plane(struct drm_plane_state *plane_state,
 			    struct drm_framebuffer *fb)
 {
-	if (plane_state->fb)
-		drm_framebuffer_unreference(plane_state->fb);
-	if (fb)
-		drm_framebuffer_reference(fb);
-	plane_state->fb = fb;
-
 	if (fb)
 		DRM_DEBUG_ATOMIC("Set [FB:%d] for plane state %p\n",
 				 fb->base.id, plane_state);
 	else
 		DRM_DEBUG_ATOMIC("Set [NOFB] for plane state %p\n",
 				 plane_state);
+
+	drm_framebuffer_assign(&plane_state->fb, fb);
 }
 EXPORT_SYMBOL(drm_atomic_set_fb_for_plane);
 
 /**
+ * drm_atomic_set_fence_for_plane - set fence for plane
+ * @plane_state: atomic state object for the plane
+ * @fence: dma_fence to use for the plane
+ *
+ * Helper to setup the plane_state fence in case it is not set yet.
+ * By using this drivers doesn't need to worry if the user choose
+ * implicit or explicit fencing.
+ *
+ * This function will not set the fence to the state if it was set
+ * via explicit fencing interfaces on the atomic ioctl. It will
+ * all drope the reference to the fence as we not storing it
+ * anywhere.
+ *
+ * Otherwise, if plane_state->fence is not set this function we
+ * just set it with the received implict fence.
+ */
+void
+drm_atomic_set_fence_for_plane(struct drm_plane_state *plane_state,
+			       struct dma_fence *fence)
+{
+	if (plane_state->fence) {
+		dma_fence_put(fence);
+		return;
+	}
+
+	plane_state->fence = fence;
+}
+EXPORT_SYMBOL(drm_atomic_set_fence_for_plane);
+
+/**
  * drm_atomic_set_crtc_for_connector - set crtc for connector
  * @conn_state: atomic state object for the connector
  * @crtc: crtc to use for the connector
@@ -1462,16 +1648,107 @@ int drm_atomic_nonblocking_commit(struct drm_atomic_state *state)
 }
 EXPORT_SYMBOL(drm_atomic_nonblocking_commit);
 
+static void drm_atomic_print_state(const struct drm_atomic_state *state)
+{
+	struct drm_printer p = drm_info_printer(state->dev->dev);
+	struct drm_plane *plane;
+	struct drm_plane_state *plane_state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *crtc_state;
+	struct drm_connector *connector;
+	struct drm_connector_state *connector_state;
+	int i;
+
+	DRM_DEBUG_ATOMIC("checking %p\n", state);
+
+	for_each_plane_in_state(state, plane, plane_state, i)
+		drm_atomic_plane_print_state(&p, plane_state);
+
+	for_each_crtc_in_state(state, crtc, crtc_state, i)
+		drm_atomic_crtc_print_state(&p, crtc_state);
+
+	for_each_connector_in_state(state, connector, connector_state, i)
+		drm_atomic_connector_print_state(&p, connector_state);
+}
+
+/**
+ * drm_state_dump - dump entire device atomic state
+ * @dev: the drm device
+ * @p: where to print the state to
+ *
+ * Just for debugging.  Drivers might want an option to dump state
+ * to dmesg in case of error irq's.  (Hint, you probably want to
+ * ratelimit this!)
+ *
+ * The caller must drm_modeset_lock_all(), or if this is called
+ * from error irq handler, it should not be enabled by default.
+ * (Ie. if you are debugging errors you might not care that this
+ * is racey.  But calling this without all modeset locks held is
+ * not inherently safe.)
+ */
+void drm_state_dump(struct drm_device *dev, struct drm_printer *p)
+{
+	struct drm_mode_config *config = &dev->mode_config;
+	struct drm_plane *plane;
+	struct drm_crtc *crtc;
+	struct drm_connector *connector;
+
+	if (!drm_core_check_feature(dev, DRIVER_ATOMIC))
+		return;
+
+	list_for_each_entry(plane, &config->plane_list, head)
+		drm_atomic_plane_print_state(p, plane->state);
+
+	list_for_each_entry(crtc, &config->crtc_list, head)
+		drm_atomic_crtc_print_state(p, crtc->state);
+
+	list_for_each_entry(connector, &config->connector_list, head)
+		drm_atomic_connector_print_state(p, connector->state);
+}
+EXPORT_SYMBOL(drm_state_dump);
+
+#ifdef CONFIG_DEBUG_FS
+static int drm_state_info(struct seq_file *m, void *data)
+{
+	struct drm_info_node *node = (struct drm_info_node *) m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct drm_printer p = drm_seq_file_printer(m);
+
+	drm_modeset_lock_all(dev);
+	drm_state_dump(dev, &p);
+	drm_modeset_unlock_all(dev);
+
+	return 0;
+}
+
+/* any use in debugfs files to dump individual planes/crtc/etc? */
+static const struct drm_info_list drm_atomic_debugfs_list[] = {
+	{"state", drm_state_info, 0},
+};
+
+int drm_atomic_debugfs_init(struct drm_minor *minor)
+{
+	return drm_debugfs_create_files(drm_atomic_debugfs_list,
+			ARRAY_SIZE(drm_atomic_debugfs_list),
+			minor->debugfs_root, minor);
+}
+
+int drm_atomic_debugfs_cleanup(struct drm_minor *minor)
+{
+	return drm_debugfs_remove_files(drm_atomic_debugfs_list,
+					ARRAY_SIZE(drm_atomic_debugfs_list),
+					minor);
+}
+#endif
+
 /*
  * The big monstor ioctl
  */
 
 static struct drm_pending_vblank_event *create_vblank_event(
-		struct drm_device *dev, struct drm_file *file_priv,
-		struct fence *fence, uint64_t user_data)
+		struct drm_device *dev, uint64_t user_data)
 {
 	struct drm_pending_vblank_event *e = NULL;
-	int ret;
 
 	e = kzalloc(sizeof *e, GFP_KERNEL);
 	if (!e)
@@ -1481,17 +1758,6 @@ static struct drm_pending_vblank_event *create_vblank_event(
 	e->event.base.length = sizeof(e->event);
 	e->event.user_data = user_data;
 
-	if (file_priv) {
-		ret = drm_event_reserve_init(dev, file_priv, &e->base,
-					     &e->event.base);
-		if (ret) {
-			kfree(e);
-			return NULL;
-		}
-	}
-
-	e->base.fence = fence;
-
 	return e;
 }
 
@@ -1596,6 +1862,203 @@ void drm_atomic_clean_old_fb(struct drm_device *dev,
 }
 EXPORT_SYMBOL(drm_atomic_clean_old_fb);
 
+/**
+ * DOC: explicit fencing properties
+ *
+ * Explicit fencing allows userspace to control the buffer synchronization
+ * between devices. A Fence or a group of fences are transfered to/from
+ * userspace using Sync File fds and there are two DRM properties for that.
+ * IN_FENCE_FD on each DRM Plane to send fences to the kernel and
+ * OUT_FENCE_PTR on each DRM CRTC to receive fences from the kernel.
+ *
+ * As a contrast, with implicit fencing the kernel keeps track of any
+ * ongoing rendering, and automatically ensures that the atomic update waits
+ * for any pending rendering to complete. For shared buffers represented with
+ * a struct &dma_buf this is tracked in &reservation_object structures.
+ * Implicit syncing is how Linux traditionally worked (e.g. DRI2/3 on X.org),
+ * whereas explicit fencing is what Android wants.
+ *
+ * "IN_FENCE_FD”:
+ *	Use this property to pass a fence that DRM should wait on before
+ *	proceeding with the Atomic Commit request and show the framebuffer for
+ *	the plane on the screen. The fence can be either a normal fence or a
+ *	merged one, the sync_file framework will handle both cases and use a
+ *	fence_array if a merged fence is received. Passing -1 here means no
+ *	fences to wait on.
+ *
+ *	If the Atomic Commit request has the DRM_MODE_ATOMIC_TEST_ONLY flag
+ *	it will only check if the Sync File is a valid one.
+ *
+ *	On the driver side the fence is stored on the @fence parameter of
+ *	struct &drm_plane_state. Drivers which also support implicit fencing
+ *	should set the implicit fence using drm_atomic_set_fence_for_plane(),
+ *	to make sure there's consistent behaviour between drivers in precedence
+ *	of implicit vs. explicit fencing.
+ *
+ * "OUT_FENCE_PTR”:
+ *	Use this property to pass a file descriptor pointer to DRM. Once the
+ *	Atomic Commit request call returns OUT_FENCE_PTR will be filled with
+ *	the file descriptor number of a Sync File. This Sync File contains the
+ *	CRTC fence that will be signaled when all framebuffers present on the
+ *	Atomic Commit * request for that given CRTC are scanned out on the
+ *	screen.
+ *
+ *	The Atomic Commit request fails if a invalid pointer is passed. If the
+ *	Atomic Commit request fails for any other reason the out fence fd
+ *	returned will be -1. On a Atomic Commit with the
+ *	DRM_MODE_ATOMIC_TEST_ONLY flag the out fence will also be set to -1.
+ *
+ *	Note that out-fences don't have a special interface to drivers and are
+ *	internally represented by a struct &drm_pending_vblank_event in struct
+ *	&drm_crtc_state, which is also used by the nonblocking atomic commit
+ *	helpers and for the DRM event handling for existing userspace.
+ */
+
+struct drm_out_fence_state {
+	s64 __user *out_fence_ptr;
+	struct sync_file *sync_file;
+	int fd;
+};
+
+static int setup_out_fence(struct drm_out_fence_state *fence_state,
+			   struct dma_fence *fence)
+{
+	fence_state->fd = get_unused_fd_flags(O_CLOEXEC);
+	if (fence_state->fd < 0)
+		return fence_state->fd;
+
+	if (put_user(fence_state->fd, fence_state->out_fence_ptr))
+		return -EFAULT;
+
+	fence_state->sync_file = sync_file_create(fence);
+	if (!fence_state->sync_file)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static int prepare_crtc_signaling(struct drm_device *dev,
+				  struct drm_atomic_state *state,
+				  struct drm_mode_atomic *arg,
+				  struct drm_file *file_priv,
+				  struct drm_out_fence_state **fence_state,
+				  unsigned int *num_fences)
+{
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *crtc_state;
+	int i, ret;
+
+	if (arg->flags & DRM_MODE_ATOMIC_TEST_ONLY)
+		return 0;
+
+	for_each_crtc_in_state(state, crtc, crtc_state, i) {
+		u64 __user *fence_ptr;
+
+		fence_ptr = get_out_fence_for_crtc(crtc_state->state, crtc);
+
+		if (arg->flags & DRM_MODE_PAGE_FLIP_EVENT || fence_ptr) {
+			struct drm_pending_vblank_event *e;
+
+			e = create_vblank_event(dev, arg->user_data);
+			if (!e)
+				return -ENOMEM;
+
+			crtc_state->event = e;
+		}
+
+		if (arg->flags & DRM_MODE_PAGE_FLIP_EVENT) {
+			struct drm_pending_vblank_event *e = crtc_state->event;
+
+			if (!file_priv)
+				continue;
+
+			ret = drm_event_reserve_init(dev, file_priv, &e->base,
+						     &e->event.base);
+			if (ret) {
+				kfree(e);
+				crtc_state->event = NULL;
+				return ret;
+			}
+		}
+
+		if (fence_ptr) {
+			struct dma_fence *fence;
+			struct drm_out_fence_state *f;
+
+			f = krealloc(*fence_state, sizeof(**fence_state) *
+				     (*num_fences + 1), GFP_KERNEL);
+			if (!f)
+				return -ENOMEM;
+
+			memset(&f[*num_fences], 0, sizeof(*f));
+
+			f[*num_fences].out_fence_ptr = fence_ptr;
+			*fence_state = f;
+
+			fence = drm_crtc_create_fence(crtc);
+			if (!fence)
+				return -ENOMEM;
+
+			ret = setup_out_fence(&f[(*num_fences)++], fence);
+			if (ret) {
+				dma_fence_put(fence);
+				return ret;
+			}
+
+			crtc_state->event->base.fence = fence;
+		}
+	}
+
+	return 0;
+}
+
+static void complete_crtc_signaling(struct drm_device *dev,
+				    struct drm_atomic_state *state,
+				    struct drm_out_fence_state *fence_state,
+				    unsigned int num_fences,
+				    bool install_fds)
+{
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *crtc_state;
+	int i;
+
+	if (install_fds) {
+		for (i = 0; i < num_fences; i++)
+			fd_install(fence_state[i].fd,
+				   fence_state[i].sync_file->file);
+
+		kfree(fence_state);
+		return;
+	}
+
+	for_each_crtc_in_state(state, crtc, crtc_state, i) {
+		/*
+		 * TEST_ONLY and PAGE_FLIP_EVENT are mutually
+		 * exclusive, if they weren't, this code should be
+		 * called on success for TEST_ONLY too.
+		 */
+		if (crtc_state->event)
+			drm_event_cancel_free(dev, &crtc_state->event->base);
+	}
+
+	if (!fence_state)
+		return;
+
+	for (i = 0; i < num_fences; i++) {
+		if (fence_state[i].sync_file)
+			fput(fence_state[i].sync_file->file);
+		if (fence_state[i].fd >= 0)
+			put_unused_fd(fence_state[i].fd);
+
+		/* If this fails log error to the user */
+		if (fence_state[i].out_fence_ptr &&
+		    put_user(-1, fence_state[i].out_fence_ptr))
+			DRM_DEBUG_ATOMIC("Couldn't clear out_fence_ptr\n");
+	}
+
+	kfree(fence_state);
+}
+
 int drm_mode_atomic_ioctl(struct drm_device *dev,
 			  void *data, struct drm_file *file_priv)
 {
@@ -1608,11 +2071,10 @@ int drm_mode_atomic_ioctl(struct drm_device *dev,
 	struct drm_atomic_state *state;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_plane *plane;
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *crtc_state;
+	struct drm_out_fence_state *fence_state = NULL;
 	unsigned plane_mask;
 	int ret = 0;
-	unsigned int i, j;
+	unsigned int i, j, num_fences = 0;
 
 	/* disallow for drivers not supporting atomic: */
 	if (!drm_core_check_feature(dev, DRIVER_ATOMIC))
@@ -1727,50 +2189,30 @@ retry:
 		drm_mode_object_unreference(obj);
 	}
 
-	if (arg->flags & DRM_MODE_PAGE_FLIP_EVENT) {
-		for_each_crtc_in_state(state, crtc, crtc_state, i) {
-			struct drm_pending_vblank_event *e;
-
-			e = create_vblank_event(dev, file_priv, NULL,
-						arg->user_data);
-			if (!e) {
-				ret = -ENOMEM;
-				goto out;
-			}
-
-			crtc_state->event = e;
-		}
-	}
+	ret = prepare_crtc_signaling(dev, state, arg, file_priv, &fence_state,
+				     &num_fences);
+	if (ret)
+		goto out;
 
 	if (arg->flags & DRM_MODE_ATOMIC_TEST_ONLY) {
 		/*
 		 * Unlike commit, check_only does not clean up state.
-		 * Below we call drm_atomic_state_free for it.
+		 * Below we call drm_atomic_state_put for it.
 		 */
 		ret = drm_atomic_check_only(state);
 	} else if (arg->flags & DRM_MODE_ATOMIC_NONBLOCK) {
 		ret = drm_atomic_nonblocking_commit(state);
 	} else {
+		if (unlikely(drm_debug & DRM_UT_STATE))
+			drm_atomic_print_state(state);
+
 		ret = drm_atomic_commit(state);
 	}
 
 out:
 	drm_atomic_clean_old_fb(dev, plane_mask, ret);
 
-	if (ret && arg->flags & DRM_MODE_PAGE_FLIP_EVENT) {
-		/*
-		 * TEST_ONLY and PAGE_FLIP_EVENT are mutually exclusive,
-		 * if they weren't, this code should be called on success
-		 * for TEST_ONLY too.
-		 */
-
-		for_each_crtc_in_state(state, crtc, crtc_state, i) {
-			if (!crtc_state->event)
-				continue;
-
-			drm_event_cancel_free(dev, &crtc_state->event->base);
-		}
-	}
+	complete_crtc_signaling(dev, state, fence_state, num_fences, !ret);
 
 	if (ret == -EDEADLK) {
 		drm_atomic_state_clear(state);
@@ -1778,8 +2220,7 @@ out:
 		goto retry;
 	}
 
-	if (ret || arg->flags & DRM_MODE_ATOMIC_TEST_ONLY)
-		drm_atomic_state_free(state);
+	drm_atomic_state_put(state);
 
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index 21f992605541..583f47f27b36 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -30,7 +30,7 @@
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_atomic_helper.h>
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 
 #include "drm_crtc_internal.h"
 
@@ -458,10 +458,11 @@ mode_fixup(struct drm_atomic_state *state)
  * removed from the crtc.
  * crtc_state->active_changed is set when crtc_state->active changes,
  * which is used for dpms.
+ * See also: drm_atomic_crtc_needs_modeset()
  *
  * IMPORTANT:
  *
- * Drivers which update ->mode_changed (e.g. in their ->atomic_check hooks if a
+ * Drivers which set ->mode_changed (e.g. in their ->atomic_check hooks if a
  * plane update can't be done without a full modeset) _must_ call this function
  * afterwards after that change. It is permitted to call this function multiple
  * times for the same update, e.g. when the ->atomic_check functions depend upon
@@ -510,9 +511,9 @@ drm_atomic_helper_check_modeset(struct drm_device *dev,
 
 	for_each_connector_in_state(state, connector, connector_state, i) {
 		/*
-		 * This only sets crtc->mode_changed for routing changes,
-		 * drivers must set crtc->mode_changed themselves when connector
-		 * properties need to be updated.
+		 * This only sets crtc->connectors_changed for routing changes,
+		 * drivers must set crtc->connectors_changed themselves when
+		 * connector properties need to be updated.
 		 */
 		ret = update_connector_routing(state, connector,
 					       connector_state);
@@ -1005,14 +1006,22 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_modeset_enables);
  * drm_atomic_helper_wait_for_fences - wait for fences stashed in plane state
  * @dev: DRM device
  * @state: atomic state object with old state structures
- * @pre_swap: if true, do an interruptible wait
+ * @pre_swap: If true, do an interruptible wait, and @state is the new state.
+ * 	Otherwise @state is the old state.
  *
  * For implicit sync, driver should fish the exclusive fence out from the
  * incoming fb's and stash it in the drm_plane_state.  This is called after
  * drm_atomic_helper_swap_state() so it uses the current plane state (and
  * just uses the atomic state to find the changed planes)
  *
- * Returns zero if success or < 0 if fence_wait() fails.
+ * Note that @pre_swap is needed since the point where we block for fences moves
+ * around depending upon whether an atomic commit is blocking or
+ * non-blocking. For async commit all waiting needs to happen after
+ * drm_atomic_helper_swap_state() is called, but for synchronous commits we want
+ * to wait **before** we do anything that can't be easily rolled back. That is
+ * before we call drm_atomic_helper_swap_state().
+ *
+ * Returns zero if success or < 0 if dma_fence_wait() fails.
  */
 int drm_atomic_helper_wait_for_fences(struct drm_device *dev,
 				      struct drm_atomic_state *state,
@@ -1036,11 +1045,11 @@ int drm_atomic_helper_wait_for_fences(struct drm_device *dev,
 		 * still interrupt the operation. Instead of blocking until the
 		 * timer expires, make the wait interruptible.
 		 */
-		ret = fence_wait(plane_state->fence, pre_swap);
+		ret = dma_fence_wait(plane_state->fence, pre_swap);
 		if (ret)
 			return ret;
 
-		fence_put(plane_state->fence);
+		dma_fence_put(plane_state->fence);
 		plane_state->fence = NULL;
 	}
 
@@ -1146,7 +1155,7 @@ EXPORT_SYMBOL(drm_atomic_helper_wait_for_vblanks);
 
 /**
  * drm_atomic_helper_commit_tail - commit atomic update to hardware
- * @state: new modeset state to be committed
+ * @old_state: atomic state object with old state structures
  *
  * This is the default implemenation for the ->atomic_commit_tail() hook of the
  * &drm_mode_config_helper_funcs vtable.
@@ -1157,53 +1166,53 @@ EXPORT_SYMBOL(drm_atomic_helper_wait_for_vblanks);
  *
  * For drivers supporting runtime PM the recommended sequence is instead ::
  *
- *     drm_atomic_helper_commit_modeset_disables(dev, state);
+ *     drm_atomic_helper_commit_modeset_disables(dev, old_state);
  *
- *     drm_atomic_helper_commit_modeset_enables(dev, state);
+ *     drm_atomic_helper_commit_modeset_enables(dev, old_state);
  *
- *     drm_atomic_helper_commit_planes(dev, state,
+ *     drm_atomic_helper_commit_planes(dev, old_state,
  *                                     DRM_PLANE_COMMIT_ACTIVE_ONLY);
  *
  * for committing the atomic update to hardware.  See the kerneldoc entries for
  * these three functions for more details.
  */
-void drm_atomic_helper_commit_tail(struct drm_atomic_state *state)
+void drm_atomic_helper_commit_tail(struct drm_atomic_state *old_state)
 {
-	struct drm_device *dev = state->dev;
+	struct drm_device *dev = old_state->dev;
 
-	drm_atomic_helper_commit_modeset_disables(dev, state);
+	drm_atomic_helper_commit_modeset_disables(dev, old_state);
 
-	drm_atomic_helper_commit_planes(dev, state, 0);
+	drm_atomic_helper_commit_planes(dev, old_state, 0);
 
-	drm_atomic_helper_commit_modeset_enables(dev, state);
+	drm_atomic_helper_commit_modeset_enables(dev, old_state);
 
-	drm_atomic_helper_commit_hw_done(state);
+	drm_atomic_helper_commit_hw_done(old_state);
 
-	drm_atomic_helper_wait_for_vblanks(dev, state);
+	drm_atomic_helper_wait_for_vblanks(dev, old_state);
 
-	drm_atomic_helper_cleanup_planes(dev, state);
+	drm_atomic_helper_cleanup_planes(dev, old_state);
 }
 EXPORT_SYMBOL(drm_atomic_helper_commit_tail);
 
-static void commit_tail(struct drm_atomic_state *state)
+static void commit_tail(struct drm_atomic_state *old_state)
 {
-	struct drm_device *dev = state->dev;
+	struct drm_device *dev = old_state->dev;
 	struct drm_mode_config_helper_funcs *funcs;
 
 	funcs = dev->mode_config.helper_private;
 
-	drm_atomic_helper_wait_for_fences(dev, state, false);
+	drm_atomic_helper_wait_for_fences(dev, old_state, false);
 
-	drm_atomic_helper_wait_for_dependencies(state);
+	drm_atomic_helper_wait_for_dependencies(old_state);
 
 	if (funcs && funcs->atomic_commit_tail)
-		funcs->atomic_commit_tail(state);
+		funcs->atomic_commit_tail(old_state);
 	else
-		drm_atomic_helper_commit_tail(state);
+		drm_atomic_helper_commit_tail(old_state);
 
-	drm_atomic_helper_commit_cleanup_done(state);
+	drm_atomic_helper_commit_cleanup_done(old_state);
 
-	drm_atomic_state_free(state);
+	drm_atomic_state_put(old_state);
 }
 
 static void commit_work(struct work_struct *work)
@@ -1225,9 +1234,6 @@ static void commit_work(struct work_struct *work)
  * function implements nonblocking commits, using
  * drm_atomic_helper_setup_commit() and related functions.
  *
- * Note that right now this function does not support nonblocking commits, hence
- * driver writers must implement their own version for now.
- *
  * Committing the actual hardware state is done through the
  * ->atomic_commit_tail() callback of the &drm_mode_config_helper_funcs vtable,
  * or it's default implementation drm_atomic_helper_commit_tail().
@@ -1285,6 +1291,7 @@ int drm_atomic_helper_commit(struct drm_device *dev,
 	 * make sure work items don't artifically stall on each another.
 	 */
 
+	drm_atomic_state_get(state);
 	if (nonblock)
 		queue_work(system_unbound_wq, &state->commit_work);
 	else
@@ -1496,10 +1503,10 @@ static struct drm_crtc_commit *preceeding_commit(struct drm_crtc *crtc)
 
 /**
  * drm_atomic_helper_wait_for_dependencies - wait for required preceeding commits
- * @state: new modeset state to be committed
+ * @old_state: atomic state object with old state structures
  *
  * This function waits for all preceeding commits that touch the same CRTC as
- * @state to both be committed to the hardware (as signalled by
+ * @old_state to both be committed to the hardware (as signalled by
  * drm_atomic_helper_commit_hw_done) and executed by the hardware (as signalled
  * by calling drm_crtc_vblank_send_event on the event member of
  * &drm_crtc_state).
@@ -1507,7 +1514,7 @@ static struct drm_crtc_commit *preceeding_commit(struct drm_crtc *crtc)
  * This is part of the atomic helper support for nonblocking commits, see
  * drm_atomic_helper_setup_commit() for an overview.
  */
-void drm_atomic_helper_wait_for_dependencies(struct drm_atomic_state *state)
+void drm_atomic_helper_wait_for_dependencies(struct drm_atomic_state *old_state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
@@ -1515,7 +1522,7 @@ void drm_atomic_helper_wait_for_dependencies(struct drm_atomic_state *state)
 	int i;
 	long ret;
 
-	for_each_crtc_in_state(state, crtc, crtc_state, i) {
+	for_each_crtc_in_state(old_state, crtc, crtc_state, i) {
 		spin_lock(&crtc->commit_lock);
 		commit = preceeding_commit(crtc);
 		if (commit)
@@ -1546,7 +1553,7 @@ EXPORT_SYMBOL(drm_atomic_helper_wait_for_dependencies);
 
 /**
  * drm_atomic_helper_commit_hw_done - setup possible nonblocking commit
- * @state: new modeset state to be committed
+ * @old_state: atomic state object with old state structures
  *
  * This function is used to signal completion of the hardware commit step. After
  * this step the driver is not allowed to read or change any permanent software
@@ -1559,15 +1566,15 @@ EXPORT_SYMBOL(drm_atomic_helper_wait_for_dependencies);
  * This is part of the atomic helper support for nonblocking commits, see
  * drm_atomic_helper_setup_commit() for an overview.
  */
-void drm_atomic_helper_commit_hw_done(struct drm_atomic_state *state)
+void drm_atomic_helper_commit_hw_done(struct drm_atomic_state *old_state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
 	struct drm_crtc_commit *commit;
 	int i;
 
-	for_each_crtc_in_state(state, crtc, crtc_state, i) {
-		commit = state->crtcs[i].commit;
+	for_each_crtc_in_state(old_state, crtc, crtc_state, i) {
+		commit = old_state->crtcs[i].commit;
 		if (!commit)
 			continue;
 
@@ -1582,16 +1589,16 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_hw_done);
 
 /**
  * drm_atomic_helper_commit_cleanup_done - signal completion of commit
- * @state: new modeset state to be committed
+ * @old_state: atomic state object with old state structures
  *
- * This signals completion of the atomic update @state, including any cleanup
- * work. If used, it must be called right before calling
- * drm_atomic_state_free().
+ * This signals completion of the atomic update @old_state, including any
+ * cleanup work. If used, it must be called right before calling
+ * drm_atomic_state_put().
  *
  * This is part of the atomic helper support for nonblocking commits, see
  * drm_atomic_helper_setup_commit() for an overview.
  */
-void drm_atomic_helper_commit_cleanup_done(struct drm_atomic_state *state)
+void drm_atomic_helper_commit_cleanup_done(struct drm_atomic_state *old_state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
@@ -1599,8 +1606,8 @@ void drm_atomic_helper_commit_cleanup_done(struct drm_atomic_state *state)
 	int i;
 	long ret;
 
-	for_each_crtc_in_state(state, crtc, crtc_state, i) {
-		commit = state->crtcs[i].commit;
+	for_each_crtc_in_state(old_state, crtc, crtc_state, i) {
+		commit = old_state->crtcs[i].commit;
 		if (WARN_ON(!commit))
 			continue;
 
@@ -2109,18 +2116,13 @@ retry:
 		state->legacy_cursor_update = true;
 
 	ret = drm_atomic_commit(state);
-	if (ret != 0)
-		goto fail;
-
-	/* Driver takes ownership of state on successful commit. */
-	return 0;
 fail:
 	if (ret == -EDEADLK)
 		goto backoff;
 
-	drm_atomic_state_free(state);
-
+	drm_atomic_state_put(state);
 	return ret;
+
 backoff:
 	drm_atomic_state_clear(state);
 	drm_atomic_legacy_backoff(state);
@@ -2182,18 +2184,13 @@ retry:
 		goto fail;
 
 	ret = drm_atomic_commit(state);
-	if (ret != 0)
-		goto fail;
-
-	/* Driver takes ownership of state on successful commit. */
-	return 0;
 fail:
 	if (ret == -EDEADLK)
 		goto backoff;
 
-	drm_atomic_state_free(state);
-
+	drm_atomic_state_put(state);
 	return ret;
+
 backoff:
 	drm_atomic_state_clear(state);
 	drm_atomic_legacy_backoff(state);
@@ -2322,18 +2319,13 @@ retry:
 		goto fail;
 
 	ret = drm_atomic_commit(state);
-	if (ret != 0)
-		goto fail;
-
-	/* Driver takes ownership of state on successful commit. */
-	return 0;
 fail:
 	if (ret == -EDEADLK)
 		goto backoff;
 
-	drm_atomic_state_free(state);
-
+	drm_atomic_state_put(state);
 	return ret;
+
 backoff:
 	drm_atomic_state_clear(state);
 	drm_atomic_legacy_backoff(state);
@@ -2408,7 +2400,7 @@ int __drm_atomic_helper_set_config(struct drm_mode_set *set,
 	primary_state->crtc_h = vdisplay;
 	primary_state->src_x = set->x << 16;
 	primary_state->src_y = set->y << 16;
-	if (primary_state->rotation & (DRM_ROTATE_90 | DRM_ROTATE_270)) {
+	if (drm_rotation_90_or_270(primary_state->rotation)) {
 		primary_state->src_w = vdisplay << 16;
 		primary_state->src_h = hdisplay << 16;
 	} else {
@@ -2475,11 +2467,8 @@ int drm_atomic_helper_disable_all(struct drm_device *dev,
 	}
 
 	err = drm_atomic_commit(state);
-
 free:
-	if (err < 0)
-		drm_atomic_state_free(state);
-
+	drm_atomic_state_put(state);
 	return err;
 }
 EXPORT_SYMBOL(drm_atomic_helper_disable_all);
@@ -2530,7 +2519,7 @@ retry:
 
 	err = drm_atomic_helper_disable_all(dev, &ctx);
 	if (err < 0) {
-		drm_atomic_state_free(state);
+		drm_atomic_state_put(state);
 		state = ERR_PTR(err);
 		goto unlock;
 	}
@@ -2619,18 +2608,13 @@ retry:
 		goto fail;
 
 	ret = drm_atomic_commit(state);
-	if (ret != 0)
-		goto fail;
-
-	/* Driver takes ownership of state on successful commit. */
-	return 0;
 fail:
 	if (ret == -EDEADLK)
 		goto backoff;
 
-	drm_atomic_state_free(state);
-
+	drm_atomic_state_put(state);
 	return ret;
+
 backoff:
 	drm_atomic_state_clear(state);
 	drm_atomic_legacy_backoff(state);
@@ -2679,18 +2663,13 @@ retry:
 		goto fail;
 
 	ret = drm_atomic_commit(state);
-	if (ret != 0)
-		goto fail;
-
-	/* Driver takes ownership of state on successful commit. */
-	return 0;
 fail:
 	if (ret == -EDEADLK)
 		goto backoff;
 
-	drm_atomic_state_free(state);
-
+	drm_atomic_state_put(state);
 	return ret;
+
 backoff:
 	drm_atomic_state_clear(state);
 	drm_atomic_legacy_backoff(state);
@@ -2739,18 +2718,13 @@ retry:
 		goto fail;
 
 	ret = drm_atomic_commit(state);
-	if (ret != 0)
-		goto fail;
-
-	/* Driver takes ownership of state on successful commit. */
-	return 0;
 fail:
 	if (ret == -EDEADLK)
 		goto backoff;
 
-	drm_atomic_state_free(state);
-
+	drm_atomic_state_put(state);
 	return ret;
+
 backoff:
 	drm_atomic_state_clear(state);
 	drm_atomic_legacy_backoff(state);
@@ -2823,18 +2797,13 @@ retry:
 	}
 
 	ret = drm_atomic_nonblocking_commit(state);
-	if (ret != 0)
-		goto fail;
-
-	/* Driver takes ownership of state on successful commit. */
-	return 0;
 fail:
 	if (ret == -EDEADLK)
 		goto backoff;
 
-	drm_atomic_state_free(state);
-
+	drm_atomic_state_put(state);
 	return ret;
+
 backoff:
 	drm_atomic_state_clear(state);
 	drm_atomic_legacy_backoff(state);
@@ -2910,19 +2879,14 @@ retry:
 	crtc_state->active = active;
 
 	ret = drm_atomic_commit(state);
-	if (ret != 0)
-		goto fail;
-
-	/* Driver takes ownership of state on successful commit. */
-	return 0;
 fail:
 	if (ret == -EDEADLK)
 		goto backoff;
-
-	connector->dpms = old_mode;
-	drm_atomic_state_free(state);
-
+	if (ret != 0)
+		connector->dpms = old_mode;
+	drm_atomic_state_put(state);
 	return ret;
+
 backoff:
 	drm_atomic_state_clear(state);
 	drm_atomic_legacy_backoff(state);
@@ -3113,6 +3077,8 @@ void __drm_atomic_helper_plane_duplicate_state(struct drm_plane *plane,
 
 	if (state->fb)
 		drm_framebuffer_reference(state->fb);
+
+	state->fence = NULL;
 }
 EXPORT_SYMBOL(__drm_atomic_helper_plane_duplicate_state);
 
@@ -3151,6 +3117,9 @@ void __drm_atomic_helper_plane_destroy_state(struct drm_plane_state *state)
 {
 	if (state->fb)
 		drm_framebuffer_unreference(state->fb);
+
+	if (state->fence)
+		dma_fence_put(state->fence);
 }
 EXPORT_SYMBOL(__drm_atomic_helper_plane_destroy_state);
 
@@ -3329,7 +3298,7 @@ drm_atomic_helper_duplicate_state(struct drm_device *dev,
 
 free:
 	if (err < 0) {
-		drm_atomic_state_free(state);
+		drm_atomic_state_put(state);
 		state = ERR_PTR(err);
 	}
 
@@ -3444,22 +3413,14 @@ retry:
 		goto fail;
 
 	ret = drm_atomic_commit(state);
-	if (ret)
-		goto fail;
-
-	/* Driver takes ownership of state on successful commit. */
-
-	drm_property_unreference_blob(blob);
-
-	return 0;
 fail:
 	if (ret == -EDEADLK)
 		goto backoff;
 
-	drm_atomic_state_free(state);
+	drm_atomic_state_put(state);
 	drm_property_unreference_blob(blob);
-
 	return ret;
+
 backoff:
 	drm_atomic_state_clear(state);
 	drm_atomic_legacy_backoff(state);
diff --git a/drivers/gpu/drm/drm_blend.c b/drivers/gpu/drm/drm_blend.c
index 85172a977bf3..1f2412c7ccfd 100644
--- a/drivers/gpu/drm/drm_blend.c
+++ b/drivers/gpu/drm/drm_blend.c
@@ -89,7 +89,7 @@
  * On top of this basic transformation additional properties can be exposed by
  * the driver:
  *
- * - Rotation is set up with drm_mode_create_rotation_property(). It adds a
+ * - Rotation is set up with drm_plane_create_rotation_property(). It adds a
  *   rotation and reflection step between the source and destination rectangles.
  *   Without this property the rectangle is only scaled, but not rotated or
  *   reflected.
@@ -105,18 +105,12 @@
  */
 
 /**
- * drm_mode_create_rotation_property - create a new rotation property
- * @dev: DRM device
+ * drm_plane_create_rotation_property - create a new rotation property
+ * @plane: drm plane
+ * @rotation: initial value of the rotation property
  * @supported_rotations: bitmask of supported rotations and reflections
  *
  * This creates a new property with the selected support for transformations.
- * The resulting property should be stored in @rotation_property in
- * &drm_mode_config. It then must be attached to each plane which supports
- * rotations using drm_object_attach_property().
- *
- * FIXME: Probably better if the rotation property is created on each plane,
- * like the zpos property. Otherwise it's not possible to allow different
- * rotation modes on different planes.
  *
  * Since a rotation by 180° degress is the same as reflecting both along the x
  * and the y axis the rotation property is somewhat redundant. Drivers can use
@@ -144,8 +138,9 @@
  * rotation. After reflection, the rotation is applied to the image sampled from
  * the source rectangle, before scaling it to fit the destination rectangle.
  */
-struct drm_property *drm_mode_create_rotation_property(struct drm_device *dev,
-						       unsigned int supported_rotations)
+int drm_plane_create_rotation_property(struct drm_plane *plane,
+				       unsigned int rotation,
+				       unsigned int supported_rotations)
 {
 	static const struct drm_prop_enum_list props[] = {
 		{ __builtin_ffs(DRM_ROTATE_0) - 1,   "rotate-0" },
@@ -155,12 +150,28 @@ struct drm_property *drm_mode_create_rotation_property(struct drm_device *dev,
 		{ __builtin_ffs(DRM_REFLECT_X) - 1,  "reflect-x" },
 		{ __builtin_ffs(DRM_REFLECT_Y) - 1,  "reflect-y" },
 	};
+	struct drm_property *prop;
+
+	WARN_ON((supported_rotations & DRM_ROTATE_MASK) == 0);
+	WARN_ON(!is_power_of_2(rotation & DRM_ROTATE_MASK));
+	WARN_ON(rotation & ~supported_rotations);
 
-	return drm_property_create_bitmask(dev, 0, "rotation",
+	prop = drm_property_create_bitmask(plane->dev, 0, "rotation",
 					   props, ARRAY_SIZE(props),
 					   supported_rotations);
+	if (!prop)
+		return -ENOMEM;
+
+	drm_object_attach_property(&plane->base, prop, rotation);
+
+	if (plane->state)
+		plane->state->rotation = rotation;
+
+	plane->rotation_property = prop;
+
+	return 0;
 }
-EXPORT_SYMBOL(drm_mode_create_rotation_property);
+EXPORT_SYMBOL(drm_plane_create_rotation_property);
 
 /**
  * drm_rotation_simplify() - Try to simplify the rotation
diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c
index d28ffdd2b929..6543ebde501a 100644
--- a/drivers/gpu/drm/drm_color_mgmt.c
+++ b/drivers/gpu/drm/drm_color_mgmt.c
@@ -41,6 +41,10 @@
  *	nor use all the elements of the LUT (for example the hardware might
  *	choose to interpolate between LUT[0] and LUT[4]).
  *
+ *	Setting this to NULL (blob property value set to 0) means a
+ *	linear/pass-thru gamma table should be used. This is generally the
+ *	driver boot-up state too.
+ *
  * “DEGAMMA_LUT_SIZE”:
  *	Unsinged range property to give the size of the lookup table to be set
  *	on the DEGAMMA_LUT property (the size depends on the underlying
@@ -54,6 +58,10 @@
  *	lookup through the gamma LUT. The data is interpreted as a struct
  *	&drm_color_ctm.
  *
+ *	Setting this to NULL (blob property value set to 0) means a
+ *	unit/pass-thru matrix should be used. This is generally the driver
+ *	boot-up state too.
+ *
  * “GAMMA_LUT”:
  *	Blob property to set the gamma lookup table (LUT) mapping pixel data
  *	after the transformation matrix to data sent to the connector. The
@@ -62,6 +70,10 @@
  *	nor use all the elements of the LUT (for example the hardware might
  *	choose to interpolate between LUT[0] and LUT[4]).
  *
+ *	Setting this to NULL (blob property value set to 0) means a
+ *	linear/pass-thru gamma table should be used. This is generally the
+ *	driver boot-up state too.
+ *
  * “GAMMA_LUT_SIZE”:
  *	Unsigned range property to give the size of the lookup table to be set
  *	on the GAMMA_LUT property (the size depends on the underlying hardware).
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index 2db7fb510b6c..5a4526289392 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -588,6 +588,50 @@ static const struct drm_prop_enum_list drm_tv_subconnector_enum_list[] = {
 DRM_ENUM_NAME_FN(drm_get_tv_subconnector_name,
 		 drm_tv_subconnector_enum_list)
 
+/**
+ * DOC: standard connector properties
+ *
+ * DRM connectors have a few standardized properties:
+ *
+ * EDID:
+ * 	Blob property which contains the current EDID read from the sink. This
+ * 	is useful to parse sink identification information like vendor, model
+ * 	and serial. Drivers should update this property by calling
+ * 	drm_mode_connector_update_edid_property(), usually after having parsed
+ * 	the EDID using drm_add_edid_modes(). Userspace cannot change this
+ * 	property.
+ * DPMS:
+ * 	Legacy property for setting the power state of the connector. For atomic
+ * 	drivers this is only provided for backwards compatibility with existing
+ * 	drivers, it remaps to controlling the "ACTIVE" property on the CRTC the
+ * 	connector is linked to. Drivers should never set this property directly,
+ * 	it is handled by the DRM core by calling the ->dpms() callback in
+ * 	&drm_connector_funcs. Atomic drivers should implement this hook using
+ * 	drm_atomic_helper_connector_dpms(). This is the only property standard
+ * 	connector property that userspace can change.
+ * PATH:
+ * 	Connector path property to identify how this sink is physically
+ * 	connected. Used by DP MST. This should be set by calling
+ * 	drm_mode_connector_set_path_property(), in the case of DP MST with the
+ * 	path property the MST manager created. Userspace cannot change this
+ * 	property.
+ * TILE:
+ * 	Connector tile group property to indicate how a set of DRM connector
+ * 	compose together into one logical screen. This is used by both high-res
+ * 	external screens (often only using a single cable, but exposing multiple
+ * 	DP MST sinks), or high-res integrated panels (like dual-link DSI) which
+ * 	are not gen-locked. Note that for tiled panels which are genlocked, like
+ * 	dual-link LVDS or dual-link DSI, the driver should try to not expose the
+ * 	tiling and virtualize both &drm_crtc and &drm_plane if needed. Drivers
+ * 	should update this value using drm_mode_connector_set_tile_property().
+ * 	Userspace cannot change this property.
+ *
+ * Connectors also have one standardized atomic property:
+ *
+ * CRTC_ID:
+ * 	Mode object ID of the &drm_crtc this connector should be connected to.
+ */
+
 int drm_connector_create_standard_properties(struct drm_device *dev)
 {
 	struct drm_property *prop;
@@ -1121,3 +1165,107 @@ out_unlock:
 	return ret;
 }
 
+
+/**
+ * DOC: Tile group
+ *
+ * Tile groups are used to represent tiled monitors with a unique integer
+ * identifier. Tiled monitors using DisplayID v1.3 have a unique 8-byte handle,
+ * we store this in a tile group, so we have a common identifier for all tiles
+ * in a monitor group. The property is called "TILE". Drivers can manage tile
+ * groups using drm_mode_create_tile_group(), drm_mode_put_tile_group() and
+ * drm_mode_get_tile_group(). But this is only needed for internal panels where
+ * the tile group information is exposed through a non-standard way.
+ */
+
+static void drm_tile_group_free(struct kref *kref)
+{
+	struct drm_tile_group *tg = container_of(kref, struct drm_tile_group, refcount);
+	struct drm_device *dev = tg->dev;
+	mutex_lock(&dev->mode_config.idr_mutex);
+	idr_remove(&dev->mode_config.tile_idr, tg->id);
+	mutex_unlock(&dev->mode_config.idr_mutex);
+	kfree(tg);
+}
+
+/**
+ * drm_mode_put_tile_group - drop a reference to a tile group.
+ * @dev: DRM device
+ * @tg: tile group to drop reference to.
+ *
+ * drop reference to tile group and free if 0.
+ */
+void drm_mode_put_tile_group(struct drm_device *dev,
+			     struct drm_tile_group *tg)
+{
+	kref_put(&tg->refcount, drm_tile_group_free);
+}
+EXPORT_SYMBOL(drm_mode_put_tile_group);
+
+/**
+ * drm_mode_get_tile_group - get a reference to an existing tile group
+ * @dev: DRM device
+ * @topology: 8-bytes unique per monitor.
+ *
+ * Use the unique bytes to get a reference to an existing tile group.
+ *
+ * RETURNS:
+ * tile group or NULL if not found.
+ */
+struct drm_tile_group *drm_mode_get_tile_group(struct drm_device *dev,
+					       char topology[8])
+{
+	struct drm_tile_group *tg;
+	int id;
+	mutex_lock(&dev->mode_config.idr_mutex);
+	idr_for_each_entry(&dev->mode_config.tile_idr, tg, id) {
+		if (!memcmp(tg->group_data, topology, 8)) {
+			if (!kref_get_unless_zero(&tg->refcount))
+				tg = NULL;
+			mutex_unlock(&dev->mode_config.idr_mutex);
+			return tg;
+		}
+	}
+	mutex_unlock(&dev->mode_config.idr_mutex);
+	return NULL;
+}
+EXPORT_SYMBOL(drm_mode_get_tile_group);
+
+/**
+ * drm_mode_create_tile_group - create a tile group from a displayid description
+ * @dev: DRM device
+ * @topology: 8-bytes unique per monitor.
+ *
+ * Create a tile group for the unique monitor, and get a unique
+ * identifier for the tile group.
+ *
+ * RETURNS:
+ * new tile group or error.
+ */
+struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev,
+						  char topology[8])
+{
+	struct drm_tile_group *tg;
+	int ret;
+
+	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
+	if (!tg)
+		return ERR_PTR(-ENOMEM);
+
+	kref_init(&tg->refcount);
+	memcpy(tg->group_data, topology, 8);
+	tg->dev = dev;
+
+	mutex_lock(&dev->mode_config.idr_mutex);
+	ret = idr_alloc(&dev->mode_config.tile_idr, tg, 1, 0, GFP_KERNEL);
+	if (ret >= 0) {
+		tg->id = ret;
+	} else {
+		kfree(tg);
+		tg = ERR_PTR(ret);
+	}
+
+	mutex_unlock(&dev->mode_config.idr_mutex);
+	return tg;
+}
+EXPORT_SYMBOL(drm_mode_create_tile_group);
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 2d7bedf28647..e75f62cd8a65 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -33,6 +33,7 @@
 #include <linux/list.h>
 #include <linux/slab.h>
 #include <linux/export.h>
+#include <linux/dma-fence.h>
 #include <drm/drmP.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
@@ -40,23 +41,11 @@
 #include <drm/drm_modeset_lock.h>
 #include <drm/drm_atomic.h>
 #include <drm/drm_auth.h>
-#include <drm/drm_framebuffer.h>
+#include <drm/drm_debugfs_crc.h>
 
 #include "drm_crtc_internal.h"
 #include "drm_internal.h"
 
-/*
- * Global properties
- */
-static const struct drm_prop_enum_list drm_plane_type_enum_list[] = {
-	{ DRM_PLANE_TYPE_OVERLAY, "Overlay" },
-	{ DRM_PLANE_TYPE_PRIMARY, "Primary" },
-	{ DRM_PLANE_TYPE_CURSOR, "Cursor" },
-};
-
-/*
- * Optional properties
- */
 /**
  * drm_crtc_force_disable - Forcibly turn off a CRTC
  * @crtc: CRTC to turn off
@@ -102,8 +91,6 @@ out:
 }
 EXPORT_SYMBOL(drm_crtc_force_disable_all);
 
-DEFINE_WW_CLASS(crtc_ww_class);
-
 static unsigned int drm_num_crtcs(struct drm_device *dev)
 {
 	unsigned int num = 0;
@@ -116,12 +103,16 @@ static unsigned int drm_num_crtcs(struct drm_device *dev)
 	return num;
 }
 
-static int drm_crtc_register_all(struct drm_device *dev)
+int drm_crtc_register_all(struct drm_device *dev)
 {
 	struct drm_crtc *crtc;
 	int ret = 0;
 
 	drm_for_each_crtc(crtc, dev) {
+		if (drm_debugfs_crtc_add(crtc))
+			DRM_ERROR("Failed to initialize debugfs entry for CRTC '%s'.\n",
+				  crtc->name);
+
 		if (crtc->funcs->late_register)
 			ret = crtc->funcs->late_register(crtc);
 		if (ret)
@@ -131,16 +122,84 @@ static int drm_crtc_register_all(struct drm_device *dev)
 	return 0;
 }
 
-static void drm_crtc_unregister_all(struct drm_device *dev)
+void drm_crtc_unregister_all(struct drm_device *dev)
 {
 	struct drm_crtc *crtc;
 
 	drm_for_each_crtc(crtc, dev) {
 		if (crtc->funcs->early_unregister)
 			crtc->funcs->early_unregister(crtc);
+		drm_debugfs_crtc_remove(crtc);
 	}
 }
 
+static int drm_crtc_crc_init(struct drm_crtc *crtc)
+{
+#ifdef CONFIG_DEBUG_FS
+	spin_lock_init(&crtc->crc.lock);
+	init_waitqueue_head(&crtc->crc.wq);
+	crtc->crc.source = kstrdup("auto", GFP_KERNEL);
+	if (!crtc->crc.source)
+		return -ENOMEM;
+#endif
+	return 0;
+}
+
+static void drm_crtc_crc_fini(struct drm_crtc *crtc)
+{
+#ifdef CONFIG_DEBUG_FS
+	kfree(crtc->crc.source);
+#endif
+}
+
+static const struct dma_fence_ops drm_crtc_fence_ops;
+
+static struct drm_crtc *fence_to_crtc(struct dma_fence *fence)
+{
+	BUG_ON(fence->ops != &drm_crtc_fence_ops);
+	return container_of(fence->lock, struct drm_crtc, fence_lock);
+}
+
+static const char *drm_crtc_fence_get_driver_name(struct dma_fence *fence)
+{
+	struct drm_crtc *crtc = fence_to_crtc(fence);
+
+	return crtc->dev->driver->name;
+}
+
+static const char *drm_crtc_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct drm_crtc *crtc = fence_to_crtc(fence);
+
+	return crtc->timeline_name;
+}
+
+static bool drm_crtc_fence_enable_signaling(struct dma_fence *fence)
+{
+	return true;
+}
+
+static const struct dma_fence_ops drm_crtc_fence_ops = {
+	.get_driver_name = drm_crtc_fence_get_driver_name,
+	.get_timeline_name = drm_crtc_fence_get_timeline_name,
+	.enable_signaling = drm_crtc_fence_enable_signaling,
+	.wait = dma_fence_default_wait,
+};
+
+struct dma_fence *drm_crtc_create_fence(struct drm_crtc *crtc)
+{
+	struct dma_fence *fence;
+
+	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
+	if (!fence)
+		return NULL;
+
+	dma_fence_init(fence, &drm_crtc_fence_ops, &crtc->fence_lock,
+		       crtc->fence_context, ++crtc->fence_seqno);
+
+	return fence;
+}
+
 /**
  * drm_crtc_init_with_planes - Initialise a new CRTC object with
  *    specified primary and cursor planes.
@@ -198,6 +257,11 @@ int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc,
 		return -ENOMEM;
 	}
 
+	crtc->fence_context = dma_fence_context_alloc(1);
+	spin_lock_init(&crtc->fence_lock);
+	snprintf(crtc->timeline_name, sizeof(crtc->timeline_name),
+		 "CRTC:%d-%s", crtc->base.id, crtc->name);
+
 	crtc->base.properties = &crtc->properties;
 
 	list_add_tail(&crtc->head, &config->crtc_list);
@@ -205,14 +269,22 @@ int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc,
 
 	crtc->primary = primary;
 	crtc->cursor = cursor;
-	if (primary)
+	if (primary && !primary->possible_crtcs)
 		primary->possible_crtcs = 1 << drm_crtc_index(crtc);
-	if (cursor)
+	if (cursor && !cursor->possible_crtcs)
 		cursor->possible_crtcs = 1 << drm_crtc_index(crtc);
 
+	ret = drm_crtc_crc_init(crtc);
+	if (ret) {
+		drm_mode_object_unregister(dev, &crtc->base);
+		return ret;
+	}
+
 	if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
 		drm_object_attach_property(&crtc->base, config->prop_active, 0);
 		drm_object_attach_property(&crtc->base, config->prop_mode_id, 0);
+		drm_object_attach_property(&crtc->base,
+					   config->prop_out_fence_ptr, 0);
 	}
 
 	return 0;
@@ -236,6 +308,8 @@ void drm_crtc_cleanup(struct drm_crtc *crtc)
 	 * the indices on the drm_crtc after us in the crtc_list.
 	 */
 
+	drm_crtc_crc_fini(crtc);
+
 	kfree(crtc->gamma_store);
 	crtc->gamma_store = NULL;
 
@@ -255,301 +329,6 @@ void drm_crtc_cleanup(struct drm_crtc *crtc)
 }
 EXPORT_SYMBOL(drm_crtc_cleanup);
 
-int drm_modeset_register_all(struct drm_device *dev)
-{
-	int ret;
-
-	ret = drm_plane_register_all(dev);
-	if (ret)
-		goto err_plane;
-
-	ret = drm_crtc_register_all(dev);
-	if  (ret)
-		goto err_crtc;
-
-	ret = drm_encoder_register_all(dev);
-	if (ret)
-		goto err_encoder;
-
-	ret = drm_connector_register_all(dev);
-	if (ret)
-		goto err_connector;
-
-	return 0;
-
-err_connector:
-	drm_encoder_unregister_all(dev);
-err_encoder:
-	drm_crtc_unregister_all(dev);
-err_crtc:
-	drm_plane_unregister_all(dev);
-err_plane:
-	return ret;
-}
-
-void drm_modeset_unregister_all(struct drm_device *dev)
-{
-	drm_connector_unregister_all(dev);
-	drm_encoder_unregister_all(dev);
-	drm_crtc_unregister_all(dev);
-	drm_plane_unregister_all(dev);
-}
-
-static int drm_mode_create_standard_properties(struct drm_device *dev)
-{
-	struct drm_property *prop;
-	int ret;
-
-	ret = drm_connector_create_standard_properties(dev);
-	if (ret)
-		return ret;
-
-	prop = drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
-					"type", drm_plane_type_enum_list,
-					ARRAY_SIZE(drm_plane_type_enum_list));
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.plane_type_property = prop;
-
-	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
-			"SRC_X", 0, UINT_MAX);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.prop_src_x = prop;
-
-	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
-			"SRC_Y", 0, UINT_MAX);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.prop_src_y = prop;
-
-	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
-			"SRC_W", 0, UINT_MAX);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.prop_src_w = prop;
-
-	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
-			"SRC_H", 0, UINT_MAX);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.prop_src_h = prop;
-
-	prop = drm_property_create_signed_range(dev, DRM_MODE_PROP_ATOMIC,
-			"CRTC_X", INT_MIN, INT_MAX);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.prop_crtc_x = prop;
-
-	prop = drm_property_create_signed_range(dev, DRM_MODE_PROP_ATOMIC,
-			"CRTC_Y", INT_MIN, INT_MAX);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.prop_crtc_y = prop;
-
-	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
-			"CRTC_W", 0, INT_MAX);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.prop_crtc_w = prop;
-
-	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
-			"CRTC_H", 0, INT_MAX);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.prop_crtc_h = prop;
-
-	prop = drm_property_create_object(dev, DRM_MODE_PROP_ATOMIC,
-			"FB_ID", DRM_MODE_OBJECT_FB);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.prop_fb_id = prop;
-
-	prop = drm_property_create_object(dev, DRM_MODE_PROP_ATOMIC,
-			"CRTC_ID", DRM_MODE_OBJECT_CRTC);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.prop_crtc_id = prop;
-
-	prop = drm_property_create_bool(dev, DRM_MODE_PROP_ATOMIC,
-			"ACTIVE");
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.prop_active = prop;
-
-	prop = drm_property_create(dev,
-			DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB,
-			"MODE_ID", 0);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.prop_mode_id = prop;
-
-	prop = drm_property_create(dev,
-			DRM_MODE_PROP_BLOB,
-			"DEGAMMA_LUT", 0);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.degamma_lut_property = prop;
-
-	prop = drm_property_create_range(dev,
-			DRM_MODE_PROP_IMMUTABLE,
-			"DEGAMMA_LUT_SIZE", 0, UINT_MAX);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.degamma_lut_size_property = prop;
-
-	prop = drm_property_create(dev,
-			DRM_MODE_PROP_BLOB,
-			"CTM", 0);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.ctm_property = prop;
-
-	prop = drm_property_create(dev,
-			DRM_MODE_PROP_BLOB,
-			"GAMMA_LUT", 0);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.gamma_lut_property = prop;
-
-	prop = drm_property_create_range(dev,
-			DRM_MODE_PROP_IMMUTABLE,
-			"GAMMA_LUT_SIZE", 0, UINT_MAX);
-	if (!prop)
-		return -ENOMEM;
-	dev->mode_config.gamma_lut_size_property = prop;
-
-	return 0;
-}
-
-/**
- * drm_mode_getresources - get graphics configuration
- * @dev: drm device for the ioctl
- * @data: data pointer for the ioctl
- * @file_priv: drm file for the ioctl call
- *
- * Construct a set of configuration description structures and return
- * them to the user, including CRTC, connector and framebuffer configuration.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_getresources(struct drm_device *dev, void *data,
-			  struct drm_file *file_priv)
-{
-	struct drm_mode_card_res *card_res = data;
-	struct list_head *lh;
-	struct drm_framebuffer *fb;
-	struct drm_connector *connector;
-	struct drm_crtc *crtc;
-	struct drm_encoder *encoder;
-	int ret = 0;
-	int connector_count = 0;
-	int crtc_count = 0;
-	int fb_count = 0;
-	int encoder_count = 0;
-	int copied = 0;
-	uint32_t __user *fb_id;
-	uint32_t __user *crtc_id;
-	uint32_t __user *connector_id;
-	uint32_t __user *encoder_id;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return -EINVAL;
-
-
-	mutex_lock(&file_priv->fbs_lock);
-	/*
-	 * For the non-control nodes we need to limit the list of resources
-	 * by IDs in the group list for this node
-	 */
-	list_for_each(lh, &file_priv->fbs)
-		fb_count++;
-
-	/* handle this in 4 parts */
-	/* FBs */
-	if (card_res->count_fbs >= fb_count) {
-		copied = 0;
-		fb_id = (uint32_t __user *)(unsigned long)card_res->fb_id_ptr;
-		list_for_each_entry(fb, &file_priv->fbs, filp_head) {
-			if (put_user(fb->base.id, fb_id + copied)) {
-				mutex_unlock(&file_priv->fbs_lock);
-				return -EFAULT;
-			}
-			copied++;
-		}
-	}
-	card_res->count_fbs = fb_count;
-	mutex_unlock(&file_priv->fbs_lock);
-
-	/* mode_config.mutex protects the connector list against e.g. DP MST
-	 * connector hot-adding. CRTC/Plane lists are invariant. */
-	mutex_lock(&dev->mode_config.mutex);
-	drm_for_each_crtc(crtc, dev)
-		crtc_count++;
-
-	drm_for_each_connector(connector, dev)
-		connector_count++;
-
-	drm_for_each_encoder(encoder, dev)
-		encoder_count++;
-
-	card_res->max_height = dev->mode_config.max_height;
-	card_res->min_height = dev->mode_config.min_height;
-	card_res->max_width = dev->mode_config.max_width;
-	card_res->min_width = dev->mode_config.min_width;
-
-	/* CRTCs */
-	if (card_res->count_crtcs >= crtc_count) {
-		copied = 0;
-		crtc_id = (uint32_t __user *)(unsigned long)card_res->crtc_id_ptr;
-		drm_for_each_crtc(crtc, dev) {
-			if (put_user(crtc->base.id, crtc_id + copied)) {
-				ret = -EFAULT;
-				goto out;
-			}
-			copied++;
-		}
-	}
-	card_res->count_crtcs = crtc_count;
-
-	/* Encoders */
-	if (card_res->count_encoders >= encoder_count) {
-		copied = 0;
-		encoder_id = (uint32_t __user *)(unsigned long)card_res->encoder_id_ptr;
-		drm_for_each_encoder(encoder, dev) {
-			if (put_user(encoder->base.id, encoder_id +
-				     copied)) {
-				ret = -EFAULT;
-				goto out;
-			}
-			copied++;
-		}
-	}
-	card_res->count_encoders = encoder_count;
-
-	/* Connectors */
-	if (card_res->count_connectors >= connector_count) {
-		copied = 0;
-		connector_id = (uint32_t __user *)(unsigned long)card_res->connector_id_ptr;
-		drm_for_each_connector(connector, dev) {
-			if (put_user(connector->base.id,
-				     connector_id + copied)) {
-				ret = -EFAULT;
-				goto out;
-			}
-			copied++;
-		}
-	}
-	card_res->count_connectors = connector_count;
-
-out:
-	mutex_unlock(&dev->mode_config.mutex);
-	return ret;
-}
-
 /**
  * drm_mode_getcrtc - get CRTC configuration
  * @dev: drm device for the ioctl
@@ -695,8 +474,7 @@ int drm_crtc_check_viewport(const struct drm_crtc *crtc,
 	drm_crtc_get_hv_timing(mode, &hdisplay, &vdisplay);
 
 	if (crtc->state &&
-	    crtc->primary->state->rotation & (DRM_ROTATE_90 |
-					      DRM_ROTATE_270))
+	    drm_rotation_90_or_270(crtc->primary->state->rotation))
 		swap(hdisplay, vdisplay);
 
 	return drm_framebuffer_check_src_coords(x << 16, y << 16,
@@ -796,9 +574,10 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 			ret = drm_plane_check_pixel_format(crtc->primary,
 							   fb->pixel_format);
 			if (ret) {
-				char *format_name = drm_get_format_name(fb->pixel_format);
-				DRM_DEBUG_KMS("Invalid pixel format %s\n", format_name);
-				kfree(format_name);
+				struct drm_format_name_buf format_name;
+				DRM_DEBUG_KMS("Invalid pixel format %s\n",
+				              drm_get_format_name(fb->pixel_format,
+				                                  &format_name));
 				goto out;
 			}
 		}
@@ -902,362 +681,3 @@ int drm_mode_crtc_set_obj_prop(struct drm_mode_object *obj,
 
 	return ret;
 }
-
-/**
- * drm_mode_config_reset - call ->reset callbacks
- * @dev: drm device
- *
- * This functions calls all the crtc's, encoder's and connector's ->reset
- * callback. Drivers can use this in e.g. their driver load or resume code to
- * reset hardware and software state.
- */
-void drm_mode_config_reset(struct drm_device *dev)
-{
-	struct drm_crtc *crtc;
-	struct drm_plane *plane;
-	struct drm_encoder *encoder;
-	struct drm_connector *connector;
-
-	drm_for_each_plane(plane, dev)
-		if (plane->funcs->reset)
-			plane->funcs->reset(plane);
-
-	drm_for_each_crtc(crtc, dev)
-		if (crtc->funcs->reset)
-			crtc->funcs->reset(crtc);
-
-	drm_for_each_encoder(encoder, dev)
-		if (encoder->funcs->reset)
-			encoder->funcs->reset(encoder);
-
-	mutex_lock(&dev->mode_config.mutex);
-	drm_for_each_connector(connector, dev)
-		if (connector->funcs->reset)
-			connector->funcs->reset(connector);
-	mutex_unlock(&dev->mode_config.mutex);
-}
-EXPORT_SYMBOL(drm_mode_config_reset);
-
-/**
- * drm_mode_create_dumb_ioctl - create a dumb backing storage buffer
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * This creates a new dumb buffer in the driver's backing storage manager (GEM,
- * TTM or something else entirely) and returns the resulting buffer handle. This
- * handle can then be wrapped up into a framebuffer modeset object.
- *
- * Note that userspace is not allowed to use such objects for render
- * acceleration - drivers must create their own private ioctls for such a use
- * case.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_create_dumb_ioctl(struct drm_device *dev,
-			       void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_create_dumb *args = data;
-	u32 cpp, stride, size;
-
-	if (!dev->driver->dumb_create)
-		return -ENOSYS;
-	if (!args->width || !args->height || !args->bpp)
-		return -EINVAL;
-
-	/* overflow checks for 32bit size calculations */
-	/* NOTE: DIV_ROUND_UP() can overflow */
-	cpp = DIV_ROUND_UP(args->bpp, 8);
-	if (!cpp || cpp > 0xffffffffU / args->width)
-		return -EINVAL;
-	stride = cpp * args->width;
-	if (args->height > 0xffffffffU / stride)
-		return -EINVAL;
-
-	/* test for wrap-around */
-	size = args->height * stride;
-	if (PAGE_ALIGN(size) == 0)
-		return -EINVAL;
-
-	/*
-	 * handle, pitch and size are output parameters. Zero them out to
-	 * prevent drivers from accidentally using uninitialized data. Since
-	 * not all existing userspace is clearing these fields properly we
-	 * cannot reject IOCTL with garbage in them.
-	 */
-	args->handle = 0;
-	args->pitch = 0;
-	args->size = 0;
-
-	return dev->driver->dumb_create(file_priv, dev, args);
-}
-
-/**
- * drm_mode_mmap_dumb_ioctl - create an mmap offset for a dumb backing storage buffer
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * Allocate an offset in the drm device node's address space to be able to
- * memory map a dumb buffer.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_mmap_dumb_ioctl(struct drm_device *dev,
-			     void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_map_dumb *args = data;
-
-	/* call driver ioctl to get mmap offset */
-	if (!dev->driver->dumb_map_offset)
-		return -ENOSYS;
-
-	return dev->driver->dumb_map_offset(file_priv, dev, args->handle, &args->offset);
-}
-
-/**
- * drm_mode_destroy_dumb_ioctl - destroy a dumb backing strage buffer
- * @dev: DRM device
- * @data: ioctl data
- * @file_priv: DRM file info
- *
- * This destroys the userspace handle for the given dumb backing storage buffer.
- * Since buffer objects must be reference counted in the kernel a buffer object
- * won't be immediately freed if a framebuffer modeset object still uses it.
- *
- * Called by the user via ioctl.
- *
- * Returns:
- * Zero on success, negative errno on failure.
- */
-int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
-				void *data, struct drm_file *file_priv)
-{
-	struct drm_mode_destroy_dumb *args = data;
-
-	if (!dev->driver->dumb_destroy)
-		return -ENOSYS;
-
-	return dev->driver->dumb_destroy(file_priv, dev, args->handle);
-}
-
-/**
- * drm_mode_config_init - initialize DRM mode_configuration structure
- * @dev: DRM device
- *
- * Initialize @dev's mode_config structure, used for tracking the graphics
- * configuration of @dev.
- *
- * Since this initializes the modeset locks, no locking is possible. Which is no
- * problem, since this should happen single threaded at init time. It is the
- * driver's problem to ensure this guarantee.
- *
- */
-void drm_mode_config_init(struct drm_device *dev)
-{
-	mutex_init(&dev->mode_config.mutex);
-	drm_modeset_lock_init(&dev->mode_config.connection_mutex);
-	mutex_init(&dev->mode_config.idr_mutex);
-	mutex_init(&dev->mode_config.fb_lock);
-	mutex_init(&dev->mode_config.blob_lock);
-	INIT_LIST_HEAD(&dev->mode_config.fb_list);
-	INIT_LIST_HEAD(&dev->mode_config.crtc_list);
-	INIT_LIST_HEAD(&dev->mode_config.connector_list);
-	INIT_LIST_HEAD(&dev->mode_config.encoder_list);
-	INIT_LIST_HEAD(&dev->mode_config.property_list);
-	INIT_LIST_HEAD(&dev->mode_config.property_blob_list);
-	INIT_LIST_HEAD(&dev->mode_config.plane_list);
-	idr_init(&dev->mode_config.crtc_idr);
-	idr_init(&dev->mode_config.tile_idr);
-	ida_init(&dev->mode_config.connector_ida);
-
-	drm_modeset_lock_all(dev);
-	drm_mode_create_standard_properties(dev);
-	drm_modeset_unlock_all(dev);
-
-	/* Just to be sure */
-	dev->mode_config.num_fb = 0;
-	dev->mode_config.num_connector = 0;
-	dev->mode_config.num_crtc = 0;
-	dev->mode_config.num_encoder = 0;
-	dev->mode_config.num_overlay_plane = 0;
-	dev->mode_config.num_total_plane = 0;
-}
-EXPORT_SYMBOL(drm_mode_config_init);
-
-/**
- * drm_mode_config_cleanup - free up DRM mode_config info
- * @dev: DRM device
- *
- * Free up all the connectors and CRTCs associated with this DRM device, then
- * free up the framebuffers and associated buffer objects.
- *
- * Note that since this /should/ happen single-threaded at driver/device
- * teardown time, no locking is required. It's the driver's job to ensure that
- * this guarantee actually holds true.
- *
- * FIXME: cleanup any dangling user buffer objects too
- */
-void drm_mode_config_cleanup(struct drm_device *dev)
-{
-	struct drm_connector *connector, *ot;
-	struct drm_crtc *crtc, *ct;
-	struct drm_encoder *encoder, *enct;
-	struct drm_framebuffer *fb, *fbt;
-	struct drm_property *property, *pt;
-	struct drm_property_blob *blob, *bt;
-	struct drm_plane *plane, *plt;
-
-	list_for_each_entry_safe(encoder, enct, &dev->mode_config.encoder_list,
-				 head) {
-		encoder->funcs->destroy(encoder);
-	}
-
-	list_for_each_entry_safe(connector, ot,
-				 &dev->mode_config.connector_list, head) {
-		connector->funcs->destroy(connector);
-	}
-
-	list_for_each_entry_safe(property, pt, &dev->mode_config.property_list,
-				 head) {
-		drm_property_destroy(dev, property);
-	}
-
-	list_for_each_entry_safe(plane, plt, &dev->mode_config.plane_list,
-				 head) {
-		plane->funcs->destroy(plane);
-	}
-
-	list_for_each_entry_safe(crtc, ct, &dev->mode_config.crtc_list, head) {
-		crtc->funcs->destroy(crtc);
-	}
-
-	list_for_each_entry_safe(blob, bt, &dev->mode_config.property_blob_list,
-				 head_global) {
-		drm_property_unreference_blob(blob);
-	}
-
-	/*
-	 * Single-threaded teardown context, so it's not required to grab the
-	 * fb_lock to protect against concurrent fb_list access. Contrary, it
-	 * would actually deadlock with the drm_framebuffer_cleanup function.
-	 *
-	 * Also, if there are any framebuffers left, that's a driver leak now,
-	 * so politely WARN about this.
-	 */
-	WARN_ON(!list_empty(&dev->mode_config.fb_list));
-	list_for_each_entry_safe(fb, fbt, &dev->mode_config.fb_list, head) {
-		drm_framebuffer_free(&fb->base.refcount);
-	}
-
-	ida_destroy(&dev->mode_config.connector_ida);
-	idr_destroy(&dev->mode_config.tile_idr);
-	idr_destroy(&dev->mode_config.crtc_idr);
-	drm_modeset_lock_fini(&dev->mode_config.connection_mutex);
-}
-EXPORT_SYMBOL(drm_mode_config_cleanup);
-
-/**
- * DOC: Tile group
- *
- * Tile groups are used to represent tiled monitors with a unique
- * integer identifier. Tiled monitors using DisplayID v1.3 have
- * a unique 8-byte handle, we store this in a tile group, so we
- * have a common identifier for all tiles in a monitor group.
- */
-static void drm_tile_group_free(struct kref *kref)
-{
-	struct drm_tile_group *tg = container_of(kref, struct drm_tile_group, refcount);
-	struct drm_device *dev = tg->dev;
-	mutex_lock(&dev->mode_config.idr_mutex);
-	idr_remove(&dev->mode_config.tile_idr, tg->id);
-	mutex_unlock(&dev->mode_config.idr_mutex);
-	kfree(tg);
-}
-
-/**
- * drm_mode_put_tile_group - drop a reference to a tile group.
- * @dev: DRM device
- * @tg: tile group to drop reference to.
- *
- * drop reference to tile group and free if 0.
- */
-void drm_mode_put_tile_group(struct drm_device *dev,
-			     struct drm_tile_group *tg)
-{
-	kref_put(&tg->refcount, drm_tile_group_free);
-}
-
-/**
- * drm_mode_get_tile_group - get a reference to an existing tile group
- * @dev: DRM device
- * @topology: 8-bytes unique per monitor.
- *
- * Use the unique bytes to get a reference to an existing tile group.
- *
- * RETURNS:
- * tile group or NULL if not found.
- */
-struct drm_tile_group *drm_mode_get_tile_group(struct drm_device *dev,
-					       char topology[8])
-{
-	struct drm_tile_group *tg;
-	int id;
-	mutex_lock(&dev->mode_config.idr_mutex);
-	idr_for_each_entry(&dev->mode_config.tile_idr, tg, id) {
-		if (!memcmp(tg->group_data, topology, 8)) {
-			if (!kref_get_unless_zero(&tg->refcount))
-				tg = NULL;
-			mutex_unlock(&dev->mode_config.idr_mutex);
-			return tg;
-		}
-	}
-	mutex_unlock(&dev->mode_config.idr_mutex);
-	return NULL;
-}
-EXPORT_SYMBOL(drm_mode_get_tile_group);
-
-/**
- * drm_mode_create_tile_group - create a tile group from a displayid description
- * @dev: DRM device
- * @topology: 8-bytes unique per monitor.
- *
- * Create a tile group for the unique monitor, and get a unique
- * identifier for the tile group.
- *
- * RETURNS:
- * new tile group or error.
- */
-struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev,
-						  char topology[8])
-{
-	struct drm_tile_group *tg;
-	int ret;
-
-	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
-	if (!tg)
-		return ERR_PTR(-ENOMEM);
-
-	kref_init(&tg->refcount);
-	memcpy(tg->group_data, topology, 8);
-	tg->dev = dev;
-
-	mutex_lock(&dev->mode_config.idr_mutex);
-	ret = idr_alloc(&dev->mode_config.tile_idr, tg, 1, 0, GFP_KERNEL);
-	if (ret >= 0) {
-		tg->id = ret;
-	} else {
-		kfree(tg);
-		tg = ERR_PTR(ret);
-	}
-
-	mutex_unlock(&dev->mode_config.idr_mutex);
-	return tg;
-}
-EXPORT_SYMBOL(drm_mode_create_tile_group);
diff --git a/drivers/gpu/drm/drm_crtc_internal.h b/drivers/gpu/drm/drm_crtc_internal.h
index c48ba02c5365..cdf6860c9d22 100644
--- a/drivers/gpu/drm/drm_crtc_internal.h
+++ b/drivers/gpu/drm/drm_crtc_internal.h
@@ -40,10 +40,29 @@ int drm_crtc_check_viewport(const struct drm_crtc *crtc,
 			    int x, int y,
 			    const struct drm_display_mode *mode,
 			    const struct drm_framebuffer *fb);
+int drm_crtc_register_all(struct drm_device *dev);
+void drm_crtc_unregister_all(struct drm_device *dev);
 
-void drm_fb_release(struct drm_file *file_priv);
+struct dma_fence *drm_crtc_create_fence(struct drm_crtc *crtc);
+
+/* IOCTLs */
+int drm_mode_getcrtc(struct drm_device *dev,
+		     void *data, struct drm_file *file_priv);
+int drm_mode_setcrtc(struct drm_device *dev,
+		     void *data, struct drm_file *file_priv);
+
+
+/* drm_mode_config.c */
+int drm_modeset_register_all(struct drm_device *dev);
+void drm_modeset_unregister_all(struct drm_device *dev);
+
+/* IOCTLs */
+int drm_mode_getresources(struct drm_device *dev,
+			  void *data, struct drm_file *file_priv);
 
-/* dumb buffer support IOCTLs */
+
+/* drm_dumb_buffers.c */
+/* IOCTLs */
 int drm_mode_create_dumb_ioctl(struct drm_device *dev,
 			       void *data, struct drm_file *file_priv);
 int drm_mode_mmap_dumb_ioctl(struct drm_device *dev,
@@ -51,14 +70,6 @@ int drm_mode_mmap_dumb_ioctl(struct drm_device *dev,
 int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
 				void *data, struct drm_file *file_priv);
 
-/* IOCTLs */
-int drm_mode_getresources(struct drm_device *dev,
-			  void *data, struct drm_file *file_priv);
-int drm_mode_getcrtc(struct drm_device *dev,
-		     void *data, struct drm_file *file_priv);
-int drm_mode_setcrtc(struct drm_device *dev,
-		     void *data, struct drm_file *file_priv);
-
 /* drm_color_mgmt.c */
 
 /* IOCTLs */
@@ -147,6 +158,8 @@ void drm_framebuffer_free(struct kref *kref);
 int drm_framebuffer_check_src_coords(uint32_t src_x, uint32_t src_y,
 				     uint32_t src_w, uint32_t src_h,
 				     const struct drm_framebuffer *fb);
+void drm_fb_release(struct drm_file *file_priv);
+
 
 /* IOCTL */
 int drm_mode_addfb(struct drm_device *dev,
@@ -166,9 +179,6 @@ int drm_atomic_get_property(struct drm_mode_object *obj,
 int drm_mode_atomic_ioctl(struct drm_device *dev,
 			  void *data, struct drm_file *file_priv);
 
-int drm_modeset_register_all(struct drm_device *dev);
-void drm_modeset_unregister_all(struct drm_device *dev);
-
 
 /* drm_plane.c */
 int drm_plane_register_all(struct drm_device *dev);
diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c
index 1205790ed960..2e3e46a53805 100644
--- a/drivers/gpu/drm/drm_debugfs.c
+++ b/drivers/gpu/drm/drm_debugfs.c
@@ -36,6 +36,7 @@
 #include <linux/export.h>
 #include <drm/drmP.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_atomic.h>
 #include "drm_internal.h"
 
 #if defined(CONFIG_DEBUG_FS)
@@ -163,6 +164,14 @@ int drm_debugfs_init(struct drm_minor *minor, int minor_id,
 		return ret;
 	}
 
+	if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
+		ret = drm_atomic_debugfs_init(minor);
+		if (ret) {
+			DRM_ERROR("Failed to create atomic debugfs files\n");
+			return ret;
+		}
+	}
+
 	if (dev->driver->debugfs_init) {
 		ret = dev->driver->debugfs_init(minor);
 		if (ret) {
@@ -219,6 +228,7 @@ EXPORT_SYMBOL(drm_debugfs_remove_files);
 int drm_debugfs_cleanup(struct drm_minor *minor)
 {
 	struct drm_device *dev = minor->dev;
+	int ret;
 
 	if (!minor->debugfs_root)
 		return 0;
@@ -226,6 +236,14 @@ int drm_debugfs_cleanup(struct drm_minor *minor)
 	if (dev->driver->debugfs_cleanup)
 		dev->driver->debugfs_cleanup(minor);
 
+	if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
+		ret = drm_atomic_debugfs_cleanup(minor);
+		if (ret) {
+			DRM_ERROR("DRM: Failed to remove atomic debugfs entries\n");
+			return ret;
+		}
+	}
+
 	drm_debugfs_remove_files(drm_debugfs_list, DRM_DEBUGFS_ENTRIES, minor);
 
 	debugfs_remove(minor->debugfs_root);
@@ -415,5 +433,37 @@ void drm_debugfs_connector_remove(struct drm_connector *connector)
 	connector->debugfs_entry = NULL;
 }
 
-#endif /* CONFIG_DEBUG_FS */
+int drm_debugfs_crtc_add(struct drm_crtc *crtc)
+{
+	struct drm_minor *minor = crtc->dev->primary;
+	struct dentry *root;
+	char *name;
+
+	name = kasprintf(GFP_KERNEL, "crtc-%d", crtc->index);
+	if (!name)
+		return -ENOMEM;
 
+	root = debugfs_create_dir(name, minor->debugfs_root);
+	kfree(name);
+	if (!root)
+		return -ENOMEM;
+
+	crtc->debugfs_entry = root;
+
+	if (drm_debugfs_crtc_crc_add(crtc))
+		goto error;
+
+	return 0;
+
+error:
+	drm_debugfs_crtc_remove(crtc);
+	return -ENOMEM;
+}
+
+void drm_debugfs_crtc_remove(struct drm_crtc *crtc)
+{
+	debugfs_remove_recursive(crtc->debugfs_entry);
+	crtc->debugfs_entry = NULL;
+}
+
+#endif /* CONFIG_DEBUG_FS */
diff --git a/drivers/gpu/drm/drm_debugfs_crc.c b/drivers/gpu/drm/drm_debugfs_crc.c
new file mode 100644
index 000000000000..00e771fb7df2
--- /dev/null
+++ b/drivers/gpu/drm/drm_debugfs_crc.c
@@ -0,0 +1,352 @@
+/*
+ * Copyright © 2008 Intel Corporation
+ * Copyright © 2016 Collabora Ltd
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Based on code from the i915 driver.
+ * Original author: Damien Lespiau <damien.lespiau@intel.com>
+ *
+ */
+
+#include <linux/circ_buf.h>
+#include <linux/ctype.h>
+#include <linux/debugfs.h>
+#include <drm/drmP.h>
+#include "drm_internal.h"
+
+/**
+ * DOC: CRC ABI
+ *
+ * DRM device drivers can provide to userspace CRC information of each frame as
+ * it reached a given hardware component (a "source").
+ *
+ * Userspace can control generation of CRCs in a given CRTC by writing to the
+ * file dri/0/crtc-N/crc/control in debugfs, with N being the index of the CRTC.
+ * Accepted values are source names (which are driver-specific) and the "auto"
+ * keyword, which will let the driver select a default source of frame CRCs
+ * for this CRTC.
+ *
+ * Once frame CRC generation is enabled, userspace can capture them by reading
+ * the dri/0/crtc-N/crc/data file. Each line in that file contains the frame
+ * number in the first field and then a number of unsigned integer fields
+ * containing the CRC data. Fields are separated by a single space and the number
+ * of CRC fields is source-specific.
+ *
+ * Note that though in some cases the CRC is computed in a specified way and on
+ * the frame contents as supplied by userspace (eDP 1.3), in general the CRC
+ * computation is performed in an unspecified way and on frame contents that have
+ * been already processed in also an unspecified way and thus userspace cannot
+ * rely on being able to generate matching CRC values for the frame contents that
+ * it submits. In this general case, the maximum userspace can do is to compare
+ * the reported CRCs of frames that should have the same contents.
+ */
+
+static int crc_control_show(struct seq_file *m, void *data)
+{
+	struct drm_crtc *crtc = m->private;
+
+	seq_printf(m, "%s\n", crtc->crc.source);
+
+	return 0;
+}
+
+static int crc_control_open(struct inode *inode, struct file *file)
+{
+	struct drm_crtc *crtc = inode->i_private;
+
+	return single_open(file, crc_control_show, crtc);
+}
+
+static ssize_t crc_control_write(struct file *file, const char __user *ubuf,
+				 size_t len, loff_t *offp)
+{
+	struct seq_file *m = file->private_data;
+	struct drm_crtc *crtc = m->private;
+	struct drm_crtc_crc *crc = &crtc->crc;
+	char *source;
+
+	if (len == 0)
+		return 0;
+
+	if (len > PAGE_SIZE - 1) {
+		DRM_DEBUG_KMS("Expected < %lu bytes into crtc crc control\n",
+			      PAGE_SIZE);
+		return -E2BIG;
+	}
+
+	source = memdup_user_nul(ubuf, len);
+	if (IS_ERR(source))
+		return PTR_ERR(source);
+
+	if (source[len] == '\n')
+		source[len] = '\0';
+
+	spin_lock_irq(&crc->lock);
+
+	if (crc->opened) {
+		spin_unlock_irq(&crc->lock);
+		kfree(source);
+		return -EBUSY;
+	}
+
+	kfree(crc->source);
+	crc->source = source;
+
+	spin_unlock_irq(&crc->lock);
+
+	*offp += len;
+	return len;
+}
+
+static const struct file_operations drm_crtc_crc_control_fops = {
+	.owner = THIS_MODULE,
+	.open = crc_control_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+	.write = crc_control_write
+};
+
+static int crtc_crc_open(struct inode *inode, struct file *filep)
+{
+	struct drm_crtc *crtc = inode->i_private;
+	struct drm_crtc_crc *crc = &crtc->crc;
+	struct drm_crtc_crc_entry *entries = NULL;
+	size_t values_cnt;
+	int ret;
+
+	if (crc->opened)
+		return -EBUSY;
+
+	ret = crtc->funcs->set_crc_source(crtc, crc->source, &values_cnt);
+	if (ret)
+		return ret;
+
+	if (WARN_ON(values_cnt > DRM_MAX_CRC_NR)) {
+		ret = -EINVAL;
+		goto err_disable;
+	}
+
+	if (WARN_ON(values_cnt == 0)) {
+		ret = -EINVAL;
+		goto err_disable;
+	}
+
+	entries = kcalloc(DRM_CRC_ENTRIES_NR, sizeof(*entries), GFP_KERNEL);
+	if (!entries) {
+		ret = -ENOMEM;
+		goto err_disable;
+	}
+
+	spin_lock_irq(&crc->lock);
+	crc->entries = entries;
+	crc->values_cnt = values_cnt;
+	crc->opened = true;
+	spin_unlock_irq(&crc->lock);
+
+	return 0;
+
+err_disable:
+	crtc->funcs->set_crc_source(crtc, NULL, &values_cnt);
+	return ret;
+}
+
+static int crtc_crc_release(struct inode *inode, struct file *filep)
+{
+	struct drm_crtc *crtc = filep->f_inode->i_private;
+	struct drm_crtc_crc *crc = &crtc->crc;
+	size_t values_cnt;
+
+	spin_lock_irq(&crc->lock);
+	kfree(crc->entries);
+	crc->entries = NULL;
+	crc->head = 0;
+	crc->tail = 0;
+	crc->values_cnt = 0;
+	crc->opened = false;
+	spin_unlock_irq(&crc->lock);
+
+	crtc->funcs->set_crc_source(crtc, NULL, &values_cnt);
+
+	return 0;
+}
+
+static int crtc_crc_data_count(struct drm_crtc_crc *crc)
+{
+	assert_spin_locked(&crc->lock);
+	return CIRC_CNT(crc->head, crc->tail, DRM_CRC_ENTRIES_NR);
+}
+
+/*
+ * 1 frame field of 10 chars plus a number of CRC fields of 10 chars each, space
+ * separated, with a newline at the end and null-terminated.
+ */
+#define LINE_LEN(values_cnt)	(10 + 11 * values_cnt + 1 + 1)
+#define MAX_LINE_LEN		(LINE_LEN(DRM_MAX_CRC_NR))
+
+static ssize_t crtc_crc_read(struct file *filep, char __user *user_buf,
+			     size_t count, loff_t *pos)
+{
+	struct drm_crtc *crtc = filep->f_inode->i_private;
+	struct drm_crtc_crc *crc = &crtc->crc;
+	struct drm_crtc_crc_entry *entry;
+	char buf[MAX_LINE_LEN];
+	int ret, i;
+
+	spin_lock_irq(&crc->lock);
+
+	if (!crc->source) {
+		spin_unlock_irq(&crc->lock);
+		return 0;
+	}
+
+	/* Nothing to read? */
+	while (crtc_crc_data_count(crc) == 0) {
+		if (filep->f_flags & O_NONBLOCK) {
+			spin_unlock_irq(&crc->lock);
+			return -EAGAIN;
+		}
+
+		ret = wait_event_interruptible_lock_irq(crc->wq,
+							crtc_crc_data_count(crc),
+							crc->lock);
+		if (ret) {
+			spin_unlock_irq(&crc->lock);
+			return ret;
+		}
+	}
+
+	/* We know we have an entry to be read */
+	entry = &crc->entries[crc->tail];
+
+	if (count < LINE_LEN(crc->values_cnt)) {
+		spin_unlock_irq(&crc->lock);
+		return -EINVAL;
+	}
+
+	BUILD_BUG_ON_NOT_POWER_OF_2(DRM_CRC_ENTRIES_NR);
+	crc->tail = (crc->tail + 1) & (DRM_CRC_ENTRIES_NR - 1);
+
+	spin_unlock_irq(&crc->lock);
+
+	if (entry->has_frame_counter)
+		sprintf(buf, "0x%08x", entry->frame);
+	else
+		sprintf(buf, "XXXXXXXXXX");
+
+	for (i = 0; i < crc->values_cnt; i++)
+		sprintf(buf + 10 + i * 11, " 0x%08x", entry->crcs[i]);
+	sprintf(buf + 10 + crc->values_cnt * 11, "\n");
+
+	if (copy_to_user(user_buf, buf, LINE_LEN(crc->values_cnt)))
+		return -EFAULT;
+
+	return LINE_LEN(crc->values_cnt);
+}
+
+static const struct file_operations drm_crtc_crc_data_fops = {
+	.owner = THIS_MODULE,
+	.open = crtc_crc_open,
+	.read = crtc_crc_read,
+	.release = crtc_crc_release,
+};
+
+/**
+ * drm_debugfs_crtc_crc_add - Add files to debugfs for capture of frame CRCs
+ * @crtc: CRTC to whom the frames will belong
+ *
+ * Adds files to debugfs directory that allows userspace to control the
+ * generation of frame CRCs and to read them.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_debugfs_crtc_crc_add(struct drm_crtc *crtc)
+{
+	struct dentry *crc_ent, *ent;
+
+	if (!crtc->funcs->set_crc_source)
+		return 0;
+
+	crc_ent = debugfs_create_dir("crc", crtc->debugfs_entry);
+	if (!crc_ent)
+		return -ENOMEM;
+
+	ent = debugfs_create_file("control", S_IRUGO, crc_ent, crtc,
+				  &drm_crtc_crc_control_fops);
+	if (!ent)
+		goto error;
+
+	ent = debugfs_create_file("data", S_IRUGO, crc_ent, crtc,
+				  &drm_crtc_crc_data_fops);
+	if (!ent)
+		goto error;
+
+	return 0;
+
+error:
+	debugfs_remove_recursive(crc_ent);
+
+	return -ENOMEM;
+}
+
+/**
+ * drm_crtc_add_crc_entry - Add entry with CRC information for a frame
+ * @crtc: CRTC to which the frame belongs
+ * @has_frame: whether this entry has a frame number to go with
+ * @frame: number of the frame these CRCs are about
+ * @crcs: array of CRC values, with length matching #drm_crtc_crc.values_cnt
+ *
+ * For each frame, the driver polls the source of CRCs for new data and calls
+ * this function to add them to the buffer from where userspace reads.
+ */
+int drm_crtc_add_crc_entry(struct drm_crtc *crtc, bool has_frame,
+			   uint32_t frame, uint32_t *crcs)
+{
+	struct drm_crtc_crc *crc = &crtc->crc;
+	struct drm_crtc_crc_entry *entry;
+	int head, tail;
+
+	assert_spin_locked(&crc->lock);
+
+	/* Caller may not have noticed yet that userspace has stopped reading */
+	if (!crc->opened)
+		return -EINVAL;
+
+	head = crc->head;
+	tail = crc->tail;
+
+	if (CIRC_SPACE(head, tail, DRM_CRC_ENTRIES_NR) < 1) {
+		DRM_ERROR("Overflow of CRC buffer, userspace reads too slow.\n");
+		return -ENOBUFS;
+	}
+
+	entry = &crc->entries[head];
+	entry->frame = frame;
+	entry->has_frame_counter = has_frame;
+	memcpy(&entry->crcs, crcs, sizeof(*crcs) * crc->values_cnt);
+
+	head = (head + 1) & (DRM_CRC_ENTRIES_NR - 1);
+	crc->head = head;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(drm_crtc_add_crc_entry);
diff --git a/drivers/gpu/drm/drm_dp_dual_mode_helper.c b/drivers/gpu/drm/drm_dp_dual_mode_helper.c
index a7b2a751f6fe..e02563966271 100644
--- a/drivers/gpu/drm/drm_dp_dual_mode_helper.c
+++ b/drivers/gpu/drm/drm_dp_dual_mode_helper.c
@@ -142,12 +142,25 @@ static bool is_hdmi_adaptor(const char hdmi_id[DP_DUAL_MODE_HDMI_ID_LEN])
 		      sizeof(dp_dual_mode_hdmi_id)) == 0;
 }
 
+static bool is_type1_adaptor(uint8_t adaptor_id)
+{
+	return adaptor_id == 0 || adaptor_id == 0xff;
+}
+
 static bool is_type2_adaptor(uint8_t adaptor_id)
 {
 	return adaptor_id == (DP_DUAL_MODE_TYPE_TYPE2 |
 			      DP_DUAL_MODE_REV_TYPE2);
 }
 
+static bool is_lspcon_adaptor(const char hdmi_id[DP_DUAL_MODE_HDMI_ID_LEN],
+			      const uint8_t adaptor_id)
+{
+	return is_hdmi_adaptor(hdmi_id) &&
+		(adaptor_id == (DP_DUAL_MODE_TYPE_TYPE2 |
+		 DP_DUAL_MODE_TYPE_HAS_DPCD));
+}
+
 /**
  * drm_dp_dual_mode_detect - Identify the DP dual mode adaptor
  * @adapter: I2C adapter for the DDC bus
@@ -185,6 +198,8 @@ enum drm_dp_dual_mode_type drm_dp_dual_mode_detect(struct i2c_adapter *adapter)
 	 */
 	ret = drm_dp_dual_mode_read(adapter, DP_DUAL_MODE_HDMI_ID,
 				    hdmi_id, sizeof(hdmi_id));
+	DRM_DEBUG_KMS("DP dual mode HDMI ID: %*pE (err %zd)\n",
+		      ret ? 0 : (int)sizeof(hdmi_id), hdmi_id, ret);
 	if (ret)
 		return DRM_DP_DUAL_MODE_UNKNOWN;
 
@@ -202,13 +217,26 @@ enum drm_dp_dual_mode_type drm_dp_dual_mode_detect(struct i2c_adapter *adapter)
 	 */
 	ret = drm_dp_dual_mode_read(adapter, DP_DUAL_MODE_ADAPTOR_ID,
 				    &adaptor_id, sizeof(adaptor_id));
+	DRM_DEBUG_KMS("DP dual mode adaptor ID: %02x (err %zd)\n",
+		      adaptor_id, ret);
 	if (ret == 0) {
+		if (is_lspcon_adaptor(hdmi_id, adaptor_id))
+			return DRM_DP_DUAL_MODE_LSPCON;
 		if (is_type2_adaptor(adaptor_id)) {
 			if (is_hdmi_adaptor(hdmi_id))
 				return DRM_DP_DUAL_MODE_TYPE2_HDMI;
 			else
 				return DRM_DP_DUAL_MODE_TYPE2_DVI;
 		}
+		/*
+		 * If neither a proper type 1 ID nor a broken type 1 adaptor
+		 * as described above, assume type 1, but let the user know
+		 * that we may have misdetected the type.
+		 */
+		if (!is_type1_adaptor(adaptor_id) && adaptor_id != hdmi_id[0])
+			DRM_ERROR("Unexpected DP dual mode adaptor ID %02x\n",
+				  adaptor_id);
+
 	}
 
 	if (is_hdmi_adaptor(hdmi_id))
@@ -364,3 +392,96 @@ const char *drm_dp_get_dual_mode_type_name(enum drm_dp_dual_mode_type type)
 	}
 }
 EXPORT_SYMBOL(drm_dp_get_dual_mode_type_name);
+
+/**
+ * drm_lspcon_get_mode: Get LSPCON's current mode of operation by
+ * reading offset (0x80, 0x41)
+ * @adapter: I2C-over-aux adapter
+ * @mode: current lspcon mode of operation output variable
+ *
+ * Returns:
+ * 0 on success, sets the current_mode value to appropriate mode
+ * -error on failure
+ */
+int drm_lspcon_get_mode(struct i2c_adapter *adapter,
+			enum drm_lspcon_mode *mode)
+{
+	u8 data;
+	int ret = 0;
+
+	if (!mode) {
+		DRM_ERROR("NULL input\n");
+		return -EINVAL;
+	}
+
+	/* Read Status: i2c over aux */
+	ret = drm_dp_dual_mode_read(adapter, DP_DUAL_MODE_LSPCON_CURRENT_MODE,
+				    &data, sizeof(data));
+	if (ret < 0) {
+		DRM_ERROR("LSPCON read(0x80, 0x41) failed\n");
+		return -EFAULT;
+	}
+
+	if (data & DP_DUAL_MODE_LSPCON_MODE_PCON)
+		*mode = DRM_LSPCON_MODE_PCON;
+	else
+		*mode = DRM_LSPCON_MODE_LS;
+	return 0;
+}
+EXPORT_SYMBOL(drm_lspcon_get_mode);
+
+/**
+ * drm_lspcon_set_mode: Change LSPCON's mode of operation by
+ * writing offset (0x80, 0x40)
+ * @adapter: I2C-over-aux adapter
+ * @mode: required mode of operation
+ *
+ * Returns:
+ * 0 on success, -error on failure/timeout
+ */
+int drm_lspcon_set_mode(struct i2c_adapter *adapter,
+			enum drm_lspcon_mode mode)
+{
+	u8 data = 0;
+	int ret;
+	int time_out = 200;
+	enum drm_lspcon_mode current_mode;
+
+	if (mode == DRM_LSPCON_MODE_PCON)
+		data = DP_DUAL_MODE_LSPCON_MODE_PCON;
+
+	/* Change mode */
+	ret = drm_dp_dual_mode_write(adapter, DP_DUAL_MODE_LSPCON_MODE_CHANGE,
+				     &data, sizeof(data));
+	if (ret < 0) {
+		DRM_ERROR("LSPCON mode change failed\n");
+		return ret;
+	}
+
+	/*
+	 * Confirm mode change by reading the status bit.
+	 * Sometimes, it takes a while to change the mode,
+	 * so wait and retry until time out or done.
+	 */
+	do {
+		ret = drm_lspcon_get_mode(adapter, &current_mode);
+		if (ret) {
+			DRM_ERROR("can't confirm LSPCON mode change\n");
+			return ret;
+		} else {
+			if (current_mode != mode) {
+				msleep(10);
+				time_out -= 10;
+			} else {
+				DRM_DEBUG_KMS("LSPCON mode changed to %s\n",
+						mode == DRM_LSPCON_MODE_LS ?
+						"LS" : "PCON");
+				return 0;
+			}
+		}
+	} while (time_out);
+
+	DRM_ERROR("LSPCON mode change timed out\n");
+	return -ETIMEDOUT;
+}
+EXPORT_SYMBOL(drm_lspcon_set_mode);
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 6efdba4993fc..a525751b4559 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -32,7 +32,10 @@
 #include <linux/moduleparam.h>
 #include <linux/mount.h>
 #include <linux/slab.h>
+
+#include <drm/drm_drv.h>
 #include <drm/drmP.h>
+
 #include "drm_crtc_internal.h"
 #include "drm_legacy.h"
 #include "drm_internal.h"
@@ -257,10 +260,7 @@ static void drm_minor_unregister(struct drm_device *dev, unsigned int type)
 	drm_debugfs_cleanup(minor);
 }
 
-/**
- * drm_minor_acquire - Acquire a DRM minor
- * @minor_id: Minor ID of the DRM-minor
- *
+/*
  * Looks up the given minor-ID and returns the respective DRM-minor object. The
  * refence-count of the underlying device is increased so you must release this
  * object with drm_minor_release().
@@ -268,10 +268,6 @@ static void drm_minor_unregister(struct drm_device *dev, unsigned int type)
  * As long as you hold this minor, it is guaranteed that the object and the
  * minor->dev pointer will stay valid! However, the device may get unplugged and
  * unregistered while you hold the minor.
- *
- * Returns:
- * Pointer to minor-object with increased device-refcount, or PTR_ERR on
- * failure.
  */
 struct drm_minor *drm_minor_acquire(unsigned int minor_id)
 {
@@ -294,12 +290,6 @@ struct drm_minor *drm_minor_acquire(unsigned int minor_id)
 	return minor;
 }
 
-/**
- * drm_minor_release - Release DRM minor
- * @minor: Pointer to DRM minor object
- *
- * Release a minor that was previously acquired via drm_minor_acquire().
- */
 void drm_minor_release(struct drm_minor *minor)
 {
 	drm_dev_unref(minor->dev);
@@ -313,9 +303,10 @@ void drm_minor_release(struct drm_minor *minor)
  * callbacks implemented by the driver. The driver then needs to initialize all
  * the various subsystems for the drm device like memory management, vblank
  * handling, modesetting support and intial output configuration plus obviously
- * initialize all the corresponding hardware bits. Finally when everything is up
- * and running and ready for userspace the device instance can be published
- * using drm_dev_register().
+ * initialize all the corresponding hardware bits. An important part of this is
+ * also calling drm_dev_set_unique() to set the userspace-visible unique name of
+ * this device instance. Finally when everything is up and running and ready for
+ * userspace the device instance can be published using drm_dev_register().
  *
  * There is also deprecated support for initalizing device instances using
  * bus-specific helpers and the ->load() callback. But due to
@@ -337,17 +328,6 @@ void drm_minor_release(struct drm_minor *minor)
  * dev_priv field of &drm_device.
  */
 
-static int drm_dev_set_unique(struct drm_device *dev, const char *name)
-{
-	if (!name)
-		return -EINVAL;
-
-	kfree(dev->unique);
-	dev->unique = kstrdup(name, GFP_KERNEL);
-
-	return dev->unique ? 0 : -ENOMEM;
-}
-
 /**
  * drm_put_dev - Unregister and release a DRM device
  * @dev: DRM device
@@ -517,12 +497,6 @@ int drm_dev_init(struct drm_device *dev,
 		goto err_free;
 	}
 
-	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
-		ret = drm_minor_alloc(dev, DRM_MINOR_CONTROL);
-		if (ret)
-			goto err_minors;
-	}
-
 	if (drm_core_check_feature(dev, DRIVER_RENDER)) {
 		ret = drm_minor_alloc(dev, DRM_MINOR_RENDER);
 		if (ret)
@@ -568,6 +542,9 @@ err_minors:
 	drm_fs_inode_free(dev->anon_inode);
 err_free:
 	mutex_destroy(&dev->master_mutex);
+	mutex_destroy(&dev->ctxlist_mutex);
+	mutex_destroy(&dev->filelist_mutex);
+	mutex_destroy(&dev->struct_mutex);
 	return ret;
 }
 EXPORT_SYMBOL(drm_dev_init);
@@ -630,6 +607,9 @@ static void drm_dev_release(struct kref *ref)
 	drm_minor_free(dev, DRM_MINOR_CONTROL);
 
 	mutex_destroy(&dev->master_mutex);
+	mutex_destroy(&dev->ctxlist_mutex);
+	mutex_destroy(&dev->filelist_mutex);
+	mutex_destroy(&dev->struct_mutex);
 	kfree(dev->unique);
 	kfree(dev);
 }
@@ -667,6 +647,62 @@ void drm_dev_unref(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_dev_unref);
 
+static int create_compat_control_link(struct drm_device *dev)
+{
+	struct drm_minor *minor;
+	char *name;
+	int ret;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return 0;
+
+	minor = *drm_minor_get_slot(dev, DRM_MINOR_PRIMARY);
+	if (!minor)
+		return 0;
+
+	/*
+	 * Some existing userspace out there uses the existing of the controlD*
+	 * sysfs files to figure out whether it's a modeset driver. It only does
+	 * readdir, hence a symlink is sufficient (and the least confusing
+	 * option). Otherwise controlD* is entirely unused.
+	 *
+	 * Old controlD chardev have been allocated in the range
+	 * 64-127.
+	 */
+	name = kasprintf(GFP_KERNEL, "controlD%d", minor->index + 64);
+	if (!name)
+		return -ENOMEM;
+
+	ret = sysfs_create_link(minor->kdev->kobj.parent,
+				&minor->kdev->kobj,
+				name);
+
+	kfree(name);
+
+	return ret;
+}
+
+static void remove_compat_control_link(struct drm_device *dev)
+{
+	struct drm_minor *minor;
+	char *name;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return;
+
+	minor = *drm_minor_get_slot(dev, DRM_MINOR_PRIMARY);
+	if (!minor)
+		return;
+
+	name = kasprintf(GFP_KERNEL, "controlD%d", minor->index);
+	if (!name)
+		return;
+
+	sysfs_remove_link(minor->kdev->kobj.parent, name);
+
+	kfree(name);
+}
+
 /**
  * drm_dev_register - Register DRM device
  * @dev: Device to register
@@ -705,6 +741,10 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags)
 	if (ret)
 		goto err_minors;
 
+	ret = create_compat_control_link(dev);
+	if (ret)
+		goto err_minors;
+
 	if (dev->driver->load) {
 		ret = dev->driver->load(dev, flags);
 		if (ret)
@@ -718,6 +758,7 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags)
 	goto out_unlock;
 
 err_minors:
+	remove_compat_control_link(dev);
 	drm_minor_unregister(dev, DRM_MINOR_PRIMARY);
 	drm_minor_unregister(dev, DRM_MINOR_RENDER);
 	drm_minor_unregister(dev, DRM_MINOR_CONTROL);
@@ -758,12 +799,33 @@ void drm_dev_unregister(struct drm_device *dev)
 	list_for_each_entry_safe(r_list, list_temp, &dev->maplist, head)
 		drm_legacy_rmmap(dev, r_list->map);
 
+	remove_compat_control_link(dev);
 	drm_minor_unregister(dev, DRM_MINOR_PRIMARY);
 	drm_minor_unregister(dev, DRM_MINOR_RENDER);
 	drm_minor_unregister(dev, DRM_MINOR_CONTROL);
 }
 EXPORT_SYMBOL(drm_dev_unregister);
 
+/**
+ * drm_dev_set_unique - Set the unique name of a DRM device
+ * @dev: device of which to set the unique name
+ * @name: unique name
+ *
+ * Sets the unique name of a DRM device using the specified string. Drivers
+ * can use this at driver probe time if the unique name of the devices they
+ * drive is static.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int drm_dev_set_unique(struct drm_device *dev, const char *name)
+{
+	kfree(dev->unique);
+	dev->unique = kstrdup(name, GFP_KERNEL);
+
+	return dev->unique ? 0 : -ENOMEM;
+}
+EXPORT_SYMBOL(drm_dev_set_unique);
+
 /*
  * DRM Core
  * The DRM core module initializes all global DRM objects and makes them
diff --git a/drivers/gpu/drm/drm_dumb_buffers.c b/drivers/gpu/drm/drm_dumb_buffers.c
new file mode 100644
index 000000000000..8ac5a1c1d811
--- /dev/null
+++ b/drivers/gpu/drm/drm_dumb_buffers.c
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2006-2008 Intel Corporation
+ * Copyright (c) 2007 Dave Airlie <airlied@linux.ie>
+ * Copyright (c) 2008 Red Hat Inc.
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#include <drm/drmP.h>
+
+#include "drm_crtc_internal.h"
+
+/**
+ * DOC: overview
+ *
+ * The KMS API doesn't standardize backing storage object creation and leaves it
+ * to driver-specific ioctls. Furthermore actually creating a buffer object even
+ * for GEM-based drivers is done through a driver-specific ioctl - GEM only has
+ * a common userspace interface for sharing and destroying objects. While not an
+ * issue for full-fledged graphics stacks that include device-specific userspace
+ * components (in libdrm for instance), this limit makes DRM-based early boot
+ * graphics unnecessarily complex.
+ *
+ * Dumb objects partly alleviate the problem by providing a standard API to
+ * create dumb buffers suitable for scanout, which can then be used to create
+ * KMS frame buffers.
+ *
+ * To support dumb objects drivers must implement the dumb_create,
+ * dumb_destroy and dumb_map_offset operations from struct &drm_driver. See
+ * there for further details.
+ *
+ * Note that dumb objects may not be used for gpu acceleration, as has been
+ * attempted on some ARM embedded platforms. Such drivers really must have
+ * a hardware-specific ioctl to allocate suitable buffer objects.
+ */
+
+int drm_mode_create_dumb_ioctl(struct drm_device *dev,
+			       void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_create_dumb *args = data;
+	u32 cpp, stride, size;
+
+	if (!dev->driver->dumb_create)
+		return -ENOSYS;
+	if (!args->width || !args->height || !args->bpp)
+		return -EINVAL;
+
+	/* overflow checks for 32bit size calculations */
+	/* NOTE: DIV_ROUND_UP() can overflow */
+	cpp = DIV_ROUND_UP(args->bpp, 8);
+	if (!cpp || cpp > 0xffffffffU / args->width)
+		return -EINVAL;
+	stride = cpp * args->width;
+	if (args->height > 0xffffffffU / stride)
+		return -EINVAL;
+
+	/* test for wrap-around */
+	size = args->height * stride;
+	if (PAGE_ALIGN(size) == 0)
+		return -EINVAL;
+
+	/*
+	 * handle, pitch and size are output parameters. Zero them out to
+	 * prevent drivers from accidentally using uninitialized data. Since
+	 * not all existing userspace is clearing these fields properly we
+	 * cannot reject IOCTL with garbage in them.
+	 */
+	args->handle = 0;
+	args->pitch = 0;
+	args->size = 0;
+
+	return dev->driver->dumb_create(file_priv, dev, args);
+}
+
+/**
+ * drm_mode_mmap_dumb_ioctl - create an mmap offset for a dumb backing storage buffer
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * Allocate an offset in the drm device node's address space to be able to
+ * memory map a dumb buffer.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_mmap_dumb_ioctl(struct drm_device *dev,
+			     void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_map_dumb *args = data;
+
+	/* call driver ioctl to get mmap offset */
+	if (!dev->driver->dumb_map_offset)
+		return -ENOSYS;
+
+	return dev->driver->dumb_map_offset(file_priv, dev, args->handle, &args->offset);
+}
+
+int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
+				void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_destroy_dumb *args = data;
+
+	if (!dev->driver->dumb_destroy)
+		return -ENOSYS;
+
+	return dev->driver->dumb_destroy(file_priv, dev, args->handle);
+}
+
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index ec77bd3e1f08..336be31ff3de 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -957,13 +957,13 @@ static const struct drm_display_mode edid_cea_modes[] = {
 		   798, 858, 0, 480, 489, 495, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
 	  .vrefresh = 240, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, },
-	/* 58 - 720(1440)x480i@240 */
+	/* 58 - 720(1440)x480i@240Hz */
 	{ DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 54000, 720, 739,
 		   801, 858, 0, 480, 488, 494, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK),
 	  .vrefresh = 240, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3, },
-	/* 59 - 720(1440)x480i@240 */
+	/* 59 - 720(1440)x480i@240Hz */
 	{ DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 54000, 720, 739,
 		   801, 858, 0, 480, 488, 494, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
@@ -1260,6 +1260,34 @@ drm_do_probe_ddc_edid(void *data, u8 *buf, unsigned int block, size_t len)
 	return ret == xfers ? 0 : -1;
 }
 
+static void connector_bad_edid(struct drm_connector *connector,
+			       u8 *edid, int num_blocks)
+{
+	int i;
+
+	if (connector->bad_edid_counter++ && !(drm_debug & DRM_UT_KMS))
+		return;
+
+	dev_warn(connector->dev->dev,
+		 "%s: EDID is invalid:\n",
+		 connector->name);
+	for (i = 0; i < num_blocks; i++) {
+		u8 *block = edid + i * EDID_LENGTH;
+		char prefix[20];
+
+		if (drm_edid_is_zero(block, EDID_LENGTH))
+			sprintf(prefix, "\t[%02x] ZERO ", i);
+		else if (!drm_edid_block_valid(block, i, false, NULL))
+			sprintf(prefix, "\t[%02x] BAD  ", i);
+		else
+			sprintf(prefix, "\t[%02x] GOOD ", i);
+
+		print_hex_dump(KERN_WARNING,
+			       prefix, DUMP_PREFIX_NONE, 16, 1,
+			       block, EDID_LENGTH, false);
+	}
+}
+
 /**
  * drm_do_get_edid - get EDID data using a custom EDID block read function
  * @connector: connector we're probing
@@ -1282,20 +1310,19 @@ struct edid *drm_do_get_edid(struct drm_connector *connector,
 	void *data)
 {
 	int i, j = 0, valid_extensions = 0;
-	u8 *block, *new;
-	bool print_bad_edid = !connector->bad_edid_counter || (drm_debug & DRM_UT_KMS);
+	u8 *edid, *new;
 
-	if ((block = kmalloc(EDID_LENGTH, GFP_KERNEL)) == NULL)
+	if ((edid = kmalloc(EDID_LENGTH, GFP_KERNEL)) == NULL)
 		return NULL;
 
 	/* base block fetch */
 	for (i = 0; i < 4; i++) {
-		if (get_edid_block(data, block, 0, EDID_LENGTH))
+		if (get_edid_block(data, edid, 0, EDID_LENGTH))
 			goto out;
-		if (drm_edid_block_valid(block, 0, print_bad_edid,
+		if (drm_edid_block_valid(edid, 0, false,
 					 &connector->edid_corrupt))
 			break;
-		if (i == 0 && drm_edid_is_zero(block, EDID_LENGTH)) {
+		if (i == 0 && drm_edid_is_zero(edid, EDID_LENGTH)) {
 			connector->null_edid_counter++;
 			goto carp;
 		}
@@ -1304,58 +1331,62 @@ struct edid *drm_do_get_edid(struct drm_connector *connector,
 		goto carp;
 
 	/* if there's no extensions, we're done */
-	if (block[0x7e] == 0)
-		return (struct edid *)block;
+	valid_extensions = edid[0x7e];
+	if (valid_extensions == 0)
+		return (struct edid *)edid;
 
-	new = krealloc(block, (block[0x7e] + 1) * EDID_LENGTH, GFP_KERNEL);
+	new = krealloc(edid, (valid_extensions + 1) * EDID_LENGTH, GFP_KERNEL);
 	if (!new)
 		goto out;
-	block = new;
+	edid = new;
+
+	for (j = 1; j <= edid[0x7e]; j++) {
+		u8 *block = edid + j * EDID_LENGTH;
 
-	for (j = 1; j <= block[0x7e]; j++) {
 		for (i = 0; i < 4; i++) {
-			if (get_edid_block(data,
-				  block + (valid_extensions + 1) * EDID_LENGTH,
-				  j, EDID_LENGTH))
+			if (get_edid_block(data, block, j, EDID_LENGTH))
 				goto out;
-			if (drm_edid_block_valid(block + (valid_extensions + 1)
-						 * EDID_LENGTH, j,
-						 print_bad_edid,
-						 NULL)) {
-				valid_extensions++;
+			if (drm_edid_block_valid(block, j, false, NULL))
 				break;
-			}
 		}
 
-		if (i == 4 && print_bad_edid) {
-			dev_warn(connector->dev->dev,
-			 "%s: Ignoring invalid EDID block %d.\n",
-			 connector->name, j);
-
-			connector->bad_edid_counter++;
-		}
+		if (i == 4)
+			valid_extensions--;
 	}
 
-	if (valid_extensions != block[0x7e]) {
-		block[EDID_LENGTH-1] += block[0x7e] - valid_extensions;
-		block[0x7e] = valid_extensions;
-		new = krealloc(block, (valid_extensions + 1) * EDID_LENGTH, GFP_KERNEL);
+	if (valid_extensions != edid[0x7e]) {
+		u8 *base;
+
+		connector_bad_edid(connector, edid, edid[0x7e] + 1);
+
+		edid[EDID_LENGTH-1] += edid[0x7e] - valid_extensions;
+		edid[0x7e] = valid_extensions;
+
+		new = kmalloc((valid_extensions + 1) * EDID_LENGTH, GFP_KERNEL);
 		if (!new)
 			goto out;
-		block = new;
-	}
 
-	return (struct edid *)block;
+		base = new;
+		for (i = 0; i <= edid[0x7e]; i++) {
+			u8 *block = edid + i * EDID_LENGTH;
 
-carp:
-	if (print_bad_edid) {
-		dev_warn(connector->dev->dev, "%s: EDID block %d invalid.\n",
-			 connector->name, j);
+			if (!drm_edid_block_valid(block, i, false, NULL))
+				continue;
+
+			memcpy(base, block, EDID_LENGTH);
+			base += EDID_LENGTH;
+		}
+
+		kfree(edid);
+		edid = new;
 	}
-	connector->bad_edid_counter++;
 
+	return (struct edid *)edid;
+
+carp:
+	connector_bad_edid(connector, edid, 1);
 out:
-	kfree(block);
+	kfree(edid);
 	return NULL;
 }
 EXPORT_SYMBOL_GPL(drm_do_get_edid);
@@ -2582,6 +2613,41 @@ cea_mode_alternate_clock(const struct drm_display_mode *cea_mode)
 	return clock;
 }
 
+static bool
+cea_mode_alternate_timings(u8 vic, struct drm_display_mode *mode)
+{
+	/*
+	 * For certain VICs the spec allows the vertical
+	 * front porch to vary by one or two lines.
+	 *
+	 * cea_modes[] stores the variant with the shortest
+	 * vertical front porch. We can adjust the mode to
+	 * get the other variants by simply increasing the
+	 * vertical front porch length.
+	 */
+	BUILD_BUG_ON(edid_cea_modes[8].vtotal != 262 ||
+		     edid_cea_modes[9].vtotal != 262 ||
+		     edid_cea_modes[12].vtotal != 262 ||
+		     edid_cea_modes[13].vtotal != 262 ||
+		     edid_cea_modes[23].vtotal != 312 ||
+		     edid_cea_modes[24].vtotal != 312 ||
+		     edid_cea_modes[27].vtotal != 312 ||
+		     edid_cea_modes[28].vtotal != 312);
+
+	if (((vic == 8 || vic == 9 ||
+	      vic == 12 || vic == 13) && mode->vtotal < 263) ||
+	    ((vic == 23 || vic == 24 ||
+	      vic == 27 || vic == 28) && mode->vtotal < 314)) {
+		mode->vsync_start++;
+		mode->vsync_end++;
+		mode->vtotal++;
+
+		return true;
+	}
+
+	return false;
+}
+
 static u8 drm_match_cea_mode_clock_tolerance(const struct drm_display_mode *to_match,
 					     unsigned int clock_tolerance)
 {
@@ -2591,19 +2657,21 @@ static u8 drm_match_cea_mode_clock_tolerance(const struct drm_display_mode *to_m
 		return 0;
 
 	for (vic = 1; vic < ARRAY_SIZE(edid_cea_modes); vic++) {
-		const struct drm_display_mode *cea_mode = &edid_cea_modes[vic];
+		struct drm_display_mode cea_mode = edid_cea_modes[vic];
 		unsigned int clock1, clock2;
 
 		/* Check both 60Hz and 59.94Hz */
-		clock1 = cea_mode->clock;
-		clock2 = cea_mode_alternate_clock(cea_mode);
+		clock1 = cea_mode.clock;
+		clock2 = cea_mode_alternate_clock(&cea_mode);
 
 		if (abs(to_match->clock - clock1) > clock_tolerance &&
 		    abs(to_match->clock - clock2) > clock_tolerance)
 			continue;
 
-		if (drm_mode_equal_no_clocks(to_match, cea_mode))
-			return vic;
+		do {
+			if (drm_mode_equal_no_clocks_no_stereo(to_match, &cea_mode))
+				return vic;
+		} while (cea_mode_alternate_timings(vic, &cea_mode));
 	}
 
 	return 0;
@@ -2624,18 +2692,23 @@ u8 drm_match_cea_mode(const struct drm_display_mode *to_match)
 		return 0;
 
 	for (vic = 1; vic < ARRAY_SIZE(edid_cea_modes); vic++) {
-		const struct drm_display_mode *cea_mode = &edid_cea_modes[vic];
+		struct drm_display_mode cea_mode = edid_cea_modes[vic];
 		unsigned int clock1, clock2;
 
 		/* Check both 60Hz and 59.94Hz */
-		clock1 = cea_mode->clock;
-		clock2 = cea_mode_alternate_clock(cea_mode);
+		clock1 = cea_mode.clock;
+		clock2 = cea_mode_alternate_clock(&cea_mode);
 
-		if ((KHZ2PICOS(to_match->clock) == KHZ2PICOS(clock1) ||
-		     KHZ2PICOS(to_match->clock) == KHZ2PICOS(clock2)) &&
-		    drm_mode_equal_no_clocks_no_stereo(to_match, cea_mode))
-			return vic;
+		if (KHZ2PICOS(to_match->clock) != KHZ2PICOS(clock1) &&
+		    KHZ2PICOS(to_match->clock) != KHZ2PICOS(clock2))
+			continue;
+
+		do {
+			if (drm_mode_equal_no_clocks_no_stereo(to_match, &cea_mode))
+				return vic;
+		} while (cea_mode_alternate_timings(vic, &cea_mode));
 	}
+
 	return 0;
 }
 EXPORT_SYMBOL(drm_match_cea_mode);
@@ -3580,32 +3653,6 @@ int drm_av_sync_delay(struct drm_connector *connector,
 EXPORT_SYMBOL(drm_av_sync_delay);
 
 /**
- * drm_select_eld - select one ELD from multiple HDMI/DP sinks
- * @encoder: the encoder just changed display mode
- *
- * It's possible for one encoder to be associated with multiple HDMI/DP sinks.
- * The policy is now hard coded to simply use the first HDMI/DP sink's ELD.
- *
- * Return: The connector associated with the first HDMI/DP sink that has ELD
- * attached to it.
- */
-struct drm_connector *drm_select_eld(struct drm_encoder *encoder)
-{
-	struct drm_connector *connector;
-	struct drm_device *dev = encoder->dev;
-
-	WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
-	WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
-
-	drm_for_each_connector(connector, dev)
-		if (connector->encoder == encoder && connector->eld[0])
-			return connector;
-
-	return NULL;
-}
-EXPORT_SYMBOL(drm_select_eld);
-
-/**
  * drm_detect_hdmi_monitor - detect whether monitor is HDMI
  * @edid: monitor EDID information
  *
diff --git a/drivers/gpu/drm/drm_encoder.c b/drivers/gpu/drm/drm_encoder.c
index 5c067719164d..992879f15f23 100644
--- a/drivers/gpu/drm/drm_encoder.c
+++ b/drivers/gpu/drm/drm_encoder.c
@@ -110,11 +110,9 @@ int drm_encoder_init(struct drm_device *dev,
 {
 	int ret;
 
-	drm_modeset_lock_all(dev);
-
 	ret = drm_mode_object_get(dev, &encoder->base, DRM_MODE_OBJECT_ENCODER);
 	if (ret)
-		goto out_unlock;
+		return ret;
 
 	encoder->dev = dev;
 	encoder->encoder_type = encoder_type;
@@ -142,9 +140,6 @@ out_put:
 	if (ret)
 		drm_mode_object_unregister(dev, &encoder->base);
 
-out_unlock:
-	drm_modeset_unlock_all(dev);
-
 	return ret;
 }
 EXPORT_SYMBOL(drm_encoder_init);
@@ -164,12 +159,10 @@ void drm_encoder_cleanup(struct drm_encoder *encoder)
 	 * the indices on the drm_encoder after us in the encoder_list.
 	 */
 
-	drm_modeset_lock_all(dev);
 	drm_mode_object_unregister(dev, &encoder->base);
 	kfree(encoder->name);
 	list_del(&encoder->head);
 	dev->mode_config.num_encoder--;
-	drm_modeset_unlock_all(dev);
 
 	memset(encoder, 0, sizeof(*encoder));
 }
diff --git a/drivers/gpu/drm/drm_fb_cma_helper.c b/drivers/gpu/drm/drm_fb_cma_helper.c
index 1fd6eac1400c..81b3558302b5 100644
--- a/drivers/gpu/drm/drm_fb_cma_helper.c
+++ b/drivers/gpu/drm/drm_fb_cma_helper.c
@@ -18,13 +18,16 @@
  */
 
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_fb_cma_helper.h>
+#include <linux/dma-buf.h>
 #include <linux/dma-mapping.h>
 #include <linux/module.h>
+#include <linux/reservation.h>
 
 #define DEFAULT_FBDEFIO_DELAY_MS 50
 
@@ -176,20 +179,20 @@ struct drm_framebuffer *drm_fb_cma_create_with_funcs(struct drm_device *dev,
 	struct drm_file *file_priv, const struct drm_mode_fb_cmd2 *mode_cmd,
 	const struct drm_framebuffer_funcs *funcs)
 {
+	const struct drm_format_info *info;
 	struct drm_fb_cma *fb_cma;
 	struct drm_gem_cma_object *objs[4];
 	struct drm_gem_object *obj;
-	unsigned int hsub;
-	unsigned int vsub;
 	int ret;
 	int i;
 
-	hsub = drm_format_horz_chroma_subsampling(mode_cmd->pixel_format);
-	vsub = drm_format_vert_chroma_subsampling(mode_cmd->pixel_format);
+	info = drm_format_info(mode_cmd->pixel_format);
+	if (!info)
+		return ERR_PTR(-EINVAL);
 
-	for (i = 0; i < drm_format_num_planes(mode_cmd->pixel_format); i++) {
-		unsigned int width = mode_cmd->width / (i ? hsub : 1);
-		unsigned int height = mode_cmd->height / (i ? vsub : 1);
+	for (i = 0; i < info->num_planes; i++) {
+		unsigned int width = mode_cmd->width / (i ? info->hsub : 1);
+		unsigned int height = mode_cmd->height / (i ? info->vsub : 1);
 		unsigned int min_size;
 
 		obj = drm_gem_object_lookup(file_priv, mode_cmd->handles[i]);
@@ -200,7 +203,7 @@ struct drm_framebuffer *drm_fb_cma_create_with_funcs(struct drm_device *dev,
 		}
 
 		min_size = (height - 1) * mode_cmd->pitches[i]
-			 + width * drm_format_plane_cpp(mode_cmd->pixel_format, i)
+			 + width * info->cpp[i]
 			 + mode_cmd->offsets[i];
 
 		if (obj->size < min_size) {
@@ -265,16 +268,51 @@ struct drm_gem_cma_object *drm_fb_cma_get_gem_obj(struct drm_framebuffer *fb,
 }
 EXPORT_SYMBOL_GPL(drm_fb_cma_get_gem_obj);
 
+/**
+ * drm_fb_cma_prepare_fb() - Prepare CMA framebuffer
+ * @plane: Which plane
+ * @state: Plane state attach fence to
+ *
+ * This should be put into prepare_fb hook of struct &drm_plane_helper_funcs .
+ *
+ * This function checks if the plane FB has an dma-buf attached, extracts
+ * the exclusive fence and attaches it to plane state for the atomic helper
+ * to wait on.
+ *
+ * There is no need for cleanup_fb for CMA based framebuffer drivers.
+ */
+int drm_fb_cma_prepare_fb(struct drm_plane *plane,
+			  struct drm_plane_state *state)
+{
+	struct dma_buf *dma_buf;
+	struct dma_fence *fence;
+
+	if ((plane->state->fb == state->fb) || !state->fb)
+		return 0;
+
+	dma_buf = drm_fb_cma_get_gem_obj(state->fb, 0)->base.dma_buf;
+	if (dma_buf) {
+		fence = reservation_object_get_excl_rcu(dma_buf->resv);
+		drm_atomic_set_fence_for_plane(state, fence);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(drm_fb_cma_prepare_fb);
+
 #ifdef CONFIG_DEBUG_FS
 static void drm_fb_cma_describe(struct drm_framebuffer *fb, struct seq_file *m)
 {
 	struct drm_fb_cma *fb_cma = to_fb_cma(fb);
-	int i, n = drm_format_num_planes(fb->pixel_format);
+	const struct drm_format_info *info;
+	int i;
 
 	seq_printf(m, "fb: %dx%d@%4.4s\n", fb->width, fb->height,
 			(char *)&fb->pixel_format);
 
-	for (i = 0; i < n; i++) {
+	info = drm_format_info(fb->pixel_format);
+
+	for (i = 0; i < info->num_planes; i++) {
 		seq_printf(m, "   %d: offset=%d pitch=%d, obj: ",
 				i, fb->offsets[i], fb->pitches[i]);
 		drm_gem_cma_describe(fb_cma->obj[i], m);
@@ -311,14 +349,10 @@ static int drm_fb_cma_mmap(struct fb_info *info, struct vm_area_struct *vma)
 
 static struct fb_ops drm_fbdev_cma_ops = {
 	.owner		= THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_fillrect	= drm_fb_helper_sys_fillrect,
 	.fb_copyarea	= drm_fb_helper_sys_copyarea,
 	.fb_imageblit	= drm_fb_helper_sys_imageblit,
-	.fb_check_var	= drm_fb_helper_check_var,
-	.fb_set_par	= drm_fb_helper_set_par,
-	.fb_blank	= drm_fb_helper_blank,
-	.fb_pan_display	= drm_fb_helper_pan_display,
-	.fb_setcmap	= drm_fb_helper_setcmap,
 	.fb_mmap	= drm_fb_cma_mmap,
 };
 
@@ -557,7 +591,8 @@ EXPORT_SYMBOL_GPL(drm_fbdev_cma_init);
 void drm_fbdev_cma_fini(struct drm_fbdev_cma *fbdev_cma)
 {
 	drm_fb_helper_unregister_fbi(&fbdev_cma->fb_helper);
-	drm_fbdev_cma_defio_fini(fbdev_cma->fb_helper.fbdev);
+	if (fbdev_cma->fb_helper.fbdev)
+		drm_fbdev_cma_defio_fini(fbdev_cma->fb_helper.fbdev);
 	drm_fb_helper_release_fbi(&fbdev_cma->fb_helper);
 
 	if (fbdev_cma->fb) {
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 6c75e62c0b22..e934b541feea 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -49,6 +49,7 @@ MODULE_PARM_DESC(fbdev_emulation,
 		 "Enable legacy fbdev emulation [default=true]");
 
 static LIST_HEAD(kernel_fb_helper_list);
+static DEFINE_MUTEX(kernel_fb_helper_lock);
 
 /**
  * DOC: fbdev helpers
@@ -97,6 +98,10 @@ static LIST_HEAD(kernel_fb_helper_list);
  * mmap page writes.
  */
 
+#define drm_fb_helper_for_each_connector(fbh, i__) \
+	for (({ lockdep_assert_held(&(fbh)->dev->mode_config.mutex); }), \
+	     i__ = 0; i__ < (fbh)->connector_count; i__++)
+
 /**
  * drm_fb_helper_single_add_all_connectors() - add all connectors to fbdev
  * 					       emulation helper
@@ -130,7 +135,7 @@ int drm_fb_helper_single_add_all_connectors(struct drm_fb_helper *fb_helper)
 	mutex_unlock(&dev->mode_config.mutex);
 	return 0;
 fail:
-	for (i = 0; i < fb_helper->connector_count; i++) {
+	drm_fb_helper_for_each_connector(fb_helper, i) {
 		struct drm_fb_helper_connector *fb_helper_connector =
 			fb_helper->connector_info[i];
 
@@ -256,6 +261,9 @@ int drm_fb_helper_debug_enter(struct fb_info *info)
 				continue;
 
 			funcs =	mode_set->crtc->helper_private;
+			if (funcs->mode_set_base_atomic == NULL)
+				continue;
+
 			drm_fb_helper_save_lut_atomic(mode_set->crtc, helper);
 			funcs->mode_set_base_atomic(mode_set->crtc,
 						    mode_set->fb,
@@ -309,6 +317,9 @@ int drm_fb_helper_debug_leave(struct fb_info *info)
 			continue;
 		}
 
+		if (funcs->mode_set_base_atomic == NULL)
+			continue;
+
 		drm_fb_helper_restore_lut_atomic(mode_set->crtc);
 		funcs->mode_set_base_atomic(mode_set->crtc, fb, crtc->x,
 					    crtc->y, LEAVE_ATOMIC_MODE_SET);
@@ -372,9 +383,7 @@ fail:
 	if (ret == -EDEADLK)
 		goto backoff;
 
-	if (ret != 0)
-		drm_atomic_state_free(state);
-
+	drm_atomic_state_put(state);
 	return ret;
 
 backoff:
@@ -399,11 +408,10 @@ static int restore_fbdev_mode(struct drm_fb_helper *fb_helper)
 		if (plane->type != DRM_PLANE_TYPE_PRIMARY)
 			drm_plane_force_disable(plane);
 
-		if (dev->mode_config.rotation_property) {
+		if (plane->rotation_property)
 			drm_mode_plane_set_obj_prop(plane,
-						    dev->mode_config.rotation_property,
+						    plane->rotation_property,
 						    DRM_ROTATE_0);
-		}
 	}
 
 	for (i = 0; i < fb_helper->crtc_count; i++) {
@@ -562,7 +570,7 @@ static void drm_fb_helper_dpms(struct fb_info *info, int dpms_mode)
 			continue;
 
 		/* Walk the connectors & encoders on this fb turning them on/off */
-		for (j = 0; j < fb_helper->connector_count; j++) {
+		drm_fb_helper_for_each_connector(fb_helper, j) {
 			connector = fb_helper->connector_info[j]->connector;
 			connector->funcs->dpms(connector, dpms_mode);
 			drm_object_property_set_value(&connector->base,
@@ -848,12 +856,14 @@ void drm_fb_helper_fini(struct drm_fb_helper *fb_helper)
 	if (!drm_fbdev_emulation)
 		return;
 
+	mutex_lock(&kernel_fb_helper_lock);
 	if (!list_empty(&fb_helper->kernel_fb_list)) {
 		list_del(&fb_helper->kernel_fb_list);
 		if (list_empty(&kernel_fb_helper_list)) {
 			unregister_sysrq_key('v', &sysrq_drm_fb_helper_restore_op);
 		}
 	}
+	mutex_unlock(&kernel_fb_helper_lock);
 
 	drm_fb_helper_crtc_free(fb_helper);
 
@@ -1238,11 +1248,14 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var,
 	if (var->pixclock != 0 || in_dbg_master())
 		return -EINVAL;
 
-	/* Need to resize the fb object !!! */
-	if (var->bits_per_pixel > fb->bits_per_pixel ||
-	    var->xres > fb->width || var->yres > fb->height ||
-	    var->xres_virtual > fb->width || var->yres_virtual > fb->height) {
-		DRM_DEBUG("fb userspace requested width/height/bpp is greater than current fb "
+	/*
+	 * Changes struct fb_var_screeninfo are currently not pushed back
+	 * to KMS, hence fail if different settings are requested.
+	 */
+	if (var->bits_per_pixel != fb->bits_per_pixel ||
+	    var->xres != fb->width || var->yres != fb->height ||
+	    var->xres_virtual != fb->width || var->yres_virtual != fb->height) {
+		DRM_DEBUG("fb userspace requested width/height/bpp different than current fb "
 			  "request %dx%d-%d (virtual %dx%d) > %dx%d-%d\n",
 			  var->xres, var->yres, var->bits_per_pixel,
 			  var->xres_virtual, var->yres_virtual,
@@ -1388,16 +1401,13 @@ retry:
 	info->var.xoffset = var->xoffset;
 	info->var.yoffset = var->yoffset;
 
-
 fail:
 	drm_atomic_clean_old_fb(dev, plane_mask, ret);
 
 	if (ret == -EDEADLK)
 		goto backoff;
 
-	if (ret != 0)
-		drm_atomic_state_free(state);
-
+	drm_atomic_state_put(state);
 	return ret;
 
 backoff:
@@ -1466,7 +1476,6 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 	int ret = 0;
 	int crtc_count = 0;
 	int i;
-	struct fb_info *info;
 	struct drm_fb_helper_surface_size sizes;
 	int gamma_size = 0;
 
@@ -1482,7 +1491,7 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 		sizes.surface_depth = sizes.surface_bpp = preferred_bpp;
 
 	/* first up get a count of crtcs now in use and new min/maxes width/heights */
-	for (i = 0; i < fb_helper->connector_count; i++) {
+	drm_fb_helper_for_each_connector(fb_helper, i) {
 		struct drm_fb_helper_connector *fb_helper_conn = fb_helper->connector_info[i];
 		struct drm_cmdline_mode *cmdline_mode;
 
@@ -1569,8 +1578,6 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 	if (ret < 0)
 		return ret;
 
-	info = fb_helper->fbdev;
-
 	/*
 	 * Set the fb pointer - usually drm_setup_crtcs does this for hotplug
 	 * events, but at init time drm_setup_crtcs needs to be called before
@@ -1582,20 +1589,6 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 		if (fb_helper->crtc_info[i].mode_set.num_connectors)
 			fb_helper->crtc_info[i].mode_set.fb = fb_helper->fb;
 
-
-	info->var.pixclock = 0;
-	if (register_framebuffer(info) < 0)
-		return -EINVAL;
-
-	dev_info(fb_helper->dev->dev, "fb%d: %s frame buffer device\n",
-			info->node, info->fix.id);
-
-	if (list_empty(&kernel_fb_helper_list)) {
-		register_sysrq_key('v', &sysrq_drm_fb_helper_restore_op);
-	}
-
-	list_add(&fb_helper->kernel_fb_list, &kernel_fb_helper_list);
-
 	return 0;
 }
 
@@ -1727,7 +1720,7 @@ static int drm_fb_helper_probe_connector_modes(struct drm_fb_helper *fb_helper,
 	int count = 0;
 	int i;
 
-	for (i = 0; i < fb_helper->connector_count; i++) {
+	drm_fb_helper_for_each_connector(fb_helper, i) {
 		connector = fb_helper->connector_info[i]->connector;
 		count += connector->funcs->fill_modes(connector, maxX, maxY);
 	}
@@ -1827,7 +1820,7 @@ static void drm_enable_connectors(struct drm_fb_helper *fb_helper,
 	struct drm_connector *connector;
 	int i = 0;
 
-	for (i = 0; i < fb_helper->connector_count; i++) {
+	drm_fb_helper_for_each_connector(fb_helper, i) {
 		connector = fb_helper->connector_info[i]->connector;
 		enabled[i] = drm_connector_enabled(connector, true);
 		DRM_DEBUG_KMS("connector %d enabled? %s\n", connector->base.id,
@@ -1838,7 +1831,7 @@ static void drm_enable_connectors(struct drm_fb_helper *fb_helper,
 	if (any_enabled)
 		return;
 
-	for (i = 0; i < fb_helper->connector_count; i++) {
+	drm_fb_helper_for_each_connector(fb_helper, i) {
 		connector = fb_helper->connector_info[i]->connector;
 		enabled[i] = drm_connector_enabled(connector, false);
 	}
@@ -1859,7 +1852,7 @@ static bool drm_target_cloned(struct drm_fb_helper *fb_helper,
 		return false;
 
 	count = 0;
-	for (i = 0; i < fb_helper->connector_count; i++) {
+	drm_fb_helper_for_each_connector(fb_helper, i) {
 		if (enabled[i])
 			count++;
 	}
@@ -1870,7 +1863,7 @@ static bool drm_target_cloned(struct drm_fb_helper *fb_helper,
 
 	/* check the command line or if nothing common pick 1024x768 */
 	can_clone = true;
-	for (i = 0; i < fb_helper->connector_count; i++) {
+	drm_fb_helper_for_each_connector(fb_helper, i) {
 		if (!enabled[i])
 			continue;
 		fb_helper_conn = fb_helper->connector_info[i];
@@ -1896,8 +1889,7 @@ static bool drm_target_cloned(struct drm_fb_helper *fb_helper,
 	can_clone = true;
 	dmt_mode = drm_mode_find_dmt(fb_helper->dev, 1024, 768, 60, false);
 
-	for (i = 0; i < fb_helper->connector_count; i++) {
-
+	drm_fb_helper_for_each_connector(fb_helper, i) {
 		if (!enabled[i])
 			continue;
 
@@ -1928,7 +1920,7 @@ static int drm_get_tile_offsets(struct drm_fb_helper *fb_helper,
 	int i;
 	int hoffset = 0, voffset = 0;
 
-	for (i = 0; i < fb_helper->connector_count; i++) {
+	drm_fb_helper_for_each_connector(fb_helper, i) {
 		fb_helper_conn = fb_helper->connector_info[i];
 		if (!fb_helper_conn->connector->has_tile)
 			continue;
@@ -1956,19 +1948,20 @@ static bool drm_target_preferred(struct drm_fb_helper *fb_helper,
 				 bool *enabled, int width, int height)
 {
 	struct drm_fb_helper_connector *fb_helper_conn;
-	int i;
-	uint64_t conn_configured = 0, mask;
+	const u64 mask = BIT_ULL(fb_helper->connector_count) - 1;
+	u64 conn_configured = 0;
 	int tile_pass = 0;
-	mask = (1 << fb_helper->connector_count) - 1;
+	int i;
+
 retry:
-	for (i = 0; i < fb_helper->connector_count; i++) {
+	drm_fb_helper_for_each_connector(fb_helper, i) {
 		fb_helper_conn = fb_helper->connector_info[i];
 
-		if (conn_configured & (1 << i))
+		if (conn_configured & BIT_ULL(i))
 			continue;
 
 		if (enabled[i] == false) {
-			conn_configured |= (1 << i);
+			conn_configured |= BIT_ULL(i);
 			continue;
 		}
 
@@ -2009,7 +2002,7 @@ retry:
 		}
 		DRM_DEBUG_KMS("found mode %s\n", modes[i] ? modes[i]->name :
 			  "none");
-		conn_configured |= (1 << i);
+		conn_configured |= BIT_ULL(i);
 	}
 
 	if ((conn_configured & mask) != mask) {
@@ -2109,20 +2102,22 @@ out:
 	return best_score;
 }
 
-static void drm_setup_crtcs(struct drm_fb_helper *fb_helper)
+static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
+			    u32 width, u32 height)
 {
 	struct drm_device *dev = fb_helper->dev;
 	struct drm_fb_helper_crtc **crtcs;
 	struct drm_display_mode **modes;
 	struct drm_fb_offset *offsets;
 	bool *enabled;
-	int width, height;
 	int i;
 
 	DRM_DEBUG_KMS("\n");
+	if (drm_fb_helper_probe_connector_modes(fb_helper, width, height) == 0)
+		DRM_DEBUG_KMS("No connectors reported connected with modes\n");
 
-	width = dev->mode_config.max_width;
-	height = dev->mode_config.max_height;
+	/* prevent concurrent modification of connector_count by hotplug */
+	lockdep_assert_held(&fb_helper->dev->mode_config.mutex);
 
 	crtcs = kcalloc(fb_helper->connector_count,
 			sizeof(struct drm_fb_helper_crtc *), GFP_KERNEL);
@@ -2137,7 +2132,6 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper)
 		goto out;
 	}
 
-
 	drm_enable_connectors(fb_helper, enabled);
 
 	if (!(fb_helper->funcs->initial_config &&
@@ -2166,7 +2160,7 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper)
 		drm_fb_helper_modeset_release(fb_helper,
 					      &fb_helper->crtc_info[i].mode_set);
 
-	for (i = 0; i < fb_helper->connector_count; i++) {
+	drm_fb_helper_for_each_connector(fb_helper, i) {
 		struct drm_display_mode *mode = modes[i];
 		struct drm_fb_helper_crtc *fb_crtc = crtcs[i];
 		struct drm_fb_offset *offset = &offsets[i];
@@ -2243,25 +2237,38 @@ out:
 int drm_fb_helper_initial_config(struct drm_fb_helper *fb_helper, int bpp_sel)
 {
 	struct drm_device *dev = fb_helper->dev;
-	int count = 0;
+	struct fb_info *info;
+	int ret;
 
 	if (!drm_fbdev_emulation)
 		return 0;
 
 	mutex_lock(&dev->mode_config.mutex);
-	count = drm_fb_helper_probe_connector_modes(fb_helper,
-						    dev->mode_config.max_width,
-						    dev->mode_config.max_height);
+	drm_setup_crtcs(fb_helper,
+			dev->mode_config.max_width,
+			dev->mode_config.max_height);
+	ret = drm_fb_helper_single_fb_probe(fb_helper, bpp_sel);
 	mutex_unlock(&dev->mode_config.mutex);
-	/*
-	 * we shouldn't end up with no modes here.
-	 */
-	if (count == 0)
-		dev_info(fb_helper->dev->dev, "No connectors reported connected with modes\n");
+	if (ret)
+		return ret;
+
+	info = fb_helper->fbdev;
+	info->var.pixclock = 0;
+	ret = register_framebuffer(info);
+	if (ret < 0)
+		return ret;
+
+	dev_info(dev->dev, "fb%d: %s frame buffer device\n",
+		 info->node, info->fix.id);
+
+	mutex_lock(&kernel_fb_helper_lock);
+	if (list_empty(&kernel_fb_helper_list))
+		register_sysrq_key('v', &sysrq_drm_fb_helper_restore_op);
 
-	drm_setup_crtcs(fb_helper);
+	list_add(&fb_helper->kernel_fb_list, &kernel_fb_helper_list);
+	mutex_unlock(&kernel_fb_helper_lock);
 
-	return drm_fb_helper_single_fb_probe(fb_helper, bpp_sel);
+	return 0;
 }
 EXPORT_SYMBOL(drm_fb_helper_initial_config);
 
@@ -2289,28 +2296,22 @@ EXPORT_SYMBOL(drm_fb_helper_initial_config);
 int drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
 {
 	struct drm_device *dev = fb_helper->dev;
-	u32 max_width, max_height;
 
 	if (!drm_fbdev_emulation)
 		return 0;
 
-	mutex_lock(&fb_helper->dev->mode_config.mutex);
+	mutex_lock(&dev->mode_config.mutex);
 	if (!fb_helper->fb || !drm_fb_helper_is_bound(fb_helper)) {
 		fb_helper->delayed_hotplug = true;
-		mutex_unlock(&fb_helper->dev->mode_config.mutex);
+		mutex_unlock(&dev->mode_config.mutex);
 		return 0;
 	}
 	DRM_DEBUG_KMS("\n");
 
-	max_width = fb_helper->fb->width;
-	max_height = fb_helper->fb->height;
+	drm_setup_crtcs(fb_helper, fb_helper->fb->width, fb_helper->fb->height);
 
-	drm_fb_helper_probe_connector_modes(fb_helper, max_width, max_height);
-	mutex_unlock(&fb_helper->dev->mode_config.mutex);
+	mutex_unlock(&dev->mode_config.mutex);
 
-	drm_modeset_lock_all(dev);
-	drm_setup_crtcs(fb_helper);
-	drm_modeset_unlock_all(dev);
 	drm_fb_helper_set_par(fb_helper->fbdev);
 
 	return 0;
diff --git a/drivers/gpu/drm/drm_fops.c b/drivers/gpu/drm/drm_fops.c
index e84faecf5225..5d96de40b63f 100644
--- a/drivers/gpu/drm/drm_fops.c
+++ b/drivers/gpu/drm/drm_fops.c
@@ -51,10 +51,11 @@ DEFINE_MUTEX(drm_global_mutex);
  * Drivers must define the file operations structure that forms the DRM
  * userspace API entry point, even though most of those operations are
  * implemented in the DRM core. The mandatory functions are drm_open(),
- * drm_read(), drm_ioctl() and drm_compat_ioctl if CONFIG_COMPAT is enabled.
- * Drivers which implement private ioctls that require 32/64 bit compatibility
- * support must provided their onw .compat_ioctl() handler that processes
- * private ioctls and calls drm_compat_ioctl() for core ioctls.
+ * drm_read(), drm_ioctl() and drm_compat_ioctl() if CONFIG_COMPAT is enabled
+ * (note that drm_compat_ioctl will be NULL if CONFIG_COMPAT=n). Drivers which
+ * implement private ioctls that require 32/64 bit compatibility support must
+ * provide their own .compat_ioctl() handler that processes private ioctls and
+ * calls drm_compat_ioctl() for core ioctls.
  *
  * In addition drm_read() and drm_poll() provide support for DRM events. DRM
  * events are a generic and extensible means to send asynchronous events to
@@ -75,9 +76,7 @@ DEFINE_MUTEX(drm_global_mutex);
  *             .open = drm_open,
  *             .release = drm_release,
  *             .unlocked_ioctl = drm_ioctl,
- *     #ifdef CONFIG_COMPAT
- *             .compat_ioctl = drm_compat_ioctl,
- *     #endif
+ *             .compat_ioctl = drm_compat_ioctl, // NULL if CONFIG_COMPAT=n
  *             .poll = drm_poll,
  *             .read = drm_read,
  *             .llseek = no_llseek,
@@ -663,6 +662,10 @@ void drm_event_cancel_free(struct drm_device *dev,
 		list_del(&p->pending_link);
 	}
 	spin_unlock_irqrestore(&dev->event_lock, flags);
+
+	if (p->fence)
+		dma_fence_put(p->fence);
+
 	kfree(p);
 }
 EXPORT_SYMBOL(drm_event_cancel_free);
@@ -692,8 +695,8 @@ void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e)
 	}
 
 	if (e->fence) {
-		fence_signal(e->fence);
-		fence_put(e->fence);
+		dma_fence_signal(e->fence);
+		dma_fence_put(e->fence);
 	}
 
 	if (!e->file_priv) {
diff --git a/drivers/gpu/drm/drm_fourcc.c b/drivers/gpu/drm/drm_fourcc.c
index 29c56b4331e0..90d2cc8da8eb 100644
--- a/drivers/gpu/drm/drm_fourcc.c
+++ b/drivers/gpu/drm/drm_fourcc.c
@@ -79,17 +79,13 @@ uint32_t drm_mode_legacy_fb_format(uint32_t bpp, uint32_t depth)
 EXPORT_SYMBOL(drm_mode_legacy_fb_format);
 
 /**
- * drm_get_format_name - return a string for drm fourcc format
+ * drm_get_format_name - fill a string with a drm fourcc format's name
  * @format: format to compute name of
- *
- * Note that the buffer returned by this function is owned by the caller
- * and will need to be freed using kfree().
+ * @buf: caller-supplied buffer
  */
-char *drm_get_format_name(uint32_t format)
+const char *drm_get_format_name(uint32_t format, struct drm_format_name_buf *buf)
 {
-	char *buf = kmalloc(32, GFP_KERNEL);
-
-	snprintf(buf, 32,
+	snprintf(buf->str, sizeof(buf->str),
 		 "%c%c%c%c %s-endian (0x%08x)",
 		 printable_char(format & 0xff),
 		 printable_char((format >> 8) & 0xff),
@@ -98,87 +94,109 @@ char *drm_get_format_name(uint32_t format)
 		 format & DRM_FORMAT_BIG_ENDIAN ? "big" : "little",
 		 format);
 
-	return buf;
+	return buf->str;
 }
 EXPORT_SYMBOL(drm_get_format_name);
 
+/*
+ * Internal function to query information for a given format. See
+ * drm_format_info() for the public API.
+ */
+const struct drm_format_info *__drm_format_info(u32 format)
+{
+	static const struct drm_format_info formats[] = {
+		{ .format = DRM_FORMAT_C8,		.depth = 8,  .num_planes = 1, .cpp = { 1, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_RGB332,		.depth = 8,  .num_planes = 1, .cpp = { 1, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_BGR233,		.depth = 8,  .num_planes = 1, .cpp = { 1, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_XRGB4444,	.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_XBGR4444,	.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_RGBX4444,	.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_BGRX4444,	.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_ARGB4444,	.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_ABGR4444,	.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_RGBA4444,	.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_BGRA4444,	.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_XRGB1555,	.depth = 15, .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_XBGR1555,	.depth = 15, .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_RGBX5551,	.depth = 15, .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_BGRX5551,	.depth = 15, .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_ARGB1555,	.depth = 15, .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_ABGR1555,	.depth = 15, .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_RGBA5551,	.depth = 15, .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_BGRA5551,	.depth = 15, .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_RGB565,		.depth = 16, .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_BGR565,		.depth = 16, .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_RGB888,		.depth = 24, .num_planes = 1, .cpp = { 3, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_BGR888,		.depth = 24, .num_planes = 1, .cpp = { 3, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_XRGB8888,	.depth = 24, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_XBGR8888,	.depth = 24, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_RGBX8888,	.depth = 24, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_BGRX8888,	.depth = 24, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_XRGB2101010,	.depth = 30, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_XBGR2101010,	.depth = 30, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_RGBX1010102,	.depth = 30, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_BGRX1010102,	.depth = 30, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_ARGB2101010,	.depth = 30, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_ABGR2101010,	.depth = 30, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_RGBA1010102,	.depth = 30, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_BGRA1010102,	.depth = 30, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_ARGB8888,	.depth = 32, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_ABGR8888,	.depth = 32, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_RGBA8888,	.depth = 32, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_BGRA8888,	.depth = 32, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_YUV410,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 4, .vsub = 4 },
+		{ .format = DRM_FORMAT_YVU410,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 4, .vsub = 4 },
+		{ .format = DRM_FORMAT_YUV411,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 4, .vsub = 1 },
+		{ .format = DRM_FORMAT_YVU411,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 4, .vsub = 1 },
+		{ .format = DRM_FORMAT_YUV420,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 2, .vsub = 2 },
+		{ .format = DRM_FORMAT_YVU420,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 2, .vsub = 2 },
+		{ .format = DRM_FORMAT_YUV422,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 2, .vsub = 1 },
+		{ .format = DRM_FORMAT_YVU422,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 2, .vsub = 1 },
+		{ .format = DRM_FORMAT_YUV444,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_YVU444,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_NV12,		.depth = 0,  .num_planes = 2, .cpp = { 1, 2, 0 }, .hsub = 2, .vsub = 2 },
+		{ .format = DRM_FORMAT_NV21,		.depth = 0,  .num_planes = 2, .cpp = { 1, 2, 0 }, .hsub = 2, .vsub = 2 },
+		{ .format = DRM_FORMAT_NV16,		.depth = 0,  .num_planes = 2, .cpp = { 1, 2, 0 }, .hsub = 2, .vsub = 1 },
+		{ .format = DRM_FORMAT_NV61,		.depth = 0,  .num_planes = 2, .cpp = { 1, 2, 0 }, .hsub = 2, .vsub = 1 },
+		{ .format = DRM_FORMAT_NV24,		.depth = 0,  .num_planes = 2, .cpp = { 1, 2, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_NV42,		.depth = 0,  .num_planes = 2, .cpp = { 1, 2, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_YUYV,		.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 2, .vsub = 1 },
+		{ .format = DRM_FORMAT_YVYU,		.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 2, .vsub = 1 },
+		{ .format = DRM_FORMAT_UYVY,		.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 2, .vsub = 1 },
+		{ .format = DRM_FORMAT_VYUY,		.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 2, .vsub = 1 },
+		{ .format = DRM_FORMAT_AYUV,		.depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1 },
+	};
+
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(formats); ++i) {
+		if (formats[i].format == format)
+			return &formats[i];
+	}
+
+	return NULL;
+}
+
 /**
- * drm_fb_get_bpp_depth - get the bpp/depth values for format
+ * drm_format_info - query information for a given format
  * @format: pixel format (DRM_FORMAT_*)
- * @depth: storage for the depth value
- * @bpp: storage for the bpp value
  *
- * This only supports RGB formats here for compat with code that doesn't use
- * pixel formats directly yet.
+ * The caller should only pass a supported pixel format to this function.
+ * Unsupported pixel formats will generate a warning in the kernel log.
+ *
+ * Returns:
+ * The instance of struct drm_format_info that describes the pixel format, or
+ * NULL if the format is unsupported.
  */
-void drm_fb_get_bpp_depth(uint32_t format, unsigned int *depth,
-			  int *bpp)
+const struct drm_format_info *drm_format_info(u32 format)
 {
-	char *format_name;
+	const struct drm_format_info *info;
 
-	switch (format) {
-	case DRM_FORMAT_C8:
-	case DRM_FORMAT_RGB332:
-	case DRM_FORMAT_BGR233:
-		*depth = 8;
-		*bpp = 8;
-		break;
-	case DRM_FORMAT_XRGB1555:
-	case DRM_FORMAT_XBGR1555:
-	case DRM_FORMAT_RGBX5551:
-	case DRM_FORMAT_BGRX5551:
-	case DRM_FORMAT_ARGB1555:
-	case DRM_FORMAT_ABGR1555:
-	case DRM_FORMAT_RGBA5551:
-	case DRM_FORMAT_BGRA5551:
-		*depth = 15;
-		*bpp = 16;
-		break;
-	case DRM_FORMAT_RGB565:
-	case DRM_FORMAT_BGR565:
-		*depth = 16;
-		*bpp = 16;
-		break;
-	case DRM_FORMAT_RGB888:
-	case DRM_FORMAT_BGR888:
-		*depth = 24;
-		*bpp = 24;
-		break;
-	case DRM_FORMAT_XRGB8888:
-	case DRM_FORMAT_XBGR8888:
-	case DRM_FORMAT_RGBX8888:
-	case DRM_FORMAT_BGRX8888:
-		*depth = 24;
-		*bpp = 32;
-		break;
-	case DRM_FORMAT_XRGB2101010:
-	case DRM_FORMAT_XBGR2101010:
-	case DRM_FORMAT_RGBX1010102:
-	case DRM_FORMAT_BGRX1010102:
-	case DRM_FORMAT_ARGB2101010:
-	case DRM_FORMAT_ABGR2101010:
-	case DRM_FORMAT_RGBA1010102:
-	case DRM_FORMAT_BGRA1010102:
-		*depth = 30;
-		*bpp = 32;
-		break;
-	case DRM_FORMAT_ARGB8888:
-	case DRM_FORMAT_ABGR8888:
-	case DRM_FORMAT_RGBA8888:
-	case DRM_FORMAT_BGRA8888:
-		*depth = 32;
-		*bpp = 32;
-		break;
-	default:
-		format_name = drm_get_format_name(format);
-		DRM_DEBUG_KMS("unsupported pixel format %s\n", format_name);
-		kfree(format_name);
-		*depth = 0;
-		*bpp = 0;
-		break;
-	}
+	info = __drm_format_info(format);
+	WARN_ON(!info);
+	return info;
 }
-EXPORT_SYMBOL(drm_fb_get_bpp_depth);
+EXPORT_SYMBOL(drm_format_info);
 
 /**
  * drm_format_num_planes - get the number of planes for format
@@ -189,28 +207,10 @@ EXPORT_SYMBOL(drm_fb_get_bpp_depth);
  */
 int drm_format_num_planes(uint32_t format)
 {
-	switch (format) {
-	case DRM_FORMAT_YUV410:
-	case DRM_FORMAT_YVU410:
-	case DRM_FORMAT_YUV411:
-	case DRM_FORMAT_YVU411:
-	case DRM_FORMAT_YUV420:
-	case DRM_FORMAT_YVU420:
-	case DRM_FORMAT_YUV422:
-	case DRM_FORMAT_YVU422:
-	case DRM_FORMAT_YUV444:
-	case DRM_FORMAT_YVU444:
-		return 3;
-	case DRM_FORMAT_NV12:
-	case DRM_FORMAT_NV21:
-	case DRM_FORMAT_NV16:
-	case DRM_FORMAT_NV61:
-	case DRM_FORMAT_NV24:
-	case DRM_FORMAT_NV42:
-		return 2;
-	default:
-		return 1;
-	}
+	const struct drm_format_info *info;
+
+	info = drm_format_info(format);
+	return info ? info->num_planes : 1;
 }
 EXPORT_SYMBOL(drm_format_num_planes);
 
@@ -224,40 +224,13 @@ EXPORT_SYMBOL(drm_format_num_planes);
  */
 int drm_format_plane_cpp(uint32_t format, int plane)
 {
-	unsigned int depth;
-	int bpp;
+	const struct drm_format_info *info;
 
-	if (plane >= drm_format_num_planes(format))
+	info = drm_format_info(format);
+	if (!info || plane >= info->num_planes)
 		return 0;
 
-	switch (format) {
-	case DRM_FORMAT_YUYV:
-	case DRM_FORMAT_YVYU:
-	case DRM_FORMAT_UYVY:
-	case DRM_FORMAT_VYUY:
-		return 2;
-	case DRM_FORMAT_NV12:
-	case DRM_FORMAT_NV21:
-	case DRM_FORMAT_NV16:
-	case DRM_FORMAT_NV61:
-	case DRM_FORMAT_NV24:
-	case DRM_FORMAT_NV42:
-		return plane ? 2 : 1;
-	case DRM_FORMAT_YUV410:
-	case DRM_FORMAT_YVU410:
-	case DRM_FORMAT_YUV411:
-	case DRM_FORMAT_YVU411:
-	case DRM_FORMAT_YUV420:
-	case DRM_FORMAT_YVU420:
-	case DRM_FORMAT_YUV422:
-	case DRM_FORMAT_YVU422:
-	case DRM_FORMAT_YUV444:
-	case DRM_FORMAT_YVU444:
-		return 1;
-	default:
-		drm_fb_get_bpp_depth(format, &depth, &bpp);
-		return bpp >> 3;
-	}
+	return info->cpp[plane];
 }
 EXPORT_SYMBOL(drm_format_plane_cpp);
 
@@ -271,28 +244,10 @@ EXPORT_SYMBOL(drm_format_plane_cpp);
  */
 int drm_format_horz_chroma_subsampling(uint32_t format)
 {
-	switch (format) {
-	case DRM_FORMAT_YUV411:
-	case DRM_FORMAT_YVU411:
-	case DRM_FORMAT_YUV410:
-	case DRM_FORMAT_YVU410:
-		return 4;
-	case DRM_FORMAT_YUYV:
-	case DRM_FORMAT_YVYU:
-	case DRM_FORMAT_UYVY:
-	case DRM_FORMAT_VYUY:
-	case DRM_FORMAT_NV12:
-	case DRM_FORMAT_NV21:
-	case DRM_FORMAT_NV16:
-	case DRM_FORMAT_NV61:
-	case DRM_FORMAT_YUV422:
-	case DRM_FORMAT_YVU422:
-	case DRM_FORMAT_YUV420:
-	case DRM_FORMAT_YVU420:
-		return 2;
-	default:
-		return 1;
-	}
+	const struct drm_format_info *info;
+
+	info = drm_format_info(format);
+	return info ? info->hsub : 1;
 }
 EXPORT_SYMBOL(drm_format_horz_chroma_subsampling);
 
@@ -306,18 +261,10 @@ EXPORT_SYMBOL(drm_format_horz_chroma_subsampling);
  */
 int drm_format_vert_chroma_subsampling(uint32_t format)
 {
-	switch (format) {
-	case DRM_FORMAT_YUV410:
-	case DRM_FORMAT_YVU410:
-		return 4;
-	case DRM_FORMAT_YUV420:
-	case DRM_FORMAT_YVU420:
-	case DRM_FORMAT_NV12:
-	case DRM_FORMAT_NV21:
-		return 2;
-	default:
-		return 1;
-	}
+	const struct drm_format_info *info;
+
+	info = drm_format_info(format);
+	return info ? info->vsub : 1;
 }
 EXPORT_SYMBOL(drm_format_vert_chroma_subsampling);
 
@@ -332,13 +279,16 @@ EXPORT_SYMBOL(drm_format_vert_chroma_subsampling);
  */
 int drm_format_plane_width(int width, uint32_t format, int plane)
 {
-	if (plane >= drm_format_num_planes(format))
+	const struct drm_format_info *info;
+
+	info = drm_format_info(format);
+	if (!info || plane >= info->num_planes)
 		return 0;
 
 	if (plane == 0)
 		return width;
 
-	return width / drm_format_horz_chroma_subsampling(format);
+	return width / info->hsub;
 }
 EXPORT_SYMBOL(drm_format_plane_width);
 
@@ -353,12 +303,15 @@ EXPORT_SYMBOL(drm_format_plane_width);
  */
 int drm_format_plane_height(int height, uint32_t format, int plane)
 {
-	if (plane >= drm_format_num_planes(format))
+	const struct drm_format_info *info;
+
+	info = drm_format_info(format);
+	if (!info || plane >= info->num_planes)
 		return 0;
 
 	if (plane == 0)
 		return height;
 
-	return height / drm_format_vert_chroma_subsampling(format);
+	return height / info->vsub;
 }
 EXPORT_SYMBOL(drm_format_plane_height);
diff --git a/drivers/gpu/drm/drm_framebuffer.c b/drivers/gpu/drm/drm_framebuffer.c
index 398efd67cb93..cbf0c893f426 100644
--- a/drivers/gpu/drm/drm_framebuffer.c
+++ b/drivers/gpu/drm/drm_framebuffer.c
@@ -126,111 +126,34 @@ int drm_mode_addfb(struct drm_device *dev,
 	return 0;
 }
 
-static int format_check(const struct drm_mode_fb_cmd2 *r)
-{
-	uint32_t format = r->pixel_format & ~DRM_FORMAT_BIG_ENDIAN;
-	char *format_name;
-
-	switch (format) {
-	case DRM_FORMAT_C8:
-	case DRM_FORMAT_RGB332:
-	case DRM_FORMAT_BGR233:
-	case DRM_FORMAT_XRGB4444:
-	case DRM_FORMAT_XBGR4444:
-	case DRM_FORMAT_RGBX4444:
-	case DRM_FORMAT_BGRX4444:
-	case DRM_FORMAT_ARGB4444:
-	case DRM_FORMAT_ABGR4444:
-	case DRM_FORMAT_RGBA4444:
-	case DRM_FORMAT_BGRA4444:
-	case DRM_FORMAT_XRGB1555:
-	case DRM_FORMAT_XBGR1555:
-	case DRM_FORMAT_RGBX5551:
-	case DRM_FORMAT_BGRX5551:
-	case DRM_FORMAT_ARGB1555:
-	case DRM_FORMAT_ABGR1555:
-	case DRM_FORMAT_RGBA5551:
-	case DRM_FORMAT_BGRA5551:
-	case DRM_FORMAT_RGB565:
-	case DRM_FORMAT_BGR565:
-	case DRM_FORMAT_RGB888:
-	case DRM_FORMAT_BGR888:
-	case DRM_FORMAT_XRGB8888:
-	case DRM_FORMAT_XBGR8888:
-	case DRM_FORMAT_RGBX8888:
-	case DRM_FORMAT_BGRX8888:
-	case DRM_FORMAT_ARGB8888:
-	case DRM_FORMAT_ABGR8888:
-	case DRM_FORMAT_RGBA8888:
-	case DRM_FORMAT_BGRA8888:
-	case DRM_FORMAT_XRGB2101010:
-	case DRM_FORMAT_XBGR2101010:
-	case DRM_FORMAT_RGBX1010102:
-	case DRM_FORMAT_BGRX1010102:
-	case DRM_FORMAT_ARGB2101010:
-	case DRM_FORMAT_ABGR2101010:
-	case DRM_FORMAT_RGBA1010102:
-	case DRM_FORMAT_BGRA1010102:
-	case DRM_FORMAT_YUYV:
-	case DRM_FORMAT_YVYU:
-	case DRM_FORMAT_UYVY:
-	case DRM_FORMAT_VYUY:
-	case DRM_FORMAT_AYUV:
-	case DRM_FORMAT_NV12:
-	case DRM_FORMAT_NV21:
-	case DRM_FORMAT_NV16:
-	case DRM_FORMAT_NV61:
-	case DRM_FORMAT_NV24:
-	case DRM_FORMAT_NV42:
-	case DRM_FORMAT_YUV410:
-	case DRM_FORMAT_YVU410:
-	case DRM_FORMAT_YUV411:
-	case DRM_FORMAT_YVU411:
-	case DRM_FORMAT_YUV420:
-	case DRM_FORMAT_YVU420:
-	case DRM_FORMAT_YUV422:
-	case DRM_FORMAT_YVU422:
-	case DRM_FORMAT_YUV444:
-	case DRM_FORMAT_YVU444:
-		return 0;
-	default:
-		format_name = drm_get_format_name(r->pixel_format);
-		DRM_DEBUG_KMS("invalid pixel format %s\n", format_name);
-		kfree(format_name);
-		return -EINVAL;
-	}
-}
-
 static int framebuffer_check(const struct drm_mode_fb_cmd2 *r)
 {
-	int ret, hsub, vsub, num_planes, i;
-
-	ret = format_check(r);
-	if (ret) {
-		char *format_name = drm_get_format_name(r->pixel_format);
-		DRM_DEBUG_KMS("bad framebuffer format %s\n", format_name);
-		kfree(format_name);
-		return ret;
+	const struct drm_format_info *info;
+	int i;
+
+	info = __drm_format_info(r->pixel_format & ~DRM_FORMAT_BIG_ENDIAN);
+	if (!info) {
+		struct drm_format_name_buf format_name;
+		DRM_DEBUG_KMS("bad framebuffer format %s\n",
+		              drm_get_format_name(r->pixel_format,
+		                                  &format_name));
+		return -EINVAL;
 	}
 
-	hsub = drm_format_horz_chroma_subsampling(r->pixel_format);
-	vsub = drm_format_vert_chroma_subsampling(r->pixel_format);
-	num_planes = drm_format_num_planes(r->pixel_format);
-
-	if (r->width == 0 || r->width % hsub) {
+	if (r->width == 0 || r->width % info->hsub) {
 		DRM_DEBUG_KMS("bad framebuffer width %u\n", r->width);
 		return -EINVAL;
 	}
 
-	if (r->height == 0 || r->height % vsub) {
+	if (r->height == 0 || r->height % info->vsub) {
 		DRM_DEBUG_KMS("bad framebuffer height %u\n", r->height);
 		return -EINVAL;
 	}
 
-	for (i = 0; i < num_planes; i++) {
-		unsigned int width = r->width / (i != 0 ? hsub : 1);
-		unsigned int height = r->height / (i != 0 ? vsub : 1);
-		unsigned int cpp = drm_format_plane_cpp(r->pixel_format, i);
+	for (i = 0; i < info->num_planes; i++) {
+		unsigned int width = r->width / (i != 0 ? info->hsub : 1);
+		unsigned int height = r->height / (i != 0 ? info->vsub : 1);
+		unsigned int cpp = info->cpp[i];
 
 		if (!r->handles[i]) {
 			DRM_DEBUG_KMS("no buffer object handle for plane %d\n", i);
@@ -254,6 +177,13 @@ static int framebuffer_check(const struct drm_mode_fb_cmd2 *r)
 			return -EINVAL;
 		}
 
+		if (r->flags & DRM_MODE_FB_MODIFIERS &&
+		    r->modifier[i] != r->modifier[0]) {
+			DRM_DEBUG_KMS("bad fb modifier %llu for plane %d\n",
+				      r->modifier[i], i);
+			return -EINVAL;
+		}
+
 		/* modifier specific checks: */
 		switch (r->modifier[i]) {
 		case DRM_FORMAT_MOD_SAMSUNG_64_32_TILE:
@@ -273,7 +203,7 @@ static int framebuffer_check(const struct drm_mode_fb_cmd2 *r)
 		}
 	}
 
-	for (i = num_planes; i < 4; i++) {
+	for (i = info->num_planes; i < 4; i++) {
 		if (r->modifier[i]) {
 			DRM_DEBUG_KMS("non-zero modifier for unused plane %d\n", i);
 			return -EINVAL;
@@ -751,6 +681,11 @@ EXPORT_SYMBOL(drm_framebuffer_lookup);
  * those used for fbdev. Note that the caller must hold a reference of it's own,
  * i.e. the object may not be destroyed through this call (since it'll lead to a
  * locking inversion).
+ *
+ * NOTE: This function is deprecated. For driver-private framebuffers it is not
+ * recommended to embed a framebuffer struct info fbdev struct, instead, a
+ * framebuffer pointer is preferred and drm_framebuffer_unreference() should be
+ * called when the framebuffer is to be cleaned up.
  */
 void drm_framebuffer_unregister_private(struct drm_framebuffer *fb)
 {
diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h
index e66af289a016..db80ec860e33 100644
--- a/drivers/gpu/drm/drm_internal.h
+++ b/drivers/gpu/drm/drm_internal.h
@@ -24,9 +24,6 @@
 #define DRM_IF_MAJOR 1
 #define DRM_IF_MINOR 4
 
-/* drm_irq.c */
-extern unsigned int drm_timestamp_monotonic;
-
 /* drm_fops.c */
 extern struct mutex drm_global_mutex;
 void drm_lastclose(struct drm_device *dev);
@@ -46,12 +43,21 @@ void drm_prime_destroy_file_private(struct drm_prime_file_private *prime_fpriv);
 void drm_prime_remove_buf_handle_locked(struct drm_prime_file_private *prime_fpriv,
 					struct dma_buf *dma_buf);
 
+/* drm_drv.c */
+struct drm_minor *drm_minor_acquire(unsigned int minor_id);
+void drm_minor_release(struct drm_minor *minor);
+
 /* drm_info.c */
 int drm_name_info(struct seq_file *m, void *data);
 int drm_clients_info(struct seq_file *m, void* data);
 int drm_gem_name_info(struct seq_file *m, void *data);
 
 /* drm_irq.c */
+extern unsigned int drm_timestamp_monotonic;
+
+/* IOCTLS */
+int drm_wait_vblank(struct drm_device *dev, void *data,
+		    struct drm_file *filp);
 int drm_control(struct drm_device *dev, void *data,
 		struct drm_file *file_priv);
 int drm_modeset_ctl(struct drm_device *dev, void *data,
@@ -100,6 +106,9 @@ int drm_debugfs_init(struct drm_minor *minor, int minor_id,
 int drm_debugfs_cleanup(struct drm_minor *minor);
 int drm_debugfs_connector_add(struct drm_connector *connector);
 void drm_debugfs_connector_remove(struct drm_connector *connector);
+int drm_debugfs_crtc_add(struct drm_crtc *crtc);
+void drm_debugfs_crtc_remove(struct drm_crtc *crtc);
+int drm_debugfs_crtc_crc_add(struct drm_crtc *crtc);
 #else
 static inline int drm_debugfs_init(struct drm_minor *minor, int minor_id,
 				   struct dentry *root)
@@ -119,4 +128,17 @@ static inline int drm_debugfs_connector_add(struct drm_connector *connector)
 static inline void drm_debugfs_connector_remove(struct drm_connector *connector)
 {
 }
+
+static inline int drm_debugfs_crtc_add(struct drm_crtc *crtc)
+{
+	return 0;
+}
+static inline void drm_debugfs_crtc_remove(struct drm_crtc *crtc)
+{
+}
+
+static inline int drm_debugfs_crtc_crc_add(struct drm_crtc *crtc)
+{
+	return 0;
+}
 #endif
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index 71c3473476c7..fed22c2b98b6 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -229,6 +229,22 @@ static int drm_getcap(struct drm_device *dev, void *data, struct drm_file *file_
 	struct drm_crtc *crtc;
 
 	req->value = 0;
+
+	/* Only some caps make sense with UMS/render-only drivers. */
+	switch (req->capability) {
+	case DRM_CAP_TIMESTAMP_MONOTONIC:
+		req->value = drm_timestamp_monotonic;
+		return 0;
+	case DRM_CAP_PRIME:
+		req->value |= dev->driver->prime_fd_to_handle ? DRM_PRIME_CAP_IMPORT : 0;
+		req->value |= dev->driver->prime_handle_to_fd ? DRM_PRIME_CAP_EXPORT : 0;
+		return 0;
+	}
+
+	/* Other caps only work with KMS drivers */
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENOTSUPP;
+
 	switch (req->capability) {
 	case DRM_CAP_DUMB_BUFFER:
 		if (dev->driver->dumb_create)
@@ -243,23 +259,14 @@ static int drm_getcap(struct drm_device *dev, void *data, struct drm_file *file_
 	case DRM_CAP_DUMB_PREFER_SHADOW:
 		req->value = dev->mode_config.prefer_shadow;
 		break;
-	case DRM_CAP_PRIME:
-		req->value |= dev->driver->prime_fd_to_handle ? DRM_PRIME_CAP_IMPORT : 0;
-		req->value |= dev->driver->prime_handle_to_fd ? DRM_PRIME_CAP_EXPORT : 0;
-		break;
-	case DRM_CAP_TIMESTAMP_MONOTONIC:
-		req->value = drm_timestamp_monotonic;
-		break;
 	case DRM_CAP_ASYNC_PAGE_FLIP:
 		req->value = dev->mode_config.async_page_flip;
 		break;
 	case DRM_CAP_PAGE_FLIP_TARGET:
-		if (drm_core_check_feature(dev, DRIVER_MODESET)) {
-			req->value = 1;
-			drm_for_each_crtc(crtc, dev) {
-				if (!crtc->funcs->page_flip_target)
-					req->value = 0;
-			}
+		req->value = 1;
+		drm_for_each_crtc(crtc, dev) {
+			if (!crtc->funcs->page_flip_target)
+				req->value = 0;
 		}
 		break;
 	case DRM_CAP_CURSOR_WIDTH:
diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index b969a64a1514..273625a85036 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -93,7 +93,7 @@ static void store_vblank(struct drm_device *dev, unsigned int pipe,
  * Reset the stored timestamp for the current vblank count to correspond
  * to the last vblank occurred.
  *
- * Only to be called from drm_vblank_on().
+ * Only to be called from drm_crtc_vblank_on().
  *
  * Note: caller must hold dev->vbl_lock since this reads & writes
  * device vblank fields.
@@ -234,6 +234,16 @@ static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
 	store_vblank(dev, pipe, diff, &t_vblank, cur_vblank);
 }
 
+static u32 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
+{
+	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
+
+	if (WARN_ON(pipe >= dev->num_crtcs))
+		return 0;
+
+	return vblank->count;
+}
+
 /**
  * drm_accurate_vblank_count - retrieve the master vblank counter
  * @crtc: which counter to retrieve
@@ -296,7 +306,7 @@ static void vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
 	 * Always update the count and timestamp to maintain the
 	 * appearance that the counter has been ticking all along until
 	 * this time. This makes the count account for the entire time
-	 * between drm_vblank_on() and drm_vblank_off().
+	 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
 	 */
 	drm_update_vblank_count(dev, pipe, 0);
 
@@ -888,31 +898,6 @@ drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
 }
 
 /**
- * drm_vblank_count - retrieve "cooked" vblank counter value
- * @dev: DRM device
- * @pipe: index of CRTC for which to retrieve the counter
- *
- * Fetches the "cooked" vblank count value that represents the number of
- * vblank events since the system was booted, including lost events due to
- * modesetting activity.
- *
- * This is the legacy version of drm_crtc_vblank_count().
- *
- * Returns:
- * The software vblank counter.
- */
-u32 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
-{
-	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
-
-	if (WARN_ON(pipe >= dev->num_crtcs))
-		return 0;
-
-	return vblank->count;
-}
-EXPORT_SYMBOL(drm_vblank_count);
-
-/**
  * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
  * @crtc: which counter to retrieve
  *
@@ -920,8 +905,6 @@ EXPORT_SYMBOL(drm_vblank_count);
  * vblank events since the system was booted, including lost events due to
  * modesetting activity.
  *
- * This is the native KMS version of drm_vblank_count().
- *
  * Returns:
  * The software vblank counter.
  */
@@ -952,8 +935,10 @@ static u32 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
 	u32 vblank_count;
 	unsigned int seq;
 
-	if (WARN_ON(pipe >= dev->num_crtcs))
+	if (WARN_ON(pipe >= dev->num_crtcs)) {
+		*vblanktime = (struct timeval) { 0 };
 		return 0;
+	}
 
 	do {
 		seq = read_seqbegin(&vblank->seqlock);
@@ -1270,21 +1255,20 @@ void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
 EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
 
 /**
- * drm_vblank_off - disable vblank events on a CRTC
- * @dev: DRM device
- * @pipe: CRTC index
+ * drm_crtc_vblank_off - disable vblank events on a CRTC
+ * @crtc: CRTC in question
  *
  * Drivers can use this function to shut down the vblank interrupt handling when
  * disabling a crtc. This function ensures that the latest vblank frame count is
- * stored so that drm_vblank_on() can restore it again.
+ * stored so that drm_vblank_on can restore it again.
  *
  * Drivers must use this function when the hardware vblank counter can get
  * reset, e.g. when suspending.
- *
- * This is the legacy version of drm_crtc_vblank_off().
  */
-void drm_vblank_off(struct drm_device *dev, unsigned int pipe)
+void drm_crtc_vblank_off(struct drm_crtc *crtc)
 {
+	struct drm_device *dev = crtc->dev;
+	unsigned int pipe = drm_crtc_index(crtc);
 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 	struct drm_pending_vblank_event *e, *t;
 	struct timeval now;
@@ -1300,7 +1284,8 @@ void drm_vblank_off(struct drm_device *dev, unsigned int pipe)
 	DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
 		      pipe, vblank->enabled, vblank->inmodeset);
 
-	/* Avoid redundant vblank disables without previous drm_vblank_on(). */
+	/* Avoid redundant vblank disables without previous
+	 * drm_crtc_vblank_on(). */
 	if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
 		vblank_disable_and_save(dev, pipe);
 
@@ -1331,25 +1316,6 @@ void drm_vblank_off(struct drm_device *dev, unsigned int pipe)
 	}
 	spin_unlock_irqrestore(&dev->event_lock, irqflags);
 }
-EXPORT_SYMBOL(drm_vblank_off);
-
-/**
- * drm_crtc_vblank_off - disable vblank events on a CRTC
- * @crtc: CRTC in question
- *
- * Drivers can use this function to shut down the vblank interrupt handling when
- * disabling a crtc. This function ensures that the latest vblank frame count is
- * stored so that drm_vblank_on can restore it again.
- *
- * Drivers must use this function when the hardware vblank counter can get
- * reset, e.g. when suspending.
- *
- * This is the native kms version of drm_vblank_off().
- */
-void drm_crtc_vblank_off(struct drm_crtc *crtc)
-{
-	drm_vblank_off(crtc->dev, drm_crtc_index(crtc));
-}
 EXPORT_SYMBOL(drm_crtc_vblank_off);
 
 /**
@@ -1385,19 +1351,18 @@ void drm_crtc_vblank_reset(struct drm_crtc *crtc)
 EXPORT_SYMBOL(drm_crtc_vblank_reset);
 
 /**
- * drm_vblank_on - enable vblank events on a CRTC
- * @dev: DRM device
- * @pipe: CRTC index
+ * drm_crtc_vblank_on - enable vblank events on a CRTC
+ * @crtc: CRTC in question
  *
  * This functions restores the vblank interrupt state captured with
- * drm_vblank_off() again. Note that calls to drm_vblank_on() and
- * drm_vblank_off() can be unbalanced and so can also be unconditionally called
+ * drm_crtc_vblank_off() again. Note that calls to drm_crtc_vblank_on() and
+ * drm_crtc_vblank_off() can be unbalanced and so can also be unconditionally called
  * in driver load code to reflect the current hardware state of the crtc.
- *
- * This is the legacy version of drm_crtc_vblank_on().
  */
-void drm_vblank_on(struct drm_device *dev, unsigned int pipe)
+void drm_crtc_vblank_on(struct drm_crtc *crtc)
 {
+	struct drm_device *dev = crtc->dev;
+	unsigned int pipe = drm_crtc_index(crtc);
 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 	unsigned long irqflags;
 
@@ -1424,49 +1389,10 @@ void drm_vblank_on(struct drm_device *dev, unsigned int pipe)
 		WARN_ON(drm_vblank_enable(dev, pipe));
 	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 }
-EXPORT_SYMBOL(drm_vblank_on);
-
-/**
- * drm_crtc_vblank_on - enable vblank events on a CRTC
- * @crtc: CRTC in question
- *
- * This functions restores the vblank interrupt state captured with
- * drm_vblank_off() again. Note that calls to drm_vblank_on() and
- * drm_vblank_off() can be unbalanced and so can also be unconditionally called
- * in driver load code to reflect the current hardware state of the crtc.
- *
- * This is the native kms version of drm_vblank_on().
- */
-void drm_crtc_vblank_on(struct drm_crtc *crtc)
-{
-	drm_vblank_on(crtc->dev, drm_crtc_index(crtc));
-}
 EXPORT_SYMBOL(drm_crtc_vblank_on);
 
-/**
- * drm_vblank_pre_modeset - account for vblanks across mode sets
- * @dev: DRM device
- * @pipe: CRTC index
- *
- * Account for vblank events across mode setting events, which will likely
- * reset the hardware frame counter.
- *
- * This is done by grabbing a temporary vblank reference to ensure that the
- * vblank interrupt keeps running across the modeset sequence. With this the
- * software-side vblank frame counting will ensure that there are no jumps or
- * discontinuities.
- *
- * Unfortunately this approach is racy and also doesn't work when the vblank
- * interrupt stops running, e.g. across system suspend resume. It is therefore
- * highly recommended that drivers use the newer drm_vblank_off() and
- * drm_vblank_on() instead. drm_vblank_pre_modeset() only works correctly when
- * using "cooked" software vblank frame counters and not relying on any hardware
- * counters.
- *
- * Drivers must call drm_vblank_post_modeset() when re-enabling the same crtc
- * again.
- */
-void drm_vblank_pre_modeset(struct drm_device *dev, unsigned int pipe)
+static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
+					  unsigned int pipe)
 {
 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 
@@ -1490,17 +1416,9 @@ void drm_vblank_pre_modeset(struct drm_device *dev, unsigned int pipe)
 			vblank->inmodeset |= 0x2;
 	}
 }
-EXPORT_SYMBOL(drm_vblank_pre_modeset);
 
-/**
- * drm_vblank_post_modeset - undo drm_vblank_pre_modeset changes
- * @dev: DRM device
- * @pipe: CRTC index
- *
- * This function again drops the temporary vblank reference acquired in
- * drm_vblank_pre_modeset.
- */
-void drm_vblank_post_modeset(struct drm_device *dev, unsigned int pipe)
+static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
+					   unsigned int pipe)
 {
 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 	unsigned long irqflags;
@@ -1523,7 +1441,6 @@ void drm_vblank_post_modeset(struct drm_device *dev, unsigned int pipe)
 		vblank->inmodeset = 0;
 	}
 }
-EXPORT_SYMBOL(drm_vblank_post_modeset);
 
 /*
  * drm_modeset_ctl - handle vblank event counter changes across mode switch
@@ -1556,10 +1473,10 @@ int drm_modeset_ctl(struct drm_device *dev, void *data,
 
 	switch (modeset->cmd) {
 	case _DRM_PRE_MODESET:
-		drm_vblank_pre_modeset(dev, pipe);
+		drm_legacy_vblank_pre_modeset(dev, pipe);
 		break;
 	case _DRM_POST_MODESET:
-		drm_vblank_post_modeset(dev, pipe);
+		drm_legacy_vblank_post_modeset(dev, pipe);
 		break;
 	default:
 		return -EINVAL;
@@ -1594,11 +1511,10 @@ static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
 	spin_lock_irqsave(&dev->event_lock, flags);
 
 	/*
-	 * drm_vblank_off() might have been called after we called
-	 * drm_vblank_get(). drm_vblank_off() holds event_lock
-	 * around the vblank disable, so no need for further locking.
-	 * The reference from drm_vblank_get() protects against
-	 * vblank disable from another source.
+	 * drm_crtc_vblank_off() might have been called after we called
+	 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
+	 * vblank disable, so no need for further locking.  The reference from
+	 * drm_vblank_get() protects against vblank disable from another source.
 	 */
 	if (!vblank->enabled) {
 		ret = -EINVAL;
diff --git a/drivers/gpu/drm/drm_lock.c b/drivers/gpu/drm/drm_lock.c
index c901f3c5b269..32d43f86a8f2 100644
--- a/drivers/gpu/drm/drm_lock.c
+++ b/drivers/gpu/drm/drm_lock.c
@@ -176,7 +176,8 @@ int drm_legacy_lock(struct drm_device *dev, void *data,
 
 	DRM_DEBUG("%d (pid %d) requests lock (0x%08x), flags = 0x%08x\n",
 		  lock->context, task_pid_nr(current),
-		  master->lock.hw_lock->lock, lock->flags);
+		  master->lock.hw_lock ? master->lock.hw_lock->lock : -1,
+		  lock->flags);
 
 	add_wait_queue(&master->lock.lock_queue, &entry);
 	spin_lock_bh(&master->lock.spinlock);
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index 11d44a1e0ab3..ca1e344f318d 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -104,6 +104,68 @@ static struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_
 						u64 end,
 						enum drm_mm_search_flags flags);
 
+#ifdef CONFIG_DRM_DEBUG_MM
+#include <linux/stackdepot.h>
+
+#define STACKDEPTH 32
+#define BUFSZ 4096
+
+static noinline void save_stack(struct drm_mm_node *node)
+{
+	unsigned long entries[STACKDEPTH];
+	struct stack_trace trace = {
+		.entries = entries,
+		.max_entries = STACKDEPTH,
+		.skip = 1
+	};
+
+	save_stack_trace(&trace);
+	if (trace.nr_entries != 0 &&
+	    trace.entries[trace.nr_entries-1] == ULONG_MAX)
+		trace.nr_entries--;
+
+	/* May be called under spinlock, so avoid sleeping */
+	node->stack = depot_save_stack(&trace, GFP_NOWAIT);
+}
+
+static void show_leaks(struct drm_mm *mm)
+{
+	struct drm_mm_node *node;
+	unsigned long entries[STACKDEPTH];
+	char *buf;
+
+	buf = kmalloc(BUFSZ, GFP_KERNEL);
+	if (!buf)
+		return;
+
+	list_for_each_entry(node, &mm->head_node.node_list, node_list) {
+		struct stack_trace trace = {
+			.entries = entries,
+			.max_entries = STACKDEPTH
+		};
+
+		if (!node->stack) {
+			DRM_ERROR("node [%08llx + %08llx]: unknown owner\n",
+				  node->start, node->size);
+			continue;
+		}
+
+		depot_fetch_stack(node->stack, &trace);
+		snprint_stack_trace(buf, BUFSZ, &trace, 0);
+		DRM_ERROR("node [%08llx + %08llx]: inserted at\n%s",
+			  node->start, node->size, buf);
+	}
+
+	kfree(buf);
+}
+
+#undef STACKDEPTH
+#undef BUFSZ
+#else
+static void save_stack(struct drm_mm_node *node) { }
+static void show_leaks(struct drm_mm *mm) { }
+#endif
+
 #define START(node) ((node)->start)
 #define LAST(node)  ((node)->start + (node)->size - 1)
 
@@ -112,19 +174,12 @@ INTERVAL_TREE_DEFINE(struct drm_mm_node, rb,
 		     START, LAST, static inline, drm_mm_interval_tree)
 
 struct drm_mm_node *
-drm_mm_interval_first(struct drm_mm *mm, u64 start, u64 last)
+__drm_mm_interval_first(struct drm_mm *mm, u64 start, u64 last)
 {
 	return drm_mm_interval_tree_iter_first(&mm->interval_tree,
 					       start, last);
 }
-EXPORT_SYMBOL(drm_mm_interval_first);
-
-struct drm_mm_node *
-drm_mm_interval_next(struct drm_mm_node *node, u64 start, u64 last)
-{
-	return drm_mm_interval_tree_iter_next(node, start, last);
-}
-EXPORT_SYMBOL(drm_mm_interval_next);
+EXPORT_SYMBOL(__drm_mm_interval_first);
 
 static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node,
 					  struct drm_mm_node *node)
@@ -228,6 +283,8 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
 		list_add(&node->hole_stack, &mm->hole_stack);
 		node->hole_follows = 1;
 	}
+
+	save_stack(node);
 }
 
 /**
@@ -249,6 +306,7 @@ int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node)
 	u64 end = node->start + node->size;
 	struct drm_mm_node *hole;
 	u64 hole_start, hole_end;
+	u64 adj_start, adj_end;
 
 	if (WARN_ON(node->size == 0))
 		return -EINVAL;
@@ -270,9 +328,13 @@ int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node)
 	if (!hole->hole_follows)
 		return -ENOSPC;
 
-	hole_start = __drm_mm_hole_node_start(hole);
-	hole_end = __drm_mm_hole_node_end(hole);
-	if (hole_start > node->start || hole_end < end)
+	adj_start = hole_start = __drm_mm_hole_node_start(hole);
+	adj_end = hole_end = __drm_mm_hole_node_end(hole);
+
+	if (mm->color_adjust)
+		mm->color_adjust(hole, node->color, &adj_start, &adj_end);
+
+	if (adj_start > node->start || adj_end < end)
 		return -ENOSPC;
 
 	node->mm = mm;
@@ -293,6 +355,8 @@ int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node)
 		node->hole_follows = 1;
 	}
 
+	save_stack(node);
+
 	return 0;
 }
 EXPORT_SYMBOL(drm_mm_reserve_node);
@@ -397,6 +461,8 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node,
 		list_add(&node->hole_stack, &mm->hole_stack);
 		node->hole_follows = 1;
 	}
+
+	save_stack(node);
 }
 
 /**
@@ -839,6 +905,7 @@ void drm_mm_init(struct drm_mm * mm, u64 start, u64 size)
 
 	/* Clever trick to avoid a special case in the free hole tracking. */
 	INIT_LIST_HEAD(&mm->head_node.node_list);
+	mm->head_node.allocated = 0;
 	mm->head_node.hole_follows = 1;
 	mm->head_node.scanned_block = 0;
 	mm->head_node.scanned_prev_free = 0;
@@ -861,10 +928,12 @@ EXPORT_SYMBOL(drm_mm_init);
  * Note that it is a bug to call this function on an allocator which is not
  * clean.
  */
-void drm_mm_takedown(struct drm_mm * mm)
+void drm_mm_takedown(struct drm_mm *mm)
 {
-	WARN(!list_empty(&mm->head_node.node_list),
-	     "Memory manager not clean during takedown.\n");
+	if (WARN(!list_empty(&mm->head_node.node_list),
+		 "Memory manager not clean during takedown.\n"))
+		show_leaks(mm);
+
 }
 EXPORT_SYMBOL(drm_mm_takedown);
 
diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c
new file mode 100644
index 000000000000..2735a5847ffa
--- /dev/null
+++ b/drivers/gpu/drm/drm_mode_config.c
@@ -0,0 +1,494 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#include <drm/drm_mode_config.h>
+#include <drm/drmP.h>
+
+#include "drm_crtc_internal.h"
+#include "drm_internal.h"
+
+int drm_modeset_register_all(struct drm_device *dev)
+{
+	int ret;
+
+	ret = drm_plane_register_all(dev);
+	if (ret)
+		goto err_plane;
+
+	ret = drm_crtc_register_all(dev);
+	if  (ret)
+		goto err_crtc;
+
+	ret = drm_encoder_register_all(dev);
+	if (ret)
+		goto err_encoder;
+
+	ret = drm_connector_register_all(dev);
+	if (ret)
+		goto err_connector;
+
+	return 0;
+
+err_connector:
+	drm_encoder_unregister_all(dev);
+err_encoder:
+	drm_crtc_unregister_all(dev);
+err_crtc:
+	drm_plane_unregister_all(dev);
+err_plane:
+	return ret;
+}
+
+void drm_modeset_unregister_all(struct drm_device *dev)
+{
+	drm_connector_unregister_all(dev);
+	drm_encoder_unregister_all(dev);
+	drm_crtc_unregister_all(dev);
+	drm_plane_unregister_all(dev);
+}
+
+/**
+ * drm_mode_getresources - get graphics configuration
+ * @dev: drm device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: drm file for the ioctl call
+ *
+ * Construct a set of configuration description structures and return
+ * them to the user, including CRTC, connector and framebuffer configuration.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_mode_getresources(struct drm_device *dev, void *data,
+			  struct drm_file *file_priv)
+{
+	struct drm_mode_card_res *card_res = data;
+	struct list_head *lh;
+	struct drm_framebuffer *fb;
+	struct drm_connector *connector;
+	struct drm_crtc *crtc;
+	struct drm_encoder *encoder;
+	int ret = 0;
+	int connector_count = 0;
+	int crtc_count = 0;
+	int fb_count = 0;
+	int encoder_count = 0;
+	int copied = 0;
+	uint32_t __user *fb_id;
+	uint32_t __user *crtc_id;
+	uint32_t __user *connector_id;
+	uint32_t __user *encoder_id;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+
+	mutex_lock(&file_priv->fbs_lock);
+	/*
+	 * For the non-control nodes we need to limit the list of resources
+	 * by IDs in the group list for this node
+	 */
+	list_for_each(lh, &file_priv->fbs)
+		fb_count++;
+
+	/* handle this in 4 parts */
+	/* FBs */
+	if (card_res->count_fbs >= fb_count) {
+		copied = 0;
+		fb_id = (uint32_t __user *)(unsigned long)card_res->fb_id_ptr;
+		list_for_each_entry(fb, &file_priv->fbs, filp_head) {
+			if (put_user(fb->base.id, fb_id + copied)) {
+				mutex_unlock(&file_priv->fbs_lock);
+				return -EFAULT;
+			}
+			copied++;
+		}
+	}
+	card_res->count_fbs = fb_count;
+	mutex_unlock(&file_priv->fbs_lock);
+
+	/* mode_config.mutex protects the connector list against e.g. DP MST
+	 * connector hot-adding. CRTC/Plane lists are invariant. */
+	mutex_lock(&dev->mode_config.mutex);
+	drm_for_each_crtc(crtc, dev)
+		crtc_count++;
+
+	drm_for_each_connector(connector, dev)
+		connector_count++;
+
+	drm_for_each_encoder(encoder, dev)
+		encoder_count++;
+
+	card_res->max_height = dev->mode_config.max_height;
+	card_res->min_height = dev->mode_config.min_height;
+	card_res->max_width = dev->mode_config.max_width;
+	card_res->min_width = dev->mode_config.min_width;
+
+	/* CRTCs */
+	if (card_res->count_crtcs >= crtc_count) {
+		copied = 0;
+		crtc_id = (uint32_t __user *)(unsigned long)card_res->crtc_id_ptr;
+		drm_for_each_crtc(crtc, dev) {
+			if (put_user(crtc->base.id, crtc_id + copied)) {
+				ret = -EFAULT;
+				goto out;
+			}
+			copied++;
+		}
+	}
+	card_res->count_crtcs = crtc_count;
+
+	/* Encoders */
+	if (card_res->count_encoders >= encoder_count) {
+		copied = 0;
+		encoder_id = (uint32_t __user *)(unsigned long)card_res->encoder_id_ptr;
+		drm_for_each_encoder(encoder, dev) {
+			if (put_user(encoder->base.id, encoder_id +
+				     copied)) {
+				ret = -EFAULT;
+				goto out;
+			}
+			copied++;
+		}
+	}
+	card_res->count_encoders = encoder_count;
+
+	/* Connectors */
+	if (card_res->count_connectors >= connector_count) {
+		copied = 0;
+		connector_id = (uint32_t __user *)(unsigned long)card_res->connector_id_ptr;
+		drm_for_each_connector(connector, dev) {
+			if (put_user(connector->base.id,
+				     connector_id + copied)) {
+				ret = -EFAULT;
+				goto out;
+			}
+			copied++;
+		}
+	}
+	card_res->count_connectors = connector_count;
+
+out:
+	mutex_unlock(&dev->mode_config.mutex);
+	return ret;
+}
+
+/**
+ * drm_mode_config_reset - call ->reset callbacks
+ * @dev: drm device
+ *
+ * This functions calls all the crtc's, encoder's and connector's ->reset
+ * callback. Drivers can use this in e.g. their driver load or resume code to
+ * reset hardware and software state.
+ */
+void drm_mode_config_reset(struct drm_device *dev)
+{
+	struct drm_crtc *crtc;
+	struct drm_plane *plane;
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
+
+	drm_for_each_plane(plane, dev)
+		if (plane->funcs->reset)
+			plane->funcs->reset(plane);
+
+	drm_for_each_crtc(crtc, dev)
+		if (crtc->funcs->reset)
+			crtc->funcs->reset(crtc);
+
+	drm_for_each_encoder(encoder, dev)
+		if (encoder->funcs->reset)
+			encoder->funcs->reset(encoder);
+
+	mutex_lock(&dev->mode_config.mutex);
+	drm_for_each_connector(connector, dev)
+		if (connector->funcs->reset)
+			connector->funcs->reset(connector);
+	mutex_unlock(&dev->mode_config.mutex);
+}
+EXPORT_SYMBOL(drm_mode_config_reset);
+
+/*
+ * Global properties
+ */
+static const struct drm_prop_enum_list drm_plane_type_enum_list[] = {
+	{ DRM_PLANE_TYPE_OVERLAY, "Overlay" },
+	{ DRM_PLANE_TYPE_PRIMARY, "Primary" },
+	{ DRM_PLANE_TYPE_CURSOR, "Cursor" },
+};
+
+static int drm_mode_create_standard_properties(struct drm_device *dev)
+{
+	struct drm_property *prop;
+	int ret;
+
+	ret = drm_connector_create_standard_properties(dev);
+	if (ret)
+		return ret;
+
+	prop = drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
+					"type", drm_plane_type_enum_list,
+					ARRAY_SIZE(drm_plane_type_enum_list));
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.plane_type_property = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+			"SRC_X", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_src_x = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+			"SRC_Y", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_src_y = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+			"SRC_W", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_src_w = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+			"SRC_H", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_src_h = prop;
+
+	prop = drm_property_create_signed_range(dev, DRM_MODE_PROP_ATOMIC,
+			"CRTC_X", INT_MIN, INT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_crtc_x = prop;
+
+	prop = drm_property_create_signed_range(dev, DRM_MODE_PROP_ATOMIC,
+			"CRTC_Y", INT_MIN, INT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_crtc_y = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+			"CRTC_W", 0, INT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_crtc_w = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+			"CRTC_H", 0, INT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_crtc_h = prop;
+
+	prop = drm_property_create_object(dev, DRM_MODE_PROP_ATOMIC,
+			"FB_ID", DRM_MODE_OBJECT_FB);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_fb_id = prop;
+
+	prop = drm_property_create_signed_range(dev, DRM_MODE_PROP_ATOMIC,
+			"IN_FENCE_FD", -1, INT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_in_fence_fd = prop;
+
+	prop = drm_property_create_range(dev, DRM_MODE_PROP_ATOMIC,
+			"OUT_FENCE_PTR", 0, U64_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_out_fence_ptr = prop;
+
+	prop = drm_property_create_object(dev, DRM_MODE_PROP_ATOMIC,
+			"CRTC_ID", DRM_MODE_OBJECT_CRTC);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_crtc_id = prop;
+
+	prop = drm_property_create_bool(dev, DRM_MODE_PROP_ATOMIC,
+			"ACTIVE");
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_active = prop;
+
+	prop = drm_property_create(dev,
+			DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB,
+			"MODE_ID", 0);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.prop_mode_id = prop;
+
+	prop = drm_property_create(dev,
+			DRM_MODE_PROP_BLOB,
+			"DEGAMMA_LUT", 0);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.degamma_lut_property = prop;
+
+	prop = drm_property_create_range(dev,
+			DRM_MODE_PROP_IMMUTABLE,
+			"DEGAMMA_LUT_SIZE", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.degamma_lut_size_property = prop;
+
+	prop = drm_property_create(dev,
+			DRM_MODE_PROP_BLOB,
+			"CTM", 0);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.ctm_property = prop;
+
+	prop = drm_property_create(dev,
+			DRM_MODE_PROP_BLOB,
+			"GAMMA_LUT", 0);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.gamma_lut_property = prop;
+
+	prop = drm_property_create_range(dev,
+			DRM_MODE_PROP_IMMUTABLE,
+			"GAMMA_LUT_SIZE", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.gamma_lut_size_property = prop;
+
+	return 0;
+}
+
+/**
+ * drm_mode_config_init - initialize DRM mode_configuration structure
+ * @dev: DRM device
+ *
+ * Initialize @dev's mode_config structure, used for tracking the graphics
+ * configuration of @dev.
+ *
+ * Since this initializes the modeset locks, no locking is possible. Which is no
+ * problem, since this should happen single threaded at init time. It is the
+ * driver's problem to ensure this guarantee.
+ *
+ */
+void drm_mode_config_init(struct drm_device *dev)
+{
+	mutex_init(&dev->mode_config.mutex);
+	drm_modeset_lock_init(&dev->mode_config.connection_mutex);
+	mutex_init(&dev->mode_config.idr_mutex);
+	mutex_init(&dev->mode_config.fb_lock);
+	mutex_init(&dev->mode_config.blob_lock);
+	INIT_LIST_HEAD(&dev->mode_config.fb_list);
+	INIT_LIST_HEAD(&dev->mode_config.crtc_list);
+	INIT_LIST_HEAD(&dev->mode_config.connector_list);
+	INIT_LIST_HEAD(&dev->mode_config.encoder_list);
+	INIT_LIST_HEAD(&dev->mode_config.property_list);
+	INIT_LIST_HEAD(&dev->mode_config.property_blob_list);
+	INIT_LIST_HEAD(&dev->mode_config.plane_list);
+	idr_init(&dev->mode_config.crtc_idr);
+	idr_init(&dev->mode_config.tile_idr);
+	ida_init(&dev->mode_config.connector_ida);
+
+	drm_modeset_lock_all(dev);
+	drm_mode_create_standard_properties(dev);
+	drm_modeset_unlock_all(dev);
+
+	/* Just to be sure */
+	dev->mode_config.num_fb = 0;
+	dev->mode_config.num_connector = 0;
+	dev->mode_config.num_crtc = 0;
+	dev->mode_config.num_encoder = 0;
+	dev->mode_config.num_overlay_plane = 0;
+	dev->mode_config.num_total_plane = 0;
+}
+EXPORT_SYMBOL(drm_mode_config_init);
+
+/**
+ * drm_mode_config_cleanup - free up DRM mode_config info
+ * @dev: DRM device
+ *
+ * Free up all the connectors and CRTCs associated with this DRM device, then
+ * free up the framebuffers and associated buffer objects.
+ *
+ * Note that since this /should/ happen single-threaded at driver/device
+ * teardown time, no locking is required. It's the driver's job to ensure that
+ * this guarantee actually holds true.
+ *
+ * FIXME: cleanup any dangling user buffer objects too
+ */
+void drm_mode_config_cleanup(struct drm_device *dev)
+{
+	struct drm_connector *connector, *ot;
+	struct drm_crtc *crtc, *ct;
+	struct drm_encoder *encoder, *enct;
+	struct drm_framebuffer *fb, *fbt;
+	struct drm_property *property, *pt;
+	struct drm_property_blob *blob, *bt;
+	struct drm_plane *plane, *plt;
+
+	list_for_each_entry_safe(encoder, enct, &dev->mode_config.encoder_list,
+				 head) {
+		encoder->funcs->destroy(encoder);
+	}
+
+	list_for_each_entry_safe(connector, ot,
+				 &dev->mode_config.connector_list, head) {
+		connector->funcs->destroy(connector);
+	}
+
+	list_for_each_entry_safe(property, pt, &dev->mode_config.property_list,
+				 head) {
+		drm_property_destroy(dev, property);
+	}
+
+	list_for_each_entry_safe(plane, plt, &dev->mode_config.plane_list,
+				 head) {
+		plane->funcs->destroy(plane);
+	}
+
+	list_for_each_entry_safe(crtc, ct, &dev->mode_config.crtc_list, head) {
+		crtc->funcs->destroy(crtc);
+	}
+
+	list_for_each_entry_safe(blob, bt, &dev->mode_config.property_blob_list,
+				 head_global) {
+		drm_property_unreference_blob(blob);
+	}
+
+	/*
+	 * Single-threaded teardown context, so it's not required to grab the
+	 * fb_lock to protect against concurrent fb_list access. Contrary, it
+	 * would actually deadlock with the drm_framebuffer_cleanup function.
+	 *
+	 * Also, if there are any framebuffers left, that's a driver leak now,
+	 * so politely WARN about this.
+	 */
+	WARN_ON(!list_empty(&dev->mode_config.fb_list));
+	list_for_each_entry_safe(fb, fbt, &dev->mode_config.fb_list, head) {
+		drm_framebuffer_free(&fb->base.refcount);
+	}
+
+	ida_destroy(&dev->mode_config.connector_ida);
+	idr_destroy(&dev->mode_config.tile_idr);
+	idr_destroy(&dev->mode_config.crtc_idr);
+	drm_modeset_lock_fini(&dev->mode_config.connection_mutex);
+}
+EXPORT_SYMBOL(drm_mode_config_cleanup);
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index 53f07ac7c174..ac6a35212501 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -49,13 +49,7 @@
  */
 void drm_mode_debug_printmodeline(const struct drm_display_mode *mode)
 {
-	DRM_DEBUG_KMS("Modeline %d:\"%s\" %d %d %d %d %d %d %d %d %d %d "
-			"0x%x 0x%x\n",
-		mode->base.id, mode->name, mode->vrefresh, mode->clock,
-		mode->hdisplay, mode->hsync_start,
-		mode->hsync_end, mode->htotal,
-		mode->vdisplay, mode->vsync_start,
-		mode->vsync_end, mode->vtotal, mode->type, mode->flags);
+	DRM_DEBUG_KMS("Modeline " DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));
 }
 EXPORT_SYMBOL(drm_mode_debug_printmodeline);
 
@@ -165,6 +159,7 @@ struct drm_display_mode *drm_cvt_mode(struct drm_device *dev, int hdisplay,
 	unsigned int vfieldrate, hperiod;
 	int hdisplay_rnd, hmargin, vdisplay_rnd, vmargin, vsync;
 	int interlace;
+	u64 tmp;
 
 	/* allocate the drm_display_mode structure. If failure, we will
 	 * return directly
@@ -322,8 +317,11 @@ struct drm_display_mode *drm_cvt_mode(struct drm_device *dev, int hdisplay,
 		drm_mode->vsync_end = drm_mode->vsync_start + vsync;
 	}
 	/* 15/13. Find pixel clock frequency (kHz for xf86) */
-	drm_mode->clock = drm_mode->htotal * HV_FACTOR * 1000 / hperiod;
-	drm_mode->clock -= drm_mode->clock % CVT_CLOCK_STEP;
+	tmp = drm_mode->htotal; /* perform intermediate calcs in u64 */
+	tmp *= HV_FACTOR * 1000;
+	do_div(tmp, hperiod);
+	tmp -= drm_mode->clock % CVT_CLOCK_STEP;
+	drm_mode->clock = tmp;
 	/* 18/16. Find actual vertical frame frequency */
 	/* ignore - just set the mode flag for interlaced */
 	if (interlaced) {
diff --git a/drivers/gpu/drm/drm_modeset_helper.c b/drivers/gpu/drm/drm_modeset_helper.c
index 1d45738f8f98..cc232ac6c950 100644
--- a/drivers/gpu/drm/drm_modeset_helper.c
+++ b/drivers/gpu/drm/drm_modeset_helper.c
@@ -38,7 +38,7 @@
  * Some userspace presumes that the first connected connector is the main
  * display, where it's supposed to display e.g. the login screen. For
  * laptops, this should be the main panel. Use this function to sort all
- * (eDP/LVDS) panels to the front of the connector list, instead of
+ * (eDP/LVDS/DSI) panels to the front of the connector list, instead of
  * painstakingly trying to initialize them in the right order.
  */
 void drm_helper_move_panel_connectors_to_head(struct drm_device *dev)
@@ -51,7 +51,8 @@ void drm_helper_move_panel_connectors_to_head(struct drm_device *dev)
 	list_for_each_entry_safe(connector, tmp,
 				 &dev->mode_config.connector_list, head) {
 		if (connector->connector_type == DRM_MODE_CONNECTOR_LVDS ||
-		    connector->connector_type == DRM_MODE_CONNECTOR_eDP)
+		    connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
+		    connector->connector_type == DRM_MODE_CONNECTOR_DSI)
 			list_move_tail(&connector->head, &panel_list);
 	}
 
@@ -70,17 +71,31 @@ EXPORT_SYMBOL(drm_helper_move_panel_connectors_to_head);
 void drm_helper_mode_fill_fb_struct(struct drm_framebuffer *fb,
 				    const struct drm_mode_fb_cmd2 *mode_cmd)
 {
+	const struct drm_format_info *info;
 	int i;
 
+	info = drm_format_info(mode_cmd->pixel_format);
+	if (!info || !info->depth) {
+		struct drm_format_name_buf format_name;
+
+		DRM_DEBUG_KMS("non-RGB pixel format %s\n",
+		              drm_get_format_name(mode_cmd->pixel_format,
+		                                  &format_name));
+
+		fb->depth = 0;
+		fb->bits_per_pixel = 0;
+	} else {
+		fb->depth = info->depth;
+		fb->bits_per_pixel = info->cpp[0] * 8;
+	}
+
 	fb->width = mode_cmd->width;
 	fb->height = mode_cmd->height;
 	for (i = 0; i < 4; i++) {
 		fb->pitches[i] = mode_cmd->pitches[i];
 		fb->offsets[i] = mode_cmd->offsets[i];
-		fb->modifier[i] = mode_cmd->modifier[i];
 	}
-	drm_fb_get_bpp_depth(mode_cmd->pixel_format, &fb->depth,
-				    &fb->bits_per_pixel);
+	fb->modifier = mode_cmd->modifier[0];
 	fb->pixel_format = mode_cmd->pixel_format;
 	fb->flags = mode_cmd->flags;
 }
diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c
index 61146f5b4f56..3551ae31f143 100644
--- a/drivers/gpu/drm/drm_modeset_lock.c
+++ b/drivers/gpu/drm/drm_modeset_lock.c
@@ -52,14 +52,16 @@
  *     drm_modeset_drop_locks(&ctx);
  *     drm_modeset_acquire_fini(&ctx);
  *
- *  On top of of these per-object locks using &ww_mutex there's also an overall
- *  dev->mode_config.lock, for protecting everything else. Mostly this means
- *  probe state of connectors, and preventing hotplug add/removal of connectors.
+ * On top of of these per-object locks using &ww_mutex there's also an overall
+ * dev->mode_config.lock, for protecting everything else. Mostly this means
+ * probe state of connectors, and preventing hotplug add/removal of connectors.
  *
- *  Finally there's a bunch of dedicated locks to protect drm core internal
- *  lists and lookup data structures.
+ * Finally there's a bunch of dedicated locks to protect drm core internal
+ * lists and lookup data structures.
  */
 
+static DEFINE_WW_CLASS(crtc_ww_class);
+
 /**
  * drm_modeset_lock_all - take all modeset locks
  * @dev: DRM device
@@ -398,6 +400,17 @@ int drm_modeset_backoff_interruptible(struct drm_modeset_acquire_ctx *ctx)
 EXPORT_SYMBOL(drm_modeset_backoff_interruptible);
 
 /**
+ * drm_modeset_lock_init - initialize lock
+ * @lock: lock to init
+ */
+void drm_modeset_lock_init(struct drm_modeset_lock *lock)
+{
+	ww_mutex_init(&lock->mutex, &crtc_ww_class);
+	INIT_LIST_HEAD(&lock->head);
+}
+EXPORT_SYMBOL(drm_modeset_lock_init);
+
+/**
  * drm_modeset_lock - take modeset lock
  * @lock: lock to take
  * @ctx: acquire ctx
diff --git a/drivers/gpu/drm/drm_of.c b/drivers/gpu/drm/drm_of.c
index bc98bb94264d..47848ed8ca48 100644
--- a/drivers/gpu/drm/drm_of.c
+++ b/drivers/gpu/drm/drm_of.c
@@ -6,6 +6,11 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_of.h>
 
+static void drm_release_of(struct device *dev, void *data)
+{
+	of_node_put(data);
+}
+
 /**
  * drm_crtc_port_mask - find the mask of a registered CRTC by port OF node
  * @dev: DRM device
@@ -64,6 +69,24 @@ uint32_t drm_of_find_possible_crtcs(struct drm_device *dev,
 EXPORT_SYMBOL(drm_of_find_possible_crtcs);
 
 /**
+ * drm_of_component_match_add - Add a component helper OF node match rule
+ * @master: master device
+ * @matchptr: component match pointer
+ * @compare: compare function used for matching component
+ * @node: of_node
+ */
+void drm_of_component_match_add(struct device *master,
+				struct component_match **matchptr,
+				int (*compare)(struct device *, void *),
+				struct device_node *node)
+{
+	of_node_get(node);
+	component_match_add_release(master, matchptr, drm_release_of,
+				    compare, node);
+}
+EXPORT_SYMBOL_GPL(drm_of_component_match_add);
+
+/**
  * drm_of_component_probe - Generic probe function for a component based master
  * @dev: master device containing the OF node
  * @compare_of: compare function used for matching components
@@ -101,7 +124,7 @@ int drm_of_component_probe(struct device *dev,
 			continue;
 		}
 
-		component_match_add(dev, &match, compare_of, port);
+		drm_of_component_match_add(dev, &match, compare_of, port);
 		of_node_put(port);
 	}
 
@@ -140,7 +163,8 @@ int drm_of_component_probe(struct device *dev,
 				continue;
 			}
 
-			component_match_add(dev, &match, compare_of, remote);
+			drm_of_component_match_add(dev, &match, compare_of,
+						   remote);
 			of_node_put(remote);
 		}
 		of_node_put(port);
diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c
index 249c0ae52c6d..62b98f386fd1 100644
--- a/drivers/gpu/drm/drm_plane.c
+++ b/drivers/gpu/drm/drm_plane.c
@@ -79,7 +79,7 @@ static unsigned int drm_num_planes(struct drm_device *dev)
  * Zero on success, error code on failure.
  */
 int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
-			     unsigned long possible_crtcs,
+			     uint32_t possible_crtcs,
 			     const struct drm_plane_funcs *funcs,
 			     const uint32_t *formats, unsigned int format_count,
 			     enum drm_plane_type type,
@@ -137,6 +137,7 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
 
 	if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
 		drm_object_attach_property(&plane->base, config->prop_fb_id, 0);
+		drm_object_attach_property(&plane->base, config->prop_in_fence_fd, -1);
 		drm_object_attach_property(&plane->base, config->prop_crtc_id, 0);
 		drm_object_attach_property(&plane->base, config->prop_crtc_x, 0);
 		drm_object_attach_property(&plane->base, config->prop_crtc_y, 0);
@@ -195,7 +196,7 @@ void drm_plane_unregister_all(struct drm_device *dev)
  * Zero on success, error code on failure.
  */
 int drm_plane_init(struct drm_device *dev, struct drm_plane *plane,
-		   unsigned long possible_crtcs,
+		   uint32_t possible_crtcs,
 		   const struct drm_plane_funcs *funcs,
 		   const uint32_t *formats, unsigned int format_count,
 		   bool is_primary)
@@ -220,7 +221,8 @@ void drm_plane_cleanup(struct drm_plane *plane)
 {
 	struct drm_device *dev = plane->dev;
 
-	drm_modeset_lock_all(dev);
+	drm_modeset_lock_fini(&plane->mutex);
+
 	kfree(plane->format_types);
 	drm_mode_object_unregister(dev, &plane->base);
 
@@ -235,7 +237,6 @@ void drm_plane_cleanup(struct drm_plane *plane)
 	dev->mode_config.num_total_plane--;
 	if (plane->type == DRM_PLANE_TYPE_OVERLAY)
 		dev->mode_config.num_overlay_plane--;
-	drm_modeset_unlock_all(dev);
 
 	WARN_ON(plane->state && !plane->funcs->atomic_destroy_state);
 	if (plane->state && plane->funcs->atomic_destroy_state)
@@ -479,9 +480,10 @@ static int __setplane_internal(struct drm_plane *plane,
 	/* Check whether this plane supports the fb pixel format. */
 	ret = drm_plane_check_pixel_format(plane, fb->pixel_format);
 	if (ret) {
-		char *format_name = drm_get_format_name(fb->pixel_format);
-		DRM_DEBUG_KMS("Invalid pixel format %s\n", format_name);
-		kfree(format_name);
+		struct drm_format_name_buf format_name;
+		DRM_DEBUG_KMS("Invalid pixel format %s\n",
+		              drm_get_format_name(fb->pixel_format,
+		                                  &format_name));
 		goto out;
 	}
 
diff --git a/drivers/gpu/drm/drm_plane_helper.c b/drivers/gpu/drm/drm_plane_helper.c
index 7899fc1dcdb0..7a7dddf604d7 100644
--- a/drivers/gpu/drm/drm_plane_helper.c
+++ b/drivers/gpu/drm/drm_plane_helper.c
@@ -130,15 +130,8 @@ int drm_plane_helper_check_state(struct drm_plane_state *state,
 	unsigned int rotation = state->rotation;
 	int hscale, vscale;
 
-	src->x1 = state->src_x;
-	src->y1 = state->src_y;
-	src->x2 = state->src_x + state->src_w;
-	src->y2 = state->src_y + state->src_h;
-
-	dst->x1 = state->crtc_x;
-	dst->y1 = state->crtc_y;
-	dst->x2 = state->crtc_x + state->crtc_w;
-	dst->y2 = state->crtc_y + state->crtc_h;
+	*src = drm_plane_state_src(state);
+	*dst = drm_plane_state_dest(state);
 
 	if (!fb) {
 		state->visible = false;
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index b22a94dd7b53..8d77b2462594 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -290,7 +290,8 @@ static void drm_gem_unmap_dma_buf(struct dma_buf_attachment *attach,
  *
  * This wraps dma_buf_export() for use by generic GEM drivers that are using
  * drm_gem_dmabuf_release(). In addition to calling dma_buf_export(), we take
- * a reference to the drm_device which is released by drm_gem_dmabuf_release().
+ * a reference to the &drm_device and the exported &drm_gem_object (stored in
+ * exp_info->priv) which is released by drm_gem_dmabuf_release().
  *
  * Returns the new dmabuf.
  */
@@ -300,8 +301,11 @@ struct dma_buf *drm_gem_dmabuf_export(struct drm_device *dev,
 	struct dma_buf *dma_buf;
 
 	dma_buf = dma_buf_export(exp_info);
-	if (!IS_ERR(dma_buf))
-		drm_dev_ref(dev);
+	if (IS_ERR(dma_buf))
+		return dma_buf;
+
+	drm_dev_ref(dev);
+	drm_gem_object_reference(exp_info->priv);
 
 	return dma_buf;
 }
@@ -472,8 +476,6 @@ static struct dma_buf *export_and_register_object(struct drm_device *dev,
 	 */
 	obj->dma_buf = dmabuf;
 	get_dma_buf(obj->dma_buf);
-	/* Grab a new ref since the callers is now used by the dma-buf */
-	drm_gem_object_reference(obj);
 
 	return dmabuf;
 }
diff --git a/drivers/gpu/drm/drm_print.c b/drivers/gpu/drm/drm_print.c
new file mode 100644
index 000000000000..ad3caaa1f48b
--- /dev/null
+++ b/drivers/gpu/drm/drm_print.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2016 Red Hat
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ * Rob Clark <robdclark@gmail.com>
+ */
+
+#include <stdarg.h>
+#include <linux/seq_file.h>
+#include <drm/drmP.h>
+#include <drm/drm_print.h>
+
+void __drm_printfn_seq_file(struct drm_printer *p, struct va_format *vaf)
+{
+	seq_printf(p->arg, "%pV", vaf);
+}
+EXPORT_SYMBOL(__drm_printfn_seq_file);
+
+void __drm_printfn_info(struct drm_printer *p, struct va_format *vaf)
+{
+	dev_printk(KERN_INFO, p->arg, "[" DRM_NAME "] %pV", vaf);
+}
+EXPORT_SYMBOL(__drm_printfn_info);
+
+/**
+ * drm_printf - print to a &drm_printer stream
+ * @p: the &drm_printer
+ * @f: format string
+ */
+void drm_printf(struct drm_printer *p, const char *f, ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	va_start(args, f);
+	vaf.fmt = f;
+	vaf.va = &args;
+	p->printfn(p, &vaf);
+	va_end(args);
+}
+EXPORT_SYMBOL(drm_printf);
diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c
index f6b64d7d3528..ac953f037be7 100644
--- a/drivers/gpu/drm/drm_probe_helper.c
+++ b/drivers/gpu/drm/drm_probe_helper.c
@@ -152,6 +152,14 @@ void drm_kms_helper_poll_enable_locked(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_kms_helper_poll_enable_locked);
 
+static enum drm_connector_status
+drm_connector_detect(struct drm_connector *connector, bool force)
+{
+	return connector->funcs->detect ?
+		connector->funcs->detect(connector, force) :
+		connector_status_connected;
+}
+
 /**
  * drm_helper_probe_single_connector_modes - get complete set of display modes
  * @connector: connector to probe
@@ -239,7 +247,7 @@ int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
 		if (connector->funcs->force)
 			connector->funcs->force(connector);
 	} else {
-		connector->status = connector->funcs->detect(connector, true);
+		connector->status = drm_connector_detect(connector, true);
 	}
 
 	/*
@@ -384,7 +392,11 @@ static void output_poll_execute(struct work_struct *work)
 	if (!drm_kms_helper_poll)
 		goto out;
 
-	mutex_lock(&dev->mode_config.mutex);
+	if (!mutex_trylock(&dev->mode_config.mutex)) {
+		repoll = true;
+		goto out;
+	}
+
 	drm_for_each_connector(connector, dev) {
 
 		/* Ignore forced connectors. */
@@ -405,7 +417,7 @@ static void output_poll_execute(struct work_struct *work)
 
 		repoll = true;
 
-		connector->status = connector->funcs->detect(connector, false);
+		connector->status = drm_connector_detect(connector, false);
 		if (old_status != connector->status) {
 			const char *old, *new;
 
@@ -565,7 +577,7 @@ bool drm_helper_hpd_irq_event(struct drm_device *dev)
 
 		old_status = connector->status;
 
-		connector->status = connector->funcs->detect(connector, false);
+		connector->status = drm_connector_detect(connector, false);
 		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n",
 			      connector->base.id,
 			      connector->name,
diff --git a/drivers/gpu/drm/drm_property.c b/drivers/gpu/drm/drm_property.c
index a4d81cf4ffa0..24be69d29964 100644
--- a/drivers/gpu/drm/drm_property.c
+++ b/drivers/gpu/drm/drm_property.c
@@ -65,9 +65,9 @@ static bool drm_property_type_valid(struct drm_property *property)
  * @num_values: number of pre-defined values
  *
  * This creates a new generic drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy(), which is done automatically when calling
- * drm_mode_config_cleanup().
+ * object with drm_object_attach_property(). The returned property object must
+ * be freed with drm_property_destroy(), which is done automatically when
+ * calling drm_mode_config_cleanup().
  *
  * Returns:
  * A pointer to the newly created property on success, NULL on failure.
@@ -125,9 +125,9 @@ EXPORT_SYMBOL(drm_property_create);
  * @num_values: number of pre-defined values
  *
  * This creates a new generic drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy(), which is done automatically when calling
- * drm_mode_config_cleanup().
+ * object with drm_object_attach_property(). The returned property object must
+ * be freed with drm_property_destroy(), which is done automatically when
+ * calling drm_mode_config_cleanup().
  *
  * Userspace is only allowed to set one of the predefined values for enumeration
  * properties.
@@ -173,9 +173,9 @@ EXPORT_SYMBOL(drm_property_create_enum);
  * @supported_bits: bitmask of all supported enumeration values
  *
  * This creates a new bitmask drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy(), which is done automatically when calling
- * drm_mode_config_cleanup().
+ * object with drm_object_attach_property(). The returned property object must
+ * be freed with drm_property_destroy(), which is done automatically when
+ * calling drm_mode_config_cleanup().
  *
  * Compared to plain enumeration properties userspace is allowed to set any
  * or'ed together combination of the predefined property bitflag values
@@ -245,9 +245,9 @@ static struct drm_property *property_create_range(struct drm_device *dev,
  * @max: maximum value of the property
  *
  * This creates a new generic drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy(), which is done automatically when calling
- * drm_mode_config_cleanup().
+ * object with drm_object_attach_property(). The returned property object must
+ * be freed with drm_property_destroy(), which is done automatically when
+ * calling drm_mode_config_cleanup().
  *
  * Userspace is allowed to set any unsigned integer value in the (min, max)
  * range inclusive.
@@ -273,9 +273,9 @@ EXPORT_SYMBOL(drm_property_create_range);
  * @max: maximum value of the property
  *
  * This creates a new generic drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy(), which is done automatically when calling
- * drm_mode_config_cleanup().
+ * object with drm_object_attach_property(). The returned property object must
+ * be freed with drm_property_destroy(), which is done automatically when
+ * calling drm_mode_config_cleanup().
  *
  * Userspace is allowed to set any signed integer value in the (min, max)
  * range inclusive.
@@ -300,9 +300,9 @@ EXPORT_SYMBOL(drm_property_create_signed_range);
  * @type: object type from DRM_MODE_OBJECT_* defines
  *
  * This creates a new generic drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy(), which is done automatically when calling
- * drm_mode_config_cleanup().
+ * object with drm_object_attach_property(). The returned property object must
+ * be freed with drm_property_destroy(), which is done automatically when
+ * calling drm_mode_config_cleanup().
  *
  * Userspace is only allowed to set this to any property value of the given
  * @type. Only useful for atomic properties, which is enforced.
@@ -338,9 +338,9 @@ EXPORT_SYMBOL(drm_property_create_object);
  * @name: name of the property
  *
  * This creates a new generic drm property which can then be attached to a drm
- * object with drm_object_attach_property. The returned property object must be
- * freed with drm_property_destroy(), which is done automatically when calling
- * drm_mode_config_cleanup().
+ * object with drm_object_attach_property(). The returned property object must
+ * be freed with drm_property_destroy(), which is done automatically when
+ * calling drm_mode_config_cleanup().
  *
  * This is implemented as a ranged property with only {0, 1} as valid values.
  *
@@ -729,7 +729,6 @@ int drm_mode_getblob_ioctl(struct drm_device *dev,
 	struct drm_mode_get_blob *out_resp = data;
 	struct drm_property_blob *blob;
 	int ret = 0;
-	void __user *blob_ptr;
 
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 		return -EINVAL;
@@ -739,8 +738,9 @@ int drm_mode_getblob_ioctl(struct drm_device *dev,
 		return -ENOENT;
 
 	if (out_resp->length == blob->length) {
-		blob_ptr = (void __user *)(unsigned long)out_resp->data;
-		if (copy_to_user(blob_ptr, blob->data, blob->length)) {
+		if (copy_to_user(u64_to_user_ptr(out_resp->data),
+				 blob->data,
+				 blob->length)) {
 			ret = -EFAULT;
 			goto unref;
 		}
@@ -757,7 +757,6 @@ int drm_mode_createblob_ioctl(struct drm_device *dev,
 {
 	struct drm_mode_create_blob *out_resp = data;
 	struct drm_property_blob *blob;
-	void __user *blob_ptr;
 	int ret = 0;
 
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
@@ -767,8 +766,9 @@ int drm_mode_createblob_ioctl(struct drm_device *dev,
 	if (IS_ERR(blob))
 		return PTR_ERR(blob);
 
-	blob_ptr = (void __user *)(unsigned long)out_resp->data;
-	if (copy_from_user(blob->data, blob_ptr, out_resp->length)) {
+	if (copy_from_user(blob->data,
+			   u64_to_user_ptr(out_resp->data),
+			   out_resp->length)) {
 		ret = -EFAULT;
 		goto out_blob;
 	}
diff --git a/drivers/gpu/drm/drm_rect.c b/drivers/gpu/drm/drm_rect.c
index 73e53a8d1b37..e6057d8cdcd5 100644
--- a/drivers/gpu/drm/drm_rect.c
+++ b/drivers/gpu/drm/drm_rect.c
@@ -281,17 +281,10 @@ EXPORT_SYMBOL(drm_rect_calc_vscale_relaxed);
  */
 void drm_rect_debug_print(const char *prefix, const struct drm_rect *r, bool fixed_point)
 {
-	int w = drm_rect_width(r);
-	int h = drm_rect_height(r);
-
 	if (fixed_point)
-		DRM_DEBUG_KMS("%s%d.%06ux%d.%06u%+d.%06u%+d.%06u\n", prefix,
-			      w >> 16, ((w & 0xffff) * 15625) >> 10,
-			      h >> 16, ((h & 0xffff) * 15625) >> 10,
-			      r->x1 >> 16, ((r->x1 & 0xffff) * 15625) >> 10,
-			      r->y1 >> 16, ((r->y1 & 0xffff) * 15625) >> 10);
+		DRM_DEBUG_KMS("%s" DRM_RECT_FP_FMT "\n", prefix, DRM_RECT_FP_ARG(r));
 	else
-		DRM_DEBUG_KMS("%s%dx%d%+d%+d\n", prefix, w, h, r->x1, r->y1);
+		DRM_DEBUG_KMS("%s" DRM_RECT_FMT "\n", prefix, DRM_RECT_ARG(r));
 }
 EXPORT_SYMBOL(drm_rect_debug_print);
 
diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c
index caa4e4ca616d..bd311c77c254 100644
--- a/drivers/gpu/drm/drm_vm.c
+++ b/drivers/gpu/drm/drm_vm.c
@@ -124,8 +124,7 @@ static int drm_do_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 		 * Using vm_pgoff as a selector forces us to use this unusual
 		 * addressing scheme.
 		 */
-		resource_size_t offset = (unsigned long)vmf->virtual_address -
-			vma->vm_start;
+		resource_size_t offset = vmf->address - vma->vm_start;
 		resource_size_t baddr = map->offset + offset;
 		struct drm_agp_mem *agpmem;
 		struct page *page;
@@ -195,7 +194,7 @@ static int drm_do_vm_shm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	if (!map)
 		return VM_FAULT_SIGBUS;	/* Nothing allocated */
 
-	offset = (unsigned long)vmf->virtual_address - vma->vm_start;
+	offset = vmf->address - vma->vm_start;
 	i = (unsigned long)map->handle + offset;
 	page = vmalloc_to_page((void *)i);
 	if (!page)
@@ -301,7 +300,8 @@ static int drm_do_vm_dma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	if (!dma->pagelist)
 		return VM_FAULT_SIGBUS;	/* Nothing allocated */
 
-	offset = (unsigned long)vmf->virtual_address - vma->vm_start;	/* vm_[pg]off[set] should be 0 */
+	offset = vmf->address - vma->vm_start;
+					/* vm_[pg]off[set] should be 0 */
 	page_nr = offset >> PAGE_SHIFT; /* page_nr could just be vmf->pgoff */
 	page = virt_to_page((void *)dma->pagelist[page_nr]);
 
@@ -337,7 +337,7 @@ static int drm_do_vm_sg_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	if (!entry->pagelist)
 		return VM_FAULT_SIGBUS;	/* Nothing allocated */
 
-	offset = (unsigned long)vmf->virtual_address - vma->vm_start;
+	offset = vmf->address - vma->vm_start;
 	map_offset = map->offset - (unsigned long)dev->sg->virtual;
 	page_offset = (offset >> PAGE_SHIFT) + (map_offset >> PAGE_SHIFT);
 	page = entry->pagelist[page_offset];
diff --git a/drivers/gpu/drm/etnaviv/cmdstream.xml.h b/drivers/gpu/drm/etnaviv/cmdstream.xml.h
index 8c44ba9a694e..65f1ba1099bd 100644
--- a/drivers/gpu/drm/etnaviv/cmdstream.xml.h
+++ b/drivers/gpu/drm/etnaviv/cmdstream.xml.h
@@ -8,10 +8,34 @@ http://0x04.net/cgit/index.cgi/rules-ng-ng
 git clone git://0x04.net/rules-ng-ng
 
 The rules-ng-ng source files this header was generated from are:
-- cmdstream.xml (  12589 bytes, from 2014-02-17 14:57:56)
-- common.xml    (  18437 bytes, from 2015-03-25 11:27:41)
-
-Copyright (C) 2014
+- cmdstream.xml (  14094 bytes, from 2016-11-11 06:55:14)
+- copyright.xml (   1597 bytes, from 2016-10-29 07:29:22)
+- common.xml    (  23344 bytes, from 2016-11-10 15:14:07)
+
+Copyright (C) 2012-2016 by the following authors:
+- Wladimir J. van der Laan <laanwj@gmail.com>
+- Christian Gmeiner <christian.gmeiner@gmail.com>
+- Lucas Stach <l.stach@pengutronix.de>
+- Russell King <rmk@arm.linux.org.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sub license,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice (including the
+next paragraph) shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
 */
 
 
@@ -26,6 +50,7 @@ Copyright (C) 2014
 #define FE_OPCODE_STALL						0x00000009
 #define FE_OPCODE_CALL						0x0000000a
 #define FE_OPCODE_RETURN					0x0000000b
+#define FE_OPCODE_DRAW_INSTANCED				0x0000000c
 #define FE_OPCODE_CHIP_SELECT					0x0000000d
 #define PRIMITIVE_TYPE_POINTS					0x00000001
 #define PRIMITIVE_TYPE_LINES					0x00000002
@@ -214,5 +239,32 @@ Copyright (C) 2014
 #define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP1			0x00000002
 #define VIV_FE_CHIP_SELECT_HEADER_ENABLE_CHIP0			0x00000001
 
+#define VIV_FE_DRAW_INSTANCED					0x00000000
+
+#define VIV_FE_DRAW_INSTANCED_HEADER				0x00000000
+#define VIV_FE_DRAW_INSTANCED_HEADER_OP__MASK			0xf8000000
+#define VIV_FE_DRAW_INSTANCED_HEADER_OP__SHIFT			27
+#define VIV_FE_DRAW_INSTANCED_HEADER_OP_DRAW_INSTANCED		0x60000000
+#define VIV_FE_DRAW_INSTANCED_HEADER_INDEXED			0x00100000
+#define VIV_FE_DRAW_INSTANCED_HEADER_TYPE__MASK			0x000f0000
+#define VIV_FE_DRAW_INSTANCED_HEADER_TYPE__SHIFT		16
+#define VIV_FE_DRAW_INSTANCED_HEADER_TYPE(x)			(((x) << VIV_FE_DRAW_INSTANCED_HEADER_TYPE__SHIFT) & VIV_FE_DRAW_INSTANCED_HEADER_TYPE__MASK)
+#define VIV_FE_DRAW_INSTANCED_HEADER_INSTANCE_COUNT_LO__MASK	0x0000ffff
+#define VIV_FE_DRAW_INSTANCED_HEADER_INSTANCE_COUNT_LO__SHIFT	0
+#define VIV_FE_DRAW_INSTANCED_HEADER_INSTANCE_COUNT_LO(x)	(((x) << VIV_FE_DRAW_INSTANCED_HEADER_INSTANCE_COUNT_LO__SHIFT) & VIV_FE_DRAW_INSTANCED_HEADER_INSTANCE_COUNT_LO__MASK)
+
+#define VIV_FE_DRAW_INSTANCED_COUNT				0x00000004
+#define VIV_FE_DRAW_INSTANCED_COUNT_INSTANCE_COUNT_HI__MASK	0xff000000
+#define VIV_FE_DRAW_INSTANCED_COUNT_INSTANCE_COUNT_HI__SHIFT	24
+#define VIV_FE_DRAW_INSTANCED_COUNT_INSTANCE_COUNT_HI(x)	(((x) << VIV_FE_DRAW_INSTANCED_COUNT_INSTANCE_COUNT_HI__SHIFT) & VIV_FE_DRAW_INSTANCED_COUNT_INSTANCE_COUNT_HI__MASK)
+#define VIV_FE_DRAW_INSTANCED_COUNT_VERTEX_COUNT__MASK		0x00ffffff
+#define VIV_FE_DRAW_INSTANCED_COUNT_VERTEX_COUNT__SHIFT		0
+#define VIV_FE_DRAW_INSTANCED_COUNT_VERTEX_COUNT(x)		(((x) << VIV_FE_DRAW_INSTANCED_COUNT_VERTEX_COUNT__SHIFT) & VIV_FE_DRAW_INSTANCED_COUNT_VERTEX_COUNT__MASK)
+
+#define VIV_FE_DRAW_INSTANCED_START				0x00000008
+#define VIV_FE_DRAW_INSTANCED_START_INDEX__MASK			0xffffffff
+#define VIV_FE_DRAW_INSTANCED_START_INDEX__SHIFT		0
+#define VIV_FE_DRAW_INSTANCED_START_INDEX(x)			(((x) << VIV_FE_DRAW_INSTANCED_START_INDEX__SHIFT) & VIV_FE_DRAW_INSTANCED_START_INDEX__MASK)
+
 
 #endif /* CMDSTREAM_XML */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c b/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c
index dcfd565c88d1..2a2e5e366ab7 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c
@@ -143,6 +143,7 @@ static bool etnaviv_validate_load_state(struct etna_validation_state *state,
 static uint8_t cmd_length[32] = {
 	[FE_OPCODE_DRAW_PRIMITIVES] = 4,
 	[FE_OPCODE_DRAW_INDEXED_PRIMITIVES] = 6,
+	[FE_OPCODE_DRAW_INSTANCED] = 4,
 	[FE_OPCODE_NOP] = 2,
 	[FE_OPCODE_STALL] = 2,
 };
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index aa687669e22b..00368b14d08d 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -16,6 +16,7 @@
 
 #include <linux/component.h>
 #include <linux/of_platform.h>
+#include <drm/drm_of.h>
 
 #include "etnaviv_drv.h"
 #include "etnaviv_gpu.h"
@@ -478,9 +479,7 @@ static const struct file_operations fops = {
 	.open               = drm_open,
 	.release            = drm_release,
 	.unlocked_ioctl     = drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl       = drm_compat_ioctl,
-#endif
 	.poll               = drm_poll,
 	.read               = drm_read,
 	.llseek             = no_llseek,
@@ -505,6 +504,7 @@ static struct drm_driver etnaviv_drm_driver = {
 	.gem_prime_import_sg_table = etnaviv_gem_prime_import_sg_table,
 	.gem_prime_vmap     = etnaviv_gem_prime_vmap,
 	.gem_prime_vunmap   = etnaviv_gem_prime_vunmap,
+	.gem_prime_mmap     = etnaviv_gem_prime_mmap,
 #ifdef CONFIG_DEBUG_FS
 	.debugfs_init       = etnaviv_debugfs_init,
 	.debugfs_cleanup    = etnaviv_debugfs_cleanup,
@@ -629,8 +629,8 @@ static int etnaviv_pdev_probe(struct platform_device *pdev)
 			if (!core_node)
 				break;
 
-			component_match_add(&pdev->dev, &match, compare_of,
-					    core_node);
+			drm_of_component_match_add(&pdev->dev, &match,
+						   compare_of, core_node);
 			of_node_put(core_node);
 		}
 	} else if (dev->platform_data) {
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
index 65e057639653..c255eda40526 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
@@ -78,6 +78,8 @@ int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset);
 struct sg_table *etnaviv_gem_prime_get_sg_table(struct drm_gem_object *obj);
 void *etnaviv_gem_prime_vmap(struct drm_gem_object *obj);
 void etnaviv_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
+int etnaviv_gem_prime_mmap(struct drm_gem_object *obj,
+			   struct vm_area_struct *vma);
 struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
 	struct dma_buf_attachment *attach, struct sg_table *sg);
 int etnaviv_gem_prime_pin(struct drm_gem_object *obj);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c
index 2bef501d4a17..af65491a78e2 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_dump.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c
@@ -160,7 +160,8 @@ void etnaviv_core_dump(struct etnaviv_gpu *gpu)
 	file_size += sizeof(*iter.hdr) * n_obj;
 
 	/* Allocate the file in vmalloc memory, it's likely to be big */
-	iter.start = vmalloc(file_size);
+	iter.start = __vmalloc(file_size, GFP_KERNEL | __GFP_HIGHMEM |
+			       __GFP_NOWARN | __GFP_NORETRY, PAGE_KERNEL);
 	if (!iter.start) {
 		dev_warn(gpu->dev, "failed to allocate devcoredump file\n");
 		return;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index 0370b842d9cc..114dddbd297b 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -202,15 +202,14 @@ int etnaviv_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	}
 
 	/* We don't use vmf->pgoff since that has the fake offset: */
-	pgoff = ((unsigned long)vmf->virtual_address -
-			vma->vm_start) >> PAGE_SHIFT;
+	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
 
 	page = pages[pgoff];
 
-	VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
+	VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address,
 	     page_to_pfn(page), page_to_pfn(page) << PAGE_SHIFT);
 
-	ret = vm_insert_page(vma, (unsigned long)vmf->virtual_address, page);
+	ret = vm_insert_page(vma, vmf->address, page);
 
 out:
 	switch (ret) {
@@ -409,20 +408,16 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
 	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
 	struct drm_device *dev = obj->dev;
 	bool write = !!(op & ETNA_PREP_WRITE);
-	int ret;
-
-	if (op & ETNA_PREP_NOSYNC) {
-		if (!reservation_object_test_signaled_rcu(etnaviv_obj->resv,
-							  write))
-			return -EBUSY;
-	} else {
-		unsigned long remain = etnaviv_timeout_to_jiffies(timeout);
+	unsigned long remain =
+		op & ETNA_PREP_NOSYNC ? 0 : etnaviv_timeout_to_jiffies(timeout);
+	long lret;
 
-		ret = reservation_object_wait_timeout_rcu(etnaviv_obj->resv,
-							  write, true, remain);
-		if (ret <= 0)
-			return ret == 0 ? -ETIMEDOUT : ret;
-	}
+	lret = reservation_object_wait_timeout_rcu(etnaviv_obj->resv,
+						   write, true, remain);
+	if (lret < 0)
+		return lret;
+	else if (lret == 0)
+		return remain == 0 ? -EBUSY : -ETIMEDOUT;
 
 	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
 		if (!etnaviv_obj->sgt) {
@@ -470,10 +465,10 @@ int etnaviv_gem_wait_bo(struct etnaviv_gpu *gpu, struct drm_gem_object *obj,
 }
 
 #ifdef CONFIG_DEBUG_FS
-static void etnaviv_gem_describe_fence(struct fence *fence,
+static void etnaviv_gem_describe_fence(struct dma_fence *fence,
 	const char *type, struct seq_file *m)
 {
-	if (!test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+	if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
 		seq_printf(m, "\t%9s: %s %s seq %u\n",
 			   type,
 			   fence->ops->get_driver_name(fence),
@@ -486,7 +481,7 @@ static void etnaviv_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
 	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
 	struct reservation_object *robj = etnaviv_obj->resv;
 	struct reservation_object_list *fobj;
-	struct fence *fence;
+	struct dma_fence *fence;
 	unsigned long off = drm_vma_node_start(&obj->vma_node);
 
 	seq_printf(m, "%08x: %c %2d (%2d) %08lx %p %zd\n",
@@ -763,7 +758,7 @@ static struct page **etnaviv_gem_userptr_do_get_pages(
 	down_read(&mm->mmap_sem);
 	while (pinned < npages) {
 		ret = get_user_pages_remote(task, mm, ptr, npages - pinned,
-					    flags, pvec + pinned, NULL);
+					    flags, pvec + pinned, NULL, NULL);
 		if (ret < 0)
 			break;
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
index b93618c1aa69..62b47972a52e 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
@@ -23,10 +23,12 @@
 struct sg_table *etnaviv_gem_prime_get_sg_table(struct drm_gem_object *obj)
 {
 	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+	int npages = obj->size >> PAGE_SHIFT;
 
-	BUG_ON(!etnaviv_obj->sgt);  /* should have already pinned! */
+	if (WARN_ON(!etnaviv_obj->pages))  /* should have already pinned! */
+		return NULL;
 
-	return etnaviv_obj->sgt;
+	return drm_prime_pages_to_sg(etnaviv_obj->pages, npages);
 }
 
 void *etnaviv_gem_prime_vmap(struct drm_gem_object *obj)
@@ -39,6 +41,19 @@ void etnaviv_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
 	/* TODO msm_gem_vunmap() */
 }
 
+int etnaviv_gem_prime_mmap(struct drm_gem_object *obj,
+			   struct vm_area_struct *vma)
+{
+	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+	int ret;
+
+	ret = drm_gem_mmap_obj(obj, obj->size, vma);
+	if (ret < 0)
+		return ret;
+
+	return etnaviv_obj->ops->mmap(etnaviv_obj, vma);
+}
+
 int etnaviv_gem_prime_pin(struct drm_gem_object *obj)
 {
 	if (!obj->import_attach) {
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index b1254f885fed..0a67124bb2a4 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -15,7 +15,7 @@
  */
 
 #include <linux/component.h>
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 #include <linux/moduleparam.h>
 #include <linux/of_device.h>
 #include "etnaviv_dump.h"
@@ -639,6 +639,10 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu)
 			gpu->memory_base = PHYS_OFFSET;
 		else
 			gpu->memory_base = dma_mask - SZ_2G + 1;
+	} else if (PHYS_OFFSET >= SZ_2G) {
+		dev_info(gpu->dev, "Need to move linear window on MC1.0, disabling TS\n");
+		gpu->memory_base = PHYS_OFFSET;
+		gpu->identity.features &= ~chipFeatures_FAST_CLEAR;
 	}
 
 	ret = etnaviv_hw_reset(gpu);
@@ -882,7 +886,7 @@ static void recover_worker(struct work_struct *work)
 	for (i = 0; i < ARRAY_SIZE(gpu->event); i++) {
 		if (!gpu->event[i].used)
 			continue;
-		fence_signal(gpu->event[i].fence);
+		dma_fence_signal(gpu->event[i].fence);
 		gpu->event[i].fence = NULL;
 		gpu->event[i].used = false;
 		complete(&gpu->event_free);
@@ -952,55 +956,55 @@ static void hangcheck_disable(struct etnaviv_gpu *gpu)
 /* fence object management */
 struct etnaviv_fence {
 	struct etnaviv_gpu *gpu;
-	struct fence base;
+	struct dma_fence base;
 };
 
-static inline struct etnaviv_fence *to_etnaviv_fence(struct fence *fence)
+static inline struct etnaviv_fence *to_etnaviv_fence(struct dma_fence *fence)
 {
 	return container_of(fence, struct etnaviv_fence, base);
 }
 
-static const char *etnaviv_fence_get_driver_name(struct fence *fence)
+static const char *etnaviv_fence_get_driver_name(struct dma_fence *fence)
 {
 	return "etnaviv";
 }
 
-static const char *etnaviv_fence_get_timeline_name(struct fence *fence)
+static const char *etnaviv_fence_get_timeline_name(struct dma_fence *fence)
 {
 	struct etnaviv_fence *f = to_etnaviv_fence(fence);
 
 	return dev_name(f->gpu->dev);
 }
 
-static bool etnaviv_fence_enable_signaling(struct fence *fence)
+static bool etnaviv_fence_enable_signaling(struct dma_fence *fence)
 {
 	return true;
 }
 
-static bool etnaviv_fence_signaled(struct fence *fence)
+static bool etnaviv_fence_signaled(struct dma_fence *fence)
 {
 	struct etnaviv_fence *f = to_etnaviv_fence(fence);
 
 	return fence_completed(f->gpu, f->base.seqno);
 }
 
-static void etnaviv_fence_release(struct fence *fence)
+static void etnaviv_fence_release(struct dma_fence *fence)
 {
 	struct etnaviv_fence *f = to_etnaviv_fence(fence);
 
 	kfree_rcu(f, base.rcu);
 }
 
-static const struct fence_ops etnaviv_fence_ops = {
+static const struct dma_fence_ops etnaviv_fence_ops = {
 	.get_driver_name = etnaviv_fence_get_driver_name,
 	.get_timeline_name = etnaviv_fence_get_timeline_name,
 	.enable_signaling = etnaviv_fence_enable_signaling,
 	.signaled = etnaviv_fence_signaled,
-	.wait = fence_default_wait,
+	.wait = dma_fence_default_wait,
 	.release = etnaviv_fence_release,
 };
 
-static struct fence *etnaviv_gpu_fence_alloc(struct etnaviv_gpu *gpu)
+static struct dma_fence *etnaviv_gpu_fence_alloc(struct etnaviv_gpu *gpu)
 {
 	struct etnaviv_fence *f;
 
@@ -1010,8 +1014,8 @@ static struct fence *etnaviv_gpu_fence_alloc(struct etnaviv_gpu *gpu)
 
 	f->gpu = gpu;
 
-	fence_init(&f->base, &etnaviv_fence_ops, &gpu->fence_spinlock,
-		   gpu->fence_context, ++gpu->next_fence);
+	dma_fence_init(&f->base, &etnaviv_fence_ops, &gpu->fence_spinlock,
+		       gpu->fence_context, ++gpu->next_fence);
 
 	return &f->base;
 }
@@ -1021,7 +1025,7 @@ int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj,
 {
 	struct reservation_object *robj = etnaviv_obj->resv;
 	struct reservation_object_list *fobj;
-	struct fence *fence;
+	struct dma_fence *fence;
 	int i, ret;
 
 	if (!exclusive) {
@@ -1039,7 +1043,7 @@ int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj,
 		/* Wait on any existing exclusive fence which isn't our own */
 		fence = reservation_object_get_excl(robj);
 		if (fence && fence->context != context) {
-			ret = fence_wait(fence, true);
+			ret = dma_fence_wait(fence, true);
 			if (ret)
 				return ret;
 		}
@@ -1052,7 +1056,7 @@ int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj,
 		fence = rcu_dereference_protected(fobj->shared[i],
 						reservation_object_held(robj));
 		if (fence->context != context) {
-			ret = fence_wait(fence, true);
+			ret = dma_fence_wait(fence, true);
 			if (ret)
 				return ret;
 		}
@@ -1158,11 +1162,11 @@ static void retire_worker(struct work_struct *work)
 
 	mutex_lock(&gpu->lock);
 	list_for_each_entry_safe(cmdbuf, tmp, &gpu->active_cmd_list, node) {
-		if (!fence_is_signaled(cmdbuf->fence))
+		if (!dma_fence_is_signaled(cmdbuf->fence))
 			break;
 
 		list_del(&cmdbuf->node);
-		fence_put(cmdbuf->fence);
+		dma_fence_put(cmdbuf->fence);
 
 		for (i = 0; i < cmdbuf->nr_bos; i++) {
 			struct etnaviv_vram_mapping *mapping = cmdbuf->bo_map[i];
@@ -1275,7 +1279,7 @@ void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu)
 int etnaviv_gpu_submit(struct etnaviv_gpu *gpu,
 	struct etnaviv_gem_submit *submit, struct etnaviv_cmdbuf *cmdbuf)
 {
-	struct fence *fence;
+	struct dma_fence *fence;
 	unsigned int event, i;
 	int ret;
 
@@ -1391,7 +1395,7 @@ static irqreturn_t irq_handler(int irq, void *data)
 		}
 
 		while ((event = ffs(intr)) != 0) {
-			struct fence *fence;
+			struct dma_fence *fence;
 
 			event -= 1;
 
@@ -1401,7 +1405,7 @@ static irqreturn_t irq_handler(int irq, void *data)
 
 			fence = gpu->event[event].fence;
 			gpu->event[event].fence = NULL;
-			fence_signal(fence);
+			dma_fence_signal(fence);
 
 			/*
 			 * Events can be processed out of order.  Eg,
@@ -1553,7 +1557,7 @@ static int etnaviv_gpu_bind(struct device *dev, struct device *master,
 		return ret;
 
 	gpu->drm = drm;
-	gpu->fence_context = fence_context_alloc(1);
+	gpu->fence_context = dma_fence_context_alloc(1);
 	spin_lock_init(&gpu->fence_spinlock);
 
 	INIT_LIST_HEAD(&gpu->active_cmd_list);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
index 73c278dc3706..8c6b824e9d0a 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
@@ -89,7 +89,7 @@ struct etnaviv_chip_identity {
 
 struct etnaviv_event {
 	bool used;
-	struct fence *fence;
+	struct dma_fence *fence;
 };
 
 struct etnaviv_cmdbuf;
@@ -163,7 +163,7 @@ struct etnaviv_cmdbuf {
 	/* vram node used if the cmdbuf is mapped through the MMUv2 */
 	struct drm_mm_node vram_node;
 	/* fence after which this buffer is to be disposed */
-	struct fence *fence;
+	struct dma_fence *fence;
 	/* target exec state */
 	u32 exec_state;
 	/* per GPU in-flight list */
diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig
index 465d344f3391..d706ca4e2f02 100644
--- a/drivers/gpu/drm/exynos/Kconfig
+++ b/drivers/gpu/drm/exynos/Kconfig
@@ -114,7 +114,7 @@ config DRM_EXYNOS_ROTATOR
 
 config DRM_EXYNOS_GSC
 	bool "GScaler"
-	depends on DRM_EXYNOS_IPP && ARCH_EXYNOS5 && !VIDEO_SAMSUNG_EXYNOS_GSC
+	depends on DRM_EXYNOS_IPP && ARCH_EXYNOS5 && VIDEO_SAMSUNG_EXYNOS_GSC=n
 	help
 	  Choose this option if you want to use Exynos GSC for DRM.
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c
index f86e7c846678..739180ac3da5 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c
@@ -69,7 +69,7 @@ static void exynos_atomic_commit_complete(struct exynos_atomic_commit *commit)
 
 	drm_atomic_helper_cleanup_planes(dev, state);
 
-	drm_atomic_state_free(state);
+	drm_atomic_state_put(state);
 
 	spin_lock(&priv->lock);
 	priv->pending &= ~commit->crtcs;
@@ -254,6 +254,7 @@ int exynos_atomic_commit(struct drm_device *dev, struct drm_atomic_state *state,
 
 	drm_atomic_helper_swap_state(state, true);
 
+	drm_atomic_state_get(state);
 	if (nonblock)
 		schedule_work(&commit->work);
 	else
@@ -365,9 +366,7 @@ static const struct file_operations exynos_drm_driver_fops = {
 	.poll		= drm_poll,
 	.read		= drm_read,
 	.unlocked_ioctl	= drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.release	= drm_release,
 };
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
index 4cfb39d543b4..9f35deb56170 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
@@ -63,15 +63,11 @@ static int exynos_drm_fb_mmap(struct fb_info *info,
 
 static struct fb_ops exynos_drm_fb_ops = {
 	.owner		= THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_mmap        = exynos_drm_fb_mmap,
 	.fb_fillrect	= drm_fb_helper_cfb_fillrect,
 	.fb_copyarea	= drm_fb_helper_cfb_copyarea,
 	.fb_imageblit	= drm_fb_helper_cfb_imageblit,
-	.fb_check_var	= drm_fb_helper_check_var,
-	.fb_set_par	= drm_fb_helper_set_par,
-	.fb_blank	= drm_fb_helper_blank,
-	.fb_pan_display	= drm_fb_helper_pan_display,
-	.fb_setcmap	= drm_fb_helper_setcmap,
 };
 
 static int exynos_drm_fbdev_update(struct drm_fb_helper *helper,
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimc.c b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
index 147ef0d298cb..95871577015d 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
@@ -1433,7 +1433,7 @@ static int fimc_ippdrv_start(struct device *dev, enum drm_exynos_ipp_cmd cmd)
 		&img_pos[EXYNOS_DRM_OPS_SRC],
 		&img_pos[EXYNOS_DRM_OPS_DST]);
 	if (ret) {
-		dev_err(dev, "failed to set precalser.\n");
+		dev_err(dev, "failed to set prescaler.\n");
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.c b/drivers/gpu/drm/exynos/exynos_drm_gem.c
index f2ae72ba7d5a..57b81460fec8 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.c
@@ -231,12 +231,12 @@ struct exynos_drm_gem *exynos_drm_gem_create(struct drm_device *dev,
 	int ret;
 
 	if (flags & ~(EXYNOS_BO_MASK)) {
-		DRM_ERROR("invalid flags.\n");
+		DRM_ERROR("invalid GEM buffer flags: %u\n", flags);
 		return ERR_PTR(-EINVAL);
 	}
 
 	if (!size) {
-		DRM_ERROR("invalid size.\n");
+		DRM_ERROR("invalid GEM buffer size: %lu\n", size);
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -455,8 +455,7 @@ int exynos_drm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	pgoff_t page_offset;
 	int ret;
 
-	page_offset = ((unsigned long)vmf->virtual_address -
-			vma->vm_start) >> PAGE_SHIFT;
+	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
 
 	if (page_offset >= (exynos_gem->size >> PAGE_SHIFT)) {
 		DRM_ERROR("invalid page offset\n");
@@ -465,8 +464,7 @@ int exynos_drm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	}
 
 	pfn = page_to_pfn(exynos_gem->pages[page_offset]);
-	ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address,
-			__pfn_to_pfn_t(pfn, PFN_DEV));
+	ret = vm_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV));
 
 out:
 	switch (ret) {
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gsc.c b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
index 52a9d269484e..bef57987759d 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gsc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
@@ -1610,7 +1610,7 @@ static int gsc_ippdrv_start(struct device *dev, enum drm_exynos_ipp_cmd cmd)
 		&img_pos[EXYNOS_DRM_OPS_SRC],
 		&img_pos[EXYNOS_DRM_OPS_DST]);
 	if (ret) {
-		dev_err(dev, "failed to set precalser.\n");
+		dev_err(dev, "failed to set prescaler.\n");
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index 38eaa63afb31..5ed8b1effe71 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -47,19 +47,6 @@
 
 #define HOTPLUG_DEBOUNCE_MS		1100
 
-/* AVI header and aspect ratio */
-#define HDMI_AVI_VERSION		0x02
-#define HDMI_AVI_LENGTH			0x0d
-
-/* AUI header info */
-#define HDMI_AUI_VERSION		0x01
-#define HDMI_AUI_LENGTH			0x0a
-
-/* AVI active format aspect ratio */
-#define AVI_SAME_AS_PIC_ASPECT_RATIO	0x08
-#define AVI_4_3_CENTER_RATIO		0x09
-#define AVI_16_9_CENTER_RATIO		0x0a
-
 enum hdmi_type {
 	HDMI_TYPE13,
 	HDMI_TYPE14,
@@ -131,7 +118,6 @@ struct hdmi_context {
 	bool				dvi_mode;
 	struct delayed_work		hotplug_work;
 	struct drm_display_mode		current_mode;
-	u8				cea_video_id;
 	const struct hdmi_driver_data	*drv_data;
 
 	void __iomem			*regs;
@@ -681,6 +667,13 @@ static inline void hdmi_reg_writev(struct hdmi_context *hdata, u32 reg_id,
 	}
 }
 
+static inline void hdmi_reg_write_buf(struct hdmi_context *hdata, u32 reg_id,
+				      u8 *buf, int size)
+{
+	for (reg_id = hdmi_map_reg(hdata, reg_id); size; --size, reg_id += 4)
+		writel(*buf++, hdata->regs + reg_id);
+}
+
 static inline void hdmi_reg_writemask(struct hdmi_context *hdata,
 				 u32 reg_id, u32 value, u32 mask)
 {
@@ -762,93 +755,50 @@ static int hdmi_clk_set_parents(struct hdmi_context *hdata, bool to_phy)
 	return ret;
 }
 
-static u8 hdmi_chksum(struct hdmi_context *hdata,
-			u32 start, u8 len, u32 hdr_sum)
-{
-	int i;
-
-	/* hdr_sum : header0 + header1 + header2
-	* start : start address of packet byte1
-	* len : packet bytes - 1 */
-	for (i = 0; i < len; ++i)
-		hdr_sum += 0xff & hdmi_reg_read(hdata, start + i * 4);
-
-	/* return 2's complement of 8 bit hdr_sum */
-	return (u8)(~(hdr_sum & 0xff) + 1);
-}
-
-static void hdmi_reg_infoframe(struct hdmi_context *hdata,
-			union hdmi_infoframe *infoframe)
+static void hdmi_reg_infoframes(struct hdmi_context *hdata)
 {
-	u32 hdr_sum;
-	u8 chksum;
-	u8 ar;
+	union hdmi_infoframe frm;
+	u8 buf[25];
+	int ret;
 
 	if (hdata->dvi_mode) {
-		hdmi_reg_writeb(hdata, HDMI_VSI_CON,
-				HDMI_VSI_CON_DO_NOT_TRANSMIT);
 		hdmi_reg_writeb(hdata, HDMI_AVI_CON,
 				HDMI_AVI_CON_DO_NOT_TRANSMIT);
+		hdmi_reg_writeb(hdata, HDMI_VSI_CON,
+				HDMI_VSI_CON_DO_NOT_TRANSMIT);
 		hdmi_reg_writeb(hdata, HDMI_AUI_CON, HDMI_AUI_CON_NO_TRAN);
 		return;
 	}
 
-	switch (infoframe->any.type) {
-	case HDMI_INFOFRAME_TYPE_AVI:
+	ret = drm_hdmi_avi_infoframe_from_display_mode(&frm.avi,
+			&hdata->current_mode);
+	if (!ret)
+		ret = hdmi_avi_infoframe_pack(&frm.avi, buf, sizeof(buf));
+	if (ret > 0) {
 		hdmi_reg_writeb(hdata, HDMI_AVI_CON, HDMI_AVI_CON_EVERY_VSYNC);
-		hdmi_reg_writeb(hdata, HDMI_AVI_HEADER0, infoframe->any.type);
-		hdmi_reg_writeb(hdata, HDMI_AVI_HEADER1,
-				infoframe->any.version);
-		hdmi_reg_writeb(hdata, HDMI_AVI_HEADER2, infoframe->any.length);
-		hdr_sum = infoframe->any.type + infoframe->any.version +
-			  infoframe->any.length;
-
-		/* Output format zero hardcoded ,RGB YBCR selection */
-		hdmi_reg_writeb(hdata, HDMI_AVI_BYTE(1), 0 << 5 |
-			AVI_ACTIVE_FORMAT_VALID |
-			AVI_UNDERSCANNED_DISPLAY_VALID);
-
-		/*
-		 * Set the aspect ratio as per the mode, mentioned in
-		 * Table 9 AVI InfoFrame Data Byte 2 of CEA-861-D Standard
-		 */
-		ar = hdata->current_mode.picture_aspect_ratio;
-		switch (ar) {
-		case HDMI_PICTURE_ASPECT_4_3:
-			ar |= AVI_4_3_CENTER_RATIO;
-			break;
-		case HDMI_PICTURE_ASPECT_16_9:
-			ar |= AVI_16_9_CENTER_RATIO;
-			break;
-		case HDMI_PICTURE_ASPECT_NONE:
-		default:
-			ar |= AVI_SAME_AS_PIC_ASPECT_RATIO;
-			break;
-		}
-		hdmi_reg_writeb(hdata, HDMI_AVI_BYTE(2), ar);
+		hdmi_reg_write_buf(hdata, HDMI_AVI_HEADER0, buf, ret);
+	} else {
+		DRM_INFO("%s: invalid AVI infoframe (%d)\n", __func__, ret);
+	}
 
-		hdmi_reg_writeb(hdata, HDMI_AVI_BYTE(4), hdata->cea_video_id);
+	ret = drm_hdmi_vendor_infoframe_from_display_mode(&frm.vendor.hdmi,
+			&hdata->current_mode);
+	if (!ret)
+		ret = hdmi_vendor_infoframe_pack(&frm.vendor.hdmi, buf,
+				sizeof(buf));
+	if (ret > 0) {
+		hdmi_reg_writeb(hdata, HDMI_VSI_CON, HDMI_VSI_CON_EVERY_VSYNC);
+		hdmi_reg_write_buf(hdata, HDMI_VSI_HEADER0, buf, ret);
+	}
 
-		chksum = hdmi_chksum(hdata, HDMI_AVI_BYTE(1),
-					infoframe->any.length, hdr_sum);
-		DRM_DEBUG_KMS("AVI checksum = 0x%x\n", chksum);
-		hdmi_reg_writeb(hdata, HDMI_AVI_CHECK_SUM, chksum);
-		break;
-	case HDMI_INFOFRAME_TYPE_AUDIO:
-		hdmi_reg_writeb(hdata, HDMI_AUI_CON, 0x02);
-		hdmi_reg_writeb(hdata, HDMI_AUI_HEADER0, infoframe->any.type);
-		hdmi_reg_writeb(hdata, HDMI_AUI_HEADER1,
-				infoframe->any.version);
-		hdmi_reg_writeb(hdata, HDMI_AUI_HEADER2, infoframe->any.length);
-		hdr_sum = infoframe->any.type + infoframe->any.version +
-			  infoframe->any.length;
-		chksum = hdmi_chksum(hdata, HDMI_AUI_BYTE(1),
-					infoframe->any.length, hdr_sum);
-		DRM_DEBUG_KMS("AUI checksum = 0x%x\n", chksum);
-		hdmi_reg_writeb(hdata, HDMI_AUI_CHECK_SUM, chksum);
-		break;
-	default:
-		break;
+	ret = hdmi_audio_infoframe_init(&frm.audio);
+	if (!ret) {
+		frm.audio.channels = 2;
+		ret = hdmi_audio_infoframe_pack(&frm.audio, buf, sizeof(buf));
+	}
+	if (ret > 0) {
+		hdmi_reg_writeb(hdata, HDMI_AUI_CON, HDMI_AUI_CON_EVERY_VSYNC);
+		hdmi_reg_write_buf(hdata, HDMI_AUI_HEADER0, buf, ret);
 	}
 }
 
@@ -1127,8 +1077,6 @@ static void hdmi_start(struct hdmi_context *hdata, bool start)
 
 static void hdmi_conf_init(struct hdmi_context *hdata)
 {
-	union hdmi_infoframe infoframe;
-
 	/* disable HPD interrupts from HDMI IP block, use GPIO instead */
 	hdmi_reg_writemask(hdata, HDMI_INTC_CON, 0, HDMI_INTC_EN_GLOBAL |
 		HDMI_INTC_EN_HPD_PLUG | HDMI_INTC_EN_HPD_UNPLUG);
@@ -1164,15 +1112,7 @@ static void hdmi_conf_init(struct hdmi_context *hdata)
 		hdmi_reg_writeb(hdata, HDMI_V13_AUI_CON, 0x02);
 		hdmi_reg_writeb(hdata, HDMI_V13_ACR_CON, 0x04);
 	} else {
-		infoframe.any.type = HDMI_INFOFRAME_TYPE_AVI;
-		infoframe.any.version = HDMI_AVI_VERSION;
-		infoframe.any.length = HDMI_AVI_LENGTH;
-		hdmi_reg_infoframe(hdata, &infoframe);
-
-		infoframe.any.type = HDMI_INFOFRAME_TYPE_AUDIO;
-		infoframe.any.version = HDMI_AUI_VERSION;
-		infoframe.any.length = HDMI_AUI_LENGTH;
-		hdmi_reg_infoframe(hdata, &infoframe);
+		hdmi_reg_infoframes(hdata);
 
 		/* enable AVI packet every vsync, fixes purple line problem */
 		hdmi_reg_writemask(hdata, HDMI_CON_1, 2, 3 << 5);
@@ -1458,7 +1398,6 @@ static void hdmi_mode_set(struct drm_encoder *encoder,
 		"INTERLACED" : "PROGRESSIVE");
 
 	drm_mode_copy(&hdata->current_mode, m);
-	hdata->cea_video_id = drm_match_cea_mode(mode);
 }
 
 static void hdmi_set_refclk(struct hdmi_context *hdata, bool on)
diff --git a/drivers/gpu/drm/exynos/regs-hdmi.h b/drivers/gpu/drm/exynos/regs-hdmi.h
index 169667a22bdc..a0507dc18d9e 100644
--- a/drivers/gpu/drm/exynos/regs-hdmi.h
+++ b/drivers/gpu/drm/exynos/regs-hdmi.h
@@ -361,9 +361,11 @@
 
 /* AUI bit definition */
 #define HDMI_AUI_CON_NO_TRAN		(0 << 0)
+#define HDMI_AUI_CON_EVERY_VSYNC	(1 << 1)
 
 /* VSI bit definition */
 #define HDMI_VSI_CON_DO_NOT_TRANSMIT	(0 << 0)
+#define HDMI_VSI_CON_EVERY_VSYNC	(1 << 1)
 
 /* HDCP related registers */
 #define HDMI_HDCP_SHA1(n)		HDMI_CORE_BASE(0x7000 + 4 * (n))
diff --git a/drivers/gpu/drm/fsl-dcu/Makefile b/drivers/gpu/drm/fsl-dcu/Makefile
index b35a292287f3..aca34f656bea 100644
--- a/drivers/gpu/drm/fsl-dcu/Makefile
+++ b/drivers/gpu/drm/fsl-dcu/Makefile
@@ -3,6 +3,5 @@ fsl-dcu-drm-y := fsl_dcu_drm_drv.o \
 		 fsl_dcu_drm_rgb.o \
 		 fsl_dcu_drm_plane.o \
 		 fsl_dcu_drm_crtc.o \
-		 fsl_dcu_drm_fbdev.o \
 		 fsl_tcon.o
 obj-$(CONFIG_DRM_FSL_DCU)	+= fsl-dcu-drm.o
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
index cc2fde2ae5ef..537ca159ffe5 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
@@ -32,6 +32,9 @@
 #include "fsl_dcu_drm_drv.h"
 #include "fsl_tcon.h"
 
+static int legacyfb_depth = 24;
+module_param(legacyfb_depth, int, 0444);
+
 static bool fsl_dcu_drm_is_volatile_reg(struct device *dev, unsigned int reg)
 {
 	if (reg == DCU_INT_STATUS || reg == DCU_UPDATE_MODE)
@@ -85,7 +88,18 @@ static int fsl_dcu_load(struct drm_device *dev, unsigned long flags)
 		goto done;
 	dev->irq_enabled = true;
 
-	fsl_dcu_fbdev_init(dev);
+	if (legacyfb_depth != 16 && legacyfb_depth != 24 &&
+	    legacyfb_depth != 32) {
+		dev_warn(dev->dev,
+			"Invalid legacyfb_depth.  Defaulting to 24bpp\n");
+		legacyfb_depth = 24;
+	}
+	fsl_dev->fbdev = drm_fbdev_cma_init(dev, legacyfb_depth, 1, 1);
+	if (IS_ERR(fsl_dev->fbdev)) {
+		ret = PTR_ERR(fsl_dev->fbdev);
+		fsl_dev->fbdev = NULL;
+		goto done;
+	}
 
 	return 0;
 done:
@@ -106,6 +120,7 @@ static int fsl_dcu_unload(struct drm_device *dev)
 {
 	struct fsl_dcu_drm_device *fsl_dev = dev->dev_private;
 
+	drm_crtc_force_disable_all(dev);
 	drm_kms_helper_poll_fini(dev);
 
 	if (fsl_dev->fbdev)
@@ -176,9 +191,7 @@ static const struct file_operations fsl_dcu_drm_fops = {
 	.open		= drm_open,
 	.release	= drm_release,
 	.unlocked_ioctl	= drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl	= drm_compat_ioctl,
-#endif
 	.poll		= drm_poll,
 	.read		= drm_read,
 	.llseek		= no_llseek,
@@ -334,11 +347,6 @@ static int fsl_dcu_drm_probe(struct platform_device *pdev)
 	fsl_dev->soc = id->data;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		dev_err(dev, "could not get memory IO resource\n");
-		return -ENODEV;
-	}
-
 	base = devm_ioremap_resource(dev, res);
 	if (IS_ERR(base)) {
 		ret = PTR_ERR(base);
@@ -348,7 +356,7 @@ static int fsl_dcu_drm_probe(struct platform_device *pdev)
 	fsl_dev->irq = platform_get_irq(pdev, 0);
 	if (fsl_dev->irq < 0) {
 		dev_err(dev, "failed to get irq\n");
-		return -ENXIO;
+		return fsl_dev->irq;
 	}
 
 	fsl_dev->regmap = devm_regmap_init_mmio(dev, base,
@@ -426,9 +434,9 @@ static int fsl_dcu_drm_remove(struct platform_device *pdev)
 {
 	struct fsl_dcu_drm_device *fsl_dev = platform_get_drvdata(pdev);
 
+	drm_put_dev(fsl_dev->drm);
 	clk_disable_unprepare(fsl_dev->clk);
 	clk_unregister(fsl_dev->pix_clk);
-	drm_put_dev(fsl_dev->drm);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h
index 3b371fe7491e..e9e9aeecf2eb 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h
@@ -197,7 +197,6 @@ struct fsl_dcu_drm_device {
 	struct drm_atomic_state *state;
 };
 
-void fsl_dcu_fbdev_init(struct drm_device *dev);
 int fsl_dcu_drm_modeset_init(struct fsl_dcu_drm_device *fsl_dev);
 
 #endif /* __FSL_DCU_DRM_DRV_H__ */
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_fbdev.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_fbdev.c
deleted file mode 100644
index 8b8b819ea704..000000000000
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_fbdev.c
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2015 Freescale Semiconductor, Inc.
- *
- * Freescale DCU drm device driver
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-
-#include <drm/drmP.h>
-#include <drm/drm_fb_cma_helper.h>
-
-#include "fsl_dcu_drm_drv.h"
-
-/* initialize fbdev helper */
-void fsl_dcu_fbdev_init(struct drm_device *dev)
-{
-	struct fsl_dcu_drm_device *fsl_dev = dev_get_drvdata(dev->dev);
-
-	fsl_dev->fbdev = drm_fbdev_cma_init(dev, 24, 1, 1);
-}
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
index e1dd75b18118..05a8ee106879 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
@@ -58,17 +58,10 @@ static void fsl_dcu_drm_connector_destroy(struct drm_connector *connector)
 	drm_connector_cleanup(connector);
 }
 
-static enum drm_connector_status
-fsl_dcu_drm_connector_detect(struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static const struct drm_connector_funcs fsl_dcu_drm_connector_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 	.destroy = fsl_dcu_drm_connector_destroy,
-	.detect = fsl_dcu_drm_connector_detect,
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.reset = drm_atomic_helper_connector_reset,
diff --git a/drivers/gpu/drm/gma500/cdv_intel_lvds.c b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
index ea733ab5b1e0..5efdb7fbb7ee 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
@@ -387,19 +387,6 @@ static void cdv_intel_lvds_mode_set(struct drm_encoder *encoder,
 }
 
 /**
- * Detect the LVDS connection.
- *
- * This always returns CONNECTOR_STATUS_CONNECTED.
- * This connector should only have
- * been set up if the LVDS was actually connected anyway.
- */
-static enum drm_connector_status cdv_intel_lvds_detect(
-				struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
-/**
  * Return the list of DDC modes if available, or the BIOS fixed mode otherwise.
  */
 static int cdv_intel_lvds_get_modes(struct drm_connector *connector)
@@ -521,7 +508,6 @@ static const struct drm_connector_helper_funcs
 
 static const struct drm_connector_funcs cdv_intel_lvds_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.detect = cdv_intel_lvds_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = cdv_intel_lvds_set_property,
 	.destroy = cdv_intel_lvds_destroy,
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index 3a44e705db53..8b44fa542562 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -124,8 +124,8 @@ static int psbfb_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	unsigned long phys_addr = (unsigned long)dev_priv->stolen_base +
 				  psbfb->gtt->offset;
 
-	page_num = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
-	address = (unsigned long)vmf->virtual_address - (vmf->pgoff << PAGE_SHIFT);
+	page_num = vma_pages(vma);
+	address = vmf->address - (vmf->pgoff << PAGE_SHIFT);
 
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
@@ -185,9 +185,7 @@ static int psbfb_mmap(struct fb_info *info, struct vm_area_struct *vma)
 
 static struct fb_ops psbfb_ops = {
 	.owner = THIS_MODULE,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
-	.fb_blank = drm_fb_helper_blank,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_setcolreg = psbfb_setcolreg,
 	.fb_fillrect = drm_fb_helper_cfb_fillrect,
 	.fb_copyarea = psbfb_copyarea,
@@ -198,9 +196,7 @@ static struct fb_ops psbfb_ops = {
 
 static struct fb_ops psbfb_roll_ops = {
 	.owner = THIS_MODULE,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
-	.fb_blank = drm_fb_helper_blank,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_setcolreg = psbfb_setcolreg,
 	.fb_fillrect = drm_fb_helper_cfb_fillrect,
 	.fb_copyarea = drm_fb_helper_cfb_copyarea,
@@ -211,9 +207,7 @@ static struct fb_ops psbfb_roll_ops = {
 
 static struct fb_ops psbfb_unaccel_ops = {
 	.owner = THIS_MODULE,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
-	.fb_blank = drm_fb_helper_blank,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_setcolreg = psbfb_setcolreg,
 	.fb_fillrect = drm_fb_helper_cfb_fillrect,
 	.fb_copyarea = drm_fb_helper_cfb_copyarea,
@@ -236,22 +230,20 @@ static int psb_framebuffer_init(struct drm_device *dev,
 					const struct drm_mode_fb_cmd2 *mode_cmd,
 					struct gtt_range *gt)
 {
-	u32 bpp, depth;
+	const struct drm_format_info *info;
 	int ret;
 
-	drm_fb_get_bpp_depth(mode_cmd->pixel_format, &depth, &bpp);
+	/*
+	 * Reject unknown formats, YUV formats, and formats with more than
+	 * 4 bytes per pixel.
+	 */
+	info = drm_format_info(mode_cmd->pixel_format);
+	if (!info || !info->depth || info->cpp[0] > 4)
+		return -EINVAL;
 
 	if (mode_cmd->pitches[0] & 63)
 		return -EINVAL;
-	switch (bpp) {
-	case 8:
-	case 16:
-	case 24:
-	case 32:
-		break;
-	default:
-		return -EINVAL;
-	}
+
 	drm_helper_mode_fill_fb_struct(&fb->base, mode_cmd);
 	fb->gtt = gt;
 	ret = drm_framebuffer_init(dev, &fb->base, &psb_fb_funcs);
@@ -298,7 +290,6 @@ static struct drm_framebuffer *psb_framebuffer_create
  *	psbfb_alloc		-	allocate frame buffer memory
  *	@dev: the DRM device
  *	@aligned_size: space needed
- *	@force: fall back to GEM buffers if need be
  *
  *	Allocate the frame buffer. In the usual case we get a GTT range that
  *	is stolen memory backed and life is simple. If there isn't sufficient
diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c
index 6d1cb6b370b1..527c62917660 100644
--- a/drivers/gpu/drm/gma500/gem.c
+++ b/drivers/gpu/drm/gma500/gem.c
@@ -197,15 +197,14 @@ int psb_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 
 	/* Page relative to the VMA start - we must calculate this ourselves
 	   because vmf->pgoff is the fake GEM offset */
-	page_offset = ((unsigned long) vmf->virtual_address - vma->vm_start)
-				>> PAGE_SHIFT;
+	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
 
 	/* CPU view of the page, don't go via the GART for CPU writes */
 	if (r->stolen)
 		pfn = (dev_priv->stolen_base + r->offset) >> PAGE_SHIFT;
 	else
 		pfn = page_to_pfn(r->pages[page_offset]);
-	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
+	ret = vm_insert_pfn(vma, vmf->address, pfn);
 
 fail:
 	mutex_unlock(&dev_priv->mmap_mutex);
diff --git a/drivers/gpu/drm/gma500/gtt.c b/drivers/gpu/drm/gma500/gtt.c
index 8f69225ce2b4..3f4f424196b2 100644
--- a/drivers/gpu/drm/gma500/gtt.c
+++ b/drivers/gpu/drm/gma500/gtt.c
@@ -76,6 +76,7 @@ static u32 __iomem *psb_gtt_entry(struct drm_device *dev, struct gtt_range *r)
  *	psb_gtt_insert	-	put an object into the GTT
  *	@dev: our DRM device
  *	@r: our GTT range
+ *	@resume: on resume
  *
  *	Take our preallocated GTT range and insert the GEM object into
  *	the GTT. This is protected via the gtt mutex which the caller
@@ -130,7 +131,7 @@ static int psb_gtt_insert(struct drm_device *dev, struct gtt_range *r,
  *	page table entries with the dummy page. This is protected via the gtt
  *	mutex which the caller must hold.
  */
-void psb_gtt_remove(struct drm_device *dev, struct gtt_range *r)
+static void psb_gtt_remove(struct drm_device *dev, struct gtt_range *r)
 {
 	struct drm_psb_private *dev_priv = dev->dev_private;
 	u32 __iomem *gtt_slot;
@@ -321,6 +322,7 @@ out:
  *	@len: length (bytes) of address space required
  *	@name: resource name
  *	@backed: resource should be backed by stolen pages
+ *	@align: requested alignment
  *
  *	Ask the kernel core to find us a suitable range of addresses
  *	to use for a GTT mapping.
diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c
index 50eb944fb78a..ff37ea585664 100644
--- a/drivers/gpu/drm/gma500/psb_drv.c
+++ b/drivers/gpu/drm/gma500/psb_drv.c
@@ -473,6 +473,7 @@ static const struct file_operations psb_gem_fops = {
 	.open = drm_open,
 	.release = drm_release,
 	.unlocked_ioctl = psb_unlocked_ioctl,
+	.compat_ioctl = drm_compat_ioctl,
 	.mmap = drm_gem_mmap,
 	.poll = drm_poll,
 	.read = drm_read,
diff --git a/drivers/gpu/drm/gma500/psb_drv.h b/drivers/gpu/drm/gma500/psb_drv.h
index b74372760d7f..05d7aaf47eea 100644
--- a/drivers/gpu/drm/gma500/psb_drv.h
+++ b/drivers/gpu/drm/gma500/psb_drv.h
@@ -753,10 +753,6 @@ extern int psb_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
 extern int psb_gem_dumb_map_gtt(struct drm_file *file, struct drm_device *dev,
 			uint32_t handle, uint64_t *offset);
 extern int psb_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
-extern int psb_gem_create_ioctl(struct drm_device *dev, void *data,
-			struct drm_file *file);
-extern int psb_gem_mmap_ioctl(struct drm_device *dev, void *data,
-					struct drm_file *file);
 
 /* psb_device.c */
 extern const struct psb_ops psb_chip_ops;
diff --git a/drivers/gpu/drm/gma500/psb_intel_lvds.c b/drivers/gpu/drm/gma500/psb_intel_lvds.c
index fd7c91254841..483fdce74e39 100644
--- a/drivers/gpu/drm/gma500/psb_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/psb_intel_lvds.c
@@ -500,19 +500,6 @@ static void psb_intel_lvds_mode_set(struct drm_encoder *encoder,
 }
 
 /*
- * Detect the LVDS connection.
- *
- * This always returns CONNECTOR_STATUS_CONNECTED.
- * This connector should only have
- * been set up if the LVDS was actually connected anyway.
- */
-static enum drm_connector_status psb_intel_lvds_detect(struct drm_connector
-						   *connector, bool force)
-{
-	return connector_status_connected;
-}
-
-/*
  * Return the list of DDC modes if available, or the BIOS fixed mode otherwise.
  */
 static int psb_intel_lvds_get_modes(struct drm_connector *connector)
@@ -643,7 +630,6 @@ const struct drm_connector_helper_funcs
 
 const struct drm_connector_funcs psb_intel_lvds_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.detect = psb_intel_lvds_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = psb_intel_lvds_set_property,
 	.destroy = psb_intel_lvds_destroy,
diff --git a/drivers/gpu/drm/hisilicon/Kconfig b/drivers/gpu/drm/hisilicon/Kconfig
index 558c61b1b8e8..2fd2724b7a7d 100644
--- a/drivers/gpu/drm/hisilicon/Kconfig
+++ b/drivers/gpu/drm/hisilicon/Kconfig
@@ -2,4 +2,5 @@
 # hisilicon drm device configuration.
 # Please keep this list sorted alphabetically
 
+source "drivers/gpu/drm/hisilicon/hibmc/Kconfig"
 source "drivers/gpu/drm/hisilicon/kirin/Kconfig"
diff --git a/drivers/gpu/drm/hisilicon/Makefile b/drivers/gpu/drm/hisilicon/Makefile
index e3f6d493c996..c8155bfb1ff1 100644
--- a/drivers/gpu/drm/hisilicon/Makefile
+++ b/drivers/gpu/drm/hisilicon/Makefile
@@ -2,4 +2,5 @@
 # Makefile for hisilicon drm drivers.
 # Please keep this list sorted alphabetically
 
+obj-$(CONFIG_DRM_HISI_HIBMC) += hibmc/
 obj-$(CONFIG_DRM_HISI_KIRIN) += kirin/
diff --git a/drivers/gpu/drm/hisilicon/hibmc/Kconfig b/drivers/gpu/drm/hisilicon/hibmc/Kconfig
new file mode 100644
index 000000000000..380622a0da35
--- /dev/null
+++ b/drivers/gpu/drm/hisilicon/hibmc/Kconfig
@@ -0,0 +1,9 @@
+config DRM_HISI_HIBMC
+	tristate "DRM Support for Hisilicon Hibmc"
+	depends on DRM && PCI
+	select DRM_KMS_HELPER
+	select DRM_TTM
+
+	help
+	  Choose this option if you have a Hisilicon Hibmc soc chipset.
+	  If M is selected the module will be called hibmc-drm.
diff --git a/drivers/gpu/drm/hisilicon/hibmc/Makefile b/drivers/gpu/drm/hisilicon/hibmc/Makefile
new file mode 100644
index 000000000000..f2e04c035673
--- /dev/null
+++ b/drivers/gpu/drm/hisilicon/hibmc/Makefile
@@ -0,0 +1,4 @@
+ccflags-y := -Iinclude/drm
+hibmc-drm-y := hibmc_drm_drv.o hibmc_drm_de.o hibmc_drm_vdac.o hibmc_drm_fbdev.o hibmc_ttm.o
+
+obj-$(CONFIG_DRM_HISI_HIBMC) += hibmc-drm.o
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
new file mode 100644
index 000000000000..2a1386e33126
--- /dev/null
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
@@ -0,0 +1,477 @@
+/* Hisilicon Hibmc SoC drm driver
+ *
+ * Based on the bochs drm driver.
+ *
+ * Copyright (c) 2016 Huawei Limited.
+ *
+ * Author:
+ *	Rongrong Zou <zourongrong@huawei.com>
+ *	Rongrong Zou <zourongrong@gmail.com>
+ *	Jianhua Li <lijianhua@huawei.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_plane_helper.h>
+
+#include "hibmc_drm_drv.h"
+#include "hibmc_drm_regs.h"
+
+struct hibmc_display_panel_pll {
+	unsigned long M;
+	unsigned long N;
+	unsigned long OD;
+	unsigned long POD;
+};
+
+struct hibmc_dislay_pll_config {
+	unsigned long hdisplay;
+	unsigned long vdisplay;
+	u32 pll1_config_value;
+	u32 pll2_config_value;
+};
+
+static const struct hibmc_dislay_pll_config hibmc_pll_table[] = {
+	{800, 600, CRT_PLL1_HS_40MHZ, CRT_PLL2_HS_40MHZ},
+	{1024, 768, CRT_PLL1_HS_65MHZ, CRT_PLL2_HS_65MHZ},
+	{1152, 864, CRT_PLL1_HS_80MHZ_1152, CRT_PLL2_HS_80MHZ},
+	{1280, 768, CRT_PLL1_HS_80MHZ, CRT_PLL2_HS_80MHZ},
+	{1280, 720, CRT_PLL1_HS_74MHZ, CRT_PLL2_HS_74MHZ},
+	{1280, 960, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ},
+	{1280, 1024, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ},
+	{1600, 1200, CRT_PLL1_HS_162MHZ, CRT_PLL2_HS_162MHZ},
+	{1920, 1080, CRT_PLL1_HS_148MHZ, CRT_PLL2_HS_148MHZ},
+	{1920, 1200, CRT_PLL1_HS_193MHZ, CRT_PLL2_HS_193MHZ},
+};
+
+#define PADDING(align, data) (((data) + (align) - 1) & (~((align) - 1)))
+
+static int hibmc_plane_atomic_check(struct drm_plane *plane,
+				    struct drm_plane_state *state)
+{
+	struct drm_framebuffer *fb = state->fb;
+	struct drm_crtc *crtc = state->crtc;
+	struct drm_crtc_state *crtc_state;
+	u32 src_w = state->src_w >> 16;
+	u32 src_h = state->src_h >> 16;
+
+	if (!crtc || !fb)
+		return 0;
+
+	crtc_state = drm_atomic_get_crtc_state(state->state, crtc);
+	if (IS_ERR(crtc_state))
+		return PTR_ERR(crtc_state);
+
+	if (src_w != state->crtc_w || src_h != state->crtc_h) {
+		DRM_DEBUG_ATOMIC("scale not support\n");
+		return -EINVAL;
+	}
+
+	if (state->crtc_x < 0 || state->crtc_y < 0) {
+		DRM_DEBUG_ATOMIC("crtc_x/y of drm_plane state is invalid\n");
+		return -EINVAL;
+	}
+
+	if (state->crtc_x + state->crtc_w >
+	    crtc_state->adjusted_mode.hdisplay ||
+	    state->crtc_y + state->crtc_h >
+	    crtc_state->adjusted_mode.vdisplay) {
+		DRM_DEBUG_ATOMIC("visible portion of plane is invalid\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void hibmc_plane_atomic_update(struct drm_plane *plane,
+				      struct drm_plane_state *old_state)
+{
+	struct drm_plane_state	*state	= plane->state;
+	u32 reg;
+	int ret;
+	u64 gpu_addr = 0;
+	unsigned int line_l;
+	struct hibmc_drm_private *priv = plane->dev->dev_private;
+	struct hibmc_framebuffer *hibmc_fb;
+	struct hibmc_bo *bo;
+
+	if (!state->fb)
+		return;
+
+	hibmc_fb = to_hibmc_framebuffer(state->fb);
+	bo = gem_to_hibmc_bo(hibmc_fb->obj);
+	ret = ttm_bo_reserve(&bo->bo, true, false, NULL);
+	if (ret) {
+		DRM_ERROR("failed to reserve ttm_bo: %d", ret);
+		return;
+	}
+
+	ret = hibmc_bo_pin(bo, TTM_PL_FLAG_VRAM, &gpu_addr);
+	ttm_bo_unreserve(&bo->bo);
+	if (ret) {
+		DRM_ERROR("failed to pin hibmc_bo: %d", ret);
+		return;
+	}
+
+	writel(gpu_addr, priv->mmio + HIBMC_CRT_FB_ADDRESS);
+
+	reg = state->fb->width * (state->fb->bits_per_pixel / 8);
+	/* now line_pad is 16 */
+	reg = PADDING(16, reg);
+
+	line_l = state->fb->width * state->fb->bits_per_pixel / 8;
+	line_l = PADDING(16, line_l);
+	writel(HIBMC_FIELD(HIBMC_CRT_FB_WIDTH_WIDTH, reg) |
+	       HIBMC_FIELD(HIBMC_CRT_FB_WIDTH_OFFS, line_l),
+	       priv->mmio + HIBMC_CRT_FB_WIDTH);
+
+	/* SET PIXEL FORMAT */
+	reg = readl(priv->mmio + HIBMC_CRT_DISP_CTL);
+	reg &= ~HIBMC_CRT_DISP_CTL_FORMAT_MASK;
+	reg |= HIBMC_FIELD(HIBMC_CRT_DISP_CTL_FORMAT,
+			   state->fb->bits_per_pixel / 16);
+	writel(reg, priv->mmio + HIBMC_CRT_DISP_CTL);
+}
+
+static const u32 channel_formats1[] = {
+	DRM_FORMAT_RGB565, DRM_FORMAT_BGR565, DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888, DRM_FORMAT_XRGB8888, DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGBA8888, DRM_FORMAT_BGRA8888, DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888
+};
+
+static struct drm_plane_funcs hibmc_plane_funcs = {
+	.update_plane	= drm_atomic_helper_update_plane,
+	.disable_plane	= drm_atomic_helper_disable_plane,
+	.set_property = drm_atomic_helper_plane_set_property,
+	.destroy = drm_plane_cleanup,
+	.reset = drm_atomic_helper_plane_reset,
+	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+};
+
+static const struct drm_plane_helper_funcs hibmc_plane_helper_funcs = {
+	.atomic_check = hibmc_plane_atomic_check,
+	.atomic_update = hibmc_plane_atomic_update,
+};
+
+static struct drm_plane *hibmc_plane_init(struct hibmc_drm_private *priv)
+{
+	struct drm_device *dev = priv->dev;
+	struct drm_plane *plane;
+	int ret = 0;
+
+	plane = devm_kzalloc(dev->dev, sizeof(*plane), GFP_KERNEL);
+	if (!plane) {
+		DRM_ERROR("failed to alloc memory when init plane\n");
+		return ERR_PTR(-ENOMEM);
+	}
+	/*
+	 * plane init
+	 * TODO: Now only support primary plane, overlay planes
+	 * need to do.
+	 */
+	ret = drm_universal_plane_init(dev, plane, 1, &hibmc_plane_funcs,
+				       channel_formats1,
+				       ARRAY_SIZE(channel_formats1),
+				       DRM_PLANE_TYPE_PRIMARY,
+				       NULL);
+	if (ret) {
+		DRM_ERROR("failed to init plane: %d\n", ret);
+		return ERR_PTR(ret);
+	}
+
+	drm_plane_helper_add(plane, &hibmc_plane_helper_funcs);
+	return plane;
+}
+
+static void hibmc_crtc_enable(struct drm_crtc *crtc)
+{
+	unsigned int reg;
+	struct hibmc_drm_private *priv = crtc->dev->dev_private;
+
+	hibmc_set_power_mode(priv, HIBMC_PW_MODE_CTL_MODE_MODE0);
+
+	/* Enable display power gate & LOCALMEM power gate*/
+	reg = readl(priv->mmio + HIBMC_CURRENT_GATE);
+	reg &= ~HIBMC_CURR_GATE_LOCALMEM_MASK;
+	reg &= ~HIBMC_CURR_GATE_DISPLAY_MASK;
+	reg |= HIBMC_CURR_GATE_LOCALMEM(1);
+	reg |= HIBMC_CURR_GATE_DISPLAY(1);
+	hibmc_set_current_gate(priv, reg);
+	drm_crtc_vblank_on(crtc);
+}
+
+static void hibmc_crtc_disable(struct drm_crtc *crtc)
+{
+	unsigned int reg;
+	struct hibmc_drm_private *priv = crtc->dev->dev_private;
+
+	drm_crtc_vblank_off(crtc);
+
+	hibmc_set_power_mode(priv, HIBMC_PW_MODE_CTL_MODE_SLEEP);
+
+	/* Enable display power gate & LOCALMEM power gate*/
+	reg = readl(priv->mmio + HIBMC_CURRENT_GATE);
+	reg &= ~HIBMC_CURR_GATE_LOCALMEM_MASK;
+	reg &= ~HIBMC_CURR_GATE_DISPLAY_MASK;
+	reg |= HIBMC_CURR_GATE_LOCALMEM(0);
+	reg |= HIBMC_CURR_GATE_DISPLAY(0);
+	hibmc_set_current_gate(priv, reg);
+}
+
+static unsigned int format_pll_reg(void)
+{
+	unsigned int pllreg = 0;
+	struct hibmc_display_panel_pll pll = {0};
+
+	/*
+	 * Note that all PLL's have the same format. Here,
+	 * we just use Panel PLL parameter to work out the bit
+	 * fields in the register.On returning a 32 bit number, the value can
+	 * be applied to any PLL in the calling function.
+	 */
+	pllreg |= HIBMC_FIELD(HIBMC_PLL_CTRL_BYPASS, 0);
+	pllreg |= HIBMC_FIELD(HIBMC_PLL_CTRL_POWER, 1);
+	pllreg |= HIBMC_FIELD(HIBMC_PLL_CTRL_INPUT, 0);
+	pllreg |= HIBMC_FIELD(HIBMC_PLL_CTRL_POD, pll.POD);
+	pllreg |= HIBMC_FIELD(HIBMC_PLL_CTRL_OD, pll.OD);
+	pllreg |= HIBMC_FIELD(HIBMC_PLL_CTRL_N, pll.N);
+	pllreg |= HIBMC_FIELD(HIBMC_PLL_CTRL_M, pll.M);
+
+	return pllreg;
+}
+
+static void set_vclock_hisilicon(struct drm_device *dev, unsigned long pll)
+{
+	u32 val;
+	struct hibmc_drm_private *priv = dev->dev_private;
+
+	val = readl(priv->mmio + CRT_PLL1_HS);
+	val &= ~(CRT_PLL1_HS_OUTER_BYPASS(1));
+	writel(val, priv->mmio + CRT_PLL1_HS);
+
+	val = CRT_PLL1_HS_INTER_BYPASS(1) | CRT_PLL1_HS_POWERON(1);
+	writel(val, priv->mmio + CRT_PLL1_HS);
+
+	writel(pll, priv->mmio + CRT_PLL1_HS);
+
+	usleep_range(1000, 2000);
+
+	val = pll & ~(CRT_PLL1_HS_POWERON(1));
+	writel(val, priv->mmio + CRT_PLL1_HS);
+
+	usleep_range(1000, 2000);
+
+	val &= ~(CRT_PLL1_HS_INTER_BYPASS(1));
+	writel(val, priv->mmio + CRT_PLL1_HS);
+
+	usleep_range(1000, 2000);
+
+	val |= CRT_PLL1_HS_OUTER_BYPASS(1);
+	writel(val, priv->mmio + CRT_PLL1_HS);
+}
+
+static void get_pll_config(unsigned long x, unsigned long y,
+			   u32 *pll1, u32 *pll2)
+{
+	int i;
+	int count = ARRAY_SIZE(hibmc_pll_table);
+
+	for (i = 0; i < count; i++) {
+		if (hibmc_pll_table[i].hdisplay == x &&
+		    hibmc_pll_table[i].vdisplay == y) {
+			*pll1 = hibmc_pll_table[i].pll1_config_value;
+			*pll2 = hibmc_pll_table[i].pll2_config_value;
+			return;
+		}
+	}
+
+	/* if found none, we use default value */
+	*pll1 = CRT_PLL1_HS_25MHZ;
+	*pll2 = CRT_PLL2_HS_25MHZ;
+}
+
+/*
+ * This function takes care the extra registers and bit fields required to
+ * setup a mode in board.
+ * Explanation about Display Control register:
+ * FPGA only supports 7 predefined pixel clocks, and clock select is
+ * in bit 4:0 of new register 0x802a8.
+ */
+static unsigned int display_ctrl_adjust(struct drm_device *dev,
+					struct drm_display_mode *mode,
+					unsigned int ctrl)
+{
+	unsigned long x, y;
+	u32 pll1; /* bit[31:0] of PLL */
+	u32 pll2; /* bit[63:32] of PLL */
+	struct hibmc_drm_private *priv = dev->dev_private;
+
+	x = mode->hdisplay;
+	y = mode->vdisplay;
+
+	get_pll_config(x, y, &pll1, &pll2);
+	writel(pll2, priv->mmio + CRT_PLL2_HS);
+	set_vclock_hisilicon(dev, pll1);
+
+	/*
+	 * Hisilicon has to set up the top-left and bottom-right
+	 * registers as well.
+	 * Note that normal chip only use those two register for
+	 * auto-centering mode.
+	 */
+	writel(HIBMC_FIELD(HIBMC_CRT_AUTO_CENTERING_TL_TOP, 0) |
+	       HIBMC_FIELD(HIBMC_CRT_AUTO_CENTERING_TL_LEFT, 0),
+	       priv->mmio + HIBMC_CRT_AUTO_CENTERING_TL);
+
+	writel(HIBMC_FIELD(HIBMC_CRT_AUTO_CENTERING_BR_BOTTOM, y - 1) |
+	       HIBMC_FIELD(HIBMC_CRT_AUTO_CENTERING_BR_RIGHT, x - 1),
+	       priv->mmio + HIBMC_CRT_AUTO_CENTERING_BR);
+
+	/*
+	 * Assume common fields in ctrl have been properly set before
+	 * calling this function.
+	 * This function only sets the extra fields in ctrl.
+	 */
+
+	/* Set bit 25 of display controller: Select CRT or VGA clock */
+	ctrl &= ~HIBMC_CRT_DISP_CTL_CRTSELECT_MASK;
+	ctrl &= ~HIBMC_CRT_DISP_CTL_CLOCK_PHASE_MASK;
+
+	ctrl |= HIBMC_CRT_DISP_CTL_CRTSELECT(HIBMC_CRTSELECT_CRT);
+
+	/* clock_phase_polarity is 0 */
+	ctrl |= HIBMC_CRT_DISP_CTL_CLOCK_PHASE(0);
+
+	writel(ctrl, priv->mmio + HIBMC_CRT_DISP_CTL);
+
+	return ctrl;
+}
+
+static void hibmc_crtc_mode_set_nofb(struct drm_crtc *crtc)
+{
+	unsigned int val;
+	struct drm_display_mode *mode = &crtc->state->mode;
+	struct drm_device *dev = crtc->dev;
+	struct hibmc_drm_private *priv = dev->dev_private;
+	int width = mode->hsync_end - mode->hsync_start;
+	int height = mode->vsync_end - mode->vsync_start;
+
+	writel(format_pll_reg(), priv->mmio + HIBMC_CRT_PLL_CTRL);
+	writel(HIBMC_FIELD(HIBMC_CRT_HORZ_TOTAL_TOTAL, mode->htotal - 1) |
+	       HIBMC_FIELD(HIBMC_CRT_HORZ_TOTAL_DISP_END, mode->hdisplay - 1),
+	       priv->mmio + HIBMC_CRT_HORZ_TOTAL);
+
+	writel(HIBMC_FIELD(HIBMC_CRT_HORZ_SYNC_WIDTH, width) |
+	       HIBMC_FIELD(HIBMC_CRT_HORZ_SYNC_START, mode->hsync_start - 1),
+	       priv->mmio + HIBMC_CRT_HORZ_SYNC);
+
+	writel(HIBMC_FIELD(HIBMC_CRT_VERT_TOTAL_TOTAL, mode->vtotal - 1) |
+	       HIBMC_FIELD(HIBMC_CRT_VERT_TOTAL_DISP_END, mode->vdisplay - 1),
+	       priv->mmio + HIBMC_CRT_VERT_TOTAL);
+
+	writel(HIBMC_FIELD(HIBMC_CRT_VERT_SYNC_HEIGHT, height) |
+	       HIBMC_FIELD(HIBMC_CRT_VERT_SYNC_START, mode->vsync_start - 1),
+	       priv->mmio + HIBMC_CRT_VERT_SYNC);
+
+	val = HIBMC_FIELD(HIBMC_CRT_DISP_CTL_VSYNC_PHASE, 0);
+	val |= HIBMC_FIELD(HIBMC_CRT_DISP_CTL_HSYNC_PHASE, 0);
+	val |= HIBMC_CRT_DISP_CTL_TIMING(1);
+	val |= HIBMC_CRT_DISP_CTL_PLANE(1);
+
+	display_ctrl_adjust(dev, mode, val);
+}
+
+static void hibmc_crtc_atomic_begin(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_state)
+{
+	unsigned int reg;
+	struct drm_device *dev = crtc->dev;
+	struct hibmc_drm_private *priv = dev->dev_private;
+
+	hibmc_set_power_mode(priv, HIBMC_PW_MODE_CTL_MODE_MODE0);
+
+	/* Enable display power gate & LOCALMEM power gate*/
+	reg = readl(priv->mmio + HIBMC_CURRENT_GATE);
+	reg &= ~HIBMC_CURR_GATE_DISPLAY_MASK;
+	reg &= ~HIBMC_CURR_GATE_LOCALMEM_MASK;
+	reg |= HIBMC_CURR_GATE_DISPLAY(1);
+	reg |= HIBMC_CURR_GATE_LOCALMEM(1);
+	hibmc_set_current_gate(priv, reg);
+
+	/* We can add more initialization as needed. */
+}
+
+static void hibmc_crtc_atomic_flush(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_state)
+
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&crtc->dev->event_lock, flags);
+	if (crtc->state->event)
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+	crtc->state->event = NULL;
+	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+}
+
+static const struct drm_crtc_funcs hibmc_crtc_funcs = {
+	.page_flip = drm_atomic_helper_page_flip,
+	.set_config = drm_atomic_helper_set_config,
+	.destroy = drm_crtc_cleanup,
+	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_duplicate_state =  drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+};
+
+static const struct drm_crtc_helper_funcs hibmc_crtc_helper_funcs = {
+	.enable		= hibmc_crtc_enable,
+	.disable	= hibmc_crtc_disable,
+	.mode_set_nofb	= hibmc_crtc_mode_set_nofb,
+	.atomic_begin	= hibmc_crtc_atomic_begin,
+	.atomic_flush	= hibmc_crtc_atomic_flush,
+};
+
+int hibmc_de_init(struct hibmc_drm_private *priv)
+{
+	struct drm_device *dev = priv->dev;
+	struct drm_crtc *crtc;
+	struct drm_plane *plane;
+	int ret;
+
+	plane = hibmc_plane_init(priv);
+	if (IS_ERR(plane)) {
+		DRM_ERROR("failed to create plane: %ld\n", PTR_ERR(plane));
+		return PTR_ERR(plane);
+	}
+
+	crtc = devm_kzalloc(dev->dev, sizeof(*crtc), GFP_KERNEL);
+	if (!crtc) {
+		DRM_ERROR("failed to alloc memory when init crtc\n");
+		return -ENOMEM;
+	}
+
+	ret = drm_crtc_init_with_planes(dev, crtc, plane,
+					NULL, &hibmc_crtc_funcs, NULL);
+	if (ret) {
+		DRM_ERROR("failed to init crtc: %d\n", ret);
+		return ret;
+	}
+
+	ret = drm_mode_crtc_set_gamma_size(crtc, 256);
+	if (ret) {
+		DRM_ERROR("failed to set gamma size: %d\n", ret);
+		return ret;
+	}
+	drm_crtc_helper_add(crtc, &hibmc_crtc_helper_funcs);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
new file mode 100644
index 000000000000..7e2043f4348c
--- /dev/null
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
@@ -0,0 +1,456 @@
+/* Hisilicon Hibmc SoC drm driver
+ *
+ * Based on the bochs drm driver.
+ *
+ * Copyright (c) 2016 Huawei Limited.
+ *
+ * Author:
+ *	Rongrong Zou <zourongrong@huawei.com>
+ *	Rongrong Zou <zourongrong@gmail.com>
+ *	Jianhua Li <lijianhua@huawei.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#include <linux/console.h>
+#include <linux/module.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+
+#include "hibmc_drm_drv.h"
+#include "hibmc_drm_regs.h"
+
+static const struct file_operations hibmc_fops = {
+	.owner		= THIS_MODULE,
+	.open		= drm_open,
+	.release	= drm_release,
+	.unlocked_ioctl	= drm_ioctl,
+	.compat_ioctl	= drm_compat_ioctl,
+	.mmap		= hibmc_mmap,
+	.poll		= drm_poll,
+	.read		= drm_read,
+	.llseek		= no_llseek,
+};
+
+static int hibmc_enable_vblank(struct drm_device *dev, unsigned int pipe)
+{
+	struct hibmc_drm_private *priv =
+		(struct hibmc_drm_private *)dev->dev_private;
+
+	writel(HIBMC_RAW_INTERRUPT_EN_VBLANK(1),
+	       priv->mmio + HIBMC_RAW_INTERRUPT_EN);
+
+	return 0;
+}
+
+static void hibmc_disable_vblank(struct drm_device *dev, unsigned int pipe)
+{
+	struct hibmc_drm_private *priv =
+		(struct hibmc_drm_private *)dev->dev_private;
+
+	writel(HIBMC_RAW_INTERRUPT_EN_VBLANK(0),
+	       priv->mmio + HIBMC_RAW_INTERRUPT_EN);
+}
+
+irqreturn_t hibmc_drm_interrupt(int irq, void *arg)
+{
+	struct drm_device *dev = (struct drm_device *)arg;
+	struct hibmc_drm_private *priv =
+		(struct hibmc_drm_private *)dev->dev_private;
+	u32 status;
+
+	status = readl(priv->mmio + HIBMC_RAW_INTERRUPT);
+
+	if (status & HIBMC_RAW_INTERRUPT_VBLANK(1)) {
+		writel(HIBMC_RAW_INTERRUPT_VBLANK(1),
+		       priv->mmio + HIBMC_RAW_INTERRUPT);
+		drm_handle_vblank(dev, 0);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static struct drm_driver hibmc_driver = {
+	.driver_features	= DRIVER_GEM | DRIVER_MODESET |
+				  DRIVER_ATOMIC | DRIVER_HAVE_IRQ,
+	.fops			= &hibmc_fops,
+	.name			= "hibmc",
+	.date			= "20160828",
+	.desc			= "hibmc drm driver",
+	.major			= 1,
+	.minor			= 0,
+	.get_vblank_counter	= drm_vblank_no_hw_counter,
+	.enable_vblank		= hibmc_enable_vblank,
+	.disable_vblank		= hibmc_disable_vblank,
+	.gem_free_object_unlocked = hibmc_gem_free_object,
+	.dumb_create            = hibmc_dumb_create,
+	.dumb_map_offset        = hibmc_dumb_mmap_offset,
+	.dumb_destroy           = drm_gem_dumb_destroy,
+	.irq_handler		= hibmc_drm_interrupt,
+};
+
+static int __maybe_unused hibmc_pm_suspend(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct drm_device *drm_dev = pci_get_drvdata(pdev);
+	struct hibmc_drm_private *priv = drm_dev->dev_private;
+
+	drm_kms_helper_poll_disable(drm_dev);
+	priv->suspend_state = drm_atomic_helper_suspend(drm_dev);
+	if (IS_ERR(priv->suspend_state)) {
+		DRM_ERROR("drm_atomic_helper_suspend failed: %ld\n",
+			  PTR_ERR(priv->suspend_state));
+		drm_kms_helper_poll_enable(drm_dev);
+		return PTR_ERR(priv->suspend_state);
+	}
+
+	return 0;
+}
+
+static int  __maybe_unused hibmc_pm_resume(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct drm_device *drm_dev = pci_get_drvdata(pdev);
+	struct hibmc_drm_private *priv = drm_dev->dev_private;
+
+	drm_atomic_helper_resume(drm_dev, priv->suspend_state);
+	drm_kms_helper_poll_enable(drm_dev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops hibmc_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(hibmc_pm_suspend,
+				hibmc_pm_resume)
+};
+
+static int hibmc_kms_init(struct hibmc_drm_private *priv)
+{
+	int ret;
+
+	drm_mode_config_init(priv->dev);
+	priv->mode_config_initialized = true;
+
+	priv->dev->mode_config.min_width = 0;
+	priv->dev->mode_config.min_height = 0;
+	priv->dev->mode_config.max_width = 1920;
+	priv->dev->mode_config.max_height = 1440;
+
+	priv->dev->mode_config.fb_base = priv->fb_base;
+	priv->dev->mode_config.preferred_depth = 24;
+	priv->dev->mode_config.prefer_shadow = 0;
+
+	priv->dev->mode_config.funcs = (void *)&hibmc_mode_funcs;
+
+	ret = hibmc_de_init(priv);
+	if (ret) {
+		DRM_ERROR("failed to init de: %d\n", ret);
+		return ret;
+	}
+
+	ret = hibmc_vdac_init(priv);
+	if (ret) {
+		DRM_ERROR("failed to init vdac: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void hibmc_kms_fini(struct hibmc_drm_private *priv)
+{
+	if (priv->mode_config_initialized) {
+		drm_mode_config_cleanup(priv->dev);
+		priv->mode_config_initialized = false;
+	}
+}
+
+/*
+ * It can operate in one of three modes: 0, 1 or Sleep.
+ */
+void hibmc_set_power_mode(struct hibmc_drm_private *priv,
+			  unsigned int power_mode)
+{
+	unsigned int control_value = 0;
+	void __iomem   *mmio = priv->mmio;
+	unsigned int input = 1;
+
+	if (power_mode > HIBMC_PW_MODE_CTL_MODE_SLEEP)
+		return;
+
+	if (power_mode == HIBMC_PW_MODE_CTL_MODE_SLEEP)
+		input = 0;
+
+	control_value = readl(mmio + HIBMC_POWER_MODE_CTRL);
+	control_value &= ~(HIBMC_PW_MODE_CTL_MODE_MASK |
+			   HIBMC_PW_MODE_CTL_OSC_INPUT_MASK);
+	control_value |= HIBMC_FIELD(HIBMC_PW_MODE_CTL_MODE, power_mode);
+	control_value |= HIBMC_FIELD(HIBMC_PW_MODE_CTL_OSC_INPUT, input);
+	writel(control_value, mmio + HIBMC_POWER_MODE_CTRL);
+}
+
+void hibmc_set_current_gate(struct hibmc_drm_private *priv, unsigned int gate)
+{
+	unsigned int gate_reg;
+	unsigned int mode;
+	void __iomem   *mmio = priv->mmio;
+
+	/* Get current power mode. */
+	mode = (readl(mmio + HIBMC_POWER_MODE_CTRL) &
+		HIBMC_PW_MODE_CTL_MODE_MASK) >> HIBMC_PW_MODE_CTL_MODE_SHIFT;
+
+	switch (mode) {
+	case HIBMC_PW_MODE_CTL_MODE_MODE0:
+		gate_reg = HIBMC_MODE0_GATE;
+		break;
+
+	case HIBMC_PW_MODE_CTL_MODE_MODE1:
+		gate_reg = HIBMC_MODE1_GATE;
+		break;
+
+	default:
+		gate_reg = HIBMC_MODE0_GATE;
+		break;
+	}
+	writel(gate, mmio + gate_reg);
+}
+
+static void hibmc_hw_config(struct hibmc_drm_private *priv)
+{
+	unsigned int reg;
+
+	/* On hardware reset, power mode 0 is default. */
+	hibmc_set_power_mode(priv, HIBMC_PW_MODE_CTL_MODE_MODE0);
+
+	/* Enable display power gate & LOCALMEM power gate*/
+	reg = readl(priv->mmio + HIBMC_CURRENT_GATE);
+	reg &= ~HIBMC_CURR_GATE_DISPLAY_MASK;
+	reg &= ~HIBMC_CURR_GATE_LOCALMEM_MASK;
+	reg |= HIBMC_CURR_GATE_DISPLAY(1);
+	reg |= HIBMC_CURR_GATE_LOCALMEM(1);
+
+	hibmc_set_current_gate(priv, reg);
+
+	/*
+	 * Reset the memory controller. If the memory controller
+	 * is not reset in chip,the system might hang when sw accesses
+	 * the memory.The memory should be resetted after
+	 * changing the MXCLK.
+	 */
+	reg = readl(priv->mmio + HIBMC_MISC_CTRL);
+	reg &= ~HIBMC_MSCCTL_LOCALMEM_RESET_MASK;
+	reg |= HIBMC_MSCCTL_LOCALMEM_RESET(0);
+	writel(reg, priv->mmio + HIBMC_MISC_CTRL);
+
+	reg &= ~HIBMC_MSCCTL_LOCALMEM_RESET_MASK;
+	reg |= HIBMC_MSCCTL_LOCALMEM_RESET(1);
+
+	writel(reg, priv->mmio + HIBMC_MISC_CTRL);
+}
+
+static int hibmc_hw_map(struct hibmc_drm_private *priv)
+{
+	struct drm_device *dev = priv->dev;
+	struct pci_dev *pdev = dev->pdev;
+	resource_size_t addr, size, ioaddr, iosize;
+
+	ioaddr = pci_resource_start(pdev, 1);
+	iosize = pci_resource_len(pdev, 1);
+	priv->mmio = devm_ioremap_nocache(dev->dev, ioaddr, iosize);
+	if (!priv->mmio) {
+		DRM_ERROR("Cannot map mmio region\n");
+		return -ENOMEM;
+	}
+
+	addr = pci_resource_start(pdev, 0);
+	size = pci_resource_len(pdev, 0);
+	priv->fb_map = devm_ioremap(dev->dev, addr, size);
+	if (!priv->fb_map) {
+		DRM_ERROR("Cannot map framebuffer\n");
+		return -ENOMEM;
+	}
+	priv->fb_base = addr;
+	priv->fb_size = size;
+
+	return 0;
+}
+
+static int hibmc_hw_init(struct hibmc_drm_private *priv)
+{
+	int ret;
+
+	ret = hibmc_hw_map(priv);
+	if (ret)
+		return ret;
+
+	hibmc_hw_config(priv);
+
+	return 0;
+}
+
+static int hibmc_unload(struct drm_device *dev)
+{
+	struct hibmc_drm_private *priv = dev->dev_private;
+
+	hibmc_fbdev_fini(priv);
+
+	if (dev->irq_enabled)
+		drm_irq_uninstall(dev);
+	if (priv->msi_enabled)
+		pci_disable_msi(dev->pdev);
+	drm_vblank_cleanup(dev);
+
+	hibmc_kms_fini(priv);
+	hibmc_mm_fini(priv);
+	dev->dev_private = NULL;
+	return 0;
+}
+
+static int hibmc_load(struct drm_device *dev)
+{
+	struct hibmc_drm_private *priv;
+	int ret;
+
+	priv = devm_kzalloc(dev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv) {
+		DRM_ERROR("no memory to allocate for hibmc_drm_private\n");
+		return -ENOMEM;
+	}
+	dev->dev_private = priv;
+	priv->dev = dev;
+
+	ret = hibmc_hw_init(priv);
+	if (ret)
+		goto err;
+
+	ret = hibmc_mm_init(priv);
+	if (ret)
+		goto err;
+
+	ret = hibmc_kms_init(priv);
+	if (ret)
+		goto err;
+
+	ret = drm_vblank_init(dev, dev->mode_config.num_crtc);
+	if (ret) {
+		DRM_ERROR("failed to initialize vblank: %d\n", ret);
+		goto err;
+	}
+
+	priv->msi_enabled = 0;
+	ret = pci_enable_msi(dev->pdev);
+	if (ret) {
+		DRM_WARN("enabling MSI failed: %d\n", ret);
+	} else {
+		priv->msi_enabled = 1;
+		ret = drm_irq_install(dev, dev->pdev->irq);
+		if (ret)
+			DRM_WARN("install irq failed: %d\n", ret);
+	}
+
+	/* reset all the states of crtc/plane/encoder/connector */
+	drm_mode_config_reset(dev);
+
+	ret = hibmc_fbdev_init(priv);
+	if (ret) {
+		DRM_ERROR("failed to initialize fbdev: %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	hibmc_unload(dev);
+	DRM_ERROR("failed to initialize drm driver: %d\n", ret);
+	return ret;
+}
+
+static int hibmc_pci_probe(struct pci_dev *pdev,
+			   const struct pci_device_id *ent)
+{
+	struct drm_device *dev;
+	int ret;
+
+	dev = drm_dev_alloc(&hibmc_driver, &pdev->dev);
+	if (IS_ERR(dev)) {
+		DRM_ERROR("failed to allocate drm_device\n");
+		return PTR_ERR(dev);
+	}
+
+	dev->pdev = pdev;
+	pci_set_drvdata(pdev, dev);
+
+	ret = pci_enable_device(pdev);
+	if (ret) {
+		DRM_ERROR("failed to enable pci device: %d\n", ret);
+		goto err_free;
+	}
+
+	ret = hibmc_load(dev);
+	if (ret) {
+		DRM_ERROR("failed to load hibmc: %d\n", ret);
+		goto err_disable;
+	}
+
+	ret = drm_dev_register(dev, 0);
+	if (ret) {
+		DRM_ERROR("failed to register drv for userspace access: %d\n",
+			  ret);
+		goto err_unload;
+	}
+	return 0;
+
+err_unload:
+	hibmc_unload(dev);
+err_disable:
+	pci_disable_device(pdev);
+err_free:
+	drm_dev_unref(dev);
+
+	return ret;
+}
+
+static void hibmc_pci_remove(struct pci_dev *pdev)
+{
+	struct drm_device *dev = pci_get_drvdata(pdev);
+
+	drm_dev_unregister(dev);
+	hibmc_unload(dev);
+	drm_dev_unref(dev);
+}
+
+static struct pci_device_id hibmc_pci_table[] = {
+	{0x19e5, 0x1711, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
+	{0,}
+};
+
+static struct pci_driver hibmc_pci_driver = {
+	.name =		"hibmc-drm",
+	.id_table =	hibmc_pci_table,
+	.probe =	hibmc_pci_probe,
+	.remove =	hibmc_pci_remove,
+	.driver.pm =    &hibmc_pm_ops,
+};
+
+static int __init hibmc_init(void)
+{
+	return pci_register_driver(&hibmc_pci_driver);
+}
+
+static void __exit hibmc_exit(void)
+{
+	return pci_unregister_driver(&hibmc_pci_driver);
+}
+
+module_init(hibmc_init);
+module_exit(hibmc_exit);
+
+MODULE_DEVICE_TABLE(pci, hibmc_pci_table);
+MODULE_AUTHOR("RongrongZou <zourongrong@huawei.com>");
+MODULE_DESCRIPTION("DRM Driver for Hisilicon Hibmc");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h
new file mode 100644
index 000000000000..e195521eb41e
--- /dev/null
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h
@@ -0,0 +1,114 @@
+/* Hisilicon Hibmc SoC drm driver
+ *
+ * Based on the bochs drm driver.
+ *
+ * Copyright (c) 2016 Huawei Limited.
+ *
+ * Author:
+ *	Rongrong Zou <zourongrong@huawei.com>
+ *	Rongrong Zou <zourongrong@gmail.com>
+ *	Jianhua Li <lijianhua@huawei.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#ifndef HIBMC_DRM_DRV_H
+#define HIBMC_DRM_DRV_H
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_gem.h>
+#include <drm/ttm/ttm_bo_driver.h>
+
+struct hibmc_framebuffer {
+	struct drm_framebuffer fb;
+	struct drm_gem_object *obj;
+};
+
+struct hibmc_fbdev {
+	struct drm_fb_helper helper;
+	struct hibmc_framebuffer *fb;
+	int size;
+};
+
+struct hibmc_drm_private {
+	/* hw */
+	void __iomem   *mmio;
+	void __iomem   *fb_map;
+	unsigned long  fb_base;
+	unsigned long  fb_size;
+	bool msi_enabled;
+
+	/* drm */
+	struct drm_device  *dev;
+	bool mode_config_initialized;
+	struct drm_atomic_state *suspend_state;
+
+	/* ttm */
+	struct drm_global_reference mem_global_ref;
+	struct ttm_bo_global_ref bo_global_ref;
+	struct ttm_bo_device bdev;
+	bool initialized;
+
+	/* fbdev */
+	struct hibmc_fbdev *fbdev;
+	bool mm_inited;
+};
+
+#define to_hibmc_framebuffer(x) container_of(x, struct hibmc_framebuffer, fb)
+
+struct hibmc_bo {
+	struct ttm_buffer_object bo;
+	struct ttm_placement placement;
+	struct ttm_bo_kmap_obj kmap;
+	struct drm_gem_object gem;
+	struct ttm_place placements[3];
+	int pin_count;
+};
+
+static inline struct hibmc_bo *hibmc_bo(struct ttm_buffer_object *bo)
+{
+	return container_of(bo, struct hibmc_bo, bo);
+}
+
+static inline struct hibmc_bo *gem_to_hibmc_bo(struct drm_gem_object *gem)
+{
+	return container_of(gem, struct hibmc_bo, gem);
+}
+
+void hibmc_set_power_mode(struct hibmc_drm_private *priv,
+			  unsigned int power_mode);
+void hibmc_set_current_gate(struct hibmc_drm_private *priv,
+			    unsigned int gate);
+
+int hibmc_de_init(struct hibmc_drm_private *priv);
+int hibmc_vdac_init(struct hibmc_drm_private *priv);
+int hibmc_fbdev_init(struct hibmc_drm_private *priv);
+void hibmc_fbdev_fini(struct hibmc_drm_private *priv);
+
+int hibmc_gem_create(struct drm_device *dev, u32 size, bool iskernel,
+		     struct drm_gem_object **obj);
+struct hibmc_framebuffer *
+hibmc_framebuffer_init(struct drm_device *dev,
+		       const struct drm_mode_fb_cmd2 *mode_cmd,
+		       struct drm_gem_object *obj);
+
+int hibmc_mm_init(struct hibmc_drm_private *hibmc);
+void hibmc_mm_fini(struct hibmc_drm_private *hibmc);
+int hibmc_bo_pin(struct hibmc_bo *bo, u32 pl_flag, u64 *gpu_addr);
+int hibmc_bo_unpin(struct hibmc_bo *bo);
+void hibmc_gem_free_object(struct drm_gem_object *obj);
+int hibmc_dumb_create(struct drm_file *file, struct drm_device *dev,
+		      struct drm_mode_create_dumb *args);
+int hibmc_dumb_mmap_offset(struct drm_file *file, struct drm_device *dev,
+			   u32 handle, u64 *offset);
+int hibmc_mmap(struct file *filp, struct vm_area_struct *vma);
+
+extern const struct drm_mode_config_funcs hibmc_mode_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_fbdev.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_fbdev.c
new file mode 100644
index 000000000000..9b0696735ba1
--- /dev/null
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_fbdev.c
@@ -0,0 +1,267 @@
+/* Hisilicon Hibmc SoC drm driver
+ *
+ * Based on the bochs drm driver.
+ *
+ * Copyright (c) 2016 Huawei Limited.
+ *
+ * Author:
+ *	Rongrong Zou <zourongrong@huawei.com>
+ *	Rongrong Zou <zourongrong@gmail.com>
+ *	Jianhua Li <lijianhua@huawei.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+
+#include "hibmc_drm_drv.h"
+
+static int hibmcfb_create_object(
+				struct hibmc_drm_private *priv,
+				const struct drm_mode_fb_cmd2 *mode_cmd,
+				struct drm_gem_object **gobj_p)
+{
+	struct drm_gem_object *gobj;
+	struct drm_device *dev = priv->dev;
+	u32 size;
+	int ret = 0;
+
+	size = mode_cmd->pitches[0] * mode_cmd->height;
+	ret = hibmc_gem_create(dev, size, true, &gobj);
+	if (ret)
+		return ret;
+
+	*gobj_p = gobj;
+	return ret;
+}
+
+static struct fb_ops hibmc_drm_fb_ops = {
+	.owner = THIS_MODULE,
+	.fb_check_var = drm_fb_helper_check_var,
+	.fb_set_par = drm_fb_helper_set_par,
+	.fb_fillrect = drm_fb_helper_sys_fillrect,
+	.fb_copyarea = drm_fb_helper_sys_copyarea,
+	.fb_imageblit = drm_fb_helper_sys_imageblit,
+	.fb_pan_display = drm_fb_helper_pan_display,
+	.fb_blank = drm_fb_helper_blank,
+	.fb_setcmap = drm_fb_helper_setcmap,
+};
+
+static int hibmc_drm_fb_create(struct drm_fb_helper *helper,
+			       struct drm_fb_helper_surface_size *sizes)
+{
+	struct hibmc_fbdev *hi_fbdev =
+		container_of(helper, struct hibmc_fbdev, helper);
+	struct hibmc_drm_private *priv = helper->dev->dev_private;
+	struct fb_info *info;
+	struct drm_mode_fb_cmd2 mode_cmd;
+	struct drm_gem_object *gobj = NULL;
+	int ret = 0;
+	int ret1;
+	size_t size;
+	unsigned int bytes_per_pixel;
+	struct hibmc_bo *bo = NULL;
+
+	DRM_DEBUG_DRIVER("surface width(%d), height(%d) and bpp(%d)\n",
+			 sizes->surface_width, sizes->surface_height,
+			 sizes->surface_bpp);
+	sizes->surface_depth = 32;
+
+	bytes_per_pixel = DIV_ROUND_UP(sizes->surface_bpp, 8);
+
+	mode_cmd.width = sizes->surface_width;
+	mode_cmd.height = sizes->surface_height;
+	mode_cmd.pitches[0] = mode_cmd.width * bytes_per_pixel;
+	mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
+							  sizes->surface_depth);
+
+	size = PAGE_ALIGN(mode_cmd.pitches[0] * mode_cmd.height);
+
+	ret = hibmcfb_create_object(priv, &mode_cmd, &gobj);
+	if (ret) {
+		DRM_ERROR("failed to create fbcon backing object: %d\n", ret);
+		return -ENOMEM;
+	}
+
+	bo = gem_to_hibmc_bo(gobj);
+
+	ret = ttm_bo_reserve(&bo->bo, true, false, NULL);
+	if (ret) {
+		DRM_ERROR("failed to reserve ttm_bo: %d\n", ret);
+		goto out_unref_gem;
+	}
+
+	ret = hibmc_bo_pin(bo, TTM_PL_FLAG_VRAM, NULL);
+	if (ret) {
+		DRM_ERROR("failed to pin fbcon: %d\n", ret);
+		goto out_unreserve_ttm_bo;
+	}
+
+	ret = ttm_bo_kmap(&bo->bo, 0, bo->bo.num_pages, &bo->kmap);
+	if (ret) {
+		DRM_ERROR("failed to kmap fbcon: %d\n", ret);
+		goto out_unpin_bo;
+	}
+	ttm_bo_unreserve(&bo->bo);
+
+	info = drm_fb_helper_alloc_fbi(helper);
+	if (IS_ERR(info)) {
+		ret = PTR_ERR(info);
+		DRM_ERROR("failed to allocate fbi: %d\n", ret);
+		goto out_release_fbi;
+	}
+
+	info->par = hi_fbdev;
+
+	hi_fbdev->fb = hibmc_framebuffer_init(priv->dev, &mode_cmd, gobj);
+	if (IS_ERR(hi_fbdev->fb)) {
+		ret = PTR_ERR(info);
+		DRM_ERROR("failed to initialize framebuffer: %d\n", ret);
+		goto out_release_fbi;
+	}
+
+	priv->fbdev->size = size;
+	hi_fbdev->helper.fb = &hi_fbdev->fb->fb;
+
+	strcpy(info->fix.id, "hibmcdrmfb");
+
+	info->flags = FBINFO_DEFAULT;
+	info->fbops = &hibmc_drm_fb_ops;
+
+	drm_fb_helper_fill_fix(info, hi_fbdev->fb->fb.pitches[0],
+			       hi_fbdev->fb->fb.depth);
+	drm_fb_helper_fill_var(info, &priv->fbdev->helper, sizes->fb_width,
+			       sizes->fb_height);
+
+	info->screen_base = bo->kmap.virtual;
+	info->screen_size = size;
+
+	info->fix.smem_start = bo->bo.mem.bus.offset + bo->bo.mem.bus.base;
+	info->fix.smem_len = size;
+	return 0;
+
+out_release_fbi:
+	drm_fb_helper_release_fbi(helper);
+	ret1 = ttm_bo_reserve(&bo->bo, true, false, NULL);
+	if (ret1) {
+		DRM_ERROR("failed to rsv ttm_bo when release fbi: %d\n", ret1);
+		goto out_unref_gem;
+	}
+	ttm_bo_kunmap(&bo->kmap);
+out_unpin_bo:
+	hibmc_bo_unpin(bo);
+out_unreserve_ttm_bo:
+	ttm_bo_unreserve(&bo->bo);
+out_unref_gem:
+	drm_gem_object_unreference_unlocked(gobj);
+
+	return ret;
+}
+
+static void hibmc_fbdev_destroy(struct hibmc_fbdev *fbdev)
+{
+	struct hibmc_framebuffer *gfb = fbdev->fb;
+	struct drm_fb_helper *fbh = &fbdev->helper;
+
+	drm_fb_helper_unregister_fbi(fbh);
+	drm_fb_helper_release_fbi(fbh);
+
+	drm_fb_helper_fini(fbh);
+
+	if (gfb)
+		drm_framebuffer_unreference(&gfb->fb);
+}
+
+static const struct drm_fb_helper_funcs hibmc_fbdev_helper_funcs = {
+	.fb_probe = hibmc_drm_fb_create,
+};
+
+int hibmc_fbdev_init(struct hibmc_drm_private *priv)
+{
+	int ret;
+	struct fb_var_screeninfo *var;
+	struct fb_fix_screeninfo *fix;
+	struct hibmc_fbdev *hifbdev;
+
+	hifbdev = devm_kzalloc(priv->dev->dev, sizeof(*hifbdev), GFP_KERNEL);
+	if (!hifbdev) {
+		DRM_ERROR("failed to allocate hibmc_fbdev\n");
+		return -ENOMEM;
+	}
+
+	priv->fbdev = hifbdev;
+	drm_fb_helper_prepare(priv->dev, &hifbdev->helper,
+			      &hibmc_fbdev_helper_funcs);
+
+	/* Now just one crtc and one channel */
+	ret = drm_fb_helper_init(priv->dev,
+				 &hifbdev->helper, 1, 1);
+	if (ret) {
+		DRM_ERROR("failed to initialize fb helper: %d\n", ret);
+		return ret;
+	}
+
+	ret = drm_fb_helper_single_add_all_connectors(&hifbdev->helper);
+	if (ret) {
+		DRM_ERROR("failed to add all connectors: %d\n", ret);
+		goto fini;
+	}
+
+	ret = drm_fb_helper_initial_config(&hifbdev->helper, 16);
+	if (ret) {
+		DRM_ERROR("failed to setup initial conn config: %d\n", ret);
+		goto fini;
+	}
+
+	var = &hifbdev->helper.fbdev->var;
+	fix = &hifbdev->helper.fbdev->fix;
+
+	DRM_DEBUG_DRIVER("Member of info->var is :\n"
+			 "xres=%d\n"
+			 "yres=%d\n"
+			 "xres_virtual=%d\n"
+			 "yres_virtual=%d\n"
+			 "xoffset=%d\n"
+			 "yoffset=%d\n"
+			 "bits_per_pixel=%d\n"
+			 "...\n", var->xres, var->yres, var->xres_virtual,
+			 var->yres_virtual, var->xoffset, var->yoffset,
+			 var->bits_per_pixel);
+	DRM_DEBUG_DRIVER("Member of info->fix is :\n"
+			 "smem_start=%lx\n"
+			 "smem_len=%d\n"
+			 "type=%d\n"
+			 "type_aux=%d\n"
+			 "visual=%d\n"
+			 "xpanstep=%d\n"
+			 "ypanstep=%d\n"
+			 "ywrapstep=%d\n"
+			 "line_length=%d\n"
+			 "accel=%d\n"
+			 "capabilities=%d\n"
+			 "...\n", fix->smem_start, fix->smem_len, fix->type,
+			 fix->type_aux, fix->visual, fix->xpanstep,
+			 fix->ypanstep, fix->ywrapstep, fix->line_length,
+			 fix->accel, fix->capabilities);
+
+	return 0;
+
+fini:
+	drm_fb_helper_fini(&hifbdev->helper);
+	return ret;
+}
+
+void hibmc_fbdev_fini(struct hibmc_drm_private *priv)
+{
+	if (!priv->fbdev)
+		return;
+
+	hibmc_fbdev_destroy(priv->fbdev);
+	priv->fbdev = NULL;
+}
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_regs.h b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_regs.h
new file mode 100644
index 000000000000..f7035bf3ec1f
--- /dev/null
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_regs.h
@@ -0,0 +1,196 @@
+/* Hisilicon Hibmc SoC drm driver
+ *
+ * Based on the bochs drm driver.
+ *
+ * Copyright (c) 2016 Huawei Limited.
+ *
+ * Author:
+ *	Rongrong Zou <zourongrong@huawei.com>
+ *	Rongrong Zou <zourongrong@gmail.com>
+ *	Jianhua Li <lijianhua@huawei.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#ifndef HIBMC_DRM_HW_H
+#define HIBMC_DRM_HW_H
+
+/* register definition */
+#define HIBMC_MISC_CTRL				0x4
+
+#define HIBMC_MSCCTL_LOCALMEM_RESET(x)		((x) << 6)
+#define HIBMC_MSCCTL_LOCALMEM_RESET_MASK	0x40
+
+#define HIBMC_CURRENT_GATE			0x000040
+#define HIBMC_CURR_GATE_DISPLAY(x)		((x) << 2)
+#define HIBMC_CURR_GATE_DISPLAY_MASK		0x4
+
+#define HIBMC_CURR_GATE_LOCALMEM(x)		((x) << 1)
+#define HIBMC_CURR_GATE_LOCALMEM_MASK		0x2
+
+#define HIBMC_MODE0_GATE			0x000044
+#define HIBMC_MODE1_GATE			0x000048
+#define HIBMC_POWER_MODE_CTRL			0x00004C
+
+#define HIBMC_PW_MODE_CTL_OSC_INPUT(x)		((x) << 3)
+#define HIBMC_PW_MODE_CTL_OSC_INPUT_MASK	0x8
+
+#define HIBMC_PW_MODE_CTL_MODE(x)		((x) << 0)
+#define HIBMC_PW_MODE_CTL_MODE_MASK		0x03
+#define HIBMC_PW_MODE_CTL_MODE_SHIFT		0
+
+#define HIBMC_PW_MODE_CTL_MODE_MODE0		0
+#define HIBMC_PW_MODE_CTL_MODE_MODE1		1
+#define HIBMC_PW_MODE_CTL_MODE_SLEEP		2
+
+#define HIBMC_PANEL_PLL_CTRL			0x00005C
+#define HIBMC_CRT_PLL_CTRL			0x000060
+
+#define HIBMC_PLL_CTRL_BYPASS(x)		((x) << 18)
+#define HIBMC_PLL_CTRL_BYPASS_MASK		0x40000
+
+#define HIBMC_PLL_CTRL_POWER(x)			((x) << 17)
+#define HIBMC_PLL_CTRL_POWER_MASK		0x20000
+
+#define HIBMC_PLL_CTRL_INPUT(x)			((x) << 16)
+#define HIBMC_PLL_CTRL_INPUT_MASK		0x10000
+
+#define HIBMC_PLL_CTRL_POD(x)			((x) << 14)
+#define HIBMC_PLL_CTRL_POD_MASK			0xC000
+
+#define HIBMC_PLL_CTRL_OD(x)			((x) << 12)
+#define HIBMC_PLL_CTRL_OD_MASK			0x3000
+
+#define HIBMC_PLL_CTRL_N(x)			((x) << 8)
+#define HIBMC_PLL_CTRL_N_MASK			0xF00
+
+#define HIBMC_PLL_CTRL_M(x)			((x) << 0)
+#define HIBMC_PLL_CTRL_M_MASK			0xFF
+
+#define HIBMC_CRT_DISP_CTL			0x80200
+
+#define HIBMC_CRT_DISP_CTL_CRTSELECT(x)		((x) << 25)
+#define HIBMC_CRT_DISP_CTL_CRTSELECT_MASK	0x2000000
+
+#define HIBMC_CRTSELECT_CRT			1
+
+#define HIBMC_CRT_DISP_CTL_CLOCK_PHASE(x)	((x) << 14)
+#define HIBMC_CRT_DISP_CTL_CLOCK_PHASE_MASK	0x4000
+
+#define HIBMC_CRT_DISP_CTL_VSYNC_PHASE(x)	((x) << 13)
+#define HIBMC_CRT_DISP_CTL_VSYNC_PHASE_MASK	0x2000
+
+#define HIBMC_CRT_DISP_CTL_HSYNC_PHASE(x)	((x) << 12)
+#define HIBMC_CRT_DISP_CTL_HSYNC_PHASE_MASK	0x1000
+
+#define HIBMC_CRT_DISP_CTL_TIMING(x)		((x) << 8)
+#define HIBMC_CRT_DISP_CTL_TIMING_MASK		0x100
+
+#define HIBMC_CRT_DISP_CTL_PLANE(x)		((x) << 2)
+#define HIBMC_CRT_DISP_CTL_PLANE_MASK		4
+
+#define HIBMC_CRT_DISP_CTL_FORMAT(x)		((x) << 0)
+#define HIBMC_CRT_DISP_CTL_FORMAT_MASK		0x03
+
+#define HIBMC_CRT_FB_ADDRESS			0x080204
+
+#define HIBMC_CRT_FB_WIDTH			0x080208
+#define HIBMC_CRT_FB_WIDTH_WIDTH(x)		((x) << 16)
+#define HIBMC_CRT_FB_WIDTH_WIDTH_MASK		0x3FFF0000
+#define HIBMC_CRT_FB_WIDTH_OFFS(x)		((x) << 0)
+#define HIBMC_CRT_FB_WIDTH_OFFS_MASK		0x3FFF
+
+#define HIBMC_CRT_HORZ_TOTAL			0x08020C
+#define HIBMC_CRT_HORZ_TOTAL_TOTAL(x)		((x) << 16)
+#define HIBMC_CRT_HORZ_TOTAL_TOTAL_MASK		0xFFF0000
+
+#define HIBMC_CRT_HORZ_TOTAL_DISP_END(x)	((x) << 0)
+#define HIBMC_CRT_HORZ_TOTAL_DISP_END_MASK	0xFFF
+
+#define HIBMC_CRT_HORZ_SYNC			0x080210
+#define HIBMC_CRT_HORZ_SYNC_WIDTH(x)		((x) << 16)
+#define HIBMC_CRT_HORZ_SYNC_WIDTH_MASK		0xFF0000
+
+#define HIBMC_CRT_HORZ_SYNC_START(x)		((x) << 0)
+#define HIBMC_CRT_HORZ_SYNC_START_MASK		0xFFF
+
+#define HIBMC_CRT_VERT_TOTAL			0x080214
+#define HIBMC_CRT_VERT_TOTAL_TOTAL(x)		((x) << 16)
+#define HIBMC_CRT_VERT_TOTAL_TOTAL_MASK		0x7FFF0000
+
+#define HIBMC_CRT_VERT_TOTAL_DISP_END(x)	((x) << 0)
+#define HIBMC_CRT_VERT_TOTAL_DISP_END_MASK	0x7FF
+
+#define HIBMC_CRT_VERT_SYNC			0x080218
+#define HIBMC_CRT_VERT_SYNC_HEIGHT(x)		((x) << 16)
+#define HIBMC_CRT_VERT_SYNC_HEIGHT_MASK		0x3F0000
+
+#define HIBMC_CRT_VERT_SYNC_START(x)		((x) << 0)
+#define HIBMC_CRT_VERT_SYNC_START_MASK		0x7FF
+
+/* Auto Centering */
+#define HIBMC_CRT_AUTO_CENTERING_TL		0x080280
+#define HIBMC_CRT_AUTO_CENTERING_TL_TOP(x)	((x) << 16)
+#define HIBMC_CRT_AUTO_CENTERING_TL_TOP_MASK	0x7FF0000
+
+#define HIBMC_CRT_AUTO_CENTERING_TL_LEFT(x)	((x) << 0)
+#define HIBMC_CRT_AUTO_CENTERING_TL_LEFT_MASK	0x7FF
+
+#define HIBMC_CRT_AUTO_CENTERING_BR		0x080284
+#define HIBMC_CRT_AUTO_CENTERING_BR_BOTTOM(x)	((x) << 16)
+#define HIBMC_CRT_AUTO_CENTERING_BR_BOTTOM_MASK	0x7FF0000
+
+#define HIBMC_CRT_AUTO_CENTERING_BR_RIGHT(x)	((x) << 0)
+#define HIBMC_CRT_AUTO_CENTERING_BR_RIGHT_MASK	0x7FF
+
+/* register to control panel output */
+#define HIBMC_DISPLAY_CONTROL_HISILE		0x80288
+#define HIBMC_DISPLAY_CONTROL_FPVDDEN(x)	((x) << 0)
+#define HIBMC_DISPLAY_CONTROL_PANELDATE(x)	((x) << 1)
+#define HIBMC_DISPLAY_CONTROL_FPEN(x)		((x) << 2)
+#define HIBMC_DISPLAY_CONTROL_VBIASEN(x)	((x) << 3)
+
+#define HIBMC_RAW_INTERRUPT			0x80290
+#define HIBMC_RAW_INTERRUPT_VBLANK(x)		((x) << 2)
+#define HIBMC_RAW_INTERRUPT_VBLANK_MASK		0x4
+
+#define HIBMC_RAW_INTERRUPT_EN			0x80298
+#define HIBMC_RAW_INTERRUPT_EN_VBLANK(x)	((x) << 2)
+#define HIBMC_RAW_INTERRUPT_EN_VBLANK_MASK	0x4
+
+/* register and values for PLL control */
+#define CRT_PLL1_HS				0x802a8
+#define CRT_PLL1_HS_OUTER_BYPASS(x)		((x) << 30)
+#define CRT_PLL1_HS_INTER_BYPASS(x)		((x) << 29)
+#define CRT_PLL1_HS_POWERON(x)			((x) << 24)
+
+#define CRT_PLL1_HS_25MHZ			0x23d40f02
+#define CRT_PLL1_HS_40MHZ			0x23940801
+#define CRT_PLL1_HS_65MHZ			0x23940d01
+#define CRT_PLL1_HS_78MHZ			0x23540F82
+#define CRT_PLL1_HS_74MHZ			0x23941dc2
+#define CRT_PLL1_HS_80MHZ			0x23941001
+#define CRT_PLL1_HS_80MHZ_1152			0x23540fc2
+#define CRT_PLL1_HS_108MHZ			0x23b41b01
+#define CRT_PLL1_HS_162MHZ			0x23480681
+#define CRT_PLL1_HS_148MHZ			0x23541dc2
+#define CRT_PLL1_HS_193MHZ			0x234807c1
+
+#define CRT_PLL2_HS				0x802ac
+#define CRT_PLL2_HS_25MHZ			0x206B851E
+#define CRT_PLL2_HS_40MHZ			0x30000000
+#define CRT_PLL2_HS_65MHZ			0x40000000
+#define CRT_PLL2_HS_78MHZ			0x50E147AE
+#define CRT_PLL2_HS_74MHZ			0x602B6AE7
+#define CRT_PLL2_HS_80MHZ			0x70000000
+#define CRT_PLL2_HS_108MHZ			0x80000000
+#define CRT_PLL2_HS_162MHZ			0xA0000000
+#define CRT_PLL2_HS_148MHZ			0xB0CCCCCD
+#define CRT_PLL2_HS_193MHZ			0xC0872B02
+
+#define HIBMC_FIELD(field, value) (field(value) & field##_MASK)
+#endif
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
new file mode 100644
index 000000000000..12a18557c5fd
--- /dev/null
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
@@ -0,0 +1,140 @@
+/* Hisilicon Hibmc SoC drm driver
+ *
+ * Based on the bochs drm driver.
+ *
+ * Copyright (c) 2016 Huawei Limited.
+ *
+ * Author:
+ *	Rongrong Zou <zourongrong@huawei.com>
+ *	Rongrong Zou <zourongrong@gmail.com>
+ *	Jianhua Li <lijianhua@huawei.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+
+#include "hibmc_drm_drv.h"
+#include "hibmc_drm_regs.h"
+
+static int hibmc_connector_get_modes(struct drm_connector *connector)
+{
+	return drm_add_modes_noedid(connector, 800, 600);
+}
+
+static int hibmc_connector_mode_valid(struct drm_connector *connector,
+				      struct drm_display_mode *mode)
+{
+	return MODE_OK;
+}
+
+static struct drm_encoder *
+hibmc_connector_best_encoder(struct drm_connector *connector)
+{
+	return drm_encoder_find(connector->dev, connector->encoder_ids[0]);
+}
+
+static const struct drm_connector_helper_funcs
+	hibmc_connector_helper_funcs = {
+	.get_modes = hibmc_connector_get_modes,
+	.mode_valid = hibmc_connector_mode_valid,
+	.best_encoder = hibmc_connector_best_encoder,
+};
+
+static const struct drm_connector_funcs hibmc_connector_funcs = {
+	.dpms = drm_atomic_helper_connector_dpms,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = drm_connector_cleanup,
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static struct drm_connector *
+hibmc_connector_init(struct hibmc_drm_private *priv)
+{
+	struct drm_device *dev = priv->dev;
+	struct drm_connector *connector;
+	int ret;
+
+	connector = devm_kzalloc(dev->dev, sizeof(*connector), GFP_KERNEL);
+	if (!connector) {
+		DRM_ERROR("failed to alloc memory when init connector\n");
+		return ERR_PTR(-ENOMEM);
+	}
+
+	ret = drm_connector_init(dev, connector,
+				 &hibmc_connector_funcs,
+				 DRM_MODE_CONNECTOR_VGA);
+	if (ret) {
+		DRM_ERROR("failed to init connector: %d\n", ret);
+		return ERR_PTR(ret);
+	}
+	drm_connector_helper_add(connector,
+				 &hibmc_connector_helper_funcs);
+
+	return connector;
+}
+
+static void hibmc_encoder_mode_set(struct drm_encoder *encoder,
+				   struct drm_display_mode *mode,
+				   struct drm_display_mode *adj_mode)
+{
+	u32 reg;
+	struct drm_device *dev = encoder->dev;
+	struct hibmc_drm_private *priv = dev->dev_private;
+
+	reg = readl(priv->mmio + HIBMC_DISPLAY_CONTROL_HISILE);
+	reg |= HIBMC_DISPLAY_CONTROL_FPVDDEN(1);
+	reg |= HIBMC_DISPLAY_CONTROL_PANELDATE(1);
+	reg |= HIBMC_DISPLAY_CONTROL_FPEN(1);
+	reg |= HIBMC_DISPLAY_CONTROL_VBIASEN(1);
+	writel(reg, priv->mmio + HIBMC_DISPLAY_CONTROL_HISILE);
+}
+
+static const struct drm_encoder_helper_funcs hibmc_encoder_helper_funcs = {
+	.mode_set = hibmc_encoder_mode_set,
+};
+
+static const struct drm_encoder_funcs hibmc_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
+int hibmc_vdac_init(struct hibmc_drm_private *priv)
+{
+	struct drm_device *dev = priv->dev;
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
+	int ret;
+
+	connector = hibmc_connector_init(priv);
+	if (IS_ERR(connector)) {
+		DRM_ERROR("failed to create connector: %ld\n",
+			  PTR_ERR(connector));
+		return PTR_ERR(connector);
+	}
+
+	encoder = devm_kzalloc(dev->dev, sizeof(*encoder), GFP_KERNEL);
+	if (!encoder) {
+		DRM_ERROR("failed to alloc memory when init encoder\n");
+		return -ENOMEM;
+	}
+
+	encoder->possible_crtcs = 0x1;
+	ret = drm_encoder_init(dev, encoder, &hibmc_encoder_funcs,
+			       DRM_MODE_ENCODER_DAC, NULL);
+	if (ret) {
+		DRM_ERROR("failed to init encoder: %d\n", ret);
+		return ret;
+	}
+
+	drm_encoder_helper_add(encoder, &hibmc_encoder_helper_funcs);
+	drm_mode_connector_attach_encoder(connector, encoder);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
new file mode 100644
index 000000000000..e76abf61edae
--- /dev/null
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
@@ -0,0 +1,558 @@
+/* Hisilicon Hibmc SoC drm driver
+ *
+ * Based on the bochs drm driver.
+ *
+ * Copyright (c) 2016 Huawei Limited.
+ *
+ * Author:
+ *	Rongrong Zou <zourongrong@huawei.com>
+ *	Rongrong Zou <zourongrong@gmail.com>
+ *	Jianhua Li <lijianhua@huawei.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#include <drm/drm_atomic_helper.h>
+#include <ttm/ttm_page_alloc.h>
+
+#include "hibmc_drm_drv.h"
+
+#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
+
+static inline struct hibmc_drm_private *
+hibmc_bdev(struct ttm_bo_device *bd)
+{
+	return container_of(bd, struct hibmc_drm_private, bdev);
+}
+
+static int
+hibmc_ttm_mem_global_init(struct drm_global_reference *ref)
+{
+	return ttm_mem_global_init(ref->object);
+}
+
+static void
+hibmc_ttm_mem_global_release(struct drm_global_reference *ref)
+{
+	ttm_mem_global_release(ref->object);
+}
+
+static int hibmc_ttm_global_init(struct hibmc_drm_private *hibmc)
+{
+	int ret;
+
+	hibmc->mem_global_ref.global_type = DRM_GLOBAL_TTM_MEM;
+	hibmc->mem_global_ref.size = sizeof(struct ttm_mem_global);
+	hibmc->mem_global_ref.init = &hibmc_ttm_mem_global_init;
+	hibmc->mem_global_ref.release = &hibmc_ttm_mem_global_release;
+	ret = drm_global_item_ref(&hibmc->mem_global_ref);
+	if (ret) {
+		DRM_ERROR("could not get ref on ttm global: %d\n", ret);
+		return ret;
+	}
+
+	hibmc->bo_global_ref.mem_glob =
+		hibmc->mem_global_ref.object;
+	hibmc->bo_global_ref.ref.global_type = DRM_GLOBAL_TTM_BO;
+	hibmc->bo_global_ref.ref.size = sizeof(struct ttm_bo_global);
+	hibmc->bo_global_ref.ref.init = &ttm_bo_global_init;
+	hibmc->bo_global_ref.ref.release = &ttm_bo_global_release;
+	ret = drm_global_item_ref(&hibmc->bo_global_ref.ref);
+	if (ret) {
+		DRM_ERROR("failed setting up TTM BO subsystem: %d\n", ret);
+		drm_global_item_unref(&hibmc->mem_global_ref);
+		return ret;
+	}
+	return 0;
+}
+
+static void
+hibmc_ttm_global_release(struct hibmc_drm_private *hibmc)
+{
+	drm_global_item_unref(&hibmc->bo_global_ref.ref);
+	drm_global_item_unref(&hibmc->mem_global_ref);
+	hibmc->mem_global_ref.release = NULL;
+}
+
+static void hibmc_bo_ttm_destroy(struct ttm_buffer_object *tbo)
+{
+	struct hibmc_bo *bo = container_of(tbo, struct hibmc_bo, bo);
+
+	drm_gem_object_release(&bo->gem);
+	kfree(bo);
+}
+
+static bool hibmc_ttm_bo_is_hibmc_bo(struct ttm_buffer_object *bo)
+{
+	return bo->destroy == &hibmc_bo_ttm_destroy;
+}
+
+static int
+hibmc_bo_init_mem_type(struct ttm_bo_device *bdev, u32 type,
+		       struct ttm_mem_type_manager *man)
+{
+	switch (type) {
+	case TTM_PL_SYSTEM:
+		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
+		man->available_caching = TTM_PL_MASK_CACHING;
+		man->default_caching = TTM_PL_FLAG_CACHED;
+		break;
+	case TTM_PL_VRAM:
+		man->func = &ttm_bo_manager_func;
+		man->flags = TTM_MEMTYPE_FLAG_FIXED |
+			TTM_MEMTYPE_FLAG_MAPPABLE;
+		man->available_caching = TTM_PL_FLAG_UNCACHED |
+			TTM_PL_FLAG_WC;
+		man->default_caching = TTM_PL_FLAG_WC;
+		break;
+	default:
+		DRM_ERROR("unsupported memory type %u\n", type);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+void hibmc_ttm_placement(struct hibmc_bo *bo, int domain)
+{
+	u32 count = 0;
+	u32 i;
+
+	bo->placement.placement = bo->placements;
+	bo->placement.busy_placement = bo->placements;
+	if (domain & TTM_PL_FLAG_VRAM)
+		bo->placements[count++].flags = TTM_PL_FLAG_WC |
+			TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_VRAM;
+	if (domain & TTM_PL_FLAG_SYSTEM)
+		bo->placements[count++].flags = TTM_PL_MASK_CACHING |
+			TTM_PL_FLAG_SYSTEM;
+	if (!count)
+		bo->placements[count++].flags = TTM_PL_MASK_CACHING |
+			TTM_PL_FLAG_SYSTEM;
+
+	bo->placement.num_placement = count;
+	bo->placement.num_busy_placement = count;
+	for (i = 0; i < count; i++) {
+		bo->placements[i].fpfn = 0;
+		bo->placements[i].lpfn = 0;
+	}
+}
+
+static void
+hibmc_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
+{
+	struct hibmc_bo *hibmcbo = hibmc_bo(bo);
+
+	if (!hibmc_ttm_bo_is_hibmc_bo(bo))
+		return;
+
+	hibmc_ttm_placement(hibmcbo, TTM_PL_FLAG_SYSTEM);
+	*pl = hibmcbo->placement;
+}
+
+static int hibmc_bo_verify_access(struct ttm_buffer_object *bo,
+				  struct file *filp)
+{
+	struct hibmc_bo *hibmcbo = hibmc_bo(bo);
+
+	return drm_vma_node_verify_access(&hibmcbo->gem.vma_node,
+					  filp->private_data);
+}
+
+static int hibmc_ttm_io_mem_reserve(struct ttm_bo_device *bdev,
+				    struct ttm_mem_reg *mem)
+{
+	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
+	struct hibmc_drm_private *hibmc = hibmc_bdev(bdev);
+
+	mem->bus.addr = NULL;
+	mem->bus.offset = 0;
+	mem->bus.size = mem->num_pages << PAGE_SHIFT;
+	mem->bus.base = 0;
+	mem->bus.is_iomem = false;
+	if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
+		return -EINVAL;
+	switch (mem->mem_type) {
+	case TTM_PL_SYSTEM:
+		/* system memory */
+		return 0;
+	case TTM_PL_VRAM:
+		mem->bus.offset = mem->start << PAGE_SHIFT;
+		mem->bus.base = pci_resource_start(hibmc->dev->pdev, 0);
+		mem->bus.is_iomem = true;
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static void hibmc_ttm_backend_destroy(struct ttm_tt *tt)
+{
+	ttm_tt_fini(tt);
+	kfree(tt);
+}
+
+static struct ttm_backend_func hibmc_tt_backend_func = {
+	.destroy = &hibmc_ttm_backend_destroy,
+};
+
+static struct ttm_tt *hibmc_ttm_tt_create(struct ttm_bo_device *bdev,
+					  unsigned long size,
+					  u32 page_flags,
+					  struct page *dummy_read_page)
+{
+	struct ttm_tt *tt;
+	int ret;
+
+	tt = kzalloc(sizeof(*tt), GFP_KERNEL);
+	if (!tt) {
+		DRM_ERROR("failed to allocate ttm_tt\n");
+		return NULL;
+	}
+	tt->func = &hibmc_tt_backend_func;
+	ret = ttm_tt_init(tt, bdev, size, page_flags, dummy_read_page);
+	if (ret) {
+		DRM_ERROR("failed to initialize ttm_tt: %d\n", ret);
+		kfree(tt);
+		return NULL;
+	}
+	return tt;
+}
+
+static int hibmc_ttm_tt_populate(struct ttm_tt *ttm)
+{
+	return ttm_pool_populate(ttm);
+}
+
+static void hibmc_ttm_tt_unpopulate(struct ttm_tt *ttm)
+{
+	ttm_pool_unpopulate(ttm);
+}
+
+struct ttm_bo_driver hibmc_bo_driver = {
+	.ttm_tt_create		= hibmc_ttm_tt_create,
+	.ttm_tt_populate	= hibmc_ttm_tt_populate,
+	.ttm_tt_unpopulate	= hibmc_ttm_tt_unpopulate,
+	.init_mem_type		= hibmc_bo_init_mem_type,
+	.evict_flags		= hibmc_bo_evict_flags,
+	.move			= NULL,
+	.verify_access		= hibmc_bo_verify_access,
+	.io_mem_reserve		= &hibmc_ttm_io_mem_reserve,
+	.io_mem_free		= NULL,
+	.lru_tail		= &ttm_bo_default_lru_tail,
+	.swap_lru_tail		= &ttm_bo_default_swap_lru_tail,
+};
+
+int hibmc_mm_init(struct hibmc_drm_private *hibmc)
+{
+	int ret;
+	struct drm_device *dev = hibmc->dev;
+	struct ttm_bo_device *bdev = &hibmc->bdev;
+
+	ret = hibmc_ttm_global_init(hibmc);
+	if (ret)
+		return ret;
+
+	ret = ttm_bo_device_init(&hibmc->bdev,
+				 hibmc->bo_global_ref.ref.object,
+				 &hibmc_bo_driver,
+				 dev->anon_inode->i_mapping,
+				 DRM_FILE_PAGE_OFFSET,
+				 true);
+	if (ret) {
+		hibmc_ttm_global_release(hibmc);
+		DRM_ERROR("error initializing bo driver: %d\n", ret);
+		return ret;
+	}
+
+	ret = ttm_bo_init_mm(bdev, TTM_PL_VRAM,
+			     hibmc->fb_size >> PAGE_SHIFT);
+	if (ret) {
+		hibmc_ttm_global_release(hibmc);
+		DRM_ERROR("failed ttm VRAM init: %d\n", ret);
+		return ret;
+	}
+
+	hibmc->mm_inited = true;
+	return 0;
+}
+
+void hibmc_mm_fini(struct hibmc_drm_private *hibmc)
+{
+	if (!hibmc->mm_inited)
+		return;
+
+	ttm_bo_device_release(&hibmc->bdev);
+	hibmc_ttm_global_release(hibmc);
+	hibmc->mm_inited = false;
+}
+
+static void hibmc_bo_unref(struct hibmc_bo **bo)
+{
+	struct ttm_buffer_object *tbo;
+
+	if ((*bo) == NULL)
+		return;
+
+	tbo = &((*bo)->bo);
+	ttm_bo_unref(&tbo);
+	*bo = NULL;
+}
+
+int hibmc_bo_create(struct drm_device *dev, int size, int align,
+		    u32 flags, struct hibmc_bo **phibmcbo)
+{
+	struct hibmc_drm_private *hibmc = dev->dev_private;
+	struct hibmc_bo *hibmcbo;
+	size_t acc_size;
+	int ret;
+
+	hibmcbo = kzalloc(sizeof(*hibmcbo), GFP_KERNEL);
+	if (!hibmcbo) {
+		DRM_ERROR("failed to allocate hibmcbo\n");
+		return -ENOMEM;
+	}
+	ret = drm_gem_object_init(dev, &hibmcbo->gem, size);
+	if (ret) {
+		DRM_ERROR("failed to initialize drm gem object: %d\n", ret);
+		kfree(hibmcbo);
+		return ret;
+	}
+
+	hibmcbo->bo.bdev = &hibmc->bdev;
+
+	hibmc_ttm_placement(hibmcbo, TTM_PL_FLAG_VRAM | TTM_PL_FLAG_SYSTEM);
+
+	acc_size = ttm_bo_dma_acc_size(&hibmc->bdev, size,
+				       sizeof(struct hibmc_bo));
+
+	ret = ttm_bo_init(&hibmc->bdev, &hibmcbo->bo, size,
+			  ttm_bo_type_device, &hibmcbo->placement,
+			  align >> PAGE_SHIFT, false, NULL, acc_size,
+			  NULL, NULL, hibmc_bo_ttm_destroy);
+	if (ret) {
+		hibmc_bo_unref(&hibmcbo);
+		DRM_ERROR("failed to initialize ttm_bo: %d\n", ret);
+		return ret;
+	}
+
+	*phibmcbo = hibmcbo;
+	return 0;
+}
+
+int hibmc_bo_pin(struct hibmc_bo *bo, u32 pl_flag, u64 *gpu_addr)
+{
+	int i, ret;
+
+	if (bo->pin_count) {
+		bo->pin_count++;
+		if (gpu_addr)
+			*gpu_addr = bo->bo.offset;
+		return 0;
+	}
+
+	hibmc_ttm_placement(bo, pl_flag);
+	for (i = 0; i < bo->placement.num_placement; i++)
+		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
+	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	if (ret)
+		return ret;
+
+	bo->pin_count = 1;
+	if (gpu_addr)
+		*gpu_addr = bo->bo.offset;
+	return 0;
+}
+
+int hibmc_bo_unpin(struct hibmc_bo *bo)
+{
+	int i, ret;
+
+	if (!bo->pin_count) {
+		DRM_ERROR("unpin bad %p\n", bo);
+		return 0;
+	}
+	bo->pin_count--;
+	if (bo->pin_count)
+		return 0;
+
+	for (i = 0; i < bo->placement.num_placement ; i++)
+		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
+	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	if (ret) {
+		DRM_ERROR("validate failed for unpin: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int hibmc_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct drm_file *file_priv;
+	struct hibmc_drm_private *hibmc;
+
+	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
+		return -EINVAL;
+
+	file_priv = filp->private_data;
+	hibmc = file_priv->minor->dev->dev_private;
+	return ttm_bo_mmap(filp, vma, &hibmc->bdev);
+}
+
+int hibmc_gem_create(struct drm_device *dev, u32 size, bool iskernel,
+		     struct drm_gem_object **obj)
+{
+	struct hibmc_bo *hibmcbo;
+	int ret;
+
+	*obj = NULL;
+
+	size = PAGE_ALIGN(size);
+	if (size == 0) {
+		DRM_ERROR("error: zero size\n");
+		return -EINVAL;
+	}
+
+	ret = hibmc_bo_create(dev, size, 0, 0, &hibmcbo);
+	if (ret) {
+		if (ret != -ERESTARTSYS)
+			DRM_ERROR("failed to allocate GEM object: %d\n", ret);
+		return ret;
+	}
+	*obj = &hibmcbo->gem;
+	return 0;
+}
+
+int hibmc_dumb_create(struct drm_file *file, struct drm_device *dev,
+		      struct drm_mode_create_dumb *args)
+{
+	struct drm_gem_object *gobj;
+	u32 handle;
+	int ret;
+
+	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 16);
+	args->size = args->pitch * args->height;
+
+	ret = hibmc_gem_create(dev, args->size, false,
+			       &gobj);
+	if (ret) {
+		DRM_ERROR("failed to create GEM object: %d\n", ret);
+		return ret;
+	}
+
+	ret = drm_gem_handle_create(file, gobj, &handle);
+	drm_gem_object_unreference_unlocked(gobj);
+	if (ret) {
+		DRM_ERROR("failed to unreference GEM object: %d\n", ret);
+		return ret;
+	}
+
+	args->handle = handle;
+	return 0;
+}
+
+void hibmc_gem_free_object(struct drm_gem_object *obj)
+{
+	struct hibmc_bo *hibmcbo = gem_to_hibmc_bo(obj);
+
+	hibmc_bo_unref(&hibmcbo);
+}
+
+static u64 hibmc_bo_mmap_offset(struct hibmc_bo *bo)
+{
+	return drm_vma_node_offset_addr(&bo->bo.vma_node);
+}
+
+int hibmc_dumb_mmap_offset(struct drm_file *file, struct drm_device *dev,
+			   u32 handle, u64 *offset)
+{
+	struct drm_gem_object *obj;
+	struct hibmc_bo *bo;
+
+	obj = drm_gem_object_lookup(file, handle);
+	if (!obj)
+		return -ENOENT;
+
+	bo = gem_to_hibmc_bo(obj);
+	*offset = hibmc_bo_mmap_offset(bo);
+
+	drm_gem_object_unreference_unlocked(obj);
+	return 0;
+}
+
+static void hibmc_user_framebuffer_destroy(struct drm_framebuffer *fb)
+{
+	struct hibmc_framebuffer *hibmc_fb = to_hibmc_framebuffer(fb);
+
+	drm_gem_object_unreference_unlocked(hibmc_fb->obj);
+	drm_framebuffer_cleanup(fb);
+	kfree(hibmc_fb);
+}
+
+static const struct drm_framebuffer_funcs hibmc_fb_funcs = {
+	.destroy = hibmc_user_framebuffer_destroy,
+};
+
+struct hibmc_framebuffer *
+hibmc_framebuffer_init(struct drm_device *dev,
+		       const struct drm_mode_fb_cmd2 *mode_cmd,
+		       struct drm_gem_object *obj)
+{
+	struct hibmc_framebuffer *hibmc_fb;
+	int ret;
+
+	hibmc_fb = kzalloc(sizeof(*hibmc_fb), GFP_KERNEL);
+	if (!hibmc_fb) {
+		DRM_ERROR("failed to allocate hibmc_fb\n");
+		return ERR_PTR(-ENOMEM);
+	}
+
+	drm_helper_mode_fill_fb_struct(&hibmc_fb->fb, mode_cmd);
+	hibmc_fb->obj = obj;
+	ret = drm_framebuffer_init(dev, &hibmc_fb->fb, &hibmc_fb_funcs);
+	if (ret) {
+		DRM_ERROR("drm_framebuffer_init failed: %d\n", ret);
+		kfree(hibmc_fb);
+		return ERR_PTR(ret);
+	}
+
+	return hibmc_fb;
+}
+
+static struct drm_framebuffer *
+hibmc_user_framebuffer_create(struct drm_device *dev,
+			      struct drm_file *filp,
+			      const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	struct drm_gem_object *obj;
+	struct hibmc_framebuffer *hibmc_fb;
+
+	DRM_DEBUG_DRIVER("%dx%d, format %c%c%c%c\n",
+			 mode_cmd->width, mode_cmd->height,
+			 (mode_cmd->pixel_format) & 0xff,
+			 (mode_cmd->pixel_format >> 8)  & 0xff,
+			 (mode_cmd->pixel_format >> 16) & 0xff,
+			 (mode_cmd->pixel_format >> 24) & 0xff);
+
+	obj = drm_gem_object_lookup(filp, mode_cmd->handles[0]);
+	if (!obj)
+		return ERR_PTR(-ENOENT);
+
+	hibmc_fb = hibmc_framebuffer_init(dev, mode_cmd, obj);
+	if (IS_ERR(hibmc_fb)) {
+		drm_gem_object_unreference_unlocked(obj);
+		return ERR_PTR((long)hibmc_fb);
+	}
+	return &hibmc_fb->fb;
+}
+
+const struct drm_mode_config_funcs hibmc_mode_funcs = {
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+	.fb_create = hibmc_user_framebuffer_create,
+};
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
index 7e7a4d43d6b6..afc2b5d2d5f0 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
@@ -608,17 +608,16 @@ static void ade_rdma_set(void __iomem *base, struct drm_framebuffer *fb,
 			 u32 ch, u32 y, u32 in_h, u32 fmt)
 {
 	struct drm_gem_cma_object *obj = drm_fb_cma_get_gem_obj(fb, 0);
-	char *format_name;
+	struct drm_format_name_buf format_name;
 	u32 reg_ctrl, reg_addr, reg_size, reg_stride, reg_space, reg_en;
 	u32 stride = fb->pitches[0];
 	u32 addr = (u32)obj->paddr + y * stride;
 
 	DRM_DEBUG_DRIVER("rdma%d: (y=%d, height=%d), stride=%d, paddr=0x%x\n",
 			 ch + 1, y, in_h, stride, (u32)obj->paddr);
-	format_name = drm_get_format_name(fb->pixel_format);
 	DRM_DEBUG_DRIVER("addr=0x%x, fb:%dx%d, pixel_format=%d(%s)\n",
-			 addr, fb->width, fb->height, fmt, format_name);
-	kfree(format_name);
+			 addr, fb->width, fb->height, fmt,
+			 drm_get_format_name(fb->pixel_format, &format_name));
 
 	/* get reg offset */
 	reg_ctrl = RD_CH_CTRL(ch);
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
index 90377a609c98..ebd5f4fe4c23 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
@@ -24,6 +24,7 @@
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_of.h>
 
 #include "kirin_drm_drv.h"
 
@@ -151,9 +152,7 @@ static const struct file_operations kirin_drm_fops = {
 	.open		= drm_open,
 	.release	= drm_release,
 	.unlocked_ioctl	= drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl	= drm_compat_ioctl,
-#endif
 	.poll		= drm_poll,
 	.read		= drm_read,
 	.llseek		= no_llseek,
@@ -260,14 +259,13 @@ static struct device_node *kirin_get_remote_node(struct device_node *np)
 		DRM_ERROR("no valid endpoint node\n");
 		return ERR_PTR(-ENODEV);
 	}
-	of_node_put(endpoint);
 
 	remote = of_graph_get_remote_port_parent(endpoint);
+	of_node_put(endpoint);
 	if (!remote) {
 		DRM_ERROR("no valid remote node\n");
 		return ERR_PTR(-ENODEV);
 	}
-	of_node_put(remote);
 
 	if (!of_device_is_available(remote)) {
 		DRM_ERROR("not available for remote node\n");
@@ -294,7 +292,8 @@ static int kirin_drm_platform_probe(struct platform_device *pdev)
 	if (IS_ERR(remote))
 		return PTR_ERR(remote);
 
-	component_match_add(dev, &match, compare_of, remote);
+	drm_of_component_match_add(dev, &match, compare_of, remote);
+	of_node_put(remote);
 
 	return component_master_add_with_match(dev, &kirin_drm_ops, match);
 
diff --git a/drivers/gpu/drm/i2c/tda998x_drv.c b/drivers/gpu/drm/i2c/tda998x_drv.c
index 9798d400d817..86f47e190309 100644
--- a/drivers/gpu/drm/i2c/tda998x_drv.c
+++ b/drivers/gpu/drm/i2c/tda998x_drv.c
@@ -41,12 +41,15 @@ struct tda998x_priv {
 	struct i2c_client *hdmi;
 	struct mutex mutex;
 	u16 rev;
+	u8 cec_addr;
 	u8 current_page;
-	int dpms;
-	bool is_hdmi_sink;
+	bool is_on;
+	bool supports_infoframes;
+	bool sink_has_audio;
 	u8 vip_cntrl_0;
 	u8 vip_cntrl_1;
 	u8 vip_cntrl_2;
+	unsigned long tmds_clock;
 	struct tda998x_audio_params audio_params;
 
 	struct platform_device *audio_pdev;
@@ -105,6 +108,8 @@ struct tda998x_priv {
 # define I2C_MASTER_DIS_FILT      (1 << 1)
 # define I2C_MASTER_APP_STRT_LAT  (1 << 2)
 #define REG_FEAT_POWERDOWN        REG(0x00, 0x0e)     /* read/write */
+# define FEAT_POWERDOWN_PREFILT   BIT(0)
+# define FEAT_POWERDOWN_CSC       BIT(1)
 # define FEAT_POWERDOWN_SPDIF     (1 << 3)
 #define REG_INT_FLAGS_0           REG(0x00, 0x0f)     /* read/write */
 #define REG_INT_FLAGS_1           REG(0x00, 0x10)     /* read/write */
@@ -370,35 +375,46 @@ struct tda998x_priv {
 static void
 cec_write(struct tda998x_priv *priv, u16 addr, u8 val)
 {
-	struct i2c_client *client = priv->cec;
 	u8 buf[] = {addr, val};
+	struct i2c_msg msg = {
+		.addr = priv->cec_addr,
+		.len = 2,
+		.buf = buf,
+	};
 	int ret;
 
-	ret = i2c_master_send(client, buf, sizeof(buf));
+	ret = i2c_transfer(priv->hdmi->adapter, &msg, 1);
 	if (ret < 0)
-		dev_err(&client->dev, "Error %d writing to cec:0x%x\n", ret, addr);
+		dev_err(&priv->hdmi->dev, "Error %d writing to cec:0x%x\n",
+			ret, addr);
 }
 
 static u8
 cec_read(struct tda998x_priv *priv, u8 addr)
 {
-	struct i2c_client *client = priv->cec;
 	u8 val;
+	struct i2c_msg msg[2] = {
+		{
+			.addr = priv->cec_addr,
+			.len = 1,
+			.buf = &addr,
+		}, {
+			.addr = priv->cec_addr,
+			.flags = I2C_M_RD,
+			.len = 1,
+			.buf = &val,
+		},
+	};
 	int ret;
 
-	ret = i2c_master_send(client, &addr, sizeof(addr));
-	if (ret < 0)
-		goto fail;
-
-	ret = i2c_master_recv(client, &val, sizeof(val));
-	if (ret < 0)
-		goto fail;
+	ret = i2c_transfer(priv->hdmi->adapter, msg, ARRAY_SIZE(msg));
+	if (ret < 0) {
+		dev_err(&priv->hdmi->dev, "Error %d reading from cec:0x%x\n",
+			ret, addr);
+		val = 0;
+	}
 
 	return val;
-
-fail:
-	dev_err(&client->dev, "Error %d reading from cec:0x%x\n", ret, addr);
-	return 0;
 }
 
 static int
@@ -579,9 +595,9 @@ tda998x_reset(struct tda998x_priv *priv)
  * HPD assertion: it needs a delay of 100ms to avoid timing out while
  * trying to read EDID data.
  *
- * However, tda998x_encoder_get_modes() may be called at any moment
+ * However, tda998x_connector_get_modes() may be called at any moment
  * after tda998x_connector_detect() indicates that we are connected, so
- * we need to delay probing modes in tda998x_encoder_get_modes() after
+ * we need to delay probing modes in tda998x_connector_get_modes() after
  * we have seen a HPD inactive->active transition.  This code implements
  * that delay.
  */
@@ -630,28 +646,30 @@ static irqreturn_t tda998x_irq_thread(int irq, void *data)
 	bool handled = false;
 
 	sta = cec_read(priv, REG_CEC_INTSTATUS);
-	cec = cec_read(priv, REG_CEC_RXSHPDINT);
-	lvl = cec_read(priv, REG_CEC_RXSHPDLEV);
-	flag0 = reg_read(priv, REG_INT_FLAGS_0);
-	flag1 = reg_read(priv, REG_INT_FLAGS_1);
-	flag2 = reg_read(priv, REG_INT_FLAGS_2);
-	DRM_DEBUG_DRIVER(
-		"tda irq sta %02x cec %02x lvl %02x f0 %02x f1 %02x f2 %02x\n",
-		sta, cec, lvl, flag0, flag1, flag2);
-
-	if (cec & CEC_RXSHPDINT_HPD) {
-		if (lvl & CEC_RXSHPDLEV_HPD)
-			tda998x_edid_delay_start(priv);
-		else
-			schedule_work(&priv->detect_work);
-
-		handled = true;
-	}
+	if (sta & CEC_INTSTATUS_HDMI) {
+		cec = cec_read(priv, REG_CEC_RXSHPDINT);
+		lvl = cec_read(priv, REG_CEC_RXSHPDLEV);
+		flag0 = reg_read(priv, REG_INT_FLAGS_0);
+		flag1 = reg_read(priv, REG_INT_FLAGS_1);
+		flag2 = reg_read(priv, REG_INT_FLAGS_2);
+		DRM_DEBUG_DRIVER(
+			"tda irq sta %02x cec %02x lvl %02x f0 %02x f1 %02x f2 %02x\n",
+			sta, cec, lvl, flag0, flag1, flag2);
+
+		if (cec & CEC_RXSHPDINT_HPD) {
+			if (lvl & CEC_RXSHPDLEV_HPD)
+				tda998x_edid_delay_start(priv);
+			else
+				schedule_work(&priv->detect_work);
+
+			handled = true;
+		}
 
-	if ((flag2 & INT_FLAGS_2_EDID_BLK_RD) && priv->wq_edid_wait) {
-		priv->wq_edid_wait = 0;
-		wake_up(&priv->wq_edid);
-		handled = true;
+		if ((flag2 & INT_FLAGS_2_EDID_BLK_RD) && priv->wq_edid_wait) {
+			priv->wq_edid_wait = 0;
+			wake_up(&priv->wq_edid);
+			handled = true;
+		}
 	}
 
 	return IRQ_RETVAL(handled);
@@ -700,6 +718,8 @@ tda998x_write_avi(struct tda998x_priv *priv, struct drm_display_mode *mode)
 	tda998x_write_if(priv, DIP_IF_FLAGS_IF2, REG_IF2_HB0, &frame);
 }
 
+/* Audio support */
+
 static void tda998x_audio_mute(struct tda998x_priv *priv, bool on)
 {
 	if (on) {
@@ -713,8 +733,7 @@ static void tda998x_audio_mute(struct tda998x_priv *priv, bool on)
 
 static int
 tda998x_configure_audio(struct tda998x_priv *priv,
-			struct tda998x_audio_params *params,
-			unsigned mode_clock)
+			struct tda998x_audio_params *params)
 {
 	u8 buf[6], clksel_aip, clksel_fs, cts_n, adiv;
 	u32 n;
@@ -771,7 +790,7 @@ tda998x_configure_audio(struct tda998x_priv *priv,
 	 * assume 100MHz requires larger divider.
 	 */
 	adiv = AUDIO_DIV_SERCLK_8;
-	if (mode_clock > 100000)
+	if (priv->tmds_clock > 100000)
 		adiv++;			/* AUDIO_DIV_SERCLK_16 */
 
 	/* S/PDIF asks for a larger divider */
@@ -819,58 +838,281 @@ tda998x_configure_audio(struct tda998x_priv *priv,
 	return tda998x_write_aif(priv, &params->cea);
 }
 
-/* DRM encoder functions */
+static int tda998x_audio_hw_params(struct device *dev, void *data,
+				   struct hdmi_codec_daifmt *daifmt,
+				   struct hdmi_codec_params *params)
+{
+	struct tda998x_priv *priv = dev_get_drvdata(dev);
+	int i, ret;
+	struct tda998x_audio_params audio = {
+		.sample_width = params->sample_width,
+		.sample_rate = params->sample_rate,
+		.cea = params->cea,
+	};
+
+	memcpy(audio.status, params->iec.status,
+	       min(sizeof(audio.status), sizeof(params->iec.status)));
 
-static void tda998x_encoder_set_config(struct tda998x_priv *priv,
-				       const struct tda998x_encoder_params *p)
+	switch (daifmt->fmt) {
+	case HDMI_I2S:
+		if (daifmt->bit_clk_inv || daifmt->frame_clk_inv ||
+		    daifmt->bit_clk_master || daifmt->frame_clk_master) {
+			dev_err(dev, "%s: Bad flags %d %d %d %d\n", __func__,
+				daifmt->bit_clk_inv, daifmt->frame_clk_inv,
+				daifmt->bit_clk_master,
+				daifmt->frame_clk_master);
+			return -EINVAL;
+		}
+		for (i = 0; i < ARRAY_SIZE(priv->audio_port); i++)
+			if (priv->audio_port[i].format == AFMT_I2S)
+				audio.config = priv->audio_port[i].config;
+		audio.format = AFMT_I2S;
+		break;
+	case HDMI_SPDIF:
+		for (i = 0; i < ARRAY_SIZE(priv->audio_port); i++)
+			if (priv->audio_port[i].format == AFMT_SPDIF)
+				audio.config = priv->audio_port[i].config;
+		audio.format = AFMT_SPDIF;
+		break;
+	default:
+		dev_err(dev, "%s: Invalid format %d\n", __func__, daifmt->fmt);
+		return -EINVAL;
+	}
+
+	if (audio.config == 0) {
+		dev_err(dev, "%s: No audio configuration found\n", __func__);
+		return -EINVAL;
+	}
+
+	mutex_lock(&priv->audio_mutex);
+	if (priv->supports_infoframes && priv->sink_has_audio)
+		ret = tda998x_configure_audio(priv, &audio);
+	else
+		ret = 0;
+
+	if (ret == 0)
+		priv->audio_params = audio;
+	mutex_unlock(&priv->audio_mutex);
+
+	return ret;
+}
+
+static void tda998x_audio_shutdown(struct device *dev, void *data)
 {
-	priv->vip_cntrl_0 = VIP_CNTRL_0_SWAP_A(p->swap_a) |
-			    (p->mirr_a ? VIP_CNTRL_0_MIRR_A : 0) |
-			    VIP_CNTRL_0_SWAP_B(p->swap_b) |
-			    (p->mirr_b ? VIP_CNTRL_0_MIRR_B : 0);
-	priv->vip_cntrl_1 = VIP_CNTRL_1_SWAP_C(p->swap_c) |
-			    (p->mirr_c ? VIP_CNTRL_1_MIRR_C : 0) |
-			    VIP_CNTRL_1_SWAP_D(p->swap_d) |
-			    (p->mirr_d ? VIP_CNTRL_1_MIRR_D : 0);
-	priv->vip_cntrl_2 = VIP_CNTRL_2_SWAP_E(p->swap_e) |
-			    (p->mirr_e ? VIP_CNTRL_2_MIRR_E : 0) |
-			    VIP_CNTRL_2_SWAP_F(p->swap_f) |
-			    (p->mirr_f ? VIP_CNTRL_2_MIRR_F : 0);
+	struct tda998x_priv *priv = dev_get_drvdata(dev);
 
-	priv->audio_params = p->audio_params;
+	mutex_lock(&priv->audio_mutex);
+
+	reg_write(priv, REG_ENA_AP, 0);
+
+	priv->audio_params.format = AFMT_UNUSED;
+
+	mutex_unlock(&priv->audio_mutex);
 }
 
-static void tda998x_encoder_dpms(struct drm_encoder *encoder, int mode)
+int tda998x_audio_digital_mute(struct device *dev, void *data, bool enable)
 {
-	struct tda998x_priv *priv = enc_to_tda998x_priv(encoder);
+	struct tda998x_priv *priv = dev_get_drvdata(dev);
 
-	/* we only care about on or off: */
-	if (mode != DRM_MODE_DPMS_ON)
-		mode = DRM_MODE_DPMS_OFF;
+	mutex_lock(&priv->audio_mutex);
 
-	if (mode == priv->dpms)
-		return;
+	tda998x_audio_mute(priv, enable);
 
-	switch (mode) {
-	case DRM_MODE_DPMS_ON:
-		/* enable video ports, audio will be enabled later */
-		reg_write(priv, REG_ENA_VP_0, 0xff);
-		reg_write(priv, REG_ENA_VP_1, 0xff);
-		reg_write(priv, REG_ENA_VP_2, 0xff);
-		/* set muxing after enabling ports: */
-		reg_write(priv, REG_VIP_CNTRL_0, priv->vip_cntrl_0);
-		reg_write(priv, REG_VIP_CNTRL_1, priv->vip_cntrl_1);
-		reg_write(priv, REG_VIP_CNTRL_2, priv->vip_cntrl_2);
-		break;
-	case DRM_MODE_DPMS_OFF:
-		/* disable video ports */
-		reg_write(priv, REG_ENA_VP_0, 0x00);
-		reg_write(priv, REG_ENA_VP_1, 0x00);
-		reg_write(priv, REG_ENA_VP_2, 0x00);
-		break;
+	mutex_unlock(&priv->audio_mutex);
+	return 0;
+}
+
+static int tda998x_audio_get_eld(struct device *dev, void *data,
+				 uint8_t *buf, size_t len)
+{
+	struct tda998x_priv *priv = dev_get_drvdata(dev);
+
+	mutex_lock(&priv->audio_mutex);
+	memcpy(buf, priv->connector.eld,
+	       min(sizeof(priv->connector.eld), len));
+	mutex_unlock(&priv->audio_mutex);
+
+	return 0;
+}
+
+static const struct hdmi_codec_ops audio_codec_ops = {
+	.hw_params = tda998x_audio_hw_params,
+	.audio_shutdown = tda998x_audio_shutdown,
+	.digital_mute = tda998x_audio_digital_mute,
+	.get_eld = tda998x_audio_get_eld,
+};
+
+static int tda998x_audio_codec_init(struct tda998x_priv *priv,
+				    struct device *dev)
+{
+	struct hdmi_codec_pdata codec_data = {
+		.ops = &audio_codec_ops,
+		.max_i2s_channels = 2,
+	};
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(priv->audio_port); i++) {
+		if (priv->audio_port[i].format == AFMT_I2S &&
+		    priv->audio_port[i].config != 0)
+			codec_data.i2s = 1;
+		if (priv->audio_port[i].format == AFMT_SPDIF &&
+		    priv->audio_port[i].config != 0)
+			codec_data.spdif = 1;
 	}
 
-	priv->dpms = mode;
+	priv->audio_pdev = platform_device_register_data(
+		dev, HDMI_CODEC_DRV_NAME, PLATFORM_DEVID_AUTO,
+		&codec_data, sizeof(codec_data));
+
+	return PTR_ERR_OR_ZERO(priv->audio_pdev);
+}
+
+/* DRM connector functions */
+
+static int tda998x_connector_dpms(struct drm_connector *connector, int mode)
+{
+	if (drm_core_check_feature(connector->dev, DRIVER_ATOMIC))
+		return drm_atomic_helper_connector_dpms(connector, mode);
+	else
+		return drm_helper_connector_dpms(connector, mode);
+}
+
+static int tda998x_connector_fill_modes(struct drm_connector *connector,
+					uint32_t maxX, uint32_t maxY)
+{
+	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
+	int ret;
+
+	mutex_lock(&priv->audio_mutex);
+	ret = drm_helper_probe_single_connector_modes(connector, maxX, maxY);
+
+	if (connector->edid_blob_ptr) {
+		struct edid *edid = (void *)connector->edid_blob_ptr->data;
+
+		priv->sink_has_audio = drm_detect_monitor_audio(edid);
+	} else {
+		priv->sink_has_audio = false;
+	}
+	mutex_unlock(&priv->audio_mutex);
+
+	return ret;
+}
+
+static enum drm_connector_status
+tda998x_connector_detect(struct drm_connector *connector, bool force)
+{
+	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
+	u8 val = cec_read(priv, REG_CEC_RXSHPDLEV);
+
+	return (val & CEC_RXSHPDLEV_HPD) ? connector_status_connected :
+			connector_status_disconnected;
+}
+
+static void tda998x_connector_destroy(struct drm_connector *connector)
+{
+	drm_connector_cleanup(connector);
+}
+
+static const struct drm_connector_funcs tda998x_connector_funcs = {
+	.dpms = tda998x_connector_dpms,
+	.reset = drm_atomic_helper_connector_reset,
+	.fill_modes = tda998x_connector_fill_modes,
+	.detect = tda998x_connector_detect,
+	.destroy = tda998x_connector_destroy,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static int read_edid_block(void *data, u8 *buf, unsigned int blk, size_t length)
+{
+	struct tda998x_priv *priv = data;
+	u8 offset, segptr;
+	int ret, i;
+
+	offset = (blk & 1) ? 128 : 0;
+	segptr = blk / 2;
+
+	reg_write(priv, REG_DDC_ADDR, 0xa0);
+	reg_write(priv, REG_DDC_OFFS, offset);
+	reg_write(priv, REG_DDC_SEGM_ADDR, 0x60);
+	reg_write(priv, REG_DDC_SEGM, segptr);
+
+	/* enable reading EDID: */
+	priv->wq_edid_wait = 1;
+	reg_write(priv, REG_EDID_CTRL, 0x1);
+
+	/* flag must be cleared by sw: */
+	reg_write(priv, REG_EDID_CTRL, 0x0);
+
+	/* wait for block read to complete: */
+	if (priv->hdmi->irq) {
+		i = wait_event_timeout(priv->wq_edid,
+					!priv->wq_edid_wait,
+					msecs_to_jiffies(100));
+		if (i < 0) {
+			dev_err(&priv->hdmi->dev, "read edid wait err %d\n", i);
+			return i;
+		}
+	} else {
+		for (i = 100; i > 0; i--) {
+			msleep(1);
+			ret = reg_read(priv, REG_INT_FLAGS_2);
+			if (ret < 0)
+				return ret;
+			if (ret & INT_FLAGS_2_EDID_BLK_RD)
+				break;
+		}
+	}
+
+	if (i == 0) {
+		dev_err(&priv->hdmi->dev, "read edid timeout\n");
+		return -ETIMEDOUT;
+	}
+
+	ret = reg_read_range(priv, REG_EDID_DATA_0, buf, length);
+	if (ret != length) {
+		dev_err(&priv->hdmi->dev, "failed to read edid block %d: %d\n",
+			blk, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int tda998x_connector_get_modes(struct drm_connector *connector)
+{
+	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
+	struct edid *edid;
+	int n;
+
+	/*
+	 * If we get killed while waiting for the HPD timeout, return
+	 * no modes found: we are not in a restartable path, so we
+	 * can't handle signals gracefully.
+	 */
+	if (tda998x_edid_delay_wait(priv))
+		return 0;
+
+	if (priv->rev == TDA19988)
+		reg_clear(priv, REG_TX4, TX4_PD_RAM);
+
+	edid = drm_do_get_edid(connector, read_edid_block, priv);
+
+	if (priv->rev == TDA19988)
+		reg_set(priv, REG_TX4, TX4_PD_RAM);
+
+	if (!edid) {
+		dev_warn(&priv->hdmi->dev, "failed to read EDID\n");
+		return 0;
+	}
+
+	drm_mode_connector_update_edid_property(connector, edid);
+	n = drm_add_edid_modes(connector, edid);
+	drm_edid_to_eld(connector, edid);
+
+	kfree(edid);
+
+	return n;
 }
 
 static int tda998x_connector_mode_valid(struct drm_connector *connector,
@@ -888,6 +1130,80 @@ static int tda998x_connector_mode_valid(struct drm_connector *connector,
 	return MODE_OK;
 }
 
+static struct drm_encoder *
+tda998x_connector_best_encoder(struct drm_connector *connector)
+{
+	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
+
+	return &priv->encoder;
+}
+
+static
+const struct drm_connector_helper_funcs tda998x_connector_helper_funcs = {
+	.get_modes = tda998x_connector_get_modes,
+	.mode_valid = tda998x_connector_mode_valid,
+	.best_encoder = tda998x_connector_best_encoder,
+};
+
+static int tda998x_connector_init(struct tda998x_priv *priv,
+				  struct drm_device *drm)
+{
+	struct drm_connector *connector = &priv->connector;
+	int ret;
+
+	connector->interlace_allowed = 1;
+
+	if (priv->hdmi->irq)
+		connector->polled = DRM_CONNECTOR_POLL_HPD;
+	else
+		connector->polled = DRM_CONNECTOR_POLL_CONNECT |
+			DRM_CONNECTOR_POLL_DISCONNECT;
+
+	drm_connector_helper_add(connector, &tda998x_connector_helper_funcs);
+	ret = drm_connector_init(drm, connector, &tda998x_connector_funcs,
+				 DRM_MODE_CONNECTOR_HDMIA);
+	if (ret)
+		return ret;
+
+	drm_mode_connector_attach_encoder(&priv->connector, &priv->encoder);
+
+	return 0;
+}
+
+/* DRM encoder functions */
+
+static void tda998x_encoder_dpms(struct drm_encoder *encoder, int mode)
+{
+	struct tda998x_priv *priv = enc_to_tda998x_priv(encoder);
+	bool on;
+
+	/* we only care about on or off: */
+	on = mode == DRM_MODE_DPMS_ON;
+
+	if (on == priv->is_on)
+		return;
+
+	if (on) {
+		/* enable video ports, audio will be enabled later */
+		reg_write(priv, REG_ENA_VP_0, 0xff);
+		reg_write(priv, REG_ENA_VP_1, 0xff);
+		reg_write(priv, REG_ENA_VP_2, 0xff);
+		/* set muxing after enabling ports: */
+		reg_write(priv, REG_VIP_CNTRL_0, priv->vip_cntrl_0);
+		reg_write(priv, REG_VIP_CNTRL_1, priv->vip_cntrl_1);
+		reg_write(priv, REG_VIP_CNTRL_2, priv->vip_cntrl_2);
+
+		priv->is_on = true;
+	} else {
+		/* disable video ports */
+		reg_write(priv, REG_ENA_VP_0, 0x00);
+		reg_write(priv, REG_ENA_VP_1, 0x00);
+		reg_write(priv, REG_ENA_VP_2, 0x00);
+
+		priv->is_on = false;
+	}
+}
+
 static void
 tda998x_encoder_mode_set(struct drm_encoder *encoder,
 			 struct drm_display_mode *mode,
@@ -971,6 +1287,8 @@ tda998x_encoder_mode_set(struct drm_encoder *encoder,
 			div = 3;
 	}
 
+	mutex_lock(&priv->audio_mutex);
+
 	/* mute the audio FIFO: */
 	reg_set(priv, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_FIFO);
 
@@ -982,6 +1300,7 @@ tda998x_encoder_mode_set(struct drm_encoder *encoder,
 	/* no pre-filter or interpolator: */
 	reg_write(priv, REG_HVF_CNTRL_0, HVF_CNTRL_0_PREFIL(0) |
 			HVF_CNTRL_0_INTPOL(0));
+	reg_set(priv, REG_FEAT_POWERDOWN, FEAT_POWERDOWN_PREFILT);
 	reg_write(priv, REG_VIP_CNTRL_5, VIP_CNTRL_5_SP_CNT(0));
 	reg_write(priv, REG_VIP_CNTRL_4, VIP_CNTRL_4_BLANKIT(0) |
 			VIP_CNTRL_4_BLC(0));
@@ -1004,6 +1323,7 @@ tda998x_encoder_mode_set(struct drm_encoder *encoder,
 	/* set color matrix bypass flag: */
 	reg_write(priv, REG_MAT_CONTRL, MAT_CONTRL_MAT_BP |
 				MAT_CONTRL_MAT_SC(1));
+	reg_set(priv, REG_FEAT_POWERDOWN, FEAT_POWERDOWN_CSC);
 
 	/* set BIAS tmds value: */
 	reg_write(priv, REG_ANA_GENERAL, 0x09);
@@ -1064,8 +1384,22 @@ tda998x_encoder_mode_set(struct drm_encoder *encoder,
 	/* must be last register set: */
 	reg_write(priv, REG_TBG_CNTRL_0, 0);
 
-	/* Only setup the info frames if the sink is HDMI */
-	if (priv->is_hdmi_sink) {
+	priv->tmds_clock = adjusted_mode->clock;
+
+	/* CEA-861B section 6 says that:
+	 * CEA version 1 (CEA-861) has no support for infoframes.
+	 * CEA version 2 (CEA-861A) supports version 1 AVI infoframes,
+	 * and optional basic audio.
+	 * CEA version 3 (CEA-861B) supports version 1 and 2 AVI infoframes,
+	 * and optional digital audio, with audio infoframes.
+	 *
+	 * Since we only support generation of version 2 AVI infoframes,
+	 * ignore CEA version 2 and below (iow, behave as if we're a
+	 * CEA-861 source.)
+	 */
+	priv->supports_infoframes = priv->connector.display_info.cea_rev >= 3;
+
+	if (priv->supports_infoframes) {
 		/* We need to turn HDMI HDCP stuff on to get audio through */
 		reg &= ~TBG_CNTRL_1_DWIN_DIS;
 		reg_write(priv, REG_TBG_CNTRL_1, reg);
@@ -1074,127 +1408,12 @@ tda998x_encoder_mode_set(struct drm_encoder *encoder,
 
 		tda998x_write_avi(priv, adjusted_mode);
 
-		if (priv->audio_params.format != AFMT_UNUSED) {
-			mutex_lock(&priv->audio_mutex);
-			tda998x_configure_audio(priv,
-						&priv->audio_params,
-						adjusted_mode->clock);
-			mutex_unlock(&priv->audio_mutex);
-		}
-	}
-}
-
-static enum drm_connector_status
-tda998x_connector_detect(struct drm_connector *connector, bool force)
-{
-	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
-	u8 val = cec_read(priv, REG_CEC_RXSHPDLEV);
-
-	return (val & CEC_RXSHPDLEV_HPD) ? connector_status_connected :
-			connector_status_disconnected;
-}
-
-static int read_edid_block(void *data, u8 *buf, unsigned int blk, size_t length)
-{
-	struct tda998x_priv *priv = data;
-	u8 offset, segptr;
-	int ret, i;
-
-	offset = (blk & 1) ? 128 : 0;
-	segptr = blk / 2;
-
-	reg_write(priv, REG_DDC_ADDR, 0xa0);
-	reg_write(priv, REG_DDC_OFFS, offset);
-	reg_write(priv, REG_DDC_SEGM_ADDR, 0x60);
-	reg_write(priv, REG_DDC_SEGM, segptr);
-
-	/* enable reading EDID: */
-	priv->wq_edid_wait = 1;
-	reg_write(priv, REG_EDID_CTRL, 0x1);
-
-	/* flag must be cleared by sw: */
-	reg_write(priv, REG_EDID_CTRL, 0x0);
-
-	/* wait for block read to complete: */
-	if (priv->hdmi->irq) {
-		i = wait_event_timeout(priv->wq_edid,
-					!priv->wq_edid_wait,
-					msecs_to_jiffies(100));
-		if (i < 0) {
-			dev_err(&priv->hdmi->dev, "read edid wait err %d\n", i);
-			return i;
-		}
-	} else {
-		for (i = 100; i > 0; i--) {
-			msleep(1);
-			ret = reg_read(priv, REG_INT_FLAGS_2);
-			if (ret < 0)
-				return ret;
-			if (ret & INT_FLAGS_2_EDID_BLK_RD)
-				break;
-		}
-	}
-
-	if (i == 0) {
-		dev_err(&priv->hdmi->dev, "read edid timeout\n");
-		return -ETIMEDOUT;
+		if (priv->audio_params.format != AFMT_UNUSED &&
+		    priv->sink_has_audio)
+			tda998x_configure_audio(priv, &priv->audio_params);
 	}
 
-	ret = reg_read_range(priv, REG_EDID_DATA_0, buf, length);
-	if (ret != length) {
-		dev_err(&priv->hdmi->dev, "failed to read edid block %d: %d\n",
-			blk, ret);
-		return ret;
-	}
-
-	return 0;
-}
-
-static int tda998x_connector_get_modes(struct drm_connector *connector)
-{
-	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
-	struct edid *edid;
-	int n;
-
-	/*
-	 * If we get killed while waiting for the HPD timeout, return
-	 * no modes found: we are not in a restartable path, so we
-	 * can't handle signals gracefully.
-	 */
-	if (tda998x_edid_delay_wait(priv))
-		return 0;
-
-	if (priv->rev == TDA19988)
-		reg_clear(priv, REG_TX4, TX4_PD_RAM);
-
-	edid = drm_do_get_edid(connector, read_edid_block, priv);
-
-	if (priv->rev == TDA19988)
-		reg_set(priv, REG_TX4, TX4_PD_RAM);
-
-	if (!edid) {
-		dev_warn(&priv->hdmi->dev, "failed to read EDID\n");
-		return 0;
-	}
-
-	drm_mode_connector_update_edid_property(connector, edid);
-	n = drm_add_edid_modes(connector, edid);
-	priv->is_hdmi_sink = drm_detect_hdmi_monitor(edid);
-	drm_edid_to_eld(connector, edid);
-
-	kfree(edid);
-
-	return n;
-}
-
-static void tda998x_encoder_set_polling(struct tda998x_priv *priv,
-					struct drm_connector *connector)
-{
-	if (priv->hdmi->irq)
-		connector->polled = DRM_CONNECTOR_POLL_HPD;
-	else
-		connector->polled = DRM_CONNECTOR_POLL_CONNECT |
-			DRM_CONNECTOR_POLL_DISCONNECT;
+	mutex_unlock(&priv->audio_mutex);
 }
 
 static void tda998x_destroy(struct tda998x_priv *priv)
@@ -1215,145 +1434,6 @@ static void tda998x_destroy(struct tda998x_priv *priv)
 	i2c_unregister_device(priv->cec);
 }
 
-static int tda998x_audio_hw_params(struct device *dev, void *data,
-				   struct hdmi_codec_daifmt *daifmt,
-				   struct hdmi_codec_params *params)
-{
-	struct tda998x_priv *priv = dev_get_drvdata(dev);
-	int i, ret;
-	struct tda998x_audio_params audio = {
-		.sample_width = params->sample_width,
-		.sample_rate = params->sample_rate,
-		.cea = params->cea,
-	};
-
-	if (!priv->encoder.crtc)
-		return -ENODEV;
-
-	memcpy(audio.status, params->iec.status,
-	       min(sizeof(audio.status), sizeof(params->iec.status)));
-
-	switch (daifmt->fmt) {
-	case HDMI_I2S:
-		if (daifmt->bit_clk_inv || daifmt->frame_clk_inv ||
-		    daifmt->bit_clk_master || daifmt->frame_clk_master) {
-			dev_err(dev, "%s: Bad flags %d %d %d %d\n", __func__,
-				daifmt->bit_clk_inv, daifmt->frame_clk_inv,
-				daifmt->bit_clk_master,
-				daifmt->frame_clk_master);
-			return -EINVAL;
-		}
-		for (i = 0; i < ARRAY_SIZE(priv->audio_port); i++)
-			if (priv->audio_port[i].format == AFMT_I2S)
-				audio.config = priv->audio_port[i].config;
-		audio.format = AFMT_I2S;
-		break;
-	case HDMI_SPDIF:
-		for (i = 0; i < ARRAY_SIZE(priv->audio_port); i++)
-			if (priv->audio_port[i].format == AFMT_SPDIF)
-				audio.config = priv->audio_port[i].config;
-		audio.format = AFMT_SPDIF;
-		break;
-	default:
-		dev_err(dev, "%s: Invalid format %d\n", __func__, daifmt->fmt);
-		return -EINVAL;
-	}
-
-	if (audio.config == 0) {
-		dev_err(dev, "%s: No audio configutation found\n", __func__);
-		return -EINVAL;
-	}
-
-	mutex_lock(&priv->audio_mutex);
-	ret = tda998x_configure_audio(priv,
-				      &audio,
-				      priv->encoder.crtc->hwmode.clock);
-
-	if (ret == 0)
-		priv->audio_params = audio;
-	mutex_unlock(&priv->audio_mutex);
-
-	return ret;
-}
-
-static void tda998x_audio_shutdown(struct device *dev, void *data)
-{
-	struct tda998x_priv *priv = dev_get_drvdata(dev);
-
-	mutex_lock(&priv->audio_mutex);
-
-	reg_write(priv, REG_ENA_AP, 0);
-
-	priv->audio_params.format = AFMT_UNUSED;
-
-	mutex_unlock(&priv->audio_mutex);
-}
-
-int tda998x_audio_digital_mute(struct device *dev, void *data, bool enable)
-{
-	struct tda998x_priv *priv = dev_get_drvdata(dev);
-
-	mutex_lock(&priv->audio_mutex);
-
-	tda998x_audio_mute(priv, enable);
-
-	mutex_unlock(&priv->audio_mutex);
-	return 0;
-}
-
-static int tda998x_audio_get_eld(struct device *dev, void *data,
-				 uint8_t *buf, size_t len)
-{
-	struct tda998x_priv *priv = dev_get_drvdata(dev);
-	struct drm_mode_config *config = &priv->encoder.dev->mode_config;
-	struct drm_connector *connector;
-	int ret = -ENODEV;
-
-	mutex_lock(&config->mutex);
-	list_for_each_entry(connector, &config->connector_list, head) {
-		if (&priv->encoder == connector->encoder) {
-			memcpy(buf, connector->eld,
-			       min(sizeof(connector->eld), len));
-			ret = 0;
-		}
-	}
-	mutex_unlock(&config->mutex);
-
-	return ret;
-}
-
-static const struct hdmi_codec_ops audio_codec_ops = {
-	.hw_params = tda998x_audio_hw_params,
-	.audio_shutdown = tda998x_audio_shutdown,
-	.digital_mute = tda998x_audio_digital_mute,
-	.get_eld = tda998x_audio_get_eld,
-};
-
-static int tda998x_audio_codec_init(struct tda998x_priv *priv,
-				    struct device *dev)
-{
-	struct hdmi_codec_pdata codec_data = {
-		.ops = &audio_codec_ops,
-		.max_i2s_channels = 2,
-	};
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(priv->audio_port); i++) {
-		if (priv->audio_port[i].format == AFMT_I2S &&
-		    priv->audio_port[i].config != 0)
-			codec_data.i2s = 1;
-		if (priv->audio_port[i].format == AFMT_SPDIF &&
-		    priv->audio_port[i].config != 0)
-			codec_data.spdif = 1;
-	}
-
-	priv->audio_pdev = platform_device_register_data(
-		dev, HDMI_CODEC_DRV_NAME, PLATFORM_DEVID_AUTO,
-		&codec_data, sizeof(codec_data));
-
-	return PTR_ERR_OR_ZERO(priv->audio_pdev);
-}
-
 /* I2C driver functions */
 
 static int tda998x_get_audio_ports(struct tda998x_priv *priv,
@@ -1403,22 +1483,21 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 	struct device_node *np = client->dev.of_node;
 	u32 video;
 	int rev_lo, rev_hi, ret;
-	unsigned short cec_addr;
+
+	mutex_init(&priv->audio_mutex); /* Protect access from audio thread */
 
 	priv->vip_cntrl_0 = VIP_CNTRL_0_SWAP_A(2) | VIP_CNTRL_0_SWAP_B(3);
 	priv->vip_cntrl_1 = VIP_CNTRL_1_SWAP_C(0) | VIP_CNTRL_1_SWAP_D(1);
 	priv->vip_cntrl_2 = VIP_CNTRL_2_SWAP_E(4) | VIP_CNTRL_2_SWAP_F(5);
 
+	/* CEC I2C address bound to TDA998x I2C addr by configuration pins */
+	priv->cec_addr = 0x34 + (client->addr & 0x03);
 	priv->current_page = 0xff;
 	priv->hdmi = client;
-	/* CEC I2C address bound to TDA998x I2C addr by configuration pins */
-	cec_addr = 0x34 + (client->addr & 0x03);
-	priv->cec = i2c_new_dummy(client->adapter, cec_addr);
+	priv->cec = i2c_new_dummy(client->adapter, priv->cec_addr);
 	if (!priv->cec)
 		return -ENODEV;
 
-	priv->dpms = DRM_MODE_DPMS_OFF;
-
 	mutex_init(&priv->mutex);	/* protect the page access */
 	init_waitqueue_head(&priv->edid_delay_waitq);
 	setup_timer(&priv->edid_delay_timer, tda998x_edid_delay_done,
@@ -1478,7 +1557,7 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 
 	/* initialize the optional IRQ */
 	if (client->irq) {
-		int irqf_trigger;
+		unsigned long irq_flags;
 
 		/* init read EDID waitqueue and HDP work */
 		init_waitqueue_head(&priv->wq_edid);
@@ -1488,11 +1567,11 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 		reg_read(priv, REG_INT_FLAGS_1);
 		reg_read(priv, REG_INT_FLAGS_2);
 
-		irqf_trigger =
+		irq_flags =
 			irqd_get_trigger_type(irq_get_irq_data(client->irq));
+		irq_flags |= IRQF_SHARED | IRQF_ONESHOT;
 		ret = request_threaded_irq(client->irq, NULL,
-					   tda998x_irq_thread,
-					   irqf_trigger | IRQF_ONESHOT,
+					   tda998x_irq_thread, irq_flags,
 					   "tda998x", priv);
 		if (ret) {
 			dev_err(&client->dev,
@@ -1519,8 +1598,6 @@ static int tda998x_create(struct i2c_client *client, struct tda998x_priv *priv)
 		priv->vip_cntrl_2 = video;
 	}
 
-	mutex_init(&priv->audio_mutex); /* Protect access from audio thread */
-
 	ret = tda998x_get_audio_ports(priv, np);
 	if (ret)
 		goto fail;
@@ -1567,45 +1644,25 @@ static const struct drm_encoder_funcs tda998x_encoder_funcs = {
 	.destroy = tda998x_encoder_destroy,
 };
 
-static struct drm_encoder *
-tda998x_connector_best_encoder(struct drm_connector *connector)
-{
-	struct tda998x_priv *priv = conn_to_tda998x_priv(connector);
-
-	return &priv->encoder;
-}
-
-static
-const struct drm_connector_helper_funcs tda998x_connector_helper_funcs = {
-	.get_modes = tda998x_connector_get_modes,
-	.mode_valid = tda998x_connector_mode_valid,
-	.best_encoder = tda998x_connector_best_encoder,
-};
-
-static void tda998x_connector_destroy(struct drm_connector *connector)
+static void tda998x_set_config(struct tda998x_priv *priv,
+			       const struct tda998x_encoder_params *p)
 {
-	drm_connector_unregister(connector);
-	drm_connector_cleanup(connector);
-}
+	priv->vip_cntrl_0 = VIP_CNTRL_0_SWAP_A(p->swap_a) |
+			    (p->mirr_a ? VIP_CNTRL_0_MIRR_A : 0) |
+			    VIP_CNTRL_0_SWAP_B(p->swap_b) |
+			    (p->mirr_b ? VIP_CNTRL_0_MIRR_B : 0);
+	priv->vip_cntrl_1 = VIP_CNTRL_1_SWAP_C(p->swap_c) |
+			    (p->mirr_c ? VIP_CNTRL_1_MIRR_C : 0) |
+			    VIP_CNTRL_1_SWAP_D(p->swap_d) |
+			    (p->mirr_d ? VIP_CNTRL_1_MIRR_D : 0);
+	priv->vip_cntrl_2 = VIP_CNTRL_2_SWAP_E(p->swap_e) |
+			    (p->mirr_e ? VIP_CNTRL_2_MIRR_E : 0) |
+			    VIP_CNTRL_2_SWAP_F(p->swap_f) |
+			    (p->mirr_f ? VIP_CNTRL_2_MIRR_F : 0);
 
-static int tda998x_connector_dpms(struct drm_connector *connector, int mode)
-{
-	if (drm_core_check_feature(connector->dev, DRIVER_ATOMIC))
-		return drm_atomic_helper_connector_dpms(connector, mode);
-	else
-		return drm_helper_connector_dpms(connector, mode);
+	priv->audio_params = p->audio_params;
 }
 
-static const struct drm_connector_funcs tda998x_connector_funcs = {
-	.dpms = tda998x_connector_dpms,
-	.reset = drm_atomic_helper_connector_reset,
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.detect = tda998x_connector_detect,
-	.destroy = tda998x_connector_destroy,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-};
-
 static int tda998x_bind(struct device *dev, struct device *master, void *data)
 {
 	struct tda998x_encoder_params *params = dev->platform_data;
@@ -1630,7 +1687,6 @@ static int tda998x_bind(struct device *dev, struct device *master, void *data)
 		crtcs = 1 << 0;
 	}
 
-	priv->connector.interlace_allowed = 1;
 	priv->encoder.possible_crtcs = crtcs;
 
 	ret = tda998x_create(client, priv);
@@ -1638,9 +1694,7 @@ static int tda998x_bind(struct device *dev, struct device *master, void *data)
 		return ret;
 
 	if (!dev->of_node && params)
-		tda998x_encoder_set_config(priv, params);
-
-	tda998x_encoder_set_polling(priv, &priv->connector);
+		tda998x_set_config(priv, params);
 
 	drm_encoder_helper_add(&priv->encoder, &tda998x_encoder_helper_funcs);
 	ret = drm_encoder_init(drm, &priv->encoder, &tda998x_encoder_funcs,
@@ -1648,24 +1702,12 @@ static int tda998x_bind(struct device *dev, struct device *master, void *data)
 	if (ret)
 		goto err_encoder;
 
-	drm_connector_helper_add(&priv->connector,
-				 &tda998x_connector_helper_funcs);
-	ret = drm_connector_init(drm, &priv->connector,
-				 &tda998x_connector_funcs,
-				 DRM_MODE_CONNECTOR_HDMIA);
+	ret = tda998x_connector_init(priv, drm);
 	if (ret)
 		goto err_connector;
 
-	ret = drm_connector_register(&priv->connector);
-	if (ret)
-		goto err_sysfs;
-
-	drm_mode_connector_attach_encoder(&priv->connector, &priv->encoder);
-
 	return 0;
 
-err_sysfs:
-	drm_connector_cleanup(&priv->connector);
 err_connector:
 	drm_encoder_cleanup(&priv->encoder);
 err_encoder:
@@ -1678,7 +1720,6 @@ static void tda998x_unbind(struct device *dev, struct device *master,
 {
 	struct tda998x_priv *priv = dev_get_drvdata(dev);
 
-	drm_connector_unregister(&priv->connector);
 	drm_connector_cleanup(&priv->connector);
 	drm_encoder_cleanup(&priv->encoder);
 	tda998x_destroy(priv);
@@ -1692,6 +1733,10 @@ static const struct component_ops tda998x_ops = {
 static int
 tda998x_probe(struct i2c_client *client, const struct i2c_device_id *id)
 {
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
+		dev_warn(&client->dev, "adapter does not support I2C\n");
+		return -EIO;
+	}
 	return component_add(&client->dev, &tda998x_ops);
 }
 
diff --git a/drivers/gpu/drm/i810/i810_dma.c b/drivers/gpu/drm/i810/i810_dma.c
index d91856779beb..ab4e6cbe1f8b 100644
--- a/drivers/gpu/drm/i810/i810_dma.c
+++ b/drivers/gpu/drm/i810/i810_dma.c
@@ -113,9 +113,7 @@ static const struct file_operations i810_buffer_fops = {
 	.release = drm_release,
 	.unlocked_ioctl = drm_ioctl,
 	.mmap = i810_mmap_buffers,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.llseek = noop_llseek,
 };
 
diff --git a/drivers/gpu/drm/i810/i810_drv.c b/drivers/gpu/drm/i810/i810_drv.c
index 0be55dc1ef4b..02504a7cfaf2 100644
--- a/drivers/gpu/drm/i810/i810_drv.c
+++ b/drivers/gpu/drm/i810/i810_drv.c
@@ -49,9 +49,7 @@ static const struct file_operations i810_driver_fops = {
 	.unlocked_ioctl = drm_ioctl,
 	.mmap = drm_legacy_mmap,
 	.poll = drm_poll,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.llseek = noop_llseek,
 };
 
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig
index 7769e469118f..5ddde7349fbd 100644
--- a/drivers/gpu/drm/i915/Kconfig
+++ b/drivers/gpu/drm/i915/Kconfig
@@ -11,6 +11,7 @@ config DRM_I915
 	select DRM_KMS_HELPER
 	select DRM_PANEL
 	select DRM_MIPI_DSI
+	select RELAY
 	# i915 depends on ACPI_VIDEO when ACPI is enabled
 	# but for select to work, need to select ACPI_VIDEO's dependencies, ick
 	select BACKLIGHT_LCD_SUPPORT if ACPI
@@ -24,28 +25,59 @@ config DRM_I915
 	  including 830M, 845G, 852GM, 855GM, 865G, 915G, 945G, 965G,
 	  G35, G41, G43, G45 chipsets and Celeron, Pentium, Core i3,
 	  Core i5, Core i7 as well as Atom CPUs with integrated graphics.
-	  If M is selected, the module will be called i915.  AGP support
-	  is required for this driver to work. This driver is used by
-	  the Intel driver in X.org 6.8 and XFree86 4.4 and above. It
-	  replaces the older i830 module that supported a subset of the
-	  hardware in older X.org releases.
+
+	  This driver is used by the Intel driver in X.org 6.8 and
+	  XFree86 4.4 and above. It replaces the older i830 module that
+	  supported a subset of the hardware in older X.org releases.
 
 	  Note that the older i810/i815 chipsets require the use of the
 	  i810 driver instead, and the Atom z5xx series has an entirely
 	  different implementation.
 
-config DRM_I915_PRELIMINARY_HW_SUPPORT
-	bool "Enable preliminary support for prerelease Intel hardware by default"
+	  If "M" is selected, the module will be called i915.
+
+config DRM_I915_ALPHA_SUPPORT
+	bool "Enable alpha quality support for new Intel hardware by default"
 	depends on DRM_I915
 	default n
 	help
-	  Choose this option if you have prerelease Intel hardware and want the
-	  i915 driver to support it by default.  You can enable such support at
-	  runtime with the module option i915.preliminary_hw_support=1; this
-	  option changes the default for that module option.
+	  Choose this option if you have new Intel hardware and want to enable
+	  the alpha quality i915 driver support for the hardware in this kernel
+	  version. You can also enable the support at runtime using the module
+	  parameter i915.alpha_support=1; this option changes the default for
+	  that module parameter.
+
+	  It is recommended to upgrade to a kernel version with proper support
+	  as soon as it is available. Generally fixes for platforms with alpha
+	  support are not backported to older kernels.
 
 	  If in doubt, say "N".
 
+config DRM_I915_CAPTURE_ERROR
+	bool "Enable capturing GPU state following a hang"
+	depends on DRM_I915
+	default y
+	help
+	  This option enables capturing the GPU state when a hang is detected.
+	  This information is vital for triaging hangs and assists in debugging.
+	  Please report any hang to
+            https://bugs.freedesktop.org/enter_bug.cgi?product=DRI
+	  for triaging.
+
+	  If in doubt, say "Y".
+
+config DRM_I915_COMPRESS_ERROR
+	bool "Compress GPU error state"
+	depends on DRM_I915_CAPTURE_ERROR
+	select ZLIB_DEFLATE
+	default y
+	help
+	  This option selects ZLIB_DEFLATE if it isn't already
+	  selected and causes any error state captured upon a GPU hang
+	  to be compressed using zlib.
+
+	  If in doubt, say "Y".
+
 config DRM_I915_USERPTR
 	bool "Always enable userptr support"
 	depends on DRM_I915
@@ -60,6 +92,7 @@ config DRM_I915_USERPTR
 config DRM_I915_GVT
         bool "Enable Intel GVT-g graphics virtualization host support"
         depends on DRM_I915
+        depends on 64BIT
         default n
         help
 	  Choose this option if you want to enable Intel GVT-g graphics
@@ -79,6 +112,15 @@ config DRM_I915_GVT
 
 	  If in doubt, say "N".
 
+config DRM_I915_GVT_KVMGT
+	tristate "Enable KVM/VFIO support for Intel GVT-g"
+	depends on DRM_I915_GVT
+	depends on KVM
+	default n
+	help
+	  Choose this option if you want to enable KVMGT support for
+	  Intel GVT-g.
+
 menu "drm/i915 Debugging"
 depends on DRM_I915
 depends on EXPERT
diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index cee87bfd10c4..51ba630a134b 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -21,6 +21,7 @@ config DRM_I915_DEBUG
         select PREEMPT_COUNT
         select X86_MSR # used by igt/pm_rpm
         select DRM_VGEM # used by igt/prime_vgem (dmabuf interop checks)
+        select DRM_DEBUG_MM if DRM=y
         default n
         help
           Choose this option to turn on extra driver debugging that may affect
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index a998c2bce70a..3dea46af9fe6 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -33,19 +33,22 @@ i915-y += i915_cmd_parser.o \
 	  i915_gem_dmabuf.o \
 	  i915_gem_evict.o \
 	  i915_gem_execbuffer.o \
-	  i915_gem_fence.o \
+	  i915_gem_fence_reg.o \
 	  i915_gem_gtt.o \
+	  i915_gem_internal.o \
 	  i915_gem.o \
 	  i915_gem_render_state.o \
 	  i915_gem_request.o \
 	  i915_gem_shrinker.o \
 	  i915_gem_stolen.o \
 	  i915_gem_tiling.o \
+	  i915_gem_timeline.o \
 	  i915_gem_userptr.o \
-	  i915_gpu_error.o \
 	  i915_trace_points.o \
+	  i915_vma.o \
 	  intel_breadcrumbs.o \
 	  intel_engine_cs.o \
+	  intel_hangcheck.o \
 	  intel_lrc.o \
 	  intel_mocs.o \
 	  intel_ringbuffer.o \
@@ -102,11 +105,15 @@ i915-y += dvo_ch7017.o \
 	  intel_dvo.o \
 	  intel_hdmi.o \
 	  intel_i2c.o \
+	  intel_lspcon.o \
 	  intel_lvds.o \
 	  intel_panel.o \
 	  intel_sdvo.o \
 	  intel_tv.o
 
+# Post-mortem debug and GPU hang state capture
+i915-$(CONFIG_DRM_I915_CAPTURE_ERROR) += i915_gpu_error.o
+
 # virtual gpu code
 i915-y += i915_vgpu.o
 
diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile
index d0f21a6ad60d..8a46a7f31d53 100644
--- a/drivers/gpu/drm/i915/gvt/Makefile
+++ b/drivers/gpu/drm/i915/gvt/Makefile
@@ -1,5 +1,10 @@
 GVT_DIR := gvt
-GVT_SOURCE := gvt.o
+GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \
+	interrupt.o gtt.o cfg_space.o opregion.o mmio.o display.o edid.o \
+	execlist.o scheduler.o sched_policy.o render.o cmd_parser.o
 
-ccflags-y                      += -I$(src) -I$(src)/$(GVT_DIR) -Wall
-i915-y			       += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE))
+ccflags-y				+= -I$(src) -I$(src)/$(GVT_DIR) -Wall
+i915-y					+= $(addprefix $(GVT_DIR)/, $(GVT_SOURCE))
+
+CFLAGS_kvmgt.o				:= -Wno-unused-function
+obj-$(CONFIG_DRM_I915_GVT_KVMGT)	+= $(GVT_DIR)/kvmgt.o
diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c
new file mode 100644
index 000000000000..0d41ebc4aea6
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c
@@ -0,0 +1,352 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Dexuan Cui
+ *
+ * Contributors:
+ *    Pei Zhang <pei.zhang@intel.com>
+ *    Min He <min.he@intel.com>
+ *    Niu Bing <bing.niu@intel.com>
+ *    Yulei Zhang <yulei.zhang@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+#define MB_TO_BYTES(mb) ((mb) << 20ULL)
+#define BYTES_TO_MB(b) ((b) >> 20ULL)
+
+#define HOST_LOW_GM_SIZE MB_TO_BYTES(128)
+#define HOST_HIGH_GM_SIZE MB_TO_BYTES(384)
+#define HOST_FENCE 4
+
+static int alloc_gm(struct intel_vgpu *vgpu, bool high_gm)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	u32 alloc_flag, search_flag;
+	u64 start, end, size;
+	struct drm_mm_node *node;
+	int retried = 0;
+	int ret;
+
+	if (high_gm) {
+		search_flag = DRM_MM_SEARCH_BELOW;
+		alloc_flag = DRM_MM_CREATE_TOP;
+		node = &vgpu->gm.high_gm_node;
+		size = vgpu_hidden_sz(vgpu);
+		start = gvt_hidden_gmadr_base(gvt);
+		end = gvt_hidden_gmadr_end(gvt);
+	} else {
+		search_flag = DRM_MM_SEARCH_DEFAULT;
+		alloc_flag = DRM_MM_CREATE_DEFAULT;
+		node = &vgpu->gm.low_gm_node;
+		size = vgpu_aperture_sz(vgpu);
+		start = gvt_aperture_gmadr_base(gvt);
+		end = gvt_aperture_gmadr_end(gvt);
+	}
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+search_again:
+	ret = drm_mm_insert_node_in_range_generic(&dev_priv->ggtt.base.mm,
+						  node, size, 4096, 0,
+						  start, end, search_flag,
+						  alloc_flag);
+	if (ret) {
+		ret = i915_gem_evict_something(&dev_priv->ggtt.base,
+					       size, 4096, 0, start, end, 0);
+		if (ret == 0 && ++retried < 3)
+			goto search_again;
+
+		gvt_err("fail to alloc %s gm space from host, retried %d\n",
+				high_gm ? "high" : "low", retried);
+	}
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+	return ret;
+}
+
+static int alloc_vgpu_gm(struct intel_vgpu *vgpu)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	int ret;
+
+	ret = alloc_gm(vgpu, false);
+	if (ret)
+		return ret;
+
+	ret = alloc_gm(vgpu, true);
+	if (ret)
+		goto out_free_aperture;
+
+	gvt_dbg_core("vgpu%d: alloc low GM start %llx size %llx\n", vgpu->id,
+		     vgpu_aperture_offset(vgpu), vgpu_aperture_sz(vgpu));
+
+	gvt_dbg_core("vgpu%d: alloc high GM start %llx size %llx\n", vgpu->id,
+		     vgpu_hidden_offset(vgpu), vgpu_hidden_sz(vgpu));
+
+	return 0;
+out_free_aperture:
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	drm_mm_remove_node(&vgpu->gm.low_gm_node);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+	return ret;
+}
+
+static void free_vgpu_gm(struct intel_vgpu *vgpu)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	drm_mm_remove_node(&vgpu->gm.low_gm_node);
+	drm_mm_remove_node(&vgpu->gm.high_gm_node);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+}
+
+/**
+ * intel_vgpu_write_fence - write fence registers owned by a vGPU
+ * @vgpu: vGPU instance
+ * @fence: vGPU fence register number
+ * @value: Fence register value to be written
+ *
+ * This function is used to write fence registers owned by a vGPU. The vGPU
+ * fence register number will be translated into HW fence register number.
+ *
+ */
+void intel_vgpu_write_fence(struct intel_vgpu *vgpu,
+		u32 fence, u64 value)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	struct drm_i915_fence_reg *reg;
+	i915_reg_t fence_reg_lo, fence_reg_hi;
+
+	assert_rpm_wakelock_held(dev_priv);
+
+	if (WARN_ON(fence > vgpu_fence_sz(vgpu)))
+		return;
+
+	reg = vgpu->fence.regs[fence];
+	if (WARN_ON(!reg))
+		return;
+
+	fence_reg_lo = FENCE_REG_GEN6_LO(reg->id);
+	fence_reg_hi = FENCE_REG_GEN6_HI(reg->id);
+
+	I915_WRITE(fence_reg_lo, 0);
+	POSTING_READ(fence_reg_lo);
+
+	I915_WRITE(fence_reg_hi, upper_32_bits(value));
+	I915_WRITE(fence_reg_lo, lower_32_bits(value));
+	POSTING_READ(fence_reg_lo);
+}
+
+static void free_vgpu_fence(struct intel_vgpu *vgpu)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	struct drm_i915_fence_reg *reg;
+	u32 i;
+
+	if (WARN_ON(!vgpu_fence_sz(vgpu)))
+		return;
+
+	intel_runtime_pm_get(dev_priv);
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	for (i = 0; i < vgpu_fence_sz(vgpu); i++) {
+		reg = vgpu->fence.regs[i];
+		intel_vgpu_write_fence(vgpu, i, 0);
+		list_add_tail(&reg->link,
+			      &dev_priv->mm.fence_list);
+	}
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+
+	intel_runtime_pm_put(dev_priv);
+}
+
+static int alloc_vgpu_fence(struct intel_vgpu *vgpu)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	struct drm_i915_fence_reg *reg;
+	int i;
+	struct list_head *pos, *q;
+
+	intel_runtime_pm_get(dev_priv);
+
+	/* Request fences from host */
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	i = 0;
+	list_for_each_safe(pos, q, &dev_priv->mm.fence_list) {
+		reg = list_entry(pos, struct drm_i915_fence_reg, link);
+		if (reg->pin_count || reg->vma)
+			continue;
+		list_del(pos);
+		vgpu->fence.regs[i] = reg;
+		intel_vgpu_write_fence(vgpu, i, 0);
+		if (++i == vgpu_fence_sz(vgpu))
+			break;
+	}
+	if (i != vgpu_fence_sz(vgpu))
+		goto out_free_fence;
+
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+	intel_runtime_pm_put(dev_priv);
+	return 0;
+out_free_fence:
+	/* Return fences to host, if fail */
+	for (i = 0; i < vgpu_fence_sz(vgpu); i++) {
+		reg = vgpu->fence.regs[i];
+		if (!reg)
+			continue;
+		list_add_tail(&reg->link,
+			      &dev_priv->mm.fence_list);
+	}
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+	intel_runtime_pm_put(dev_priv);
+	return -ENOSPC;
+}
+
+static void free_resource(struct intel_vgpu *vgpu)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+
+	gvt->gm.vgpu_allocated_low_gm_size -= vgpu_aperture_sz(vgpu);
+	gvt->gm.vgpu_allocated_high_gm_size -= vgpu_hidden_sz(vgpu);
+	gvt->fence.vgpu_allocated_fence_num -= vgpu_fence_sz(vgpu);
+}
+
+static int alloc_resource(struct intel_vgpu *vgpu,
+		struct intel_vgpu_creation_params *param)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	unsigned long request, avail, max, taken;
+	const char *item;
+
+	if (!param->low_gm_sz || !param->high_gm_sz || !param->fence_sz) {
+		gvt_err("Invalid vGPU creation params\n");
+		return -EINVAL;
+	}
+
+	item = "low GM space";
+	max = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE;
+	taken = gvt->gm.vgpu_allocated_low_gm_size;
+	avail = max - taken;
+	request = MB_TO_BYTES(param->low_gm_sz);
+
+	if (request > avail)
+		goto no_enough_resource;
+
+	vgpu_aperture_sz(vgpu) = request;
+
+	item = "high GM space";
+	max = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE;
+	taken = gvt->gm.vgpu_allocated_high_gm_size;
+	avail = max - taken;
+	request = MB_TO_BYTES(param->high_gm_sz);
+
+	if (request > avail)
+		goto no_enough_resource;
+
+	vgpu_hidden_sz(vgpu) = request;
+
+	item = "fence";
+	max = gvt_fence_sz(gvt) - HOST_FENCE;
+	taken = gvt->fence.vgpu_allocated_fence_num;
+	avail = max - taken;
+	request = param->fence_sz;
+
+	if (request > avail)
+		goto no_enough_resource;
+
+	vgpu_fence_sz(vgpu) = request;
+
+	gvt->gm.vgpu_allocated_low_gm_size += MB_TO_BYTES(param->low_gm_sz);
+	gvt->gm.vgpu_allocated_high_gm_size += MB_TO_BYTES(param->high_gm_sz);
+	gvt->fence.vgpu_allocated_fence_num += param->fence_sz;
+	return 0;
+
+no_enough_resource:
+	gvt_err("vgpu%d: fail to allocate resource %s\n", vgpu->id, item);
+	gvt_err("vgpu%d: request %luMB avail %luMB max %luMB taken %luMB\n",
+		vgpu->id, BYTES_TO_MB(request), BYTES_TO_MB(avail),
+		BYTES_TO_MB(max), BYTES_TO_MB(taken));
+	return -ENOSPC;
+}
+
+/**
+ * inte_gvt_free_vgpu_resource - free HW resource owned by a vGPU
+ * @vgpu: a vGPU
+ *
+ * This function is used to free the HW resource owned by a vGPU.
+ *
+ */
+void intel_vgpu_free_resource(struct intel_vgpu *vgpu)
+{
+	free_vgpu_gm(vgpu);
+	free_vgpu_fence(vgpu);
+	free_resource(vgpu);
+}
+
+/**
+ * intel_alloc_vgpu_resource - allocate HW resource for a vGPU
+ * @vgpu: vGPU
+ * @param: vGPU creation params
+ *
+ * This function is used to allocate HW resource for a vGPU. User specifies
+ * the resource configuration through the creation params.
+ *
+ * Returns:
+ * zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_alloc_resource(struct intel_vgpu *vgpu,
+		struct intel_vgpu_creation_params *param)
+{
+	int ret;
+
+	ret = alloc_resource(vgpu, param);
+	if (ret)
+		return ret;
+
+	ret = alloc_vgpu_gm(vgpu);
+	if (ret)
+		goto out_free_resource;
+
+	ret = alloc_vgpu_fence(vgpu);
+	if (ret)
+		goto out_free_vgpu_gm;
+
+	return 0;
+
+out_free_vgpu_gm:
+	free_vgpu_gm(vgpu);
+out_free_resource:
+	free_resource(vgpu);
+	return ret;
+}
diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c
new file mode 100644
index 000000000000..db516382a4d4
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/cfg_space.c
@@ -0,0 +1,284 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Jike Song <jike.song@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+enum {
+	INTEL_GVT_PCI_BAR_GTTMMIO = 0,
+	INTEL_GVT_PCI_BAR_APERTURE,
+	INTEL_GVT_PCI_BAR_PIO,
+	INTEL_GVT_PCI_BAR_MAX,
+};
+
+/**
+ * intel_vgpu_emulate_cfg_read - emulate vGPU configuration space read
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset,
+	void *p_data, unsigned int bytes)
+{
+	if (WARN_ON(bytes > 4))
+		return -EINVAL;
+
+	if (WARN_ON(offset + bytes > INTEL_GVT_MAX_CFG_SPACE_SZ))
+		return -EINVAL;
+
+	memcpy(p_data, vgpu_cfg_space(vgpu) + offset, bytes);
+	return 0;
+}
+
+static int map_aperture(struct intel_vgpu *vgpu, bool map)
+{
+	u64 first_gfn, first_mfn;
+	u64 val;
+	int ret;
+
+	if (map == vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked)
+		return 0;
+
+	val = vgpu_cfg_space(vgpu)[PCI_BASE_ADDRESS_2];
+	if (val & PCI_BASE_ADDRESS_MEM_TYPE_64)
+		val = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2);
+	else
+		val = *(u32 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2);
+
+	first_gfn = (val + vgpu_aperture_offset(vgpu)) >> PAGE_SHIFT;
+	first_mfn = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT;
+
+	ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, first_gfn,
+						  first_mfn,
+						  vgpu_aperture_sz(vgpu) >>
+						  PAGE_SHIFT, map);
+	if (ret)
+		return ret;
+
+	vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked = map;
+	return 0;
+}
+
+static int trap_gttmmio(struct intel_vgpu *vgpu, bool trap)
+{
+	u64 start, end;
+	u64 val;
+	int ret;
+
+	if (trap == vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].tracked)
+		return 0;
+
+	val = vgpu_cfg_space(vgpu)[PCI_BASE_ADDRESS_0];
+	if (val & PCI_BASE_ADDRESS_MEM_TYPE_64)
+		start = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_0);
+	else
+		start = *(u32 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_0);
+
+	start &= ~GENMASK(3, 0);
+	end = start + vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].size - 1;
+
+	ret = intel_gvt_hypervisor_set_trap_area(vgpu, start, end, trap);
+	if (ret)
+		return ret;
+
+	vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].tracked = trap;
+	return 0;
+}
+
+static int emulate_pci_command_write(struct intel_vgpu *vgpu,
+	unsigned int offset, void *p_data, unsigned int bytes)
+{
+	u8 old = vgpu_cfg_space(vgpu)[offset];
+	u8 new = *(u8 *)p_data;
+	u8 changed = old ^ new;
+	int ret;
+
+	if (!(changed & PCI_COMMAND_MEMORY))
+		return 0;
+
+	if (old & PCI_COMMAND_MEMORY) {
+		ret = trap_gttmmio(vgpu, false);
+		if (ret)
+			return ret;
+		ret = map_aperture(vgpu, false);
+		if (ret)
+			return ret;
+	} else {
+		ret = trap_gttmmio(vgpu, true);
+		if (ret)
+			return ret;
+		ret = map_aperture(vgpu, true);
+		if (ret)
+			return ret;
+	}
+
+	memcpy(vgpu_cfg_space(vgpu) + offset, p_data, bytes);
+	return 0;
+}
+
+static int emulate_pci_bar_write(struct intel_vgpu *vgpu, unsigned int offset,
+	void *p_data, unsigned int bytes)
+{
+	unsigned int bar_index =
+		(rounddown(offset, 8) % PCI_BASE_ADDRESS_0) / 8;
+	u32 new = *(u32 *)(p_data);
+	bool lo = IS_ALIGNED(offset, 8);
+	u64 size;
+	int ret = 0;
+	bool mmio_enabled =
+		vgpu_cfg_space(vgpu)[PCI_COMMAND] & PCI_COMMAND_MEMORY;
+
+	if (WARN_ON(bar_index >= INTEL_GVT_PCI_BAR_MAX))
+		return -EINVAL;
+
+	if (new == 0xffffffff) {
+		/*
+		 * Power-up software can determine how much address
+		 * space the device requires by writing a value of
+		 * all 1's to the register and then reading the value
+		 * back. The device will return 0's in all don't-care
+		 * address bits.
+		 */
+		size = vgpu->cfg_space.bar[bar_index].size;
+		if (lo) {
+			new = rounddown(new, size);
+		} else {
+			u32 val = vgpu_cfg_space(vgpu)[rounddown(offset, 8)];
+			/* for 32bit mode bar it returns all-0 in upper 32
+			 * bit, for 64bit mode bar it will calculate the
+			 * size with lower 32bit and return the corresponding
+			 * value
+			 */
+			if (val & PCI_BASE_ADDRESS_MEM_TYPE_64)
+				new &= (~(size-1)) >> 32;
+			else
+				new = 0;
+		}
+		/*
+		 * Unmapp & untrap the BAR, since guest hasn't configured a
+		 * valid GPA
+		 */
+		switch (bar_index) {
+		case INTEL_GVT_PCI_BAR_GTTMMIO:
+			ret = trap_gttmmio(vgpu, false);
+			break;
+		case INTEL_GVT_PCI_BAR_APERTURE:
+			ret = map_aperture(vgpu, false);
+			break;
+		}
+		intel_vgpu_write_pci_bar(vgpu, offset, new, lo);
+	} else {
+		/*
+		 * Unmapp & untrap the old BAR first, since guest has
+		 * re-configured the BAR
+		 */
+		switch (bar_index) {
+		case INTEL_GVT_PCI_BAR_GTTMMIO:
+			ret = trap_gttmmio(vgpu, false);
+			break;
+		case INTEL_GVT_PCI_BAR_APERTURE:
+			ret = map_aperture(vgpu, false);
+			break;
+		}
+		intel_vgpu_write_pci_bar(vgpu, offset, new, lo);
+		/* Track the new BAR */
+		if (mmio_enabled) {
+			switch (bar_index) {
+			case INTEL_GVT_PCI_BAR_GTTMMIO:
+				ret = trap_gttmmio(vgpu, true);
+				break;
+			case INTEL_GVT_PCI_BAR_APERTURE:
+				ret = map_aperture(vgpu, true);
+				break;
+			}
+		}
+	}
+	return ret;
+}
+
+/**
+ * intel_vgpu_emulate_cfg_read - emulate vGPU configuration space write
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset,
+	void *p_data, unsigned int bytes)
+{
+	int ret;
+
+	if (WARN_ON(bytes > 4))
+		return -EINVAL;
+
+	if (WARN_ON(offset + bytes >= INTEL_GVT_MAX_CFG_SPACE_SZ))
+		return -EINVAL;
+
+	/* First check if it's PCI_COMMAND */
+	if (IS_ALIGNED(offset, 2) && offset == PCI_COMMAND) {
+		if (WARN_ON(bytes > 2))
+			return -EINVAL;
+		return emulate_pci_command_write(vgpu, offset, p_data, bytes);
+	}
+
+	switch (rounddown(offset, 4)) {
+	case PCI_BASE_ADDRESS_0:
+	case PCI_BASE_ADDRESS_1:
+	case PCI_BASE_ADDRESS_2:
+	case PCI_BASE_ADDRESS_3:
+		if (WARN_ON(!IS_ALIGNED(offset, 4)))
+			return -EINVAL;
+		return emulate_pci_bar_write(vgpu, offset, p_data, bytes);
+
+	case INTEL_GVT_PCI_SWSCI:
+		if (WARN_ON(!IS_ALIGNED(offset, 4)))
+			return -EINVAL;
+		ret = intel_vgpu_emulate_opregion_request(vgpu, *(u32 *)p_data);
+		if (ret)
+			return ret;
+		break;
+
+	case INTEL_GVT_PCI_OPREGION:
+		if (WARN_ON(!IS_ALIGNED(offset, 4)))
+			return -EINVAL;
+		ret = intel_vgpu_init_opregion(vgpu, *(u32 *)p_data);
+		if (ret)
+			return ret;
+
+		memcpy(vgpu_cfg_space(vgpu) + offset, p_data, bytes);
+		break;
+	default:
+		memcpy(vgpu_cfg_space(vgpu) + offset, p_data, bytes);
+		break;
+	}
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
new file mode 100644
index 000000000000..d26a092c70e8
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -0,0 +1,2831 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Yulei Zhang <yulei.zhang@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include <linux/slab.h>
+#include "i915_drv.h"
+#include "gvt.h"
+#include "i915_pvinfo.h"
+#include "trace.h"
+
+#define INVALID_OP    (~0U)
+
+#define OP_LEN_MI           9
+#define OP_LEN_2D           10
+#define OP_LEN_3D_MEDIA     16
+#define OP_LEN_MFX_VC       16
+#define OP_LEN_VEBOX	    16
+
+#define CMD_TYPE(cmd)	(((cmd) >> 29) & 7)
+
+struct sub_op_bits {
+	int hi;
+	int low;
+};
+struct decode_info {
+	char *name;
+	int op_len;
+	int nr_sub_op;
+	struct sub_op_bits *sub_op;
+};
+
+#define   MAX_CMD_BUDGET			0x7fffffff
+#define   MI_WAIT_FOR_PLANE_C_FLIP_PENDING      (1<<15)
+#define   MI_WAIT_FOR_PLANE_B_FLIP_PENDING      (1<<9)
+#define   MI_WAIT_FOR_PLANE_A_FLIP_PENDING      (1<<1)
+
+#define   MI_WAIT_FOR_SPRITE_C_FLIP_PENDING      (1<<20)
+#define   MI_WAIT_FOR_SPRITE_B_FLIP_PENDING      (1<<10)
+#define   MI_WAIT_FOR_SPRITE_A_FLIP_PENDING      (1<<2)
+
+/* Render Command Map */
+
+/* MI_* command Opcode (28:23) */
+#define OP_MI_NOOP                          0x0
+#define OP_MI_SET_PREDICATE                 0x1  /* HSW+ */
+#define OP_MI_USER_INTERRUPT                0x2
+#define OP_MI_WAIT_FOR_EVENT                0x3
+#define OP_MI_FLUSH                         0x4
+#define OP_MI_ARB_CHECK                     0x5
+#define OP_MI_RS_CONTROL                    0x6  /* HSW+ */
+#define OP_MI_REPORT_HEAD                   0x7
+#define OP_MI_ARB_ON_OFF                    0x8
+#define OP_MI_URB_ATOMIC_ALLOC              0x9  /* HSW+ */
+#define OP_MI_BATCH_BUFFER_END              0xA
+#define OP_MI_SUSPEND_FLUSH                 0xB
+#define OP_MI_PREDICATE                     0xC  /* IVB+ */
+#define OP_MI_TOPOLOGY_FILTER               0xD  /* IVB+ */
+#define OP_MI_SET_APPID                     0xE  /* IVB+ */
+#define OP_MI_RS_CONTEXT                    0xF  /* HSW+ */
+#define OP_MI_LOAD_SCAN_LINES_INCL          0x12 /* HSW+ */
+#define OP_MI_DISPLAY_FLIP                  0x14
+#define OP_MI_SEMAPHORE_MBOX                0x16
+#define OP_MI_SET_CONTEXT                   0x18
+#define OP_MI_MATH                          0x1A
+#define OP_MI_URB_CLEAR                     0x19
+#define OP_MI_SEMAPHORE_SIGNAL		    0x1B  /* BDW+ */
+#define OP_MI_SEMAPHORE_WAIT		    0x1C  /* BDW+ */
+
+#define OP_MI_STORE_DATA_IMM                0x20
+#define OP_MI_STORE_DATA_INDEX              0x21
+#define OP_MI_LOAD_REGISTER_IMM             0x22
+#define OP_MI_UPDATE_GTT                    0x23
+#define OP_MI_STORE_REGISTER_MEM            0x24
+#define OP_MI_FLUSH_DW                      0x26
+#define OP_MI_CLFLUSH                       0x27
+#define OP_MI_REPORT_PERF_COUNT             0x28
+#define OP_MI_LOAD_REGISTER_MEM             0x29  /* HSW+ */
+#define OP_MI_LOAD_REGISTER_REG             0x2A  /* HSW+ */
+#define OP_MI_RS_STORE_DATA_IMM             0x2B  /* HSW+ */
+#define OP_MI_LOAD_URB_MEM                  0x2C  /* HSW+ */
+#define OP_MI_STORE_URM_MEM                 0x2D  /* HSW+ */
+#define OP_MI_2E			    0x2E  /* BDW+ */
+#define OP_MI_2F			    0x2F  /* BDW+ */
+#define OP_MI_BATCH_BUFFER_START            0x31
+
+/* Bit definition for dword 0 */
+#define _CMDBIT_BB_START_IN_PPGTT	(1UL << 8)
+
+#define OP_MI_CONDITIONAL_BATCH_BUFFER_END  0x36
+
+#define BATCH_BUFFER_ADDR_MASK ((1UL << 32) - (1U << 2))
+#define BATCH_BUFFER_ADDR_HIGH_MASK ((1UL << 16) - (1U))
+#define BATCH_BUFFER_ADR_SPACE_BIT(x)	(((x) >> 8) & 1U)
+#define BATCH_BUFFER_2ND_LEVEL_BIT(x)   ((x) >> 22 & 1U)
+
+/* 2D command: Opcode (28:22) */
+#define OP_2D(x)    ((2<<7) | x)
+
+#define OP_XY_SETUP_BLT                             OP_2D(0x1)
+#define OP_XY_SETUP_CLIP_BLT                        OP_2D(0x3)
+#define OP_XY_SETUP_MONO_PATTERN_SL_BLT             OP_2D(0x11)
+#define OP_XY_PIXEL_BLT                             OP_2D(0x24)
+#define OP_XY_SCANLINES_BLT                         OP_2D(0x25)
+#define OP_XY_TEXT_BLT                              OP_2D(0x26)
+#define OP_XY_TEXT_IMMEDIATE_BLT                    OP_2D(0x31)
+#define OP_XY_COLOR_BLT                             OP_2D(0x50)
+#define OP_XY_PAT_BLT                               OP_2D(0x51)
+#define OP_XY_MONO_PAT_BLT                          OP_2D(0x52)
+#define OP_XY_SRC_COPY_BLT                          OP_2D(0x53)
+#define OP_XY_MONO_SRC_COPY_BLT                     OP_2D(0x54)
+#define OP_XY_FULL_BLT                              OP_2D(0x55)
+#define OP_XY_FULL_MONO_SRC_BLT                     OP_2D(0x56)
+#define OP_XY_FULL_MONO_PATTERN_BLT                 OP_2D(0x57)
+#define OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT        OP_2D(0x58)
+#define OP_XY_MONO_PAT_FIXED_BLT                    OP_2D(0x59)
+#define OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT           OP_2D(0x71)
+#define OP_XY_PAT_BLT_IMMEDIATE                     OP_2D(0x72)
+#define OP_XY_SRC_COPY_CHROMA_BLT                   OP_2D(0x73)
+#define OP_XY_FULL_IMMEDIATE_PATTERN_BLT            OP_2D(0x74)
+#define OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT   OP_2D(0x75)
+#define OP_XY_PAT_CHROMA_BLT                        OP_2D(0x76)
+#define OP_XY_PAT_CHROMA_BLT_IMMEDIATE              OP_2D(0x77)
+
+/* 3D/Media Command: Pipeline Type(28:27) Opcode(26:24) Sub Opcode(23:16) */
+#define OP_3D_MEDIA(sub_type, opcode, sub_opcode) \
+	((3 << 13) | ((sub_type) << 11) | ((opcode) << 8) | (sub_opcode))
+
+#define OP_STATE_PREFETCH                       OP_3D_MEDIA(0x0, 0x0, 0x03)
+
+#define OP_STATE_BASE_ADDRESS                   OP_3D_MEDIA(0x0, 0x1, 0x01)
+#define OP_STATE_SIP                            OP_3D_MEDIA(0x0, 0x1, 0x02)
+#define OP_3D_MEDIA_0_1_4			OP_3D_MEDIA(0x0, 0x1, 0x04)
+
+#define OP_3DSTATE_VF_STATISTICS_GM45           OP_3D_MEDIA(0x1, 0x0, 0x0B)
+
+#define OP_PIPELINE_SELECT                      OP_3D_MEDIA(0x1, 0x1, 0x04)
+
+#define OP_MEDIA_VFE_STATE                      OP_3D_MEDIA(0x2, 0x0, 0x0)
+#define OP_MEDIA_CURBE_LOAD                     OP_3D_MEDIA(0x2, 0x0, 0x1)
+#define OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD      OP_3D_MEDIA(0x2, 0x0, 0x2)
+#define OP_MEDIA_GATEWAY_STATE                  OP_3D_MEDIA(0x2, 0x0, 0x3)
+#define OP_MEDIA_STATE_FLUSH                    OP_3D_MEDIA(0x2, 0x0, 0x4)
+
+#define OP_MEDIA_OBJECT                         OP_3D_MEDIA(0x2, 0x1, 0x0)
+#define OP_MEDIA_OBJECT_PRT                     OP_3D_MEDIA(0x2, 0x1, 0x2)
+#define OP_MEDIA_OBJECT_WALKER                  OP_3D_MEDIA(0x2, 0x1, 0x3)
+#define OP_GPGPU_WALKER                         OP_3D_MEDIA(0x2, 0x1, 0x5)
+
+#define OP_3DSTATE_CLEAR_PARAMS                 OP_3D_MEDIA(0x3, 0x0, 0x04) /* IVB+ */
+#define OP_3DSTATE_DEPTH_BUFFER                 OP_3D_MEDIA(0x3, 0x0, 0x05) /* IVB+ */
+#define OP_3DSTATE_STENCIL_BUFFER               OP_3D_MEDIA(0x3, 0x0, 0x06) /* IVB+ */
+#define OP_3DSTATE_HIER_DEPTH_BUFFER            OP_3D_MEDIA(0x3, 0x0, 0x07) /* IVB+ */
+#define OP_3DSTATE_VERTEX_BUFFERS               OP_3D_MEDIA(0x3, 0x0, 0x08)
+#define OP_3DSTATE_VERTEX_ELEMENTS              OP_3D_MEDIA(0x3, 0x0, 0x09)
+#define OP_3DSTATE_INDEX_BUFFER                 OP_3D_MEDIA(0x3, 0x0, 0x0A)
+#define OP_3DSTATE_VF_STATISTICS                OP_3D_MEDIA(0x3, 0x0, 0x0B)
+#define OP_3DSTATE_VF                           OP_3D_MEDIA(0x3, 0x0, 0x0C)  /* HSW+ */
+#define OP_3DSTATE_CC_STATE_POINTERS            OP_3D_MEDIA(0x3, 0x0, 0x0E)
+#define OP_3DSTATE_SCISSOR_STATE_POINTERS       OP_3D_MEDIA(0x3, 0x0, 0x0F)
+#define OP_3DSTATE_VS                           OP_3D_MEDIA(0x3, 0x0, 0x10)
+#define OP_3DSTATE_GS                           OP_3D_MEDIA(0x3, 0x0, 0x11)
+#define OP_3DSTATE_CLIP                         OP_3D_MEDIA(0x3, 0x0, 0x12)
+#define OP_3DSTATE_SF                           OP_3D_MEDIA(0x3, 0x0, 0x13)
+#define OP_3DSTATE_WM                           OP_3D_MEDIA(0x3, 0x0, 0x14)
+#define OP_3DSTATE_CONSTANT_VS                  OP_3D_MEDIA(0x3, 0x0, 0x15)
+#define OP_3DSTATE_CONSTANT_GS                  OP_3D_MEDIA(0x3, 0x0, 0x16)
+#define OP_3DSTATE_CONSTANT_PS                  OP_3D_MEDIA(0x3, 0x0, 0x17)
+#define OP_3DSTATE_SAMPLE_MASK                  OP_3D_MEDIA(0x3, 0x0, 0x18)
+#define OP_3DSTATE_CONSTANT_HS                  OP_3D_MEDIA(0x3, 0x0, 0x19) /* IVB+ */
+#define OP_3DSTATE_CONSTANT_DS                  OP_3D_MEDIA(0x3, 0x0, 0x1A) /* IVB+ */
+#define OP_3DSTATE_HS                           OP_3D_MEDIA(0x3, 0x0, 0x1B) /* IVB+ */
+#define OP_3DSTATE_TE                           OP_3D_MEDIA(0x3, 0x0, 0x1C) /* IVB+ */
+#define OP_3DSTATE_DS                           OP_3D_MEDIA(0x3, 0x0, 0x1D) /* IVB+ */
+#define OP_3DSTATE_STREAMOUT                    OP_3D_MEDIA(0x3, 0x0, 0x1E) /* IVB+ */
+#define OP_3DSTATE_SBE                          OP_3D_MEDIA(0x3, 0x0, 0x1F) /* IVB+ */
+#define OP_3DSTATE_PS                           OP_3D_MEDIA(0x3, 0x0, 0x20) /* IVB+ */
+#define OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP OP_3D_MEDIA(0x3, 0x0, 0x21) /* IVB+ */
+#define OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC   OP_3D_MEDIA(0x3, 0x0, 0x23) /* IVB+ */
+#define OP_3DSTATE_BLEND_STATE_POINTERS         OP_3D_MEDIA(0x3, 0x0, 0x24) /* IVB+ */
+#define OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x25) /* IVB+ */
+#define OP_3DSTATE_BINDING_TABLE_POINTERS_VS    OP_3D_MEDIA(0x3, 0x0, 0x26) /* IVB+ */
+#define OP_3DSTATE_BINDING_TABLE_POINTERS_HS    OP_3D_MEDIA(0x3, 0x0, 0x27) /* IVB+ */
+#define OP_3DSTATE_BINDING_TABLE_POINTERS_DS    OP_3D_MEDIA(0x3, 0x0, 0x28) /* IVB+ */
+#define OP_3DSTATE_BINDING_TABLE_POINTERS_GS    OP_3D_MEDIA(0x3, 0x0, 0x29) /* IVB+ */
+#define OP_3DSTATE_BINDING_TABLE_POINTERS_PS    OP_3D_MEDIA(0x3, 0x0, 0x2A) /* IVB+ */
+#define OP_3DSTATE_SAMPLER_STATE_POINTERS_VS    OP_3D_MEDIA(0x3, 0x0, 0x2B) /* IVB+ */
+#define OP_3DSTATE_SAMPLER_STATE_POINTERS_HS    OP_3D_MEDIA(0x3, 0x0, 0x2C) /* IVB+ */
+#define OP_3DSTATE_SAMPLER_STATE_POINTERS_DS    OP_3D_MEDIA(0x3, 0x0, 0x2D) /* IVB+ */
+#define OP_3DSTATE_SAMPLER_STATE_POINTERS_GS    OP_3D_MEDIA(0x3, 0x0, 0x2E) /* IVB+ */
+#define OP_3DSTATE_SAMPLER_STATE_POINTERS_PS    OP_3D_MEDIA(0x3, 0x0, 0x2F) /* IVB+ */
+#define OP_3DSTATE_URB_VS                       OP_3D_MEDIA(0x3, 0x0, 0x30) /* IVB+ */
+#define OP_3DSTATE_URB_HS                       OP_3D_MEDIA(0x3, 0x0, 0x31) /* IVB+ */
+#define OP_3DSTATE_URB_DS                       OP_3D_MEDIA(0x3, 0x0, 0x32) /* IVB+ */
+#define OP_3DSTATE_URB_GS                       OP_3D_MEDIA(0x3, 0x0, 0x33) /* IVB+ */
+#define OP_3DSTATE_GATHER_CONSTANT_VS           OP_3D_MEDIA(0x3, 0x0, 0x34) /* HSW+ */
+#define OP_3DSTATE_GATHER_CONSTANT_GS           OP_3D_MEDIA(0x3, 0x0, 0x35) /* HSW+ */
+#define OP_3DSTATE_GATHER_CONSTANT_HS           OP_3D_MEDIA(0x3, 0x0, 0x36) /* HSW+ */
+#define OP_3DSTATE_GATHER_CONSTANT_DS           OP_3D_MEDIA(0x3, 0x0, 0x37) /* HSW+ */
+#define OP_3DSTATE_GATHER_CONSTANT_PS           OP_3D_MEDIA(0x3, 0x0, 0x38) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANTF_VS             OP_3D_MEDIA(0x3, 0x0, 0x39) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANTF_PS             OP_3D_MEDIA(0x3, 0x0, 0x3A) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANTI_VS             OP_3D_MEDIA(0x3, 0x0, 0x3B) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANTI_PS             OP_3D_MEDIA(0x3, 0x0, 0x3C) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANTB_VS             OP_3D_MEDIA(0x3, 0x0, 0x3D) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANTB_PS             OP_3D_MEDIA(0x3, 0x0, 0x3E) /* HSW+ */
+#define OP_3DSTATE_DX9_LOCAL_VALID_VS           OP_3D_MEDIA(0x3, 0x0, 0x3F) /* HSW+ */
+#define OP_3DSTATE_DX9_LOCAL_VALID_PS           OP_3D_MEDIA(0x3, 0x0, 0x40) /* HSW+ */
+#define OP_3DSTATE_DX9_GENERATE_ACTIVE_VS       OP_3D_MEDIA(0x3, 0x0, 0x41) /* HSW+ */
+#define OP_3DSTATE_DX9_GENERATE_ACTIVE_PS       OP_3D_MEDIA(0x3, 0x0, 0x42) /* HSW+ */
+#define OP_3DSTATE_BINDING_TABLE_EDIT_VS        OP_3D_MEDIA(0x3, 0x0, 0x43) /* HSW+ */
+#define OP_3DSTATE_BINDING_TABLE_EDIT_GS        OP_3D_MEDIA(0x3, 0x0, 0x44) /* HSW+ */
+#define OP_3DSTATE_BINDING_TABLE_EDIT_HS        OP_3D_MEDIA(0x3, 0x0, 0x45) /* HSW+ */
+#define OP_3DSTATE_BINDING_TABLE_EDIT_DS        OP_3D_MEDIA(0x3, 0x0, 0x46) /* HSW+ */
+#define OP_3DSTATE_BINDING_TABLE_EDIT_PS        OP_3D_MEDIA(0x3, 0x0, 0x47) /* HSW+ */
+
+#define OP_3DSTATE_VF_INSTANCING 		OP_3D_MEDIA(0x3, 0x0, 0x49) /* BDW+ */
+#define OP_3DSTATE_VF_SGVS  			OP_3D_MEDIA(0x3, 0x0, 0x4A) /* BDW+ */
+#define OP_3DSTATE_VF_TOPOLOGY   		OP_3D_MEDIA(0x3, 0x0, 0x4B) /* BDW+ */
+#define OP_3DSTATE_WM_CHROMAKEY   		OP_3D_MEDIA(0x3, 0x0, 0x4C) /* BDW+ */
+#define OP_3DSTATE_PS_BLEND   			OP_3D_MEDIA(0x3, 0x0, 0x4D) /* BDW+ */
+#define OP_3DSTATE_WM_DEPTH_STENCIL   		OP_3D_MEDIA(0x3, 0x0, 0x4E) /* BDW+ */
+#define OP_3DSTATE_PS_EXTRA   			OP_3D_MEDIA(0x3, 0x0, 0x4F) /* BDW+ */
+#define OP_3DSTATE_RASTER   			OP_3D_MEDIA(0x3, 0x0, 0x50) /* BDW+ */
+#define OP_3DSTATE_SBE_SWIZ   			OP_3D_MEDIA(0x3, 0x0, 0x51) /* BDW+ */
+#define OP_3DSTATE_WM_HZ_OP   			OP_3D_MEDIA(0x3, 0x0, 0x52) /* BDW+ */
+#define OP_3DSTATE_COMPONENT_PACKING		OP_3D_MEDIA(0x3, 0x0, 0x55) /* SKL+ */
+
+#define OP_3DSTATE_DRAWING_RECTANGLE            OP_3D_MEDIA(0x3, 0x1, 0x00)
+#define OP_3DSTATE_SAMPLER_PALETTE_LOAD0        OP_3D_MEDIA(0x3, 0x1, 0x02)
+#define OP_3DSTATE_CHROMA_KEY                   OP_3D_MEDIA(0x3, 0x1, 0x04)
+#define OP_SNB_3DSTATE_DEPTH_BUFFER             OP_3D_MEDIA(0x3, 0x1, 0x05)
+#define OP_3DSTATE_POLY_STIPPLE_OFFSET          OP_3D_MEDIA(0x3, 0x1, 0x06)
+#define OP_3DSTATE_POLY_STIPPLE_PATTERN         OP_3D_MEDIA(0x3, 0x1, 0x07)
+#define OP_3DSTATE_LINE_STIPPLE                 OP_3D_MEDIA(0x3, 0x1, 0x08)
+#define OP_3DSTATE_AA_LINE_PARAMS               OP_3D_MEDIA(0x3, 0x1, 0x0A)
+#define OP_3DSTATE_GS_SVB_INDEX                 OP_3D_MEDIA(0x3, 0x1, 0x0B)
+#define OP_3DSTATE_SAMPLER_PALETTE_LOAD1        OP_3D_MEDIA(0x3, 0x1, 0x0C)
+#define OP_3DSTATE_MULTISAMPLE_BDW		OP_3D_MEDIA(0x3, 0x0, 0x0D)
+#define OP_SNB_3DSTATE_STENCIL_BUFFER           OP_3D_MEDIA(0x3, 0x1, 0x0E)
+#define OP_SNB_3DSTATE_HIER_DEPTH_BUFFER        OP_3D_MEDIA(0x3, 0x1, 0x0F)
+#define OP_SNB_3DSTATE_CLEAR_PARAMS             OP_3D_MEDIA(0x3, 0x1, 0x10)
+#define OP_3DSTATE_MONOFILTER_SIZE              OP_3D_MEDIA(0x3, 0x1, 0x11)
+#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS       OP_3D_MEDIA(0x3, 0x1, 0x12) /* IVB+ */
+#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS       OP_3D_MEDIA(0x3, 0x1, 0x13) /* IVB+ */
+#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS       OP_3D_MEDIA(0x3, 0x1, 0x14) /* IVB+ */
+#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS       OP_3D_MEDIA(0x3, 0x1, 0x15) /* IVB+ */
+#define OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS       OP_3D_MEDIA(0x3, 0x1, 0x16) /* IVB+ */
+#define OP_3DSTATE_SO_DECL_LIST                 OP_3D_MEDIA(0x3, 0x1, 0x17)
+#define OP_3DSTATE_SO_BUFFER                    OP_3D_MEDIA(0x3, 0x1, 0x18)
+#define OP_3DSTATE_BINDING_TABLE_POOL_ALLOC     OP_3D_MEDIA(0x3, 0x1, 0x19) /* HSW+ */
+#define OP_3DSTATE_GATHER_POOL_ALLOC            OP_3D_MEDIA(0x3, 0x1, 0x1A) /* HSW+ */
+#define OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x1B) /* HSW+ */
+#define OP_3DSTATE_SAMPLE_PATTERN               OP_3D_MEDIA(0x3, 0x1, 0x1C)
+#define OP_PIPE_CONTROL                         OP_3D_MEDIA(0x3, 0x2, 0x00)
+#define OP_3DPRIMITIVE                          OP_3D_MEDIA(0x3, 0x3, 0x00)
+
+/* VCCP Command Parser */
+
+/*
+ * Below MFX and VBE cmd definition is from vaapi intel driver project (BSD License)
+ * git://anongit.freedesktop.org/vaapi/intel-driver
+ * src/i965_defines.h
+ *
+ */
+
+#define OP_MFX(pipeline, op, sub_opa, sub_opb)     \
+	(3 << 13 | \
+	 (pipeline) << 11 | \
+	 (op) << 8 | \
+	 (sub_opa) << 5 | \
+	 (sub_opb))
+
+#define OP_MFX_PIPE_MODE_SELECT                    OP_MFX(2, 0, 0, 0)  /* ALL */
+#define OP_MFX_SURFACE_STATE                       OP_MFX(2, 0, 0, 1)  /* ALL */
+#define OP_MFX_PIPE_BUF_ADDR_STATE                 OP_MFX(2, 0, 0, 2)  /* ALL */
+#define OP_MFX_IND_OBJ_BASE_ADDR_STATE             OP_MFX(2, 0, 0, 3)  /* ALL */
+#define OP_MFX_BSP_BUF_BASE_ADDR_STATE             OP_MFX(2, 0, 0, 4)  /* ALL */
+#define OP_2_0_0_5                                 OP_MFX(2, 0, 0, 5)  /* ALL */
+#define OP_MFX_STATE_POINTER                       OP_MFX(2, 0, 0, 6)  /* ALL */
+#define OP_MFX_QM_STATE                            OP_MFX(2, 0, 0, 7)  /* IVB+ */
+#define OP_MFX_FQM_STATE                           OP_MFX(2, 0, 0, 8)  /* IVB+ */
+#define OP_MFX_PAK_INSERT_OBJECT                   OP_MFX(2, 0, 2, 8)  /* IVB+ */
+#define OP_MFX_STITCH_OBJECT                       OP_MFX(2, 0, 2, 0xA)  /* IVB+ */
+
+#define OP_MFD_IT_OBJECT                           OP_MFX(2, 0, 1, 9) /* ALL */
+
+#define OP_MFX_WAIT                                OP_MFX(1, 0, 0, 0) /* IVB+ */
+#define OP_MFX_AVC_IMG_STATE                       OP_MFX(2, 1, 0, 0) /* ALL */
+#define OP_MFX_AVC_QM_STATE                        OP_MFX(2, 1, 0, 1) /* ALL */
+#define OP_MFX_AVC_DIRECTMODE_STATE                OP_MFX(2, 1, 0, 2) /* ALL */
+#define OP_MFX_AVC_SLICE_STATE                     OP_MFX(2, 1, 0, 3) /* ALL */
+#define OP_MFX_AVC_REF_IDX_STATE                   OP_MFX(2, 1, 0, 4) /* ALL */
+#define OP_MFX_AVC_WEIGHTOFFSET_STATE              OP_MFX(2, 1, 0, 5) /* ALL */
+#define OP_MFD_AVC_PICID_STATE                     OP_MFX(2, 1, 1, 5) /* HSW+ */
+#define OP_MFD_AVC_DPB_STATE			   OP_MFX(2, 1, 1, 6) /* IVB+ */
+#define OP_MFD_AVC_SLICEADDR                       OP_MFX(2, 1, 1, 7) /* IVB+ */
+#define OP_MFD_AVC_BSD_OBJECT                      OP_MFX(2, 1, 1, 8) /* ALL */
+#define OP_MFC_AVC_PAK_OBJECT                      OP_MFX(2, 1, 2, 9) /* ALL */
+
+#define OP_MFX_VC1_PRED_PIPE_STATE                 OP_MFX(2, 2, 0, 1) /* ALL */
+#define OP_MFX_VC1_DIRECTMODE_STATE                OP_MFX(2, 2, 0, 2) /* ALL */
+#define OP_MFD_VC1_SHORT_PIC_STATE                 OP_MFX(2, 2, 1, 0) /* IVB+ */
+#define OP_MFD_VC1_LONG_PIC_STATE                  OP_MFX(2, 2, 1, 1) /* IVB+ */
+#define OP_MFD_VC1_BSD_OBJECT                      OP_MFX(2, 2, 1, 8) /* ALL */
+
+#define OP_MFX_MPEG2_PIC_STATE                     OP_MFX(2, 3, 0, 0) /* ALL */
+#define OP_MFX_MPEG2_QM_STATE                      OP_MFX(2, 3, 0, 1) /* ALL */
+#define OP_MFD_MPEG2_BSD_OBJECT                    OP_MFX(2, 3, 1, 8) /* ALL */
+#define OP_MFC_MPEG2_SLICEGROUP_STATE              OP_MFX(2, 3, 2, 3) /* ALL */
+#define OP_MFC_MPEG2_PAK_OBJECT                    OP_MFX(2, 3, 2, 9) /* ALL */
+
+#define OP_MFX_2_6_0_0                             OP_MFX(2, 6, 0, 0) /* IVB+ */
+#define OP_MFX_2_6_0_8                             OP_MFX(2, 6, 0, 8) /* IVB+ */
+#define OP_MFX_2_6_0_9                             OP_MFX(2, 6, 0, 9) /* IVB+ */
+
+#define OP_MFX_JPEG_PIC_STATE                      OP_MFX(2, 7, 0, 0)
+#define OP_MFX_JPEG_HUFF_TABLE_STATE               OP_MFX(2, 7, 0, 2)
+#define OP_MFD_JPEG_BSD_OBJECT                     OP_MFX(2, 7, 1, 8)
+
+#define OP_VEB(pipeline, op, sub_opa, sub_opb) \
+	(3 << 13 | \
+	 (pipeline) << 11 | \
+	 (op) << 8 | \
+	 (sub_opa) << 5 | \
+	 (sub_opb))
+
+#define OP_VEB_SURFACE_STATE                       OP_VEB(2, 4, 0, 0)
+#define OP_VEB_STATE                               OP_VEB(2, 4, 0, 2)
+#define OP_VEB_DNDI_IECP_STATE                     OP_VEB(2, 4, 0, 3)
+
+struct parser_exec_state;
+
+typedef int (*parser_cmd_handler)(struct parser_exec_state *s);
+
+#define GVT_CMD_HASH_BITS   7
+
+/* which DWords need address fix */
+#define ADDR_FIX_1(x1)			(1 << (x1))
+#define ADDR_FIX_2(x1, x2)		(ADDR_FIX_1(x1) | ADDR_FIX_1(x2))
+#define ADDR_FIX_3(x1, x2, x3)		(ADDR_FIX_1(x1) | ADDR_FIX_2(x2, x3))
+#define ADDR_FIX_4(x1, x2, x3, x4)	(ADDR_FIX_1(x1) | ADDR_FIX_3(x2, x3, x4))
+#define ADDR_FIX_5(x1, x2, x3, x4, x5)  (ADDR_FIX_1(x1) | ADDR_FIX_4(x2, x3, x4, x5))
+
+struct cmd_info {
+	char *name;
+	u32 opcode;
+
+#define F_LEN_MASK	(1U<<0)
+#define F_LEN_CONST  1U
+#define F_LEN_VAR    0U
+
+/*
+ * command has its own ip advance logic
+ * e.g. MI_BATCH_START, MI_BATCH_END
+ */
+#define F_IP_ADVANCE_CUSTOM (1<<1)
+
+#define F_POST_HANDLE	(1<<2)
+	u32 flag;
+
+#define R_RCS	(1 << RCS)
+#define R_VCS1  (1 << VCS)
+#define R_VCS2  (1 << VCS2)
+#define R_VCS	(R_VCS1 | R_VCS2)
+#define R_BCS	(1 << BCS)
+#define R_VECS	(1 << VECS)
+#define R_ALL (R_RCS | R_VCS | R_BCS | R_VECS)
+	/* rings that support this cmd: BLT/RCS/VCS/VECS */
+	uint16_t rings;
+
+	/* devices that support this cmd: SNB/IVB/HSW/... */
+	uint16_t devices;
+
+	/* which DWords are address that need fix up.
+	 * bit 0 means a 32-bit non address operand in command
+	 * bit 1 means address operand, which could be 32-bit
+	 * or 64-bit depending on different architectures.(
+	 * defined by "gmadr_bytes_in_cmd" in intel_gvt.
+	 * No matter the address length, each address only takes
+	 * one bit in the bitmap.
+	 */
+	uint16_t addr_bitmap;
+
+	/* flag == F_LEN_CONST : command length
+	 * flag == F_LEN_VAR : length bias bits
+	 * Note: length is in DWord
+	 */
+	uint8_t	len;
+
+	parser_cmd_handler handler;
+};
+
+struct cmd_entry {
+	struct hlist_node hlist;
+	struct cmd_info *info;
+};
+
+enum {
+	RING_BUFFER_INSTRUCTION,
+	BATCH_BUFFER_INSTRUCTION,
+	BATCH_BUFFER_2ND_LEVEL,
+};
+
+enum {
+	GTT_BUFFER,
+	PPGTT_BUFFER
+};
+
+struct parser_exec_state {
+	struct intel_vgpu *vgpu;
+	int ring_id;
+
+	int buf_type;
+
+	/* batch buffer address type */
+	int buf_addr_type;
+
+	/* graphics memory address of ring buffer start */
+	unsigned long ring_start;
+	unsigned long ring_size;
+	unsigned long ring_head;
+	unsigned long ring_tail;
+
+	/* instruction graphics memory address */
+	unsigned long ip_gma;
+
+	/* mapped va of the instr_gma */
+	void *ip_va;
+	void *rb_va;
+
+	void *ret_bb_va;
+	/* next instruction when return from  batch buffer to ring buffer */
+	unsigned long ret_ip_gma_ring;
+
+	/* next instruction when return from 2nd batch buffer to batch buffer */
+	unsigned long ret_ip_gma_bb;
+
+	/* batch buffer address type (GTT or PPGTT)
+	 * used when ret from 2nd level batch buffer
+	 */
+	int saved_buf_addr_type;
+
+	struct cmd_info *info;
+
+	struct intel_vgpu_workload *workload;
+};
+
+#define gmadr_dw_number(s)	\
+	(s->vgpu->gvt->device_info.gmadr_bytes_in_cmd >> 2)
+
+static unsigned long bypass_scan_mask = 0;
+static bool bypass_batch_buffer_scan = true;
+
+/* ring ALL, type = 0 */
+static struct sub_op_bits sub_op_mi[] = {
+	{31, 29},
+	{28, 23},
+};
+
+static struct decode_info decode_info_mi = {
+	"MI",
+	OP_LEN_MI,
+	ARRAY_SIZE(sub_op_mi),
+	sub_op_mi,
+};
+
+/* ring RCS, command type 2 */
+static struct sub_op_bits sub_op_2d[] = {
+	{31, 29},
+	{28, 22},
+};
+
+static struct decode_info decode_info_2d = {
+	"2D",
+	OP_LEN_2D,
+	ARRAY_SIZE(sub_op_2d),
+	sub_op_2d,
+};
+
+/* ring RCS, command type 3 */
+static struct sub_op_bits sub_op_3d_media[] = {
+	{31, 29},
+	{28, 27},
+	{26, 24},
+	{23, 16},
+};
+
+static struct decode_info decode_info_3d_media = {
+	"3D_Media",
+	OP_LEN_3D_MEDIA,
+	ARRAY_SIZE(sub_op_3d_media),
+	sub_op_3d_media,
+};
+
+/* ring VCS, command type 3 */
+static struct sub_op_bits sub_op_mfx_vc[] = {
+	{31, 29},
+	{28, 27},
+	{26, 24},
+	{23, 21},
+	{20, 16},
+};
+
+static struct decode_info decode_info_mfx_vc = {
+	"MFX_VC",
+	OP_LEN_MFX_VC,
+	ARRAY_SIZE(sub_op_mfx_vc),
+	sub_op_mfx_vc,
+};
+
+/* ring VECS, command type 3 */
+static struct sub_op_bits sub_op_vebox[] = {
+	{31, 29},
+	{28, 27},
+	{26, 24},
+	{23, 21},
+	{20, 16},
+};
+
+static struct decode_info decode_info_vebox = {
+	"VEBOX",
+	OP_LEN_VEBOX,
+	ARRAY_SIZE(sub_op_vebox),
+	sub_op_vebox,
+};
+
+static struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
+	[RCS] = {
+		&decode_info_mi,
+		NULL,
+		NULL,
+		&decode_info_3d_media,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+	},
+
+	[VCS] = {
+		&decode_info_mi,
+		NULL,
+		NULL,
+		&decode_info_mfx_vc,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+	},
+
+	[BCS] = {
+		&decode_info_mi,
+		NULL,
+		&decode_info_2d,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+	},
+
+	[VECS] = {
+		&decode_info_mi,
+		NULL,
+		NULL,
+		&decode_info_vebox,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+	},
+
+	[VCS2] = {
+		&decode_info_mi,
+		NULL,
+		NULL,
+		&decode_info_mfx_vc,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+	},
+};
+
+static inline u32 get_opcode(u32 cmd, int ring_id)
+{
+	struct decode_info *d_info;
+
+	if (ring_id >= I915_NUM_ENGINES)
+		return INVALID_OP;
+
+	d_info = ring_decode_info[ring_id][CMD_TYPE(cmd)];
+	if (d_info == NULL)
+		return INVALID_OP;
+
+	return cmd >> (32 - d_info->op_len);
+}
+
+static inline struct cmd_info *find_cmd_entry(struct intel_gvt *gvt,
+		unsigned int opcode, int ring_id)
+{
+	struct cmd_entry *e;
+
+	hash_for_each_possible(gvt->cmd_table, e, hlist, opcode) {
+		if ((opcode == e->info->opcode) &&
+				(e->info->rings & (1 << ring_id)))
+			return e->info;
+	}
+	return NULL;
+}
+
+static inline struct cmd_info *get_cmd_info(struct intel_gvt *gvt,
+		u32 cmd, int ring_id)
+{
+	u32 opcode;
+
+	opcode = get_opcode(cmd, ring_id);
+	if (opcode == INVALID_OP)
+		return NULL;
+
+	return find_cmd_entry(gvt, opcode, ring_id);
+}
+
+static inline u32 sub_op_val(u32 cmd, u32 hi, u32 low)
+{
+	return (cmd >> low) & ((1U << (hi - low + 1)) - 1);
+}
+
+static inline void print_opcode(u32 cmd, int ring_id)
+{
+	struct decode_info *d_info;
+	int i;
+
+	if (ring_id >= I915_NUM_ENGINES)
+		return;
+
+	d_info = ring_decode_info[ring_id][CMD_TYPE(cmd)];
+	if (d_info == NULL)
+		return;
+
+	gvt_err("opcode=0x%x %s sub_ops:",
+			cmd >> (32 - d_info->op_len), d_info->name);
+
+	for (i = 0; i < d_info->nr_sub_op; i++)
+		pr_err("0x%x ", sub_op_val(cmd, d_info->sub_op[i].hi,
+					d_info->sub_op[i].low));
+
+	pr_err("\n");
+}
+
+static inline u32 *cmd_ptr(struct parser_exec_state *s, int index)
+{
+	return s->ip_va + (index << 2);
+}
+
+static inline u32 cmd_val(struct parser_exec_state *s, int index)
+{
+	return *cmd_ptr(s, index);
+}
+
+static void parser_exec_state_dump(struct parser_exec_state *s)
+{
+	int cnt = 0;
+	int i;
+
+	gvt_err("  vgpu%d RING%d: ring_start(%08lx) ring_end(%08lx)"
+			" ring_head(%08lx) ring_tail(%08lx)\n", s->vgpu->id,
+			s->ring_id, s->ring_start, s->ring_start + s->ring_size,
+			s->ring_head, s->ring_tail);
+
+	gvt_err("  %s %s ip_gma(%08lx) ",
+			s->buf_type == RING_BUFFER_INSTRUCTION ?
+			"RING_BUFFER" : "BATCH_BUFFER",
+			s->buf_addr_type == GTT_BUFFER ?
+			"GTT" : "PPGTT", s->ip_gma);
+
+	if (s->ip_va == NULL) {
+		gvt_err(" ip_va(NULL)");
+		return;
+	}
+
+	gvt_err("  ip_va=%p: %08x %08x %08x %08x\n",
+			s->ip_va, cmd_val(s, 0), cmd_val(s, 1),
+			cmd_val(s, 2), cmd_val(s, 3));
+
+	print_opcode(cmd_val(s, 0), s->ring_id);
+
+	/* print the whole page to trace */
+	pr_err("    ip_va=%p: %08x %08x %08x %08x\n",
+			s->ip_va, cmd_val(s, 0), cmd_val(s, 1),
+			cmd_val(s, 2), cmd_val(s, 3));
+
+	s->ip_va = (u32 *)((((u64)s->ip_va) >> 12) << 12);
+
+	while (cnt < 1024) {
+		pr_err("ip_va=%p: ", s->ip_va);
+		for (i = 0; i < 8; i++)
+			pr_err("%08x ", cmd_val(s, i));
+		pr_err("\n");
+
+		s->ip_va += 8 * sizeof(u32);
+		cnt += 8;
+	}
+}
+
+static inline void update_ip_va(struct parser_exec_state *s)
+{
+	unsigned long len = 0;
+
+	if (WARN_ON(s->ring_head == s->ring_tail))
+		return;
+
+	if (s->buf_type == RING_BUFFER_INSTRUCTION) {
+		unsigned long ring_top = s->ring_start + s->ring_size;
+
+		if (s->ring_head > s->ring_tail) {
+			if (s->ip_gma >= s->ring_head && s->ip_gma < ring_top)
+				len = (s->ip_gma - s->ring_head);
+			else if (s->ip_gma >= s->ring_start &&
+					s->ip_gma <= s->ring_tail)
+				len = (ring_top - s->ring_head) +
+					(s->ip_gma - s->ring_start);
+		} else
+			len = (s->ip_gma - s->ring_head);
+
+		s->ip_va = s->rb_va + len;
+	} else {/* shadow batch buffer */
+		s->ip_va = s->ret_bb_va;
+	}
+}
+
+static inline int ip_gma_set(struct parser_exec_state *s,
+		unsigned long ip_gma)
+{
+	WARN_ON(!IS_ALIGNED(ip_gma, 4));
+
+	s->ip_gma = ip_gma;
+	update_ip_va(s);
+	return 0;
+}
+
+static inline int ip_gma_advance(struct parser_exec_state *s,
+		unsigned int dw_len)
+{
+	s->ip_gma += (dw_len << 2);
+
+	if (s->buf_type == RING_BUFFER_INSTRUCTION) {
+		if (s->ip_gma >= s->ring_start + s->ring_size)
+			s->ip_gma -= s->ring_size;
+		update_ip_va(s);
+	} else {
+		s->ip_va += (dw_len << 2);
+	}
+
+	return 0;
+}
+
+static inline int get_cmd_length(struct cmd_info *info, u32 cmd)
+{
+	if ((info->flag & F_LEN_MASK) == F_LEN_CONST)
+		return info->len;
+	else
+		return (cmd & ((1U << info->len) - 1)) + 2;
+	return 0;
+}
+
+static inline int cmd_length(struct parser_exec_state *s)
+{
+	return get_cmd_length(s->info, cmd_val(s, 0));
+}
+
+/* do not remove this, some platform may need clflush here */
+#define patch_value(s, addr, val) do { \
+	*addr = val; \
+} while (0)
+
+static bool is_shadowed_mmio(unsigned int offset)
+{
+	bool ret = false;
+
+	if ((offset == 0x2168) || /*BB current head register UDW */
+	    (offset == 0x2140) || /*BB current header register */
+	    (offset == 0x211c) || /*second BB header register UDW */
+	    (offset == 0x2114)) { /*second BB header register UDW */
+		ret = true;
+	}
+	return ret;
+}
+
+static int cmd_reg_handler(struct parser_exec_state *s,
+	unsigned int offset, unsigned int index, char *cmd)
+{
+	struct intel_vgpu *vgpu = s->vgpu;
+	struct intel_gvt *gvt = vgpu->gvt;
+
+	if (offset + 4 > gvt->device_info.mmio_size) {
+		gvt_err("%s access to (%x) outside of MMIO range\n",
+				cmd, offset);
+		return -EINVAL;
+	}
+
+	if (!intel_gvt_mmio_is_cmd_access(gvt, offset)) {
+		gvt_err("vgpu%d: %s access to non-render register (%x)\n",
+				s->vgpu->id, cmd, offset);
+		return 0;
+	}
+
+	if (is_shadowed_mmio(offset)) {
+		gvt_err("vgpu%d: found access of shadowed MMIO %x\n",
+				s->vgpu->id, offset);
+		return 0;
+	}
+
+	if (offset == i915_mmio_reg_offset(DERRMR) ||
+		offset == i915_mmio_reg_offset(FORCEWAKE_MT)) {
+		/* Writing to HW VGT_PVINFO_PAGE offset will be discarded */
+		patch_value(s, cmd_ptr(s, index), VGT_PVINFO_PAGE);
+	}
+
+	/* TODO: Update the global mask if this MMIO is a masked-MMIO */
+	intel_gvt_mmio_set_cmd_accessed(gvt, offset);
+	return 0;
+}
+
+#define cmd_reg(s, i) \
+	(cmd_val(s, i) & GENMASK(22, 2))
+
+#define cmd_reg_inhibit(s, i) \
+	(cmd_val(s, i) & GENMASK(22, 18))
+
+#define cmd_gma(s, i) \
+	(cmd_val(s, i) & GENMASK(31, 2))
+
+#define cmd_gma_hi(s, i) \
+	(cmd_val(s, i) & GENMASK(15, 0))
+
+static int cmd_handler_lri(struct parser_exec_state *s)
+{
+	int i, ret = 0;
+	int cmd_len = cmd_length(s);
+	struct intel_gvt *gvt = s->vgpu->gvt;
+
+	for (i = 1; i < cmd_len; i += 2) {
+		if (IS_BROADWELL(gvt->dev_priv) &&
+				(s->ring_id != RCS)) {
+			if (s->ring_id == BCS &&
+					cmd_reg(s, i) ==
+					i915_mmio_reg_offset(DERRMR))
+				ret |= 0;
+			else
+				ret |= (cmd_reg_inhibit(s, i)) ? -EINVAL : 0;
+		}
+		if (ret)
+			break;
+		ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lri");
+	}
+	return ret;
+}
+
+static int cmd_handler_lrr(struct parser_exec_state *s)
+{
+	int i, ret = 0;
+	int cmd_len = cmd_length(s);
+
+	for (i = 1; i < cmd_len; i += 2) {
+		if (IS_BROADWELL(s->vgpu->gvt->dev_priv))
+			ret |= ((cmd_reg_inhibit(s, i) ||
+					(cmd_reg_inhibit(s, i + 1)))) ?
+				-EINVAL : 0;
+		if (ret)
+			break;
+		ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrr-src");
+		ret |= cmd_reg_handler(s, cmd_reg(s, i + 1), i, "lrr-dst");
+	}
+	return ret;
+}
+
+static inline int cmd_address_audit(struct parser_exec_state *s,
+		unsigned long guest_gma, int op_size, bool index_mode);
+
+static int cmd_handler_lrm(struct parser_exec_state *s)
+{
+	struct intel_gvt *gvt = s->vgpu->gvt;
+	int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd;
+	unsigned long gma;
+	int i, ret = 0;
+	int cmd_len = cmd_length(s);
+
+	for (i = 1; i < cmd_len;) {
+		if (IS_BROADWELL(gvt->dev_priv))
+			ret |= (cmd_reg_inhibit(s, i)) ? -EINVAL : 0;
+		if (ret)
+			break;
+		ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrm");
+		if (cmd_val(s, 0) & (1 << 22)) {
+			gma = cmd_gma(s, i + 1);
+			if (gmadr_bytes == 8)
+				gma |= (cmd_gma_hi(s, i + 2)) << 32;
+			ret |= cmd_address_audit(s, gma, sizeof(u32), false);
+		}
+		i += gmadr_dw_number(s) + 1;
+	}
+	return ret;
+}
+
+static int cmd_handler_srm(struct parser_exec_state *s)
+{
+	int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+	unsigned long gma;
+	int i, ret = 0;
+	int cmd_len = cmd_length(s);
+
+	for (i = 1; i < cmd_len;) {
+		ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "srm");
+		if (cmd_val(s, 0) & (1 << 22)) {
+			gma = cmd_gma(s, i + 1);
+			if (gmadr_bytes == 8)
+				gma |= (cmd_gma_hi(s, i + 2)) << 32;
+			ret |= cmd_address_audit(s, gma, sizeof(u32), false);
+		}
+		i += gmadr_dw_number(s) + 1;
+	}
+	return ret;
+}
+
+struct cmd_interrupt_event {
+	int pipe_control_notify;
+	int mi_flush_dw;
+	int mi_user_interrupt;
+};
+
+static struct cmd_interrupt_event cmd_interrupt_events[] = {
+	[RCS] = {
+		.pipe_control_notify = RCS_PIPE_CONTROL,
+		.mi_flush_dw = INTEL_GVT_EVENT_RESERVED,
+		.mi_user_interrupt = RCS_MI_USER_INTERRUPT,
+	},
+	[BCS] = {
+		.pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
+		.mi_flush_dw = BCS_MI_FLUSH_DW,
+		.mi_user_interrupt = BCS_MI_USER_INTERRUPT,
+	},
+	[VCS] = {
+		.pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
+		.mi_flush_dw = VCS_MI_FLUSH_DW,
+		.mi_user_interrupt = VCS_MI_USER_INTERRUPT,
+	},
+	[VCS2] = {
+		.pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
+		.mi_flush_dw = VCS2_MI_FLUSH_DW,
+		.mi_user_interrupt = VCS2_MI_USER_INTERRUPT,
+	},
+	[VECS] = {
+		.pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
+		.mi_flush_dw = VECS_MI_FLUSH_DW,
+		.mi_user_interrupt = VECS_MI_USER_INTERRUPT,
+	},
+};
+
+static int cmd_handler_pipe_control(struct parser_exec_state *s)
+{
+	int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+	unsigned long gma;
+	bool index_mode = false;
+	unsigned int post_sync;
+	int ret = 0;
+
+	post_sync = (cmd_val(s, 1) & PIPE_CONTROL_POST_SYNC_OP_MASK) >> 14;
+
+	/* LRI post sync */
+	if (cmd_val(s, 1) & PIPE_CONTROL_MMIO_WRITE)
+		ret = cmd_reg_handler(s, cmd_reg(s, 2), 1, "pipe_ctrl");
+	/* post sync */
+	else if (post_sync) {
+		if (post_sync == 2)
+			ret = cmd_reg_handler(s, 0x2350, 1, "pipe_ctrl");
+		else if (post_sync == 3)
+			ret = cmd_reg_handler(s, 0x2358, 1, "pipe_ctrl");
+		else if (post_sync == 1) {
+			/* check ggtt*/
+			if ((cmd_val(s, 2) & (1 << 2))) {
+				gma = cmd_val(s, 2) & GENMASK(31, 3);
+				if (gmadr_bytes == 8)
+					gma |= (cmd_gma_hi(s, 3)) << 32;
+				/* Store Data Index */
+				if (cmd_val(s, 1) & (1 << 21))
+					index_mode = true;
+				ret |= cmd_address_audit(s, gma, sizeof(u64),
+						index_mode);
+			}
+		}
+	}
+
+	if (ret)
+		return ret;
+
+	if (cmd_val(s, 1) & PIPE_CONTROL_NOTIFY)
+		set_bit(cmd_interrupt_events[s->ring_id].pipe_control_notify,
+				s->workload->pending_events);
+	return 0;
+}
+
+static int cmd_handler_mi_user_interrupt(struct parser_exec_state *s)
+{
+	set_bit(cmd_interrupt_events[s->ring_id].mi_user_interrupt,
+			s->workload->pending_events);
+	return 0;
+}
+
+static int cmd_advance_default(struct parser_exec_state *s)
+{
+	return ip_gma_advance(s, cmd_length(s));
+}
+
+static int cmd_handler_mi_batch_buffer_end(struct parser_exec_state *s)
+{
+	int ret;
+
+	if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) {
+		s->buf_type = BATCH_BUFFER_INSTRUCTION;
+		ret = ip_gma_set(s, s->ret_ip_gma_bb);
+		s->buf_addr_type = s->saved_buf_addr_type;
+	} else {
+		s->buf_type = RING_BUFFER_INSTRUCTION;
+		s->buf_addr_type = GTT_BUFFER;
+		if (s->ret_ip_gma_ring >= s->ring_start + s->ring_size)
+			s->ret_ip_gma_ring -= s->ring_size;
+		ret = ip_gma_set(s, s->ret_ip_gma_ring);
+	}
+	return ret;
+}
+
+struct mi_display_flip_command_info {
+	int pipe;
+	int plane;
+	int event;
+	i915_reg_t stride_reg;
+	i915_reg_t ctrl_reg;
+	i915_reg_t surf_reg;
+	u64 stride_val;
+	u64 tile_val;
+	u64 surf_val;
+	bool async_flip;
+};
+
+struct plane_code_mapping {
+	int pipe;
+	int plane;
+	int event;
+};
+
+static int gen8_decode_mi_display_flip(struct parser_exec_state *s,
+		struct mi_display_flip_command_info *info)
+{
+	struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+	struct plane_code_mapping gen8_plane_code[] = {
+		[0] = {PIPE_A, PLANE_A, PRIMARY_A_FLIP_DONE},
+		[1] = {PIPE_B, PLANE_A, PRIMARY_B_FLIP_DONE},
+		[2] = {PIPE_A, PLANE_B, SPRITE_A_FLIP_DONE},
+		[3] = {PIPE_B, PLANE_B, SPRITE_B_FLIP_DONE},
+		[4] = {PIPE_C, PLANE_A, PRIMARY_C_FLIP_DONE},
+		[5] = {PIPE_C, PLANE_B, SPRITE_C_FLIP_DONE},
+	};
+	u32 dword0, dword1, dword2;
+	u32 v;
+
+	dword0 = cmd_val(s, 0);
+	dword1 = cmd_val(s, 1);
+	dword2 = cmd_val(s, 2);
+
+	v = (dword0 & GENMASK(21, 19)) >> 19;
+	if (WARN_ON(v >= ARRAY_SIZE(gen8_plane_code)))
+		return -EINVAL;
+
+	info->pipe = gen8_plane_code[v].pipe;
+	info->plane = gen8_plane_code[v].plane;
+	info->event = gen8_plane_code[v].event;
+	info->stride_val = (dword1 & GENMASK(15, 6)) >> 6;
+	info->tile_val = (dword1 & 0x1);
+	info->surf_val = (dword2 & GENMASK(31, 12)) >> 12;
+	info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1);
+
+	if (info->plane == PLANE_A) {
+		info->ctrl_reg = DSPCNTR(info->pipe);
+		info->stride_reg = DSPSTRIDE(info->pipe);
+		info->surf_reg = DSPSURF(info->pipe);
+	} else if (info->plane == PLANE_B) {
+		info->ctrl_reg = SPRCTL(info->pipe);
+		info->stride_reg = SPRSTRIDE(info->pipe);
+		info->surf_reg = SPRSURF(info->pipe);
+	} else {
+		WARN_ON(1);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int skl_decode_mi_display_flip(struct parser_exec_state *s,
+		struct mi_display_flip_command_info *info)
+{
+	struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+	u32 dword0 = cmd_val(s, 0);
+	u32 dword1 = cmd_val(s, 1);
+	u32 dword2 = cmd_val(s, 2);
+	u32 plane = (dword0 & GENMASK(12, 8)) >> 8;
+
+	switch (plane) {
+	case MI_DISPLAY_FLIP_SKL_PLANE_1_A:
+		info->pipe = PIPE_A;
+		info->event = PRIMARY_A_FLIP_DONE;
+		break;
+	case MI_DISPLAY_FLIP_SKL_PLANE_1_B:
+		info->pipe = PIPE_B;
+		info->event = PRIMARY_B_FLIP_DONE;
+		break;
+	case MI_DISPLAY_FLIP_SKL_PLANE_1_C:
+		info->pipe = PIPE_C;
+		info->event = PRIMARY_C_FLIP_DONE;
+		break;
+	default:
+		gvt_err("unknown plane code %d\n", plane);
+		return -EINVAL;
+	}
+
+	info->pipe = PRIMARY_PLANE;
+	info->stride_val = (dword1 & GENMASK(15, 6)) >> 6;
+	info->tile_val = (dword1 & GENMASK(2, 0));
+	info->surf_val = (dword2 & GENMASK(31, 12)) >> 12;
+	info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1);
+
+	info->ctrl_reg = DSPCNTR(info->pipe);
+	info->stride_reg = DSPSTRIDE(info->pipe);
+	info->surf_reg = DSPSURF(info->pipe);
+
+	return 0;
+}
+
+static int gen8_check_mi_display_flip(struct parser_exec_state *s,
+		struct mi_display_flip_command_info *info)
+{
+	struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+	u32 stride, tile;
+
+	if (!info->async_flip)
+		return 0;
+
+	if (IS_SKYLAKE(dev_priv)) {
+		stride = vgpu_vreg(s->vgpu, info->stride_reg) & GENMASK(9, 0);
+		tile = (vgpu_vreg(s->vgpu, info->ctrl_reg) &
+				GENMASK(12, 10)) >> 10;
+	} else {
+		stride = (vgpu_vreg(s->vgpu, info->stride_reg) &
+				GENMASK(15, 6)) >> 6;
+		tile = (vgpu_vreg(s->vgpu, info->ctrl_reg) & (1 << 10)) >> 10;
+	}
+
+	if (stride != info->stride_val)
+		gvt_dbg_cmd("cannot change stride during async flip\n");
+
+	if (tile != info->tile_val)
+		gvt_dbg_cmd("cannot change tile during async flip\n");
+
+	return 0;
+}
+
+static int gen8_update_plane_mmio_from_mi_display_flip(
+		struct parser_exec_state *s,
+		struct mi_display_flip_command_info *info)
+{
+	struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+	struct intel_vgpu *vgpu = s->vgpu;
+
+	set_mask_bits(&vgpu_vreg(vgpu, info->surf_reg), GENMASK(31, 12),
+		      info->surf_val << 12);
+	if (IS_SKYLAKE(dev_priv)) {
+		set_mask_bits(&vgpu_vreg(vgpu, info->stride_reg), GENMASK(9, 0),
+			      info->stride_val);
+		set_mask_bits(&vgpu_vreg(vgpu, info->ctrl_reg), GENMASK(12, 10),
+			      info->tile_val << 10);
+	} else {
+		set_mask_bits(&vgpu_vreg(vgpu, info->stride_reg), GENMASK(15, 6),
+			      info->stride_val << 6);
+		set_mask_bits(&vgpu_vreg(vgpu, info->ctrl_reg), GENMASK(10, 10),
+			      info->tile_val << 10);
+	}
+
+	vgpu_vreg(vgpu, PIPE_FRMCOUNT_G4X(info->pipe))++;
+	intel_vgpu_trigger_virtual_event(vgpu, info->event);
+	return 0;
+}
+
+static int decode_mi_display_flip(struct parser_exec_state *s,
+		struct mi_display_flip_command_info *info)
+{
+	struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+
+	if (IS_BROADWELL(dev_priv))
+		return gen8_decode_mi_display_flip(s, info);
+	if (IS_SKYLAKE(dev_priv))
+		return skl_decode_mi_display_flip(s, info);
+
+	return -ENODEV;
+}
+
+static int check_mi_display_flip(struct parser_exec_state *s,
+		struct mi_display_flip_command_info *info)
+{
+	struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+
+	if (IS_BROADWELL(dev_priv) || IS_SKYLAKE(dev_priv))
+		return gen8_check_mi_display_flip(s, info);
+	return -ENODEV;
+}
+
+static int update_plane_mmio_from_mi_display_flip(
+		struct parser_exec_state *s,
+		struct mi_display_flip_command_info *info)
+{
+	struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
+
+	if (IS_BROADWELL(dev_priv) || IS_SKYLAKE(dev_priv))
+		return gen8_update_plane_mmio_from_mi_display_flip(s, info);
+	return -ENODEV;
+}
+
+static int cmd_handler_mi_display_flip(struct parser_exec_state *s)
+{
+	struct mi_display_flip_command_info info;
+	int ret;
+	int i;
+	int len = cmd_length(s);
+
+	ret = decode_mi_display_flip(s, &info);
+	if (ret) {
+		gvt_err("fail to decode MI display flip command\n");
+		return ret;
+	}
+
+	ret = check_mi_display_flip(s, &info);
+	if (ret) {
+		gvt_err("invalid MI display flip command\n");
+		return ret;
+	}
+
+	ret = update_plane_mmio_from_mi_display_flip(s, &info);
+	if (ret) {
+		gvt_err("fail to update plane mmio\n");
+		return ret;
+	}
+
+	for (i = 0; i < len; i++)
+		patch_value(s, cmd_ptr(s, i), MI_NOOP);
+	return 0;
+}
+
+static bool is_wait_for_flip_pending(u32 cmd)
+{
+	return cmd & (MI_WAIT_FOR_PLANE_A_FLIP_PENDING |
+			MI_WAIT_FOR_PLANE_B_FLIP_PENDING |
+			MI_WAIT_FOR_PLANE_C_FLIP_PENDING |
+			MI_WAIT_FOR_SPRITE_A_FLIP_PENDING |
+			MI_WAIT_FOR_SPRITE_B_FLIP_PENDING |
+			MI_WAIT_FOR_SPRITE_C_FLIP_PENDING);
+}
+
+static int cmd_handler_mi_wait_for_event(struct parser_exec_state *s)
+{
+	u32 cmd = cmd_val(s, 0);
+
+	if (!is_wait_for_flip_pending(cmd))
+		return 0;
+
+	patch_value(s, cmd_ptr(s, 0), MI_NOOP);
+	return 0;
+}
+
+static unsigned long get_gma_bb_from_cmd(struct parser_exec_state *s, int index)
+{
+	unsigned long addr;
+	unsigned long gma_high, gma_low;
+	int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+
+	if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8))
+		return INTEL_GVT_INVALID_ADDR;
+
+	gma_low = cmd_val(s, index) & BATCH_BUFFER_ADDR_MASK;
+	if (gmadr_bytes == 4) {
+		addr = gma_low;
+	} else {
+		gma_high = cmd_val(s, index + 1) & BATCH_BUFFER_ADDR_HIGH_MASK;
+		addr = (((unsigned long)gma_high) << 32) | gma_low;
+	}
+	return addr;
+}
+
+static inline int cmd_address_audit(struct parser_exec_state *s,
+		unsigned long guest_gma, int op_size, bool index_mode)
+{
+	struct intel_vgpu *vgpu = s->vgpu;
+	u32 max_surface_size = vgpu->gvt->device_info.max_surface_size;
+	int i;
+	int ret;
+
+	if (op_size > max_surface_size) {
+		gvt_err("command address audit fail name %s\n", s->info->name);
+		return -EINVAL;
+	}
+
+	if (index_mode)	{
+		if (guest_gma >= GTT_PAGE_SIZE / sizeof(u64)) {
+			ret = -EINVAL;
+			goto err;
+		}
+	} else if ((!vgpu_gmadr_is_valid(s->vgpu, guest_gma)) ||
+			(!vgpu_gmadr_is_valid(s->vgpu,
+					      guest_gma + op_size - 1))) {
+		ret = -EINVAL;
+		goto err;
+	}
+	return 0;
+err:
+	gvt_err("cmd_parser: Malicious %s detected, addr=0x%lx, len=%d!\n",
+			s->info->name, guest_gma, op_size);
+
+	pr_err("cmd dump: ");
+	for (i = 0; i < cmd_length(s); i++) {
+		if (!(i % 4))
+			pr_err("\n%08x ", cmd_val(s, i));
+		else
+			pr_err("%08x ", cmd_val(s, i));
+	}
+	pr_err("\nvgpu%d: aperture 0x%llx - 0x%llx, hidden 0x%llx - 0x%llx\n",
+			vgpu->id,
+			vgpu_aperture_gmadr_base(vgpu),
+			vgpu_aperture_gmadr_end(vgpu),
+			vgpu_hidden_gmadr_base(vgpu),
+			vgpu_hidden_gmadr_end(vgpu));
+	return ret;
+}
+
+static int cmd_handler_mi_store_data_imm(struct parser_exec_state *s)
+{
+	int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+	int op_size = (cmd_length(s) - 3) * sizeof(u32);
+	int core_id = (cmd_val(s, 2) & (1 << 0)) ? 1 : 0;
+	unsigned long gma, gma_low, gma_high;
+	int ret = 0;
+
+	/* check ppggt */
+	if (!(cmd_val(s, 0) & (1 << 22)))
+		return 0;
+
+	gma = cmd_val(s, 2) & GENMASK(31, 2);
+
+	if (gmadr_bytes == 8) {
+		gma_low = cmd_val(s, 1) & GENMASK(31, 2);
+		gma_high = cmd_val(s, 2) & GENMASK(15, 0);
+		gma = (gma_high << 32) | gma_low;
+		core_id = (cmd_val(s, 1) & (1 << 0)) ? 1 : 0;
+	}
+	ret = cmd_address_audit(s, gma + op_size * core_id, op_size, false);
+	return ret;
+}
+
+static inline int unexpected_cmd(struct parser_exec_state *s)
+{
+	gvt_err("vgpu%d: Unexpected %s in command buffer!\n",
+			s->vgpu->id, s->info->name);
+	return -EINVAL;
+}
+
+static int cmd_handler_mi_semaphore_wait(struct parser_exec_state *s)
+{
+	return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_report_perf_count(struct parser_exec_state *s)
+{
+	return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_op_2e(struct parser_exec_state *s)
+{
+	return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_op_2f(struct parser_exec_state *s)
+{
+	int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+	int op_size = (1 << ((cmd_val(s, 0) & GENMASK(20, 19)) >> 19)) *
+			sizeof(u32);
+	unsigned long gma, gma_high;
+	int ret = 0;
+
+	if (!(cmd_val(s, 0) & (1 << 22)))
+		return ret;
+
+	gma = cmd_val(s, 1) & GENMASK(31, 2);
+	if (gmadr_bytes == 8) {
+		gma_high = cmd_val(s, 2) & GENMASK(15, 0);
+		gma = (gma_high << 32) | gma;
+	}
+	ret = cmd_address_audit(s, gma, op_size, false);
+	return ret;
+}
+
+static int cmd_handler_mi_store_data_index(struct parser_exec_state *s)
+{
+	return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_clflush(struct parser_exec_state *s)
+{
+	return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_conditional_batch_buffer_end(
+		struct parser_exec_state *s)
+{
+	return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_update_gtt(struct parser_exec_state *s)
+{
+	return unexpected_cmd(s);
+}
+
+static int cmd_handler_mi_flush_dw(struct parser_exec_state *s)
+{
+	int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+	unsigned long gma;
+	bool index_mode = false;
+	int ret = 0;
+
+	/* Check post-sync and ppgtt bit */
+	if (((cmd_val(s, 0) >> 14) & 0x3) && (cmd_val(s, 1) & (1 << 2))) {
+		gma = cmd_val(s, 1) & GENMASK(31, 3);
+		if (gmadr_bytes == 8)
+			gma |= (cmd_val(s, 2) & GENMASK(15, 0)) << 32;
+		/* Store Data Index */
+		if (cmd_val(s, 0) & (1 << 21))
+			index_mode = true;
+		ret = cmd_address_audit(s, gma, sizeof(u64), index_mode);
+	}
+	/* Check notify bit */
+	if ((cmd_val(s, 0) & (1 << 8)))
+		set_bit(cmd_interrupt_events[s->ring_id].mi_flush_dw,
+				s->workload->pending_events);
+	return ret;
+}
+
+static void addr_type_update_snb(struct parser_exec_state *s)
+{
+	if ((s->buf_type == RING_BUFFER_INSTRUCTION) &&
+			(BATCH_BUFFER_ADR_SPACE_BIT(cmd_val(s, 0)) == 1)) {
+		s->buf_addr_type = PPGTT_BUFFER;
+	}
+}
+
+
+static int copy_gma_to_hva(struct intel_vgpu *vgpu, struct intel_vgpu_mm *mm,
+		unsigned long gma, unsigned long end_gma, void *va)
+{
+	unsigned long copy_len, offset;
+	unsigned long len = 0;
+	unsigned long gpa;
+
+	while (gma != end_gma) {
+		gpa = intel_vgpu_gma_to_gpa(mm, gma);
+		if (gpa == INTEL_GVT_INVALID_ADDR) {
+			gvt_err("invalid gma address: %lx\n", gma);
+			return -EFAULT;
+		}
+
+		offset = gma & (GTT_PAGE_SIZE - 1);
+
+		copy_len = (end_gma - gma) >= (GTT_PAGE_SIZE - offset) ?
+			GTT_PAGE_SIZE - offset : end_gma - gma;
+
+		intel_gvt_hypervisor_read_gpa(vgpu, gpa, va + len, copy_len);
+
+		len += copy_len;
+		gma += copy_len;
+	}
+	return 0;
+}
+
+
+/*
+ * Check whether a batch buffer needs to be scanned. Currently
+ * the only criteria is based on privilege.
+ */
+static int batch_buffer_needs_scan(struct parser_exec_state *s)
+{
+	struct intel_gvt *gvt = s->vgpu->gvt;
+
+	if (bypass_batch_buffer_scan)
+		return 0;
+
+	if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)) {
+		/* BDW decides privilege based on address space */
+		if (cmd_val(s, 0) & (1 << 8))
+			return 0;
+	}
+	return 1;
+}
+
+static uint32_t find_bb_size(struct parser_exec_state *s)
+{
+	unsigned long gma = 0;
+	struct cmd_info *info;
+	uint32_t bb_size = 0;
+	uint32_t cmd_len = 0;
+	bool met_bb_end = false;
+	u32 cmd;
+
+	/* get the start gm address of the batch buffer */
+	gma = get_gma_bb_from_cmd(s, 1);
+	cmd = cmd_val(s, 0);
+
+	info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
+	if (info == NULL) {
+		gvt_err("unknown cmd 0x%x, opcode=0x%x\n",
+				cmd, get_opcode(cmd, s->ring_id));
+		return -EINVAL;
+	}
+	do {
+		copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
+				gma, gma + 4, &cmd);
+		info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
+		if (info == NULL) {
+			gvt_err("unknown cmd 0x%x, opcode=0x%x\n",
+				cmd, get_opcode(cmd, s->ring_id));
+			return -EINVAL;
+		}
+
+		if (info->opcode == OP_MI_BATCH_BUFFER_END) {
+			met_bb_end = true;
+		} else if (info->opcode == OP_MI_BATCH_BUFFER_START) {
+			if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0) {
+				/* chained batch buffer */
+				met_bb_end = true;
+			}
+		}
+		cmd_len = get_cmd_length(info, cmd) << 2;
+		bb_size += cmd_len;
+		gma += cmd_len;
+
+	} while (!met_bb_end);
+
+	return bb_size;
+}
+
+static int perform_bb_shadow(struct parser_exec_state *s)
+{
+	struct intel_shadow_bb_entry *entry_obj;
+	unsigned long gma = 0;
+	uint32_t bb_size;
+	void *dst = NULL;
+	int ret = 0;
+
+	/* get the start gm address of the batch buffer */
+	gma = get_gma_bb_from_cmd(s, 1);
+
+	/* get the size of the batch buffer */
+	bb_size = find_bb_size(s);
+
+	/* allocate shadow batch buffer */
+	entry_obj = kmalloc(sizeof(*entry_obj), GFP_KERNEL);
+	if (entry_obj == NULL)
+		return -ENOMEM;
+
+	entry_obj->obj =
+		i915_gem_object_create(&(s->vgpu->gvt->dev_priv->drm),
+				       roundup(bb_size, PAGE_SIZE));
+	if (IS_ERR(entry_obj->obj)) {
+		ret = PTR_ERR(entry_obj->obj);
+		goto free_entry;
+	}
+	entry_obj->len = bb_size;
+	INIT_LIST_HEAD(&entry_obj->list);
+
+	dst = i915_gem_object_pin_map(entry_obj->obj, I915_MAP_WB);
+	if (IS_ERR(dst)) {
+		ret = PTR_ERR(dst);
+		goto put_obj;
+	}
+
+	ret = i915_gem_object_set_to_cpu_domain(entry_obj->obj, false);
+	if (ret) {
+		gvt_err("failed to set shadow batch to CPU\n");
+		goto unmap_src;
+	}
+
+	entry_obj->va = dst;
+	entry_obj->bb_start_cmd_va = s->ip_va;
+
+	/* copy batch buffer to shadow batch buffer*/
+	ret = copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
+			      gma, gma + bb_size,
+			      dst);
+	if (ret) {
+		gvt_err("fail to copy guest ring buffer\n");
+		goto unmap_src;
+	}
+
+	list_add(&entry_obj->list, &s->workload->shadow_bb);
+	/*
+	 * ip_va saves the virtual address of the shadow batch buffer, while
+	 * ip_gma saves the graphics address of the original batch buffer.
+	 * As the shadow batch buffer is just a copy from the originial one,
+	 * it should be right to use shadow batch buffer'va and original batch
+	 * buffer's gma in pair. After all, we don't want to pin the shadow
+	 * buffer here (too early).
+	 */
+	s->ip_va = dst;
+	s->ip_gma = gma;
+
+	return 0;
+
+unmap_src:
+	i915_gem_object_unpin_map(entry_obj->obj);
+put_obj:
+	i915_gem_object_put(entry_obj->obj);
+free_entry:
+	kfree(entry_obj);
+	return ret;
+}
+
+static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s)
+{
+	bool second_level;
+	int ret = 0;
+
+	if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) {
+		gvt_err("Found MI_BATCH_BUFFER_START in 2nd level BB\n");
+		return -EINVAL;
+	}
+
+	second_level = BATCH_BUFFER_2ND_LEVEL_BIT(cmd_val(s, 0)) == 1;
+	if (second_level && (s->buf_type != BATCH_BUFFER_INSTRUCTION)) {
+		gvt_err("Jumping to 2nd level BB from RB is not allowed\n");
+		return -EINVAL;
+	}
+
+	s->saved_buf_addr_type = s->buf_addr_type;
+	addr_type_update_snb(s);
+	if (s->buf_type == RING_BUFFER_INSTRUCTION) {
+		s->ret_ip_gma_ring = s->ip_gma + cmd_length(s) * sizeof(u32);
+		s->buf_type = BATCH_BUFFER_INSTRUCTION;
+	} else if (second_level) {
+		s->buf_type = BATCH_BUFFER_2ND_LEVEL;
+		s->ret_ip_gma_bb = s->ip_gma + cmd_length(s) * sizeof(u32);
+		s->ret_bb_va = s->ip_va + cmd_length(s) * sizeof(u32);
+	}
+
+	if (batch_buffer_needs_scan(s)) {
+		ret = perform_bb_shadow(s);
+		if (ret < 0)
+			gvt_err("invalid shadow batch buffer\n");
+	} else {
+		/* emulate a batch buffer end to do return right */
+		ret = cmd_handler_mi_batch_buffer_end(s);
+		if (ret < 0)
+			return ret;
+	}
+
+	return ret;
+}
+
+static struct cmd_info cmd_info[] = {
+	{"MI_NOOP", OP_MI_NOOP, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL},
+
+	{"MI_SET_PREDICATE", OP_MI_SET_PREDICATE, F_LEN_CONST, R_ALL, D_ALL,
+		0, 1, NULL},
+
+	{"MI_USER_INTERRUPT", OP_MI_USER_INTERRUPT, F_LEN_CONST, R_ALL, D_ALL,
+		0, 1, cmd_handler_mi_user_interrupt},
+
+	{"MI_WAIT_FOR_EVENT", OP_MI_WAIT_FOR_EVENT, F_LEN_CONST, R_RCS | R_BCS,
+		D_ALL, 0, 1, cmd_handler_mi_wait_for_event},
+
+	{"MI_FLUSH", OP_MI_FLUSH, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL},
+
+	{"MI_ARB_CHECK", OP_MI_ARB_CHECK, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
+		NULL},
+
+	{"MI_RS_CONTROL", OP_MI_RS_CONTROL, F_LEN_CONST, R_RCS, D_ALL, 0, 1,
+		NULL},
+
+	{"MI_REPORT_HEAD", OP_MI_REPORT_HEAD, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
+		NULL},
+
+	{"MI_ARB_ON_OFF", OP_MI_ARB_ON_OFF, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
+		NULL},
+
+	{"MI_URB_ATOMIC_ALLOC", OP_MI_URB_ATOMIC_ALLOC, F_LEN_CONST, R_RCS,
+		D_ALL, 0, 1, NULL},
+
+	{"MI_BATCH_BUFFER_END", OP_MI_BATCH_BUFFER_END,
+		F_IP_ADVANCE_CUSTOM | F_LEN_CONST, R_ALL, D_ALL, 0, 1,
+		cmd_handler_mi_batch_buffer_end},
+
+	{"MI_SUSPEND_FLUSH", OP_MI_SUSPEND_FLUSH, F_LEN_CONST, R_ALL, D_ALL,
+		0, 1, NULL},
+
+	{"MI_PREDICATE", OP_MI_PREDICATE, F_LEN_CONST, R_RCS, D_ALL, 0, 1,
+		NULL},
+
+	{"MI_TOPOLOGY_FILTER", OP_MI_TOPOLOGY_FILTER, F_LEN_CONST, R_ALL,
+		D_ALL, 0, 1, NULL},
+
+	{"MI_SET_APPID", OP_MI_SET_APPID, F_LEN_CONST, R_ALL, D_ALL, 0, 1,
+		NULL},
+
+	{"MI_RS_CONTEXT", OP_MI_RS_CONTEXT, F_LEN_CONST, R_RCS, D_ALL, 0, 1,
+		NULL},
+
+	{"MI_DISPLAY_FLIP", OP_MI_DISPLAY_FLIP, F_LEN_VAR | F_POST_HANDLE,
+		R_RCS | R_BCS, D_ALL, 0, 8, cmd_handler_mi_display_flip},
+
+	{"MI_SEMAPHORE_MBOX", OP_MI_SEMAPHORE_MBOX, F_LEN_VAR, R_ALL, D_ALL,
+		0, 8, NULL},
+
+	{"MI_MATH", OP_MI_MATH, F_LEN_VAR, R_ALL, D_ALL, 0, 8, NULL},
+
+	{"MI_URB_CLEAR", OP_MI_URB_CLEAR, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"ME_SEMAPHORE_SIGNAL", OP_MI_SEMAPHORE_SIGNAL, F_LEN_VAR, R_ALL,
+		D_BDW_PLUS, 0, 8, NULL},
+
+	{"ME_SEMAPHORE_WAIT", OP_MI_SEMAPHORE_WAIT, F_LEN_VAR, R_ALL, D_BDW_PLUS,
+		ADDR_FIX_1(2), 8, cmd_handler_mi_semaphore_wait},
+
+	{"MI_STORE_DATA_IMM", OP_MI_STORE_DATA_IMM, F_LEN_VAR, R_ALL, D_BDW_PLUS,
+		ADDR_FIX_1(1), 10, cmd_handler_mi_store_data_imm},
+
+	{"MI_STORE_DATA_INDEX", OP_MI_STORE_DATA_INDEX, F_LEN_VAR, R_ALL, D_ALL,
+		0, 8, cmd_handler_mi_store_data_index},
+
+	{"MI_LOAD_REGISTER_IMM", OP_MI_LOAD_REGISTER_IMM, F_LEN_VAR, R_ALL,
+		D_ALL, 0, 8, cmd_handler_lri},
+
+	{"MI_UPDATE_GTT", OP_MI_UPDATE_GTT, F_LEN_VAR, R_ALL, D_BDW_PLUS, 0, 10,
+		cmd_handler_mi_update_gtt},
+
+	{"MI_STORE_REGISTER_MEM", OP_MI_STORE_REGISTER_MEM, F_LEN_VAR, R_ALL,
+		D_ALL, ADDR_FIX_1(2), 8, cmd_handler_srm},
+
+	{"MI_FLUSH_DW", OP_MI_FLUSH_DW, F_LEN_VAR, R_ALL, D_ALL, 0, 6,
+		cmd_handler_mi_flush_dw},
+
+	{"MI_CLFLUSH", OP_MI_CLFLUSH, F_LEN_VAR, R_ALL, D_ALL, ADDR_FIX_1(1),
+		10, cmd_handler_mi_clflush},
+
+	{"MI_REPORT_PERF_COUNT", OP_MI_REPORT_PERF_COUNT, F_LEN_VAR, R_ALL,
+		D_ALL, ADDR_FIX_1(1), 6, cmd_handler_mi_report_perf_count},
+
+	{"MI_LOAD_REGISTER_MEM", OP_MI_LOAD_REGISTER_MEM, F_LEN_VAR, R_ALL,
+		D_ALL, ADDR_FIX_1(2), 8, cmd_handler_lrm},
+
+	{"MI_LOAD_REGISTER_REG", OP_MI_LOAD_REGISTER_REG, F_LEN_VAR, R_ALL,
+		D_ALL, 0, 8, cmd_handler_lrr},
+
+	{"MI_RS_STORE_DATA_IMM", OP_MI_RS_STORE_DATA_IMM, F_LEN_VAR, R_RCS,
+		D_ALL, 0, 8, NULL},
+
+	{"MI_LOAD_URB_MEM", OP_MI_LOAD_URB_MEM, F_LEN_VAR, R_RCS, D_ALL,
+		ADDR_FIX_1(2), 8, NULL},
+
+	{"MI_STORE_URM_MEM", OP_MI_STORE_URM_MEM, F_LEN_VAR, R_RCS, D_ALL,
+		ADDR_FIX_1(2), 8, NULL},
+
+	{"MI_OP_2E", OP_MI_2E, F_LEN_VAR, R_ALL, D_BDW_PLUS, ADDR_FIX_2(1, 2),
+		8, cmd_handler_mi_op_2e},
+
+	{"MI_OP_2F", OP_MI_2F, F_LEN_VAR, R_ALL, D_BDW_PLUS, ADDR_FIX_1(1),
+		8, cmd_handler_mi_op_2f},
+
+	{"MI_BATCH_BUFFER_START", OP_MI_BATCH_BUFFER_START,
+		F_IP_ADVANCE_CUSTOM, R_ALL, D_ALL, 0, 8,
+		cmd_handler_mi_batch_buffer_start},
+
+	{"MI_CONDITIONAL_BATCH_BUFFER_END", OP_MI_CONDITIONAL_BATCH_BUFFER_END,
+		F_LEN_VAR, R_ALL, D_ALL, ADDR_FIX_1(2), 8,
+		cmd_handler_mi_conditional_batch_buffer_end},
+
+	{"MI_LOAD_SCAN_LINES_INCL", OP_MI_LOAD_SCAN_LINES_INCL, F_LEN_CONST,
+		R_RCS | R_BCS, D_ALL, 0, 2, NULL},
+
+	{"XY_SETUP_BLT", OP_XY_SETUP_BLT, F_LEN_VAR, R_BCS, D_ALL,
+		ADDR_FIX_2(4, 7), 8, NULL},
+
+	{"XY_SETUP_CLIP_BLT", OP_XY_SETUP_CLIP_BLT, F_LEN_VAR, R_BCS, D_ALL,
+		0, 8, NULL},
+
+	{"XY_SETUP_MONO_PATTERN_SL_BLT", OP_XY_SETUP_MONO_PATTERN_SL_BLT,
+		F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL},
+
+	{"XY_PIXEL_BLT", OP_XY_PIXEL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL},
+
+	{"XY_SCANLINES_BLT", OP_XY_SCANLINES_BLT, F_LEN_VAR, R_BCS, D_ALL,
+		0, 8, NULL},
+
+	{"XY_TEXT_BLT", OP_XY_TEXT_BLT, F_LEN_VAR, R_BCS, D_ALL,
+		ADDR_FIX_1(3), 8, NULL},
+
+	{"XY_TEXT_IMMEDIATE_BLT", OP_XY_TEXT_IMMEDIATE_BLT, F_LEN_VAR, R_BCS,
+		D_ALL, 0, 8, NULL},
+
+	{"XY_COLOR_BLT", OP_XY_COLOR_BLT, F_LEN_VAR, R_BCS, D_ALL,
+		ADDR_FIX_1(4), 8, NULL},
+
+	{"XY_PAT_BLT", OP_XY_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL,
+		ADDR_FIX_2(4, 5), 8, NULL},
+
+	{"XY_MONO_PAT_BLT", OP_XY_MONO_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL,
+		ADDR_FIX_1(4), 8, NULL},
+
+	{"XY_SRC_COPY_BLT", OP_XY_SRC_COPY_BLT, F_LEN_VAR, R_BCS, D_ALL,
+		ADDR_FIX_2(4, 7), 8, NULL},
+
+	{"XY_MONO_SRC_COPY_BLT", OP_XY_MONO_SRC_COPY_BLT, F_LEN_VAR, R_BCS,
+		D_ALL, ADDR_FIX_2(4, 5), 8, NULL},
+
+	{"XY_FULL_BLT", OP_XY_FULL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL},
+
+	{"XY_FULL_MONO_SRC_BLT", OP_XY_FULL_MONO_SRC_BLT, F_LEN_VAR, R_BCS,
+		D_ALL, ADDR_FIX_3(4, 5, 8), 8, NULL},
+
+	{"XY_FULL_MONO_PATTERN_BLT", OP_XY_FULL_MONO_PATTERN_BLT, F_LEN_VAR,
+		R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL},
+
+	{"XY_FULL_MONO_PATTERN_MONO_SRC_BLT",
+		OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT,
+		F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL},
+
+	{"XY_MONO_PAT_FIXED_BLT", OP_XY_MONO_PAT_FIXED_BLT, F_LEN_VAR, R_BCS,
+		D_ALL, ADDR_FIX_1(4), 8, NULL},
+
+	{"XY_MONO_SRC_COPY_IMMEDIATE_BLT", OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT,
+		F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL},
+
+	{"XY_PAT_BLT_IMMEDIATE", OP_XY_PAT_BLT_IMMEDIATE, F_LEN_VAR, R_BCS,
+		D_ALL, ADDR_FIX_1(4), 8, NULL},
+
+	{"XY_SRC_COPY_CHROMA_BLT", OP_XY_SRC_COPY_CHROMA_BLT, F_LEN_VAR, R_BCS,
+		D_ALL, ADDR_FIX_2(4, 7), 8, NULL},
+
+	{"XY_FULL_IMMEDIATE_PATTERN_BLT", OP_XY_FULL_IMMEDIATE_PATTERN_BLT,
+		F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL},
+
+	{"XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT",
+		OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT,
+		F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL},
+
+	{"XY_PAT_CHROMA_BLT", OP_XY_PAT_CHROMA_BLT, F_LEN_VAR, R_BCS, D_ALL,
+		ADDR_FIX_2(4, 5), 8, NULL},
+
+	{"XY_PAT_CHROMA_BLT_IMMEDIATE", OP_XY_PAT_CHROMA_BLT_IMMEDIATE,
+		F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL},
+
+	{"3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP",
+		OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_VIEWPORT_STATE_POINTERS_CC",
+		OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_BLEND_STATE_POINTERS",
+		OP_3DSTATE_BLEND_STATE_POINTERS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_DEPTH_STENCIL_STATE_POINTERS",
+		OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_BINDING_TABLE_POINTERS_VS",
+		OP_3DSTATE_BINDING_TABLE_POINTERS_VS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_BINDING_TABLE_POINTERS_HS",
+		OP_3DSTATE_BINDING_TABLE_POINTERS_HS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_BINDING_TABLE_POINTERS_DS",
+		OP_3DSTATE_BINDING_TABLE_POINTERS_DS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_BINDING_TABLE_POINTERS_GS",
+		OP_3DSTATE_BINDING_TABLE_POINTERS_GS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_BINDING_TABLE_POINTERS_PS",
+		OP_3DSTATE_BINDING_TABLE_POINTERS_PS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_SAMPLER_STATE_POINTERS_VS",
+		OP_3DSTATE_SAMPLER_STATE_POINTERS_VS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_SAMPLER_STATE_POINTERS_HS",
+		OP_3DSTATE_SAMPLER_STATE_POINTERS_HS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_SAMPLER_STATE_POINTERS_DS",
+		OP_3DSTATE_SAMPLER_STATE_POINTERS_DS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_SAMPLER_STATE_POINTERS_GS",
+		OP_3DSTATE_SAMPLER_STATE_POINTERS_GS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_SAMPLER_STATE_POINTERS_PS",
+		OP_3DSTATE_SAMPLER_STATE_POINTERS_PS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_URB_VS", OP_3DSTATE_URB_VS, F_LEN_VAR, R_RCS, D_ALL,
+		0, 8, NULL},
+
+	{"3DSTATE_URB_HS", OP_3DSTATE_URB_HS, F_LEN_VAR, R_RCS, D_ALL,
+		0, 8, NULL},
+
+	{"3DSTATE_URB_DS", OP_3DSTATE_URB_DS, F_LEN_VAR, R_RCS, D_ALL,
+		0, 8, NULL},
+
+	{"3DSTATE_URB_GS", OP_3DSTATE_URB_GS, F_LEN_VAR, R_RCS, D_ALL,
+		0, 8, NULL},
+
+	{"3DSTATE_GATHER_CONSTANT_VS", OP_3DSTATE_GATHER_CONSTANT_VS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_GATHER_CONSTANT_GS", OP_3DSTATE_GATHER_CONSTANT_GS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_GATHER_CONSTANT_HS", OP_3DSTATE_GATHER_CONSTANT_HS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_GATHER_CONSTANT_DS", OP_3DSTATE_GATHER_CONSTANT_DS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_GATHER_CONSTANT_PS", OP_3DSTATE_GATHER_CONSTANT_PS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_DX9_CONSTANTF_VS", OP_3DSTATE_DX9_CONSTANTF_VS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL},
+
+	{"3DSTATE_DX9_CONSTANTF_PS", OP_3DSTATE_DX9_CONSTANTF_PS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL},
+
+	{"3DSTATE_DX9_CONSTANTI_VS", OP_3DSTATE_DX9_CONSTANTI_VS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_DX9_CONSTANTI_PS", OP_3DSTATE_DX9_CONSTANTI_PS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_DX9_CONSTANTB_VS", OP_3DSTATE_DX9_CONSTANTB_VS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_DX9_CONSTANTB_PS", OP_3DSTATE_DX9_CONSTANTB_PS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_DX9_LOCAL_VALID_VS", OP_3DSTATE_DX9_LOCAL_VALID_VS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_DX9_LOCAL_VALID_PS", OP_3DSTATE_DX9_LOCAL_VALID_PS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_DX9_GENERATE_ACTIVE_VS", OP_3DSTATE_DX9_GENERATE_ACTIVE_VS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_DX9_GENERATE_ACTIVE_PS", OP_3DSTATE_DX9_GENERATE_ACTIVE_PS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_BINDING_TABLE_EDIT_VS", OP_3DSTATE_BINDING_TABLE_EDIT_VS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
+
+	{"3DSTATE_BINDING_TABLE_EDIT_GS", OP_3DSTATE_BINDING_TABLE_EDIT_GS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
+
+	{"3DSTATE_BINDING_TABLE_EDIT_HS", OP_3DSTATE_BINDING_TABLE_EDIT_HS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
+
+	{"3DSTATE_BINDING_TABLE_EDIT_DS", OP_3DSTATE_BINDING_TABLE_EDIT_DS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
+
+	{"3DSTATE_BINDING_TABLE_EDIT_PS", OP_3DSTATE_BINDING_TABLE_EDIT_PS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL},
+
+	{"3DSTATE_VF_INSTANCING", OP_3DSTATE_VF_INSTANCING, F_LEN_VAR, R_RCS,
+		D_BDW_PLUS, 0, 8, NULL},
+
+	{"3DSTATE_VF_SGVS", OP_3DSTATE_VF_SGVS, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
+		NULL},
+
+	{"3DSTATE_VF_TOPOLOGY", OP_3DSTATE_VF_TOPOLOGY, F_LEN_VAR, R_RCS,
+		D_BDW_PLUS, 0, 8, NULL},
+
+	{"3DSTATE_WM_CHROMAKEY", OP_3DSTATE_WM_CHROMAKEY, F_LEN_VAR, R_RCS,
+		D_BDW_PLUS, 0, 8, NULL},
+
+	{"3DSTATE_PS_BLEND", OP_3DSTATE_PS_BLEND, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0,
+		8, NULL},
+
+	{"3DSTATE_WM_DEPTH_STENCIL", OP_3DSTATE_WM_DEPTH_STENCIL, F_LEN_VAR,
+		R_RCS, D_BDW_PLUS, 0, 8, NULL},
+
+	{"3DSTATE_PS_EXTRA", OP_3DSTATE_PS_EXTRA, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0,
+		8, NULL},
+
+	{"3DSTATE_RASTER", OP_3DSTATE_RASTER, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
+		NULL},
+
+	{"3DSTATE_SBE_SWIZ", OP_3DSTATE_SBE_SWIZ, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
+		NULL},
+
+	{"3DSTATE_WM_HZ_OP", OP_3DSTATE_WM_HZ_OP, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8,
+		NULL},
+
+	{"3DSTATE_VERTEX_BUFFERS", OP_3DSTATE_VERTEX_BUFFERS, F_LEN_VAR, R_RCS,
+		D_BDW_PLUS, 0, 8, NULL},
+
+	{"3DSTATE_VERTEX_ELEMENTS", OP_3DSTATE_VERTEX_ELEMENTS, F_LEN_VAR,
+		R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_INDEX_BUFFER", OP_3DSTATE_INDEX_BUFFER, F_LEN_VAR, R_RCS,
+		D_BDW_PLUS, ADDR_FIX_1(2), 8, NULL},
+
+	{"3DSTATE_VF_STATISTICS", OP_3DSTATE_VF_STATISTICS, F_LEN_CONST,
+		R_RCS, D_ALL, 0, 1, NULL},
+
+	{"3DSTATE_VF", OP_3DSTATE_VF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_CC_STATE_POINTERS", OP_3DSTATE_CC_STATE_POINTERS, F_LEN_VAR,
+		R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_SCISSOR_STATE_POINTERS", OP_3DSTATE_SCISSOR_STATE_POINTERS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_GS", OP_3DSTATE_GS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_CLIP", OP_3DSTATE_CLIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_WM", OP_3DSTATE_WM, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_CONSTANT_GS", OP_3DSTATE_CONSTANT_GS, F_LEN_VAR, R_RCS,
+		D_BDW_PLUS, 0, 8, NULL},
+
+	{"3DSTATE_CONSTANT_PS", OP_3DSTATE_CONSTANT_PS, F_LEN_VAR, R_RCS,
+		D_BDW_PLUS, 0, 8, NULL},
+
+	{"3DSTATE_SAMPLE_MASK", OP_3DSTATE_SAMPLE_MASK, F_LEN_VAR, R_RCS,
+		D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_CONSTANT_HS", OP_3DSTATE_CONSTANT_HS, F_LEN_VAR, R_RCS,
+		D_BDW_PLUS, 0, 8, NULL},
+
+	{"3DSTATE_CONSTANT_DS", OP_3DSTATE_CONSTANT_DS, F_LEN_VAR, R_RCS,
+		D_BDW_PLUS, 0, 8, NULL},
+
+	{"3DSTATE_HS", OP_3DSTATE_HS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_TE", OP_3DSTATE_TE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_DS", OP_3DSTATE_DS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_STREAMOUT", OP_3DSTATE_STREAMOUT, F_LEN_VAR, R_RCS,
+		D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_SBE", OP_3DSTATE_SBE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_PS", OP_3DSTATE_PS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_DRAWING_RECTANGLE", OP_3DSTATE_DRAWING_RECTANGLE, F_LEN_VAR,
+		R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_SAMPLER_PALETTE_LOAD0", OP_3DSTATE_SAMPLER_PALETTE_LOAD0,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_CHROMA_KEY", OP_3DSTATE_CHROMA_KEY, F_LEN_VAR, R_RCS, D_ALL,
+		0, 8, NULL},
+
+	{"3DSTATE_DEPTH_BUFFER", OP_3DSTATE_DEPTH_BUFFER, F_LEN_VAR, R_RCS,
+		D_ALL, ADDR_FIX_1(2), 8, NULL},
+
+	{"3DSTATE_POLY_STIPPLE_OFFSET", OP_3DSTATE_POLY_STIPPLE_OFFSET,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_POLY_STIPPLE_PATTERN", OP_3DSTATE_POLY_STIPPLE_PATTERN,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_LINE_STIPPLE", OP_3DSTATE_LINE_STIPPLE, F_LEN_VAR, R_RCS,
+		D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_AA_LINE_PARAMS", OP_3DSTATE_AA_LINE_PARAMS, F_LEN_VAR, R_RCS,
+		D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_GS_SVB_INDEX", OP_3DSTATE_GS_SVB_INDEX, F_LEN_VAR, R_RCS,
+		D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_SAMPLER_PALETTE_LOAD1", OP_3DSTATE_SAMPLER_PALETTE_LOAD1,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_MULTISAMPLE", OP_3DSTATE_MULTISAMPLE_BDW, F_LEN_VAR, R_RCS,
+		D_BDW_PLUS, 0, 8, NULL},
+
+	{"3DSTATE_STENCIL_BUFFER", OP_3DSTATE_STENCIL_BUFFER, F_LEN_VAR, R_RCS,
+		D_ALL, ADDR_FIX_1(2), 8, NULL},
+
+	{"3DSTATE_HIER_DEPTH_BUFFER", OP_3DSTATE_HIER_DEPTH_BUFFER, F_LEN_VAR,
+		R_RCS, D_ALL, ADDR_FIX_1(2), 8, NULL},
+
+	{"3DSTATE_CLEAR_PARAMS", OP_3DSTATE_CLEAR_PARAMS, F_LEN_VAR,
+		R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_PUSH_CONSTANT_ALLOC_VS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_PUSH_CONSTANT_ALLOC_HS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_PUSH_CONSTANT_ALLOC_DS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_PUSH_CONSTANT_ALLOC_GS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_PUSH_CONSTANT_ALLOC_PS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_MONOFILTER_SIZE", OP_3DSTATE_MONOFILTER_SIZE, F_LEN_VAR,
+		R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_SO_DECL_LIST", OP_3DSTATE_SO_DECL_LIST, F_LEN_VAR, R_RCS,
+		D_ALL, 0, 9, NULL},
+
+	{"3DSTATE_SO_BUFFER", OP_3DSTATE_SO_BUFFER, F_LEN_VAR, R_RCS, D_BDW_PLUS,
+		ADDR_FIX_2(2, 4), 8, NULL},
+
+	{"3DSTATE_BINDING_TABLE_POOL_ALLOC",
+		OP_3DSTATE_BINDING_TABLE_POOL_ALLOC,
+		F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL},
+
+	{"3DSTATE_GATHER_POOL_ALLOC", OP_3DSTATE_GATHER_POOL_ALLOC,
+		F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL},
+
+	{"3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC",
+		OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC,
+		F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL},
+
+	{"3DSTATE_SAMPLE_PATTERN", OP_3DSTATE_SAMPLE_PATTERN, F_LEN_VAR, R_RCS,
+		D_BDW_PLUS, 0, 8, NULL},
+
+	{"PIPE_CONTROL", OP_PIPE_CONTROL, F_LEN_VAR, R_RCS, D_ALL,
+		ADDR_FIX_1(2), 8, cmd_handler_pipe_control},
+
+	{"3DPRIMITIVE", OP_3DPRIMITIVE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"PIPELINE_SELECT", OP_PIPELINE_SELECT, F_LEN_CONST, R_RCS, D_ALL, 0,
+		1, NULL},
+
+	{"STATE_PREFETCH", OP_STATE_PREFETCH, F_LEN_VAR, R_RCS, D_ALL,
+		ADDR_FIX_1(1), 8, NULL},
+
+	{"STATE_SIP", OP_STATE_SIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"STATE_BASE_ADDRESS", OP_STATE_BASE_ADDRESS, F_LEN_VAR, R_RCS, D_BDW_PLUS,
+		ADDR_FIX_5(1, 3, 4, 5, 6), 8, NULL},
+
+	{"OP_3D_MEDIA_0_1_4", OP_3D_MEDIA_0_1_4, F_LEN_VAR, R_RCS, D_ALL,
+		ADDR_FIX_1(1), 8, NULL},
+
+	{"3DSTATE_VS", OP_3DSTATE_VS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_SF", OP_3DSTATE_SF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL},
+
+	{"3DSTATE_CONSTANT_VS", OP_3DSTATE_CONSTANT_VS, F_LEN_VAR, R_RCS, D_BDW_PLUS,
+		0, 8, NULL},
+
+	{"3DSTATE_COMPONENT_PACKING", OP_3DSTATE_COMPONENT_PACKING, F_LEN_VAR, R_RCS,
+		D_SKL_PLUS, 0, 8, NULL},
+
+	{"MEDIA_INTERFACE_DESCRIPTOR_LOAD", OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD,
+		F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL},
+
+	{"MEDIA_GATEWAY_STATE", OP_MEDIA_GATEWAY_STATE, F_LEN_VAR, R_RCS, D_ALL,
+		0, 16, NULL},
+
+	{"MEDIA_STATE_FLUSH", OP_MEDIA_STATE_FLUSH, F_LEN_VAR, R_RCS, D_ALL,
+		0, 16, NULL},
+
+	{"MEDIA_OBJECT", OP_MEDIA_OBJECT, F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL},
+
+	{"MEDIA_CURBE_LOAD", OP_MEDIA_CURBE_LOAD, F_LEN_VAR, R_RCS, D_ALL,
+		0, 16, NULL},
+
+	{"MEDIA_OBJECT_PRT", OP_MEDIA_OBJECT_PRT, F_LEN_VAR, R_RCS, D_ALL,
+		0, 16, NULL},
+
+	{"MEDIA_OBJECT_WALKER", OP_MEDIA_OBJECT_WALKER, F_LEN_VAR, R_RCS, D_ALL,
+		0, 16, NULL},
+
+	{"GPGPU_WALKER", OP_GPGPU_WALKER, F_LEN_VAR, R_RCS, D_ALL,
+		0, 8, NULL},
+
+	{"MEDIA_VFE_STATE", OP_MEDIA_VFE_STATE, F_LEN_VAR, R_RCS, D_ALL, 0, 16,
+		NULL},
+
+	{"3DSTATE_VF_STATISTICS_GM45", OP_3DSTATE_VF_STATISTICS_GM45,
+		F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL},
+
+	{"MFX_PIPE_MODE_SELECT", OP_MFX_PIPE_MODE_SELECT, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_SURFACE_STATE", OP_MFX_SURFACE_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_PIPE_BUF_ADDR_STATE", OP_MFX_PIPE_BUF_ADDR_STATE, F_LEN_VAR,
+		R_VCS, D_BDW_PLUS, 0, 12, NULL},
+
+	{"MFX_IND_OBJ_BASE_ADDR_STATE", OP_MFX_IND_OBJ_BASE_ADDR_STATE,
+		F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL},
+
+	{"MFX_BSP_BUF_BASE_ADDR_STATE", OP_MFX_BSP_BUF_BASE_ADDR_STATE,
+		F_LEN_VAR, R_VCS, D_BDW_PLUS, ADDR_FIX_3(1, 3, 5), 12, NULL},
+
+	{"OP_2_0_0_5", OP_2_0_0_5, F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL},
+
+	{"MFX_STATE_POINTER", OP_MFX_STATE_POINTER, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_QM_STATE", OP_MFX_QM_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_FQM_STATE", OP_MFX_FQM_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_PAK_INSERT_OBJECT", OP_MFX_PAK_INSERT_OBJECT, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_STITCH_OBJECT", OP_MFX_STITCH_OBJECT, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFD_IT_OBJECT", OP_MFD_IT_OBJECT, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_WAIT", OP_MFX_WAIT, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 6, NULL},
+
+	{"MFX_AVC_IMG_STATE", OP_MFX_AVC_IMG_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_AVC_QM_STATE", OP_MFX_AVC_QM_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_AVC_DIRECTMODE_STATE", OP_MFX_AVC_DIRECTMODE_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_AVC_SLICE_STATE", OP_MFX_AVC_SLICE_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_AVC_REF_IDX_STATE", OP_MFX_AVC_REF_IDX_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_AVC_WEIGHTOFFSET_STATE", OP_MFX_AVC_WEIGHTOFFSET_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFD_AVC_PICID_STATE", OP_MFD_AVC_PICID_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+	{"MFD_AVC_DPB_STATE", OP_MFD_AVC_DPB_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFD_AVC_BSD_OBJECT", OP_MFD_AVC_BSD_OBJECT, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFD_AVC_SLICEADDR", OP_MFD_AVC_SLICEADDR, F_LEN_VAR,
+		R_VCS, D_ALL, ADDR_FIX_1(2), 12, NULL},
+
+	{"MFC_AVC_PAK_OBJECT", OP_MFC_AVC_PAK_OBJECT, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_VC1_PRED_PIPE_STATE", OP_MFX_VC1_PRED_PIPE_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_VC1_DIRECTMODE_STATE", OP_MFX_VC1_DIRECTMODE_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFD_VC1_SHORT_PIC_STATE", OP_MFD_VC1_SHORT_PIC_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFD_VC1_LONG_PIC_STATE", OP_MFD_VC1_LONG_PIC_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFD_VC1_BSD_OBJECT", OP_MFD_VC1_BSD_OBJECT, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFC_MPEG2_SLICEGROUP_STATE", OP_MFC_MPEG2_SLICEGROUP_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFC_MPEG2_PAK_OBJECT", OP_MFC_MPEG2_PAK_OBJECT, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_MPEG2_PIC_STATE", OP_MFX_MPEG2_PIC_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_MPEG2_QM_STATE", OP_MFX_MPEG2_QM_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFD_MPEG2_BSD_OBJECT", OP_MFD_MPEG2_BSD_OBJECT, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_2_6_0_0", OP_MFX_2_6_0_0, F_LEN_VAR, R_VCS, D_ALL,
+		0, 16, NULL},
+
+	{"MFX_2_6_0_9", OP_MFX_2_6_0_9, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL},
+
+	{"MFX_2_6_0_8", OP_MFX_2_6_0_8, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL},
+
+	{"MFX_JPEG_PIC_STATE", OP_MFX_JPEG_PIC_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFX_JPEG_HUFF_TABLE_STATE", OP_MFX_JPEG_HUFF_TABLE_STATE, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"MFD_JPEG_BSD_OBJECT", OP_MFD_JPEG_BSD_OBJECT, F_LEN_VAR,
+		R_VCS, D_ALL, 0, 12, NULL},
+
+	{"VEBOX_STATE", OP_VEB_STATE, F_LEN_VAR, R_VECS, D_ALL, 0, 12, NULL},
+
+	{"VEBOX_SURFACE_STATE", OP_VEB_SURFACE_STATE, F_LEN_VAR, R_VECS, D_ALL,
+		0, 12, NULL},
+
+	{"VEB_DI_IECP", OP_VEB_DNDI_IECP_STATE, F_LEN_VAR, R_VECS, D_BDW_PLUS,
+		0, 20, NULL},
+};
+
+static void add_cmd_entry(struct intel_gvt *gvt, struct cmd_entry *e)
+{
+	hash_add(gvt->cmd_table, &e->hlist, e->info->opcode);
+}
+
+#define GVT_MAX_CMD_LENGTH     20  /* In Dword */
+
+static void trace_cs_command(struct parser_exec_state *s,
+		cycles_t cost_pre_cmd_handler, cycles_t cost_cmd_handler)
+{
+	/* This buffer is used by ftrace to store all commands copied from
+	 * guest gma space. Sometimes commands can cross pages, this should
+	 * not be handled in ftrace logic. So this is just used as a
+	 * 'bounce buffer'
+	 */
+	u32 cmd_trace_buf[GVT_MAX_CMD_LENGTH];
+	int i;
+	u32 cmd_len = cmd_length(s);
+	/* The chosen value of GVT_MAX_CMD_LENGTH are just based on
+	 * following two considerations:
+	 * 1) From observation, most common ring commands is not that long.
+	 *    But there are execeptions. So it indeed makes sence to observe
+	 *    longer commands.
+	 * 2) From the performance and debugging point of view, dumping all
+	 *    contents of very commands is not necessary.
+	 * We mgith shrink GVT_MAX_CMD_LENGTH or remove this trace event in
+	 * future for performance considerations.
+	 */
+	if (unlikely(cmd_len > GVT_MAX_CMD_LENGTH)) {
+		gvt_dbg_cmd("cmd length exceed tracing limitation!\n");
+		cmd_len = GVT_MAX_CMD_LENGTH;
+	}
+
+	for (i = 0; i < cmd_len; i++)
+		cmd_trace_buf[i] = cmd_val(s, i);
+
+	trace_gvt_command(s->vgpu->id, s->ring_id, s->ip_gma, cmd_trace_buf,
+			cmd_len, s->buf_type == RING_BUFFER_INSTRUCTION,
+			cost_pre_cmd_handler, cost_cmd_handler);
+}
+
+/* call the cmd handler, and advance ip */
+static int cmd_parser_exec(struct parser_exec_state *s)
+{
+	struct cmd_info *info;
+	u32 cmd;
+	int ret = 0;
+	cycles_t t0, t1, t2;
+	struct parser_exec_state s_before_advance_custom;
+
+	t0 = get_cycles();
+
+	cmd = cmd_val(s, 0);
+
+	info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
+	if (info == NULL) {
+		gvt_err("unknown cmd 0x%x, opcode=0x%x\n",
+				cmd, get_opcode(cmd, s->ring_id));
+		return -EINVAL;
+	}
+
+	gvt_dbg_cmd("%s\n", info->name);
+
+	s->info = info;
+
+	t1 = get_cycles();
+
+	memcpy(&s_before_advance_custom, s, sizeof(struct parser_exec_state));
+
+	if (info->handler) {
+		ret = info->handler(s);
+		if (ret < 0) {
+			gvt_err("%s handler error\n", info->name);
+			return ret;
+		}
+	}
+	t2 = get_cycles();
+
+	trace_cs_command(&s_before_advance_custom, t1 - t0, t2 - t1);
+
+	if (!(info->flag & F_IP_ADVANCE_CUSTOM)) {
+		ret = cmd_advance_default(s);
+		if (ret) {
+			gvt_err("%s IP advance error\n", info->name);
+			return ret;
+		}
+	}
+	return 0;
+}
+
+static inline bool gma_out_of_range(unsigned long gma,
+		unsigned long gma_head, unsigned int gma_tail)
+{
+	if (gma_tail >= gma_head)
+		return (gma < gma_head) || (gma > gma_tail);
+	else
+		return (gma > gma_tail) && (gma < gma_head);
+}
+
+static int command_scan(struct parser_exec_state *s,
+		unsigned long rb_head, unsigned long rb_tail,
+		unsigned long rb_start, unsigned long rb_len)
+{
+
+	unsigned long gma_head, gma_tail, gma_bottom;
+	int ret = 0;
+
+	gma_head = rb_start + rb_head;
+	gma_tail = rb_start + rb_tail;
+	gma_bottom = rb_start +  rb_len;
+
+	gvt_dbg_cmd("scan_start: start=%lx end=%lx\n", gma_head, gma_tail);
+
+	while (s->ip_gma != gma_tail) {
+		if (s->buf_type == RING_BUFFER_INSTRUCTION) {
+			if (!(s->ip_gma >= rb_start) ||
+				!(s->ip_gma < gma_bottom)) {
+				gvt_err("ip_gma %lx out of ring scope."
+					"(base:0x%lx, bottom: 0x%lx)\n",
+					s->ip_gma, rb_start,
+					gma_bottom);
+				parser_exec_state_dump(s);
+				return -EINVAL;
+			}
+			if (gma_out_of_range(s->ip_gma, gma_head, gma_tail)) {
+				gvt_err("ip_gma %lx out of range."
+					"base 0x%lx head 0x%lx tail 0x%lx\n",
+					s->ip_gma, rb_start,
+					rb_head, rb_tail);
+				parser_exec_state_dump(s);
+				break;
+			}
+		}
+		ret = cmd_parser_exec(s);
+		if (ret) {
+			gvt_err("cmd parser error\n");
+			parser_exec_state_dump(s);
+			break;
+		}
+	}
+
+	gvt_dbg_cmd("scan_end\n");
+
+	return ret;
+}
+
+static int scan_workload(struct intel_vgpu_workload *workload)
+{
+	unsigned long gma_head, gma_tail, gma_bottom;
+	struct parser_exec_state s;
+	int ret = 0;
+
+	/* ring base is page aligned */
+	if (WARN_ON(!IS_ALIGNED(workload->rb_start, GTT_PAGE_SIZE)))
+		return -EINVAL;
+
+	gma_head = workload->rb_start + workload->rb_head;
+	gma_tail = workload->rb_start + workload->rb_tail;
+	gma_bottom = workload->rb_start +  _RING_CTL_BUF_SIZE(workload->rb_ctl);
+
+	s.buf_type = RING_BUFFER_INSTRUCTION;
+	s.buf_addr_type = GTT_BUFFER;
+	s.vgpu = workload->vgpu;
+	s.ring_id = workload->ring_id;
+	s.ring_start = workload->rb_start;
+	s.ring_size = _RING_CTL_BUF_SIZE(workload->rb_ctl);
+	s.ring_head = gma_head;
+	s.ring_tail = gma_tail;
+	s.rb_va = workload->shadow_ring_buffer_va;
+	s.workload = workload;
+
+	if ((bypass_scan_mask & (1 << workload->ring_id)) ||
+		gma_head == gma_tail)
+		return 0;
+
+	ret = ip_gma_set(&s, gma_head);
+	if (ret)
+		goto out;
+
+	ret = command_scan(&s, workload->rb_head, workload->rb_tail,
+		workload->rb_start, _RING_CTL_BUF_SIZE(workload->rb_ctl));
+
+out:
+	return ret;
+}
+
+static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+
+	unsigned long gma_head, gma_tail, gma_bottom, ring_size, ring_tail;
+	struct parser_exec_state s;
+	int ret = 0;
+
+	/* ring base is page aligned */
+	if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma, GTT_PAGE_SIZE)))
+		return -EINVAL;
+
+	ring_tail = wa_ctx->indirect_ctx.size + 3 * sizeof(uint32_t);
+	ring_size = round_up(wa_ctx->indirect_ctx.size + CACHELINE_BYTES,
+			PAGE_SIZE);
+	gma_head = wa_ctx->indirect_ctx.guest_gma;
+	gma_tail = wa_ctx->indirect_ctx.guest_gma + ring_tail;
+	gma_bottom = wa_ctx->indirect_ctx.guest_gma + ring_size;
+
+	s.buf_type = RING_BUFFER_INSTRUCTION;
+	s.buf_addr_type = GTT_BUFFER;
+	s.vgpu = wa_ctx->workload->vgpu;
+	s.ring_id = wa_ctx->workload->ring_id;
+	s.ring_start = wa_ctx->indirect_ctx.guest_gma;
+	s.ring_size = ring_size;
+	s.ring_head = gma_head;
+	s.ring_tail = gma_tail;
+	s.rb_va = wa_ctx->indirect_ctx.shadow_va;
+	s.workload = wa_ctx->workload;
+
+	ret = ip_gma_set(&s, gma_head);
+	if (ret)
+		goto out;
+
+	ret = command_scan(&s, 0, ring_tail,
+		wa_ctx->indirect_ctx.guest_gma, ring_size);
+out:
+	return ret;
+}
+
+static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload)
+{
+	struct intel_vgpu *vgpu = workload->vgpu;
+	int ring_id = workload->ring_id;
+	struct i915_gem_context *shadow_ctx = vgpu->shadow_ctx;
+	struct intel_ring *ring = shadow_ctx->engine[ring_id].ring;
+	unsigned long gma_head, gma_tail, gma_top, guest_rb_size;
+	unsigned int copy_len = 0;
+	int ret;
+
+	guest_rb_size = _RING_CTL_BUF_SIZE(workload->rb_ctl);
+
+	/* calculate workload ring buffer size */
+	workload->rb_len = (workload->rb_tail + guest_rb_size -
+			workload->rb_head) % guest_rb_size;
+
+	gma_head = workload->rb_start + workload->rb_head;
+	gma_tail = workload->rb_start + workload->rb_tail;
+	gma_top = workload->rb_start + guest_rb_size;
+
+	/* allocate shadow ring buffer */
+	ret = intel_ring_begin(workload->req, workload->rb_len / 4);
+	if (ret)
+		return ret;
+
+	/* get shadow ring buffer va */
+	workload->shadow_ring_buffer_va = ring->vaddr + ring->tail;
+
+	/* head > tail --> copy head <-> top */
+	if (gma_head > gma_tail) {
+		ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm,
+				gma_head, gma_top,
+				workload->shadow_ring_buffer_va);
+		if (ret) {
+			gvt_err("fail to copy guest ring buffer\n");
+			return ret;
+		}
+		copy_len = gma_top - gma_head;
+		gma_head = workload->rb_start;
+	}
+
+	/* copy head or start <-> tail */
+	ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm,
+			gma_head, gma_tail,
+			workload->shadow_ring_buffer_va + copy_len);
+	if (ret) {
+		gvt_err("fail to copy guest ring buffer\n");
+		return ret;
+	}
+	ring->tail += workload->rb_len;
+	intel_ring_advance(ring);
+	return 0;
+}
+
+int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
+{
+	int ret;
+
+	ret = shadow_workload_ring_buffer(workload);
+	if (ret) {
+		gvt_err("fail to shadow workload ring_buffer\n");
+		return ret;
+	}
+
+	ret = scan_workload(workload);
+	if (ret) {
+		gvt_err("scan workload error\n");
+		return ret;
+	}
+	return 0;
+}
+
+static int shadow_indirect_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+	struct drm_device *dev = &wa_ctx->workload->vgpu->gvt->dev_priv->drm;
+	int ctx_size = wa_ctx->indirect_ctx.size;
+	unsigned long guest_gma = wa_ctx->indirect_ctx.guest_gma;
+	struct drm_i915_gem_object *obj;
+	int ret = 0;
+	void *map;
+
+	obj = i915_gem_object_create(dev,
+				     roundup(ctx_size + CACHELINE_BYTES,
+					     PAGE_SIZE));
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
+
+	/* get the va of the shadow batch buffer */
+	map = i915_gem_object_pin_map(obj, I915_MAP_WB);
+	if (IS_ERR(map)) {
+		gvt_err("failed to vmap shadow indirect ctx\n");
+		ret = PTR_ERR(map);
+		goto put_obj;
+	}
+
+	ret = i915_gem_object_set_to_cpu_domain(obj, false);
+	if (ret) {
+		gvt_err("failed to set shadow indirect ctx to CPU\n");
+		goto unmap_src;
+	}
+
+	ret = copy_gma_to_hva(wa_ctx->workload->vgpu,
+				wa_ctx->workload->vgpu->gtt.ggtt_mm,
+				guest_gma, guest_gma + ctx_size,
+				map);
+	if (ret) {
+		gvt_err("fail to copy guest indirect ctx\n");
+		goto unmap_src;
+	}
+
+	wa_ctx->indirect_ctx.obj = obj;
+	wa_ctx->indirect_ctx.shadow_va = map;
+	return 0;
+
+unmap_src:
+	i915_gem_object_unpin_map(obj);
+put_obj:
+	i915_gem_object_put(wa_ctx->indirect_ctx.obj);
+	return ret;
+}
+
+static int combine_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+	uint32_t per_ctx_start[CACHELINE_DWORDS] = {0};
+	unsigned char *bb_start_sva;
+
+	per_ctx_start[0] = 0x18800001;
+	per_ctx_start[1] = wa_ctx->per_ctx.guest_gma;
+
+	bb_start_sva = (unsigned char *)wa_ctx->indirect_ctx.shadow_va +
+				wa_ctx->indirect_ctx.size;
+
+	memcpy(bb_start_sva, per_ctx_start, CACHELINE_BYTES);
+
+	return 0;
+}
+
+int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+	int ret;
+
+	if (wa_ctx->indirect_ctx.size == 0)
+		return 0;
+
+	ret = shadow_indirect_ctx(wa_ctx);
+	if (ret) {
+		gvt_err("fail to shadow indirect ctx\n");
+		return ret;
+	}
+
+	combine_wa_ctx(wa_ctx);
+
+	ret = scan_wa_ctx(wa_ctx);
+	if (ret) {
+		gvt_err("scan wa ctx error\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static struct cmd_info *find_cmd_entry_any_ring(struct intel_gvt *gvt,
+		unsigned int opcode, int rings)
+{
+	struct cmd_info *info = NULL;
+	unsigned int ring;
+
+	for_each_set_bit(ring, (unsigned long *)&rings, I915_NUM_ENGINES) {
+		info = find_cmd_entry(gvt, opcode, ring);
+		if (info)
+			break;
+	}
+	return info;
+}
+
+static int init_cmd_table(struct intel_gvt *gvt)
+{
+	int i;
+	struct cmd_entry *e;
+	struct cmd_info	*info;
+	unsigned int gen_type;
+
+	gen_type = intel_gvt_get_device_type(gvt);
+
+	for (i = 0; i < ARRAY_SIZE(cmd_info); i++) {
+		if (!(cmd_info[i].devices & gen_type))
+			continue;
+
+		e = kzalloc(sizeof(*e), GFP_KERNEL);
+		if (!e)
+			return -ENOMEM;
+
+		e->info = &cmd_info[i];
+		info = find_cmd_entry_any_ring(gvt,
+				e->info->opcode, e->info->rings);
+		if (info) {
+			gvt_err("%s %s duplicated\n", e->info->name,
+					info->name);
+			return -EEXIST;
+		}
+
+		INIT_HLIST_NODE(&e->hlist);
+		add_cmd_entry(gvt, e);
+		gvt_dbg_cmd("add %-30s op %04x flag %x devs %02x rings %02x\n",
+				e->info->name, e->info->opcode, e->info->flag,
+				e->info->devices, e->info->rings);
+	}
+	return 0;
+}
+
+static void clean_cmd_table(struct intel_gvt *gvt)
+{
+	struct hlist_node *tmp;
+	struct cmd_entry *e;
+	int i;
+
+	hash_for_each_safe(gvt->cmd_table, i, tmp, e, hlist)
+		kfree(e);
+
+	hash_init(gvt->cmd_table);
+}
+
+void intel_gvt_clean_cmd_parser(struct intel_gvt *gvt)
+{
+	clean_cmd_table(gvt);
+}
+
+int intel_gvt_init_cmd_parser(struct intel_gvt *gvt)
+{
+	int ret;
+
+	ret = init_cmd_table(gvt);
+	if (ret) {
+		intel_gvt_clean_cmd_parser(gvt);
+		return ret;
+	}
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.h b/drivers/gpu/drm/i915/gvt/cmd_parser.h
new file mode 100644
index 000000000000..bed33514103c
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Yulei Zhang <yulei.zhang@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+#ifndef _GVT_CMD_PARSER_H_
+#define _GVT_CMD_PARSER_H_
+
+#define GVT_CMD_HASH_BITS 7
+
+void intel_gvt_clean_cmd_parser(struct intel_gvt *gvt);
+
+int intel_gvt_init_cmd_parser(struct intel_gvt *gvt);
+
+int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload);
+
+int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx);
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/debug.h b/drivers/gpu/drm/i915/gvt/debug.h
index 7ef412be665f..68cba7bd980a 100644
--- a/drivers/gpu/drm/i915/gvt/debug.h
+++ b/drivers/gpu/drm/i915/gvt/debug.h
@@ -24,11 +24,34 @@
 #ifndef __GVT_DEBUG_H__
 #define __GVT_DEBUG_H__
 
+#define gvt_err(fmt, args...) \
+	DRM_ERROR("gvt: "fmt, ##args)
+
 #define gvt_dbg_core(fmt, args...) \
 	DRM_DEBUG_DRIVER("gvt: core: "fmt, ##args)
 
-/*
- * Other GVT debug stuff will be introduced in the GVT device model patches.
- */
+#define gvt_dbg_irq(fmt, args...) \
+	DRM_DEBUG_DRIVER("gvt: irq: "fmt, ##args)
+
+#define gvt_dbg_mm(fmt, args...) \
+	DRM_DEBUG_DRIVER("gvt: mm: "fmt, ##args)
+
+#define gvt_dbg_mmio(fmt, args...) \
+	DRM_DEBUG_DRIVER("gvt: mmio: "fmt, ##args)
+
+#define gvt_dbg_dpy(fmt, args...) \
+	DRM_DEBUG_DRIVER("gvt: dpy: "fmt, ##args)
+
+#define gvt_dbg_el(fmt, args...) \
+	DRM_DEBUG_DRIVER("gvt: el: "fmt, ##args)
+
+#define gvt_dbg_sched(fmt, args...) \
+	DRM_DEBUG_DRIVER("gvt: sched: "fmt, ##args)
+
+#define gvt_dbg_render(fmt, args...) \
+	DRM_DEBUG_DRIVER("gvt: render: "fmt, ##args)
+
+#define gvt_dbg_cmd(fmt, args...) \
+	DRM_DEBUG_DRIVER("gvt: cmd: "fmt, ##args)
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c
new file mode 100644
index 000000000000..c0c884aeb30e
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/display.c
@@ -0,0 +1,330 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Terrence Xu <terrence.xu@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+static int get_edp_pipe(struct intel_vgpu *vgpu)
+{
+	u32 data = vgpu_vreg(vgpu, _TRANS_DDI_FUNC_CTL_EDP);
+	int pipe = -1;
+
+	switch (data & TRANS_DDI_EDP_INPUT_MASK) {
+	case TRANS_DDI_EDP_INPUT_A_ON:
+	case TRANS_DDI_EDP_INPUT_A_ONOFF:
+		pipe = PIPE_A;
+		break;
+	case TRANS_DDI_EDP_INPUT_B_ONOFF:
+		pipe = PIPE_B;
+		break;
+	case TRANS_DDI_EDP_INPUT_C_ONOFF:
+		pipe = PIPE_C;
+		break;
+	}
+	return pipe;
+}
+
+static int edp_pipe_is_enabled(struct intel_vgpu *vgpu)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+	if (!(vgpu_vreg(vgpu, PIPECONF(_PIPE_EDP)) & PIPECONF_ENABLE))
+		return 0;
+
+	if (!(vgpu_vreg(vgpu, _TRANS_DDI_FUNC_CTL_EDP) & TRANS_DDI_FUNC_ENABLE))
+		return 0;
+	return 1;
+}
+
+static int pipe_is_enabled(struct intel_vgpu *vgpu, int pipe)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+	if (WARN_ON(pipe < PIPE_A || pipe >= I915_MAX_PIPES))
+		return -EINVAL;
+
+	if (vgpu_vreg(vgpu, PIPECONF(pipe)) & PIPECONF_ENABLE)
+		return 1;
+
+	if (edp_pipe_is_enabled(vgpu) &&
+			get_edp_pipe(vgpu) == pipe)
+		return 1;
+	return 0;
+}
+
+/* EDID with 1024x768 as its resolution */
+static unsigned char virtual_dp_monitor_edid[] = {
+	/*Header*/
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00,
+	/* Vendor & Product Identification */
+	0x22, 0xf0, 0x54, 0x29, 0x00, 0x00, 0x00, 0x00, 0x04, 0x17,
+	/* Version & Revision */
+	0x01, 0x04,
+	/* Basic Display Parameters & Features */
+	0xa5, 0x34, 0x20, 0x78, 0x23,
+	/* Color Characteristics */
+	0xfc, 0x81, 0xa4, 0x55, 0x4d, 0x9d, 0x25, 0x12, 0x50, 0x54,
+	/* Established Timings: maximum resolution is 1024x768 */
+	0x21, 0x08, 0x00,
+	/* Standard Timings. All invalid */
+	0x00, 0xc0, 0x00, 0xc0, 0x00, 0x40, 0x00, 0x80, 0x00, 0x00,
+	0x00, 0x40, 0x00, 0x00, 0x00, 0x01,
+	/* 18 Byte Data Blocks 1: invalid */
+	0x00, 0x00, 0x80, 0xa0, 0x70, 0xb0,
+	0x23, 0x40, 0x30, 0x20, 0x36, 0x00, 0x06, 0x44, 0x21, 0x00, 0x00, 0x1a,
+	/* 18 Byte Data Blocks 2: invalid */
+	0x00, 0x00, 0x00, 0xfd, 0x00, 0x18, 0x3c, 0x18, 0x50, 0x11, 0x00, 0x0a,
+	0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+	/* 18 Byte Data Blocks 3: invalid */
+	0x00, 0x00, 0x00, 0xfc, 0x00, 0x48,
+	0x50, 0x20, 0x5a, 0x52, 0x32, 0x34, 0x34, 0x30, 0x77, 0x0a, 0x20, 0x20,
+	/* 18 Byte Data Blocks 4: invalid */
+	0x00, 0x00, 0x00, 0xff, 0x00, 0x43, 0x4e, 0x34, 0x33, 0x30, 0x34, 0x30,
+	0x44, 0x58, 0x51, 0x0a, 0x20, 0x20,
+	/* Extension Block Count */
+	0x00,
+	/* Checksum */
+	0xef,
+};
+
+#define DPCD_HEADER_SIZE        0xb
+
+static u8 dpcd_fix_data[DPCD_HEADER_SIZE] = {
+	0x11, 0x0a, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+static void emulate_monitor_status_change(struct intel_vgpu *vgpu)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	vgpu_vreg(vgpu, SDEISR) &= ~(SDE_PORTB_HOTPLUG_CPT |
+			SDE_PORTC_HOTPLUG_CPT |
+			SDE_PORTD_HOTPLUG_CPT);
+
+	if (IS_SKYLAKE(dev_priv))
+		vgpu_vreg(vgpu, SDEISR) &= ~(SDE_PORTA_HOTPLUG_SPT |
+				SDE_PORTE_HOTPLUG_SPT);
+
+	if (intel_vgpu_has_monitor_on_port(vgpu, PORT_B))
+		vgpu_vreg(vgpu, SDEISR) |= SDE_PORTB_HOTPLUG_CPT;
+
+	if (intel_vgpu_has_monitor_on_port(vgpu, PORT_C))
+		vgpu_vreg(vgpu, SDEISR) |= SDE_PORTC_HOTPLUG_CPT;
+
+	if (intel_vgpu_has_monitor_on_port(vgpu, PORT_D))
+		vgpu_vreg(vgpu, SDEISR) |= SDE_PORTD_HOTPLUG_CPT;
+
+	if (IS_SKYLAKE(dev_priv) &&
+			intel_vgpu_has_monitor_on_port(vgpu, PORT_E)) {
+		vgpu_vreg(vgpu, SDEISR) |= SDE_PORTE_HOTPLUG_SPT;
+	}
+
+	if (intel_vgpu_has_monitor_on_port(vgpu, PORT_A)) {
+		if (IS_BROADWELL(dev_priv))
+			vgpu_vreg(vgpu, GEN8_DE_PORT_ISR) |=
+				GEN8_PORT_DP_A_HOTPLUG;
+		else
+			vgpu_vreg(vgpu, SDEISR) |= SDE_PORTA_HOTPLUG_SPT;
+	}
+}
+
+static void clean_virtual_dp_monitor(struct intel_vgpu *vgpu, int port_num)
+{
+	struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
+
+	kfree(port->edid);
+	port->edid = NULL;
+
+	kfree(port->dpcd);
+	port->dpcd = NULL;
+}
+
+static int setup_virtual_dp_monitor(struct intel_vgpu *vgpu, int port_num,
+		int type)
+{
+	struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
+
+	port->edid = kzalloc(sizeof(*(port->edid)), GFP_KERNEL);
+	if (!port->edid)
+		return -ENOMEM;
+
+	port->dpcd = kzalloc(sizeof(*(port->dpcd)), GFP_KERNEL);
+	if (!port->dpcd) {
+		kfree(port->edid);
+		return -ENOMEM;
+	}
+
+	memcpy(port->edid->edid_block, virtual_dp_monitor_edid,
+			EDID_SIZE);
+	port->edid->data_valid = true;
+
+	memcpy(port->dpcd->data, dpcd_fix_data, DPCD_HEADER_SIZE);
+	port->dpcd->data_valid = true;
+	port->dpcd->data[DPCD_SINK_COUNT] = 0x1;
+	port->type = type;
+
+	emulate_monitor_status_change(vgpu);
+	return 0;
+}
+
+/**
+ * intel_gvt_check_vblank_emulation - check if vblank emulation timer should
+ * be turned on/off when a virtual pipe is enabled/disabled.
+ * @gvt: a GVT device
+ *
+ * This function is used to turn on/off vblank timer according to currently
+ * enabled/disabled virtual pipes.
+ *
+ */
+void intel_gvt_check_vblank_emulation(struct intel_gvt *gvt)
+{
+	struct intel_gvt_irq *irq = &gvt->irq;
+	struct intel_vgpu *vgpu;
+	bool have_enabled_pipe = false;
+	int pipe, id;
+
+	if (WARN_ON(!mutex_is_locked(&gvt->lock)))
+		return;
+
+	hrtimer_cancel(&irq->vblank_timer.timer);
+
+	for_each_active_vgpu(gvt, vgpu, id) {
+		for (pipe = 0; pipe < I915_MAX_PIPES; pipe++) {
+			have_enabled_pipe =
+				pipe_is_enabled(vgpu, pipe);
+			if (have_enabled_pipe)
+				break;
+		}
+	}
+
+	if (have_enabled_pipe)
+		hrtimer_start(&irq->vblank_timer.timer,
+			ktime_add_ns(ktime_get(), irq->vblank_timer.period),
+			HRTIMER_MODE_ABS);
+}
+
+static void emulate_vblank_on_pipe(struct intel_vgpu *vgpu, int pipe)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	struct intel_vgpu_irq *irq = &vgpu->irq;
+	int vblank_event[] = {
+		[PIPE_A] = PIPE_A_VBLANK,
+		[PIPE_B] = PIPE_B_VBLANK,
+		[PIPE_C] = PIPE_C_VBLANK,
+	};
+	int event;
+
+	if (pipe < PIPE_A || pipe > PIPE_C)
+		return;
+
+	for_each_set_bit(event, irq->flip_done_event[pipe],
+			INTEL_GVT_EVENT_MAX) {
+		clear_bit(event, irq->flip_done_event[pipe]);
+		if (!pipe_is_enabled(vgpu, pipe))
+			continue;
+
+		vgpu_vreg(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++;
+		intel_vgpu_trigger_virtual_event(vgpu, event);
+	}
+
+	if (pipe_is_enabled(vgpu, pipe)) {
+		vgpu_vreg(vgpu, PIPE_FRMCOUNT_G4X(pipe))++;
+		intel_vgpu_trigger_virtual_event(vgpu, vblank_event[pipe]);
+	}
+}
+
+static void emulate_vblank(struct intel_vgpu *vgpu)
+{
+	int pipe;
+
+	for_each_pipe(vgpu->gvt->dev_priv, pipe)
+		emulate_vblank_on_pipe(vgpu, pipe);
+}
+
+/**
+ * intel_gvt_emulate_vblank - trigger vblank events for vGPUs on GVT device
+ * @gvt: a GVT device
+ *
+ * This function is used to trigger vblank interrupts for vGPUs on GVT device
+ *
+ */
+void intel_gvt_emulate_vblank(struct intel_gvt *gvt)
+{
+	struct intel_vgpu *vgpu;
+	int id;
+
+	if (WARN_ON(!mutex_is_locked(&gvt->lock)))
+		return;
+
+	for_each_active_vgpu(gvt, vgpu, id)
+		emulate_vblank(vgpu);
+}
+
+/**
+ * intel_vgpu_clean_display - clean vGPU virtual display emulation
+ * @vgpu: a vGPU
+ *
+ * This function is used to clean vGPU virtual display emulation stuffs
+ *
+ */
+void intel_vgpu_clean_display(struct intel_vgpu *vgpu)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+	if (IS_SKYLAKE(dev_priv))
+		clean_virtual_dp_monitor(vgpu, PORT_D);
+	else
+		clean_virtual_dp_monitor(vgpu, PORT_B);
+}
+
+/**
+ * intel_vgpu_init_display- initialize vGPU virtual display emulation
+ * @vgpu: a vGPU
+ *
+ * This function is used to initialize vGPU virtual display emulation stuffs
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_init_display(struct intel_vgpu *vgpu)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+	intel_vgpu_init_i2c_edid(vgpu);
+
+	if (IS_SKYLAKE(dev_priv))
+		return setup_virtual_dp_monitor(vgpu, PORT_D, GVT_DP_D);
+	else
+		return setup_virtual_dp_monitor(vgpu, PORT_B, GVT_DP_B);
+}
diff --git a/drivers/gpu/drm/i915/gvt/display.h b/drivers/gpu/drm/i915/gvt/display.h
new file mode 100644
index 000000000000..7a60cb848268
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/display.h
@@ -0,0 +1,163 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Terrence Xu <terrence.xu@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#ifndef _GVT_DISPLAY_H_
+#define _GVT_DISPLAY_H_
+
+#define SBI_REG_MAX	20
+#define DPCD_SIZE	0x700
+
+#define intel_vgpu_port(vgpu, port) \
+	(&(vgpu->display.ports[port]))
+
+#define intel_vgpu_has_monitor_on_port(vgpu, port) \
+	(intel_vgpu_port(vgpu, port)->edid && \
+		intel_vgpu_port(vgpu, port)->edid->data_valid)
+
+#define intel_vgpu_port_is_dp(vgpu, port) \
+	((intel_vgpu_port(vgpu, port)->type == GVT_DP_A) || \
+	(intel_vgpu_port(vgpu, port)->type == GVT_DP_B) || \
+	(intel_vgpu_port(vgpu, port)->type == GVT_DP_C) || \
+	(intel_vgpu_port(vgpu, port)->type == GVT_DP_D))
+
+#define INTEL_GVT_MAX_UEVENT_VARS	3
+
+/* DPCD start */
+#define DPCD_SIZE	0x700
+
+/* DPCD */
+#define DP_SET_POWER            0x600
+#define DP_SET_POWER_D0         0x1
+#define AUX_NATIVE_WRITE        0x8
+#define AUX_NATIVE_READ         0x9
+
+#define AUX_NATIVE_REPLY_MASK   (0x3 << 4)
+#define AUX_NATIVE_REPLY_ACK    (0x0 << 4)
+#define AUX_NATIVE_REPLY_NAK    (0x1 << 4)
+#define AUX_NATIVE_REPLY_DEFER  (0x2 << 4)
+
+#define AUX_BURST_SIZE          16
+
+/* DPCD addresses */
+#define DPCD_REV			0x000
+#define DPCD_MAX_LINK_RATE		0x001
+#define DPCD_MAX_LANE_COUNT		0x002
+
+#define DPCD_TRAINING_PATTERN_SET	0x102
+#define	DPCD_SINK_COUNT			0x200
+#define DPCD_LANE0_1_STATUS		0x202
+#define DPCD_LANE2_3_STATUS		0x203
+#define DPCD_LANE_ALIGN_STATUS_UPDATED	0x204
+#define DPCD_SINK_STATUS		0x205
+
+/* link training */
+#define DPCD_TRAINING_PATTERN_SET_MASK	0x03
+#define DPCD_LINK_TRAINING_DISABLED	0x00
+#define DPCD_TRAINING_PATTERN_1		0x01
+#define DPCD_TRAINING_PATTERN_2		0x02
+
+#define DPCD_CP_READY_MASK		(1 << 6)
+
+/* lane status */
+#define DPCD_LANES_CR_DONE		0x11
+#define DPCD_LANES_EQ_DONE		0x22
+#define DPCD_SYMBOL_LOCKED		0x44
+
+#define DPCD_INTERLANE_ALIGN_DONE	0x01
+
+#define DPCD_SINK_IN_SYNC		0x03
+/* DPCD end */
+
+#define SBI_RESPONSE_MASK               0x3
+#define SBI_RESPONSE_SHIFT              0x1
+#define SBI_STAT_MASK                   0x1
+#define SBI_STAT_SHIFT                  0x0
+#define SBI_OPCODE_SHIFT                8
+#define SBI_OPCODE_MASK			(0xff << SBI_OPCODE_SHIFT)
+#define SBI_CMD_IORD                    2
+#define SBI_CMD_IOWR                    3
+#define SBI_CMD_CRRD                    6
+#define SBI_CMD_CRWR                    7
+#define SBI_ADDR_OFFSET_SHIFT           16
+#define SBI_ADDR_OFFSET_MASK            (0xffff << SBI_ADDR_OFFSET_SHIFT)
+
+struct intel_vgpu_sbi_register {
+	unsigned int offset;
+	u32 value;
+};
+
+struct intel_vgpu_sbi {
+	int number;
+	struct intel_vgpu_sbi_register registers[SBI_REG_MAX];
+};
+
+enum intel_gvt_plane_type {
+	PRIMARY_PLANE = 0,
+	CURSOR_PLANE,
+	SPRITE_PLANE,
+	MAX_PLANE
+};
+
+struct intel_vgpu_dpcd_data {
+	bool data_valid;
+	u8 data[DPCD_SIZE];
+};
+
+enum intel_vgpu_port_type {
+	GVT_CRT = 0,
+	GVT_DP_A,
+	GVT_DP_B,
+	GVT_DP_C,
+	GVT_DP_D,
+	GVT_HDMI_B,
+	GVT_HDMI_C,
+	GVT_HDMI_D,
+	GVT_PORT_MAX
+};
+
+struct intel_vgpu_port {
+	/* per display EDID information */
+	struct intel_vgpu_edid_data *edid;
+	/* per display DPCD information */
+	struct intel_vgpu_dpcd_data *dpcd;
+	int type;
+};
+
+void intel_gvt_emulate_vblank(struct intel_gvt *gvt);
+void intel_gvt_check_vblank_emulation(struct intel_gvt *gvt);
+
+int intel_vgpu_init_display(struct intel_vgpu *vgpu);
+void intel_vgpu_clean_display(struct intel_vgpu *vgpu);
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c
new file mode 100644
index 000000000000..bda85dff7b2a
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/edid.c
@@ -0,0 +1,531 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Terrence Xu <terrence.xu@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+#define GMBUS1_TOTAL_BYTES_SHIFT 16
+#define GMBUS1_TOTAL_BYTES_MASK 0x1ff
+#define gmbus1_total_byte_count(v) (((v) >> \
+	GMBUS1_TOTAL_BYTES_SHIFT) & GMBUS1_TOTAL_BYTES_MASK)
+#define gmbus1_slave_addr(v) (((v) & 0xff) >> 1)
+#define gmbus1_slave_index(v) (((v) >> 8) & 0xff)
+#define gmbus1_bus_cycle(v) (((v) >> 25) & 0x7)
+
+/* GMBUS0 bits definitions */
+#define _GMBUS_PIN_SEL_MASK     (0x7)
+
+static unsigned char edid_get_byte(struct intel_vgpu *vgpu)
+{
+	struct intel_vgpu_i2c_edid *edid = &vgpu->display.i2c_edid;
+	unsigned char chr = 0;
+
+	if (edid->state == I2C_NOT_SPECIFIED || !edid->slave_selected) {
+		gvt_err("Driver tries to read EDID without proper sequence!\n");
+		return 0;
+	}
+	if (edid->current_edid_read >= EDID_SIZE) {
+		gvt_err("edid_get_byte() exceeds the size of EDID!\n");
+		return 0;
+	}
+
+	if (!edid->edid_available) {
+		gvt_err("Reading EDID but EDID is not available!\n");
+		return 0;
+	}
+
+	if (intel_vgpu_has_monitor_on_port(vgpu, edid->port)) {
+		struct intel_vgpu_edid_data *edid_data =
+			intel_vgpu_port(vgpu, edid->port)->edid;
+
+		chr = edid_data->edid_block[edid->current_edid_read];
+		edid->current_edid_read++;
+	} else {
+		gvt_err("No EDID available during the reading?\n");
+	}
+	return chr;
+}
+
+static inline int get_port_from_gmbus0(u32 gmbus0)
+{
+	int port_select = gmbus0 & _GMBUS_PIN_SEL_MASK;
+	int port = -EINVAL;
+
+	if (port_select == 2)
+		port = PORT_E;
+	else if (port_select == 4)
+		port = PORT_C;
+	else if (port_select == 5)
+		port = PORT_B;
+	else if (port_select == 6)
+		port = PORT_D;
+	return port;
+}
+
+static void reset_gmbus_controller(struct intel_vgpu *vgpu)
+{
+	vgpu_vreg(vgpu, PCH_GMBUS2) = GMBUS_HW_RDY;
+	if (!vgpu->display.i2c_edid.edid_available)
+		vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_SATOER;
+	vgpu->display.i2c_edid.gmbus.phase = GMBUS_IDLE_PHASE;
+}
+
+/* GMBUS0 */
+static int gmbus0_mmio_write(struct intel_vgpu *vgpu,
+			unsigned int offset, void *p_data, unsigned int bytes)
+{
+	int port, pin_select;
+
+	memcpy(&vgpu_vreg(vgpu, offset), p_data, bytes);
+
+	pin_select = vgpu_vreg(vgpu, offset) & _GMBUS_PIN_SEL_MASK;
+
+	intel_vgpu_init_i2c_edid(vgpu);
+
+	if (pin_select == 0)
+		return 0;
+
+	port = get_port_from_gmbus0(pin_select);
+	if (WARN_ON(port < 0))
+		return 0;
+
+	vgpu->display.i2c_edid.state = I2C_GMBUS;
+	vgpu->display.i2c_edid.gmbus.phase = GMBUS_IDLE_PHASE;
+
+	vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE;
+	vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY | GMBUS_HW_WAIT_PHASE;
+
+	if (intel_vgpu_has_monitor_on_port(vgpu, port) &&
+			!intel_vgpu_port_is_dp(vgpu, port)) {
+		vgpu->display.i2c_edid.port = port;
+		vgpu->display.i2c_edid.edid_available = true;
+		vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_SATOER;
+	} else
+		vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_SATOER;
+	return 0;
+}
+
+static int gmbus1_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	struct intel_vgpu_i2c_edid *i2c_edid = &vgpu->display.i2c_edid;
+	u32 slave_addr;
+	u32 wvalue = *(u32 *)p_data;
+
+	if (vgpu_vreg(vgpu, offset) & GMBUS_SW_CLR_INT) {
+		if (!(wvalue & GMBUS_SW_CLR_INT)) {
+			vgpu_vreg(vgpu, offset) &= ~GMBUS_SW_CLR_INT;
+			reset_gmbus_controller(vgpu);
+		}
+		/*
+		 * TODO: "This bit is cleared to zero when an event
+		 * causes the HW_RDY bit transition to occur "
+		 */
+	} else {
+		/*
+		 * per bspec setting this bit can cause:
+		 * 1) INT status bit cleared
+		 * 2) HW_RDY bit asserted
+		 */
+		if (wvalue & GMBUS_SW_CLR_INT) {
+			vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_INT;
+			vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY;
+		}
+
+		/* For virtualization, we suppose that HW is always ready,
+		 * so GMBUS_SW_RDY should always be cleared
+		 */
+		if (wvalue & GMBUS_SW_RDY)
+			wvalue &= ~GMBUS_SW_RDY;
+
+		i2c_edid->gmbus.total_byte_count =
+			gmbus1_total_byte_count(wvalue);
+		slave_addr = gmbus1_slave_addr(wvalue);
+
+		/* vgpu gmbus only support EDID */
+		if (slave_addr == EDID_ADDR) {
+			i2c_edid->slave_selected = true;
+		} else if (slave_addr != 0) {
+			gvt_dbg_dpy(
+				"vgpu%d: unsupported gmbus slave addr(0x%x)\n"
+				"	gmbus operations will be ignored.\n",
+					vgpu->id, slave_addr);
+		}
+
+		if (wvalue & GMBUS_CYCLE_INDEX)
+			i2c_edid->current_edid_read =
+				gmbus1_slave_index(wvalue);
+
+		i2c_edid->gmbus.cycle_type = gmbus1_bus_cycle(wvalue);
+		switch (gmbus1_bus_cycle(wvalue)) {
+		case GMBUS_NOCYCLE:
+			break;
+		case GMBUS_STOP:
+			/* From spec:
+			 * This can only cause a STOP to be generated
+			 * if a GMBUS cycle is generated, the GMBUS is
+			 * currently in a data/wait/idle phase, or it is in a
+			 * WAIT phase
+			 */
+			if (gmbus1_bus_cycle(vgpu_vreg(vgpu, offset))
+				!= GMBUS_NOCYCLE) {
+				intel_vgpu_init_i2c_edid(vgpu);
+				/* After the 'stop' cycle, hw state would become
+				 * 'stop phase' and then 'idle phase' after a
+				 * few milliseconds. In emulation, we just set
+				 * it as 'idle phase' ('stop phase' is not
+				 * visible in gmbus interface)
+				 */
+				i2c_edid->gmbus.phase = GMBUS_IDLE_PHASE;
+				vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE;
+			}
+			break;
+		case NIDX_NS_W:
+		case IDX_NS_W:
+		case NIDX_STOP:
+		case IDX_STOP:
+			/* From hw spec the GMBUS phase
+			 * transition like this:
+			 * START (-->INDEX) -->DATA
+			 */
+			i2c_edid->gmbus.phase = GMBUS_DATA_PHASE;
+			vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_ACTIVE;
+			break;
+		default:
+			gvt_err("Unknown/reserved GMBUS cycle detected!\n");
+			break;
+		}
+		/*
+		 * From hw spec the WAIT state will be
+		 * cleared:
+		 * (1) in a new GMBUS cycle
+		 * (2) by generating a stop
+		 */
+		vgpu_vreg(vgpu, offset) = wvalue;
+	}
+	return 0;
+}
+
+static int gmbus3_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+	void *p_data, unsigned int bytes)
+{
+	WARN_ON(1);
+	return 0;
+}
+
+static int gmbus3_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	int i;
+	unsigned char byte_data;
+	struct intel_vgpu_i2c_edid *i2c_edid = &vgpu->display.i2c_edid;
+	int byte_left = i2c_edid->gmbus.total_byte_count -
+				i2c_edid->current_edid_read;
+	int byte_count = byte_left;
+	u32 reg_data = 0;
+
+	/* Data can only be recevied if previous settings correct */
+	if (vgpu_vreg(vgpu, PCH_GMBUS1) & GMBUS_SLAVE_READ) {
+		if (byte_left <= 0) {
+			memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes);
+			return 0;
+		}
+
+		if (byte_count > 4)
+			byte_count = 4;
+		for (i = 0; i < byte_count; i++) {
+			byte_data = edid_get_byte(vgpu);
+			reg_data |= (byte_data << (i << 3));
+		}
+
+		memcpy(&vgpu_vreg(vgpu, offset), &reg_data, byte_count);
+		memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes);
+
+		if (byte_left <= 4) {
+			switch (i2c_edid->gmbus.cycle_type) {
+			case NIDX_STOP:
+			case IDX_STOP:
+				i2c_edid->gmbus.phase = GMBUS_IDLE_PHASE;
+				break;
+			case NIDX_NS_W:
+			case IDX_NS_W:
+			default:
+				i2c_edid->gmbus.phase = GMBUS_WAIT_PHASE;
+				break;
+			}
+			intel_vgpu_init_i2c_edid(vgpu);
+		}
+		/*
+		 * Read GMBUS3 during send operation,
+		 * return the latest written value
+		 */
+	} else {
+		memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes);
+		gvt_err("vgpu%d: warning: gmbus3 read with nothing returned\n",
+				vgpu->id);
+	}
+	return 0;
+}
+
+static int gmbus2_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 value = vgpu_vreg(vgpu, offset);
+
+	if (!(vgpu_vreg(vgpu, offset) & GMBUS_INUSE))
+		vgpu_vreg(vgpu, offset) |= GMBUS_INUSE;
+	memcpy(p_data, (void *)&value, bytes);
+	return 0;
+}
+
+static int gmbus2_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 wvalue = *(u32 *)p_data;
+
+	if (wvalue & GMBUS_INUSE)
+		vgpu_vreg(vgpu, offset) &= ~GMBUS_INUSE;
+	/* All other bits are read-only */
+	return 0;
+}
+
+/**
+ * intel_gvt_i2c_handle_gmbus_read - emulate gmbus register mmio read
+ * @vgpu: a vGPU
+ *
+ * This function is used to emulate gmbus register mmio read
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_gvt_i2c_handle_gmbus_read(struct intel_vgpu *vgpu,
+	unsigned int offset, void *p_data, unsigned int bytes)
+{
+	if (WARN_ON(bytes > 8 && (offset & (bytes - 1))))
+		return -EINVAL;
+
+	if (offset == i915_mmio_reg_offset(PCH_GMBUS2))
+		return gmbus2_mmio_read(vgpu, offset, p_data, bytes);
+	else if (offset == i915_mmio_reg_offset(PCH_GMBUS3))
+		return gmbus3_mmio_read(vgpu, offset, p_data, bytes);
+
+	memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes);
+	return 0;
+}
+
+/**
+ * intel_gvt_i2c_handle_gmbus_write - emulate gmbus register mmio write
+ * @vgpu: a vGPU
+ *
+ * This function is used to emulate gmbus register mmio write
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_gvt_i2c_handle_gmbus_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	if (WARN_ON(bytes > 8 && (offset & (bytes - 1))))
+		return -EINVAL;
+
+	if (offset == i915_mmio_reg_offset(PCH_GMBUS0))
+		return gmbus0_mmio_write(vgpu, offset, p_data, bytes);
+	else if (offset == i915_mmio_reg_offset(PCH_GMBUS1))
+		return gmbus1_mmio_write(vgpu, offset, p_data, bytes);
+	else if (offset == i915_mmio_reg_offset(PCH_GMBUS2))
+		return gmbus2_mmio_write(vgpu, offset, p_data, bytes);
+	else if (offset == i915_mmio_reg_offset(PCH_GMBUS3))
+		return gmbus3_mmio_write(vgpu, offset, p_data, bytes);
+
+	memcpy(&vgpu_vreg(vgpu, offset), p_data, bytes);
+	return 0;
+}
+
+enum {
+	AUX_CH_CTL = 0,
+	AUX_CH_DATA1,
+	AUX_CH_DATA2,
+	AUX_CH_DATA3,
+	AUX_CH_DATA4,
+	AUX_CH_DATA5
+};
+
+static inline int get_aux_ch_reg(unsigned int offset)
+{
+	int reg;
+
+	switch (offset & 0xff) {
+	case 0x10:
+		reg = AUX_CH_CTL;
+		break;
+	case 0x14:
+		reg = AUX_CH_DATA1;
+		break;
+	case 0x18:
+		reg = AUX_CH_DATA2;
+		break;
+	case 0x1c:
+		reg = AUX_CH_DATA3;
+		break;
+	case 0x20:
+		reg = AUX_CH_DATA4;
+		break;
+	case 0x24:
+		reg = AUX_CH_DATA5;
+		break;
+	default:
+		reg = -1;
+		break;
+	}
+	return reg;
+}
+
+#define AUX_CTL_MSG_LENGTH(reg) \
+	((reg & DP_AUX_CH_CTL_MESSAGE_SIZE_MASK) >> \
+		DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT)
+
+/**
+ * intel_gvt_i2c_handle_aux_ch_write - emulate AUX channel register write
+ * @vgpu: a vGPU
+ *
+ * This function is used to emulate AUX channel register write
+ *
+ */
+void intel_gvt_i2c_handle_aux_ch_write(struct intel_vgpu *vgpu,
+				int port_idx,
+				unsigned int offset,
+				void *p_data)
+{
+	struct intel_vgpu_i2c_edid *i2c_edid = &vgpu->display.i2c_edid;
+	int msg_length, ret_msg_size;
+	int msg, addr, ctrl, op;
+	u32 value = *(u32 *)p_data;
+	int aux_data_for_write = 0;
+	int reg = get_aux_ch_reg(offset);
+
+	if (reg != AUX_CH_CTL) {
+		vgpu_vreg(vgpu, offset) = value;
+		return;
+	}
+
+	msg_length = AUX_CTL_MSG_LENGTH(value);
+	// check the msg in DATA register.
+	msg = vgpu_vreg(vgpu, offset + 4);
+	addr = (msg >> 8) & 0xffff;
+	ctrl = (msg >> 24) & 0xff;
+	op = ctrl >> 4;
+	if (!(value & DP_AUX_CH_CTL_SEND_BUSY)) {
+		/* The ctl write to clear some states */
+		return;
+	}
+
+	/* Always set the wanted value for vms. */
+	ret_msg_size = (((op & 0x1) == GVT_AUX_I2C_READ) ? 2 : 1);
+	vgpu_vreg(vgpu, offset) =
+		DP_AUX_CH_CTL_DONE |
+		((ret_msg_size << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) &
+		DP_AUX_CH_CTL_MESSAGE_SIZE_MASK);
+
+	if (msg_length == 3) {
+		if (!(op & GVT_AUX_I2C_MOT)) {
+			/* stop */
+			intel_vgpu_init_i2c_edid(vgpu);
+		} else {
+			/* start or restart */
+			i2c_edid->aux_ch.i2c_over_aux_ch = true;
+			i2c_edid->aux_ch.aux_ch_mot = true;
+			if (addr == 0) {
+				/* reset the address */
+				intel_vgpu_init_i2c_edid(vgpu);
+			} else if (addr == EDID_ADDR) {
+				i2c_edid->state = I2C_AUX_CH;
+				i2c_edid->port = port_idx;
+				i2c_edid->slave_selected = true;
+				if (intel_vgpu_has_monitor_on_port(vgpu,
+					port_idx) &&
+					intel_vgpu_port_is_dp(vgpu, port_idx))
+					i2c_edid->edid_available = true;
+			}
+		}
+	} else if ((op & 0x1) == GVT_AUX_I2C_WRITE) {
+		/* TODO
+		 * We only support EDID reading from I2C_over_AUX. And
+		 * we do not expect the index mode to be used. Right now
+		 * the WRITE operation is ignored. It is good enough to
+		 * support the gfx driver to do EDID access.
+		 */
+	} else {
+		if (WARN_ON((op & 0x1) != GVT_AUX_I2C_READ))
+			return;
+		if (WARN_ON(msg_length != 4))
+			return;
+		if (i2c_edid->edid_available && i2c_edid->slave_selected) {
+			unsigned char val = edid_get_byte(vgpu);
+
+			aux_data_for_write = (val << 16);
+		}
+	}
+	/* write the return value in AUX_CH_DATA reg which includes:
+	 * ACK of I2C_WRITE
+	 * returned byte if it is READ
+	 */
+	aux_data_for_write |= GVT_AUX_I2C_REPLY_ACK << 24;
+	vgpu_vreg(vgpu, offset + 4) = aux_data_for_write;
+}
+
+/**
+ * intel_vgpu_init_i2c_edid - initialize vGPU i2c edid emulation
+ * @vgpu: a vGPU
+ *
+ * This function is used to initialize vGPU i2c edid emulation stuffs
+ *
+ */
+void intel_vgpu_init_i2c_edid(struct intel_vgpu *vgpu)
+{
+	struct intel_vgpu_i2c_edid *edid = &vgpu->display.i2c_edid;
+
+	edid->state = I2C_NOT_SPECIFIED;
+
+	edid->port = -1;
+	edid->slave_selected = false;
+	edid->edid_available = false;
+	edid->current_edid_read = 0;
+
+	memset(&edid->gmbus, 0, sizeof(struct intel_vgpu_i2c_gmbus));
+
+	edid->aux_ch.i2c_over_aux_ch = false;
+	edid->aux_ch.aux_ch_mot = false;
+}
diff --git a/drivers/gpu/drm/i915/gvt/edid.h b/drivers/gpu/drm/i915/gvt/edid.h
new file mode 100644
index 000000000000..f6dfc8b795ec
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/edid.h
@@ -0,0 +1,150 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Terrence Xu <terrence.xu@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#ifndef _GVT_EDID_H_
+#define _GVT_EDID_H_
+
+#define EDID_SIZE		128
+#define EDID_ADDR		0x50 /* Linux hvm EDID addr */
+
+#define GVT_AUX_NATIVE_WRITE			0x8
+#define GVT_AUX_NATIVE_READ			0x9
+#define GVT_AUX_I2C_WRITE			0x0
+#define GVT_AUX_I2C_READ			0x1
+#define GVT_AUX_I2C_STATUS			0x2
+#define GVT_AUX_I2C_MOT				0x4
+#define GVT_AUX_I2C_REPLY_ACK			0x0
+
+struct intel_vgpu_edid_data {
+	bool data_valid;
+	unsigned char edid_block[EDID_SIZE];
+};
+
+enum gmbus_cycle_type {
+	GMBUS_NOCYCLE	= 0x0,
+	NIDX_NS_W	= 0x1,
+	IDX_NS_W	= 0x3,
+	GMBUS_STOP	= 0x4,
+	NIDX_STOP	= 0x5,
+	IDX_STOP	= 0x7
+};
+
+/*
+ * States of GMBUS
+ *
+ * GMBUS0-3 could be related to the EDID virtualization. Another two GMBUS
+ * registers, GMBUS4 (interrupt mask) and GMBUS5 (2 byte indes register), are
+ * not considered here. Below describes the usage of GMBUS registers that are
+ * cared by the EDID virtualization
+ *
+ * GMBUS0:
+ *      R/W
+ *      port selection. value of bit0 - bit2 corresponds to the GPIO registers.
+ *
+ * GMBUS1:
+ *      R/W Protect
+ *      Command and Status.
+ *      bit0 is the direction bit: 1 is read; 0 is write.
+ *      bit1 - bit7 is slave 7-bit address.
+ *      bit16 - bit24 total byte count (ignore?)
+ *
+ * GMBUS2:
+ *      Most of bits are read only except bit 15 (IN_USE)
+ *      Status register
+ *      bit0 - bit8 current byte count
+ *      bit 11: hardware ready;
+ *
+ * GMBUS3:
+ *      Read/Write
+ *      Data for transfer
+ */
+
+/* From hw specs, Other phases like START, ADDRESS, INDEX
+ * are invisible to GMBUS MMIO interface. So no definitions
+ * in below enum types
+ */
+enum gvt_gmbus_phase {
+	GMBUS_IDLE_PHASE = 0,
+	GMBUS_DATA_PHASE,
+	GMBUS_WAIT_PHASE,
+	//GMBUS_STOP_PHASE,
+	GMBUS_MAX_PHASE
+};
+
+struct intel_vgpu_i2c_gmbus {
+	unsigned int total_byte_count; /* from GMBUS1 */
+	enum gmbus_cycle_type cycle_type;
+	enum gvt_gmbus_phase phase;
+};
+
+struct intel_vgpu_i2c_aux_ch {
+	bool i2c_over_aux_ch;
+	bool aux_ch_mot;
+};
+
+enum i2c_state {
+	I2C_NOT_SPECIFIED = 0,
+	I2C_GMBUS = 1,
+	I2C_AUX_CH = 2
+};
+
+/* I2C sequences cannot interleave.
+ * GMBUS and AUX_CH sequences cannot interleave.
+ */
+struct intel_vgpu_i2c_edid {
+	enum i2c_state state;
+
+	unsigned int port;
+	bool slave_selected;
+	bool edid_available;
+	unsigned int current_edid_read;
+
+	struct intel_vgpu_i2c_gmbus gmbus;
+	struct intel_vgpu_i2c_aux_ch aux_ch;
+};
+
+void intel_vgpu_init_i2c_edid(struct intel_vgpu *vgpu);
+
+int intel_gvt_i2c_handle_gmbus_read(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes);
+
+int intel_gvt_i2c_handle_gmbus_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes);
+
+void intel_gvt_i2c_handle_aux_ch_write(struct intel_vgpu *vgpu,
+		int port_idx,
+		unsigned int offset,
+		void *p_data);
+
+#endif /*_GVT_EDID_H_*/
diff --git a/drivers/gpu/drm/i915/gvt/execlist.c b/drivers/gpu/drm/i915/gvt/execlist.c
new file mode 100644
index 000000000000..f32bb6f6495c
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/execlist.c
@@ -0,0 +1,858 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+#define _EL_OFFSET_STATUS       0x234
+#define _EL_OFFSET_STATUS_BUF   0x370
+#define _EL_OFFSET_STATUS_PTR   0x3A0
+
+#define execlist_ring_mmio(gvt, ring_id, offset) \
+	(gvt->dev_priv->engine[ring_id]->mmio_base + (offset))
+
+#define valid_context(ctx) ((ctx)->valid)
+#define same_context(a, b) (((a)->context_id == (b)->context_id) && \
+		((a)->lrca == (b)->lrca))
+
+static int context_switch_events[] = {
+	[RCS] = RCS_AS_CONTEXT_SWITCH,
+	[BCS] = BCS_AS_CONTEXT_SWITCH,
+	[VCS] = VCS_AS_CONTEXT_SWITCH,
+	[VCS2] = VCS2_AS_CONTEXT_SWITCH,
+	[VECS] = VECS_AS_CONTEXT_SWITCH,
+};
+
+static int ring_id_to_context_switch_event(int ring_id)
+{
+	if (WARN_ON(ring_id < RCS && ring_id >
+				ARRAY_SIZE(context_switch_events)))
+		return -EINVAL;
+
+	return context_switch_events[ring_id];
+}
+
+static void switch_virtual_execlist_slot(struct intel_vgpu_execlist *execlist)
+{
+	gvt_dbg_el("[before] running slot %d/context %x pending slot %d\n",
+			execlist->running_slot ?
+			execlist->running_slot->index : -1,
+			execlist->running_context ?
+			execlist->running_context->context_id : 0,
+			execlist->pending_slot ?
+			execlist->pending_slot->index : -1);
+
+	execlist->running_slot = execlist->pending_slot;
+	execlist->pending_slot = NULL;
+	execlist->running_context = execlist->running_context ?
+		&execlist->running_slot->ctx[0] : NULL;
+
+	gvt_dbg_el("[after] running slot %d/context %x pending slot %d\n",
+			execlist->running_slot ?
+			execlist->running_slot->index : -1,
+			execlist->running_context ?
+			execlist->running_context->context_id : 0,
+			execlist->pending_slot ?
+			execlist->pending_slot->index : -1);
+}
+
+static void emulate_execlist_status(struct intel_vgpu_execlist *execlist)
+{
+	struct intel_vgpu_execlist_slot *running = execlist->running_slot;
+	struct intel_vgpu_execlist_slot *pending = execlist->pending_slot;
+	struct execlist_ctx_descriptor_format *desc = execlist->running_context;
+	struct intel_vgpu *vgpu = execlist->vgpu;
+	struct execlist_status_format status;
+	int ring_id = execlist->ring_id;
+	u32 status_reg = execlist_ring_mmio(vgpu->gvt,
+			ring_id, _EL_OFFSET_STATUS);
+
+	status.ldw = vgpu_vreg(vgpu, status_reg);
+	status.udw = vgpu_vreg(vgpu, status_reg + 4);
+
+	if (running) {
+		status.current_execlist_pointer = !!running->index;
+		status.execlist_write_pointer = !!!running->index;
+		status.execlist_0_active = status.execlist_0_valid =
+			!!!(running->index);
+		status.execlist_1_active = status.execlist_1_valid =
+			!!(running->index);
+	} else {
+		status.context_id = 0;
+		status.execlist_0_active = status.execlist_0_valid = 0;
+		status.execlist_1_active = status.execlist_1_valid = 0;
+	}
+
+	status.context_id = desc ? desc->context_id : 0;
+	status.execlist_queue_full = !!(pending);
+
+	vgpu_vreg(vgpu, status_reg) = status.ldw;
+	vgpu_vreg(vgpu, status_reg + 4) = status.udw;
+
+	gvt_dbg_el("vgpu%d: status reg offset %x ldw %x udw %x\n",
+		vgpu->id, status_reg, status.ldw, status.udw);
+}
+
+static void emulate_csb_update(struct intel_vgpu_execlist *execlist,
+		struct execlist_context_status_format *status,
+		bool trigger_interrupt_later)
+{
+	struct intel_vgpu *vgpu = execlist->vgpu;
+	int ring_id = execlist->ring_id;
+	struct execlist_context_status_pointer_format ctx_status_ptr;
+	u32 write_pointer;
+	u32 ctx_status_ptr_reg, ctx_status_buf_reg, offset;
+
+	ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id,
+			_EL_OFFSET_STATUS_PTR);
+	ctx_status_buf_reg = execlist_ring_mmio(vgpu->gvt, ring_id,
+			_EL_OFFSET_STATUS_BUF);
+
+	ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg);
+
+	write_pointer = ctx_status_ptr.write_ptr;
+
+	if (write_pointer == 0x7)
+		write_pointer = 0;
+	else {
+		++write_pointer;
+		write_pointer %= 0x6;
+	}
+
+	offset = ctx_status_buf_reg + write_pointer * 8;
+
+	vgpu_vreg(vgpu, offset) = status->ldw;
+	vgpu_vreg(vgpu, offset + 4) = status->udw;
+
+	ctx_status_ptr.write_ptr = write_pointer;
+	vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
+
+	gvt_dbg_el("vgpu%d: w pointer %u reg %x csb l %x csb h %x\n",
+		vgpu->id, write_pointer, offset, status->ldw, status->udw);
+
+	if (trigger_interrupt_later)
+		return;
+
+	intel_vgpu_trigger_virtual_event(vgpu,
+			ring_id_to_context_switch_event(execlist->ring_id));
+}
+
+static int emulate_execlist_ctx_schedule_out(
+		struct intel_vgpu_execlist *execlist,
+		struct execlist_ctx_descriptor_format *ctx)
+{
+	struct intel_vgpu_execlist_slot *running = execlist->running_slot;
+	struct intel_vgpu_execlist_slot *pending = execlist->pending_slot;
+	struct execlist_ctx_descriptor_format *ctx0 = &running->ctx[0];
+	struct execlist_ctx_descriptor_format *ctx1 = &running->ctx[1];
+	struct execlist_context_status_format status;
+
+	memset(&status, 0, sizeof(status));
+
+	gvt_dbg_el("schedule out context id %x\n", ctx->context_id);
+
+	if (WARN_ON(!same_context(ctx, execlist->running_context))) {
+		gvt_err("schedule out context is not running context,"
+				"ctx id %x running ctx id %x\n",
+				ctx->context_id,
+				execlist->running_context->context_id);
+		return -EINVAL;
+	}
+
+	/* ctx1 is valid, ctx0/ctx is scheduled-out -> element switch */
+	if (valid_context(ctx1) && same_context(ctx0, ctx)) {
+		gvt_dbg_el("ctx 1 valid, ctx/ctx 0 is scheduled-out\n");
+
+		execlist->running_context = ctx1;
+
+		emulate_execlist_status(execlist);
+
+		status.context_complete = status.element_switch = 1;
+		status.context_id = ctx->context_id;
+
+		emulate_csb_update(execlist, &status, false);
+		/*
+		 * ctx1 is not valid, ctx == ctx0
+		 * ctx1 is valid, ctx1 == ctx
+		 *	--> last element is finished
+		 * emulate:
+		 *	active-to-idle if there is *no* pending execlist
+		 *	context-complete if there *is* pending execlist
+		 */
+	} else if ((!valid_context(ctx1) && same_context(ctx0, ctx))
+			|| (valid_context(ctx1) && same_context(ctx1, ctx))) {
+		gvt_dbg_el("need to switch virtual execlist slot\n");
+
+		switch_virtual_execlist_slot(execlist);
+
+		emulate_execlist_status(execlist);
+
+		status.context_complete = status.active_to_idle = 1;
+		status.context_id = ctx->context_id;
+
+		if (!pending) {
+			emulate_csb_update(execlist, &status, false);
+		} else {
+			emulate_csb_update(execlist, &status, true);
+
+			memset(&status, 0, sizeof(status));
+
+			status.idle_to_active = 1;
+			status.context_id = 0;
+
+			emulate_csb_update(execlist, &status, false);
+		}
+	} else {
+		WARN_ON(1);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static struct intel_vgpu_execlist_slot *get_next_execlist_slot(
+		struct intel_vgpu_execlist *execlist)
+{
+	struct intel_vgpu *vgpu = execlist->vgpu;
+	int ring_id = execlist->ring_id;
+	u32 status_reg = execlist_ring_mmio(vgpu->gvt, ring_id,
+			_EL_OFFSET_STATUS);
+	struct execlist_status_format status;
+
+	status.ldw = vgpu_vreg(vgpu, status_reg);
+	status.udw = vgpu_vreg(vgpu, status_reg + 4);
+
+	if (status.execlist_queue_full) {
+		gvt_err("virtual execlist slots are full\n");
+		return NULL;
+	}
+
+	return &execlist->slot[status.execlist_write_pointer];
+}
+
+static int emulate_execlist_schedule_in(struct intel_vgpu_execlist *execlist,
+		struct execlist_ctx_descriptor_format ctx[2])
+{
+	struct intel_vgpu_execlist_slot *running = execlist->running_slot;
+	struct intel_vgpu_execlist_slot *slot =
+		get_next_execlist_slot(execlist);
+
+	struct execlist_ctx_descriptor_format *ctx0, *ctx1;
+	struct execlist_context_status_format status;
+
+	gvt_dbg_el("emulate schedule-in\n");
+
+	if (!slot) {
+		gvt_err("no available execlist slot\n");
+		return -EINVAL;
+	}
+
+	memset(&status, 0, sizeof(status));
+	memset(slot->ctx, 0, sizeof(slot->ctx));
+
+	slot->ctx[0] = ctx[0];
+	slot->ctx[1] = ctx[1];
+
+	gvt_dbg_el("alloc slot index %d ctx 0 %x ctx 1 %x\n",
+			slot->index, ctx[0].context_id,
+			ctx[1].context_id);
+
+	/*
+	 * no running execlist, make this write bundle as running execlist
+	 * -> idle-to-active
+	 */
+	if (!running) {
+		gvt_dbg_el("no current running execlist\n");
+
+		execlist->running_slot = slot;
+		execlist->pending_slot = NULL;
+		execlist->running_context = &slot->ctx[0];
+
+		gvt_dbg_el("running slot index %d running context %x\n",
+				execlist->running_slot->index,
+				execlist->running_context->context_id);
+
+		emulate_execlist_status(execlist);
+
+		status.idle_to_active = 1;
+		status.context_id = 0;
+
+		emulate_csb_update(execlist, &status, false);
+		return 0;
+	}
+
+	ctx0 = &running->ctx[0];
+	ctx1 = &running->ctx[1];
+
+	gvt_dbg_el("current running slot index %d ctx 0 %x ctx 1 %x\n",
+		running->index, ctx0->context_id, ctx1->context_id);
+
+	/*
+	 * already has an running execlist
+	 *	a. running ctx1 is valid,
+	 *	   ctx0 is finished, and running ctx1 == new execlist ctx[0]
+	 *	b. running ctx1 is not valid,
+	 *	   ctx0 == new execlist ctx[0]
+	 * ----> lite-restore + preempted
+	 */
+	if ((valid_context(ctx1) && same_context(ctx1, &slot->ctx[0]) &&
+		/* condition a */
+		(!same_context(ctx0, execlist->running_context))) ||
+			(!valid_context(ctx1) &&
+			 same_context(ctx0, &slot->ctx[0]))) { /* condition b */
+		gvt_dbg_el("need to switch virtual execlist slot\n");
+
+		execlist->pending_slot = slot;
+		switch_virtual_execlist_slot(execlist);
+
+		emulate_execlist_status(execlist);
+
+		status.lite_restore = status.preempted = 1;
+		status.context_id = ctx[0].context_id;
+
+		emulate_csb_update(execlist, &status, false);
+	} else {
+		gvt_dbg_el("emulate as pending slot\n");
+		/*
+		 * otherwise
+		 * --> emulate pending execlist exist + but no preemption case
+		 */
+		execlist->pending_slot = slot;
+		emulate_execlist_status(execlist);
+	}
+	return 0;
+}
+
+static void free_workload(struct intel_vgpu_workload *workload)
+{
+	intel_vgpu_unpin_mm(workload->shadow_mm);
+	intel_gvt_mm_unreference(workload->shadow_mm);
+	kmem_cache_free(workload->vgpu->workloads, workload);
+}
+
+#define get_desc_from_elsp_dwords(ed, i) \
+	((struct execlist_ctx_descriptor_format *)&((ed)->data[i * 2]))
+
+
+#define BATCH_BUFFER_ADDR_MASK ((1UL << 32) - (1U << 2))
+#define BATCH_BUFFER_ADDR_HIGH_MASK ((1UL << 16) - (1U))
+static int set_gma_to_bb_cmd(struct intel_shadow_bb_entry *entry_obj,
+			     unsigned long add, int gmadr_bytes)
+{
+	if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8))
+		return -1;
+
+	*((u32 *)(entry_obj->bb_start_cmd_va + (1 << 2))) = add &
+		BATCH_BUFFER_ADDR_MASK;
+	if (gmadr_bytes == 8) {
+		*((u32 *)(entry_obj->bb_start_cmd_va + (2 << 2))) =
+			add & BATCH_BUFFER_ADDR_HIGH_MASK;
+	}
+
+	return 0;
+}
+
+static void prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
+{
+	int gmadr_bytes = workload->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+
+	/* pin the gem object to ggtt */
+	if (!list_empty(&workload->shadow_bb)) {
+		struct intel_shadow_bb_entry *entry_obj =
+			list_first_entry(&workload->shadow_bb,
+					 struct intel_shadow_bb_entry,
+					 list);
+		struct intel_shadow_bb_entry *temp;
+
+		list_for_each_entry_safe(entry_obj, temp, &workload->shadow_bb,
+				list) {
+			struct i915_vma *vma;
+
+			vma = i915_gem_object_ggtt_pin(entry_obj->obj, NULL, 0,
+						       4, 0);
+			if (IS_ERR(vma)) {
+				gvt_err("Cannot pin\n");
+				return;
+			}
+
+			/* FIXME: we are not tracking our pinned VMA leaving it
+			 * up to the core to fix up the stray pin_count upon
+			 * free.
+			 */
+
+			/* update the relocate gma with shadow batch buffer*/
+			set_gma_to_bb_cmd(entry_obj,
+					  i915_ggtt_offset(vma),
+					  gmadr_bytes);
+		}
+	}
+}
+
+static int update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+	int ring_id = wa_ctx->workload->ring_id;
+	struct i915_gem_context *shadow_ctx =
+		wa_ctx->workload->vgpu->shadow_ctx;
+	struct drm_i915_gem_object *ctx_obj =
+		shadow_ctx->engine[ring_id].state->obj;
+	struct execlist_ring_context *shadow_ring_context;
+	struct page *page;
+
+	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+	shadow_ring_context = kmap_atomic(page);
+
+	shadow_ring_context->bb_per_ctx_ptr.val =
+		(shadow_ring_context->bb_per_ctx_ptr.val &
+		(~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma;
+	shadow_ring_context->rcs_indirect_ctx.val =
+		(shadow_ring_context->rcs_indirect_ctx.val &
+		(~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma;
+
+	kunmap_atomic(shadow_ring_context);
+	return 0;
+}
+
+static void prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+	struct i915_vma *vma;
+	unsigned char *per_ctx_va =
+		(unsigned char *)wa_ctx->indirect_ctx.shadow_va +
+		wa_ctx->indirect_ctx.size;
+
+	if (wa_ctx->indirect_ctx.size == 0)
+		return;
+
+	vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL,
+				       0, CACHELINE_BYTES, 0);
+	if (IS_ERR(vma)) {
+		gvt_err("Cannot pin indirect ctx obj\n");
+		return;
+	}
+
+	/* FIXME: we are not tracking our pinned VMA leaving it
+	 * up to the core to fix up the stray pin_count upon
+	 * free.
+	 */
+
+	wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma);
+
+	wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1);
+	memset(per_ctx_va, 0, CACHELINE_BYTES);
+
+	update_wa_ctx_2_shadow_ctx(wa_ctx);
+}
+
+static int prepare_execlist_workload(struct intel_vgpu_workload *workload)
+{
+	struct intel_vgpu *vgpu = workload->vgpu;
+	struct execlist_ctx_descriptor_format ctx[2];
+	int ring_id = workload->ring_id;
+
+	intel_vgpu_pin_mm(workload->shadow_mm);
+	intel_vgpu_sync_oos_pages(workload->vgpu);
+	intel_vgpu_flush_post_shadow(workload->vgpu);
+	prepare_shadow_batch_buffer(workload);
+	prepare_shadow_wa_ctx(&workload->wa_ctx);
+	if (!workload->emulate_schedule_in)
+		return 0;
+
+	ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1);
+	ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0);
+
+	return emulate_execlist_schedule_in(&vgpu->execlist[ring_id], ctx);
+}
+
+static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)
+{
+	/* release all the shadow batch buffer */
+	if (!list_empty(&workload->shadow_bb)) {
+		struct intel_shadow_bb_entry *entry_obj =
+			list_first_entry(&workload->shadow_bb,
+					 struct intel_shadow_bb_entry,
+					 list);
+		struct intel_shadow_bb_entry *temp;
+
+		list_for_each_entry_safe(entry_obj, temp, &workload->shadow_bb,
+					 list) {
+			i915_gem_object_unpin_map(entry_obj->obj);
+			i915_gem_object_put(entry_obj->obj);
+			list_del(&entry_obj->list);
+			kfree(entry_obj);
+		}
+	}
+}
+
+static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+	if (wa_ctx->indirect_ctx.size == 0)
+		return;
+
+	i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj);
+	i915_gem_object_put(wa_ctx->indirect_ctx.obj);
+}
+
+static int complete_execlist_workload(struct intel_vgpu_workload *workload)
+{
+	struct intel_vgpu *vgpu = workload->vgpu;
+	struct intel_vgpu_execlist *execlist =
+		&vgpu->execlist[workload->ring_id];
+	struct intel_vgpu_workload *next_workload;
+	struct list_head *next = workload_q_head(vgpu, workload->ring_id)->next;
+	bool lite_restore = false;
+	int ret;
+
+	gvt_dbg_el("complete workload %p status %d\n", workload,
+			workload->status);
+
+	release_shadow_batch_buffer(workload);
+	release_shadow_wa_ctx(&workload->wa_ctx);
+
+	if (workload->status || vgpu->resetting)
+		goto out;
+
+	if (!list_empty(workload_q_head(vgpu, workload->ring_id))) {
+		struct execlist_ctx_descriptor_format *this_desc, *next_desc;
+
+		next_workload = container_of(next,
+				struct intel_vgpu_workload, list);
+		this_desc = &workload->ctx_desc;
+		next_desc = &next_workload->ctx_desc;
+
+		lite_restore = same_context(this_desc, next_desc);
+	}
+
+	if (lite_restore) {
+		gvt_dbg_el("next context == current - no schedule-out\n");
+		free_workload(workload);
+		return 0;
+	}
+
+	ret = emulate_execlist_ctx_schedule_out(execlist, &workload->ctx_desc);
+	if (ret)
+		goto err;
+out:
+	free_workload(workload);
+	return 0;
+err:
+	free_workload(workload);
+	return ret;
+}
+
+#define RING_CTX_OFF(x) \
+	offsetof(struct execlist_ring_context, x)
+
+static void read_guest_pdps(struct intel_vgpu *vgpu,
+		u64 ring_context_gpa, u32 pdp[8])
+{
+	u64 gpa;
+	int i;
+
+	gpa = ring_context_gpa + RING_CTX_OFF(pdp3_UDW.val);
+
+	for (i = 0; i < 8; i++)
+		intel_gvt_hypervisor_read_gpa(vgpu,
+				gpa + i * 8, &pdp[7 - i], 4);
+}
+
+static int prepare_mm(struct intel_vgpu_workload *workload)
+{
+	struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc;
+	struct intel_vgpu_mm *mm;
+	int page_table_level;
+	u32 pdp[8];
+
+	if (desc->addressing_mode == 1) { /* legacy 32-bit */
+		page_table_level = 3;
+	} else if (desc->addressing_mode == 3) { /* legacy 64 bit */
+		page_table_level = 4;
+	} else {
+		gvt_err("Advanced Context mode(SVM) is not supported!\n");
+		return -EINVAL;
+	}
+
+	read_guest_pdps(workload->vgpu, workload->ring_context_gpa, pdp);
+
+	mm = intel_vgpu_find_ppgtt_mm(workload->vgpu, page_table_level, pdp);
+	if (mm) {
+		intel_gvt_mm_reference(mm);
+	} else {
+
+		mm = intel_vgpu_create_mm(workload->vgpu, INTEL_GVT_MM_PPGTT,
+				pdp, page_table_level, 0);
+		if (IS_ERR(mm)) {
+			gvt_err("fail to create mm object.\n");
+			return PTR_ERR(mm);
+		}
+	}
+	workload->shadow_mm = mm;
+	return 0;
+}
+
+#define get_last_workload(q) \
+	(list_empty(q) ? NULL : container_of(q->prev, \
+	struct intel_vgpu_workload, list))
+
+static int submit_context(struct intel_vgpu *vgpu, int ring_id,
+		struct execlist_ctx_descriptor_format *desc,
+		bool emulate_schedule_in)
+{
+	struct list_head *q = workload_q_head(vgpu, ring_id);
+	struct intel_vgpu_workload *last_workload = get_last_workload(q);
+	struct intel_vgpu_workload *workload = NULL;
+	u64 ring_context_gpa;
+	u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx;
+	int ret;
+
+	ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
+			(u32)((desc->lrca + 1) << GTT_PAGE_SHIFT));
+	if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) {
+		gvt_err("invalid guest context LRCA: %x\n", desc->lrca);
+		return -EINVAL;
+	}
+
+	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(ring_header.val), &head, 4);
+
+	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(ring_tail.val), &tail, 4);
+
+	head &= RB_HEAD_OFF_MASK;
+	tail &= RB_TAIL_OFF_MASK;
+
+	if (last_workload && same_context(&last_workload->ctx_desc, desc)) {
+		gvt_dbg_el("ring id %d cur workload == last\n", ring_id);
+		gvt_dbg_el("ctx head %x real head %lx\n", head,
+				last_workload->rb_tail);
+		/*
+		 * cannot use guest context head pointer here,
+		 * as it might not be updated at this time
+		 */
+		head = last_workload->rb_tail;
+	}
+
+	gvt_dbg_el("ring id %d begin a new workload\n", ring_id);
+
+	workload = kmem_cache_zalloc(vgpu->workloads, GFP_KERNEL);
+	if (!workload)
+		return -ENOMEM;
+
+	/* record some ring buffer register values for scan and shadow */
+	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(rb_start.val), &start, 4);
+	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(rb_ctrl.val), &ctl, 4);
+	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4);
+
+	INIT_LIST_HEAD(&workload->list);
+	INIT_LIST_HEAD(&workload->shadow_bb);
+
+	init_waitqueue_head(&workload->shadow_ctx_status_wq);
+	atomic_set(&workload->shadow_ctx_active, 0);
+
+	workload->vgpu = vgpu;
+	workload->ring_id = ring_id;
+	workload->ctx_desc = *desc;
+	workload->ring_context_gpa = ring_context_gpa;
+	workload->rb_head = head;
+	workload->rb_tail = tail;
+	workload->rb_start = start;
+	workload->rb_ctl = ctl;
+	workload->prepare = prepare_execlist_workload;
+	workload->complete = complete_execlist_workload;
+	workload->status = -EINPROGRESS;
+	workload->emulate_schedule_in = emulate_schedule_in;
+
+	if (ring_id == RCS) {
+		intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4);
+		intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4);
+
+		workload->wa_ctx.indirect_ctx.guest_gma =
+			indirect_ctx & INDIRECT_CTX_ADDR_MASK;
+		workload->wa_ctx.indirect_ctx.size =
+			(indirect_ctx & INDIRECT_CTX_SIZE_MASK) *
+			CACHELINE_BYTES;
+		workload->wa_ctx.per_ctx.guest_gma =
+			per_ctx & PER_CTX_ADDR_MASK;
+		workload->wa_ctx.workload = workload;
+
+		WARN_ON(workload->wa_ctx.indirect_ctx.size && !(per_ctx & 0x1));
+	}
+
+	if (emulate_schedule_in)
+		memcpy(&workload->elsp_dwords,
+				&vgpu->execlist[ring_id].elsp_dwords,
+				sizeof(workload->elsp_dwords));
+
+	gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n",
+			workload, ring_id, head, tail, start, ctl);
+
+	gvt_dbg_el("workload %p emulate schedule_in %d\n", workload,
+			emulate_schedule_in);
+
+	ret = prepare_mm(workload);
+	if (ret) {
+		kmem_cache_free(vgpu->workloads, workload);
+		return ret;
+	}
+
+	queue_workload(workload);
+	return 0;
+}
+
+int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id)
+{
+	struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id];
+	struct execlist_ctx_descriptor_format *desc[2], valid_desc[2];
+	unsigned long valid_desc_bitmap = 0;
+	bool emulate_schedule_in = true;
+	int ret;
+	int i;
+
+	memset(valid_desc, 0, sizeof(valid_desc));
+
+	desc[0] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1);
+	desc[1] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0);
+
+	for (i = 0; i < 2; i++) {
+		if (!desc[i]->valid)
+			continue;
+
+		if (!desc[i]->privilege_access) {
+			gvt_err("vgpu%d: unexpected GGTT elsp submission\n",
+					vgpu->id);
+			return -EINVAL;
+		}
+
+		/* TODO: add another guest context checks here. */
+		set_bit(i, &valid_desc_bitmap);
+		valid_desc[i] = *desc[i];
+	}
+
+	if (!valid_desc_bitmap) {
+		gvt_err("vgpu%d: no valid desc in a elsp submission\n",
+				vgpu->id);
+		return -EINVAL;
+	}
+
+	if (!test_bit(0, (void *)&valid_desc_bitmap) &&
+			test_bit(1, (void *)&valid_desc_bitmap)) {
+		gvt_err("vgpu%d: weird elsp submission, desc 0 is not valid\n",
+				vgpu->id);
+		return -EINVAL;
+	}
+
+	/* submit workload */
+	for_each_set_bit(i, (void *)&valid_desc_bitmap, 2) {
+		ret = submit_context(vgpu, ring_id, &valid_desc[i],
+				emulate_schedule_in);
+		if (ret) {
+			gvt_err("vgpu%d: fail to schedule workload\n",
+					vgpu->id);
+			return ret;
+		}
+		emulate_schedule_in = false;
+	}
+	return 0;
+}
+
+static void init_vgpu_execlist(struct intel_vgpu *vgpu, int ring_id)
+{
+	struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id];
+	struct execlist_context_status_pointer_format ctx_status_ptr;
+	u32 ctx_status_ptr_reg;
+
+	memset(execlist, 0, sizeof(*execlist));
+
+	execlist->vgpu = vgpu;
+	execlist->ring_id = ring_id;
+	execlist->slot[0].index = 0;
+	execlist->slot[1].index = 1;
+
+	ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id,
+			_EL_OFFSET_STATUS_PTR);
+
+	ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg);
+	ctx_status_ptr.read_ptr = ctx_status_ptr.write_ptr = 0x7;
+	vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
+}
+
+void intel_vgpu_clean_execlist(struct intel_vgpu *vgpu)
+{
+	kmem_cache_destroy(vgpu->workloads);
+}
+
+int intel_vgpu_init_execlist(struct intel_vgpu *vgpu)
+{
+	enum intel_engine_id i;
+	struct intel_engine_cs *engine;
+
+	/* each ring has a virtual execlist engine */
+	for_each_engine(engine, vgpu->gvt->dev_priv, i) {
+		init_vgpu_execlist(vgpu, i);
+		INIT_LIST_HEAD(&vgpu->workload_q_head[i]);
+	}
+
+	vgpu->workloads = kmem_cache_create("gvt-g vgpu workload",
+			sizeof(struct intel_vgpu_workload), 0,
+			SLAB_HWCACHE_ALIGN,
+			NULL);
+
+	if (!vgpu->workloads)
+		return -ENOMEM;
+
+	return 0;
+}
+
+void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu,
+		unsigned long engine_mask)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	struct intel_engine_cs *engine;
+	struct intel_vgpu_workload *pos, *n;
+	unsigned int tmp;
+
+	for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
+		/* free the unsubmited workload in the queue */
+		list_for_each_entry_safe(pos, n,
+			&vgpu->workload_q_head[engine->id], list) {
+			list_del_init(&pos->list);
+			free_workload(pos);
+		}
+
+		init_vgpu_execlist(vgpu, engine->id);
+	}
+}
diff --git a/drivers/gpu/drm/i915/gvt/execlist.h b/drivers/gpu/drm/i915/gvt/execlist.h
new file mode 100644
index 000000000000..7eced40a1e30
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/execlist.h
@@ -0,0 +1,188 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *
+ */
+
+#ifndef _GVT_EXECLIST_H_
+#define _GVT_EXECLIST_H_
+
+struct execlist_ctx_descriptor_format {
+	union {
+		u32 udw;
+		u32 context_id;
+	};
+	union {
+		u32 ldw;
+		struct {
+			u32 valid                  : 1;
+			u32 force_pd_restore       : 1;
+			u32 force_restore          : 1;
+			u32 addressing_mode        : 2;
+			u32 llc_coherency          : 1;
+			u32 fault_handling         : 2;
+			u32 privilege_access       : 1;
+			u32 reserved               : 3;
+			u32 lrca                   : 20;
+		};
+	};
+};
+
+struct execlist_status_format {
+	union {
+		u32 ldw;
+		struct {
+			u32 current_execlist_pointer       :1;
+			u32 execlist_write_pointer         :1;
+			u32 execlist_queue_full            :1;
+			u32 execlist_1_valid               :1;
+			u32 execlist_0_valid               :1;
+			u32 last_ctx_switch_reason         :9;
+			u32 current_active_elm_status      :2;
+			u32 arbitration_enable             :1;
+			u32 execlist_1_active              :1;
+			u32 execlist_0_active              :1;
+			u32 reserved                       :13;
+		};
+	};
+	union {
+		u32 udw;
+		u32 context_id;
+	};
+};
+
+struct execlist_context_status_pointer_format {
+	union {
+		u32 dw;
+		struct {
+			u32 write_ptr              :3;
+			u32 reserved               :5;
+			u32 read_ptr               :3;
+			u32 reserved2              :5;
+			u32 mask                   :16;
+		};
+	};
+};
+
+struct execlist_context_status_format {
+	union {
+		u32 ldw;
+		struct {
+			u32 idle_to_active         :1;
+			u32 preempted              :1;
+			u32 element_switch         :1;
+			u32 active_to_idle         :1;
+			u32 context_complete       :1;
+			u32 wait_on_sync_flip      :1;
+			u32 wait_on_vblank         :1;
+			u32 wait_on_semaphore      :1;
+			u32 wait_on_scanline       :1;
+			u32 reserved               :2;
+			u32 semaphore_wait_mode    :1;
+			u32 display_plane          :3;
+			u32 lite_restore           :1;
+			u32 reserved_2             :16;
+		};
+	};
+	union {
+		u32 udw;
+		u32 context_id;
+	};
+};
+
+struct execlist_mmio_pair {
+	u32 addr;
+	u32 val;
+};
+
+/* The first 52 dwords in register state context */
+struct execlist_ring_context {
+	u32 nop1;
+	u32 lri_cmd_1;
+	struct execlist_mmio_pair ctx_ctrl;
+	struct execlist_mmio_pair ring_header;
+	struct execlist_mmio_pair ring_tail;
+	struct execlist_mmio_pair rb_start;
+	struct execlist_mmio_pair rb_ctrl;
+	struct execlist_mmio_pair bb_cur_head_UDW;
+	struct execlist_mmio_pair bb_cur_head_LDW;
+	struct execlist_mmio_pair bb_state;
+	struct execlist_mmio_pair second_bb_addr_UDW;
+	struct execlist_mmio_pair second_bb_addr_LDW;
+	struct execlist_mmio_pair second_bb_state;
+	struct execlist_mmio_pair bb_per_ctx_ptr;
+	struct execlist_mmio_pair rcs_indirect_ctx;
+	struct execlist_mmio_pair rcs_indirect_ctx_offset;
+	u32 nop2;
+	u32 nop3;
+	u32 nop4;
+	u32 lri_cmd_2;
+	struct execlist_mmio_pair ctx_timestamp;
+	struct execlist_mmio_pair pdp3_UDW;
+	struct execlist_mmio_pair pdp3_LDW;
+	struct execlist_mmio_pair pdp2_UDW;
+	struct execlist_mmio_pair pdp2_LDW;
+	struct execlist_mmio_pair pdp1_UDW;
+	struct execlist_mmio_pair pdp1_LDW;
+	struct execlist_mmio_pair pdp0_UDW;
+	struct execlist_mmio_pair pdp0_LDW;
+};
+
+struct intel_vgpu_elsp_dwords {
+	u32 data[4];
+	u32 index;
+};
+
+struct intel_vgpu_execlist_slot {
+	struct execlist_ctx_descriptor_format ctx[2];
+	u32 index;
+};
+
+struct intel_vgpu_execlist {
+	struct intel_vgpu_execlist_slot slot[2];
+	struct intel_vgpu_execlist_slot *running_slot;
+	struct intel_vgpu_execlist_slot *pending_slot;
+	struct execlist_ctx_descriptor_format *running_context;
+	int ring_id;
+	struct intel_vgpu *vgpu;
+	struct intel_vgpu_elsp_dwords elsp_dwords;
+};
+
+void intel_vgpu_clean_execlist(struct intel_vgpu *vgpu);
+
+int intel_vgpu_init_execlist(struct intel_vgpu *vgpu);
+
+int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id);
+
+void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu,
+		unsigned long engine_mask);
+
+#endif /*_GVT_EXECLIST_H_*/
diff --git a/drivers/gpu/drm/i915/gvt/firmware.c b/drivers/gpu/drm/i915/gvt/firmware.c
new file mode 100644
index 000000000000..2fae2a2ca96f
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/firmware.c
@@ -0,0 +1,312 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Changbin Du <changbin.du@intel.com>
+ *
+ */
+
+#include <linux/firmware.h>
+#include <linux/crc32.h>
+
+#include "i915_drv.h"
+#include "gvt.h"
+#include "i915_pvinfo.h"
+
+#define FIRMWARE_VERSION (0x0)
+
+struct gvt_firmware_header {
+	u64 magic;
+	u32 crc32;		/* protect the data after this field */
+	u32 version;
+	u64 cfg_space_size;
+	u64 cfg_space_offset;	/* offset in the file */
+	u64 mmio_size;
+	u64 mmio_offset;	/* offset in the file */
+	unsigned char data[1];
+};
+
+#define RD(offset) (readl(mmio + offset.reg))
+#define WR(v, offset) (writel(v, mmio + offset.reg))
+
+static void bdw_forcewake_get(void __iomem *mmio)
+{
+	WR(_MASKED_BIT_DISABLE(0xffff), FORCEWAKE_MT);
+
+	RD(ECOBUS);
+
+	if (wait_for((RD(FORCEWAKE_ACK_HSW) & FORCEWAKE_KERNEL) == 0, 50))
+		gvt_err("fail to wait forcewake idle\n");
+
+	WR(_MASKED_BIT_ENABLE(FORCEWAKE_KERNEL), FORCEWAKE_MT);
+
+	if (wait_for((RD(FORCEWAKE_ACK_HSW) & FORCEWAKE_KERNEL), 50))
+		gvt_err("fail to wait forcewake ack\n");
+
+	if (wait_for((RD(GEN6_GT_THREAD_STATUS_REG) &
+		      GEN6_GT_THREAD_STATUS_CORE_MASK) == 0, 50))
+		gvt_err("fail to wait c0 wake up\n");
+}
+
+#undef RD
+#undef WR
+
+#define dev_to_drm_minor(d) dev_get_drvdata((d))
+
+static ssize_t
+gvt_firmware_read(struct file *filp, struct kobject *kobj,
+	     struct bin_attribute *attr, char *buf,
+	     loff_t offset, size_t count)
+{
+	memcpy(buf, attr->private + offset, count);
+	return count;
+}
+
+static struct bin_attribute firmware_attr = {
+	.attr = {.name = "gvt_firmware", .mode = (S_IRUSR)},
+	.read = gvt_firmware_read,
+	.write = NULL,
+	.mmap = NULL,
+};
+
+static int expose_firmware_sysfs(struct intel_gvt *gvt,
+					void __iomem *mmio)
+{
+	struct intel_gvt_device_info *info = &gvt->device_info;
+	struct pci_dev *pdev = gvt->dev_priv->drm.pdev;
+	struct intel_gvt_mmio_info *e;
+	struct gvt_firmware_header *h;
+	void *firmware;
+	void *p;
+	unsigned long size;
+	int i;
+	int ret;
+
+	size = sizeof(*h) + info->mmio_size + info->cfg_space_size - 1;
+	firmware = vmalloc(size);
+	if (!firmware)
+		return -ENOMEM;
+
+	h = firmware;
+
+	h->magic = VGT_MAGIC;
+	h->version = FIRMWARE_VERSION;
+	h->cfg_space_size = info->cfg_space_size;
+	h->cfg_space_offset = offsetof(struct gvt_firmware_header, data);
+	h->mmio_size = info->mmio_size;
+	h->mmio_offset = h->cfg_space_offset + h->cfg_space_size;
+
+	p = firmware + h->cfg_space_offset;
+
+	for (i = 0; i < h->cfg_space_size; i += 4)
+		pci_read_config_dword(pdev, i, p + i);
+
+	memcpy(gvt->firmware.cfg_space, p, info->cfg_space_size);
+
+	p = firmware + h->mmio_offset;
+
+	hash_for_each(gvt->mmio.mmio_info_table, i, e, node) {
+		int j;
+
+		for (j = 0; j < e->length; j += 4)
+			*(u32 *)(p + e->offset + j) =
+				readl(mmio + e->offset + j);
+	}
+
+	memcpy(gvt->firmware.mmio, p, info->mmio_size);
+
+	firmware_attr.size = size;
+	firmware_attr.private = firmware;
+
+	ret = device_create_bin_file(&pdev->dev, &firmware_attr);
+	if (ret) {
+		vfree(firmware);
+		return ret;
+	}
+	return 0;
+}
+
+static void clean_firmware_sysfs(struct intel_gvt *gvt)
+{
+	struct pci_dev *pdev = gvt->dev_priv->drm.pdev;
+
+	device_remove_bin_file(&pdev->dev, &firmware_attr);
+	vfree(firmware_attr.private);
+}
+
+/**
+ * intel_gvt_free_firmware - free GVT firmware
+ * @gvt: intel gvt device
+ *
+ */
+void intel_gvt_free_firmware(struct intel_gvt *gvt)
+{
+	if (!gvt->firmware.firmware_loaded)
+		clean_firmware_sysfs(gvt);
+
+	kfree(gvt->firmware.cfg_space);
+	kfree(gvt->firmware.mmio);
+}
+
+static int verify_firmware(struct intel_gvt *gvt,
+			   const struct firmware *fw)
+{
+	struct intel_gvt_device_info *info = &gvt->device_info;
+	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
+	struct gvt_firmware_header *h;
+	unsigned long id, crc32_start;
+	const void *mem;
+	const char *item;
+	u64 file, request;
+
+	h = (struct gvt_firmware_header *)fw->data;
+
+	crc32_start = offsetof(struct gvt_firmware_header, crc32) + 4;
+	mem = fw->data + crc32_start;
+
+#define VERIFY(s, a, b) do { \
+	item = (s); file = (u64)(a); request = (u64)(b); \
+	if ((a) != (b)) \
+		goto invalid_firmware; \
+} while (0)
+
+	VERIFY("magic number", h->magic, VGT_MAGIC);
+	VERIFY("version", h->version, FIRMWARE_VERSION);
+	VERIFY("crc32", h->crc32, crc32_le(0, mem, fw->size - crc32_start));
+	VERIFY("cfg space size", h->cfg_space_size, info->cfg_space_size);
+	VERIFY("mmio size", h->mmio_size, info->mmio_size);
+
+	mem = (fw->data + h->cfg_space_offset);
+
+	id = *(u16 *)(mem + PCI_VENDOR_ID);
+	VERIFY("vender id", id, pdev->vendor);
+
+	id = *(u16 *)(mem + PCI_DEVICE_ID);
+	VERIFY("device id", id, pdev->device);
+
+	id = *(u8 *)(mem + PCI_REVISION_ID);
+	VERIFY("revision id", id, pdev->revision);
+
+#undef VERIFY
+	return 0;
+
+invalid_firmware:
+	gvt_dbg_core("Invalid firmware: %s [file] 0x%llx [request] 0x%llx\n",
+		     item, file, request);
+	return -EINVAL;
+}
+
+#define GVT_FIRMWARE_PATH "i915/gvt"
+
+/**
+ * intel_gvt_load_firmware - load GVT firmware
+ * @gvt: intel gvt device
+ *
+ */
+int intel_gvt_load_firmware(struct intel_gvt *gvt)
+{
+	struct intel_gvt_device_info *info = &gvt->device_info;
+	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
+	struct intel_gvt_firmware *firmware = &gvt->firmware;
+	struct gvt_firmware_header *h;
+	const struct firmware *fw;
+	char *path;
+	void __iomem *mmio;
+	void *mem;
+	int ret;
+
+	path = kmalloc(PATH_MAX, GFP_KERNEL);
+	if (!path)
+		return -ENOMEM;
+
+	mem = kmalloc(info->cfg_space_size, GFP_KERNEL);
+	if (!mem) {
+		kfree(path);
+		return -ENOMEM;
+	}
+
+	firmware->cfg_space = mem;
+
+	mem = kmalloc(info->mmio_size, GFP_KERNEL);
+	if (!mem) {
+		kfree(path);
+		kfree(firmware->cfg_space);
+		return -ENOMEM;
+	}
+
+	firmware->mmio = mem;
+
+	mmio = pci_iomap(pdev, info->mmio_bar, info->mmio_size);
+	if (!mmio) {
+		kfree(path);
+		kfree(firmware->cfg_space);
+		kfree(firmware->mmio);
+		return -EINVAL;
+	}
+
+	if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv))
+		bdw_forcewake_get(mmio);
+
+	sprintf(path, "%s/vid_0x%04x_did_0x%04x_rid_0x%04x.golden_hw_state",
+		 GVT_FIRMWARE_PATH, pdev->vendor, pdev->device,
+		 pdev->revision);
+
+	gvt_dbg_core("request hw state firmware %s...\n", path);
+
+	ret = request_firmware(&fw, path, &dev_priv->drm.pdev->dev);
+	kfree(path);
+
+	if (ret)
+		goto expose_firmware;
+
+	gvt_dbg_core("success.\n");
+
+	ret = verify_firmware(gvt, fw);
+	if (ret)
+		goto out_free_fw;
+
+	gvt_dbg_core("verified.\n");
+
+	h = (struct gvt_firmware_header *)fw->data;
+
+	memcpy(firmware->cfg_space, fw->data + h->cfg_space_offset,
+	       h->cfg_space_size);
+	memcpy(firmware->mmio, fw->data + h->mmio_offset,
+	       h->mmio_size);
+
+	release_firmware(fw);
+	firmware->firmware_loaded = true;
+	pci_iounmap(pdev, mmio);
+	return 0;
+
+out_free_fw:
+	release_firmware(fw);
+expose_firmware:
+	expose_firmware_sysfs(gvt, mmio);
+	pci_iounmap(pdev, mmio);
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c
new file mode 100644
index 000000000000..7eaaf1c9ed2b
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/gtt.c
@@ -0,0 +1,2244 @@
+/*
+ * GTT virtualization
+ *
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *    Xiao Zheng <xiao.zheng@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+#include "i915_pvinfo.h"
+#include "trace.h"
+
+static bool enable_out_of_sync = false;
+static int preallocated_oos_pages = 8192;
+
+/*
+ * validate a gm address and related range size,
+ * translate it to host gm address
+ */
+bool intel_gvt_ggtt_validate_range(struct intel_vgpu *vgpu, u64 addr, u32 size)
+{
+	if ((!vgpu_gmadr_is_valid(vgpu, addr)) || (size
+			&& !vgpu_gmadr_is_valid(vgpu, addr + size - 1))) {
+		gvt_err("vgpu%d: invalid range gmadr 0x%llx size 0x%x\n",
+				vgpu->id, addr, size);
+		return false;
+	}
+	return true;
+}
+
+/* translate a guest gmadr to host gmadr */
+int intel_gvt_ggtt_gmadr_g2h(struct intel_vgpu *vgpu, u64 g_addr, u64 *h_addr)
+{
+	if (WARN(!vgpu_gmadr_is_valid(vgpu, g_addr),
+		 "invalid guest gmadr %llx\n", g_addr))
+		return -EACCES;
+
+	if (vgpu_gmadr_is_aperture(vgpu, g_addr))
+		*h_addr = vgpu_aperture_gmadr_base(vgpu)
+			  + (g_addr - vgpu_aperture_offset(vgpu));
+	else
+		*h_addr = vgpu_hidden_gmadr_base(vgpu)
+			  + (g_addr - vgpu_hidden_offset(vgpu));
+	return 0;
+}
+
+/* translate a host gmadr to guest gmadr */
+int intel_gvt_ggtt_gmadr_h2g(struct intel_vgpu *vgpu, u64 h_addr, u64 *g_addr)
+{
+	if (WARN(!gvt_gmadr_is_valid(vgpu->gvt, h_addr),
+		 "invalid host gmadr %llx\n", h_addr))
+		return -EACCES;
+
+	if (gvt_gmadr_is_aperture(vgpu->gvt, h_addr))
+		*g_addr = vgpu_aperture_gmadr_base(vgpu)
+			+ (h_addr - gvt_aperture_gmadr_base(vgpu->gvt));
+	else
+		*g_addr = vgpu_hidden_gmadr_base(vgpu)
+			+ (h_addr - gvt_hidden_gmadr_base(vgpu->gvt));
+	return 0;
+}
+
+int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index,
+			     unsigned long *h_index)
+{
+	u64 h_addr;
+	int ret;
+
+	ret = intel_gvt_ggtt_gmadr_g2h(vgpu, g_index << GTT_PAGE_SHIFT,
+				       &h_addr);
+	if (ret)
+		return ret;
+
+	*h_index = h_addr >> GTT_PAGE_SHIFT;
+	return 0;
+}
+
+int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
+			     unsigned long *g_index)
+{
+	u64 g_addr;
+	int ret;
+
+	ret = intel_gvt_ggtt_gmadr_h2g(vgpu, h_index << GTT_PAGE_SHIFT,
+				       &g_addr);
+	if (ret)
+		return ret;
+
+	*g_index = g_addr >> GTT_PAGE_SHIFT;
+	return 0;
+}
+
+#define gtt_type_is_entry(type) \
+	(type > GTT_TYPE_INVALID && type < GTT_TYPE_PPGTT_ENTRY \
+	 && type != GTT_TYPE_PPGTT_PTE_ENTRY \
+	 && type != GTT_TYPE_PPGTT_ROOT_ENTRY)
+
+#define gtt_type_is_pt(type) \
+	(type >= GTT_TYPE_PPGTT_PTE_PT && type < GTT_TYPE_MAX)
+
+#define gtt_type_is_pte_pt(type) \
+	(type == GTT_TYPE_PPGTT_PTE_PT)
+
+#define gtt_type_is_root_pointer(type) \
+	(gtt_type_is_entry(type) && type > GTT_TYPE_PPGTT_ROOT_ENTRY)
+
+#define gtt_init_entry(e, t, p, v) do { \
+	(e)->type = t; \
+	(e)->pdev = p; \
+	memcpy(&(e)->val64, &v, sizeof(v)); \
+} while (0)
+
+/*
+ * Mappings between GTT_TYPE* enumerations.
+ * Following information can be found according to the given type:
+ * - type of next level page table
+ * - type of entry inside this level page table
+ * - type of entry with PSE set
+ *
+ * If the given type doesn't have such a kind of information,
+ * e.g. give a l4 root entry type, then request to get its PSE type,
+ * give a PTE page table type, then request to get its next level page
+ * table type, as we know l4 root entry doesn't have a PSE bit,
+ * and a PTE page table doesn't have a next level page table type,
+ * GTT_TYPE_INVALID will be returned. This is useful when traversing a
+ * page table.
+ */
+
+struct gtt_type_table_entry {
+	int entry_type;
+	int next_pt_type;
+	int pse_entry_type;
+};
+
+#define GTT_TYPE_TABLE_ENTRY(type, e_type, npt_type, pse_type) \
+	[type] = { \
+		.entry_type = e_type, \
+		.next_pt_type = npt_type, \
+		.pse_entry_type = pse_type, \
+	}
+
+static struct gtt_type_table_entry gtt_type_table[] = {
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
+			GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
+			GTT_TYPE_PPGTT_PML4_PT,
+			GTT_TYPE_INVALID),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_PT,
+			GTT_TYPE_PPGTT_PML4_ENTRY,
+			GTT_TYPE_PPGTT_PDP_PT,
+			GTT_TYPE_INVALID),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_ENTRY,
+			GTT_TYPE_PPGTT_PML4_ENTRY,
+			GTT_TYPE_PPGTT_PDP_PT,
+			GTT_TYPE_INVALID),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_PT,
+			GTT_TYPE_PPGTT_PDP_ENTRY,
+			GTT_TYPE_PPGTT_PDE_PT,
+			GTT_TYPE_PPGTT_PTE_1G_ENTRY),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
+			GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
+			GTT_TYPE_PPGTT_PDE_PT,
+			GTT_TYPE_PPGTT_PTE_1G_ENTRY),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_ENTRY,
+			GTT_TYPE_PPGTT_PDP_ENTRY,
+			GTT_TYPE_PPGTT_PDE_PT,
+			GTT_TYPE_PPGTT_PTE_1G_ENTRY),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_PT,
+			GTT_TYPE_PPGTT_PDE_ENTRY,
+			GTT_TYPE_PPGTT_PTE_PT,
+			GTT_TYPE_PPGTT_PTE_2M_ENTRY),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_ENTRY,
+			GTT_TYPE_PPGTT_PDE_ENTRY,
+			GTT_TYPE_PPGTT_PTE_PT,
+			GTT_TYPE_PPGTT_PTE_2M_ENTRY),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_PT,
+			GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+			GTT_TYPE_INVALID,
+			GTT_TYPE_INVALID),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+			GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+			GTT_TYPE_INVALID,
+			GTT_TYPE_INVALID),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_2M_ENTRY,
+			GTT_TYPE_PPGTT_PDE_ENTRY,
+			GTT_TYPE_INVALID,
+			GTT_TYPE_PPGTT_PTE_2M_ENTRY),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_1G_ENTRY,
+			GTT_TYPE_PPGTT_PDP_ENTRY,
+			GTT_TYPE_INVALID,
+			GTT_TYPE_PPGTT_PTE_1G_ENTRY),
+	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_GGTT_PTE,
+			GTT_TYPE_GGTT_PTE,
+			GTT_TYPE_INVALID,
+			GTT_TYPE_INVALID),
+};
+
+static inline int get_next_pt_type(int type)
+{
+	return gtt_type_table[type].next_pt_type;
+}
+
+static inline int get_entry_type(int type)
+{
+	return gtt_type_table[type].entry_type;
+}
+
+static inline int get_pse_type(int type)
+{
+	return gtt_type_table[type].pse_entry_type;
+}
+
+static u64 read_pte64(struct drm_i915_private *dev_priv, unsigned long index)
+{
+	void __iomem *addr = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + index;
+	u64 pte;
+
+#ifdef readq
+	pte = readq(addr);
+#else
+	pte = ioread32(addr);
+	pte |= (u64)ioread32(addr + 4) << 32;
+#endif
+	return pte;
+}
+
+static void write_pte64(struct drm_i915_private *dev_priv,
+		unsigned long index, u64 pte)
+{
+	void __iomem *addr = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + index;
+
+#ifdef writeq
+	writeq(pte, addr);
+#else
+	iowrite32((u32)pte, addr);
+	iowrite32(pte >> 32, addr + 4);
+#endif
+	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
+	POSTING_READ(GFX_FLSH_CNTL_GEN6);
+}
+
+static inline struct intel_gvt_gtt_entry *gtt_get_entry64(void *pt,
+		struct intel_gvt_gtt_entry *e,
+		unsigned long index, bool hypervisor_access, unsigned long gpa,
+		struct intel_vgpu *vgpu)
+{
+	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+	int ret;
+
+	if (WARN_ON(info->gtt_entry_size != 8))
+		return e;
+
+	if (hypervisor_access) {
+		ret = intel_gvt_hypervisor_read_gpa(vgpu, gpa +
+				(index << info->gtt_entry_size_shift),
+				&e->val64, 8);
+		WARN_ON(ret);
+	} else if (!pt) {
+		e->val64 = read_pte64(vgpu->gvt->dev_priv, index);
+	} else {
+		e->val64 = *((u64 *)pt + index);
+	}
+	return e;
+}
+
+static inline struct intel_gvt_gtt_entry *gtt_set_entry64(void *pt,
+		struct intel_gvt_gtt_entry *e,
+		unsigned long index, bool hypervisor_access, unsigned long gpa,
+		struct intel_vgpu *vgpu)
+{
+	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+	int ret;
+
+	if (WARN_ON(info->gtt_entry_size != 8))
+		return e;
+
+	if (hypervisor_access) {
+		ret = intel_gvt_hypervisor_write_gpa(vgpu, gpa +
+				(index << info->gtt_entry_size_shift),
+				&e->val64, 8);
+		WARN_ON(ret);
+	} else if (!pt) {
+		write_pte64(vgpu->gvt->dev_priv, index, e->val64);
+	} else {
+		*((u64 *)pt + index) = e->val64;
+	}
+	return e;
+}
+
+#define GTT_HAW 46
+
+#define ADDR_1G_MASK (((1UL << (GTT_HAW - 30 + 1)) - 1) << 30)
+#define ADDR_2M_MASK (((1UL << (GTT_HAW - 21 + 1)) - 1) << 21)
+#define ADDR_4K_MASK (((1UL << (GTT_HAW - 12 + 1)) - 1) << 12)
+
+static unsigned long gen8_gtt_get_pfn(struct intel_gvt_gtt_entry *e)
+{
+	unsigned long pfn;
+
+	if (e->type == GTT_TYPE_PPGTT_PTE_1G_ENTRY)
+		pfn = (e->val64 & ADDR_1G_MASK) >> 12;
+	else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY)
+		pfn = (e->val64 & ADDR_2M_MASK) >> 12;
+	else
+		pfn = (e->val64 & ADDR_4K_MASK) >> 12;
+	return pfn;
+}
+
+static void gen8_gtt_set_pfn(struct intel_gvt_gtt_entry *e, unsigned long pfn)
+{
+	if (e->type == GTT_TYPE_PPGTT_PTE_1G_ENTRY) {
+		e->val64 &= ~ADDR_1G_MASK;
+		pfn &= (ADDR_1G_MASK >> 12);
+	} else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY) {
+		e->val64 &= ~ADDR_2M_MASK;
+		pfn &= (ADDR_2M_MASK >> 12);
+	} else {
+		e->val64 &= ~ADDR_4K_MASK;
+		pfn &= (ADDR_4K_MASK >> 12);
+	}
+
+	e->val64 |= (pfn << 12);
+}
+
+static bool gen8_gtt_test_pse(struct intel_gvt_gtt_entry *e)
+{
+	/* Entry doesn't have PSE bit. */
+	if (get_pse_type(e->type) == GTT_TYPE_INVALID)
+		return false;
+
+	e->type = get_entry_type(e->type);
+	if (!(e->val64 & (1 << 7)))
+		return false;
+
+	e->type = get_pse_type(e->type);
+	return true;
+}
+
+static bool gen8_gtt_test_present(struct intel_gvt_gtt_entry *e)
+{
+	/*
+	 * i915 writes PDP root pointer registers without present bit,
+	 * it also works, so we need to treat root pointer entry
+	 * specifically.
+	 */
+	if (e->type == GTT_TYPE_PPGTT_ROOT_L3_ENTRY
+			|| e->type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY)
+		return (e->val64 != 0);
+	else
+		return (e->val64 & (1 << 0));
+}
+
+static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e)
+{
+	e->val64 &= ~(1 << 0);
+}
+
+/*
+ * Per-platform GMA routines.
+ */
+static unsigned long gma_to_ggtt_pte_index(unsigned long gma)
+{
+	unsigned long x = (gma >> GTT_PAGE_SHIFT);
+
+	trace_gma_index(__func__, gma, x);
+	return x;
+}
+
+#define DEFINE_PPGTT_GMA_TO_INDEX(prefix, ename, exp) \
+static unsigned long prefix##_gma_to_##ename##_index(unsigned long gma) \
+{ \
+	unsigned long x = (exp); \
+	trace_gma_index(__func__, gma, x); \
+	return x; \
+}
+
+DEFINE_PPGTT_GMA_TO_INDEX(gen8, pte, (gma >> 12 & 0x1ff));
+DEFINE_PPGTT_GMA_TO_INDEX(gen8, pde, (gma >> 21 & 0x1ff));
+DEFINE_PPGTT_GMA_TO_INDEX(gen8, l3_pdp, (gma >> 30 & 0x3));
+DEFINE_PPGTT_GMA_TO_INDEX(gen8, l4_pdp, (gma >> 30 & 0x1ff));
+DEFINE_PPGTT_GMA_TO_INDEX(gen8, pml4, (gma >> 39 & 0x1ff));
+
+static struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = {
+	.get_entry = gtt_get_entry64,
+	.set_entry = gtt_set_entry64,
+	.clear_present = gtt_entry_clear_present,
+	.test_present = gen8_gtt_test_present,
+	.test_pse = gen8_gtt_test_pse,
+	.get_pfn = gen8_gtt_get_pfn,
+	.set_pfn = gen8_gtt_set_pfn,
+};
+
+static struct intel_gvt_gtt_gma_ops gen8_gtt_gma_ops = {
+	.gma_to_ggtt_pte_index = gma_to_ggtt_pte_index,
+	.gma_to_pte_index = gen8_gma_to_pte_index,
+	.gma_to_pde_index = gen8_gma_to_pde_index,
+	.gma_to_l3_pdp_index = gen8_gma_to_l3_pdp_index,
+	.gma_to_l4_pdp_index = gen8_gma_to_l4_pdp_index,
+	.gma_to_pml4_index = gen8_gma_to_pml4_index,
+};
+
+static int gtt_entry_p2m(struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *p,
+		struct intel_gvt_gtt_entry *m)
+{
+	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+	unsigned long gfn, mfn;
+
+	*m = *p;
+
+	if (!ops->test_present(p))
+		return 0;
+
+	gfn = ops->get_pfn(p);
+
+	mfn = intel_gvt_hypervisor_gfn_to_mfn(vgpu, gfn);
+	if (mfn == INTEL_GVT_INVALID_ADDR) {
+		gvt_err("fail to translate gfn: 0x%lx\n", gfn);
+		return -ENXIO;
+	}
+
+	ops->set_pfn(m, mfn);
+	return 0;
+}
+
+/*
+ * MM helpers.
+ */
+struct intel_gvt_gtt_entry *intel_vgpu_mm_get_entry(struct intel_vgpu_mm *mm,
+		void *page_table, struct intel_gvt_gtt_entry *e,
+		unsigned long index)
+{
+	struct intel_gvt *gvt = mm->vgpu->gvt;
+	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+
+	e->type = mm->page_table_entry_type;
+
+	ops->get_entry(page_table, e, index, false, 0, mm->vgpu);
+	ops->test_pse(e);
+	return e;
+}
+
+struct intel_gvt_gtt_entry *intel_vgpu_mm_set_entry(struct intel_vgpu_mm *mm,
+		void *page_table, struct intel_gvt_gtt_entry *e,
+		unsigned long index)
+{
+	struct intel_gvt *gvt = mm->vgpu->gvt;
+	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+
+	return ops->set_entry(page_table, e, index, false, 0, mm->vgpu);
+}
+
+/*
+ * PPGTT shadow page table helpers.
+ */
+static inline struct intel_gvt_gtt_entry *ppgtt_spt_get_entry(
+		struct intel_vgpu_ppgtt_spt *spt,
+		void *page_table, int type,
+		struct intel_gvt_gtt_entry *e, unsigned long index,
+		bool guest)
+{
+	struct intel_gvt *gvt = spt->vgpu->gvt;
+	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+
+	e->type = get_entry_type(type);
+
+	if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n"))
+		return e;
+
+	ops->get_entry(page_table, e, index, guest,
+			spt->guest_page.gfn << GTT_PAGE_SHIFT,
+			spt->vgpu);
+	ops->test_pse(e);
+	return e;
+}
+
+static inline struct intel_gvt_gtt_entry *ppgtt_spt_set_entry(
+		struct intel_vgpu_ppgtt_spt *spt,
+		void *page_table, int type,
+		struct intel_gvt_gtt_entry *e, unsigned long index,
+		bool guest)
+{
+	struct intel_gvt *gvt = spt->vgpu->gvt;
+	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+
+	if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n"))
+		return e;
+
+	return ops->set_entry(page_table, e, index, guest,
+			spt->guest_page.gfn << GTT_PAGE_SHIFT,
+			spt->vgpu);
+}
+
+#define ppgtt_get_guest_entry(spt, e, index) \
+	ppgtt_spt_get_entry(spt, NULL, \
+		spt->guest_page_type, e, index, true)
+
+#define ppgtt_set_guest_entry(spt, e, index) \
+	ppgtt_spt_set_entry(spt, NULL, \
+		spt->guest_page_type, e, index, true)
+
+#define ppgtt_get_shadow_entry(spt, e, index) \
+	ppgtt_spt_get_entry(spt, spt->shadow_page.vaddr, \
+		spt->shadow_page.type, e, index, false)
+
+#define ppgtt_set_shadow_entry(spt, e, index) \
+	ppgtt_spt_set_entry(spt, spt->shadow_page.vaddr, \
+		spt->shadow_page.type, e, index, false)
+
+/**
+ * intel_vgpu_init_guest_page - init a guest page data structure
+ * @vgpu: a vGPU
+ * @p: a guest page data structure
+ * @gfn: guest memory page frame number
+ * @handler: function will be called when target guest memory page has
+ * been modified.
+ *
+ * This function is called when user wants to track a guest memory page.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_init_guest_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *p,
+		unsigned long gfn,
+		int (*handler)(void *, u64, void *, int),
+		void *data)
+{
+	INIT_HLIST_NODE(&p->node);
+
+	p->writeprotection = false;
+	p->gfn = gfn;
+	p->handler = handler;
+	p->data = data;
+	p->oos_page = NULL;
+	p->write_cnt = 0;
+
+	hash_add(vgpu->gtt.guest_page_hash_table, &p->node, p->gfn);
+	return 0;
+}
+
+static int detach_oos_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_oos_page *oos_page);
+
+/**
+ * intel_vgpu_clean_guest_page - release the resource owned by guest page data
+ * structure
+ * @vgpu: a vGPU
+ * @p: a tracked guest page
+ *
+ * This function is called when user tries to stop tracking a guest memory
+ * page.
+ */
+void intel_vgpu_clean_guest_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *p)
+{
+	if (!hlist_unhashed(&p->node))
+		hash_del(&p->node);
+
+	if (p->oos_page)
+		detach_oos_page(vgpu, p->oos_page);
+
+	if (p->writeprotection)
+		intel_gvt_hypervisor_unset_wp_page(vgpu, p);
+}
+
+/**
+ * intel_vgpu_find_guest_page - find a guest page data structure by GFN.
+ * @vgpu: a vGPU
+ * @gfn: guest memory page frame number
+ *
+ * This function is called when emulation logic wants to know if a trapped GFN
+ * is a tracked guest page.
+ *
+ * Returns:
+ * Pointer to guest page data structure, NULL if failed.
+ */
+struct intel_vgpu_guest_page *intel_vgpu_find_guest_page(
+		struct intel_vgpu *vgpu, unsigned long gfn)
+{
+	struct intel_vgpu_guest_page *p;
+
+	hash_for_each_possible(vgpu->gtt.guest_page_hash_table,
+		p, node, gfn) {
+		if (p->gfn == gfn)
+			return p;
+	}
+	return NULL;
+}
+
+static inline int init_shadow_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_shadow_page *p, int type)
+{
+	p->vaddr = page_address(p->page);
+	p->type = type;
+
+	INIT_HLIST_NODE(&p->node);
+
+	p->mfn = intel_gvt_hypervisor_virt_to_mfn(p->vaddr);
+	if (p->mfn == INTEL_GVT_INVALID_ADDR)
+		return -EFAULT;
+
+	hash_add(vgpu->gtt.shadow_page_hash_table, &p->node, p->mfn);
+	return 0;
+}
+
+static inline void clean_shadow_page(struct intel_vgpu_shadow_page *p)
+{
+	if (!hlist_unhashed(&p->node))
+		hash_del(&p->node);
+}
+
+static inline struct intel_vgpu_shadow_page *find_shadow_page(
+		struct intel_vgpu *vgpu, unsigned long mfn)
+{
+	struct intel_vgpu_shadow_page *p;
+
+	hash_for_each_possible(vgpu->gtt.shadow_page_hash_table,
+		p, node, mfn) {
+		if (p->mfn == mfn)
+			return p;
+	}
+	return NULL;
+}
+
+#define guest_page_to_ppgtt_spt(ptr) \
+	container_of(ptr, struct intel_vgpu_ppgtt_spt, guest_page)
+
+#define shadow_page_to_ppgtt_spt(ptr) \
+	container_of(ptr, struct intel_vgpu_ppgtt_spt, shadow_page)
+
+static void *alloc_spt(gfp_t gfp_mask)
+{
+	struct intel_vgpu_ppgtt_spt *spt;
+
+	spt = kzalloc(sizeof(*spt), gfp_mask);
+	if (!spt)
+		return NULL;
+
+	spt->shadow_page.page = alloc_page(gfp_mask);
+	if (!spt->shadow_page.page) {
+		kfree(spt);
+		return NULL;
+	}
+	return spt;
+}
+
+static void free_spt(struct intel_vgpu_ppgtt_spt *spt)
+{
+	__free_page(spt->shadow_page.page);
+	kfree(spt);
+}
+
+static void ppgtt_free_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
+{
+	trace_spt_free(spt->vgpu->id, spt, spt->shadow_page.type);
+
+	clean_shadow_page(&spt->shadow_page);
+	intel_vgpu_clean_guest_page(spt->vgpu, &spt->guest_page);
+	list_del_init(&spt->post_shadow_list);
+
+	free_spt(spt);
+}
+
+static void ppgtt_free_all_shadow_page(struct intel_vgpu *vgpu)
+{
+	struct hlist_node *n;
+	struct intel_vgpu_shadow_page *sp;
+	int i;
+
+	hash_for_each_safe(vgpu->gtt.shadow_page_hash_table, i, n, sp, node)
+		ppgtt_free_shadow_page(shadow_page_to_ppgtt_spt(sp));
+}
+
+static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
+		u64 pa, void *p_data, int bytes);
+
+static int ppgtt_write_protection_handler(void *gp, u64 pa,
+		void *p_data, int bytes)
+{
+	struct intel_vgpu_guest_page *gpt = (struct intel_vgpu_guest_page *)gp;
+	int ret;
+
+	if (bytes != 4 && bytes != 8)
+		return -EINVAL;
+
+	if (!gpt->writeprotection)
+		return -EINVAL;
+
+	ret = ppgtt_handle_guest_write_page_table_bytes(gp,
+		pa, p_data, bytes);
+	if (ret)
+		return ret;
+	return ret;
+}
+
+static int reclaim_one_mm(struct intel_gvt *gvt);
+
+static struct intel_vgpu_ppgtt_spt *ppgtt_alloc_shadow_page(
+		struct intel_vgpu *vgpu, int type, unsigned long gfn)
+{
+	struct intel_vgpu_ppgtt_spt *spt = NULL;
+	int ret;
+
+retry:
+	spt = alloc_spt(GFP_KERNEL | __GFP_ZERO);
+	if (!spt) {
+		if (reclaim_one_mm(vgpu->gvt))
+			goto retry;
+
+		gvt_err("fail to allocate ppgtt shadow page\n");
+		return ERR_PTR(-ENOMEM);
+	}
+
+	spt->vgpu = vgpu;
+	spt->guest_page_type = type;
+	atomic_set(&spt->refcount, 1);
+	INIT_LIST_HEAD(&spt->post_shadow_list);
+
+	/*
+	 * TODO: guest page type may be different with shadow page type,
+	 *	 when we support PSE page in future.
+	 */
+	ret = init_shadow_page(vgpu, &spt->shadow_page, type);
+	if (ret) {
+		gvt_err("fail to initialize shadow page for spt\n");
+		goto err;
+	}
+
+	ret = intel_vgpu_init_guest_page(vgpu, &spt->guest_page,
+			gfn, ppgtt_write_protection_handler, NULL);
+	if (ret) {
+		gvt_err("fail to initialize guest page for spt\n");
+		goto err;
+	}
+
+	trace_spt_alloc(vgpu->id, spt, type, spt->shadow_page.mfn, gfn);
+	return spt;
+err:
+	ppgtt_free_shadow_page(spt);
+	return ERR_PTR(ret);
+}
+
+static struct intel_vgpu_ppgtt_spt *ppgtt_find_shadow_page(
+		struct intel_vgpu *vgpu, unsigned long mfn)
+{
+	struct intel_vgpu_shadow_page *p = find_shadow_page(vgpu, mfn);
+
+	if (p)
+		return shadow_page_to_ppgtt_spt(p);
+
+	gvt_err("vgpu%d: fail to find ppgtt shadow page: 0x%lx\n",
+			vgpu->id, mfn);
+	return NULL;
+}
+
+#define pt_entry_size_shift(spt) \
+	((spt)->vgpu->gvt->device_info.gtt_entry_size_shift)
+
+#define pt_entries(spt) \
+	(GTT_PAGE_SIZE >> pt_entry_size_shift(spt))
+
+#define for_each_present_guest_entry(spt, e, i) \
+	for (i = 0; i < pt_entries(spt); i++) \
+	if (spt->vgpu->gvt->gtt.pte_ops->test_present( \
+		ppgtt_get_guest_entry(spt, e, i)))
+
+#define for_each_present_shadow_entry(spt, e, i) \
+	for (i = 0; i < pt_entries(spt); i++) \
+	if (spt->vgpu->gvt->gtt.pte_ops->test_present( \
+		ppgtt_get_shadow_entry(spt, e, i)))
+
+static void ppgtt_get_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
+{
+	int v = atomic_read(&spt->refcount);
+
+	trace_spt_refcount(spt->vgpu->id, "inc", spt, v, (v + 1));
+
+	atomic_inc(&spt->refcount);
+}
+
+static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt);
+
+static int ppgtt_invalidate_shadow_page_by_shadow_entry(struct intel_vgpu *vgpu,
+		struct intel_gvt_gtt_entry *e)
+{
+	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+	struct intel_vgpu_ppgtt_spt *s;
+	intel_gvt_gtt_type_t cur_pt_type;
+
+	if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(e->type))))
+		return -EINVAL;
+
+	if (e->type != GTT_TYPE_PPGTT_ROOT_L3_ENTRY
+		&& e->type != GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
+		cur_pt_type = get_next_pt_type(e->type) + 1;
+		if (ops->get_pfn(e) ==
+			vgpu->gtt.scratch_pt[cur_pt_type].page_mfn)
+			return 0;
+	}
+	s = ppgtt_find_shadow_page(vgpu, ops->get_pfn(e));
+	if (!s) {
+		gvt_err("vgpu%d: fail to find shadow page: mfn: 0x%lx\n",
+				vgpu->id, ops->get_pfn(e));
+		return -ENXIO;
+	}
+	return ppgtt_invalidate_shadow_page(s);
+}
+
+static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
+{
+	struct intel_gvt_gtt_entry e;
+	unsigned long index;
+	int ret;
+	int v = atomic_read(&spt->refcount);
+
+	trace_spt_change(spt->vgpu->id, "die", spt,
+			spt->guest_page.gfn, spt->shadow_page.type);
+
+	trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1));
+
+	if (atomic_dec_return(&spt->refcount) > 0)
+		return 0;
+
+	if (gtt_type_is_pte_pt(spt->shadow_page.type))
+		goto release;
+
+	for_each_present_shadow_entry(spt, &e, index) {
+		if (!gtt_type_is_pt(get_next_pt_type(e.type))) {
+			gvt_err("GVT doesn't support pse bit for now\n");
+			return -EINVAL;
+		}
+		ret = ppgtt_invalidate_shadow_page_by_shadow_entry(
+				spt->vgpu, &e);
+		if (ret)
+			goto fail;
+	}
+release:
+	trace_spt_change(spt->vgpu->id, "release", spt,
+			spt->guest_page.gfn, spt->shadow_page.type);
+	ppgtt_free_shadow_page(spt);
+	return 0;
+fail:
+	gvt_err("vgpu%d: fail: shadow page %p shadow entry 0x%llx type %d\n",
+			spt->vgpu->id, spt, e.val64, e.type);
+	return ret;
+}
+
+static int ppgtt_populate_shadow_page(struct intel_vgpu_ppgtt_spt *spt);
+
+static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry(
+		struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *we)
+{
+	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+	struct intel_vgpu_ppgtt_spt *s = NULL;
+	struct intel_vgpu_guest_page *g;
+	int ret;
+
+	if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(we->type)))) {
+		ret = -EINVAL;
+		goto fail;
+	}
+
+	g = intel_vgpu_find_guest_page(vgpu, ops->get_pfn(we));
+	if (g) {
+		s = guest_page_to_ppgtt_spt(g);
+		ppgtt_get_shadow_page(s);
+	} else {
+		int type = get_next_pt_type(we->type);
+
+		s = ppgtt_alloc_shadow_page(vgpu, type, ops->get_pfn(we));
+		if (IS_ERR(s)) {
+			ret = PTR_ERR(s);
+			goto fail;
+		}
+
+		ret = intel_gvt_hypervisor_set_wp_page(vgpu, &s->guest_page);
+		if (ret)
+			goto fail;
+
+		ret = ppgtt_populate_shadow_page(s);
+		if (ret)
+			goto fail;
+
+		trace_spt_change(vgpu->id, "new", s, s->guest_page.gfn,
+			s->shadow_page.type);
+	}
+	return s;
+fail:
+	gvt_err("vgpu%d: fail: shadow page %p guest entry 0x%llx type %d\n",
+			vgpu->id, s, we->val64, we->type);
+	return ERR_PTR(ret);
+}
+
+static inline void ppgtt_generate_shadow_entry(struct intel_gvt_gtt_entry *se,
+		struct intel_vgpu_ppgtt_spt *s, struct intel_gvt_gtt_entry *ge)
+{
+	struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops;
+
+	se->type = ge->type;
+	se->val64 = ge->val64;
+
+	ops->set_pfn(se, s->shadow_page.mfn);
+}
+
+static int ppgtt_populate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
+{
+	struct intel_vgpu *vgpu = spt->vgpu;
+	struct intel_vgpu_ppgtt_spt *s;
+	struct intel_gvt_gtt_entry se, ge;
+	unsigned long i;
+	int ret;
+
+	trace_spt_change(spt->vgpu->id, "born", spt,
+			spt->guest_page.gfn, spt->shadow_page.type);
+
+	if (gtt_type_is_pte_pt(spt->shadow_page.type)) {
+		for_each_present_guest_entry(spt, &ge, i) {
+			ret = gtt_entry_p2m(vgpu, &ge, &se);
+			if (ret)
+				goto fail;
+			ppgtt_set_shadow_entry(spt, &se, i);
+		}
+		return 0;
+	}
+
+	for_each_present_guest_entry(spt, &ge, i) {
+		if (!gtt_type_is_pt(get_next_pt_type(ge.type))) {
+			gvt_err("GVT doesn't support pse bit now\n");
+			ret = -EINVAL;
+			goto fail;
+		}
+
+		s = ppgtt_populate_shadow_page_by_guest_entry(vgpu, &ge);
+		if (IS_ERR(s)) {
+			ret = PTR_ERR(s);
+			goto fail;
+		}
+		ppgtt_get_shadow_entry(spt, &se, i);
+		ppgtt_generate_shadow_entry(&se, s, &ge);
+		ppgtt_set_shadow_entry(spt, &se, i);
+	}
+	return 0;
+fail:
+	gvt_err("vgpu%d: fail: shadow page %p guest entry 0x%llx type %d\n",
+			vgpu->id, spt, ge.val64, ge.type);
+	return ret;
+}
+
+static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_guest_page *gpt,
+		unsigned long index)
+{
+	struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
+	struct intel_vgpu_shadow_page *sp = &spt->shadow_page;
+	struct intel_vgpu *vgpu = spt->vgpu;
+	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+	struct intel_gvt_gtt_entry e;
+	int ret;
+
+	ppgtt_get_shadow_entry(spt, &e, index);
+
+	trace_gpt_change(spt->vgpu->id, "remove", spt, sp->type, e.val64,
+			 index);
+
+	if (!ops->test_present(&e))
+		return 0;
+
+	if (ops->get_pfn(&e) == vgpu->gtt.scratch_pt[sp->type].page_mfn)
+		return 0;
+
+	if (gtt_type_is_pt(get_next_pt_type(e.type))) {
+		struct intel_vgpu_ppgtt_spt *s =
+			ppgtt_find_shadow_page(vgpu, ops->get_pfn(&e));
+		if (!s) {
+			gvt_err("fail to find guest page\n");
+			ret = -ENXIO;
+			goto fail;
+		}
+		ret = ppgtt_invalidate_shadow_page(s);
+		if (ret)
+			goto fail;
+	}
+	ops->set_pfn(&e, vgpu->gtt.scratch_pt[sp->type].page_mfn);
+	ppgtt_set_shadow_entry(spt, &e, index);
+	return 0;
+fail:
+	gvt_err("vgpu%d: fail: shadow page %p guest entry 0x%llx type %d\n",
+			vgpu->id, spt, e.val64, e.type);
+	return ret;
+}
+
+static int ppgtt_handle_guest_entry_add(struct intel_vgpu_guest_page *gpt,
+		struct intel_gvt_gtt_entry *we, unsigned long index)
+{
+	struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
+	struct intel_vgpu_shadow_page *sp = &spt->shadow_page;
+	struct intel_vgpu *vgpu = spt->vgpu;
+	struct intel_gvt_gtt_entry m;
+	struct intel_vgpu_ppgtt_spt *s;
+	int ret;
+
+	trace_gpt_change(spt->vgpu->id, "add", spt, sp->type,
+		we->val64, index);
+
+	if (gtt_type_is_pt(get_next_pt_type(we->type))) {
+		s = ppgtt_populate_shadow_page_by_guest_entry(vgpu, we);
+		if (IS_ERR(s)) {
+			ret = PTR_ERR(s);
+			goto fail;
+		}
+		ppgtt_get_shadow_entry(spt, &m, index);
+		ppgtt_generate_shadow_entry(&m, s, we);
+		ppgtt_set_shadow_entry(spt, &m, index);
+	} else {
+		ret = gtt_entry_p2m(vgpu, we, &m);
+		if (ret)
+			goto fail;
+		ppgtt_set_shadow_entry(spt, &m, index);
+	}
+	return 0;
+fail:
+	gvt_err("vgpu%d: fail: spt %p guest entry 0x%llx type %d\n", vgpu->id,
+			spt, we->val64, we->type);
+	return ret;
+}
+
+static int sync_oos_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_oos_page *oos_page)
+{
+	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+	struct intel_vgpu_ppgtt_spt *spt =
+		guest_page_to_ppgtt_spt(oos_page->guest_page);
+	struct intel_gvt_gtt_entry old, new, m;
+	int index;
+	int ret;
+
+	trace_oos_change(vgpu->id, "sync", oos_page->id,
+			oos_page->guest_page, spt->guest_page_type);
+
+	old.type = new.type = get_entry_type(spt->guest_page_type);
+	old.val64 = new.val64 = 0;
+
+	for (index = 0; index < (GTT_PAGE_SIZE >> info->gtt_entry_size_shift);
+		index++) {
+		ops->get_entry(oos_page->mem, &old, index, false, 0, vgpu);
+		ops->get_entry(NULL, &new, index, true,
+			oos_page->guest_page->gfn << PAGE_SHIFT, vgpu);
+
+		if (old.val64 == new.val64
+			&& !test_and_clear_bit(index, spt->post_shadow_bitmap))
+			continue;
+
+		trace_oos_sync(vgpu->id, oos_page->id,
+				oos_page->guest_page, spt->guest_page_type,
+				new.val64, index);
+
+		ret = gtt_entry_p2m(vgpu, &new, &m);
+		if (ret)
+			return ret;
+
+		ops->set_entry(oos_page->mem, &new, index, false, 0, vgpu);
+		ppgtt_set_shadow_entry(spt, &m, index);
+	}
+
+	oos_page->guest_page->write_cnt = 0;
+	list_del_init(&spt->post_shadow_list);
+	return 0;
+}
+
+static int detach_oos_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_oos_page *oos_page)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_vgpu_ppgtt_spt *spt =
+		guest_page_to_ppgtt_spt(oos_page->guest_page);
+
+	trace_oos_change(vgpu->id, "detach", oos_page->id,
+			oos_page->guest_page, spt->guest_page_type);
+
+	oos_page->guest_page->write_cnt = 0;
+	oos_page->guest_page->oos_page = NULL;
+	oos_page->guest_page = NULL;
+
+	list_del_init(&oos_page->vm_list);
+	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_free_list_head);
+
+	return 0;
+}
+
+static int attach_oos_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_oos_page *oos_page,
+		struct intel_vgpu_guest_page *gpt)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	int ret;
+
+	ret = intel_gvt_hypervisor_read_gpa(vgpu, gpt->gfn << GTT_PAGE_SHIFT,
+		oos_page->mem, GTT_PAGE_SIZE);
+	if (ret)
+		return ret;
+
+	oos_page->guest_page = gpt;
+	gpt->oos_page = oos_page;
+
+	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_use_list_head);
+
+	trace_oos_change(vgpu->id, "attach", gpt->oos_page->id,
+			gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type);
+	return 0;
+}
+
+static int ppgtt_set_guest_page_sync(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *gpt)
+{
+	int ret;
+
+	ret = intel_gvt_hypervisor_set_wp_page(vgpu, gpt);
+	if (ret)
+		return ret;
+
+	trace_oos_change(vgpu->id, "set page sync", gpt->oos_page->id,
+			gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type);
+
+	list_del_init(&gpt->oos_page->vm_list);
+	return sync_oos_page(vgpu, gpt->oos_page);
+}
+
+static int ppgtt_allocate_oos_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *gpt)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_gtt *gtt = &gvt->gtt;
+	struct intel_vgpu_oos_page *oos_page = gpt->oos_page;
+	int ret;
+
+	WARN(oos_page, "shadow PPGTT page has already has a oos page\n");
+
+	if (list_empty(&gtt->oos_page_free_list_head)) {
+		oos_page = container_of(gtt->oos_page_use_list_head.next,
+			struct intel_vgpu_oos_page, list);
+		ret = ppgtt_set_guest_page_sync(vgpu, oos_page->guest_page);
+		if (ret)
+			return ret;
+		ret = detach_oos_page(vgpu, oos_page);
+		if (ret)
+			return ret;
+	} else
+		oos_page = container_of(gtt->oos_page_free_list_head.next,
+			struct intel_vgpu_oos_page, list);
+	return attach_oos_page(vgpu, oos_page, gpt);
+}
+
+static int ppgtt_set_guest_page_oos(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *gpt)
+{
+	struct intel_vgpu_oos_page *oos_page = gpt->oos_page;
+
+	if (WARN(!oos_page, "shadow PPGTT page should have a oos page\n"))
+		return -EINVAL;
+
+	trace_oos_change(vgpu->id, "set page out of sync", gpt->oos_page->id,
+			gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type);
+
+	list_add_tail(&oos_page->vm_list, &vgpu->gtt.oos_page_list_head);
+	return intel_gvt_hypervisor_unset_wp_page(vgpu, gpt);
+}
+
+/**
+ * intel_vgpu_sync_oos_pages - sync all the out-of-synced shadow for vGPU
+ * @vgpu: a vGPU
+ *
+ * This function is called before submitting a guest workload to host,
+ * to sync all the out-of-synced shadow for vGPU
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu)
+{
+	struct list_head *pos, *n;
+	struct intel_vgpu_oos_page *oos_page;
+	int ret;
+
+	if (!enable_out_of_sync)
+		return 0;
+
+	list_for_each_safe(pos, n, &vgpu->gtt.oos_page_list_head) {
+		oos_page = container_of(pos,
+				struct intel_vgpu_oos_page, vm_list);
+		ret = ppgtt_set_guest_page_sync(vgpu, oos_page->guest_page);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+/*
+ * The heart of PPGTT shadow page table.
+ */
+static int ppgtt_handle_guest_write_page_table(
+		struct intel_vgpu_guest_page *gpt,
+		struct intel_gvt_gtt_entry *we, unsigned long index)
+{
+	struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
+	struct intel_vgpu *vgpu = spt->vgpu;
+	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+
+	int ret;
+	int new_present;
+
+	new_present = ops->test_present(we);
+
+	ret = ppgtt_handle_guest_entry_removal(gpt, index);
+	if (ret)
+		goto fail;
+
+	if (new_present) {
+		ret = ppgtt_handle_guest_entry_add(gpt, we, index);
+		if (ret)
+			goto fail;
+	}
+	return 0;
+fail:
+	gvt_err("vgpu%d: fail: shadow page %p guest entry 0x%llx type %d.\n",
+			vgpu->id, spt, we->val64, we->type);
+	return ret;
+}
+
+static inline bool can_do_out_of_sync(struct intel_vgpu_guest_page *gpt)
+{
+	return enable_out_of_sync
+		&& gtt_type_is_pte_pt(
+			guest_page_to_ppgtt_spt(gpt)->guest_page_type)
+		&& gpt->write_cnt >= 2;
+}
+
+static void ppgtt_set_post_shadow(struct intel_vgpu_ppgtt_spt *spt,
+		unsigned long index)
+{
+	set_bit(index, spt->post_shadow_bitmap);
+	if (!list_empty(&spt->post_shadow_list))
+		return;
+
+	list_add_tail(&spt->post_shadow_list,
+			&spt->vgpu->gtt.post_shadow_list_head);
+}
+
+/**
+ * intel_vgpu_flush_post_shadow - flush the post shadow transactions
+ * @vgpu: a vGPU
+ *
+ * This function is called before submitting a guest workload to host,
+ * to flush all the post shadows for a vGPU.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu)
+{
+	struct list_head *pos, *n;
+	struct intel_vgpu_ppgtt_spt *spt;
+	struct intel_gvt_gtt_entry ge;
+	unsigned long index;
+	int ret;
+
+	list_for_each_safe(pos, n, &vgpu->gtt.post_shadow_list_head) {
+		spt = container_of(pos, struct intel_vgpu_ppgtt_spt,
+				post_shadow_list);
+
+		for_each_set_bit(index, spt->post_shadow_bitmap,
+				GTT_ENTRY_NUM_IN_ONE_PAGE) {
+			ppgtt_get_guest_entry(spt, &ge, index);
+
+			ret = ppgtt_handle_guest_write_page_table(
+					&spt->guest_page, &ge, index);
+			if (ret)
+				return ret;
+			clear_bit(index, spt->post_shadow_bitmap);
+		}
+		list_del_init(&spt->post_shadow_list);
+	}
+	return 0;
+}
+
+static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
+		u64 pa, void *p_data, int bytes)
+{
+	struct intel_vgpu_guest_page *gpt = (struct intel_vgpu_guest_page *)gp;
+	struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
+	struct intel_vgpu *vgpu = spt->vgpu;
+	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+	struct intel_gvt_gtt_entry we;
+	unsigned long index;
+	int ret;
+
+	index = (pa & (PAGE_SIZE - 1)) >> info->gtt_entry_size_shift;
+
+	ppgtt_get_guest_entry(spt, &we, index);
+
+	ops->test_pse(&we);
+
+	if (bytes == info->gtt_entry_size) {
+		ret = ppgtt_handle_guest_write_page_table(gpt, &we, index);
+		if (ret)
+			return ret;
+	} else {
+		if (!test_bit(index, spt->post_shadow_bitmap)) {
+			ret = ppgtt_handle_guest_entry_removal(gpt, index);
+			if (ret)
+				return ret;
+		}
+
+		ppgtt_set_post_shadow(spt, index);
+	}
+
+	if (!enable_out_of_sync)
+		return 0;
+
+	gpt->write_cnt++;
+
+	if (gpt->oos_page)
+		ops->set_entry(gpt->oos_page->mem, &we, index,
+				false, 0, vgpu);
+
+	if (can_do_out_of_sync(gpt)) {
+		if (!gpt->oos_page)
+			ppgtt_allocate_oos_page(vgpu, gpt);
+
+		ret = ppgtt_set_guest_page_oos(vgpu, gpt);
+		if (ret < 0)
+			return ret;
+	}
+	return 0;
+}
+
+/*
+ * mm page table allocation policy for bdw+
+ *  - for ggtt, only virtual page table will be allocated.
+ *  - for ppgtt, dedicated virtual/shadow page table will be allocated.
+ */
+static int gen8_mm_alloc_page_table(struct intel_vgpu_mm *mm)
+{
+	struct intel_vgpu *vgpu = mm->vgpu;
+	struct intel_gvt *gvt = vgpu->gvt;
+	const struct intel_gvt_device_info *info = &gvt->device_info;
+	void *mem;
+
+	if (mm->type == INTEL_GVT_MM_PPGTT) {
+		mm->page_table_entry_cnt = 4;
+		mm->page_table_entry_size = mm->page_table_entry_cnt *
+			info->gtt_entry_size;
+		mem = kzalloc(mm->has_shadow_page_table ?
+			mm->page_table_entry_size * 2
+				: mm->page_table_entry_size,
+			GFP_ATOMIC);
+		if (!mem)
+			return -ENOMEM;
+		mm->virtual_page_table = mem;
+		if (!mm->has_shadow_page_table)
+			return 0;
+		mm->shadow_page_table = mem + mm->page_table_entry_size;
+	} else if (mm->type == INTEL_GVT_MM_GGTT) {
+		mm->page_table_entry_cnt =
+			(gvt_ggtt_gm_sz(gvt) >> GTT_PAGE_SHIFT);
+		mm->page_table_entry_size = mm->page_table_entry_cnt *
+			info->gtt_entry_size;
+		mem = vzalloc(mm->page_table_entry_size);
+		if (!mem)
+			return -ENOMEM;
+		mm->virtual_page_table = mem;
+	}
+	return 0;
+}
+
+static void gen8_mm_free_page_table(struct intel_vgpu_mm *mm)
+{
+	if (mm->type == INTEL_GVT_MM_PPGTT) {
+		kfree(mm->virtual_page_table);
+	} else if (mm->type == INTEL_GVT_MM_GGTT) {
+		if (mm->virtual_page_table)
+			vfree(mm->virtual_page_table);
+	}
+	mm->virtual_page_table = mm->shadow_page_table = NULL;
+}
+
+static void invalidate_mm(struct intel_vgpu_mm *mm)
+{
+	struct intel_vgpu *vgpu = mm->vgpu;
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_gtt *gtt = &gvt->gtt;
+	struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;
+	struct intel_gvt_gtt_entry se;
+	int i;
+
+	if (WARN_ON(!mm->has_shadow_page_table || !mm->shadowed))
+		return;
+
+	for (i = 0; i < mm->page_table_entry_cnt; i++) {
+		ppgtt_get_shadow_root_entry(mm, &se, i);
+		if (!ops->test_present(&se))
+			continue;
+		ppgtt_invalidate_shadow_page_by_shadow_entry(
+				vgpu, &se);
+		se.val64 = 0;
+		ppgtt_set_shadow_root_entry(mm, &se, i);
+
+		trace_gpt_change(vgpu->id, "destroy root pointer",
+				NULL, se.type, se.val64, i);
+	}
+	mm->shadowed = false;
+}
+
+/**
+ * intel_vgpu_destroy_mm - destroy a mm object
+ * @mm: a kref object
+ *
+ * This function is used to destroy a mm object for vGPU
+ *
+ */
+void intel_vgpu_destroy_mm(struct kref *mm_ref)
+{
+	struct intel_vgpu_mm *mm = container_of(mm_ref, typeof(*mm), ref);
+	struct intel_vgpu *vgpu = mm->vgpu;
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_gtt *gtt = &gvt->gtt;
+
+	if (!mm->initialized)
+		goto out;
+
+	list_del(&mm->list);
+	list_del(&mm->lru_list);
+
+	if (mm->has_shadow_page_table)
+		invalidate_mm(mm);
+
+	gtt->mm_free_page_table(mm);
+out:
+	kfree(mm);
+}
+
+static int shadow_mm(struct intel_vgpu_mm *mm)
+{
+	struct intel_vgpu *vgpu = mm->vgpu;
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_gtt *gtt = &gvt->gtt;
+	struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;
+	struct intel_vgpu_ppgtt_spt *spt;
+	struct intel_gvt_gtt_entry ge, se;
+	int i;
+	int ret;
+
+	if (WARN_ON(!mm->has_shadow_page_table || mm->shadowed))
+		return 0;
+
+	mm->shadowed = true;
+
+	for (i = 0; i < mm->page_table_entry_cnt; i++) {
+		ppgtt_get_guest_root_entry(mm, &ge, i);
+		if (!ops->test_present(&ge))
+			continue;
+
+		trace_gpt_change(vgpu->id, __func__, NULL,
+				ge.type, ge.val64, i);
+
+		spt = ppgtt_populate_shadow_page_by_guest_entry(vgpu, &ge);
+		if (IS_ERR(spt)) {
+			gvt_err("fail to populate guest root pointer\n");
+			ret = PTR_ERR(spt);
+			goto fail;
+		}
+		ppgtt_generate_shadow_entry(&se, spt, &ge);
+		ppgtt_set_shadow_root_entry(mm, &se, i);
+
+		trace_gpt_change(vgpu->id, "populate root pointer",
+				NULL, se.type, se.val64, i);
+	}
+	return 0;
+fail:
+	invalidate_mm(mm);
+	return ret;
+}
+
+/**
+ * intel_vgpu_create_mm - create a mm object for a vGPU
+ * @vgpu: a vGPU
+ * @mm_type: mm object type, should be PPGTT or GGTT
+ * @virtual_page_table: page table root pointers. Could be NULL if user wants
+ *	to populate shadow later.
+ * @page_table_level: describe the page table level of the mm object
+ * @pde_base_index: pde root pointer base in GGTT MMIO.
+ *
+ * This function is used to create a mm object for a vGPU.
+ *
+ * Returns:
+ * Zero on success, negative error code in pointer if failed.
+ */
+struct intel_vgpu_mm *intel_vgpu_create_mm(struct intel_vgpu *vgpu,
+		int mm_type, void *virtual_page_table, int page_table_level,
+		u32 pde_base_index)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_gtt *gtt = &gvt->gtt;
+	struct intel_vgpu_mm *mm;
+	int ret;
+
+	mm = kzalloc(sizeof(*mm), GFP_ATOMIC);
+	if (!mm) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	mm->type = mm_type;
+
+	if (page_table_level == 1)
+		mm->page_table_entry_type = GTT_TYPE_GGTT_PTE;
+	else if (page_table_level == 3)
+		mm->page_table_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY;
+	else if (page_table_level == 4)
+		mm->page_table_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY;
+	else {
+		WARN_ON(1);
+		ret = -EINVAL;
+		goto fail;
+	}
+
+	mm->page_table_level = page_table_level;
+	mm->pde_base_index = pde_base_index;
+
+	mm->vgpu = vgpu;
+	mm->has_shadow_page_table = !!(mm_type == INTEL_GVT_MM_PPGTT);
+
+	kref_init(&mm->ref);
+	atomic_set(&mm->pincount, 0);
+	INIT_LIST_HEAD(&mm->list);
+	INIT_LIST_HEAD(&mm->lru_list);
+	list_add_tail(&mm->list, &vgpu->gtt.mm_list_head);
+
+	ret = gtt->mm_alloc_page_table(mm);
+	if (ret) {
+		gvt_err("fail to allocate page table for mm\n");
+		goto fail;
+	}
+
+	mm->initialized = true;
+
+	if (virtual_page_table)
+		memcpy(mm->virtual_page_table, virtual_page_table,
+				mm->page_table_entry_size);
+
+	if (mm->has_shadow_page_table) {
+		ret = shadow_mm(mm);
+		if (ret)
+			goto fail;
+		list_add_tail(&mm->lru_list, &gvt->gtt.mm_lru_list_head);
+	}
+	return mm;
+fail:
+	gvt_err("fail to create mm\n");
+	if (mm)
+		intel_gvt_mm_unreference(mm);
+	return ERR_PTR(ret);
+}
+
+/**
+ * intel_vgpu_unpin_mm - decrease the pin count of a vGPU mm object
+ * @mm: a vGPU mm object
+ *
+ * This function is called when user doesn't want to use a vGPU mm object
+ */
+void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm)
+{
+	if (WARN_ON(mm->type != INTEL_GVT_MM_PPGTT))
+		return;
+
+	atomic_dec(&mm->pincount);
+}
+
+/**
+ * intel_vgpu_pin_mm - increase the pin count of a vGPU mm object
+ * @vgpu: a vGPU
+ *
+ * This function is called when user wants to use a vGPU mm object. If this
+ * mm object hasn't been shadowed yet, the shadow will be populated at this
+ * time.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm)
+{
+	int ret;
+
+	if (WARN_ON(mm->type != INTEL_GVT_MM_PPGTT))
+		return 0;
+
+	atomic_inc(&mm->pincount);
+
+	if (!mm->shadowed) {
+		ret = shadow_mm(mm);
+		if (ret)
+			return ret;
+	}
+
+	list_del_init(&mm->lru_list);
+	list_add_tail(&mm->lru_list, &mm->vgpu->gvt->gtt.mm_lru_list_head);
+	return 0;
+}
+
+static int reclaim_one_mm(struct intel_gvt *gvt)
+{
+	struct intel_vgpu_mm *mm;
+	struct list_head *pos, *n;
+
+	list_for_each_safe(pos, n, &gvt->gtt.mm_lru_list_head) {
+		mm = container_of(pos, struct intel_vgpu_mm, lru_list);
+
+		if (mm->type != INTEL_GVT_MM_PPGTT)
+			continue;
+		if (atomic_read(&mm->pincount))
+			continue;
+
+		list_del_init(&mm->lru_list);
+		invalidate_mm(mm);
+		return 1;
+	}
+	return 0;
+}
+
+/*
+ * GMA translation APIs.
+ */
+static inline int ppgtt_get_next_level_entry(struct intel_vgpu_mm *mm,
+		struct intel_gvt_gtt_entry *e, unsigned long index, bool guest)
+{
+	struct intel_vgpu *vgpu = mm->vgpu;
+	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+	struct intel_vgpu_ppgtt_spt *s;
+
+	if (WARN_ON(!mm->has_shadow_page_table))
+		return -EINVAL;
+
+	s = ppgtt_find_shadow_page(vgpu, ops->get_pfn(e));
+	if (!s)
+		return -ENXIO;
+
+	if (!guest)
+		ppgtt_get_shadow_entry(s, e, index);
+	else
+		ppgtt_get_guest_entry(s, e, index);
+	return 0;
+}
+
+/**
+ * intel_vgpu_gma_to_gpa - translate a gma to GPA
+ * @mm: mm object. could be a PPGTT or GGTT mm object
+ * @gma: graphics memory address in this mm object
+ *
+ * This function is used to translate a graphics memory address in specific
+ * graphics memory space to guest physical address.
+ *
+ * Returns:
+ * Guest physical address on success, INTEL_GVT_INVALID_ADDR if failed.
+ */
+unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
+{
+	struct intel_vgpu *vgpu = mm->vgpu;
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops;
+	struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops;
+	unsigned long gpa = INTEL_GVT_INVALID_ADDR;
+	unsigned long gma_index[4];
+	struct intel_gvt_gtt_entry e;
+	int i, index;
+	int ret;
+
+	if (mm->type != INTEL_GVT_MM_GGTT && mm->type != INTEL_GVT_MM_PPGTT)
+		return INTEL_GVT_INVALID_ADDR;
+
+	if (mm->type == INTEL_GVT_MM_GGTT) {
+		if (!vgpu_gmadr_is_valid(vgpu, gma))
+			goto err;
+
+		ggtt_get_guest_entry(mm, &e,
+			gma_ops->gma_to_ggtt_pte_index(gma));
+		gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT)
+			+ (gma & ~GTT_PAGE_MASK);
+
+		trace_gma_translate(vgpu->id, "ggtt", 0, 0, gma, gpa);
+		return gpa;
+	}
+
+	switch (mm->page_table_level) {
+	case 4:
+		ppgtt_get_shadow_root_entry(mm, &e, 0);
+		gma_index[0] = gma_ops->gma_to_pml4_index(gma);
+		gma_index[1] = gma_ops->gma_to_l4_pdp_index(gma);
+		gma_index[2] = gma_ops->gma_to_pde_index(gma);
+		gma_index[3] = gma_ops->gma_to_pte_index(gma);
+		index = 4;
+		break;
+	case 3:
+		ppgtt_get_shadow_root_entry(mm, &e,
+				gma_ops->gma_to_l3_pdp_index(gma));
+		gma_index[0] = gma_ops->gma_to_pde_index(gma);
+		gma_index[1] = gma_ops->gma_to_pte_index(gma);
+		index = 2;
+		break;
+	case 2:
+		ppgtt_get_shadow_root_entry(mm, &e,
+				gma_ops->gma_to_pde_index(gma));
+		gma_index[0] = gma_ops->gma_to_pte_index(gma);
+		index = 1;
+		break;
+	default:
+		WARN_ON(1);
+		goto err;
+	}
+
+	/* walk into the shadow page table and get gpa from guest entry */
+	for (i = 0; i < index; i++) {
+		ret = ppgtt_get_next_level_entry(mm, &e, gma_index[i],
+			(i == index - 1));
+		if (ret)
+			goto err;
+	}
+
+	gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT)
+		+ (gma & ~GTT_PAGE_MASK);
+
+	trace_gma_translate(vgpu->id, "ppgtt", 0,
+			mm->page_table_level, gma, gpa);
+	return gpa;
+err:
+	gvt_err("invalid mm type: %d gma %lx\n", mm->type, gma);
+	return INTEL_GVT_INVALID_ADDR;
+}
+
+static int emulate_gtt_mmio_read(struct intel_vgpu *vgpu,
+	unsigned int off, void *p_data, unsigned int bytes)
+{
+	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;
+	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+	unsigned long index = off >> info->gtt_entry_size_shift;
+	struct intel_gvt_gtt_entry e;
+
+	if (bytes != 4 && bytes != 8)
+		return -EINVAL;
+
+	ggtt_get_guest_entry(ggtt_mm, &e, index);
+	memcpy(p_data, (void *)&e.val64 + (off & (info->gtt_entry_size - 1)),
+			bytes);
+	return 0;
+}
+
+/**
+ * intel_vgpu_emulate_gtt_mmio_read - emulate GTT MMIO register read
+ * @vgpu: a vGPU
+ * @off: register offset
+ * @p_data: data will be returned to guest
+ * @bytes: data length
+ *
+ * This function is used to emulate the GTT MMIO register read
+ *
+ * Returns:
+ * Zero on success, error code if failed.
+ */
+int intel_vgpu_emulate_gtt_mmio_read(struct intel_vgpu *vgpu, unsigned int off,
+	void *p_data, unsigned int bytes)
+{
+	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+	int ret;
+
+	if (bytes != 4 && bytes != 8)
+		return -EINVAL;
+
+	off -= info->gtt_start_offset;
+	ret = emulate_gtt_mmio_read(vgpu, off, p_data, bytes);
+	return ret;
+}
+
+static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
+	void *p_data, unsigned int bytes)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	const struct intel_gvt_device_info *info = &gvt->device_info;
+	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;
+	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+	unsigned long g_gtt_index = off >> info->gtt_entry_size_shift;
+	unsigned long gma;
+	struct intel_gvt_gtt_entry e, m;
+	int ret;
+
+	if (bytes != 4 && bytes != 8)
+		return -EINVAL;
+
+	gma = g_gtt_index << GTT_PAGE_SHIFT;
+
+	/* the VM may configure the whole GM space when ballooning is used */
+	if (WARN_ONCE(!vgpu_gmadr_is_valid(vgpu, gma),
+				"vgpu%d: found oob ggtt write, offset %x\n",
+				vgpu->id, off)) {
+		return 0;
+	}
+
+	ggtt_get_guest_entry(ggtt_mm, &e, g_gtt_index);
+
+	memcpy((void *)&e.val64 + (off & (info->gtt_entry_size - 1)), p_data,
+			bytes);
+
+	if (ops->test_present(&e)) {
+		ret = gtt_entry_p2m(vgpu, &e, &m);
+		if (ret) {
+			gvt_err("vgpu%d: fail to translate guest gtt entry\n",
+					vgpu->id);
+			return ret;
+		}
+	} else {
+		m = e;
+		m.val64 = 0;
+	}
+
+	ggtt_set_shadow_entry(ggtt_mm, &m, g_gtt_index);
+	ggtt_set_guest_entry(ggtt_mm, &e, g_gtt_index);
+	return 0;
+}
+
+/*
+ * intel_vgpu_emulate_gtt_mmio_write - emulate GTT MMIO register write
+ * @vgpu: a vGPU
+ * @off: register offset
+ * @p_data: data from guest write
+ * @bytes: data length
+ *
+ * This function is used to emulate the GTT MMIO register write
+ *
+ * Returns:
+ * Zero on success, error code if failed.
+ */
+int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
+	void *p_data, unsigned int bytes)
+{
+	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
+	int ret;
+
+	if (bytes != 4 && bytes != 8)
+		return -EINVAL;
+
+	off -= info->gtt_start_offset;
+	ret = emulate_gtt_mmio_write(vgpu, off, p_data, bytes);
+	return ret;
+}
+
+static int alloc_scratch_pages(struct intel_vgpu *vgpu,
+		intel_gvt_gtt_type_t type)
+{
+	struct intel_vgpu_gtt *gtt = &vgpu->gtt;
+	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+	int page_entry_num = GTT_PAGE_SIZE >>
+				vgpu->gvt->device_info.gtt_entry_size_shift;
+	struct page *scratch_pt;
+	unsigned long mfn;
+	int i;
+	void *p;
+
+	if (WARN_ON(type < GTT_TYPE_PPGTT_PTE_PT || type >= GTT_TYPE_MAX))
+		return -EINVAL;
+
+	scratch_pt = alloc_page(GFP_KERNEL | GFP_ATOMIC | __GFP_ZERO);
+	if (!scratch_pt) {
+		gvt_err("fail to allocate scratch page\n");
+		return -ENOMEM;
+	}
+
+	p = kmap_atomic(scratch_pt);
+	mfn = intel_gvt_hypervisor_virt_to_mfn(p);
+	if (mfn == INTEL_GVT_INVALID_ADDR) {
+		gvt_err("fail to translate vaddr:0x%llx\n", (u64)p);
+		kunmap_atomic(p);
+		__free_page(scratch_pt);
+		return -EFAULT;
+	}
+	gtt->scratch_pt[type].page_mfn = mfn;
+	gtt->scratch_pt[type].page = scratch_pt;
+	gvt_dbg_mm("vgpu%d create scratch_pt: type %d mfn=0x%lx\n",
+			vgpu->id, type, mfn);
+
+	/* Build the tree by full filled the scratch pt with the entries which
+	 * point to the next level scratch pt or scratch page. The
+	 * scratch_pt[type] indicate the scratch pt/scratch page used by the
+	 * 'type' pt.
+	 * e.g. scratch_pt[GTT_TYPE_PPGTT_PDE_PT] is used by
+	 * GTT_TYPE_PPGTT_PDE_PT level pt, that means this scatch_pt it self
+	 * is GTT_TYPE_PPGTT_PTE_PT, and full filled by scratch page mfn.
+	 */
+	if (type > GTT_TYPE_PPGTT_PTE_PT && type < GTT_TYPE_MAX) {
+		struct intel_gvt_gtt_entry se;
+
+		memset(&se, 0, sizeof(struct intel_gvt_gtt_entry));
+		se.type = get_entry_type(type - 1);
+		ops->set_pfn(&se, gtt->scratch_pt[type - 1].page_mfn);
+
+		/* The entry parameters like present/writeable/cache type
+		 * set to the same as i915's scratch page tree.
+		 */
+		se.val64 |= _PAGE_PRESENT | _PAGE_RW;
+		if (type == GTT_TYPE_PPGTT_PDE_PT)
+			se.val64 |= PPAT_CACHED_INDEX;
+
+		for (i = 0; i < page_entry_num; i++)
+			ops->set_entry(p, &se, i, false, 0, vgpu);
+	}
+
+	kunmap_atomic(p);
+
+	return 0;
+}
+
+static int release_scratch_page_tree(struct intel_vgpu *vgpu)
+{
+	int i;
+
+	for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) {
+		if (vgpu->gtt.scratch_pt[i].page != NULL) {
+			__free_page(vgpu->gtt.scratch_pt[i].page);
+			vgpu->gtt.scratch_pt[i].page = NULL;
+			vgpu->gtt.scratch_pt[i].page_mfn = 0;
+		}
+	}
+
+	return 0;
+}
+
+static int create_scratch_page_tree(struct intel_vgpu *vgpu)
+{
+	int i, ret;
+
+	for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) {
+		ret = alloc_scratch_pages(vgpu, i);
+		if (ret)
+			goto err;
+	}
+
+	return 0;
+
+err:
+	release_scratch_page_tree(vgpu);
+	return ret;
+}
+
+/**
+ * intel_vgpu_init_gtt - initialize per-vGPU graphics memory virulization
+ * @vgpu: a vGPU
+ *
+ * This function is used to initialize per-vGPU graphics memory virtualization
+ * components.
+ *
+ * Returns:
+ * Zero on success, error code if failed.
+ */
+int intel_vgpu_init_gtt(struct intel_vgpu *vgpu)
+{
+	struct intel_vgpu_gtt *gtt = &vgpu->gtt;
+	struct intel_vgpu_mm *ggtt_mm;
+
+	hash_init(gtt->guest_page_hash_table);
+	hash_init(gtt->shadow_page_hash_table);
+
+	INIT_LIST_HEAD(&gtt->mm_list_head);
+	INIT_LIST_HEAD(&gtt->oos_page_list_head);
+	INIT_LIST_HEAD(&gtt->post_shadow_list_head);
+
+	ggtt_mm = intel_vgpu_create_mm(vgpu, INTEL_GVT_MM_GGTT,
+			NULL, 1, 0);
+	if (IS_ERR(ggtt_mm)) {
+		gvt_err("fail to create mm for ggtt.\n");
+		return PTR_ERR(ggtt_mm);
+	}
+
+	gtt->ggtt_mm = ggtt_mm;
+
+	return create_scratch_page_tree(vgpu);
+}
+
+/**
+ * intel_vgpu_clean_gtt - clean up per-vGPU graphics memory virulization
+ * @vgpu: a vGPU
+ *
+ * This function is used to clean up per-vGPU graphics memory virtualization
+ * components.
+ *
+ * Returns:
+ * Zero on success, error code if failed.
+ */
+void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu)
+{
+	struct list_head *pos, *n;
+	struct intel_vgpu_mm *mm;
+
+	ppgtt_free_all_shadow_page(vgpu);
+	release_scratch_page_tree(vgpu);
+
+	list_for_each_safe(pos, n, &vgpu->gtt.mm_list_head) {
+		mm = container_of(pos, struct intel_vgpu_mm, list);
+		vgpu->gvt->gtt.mm_free_page_table(mm);
+		list_del(&mm->list);
+		list_del(&mm->lru_list);
+		kfree(mm);
+	}
+}
+
+static void clean_spt_oos(struct intel_gvt *gvt)
+{
+	struct intel_gvt_gtt *gtt = &gvt->gtt;
+	struct list_head *pos, *n;
+	struct intel_vgpu_oos_page *oos_page;
+
+	WARN(!list_empty(&gtt->oos_page_use_list_head),
+		"someone is still using oos page\n");
+
+	list_for_each_safe(pos, n, &gtt->oos_page_free_list_head) {
+		oos_page = container_of(pos, struct intel_vgpu_oos_page, list);
+		list_del(&oos_page->list);
+		kfree(oos_page);
+	}
+}
+
+static int setup_spt_oos(struct intel_gvt *gvt)
+{
+	struct intel_gvt_gtt *gtt = &gvt->gtt;
+	struct intel_vgpu_oos_page *oos_page;
+	int i;
+	int ret;
+
+	INIT_LIST_HEAD(&gtt->oos_page_free_list_head);
+	INIT_LIST_HEAD(&gtt->oos_page_use_list_head);
+
+	for (i = 0; i < preallocated_oos_pages; i++) {
+		oos_page = kzalloc(sizeof(*oos_page), GFP_KERNEL);
+		if (!oos_page) {
+			gvt_err("fail to pre-allocate oos page\n");
+			ret = -ENOMEM;
+			goto fail;
+		}
+
+		INIT_LIST_HEAD(&oos_page->list);
+		INIT_LIST_HEAD(&oos_page->vm_list);
+		oos_page->id = i;
+		list_add_tail(&oos_page->list, &gtt->oos_page_free_list_head);
+	}
+
+	gvt_dbg_mm("%d oos pages preallocated\n", i);
+
+	return 0;
+fail:
+	clean_spt_oos(gvt);
+	return ret;
+}
+
+/**
+ * intel_vgpu_find_ppgtt_mm - find a PPGTT mm object
+ * @vgpu: a vGPU
+ * @page_table_level: PPGTT page table level
+ * @root_entry: PPGTT page table root pointers
+ *
+ * This function is used to find a PPGTT mm object from mm object pool
+ *
+ * Returns:
+ * pointer to mm object on success, NULL if failed.
+ */
+struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu,
+		int page_table_level, void *root_entry)
+{
+	struct list_head *pos;
+	struct intel_vgpu_mm *mm;
+	u64 *src, *dst;
+
+	list_for_each(pos, &vgpu->gtt.mm_list_head) {
+		mm = container_of(pos, struct intel_vgpu_mm, list);
+		if (mm->type != INTEL_GVT_MM_PPGTT)
+			continue;
+
+		if (mm->page_table_level != page_table_level)
+			continue;
+
+		src = root_entry;
+		dst = mm->virtual_page_table;
+
+		if (page_table_level == 3) {
+			if (src[0] == dst[0]
+					&& src[1] == dst[1]
+					&& src[2] == dst[2]
+					&& src[3] == dst[3])
+				return mm;
+		} else {
+			if (src[0] == dst[0])
+				return mm;
+		}
+	}
+	return NULL;
+}
+
+/**
+ * intel_vgpu_g2v_create_ppgtt_mm - create a PPGTT mm object from
+ * g2v notification
+ * @vgpu: a vGPU
+ * @page_table_level: PPGTT page table level
+ *
+ * This function is used to create a PPGTT mm object from a guest to GVT-g
+ * notification.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_g2v_create_ppgtt_mm(struct intel_vgpu *vgpu,
+		int page_table_level)
+{
+	u64 *pdp = (u64 *)&vgpu_vreg64(vgpu, vgtif_reg(pdp[0]));
+	struct intel_vgpu_mm *mm;
+
+	if (WARN_ON((page_table_level != 4) && (page_table_level != 3)))
+		return -EINVAL;
+
+	mm = intel_vgpu_find_ppgtt_mm(vgpu, page_table_level, pdp);
+	if (mm) {
+		intel_gvt_mm_reference(mm);
+	} else {
+		mm = intel_vgpu_create_mm(vgpu, INTEL_GVT_MM_PPGTT,
+				pdp, page_table_level, 0);
+		if (IS_ERR(mm)) {
+			gvt_err("fail to create mm\n");
+			return PTR_ERR(mm);
+		}
+	}
+	return 0;
+}
+
+/**
+ * intel_vgpu_g2v_destroy_ppgtt_mm - destroy a PPGTT mm object from
+ * g2v notification
+ * @vgpu: a vGPU
+ * @page_table_level: PPGTT page table level
+ *
+ * This function is used to create a PPGTT mm object from a guest to GVT-g
+ * notification.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_g2v_destroy_ppgtt_mm(struct intel_vgpu *vgpu,
+		int page_table_level)
+{
+	u64 *pdp = (u64 *)&vgpu_vreg64(vgpu, vgtif_reg(pdp[0]));
+	struct intel_vgpu_mm *mm;
+
+	if (WARN_ON((page_table_level != 4) && (page_table_level != 3)))
+		return -EINVAL;
+
+	mm = intel_vgpu_find_ppgtt_mm(vgpu, page_table_level, pdp);
+	if (!mm) {
+		gvt_err("fail to find ppgtt instance.\n");
+		return -EINVAL;
+	}
+	intel_gvt_mm_unreference(mm);
+	return 0;
+}
+
+/**
+ * intel_gvt_init_gtt - initialize mm components of a GVT device
+ * @gvt: GVT device
+ *
+ * This function is called at the initialization stage, to initialize
+ * the mm components of a GVT device.
+ *
+ * Returns:
+ * zero on success, negative error code if failed.
+ */
+int intel_gvt_init_gtt(struct intel_gvt *gvt)
+{
+	int ret;
+
+	gvt_dbg_core("init gtt\n");
+
+	if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)) {
+		gvt->gtt.pte_ops = &gen8_gtt_pte_ops;
+		gvt->gtt.gma_ops = &gen8_gtt_gma_ops;
+		gvt->gtt.mm_alloc_page_table = gen8_mm_alloc_page_table;
+		gvt->gtt.mm_free_page_table = gen8_mm_free_page_table;
+	} else {
+		return -ENODEV;
+	}
+
+	if (enable_out_of_sync) {
+		ret = setup_spt_oos(gvt);
+		if (ret) {
+			gvt_err("fail to initialize SPT oos\n");
+			return ret;
+		}
+	}
+	INIT_LIST_HEAD(&gvt->gtt.mm_lru_list_head);
+	return 0;
+}
+
+/**
+ * intel_gvt_clean_gtt - clean up mm components of a GVT device
+ * @gvt: GVT device
+ *
+ * This function is called at the driver unloading stage, to clean up the
+ * the mm components of a GVT device.
+ *
+ */
+void intel_gvt_clean_gtt(struct intel_gvt *gvt)
+{
+	if (enable_out_of_sync)
+		clean_spt_oos(gvt);
+}
diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h
new file mode 100644
index 000000000000..d250013bc37b
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/gtt.h
@@ -0,0 +1,306 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *    Xiao Zheng <xiao.zheng@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#ifndef _GVT_GTT_H_
+#define _GVT_GTT_H_
+
+#define GTT_PAGE_SHIFT		12
+#define GTT_PAGE_SIZE		(1UL << GTT_PAGE_SHIFT)
+#define GTT_PAGE_MASK		(~(GTT_PAGE_SIZE-1))
+
+struct intel_vgpu_mm;
+
+#define INTEL_GVT_GTT_HASH_BITS 8
+#define INTEL_GVT_INVALID_ADDR (~0UL)
+
+struct intel_gvt_gtt_entry {
+	u64 val64;
+	int type;
+};
+
+struct intel_gvt_gtt_pte_ops {
+	struct intel_gvt_gtt_entry *(*get_entry)(void *pt,
+		struct intel_gvt_gtt_entry *e,
+		unsigned long index, bool hypervisor_access, unsigned long gpa,
+		struct intel_vgpu *vgpu);
+	struct intel_gvt_gtt_entry *(*set_entry)(void *pt,
+		struct intel_gvt_gtt_entry *e,
+		unsigned long index, bool hypervisor_access, unsigned long gpa,
+		struct intel_vgpu *vgpu);
+	bool (*test_present)(struct intel_gvt_gtt_entry *e);
+	void (*clear_present)(struct intel_gvt_gtt_entry *e);
+	bool (*test_pse)(struct intel_gvt_gtt_entry *e);
+	void (*set_pfn)(struct intel_gvt_gtt_entry *e, unsigned long pfn);
+	unsigned long (*get_pfn)(struct intel_gvt_gtt_entry *e);
+};
+
+struct intel_gvt_gtt_gma_ops {
+	unsigned long (*gma_to_ggtt_pte_index)(unsigned long gma);
+	unsigned long (*gma_to_pte_index)(unsigned long gma);
+	unsigned long (*gma_to_pde_index)(unsigned long gma);
+	unsigned long (*gma_to_l3_pdp_index)(unsigned long gma);
+	unsigned long (*gma_to_l4_pdp_index)(unsigned long gma);
+	unsigned long (*gma_to_pml4_index)(unsigned long gma);
+};
+
+struct intel_gvt_gtt {
+	struct intel_gvt_gtt_pte_ops *pte_ops;
+	struct intel_gvt_gtt_gma_ops *gma_ops;
+	int (*mm_alloc_page_table)(struct intel_vgpu_mm *mm);
+	void (*mm_free_page_table)(struct intel_vgpu_mm *mm);
+	struct list_head oos_page_use_list_head;
+	struct list_head oos_page_free_list_head;
+	struct list_head mm_lru_list_head;
+};
+
+enum {
+	INTEL_GVT_MM_GGTT = 0,
+	INTEL_GVT_MM_PPGTT,
+};
+
+typedef enum {
+	GTT_TYPE_INVALID = -1,
+
+	GTT_TYPE_GGTT_PTE,
+
+	GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+	GTT_TYPE_PPGTT_PTE_2M_ENTRY,
+	GTT_TYPE_PPGTT_PTE_1G_ENTRY,
+
+	GTT_TYPE_PPGTT_PTE_ENTRY,
+
+	GTT_TYPE_PPGTT_PDE_ENTRY,
+	GTT_TYPE_PPGTT_PDP_ENTRY,
+	GTT_TYPE_PPGTT_PML4_ENTRY,
+
+	GTT_TYPE_PPGTT_ROOT_ENTRY,
+
+	GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
+	GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
+
+	GTT_TYPE_PPGTT_ENTRY,
+
+	GTT_TYPE_PPGTT_PTE_PT,
+	GTT_TYPE_PPGTT_PDE_PT,
+	GTT_TYPE_PPGTT_PDP_PT,
+	GTT_TYPE_PPGTT_PML4_PT,
+
+	GTT_TYPE_MAX,
+} intel_gvt_gtt_type_t;
+
+struct intel_vgpu_mm {
+	int type;
+	bool initialized;
+	bool shadowed;
+
+	int page_table_entry_type;
+	u32 page_table_entry_size;
+	u32 page_table_entry_cnt;
+	void *virtual_page_table;
+	void *shadow_page_table;
+
+	int page_table_level;
+	bool has_shadow_page_table;
+	u32 pde_base_index;
+
+	struct list_head list;
+	struct kref ref;
+	atomic_t pincount;
+	struct list_head lru_list;
+	struct intel_vgpu *vgpu;
+};
+
+extern struct intel_gvt_gtt_entry *intel_vgpu_mm_get_entry(
+		struct intel_vgpu_mm *mm,
+		void *page_table, struct intel_gvt_gtt_entry *e,
+		unsigned long index);
+
+extern struct intel_gvt_gtt_entry *intel_vgpu_mm_set_entry(
+		struct intel_vgpu_mm *mm,
+		void *page_table, struct intel_gvt_gtt_entry *e,
+		unsigned long index);
+
+#define ggtt_get_guest_entry(mm, e, index) \
+	intel_vgpu_mm_get_entry(mm, mm->virtual_page_table, e, index)
+
+#define ggtt_set_guest_entry(mm, e, index) \
+	intel_vgpu_mm_set_entry(mm, mm->virtual_page_table, e, index)
+
+#define ggtt_get_shadow_entry(mm, e, index) \
+	intel_vgpu_mm_get_entry(mm, mm->shadow_page_table, e, index)
+
+#define ggtt_set_shadow_entry(mm, e, index) \
+	intel_vgpu_mm_set_entry(mm, mm->shadow_page_table, e, index)
+
+#define ppgtt_get_guest_root_entry(mm, e, index) \
+	intel_vgpu_mm_get_entry(mm, mm->virtual_page_table, e, index)
+
+#define ppgtt_set_guest_root_entry(mm, e, index) \
+	intel_vgpu_mm_set_entry(mm, mm->virtual_page_table, e, index)
+
+#define ppgtt_get_shadow_root_entry(mm, e, index) \
+	intel_vgpu_mm_get_entry(mm, mm->shadow_page_table, e, index)
+
+#define ppgtt_set_shadow_root_entry(mm, e, index) \
+	intel_vgpu_mm_set_entry(mm, mm->shadow_page_table, e, index)
+
+extern struct intel_vgpu_mm *intel_vgpu_create_mm(struct intel_vgpu *vgpu,
+		int mm_type, void *virtual_page_table, int page_table_level,
+		u32 pde_base_index);
+extern void intel_vgpu_destroy_mm(struct kref *mm_ref);
+
+struct intel_vgpu_guest_page;
+
+struct intel_vgpu_scratch_pt {
+	struct page *page;
+	unsigned long page_mfn;
+};
+
+
+struct intel_vgpu_gtt {
+	struct intel_vgpu_mm *ggtt_mm;
+	unsigned long active_ppgtt_mm_bitmap;
+	struct list_head mm_list_head;
+	DECLARE_HASHTABLE(shadow_page_hash_table, INTEL_GVT_GTT_HASH_BITS);
+	DECLARE_HASHTABLE(guest_page_hash_table, INTEL_GVT_GTT_HASH_BITS);
+	atomic_t n_write_protected_guest_page;
+	struct list_head oos_page_list_head;
+	struct list_head post_shadow_list_head;
+	struct intel_vgpu_scratch_pt scratch_pt[GTT_TYPE_MAX];
+
+};
+
+extern int intel_vgpu_init_gtt(struct intel_vgpu *vgpu);
+extern void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu);
+
+extern int intel_gvt_init_gtt(struct intel_gvt *gvt);
+extern void intel_gvt_clean_gtt(struct intel_gvt *gvt);
+
+extern struct intel_vgpu_mm *intel_gvt_find_ppgtt_mm(struct intel_vgpu *vgpu,
+		int page_table_level, void *root_entry);
+
+struct intel_vgpu_oos_page;
+
+struct intel_vgpu_shadow_page {
+	void *vaddr;
+	struct page *page;
+	int type;
+	struct hlist_node node;
+	unsigned long mfn;
+};
+
+struct intel_vgpu_guest_page {
+	struct hlist_node node;
+	bool writeprotection;
+	unsigned long gfn;
+	int (*handler)(void *, u64, void *, int);
+	void *data;
+	unsigned long write_cnt;
+	struct intel_vgpu_oos_page *oos_page;
+};
+
+struct intel_vgpu_oos_page {
+	struct intel_vgpu_guest_page *guest_page;
+	struct list_head list;
+	struct list_head vm_list;
+	int id;
+	unsigned char mem[GTT_PAGE_SIZE];
+};
+
+#define GTT_ENTRY_NUM_IN_ONE_PAGE 512
+
+struct intel_vgpu_ppgtt_spt {
+	struct intel_vgpu_shadow_page shadow_page;
+	struct intel_vgpu_guest_page guest_page;
+	int guest_page_type;
+	atomic_t refcount;
+	struct intel_vgpu *vgpu;
+	DECLARE_BITMAP(post_shadow_bitmap, GTT_ENTRY_NUM_IN_ONE_PAGE);
+	struct list_head post_shadow_list;
+};
+
+int intel_vgpu_init_guest_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *guest_page,
+		unsigned long gfn,
+		int (*handler)(void *gp, u64, void *, int),
+		void *data);
+
+void intel_vgpu_clean_guest_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *guest_page);
+
+int intel_vgpu_set_guest_page_writeprotection(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *guest_page);
+
+void intel_vgpu_clear_guest_page_writeprotection(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *guest_page);
+
+struct intel_vgpu_guest_page *intel_vgpu_find_guest_page(
+		struct intel_vgpu *vgpu, unsigned long gfn);
+
+int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu);
+
+int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu);
+
+static inline void intel_gvt_mm_reference(struct intel_vgpu_mm *mm)
+{
+	kref_get(&mm->ref);
+}
+
+static inline void intel_gvt_mm_unreference(struct intel_vgpu_mm *mm)
+{
+	kref_put(&mm->ref, intel_vgpu_destroy_mm);
+}
+
+int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm);
+
+void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm);
+
+unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm,
+		unsigned long gma);
+
+struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu,
+		int page_table_level, void *root_entry);
+
+int intel_vgpu_g2v_create_ppgtt_mm(struct intel_vgpu *vgpu,
+		int page_table_level);
+
+int intel_vgpu_g2v_destroy_ppgtt_mm(struct intel_vgpu *vgpu,
+		int page_table_level);
+
+int intel_vgpu_emulate_gtt_mmio_read(struct intel_vgpu *vgpu,
+	unsigned int off, void *p_data, unsigned int bytes);
+
+int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu,
+	unsigned int off, void *p_data, unsigned int bytes);
+
+#endif /* _GVT_GTT_H_ */
diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c
index 927f4579f5b6..398877c3d2fd 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.c
+++ b/drivers/gpu/drm/i915/gvt/gvt.c
@@ -19,12 +19,23 @@
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Eddie Dong <eddie.dong@intel.com>
+ *
+ * Contributors:
+ *    Niu Bing <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
  */
 
 #include <linux/types.h>
 #include <xen/xen.h>
+#include <linux/kthread.h>
 
 #include "i915_drv.h"
+#include "gvt.h"
 
 struct intel_gvt_host intel_gvt_host;
 
@@ -33,6 +44,16 @@ static const char * const supported_hypervisors[] = {
 	[INTEL_GVT_HYPERVISOR_KVM] = "KVM",
 };
 
+static const struct intel_gvt_ops intel_gvt_ops = {
+	.emulate_cfg_read = intel_vgpu_emulate_cfg_read,
+	.emulate_cfg_write = intel_vgpu_emulate_cfg_write,
+	.emulate_mmio_read = intel_vgpu_emulate_mmio_read,
+	.emulate_mmio_write = intel_vgpu_emulate_mmio_write,
+	.vgpu_create = intel_gvt_create_vgpu,
+	.vgpu_destroy = intel_gvt_destroy_vgpu,
+	.vgpu_reset = intel_gvt_reset_vgpu,
+};
+
 /**
  * intel_gvt_init_host - Load MPT modules and detect if we're running in host
  * @gvt: intel gvt device
@@ -47,6 +68,8 @@ static const char * const supported_hypervisors[] = {
  */
 int intel_gvt_init_host(void)
 {
+	int ret;
+
 	if (intel_gvt_host.initialized)
 		return 0;
 
@@ -61,10 +84,12 @@ int intel_gvt_init_host(void)
 				symbol_get(xengt_mpt), "xengt");
 		intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_XEN;
 	} else {
+#if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT)
 		/* not in Xen. Try KVMGT */
 		intel_gvt_host.mpt = try_then_request_module(
-				symbol_get(kvmgt_mpt), "kvm");
+				symbol_get(kvmgt_mpt), "kvmgt");
 		intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_KVM;
+#endif
 	}
 
 	/* Fail to load MPT modules - bail out */
@@ -72,7 +97,8 @@ int intel_gvt_init_host(void)
 		return -EINVAL;
 
 	/* Try to detect if we're running in host instead of VM. */
-	if (!intel_gvt_hypervisor_detect_host())
+	ret = intel_gvt_hypervisor_detect_host();
+	if (ret)
 		return -ENODEV;
 
 	gvt_dbg_core("Running with hypervisor %s in host mode\n",
@@ -84,9 +110,67 @@ int intel_gvt_init_host(void)
 
 static void init_device_info(struct intel_gvt *gvt)
 {
-	if (IS_BROADWELL(gvt->dev_priv))
-		gvt->device_info.max_support_vgpus = 8;
-	/* This function will grow large in GVT device model patches. */
+	struct intel_gvt_device_info *info = &gvt->device_info;
+	struct pci_dev *pdev = gvt->dev_priv->drm.pdev;
+
+	if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)) {
+		info->max_support_vgpus = 8;
+		info->cfg_space_size = 256;
+		info->mmio_size = 2 * 1024 * 1024;
+		info->mmio_bar = 0;
+		info->gtt_start_offset = 8 * 1024 * 1024;
+		info->gtt_entry_size = 8;
+		info->gtt_entry_size_shift = 3;
+		info->gmadr_bytes_in_cmd = 8;
+		info->max_surface_size = 36 * 1024 * 1024;
+	}
+	info->msi_cap_offset = pdev->msi_cap;
+}
+
+static int gvt_service_thread(void *data)
+{
+	struct intel_gvt *gvt = (struct intel_gvt *)data;
+	int ret;
+
+	gvt_dbg_core("service thread start\n");
+
+	while (!kthread_should_stop()) {
+		ret = wait_event_interruptible(gvt->service_thread_wq,
+				kthread_should_stop() || gvt->service_request);
+
+		if (kthread_should_stop())
+			break;
+
+		if (WARN_ONCE(ret, "service thread is waken up by signal.\n"))
+			continue;
+
+		if (test_and_clear_bit(INTEL_GVT_REQUEST_EMULATE_VBLANK,
+					(void *)&gvt->service_request)) {
+			mutex_lock(&gvt->lock);
+			intel_gvt_emulate_vblank(gvt);
+			mutex_unlock(&gvt->lock);
+		}
+	}
+
+	return 0;
+}
+
+static void clean_service_thread(struct intel_gvt *gvt)
+{
+	kthread_stop(gvt->service_thread);
+}
+
+static int init_service_thread(struct intel_gvt *gvt)
+{
+	init_waitqueue_head(&gvt->service_thread_wq);
+
+	gvt->service_thread = kthread_run(gvt_service_thread,
+			gvt, "gvt_service_thread");
+	if (IS_ERR(gvt->service_thread)) {
+		gvt_err("fail to start service thread.\n");
+		return PTR_ERR(gvt->service_thread);
+	}
+	return 0;
 }
 
 /**
@@ -99,14 +183,26 @@ static void init_device_info(struct intel_gvt *gvt)
  */
 void intel_gvt_clean_device(struct drm_i915_private *dev_priv)
 {
-	struct intel_gvt *gvt = &dev_priv->gvt;
+	struct intel_gvt *gvt = to_gvt(dev_priv);
 
-	if (WARN_ON(!gvt->initialized))
+	if (WARN_ON(!gvt))
 		return;
 
-	/* Other de-initialization of GVT components will be introduced. */
+	clean_service_thread(gvt);
+	intel_gvt_clean_cmd_parser(gvt);
+	intel_gvt_clean_sched_policy(gvt);
+	intel_gvt_clean_workload_scheduler(gvt);
+	intel_gvt_clean_opregion(gvt);
+	intel_gvt_clean_gtt(gvt);
+	intel_gvt_clean_irq(gvt);
+	intel_gvt_clean_mmio_info(gvt);
+	intel_gvt_free_firmware(gvt);
+
+	intel_gvt_hypervisor_host_exit(&dev_priv->drm.pdev->dev, gvt);
+	intel_gvt_clean_vgpu_types(gvt);
 
-	gvt->initialized = false;
+	kfree(dev_priv->gvt);
+	dev_priv->gvt = NULL;
 }
 
 /**
@@ -122,7 +218,9 @@ void intel_gvt_clean_device(struct drm_i915_private *dev_priv)
  */
 int intel_gvt_init_device(struct drm_i915_private *dev_priv)
 {
-	struct intel_gvt *gvt = &dev_priv->gvt;
+	struct intel_gvt *gvt;
+	int ret;
+
 	/*
 	 * Cannot initialize GVT device without intel_gvt_host gets
 	 * initialized first.
@@ -130,16 +228,91 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv)
 	if (WARN_ON(!intel_gvt_host.initialized))
 		return -EINVAL;
 
-	if (WARN_ON(gvt->initialized))
+	if (WARN_ON(dev_priv->gvt))
 		return -EEXIST;
 
+	gvt = kzalloc(sizeof(struct intel_gvt), GFP_KERNEL);
+	if (!gvt)
+		return -ENOMEM;
+
 	gvt_dbg_core("init gvt device\n");
 
+	mutex_init(&gvt->lock);
+	gvt->dev_priv = dev_priv;
+
 	init_device_info(gvt);
-	/*
-	 * Other initialization of GVT components will be introduce here.
-	 */
-	gvt_dbg_core("gvt device creation is done\n");
-	gvt->initialized = true;
+
+	ret = intel_gvt_setup_mmio_info(gvt);
+	if (ret)
+		return ret;
+
+	ret = intel_gvt_load_firmware(gvt);
+	if (ret)
+		goto out_clean_mmio_info;
+
+	ret = intel_gvt_init_irq(gvt);
+	if (ret)
+		goto out_free_firmware;
+
+	ret = intel_gvt_init_gtt(gvt);
+	if (ret)
+		goto out_clean_irq;
+
+	ret = intel_gvt_init_opregion(gvt);
+	if (ret)
+		goto out_clean_gtt;
+
+	ret = intel_gvt_init_workload_scheduler(gvt);
+	if (ret)
+		goto out_clean_opregion;
+
+	ret = intel_gvt_init_sched_policy(gvt);
+	if (ret)
+		goto out_clean_workload_scheduler;
+
+	ret = intel_gvt_init_cmd_parser(gvt);
+	if (ret)
+		goto out_clean_sched_policy;
+
+	ret = init_service_thread(gvt);
+	if (ret)
+		goto out_clean_cmd_parser;
+
+	ret = intel_gvt_init_vgpu_types(gvt);
+	if (ret)
+		goto out_clean_thread;
+
+	ret = intel_gvt_hypervisor_host_init(&dev_priv->drm.pdev->dev, gvt,
+				&intel_gvt_ops);
+	if (ret) {
+		gvt_err("failed to register gvt-g host device: %d\n", ret);
+		goto out_clean_types;
+	}
+
+	gvt_dbg_core("gvt device initialization is done\n");
+	dev_priv->gvt = gvt;
 	return 0;
+
+out_clean_types:
+	intel_gvt_clean_vgpu_types(gvt);
+out_clean_thread:
+	clean_service_thread(gvt);
+out_clean_cmd_parser:
+	intel_gvt_clean_cmd_parser(gvt);
+out_clean_sched_policy:
+	intel_gvt_clean_sched_policy(gvt);
+out_clean_workload_scheduler:
+	intel_gvt_clean_workload_scheduler(gvt);
+out_clean_opregion:
+	intel_gvt_clean_opregion(gvt);
+out_clean_gtt:
+	intel_gvt_clean_gtt(gvt);
+out_clean_irq:
+	intel_gvt_clean_irq(gvt);
+out_free_firmware:
+	intel_gvt_free_firmware(gvt);
+out_clean_mmio_info:
+	intel_gvt_clean_mmio_info(gvt);
+	kfree(gvt);
+	return ret;
 }
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index fb619a6e519d..b1a7c8dd4b5f 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -19,6 +19,15 @@
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Eddie Dong <eddie.dong@intel.com>
+ *
+ * Contributors:
+ *    Niu Bing <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
  */
 
 #ifndef _GVT_H_
@@ -26,6 +35,17 @@
 
 #include "debug.h"
 #include "hypercall.h"
+#include "mmio.h"
+#include "reg.h"
+#include "interrupt.h"
+#include "gtt.h"
+#include "display.h"
+#include "edid.h"
+#include "execlist.h"
+#include "scheduler.h"
+#include "sched_policy.h"
+#include "render.h"
+#include "cmd_parser.h"
 
 #define GVT_MAX_VGPU 8
 
@@ -45,25 +65,381 @@ extern struct intel_gvt_host intel_gvt_host;
 /* Describe per-platform limitations. */
 struct intel_gvt_device_info {
 	u32 max_support_vgpus;
-	/* This data structure will grow bigger in GVT device model patches */
+	u32 cfg_space_size;
+	u32 mmio_size;
+	u32 mmio_bar;
+	unsigned long msi_cap_offset;
+	u32 gtt_start_offset;
+	u32 gtt_entry_size;
+	u32 gtt_entry_size_shift;
+	int gmadr_bytes_in_cmd;
+	u32 max_surface_size;
+};
+
+/* GM resources owned by a vGPU */
+struct intel_vgpu_gm {
+	u64 aperture_sz;
+	u64 hidden_sz;
+	struct drm_mm_node low_gm_node;
+	struct drm_mm_node high_gm_node;
+};
+
+#define INTEL_GVT_MAX_NUM_FENCES 32
+
+/* Fences owned by a vGPU */
+struct intel_vgpu_fence {
+	struct drm_i915_fence_reg *regs[INTEL_GVT_MAX_NUM_FENCES];
+	u32 base;
+	u32 size;
+};
+
+struct intel_vgpu_mmio {
+	void *vreg;
+	void *sreg;
+	bool disable_warn_untrack;
+};
+
+#define INTEL_GVT_MAX_CFG_SPACE_SZ 256
+#define INTEL_GVT_MAX_BAR_NUM 4
+
+struct intel_vgpu_pci_bar {
+	u64 size;
+	bool tracked;
+};
+
+struct intel_vgpu_cfg_space {
+	unsigned char virtual_cfg_space[INTEL_GVT_MAX_CFG_SPACE_SZ];
+	struct intel_vgpu_pci_bar bar[INTEL_GVT_MAX_BAR_NUM];
+};
+
+#define vgpu_cfg_space(vgpu) ((vgpu)->cfg_space.virtual_cfg_space)
+
+#define INTEL_GVT_MAX_PIPE 4
+
+struct intel_vgpu_irq {
+	bool irq_warn_once[INTEL_GVT_EVENT_MAX];
+	DECLARE_BITMAP(flip_done_event[INTEL_GVT_MAX_PIPE],
+		       INTEL_GVT_EVENT_MAX);
+};
+
+struct intel_vgpu_opregion {
+	void *va;
+	u32 gfn[INTEL_GVT_OPREGION_PAGES];
+	struct page *pages[INTEL_GVT_OPREGION_PAGES];
+};
+
+#define vgpu_opregion(vgpu) (&(vgpu->opregion))
+
+#define INTEL_GVT_MAX_PORT 5
+
+struct intel_vgpu_display {
+	struct intel_vgpu_i2c_edid i2c_edid;
+	struct intel_vgpu_port ports[INTEL_GVT_MAX_PORT];
+	struct intel_vgpu_sbi sbi;
 };
 
 struct intel_vgpu {
 	struct intel_gvt *gvt;
 	int id;
 	unsigned long handle; /* vGPU handle used by hypervisor MPT modules */
+	bool active;
+	bool resetting;
+	void *sched_data;
+
+	struct intel_vgpu_fence fence;
+	struct intel_vgpu_gm gm;
+	struct intel_vgpu_cfg_space cfg_space;
+	struct intel_vgpu_mmio mmio;
+	struct intel_vgpu_irq irq;
+	struct intel_vgpu_gtt gtt;
+	struct intel_vgpu_opregion opregion;
+	struct intel_vgpu_display display;
+	struct intel_vgpu_execlist execlist[I915_NUM_ENGINES];
+	struct list_head workload_q_head[I915_NUM_ENGINES];
+	struct kmem_cache *workloads;
+	atomic_t running_workload_num;
+	DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES);
+	struct i915_gem_context *shadow_ctx;
+	struct notifier_block shadow_ctx_notifier_block;
+
+#if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT)
+	struct {
+		struct device *mdev;
+		struct vfio_region *region;
+		int num_regions;
+		struct eventfd_ctx *intx_trigger;
+		struct eventfd_ctx *msi_trigger;
+		struct rb_root cache;
+		struct mutex cache_lock;
+		void *vfio_group;
+		struct notifier_block iommu_notifier;
+	} vdev;
+#endif
+};
+
+struct intel_gvt_gm {
+	unsigned long vgpu_allocated_low_gm_size;
+	unsigned long vgpu_allocated_high_gm_size;
+};
+
+struct intel_gvt_fence {
+	unsigned long vgpu_allocated_fence_num;
+};
+
+#define INTEL_GVT_MMIO_HASH_BITS 9
+
+struct intel_gvt_mmio {
+	u32 *mmio_attribute;
+	DECLARE_HASHTABLE(mmio_info_table, INTEL_GVT_MMIO_HASH_BITS);
+};
+
+struct intel_gvt_firmware {
+	void *cfg_space;
+	void *mmio;
+	bool firmware_loaded;
+};
+
+struct intel_gvt_opregion {
+	void __iomem *opregion_va;
+	u32 opregion_pa;
+};
+
+#define NR_MAX_INTEL_VGPU_TYPES 20
+struct intel_vgpu_type {
+	char name[16];
+	unsigned int max_instance;
+	unsigned int avail_instance;
+	unsigned int low_gm_size;
+	unsigned int high_gm_size;
+	unsigned int fence;
 };
 
 struct intel_gvt {
 	struct mutex lock;
-	bool initialized;
-
 	struct drm_i915_private *dev_priv;
 	struct idr vgpu_idr;	/* vGPU IDR pool */
 
 	struct intel_gvt_device_info device_info;
+	struct intel_gvt_gm gm;
+	struct intel_gvt_fence fence;
+	struct intel_gvt_mmio mmio;
+	struct intel_gvt_firmware firmware;
+	struct intel_gvt_irq irq;
+	struct intel_gvt_gtt gtt;
+	struct intel_gvt_opregion opregion;
+	struct intel_gvt_workload_scheduler scheduler;
+	DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS);
+	struct intel_vgpu_type *types;
+	unsigned int num_types;
+
+	struct task_struct *service_thread;
+	wait_queue_head_t service_thread_wq;
+	unsigned long service_request;
+};
+
+static inline struct intel_gvt *to_gvt(struct drm_i915_private *i915)
+{
+	return i915->gvt;
+}
+
+enum {
+	INTEL_GVT_REQUEST_EMULATE_VBLANK = 0,
 };
 
+static inline void intel_gvt_request_service(struct intel_gvt *gvt,
+		int service)
+{
+	set_bit(service, (void *)&gvt->service_request);
+	wake_up(&gvt->service_thread_wq);
+}
+
+void intel_gvt_free_firmware(struct intel_gvt *gvt);
+int intel_gvt_load_firmware(struct intel_gvt *gvt);
+
+/* Aperture/GM space definitions for GVT device */
+#define MB_TO_BYTES(mb) ((mb) << 20ULL)
+#define BYTES_TO_MB(b) ((b) >> 20ULL)
+
+#define HOST_LOW_GM_SIZE MB_TO_BYTES(128)
+#define HOST_HIGH_GM_SIZE MB_TO_BYTES(384)
+#define HOST_FENCE 4
+
+/* Aperture/GM space definitions for GVT device */
+#define gvt_aperture_sz(gvt)	  (gvt->dev_priv->ggtt.mappable_end)
+#define gvt_aperture_pa_base(gvt) (gvt->dev_priv->ggtt.mappable_base)
+
+#define gvt_ggtt_gm_sz(gvt)	  (gvt->dev_priv->ggtt.base.total)
+#define gvt_ggtt_sz(gvt) \
+	((gvt->dev_priv->ggtt.base.total >> PAGE_SHIFT) << 3)
+#define gvt_hidden_sz(gvt)	  (gvt_ggtt_gm_sz(gvt) - gvt_aperture_sz(gvt))
+
+#define gvt_aperture_gmadr_base(gvt) (0)
+#define gvt_aperture_gmadr_end(gvt) (gvt_aperture_gmadr_base(gvt) \
+				     + gvt_aperture_sz(gvt) - 1)
+
+#define gvt_hidden_gmadr_base(gvt) (gvt_aperture_gmadr_base(gvt) \
+				    + gvt_aperture_sz(gvt))
+#define gvt_hidden_gmadr_end(gvt) (gvt_hidden_gmadr_base(gvt) \
+				   + gvt_hidden_sz(gvt) - 1)
+
+#define gvt_fence_sz(gvt) (gvt->dev_priv->num_fence_regs)
+
+/* Aperture/GM space definitions for vGPU */
+#define vgpu_aperture_offset(vgpu)	((vgpu)->gm.low_gm_node.start)
+#define vgpu_hidden_offset(vgpu)	((vgpu)->gm.high_gm_node.start)
+#define vgpu_aperture_sz(vgpu)		((vgpu)->gm.aperture_sz)
+#define vgpu_hidden_sz(vgpu)		((vgpu)->gm.hidden_sz)
+
+#define vgpu_aperture_pa_base(vgpu) \
+	(gvt_aperture_pa_base(vgpu->gvt) + vgpu_aperture_offset(vgpu))
+
+#define vgpu_ggtt_gm_sz(vgpu) ((vgpu)->gm.aperture_sz + (vgpu)->gm.hidden_sz)
+
+#define vgpu_aperture_pa_end(vgpu) \
+	(vgpu_aperture_pa_base(vgpu) + vgpu_aperture_sz(vgpu) - 1)
+
+#define vgpu_aperture_gmadr_base(vgpu) (vgpu_aperture_offset(vgpu))
+#define vgpu_aperture_gmadr_end(vgpu) \
+	(vgpu_aperture_gmadr_base(vgpu) + vgpu_aperture_sz(vgpu) - 1)
+
+#define vgpu_hidden_gmadr_base(vgpu) (vgpu_hidden_offset(vgpu))
+#define vgpu_hidden_gmadr_end(vgpu) \
+	(vgpu_hidden_gmadr_base(vgpu) + vgpu_hidden_sz(vgpu) - 1)
+
+#define vgpu_fence_base(vgpu) (vgpu->fence.base)
+#define vgpu_fence_sz(vgpu) (vgpu->fence.size)
+
+struct intel_vgpu_creation_params {
+	__u64 handle;
+	__u64 low_gm_sz;  /* in MB */
+	__u64 high_gm_sz; /* in MB */
+	__u64 fence_sz;
+	__s32 primary;
+	__u64 vgpu_id;
+};
+
+int intel_vgpu_alloc_resource(struct intel_vgpu *vgpu,
+			      struct intel_vgpu_creation_params *param);
+void intel_vgpu_free_resource(struct intel_vgpu *vgpu);
+void intel_vgpu_write_fence(struct intel_vgpu *vgpu,
+	u32 fence, u64 value);
+
+/* Macros for easily accessing vGPU virtual/shadow register */
+#define vgpu_vreg(vgpu, reg) \
+	(*(u32 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_vreg8(vgpu, reg) \
+	(*(u8 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_vreg16(vgpu, reg) \
+	(*(u16 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_vreg64(vgpu, reg) \
+	(*(u64 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_sreg(vgpu, reg) \
+	(*(u32 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_sreg8(vgpu, reg) \
+	(*(u8 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_sreg16(vgpu, reg) \
+	(*(u16 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+#define vgpu_sreg64(vgpu, reg) \
+	(*(u64 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+
+#define for_each_active_vgpu(gvt, vgpu, id) \
+	idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) \
+		for_each_if(vgpu->active)
+
+static inline void intel_vgpu_write_pci_bar(struct intel_vgpu *vgpu,
+					    u32 offset, u32 val, bool low)
+{
+	u32 *pval;
+
+	/* BAR offset should be 32 bits algiend */
+	offset = rounddown(offset, 4);
+	pval = (u32 *)(vgpu_cfg_space(vgpu) + offset);
+
+	if (low) {
+		/*
+		 * only update bit 31 - bit 4,
+		 * leave the bit 3 - bit 0 unchanged.
+		 */
+		*pval = (val & GENMASK(31, 4)) | (*pval & GENMASK(3, 0));
+	} else {
+		*pval = val;
+	}
+}
+
+int intel_gvt_init_vgpu_types(struct intel_gvt *gvt);
+void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt);
+
+struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
+					 struct intel_vgpu_type *type);
+void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu);
+void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu);
+
+
+/* validating GM functions */
+#define vgpu_gmadr_is_aperture(vgpu, gmadr) \
+	((gmadr >= vgpu_aperture_gmadr_base(vgpu)) && \
+	 (gmadr <= vgpu_aperture_gmadr_end(vgpu)))
+
+#define vgpu_gmadr_is_hidden(vgpu, gmadr) \
+	((gmadr >= vgpu_hidden_gmadr_base(vgpu)) && \
+	 (gmadr <= vgpu_hidden_gmadr_end(vgpu)))
+
+#define vgpu_gmadr_is_valid(vgpu, gmadr) \
+	 ((vgpu_gmadr_is_aperture(vgpu, gmadr) || \
+	  (vgpu_gmadr_is_hidden(vgpu, gmadr))))
+
+#define gvt_gmadr_is_aperture(gvt, gmadr) \
+	 ((gmadr >= gvt_aperture_gmadr_base(gvt)) && \
+	  (gmadr <= gvt_aperture_gmadr_end(gvt)))
+
+#define gvt_gmadr_is_hidden(gvt, gmadr) \
+	  ((gmadr >= gvt_hidden_gmadr_base(gvt)) && \
+	   (gmadr <= gvt_hidden_gmadr_end(gvt)))
+
+#define gvt_gmadr_is_valid(gvt, gmadr) \
+	  (gvt_gmadr_is_aperture(gvt, gmadr) || \
+	    gvt_gmadr_is_hidden(gvt, gmadr))
+
+bool intel_gvt_ggtt_validate_range(struct intel_vgpu *vgpu, u64 addr, u32 size);
+int intel_gvt_ggtt_gmadr_g2h(struct intel_vgpu *vgpu, u64 g_addr, u64 *h_addr);
+int intel_gvt_ggtt_gmadr_h2g(struct intel_vgpu *vgpu, u64 h_addr, u64 *g_addr);
+int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index,
+			     unsigned long *h_index);
+int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
+			     unsigned long *g_index);
+
+int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes);
+
+int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes);
+
+void intel_gvt_clean_opregion(struct intel_gvt *gvt);
+int intel_gvt_init_opregion(struct intel_gvt *gvt);
+
+void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu);
+int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa);
+
+int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci);
+int setup_vgpu_mmio(struct intel_vgpu *vgpu);
+void populate_pvinfo_page(struct intel_vgpu *vgpu);
+
+struct intel_gvt_ops {
+	int (*emulate_cfg_read)(struct intel_vgpu *, unsigned int, void *,
+				unsigned int);
+	int (*emulate_cfg_write)(struct intel_vgpu *, unsigned int, void *,
+				unsigned int);
+	int (*emulate_mmio_read)(struct intel_vgpu *, u64, void *,
+				unsigned int);
+	int (*emulate_mmio_write)(struct intel_vgpu *, u64, void *,
+				unsigned int);
+	struct intel_vgpu *(*vgpu_create)(struct intel_gvt *,
+				struct intel_vgpu_type *);
+	void (*vgpu_destroy)(struct intel_vgpu *);
+	void (*vgpu_reset)(struct intel_vgpu *);
+};
+
+
 #include "mpt.h"
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
new file mode 100644
index 000000000000..522809710312
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -0,0 +1,2848 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Pei Zhang <pei.zhang@intel.com>
+ *    Niu Bing <bing.niu@intel.com>
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+#include "i915_pvinfo.h"
+
+/* XXX FIXME i915 has changed PP_XXX definition */
+#define PCH_PP_STATUS  _MMIO(0xc7200)
+#define PCH_PP_CONTROL _MMIO(0xc7204)
+#define PCH_PP_ON_DELAYS _MMIO(0xc7208)
+#define PCH_PP_OFF_DELAYS _MMIO(0xc720c)
+#define PCH_PP_DIVISOR _MMIO(0xc7210)
+
+/* Register contains RO bits */
+#define F_RO		(1 << 0)
+/* Register contains graphics address */
+#define F_GMADR		(1 << 1)
+/* Mode mask registers with high 16 bits as the mask bits */
+#define F_MODE_MASK	(1 << 2)
+/* This reg can be accessed by GPU commands */
+#define F_CMD_ACCESS	(1 << 3)
+/* This reg has been accessed by a VM */
+#define F_ACCESSED	(1 << 4)
+/* This reg has been accessed through GPU commands */
+#define F_CMD_ACCESSED	(1 << 5)
+/* This reg could be accessed by unaligned address */
+#define F_UNALIGN	(1 << 6)
+
+unsigned long intel_gvt_get_device_type(struct intel_gvt *gvt)
+{
+	if (IS_BROADWELL(gvt->dev_priv))
+		return D_BDW;
+	else if (IS_SKYLAKE(gvt->dev_priv))
+		return D_SKL;
+
+	return 0;
+}
+
+bool intel_gvt_match_device(struct intel_gvt *gvt,
+		unsigned long device)
+{
+	return intel_gvt_get_device_type(gvt) & device;
+}
+
+static void read_vreg(struct intel_vgpu *vgpu, unsigned int offset,
+	void *p_data, unsigned int bytes)
+{
+	memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes);
+}
+
+static void write_vreg(struct intel_vgpu *vgpu, unsigned int offset,
+	void *p_data, unsigned int bytes)
+{
+	memcpy(&vgpu_vreg(vgpu, offset), p_data, bytes);
+}
+
+static int new_mmio_info(struct intel_gvt *gvt,
+		u32 offset, u32 flags, u32 size,
+		u32 addr_mask, u32 ro_mask, u32 device,
+		void *read, void *write)
+{
+	struct intel_gvt_mmio_info *info, *p;
+	u32 start, end, i;
+
+	if (!intel_gvt_match_device(gvt, device))
+		return 0;
+
+	if (WARN_ON(!IS_ALIGNED(offset, 4)))
+		return -EINVAL;
+
+	start = offset;
+	end = offset + size;
+
+	for (i = start; i < end; i += 4) {
+		info = kzalloc(sizeof(*info), GFP_KERNEL);
+		if (!info)
+			return -ENOMEM;
+
+		info->offset = i;
+		p = intel_gvt_find_mmio_info(gvt, info->offset);
+		if (p)
+			gvt_err("dup mmio definition offset %x\n",
+				info->offset);
+		info->size = size;
+		info->length = (i + 4) < end ? 4 : (end - i);
+		info->addr_mask = addr_mask;
+		info->device = device;
+		info->read = read ? read : intel_vgpu_default_mmio_read;
+		info->write = write ? write : intel_vgpu_default_mmio_write;
+		gvt->mmio.mmio_attribute[info->offset / 4] = flags;
+		INIT_HLIST_NODE(&info->node);
+		hash_add(gvt->mmio.mmio_info_table, &info->node, info->offset);
+	}
+	return 0;
+}
+
+static int render_mmio_to_ring_id(struct intel_gvt *gvt, unsigned int reg)
+{
+	enum intel_engine_id id;
+	struct intel_engine_cs *engine;
+
+	reg &= ~GENMASK(11, 0);
+	for_each_engine(engine, gvt->dev_priv, id) {
+		if (engine->mmio_base == reg)
+			return id;
+	}
+	return -1;
+}
+
+#define offset_to_fence_num(offset) \
+	((offset - i915_mmio_reg_offset(FENCE_REG_GEN6_LO(0))) >> 3)
+
+#define fence_num_to_offset(num) \
+	(num * 8 + i915_mmio_reg_offset(FENCE_REG_GEN6_LO(0)))
+
+static int sanitize_fence_mmio_access(struct intel_vgpu *vgpu,
+		unsigned int fence_num, void *p_data, unsigned int bytes)
+{
+	if (fence_num >= vgpu_fence_sz(vgpu)) {
+		gvt_err("vgpu%d: found oob fence register access\n",
+				vgpu->id);
+		gvt_err("vgpu%d: total fence num %d access fence num %d\n",
+				vgpu->id, vgpu_fence_sz(vgpu), fence_num);
+		memset(p_data, 0, bytes);
+	}
+	return 0;
+}
+
+static int fence_mmio_read(struct intel_vgpu *vgpu, unsigned int off,
+		void *p_data, unsigned int bytes)
+{
+	int ret;
+
+	ret = sanitize_fence_mmio_access(vgpu, offset_to_fence_num(off),
+			p_data, bytes);
+	if (ret)
+		return ret;
+	read_vreg(vgpu, off, p_data, bytes);
+	return 0;
+}
+
+static int fence_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
+		void *p_data, unsigned int bytes)
+{
+	unsigned int fence_num = offset_to_fence_num(off);
+	int ret;
+
+	ret = sanitize_fence_mmio_access(vgpu, fence_num, p_data, bytes);
+	if (ret)
+		return ret;
+	write_vreg(vgpu, off, p_data, bytes);
+
+	intel_vgpu_write_fence(vgpu, fence_num,
+			vgpu_vreg64(vgpu, fence_num_to_offset(fence_num)));
+	return 0;
+}
+
+#define CALC_MODE_MASK_REG(old, new) \
+	(((new) & GENMASK(31, 16)) \
+	 | ((((old) & GENMASK(15, 0)) & ~((new) >> 16)) \
+	 | ((new) & ((new) >> 16))))
+
+static int mul_force_wake_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	u32 old, new;
+	uint32_t ack_reg_offset;
+
+	old = vgpu_vreg(vgpu, offset);
+	new = CALC_MODE_MASK_REG(old, *(u32 *)p_data);
+
+	if (IS_SKYLAKE(vgpu->gvt->dev_priv)) {
+		switch (offset) {
+		case FORCEWAKE_RENDER_GEN9_REG:
+			ack_reg_offset = FORCEWAKE_ACK_RENDER_GEN9_REG;
+			break;
+		case FORCEWAKE_BLITTER_GEN9_REG:
+			ack_reg_offset = FORCEWAKE_ACK_BLITTER_GEN9_REG;
+			break;
+		case FORCEWAKE_MEDIA_GEN9_REG:
+			ack_reg_offset = FORCEWAKE_ACK_MEDIA_GEN9_REG;
+			break;
+		default:
+			/*should not hit here*/
+			gvt_err("invalid forcewake offset 0x%x\n", offset);
+			return 1;
+		}
+	} else {
+		ack_reg_offset = FORCEWAKE_ACK_HSW_REG;
+	}
+
+	vgpu_vreg(vgpu, offset) = new;
+	vgpu_vreg(vgpu, ack_reg_offset) = (new & GENMASK(15, 0));
+	return 0;
+}
+
+static int handle_device_reset(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes, unsigned long bitmap)
+{
+	struct intel_gvt_workload_scheduler *scheduler =
+		&vgpu->gvt->scheduler;
+
+	vgpu->resetting = true;
+
+	intel_vgpu_stop_schedule(vgpu);
+	/*
+	 * The current_vgpu will set to NULL after stopping the
+	 * scheduler when the reset is triggered by current vgpu.
+	 */
+	if (scheduler->current_vgpu == NULL) {
+		mutex_unlock(&vgpu->gvt->lock);
+		intel_gvt_wait_vgpu_idle(vgpu);
+		mutex_lock(&vgpu->gvt->lock);
+	}
+
+	intel_vgpu_reset_execlist(vgpu, bitmap);
+
+	/* full GPU reset */
+	if (bitmap == 0xff) {
+		mutex_unlock(&vgpu->gvt->lock);
+		intel_vgpu_clean_gtt(vgpu);
+		mutex_lock(&vgpu->gvt->lock);
+		setup_vgpu_mmio(vgpu);
+		populate_pvinfo_page(vgpu);
+		intel_vgpu_init_gtt(vgpu);
+	}
+
+	vgpu->resetting = false;
+
+	return 0;
+}
+
+static int gdrst_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 data;
+	u64 bitmap = 0;
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	data = vgpu_vreg(vgpu, offset);
+
+	if (data & GEN6_GRDOM_FULL) {
+		gvt_dbg_mmio("vgpu%d: request full GPU reset\n", vgpu->id);
+		bitmap = 0xff;
+	}
+	if (data & GEN6_GRDOM_RENDER) {
+		gvt_dbg_mmio("vgpu%d: request RCS reset\n", vgpu->id);
+		bitmap |= (1 << RCS);
+	}
+	if (data & GEN6_GRDOM_MEDIA) {
+		gvt_dbg_mmio("vgpu%d: request VCS reset\n", vgpu->id);
+		bitmap |= (1 << VCS);
+	}
+	if (data & GEN6_GRDOM_BLT) {
+		gvt_dbg_mmio("vgpu%d: request BCS Reset\n", vgpu->id);
+		bitmap |= (1 << BCS);
+	}
+	if (data & GEN6_GRDOM_VECS) {
+		gvt_dbg_mmio("vgpu%d: request VECS Reset\n", vgpu->id);
+		bitmap |= (1 << VECS);
+	}
+	if (data & GEN8_GRDOM_MEDIA2) {
+		gvt_dbg_mmio("vgpu%d: request VCS2 Reset\n", vgpu->id);
+		if (HAS_BSD2(vgpu->gvt->dev_priv))
+			bitmap |= (1 << VCS2);
+	}
+	return handle_device_reset(vgpu, offset, p_data, bytes, bitmap);
+}
+
+static int gmbus_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	return intel_gvt_i2c_handle_gmbus_read(vgpu, offset, p_data, bytes);
+}
+
+static int gmbus_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	return intel_gvt_i2c_handle_gmbus_write(vgpu, offset, p_data, bytes);
+}
+
+static int pch_pp_control_mmio_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	write_vreg(vgpu, offset, p_data, bytes);
+
+	if (vgpu_vreg(vgpu, offset) & PANEL_POWER_ON) {
+		vgpu_vreg(vgpu, PCH_PP_STATUS) |= PP_ON;
+		vgpu_vreg(vgpu, PCH_PP_STATUS) |= PP_SEQUENCE_STATE_ON_IDLE;
+		vgpu_vreg(vgpu, PCH_PP_STATUS) &= ~PP_SEQUENCE_POWER_DOWN;
+		vgpu_vreg(vgpu, PCH_PP_STATUS) &= ~PP_CYCLE_DELAY_ACTIVE;
+
+	} else
+		vgpu_vreg(vgpu, PCH_PP_STATUS) &=
+			~(PP_ON | PP_SEQUENCE_POWER_DOWN
+					| PP_CYCLE_DELAY_ACTIVE);
+	return 0;
+}
+
+static int transconf_mmio_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	write_vreg(vgpu, offset, p_data, bytes);
+
+	if (vgpu_vreg(vgpu, offset) & TRANS_ENABLE)
+		vgpu_vreg(vgpu, offset) |= TRANS_STATE_ENABLE;
+	else
+		vgpu_vreg(vgpu, offset) &= ~TRANS_STATE_ENABLE;
+	return 0;
+}
+
+static int lcpll_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	write_vreg(vgpu, offset, p_data, bytes);
+
+	if (vgpu_vreg(vgpu, offset) & LCPLL_PLL_DISABLE)
+		vgpu_vreg(vgpu, offset) &= ~LCPLL_PLL_LOCK;
+	else
+		vgpu_vreg(vgpu, offset) |= LCPLL_PLL_LOCK;
+
+	if (vgpu_vreg(vgpu, offset) & LCPLL_CD_SOURCE_FCLK)
+		vgpu_vreg(vgpu, offset) |= LCPLL_CD_SOURCE_FCLK_DONE;
+	else
+		vgpu_vreg(vgpu, offset) &= ~LCPLL_CD_SOURCE_FCLK_DONE;
+
+	return 0;
+}
+
+static int dpy_reg_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	*(u32 *)p_data = (1 << 17);
+	return 0;
+}
+
+static int dpy_reg_mmio_read_2(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	*(u32 *)p_data = 3;
+	return 0;
+}
+
+static int dpy_reg_mmio_read_3(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	*(u32 *)p_data = (0x2f << 16);
+	return 0;
+}
+
+static int pipeconf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 data;
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	data = vgpu_vreg(vgpu, offset);
+
+	if (data & PIPECONF_ENABLE)
+		vgpu_vreg(vgpu, offset) |= I965_PIPECONF_ACTIVE;
+	else
+		vgpu_vreg(vgpu, offset) &= ~I965_PIPECONF_ACTIVE;
+	intel_gvt_check_vblank_emulation(vgpu->gvt);
+	return 0;
+}
+
+static int ddi_buf_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	write_vreg(vgpu, offset, p_data, bytes);
+
+	if (vgpu_vreg(vgpu, offset) & DDI_BUF_CTL_ENABLE) {
+		vgpu_vreg(vgpu, offset) &= ~DDI_BUF_IS_IDLE;
+	} else {
+		vgpu_vreg(vgpu, offset) |= DDI_BUF_IS_IDLE;
+		if (offset == i915_mmio_reg_offset(DDI_BUF_CTL(PORT_E)))
+			vgpu_vreg(vgpu, DP_TP_STATUS(PORT_E))
+				&= ~DP_TP_STATUS_AUTOTRAIN_DONE;
+	}
+	return 0;
+}
+
+static int fdi_rx_iir_mmio_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	vgpu_vreg(vgpu, offset) &= ~*(u32 *)p_data;
+	return 0;
+}
+
+#define FDI_LINK_TRAIN_PATTERN1         0
+#define FDI_LINK_TRAIN_PATTERN2         1
+
+static int fdi_auto_training_started(struct intel_vgpu *vgpu)
+{
+	u32 ddi_buf_ctl = vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_E));
+	u32 rx_ctl = vgpu_vreg(vgpu, _FDI_RXA_CTL);
+	u32 tx_ctl = vgpu_vreg(vgpu, DP_TP_CTL(PORT_E));
+
+	if ((ddi_buf_ctl & DDI_BUF_CTL_ENABLE) &&
+			(rx_ctl & FDI_RX_ENABLE) &&
+			(rx_ctl & FDI_AUTO_TRAINING) &&
+			(tx_ctl & DP_TP_CTL_ENABLE) &&
+			(tx_ctl & DP_TP_CTL_FDI_AUTOTRAIN))
+		return 1;
+	else
+		return 0;
+}
+
+static int check_fdi_rx_train_status(struct intel_vgpu *vgpu,
+		enum pipe pipe, unsigned int train_pattern)
+{
+	i915_reg_t fdi_rx_imr, fdi_tx_ctl, fdi_rx_ctl;
+	unsigned int fdi_rx_check_bits, fdi_tx_check_bits;
+	unsigned int fdi_rx_train_bits, fdi_tx_train_bits;
+	unsigned int fdi_iir_check_bits;
+
+	fdi_rx_imr = FDI_RX_IMR(pipe);
+	fdi_tx_ctl = FDI_TX_CTL(pipe);
+	fdi_rx_ctl = FDI_RX_CTL(pipe);
+
+	if (train_pattern == FDI_LINK_TRAIN_PATTERN1) {
+		fdi_rx_train_bits = FDI_LINK_TRAIN_PATTERN_1_CPT;
+		fdi_tx_train_bits = FDI_LINK_TRAIN_PATTERN_1;
+		fdi_iir_check_bits = FDI_RX_BIT_LOCK;
+	} else if (train_pattern == FDI_LINK_TRAIN_PATTERN2) {
+		fdi_rx_train_bits = FDI_LINK_TRAIN_PATTERN_2_CPT;
+		fdi_tx_train_bits = FDI_LINK_TRAIN_PATTERN_2;
+		fdi_iir_check_bits = FDI_RX_SYMBOL_LOCK;
+	} else {
+		gvt_err("Invalid train pattern %d\n", train_pattern);
+		return -EINVAL;
+	}
+
+	fdi_rx_check_bits = FDI_RX_ENABLE | fdi_rx_train_bits;
+	fdi_tx_check_bits = FDI_TX_ENABLE | fdi_tx_train_bits;
+
+	/* If imr bit has been masked */
+	if (vgpu_vreg(vgpu, fdi_rx_imr) & fdi_iir_check_bits)
+		return 0;
+
+	if (((vgpu_vreg(vgpu, fdi_tx_ctl) & fdi_tx_check_bits)
+			== fdi_tx_check_bits)
+		&& ((vgpu_vreg(vgpu, fdi_rx_ctl) & fdi_rx_check_bits)
+			== fdi_rx_check_bits))
+		return 1;
+	else
+		return 0;
+}
+
+#define INVALID_INDEX (~0U)
+
+static unsigned int calc_index(unsigned int offset, unsigned int start,
+	unsigned int next, unsigned int end, i915_reg_t i915_end)
+{
+	unsigned int range = next - start;
+
+	if (!end)
+		end = i915_mmio_reg_offset(i915_end);
+	if (offset < start || offset > end)
+		return INVALID_INDEX;
+	offset -= start;
+	return offset / range;
+}
+
+#define FDI_RX_CTL_TO_PIPE(offset) \
+	calc_index(offset, _FDI_RXA_CTL, _FDI_RXB_CTL, 0, FDI_RX_CTL(PIPE_C))
+
+#define FDI_TX_CTL_TO_PIPE(offset) \
+	calc_index(offset, _FDI_TXA_CTL, _FDI_TXB_CTL, 0, FDI_TX_CTL(PIPE_C))
+
+#define FDI_RX_IMR_TO_PIPE(offset) \
+	calc_index(offset, _FDI_RXA_IMR, _FDI_RXB_IMR, 0, FDI_RX_IMR(PIPE_C))
+
+static int update_fdi_rx_iir_status(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	i915_reg_t fdi_rx_iir;
+	unsigned int index;
+	int ret;
+
+	if (FDI_RX_CTL_TO_PIPE(offset) != INVALID_INDEX)
+		index = FDI_RX_CTL_TO_PIPE(offset);
+	else if (FDI_TX_CTL_TO_PIPE(offset) != INVALID_INDEX)
+		index = FDI_TX_CTL_TO_PIPE(offset);
+	else if (FDI_RX_IMR_TO_PIPE(offset) != INVALID_INDEX)
+		index = FDI_RX_IMR_TO_PIPE(offset);
+	else {
+		gvt_err("Unsupport registers %x\n", offset);
+		return -EINVAL;
+	}
+
+	write_vreg(vgpu, offset, p_data, bytes);
+
+	fdi_rx_iir = FDI_RX_IIR(index);
+
+	ret = check_fdi_rx_train_status(vgpu, index, FDI_LINK_TRAIN_PATTERN1);
+	if (ret < 0)
+		return ret;
+	if (ret)
+		vgpu_vreg(vgpu, fdi_rx_iir) |= FDI_RX_BIT_LOCK;
+
+	ret = check_fdi_rx_train_status(vgpu, index, FDI_LINK_TRAIN_PATTERN2);
+	if (ret < 0)
+		return ret;
+	if (ret)
+		vgpu_vreg(vgpu, fdi_rx_iir) |= FDI_RX_SYMBOL_LOCK;
+
+	if (offset == _FDI_RXA_CTL)
+		if (fdi_auto_training_started(vgpu))
+			vgpu_vreg(vgpu, DP_TP_STATUS(PORT_E)) |=
+				DP_TP_STATUS_AUTOTRAIN_DONE;
+	return 0;
+}
+
+#define DP_TP_CTL_TO_PORT(offset) \
+	calc_index(offset, _DP_TP_CTL_A, _DP_TP_CTL_B, 0, DP_TP_CTL(PORT_E))
+
+static int dp_tp_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	i915_reg_t status_reg;
+	unsigned int index;
+	u32 data;
+
+	write_vreg(vgpu, offset, p_data, bytes);
+
+	index = DP_TP_CTL_TO_PORT(offset);
+	data = (vgpu_vreg(vgpu, offset) & GENMASK(10, 8)) >> 8;
+	if (data == 0x2) {
+		status_reg = DP_TP_STATUS(index);
+		vgpu_vreg(vgpu, status_reg) |= (1 << 25);
+	}
+	return 0;
+}
+
+static int dp_tp_status_mmio_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	u32 reg_val;
+	u32 sticky_mask;
+
+	reg_val = *((u32 *)p_data);
+	sticky_mask = GENMASK(27, 26) | (1 << 24);
+
+	vgpu_vreg(vgpu, offset) = (reg_val & ~sticky_mask) |
+		(vgpu_vreg(vgpu, offset) & sticky_mask);
+	vgpu_vreg(vgpu, offset) &= ~(reg_val & sticky_mask);
+	return 0;
+}
+
+static int pch_adpa_mmio_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	u32 data;
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	data = vgpu_vreg(vgpu, offset);
+
+	if (data & ADPA_CRT_HOTPLUG_FORCE_TRIGGER)
+		vgpu_vreg(vgpu, offset) &= ~ADPA_CRT_HOTPLUG_FORCE_TRIGGER;
+	return 0;
+}
+
+static int south_chicken2_mmio_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	u32 data;
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	data = vgpu_vreg(vgpu, offset);
+
+	if (data & FDI_MPHY_IOSFSB_RESET_CTL)
+		vgpu_vreg(vgpu, offset) |= FDI_MPHY_IOSFSB_RESET_STATUS;
+	else
+		vgpu_vreg(vgpu, offset) &= ~FDI_MPHY_IOSFSB_RESET_STATUS;
+	return 0;
+}
+
+#define DSPSURF_TO_PIPE(offset) \
+	calc_index(offset, _DSPASURF, _DSPBSURF, 0, DSPSURF(PIPE_C))
+
+static int pri_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	unsigned int index = DSPSURF_TO_PIPE(offset);
+	i915_reg_t surflive_reg = DSPSURFLIVE(index);
+	int flip_event[] = {
+		[PIPE_A] = PRIMARY_A_FLIP_DONE,
+		[PIPE_B] = PRIMARY_B_FLIP_DONE,
+		[PIPE_C] = PRIMARY_C_FLIP_DONE,
+	};
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	vgpu_vreg(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset);
+
+	set_bit(flip_event[index], vgpu->irq.flip_done_event[index]);
+	return 0;
+}
+
+#define SPRSURF_TO_PIPE(offset) \
+	calc_index(offset, _SPRA_SURF, _SPRB_SURF, 0, SPRSURF(PIPE_C))
+
+static int spr_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	unsigned int index = SPRSURF_TO_PIPE(offset);
+	i915_reg_t surflive_reg = SPRSURFLIVE(index);
+	int flip_event[] = {
+		[PIPE_A] = SPRITE_A_FLIP_DONE,
+		[PIPE_B] = SPRITE_B_FLIP_DONE,
+		[PIPE_C] = SPRITE_C_FLIP_DONE,
+	};
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	vgpu_vreg(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset);
+
+	set_bit(flip_event[index], vgpu->irq.flip_done_event[index]);
+	return 0;
+}
+
+static int trigger_aux_channel_interrupt(struct intel_vgpu *vgpu,
+		unsigned int reg)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	enum intel_gvt_event_type event;
+
+	if (reg == _DPA_AUX_CH_CTL)
+		event = AUX_CHANNEL_A;
+	else if (reg == _PCH_DPB_AUX_CH_CTL || reg == _DPB_AUX_CH_CTL)
+		event = AUX_CHANNEL_B;
+	else if (reg == _PCH_DPC_AUX_CH_CTL || reg == _DPC_AUX_CH_CTL)
+		event = AUX_CHANNEL_C;
+	else if (reg == _PCH_DPD_AUX_CH_CTL || reg == _DPD_AUX_CH_CTL)
+		event = AUX_CHANNEL_D;
+	else {
+		WARN_ON(true);
+		return -EINVAL;
+	}
+
+	intel_vgpu_trigger_virtual_event(vgpu, event);
+	return 0;
+}
+
+static int dp_aux_ch_ctl_trans_done(struct intel_vgpu *vgpu, u32 value,
+		unsigned int reg, int len, bool data_valid)
+{
+	/* mark transaction done */
+	value |= DP_AUX_CH_CTL_DONE;
+	value &= ~DP_AUX_CH_CTL_SEND_BUSY;
+	value &= ~DP_AUX_CH_CTL_RECEIVE_ERROR;
+
+	if (data_valid)
+		value &= ~DP_AUX_CH_CTL_TIME_OUT_ERROR;
+	else
+		value |= DP_AUX_CH_CTL_TIME_OUT_ERROR;
+
+	/* message size */
+	value &= ~(0xf << 20);
+	value |= (len << 20);
+	vgpu_vreg(vgpu, reg) = value;
+
+	if (value & DP_AUX_CH_CTL_INTERRUPT)
+		return trigger_aux_channel_interrupt(vgpu, reg);
+	return 0;
+}
+
+static void dp_aux_ch_ctl_link_training(struct intel_vgpu_dpcd_data *dpcd,
+		uint8_t t)
+{
+	if ((t & DPCD_TRAINING_PATTERN_SET_MASK) == DPCD_TRAINING_PATTERN_1) {
+		/* training pattern 1 for CR */
+		/* set LANE0_CR_DONE, LANE1_CR_DONE */
+		dpcd->data[DPCD_LANE0_1_STATUS] |= DPCD_LANES_CR_DONE;
+		/* set LANE2_CR_DONE, LANE3_CR_DONE */
+		dpcd->data[DPCD_LANE2_3_STATUS] |= DPCD_LANES_CR_DONE;
+	} else if ((t & DPCD_TRAINING_PATTERN_SET_MASK) ==
+			DPCD_TRAINING_PATTERN_2) {
+		/* training pattern 2 for EQ */
+		/* Set CHANNEL_EQ_DONE and  SYMBOL_LOCKED for Lane0_1 */
+		dpcd->data[DPCD_LANE0_1_STATUS] |= DPCD_LANES_EQ_DONE;
+		dpcd->data[DPCD_LANE0_1_STATUS] |= DPCD_SYMBOL_LOCKED;
+		/* Set CHANNEL_EQ_DONE and  SYMBOL_LOCKED for Lane2_3 */
+		dpcd->data[DPCD_LANE2_3_STATUS] |= DPCD_LANES_EQ_DONE;
+		dpcd->data[DPCD_LANE2_3_STATUS] |= DPCD_SYMBOL_LOCKED;
+		/* set INTERLANE_ALIGN_DONE */
+		dpcd->data[DPCD_LANE_ALIGN_STATUS_UPDATED] |=
+			DPCD_INTERLANE_ALIGN_DONE;
+	} else if ((t & DPCD_TRAINING_PATTERN_SET_MASK) ==
+			DPCD_LINK_TRAINING_DISABLED) {
+		/* finish link training */
+		/* set sink status as synchronized */
+		dpcd->data[DPCD_SINK_STATUS] = DPCD_SINK_IN_SYNC;
+	}
+}
+
+#define _REG_HSW_DP_AUX_CH_CTL(dp) \
+	((dp) ? (_PCH_DPB_AUX_CH_CTL + ((dp)-1)*0x100) : 0x64010)
+
+#define _REG_SKL_DP_AUX_CH_CTL(dp) (0x64010 + (dp) * 0x100)
+
+#define OFFSET_TO_DP_AUX_PORT(offset) (((offset) & 0xF00) >> 8)
+
+#define dpy_is_valid_port(port)	\
+		(((port) >= PORT_A) && ((port) < I915_MAX_PORTS))
+
+static int dp_aux_ch_ctl_mmio_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	struct intel_vgpu_display *display = &vgpu->display;
+	int msg, addr, ctrl, op, len;
+	int port_index = OFFSET_TO_DP_AUX_PORT(offset);
+	struct intel_vgpu_dpcd_data *dpcd = NULL;
+	struct intel_vgpu_port *port = NULL;
+	u32 data;
+
+	if (!dpy_is_valid_port(port_index)) {
+		gvt_err("GVT(%d): Unsupported DP port access!\n", vgpu->id);
+		return 0;
+	}
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	data = vgpu_vreg(vgpu, offset);
+
+	if (IS_SKYLAKE(vgpu->gvt->dev_priv) &&
+	    offset != _REG_SKL_DP_AUX_CH_CTL(port_index)) {
+		/* SKL DPB/C/D aux ctl register changed */
+		return 0;
+	} else if (IS_BROADWELL(vgpu->gvt->dev_priv) &&
+		   offset != _REG_HSW_DP_AUX_CH_CTL(port_index)) {
+		/* write to the data registers */
+		return 0;
+	}
+
+	if (!(data & DP_AUX_CH_CTL_SEND_BUSY)) {
+		/* just want to clear the sticky bits */
+		vgpu_vreg(vgpu, offset) = 0;
+		return 0;
+	}
+
+	port = &display->ports[port_index];
+	dpcd = port->dpcd;
+
+	/* read out message from DATA1 register */
+	msg = vgpu_vreg(vgpu, offset + 4);
+	addr = (msg >> 8) & 0xffff;
+	ctrl = (msg >> 24) & 0xff;
+	len = msg & 0xff;
+	op = ctrl >> 4;
+
+	if (op == GVT_AUX_NATIVE_WRITE) {
+		int t;
+		uint8_t buf[16];
+
+		if ((addr + len + 1) >= DPCD_SIZE) {
+			/*
+			 * Write request exceeds what we supported,
+			 * DCPD spec: When a Source Device is writing a DPCD
+			 * address not supported by the Sink Device, the Sink
+			 * Device shall reply with AUX NACK and “M” equal to
+			 * zero.
+			 */
+
+			/* NAK the write */
+			vgpu_vreg(vgpu, offset + 4) = AUX_NATIVE_REPLY_NAK;
+			dp_aux_ch_ctl_trans_done(vgpu, data, offset, 2, true);
+			return 0;
+		}
+
+		/*
+		 * Write request format: (command + address) occupies
+		 * 3 bytes, followed by (len + 1) bytes of data.
+		 */
+		if (WARN_ON((len + 4) > AUX_BURST_SIZE))
+			return -EINVAL;
+
+		/* unpack data from vreg to buf */
+		for (t = 0; t < 4; t++) {
+			u32 r = vgpu_vreg(vgpu, offset + 8 + t * 4);
+
+			buf[t * 4] = (r >> 24) & 0xff;
+			buf[t * 4 + 1] = (r >> 16) & 0xff;
+			buf[t * 4 + 2] = (r >> 8) & 0xff;
+			buf[t * 4 + 3] = r & 0xff;
+		}
+
+		/* write to virtual DPCD */
+		if (dpcd && dpcd->data_valid) {
+			for (t = 0; t <= len; t++) {
+				int p = addr + t;
+
+				dpcd->data[p] = buf[t];
+				/* check for link training */
+				if (p == DPCD_TRAINING_PATTERN_SET)
+					dp_aux_ch_ctl_link_training(dpcd,
+							buf[t]);
+			}
+		}
+
+		/* ACK the write */
+		vgpu_vreg(vgpu, offset + 4) = 0;
+		dp_aux_ch_ctl_trans_done(vgpu, data, offset, 1,
+				dpcd && dpcd->data_valid);
+		return 0;
+	}
+
+	if (op == GVT_AUX_NATIVE_READ) {
+		int idx, i, ret = 0;
+
+		if ((addr + len + 1) >= DPCD_SIZE) {
+			/*
+			 * read request exceeds what we supported
+			 * DPCD spec: A Sink Device receiving a Native AUX CH
+			 * read request for an unsupported DPCD address must
+			 * reply with an AUX ACK and read data set equal to
+			 * zero instead of replying with AUX NACK.
+			 */
+
+			/* ACK the READ*/
+			vgpu_vreg(vgpu, offset + 4) = 0;
+			vgpu_vreg(vgpu, offset + 8) = 0;
+			vgpu_vreg(vgpu, offset + 12) = 0;
+			vgpu_vreg(vgpu, offset + 16) = 0;
+			vgpu_vreg(vgpu, offset + 20) = 0;
+
+			dp_aux_ch_ctl_trans_done(vgpu, data, offset, len + 2,
+					true);
+			return 0;
+		}
+
+		for (idx = 1; idx <= 5; idx++) {
+			/* clear the data registers */
+			vgpu_vreg(vgpu, offset + 4 * idx) = 0;
+		}
+
+		/*
+		 * Read reply format: ACK (1 byte) plus (len + 1) bytes of data.
+		 */
+		if (WARN_ON((len + 2) > AUX_BURST_SIZE))
+			return -EINVAL;
+
+		/* read from virtual DPCD to vreg */
+		/* first 4 bytes: [ACK][addr][addr+1][addr+2] */
+		if (dpcd && dpcd->data_valid) {
+			for (i = 1; i <= (len + 1); i++) {
+				int t;
+
+				t = dpcd->data[addr + i - 1];
+				t <<= (24 - 8 * (i % 4));
+				ret |= t;
+
+				if ((i % 4 == 3) || (i == (len + 1))) {
+					vgpu_vreg(vgpu, offset +
+							(i / 4 + 1) * 4) = ret;
+					ret = 0;
+				}
+			}
+		}
+		dp_aux_ch_ctl_trans_done(vgpu, data, offset, len + 2,
+				dpcd && dpcd->data_valid);
+		return 0;
+	}
+
+	/* i2c transaction starts */
+	intel_gvt_i2c_handle_aux_ch_write(vgpu, port_index, offset, p_data);
+
+	if (data & DP_AUX_CH_CTL_INTERRUPT)
+		trigger_aux_channel_interrupt(vgpu, offset);
+	return 0;
+}
+
+static int vga_control_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	bool vga_disable;
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	vga_disable = vgpu_vreg(vgpu, offset) & VGA_DISP_DISABLE;
+
+	gvt_dbg_core("vgpu%d: %s VGA mode\n", vgpu->id,
+			vga_disable ? "Disable" : "Enable");
+	return 0;
+}
+
+static u32 read_virtual_sbi_register(struct intel_vgpu *vgpu,
+		unsigned int sbi_offset)
+{
+	struct intel_vgpu_display *display = &vgpu->display;
+	int num = display->sbi.number;
+	int i;
+
+	for (i = 0; i < num; ++i)
+		if (display->sbi.registers[i].offset == sbi_offset)
+			break;
+
+	if (i == num)
+		return 0;
+
+	return display->sbi.registers[i].value;
+}
+
+static void write_virtual_sbi_register(struct intel_vgpu *vgpu,
+		unsigned int offset, u32 value)
+{
+	struct intel_vgpu_display *display = &vgpu->display;
+	int num = display->sbi.number;
+	int i;
+
+	for (i = 0; i < num; ++i) {
+		if (display->sbi.registers[i].offset == offset)
+			break;
+	}
+
+	if (i == num) {
+		if (num == SBI_REG_MAX) {
+			gvt_err("vgpu%d: SBI caching meets maximum limits\n",
+					vgpu->id);
+			return;
+		}
+		display->sbi.number++;
+	}
+
+	display->sbi.registers[i].offset = offset;
+	display->sbi.registers[i].value = value;
+}
+
+static int sbi_data_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	if (((vgpu_vreg(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >>
+				SBI_OPCODE_SHIFT) == SBI_CMD_CRRD) {
+		unsigned int sbi_offset = (vgpu_vreg(vgpu, SBI_ADDR) &
+				SBI_ADDR_OFFSET_MASK) >> SBI_ADDR_OFFSET_SHIFT;
+		vgpu_vreg(vgpu, offset) = read_virtual_sbi_register(vgpu,
+				sbi_offset);
+	}
+	read_vreg(vgpu, offset, p_data, bytes);
+	return 0;
+}
+
+static bool sbi_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 data;
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	data = vgpu_vreg(vgpu, offset);
+
+	data &= ~(SBI_STAT_MASK << SBI_STAT_SHIFT);
+	data |= SBI_READY;
+
+	data &= ~(SBI_RESPONSE_MASK << SBI_RESPONSE_SHIFT);
+	data |= SBI_RESPONSE_SUCCESS;
+
+	vgpu_vreg(vgpu, offset) = data;
+
+	if (((vgpu_vreg(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >>
+				SBI_OPCODE_SHIFT) == SBI_CMD_CRWR) {
+		unsigned int sbi_offset = (vgpu_vreg(vgpu, SBI_ADDR) &
+				SBI_ADDR_OFFSET_MASK) >> SBI_ADDR_OFFSET_SHIFT;
+
+		write_virtual_sbi_register(vgpu, sbi_offset,
+				vgpu_vreg(vgpu, SBI_DATA));
+	}
+	return 0;
+}
+
+#define _vgtif_reg(x) \
+	(VGT_PVINFO_PAGE + offsetof(struct vgt_if, x))
+
+static int pvinfo_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	bool invalid_read = false;
+
+	read_vreg(vgpu, offset, p_data, bytes);
+
+	switch (offset) {
+	case _vgtif_reg(magic) ... _vgtif_reg(vgt_id):
+		if (offset + bytes > _vgtif_reg(vgt_id) + 4)
+			invalid_read = true;
+		break;
+	case _vgtif_reg(avail_rs.mappable_gmadr.base) ...
+		_vgtif_reg(avail_rs.fence_num):
+		if (offset + bytes >
+			_vgtif_reg(avail_rs.fence_num) + 4)
+			invalid_read = true;
+		break;
+	case 0x78010:	/* vgt_caps */
+	case 0x7881c:
+		break;
+	default:
+		invalid_read = true;
+		break;
+	}
+	if (invalid_read)
+		gvt_err("invalid pvinfo read: [%x:%x] = %x\n",
+				offset, bytes, *(u32 *)p_data);
+	return 0;
+}
+
+static int handle_g2v_notification(struct intel_vgpu *vgpu, int notification)
+{
+	int ret = 0;
+
+	switch (notification) {
+	case VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE:
+		ret = intel_vgpu_g2v_create_ppgtt_mm(vgpu, 3);
+		break;
+	case VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY:
+		ret = intel_vgpu_g2v_destroy_ppgtt_mm(vgpu, 3);
+		break;
+	case VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE:
+		ret = intel_vgpu_g2v_create_ppgtt_mm(vgpu, 4);
+		break;
+	case VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY:
+		ret = intel_vgpu_g2v_destroy_ppgtt_mm(vgpu, 4);
+		break;
+	case VGT_G2V_EXECLIST_CONTEXT_CREATE:
+	case VGT_G2V_EXECLIST_CONTEXT_DESTROY:
+	case 1:	/* Remove this in guest driver. */
+		break;
+	default:
+		gvt_err("Invalid PV notification %d\n", notification);
+	}
+	return ret;
+}
+
+static int send_display_ready_uevent(struct intel_vgpu *vgpu, int ready)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj;
+	char *env[3] = {NULL, NULL, NULL};
+	char vmid_str[20];
+	char display_ready_str[20];
+
+	snprintf(display_ready_str, 20, "GVT_DISPLAY_READY=%d\n", ready);
+	env[0] = display_ready_str;
+
+	snprintf(vmid_str, 20, "VMID=%d", vgpu->id);
+	env[1] = vmid_str;
+
+	return kobject_uevent_env(kobj, KOBJ_ADD, env);
+}
+
+static int pvinfo_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 data;
+	int ret;
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	data = vgpu_vreg(vgpu, offset);
+
+	switch (offset) {
+	case _vgtif_reg(display_ready):
+		send_display_ready_uevent(vgpu, data ? 1 : 0);
+		break;
+	case _vgtif_reg(g2v_notify):
+		ret = handle_g2v_notification(vgpu, data);
+		break;
+	/* add xhot and yhot to handled list to avoid error log */
+	case 0x78830:
+	case 0x78834:
+	case _vgtif_reg(pdp[0].lo):
+	case _vgtif_reg(pdp[0].hi):
+	case _vgtif_reg(pdp[1].lo):
+	case _vgtif_reg(pdp[1].hi):
+	case _vgtif_reg(pdp[2].lo):
+	case _vgtif_reg(pdp[2].hi):
+	case _vgtif_reg(pdp[3].lo):
+	case _vgtif_reg(pdp[3].hi):
+	case _vgtif_reg(execlist_context_descriptor_lo):
+	case _vgtif_reg(execlist_context_descriptor_hi):
+		break;
+	default:
+		gvt_err("invalid pvinfo write offset %x bytes %x data %x\n",
+				offset, bytes, data);
+		break;
+	}
+	return 0;
+}
+
+static int pf_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	u32 val = *(u32 *)p_data;
+
+	if ((offset == _PS_1A_CTRL || offset == _PS_2A_CTRL ||
+	   offset == _PS_1B_CTRL || offset == _PS_2B_CTRL ||
+	   offset == _PS_1C_CTRL) && (val & PS_PLANE_SEL_MASK) != 0) {
+		WARN_ONCE(true, "VM(%d): guest is trying to scaling a plane\n",
+			  vgpu->id);
+		return 0;
+	}
+
+	return intel_vgpu_default_mmio_write(vgpu, offset, p_data, bytes);
+}
+
+static int power_well_ctl_mmio_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	write_vreg(vgpu, offset, p_data, bytes);
+
+	if (vgpu_vreg(vgpu, offset) & HSW_PWR_WELL_ENABLE_REQUEST)
+		vgpu_vreg(vgpu, offset) |= HSW_PWR_WELL_STATE_ENABLED;
+	else
+		vgpu_vreg(vgpu, offset) &= ~HSW_PWR_WELL_STATE_ENABLED;
+	return 0;
+}
+
+static int fpga_dbg_mmio_write(struct intel_vgpu *vgpu,
+	unsigned int offset, void *p_data, unsigned int bytes)
+{
+	write_vreg(vgpu, offset, p_data, bytes);
+
+	if (vgpu_vreg(vgpu, offset) & FPGA_DBG_RM_NOCLAIM)
+		vgpu_vreg(vgpu, offset) &= ~FPGA_DBG_RM_NOCLAIM;
+	return 0;
+}
+
+static int dma_ctrl_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 mode;
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	mode = vgpu_vreg(vgpu, offset);
+
+	if (GFX_MODE_BIT_SET_IN_MASK(mode, START_DMA)) {
+		WARN_ONCE(1, "VM(%d): iGVT-g doesn't supporte GuC\n",
+				vgpu->id);
+		return 0;
+	}
+
+	return 0;
+}
+
+static int gen9_trtte_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	u32 trtte = *(u32 *)p_data;
+
+	if ((trtte & 1) && (trtte & (1 << 1)) == 0) {
+		WARN(1, "VM(%d): Use physical address for TRTT!\n",
+				vgpu->id);
+		return -EINVAL;
+	}
+	write_vreg(vgpu, offset, p_data, bytes);
+	/* TRTTE is not per-context */
+	I915_WRITE(_MMIO(offset), vgpu_vreg(vgpu, offset));
+
+	return 0;
+}
+
+static int gen9_trtt_chicken_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	u32 val = *(u32 *)p_data;
+
+	if (val & 1) {
+		/* unblock hw logic */
+		I915_WRITE(_MMIO(offset), val);
+	}
+	write_vreg(vgpu, offset, p_data, bytes);
+	return 0;
+}
+
+static int dpll_status_read(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 v = 0;
+
+	if (vgpu_vreg(vgpu, 0x46010) & (1 << 31))
+		v |= (1 << 0);
+
+	if (vgpu_vreg(vgpu, 0x46014) & (1 << 31))
+		v |= (1 << 8);
+
+	if (vgpu_vreg(vgpu, 0x46040) & (1 << 31))
+		v |= (1 << 16);
+
+	if (vgpu_vreg(vgpu, 0x46060) & (1 << 31))
+		v |= (1 << 24);
+
+	vgpu_vreg(vgpu, offset) = v;
+
+	return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes);
+}
+
+static int mailbox_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 value = *(u32 *)p_data;
+	u32 cmd = value & 0xff;
+	u32 *data0 = &vgpu_vreg(vgpu, GEN6_PCODE_DATA);
+
+	switch (cmd) {
+	case 0x6:
+		/**
+		 * "Read memory latency" command on gen9.
+		 * Below memory latency values are read
+		 * from skylake platform.
+		 */
+		if (!*data0)
+			*data0 = 0x1e1a1100;
+		else
+			*data0 = 0x61514b3d;
+		break;
+	case 0x5:
+		*data0 |= 0x1;
+		break;
+	}
+
+	gvt_dbg_core("VM(%d) write %x to mailbox, return data0 %x\n",
+		     vgpu->id, value, *data0);
+
+	value &= ~(1 << 31);
+	return intel_vgpu_default_mmio_write(vgpu, offset, &value, bytes);
+}
+
+static int skl_power_well_ctl_write(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	u32 v = *(u32 *)p_data;
+
+	v &= (1 << 31) | (1 << 29) | (1 << 9) |
+	     (1 << 7) | (1 << 5) | (1 << 3) | (1 << 1);
+	v |= (v >> 1);
+
+	return intel_vgpu_default_mmio_write(vgpu, offset, &v, bytes);
+}
+
+static int skl_misc_ctl_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	i915_reg_t reg = {.reg = offset};
+
+	switch (offset) {
+	case 0x4ddc:
+		vgpu_vreg(vgpu, offset) = 0x8000003c;
+		/* WaCompressedResourceSamplerPbeMediaNewHashMode:skl */
+		I915_WRITE(reg, vgpu_vreg(vgpu, offset));
+		break;
+	case 0x42080:
+		vgpu_vreg(vgpu, offset) = 0x8000;
+		/* WaCompressedResourceDisplayNewHashMode:skl */
+		I915_WRITE(reg, vgpu_vreg(vgpu, offset));
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int skl_lcpll_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 v = *(u32 *)p_data;
+
+	/* other bits are MBZ. */
+	v &= (1 << 31) | (1 << 30);
+	v & (1 << 31) ? (v |= (1 << 30)) : (v &= ~(1 << 30));
+
+	vgpu_vreg(vgpu, offset) = v;
+
+	return 0;
+}
+
+static int ring_timestamp_mmio_read(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+	vgpu_vreg(vgpu, offset) = I915_READ(_MMIO(offset));
+	return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes);
+}
+
+static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	int ring_id = render_mmio_to_ring_id(vgpu->gvt, offset);
+	struct intel_vgpu_execlist *execlist;
+	u32 data = *(u32 *)p_data;
+	int ret = 0;
+
+	if (WARN_ON(ring_id < 0 || ring_id > I915_NUM_ENGINES - 1))
+		return -EINVAL;
+
+	execlist = &vgpu->execlist[ring_id];
+
+	execlist->elsp_dwords.data[execlist->elsp_dwords.index] = data;
+	if (execlist->elsp_dwords.index == 3) {
+		ret = intel_vgpu_submit_execlist(vgpu, ring_id);
+		if(ret)
+			gvt_err("fail submit workload on ring %d\n", ring_id);
+	}
+
+	++execlist->elsp_dwords.index;
+	execlist->elsp_dwords.index &= 0x3;
+	return ret;
+}
+
+static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 data = *(u32 *)p_data;
+	int ring_id = render_mmio_to_ring_id(vgpu->gvt, offset);
+	bool enable_execlist;
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	if ((data & _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE))
+			|| (data & _MASKED_BIT_DISABLE(GFX_RUN_LIST_ENABLE))) {
+		enable_execlist = !!(data & GFX_RUN_LIST_ENABLE);
+
+		gvt_dbg_core("EXECLIST %s on ring %d\n",
+				(enable_execlist ? "enabling" : "disabling"),
+				ring_id);
+
+		if (enable_execlist)
+			intel_vgpu_start_schedule(vgpu);
+	}
+	return 0;
+}
+
+static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu,
+		unsigned int offset, void *p_data, unsigned int bytes)
+{
+	int rc = 0;
+	unsigned int id = 0;
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	vgpu_vreg(vgpu, offset) = 0;
+
+	switch (offset) {
+	case 0x4260:
+		id = RCS;
+		break;
+	case 0x4264:
+		id = VCS;
+		break;
+	case 0x4268:
+		id = VCS2;
+		break;
+	case 0x426c:
+		id = BCS;
+		break;
+	case 0x4270:
+		id = VECS;
+		break;
+	default:
+		rc = -EINVAL;
+		break;
+	}
+	set_bit(id, (void *)vgpu->tlb_handle_pending);
+
+	return rc;
+}
+
+static int ring_reset_ctl_write(struct intel_vgpu *vgpu,
+	unsigned int offset, void *p_data, unsigned int bytes)
+{
+	u32 data;
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	data = vgpu_vreg(vgpu, offset);
+
+	if (data & _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET))
+		data |= RESET_CTL_READY_TO_RESET;
+	else if (data & _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET))
+		data &= ~RESET_CTL_READY_TO_RESET;
+
+	vgpu_vreg(vgpu, offset) = data;
+	return 0;
+}
+
+#define MMIO_F(reg, s, f, am, rm, d, r, w) do { \
+	ret = new_mmio_info(gvt, INTEL_GVT_MMIO_OFFSET(reg), \
+		f, s, am, rm, d, r, w); \
+	if (ret) \
+		return ret; \
+} while (0)
+
+#define MMIO_D(reg, d) \
+	MMIO_F(reg, 4, 0, 0, 0, d, NULL, NULL)
+
+#define MMIO_DH(reg, d, r, w) \
+	MMIO_F(reg, 4, 0, 0, 0, d, r, w)
+
+#define MMIO_DFH(reg, d, f, r, w) \
+	MMIO_F(reg, 4, f, 0, 0, d, r, w)
+
+#define MMIO_GM(reg, d, r, w) \
+	MMIO_F(reg, 4, F_GMADR, 0xFFFFF000, 0, d, r, w)
+
+#define MMIO_RO(reg, d, f, rm, r, w) \
+	MMIO_F(reg, 4, F_RO | f, 0, rm, d, r, w)
+
+#define MMIO_RING_F(prefix, s, f, am, rm, d, r, w) do { \
+	MMIO_F(prefix(RENDER_RING_BASE), s, f, am, rm, d, r, w); \
+	MMIO_F(prefix(BLT_RING_BASE), s, f, am, rm, d, r, w); \
+	MMIO_F(prefix(GEN6_BSD_RING_BASE), s, f, am, rm, d, r, w); \
+	MMIO_F(prefix(VEBOX_RING_BASE), s, f, am, rm, d, r, w); \
+} while (0)
+
+#define MMIO_RING_D(prefix, d) \
+	MMIO_RING_F(prefix, 4, 0, 0, 0, d, NULL, NULL)
+
+#define MMIO_RING_DFH(prefix, d, f, r, w) \
+	MMIO_RING_F(prefix, 4, f, 0, 0, d, r, w)
+
+#define MMIO_RING_GM(prefix, d, r, w) \
+	MMIO_RING_F(prefix, 4, F_GMADR, 0xFFFF0000, 0, d, r, w)
+
+#define MMIO_RING_RO(prefix, d, f, rm, r, w) \
+	MMIO_RING_F(prefix, 4, F_RO | f, 0, rm, d, r, w)
+
+static int init_generic_mmio_info(struct intel_gvt *gvt)
+{
+	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	int ret;
+
+	MMIO_RING_DFH(RING_IMR, D_ALL, 0, NULL, intel_vgpu_reg_imr_handler);
+
+	MMIO_DFH(SDEIMR, D_ALL, 0, NULL, intel_vgpu_reg_imr_handler);
+	MMIO_DFH(SDEIER, D_ALL, 0, NULL, intel_vgpu_reg_ier_handler);
+	MMIO_DFH(SDEIIR, D_ALL, 0, NULL, intel_vgpu_reg_iir_handler);
+	MMIO_D(SDEISR, D_ALL);
+
+	MMIO_RING_D(RING_HWSTAM, D_ALL);
+
+	MMIO_GM(RENDER_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+	MMIO_GM(BSD_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+	MMIO_GM(BLT_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+	MMIO_GM(VEBOX_HWS_PGA_GEN7, D_ALL, NULL, NULL);
+
+#define RING_REG(base) (base + 0x28)
+	MMIO_RING_D(RING_REG, D_ALL);
+#undef RING_REG
+
+#define RING_REG(base) (base + 0x134)
+	MMIO_RING_D(RING_REG, D_ALL);
+#undef RING_REG
+
+	MMIO_GM(0x2148, D_ALL, NULL, NULL);
+	MMIO_GM(CCID, D_ALL, NULL, NULL);
+	MMIO_GM(0x12198, D_ALL, NULL, NULL);
+	MMIO_D(GEN7_CXT_SIZE, D_ALL);
+
+	MMIO_RING_D(RING_TAIL, D_ALL);
+	MMIO_RING_D(RING_HEAD, D_ALL);
+	MMIO_RING_D(RING_CTL, D_ALL);
+	MMIO_RING_D(RING_ACTHD, D_ALL);
+	MMIO_RING_GM(RING_START, D_ALL, NULL, NULL);
+
+	/* RING MODE */
+#define RING_REG(base) (base + 0x29c)
+	MMIO_RING_DFH(RING_REG, D_ALL, F_MODE_MASK, NULL, ring_mode_mmio_write);
+#undef RING_REG
+
+	MMIO_RING_DFH(RING_MI_MODE, D_ALL, F_MODE_MASK, NULL, NULL);
+	MMIO_RING_DFH(RING_INSTPM, D_ALL, F_MODE_MASK, NULL, NULL);
+	MMIO_RING_DFH(RING_TIMESTAMP, D_ALL, F_CMD_ACCESS,
+			ring_timestamp_mmio_read, NULL);
+	MMIO_RING_DFH(RING_TIMESTAMP_UDW, D_ALL, F_CMD_ACCESS,
+			ring_timestamp_mmio_read, NULL);
+
+	MMIO_DFH(GEN7_GT_MODE, D_ALL, F_MODE_MASK, NULL, NULL);
+	MMIO_DFH(CACHE_MODE_0_GEN7, D_ALL, F_MODE_MASK, NULL, NULL);
+	MMIO_DFH(CACHE_MODE_1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+
+	MMIO_DFH(0x20dc, D_ALL, F_MODE_MASK, NULL, NULL);
+	MMIO_DFH(_3D_CHICKEN3, D_ALL, F_MODE_MASK, NULL, NULL);
+	MMIO_DFH(0x2088, D_ALL, F_MODE_MASK, NULL, NULL);
+	MMIO_DFH(0x20e4, D_ALL, F_MODE_MASK, NULL, NULL);
+	MMIO_DFH(0x2470, D_ALL, F_MODE_MASK, NULL, NULL);
+	MMIO_D(GAM_ECOCHK, D_ALL);
+	MMIO_DFH(GEN7_COMMON_SLICE_CHICKEN1, D_ALL, F_MODE_MASK, NULL, NULL);
+	MMIO_DFH(COMMON_SLICE_CHICKEN2, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_D(0x9030, D_ALL);
+	MMIO_D(0x20a0, D_ALL);
+	MMIO_D(0x2420, D_ALL);
+	MMIO_D(0x2430, D_ALL);
+	MMIO_D(0x2434, D_ALL);
+	MMIO_D(0x2438, D_ALL);
+	MMIO_D(0x243c, D_ALL);
+	MMIO_DFH(0x7018, D_ALL, F_MODE_MASK, NULL, NULL);
+	MMIO_DFH(HALF_SLICE_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(0xe100, D_ALL, F_MODE_MASK, NULL, NULL);
+
+	/* display */
+	MMIO_F(0x60220, 0x20, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_D(0x602a0, D_ALL);
+
+	MMIO_D(0x65050, D_ALL);
+	MMIO_D(0x650b4, D_ALL);
+
+	MMIO_D(0xc4040, D_ALL);
+	MMIO_D(DERRMR, D_ALL);
+
+	MMIO_D(PIPEDSL(PIPE_A), D_ALL);
+	MMIO_D(PIPEDSL(PIPE_B), D_ALL);
+	MMIO_D(PIPEDSL(PIPE_C), D_ALL);
+	MMIO_D(PIPEDSL(_PIPE_EDP), D_ALL);
+
+	MMIO_DH(PIPECONF(PIPE_A), D_ALL, NULL, pipeconf_mmio_write);
+	MMIO_DH(PIPECONF(PIPE_B), D_ALL, NULL, pipeconf_mmio_write);
+	MMIO_DH(PIPECONF(PIPE_C), D_ALL, NULL, pipeconf_mmio_write);
+	MMIO_DH(PIPECONF(_PIPE_EDP), D_ALL, NULL, pipeconf_mmio_write);
+
+	MMIO_D(PIPESTAT(PIPE_A), D_ALL);
+	MMIO_D(PIPESTAT(PIPE_B), D_ALL);
+	MMIO_D(PIPESTAT(PIPE_C), D_ALL);
+	MMIO_D(PIPESTAT(_PIPE_EDP), D_ALL);
+
+	MMIO_D(PIPE_FLIPCOUNT_G4X(PIPE_A), D_ALL);
+	MMIO_D(PIPE_FLIPCOUNT_G4X(PIPE_B), D_ALL);
+	MMIO_D(PIPE_FLIPCOUNT_G4X(PIPE_C), D_ALL);
+	MMIO_D(PIPE_FLIPCOUNT_G4X(_PIPE_EDP), D_ALL);
+
+	MMIO_D(PIPE_FRMCOUNT_G4X(PIPE_A), D_ALL);
+	MMIO_D(PIPE_FRMCOUNT_G4X(PIPE_B), D_ALL);
+	MMIO_D(PIPE_FRMCOUNT_G4X(PIPE_C), D_ALL);
+	MMIO_D(PIPE_FRMCOUNT_G4X(_PIPE_EDP), D_ALL);
+
+	MMIO_D(CURCNTR(PIPE_A), D_ALL);
+	MMIO_D(CURCNTR(PIPE_B), D_ALL);
+	MMIO_D(CURCNTR(PIPE_C), D_ALL);
+
+	MMIO_D(CURPOS(PIPE_A), D_ALL);
+	MMIO_D(CURPOS(PIPE_B), D_ALL);
+	MMIO_D(CURPOS(PIPE_C), D_ALL);
+
+	MMIO_D(CURBASE(PIPE_A), D_ALL);
+	MMIO_D(CURBASE(PIPE_B), D_ALL);
+	MMIO_D(CURBASE(PIPE_C), D_ALL);
+
+	MMIO_D(0x700ac, D_ALL);
+	MMIO_D(0x710ac, D_ALL);
+	MMIO_D(0x720ac, D_ALL);
+
+	MMIO_D(0x70090, D_ALL);
+	MMIO_D(0x70094, D_ALL);
+	MMIO_D(0x70098, D_ALL);
+	MMIO_D(0x7009c, D_ALL);
+
+	MMIO_D(DSPCNTR(PIPE_A), D_ALL);
+	MMIO_D(DSPADDR(PIPE_A), D_ALL);
+	MMIO_D(DSPSTRIDE(PIPE_A), D_ALL);
+	MMIO_D(DSPPOS(PIPE_A), D_ALL);
+	MMIO_D(DSPSIZE(PIPE_A), D_ALL);
+	MMIO_DH(DSPSURF(PIPE_A), D_ALL, NULL, pri_surf_mmio_write);
+	MMIO_D(DSPOFFSET(PIPE_A), D_ALL);
+	MMIO_D(DSPSURFLIVE(PIPE_A), D_ALL);
+
+	MMIO_D(DSPCNTR(PIPE_B), D_ALL);
+	MMIO_D(DSPADDR(PIPE_B), D_ALL);
+	MMIO_D(DSPSTRIDE(PIPE_B), D_ALL);
+	MMIO_D(DSPPOS(PIPE_B), D_ALL);
+	MMIO_D(DSPSIZE(PIPE_B), D_ALL);
+	MMIO_DH(DSPSURF(PIPE_B), D_ALL, NULL, pri_surf_mmio_write);
+	MMIO_D(DSPOFFSET(PIPE_B), D_ALL);
+	MMIO_D(DSPSURFLIVE(PIPE_B), D_ALL);
+
+	MMIO_D(DSPCNTR(PIPE_C), D_ALL);
+	MMIO_D(DSPADDR(PIPE_C), D_ALL);
+	MMIO_D(DSPSTRIDE(PIPE_C), D_ALL);
+	MMIO_D(DSPPOS(PIPE_C), D_ALL);
+	MMIO_D(DSPSIZE(PIPE_C), D_ALL);
+	MMIO_DH(DSPSURF(PIPE_C), D_ALL, NULL, pri_surf_mmio_write);
+	MMIO_D(DSPOFFSET(PIPE_C), D_ALL);
+	MMIO_D(DSPSURFLIVE(PIPE_C), D_ALL);
+
+	MMIO_D(SPRCTL(PIPE_A), D_ALL);
+	MMIO_D(SPRLINOFF(PIPE_A), D_ALL);
+	MMIO_D(SPRSTRIDE(PIPE_A), D_ALL);
+	MMIO_D(SPRPOS(PIPE_A), D_ALL);
+	MMIO_D(SPRSIZE(PIPE_A), D_ALL);
+	MMIO_D(SPRKEYVAL(PIPE_A), D_ALL);
+	MMIO_D(SPRKEYMSK(PIPE_A), D_ALL);
+	MMIO_DH(SPRSURF(PIPE_A), D_ALL, NULL, spr_surf_mmio_write);
+	MMIO_D(SPRKEYMAX(PIPE_A), D_ALL);
+	MMIO_D(SPROFFSET(PIPE_A), D_ALL);
+	MMIO_D(SPRSCALE(PIPE_A), D_ALL);
+	MMIO_D(SPRSURFLIVE(PIPE_A), D_ALL);
+
+	MMIO_D(SPRCTL(PIPE_B), D_ALL);
+	MMIO_D(SPRLINOFF(PIPE_B), D_ALL);
+	MMIO_D(SPRSTRIDE(PIPE_B), D_ALL);
+	MMIO_D(SPRPOS(PIPE_B), D_ALL);
+	MMIO_D(SPRSIZE(PIPE_B), D_ALL);
+	MMIO_D(SPRKEYVAL(PIPE_B), D_ALL);
+	MMIO_D(SPRKEYMSK(PIPE_B), D_ALL);
+	MMIO_DH(SPRSURF(PIPE_B), D_ALL, NULL, spr_surf_mmio_write);
+	MMIO_D(SPRKEYMAX(PIPE_B), D_ALL);
+	MMIO_D(SPROFFSET(PIPE_B), D_ALL);
+	MMIO_D(SPRSCALE(PIPE_B), D_ALL);
+	MMIO_D(SPRSURFLIVE(PIPE_B), D_ALL);
+
+	MMIO_D(SPRCTL(PIPE_C), D_ALL);
+	MMIO_D(SPRLINOFF(PIPE_C), D_ALL);
+	MMIO_D(SPRSTRIDE(PIPE_C), D_ALL);
+	MMIO_D(SPRPOS(PIPE_C), D_ALL);
+	MMIO_D(SPRSIZE(PIPE_C), D_ALL);
+	MMIO_D(SPRKEYVAL(PIPE_C), D_ALL);
+	MMIO_D(SPRKEYMSK(PIPE_C), D_ALL);
+	MMIO_DH(SPRSURF(PIPE_C), D_ALL, NULL, spr_surf_mmio_write);
+	MMIO_D(SPRKEYMAX(PIPE_C), D_ALL);
+	MMIO_D(SPROFFSET(PIPE_C), D_ALL);
+	MMIO_D(SPRSCALE(PIPE_C), D_ALL);
+	MMIO_D(SPRSURFLIVE(PIPE_C), D_ALL);
+
+	MMIO_F(LGC_PALETTE(PIPE_A, 0), 4 * 256, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(LGC_PALETTE(PIPE_B, 0), 4 * 256, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(LGC_PALETTE(PIPE_C, 0), 4 * 256, 0, 0, 0, D_ALL, NULL, NULL);
+
+	MMIO_D(HTOTAL(TRANSCODER_A), D_ALL);
+	MMIO_D(HBLANK(TRANSCODER_A), D_ALL);
+	MMIO_D(HSYNC(TRANSCODER_A), D_ALL);
+	MMIO_D(VTOTAL(TRANSCODER_A), D_ALL);
+	MMIO_D(VBLANK(TRANSCODER_A), D_ALL);
+	MMIO_D(VSYNC(TRANSCODER_A), D_ALL);
+	MMIO_D(BCLRPAT(TRANSCODER_A), D_ALL);
+	MMIO_D(VSYNCSHIFT(TRANSCODER_A), D_ALL);
+	MMIO_D(PIPESRC(TRANSCODER_A), D_ALL);
+
+	MMIO_D(HTOTAL(TRANSCODER_B), D_ALL);
+	MMIO_D(HBLANK(TRANSCODER_B), D_ALL);
+	MMIO_D(HSYNC(TRANSCODER_B), D_ALL);
+	MMIO_D(VTOTAL(TRANSCODER_B), D_ALL);
+	MMIO_D(VBLANK(TRANSCODER_B), D_ALL);
+	MMIO_D(VSYNC(TRANSCODER_B), D_ALL);
+	MMIO_D(BCLRPAT(TRANSCODER_B), D_ALL);
+	MMIO_D(VSYNCSHIFT(TRANSCODER_B), D_ALL);
+	MMIO_D(PIPESRC(TRANSCODER_B), D_ALL);
+
+	MMIO_D(HTOTAL(TRANSCODER_C), D_ALL);
+	MMIO_D(HBLANK(TRANSCODER_C), D_ALL);
+	MMIO_D(HSYNC(TRANSCODER_C), D_ALL);
+	MMIO_D(VTOTAL(TRANSCODER_C), D_ALL);
+	MMIO_D(VBLANK(TRANSCODER_C), D_ALL);
+	MMIO_D(VSYNC(TRANSCODER_C), D_ALL);
+	MMIO_D(BCLRPAT(TRANSCODER_C), D_ALL);
+	MMIO_D(VSYNCSHIFT(TRANSCODER_C), D_ALL);
+	MMIO_D(PIPESRC(TRANSCODER_C), D_ALL);
+
+	MMIO_D(HTOTAL(TRANSCODER_EDP), D_ALL);
+	MMIO_D(HBLANK(TRANSCODER_EDP), D_ALL);
+	MMIO_D(HSYNC(TRANSCODER_EDP), D_ALL);
+	MMIO_D(VTOTAL(TRANSCODER_EDP), D_ALL);
+	MMIO_D(VBLANK(TRANSCODER_EDP), D_ALL);
+	MMIO_D(VSYNC(TRANSCODER_EDP), D_ALL);
+	MMIO_D(BCLRPAT(TRANSCODER_EDP), D_ALL);
+	MMIO_D(VSYNCSHIFT(TRANSCODER_EDP), D_ALL);
+
+	MMIO_D(PIPE_DATA_M1(TRANSCODER_A), D_ALL);
+	MMIO_D(PIPE_DATA_N1(TRANSCODER_A), D_ALL);
+	MMIO_D(PIPE_DATA_M2(TRANSCODER_A), D_ALL);
+	MMIO_D(PIPE_DATA_N2(TRANSCODER_A), D_ALL);
+	MMIO_D(PIPE_LINK_M1(TRANSCODER_A), D_ALL);
+	MMIO_D(PIPE_LINK_N1(TRANSCODER_A), D_ALL);
+	MMIO_D(PIPE_LINK_M2(TRANSCODER_A), D_ALL);
+	MMIO_D(PIPE_LINK_N2(TRANSCODER_A), D_ALL);
+
+	MMIO_D(PIPE_DATA_M1(TRANSCODER_B), D_ALL);
+	MMIO_D(PIPE_DATA_N1(TRANSCODER_B), D_ALL);
+	MMIO_D(PIPE_DATA_M2(TRANSCODER_B), D_ALL);
+	MMIO_D(PIPE_DATA_N2(TRANSCODER_B), D_ALL);
+	MMIO_D(PIPE_LINK_M1(TRANSCODER_B), D_ALL);
+	MMIO_D(PIPE_LINK_N1(TRANSCODER_B), D_ALL);
+	MMIO_D(PIPE_LINK_M2(TRANSCODER_B), D_ALL);
+	MMIO_D(PIPE_LINK_N2(TRANSCODER_B), D_ALL);
+
+	MMIO_D(PIPE_DATA_M1(TRANSCODER_C), D_ALL);
+	MMIO_D(PIPE_DATA_N1(TRANSCODER_C), D_ALL);
+	MMIO_D(PIPE_DATA_M2(TRANSCODER_C), D_ALL);
+	MMIO_D(PIPE_DATA_N2(TRANSCODER_C), D_ALL);
+	MMIO_D(PIPE_LINK_M1(TRANSCODER_C), D_ALL);
+	MMIO_D(PIPE_LINK_N1(TRANSCODER_C), D_ALL);
+	MMIO_D(PIPE_LINK_M2(TRANSCODER_C), D_ALL);
+	MMIO_D(PIPE_LINK_N2(TRANSCODER_C), D_ALL);
+
+	MMIO_D(PIPE_DATA_M1(TRANSCODER_EDP), D_ALL);
+	MMIO_D(PIPE_DATA_N1(TRANSCODER_EDP), D_ALL);
+	MMIO_D(PIPE_DATA_M2(TRANSCODER_EDP), D_ALL);
+	MMIO_D(PIPE_DATA_N2(TRANSCODER_EDP), D_ALL);
+	MMIO_D(PIPE_LINK_M1(TRANSCODER_EDP), D_ALL);
+	MMIO_D(PIPE_LINK_N1(TRANSCODER_EDP), D_ALL);
+	MMIO_D(PIPE_LINK_M2(TRANSCODER_EDP), D_ALL);
+	MMIO_D(PIPE_LINK_N2(TRANSCODER_EDP), D_ALL);
+
+	MMIO_D(PF_CTL(PIPE_A), D_ALL);
+	MMIO_D(PF_WIN_SZ(PIPE_A), D_ALL);
+	MMIO_D(PF_WIN_POS(PIPE_A), D_ALL);
+	MMIO_D(PF_VSCALE(PIPE_A), D_ALL);
+	MMIO_D(PF_HSCALE(PIPE_A), D_ALL);
+
+	MMIO_D(PF_CTL(PIPE_B), D_ALL);
+	MMIO_D(PF_WIN_SZ(PIPE_B), D_ALL);
+	MMIO_D(PF_WIN_POS(PIPE_B), D_ALL);
+	MMIO_D(PF_VSCALE(PIPE_B), D_ALL);
+	MMIO_D(PF_HSCALE(PIPE_B), D_ALL);
+
+	MMIO_D(PF_CTL(PIPE_C), D_ALL);
+	MMIO_D(PF_WIN_SZ(PIPE_C), D_ALL);
+	MMIO_D(PF_WIN_POS(PIPE_C), D_ALL);
+	MMIO_D(PF_VSCALE(PIPE_C), D_ALL);
+	MMIO_D(PF_HSCALE(PIPE_C), D_ALL);
+
+	MMIO_D(WM0_PIPEA_ILK, D_ALL);
+	MMIO_D(WM0_PIPEB_ILK, D_ALL);
+	MMIO_D(WM0_PIPEC_IVB, D_ALL);
+	MMIO_D(WM1_LP_ILK, D_ALL);
+	MMIO_D(WM2_LP_ILK, D_ALL);
+	MMIO_D(WM3_LP_ILK, D_ALL);
+	MMIO_D(WM1S_LP_ILK, D_ALL);
+	MMIO_D(WM2S_LP_IVB, D_ALL);
+	MMIO_D(WM3S_LP_IVB, D_ALL);
+
+	MMIO_D(BLC_PWM_CPU_CTL2, D_ALL);
+	MMIO_D(BLC_PWM_CPU_CTL, D_ALL);
+	MMIO_D(BLC_PWM_PCH_CTL1, D_ALL);
+	MMIO_D(BLC_PWM_PCH_CTL2, D_ALL);
+
+	MMIO_D(0x48268, D_ALL);
+
+	MMIO_F(PCH_GMBUS0, 4 * 4, 0, 0, 0, D_ALL, gmbus_mmio_read,
+		gmbus_mmio_write);
+	MMIO_F(PCH_GPIOA, 6 * 4, F_UNALIGN, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(0xe4f00, 0x28, 0, 0, 0, D_ALL, NULL, NULL);
+
+	MMIO_F(_PCH_DPB_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_PRE_SKL, NULL,
+		dp_aux_ch_ctl_mmio_write);
+	MMIO_F(_PCH_DPC_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_PRE_SKL, NULL,
+		dp_aux_ch_ctl_mmio_write);
+	MMIO_F(_PCH_DPD_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_PRE_SKL, NULL,
+		dp_aux_ch_ctl_mmio_write);
+
+	MMIO_RO(PCH_ADPA, D_ALL, 0, ADPA_CRT_HOTPLUG_MONITOR_MASK, NULL, pch_adpa_mmio_write);
+
+	MMIO_DH(_PCH_TRANSACONF, D_ALL, NULL, transconf_mmio_write);
+	MMIO_DH(_PCH_TRANSBCONF, D_ALL, NULL, transconf_mmio_write);
+
+	MMIO_DH(FDI_RX_IIR(PIPE_A), D_ALL, NULL, fdi_rx_iir_mmio_write);
+	MMIO_DH(FDI_RX_IIR(PIPE_B), D_ALL, NULL, fdi_rx_iir_mmio_write);
+	MMIO_DH(FDI_RX_IIR(PIPE_C), D_ALL, NULL, fdi_rx_iir_mmio_write);
+	MMIO_DH(FDI_RX_IMR(PIPE_A), D_ALL, NULL, update_fdi_rx_iir_status);
+	MMIO_DH(FDI_RX_IMR(PIPE_B), D_ALL, NULL, update_fdi_rx_iir_status);
+	MMIO_DH(FDI_RX_IMR(PIPE_C), D_ALL, NULL, update_fdi_rx_iir_status);
+	MMIO_DH(FDI_RX_CTL(PIPE_A), D_ALL, NULL, update_fdi_rx_iir_status);
+	MMIO_DH(FDI_RX_CTL(PIPE_B), D_ALL, NULL, update_fdi_rx_iir_status);
+	MMIO_DH(FDI_RX_CTL(PIPE_C), D_ALL, NULL, update_fdi_rx_iir_status);
+
+	MMIO_D(_PCH_TRANS_HTOTAL_A, D_ALL);
+	MMIO_D(_PCH_TRANS_HBLANK_A, D_ALL);
+	MMIO_D(_PCH_TRANS_HSYNC_A, D_ALL);
+	MMIO_D(_PCH_TRANS_VTOTAL_A, D_ALL);
+	MMIO_D(_PCH_TRANS_VBLANK_A, D_ALL);
+	MMIO_D(_PCH_TRANS_VSYNC_A, D_ALL);
+	MMIO_D(_PCH_TRANS_VSYNCSHIFT_A, D_ALL);
+
+	MMIO_D(_PCH_TRANS_HTOTAL_B, D_ALL);
+	MMIO_D(_PCH_TRANS_HBLANK_B, D_ALL);
+	MMIO_D(_PCH_TRANS_HSYNC_B, D_ALL);
+	MMIO_D(_PCH_TRANS_VTOTAL_B, D_ALL);
+	MMIO_D(_PCH_TRANS_VBLANK_B, D_ALL);
+	MMIO_D(_PCH_TRANS_VSYNC_B, D_ALL);
+	MMIO_D(_PCH_TRANS_VSYNCSHIFT_B, D_ALL);
+
+	MMIO_D(_PCH_TRANSA_DATA_M1, D_ALL);
+	MMIO_D(_PCH_TRANSA_DATA_N1, D_ALL);
+	MMIO_D(_PCH_TRANSA_DATA_M2, D_ALL);
+	MMIO_D(_PCH_TRANSA_DATA_N2, D_ALL);
+	MMIO_D(_PCH_TRANSA_LINK_M1, D_ALL);
+	MMIO_D(_PCH_TRANSA_LINK_N1, D_ALL);
+	MMIO_D(_PCH_TRANSA_LINK_M2, D_ALL);
+	MMIO_D(_PCH_TRANSA_LINK_N2, D_ALL);
+
+	MMIO_D(TRANS_DP_CTL(PIPE_A), D_ALL);
+	MMIO_D(TRANS_DP_CTL(PIPE_B), D_ALL);
+	MMIO_D(TRANS_DP_CTL(PIPE_C), D_ALL);
+
+	MMIO_D(TVIDEO_DIP_CTL(PIPE_A), D_ALL);
+	MMIO_D(TVIDEO_DIP_DATA(PIPE_A), D_ALL);
+	MMIO_D(TVIDEO_DIP_GCP(PIPE_A), D_ALL);
+
+	MMIO_D(TVIDEO_DIP_CTL(PIPE_B), D_ALL);
+	MMIO_D(TVIDEO_DIP_DATA(PIPE_B), D_ALL);
+	MMIO_D(TVIDEO_DIP_GCP(PIPE_B), D_ALL);
+
+	MMIO_D(TVIDEO_DIP_CTL(PIPE_C), D_ALL);
+	MMIO_D(TVIDEO_DIP_DATA(PIPE_C), D_ALL);
+	MMIO_D(TVIDEO_DIP_GCP(PIPE_C), D_ALL);
+
+	MMIO_D(_FDI_RXA_MISC, D_ALL);
+	MMIO_D(_FDI_RXB_MISC, D_ALL);
+	MMIO_D(_FDI_RXA_TUSIZE1, D_ALL);
+	MMIO_D(_FDI_RXA_TUSIZE2, D_ALL);
+	MMIO_D(_FDI_RXB_TUSIZE1, D_ALL);
+	MMIO_D(_FDI_RXB_TUSIZE2, D_ALL);
+
+	MMIO_DH(PCH_PP_CONTROL, D_ALL, NULL, pch_pp_control_mmio_write);
+	MMIO_D(PCH_PP_DIVISOR, D_ALL);
+	MMIO_D(PCH_PP_STATUS,  D_ALL);
+	MMIO_D(PCH_LVDS, D_ALL);
+	MMIO_D(_PCH_DPLL_A, D_ALL);
+	MMIO_D(_PCH_DPLL_B, D_ALL);
+	MMIO_D(_PCH_FPA0, D_ALL);
+	MMIO_D(_PCH_FPA1, D_ALL);
+	MMIO_D(_PCH_FPB0, D_ALL);
+	MMIO_D(_PCH_FPB1, D_ALL);
+	MMIO_D(PCH_DREF_CONTROL, D_ALL);
+	MMIO_D(PCH_RAWCLK_FREQ, D_ALL);
+	MMIO_D(PCH_DPLL_SEL, D_ALL);
+
+	MMIO_D(0x61208, D_ALL);
+	MMIO_D(0x6120c, D_ALL);
+	MMIO_D(PCH_PP_ON_DELAYS, D_ALL);
+	MMIO_D(PCH_PP_OFF_DELAYS, D_ALL);
+
+	MMIO_DH(0xe651c, D_ALL, dpy_reg_mmio_read, NULL);
+	MMIO_DH(0xe661c, D_ALL, dpy_reg_mmio_read, NULL);
+	MMIO_DH(0xe671c, D_ALL, dpy_reg_mmio_read, NULL);
+	MMIO_DH(0xe681c, D_ALL, dpy_reg_mmio_read, NULL);
+	MMIO_DH(0xe6c04, D_ALL, dpy_reg_mmio_read_2, NULL);
+	MMIO_DH(0xe6e1c, D_ALL, dpy_reg_mmio_read_3, NULL);
+
+	MMIO_RO(PCH_PORT_HOTPLUG, D_ALL, 0,
+		PORTA_HOTPLUG_STATUS_MASK
+		| PORTB_HOTPLUG_STATUS_MASK
+		| PORTC_HOTPLUG_STATUS_MASK
+		| PORTD_HOTPLUG_STATUS_MASK,
+		NULL, NULL);
+
+	MMIO_DH(LCPLL_CTL, D_ALL, NULL, lcpll_ctl_mmio_write);
+	MMIO_D(FUSE_STRAP, D_ALL);
+	MMIO_D(DIGITAL_PORT_HOTPLUG_CNTRL, D_ALL);
+
+	MMIO_D(DISP_ARB_CTL, D_ALL);
+	MMIO_D(DISP_ARB_CTL2, D_ALL);
+
+	MMIO_D(ILK_DISPLAY_CHICKEN1, D_ALL);
+	MMIO_D(ILK_DISPLAY_CHICKEN2, D_ALL);
+	MMIO_D(ILK_DSPCLK_GATE_D, D_ALL);
+
+	MMIO_D(SOUTH_CHICKEN1, D_ALL);
+	MMIO_DH(SOUTH_CHICKEN2, D_ALL, NULL, south_chicken2_mmio_write);
+	MMIO_D(_TRANSA_CHICKEN1, D_ALL);
+	MMIO_D(_TRANSB_CHICKEN1, D_ALL);
+	MMIO_D(SOUTH_DSPCLK_GATE_D, D_ALL);
+	MMIO_D(_TRANSA_CHICKEN2, D_ALL);
+	MMIO_D(_TRANSB_CHICKEN2, D_ALL);
+
+	MMIO_D(ILK_DPFC_CB_BASE, D_ALL);
+	MMIO_D(ILK_DPFC_CONTROL, D_ALL);
+	MMIO_D(ILK_DPFC_RECOMP_CTL, D_ALL);
+	MMIO_D(ILK_DPFC_STATUS, D_ALL);
+	MMIO_D(ILK_DPFC_FENCE_YOFF, D_ALL);
+	MMIO_D(ILK_DPFC_CHICKEN, D_ALL);
+	MMIO_D(ILK_FBC_RT_BASE, D_ALL);
+
+	MMIO_D(IPS_CTL, D_ALL);
+
+	MMIO_D(PIPE_CSC_COEFF_RY_GY(PIPE_A), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_BY(PIPE_A), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_RU_GU(PIPE_A), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_BU(PIPE_A), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_RV_GV(PIPE_A), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_BV(PIPE_A), D_ALL);
+	MMIO_D(PIPE_CSC_MODE(PIPE_A), D_ALL);
+	MMIO_D(PIPE_CSC_PREOFF_HI(PIPE_A), D_ALL);
+	MMIO_D(PIPE_CSC_PREOFF_ME(PIPE_A), D_ALL);
+	MMIO_D(PIPE_CSC_PREOFF_LO(PIPE_A), D_ALL);
+	MMIO_D(PIPE_CSC_POSTOFF_HI(PIPE_A), D_ALL);
+	MMIO_D(PIPE_CSC_POSTOFF_ME(PIPE_A), D_ALL);
+	MMIO_D(PIPE_CSC_POSTOFF_LO(PIPE_A), D_ALL);
+
+	MMIO_D(PIPE_CSC_COEFF_RY_GY(PIPE_B), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_BY(PIPE_B), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_RU_GU(PIPE_B), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_BU(PIPE_B), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_RV_GV(PIPE_B), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_BV(PIPE_B), D_ALL);
+	MMIO_D(PIPE_CSC_MODE(PIPE_B), D_ALL);
+	MMIO_D(PIPE_CSC_PREOFF_HI(PIPE_B), D_ALL);
+	MMIO_D(PIPE_CSC_PREOFF_ME(PIPE_B), D_ALL);
+	MMIO_D(PIPE_CSC_PREOFF_LO(PIPE_B), D_ALL);
+	MMIO_D(PIPE_CSC_POSTOFF_HI(PIPE_B), D_ALL);
+	MMIO_D(PIPE_CSC_POSTOFF_ME(PIPE_B), D_ALL);
+	MMIO_D(PIPE_CSC_POSTOFF_LO(PIPE_B), D_ALL);
+
+	MMIO_D(PIPE_CSC_COEFF_RY_GY(PIPE_C), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_BY(PIPE_C), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_RU_GU(PIPE_C), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_BU(PIPE_C), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_RV_GV(PIPE_C), D_ALL);
+	MMIO_D(PIPE_CSC_COEFF_BV(PIPE_C), D_ALL);
+	MMIO_D(PIPE_CSC_MODE(PIPE_C), D_ALL);
+	MMIO_D(PIPE_CSC_PREOFF_HI(PIPE_C), D_ALL);
+	MMIO_D(PIPE_CSC_PREOFF_ME(PIPE_C), D_ALL);
+	MMIO_D(PIPE_CSC_PREOFF_LO(PIPE_C), D_ALL);
+	MMIO_D(PIPE_CSC_POSTOFF_HI(PIPE_C), D_ALL);
+	MMIO_D(PIPE_CSC_POSTOFF_ME(PIPE_C), D_ALL);
+	MMIO_D(PIPE_CSC_POSTOFF_LO(PIPE_C), D_ALL);
+
+	MMIO_D(PREC_PAL_INDEX(PIPE_A), D_ALL);
+	MMIO_D(PREC_PAL_DATA(PIPE_A), D_ALL);
+	MMIO_F(PREC_PAL_GC_MAX(PIPE_A, 0), 4 * 3, 0, 0, 0, D_ALL, NULL, NULL);
+
+	MMIO_D(PREC_PAL_INDEX(PIPE_B), D_ALL);
+	MMIO_D(PREC_PAL_DATA(PIPE_B), D_ALL);
+	MMIO_F(PREC_PAL_GC_MAX(PIPE_B, 0), 4 * 3, 0, 0, 0, D_ALL, NULL, NULL);
+
+	MMIO_D(PREC_PAL_INDEX(PIPE_C), D_ALL);
+	MMIO_D(PREC_PAL_DATA(PIPE_C), D_ALL);
+	MMIO_F(PREC_PAL_GC_MAX(PIPE_C, 0), 4 * 3, 0, 0, 0, D_ALL, NULL, NULL);
+
+	MMIO_D(0x60110, D_ALL);
+	MMIO_D(0x61110, D_ALL);
+	MMIO_F(0x70400, 0x40, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(0x71400, 0x40, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(0x72400, 0x40, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(0x70440, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+	MMIO_F(0x71440, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+	MMIO_F(0x72440, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+	MMIO_F(0x7044c, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+	MMIO_F(0x7144c, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+	MMIO_F(0x7244c, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+
+	MMIO_D(PIPE_WM_LINETIME(PIPE_A), D_ALL);
+	MMIO_D(PIPE_WM_LINETIME(PIPE_B), D_ALL);
+	MMIO_D(PIPE_WM_LINETIME(PIPE_C), D_ALL);
+	MMIO_D(SPLL_CTL, D_ALL);
+	MMIO_D(_WRPLL_CTL1, D_ALL);
+	MMIO_D(_WRPLL_CTL2, D_ALL);
+	MMIO_D(PORT_CLK_SEL(PORT_A), D_ALL);
+	MMIO_D(PORT_CLK_SEL(PORT_B), D_ALL);
+	MMIO_D(PORT_CLK_SEL(PORT_C), D_ALL);
+	MMIO_D(PORT_CLK_SEL(PORT_D), D_ALL);
+	MMIO_D(PORT_CLK_SEL(PORT_E), D_ALL);
+	MMIO_D(TRANS_CLK_SEL(TRANSCODER_A), D_ALL);
+	MMIO_D(TRANS_CLK_SEL(TRANSCODER_B), D_ALL);
+	MMIO_D(TRANS_CLK_SEL(TRANSCODER_C), D_ALL);
+
+	MMIO_D(HSW_NDE_RSTWRN_OPT, D_ALL);
+	MMIO_D(0x46508, D_ALL);
+
+	MMIO_D(0x49080, D_ALL);
+	MMIO_D(0x49180, D_ALL);
+	MMIO_D(0x49280, D_ALL);
+
+	MMIO_F(0x49090, 0x14, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(0x49190, 0x14, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(0x49290, 0x14, 0, 0, 0, D_ALL, NULL, NULL);
+
+	MMIO_D(GAMMA_MODE(PIPE_A), D_ALL);
+	MMIO_D(GAMMA_MODE(PIPE_B), D_ALL);
+	MMIO_D(GAMMA_MODE(PIPE_C), D_ALL);
+
+	MMIO_D(PIPE_MULT(PIPE_A), D_ALL);
+	MMIO_D(PIPE_MULT(PIPE_B), D_ALL);
+	MMIO_D(PIPE_MULT(PIPE_C), D_ALL);
+
+	MMIO_D(HSW_TVIDEO_DIP_CTL(TRANSCODER_A), D_ALL);
+	MMIO_D(HSW_TVIDEO_DIP_CTL(TRANSCODER_B), D_ALL);
+	MMIO_D(HSW_TVIDEO_DIP_CTL(TRANSCODER_C), D_ALL);
+
+	MMIO_DH(SFUSE_STRAP, D_ALL, NULL, NULL);
+	MMIO_D(SBI_ADDR, D_ALL);
+	MMIO_DH(SBI_DATA, D_ALL, sbi_data_mmio_read, NULL);
+	MMIO_DH(SBI_CTL_STAT, D_ALL, NULL, sbi_ctl_mmio_write);
+	MMIO_D(PIXCLK_GATE, D_ALL);
+
+	MMIO_F(_DPA_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_ALL, NULL,
+		dp_aux_ch_ctl_mmio_write);
+
+	MMIO_DH(DDI_BUF_CTL(PORT_A), D_ALL, NULL, ddi_buf_ctl_mmio_write);
+	MMIO_DH(DDI_BUF_CTL(PORT_B), D_ALL, NULL, ddi_buf_ctl_mmio_write);
+	MMIO_DH(DDI_BUF_CTL(PORT_C), D_ALL, NULL, ddi_buf_ctl_mmio_write);
+	MMIO_DH(DDI_BUF_CTL(PORT_D), D_ALL, NULL, ddi_buf_ctl_mmio_write);
+	MMIO_DH(DDI_BUF_CTL(PORT_E), D_ALL, NULL, ddi_buf_ctl_mmio_write);
+
+	MMIO_DH(DP_TP_CTL(PORT_A), D_ALL, NULL, dp_tp_ctl_mmio_write);
+	MMIO_DH(DP_TP_CTL(PORT_B), D_ALL, NULL, dp_tp_ctl_mmio_write);
+	MMIO_DH(DP_TP_CTL(PORT_C), D_ALL, NULL, dp_tp_ctl_mmio_write);
+	MMIO_DH(DP_TP_CTL(PORT_D), D_ALL, NULL, dp_tp_ctl_mmio_write);
+	MMIO_DH(DP_TP_CTL(PORT_E), D_ALL, NULL, dp_tp_ctl_mmio_write);
+
+	MMIO_DH(DP_TP_STATUS(PORT_A), D_ALL, NULL, dp_tp_status_mmio_write);
+	MMIO_DH(DP_TP_STATUS(PORT_B), D_ALL, NULL, dp_tp_status_mmio_write);
+	MMIO_DH(DP_TP_STATUS(PORT_C), D_ALL, NULL, dp_tp_status_mmio_write);
+	MMIO_DH(DP_TP_STATUS(PORT_D), D_ALL, NULL, dp_tp_status_mmio_write);
+	MMIO_DH(DP_TP_STATUS(PORT_E), D_ALL, NULL, NULL);
+
+	MMIO_F(_DDI_BUF_TRANS_A, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(0x64e60, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(0x64eC0, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(0x64f20, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(0x64f80, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+
+	MMIO_D(HSW_AUD_CFG(PIPE_A), D_ALL);
+	MMIO_D(HSW_AUD_PIN_ELD_CP_VLD, D_ALL);
+
+	MMIO_DH(_TRANS_DDI_FUNC_CTL_A, D_ALL, NULL, NULL);
+	MMIO_DH(_TRANS_DDI_FUNC_CTL_B, D_ALL, NULL, NULL);
+	MMIO_DH(_TRANS_DDI_FUNC_CTL_C, D_ALL, NULL, NULL);
+	MMIO_DH(_TRANS_DDI_FUNC_CTL_EDP, D_ALL, NULL, NULL);
+
+	MMIO_D(_TRANSA_MSA_MISC, D_ALL);
+	MMIO_D(_TRANSB_MSA_MISC, D_ALL);
+	MMIO_D(_TRANSC_MSA_MISC, D_ALL);
+	MMIO_D(_TRANS_EDP_MSA_MISC, D_ALL);
+
+	MMIO_DH(FORCEWAKE, D_ALL, NULL, NULL);
+	MMIO_D(FORCEWAKE_ACK, D_ALL);
+	MMIO_D(GEN6_GT_CORE_STATUS, D_ALL);
+	MMIO_D(GEN6_GT_THREAD_STATUS_REG, D_ALL);
+	MMIO_D(GTFIFODBG, D_ALL);
+	MMIO_D(GTFIFOCTL, D_ALL);
+	MMIO_DH(FORCEWAKE_MT, D_PRE_SKL, NULL, mul_force_wake_write);
+	MMIO_DH(FORCEWAKE_ACK_HSW, D_HSW | D_BDW, NULL, NULL);
+	MMIO_D(ECOBUS, D_ALL);
+	MMIO_DH(GEN6_RC_CONTROL, D_ALL, NULL, NULL);
+	MMIO_DH(GEN6_RC_STATE, D_ALL, NULL, NULL);
+	MMIO_D(GEN6_RPNSWREQ, D_ALL);
+	MMIO_D(GEN6_RC_VIDEO_FREQ, D_ALL);
+	MMIO_D(GEN6_RP_DOWN_TIMEOUT, D_ALL);
+	MMIO_D(GEN6_RP_INTERRUPT_LIMITS, D_ALL);
+	MMIO_D(GEN6_RPSTAT1, D_ALL);
+	MMIO_D(GEN6_RP_CONTROL, D_ALL);
+	MMIO_D(GEN6_RP_UP_THRESHOLD, D_ALL);
+	MMIO_D(GEN6_RP_DOWN_THRESHOLD, D_ALL);
+	MMIO_D(GEN6_RP_CUR_UP_EI, D_ALL);
+	MMIO_D(GEN6_RP_CUR_UP, D_ALL);
+	MMIO_D(GEN6_RP_PREV_UP, D_ALL);
+	MMIO_D(GEN6_RP_CUR_DOWN_EI, D_ALL);
+	MMIO_D(GEN6_RP_CUR_DOWN, D_ALL);
+	MMIO_D(GEN6_RP_PREV_DOWN, D_ALL);
+	MMIO_D(GEN6_RP_UP_EI, D_ALL);
+	MMIO_D(GEN6_RP_DOWN_EI, D_ALL);
+	MMIO_D(GEN6_RP_IDLE_HYSTERSIS, D_ALL);
+	MMIO_D(GEN6_RC1_WAKE_RATE_LIMIT, D_ALL);
+	MMIO_D(GEN6_RC6_WAKE_RATE_LIMIT, D_ALL);
+	MMIO_D(GEN6_RC6pp_WAKE_RATE_LIMIT, D_ALL);
+	MMIO_D(GEN6_RC_EVALUATION_INTERVAL, D_ALL);
+	MMIO_D(GEN6_RC_IDLE_HYSTERSIS, D_ALL);
+	MMIO_D(GEN6_RC_SLEEP, D_ALL);
+	MMIO_D(GEN6_RC1e_THRESHOLD, D_ALL);
+	MMIO_D(GEN6_RC6_THRESHOLD, D_ALL);
+	MMIO_D(GEN6_RC6p_THRESHOLD, D_ALL);
+	MMIO_D(GEN6_RC6pp_THRESHOLD, D_ALL);
+	MMIO_D(GEN6_PMINTRMSK, D_ALL);
+	MMIO_DH(HSW_PWR_WELL_BIOS, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_DRIVER, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_KVMR, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_DEBUG, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_CTL5, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_CTL6, D_HSW | D_BDW, NULL, power_well_ctl_mmio_write);
+
+	MMIO_D(RSTDBYCTL, D_ALL);
+
+	MMIO_DH(GEN6_GDRST, D_ALL, NULL, gdrst_mmio_write);
+	MMIO_F(FENCE_REG_GEN6_LO(0), 0x80, 0, 0, 0, D_ALL, fence_mmio_read, fence_mmio_write);
+	MMIO_F(VGT_PVINFO_PAGE, VGT_PVINFO_SIZE, F_UNALIGN, 0, 0, D_ALL, pvinfo_mmio_read, pvinfo_mmio_write);
+	MMIO_DH(CPU_VGACNTRL, D_ALL, NULL, vga_control_mmio_write);
+
+	MMIO_F(MCHBAR_MIRROR_BASE_SNB, 0x40000, 0, 0, 0, D_ALL, NULL, NULL);
+
+	MMIO_D(TILECTL, D_ALL);
+
+	MMIO_D(GEN6_UCGCTL1, D_ALL);
+	MMIO_D(GEN6_UCGCTL2, D_ALL);
+
+	MMIO_F(0x4f000, 0x90, 0, 0, 0, D_ALL, NULL, NULL);
+
+	MMIO_D(GEN6_PCODE_MAILBOX, D_PRE_SKL);
+	MMIO_D(GEN6_PCODE_DATA, D_ALL);
+	MMIO_D(0x13812c, D_ALL);
+	MMIO_DH(GEN7_ERR_INT, D_ALL, NULL, NULL);
+	MMIO_D(HSW_EDRAM_CAP, D_ALL);
+	MMIO_D(HSW_IDICR, D_ALL);
+	MMIO_DH(GFX_FLSH_CNTL_GEN6, D_ALL, NULL, NULL);
+
+	MMIO_D(0x3c, D_ALL);
+	MMIO_D(0x860, D_ALL);
+	MMIO_D(ECOSKPD, D_ALL);
+	MMIO_D(0x121d0, D_ALL);
+	MMIO_D(GEN6_BLITTER_ECOSKPD, D_ALL);
+	MMIO_D(0x41d0, D_ALL);
+	MMIO_D(GAC_ECO_BITS, D_ALL);
+	MMIO_D(0x6200, D_ALL);
+	MMIO_D(0x6204, D_ALL);
+	MMIO_D(0x6208, D_ALL);
+	MMIO_D(0x7118, D_ALL);
+	MMIO_D(0x7180, D_ALL);
+	MMIO_D(0x7408, D_ALL);
+	MMIO_D(0x7c00, D_ALL);
+	MMIO_D(GEN6_MBCTL, D_ALL);
+	MMIO_D(0x911c, D_ALL);
+	MMIO_D(0x9120, D_ALL);
+	MMIO_DFH(GEN7_UCGCTL4, D_ALL, F_CMD_ACCESS, NULL, NULL);
+
+	MMIO_D(GAB_CTL, D_ALL);
+	MMIO_D(0x48800, D_ALL);
+	MMIO_D(0xce044, D_ALL);
+	MMIO_D(0xe6500, D_ALL);
+	MMIO_D(0xe6504, D_ALL);
+	MMIO_D(0xe6600, D_ALL);
+	MMIO_D(0xe6604, D_ALL);
+	MMIO_D(0xe6700, D_ALL);
+	MMIO_D(0xe6704, D_ALL);
+	MMIO_D(0xe6800, D_ALL);
+	MMIO_D(0xe6804, D_ALL);
+	MMIO_D(PCH_GMBUS4, D_ALL);
+	MMIO_D(PCH_GMBUS5, D_ALL);
+
+	MMIO_D(0x902c, D_ALL);
+	MMIO_D(0xec008, D_ALL);
+	MMIO_D(0xec00c, D_ALL);
+	MMIO_D(0xec008 + 0x18, D_ALL);
+	MMIO_D(0xec00c + 0x18, D_ALL);
+	MMIO_D(0xec008 + 0x18 * 2, D_ALL);
+	MMIO_D(0xec00c + 0x18 * 2, D_ALL);
+	MMIO_D(0xec008 + 0x18 * 3, D_ALL);
+	MMIO_D(0xec00c + 0x18 * 3, D_ALL);
+	MMIO_D(0xec408, D_ALL);
+	MMIO_D(0xec40c, D_ALL);
+	MMIO_D(0xec408 + 0x18, D_ALL);
+	MMIO_D(0xec40c + 0x18, D_ALL);
+	MMIO_D(0xec408 + 0x18 * 2, D_ALL);
+	MMIO_D(0xec40c + 0x18 * 2, D_ALL);
+	MMIO_D(0xec408 + 0x18 * 3, D_ALL);
+	MMIO_D(0xec40c + 0x18 * 3, D_ALL);
+	MMIO_D(0xfc810, D_ALL);
+	MMIO_D(0xfc81c, D_ALL);
+	MMIO_D(0xfc828, D_ALL);
+	MMIO_D(0xfc834, D_ALL);
+	MMIO_D(0xfcc00, D_ALL);
+	MMIO_D(0xfcc0c, D_ALL);
+	MMIO_D(0xfcc18, D_ALL);
+	MMIO_D(0xfcc24, D_ALL);
+	MMIO_D(0xfd000, D_ALL);
+	MMIO_D(0xfd00c, D_ALL);
+	MMIO_D(0xfd018, D_ALL);
+	MMIO_D(0xfd024, D_ALL);
+	MMIO_D(0xfd034, D_ALL);
+
+	MMIO_DH(FPGA_DBG, D_ALL, NULL, fpga_dbg_mmio_write);
+	MMIO_D(0x2054, D_ALL);
+	MMIO_D(0x12054, D_ALL);
+	MMIO_D(0x22054, D_ALL);
+	MMIO_D(0x1a054, D_ALL);
+
+	MMIO_D(0x44070, D_ALL);
+
+	MMIO_D(0x215c, D_HSW_PLUS);
+	MMIO_DFH(0x2178, D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(0x217c, D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(0x12178, D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(0x1217c, D_ALL, F_CMD_ACCESS, NULL, NULL);
+
+	MMIO_F(0x2290, 8, 0, 0, 0, D_HSW_PLUS, NULL, NULL);
+	MMIO_D(OACONTROL, D_HSW);
+	MMIO_D(0x2b00, D_BDW_PLUS);
+	MMIO_D(0x2360, D_BDW_PLUS);
+	MMIO_F(0x5200, 32, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(0x5240, 32, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(0x5280, 16, 0, 0, 0, D_ALL, NULL, NULL);
+
+	MMIO_DFH(0x1c17c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(0x1c178, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_D(BCS_SWCTRL, D_ALL);
+
+	MMIO_F(HS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(DS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(IA_VERTICES_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(IA_PRIMITIVES_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(VS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(GS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(GS_PRIMITIVES_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(CL_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(CL_PRIMITIVES_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(PS_INVOCATION_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(PS_DEPTH_COUNT, 8, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_DH(0x4260, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+	MMIO_DH(0x4264, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+	MMIO_DH(0x4268, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+	MMIO_DH(0x426c, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+	MMIO_DH(0x4270, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+	MMIO_DFH(0x4094, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+
+	return 0;
+}
+
+static int init_broadwell_mmio_info(struct intel_gvt *gvt)
+{
+	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	int ret;
+
+	MMIO_DH(RING_IMR(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL,
+			intel_vgpu_reg_imr_handler);
+
+	MMIO_DH(GEN8_GT_IMR(0), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+	MMIO_DH(GEN8_GT_IER(0), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+	MMIO_DH(GEN8_GT_IIR(0), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+	MMIO_D(GEN8_GT_ISR(0), D_BDW_PLUS);
+
+	MMIO_DH(GEN8_GT_IMR(1), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+	MMIO_DH(GEN8_GT_IER(1), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+	MMIO_DH(GEN8_GT_IIR(1), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+	MMIO_D(GEN8_GT_ISR(1), D_BDW_PLUS);
+
+	MMIO_DH(GEN8_GT_IMR(2), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+	MMIO_DH(GEN8_GT_IER(2), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+	MMIO_DH(GEN8_GT_IIR(2), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+	MMIO_D(GEN8_GT_ISR(2), D_BDW_PLUS);
+
+	MMIO_DH(GEN8_GT_IMR(3), D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+	MMIO_DH(GEN8_GT_IER(3), D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+	MMIO_DH(GEN8_GT_IIR(3), D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+	MMIO_D(GEN8_GT_ISR(3), D_BDW_PLUS);
+
+	MMIO_DH(GEN8_DE_PIPE_IMR(PIPE_A), D_BDW_PLUS, NULL,
+		intel_vgpu_reg_imr_handler);
+	MMIO_DH(GEN8_DE_PIPE_IER(PIPE_A), D_BDW_PLUS, NULL,
+		intel_vgpu_reg_ier_handler);
+	MMIO_DH(GEN8_DE_PIPE_IIR(PIPE_A), D_BDW_PLUS, NULL,
+		intel_vgpu_reg_iir_handler);
+	MMIO_D(GEN8_DE_PIPE_ISR(PIPE_A), D_BDW_PLUS);
+
+	MMIO_DH(GEN8_DE_PIPE_IMR(PIPE_B), D_BDW_PLUS, NULL,
+		intel_vgpu_reg_imr_handler);
+	MMIO_DH(GEN8_DE_PIPE_IER(PIPE_B), D_BDW_PLUS, NULL,
+		intel_vgpu_reg_ier_handler);
+	MMIO_DH(GEN8_DE_PIPE_IIR(PIPE_B), D_BDW_PLUS, NULL,
+		intel_vgpu_reg_iir_handler);
+	MMIO_D(GEN8_DE_PIPE_ISR(PIPE_B), D_BDW_PLUS);
+
+	MMIO_DH(GEN8_DE_PIPE_IMR(PIPE_C), D_BDW_PLUS, NULL,
+		intel_vgpu_reg_imr_handler);
+	MMIO_DH(GEN8_DE_PIPE_IER(PIPE_C), D_BDW_PLUS, NULL,
+		intel_vgpu_reg_ier_handler);
+	MMIO_DH(GEN8_DE_PIPE_IIR(PIPE_C), D_BDW_PLUS, NULL,
+		intel_vgpu_reg_iir_handler);
+	MMIO_D(GEN8_DE_PIPE_ISR(PIPE_C), D_BDW_PLUS);
+
+	MMIO_DH(GEN8_DE_PORT_IMR, D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+	MMIO_DH(GEN8_DE_PORT_IER, D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+	MMIO_DH(GEN8_DE_PORT_IIR, D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+	MMIO_D(GEN8_DE_PORT_ISR, D_BDW_PLUS);
+
+	MMIO_DH(GEN8_DE_MISC_IMR, D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+	MMIO_DH(GEN8_DE_MISC_IER, D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+	MMIO_DH(GEN8_DE_MISC_IIR, D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+	MMIO_D(GEN8_DE_MISC_ISR, D_BDW_PLUS);
+
+	MMIO_DH(GEN8_PCU_IMR, D_BDW_PLUS, NULL, intel_vgpu_reg_imr_handler);
+	MMIO_DH(GEN8_PCU_IER, D_BDW_PLUS, NULL, intel_vgpu_reg_ier_handler);
+	MMIO_DH(GEN8_PCU_IIR, D_BDW_PLUS, NULL, intel_vgpu_reg_iir_handler);
+	MMIO_D(GEN8_PCU_ISR, D_BDW_PLUS);
+
+	MMIO_DH(GEN8_MASTER_IRQ, D_BDW_PLUS, NULL,
+		intel_vgpu_reg_master_irq_handler);
+
+	MMIO_D(RING_HWSTAM(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+	MMIO_D(0x1c134, D_BDW_PLUS);
+
+	MMIO_D(RING_TAIL(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+	MMIO_D(RING_HEAD(GEN8_BSD2_RING_BASE),  D_BDW_PLUS);
+	MMIO_GM(RING_START(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL, NULL);
+	MMIO_D(RING_CTL(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+	MMIO_D(RING_ACTHD(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+	MMIO_D(RING_ACTHD_UDW(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+	MMIO_DFH(0x1c29c, D_BDW_PLUS, F_MODE_MASK, NULL, ring_mode_mmio_write);
+	MMIO_DFH(RING_MI_MODE(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_MODE_MASK,
+			NULL, NULL);
+	MMIO_DFH(RING_INSTPM(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_MODE_MASK,
+			NULL, NULL);
+	MMIO_DFH(RING_TIMESTAMP(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_CMD_ACCESS,
+			ring_timestamp_mmio_read, NULL);
+
+	MMIO_RING_D(RING_ACTHD_UDW, D_BDW_PLUS);
+
+#define RING_REG(base) (base + 0xd0)
+	MMIO_RING_F(RING_REG, 4, F_RO, 0,
+		~_MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET), D_BDW_PLUS, NULL,
+		ring_reset_ctl_write);
+	MMIO_F(RING_REG(GEN8_BSD2_RING_BASE), 4, F_RO, 0,
+		~_MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET), D_BDW_PLUS, NULL,
+		ring_reset_ctl_write);
+#undef RING_REG
+
+#define RING_REG(base) (base + 0x230)
+	MMIO_RING_DFH(RING_REG, D_BDW_PLUS, 0, NULL, elsp_mmio_write);
+	MMIO_DH(RING_REG(GEN8_BSD2_RING_BASE), D_BDW_PLUS, NULL, elsp_mmio_write);
+#undef RING_REG
+
+#define RING_REG(base) (base + 0x234)
+	MMIO_RING_F(RING_REG, 8, F_RO, 0, ~0, D_BDW_PLUS, NULL, NULL);
+	MMIO_F(RING_REG(GEN8_BSD2_RING_BASE), 4, F_RO, 0, ~0LL, D_BDW_PLUS, NULL, NULL);
+#undef RING_REG
+
+#define RING_REG(base) (base + 0x244)
+	MMIO_RING_D(RING_REG, D_BDW_PLUS);
+	MMIO_D(RING_REG(GEN8_BSD2_RING_BASE), D_BDW_PLUS);
+#undef RING_REG
+
+#define RING_REG(base) (base + 0x370)
+	MMIO_RING_F(RING_REG, 48, F_RO, 0, ~0, D_BDW_PLUS, NULL, NULL);
+	MMIO_F(RING_REG(GEN8_BSD2_RING_BASE), 48, F_RO, 0, ~0, D_BDW_PLUS,
+			NULL, NULL);
+#undef RING_REG
+
+#define RING_REG(base) (base + 0x3a0)
+	MMIO_RING_DFH(RING_REG, D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
+	MMIO_DFH(RING_REG(GEN8_BSD2_RING_BASE), D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
+#undef RING_REG
+
+	MMIO_D(PIPEMISC(PIPE_A), D_BDW_PLUS);
+	MMIO_D(PIPEMISC(PIPE_B), D_BDW_PLUS);
+	MMIO_D(PIPEMISC(PIPE_C), D_BDW_PLUS);
+	MMIO_D(0x1c1d0, D_BDW_PLUS);
+	MMIO_D(GEN6_MBCUNIT_SNPCR, D_BDW_PLUS);
+	MMIO_D(GEN7_MISCCPCTL, D_BDW_PLUS);
+	MMIO_D(0x1c054, D_BDW_PLUS);
+
+	MMIO_D(GEN8_PRIVATE_PAT_LO, D_BDW_PLUS);
+	MMIO_D(GEN8_PRIVATE_PAT_HI, D_BDW_PLUS);
+
+	MMIO_D(GAMTARBMODE, D_BDW_PLUS);
+
+#define RING_REG(base) (base + 0x270)
+	MMIO_RING_F(RING_REG, 32, 0, 0, 0, D_BDW_PLUS, NULL, NULL);
+	MMIO_F(RING_REG(GEN8_BSD2_RING_BASE), 32, 0, 0, 0, D_BDW_PLUS, NULL, NULL);
+#undef RING_REG
+
+	MMIO_RING_GM(RING_HWS_PGA, D_BDW_PLUS, NULL, NULL);
+	MMIO_GM(0x1c080, D_BDW_PLUS, NULL, NULL);
+
+	MMIO_DFH(HDC_CHICKEN0, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+
+	MMIO_D(CHICKEN_PIPESL_1(PIPE_A), D_BDW);
+	MMIO_D(CHICKEN_PIPESL_1(PIPE_B), D_BDW);
+	MMIO_D(CHICKEN_PIPESL_1(PIPE_C), D_BDW);
+
+	MMIO_D(WM_MISC, D_BDW);
+	MMIO_D(BDW_EDP_PSR_BASE, D_BDW);
+
+	MMIO_D(0x66c00, D_BDW_PLUS);
+	MMIO_D(0x66c04, D_BDW_PLUS);
+
+	MMIO_D(HSW_GTT_CACHE_EN, D_BDW_PLUS);
+
+	MMIO_D(GEN8_EU_DISABLE0, D_BDW_PLUS);
+	MMIO_D(GEN8_EU_DISABLE1, D_BDW_PLUS);
+	MMIO_D(GEN8_EU_DISABLE2, D_BDW_PLUS);
+
+	MMIO_D(0xfdc, D_BDW);
+	MMIO_DFH(GEN8_ROW_CHICKEN, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_D(GEN7_ROW_CHICKEN2, D_BDW_PLUS);
+	MMIO_D(GEN8_UCGCTL6, D_BDW_PLUS);
+
+	MMIO_D(0xb1f0, D_BDW);
+	MMIO_D(0xb1c0, D_BDW);
+	MMIO_DFH(GEN8_L3SQCREG4, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_D(0xb100, D_BDW);
+	MMIO_D(0xb10c, D_BDW);
+	MMIO_D(0xb110, D_BDW);
+
+	MMIO_DFH(0x24d0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(0x24d4, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(0x24d8, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(0x24dc, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+
+	MMIO_D(0x83a4, D_BDW);
+	MMIO_D(GEN8_L3_LRA_1_GPGPU, D_BDW_PLUS);
+
+	MMIO_D(0x8430, D_BDW);
+
+	MMIO_D(0x110000, D_BDW_PLUS);
+
+	MMIO_D(0x48400, D_BDW_PLUS);
+
+	MMIO_D(0x6e570, D_BDW_PLUS);
+	MMIO_D(0x65f10, D_BDW_PLUS);
+
+	MMIO_DFH(0xe194, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(0xe188, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(HALF_SLICE_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(0x2580, D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
+
+	MMIO_D(0x2248, D_BDW);
+
+	return 0;
+}
+
+static int init_skl_mmio_info(struct intel_gvt *gvt)
+{
+	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	int ret;
+
+	MMIO_DH(FORCEWAKE_RENDER_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write);
+	MMIO_DH(FORCEWAKE_ACK_RENDER_GEN9, D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(FORCEWAKE_BLITTER_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write);
+	MMIO_DH(FORCEWAKE_ACK_BLITTER_GEN9, D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(FORCEWAKE_MEDIA_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write);
+	MMIO_DH(FORCEWAKE_ACK_MEDIA_GEN9, D_SKL_PLUS, NULL, NULL);
+
+	MMIO_F(_DPB_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL, NULL, dp_aux_ch_ctl_mmio_write);
+	MMIO_F(_DPC_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL, NULL, dp_aux_ch_ctl_mmio_write);
+	MMIO_F(_DPD_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL, NULL, dp_aux_ch_ctl_mmio_write);
+
+	MMIO_D(HSW_PWR_WELL_BIOS, D_SKL);
+	MMIO_DH(HSW_PWR_WELL_DRIVER, D_SKL, NULL, skl_power_well_ctl_write);
+
+	MMIO_DH(GEN6_PCODE_MAILBOX, D_SKL, NULL, mailbox_write);
+	MMIO_D(0xa210, D_SKL_PLUS);
+	MMIO_D(GEN9_MEDIA_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
+	MMIO_D(GEN9_RENDER_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
+	MMIO_DFH(GEN9_GAMT_ECO_REG_RW_IA, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DH(0x4ddc, D_SKL, NULL, skl_misc_ctl_write);
+	MMIO_DH(0x42080, D_SKL, NULL, skl_misc_ctl_write);
+	MMIO_D(0x45504, D_SKL);
+	MMIO_D(0x45520, D_SKL);
+	MMIO_D(0x46000, D_SKL);
+	MMIO_DH(0x46010, D_SKL, NULL, skl_lcpll_write);
+	MMIO_DH(0x46014, D_SKL, NULL, skl_lcpll_write);
+	MMIO_D(0x6C040, D_SKL);
+	MMIO_D(0x6C048, D_SKL);
+	MMIO_D(0x6C050, D_SKL);
+	MMIO_D(0x6C044, D_SKL);
+	MMIO_D(0x6C04C, D_SKL);
+	MMIO_D(0x6C054, D_SKL);
+	MMIO_D(0x6c058, D_SKL);
+	MMIO_D(0x6c05c, D_SKL);
+	MMIO_DH(0X6c060, D_SKL, dpll_status_read, NULL);
+
+	MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 0), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 1), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_WIN_POS(PIPE_B, 0), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_WIN_POS(PIPE_B, 1), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_WIN_POS(PIPE_C, 0), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_WIN_POS(PIPE_C, 1), D_SKL, NULL, pf_write);
+
+	MMIO_DH(SKL_PS_WIN_SZ(PIPE_A, 0), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_WIN_SZ(PIPE_A, 1), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_WIN_SZ(PIPE_B, 0), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_WIN_SZ(PIPE_B, 1), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_WIN_SZ(PIPE_C, 0), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_WIN_SZ(PIPE_C, 1), D_SKL, NULL, pf_write);
+
+	MMIO_DH(SKL_PS_CTRL(PIPE_A, 0), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_CTRL(PIPE_A, 1), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_CTRL(PIPE_B, 0), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_CTRL(PIPE_B, 1), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_CTRL(PIPE_C, 0), D_SKL, NULL, pf_write);
+	MMIO_DH(SKL_PS_CTRL(PIPE_C, 1), D_SKL, NULL, pf_write);
+
+	MMIO_DH(PLANE_BUF_CFG(PIPE_A, 0), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_BUF_CFG(PIPE_A, 1), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_BUF_CFG(PIPE_A, 2), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_BUF_CFG(PIPE_A, 3), D_SKL, NULL, NULL);
+
+	MMIO_DH(PLANE_BUF_CFG(PIPE_B, 0), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_BUF_CFG(PIPE_B, 1), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_BUF_CFG(PIPE_B, 2), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_BUF_CFG(PIPE_B, 3), D_SKL, NULL, NULL);
+
+	MMIO_DH(PLANE_BUF_CFG(PIPE_C, 0), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_BUF_CFG(PIPE_C, 1), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_BUF_CFG(PIPE_C, 2), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_BUF_CFG(PIPE_C, 3), D_SKL, NULL, NULL);
+
+	MMIO_DH(CUR_BUF_CFG(PIPE_A), D_SKL, NULL, NULL);
+	MMIO_DH(CUR_BUF_CFG(PIPE_B), D_SKL, NULL, NULL);
+	MMIO_DH(CUR_BUF_CFG(PIPE_C), D_SKL, NULL, NULL);
+
+	MMIO_F(PLANE_WM(PIPE_A, 0, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+	MMIO_F(PLANE_WM(PIPE_A, 1, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+	MMIO_F(PLANE_WM(PIPE_A, 2, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+
+	MMIO_F(PLANE_WM(PIPE_B, 0, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+	MMIO_F(PLANE_WM(PIPE_B, 1, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+	MMIO_F(PLANE_WM(PIPE_B, 2, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+
+	MMIO_F(PLANE_WM(PIPE_C, 0, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+	MMIO_F(PLANE_WM(PIPE_C, 1, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+	MMIO_F(PLANE_WM(PIPE_C, 2, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+
+	MMIO_F(CUR_WM(PIPE_A, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+	MMIO_F(CUR_WM(PIPE_B, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+	MMIO_F(CUR_WM(PIPE_C, 0), 4 * 8, 0, 0, 0, D_SKL, NULL, NULL);
+
+	MMIO_DH(PLANE_WM_TRANS(PIPE_A, 0), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_WM_TRANS(PIPE_A, 1), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_WM_TRANS(PIPE_A, 2), D_SKL, NULL, NULL);
+
+	MMIO_DH(PLANE_WM_TRANS(PIPE_B, 0), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_WM_TRANS(PIPE_B, 1), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_WM_TRANS(PIPE_B, 2), D_SKL, NULL, NULL);
+
+	MMIO_DH(PLANE_WM_TRANS(PIPE_C, 0), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_WM_TRANS(PIPE_C, 1), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_WM_TRANS(PIPE_C, 2), D_SKL, NULL, NULL);
+
+	MMIO_DH(CUR_WM_TRANS(PIPE_A), D_SKL, NULL, NULL);
+	MMIO_DH(CUR_WM_TRANS(PIPE_B), D_SKL, NULL, NULL);
+	MMIO_DH(CUR_WM_TRANS(PIPE_C), D_SKL, NULL, NULL);
+
+	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 0), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 1), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 2), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_A, 3), D_SKL, NULL, NULL);
+
+	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 0), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 1), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 2), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_B, 3), D_SKL, NULL, NULL);
+
+	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 0), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 1), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 2), D_SKL, NULL, NULL);
+	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 3), D_SKL, NULL, NULL);
+
+	MMIO_DH(_REG_701C0(PIPE_A, 1), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C0(PIPE_A, 2), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C0(PIPE_A, 3), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C0(PIPE_A, 4), D_SKL, NULL, NULL);
+
+	MMIO_DH(_REG_701C0(PIPE_B, 1), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C0(PIPE_B, 2), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C0(PIPE_B, 3), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C0(PIPE_B, 4), D_SKL, NULL, NULL);
+
+	MMIO_DH(_REG_701C0(PIPE_C, 1), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C0(PIPE_C, 2), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C0(PIPE_C, 3), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C0(PIPE_C, 4), D_SKL, NULL, NULL);
+
+	MMIO_DH(_REG_701C4(PIPE_A, 1), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C4(PIPE_A, 2), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C4(PIPE_A, 3), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C4(PIPE_A, 4), D_SKL, NULL, NULL);
+
+	MMIO_DH(_REG_701C4(PIPE_B, 1), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C4(PIPE_B, 2), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C4(PIPE_B, 3), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C4(PIPE_B, 4), D_SKL, NULL, NULL);
+
+	MMIO_DH(_REG_701C4(PIPE_C, 1), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C4(PIPE_C, 2), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C4(PIPE_C, 3), D_SKL, NULL, NULL);
+	MMIO_DH(_REG_701C4(PIPE_C, 4), D_SKL, NULL, NULL);
+
+	MMIO_D(0x70380, D_SKL);
+	MMIO_D(0x71380, D_SKL);
+	MMIO_D(0x72380, D_SKL);
+	MMIO_D(0x7039c, D_SKL);
+
+	MMIO_F(0x80000, 0x3000, 0, 0, 0, D_SKL, NULL, NULL);
+	MMIO_D(0x8f074, D_SKL);
+	MMIO_D(0x8f004, D_SKL);
+	MMIO_D(0x8f034, D_SKL);
+
+	MMIO_D(0xb11c, D_SKL);
+
+	MMIO_D(0x51000, D_SKL);
+	MMIO_D(0x6c00c, D_SKL);
+
+	MMIO_F(0xc800, 0x7f8, F_CMD_ACCESS, 0, 0, D_SKL, NULL, NULL);
+	MMIO_F(0xb020, 0x80, F_CMD_ACCESS, 0, 0, D_SKL, NULL, NULL);
+
+	MMIO_D(0xd08, D_SKL);
+	MMIO_D(0x20e0, D_SKL);
+	MMIO_D(0x20ec, D_SKL);
+
+	/* TRTT */
+	MMIO_D(0x4de0, D_SKL);
+	MMIO_D(0x4de4, D_SKL);
+	MMIO_D(0x4de8, D_SKL);
+	MMIO_D(0x4dec, D_SKL);
+	MMIO_D(0x4df0, D_SKL);
+	MMIO_DH(0x4df4, D_SKL, NULL, gen9_trtte_write);
+	MMIO_DH(0x4dfc, D_SKL, NULL, gen9_trtt_chicken_write);
+
+	MMIO_D(0x45008, D_SKL);
+
+	MMIO_D(0x46430, D_SKL);
+
+	MMIO_D(0x46520, D_SKL);
+
+	MMIO_D(0xc403c, D_SKL);
+	MMIO_D(0xb004, D_SKL);
+	MMIO_DH(DMA_CTRL, D_SKL_PLUS, NULL, dma_ctrl_write);
+
+	MMIO_D(0x65900, D_SKL);
+	MMIO_D(0x1082c0, D_SKL);
+	MMIO_D(0x4068, D_SKL);
+	MMIO_D(0x67054, D_SKL);
+	MMIO_D(0x6e560, D_SKL);
+	MMIO_D(0x6e554, D_SKL);
+	MMIO_D(0x2b20, D_SKL);
+	MMIO_D(0x65f00, D_SKL);
+	MMIO_D(0x65f08, D_SKL);
+	MMIO_D(0x320f0, D_SKL);
+
+	MMIO_D(_REG_VCS2_EXCC, D_SKL);
+	MMIO_D(0x70034, D_SKL);
+	MMIO_D(0x71034, D_SKL);
+	MMIO_D(0x72034, D_SKL);
+
+	MMIO_D(_PLANE_KEYVAL_1(PIPE_A), D_SKL);
+	MMIO_D(_PLANE_KEYVAL_1(PIPE_B), D_SKL);
+	MMIO_D(_PLANE_KEYVAL_1(PIPE_C), D_SKL);
+	MMIO_D(_PLANE_KEYMSK_1(PIPE_A), D_SKL);
+	MMIO_D(_PLANE_KEYMSK_1(PIPE_B), D_SKL);
+	MMIO_D(_PLANE_KEYMSK_1(PIPE_C), D_SKL);
+
+	MMIO_D(0x44500, D_SKL);
+	return 0;
+}
+
+/**
+ * intel_gvt_find_mmio_info - find MMIO information entry by aligned offset
+ * @gvt: GVT device
+ * @offset: register offset
+ *
+ * This function is used to find the MMIO information entry from hash table
+ *
+ * Returns:
+ * pointer to MMIO information entry, NULL if not exists
+ */
+struct intel_gvt_mmio_info *intel_gvt_find_mmio_info(struct intel_gvt *gvt,
+	unsigned int offset)
+{
+	struct intel_gvt_mmio_info *e;
+
+	WARN_ON(!IS_ALIGNED(offset, 4));
+
+	hash_for_each_possible(gvt->mmio.mmio_info_table, e, node, offset) {
+		if (e->offset == offset)
+			return e;
+	}
+	return NULL;
+}
+
+/**
+ * intel_gvt_clean_mmio_info - clean up MMIO information table for GVT device
+ * @gvt: GVT device
+ *
+ * This function is called at the driver unloading stage, to clean up the MMIO
+ * information table of GVT device
+ *
+ */
+void intel_gvt_clean_mmio_info(struct intel_gvt *gvt)
+{
+	struct hlist_node *tmp;
+	struct intel_gvt_mmio_info *e;
+	int i;
+
+	hash_for_each_safe(gvt->mmio.mmio_info_table, i, tmp, e, node)
+		kfree(e);
+
+	vfree(gvt->mmio.mmio_attribute);
+	gvt->mmio.mmio_attribute = NULL;
+}
+
+/**
+ * intel_gvt_setup_mmio_info - setup MMIO information table for GVT device
+ * @gvt: GVT device
+ *
+ * This function is called at the initialization stage, to setup the MMIO
+ * information table for GVT device
+ *
+ * Returns:
+ * zero on success, negative if failed.
+ */
+int intel_gvt_setup_mmio_info(struct intel_gvt *gvt)
+{
+	struct intel_gvt_device_info *info = &gvt->device_info;
+	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	int ret;
+
+	gvt->mmio.mmio_attribute = vzalloc(info->mmio_size);
+	if (!gvt->mmio.mmio_attribute)
+		return -ENOMEM;
+
+	ret = init_generic_mmio_info(gvt);
+	if (ret)
+		goto err;
+
+	if (IS_BROADWELL(dev_priv)) {
+		ret = init_broadwell_mmio_info(gvt);
+		if (ret)
+			goto err;
+	} else if (IS_SKYLAKE(dev_priv)) {
+		ret = init_broadwell_mmio_info(gvt);
+		if (ret)
+			goto err;
+		ret = init_skl_mmio_info(gvt);
+		if (ret)
+			goto err;
+	}
+	return 0;
+err:
+	intel_gvt_clean_mmio_info(gvt);
+	return ret;
+}
+
+/**
+ * intel_gvt_mmio_set_accessed - mark a MMIO has been accessed
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+void intel_gvt_mmio_set_accessed(struct intel_gvt *gvt, unsigned int offset)
+{
+	gvt->mmio.mmio_attribute[offset >> 2] |=
+		F_ACCESSED;
+}
+
+/**
+ * intel_gvt_mmio_is_cmd_accessed - mark a MMIO could be accessed by command
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+bool intel_gvt_mmio_is_cmd_access(struct intel_gvt *gvt,
+		unsigned int offset)
+{
+	return gvt->mmio.mmio_attribute[offset >> 2] &
+		F_CMD_ACCESS;
+}
+
+/**
+ * intel_gvt_mmio_is_unalign - mark a MMIO could be accessed unaligned
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+bool intel_gvt_mmio_is_unalign(struct intel_gvt *gvt,
+		unsigned int offset)
+{
+	return gvt->mmio.mmio_attribute[offset >> 2] &
+		F_UNALIGN;
+}
+
+/**
+ * intel_gvt_mmio_set_cmd_accessed - mark a MMIO has been accessed by command
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ */
+void intel_gvt_mmio_set_cmd_accessed(struct intel_gvt *gvt,
+		unsigned int offset)
+{
+	gvt->mmio.mmio_attribute[offset >> 2] |=
+		F_CMD_ACCESSED;
+}
+
+/**
+ * intel_gvt_mmio_has_mode_mask - if a MMIO has a mode mask
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ * Returns:
+ * True if a MMIO has a mode mask in its higher 16 bits, false if it isn't.
+ *
+ */
+bool intel_gvt_mmio_has_mode_mask(struct intel_gvt *gvt, unsigned int offset)
+{
+	return gvt->mmio.mmio_attribute[offset >> 2] &
+		F_MODE_MASK;
+}
+
+/**
+ * intel_vgpu_default_mmio_read - default MMIO read handler
+ * @vgpu: a vGPU
+ * @offset: access offset
+ * @p_data: data return buffer
+ * @bytes: access data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_default_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	read_vreg(vgpu, offset, p_data, bytes);
+	return 0;
+}
+
+/**
+ * intel_t_default_mmio_write - default MMIO write handler
+ * @vgpu: a vGPU
+ * @offset: access offset
+ * @p_data: write data buffer
+ * @bytes: access data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_default_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	write_vreg(vgpu, offset, p_data, bytes);
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h
index 254df8bf1f35..30e543f5a703 100644
--- a/drivers/gpu/drm/i915/gvt/hypercall.h
+++ b/drivers/gpu/drm/i915/gvt/hypercall.h
@@ -19,6 +19,15 @@
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Dexuan Cui
+ *    Jike Song <jike.song@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
  */
 
 #ifndef _GVT_HYPERCALL_H_
@@ -30,6 +39,23 @@
  */
 struct intel_gvt_mpt {
 	int (*detect_host)(void);
+	int (*host_init)(struct device *dev, void *gvt, const void *ops);
+	void (*host_exit)(struct device *dev, void *gvt);
+	int (*attach_vgpu)(void *vgpu, unsigned long *handle);
+	void (*detach_vgpu)(unsigned long handle);
+	int (*inject_msi)(unsigned long handle, u32 addr, u16 data);
+	unsigned long (*from_virt_to_mfn)(void *p);
+	int (*set_wp_page)(unsigned long handle, u64 gfn);
+	int (*unset_wp_page)(unsigned long handle, u64 gfn);
+	int (*read_gpa)(unsigned long handle, unsigned long gpa, void *buf,
+			unsigned long len);
+	int (*write_gpa)(unsigned long handle, unsigned long gpa, void *buf,
+			 unsigned long len);
+	unsigned long (*gfn_to_mfn)(unsigned long handle, unsigned long gfn);
+	int (*map_gfn_to_mfn)(unsigned long handle, unsigned long gfn,
+			      unsigned long mfn, unsigned int nr, bool map);
+	int (*set_trap_area)(unsigned long handle, u64 start, u64 end,
+			     bool map);
 };
 
 extern struct intel_gvt_mpt xengt_mpt;
diff --git a/drivers/gpu/drm/i915/gvt/interrupt.c b/drivers/gpu/drm/i915/gvt/interrupt.c
new file mode 100644
index 000000000000..f7be02ac4be1
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/interrupt.c
@@ -0,0 +1,741 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Min he <min.he@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+/* common offset among interrupt control registers */
+#define regbase_to_isr(base)	(base)
+#define regbase_to_imr(base)	(base + 0x4)
+#define regbase_to_iir(base)	(base + 0x8)
+#define regbase_to_ier(base)	(base + 0xC)
+
+#define iir_to_regbase(iir)    (iir - 0x8)
+#define ier_to_regbase(ier)    (ier - 0xC)
+
+#define get_event_virt_handler(irq, e)	(irq->events[e].v_handler)
+#define get_irq_info(irq, e)		(irq->events[e].info)
+
+#define irq_to_gvt(irq) \
+	container_of(irq, struct intel_gvt, irq)
+
+static void update_upstream_irq(struct intel_vgpu *vgpu,
+		struct intel_gvt_irq_info *info);
+
+static const char * const irq_name[INTEL_GVT_EVENT_MAX] = {
+	[RCS_MI_USER_INTERRUPT] = "Render CS MI USER INTERRUPT",
+	[RCS_DEBUG] = "Render EU debug from SVG",
+	[RCS_MMIO_SYNC_FLUSH] = "Render MMIO sync flush status",
+	[RCS_CMD_STREAMER_ERR] = "Render CS error interrupt",
+	[RCS_PIPE_CONTROL] = "Render PIPE CONTROL notify",
+	[RCS_WATCHDOG_EXCEEDED] = "Render CS Watchdog counter exceeded",
+	[RCS_PAGE_DIRECTORY_FAULT] = "Render page directory faults",
+	[RCS_AS_CONTEXT_SWITCH] = "Render AS Context Switch Interrupt",
+
+	[VCS_MI_USER_INTERRUPT] = "Video CS MI USER INTERRUPT",
+	[VCS_MMIO_SYNC_FLUSH] = "Video MMIO sync flush status",
+	[VCS_CMD_STREAMER_ERR] = "Video CS error interrupt",
+	[VCS_MI_FLUSH_DW] = "Video MI FLUSH DW notify",
+	[VCS_WATCHDOG_EXCEEDED] = "Video CS Watchdog counter exceeded",
+	[VCS_PAGE_DIRECTORY_FAULT] = "Video page directory faults",
+	[VCS_AS_CONTEXT_SWITCH] = "Video AS Context Switch Interrupt",
+	[VCS2_MI_USER_INTERRUPT] = "VCS2 Video CS MI USER INTERRUPT",
+	[VCS2_MI_FLUSH_DW] = "VCS2 Video MI FLUSH DW notify",
+	[VCS2_AS_CONTEXT_SWITCH] = "VCS2 Context Switch Interrupt",
+
+	[BCS_MI_USER_INTERRUPT] = "Blitter CS MI USER INTERRUPT",
+	[BCS_MMIO_SYNC_FLUSH] = "Billter MMIO sync flush status",
+	[BCS_CMD_STREAMER_ERR] = "Blitter CS error interrupt",
+	[BCS_MI_FLUSH_DW] = "Blitter MI FLUSH DW notify",
+	[BCS_PAGE_DIRECTORY_FAULT] = "Blitter page directory faults",
+	[BCS_AS_CONTEXT_SWITCH] = "Blitter AS Context Switch Interrupt",
+
+	[VECS_MI_FLUSH_DW] = "Video Enhanced Streamer MI FLUSH DW notify",
+	[VECS_AS_CONTEXT_SWITCH] = "VECS Context Switch Interrupt",
+
+	[PIPE_A_FIFO_UNDERRUN] = "Pipe A FIFO underrun",
+	[PIPE_A_CRC_ERR] = "Pipe A CRC error",
+	[PIPE_A_CRC_DONE] = "Pipe A CRC done",
+	[PIPE_A_VSYNC] = "Pipe A vsync",
+	[PIPE_A_LINE_COMPARE] = "Pipe A line compare",
+	[PIPE_A_ODD_FIELD] = "Pipe A odd field",
+	[PIPE_A_EVEN_FIELD] = "Pipe A even field",
+	[PIPE_A_VBLANK] = "Pipe A vblank",
+	[PIPE_B_FIFO_UNDERRUN] = "Pipe B FIFO underrun",
+	[PIPE_B_CRC_ERR] = "Pipe B CRC error",
+	[PIPE_B_CRC_DONE] = "Pipe B CRC done",
+	[PIPE_B_VSYNC] = "Pipe B vsync",
+	[PIPE_B_LINE_COMPARE] = "Pipe B line compare",
+	[PIPE_B_ODD_FIELD] = "Pipe B odd field",
+	[PIPE_B_EVEN_FIELD] = "Pipe B even field",
+	[PIPE_B_VBLANK] = "Pipe B vblank",
+	[PIPE_C_VBLANK] = "Pipe C vblank",
+	[DPST_PHASE_IN] = "DPST phase in event",
+	[DPST_HISTOGRAM] = "DPST histogram event",
+	[GSE] = "GSE",
+	[DP_A_HOTPLUG] = "DP A Hotplug",
+	[AUX_CHANNEL_A] = "AUX Channel A",
+	[PERF_COUNTER] = "Performance counter",
+	[POISON] = "Poison",
+	[GTT_FAULT] = "GTT fault",
+	[PRIMARY_A_FLIP_DONE] = "Primary Plane A flip done",
+	[PRIMARY_B_FLIP_DONE] = "Primary Plane B flip done",
+	[PRIMARY_C_FLIP_DONE] = "Primary Plane C flip done",
+	[SPRITE_A_FLIP_DONE] = "Sprite Plane A flip done",
+	[SPRITE_B_FLIP_DONE] = "Sprite Plane B flip done",
+	[SPRITE_C_FLIP_DONE] = "Sprite Plane C flip done",
+
+	[PCU_THERMAL] = "PCU Thermal Event",
+	[PCU_PCODE2DRIVER_MAILBOX] = "PCU pcode2driver mailbox event",
+
+	[FDI_RX_INTERRUPTS_TRANSCODER_A] = "FDI RX Interrupts Combined A",
+	[AUDIO_CP_CHANGE_TRANSCODER_A] = "Audio CP Change Transcoder A",
+	[AUDIO_CP_REQUEST_TRANSCODER_A] = "Audio CP Request Transcoder A",
+	[FDI_RX_INTERRUPTS_TRANSCODER_B] = "FDI RX Interrupts Combined B",
+	[AUDIO_CP_CHANGE_TRANSCODER_B] = "Audio CP Change Transcoder B",
+	[AUDIO_CP_REQUEST_TRANSCODER_B] = "Audio CP Request Transcoder B",
+	[FDI_RX_INTERRUPTS_TRANSCODER_C] = "FDI RX Interrupts Combined C",
+	[AUDIO_CP_CHANGE_TRANSCODER_C] = "Audio CP Change Transcoder C",
+	[AUDIO_CP_REQUEST_TRANSCODER_C] = "Audio CP Request Transcoder C",
+	[ERR_AND_DBG] = "South Error and Debug Interupts Combined",
+	[GMBUS] = "Gmbus",
+	[SDVO_B_HOTPLUG] = "SDVO B hotplug",
+	[CRT_HOTPLUG] = "CRT Hotplug",
+	[DP_B_HOTPLUG] = "DisplayPort/HDMI/DVI B Hotplug",
+	[DP_C_HOTPLUG] = "DisplayPort/HDMI/DVI C Hotplug",
+	[DP_D_HOTPLUG] = "DisplayPort/HDMI/DVI D Hotplug",
+	[AUX_CHANNEL_B] = "AUX Channel B",
+	[AUX_CHANNEL_C] = "AUX Channel C",
+	[AUX_CHANNEL_D] = "AUX Channel D",
+	[AUDIO_POWER_STATE_CHANGE_B] = "Audio Power State change Port B",
+	[AUDIO_POWER_STATE_CHANGE_C] = "Audio Power State change Port C",
+	[AUDIO_POWER_STATE_CHANGE_D] = "Audio Power State change Port D",
+
+	[INTEL_GVT_EVENT_RESERVED] = "RESERVED EVENTS!!!",
+};
+
+static inline struct intel_gvt_irq_info *regbase_to_irq_info(
+		struct intel_gvt *gvt,
+		unsigned int reg)
+{
+	struct intel_gvt_irq *irq = &gvt->irq;
+	int i;
+
+	for_each_set_bit(i, irq->irq_info_bitmap, INTEL_GVT_IRQ_INFO_MAX) {
+		if (i915_mmio_reg_offset(irq->info[i]->reg_base) == reg)
+			return irq->info[i];
+	}
+
+	return NULL;
+}
+
+/**
+ * intel_vgpu_reg_imr_handler - Generic IMR register emulation write handler
+ * @vgpu: a vGPU
+ * @reg: register offset written by guest
+ * @p_data: register data written by guest
+ * @bytes: register data length
+ *
+ * This function is used to emulate the generic IMR register bit change
+ * behavior.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_reg_imr_handler(struct intel_vgpu *vgpu,
+	unsigned int reg, void *p_data, unsigned int bytes)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_irq_ops *ops = gvt->irq.ops;
+	u32 changed, masked, unmasked;
+	u32 imr = *(u32 *)p_data;
+
+	gvt_dbg_irq("write IMR %x with val %x\n",
+		reg, imr);
+
+	gvt_dbg_irq("old vIMR %x\n", vgpu_vreg(vgpu, reg));
+
+	/* figure out newly masked/unmasked bits */
+	changed = vgpu_vreg(vgpu, reg) ^ imr;
+	masked = (vgpu_vreg(vgpu, reg) & changed) ^ changed;
+	unmasked = masked ^ changed;
+
+	gvt_dbg_irq("changed %x, masked %x, unmasked %x\n",
+		changed, masked, unmasked);
+
+	vgpu_vreg(vgpu, reg) = imr;
+
+	ops->check_pending_irq(vgpu);
+	gvt_dbg_irq("IRQ: new vIMR %x\n", vgpu_vreg(vgpu, reg));
+	return 0;
+}
+
+/**
+ * intel_vgpu_reg_master_irq_handler - master IRQ write emulation handler
+ * @vgpu: a vGPU
+ * @reg: register offset written by guest
+ * @p_data: register data written by guest
+ * @bytes: register data length
+ *
+ * This function is used to emulate the master IRQ register on gen8+.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_reg_master_irq_handler(struct intel_vgpu *vgpu,
+	unsigned int reg, void *p_data, unsigned int bytes)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_irq_ops *ops = gvt->irq.ops;
+	u32 changed, enabled, disabled;
+	u32 ier = *(u32 *)p_data;
+	u32 virtual_ier = vgpu_vreg(vgpu, reg);
+
+	gvt_dbg_irq("write master irq reg %x with val %x\n",
+		reg, ier);
+
+	gvt_dbg_irq("old vreg %x\n", vgpu_vreg(vgpu, reg));
+
+	/*
+	 * GEN8_MASTER_IRQ is a special irq register,
+	 * only bit 31 is allowed to be modified
+	 * and treated as an IER bit.
+	 */
+	ier &= GEN8_MASTER_IRQ_CONTROL;
+	virtual_ier &= GEN8_MASTER_IRQ_CONTROL;
+	vgpu_vreg(vgpu, reg) &= ~GEN8_MASTER_IRQ_CONTROL;
+	vgpu_vreg(vgpu, reg) |= ier;
+
+	/* figure out newly enabled/disable bits */
+	changed = virtual_ier ^ ier;
+	enabled = (virtual_ier & changed) ^ changed;
+	disabled = enabled ^ changed;
+
+	gvt_dbg_irq("changed %x, enabled %x, disabled %x\n",
+			changed, enabled, disabled);
+
+	ops->check_pending_irq(vgpu);
+	gvt_dbg_irq("new vreg %x\n", vgpu_vreg(vgpu, reg));
+	return 0;
+}
+
+/**
+ * intel_vgpu_reg_ier_handler - Generic IER write emulation handler
+ * @vgpu: a vGPU
+ * @reg: register offset written by guest
+ * @p_data: register data written by guest
+ * @bytes: register data length
+ *
+ * This function is used to emulate the generic IER register behavior.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_reg_ier_handler(struct intel_vgpu *vgpu,
+	unsigned int reg, void *p_data, unsigned int bytes)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_irq_ops *ops = gvt->irq.ops;
+	struct intel_gvt_irq_info *info;
+	u32 changed, enabled, disabled;
+	u32 ier = *(u32 *)p_data;
+
+	gvt_dbg_irq("write IER %x with val %x\n",
+		reg, ier);
+
+	gvt_dbg_irq("old vIER %x\n", vgpu_vreg(vgpu, reg));
+
+	/* figure out newly enabled/disable bits */
+	changed = vgpu_vreg(vgpu, reg) ^ ier;
+	enabled = (vgpu_vreg(vgpu, reg) & changed) ^ changed;
+	disabled = enabled ^ changed;
+
+	gvt_dbg_irq("changed %x, enabled %x, disabled %x\n",
+			changed, enabled, disabled);
+	vgpu_vreg(vgpu, reg) = ier;
+
+	info = regbase_to_irq_info(gvt, ier_to_regbase(reg));
+	if (WARN_ON(!info))
+		return -EINVAL;
+
+	if (info->has_upstream_irq)
+		update_upstream_irq(vgpu, info);
+
+	ops->check_pending_irq(vgpu);
+	gvt_dbg_irq("new vIER %x\n", vgpu_vreg(vgpu, reg));
+	return 0;
+}
+
+/**
+ * intel_vgpu_reg_iir_handler - Generic IIR write emulation handler
+ * @vgpu: a vGPU
+ * @reg: register offset written by guest
+ * @p_data: register data written by guest
+ * @bytes: register data length
+ *
+ * This function is used to emulate the generic IIR register behavior.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_reg_iir_handler(struct intel_vgpu *vgpu, unsigned int reg,
+	void *p_data, unsigned int bytes)
+{
+	struct intel_gvt_irq_info *info = regbase_to_irq_info(vgpu->gvt,
+		iir_to_regbase(reg));
+	u32 iir = *(u32 *)p_data;
+
+	gvt_dbg_irq("write IIR %x with val %x\n", reg, iir);
+
+	if (WARN_ON(!info))
+		return -EINVAL;
+
+	vgpu_vreg(vgpu, reg) &= ~iir;
+
+	if (info->has_upstream_irq)
+		update_upstream_irq(vgpu, info);
+	return 0;
+}
+
+static struct intel_gvt_irq_map gen8_irq_map[] = {
+	{ INTEL_GVT_IRQ_INFO_MASTER, 0, INTEL_GVT_IRQ_INFO_GT0, 0xffff },
+	{ INTEL_GVT_IRQ_INFO_MASTER, 1, INTEL_GVT_IRQ_INFO_GT0, 0xffff0000 },
+	{ INTEL_GVT_IRQ_INFO_MASTER, 2, INTEL_GVT_IRQ_INFO_GT1, 0xffff },
+	{ INTEL_GVT_IRQ_INFO_MASTER, 3, INTEL_GVT_IRQ_INFO_GT1, 0xffff0000 },
+	{ INTEL_GVT_IRQ_INFO_MASTER, 4, INTEL_GVT_IRQ_INFO_GT2, 0xffff },
+	{ INTEL_GVT_IRQ_INFO_MASTER, 6, INTEL_GVT_IRQ_INFO_GT3, 0xffff },
+	{ INTEL_GVT_IRQ_INFO_MASTER, 16, INTEL_GVT_IRQ_INFO_DE_PIPE_A, ~0 },
+	{ INTEL_GVT_IRQ_INFO_MASTER, 17, INTEL_GVT_IRQ_INFO_DE_PIPE_B, ~0 },
+	{ INTEL_GVT_IRQ_INFO_MASTER, 18, INTEL_GVT_IRQ_INFO_DE_PIPE_C, ~0 },
+	{ INTEL_GVT_IRQ_INFO_MASTER, 20, INTEL_GVT_IRQ_INFO_DE_PORT, ~0 },
+	{ INTEL_GVT_IRQ_INFO_MASTER, 22, INTEL_GVT_IRQ_INFO_DE_MISC, ~0 },
+	{ INTEL_GVT_IRQ_INFO_MASTER, 23, INTEL_GVT_IRQ_INFO_PCH, ~0 },
+	{ INTEL_GVT_IRQ_INFO_MASTER, 30, INTEL_GVT_IRQ_INFO_PCU, ~0 },
+	{ -1, -1, ~0 },
+};
+
+static void update_upstream_irq(struct intel_vgpu *vgpu,
+		struct intel_gvt_irq_info *info)
+{
+	struct intel_gvt_irq *irq = &vgpu->gvt->irq;
+	struct intel_gvt_irq_map *map = irq->irq_map;
+	struct intel_gvt_irq_info *up_irq_info = NULL;
+	u32 set_bits = 0;
+	u32 clear_bits = 0;
+	int bit;
+	u32 val = vgpu_vreg(vgpu,
+			regbase_to_iir(i915_mmio_reg_offset(info->reg_base)))
+		& vgpu_vreg(vgpu,
+			regbase_to_ier(i915_mmio_reg_offset(info->reg_base)));
+
+	if (!info->has_upstream_irq)
+		return;
+
+	for (map = irq->irq_map; map->up_irq_bit != -1; map++) {
+		if (info->group != map->down_irq_group)
+			continue;
+
+		if (!up_irq_info)
+			up_irq_info = irq->info[map->up_irq_group];
+		else
+			WARN_ON(up_irq_info != irq->info[map->up_irq_group]);
+
+		bit = map->up_irq_bit;
+
+		if (val & map->down_irq_bitmask)
+			set_bits |= (1 << bit);
+		else
+			clear_bits |= (1 << bit);
+	}
+
+	WARN_ON(!up_irq_info);
+
+	if (up_irq_info->group == INTEL_GVT_IRQ_INFO_MASTER) {
+		u32 isr = i915_mmio_reg_offset(up_irq_info->reg_base);
+
+		vgpu_vreg(vgpu, isr) &= ~clear_bits;
+		vgpu_vreg(vgpu, isr) |= set_bits;
+	} else {
+		u32 iir = regbase_to_iir(
+			i915_mmio_reg_offset(up_irq_info->reg_base));
+		u32 imr = regbase_to_imr(
+			i915_mmio_reg_offset(up_irq_info->reg_base));
+
+		vgpu_vreg(vgpu, iir) |= (set_bits & ~vgpu_vreg(vgpu, imr));
+	}
+
+	if (up_irq_info->has_upstream_irq)
+		update_upstream_irq(vgpu, up_irq_info);
+}
+
+static void init_irq_map(struct intel_gvt_irq *irq)
+{
+	struct intel_gvt_irq_map *map;
+	struct intel_gvt_irq_info *up_info, *down_info;
+	int up_bit;
+
+	for (map = irq->irq_map; map->up_irq_bit != -1; map++) {
+		up_info = irq->info[map->up_irq_group];
+		up_bit = map->up_irq_bit;
+		down_info = irq->info[map->down_irq_group];
+
+		set_bit(up_bit, up_info->downstream_irq_bitmap);
+		down_info->has_upstream_irq = true;
+
+		gvt_dbg_irq("[up] grp %d bit %d -> [down] grp %d bitmask %x\n",
+			up_info->group, up_bit,
+			down_info->group, map->down_irq_bitmask);
+	}
+}
+
+/* =======================vEvent injection===================== */
+static int inject_virtual_interrupt(struct intel_vgpu *vgpu)
+{
+	return intel_gvt_hypervisor_inject_msi(vgpu);
+}
+
+static void propagate_event(struct intel_gvt_irq *irq,
+	enum intel_gvt_event_type event, struct intel_vgpu *vgpu)
+{
+	struct intel_gvt_irq_info *info;
+	unsigned int reg_base;
+	int bit;
+
+	info = get_irq_info(irq, event);
+	if (WARN_ON(!info))
+		return;
+
+	reg_base = i915_mmio_reg_offset(info->reg_base);
+	bit = irq->events[event].bit;
+
+	if (!test_bit(bit, (void *)&vgpu_vreg(vgpu,
+					regbase_to_imr(reg_base)))) {
+		gvt_dbg_irq("set bit (%d) for (%s) for vgpu (%d)\n",
+				bit, irq_name[event], vgpu->id);
+		set_bit(bit, (void *)&vgpu_vreg(vgpu,
+					regbase_to_iir(reg_base)));
+	}
+}
+
+/* =======================vEvent Handlers===================== */
+static void handle_default_event_virt(struct intel_gvt_irq *irq,
+	enum intel_gvt_event_type event, struct intel_vgpu *vgpu)
+{
+	if (!vgpu->irq.irq_warn_once[event]) {
+		gvt_dbg_core("vgpu%d: IRQ receive event %d (%s)\n",
+			vgpu->id, event, irq_name[event]);
+		vgpu->irq.irq_warn_once[event] = true;
+	}
+	propagate_event(irq, event, vgpu);
+}
+
+/* =====================GEN specific logic======================= */
+/* GEN8 interrupt routines. */
+
+#define DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(regname, regbase) \
+static struct intel_gvt_irq_info gen8_##regname##_info = { \
+	.name = #regname"-IRQ", \
+	.reg_base = (regbase), \
+	.bit_to_event = {[0 ... INTEL_GVT_IRQ_BITWIDTH-1] = \
+		INTEL_GVT_EVENT_RESERVED}, \
+}
+
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt0, GEN8_GT_ISR(0));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt1, GEN8_GT_ISR(1));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt2, GEN8_GT_ISR(2));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt3, GEN8_GT_ISR(3));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_pipe_a, GEN8_DE_PIPE_ISR(PIPE_A));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_pipe_b, GEN8_DE_PIPE_ISR(PIPE_B));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_pipe_c, GEN8_DE_PIPE_ISR(PIPE_C));
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_port, GEN8_DE_PORT_ISR);
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_misc, GEN8_DE_MISC_ISR);
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(pcu, GEN8_PCU_ISR);
+DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(master, GEN8_MASTER_IRQ);
+
+static struct intel_gvt_irq_info gvt_base_pch_info = {
+	.name = "PCH-IRQ",
+	.reg_base = SDEISR,
+	.bit_to_event = {[0 ... INTEL_GVT_IRQ_BITWIDTH-1] =
+		INTEL_GVT_EVENT_RESERVED},
+};
+
+static void gen8_check_pending_irq(struct intel_vgpu *vgpu)
+{
+	struct intel_gvt_irq *irq = &vgpu->gvt->irq;
+	int i;
+
+	if (!(vgpu_vreg(vgpu, i915_mmio_reg_offset(GEN8_MASTER_IRQ)) &
+				GEN8_MASTER_IRQ_CONTROL))
+		return;
+
+	for_each_set_bit(i, irq->irq_info_bitmap, INTEL_GVT_IRQ_INFO_MAX) {
+		struct intel_gvt_irq_info *info = irq->info[i];
+		u32 reg_base;
+
+		if (!info->has_upstream_irq)
+			continue;
+
+		reg_base = i915_mmio_reg_offset(info->reg_base);
+		if ((vgpu_vreg(vgpu, regbase_to_iir(reg_base))
+				& vgpu_vreg(vgpu, regbase_to_ier(reg_base))))
+			update_upstream_irq(vgpu, info);
+	}
+
+	if (vgpu_vreg(vgpu, i915_mmio_reg_offset(GEN8_MASTER_IRQ))
+			& ~GEN8_MASTER_IRQ_CONTROL)
+		inject_virtual_interrupt(vgpu);
+}
+
+static void gen8_init_irq(
+		struct intel_gvt_irq *irq)
+{
+	struct intel_gvt *gvt = irq_to_gvt(irq);
+
+#define SET_BIT_INFO(s, b, e, i)		\
+	do {					\
+		s->events[e].bit = b;		\
+		s->events[e].info = s->info[i];	\
+		s->info[i]->bit_to_event[b] = e;\
+	} while (0)
+
+#define SET_IRQ_GROUP(s, g, i) \
+	do { \
+		s->info[g] = i; \
+		(i)->group = g; \
+		set_bit(g, s->irq_info_bitmap); \
+	} while (0)
+
+	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_MASTER, &gen8_master_info);
+	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT0, &gen8_gt0_info);
+	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT1, &gen8_gt1_info);
+	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT2, &gen8_gt2_info);
+	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT3, &gen8_gt3_info);
+	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PIPE_A, &gen8_de_pipe_a_info);
+	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PIPE_B, &gen8_de_pipe_b_info);
+	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PIPE_C, &gen8_de_pipe_c_info);
+	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PORT, &gen8_de_port_info);
+	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_MISC, &gen8_de_misc_info);
+	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_PCU, &gen8_pcu_info);
+	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_PCH, &gvt_base_pch_info);
+
+	/* GEN8 level 2 interrupts. */
+
+	/* GEN8 interrupt GT0 events */
+	SET_BIT_INFO(irq, 0, RCS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT0);
+	SET_BIT_INFO(irq, 4, RCS_PIPE_CONTROL, INTEL_GVT_IRQ_INFO_GT0);
+	SET_BIT_INFO(irq, 8, RCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT0);
+
+	SET_BIT_INFO(irq, 16, BCS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT0);
+	SET_BIT_INFO(irq, 20, BCS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT0);
+	SET_BIT_INFO(irq, 24, BCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT0);
+
+	/* GEN8 interrupt GT1 events */
+	SET_BIT_INFO(irq, 0, VCS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT1);
+	SET_BIT_INFO(irq, 4, VCS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT1);
+	SET_BIT_INFO(irq, 8, VCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT1);
+
+	if (HAS_BSD2(gvt->dev_priv)) {
+		SET_BIT_INFO(irq, 16, VCS2_MI_USER_INTERRUPT,
+			INTEL_GVT_IRQ_INFO_GT1);
+		SET_BIT_INFO(irq, 20, VCS2_MI_FLUSH_DW,
+			INTEL_GVT_IRQ_INFO_GT1);
+		SET_BIT_INFO(irq, 24, VCS2_AS_CONTEXT_SWITCH,
+			INTEL_GVT_IRQ_INFO_GT1);
+	}
+
+	/* GEN8 interrupt GT3 events */
+	SET_BIT_INFO(irq, 0, VECS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT3);
+	SET_BIT_INFO(irq, 4, VECS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT3);
+	SET_BIT_INFO(irq, 8, VECS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT3);
+
+	SET_BIT_INFO(irq, 0, PIPE_A_VBLANK, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
+	SET_BIT_INFO(irq, 0, PIPE_B_VBLANK, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
+	SET_BIT_INFO(irq, 0, PIPE_C_VBLANK, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
+
+	/* GEN8 interrupt DE PORT events */
+	SET_BIT_INFO(irq, 0, AUX_CHANNEL_A, INTEL_GVT_IRQ_INFO_DE_PORT);
+	SET_BIT_INFO(irq, 3, DP_A_HOTPLUG, INTEL_GVT_IRQ_INFO_DE_PORT);
+
+	/* GEN8 interrupt DE MISC events */
+	SET_BIT_INFO(irq, 0, GSE, INTEL_GVT_IRQ_INFO_DE_MISC);
+
+	/* PCH events */
+	SET_BIT_INFO(irq, 17, GMBUS, INTEL_GVT_IRQ_INFO_PCH);
+	SET_BIT_INFO(irq, 19, CRT_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH);
+	SET_BIT_INFO(irq, 21, DP_B_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH);
+	SET_BIT_INFO(irq, 22, DP_C_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH);
+	SET_BIT_INFO(irq, 23, DP_D_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH);
+
+	if (IS_BROADWELL(gvt->dev_priv)) {
+		SET_BIT_INFO(irq, 25, AUX_CHANNEL_B, INTEL_GVT_IRQ_INFO_PCH);
+		SET_BIT_INFO(irq, 26, AUX_CHANNEL_C, INTEL_GVT_IRQ_INFO_PCH);
+		SET_BIT_INFO(irq, 27, AUX_CHANNEL_D, INTEL_GVT_IRQ_INFO_PCH);
+
+		SET_BIT_INFO(irq, 4, PRIMARY_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
+		SET_BIT_INFO(irq, 5, SPRITE_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
+
+		SET_BIT_INFO(irq, 4, PRIMARY_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
+		SET_BIT_INFO(irq, 5, SPRITE_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
+
+		SET_BIT_INFO(irq, 4, PRIMARY_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
+		SET_BIT_INFO(irq, 5, SPRITE_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
+	} else if (IS_SKYLAKE(gvt->dev_priv)) {
+		SET_BIT_INFO(irq, 25, AUX_CHANNEL_B, INTEL_GVT_IRQ_INFO_DE_PORT);
+		SET_BIT_INFO(irq, 26, AUX_CHANNEL_C, INTEL_GVT_IRQ_INFO_DE_PORT);
+		SET_BIT_INFO(irq, 27, AUX_CHANNEL_D, INTEL_GVT_IRQ_INFO_DE_PORT);
+
+		SET_BIT_INFO(irq, 3, PRIMARY_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
+		SET_BIT_INFO(irq, 3, PRIMARY_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
+		SET_BIT_INFO(irq, 3, PRIMARY_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
+	}
+
+	/* GEN8 interrupt PCU events */
+	SET_BIT_INFO(irq, 24, PCU_THERMAL, INTEL_GVT_IRQ_INFO_PCU);
+	SET_BIT_INFO(irq, 25, PCU_PCODE2DRIVER_MAILBOX, INTEL_GVT_IRQ_INFO_PCU);
+}
+
+static struct intel_gvt_irq_ops gen8_irq_ops = {
+	.init_irq = gen8_init_irq,
+	.check_pending_irq = gen8_check_pending_irq,
+};
+
+/**
+ * intel_vgpu_trigger_virtual_event - Trigger a virtual event for a vGPU
+ * @vgpu: a vGPU
+ * @event: interrupt event
+ *
+ * This function is used to trigger a virtual interrupt event for vGPU.
+ * The caller provides the event to be triggered, the framework itself
+ * will emulate the IRQ register bit change.
+ *
+ */
+void intel_vgpu_trigger_virtual_event(struct intel_vgpu *vgpu,
+	enum intel_gvt_event_type event)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_irq *irq = &gvt->irq;
+	gvt_event_virt_handler_t handler;
+	struct intel_gvt_irq_ops *ops = gvt->irq.ops;
+
+	handler = get_event_virt_handler(irq, event);
+	WARN_ON(!handler);
+
+	handler(irq, event, vgpu);
+
+	ops->check_pending_irq(vgpu);
+}
+
+static void init_events(
+	struct intel_gvt_irq *irq)
+{
+	int i;
+
+	for (i = 0; i < INTEL_GVT_EVENT_MAX; i++) {
+		irq->events[i].info = NULL;
+		irq->events[i].v_handler = handle_default_event_virt;
+	}
+}
+
+static enum hrtimer_restart vblank_timer_fn(struct hrtimer *data)
+{
+	struct intel_gvt_vblank_timer *vblank_timer;
+	struct intel_gvt_irq *irq;
+	struct intel_gvt *gvt;
+
+	vblank_timer = container_of(data, struct intel_gvt_vblank_timer, timer);
+	irq = container_of(vblank_timer, struct intel_gvt_irq, vblank_timer);
+	gvt = container_of(irq, struct intel_gvt, irq);
+
+	intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EMULATE_VBLANK);
+	hrtimer_add_expires_ns(&vblank_timer->timer, vblank_timer->period);
+	return HRTIMER_RESTART;
+}
+
+/**
+ * intel_gvt_clean_irq - clean up GVT-g IRQ emulation subsystem
+ * @gvt: a GVT device
+ *
+ * This function is called at driver unloading stage, to clean up GVT-g IRQ
+ * emulation subsystem.
+ *
+ */
+void intel_gvt_clean_irq(struct intel_gvt *gvt)
+{
+	struct intel_gvt_irq *irq = &gvt->irq;
+
+	hrtimer_cancel(&irq->vblank_timer.timer);
+}
+
+#define VBLNAK_TIMER_PERIOD 16000000
+
+/**
+ * intel_gvt_init_irq - initialize GVT-g IRQ emulation subsystem
+ * @gvt: a GVT device
+ *
+ * This function is called at driver loading stage, to initialize the GVT-g IRQ
+ * emulation subsystem.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_gvt_init_irq(struct intel_gvt *gvt)
+{
+	struct intel_gvt_irq *irq = &gvt->irq;
+	struct intel_gvt_vblank_timer *vblank_timer = &irq->vblank_timer;
+
+	gvt_dbg_core("init irq framework\n");
+
+	if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)) {
+		irq->ops = &gen8_irq_ops;
+		irq->irq_map = gen8_irq_map;
+	} else {
+		WARN_ON(1);
+		return -ENODEV;
+	}
+
+	/* common event initialization */
+	init_events(irq);
+
+	/* gen specific initialization */
+	irq->ops->init_irq(irq);
+
+	init_irq_map(irq);
+
+	hrtimer_init(&vblank_timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
+	vblank_timer->timer.function = vblank_timer_fn;
+	vblank_timer->period = VBLNAK_TIMER_PERIOD;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/interrupt.h b/drivers/gpu/drm/i915/gvt/interrupt.h
new file mode 100644
index 000000000000..5313fb1b33e1
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/interrupt.h
@@ -0,0 +1,233 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Min he <min.he@intel.com>
+ *
+ */
+
+#ifndef _GVT_INTERRUPT_H_
+#define _GVT_INTERRUPT_H_
+
+enum intel_gvt_event_type {
+	RCS_MI_USER_INTERRUPT = 0,
+	RCS_DEBUG,
+	RCS_MMIO_SYNC_FLUSH,
+	RCS_CMD_STREAMER_ERR,
+	RCS_PIPE_CONTROL,
+	RCS_L3_PARITY_ERR,
+	RCS_WATCHDOG_EXCEEDED,
+	RCS_PAGE_DIRECTORY_FAULT,
+	RCS_AS_CONTEXT_SWITCH,
+	RCS_MONITOR_BUFF_HALF_FULL,
+
+	VCS_MI_USER_INTERRUPT,
+	VCS_MMIO_SYNC_FLUSH,
+	VCS_CMD_STREAMER_ERR,
+	VCS_MI_FLUSH_DW,
+	VCS_WATCHDOG_EXCEEDED,
+	VCS_PAGE_DIRECTORY_FAULT,
+	VCS_AS_CONTEXT_SWITCH,
+
+	VCS2_MI_USER_INTERRUPT,
+	VCS2_MI_FLUSH_DW,
+	VCS2_AS_CONTEXT_SWITCH,
+
+	BCS_MI_USER_INTERRUPT,
+	BCS_MMIO_SYNC_FLUSH,
+	BCS_CMD_STREAMER_ERR,
+	BCS_MI_FLUSH_DW,
+	BCS_PAGE_DIRECTORY_FAULT,
+	BCS_AS_CONTEXT_SWITCH,
+
+	VECS_MI_USER_INTERRUPT,
+	VECS_MI_FLUSH_DW,
+	VECS_AS_CONTEXT_SWITCH,
+
+	PIPE_A_FIFO_UNDERRUN,
+	PIPE_B_FIFO_UNDERRUN,
+	PIPE_A_CRC_ERR,
+	PIPE_B_CRC_ERR,
+	PIPE_A_CRC_DONE,
+	PIPE_B_CRC_DONE,
+	PIPE_A_ODD_FIELD,
+	PIPE_B_ODD_FIELD,
+	PIPE_A_EVEN_FIELD,
+	PIPE_B_EVEN_FIELD,
+	PIPE_A_LINE_COMPARE,
+	PIPE_B_LINE_COMPARE,
+	PIPE_C_LINE_COMPARE,
+	PIPE_A_VBLANK,
+	PIPE_B_VBLANK,
+	PIPE_C_VBLANK,
+	PIPE_A_VSYNC,
+	PIPE_B_VSYNC,
+	PIPE_C_VSYNC,
+	PRIMARY_A_FLIP_DONE,
+	PRIMARY_B_FLIP_DONE,
+	PRIMARY_C_FLIP_DONE,
+	SPRITE_A_FLIP_DONE,
+	SPRITE_B_FLIP_DONE,
+	SPRITE_C_FLIP_DONE,
+
+	PCU_THERMAL,
+	PCU_PCODE2DRIVER_MAILBOX,
+
+	DPST_PHASE_IN,
+	DPST_HISTOGRAM,
+	GSE,
+	DP_A_HOTPLUG,
+	AUX_CHANNEL_A,
+	PERF_COUNTER,
+	POISON,
+	GTT_FAULT,
+	ERROR_INTERRUPT_COMBINED,
+
+	FDI_RX_INTERRUPTS_TRANSCODER_A,
+	AUDIO_CP_CHANGE_TRANSCODER_A,
+	AUDIO_CP_REQUEST_TRANSCODER_A,
+	FDI_RX_INTERRUPTS_TRANSCODER_B,
+	AUDIO_CP_CHANGE_TRANSCODER_B,
+	AUDIO_CP_REQUEST_TRANSCODER_B,
+	FDI_RX_INTERRUPTS_TRANSCODER_C,
+	AUDIO_CP_CHANGE_TRANSCODER_C,
+	AUDIO_CP_REQUEST_TRANSCODER_C,
+	ERR_AND_DBG,
+	GMBUS,
+	SDVO_B_HOTPLUG,
+	CRT_HOTPLUG,
+	DP_B_HOTPLUG,
+	DP_C_HOTPLUG,
+	DP_D_HOTPLUG,
+	AUX_CHANNEL_B,
+	AUX_CHANNEL_C,
+	AUX_CHANNEL_D,
+	AUDIO_POWER_STATE_CHANGE_B,
+	AUDIO_POWER_STATE_CHANGE_C,
+	AUDIO_POWER_STATE_CHANGE_D,
+
+	INTEL_GVT_EVENT_RESERVED,
+	INTEL_GVT_EVENT_MAX,
+};
+
+struct intel_gvt_irq;
+struct intel_gvt;
+
+typedef void (*gvt_event_virt_handler_t)(struct intel_gvt_irq *irq,
+	enum intel_gvt_event_type event, struct intel_vgpu *vgpu);
+
+struct intel_gvt_irq_ops {
+	void (*init_irq)(struct intel_gvt_irq *irq);
+	void (*check_pending_irq)(struct intel_vgpu *vgpu);
+};
+
+/* the list of physical interrupt control register groups */
+enum intel_gvt_irq_type {
+	INTEL_GVT_IRQ_INFO_GT,
+	INTEL_GVT_IRQ_INFO_DPY,
+	INTEL_GVT_IRQ_INFO_PCH,
+	INTEL_GVT_IRQ_INFO_PM,
+
+	INTEL_GVT_IRQ_INFO_MASTER,
+	INTEL_GVT_IRQ_INFO_GT0,
+	INTEL_GVT_IRQ_INFO_GT1,
+	INTEL_GVT_IRQ_INFO_GT2,
+	INTEL_GVT_IRQ_INFO_GT3,
+	INTEL_GVT_IRQ_INFO_DE_PIPE_A,
+	INTEL_GVT_IRQ_INFO_DE_PIPE_B,
+	INTEL_GVT_IRQ_INFO_DE_PIPE_C,
+	INTEL_GVT_IRQ_INFO_DE_PORT,
+	INTEL_GVT_IRQ_INFO_DE_MISC,
+	INTEL_GVT_IRQ_INFO_AUD,
+	INTEL_GVT_IRQ_INFO_PCU,
+
+	INTEL_GVT_IRQ_INFO_MAX,
+};
+
+#define INTEL_GVT_IRQ_BITWIDTH	32
+
+/* device specific interrupt bit definitions */
+struct intel_gvt_irq_info {
+	char *name;
+	i915_reg_t reg_base;
+	enum intel_gvt_event_type bit_to_event[INTEL_GVT_IRQ_BITWIDTH];
+	unsigned long warned;
+	int group;
+	DECLARE_BITMAP(downstream_irq_bitmap, INTEL_GVT_IRQ_BITWIDTH);
+	bool has_upstream_irq;
+};
+
+/* per-event information */
+struct intel_gvt_event_info {
+	int bit;				/* map to register bit */
+	int policy;				/* forwarding policy */
+	struct intel_gvt_irq_info *info;	/* register info */
+	gvt_event_virt_handler_t v_handler;	/* for v_event */
+};
+
+struct intel_gvt_irq_map {
+	int up_irq_group;
+	int up_irq_bit;
+	int down_irq_group;
+	u32 down_irq_bitmask;
+};
+
+struct intel_gvt_vblank_timer {
+	struct hrtimer timer;
+	u64 period;
+};
+
+/* structure containing device specific IRQ state */
+struct intel_gvt_irq {
+	struct intel_gvt_irq_ops *ops;
+	struct intel_gvt_irq_info *info[INTEL_GVT_IRQ_INFO_MAX];
+	DECLARE_BITMAP(irq_info_bitmap, INTEL_GVT_IRQ_INFO_MAX);
+	struct intel_gvt_event_info events[INTEL_GVT_EVENT_MAX];
+	DECLARE_BITMAP(pending_events, INTEL_GVT_EVENT_MAX);
+	struct intel_gvt_irq_map *irq_map;
+	struct intel_gvt_vblank_timer vblank_timer;
+};
+
+int intel_gvt_init_irq(struct intel_gvt *gvt);
+void intel_gvt_clean_irq(struct intel_gvt *gvt);
+
+void intel_vgpu_trigger_virtual_event(struct intel_vgpu *vgpu,
+	enum intel_gvt_event_type event);
+
+int intel_vgpu_reg_iir_handler(struct intel_vgpu *vgpu, unsigned int reg,
+	void *p_data, unsigned int bytes);
+int intel_vgpu_reg_ier_handler(struct intel_vgpu *vgpu,
+	unsigned int reg, void *p_data, unsigned int bytes);
+int intel_vgpu_reg_master_irq_handler(struct intel_vgpu *vgpu,
+	unsigned int reg, void *p_data, unsigned int bytes);
+int intel_vgpu_reg_imr_handler(struct intel_vgpu *vgpu,
+	unsigned int reg, void *p_data, unsigned int bytes);
+
+int gvt_ring_id_to_pipe_control_notify_event(int ring_id);
+int gvt_ring_id_to_mi_flush_dw_event(int ring_id);
+int gvt_ring_id_to_mi_user_interrupt_event(int ring_id);
+
+#endif /* _GVT_INTERRUPT_H_ */
diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c
new file mode 100644
index 000000000000..dc0365033157
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/kvmgt.c
@@ -0,0 +1,597 @@
+/*
+ * KVMGT - the implementation of Intel mediated pass-through framework for KVM
+ *
+ * Copyright(c) 2014-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Jike Song <jike.song@intel.com>
+ *    Xiaoguang Chen <xiaoguang.chen@intel.com>
+ */
+
+#include <linux/init.h>
+#include <linux/device.h>
+#include <linux/mm.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/rbtree.h>
+#include <linux/spinlock.h>
+#include <linux/eventfd.h>
+#include <linux/uuid.h>
+#include <linux/kvm_host.h>
+#include <linux/vfio.h>
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+static inline long kvmgt_pin_pages(struct device *dev, unsigned long *user_pfn,
+			long npage, int prot, unsigned long *phys_pfn)
+{
+	return 0;
+}
+static inline long kvmgt_unpin_pages(struct device *dev, unsigned long *pfn,
+			long npage)
+{
+	return 0;
+}
+
+static const struct intel_gvt_ops *intel_gvt_ops;
+
+
+/* helper macros copied from vfio-pci */
+#define VFIO_PCI_OFFSET_SHIFT   40
+#define VFIO_PCI_OFFSET_TO_INDEX(off)   (off >> VFIO_PCI_OFFSET_SHIFT)
+#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
+#define VFIO_PCI_OFFSET_MASK    (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
+
+struct vfio_region {
+	u32				type;
+	u32				subtype;
+	size_t				size;
+	u32				flags;
+};
+
+struct kvmgt_pgfn {
+	gfn_t gfn;
+	struct hlist_node hnode;
+};
+
+struct kvmgt_guest_info {
+	struct kvm *kvm;
+	struct intel_vgpu *vgpu;
+	struct kvm_page_track_notifier_node track_node;
+#define NR_BKT (1 << 18)
+	struct hlist_head ptable[NR_BKT];
+#undef NR_BKT
+};
+
+struct gvt_dma {
+	struct rb_node node;
+	gfn_t gfn;
+	kvm_pfn_t pfn;
+};
+
+static struct gvt_dma *__gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn)
+{
+	struct rb_node *node = vgpu->vdev.cache.rb_node;
+	struct gvt_dma *ret = NULL;
+
+	while (node) {
+		struct gvt_dma *itr = rb_entry(node, struct gvt_dma, node);
+
+		if (gfn < itr->gfn)
+			node = node->rb_left;
+		else if (gfn > itr->gfn)
+			node = node->rb_right;
+		else {
+			ret = itr;
+			goto out;
+		}
+	}
+
+out:
+	return ret;
+}
+
+static kvm_pfn_t gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn)
+{
+	struct gvt_dma *entry;
+
+	mutex_lock(&vgpu->vdev.cache_lock);
+	entry = __gvt_cache_find(vgpu, gfn);
+	mutex_unlock(&vgpu->vdev.cache_lock);
+
+	return entry == NULL ? 0 : entry->pfn;
+}
+
+static void gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, kvm_pfn_t pfn)
+{
+	struct gvt_dma *new, *itr;
+	struct rb_node **link = &vgpu->vdev.cache.rb_node, *parent = NULL;
+
+	new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);
+	if (!new)
+		return;
+
+	new->gfn = gfn;
+	new->pfn = pfn;
+
+	mutex_lock(&vgpu->vdev.cache_lock);
+	while (*link) {
+		parent = *link;
+		itr = rb_entry(parent, struct gvt_dma, node);
+
+		if (gfn == itr->gfn)
+			goto out;
+		else if (gfn < itr->gfn)
+			link = &parent->rb_left;
+		else
+			link = &parent->rb_right;
+	}
+
+	rb_link_node(&new->node, parent, link);
+	rb_insert_color(&new->node, &vgpu->vdev.cache);
+	mutex_unlock(&vgpu->vdev.cache_lock);
+	return;
+
+out:
+	mutex_unlock(&vgpu->vdev.cache_lock);
+	kfree(new);
+}
+
+static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,
+				struct gvt_dma *entry)
+{
+	rb_erase(&entry->node, &vgpu->vdev.cache);
+	kfree(entry);
+}
+
+static void gvt_cache_remove(struct intel_vgpu *vgpu, gfn_t gfn)
+{
+	struct device *dev = vgpu->vdev.mdev;
+	struct gvt_dma *this;
+	unsigned long pfn;
+
+	mutex_lock(&vgpu->vdev.cache_lock);
+	this  = __gvt_cache_find(vgpu, gfn);
+	if (!this) {
+		mutex_unlock(&vgpu->vdev.cache_lock);
+		return;
+	}
+
+	pfn = this->pfn;
+	WARN_ON((kvmgt_unpin_pages(dev, &pfn, 1) != 1));
+	__gvt_cache_remove_entry(vgpu, this);
+	mutex_unlock(&vgpu->vdev.cache_lock);
+}
+
+static void gvt_cache_init(struct intel_vgpu *vgpu)
+{
+	vgpu->vdev.cache = RB_ROOT;
+	mutex_init(&vgpu->vdev.cache_lock);
+}
+
+static void gvt_cache_destroy(struct intel_vgpu *vgpu)
+{
+	struct gvt_dma *dma;
+	struct rb_node *node = NULL;
+	struct device *dev = vgpu->vdev.mdev;
+	unsigned long pfn;
+
+	mutex_lock(&vgpu->vdev.cache_lock);
+	while ((node = rb_first(&vgpu->vdev.cache))) {
+		dma = rb_entry(node, struct gvt_dma, node);
+		pfn = dma->pfn;
+
+		kvmgt_unpin_pages(dev, &pfn, 1);
+		__gvt_cache_remove_entry(vgpu, dma);
+	}
+	mutex_unlock(&vgpu->vdev.cache_lock);
+}
+
+static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt,
+		const char *name)
+{
+	int i;
+	struct intel_vgpu_type *t;
+	const char *driver_name = dev_driver_string(
+			&gvt->dev_priv->drm.pdev->dev);
+
+	for (i = 0; i < gvt->num_types; i++) {
+		t = &gvt->types[i];
+		if (!strncmp(t->name, name + strlen(driver_name) + 1,
+			sizeof(t->name)))
+			return t;
+	}
+
+	return NULL;
+}
+
+static struct attribute *type_attrs[] = {
+	NULL,
+};
+
+static struct attribute_group *intel_vgpu_type_groups[] = {
+	[0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL,
+};
+
+static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt)
+{
+	int i, j;
+	struct intel_vgpu_type *type;
+	struct attribute_group *group;
+
+	for (i = 0; i < gvt->num_types; i++) {
+		type = &gvt->types[i];
+
+		group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL);
+		if (WARN_ON(!group))
+			goto unwind;
+
+		group->name = type->name;
+		group->attrs = type_attrs;
+		intel_vgpu_type_groups[i] = group;
+	}
+
+	return true;
+
+unwind:
+	for (j = 0; j < i; j++) {
+		group = intel_vgpu_type_groups[j];
+		kfree(group);
+	}
+
+	return false;
+}
+
+static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt)
+{
+	int i;
+	struct attribute_group *group;
+
+	for (i = 0; i < gvt->num_types; i++) {
+		group = intel_vgpu_type_groups[i];
+		kfree(group);
+	}
+}
+
+static void kvmgt_protect_table_init(struct kvmgt_guest_info *info)
+{
+	hash_init(info->ptable);
+}
+
+static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info)
+{
+	struct kvmgt_pgfn *p;
+	struct hlist_node *tmp;
+	int i;
+
+	hash_for_each_safe(info->ptable, i, tmp, p, hnode) {
+		hash_del(&p->hnode);
+		kfree(p);
+	}
+}
+
+static struct kvmgt_pgfn *
+__kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn)
+{
+	struct kvmgt_pgfn *p, *res = NULL;
+
+	hash_for_each_possible(info->ptable, p, hnode, gfn) {
+		if (gfn == p->gfn) {
+			res = p;
+			break;
+		}
+	}
+
+	return res;
+}
+
+static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info,
+				gfn_t gfn)
+{
+	struct kvmgt_pgfn *p;
+
+	p = __kvmgt_protect_table_find(info, gfn);
+	return !!p;
+}
+
+static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn)
+{
+	struct kvmgt_pgfn *p;
+
+	if (kvmgt_gfn_is_write_protected(info, gfn))
+		return;
+
+	p = kmalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC);
+	if (WARN(!p, "gfn: 0x%llx\n", gfn))
+		return;
+
+	p->gfn = gfn;
+	hash_add(info->ptable, &p->hnode, gfn);
+}
+
+static void kvmgt_protect_table_del(struct kvmgt_guest_info *info,
+				gfn_t gfn)
+{
+	struct kvmgt_pgfn *p;
+
+	p = __kvmgt_protect_table_find(info, gfn);
+	if (p) {
+		hash_del(&p->hnode);
+		kfree(p);
+	}
+}
+
+static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops)
+{
+	if (!intel_gvt_init_vgpu_type_groups(gvt))
+		return -EFAULT;
+
+	intel_gvt_ops = ops;
+
+	/* MDEV is not yet available */
+	return -ENODEV;
+}
+
+static void kvmgt_host_exit(struct device *dev, void *gvt)
+{
+	intel_gvt_cleanup_vgpu_type_groups(gvt);
+}
+
+static int kvmgt_write_protect_add(unsigned long handle, u64 gfn)
+{
+	struct kvmgt_guest_info *info = (struct kvmgt_guest_info *)handle;
+	struct kvm *kvm = info->kvm;
+	struct kvm_memory_slot *slot;
+	int idx;
+
+	idx = srcu_read_lock(&kvm->srcu);
+	slot = gfn_to_memslot(kvm, gfn);
+
+	spin_lock(&kvm->mmu_lock);
+
+	if (kvmgt_gfn_is_write_protected(info, gfn))
+		goto out;
+
+	kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
+	kvmgt_protect_table_add(info, gfn);
+
+out:
+	spin_unlock(&kvm->mmu_lock);
+	srcu_read_unlock(&kvm->srcu, idx);
+	return 0;
+}
+
+static int kvmgt_write_protect_remove(unsigned long handle, u64 gfn)
+{
+	struct kvmgt_guest_info *info = (struct kvmgt_guest_info *)handle;
+	struct kvm *kvm = info->kvm;
+	struct kvm_memory_slot *slot;
+	int idx;
+
+	idx = srcu_read_lock(&kvm->srcu);
+	slot = gfn_to_memslot(kvm, gfn);
+
+	spin_lock(&kvm->mmu_lock);
+
+	if (!kvmgt_gfn_is_write_protected(info, gfn))
+		goto out;
+
+	kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
+	kvmgt_protect_table_del(info, gfn);
+
+out:
+	spin_unlock(&kvm->mmu_lock);
+	srcu_read_unlock(&kvm->srcu, idx);
+	return 0;
+}
+
+static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa,
+		const u8 *val, int len,
+		struct kvm_page_track_notifier_node *node)
+{
+	struct kvmgt_guest_info *info = container_of(node,
+					struct kvmgt_guest_info, track_node);
+
+	if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa)))
+		intel_gvt_ops->emulate_mmio_write(info->vgpu, gpa,
+					(void *)val, len);
+}
+
+static void kvmgt_page_track_flush_slot(struct kvm *kvm,
+		struct kvm_memory_slot *slot,
+		struct kvm_page_track_notifier_node *node)
+{
+	int i;
+	gfn_t gfn;
+	struct kvmgt_guest_info *info = container_of(node,
+					struct kvmgt_guest_info, track_node);
+
+	spin_lock(&kvm->mmu_lock);
+	for (i = 0; i < slot->npages; i++) {
+		gfn = slot->base_gfn + i;
+		if (kvmgt_gfn_is_write_protected(info, gfn)) {
+			kvm_slot_page_track_remove_page(kvm, slot, gfn,
+						KVM_PAGE_TRACK_WRITE);
+			kvmgt_protect_table_del(info, gfn);
+		}
+	}
+	spin_unlock(&kvm->mmu_lock);
+}
+
+static bool kvmgt_check_guest(void)
+{
+	unsigned int eax, ebx, ecx, edx;
+	char s[12];
+	unsigned int *i;
+
+	eax = KVM_CPUID_SIGNATURE;
+	ebx = ecx = edx = 0;
+
+	asm volatile ("cpuid"
+		      : "+a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
+		      :
+		      : "cc", "memory");
+	i = (unsigned int *)s;
+	i[0] = ebx;
+	i[1] = ecx;
+	i[2] = edx;
+
+	return !strncmp(s, "KVMKVMKVM", strlen("KVMKVMKVM"));
+}
+
+/**
+ * NOTE:
+ * It's actually impossible to check if we are running in KVM host,
+ * since the "KVM host" is simply native. So we only dectect guest here.
+ */
+static int kvmgt_detect_host(void)
+{
+#ifdef CONFIG_INTEL_IOMMU
+	if (intel_iommu_gfx_mapped) {
+		gvt_err("Hardware IOMMU compatibility not yet supported, try to boot with intel_iommu=igfx_off\n");
+		return -ENODEV;
+	}
+#endif
+	return kvmgt_check_guest() ? -ENODEV : 0;
+}
+
+static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle)
+{
+	/* nothing to do here */
+	return 0;
+}
+
+static void kvmgt_detach_vgpu(unsigned long handle)
+{
+	/* nothing to do here */
+}
+
+static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data)
+{
+	struct kvmgt_guest_info *info = (struct kvmgt_guest_info *)handle;
+	struct intel_vgpu *vgpu = info->vgpu;
+
+	if (vgpu->vdev.msi_trigger)
+		return eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1;
+
+	return false;
+}
+
+static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn)
+{
+	unsigned long pfn;
+	struct kvmgt_guest_info *info = (struct kvmgt_guest_info *)handle;
+	int rc;
+
+	pfn = gvt_cache_find(info->vgpu, gfn);
+	if (pfn != 0)
+		return pfn;
+
+	rc = kvmgt_pin_pages(info->vgpu->vdev.mdev, &gfn, 1,
+			     IOMMU_READ | IOMMU_WRITE, &pfn);
+	if (rc != 1) {
+		gvt_err("vfio_pin_pages failed for gfn: 0x%lx\n", gfn);
+		return 0;
+	}
+
+	gvt_cache_add(info->vgpu, gfn, pfn);
+	return pfn;
+}
+
+static void *kvmgt_gpa_to_hva(unsigned long handle, unsigned long gpa)
+{
+	unsigned long pfn;
+	gfn_t gfn = gpa_to_gfn(gpa);
+
+	pfn = kvmgt_gfn_to_pfn(handle, gfn);
+	if (!pfn)
+		return NULL;
+
+	return (char *)pfn_to_kaddr(pfn) + offset_in_page(gpa);
+}
+
+static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa,
+			void *buf, unsigned long len, bool write)
+{
+	void *hva = NULL;
+
+	hva = kvmgt_gpa_to_hva(handle, gpa);
+	if (!hva)
+		return -EFAULT;
+
+	if (write)
+		memcpy(hva, buf, len);
+	else
+		memcpy(buf, hva, len);
+
+	return 0;
+}
+
+static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa,
+			void *buf, unsigned long len)
+{
+	return kvmgt_rw_gpa(handle, gpa, buf, len, false);
+}
+
+static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa,
+			void *buf, unsigned long len)
+{
+	return kvmgt_rw_gpa(handle, gpa, buf, len, true);
+}
+
+static unsigned long kvmgt_virt_to_pfn(void *addr)
+{
+	return PFN_DOWN(__pa(addr));
+}
+
+struct intel_gvt_mpt kvmgt_mpt = {
+	.detect_host = kvmgt_detect_host,
+	.host_init = kvmgt_host_init,
+	.host_exit = kvmgt_host_exit,
+	.attach_vgpu = kvmgt_attach_vgpu,
+	.detach_vgpu = kvmgt_detach_vgpu,
+	.inject_msi = kvmgt_inject_msi,
+	.from_virt_to_mfn = kvmgt_virt_to_pfn,
+	.set_wp_page = kvmgt_write_protect_add,
+	.unset_wp_page = kvmgt_write_protect_remove,
+	.read_gpa = kvmgt_read_gpa,
+	.write_gpa = kvmgt_write_gpa,
+	.gfn_to_mfn = kvmgt_gfn_to_pfn,
+};
+EXPORT_SYMBOL_GPL(kvmgt_mpt);
+
+static int __init kvmgt_init(void)
+{
+	return 0;
+}
+
+static void __exit kvmgt_exit(void)
+{
+}
+
+module_init(kvmgt_init);
+module_exit(kvmgt_exit);
+
+MODULE_LICENSE("GPL and additional rights");
+MODULE_AUTHOR("Intel Corporation");
diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
new file mode 100644
index 000000000000..09c9450a1946
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/mmio.c
@@ -0,0 +1,304 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Dexuan Cui
+ *
+ * Contributors:
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Min He <min.he@intel.com>
+ *    Niu Bing <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+/**
+ * intel_vgpu_gpa_to_mmio_offset - translate a GPA to MMIO offset
+ * @vgpu: a vGPU
+ *
+ * Returns:
+ * Zero on success, negative error code if failed
+ */
+int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa)
+{
+	u64 gttmmio_gpa = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_0) &
+			  ~GENMASK(3, 0);
+	return gpa - gttmmio_gpa;
+}
+
+#define reg_is_mmio(gvt, reg)  \
+	(reg >= 0 && reg < gvt->device_info.mmio_size)
+
+#define reg_is_gtt(gvt, reg)   \
+	(reg >= gvt->device_info.gtt_start_offset \
+	 && reg < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt))
+
+/**
+ * intel_vgpu_emulate_mmio_read - emulate MMIO read
+ * @vgpu: a vGPU
+ * @pa: guest physical address
+ * @p_data: data return buffer
+ * @bytes: access data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed
+ */
+int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa,
+		void *p_data, unsigned int bytes)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_mmio_info *mmio;
+	unsigned int offset = 0;
+	int ret = -EINVAL;
+
+	mutex_lock(&gvt->lock);
+
+	if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
+		struct intel_vgpu_guest_page *gp;
+
+		gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT);
+		if (gp) {
+			ret = intel_gvt_hypervisor_read_gpa(vgpu, pa,
+					p_data, bytes);
+			if (ret) {
+				gvt_err("vgpu%d: guest page read error %d, "
+					"gfn 0x%lx, pa 0x%llx, var 0x%x, len %d\n",
+					vgpu->id, ret,
+					gp->gfn, pa, *(u32 *)p_data, bytes);
+			}
+			mutex_unlock(&gvt->lock);
+			return ret;
+		}
+	}
+
+	offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa);
+
+	if (WARN_ON(bytes > 8))
+		goto err;
+
+	if (reg_is_gtt(gvt, offset)) {
+		if (WARN_ON(!IS_ALIGNED(offset, 4) && !IS_ALIGNED(offset, 8)))
+			goto err;
+		if (WARN_ON(bytes != 4 && bytes != 8))
+			goto err;
+		if (WARN_ON(!reg_is_gtt(gvt, offset + bytes - 1)))
+			goto err;
+
+		ret = intel_vgpu_emulate_gtt_mmio_read(vgpu, offset,
+				p_data, bytes);
+		if (ret)
+			goto err;
+		mutex_unlock(&gvt->lock);
+		return ret;
+	}
+
+	if (WARN_ON_ONCE(!reg_is_mmio(gvt, offset))) {
+		ret = intel_gvt_hypervisor_read_gpa(vgpu, pa, p_data, bytes);
+		mutex_unlock(&gvt->lock);
+		return ret;
+	}
+
+	if (WARN_ON(!reg_is_mmio(gvt, offset + bytes - 1)))
+		goto err;
+
+	mmio = intel_gvt_find_mmio_info(gvt, rounddown(offset, 4));
+	if (!mmio && !vgpu->mmio.disable_warn_untrack) {
+		gvt_err("vgpu%d: read untracked MMIO %x len %d val %x\n",
+				vgpu->id, offset, bytes, *(u32 *)p_data);
+
+		if (offset == 0x206c) {
+			gvt_err("------------------------------------------\n");
+			gvt_err("vgpu%d: likely triggers a gfx reset\n",
+			vgpu->id);
+			gvt_err("------------------------------------------\n");
+			vgpu->mmio.disable_warn_untrack = true;
+		}
+	}
+
+	if (!intel_gvt_mmio_is_unalign(gvt, offset)) {
+		if (WARN_ON(!IS_ALIGNED(offset, bytes)))
+			goto err;
+	}
+
+	if (mmio) {
+		if (!intel_gvt_mmio_is_unalign(gvt, mmio->offset)) {
+			if (WARN_ON(offset + bytes > mmio->offset + mmio->size))
+				goto err;
+			if (WARN_ON(mmio->offset != offset))
+				goto err;
+		}
+		ret = mmio->read(vgpu, offset, p_data, bytes);
+	} else
+		ret = intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes);
+
+	if (ret)
+		goto err;
+
+	intel_gvt_mmio_set_accessed(gvt, offset);
+	mutex_unlock(&gvt->lock);
+	return 0;
+err:
+	gvt_err("vgpu%d: fail to emulate MMIO read %08x len %d\n",
+			vgpu->id, offset, bytes);
+	mutex_unlock(&gvt->lock);
+	return ret;
+}
+
+/**
+ * intel_vgpu_emulate_mmio_write - emulate MMIO write
+ * @vgpu: a vGPU
+ * @pa: guest physical address
+ * @p_data: write data buffer
+ * @bytes: access data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed
+ */
+int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa,
+		void *p_data, unsigned int bytes)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_mmio_info *mmio;
+	unsigned int offset = 0;
+	u32 old_vreg = 0, old_sreg = 0;
+	int ret = -EINVAL;
+
+	mutex_lock(&gvt->lock);
+
+	if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
+		struct intel_vgpu_guest_page *gp;
+
+		gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT);
+		if (gp) {
+			ret = gp->handler(gp, pa, p_data, bytes);
+			if (ret) {
+				gvt_err("vgpu%d: guest page write error %d, "
+					"gfn 0x%lx, pa 0x%llx, var 0x%x, len %d\n",
+					vgpu->id, ret,
+					gp->gfn, pa, *(u32 *)p_data, bytes);
+			}
+			mutex_unlock(&gvt->lock);
+			return ret;
+		}
+	}
+
+	offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa);
+
+	if (WARN_ON(bytes > 8))
+		goto err;
+
+	if (reg_is_gtt(gvt, offset)) {
+		if (WARN_ON(!IS_ALIGNED(offset, 4) && !IS_ALIGNED(offset, 8)))
+			goto err;
+		if (WARN_ON(bytes != 4 && bytes != 8))
+			goto err;
+		if (WARN_ON(!reg_is_gtt(gvt, offset + bytes - 1)))
+			goto err;
+
+		ret = intel_vgpu_emulate_gtt_mmio_write(vgpu, offset,
+				p_data, bytes);
+		if (ret)
+			goto err;
+		mutex_unlock(&gvt->lock);
+		return ret;
+	}
+
+	if (WARN_ON_ONCE(!reg_is_mmio(gvt, offset))) {
+		ret = intel_gvt_hypervisor_write_gpa(vgpu, pa, p_data, bytes);
+		mutex_unlock(&gvt->lock);
+		return ret;
+	}
+
+	mmio = intel_gvt_find_mmio_info(gvt, rounddown(offset, 4));
+	if (!mmio && !vgpu->mmio.disable_warn_untrack)
+		gvt_err("vgpu%d: write untracked MMIO %x len %d val %x\n",
+				vgpu->id, offset, bytes, *(u32 *)p_data);
+
+	if (!intel_gvt_mmio_is_unalign(gvt, offset)) {
+		if (WARN_ON(!IS_ALIGNED(offset, bytes)))
+			goto err;
+	}
+
+	if (mmio) {
+		u64 ro_mask = mmio->ro_mask;
+
+		if (!intel_gvt_mmio_is_unalign(gvt, mmio->offset)) {
+			if (WARN_ON(offset + bytes > mmio->offset + mmio->size))
+				goto err;
+			if (WARN_ON(mmio->offset != offset))
+				goto err;
+		}
+
+		if (intel_gvt_mmio_has_mode_mask(gvt, mmio->offset)) {
+			old_vreg = vgpu_vreg(vgpu, offset);
+			old_sreg = vgpu_sreg(vgpu, offset);
+		}
+
+		if (!ro_mask) {
+			ret = mmio->write(vgpu, offset, p_data, bytes);
+		} else {
+			/* Protect RO bits like HW */
+			u64 data = 0;
+
+			/* all register bits are RO. */
+			if (ro_mask == ~(u64)0) {
+				gvt_err("vgpu%d: try to write RO reg %x\n",
+						vgpu->id, offset);
+				ret = 0;
+				goto out;
+			}
+			/* keep the RO bits in the virtual register */
+			memcpy(&data, p_data, bytes);
+			data &= ~mmio->ro_mask;
+			data |= vgpu_vreg(vgpu, offset) & mmio->ro_mask;
+			ret = mmio->write(vgpu, offset, &data, bytes);
+		}
+
+		/* higher 16bits of mode ctl regs are mask bits for change */
+		if (intel_gvt_mmio_has_mode_mask(gvt, mmio->offset)) {
+			u32 mask = vgpu_vreg(vgpu, offset) >> 16;
+
+			vgpu_vreg(vgpu, offset) = (old_vreg & ~mask)
+				| (vgpu_vreg(vgpu, offset) & mask);
+			vgpu_sreg(vgpu, offset) = (old_sreg & ~mask)
+				| (vgpu_sreg(vgpu, offset) & mask);
+		}
+	} else
+		ret = intel_vgpu_default_mmio_write(vgpu, offset, p_data,
+				bytes);
+	if (ret)
+		goto err;
+out:
+	intel_gvt_mmio_set_accessed(gvt, offset);
+	mutex_unlock(&gvt->lock);
+	return 0;
+err:
+	gvt_err("vgpu%d: fail to emulate MMIO write %08x len %d\n",
+			vgpu->id, offset, bytes);
+	mutex_unlock(&gvt->lock);
+	return ret;
+}
diff --git a/drivers/gpu/drm/i915/gvt/mmio.h b/drivers/gpu/drm/i915/gvt/mmio.h
new file mode 100644
index 000000000000..87d5b5e366a3
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/mmio.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Ke Yu
+ *    Kevin Tian <kevin.tian@intel.com>
+ *    Dexuan Cui
+ *
+ * Contributors:
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Min He <min.he@intel.com>
+ *    Niu Bing <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#ifndef _GVT_MMIO_H_
+#define _GVT_MMIO_H_
+
+struct intel_gvt;
+struct intel_vgpu;
+
+#define D_SNB   (1 << 0)
+#define D_IVB   (1 << 1)
+#define D_HSW   (1 << 2)
+#define D_BDW   (1 << 3)
+#define D_SKL	(1 << 4)
+
+#define D_GEN9PLUS	(D_SKL)
+#define D_GEN8PLUS	(D_BDW | D_SKL)
+#define D_GEN75PLUS	(D_HSW | D_BDW | D_SKL)
+#define D_GEN7PLUS	(D_IVB | D_HSW | D_BDW | D_SKL)
+
+#define D_SKL_PLUS	(D_SKL)
+#define D_BDW_PLUS	(D_BDW | D_SKL)
+#define D_HSW_PLUS	(D_HSW | D_BDW | D_SKL)
+#define D_IVB_PLUS	(D_IVB | D_HSW | D_BDW | D_SKL)
+
+#define D_PRE_BDW	(D_SNB | D_IVB | D_HSW)
+#define D_PRE_SKL	(D_SNB | D_IVB | D_HSW | D_BDW)
+#define D_ALL		(D_SNB | D_IVB | D_HSW | D_BDW | D_SKL)
+
+struct intel_gvt_mmio_info {
+	u32 offset;
+	u32 size;
+	u32 length;
+	u32 addr_mask;
+	u64 ro_mask;
+	u32 device;
+	int (*read)(struct intel_vgpu *, unsigned int, void *, unsigned int);
+	int (*write)(struct intel_vgpu *, unsigned int, void *, unsigned int);
+	u32 addr_range;
+	struct hlist_node node;
+};
+
+unsigned long intel_gvt_get_device_type(struct intel_gvt *gvt);
+bool intel_gvt_match_device(struct intel_gvt *gvt, unsigned long device);
+
+int intel_gvt_setup_mmio_info(struct intel_gvt *gvt);
+void intel_gvt_clean_mmio_info(struct intel_gvt *gvt);
+
+struct intel_gvt_mmio_info *intel_gvt_find_mmio_info(struct intel_gvt *gvt,
+						     unsigned int offset);
+#define INTEL_GVT_MMIO_OFFSET(reg) ({ \
+	typeof(reg) __reg = reg; \
+	u32 *offset = (u32 *)&__reg; \
+	*offset; \
+})
+
+int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa);
+
+int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, u64 pa,
+				void *p_data, unsigned int bytes);
+int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, u64 pa,
+				void *p_data, unsigned int bytes);
+bool intel_gvt_mmio_is_cmd_access(struct intel_gvt *gvt,
+				  unsigned int offset);
+bool intel_gvt_mmio_is_unalign(struct intel_gvt *gvt, unsigned int offset);
+void intel_gvt_mmio_set_accessed(struct intel_gvt *gvt, unsigned int offset);
+void intel_gvt_mmio_set_cmd_accessed(struct intel_gvt *gvt,
+				     unsigned int offset);
+bool intel_gvt_mmio_has_mode_mask(struct intel_gvt *gvt, unsigned int offset);
+int intel_vgpu_default_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
+				 void *p_data, unsigned int bytes);
+int intel_vgpu_default_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
+				  void *p_data, unsigned int bytes);
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h
index 03601e3ffa7c..1af5830c0a56 100644
--- a/drivers/gpu/drm/i915/gvt/mpt.h
+++ b/drivers/gpu/drm/i915/gvt/mpt.h
@@ -19,6 +19,15 @@
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Dexuan Cui
+ *    Jike Song <jike.song@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
  */
 
 #ifndef _GVT_MPT_H_
@@ -46,4 +55,254 @@ static inline int intel_gvt_hypervisor_detect_host(void)
 	return intel_gvt_host.mpt->detect_host();
 }
 
+/**
+ * intel_gvt_hypervisor_host_init - init GVT-g host side
+ *
+ * Returns:
+ * Zero on success, negative error code if failed
+ */
+static inline int intel_gvt_hypervisor_host_init(struct device *dev,
+			void *gvt, const void *ops)
+{
+	/* optional to provide */
+	if (!intel_gvt_host.mpt->host_init)
+		return 0;
+
+	return intel_gvt_host.mpt->host_init(dev, gvt, ops);
+}
+
+/**
+ * intel_gvt_hypervisor_host_exit - exit GVT-g host side
+ */
+static inline void intel_gvt_hypervisor_host_exit(struct device *dev,
+			void *gvt)
+{
+	/* optional to provide */
+	if (!intel_gvt_host.mpt->host_exit)
+		return;
+
+	intel_gvt_host.mpt->host_exit(dev, gvt);
+}
+
+/**
+ * intel_gvt_hypervisor_attach_vgpu - call hypervisor to initialize vGPU
+ * related stuffs inside hypervisor.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_attach_vgpu(struct intel_vgpu *vgpu)
+{
+	/* optional to provide */
+	if (!intel_gvt_host.mpt->attach_vgpu)
+		return 0;
+
+	return intel_gvt_host.mpt->attach_vgpu(vgpu, &vgpu->handle);
+}
+
+/**
+ * intel_gvt_hypervisor_detach_vgpu - call hypervisor to release vGPU
+ * related stuffs inside hypervisor.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline void intel_gvt_hypervisor_detach_vgpu(struct intel_vgpu *vgpu)
+{
+	/* optional to provide */
+	if (!intel_gvt_host.mpt->detach_vgpu)
+		return;
+
+	intel_gvt_host.mpt->detach_vgpu(vgpu->handle);
+}
+
+#define MSI_CAP_CONTROL(offset) (offset + 2)
+#define MSI_CAP_ADDRESS(offset) (offset + 4)
+#define MSI_CAP_DATA(offset) (offset + 8)
+#define MSI_CAP_EN 0x1
+
+/**
+ * intel_gvt_hypervisor_inject_msi - inject a MSI interrupt into vGPU
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_inject_msi(struct intel_vgpu *vgpu)
+{
+	unsigned long offset = vgpu->gvt->device_info.msi_cap_offset;
+	u16 control, data;
+	u32 addr;
+	int ret;
+
+	control = *(u16 *)(vgpu_cfg_space(vgpu) + MSI_CAP_CONTROL(offset));
+	addr = *(u32 *)(vgpu_cfg_space(vgpu) + MSI_CAP_ADDRESS(offset));
+	data = *(u16 *)(vgpu_cfg_space(vgpu) + MSI_CAP_DATA(offset));
+
+	/* Do not generate MSI if MSIEN is disable */
+	if (!(control & MSI_CAP_EN))
+		return 0;
+
+	if (WARN(control & GENMASK(15, 1), "only support one MSI format\n"))
+		return -EINVAL;
+
+	gvt_dbg_irq("vgpu%d: inject msi address %x data%x\n", vgpu->id, addr,
+		    data);
+
+	ret = intel_gvt_host.mpt->inject_msi(vgpu->handle, addr, data);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+/**
+ * intel_gvt_hypervisor_set_wp_page - translate a host VA into MFN
+ * @p: host kernel virtual address
+ *
+ * Returns:
+ * MFN on success, INTEL_GVT_INVALID_ADDR if failed.
+ */
+static inline unsigned long intel_gvt_hypervisor_virt_to_mfn(void *p)
+{
+	return intel_gvt_host.mpt->from_virt_to_mfn(p);
+}
+
+/**
+ * intel_gvt_hypervisor_set_wp_page - set a guest page to write-protected
+ * @vgpu: a vGPU
+ * @p: intel_vgpu_guest_page
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_set_wp_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *p)
+{
+	int ret;
+
+	if (p->writeprotection)
+		return 0;
+
+	ret = intel_gvt_host.mpt->set_wp_page(vgpu->handle, p->gfn);
+	if (ret)
+		return ret;
+	p->writeprotection = true;
+	atomic_inc(&vgpu->gtt.n_write_protected_guest_page);
+	return 0;
+}
+
+/**
+ * intel_gvt_hypervisor_unset_wp_page - remove the write-protection of a
+ * guest page
+ * @vgpu: a vGPU
+ * @p: intel_vgpu_guest_page
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_unset_wp_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *p)
+{
+	int ret;
+
+	if (!p->writeprotection)
+		return 0;
+
+	ret = intel_gvt_host.mpt->unset_wp_page(vgpu->handle, p->gfn);
+	if (ret)
+		return ret;
+	p->writeprotection = false;
+	atomic_dec(&vgpu->gtt.n_write_protected_guest_page);
+	return 0;
+}
+
+/**
+ * intel_gvt_hypervisor_read_gpa - copy data from GPA to host data buffer
+ * @vgpu: a vGPU
+ * @gpa: guest physical address
+ * @buf: host data buffer
+ * @len: data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_read_gpa(struct intel_vgpu *vgpu,
+		unsigned long gpa, void *buf, unsigned long len)
+{
+	return intel_gvt_host.mpt->read_gpa(vgpu->handle, gpa, buf, len);
+}
+
+/**
+ * intel_gvt_hypervisor_write_gpa - copy data from host data buffer to GPA
+ * @vgpu: a vGPU
+ * @gpa: guest physical address
+ * @buf: host data buffer
+ * @len: data length
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_write_gpa(struct intel_vgpu *vgpu,
+		unsigned long gpa, void *buf, unsigned long len)
+{
+	return intel_gvt_host.mpt->write_gpa(vgpu->handle, gpa, buf, len);
+}
+
+/**
+ * intel_gvt_hypervisor_gfn_to_mfn - translate a GFN to MFN
+ * @vgpu: a vGPU
+ * @gpfn: guest pfn
+ *
+ * Returns:
+ * MFN on success, INTEL_GVT_INVALID_ADDR if failed.
+ */
+static inline unsigned long intel_gvt_hypervisor_gfn_to_mfn(
+		struct intel_vgpu *vgpu, unsigned long gfn)
+{
+	return intel_gvt_host.mpt->gfn_to_mfn(vgpu->handle, gfn);
+}
+
+/**
+ * intel_gvt_hypervisor_map_gfn_to_mfn - map a GFN region to MFN
+ * @vgpu: a vGPU
+ * @gfn: guest PFN
+ * @mfn: host PFN
+ * @nr: amount of PFNs
+ * @map: map or unmap
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_map_gfn_to_mfn(
+		struct intel_vgpu *vgpu, unsigned long gfn,
+		unsigned long mfn, unsigned int nr,
+		bool map)
+{
+	/* a MPT implementation could have MMIO mapped elsewhere */
+	if (!intel_gvt_host.mpt->map_gfn_to_mfn)
+		return 0;
+
+	return intel_gvt_host.mpt->map_gfn_to_mfn(vgpu->handle, gfn, mfn, nr,
+						  map);
+}
+
+/**
+ * intel_gvt_hypervisor_set_trap_area - Trap a guest PA region
+ * @vgpu: a vGPU
+ * @start: the beginning of the guest physical address region
+ * @end: the end of the guest physical address region
+ * @map: map or unmap
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_set_trap_area(
+		struct intel_vgpu *vgpu, u64 start, u64 end, bool map)
+{
+	/* a MPT implementation could have MMIO trapped elsewhere */
+	if (!intel_gvt_host.mpt->set_trap_area)
+		return 0;
+
+	return intel_gvt_host.mpt->set_trap_area(vgpu->handle, start, end, map);
+}
+
 #endif /* _GVT_MPT_H_ */
diff --git a/drivers/gpu/drm/i915/gvt/opregion.c b/drivers/gpu/drm/i915/gvt/opregion.c
new file mode 100644
index 000000000000..d2a0fbc896c3
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/opregion.c
@@ -0,0 +1,320 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/acpi.h>
+#include "i915_drv.h"
+#include "gvt.h"
+
+static int init_vgpu_opregion(struct intel_vgpu *vgpu, u32 gpa)
+{
+	void __iomem *host_va = vgpu->gvt->opregion.opregion_va;
+	u8 *buf;
+	int i;
+
+	if (WARN((vgpu_opregion(vgpu)->va),
+			"vgpu%d: opregion has been initialized already.\n",
+			vgpu->id))
+		return -EINVAL;
+
+	vgpu_opregion(vgpu)->va = (void *)__get_free_pages(GFP_ATOMIC |
+			GFP_DMA32 | __GFP_ZERO,
+			INTEL_GVT_OPREGION_PORDER);
+
+	if (!vgpu_opregion(vgpu)->va)
+		return -ENOMEM;
+
+	memcpy_fromio(vgpu_opregion(vgpu)->va, host_va,
+			INTEL_GVT_OPREGION_SIZE);
+
+	for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++)
+		vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i;
+
+	/* for unknown reason, the value in LID field is incorrect
+	 * which block the windows guest, so workaround it by force
+	 * setting it to "OPEN"
+	 */
+	buf = (u8 *)vgpu_opregion(vgpu)->va;
+	buf[INTEL_GVT_OPREGION_CLID] = 0x3;
+
+	return 0;
+}
+
+static int map_vgpu_opregion(struct intel_vgpu *vgpu, bool map)
+{
+	u64 mfn;
+	int i, ret;
+
+	for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++) {
+		mfn = intel_gvt_hypervisor_virt_to_mfn(vgpu_opregion(vgpu)
+			+ i * PAGE_SIZE);
+		if (mfn == INTEL_GVT_INVALID_ADDR) {
+			gvt_err("fail to get MFN from VA\n");
+			return -EINVAL;
+		}
+		ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu,
+				vgpu_opregion(vgpu)->gfn[i],
+				mfn, 1, map);
+		if (ret) {
+			gvt_err("fail to map GFN to MFN, errno: %d\n", ret);
+			return ret;
+		}
+	}
+	return 0;
+}
+
+/**
+ * intel_vgpu_clean_opregion - clean the stuff used to emulate opregion
+ * @vgpu: a vGPU
+ *
+ */
+void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu)
+{
+	gvt_dbg_core("vgpu%d: clean vgpu opregion\n", vgpu->id);
+
+	if (!vgpu_opregion(vgpu)->va)
+		return;
+
+	if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_XEN) {
+		map_vgpu_opregion(vgpu, false);
+		free_pages((unsigned long)vgpu_opregion(vgpu)->va,
+				INTEL_GVT_OPREGION_PORDER);
+
+		vgpu_opregion(vgpu)->va = NULL;
+	}
+}
+
+/**
+ * intel_vgpu_init_opregion - initialize the stuff used to emulate opregion
+ * @vgpu: a vGPU
+ * @gpa: guest physical address of opregion
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa)
+{
+	int ret;
+
+	gvt_dbg_core("vgpu%d: init vgpu opregion\n", vgpu->id);
+
+	if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_XEN) {
+		gvt_dbg_core("emulate opregion from kernel\n");
+
+		ret = init_vgpu_opregion(vgpu, gpa);
+		if (ret)
+			return ret;
+
+		ret = map_vgpu_opregion(vgpu, true);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * intel_gvt_clean_opregion - clean host opergion related stuffs
+ * @gvt: a GVT device
+ *
+ */
+void intel_gvt_clean_opregion(struct intel_gvt *gvt)
+{
+	memunmap(gvt->opregion.opregion_va);
+	gvt->opregion.opregion_va = NULL;
+}
+
+/**
+ * intel_gvt_init_opregion - initialize host opergion related stuffs
+ * @gvt: a GVT device
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_gvt_init_opregion(struct intel_gvt *gvt)
+{
+	gvt_dbg_core("init host opregion\n");
+
+	pci_read_config_dword(gvt->dev_priv->drm.pdev, INTEL_GVT_PCI_OPREGION,
+			&gvt->opregion.opregion_pa);
+
+	gvt->opregion.opregion_va = memremap(gvt->opregion.opregion_pa,
+					     INTEL_GVT_OPREGION_SIZE, MEMREMAP_WB);
+	if (!gvt->opregion.opregion_va) {
+		gvt_err("fail to map host opregion\n");
+		return -EFAULT;
+	}
+	return 0;
+}
+
+#define GVT_OPREGION_FUNC(scic)					\
+	({							\
+	 u32 __ret;						\
+	 __ret = (scic & OPREGION_SCIC_FUNC_MASK) >>		\
+	 OPREGION_SCIC_FUNC_SHIFT;				\
+	 __ret;							\
+	 })
+
+#define GVT_OPREGION_SUBFUNC(scic)				\
+	({							\
+	 u32 __ret;						\
+	 __ret = (scic & OPREGION_SCIC_SUBFUNC_MASK) >>		\
+	 OPREGION_SCIC_SUBFUNC_SHIFT;				\
+	 __ret;							\
+	 })
+
+static const char *opregion_func_name(u32 func)
+{
+	const char *name = NULL;
+
+	switch (func) {
+	case 0 ... 3:
+	case 5:
+	case 7 ... 15:
+		name = "Reserved";
+		break;
+
+	case 4:
+		name = "Get BIOS Data";
+		break;
+
+	case 6:
+		name = "System BIOS Callbacks";
+		break;
+
+	default:
+		name = "Unknown";
+		break;
+	}
+	return name;
+}
+
+static const char *opregion_subfunc_name(u32 subfunc)
+{
+	const char *name = NULL;
+
+	switch (subfunc) {
+	case 0:
+		name = "Supported Calls";
+		break;
+
+	case 1:
+		name = "Requested Callbacks";
+		break;
+
+	case 2 ... 3:
+	case 8 ... 9:
+		name = "Reserved";
+		break;
+
+	case 5:
+		name = "Boot Display";
+		break;
+
+	case 6:
+		name = "TV-Standard/Video-Connector";
+		break;
+
+	case 7:
+		name = "Internal Graphics";
+		break;
+
+	case 10:
+		name = "Spread Spectrum Clocks";
+		break;
+
+	case 11:
+		name = "Get AKSV";
+		break;
+
+	default:
+		name = "Unknown";
+		break;
+	}
+	return name;
+};
+
+static bool querying_capabilities(u32 scic)
+{
+	u32 func, subfunc;
+
+	func = GVT_OPREGION_FUNC(scic);
+	subfunc = GVT_OPREGION_SUBFUNC(scic);
+
+	if ((func == INTEL_GVT_OPREGION_SCIC_F_GETBIOSDATA &&
+		subfunc == INTEL_GVT_OPREGION_SCIC_SF_SUPPRTEDCALLS)
+		|| (func == INTEL_GVT_OPREGION_SCIC_F_GETBIOSDATA &&
+		 subfunc == INTEL_GVT_OPREGION_SCIC_SF_REQEUSTEDCALLBACKS)
+		|| (func == INTEL_GVT_OPREGION_SCIC_F_GETBIOSCALLBACKS &&
+		 subfunc == INTEL_GVT_OPREGION_SCIC_SF_SUPPRTEDCALLS)) {
+		return true;
+	}
+	return false;
+}
+
+/**
+ * intel_vgpu_emulate_opregion_request - emulating OpRegion request
+ * @vgpu: a vGPU
+ * @swsci: SWSCI request
+ *
+ * Returns:
+ * Zero on success, negative error code if failed
+ */
+int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci)
+{
+	u32 *scic, *parm;
+	u32 func, subfunc;
+
+	scic = vgpu_opregion(vgpu)->va + INTEL_GVT_OPREGION_SCIC;
+	parm = vgpu_opregion(vgpu)->va + INTEL_GVT_OPREGION_PARM;
+
+	if (!(swsci & SWSCI_SCI_SELECT)) {
+		gvt_err("vgpu%d: requesting SMI service\n", vgpu->id);
+		return 0;
+	}
+	/* ignore non 0->1 trasitions */
+	if ((vgpu_cfg_space(vgpu)[INTEL_GVT_PCI_SWSCI]
+				& SWSCI_SCI_TRIGGER) ||
+			!(swsci & SWSCI_SCI_TRIGGER)) {
+		return 0;
+	}
+
+	func = GVT_OPREGION_FUNC(*scic);
+	subfunc = GVT_OPREGION_SUBFUNC(*scic);
+	if (!querying_capabilities(*scic)) {
+		gvt_err("vgpu%d: requesting runtime service: func \"%s\","
+				" subfunc \"%s\"\n",
+				vgpu->id,
+				opregion_func_name(func),
+				opregion_subfunc_name(subfunc));
+		/*
+		 * emulate exit status of function call, '0' means
+		 * "failure, generic, unsupported or unknown cause"
+		 */
+		*scic &= ~OPREGION_SCIC_EXIT_MASK;
+		return 0;
+	}
+
+	*scic = 0;
+	*parm = 0;
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/reg.h b/drivers/gpu/drm/i915/gvt/reg.h
new file mode 100644
index 000000000000..0dfe789d8f02
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/reg.h
@@ -0,0 +1,80 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _GVT_REG_H
+#define _GVT_REG_H
+
+#define INTEL_GVT_PCI_CLASS_VGA_OTHER   0x80
+
+#define INTEL_GVT_PCI_GMCH_CONTROL	0x50
+#define   BDW_GMCH_GMS_SHIFT		8
+#define   BDW_GMCH_GMS_MASK		0xff
+
+#define INTEL_GVT_PCI_SWSCI		0xe8
+#define   SWSCI_SCI_SELECT		(1 << 15)
+#define   SWSCI_SCI_TRIGGER		1
+
+#define INTEL_GVT_PCI_OPREGION		0xfc
+
+#define INTEL_GVT_OPREGION_CLID		0x1AC
+#define INTEL_GVT_OPREGION_SCIC		0x200
+#define   OPREGION_SCIC_FUNC_MASK	0x1E
+#define   OPREGION_SCIC_FUNC_SHIFT	1
+#define   OPREGION_SCIC_SUBFUNC_MASK	0xFF00
+#define   OPREGION_SCIC_SUBFUNC_SHIFT	8
+#define   OPREGION_SCIC_EXIT_MASK	0xE0
+#define INTEL_GVT_OPREGION_SCIC_F_GETBIOSDATA         4
+#define INTEL_GVT_OPREGION_SCIC_F_GETBIOSCALLBACKS    6
+#define INTEL_GVT_OPREGION_SCIC_SF_SUPPRTEDCALLS      0
+#define INTEL_GVT_OPREGION_SCIC_SF_REQEUSTEDCALLBACKS 1
+#define INTEL_GVT_OPREGION_PARM                   0x204
+
+#define INTEL_GVT_OPREGION_PAGES	2
+#define INTEL_GVT_OPREGION_PORDER	1
+#define INTEL_GVT_OPREGION_SIZE		(2 * 4096)
+
+#define VGT_SPRSTRIDE(pipe)	_PIPE(pipe, _SPRA_STRIDE, _PLANE_STRIDE_2_B)
+
+#define _REG_VECS_EXCC		0x1A028
+#define _REG_VCS2_EXCC		0x1c028
+
+#define _REG_701C0(pipe, plane) (0x701c0 + pipe * 0x1000 + (plane - 1) * 0x100)
+#define _REG_701C4(pipe, plane) (0x701c4 + pipe * 0x1000 + (plane - 1) * 0x100)
+
+#define GFX_MODE_BIT_SET_IN_MASK(val, bit) \
+		((((bit) & 0xffff0000) == 0) && !!((val) & (((bit) << 16))))
+
+#define FORCEWAKE_RENDER_GEN9_REG 0xa278
+#define FORCEWAKE_ACK_RENDER_GEN9_REG 0x0D84
+#define FORCEWAKE_BLITTER_GEN9_REG 0xa188
+#define FORCEWAKE_ACK_BLITTER_GEN9_REG 0x130044
+#define FORCEWAKE_MEDIA_GEN9_REG 0xa270
+#define FORCEWAKE_ACK_MEDIA_GEN9_REG 0x0D88
+#define FORCEWAKE_ACK_HSW_REG 0x130044
+
+#define RB_HEAD_OFF_MASK	((1U << 21) - (1U << 2))
+#define RB_TAIL_OFF_MASK	((1U << 21) - (1U << 3))
+#define RB_TAIL_SIZE_MASK	((1U << 21) - (1U << 12))
+#define _RING_CTL_BUF_SIZE(ctl) (((ctl) & RB_TAIL_SIZE_MASK) + GTT_PAGE_SIZE)
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/render.c b/drivers/gpu/drm/i915/gvt/render.c
new file mode 100644
index 000000000000..44136b1f3aab
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/render.c
@@ -0,0 +1,310 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+struct render_mmio {
+	int ring_id;
+	i915_reg_t reg;
+	u32 mask;
+	bool in_context;
+	u32 value;
+};
+
+static struct render_mmio gen8_render_mmio_list[] = {
+	{RCS, _MMIO(0x229c), 0xffff, false},
+	{RCS, _MMIO(0x2248), 0x0, false},
+	{RCS, _MMIO(0x2098), 0x0, false},
+	{RCS, _MMIO(0x20c0), 0xffff, true},
+	{RCS, _MMIO(0x24d0), 0, false},
+	{RCS, _MMIO(0x24d4), 0, false},
+	{RCS, _MMIO(0x24d8), 0, false},
+	{RCS, _MMIO(0x24dc), 0, false},
+	{RCS, _MMIO(0x7004), 0xffff, true},
+	{RCS, _MMIO(0x7008), 0xffff, true},
+	{RCS, _MMIO(0x7000), 0xffff, true},
+	{RCS, _MMIO(0x7010), 0xffff, true},
+	{RCS, _MMIO(0x7300), 0xffff, true},
+	{RCS, _MMIO(0x83a4), 0xffff, true},
+
+	{BCS, _MMIO(0x2229c), 0xffff, false},
+	{BCS, _MMIO(0x2209c), 0xffff, false},
+	{BCS, _MMIO(0x220c0), 0xffff, false},
+	{BCS, _MMIO(0x22098), 0x0, false},
+	{BCS, _MMIO(0x22028), 0x0, false},
+};
+
+static struct render_mmio gen9_render_mmio_list[] = {
+	{RCS, _MMIO(0x229c), 0xffff, false},
+	{RCS, _MMIO(0x2248), 0x0, false},
+	{RCS, _MMIO(0x2098), 0x0, false},
+	{RCS, _MMIO(0x20c0), 0xffff, true},
+	{RCS, _MMIO(0x24d0), 0, false},
+	{RCS, _MMIO(0x24d4), 0, false},
+	{RCS, _MMIO(0x24d8), 0, false},
+	{RCS, _MMIO(0x24dc), 0, false},
+	{RCS, _MMIO(0x7004), 0xffff, true},
+	{RCS, _MMIO(0x7008), 0xffff, true},
+	{RCS, _MMIO(0x7000), 0xffff, true},
+	{RCS, _MMIO(0x7010), 0xffff, true},
+	{RCS, _MMIO(0x7300), 0xffff, true},
+	{RCS, _MMIO(0x83a4), 0xffff, true},
+
+	{RCS, _MMIO(0x40e0), 0, false},
+	{RCS, _MMIO(0x40e4), 0, false},
+	{RCS, _MMIO(0x2580), 0xffff, true},
+	{RCS, _MMIO(0x7014), 0xffff, true},
+	{RCS, _MMIO(0x20ec), 0xffff, false},
+	{RCS, _MMIO(0xb118), 0, false},
+	{RCS, _MMIO(0xe100), 0xffff, true},
+	{RCS, _MMIO(0xe180), 0xffff, true},
+	{RCS, _MMIO(0xe184), 0xffff, true},
+	{RCS, _MMIO(0xe188), 0xffff, true},
+	{RCS, _MMIO(0xe194), 0xffff, true},
+	{RCS, _MMIO(0x4de0), 0, false},
+	{RCS, _MMIO(0x4de4), 0, false},
+	{RCS, _MMIO(0x4de8), 0, false},
+	{RCS, _MMIO(0x4dec), 0, false},
+	{RCS, _MMIO(0x4df0), 0, false},
+	{RCS, _MMIO(0x4df4), 0, false},
+
+	{BCS, _MMIO(0x2229c), 0xffff, false},
+	{BCS, _MMIO(0x2209c), 0xffff, false},
+	{BCS, _MMIO(0x220c0), 0xffff, false},
+	{BCS, _MMIO(0x22098), 0x0, false},
+	{BCS, _MMIO(0x22028), 0x0, false},
+
+	{VCS2, _MMIO(0x1c028), 0xffff, false},
+
+	{VECS, _MMIO(0x1a028), 0xffff, false},
+};
+
+static u32 gen9_render_mocs[I915_NUM_ENGINES][64];
+static u32 gen9_render_mocs_L3[32];
+
+static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	enum forcewake_domains fw;
+	i915_reg_t reg;
+	u32 regs[] = {
+		[RCS] = 0x4260,
+		[VCS] = 0x4264,
+		[VCS2] = 0x4268,
+		[BCS] = 0x426c,
+		[VECS] = 0x4270,
+	};
+
+	if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
+		return;
+
+	if (!test_and_clear_bit(ring_id, (void *)vgpu->tlb_handle_pending))
+		return;
+
+	reg = _MMIO(regs[ring_id]);
+
+	/* WaForceWakeRenderDuringMmioTLBInvalidate:skl
+	 * we need to put a forcewake when invalidating RCS TLB caches,
+	 * otherwise device can go to RC6 state and interrupt invalidation
+	 * process
+	 */
+	fw = intel_uncore_forcewake_for_reg(dev_priv, reg,
+					    FW_REG_READ | FW_REG_WRITE);
+	if (ring_id == RCS && IS_SKYLAKE(dev_priv))
+		fw |= FORCEWAKE_RENDER;
+
+	intel_uncore_forcewake_get(dev_priv, fw);
+
+	I915_WRITE_FW(reg, 0x1);
+
+	if (wait_for_atomic((I915_READ_FW(reg) == 0), 50))
+		gvt_err("timeout in invalidate ring (%d) tlb\n", ring_id);
+	else
+		vgpu_vreg(vgpu, regs[ring_id]) = 0;
+
+	intel_uncore_forcewake_put(dev_priv, fw);
+
+	gvt_dbg_core("invalidate TLB for ring %d\n", ring_id);
+}
+
+static void load_mocs(struct intel_vgpu *vgpu, int ring_id)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	i915_reg_t offset, l3_offset;
+	u32 regs[] = {
+		[RCS] = 0xc800,
+		[VCS] = 0xc900,
+		[VCS2] = 0xca00,
+		[BCS] = 0xcc00,
+		[VECS] = 0xcb00,
+	};
+	int i;
+
+	if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
+		return;
+
+	if (!IS_SKYLAKE(dev_priv))
+		return;
+
+	offset.reg = regs[ring_id];
+	for (i = 0; i < 64; i++) {
+		gen9_render_mocs[ring_id][i] = I915_READ(offset);
+		I915_WRITE(offset, vgpu_vreg(vgpu, offset));
+		POSTING_READ(offset);
+		offset.reg += 4;
+	}
+
+	if (ring_id == RCS) {
+		l3_offset.reg = 0xb020;
+		for (i = 0; i < 32; i++) {
+			gen9_render_mocs_L3[i] = I915_READ(l3_offset);
+			I915_WRITE(l3_offset, vgpu_vreg(vgpu, offset));
+			POSTING_READ(l3_offset);
+			l3_offset.reg += 4;
+		}
+	}
+}
+
+static void restore_mocs(struct intel_vgpu *vgpu, int ring_id)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	i915_reg_t offset, l3_offset;
+	u32 regs[] = {
+		[RCS] = 0xc800,
+		[VCS] = 0xc900,
+		[VCS2] = 0xca00,
+		[BCS] = 0xcc00,
+		[VECS] = 0xcb00,
+	};
+	int i;
+
+	if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
+		return;
+
+	if (!IS_SKYLAKE(dev_priv))
+		return;
+
+	offset.reg = regs[ring_id];
+	for (i = 0; i < 64; i++) {
+		vgpu_vreg(vgpu, offset) = I915_READ(offset);
+		I915_WRITE(offset, gen9_render_mocs[ring_id][i]);
+		POSTING_READ(offset);
+		offset.reg += 4;
+	}
+
+	if (ring_id == RCS) {
+		l3_offset.reg = 0xb020;
+		for (i = 0; i < 32; i++) {
+			vgpu_vreg(vgpu, l3_offset) = I915_READ(l3_offset);
+			I915_WRITE(l3_offset, gen9_render_mocs_L3[i]);
+			POSTING_READ(l3_offset);
+			l3_offset.reg += 4;
+		}
+	}
+}
+
+void intel_gvt_load_render_mmio(struct intel_vgpu *vgpu, int ring_id)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	struct render_mmio *mmio;
+	u32 v;
+	int i, array_size;
+
+	if (IS_SKYLAKE(vgpu->gvt->dev_priv)) {
+		mmio = gen9_render_mmio_list;
+		array_size = ARRAY_SIZE(gen9_render_mmio_list);
+		load_mocs(vgpu, ring_id);
+	} else {
+		mmio = gen8_render_mmio_list;
+		array_size = ARRAY_SIZE(gen8_render_mmio_list);
+	}
+
+	for (i = 0; i < array_size; i++, mmio++) {
+		if (mmio->ring_id != ring_id)
+			continue;
+
+		mmio->value = I915_READ(mmio->reg);
+		if (mmio->mask)
+			v = vgpu_vreg(vgpu, mmio->reg) | (mmio->mask << 16);
+		else
+			v = vgpu_vreg(vgpu, mmio->reg);
+
+		I915_WRITE(mmio->reg, v);
+		POSTING_READ(mmio->reg);
+
+		gvt_dbg_render("load reg %x old %x new %x\n",
+				i915_mmio_reg_offset(mmio->reg),
+				mmio->value, v);
+	}
+	handle_tlb_pending_event(vgpu, ring_id);
+}
+
+void intel_gvt_restore_render_mmio(struct intel_vgpu *vgpu, int ring_id)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	struct render_mmio *mmio;
+	u32 v;
+	int i, array_size;
+
+	if (IS_SKYLAKE(dev_priv)) {
+		mmio = gen9_render_mmio_list;
+		array_size = ARRAY_SIZE(gen9_render_mmio_list);
+		restore_mocs(vgpu, ring_id);
+	} else {
+		mmio = gen8_render_mmio_list;
+		array_size = ARRAY_SIZE(gen8_render_mmio_list);
+	}
+
+	for (i = 0; i < array_size; i++, mmio++) {
+		if (mmio->ring_id != ring_id)
+			continue;
+
+		vgpu_vreg(vgpu, mmio->reg) = I915_READ(mmio->reg);
+
+		if (mmio->mask) {
+			vgpu_vreg(vgpu, mmio->reg) &= ~(mmio->mask << 16);
+			v = mmio->value | (mmio->mask << 16);
+		} else
+			v = mmio->value;
+
+		I915_WRITE(mmio->reg, v);
+		POSTING_READ(mmio->reg);
+
+		gvt_dbg_render("restore reg %x old %x new %x\n",
+				i915_mmio_reg_offset(mmio->reg),
+				mmio->value, v);
+	}
+}
diff --git a/drivers/gpu/drm/i915/gvt/render.h b/drivers/gpu/drm/i915/gvt/render.h
new file mode 100644
index 000000000000..dac1a3cc458b
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/render.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#ifndef __GVT_RENDER_H__
+#define __GVT_RENDER_H__
+
+void intel_gvt_load_render_mmio(struct intel_vgpu *vgpu, int ring_id);
+
+void intel_gvt_restore_render_mmio(struct intel_vgpu *vgpu, int ring_id);
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.c b/drivers/gpu/drm/i915/gvt/sched_policy.c
new file mode 100644
index 000000000000..678b0be85376
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/sched_policy.c
@@ -0,0 +1,292 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Anhua Xu
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+static bool vgpu_has_pending_workload(struct intel_vgpu *vgpu)
+{
+	enum intel_engine_id i;
+	struct intel_engine_cs *engine;
+
+	for_each_engine(engine, vgpu->gvt->dev_priv, i) {
+		if (!list_empty(workload_q_head(vgpu, i)))
+			return true;
+	}
+
+	return false;
+}
+
+static void try_to_schedule_next_vgpu(struct intel_gvt *gvt)
+{
+	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+	enum intel_engine_id i;
+	struct intel_engine_cs *engine;
+
+	/* no target to schedule */
+	if (!scheduler->next_vgpu)
+		return;
+
+	gvt_dbg_sched("try to schedule next vgpu %d\n",
+			scheduler->next_vgpu->id);
+
+	/*
+	 * after the flag is set, workload dispatch thread will
+	 * stop dispatching workload for current vgpu
+	 */
+	scheduler->need_reschedule = true;
+
+	/* still have uncompleted workload? */
+	for_each_engine(engine, gvt->dev_priv, i) {
+		if (scheduler->current_workload[i]) {
+			gvt_dbg_sched("still have running workload\n");
+			return;
+		}
+	}
+
+	gvt_dbg_sched("switch to next vgpu %d\n",
+			scheduler->next_vgpu->id);
+
+	/* switch current vgpu */
+	scheduler->current_vgpu = scheduler->next_vgpu;
+	scheduler->next_vgpu = NULL;
+
+	scheduler->need_reschedule = false;
+
+	/* wake up workload dispatch thread */
+	for_each_engine(engine, gvt->dev_priv, i)
+		wake_up(&scheduler->waitq[i]);
+}
+
+struct tbs_vgpu_data {
+	struct list_head list;
+	struct intel_vgpu *vgpu;
+	/* put some per-vgpu sched stats here */
+};
+
+struct tbs_sched_data {
+	struct intel_gvt *gvt;
+	struct delayed_work work;
+	unsigned long period;
+	struct list_head runq_head;
+};
+
+#define GVT_DEFAULT_TIME_SLICE (1 * HZ / 1000)
+
+static void tbs_sched_func(struct work_struct *work)
+{
+	struct tbs_sched_data *sched_data = container_of(work,
+			struct tbs_sched_data, work.work);
+	struct tbs_vgpu_data *vgpu_data;
+
+	struct intel_gvt *gvt = sched_data->gvt;
+	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+
+	struct intel_vgpu *vgpu = NULL;
+	struct list_head *pos, *head;
+
+	mutex_lock(&gvt->lock);
+
+	/* no vgpu or has already had a target */
+	if (list_empty(&sched_data->runq_head) || scheduler->next_vgpu)
+		goto out;
+
+	if (scheduler->current_vgpu) {
+		vgpu_data = scheduler->current_vgpu->sched_data;
+		head = &vgpu_data->list;
+	} else {
+		gvt_dbg_sched("no current vgpu search from q head\n");
+		head = &sched_data->runq_head;
+	}
+
+	/* search a vgpu with pending workload */
+	list_for_each(pos, head) {
+		if (pos == &sched_data->runq_head)
+			continue;
+
+		vgpu_data = container_of(pos, struct tbs_vgpu_data, list);
+		if (!vgpu_has_pending_workload(vgpu_data->vgpu))
+			continue;
+
+		vgpu = vgpu_data->vgpu;
+		break;
+	}
+
+	if (vgpu) {
+		scheduler->next_vgpu = vgpu;
+		gvt_dbg_sched("pick next vgpu %d\n", vgpu->id);
+	}
+out:
+	if (scheduler->next_vgpu) {
+		gvt_dbg_sched("try to schedule next vgpu %d\n",
+				scheduler->next_vgpu->id);
+		try_to_schedule_next_vgpu(gvt);
+	}
+
+	/*
+	 * still have vgpu on runq
+	 * or last schedule haven't finished due to running workload
+	 */
+	if (!list_empty(&sched_data->runq_head) || scheduler->next_vgpu)
+		schedule_delayed_work(&sched_data->work, sched_data->period);
+
+	mutex_unlock(&gvt->lock);
+}
+
+static int tbs_sched_init(struct intel_gvt *gvt)
+{
+	struct intel_gvt_workload_scheduler *scheduler =
+		&gvt->scheduler;
+
+	struct tbs_sched_data *data;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&data->runq_head);
+	INIT_DELAYED_WORK(&data->work, tbs_sched_func);
+	data->period = GVT_DEFAULT_TIME_SLICE;
+	data->gvt = gvt;
+
+	scheduler->sched_data = data;
+	return 0;
+}
+
+static void tbs_sched_clean(struct intel_gvt *gvt)
+{
+	struct intel_gvt_workload_scheduler *scheduler =
+		&gvt->scheduler;
+	struct tbs_sched_data *data = scheduler->sched_data;
+
+	cancel_delayed_work(&data->work);
+	kfree(data);
+	scheduler->sched_data = NULL;
+}
+
+static int tbs_sched_init_vgpu(struct intel_vgpu *vgpu)
+{
+	struct tbs_vgpu_data *data;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->vgpu = vgpu;
+	INIT_LIST_HEAD(&data->list);
+
+	vgpu->sched_data = data;
+	return 0;
+}
+
+static void tbs_sched_clean_vgpu(struct intel_vgpu *vgpu)
+{
+	kfree(vgpu->sched_data);
+	vgpu->sched_data = NULL;
+}
+
+static void tbs_sched_start_schedule(struct intel_vgpu *vgpu)
+{
+	struct tbs_sched_data *sched_data = vgpu->gvt->scheduler.sched_data;
+	struct tbs_vgpu_data *vgpu_data = vgpu->sched_data;
+
+	if (!list_empty(&vgpu_data->list))
+		return;
+
+	list_add_tail(&vgpu_data->list, &sched_data->runq_head);
+	schedule_delayed_work(&sched_data->work, sched_data->period);
+}
+
+static void tbs_sched_stop_schedule(struct intel_vgpu *vgpu)
+{
+	struct tbs_vgpu_data *vgpu_data = vgpu->sched_data;
+
+	list_del_init(&vgpu_data->list);
+}
+
+static struct intel_gvt_sched_policy_ops tbs_schedule_ops = {
+	.init = tbs_sched_init,
+	.clean = tbs_sched_clean,
+	.init_vgpu = tbs_sched_init_vgpu,
+	.clean_vgpu = tbs_sched_clean_vgpu,
+	.start_schedule = tbs_sched_start_schedule,
+	.stop_schedule = tbs_sched_stop_schedule,
+};
+
+int intel_gvt_init_sched_policy(struct intel_gvt *gvt)
+{
+	gvt->scheduler.sched_ops = &tbs_schedule_ops;
+
+	return gvt->scheduler.sched_ops->init(gvt);
+}
+
+void intel_gvt_clean_sched_policy(struct intel_gvt *gvt)
+{
+	gvt->scheduler.sched_ops->clean(gvt);
+}
+
+int intel_vgpu_init_sched_policy(struct intel_vgpu *vgpu)
+{
+	return vgpu->gvt->scheduler.sched_ops->init_vgpu(vgpu);
+}
+
+void intel_vgpu_clean_sched_policy(struct intel_vgpu *vgpu)
+{
+	vgpu->gvt->scheduler.sched_ops->clean_vgpu(vgpu);
+}
+
+void intel_vgpu_start_schedule(struct intel_vgpu *vgpu)
+{
+	gvt_dbg_core("vgpu%d: start schedule\n", vgpu->id);
+
+	vgpu->gvt->scheduler.sched_ops->start_schedule(vgpu);
+}
+
+void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu)
+{
+	struct intel_gvt_workload_scheduler *scheduler =
+		&vgpu->gvt->scheduler;
+
+	gvt_dbg_core("vgpu%d: stop schedule\n", vgpu->id);
+
+	scheduler->sched_ops->stop_schedule(vgpu);
+
+	if (scheduler->next_vgpu == vgpu)
+		scheduler->next_vgpu = NULL;
+
+	if (scheduler->current_vgpu == vgpu) {
+		/* stop workload dispatching */
+		scheduler->need_reschedule = true;
+		scheduler->current_vgpu = NULL;
+	}
+}
diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.h b/drivers/gpu/drm/i915/gvt/sched_policy.h
new file mode 100644
index 000000000000..bb8b9097e41a
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/sched_policy.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Anhua Xu
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#ifndef __GVT_SCHED_POLICY__
+#define __GVT_SCHED_POLICY__
+
+struct intel_gvt_sched_policy_ops {
+	int (*init)(struct intel_gvt *gvt);
+	void (*clean)(struct intel_gvt *gvt);
+	int (*init_vgpu)(struct intel_vgpu *vgpu);
+	void (*clean_vgpu)(struct intel_vgpu *vgpu);
+	void (*start_schedule)(struct intel_vgpu *vgpu);
+	void (*stop_schedule)(struct intel_vgpu *vgpu);
+};
+
+int intel_gvt_init_sched_policy(struct intel_gvt *gvt);
+
+void intel_gvt_clean_sched_policy(struct intel_gvt *gvt);
+
+int intel_vgpu_init_sched_policy(struct intel_vgpu *vgpu);
+
+void intel_vgpu_clean_sched_policy(struct intel_vgpu *vgpu);
+
+void intel_vgpu_start_schedule(struct intel_vgpu *vgpu);
+
+void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu);
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
new file mode 100644
index 000000000000..4db242250235
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -0,0 +1,583 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Chanbin Du <changbin.du@intel.com>
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *
+ */
+
+#include <linux/kthread.h>
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+#define RING_CTX_OFF(x) \
+	offsetof(struct execlist_ring_context, x)
+
+static void set_context_pdp_root_pointer(
+		struct execlist_ring_context *ring_context,
+		u32 pdp[8])
+{
+	struct execlist_mmio_pair *pdp_pair = &ring_context->pdp3_UDW;
+	int i;
+
+	for (i = 0; i < 8; i++)
+		pdp_pair[i].val = pdp[7 - i];
+}
+
+static int populate_shadow_context(struct intel_vgpu_workload *workload)
+{
+	struct intel_vgpu *vgpu = workload->vgpu;
+	struct intel_gvt *gvt = vgpu->gvt;
+	int ring_id = workload->ring_id;
+	struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
+	struct drm_i915_gem_object *ctx_obj =
+		shadow_ctx->engine[ring_id].state->obj;
+	struct execlist_ring_context *shadow_ring_context;
+	struct page *page;
+	void *dst;
+	unsigned long context_gpa, context_page_num;
+	int i;
+
+	gvt_dbg_sched("ring id %d workload lrca %x", ring_id,
+			workload->ctx_desc.lrca);
+
+	context_page_num = intel_lr_context_size(
+			gvt->dev_priv->engine[ring_id]);
+
+	context_page_num = context_page_num >> PAGE_SHIFT;
+
+	if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS)
+		context_page_num = 19;
+
+	i = 2;
+
+	while (i < context_page_num) {
+		context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
+				(u32)((workload->ctx_desc.lrca + i) <<
+				GTT_PAGE_SHIFT));
+		if (context_gpa == INTEL_GVT_INVALID_ADDR) {
+			gvt_err("Invalid guest context descriptor\n");
+			return -EINVAL;
+		}
+
+		page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i);
+		dst = kmap(page);
+		intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst,
+				GTT_PAGE_SIZE);
+		kunmap(page);
+		i++;
+	}
+
+	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+	shadow_ring_context = kmap(page);
+
+#define COPY_REG(name) \
+	intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
+		+ RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
+
+	COPY_REG(ctx_ctrl);
+	COPY_REG(ctx_timestamp);
+
+	if (ring_id == RCS) {
+		COPY_REG(bb_per_ctx_ptr);
+		COPY_REG(rcs_indirect_ctx);
+		COPY_REG(rcs_indirect_ctx_offset);
+	}
+#undef COPY_REG
+
+	set_context_pdp_root_pointer(shadow_ring_context,
+				     workload->shadow_mm->shadow_page_table);
+
+	intel_gvt_hypervisor_read_gpa(vgpu,
+			workload->ring_context_gpa +
+			sizeof(*shadow_ring_context),
+			(void *)shadow_ring_context +
+			sizeof(*shadow_ring_context),
+			GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
+
+	kunmap(page);
+	return 0;
+}
+
+static int shadow_context_status_change(struct notifier_block *nb,
+		unsigned long action, void *data)
+{
+	struct intel_vgpu *vgpu = container_of(nb,
+			struct intel_vgpu, shadow_ctx_notifier_block);
+	struct drm_i915_gem_request *req =
+		(struct drm_i915_gem_request *)data;
+	struct intel_gvt_workload_scheduler *scheduler =
+		&vgpu->gvt->scheduler;
+	struct intel_vgpu_workload *workload =
+		scheduler->current_workload[req->engine->id];
+
+	switch (action) {
+	case INTEL_CONTEXT_SCHEDULE_IN:
+		intel_gvt_load_render_mmio(workload->vgpu,
+					   workload->ring_id);
+		atomic_set(&workload->shadow_ctx_active, 1);
+		break;
+	case INTEL_CONTEXT_SCHEDULE_OUT:
+		intel_gvt_restore_render_mmio(workload->vgpu,
+					      workload->ring_id);
+		atomic_set(&workload->shadow_ctx_active, 0);
+		break;
+	default:
+		WARN_ON(1);
+		return NOTIFY_OK;
+	}
+	wake_up(&workload->shadow_ctx_status_wq);
+	return NOTIFY_OK;
+}
+
+static int dispatch_workload(struct intel_vgpu_workload *workload)
+{
+	int ring_id = workload->ring_id;
+	struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
+	struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
+	struct drm_i915_gem_request *rq;
+	int ret;
+
+	gvt_dbg_sched("ring id %d prepare to dispatch workload %p\n",
+		ring_id, workload);
+
+	shadow_ctx->desc_template = workload->ctx_desc.addressing_mode <<
+				    GEN8_CTX_ADDRESSING_MODE_SHIFT;
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+
+	rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx);
+	if (IS_ERR(rq)) {
+		gvt_err("fail to allocate gem request\n");
+		ret = PTR_ERR(rq);
+		goto out;
+	}
+
+	gvt_dbg_sched("ring id %d get i915 gem request %p\n", ring_id, rq);
+
+	workload->req = i915_gem_request_get(rq);
+
+	ret = intel_gvt_scan_and_shadow_workload(workload);
+	if (ret)
+		goto out;
+
+	ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx);
+	if (ret)
+		goto out;
+
+	ret = populate_shadow_context(workload);
+	if (ret)
+		goto out;
+
+	if (workload->prepare) {
+		ret = workload->prepare(workload);
+		if (ret)
+			goto out;
+	}
+
+	gvt_dbg_sched("ring id %d submit workload to i915 %p\n",
+			ring_id, workload->req);
+
+	ret = 0;
+	workload->dispatched = true;
+out:
+	if (ret)
+		workload->status = ret;
+
+	if (!IS_ERR_OR_NULL(rq))
+		i915_add_request_no_flush(rq);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+	return ret;
+}
+
+static struct intel_vgpu_workload *pick_next_workload(
+		struct intel_gvt *gvt, int ring_id)
+{
+	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+	struct intel_vgpu_workload *workload = NULL;
+
+	mutex_lock(&gvt->lock);
+
+	/*
+	 * no current vgpu / will be scheduled out / no workload
+	 * bail out
+	 */
+	if (!scheduler->current_vgpu) {
+		gvt_dbg_sched("ring id %d stop - no current vgpu\n", ring_id);
+		goto out;
+	}
+
+	if (scheduler->need_reschedule) {
+		gvt_dbg_sched("ring id %d stop - will reschedule\n", ring_id);
+		goto out;
+	}
+
+	if (list_empty(workload_q_head(scheduler->current_vgpu, ring_id))) {
+		gvt_dbg_sched("ring id %d stop - no available workload\n",
+				ring_id);
+		goto out;
+	}
+
+	/*
+	 * still have current workload, maybe the workload disptacher
+	 * fail to submit it for some reason, resubmit it.
+	 */
+	if (scheduler->current_workload[ring_id]) {
+		workload = scheduler->current_workload[ring_id];
+		gvt_dbg_sched("ring id %d still have current workload %p\n",
+				ring_id, workload);
+		goto out;
+	}
+
+	/*
+	 * pick a workload as current workload
+	 * once current workload is set, schedule policy routines
+	 * will wait the current workload is finished when trying to
+	 * schedule out a vgpu.
+	 */
+	scheduler->current_workload[ring_id] = container_of(
+			workload_q_head(scheduler->current_vgpu, ring_id)->next,
+			struct intel_vgpu_workload, list);
+
+	workload = scheduler->current_workload[ring_id];
+
+	gvt_dbg_sched("ring id %d pick new workload %p\n", ring_id, workload);
+
+	atomic_inc(&workload->vgpu->running_workload_num);
+out:
+	mutex_unlock(&gvt->lock);
+	return workload;
+}
+
+static void update_guest_context(struct intel_vgpu_workload *workload)
+{
+	struct intel_vgpu *vgpu = workload->vgpu;
+	struct intel_gvt *gvt = vgpu->gvt;
+	int ring_id = workload->ring_id;
+	struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
+	struct drm_i915_gem_object *ctx_obj =
+		shadow_ctx->engine[ring_id].state->obj;
+	struct execlist_ring_context *shadow_ring_context;
+	struct page *page;
+	void *src;
+	unsigned long context_gpa, context_page_num;
+	int i;
+
+	gvt_dbg_sched("ring id %d workload lrca %x\n", ring_id,
+			workload->ctx_desc.lrca);
+
+	context_page_num = intel_lr_context_size(
+			gvt->dev_priv->engine[ring_id]);
+
+	context_page_num = context_page_num >> PAGE_SHIFT;
+
+	if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS)
+		context_page_num = 19;
+
+	i = 2;
+
+	while (i < context_page_num) {
+		context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
+				(u32)((workload->ctx_desc.lrca + i) <<
+					GTT_PAGE_SHIFT));
+		if (context_gpa == INTEL_GVT_INVALID_ADDR) {
+			gvt_err("invalid guest context descriptor\n");
+			return;
+		}
+
+		page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i);
+		src = kmap(page);
+		intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src,
+				GTT_PAGE_SIZE);
+		kunmap(page);
+		i++;
+	}
+
+	intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa +
+		RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4);
+
+	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+	shadow_ring_context = kmap(page);
+
+#define COPY_REG(name) \
+	intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \
+		RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
+
+	COPY_REG(ctx_ctrl);
+	COPY_REG(ctx_timestamp);
+
+#undef COPY_REG
+
+	intel_gvt_hypervisor_write_gpa(vgpu,
+			workload->ring_context_gpa +
+			sizeof(*shadow_ring_context),
+			(void *)shadow_ring_context +
+			sizeof(*shadow_ring_context),
+			GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
+
+	kunmap(page);
+}
+
+static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
+{
+	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+	struct intel_vgpu_workload *workload;
+	int event;
+
+	mutex_lock(&gvt->lock);
+
+	workload = scheduler->current_workload[ring_id];
+
+	if (!workload->status && !workload->vgpu->resetting) {
+		wait_event(workload->shadow_ctx_status_wq,
+			   !atomic_read(&workload->shadow_ctx_active));
+
+		update_guest_context(workload);
+
+		for_each_set_bit(event, workload->pending_events,
+				 INTEL_GVT_EVENT_MAX)
+			intel_vgpu_trigger_virtual_event(workload->vgpu,
+					event);
+	}
+
+	gvt_dbg_sched("ring id %d complete workload %p status %d\n",
+			ring_id, workload, workload->status);
+
+	scheduler->current_workload[ring_id] = NULL;
+
+	atomic_dec(&workload->vgpu->running_workload_num);
+
+	list_del_init(&workload->list);
+	workload->complete(workload);
+
+	wake_up(&scheduler->workload_complete_wq);
+	mutex_unlock(&gvt->lock);
+}
+
+struct workload_thread_param {
+	struct intel_gvt *gvt;
+	int ring_id;
+};
+
+static DEFINE_MUTEX(scheduler_mutex);
+
+static int workload_thread(void *priv)
+{
+	struct workload_thread_param *p = (struct workload_thread_param *)priv;
+	struct intel_gvt *gvt = p->gvt;
+	int ring_id = p->ring_id;
+	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+	struct intel_vgpu_workload *workload = NULL;
+	long lret;
+	int ret;
+	bool need_force_wake = IS_SKYLAKE(gvt->dev_priv);
+	DEFINE_WAIT_FUNC(wait, woken_wake_function);
+
+	kfree(p);
+
+	gvt_dbg_core("workload thread for ring %d started\n", ring_id);
+
+	while (!kthread_should_stop()) {
+		add_wait_queue(&scheduler->waitq[ring_id], &wait);
+		do {
+			workload = pick_next_workload(gvt, ring_id);
+			if (workload)
+				break;
+			wait_woken(&wait, TASK_INTERRUPTIBLE,
+				   MAX_SCHEDULE_TIMEOUT);
+		} while (!kthread_should_stop());
+		remove_wait_queue(&scheduler->waitq[ring_id], &wait);
+
+		if (!workload)
+			break;
+
+		mutex_lock(&scheduler_mutex);
+
+		gvt_dbg_sched("ring id %d next workload %p vgpu %d\n",
+				workload->ring_id, workload,
+				workload->vgpu->id);
+
+		intel_runtime_pm_get(gvt->dev_priv);
+
+		gvt_dbg_sched("ring id %d will dispatch workload %p\n",
+				workload->ring_id, workload);
+
+		if (need_force_wake)
+			intel_uncore_forcewake_get(gvt->dev_priv,
+					FORCEWAKE_ALL);
+
+		mutex_lock(&gvt->lock);
+		ret = dispatch_workload(workload);
+		mutex_unlock(&gvt->lock);
+
+		if (ret) {
+			gvt_err("fail to dispatch workload, skip\n");
+			goto complete;
+		}
+
+		gvt_dbg_sched("ring id %d wait workload %p\n",
+				workload->ring_id, workload);
+
+		lret = i915_wait_request(workload->req,
+					 0, MAX_SCHEDULE_TIMEOUT);
+		if (lret < 0) {
+			workload->status = lret;
+			gvt_err("fail to wait workload, skip\n");
+		} else {
+			workload->status = 0;
+		}
+
+complete:
+		gvt_dbg_sched("will complete workload %p\n, status: %d\n",
+				workload, workload->status);
+
+		complete_current_workload(gvt, ring_id);
+
+		if (workload->req)
+			i915_gem_request_put(fetch_and_zero(&workload->req));
+
+		if (need_force_wake)
+			intel_uncore_forcewake_put(gvt->dev_priv,
+					FORCEWAKE_ALL);
+
+		intel_runtime_pm_put(gvt->dev_priv);
+
+		mutex_unlock(&scheduler_mutex);
+
+	}
+	return 0;
+}
+
+void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+
+	if (atomic_read(&vgpu->running_workload_num)) {
+		gvt_dbg_sched("wait vgpu idle\n");
+
+		wait_event(scheduler->workload_complete_wq,
+				!atomic_read(&vgpu->running_workload_num));
+	}
+}
+
+void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt)
+{
+	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+	int i;
+
+	gvt_dbg_core("clean workload scheduler\n");
+
+	for (i = 0; i < I915_NUM_ENGINES; i++) {
+		if (scheduler->thread[i]) {
+			kthread_stop(scheduler->thread[i]);
+			scheduler->thread[i] = NULL;
+		}
+	}
+}
+
+int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt)
+{
+	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
+	struct workload_thread_param *param = NULL;
+	int ret;
+	int i;
+
+	gvt_dbg_core("init workload scheduler\n");
+
+	init_waitqueue_head(&scheduler->workload_complete_wq);
+
+	for (i = 0; i < I915_NUM_ENGINES; i++) {
+		/* check ring mask at init time */
+		if (!HAS_ENGINE(gvt->dev_priv, i))
+			continue;
+
+		init_waitqueue_head(&scheduler->waitq[i]);
+
+		param = kzalloc(sizeof(*param), GFP_KERNEL);
+		if (!param) {
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		param->gvt = gvt;
+		param->ring_id = i;
+
+		scheduler->thread[i] = kthread_run(workload_thread, param,
+			"gvt workload %d", i);
+		if (IS_ERR(scheduler->thread[i])) {
+			gvt_err("fail to create workload thread\n");
+			ret = PTR_ERR(scheduler->thread[i]);
+			goto err;
+		}
+	}
+	return 0;
+err:
+	intel_gvt_clean_workload_scheduler(gvt);
+	kfree(param);
+	param = NULL;
+	return ret;
+}
+
+void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+	atomic_notifier_chain_unregister(&vgpu->shadow_ctx->status_notifier,
+			&vgpu->shadow_ctx_notifier_block);
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+
+	/* a little hacky to mark as ctx closed */
+	vgpu->shadow_ctx->closed = true;
+	i915_gem_context_put(vgpu->shadow_ctx);
+
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+}
+
+int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu)
+{
+	atomic_set(&vgpu->running_workload_num, 0);
+
+	vgpu->shadow_ctx = i915_gem_context_create_gvt(
+			&vgpu->gvt->dev_priv->drm);
+	if (IS_ERR(vgpu->shadow_ctx))
+		return PTR_ERR(vgpu->shadow_ctx);
+
+	vgpu->shadow_ctx->engine[RCS].initialised = true;
+
+	vgpu->shadow_ctx_notifier_block.notifier_call =
+		shadow_context_status_change;
+
+	atomic_notifier_chain_register(&vgpu->shadow_ctx->status_notifier,
+				       &vgpu->shadow_ctx_notifier_block);
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h
new file mode 100644
index 000000000000..3b30c28bff51
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/scheduler.h
@@ -0,0 +1,139 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ * Contributors:
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Chanbin Du <changbin.du@intel.com>
+ *    Min He <min.he@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *
+ */
+
+#ifndef _GVT_SCHEDULER_H_
+#define _GVT_SCHEDULER_H_
+
+struct intel_gvt_workload_scheduler {
+	struct intel_vgpu *current_vgpu;
+	struct intel_vgpu *next_vgpu;
+	struct intel_vgpu_workload *current_workload[I915_NUM_ENGINES];
+	bool need_reschedule;
+
+	wait_queue_head_t workload_complete_wq;
+	struct task_struct *thread[I915_NUM_ENGINES];
+	wait_queue_head_t waitq[I915_NUM_ENGINES];
+
+	void *sched_data;
+	struct intel_gvt_sched_policy_ops *sched_ops;
+};
+
+#define INDIRECT_CTX_ADDR_MASK 0xffffffc0
+#define INDIRECT_CTX_SIZE_MASK 0x3f
+struct shadow_indirect_ctx {
+	struct drm_i915_gem_object *obj;
+	unsigned long guest_gma;
+	unsigned long shadow_gma;
+	void *shadow_va;
+	uint32_t size;
+};
+
+#define PER_CTX_ADDR_MASK 0xfffff000
+struct shadow_per_ctx {
+	unsigned long guest_gma;
+	unsigned long shadow_gma;
+};
+
+struct intel_shadow_wa_ctx {
+	struct intel_vgpu_workload *workload;
+	struct shadow_indirect_ctx indirect_ctx;
+	struct shadow_per_ctx per_ctx;
+
+};
+
+struct intel_vgpu_workload {
+	struct intel_vgpu *vgpu;
+	int ring_id;
+	struct drm_i915_gem_request *req;
+	/* if this workload has been dispatched to i915? */
+	bool dispatched;
+	int status;
+
+	struct intel_vgpu_mm *shadow_mm;
+
+	/* different submission model may need different handler */
+	int (*prepare)(struct intel_vgpu_workload *);
+	int (*complete)(struct intel_vgpu_workload *);
+	struct list_head list;
+
+	DECLARE_BITMAP(pending_events, INTEL_GVT_EVENT_MAX);
+	void *shadow_ring_buffer_va;
+
+	/* execlist context information */
+	struct execlist_ctx_descriptor_format ctx_desc;
+	struct execlist_ring_context *ring_context;
+	unsigned long rb_head, rb_tail, rb_ctl, rb_start, rb_len;
+	bool restore_inhibit;
+	struct intel_vgpu_elsp_dwords elsp_dwords;
+	bool emulate_schedule_in;
+	atomic_t shadow_ctx_active;
+	wait_queue_head_t shadow_ctx_status_wq;
+	u64 ring_context_gpa;
+
+	/* shadow batch buffer */
+	struct list_head shadow_bb;
+	struct intel_shadow_wa_ctx wa_ctx;
+};
+
+/* Intel shadow batch buffer is a i915 gem object */
+struct intel_shadow_bb_entry {
+	struct list_head list;
+	struct drm_i915_gem_object *obj;
+	void *va;
+	unsigned long len;
+	void *bb_start_cmd_va;
+};
+
+#define workload_q_head(vgpu, ring_id) \
+	(&(vgpu->workload_q_head[ring_id]))
+
+#define queue_workload(workload) do { \
+	list_add_tail(&workload->list, \
+	workload_q_head(workload->vgpu, workload->ring_id)); \
+	wake_up(&workload->vgpu->gvt-> \
+	scheduler.waitq[workload->ring_id]); \
+} while (0)
+
+int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt);
+
+void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt);
+
+void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu);
+
+int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu);
+
+void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu);
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/trace.h b/drivers/gpu/drm/i915/gvt/trace.h
new file mode 100644
index 000000000000..53a2d10cf3f1
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/trace.h
@@ -0,0 +1,286 @@
+/*
+ * Copyright © 2011-2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Jike Song <jike.song@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *
+ */
+
+#if !defined(_GVT_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _GVT_TRACE_H_
+
+#include <linux/types.h>
+#include <linux/stringify.h>
+#include <linux/tracepoint.h>
+#include <asm/tsc.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM gvt
+
+TRACE_EVENT(spt_alloc,
+	TP_PROTO(int id, void *spt, int type, unsigned long mfn,
+		unsigned long gpt_gfn),
+
+	TP_ARGS(id, spt, type, mfn, gpt_gfn),
+
+	TP_STRUCT__entry(
+		__field(int, id)
+		__field(void *, spt)
+		__field(int, type)
+		__field(unsigned long, mfn)
+		__field(unsigned long, gpt_gfn)
+		),
+
+	TP_fast_assign(
+		__entry->id = id;
+		__entry->spt = spt;
+		__entry->type = type;
+		__entry->mfn = mfn;
+		__entry->gpt_gfn = gpt_gfn;
+	),
+
+	TP_printk("VM%d [alloc] spt %p type %d mfn 0x%lx gfn 0x%lx\n",
+		__entry->id,
+		__entry->spt,
+		__entry->type,
+		__entry->mfn,
+		__entry->gpt_gfn)
+);
+
+TRACE_EVENT(spt_free,
+	TP_PROTO(int id, void *spt, int type),
+
+	TP_ARGS(id, spt, type),
+
+	TP_STRUCT__entry(
+		__field(int, id)
+		__field(void *, spt)
+		__field(int, type)
+		),
+
+	TP_fast_assign(
+		__entry->id = id;
+		__entry->spt = spt;
+		__entry->type = type;
+	),
+
+	TP_printk("VM%u [free] spt %p type %d\n",
+		__entry->id,
+		__entry->spt,
+		__entry->type)
+);
+
+#define MAX_BUF_LEN 256
+
+TRACE_EVENT(gma_index,
+	TP_PROTO(const char *prefix, unsigned long gma,
+		unsigned long index),
+
+	TP_ARGS(prefix, gma, index),
+
+	TP_STRUCT__entry(
+		__array(char, buf, MAX_BUF_LEN)
+	),
+
+	TP_fast_assign(
+		snprintf(__entry->buf, MAX_BUF_LEN,
+			"%s gma 0x%lx index 0x%lx\n", prefix, gma, index);
+	),
+
+	TP_printk("%s", __entry->buf)
+);
+
+TRACE_EVENT(gma_translate,
+	TP_PROTO(int id, char *type, int ring_id, int pt_level,
+		unsigned long gma, unsigned long gpa),
+
+	TP_ARGS(id, type, ring_id, pt_level, gma, gpa),
+
+	TP_STRUCT__entry(
+		__array(char, buf, MAX_BUF_LEN)
+	),
+
+	TP_fast_assign(
+		snprintf(__entry->buf, MAX_BUF_LEN,
+			"VM%d %s ring %d pt_level %d gma 0x%lx -> gpa 0x%lx\n",
+				id, type, ring_id, pt_level, gma, gpa);
+	),
+
+	TP_printk("%s", __entry->buf)
+);
+
+TRACE_EVENT(spt_refcount,
+	TP_PROTO(int id, char *action, void *spt, int before, int after),
+
+	TP_ARGS(id, action, spt, before, after),
+
+	TP_STRUCT__entry(
+		__array(char, buf, MAX_BUF_LEN)
+	),
+
+	TP_fast_assign(
+		snprintf(__entry->buf, MAX_BUF_LEN,
+			"VM%d [%s] spt %p before %d -> after %d\n",
+				id, action, spt, before, after);
+	),
+
+	TP_printk("%s", __entry->buf)
+);
+
+TRACE_EVENT(spt_change,
+	TP_PROTO(int id, char *action, void *spt, unsigned long gfn,
+		int type),
+
+	TP_ARGS(id, action, spt, gfn, type),
+
+	TP_STRUCT__entry(
+		__array(char, buf, MAX_BUF_LEN)
+	),
+
+	TP_fast_assign(
+		snprintf(__entry->buf, MAX_BUF_LEN,
+			"VM%d [%s] spt %p gfn 0x%lx type %d\n",
+				id, action, spt, gfn, type);
+	),
+
+	TP_printk("%s", __entry->buf)
+);
+
+TRACE_EVENT(gpt_change,
+	TP_PROTO(int id, const char *tag, void *spt, int type, u64 v,
+		unsigned long index),
+
+	TP_ARGS(id, tag, spt, type, v, index),
+
+	TP_STRUCT__entry(
+		__array(char, buf, MAX_BUF_LEN)
+	),
+
+	TP_fast_assign(
+		snprintf(__entry->buf, MAX_BUF_LEN,
+		"VM%d [%s] spt %p type %d entry 0x%llx index 0x%lx\n",
+			id, tag, spt, type, v, index);
+	),
+
+	TP_printk("%s", __entry->buf)
+);
+
+TRACE_EVENT(oos_change,
+	TP_PROTO(int id, const char *tag, int page_id, void *gpt, int type),
+
+	TP_ARGS(id, tag, page_id, gpt, type),
+
+	TP_STRUCT__entry(
+		__array(char, buf, MAX_BUF_LEN)
+	),
+
+	TP_fast_assign(
+		snprintf(__entry->buf, MAX_BUF_LEN,
+		"VM%d [oos %s] page id %d gpt %p type %d\n",
+			id, tag, page_id, gpt, type);
+	),
+
+	TP_printk("%s", __entry->buf)
+);
+
+TRACE_EVENT(oos_sync,
+	TP_PROTO(int id, int page_id, void *gpt, int type, u64 v,
+		unsigned long index),
+
+	TP_ARGS(id, page_id, gpt, type, v, index),
+
+	TP_STRUCT__entry(
+		__array(char, buf, MAX_BUF_LEN)
+	),
+
+	TP_fast_assign(
+	snprintf(__entry->buf, MAX_BUF_LEN,
+	"VM%d [oos sync] page id %d gpt %p type %d entry 0x%llx index 0x%lx\n",
+				id, page_id, gpt, type, v, index);
+	),
+
+	TP_printk("%s", __entry->buf)
+);
+
+#define MAX_CMD_STR_LEN	256
+TRACE_EVENT(gvt_command,
+		TP_PROTO(u8 vm_id, u8 ring_id, u32 ip_gma, u32 *cmd_va, u32 cmd_len, bool ring_buffer_cmd, cycles_t cost_pre_cmd_handler, cycles_t cost_cmd_handler),
+
+		TP_ARGS(vm_id, ring_id, ip_gma, cmd_va, cmd_len, ring_buffer_cmd, cost_pre_cmd_handler, cost_cmd_handler),
+
+		TP_STRUCT__entry(
+			__field(u8, vm_id)
+			__field(u8, ring_id)
+			__field(int, i)
+			__array(char, tmp_buf, MAX_CMD_STR_LEN)
+			__array(char, cmd_str, MAX_CMD_STR_LEN)
+			),
+
+		TP_fast_assign(
+			__entry->vm_id = vm_id;
+			__entry->ring_id = ring_id;
+			__entry->cmd_str[0] = '\0';
+			snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "VM(%d) Ring(%d): %s ip(%08x) pre handler cost (%llu), handler cost (%llu) ", vm_id, ring_id, ring_buffer_cmd ? "RB":"BB", ip_gma, cost_pre_cmd_handler, cost_cmd_handler);
+			strcat(__entry->cmd_str, __entry->tmp_buf);
+			entry->i = 0;
+			while (cmd_len > 0) {
+				if (cmd_len >= 8) {
+					snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "%08x %08x %08x %08x %08x %08x %08x %08x ",
+						cmd_va[__entry->i], cmd_va[__entry->i+1], cmd_va[__entry->i+2], cmd_va[__entry->i+3],
+						cmd_va[__entry->i+4], cmd_va[__entry->i+5], cmd_va[__entry->i+6], cmd_va[__entry->i+7]);
+					__entry->i += 8;
+					cmd_len -= 8;
+					strcat(__entry->cmd_str, __entry->tmp_buf);
+				} else if (cmd_len >= 4) {
+					snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "%08x %08x %08x %08x ",
+						cmd_va[__entry->i], cmd_va[__entry->i+1], cmd_va[__entry->i+2], cmd_va[__entry->i+3]);
+					__entry->i += 4;
+					cmd_len -= 4;
+					strcat(__entry->cmd_str, __entry->tmp_buf);
+				} else if (cmd_len >= 2) {
+					snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "%08x %08x ", cmd_va[__entry->i], cmd_va[__entry->i+1]);
+					__entry->i += 2;
+					cmd_len -= 2;
+					strcat(__entry->cmd_str, __entry->tmp_buf);
+				} else if (cmd_len == 1) {
+					snprintf(__entry->tmp_buf, MAX_CMD_STR_LEN, "%08x ", cmd_va[__entry->i]);
+					__entry->i += 1;
+					cmd_len -= 1;
+					strcat(__entry->cmd_str, __entry->tmp_buf);
+				}
+			}
+			strcat(__entry->cmd_str, "\n");
+		),
+
+		TP_printk("%s", __entry->cmd_str)
+);
+#endif /* _GVT_TRACE_H_ */
+
+/* This part must be out of protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.h b/drivers/gpu/drm/i915/gvt/trace_points.c
index 91315557e421..a3deed692b9c 100644
--- a/drivers/gpu/drm/i915/i915_gem_dmabuf.h
+++ b/drivers/gpu/drm/i915/gvt/trace_points.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 Intel Corporation
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -16,30 +16,21 @@
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Jike Song <jike.song@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
  *
  */
 
-#ifndef _I915_GEM_DMABUF_H_
-#define _I915_GEM_DMABUF_H_
-
-#include <linux/dma-buf.h>
-
-static inline struct reservation_object *
-i915_gem_object_get_dmabuf_resv(struct drm_i915_gem_object *obj)
-{
-	struct dma_buf *dma_buf;
-
-	if (obj->base.dma_buf)
-		dma_buf = obj->base.dma_buf;
-	else if (obj->base.import_attach)
-		dma_buf = obj->base.import_attach->dmabuf;
-	else
-		return NULL;
-
-	return dma_buf->resv;
-}
+#include "trace.h"
 
+#ifndef __CHECKER__
+#define CREATE_TRACE_POINTS
+#include "trace.h"
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
new file mode 100644
index 000000000000..536d2b9d5777
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/vgpu.c
@@ -0,0 +1,409 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+#include "i915_pvinfo.h"
+
+static void clean_vgpu_mmio(struct intel_vgpu *vgpu)
+{
+	vfree(vgpu->mmio.vreg);
+	vgpu->mmio.vreg = vgpu->mmio.sreg = NULL;
+}
+
+int setup_vgpu_mmio(struct intel_vgpu *vgpu)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	const struct intel_gvt_device_info *info = &gvt->device_info;
+
+	if (vgpu->mmio.vreg)
+		memset(vgpu->mmio.vreg, 0, info->mmio_size * 2);
+	else {
+		vgpu->mmio.vreg = vzalloc(info->mmio_size * 2);
+		if (!vgpu->mmio.vreg)
+			return -ENOMEM;
+	}
+
+	vgpu->mmio.sreg = vgpu->mmio.vreg + info->mmio_size;
+
+	memcpy(vgpu->mmio.vreg, gvt->firmware.mmio, info->mmio_size);
+	memcpy(vgpu->mmio.sreg, gvt->firmware.mmio, info->mmio_size);
+
+	vgpu_vreg(vgpu, GEN6_GT_THREAD_STATUS_REG) = 0;
+
+	/* set the bit 0:2(Core C-State ) to C0 */
+	vgpu_vreg(vgpu, GEN6_GT_CORE_STATUS) = 0;
+	return 0;
+}
+
+static void setup_vgpu_cfg_space(struct intel_vgpu *vgpu,
+	struct intel_vgpu_creation_params *param)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	const struct intel_gvt_device_info *info = &gvt->device_info;
+	u16 *gmch_ctl;
+	int i;
+
+	memcpy(vgpu_cfg_space(vgpu), gvt->firmware.cfg_space,
+	       info->cfg_space_size);
+
+	if (!param->primary) {
+		vgpu_cfg_space(vgpu)[PCI_CLASS_DEVICE] =
+			INTEL_GVT_PCI_CLASS_VGA_OTHER;
+		vgpu_cfg_space(vgpu)[PCI_CLASS_PROG] =
+			INTEL_GVT_PCI_CLASS_VGA_OTHER;
+	}
+
+	/* Show guest that there isn't any stolen memory.*/
+	gmch_ctl = (u16 *)(vgpu_cfg_space(vgpu) + INTEL_GVT_PCI_GMCH_CONTROL);
+	*gmch_ctl &= ~(BDW_GMCH_GMS_MASK << BDW_GMCH_GMS_SHIFT);
+
+	intel_vgpu_write_pci_bar(vgpu, PCI_BASE_ADDRESS_2,
+				 gvt_aperture_pa_base(gvt), true);
+
+	vgpu_cfg_space(vgpu)[PCI_COMMAND] &= ~(PCI_COMMAND_IO
+					     | PCI_COMMAND_MEMORY
+					     | PCI_COMMAND_MASTER);
+	/*
+	 * Clear the bar upper 32bit and let guest to assign the new value
+	 */
+	memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_1, 0, 4);
+	memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_3, 0, 4);
+	memset(vgpu_cfg_space(vgpu) + INTEL_GVT_PCI_OPREGION, 0, 4);
+
+	for (i = 0; i < INTEL_GVT_MAX_BAR_NUM; i++) {
+		vgpu->cfg_space.bar[i].size = pci_resource_len(
+					      gvt->dev_priv->drm.pdev, i * 2);
+		vgpu->cfg_space.bar[i].tracked = false;
+	}
+}
+
+void populate_pvinfo_page(struct intel_vgpu *vgpu)
+{
+	/* setup the ballooning information */
+	vgpu_vreg64(vgpu, vgtif_reg(magic)) = VGT_MAGIC;
+	vgpu_vreg(vgpu, vgtif_reg(version_major)) = 1;
+	vgpu_vreg(vgpu, vgtif_reg(version_minor)) = 0;
+	vgpu_vreg(vgpu, vgtif_reg(display_ready)) = 0;
+	vgpu_vreg(vgpu, vgtif_reg(vgt_id)) = vgpu->id;
+	vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.base)) =
+		vgpu_aperture_gmadr_base(vgpu);
+	vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.size)) =
+		vgpu_aperture_sz(vgpu);
+	vgpu_vreg(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.base)) =
+		vgpu_hidden_gmadr_base(vgpu);
+	vgpu_vreg(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.size)) =
+		vgpu_hidden_sz(vgpu);
+
+	vgpu_vreg(vgpu, vgtif_reg(avail_rs.fence_num)) = vgpu_fence_sz(vgpu);
+
+	gvt_dbg_core("Populate PVINFO PAGE for vGPU %d\n", vgpu->id);
+	gvt_dbg_core("aperture base [GMADR] 0x%llx size 0x%llx\n",
+		vgpu_aperture_gmadr_base(vgpu), vgpu_aperture_sz(vgpu));
+	gvt_dbg_core("hidden base [GMADR] 0x%llx size=0x%llx\n",
+		vgpu_hidden_gmadr_base(vgpu), vgpu_hidden_sz(vgpu));
+	gvt_dbg_core("fence size %d\n", vgpu_fence_sz(vgpu));
+
+	WARN_ON(sizeof(struct vgt_if) != VGT_PVINFO_SIZE);
+}
+
+/**
+ * intel_gvt_init_vgpu_types - initialize vGPU type list
+ * @gvt : GVT device
+ *
+ * Initialize vGPU type list based on available resource.
+ *
+ */
+int intel_gvt_init_vgpu_types(struct intel_gvt *gvt)
+{
+	unsigned int num_types;
+	unsigned int i, low_avail;
+	unsigned int min_low;
+
+	/* vGPU type name is defined as GVTg_Vx_y which contains
+	 * physical GPU generation type and 'y' means maximum vGPU
+	 * instances user can create on one physical GPU for this
+	 * type.
+	 *
+	 * Depend on physical SKU resource, might see vGPU types like
+	 * GVTg_V4_8, GVTg_V4_4, GVTg_V4_2, etc. We can create
+	 * different types of vGPU on same physical GPU depending on
+	 * available resource. Each vGPU type will have "avail_instance"
+	 * to indicate how many vGPU instance can be created for this
+	 * type.
+	 *
+	 * Currently use static size here as we init type earlier..
+	 */
+	low_avail = MB_TO_BYTES(256) - HOST_LOW_GM_SIZE;
+	num_types = 4;
+
+	gvt->types = kzalloc(num_types * sizeof(struct intel_vgpu_type),
+			     GFP_KERNEL);
+	if (!gvt->types)
+		return -ENOMEM;
+
+	min_low = MB_TO_BYTES(32);
+	for (i = 0; i < num_types; ++i) {
+		if (low_avail / min_low == 0)
+			break;
+		gvt->types[i].low_gm_size = min_low;
+		gvt->types[i].high_gm_size = 3 * gvt->types[i].low_gm_size;
+		gvt->types[i].fence = 4;
+		gvt->types[i].max_instance = low_avail / min_low;
+		gvt->types[i].avail_instance = gvt->types[i].max_instance;
+
+		if (IS_GEN8(gvt->dev_priv))
+			sprintf(gvt->types[i].name, "GVTg_V4_%u",
+						gvt->types[i].max_instance);
+		else if (IS_GEN9(gvt->dev_priv))
+			sprintf(gvt->types[i].name, "GVTg_V5_%u",
+						gvt->types[i].max_instance);
+
+		min_low <<= 1;
+		gvt_dbg_core("type[%d]: %s max %u avail %u low %u high %u fence %u\n",
+			     i, gvt->types[i].name, gvt->types[i].max_instance,
+			     gvt->types[i].avail_instance,
+			     gvt->types[i].low_gm_size,
+			     gvt->types[i].high_gm_size, gvt->types[i].fence);
+	}
+
+	gvt->num_types = i;
+	return 0;
+}
+
+void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt)
+{
+	kfree(gvt->types);
+}
+
+static void intel_gvt_update_vgpu_types(struct intel_gvt *gvt)
+{
+	int i;
+	unsigned int low_gm_avail, high_gm_avail, fence_avail;
+	unsigned int low_gm_min, high_gm_min, fence_min, total_min;
+
+	/* Need to depend on maxium hw resource size but keep on
+	 * static config for now.
+	 */
+	low_gm_avail = MB_TO_BYTES(256) - HOST_LOW_GM_SIZE -
+		gvt->gm.vgpu_allocated_low_gm_size;
+	high_gm_avail = MB_TO_BYTES(256) * 3 - HOST_HIGH_GM_SIZE -
+		gvt->gm.vgpu_allocated_high_gm_size;
+	fence_avail = gvt_fence_sz(gvt) - HOST_FENCE -
+		gvt->fence.vgpu_allocated_fence_num;
+
+	for (i = 0; i < gvt->num_types; i++) {
+		low_gm_min = low_gm_avail / gvt->types[i].low_gm_size;
+		high_gm_min = high_gm_avail / gvt->types[i].high_gm_size;
+		fence_min = fence_avail / gvt->types[i].fence;
+		total_min = min(min(low_gm_min, high_gm_min), fence_min);
+		gvt->types[i].avail_instance = min(gvt->types[i].max_instance,
+						   total_min);
+
+		gvt_dbg_core("update type[%d]: %s max %u avail %u low %u high %u fence %u\n",
+		       i, gvt->types[i].name, gvt->types[i].max_instance,
+		       gvt->types[i].avail_instance, gvt->types[i].low_gm_size,
+		       gvt->types[i].high_gm_size, gvt->types[i].fence);
+	}
+}
+
+/**
+ * intel_gvt_destroy_vgpu - destroy a virtual GPU
+ * @vgpu: virtual GPU
+ *
+ * This function is called when user wants to destroy a virtual GPU.
+ *
+ */
+void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+
+	mutex_lock(&gvt->lock);
+
+	vgpu->active = false;
+	idr_remove(&gvt->vgpu_idr, vgpu->id);
+
+	if (atomic_read(&vgpu->running_workload_num)) {
+		mutex_unlock(&gvt->lock);
+		intel_gvt_wait_vgpu_idle(vgpu);
+		mutex_lock(&gvt->lock);
+	}
+
+	intel_vgpu_stop_schedule(vgpu);
+	intel_vgpu_clean_sched_policy(vgpu);
+	intel_vgpu_clean_gvt_context(vgpu);
+	intel_vgpu_clean_execlist(vgpu);
+	intel_vgpu_clean_display(vgpu);
+	intel_vgpu_clean_opregion(vgpu);
+	intel_vgpu_clean_gtt(vgpu);
+	intel_gvt_hypervisor_detach_vgpu(vgpu);
+	intel_vgpu_free_resource(vgpu);
+	clean_vgpu_mmio(vgpu);
+	vfree(vgpu);
+
+	intel_gvt_update_vgpu_types(gvt);
+	mutex_unlock(&gvt->lock);
+}
+
+static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt,
+		struct intel_vgpu_creation_params *param)
+{
+	struct intel_vgpu *vgpu;
+	int ret;
+
+	gvt_dbg_core("handle %llu low %llu MB high %llu MB fence %llu\n",
+			param->handle, param->low_gm_sz, param->high_gm_sz,
+			param->fence_sz);
+
+	vgpu = vzalloc(sizeof(*vgpu));
+	if (!vgpu)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_lock(&gvt->lock);
+
+	ret = idr_alloc(&gvt->vgpu_idr, vgpu, 1, GVT_MAX_VGPU, GFP_KERNEL);
+	if (ret < 0)
+		goto out_free_vgpu;
+
+	vgpu->id = ret;
+	vgpu->handle = param->handle;
+	vgpu->gvt = gvt;
+	bitmap_zero(vgpu->tlb_handle_pending, I915_NUM_ENGINES);
+
+	setup_vgpu_cfg_space(vgpu, param);
+
+	ret = setup_vgpu_mmio(vgpu);
+	if (ret)
+		goto out_free_vgpu;
+
+	ret = intel_vgpu_alloc_resource(vgpu, param);
+	if (ret)
+		goto out_clean_vgpu_mmio;
+
+	populate_pvinfo_page(vgpu);
+
+	ret = intel_gvt_hypervisor_attach_vgpu(vgpu);
+	if (ret)
+		goto out_clean_vgpu_resource;
+
+	ret = intel_vgpu_init_gtt(vgpu);
+	if (ret)
+		goto out_detach_hypervisor_vgpu;
+
+	ret = intel_vgpu_init_display(vgpu);
+	if (ret)
+		goto out_clean_gtt;
+
+	ret = intel_vgpu_init_execlist(vgpu);
+	if (ret)
+		goto out_clean_display;
+
+	ret = intel_vgpu_init_gvt_context(vgpu);
+	if (ret)
+		goto out_clean_execlist;
+
+	ret = intel_vgpu_init_sched_policy(vgpu);
+	if (ret)
+		goto out_clean_shadow_ctx;
+
+	vgpu->active = true;
+	mutex_unlock(&gvt->lock);
+
+	return vgpu;
+
+out_clean_shadow_ctx:
+	intel_vgpu_clean_gvt_context(vgpu);
+out_clean_execlist:
+	intel_vgpu_clean_execlist(vgpu);
+out_clean_display:
+	intel_vgpu_clean_display(vgpu);
+out_clean_gtt:
+	intel_vgpu_clean_gtt(vgpu);
+out_detach_hypervisor_vgpu:
+	intel_gvt_hypervisor_detach_vgpu(vgpu);
+out_clean_vgpu_resource:
+	intel_vgpu_free_resource(vgpu);
+out_clean_vgpu_mmio:
+	clean_vgpu_mmio(vgpu);
+out_free_vgpu:
+	vfree(vgpu);
+	mutex_unlock(&gvt->lock);
+	return ERR_PTR(ret);
+}
+
+/**
+ * intel_gvt_create_vgpu - create a virtual GPU
+ * @gvt: GVT device
+ * @type: type of the vGPU to create
+ *
+ * This function is called when user wants to create a virtual GPU.
+ *
+ * Returns:
+ * pointer to intel_vgpu, error pointer if failed.
+ */
+struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
+				struct intel_vgpu_type *type)
+{
+	struct intel_vgpu_creation_params param;
+	struct intel_vgpu *vgpu;
+
+	param.handle = 0;
+	param.primary = 1;
+	param.low_gm_sz = type->low_gm_size;
+	param.high_gm_sz = type->high_gm_size;
+	param.fence_sz = type->fence;
+
+	/* XXX current param based on MB */
+	param.low_gm_sz = BYTES_TO_MB(param.low_gm_sz);
+	param.high_gm_sz = BYTES_TO_MB(param.high_gm_sz);
+
+	vgpu = __intel_gvt_create_vgpu(gvt, &param);
+	if (IS_ERR(vgpu))
+		return vgpu;
+
+	/* calculate left instance change for types */
+	intel_gvt_update_vgpu_types(gvt);
+
+	return vgpu;
+}
+
+/**
+ * intel_gvt_reset_vgpu - reset a virtual GPU
+ * @vgpu: virtual GPU
+ *
+ * This function is called when user wants to reset a virtual GPU.
+ *
+ */
+void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu)
+{
+}
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index 70980f82a15b..f5039f4f988f 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -1290,7 +1290,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
 	}
 
 	if (ret == 0 && needs_clflush_after)
-		drm_clflush_virt_range(shadow_batch_obj->mapping, batch_len);
+		drm_clflush_virt_range(shadow_batch_obj->mm.mapping, batch_len);
 	i915_gem_object_unpin_map(shadow_batch_obj);
 
 	return ret;
@@ -1308,10 +1308,11 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
 int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	bool active = false;
 
 	/* If the command parser is not enabled, report 0 - unsupported */
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		if (intel_engine_needs_cmd_parser(engine)) {
 			active = true;
 			break;
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 27b0e34dadec..791bfc760075 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -79,10 +79,8 @@ static int i915_capabilities(struct seq_file *m, void *data)
 	seq_printf(m, "gen: %d\n", INTEL_GEN(dev_priv));
 	seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(dev_priv));
 #define PRINT_FLAG(x)  seq_printf(m, #x ": %s\n", yesno(info->x))
-#define SEP_SEMICOLON ;
-	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_SEMICOLON);
+	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG);
 #undef PRINT_FLAG
-#undef SEP_SEMICOLON
 
 	return 0;
 }
@@ -109,12 +107,12 @@ static char get_tiling_flag(struct drm_i915_gem_object *obj)
 
 static char get_global_flag(struct drm_i915_gem_object *obj)
 {
-	return i915_gem_object_to_ggtt(obj, NULL) ?  'g' : ' ';
+	return !list_empty(&obj->userfault_link) ? 'g' : ' ';
 }
 
 static char get_pin_mapped_flag(struct drm_i915_gem_object *obj)
 {
-	return obj->mapping ? 'M' : ' ';
+	return obj->mm.mapping ? 'M' : ' ';
 }
 
 static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj)
@@ -138,11 +136,10 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 	struct i915_vma *vma;
 	unsigned int frontbuffer_bits;
 	int pin_count = 0;
-	enum intel_engine_id id;
 
 	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
-	seq_printf(m, "%pK: %c%c%c%c%c %8zdKiB %02x %02x [ ",
+	seq_printf(m, "%pK: %c%c%c%c%c %8zdKiB %02x %02x %s%s%s",
 		   &obj->base,
 		   get_active_flag(obj),
 		   get_pin_flag(obj),
@@ -151,17 +148,10 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 		   get_pin_mapped_flag(obj),
 		   obj->base.size / 1024,
 		   obj->base.read_domains,
-		   obj->base.write_domain);
-	for_each_engine_id(engine, dev_priv, id)
-		seq_printf(m, "%x ",
-			   i915_gem_active_get_seqno(&obj->last_read[id],
-						     &obj->base.dev->struct_mutex));
-	seq_printf(m, "] %x %s%s%s",
-		   i915_gem_active_get_seqno(&obj->last_write,
-					     &obj->base.dev->struct_mutex),
+		   obj->base.write_domain,
 		   i915_cache_level_str(dev_priv, obj->cache_level),
-		   obj->dirty ? " dirty" : "",
-		   obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
+		   obj->mm.dirty ? " dirty" : "",
+		   obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : "");
 	if (obj->base.name)
 		seq_printf(m, " (name: %d)", obj->base.name);
 	list_for_each_entry(vma, &obj->vma_list, obj_link) {
@@ -188,18 +178,8 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 	}
 	if (obj->stolen)
 		seq_printf(m, " (stolen: %08llx)", obj->stolen->start);
-	if (obj->pin_display || obj->fault_mappable) {
-		char s[3], *t = s;
-		if (obj->pin_display)
-			*t++ = 'p';
-		if (obj->fault_mappable)
-			*t++ = 'f';
-		*t = '\0';
-		seq_printf(m, " (%s mappable)", s);
-	}
-
-	engine = i915_gem_active_get_engine(&obj->last_write,
-					    &dev_priv->drm.struct_mutex);
+
+	engine = i915_gem_object_last_write_engine(obj);
 	if (engine)
 		seq_printf(m, " (%s)", engine->name);
 
@@ -237,7 +217,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
 		return ret;
 
 	total_obj_size = total_gtt_size = count = 0;
-	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
+	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
 		if (obj->stolen == NULL)
 			continue;
 
@@ -247,7 +227,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
 		total_gtt_size += i915_gem_obj_total_ggtt_size(obj);
 		count++;
 	}
-	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
+	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) {
 		if (obj->stolen == NULL)
 			continue;
 
@@ -334,11 +314,12 @@ static void print_batch_pool_stats(struct seq_file *m,
 	struct drm_i915_gem_object *obj;
 	struct file_stats stats;
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	int j;
 
 	memset(&stats, 0, sizeof(stats));
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		for (j = 0; j < ARRAY_SIZE(engine->batch_pool.cache_list); j++) {
 			list_for_each_entry(obj,
 					    &engine->batch_pool.cache_list[j],
@@ -402,23 +383,23 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
 	if (ret)
 		return ret;
 
-	seq_printf(m, "%u objects, %zu bytes\n",
+	seq_printf(m, "%u objects, %llu bytes\n",
 		   dev_priv->mm.object_count,
 		   dev_priv->mm.object_memory);
 
 	size = count = 0;
 	mapped_size = mapped_count = 0;
 	purgeable_size = purgeable_count = 0;
-	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
+	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) {
 		size += obj->base.size;
 		++count;
 
-		if (obj->madv == I915_MADV_DONTNEED) {
+		if (obj->mm.madv == I915_MADV_DONTNEED) {
 			purgeable_size += obj->base.size;
 			++purgeable_count;
 		}
 
-		if (obj->mapping) {
+		if (obj->mm.mapping) {
 			mapped_count++;
 			mapped_size += obj->base.size;
 		}
@@ -426,7 +407,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
 	seq_printf(m, "%u unbound objects, %llu bytes\n", count, size);
 
 	size = count = dpy_size = dpy_count = 0;
-	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
+	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
 		size += obj->base.size;
 		++count;
 
@@ -435,12 +416,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
 			++dpy_count;
 		}
 
-		if (obj->madv == I915_MADV_DONTNEED) {
+		if (obj->mm.madv == I915_MADV_DONTNEED) {
 			purgeable_size += obj->base.size;
 			++purgeable_count;
 		}
 
-		if (obj->mapping) {
+		if (obj->mm.mapping) {
 			mapped_count++;
 			mapped_size += obj->base.size;
 		}
@@ -512,7 +493,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data)
 		return ret;
 
 	total_obj_size = total_gtt_size = count = 0;
-	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
+	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
 		if (show_pin_display_only && !obj->pin_display)
 			continue;
 
@@ -566,12 +547,12 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
 					   pipe, plane);
 			}
 			if (work->flip_queued_req) {
-				struct intel_engine_cs *engine = i915_gem_request_get_engine(work->flip_queued_req);
+				struct intel_engine_cs *engine = work->flip_queued_req->engine;
 
 				seq_printf(m, "Flip queued on %s at seqno %x, next seqno %x [current breadcrumb %x], completed? %d\n",
 					   engine->name,
-					   i915_gem_request_get_seqno(work->flip_queued_req),
-					   dev_priv->next_seqno,
+					   work->flip_queued_req->global_seqno,
+					   atomic_read(&dev_priv->gt.global_timeline.next_seqno),
 					   intel_engine_get_seqno(engine),
 					   i915_gem_request_completed(work->flip_queued_req));
 			} else
@@ -607,6 +588,7 @@ static int i915_gem_batch_pool_info(struct seq_file *m, void *data)
 	struct drm_device *dev = &dev_priv->drm;
 	struct drm_i915_gem_object *obj;
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	int total = 0;
 	int ret, j;
 
@@ -614,7 +596,7 @@ static int i915_gem_batch_pool_info(struct seq_file *m, void *data)
 	if (ret)
 		return ret;
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		for (j = 0; j < ARRAY_SIZE(engine->batch_pool.cache_list); j++) {
 			int count;
 
@@ -645,12 +627,24 @@ static int i915_gem_batch_pool_info(struct seq_file *m, void *data)
 	return 0;
 }
 
+static void print_request(struct seq_file *m,
+			  struct drm_i915_gem_request *rq,
+			  const char *prefix)
+{
+	seq_printf(m, "%s%x [%x:%x] prio=%d @ %dms: %s\n", prefix,
+		   rq->global_seqno, rq->ctx->hw_id, rq->fence.seqno,
+		   rq->priotree.priority,
+		   jiffies_to_msecs(jiffies - rq->emitted_jiffies),
+		   rq->timeline->common->name);
+}
+
 static int i915_gem_request_info(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct drm_device *dev = &dev_priv->drm;
-	struct intel_engine_cs *engine;
 	struct drm_i915_gem_request *req;
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	int ret, any;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -658,29 +652,18 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
 		return ret;
 
 	any = 0;
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		int count;
 
 		count = 0;
-		list_for_each_entry(req, &engine->request_list, link)
+		list_for_each_entry(req, &engine->timeline->requests, link)
 			count++;
 		if (count == 0)
 			continue;
 
 		seq_printf(m, "%s requests: %d\n", engine->name, count);
-		list_for_each_entry(req, &engine->request_list, link) {
-			struct pid *pid = req->ctx->pid;
-			struct task_struct *task;
-
-			rcu_read_lock();
-			task = pid ? pid_task(pid, PIDTYPE_PID) : NULL;
-			seq_printf(m, "    %x @ %d: %s [%d]\n",
-				   req->fence.seqno,
-				   (int) (jiffies - req->emitted_jiffies),
-				   task ? task->comm : "<unknown>",
-				   task ? task->pid : -1);
-			rcu_read_unlock();
-		}
+		list_for_each_entry(req, &engine->timeline->requests, link)
+			print_request(m, req, "    ");
 
 		any++;
 	}
@@ -701,22 +684,23 @@ static void i915_ring_seqno_info(struct seq_file *m,
 	seq_printf(m, "Current sequence (%s): %x\n",
 		   engine->name, intel_engine_get_seqno(engine));
 
-	spin_lock(&b->lock);
+	spin_lock_irq(&b->lock);
 	for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
 		struct intel_wait *w = container_of(rb, typeof(*w), node);
 
 		seq_printf(m, "Waiting (%s): %s [%d] on %x\n",
 			   engine->name, w->tsk->comm, w->tsk->pid, w->seqno);
 	}
-	spin_unlock(&b->lock);
+	spin_unlock_irq(&b->lock);
 }
 
 static int i915_gem_seqno_info(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		i915_ring_seqno_info(m, engine);
 
 	return 0;
@@ -727,6 +711,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	int i, pipe;
 
 	intel_runtime_pm_get(dev_priv);
@@ -743,17 +728,32 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 			   I915_READ(VLV_IIR_RW));
 		seq_printf(m, "Display IMR:\t%08x\n",
 			   I915_READ(VLV_IMR));
-		for_each_pipe(dev_priv, pipe)
+		for_each_pipe(dev_priv, pipe) {
+			enum intel_display_power_domain power_domain;
+
+			power_domain = POWER_DOMAIN_PIPE(pipe);
+			if (!intel_display_power_get_if_enabled(dev_priv,
+								power_domain)) {
+				seq_printf(m, "Pipe %c power disabled\n",
+					   pipe_name(pipe));
+				continue;
+			}
+
 			seq_printf(m, "Pipe %c stat:\t%08x\n",
 				   pipe_name(pipe),
 				   I915_READ(PIPESTAT(pipe)));
 
+			intel_display_power_put(dev_priv, power_domain);
+		}
+
+		intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
 		seq_printf(m, "Port hotplug:\t%08x\n",
 			   I915_READ(PORT_HOTPLUG_EN));
 		seq_printf(m, "DPFLIPSTAT:\t%08x\n",
 			   I915_READ(VLV_DPFLIPSTAT));
 		seq_printf(m, "DPINVGTT:\t%08x\n",
 			   I915_READ(DPINVGTT));
+		intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
 
 		for (i = 0; i < 4; i++) {
 			seq_printf(m, "GT Interrupt IMR %d:\t%08x\n",
@@ -895,7 +895,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 		seq_printf(m, "Graphics Interrupt mask:		%08x\n",
 			   I915_READ(GTIMR));
 	}
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		if (INTEL_GEN(dev_priv) >= 6) {
 			seq_printf(m,
 				   "Graphics Interrupt mask (%s):	%08x\n",
@@ -935,26 +935,7 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
 	return 0;
 }
 
-static int i915_hws_info(struct seq_file *m, void *data)
-{
-	struct drm_info_node *node = m->private;
-	struct drm_i915_private *dev_priv = node_to_i915(node);
-	struct intel_engine_cs *engine;
-	const u32 *hws;
-	int i;
-
-	engine = &dev_priv->engine[(uintptr_t)node->info_ent->data];
-	hws = engine->status_page.page_addr;
-	if (hws == NULL)
-		return 0;
-
-	for (i = 0; i < 4096 / sizeof(u32) / 4; i += 4) {
-		seq_printf(m, "0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n",
-			   i * 4,
-			   hws[i], hws[i + 1], hws[i + 2], hws[i + 3]);
-	}
-	return 0;
-}
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
 
 static ssize_t
 i915_error_state_write(struct file *filp,
@@ -1038,19 +1019,14 @@ static const struct file_operations i915_error_state_fops = {
 	.release = i915_error_state_release,
 };
 
+#endif
+
 static int
 i915_next_seqno_get(void *data, u64 *val)
 {
 	struct drm_i915_private *dev_priv = data;
-	int ret;
-
-	ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
-	if (ret)
-		return ret;
-
-	*val = dev_priv->next_seqno;
-	mutex_unlock(&dev_priv->drm.struct_mutex);
 
+	*val = 1 + atomic_read(&dev_priv->gt.global_timeline.next_seqno);
 	return 0;
 }
 
@@ -1065,7 +1041,7 @@ i915_next_seqno_set(void *data, u64 val)
 	if (ret)
 		return ret;
 
-	ret = i915_gem_set_seqno(dev, val);
+	ret = i915_gem_set_global_seqno(dev, val);
 	mutex_unlock(&dev->struct_mutex);
 
 	return ret;
@@ -1277,15 +1253,42 @@ out:
 	return ret;
 }
 
+static void i915_instdone_info(struct drm_i915_private *dev_priv,
+			       struct seq_file *m,
+			       struct intel_instdone *instdone)
+{
+	int slice;
+	int subslice;
+
+	seq_printf(m, "\t\tINSTDONE: 0x%08x\n",
+		   instdone->instdone);
+
+	if (INTEL_GEN(dev_priv) <= 3)
+		return;
+
+	seq_printf(m, "\t\tSC_INSTDONE: 0x%08x\n",
+		   instdone->slice_common);
+
+	if (INTEL_GEN(dev_priv) <= 6)
+		return;
+
+	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
+		seq_printf(m, "\t\tSAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
+			   slice, subslice, instdone->sampler[slice][subslice]);
+
+	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
+		seq_printf(m, "\t\tROW_INSTDONE[%d][%d]: 0x%08x\n",
+			   slice, subslice, instdone->row[slice][subslice]);
+}
+
 static int i915_hangcheck_info(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct intel_engine_cs *engine;
 	u64 acthd[I915_NUM_ENGINES];
 	u32 seqno[I915_NUM_ENGINES];
-	u32 instdone[I915_NUM_INSTDONE_REG];
+	struct intel_instdone instdone;
 	enum intel_engine_id id;
-	int j;
 
 	if (test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
 		seq_printf(m, "Wedged\n");
@@ -1303,12 +1306,12 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 
 	intel_runtime_pm_get(dev_priv);
 
-	for_each_engine_id(engine, dev_priv, id) {
+	for_each_engine(engine, dev_priv, id) {
 		acthd[id] = intel_engine_get_active_head(engine);
 		seqno[id] = intel_engine_get_seqno(engine);
 	}
 
-	i915_get_extra_instdone(dev_priv, instdone);
+	intel_engine_get_instdone(dev_priv->engine[RCS], &instdone);
 
 	intel_runtime_pm_put(dev_priv);
 
@@ -1319,16 +1322,27 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 	} else
 		seq_printf(m, "Hangcheck inactive\n");
 
-	for_each_engine_id(engine, dev_priv, id) {
+	for_each_engine(engine, dev_priv, id) {
+		struct intel_breadcrumbs *b = &engine->breadcrumbs;
+		struct rb_node *rb;
+
 		seq_printf(m, "%s:\n", engine->name);
 		seq_printf(m, "\tseqno = %x [current %x, last %x]\n",
-			   engine->hangcheck.seqno,
-			   seqno[id],
-			   engine->last_submitted_seqno);
+			   engine->hangcheck.seqno, seqno[id],
+			   intel_engine_last_submit(engine));
 		seq_printf(m, "\twaiters? %s, fake irq active? %s\n",
 			   yesno(intel_engine_has_waiter(engine)),
 			   yesno(test_bit(engine->id,
 					  &dev_priv->gpu_error.missed_irq_rings)));
+		spin_lock_irq(&b->lock);
+		for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
+			struct intel_wait *w = container_of(rb, typeof(*w), node);
+
+			seq_printf(m, "\t%s [%d] waiting for %x\n",
+				   w->tsk->comm, w->tsk->pid, w->seqno);
+		}
+		spin_unlock_irq(&b->lock);
+
 		seq_printf(m, "\tACTHD = 0x%08llx [current 0x%08llx]\n",
 			   (long long)engine->hangcheck.acthd,
 			   (long long)acthd[id]);
@@ -1336,18 +1350,14 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 		seq_printf(m, "\taction = %d\n", engine->hangcheck.action);
 
 		if (engine->id == RCS) {
-			seq_puts(m, "\tinstdone read =");
+			seq_puts(m, "\tinstdone read =\n");
 
-			for (j = 0; j < I915_NUM_INSTDONE_REG; j++)
-				seq_printf(m, " 0x%08x", instdone[j]);
+			i915_instdone_info(dev_priv, m, &instdone);
 
-			seq_puts(m, "\n\tinstdone accu =");
+			seq_puts(m, "\tinstdone accu =\n");
 
-			for (j = 0; j < I915_NUM_INSTDONE_REG; j++)
-				seq_printf(m, " 0x%08x",
-					   engine->hangcheck.instdone[j]);
-
-			seq_puts(m, "\n");
+			i915_instdone_info(dev_priv, m,
+					   &engine->hangcheck.instdone);
 		}
 	}
 
@@ -1357,14 +1367,9 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 static int ironlake_drpc_info(struct seq_file *m)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	struct drm_device *dev = &dev_priv->drm;
 	u32 rgvmodectl, rstdbyctl;
 	u16 crstandvid;
-	int ret;
 
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
 	intel_runtime_pm_get(dev_priv);
 
 	rgvmodectl = I915_READ(MEMMODECTL);
@@ -1372,7 +1377,6 @@ static int ironlake_drpc_info(struct seq_file *m)
 	crstandvid = I915_READ16(CRSTANDVID);
 
 	intel_runtime_pm_put(dev_priv);
-	mutex_unlock(&dev->struct_mutex);
 
 	seq_printf(m, "HD boost: %s\n", yesno(rgvmodectl & MEMMODE_BOOST_EN));
 	seq_printf(m, "Boost freq: %d\n",
@@ -1635,10 +1639,13 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
 		seq_printf(m, "FBC disabled: %s\n",
 			   dev_priv->fbc.no_fbc_reason);
 
-	if (INTEL_GEN(dev_priv) >= 7)
+	if (intel_fbc_is_active(dev_priv) && INTEL_GEN(dev_priv) >= 7) {
+		uint32_t mask = INTEL_GEN(dev_priv) >= 8 ?
+				BDW_FBC_COMPRESSION_MASK :
+				IVB_FBC_COMPRESSION_MASK;
 		seq_printf(m, "Compressing: %s\n",
-			   yesno(I915_READ(FBC_STATUS2) &
-				 FBC_COMPRESSION_MASK));
+			   yesno(I915_READ(FBC_STATUS2) & mask));
+	}
 
 	mutex_unlock(&dev_priv->fbc.lock);
 	intel_runtime_pm_put(dev_priv);
@@ -1717,6 +1724,7 @@ static int i915_sr_status(struct seq_file *m, void *unused)
 	bool sr_enabled = false;
 
 	intel_runtime_pm_get(dev_priv);
+	intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
 
 	if (HAS_PCH_SPLIT(dev_priv))
 		sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN;
@@ -1730,10 +1738,10 @@ static int i915_sr_status(struct seq_file *m, void *unused)
 	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		sr_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 
+	intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
 	intel_runtime_pm_put(dev_priv);
 
-	seq_printf(m, "self-refresh: %s\n",
-		   sr_enabled ? "enabled" : "disabled");
+	seq_printf(m, "self-refresh: %s\n", enableddisabled(sr_enabled));
 
 	return 0;
 }
@@ -1867,7 +1875,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
 			   fbdev_fb->base.height,
 			   fbdev_fb->base.depth,
 			   fbdev_fb->base.bits_per_pixel,
-			   fbdev_fb->base.modifier[0],
+			   fbdev_fb->base.modifier,
 			   drm_framebuffer_read_refcount(&fbdev_fb->base));
 		describe_obj(m, fbdev_fb->obj);
 		seq_putc(m, '\n');
@@ -1885,7 +1893,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
 			   fb->base.height,
 			   fb->base.depth,
 			   fb->base.bits_per_pixel,
-			   fb->base.modifier[0],
+			   fb->base.modifier,
 			   drm_framebuffer_read_refcount(&fb->base));
 		describe_obj(m, fb->obj);
 		seq_putc(m, '\n');
@@ -1909,6 +1917,7 @@ static int i915_context_status(struct seq_file *m, void *unused)
 	struct drm_device *dev = &dev_priv->drm;
 	struct intel_engine_cs *engine;
 	struct i915_gem_context *ctx;
+	enum intel_engine_id id;
 	int ret;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -1935,7 +1944,7 @@ static int i915_context_status(struct seq_file *m, void *unused)
 		seq_putc(m, ctx->remap_slice ? 'R' : 'r');
 		seq_putc(m, '\n');
 
-		for_each_engine(engine, dev_priv) {
+		for_each_engine(engine, dev_priv, id) {
 			struct intel_context *ce = &ctx->engine[engine->id];
 
 			seq_printf(m, "%s: ", engine->name);
@@ -1974,7 +1983,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,
 		seq_printf(m, "\tBound in GGTT at 0x%08x\n",
 			   i915_ggtt_offset(vma));
 
-	if (i915_gem_object_get_pages(vma->obj)) {
+	if (i915_gem_object_pin_pages(vma->obj)) {
 		seq_puts(m, "\tFailed to get pages for context object\n\n");
 		return;
 	}
@@ -1993,6 +2002,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,
 		kunmap_atomic(reg_state);
 	}
 
+	i915_gem_object_unpin_pages(vma->obj);
 	seq_putc(m, '\n');
 }
 
@@ -2002,6 +2012,7 @@ static int i915_dump_lrc(struct seq_file *m, void *unused)
 	struct drm_device *dev = &dev_priv->drm;
 	struct intel_engine_cs *engine;
 	struct i915_gem_context *ctx;
+	enum intel_engine_id id;
 	int ret;
 
 	if (!i915.enable_execlists) {
@@ -2014,7 +2025,7 @@ static int i915_dump_lrc(struct seq_file *m, void *unused)
 		return ret;
 
 	list_for_each_entry(ctx, &dev_priv->context_list, link)
-		for_each_engine(engine, dev_priv)
+		for_each_engine(engine, dev_priv, id)
 			i915_dump_lrc_obj(m, ctx, engine);
 
 	mutex_unlock(&dev->struct_mutex);
@@ -2022,84 +2033,6 @@ static int i915_dump_lrc(struct seq_file *m, void *unused)
 	return 0;
 }
 
-static int i915_execlists(struct seq_file *m, void *data)
-{
-	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	struct drm_device *dev = &dev_priv->drm;
-	struct intel_engine_cs *engine;
-	u32 status_pointer;
-	u8 read_pointer;
-	u8 write_pointer;
-	u32 status;
-	u32 ctx_id;
-	struct list_head *cursor;
-	int i, ret;
-
-	if (!i915.enable_execlists) {
-		seq_puts(m, "Logical Ring Contexts are disabled\n");
-		return 0;
-	}
-
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
-
-	intel_runtime_pm_get(dev_priv);
-
-	for_each_engine(engine, dev_priv) {
-		struct drm_i915_gem_request *head_req = NULL;
-		int count = 0;
-
-		seq_printf(m, "%s\n", engine->name);
-
-		status = I915_READ(RING_EXECLIST_STATUS_LO(engine));
-		ctx_id = I915_READ(RING_EXECLIST_STATUS_HI(engine));
-		seq_printf(m, "\tExeclist status: 0x%08X, context: %u\n",
-			   status, ctx_id);
-
-		status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
-		seq_printf(m, "\tStatus pointer: 0x%08X\n", status_pointer);
-
-		read_pointer = GEN8_CSB_READ_PTR(status_pointer);
-		write_pointer = GEN8_CSB_WRITE_PTR(status_pointer);
-		if (read_pointer > write_pointer)
-			write_pointer += GEN8_CSB_ENTRIES;
-		seq_printf(m, "\tRead pointer: 0x%08X, write pointer 0x%08X\n",
-			   read_pointer, write_pointer);
-
-		for (i = 0; i < GEN8_CSB_ENTRIES; i++) {
-			status = I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, i));
-			ctx_id = I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, i));
-
-			seq_printf(m, "\tStatus buffer %d: 0x%08X, context: %u\n",
-				   i, status, ctx_id);
-		}
-
-		spin_lock_bh(&engine->execlist_lock);
-		list_for_each(cursor, &engine->execlist_queue)
-			count++;
-		head_req = list_first_entry_or_null(&engine->execlist_queue,
-						    struct drm_i915_gem_request,
-						    execlist_link);
-		spin_unlock_bh(&engine->execlist_lock);
-
-		seq_printf(m, "\t%d requests in queue\n", count);
-		if (head_req) {
-			seq_printf(m, "\tHead request context: %u\n",
-				   head_req->ctx->hw_id);
-			seq_printf(m, "\tHead request tail: %u\n",
-				   head_req->tail);
-		}
-
-		seq_putc(m, '\n');
-	}
-
-	intel_runtime_pm_put(dev_priv);
-	mutex_unlock(&dev->struct_mutex);
-
-	return 0;
-}
-
 static const char *swizzle_string(unsigned swizzle)
 {
 	switch (swizzle) {
@@ -2127,12 +2060,7 @@ static const char *swizzle_string(unsigned swizzle)
 static int i915_swizzle_info(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	struct drm_device *dev = &dev_priv->drm;
-	int ret;
 
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
 	intel_runtime_pm_get(dev_priv);
 
 	seq_printf(m, "bit6 swizzle for X-tiling = %s\n",
@@ -2172,7 +2100,6 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
 		seq_puts(m, "L-shaped memory detected\n");
 
 	intel_runtime_pm_put(dev_priv);
-	mutex_unlock(&dev->struct_mutex);
 
 	return 0;
 }
@@ -2201,14 +2128,15 @@ static int per_file_ctx(int id, void *ptr, void *data)
 static void gen8_ppgtt_info(struct seq_file *m,
 			    struct drm_i915_private *dev_priv)
 {
-	struct intel_engine_cs *engine;
 	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	int i;
 
 	if (!ppgtt)
 		return;
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		seq_printf(m, "%s\n", engine->name);
 		for (i = 0; i < 4; i++) {
 			u64 pdp = I915_READ(GEN8_RING_PDP_UDW(engine, i));
@@ -2223,11 +2151,12 @@ static void gen6_ppgtt_info(struct seq_file *m,
 			    struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 
 	if (IS_GEN6(dev_priv))
 		seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(GFX_MODE));
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		seq_printf(m, "%s\n", engine->name);
 		if (IS_GEN7(dev_priv))
 			seq_printf(m, "GFX_MODE: 0x%08x\n",
@@ -2296,9 +2225,10 @@ out_unlock:
 static int count_irq_waiters(struct drm_i915_private *i915)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	int count = 0;
 
-	for_each_engine(engine, i915)
+	for_each_engine(engine, i915, id)
 		count += intel_engine_has_waiter(engine);
 
 	return count;
@@ -2325,8 +2255,8 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
 	struct drm_file *file;
 
 	seq_printf(m, "RPS enabled? %d\n", dev_priv->rps.enabled);
-	seq_printf(m, "GPU busy? %s [%x]\n",
-		   yesno(dev_priv->gt.awake), dev_priv->gt.active_engines);
+	seq_printf(m, "GPU busy? %s [%d requests]\n",
+		   yesno(dev_priv->gt.awake), dev_priv->gt.active_requests);
 	seq_printf(m, "CPU waiting? %d\n", count_irq_waiters(dev_priv));
 	seq_printf(m, "Frequency requested %d\n",
 		   intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq));
@@ -2361,7 +2291,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
 
 	if (INTEL_GEN(dev_priv) >= 6 &&
 	    dev_priv->rps.enabled &&
-	    dev_priv->gt.active_engines) {
+	    dev_priv->gt.active_requests) {
 		u32 rpup, rpupei;
 		u32 rpdown, rpdownei;
 
@@ -2442,6 +2372,32 @@ static int i915_guc_load_status_info(struct seq_file *m, void *data)
 	return 0;
 }
 
+static void i915_guc_log_info(struct seq_file *m,
+			      struct drm_i915_private *dev_priv)
+{
+	struct intel_guc *guc = &dev_priv->guc;
+
+	seq_puts(m, "\nGuC logging stats:\n");
+
+	seq_printf(m, "\tISR:   flush count %10u, overflow count %10u\n",
+		   guc->log.flush_count[GUC_ISR_LOG_BUFFER],
+		   guc->log.total_overflow_count[GUC_ISR_LOG_BUFFER]);
+
+	seq_printf(m, "\tDPC:   flush count %10u, overflow count %10u\n",
+		   guc->log.flush_count[GUC_DPC_LOG_BUFFER],
+		   guc->log.total_overflow_count[GUC_DPC_LOG_BUFFER]);
+
+	seq_printf(m, "\tCRASH: flush count %10u, overflow count %10u\n",
+		   guc->log.flush_count[GUC_CRASH_DUMP_LOG_BUFFER],
+		   guc->log.total_overflow_count[GUC_CRASH_DUMP_LOG_BUFFER]);
+
+	seq_printf(m, "\tTotal flush interrupt count: %u\n",
+		   guc->log.flush_interrupt_count);
+
+	seq_printf(m, "\tCapture miss count: %u\n",
+		   guc->log.capture_miss_count);
+}
+
 static void i915_guc_client_info(struct seq_file *m,
 				 struct drm_i915_private *dev_priv,
 				 struct i915_guc_client *client)
@@ -2461,7 +2417,7 @@ static void i915_guc_client_info(struct seq_file *m,
 	seq_printf(m, "\tFailed doorbell: %u\n", client->b_fail);
 	seq_printf(m, "\tLast submission result: %d\n", client->retcode);
 
-	for_each_engine_id(engine, dev_priv, id) {
+	for_each_engine(engine, dev_priv, id) {
 		u64 submissions = client->submissions[id];
 		tot += submissions;
 		seq_printf(m, "\tSubmissions: %llu %s\n",
@@ -2504,7 +2460,7 @@ static int i915_guc_info(struct seq_file *m, void *data)
 	seq_printf(m, "GuC last action error code: %d\n", guc.action_err);
 
 	seq_printf(m, "\nGuC submissions:\n");
-	for_each_engine_id(engine, dev_priv, id) {
+	for_each_engine(engine, dev_priv, id) {
 		u64 submissions = guc.submissions[id];
 		total += submissions;
 		seq_printf(m, "\t%-24s: %10llu, last seqno 0x%08x\n",
@@ -2515,6 +2471,8 @@ static int i915_guc_info(struct seq_file *m, void *data)
 	seq_printf(m, "\nGuC execbuf client @ %p:\n", guc.execbuf_client);
 	i915_guc_client_info(m, dev_priv, &client);
 
+	i915_guc_log_info(m, dev_priv);
+
 	/* Add more as required ... */
 
 	return 0;
@@ -2526,10 +2484,10 @@ static int i915_guc_log_dump(struct seq_file *m, void *data)
 	struct drm_i915_gem_object *obj;
 	int i = 0, pg;
 
-	if (!dev_priv->guc.log_vma)
+	if (!dev_priv->guc.log.vma)
 		return 0;
 
-	obj = dev_priv->guc.log_vma->obj;
+	obj = dev_priv->guc.log.vma->obj;
 	for (pg = 0; pg < obj->base.size / PAGE_SIZE; pg++) {
 		u32 *log = kmap_atomic(i915_gem_object_get_page(obj, pg));
 
@@ -2546,6 +2504,44 @@ static int i915_guc_log_dump(struct seq_file *m, void *data)
 	return 0;
 }
 
+static int i915_guc_log_control_get(void *data, u64 *val)
+{
+	struct drm_device *dev = data;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+
+	if (!dev_priv->guc.log.vma)
+		return -EINVAL;
+
+	*val = i915.guc_log_level;
+
+	return 0;
+}
+
+static int i915_guc_log_control_set(void *data, u64 val)
+{
+	struct drm_device *dev = data;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	int ret;
+
+	if (!dev_priv->guc.log.vma)
+		return -EINVAL;
+
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	intel_runtime_pm_get(dev_priv);
+	ret = i915_guc_log_control(dev_priv, val);
+	intel_runtime_pm_put(dev_priv);
+
+	mutex_unlock(&dev->struct_mutex);
+	return ret;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(i915_guc_log_control_fops,
+			i915_guc_log_control_get, i915_guc_log_control_set,
+			"%lld\n");
+
 static int i915_edp_psr_status(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -2575,11 +2571,22 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 		enabled = I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE;
 	else {
 		for_each_pipe(dev_priv, pipe) {
+			enum transcoder cpu_transcoder =
+				intel_pipe_to_cpu_transcoder(dev_priv, pipe);
+			enum intel_display_power_domain power_domain;
+
+			power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
+			if (!intel_display_power_get_if_enabled(dev_priv,
+								power_domain))
+				continue;
+
 			stat[pipe] = I915_READ(VLV_PSRSTAT(pipe)) &
 				VLV_EDP_PSR_CURR_STATE_MASK;
 			if ((stat[pipe] == VLV_EDP_PSR_ACTIVE_NORFB_UP) ||
 			    (stat[pipe] == VLV_EDP_PSR_ACTIVE_SF_UPDATE))
 				enabled = true;
+
+			intel_display_power_put(dev_priv, power_domain);
 		}
 	}
 
@@ -3004,7 +3011,7 @@ static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc)
 	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
 		struct drm_plane_state *state;
 		struct drm_plane *plane = &intel_plane->base;
-		char *format_name;
+		struct drm_format_name_buf format_name;
 
 		if (!plane->state) {
 			seq_puts(m, "plane->state is NULL!\n");
@@ -3014,9 +3021,9 @@ static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc)
 		state = plane->state;
 
 		if (state->fb) {
-			format_name = drm_get_format_name(state->fb->pixel_format);
+			drm_get_format_name(state->fb->pixel_format, &format_name);
 		} else {
-			format_name = kstrdup("N/A", GFP_KERNEL);
+			sprintf(format_name.str, "N/A");
 		}
 
 		seq_printf(m, "\t--Plane id %d: type=%s, crtc_pos=%4dx%4d, crtc_size=%4dx%4d, src_pos=%d.%04ux%d.%04u, src_size=%d.%04ux%d.%04u, format=%s, rotation=%s\n",
@@ -3032,10 +3039,8 @@ static void intel_plane_info(struct seq_file *m, struct intel_crtc *intel_crtc)
 			   ((state->src_w & 0xffff) * 15625) >> 10,
 			   (state->src_h >> 16),
 			   ((state->src_h & 0xffff) * 15625) >> 10,
-			   format_name,
+			   format_name.str,
 			   plane_rotation(state->rotation));
-
-		kfree(format_name);
 	}
 }
 
@@ -3121,6 +3126,146 @@ static int i915_display_info(struct seq_file *m, void *unused)
 	return 0;
 }
 
+static int i915_engine_info(struct seq_file *m, void *unused)
+{
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	intel_runtime_pm_get(dev_priv);
+
+	for_each_engine(engine, dev_priv, id) {
+		struct intel_breadcrumbs *b = &engine->breadcrumbs;
+		struct drm_i915_gem_request *rq;
+		struct rb_node *rb;
+		u64 addr;
+
+		seq_printf(m, "%s\n", engine->name);
+		seq_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [score %d]\n",
+			   intel_engine_get_seqno(engine),
+			   intel_engine_last_submit(engine),
+			   engine->hangcheck.seqno,
+			   engine->hangcheck.score);
+
+		rcu_read_lock();
+
+		seq_printf(m, "\tRequests:\n");
+
+		rq = list_first_entry(&engine->timeline->requests,
+				      struct drm_i915_gem_request, link);
+		if (&rq->link != &engine->timeline->requests)
+			print_request(m, rq, "\t\tfirst  ");
+
+		rq = list_last_entry(&engine->timeline->requests,
+				     struct drm_i915_gem_request, link);
+		if (&rq->link != &engine->timeline->requests)
+			print_request(m, rq, "\t\tlast   ");
+
+		rq = i915_gem_find_active_request(engine);
+		if (rq) {
+			print_request(m, rq, "\t\tactive ");
+			seq_printf(m,
+				   "\t\t[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]\n",
+				   rq->head, rq->postfix, rq->tail,
+				   rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u,
+				   rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u);
+		}
+
+		seq_printf(m, "\tRING_START: 0x%08x [0x%08x]\n",
+			   I915_READ(RING_START(engine->mmio_base)),
+			   rq ? i915_ggtt_offset(rq->ring->vma) : 0);
+		seq_printf(m, "\tRING_HEAD:  0x%08x [0x%08x]\n",
+			   I915_READ(RING_HEAD(engine->mmio_base)) & HEAD_ADDR,
+			   rq ? rq->ring->head : 0);
+		seq_printf(m, "\tRING_TAIL:  0x%08x [0x%08x]\n",
+			   I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR,
+			   rq ? rq->ring->tail : 0);
+		seq_printf(m, "\tRING_CTL:   0x%08x [%s]\n",
+			   I915_READ(RING_CTL(engine->mmio_base)),
+			   I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? "waiting" : "");
+
+		rcu_read_unlock();
+
+		addr = intel_engine_get_active_head(engine);
+		seq_printf(m, "\tACTHD:  0x%08x_%08x\n",
+			   upper_32_bits(addr), lower_32_bits(addr));
+		addr = intel_engine_get_last_batch_head(engine);
+		seq_printf(m, "\tBBADDR: 0x%08x_%08x\n",
+			   upper_32_bits(addr), lower_32_bits(addr));
+
+		if (i915.enable_execlists) {
+			u32 ptr, read, write;
+			struct rb_node *rb;
+
+			seq_printf(m, "\tExeclist status: 0x%08x %08x\n",
+				   I915_READ(RING_EXECLIST_STATUS_LO(engine)),
+				   I915_READ(RING_EXECLIST_STATUS_HI(engine)));
+
+			ptr = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
+			read = GEN8_CSB_READ_PTR(ptr);
+			write = GEN8_CSB_WRITE_PTR(ptr);
+			seq_printf(m, "\tExeclist CSB read %d, write %d\n",
+				   read, write);
+			if (read >= GEN8_CSB_ENTRIES)
+				read = 0;
+			if (write >= GEN8_CSB_ENTRIES)
+				write = 0;
+			if (read > write)
+				write += GEN8_CSB_ENTRIES;
+			while (read < write) {
+				unsigned int idx = ++read % GEN8_CSB_ENTRIES;
+
+				seq_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
+					   idx,
+					   I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)),
+					   I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)));
+			}
+
+			rcu_read_lock();
+			rq = READ_ONCE(engine->execlist_port[0].request);
+			if (rq)
+				print_request(m, rq, "\t\tELSP[0] ");
+			else
+				seq_printf(m, "\t\tELSP[0] idle\n");
+			rq = READ_ONCE(engine->execlist_port[1].request);
+			if (rq)
+				print_request(m, rq, "\t\tELSP[1] ");
+			else
+				seq_printf(m, "\t\tELSP[1] idle\n");
+			rcu_read_unlock();
+
+			spin_lock_irq(&engine->timeline->lock);
+			for (rb = engine->execlist_first; rb; rb = rb_next(rb)) {
+				rq = rb_entry(rb, typeof(*rq), priotree.node);
+				print_request(m, rq, "\t\tQ ");
+			}
+			spin_unlock_irq(&engine->timeline->lock);
+		} else if (INTEL_GEN(dev_priv) > 6) {
+			seq_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
+				   I915_READ(RING_PP_DIR_BASE(engine)));
+			seq_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
+				   I915_READ(RING_PP_DIR_BASE_READ(engine)));
+			seq_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
+				   I915_READ(RING_PP_DIR_DCLV(engine)));
+		}
+
+		spin_lock_irq(&b->lock);
+		for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
+			struct intel_wait *w = container_of(rb, typeof(*w), node);
+
+			seq_printf(m, "\t%s [%d] waiting for %x\n",
+				   w->tsk->comm, w->tsk->pid, w->seqno);
+		}
+		spin_unlock_irq(&b->lock);
+
+		seq_puts(m, "\n");
+	}
+
+	intel_runtime_pm_put(dev_priv);
+
+	return 0;
+}
+
 static int i915_semaphore_status(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -3147,7 +3292,7 @@ static int i915_semaphore_status(struct seq_file *m, void *unused)
 		page = i915_gem_object_get_page(dev_priv->semaphore->obj, 0);
 
 		seqno = (uint64_t *)kmap_atomic(page);
-		for_each_engine_id(engine, dev_priv, id) {
+		for_each_engine(engine, dev_priv, id) {
 			uint64_t offset;
 
 			seq_printf(m, "%s\n", engine->name);
@@ -3172,22 +3317,13 @@ static int i915_semaphore_status(struct seq_file *m, void *unused)
 		kunmap_atomic(seqno);
 	} else {
 		seq_puts(m, "  Last signal:");
-		for_each_engine(engine, dev_priv)
+		for_each_engine(engine, dev_priv, id)
 			for (j = 0; j < num_rings; j++)
 				seq_printf(m, "0x%08x\n",
 					   I915_READ(engine->semaphore.mbox.signal[j]));
 		seq_putc(m, '\n');
 	}
 
-	seq_puts(m, "\nSync seqno:\n");
-	for_each_engine(engine, dev_priv) {
-		for (j = 0; j < num_rings; j++)
-			seq_printf(m, "  0x%08x ",
-				   engine->semaphore.sync_seqno[j]);
-		seq_putc(m, '\n');
-	}
-	seq_putc(m, '\n');
-
 	intel_runtime_pm_put(dev_priv);
 	mutex_unlock(&dev->struct_mutex);
 	return 0;
@@ -3236,7 +3372,7 @@ static int i915_wa_registers(struct seq_file *m, void *unused)
 	intel_runtime_pm_get(dev_priv);
 
 	seq_printf(m, "Workarounds applied: %d\n", workarounds->count);
-	for_each_engine_id(engine, dev_priv, id)
+	for_each_engine(engine, dev_priv, id)
 		seq_printf(m, "HW whitelist count for %s: %d\n",
 			   engine->name, workarounds->hw_whitelist_count[id]);
 	for (i = 0; i < workarounds->count; ++i) {
@@ -3280,7 +3416,7 @@ static int i915_ddb_info(struct seq_file *m, void *unused)
 	for_each_pipe(dev_priv, pipe) {
 		seq_printf(m, "Pipe %c\n", pipe_name(pipe));
 
-		for_each_plane(dev_priv, pipe, plane) {
+		for_each_universal_plane(dev_priv, pipe, plane) {
 			entry = &ddb->plane[pipe][plane];
 			seq_printf(m, "  Plane%-8d%8u%8u%8u\n", plane + 1,
 				   entry->start, entry->end,
@@ -3914,8 +4050,7 @@ static void hsw_trans_edp_pipe_A_crc_wa(struct drm_i915_private *dev_priv,
 					bool enable)
 {
 	struct drm_device *dev = &dev_priv->drm;
-	struct intel_crtc *crtc =
-		to_intel_crtc(dev_priv->pipe_to_crtc_mapping[PIPE_A]);
+	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
 	struct intel_crtc_state *pipe_config;
 	struct drm_atomic_state *state;
 	int ret = 0;
@@ -3941,10 +4076,9 @@ static void hsw_trans_edp_pipe_A_crc_wa(struct drm_i915_private *dev_priv,
 
 	ret = drm_atomic_commit(state);
 out:
-	drm_modeset_unlock_all(dev);
 	WARN(ret, "Toggling workaround to %i returns %i\n", enable, ret);
-	if (ret)
-		drm_atomic_state_free(state);
+	drm_modeset_unlock_all(dev);
+	drm_atomic_state_put(state);
 }
 
 static int ivb_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
@@ -3982,10 +4116,8 @@ static int pipe_crc_set_source(struct drm_i915_private *dev_priv,
 			       enum pipe pipe,
 			       enum intel_pipe_crc_source source)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
-	struct intel_crtc *crtc =
-			to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe));
+	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 	enum intel_display_power_domain power_domain;
 	u32 val = 0; /* shut up gcc */
 	int ret;
@@ -4056,15 +4188,15 @@ static int pipe_crc_set_source(struct drm_i915_private *dev_priv,
 	/* real source -> none transition */
 	if (source == INTEL_PIPE_CRC_SOURCE_NONE) {
 		struct intel_pipe_crc_entry *entries;
-		struct intel_crtc *crtc =
-			to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
+		struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv,
+								  pipe);
 
 		DRM_DEBUG_DRIVER("stopping CRCs for pipe %c\n",
 				 pipe_name(pipe));
 
 		drm_modeset_lock(&crtc->base.mutex, NULL);
 		if (crtc->base.state->active)
-			intel_wait_for_vblank(dev, pipe);
+			intel_wait_for_vblank(dev_priv, pipe);
 		drm_modeset_unlock(&crtc->base.mutex);
 
 		spin_lock_irq(&pipe_crc->lock);
@@ -4463,7 +4595,7 @@ static void wm_latency_show(struct seq_file *m, const uint16_t wm[8])
 	else if (IS_VALLEYVIEW(dev_priv))
 		num_levels = 1;
 	else
-		num_levels = ilk_wm_max_level(dev) + 1;
+		num_levels = ilk_wm_max_level(dev_priv) + 1;
 
 	drm_modeset_lock_all(dev);
 
@@ -4579,7 +4711,7 @@ static ssize_t wm_latency_write(struct file *file, const char __user *ubuf,
 	else if (IS_VALLEYVIEW(dev_priv))
 		num_levels = 1;
 	else
-		num_levels = ilk_wm_max_level(dev) + 1;
+		num_levels = ilk_wm_max_level(dev_priv) + 1;
 
 	if (len >= sizeof(tmp))
 		return -EINVAL;
@@ -4704,13 +4836,9 @@ i915_wedged_set(void *data, u64 val)
 	if (i915_reset_in_progress(&dev_priv->gpu_error))
 		return -EAGAIN;
 
-	intel_runtime_pm_get(dev_priv);
-
 	i915_handle_error(dev_priv, val,
 			  "Manually setting wedged to %llu", val);
 
-	intel_runtime_pm_put(dev_priv);
-
 	return 0;
 }
 
@@ -4778,10 +4906,12 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_ring_test_irq_fops,
 #define DROP_BOUND 0x2
 #define DROP_RETIRE 0x4
 #define DROP_ACTIVE 0x8
-#define DROP_ALL (DROP_UNBOUND | \
-		  DROP_BOUND | \
-		  DROP_RETIRE | \
-		  DROP_ACTIVE)
+#define DROP_FREED 0x10
+#define DROP_ALL (DROP_UNBOUND	| \
+		  DROP_BOUND	| \
+		  DROP_RETIRE	| \
+		  DROP_ACTIVE	| \
+		  DROP_FREED)
 static int
 i915_drop_caches_get(void *data, u64 *val)
 {
@@ -4825,6 +4955,11 @@ i915_drop_caches_set(void *data, u64 val)
 unlock:
 	mutex_unlock(&dev->struct_mutex);
 
+	if (val & DROP_FREED) {
+		synchronize_rcu();
+		flush_work(&dev_priv->mm.free_work);
+	}
+
 	return ret;
 }
 
@@ -4945,22 +5080,16 @@ static int
 i915_cache_sharing_get(void *data, u64 *val)
 {
 	struct drm_i915_private *dev_priv = data;
-	struct drm_device *dev = &dev_priv->drm;
 	u32 snpcr;
-	int ret;
 
 	if (!(IS_GEN6(dev_priv) || IS_GEN7(dev_priv)))
 		return -ENODEV;
 
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
 	intel_runtime_pm_get(dev_priv);
 
 	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
 
 	intel_runtime_pm_put(dev_priv);
-	mutex_unlock(&dev->struct_mutex);
 
 	*val = (snpcr & GEN6_MBC_SNPCR_MASK) >> GEN6_MBC_SNPCR_SHIFT;
 
@@ -5253,10 +5382,6 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_gem_seqno", i915_gem_seqno_info, 0},
 	{"i915_gem_fence_regs", i915_gem_fence_regs_info, 0},
 	{"i915_gem_interrupt", i915_interrupt_info, 0},
-	{"i915_gem_hws", i915_hws_info, 0, (void *)RCS},
-	{"i915_gem_hws_blt", i915_hws_info, 0, (void *)BCS},
-	{"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS},
-	{"i915_gem_hws_vebox", i915_hws_info, 0, (void *)VECS},
 	{"i915_gem_batch_pool", i915_gem_batch_pool_info, 0},
 	{"i915_guc_info", i915_guc_info, 0},
 	{"i915_guc_load_status", i915_guc_load_status_info, 0},
@@ -5275,7 +5400,6 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_gem_framebuffer", i915_gem_framebuffer_info, 0},
 	{"i915_context_status", i915_context_status, 0},
 	{"i915_dump_lrc", i915_dump_lrc, 0},
-	{"i915_execlists", i915_execlists, 0},
 	{"i915_forcewake_domains", i915_forcewake_domains, 0},
 	{"i915_swizzle_info", i915_swizzle_info, 0},
 	{"i915_ppgtt_info", i915_ppgtt_info, 0},
@@ -5287,6 +5411,7 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_power_domain_info", i915_power_domain_info, 0},
 	{"i915_dmc_info", i915_dmc_info, 0},
 	{"i915_display_info", i915_display_info, 0},
+	{"i915_engine_info", i915_engine_info, 0},
 	{"i915_semaphore_status", i915_semaphore_status, 0},
 	{"i915_shared_dplls_info", i915_shared_dplls_info, 0},
 	{"i915_dp_mst_info", i915_dp_mst_info, 0},
@@ -5309,7 +5434,9 @@ static const struct i915_debugfs_files {
 	{"i915_ring_missed_irq", &i915_ring_missed_irq_fops},
 	{"i915_ring_test_irq", &i915_ring_test_irq_fops},
 	{"i915_gem_drop_caches", &i915_drop_caches_fops},
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
 	{"i915_error_state", &i915_error_state_fops},
+#endif
 	{"i915_next_seqno", &i915_next_seqno_fops},
 	{"i915_display_crc_ctl", &i915_display_crc_ctl_fops},
 	{"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
@@ -5318,7 +5445,8 @@ static const struct i915_debugfs_files {
 	{"i915_fbc_false_color", &i915_fbc_fc_fops},
 	{"i915_dp_test_data", &i915_displayport_test_data_fops},
 	{"i915_dp_test_type", &i915_displayport_test_type_fops},
-	{"i915_dp_test_active", &i915_displayport_test_active_fops}
+	{"i915_dp_test_active", &i915_displayport_test_active_fops},
+	{"i915_guc_log_control", &i915_guc_log_control_fops}
 };
 
 void intel_display_crc_init(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 18dfdd5c1b3b..445fec9c2841 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -114,7 +114,7 @@ static bool i915_error_injected(struct drm_i915_private *dev_priv)
 		      fmt, ##__VA_ARGS__)
 
 
-static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
+static enum intel_pch intel_virt_detect_pch(struct drm_i915_private *dev_priv)
 {
 	enum intel_pch ret = PCH_NOP;
 
@@ -125,16 +125,16 @@ static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
 	 * make an educated guess as to which PCH is really there.
 	 */
 
-	if (IS_GEN5(dev)) {
+	if (IS_GEN5(dev_priv)) {
 		ret = PCH_IBX;
 		DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n");
-	} else if (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
+	} else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
 		ret = PCH_CPT;
 		DRM_DEBUG_KMS("Assuming CouarPoint PCH\n");
-	} else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
 		ret = PCH_LPT;
 		DRM_DEBUG_KMS("Assuming LynxPoint PCH\n");
-	} else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+	} else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
 		ret = PCH_SPT;
 		DRM_DEBUG_KMS("Assuming SunrisePoint PCH\n");
 	}
@@ -150,7 +150,7 @@ static void intel_detect_pch(struct drm_device *dev)
 	/* In all current cases, num_pipes is equivalent to the PCH_NOP setting
 	 * (which really amounts to a PCH but no South Display).
 	 */
-	if (INTEL_INFO(dev)->num_pipes == 0) {
+	if (INTEL_INFO(dev_priv)->num_pipes == 0) {
 		dev_priv->pch_type = PCH_NOP;
 		return;
 	}
@@ -174,40 +174,46 @@ static void intel_detect_pch(struct drm_device *dev)
 			if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_IBX;
 				DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
-				WARN_ON(!IS_GEN5(dev));
+				WARN_ON(!IS_GEN5(dev_priv));
 			} else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_CPT;
 				DRM_DEBUG_KMS("Found CougarPoint PCH\n");
-				WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
+				WARN_ON(!(IS_GEN6(dev_priv) ||
+					IS_IVYBRIDGE(dev_priv)));
 			} else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
 				/* PantherPoint is CPT compatible */
 				dev_priv->pch_type = PCH_CPT;
 				DRM_DEBUG_KMS("Found PantherPoint PCH\n");
-				WARN_ON(!(IS_GEN6(dev) || IS_IVYBRIDGE(dev)));
+				WARN_ON(!(IS_GEN6(dev_priv) ||
+					IS_IVYBRIDGE(dev_priv)));
 			} else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_LPT;
 				DRM_DEBUG_KMS("Found LynxPoint PCH\n");
-				WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
-				WARN_ON(IS_HSW_ULT(dev) || IS_BDW_ULT(dev));
+				WARN_ON(!IS_HASWELL(dev_priv) &&
+					!IS_BROADWELL(dev_priv));
+				WARN_ON(IS_HSW_ULT(dev_priv) ||
+					IS_BDW_ULT(dev_priv));
 			} else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_LPT;
 				DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
-				WARN_ON(!IS_HASWELL(dev) && !IS_BROADWELL(dev));
-				WARN_ON(!IS_HSW_ULT(dev) && !IS_BDW_ULT(dev));
+				WARN_ON(!IS_HASWELL(dev_priv) &&
+					!IS_BROADWELL(dev_priv));
+				WARN_ON(!IS_HSW_ULT(dev_priv) &&
+					!IS_BDW_ULT(dev_priv));
 			} else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_SPT;
 				DRM_DEBUG_KMS("Found SunrisePoint PCH\n");
-				WARN_ON(!IS_SKYLAKE(dev) &&
-					!IS_KABYLAKE(dev));
+				WARN_ON(!IS_SKYLAKE(dev_priv) &&
+					!IS_KABYLAKE(dev_priv));
 			} else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_SPT;
 				DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
-				WARN_ON(!IS_SKYLAKE(dev) &&
-					!IS_KABYLAKE(dev));
+				WARN_ON(!IS_SKYLAKE(dev_priv) &&
+					!IS_KABYLAKE(dev_priv));
 			} else if (id == INTEL_PCH_KBP_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_KBP;
 				DRM_DEBUG_KMS("Found KabyPoint PCH\n");
-				WARN_ON(!IS_KABYLAKE(dev));
+				WARN_ON(!IS_KABYLAKE(dev_priv));
 			} else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) ||
 				   (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) ||
 				   ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) &&
@@ -215,7 +221,8 @@ static void intel_detect_pch(struct drm_device *dev)
 					    PCI_SUBVENDOR_ID_REDHAT_QUMRANET &&
 				    pch->subsystem_device ==
 					    PCI_SUBDEVICE_ID_QEMU)) {
-				dev_priv->pch_type = intel_virt_detect_pch(dev);
+				dev_priv->pch_type =
+					intel_virt_detect_pch(dev_priv);
 			} else
 				continue;
 
@@ -255,16 +262,16 @@ static int i915_getparam(struct drm_device *dev, void *data,
 		value = dev_priv->overlay ? 1 : 0;
 		break;
 	case I915_PARAM_HAS_BSD:
-		value = intel_engine_initialized(&dev_priv->engine[VCS]);
+		value = !!dev_priv->engine[VCS];
 		break;
 	case I915_PARAM_HAS_BLT:
-		value = intel_engine_initialized(&dev_priv->engine[BCS]);
+		value = !!dev_priv->engine[BCS];
 		break;
 	case I915_PARAM_HAS_VEBOX:
-		value = intel_engine_initialized(&dev_priv->engine[VECS]);
+		value = !!dev_priv->engine[VECS];
 		break;
 	case I915_PARAM_HAS_BSD2:
-		value = intel_engine_initialized(&dev_priv->engine[VCS2]);
+		value = !!dev_priv->engine[VCS2];
 		break;
 	case I915_PARAM_HAS_EXEC_CONSTANTS:
 		value = INTEL_GEN(dev_priv) >= 4;
@@ -316,6 +323,10 @@ static int i915_getparam(struct drm_device *dev, void *data,
 		 */
 		value = i915_gem_mmap_gtt_version();
 		break;
+	case I915_PARAM_HAS_SCHEDULER:
+		value = dev_priv->engine[RCS] &&
+			dev_priv->engine[RCS]->schedule;
+		break;
 	case I915_PARAM_MMAP_VERSION:
 		/* Remember to bump this if the version changes! */
 	case I915_PARAM_HAS_GEM:
@@ -367,12 +378,12 @@ static int
 intel_alloc_mchbar_resource(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
+	int reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
 	u32 temp_lo, temp_hi = 0;
 	u64 mchbar_addr;
 	int ret;
 
-	if (INTEL_INFO(dev)->gen >= 4)
+	if (INTEL_GEN(dev_priv) >= 4)
 		pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
 	pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
 	mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
@@ -399,7 +410,7 @@ intel_alloc_mchbar_resource(struct drm_device *dev)
 		return ret;
 	}
 
-	if (INTEL_INFO(dev)->gen >= 4)
+	if (INTEL_GEN(dev_priv) >= 4)
 		pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
 				       upper_32_bits(dev_priv->mch_res.start));
 
@@ -413,16 +424,16 @@ static void
 intel_setup_mchbar(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
+	int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
 	u32 temp;
 	bool enabled;
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		return;
 
 	dev_priv->mchbar_need_disable = false;
 
-	if (IS_I915G(dev) || IS_I915GM(dev)) {
+	if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
 		pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp);
 		enabled = !!(temp & DEVEN_MCHBAR_EN);
 	} else {
@@ -440,7 +451,7 @@ intel_setup_mchbar(struct drm_device *dev)
 	dev_priv->mchbar_need_disable = true;
 
 	/* Space is allocated or reserved, so enable it. */
-	if (IS_I915G(dev) || IS_I915GM(dev)) {
+	if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
 		pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
 				       temp | DEVEN_MCHBAR_EN);
 	} else {
@@ -453,10 +464,10 @@ static void
 intel_teardown_mchbar(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
+	int mchbar_reg = INTEL_GEN(dev_priv) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
 
 	if (dev_priv->mchbar_need_disable) {
-		if (IS_I915G(dev) || IS_I915GM(dev)) {
+		if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
 			u32 deven_val;
 
 			pci_read_config_dword(dev_priv->bridge_dev, DEVEN,
@@ -484,7 +495,7 @@ static unsigned int i915_vga_set_decode(void *cookie, bool state)
 {
 	struct drm_device *dev = cookie;
 
-	intel_modeset_vga_set_state(dev, state);
+	intel_modeset_vga_set_state(to_i915(dev), state);
 	if (state)
 		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
 		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
@@ -530,40 +541,17 @@ static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
 	.can_switch = i915_switcheroo_can_switch,
 };
 
-static void i915_gem_fini(struct drm_device *dev)
+static void i915_gem_fini(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	i915_gem_cleanup_engines(&dev_priv->drm);
+	i915_gem_context_fini(&dev_priv->drm);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
 
-	/*
-	 * Neither the BIOS, ourselves or any other kernel
-	 * expects the system to be in execlists mode on startup,
-	 * so we need to reset the GPU back to legacy mode. And the only
-	 * known way to disable logical contexts is through a GPU reset.
-	 *
-	 * So in order to leave the system in a known default configuration,
-	 * always reset the GPU upon unload. Afterwards we then clean up the
-	 * GEM state tracking, flushing off the requests and leaving the
-	 * system in a known idle state.
-	 *
-	 * Note that is of the upmost importance that the GPU is idle and
-	 * all stray writes are flushed *before* we dismantle the backing
-	 * storage for the pinned objects.
-	 *
-	 * However, since we are uncertain that reseting the GPU on older
-	 * machines is a good idea, we don't - just in case it leaves the
-	 * machine in an unusable condition.
-	 */
-	if (HAS_HW_CONTEXTS(dev)) {
-		int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
-		WARN_ON(reset && reset != -ENODEV);
-	}
-
-	mutex_lock(&dev->struct_mutex);
-	i915_gem_cleanup_engines(dev);
-	i915_gem_context_fini(dev);
-	mutex_unlock(&dev->struct_mutex);
+	rcu_barrier();
+	flush_work(&dev_priv->mm.free_work);
 
-	WARN_ON(!list_empty(&to_i915(dev)->context_list));
+	WARN_ON(!list_empty(&dev_priv->context_list));
 }
 
 static int i915_load_modeset_init(struct drm_device *dev)
@@ -611,7 +599,9 @@ static int i915_load_modeset_init(struct drm_device *dev)
 
 	/* Important: The output setup functions called by modeset_init need
 	 * working irqs for e.g. gmbus and dp aux transfers. */
-	intel_modeset_init(dev);
+	ret = intel_modeset_init(dev);
+	if (ret)
+		goto cleanup_irq;
 
 	intel_guc_init(dev);
 
@@ -621,7 +611,7 @@ static int i915_load_modeset_init(struct drm_device *dev)
 
 	intel_modeset_gem_init(dev);
 
-	if (INTEL_INFO(dev)->num_pipes == 0)
+	if (INTEL_INFO(dev_priv)->num_pipes == 0)
 		return 0;
 
 	ret = intel_fbdev_init(dev);
@@ -636,7 +626,9 @@ static int i915_load_modeset_init(struct drm_device *dev)
 	return 0;
 
 cleanup_gem:
-	i915_gem_fini(dev);
+	if (i915_gem_suspend(dev))
+		DRM_ERROR("failed to idle hardware; continuing to unload!\n");
+	i915_gem_fini(dev_priv);
 cleanup_irq:
 	intel_guc_fini(dev);
 	drm_irq_uninstall(dev);
@@ -771,6 +763,19 @@ static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
 	destroy_workqueue(dev_priv->wq);
 }
 
+/*
+ * We don't keep the workarounds for pre-production hardware, so we expect our
+ * driver to fail on these machines in one way or another. A little warning on
+ * dmesg may help both the user and the bug triagers.
+ */
+static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)
+{
+	if (IS_HSW_EARLY_SDV(dev_priv) ||
+	    IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0))
+		DRM_ERROR("This is a pre-production stepping. "
+			  "It may not be fully functional.\n");
+}
+
 /**
  * i915_driver_init_early - setup state not requiring device access
  * @dev_priv: device private
@@ -829,25 +834,24 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 	intel_init_dpio(dev_priv);
 	intel_power_domains_init(dev_priv);
 	intel_irq_init(dev_priv);
+	intel_hangcheck_init(dev_priv);
 	intel_init_display_hooks(dev_priv);
 	intel_init_clock_gating_hooks(dev_priv);
 	intel_init_audio_hooks(dev_priv);
-	i915_gem_load_init(&dev_priv->drm);
+	ret = i915_gem_load_init(&dev_priv->drm);
+	if (ret < 0)
+		goto err_gvt;
 
 	intel_display_crc_init(dev_priv);
 
 	intel_device_info_dump(dev_priv);
 
-	/* Not all pre-production machines fall into this category, only the
-	 * very first ones. Almost everything should work, except for maybe
-	 * suspend/resume. And we don't implement workarounds that affect only
-	 * pre-production machines. */
-	if (IS_HSW_EARLY_SDV(dev_priv))
-		DRM_INFO("This is an early pre-production Haswell machine. "
-			 "It may not be fully functional.\n");
+	intel_detect_preproduction_hw(dev_priv);
 
 	return 0;
 
+err_gvt:
+	intel_gvt_cleanup(dev_priv);
 err_workqueues:
 	i915_workqueues_cleanup(dev_priv);
 	return ret;
@@ -870,7 +874,7 @@ static int i915_mmio_setup(struct drm_device *dev)
 	int mmio_bar;
 	int mmio_size;
 
-	mmio_bar = IS_GEN2(dev) ? 1 : 0;
+	mmio_bar = IS_GEN2(dev_priv) ? 1 : 0;
 	/*
 	 * Before gen4, the registers and the GTT are behind different BARs.
 	 * However, from gen4 onwards, the registers and the GTT are shared
@@ -879,7 +883,7 @@ static int i915_mmio_setup(struct drm_device *dev)
 	 * the register BAR remains the same size for all the earlier
 	 * generations up to Ironlake.
 	 */
-	if (INTEL_INFO(dev)->gen < 5)
+	if (INTEL_GEN(dev_priv) < 5)
 		mmio_size = 512 * 1024;
 	else
 		mmio_size = 2 * 1024 * 1024;
@@ -982,7 +986,6 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv)
 static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 {
 	struct pci_dev *pdev = dev_priv->drm.pdev;
-	struct drm_device *dev = &dev_priv->drm;
 	int ret;
 
 	if (i915_inject_load_failure())
@@ -1023,7 +1026,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 	pci_set_master(pdev);
 
 	/* overlay on gen2 is broken and can't address above 1G */
-	if (IS_GEN2(dev)) {
+	if (IS_GEN2(dev_priv)) {
 		ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30));
 		if (ret) {
 			DRM_ERROR("failed to set DMA mask\n");
@@ -1040,7 +1043,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 	 * behaviour if any general state is accessed within a page above 4GB,
 	 * which also needs to be handled carefully.
 	 */
-	if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) {
+	if (IS_BROADWATER(dev_priv) || IS_CRESTLINE(dev_priv)) {
 		ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
 
 		if (ret) {
@@ -1070,7 +1073,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 	 * be lost or delayed, but we use them anyways to avoid
 	 * stuck interrupts on some machines.
 	 */
-	if (!IS_I945G(dev) && !IS_I945GM(dev)) {
+	if (!IS_I945G(dev_priv) && !IS_I945GM(dev_priv)) {
 		if (pci_enable_msi(pdev) < 0)
 			DRM_DEBUG_DRIVER("can't enable MSI");
 	}
@@ -1121,6 +1124,7 @@ static void i915_driver_register(struct drm_i915_private *dev_priv)
 	/* Reveal our presence to userspace */
 	if (drm_dev_register(dev, 0) == 0) {
 		i915_debugfs_register(dev_priv);
+		i915_guc_register(dev_priv);
 		i915_setup_sysfs(dev_priv);
 	} else
 		DRM_ERROR("Failed to register driver for userspace access!\n");
@@ -1159,6 +1163,7 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv)
 	intel_opregion_unregister(dev_priv);
 
 	i915_teardown_sysfs(dev_priv);
+	i915_guc_unregister(dev_priv);
 	i915_debugfs_unregister(dev_priv);
 	drm_dev_unregister(&dev_priv->drm);
 
@@ -1167,8 +1172,8 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv)
 
 /**
  * i915_driver_load - setup chip and create an initial config
- * @dev: DRM device
- * @flags: startup flags
+ * @pdev: PCI device
+ * @ent: matching PCI ID entry
  *
  * The driver load routine has to do several things:
  *   - drive output discovery via intel_modeset_init()
@@ -1242,6 +1247,10 @@ int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
 	DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n",
 		 driver.name, driver.major, driver.minor, driver.patchlevel,
 		 driver.date, pci_name(pdev), dev_priv->drm.primary->index);
+	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG))
+		DRM_INFO("DRM_I915_DEBUG enabled\n");
+	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
+		DRM_INFO("DRM_I915_DEBUG_GEM enabled\n");
 
 	intel_runtime_pm_put(dev_priv);
 
@@ -1309,7 +1318,7 @@ void i915_driver_unload(struct drm_device *dev)
 	drain_workqueue(dev_priv->wq);
 
 	intel_guc_fini(dev);
-	i915_gem_fini(dev);
+	i915_gem_fini(dev_priv);
 	intel_fbc_cleanup_cfb(dev_priv);
 
 	intel_power_domains_fini(dev_priv);
@@ -1431,9 +1440,9 @@ static int i915_drm_suspend(struct drm_device *dev)
 
 	intel_suspend_encoders(dev_priv);
 
-	intel_suspend_hw(dev);
+	intel_suspend_hw(dev_priv);
 
-	i915_gem_suspend_gtt_mappings(dev);
+	i915_gem_suspend_gtt_mappings(dev_priv);
 
 	i915_save_state(dev);
 
@@ -1507,7 +1516,7 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
 	 * Fujitsu FSC S7110
 	 * Acer Aspire 1830T
 	 */
-	if (!(hibernation && INTEL_INFO(dev_priv)->gen < 6))
+	if (!(hibernation && INTEL_GEN(dev_priv) < 6))
 		pci_set_power_state(pdev, PCI_D3hot);
 
 	dev_priv->suspended_to_idle = suspend_to_idle(dev_priv);
@@ -1595,6 +1604,8 @@ static int i915_drm_resume(struct drm_device *dev)
 
 	intel_display_resume(dev);
 
+	drm_kms_helper_poll_enable(dev);
+
 	/*
 	 * ... but also need to make sure that hotplug processing
 	 * doesn't cause havoc. Like in the driver load code we don't
@@ -1602,8 +1613,6 @@ static int i915_drm_resume(struct drm_device *dev)
 	 * notifications.
 	 * */
 	intel_hpd_init(dev_priv);
-	/* Config may have changed between suspend and resume */
-	drm_helper_hpd_irq_event(dev);
 
 	intel_opregion_register(dev_priv);
 
@@ -1616,7 +1625,6 @@ static int i915_drm_resume(struct drm_device *dev)
 	intel_opregion_notify_adapter(dev_priv, PCI_D0);
 
 	intel_autoenable_gt_powersave(dev_priv);
-	drm_kms_helper_poll_enable(dev);
 
 	enable_rpm_wakeref_asserts(dev_priv);
 
@@ -1721,6 +1729,22 @@ int i915_resume_switcheroo(struct drm_device *dev)
 	return i915_drm_resume(dev);
 }
 
+static void disable_engines_irq(struct drm_i915_private *dev_priv)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	/* Ensure irq handler finishes, and not run again. */
+	disable_irq(dev_priv->drm.irq);
+	for_each_engine(engine, dev_priv, id)
+		tasklet_kill(&engine->irq_tasklet);
+}
+
+static void enable_engines_irq(struct drm_i915_private *dev_priv)
+{
+	enable_irq(dev_priv->drm.irq);
+}
+
 /**
  * i915_reset - reset chip after a hang
  * @dev: drm device to reset
@@ -1754,7 +1778,11 @@ void i915_reset(struct drm_i915_private *dev_priv)
 	error->reset_count++;
 
 	pr_notice("drm/i915: Resetting chip after gpu hang\n");
+
+	disable_engines_irq(dev_priv);
 	ret = intel_gpu_reset(dev_priv, ALL_ENGINES);
+	enable_engines_irq(dev_priv);
+
 	if (ret) {
 		if (ret != -ENODEV)
 			DRM_ERROR("Failed to reset chip: %i\n", ret);
@@ -2240,7 +2268,6 @@ err1:
 static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
 				bool rpm_resume)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	int err;
 	int ret;
 
@@ -2264,10 +2291,8 @@ static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
 
 	vlv_check_no_gt_access(dev_priv);
 
-	if (rpm_resume) {
-		intel_init_clock_gating(dev);
-		i915_gem_restore_fences(dev);
-	}
+	if (rpm_resume)
+		intel_init_clock_gating(dev_priv);
 
 	return ret;
 }
@@ -2282,37 +2307,18 @@ static int intel_runtime_suspend(struct device *kdev)
 	if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6())))
 		return -ENODEV;
 
-	if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
+	if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv)))
 		return -ENODEV;
 
 	DRM_DEBUG_KMS("Suspending device\n");
 
-	/*
-	 * We could deadlock here in case another thread holding struct_mutex
-	 * calls RPM suspend concurrently, since the RPM suspend will wait
-	 * first for this RPM suspend to finish. In this case the concurrent
-	 * RPM resume will be followed by its RPM suspend counterpart. Still
-	 * for consistency return -EAGAIN, which will reschedule this suspend.
-	 */
-	if (!mutex_trylock(&dev->struct_mutex)) {
-		DRM_DEBUG_KMS("device lock contention, deffering suspend\n");
-		/*
-		 * Bump the expiration timestamp, otherwise the suspend won't
-		 * be rescheduled.
-		 */
-		pm_runtime_mark_last_busy(kdev);
-
-		return -EAGAIN;
-	}
-
 	disable_rpm_wakeref_asserts(dev_priv);
 
 	/*
 	 * We are safe here against re-faults, since the fault handler takes
 	 * an RPM reference.
 	 */
-	i915_gem_release_all_mmaps(dev_priv);
-	mutex_unlock(&dev->struct_mutex);
+	i915_gem_runtime_suspend(dev_priv);
 
 	intel_guc_suspend(dev);
 
@@ -2386,7 +2392,7 @@ static int intel_runtime_resume(struct device *kdev)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	int ret = 0;
 
-	if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
+	if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv)))
 		return -ENODEV;
 
 	DRM_DEBUG_KMS("Resuming device\n");
@@ -2404,7 +2410,7 @@ static int intel_runtime_resume(struct device *kdev)
 	if (IS_GEN6(dev_priv))
 		intel_init_pch_refclk(dev);
 
-	if (IS_BROXTON(dev)) {
+	if (IS_BROXTON(dev_priv)) {
 		bxt_disable_dc9(dev_priv);
 		bxt_display_core_init(dev_priv, true);
 		if (dev_priv->csr.dmc_payload &&
@@ -2420,7 +2426,7 @@ static int intel_runtime_resume(struct device *kdev)
 	 * No point of rolling back things in case of an error, as the best
 	 * we can do is to hope that things will still work (and disable RPM).
 	 */
-	i915_gem_init_swizzling(dev);
+	i915_gem_init_swizzling(dev_priv);
 
 	intel_runtime_pm_enable_interrupts(dev_priv);
 
@@ -2495,9 +2501,7 @@ static const struct file_operations i915_driver_fops = {
 	.mmap = drm_gem_mmap,
 	.poll = drm_poll,
 	.read = drm_read,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = i915_compat_ioctl,
-#endif
 	.llseek = noop_llseek,
 };
 
@@ -2577,7 +2581,7 @@ static struct drm_driver driver = {
 	.set_busid = drm_pci_set_busid,
 
 	.gem_close_object = i915_gem_close_object,
-	.gem_free_object = i915_gem_free_object,
+	.gem_free_object_unlocked = i915_gem_free_object,
 	.gem_vm_ops = &i915_gem_vm_ops,
 
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 685e9e065287..56002a52936d 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -41,6 +41,7 @@
 #include <linux/intel-iommu.h>
 #include <linux/kref.h>
 #include <linux/pm_qos.h>
+#include <linux/reservation.h>
 #include <linux/shmem_fs.h>
 
 #include <drm/drmP.h>
@@ -59,9 +60,14 @@
 #include "intel_ringbuffer.h"
 
 #include "i915_gem.h"
+#include "i915_gem_fence_reg.h"
+#include "i915_gem_object.h"
 #include "i915_gem_gtt.h"
 #include "i915_gem_render_state.h"
 #include "i915_gem_request.h"
+#include "i915_gem_timeline.h"
+
+#include "i915_vma.h"
 
 #include "intel_gvt.h"
 
@@ -70,7 +76,8 @@
 
 #define DRIVER_NAME		"i915"
 #define DRIVER_DESC		"Intel Graphics"
-#define DRIVER_DATE		"20160919"
+#define DRIVER_DATE		"20161121"
+#define DRIVER_TIMESTAMP	1479717903
 
 #undef WARN_ON
 /* Many gcc seem to no see through this and fall over :( */
@@ -122,6 +129,11 @@ static inline const char *onoff(bool v)
 	return v ? "on" : "off";
 }
 
+static inline const char *enableddisabled(bool v)
+{
+	return v ? "enabled" : "disabled";
+}
+
 enum pipe {
 	INVALID_PIPE = -1,
 	PIPE_A = 0,
@@ -182,9 +194,10 @@ enum plane {
 };
 #define plane_name(p) ((p) + 'A')
 
-#define sprite_name(p, s) ((p) * INTEL_INFO(dev)->num_sprites[(p)] + (s) + 'A')
+#define sprite_name(p, s) ((p) * INTEL_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A')
 
 enum port {
+	PORT_NONE = -1,
 	PORT_A = 0,
 	PORT_B,
 	PORT_C,
@@ -310,7 +323,7 @@ struct i915_hotplug {
 #define for_each_pipe_masked(__dev_priv, __p, __mask) \
 	for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) \
 		for_each_if ((__mask) & (1 << (__p)))
-#define for_each_plane(__dev_priv, __pipe, __p)				\
+#define for_each_universal_plane(__dev_priv, __pipe, __p)		\
 	for ((__p) = 0;							\
 	     (__p) < INTEL_INFO(__dev_priv)->num_sprites[(__pipe)] + 1;	\
 	     (__p)++)
@@ -455,23 +468,6 @@ struct intel_opregion {
 struct intel_overlay;
 struct intel_overlay_error_state;
 
-struct drm_i915_fence_reg {
-	struct list_head link;
-	struct drm_i915_private *i915;
-	struct i915_vma *vma;
-	int pin_count;
-	int id;
-	/**
-	 * Whether the tiling parameters for the currently
-	 * associated fence register have changed. Note that
-	 * for the purposes of tracking tiling changes we also
-	 * treat the unfenced register, the register slot that
-	 * the object occupies whilst it executes a fenced
-	 * command (such as BLT on gen2/3), as a "fence".
-	 */
-	bool dirty;
-};
-
 struct sdvo_device_mapping {
 	u8 initialized;
 	u8 dvo_port;
@@ -483,6 +479,7 @@ struct sdvo_device_mapping {
 
 struct intel_connector;
 struct intel_encoder;
+struct intel_atomic_state;
 struct intel_crtc_state;
 struct intel_initial_plane_config;
 struct intel_crtc;
@@ -490,16 +487,20 @@ struct intel_limit;
 struct dpll;
 
 struct drm_i915_display_funcs {
-	int (*get_display_clock_speed)(struct drm_device *dev);
-	int (*get_fifo_size)(struct drm_device *dev, int plane);
+	int (*get_display_clock_speed)(struct drm_i915_private *dev_priv);
+	int (*get_fifo_size)(struct drm_i915_private *dev_priv, int plane);
 	int (*compute_pipe_wm)(struct intel_crtc_state *cstate);
 	int (*compute_intermediate_wm)(struct drm_device *dev,
 				       struct intel_crtc *intel_crtc,
 				       struct intel_crtc_state *newstate);
-	void (*initial_watermarks)(struct intel_crtc_state *cstate);
-	void (*optimize_watermarks)(struct intel_crtc_state *cstate);
+	void (*initial_watermarks)(struct intel_atomic_state *state,
+				   struct intel_crtc_state *cstate);
+	void (*atomic_update_watermarks)(struct intel_atomic_state *state,
+					 struct intel_crtc_state *cstate);
+	void (*optimize_watermarks)(struct intel_atomic_state *state,
+				    struct intel_crtc_state *cstate);
 	int (*compute_global_watermarks)(struct drm_atomic_state *state);
-	void (*update_wm)(struct drm_crtc *crtc);
+	void (*update_wm)(struct intel_crtc *crtc);
 	int (*modeset_calc_cdclk)(struct drm_atomic_state *state);
 	void (*modeset_commit_cdclk)(struct drm_atomic_state *state);
 	/* Returns the active state of the crtc, and if the crtc is active,
@@ -521,7 +522,7 @@ struct drm_i915_display_funcs {
 				   const struct drm_display_mode *adjusted_mode);
 	void (*audio_codec_disable)(struct intel_encoder *encoder);
 	void (*fdi_link_train)(struct drm_crtc *crtc);
-	void (*init_clock_gating)(struct drm_device *dev);
+	void (*init_clock_gating)(struct drm_i915_private *dev_priv);
 	int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc,
 			  struct drm_framebuffer *fb,
 			  struct drm_i915_gem_object *obj,
@@ -558,6 +559,18 @@ enum forcewake_domains {
 #define FW_REG_READ  (1)
 #define FW_REG_WRITE (2)
 
+enum decoupled_power_domain {
+	GEN9_DECOUPLED_PD_BLITTER = 0,
+	GEN9_DECOUPLED_PD_RENDER,
+	GEN9_DECOUPLED_PD_MEDIA,
+	GEN9_DECOUPLED_PD_ALL
+};
+
+enum decoupled_ops {
+	GEN9_DECOUPLED_OP_WRITE = 0,
+	GEN9_DECOUPLED_OP_READ
+};
+
 enum forcewake_domains
 intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
 			       i915_reg_t reg, unsigned int op);
@@ -581,13 +594,25 @@ struct intel_uncore_funcs {
 				uint32_t val, bool trace);
 };
 
+struct intel_forcewake_range {
+	u32 start;
+	u32 end;
+
+	enum forcewake_domains domains;
+};
+
 struct intel_uncore {
 	spinlock_t lock; /** lock is also taken in irq contexts. */
 
+	const struct intel_forcewake_range *fw_domains_table;
+	unsigned int fw_domains_table_entries;
+
 	struct intel_uncore_funcs funcs;
 
 	unsigned fifo_count;
+
 	enum forcewake_domains fw_domains;
+	enum forcewake_domains fw_domains_active;
 
 	struct intel_uncore_forcewake_domain {
 		struct drm_i915_private *i915;
@@ -633,54 +658,55 @@ struct intel_csr {
 	uint32_t allowed_dc_mask;
 };
 
-#define DEV_INFO_FOR_EACH_FLAG(func, sep) \
-	func(is_mobile) sep \
-	func(is_i85x) sep \
-	func(is_i915g) sep \
-	func(is_i945gm) sep \
-	func(is_g33) sep \
-	func(hws_needs_physical) sep \
-	func(is_g4x) sep \
-	func(is_pineview) sep \
-	func(is_broadwater) sep \
-	func(is_crestline) sep \
-	func(is_ivybridge) sep \
-	func(is_valleyview) sep \
-	func(is_cherryview) sep \
-	func(is_haswell) sep \
-	func(is_broadwell) sep \
-	func(is_skylake) sep \
-	func(is_broxton) sep \
-	func(is_kabylake) sep \
-	func(is_preliminary) sep \
-	func(has_fbc) sep \
-	func(has_psr) sep \
-	func(has_runtime_pm) sep \
-	func(has_csr) sep \
-	func(has_resource_streamer) sep \
-	func(has_rc6) sep \
-	func(has_rc6p) sep \
-	func(has_dp_mst) sep \
-	func(has_gmbus_irq) sep \
-	func(has_hw_contexts) sep \
-	func(has_logical_ring_contexts) sep \
-	func(has_l3_dpf) sep \
-	func(has_gmch_display) sep \
-	func(has_guc) sep \
-	func(has_pipe_cxsr) sep \
-	func(has_hotplug) sep \
-	func(cursor_needs_physical) sep \
-	func(has_overlay) sep \
-	func(overlay_needs_physical) sep \
-	func(supports_tv) sep \
-	func(has_llc) sep \
-	func(has_snoop) sep \
-	func(has_ddi) sep \
-	func(has_fpga_dbg) sep \
-	func(has_pooled_eu)
-
-#define DEFINE_FLAG(name) u8 name:1
-#define SEP_SEMICOLON ;
+#define DEV_INFO_FOR_EACH_FLAG(func) \
+	/* Keep is_* in chronological order */ \
+	func(is_mobile); \
+	func(is_i85x); \
+	func(is_i915g); \
+	func(is_i945gm); \
+	func(is_g33); \
+	func(is_g4x); \
+	func(is_pineview); \
+	func(is_broadwater); \
+	func(is_crestline); \
+	func(is_ivybridge); \
+	func(is_valleyview); \
+	func(is_cherryview); \
+	func(is_haswell); \
+	func(is_broadwell); \
+	func(is_skylake); \
+	func(is_broxton); \
+	func(is_kabylake); \
+	func(is_alpha_support); \
+	/* Keep has_* in alphabetical order */ \
+	func(has_64bit_reloc); \
+	func(has_csr); \
+	func(has_ddi); \
+	func(has_dp_mst); \
+	func(has_fbc); \
+	func(has_fpga_dbg); \
+	func(has_gmbus_irq); \
+	func(has_gmch_display); \
+	func(has_guc); \
+	func(has_hotplug); \
+	func(has_hw_contexts); \
+	func(has_l3_dpf); \
+	func(has_llc); \
+	func(has_logical_ring_contexts); \
+	func(has_overlay); \
+	func(has_pipe_cxsr); \
+	func(has_pooled_eu); \
+	func(has_psr); \
+	func(has_rc6); \
+	func(has_rc6p); \
+	func(has_resource_streamer); \
+	func(has_runtime_pm); \
+	func(has_snoop); \
+	func(cursor_needs_physical); \
+	func(hws_needs_physical); \
+	func(overlay_needs_physical); \
+	func(supports_tv); \
+	func(has_decoupled_mmio)
 
 struct sseu_dev_info {
 	u8 slice_mask;
@@ -709,7 +735,9 @@ struct intel_device_info {
 	u16 gen_mask;
 	u8 ring_mask; /* Rings supported by the HW */
 	u8 num_rings;
-	DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG, SEP_SEMICOLON);
+#define DEFINE_FLAG(name) u8 name:1
+	DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG);
+#undef DEFINE_FLAG
 	u16 ddb_size; /* in blocks */
 	/* Register offsets for the various display pipes and transcoders */
 	int pipe_offsets[I915_MAX_TRANSCODERS];
@@ -726,14 +754,15 @@ struct intel_device_info {
 	} color;
 };
 
-#undef DEFINE_FLAG
-#undef SEP_SEMICOLON
-
 struct intel_display_error_state;
 
 struct drm_i915_error_state {
 	struct kref ref;
 	struct timeval time;
+	struct timeval boottime;
+	struct timeval uptime;
+
+	struct drm_i915_private *i915;
 
 	char error_msg[128];
 	bool simulated;
@@ -759,11 +788,12 @@ struct drm_i915_error_state {
 	u32 gam_ecochk;
 	u32 gab_ctl;
 	u32 gfx_mode;
-	u32 extra_instdone[I915_NUM_INSTDONE_REG];
+
 	u64 fence[I915_MAX_NUM_FENCES];
 	struct intel_overlay_error_state *overlay;
 	struct intel_display_error_state *display;
 	struct drm_i915_error_object *semaphore;
+	struct drm_i915_error_object *guc_log;
 
 	struct drm_i915_error_engine {
 		int engine_id;
@@ -775,12 +805,14 @@ struct drm_i915_error_state {
 		struct i915_address_space *vm;
 		int num_requests;
 
+		/* position of active request inside the ring */
+		u32 rq_head, rq_post, rq_tail;
+
 		/* our own tracking of ring head and tail */
 		u32 cpu_ring_head;
 		u32 cpu_ring_tail;
 
 		u32 last_seqno;
-		u32 semaphore_seqno[I915_NUM_ENGINES - 1];
 
 		/* Register state */
 		u32 start;
@@ -791,7 +823,6 @@ struct drm_i915_error_state {
 		u32 hws;
 		u32 ipeir;
 		u32 ipehr;
-		u32 instdone;
 		u32 bbstate;
 		u32 instpm;
 		u32 instps;
@@ -802,11 +833,13 @@ struct drm_i915_error_state {
 		u64 faddr;
 		u32 rc_psmi; /* sleep state */
 		u32 semaphore_mboxes[I915_NUM_ENGINES - 1];
+		struct intel_instdone instdone;
 
 		struct drm_i915_error_object {
-			int page_count;
 			u64 gtt_offset;
 			u64 gtt_size;
+			int page_count;
+			int unused;
 			u32 *pages[0];
 		} *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
 
@@ -815,10 +848,11 @@ struct drm_i915_error_state {
 		struct drm_i915_error_request {
 			long jiffies;
 			pid_t pid;
+			u32 context;
 			u32 seqno;
 			u32 head;
 			u32 tail;
-		} *requests;
+		} *requests, execlist[2];
 
 		struct drm_i915_error_waiter {
 			char comm[TASK_COMM_LEN];
@@ -914,6 +948,7 @@ struct i915_gem_context {
 	struct drm_i915_file_private *file_priv;
 	struct i915_hw_ppgtt *ppgtt;
 	struct pid *pid;
+	const char *name;
 
 	struct i915_ctx_hang_stats hang_stats;
 
@@ -924,6 +959,7 @@ struct i915_gem_context {
 	/* Unique identifier for this context, used by the hw for tracking */
 	unsigned int hw_id;
 	u32 user_handle;
+	int priority; /* greater priorities are serviced first */
 
 	u32 ggtt_alignment;
 
@@ -972,6 +1008,9 @@ struct intel_fbc {
 	bool enabled;
 	bool active;
 
+	bool underrun_detected;
+	struct work_struct underrun_work;
+
 	struct intel_fbc_state_cache {
 		struct {
 			unsigned int mode_flags;
@@ -1297,6 +1336,12 @@ struct i915_power_well {
 	/* cached hw enabled state */
 	bool hw_enabled;
 	unsigned long domains;
+	/* unique identifier for this power well */
+	unsigned long id;
+	/*
+	 * Arbitraty data associated with this power well. Platform and power
+	 * well specific.
+	 */
 	unsigned long data;
 	const struct i915_power_well_ops *ops;
 };
@@ -1334,11 +1379,22 @@ struct i915_gem_mm {
 	struct list_head bound_list;
 	/**
 	 * List of objects which are not bound to the GTT (thus
-	 * are idle and not used by the GPU) but still have
-	 * (presumably uncached) pages still attached.
+	 * are idle and not used by the GPU). These objects may or may
+	 * not actually have any pages attached.
 	 */
 	struct list_head unbound_list;
 
+	/** List of all objects in gtt_space, currently mmaped by userspace.
+	 * All objects within this list must also be on bound_list.
+	 */
+	struct list_head userfault_list;
+
+	/**
+	 * List of objects which are pending destruction.
+	 */
+	struct llist_head free_list;
+	struct work_struct free_work;
+
 	/** Usable portion of the GTT for GEM */
 	unsigned long stolen_base; /* limited to low memory (32-bit) */
 
@@ -1368,7 +1424,7 @@ struct i915_gem_mm {
 
 	/* accounting, useful for userland debugging */
 	spinlock_t object_stat_lock;
-	size_t object_memory;
+	u64 object_memory;
 	u32 object_count;
 };
 
@@ -1387,6 +1443,9 @@ struct i915_error_state_file_priv {
 	struct drm_i915_error_state *error;
 };
 
+#define I915_RESET_TIMEOUT (10 * HZ) /* 10s */
+#define I915_FENCE_TIMEOUT (10 * HZ) /* 10s */
+
 struct i915_gpu_error {
 	/* For hangcheck timer */
 #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
@@ -1620,7 +1679,6 @@ static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1,
 }
 
 struct skl_ddb_allocation {
-	struct skl_ddb_entry pipe[I915_MAX_PIPES];
 	struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES]; /* packed/uv */
 	struct skl_ddb_entry y_plane[I915_MAX_PIPES][I915_MAX_PLANES];
 };
@@ -1628,15 +1686,12 @@ struct skl_ddb_allocation {
 struct skl_wm_values {
 	unsigned dirty_pipes;
 	struct skl_ddb_allocation ddb;
-	uint32_t wm_linetime[I915_MAX_PIPES];
-	uint32_t plane[I915_MAX_PIPES][I915_MAX_PLANES][8];
-	uint32_t plane_trans[I915_MAX_PIPES][I915_MAX_PLANES];
 };
 
 struct skl_wm_level {
-	bool plane_en[I915_MAX_PLANES];
-	uint16_t plane_res_b[I915_MAX_PLANES];
-	uint8_t plane_res_l[I915_MAX_PLANES];
+	bool plane_en;
+	uint16_t plane_res_b;
+	uint8_t plane_res_l;
 };
 
 /*
@@ -1664,7 +1719,6 @@ struct skl_wm_level {
  */
 struct i915_runtime_pm {
 	atomic_t wakeref_count;
-	atomic_t atomic_seq;
 	bool suspended;
 	bool irqs_enabled;
 };
@@ -1748,6 +1802,7 @@ struct drm_i915_private {
 	struct kmem_cache *objects;
 	struct kmem_cache *vmas;
 	struct kmem_cache *requests;
+	struct kmem_cache *dependencies;
 
 	const struct intel_device_info info;
 
@@ -1759,7 +1814,7 @@ struct drm_i915_private {
 
 	struct i915_virtual_gpu vgpu;
 
-	struct intel_gvt gvt;
+	struct intel_gvt *gvt;
 
 	struct intel_guc guc;
 
@@ -1787,9 +1842,8 @@ struct drm_i915_private {
 
 	struct pci_dev *bridge_dev;
 	struct i915_gem_context *kernel_context;
-	struct intel_engine_cs engine[I915_NUM_ENGINES];
+	struct intel_engine_cs *engine[I915_NUM_ENGINES];
 	struct i915_vma *semaphore;
-	u32 next_seqno;
 
 	struct drm_dma_handle *status_page_dmah;
 	struct resource mch_res;
@@ -1814,8 +1868,10 @@ struct drm_i915_private {
 		u32 de_irq_mask[I915_MAX_PIPES];
 	};
 	u32 gt_irq_mask;
-	u32 pm_irq_mask;
+	u32 pm_imr;
+	u32 pm_ier;
 	u32 pm_rps_events;
+	u32 pm_guc_events;
 	u32 pipestat_irq_mask[I915_MAX_PIPES];
 
 	struct i915_hotplug hotplug;
@@ -1892,8 +1948,8 @@ struct drm_i915_private {
 
 	/* Kernel Modesetting */
 
-	struct drm_crtc *plane_to_crtc_mapping[I915_MAX_PIPES];
-	struct drm_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES];
+	struct intel_crtc *plane_to_crtc_mapping[I915_MAX_PIPES];
+	struct intel_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES];
 	wait_queue_head_t pending_flip_queue;
 
 #ifdef CONFIG_DEBUG_FS
@@ -2009,13 +2065,6 @@ struct drm_i915_private {
 		 */
 		uint16_t skl_latency[8];
 
-		/*
-		 * The skl_wm_values structure is a bit too big for stack
-		 * allocation, so we keep the staging struct where we store
-		 * intermediate results here instead.
-		 */
-		struct skl_wm_values skl_results;
-
 		/* current hardware state */
 		union {
 			struct ilk_wm_values hw;
@@ -2047,6 +2096,10 @@ struct drm_i915_private {
 		void (*resume)(struct drm_i915_private *);
 		void (*cleanup_engine)(struct intel_engine_cs *engine);
 
+		struct list_head timelines;
+		struct i915_gem_timeline global_timeline;
+		u32 active_requests;
+
 		/**
 		 * Is the GPU currently considered idle, or busy executing
 		 * userspace requests? Whilst idle, we allow runtime power
@@ -2054,7 +2107,6 @@ struct drm_i915_private {
 		 * In order to reduce the effect on performance, there
 		 * is a slight delay before we do so.
 		 */
-		unsigned int active_engines;
 		bool awake;
 
 		/**
@@ -2074,12 +2126,15 @@ struct drm_i915_private {
 		 * off the idle_work.
 		 */
 		struct delayed_work idle_work;
+
+		ktime_t last_init_time;
 	} gt;
 
 	/* perform PHY state sanity checks? */
 	bool chv_phy_assert[2];
 
-	struct intel_encoder *dig_port_map[I915_MAX_PORTS];
+	/* Used to save the pipe-to-encoder mapping for audio */
+	struct intel_encoder *av_enc_map[I915_MAX_PIPES];
 
 	/*
 	 * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch
@@ -2103,19 +2158,11 @@ static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc)
 }
 
 /* Simple iterator over all initialised engines */
-#define for_each_engine(engine__, dev_priv__) \
-	for ((engine__) = &(dev_priv__)->engine[0]; \
-	     (engine__) < &(dev_priv__)->engine[I915_NUM_ENGINES]; \
-	     (engine__)++) \
-		for_each_if (intel_engine_initialized(engine__))
-
-/* Iterator with engine_id */
-#define for_each_engine_id(engine__, dev_priv__, id__) \
-	for ((engine__) = &(dev_priv__)->engine[0], (id__) = 0; \
-	     (engine__) < &(dev_priv__)->engine[I915_NUM_ENGINES]; \
-	     (engine__)++) \
-		for_each_if (((id__) = (engine__)->id, \
-			      intel_engine_initialized(engine__)))
+#define for_each_engine(engine__, dev_priv__, id__) \
+	for ((id__) = 0; \
+	     (id__) < I915_NUM_ENGINES; \
+	     (id__)++) \
+		for_each_if ((engine__) = (dev_priv__)->engine[(id__)])
 
 #define __mask_next_bit(mask) ({					\
 	int __idx = ffs(mask) - 1;					\
@@ -2126,7 +2173,7 @@ static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc)
 /* Iterator over subset of engines selected by mask */
 #define for_each_engine_masked(engine__, dev_priv__, mask__, tmp__) \
 	for (tmp__ = mask__ & INTEL_INFO(dev_priv__)->ring_mask;	\
-	     tmp__ ? (engine__ = &(dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : 0; )
+	     tmp__ ? (engine__ = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : 0; )
 
 enum hdmi_force_audio {
 	HDMI_AUDIO_OFF_DVI = -2,	/* no aux data for HDMI-DVI converter */
@@ -2137,30 +2184,6 @@ enum hdmi_force_audio {
 
 #define I915_GTT_OFFSET_NONE ((u32)-1)
 
-struct drm_i915_gem_object_ops {
-	unsigned int flags;
-#define I915_GEM_OBJECT_HAS_STRUCT_PAGE 0x1
-
-	/* Interface between the GEM object and its backing storage.
-	 * get_pages() is called once prior to the use of the associated set
-	 * of pages before to binding them into the GTT, and put_pages() is
-	 * called after we no longer need them. As we expect there to be
-	 * associated cost with migrating pages between the backing storage
-	 * and making them available for the GPU (e.g. clflush), we may hold
-	 * onto the pages after they are no longer referenced by the GPU
-	 * in case they may be used again shortly (for example migrating the
-	 * pages to a different memory domain within the GTT). put_pages()
-	 * will therefore most likely be called when the object itself is
-	 * being released or under memory pressure (where we attempt to
-	 * reap pages for the shrinker).
-	 */
-	int (*get_pages)(struct drm_i915_gem_object *);
-	void (*put_pages)(struct drm_i915_gem_object *);
-
-	int (*dmabuf_export)(struct drm_i915_gem_object *);
-	void (*release)(struct drm_i915_gem_object *);
-};
-
 /*
  * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is
  * considered to be the frontbuffer for the given plane interface-wise. This
@@ -2182,232 +2205,6 @@ struct drm_i915_gem_object_ops {
 #define INTEL_FRONTBUFFER_ALL_MASK(pipe) \
 	(0xff << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))
 
-struct drm_i915_gem_object {
-	struct drm_gem_object base;
-
-	const struct drm_i915_gem_object_ops *ops;
-
-	/** List of VMAs backed by this object */
-	struct list_head vma_list;
-
-	/** Stolen memory for this object, instead of being backed by shmem. */
-	struct drm_mm_node *stolen;
-	struct list_head global_list;
-
-	/** Used in execbuf to temporarily hold a ref */
-	struct list_head obj_exec_link;
-
-	struct list_head batch_pool_link;
-
-	unsigned long flags;
-	/**
-	 * This is set if the object is on the active lists (has pending
-	 * rendering and so a non-zero seqno), and is not set if it i s on
-	 * inactive (ready to be unbound) list.
-	 */
-#define I915_BO_ACTIVE_SHIFT 0
-#define I915_BO_ACTIVE_MASK ((1 << I915_NUM_ENGINES) - 1)
-#define __I915_BO_ACTIVE(bo) \
-	((READ_ONCE((bo)->flags) >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK)
-
-	/**
-	 * This is set if the object has been written to since last bound
-	 * to the GTT
-	 */
-	unsigned int dirty:1;
-
-	/**
-	 * Advice: are the backing pages purgeable?
-	 */
-	unsigned int madv:2;
-
-	/**
-	 * Whether the current gtt mapping needs to be mappable (and isn't just
-	 * mappable by accident). Track pin and fault separate for a more
-	 * accurate mappable working set.
-	 */
-	unsigned int fault_mappable:1;
-
-	/*
-	 * Is the object to be mapped as read-only to the GPU
-	 * Only honoured if hardware has relevant pte bit
-	 */
-	unsigned long gt_ro:1;
-	unsigned int cache_level:3;
-	unsigned int cache_dirty:1;
-
-	atomic_t frontbuffer_bits;
-	unsigned int frontbuffer_ggtt_origin; /* write once */
-
-	/** Current tiling stride for the object, if it's tiled. */
-	unsigned int tiling_and_stride;
-#define FENCE_MINIMUM_STRIDE 128 /* See i915_tiling_ok() */
-#define TILING_MASK (FENCE_MINIMUM_STRIDE-1)
-#define STRIDE_MASK (~TILING_MASK)
-
-	/** Count of VMA actually bound by this object */
-	unsigned int bind_count;
-	unsigned int pin_display;
-
-	struct sg_table *pages;
-	int pages_pin_count;
-	struct get_page {
-		struct scatterlist *sg;
-		int last;
-	} get_page;
-	void *mapping;
-
-	/** Breadcrumb of last rendering to the buffer.
-	 * There can only be one writer, but we allow for multiple readers.
-	 * If there is a writer that necessarily implies that all other
-	 * read requests are complete - but we may only be lazily clearing
-	 * the read requests. A read request is naturally the most recent
-	 * request on a ring, so we may have two different write and read
-	 * requests on one ring where the write request is older than the
-	 * read request. This allows for the CPU to read from an active
-	 * buffer by only waiting for the write to complete.
-	 */
-	struct i915_gem_active last_read[I915_NUM_ENGINES];
-	struct i915_gem_active last_write;
-
-	/** References from framebuffers, locks out tiling changes. */
-	unsigned long framebuffer_references;
-
-	/** Record of address bit 17 of each page at last unbind. */
-	unsigned long *bit_17;
-
-	struct i915_gem_userptr {
-		uintptr_t ptr;
-		unsigned read_only :1;
-		unsigned workers :4;
-#define I915_GEM_USERPTR_MAX_WORKERS 15
-
-		struct i915_mm_struct *mm;
-		struct i915_mmu_object *mmu_object;
-		struct work_struct *work;
-	} userptr;
-
-	/** for phys allocated objects */
-	struct drm_dma_handle *phys_handle;
-};
-
-static inline struct drm_i915_gem_object *
-to_intel_bo(struct drm_gem_object *gem)
-{
-	/* Assert that to_intel_bo(NULL) == NULL */
-	BUILD_BUG_ON(offsetof(struct drm_i915_gem_object, base));
-
-	return container_of(gem, struct drm_i915_gem_object, base);
-}
-
-static inline struct drm_i915_gem_object *
-i915_gem_object_lookup(struct drm_file *file, u32 handle)
-{
-	return to_intel_bo(drm_gem_object_lookup(file, handle));
-}
-
-__deprecated
-extern struct drm_gem_object *
-drm_gem_object_lookup(struct drm_file *file, u32 handle);
-
-__attribute__((nonnull))
-static inline struct drm_i915_gem_object *
-i915_gem_object_get(struct drm_i915_gem_object *obj)
-{
-	drm_gem_object_reference(&obj->base);
-	return obj;
-}
-
-__deprecated
-extern void drm_gem_object_reference(struct drm_gem_object *);
-
-__attribute__((nonnull))
-static inline void
-i915_gem_object_put(struct drm_i915_gem_object *obj)
-{
-	drm_gem_object_unreference(&obj->base);
-}
-
-__deprecated
-extern void drm_gem_object_unreference(struct drm_gem_object *);
-
-__attribute__((nonnull))
-static inline void
-i915_gem_object_put_unlocked(struct drm_i915_gem_object *obj)
-{
-	drm_gem_object_unreference_unlocked(&obj->base);
-}
-
-__deprecated
-extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);
-
-static inline bool
-i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
-{
-	return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE;
-}
-
-static inline unsigned long
-i915_gem_object_get_active(const struct drm_i915_gem_object *obj)
-{
-	return (obj->flags >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK;
-}
-
-static inline bool
-i915_gem_object_is_active(const struct drm_i915_gem_object *obj)
-{
-	return i915_gem_object_get_active(obj);
-}
-
-static inline void
-i915_gem_object_set_active(struct drm_i915_gem_object *obj, int engine)
-{
-	obj->flags |= BIT(engine + I915_BO_ACTIVE_SHIFT);
-}
-
-static inline void
-i915_gem_object_clear_active(struct drm_i915_gem_object *obj, int engine)
-{
-	obj->flags &= ~BIT(engine + I915_BO_ACTIVE_SHIFT);
-}
-
-static inline bool
-i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj,
-				  int engine)
-{
-	return obj->flags & BIT(engine + I915_BO_ACTIVE_SHIFT);
-}
-
-static inline unsigned int
-i915_gem_object_get_tiling(struct drm_i915_gem_object *obj)
-{
-	return obj->tiling_and_stride & TILING_MASK;
-}
-
-static inline bool
-i915_gem_object_is_tiled(struct drm_i915_gem_object *obj)
-{
-	return i915_gem_object_get_tiling(obj) != I915_TILING_NONE;
-}
-
-static inline unsigned int
-i915_gem_object_get_stride(struct drm_i915_gem_object *obj)
-{
-	return obj->tiling_and_stride & STRIDE_MASK;
-}
-
-static inline struct i915_vma *i915_vma_get(struct i915_vma *vma)
-{
-	i915_gem_object_get(vma->obj);
-	return vma;
-}
-
-static inline void i915_vma_put(struct i915_vma *vma)
-{
-	lockdep_assert_held(&vma->vm->dev->struct_mutex);
-	i915_gem_object_put(vma->obj);
-}
-
 /*
  * Optimised SGL iterator for GEM objects
  */
@@ -2434,6 +2231,14 @@ static __always_inline struct sgt_iter {
 	return s;
 }
 
+static inline struct scatterlist *____sg_next(struct scatterlist *sg)
+{
+	++sg;
+	if (unlikely(sg_is_chain(sg)))
+		sg = sg_chain_ptr(sg);
+	return sg;
+}
+
 /**
  * __sg_next - return the next scatterlist entry in a list
  * @sg:		The current sg entry
@@ -2448,9 +2253,7 @@ static inline struct scatterlist *__sg_next(struct scatterlist *sg)
 #ifdef CONFIG_DEBUG_SG
 	BUG_ON(sg->sg_magic != SG_MAGIC);
 #endif
-	return sg_is_last(sg) ? NULL :
-		likely(!sg_is_chain(++sg)) ? sg :
-		sg_chain_ptr(sg);
+	return sg_is_last(sg) ? NULL : ____sg_next(sg);
 }
 
 /**
@@ -2574,23 +2377,19 @@ struct drm_i915_cmd_table {
 	int count;
 };
 
-/* Note that the (struct drm_i915_private *) cast is just to shut up gcc. */
-#define __I915__(p) ({ \
-	struct drm_i915_private *__p; \
-	if (__builtin_types_compatible_p(typeof(*p), struct drm_i915_private)) \
-		__p = (struct drm_i915_private *)p; \
-	else if (__builtin_types_compatible_p(typeof(*p), struct drm_device)) \
-		__p = to_i915((struct drm_device *)p); \
-	else \
-		BUILD_BUG(); \
-	__p; \
-})
-#define INTEL_INFO(p)	(&__I915__(p)->info)
-#define INTEL_GEN(p)	(INTEL_INFO(p)->gen)
-#define INTEL_DEVID(p)	(INTEL_INFO(p)->device_id)
+static inline const struct intel_device_info *
+intel_info(const struct drm_i915_private *dev_priv)
+{
+	return &dev_priv->info;
+}
+
+#define INTEL_INFO(dev_priv)	intel_info((dev_priv))
+
+#define INTEL_GEN(dev_priv)	((dev_priv)->info.gen)
+#define INTEL_DEVID(dev_priv)	((dev_priv)->info.device_id)
 
 #define REVID_FOREVER		0xff
-#define INTEL_REVID(p)	(__I915__(p)->drm.pdev->revision)
+#define INTEL_REVID(dev_priv)	((dev_priv)->drm.pdev->revision)
 
 #define GEN_FOREVER (0)
 /*
@@ -2598,7 +2397,7 @@ struct drm_i915_cmd_table {
  *
  * Use GEN_FOREVER for unbound start and or end.
  */
-#define IS_GEN(p, s, e) ({ \
+#define IS_GEN(dev_priv, s, e) ({ \
 	unsigned int __s = (s), __e = (e); \
 	BUILD_BUG_ON(!__builtin_constant_p(s)); \
 	BUILD_BUG_ON(!__builtin_constant_p(e)); \
@@ -2608,7 +2407,7 @@ struct drm_i915_cmd_table {
 		__e = BITS_PER_LONG - 1; \
 	else \
 		__e = (e) - 1; \
-	!!(INTEL_INFO(p)->gen_mask & GENMASK((__e), (__s))); \
+	!!((dev_priv)->info.gen_mask & GENMASK((__e), (__s))); \
 })
 
 /*
@@ -2619,75 +2418,75 @@ struct drm_i915_cmd_table {
 #define IS_REVID(p, since, until) \
 	(INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until))
 
-#define IS_I830(dev)		(INTEL_DEVID(dev) == 0x3577)
-#define IS_845G(dev)		(INTEL_DEVID(dev) == 0x2562)
-#define IS_I85X(dev)		(INTEL_INFO(dev)->is_i85x)
-#define IS_I865G(dev)		(INTEL_DEVID(dev) == 0x2572)
-#define IS_I915G(dev)		(INTEL_INFO(dev)->is_i915g)
-#define IS_I915GM(dev)		(INTEL_DEVID(dev) == 0x2592)
-#define IS_I945G(dev)		(INTEL_DEVID(dev) == 0x2772)
-#define IS_I945GM(dev)		(INTEL_INFO(dev)->is_i945gm)
-#define IS_BROADWATER(dev)	(INTEL_INFO(dev)->is_broadwater)
-#define IS_CRESTLINE(dev)	(INTEL_INFO(dev)->is_crestline)
-#define IS_GM45(dev)		(INTEL_DEVID(dev) == 0x2A42)
-#define IS_G4X(dev)		(INTEL_INFO(dev)->is_g4x)
-#define IS_PINEVIEW_G(dev)	(INTEL_DEVID(dev) == 0xa001)
-#define IS_PINEVIEW_M(dev)	(INTEL_DEVID(dev) == 0xa011)
-#define IS_PINEVIEW(dev)	(INTEL_INFO(dev)->is_pineview)
-#define IS_G33(dev)		(INTEL_INFO(dev)->is_g33)
-#define IS_IRONLAKE_M(dev)	(INTEL_DEVID(dev) == 0x0046)
-#define IS_IVYBRIDGE(dev)	(INTEL_INFO(dev)->is_ivybridge)
-#define IS_IVB_GT1(dev)		(INTEL_DEVID(dev) == 0x0156 || \
-				 INTEL_DEVID(dev) == 0x0152 || \
-				 INTEL_DEVID(dev) == 0x015a)
-#define IS_VALLEYVIEW(dev)	(INTEL_INFO(dev)->is_valleyview)
-#define IS_CHERRYVIEW(dev)	(INTEL_INFO(dev)->is_cherryview)
-#define IS_HASWELL(dev)	(INTEL_INFO(dev)->is_haswell)
-#define IS_BROADWELL(dev)	(INTEL_INFO(dev)->is_broadwell)
-#define IS_SKYLAKE(dev)	(INTEL_INFO(dev)->is_skylake)
-#define IS_BROXTON(dev)		(INTEL_INFO(dev)->is_broxton)
-#define IS_KABYLAKE(dev)	(INTEL_INFO(dev)->is_kabylake)
-#define IS_MOBILE(dev)		(INTEL_INFO(dev)->is_mobile)
-#define IS_HSW_EARLY_SDV(dev)	(IS_HASWELL(dev) && \
-				 (INTEL_DEVID(dev) & 0xFF00) == 0x0C00)
-#define IS_BDW_ULT(dev)		(IS_BROADWELL(dev) && \
-				 ((INTEL_DEVID(dev) & 0xf) == 0x6 ||	\
-				 (INTEL_DEVID(dev) & 0xf) == 0xb ||	\
-				 (INTEL_DEVID(dev) & 0xf) == 0xe))
+#define IS_I830(dev_priv)	(INTEL_DEVID(dev_priv) == 0x3577)
+#define IS_845G(dev_priv)	(INTEL_DEVID(dev_priv) == 0x2562)
+#define IS_I85X(dev_priv)	((dev_priv)->info.is_i85x)
+#define IS_I865G(dev_priv)	(INTEL_DEVID(dev_priv) == 0x2572)
+#define IS_I915G(dev_priv)	((dev_priv)->info.is_i915g)
+#define IS_I915GM(dev_priv)	(INTEL_DEVID(dev_priv) == 0x2592)
+#define IS_I945G(dev_priv)	(INTEL_DEVID(dev_priv) == 0x2772)
+#define IS_I945GM(dev_priv)	((dev_priv)->info.is_i945gm)
+#define IS_BROADWATER(dev_priv)	((dev_priv)->info.is_broadwater)
+#define IS_CRESTLINE(dev_priv)	((dev_priv)->info.is_crestline)
+#define IS_GM45(dev_priv)	(INTEL_DEVID(dev_priv) == 0x2A42)
+#define IS_G4X(dev_priv)	((dev_priv)->info.is_g4x)
+#define IS_PINEVIEW_G(dev_priv)	(INTEL_DEVID(dev_priv) == 0xa001)
+#define IS_PINEVIEW_M(dev_priv)	(INTEL_DEVID(dev_priv) == 0xa011)
+#define IS_PINEVIEW(dev_priv)	((dev_priv)->info.is_pineview)
+#define IS_G33(dev_priv)	((dev_priv)->info.is_g33)
+#define IS_IRONLAKE_M(dev_priv)	(INTEL_DEVID(dev_priv) == 0x0046)
+#define IS_IVYBRIDGE(dev_priv)	((dev_priv)->info.is_ivybridge)
+#define IS_IVB_GT1(dev_priv)	(INTEL_DEVID(dev_priv) == 0x0156 || \
+				 INTEL_DEVID(dev_priv) == 0x0152 || \
+				 INTEL_DEVID(dev_priv) == 0x015a)
+#define IS_VALLEYVIEW(dev_priv)	((dev_priv)->info.is_valleyview)
+#define IS_CHERRYVIEW(dev_priv)	((dev_priv)->info.is_cherryview)
+#define IS_HASWELL(dev_priv)	((dev_priv)->info.is_haswell)
+#define IS_BROADWELL(dev_priv)	((dev_priv)->info.is_broadwell)
+#define IS_SKYLAKE(dev_priv)	((dev_priv)->info.is_skylake)
+#define IS_BROXTON(dev_priv)	((dev_priv)->info.is_broxton)
+#define IS_KABYLAKE(dev_priv)	((dev_priv)->info.is_kabylake)
+#define IS_MOBILE(dev_priv)	((dev_priv)->info.is_mobile)
+#define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \
+				    (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00)
+#define IS_BDW_ULT(dev_priv)	(IS_BROADWELL(dev_priv) && \
+				 ((INTEL_DEVID(dev_priv) & 0xf) == 0x6 ||	\
+				 (INTEL_DEVID(dev_priv) & 0xf) == 0xb ||	\
+				 (INTEL_DEVID(dev_priv) & 0xf) == 0xe))
 /* ULX machines are also considered ULT. */
-#define IS_BDW_ULX(dev)		(IS_BROADWELL(dev) && \
-				 (INTEL_DEVID(dev) & 0xf) == 0xe)
-#define IS_BDW_GT3(dev)		(IS_BROADWELL(dev) && \
-				 (INTEL_DEVID(dev) & 0x00F0) == 0x0020)
-#define IS_HSW_ULT(dev)		(IS_HASWELL(dev) && \
-				 (INTEL_DEVID(dev) & 0xFF00) == 0x0A00)
-#define IS_HSW_GT3(dev)		(IS_HASWELL(dev) && \
-				 (INTEL_DEVID(dev) & 0x00F0) == 0x0020)
+#define IS_BDW_ULX(dev_priv)	(IS_BROADWELL(dev_priv) && \
+				 (INTEL_DEVID(dev_priv) & 0xf) == 0xe)
+#define IS_BDW_GT3(dev_priv)	(IS_BROADWELL(dev_priv) && \
+				 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020)
+#define IS_HSW_ULT(dev_priv)	(IS_HASWELL(dev_priv) && \
+				 (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0A00)
+#define IS_HSW_GT3(dev_priv)	(IS_HASWELL(dev_priv) && \
+				 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020)
 /* ULX machines are also considered ULT. */
-#define IS_HSW_ULX(dev)		(INTEL_DEVID(dev) == 0x0A0E || \
-				 INTEL_DEVID(dev) == 0x0A1E)
-#define IS_SKL_ULT(dev)		(INTEL_DEVID(dev) == 0x1906 || \
-				 INTEL_DEVID(dev) == 0x1913 || \
-				 INTEL_DEVID(dev) == 0x1916 || \
-				 INTEL_DEVID(dev) == 0x1921 || \
-				 INTEL_DEVID(dev) == 0x1926)
-#define IS_SKL_ULX(dev)		(INTEL_DEVID(dev) == 0x190E || \
-				 INTEL_DEVID(dev) == 0x1915 || \
-				 INTEL_DEVID(dev) == 0x191E)
-#define IS_KBL_ULT(dev)		(INTEL_DEVID(dev) == 0x5906 || \
-				 INTEL_DEVID(dev) == 0x5913 || \
-				 INTEL_DEVID(dev) == 0x5916 || \
-				 INTEL_DEVID(dev) == 0x5921 || \
-				 INTEL_DEVID(dev) == 0x5926)
-#define IS_KBL_ULX(dev)		(INTEL_DEVID(dev) == 0x590E || \
-				 INTEL_DEVID(dev) == 0x5915 || \
-				 INTEL_DEVID(dev) == 0x591E)
-#define IS_SKL_GT3(dev)		(IS_SKYLAKE(dev) && \
-				 (INTEL_DEVID(dev) & 0x00F0) == 0x0020)
-#define IS_SKL_GT4(dev)		(IS_SKYLAKE(dev) && \
-				 (INTEL_DEVID(dev) & 0x00F0) == 0x0030)
-
-#define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary)
+#define IS_HSW_ULX(dev_priv)	(INTEL_DEVID(dev_priv) == 0x0A0E || \
+				 INTEL_DEVID(dev_priv) == 0x0A1E)
+#define IS_SKL_ULT(dev_priv)	(INTEL_DEVID(dev_priv) == 0x1906 || \
+				 INTEL_DEVID(dev_priv) == 0x1913 || \
+				 INTEL_DEVID(dev_priv) == 0x1916 || \
+				 INTEL_DEVID(dev_priv) == 0x1921 || \
+				 INTEL_DEVID(dev_priv) == 0x1926)
+#define IS_SKL_ULX(dev_priv)	(INTEL_DEVID(dev_priv) == 0x190E || \
+				 INTEL_DEVID(dev_priv) == 0x1915 || \
+				 INTEL_DEVID(dev_priv) == 0x191E)
+#define IS_KBL_ULT(dev_priv)	(INTEL_DEVID(dev_priv) == 0x5906 || \
+				 INTEL_DEVID(dev_priv) == 0x5913 || \
+				 INTEL_DEVID(dev_priv) == 0x5916 || \
+				 INTEL_DEVID(dev_priv) == 0x5921 || \
+				 INTEL_DEVID(dev_priv) == 0x5926)
+#define IS_KBL_ULX(dev_priv)	(INTEL_DEVID(dev_priv) == 0x590E || \
+				 INTEL_DEVID(dev_priv) == 0x5915 || \
+				 INTEL_DEVID(dev_priv) == 0x591E)
+#define IS_SKL_GT3(dev_priv)	(IS_SKYLAKE(dev_priv) && \
+				 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020)
+#define IS_SKL_GT4(dev_priv)	(IS_SKYLAKE(dev_priv) && \
+				 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0030)
+
+#define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support)
 
 #define SKL_REVID_A0		0x0
 #define SKL_REVID_B0		0x1
@@ -2705,7 +2504,8 @@ struct drm_i915_cmd_table {
 #define BXT_REVID_B0		0x3
 #define BXT_REVID_C0		0x9
 
-#define IS_BXT_REVID(p, since, until) (IS_BROXTON(p) && IS_REVID(p, since, until))
+#define IS_BXT_REVID(dev_priv, since, until) \
+	(IS_BROXTON(dev_priv) && IS_REVID(dev_priv, since, until))
 
 #define KBL_REVID_A0		0x0
 #define KBL_REVID_B0		0x1
@@ -2713,8 +2513,8 @@ struct drm_i915_cmd_table {
 #define KBL_REVID_D0		0x3
 #define KBL_REVID_E0		0x4
 
-#define IS_KBL_REVID(p, since, until) \
-	(IS_KABYLAKE(p) && IS_REVID(p, since, until))
+#define IS_KBL_REVID(dev_priv, since, until) \
+	(IS_KABYLAKE(dev_priv) && IS_REVID(dev_priv, since, until))
 
 /*
  * The genX designation typically refers to the render engine, so render
@@ -2722,14 +2522,14 @@ struct drm_i915_cmd_table {
  * have their own (e.g. HAS_PCH_SPLIT for ILK+ display, IS_foo for particular
  * chips, etc.).
  */
-#define IS_GEN2(dev)	(!!(INTEL_INFO(dev)->gen_mask & BIT(1)))
-#define IS_GEN3(dev)	(!!(INTEL_INFO(dev)->gen_mask & BIT(2)))
-#define IS_GEN4(dev)	(!!(INTEL_INFO(dev)->gen_mask & BIT(3)))
-#define IS_GEN5(dev)	(!!(INTEL_INFO(dev)->gen_mask & BIT(4)))
-#define IS_GEN6(dev)	(!!(INTEL_INFO(dev)->gen_mask & BIT(5)))
-#define IS_GEN7(dev)	(!!(INTEL_INFO(dev)->gen_mask & BIT(6)))
-#define IS_GEN8(dev)	(!!(INTEL_INFO(dev)->gen_mask & BIT(7)))
-#define IS_GEN9(dev)	(!!(INTEL_INFO(dev)->gen_mask & BIT(8)))
+#define IS_GEN2(dev_priv)	(!!((dev_priv)->info.gen_mask & BIT(1)))
+#define IS_GEN3(dev_priv)	(!!((dev_priv)->info.gen_mask & BIT(2)))
+#define IS_GEN4(dev_priv)	(!!((dev_priv)->info.gen_mask & BIT(3)))
+#define IS_GEN5(dev_priv)	(!!((dev_priv)->info.gen_mask & BIT(4)))
+#define IS_GEN6(dev_priv)	(!!((dev_priv)->info.gen_mask & BIT(5)))
+#define IS_GEN7(dev_priv)	(!!((dev_priv)->info.gen_mask & BIT(6)))
+#define IS_GEN8(dev_priv)	(!!((dev_priv)->info.gen_mask & BIT(7)))
+#define IS_GEN9(dev_priv)	(!!((dev_priv)->info.gen_mask & BIT(8)))
 
 #define ENGINE_MASK(id)	BIT(id)
 #define RENDER_RING	ENGINE_MASK(RCS)
@@ -2740,31 +2540,34 @@ struct drm_i915_cmd_table {
 #define ALL_ENGINES	(~0)
 
 #define HAS_ENGINE(dev_priv, id) \
-	(!!(INTEL_INFO(dev_priv)->ring_mask & ENGINE_MASK(id)))
+	(!!((dev_priv)->info.ring_mask & ENGINE_MASK(id)))
 
 #define HAS_BSD(dev_priv)	HAS_ENGINE(dev_priv, VCS)
 #define HAS_BSD2(dev_priv)	HAS_ENGINE(dev_priv, VCS2)
 #define HAS_BLT(dev_priv)	HAS_ENGINE(dev_priv, BCS)
 #define HAS_VEBOX(dev_priv)	HAS_ENGINE(dev_priv, VECS)
 
-#define HAS_LLC(dev)		(INTEL_INFO(dev)->has_llc)
-#define HAS_SNOOP(dev)		(INTEL_INFO(dev)->has_snoop)
-#define HAS_EDRAM(dev)		(!!(__I915__(dev)->edram_cap & EDRAM_ENABLED))
-#define HAS_WT(dev)		((IS_HASWELL(dev) || IS_BROADWELL(dev)) && \
-				 HAS_EDRAM(dev))
-#define HWS_NEEDS_PHYSICAL(dev)	(INTEL_INFO(dev)->hws_needs_physical)
+#define HAS_LLC(dev_priv)	((dev_priv)->info.has_llc)
+#define HAS_SNOOP(dev_priv)	((dev_priv)->info.has_snoop)
+#define HAS_EDRAM(dev_priv)	(!!((dev_priv)->edram_cap & EDRAM_ENABLED))
+#define HAS_WT(dev_priv)	((IS_HASWELL(dev_priv) || \
+				 IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv))
 
-#define HAS_HW_CONTEXTS(dev)	(INTEL_INFO(dev)->has_hw_contexts)
-#define HAS_LOGICAL_RING_CONTEXTS(dev)	(INTEL_INFO(dev)->has_logical_ring_contexts)
-#define USES_PPGTT(dev)		(i915.enable_ppgtt)
-#define USES_FULL_PPGTT(dev)	(i915.enable_ppgtt >= 2)
-#define USES_FULL_48BIT_PPGTT(dev)	(i915.enable_ppgtt == 3)
+#define HWS_NEEDS_PHYSICAL(dev_priv)	((dev_priv)->info.hws_needs_physical)
 
-#define HAS_OVERLAY(dev)		(INTEL_INFO(dev)->has_overlay)
-#define OVERLAY_NEEDS_PHYSICAL(dev)	(INTEL_INFO(dev)->overlay_needs_physical)
+#define HAS_HW_CONTEXTS(dev_priv)	    ((dev_priv)->info.has_hw_contexts)
+#define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \
+		((dev_priv)->info.has_logical_ring_contexts)
+#define USES_PPGTT(dev_priv)		(i915.enable_ppgtt)
+#define USES_FULL_PPGTT(dev_priv)	(i915.enable_ppgtt >= 2)
+#define USES_FULL_48BIT_PPGTT(dev_priv)	(i915.enable_ppgtt == 3)
+
+#define HAS_OVERLAY(dev_priv)		 ((dev_priv)->info.has_overlay)
+#define OVERLAY_NEEDS_PHYSICAL(dev_priv) \
+		((dev_priv)->info.overlay_needs_physical)
 
 /* Early gen2 have a totally busted CS tlb and require pinned batches. */
-#define HAS_BROKEN_CS_TLB(dev)		(IS_I830(dev) || IS_845G(dev))
+#define HAS_BROKEN_CS_TLB(dev_priv)	(IS_I830(dev_priv) || IS_845G(dev_priv))
 
 /* WaRsDisableCoarsePowerGating:skl,bxt */
 #define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \
@@ -2778,46 +2581,49 @@ struct drm_i915_cmd_table {
  * legacy irq no. is shared with another device. The kernel then disables that
  * interrupt source and so prevents the other device from working properly.
  */
-#define HAS_AUX_IRQ(dev) (INTEL_INFO(dev)->gen >= 5)
-#define HAS_GMBUS_IRQ(dev) (INTEL_INFO(dev)->has_gmbus_irq)
+#define HAS_AUX_IRQ(dev_priv)   ((dev_priv)->info.gen >= 5)
+#define HAS_GMBUS_IRQ(dev_priv) ((dev_priv)->info.has_gmbus_irq)
 
 /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
  * rows, which changed the alignment requirements and fence programming.
  */
-#define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \
-						      IS_I915GM(dev)))
-#define SUPPORTS_TV(dev)		(INTEL_INFO(dev)->supports_tv)
-#define I915_HAS_HOTPLUG(dev)		 (INTEL_INFO(dev)->has_hotplug)
+#define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN2(dev_priv) && \
+					 !(IS_I915G(dev_priv) || \
+					 IS_I915GM(dev_priv)))
+#define SUPPORTS_TV(dev_priv)		((dev_priv)->info.supports_tv)
+#define I915_HAS_HOTPLUG(dev_priv)	((dev_priv)->info.has_hotplug)
+
+#define HAS_FW_BLC(dev_priv) 	(INTEL_GEN(dev_priv) > 2)
+#define HAS_PIPE_CXSR(dev_priv) ((dev_priv)->info.has_pipe_cxsr)
+#define HAS_FBC(dev_priv)	((dev_priv)->info.has_fbc)
 
-#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2)
-#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
-#define HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
+#define HAS_IPS(dev_priv)	(IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv))
 
-#define HAS_IPS(dev)		(IS_HSW_ULT(dev) || IS_BROADWELL(dev))
+#define HAS_DP_MST(dev_priv)	((dev_priv)->info.has_dp_mst)
 
-#define HAS_DP_MST(dev)	(INTEL_INFO(dev)->has_dp_mst)
+#define HAS_DDI(dev_priv)		 ((dev_priv)->info.has_ddi)
+#define HAS_FPGA_DBG_UNCLAIMED(dev_priv) ((dev_priv)->info.has_fpga_dbg)
+#define HAS_PSR(dev_priv)		 ((dev_priv)->info.has_psr)
+#define HAS_RC6(dev_priv)		 ((dev_priv)->info.has_rc6)
+#define HAS_RC6p(dev_priv)		 ((dev_priv)->info.has_rc6p)
 
-#define HAS_DDI(dev)		(INTEL_INFO(dev)->has_ddi)
-#define HAS_FPGA_DBG_UNCLAIMED(dev)	(INTEL_INFO(dev)->has_fpga_dbg)
-#define HAS_PSR(dev)		(INTEL_INFO(dev)->has_psr)
-#define HAS_RUNTIME_PM(dev)	(INTEL_INFO(dev)->has_runtime_pm)
-#define HAS_RC6(dev)		(INTEL_INFO(dev)->has_rc6)
-#define HAS_RC6p(dev)		(INTEL_INFO(dev)->has_rc6p)
+#define HAS_CSR(dev_priv)	((dev_priv)->info.has_csr)
 
-#define HAS_CSR(dev)	(INTEL_INFO(dev)->has_csr)
+#define HAS_RUNTIME_PM(dev_priv) ((dev_priv)->info.has_runtime_pm)
+#define HAS_64BIT_RELOC(dev_priv) ((dev_priv)->info.has_64bit_reloc)
 
 /*
  * For now, anything with a GuC requires uCode loading, and then supports
  * command submission once loaded. But these are logically independent
  * properties, so we have separate macros to test them.
  */
-#define HAS_GUC(dev)		(INTEL_INFO(dev)->has_guc)
-#define HAS_GUC_UCODE(dev)	(HAS_GUC(dev))
-#define HAS_GUC_SCHED(dev)	(HAS_GUC(dev))
+#define HAS_GUC(dev_priv)	((dev_priv)->info.has_guc)
+#define HAS_GUC_UCODE(dev_priv)	(HAS_GUC(dev_priv))
+#define HAS_GUC_SCHED(dev_priv)	(HAS_GUC(dev_priv))
 
-#define HAS_RESOURCE_STREAMER(dev) (INTEL_INFO(dev)->has_resource_streamer)
+#define HAS_RESOURCE_STREAMER(dev_priv) ((dev_priv)->info.has_resource_streamer)
 
-#define HAS_POOLED_EU(dev)	(INTEL_INFO(dev)->has_pooled_eu)
+#define HAS_POOLED_EU(dev_priv)	((dev_priv)->info.has_pooled_eu)
 
 #define INTEL_PCH_DEVICE_ID_MASK		0xff00
 #define INTEL_PCH_IBX_DEVICE_ID_TYPE		0x3b00
@@ -2832,26 +2638,33 @@ struct drm_i915_cmd_table {
 #define INTEL_PCH_P3X_DEVICE_ID_TYPE		0x7000
 #define INTEL_PCH_QEMU_DEVICE_ID_TYPE		0x2900 /* qemu q35 has 2918 */
 
-#define INTEL_PCH_TYPE(dev) (__I915__(dev)->pch_type)
-#define HAS_PCH_KBP(dev) (INTEL_PCH_TYPE(dev) == PCH_KBP)
-#define HAS_PCH_SPT(dev) (INTEL_PCH_TYPE(dev) == PCH_SPT)
-#define HAS_PCH_LPT(dev) (INTEL_PCH_TYPE(dev) == PCH_LPT)
-#define HAS_PCH_LPT_LP(dev) (__I915__(dev)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
-#define HAS_PCH_LPT_H(dev) (__I915__(dev)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE)
-#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
-#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
-#define HAS_PCH_NOP(dev) (INTEL_PCH_TYPE(dev) == PCH_NOP)
-#define HAS_PCH_SPLIT(dev) (INTEL_PCH_TYPE(dev) != PCH_NONE)
+#define INTEL_PCH_TYPE(dev_priv) ((dev_priv)->pch_type)
+#define HAS_PCH_KBP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_KBP)
+#define HAS_PCH_SPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_SPT)
+#define HAS_PCH_LPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_LPT)
+#define HAS_PCH_LPT_LP(dev_priv) \
+	((dev_priv)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
+#define HAS_PCH_LPT_H(dev_priv) \
+	((dev_priv)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE)
+#define HAS_PCH_CPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CPT)
+#define HAS_PCH_IBX(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_IBX)
+#define HAS_PCH_NOP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_NOP)
+#define HAS_PCH_SPLIT(dev_priv) (INTEL_PCH_TYPE(dev_priv) != PCH_NONE)
 
-#define HAS_GMCH_DISPLAY(dev) (INTEL_INFO(dev)->has_gmch_display)
+#define HAS_GMCH_DISPLAY(dev_priv) ((dev_priv)->info.has_gmch_display)
+
+#define HAS_LSPCON(dev_priv) (IS_GEN9(dev_priv))
 
 /* DPF == dynamic parity feature */
-#define HAS_L3_DPF(dev) (INTEL_INFO(dev)->has_l3_dpf)
-#define NUM_L3_SLICES(dev) (IS_HSW_GT3(dev) ? 2 : HAS_L3_DPF(dev))
+#define HAS_L3_DPF(dev_priv) ((dev_priv)->info.has_l3_dpf)
+#define NUM_L3_SLICES(dev_priv) (IS_HSW_GT3(dev_priv) ? \
+				 2 : HAS_L3_DPF(dev_priv))
 
 #define GT_FREQUENCY_MULTIPLIER 50
 #define GEN9_FREQ_SCALER 3
 
+#define HAS_DECOUPLED_MMIO(dev_priv) (INTEL_INFO(dev_priv)->has_decoupled_mmio)
+
 #include "i915_trace.h"
 
 static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv)
@@ -2882,6 +2695,8 @@ __i915_printk(struct drm_i915_private *dev_priv, const char *level,
 #ifdef CONFIG_COMPAT
 extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
 			      unsigned long arg);
+#else
+#define i915_compat_ioctl NULL
 #endif
 extern const struct dev_pm_ops i915_pm_ops;
 
@@ -2893,6 +2708,7 @@ extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv);
 extern void i915_reset(struct drm_i915_private *dev_priv);
 extern int intel_guc_reset(struct drm_i915_private *dev_priv);
 extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine);
+extern void intel_hangcheck_init(struct drm_i915_private *dev_priv);
 extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
 extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv);
 extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv);
@@ -2974,7 +2790,7 @@ int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
 
 static inline bool intel_gvt_active(struct drm_i915_private *dev_priv)
 {
-	return dev_priv->gvt.initialized;
+	return dev_priv->gvt;
 }
 
 static inline bool intel_vgpu_active(struct drm_i915_private *dev_priv)
@@ -3076,7 +2892,7 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *file_priv);
 int i915_gem_wait_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
-void i915_gem_load_init(struct drm_device *dev);
+int i915_gem_load_init(struct drm_device *dev);
 void i915_gem_load_cleanup(struct drm_device *dev);
 void i915_gem_load_init_fences(struct drm_i915_private *dev_priv);
 int i915_gem_freeze(struct drm_i915_private *dev_priv);
@@ -3087,7 +2903,7 @@ void i915_gem_object_free(struct drm_i915_gem_object *obj);
 void i915_gem_object_init(struct drm_i915_gem_object *obj,
 			 const struct drm_i915_gem_object_ops *ops);
 struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
-						  size_t size);
+						   u64 size);
 struct drm_i915_gem_object *i915_gem_object_create_from_data(
 		struct drm_device *dev, const void *data, size_t size);
 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file);
@@ -3100,77 +2916,86 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 			 u64 alignment,
 			 u64 flags);
 
-int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
-		  u32 flags);
-void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
-int __must_check i915_vma_unbind(struct i915_vma *vma);
-void i915_vma_close(struct i915_vma *vma);
-void i915_vma_destroy(struct i915_vma *vma);
-
 int i915_gem_object_unbind(struct drm_i915_gem_object *obj);
-int i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
-void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv);
 void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
 
-int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
+void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv);
 
-static inline int __sg_page_count(struct scatterlist *sg)
+static inline int __sg_page_count(const struct scatterlist *sg)
 {
 	return sg->length >> PAGE_SHIFT;
 }
 
+struct scatterlist *
+i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
+		       unsigned int n, unsigned int *offset);
+
+struct page *
+i915_gem_object_get_page(struct drm_i915_gem_object *obj,
+			 unsigned int n);
+
 struct page *
-i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n);
+i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
+			       unsigned int n);
 
-static inline dma_addr_t
-i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, int n)
+dma_addr_t
+i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
+				unsigned long n);
+
+void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
+				 struct sg_table *pages);
+int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
+
+static inline int __must_check
+i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
 {
-	if (n < obj->get_page.last) {
-		obj->get_page.sg = obj->pages->sgl;
-		obj->get_page.last = 0;
-	}
+	might_lock(&obj->mm.lock);
 
-	while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) {
-		obj->get_page.last += __sg_page_count(obj->get_page.sg++);
-		if (unlikely(sg_is_chain(obj->get_page.sg)))
-			obj->get_page.sg = sg_chain_ptr(obj->get_page.sg);
-	}
+	if (atomic_inc_not_zero(&obj->mm.pages_pin_count))
+		return 0;
 
-	return sg_dma_address(obj->get_page.sg) + ((n - obj->get_page.last) << PAGE_SHIFT);
+	return __i915_gem_object_get_pages(obj);
 }
 
-static inline struct page *
-i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n)
+static inline void
+__i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
 {
-	if (WARN_ON(n >= obj->base.size >> PAGE_SHIFT))
-		return NULL;
-
-	if (n < obj->get_page.last) {
-		obj->get_page.sg = obj->pages->sgl;
-		obj->get_page.last = 0;
-	}
+	GEM_BUG_ON(!obj->mm.pages);
 
-	while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) {
-		obj->get_page.last += __sg_page_count(obj->get_page.sg++);
-		if (unlikely(sg_is_chain(obj->get_page.sg)))
-			obj->get_page.sg = sg_chain_ptr(obj->get_page.sg);
-	}
+	atomic_inc(&obj->mm.pages_pin_count);
+}
 
-	return nth_page(sg_page(obj->get_page.sg), n - obj->get_page.last);
+static inline bool
+i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj)
+{
+	return atomic_read(&obj->mm.pages_pin_count);
 }
 
-static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
+static inline void
+__i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
 {
-	BUG_ON(obj->pages == NULL);
-	obj->pages_pin_count++;
+	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
+	GEM_BUG_ON(!obj->mm.pages);
+
+	atomic_dec(&obj->mm.pages_pin_count);
+	GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count);
 }
 
-static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
+static inline void
+i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
 {
-	BUG_ON(obj->pages_pin_count == 0);
-	obj->pages_pin_count--;
+	__i915_gem_object_unpin_pages(obj);
 }
 
+enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock */
+	I915_MM_NORMAL = 0,
+	I915_MM_SHRINKER
+};
+
+void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj,
+				 enum i915_mm_subclass subclass);
+void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj);
+
 enum i915_map_type {
 	I915_MAP_WB = 0,
 	I915_MAP_WC,
@@ -3186,8 +3011,8 @@ enum i915_map_type {
  * the kernel address space. Based on the @type of mapping, the PTE will be
  * set to either WriteBack or WriteCombine (via pgprot_t).
  *
- * The caller must hold the struct_mutex, and is responsible for calling
- * i915_gem_object_unpin_map() when the mapping is no longer required.
+ * The caller is responsible for calling i915_gem_object_unpin_map() when the
+ * mapping is no longer required.
  *
  * Returns the pointer through which to access the mapped object, or an
  * ERR_PTR() on error.
@@ -3203,12 +3028,9 @@ void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
  * with your access, call i915_gem_object_unpin_map() to release the pin
  * upon the mapping. Once the pin count reaches zero, that mapping may be
  * removed.
- *
- * The caller must hold the struct_mutex.
  */
 static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj)
 {
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
 	i915_gem_object_unpin_pages(obj);
 }
 
@@ -3241,7 +3063,7 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
 		       struct drm_i915_gem_object *new,
 		       unsigned frontbuffer_bits);
 
-int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno);
+int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno);
 
 struct drm_i915_gem_request *
 i915_gem_find_active_request(struct intel_engine_cs *engine);
@@ -3270,19 +3092,25 @@ static inline u32 i915_reset_count(struct i915_gpu_error *error)
 
 void i915_gem_reset(struct drm_i915_private *dev_priv);
 void i915_gem_set_wedged(struct drm_i915_private *dev_priv);
-bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force);
+void i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force);
 int __must_check i915_gem_init(struct drm_device *dev);
 int __must_check i915_gem_init_hw(struct drm_device *dev);
-void i915_gem_init_swizzling(struct drm_device *dev);
+void i915_gem_init_swizzling(struct drm_i915_private *dev_priv);
 void i915_gem_cleanup_engines(struct drm_device *dev);
 int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
 					unsigned int flags);
 int __must_check i915_gem_suspend(struct drm_device *dev);
 void i915_gem_resume(struct drm_device *dev);
 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
-int __must_check
-i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
-			       bool readonly);
+int i915_gem_object_wait(struct drm_i915_gem_object *obj,
+			 unsigned int flags,
+			 long timeout,
+			 struct intel_rps_client *rps);
+int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
+				  unsigned int flags,
+				  int priority);
+#define I915_PRIORITY_DISPLAY I915_PRIORITY_MAX
+
 int __must_check
 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
 				  bool write);
@@ -3342,57 +3170,17 @@ i915_gem_object_ggtt_offset(struct drm_i915_gem_object *o,
 	return i915_ggtt_offset(i915_gem_object_to_ggtt(o, view));
 }
 
-/* i915_gem_fence.c */
+/* i915_gem_fence_reg.c */
 int __must_check i915_vma_get_fence(struct i915_vma *vma);
 int __must_check i915_vma_put_fence(struct i915_vma *vma);
 
-/**
- * i915_vma_pin_fence - pin fencing state
- * @vma: vma to pin fencing for
- *
- * This pins the fencing state (whether tiled or untiled) to make sure the
- * vma (and its object) is ready to be used as a scanout target. Fencing
- * status must be synchronize first by calling i915_vma_get_fence():
- *
- * The resulting fence pin reference must be released again with
- * i915_vma_unpin_fence().
- *
- * Returns:
- *
- * True if the vma has a fence, false otherwise.
- */
-static inline bool
-i915_vma_pin_fence(struct i915_vma *vma)
-{
-	if (vma->fence) {
-		vma->fence->pin_count++;
-		return true;
-	} else
-		return false;
-}
-
-/**
- * i915_vma_unpin_fence - unpin fencing state
- * @vma: vma to unpin fencing for
- *
- * This releases the fence pin reference acquired through
- * i915_vma_pin_fence. It will handle both objects with and without an
- * attached fence correctly, callers do not need to distinguish this.
- */
-static inline void
-i915_vma_unpin_fence(struct i915_vma *vma)
-{
-	if (vma->fence) {
-		GEM_BUG_ON(vma->fence->pin_count <= 0);
-		vma->fence->pin_count--;
-	}
-}
-
-void i915_gem_restore_fences(struct drm_device *dev);
+void i915_gem_restore_fences(struct drm_i915_private *dev_priv);
 
-void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
-void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj);
-void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
+void i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv);
+void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj,
+				       struct sg_table *pages);
+void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj,
+					 struct sg_table *pages);
 
 /* i915_gem_context.c */
 int __must_check i915_gem_context_init(struct drm_device *dev);
@@ -3402,6 +3190,9 @@ int i915_gem_context_open(struct drm_device *dev, struct drm_file *file);
 void i915_gem_context_close(struct drm_device *dev, struct drm_file *file);
 int i915_switch_context(struct drm_i915_gem_request *req);
 int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv);
+struct i915_vma *
+i915_gem_context_pin_legacy(struct i915_gem_context *ctx,
+			    unsigned int flags);
 void i915_gem_context_free(struct kref *ctx_ref);
 struct drm_i915_gem_object *
 i915_gem_alloc_context_obj(struct drm_device *dev, size_t size);
@@ -3435,6 +3226,16 @@ static inline void i915_gem_context_put(struct i915_gem_context *ctx)
 	kref_put(&ctx->ref, i915_gem_context_free);
 }
 
+static inline struct intel_timeline *
+i915_gem_context_lookup_timeline(struct i915_gem_context *ctx,
+				 struct intel_engine_cs *engine)
+{
+	struct i915_address_space *vm;
+
+	vm = ctx->ppgtt ? &ctx->ppgtt->base : &ctx->i915->ggtt.base;
+	return &vm->timeline.engine[engine->id];
+}
+
 static inline bool i915_gem_context_is_default(const struct i915_gem_context *c)
 {
 	return c->user_handle == DEFAULT_CONTEXT_HANDLE;
@@ -3478,7 +3279,7 @@ int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv,
 					 u64 end);
 void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
 				 struct drm_mm_node *node);
-int i915_gem_init_stolen(struct drm_device *dev);
+int i915_gem_init_stolen(struct drm_i915_private *dev_priv);
 void i915_gem_cleanup_stolen(struct drm_device *dev);
 struct drm_i915_gem_object *
 i915_gem_object_create_stolen(struct drm_device *dev, u32 size);
@@ -3488,6 +3289,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
 					       u32 gtt_offset,
 					       u32 size);
 
+/* i915_gem_internal.c */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_i915_private *dev_priv,
+				unsigned int size);
+
 /* i915_gem_shrinker.c */
 unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
 			      unsigned long target,
@@ -3526,6 +3332,8 @@ static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {}
 #endif
 
 /* i915_gpu_error.c */
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+
 __printf(2, 3)
 void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...);
 int i915_error_state_to_str(struct drm_i915_error_state_buf *estr,
@@ -3546,7 +3354,20 @@ void i915_error_state_get(struct drm_device *dev,
 void i915_error_state_put(struct i915_error_state_file_priv *error_priv);
 void i915_destroy_error_state(struct drm_device *dev);
 
-void i915_get_extra_instdone(struct drm_i915_private *dev_priv, uint32_t *instdone);
+#else
+
+static inline void i915_capture_error_state(struct drm_i915_private *dev_priv,
+					    u32 engine_mask,
+					    const char *error_msg)
+{
+}
+
+static inline void i915_destroy_error_state(struct drm_device *dev)
+{
+}
+
+#endif
+
 const char *i915_cache_level_str(struct drm_i915_private *i915, int type);
 
 /* i915_cmd_parser.c */
@@ -3596,6 +3417,9 @@ bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, enum por
 bool intel_bios_is_dsi_present(struct drm_i915_private *dev_priv, enum port *port);
 bool intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv,
 				     enum port port);
+bool intel_bios_is_lspcon_present(struct drm_i915_private *dev_priv,
+				enum port port);
+
 
 /* intel_opregion.c */
 #ifdef CONFIG_ACPI
@@ -3652,15 +3476,16 @@ void intel_device_info_dump(struct drm_i915_private *dev_priv);
 
 /* modesetting */
 extern void intel_modeset_init_hw(struct drm_device *dev);
-extern void intel_modeset_init(struct drm_device *dev);
+extern int intel_modeset_init(struct drm_device *dev);
 extern void intel_modeset_gem_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
 extern int intel_connector_register(struct drm_connector *);
 extern void intel_connector_unregister(struct drm_connector *);
-extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state);
+extern int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv,
+				       bool state);
 extern void intel_display_resume(struct drm_device *dev);
-extern void i915_redisable_vga(struct drm_device *dev);
-extern void i915_redisable_vga_power_on(struct drm_device *dev);
+extern void i915_redisable_vga(struct drm_i915_private *dev_priv);
+extern void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv);
 extern bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val);
 extern void intel_init_pch_refclk(struct drm_device *dev);
 extern void intel_set_rps(struct drm_i915_private *dev_priv, u8 val);
@@ -3679,7 +3504,7 @@ extern void intel_overlay_print_error_state(struct drm_i915_error_state_buf *e,
 extern struct intel_display_error_state *
 intel_display_capture_error_state(struct drm_i915_private *dev_priv);
 extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e,
-					    struct drm_device *dev,
+					    struct drm_i915_private *dev_priv,
 					    struct intel_display_error_state *error);
 
 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val);
@@ -3707,6 +3532,23 @@ u32 vlv_flisdsi_read(struct drm_i915_private *dev_priv, u32 reg);
 void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val);
 
 /* intel_dpio_phy.c */
+void bxt_port_to_phy_channel(enum port port,
+			     enum dpio_phy *phy, enum dpio_channel *ch);
+void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv,
+				  enum port port, u32 margin, u32 scale,
+				  u32 enable, u32 deemphasis);
+void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy);
+void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy);
+bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv,
+			    enum dpio_phy phy);
+bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
+			      enum dpio_phy phy);
+uint8_t bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder,
+					     uint8_t lane_count);
+void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder,
+				     uint8_t lane_lat_optim_mask);
+uint8_t bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder);
+
 void chv_set_phy_signal_level(struct intel_encoder *encoder,
 			      u32 deemph_reg_value, u32 margin_reg_value,
 			      bool uniq_trans_scale);
@@ -3796,11 +3638,30 @@ __raw_write(64, q)
 #undef __raw_write
 
 /* These are untraced mmio-accessors that are only valid to be used inside
- * critical sections inside IRQ handlers where forcewake is explicitly
+ * critical sections, such as inside IRQ handlers, where forcewake is explicitly
  * controlled.
+ *
  * Think twice, and think again, before using these.
- * Note: Should only be used between intel_uncore_forcewake_irqlock() and
- * intel_uncore_forcewake_irqunlock().
+ *
+ * As an example, these accessors can possibly be used between:
+ *
+ * spin_lock_irq(&dev_priv->uncore.lock);
+ * intel_uncore_forcewake_get__locked();
+ *
+ * and
+ *
+ * intel_uncore_forcewake_put__locked();
+ * spin_unlock_irq(&dev_priv->uncore.lock);
+ *
+ *
+ * Note: some registers may not need forcewake held, so
+ * intel_uncore_forcewake_{get,put} can be omitted, see
+ * intel_uncore_forcewake_for_reg().
+ *
+ * Certain architectures will die if the same cacheline is concurrently accessed
+ * by different clients (e.g. on Ivybridge). Access to registers should
+ * therefore generally be serialised, by either the dev_priv->uncore.lock or
+ * a more localised lock guarding all access to that bank of registers.
  */
 #define I915_READ_FW(reg__) __raw_i915_read32(dev_priv, (reg__))
 #define I915_WRITE_FW(reg__, val__) __raw_i915_write32(dev_priv, (reg__), (val__))
@@ -3812,11 +3673,11 @@ __raw_write(64, q)
 #define INTEL_BROADCAST_RGB_FULL 1
 #define INTEL_BROADCAST_RGB_LIMITED 2
 
-static inline i915_reg_t i915_vgacntrl_reg(struct drm_device *dev)
+static inline i915_reg_t i915_vgacntrl_reg(struct drm_i915_private *dev_priv)
 {
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		return VLV_VGACNTRL;
-	else if (INTEL_INFO(dev)->gen >= 5)
+	else if (INTEL_GEN(dev_priv) >= 5)
 		return CPU_VGACNTRL;
 	else
 		return VGACNTRL;
@@ -3877,7 +3738,7 @@ __i915_request_irq_complete(struct drm_i915_gem_request *req)
 	/* Before we do the heavier coherent read of the seqno,
 	 * check the value (hopefully) in the CPU cacheline.
 	 */
-	if (i915_gem_request_completed(req))
+	if (__i915_gem_request_completed(req))
 		return true;
 
 	/* Ensure our read of the seqno is coherent so that we
@@ -3928,7 +3789,7 @@ __i915_request_irq_complete(struct drm_i915_gem_request *req)
 			wake_up_process(tsk);
 		rcu_read_unlock();
 
-		if (i915_gem_request_completed(req))
+		if (__i915_gem_request_completed(req))
 			return true;
 	}
 
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 00eb4814b913..412f3513f269 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -29,12 +29,12 @@
 #include <drm/drm_vma_manager.h>
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
-#include "i915_gem_dmabuf.h"
 #include "i915_vgpu.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
 #include "intel_frontbuffer.h"
 #include "intel_mocs.h"
+#include <linux/dma-fence-array.h>
 #include <linux/reservation.h>
 #include <linux/shmem_fs.h>
 #include <linux/slab.h>
@@ -42,13 +42,14 @@
 #include <linux/pci.h>
 #include <linux/dma-buf.h>
 
+static void i915_gem_flush_free_objects(struct drm_i915_private *i915);
 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
 
 static bool cpu_cache_is_coherent(struct drm_device *dev,
 				  enum i915_cache_level level)
 {
-	return HAS_LLC(dev) || level != I915_CACHE_NONE;
+	return HAS_LLC(to_i915(dev)) || level != I915_CACHE_NONE;
 }
 
 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
@@ -63,13 +64,13 @@ static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
 }
 
 static int
-insert_mappable_node(struct drm_i915_private *i915,
+insert_mappable_node(struct i915_ggtt *ggtt,
                      struct drm_mm_node *node, u32 size)
 {
 	memset(node, 0, sizeof(*node));
-	return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node,
-						   size, 0, 0, 0,
-						   i915->ggtt.mappable_end,
+	return drm_mm_insert_node_in_range_generic(&ggtt->base.mm, node,
+						   size, 0, -1,
+						   0, ggtt->mappable_end,
 						   DRM_MM_SEARCH_DEFAULT,
 						   DRM_MM_CREATE_DEFAULT);
 }
@@ -82,7 +83,7 @@ remove_mappable_node(struct drm_mm_node *node)
 
 /* some bookkeeping */
 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
-				  size_t size)
+				  u64 size)
 {
 	spin_lock(&dev_priv->mm.object_stat_lock);
 	dev_priv->mm.object_count++;
@@ -91,7 +92,7 @@ static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
 }
 
 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
-				     size_t size)
+				     u64 size)
 {
 	spin_lock(&dev_priv->mm.object_stat_lock);
 	dev_priv->mm.object_count--;
@@ -104,6 +105,8 @@ i915_gem_wait_for_error(struct i915_gpu_error *error)
 {
 	int ret;
 
+	might_sleep();
+
 	if (!i915_reset_in_progress(error))
 		return 0;
 
@@ -114,7 +117,7 @@ i915_gem_wait_for_error(struct i915_gpu_error *error)
 	 */
 	ret = wait_event_interruptible_timeout(error->reset_queue,
 					       !i915_reset_in_progress(error),
-					       10*HZ);
+					       I915_RESET_TIMEOUT);
 	if (ret == 0) {
 		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 		return -EIO;
@@ -167,7 +170,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 	return 0;
 }
 
-static int
+static struct sg_table *
 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 {
 	struct address_space *mapping = obj->base.filp->f_mapping;
@@ -177,7 +180,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 	int i;
 
 	if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
-		return -EINVAL;
+		return ERR_PTR(-EINVAL);
 
 	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 		struct page *page;
@@ -185,7 +188,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 
 		page = shmem_read_mapping_page(mapping, i);
 		if (IS_ERR(page))
-			return PTR_ERR(page);
+			return ERR_CAST(page);
 
 		src = kmap_atomic(page);
 		memcpy(vaddr, src, PAGE_SIZE);
@@ -200,11 +203,11 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 
 	st = kmalloc(sizeof(*st), GFP_KERNEL);
 	if (st == NULL)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 		kfree(st);
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 	}
 
 	sg = st->sgl;
@@ -214,29 +217,33 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 	sg_dma_address(sg) = obj->phys_handle->busaddr;
 	sg_dma_len(sg) = obj->base.size;
 
-	obj->pages = st;
-	return 0;
+	return st;
 }
 
 static void
-i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
+__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
+				struct sg_table *pages)
 {
-	int ret;
+	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
 
-	BUG_ON(obj->madv == __I915_MADV_PURGED);
+	if (obj->mm.madv == I915_MADV_DONTNEED)
+		obj->mm.dirty = false;
 
-	ret = i915_gem_object_set_to_cpu_domain(obj, true);
-	if (WARN_ON(ret)) {
-		/* In the event of a disaster, abandon all caches and
-		 * hope for the best.
-		 */
-		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
-	}
+	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
+	    !cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
+		drm_clflush_sg(pages);
+
+	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+}
 
-	if (obj->madv == I915_MADV_DONTNEED)
-		obj->dirty = 0;
+static void
+i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
+			       struct sg_table *pages)
+{
+	__i915_gem_object_release_shmem(obj, pages);
 
-	if (obj->dirty) {
+	if (obj->mm.dirty) {
 		struct address_space *mapping = obj->base.filp->f_mapping;
 		char *vaddr = obj->phys_handle->vaddr;
 		int i;
@@ -255,22 +262,23 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 			kunmap_atomic(dst);
 
 			set_page_dirty(page);
-			if (obj->madv == I915_MADV_WILLNEED)
+			if (obj->mm.madv == I915_MADV_WILLNEED)
 				mark_page_accessed(page);
 			put_page(page);
 			vaddr += PAGE_SIZE;
 		}
-		obj->dirty = 0;
+		obj->mm.dirty = false;
 	}
 
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(pages);
+	kfree(pages);
 }
 
 static void
 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
 {
 	drm_pci_free(obj->base.dev, obj->phys_handle);
+	i915_gem_object_unpin_pages(obj);
 }
 
 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
@@ -292,7 +300,12 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 	 * must wait for all rendering to complete to the object (as unbinding
 	 * must anyway), and retire the requests.
 	 */
-	ret = i915_gem_object_wait_rendering(obj, false);
+	ret = i915_gem_object_wait(obj,
+				   I915_WAIT_INTERRUPTIBLE |
+				   I915_WAIT_LOCKED |
+				   I915_WAIT_ALL,
+				   MAX_SCHEDULE_TIMEOUT,
+				   NULL);
 	if (ret)
 		return ret;
 
@@ -311,90 +324,209 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 	return ret;
 }
 
-/**
- * Ensures that all rendering to the object has completed and the object is
- * safe to unbind from the GTT or access from the CPU.
- * @obj: i915 gem object
- * @readonly: waiting for just read access or read-write access
- */
-int
-i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
-			       bool readonly)
+static long
+i915_gem_object_wait_fence(struct dma_fence *fence,
+			   unsigned int flags,
+			   long timeout,
+			   struct intel_rps_client *rps)
 {
-	struct reservation_object *resv;
-	struct i915_gem_active *active;
-	unsigned long active_mask;
-	int idx;
+	struct drm_i915_gem_request *rq;
 
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
 
-	if (!readonly) {
-		active = obj->last_read;
-		active_mask = i915_gem_object_get_active(obj);
-	} else {
-		active_mask = 1;
-		active = &obj->last_write;
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+		return timeout;
+
+	if (!dma_fence_is_i915(fence))
+		return dma_fence_wait_timeout(fence,
+					      flags & I915_WAIT_INTERRUPTIBLE,
+					      timeout);
+
+	rq = to_request(fence);
+	if (i915_gem_request_completed(rq))
+		goto out;
+
+	/* This client is about to stall waiting for the GPU. In many cases
+	 * this is undesirable and limits the throughput of the system, as
+	 * many clients cannot continue processing user input/output whilst
+	 * blocked. RPS autotuning may take tens of milliseconds to respond
+	 * to the GPU load and thus incurs additional latency for the client.
+	 * We can circumvent that by promoting the GPU frequency to maximum
+	 * before we wait. This makes the GPU throttle up much more quickly
+	 * (good for benchmarks and user experience, e.g. window animations),
+	 * but at a cost of spending more power processing the workload
+	 * (bad for battery). Not all clients even want their results
+	 * immediately and for them we should just let the GPU select its own
+	 * frequency to maximise efficiency. To prevent a single client from
+	 * forcing the clocks too high for the whole system, we only allow
+	 * each client to waitboost once in a busy period.
+	 */
+	if (rps) {
+		if (INTEL_GEN(rq->i915) >= 6)
+			gen6_rps_boost(rq->i915, rps, rq->emitted_jiffies);
+		else
+			rps = NULL;
 	}
 
-	for_each_active(active_mask, idx) {
+	timeout = i915_wait_request(rq, flags, timeout);
+
+out:
+	if (flags & I915_WAIT_LOCKED && i915_gem_request_completed(rq))
+		i915_gem_request_retire_upto(rq);
+
+	if (rps && rq->global_seqno == intel_engine_last_submit(rq->engine)) {
+		/* The GPU is now idle and this client has stalled.
+		 * Since no other client has submitted a request in the
+		 * meantime, assume that this client is the only one
+		 * supplying work to the GPU but is unable to keep that
+		 * work supplied because it is waiting. Since the GPU is
+		 * then never kept fully busy, RPS autoclocking will
+		 * keep the clocks relatively low, causing further delays.
+		 * Compensate by giving the synchronous client credit for
+		 * a waitboost next time.
+		 */
+		spin_lock(&rq->i915->rps.client_lock);
+		list_del_init(&rps->link);
+		spin_unlock(&rq->i915->rps.client_lock);
+	}
+
+	return timeout;
+}
+
+static long
+i915_gem_object_wait_reservation(struct reservation_object *resv,
+				 unsigned int flags,
+				 long timeout,
+				 struct intel_rps_client *rps)
+{
+	struct dma_fence *excl;
+
+	if (flags & I915_WAIT_ALL) {
+		struct dma_fence **shared;
+		unsigned int count, i;
 		int ret;
 
-		ret = i915_gem_active_wait(&active[idx],
-					   &obj->base.dev->struct_mutex);
+		ret = reservation_object_get_fences_rcu(resv,
+							&excl, &count, &shared);
 		if (ret)
 			return ret;
-	}
 
-	resv = i915_gem_object_get_dmabuf_resv(obj);
-	if (resv) {
-		long err;
+		for (i = 0; i < count; i++) {
+			timeout = i915_gem_object_wait_fence(shared[i],
+							     flags, timeout,
+							     rps);
+			if (timeout <= 0)
+				break;
 
-		err = reservation_object_wait_timeout_rcu(resv, !readonly, true,
-							  MAX_SCHEDULE_TIMEOUT);
-		if (err < 0)
-			return err;
+			dma_fence_put(shared[i]);
+		}
+
+		for (; i < count; i++)
+			dma_fence_put(shared[i]);
+		kfree(shared);
+	} else {
+		excl = reservation_object_get_excl_rcu(resv);
 	}
 
-	return 0;
+	if (excl && timeout > 0)
+		timeout = i915_gem_object_wait_fence(excl, flags, timeout, rps);
+
+	dma_fence_put(excl);
+
+	return timeout;
 }
 
-/* A nonblocking variant of the above wait. Must be called prior to
- * acquiring the mutex for the object, as the object state may change
- * during this call. A reference must be held by the caller for the object.
- */
-static __must_check int
-__unsafe_wait_rendering(struct drm_i915_gem_object *obj,
-			struct intel_rps_client *rps,
-			bool readonly)
+static void __fence_set_priority(struct dma_fence *fence, int prio)
 {
-	struct i915_gem_active *active;
-	unsigned long active_mask;
-	int idx;
+	struct drm_i915_gem_request *rq;
+	struct intel_engine_cs *engine;
 
-	active_mask = __I915_BO_ACTIVE(obj);
-	if (!active_mask)
-		return 0;
+	if (!dma_fence_is_i915(fence))
+		return;
+
+	rq = to_request(fence);
+	engine = rq->engine;
+	if (!engine->schedule)
+		return;
+
+	engine->schedule(rq, prio);
+}
+
+static void fence_set_priority(struct dma_fence *fence, int prio)
+{
+	/* Recurse once into a fence-array */
+	if (dma_fence_is_array(fence)) {
+		struct dma_fence_array *array = to_dma_fence_array(fence);
+		int i;
 
-	if (!readonly) {
-		active = obj->last_read;
+		for (i = 0; i < array->num_fences; i++)
+			__fence_set_priority(array->fences[i], prio);
 	} else {
-		active_mask = 1;
-		active = &obj->last_write;
+		__fence_set_priority(fence, prio);
 	}
+}
+
+int
+i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
+			      unsigned int flags,
+			      int prio)
+{
+	struct dma_fence *excl;
 
-	for_each_active(active_mask, idx) {
+	if (flags & I915_WAIT_ALL) {
+		struct dma_fence **shared;
+		unsigned int count, i;
 		int ret;
 
-		ret = i915_gem_active_wait_unlocked(&active[idx],
-						    I915_WAIT_INTERRUPTIBLE,
-						    NULL, rps);
+		ret = reservation_object_get_fences_rcu(obj->resv,
+							&excl, &count, &shared);
 		if (ret)
 			return ret;
+
+		for (i = 0; i < count; i++) {
+			fence_set_priority(shared[i], prio);
+			dma_fence_put(shared[i]);
+		}
+
+		kfree(shared);
+	} else {
+		excl = reservation_object_get_excl_rcu(obj->resv);
 	}
 
+	if (excl) {
+		fence_set_priority(excl, prio);
+		dma_fence_put(excl);
+	}
 	return 0;
 }
 
+/**
+ * Waits for rendering to the object to be completed
+ * @obj: i915 gem object
+ * @flags: how to wait (under a lock, for all rendering or just for writes etc)
+ * @timeout: how long to wait
+ * @rps: client (user process) to charge for any waitboosting
+ */
+int
+i915_gem_object_wait(struct drm_i915_gem_object *obj,
+		     unsigned int flags,
+		     long timeout,
+		     struct intel_rps_client *rps)
+{
+	might_sleep();
+#if IS_ENABLED(CONFIG_LOCKDEP)
+	GEM_BUG_ON(debug_locks &&
+		   !!lockdep_is_held(&obj->base.dev->struct_mutex) !=
+		   !!(flags & I915_WAIT_LOCKED));
+#endif
+	GEM_BUG_ON(timeout < 0);
+
+	timeout = i915_gem_object_wait_reservation(obj->resv,
+						   flags, timeout,
+						   rps);
+	return timeout < 0 ? timeout : 0;
+}
+
 static struct intel_rps_client *to_rps_client(struct drm_file *file)
 {
 	struct drm_i915_file_private *fpriv = file->driver_priv;
@@ -416,7 +548,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 		return 0;
 	}
 
-	if (obj->madv != I915_MADV_WILLNEED)
+	if (obj->mm.madv != I915_MADV_WILLNEED)
 		return -EFAULT;
 
 	if (obj->base.filp == NULL)
@@ -426,9 +558,9 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 	if (ret)
 		return ret;
 
-	ret = i915_gem_object_put_pages(obj);
-	if (ret)
-		return ret;
+	__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
+	if (obj->mm.pages)
+		return -EBUSY;
 
 	/* create a new object */
 	phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
@@ -438,23 +570,29 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 	obj->phys_handle = phys;
 	obj->ops = &i915_gem_phys_ops;
 
-	return i915_gem_object_get_pages(obj);
+	return i915_gem_object_pin_pages(obj);
 }
 
 static int
 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 		     struct drm_i915_gem_pwrite *args,
-		     struct drm_file *file_priv)
+		     struct drm_file *file)
 {
 	struct drm_device *dev = obj->base.dev;
 	void *vaddr = obj->phys_handle->vaddr + args->offset;
 	char __user *user_data = u64_to_user_ptr(args->data_ptr);
-	int ret = 0;
+	int ret;
 
 	/* We manually control the domain here and pretend that it
 	 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 	 */
-	ret = i915_gem_object_wait_rendering(obj, false);
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	ret = i915_gem_object_wait(obj,
+				   I915_WAIT_INTERRUPTIBLE |
+				   I915_WAIT_LOCKED |
+				   I915_WAIT_ALL,
+				   MAX_SCHEDULE_TIMEOUT,
+				   to_rps_client(file));
 	if (ret)
 		return ret;
 
@@ -516,7 +654,7 @@ i915_gem_create(struct drm_file *file,
 
 	ret = drm_gem_handle_create(file, &obj->base, &handle);
 	/* drop reference from allocate - handle holds it now */
-	i915_gem_object_put_unlocked(obj);
+	i915_gem_object_put(obj);
 	if (ret)
 		return ret;
 
@@ -548,6 +686,8 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_i915_gem_create *args = data;
 
+	i915_gem_flush_free_objects(to_i915(dev));
+
 	return i915_gem_create(file, dev,
 			       args->size, &args->handle);
 }
@@ -614,21 +754,24 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 {
 	int ret;
 
-	*needs_clflush = 0;
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
+	*needs_clflush = 0;
 	if (!i915_gem_object_has_struct_page(obj))
 		return -ENODEV;
 
-	ret = i915_gem_object_wait_rendering(obj, true);
+	ret = i915_gem_object_wait(obj,
+				   I915_WAIT_INTERRUPTIBLE |
+				   I915_WAIT_LOCKED,
+				   MAX_SCHEDULE_TIMEOUT,
+				   NULL);
 	if (ret)
 		return ret;
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
 		return ret;
 
-	i915_gem_object_pin_pages(obj);
-
 	i915_gem_object_flush_gtt_write_domain(obj);
 
 	/* If we're not in the cpu read domain, set ourself into the gtt
@@ -661,20 +804,25 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
 {
 	int ret;
 
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
 	*needs_clflush = 0;
 	if (!i915_gem_object_has_struct_page(obj))
 		return -ENODEV;
 
-	ret = i915_gem_object_wait_rendering(obj, false);
+	ret = i915_gem_object_wait(obj,
+				   I915_WAIT_INTERRUPTIBLE |
+				   I915_WAIT_LOCKED |
+				   I915_WAIT_ALL,
+				   MAX_SCHEDULE_TIMEOUT,
+				   NULL);
 	if (ret)
 		return ret;
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
 		return ret;
 
-	i915_gem_object_pin_pages(obj);
-
 	i915_gem_object_flush_gtt_write_domain(obj);
 
 	/* If we're not in the cpu write domain, set ourself into the
@@ -704,7 +852,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
 		obj->cache_dirty = true;
 
 	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
-	obj->dirty = 1;
+	obj->mm.dirty = true;
 	/* return with the pages pinned */
 	return 0;
 
@@ -713,32 +861,6 @@ err_unpin:
 	return ret;
 }
 
-/* Per-page copy function for the shmem pread fastpath.
- * Flushes invalid cachelines before reading the target if
- * needs_clflush is set. */
-static int
-shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
-		 char __user *user_data,
-		 bool page_do_bit17_swizzling, bool needs_clflush)
-{
-	char *vaddr;
-	int ret;
-
-	if (unlikely(page_do_bit17_swizzling))
-		return -EINVAL;
-
-	vaddr = kmap_atomic(page);
-	if (needs_clflush)
-		drm_clflush_virt_range(vaddr + shmem_page_offset,
-				       page_length);
-	ret = __copy_to_user_inatomic(user_data,
-				      vaddr + shmem_page_offset,
-				      page_length);
-	kunmap_atomic(vaddr);
-
-	return ret ? -EFAULT : 0;
-}
-
 static void
 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 			     bool swizzled)
@@ -764,7 +886,7 @@ shmem_clflush_swizzled_range(char *addr, unsigned long length,
 /* Only difference to the fast-path function is that this can handle bit17
  * and uses non-atomic copy and kmap functions. */
 static int
-shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
+shmem_pread_slow(struct page *page, int offset, int length,
 		 char __user *user_data,
 		 bool page_do_bit17_swizzling, bool needs_clflush)
 {
@@ -773,60 +895,130 @@ shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 
 	vaddr = kmap(page);
 	if (needs_clflush)
-		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
-					     page_length,
+		shmem_clflush_swizzled_range(vaddr + offset, length,
 					     page_do_bit17_swizzling);
 
 	if (page_do_bit17_swizzling)
-		ret = __copy_to_user_swizzled(user_data,
-					      vaddr, shmem_page_offset,
-					      page_length);
+		ret = __copy_to_user_swizzled(user_data, vaddr, offset, length);
 	else
-		ret = __copy_to_user(user_data,
-				     vaddr + shmem_page_offset,
-				     page_length);
+		ret = __copy_to_user(user_data, vaddr + offset, length);
 	kunmap(page);
 
 	return ret ? - EFAULT : 0;
 }
 
-static inline unsigned long
-slow_user_access(struct io_mapping *mapping,
-		 uint64_t page_base, int page_offset,
-		 char __user *user_data,
-		 unsigned long length, bool pwrite)
+static int
+shmem_pread(struct page *page, int offset, int length, char __user *user_data,
+	    bool page_do_bit17_swizzling, bool needs_clflush)
+{
+	int ret;
+
+	ret = -ENODEV;
+	if (!page_do_bit17_swizzling) {
+		char *vaddr = kmap_atomic(page);
+
+		if (needs_clflush)
+			drm_clflush_virt_range(vaddr + offset, length);
+		ret = __copy_to_user_inatomic(user_data, vaddr + offset, length);
+		kunmap_atomic(vaddr);
+	}
+	if (ret == 0)
+		return 0;
+
+	return shmem_pread_slow(page, offset, length, user_data,
+				page_do_bit17_swizzling, needs_clflush);
+}
+
+static int
+i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
+		     struct drm_i915_gem_pread *args)
+{
+	char __user *user_data;
+	u64 remain;
+	unsigned int obj_do_bit17_swizzling;
+	unsigned int needs_clflush;
+	unsigned int idx, offset;
+	int ret;
+
+	obj_do_bit17_swizzling = 0;
+	if (i915_gem_object_needs_bit17_swizzle(obj))
+		obj_do_bit17_swizzling = BIT(17);
+
+	ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
+	mutex_unlock(&obj->base.dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	remain = args->size;
+	user_data = u64_to_user_ptr(args->data_ptr);
+	offset = offset_in_page(args->offset);
+	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
+		struct page *page = i915_gem_object_get_page(obj, idx);
+		int length;
+
+		length = remain;
+		if (offset + length > PAGE_SIZE)
+			length = PAGE_SIZE - offset;
+
+		ret = shmem_pread(page, offset, length, user_data,
+				  page_to_phys(page) & obj_do_bit17_swizzling,
+				  needs_clflush);
+		if (ret)
+			break;
+
+		remain -= length;
+		user_data += length;
+		offset = 0;
+	}
+
+	i915_gem_obj_finish_shmem_access(obj);
+	return ret;
+}
+
+static inline bool
+gtt_user_read(struct io_mapping *mapping,
+	      loff_t base, int offset,
+	      char __user *user_data, int length)
 {
-	void __iomem *ioaddr;
 	void *vaddr;
-	uint64_t unwritten;
+	unsigned long unwritten;
 
-	ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
 	/* We can use the cpu mem copy function because this is X86. */
-	vaddr = (void __force *)ioaddr + page_offset;
-	if (pwrite)
-		unwritten = __copy_from_user(vaddr, user_data, length);
-	else
-		unwritten = __copy_to_user(user_data, vaddr, length);
-
-	io_mapping_unmap(ioaddr);
+	vaddr = (void __force *)io_mapping_map_atomic_wc(mapping, base);
+	unwritten = __copy_to_user_inatomic(user_data, vaddr + offset, length);
+	io_mapping_unmap_atomic(vaddr);
+	if (unwritten) {
+		vaddr = (void __force *)
+			io_mapping_map_wc(mapping, base, PAGE_SIZE);
+		unwritten = copy_to_user(user_data, vaddr + offset, length);
+		io_mapping_unmap(vaddr);
+	}
 	return unwritten;
 }
 
 static int
-i915_gem_gtt_pread(struct drm_device *dev,
-		   struct drm_i915_gem_object *obj, uint64_t size,
-		   uint64_t data_offset, uint64_t data_ptr)
+i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
+		   const struct drm_i915_gem_pread *args)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	struct i915_vma *vma;
+	struct drm_i915_private *i915 = to_i915(obj->base.dev);
+	struct i915_ggtt *ggtt = &i915->ggtt;
 	struct drm_mm_node node;
-	char __user *user_data;
-	uint64_t remain;
-	uint64_t offset;
+	struct i915_vma *vma;
+	void __user *user_data;
+	u64 remain, offset;
 	int ret;
 
-	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
+	ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
+	if (ret)
+		return ret;
+
+	intel_runtime_pm_get(i915);
+	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
+				       PIN_MAPPABLE | PIN_NONBLOCK);
 	if (!IS_ERR(vma)) {
 		node.start = i915_ggtt_offset(vma);
 		node.allocated = false;
@@ -837,35 +1029,21 @@ i915_gem_gtt_pread(struct drm_device *dev,
 		}
 	}
 	if (IS_ERR(vma)) {
-		ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE);
+		ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
 		if (ret)
-			goto out;
-
-		ret = i915_gem_object_get_pages(obj);
-		if (ret) {
-			remove_mappable_node(&node);
-			goto out;
-		}
-
-		i915_gem_object_pin_pages(obj);
+			goto out_unlock;
+		GEM_BUG_ON(!node.allocated);
 	}
 
 	ret = i915_gem_object_set_to_gtt_domain(obj, false);
 	if (ret)
 		goto out_unpin;
 
-	user_data = u64_to_user_ptr(data_ptr);
-	remain = size;
-	offset = data_offset;
+	mutex_unlock(&i915->drm.struct_mutex);
 
-	mutex_unlock(&dev->struct_mutex);
-	if (likely(!i915.prefault_disable)) {
-		ret = fault_in_pages_writeable(user_data, remain);
-		if (ret) {
-			mutex_lock(&dev->struct_mutex);
-			goto out_unpin;
-		}
-	}
+	user_data = u64_to_user_ptr(args->data_ptr);
+	remain = args->size;
+	offset = args->offset;
 
 	while (remain > 0) {
 		/* Operation in this page
@@ -882,19 +1060,14 @@ i915_gem_gtt_pread(struct drm_device *dev,
 			wmb();
 			ggtt->base.insert_page(&ggtt->base,
 					       i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
-					       node.start,
-					       I915_CACHE_NONE, 0);
+					       node.start, I915_CACHE_NONE, 0);
 			wmb();
 		} else {
 			page_base += offset & PAGE_MASK;
 		}
-		/* This is a slow read/write as it tries to read from
-		 * and write to user memory which may result into page
-		 * faults, and so we cannot perform this under struct_mutex.
-		 */
-		if (slow_user_access(&ggtt->mappable, page_base,
-				     page_offset, user_data,
-				     page_length, false)) {
+
+		if (gtt_user_read(&ggtt->mappable, page_base, page_offset,
+				  user_data, page_length)) {
 			ret = -EFAULT;
 			break;
 		}
@@ -904,111 +1077,19 @@ i915_gem_gtt_pread(struct drm_device *dev,
 		offset += page_length;
 	}
 
-	mutex_lock(&dev->struct_mutex);
-	if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
-		/* The user has modified the object whilst we tried
-		 * reading from it, and we now have no idea what domain
-		 * the pages should be in. As we have just been touching
-		 * them directly, flush everything back to the GTT
-		 * domain.
-		 */
-		ret = i915_gem_object_set_to_gtt_domain(obj, false);
-	}
-
+	mutex_lock(&i915->drm.struct_mutex);
 out_unpin:
 	if (node.allocated) {
 		wmb();
 		ggtt->base.clear_range(&ggtt->base,
-				       node.start, node.size,
-				       true);
-		i915_gem_object_unpin_pages(obj);
+				       node.start, node.size);
 		remove_mappable_node(&node);
 	} else {
 		i915_vma_unpin(vma);
 	}
-out:
-	return ret;
-}
-
-static int
-i915_gem_shmem_pread(struct drm_device *dev,
-		     struct drm_i915_gem_object *obj,
-		     struct drm_i915_gem_pread *args,
-		     struct drm_file *file)
-{
-	char __user *user_data;
-	ssize_t remain;
-	loff_t offset;
-	int shmem_page_offset, page_length, ret = 0;
-	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
-	int prefaulted = 0;
-	int needs_clflush = 0;
-	struct sg_page_iter sg_iter;
-
-	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
-	if (ret)
-		return ret;
-
-	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
-	user_data = u64_to_user_ptr(args->data_ptr);
-	offset = args->offset;
-	remain = args->size;
-
-	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
-			 offset >> PAGE_SHIFT) {
-		struct page *page = sg_page_iter_page(&sg_iter);
-
-		if (remain <= 0)
-			break;
-
-		/* Operation in this page
-		 *
-		 * shmem_page_offset = offset within page in shmem file
-		 * page_length = bytes to copy for this page
-		 */
-		shmem_page_offset = offset_in_page(offset);
-		page_length = remain;
-		if ((shmem_page_offset + page_length) > PAGE_SIZE)
-			page_length = PAGE_SIZE - shmem_page_offset;
-
-		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
-			(page_to_phys(page) & (1 << 17)) != 0;
-
-		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
-				       user_data, page_do_bit17_swizzling,
-				       needs_clflush);
-		if (ret == 0)
-			goto next_page;
-
-		mutex_unlock(&dev->struct_mutex);
-
-		if (likely(!i915.prefault_disable) && !prefaulted) {
-			ret = fault_in_pages_writeable(user_data, remain);
-			/* Userspace is tricking us, but we've already clobbered
-			 * its pages with the prefault and promised to write the
-			 * data up to the first fault. Hence ignore any errors
-			 * and just continue. */
-			(void)ret;
-			prefaulted = 1;
-		}
-
-		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
-				       user_data, page_do_bit17_swizzling,
-				       needs_clflush);
-
-		mutex_lock(&dev->struct_mutex);
-
-		if (ret)
-			goto out;
-
-next_page:
-		remain -= page_length;
-		user_data += page_length;
-		offset += page_length;
-	}
-
-out:
-	i915_gem_obj_finish_shmem_access(obj);
+out_unlock:
+	intel_runtime_pm_put(i915);
+	mutex_unlock(&i915->drm.struct_mutex);
 
 	return ret;
 }
@@ -1027,7 +1108,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_i915_gem_pread *args = data;
 	struct drm_i915_gem_object *obj;
-	int ret = 0;
+	int ret;
 
 	if (args->size == 0)
 		return 0;
@@ -1045,36 +1126,29 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 	if (args->offset > obj->base.size ||
 	    args->size > obj->base.size - args->offset) {
 		ret = -EINVAL;
-		goto err;
+		goto out;
 	}
 
 	trace_i915_gem_object_pread(obj, args->offset, args->size);
 
-	ret = __unsafe_wait_rendering(obj, to_rps_client(file), true);
+	ret = i915_gem_object_wait(obj,
+				   I915_WAIT_INTERRUPTIBLE,
+				   MAX_SCHEDULE_TIMEOUT,
+				   to_rps_client(file));
 	if (ret)
-		goto err;
+		goto out;
 
-	ret = i915_mutex_lock_interruptible(dev);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
-		goto err;
-
-	ret = i915_gem_shmem_pread(dev, obj, args, file);
+		goto out;
 
-	/* pread for non shmem backed objects */
-	if (ret == -EFAULT || ret == -ENODEV) {
-		intel_runtime_pm_get(to_i915(dev));
-		ret = i915_gem_gtt_pread(dev, obj, args->size,
-					args->offset, args->data_ptr);
-		intel_runtime_pm_put(to_i915(dev));
-	}
+	ret = i915_gem_shmem_pread(obj, args);
+	if (ret == -EFAULT || ret == -ENODEV)
+		ret = i915_gem_gtt_pread(obj, args);
 
+	i915_gem_object_unpin_pages(obj);
+out:
 	i915_gem_object_put(obj);
-	mutex_unlock(&dev->struct_mutex);
-
-	return ret;
-
-err:
-	i915_gem_object_put_unlocked(obj);
 	return ret;
 }
 
@@ -1082,51 +1156,52 @@ err:
  * page faults in the source data
  */
 
-static inline int
-fast_user_write(struct io_mapping *mapping,
-		loff_t page_base, int page_offset,
-		char __user *user_data,
-		int length)
+static inline bool
+ggtt_write(struct io_mapping *mapping,
+	   loff_t base, int offset,
+	   char __user *user_data, int length)
 {
-	void __iomem *vaddr_atomic;
 	void *vaddr;
 	unsigned long unwritten;
 
-	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 	/* We can use the cpu mem copy function because this is X86. */
-	vaddr = (void __force*)vaddr_atomic + page_offset;
-	unwritten = __copy_from_user_inatomic_nocache(vaddr,
+	vaddr = (void __force *)io_mapping_map_atomic_wc(mapping, base);
+	unwritten = __copy_from_user_inatomic_nocache(vaddr + offset,
 						      user_data, length);
-	io_mapping_unmap_atomic(vaddr_atomic);
+	io_mapping_unmap_atomic(vaddr);
+	if (unwritten) {
+		vaddr = (void __force *)
+			io_mapping_map_wc(mapping, base, PAGE_SIZE);
+		unwritten = copy_from_user(vaddr + offset, user_data, length);
+		io_mapping_unmap(vaddr);
+	}
+
 	return unwritten;
 }
 
 /**
  * This is the fast pwrite path, where we copy the data directly from the
  * user into the GTT, uncached.
- * @i915: i915 device private data
- * @obj: i915 gem object
+ * @obj: i915 GEM object
  * @args: pwrite arguments structure
- * @file: drm file pointer
  */
 static int
-i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
-			 struct drm_i915_gem_object *obj,
-			 struct drm_i915_gem_pwrite *args,
-			 struct drm_file *file)
+i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
+			 const struct drm_i915_gem_pwrite *args)
 {
+	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 	struct i915_ggtt *ggtt = &i915->ggtt;
-	struct drm_device *dev = obj->base.dev;
-	struct i915_vma *vma;
 	struct drm_mm_node node;
-	uint64_t remain, offset;
-	char __user *user_data;
+	struct i915_vma *vma;
+	u64 remain, offset;
+	void __user *user_data;
 	int ret;
-	bool hit_slow_path = false;
 
-	if (i915_gem_object_is_tiled(obj))
-		return -EFAULT;
+	ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
+	if (ret)
+		return ret;
 
+	intel_runtime_pm_get(i915);
 	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
 				       PIN_MAPPABLE | PIN_NONBLOCK);
 	if (!IS_ERR(vma)) {
@@ -1139,25 +1214,19 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 		}
 	}
 	if (IS_ERR(vma)) {
-		ret = insert_mappable_node(i915, &node, PAGE_SIZE);
+		ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
 		if (ret)
-			goto out;
-
-		ret = i915_gem_object_get_pages(obj);
-		if (ret) {
-			remove_mappable_node(&node);
-			goto out;
-		}
-
-		i915_gem_object_pin_pages(obj);
+			goto out_unlock;
+		GEM_BUG_ON(!node.allocated);
 	}
 
 	ret = i915_gem_object_set_to_gtt_domain(obj, true);
 	if (ret)
 		goto out_unpin;
 
+	mutex_unlock(&i915->drm.struct_mutex);
+
 	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
-	obj->dirty = true;
 
 	user_data = u64_to_user_ptr(args->data_ptr);
 	offset = args->offset;
@@ -1170,8 +1239,8 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 		 * page_length = bytes to copy for this page
 		 */
 		u32 page_base = node.start;
-		unsigned page_offset = offset_in_page(offset);
-		unsigned page_length = PAGE_SIZE - page_offset;
+		unsigned int page_offset = offset_in_page(offset);
+		unsigned int page_length = PAGE_SIZE - page_offset;
 		page_length = remain < page_length ? remain : page_length;
 		if (node.allocated) {
 			wmb(); /* flush the write before we modify the GGTT */
@@ -1188,92 +1257,36 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 		 * If the object is non-shmem backed, we retry again with the
 		 * path that handles page fault.
 		 */
-		if (fast_user_write(&ggtt->mappable, page_base,
-				    page_offset, user_data, page_length)) {
-			hit_slow_path = true;
-			mutex_unlock(&dev->struct_mutex);
-			if (slow_user_access(&ggtt->mappable,
-					     page_base,
-					     page_offset, user_data,
-					     page_length, true)) {
-				ret = -EFAULT;
-				mutex_lock(&dev->struct_mutex);
-				goto out_flush;
-			}
-
-			mutex_lock(&dev->struct_mutex);
+		if (ggtt_write(&ggtt->mappable, page_base, page_offset,
+			       user_data, page_length)) {
+			ret = -EFAULT;
+			break;
 		}
 
 		remain -= page_length;
 		user_data += page_length;
 		offset += page_length;
 	}
-
-out_flush:
-	if (hit_slow_path) {
-		if (ret == 0 &&
-		    (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
-			/* The user has modified the object whilst we tried
-			 * reading from it, and we now have no idea what domain
-			 * the pages should be in. As we have just been touching
-			 * them directly, flush everything back to the GTT
-			 * domain.
-			 */
-			ret = i915_gem_object_set_to_gtt_domain(obj, false);
-		}
-	}
-
 	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
+
+	mutex_lock(&i915->drm.struct_mutex);
 out_unpin:
 	if (node.allocated) {
 		wmb();
 		ggtt->base.clear_range(&ggtt->base,
-				       node.start, node.size,
-				       true);
-		i915_gem_object_unpin_pages(obj);
+				       node.start, node.size);
 		remove_mappable_node(&node);
 	} else {
 		i915_vma_unpin(vma);
 	}
-out:
+out_unlock:
+	intel_runtime_pm_put(i915);
+	mutex_unlock(&i915->drm.struct_mutex);
 	return ret;
 }
 
-/* Per-page copy function for the shmem pwrite fastpath.
- * Flushes invalid cachelines before writing to the target if
- * needs_clflush_before is set and flushes out any written cachelines after
- * writing if needs_clflush is set. */
-static int
-shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
-		  char __user *user_data,
-		  bool page_do_bit17_swizzling,
-		  bool needs_clflush_before,
-		  bool needs_clflush_after)
-{
-	char *vaddr;
-	int ret;
-
-	if (unlikely(page_do_bit17_swizzling))
-		return -EINVAL;
-
-	vaddr = kmap_atomic(page);
-	if (needs_clflush_before)
-		drm_clflush_virt_range(vaddr + shmem_page_offset,
-				       page_length);
-	ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
-					user_data, page_length);
-	if (needs_clflush_after)
-		drm_clflush_virt_range(vaddr + shmem_page_offset,
-				       page_length);
-	kunmap_atomic(vaddr);
-
-	return ret ? -EFAULT : 0;
-}
-
-/* Only difference to the fast-path function is that this can handle bit17
- * and uses non-atomic copy and kmap functions. */
 static int
-shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
+shmem_pwrite_slow(struct page *page, int offset, int length,
 		  char __user *user_data,
 		  bool page_do_bit17_swizzling,
 		  bool needs_clflush_before,
@@ -1284,124 +1297,114 @@ shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 
 	vaddr = kmap(page);
 	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
-		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
-					     page_length,
+		shmem_clflush_swizzled_range(vaddr + offset, length,
 					     page_do_bit17_swizzling);
 	if (page_do_bit17_swizzling)
-		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
-						user_data,
-						page_length);
+		ret = __copy_from_user_swizzled(vaddr, offset, user_data,
+						length);
 	else
-		ret = __copy_from_user(vaddr + shmem_page_offset,
-				       user_data,
-				       page_length);
+		ret = __copy_from_user(vaddr + offset, user_data, length);
 	if (needs_clflush_after)
-		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
-					     page_length,
+		shmem_clflush_swizzled_range(vaddr + offset, length,
 					     page_do_bit17_swizzling);
 	kunmap(page);
 
 	return ret ? -EFAULT : 0;
 }
 
+/* Per-page copy function for the shmem pwrite fastpath.
+ * Flushes invalid cachelines before writing to the target if
+ * needs_clflush_before is set and flushes out any written cachelines after
+ * writing if needs_clflush is set.
+ */
 static int
-i915_gem_shmem_pwrite(struct drm_device *dev,
-		      struct drm_i915_gem_object *obj,
-		      struct drm_i915_gem_pwrite *args,
-		      struct drm_file *file)
+shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
+	     bool page_do_bit17_swizzling,
+	     bool needs_clflush_before,
+	     bool needs_clflush_after)
 {
-	ssize_t remain;
-	loff_t offset;
-	char __user *user_data;
-	int shmem_page_offset, page_length, ret = 0;
-	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
-	int hit_slowpath = 0;
-	unsigned int needs_clflush;
-	struct sg_page_iter sg_iter;
+	int ret;
 
-	ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
-	if (ret)
-		return ret;
+	ret = -ENODEV;
+	if (!page_do_bit17_swizzling) {
+		char *vaddr = kmap_atomic(page);
 
-	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
-	user_data = u64_to_user_ptr(args->data_ptr);
-	offset = args->offset;
-	remain = args->size;
+		if (needs_clflush_before)
+			drm_clflush_virt_range(vaddr + offset, len);
+		ret = __copy_from_user_inatomic(vaddr + offset, user_data, len);
+		if (needs_clflush_after)
+			drm_clflush_virt_range(vaddr + offset, len);
 
-	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
-			 offset >> PAGE_SHIFT) {
-		struct page *page = sg_page_iter_page(&sg_iter);
-		int partial_cacheline_write;
+		kunmap_atomic(vaddr);
+	}
+	if (ret == 0)
+		return ret;
 
-		if (remain <= 0)
-			break;
+	return shmem_pwrite_slow(page, offset, len, user_data,
+				 page_do_bit17_swizzling,
+				 needs_clflush_before,
+				 needs_clflush_after);
+}
 
-		/* Operation in this page
-		 *
-		 * shmem_page_offset = offset within page in shmem file
-		 * page_length = bytes to copy for this page
-		 */
-		shmem_page_offset = offset_in_page(offset);
-
-		page_length = remain;
-		if ((shmem_page_offset + page_length) > PAGE_SIZE)
-			page_length = PAGE_SIZE - shmem_page_offset;
-
-		/* If we don't overwrite a cacheline completely we need to be
-		 * careful to have up-to-date data by first clflushing. Don't
-		 * overcomplicate things and flush the entire patch. */
-		partial_cacheline_write = needs_clflush & CLFLUSH_BEFORE &&
-			((shmem_page_offset | page_length)
-				& (boot_cpu_data.x86_clflush_size - 1));
-
-		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
-			(page_to_phys(page) & (1 << 17)) != 0;
-
-		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
-					user_data, page_do_bit17_swizzling,
-					partial_cacheline_write,
-					needs_clflush & CLFLUSH_AFTER);
-		if (ret == 0)
-			goto next_page;
-
-		hit_slowpath = 1;
-		mutex_unlock(&dev->struct_mutex);
-		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
-					user_data, page_do_bit17_swizzling,
-					partial_cacheline_write,
-					needs_clflush & CLFLUSH_AFTER);
+static int
+i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
+		      const struct drm_i915_gem_pwrite *args)
+{
+	struct drm_i915_private *i915 = to_i915(obj->base.dev);
+	void __user *user_data;
+	u64 remain;
+	unsigned int obj_do_bit17_swizzling;
+	unsigned int partial_cacheline_write;
+	unsigned int needs_clflush;
+	unsigned int offset, idx;
+	int ret;
 
-		mutex_lock(&dev->struct_mutex);
+	ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
+	if (ret)
+		return ret;
 
-		if (ret)
-			goto out;
+	ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
+	mutex_unlock(&i915->drm.struct_mutex);
+	if (ret)
+		return ret;
 
-next_page:
-		remain -= page_length;
-		user_data += page_length;
-		offset += page_length;
-	}
+	obj_do_bit17_swizzling = 0;
+	if (i915_gem_object_needs_bit17_swizzle(obj))
+		obj_do_bit17_swizzling = BIT(17);
 
-out:
-	i915_gem_obj_finish_shmem_access(obj);
+	/* If we don't overwrite a cacheline completely we need to be
+	 * careful to have up-to-date data by first clflushing. Don't
+	 * overcomplicate things and flush the entire patch.
+	 */
+	partial_cacheline_write = 0;
+	if (needs_clflush & CLFLUSH_BEFORE)
+		partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
 
-	if (hit_slowpath) {
-		/*
-		 * Fixup: Flush cpu caches in case we didn't flush the dirty
-		 * cachelines in-line while writing and the object moved
-		 * out of the cpu write domain while we've dropped the lock.
-		 */
-		if (!(needs_clflush & CLFLUSH_AFTER) &&
-		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
-			if (i915_gem_clflush_object(obj, obj->pin_display))
-				needs_clflush |= CLFLUSH_AFTER;
-		}
-	}
+	user_data = u64_to_user_ptr(args->data_ptr);
+	remain = args->size;
+	offset = offset_in_page(args->offset);
+	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
+		struct page *page = i915_gem_object_get_page(obj, idx);
+		int length;
+
+		length = remain;
+		if (offset + length > PAGE_SIZE)
+			length = PAGE_SIZE - offset;
+
+		ret = shmem_pwrite(page, offset, length, user_data,
+				   page_to_phys(page) & obj_do_bit17_swizzling,
+				   (offset | length) & partial_cacheline_write,
+				   needs_clflush & CLFLUSH_AFTER);
+		if (ret)
+			break;
 
-	if (needs_clflush & CLFLUSH_AFTER)
-		i915_gem_chipset_flush(to_i915(dev));
+		remain -= length;
+		user_data += length;
+		offset = 0;
+	}
 
 	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
+	i915_gem_obj_finish_shmem_access(obj);
 	return ret;
 }
 
@@ -1417,7 +1420,6 @@ int
 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 		      struct drm_file *file)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_pwrite *args = data;
 	struct drm_i915_gem_object *obj;
 	int ret;
@@ -1430,13 +1432,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 		       args->size))
 		return -EFAULT;
 
-	if (likely(!i915.prefault_disable)) {
-		ret = fault_in_pages_readable(u64_to_user_ptr(args->data_ptr),
-						   args->size);
-		if (ret)
-			return -EFAULT;
-	}
-
 	obj = i915_gem_object_lookup(file, args->handle);
 	if (!obj)
 		return -ENOENT;
@@ -1450,15 +1445,17 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 
 	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 
-	ret = __unsafe_wait_rendering(obj, to_rps_client(file), false);
+	ret = i915_gem_object_wait(obj,
+				   I915_WAIT_INTERRUPTIBLE |
+				   I915_WAIT_ALL,
+				   MAX_SCHEDULE_TIMEOUT,
+				   to_rps_client(file));
 	if (ret)
 		goto err;
 
-	intel_runtime_pm_get(dev_priv);
-
-	ret = i915_mutex_lock_interruptible(dev);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
-		goto err_rpm;
+		goto err;
 
 	ret = -EFAULT;
 	/* We can only do the GTT pwrite on untiled buffers, as otherwise
@@ -1468,30 +1465,23 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 	 * perspective, requiring manual detiling by the client.
 	 */
 	if (!i915_gem_object_has_struct_page(obj) ||
-	    cpu_write_needs_clflush(obj)) {
-		ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file);
+	    cpu_write_needs_clflush(obj))
 		/* Note that the gtt paths might fail with non-page-backed user
 		 * pointers (e.g. gtt mappings when moving data between
-		 * textures). Fallback to the shmem path in that case. */
-	}
+		 * textures). Fallback to the shmem path in that case.
+		 */
+		ret = i915_gem_gtt_pwrite_fast(obj, args);
 
 	if (ret == -EFAULT || ret == -ENOSPC) {
 		if (obj->phys_handle)
 			ret = i915_gem_phys_pwrite(obj, args, file);
 		else
-			ret = i915_gem_shmem_pwrite(dev, obj, args, file);
+			ret = i915_gem_shmem_pwrite(obj, args);
 	}
 
-	i915_gem_object_put(obj);
-	mutex_unlock(&dev->struct_mutex);
-	intel_runtime_pm_put(dev_priv);
-
-	return ret;
-
-err_rpm:
-	intel_runtime_pm_put(dev_priv);
+	i915_gem_object_unpin_pages(obj);
 err:
-	i915_gem_object_put_unlocked(obj);
+	i915_gem_object_put(obj);
 	return ret;
 }
 
@@ -1502,6 +1492,30 @@ write_origin(struct drm_i915_gem_object *obj, unsigned domain)
 		obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
 }
 
+static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
+{
+	struct drm_i915_private *i915;
+	struct list_head *list;
+	struct i915_vma *vma;
+
+	list_for_each_entry(vma, &obj->vma_list, obj_link) {
+		if (!i915_vma_is_ggtt(vma))
+			continue;
+
+		if (i915_vma_is_active(vma))
+			continue;
+
+		if (!drm_mm_node_allocated(&vma->node))
+			continue;
+
+		list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
+	}
+
+	i915 = to_i915(obj->base.dev);
+	list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list;
+	list_move_tail(&obj->global_link, list);
+}
+
 /**
  * Called when user space prepares to use an object with the CPU, either
  * through the mmap ioctl's mapping or a GTT mapping.
@@ -1517,7 +1531,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	struct drm_i915_gem_object *obj;
 	uint32_t read_domains = args->read_domains;
 	uint32_t write_domain = args->write_domain;
-	int ret;
+	int err;
 
 	/* Only handle setting domains to types used by the CPU. */
 	if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
@@ -1537,29 +1551,48 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	 * We will repeat the flush holding the lock in the normal manner
 	 * to catch cases where we are gazumped.
 	 */
-	ret = __unsafe_wait_rendering(obj, to_rps_client(file), !write_domain);
-	if (ret)
-		goto err;
+	err = i915_gem_object_wait(obj,
+				   I915_WAIT_INTERRUPTIBLE |
+				   (write_domain ? I915_WAIT_ALL : 0),
+				   MAX_SCHEDULE_TIMEOUT,
+				   to_rps_client(file));
+	if (err)
+		goto out;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		goto err;
+	/* Flush and acquire obj->pages so that we are coherent through
+	 * direct access in memory with previous cached writes through
+	 * shmemfs and that our cache domain tracking remains valid.
+	 * For example, if the obj->filp was moved to swap without us
+	 * being notified and releasing the pages, we would mistakenly
+	 * continue to assume that the obj remained out of the CPU cached
+	 * domain.
+	 */
+	err = i915_gem_object_pin_pages(obj);
+	if (err)
+		goto out;
+
+	err = i915_mutex_lock_interruptible(dev);
+	if (err)
+		goto out_unpin;
 
 	if (read_domains & I915_GEM_DOMAIN_GTT)
-		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
+		err = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 	else
-		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
+		err = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
+
+	/* And bump the LRU for this access */
+	i915_gem_object_bump_inactive_ggtt(obj);
+
+	mutex_unlock(&dev->struct_mutex);
 
 	if (write_domain != 0)
 		intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
 
+out_unpin:
+	i915_gem_object_unpin_pages(obj);
+out:
 	i915_gem_object_put(obj);
-	mutex_unlock(&dev->struct_mutex);
-	return ret;
-
-err:
-	i915_gem_object_put_unlocked(obj);
-	return ret;
+	return err;
 }
 
 /**
@@ -1589,7 +1622,7 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 		}
 	}
 
-	i915_gem_object_put_unlocked(obj);
+	i915_gem_object_put(obj);
 	return err;
 }
 
@@ -1635,7 +1668,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 	 * pages from.
 	 */
 	if (!obj->base.filp) {
-		i915_gem_object_put_unlocked(obj);
+		i915_gem_object_put(obj);
 		return -EINVAL;
 	}
 
@@ -1647,7 +1680,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 		struct vm_area_struct *vma;
 
 		if (down_write_killable(&mm->mmap_sem)) {
-			i915_gem_object_put_unlocked(obj);
+			i915_gem_object_put(obj);
 			return -EINTR;
 		}
 		vma = find_vma(mm, addr);
@@ -1661,7 +1694,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 		/* This may race, but that's ok, it only gets set */
 		WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU);
 	}
-	i915_gem_object_put_unlocked(obj);
+	i915_gem_object_put(obj);
 	if (IS_ERR((void *)addr))
 		return addr;
 
@@ -1763,8 +1796,7 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf)
 	int ret;
 
 	/* We don't use vmf->pgoff since that has the fake offset */
-	page_offset = ((unsigned long)vmf->virtual_address - area->vm_start) >>
-		PAGE_SHIFT;
+	page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
 
 	trace_i915_gem_object_fault(obj, page_offset, true, write);
 
@@ -1773,7 +1805,14 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf)
 	 * repeat the flush holding the lock in the normal manner to catch cases
 	 * where we are gazumped.
 	 */
-	ret = __unsafe_wait_rendering(obj, NULL, !write);
+	ret = i915_gem_object_wait(obj,
+				   I915_WAIT_INTERRUPTIBLE,
+				   MAX_SCHEDULE_TIMEOUT,
+				   NULL);
+	if (ret)
+		goto err;
+
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
 		goto err;
 
@@ -1784,7 +1823,7 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf)
 		goto err_rpm;
 
 	/* Access to snoopable pages through the GTT is incoherent. */
-	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
+	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) {
 		ret = -EFAULT;
 		goto err_unlock;
 	}
@@ -1813,8 +1852,7 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf)
 		view.params.partial.offset = rounddown(page_offset, chunk_size);
 		view.params.partial.size =
 			min_t(unsigned int, chunk_size,
-			      (area->vm_end - area->vm_start) / PAGE_SIZE -
-			      view.params.partial.offset);
+			      vma_pages(area) - view.params.partial.offset);
 
 		/* If the partial covers the entire object, just create a
 		 * normal VMA.
@@ -1842,22 +1880,25 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf)
 	if (ret)
 		goto err_unpin;
 
+	/* Mark as being mmapped into userspace for later revocation */
+	assert_rpm_wakelock_held(dev_priv);
+	if (list_empty(&obj->userfault_link))
+		list_add(&obj->userfault_link, &dev_priv->mm.userfault_list);
+
 	/* Finally, remap it using the new GTT offset */
 	ret = remap_io_mapping(area,
 			       area->vm_start + (vma->ggtt_view.params.partial.offset << PAGE_SHIFT),
 			       (ggtt->mappable_base + vma->node.start) >> PAGE_SHIFT,
 			       min_t(u64, vma->size, area->vm_end - area->vm_start),
 			       &ggtt->mappable);
-	if (ret)
-		goto err_unpin;
 
-	obj->fault_mappable = true;
 err_unpin:
 	__i915_vma_unpin(vma);
 err_unlock:
 	mutex_unlock(&dev->struct_mutex);
 err_rpm:
 	intel_runtime_pm_put(dev_priv);
+	i915_gem_object_unpin_pages(obj);
 err:
 	switch (ret) {
 	case -EIO:
@@ -1919,15 +1960,23 @@ err:
 void
 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
 {
+	struct drm_i915_private *i915 = to_i915(obj->base.dev);
+
 	/* Serialisation between user GTT access and our code depends upon
 	 * revoking the CPU's PTE whilst the mutex is held. The next user
 	 * pagefault then has to wait until we release the mutex.
+	 *
+	 * Note that RPM complicates somewhat by adding an additional
+	 * requirement that operations to the GGTT be made holding the RPM
+	 * wakeref.
 	 */
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	lockdep_assert_held(&i915->drm.struct_mutex);
+	intel_runtime_pm_get(i915);
 
-	if (!obj->fault_mappable)
-		return;
+	if (list_empty(&obj->userfault_link))
+		goto out;
 
+	list_del_init(&obj->userfault_link);
 	drm_vma_node_unmap(&obj->base.vma_node,
 			   obj->base.dev->anon_inode->i_mapping);
 
@@ -1940,16 +1989,45 @@ i915_gem_release_mmap(struct drm_i915_gem_object *obj)
 	 */
 	wmb();
 
-	obj->fault_mappable = false;
+out:
+	intel_runtime_pm_put(i915);
 }
 
-void
-i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
+void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_gem_object *obj;
+	struct drm_i915_gem_object *obj, *on;
+	int i;
 
-	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
-		i915_gem_release_mmap(obj);
+	/*
+	 * Only called during RPM suspend. All users of the userfault_list
+	 * must be holding an RPM wakeref to ensure that this can not
+	 * run concurrently with themselves (and use the struct_mutex for
+	 * protection between themselves).
+	 */
+
+	list_for_each_entry_safe(obj, on,
+				 &dev_priv->mm.userfault_list, userfault_link) {
+		list_del_init(&obj->userfault_link);
+		drm_vma_node_unmap(&obj->base.vma_node,
+				   obj->base.dev->anon_inode->i_mapping);
+	}
+
+	/* The fence will be lost when the device powers down. If any were
+	 * in use by hardware (i.e. they are pinned), we should not be powering
+	 * down! All other fences will be reacquired by the user upon waking.
+	 */
+	for (i = 0; i < dev_priv->num_fence_regs; i++) {
+		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
+
+		if (WARN_ON(reg->pin_count))
+			continue;
+
+		if (!reg->vma)
+			continue;
+
+		GEM_BUG_ON(!list_empty(&reg->vma->obj->userfault_link));
+		reg->dirty = true;
+	}
 }
 
 /**
@@ -2063,7 +2141,7 @@ i915_gem_mmap_gtt(struct drm_file *file,
 	if (ret == 0)
 		*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
 
-	i915_gem_object_put_unlocked(obj);
+	i915_gem_object_put(obj);
 	return ret;
 }
 
@@ -2106,16 +2184,18 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj)
 	 * backing pages, *now*.
 	 */
 	shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
-	obj->madv = __I915_MADV_PURGED;
+	obj->mm.madv = __I915_MADV_PURGED;
 }
 
 /* Try to discard unwanted pages */
-static void
-i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
+void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
 {
 	struct address_space *mapping;
 
-	switch (obj->madv) {
+	lockdep_assert_held(&obj->mm.lock);
+	GEM_BUG_ON(obj->mm.pages);
+
+	switch (obj->mm.madv) {
 	case I915_MADV_DONTNEED:
 		i915_gem_object_truncate(obj);
 	case __I915_MADV_PURGED:
@@ -2130,85 +2210,119 @@ i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
 }
 
 static void
-i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
+i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
+			      struct sg_table *pages)
 {
 	struct sgt_iter sgt_iter;
 	struct page *page;
-	int ret;
 
-	BUG_ON(obj->madv == __I915_MADV_PURGED);
+	__i915_gem_object_release_shmem(obj, pages);
 
-	ret = i915_gem_object_set_to_cpu_domain(obj, true);
-	if (WARN_ON(ret)) {
-		/* In the event of a disaster, abandon all caches and
-		 * hope for the best.
-		 */
-		i915_gem_clflush_object(obj, true);
-		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
-	}
-
-	i915_gem_gtt_finish_object(obj);
+	i915_gem_gtt_finish_pages(obj, pages);
 
 	if (i915_gem_object_needs_bit17_swizzle(obj))
-		i915_gem_object_save_bit_17_swizzle(obj);
-
-	if (obj->madv == I915_MADV_DONTNEED)
-		obj->dirty = 0;
+		i915_gem_object_save_bit_17_swizzle(obj, pages);
 
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
-		if (obj->dirty)
+	for_each_sgt_page(page, sgt_iter, pages) {
+		if (obj->mm.dirty)
 			set_page_dirty(page);
 
-		if (obj->madv == I915_MADV_WILLNEED)
+		if (obj->mm.madv == I915_MADV_WILLNEED)
 			mark_page_accessed(page);
 
 		put_page(page);
 	}
-	obj->dirty = 0;
+	obj->mm.dirty = false;
 
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(pages);
+	kfree(pages);
 }
 
-int
-i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
+static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
 {
-	const struct drm_i915_gem_object_ops *ops = obj->ops;
+	struct radix_tree_iter iter;
+	void **slot;
 
-	if (obj->pages == NULL)
-		return 0;
+	radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
+		radix_tree_delete(&obj->mm.get_page.radix, iter.index);
+}
 
-	if (obj->pages_pin_count)
-		return -EBUSY;
+void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj,
+				 enum i915_mm_subclass subclass)
+{
+	struct sg_table *pages;
+
+	if (i915_gem_object_has_pinned_pages(obj))
+		return;
 
 	GEM_BUG_ON(obj->bind_count);
+	if (!READ_ONCE(obj->mm.pages))
+		return;
+
+	/* May be called by shrinker from within get_pages() (on another bo) */
+	mutex_lock_nested(&obj->mm.lock, subclass);
+	if (unlikely(atomic_read(&obj->mm.pages_pin_count)))
+		goto unlock;
 
 	/* ->put_pages might need to allocate memory for the bit17 swizzle
 	 * array, hence protect them from being reaped by removing them from gtt
 	 * lists early. */
-	list_del(&obj->global_list);
+	pages = fetch_and_zero(&obj->mm.pages);
+	GEM_BUG_ON(!pages);
 
-	if (obj->mapping) {
+	if (obj->mm.mapping) {
 		void *ptr;
 
-		ptr = ptr_mask_bits(obj->mapping);
+		ptr = ptr_mask_bits(obj->mm.mapping);
 		if (is_vmalloc_addr(ptr))
 			vunmap(ptr);
 		else
 			kunmap(kmap_to_page(ptr));
 
-		obj->mapping = NULL;
+		obj->mm.mapping = NULL;
 	}
 
-	ops->put_pages(obj);
-	obj->pages = NULL;
+	__i915_gem_object_reset_page_iter(obj);
 
-	i915_gem_object_invalidate(obj);
+	obj->ops->put_pages(obj, pages);
+unlock:
+	mutex_unlock(&obj->mm.lock);
+}
 
+static unsigned int swiotlb_max_size(void)
+{
+#if IS_ENABLED(CONFIG_SWIOTLB)
+	return rounddown(swiotlb_nr_tbl() << IO_TLB_SHIFT, PAGE_SIZE);
+#else
 	return 0;
+#endif
 }
 
-static int
+static void i915_sg_trim(struct sg_table *orig_st)
+{
+	struct sg_table new_st;
+	struct scatterlist *sg, *new_sg;
+	unsigned int i;
+
+	if (orig_st->nents == orig_st->orig_nents)
+		return;
+
+	if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL))
+		return;
+
+	new_sg = new_st.sgl;
+	for_each_sg(orig_st->sgl, sg, orig_st->nents, i) {
+		sg_set_page(new_sg, sg_page(sg), sg->length, 0);
+		/* called before being DMA mapped, no need to copy sg->dma_* */
+		new_sg = sg_next(new_sg);
+	}
+
+	sg_free_table(orig_st);
+
+	*orig_st = new_st;
+}
+
+static struct sg_table *
 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 {
 	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
@@ -2219,6 +2333,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 	struct sgt_iter sgt_iter;
 	struct page *page;
 	unsigned long last_pfn = 0;	/* suppress gcc warning */
+	unsigned int max_segment;
 	int ret;
 	gfp_t gfp;
 
@@ -2226,17 +2341,21 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 	 * wasn't in the GTT, there shouldn't be any way it could have been in
 	 * a GPU cache
 	 */
-	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
-	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
+	GEM_BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
+	GEM_BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
+
+	max_segment = swiotlb_max_size();
+	if (!max_segment)
+		max_segment = rounddown(UINT_MAX, PAGE_SIZE);
 
 	st = kmalloc(sizeof(*st), GFP_KERNEL);
 	if (st == NULL)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	page_count = obj->base.size / PAGE_SIZE;
 	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
 		kfree(st);
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 	}
 
 	/* Get the list of pages out of our struct file.  They'll be pinned
@@ -2264,22 +2383,15 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 			 * our own buffer, now let the real VM do its job and
 			 * go down in flames if truly OOM.
 			 */
-			i915_gem_shrink_all(dev_priv);
 			page = shmem_read_mapping_page(mapping, i);
 			if (IS_ERR(page)) {
 				ret = PTR_ERR(page);
 				goto err_sg;
 			}
 		}
-#ifdef CONFIG_SWIOTLB
-		if (swiotlb_nr_tbl()) {
-			st->nents++;
-			sg_set_page(sg, page, PAGE_SIZE, 0);
-			sg = sg_next(sg);
-			continue;
-		}
-#endif
-		if (!i || page_to_pfn(page) != last_pfn + 1) {
+		if (!i ||
+		    sg->length >= max_segment ||
+		    page_to_pfn(page) != last_pfn + 1) {
 			if (i)
 				sg = sg_next(sg);
 			st->nents++;
@@ -2292,24 +2404,20 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 		/* Check that the i965g/gm workaround works. */
 		WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
 	}
-#ifdef CONFIG_SWIOTLB
-	if (!swiotlb_nr_tbl())
-#endif
+	if (sg) /* loop terminated early; short sg table */
 		sg_mark_end(sg);
-	obj->pages = st;
 
-	ret = i915_gem_gtt_prepare_object(obj);
+	/* Trim unused sg entries to avoid wasting memory. */
+	i915_sg_trim(st);
+
+	ret = i915_gem_gtt_prepare_pages(obj, st);
 	if (ret)
 		goto err_pages;
 
 	if (i915_gem_object_needs_bit17_swizzle(obj))
-		i915_gem_object_do_bit_17_swizzle(obj);
+		i915_gem_object_do_bit_17_swizzle(obj, st);
 
-	if (i915_gem_object_is_tiled(obj) &&
-	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
-		i915_gem_object_pin_pages(obj);
-
-	return 0;
+	return st;
 
 err_sg:
 	sg_mark_end(sg);
@@ -2330,43 +2438,73 @@ err_pages:
 	if (ret == -ENOSPC)
 		ret = -ENOMEM;
 
-	return ret;
+	return ERR_PTR(ret);
 }
 
-/* Ensure that the associated pages are gathered from the backing storage
- * and pinned into our object. i915_gem_object_get_pages() may be called
- * multiple times before they are released by a single call to
- * i915_gem_object_put_pages() - once the pages are no longer referenced
- * either as a result of memory pressure (reaping pages under the shrinker)
- * or as the object is itself released.
- */
-int
-i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
+void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
+				 struct sg_table *pages)
 {
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-	const struct drm_i915_gem_object_ops *ops = obj->ops;
-	int ret;
+	lockdep_assert_held(&obj->mm.lock);
 
-	if (obj->pages)
-		return 0;
+	obj->mm.get_page.sg_pos = pages->sgl;
+	obj->mm.get_page.sg_idx = 0;
+
+	obj->mm.pages = pages;
 
-	if (obj->madv != I915_MADV_WILLNEED) {
+	if (i915_gem_object_is_tiled(obj) &&
+	    to_i915(obj->base.dev)->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
+		GEM_BUG_ON(obj->mm.quirked);
+		__i915_gem_object_pin_pages(obj);
+		obj->mm.quirked = true;
+	}
+}
+
+static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
+{
+	struct sg_table *pages;
+
+	GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
+
+	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
 		DRM_DEBUG("Attempting to obtain a purgeable object\n");
 		return -EFAULT;
 	}
 
-	BUG_ON(obj->pages_pin_count);
+	pages = obj->ops->get_pages(obj);
+	if (unlikely(IS_ERR(pages)))
+		return PTR_ERR(pages);
 
-	ret = ops->get_pages(obj);
-	if (ret)
-		return ret;
+	__i915_gem_object_set_pages(obj, pages);
+	return 0;
+}
+
+/* Ensure that the associated pages are gathered from the backing storage
+ * and pinned into our object. i915_gem_object_pin_pages() may be called
+ * multiple times before they are released by a single call to
+ * i915_gem_object_unpin_pages() - once the pages are no longer referenced
+ * either as a result of memory pressure (reaping pages under the shrinker)
+ * or as the object is itself released.
+ */
+int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
+{
+	int err;
 
-	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
+	err = mutex_lock_interruptible(&obj->mm.lock);
+	if (err)
+		return err;
 
-	obj->get_page.sg = obj->pages->sgl;
-	obj->get_page.last = 0;
+	if (unlikely(!obj->mm.pages)) {
+		err = ____i915_gem_object_get_pages(obj);
+		if (err)
+			goto unlock;
 
-	return 0;
+		smp_mb__before_atomic();
+	}
+	atomic_inc(&obj->mm.pages_pin_count);
+
+unlock:
+	mutex_unlock(&obj->mm.lock);
+	return err;
 }
 
 /* The 'mapping' part of i915_gem_object_pin_map() below */
@@ -2374,7 +2512,7 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
 				 enum i915_map_type type)
 {
 	unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
-	struct sg_table *sgt = obj->pages;
+	struct sg_table *sgt = obj->mm.pages;
 	struct sgt_iter sgt_iter;
 	struct page *page;
 	struct page *stack_pages[32];
@@ -2425,21 +2563,31 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
 	void *ptr;
 	int ret;
 
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
 	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = mutex_lock_interruptible(&obj->mm.lock);
 	if (ret)
 		return ERR_PTR(ret);
 
-	i915_gem_object_pin_pages(obj);
-	pinned = obj->pages_pin_count > 1;
+	pinned = true;
+	if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
+		if (unlikely(!obj->mm.pages)) {
+			ret = ____i915_gem_object_get_pages(obj);
+			if (ret)
+				goto err_unlock;
+
+			smp_mb__before_atomic();
+		}
+		atomic_inc(&obj->mm.pages_pin_count);
+		pinned = false;
+	}
+	GEM_BUG_ON(!obj->mm.pages);
 
-	ptr = ptr_unpack_bits(obj->mapping, has_type);
+	ptr = ptr_unpack_bits(obj->mm.mapping, has_type);
 	if (ptr && has_type != type) {
 		if (pinned) {
 			ret = -EBUSY;
-			goto err;
+			goto err_unpin;
 		}
 
 		if (is_vmalloc_addr(ptr))
@@ -2447,59 +2595,28 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
 		else
 			kunmap(kmap_to_page(ptr));
 
-		ptr = obj->mapping = NULL;
+		ptr = obj->mm.mapping = NULL;
 	}
 
 	if (!ptr) {
 		ptr = i915_gem_object_map(obj, type);
 		if (!ptr) {
 			ret = -ENOMEM;
-			goto err;
+			goto err_unpin;
 		}
 
-		obj->mapping = ptr_pack_bits(ptr, type);
+		obj->mm.mapping = ptr_pack_bits(ptr, type);
 	}
 
+out_unlock:
+	mutex_unlock(&obj->mm.lock);
 	return ptr;
 
-err:
-	i915_gem_object_unpin_pages(obj);
-	return ERR_PTR(ret);
-}
-
-static void
-i915_gem_object_retire__write(struct i915_gem_active *active,
-			      struct drm_i915_gem_request *request)
-{
-	struct drm_i915_gem_object *obj =
-		container_of(active, struct drm_i915_gem_object, last_write);
-
-	intel_fb_obj_flush(obj, true, ORIGIN_CS);
-}
-
-static void
-i915_gem_object_retire__read(struct i915_gem_active *active,
-			     struct drm_i915_gem_request *request)
-{
-	int idx = request->engine->id;
-	struct drm_i915_gem_object *obj =
-		container_of(active, struct drm_i915_gem_object, last_read[idx]);
-
-	GEM_BUG_ON(!i915_gem_object_has_active_engine(obj, idx));
-
-	i915_gem_object_clear_active(obj, idx);
-	if (i915_gem_object_is_active(obj))
-		return;
-
-	/* Bump our place on the bound list to keep it roughly in LRU order
-	 * so that we don't steal from recently used but inactive objects
-	 * (unless we are forced to ofc!)
-	 */
-	if (obj->bind_count)
-		list_move_tail(&obj->global_list,
-			       &request->i915->mm.bound_list);
-
-	i915_gem_object_put(obj);
+err_unpin:
+	atomic_dec(&obj->mm.pages_pin_count);
+err_unlock:
+	ptr = ERR_PTR(ret);
+	goto out_unlock;
 }
 
 static bool i915_context_is_banned(const struct i915_gem_context *ctx)
@@ -2546,13 +2663,10 @@ i915_gem_find_active_request(struct intel_engine_cs *engine)
 	 * extra delay for a recent interrupt is pointless. Hence, we do
 	 * not need an engine->irq_seqno_barrier() before the seqno reads.
 	 */
-	list_for_each_entry(request, &engine->request_list, link) {
-		if (i915_gem_request_completed(request))
+	list_for_each_entry(request, &engine->timeline->requests, link) {
+		if (__i915_gem_request_completed(request))
 			continue;
 
-		if (!i915_sw_fence_done(&request->submit))
-			break;
-
 		return request;
 	}
 
@@ -2580,10 +2694,9 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine)
 {
 	struct drm_i915_gem_request *request;
 	struct i915_gem_context *incomplete_ctx;
+	struct intel_timeline *timeline;
 	bool ring_hung;
 
-	/* Ensure irq handler finishes, and not run again. */
-	tasklet_kill(&engine->irq_tasklet);
 	if (engine->irq_seqno_barrier)
 		engine->irq_seqno_barrier(engine);
 
@@ -2592,12 +2705,15 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine)
 		return;
 
 	ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
+	if (engine->hangcheck.seqno != intel_engine_get_seqno(engine))
+		ring_hung = false;
+
 	i915_set_reset_status(request->ctx, ring_hung);
 	if (!ring_hung)
 		return;
 
 	DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n",
-			 engine->name, request->fence.seqno);
+			 engine->name, request->global_seqno);
 
 	/* Setup the CS to resume from the breadcrumb of the hung request */
 	engine->reset_hw(engine, request);
@@ -2614,21 +2730,28 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine)
 	if (i915_gem_context_is_default(incomplete_ctx))
 		return;
 
-	list_for_each_entry_continue(request, &engine->request_list, link)
+	list_for_each_entry_continue(request, &engine->timeline->requests, link)
 		if (request->ctx == incomplete_ctx)
 			reset_request(request);
+
+	timeline = i915_gem_context_lookup_timeline(incomplete_ctx, engine);
+	list_for_each_entry(request, &timeline->requests, link)
+		reset_request(request);
 }
 
 void i915_gem_reset(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
 	i915_gem_retire_requests(dev_priv);
 
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		i915_gem_reset_engine(engine);
 
-	i915_gem_restore_fences(&dev_priv->drm);
+	i915_gem_restore_fences(dev_priv);
 
 	if (dev_priv->gt.awake) {
 		intel_sanitize_gt_powersave(dev_priv);
@@ -2640,6 +2763,8 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
 
 static void nop_submit_request(struct drm_i915_gem_request *request)
 {
+	i915_gem_request_submit(request);
+	intel_engine_init_global_seqno(request->engine, request->global_seqno);
 }
 
 static void i915_gem_cleanup_engine(struct intel_engine_cs *engine)
@@ -2650,7 +2775,8 @@ static void i915_gem_cleanup_engine(struct intel_engine_cs *engine)
 	 * (lockless) lookup doesn't try and wait upon the request as we
 	 * reset it.
 	 */
-	intel_engine_init_seqno(engine, engine->last_submitted_seqno);
+	intel_engine_init_global_seqno(engine,
+				       intel_engine_last_submit(engine));
 
 	/*
 	 * Clear the execlists queue up before freeing the requests, as those
@@ -2659,26 +2785,30 @@ static void i915_gem_cleanup_engine(struct intel_engine_cs *engine)
 	 */
 
 	if (i915.enable_execlists) {
-		spin_lock(&engine->execlist_lock);
-		INIT_LIST_HEAD(&engine->execlist_queue);
+		unsigned long flags;
+
+		spin_lock_irqsave(&engine->timeline->lock, flags);
+
 		i915_gem_request_put(engine->execlist_port[0].request);
 		i915_gem_request_put(engine->execlist_port[1].request);
 		memset(engine->execlist_port, 0, sizeof(engine->execlist_port));
-		spin_unlock(&engine->execlist_lock);
-	}
+		engine->execlist_queue = RB_ROOT;
+		engine->execlist_first = NULL;
 
-	engine->i915->gt.active_engines &= ~intel_engine_flag(engine);
+		spin_unlock_irqrestore(&engine->timeline->lock, flags);
+	}
 }
 
 void i915_gem_set_wedged(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 	set_bit(I915_WEDGED, &dev_priv->gpu_error.flags);
 
 	i915_gem_context_lost(dev_priv);
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		i915_gem_cleanup_engine(engine);
 	mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0);
 
@@ -2717,12 +2847,20 @@ i915_gem_idle_work_handler(struct work_struct *work)
 		container_of(work, typeof(*dev_priv), gt.idle_work.work);
 	struct drm_device *dev = &dev_priv->drm;
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	bool rearm_hangcheck;
 
 	if (!READ_ONCE(dev_priv->gt.awake))
 		return;
 
-	if (READ_ONCE(dev_priv->gt.active_engines))
+	/*
+	 * Wait for last execlists context complete, but bail out in case a
+	 * new request is submitted.
+	 */
+	wait_for(READ_ONCE(dev_priv->gt.active_requests) ||
+		 intel_execlists_idle(dev_priv), 10);
+
+	if (READ_ONCE(dev_priv->gt.active_requests))
 		return;
 
 	rearm_hangcheck =
@@ -2736,10 +2874,20 @@ i915_gem_idle_work_handler(struct work_struct *work)
 		goto out_rearm;
 	}
 
-	if (dev_priv->gt.active_engines)
+	/*
+	 * New request retired after this work handler started, extend active
+	 * period until next instance of the work.
+	 */
+	if (work_pending(work))
+		goto out_unlock;
+
+	if (dev_priv->gt.active_requests)
 		goto out_unlock;
 
-	for_each_engine(engine, dev_priv)
+	if (wait_for(intel_execlists_idle(dev_priv), 10))
+		DRM_ERROR("Timeout waiting for engines to idle\n");
+
+	for_each_engine(engine, dev_priv, id)
 		i915_gem_batch_pool_fini(&engine->batch_pool);
 
 	GEM_BUG_ON(!dev_priv->gt.awake);
@@ -2769,9 +2917,26 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
 	list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link)
 		if (vma->vm->file == fpriv)
 			i915_vma_close(vma);
+
+	if (i915_gem_object_is_active(obj) &&
+	    !i915_gem_object_has_active_reference(obj)) {
+		i915_gem_object_set_active_reference(obj);
+		i915_gem_object_get(obj);
+	}
 	mutex_unlock(&obj->base.dev->struct_mutex);
 }
 
+static unsigned long to_wait_timeout(s64 timeout_ns)
+{
+	if (timeout_ns < 0)
+		return MAX_SCHEDULE_TIMEOUT;
+
+	if (timeout_ns == 0)
+		return 0;
+
+	return nsecs_to_jiffies_timeout(timeout_ns);
+}
+
 /**
  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
  * @dev: drm device pointer
@@ -2800,10 +2965,9 @@ int
 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 {
 	struct drm_i915_gem_wait *args = data;
-	struct intel_rps_client *rps = to_rps_client(file);
 	struct drm_i915_gem_object *obj;
-	unsigned long active;
-	int idx, ret = 0;
+	ktime_t start;
+	long ret;
 
 	if (args->flags != 0)
 		return -EINVAL;
@@ -2812,133 +2976,29 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 	if (!obj)
 		return -ENOENT;
 
-	active = __I915_BO_ACTIVE(obj);
-	for_each_active(active, idx) {
-		s64 *timeout = args->timeout_ns >= 0 ? &args->timeout_ns : NULL;
-		ret = i915_gem_active_wait_unlocked(&obj->last_read[idx],
-						    I915_WAIT_INTERRUPTIBLE,
-						    timeout, rps);
-		if (ret)
-			break;
-	}
-
-	i915_gem_object_put_unlocked(obj);
-	return ret;
-}
-
-static void __i915_vma_iounmap(struct i915_vma *vma)
-{
-	GEM_BUG_ON(i915_vma_is_pinned(vma));
-
-	if (vma->iomap == NULL)
-		return;
-
-	io_mapping_unmap(vma->iomap);
-	vma->iomap = NULL;
-}
-
-int i915_vma_unbind(struct i915_vma *vma)
-{
-	struct drm_i915_gem_object *obj = vma->obj;
-	unsigned long active;
-	int ret;
-
-	/* First wait upon any activity as retiring the request may
-	 * have side-effects such as unpinning or even unbinding this vma.
-	 */
-	active = i915_vma_get_active(vma);
-	if (active) {
-		int idx;
-
-		/* When a closed VMA is retired, it is unbound - eek.
-		 * In order to prevent it from being recursively closed,
-		 * take a pin on the vma so that the second unbind is
-		 * aborted.
-		 */
-		__i915_vma_pin(vma);
+	start = ktime_get();
 
-		for_each_active(active, idx) {
-			ret = i915_gem_active_retire(&vma->last_read[idx],
-						   &vma->vm->dev->struct_mutex);
-			if (ret)
-				break;
-		}
-
-		__i915_vma_unpin(vma);
-		if (ret)
-			return ret;
+	ret = i915_gem_object_wait(obj,
+				   I915_WAIT_INTERRUPTIBLE | I915_WAIT_ALL,
+				   to_wait_timeout(args->timeout_ns),
+				   to_rps_client(file));
 
-		GEM_BUG_ON(i915_vma_is_active(vma));
+	if (args->timeout_ns > 0) {
+		args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
+		if (args->timeout_ns < 0)
+			args->timeout_ns = 0;
 	}
 
-	if (i915_vma_is_pinned(vma))
-		return -EBUSY;
-
-	if (!drm_mm_node_allocated(&vma->node))
-		goto destroy;
-
-	GEM_BUG_ON(obj->bind_count == 0);
-	GEM_BUG_ON(!obj->pages);
-
-	if (i915_vma_is_map_and_fenceable(vma)) {
-		/* release the fence reg _after_ flushing */
-		ret = i915_vma_put_fence(vma);
-		if (ret)
-			return ret;
-
-		/* Force a pagefault for domain tracking on next user access */
-		i915_gem_release_mmap(obj);
-
-		__i915_vma_iounmap(vma);
-		vma->flags &= ~I915_VMA_CAN_FENCE;
-	}
-
-	if (likely(!vma->vm->closed)) {
-		trace_i915_vma_unbind(vma);
-		vma->vm->unbind_vma(vma);
-	}
-	vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
-
-	drm_mm_remove_node(&vma->node);
-	list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
-
-	if (vma->pages != obj->pages) {
-		GEM_BUG_ON(!vma->pages);
-		sg_free_table(vma->pages);
-		kfree(vma->pages);
-	}
-	vma->pages = NULL;
-
-	/* Since the unbound list is global, only move to that list if
-	 * no more VMAs exist. */
-	if (--obj->bind_count == 0)
-		list_move_tail(&obj->global_list,
-			       &to_i915(obj->base.dev)->mm.unbound_list);
-
-	/* And finally now the object is completely decoupled from this vma,
-	 * we can drop its hold on the backing storage and allow it to be
-	 * reaped by the shrinker.
-	 */
-	i915_gem_object_unpin_pages(obj);
-
-destroy:
-	if (unlikely(i915_vma_is_closed(vma)))
-		i915_vma_destroy(vma);
-
-	return 0;
+	i915_gem_object_put(obj);
+	return ret;
 }
 
-int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
-			   unsigned int flags)
+static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags)
 {
-	struct intel_engine_cs *engine;
-	int ret;
-
-	for_each_engine(engine, dev_priv) {
-		if (engine->last_context == NULL)
-			continue;
+	int ret, i;
 
-		ret = intel_engine_idle(engine, flags);
+	for (i = 0; i < ARRAY_SIZE(tl->engine); i++) {
+		ret = i915_gem_active_wait(&tl->engine[i].last_request, flags);
 		if (ret)
 			return ret;
 	}
@@ -2946,187 +3006,45 @@ int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
 	return 0;
 }
 
-static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
-				     unsigned long cache_level)
-{
-	struct drm_mm_node *gtt_space = &vma->node;
-	struct drm_mm_node *other;
-
-	/*
-	 * On some machines we have to be careful when putting differing types
-	 * of snoopable memory together to avoid the prefetcher crossing memory
-	 * domains and dying. During vm initialisation, we decide whether or not
-	 * these constraints apply and set the drm_mm.color_adjust
-	 * appropriately.
-	 */
-	if (vma->vm->mm.color_adjust == NULL)
-		return true;
-
-	if (!drm_mm_node_allocated(gtt_space))
-		return true;
-
-	if (list_empty(&gtt_space->node_list))
-		return true;
-
-	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
-	if (other->allocated && !other->hole_follows && other->color != cache_level)
-		return false;
-
-	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
-	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
-		return false;
-
-	return true;
-}
-
-/**
- * i915_vma_insert - finds a slot for the vma in its address space
- * @vma: the vma
- * @size: requested size in bytes (can be larger than the VMA)
- * @alignment: required alignment
- * @flags: mask of PIN_* flags to use
- *
- * First we try to allocate some free space that meets the requirements for
- * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
- * preferrably the oldest idle entry to make room for the new VMA.
- *
- * Returns:
- * 0 on success, negative error code otherwise.
- */
-static int
-i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
+int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
 {
-	struct drm_i915_private *dev_priv = to_i915(vma->vm->dev);
-	struct drm_i915_gem_object *obj = vma->obj;
-	u64 start, end;
 	int ret;
 
-	GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
-	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
-
-	size = max(size, vma->size);
-	if (flags & PIN_MAPPABLE)
-		size = i915_gem_get_ggtt_size(dev_priv, size,
-					      i915_gem_object_get_tiling(obj));
-
-	alignment = max(max(alignment, vma->display_alignment),
-			i915_gem_get_ggtt_alignment(dev_priv, size,
-						    i915_gem_object_get_tiling(obj),
-						    flags & PIN_MAPPABLE));
-
-	start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
-
-	end = vma->vm->total;
-	if (flags & PIN_MAPPABLE)
-		end = min_t(u64, end, dev_priv->ggtt.mappable_end);
-	if (flags & PIN_ZONE_4G)
-		end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
-
-	/* If binding the object/GGTT view requires more space than the entire
-	 * aperture has, reject it early before evicting everything in a vain
-	 * attempt to find space.
-	 */
-	if (size > end) {
-		DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n",
-			  size, obj->base.size,
-			  flags & PIN_MAPPABLE ? "mappable" : "total",
-			  end);
-		return -E2BIG;
-	}
-
-	ret = i915_gem_object_get_pages(obj);
-	if (ret)
-		return ret;
-
-	i915_gem_object_pin_pages(obj);
+	if (flags & I915_WAIT_LOCKED) {
+		struct i915_gem_timeline *tl;
 
-	if (flags & PIN_OFFSET_FIXED) {
-		u64 offset = flags & PIN_OFFSET_MASK;
-		if (offset & (alignment - 1) || offset > end - size) {
-			ret = -EINVAL;
-			goto err_unpin;
-		}
+		lockdep_assert_held(&i915->drm.struct_mutex);
 
-		vma->node.start = offset;
-		vma->node.size = size;
-		vma->node.color = obj->cache_level;
-		ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
-		if (ret) {
-			ret = i915_gem_evict_for_vma(vma);
-			if (ret == 0)
-				ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
+		list_for_each_entry(tl, &i915->gt.timelines, link) {
+			ret = wait_for_timeline(tl, flags);
 			if (ret)
-				goto err_unpin;
+				return ret;
 		}
 	} else {
-		u32 search_flag, alloc_flag;
-
-		if (flags & PIN_HIGH) {
-			search_flag = DRM_MM_SEARCH_BELOW;
-			alloc_flag = DRM_MM_CREATE_TOP;
-		} else {
-			search_flag = DRM_MM_SEARCH_DEFAULT;
-			alloc_flag = DRM_MM_CREATE_DEFAULT;
-		}
-
-		/* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
-		 * so we know that we always have a minimum alignment of 4096.
-		 * The drm_mm range manager is optimised to return results
-		 * with zero alignment, so where possible use the optimal
-		 * path.
-		 */
-		if (alignment <= 4096)
-			alignment = 0;
-
-search_free:
-		ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm,
-							  &vma->node,
-							  size, alignment,
-							  obj->cache_level,
-							  start, end,
-							  search_flag,
-							  alloc_flag);
-		if (ret) {
-			ret = i915_gem_evict_something(vma->vm, size, alignment,
-						       obj->cache_level,
-						       start, end,
-						       flags);
-			if (ret == 0)
-				goto search_free;
-
-			goto err_unpin;
-		}
+		ret = wait_for_timeline(&i915->gt.global_timeline, flags);
+		if (ret)
+			return ret;
 	}
-	GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level));
-
-	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
-	list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
-	obj->bind_count++;
 
 	return 0;
-
-err_unpin:
-	i915_gem_object_unpin_pages(obj);
-	return ret;
 }
 
-bool
-i915_gem_clflush_object(struct drm_i915_gem_object *obj,
-			bool force)
+void i915_gem_clflush_object(struct drm_i915_gem_object *obj,
+			     bool force)
 {
 	/* If we don't have a page list set up, then we're not pinned
 	 * to GPU, and we can ignore the cache flush because it'll happen
 	 * again at bind time.
 	 */
-	if (obj->pages == NULL)
-		return false;
+	if (!obj->mm.pages)
+		return;
 
 	/*
 	 * Stolen memory is always coherent with the GPU as it is explicitly
 	 * marked as wc by the system, or the system is cache-coherent.
 	 */
 	if (obj->stolen || obj->phys_handle)
-		return false;
+		return;
 
 	/* If the GPU is snooping the contents of the CPU cache,
 	 * we do not need to manually clear the CPU cache lines.  However,
@@ -3138,14 +3056,12 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj,
 	 */
 	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
 		obj->cache_dirty = true;
-		return false;
+		return;
 	}
 
 	trace_i915_gem_object_clflush(obj);
-	drm_clflush_sg(obj->pages);
+	drm_clflush_sg(obj->mm.pages);
 	obj->cache_dirty = false;
-
-	return true;
 }
 
 /** Flushes the GTT write domain for the object if it's dirty. */
@@ -3174,7 +3090,7 @@ i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
 	 */
 	wmb();
 	if (INTEL_GEN(dev_priv) >= 6 && !HAS_LLC(dev_priv))
-		POSTING_READ(RING_ACTHD(dev_priv->engine[RCS].mmio_base));
+		POSTING_READ(RING_ACTHD(dev_priv->engine[RCS]->mmio_base));
 
 	intel_fb_obj_flush(obj, false, write_origin(obj, I915_GEM_DOMAIN_GTT));
 
@@ -3191,9 +3107,7 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
 	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
 		return;
 
-	if (i915_gem_clflush_object(obj, obj->pin_display))
-		i915_gem_chipset_flush(to_i915(obj->base.dev));
-
+	i915_gem_clflush_object(obj, obj->pin_display);
 	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 
 	obj->base.write_domain = 0;
@@ -3202,24 +3116,6 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
 					    I915_GEM_DOMAIN_CPU);
 }
 
-static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
-{
-	struct i915_vma *vma;
-
-	list_for_each_entry(vma, &obj->vma_list, obj_link) {
-		if (!i915_vma_is_ggtt(vma))
-			continue;
-
-		if (i915_vma_is_active(vma))
-			continue;
-
-		if (!drm_mm_node_allocated(&vma->node))
-			continue;
-
-		list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
-	}
-}
-
 /**
  * Moves a single object to the GTT read, and possibly write domain.
  * @obj: object to act on
@@ -3234,7 +3130,14 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 	uint32_t old_write_domain, old_read_domains;
 	int ret;
 
-	ret = i915_gem_object_wait_rendering(obj, !write);
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
+	ret = i915_gem_object_wait(obj,
+				   I915_WAIT_INTERRUPTIBLE |
+				   I915_WAIT_LOCKED |
+				   (write ? I915_WAIT_ALL : 0),
+				   MAX_SCHEDULE_TIMEOUT,
+				   NULL);
 	if (ret)
 		return ret;
 
@@ -3249,7 +3152,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 	 * continue to assume that the obj remained out of the CPU cached
 	 * domain.
 	 */
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
 		return ret;
 
@@ -3268,21 +3171,19 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 	/* It should now be out of any other write domains, and we can update
 	 * the domain values for our changes.
 	 */
-	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
+	GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
 	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
 	if (write) {
 		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
 		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
-		obj->dirty = 1;
+		obj->mm.dirty = true;
 	}
 
 	trace_i915_gem_object_change_domain(obj,
 					    old_read_domains,
 					    old_write_domain);
 
-	/* And bump the LRU for this access */
-	i915_gem_object_bump_inactive_ggtt(obj);
-
+	i915_gem_object_unpin_pages(obj);
 	return 0;
 }
 
@@ -3305,10 +3206,12 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 				    enum i915_cache_level cache_level)
 {
 	struct i915_vma *vma;
-	int ret = 0;
+	int ret;
+
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
 	if (obj->cache_level == cache_level)
-		goto out;
+		return 0;
 
 	/* Inspect the list of currently bound VMA and unbind any that would
 	 * be invalid given the new cache-level. This is principally to
@@ -3351,11 +3254,17 @@ restart:
 		 * If we wait upon the object, we know that all the bound
 		 * VMA are no longer active.
 		 */
-		ret = i915_gem_object_wait_rendering(obj, false);
+		ret = i915_gem_object_wait(obj,
+					   I915_WAIT_INTERRUPTIBLE |
+					   I915_WAIT_LOCKED |
+					   I915_WAIT_ALL,
+					   MAX_SCHEDULE_TIMEOUT,
+					   NULL);
 		if (ret)
 			return ret;
 
-		if (!HAS_LLC(obj->base.dev) && cache_level != I915_CACHE_NONE) {
+		if (!HAS_LLC(to_i915(obj->base.dev)) &&
+		    cache_level != I915_CACHE_NONE) {
 			/* Access to snoopable pages through the GTT is
 			 * incoherent and on some machines causes a hard
 			 * lockup. Relinquish the CPU mmaping to force
@@ -3397,20 +3306,14 @@ restart:
 		}
 	}
 
+	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU &&
+	    cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
+		obj->cache_dirty = true;
+
 	list_for_each_entry(vma, &obj->vma_list, obj_link)
 		vma->node.color = cache_level;
 	obj->cache_level = cache_level;
 
-out:
-	/* Flush the dirty CPU caches to the backing storage so that the
-	 * object is now coherent at its new cache level (with respect
-	 * to the access domain).
-	 */
-	if (obj->cache_dirty && cpu_write_needs_clflush(obj)) {
-		if (i915_gem_clflush_object(obj, true))
-			i915_gem_chipset_flush(to_i915(obj->base.dev));
-	}
-
 	return 0;
 }
 
@@ -3419,10 +3322,14 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_i915_gem_caching *args = data;
 	struct drm_i915_gem_object *obj;
+	int err = 0;
 
-	obj = i915_gem_object_lookup(file, args->handle);
-	if (!obj)
-		return -ENOENT;
+	rcu_read_lock();
+	obj = i915_gem_object_lookup_rcu(file, args->handle);
+	if (!obj) {
+		err = -ENOENT;
+		goto out;
+	}
 
 	switch (obj->cache_level) {
 	case I915_CACHE_LLC:
@@ -3438,15 +3345,15 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
 		args->caching = I915_CACHING_NONE;
 		break;
 	}
-
-	i915_gem_object_put_unlocked(obj);
-	return 0;
+out:
+	rcu_read_unlock();
+	return err;
 }
 
 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
 			       struct drm_file *file)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *i915 = to_i915(dev);
 	struct drm_i915_gem_caching *args = data;
 	struct drm_i915_gem_object *obj;
 	enum i915_cache_level level;
@@ -3463,23 +3370,21 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
 		 * cacheline, whereas normally such cachelines would get
 		 * invalidated.
 		 */
-		if (!HAS_LLC(dev) && !HAS_SNOOP(dev))
+		if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
 			return -ENODEV;
 
 		level = I915_CACHE_LLC;
 		break;
 	case I915_CACHING_DISPLAY:
-		level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
+		level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
 		break;
 	default:
 		return -EINVAL;
 	}
 
-	intel_runtime_pm_get(dev_priv);
-
 	ret = i915_mutex_lock_interruptible(dev);
 	if (ret)
-		goto rpm_put;
+		return ret;
 
 	obj = i915_gem_object_lookup(file, args->handle);
 	if (!obj) {
@@ -3488,13 +3393,9 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
 	}
 
 	ret = i915_gem_object_set_cache_level(obj, level);
-
 	i915_gem_object_put(obj);
 unlock:
 	mutex_unlock(&dev->struct_mutex);
-rpm_put:
-	intel_runtime_pm_put(dev_priv);
-
 	return ret;
 }
 
@@ -3512,6 +3413,8 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 	u32 old_read_domains, old_write_domain;
 	int ret;
 
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
 	/* Mark the pin_display early so that we account for the
 	 * display coherency whilst setting up the cache domains.
 	 */
@@ -3527,7 +3430,8 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 	 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
 	 */
 	ret = i915_gem_object_set_cache_level(obj,
-					      HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
+					      HAS_WT(to_i915(obj->base.dev)) ?
+					      I915_CACHE_WT : I915_CACHE_NONE);
 	if (ret) {
 		vma = ERR_PTR(ret);
 		goto err_unpin_display;
@@ -3565,7 +3469,11 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 
 	vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
 
-	i915_gem_object_flush_cpu_write_domain(obj);
+	/* Treat this as an end-of-frame, like intel_user_framebuffer_dirty() */
+	if (obj->cache_dirty) {
+		i915_gem_clflush_object(obj, true);
+		intel_fb_obj_flush(obj, false, ORIGIN_DIRTYFB);
+	}
 
 	old_write_domain = obj->base.write_domain;
 	old_read_domains = obj->base.read_domains;
@@ -3590,6 +3498,8 @@ err_unpin_display:
 void
 i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
 {
+	lockdep_assert_held(&vma->vm->dev->struct_mutex);
+
 	if (WARN_ON(vma->obj->pin_display == 0))
 		return;
 
@@ -3617,7 +3527,14 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 	uint32_t old_write_domain, old_read_domains;
 	int ret;
 
-	ret = i915_gem_object_wait_rendering(obj, !write);
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
+	ret = i915_gem_object_wait(obj,
+				   I915_WAIT_INTERRUPTIBLE |
+				   I915_WAIT_LOCKED |
+				   (write ? I915_WAIT_ALL : 0),
+				   MAX_SCHEDULE_TIMEOUT,
+				   NULL);
 	if (ret)
 		return ret;
 
@@ -3639,7 +3556,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 	/* It should now be out of any other write domains, and we can update
 	 * the domain values for our changes.
 	 */
-	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
+	GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
 
 	/* If we're writing through the CPU, then the GPU read domains will
 	 * need to be invalidated at next use.
@@ -3673,11 +3590,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
 	struct drm_i915_file_private *file_priv = file->driver_priv;
 	unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
 	struct drm_i915_gem_request *request, *target = NULL;
-	int ret;
-
-	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
-	if (ret)
-		return ret;
+	long ret;
 
 	/* ABI: return -EIO if already wedged */
 	if (i915_terminally_wedged(&dev_priv->gpu_error))
@@ -3704,103 +3617,12 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
 	if (target == NULL)
 		return 0;
 
-	ret = i915_wait_request(target, I915_WAIT_INTERRUPTIBLE, NULL, NULL);
+	ret = i915_wait_request(target,
+				I915_WAIT_INTERRUPTIBLE,
+				MAX_SCHEDULE_TIMEOUT);
 	i915_gem_request_put(target);
 
-	return ret;
-}
-
-static bool
-i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
-{
-	if (!drm_mm_node_allocated(&vma->node))
-		return false;
-
-	if (vma->node.size < size)
-		return true;
-
-	if (alignment && vma->node.start & (alignment - 1))
-		return true;
-
-	if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
-		return true;
-
-	if (flags & PIN_OFFSET_BIAS &&
-	    vma->node.start < (flags & PIN_OFFSET_MASK))
-		return true;
-
-	if (flags & PIN_OFFSET_FIXED &&
-	    vma->node.start != (flags & PIN_OFFSET_MASK))
-		return true;
-
-	return false;
-}
-
-void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
-{
-	struct drm_i915_gem_object *obj = vma->obj;
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-	bool mappable, fenceable;
-	u32 fence_size, fence_alignment;
-
-	fence_size = i915_gem_get_ggtt_size(dev_priv,
-					    vma->size,
-					    i915_gem_object_get_tiling(obj));
-	fence_alignment = i915_gem_get_ggtt_alignment(dev_priv,
-						      vma->size,
-						      i915_gem_object_get_tiling(obj),
-						      true);
-
-	fenceable = (vma->node.size == fence_size &&
-		     (vma->node.start & (fence_alignment - 1)) == 0);
-
-	mappable = (vma->node.start + fence_size <=
-		    dev_priv->ggtt.mappable_end);
-
-	/*
-	 * Explicitly disable for rotated VMA since the display does not
-	 * need the fence and the VMA is not accessible to other users.
-	 */
-	if (mappable && fenceable &&
-	    vma->ggtt_view.type != I915_GGTT_VIEW_ROTATED)
-		vma->flags |= I915_VMA_CAN_FENCE;
-	else
-		vma->flags &= ~I915_VMA_CAN_FENCE;
-}
-
-int __i915_vma_do_pin(struct i915_vma *vma,
-		      u64 size, u64 alignment, u64 flags)
-{
-	unsigned int bound = vma->flags;
-	int ret;
-
-	GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
-	GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
-
-	if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
-		ret = -EBUSY;
-		goto err;
-	}
-
-	if ((bound & I915_VMA_BIND_MASK) == 0) {
-		ret = i915_vma_insert(vma, size, alignment, flags);
-		if (ret)
-			goto err;
-	}
-
-	ret = i915_vma_bind(vma, vma->obj->cache_level, flags);
-	if (ret)
-		goto err;
-
-	if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND)
-		__i915_vma_set_map_and_fenceable(vma);
-
-	GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
-	return 0;
-
-err:
-	__i915_vma_unpin(vma);
-	return ret;
+	return ret < 0 ? ret : 0;
 }
 
 struct i915_vma *
@@ -3810,10 +3632,13 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 			 u64 alignment,
 			 u64 flags)
 {
-	struct i915_address_space *vm = &to_i915(obj->base.dev)->ggtt.base;
+	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+	struct i915_address_space *vm = &dev_priv->ggtt.base;
 	struct i915_vma *vma;
 	int ret;
 
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
 	vma = i915_gem_obj_lookup_or_create_vma(obj, vm, view);
 	if (IS_ERR(vma))
 		return vma;
@@ -3823,6 +3648,41 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 		    (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)))
 			return ERR_PTR(-ENOSPC);
 
+		if (flags & PIN_MAPPABLE) {
+			u32 fence_size;
+
+			fence_size = i915_gem_get_ggtt_size(dev_priv, vma->size,
+							    i915_gem_object_get_tiling(obj));
+			/* If the required space is larger than the available
+			 * aperture, we will not able to find a slot for the
+			 * object and unbinding the object now will be in
+			 * vain. Worse, doing so may cause us to ping-pong
+			 * the object in and out of the Global GTT and
+			 * waste a lot of cycles under the mutex.
+			 */
+			if (fence_size > dev_priv->ggtt.mappable_end)
+				return ERR_PTR(-E2BIG);
+
+			/* If NONBLOCK is set the caller is optimistically
+			 * trying to cache the full object within the mappable
+			 * aperture, and *must* have a fallback in place for
+			 * situations where we cannot bind the object. We
+			 * can be a little more lax here and use the fallback
+			 * more often to avoid costly migrations of ourselves
+			 * and other objects within the aperture.
+			 *
+			 * Half-the-aperture is used as a simple heuristic.
+			 * More interesting would to do search for a free
+			 * block prior to making the commitment to unbind.
+			 * That caters for the self-harm case, and with a
+			 * little more heuristics (e.g. NOFAULT, NOEVICT)
+			 * we could try to minimise harm to others.
+			 */
+			if (flags & PIN_NONBLOCK &&
+			    fence_size > dev_priv->ggtt.mappable_end / 2)
+				return ERR_PTR(-ENOSPC);
+		}
+
 		WARN(i915_vma_is_pinned(vma),
 		     "bo is already pinned in ggtt with incorrect alignment:"
 		     " offset=%08x, req.alignment=%llx,"
@@ -3869,83 +3729,42 @@ static __always_inline unsigned int __busy_write_id(unsigned int id)
 }
 
 static __always_inline unsigned int
-__busy_set_if_active(const struct i915_gem_active *active,
+__busy_set_if_active(const struct dma_fence *fence,
 		     unsigned int (*flag)(unsigned int id))
 {
-	struct drm_i915_gem_request *request;
+	struct drm_i915_gem_request *rq;
 
-	request = rcu_dereference(active->request);
-	if (!request || i915_gem_request_completed(request))
-		return 0;
-
-	/* This is racy. See __i915_gem_active_get_rcu() for an in detail
-	 * discussion of how to handle the race correctly, but for reporting
-	 * the busy state we err on the side of potentially reporting the
-	 * wrong engine as being busy (but we guarantee that the result
-	 * is at least self-consistent).
-	 *
-	 * As we use SLAB_DESTROY_BY_RCU, the request may be reallocated
-	 * whilst we are inspecting it, even under the RCU read lock as we are.
-	 * This means that there is a small window for the engine and/or the
-	 * seqno to have been overwritten. The seqno will always be in the
-	 * future compared to the intended, and so we know that if that
-	 * seqno is idle (on whatever engine) our request is idle and the
-	 * return 0 above is correct.
-	 *
-	 * The issue is that if the engine is switched, it is just as likely
-	 * to report that it is busy (but since the switch happened, we know
-	 * the request should be idle). So there is a small chance that a busy
-	 * result is actually the wrong engine.
-	 *
-	 * So why don't we care?
+	/* We have to check the current hw status of the fence as the uABI
+	 * guarantees forward progress. We could rely on the idle worker
+	 * to eventually flush us, but to minimise latency just ask the
+	 * hardware.
 	 *
-	 * For starters, the busy ioctl is a heuristic that is by definition
-	 * racy. Even with perfect serialisation in the driver, the hardware
-	 * state is constantly advancing - the state we report to the user
-	 * is stale.
-	 *
-	 * The critical information for the busy-ioctl is whether the object
-	 * is idle as userspace relies on that to detect whether its next
-	 * access will stall, or if it has missed submitting commands to
-	 * the hardware allowing the GPU to stall. We never generate a
-	 * false-positive for idleness, thus busy-ioctl is reliable at the
-	 * most fundamental level, and we maintain the guarantee that a
-	 * busy object left to itself will eventually become idle (and stay
-	 * idle!).
-	 *
-	 * We allow ourselves the leeway of potentially misreporting the busy
-	 * state because that is an optimisation heuristic that is constantly
-	 * in flux. Being quickly able to detect the busy/idle state is much
-	 * more important than accurate logging of exactly which engines were
-	 * busy.
-	 *
-	 * For accuracy in reporting the engine, we could use
-	 *
-	 *	result = 0;
-	 *	request = __i915_gem_active_get_rcu(active);
-	 *	if (request) {
-	 *		if (!i915_gem_request_completed(request))
-	 *			result = flag(request->engine->exec_id);
-	 *		i915_gem_request_put(request);
-	 *	}
-	 *
-	 * but that still remains susceptible to both hardware and userspace
-	 * races. So we accept making the result of that race slightly worse,
-	 * given the rarity of the race and its low impact on the result.
+	 * Note we only report on the status of native fences.
 	 */
-	return flag(READ_ONCE(request->engine->exec_id));
+	if (!dma_fence_is_i915(fence))
+		return 0;
+
+	/* opencode to_request() in order to avoid const warnings */
+	rq = container_of(fence, struct drm_i915_gem_request, fence);
+	if (i915_gem_request_completed(rq))
+		return 0;
+
+	return flag(rq->engine->exec_id);
 }
 
 static __always_inline unsigned int
-busy_check_reader(const struct i915_gem_active *active)
+busy_check_reader(const struct dma_fence *fence)
 {
-	return __busy_set_if_active(active, __busy_read_flag);
+	return __busy_set_if_active(fence, __busy_read_flag);
 }
 
 static __always_inline unsigned int
-busy_check_writer(const struct i915_gem_active *active)
+busy_check_writer(const struct dma_fence *fence)
 {
-	return __busy_set_if_active(active, __busy_write_id);
+	if (!fence)
+		return 0;
+
+	return __busy_set_if_active(fence, __busy_write_id);
 }
 
 int
@@ -3954,64 +3773,58 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_i915_gem_busy *args = data;
 	struct drm_i915_gem_object *obj;
-	unsigned long active;
+	struct reservation_object_list *list;
+	unsigned int seq;
+	int err;
 
-	obj = i915_gem_object_lookup(file, args->handle);
+	err = -ENOENT;
+	rcu_read_lock();
+	obj = i915_gem_object_lookup_rcu(file, args->handle);
 	if (!obj)
-		return -ENOENT;
+		goto out;
 
-	args->busy = 0;
-	active = __I915_BO_ACTIVE(obj);
-	if (active) {
-		int idx;
+	/* A discrepancy here is that we do not report the status of
+	 * non-i915 fences, i.e. even though we may report the object as idle,
+	 * a call to set-domain may still stall waiting for foreign rendering.
+	 * This also means that wait-ioctl may report an object as busy,
+	 * where busy-ioctl considers it idle.
+	 *
+	 * We trade the ability to warn of foreign fences to report on which
+	 * i915 engines are active for the object.
+	 *
+	 * Alternatively, we can trade that extra information on read/write
+	 * activity with
+	 *	args->busy =
+	 *		!reservation_object_test_signaled_rcu(obj->resv, true);
+	 * to report the overall busyness. This is what the wait-ioctl does.
+	 *
+	 */
+retry:
+	seq = raw_read_seqcount(&obj->resv->seq);
 
-		/* Yes, the lookups are intentionally racy.
-		 *
-		 * First, we cannot simply rely on __I915_BO_ACTIVE. We have
-		 * to regard the value as stale and as our ABI guarantees
-		 * forward progress, we confirm the status of each active
-		 * request with the hardware.
-		 *
-		 * Even though we guard the pointer lookup by RCU, that only
-		 * guarantees that the pointer and its contents remain
-		 * dereferencable and does *not* mean that the request we
-		 * have is the same as the one being tracked by the object.
-		 *
-		 * Consider that we lookup the request just as it is being
-		 * retired and freed. We take a local copy of the pointer,
-		 * but before we add its engine into the busy set, the other
-		 * thread reallocates it and assigns it to a task on another
-		 * engine with a fresh and incomplete seqno. Guarding against
-		 * that requires careful serialisation and reference counting,
-		 * i.e. using __i915_gem_active_get_request_rcu(). We don't,
-		 * instead we expect that if the result is busy, which engines
-		 * are busy is not completely reliable - we only guarantee
-		 * that the object was busy.
-		 */
-		rcu_read_lock();
+	/* Translate the exclusive fence to the READ *and* WRITE engine */
+	args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl));
 
-		for_each_active(active, idx)
-			args->busy |= busy_check_reader(&obj->last_read[idx]);
+	/* Translate shared fences to READ set of engines */
+	list = rcu_dereference(obj->resv->fence);
+	if (list) {
+		unsigned int shared_count = list->shared_count, i;
 
-		/* For ABI sanity, we only care that the write engine is in
-		 * the set of read engines. This should be ensured by the
-		 * ordering of setting last_read/last_write in
-		 * i915_vma_move_to_active(), and then in reverse in retire.
-		 * However, for good measure, we always report the last_write
-		 * request as a busy read as well as being a busy write.
-		 *
-		 * We don't care that the set of active read/write engines
-		 * may change during construction of the result, as it is
-		 * equally liable to change before userspace can inspect
-		 * the result.
-		 */
-		args->busy |= busy_check_writer(&obj->last_write);
+		for (i = 0; i < shared_count; ++i) {
+			struct dma_fence *fence =
+				rcu_dereference(list->shared[i]);
 
-		rcu_read_unlock();
+			args->busy |= busy_check_reader(fence);
+		}
 	}
 
-	i915_gem_object_put_unlocked(obj);
-	return 0;
+	if (args->busy && read_seqcount_retry(&obj->resv->seq, seq))
+		goto retry;
+
+	err = 0;
+out:
+	rcu_read_unlock();
+	return err;
 }
 
 int
@@ -4028,7 +3841,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_madvise *args = data;
 	struct drm_i915_gem_object *obj;
-	int ret;
+	int err;
 
 	switch (args->madv) {
 	case I915_MADV_DONTNEED:
@@ -4038,77 +3851,111 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 	    return -EINVAL;
 	}
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
-
 	obj = i915_gem_object_lookup(file_priv, args->handle);
-	if (!obj) {
-		ret = -ENOENT;
-		goto unlock;
-	}
+	if (!obj)
+		return -ENOENT;
 
-	if (obj->pages &&
+	err = mutex_lock_interruptible(&obj->mm.lock);
+	if (err)
+		goto out;
+
+	if (obj->mm.pages &&
 	    i915_gem_object_is_tiled(obj) &&
 	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
-		if (obj->madv == I915_MADV_WILLNEED)
-			i915_gem_object_unpin_pages(obj);
-		if (args->madv == I915_MADV_WILLNEED)
-			i915_gem_object_pin_pages(obj);
+		if (obj->mm.madv == I915_MADV_WILLNEED) {
+			GEM_BUG_ON(!obj->mm.quirked);
+			__i915_gem_object_unpin_pages(obj);
+			obj->mm.quirked = false;
+		}
+		if (args->madv == I915_MADV_WILLNEED) {
+			GEM_BUG_ON(obj->mm.quirked);
+			__i915_gem_object_pin_pages(obj);
+			obj->mm.quirked = true;
+		}
 	}
 
-	if (obj->madv != __I915_MADV_PURGED)
-		obj->madv = args->madv;
+	if (obj->mm.madv != __I915_MADV_PURGED)
+		obj->mm.madv = args->madv;
 
 	/* if the object is no longer attached, discard its backing storage */
-	if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
+	if (obj->mm.madv == I915_MADV_DONTNEED && !obj->mm.pages)
 		i915_gem_object_truncate(obj);
 
-	args->retained = obj->madv != __I915_MADV_PURGED;
+	args->retained = obj->mm.madv != __I915_MADV_PURGED;
+	mutex_unlock(&obj->mm.lock);
 
+out:
 	i915_gem_object_put(obj);
-unlock:
-	mutex_unlock(&dev->struct_mutex);
-	return ret;
+	return err;
+}
+
+static void
+frontbuffer_retire(struct i915_gem_active *active,
+		   struct drm_i915_gem_request *request)
+{
+	struct drm_i915_gem_object *obj =
+		container_of(active, typeof(*obj), frontbuffer_write);
+
+	intel_fb_obj_flush(obj, true, ORIGIN_CS);
 }
 
 void i915_gem_object_init(struct drm_i915_gem_object *obj,
 			  const struct drm_i915_gem_object_ops *ops)
 {
-	int i;
+	mutex_init(&obj->mm.lock);
 
-	INIT_LIST_HEAD(&obj->global_list);
-	for (i = 0; i < I915_NUM_ENGINES; i++)
-		init_request_active(&obj->last_read[i],
-				    i915_gem_object_retire__read);
-	init_request_active(&obj->last_write,
-			    i915_gem_object_retire__write);
+	INIT_LIST_HEAD(&obj->global_link);
+	INIT_LIST_HEAD(&obj->userfault_link);
 	INIT_LIST_HEAD(&obj->obj_exec_link);
 	INIT_LIST_HEAD(&obj->vma_list);
 	INIT_LIST_HEAD(&obj->batch_pool_link);
 
 	obj->ops = ops;
 
+	reservation_object_init(&obj->__builtin_resv);
+	obj->resv = &obj->__builtin_resv;
+
 	obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
-	obj->madv = I915_MADV_WILLNEED;
+	init_request_active(&obj->frontbuffer_write, frontbuffer_retire);
+
+	obj->mm.madv = I915_MADV_WILLNEED;
+	INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN);
+	mutex_init(&obj->mm.get_page.lock);
 
 	i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
 }
 
 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
-	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
+	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
+		 I915_GEM_OBJECT_IS_SHRINKABLE,
 	.get_pages = i915_gem_object_get_pages_gtt,
 	.put_pages = i915_gem_object_put_pages_gtt,
 };
 
-struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
-						  size_t size)
+/* Note we don't consider signbits :| */
+#define overflows_type(x, T) \
+	(sizeof(x) > sizeof(T) && (x) >> (sizeof(T) * BITS_PER_BYTE))
+
+struct drm_i915_gem_object *
+i915_gem_object_create(struct drm_device *dev, u64 size)
 {
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_object *obj;
 	struct address_space *mapping;
 	gfp_t mask;
 	int ret;
 
+	/* There is a prevalence of the assumption that we fit the object's
+	 * page count inside a 32bit _signed_ variable. Let's document this and
+	 * catch if we ever need to fix it. In the meantime, if you do spot
+	 * such a local variable, please consider fixing!
+	 */
+	if (WARN_ON(size >> PAGE_SHIFT > INT_MAX))
+		return ERR_PTR(-E2BIG);
+
+	if (overflows_type(size, obj->base.size))
+		return ERR_PTR(-E2BIG);
+
 	obj = i915_gem_object_alloc(dev);
 	if (obj == NULL)
 		return ERR_PTR(-ENOMEM);
@@ -4118,7 +3965,7 @@ struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
 		goto fail;
 
 	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
-	if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
+	if (IS_CRESTLINE(dev_priv) || IS_BROADWATER(dev_priv)) {
 		/* 965gm cannot relocate objects above 4GiB. */
 		mask &= ~__GFP_HIGHMEM;
 		mask |= __GFP_DMA32;
@@ -4132,7 +3979,7 @@ struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
 	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 
-	if (HAS_LLC(dev)) {
+	if (HAS_LLC(dev_priv)) {
 		/* On some devices, we can have the GPU use the LLC (the CPU
 		 * cache) for about a 10% performance improvement
 		 * compared to uncached.  Graphics requests other than
@@ -4155,7 +4002,6 @@ struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
 
 fail:
 	i915_gem_object_free(obj);
-
 	return ERR_PTR(ret);
 }
 
@@ -4167,7 +4013,7 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj)
 	 * back the contents from the GPU.
 	 */
 
-	if (obj->madv != I915_MADV_WILLNEED)
+	if (obj->mm.madv != I915_MADV_WILLNEED)
 		return false;
 
 	if (obj->base.filp == NULL)
@@ -4183,16 +4029,72 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj)
 	return atomic_long_read(&obj->base.filp->f_count) == 1;
 }
 
-void i915_gem_free_object(struct drm_gem_object *gem_obj)
+static void __i915_gem_free_objects(struct drm_i915_private *i915,
+				    struct llist_node *freed)
 {
-	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_vma *vma, *next;
+	struct drm_i915_gem_object *obj, *on;
 
-	intel_runtime_pm_get(dev_priv);
+	mutex_lock(&i915->drm.struct_mutex);
+	intel_runtime_pm_get(i915);
+	llist_for_each_entry(obj, freed, freed) {
+		struct i915_vma *vma, *vn;
+
+		trace_i915_gem_object_destroy(obj);
+
+		GEM_BUG_ON(i915_gem_object_is_active(obj));
+		list_for_each_entry_safe(vma, vn,
+					 &obj->vma_list, obj_link) {
+			GEM_BUG_ON(!i915_vma_is_ggtt(vma));
+			GEM_BUG_ON(i915_vma_is_active(vma));
+			vma->flags &= ~I915_VMA_PIN_MASK;
+			i915_vma_close(vma);
+		}
+		GEM_BUG_ON(!list_empty(&obj->vma_list));
+		GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree));
+
+		list_del(&obj->global_link);
+	}
+	intel_runtime_pm_put(i915);
+	mutex_unlock(&i915->drm.struct_mutex);
+
+	llist_for_each_entry_safe(obj, on, freed, freed) {
+		GEM_BUG_ON(obj->bind_count);
+		GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits));
+
+		if (obj->ops->release)
+			obj->ops->release(obj);
+
+		if (WARN_ON(i915_gem_object_has_pinned_pages(obj)))
+			atomic_set(&obj->mm.pages_pin_count, 0);
+		__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
+		GEM_BUG_ON(obj->mm.pages);
+
+		if (obj->base.import_attach)
+			drm_prime_gem_destroy(&obj->base, NULL);
+
+		reservation_object_fini(&obj->__builtin_resv);
+		drm_gem_object_release(&obj->base);
+		i915_gem_info_remove_obj(i915, obj->base.size);
+
+		kfree(obj->bit_17);
+		i915_gem_object_free(obj);
+	}
+}
+
+static void i915_gem_flush_free_objects(struct drm_i915_private *i915)
+{
+	struct llist_node *freed;
+
+	freed = llist_del_all(&i915->mm.free_list);
+	if (unlikely(freed))
+		__i915_gem_free_objects(i915, freed);
+}
 
-	trace_i915_gem_object_destroy(obj);
+static void __i915_gem_free_work(struct work_struct *work)
+{
+	struct drm_i915_private *i915 =
+		container_of(work, struct drm_i915_private, mm.free_work);
+	struct llist_node *freed;
 
 	/* All file-owned VMA should have been released by this point through
 	 * i915_gem_close_object(), or earlier by i915_gem_context_close().
@@ -4201,47 +4103,62 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
 	 * the GTT either for the user or for scanout). Those VMA still need to
 	 * unbound now.
 	 */
-	list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
-		GEM_BUG_ON(!i915_vma_is_ggtt(vma));
-		GEM_BUG_ON(i915_vma_is_active(vma));
-		vma->flags &= ~I915_VMA_PIN_MASK;
-		i915_vma_close(vma);
-	}
-	GEM_BUG_ON(obj->bind_count);
 
-	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
-	 * before progressing. */
-	if (obj->stolen)
-		i915_gem_object_unpin_pages(obj);
+	while ((freed = llist_del_all(&i915->mm.free_list)))
+		__i915_gem_free_objects(i915, freed);
+}
 
-	WARN_ON(atomic_read(&obj->frontbuffer_bits));
+static void __i915_gem_free_object_rcu(struct rcu_head *head)
+{
+	struct drm_i915_gem_object *obj =
+		container_of(head, typeof(*obj), rcu);
+	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 
-	if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
-	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
-	    i915_gem_object_is_tiled(obj))
-		i915_gem_object_unpin_pages(obj);
+	/* We can't simply use call_rcu() from i915_gem_free_object()
+	 * as we need to block whilst unbinding, and the call_rcu
+	 * task may be called from softirq context. So we take a
+	 * detour through a worker.
+	 */
+	if (llist_add(&obj->freed, &i915->mm.free_list))
+		schedule_work(&i915->mm.free_work);
+}
 
-	if (WARN_ON(obj->pages_pin_count))
-		obj->pages_pin_count = 0;
-	if (discard_backing_storage(obj))
-		obj->madv = I915_MADV_DONTNEED;
-	i915_gem_object_put_pages(obj);
+void i915_gem_free_object(struct drm_gem_object *gem_obj)
+{
+	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
 
-	BUG_ON(obj->pages);
+	if (obj->mm.quirked)
+		__i915_gem_object_unpin_pages(obj);
 
-	if (obj->base.import_attach)
-		drm_prime_gem_destroy(&obj->base, NULL);
+	if (discard_backing_storage(obj))
+		obj->mm.madv = I915_MADV_DONTNEED;
+
+	/* Before we free the object, make sure any pure RCU-only
+	 * read-side critical sections are complete, e.g.
+	 * i915_gem_busy_ioctl(). For the corresponding synchronized
+	 * lookup see i915_gem_object_lookup_rcu().
+	 */
+	call_rcu(&obj->rcu, __i915_gem_free_object_rcu);
+}
 
-	if (obj->ops->release)
-		obj->ops->release(obj);
+void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
-	drm_gem_object_release(&obj->base);
-	i915_gem_info_remove_obj(dev_priv, obj->base.size);
+	GEM_BUG_ON(i915_gem_object_has_active_reference(obj));
+	if (i915_gem_object_is_active(obj))
+		i915_gem_object_set_active_reference(obj);
+	else
+		i915_gem_object_put(obj);
+}
 
-	kfree(obj->bit_17);
-	i915_gem_object_free(obj);
+static void assert_kernel_context_is_current(struct drm_i915_private *dev_priv)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 
-	intel_runtime_pm_put(dev_priv);
+	for_each_engine(engine, dev_priv, id)
+		GEM_BUG_ON(engine->last_context != dev_priv->kernel_context);
 }
 
 int i915_gem_suspend(struct drm_device *dev)
@@ -4272,18 +4189,46 @@ int i915_gem_suspend(struct drm_device *dev)
 		goto err;
 
 	i915_gem_retire_requests(dev_priv);
+	GEM_BUG_ON(dev_priv->gt.active_requests);
 
+	assert_kernel_context_is_current(dev_priv);
 	i915_gem_context_lost(dev_priv);
 	mutex_unlock(&dev->struct_mutex);
 
 	cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
 	cancel_delayed_work_sync(&dev_priv->gt.retire_work);
 	flush_delayed_work(&dev_priv->gt.idle_work);
+	flush_work(&dev_priv->mm.free_work);
 
 	/* Assert that we sucessfully flushed all the work and
 	 * reset the GPU back to its idle, low power state.
 	 */
 	WARN_ON(dev_priv->gt.awake);
+	WARN_ON(!intel_execlists_idle(dev_priv));
+
+	/*
+	 * Neither the BIOS, ourselves or any other kernel
+	 * expects the system to be in execlists mode on startup,
+	 * so we need to reset the GPU back to legacy mode. And the only
+	 * known way to disable logical contexts is through a GPU reset.
+	 *
+	 * So in order to leave the system in a known default configuration,
+	 * always reset the GPU upon unload and suspend. Afterwards we then
+	 * clean up the GEM state tracking, flushing off the requests and
+	 * leaving the system in a known idle state.
+	 *
+	 * Note that is of the upmost importance that the GPU is idle and
+	 * all stray writes are flushed *before* we dismantle the backing
+	 * storage for the pinned objects.
+	 *
+	 * However, since we are uncertain that resetting the GPU on older
+	 * machines is a good idea, we don't - just in case it leaves the
+	 * machine in an unusable condition.
+	 */
+	if (HAS_HW_CONTEXTS(dev_priv)) {
+		int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
+		WARN_ON(reset && reset != -ENODEV);
+	}
 
 	return 0;
 
@@ -4296,8 +4241,10 @@ void i915_gem_resume(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
+	WARN_ON(dev_priv->gt.awake);
+
 	mutex_lock(&dev->struct_mutex);
-	i915_gem_restore_gtt_mappings(dev);
+	i915_gem_restore_gtt_mappings(dev_priv);
 
 	/* As we didn't flush the kernel context before suspend, we cannot
 	 * guarantee that the context image is complete. So let's just reset
@@ -4308,55 +4255,51 @@ void i915_gem_resume(struct drm_device *dev)
 	mutex_unlock(&dev->struct_mutex);
 }
 
-void i915_gem_init_swizzling(struct drm_device *dev)
+void i915_gem_init_swizzling(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	if (INTEL_INFO(dev)->gen < 5 ||
+	if (INTEL_GEN(dev_priv) < 5 ||
 	    dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
 		return;
 
 	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
 				 DISP_TILE_SURFACE_SWIZZLING);
 
-	if (IS_GEN5(dev))
+	if (IS_GEN5(dev_priv))
 		return;
 
 	I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
-	if (IS_GEN6(dev))
+	if (IS_GEN6(dev_priv))
 		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
-	else if (IS_GEN7(dev))
+	else if (IS_GEN7(dev_priv))
 		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
-	else if (IS_GEN8(dev))
+	else if (IS_GEN8(dev_priv))
 		I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
 	else
 		BUG();
 }
 
-static void init_unused_ring(struct drm_device *dev, u32 base)
+static void init_unused_ring(struct drm_i915_private *dev_priv, u32 base)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	I915_WRITE(RING_CTL(base), 0);
 	I915_WRITE(RING_HEAD(base), 0);
 	I915_WRITE(RING_TAIL(base), 0);
 	I915_WRITE(RING_START(base), 0);
 }
 
-static void init_unused_rings(struct drm_device *dev)
-{
-	if (IS_I830(dev)) {
-		init_unused_ring(dev, PRB1_BASE);
-		init_unused_ring(dev, SRB0_BASE);
-		init_unused_ring(dev, SRB1_BASE);
-		init_unused_ring(dev, SRB2_BASE);
-		init_unused_ring(dev, SRB3_BASE);
-	} else if (IS_GEN2(dev)) {
-		init_unused_ring(dev, SRB0_BASE);
-		init_unused_ring(dev, SRB1_BASE);
-	} else if (IS_GEN3(dev)) {
-		init_unused_ring(dev, PRB1_BASE);
-		init_unused_ring(dev, PRB2_BASE);
+static void init_unused_rings(struct drm_i915_private *dev_priv)
+{
+	if (IS_I830(dev_priv)) {
+		init_unused_ring(dev_priv, PRB1_BASE);
+		init_unused_ring(dev_priv, SRB0_BASE);
+		init_unused_ring(dev_priv, SRB1_BASE);
+		init_unused_ring(dev_priv, SRB2_BASE);
+		init_unused_ring(dev_priv, SRB3_BASE);
+	} else if (IS_GEN2(dev_priv)) {
+		init_unused_ring(dev_priv, SRB0_BASE);
+		init_unused_ring(dev_priv, SRB1_BASE);
+	} else if (IS_GEN3(dev_priv)) {
+		init_unused_ring(dev_priv, PRB1_BASE);
+		init_unused_ring(dev_priv, PRB2_BASE);
 	}
 }
 
@@ -4365,31 +4308,34 @@ i915_gem_init_hw(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	int ret;
 
+	dev_priv->gt.last_init_time = ktime_get();
+
 	/* Double layer security blanket, see i915_gem_init() */
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
-	if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
+	if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9)
 		I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
 
-	if (IS_HASWELL(dev))
-		I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
+	if (IS_HASWELL(dev_priv))
+		I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev_priv) ?
 			   LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
 
-	if (HAS_PCH_NOP(dev)) {
-		if (IS_IVYBRIDGE(dev)) {
+	if (HAS_PCH_NOP(dev_priv)) {
+		if (IS_IVYBRIDGE(dev_priv)) {
 			u32 temp = I915_READ(GEN7_MSG_CTL);
 			temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
 			I915_WRITE(GEN7_MSG_CTL, temp);
-		} else if (INTEL_INFO(dev)->gen >= 7) {
+		} else if (INTEL_GEN(dev_priv) >= 7) {
 			u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
 			temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
 			I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
 		}
 	}
 
-	i915_gem_init_swizzling(dev);
+	i915_gem_init_swizzling(dev_priv);
 
 	/*
 	 * At least 830 can leave some of the unused rings
@@ -4397,18 +4343,18 @@ i915_gem_init_hw(struct drm_device *dev)
 	 * will prevent c3 entry. Makes sure all unused rings
 	 * are totally idle.
 	 */
-	init_unused_rings(dev);
+	init_unused_rings(dev_priv);
 
 	BUG_ON(!dev_priv->kernel_context);
 
-	ret = i915_ppgtt_init_hw(dev);
+	ret = i915_ppgtt_init_hw(dev_priv);
 	if (ret) {
 		DRM_ERROR("PPGTT enable HW failed %d\n", ret);
 		goto out;
 	}
 
 	/* Need to do basic initialisation of all rings first: */
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		ret = engine->init_hw(engine);
 		if (ret)
 			goto out;
@@ -4507,21 +4453,15 @@ i915_gem_cleanup_engines(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		dev_priv->gt.cleanup_engine(engine);
 }
 
-static void
-init_engine_lists(struct intel_engine_cs *engine)
-{
-	INIT_LIST_HEAD(&engine->request_list);
-}
-
 void
 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	int i;
 
 	if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
@@ -4545,41 +4485,52 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
 		fence->id = i;
 		list_add_tail(&fence->link, &dev_priv->mm.fence_list);
 	}
-	i915_gem_restore_fences(dev);
+	i915_gem_restore_fences(dev_priv);
 
-	i915_gem_detect_bit_6_swizzle(dev);
+	i915_gem_detect_bit_6_swizzle(dev_priv);
 }
 
-void
+int
 i915_gem_load_init(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	int i;
+	int err = -ENOMEM;
+
+	dev_priv->objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);
+	if (!dev_priv->objects)
+		goto err_out;
+
+	dev_priv->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
+	if (!dev_priv->vmas)
+		goto err_objects;
+
+	dev_priv->requests = KMEM_CACHE(drm_i915_gem_request,
+					SLAB_HWCACHE_ALIGN |
+					SLAB_RECLAIM_ACCOUNT |
+					SLAB_DESTROY_BY_RCU);
+	if (!dev_priv->requests)
+		goto err_vmas;
 
-	dev_priv->objects =
-		kmem_cache_create("i915_gem_object",
-				  sizeof(struct drm_i915_gem_object), 0,
-				  SLAB_HWCACHE_ALIGN,
-				  NULL);
-	dev_priv->vmas =
-		kmem_cache_create("i915_gem_vma",
-				  sizeof(struct i915_vma), 0,
-				  SLAB_HWCACHE_ALIGN,
-				  NULL);
-	dev_priv->requests =
-		kmem_cache_create("i915_gem_request",
-				  sizeof(struct drm_i915_gem_request), 0,
-				  SLAB_HWCACHE_ALIGN |
-				  SLAB_RECLAIM_ACCOUNT |
-				  SLAB_DESTROY_BY_RCU,
-				  NULL);
+	dev_priv->dependencies = KMEM_CACHE(i915_dependency,
+					    SLAB_HWCACHE_ALIGN |
+					    SLAB_RECLAIM_ACCOUNT);
+	if (!dev_priv->dependencies)
+		goto err_requests;
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	INIT_LIST_HEAD(&dev_priv->gt.timelines);
+	err = i915_gem_timeline_init__global(dev_priv);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+	if (err)
+		goto err_dependencies;
 
 	INIT_LIST_HEAD(&dev_priv->context_list);
+	INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work);
+	init_llist_head(&dev_priv->mm.free_list);
 	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
 	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
 	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
-	for (i = 0; i < I915_NUM_ENGINES; i++)
-		init_engine_lists(&dev_priv->engine[i]);
+	INIT_LIST_HEAD(&dev_priv->mm.userfault_list);
 	INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
 			  i915_gem_retire_work_handler);
 	INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
@@ -4596,12 +4547,33 @@ i915_gem_load_init(struct drm_device *dev)
 	atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);
 
 	spin_lock_init(&dev_priv->fb_tracking.lock);
+
+	return 0;
+
+err_dependencies:
+	kmem_cache_destroy(dev_priv->dependencies);
+err_requests:
+	kmem_cache_destroy(dev_priv->requests);
+err_vmas:
+	kmem_cache_destroy(dev_priv->vmas);
+err_objects:
+	kmem_cache_destroy(dev_priv->objects);
+err_out:
+	return err;
 }
 
 void i915_gem_load_cleanup(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
+	WARN_ON(!llist_empty(&dev_priv->mm.free_list));
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	i915_gem_timeline_fini(&dev_priv->gt.global_timeline);
+	WARN_ON(!list_empty(&dev_priv->gt.timelines));
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+
+	kmem_cache_destroy(dev_priv->dependencies);
 	kmem_cache_destroy(dev_priv->requests);
 	kmem_cache_destroy(dev_priv->vmas);
 	kmem_cache_destroy(dev_priv->objects);
@@ -4650,7 +4622,7 @@ int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
 	i915_gem_shrink(dev_priv, -1UL, I915_SHRINK_UNBOUND);
 
 	for (p = phases; *p; p++) {
-		list_for_each_entry(obj, *p, global_list) {
+		list_for_each_entry(obj, *p, global_link) {
 			obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 			obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 		}
@@ -4686,7 +4658,7 @@ int i915_gem_open(struct drm_device *dev, struct drm_file *file)
 	struct drm_i915_file_private *file_priv;
 	int ret;
 
-	DRM_DEBUG_DRIVER("\n");
+	DRM_DEBUG("\n");
 
 	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
 	if (!file_priv)
@@ -4742,21 +4714,6 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
 	}
 }
 
-/* Like i915_gem_object_get_page(), but mark the returned page dirty */
-struct page *
-i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
-{
-	struct page *page;
-
-	/* Only default objects have per-page dirty tracking */
-	if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
-		return NULL;
-
-	page = i915_gem_object_get_page(obj, n);
-	set_page_dirty(page);
-	return page;
-}
-
 /* Allocate a new GEM object and fill it with the supplied data */
 struct drm_i915_gem_object *
 i915_gem_object_create_from_data(struct drm_device *dev,
@@ -4775,14 +4732,13 @@ i915_gem_object_create_from_data(struct drm_device *dev,
 	if (ret)
 		goto fail;
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
 		goto fail;
 
-	i915_gem_object_pin_pages(obj);
-	sg = obj->pages;
+	sg = obj->mm.pages;
 	bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
-	obj->dirty = 1;		/* Backing store is now out of date */
+	obj->mm.dirty = true; /* Backing store is now out of date */
 	i915_gem_object_unpin_pages(obj);
 
 	if (WARN_ON(bytes != size)) {
@@ -4797,3 +4753,156 @@ fail:
 	i915_gem_object_put(obj);
 	return ERR_PTR(ret);
 }
+
+struct scatterlist *
+i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
+		       unsigned int n,
+		       unsigned int *offset)
+{
+	struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
+	struct scatterlist *sg;
+	unsigned int idx, count;
+
+	might_sleep();
+	GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
+	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
+
+	/* As we iterate forward through the sg, we record each entry in a
+	 * radixtree for quick repeated (backwards) lookups. If we have seen
+	 * this index previously, we will have an entry for it.
+	 *
+	 * Initial lookup is O(N), but this is amortized to O(1) for
+	 * sequential page access (where each new request is consecutive
+	 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
+	 * i.e. O(1) with a large constant!
+	 */
+	if (n < READ_ONCE(iter->sg_idx))
+		goto lookup;
+
+	mutex_lock(&iter->lock);
+
+	/* We prefer to reuse the last sg so that repeated lookup of this
+	 * (or the subsequent) sg are fast - comparing against the last
+	 * sg is faster than going through the radixtree.
+	 */
+
+	sg = iter->sg_pos;
+	idx = iter->sg_idx;
+	count = __sg_page_count(sg);
+
+	while (idx + count <= n) {
+		unsigned long exception, i;
+		int ret;
+
+		/* If we cannot allocate and insert this entry, or the
+		 * individual pages from this range, cancel updating the
+		 * sg_idx so that on this lookup we are forced to linearly
+		 * scan onwards, but on future lookups we will try the
+		 * insertion again (in which case we need to be careful of
+		 * the error return reporting that we have already inserted
+		 * this index).
+		 */
+		ret = radix_tree_insert(&iter->radix, idx, sg);
+		if (ret && ret != -EEXIST)
+			goto scan;
+
+		exception =
+			RADIX_TREE_EXCEPTIONAL_ENTRY |
+			idx << RADIX_TREE_EXCEPTIONAL_SHIFT;
+		for (i = 1; i < count; i++) {
+			ret = radix_tree_insert(&iter->radix, idx + i,
+						(void *)exception);
+			if (ret && ret != -EEXIST)
+				goto scan;
+		}
+
+		idx += count;
+		sg = ____sg_next(sg);
+		count = __sg_page_count(sg);
+	}
+
+scan:
+	iter->sg_pos = sg;
+	iter->sg_idx = idx;
+
+	mutex_unlock(&iter->lock);
+
+	if (unlikely(n < idx)) /* insertion completed by another thread */
+		goto lookup;
+
+	/* In case we failed to insert the entry into the radixtree, we need
+	 * to look beyond the current sg.
+	 */
+	while (idx + count <= n) {
+		idx += count;
+		sg = ____sg_next(sg);
+		count = __sg_page_count(sg);
+	}
+
+	*offset = n - idx;
+	return sg;
+
+lookup:
+	rcu_read_lock();
+
+	sg = radix_tree_lookup(&iter->radix, n);
+	GEM_BUG_ON(!sg);
+
+	/* If this index is in the middle of multi-page sg entry,
+	 * the radixtree will contain an exceptional entry that points
+	 * to the start of that range. We will return the pointer to
+	 * the base page and the offset of this page within the
+	 * sg entry's range.
+	 */
+	*offset = 0;
+	if (unlikely(radix_tree_exception(sg))) {
+		unsigned long base =
+			(unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT;
+
+		sg = radix_tree_lookup(&iter->radix, base);
+		GEM_BUG_ON(!sg);
+
+		*offset = n - base;
+	}
+
+	rcu_read_unlock();
+
+	return sg;
+}
+
+struct page *
+i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
+{
+	struct scatterlist *sg;
+	unsigned int offset;
+
+	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
+
+	sg = i915_gem_object_get_sg(obj, n, &offset);
+	return nth_page(sg_page(sg), offset);
+}
+
+/* Like i915_gem_object_get_page(), but mark the returned page dirty */
+struct page *
+i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
+			       unsigned int n)
+{
+	struct page *page;
+
+	page = i915_gem_object_get_page(obj, n);
+	if (!obj->mm.dirty)
+		set_page_dirty(page);
+
+	return page;
+}
+
+dma_addr_t
+i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
+				unsigned long n)
+{
+	struct scatterlist *sg;
+	unsigned int offset;
+
+	sg = i915_gem_object_get_sg(obj, n, &offset);
+	return sg_dma_address(sg) + (offset << PAGE_SHIFT);
+}
diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h
index 8292e797d9b5..51ec793f2e20 100644
--- a/drivers/gpu/drm/i915/i915_gem.h
+++ b/drivers/gpu/drm/i915/i915_gem.h
@@ -28,7 +28,9 @@
 #ifdef CONFIG_DRM_I915_DEBUG_GEM
 #define GEM_BUG_ON(expr) BUG_ON(expr)
 #else
-#define GEM_BUG_ON(expr)
+#define GEM_BUG_ON(expr) do { } while (0)
 #endif
 
+#define I915_NUM_ENGINES 5
+
 #endif /* __I915_GEM_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
index ed989596d9a3..b3bc119ec1bb 100644
--- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c
+++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
@@ -73,7 +73,7 @@ void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool)
 		list_for_each_entry_safe(obj, next,
 					 &pool->cache_list[n],
 					 batch_pool_link)
-			i915_gem_object_put(obj);
+			__i915_gem_object_release_unless_active(obj);
 
 		INIT_LIST_HEAD(&pool->cache_list[n]);
 	}
@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 			size_t size)
 {
 	struct drm_i915_gem_object *obj = NULL;
-	struct drm_i915_gem_object *tmp, *next;
+	struct drm_i915_gem_object *tmp;
 	struct list_head *list;
-	int n;
+	int n, ret;
 
 	lockdep_assert_held(&pool->engine->i915->drm.struct_mutex);
 
@@ -112,40 +112,35 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 		n = ARRAY_SIZE(pool->cache_list) - 1;
 	list = &pool->cache_list[n];
 
-	list_for_each_entry_safe(tmp, next, list, batch_pool_link) {
+	list_for_each_entry(tmp, list, batch_pool_link) {
 		/* The batches are strictly LRU ordered */
-		if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id],
-					     &tmp->base.dev->struct_mutex))
+		if (i915_gem_object_is_active(tmp))
 			break;
 
-		/* While we're looping, do some clean up */
-		if (tmp->madv == __I915_MADV_PURGED) {
-			list_del(&tmp->batch_pool_link);
-			i915_gem_object_put(tmp);
-			continue;
-		}
+		GEM_BUG_ON(!reservation_object_test_signaled_rcu(tmp->resv,
+								 true));
 
 		if (tmp->base.size >= size) {
+			/* Clear the set of shared fences early */
+			ww_mutex_lock(&tmp->resv->lock, NULL);
+			reservation_object_add_excl_fence(tmp->resv, NULL);
+			ww_mutex_unlock(&tmp->resv->lock);
+
 			obj = tmp;
 			break;
 		}
 	}
 
 	if (obj == NULL) {
-		int ret;
-
-		obj = i915_gem_object_create(&pool->engine->i915->drm, size);
+		obj = i915_gem_object_create_internal(pool->engine->i915, size);
 		if (IS_ERR(obj))
 			return obj;
-
-		ret = i915_gem_object_get_pages(obj);
-		if (ret)
-			return ERR_PTR(ret);
-
-		obj->madv = I915_MADV_DONTNEED;
 	}
 
+	ret = i915_gem_object_pin_pages(obj);
+	if (ret)
+		return ERR_PTR(ret);
+
 	list_move_tail(&obj->batch_pool_link, list);
-	i915_gem_object_pin_pages(obj);
 	return obj;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index df10f4e95736..1f94b8d6d83d 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -155,9 +155,10 @@ void i915_gem_context_free(struct kref *ctx_ref)
 		if (ce->ring)
 			intel_ring_free(ce->ring);
 
-		i915_vma_put(ce->state);
+		__i915_gem_object_release_unless_active(ce->state->obj);
 	}
 
+	kfree(ctx->name);
 	put_pid(ctx->pid);
 	list_del(&ctx->link);
 
@@ -192,7 +193,7 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
 	 * This is only applicable for Ivy Bridge devices since
 	 * later platforms don't have L3 control bits in the PTE.
 	 */
-	if (IS_IVYBRIDGE(dev)) {
+	if (IS_IVYBRIDGE(to_i915(dev))) {
 		ret = i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC);
 		/* Failure shouldn't ever happen this early */
 		if (WARN_ON(ret)) {
@@ -303,19 +304,28 @@ __create_hw_context(struct drm_device *dev,
 	}
 
 	/* Default context will never have a file_priv */
-	if (file_priv != NULL) {
+	ret = DEFAULT_CONTEXT_HANDLE;
+	if (file_priv) {
 		ret = idr_alloc(&file_priv->context_idr, ctx,
 				DEFAULT_CONTEXT_HANDLE, 0, GFP_KERNEL);
 		if (ret < 0)
 			goto err_out;
-	} else
-		ret = DEFAULT_CONTEXT_HANDLE;
+	}
+	ctx->user_handle = ret;
 
 	ctx->file_priv = file_priv;
-	if (file_priv)
+	if (file_priv) {
 		ctx->pid = get_task_pid(current, PIDTYPE_PID);
+		ctx->name = kasprintf(GFP_KERNEL, "%s[%d]/%x",
+				      current->comm,
+				      pid_nr(ctx->pid),
+				      ctx->user_handle);
+		if (!ctx->name) {
+			ret = -ENOMEM;
+			goto err_pid;
+		}
+	}
 
-	ctx->user_handle = ret;
 	/* NB: Mark all slices as needing a remap so that when the context first
 	 * loads it will restore whatever remap state already exists. If there
 	 * is no remap info, it will be a NOP. */
@@ -329,6 +339,9 @@ __create_hw_context(struct drm_device *dev,
 
 	return ctx;
 
+err_pid:
+	put_pid(ctx->pid);
+	idr_remove(&file_priv->context_idr, ctx->user_handle);
 err_out:
 	context_close(ctx);
 	return ERR_PTR(ret);
@@ -352,9 +365,9 @@ i915_gem_create_context(struct drm_device *dev,
 		return ctx;
 
 	if (USES_FULL_PPGTT(dev)) {
-		struct i915_hw_ppgtt *ppgtt =
-			i915_ppgtt_create(to_i915(dev), file_priv);
+		struct i915_hw_ppgtt *ppgtt;
 
+		ppgtt = i915_ppgtt_create(to_i915(dev), file_priv, ctx->name);
 		if (IS_ERR(ppgtt)) {
 			DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n",
 					 PTR_ERR(ppgtt));
@@ -463,6 +476,7 @@ int i915_gem_context_init(struct drm_device *dev)
 		return PTR_ERR(ctx);
 	}
 
+	ctx->priority = I915_PRIORITY_MIN; /* lowest priority; idle task */
 	dev_priv->kernel_context = ctx;
 
 	DRM_DEBUG_DRIVER("%s context support initialized\n",
@@ -474,10 +488,11 @@ int i915_gem_context_init(struct drm_device *dev)
 void i915_gem_context_lost(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		if (engine->last_context) {
 			i915_gem_context_unpin(engine->last_context, engine);
 			engine->last_context = NULL;
@@ -492,13 +507,13 @@ void i915_gem_context_lost(struct drm_i915_private *dev_priv)
 			if (!i915_gem_context_is_default(ctx))
 				continue;
 
-			for_each_engine(engine, dev_priv)
+			for_each_engine(engine, dev_priv, id)
 				ctx->engine[engine->id].initialised = false;
 
 			ctx->remap_slice = ALL_L3_SLICES(dev_priv);
 		}
 
-		for_each_engine(engine, dev_priv) {
+		for_each_engine(engine, dev_priv, id) {
 			struct intel_context *kce =
 				&dev_priv->kernel_context->engine[engine->id];
 
@@ -563,6 +578,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
 	struct drm_i915_private *dev_priv = req->i915;
 	struct intel_ring *ring = req->ring;
 	struct intel_engine_cs *engine = req->engine;
+	enum intel_engine_id id;
 	u32 flags = hw_flags | MI_MM_SPACE_GTT;
 	const int num_rings =
 		/* Use an extended w/a on ivb+ if signalling from other rings */
@@ -605,7 +621,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
 
 			intel_ring_emit(ring,
 					MI_LOAD_REGISTER_IMM(num_rings));
-			for_each_engine(signaller, dev_priv) {
+			for_each_engine(signaller, dev_priv, id) {
 				if (signaller == engine)
 					continue;
 
@@ -634,7 +650,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
 
 			intel_ring_emit(ring,
 					MI_LOAD_REGISTER_IMM(num_rings));
-			for_each_engine(signaller, dev_priv) {
+			for_each_engine(signaller, dev_priv, id) {
 				if (signaller == engine)
 					continue;
 
@@ -749,12 +765,36 @@ needs_pd_load_post(struct i915_hw_ppgtt *ppgtt,
 	return false;
 }
 
+struct i915_vma *
+i915_gem_context_pin_legacy(struct i915_gem_context *ctx,
+			    unsigned int flags)
+{
+	struct i915_vma *vma = ctx->engine[RCS].state;
+	int ret;
+
+	/* Clear this page out of any CPU caches for coherent swap-in/out.
+	 * We only want to do this on the first bind so that we do not stall
+	 * on an active context (which by nature is already on the GPU).
+	 */
+	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
+		ret = i915_gem_object_set_to_gtt_domain(vma->obj, false);
+		if (ret)
+			return ERR_PTR(ret);
+	}
+
+	ret = i915_vma_pin(vma, 0, ctx->ggtt_alignment, PIN_GLOBAL | flags);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return vma;
+}
+
 static int do_rcs_switch(struct drm_i915_gem_request *req)
 {
 	struct i915_gem_context *to = req->ctx;
 	struct intel_engine_cs *engine = req->engine;
 	struct i915_hw_ppgtt *ppgtt = to->ppgtt ?: req->i915->mm.aliasing_ppgtt;
-	struct i915_vma *vma = to->engine[RCS].state;
+	struct i915_vma *vma;
 	struct i915_gem_context *from;
 	u32 hw_flags;
 	int ret, i;
@@ -762,17 +802,10 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
 	if (skip_rcs_switch(ppgtt, engine, to))
 		return 0;
 
-	/* Clear this page out of any CPU caches for coherent swap-in/out. */
-	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
-		ret = i915_gem_object_set_to_gtt_domain(vma->obj, false);
-		if (ret)
-			return ret;
-	}
-
 	/* Trying to pin first makes error handling easier. */
-	ret = i915_vma_pin(vma, 0, to->ggtt_alignment, PIN_GLOBAL);
-	if (ret)
-		return ret;
+	vma = i915_gem_context_pin_legacy(to, 0);
+	if (IS_ERR(vma))
+		return PTR_ERR(vma);
 
 	/*
 	 * Pin can switch back to the default context if we end up calling into
@@ -929,21 +962,33 @@ int i915_switch_context(struct drm_i915_gem_request *req)
 int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	struct i915_gem_timeline *timeline;
+	enum intel_engine_id id;
+
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		struct drm_i915_gem_request *req;
 		int ret;
 
-		if (engine->last_context == NULL)
-			continue;
-
-		if (engine->last_context == dev_priv->kernel_context)
-			continue;
-
 		req = i915_gem_request_alloc(engine, dev_priv->kernel_context);
 		if (IS_ERR(req))
 			return PTR_ERR(req);
 
+		/* Queue this switch after all other activity */
+		list_for_each_entry(timeline, &dev_priv->gt.timelines, link) {
+			struct drm_i915_gem_request *prev;
+			struct intel_timeline *tl;
+
+			tl = &timeline->engine[engine->id];
+			prev = i915_gem_active_raw(&tl->last_request,
+						   &dev_priv->drm.struct_mutex);
+			if (prev)
+				i915_sw_fence_await_sw_fence_gfp(&req->submit,
+								 &prev->submit,
+								 GFP_KERNEL);
+		}
+
 		ret = i915_switch_context(req);
 		i915_add_request_no_flush(req);
 		if (ret)
diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/i915_gem_dmabuf.c
index 97c9d68b45df..5e38299b5df6 100644
--- a/drivers/gpu/drm/i915/i915_gem_dmabuf.c
+++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.c
@@ -44,51 +44,42 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme
 	struct scatterlist *src, *dst;
 	int ret, i;
 
-	ret = i915_mutex_lock_interruptible(obj->base.dev);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
 		goto err;
 
-	ret = i915_gem_object_get_pages(obj);
-	if (ret)
-		goto err_unlock;
-
-	i915_gem_object_pin_pages(obj);
-
 	/* Copy sg so that we make an independent mapping */
 	st = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
 	if (st == NULL) {
 		ret = -ENOMEM;
-		goto err_unpin;
+		goto err_unpin_pages;
 	}
 
-	ret = sg_alloc_table(st, obj->pages->nents, GFP_KERNEL);
+	ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL);
 	if (ret)
 		goto err_free;
 
-	src = obj->pages->sgl;
+	src = obj->mm.pages->sgl;
 	dst = st->sgl;
-	for (i = 0; i < obj->pages->nents; i++) {
+	for (i = 0; i < obj->mm.pages->nents; i++) {
 		sg_set_page(dst, sg_page(src), src->length, 0);
 		dst = sg_next(dst);
 		src = sg_next(src);
 	}
 
 	if (!dma_map_sg(attachment->dev, st->sgl, st->nents, dir)) {
-		ret =-ENOMEM;
+		ret = -ENOMEM;
 		goto err_free_sg;
 	}
 
-	mutex_unlock(&obj->base.dev->struct_mutex);
 	return st;
 
 err_free_sg:
 	sg_free_table(st);
 err_free:
 	kfree(st);
-err_unpin:
+err_unpin_pages:
 	i915_gem_object_unpin_pages(obj);
-err_unlock:
-	mutex_unlock(&obj->base.dev->struct_mutex);
 err:
 	return ERR_PTR(ret);
 }
@@ -103,36 +94,21 @@ static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment,
 	sg_free_table(sg);
 	kfree(sg);
 
-	mutex_lock(&obj->base.dev->struct_mutex);
 	i915_gem_object_unpin_pages(obj);
-	mutex_unlock(&obj->base.dev->struct_mutex);
 }
 
 static void *i915_gem_dmabuf_vmap(struct dma_buf *dma_buf)
 {
 	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
-	struct drm_device *dev = obj->base.dev;
-	void *addr;
-	int ret;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ERR_PTR(ret);
-
-	addr = i915_gem_object_pin_map(obj, I915_MAP_WB);
-	mutex_unlock(&dev->struct_mutex);
-
-	return addr;
+	return i915_gem_object_pin_map(obj, I915_MAP_WB);
 }
 
 static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr)
 {
 	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
-	struct drm_device *dev = obj->base.dev;
 
-	mutex_lock(&dev->struct_mutex);
 	i915_gem_object_unpin_map(obj);
-	mutex_unlock(&dev->struct_mutex);
 }
 
 static void *i915_gem_dmabuf_kmap_atomic(struct dma_buf *dma_buf, unsigned long page_num)
@@ -179,32 +155,45 @@ static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_dire
 {
 	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
 	struct drm_device *dev = obj->base.dev;
-	int ret;
 	bool write = (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE);
+	int err;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
+	err = i915_gem_object_pin_pages(obj);
+	if (err)
+		return err;
+
+	err = i915_mutex_lock_interruptible(dev);
+	if (err)
+		goto out;
 
-	ret = i915_gem_object_set_to_cpu_domain(obj, write);
+	err = i915_gem_object_set_to_cpu_domain(obj, write);
 	mutex_unlock(&dev->struct_mutex);
-	return ret;
+
+out:
+	i915_gem_object_unpin_pages(obj);
+	return err;
 }
 
 static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction)
 {
 	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
 	struct drm_device *dev = obj->base.dev;
-	int ret;
+	int err;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
+	err = i915_gem_object_pin_pages(obj);
+	if (err)
+		return err;
 
-	ret = i915_gem_object_set_to_gtt_domain(obj, false);
+	err = i915_mutex_lock_interruptible(dev);
+	if (err)
+		goto out;
+
+	err = i915_gem_object_set_to_gtt_domain(obj, false);
 	mutex_unlock(&dev->struct_mutex);
 
-	return ret;
+out:
+	i915_gem_object_unpin_pages(obj);
+	return err;
 }
 
 static const struct dma_buf_ops i915_dmabuf_ops =  {
@@ -222,60 +211,17 @@ static const struct dma_buf_ops i915_dmabuf_ops =  {
 	.end_cpu_access = i915_gem_end_cpu_access,
 };
 
-static void export_fences(struct drm_i915_gem_object *obj,
-			  struct dma_buf *dma_buf)
-{
-	struct reservation_object *resv = dma_buf->resv;
-	struct drm_i915_gem_request *req;
-	unsigned long active;
-	int idx;
-
-	active = __I915_BO_ACTIVE(obj);
-	if (!active)
-		return;
-
-	/* Serialise with execbuf to prevent concurrent fence-loops */
-	mutex_lock(&obj->base.dev->struct_mutex);
-
-	/* Mark the object for future fences before racily adding old fences */
-	obj->base.dma_buf = dma_buf;
-
-	ww_mutex_lock(&resv->lock, NULL);
-
-	for_each_active(active, idx) {
-		req = i915_gem_active_get(&obj->last_read[idx],
-					  &obj->base.dev->struct_mutex);
-		if (!req)
-			continue;
-
-		if (reservation_object_reserve_shared(resv) == 0)
-			reservation_object_add_shared_fence(resv, &req->fence);
-
-		i915_gem_request_put(req);
-	}
-
-	req = i915_gem_active_get(&obj->last_write,
-				  &obj->base.dev->struct_mutex);
-	if (req) {
-		reservation_object_add_excl_fence(resv, &req->fence);
-		i915_gem_request_put(req);
-	}
-
-	ww_mutex_unlock(&resv->lock);
-	mutex_unlock(&obj->base.dev->struct_mutex);
-}
-
 struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
 				      struct drm_gem_object *gem_obj, int flags)
 {
 	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
-	struct dma_buf *dma_buf;
 
 	exp_info.ops = &i915_dmabuf_ops;
 	exp_info.size = gem_obj->size;
 	exp_info.flags = flags;
 	exp_info.priv = gem_obj;
+	exp_info.resv = obj->resv;
 
 	if (obj->ops->dmabuf_export) {
 		int ret = obj->ops->dmabuf_export(obj);
@@ -283,30 +229,21 @@ struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
 			return ERR_PTR(ret);
 	}
 
-	dma_buf = drm_gem_dmabuf_export(dev, &exp_info);
-	if (IS_ERR(dma_buf))
-		return dma_buf;
-
-	export_fences(obj, dma_buf);
-	return dma_buf;
+	return drm_gem_dmabuf_export(dev, &exp_info);
 }
 
-static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
+static struct sg_table *
+i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
 {
-	struct sg_table *sg;
-
-	sg = dma_buf_map_attachment(obj->base.import_attach, DMA_BIDIRECTIONAL);
-	if (IS_ERR(sg))
-		return PTR_ERR(sg);
-
-	obj->pages = sg;
-	return 0;
+	return dma_buf_map_attachment(obj->base.import_attach,
+				      DMA_BIDIRECTIONAL);
 }
 
-static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj)
+static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj,
+					     struct sg_table *pages)
 {
-	dma_buf_unmap_attachment(obj->base.import_attach,
-				 obj->pages, DMA_BIDIRECTIONAL);
+	dma_buf_unmap_attachment(obj->base.import_attach, pages,
+				 DMA_BIDIRECTIONAL);
 }
 
 static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {
@@ -350,6 +287,7 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev,
 	drm_gem_private_object_init(dev, &obj->base, dma_buf->size);
 	i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops);
 	obj->base.import_attach = attach;
+	obj->resv = dma_buf->resv;
 
 	/* We use GTT as shorthand for a coherent domain, one that is
 	 * neither in the GPU cache nor in the CPU cache, where all
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 5b6f81c1dbca..bd08814b015c 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -33,13 +33,17 @@
 #include "intel_drv.h"
 #include "i915_trace.h"
 
-static bool
-gpu_is_idle(struct drm_i915_private *dev_priv)
+static bool ggtt_is_idle(struct drm_i915_private *dev_priv)
 {
+	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	for_each_engine(engine, dev_priv, id) {
+		struct intel_timeline *tl;
 
-	for_each_engine(engine, dev_priv) {
-		if (intel_engine_is_active(engine))
+		tl = &ggtt->base.timeline.engine[engine->id];
+		if (i915_gem_active_isset(&tl->last_request))
 			return false;
 	}
 
@@ -55,7 +59,7 @@ mark_free(struct i915_vma *vma, unsigned int flags, struct list_head *unwind)
 	if (WARN_ON(!list_empty(&vma->exec_list)))
 		return false;
 
-	if (flags & PIN_NONFAULT && vma->obj->fault_mappable)
+	if (flags & PIN_NONFAULT && !list_empty(&vma->obj->userfault_link))
 		return false;
 
 	list_add(&vma->exec_list, unwind);
@@ -102,6 +106,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
 	struct i915_vma *vma, *next;
 	int ret;
 
+	lockdep_assert_held(&vm->dev->struct_mutex);
 	trace_i915_gem_evict(vm, min_size, alignment, flags);
 
 	/*
@@ -152,7 +157,7 @@ search_again:
 	if (!i915_is_ggtt(vm) || flags & PIN_NONBLOCK)
 		return -ENOSPC;
 
-	if (gpu_is_idle(dev_priv)) {
+	if (ggtt_is_idle(dev_priv)) {
 		/* If we still have pending pageflip completions, drop
 		 * back to userspace to give our workqueues time to
 		 * acquire our locks and unpin the old scanouts.
@@ -212,6 +217,8 @@ i915_gem_evict_for_vma(struct i915_vma *target)
 {
 	struct drm_mm_node *node, *next;
 
+	lockdep_assert_held(&target->vm->dev->struct_mutex);
+
 	list_for_each_entry_safe(node, next,
 			&target->vm->mm.head_node.node_list,
 			node_list) {
@@ -265,7 +272,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
 	struct i915_vma *vma, *next;
 	int ret;
 
-	WARN_ON(!mutex_is_locked(&vm->dev->struct_mutex));
+	lockdep_assert_held(&vm->dev->struct_mutex);
 	trace_i915_gem_evict_vm(vm);
 
 	if (do_idle) {
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index a218c2e395e7..097d9d8c2315 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -34,7 +34,6 @@
 #include <drm/i915_drm.h>
 
 #include "i915_drv.h"
-#include "i915_gem_dmabuf.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
 #include "intel_frontbuffer.h"
@@ -288,7 +287,7 @@ static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
 	if (DBG_USE_CPU_RELOC)
 		return DBG_USE_CPU_RELOC > 0;
 
-	return (HAS_LLC(obj->base.dev) ||
+	return (HAS_LLC(to_i915(obj->base.dev)) ||
 		obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
 		obj->cache_level != I915_CACHE_NONE);
 }
@@ -332,7 +331,8 @@ static void reloc_cache_init(struct reloc_cache *cache,
 	cache->page = -1;
 	cache->vaddr = 0;
 	cache->i915 = i915;
-	cache->use_64bit_reloc = INTEL_GEN(cache->i915) >= 8;
+	/* Must be a variable in the struct to allow GCC to unroll. */
+	cache->use_64bit_reloc = HAS_64BIT_RELOC(i915);
 	cache->node.allocated = false;
 }
 
@@ -370,8 +370,7 @@ static void reloc_cache_fini(struct reloc_cache *cache)
 
 			ggtt->base.clear_range(&ggtt->base,
 					       cache->node.start,
-					       cache->node.size,
-					       true);
+					       cache->node.size);
 			drm_mm_remove_node(&cache->node);
 		} else {
 			i915_vma_unpin((struct i915_vma *)cache->node.mm);
@@ -419,17 +418,8 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
 	unsigned long offset;
 	void *vaddr;
 
-	if (cache->node.allocated) {
-		wmb();
-		ggtt->base.insert_page(&ggtt->base,
-				       i915_gem_object_get_dma_address(obj, page),
-				       cache->node.start, I915_CACHE_NONE, 0);
-		cache->page = page;
-		return unmask_page(cache->vaddr);
-	}
-
 	if (cache->vaddr) {
-		io_mapping_unmap_atomic(unmask_page(cache->vaddr));
+		io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));
 	} else {
 		struct i915_vma *vma;
 		int ret;
@@ -467,6 +457,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
 
 	offset = cache->node.start;
 	if (cache->node.allocated) {
+		wmb();
 		ggtt->base.insert_page(&ggtt->base,
 				       i915_gem_object_get_dma_address(obj, page),
 				       offset, I915_CACHE_NONE, 0);
@@ -474,7 +465,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
 		offset += page << PAGE_SHIFT;
 	}
 
-	vaddr = io_mapping_map_atomic_wc(&cache->i915->ggtt.mappable, offset);
+	vaddr = (void __force *) io_mapping_map_atomic_wc(&cache->i915->ggtt.mappable, offset);
 	cache->page = page;
 	cache->vaddr = (unsigned long)vaddr;
 
@@ -552,27 +543,13 @@ repeat:
 	return 0;
 }
 
-static bool object_is_idle(struct drm_i915_gem_object *obj)
-{
-	unsigned long active = i915_gem_object_get_active(obj);
-	int idx;
-
-	for_each_active(active, idx) {
-		if (!i915_gem_active_is_idle(&obj->last_read[idx],
-					     &obj->base.dev->struct_mutex))
-			return false;
-	}
-
-	return true;
-}
-
 static int
 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 				   struct eb_vmas *eb,
 				   struct drm_i915_gem_relocation_entry *reloc,
 				   struct reloc_cache *cache)
 {
-	struct drm_device *dev = obj->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 	struct drm_gem_object *target_obj;
 	struct drm_i915_gem_object *target_i915_obj;
 	struct i915_vma *target_vma;
@@ -591,7 +568,7 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 	/* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
 	 * pipe_control writes because the gpu doesn't properly redirect them
 	 * through the ppgtt for non_secure batchbuffers. */
-	if (unlikely(IS_GEN6(dev) &&
+	if (unlikely(IS_GEN6(dev_priv) &&
 	    reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
 		ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
 				    PIN_GLOBAL);
@@ -649,10 +626,6 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 		return -EINVAL;
 	}
 
-	/* We can't wait for rendering with pagefaults disabled */
-	if (pagefault_disabled() && !object_is_idle(obj))
-		return -EFAULT;
-
 	ret = relocate_entry(obj, reloc, cache, target_offset);
 	if (ret)
 		return ret;
@@ -679,12 +652,23 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
 	remain = entry->relocation_count;
 	while (remain) {
 		struct drm_i915_gem_relocation_entry *r = stack_reloc;
-		int count = remain;
-		if (count > ARRAY_SIZE(stack_reloc))
-			count = ARRAY_SIZE(stack_reloc);
+		unsigned long unwritten;
+		unsigned int count;
+
+		count = min_t(unsigned int, remain, ARRAY_SIZE(stack_reloc));
 		remain -= count;
 
-		if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0]))) {
+		/* This is the fast path and we cannot handle a pagefault
+		 * whilst holding the struct mutex lest the user pass in the
+		 * relocations contained within a mmaped bo. For in such a case
+		 * we, the page fault handler would call i915_gem_fault() and
+		 * we would try to acquire the struct mutex again. Obviously
+		 * this is bad and so lockdep complains vehemently.
+		 */
+		pagefault_disable();
+		unwritten = __copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0]));
+		pagefault_enable();
+		if (unlikely(unwritten)) {
 			ret = -EFAULT;
 			goto out;
 		}
@@ -696,11 +680,26 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
 			if (ret)
 				goto out;
 
-			if (r->presumed_offset != offset &&
-			    __put_user(r->presumed_offset,
-				       &user_relocs->presumed_offset)) {
-				ret = -EFAULT;
-				goto out;
+			if (r->presumed_offset != offset) {
+				pagefault_disable();
+				unwritten = __put_user(r->presumed_offset,
+						       &user_relocs->presumed_offset);
+				pagefault_enable();
+				if (unlikely(unwritten)) {
+					/* Note that reporting an error now
+					 * leaves everything in an inconsistent
+					 * state as we have *already* changed
+					 * the relocation value inside the
+					 * object. As we have not changed the
+					 * reloc.presumed_offset or will not
+					 * change the execobject.offset, on the
+					 * call we may not rewrite the value
+					 * inside the object, leaving it
+					 * dangling and causing a GPU hang.
+					 */
+					ret = -EFAULT;
+					goto out;
+				}
 			}
 
 			user_relocs++;
@@ -740,20 +739,11 @@ i915_gem_execbuffer_relocate(struct eb_vmas *eb)
 	struct i915_vma *vma;
 	int ret = 0;
 
-	/* This is the fast path and we cannot handle a pagefault whilst
-	 * holding the struct mutex lest the user pass in the relocations
-	 * contained within a mmaped bo. For in such a case we, the page
-	 * fault handler would call i915_gem_fault() and we would try to
-	 * acquire the struct mutex again. Obviously this is bad and so
-	 * lockdep complains vehemently.
-	 */
-	pagefault_disable();
 	list_for_each_entry(vma, &eb->vmas, exec_list) {
 		ret = i915_gem_execbuffer_relocate_vma(vma, eb);
 		if (ret)
 			break;
 	}
-	pagefault_enable();
 
 	return ret;
 }
@@ -843,7 +833,7 @@ need_reloc_mappable(struct i915_vma *vma)
 		return false;
 
 	/* See also use_cpu_reloc() */
-	if (HAS_LLC(vma->obj->base.dev))
+	if (HAS_LLC(to_i915(vma->obj->base.dev)))
 		return false;
 
 	if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
@@ -1111,44 +1101,20 @@ err:
 	return ret;
 }
 
-static unsigned int eb_other_engines(struct drm_i915_gem_request *req)
-{
-	unsigned int mask;
-
-	mask = ~intel_engine_flag(req->engine) & I915_BO_ACTIVE_MASK;
-	mask <<= I915_BO_ACTIVE_SHIFT;
-
-	return mask;
-}
-
 static int
 i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
 				struct list_head *vmas)
 {
-	const unsigned int other_rings = eb_other_engines(req);
 	struct i915_vma *vma;
 	int ret;
 
 	list_for_each_entry(vma, vmas, exec_list) {
 		struct drm_i915_gem_object *obj = vma->obj;
-		struct reservation_object *resv;
 
-		if (obj->flags & other_rings) {
-			ret = i915_gem_request_await_object
-				(req, obj, obj->base.pending_write_domain);
-			if (ret)
-				return ret;
-		}
-
-		resv = i915_gem_object_get_dmabuf_resv(obj);
-		if (resv) {
-			ret = i915_sw_fence_await_reservation
-				(&req->submit, resv, &i915_fence_ops,
-				 obj->base.pending_write_domain, 10*HZ,
-				 GFP_KERNEL | __GFP_NOWARN);
-			if (ret < 0)
-				return ret;
-		}
+		ret = i915_gem_request_await_object
+			(req, obj, obj->base.pending_write_domain);
+		if (ret)
+			return ret;
 
 		if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
 			i915_gem_clflush_object(obj, false);
@@ -1296,8 +1262,6 @@ void i915_vma_move_to_active(struct i915_vma *vma,
 
 	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 
-	obj->dirty = 1; /* be paranoid  */
-
 	/* Add a reference if we're newly entering the active list.
 	 * The order in which we add operations to the retirement queue is
 	 * vital here: mark_active adds to the start of the callback list,
@@ -1305,15 +1269,15 @@ void i915_vma_move_to_active(struct i915_vma *vma,
 	 * add the active reference first and queue for it to be dropped
 	 * *last*.
 	 */
-	if (!i915_gem_object_is_active(obj))
-		i915_gem_object_get(obj);
-	i915_gem_object_set_active(obj, idx);
-	i915_gem_active_set(&obj->last_read[idx], req);
+	if (!i915_vma_is_active(vma))
+		obj->active_count++;
+	i915_vma_set_active(vma, idx);
+	i915_gem_active_set(&vma->last_read[idx], req);
+	list_move_tail(&vma->vm_link, &vma->vm->active_list);
 
 	if (flags & EXEC_OBJECT_WRITE) {
-		i915_gem_active_set(&obj->last_write, req);
-
-		intel_fb_obj_invalidate(obj, ORIGIN_CS);
+		if (intel_fb_obj_invalidate(obj, ORIGIN_CS))
+			i915_gem_active_set(&obj->frontbuffer_write, req);
 
 		/* update for the implicit flush after a batch */
 		obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
@@ -1323,21 +1287,13 @@ void i915_vma_move_to_active(struct i915_vma *vma,
 
 	if (flags & EXEC_OBJECT_NEEDS_FENCE)
 		i915_gem_active_set(&vma->last_fence, req);
-
-	i915_vma_set_active(vma, idx);
-	i915_gem_active_set(&vma->last_read[idx], req);
-	list_move_tail(&vma->vm_link, &vma->vm->active_list);
 }
 
 static void eb_export_fence(struct drm_i915_gem_object *obj,
 			    struct drm_i915_gem_request *req,
 			    unsigned int flags)
 {
-	struct reservation_object *resv;
-
-	resv = i915_gem_object_get_dmabuf_resv(obj);
-	if (!resv)
-		return;
+	struct reservation_object *resv = obj->resv;
 
 	/* Ignore errors from failing to allocate the new fence, we can't
 	 * handle an error right now. Worst case should be missed
@@ -1607,12 +1563,12 @@ eb_select_engine(struct drm_i915_private *dev_priv,
 			return NULL;
 		}
 
-		engine = &dev_priv->engine[_VCS(bsd_idx)];
+		engine = dev_priv->engine[_VCS(bsd_idx)];
 	} else {
-		engine = &dev_priv->engine[user_ring_map[user_ring_id]];
+		engine = dev_priv->engine[user_ring_map[user_ring_id]];
 	}
 
-	if (!intel_engine_initialized(engine)) {
+	if (!engine) {
 		DRM_DEBUG("execbuf with invalid ring: %u\n", user_ring_id);
 		return NULL;
 	}
@@ -1667,7 +1623,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 	}
 
 	if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
-		if (!HAS_RESOURCE_STREAMER(dev)) {
+		if (!HAS_RESOURCE_STREAMER(dev_priv)) {
 			DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
 			return -EINVAL;
 		}
@@ -1921,7 +1877,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 		exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
 		exec2_list[i].alignment = exec_list[i].alignment;
 		exec2_list[i].offset = exec_list[i].offset;
-		if (INTEL_INFO(dev)->gen < 4)
+		if (INTEL_GEN(to_i915(dev)) < 4)
 			exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
 		else
 			exec2_list[i].flags = 0;
diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence_reg.c
index 2c7ba0ee127c..0efa3571afc3 100644
--- a/drivers/gpu/drm/i915/i915_gem_fence.c
+++ b/drivers/gpu/drm/i915/i915_gem_fence_reg.c
@@ -343,6 +343,9 @@ i915_vma_get_fence(struct i915_vma *vma)
 	struct drm_i915_fence_reg *fence;
 	struct i915_vma *set = i915_gem_object_is_tiled(vma->obj) ? vma : NULL;
 
+	/* Note that we revoke fences on runtime suspend. Therefore the user
+	 * must keep the device awake whilst using the fence.
+	 */
 	assert_rpm_wakelock_held(to_i915(vma->vm->dev));
 
 	/* Just update our place in the LRU if our fence is getting reused. */
@@ -365,22 +368,16 @@ i915_vma_get_fence(struct i915_vma *vma)
 
 /**
  * i915_gem_restore_fences - restore fence state
- * @dev: DRM device
+ * @dev_priv: i915 device private
  *
  * Restore the hw fence state to match the software tracking again, to be called
- * after a gpu reset and on resume.
+ * after a gpu reset and on resume. Note that on runtime suspend we only cancel
+ * the fences, to be reacquired by the user later.
  */
-void i915_gem_restore_fences(struct drm_device *dev)
+void i915_gem_restore_fences(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	int i;
 
-	/* Note that this may be called outside of struct_mutex, by
-	 * runtime suspend/resume. The barrier we require is enforced by
-	 * rpm itself - all access to fences/GTT are only within an rpm
-	 * wakeref, and to acquire that wakeref you must pass through here.
-	 */
-
 	for (i = 0; i < dev_priv->num_fence_regs; i++) {
 		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
 		struct i915_vma *vma = reg->vma;
@@ -391,7 +388,7 @@ void i915_gem_restore_fences(struct drm_device *dev)
 		 */
 		if (vma && !i915_gem_object_is_tiled(vma->obj)) {
 			GEM_BUG_ON(!reg->dirty);
-			GEM_BUG_ON(vma->obj->fault_mappable);
+			GEM_BUG_ON(!list_empty(&vma->obj->userfault_link));
 
 			list_move(&reg->link, &dev_priv->mm.fence_list);
 			vma->fence = NULL;
@@ -453,19 +450,18 @@ void i915_gem_restore_fences(struct drm_device *dev)
 
 /**
  * i915_gem_detect_bit_6_swizzle - detect bit 6 swizzling pattern
- * @dev: DRM device
+ * @dev_priv: i915 device private
  *
  * Detects bit 6 swizzling of address lookup between IGD access and CPU
  * access through main memory.
  */
 void
-i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
+i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
 	uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
 
-	if (INTEL_INFO(dev)->gen >= 8 || IS_VALLEYVIEW(dev)) {
+	if (INTEL_GEN(dev_priv) >= 8 || IS_VALLEYVIEW(dev_priv)) {
 		/*
 		 * On BDW+, swizzling is not used. We leave the CPU memory
 		 * controller in charge of optimizing memory accesses without
@@ -475,7 +471,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
 		 */
 		swizzle_x = I915_BIT_6_SWIZZLE_NONE;
 		swizzle_y = I915_BIT_6_SWIZZLE_NONE;
-	} else if (INTEL_INFO(dev)->gen >= 6) {
+	} else if (INTEL_GEN(dev_priv) >= 6) {
 		if (dev_priv->preserve_bios_swizzle) {
 			if (I915_READ(DISP_ARB_CTL) &
 			    DISP_TILE_SURFACE_SWIZZLING) {
@@ -504,19 +500,20 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
 				swizzle_y = I915_BIT_6_SWIZZLE_NONE;
 			}
 		}
-	} else if (IS_GEN5(dev)) {
+	} else if (IS_GEN5(dev_priv)) {
 		/* On Ironlake whatever DRAM config, GPU always do
 		 * same swizzling setup.
 		 */
 		swizzle_x = I915_BIT_6_SWIZZLE_9_10;
 		swizzle_y = I915_BIT_6_SWIZZLE_9;
-	} else if (IS_GEN2(dev)) {
+	} else if (IS_GEN2(dev_priv)) {
 		/* As far as we know, the 865 doesn't have these bit 6
 		 * swizzling issues.
 		 */
 		swizzle_x = I915_BIT_6_SWIZZLE_NONE;
 		swizzle_y = I915_BIT_6_SWIZZLE_NONE;
-	} else if (IS_MOBILE(dev) || (IS_GEN3(dev) && !IS_G33(dev))) {
+	} else if (IS_MOBILE(dev_priv) || (IS_GEN3(dev_priv) &&
+		   !IS_G33(dev_priv))) {
 		uint32_t dcc;
 
 		/* On 9xx chipsets, channel interleave by the CPU is
@@ -554,7 +551,7 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
 		}
 
 		/* check for L-shaped memory aka modified enhanced addressing */
-		if (IS_GEN4(dev) &&
+		if (IS_GEN4(dev_priv) &&
 		    !(I915_READ(DCC2) & DCC2_MODIFIED_ENHANCED_DISABLE)) {
 			swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
 			swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
@@ -645,6 +642,7 @@ i915_gem_swizzle_page(struct page *page)
 /**
  * i915_gem_object_do_bit_17_swizzle - fixup bit 17 swizzling
  * @obj: i915 GEM buffer object
+ * @pages: the scattergather list of physical pages
  *
  * This function fixes up the swizzling in case any page frame number for this
  * object has changed in bit 17 since that state has been saved with
@@ -655,7 +653,8 @@ i915_gem_swizzle_page(struct page *page)
  * by swapping them out and back in again).
  */
 void
-i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
+i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj,
+				  struct sg_table *pages)
 {
 	struct sgt_iter sgt_iter;
 	struct page *page;
@@ -665,10 +664,9 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
 		return;
 
 	i = 0;
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
+	for_each_sgt_page(page, sgt_iter, pages) {
 		char new_bit_17 = page_to_phys(page) >> 17;
-		if ((new_bit_17 & 0x1) !=
-		    (test_bit(i, obj->bit_17) != 0)) {
+		if ((new_bit_17 & 0x1) != (test_bit(i, obj->bit_17) != 0)) {
 			i915_gem_swizzle_page(page);
 			set_page_dirty(page);
 		}
@@ -679,17 +677,19 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
 /**
  * i915_gem_object_save_bit_17_swizzle - save bit 17 swizzling
  * @obj: i915 GEM buffer object
+ * @pages: the scattergather list of physical pages
  *
  * This function saves the bit 17 of each page frame number so that swizzling
  * can be fixed up later on with i915_gem_object_do_bit_17_swizzle(). This must
  * be called before the backing storage can be unpinned.
  */
 void
-i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
+i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj,
+				    struct sg_table *pages)
 {
+	const unsigned int page_count = obj->base.size >> PAGE_SHIFT;
 	struct sgt_iter sgt_iter;
 	struct page *page;
-	int page_count = obj->base.size >> PAGE_SHIFT;
 	int i;
 
 	if (obj->bit_17 == NULL) {
@@ -704,7 +704,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
 
 	i = 0;
 
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
+	for_each_sgt_page(page, sgt_iter, pages) {
 		if (page_to_phys(page) & (1 << 17))
 			__set_bit(i, obj->bit_17);
 		else
diff --git a/drivers/gpu/drm/i915/i915_gem_fence_reg.h b/drivers/gpu/drm/i915/i915_gem_fence_reg.h
new file mode 100644
index 000000000000..22c4a2d01adf
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_fence_reg.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_FENCE_REG_H__
+#define __I915_FENCE_REG_H__
+
+#include <linux/list.h>
+
+struct drm_i915_private;
+struct i915_vma;
+
+struct drm_i915_fence_reg {
+	struct list_head link;
+	struct drm_i915_private *i915;
+	struct i915_vma *vma;
+	int pin_count;
+	int id;
+	/**
+	 * Whether the tiling parameters for the currently
+	 * associated fence register have changed. Note that
+	 * for the purposes of tracking tiling changes we also
+	 * treat the unfenced register, the register slot that
+	 * the object occupies whilst it executes a fenced
+	 * command (such as BLT on gen2/3), as a "fence".
+	 */
+	bool dirty;
+};
+
+#endif
+
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 0bb4232f66bc..b4bde1452f2a 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -31,6 +31,7 @@
 #include "i915_vgpu.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
+#include "intel_frontbuffer.h"
 
 #define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM)
 
@@ -95,13 +96,6 @@
  *
  */
 
-static inline struct i915_ggtt *
-i915_vm_to_ggtt(struct i915_address_space *vm)
-{
-	GEM_BUG_ON(!i915_is_ggtt(vm));
-	return container_of(vm, struct i915_ggtt, base);
-}
-
 static int
 i915_get_ggtt_vma_pages(struct i915_vma *vma);
 
@@ -175,7 +169,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
 {
 	u32 pte_flags = 0;
 
-	vma->pages = vma->obj->pages;
+	vma->pages = vma->obj->mm.pages;
 
 	/* Currently applicable only to VLV */
 	if (vma->obj->gt_ro)
@@ -191,15 +185,13 @@ static void ppgtt_unbind_vma(struct i915_vma *vma)
 {
 	vma->vm->clear_range(vma->vm,
 			     vma->node.start,
-			     vma->size,
-			     true);
+			     vma->size);
 }
 
 static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
-				  enum i915_cache_level level,
-				  bool valid)
+				  enum i915_cache_level level)
 {
-	gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
+	gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW;
 	pte |= addr;
 
 	switch (level) {
@@ -234,9 +226,9 @@ static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
 
 static gen6_pte_t snb_pte_encode(dma_addr_t addr,
 				 enum i915_cache_level level,
-				 bool valid, u32 unused)
+				 u32 unused)
 {
-	gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
+	gen6_pte_t pte = GEN6_PTE_VALID;
 	pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
 	switch (level) {
@@ -256,9 +248,9 @@ static gen6_pte_t snb_pte_encode(dma_addr_t addr,
 
 static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
 				 enum i915_cache_level level,
-				 bool valid, u32 unused)
+				 u32 unused)
 {
-	gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
+	gen6_pte_t pte = GEN6_PTE_VALID;
 	pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
 	switch (level) {
@@ -280,9 +272,9 @@ static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
 
 static gen6_pte_t byt_pte_encode(dma_addr_t addr,
 				 enum i915_cache_level level,
-				 bool valid, u32 flags)
+				 u32 flags)
 {
-	gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
+	gen6_pte_t pte = GEN6_PTE_VALID;
 	pte |= GEN6_PTE_ADDR_ENCODE(addr);
 
 	if (!(flags & PTE_READ_ONLY))
@@ -296,9 +288,9 @@ static gen6_pte_t byt_pte_encode(dma_addr_t addr,
 
 static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
 				 enum i915_cache_level level,
-				 bool valid, u32 unused)
+				 u32 unused)
 {
-	gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
+	gen6_pte_t pte = GEN6_PTE_VALID;
 	pte |= HSW_PTE_ADDR_ENCODE(addr);
 
 	if (level != I915_CACHE_NONE)
@@ -309,9 +301,9 @@ static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
 
 static gen6_pte_t iris_pte_encode(dma_addr_t addr,
 				  enum i915_cache_level level,
-				  bool valid, u32 unused)
+				  u32 unused)
 {
-	gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
+	gen6_pte_t pte = GEN6_PTE_VALID;
 	pte |= HSW_PTE_ADDR_ENCODE(addr);
 
 	switch (level) {
@@ -328,10 +320,10 @@ static gen6_pte_t iris_pte_encode(dma_addr_t addr,
 	return pte;
 }
 
-static int __setup_page_dma(struct drm_device *dev,
+static int __setup_page_dma(struct drm_i915_private *dev_priv,
 			    struct i915_page_dma *p, gfp_t flags)
 {
-	struct device *kdev = &dev->pdev->dev;
+	struct device *kdev = &dev_priv->drm.pdev->dev;
 
 	p->page = alloc_page(flags);
 	if (!p->page)
@@ -348,14 +340,16 @@ static int __setup_page_dma(struct drm_device *dev,
 	return 0;
 }
 
-static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
+static int setup_page_dma(struct drm_i915_private *dev_priv,
+			  struct i915_page_dma *p)
 {
-	return __setup_page_dma(dev, p, I915_GFP_DMA);
+	return __setup_page_dma(dev_priv, p, I915_GFP_DMA);
 }
 
-static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
+static void cleanup_page_dma(struct drm_i915_private *dev_priv,
+			     struct i915_page_dma *p)
 {
-	struct pci_dev *pdev = dev->pdev;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 
 	if (WARN_ON(!p->page))
 		return;
@@ -373,27 +367,29 @@ static void *kmap_page_dma(struct i915_page_dma *p)
 /* We use the flushing unmap only with ppgtt structures:
  * page directories, page tables and scratch pages.
  */
-static void kunmap_page_dma(struct drm_device *dev, void *vaddr)
+static void kunmap_page_dma(struct drm_i915_private *dev_priv, void *vaddr)
 {
 	/* There are only few exceptions for gen >=6. chv and bxt.
 	 * And we are not sure about the latter so play safe for now.
 	 */
-	if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
+	if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv))
 		drm_clflush_virt_range(vaddr, PAGE_SIZE);
 
 	kunmap_atomic(vaddr);
 }
 
 #define kmap_px(px) kmap_page_dma(px_base(px))
-#define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr))
+#define kunmap_px(ppgtt, vaddr) \
+		kunmap_page_dma(to_i915((ppgtt)->base.dev), (vaddr))
 
-#define setup_px(dev, px) setup_page_dma((dev), px_base(px))
-#define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px))
-#define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v))
-#define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v))
+#define setup_px(dev_priv, px) setup_page_dma((dev_priv), px_base(px))
+#define cleanup_px(dev_priv, px) cleanup_page_dma((dev_priv), px_base(px))
+#define fill_px(dev_priv, px, v) fill_page_dma((dev_priv), px_base(px), (v))
+#define fill32_px(dev_priv, px, v) \
+		fill_page_dma_32((dev_priv), px_base(px), (v))
 
-static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p,
-			  const uint64_t val)
+static void fill_page_dma(struct drm_i915_private *dev_priv,
+			  struct i915_page_dma *p, const uint64_t val)
 {
 	int i;
 	uint64_t * const vaddr = kmap_page_dma(p);
@@ -401,38 +397,37 @@ static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p,
 	for (i = 0; i < 512; i++)
 		vaddr[i] = val;
 
-	kunmap_page_dma(dev, vaddr);
+	kunmap_page_dma(dev_priv, vaddr);
 }
 
-static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p,
-			     const uint32_t val32)
+static void fill_page_dma_32(struct drm_i915_private *dev_priv,
+			     struct i915_page_dma *p, const uint32_t val32)
 {
 	uint64_t v = val32;
 
 	v = v << 32 | val32;
 
-	fill_page_dma(dev, p, v);
+	fill_page_dma(dev_priv, p, v);
 }
 
 static int
-setup_scratch_page(struct drm_device *dev,
+setup_scratch_page(struct drm_i915_private *dev_priv,
 		   struct i915_page_dma *scratch,
 		   gfp_t gfp)
 {
-	return __setup_page_dma(dev, scratch, gfp | __GFP_ZERO);
+	return __setup_page_dma(dev_priv, scratch, gfp | __GFP_ZERO);
 }
 
-static void cleanup_scratch_page(struct drm_device *dev,
+static void cleanup_scratch_page(struct drm_i915_private *dev_priv,
 				 struct i915_page_dma *scratch)
 {
-	cleanup_page_dma(dev, scratch);
+	cleanup_page_dma(dev_priv, scratch);
 }
 
-static struct i915_page_table *alloc_pt(struct drm_device *dev)
+static struct i915_page_table *alloc_pt(struct drm_i915_private *dev_priv)
 {
 	struct i915_page_table *pt;
-	const size_t count = INTEL_INFO(dev)->gen >= 8 ?
-		GEN8_PTES : GEN6_PTES;
+	const size_t count = INTEL_GEN(dev_priv) >= 8 ? GEN8_PTES : GEN6_PTES;
 	int ret = -ENOMEM;
 
 	pt = kzalloc(sizeof(*pt), GFP_KERNEL);
@@ -445,7 +440,7 @@ static struct i915_page_table *alloc_pt(struct drm_device *dev)
 	if (!pt->used_ptes)
 		goto fail_bitmap;
 
-	ret = setup_px(dev, pt);
+	ret = setup_px(dev_priv, pt);
 	if (ret)
 		goto fail_page_m;
 
@@ -459,9 +454,10 @@ fail_bitmap:
 	return ERR_PTR(ret);
 }
 
-static void free_pt(struct drm_device *dev, struct i915_page_table *pt)
+static void free_pt(struct drm_i915_private *dev_priv,
+		    struct i915_page_table *pt)
 {
-	cleanup_px(dev, pt);
+	cleanup_px(dev_priv, pt);
 	kfree(pt->used_ptes);
 	kfree(pt);
 }
@@ -472,9 +468,9 @@ static void gen8_initialize_pt(struct i915_address_space *vm,
 	gen8_pte_t scratch_pte;
 
 	scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
-				      I915_CACHE_LLC, true);
+				      I915_CACHE_LLC);
 
-	fill_px(vm->dev, pt, scratch_pte);
+	fill_px(to_i915(vm->dev), pt, scratch_pte);
 }
 
 static void gen6_initialize_pt(struct i915_address_space *vm,
@@ -485,12 +481,12 @@ static void gen6_initialize_pt(struct i915_address_space *vm,
 	WARN_ON(vm->scratch_page.daddr == 0);
 
 	scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
-				     I915_CACHE_LLC, true, 0);
+				     I915_CACHE_LLC, 0);
 
-	fill32_px(vm->dev, pt, scratch_pte);
+	fill32_px(to_i915(vm->dev), pt, scratch_pte);
 }
 
-static struct i915_page_directory *alloc_pd(struct drm_device *dev)
+static struct i915_page_directory *alloc_pd(struct drm_i915_private *dev_priv)
 {
 	struct i915_page_directory *pd;
 	int ret = -ENOMEM;
@@ -504,7 +500,7 @@ static struct i915_page_directory *alloc_pd(struct drm_device *dev)
 	if (!pd->used_pdes)
 		goto fail_bitmap;
 
-	ret = setup_px(dev, pd);
+	ret = setup_px(dev_priv, pd);
 	if (ret)
 		goto fail_page_m;
 
@@ -518,10 +514,11 @@ fail_bitmap:
 	return ERR_PTR(ret);
 }
 
-static void free_pd(struct drm_device *dev, struct i915_page_directory *pd)
+static void free_pd(struct drm_i915_private *dev_priv,
+		    struct i915_page_directory *pd)
 {
 	if (px_page(pd)) {
-		cleanup_px(dev, pd);
+		cleanup_px(dev_priv, pd);
 		kfree(pd->used_pdes);
 		kfree(pd);
 	}
@@ -534,13 +531,13 @@ static void gen8_initialize_pd(struct i915_address_space *vm,
 
 	scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC);
 
-	fill_px(vm->dev, pd, scratch_pde);
+	fill_px(to_i915(vm->dev), pd, scratch_pde);
 }
 
-static int __pdp_init(struct drm_device *dev,
+static int __pdp_init(struct drm_i915_private *dev_priv,
 		      struct i915_page_directory_pointer *pdp)
 {
-	size_t pdpes = I915_PDPES_PER_PDP(dev);
+	size_t pdpes = I915_PDPES_PER_PDP(dev_priv);
 
 	pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
 				  sizeof(unsigned long),
@@ -569,22 +566,22 @@ static void __pdp_fini(struct i915_page_directory_pointer *pdp)
 }
 
 static struct
-i915_page_directory_pointer *alloc_pdp(struct drm_device *dev)
+i915_page_directory_pointer *alloc_pdp(struct drm_i915_private *dev_priv)
 {
 	struct i915_page_directory_pointer *pdp;
 	int ret = -ENOMEM;
 
-	WARN_ON(!USES_FULL_48BIT_PPGTT(dev));
+	WARN_ON(!USES_FULL_48BIT_PPGTT(dev_priv));
 
 	pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
 	if (!pdp)
 		return ERR_PTR(-ENOMEM);
 
-	ret = __pdp_init(dev, pdp);
+	ret = __pdp_init(dev_priv, pdp);
 	if (ret)
 		goto fail_bitmap;
 
-	ret = setup_px(dev, pdp);
+	ret = setup_px(dev_priv, pdp);
 	if (ret)
 		goto fail_page_m;
 
@@ -598,12 +595,12 @@ fail_bitmap:
 	return ERR_PTR(ret);
 }
 
-static void free_pdp(struct drm_device *dev,
+static void free_pdp(struct drm_i915_private *dev_priv,
 		     struct i915_page_directory_pointer *pdp)
 {
 	__pdp_fini(pdp);
-	if (USES_FULL_48BIT_PPGTT(dev)) {
-		cleanup_px(dev, pdp);
+	if (USES_FULL_48BIT_PPGTT(dev_priv)) {
+		cleanup_px(dev_priv, pdp);
 		kfree(pdp);
 	}
 }
@@ -615,7 +612,7 @@ static void gen8_initialize_pdp(struct i915_address_space *vm,
 
 	scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
 
-	fill_px(vm->dev, pdp, scratch_pdpe);
+	fill_px(to_i915(vm->dev), pdp, scratch_pdpe);
 }
 
 static void gen8_initialize_pml4(struct i915_address_space *vm,
@@ -626,7 +623,7 @@ static void gen8_initialize_pml4(struct i915_address_space *vm,
 	scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp),
 					  I915_CACHE_LLC);
 
-	fill_px(vm->dev, pml4, scratch_pml4e);
+	fill_px(to_i915(vm->dev), pml4, scratch_pml4e);
 }
 
 static void
@@ -637,7 +634,7 @@ gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt,
 {
 	gen8_ppgtt_pdpe_t *page_directorypo;
 
-	if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev))
+	if (!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev)))
 		return;
 
 	page_directorypo = kmap_px(pdp);
@@ -653,7 +650,7 @@ gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt,
 {
 	gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4);
 
-	WARN_ON(!USES_FULL_48BIT_PPGTT(ppgtt->base.dev));
+	WARN_ON(!USES_FULL_48BIT_PPGTT(to_i915(ppgtt->base.dev)));
 	pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
 	kunmap_px(ppgtt, pagemap);
 }
@@ -706,85 +703,172 @@ static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt,
 	return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4));
 }
 
-static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm,
-				       struct i915_page_directory_pointer *pdp,
-				       uint64_t start,
-				       uint64_t length,
-				       gen8_pte_t scratch_pte)
+/* PDE TLBs are a pain to invalidate on GEN8+. When we modify
+ * the page table structures, we mark them dirty so that
+ * context switching/execlist queuing code takes extra steps
+ * to ensure that tlbs are flushed.
+ */
+static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
+{
+	ppgtt->pd_dirty_rings = INTEL_INFO(to_i915(ppgtt->base.dev))->ring_mask;
+}
+
+/* Removes entries from a single page table, releasing it if it's empty.
+ * Caller can use the return value to update higher-level entries.
+ */
+static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
+				struct i915_page_table *pt,
+				uint64_t start,
+				uint64_t length)
 {
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
+	unsigned int num_entries = gen8_pte_count(start, length);
+	unsigned int pte = gen8_pte_index(start);
+	unsigned int pte_end = pte + num_entries;
 	gen8_pte_t *pt_vaddr;
-	unsigned pdpe = gen8_pdpe_index(start);
-	unsigned pde = gen8_pde_index(start);
-	unsigned pte = gen8_pte_index(start);
-	unsigned num_entries = length >> PAGE_SHIFT;
-	unsigned last_pte, i;
+	gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
+						 I915_CACHE_LLC);
 
-	if (WARN_ON(!pdp))
-		return;
+	if (WARN_ON(!px_page(pt)))
+		return false;
 
-	while (num_entries) {
-		struct i915_page_directory *pd;
-		struct i915_page_table *pt;
+	GEM_BUG_ON(pte_end > GEN8_PTES);
 
-		if (WARN_ON(!pdp->page_directory[pdpe]))
-			break;
+	bitmap_clear(pt->used_ptes, pte, num_entries);
+
+	if (bitmap_empty(pt->used_ptes, GEN8_PTES)) {
+		free_pt(to_i915(vm->dev), pt);
+		return true;
+	}
 
-		pd = pdp->page_directory[pdpe];
+	pt_vaddr = kmap_px(pt);
 
+	while (pte < pte_end)
+		pt_vaddr[pte++] = scratch_pte;
+
+	kunmap_px(ppgtt, pt_vaddr);
+
+	return false;
+}
+
+/* Removes entries from a single page dir, releasing it if it's empty.
+ * Caller can use the return value to update higher-level entries
+ */
+static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
+				struct i915_page_directory *pd,
+				uint64_t start,
+				uint64_t length)
+{
+	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
+	struct i915_page_table *pt;
+	uint64_t pde;
+	gen8_pde_t *pde_vaddr;
+	gen8_pde_t scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt),
+						 I915_CACHE_LLC);
+
+	gen8_for_each_pde(pt, pd, start, length, pde) {
 		if (WARN_ON(!pd->page_table[pde]))
 			break;
 
-		pt = pd->page_table[pde];
+		if (gen8_ppgtt_clear_pt(vm, pt, start, length)) {
+			__clear_bit(pde, pd->used_pdes);
+			pde_vaddr = kmap_px(pd);
+			pde_vaddr[pde] = scratch_pde;
+			kunmap_px(ppgtt, pde_vaddr);
+		}
+	}
 
-		if (WARN_ON(!px_page(pt)))
-			break;
+	if (bitmap_empty(pd->used_pdes, I915_PDES)) {
+		free_pd(to_i915(vm->dev), pd);
+		return true;
+	}
 
-		last_pte = pte + num_entries;
-		if (last_pte > GEN8_PTES)
-			last_pte = GEN8_PTES;
+	return false;
+}
 
-		pt_vaddr = kmap_px(pt);
+/* Removes entries from a single page dir pointer, releasing it if it's empty.
+ * Caller can use the return value to update higher-level entries
+ */
+static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
+				 struct i915_page_directory_pointer *pdp,
+				 uint64_t start,
+				 uint64_t length)
+{
+	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
+	struct i915_page_directory *pd;
+	uint64_t pdpe;
+	gen8_ppgtt_pdpe_t *pdpe_vaddr;
+	gen8_ppgtt_pdpe_t scratch_pdpe =
+		gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
 
-		for (i = pte; i < last_pte; i++) {
-			pt_vaddr[i] = scratch_pte;
-			num_entries--;
+	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
+		if (WARN_ON(!pdp->page_directory[pdpe]))
+			break;
+
+		if (gen8_ppgtt_clear_pd(vm, pd, start, length)) {
+			__clear_bit(pdpe, pdp->used_pdpes);
+			if (USES_FULL_48BIT_PPGTT(dev_priv)) {
+				pdpe_vaddr = kmap_px(pdp);
+				pdpe_vaddr[pdpe] = scratch_pdpe;
+				kunmap_px(ppgtt, pdpe_vaddr);
+			}
 		}
+	}
 
-		kunmap_px(ppgtt, pt_vaddr);
+	mark_tlbs_dirty(ppgtt);
 
-		pte = 0;
-		if (++pde == I915_PDES) {
-			if (++pdpe == I915_PDPES_PER_PDP(vm->dev))
-				break;
-			pde = 0;
-		}
+	if (USES_FULL_48BIT_PPGTT(dev_priv) &&
+	    bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv))) {
+		free_pdp(dev_priv, pdp);
+		return true;
 	}
+
+	return false;
 }
 
-static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
-				   uint64_t start,
-				   uint64_t length,
-				   bool use_scratch)
+/* Removes entries from a single pml4.
+ * This is the top-level structure in 4-level page tables used on gen8+.
+ * Empty entries are always scratch pml4e.
+ */
+static void gen8_ppgtt_clear_pml4(struct i915_address_space *vm,
+				  struct i915_pml4 *pml4,
+				  uint64_t start,
+				  uint64_t length)
 {
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
-	gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
-						 I915_CACHE_LLC, use_scratch);
+	struct i915_page_directory_pointer *pdp;
+	uint64_t pml4e;
+	gen8_ppgtt_pml4e_t *pml4e_vaddr;
+	gen8_ppgtt_pml4e_t scratch_pml4e =
+		gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC);
 
-	if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
-		gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length,
-					   scratch_pte);
-	} else {
-		uint64_t pml4e;
-		struct i915_page_directory_pointer *pdp;
+	GEM_BUG_ON(!USES_FULL_48BIT_PPGTT(to_i915(vm->dev)));
 
-		gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, pml4e) {
-			gen8_ppgtt_clear_pte_range(vm, pdp, start, length,
-						   scratch_pte);
+	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
+		if (WARN_ON(!pml4->pdps[pml4e]))
+			break;
+
+		if (gen8_ppgtt_clear_pdp(vm, pdp, start, length)) {
+			__clear_bit(pml4e, pml4->used_pml4es);
+			pml4e_vaddr = kmap_px(pml4);
+			pml4e_vaddr[pml4e] = scratch_pml4e;
+			kunmap_px(ppgtt, pml4e_vaddr);
 		}
 	}
 }
 
+static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
+				   uint64_t start, uint64_t length)
+{
+	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
+
+	if (USES_FULL_48BIT_PPGTT(to_i915(vm->dev)))
+		gen8_ppgtt_clear_pml4(vm, &ppgtt->pml4, start, length);
+	else
+		gen8_ppgtt_clear_pdp(vm, &ppgtt->pdp, start, length);
+}
+
 static void
 gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm,
 			      struct i915_page_directory_pointer *pdp,
@@ -809,12 +893,12 @@ gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm,
 
 		pt_vaddr[pte] =
 			gen8_pte_encode(sg_page_iter_dma_address(sg_iter),
-					cache_level, true);
+					cache_level);
 		if (++pte == GEN8_PTES) {
 			kunmap_px(ppgtt, pt_vaddr);
 			pt_vaddr = NULL;
 			if (++pde == I915_PDES) {
-				if (++pdpe == I915_PDPES_PER_PDP(vm->dev))
+				if (++pdpe == I915_PDPES_PER_PDP(to_i915(vm->dev)))
 					break;
 				pde = 0;
 			}
@@ -837,7 +921,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 
 	__sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0);
 
-	if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
+	if (!USES_FULL_48BIT_PPGTT(to_i915(vm->dev))) {
 		gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start,
 					      cache_level);
 	} else {
@@ -852,7 +936,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 	}
 }
 
-static void gen8_free_page_tables(struct drm_device *dev,
+static void gen8_free_page_tables(struct drm_i915_private *dev_priv,
 				  struct i915_page_directory *pd)
 {
 	int i;
@@ -864,34 +948,34 @@ static void gen8_free_page_tables(struct drm_device *dev,
 		if (WARN_ON(!pd->page_table[i]))
 			continue;
 
-		free_pt(dev, pd->page_table[i]);
+		free_pt(dev_priv, pd->page_table[i]);
 		pd->page_table[i] = NULL;
 	}
 }
 
 static int gen8_init_scratch(struct i915_address_space *vm)
 {
-	struct drm_device *dev = vm->dev;
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	int ret;
 
-	ret = setup_scratch_page(dev, &vm->scratch_page, I915_GFP_DMA);
+	ret = setup_scratch_page(dev_priv, &vm->scratch_page, I915_GFP_DMA);
 	if (ret)
 		return ret;
 
-	vm->scratch_pt = alloc_pt(dev);
+	vm->scratch_pt = alloc_pt(dev_priv);
 	if (IS_ERR(vm->scratch_pt)) {
 		ret = PTR_ERR(vm->scratch_pt);
 		goto free_scratch_page;
 	}
 
-	vm->scratch_pd = alloc_pd(dev);
+	vm->scratch_pd = alloc_pd(dev_priv);
 	if (IS_ERR(vm->scratch_pd)) {
 		ret = PTR_ERR(vm->scratch_pd);
 		goto free_pt;
 	}
 
-	if (USES_FULL_48BIT_PPGTT(dev)) {
-		vm->scratch_pdp = alloc_pdp(dev);
+	if (USES_FULL_48BIT_PPGTT(dev_priv)) {
+		vm->scratch_pdp = alloc_pdp(dev_priv);
 		if (IS_ERR(vm->scratch_pdp)) {
 			ret = PTR_ERR(vm->scratch_pdp);
 			goto free_pd;
@@ -900,17 +984,17 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 
 	gen8_initialize_pt(vm, vm->scratch_pt);
 	gen8_initialize_pd(vm, vm->scratch_pd);
-	if (USES_FULL_48BIT_PPGTT(dev))
+	if (USES_FULL_48BIT_PPGTT(dev_priv))
 		gen8_initialize_pdp(vm, vm->scratch_pdp);
 
 	return 0;
 
 free_pd:
-	free_pd(dev, vm->scratch_pd);
+	free_pd(dev_priv, vm->scratch_pd);
 free_pt:
-	free_pt(dev, vm->scratch_pt);
+	free_pt(dev_priv, vm->scratch_pt);
 free_scratch_page:
-	cleanup_scratch_page(dev, &vm->scratch_page);
+	cleanup_scratch_page(dev_priv, &vm->scratch_page);
 
 	return ret;
 }
@@ -948,54 +1032,56 @@ static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
 
 static void gen8_free_scratch(struct i915_address_space *vm)
 {
-	struct drm_device *dev = vm->dev;
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 
-	if (USES_FULL_48BIT_PPGTT(dev))
-		free_pdp(dev, vm->scratch_pdp);
-	free_pd(dev, vm->scratch_pd);
-	free_pt(dev, vm->scratch_pt);
-	cleanup_scratch_page(dev, &vm->scratch_page);
+	if (USES_FULL_48BIT_PPGTT(dev_priv))
+		free_pdp(dev_priv, vm->scratch_pdp);
+	free_pd(dev_priv, vm->scratch_pd);
+	free_pt(dev_priv, vm->scratch_pt);
+	cleanup_scratch_page(dev_priv, &vm->scratch_page);
 }
 
-static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev,
+static void gen8_ppgtt_cleanup_3lvl(struct drm_i915_private *dev_priv,
 				    struct i915_page_directory_pointer *pdp)
 {
 	int i;
 
-	for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) {
+	for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev_priv)) {
 		if (WARN_ON(!pdp->page_directory[i]))
 			continue;
 
-		gen8_free_page_tables(dev, pdp->page_directory[i]);
-		free_pd(dev, pdp->page_directory[i]);
+		gen8_free_page_tables(dev_priv, pdp->page_directory[i]);
+		free_pd(dev_priv, pdp->page_directory[i]);
 	}
 
-	free_pdp(dev, pdp);
+	free_pdp(dev_priv, pdp);
 }
 
 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
 {
+	struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
 	int i;
 
 	for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) {
 		if (WARN_ON(!ppgtt->pml4.pdps[i]))
 			continue;
 
-		gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, ppgtt->pml4.pdps[i]);
+		gen8_ppgtt_cleanup_3lvl(dev_priv, ppgtt->pml4.pdps[i]);
 	}
 
-	cleanup_px(ppgtt->base.dev, &ppgtt->pml4);
+	cleanup_px(dev_priv, &ppgtt->pml4);
 }
 
 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 {
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 
-	if (intel_vgpu_active(to_i915(vm->dev)))
+	if (intel_vgpu_active(dev_priv))
 		gen8_ppgtt_notify_vgt(ppgtt, false);
 
-	if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev))
-		gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp);
+	if (!USES_FULL_48BIT_PPGTT(dev_priv))
+		gen8_ppgtt_cleanup_3lvl(dev_priv, &ppgtt->pdp);
 	else
 		gen8_ppgtt_cleanup_4lvl(ppgtt);
 
@@ -1026,7 +1112,7 @@ static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
 				     uint64_t length,
 				     unsigned long *new_pts)
 {
-	struct drm_device *dev = vm->dev;
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	struct i915_page_table *pt;
 	uint32_t pde;
 
@@ -1038,7 +1124,7 @@ static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
 			continue;
 		}
 
-		pt = alloc_pt(dev);
+		pt = alloc_pt(dev_priv);
 		if (IS_ERR(pt))
 			goto unwind_out;
 
@@ -1052,7 +1138,7 @@ static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
 
 unwind_out:
 	for_each_set_bit(pde, new_pts, I915_PDES)
-		free_pt(dev, pd->page_table[pde]);
+		free_pt(dev_priv, pd->page_table[pde]);
 
 	return -ENOMEM;
 }
@@ -1087,10 +1173,10 @@ gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
 				  uint64_t length,
 				  unsigned long *new_pds)
 {
-	struct drm_device *dev = vm->dev;
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	struct i915_page_directory *pd;
 	uint32_t pdpe;
-	uint32_t pdpes = I915_PDPES_PER_PDP(dev);
+	uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv);
 
 	WARN_ON(!bitmap_empty(new_pds, pdpes));
 
@@ -1098,7 +1184,7 @@ gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
 		if (test_bit(pdpe, pdp->used_pdpes))
 			continue;
 
-		pd = alloc_pd(dev);
+		pd = alloc_pd(dev_priv);
 		if (IS_ERR(pd))
 			goto unwind_out;
 
@@ -1112,7 +1198,7 @@ gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
 
 unwind_out:
 	for_each_set_bit(pdpe, new_pds, pdpes)
-		free_pd(dev, pdp->page_directory[pdpe]);
+		free_pd(dev_priv, pdp->page_directory[pdpe]);
 
 	return -ENOMEM;
 }
@@ -1140,7 +1226,7 @@ gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
 				  uint64_t length,
 				  unsigned long *new_pdps)
 {
-	struct drm_device *dev = vm->dev;
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	struct i915_page_directory_pointer *pdp;
 	uint32_t pml4e;
 
@@ -1148,7 +1234,7 @@ gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
 
 	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
 		if (!test_bit(pml4e, pml4->used_pml4es)) {
-			pdp = alloc_pdp(dev);
+			pdp = alloc_pdp(dev_priv);
 			if (IS_ERR(pdp))
 				goto unwind_out;
 
@@ -1166,7 +1252,7 @@ gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
 
 unwind_out:
 	for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
-		free_pdp(dev, pml4->pdps[pml4e]);
+		free_pdp(dev_priv, pml4->pdps[pml4e]);
 
 	return -ENOMEM;
 }
@@ -1208,16 +1294,6 @@ err_out:
 	return -ENOMEM;
 }
 
-/* PDE TLBs are a pain to invalidate on GEN8+. When we modify
- * the page table structures, we mark them dirty so that
- * context switching/execlist queuing code takes extra steps
- * to ensure that tlbs are flushed.
- */
-static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
-{
-	ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask;
-}
-
 static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
 				    struct i915_page_directory_pointer *pdp,
 				    uint64_t start,
@@ -1225,12 +1301,12 @@ static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
 {
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 	unsigned long *new_page_dirs, *new_page_tables;
-	struct drm_device *dev = vm->dev;
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	struct i915_page_directory *pd;
 	const uint64_t orig_start = start;
 	const uint64_t orig_length = length;
 	uint32_t pdpe;
-	uint32_t pdpes = I915_PDPES_PER_PDP(dev);
+	uint32_t pdpes = I915_PDPES_PER_PDP(dev_priv);
 	int ret;
 
 	/* Wrap is never okay since we can only represent 48b, and we don't
@@ -1318,11 +1394,12 @@ err_out:
 
 		for_each_set_bit(temp, new_page_tables + pdpe *
 				BITS_TO_LONGS(I915_PDES), I915_PDES)
-			free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]);
+			free_pt(dev_priv,
+				pdp->page_directory[pdpe]->page_table[temp]);
 	}
 
 	for_each_set_bit(pdpe, new_page_dirs, pdpes)
-		free_pd(dev, pdp->page_directory[pdpe]);
+		free_pd(dev_priv, pdp->page_directory[pdpe]);
 
 	free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
 	mark_tlbs_dirty(ppgtt);
@@ -1373,7 +1450,7 @@ static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
 
 err_out:
 	for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
-		gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]);
+		gen8_ppgtt_cleanup_3lvl(to_i915(vm->dev), pml4->pdps[pml4e]);
 
 	return ret;
 }
@@ -1383,7 +1460,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,
 {
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 
-	if (USES_FULL_48BIT_PPGTT(vm->dev))
+	if (USES_FULL_48BIT_PPGTT(to_i915(vm->dev)))
 		return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length);
 	else
 		return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
@@ -1452,9 +1529,9 @@ static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 	uint64_t start = ppgtt->base.start;
 	uint64_t length = ppgtt->base.total;
 	gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
-						 I915_CACHE_LLC, true);
+						 I915_CACHE_LLC);
 
-	if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
+	if (!USES_FULL_48BIT_PPGTT(to_i915(vm->dev))) {
 		gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m);
 	} else {
 		uint64_t pml4e;
@@ -1474,7 +1551,7 @@ static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt)
 {
 	unsigned long *new_page_dirs, *new_page_tables;
-	uint32_t pdpes = I915_PDPES_PER_PDP(dev);
+	uint32_t pdpes = I915_PDPES_PER_PDP(to_i915(ppgtt->base.dev));
 	int ret;
 
 	/* We allocate temp bitmap for page tables for no gain
@@ -1507,6 +1584,7 @@ static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt)
  */
 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 {
+	struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
 	int ret;
 
 	ret = gen8_init_scratch(&ppgtt->base);
@@ -1522,8 +1600,8 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 	ppgtt->base.bind_vma = ppgtt_bind_vma;
 	ppgtt->debug_dump = gen8_dump_ppgtt;
 
-	if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
-		ret = setup_px(ppgtt->base.dev, &ppgtt->pml4);
+	if (USES_FULL_48BIT_PPGTT(dev_priv)) {
+		ret = setup_px(dev_priv, &ppgtt->pml4);
 		if (ret)
 			goto free_scratch;
 
@@ -1532,7 +1610,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 		ppgtt->base.total = 1ULL << 48;
 		ppgtt->switch_mm = gen8_48b_mm_switch;
 	} else {
-		ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp);
+		ret = __pdp_init(dev_priv, &ppgtt->pdp);
 		if (ret)
 			goto free_scratch;
 
@@ -1542,14 +1620,14 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 							      0, 0,
 							      GEN8_PML4E_SHIFT);
 
-		if (intel_vgpu_active(to_i915(ppgtt->base.dev))) {
+		if (intel_vgpu_active(dev_priv)) {
 			ret = gen8_preallocate_top_level_pdps(ppgtt);
 			if (ret)
 				goto free_scratch;
 		}
 	}
 
-	if (intel_vgpu_active(to_i915(ppgtt->base.dev)))
+	if (intel_vgpu_active(dev_priv))
 		gen8_ppgtt_notify_vgt(ppgtt, true);
 
 	return 0;
@@ -1569,7 +1647,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 	uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
 
 	scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
-				     I915_CACHE_LLC, true, 0);
+				     I915_CACHE_LLC, 0);
 
 	gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) {
 		u32 expected;
@@ -1724,29 +1802,30 @@ static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
 	return 0;
 }
 
-static void gen8_ppgtt_enable(struct drm_device *dev)
+static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 
-	for_each_engine(engine, dev_priv) {
-		u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0;
+	for_each_engine(engine, dev_priv, id) {
+		u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ?
+				 GEN8_GFX_PPGTT_48B : 0;
 		I915_WRITE(RING_MODE_GEN7(engine),
 			   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
 	}
 }
 
-static void gen7_ppgtt_enable(struct drm_device *dev)
+static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_engine_cs *engine;
 	uint32_t ecochk, ecobits;
+	enum intel_engine_id id;
 
 	ecobits = I915_READ(GAC_ECO_BITS);
 	I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
 
 	ecochk = I915_READ(GAM_ECOCHK);
-	if (IS_HASWELL(dev)) {
+	if (IS_HASWELL(dev_priv)) {
 		ecochk |= ECOCHK_PPGTT_WB_HSW;
 	} else {
 		ecochk |= ECOCHK_PPGTT_LLC_IVB;
@@ -1754,16 +1833,15 @@ static void gen7_ppgtt_enable(struct drm_device *dev)
 	}
 	I915_WRITE(GAM_ECOCHK, ecochk);
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		/* GFX_MODE is per-ring on gen7+ */
 		I915_WRITE(RING_MODE_GEN7(engine),
 			   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 	}
 }
 
-static void gen6_ppgtt_enable(struct drm_device *dev)
+static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t ecochk, gab_ctl, ecobits;
 
 	ecobits = I915_READ(GAC_ECO_BITS);
@@ -1782,8 +1860,7 @@ static void gen6_ppgtt_enable(struct drm_device *dev)
 /* PPGTT support for Sandybdrige/Gen6 and later */
 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 				   uint64_t start,
-				   uint64_t length,
-				   bool use_scratch)
+				   uint64_t length)
 {
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 	gen6_pte_t *pt_vaddr, scratch_pte;
@@ -1794,7 +1871,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 	unsigned last_pte, i;
 
 	scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
-				     I915_CACHE_LLC, true, 0);
+				     I915_CACHE_LLC, 0);
 
 	while (num_entries) {
 		last_pte = first_pte + num_entries;
@@ -1832,7 +1909,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 			pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
 
 		pt_vaddr[act_pte] =
-			vm->pte_encode(addr, cache_level, true, flags);
+			vm->pte_encode(addr, cache_level, flags);
 
 		if (++act_pte == GEN6_PTES) {
 			kunmap_px(ppgtt, pt_vaddr);
@@ -1850,8 +1927,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
 			       uint64_t start_in, uint64_t length_in)
 {
 	DECLARE_BITMAP(new_page_tables, I915_PDES);
-	struct drm_device *dev = vm->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 	struct i915_page_table *pt;
@@ -1881,7 +1957,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
 		/* We've already allocated a page table */
 		WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES));
 
-		pt = alloc_pt(dev);
+		pt = alloc_pt(dev_priv);
 		if (IS_ERR(pt)) {
 			ret = PTR_ERR(pt);
 			goto unwind_out;
@@ -1929,7 +2005,7 @@ unwind_out:
 		struct i915_page_table *pt = ppgtt->pd.page_table[pde];
 
 		ppgtt->pd.page_table[pde] = vm->scratch_pt;
-		free_pt(vm->dev, pt);
+		free_pt(dev_priv, pt);
 	}
 
 	mark_tlbs_dirty(ppgtt);
@@ -1938,16 +2014,16 @@ unwind_out:
 
 static int gen6_init_scratch(struct i915_address_space *vm)
 {
-	struct drm_device *dev = vm->dev;
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	int ret;
 
-	ret = setup_scratch_page(dev, &vm->scratch_page, I915_GFP_DMA);
+	ret = setup_scratch_page(dev_priv, &vm->scratch_page, I915_GFP_DMA);
 	if (ret)
 		return ret;
 
-	vm->scratch_pt = alloc_pt(dev);
+	vm->scratch_pt = alloc_pt(dev_priv);
 	if (IS_ERR(vm->scratch_pt)) {
-		cleanup_scratch_page(dev, &vm->scratch_page);
+		cleanup_scratch_page(dev_priv, &vm->scratch_page);
 		return PTR_ERR(vm->scratch_pt);
 	}
 
@@ -1958,17 +2034,17 @@ static int gen6_init_scratch(struct i915_address_space *vm)
 
 static void gen6_free_scratch(struct i915_address_space *vm)
 {
-	struct drm_device *dev = vm->dev;
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 
-	free_pt(dev, vm->scratch_pt);
-	cleanup_scratch_page(dev, &vm->scratch_page);
+	free_pt(dev_priv, vm->scratch_pt);
+	cleanup_scratch_page(dev_priv, &vm->scratch_page);
 }
 
 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 {
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 	struct i915_page_directory *pd = &ppgtt->pd;
-	struct drm_device *dev = vm->dev;
+	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	struct i915_page_table *pt;
 	uint32_t pde;
 
@@ -1976,7 +2052,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 
 	gen6_for_all_pdes(pt, pd, pde)
 		if (pt != vm->scratch_pt)
-			free_pt(dev, pt);
+			free_pt(dev_priv, pt);
 
 	gen6_free_scratch(vm);
 }
@@ -1984,8 +2060,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
 {
 	struct i915_address_space *vm = &ppgtt->base;
-	struct drm_device *dev = ppgtt->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	bool retried = false;
 	int ret;
@@ -2050,17 +2125,16 @@ static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
 
 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 {
-	struct drm_device *dev = ppgtt->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	int ret;
 
 	ppgtt->base.pte_encode = ggtt->base.pte_encode;
-	if (intel_vgpu_active(dev_priv) || IS_GEN6(dev))
+	if (intel_vgpu_active(dev_priv) || IS_GEN6(dev_priv))
 		ppgtt->switch_mm = gen6_mm_switch;
-	else if (IS_HASWELL(dev))
+	else if (IS_HASWELL(dev_priv))
 		ppgtt->switch_mm = hsw_mm_switch;
-	else if (IS_GEN7(dev))
+	else if (IS_GEN7(dev_priv))
 		ppgtt->switch_mm = gen7_mm_switch;
 	else
 		BUG();
@@ -2111,8 +2185,10 @@ static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
 }
 
 static void i915_address_space_init(struct i915_address_space *vm,
-				    struct drm_i915_private *dev_priv)
+				    struct drm_i915_private *dev_priv,
+				    const char *name)
 {
+	i915_gem_timeline_init(dev_priv, &vm->timeline, name);
 	drm_mm_init(&vm->mm, vm->start, vm->total);
 	INIT_LIST_HEAD(&vm->active_list);
 	INIT_LIST_HEAD(&vm->inactive_list);
@@ -2120,44 +2196,50 @@ static void i915_address_space_init(struct i915_address_space *vm,
 	list_add_tail(&vm->global_link, &dev_priv->vm_list);
 }
 
-static void gtt_write_workarounds(struct drm_device *dev)
+static void i915_address_space_fini(struct i915_address_space *vm)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	i915_gem_timeline_fini(&vm->timeline);
+	drm_mm_takedown(&vm->mm);
+	list_del(&vm->global_link);
+}
 
+static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
+{
 	/* This function is for gtt related workarounds. This function is
 	 * called on driver load and after a GPU reset, so you can place
 	 * workarounds here even if they get overwritten by GPU reset.
 	 */
 	/* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt */
-	if (IS_BROADWELL(dev))
+	if (IS_BROADWELL(dev_priv))
 		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
-	else if (IS_CHERRYVIEW(dev))
+	else if (IS_CHERRYVIEW(dev_priv))
 		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
-	else if (IS_SKYLAKE(dev))
+	else if (IS_SKYLAKE(dev_priv))
 		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
-	else if (IS_BROXTON(dev))
+	else if (IS_BROXTON(dev_priv))
 		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
 }
 
 static int i915_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
 			   struct drm_i915_private *dev_priv,
-			   struct drm_i915_file_private *file_priv)
+			   struct drm_i915_file_private *file_priv,
+			   const char *name)
 {
 	int ret;
 
 	ret = __hw_ppgtt_init(ppgtt, dev_priv);
 	if (ret == 0) {
 		kref_init(&ppgtt->ref);
-		i915_address_space_init(&ppgtt->base, dev_priv);
+		i915_address_space_init(&ppgtt->base, dev_priv, name);
 		ppgtt->base.file = file_priv;
 	}
 
 	return ret;
 }
 
-int i915_ppgtt_init_hw(struct drm_device *dev)
+int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv)
 {
-	gtt_write_workarounds(dev);
+	gtt_write_workarounds(dev_priv);
 
 	/* In the case of execlists, PPGTT is enabled by the context descriptor
 	 * and the PDPs are contained within the context itself.  We don't
@@ -2165,24 +2247,25 @@ int i915_ppgtt_init_hw(struct drm_device *dev)
 	if (i915.enable_execlists)
 		return 0;
 
-	if (!USES_PPGTT(dev))
+	if (!USES_PPGTT(dev_priv))
 		return 0;
 
-	if (IS_GEN6(dev))
-		gen6_ppgtt_enable(dev);
-	else if (IS_GEN7(dev))
-		gen7_ppgtt_enable(dev);
-	else if (INTEL_INFO(dev)->gen >= 8)
-		gen8_ppgtt_enable(dev);
+	if (IS_GEN6(dev_priv))
+		gen6_ppgtt_enable(dev_priv);
+	else if (IS_GEN7(dev_priv))
+		gen7_ppgtt_enable(dev_priv);
+	else if (INTEL_GEN(dev_priv) >= 8)
+		gen8_ppgtt_enable(dev_priv);
 	else
-		MISSING_CASE(INTEL_INFO(dev)->gen);
+		MISSING_CASE(INTEL_GEN(dev_priv));
 
 	return 0;
 }
 
 struct i915_hw_ppgtt *
 i915_ppgtt_create(struct drm_i915_private *dev_priv,
-		  struct drm_i915_file_private *fpriv)
+		  struct drm_i915_file_private *fpriv,
+		  const char *name)
 {
 	struct i915_hw_ppgtt *ppgtt;
 	int ret;
@@ -2191,7 +2274,7 @@ i915_ppgtt_create(struct drm_i915_private *dev_priv,
 	if (!ppgtt)
 		return ERR_PTR(-ENOMEM);
 
-	ret = i915_ppgtt_init(ppgtt, dev_priv, fpriv);
+	ret = i915_ppgtt_init(ppgtt, dev_priv, fpriv, name);
 	if (ret) {
 		kfree(ppgtt);
 		return ERR_PTR(ret);
@@ -2202,7 +2285,7 @@ i915_ppgtt_create(struct drm_i915_private *dev_priv,
 	return ppgtt;
 }
 
-void  i915_ppgtt_release(struct kref *kref)
+void i915_ppgtt_release(struct kref *kref)
 {
 	struct i915_hw_ppgtt *ppgtt =
 		container_of(kref, struct i915_hw_ppgtt, ref);
@@ -2214,8 +2297,7 @@ void  i915_ppgtt_release(struct kref *kref)
 	WARN_ON(!list_empty(&ppgtt->base.inactive_list));
 	WARN_ON(!list_empty(&ppgtt->base.unbound_list));
 
-	list_del(&ppgtt->base.global_link);
-	drm_mm_takedown(&ppgtt->base.mm);
+	i915_address_space_fini(&ppgtt->base);
 
 	ppgtt->base.cleanup(&ppgtt->base);
 	kfree(ppgtt);
@@ -2239,11 +2321,12 @@ static bool needs_idle_maps(struct drm_i915_private *dev_priv)
 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 
 	if (INTEL_INFO(dev_priv)->gen < 6)
 		return;
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		u32 fault_reg;
 		fault_reg = I915_READ(RING_FAULT_REG(engine));
 		if (fault_reg & RING_FAULT_VALID) {
@@ -2260,7 +2343,10 @@ void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
 				   fault_reg & ~RING_FAULT_VALID);
 		}
 	}
-	POSTING_READ(RING_FAULT_REG(&dev_priv->engine[RCS]));
+
+	/* Engine specific init may not have been done till this point. */
+	if (dev_priv->engine[RCS])
+		POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
 }
 
 static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
@@ -2273,33 +2359,32 @@ static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
 	}
 }
 
-void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
+void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 
 	/* Don't bother messing with faults pre GEN6 as we have little
 	 * documentation supporting that it's a good idea.
 	 */
-	if (INTEL_INFO(dev)->gen < 6)
+	if (INTEL_GEN(dev_priv) < 6)
 		return;
 
 	i915_check_and_clear_faults(dev_priv);
 
-	ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total,
-			     true);
+	ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total);
 
 	i915_ggtt_flush(dev_priv);
 }
 
-int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
+int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
+			       struct sg_table *pages)
 {
-	if (!dma_map_sg(&obj->base.dev->pdev->dev,
-			obj->pages->sgl, obj->pages->nents,
-			PCI_DMA_BIDIRECTIONAL))
-		return -ENOSPC;
+	if (dma_map_sg(&obj->base.dev->pdev->dev,
+		       pages->sgl, pages->nents,
+		       PCI_DMA_BIDIRECTIONAL))
+		return 0;
 
-	return 0;
+	return -ENOSPC;
 }
 
 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
@@ -2317,16 +2402,11 @@ static void gen8_ggtt_insert_page(struct i915_address_space *vm,
 	gen8_pte_t __iomem *pte =
 		(gen8_pte_t __iomem *)dev_priv->ggtt.gsm +
 		(offset >> PAGE_SHIFT);
-	int rpm_atomic_seq;
-
-	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
-	gen8_set_pte(pte, gen8_pte_encode(addr, level, true));
+	gen8_set_pte(pte, gen8_pte_encode(addr, level));
 
 	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 	POSTING_READ(GFX_FLSH_CNTL_GEN6);
-
-	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
@@ -2340,15 +2420,12 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 	gen8_pte_t __iomem *gtt_entries;
 	gen8_pte_t gtt_entry;
 	dma_addr_t addr;
-	int rpm_atomic_seq;
 	int i = 0;
 
-	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
-
 	gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT);
 
 	for_each_sgt_dma(addr, sgt_iter, st) {
-		gtt_entry = gen8_pte_encode(addr, level, true);
+		gtt_entry = gen8_pte_encode(addr, level);
 		gen8_set_pte(&gtt_entries[i++], gtt_entry);
 	}
 
@@ -2368,8 +2445,6 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 	 */
 	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 	POSTING_READ(GFX_FLSH_CNTL_GEN6);
-
-	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
 struct insert_entries {
@@ -2408,16 +2483,11 @@ static void gen6_ggtt_insert_page(struct i915_address_space *vm,
 	gen6_pte_t __iomem *pte =
 		(gen6_pte_t __iomem *)dev_priv->ggtt.gsm +
 		(offset >> PAGE_SHIFT);
-	int rpm_atomic_seq;
-
-	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
-	iowrite32(vm->pte_encode(addr, level, true, flags), pte);
+	iowrite32(vm->pte_encode(addr, level, flags), pte);
 
 	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 	POSTING_READ(GFX_FLSH_CNTL_GEN6);
-
-	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
 /*
@@ -2437,15 +2507,12 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 	gen6_pte_t __iomem *gtt_entries;
 	gen6_pte_t gtt_entry;
 	dma_addr_t addr;
-	int rpm_atomic_seq;
 	int i = 0;
 
-	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
-
 	gtt_entries = (gen6_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT);
 
 	for_each_sgt_dma(addr, sgt_iter, st) {
-		gtt_entry = vm->pte_encode(addr, level, true, flags);
+		gtt_entry = vm->pte_encode(addr, level, flags);
 		iowrite32(gtt_entry, &gtt_entries[i++]);
 	}
 
@@ -2464,23 +2531,16 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 	 */
 	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 	POSTING_READ(GFX_FLSH_CNTL_GEN6);
-
-	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
 static void nop_clear_range(struct i915_address_space *vm,
-			    uint64_t start,
-			    uint64_t length,
-			    bool use_scratch)
+			    uint64_t start, uint64_t length)
 {
 }
 
 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
-				  uint64_t start,
-				  uint64_t length,
-				  bool use_scratch)
+				  uint64_t start, uint64_t length)
 {
-	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 	unsigned first_entry = start >> PAGE_SHIFT;
 	unsigned num_entries = length >> PAGE_SHIFT;
@@ -2488,9 +2548,6 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 		(gen8_pte_t __iomem *)ggtt->gsm + first_entry;
 	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
 	int i;
-	int rpm_atomic_seq;
-
-	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
 	if (WARN(num_entries > max_entries,
 		 "First entry = %d; Num entries = %d (max=%d)\n",
@@ -2498,21 +2555,16 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 		num_entries = max_entries;
 
 	scratch_pte = gen8_pte_encode(vm->scratch_page.daddr,
-				      I915_CACHE_LLC,
-				      use_scratch);
+				      I915_CACHE_LLC);
 	for (i = 0; i < num_entries; i++)
 		gen8_set_pte(&gtt_base[i], scratch_pte);
 	readl(gtt_base);
-
-	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 				  uint64_t start,
-				  uint64_t length,
-				  bool use_scratch)
+				  uint64_t length)
 {
-	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 	unsigned first_entry = start >> PAGE_SHIFT;
 	unsigned num_entries = length >> PAGE_SHIFT;
@@ -2520,9 +2572,6 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 		(gen6_pte_t __iomem *)ggtt->gsm + first_entry;
 	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
 	int i;
-	int rpm_atomic_seq;
-
-	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
 	if (WARN(num_entries > max_entries,
 		 "First entry = %d; Num entries = %d (max=%d)\n",
@@ -2530,13 +2579,11 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 		num_entries = max_entries;
 
 	scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
-				     I915_CACHE_LLC, use_scratch, 0);
+				     I915_CACHE_LLC, 0);
 
 	for (i = 0; i < num_entries; i++)
 		iowrite32(scratch_pte, &gtt_base[i]);
 	readl(gtt_base);
-
-	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
 static void i915_ggtt_insert_page(struct i915_address_space *vm,
@@ -2545,16 +2592,10 @@ static void i915_ggtt_insert_page(struct i915_address_space *vm,
 				  enum i915_cache_level cache_level,
 				  u32 unused)
 {
-	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
 		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
-	int rpm_atomic_seq;
-
-	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
 	intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
-
-	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
@@ -2562,40 +2603,25 @@ static void i915_ggtt_insert_entries(struct i915_address_space *vm,
 				     uint64_t start,
 				     enum i915_cache_level cache_level, u32 unused)
 {
-	struct drm_i915_private *dev_priv = to_i915(vm->dev);
 	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
 		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
-	int rpm_atomic_seq;
-
-	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
 	intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags);
 
-	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
-
 }
 
 static void i915_ggtt_clear_range(struct i915_address_space *vm,
 				  uint64_t start,
-				  uint64_t length,
-				  bool unused)
+				  uint64_t length)
 {
-	struct drm_i915_private *dev_priv = to_i915(vm->dev);
-	unsigned first_entry = start >> PAGE_SHIFT;
-	unsigned num_entries = length >> PAGE_SHIFT;
-	int rpm_atomic_seq;
-
-	rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
-
-	intel_gtt_clear_range(first_entry, num_entries);
-
-	assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
+	intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
 }
 
 static int ggtt_bind_vma(struct i915_vma *vma,
 			 enum i915_cache_level cache_level,
 			 u32 flags)
 {
+	struct drm_i915_private *i915 = to_i915(vma->vm->dev);
 	struct drm_i915_gem_object *obj = vma->obj;
 	u32 pte_flags = 0;
 	int ret;
@@ -2608,8 +2634,10 @@ static int ggtt_bind_vma(struct i915_vma *vma,
 	if (obj->gt_ro)
 		pte_flags |= PTE_READ_ONLY;
 
+	intel_runtime_pm_get(i915);
 	vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
 				cache_level, pte_flags);
+	intel_runtime_pm_put(i915);
 
 	/*
 	 * Without aliasing PPGTT there's no difference between
@@ -2625,6 +2653,7 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma,
 				 enum i915_cache_level cache_level,
 				 u32 flags)
 {
+	struct drm_i915_private *i915 = to_i915(vma->vm->dev);
 	u32 pte_flags;
 	int ret;
 
@@ -2639,14 +2668,15 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma,
 
 
 	if (flags & I915_VMA_GLOBAL_BIND) {
+		intel_runtime_pm_get(i915);
 		vma->vm->insert_entries(vma->vm,
 					vma->pages, vma->node.start,
 					cache_level, pte_flags);
+		intel_runtime_pm_put(i915);
 	}
 
 	if (flags & I915_VMA_LOCAL_BIND) {
-		struct i915_hw_ppgtt *appgtt =
-			to_i915(vma->vm->dev)->mm.aliasing_ppgtt;
+		struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
 		appgtt->base.insert_entries(&appgtt->base,
 					    vma->pages, vma->node.start,
 					    cache_level, pte_flags);
@@ -2657,21 +2687,24 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma,
 
 static void ggtt_unbind_vma(struct i915_vma *vma)
 {
-	struct i915_hw_ppgtt *appgtt = to_i915(vma->vm->dev)->mm.aliasing_ppgtt;
+	struct drm_i915_private *i915 = to_i915(vma->vm->dev);
+	struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
 	const u64 size = min(vma->size, vma->node.size);
 
-	if (vma->flags & I915_VMA_GLOBAL_BIND)
+	if (vma->flags & I915_VMA_GLOBAL_BIND) {
+		intel_runtime_pm_get(i915);
 		vma->vm->clear_range(vma->vm,
-				     vma->node.start, size,
-				     true);
+				     vma->node.start, size);
+		intel_runtime_pm_put(i915);
+	}
 
 	if (vma->flags & I915_VMA_LOCAL_BIND && appgtt)
 		appgtt->base.clear_range(&appgtt->base,
-					 vma->node.start, size,
-					 true);
+					 vma->node.start, size);
 }
 
-void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
+void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
+			       struct sg_table *pages)
 {
 	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 	struct device *kdev = &dev_priv->drm.pdev->dev;
@@ -2685,8 +2718,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 		}
 	}
 
-	dma_unmap_sg(kdev, obj->pages->sgl, obj->pages->nents,
-		     PCI_DMA_BIDIRECTIONAL);
+	dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
 }
 
 static void i915_gtt_color_adjust(struct drm_mm_node *node,
@@ -2717,6 +2749,7 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
 	 */
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	unsigned long hole_start, hole_end;
+	struct i915_hw_ppgtt *ppgtt;
 	struct drm_mm_node *entry;
 	int ret;
 
@@ -2724,45 +2757,48 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
 	if (ret)
 		return ret;
 
+	/* Reserve a mappable slot for our lockless error capture */
+	ret = drm_mm_insert_node_in_range_generic(&ggtt->base.mm,
+						  &ggtt->error_capture,
+						  4096, 0, -1,
+						  0, ggtt->mappable_end,
+						  0, 0);
+	if (ret)
+		return ret;
+
 	/* Clear any non-preallocated blocks */
 	drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) {
 		DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
 			      hole_start, hole_end);
 		ggtt->base.clear_range(&ggtt->base, hole_start,
-				     hole_end - hole_start, true);
+				       hole_end - hole_start);
 	}
 
 	/* And finally clear the reserved guard page */
 	ggtt->base.clear_range(&ggtt->base,
-			       ggtt->base.total - PAGE_SIZE, PAGE_SIZE,
-			       true);
+			       ggtt->base.total - PAGE_SIZE, PAGE_SIZE);
 
 	if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) {
-		struct i915_hw_ppgtt *ppgtt;
-
 		ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
-		if (!ppgtt)
-			return -ENOMEM;
+		if (!ppgtt) {
+			ret = -ENOMEM;
+			goto err;
+		}
 
 		ret = __hw_ppgtt_init(ppgtt, dev_priv);
-		if (ret) {
-			kfree(ppgtt);
-			return ret;
-		}
+		if (ret)
+			goto err_ppgtt;
 
-		if (ppgtt->base.allocate_va_range)
+		if (ppgtt->base.allocate_va_range) {
 			ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0,
 							    ppgtt->base.total);
-		if (ret) {
-			ppgtt->base.cleanup(&ppgtt->base);
-			kfree(ppgtt);
-			return ret;
+			if (ret)
+				goto err_ppgtt_cleanup;
 		}
 
 		ppgtt->base.clear_range(&ppgtt->base,
 					ppgtt->base.start,
-					ppgtt->base.total,
-					true);
+					ppgtt->base.total);
 
 		dev_priv->mm.aliasing_ppgtt = ppgtt;
 		WARN_ON(ggtt->base.bind_vma != ggtt_bind_vma);
@@ -2770,6 +2806,14 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
 	}
 
 	return 0;
+
+err_ppgtt_cleanup:
+	ppgtt->base.cleanup(&ppgtt->base);
+err_ppgtt:
+	kfree(ppgtt);
+err:
+	drm_mm_remove_node(&ggtt->error_capture);
+	return ret;
 }
 
 /**
@@ -2788,11 +2832,15 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
 
 	i915_gem_cleanup_stolen(&dev_priv->drm);
 
+	if (drm_mm_node_allocated(&ggtt->error_capture))
+		drm_mm_remove_node(&ggtt->error_capture);
+
 	if (drm_mm_initialized(&ggtt->base.mm)) {
 		intel_vgt_deballoon(dev_priv);
 
-		drm_mm_takedown(&ggtt->base.mm);
-		list_del(&ggtt->base.global_link);
+		mutex_lock(&dev_priv->drm.struct_mutex);
+		i915_address_space_fini(&ggtt->base);
+		mutex_unlock(&dev_priv->drm.struct_mutex);
 	}
 
 	ggtt->base.cleanup(&ggtt->base);
@@ -2881,6 +2929,7 @@ static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
 
 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
 {
+	struct drm_i915_private *dev_priv = to_i915(ggtt->base.dev);
 	struct pci_dev *pdev = ggtt->base.dev->pdev;
 	phys_addr_t phys_addr;
 	int ret;
@@ -2895,7 +2944,7 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
 	 * resort to an uncached mapping. The WC issue is easily caught by the
 	 * readback check when writing GTT PTE entries.
 	 */
-	if (IS_BROXTON(ggtt->base.dev))
+	if (IS_BROXTON(dev_priv))
 		ggtt->gsm = ioremap_nocache(phys_addr, size);
 	else
 		ggtt->gsm = ioremap_wc(phys_addr, size);
@@ -2904,9 +2953,7 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
 		return -ENOMEM;
 	}
 
-	ret = setup_scratch_page(ggtt->base.dev,
-				 &ggtt->base.scratch_page,
-				 GFP_DMA32);
+	ret = setup_scratch_page(dev_priv, &ggtt->base.scratch_page, GFP_DMA32);
 	if (ret) {
 		DRM_ERROR("Scratch setup failed\n");
 		/* iounmap will also get called at remove, but meh */
@@ -2995,7 +3042,7 @@ static void gen6_gmch_remove(struct i915_address_space *vm)
 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 
 	iounmap(ggtt->gsm);
-	cleanup_scratch_page(vm->dev, &vm->scratch_page);
+	cleanup_scratch_page(to_i915(vm->dev), &vm->scratch_page);
 }
 
 static int gen8_gmch_probe(struct i915_ggtt *ggtt)
@@ -3190,11 +3237,13 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
 	/* Subtract the guard page before address space initialization to
 	 * shrink the range used by drm_mm.
 	 */
+	mutex_lock(&dev_priv->drm.struct_mutex);
 	ggtt->base.total -= PAGE_SIZE;
-	i915_address_space_init(&ggtt->base, dev_priv);
+	i915_address_space_init(&ggtt->base, dev_priv, "[global]");
 	ggtt->base.total += PAGE_SIZE;
 	if (!HAS_LLC(dev_priv))
 		ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
+	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	if (!io_mapping_init_wc(&dev_priv->ggtt.mappable,
 				dev_priv->ggtt.mappable_base,
@@ -3209,7 +3258,7 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
 	 * Initialise stolen early so that we may reserve preallocated
 	 * objects for the BIOS to KMS transition.
 	 */
-	ret = i915_gem_init_stolen(&dev_priv->drm);
+	ret = i915_gem_init_stolen(dev_priv);
 	if (ret)
 		goto out_gtt_cleanup;
 
@@ -3228,23 +3277,21 @@ int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
 	return 0;
 }
 
-void i915_gem_restore_gtt_mappings(struct drm_device *dev)
+void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	struct drm_i915_gem_object *obj, *on;
 
 	i915_check_and_clear_faults(dev_priv);
 
 	/* First fill our portion of the GTT with scratch pages */
-	ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total,
-			       true);
+	ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total);
 
 	ggtt->base.closed = true; /* skip rewriting PTE on VMA unbind */
 
 	/* clflush objects bound into the GGTT and rebind them. */
 	list_for_each_entry_safe(obj, on,
-				 &dev_priv->mm.bound_list, global_list) {
+				 &dev_priv->mm.bound_list, global_link) {
 		bool ggtt_bound = false;
 		struct i915_vma *vma;
 
@@ -3266,8 +3313,8 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 
 	ggtt->base.closed = false;
 
-	if (INTEL_INFO(dev)->gen >= 8) {
-		if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
+	if (INTEL_GEN(dev_priv) >= 8) {
+		if (IS_CHERRYVIEW(dev_priv) || IS_BROXTON(dev_priv))
 			chv_setup_private_ppat(dev_priv);
 		else
 			bdw_setup_private_ppat(dev_priv);
@@ -3275,7 +3322,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 		return;
 	}
 
-	if (USES_PPGTT(dev)) {
+	if (USES_PPGTT(dev_priv)) {
 		struct i915_address_space *vm;
 
 		list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
@@ -3296,137 +3343,28 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 	i915_ggtt_flush(dev_priv);
 }
 
-static void
-i915_vma_retire(struct i915_gem_active *active,
-		struct drm_i915_gem_request *rq)
-{
-	const unsigned int idx = rq->engine->id;
-	struct i915_vma *vma =
-		container_of(active, struct i915_vma, last_read[idx]);
-
-	GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx));
-
-	i915_vma_clear_active(vma, idx);
-	if (i915_vma_is_active(vma))
-		return;
-
-	list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
-	if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma)))
-		WARN_ON(i915_vma_unbind(vma));
-}
-
-void i915_vma_destroy(struct i915_vma *vma)
-{
-	GEM_BUG_ON(vma->node.allocated);
-	GEM_BUG_ON(i915_vma_is_active(vma));
-	GEM_BUG_ON(!i915_vma_is_closed(vma));
-	GEM_BUG_ON(vma->fence);
-
-	list_del(&vma->vm_link);
-	if (!i915_vma_is_ggtt(vma))
-		i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
-
-	kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
-}
-
-void i915_vma_close(struct i915_vma *vma)
-{
-	GEM_BUG_ON(i915_vma_is_closed(vma));
-	vma->flags |= I915_VMA_CLOSED;
-
-	list_del_init(&vma->obj_link);
-	if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma))
-		WARN_ON(i915_vma_unbind(vma));
-}
-
-static struct i915_vma *
-__i915_vma_create(struct drm_i915_gem_object *obj,
-		  struct i915_address_space *vm,
-		  const struct i915_ggtt_view *view)
-{
-	struct i915_vma *vma;
-	int i;
-
-	GEM_BUG_ON(vm->closed);
-
-	vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL);
-	if (vma == NULL)
-		return ERR_PTR(-ENOMEM);
-
-	INIT_LIST_HEAD(&vma->exec_list);
-	for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
-		init_request_active(&vma->last_read[i], i915_vma_retire);
-	init_request_active(&vma->last_fence, NULL);
-	list_add(&vma->vm_link, &vm->unbound_list);
-	vma->vm = vm;
-	vma->obj = obj;
-	vma->size = obj->base.size;
-
-	if (view) {
-		vma->ggtt_view = *view;
-		if (view->type == I915_GGTT_VIEW_PARTIAL) {
-			vma->size = view->params.partial.size;
-			vma->size <<= PAGE_SHIFT;
-		} else if (view->type == I915_GGTT_VIEW_ROTATED) {
-			vma->size =
-				intel_rotation_info_size(&view->params.rotated);
-			vma->size <<= PAGE_SHIFT;
-		}
-	}
-
-	if (i915_is_ggtt(vm)) {
-		vma->flags |= I915_VMA_GGTT;
-	} else {
-		i915_ppgtt_get(i915_vm_to_ppgtt(vm));
-	}
-
-	list_add_tail(&vma->obj_link, &obj->vma_list);
-	return vma;
-}
-
-static inline bool vma_matches(struct i915_vma *vma,
-			       struct i915_address_space *vm,
-			       const struct i915_ggtt_view *view)
-{
-	if (vma->vm != vm)
-		return false;
-
-	if (!i915_vma_is_ggtt(vma))
-		return true;
-
-	if (!view)
-		return vma->ggtt_view.type == 0;
-
-	if (vma->ggtt_view.type != view->type)
-		return false;
-
-	return memcmp(&vma->ggtt_view.params,
-		      &view->params,
-		      sizeof(view->params)) == 0;
-}
-
-struct i915_vma *
-i915_vma_create(struct drm_i915_gem_object *obj,
-		struct i915_address_space *vm,
-		const struct i915_ggtt_view *view)
-{
-	GEM_BUG_ON(view && !i915_is_ggtt(vm));
-	GEM_BUG_ON(i915_gem_obj_to_vma(obj, vm, view));
-
-	return __i915_vma_create(obj, vm, view);
-}
-
 struct i915_vma *
 i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
 		    struct i915_address_space *vm,
 		    const struct i915_ggtt_view *view)
 {
-	struct i915_vma *vma;
+	struct rb_node *rb;
 
-	list_for_each_entry_reverse(vma, &obj->vma_list, obj_link)
-		if (vma_matches(vma, vm, view))
+	rb = obj->vma_tree.rb_node;
+	while (rb) {
+		struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node);
+		long cmp;
+
+		cmp = i915_vma_compare(vma, vm, view);
+		if (cmp == 0)
 			return vma;
 
+		if (cmp < 0)
+			rb = rb->rb_right;
+		else
+			rb = rb->rb_left;
+	}
+
 	return NULL;
 }
 
@@ -3437,11 +3375,14 @@ i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
 {
 	struct i915_vma *vma;
 
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
 	GEM_BUG_ON(view && !i915_is_ggtt(vm));
 
 	vma = i915_gem_obj_to_vma(obj, vm, view);
-	if (!vma)
-		vma = __i915_vma_create(obj, vm, view);
+	if (!vma) {
+		vma = i915_vma_create(obj, vm, view);
+		GEM_BUG_ON(vma != i915_gem_obj_to_vma(obj, vm, view));
+	}
 
 	GEM_BUG_ON(i915_vma_is_closed(vma));
 	return vma;
@@ -3507,7 +3448,7 @@ intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info,
 
 	/* Populate source page list from the object. */
 	i = 0;
-	for_each_sgt_dma(dma_addr, sgt_iter, obj->pages)
+	for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages)
 		page_addr_list[i++] = dma_addr;
 
 	GEM_BUG_ON(i != n_pages);
@@ -3543,35 +3484,47 @@ intel_partial_pages(const struct i915_ggtt_view *view,
 		    struct drm_i915_gem_object *obj)
 {
 	struct sg_table *st;
-	struct scatterlist *sg;
-	struct sg_page_iter obj_sg_iter;
+	struct scatterlist *sg, *iter;
+	unsigned int count = view->params.partial.size;
+	unsigned int offset;
 	int ret = -ENOMEM;
 
 	st = kmalloc(sizeof(*st), GFP_KERNEL);
 	if (!st)
 		goto err_st_alloc;
 
-	ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL);
+	ret = sg_alloc_table(st, count, GFP_KERNEL);
 	if (ret)
 		goto err_sg_alloc;
 
+	iter = i915_gem_object_get_sg(obj,
+				      view->params.partial.offset,
+				      &offset);
+	GEM_BUG_ON(!iter);
+
 	sg = st->sgl;
 	st->nents = 0;
-	for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents,
-		view->params.partial.offset)
-	{
-		if (st->nents >= view->params.partial.size)
-			break;
+	do {
+		unsigned int len;
 
-		sg_set_page(sg, NULL, PAGE_SIZE, 0);
-		sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter);
-		sg_dma_len(sg) = PAGE_SIZE;
+		len = min(iter->length - (offset << PAGE_SHIFT),
+			  count << PAGE_SHIFT);
+		sg_set_page(sg, NULL, len, 0);
+		sg_dma_address(sg) =
+			sg_dma_address(iter) + (offset << PAGE_SHIFT);
+		sg_dma_len(sg) = len;
 
-		sg = sg_next(sg);
 		st->nents++;
-	}
+		count -= len >> PAGE_SHIFT;
+		if (count == 0) {
+			sg_mark_end(sg);
+			return st;
+		}
 
-	return st;
+		sg = __sg_next(sg);
+		iter = __sg_next(iter);
+		offset = 0;
+	} while (1);
 
 err_sg_alloc:
 	kfree(st);
@@ -3584,11 +3537,18 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)
 {
 	int ret = 0;
 
+	/* The vma->pages are only valid within the lifespan of the borrowed
+	 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
+	 * must be the vma->pages. A simple rule is that vma->pages must only
+	 * be accessed when the obj->mm.pages are pinned.
+	 */
+	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
+
 	if (vma->pages)
 		return 0;
 
 	if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
-		vma->pages = vma->obj->pages;
+		vma->pages = vma->obj->mm.pages;
 	else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
 		vma->pages =
 			intel_rotate_fb_obj_pages(&vma->ggtt_view.params.rotated, vma->obj);
@@ -3612,94 +3572,3 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)
 	return ret;
 }
 
-/**
- * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
- * @vma: VMA to map
- * @cache_level: mapping cache level
- * @flags: flags like global or local mapping
- *
- * DMA addresses are taken from the scatter-gather table of this object (or of
- * this VMA in case of non-default GGTT views) and PTE entries set up.
- * Note that DMA addresses are also the only part of the SG table we care about.
- */
-int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
-		  u32 flags)
-{
-	u32 bind_flags;
-	u32 vma_flags;
-	int ret;
-
-	if (WARN_ON(flags == 0))
-		return -EINVAL;
-
-	bind_flags = 0;
-	if (flags & PIN_GLOBAL)
-		bind_flags |= I915_VMA_GLOBAL_BIND;
-	if (flags & PIN_USER)
-		bind_flags |= I915_VMA_LOCAL_BIND;
-
-	vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
-	if (flags & PIN_UPDATE)
-		bind_flags |= vma_flags;
-	else
-		bind_flags &= ~vma_flags;
-	if (bind_flags == 0)
-		return 0;
-
-	if (vma_flags == 0 && vma->vm->allocate_va_range) {
-		trace_i915_va_alloc(vma);
-		ret = vma->vm->allocate_va_range(vma->vm,
-						 vma->node.start,
-						 vma->node.size);
-		if (ret)
-			return ret;
-	}
-
-	ret = vma->vm->bind_vma(vma, cache_level, bind_flags);
-	if (ret)
-		return ret;
-
-	vma->flags |= bind_flags;
-	return 0;
-}
-
-void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
-{
-	void __iomem *ptr;
-
-	/* Access through the GTT requires the device to be awake. */
-	assert_rpm_wakelock_held(to_i915(vma->vm->dev));
-
-	lockdep_assert_held(&vma->vm->dev->struct_mutex);
-	if (WARN_ON(!i915_vma_is_map_and_fenceable(vma)))
-		return IO_ERR_PTR(-ENODEV);
-
-	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
-	GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0);
-
-	ptr = vma->iomap;
-	if (ptr == NULL) {
-		ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->mappable,
-					vma->node.start,
-					vma->node.size);
-		if (ptr == NULL)
-			return IO_ERR_PTR(-ENOMEM);
-
-		vma->iomap = ptr;
-	}
-
-	__i915_vma_pin(vma);
-	return ptr;
-}
-
-void i915_vma_unpin_and_release(struct i915_vma **p_vma)
-{
-	struct i915_vma *vma;
-
-	vma = fetch_and_zero(p_vma);
-	if (!vma)
-		return;
-
-	i915_vma_unpin(vma);
-	i915_vma_put(vma);
-}
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index ec78be2f8c77..4f35be4c26c7 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -35,7 +35,9 @@
 #define __I915_GEM_GTT_H__
 
 #include <linux/io-mapping.h>
+#include <linux/mm.h>
 
+#include "i915_gem_timeline.h"
 #include "i915_gem_request.h"
 
 #define I915_FENCE_REG_NONE -1
@@ -118,8 +120,8 @@ typedef uint64_t gen8_ppgtt_pml4e_t;
 #define GEN8_LEGACY_PDPES		4
 #define GEN8_PTES			I915_PTES(sizeof(gen8_pte_t))
 
-#define I915_PDPES_PER_PDP(dev) (USES_FULL_48BIT_PPGTT(dev) ?\
-				 GEN8_PML4ES_PER_PML4 : GEN8_LEGACY_PDPES)
+#define I915_PDPES_PER_PDP(dev_priv)	(USES_FULL_48BIT_PPGTT(dev_priv) ?\
+					GEN8_PML4ES_PER_PML4 : GEN8_LEGACY_PDPES)
 
 #define PPAT_UNCACHED_INDEX		(_PAGE_PWT | _PAGE_PCD)
 #define PPAT_CACHED_PDE_INDEX		0 /* WB LLC */
@@ -138,6 +140,8 @@ typedef uint64_t gen8_ppgtt_pml4e_t;
 #define GEN8_PPAT_ELLC_OVERRIDE		(0<<2)
 #define GEN8_PPAT(i, x)			((uint64_t) (x) << ((i) * 8))
 
+struct sg_table;
+
 enum i915_ggtt_view_type {
 	I915_GGTT_VIEW_NORMAL = 0,
 	I915_GGTT_VIEW_ROTATED,
@@ -168,133 +172,7 @@ extern const struct i915_ggtt_view i915_ggtt_view_rotated;
 
 enum i915_cache_level;
 
-/**
- * A VMA represents a GEM BO that is bound into an address space. Therefore, a
- * VMA's presence cannot be guaranteed before binding, or after unbinding the
- * object into/from the address space.
- *
- * To make things as simple as possible (ie. no refcounting), a VMA's lifetime
- * will always be <= an objects lifetime. So object refcounting should cover us.
- */
-struct i915_vma {
-	struct drm_mm_node node;
-	struct drm_i915_gem_object *obj;
-	struct i915_address_space *vm;
-	struct drm_i915_fence_reg *fence;
-	struct sg_table *pages;
-	void __iomem *iomap;
-	u64 size;
-	u64 display_alignment;
-
-	unsigned int flags;
-	/**
-	 * How many users have pinned this object in GTT space. The following
-	 * users can each hold at most one reference: pwrite/pread, execbuffer
-	 * (objects are not allowed multiple times for the same batchbuffer),
-	 * and the framebuffer code. When switching/pageflipping, the
-	 * framebuffer code has at most two buffers pinned per crtc.
-	 *
-	 * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
-	 * bits with absolutely no headroom. So use 4 bits.
-	 */
-#define I915_VMA_PIN_MASK 0xf
-#define I915_VMA_PIN_OVERFLOW	BIT(5)
-
-	/** Flags and address space this VMA is bound to */
-#define I915_VMA_GLOBAL_BIND	BIT(6)
-#define I915_VMA_LOCAL_BIND	BIT(7)
-#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
-
-#define I915_VMA_GGTT		BIT(8)
-#define I915_VMA_CAN_FENCE	BIT(9)
-#define I915_VMA_CLOSED		BIT(10)
-
-	unsigned int active;
-	struct i915_gem_active last_read[I915_NUM_ENGINES];
-	struct i915_gem_active last_fence;
-
-	/**
-	 * Support different GGTT views into the same object.
-	 * This means there can be multiple VMA mappings per object and per VM.
-	 * i915_ggtt_view_type is used to distinguish between those entries.
-	 * The default one of zero (I915_GGTT_VIEW_NORMAL) is default and also
-	 * assumed in GEM functions which take no ggtt view parameter.
-	 */
-	struct i915_ggtt_view ggtt_view;
-
-	/** This object's place on the active/inactive lists */
-	struct list_head vm_link;
-
-	struct list_head obj_link; /* Link in the object's VMA list */
-
-	/** This vma's place in the batchbuffer or on the eviction list */
-	struct list_head exec_list;
-
-	/**
-	 * Used for performing relocations during execbuffer insertion.
-	 */
-	struct hlist_node exec_node;
-	unsigned long exec_handle;
-	struct drm_i915_gem_exec_object2 *exec_entry;
-};
-
-struct i915_vma *
-i915_vma_create(struct drm_i915_gem_object *obj,
-		struct i915_address_space *vm,
-		const struct i915_ggtt_view *view);
-void i915_vma_unpin_and_release(struct i915_vma **p_vma);
-
-static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
-{
-	return vma->flags & I915_VMA_GGTT;
-}
-
-static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
-{
-	return vma->flags & I915_VMA_CAN_FENCE;
-}
-
-static inline bool i915_vma_is_closed(const struct i915_vma *vma)
-{
-	return vma->flags & I915_VMA_CLOSED;
-}
-
-static inline unsigned int i915_vma_get_active(const struct i915_vma *vma)
-{
-	return vma->active;
-}
-
-static inline bool i915_vma_is_active(const struct i915_vma *vma)
-{
-	return i915_vma_get_active(vma);
-}
-
-static inline void i915_vma_set_active(struct i915_vma *vma,
-				       unsigned int engine)
-{
-	vma->active |= BIT(engine);
-}
-
-static inline void i915_vma_clear_active(struct i915_vma *vma,
-					 unsigned int engine)
-{
-	vma->active &= ~BIT(engine);
-}
-
-static inline bool i915_vma_has_active_engine(const struct i915_vma *vma,
-					      unsigned int engine)
-{
-	return vma->active & BIT(engine);
-}
-
-static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
-{
-	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
-	GEM_BUG_ON(!vma->node.allocated);
-	GEM_BUG_ON(upper_32_bits(vma->node.start));
-	GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1));
-	return lower_32_bits(vma->node.start);
-}
+struct i915_vma;
 
 struct i915_page_dma {
 	struct page *page;
@@ -341,6 +219,7 @@ struct i915_pml4 {
 
 struct i915_address_space {
 	struct drm_mm mm;
+	struct i915_gem_timeline timeline;
 	struct drm_device *dev;
 	/* Every address space belongs to a struct file - except for the global
 	 * GTT that is owned by the driver (and so @file is set to NULL). In
@@ -395,7 +274,7 @@ struct i915_address_space {
 	/* FIXME: Need a more generic return type */
 	gen6_pte_t (*pte_encode)(dma_addr_t addr,
 				 enum i915_cache_level level,
-				 bool valid, u32 flags); /* Create a valid PTE */
+				 u32 flags); /* Create a valid PTE */
 	/* flags for pte_encode */
 #define PTE_READ_ONLY	(1<<0)
 	int (*allocate_va_range)(struct i915_address_space *vm,
@@ -403,8 +282,7 @@ struct i915_address_space {
 				 uint64_t length);
 	void (*clear_range)(struct i915_address_space *vm,
 			    uint64_t start,
-			    uint64_t length,
-			    bool use_scratch);
+			    uint64_t length);
 	void (*insert_page)(struct i915_address_space *vm,
 			    dma_addr_t addr,
 			    uint64_t offset,
@@ -450,6 +328,8 @@ struct i915_ggtt {
 	bool do_idle_maps;
 
 	int mtrr;
+
+	struct drm_mm_node error_capture;
 };
 
 struct i915_hw_ppgtt {
@@ -602,16 +482,24 @@ i915_page_dir_dma_addr(const struct i915_hw_ppgtt *ppgtt, const unsigned n)
 		px_dma(ppgtt->base.scratch_pd);
 }
 
+static inline struct i915_ggtt *
+i915_vm_to_ggtt(struct i915_address_space *vm)
+{
+	GEM_BUG_ON(!i915_is_ggtt(vm));
+	return container_of(vm, struct i915_ggtt, base);
+}
+
 int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv);
 int i915_ggtt_init_hw(struct drm_i915_private *dev_priv);
 int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv);
 int i915_gem_init_ggtt(struct drm_i915_private *dev_priv);
 void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv);
 
-int i915_ppgtt_init_hw(struct drm_device *dev);
+int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv);
 void i915_ppgtt_release(struct kref *kref);
 struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv,
-					struct drm_i915_file_private *fpriv);
+					struct drm_i915_file_private *fpriv,
+					const char *name);
 static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt)
 {
 	if (ppgtt)
@@ -624,11 +512,13 @@ static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt)
 }
 
 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv);
-void i915_gem_suspend_gtt_mappings(struct drm_device *dev);
-void i915_gem_restore_gtt_mappings(struct drm_device *dev);
+void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv);
+void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv);
 
-int __must_check i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj);
-void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj);
+int __must_check i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
+					    struct sg_table *pages);
+void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
+			       struct sg_table *pages);
 
 /* Flags used by pin/bind&friends. */
 #define PIN_NONBLOCK		BIT(0)
@@ -646,88 +536,4 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj);
 #define PIN_OFFSET_FIXED	BIT(11)
 #define PIN_OFFSET_MASK		(~4095)
 
-int __i915_vma_do_pin(struct i915_vma *vma,
-		      u64 size, u64 alignment, u64 flags);
-static inline int __must_check
-i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
-{
-	BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
-	BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
-	BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
-
-	/* Pin early to prevent the shrinker/eviction logic from destroying
-	 * our vma as we insert and bind.
-	 */
-	if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0))
-		return 0;
-
-	return __i915_vma_do_pin(vma, size, alignment, flags);
-}
-
-static inline int i915_vma_pin_count(const struct i915_vma *vma)
-{
-	return vma->flags & I915_VMA_PIN_MASK;
-}
-
-static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
-{
-	return i915_vma_pin_count(vma);
-}
-
-static inline void __i915_vma_pin(struct i915_vma *vma)
-{
-	vma->flags++;
-	GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
-}
-
-static inline void __i915_vma_unpin(struct i915_vma *vma)
-{
-	GEM_BUG_ON(!i915_vma_is_pinned(vma));
-	vma->flags--;
-}
-
-static inline void i915_vma_unpin(struct i915_vma *vma)
-{
-	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
-	__i915_vma_unpin(vma);
-}
-
-/**
- * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture
- * @vma: VMA to iomap
- *
- * The passed in VMA has to be pinned in the global GTT mappable region.
- * An extra pinning of the VMA is acquired for the return iomapping,
- * the caller must call i915_vma_unpin_iomap to relinquish the pinning
- * after the iomapping is no longer required.
- *
- * Callers must hold the struct_mutex.
- *
- * Returns a valid iomapped pointer or ERR_PTR.
- */
-void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
-#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x))
-
-/**
- * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap
- * @vma: VMA to unpin
- *
- * Unpins the previously iomapped VMA from i915_vma_pin_iomap().
- *
- * Callers must hold the struct_mutex. This function is only valid to be
- * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap().
- */
-static inline void i915_vma_unpin_iomap(struct i915_vma *vma)
-{
-	lockdep_assert_held(&vma->vm->dev->struct_mutex);
-	GEM_BUG_ON(vma->iomap == NULL);
-	i915_vma_unpin(vma);
-}
-
-static inline struct page *i915_vma_first_page(struct i915_vma *vma)
-{
-	GEM_BUG_ON(!vma->pages);
-	return sg_page(vma->pages->sgl);
-}
-
 #endif
diff --git a/drivers/gpu/drm/i915/i915_gem_internal.c b/drivers/gpu/drm/i915/i915_gem_internal.c
new file mode 100644
index 000000000000..4b3ff3e5b911
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_internal.c
@@ -0,0 +1,170 @@
+/*
+ * Copyright © 2014-2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
+#include "i915_drv.h"
+
+#define QUIET (__GFP_NORETRY | __GFP_NOWARN)
+
+/* convert swiotlb segment size into sensible units (pages)! */
+#define IO_TLB_SEGPAGES (IO_TLB_SEGSIZE << IO_TLB_SHIFT >> PAGE_SHIFT)
+
+static void internal_free_pages(struct sg_table *st)
+{
+	struct scatterlist *sg;
+
+	for (sg = st->sgl; sg; sg = __sg_next(sg))
+		__free_pages(sg_page(sg), get_order(sg->length));
+
+	sg_free_table(st);
+	kfree(st);
+}
+
+static struct sg_table *
+i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
+{
+	struct drm_i915_private *i915 = to_i915(obj->base.dev);
+	unsigned int npages = obj->base.size / PAGE_SIZE;
+	struct sg_table *st;
+	struct scatterlist *sg;
+	int max_order;
+	gfp_t gfp;
+
+	st = kmalloc(sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return ERR_PTR(-ENOMEM);
+
+	if (sg_alloc_table(st, npages, GFP_KERNEL)) {
+		kfree(st);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	sg = st->sgl;
+	st->nents = 0;
+
+	max_order = MAX_ORDER;
+#ifdef CONFIG_SWIOTLB
+	if (swiotlb_nr_tbl()) /* minimum max swiotlb size is IO_TLB_SEGSIZE */
+		max_order = min(max_order, ilog2(IO_TLB_SEGPAGES));
+#endif
+
+	gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE;
+	if (IS_CRESTLINE(i915) || IS_BROADWATER(i915)) {
+		/* 965gm cannot relocate objects above 4GiB. */
+		gfp &= ~__GFP_HIGHMEM;
+		gfp |= __GFP_DMA32;
+	}
+
+	do {
+		int order = min(fls(npages) - 1, max_order);
+		struct page *page;
+
+		do {
+			page = alloc_pages(gfp | (order ? QUIET : 0), order);
+			if (page)
+				break;
+			if (!order--)
+				goto err;
+
+			/* Limit subsequent allocations as well */
+			max_order = order;
+		} while (1);
+
+		sg_set_page(sg, page, PAGE_SIZE << order, 0);
+		st->nents++;
+
+		npages -= 1 << order;
+		if (!npages) {
+			sg_mark_end(sg);
+			break;
+		}
+
+		sg = __sg_next(sg);
+	} while (1);
+
+	if (i915_gem_gtt_prepare_pages(obj, st))
+		goto err;
+
+	/* Mark the pages as dontneed whilst they are still pinned. As soon
+	 * as they are unpinned they are allowed to be reaped by the shrinker,
+	 * and the caller is expected to repopulate - the contents of this
+	 * object are only valid whilst active and pinned.
+	 */
+	obj->mm.madv = I915_MADV_DONTNEED;
+	return st;
+
+err:
+	sg_mark_end(sg);
+	internal_free_pages(st);
+	return ERR_PTR(-ENOMEM);
+}
+
+static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj,
+					       struct sg_table *pages)
+{
+	i915_gem_gtt_finish_pages(obj, pages);
+	internal_free_pages(pages);
+
+	obj->mm.dirty = false;
+	obj->mm.madv = I915_MADV_WILLNEED;
+}
+
+static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {
+	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
+		 I915_GEM_OBJECT_IS_SHRINKABLE,
+	.get_pages = i915_gem_object_get_pages_internal,
+	.put_pages = i915_gem_object_put_pages_internal,
+};
+
+/**
+ * Creates a new object that wraps some internal memory for private use.
+ * This object is not backed by swappable storage, and as such its contents
+ * are volatile and only valid whilst pinned. If the object is reaped by the
+ * shrinker, its pages and data will be discarded. Equally, it is not a full
+ * GEM object and so not valid for access from userspace. This makes it useful
+ * for hardware interfaces like ringbuffers (which are pinned from the time
+ * the request is written to the time the hardware stops accessing it), but
+ * not for contexts (which need to be preserved when not active for later
+ * reuse). Note that it is not cleared upon allocation.
+ */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_i915_private *i915,
+				unsigned int size)
+{
+	struct drm_i915_gem_object *obj;
+
+	obj = i915_gem_object_alloc(&i915->drm);
+	if (!obj)
+		return ERR_PTR(-ENOMEM);
+
+	drm_gem_private_object_init(&i915->drm, &obj->base, size);
+	i915_gem_object_init(obj, &i915_gem_object_internal_ops);
+
+	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+	obj->cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
+
+	return obj;
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h
new file mode 100644
index 000000000000..6a368de9d81e
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_object.h
@@ -0,0 +1,338 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_GEM_OBJECT_H__
+#define __I915_GEM_OBJECT_H__
+
+#include <linux/reservation.h>
+
+#include <drm/drm_vma_manager.h>
+#include <drm/drm_gem.h>
+#include <drm/drmP.h>
+
+#include <drm/i915_drm.h>
+
+struct drm_i915_gem_object_ops {
+	unsigned int flags;
+#define I915_GEM_OBJECT_HAS_STRUCT_PAGE 0x1
+#define I915_GEM_OBJECT_IS_SHRINKABLE   0x2
+
+	/* Interface between the GEM object and its backing storage.
+	 * get_pages() is called once prior to the use of the associated set
+	 * of pages before to binding them into the GTT, and put_pages() is
+	 * called after we no longer need them. As we expect there to be
+	 * associated cost with migrating pages between the backing storage
+	 * and making them available for the GPU (e.g. clflush), we may hold
+	 * onto the pages after they are no longer referenced by the GPU
+	 * in case they may be used again shortly (for example migrating the
+	 * pages to a different memory domain within the GTT). put_pages()
+	 * will therefore most likely be called when the object itself is
+	 * being released or under memory pressure (where we attempt to
+	 * reap pages for the shrinker).
+	 */
+	struct sg_table *(*get_pages)(struct drm_i915_gem_object *);
+	void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *);
+
+	int (*dmabuf_export)(struct drm_i915_gem_object *);
+	void (*release)(struct drm_i915_gem_object *);
+};
+
+struct drm_i915_gem_object {
+	struct drm_gem_object base;
+
+	const struct drm_i915_gem_object_ops *ops;
+
+	/** List of VMAs backed by this object */
+	struct list_head vma_list;
+	struct rb_root vma_tree;
+
+	/** Stolen memory for this object, instead of being backed by shmem. */
+	struct drm_mm_node *stolen;
+	struct list_head global_link;
+	union {
+		struct rcu_head rcu;
+		struct llist_node freed;
+	};
+
+	/**
+	 * Whether the object is currently in the GGTT mmap.
+	 */
+	struct list_head userfault_link;
+
+	/** Used in execbuf to temporarily hold a ref */
+	struct list_head obj_exec_link;
+
+	struct list_head batch_pool_link;
+
+	unsigned long flags;
+
+	/**
+	 * Have we taken a reference for the object for incomplete GPU
+	 * activity?
+	 */
+#define I915_BO_ACTIVE_REF 0
+
+	/*
+	 * Is the object to be mapped as read-only to the GPU
+	 * Only honoured if hardware has relevant pte bit
+	 */
+	unsigned long gt_ro:1;
+	unsigned int cache_level:3;
+	unsigned int cache_dirty:1;
+
+	atomic_t frontbuffer_bits;
+	unsigned int frontbuffer_ggtt_origin; /* write once */
+	struct i915_gem_active frontbuffer_write;
+
+	/** Current tiling stride for the object, if it's tiled. */
+	unsigned int tiling_and_stride;
+#define FENCE_MINIMUM_STRIDE 128 /* See i915_tiling_ok() */
+#define TILING_MASK (FENCE_MINIMUM_STRIDE-1)
+#define STRIDE_MASK (~TILING_MASK)
+
+	/** Count of VMA actually bound by this object */
+	unsigned int bind_count;
+	unsigned int active_count;
+	unsigned int pin_display;
+
+	struct {
+		struct mutex lock; /* protects the pages and their use */
+		atomic_t pages_pin_count;
+
+		struct sg_table *pages;
+		void *mapping;
+
+		struct i915_gem_object_page_iter {
+			struct scatterlist *sg_pos;
+			unsigned int sg_idx; /* in pages, but 32bit eek! */
+
+			struct radix_tree_root radix;
+			struct mutex lock; /* protects this cache */
+		} get_page;
+
+		/**
+		 * Advice: are the backing pages purgeable?
+		 */
+		unsigned int madv:2;
+
+		/**
+		 * This is set if the object has been written to since the
+		 * pages were last acquired.
+		 */
+		bool dirty:1;
+
+		/**
+		 * This is set if the object has been pinned due to unknown
+		 * swizzling.
+		 */
+		bool quirked:1;
+	} mm;
+
+	/** Breadcrumb of last rendering to the buffer.
+	 * There can only be one writer, but we allow for multiple readers.
+	 * If there is a writer that necessarily implies that all other
+	 * read requests are complete - but we may only be lazily clearing
+	 * the read requests. A read request is naturally the most recent
+	 * request on a ring, so we may have two different write and read
+	 * requests on one ring where the write request is older than the
+	 * read request. This allows for the CPU to read from an active
+	 * buffer by only waiting for the write to complete.
+	 */
+	struct reservation_object *resv;
+
+	/** References from framebuffers, locks out tiling changes. */
+	unsigned long framebuffer_references;
+
+	/** Record of address bit 17 of each page at last unbind. */
+	unsigned long *bit_17;
+
+	struct i915_gem_userptr {
+		uintptr_t ptr;
+		unsigned read_only :1;
+
+		struct i915_mm_struct *mm;
+		struct i915_mmu_object *mmu_object;
+		struct work_struct *work;
+	} userptr;
+
+	/** for phys allocated objects */
+	struct drm_dma_handle *phys_handle;
+
+	struct reservation_object __builtin_resv;
+};
+
+static inline struct drm_i915_gem_object *
+to_intel_bo(struct drm_gem_object *gem)
+{
+	/* Assert that to_intel_bo(NULL) == NULL */
+	BUILD_BUG_ON(offsetof(struct drm_i915_gem_object, base));
+
+	return container_of(gem, struct drm_i915_gem_object, base);
+}
+
+/**
+ * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle
+ * @filp: DRM file private date
+ * @handle: userspace handle
+ *
+ * Returns:
+ *
+ * A pointer to the object named by the handle if such exists on @filp, NULL
+ * otherwise. This object is only valid whilst under the RCU read lock, and
+ * note carefully the object may be in the process of being destroyed.
+ */
+static inline struct drm_i915_gem_object *
+i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle)
+{
+#ifdef CONFIG_LOCKDEP
+	WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map));
+#endif
+	return idr_find(&file->object_idr, handle);
+}
+
+static inline struct drm_i915_gem_object *
+i915_gem_object_lookup(struct drm_file *file, u32 handle)
+{
+	struct drm_i915_gem_object *obj;
+
+	rcu_read_lock();
+	obj = i915_gem_object_lookup_rcu(file, handle);
+	if (obj && !kref_get_unless_zero(&obj->base.refcount))
+		obj = NULL;
+	rcu_read_unlock();
+
+	return obj;
+}
+
+__deprecated
+extern struct drm_gem_object *
+drm_gem_object_lookup(struct drm_file *file, u32 handle);
+
+__attribute__((nonnull))
+static inline struct drm_i915_gem_object *
+i915_gem_object_get(struct drm_i915_gem_object *obj)
+{
+	drm_gem_object_reference(&obj->base);
+	return obj;
+}
+
+__deprecated
+extern void drm_gem_object_reference(struct drm_gem_object *);
+
+__attribute__((nonnull))
+static inline void
+i915_gem_object_put(struct drm_i915_gem_object *obj)
+{
+	__drm_gem_object_unreference(&obj->base);
+}
+
+__deprecated
+extern void drm_gem_object_unreference(struct drm_gem_object *);
+
+__deprecated
+extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);
+
+static inline bool
+i915_gem_object_is_dead(const struct drm_i915_gem_object *obj)
+{
+	return atomic_read(&obj->base.refcount.refcount) == 0;
+}
+
+static inline bool
+i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
+{
+	return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE;
+}
+
+static inline bool
+i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj)
+{
+	return obj->ops->flags & I915_GEM_OBJECT_IS_SHRINKABLE;
+}
+
+static inline bool
+i915_gem_object_is_active(const struct drm_i915_gem_object *obj)
+{
+	return obj->active_count;
+}
+
+static inline bool
+i915_gem_object_has_active_reference(const struct drm_i915_gem_object *obj)
+{
+	return test_bit(I915_BO_ACTIVE_REF, &obj->flags);
+}
+
+static inline void
+i915_gem_object_set_active_reference(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	__set_bit(I915_BO_ACTIVE_REF, &obj->flags);
+}
+
+static inline void
+i915_gem_object_clear_active_reference(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	__clear_bit(I915_BO_ACTIVE_REF, &obj->flags);
+}
+
+void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj);
+
+static inline unsigned int
+i915_gem_object_get_tiling(struct drm_i915_gem_object *obj)
+{
+	return obj->tiling_and_stride & TILING_MASK;
+}
+
+static inline bool
+i915_gem_object_is_tiled(struct drm_i915_gem_object *obj)
+{
+	return i915_gem_object_get_tiling(obj) != I915_TILING_NONE;
+}
+
+static inline unsigned int
+i915_gem_object_get_stride(struct drm_i915_gem_object *obj)
+{
+	return obj->tiling_and_stride & STRIDE_MASK;
+}
+
+static inline struct intel_engine_cs *
+i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj)
+{
+	struct intel_engine_cs *engine = NULL;
+	struct dma_fence *fence;
+
+	rcu_read_lock();
+	fence = reservation_object_get_excl_rcu(obj->resv);
+	rcu_read_unlock();
+
+	if (fence && dma_fence_is_i915(fence) && !dma_fence_is_signaled(fence))
+		engine = to_request(fence)->engine;
+	dma_fence_put(fence);
+
+	return engine;
+}
+
+#endif
+
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index 95b7e9afd5f8..5af19b0bf713 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -28,17 +28,19 @@
 #include "i915_drv.h"
 #include "intel_renderstate.h"
 
-struct render_state {
+struct intel_render_state {
 	const struct intel_renderstate_rodata *rodata;
 	struct i915_vma *vma;
-	u32 aux_batch_size;
-	u32 aux_batch_offset;
+	u32 batch_offset;
+	u32 batch_size;
+	u32 aux_offset;
+	u32 aux_size;
 };
 
 static const struct intel_renderstate_rodata *
-render_state_get_rodata(const struct drm_i915_gem_request *req)
+render_state_get_rodata(const struct intel_engine_cs *engine)
 {
-	switch (INTEL_GEN(req->i915)) {
+	switch (INTEL_GEN(engine->i915)) {
 	case 6:
 		return &gen6_null_state;
 	case 7:
@@ -63,29 +65,26 @@ render_state_get_rodata(const struct drm_i915_gem_request *req)
  */
 #define OUT_BATCH(batch, i, val)				\
 	do {							\
-		if (WARN_ON((i) >= PAGE_SIZE / sizeof(u32))) {	\
-			ret = -ENOSPC;				\
-			goto err_out;				\
-		}						\
+		if ((i) >= PAGE_SIZE / sizeof(u32))		\
+			goto err;				\
 		(batch)[(i)++] = (val);				\
 	} while(0)
 
-static int render_state_setup(struct render_state *so)
+static int render_state_setup(struct intel_render_state *so,
+			      struct drm_i915_private *i915)
 {
-	struct drm_device *dev = so->vma->vm->dev;
 	const struct intel_renderstate_rodata *rodata = so->rodata;
-	const bool has_64bit_reloc = INTEL_GEN(dev) >= 8;
+	struct drm_i915_gem_object *obj = so->vma->obj;
 	unsigned int i = 0, reloc_index = 0;
-	struct page *page;
+	unsigned int needs_clflush;
 	u32 *d;
 	int ret;
 
-	ret = i915_gem_object_set_to_cpu_domain(so->vma->obj, true);
+	ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
 	if (ret)
 		return ret;
 
-	page = i915_gem_object_get_dirty_page(so->vma->obj, 0);
-	d = kmap(page);
+	d = kmap_atomic(i915_gem_object_get_dirty_page(obj, 0));
 
 	while (i < rodata->batch_items) {
 		u32 s = rodata->batch[i];
@@ -93,12 +92,10 @@ static int render_state_setup(struct render_state *so)
 		if (i * 4  == rodata->reloc[reloc_index]) {
 			u64 r = s + so->vma->node.start;
 			s = lower_32_bits(r);
-			if (has_64bit_reloc) {
+			if (HAS_64BIT_RELOC(i915)) {
 				if (i + 1 >= rodata->batch_items ||
-				    rodata->batch[i + 1] != 0) {
-					ret = -EINVAL;
-					goto err_out;
-				}
+				    rodata->batch[i + 1] != 0)
+					goto err;
 
 				d[i++] = s;
 				s = upper_32_bits(r);
@@ -110,12 +107,20 @@ static int render_state_setup(struct render_state *so)
 		d[i++] = s;
 	}
 
+	if (rodata->reloc[reloc_index] != -1) {
+		DRM_ERROR("only %d relocs resolved\n", reloc_index);
+		goto err;
+	}
+
+	so->batch_offset = so->vma->node.start;
+	so->batch_size = rodata->batch_items * sizeof(u32);
+
 	while (i % CACHELINE_DWORDS)
 		OUT_BATCH(d, i, MI_NOOP);
 
-	so->aux_batch_offset = i * sizeof(u32);
+	so->aux_offset = i * sizeof(u32);
 
-	if (HAS_POOLED_EU(dev)) {
+	if (HAS_POOLED_EU(i915)) {
 		/*
 		 * We always program 3x6 pool config but depending upon which
 		 * subslice is disabled HW drops down to appropriate config
@@ -143,88 +148,133 @@ static int render_state_setup(struct render_state *so)
 	}
 
 	OUT_BATCH(d, i, MI_BATCH_BUFFER_END);
-	so->aux_batch_size = (i * sizeof(u32)) - so->aux_batch_offset;
-
+	so->aux_size = i * sizeof(u32) - so->aux_offset;
+	so->aux_offset += so->batch_offset;
 	/*
 	 * Since we are sending length, we need to strictly conform to
 	 * all requirements. For Gen2 this must be a multiple of 8.
 	 */
-	so->aux_batch_size = ALIGN(so->aux_batch_size, 8);
-
-	kunmap(page);
-
-	ret = i915_gem_object_set_to_gtt_domain(so->vma->obj, false);
-	if (ret)
-		return ret;
-
-	if (rodata->reloc[reloc_index] != -1) {
-		DRM_ERROR("only %d relocs resolved\n", reloc_index);
-		return -EINVAL;
-	}
+	so->aux_size = ALIGN(so->aux_size, 8);
 
-	return 0;
+	if (needs_clflush)
+		drm_clflush_virt_range(d, i * sizeof(u32));
+	kunmap_atomic(d);
 
-err_out:
-	kunmap(page);
+	ret = i915_gem_object_set_to_gtt_domain(obj, false);
+out:
+	i915_gem_obj_finish_shmem_access(obj);
 	return ret;
+
+err:
+	kunmap_atomic(d);
+	ret = -EINVAL;
+	goto out;
 }
 
 #undef OUT_BATCH
 
-int i915_gem_render_state_init(struct drm_i915_gem_request *req)
+int i915_gem_render_state_init(struct intel_engine_cs *engine)
 {
-	struct render_state so;
+	struct intel_render_state *so;
+	const struct intel_renderstate_rodata *rodata;
 	struct drm_i915_gem_object *obj;
 	int ret;
 
-	if (WARN_ON(req->engine->id != RCS))
-		return -ENOENT;
+	if (engine->id != RCS)
+		return 0;
 
-	so.rodata = render_state_get_rodata(req);
-	if (!so.rodata)
+	rodata = render_state_get_rodata(engine);
+	if (!rodata)
 		return 0;
 
-	if (so.rodata->batch_items * 4 > 4096)
+	if (rodata->batch_items * 4 > 4096)
 		return -EINVAL;
 
-	obj = i915_gem_object_create(&req->i915->drm, 4096);
-	if (IS_ERR(obj))
-		return PTR_ERR(obj);
+	so = kmalloc(sizeof(*so), GFP_KERNEL);
+	if (!so)
+		return -ENOMEM;
 
-	so.vma = i915_vma_create(obj, &req->i915->ggtt.base, NULL);
-	if (IS_ERR(so.vma)) {
-		ret = PTR_ERR(so.vma);
-		goto err_obj;
+	obj = i915_gem_object_create_internal(engine->i915, 4096);
+	if (IS_ERR(obj)) {
+		ret = PTR_ERR(obj);
+		goto err_free;
 	}
 
-	ret = i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL);
-	if (ret)
+	so->vma = i915_vma_create(obj, &engine->i915->ggtt.base, NULL);
+	if (IS_ERR(so->vma)) {
+		ret = PTR_ERR(so->vma);
 		goto err_obj;
+	}
+
+	so->rodata = rodata;
+	engine->render_state = so;
+	return 0;
+
+err_obj:
+	i915_gem_object_put(obj);
+err_free:
+	kfree(so);
+	return ret;
+}
+
+int i915_gem_render_state_emit(struct drm_i915_gem_request *req)
+{
+	struct intel_render_state *so;
+	int ret;
+
+	lockdep_assert_held(&req->i915->drm.struct_mutex);
 
-	ret = render_state_setup(&so);
+	so = req->engine->render_state;
+	if (!so)
+		return 0;
+
+	/* Recreate the page after shrinking */
+	if (!so->vma->obj->mm.pages)
+		so->batch_offset = -1;
+
+	ret = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
 	if (ret)
-		goto err_unpin;
+		return ret;
+
+	if (so->vma->node.start != so->batch_offset) {
+		ret = render_state_setup(so, req->i915);
+		if (ret)
+			goto err_unpin;
+	}
 
-	ret = req->engine->emit_bb_start(req, so.vma->node.start,
-					 so.rodata->batch_items * 4,
+	ret = req->engine->emit_bb_start(req,
+					 so->batch_offset, so->batch_size,
 					 I915_DISPATCH_SECURE);
 	if (ret)
 		goto err_unpin;
 
-	if (so.aux_batch_size > 8) {
+	if (so->aux_size > 8) {
 		ret = req->engine->emit_bb_start(req,
-						 (so.vma->node.start +
-						  so.aux_batch_offset),
-						 so.aux_batch_size,
+						 so->aux_offset, so->aux_size,
 						 I915_DISPATCH_SECURE);
 		if (ret)
 			goto err_unpin;
 	}
 
-	i915_vma_move_to_active(so.vma, req, 0);
+	i915_vma_move_to_active(so->vma, req, 0);
 err_unpin:
-	i915_vma_unpin(so.vma);
-err_obj:
-	i915_gem_object_put(obj);
+	i915_vma_unpin(so->vma);
 	return ret;
 }
+
+void i915_gem_render_state_fini(struct intel_engine_cs *engine)
+{
+	struct intel_render_state *so;
+	struct drm_i915_gem_object *obj;
+
+	so = fetch_and_zero(&engine->render_state);
+	if (!so)
+		return;
+
+	obj = so->vma->obj;
+
+	i915_vma_close(so->vma);
+	__i915_gem_object_release_unless_active(obj);
+
+	kfree(so);
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.h b/drivers/gpu/drm/i915/i915_gem_render_state.h
index 18cce3f06e9c..87481845799d 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.h
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.h
@@ -26,6 +26,8 @@
 
 struct drm_i915_gem_request;
 
-int i915_gem_render_state_init(struct drm_i915_gem_request *req);
+int i915_gem_render_state_init(struct intel_engine_cs *engine);
+int i915_gem_render_state_emit(struct drm_i915_gem_request *req);
+void i915_gem_render_state_fini(struct intel_engine_cs *engine);
 
 #endif /* _I915_GEM_RENDER_STATE_H_ */
diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
index 383d13416442..b8f403faadbb 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/i915_gem_request.c
@@ -23,31 +23,26 @@
  */
 
 #include <linux/prefetch.h>
+#include <linux/dma-fence-array.h>
 
 #include "i915_drv.h"
 
-static const char *i915_fence_get_driver_name(struct fence *fence)
+static const char *i915_fence_get_driver_name(struct dma_fence *fence)
 {
 	return "i915";
 }
 
-static const char *i915_fence_get_timeline_name(struct fence *fence)
+static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
 {
-	/* Timelines are bound by eviction to a VM. However, since
-	 * we only have a global seqno at the moment, we only have
-	 * a single timeline. Note that each timeline will have
-	 * multiple execution contexts (fence contexts) as we allow
-	 * engines within a single timeline to execute in parallel.
-	 */
-	return "global";
+	return to_request(fence)->timeline->common->name;
 }
 
-static bool i915_fence_signaled(struct fence *fence)
+static bool i915_fence_signaled(struct dma_fence *fence)
 {
 	return i915_gem_request_completed(to_request(fence));
 }
 
-static bool i915_fence_enable_signaling(struct fence *fence)
+static bool i915_fence_enable_signaling(struct dma_fence *fence)
 {
 	if (i915_fence_signaled(fence))
 		return false;
@@ -56,63 +51,27 @@ static bool i915_fence_enable_signaling(struct fence *fence)
 	return true;
 }
 
-static signed long i915_fence_wait(struct fence *fence,
+static signed long i915_fence_wait(struct dma_fence *fence,
 				   bool interruptible,
-				   signed long timeout_jiffies)
+				   signed long timeout)
 {
-	s64 timeout_ns, *timeout;
-	int ret;
-
-	if (timeout_jiffies != MAX_SCHEDULE_TIMEOUT) {
-		timeout_ns = jiffies_to_nsecs(timeout_jiffies);
-		timeout = &timeout_ns;
-	} else {
-		timeout = NULL;
-	}
-
-	ret = i915_wait_request(to_request(fence),
-				interruptible, timeout,
-				NO_WAITBOOST);
-	if (ret == -ETIME)
-		return 0;
-
-	if (ret < 0)
-		return ret;
-
-	if (timeout_jiffies != MAX_SCHEDULE_TIMEOUT)
-		timeout_jiffies = nsecs_to_jiffies(timeout_ns);
-
-	return timeout_jiffies;
+	return i915_wait_request(to_request(fence), interruptible, timeout);
 }
 
-static void i915_fence_value_str(struct fence *fence, char *str, int size)
-{
-	snprintf(str, size, "%u", fence->seqno);
-}
-
-static void i915_fence_timeline_value_str(struct fence *fence, char *str,
-					  int size)
-{
-	snprintf(str, size, "%u",
-		 intel_engine_get_seqno(to_request(fence)->engine));
-}
-
-static void i915_fence_release(struct fence *fence)
+static void i915_fence_release(struct dma_fence *fence)
 {
 	struct drm_i915_gem_request *req = to_request(fence);
 
 	kmem_cache_free(req->i915->requests, req);
 }
 
-const struct fence_ops i915_fence_ops = {
+const struct dma_fence_ops i915_fence_ops = {
 	.get_driver_name = i915_fence_get_driver_name,
 	.get_timeline_name = i915_fence_get_timeline_name,
 	.enable_signaling = i915_fence_enable_signaling,
 	.signaled = i915_fence_signaled,
 	.wait = i915_fence_wait,
 	.release = i915_fence_release,
-	.fence_value_str = i915_fence_value_str,
-	.timeline_value_str = i915_fence_timeline_value_str,
 };
 
 int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
@@ -154,6 +113,82 @@ i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
 	spin_unlock(&file_priv->mm.lock);
 }
 
+static struct i915_dependency *
+i915_dependency_alloc(struct drm_i915_private *i915)
+{
+	return kmem_cache_alloc(i915->dependencies, GFP_KERNEL);
+}
+
+static void
+i915_dependency_free(struct drm_i915_private *i915,
+		     struct i915_dependency *dep)
+{
+	kmem_cache_free(i915->dependencies, dep);
+}
+
+static void
+__i915_priotree_add_dependency(struct i915_priotree *pt,
+			       struct i915_priotree *signal,
+			       struct i915_dependency *dep,
+			       unsigned long flags)
+{
+	INIT_LIST_HEAD(&dep->dfs_link);
+	list_add(&dep->wait_link, &signal->waiters_list);
+	list_add(&dep->signal_link, &pt->signalers_list);
+	dep->signaler = signal;
+	dep->flags = flags;
+}
+
+static int
+i915_priotree_add_dependency(struct drm_i915_private *i915,
+			     struct i915_priotree *pt,
+			     struct i915_priotree *signal)
+{
+	struct i915_dependency *dep;
+
+	dep = i915_dependency_alloc(i915);
+	if (!dep)
+		return -ENOMEM;
+
+	__i915_priotree_add_dependency(pt, signal, dep, I915_DEPENDENCY_ALLOC);
+	return 0;
+}
+
+static void
+i915_priotree_fini(struct drm_i915_private *i915, struct i915_priotree *pt)
+{
+	struct i915_dependency *dep, *next;
+
+	GEM_BUG_ON(!RB_EMPTY_NODE(&pt->node));
+
+	/* Everyone we depended upon (the fences we wait to be signaled)
+	 * should retire before us and remove themselves from our list.
+	 * However, retirement is run independently on each timeline and
+	 * so we may be called out-of-order.
+	 */
+	list_for_each_entry_safe(dep, next, &pt->signalers_list, signal_link) {
+		list_del(&dep->wait_link);
+		if (dep->flags & I915_DEPENDENCY_ALLOC)
+			i915_dependency_free(i915, dep);
+	}
+
+	/* Remove ourselves from everyone who depends upon us */
+	list_for_each_entry_safe(dep, next, &pt->waiters_list, wait_link) {
+		list_del(&dep->signal_link);
+		if (dep->flags & I915_DEPENDENCY_ALLOC)
+			i915_dependency_free(i915, dep);
+	}
+}
+
+static void
+i915_priotree_init(struct i915_priotree *pt)
+{
+	INIT_LIST_HEAD(&pt->signalers_list);
+	INIT_LIST_HEAD(&pt->waiters_list);
+	RB_CLEAR_NODE(&pt->node);
+	pt->priority = INT_MIN;
+}
+
 void i915_gem_retire_noop(struct i915_gem_active *active,
 			  struct drm_i915_gem_request *request)
 {
@@ -164,8 +199,17 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
 {
 	struct i915_gem_active *active, *next;
 
+	lockdep_assert_held(&request->i915->drm.struct_mutex);
+	GEM_BUG_ON(!i915_sw_fence_signaled(&request->submit));
+	GEM_BUG_ON(!i915_sw_fence_signaled(&request->execute));
+	GEM_BUG_ON(!i915_gem_request_completed(request));
+	GEM_BUG_ON(!request->i915->gt.active_requests);
+
 	trace_i915_gem_request_retire(request);
-	list_del(&request->link);
+
+	spin_lock_irq(&request->engine->timeline->lock);
+	list_del_init(&request->link);
+	spin_unlock_irq(&request->engine->timeline->lock);
 
 	/* We know the GPU must have read the request to have
 	 * sent us the seqno + interrupt, so use the position
@@ -177,6 +221,12 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
 	 */
 	list_del(&request->ring_link);
 	request->ring->last_retired_head = request->postfix;
+	if (!--request->i915->gt.active_requests) {
+		GEM_BUG_ON(!request->i915->gt.awake);
+		mod_delayed_work(request->i915->wq,
+				 &request->i915->gt.idle_work,
+				 msecs_to_jiffies(100));
+	}
 
 	/* Walk through the active list, calling retire on each. This allows
 	 * objects to track their GPU activity and mark themselves as idle
@@ -214,6 +264,10 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
 	}
 
 	i915_gem_context_put(request->ctx);
+
+	dma_fence_signal(&request->fence);
+
+	i915_priotree_fini(request->i915, &request->priotree);
 	i915_gem_request_put(request);
 }
 
@@ -223,10 +277,11 @@ void i915_gem_request_retire_upto(struct drm_i915_gem_request *req)
 	struct drm_i915_gem_request *tmp;
 
 	lockdep_assert_held(&req->i915->drm.struct_mutex);
-	GEM_BUG_ON(list_empty(&req->link));
+	if (list_empty(&req->link))
+		return;
 
 	do {
-		tmp = list_first_entry(&engine->request_list,
+		tmp = list_first_entry(&engine->timeline->requests,
 				       typeof(*tmp), link);
 
 		i915_gem_request_retire(tmp);
@@ -253,39 +308,50 @@ static int i915_gem_check_wedge(struct drm_i915_private *dev_priv)
 	return 0;
 }
 
-static int i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
+static int i915_gem_init_global_seqno(struct drm_i915_private *i915, u32 seqno)
 {
+	struct i915_gem_timeline *timeline = &i915->gt.global_timeline;
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	int ret;
 
 	/* Carefully retire all requests without writing to the rings */
-	for_each_engine(engine, dev_priv) {
-		ret = intel_engine_idle(engine,
-					I915_WAIT_INTERRUPTIBLE |
-					I915_WAIT_LOCKED);
-		if (ret)
-			return ret;
-	}
-	i915_gem_retire_requests(dev_priv);
+	ret = i915_gem_wait_for_idle(i915,
+				     I915_WAIT_INTERRUPTIBLE |
+				     I915_WAIT_LOCKED);
+	if (ret)
+		return ret;
+
+	i915_gem_retire_requests(i915);
+	GEM_BUG_ON(i915->gt.active_requests > 1);
 
 	/* If the seqno wraps around, we need to clear the breadcrumb rbtree */
-	if (!i915_seqno_passed(seqno, dev_priv->next_seqno)) {
-		while (intel_kick_waiters(dev_priv) ||
-		       intel_kick_signalers(dev_priv))
-			yield();
+	if (!i915_seqno_passed(seqno, atomic_read(&timeline->next_seqno))) {
+		while (intel_breadcrumbs_busy(i915))
+			cond_resched(); /* spin until threads are complete */
 	}
+	atomic_set(&timeline->next_seqno, seqno);
 
 	/* Finally reset hw state */
-	for_each_engine(engine, dev_priv)
-		intel_engine_init_seqno(engine, seqno);
+	for_each_engine(engine, i915, id)
+		intel_engine_init_global_seqno(engine, seqno);
+
+	list_for_each_entry(timeline, &i915->gt.timelines, link) {
+		for_each_engine(engine, i915, id) {
+			struct intel_timeline *tl = &timeline->engine[id];
+
+			memset(tl->sync_seqno, 0, sizeof(tl->sync_seqno));
+		}
+	}
 
 	return 0;
 }
 
-int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
+int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	int ret;
+
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
 	if (seqno == 0)
 		return -EINVAL;
@@ -293,29 +359,87 @@ int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
 	/* HWS page needs to be set less than what we
 	 * will inject to ring
 	 */
-	ret = i915_gem_init_seqno(dev_priv, seqno - 1);
-	if (ret)
+	return i915_gem_init_global_seqno(dev_priv, seqno - 1);
+}
+
+static int reserve_global_seqno(struct drm_i915_private *i915)
+{
+	u32 active_requests = ++i915->gt.active_requests;
+	u32 next_seqno = atomic_read(&i915->gt.global_timeline.next_seqno);
+	int ret;
+
+	/* Reservation is fine until we need to wrap around */
+	if (likely(next_seqno + active_requests > next_seqno))
+		return 0;
+
+	ret = i915_gem_init_global_seqno(i915, 0);
+	if (ret) {
+		i915->gt.active_requests--;
 		return ret;
+	}
 
-	dev_priv->next_seqno = seqno;
 	return 0;
 }
 
-static int i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno)
+static u32 __timeline_get_seqno(struct i915_gem_timeline *tl)
 {
-	/* reserve 0 for non-seqno */
-	if (unlikely(dev_priv->next_seqno == 0)) {
-		int ret;
+	/* next_seqno only incremented under a mutex */
+	return ++tl->next_seqno.counter;
+}
 
-		ret = i915_gem_init_seqno(dev_priv, 0);
-		if (ret)
-			return ret;
+static u32 timeline_get_seqno(struct i915_gem_timeline *tl)
+{
+	return atomic_inc_return(&tl->next_seqno);
+}
 
-		dev_priv->next_seqno = 1;
-	}
+void __i915_gem_request_submit(struct drm_i915_gem_request *request)
+{
+	struct intel_engine_cs *engine = request->engine;
+	struct intel_timeline *timeline;
+	u32 seqno;
 
-	*seqno = dev_priv->next_seqno++;
-	return 0;
+	/* Transfer from per-context onto the global per-engine timeline */
+	timeline = engine->timeline;
+	GEM_BUG_ON(timeline == request->timeline);
+	assert_spin_locked(&timeline->lock);
+
+	seqno = timeline_get_seqno(timeline->common);
+	GEM_BUG_ON(!seqno);
+	GEM_BUG_ON(i915_seqno_passed(intel_engine_get_seqno(engine), seqno));
+
+	GEM_BUG_ON(i915_seqno_passed(timeline->last_submitted_seqno, seqno));
+	request->previous_seqno = timeline->last_submitted_seqno;
+	timeline->last_submitted_seqno = seqno;
+
+	/* We may be recursing from the signal callback of another i915 fence */
+	spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
+	request->global_seqno = seqno;
+	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
+		intel_engine_enable_signaling(request);
+	spin_unlock(&request->lock);
+
+	GEM_BUG_ON(!request->global_seqno);
+	engine->emit_breadcrumb(request,
+				request->ring->vaddr + request->postfix);
+
+	spin_lock(&request->timeline->lock);
+	list_move_tail(&request->link, &timeline->requests);
+	spin_unlock(&request->timeline->lock);
+
+	i915_sw_fence_commit(&request->execute);
+}
+
+void i915_gem_request_submit(struct drm_i915_gem_request *request)
+{
+	struct intel_engine_cs *engine = request->engine;
+	unsigned long flags;
+
+	/* Will be called from irq-context when using foreign fences. */
+	spin_lock_irqsave(&engine->timeline->lock, flags);
+
+	__i915_gem_request_submit(request);
+
+	spin_unlock_irqrestore(&engine->timeline->lock, flags);
 }
 
 static int __i915_sw_fence_call
@@ -324,15 +448,31 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
 	struct drm_i915_gem_request *request =
 		container_of(fence, typeof(*request), submit);
 
-	/* Will be called from irq-context when using foreign DMA fences */
-
 	switch (state) {
 	case FENCE_COMPLETE:
-		request->engine->last_submitted_seqno = request->fence.seqno;
 		request->engine->submit_request(request);
 		break;
 
 	case FENCE_FREE:
+		i915_gem_request_put(request);
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
+static int __i915_sw_fence_call
+execute_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
+{
+	struct drm_i915_gem_request *request =
+		container_of(fence, typeof(*request), execute);
+
+	switch (state) {
+	case FENCE_COMPLETE:
+		break;
+
+	case FENCE_FREE:
+		i915_gem_request_put(request);
 		break;
 	}
 
@@ -357,9 +497,10 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 	struct drm_i915_gem_request *req;
-	u32 seqno;
 	int ret;
 
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+
 	/* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
 	 * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
 	 * and restart.
@@ -368,10 +509,14 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 	if (ret)
 		return ERR_PTR(ret);
 
+	ret = reserve_global_seqno(dev_priv);
+	if (ret)
+		return ERR_PTR(ret);
+
 	/* Move the oldest request to the slab-cache (if not in use!) */
-	req = list_first_entry_or_null(&engine->request_list,
+	req = list_first_entry_or_null(&engine->timeline->requests,
 				       typeof(*req), link);
-	if (req && i915_gem_request_completed(req))
+	if (req && __i915_gem_request_completed(req))
 		i915_gem_request_retire(req);
 
 	/* Beware: Dragons be flying overhead.
@@ -382,13 +527,13 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 	 * of being read by __i915_gem_active_get_rcu(). As such,
 	 * we have to be very careful when overwriting the contents. During
 	 * the RCU lookup, we change chase the request->engine pointer,
-	 * read the request->fence.seqno and increment the reference count.
+	 * read the request->global_seqno and increment the reference count.
 	 *
 	 * The reference count is incremented atomically. If it is zero,
 	 * the lookup knows the request is unallocated and complete. Otherwise,
 	 * it is either still in use, or has been reallocated and reset
-	 * with fence_init(). This increment is safe for release as we check
-	 * that the request we have a reference to and matches the active
+	 * with dma_fence_init(). This increment is safe for release as we
+	 * check that the request we have a reference to and matches the active
 	 * request.
 	 *
 	 * Before we increment the refcount, we chase the request->engine
@@ -403,21 +548,32 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 	 * Do not use kmem_cache_zalloc() here!
 	 */
 	req = kmem_cache_alloc(dev_priv->requests, GFP_KERNEL);
-	if (!req)
-		return ERR_PTR(-ENOMEM);
+	if (!req) {
+		ret = -ENOMEM;
+		goto err_unreserve;
+	}
 
-	ret = i915_gem_get_seqno(dev_priv, &seqno);
-	if (ret)
-		goto err;
+	req->timeline = i915_gem_context_lookup_timeline(ctx, engine);
+	GEM_BUG_ON(req->timeline == engine->timeline);
 
 	spin_lock_init(&req->lock);
-	fence_init(&req->fence,
-		   &i915_fence_ops,
-		   &req->lock,
-		   engine->fence_context,
-		   seqno);
+	dma_fence_init(&req->fence,
+		       &i915_fence_ops,
+		       &req->lock,
+		       req->timeline->fence_context,
+		       __timeline_get_seqno(req->timeline->common));
+
+	/* We bump the ref for the fence chain */
+	i915_sw_fence_init(&i915_gem_request_get(req)->submit, submit_notify);
+	i915_sw_fence_init(&i915_gem_request_get(req)->execute, execute_notify);
+
+	/* Ensure that the execute fence completes after the submit fence -
+	 * as we complete the execute fence from within the submit fence
+	 * callback, its completion would otherwise be visible first.
+	 */
+	i915_sw_fence_await_sw_fence(&req->execute, &req->submit, &req->execq);
 
-	i915_sw_fence_init(&req->submit, submit_notify);
+	i915_priotree_init(&req->priotree);
 
 	INIT_LIST_HEAD(&req->active_list);
 	req->i915 = dev_priv;
@@ -425,6 +581,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 	req->ctx = i915_gem_context_get(ctx);
 
 	/* No zalloc, must clear what we need by hand */
+	req->global_seqno = 0;
 	req->previous_context = NULL;
 	req->file_priv = NULL;
 	req->batch = NULL;
@@ -437,6 +594,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 	 * away, e.g. because a GPU scheduler has deferred it.
 	 */
 	req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
+	GEM_BUG_ON(req->reserved_space < engine->emit_breadcrumb_sz);
 
 	if (i915.enable_execlists)
 		ret = intel_logical_ring_alloc_request_extras(req);
@@ -456,8 +614,9 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 
 err_ctx:
 	i915_gem_context_put(ctx);
-err:
 	kmem_cache_free(dev_priv->requests, req);
+err_unreserve:
+	dev_priv->gt.active_requests--;
 	return ERR_PTR(ret);
 }
 
@@ -465,15 +624,36 @@ static int
 i915_gem_request_await_request(struct drm_i915_gem_request *to,
 			       struct drm_i915_gem_request *from)
 {
-	int idx, ret;
+	int ret;
 
 	GEM_BUG_ON(to == from);
 
-	if (to->engine == from->engine)
+	if (to->engine->schedule) {
+		ret = i915_priotree_add_dependency(to->i915,
+						   &to->priotree,
+						   &from->priotree);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (to->timeline == from->timeline)
 		return 0;
 
-	idx = intel_engine_sync_index(from->engine, to->engine);
-	if (from->fence.seqno <= from->engine->semaphore.sync_seqno[idx])
+	if (to->engine == from->engine) {
+		ret = i915_sw_fence_await_sw_fence_gfp(&to->submit,
+						       &from->submit,
+						       GFP_KERNEL);
+		return ret < 0 ? ret : 0;
+	}
+
+	if (!from->global_seqno) {
+		ret = i915_sw_fence_await_dma_fence(&to->submit,
+						    &from->fence, 0,
+						    GFP_KERNEL);
+		return ret < 0 ? ret : 0;
+	}
+
+	if (from->global_seqno <= to->timeline->sync_seqno[from->engine->id])
 		return 0;
 
 	trace_i915_gem_ring_sync_to(to, from);
@@ -491,7 +671,54 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,
 			return ret;
 	}
 
-	from->engine->semaphore.sync_seqno[idx] = from->fence.seqno;
+	to->timeline->sync_seqno[from->engine->id] = from->global_seqno;
+	return 0;
+}
+
+int
+i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
+				 struct dma_fence *fence)
+{
+	struct dma_fence_array *array;
+	int ret;
+	int i;
+
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+		return 0;
+
+	if (dma_fence_is_i915(fence))
+		return i915_gem_request_await_request(req, to_request(fence));
+
+	if (!dma_fence_is_array(fence)) {
+		ret = i915_sw_fence_await_dma_fence(&req->submit,
+						    fence, I915_FENCE_TIMEOUT,
+						    GFP_KERNEL);
+		return ret < 0 ? ret : 0;
+	}
+
+	/* Note that if the fence-array was created in signal-on-any mode,
+	 * we should *not* decompose it into its individual fences. However,
+	 * we don't currently store which mode the fence-array is operating
+	 * in. Fortunately, the only user of signal-on-any is private to
+	 * amdgpu and we should not see any incoming fence-array from
+	 * sync-file being in signal-on-any mode.
+	 */
+
+	array = to_dma_fence_array(fence);
+	for (i = 0; i < array->num_fences; i++) {
+		struct dma_fence *child = array->fences[i];
+
+		if (dma_fence_is_i915(child))
+			ret = i915_gem_request_await_request(req,
+							     to_request(child));
+		else
+			ret = i915_sw_fence_await_dma_fence(&req->submit,
+							    child, I915_FENCE_TIMEOUT,
+							    GFP_KERNEL);
+		if (ret < 0)
+			return ret;
+	}
+
 	return 0;
 }
 
@@ -520,43 +747,52 @@ i915_gem_request_await_object(struct drm_i915_gem_request *to,
 			      struct drm_i915_gem_object *obj,
 			      bool write)
 {
-	struct i915_gem_active *active;
-	unsigned long active_mask;
-	int idx;
+	struct dma_fence *excl;
+	int ret = 0;
 
 	if (write) {
-		active_mask = i915_gem_object_get_active(obj);
-		active = obj->last_read;
+		struct dma_fence **shared;
+		unsigned int count, i;
+
+		ret = reservation_object_get_fences_rcu(obj->resv,
+							&excl, &count, &shared);
+		if (ret)
+			return ret;
+
+		for (i = 0; i < count; i++) {
+			ret = i915_gem_request_await_dma_fence(to, shared[i]);
+			if (ret)
+				break;
+
+			dma_fence_put(shared[i]);
+		}
+
+		for (; i < count; i++)
+			dma_fence_put(shared[i]);
+		kfree(shared);
 	} else {
-		active_mask = 1;
-		active = &obj->last_write;
+		excl = reservation_object_get_excl_rcu(obj->resv);
 	}
 
-	for_each_active(active_mask, idx) {
-		struct drm_i915_gem_request *request;
-		int ret;
-
-		request = i915_gem_active_peek(&active[idx],
-					       &obj->base.dev->struct_mutex);
-		if (!request)
-			continue;
+	if (excl) {
+		if (ret == 0)
+			ret = i915_gem_request_await_dma_fence(to, excl);
 
-		ret = i915_gem_request_await_request(to, request);
-		if (ret)
-			return ret;
+		dma_fence_put(excl);
 	}
 
-	return 0;
+	return ret;
 }
 
 static void i915_gem_mark_busy(const struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 
-	dev_priv->gt.active_engines |= intel_engine_flag(engine);
 	if (dev_priv->gt.awake)
 		return;
 
+	GEM_BUG_ON(!dev_priv->gt.active_requests);
+
 	intel_runtime_pm_get_noresume(dev_priv);
 	dev_priv->gt.awake = true;
 
@@ -579,11 +815,11 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
 {
 	struct intel_engine_cs *engine = request->engine;
 	struct intel_ring *ring = request->ring;
+	struct intel_timeline *timeline = request->timeline;
 	struct drm_i915_gem_request *prev;
-	u32 request_start;
-	u32 reserved_tail;
-	int ret;
+	int err;
 
+	lockdep_assert_held(&request->i915->drm.struct_mutex);
 	trace_i915_gem_request_add(request);
 
 	/*
@@ -591,8 +827,6 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
 	 * should already have been reserved in the ring buffer. Let the ring
 	 * know that it is time to use that space up.
 	 */
-	request_start = ring->tail;
-	reserved_tail = request->reserved_space;
 	request->reserved_space = 0;
 
 	/*
@@ -603,10 +837,10 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
 	 * what.
 	 */
 	if (flush_caches) {
-		ret = engine->emit_flush(request, EMIT_FLUSH);
+		err = engine->emit_flush(request, EMIT_FLUSH);
 
 		/* Not allowed to fail! */
-		WARN(ret, "engine->emit_flush() failed: %d!\n", ret);
+		WARN(err, "engine->emit_flush() failed: %d!\n", err);
 	}
 
 	/* Record the position of the start of the breadcrumb so that
@@ -614,20 +848,10 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
 	 * GPU processing the request, we never over-estimate the
 	 * position of the ring's HEAD.
 	 */
+	err = intel_ring_begin(request, engine->emit_breadcrumb_sz);
+	GEM_BUG_ON(err);
 	request->postfix = ring->tail;
-
-	/* Not allowed to fail! */
-	ret = engine->emit_request(request);
-	WARN(ret, "(%s)->emit_request failed: %d!\n", engine->name, ret);
-
-	/* Sanity check that the reserved size was large enough. */
-	ret = ring->tail - request_start;
-	if (ret < 0)
-		ret += ring->size;
-	WARN_ONCE(ret > reserved_tail,
-		  "Not enough space reserved (%d bytes) "
-		  "for adding the request (%d bytes)\n",
-		  reserved_tail, ret);
+	ring->tail += engine->emit_breadcrumb_sz * sizeof(u32);
 
 	/* Seal the request and mark it as pending execution. Note that
 	 * we may inspect this state, without holding any locks, during
@@ -635,21 +859,46 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
 	 * see a more recent value in the hws than we are tracking.
 	 */
 
-	prev = i915_gem_active_raw(&engine->last_request,
+	prev = i915_gem_active_raw(&timeline->last_request,
 				   &request->i915->drm.struct_mutex);
-	if (prev)
+	if (prev) {
 		i915_sw_fence_await_sw_fence(&request->submit, &prev->submit,
 					     &request->submitq);
+		if (engine->schedule)
+			__i915_priotree_add_dependency(&request->priotree,
+						       &prev->priotree,
+						       &request->dep,
+						       0);
+	}
+
+	spin_lock_irq(&timeline->lock);
+	list_add_tail(&request->link, &timeline->requests);
+	spin_unlock_irq(&timeline->lock);
+
+	GEM_BUG_ON(i915_seqno_passed(timeline->last_submitted_seqno,
+				     request->fence.seqno));
+
+	timeline->last_submitted_seqno = request->fence.seqno;
+	i915_gem_active_set(&timeline->last_request, request);
 
-	request->emitted_jiffies = jiffies;
-	request->previous_seqno = engine->last_pending_seqno;
-	engine->last_pending_seqno = request->fence.seqno;
-	i915_gem_active_set(&engine->last_request, request);
-	list_add_tail(&request->link, &engine->request_list);
 	list_add_tail(&request->ring_link, &ring->request_list);
+	request->emitted_jiffies = jiffies;
 
 	i915_gem_mark_busy(engine);
 
+	/* Let the backend know a new request has arrived that may need
+	 * to adjust the existing execution schedule due to a high priority
+	 * request - i.e. we may want to preempt the current request in order
+	 * to run a high priority dependency chain *before* we can execute this
+	 * request.
+	 *
+	 * This is called before the request is ready to run so that we can
+	 * decide whether to preempt the entire chain so that it is ready to
+	 * run at the earliest possible convenience.
+	 */
+	if (engine->schedule)
+		engine->schedule(request, request->ctx->priority);
+
 	local_bh_disable();
 	i915_sw_fence_commit(&request->submit);
 	local_bh_enable(); /* Kick the execlists tasklet if just scheduled */
@@ -714,7 +963,7 @@ bool __i915_spin_request(const struct drm_i915_gem_request *req,
 
 	timeout_us += local_clock_us(&cpu);
 	do {
-		if (i915_gem_request_completed(req))
+		if (__i915_gem_request_completed(req))
 			return true;
 
 		if (signal_pending_state(state, current))
@@ -729,76 +978,102 @@ bool __i915_spin_request(const struct drm_i915_gem_request *req,
 	return false;
 }
 
+static long
+__i915_request_wait_for_execute(struct drm_i915_gem_request *request,
+				unsigned int flags,
+				long timeout)
+{
+	const int state = flags & I915_WAIT_INTERRUPTIBLE ?
+		TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
+	wait_queue_head_t *q = &request->i915->gpu_error.wait_queue;
+	DEFINE_WAIT(reset);
+	DEFINE_WAIT(wait);
+
+	if (flags & I915_WAIT_LOCKED)
+		add_wait_queue(q, &reset);
+
+	do {
+		prepare_to_wait(&request->execute.wait, &wait, state);
+
+		if (i915_sw_fence_done(&request->execute))
+			break;
+
+		if (flags & I915_WAIT_LOCKED &&
+		    i915_reset_in_progress(&request->i915->gpu_error)) {
+			__set_current_state(TASK_RUNNING);
+			i915_reset(request->i915);
+			reset_wait_queue(q, &reset);
+			continue;
+		}
+
+		if (signal_pending_state(state, current)) {
+			timeout = -ERESTARTSYS;
+			break;
+		}
+
+		timeout = io_schedule_timeout(timeout);
+	} while (timeout);
+	finish_wait(&request->execute.wait, &wait);
+
+	if (flags & I915_WAIT_LOCKED)
+		remove_wait_queue(q, &reset);
+
+	return timeout;
+}
+
 /**
  * i915_wait_request - wait until execution of request has finished
- * @req: duh!
+ * @req: the request to wait upon
  * @flags: how to wait
- * @timeout: in - how long to wait (NULL forever); out - how much time remaining
- * @rps: client to charge for RPS boosting
+ * @timeout: how long to wait in jiffies
+ *
+ * i915_wait_request() waits for the request to be completed, for a
+ * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an
+ * unbounded wait).
  *
- * Note: It is of utmost importance that the passed in seqno and reset_counter
- * values have been read by the caller in an smp safe manner. Where read-side
- * locks are involved, it is sufficient to read the reset_counter before
- * unlocking the lock that protects the seqno. For lockless tricks, the
- * reset_counter _must_ be read before, and an appropriate smp_rmb must be
- * inserted.
+ * If the caller holds the struct_mutex, the caller must pass I915_WAIT_LOCKED
+ * in via the flags, and vice versa if the struct_mutex is not held, the caller
+ * must not specify that the wait is locked.
  *
- * Returns 0 if the request was found within the alloted time. Else returns the
- * errno with remaining time filled in timeout argument.
+ * Returns the remaining time (in jiffies) if the request completed, which may
+ * be zero or -ETIME if the request is unfinished after the timeout expires.
+ * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
+ * pending before the request completes.
  */
-int i915_wait_request(struct drm_i915_gem_request *req,
-		      unsigned int flags,
-		      s64 *timeout,
-		      struct intel_rps_client *rps)
+long i915_wait_request(struct drm_i915_gem_request *req,
+		       unsigned int flags,
+		       long timeout)
 {
 	const int state = flags & I915_WAIT_INTERRUPTIBLE ?
 		TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
 	DEFINE_WAIT(reset);
 	struct intel_wait wait;
-	unsigned long timeout_remain;
-	int ret = 0;
 
 	might_sleep();
 #if IS_ENABLED(CONFIG_LOCKDEP)
-	GEM_BUG_ON(!!lockdep_is_held(&req->i915->drm.struct_mutex) !=
+	GEM_BUG_ON(debug_locks &&
+		   !!lockdep_is_held(&req->i915->drm.struct_mutex) !=
 		   !!(flags & I915_WAIT_LOCKED));
 #endif
+	GEM_BUG_ON(timeout < 0);
 
 	if (i915_gem_request_completed(req))
-		return 0;
-
-	timeout_remain = MAX_SCHEDULE_TIMEOUT;
-	if (timeout) {
-		if (WARN_ON(*timeout < 0))
-			return -EINVAL;
+		return timeout;
 
-		if (*timeout == 0)
-			return -ETIME;
-
-		/* Record current time in case interrupted, or wedged */
-		timeout_remain = nsecs_to_jiffies_timeout(*timeout);
-		*timeout += ktime_get_raw_ns();
-	}
+	if (!timeout)
+		return -ETIME;
 
 	trace_i915_gem_request_wait_begin(req);
 
-	/* This client is about to stall waiting for the GPU. In many cases
-	 * this is undesirable and limits the throughput of the system, as
-	 * many clients cannot continue processing user input/output whilst
-	 * blocked. RPS autotuning may take tens of milliseconds to respond
-	 * to the GPU load and thus incurs additional latency for the client.
-	 * We can circumvent that by promoting the GPU frequency to maximum
-	 * before we wait. This makes the GPU throttle up much more quickly
-	 * (good for benchmarks and user experience, e.g. window animations),
-	 * but at a cost of spending more power processing the workload
-	 * (bad for battery). Not all clients even want their results
-	 * immediately and for them we should just let the GPU select its own
-	 * frequency to maximise efficiency. To prevent a single client from
-	 * forcing the clocks too high for the whole system, we only allow
-	 * each client to waitboost once in a busy period.
-	 */
-	if (IS_RPS_CLIENT(rps) && INTEL_GEN(req->i915) >= 6)
-		gen6_rps_boost(req->i915, rps, req->emitted_jiffies);
+	if (!i915_sw_fence_done(&req->execute)) {
+		timeout = __i915_request_wait_for_execute(req, flags, timeout);
+		if (timeout < 0)
+			goto complete;
+
+		GEM_BUG_ON(!i915_sw_fence_done(&req->execute));
+	}
+	GEM_BUG_ON(!i915_sw_fence_done(&req->submit));
+	GEM_BUG_ON(!req->global_seqno);
 
 	/* Optimistic short spin before touching IRQs */
 	if (i915_spin_request(req, state, 5))
@@ -808,7 +1083,7 @@ int i915_wait_request(struct drm_i915_gem_request *req,
 	if (flags & I915_WAIT_LOCKED)
 		add_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
 
-	intel_wait_init(&wait, req->fence.seqno);
+	intel_wait_init(&wait, req->global_seqno);
 	if (intel_engine_add_wait(req->engine, &wait))
 		/* In order to check that we haven't missed the interrupt
 		 * as we enabled it, we need to kick ourselves to do a
@@ -818,16 +1093,17 @@ int i915_wait_request(struct drm_i915_gem_request *req,
 
 	for (;;) {
 		if (signal_pending_state(state, current)) {
-			ret = -ERESTARTSYS;
+			timeout = -ERESTARTSYS;
 			break;
 		}
 
-		timeout_remain = io_schedule_timeout(timeout_remain);
-		if (timeout_remain == 0) {
-			ret = -ETIME;
+		if (!timeout) {
+			timeout = -ETIME;
 			break;
 		}
 
+		timeout = io_schedule_timeout(timeout);
+
 		if (intel_wait_complete(&wait))
 			break;
 
@@ -874,74 +1150,32 @@ wakeup:
 complete:
 	trace_i915_gem_request_wait_end(req);
 
-	if (timeout) {
-		*timeout -= ktime_get_raw_ns();
-		if (*timeout < 0)
-			*timeout = 0;
-
-		/*
-		 * Apparently ktime isn't accurate enough and occasionally has a
-		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
-		 * things up to make the test happy. We allow up to 1 jiffy.
-		 *
-		 * This is a regrssion from the timespec->ktime conversion.
-		 */
-		if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
-			*timeout = 0;
-	}
-
-	if (IS_RPS_USER(rps) &&
-	    req->fence.seqno == req->engine->last_submitted_seqno) {
-		/* The GPU is now idle and this client has stalled.
-		 * Since no other client has submitted a request in the
-		 * meantime, assume that this client is the only one
-		 * supplying work to the GPU but is unable to keep that
-		 * work supplied because it is waiting. Since the GPU is
-		 * then never kept fully busy, RPS autoclocking will
-		 * keep the clocks relatively low, causing further delays.
-		 * Compensate by giving the synchronous client credit for
-		 * a waitboost next time.
-		 */
-		spin_lock(&req->i915->rps.client_lock);
-		list_del_init(&rps->link);
-		spin_unlock(&req->i915->rps.client_lock);
-	}
-
-	return ret;
+	return timeout;
 }
 
-static bool engine_retire_requests(struct intel_engine_cs *engine)
+static void engine_retire_requests(struct intel_engine_cs *engine)
 {
 	struct drm_i915_gem_request *request, *next;
 
-	list_for_each_entry_safe(request, next, &engine->request_list, link) {
-		if (!i915_gem_request_completed(request))
-			return false;
+	list_for_each_entry_safe(request, next,
+				 &engine->timeline->requests, link) {
+		if (!__i915_gem_request_completed(request))
+			return;
 
 		i915_gem_request_retire(request);
 	}
-
-	return true;
 }
 
 void i915_gem_retire_requests(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
-	unsigned int tmp;
+	enum intel_engine_id id;
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
-	if (dev_priv->gt.active_engines == 0)
+	if (!dev_priv->gt.active_requests)
 		return;
 
-	GEM_BUG_ON(!dev_priv->gt.awake);
-
-	for_each_engine_masked(engine, dev_priv, dev_priv->gt.active_engines, tmp)
-		if (engine_retire_requests(engine))
-			dev_priv->gt.active_engines &= ~intel_engine_flag(engine);
-
-	if (dev_priv->gt.active_engines == 0)
-		queue_delayed_work(dev_priv->wq,
-				   &dev_priv->gt.idle_work,
-				   msecs_to_jiffies(100));
+	for_each_engine(engine, dev_priv, id)
+		engine_retire_requests(engine);
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h
index 974bd7bcc801..e2b077df2da0 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.h
+++ b/drivers/gpu/drm/i915/i915_gem_request.h
@@ -25,11 +25,14 @@
 #ifndef I915_GEM_REQUEST_H
 #define I915_GEM_REQUEST_H
 
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 
 #include "i915_gem.h"
 #include "i915_sw_fence.h"
 
+struct drm_file;
+struct drm_i915_gem_object;
+
 struct intel_wait {
 	struct rb_node node;
 	struct task_struct *tsk;
@@ -41,6 +44,33 @@ struct intel_signal_node {
 	struct intel_wait wait;
 };
 
+struct i915_dependency {
+	struct i915_priotree *signaler;
+	struct list_head signal_link;
+	struct list_head wait_link;
+	struct list_head dfs_link;
+	unsigned long flags;
+#define I915_DEPENDENCY_ALLOC BIT(0)
+};
+
+/* Requests exist in a complex web of interdependencies. Each request
+ * has to wait for some other request to complete before it is ready to be run
+ * (e.g. we have to wait until the pixels have been rendering into a texture
+ * before we can copy from it). We track the readiness of a request in terms
+ * of fences, but we also need to keep the dependency tree for the lifetime
+ * of the request (beyond the life of an individual fence). We use the tree
+ * at various points to reorder the requests whilst keeping the requests
+ * in order with respect to their various dependencies.
+ */
+struct i915_priotree {
+	struct list_head signalers_list; /* those before us, we depend upon */
+	struct list_head waiters_list; /* those after us, they depend upon us */
+	struct rb_node node;
+	int priority;
+#define I915_PRIORITY_MAX 1024
+#define I915_PRIORITY_MIN (-I915_PRIORITY_MAX)
+};
+
 /**
  * Request queue structure.
  *
@@ -62,7 +92,7 @@ struct intel_signal_node {
  * The requests are reference counted.
  */
 struct drm_i915_gem_request {
-	struct fence fence;
+	struct dma_fence fence;
 	spinlock_t lock;
 
 	/** On Which ring this request was generated */
@@ -81,10 +111,39 @@ struct drm_i915_gem_request {
 	struct i915_gem_context *ctx;
 	struct intel_engine_cs *engine;
 	struct intel_ring *ring;
+	struct intel_timeline *timeline;
 	struct intel_signal_node signaling;
 
+	/* Fences for the various phases in the request's lifetime.
+	 *
+	 * The submit fence is used to await upon all of the request's
+	 * dependencies. When it is signaled, the request is ready to run.
+	 * It is used by the driver to then queue the request for execution.
+	 *
+	 * The execute fence is used to signal when the request has been
+	 * sent to hardware.
+	 *
+	 * It is illegal for the submit fence of one request to wait upon the
+	 * execute fence of an earlier request. It should be sufficient to
+	 * wait upon the submit fence of the earlier request.
+	 */
 	struct i915_sw_fence submit;
+	struct i915_sw_fence execute;
 	wait_queue_t submitq;
+	wait_queue_t execq;
+
+	/* A list of everyone we wait upon, and everyone who waits upon us.
+	 * Even though we will not be submitted to the hardware before the
+	 * submit fence is signaled (it waits for all external events as well
+	 * as our own requests), the scheduler still needs to know the
+	 * dependency tree for the lifetime of the request (from execbuf
+	 * to retirement), i.e. bidirectional dependency information for the
+	 * request not tied to individual fences.
+	 */
+	struct i915_priotree priotree;
+	struct i915_dependency dep;
+
+	u32 global_seqno;
 
 	/** GEM sequence number associated with the previous request,
 	 * when the HWS breadcrumb is equal to this the GPU is processing
@@ -140,14 +199,11 @@ struct drm_i915_gem_request {
 	struct drm_i915_file_private *file_priv;
 	/** file_priv list entry for this request */
 	struct list_head client_list;
-
-	/** Link in the execlist submission queue, guarded by execlist_lock. */
-	struct list_head execlist_link;
 };
 
-extern const struct fence_ops i915_fence_ops;
+extern const struct dma_fence_ops i915_fence_ops;
 
-static inline bool fence_is_i915(struct fence *fence)
+static inline bool dma_fence_is_i915(const struct dma_fence *fence)
 {
 	return fence->ops == &i915_fence_ops;
 }
@@ -159,43 +215,31 @@ int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
 				   struct drm_file *file);
 void i915_gem_request_retire_upto(struct drm_i915_gem_request *req);
 
-static inline u32
-i915_gem_request_get_seqno(struct drm_i915_gem_request *req)
-{
-	return req ? req->fence.seqno : 0;
-}
-
-static inline struct intel_engine_cs *
-i915_gem_request_get_engine(struct drm_i915_gem_request *req)
-{
-	return req ? req->engine : NULL;
-}
-
 static inline struct drm_i915_gem_request *
-to_request(struct fence *fence)
+to_request(struct dma_fence *fence)
 {
 	/* We assume that NULL fence/request are interoperable */
 	BUILD_BUG_ON(offsetof(struct drm_i915_gem_request, fence) != 0);
-	GEM_BUG_ON(fence && !fence_is_i915(fence));
+	GEM_BUG_ON(fence && !dma_fence_is_i915(fence));
 	return container_of(fence, struct drm_i915_gem_request, fence);
 }
 
 static inline struct drm_i915_gem_request *
 i915_gem_request_get(struct drm_i915_gem_request *req)
 {
-	return to_request(fence_get(&req->fence));
+	return to_request(dma_fence_get(&req->fence));
 }
 
 static inline struct drm_i915_gem_request *
 i915_gem_request_get_rcu(struct drm_i915_gem_request *req)
 {
-	return to_request(fence_get_rcu(&req->fence));
+	return to_request(dma_fence_get_rcu(&req->fence));
 }
 
 static inline void
 i915_gem_request_put(struct drm_i915_gem_request *req)
 {
-	fence_put(&req->fence);
+	dma_fence_put(&req->fence);
 }
 
 static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst,
@@ -214,6 +258,8 @@ int
 i915_gem_request_await_object(struct drm_i915_gem_request *to,
 			      struct drm_i915_gem_object *obj,
 			      bool write);
+int i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
+				     struct dma_fence *fence);
 
 void __i915_add_request(struct drm_i915_gem_request *req, bool flush_caches);
 #define i915_add_request(req) \
@@ -221,18 +267,21 @@ void __i915_add_request(struct drm_i915_gem_request *req, bool flush_caches);
 #define i915_add_request_no_flush(req) \
 	__i915_add_request(req, false)
 
+void __i915_gem_request_submit(struct drm_i915_gem_request *request);
+void i915_gem_request_submit(struct drm_i915_gem_request *request);
+
 struct intel_rps_client;
 #define NO_WAITBOOST ERR_PTR(-1)
 #define IS_RPS_CLIENT(p) (!IS_ERR(p))
 #define IS_RPS_USER(p) (!IS_ERR_OR_NULL(p))
 
-int i915_wait_request(struct drm_i915_gem_request *req,
-		      unsigned int flags,
-		      s64 *timeout,
-		      struct intel_rps_client *rps)
+long i915_wait_request(struct drm_i915_gem_request *req,
+		       unsigned int flags,
+		       long timeout)
 	__attribute__((nonnull(1)));
 #define I915_WAIT_INTERRUPTIBLE	BIT(0)
 #define I915_WAIT_LOCKED	BIT(1) /* struct_mutex held, handle GPU reset */
+#define I915_WAIT_ALL		BIT(2) /* used by i915_gem_object_wait() */
 
 static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine);
 
@@ -245,17 +294,37 @@ static inline bool i915_seqno_passed(u32 seq1, u32 seq2)
 }
 
 static inline bool
-i915_gem_request_started(const struct drm_i915_gem_request *req)
+__i915_gem_request_started(const struct drm_i915_gem_request *req)
 {
+	GEM_BUG_ON(!req->global_seqno);
 	return i915_seqno_passed(intel_engine_get_seqno(req->engine),
 				 req->previous_seqno);
 }
 
 static inline bool
-i915_gem_request_completed(const struct drm_i915_gem_request *req)
+i915_gem_request_started(const struct drm_i915_gem_request *req)
 {
+	if (!req->global_seqno)
+		return false;
+
+	return __i915_gem_request_started(req);
+}
+
+static inline bool
+__i915_gem_request_completed(const struct drm_i915_gem_request *req)
+{
+	GEM_BUG_ON(!req->global_seqno);
 	return i915_seqno_passed(intel_engine_get_seqno(req->engine),
-				 req->fence.seqno);
+				 req->global_seqno);
+}
+
+static inline bool
+i915_gem_request_completed(const struct drm_i915_gem_request *req)
+{
+	if (!req->global_seqno)
+		return false;
+
+	return __i915_gem_request_completed(req);
 }
 
 bool __i915_spin_request(const struct drm_i915_gem_request *request,
@@ -263,7 +332,7 @@ bool __i915_spin_request(const struct drm_i915_gem_request *request,
 static inline bool i915_spin_request(const struct drm_i915_gem_request *request,
 				     int state, unsigned long timeout_us)
 {
-	return (i915_gem_request_started(request) &&
+	return (__i915_gem_request_started(request) &&
 		__i915_spin_request(request, state, timeout_us));
 }
 
@@ -497,7 +566,7 @@ __i915_gem_active_get_rcu(const struct i915_gem_active *active)
 		 * compiler.
 		 *
 		 * The atomic operation at the heart of
-		 * i915_gem_request_get_rcu(), see fence_get_rcu(), is
+		 * i915_gem_request_get_rcu(), see dma_fence_get_rcu(), is
 		 * atomic_inc_not_zero() which is only a full memory barrier
 		 * when successful. That is, if i915_gem_request_get_rcu()
 		 * returns the request (and so with the reference counted
@@ -552,53 +621,13 @@ i915_gem_active_isset(const struct i915_gem_active *active)
 }
 
 /**
- * i915_gem_active_is_idle - report whether the active tracker is idle
- * @active - the active tracker
- *
- * i915_gem_active_is_idle() returns true if the active tracker is currently
- * unassigned or if the request is complete (but not yet retired). Requires
- * the caller to hold struct_mutex (but that can be relaxed if desired).
- */
-static inline bool
-i915_gem_active_is_idle(const struct i915_gem_active *active,
-			struct mutex *mutex)
-{
-	return !i915_gem_active_peek(active, mutex);
-}
-
-/**
  * i915_gem_active_wait - waits until the request is completed
  * @active - the active request on which to wait
- *
- * i915_gem_active_wait() waits until the request is completed before
- * returning. Note that it does not guarantee that the request is
- * retired first, see i915_gem_active_retire().
- *
- * i915_gem_active_wait() returns immediately if the active
- * request is already complete.
- */
-static inline int __must_check
-i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex)
-{
-	struct drm_i915_gem_request *request;
-
-	request = i915_gem_active_peek(active, mutex);
-	if (!request)
-		return 0;
-
-	return i915_wait_request(request,
-				 I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
-				 NULL, NULL);
-}
-
-/**
- * i915_gem_active_wait_unlocked - waits until the request is completed
- * @active - the active request on which to wait
  * @flags - how to wait
  * @timeout - how long to wait at most
  * @rps - userspace client to charge for a waitboost
  *
- * i915_gem_active_wait_unlocked() waits until the request is completed before
+ * i915_gem_active_wait() waits until the request is completed before
  * returning, without requiring any locks to be held. Note that it does not
  * retire any requests before returning.
  *
@@ -614,21 +643,18 @@ i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex)
  * Returns 0 if successful, or a negative error code.
  */
 static inline int
-i915_gem_active_wait_unlocked(const struct i915_gem_active *active,
-			      unsigned int flags,
-			      s64 *timeout,
-			      struct intel_rps_client *rps)
+i915_gem_active_wait(const struct i915_gem_active *active, unsigned int flags)
 {
 	struct drm_i915_gem_request *request;
-	int ret = 0;
+	long ret = 0;
 
 	request = i915_gem_active_get_unlocked(active);
 	if (request) {
-		ret = i915_wait_request(request, flags, timeout, rps);
+		ret = i915_wait_request(request, flags, MAX_SCHEDULE_TIMEOUT);
 		i915_gem_request_put(request);
 	}
 
-	return ret;
+	return ret < 0 ? ret : 0;
 }
 
 /**
@@ -645,7 +671,7 @@ i915_gem_active_retire(struct i915_gem_active *active,
 		       struct mutex *mutex)
 {
 	struct drm_i915_gem_request *request;
-	int ret;
+	long ret;
 
 	request = i915_gem_active_raw(active, mutex);
 	if (!request)
@@ -653,8 +679,8 @@ i915_gem_active_retire(struct i915_gem_active *active,
 
 	ret = i915_wait_request(request,
 				I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
-				NULL, NULL);
-	if (ret)
+				MAX_SCHEDULE_TIMEOUT);
+	if (ret < 0)
 		return ret;
 
 	list_del_init(&active->link);
@@ -665,24 +691,6 @@ i915_gem_active_retire(struct i915_gem_active *active,
 	return 0;
 }
 
-/* Convenience functions for peeking at state inside active's request whilst
- * guarded by the struct_mutex.
- */
-
-static inline uint32_t
-i915_gem_active_get_seqno(const struct i915_gem_active *active,
-			  struct mutex *mutex)
-{
-	return i915_gem_request_get_seqno(i915_gem_active_peek(active, mutex));
-}
-
-static inline struct intel_engine_cs *
-i915_gem_active_get_engine(const struct i915_gem_active *active,
-			   struct mutex *mutex)
-{
-	return i915_gem_request_get_engine(i915_gem_active_peek(active, mutex));
-}
-
 #define for_each_active(mask, idx) \
 	for (; mask ? idx = ffs(mask) - 1, 1 : 0; mask &= ~BIT(idx))
 
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index c450076d2f9b..401006b4c6a3 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -35,6 +35,24 @@
 #include "i915_drv.h"
 #include "i915_trace.h"
 
+static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
+{
+	switch (mutex_trylock_recursive(&dev->struct_mutex)) {
+	case MUTEX_TRYLOCK_FAILED:
+		return false;
+
+	case MUTEX_TRYLOCK_SUCCESS:
+		*unlock = true;
+		return true;
+
+	case MUTEX_TRYLOCK_RECURSIVE:
+		*unlock = false;
+		return true;
+	}
+
+	BUG();
+}
+
 static bool any_vma_pinned(struct drm_i915_gem_object *obj)
 {
 	struct i915_vma *vma;
@@ -53,8 +71,11 @@ static bool swap_available(void)
 
 static bool can_release_pages(struct drm_i915_gem_object *obj)
 {
-	/* Only shmemfs objects are backed by swap */
-	if (!obj->base.filp)
+	if (!obj->mm.pages)
+		return false;
+
+	/* Consider only shrinkable ojects. */
+	if (!i915_gem_object_is_shrinkable(obj))
 		return false;
 
 	/* Only report true if by unbinding the object and putting its pages
@@ -65,7 +86,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
 	 * to the GPU, simply unbinding from the GPU is not going to succeed
 	 * in releasing our pin count on the pages themselves.
 	 */
-	if (obj->pages_pin_count > obj->bind_count)
+	if (atomic_read(&obj->mm.pages_pin_count) > obj->bind_count)
 		return false;
 
 	if (any_vma_pinned(obj))
@@ -75,7 +96,14 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
 	 * discard the contents (because the user has marked them as being
 	 * purgeable) or if we can move their contents out to swap.
 	 */
-	return swap_available() || obj->madv == I915_MADV_DONTNEED;
+	return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
+}
+
+static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
+{
+	if (i915_gem_object_unbind(obj) == 0)
+		__i915_gem_object_put_pages(obj, I915_MM_SHRINKER);
+	return !READ_ONCE(obj->mm.pages);
 }
 
 /**
@@ -115,6 +143,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 		{ NULL, 0 },
 	}, *phase;
 	unsigned long count = 0;
+	bool unlock;
+
+	if (!i915_gem_shrinker_lock(&dev_priv->drm, &unlock))
+		return 0;
 
 	trace_i915_gem_shrink(dev_priv, target, flags);
 	i915_gem_retire_requests(dev_priv);
@@ -158,40 +190,51 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 		while (count < target &&
 		       (obj = list_first_entry_or_null(phase->list,
 						       typeof(*obj),
-						       global_list))) {
-			list_move_tail(&obj->global_list, &still_in_list);
+						       global_link))) {
+			list_move_tail(&obj->global_link, &still_in_list);
+			if (!obj->mm.pages) {
+				list_del_init(&obj->global_link);
+				continue;
+			}
 
 			if (flags & I915_SHRINK_PURGEABLE &&
-			    obj->madv != I915_MADV_DONTNEED)
+			    obj->mm.madv != I915_MADV_DONTNEED)
 				continue;
 
 			if (flags & I915_SHRINK_VMAPS &&
-			    !is_vmalloc_addr(obj->mapping))
+			    !is_vmalloc_addr(obj->mm.mapping))
 				continue;
 
-			if ((flags & I915_SHRINK_ACTIVE) == 0 &&
-			    i915_gem_object_is_active(obj))
+			if (!(flags & I915_SHRINK_ACTIVE) &&
+			    (i915_gem_object_is_active(obj) ||
+			     obj->framebuffer_references))
 				continue;
 
 			if (!can_release_pages(obj))
 				continue;
 
-			i915_gem_object_get(obj);
-
-			/* For the unbound phase, this should be a no-op! */
-			i915_gem_object_unbind(obj);
-			if (i915_gem_object_put_pages(obj) == 0)
-				count += obj->base.size >> PAGE_SHIFT;
-
-			i915_gem_object_put(obj);
+			if (unsafe_drop_pages(obj)) {
+				/* May arrive from get_pages on another bo */
+				mutex_lock_nested(&obj->mm.lock,
+						  I915_MM_SHRINKER);
+				if (!obj->mm.pages) {
+					__i915_gem_object_invalidate(obj);
+					list_del_init(&obj->global_link);
+					count += obj->base.size >> PAGE_SHIFT;
+				}
+				mutex_unlock(&obj->mm.lock);
+			}
 		}
-		list_splice(&still_in_list, phase->list);
+		list_splice_tail(&still_in_list, phase->list);
 	}
 
 	if (flags & I915_SHRINK_BOUND)
 		intel_runtime_pm_put(dev_priv);
 
 	i915_gem_retire_requests(dev_priv);
+	if (unlock)
+		mutex_unlock(&dev_priv->drm.struct_mutex);
+
 	/* expedite the RCU grace period to free some request slabs */
 	synchronize_rcu_expedited();
 
@@ -225,24 +268,6 @@ unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
 	return freed;
 }
 
-static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
-{
-	switch (mutex_trylock_recursive(&dev->struct_mutex)) {
-	case MUTEX_TRYLOCK_FAILED:
-		return false;
-
-	case MUTEX_TRYLOCK_SUCCESS:
-		*unlock = true;
-		return true;
-
-	case MUTEX_TRYLOCK_RECURSIVE:
-		*unlock = false;
-		return true;
-	}
-
-	BUG();
-}
-
 static unsigned long
 i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
 {
@@ -259,11 +284,11 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
 	i915_gem_retire_requests(dev_priv);
 
 	count = 0;
-	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
+	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link)
 		if (can_release_pages(obj))
 			count += obj->base.size >> PAGE_SHIFT;
 
-	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
+	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
 		if (!i915_gem_object_is_active(obj) && can_release_pages(obj))
 			count += obj->base.size >> PAGE_SHIFT;
 	}
@@ -364,13 +389,19 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
 	 * being pointed to by hardware.
 	 */
 	unbound = bound = unevictable = 0;
-	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
+	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) {
+		if (!obj->mm.pages)
+			continue;
+
 		if (!can_release_pages(obj))
 			unevictable += obj->base.size >> PAGE_SHIFT;
 		else
 			unbound += obj->base.size >> PAGE_SHIFT;
 	}
-	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
+	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) {
+		if (!obj->mm.pages)
+			continue;
+
 		if (!can_release_pages(obj))
 			unevictable += obj->base.size >> PAGE_SHIFT;
 		else
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 59989e8ee5dc..ebaa941c83af 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -89,9 +89,8 @@ void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
 	mutex_unlock(&dev_priv->mm.stolen_lock);
 }
 
-static unsigned long i915_stolen_to_physical(struct drm_device *dev)
+static unsigned long i915_stolen_to_physical(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct pci_dev *pdev = dev_priv->drm.pdev;
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	struct resource *r;
@@ -109,13 +108,13 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 	 *
 	 */
 	base = 0;
-	if (INTEL_INFO(dev)->gen >= 3) {
+	if (INTEL_GEN(dev_priv) >= 3) {
 		u32 bsm;
 
 		pci_read_config_dword(pdev, INTEL_BSM, &bsm);
 
 		base = bsm & INTEL_BSM_MASK;
-	} else if (IS_I865G(dev)) {
+	} else if (IS_I865G(dev_priv)) {
 		u32 tseg_size = 0;
 		u16 toud = 0;
 		u8 tmp;
@@ -138,7 +137,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 					 I865_TOUD, &toud);
 
 		base = (toud << 16) + tseg_size;
-	} else if (IS_I85X(dev)) {
+	} else if (IS_I85X(dev_priv)) {
 		u32 tseg_size = 0;
 		u32 tom;
 		u8 tmp;
@@ -154,7 +153,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 		tom = tmp * MB(32);
 
 		base = tom - tseg_size - ggtt->stolen_size;
-	} else if (IS_845G(dev)) {
+	} else if (IS_845G(dev_priv)) {
 		u32 tseg_size = 0;
 		u32 tom;
 		u8 tmp;
@@ -178,7 +177,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 		tom = tmp * MB(32);
 
 		base = tom - tseg_size - ggtt->stolen_size;
-	} else if (IS_I830(dev)) {
+	} else if (IS_I830(dev_priv)) {
 		u32 tseg_size = 0;
 		u32 tom;
 		u8 tmp;
@@ -204,7 +203,8 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 		return 0;
 
 	/* make sure we don't clobber the GTT if it's within stolen memory */
-	if (INTEL_INFO(dev)->gen <= 4 && !IS_G33(dev) && !IS_G4X(dev)) {
+	if (INTEL_GEN(dev_priv) <= 4 && !IS_G33(dev_priv) &&
+	    !IS_G4X(dev_priv)) {
 		struct {
 			u32 start, end;
 		} stolen[2] = {
@@ -214,7 +214,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 		u64 ggtt_start, ggtt_end;
 
 		ggtt_start = I915_READ(PGTBL_CTL);
-		if (IS_GEN4(dev))
+		if (IS_GEN4(dev_priv))
 			ggtt_start = (ggtt_start & PGTBL_ADDRESS_LO_MASK) |
 				     (ggtt_start & PGTBL_ADDRESS_HI_MASK) << 28;
 		else
@@ -252,7 +252,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 	 * kernel. So if the region is already marked as busy, something
 	 * is seriously wrong.
 	 */
-	r = devm_request_mem_region(dev->dev, base, ggtt->stolen_size,
+	r = devm_request_mem_region(dev_priv->drm.dev, base, ggtt->stolen_size,
 				    "Graphics Stolen Memory");
 	if (r == NULL) {
 		/*
@@ -263,14 +263,14 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 		 * PCI bus, but have an off-by-one error. Hence retry the
 		 * reservation starting from 1 instead of 0.
 		 */
-		r = devm_request_mem_region(dev->dev, base + 1,
+		r = devm_request_mem_region(dev_priv->drm.dev, base + 1,
 					    ggtt->stolen_size - 1,
 					    "Graphics Stolen Memory");
 		/*
 		 * GEN3 firmware likes to smash pci bridges into the stolen
 		 * range. Apparently this works.
 		 */
-		if (r == NULL && !IS_GEN3(dev)) {
+		if (r == NULL && !IS_GEN3(dev_priv)) {
 			DRM_ERROR("conflict detected with stolen region: [0x%08x - 0x%08x]\n",
 				  base, base + (uint32_t)ggtt->stolen_size);
 			base = 0;
@@ -407,9 +407,8 @@ static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv,
 		*size = stolen_top - *base;
 }
 
-int i915_gem_init_stolen(struct drm_device *dev)
+int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	unsigned long reserved_total, reserved_base = 0, reserved_size;
 	unsigned long stolen_top;
@@ -417,7 +416,7 @@ int i915_gem_init_stolen(struct drm_device *dev)
 	mutex_init(&dev_priv->mm.stolen_lock);
 
 #ifdef CONFIG_INTEL_IOMMU
-	if (intel_iommu_gfx_mapped && INTEL_INFO(dev)->gen < 8) {
+	if (intel_iommu_gfx_mapped && INTEL_GEN(dev_priv) < 8) {
 		DRM_INFO("DMAR active, disabling use of stolen memory\n");
 		return 0;
 	}
@@ -426,7 +425,7 @@ int i915_gem_init_stolen(struct drm_device *dev)
 	if (ggtt->stolen_size == 0)
 		return 0;
 
-	dev_priv->mm.stolen_base = i915_stolen_to_physical(dev);
+	dev_priv->mm.stolen_base = i915_stolen_to_physical(dev_priv);
 	if (dev_priv->mm.stolen_base == 0)
 		return 0;
 
@@ -437,7 +436,7 @@ int i915_gem_init_stolen(struct drm_device *dev)
 	case 3:
 		break;
 	case 4:
-		if (IS_G4X(dev))
+		if (IS_G4X(dev_priv))
 			g4x_get_stolen_reserved(dev_priv, &reserved_base,
 						&reserved_size);
 		break;
@@ -456,7 +455,7 @@ int i915_gem_init_stolen(struct drm_device *dev)
 		break;
 	default:
 		if (IS_BROADWELL(dev_priv) ||
-		    IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev))
+		    IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
 			bdw_get_stolen_reserved(dev_priv, &reserved_base,
 						&reserved_size);
 		else
@@ -514,12 +513,10 @@ i915_pages_create_for_stolen(struct drm_device *dev,
 			     u32 offset, u32 size)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	struct sg_table *st;
 	struct scatterlist *sg;
 
-	DRM_DEBUG_DRIVER("offset=0x%x, size=%d\n", offset, size);
-	BUG_ON(offset > ggtt->stolen_size - size);
+	GEM_BUG_ON(offset > dev_priv->ggtt.stolen_size - size);
 
 	/* We hide that we have no struct page backing our stolen object
 	 * by wrapping the contiguous physical allocation with a fake
@@ -528,11 +525,11 @@ i915_pages_create_for_stolen(struct drm_device *dev,
 
 	st = kmalloc(sizeof(*st), GFP_KERNEL);
 	if (st == NULL)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 		kfree(st);
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 	}
 
 	sg = st->sgl;
@@ -545,31 +542,36 @@ i915_pages_create_for_stolen(struct drm_device *dev,
 	return st;
 }
 
-static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
+static struct sg_table *
+i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
 {
-	BUG();
-	return -EINVAL;
+	return i915_pages_create_for_stolen(obj->base.dev,
+					    obj->stolen->start,
+					    obj->stolen->size);
 }
 
-static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj)
+static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj,
+					     struct sg_table *pages)
 {
-	/* Should only be called during free */
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	/* Should only be called from i915_gem_object_release_stolen() */
+	sg_free_table(pages);
+	kfree(pages);
 }
 
-
 static void
 i915_gem_object_release_stolen(struct drm_i915_gem_object *obj)
 {
 	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+	struct drm_mm_node *stolen = fetch_and_zero(&obj->stolen);
 
-	if (obj->stolen) {
-		i915_gem_stolen_remove_node(dev_priv, obj->stolen);
-		kfree(obj->stolen);
-		obj->stolen = NULL;
-	}
+	GEM_BUG_ON(!stolen);
+
+	__i915_gem_object_unpin_pages(obj);
+
+	i915_gem_stolen_remove_node(dev_priv, stolen);
+	kfree(stolen);
 }
+
 static const struct drm_i915_gem_object_ops i915_gem_object_stolen_ops = {
 	.get_pages = i915_gem_object_get_pages_stolen,
 	.put_pages = i915_gem_object_put_pages_stolen,
@@ -589,19 +591,13 @@ _i915_gem_object_create_stolen(struct drm_device *dev,
 	drm_gem_private_object_init(dev, &obj->base, stolen->size);
 	i915_gem_object_init(obj, &i915_gem_object_stolen_ops);
 
-	obj->pages = i915_pages_create_for_stolen(dev,
-						  stolen->start, stolen->size);
-	if (obj->pages == NULL)
-		goto cleanup;
-
-	obj->get_page.sg = obj->pages->sgl;
-	obj->get_page.last = 0;
-
-	i915_gem_object_pin_pages(obj);
 	obj->stolen = stolen;
-
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
-	obj->cache_level = HAS_LLC(dev) ? I915_CACHE_LLC : I915_CACHE_NONE;
+	obj->cache_level = HAS_LLC(to_i915(dev)) ?
+			   I915_CACHE_LLC : I915_CACHE_NONE;
+
+	if (i915_gem_object_pin_pages(obj))
+		goto cleanup;
 
 	return obj;
 
@@ -621,7 +617,6 @@ i915_gem_object_create_stolen(struct drm_device *dev, u32 size)
 	if (!drm_mm_initialized(&dev_priv->mm.stolen))
 		return NULL;
 
-	DRM_DEBUG_KMS("creating stolen object: size=%x\n", size);
 	if (size == 0)
 		return NULL;
 
@@ -697,10 +692,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
 	if (gtt_offset == I915_GTT_OFFSET_NONE)
 		return obj;
 
+	ret = i915_gem_object_pin_pages(obj);
+	if (ret)
+		goto err;
+
 	vma = i915_gem_obj_lookup_or_create_vma(obj, &ggtt->base, NULL);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
-		goto err;
+		goto err_pages;
 	}
 
 	/* To simplify the initialisation sequence between KMS and GTT,
@@ -714,20 +713,20 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
 	ret = drm_mm_reserve_node(&ggtt->base.mm, &vma->node);
 	if (ret) {
 		DRM_DEBUG_KMS("failed to allocate stolen GTT space\n");
-		goto err;
+		goto err_pages;
 	}
 
-	vma->pages = obj->pages;
+	vma->pages = obj->mm.pages;
 	vma->flags |= I915_VMA_GLOBAL_BIND;
 	__i915_vma_set_map_and_fenceable(vma);
 	list_move_tail(&vma->vm_link, &ggtt->base.inactive_list);
+	list_move_tail(&obj->global_link, &dev_priv->mm.bound_list);
 	obj->bind_count++;
 
-	list_add_tail(&obj->global_list, &dev_priv->mm.bound_list);
-	i915_gem_object_pin_pages(obj);
-
 	return obj;
 
+err_pages:
+	i915_gem_object_unpin_pages(obj);
 err:
 	i915_gem_object_put(obj);
 	return NULL;
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index a14b1e3d4c78..c85e7b06bdba 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -60,7 +60,8 @@
 
 /* Check pitch constriants for all chips & tiling formats */
 static bool
-i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
+i915_tiling_ok(struct drm_i915_private *dev_priv,
+	       int stride, int size, int tiling_mode)
 {
 	int tile_width;
 
@@ -71,8 +72,8 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
 	if (tiling_mode > I915_TILING_LAST)
 		return false;
 
-	if (IS_GEN2(dev) ||
-	    (tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
+	if (IS_GEN2(dev_priv) ||
+	    (tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev_priv)))
 		tile_width = 128;
 	else
 		tile_width = 512;
@@ -80,17 +81,17 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
 	/* check maximum stride & object size */
 	/* i965+ stores the end address of the gtt mapping in the fence
 	 * reg, so dont bother to check the size */
-	if (INTEL_INFO(dev)->gen >= 7) {
+	if (INTEL_GEN(dev_priv) >= 7) {
 		if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL)
 			return false;
-	} else if (INTEL_INFO(dev)->gen >= 4) {
+	} else if (INTEL_GEN(dev_priv) >= 4) {
 		if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
 			return false;
 	} else {
 		if (stride > 8192)
 			return false;
 
-		if (IS_GEN3(dev)) {
+		if (IS_GEN3(dev_priv)) {
 			if (size > I830_FENCE_MAX_SIZE_VAL << 20)
 				return false;
 		} else {
@@ -103,7 +104,7 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
 		return false;
 
 	/* 965+ just needs multiples of tile width */
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_GEN(dev_priv) >= 4) {
 		if (stride & (tile_width - 1))
 			return false;
 		return true;
@@ -198,14 +199,12 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 	if (!obj)
 		return -ENOENT;
 
-	if (!i915_tiling_ok(dev,
+	if (!i915_tiling_ok(dev_priv,
 			    args->stride, obj->base.size, args->tiling_mode)) {
-		i915_gem_object_put_unlocked(obj);
+		i915_gem_object_put(obj);
 		return -EINVAL;
 	}
 
-	intel_runtime_pm_get(dev_priv);
-
 	mutex_lock(&dev->struct_mutex);
 	if (obj->pin_display || obj->framebuffer_references) {
 		err = -EBUSY;
@@ -260,14 +259,22 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 		if (!err) {
 			struct i915_vma *vma;
 
-			if (obj->pages &&
-			    obj->madv == I915_MADV_WILLNEED &&
+			mutex_lock(&obj->mm.lock);
+			if (obj->mm.pages &&
+			    obj->mm.madv == I915_MADV_WILLNEED &&
 			    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
-				if (args->tiling_mode == I915_TILING_NONE)
-					i915_gem_object_unpin_pages(obj);
-				if (!i915_gem_object_is_tiled(obj))
-					i915_gem_object_pin_pages(obj);
+				if (args->tiling_mode == I915_TILING_NONE) {
+					GEM_BUG_ON(!obj->mm.quirked);
+					__i915_gem_object_unpin_pages(obj);
+					obj->mm.quirked = false;
+				}
+				if (!i915_gem_object_is_tiled(obj)) {
+					GEM_BUG_ON(!obj->mm.quirked);
+					__i915_gem_object_pin_pages(obj);
+					obj->mm.quirked = true;
+				}
 			}
+			mutex_unlock(&obj->mm.lock);
 
 			list_for_each_entry(vma, &obj->vma_list, obj_link) {
 				if (!vma->fence)
@@ -301,8 +308,6 @@ err:
 	i915_gem_object_put(obj);
 	mutex_unlock(&dev->struct_mutex);
 
-	intel_runtime_pm_put(dev_priv);
-
 	return err;
 }
 
@@ -326,12 +331,19 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
 	struct drm_i915_gem_get_tiling *args = data;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_object *obj;
+	int err = -ENOENT;
+
+	rcu_read_lock();
+	obj = i915_gem_object_lookup_rcu(file, args->handle);
+	if (obj) {
+		args->tiling_mode =
+			READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
+		err = 0;
+	}
+	rcu_read_unlock();
+	if (unlikely(err))
+		return err;
 
-	obj = i915_gem_object_lookup(file, args->handle);
-	if (!obj)
-		return -ENOENT;
-
-	args->tiling_mode = READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
 	switch (args->tiling_mode) {
 	case I915_TILING_X:
 		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
@@ -339,11 +351,10 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
 	case I915_TILING_Y:
 		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
 		break;
+	default:
 	case I915_TILING_NONE:
 		args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
 		break;
-	default:
-		DRM_ERROR("unknown tiling mode\n");
 	}
 
 	/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
@@ -356,6 +367,5 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
 	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
 		args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
 
-	i915_gem_object_put_unlocked(obj);
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.c b/drivers/gpu/drm/i915/i915_gem_timeline.c
new file mode 100644
index 000000000000..bf8a471b61e6
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_timeline.c
@@ -0,0 +1,90 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "i915_drv.h"
+
+static int __i915_gem_timeline_init(struct drm_i915_private *i915,
+				    struct i915_gem_timeline *timeline,
+				    const char *name,
+				    struct lock_class_key *lockclass,
+				    const char *lockname)
+{
+	unsigned int i;
+	u64 fences;
+
+	lockdep_assert_held(&i915->drm.struct_mutex);
+
+	timeline->i915 = i915;
+	timeline->name = kstrdup(name ?: "[kernel]", GFP_KERNEL);
+	if (!timeline->name)
+		return -ENOMEM;
+
+	list_add(&timeline->link, &i915->gt.timelines);
+
+	/* Called during early_init before we know how many engines there are */
+	fences = dma_fence_context_alloc(ARRAY_SIZE(timeline->engine));
+	for (i = 0; i < ARRAY_SIZE(timeline->engine); i++) {
+		struct intel_timeline *tl = &timeline->engine[i];
+
+		tl->fence_context = fences++;
+		tl->common = timeline;
+#ifdef CONFIG_DEBUG_SPINLOCK
+		__raw_spin_lock_init(&tl->lock.rlock, lockname, lockclass);
+#else
+		spin_lock_init(&tl->lock);
+#endif
+		init_request_active(&tl->last_request, NULL);
+		INIT_LIST_HEAD(&tl->requests);
+	}
+
+	return 0;
+}
+
+int i915_gem_timeline_init(struct drm_i915_private *i915,
+			   struct i915_gem_timeline *timeline,
+			   const char *name)
+{
+	static struct lock_class_key class;
+
+	return __i915_gem_timeline_init(i915, timeline, name,
+					&class, "&timeline->lock");
+}
+
+int i915_gem_timeline_init__global(struct drm_i915_private *i915)
+{
+	static struct lock_class_key class;
+
+	return __i915_gem_timeline_init(i915,
+					&i915->gt.global_timeline,
+					"[execution]",
+					&class, "&global_timeline->lock");
+}
+
+void i915_gem_timeline_fini(struct i915_gem_timeline *tl)
+{
+	lockdep_assert_held(&tl->i915->drm.struct_mutex);
+
+	list_del(&tl->link);
+	kfree(tl->name);
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.h b/drivers/gpu/drm/i915/i915_gem_timeline.h
new file mode 100644
index 000000000000..98d99a62b4ae
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_timeline.h
@@ -0,0 +1,73 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef I915_GEM_TIMELINE_H
+#define I915_GEM_TIMELINE_H
+
+#include <linux/list.h>
+
+#include "i915_gem_request.h"
+
+struct i915_gem_timeline;
+
+struct intel_timeline {
+	u64 fence_context;
+	u32 last_submitted_seqno;
+
+	spinlock_t lock;
+
+	/**
+	 * List of breadcrumbs associated with GPU requests currently
+	 * outstanding.
+	 */
+	struct list_head requests;
+
+	/* Contains an RCU guarded pointer to the last request. No reference is
+	 * held to the request, users must carefully acquire a reference to
+	 * the request using i915_gem_active_get_request_rcu(), or hold the
+	 * struct_mutex.
+	 */
+	struct i915_gem_active last_request;
+	u32 sync_seqno[I915_NUM_ENGINES];
+
+	struct i915_gem_timeline *common;
+};
+
+struct i915_gem_timeline {
+	struct list_head link;
+	atomic_t next_seqno;
+
+	struct drm_i915_private *i915;
+	const char *name;
+
+	struct intel_timeline engine[I915_NUM_ENGINES];
+};
+
+int i915_gem_timeline_init(struct drm_i915_private *i915,
+			   struct i915_gem_timeline *tl,
+			   const char *name);
+int i915_gem_timeline_init__global(struct drm_i915_private *i915);
+void i915_gem_timeline_fini(struct i915_gem_timeline *tl);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index c6f780f5abc9..d068af2ec3a3 100644
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -61,33 +61,26 @@ struct i915_mmu_object {
 	bool attached;
 };
 
-static void wait_rendering(struct drm_i915_gem_object *obj)
-{
-	unsigned long active = __I915_BO_ACTIVE(obj);
-	int idx;
-
-	for_each_active(active, idx)
-		i915_gem_active_wait_unlocked(&obj->last_read[idx],
-					      0, NULL, NULL);
-}
-
 static void cancel_userptr(struct work_struct *work)
 {
 	struct i915_mmu_object *mo = container_of(work, typeof(*mo), work);
 	struct drm_i915_gem_object *obj = mo->obj;
 	struct drm_device *dev = obj->base.dev;
 
-	wait_rendering(obj);
+	i915_gem_object_wait(obj, I915_WAIT_ALL, MAX_SCHEDULE_TIMEOUT, NULL);
 
 	mutex_lock(&dev->struct_mutex);
 	/* Cancel any active worker and force us to re-evaluate gup */
 	obj->userptr.work = NULL;
 
-	if (obj->pages != NULL) {
-		/* We are inside a kthread context and can't be interrupted */
-		WARN_ON(i915_gem_object_unbind(obj));
-		WARN_ON(i915_gem_object_put_pages(obj));
-	}
+	/* We are inside a kthread context and can't be interrupted */
+	if (i915_gem_object_unbind(obj) == 0)
+		__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
+	WARN_ONCE(obj->mm.pages,
+		  "Failed to release pages: bind_count=%d, pages_pin_count=%d, pin_display=%d\n",
+		  obj->bind_count,
+		  atomic_read(&obj->mm.pages_pin_count),
+		  obj->pin_display);
 
 	i915_gem_object_put(obj);
 	mutex_unlock(&dev->struct_mutex);
@@ -436,24 +429,25 @@ err:
 	return ret;
 }
 
-static int
+static struct sg_table *
 __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj,
 			     struct page **pvec, int num_pages)
 {
+	struct sg_table *pages;
 	int ret;
 
-	ret = st_set_pages(&obj->pages, pvec, num_pages);
+	ret = st_set_pages(&pages, pvec, num_pages);
 	if (ret)
-		return ret;
+		return ERR_PTR(ret);
 
-	ret = i915_gem_gtt_prepare_object(obj);
+	ret = i915_gem_gtt_prepare_pages(obj, pages);
 	if (ret) {
-		sg_free_table(obj->pages);
-		kfree(obj->pages);
-		obj->pages = NULL;
+		sg_free_table(pages);
+		kfree(pages);
+		return ERR_PTR(ret);
 	}
 
-	return ret;
+	return pages;
 }
 
 static int
@@ -497,7 +491,6 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 {
 	struct get_pages_work *work = container_of(_work, typeof(*work), work);
 	struct drm_i915_gem_object *obj = work->obj;
-	struct drm_device *dev = obj->base.dev;
 	const int npages = obj->base.size >> PAGE_SHIFT;
 	struct page **pvec;
 	int pinned, ret;
@@ -522,7 +515,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 					 obj->userptr.ptr + pinned * PAGE_SIZE,
 					 npages - pinned,
 					 flags,
-					 pvec + pinned, NULL);
+					 pvec + pinned, NULL, NULL);
 				if (ret < 0)
 					break;
 
@@ -533,33 +526,32 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 		}
 	}
 
-	mutex_lock(&dev->struct_mutex);
+	mutex_lock(&obj->mm.lock);
 	if (obj->userptr.work == &work->work) {
+		struct sg_table *pages = ERR_PTR(ret);
+
 		if (pinned == npages) {
-			ret = __i915_gem_userptr_set_pages(obj, pvec, npages);
-			if (ret == 0) {
-				list_add_tail(&obj->global_list,
-					      &to_i915(dev)->mm.unbound_list);
-				obj->get_page.sg = obj->pages->sgl;
-				obj->get_page.last = 0;
+			pages = __i915_gem_userptr_set_pages(obj, pvec, npages);
+			if (!IS_ERR(pages)) {
+				__i915_gem_object_set_pages(obj, pages);
 				pinned = 0;
+				pages = NULL;
 			}
 		}
-		obj->userptr.work = ERR_PTR(ret);
-	}
 
-	obj->userptr.workers--;
-	i915_gem_object_put(obj);
-	mutex_unlock(&dev->struct_mutex);
+		obj->userptr.work = ERR_CAST(pages);
+	}
+	mutex_unlock(&obj->mm.lock);
 
 	release_pages(pvec, pinned, 0);
 	drm_free_large(pvec);
 
+	i915_gem_object_put(obj);
 	put_task_struct(work->task);
 	kfree(work);
 }
 
-static int
+static struct sg_table *
 __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj,
 				      bool *active)
 {
@@ -584,15 +576,11 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj,
 	 * that error back to this function through
 	 * obj->userptr.work = ERR_PTR.
 	 */
-	if (obj->userptr.workers >= I915_GEM_USERPTR_MAX_WORKERS)
-		return -EAGAIN;
-
 	work = kmalloc(sizeof(*work), GFP_KERNEL);
 	if (work == NULL)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	obj->userptr.work = &work->work;
-	obj->userptr.workers++;
 
 	work->obj = i915_gem_object_get(obj);
 
@@ -603,14 +591,15 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj,
 	schedule_work(&work->work);
 
 	*active = true;
-	return -EAGAIN;
+	return ERR_PTR(-EAGAIN);
 }
 
-static int
+static struct sg_table *
 i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
 {
 	const int num_pages = obj->base.size >> PAGE_SHIFT;
 	struct page **pvec;
+	struct sg_table *pages;
 	int pinned, ret;
 	bool active;
 
@@ -634,15 +623,15 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
 	if (obj->userptr.work) {
 		/* active flag should still be held for the pending work */
 		if (IS_ERR(obj->userptr.work))
-			return PTR_ERR(obj->userptr.work);
+			return ERR_CAST(obj->userptr.work);
 		else
-			return -EAGAIN;
+			return ERR_PTR(-EAGAIN);
 	}
 
 	/* Let the mmu-notifier know that we have begun and need cancellation */
 	ret = __i915_gem_userptr_set_active(obj, true);
 	if (ret)
-		return ret;
+		return ERR_PTR(ret);
 
 	pvec = NULL;
 	pinned = 0;
@@ -651,7 +640,7 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
 				      GFP_TEMPORARY);
 		if (pvec == NULL) {
 			__i915_gem_userptr_set_active(obj, false);
-			return -ENOMEM;
+			return ERR_PTR(-ENOMEM);
 		}
 
 		pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages,
@@ -660,21 +649,22 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
 
 	active = false;
 	if (pinned < 0)
-		ret = pinned, pinned = 0;
+		pages = ERR_PTR(pinned), pinned = 0;
 	else if (pinned < num_pages)
-		ret = __i915_gem_userptr_get_pages_schedule(obj, &active);
+		pages = __i915_gem_userptr_get_pages_schedule(obj, &active);
 	else
-		ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages);
-	if (ret) {
+		pages = __i915_gem_userptr_set_pages(obj, pvec, num_pages);
+	if (IS_ERR(pages)) {
 		__i915_gem_userptr_set_active(obj, active);
 		release_pages(pvec, pinned, 0);
 	}
 	drm_free_large(pvec);
-	return ret;
+	return pages;
 }
 
 static void
-i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
+i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj,
+			   struct sg_table *pages)
 {
 	struct sgt_iter sgt_iter;
 	struct page *page;
@@ -682,22 +672,22 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
 	BUG_ON(obj->userptr.work != NULL);
 	__i915_gem_userptr_set_active(obj, false);
 
-	if (obj->madv != I915_MADV_WILLNEED)
-		obj->dirty = 0;
+	if (obj->mm.madv != I915_MADV_WILLNEED)
+		obj->mm.dirty = false;
 
-	i915_gem_gtt_finish_object(obj);
+	i915_gem_gtt_finish_pages(obj, pages);
 
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
-		if (obj->dirty)
+	for_each_sgt_page(page, sgt_iter, pages) {
+		if (obj->mm.dirty)
 			set_page_dirty(page);
 
 		mark_page_accessed(page);
 		put_page(page);
 	}
-	obj->dirty = 0;
+	obj->mm.dirty = false;
 
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(pages);
+	kfree(pages);
 }
 
 static void
@@ -717,7 +707,8 @@ i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)
 }
 
 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
-	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
+	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
+		 I915_GEM_OBJECT_IS_SHRINKABLE,
 	.get_pages = i915_gem_userptr_get_pages,
 	.put_pages = i915_gem_userptr_put_pages,
 	.dmabuf_export = i915_gem_userptr_dmabuf_export,
@@ -762,12 +753,13 @@ static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
 int
 i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 {
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_userptr *args = data;
 	struct drm_i915_gem_object *obj;
 	int ret;
 	u32 handle;
 
-	if (!HAS_LLC(dev) && !HAS_SNOOP(dev)) {
+	if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) {
 		/* We cannot support coherent userptr objects on hw without
 		 * LLC and broken snooping.
 		 */
@@ -816,7 +808,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file
 		ret = drm_gem_handle_create(file, &obj->base, &handle);
 
 	/* drop reference from allocate - handle holds it now */
-	i915_gem_object_put_unlocked(obj);
+	i915_gem_object_put(obj);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 334f15df7c8d..ae84aa4b1467 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -28,6 +28,8 @@
  */
 
 #include <generated/utsrelease.h>
+#include <linux/stop_machine.h>
+#include <linux/zlib.h>
 #include "i915_drv.h"
 
 static const char *engine_str(int engine)
@@ -172,6 +174,110 @@ static void i915_error_puts(struct drm_i915_error_state_buf *e,
 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
 #define err_puts(e, s) i915_error_puts(e, s)
 
+#ifdef CONFIG_DRM_I915_COMPRESS_ERROR
+
+static bool compress_init(struct z_stream_s *zstream)
+{
+	memset(zstream, 0, sizeof(*zstream));
+
+	zstream->workspace =
+		kmalloc(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL),
+			GFP_ATOMIC | __GFP_NOWARN);
+	if (!zstream->workspace)
+		return false;
+
+	if (zlib_deflateInit(zstream, Z_DEFAULT_COMPRESSION) != Z_OK) {
+		kfree(zstream->workspace);
+		return false;
+	}
+
+	return true;
+}
+
+static int compress_page(struct z_stream_s *zstream,
+			 void *src,
+			 struct drm_i915_error_object *dst)
+{
+	zstream->next_in = src;
+	zstream->avail_in = PAGE_SIZE;
+
+	do {
+		if (zstream->avail_out == 0) {
+			unsigned long page;
+
+			page = __get_free_page(GFP_ATOMIC | __GFP_NOWARN);
+			if (!page)
+				return -ENOMEM;
+
+			dst->pages[dst->page_count++] = (void *)page;
+
+			zstream->next_out = (void *)page;
+			zstream->avail_out = PAGE_SIZE;
+		}
+
+		if (zlib_deflate(zstream, Z_SYNC_FLUSH) != Z_OK)
+			return -EIO;
+	} while (zstream->avail_in);
+
+	/* Fallback to uncompressed if we increase size? */
+	if (0 && zstream->total_out > zstream->total_in)
+		return -E2BIG;
+
+	return 0;
+}
+
+static void compress_fini(struct z_stream_s *zstream,
+			  struct drm_i915_error_object *dst)
+{
+	if (dst) {
+		zlib_deflate(zstream, Z_FINISH);
+		dst->unused = zstream->avail_out;
+	}
+
+	zlib_deflateEnd(zstream);
+	kfree(zstream->workspace);
+}
+
+static void err_compression_marker(struct drm_i915_error_state_buf *m)
+{
+	err_puts(m, ":");
+}
+
+#else
+
+static bool compress_init(struct z_stream_s *zstream)
+{
+	return true;
+}
+
+static int compress_page(struct z_stream_s *zstream,
+			 void *src,
+			 struct drm_i915_error_object *dst)
+{
+	unsigned long page;
+
+	page = __get_free_page(GFP_ATOMIC | __GFP_NOWARN);
+	if (!page)
+		return -ENOMEM;
+
+	dst->pages[dst->page_count++] =
+		memcpy((void *)page, src, PAGE_SIZE);
+
+	return 0;
+}
+
+static void compress_fini(struct z_stream_s *zstream,
+			  struct drm_i915_error_object *dst)
+{
+}
+
+static void err_compression_marker(struct drm_i915_error_state_buf *m)
+{
+	err_puts(m, "~");
+}
+
+#endif
+
 static void print_error_buffers(struct drm_i915_error_state_buf *m,
 				const char *name,
 				struct drm_i915_error_buffer *err,
@@ -228,13 +334,57 @@ static const char *hangcheck_action_to_str(enum intel_engine_hangcheck_action a)
 	return "unknown";
 }
 
+static void error_print_instdone(struct drm_i915_error_state_buf *m,
+				 struct drm_i915_error_engine *ee)
+{
+	int slice;
+	int subslice;
+
+	err_printf(m, "  INSTDONE: 0x%08x\n",
+		   ee->instdone.instdone);
+
+	if (ee->engine_id != RCS || INTEL_GEN(m->i915) <= 3)
+		return;
+
+	err_printf(m, "  SC_INSTDONE: 0x%08x\n",
+		   ee->instdone.slice_common);
+
+	if (INTEL_GEN(m->i915) <= 6)
+		return;
+
+	for_each_instdone_slice_subslice(m->i915, slice, subslice)
+		err_printf(m, "  SAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
+			   slice, subslice,
+			   ee->instdone.sampler[slice][subslice]);
+
+	for_each_instdone_slice_subslice(m->i915, slice, subslice)
+		err_printf(m, "  ROW_INSTDONE[%d][%d]: 0x%08x\n",
+			   slice, subslice,
+			   ee->instdone.row[slice][subslice]);
+}
+
+static void error_print_request(struct drm_i915_error_state_buf *m,
+				const char *prefix,
+				struct drm_i915_error_request *erq)
+{
+	if (!erq->seqno)
+		return;
+
+	err_printf(m, "%s pid %d, seqno %8x:%08x, emitted %dms ago, head %08x, tail %08x\n",
+		   prefix, erq->pid,
+		   erq->context, erq->seqno,
+		   jiffies_to_msecs(jiffies - erq->jiffies),
+		   erq->head, erq->tail);
+}
+
 static void error_print_engine(struct drm_i915_error_state_buf *m,
 			       struct drm_i915_error_engine *ee)
 {
 	err_printf(m, "%s command stream:\n", engine_str(ee->engine_id));
 	err_printf(m, "  START: 0x%08x\n", ee->start);
-	err_printf(m, "  HEAD:  0x%08x\n", ee->head);
-	err_printf(m, "  TAIL:  0x%08x\n", ee->tail);
+	err_printf(m, "  HEAD:  0x%08x [0x%08x]\n", ee->head, ee->rq_head);
+	err_printf(m, "  TAIL:  0x%08x [0x%08x, 0x%08x]\n",
+		   ee->tail, ee->rq_post, ee->rq_tail);
 	err_printf(m, "  CTL:   0x%08x\n", ee->ctl);
 	err_printf(m, "  MODE:  0x%08x\n", ee->mode);
 	err_printf(m, "  HWS:   0x%08x\n", ee->hws);
@@ -242,7 +392,9 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
 		   (u32)(ee->acthd>>32), (u32)ee->acthd);
 	err_printf(m, "  IPEIR: 0x%08x\n", ee->ipeir);
 	err_printf(m, "  IPEHR: 0x%08x\n", ee->ipehr);
-	err_printf(m, "  INSTDONE: 0x%08x\n", ee->instdone);
+
+	error_print_instdone(m, ee);
+
 	if (ee->batchbuffer) {
 		u64 start = ee->batchbuffer->gtt_offset;
 		u64 end = start + ee->batchbuffer->gtt_size;
@@ -263,17 +415,13 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
 	if (INTEL_GEN(m->i915) >= 6) {
 		err_printf(m, "  RC PSMI: 0x%08x\n", ee->rc_psmi);
 		err_printf(m, "  FAULT_REG: 0x%08x\n", ee->fault_reg);
-		err_printf(m, "  SYNC_0: 0x%08x [last synced 0x%08x]\n",
-			   ee->semaphore_mboxes[0],
-			   ee->semaphore_seqno[0]);
-		err_printf(m, "  SYNC_1: 0x%08x [last synced 0x%08x]\n",
-			   ee->semaphore_mboxes[1],
-			   ee->semaphore_seqno[1]);
-		if (HAS_VEBOX(m->i915)) {
-			err_printf(m, "  SYNC_2: 0x%08x [last synced 0x%08x]\n",
-				   ee->semaphore_mboxes[2],
-				   ee->semaphore_seqno[2]);
-		}
+		err_printf(m, "  SYNC_0: 0x%08x\n",
+			   ee->semaphore_mboxes[0]);
+		err_printf(m, "  SYNC_1: 0x%08x\n",
+			   ee->semaphore_mboxes[1]);
+		if (HAS_VEBOX(m->i915))
+			err_printf(m, "  SYNC_2: 0x%08x\n",
+				   ee->semaphore_mboxes[2]);
 	}
 	if (USES_PPGTT(m->i915)) {
 		err_printf(m, "  GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode);
@@ -296,6 +444,8 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
 	err_printf(m, "  hangcheck: %s [%d]\n",
 		   hangcheck_action_to_str(ee->hangcheck_action),
 		   ee->hangcheck_score);
+	error_print_request(m, "  ELSP[0]: ", &ee->execlist[0]);
+	error_print_request(m, "  ELSP[1]: ", &ee->execlist[1]);
 }
 
 void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
@@ -307,40 +457,83 @@ void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
 	va_end(args);
 }
 
+static int
+ascii85_encode_len(int len)
+{
+	return DIV_ROUND_UP(len, 4);
+}
+
+static bool
+ascii85_encode(u32 in, char *out)
+{
+	int i;
+
+	if (in == 0)
+		return false;
+
+	out[5] = '\0';
+	for (i = 5; i--; ) {
+		out[i] = '!' + in % 85;
+		in /= 85;
+	}
+
+	return true;
+}
+
 static void print_error_obj(struct drm_i915_error_state_buf *m,
+			    struct intel_engine_cs *engine,
+			    const char *name,
 			    struct drm_i915_error_object *obj)
 {
-	int page, offset, elt;
+	char out[6];
+	int page;
+
+	if (!obj)
+		return;
 
-	for (page = offset = 0; page < obj->page_count; page++) {
-		for (elt = 0; elt < PAGE_SIZE/4; elt++) {
-			err_printf(m, "%08x :  %08x\n", offset,
-				   obj->pages[page][elt]);
-			offset += 4;
+	if (name) {
+		err_printf(m, "%s --- %s = 0x%08x %08x\n",
+			   engine ? engine->name : "global", name,
+			   upper_32_bits(obj->gtt_offset),
+			   lower_32_bits(obj->gtt_offset));
+	}
+
+	err_compression_marker(m);
+	for (page = 0; page < obj->page_count; page++) {
+		int i, len;
+
+		len = PAGE_SIZE;
+		if (page == obj->page_count - 1)
+			len -= obj->unused;
+		len = ascii85_encode_len(len);
+
+		for (i = 0; i < len; i++) {
+			if (ascii85_encode(obj->pages[page][i], out))
+				err_puts(m, out);
+			else
+				err_puts(m, "z");
 		}
 	}
+	err_puts(m, "\n");
 }
 
 static void err_print_capabilities(struct drm_i915_error_state_buf *m,
 				   const struct intel_device_info *info)
 {
 #define PRINT_FLAG(x)  err_printf(m, #x ": %s\n", yesno(info->x))
-#define SEP_SEMICOLON ;
-	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_SEMICOLON);
+	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG);
 #undef PRINT_FLAG
-#undef SEP_SEMICOLON
 }
 
 int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 			    const struct i915_error_state_file_priv *error_priv)
 {
-	struct drm_device *dev = error_priv->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(error_priv->dev);
 	struct pci_dev *pdev = dev_priv->drm.pdev;
 	struct drm_i915_error_state *error = error_priv->error;
 	struct drm_i915_error_object *obj;
-	int i, j, offset, elt;
 	int max_hangcheck_score;
+	int i, j;
 
 	if (!error) {
 		err_printf(m, "no error state collected\n");
@@ -348,9 +541,13 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 	}
 
 	err_printf(m, "%s\n", error->error_msg);
-	err_printf(m, "Time: %ld s %ld us\n", error->time.tv_sec,
-		   error->time.tv_usec);
 	err_printf(m, "Kernel: " UTS_RELEASE "\n");
+	err_printf(m, "Time: %ld s %ld us\n",
+		   error->time.tv_sec, error->time.tv_usec);
+	err_printf(m, "Boottime: %ld s %ld us\n",
+		   error->boottime.tv_sec, error->boottime.tv_usec);
+	err_printf(m, "Uptime: %ld s %ld us\n",
+		   error->uptime.tv_sec, error->uptime.tv_usec);
 	err_print_capabilities(m, &error->device_info);
 	max_hangcheck_score = 0;
 	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
@@ -375,7 +572,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 		   pdev->subsystem_device);
 	err_printf(m, "IOMMU enabled?: %d\n", error->iommu);
 
-	if (HAS_CSR(dev)) {
+	if (HAS_CSR(dev_priv)) {
 		struct intel_csr *csr = &dev_priv->csr;
 
 		err_printf(m, "DMC loaded: %s\n",
@@ -387,11 +584,11 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 
 	err_printf(m, "EIR: 0x%08x\n", error->eir);
 	err_printf(m, "IER: 0x%08x\n", error->ier);
-	if (INTEL_INFO(dev)->gen >= 8) {
+	if (INTEL_GEN(dev_priv) >= 8) {
 		for (i = 0; i < 4; i++)
 			err_printf(m, "GTIER gt %d: 0x%08x\n", i,
 				   error->gtier[i]);
-	} else if (HAS_PCH_SPLIT(dev) || IS_VALLEYVIEW(dev))
+	} else if (HAS_PCH_SPLIT(dev_priv) || IS_VALLEYVIEW(dev_priv))
 		err_printf(m, "GTIER: 0x%08x\n", error->gtier[0]);
 	err_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
 	err_printf(m, "FORCEWAKE: 0x%08x\n", error->forcewake);
@@ -402,21 +599,17 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 	for (i = 0; i < dev_priv->num_fence_regs; i++)
 		err_printf(m, "  fence[%d] = %08llx\n", i, error->fence[i]);
 
-	for (i = 0; i < ARRAY_SIZE(error->extra_instdone); i++)
-		err_printf(m, "  INSTDONE_%d: 0x%08x\n", i,
-			   error->extra_instdone[i]);
-
-	if (INTEL_INFO(dev)->gen >= 6) {
+	if (INTEL_GEN(dev_priv) >= 6) {
 		err_printf(m, "ERROR: 0x%08x\n", error->error);
 
-		if (INTEL_INFO(dev)->gen >= 8)
+		if (INTEL_GEN(dev_priv) >= 8)
 			err_printf(m, "FAULT_TLB_DATA: 0x%08x 0x%08x\n",
 				   error->fault_data1, error->fault_data0);
 
 		err_printf(m, "DONE_REG: 0x%08x\n", error->done_reg);
 	}
 
-	if (IS_GEN7(dev))
+	if (IS_GEN7(dev_priv))
 		err_printf(m, "ERR_INT: 0x%08x\n", error->err_int);
 
 	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
@@ -438,7 +631,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 
 			len += scnprintf(buf + len, sizeof(buf), "%s%s",
 					 first ? "" : ", ",
-					 dev_priv->engine[j].name);
+					 dev_priv->engine[j]->name);
 			first = 0;
 		}
 		scnprintf(buf + len, sizeof(buf), ")");
@@ -456,7 +649,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 
 		obj = ee->batchbuffer;
 		if (obj) {
-			err_puts(m, dev_priv->engine[i].name);
+			err_puts(m, dev_priv->engine[i]->name);
 			if (ee->pid != -1)
 				err_printf(m, " (submitted by %s [%d])",
 					   ee->comm,
@@ -464,37 +657,23 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 			err_printf(m, " --- gtt_offset = 0x%08x %08x\n",
 				   upper_32_bits(obj->gtt_offset),
 				   lower_32_bits(obj->gtt_offset));
-			print_error_obj(m, obj);
-		}
-
-		obj = ee->wa_batchbuffer;
-		if (obj) {
-			err_printf(m, "%s (w/a) --- gtt_offset = 0x%08x\n",
-				   dev_priv->engine[i].name,
-				   lower_32_bits(obj->gtt_offset));
-			print_error_obj(m, obj);
+			print_error_obj(m, dev_priv->engine[i], NULL, obj);
 		}
 
 		if (ee->num_requests) {
 			err_printf(m, "%s --- %d requests\n",
-				   dev_priv->engine[i].name,
+				   dev_priv->engine[i]->name,
 				   ee->num_requests);
-			for (j = 0; j < ee->num_requests; j++) {
-				err_printf(m, "  pid %d, seqno 0x%08x, emitted %ld, head 0x%08x, tail 0x%08x\n",
-					   ee->requests[j].pid,
-					   ee->requests[j].seqno,
-					   ee->requests[j].jiffies,
-					   ee->requests[j].head,
-					   ee->requests[j].tail);
-			}
+			for (j = 0; j < ee->num_requests; j++)
+				error_print_request(m, " ", &ee->requests[j]);
 		}
 
 		if (IS_ERR(ee->waiters)) {
 			err_printf(m, "%s --- ? waiters [unable to acquire spinlock]\n",
-				   dev_priv->engine[i].name);
+				   dev_priv->engine[i]->name);
 		} else if (ee->num_waiters) {
 			err_printf(m, "%s --- %d waiters\n",
-				   dev_priv->engine[i].name,
+				   dev_priv->engine[i]->name,
 				   ee->num_waiters);
 			for (j = 0; j < ee->num_waiters; j++) {
 				err_printf(m, " seqno 0x%08x for %s [%d]\n",
@@ -504,83 +683,31 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 			}
 		}
 
-		if ((obj = ee->ringbuffer)) {
-			err_printf(m, "%s --- ringbuffer = 0x%08x\n",
-				   dev_priv->engine[i].name,
-				   lower_32_bits(obj->gtt_offset));
-			print_error_obj(m, obj);
-		}
+		print_error_obj(m, dev_priv->engine[i],
+				"ringbuffer", ee->ringbuffer);
 
-		if ((obj = ee->hws_page)) {
-			u64 hws_offset = obj->gtt_offset;
-			u32 *hws_page = &obj->pages[0][0];
+		print_error_obj(m, dev_priv->engine[i],
+				"HW Status", ee->hws_page);
 
-			if (i915.enable_execlists) {
-				hws_offset += LRC_PPHWSP_PN * PAGE_SIZE;
-				hws_page = &obj->pages[LRC_PPHWSP_PN][0];
-			}
-			err_printf(m, "%s --- HW Status = 0x%08llx\n",
-				   dev_priv->engine[i].name, hws_offset);
-			offset = 0;
-			for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
-				err_printf(m, "[%04x] %08x %08x %08x %08x\n",
-					   offset,
-					   hws_page[elt],
-					   hws_page[elt+1],
-					   hws_page[elt+2],
-					   hws_page[elt+3]);
-				offset += 16;
-			}
-		}
+		print_error_obj(m, dev_priv->engine[i],
+				"HW context", ee->ctx);
 
-		obj = ee->wa_ctx;
-		if (obj) {
-			u64 wa_ctx_offset = obj->gtt_offset;
-			u32 *wa_ctx_page = &obj->pages[0][0];
-			struct intel_engine_cs *engine = &dev_priv->engine[RCS];
-			u32 wa_ctx_size = (engine->wa_ctx.indirect_ctx.size +
-					   engine->wa_ctx.per_ctx.size);
-
-			err_printf(m, "%s --- WA ctx batch buffer = 0x%08llx\n",
-				   dev_priv->engine[i].name, wa_ctx_offset);
-			offset = 0;
-			for (elt = 0; elt < wa_ctx_size; elt += 4) {
-				err_printf(m, "[%04x] %08x %08x %08x %08x\n",
-					   offset,
-					   wa_ctx_page[elt + 0],
-					   wa_ctx_page[elt + 1],
-					   wa_ctx_page[elt + 2],
-					   wa_ctx_page[elt + 3]);
-				offset += 16;
-			}
-		}
+		print_error_obj(m, dev_priv->engine[i],
+				"WA context", ee->wa_ctx);
 
-		if ((obj = ee->ctx)) {
-			err_printf(m, "%s --- HW Context = 0x%08x\n",
-				   dev_priv->engine[i].name,
-				   lower_32_bits(obj->gtt_offset));
-			print_error_obj(m, obj);
-		}
+		print_error_obj(m, dev_priv->engine[i],
+				"WA batchbuffer", ee->wa_batchbuffer);
 	}
 
-	if ((obj = error->semaphore)) {
-		err_printf(m, "Semaphore page = 0x%08x\n",
-			   lower_32_bits(obj->gtt_offset));
-		for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
-			err_printf(m, "[%04x] %08x %08x %08x %08x\n",
-				   elt * 4,
-				   obj->pages[0][elt],
-				   obj->pages[0][elt+1],
-				   obj->pages[0][elt+2],
-				   obj->pages[0][elt+3]);
-		}
-	}
+	print_error_obj(m, NULL, "Semaphores", error->semaphore);
+
+	print_error_obj(m, NULL, "GuC log buffer", error->guc_log);
 
 	if (error->overlay)
 		intel_overlay_print_error_state(m, error->overlay);
 
 	if (error->display)
-		intel_display_print_error_state(m, dev, error->display);
+		intel_display_print_error_state(m, dev_priv, error->display);
 
 out:
 	if (m->bytes == 0 && m->err)
@@ -629,7 +756,7 @@ static void i915_error_object_free(struct drm_i915_error_object *obj)
 		return;
 
 	for (page = 0; page < obj->page_count; page++)
-		kfree(obj->pages[page]);
+		free_page((unsigned long)obj->pages[page]);
 
 	kfree(obj);
 }
@@ -656,6 +783,7 @@ static void i915_error_state_free(struct kref *error_ref)
 	}
 
 	i915_error_object_free(error->semaphore);
+	i915_error_object_free(error->guc_log);
 
 	for (i = 0; i < ARRAY_SIZE(error->active_bo); i++)
 		kfree(error->active_bo[i]);
@@ -667,104 +795,63 @@ static void i915_error_state_free(struct kref *error_ref)
 }
 
 static struct drm_i915_error_object *
-i915_error_object_create(struct drm_i915_private *dev_priv,
+i915_error_object_create(struct drm_i915_private *i915,
 			 struct i915_vma *vma)
 {
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	struct drm_i915_gem_object *src;
+	struct i915_ggtt *ggtt = &i915->ggtt;
+	const u64 slot = ggtt->error_capture.start;
 	struct drm_i915_error_object *dst;
-	int num_pages;
-	bool use_ggtt;
-	int i = 0;
-	u64 reloc_offset;
+	struct z_stream_s zstream;
+	unsigned long num_pages;
+	struct sgt_iter iter;
+	dma_addr_t dma;
 
 	if (!vma)
 		return NULL;
 
-	src = vma->obj;
-	if (!src->pages)
-		return NULL;
-
-	num_pages = src->base.size >> PAGE_SHIFT;
-
-	dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *), GFP_ATOMIC);
+	num_pages = min_t(u64, vma->size, vma->obj->base.size) >> PAGE_SHIFT;
+	num_pages = DIV_ROUND_UP(10 * num_pages, 8); /* worstcase zlib growth */
+	dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *),
+		      GFP_ATOMIC | __GFP_NOWARN);
 	if (!dst)
 		return NULL;
 
 	dst->gtt_offset = vma->node.start;
 	dst->gtt_size = vma->node.size;
+	dst->page_count = 0;
+	dst->unused = 0;
 
-	reloc_offset = dst->gtt_offset;
-	use_ggtt = (src->cache_level == I915_CACHE_NONE &&
-		   (vma->flags & I915_VMA_GLOBAL_BIND) &&
-		   reloc_offset + num_pages * PAGE_SIZE <= ggtt->mappable_end);
-
-	/* Cannot access stolen address directly, try to use the aperture */
-	if (src->stolen) {
-		use_ggtt = true;
-
-		if (!(vma->flags & I915_VMA_GLOBAL_BIND))
-			goto unwind;
-
-		reloc_offset = vma->node.start;
-		if (reloc_offset + num_pages * PAGE_SIZE > ggtt->mappable_end)
-			goto unwind;
+	if (!compress_init(&zstream)) {
+		kfree(dst);
+		return NULL;
 	}
 
-	/* Cannot access snooped pages through the aperture */
-	if (use_ggtt && src->cache_level != I915_CACHE_NONE &&
-	    !HAS_LLC(dev_priv))
-		goto unwind;
-
-	dst->page_count = num_pages;
-	while (num_pages--) {
-		unsigned long flags;
-		void *d;
-
-		d = kmalloc(PAGE_SIZE, GFP_ATOMIC);
-		if (d == NULL)
-			goto unwind;
-
-		local_irq_save(flags);
-		if (use_ggtt) {
-			void __iomem *s;
-
-			/* Simply ignore tiling or any overlapping fence.
-			 * It's part of the error state, and this hopefully
-			 * captures what the GPU read.
-			 */
-
-			s = io_mapping_map_atomic_wc(&ggtt->mappable,
-						     reloc_offset);
-			memcpy_fromio(d, s, PAGE_SIZE);
-			io_mapping_unmap_atomic(s);
-		} else {
-			struct page *page;
-			void *s;
-
-			page = i915_gem_object_get_page(src, i);
-
-			drm_clflush_pages(&page, 1);
+	for_each_sgt_dma(dma, iter, vma->pages) {
+		void __iomem *s;
+		int ret;
 
-			s = kmap_atomic(page);
-			memcpy(d, s, PAGE_SIZE);
-			kunmap_atomic(s);
+		ggtt->base.insert_page(&ggtt->base, dma, slot,
+				       I915_CACHE_NONE, 0);
 
-			drm_clflush_pages(&page, 1);
-		}
-		local_irq_restore(flags);
+		s = io_mapping_map_atomic_wc(&ggtt->mappable, slot);
+		ret = compress_page(&zstream, (void  __force *)s, dst);
+		io_mapping_unmap_atomic(s);
 
-		dst->pages[i++] = d;
-		reloc_offset += PAGE_SIZE;
+		if (ret)
+			goto unwind;
 	}
-
-	return dst;
+	goto out;
 
 unwind:
-	while (i--)
-		kfree(dst->pages[i]);
+	while (dst->page_count--)
+		free_page((unsigned long)dst->pages[dst->page_count]);
 	kfree(dst);
-	return NULL;
+	dst = NULL;
+
+out:
+	compress_fini(&zstream, dst);
+	ggtt->base.clear_range(&ggtt->base, slot, PAGE_SIZE);
+	return dst;
 }
 
 /* The error capture is special as tries to run underneath the normal
@@ -773,16 +860,19 @@ unwind:
 static inline uint32_t
 __active_get_seqno(struct i915_gem_active *active)
 {
-	return i915_gem_request_get_seqno(__i915_gem_active_peek(active));
+	struct drm_i915_gem_request *request;
+
+	request = __i915_gem_active_peek(active);
+	return request ? request->global_seqno : 0;
 }
 
 static inline int
 __active_get_engine_id(struct i915_gem_active *active)
 {
-	struct intel_engine_cs *engine;
+	struct drm_i915_gem_request *request;
 
-	engine = i915_gem_request_get_engine(__i915_gem_active_peek(active));
-	return engine ? engine->id : -1;
+	request = __i915_gem_active_peek(active);
+	return request ? request->engine->id : -1;
 }
 
 static void capture_bo(struct drm_i915_error_buffer *err,
@@ -795,17 +885,17 @@ static void capture_bo(struct drm_i915_error_buffer *err,
 	err->name = obj->base.name;
 
 	for (i = 0; i < I915_NUM_ENGINES; i++)
-		err->rseqno[i] = __active_get_seqno(&obj->last_read[i]);
-	err->wseqno = __active_get_seqno(&obj->last_write);
-	err->engine = __active_get_engine_id(&obj->last_write);
+		err->rseqno[i] = __active_get_seqno(&vma->last_read[i]);
+	err->wseqno = __active_get_seqno(&obj->frontbuffer_write);
+	err->engine = __active_get_engine_id(&obj->frontbuffer_write);
 
 	err->gtt_offset = vma->node.start;
 	err->read_domains = obj->base.read_domains;
 	err->write_domain = obj->base.write_domain;
 	err->fence_reg = vma->fence ? vma->fence->id : -1;
 	err->tiling = i915_gem_object_get_tiling(obj);
-	err->dirty = obj->dirty;
-	err->purgeable = obj->madv != I915_MADV_WILLNEED;
+	err->dirty = obj->mm.dirty;
+	err->purgeable = obj->mm.madv != I915_MADV_WILLNEED;
 	err->userptr = obj->userptr.mm != NULL;
 	err->cache_level = obj->cache_level;
 }
@@ -855,7 +945,8 @@ static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv,
 			if (engine_id)
 				*engine_id = i;
 
-			return error->engine[i].ipehr ^ error->engine[i].instdone;
+			return error->engine[i].ipehr ^
+			       error->engine[i].instdone.instdone;
 		}
 	}
 
@@ -879,6 +970,26 @@ static void i915_gem_record_fences(struct drm_i915_private *dev_priv,
 	}
 }
 
+static inline u32
+gen8_engine_sync_index(struct intel_engine_cs *engine,
+		       struct intel_engine_cs *other)
+{
+	int idx;
+
+	/*
+	 * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2;
+	 * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs;
+	 * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs;
+	 * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs;
+	 * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
+	 */
+
+	idx = (other - engine) - 1;
+	if (idx < 0)
+		idx += I915_NUM_ENGINES;
+
+	return idx;
+}
 
 static void gen8_record_semaphore_state(struct drm_i915_error_state *error,
 					struct intel_engine_cs *engine,
@@ -891,7 +1002,7 @@ static void gen8_record_semaphore_state(struct drm_i915_error_state *error,
 	if (!error->semaphore)
 		return;
 
-	for_each_engine_id(to, dev_priv, id) {
+	for_each_engine(to, dev_priv, id) {
 		int idx;
 		u16 signal_offset;
 		u32 *tmp;
@@ -902,10 +1013,9 @@ static void gen8_record_semaphore_state(struct drm_i915_error_state *error,
 		signal_offset =
 			(GEN8_SIGNAL_OFFSET(engine, id) & (PAGE_SIZE - 1)) / 4;
 		tmp = error->semaphore->pages[0];
-		idx = intel_engine_sync_index(engine, to);
+		idx = gen8_engine_sync_index(engine, to);
 
 		ee->semaphore_mboxes[idx] = tmp[signal_offset];
-		ee->semaphore_seqno[idx] = engine->semaphore.sync_seqno[idx];
 	}
 }
 
@@ -916,14 +1026,9 @@ static void gen6_record_semaphore_state(struct intel_engine_cs *engine,
 
 	ee->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(engine->mmio_base));
 	ee->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(engine->mmio_base));
-	ee->semaphore_seqno[0] = engine->semaphore.sync_seqno[0];
-	ee->semaphore_seqno[1] = engine->semaphore.sync_seqno[1];
-
-	if (HAS_VEBOX(dev_priv)) {
+	if (HAS_VEBOX(dev_priv))
 		ee->semaphore_mboxes[2] =
 			I915_READ(RING_SYNC_2(engine->mmio_base));
-		ee->semaphore_seqno[2] = engine->semaphore.sync_seqno[2];
-	}
 }
 
 static void error_record_engine_waiters(struct intel_engine_cs *engine,
@@ -940,7 +1045,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine,
 	if (RB_EMPTY_ROOT(&b->waiters))
 		return;
 
-	if (!spin_trylock(&b->lock)) {
+	if (!spin_trylock_irq(&b->lock)) {
 		ee->waiters = ERR_PTR(-EDEADLK);
 		return;
 	}
@@ -948,7 +1053,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine,
 	count = 0;
 	for (rb = rb_first(&b->waiters); rb != NULL; rb = rb_next(rb))
 		count++;
-	spin_unlock(&b->lock);
+	spin_unlock_irq(&b->lock);
 
 	waiter = NULL;
 	if (count)
@@ -958,7 +1063,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine,
 	if (!waiter)
 		return;
 
-	if (!spin_trylock(&b->lock)) {
+	if (!spin_trylock_irq(&b->lock)) {
 		kfree(waiter);
 		ee->waiters = ERR_PTR(-EDEADLK);
 		return;
@@ -976,7 +1081,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine,
 		if (++ee->num_waiters == count)
 			break;
 	}
-	spin_unlock(&b->lock);
+	spin_unlock_irq(&b->lock);
 }
 
 static void error_record_engine_registers(struct drm_i915_error_state *error,
@@ -998,7 +1103,6 @@ static void error_record_engine_registers(struct drm_i915_error_state *error,
 		ee->faddr = I915_READ(RING_DMA_FADD(engine->mmio_base));
 		ee->ipeir = I915_READ(RING_IPEIR(engine->mmio_base));
 		ee->ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
-		ee->instdone = I915_READ(RING_INSTDONE(engine->mmio_base));
 		ee->instps = I915_READ(RING_INSTPS(engine->mmio_base));
 		ee->bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
 		if (INTEL_GEN(dev_priv) >= 8) {
@@ -1010,14 +1114,15 @@ static void error_record_engine_registers(struct drm_i915_error_state *error,
 		ee->faddr = I915_READ(DMA_FADD_I8XX);
 		ee->ipeir = I915_READ(IPEIR);
 		ee->ipehr = I915_READ(IPEHR);
-		ee->instdone = I915_READ(GEN2_INSTDONE);
 	}
 
+	intel_engine_get_instdone(engine, &ee->instdone);
+
 	ee->waiting = intel_engine_has_waiter(engine);
 	ee->instpm = I915_READ(RING_INSTPM(engine->mmio_base));
 	ee->acthd = intel_engine_get_active_head(engine);
 	ee->seqno = intel_engine_get_seqno(engine);
-	ee->last_seqno = engine->last_submitted_seqno;
+	ee->last_seqno = intel_engine_last_submit(engine);
 	ee->start = I915_READ_START(engine);
 	ee->head = I915_READ_HEAD(engine);
 	ee->tail = I915_READ_TAIL(engine);
@@ -1079,6 +1184,20 @@ static void error_record_engine_registers(struct drm_i915_error_state *error,
 	}
 }
 
+static void record_request(struct drm_i915_gem_request *request,
+			   struct drm_i915_error_request *erq)
+{
+	erq->context = request->ctx->hw_id;
+	erq->seqno = request->global_seqno;
+	erq->jiffies = request->emitted_jiffies;
+	erq->head = request->head;
+	erq->tail = request->tail;
+
+	rcu_read_lock();
+	erq->pid = request->ctx->pid ? pid_nr(request->ctx->pid) : 0;
+	rcu_read_unlock();
+}
+
 static void engine_record_requests(struct intel_engine_cs *engine,
 				   struct drm_i915_gem_request *first,
 				   struct drm_i915_error_engine *ee)
@@ -1088,7 +1207,7 @@ static void engine_record_requests(struct intel_engine_cs *engine,
 
 	count = 0;
 	request = first;
-	list_for_each_entry_from(request, &engine->request_list, link)
+	list_for_each_entry_from(request, &engine->timeline->requests, link)
 		count++;
 	if (!count)
 		return;
@@ -1101,9 +1220,7 @@ static void engine_record_requests(struct intel_engine_cs *engine,
 
 	count = 0;
 	request = first;
-	list_for_each_entry_from(request, &engine->request_list, link) {
-		struct drm_i915_error_request *erq;
-
+	list_for_each_entry_from(request, &engine->timeline->requests, link) {
 		if (count >= ee->num_requests) {
 			/*
 			 * If the ring request list was changed in
@@ -1123,19 +1240,22 @@ static void engine_record_requests(struct intel_engine_cs *engine,
 			break;
 		}
 
-		erq = &ee->requests[count++];
-		erq->seqno = request->fence.seqno;
-		erq->jiffies = request->emitted_jiffies;
-		erq->head = request->head;
-		erq->tail = request->tail;
-
-		rcu_read_lock();
-		erq->pid = request->ctx->pid ? pid_nr(request->ctx->pid) : 0;
-		rcu_read_unlock();
+		record_request(request, &ee->requests[count++]);
 	}
 	ee->num_requests = count;
 }
 
+static void error_record_engine_execlists(struct intel_engine_cs *engine,
+					  struct drm_i915_error_engine *ee)
+{
+	unsigned int n;
+
+	for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++)
+		if (engine->execlist_port[n].request)
+			record_request(engine->execlist_port[n].request,
+				       &ee->execlist[n]);
+}
+
 static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
 				  struct drm_i915_error_state *error)
 {
@@ -1146,20 +1266,21 @@ static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
 		i915_error_object_create(dev_priv, dev_priv->semaphore);
 
 	for (i = 0; i < I915_NUM_ENGINES; i++) {
-		struct intel_engine_cs *engine = &dev_priv->engine[i];
+		struct intel_engine_cs *engine = dev_priv->engine[i];
 		struct drm_i915_error_engine *ee = &error->engine[i];
 		struct drm_i915_gem_request *request;
 
 		ee->pid = -1;
 		ee->engine_id = -1;
 
-		if (!intel_engine_initialized(engine))
+		if (!engine)
 			continue;
 
 		ee->engine_id = i;
 
 		error_record_engine_registers(error, engine, ee);
 		error_record_engine_waiters(engine, ee);
+		error_record_engine_execlists(engine, ee);
 
 		request = i915_gem_find_active_request(engine);
 		if (request) {
@@ -1202,6 +1323,10 @@ static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
 			error->simulated |=
 				request->ctx->flags & CONTEXT_NO_ERROR_CAPTURE;
 
+			ee->rq_head = request->head;
+			ee->rq_post = request->postfix;
+			ee->rq_tail = request->tail;
+
 			ring = request->ring;
 			ee->cpu_ring_head = ring->head;
 			ee->cpu_ring_tail = ring->tail;
@@ -1302,11 +1427,21 @@ static void i915_capture_pinned_buffers(struct drm_i915_private *dev_priv,
 	error->pinned_bo = bo;
 }
 
+static void i915_gem_capture_guc_log_buffer(struct drm_i915_private *dev_priv,
+					    struct drm_i915_error_state *error)
+{
+	/* Capturing log buf contents won't be useful if logging was disabled */
+	if (!dev_priv->guc.log.vma || (i915.guc_log_level < 0))
+		return;
+
+	error->guc_log = i915_error_object_create(dev_priv,
+						  dev_priv->guc.log.vma);
+}
+
 /* Capture all registers which don't fit into another category. */
 static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
 				   struct drm_i915_error_state *error)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	int i;
 
 	/* General organization
@@ -1318,62 +1453,60 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
 	 */
 
 	/* 1: Registers specific to a single generation */
-	if (IS_VALLEYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev_priv)) {
 		error->gtier[0] = I915_READ(GTIER);
 		error->ier = I915_READ(VLV_IER);
 		error->forcewake = I915_READ_FW(FORCEWAKE_VLV);
 	}
 
-	if (IS_GEN7(dev))
+	if (IS_GEN7(dev_priv))
 		error->err_int = I915_READ(GEN7_ERR_INT);
 
-	if (INTEL_INFO(dev)->gen >= 8) {
+	if (INTEL_GEN(dev_priv) >= 8) {
 		error->fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0);
 		error->fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1);
 	}
 
-	if (IS_GEN6(dev)) {
+	if (IS_GEN6(dev_priv)) {
 		error->forcewake = I915_READ_FW(FORCEWAKE);
 		error->gab_ctl = I915_READ(GAB_CTL);
 		error->gfx_mode = I915_READ(GFX_MODE);
 	}
 
 	/* 2: Registers which belong to multiple generations */
-	if (INTEL_INFO(dev)->gen >= 7)
+	if (INTEL_GEN(dev_priv) >= 7)
 		error->forcewake = I915_READ_FW(FORCEWAKE_MT);
 
-	if (INTEL_INFO(dev)->gen >= 6) {
+	if (INTEL_GEN(dev_priv) >= 6) {
 		error->derrmr = I915_READ(DERRMR);
 		error->error = I915_READ(ERROR_GEN6);
 		error->done_reg = I915_READ(DONE_REG);
 	}
 
 	/* 3: Feature specific registers */
-	if (IS_GEN6(dev) || IS_GEN7(dev)) {
+	if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
 		error->gam_ecochk = I915_READ(GAM_ECOCHK);
 		error->gac_eco = I915_READ(GAC_ECO_BITS);
 	}
 
 	/* 4: Everything else */
-	if (HAS_HW_CONTEXTS(dev))
+	if (HAS_HW_CONTEXTS(dev_priv))
 		error->ccid = I915_READ(CCID);
 
-	if (INTEL_INFO(dev)->gen >= 8) {
+	if (INTEL_GEN(dev_priv) >= 8) {
 		error->ier = I915_READ(GEN8_DE_MISC_IER);
 		for (i = 0; i < 4; i++)
 			error->gtier[i] = I915_READ(GEN8_GT_IER(i));
-	} else if (HAS_PCH_SPLIT(dev)) {
+	} else if (HAS_PCH_SPLIT(dev_priv)) {
 		error->ier = I915_READ(DEIER);
 		error->gtier[0] = I915_READ(GTIER);
-	} else if (IS_GEN2(dev)) {
+	} else if (IS_GEN2(dev_priv)) {
 		error->ier = I915_READ16(IER);
-	} else if (!IS_VALLEYVIEW(dev)) {
+	} else if (!IS_VALLEYVIEW(dev_priv)) {
 		error->ier = I915_READ(IER);
 	}
 	error->eir = I915_READ(EIR);
 	error->pgtbl_er = I915_READ(PGTBL_ER);
-
-	i915_get_extra_instdone(dev_priv, error->extra_instdone);
 }
 
 static void i915_error_capture_msg(struct drm_i915_private *dev_priv,
@@ -1418,6 +1551,32 @@ static void i915_capture_gen_state(struct drm_i915_private *dev_priv,
 	       sizeof(error->device_info));
 }
 
+static int capture(void *data)
+{
+	struct drm_i915_error_state *error = data;
+
+	i915_capture_gen_state(error->i915, error);
+	i915_capture_reg_state(error->i915, error);
+	i915_gem_record_fences(error->i915, error);
+	i915_gem_record_rings(error->i915, error);
+	i915_capture_active_buffers(error->i915, error);
+	i915_capture_pinned_buffers(error->i915, error);
+	i915_gem_capture_guc_log_buffer(error->i915, error);
+
+	do_gettimeofday(&error->time);
+	error->boottime = ktime_to_timeval(ktime_get_boottime());
+	error->uptime =
+		ktime_to_timeval(ktime_sub(ktime_get(),
+					   error->i915->gt.last_init_time));
+
+	error->overlay = intel_overlay_capture_error_state(error->i915);
+	error->display = intel_display_capture_error_state(error->i915);
+
+	return 0;
+}
+
+#define DAY_AS_SECONDS(x) (24 * 60 * 60 * (x))
+
 /**
  * i915_capture_error_state - capture an error record for later analysis
  * @dev: drm device
@@ -1435,6 +1594,9 @@ void i915_capture_error_state(struct drm_i915_private *dev_priv,
 	struct drm_i915_error_state *error;
 	unsigned long flags;
 
+	if (!i915.error_capture)
+		return;
+
 	if (READ_ONCE(dev_priv->gpu_error.first_error))
 		return;
 
@@ -1446,18 +1608,9 @@ void i915_capture_error_state(struct drm_i915_private *dev_priv,
 	}
 
 	kref_init(&error->ref);
+	error->i915 = dev_priv;
 
-	i915_capture_gen_state(dev_priv, error);
-	i915_capture_reg_state(dev_priv, error);
-	i915_gem_record_fences(dev_priv, error);
-	i915_gem_record_rings(dev_priv, error);
-	i915_capture_active_buffers(dev_priv, error);
-	i915_capture_pinned_buffers(dev_priv, error);
-
-	do_gettimeofday(&error->time);
-
-	error->overlay = intel_overlay_capture_error_state(dev_priv);
-	error->display = intel_display_capture_error_state(dev_priv);
+	stop_machine(capture, error, NULL);
 
 	i915_error_capture_msg(dev_priv, error, engine_mask, error_msg);
 	DRM_INFO("%s\n", error->error_msg);
@@ -1476,7 +1629,8 @@ void i915_capture_error_state(struct drm_i915_private *dev_priv,
 		return;
 	}
 
-	if (!warned) {
+	if (!warned &&
+	    ktime_get_real_seconds() - DRIVER_TIMESTAMP < DAY_AS_SECONDS(180)) {
 		DRM_INFO("GPU hangs can indicate a bug anywhere in the entire gfx stack, including userspace.\n");
 		DRM_INFO("Please file a _new_ bug report on bugs.freedesktop.org against DRI -> DRM/Intel\n");
 		DRM_INFO("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n");
@@ -1497,7 +1651,6 @@ void i915_error_state_get(struct drm_device *dev,
 	if (error_priv->error)
 		kref_get(&error_priv->error->ref);
 	spin_unlock_irq(&dev_priv->gpu_error.lock);
-
 }
 
 void i915_error_state_put(struct i915_error_state_file_priv *error_priv)
@@ -1519,33 +1672,3 @@ void i915_destroy_error_state(struct drm_device *dev)
 	if (error)
 		kref_put(&error->ref, i915_error_state_free);
 }
-
-const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
-{
-	switch (type) {
-	case I915_CACHE_NONE: return " uncached";
-	case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
-	case I915_CACHE_L3_LLC: return " L3+LLC";
-	case I915_CACHE_WT: return " WT";
-	default: return "";
-	}
-}
-
-/* NB: please notice the memset */
-void i915_get_extra_instdone(struct drm_i915_private *dev_priv,
-			     uint32_t *instdone)
-{
-	memset(instdone, 0, sizeof(*instdone) * I915_NUM_INSTDONE_REG);
-
-	if (IS_GEN2(dev_priv) || IS_GEN3(dev_priv))
-		instdone[0] = I915_READ(GEN2_INSTDONE);
-	else if (IS_GEN4(dev_priv) || IS_GEN5(dev_priv) || IS_GEN6(dev_priv)) {
-		instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE));
-		instdone[1] = I915_READ(GEN4_INSTDONE1);
-	} else if (INTEL_GEN(dev_priv) >= 7) {
-		instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE));
-		instdone[1] = I915_READ(GEN7_SC_INSTDONE);
-		instdone[2] = I915_READ(GEN7_SAMPLER_INSTDONE);
-		instdone[3] = I915_READ(GEN7_ROW_INSTDONE);
-	}
-}
diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c
index 3106dcc06fe9..4462112725ef 100644
--- a/drivers/gpu/drm/i915/i915_guc_submission.c
+++ b/drivers/gpu/drm/i915/i915_guc_submission.c
@@ -23,6 +23,8 @@
  */
 #include <linux/firmware.h>
 #include <linux/circ_buf.h>
+#include <linux/debugfs.h>
+#include <linux/relay.h>
 #include "i915_drv.h"
 #include "intel_guc.h"
 
@@ -85,6 +87,7 @@ static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len)
 	if (WARN_ON(len < 1 || len > 15))
 		return -EINVAL;
 
+	mutex_lock(&guc->action_lock);
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
 	dev_priv->guc.action_count += 1;
@@ -123,6 +126,7 @@ static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len)
 	dev_priv->guc.action_status = status;
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	mutex_unlock(&guc->action_lock);
 
 	return ret;
 }
@@ -170,6 +174,35 @@ static int host2guc_sample_forcewake(struct intel_guc *guc,
 	return host2guc_action(guc, data, ARRAY_SIZE(data));
 }
 
+static int host2guc_logbuffer_flush_complete(struct intel_guc *guc)
+{
+	u32 data[1];
+
+	data[0] = HOST2GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE;
+
+	return host2guc_action(guc, data, 1);
+}
+
+static int host2guc_force_logbuffer_flush(struct intel_guc *guc)
+{
+	u32 data[2];
+
+	data[0] = HOST2GUC_ACTION_FORCE_LOG_BUFFER_FLUSH;
+	data[1] = 0;
+
+	return host2guc_action(guc, data, 2);
+}
+
+static int host2guc_logging_control(struct intel_guc *guc, u32 control_val)
+{
+	u32 data[2];
+
+	data[0] = HOST2GUC_ACTION_UK_LOG_ENABLE_LOGGING;
+	data[1] = control_val;
+
+	return host2guc_action(guc, data, 2);
+}
+
 /*
  * Initialise, update, or clear doorbell data shared with the GuC
  *
@@ -187,7 +220,7 @@ static int guc_update_doorbell_id(struct intel_guc *guc,
 	struct guc_context_desc desc;
 	size_t len;
 
-	doorbell = client->client_base + client->doorbell_offset;
+	doorbell = client->vaddr + client->doorbell_offset;
 
 	if (client->doorbell_id != GUC_INVALID_DOORBELL_ID &&
 	    test_bit(client->doorbell_id, doorbell_bitmap)) {
@@ -293,7 +326,7 @@ static void guc_proc_desc_init(struct intel_guc *guc,
 {
 	struct guc_process_desc *desc;
 
-	desc = client->client_base + client->proc_desc_offset;
+	desc = client->vaddr + client->proc_desc_offset;
 
 	memset(desc, 0, sizeof(*desc));
 
@@ -380,8 +413,8 @@ static void guc_ctx_desc_init(struct intel_guc *guc,
 	gfx_addr = i915_ggtt_offset(client->vma);
 	desc.db_trigger_phy = sg_dma_address(client->vma->pages->sgl) +
 				client->doorbell_offset;
-	desc.db_trigger_cpu = (uintptr_t)client->client_base +
-				client->doorbell_offset;
+	desc.db_trigger_cpu =
+		(uintptr_t)client->vaddr + client->doorbell_offset;
 	desc.db_trigger_uk = gfx_addr + client->doorbell_offset;
 	desc.process_desc = gfx_addr + client->proc_desc_offset;
 	desc.wq_addr = gfx_addr + client->wq_offset;
@@ -432,7 +465,7 @@ int i915_guc_wq_reserve(struct drm_i915_gem_request *request)
 {
 	const size_t wqi_size = sizeof(struct guc_wq_item);
 	struct i915_guc_client *gc = request->i915->guc.execbuf_client;
-	struct guc_process_desc *desc = gc->client_base + gc->proc_desc_offset;
+	struct guc_process_desc *desc = gc->vaddr + gc->proc_desc_offset;
 	u32 freespace;
 	int ret;
 
@@ -473,10 +506,9 @@ static void guc_wq_item_append(struct i915_guc_client *gc,
 	struct intel_engine_cs *engine = rq->engine;
 	struct guc_process_desc *desc;
 	struct guc_wq_item *wqi;
-	void *base;
-	u32 freespace, tail, wq_off, wq_page;
+	u32 freespace, tail, wq_off;
 
-	desc = gc->client_base + gc->proc_desc_offset;
+	desc = gc->vaddr + gc->proc_desc_offset;
 
 	/* Free space is guaranteed, see i915_guc_wq_reserve() above */
 	freespace = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size);
@@ -506,10 +538,7 @@ static void guc_wq_item_append(struct i915_guc_client *gc,
 	gc->wq_rsvd -= wqi_size;
 
 	/* WQ starts from the page after doorbell / process_desc */
-	wq_page = (wq_off + GUC_DB_SIZE) >> PAGE_SHIFT;
-	wq_off &= PAGE_SIZE - 1;
-	base = kmap_atomic(i915_gem_object_get_page(gc->vma->obj, wq_page));
-	wqi = (struct guc_wq_item *)((char *)base + wq_off);
+	wqi = gc->vaddr + wq_off + GUC_DB_SIZE;
 
 	/* Now fill in the 4-word work queue item */
 	wqi->header = WQ_TYPE_INORDER |
@@ -521,9 +550,7 @@ static void guc_wq_item_append(struct i915_guc_client *gc,
 	wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx, engine);
 
 	wqi->ring_tail = tail << WQ_RING_TAIL_SHIFT;
-	wqi->fence_id = rq->fence.seqno;
-
-	kunmap_atomic(base);
+	wqi->fence_id = rq->global_seqno;
 }
 
 static int guc_ring_doorbell(struct i915_guc_client *gc)
@@ -533,7 +560,7 @@ static int guc_ring_doorbell(struct i915_guc_client *gc)
 	union guc_doorbell_qw *db;
 	int attempt = 2, ret = -EAGAIN;
 
-	desc = gc->client_base + gc->proc_desc_offset;
+	desc = gc->vaddr + gc->proc_desc_offset;
 
 	/* Update the tail so it is visible to GuC */
 	desc->tail = gc->wq_tail;
@@ -549,7 +576,7 @@ static int guc_ring_doorbell(struct i915_guc_client *gc)
 		db_exc.cookie = 1;
 
 	/* pointer of current doorbell cacheline */
-	db = gc->client_base + gc->doorbell_offset;
+	db = gc->vaddr + gc->doorbell_offset;
 
 	while (attempt--) {
 		/* lets ring the doorbell */
@@ -601,13 +628,31 @@ static int guc_ring_doorbell(struct i915_guc_client *gc)
  */
 static void i915_guc_submit(struct drm_i915_gem_request *rq)
 {
-	unsigned int engine_id = rq->engine->id;
+	struct drm_i915_private *dev_priv = rq->i915;
+	struct intel_engine_cs *engine = rq->engine;
+	unsigned int engine_id = engine->id;
 	struct intel_guc *guc = &rq->i915->guc;
 	struct i915_guc_client *client = guc->execbuf_client;
 	int b_ret;
 
+	/* We keep the previous context alive until we retire the following
+	 * request. This ensures that any the context object is still pinned
+	 * for any residual writes the HW makes into it on the context switch
+	 * into the next object following the breadcrumb. Otherwise, we may
+	 * retire the context too early.
+	 */
+	rq->previous_context = engine->last_context;
+	engine->last_context = rq->ctx;
+
+	i915_gem_request_submit(rq);
+
 	spin_lock(&client->wq_lock);
 	guc_wq_item_append(client, rq);
+
+	/* WA to flush out the pending GMADR writes to ring buffer. */
+	if (i915_vma_is_map_and_fenceable(rq->ring->vma))
+		POSTING_READ_FW(GUC_STATUS);
+
 	b_ret = guc_ring_doorbell(client);
 
 	client->submissions[engine_id] += 1;
@@ -616,7 +661,7 @@ static void i915_guc_submit(struct drm_i915_gem_request *rq)
 		client->b_fail += 1;
 
 	guc->submissions[engine_id] += 1;
-	guc->last_seqno[engine_id] = rq->fence.seqno;
+	guc->last_seqno[engine_id] = rq->global_seqno;
 	spin_unlock(&client->wq_lock);
 }
 
@@ -685,14 +730,14 @@ guc_client_free(struct drm_i915_private *dev_priv,
 	 * Be sure to drop any locks
 	 */
 
-	if (client->client_base) {
+	if (client->vaddr) {
 		/*
 		 * If we got as far as setting up a doorbell, make sure we
 		 * shut it down before unmapping & deallocating the memory.
 		 */
 		guc_disable_doorbell(guc, client);
 
-		kunmap(kmap_to_page(client->client_base));
+		i915_gem_object_unpin_map(client->vma->obj);
 	}
 
 	i915_vma_unpin_and_release(&client->vma);
@@ -781,6 +826,7 @@ guc_client_alloc(struct drm_i915_private *dev_priv,
 	struct i915_guc_client *client;
 	struct intel_guc *guc = &dev_priv->guc;
 	struct i915_vma *vma;
+	void *vaddr;
 	uint16_t db_id;
 
 	client = kzalloc(sizeof(*client), GFP_KERNEL);
@@ -807,7 +853,12 @@ guc_client_alloc(struct drm_i915_private *dev_priv,
 
 	/* We'll keep just the first (doorbell/proc) page permanently kmap'd. */
 	client->vma = vma;
-	client->client_base = kmap(i915_vma_first_page(vma));
+
+	vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
+	if (IS_ERR(vaddr))
+		goto err;
+
+	client->vaddr = vaddr;
 
 	spin_lock_init(&client->wq_lock);
 	client->wq_offset = GUC_DB_SIZE;
@@ -847,15 +898,411 @@ err:
 	return NULL;
 }
 
+/*
+ * Sub buffer switch callback. Called whenever relay has to switch to a new
+ * sub buffer, relay stays on the same sub buffer if 0 is returned.
+ */
+static int subbuf_start_callback(struct rchan_buf *buf,
+				 void *subbuf,
+				 void *prev_subbuf,
+				 size_t prev_padding)
+{
+	/* Use no-overwrite mode by default, where relay will stop accepting
+	 * new data if there are no empty sub buffers left.
+	 * There is no strict synchronization enforced by relay between Consumer
+	 * and Producer. In overwrite mode, there is a possibility of getting
+	 * inconsistent/garbled data, the producer could be writing on to the
+	 * same sub buffer from which Consumer is reading. This can't be avoided
+	 * unless Consumer is fast enough and can always run in tandem with
+	 * Producer.
+	 */
+	if (relay_buf_full(buf))
+		return 0;
+
+	return 1;
+}
+
+/*
+ * file_create() callback. Creates relay file in debugfs.
+ */
+static struct dentry *create_buf_file_callback(const char *filename,
+					       struct dentry *parent,
+					       umode_t mode,
+					       struct rchan_buf *buf,
+					       int *is_global)
+{
+	struct dentry *buf_file;
+
+	/* This to enable the use of a single buffer for the relay channel and
+	 * correspondingly have a single file exposed to User, through which
+	 * it can collect the logs in order without any post-processing.
+	 * Need to set 'is_global' even if parent is NULL for early logging.
+	 */
+	*is_global = 1;
+
+	if (!parent)
+		return NULL;
+
+	/* Not using the channel filename passed as an argument, since for each
+	 * channel relay appends the corresponding CPU number to the filename
+	 * passed in relay_open(). This should be fine as relay just needs a
+	 * dentry of the file associated with the channel buffer and that file's
+	 * name need not be same as the filename passed as an argument.
+	 */
+	buf_file = debugfs_create_file("guc_log", mode,
+				       parent, buf, &relay_file_operations);
+	return buf_file;
+}
+
+/*
+ * file_remove() default callback. Removes relay file in debugfs.
+ */
+static int remove_buf_file_callback(struct dentry *dentry)
+{
+	debugfs_remove(dentry);
+	return 0;
+}
+
+/* relay channel callbacks */
+static struct rchan_callbacks relay_callbacks = {
+	.subbuf_start = subbuf_start_callback,
+	.create_buf_file = create_buf_file_callback,
+	.remove_buf_file = remove_buf_file_callback,
+};
+
+static void guc_log_remove_relay_file(struct intel_guc *guc)
+{
+	relay_close(guc->log.relay_chan);
+}
+
+static int guc_log_create_relay_channel(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	struct rchan *guc_log_relay_chan;
+	size_t n_subbufs, subbuf_size;
+
+	/* Keep the size of sub buffers same as shared log buffer */
+	subbuf_size = guc->log.vma->obj->base.size;
+
+	/* Store up to 8 snapshots, which is large enough to buffer sufficient
+	 * boot time logs and provides enough leeway to User, in terms of
+	 * latency, for consuming the logs from relay. Also doesn't take
+	 * up too much memory.
+	 */
+	n_subbufs = 8;
+
+	guc_log_relay_chan = relay_open(NULL, NULL, subbuf_size,
+					n_subbufs, &relay_callbacks, dev_priv);
+	if (!guc_log_relay_chan) {
+		DRM_ERROR("Couldn't create relay chan for GuC logging\n");
+		return -ENOMEM;
+	}
+
+	GEM_BUG_ON(guc_log_relay_chan->subbuf_size < subbuf_size);
+	guc->log.relay_chan = guc_log_relay_chan;
+	return 0;
+}
+
+static int guc_log_create_relay_file(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	struct dentry *log_dir;
+	int ret;
+
+	/* For now create the log file in /sys/kernel/debug/dri/0 dir */
+	log_dir = dev_priv->drm.primary->debugfs_root;
+
+	/* If /sys/kernel/debug/dri/0 location do not exist, then debugfs is
+	 * not mounted and so can't create the relay file.
+	 * The relay API seems to fit well with debugfs only, for availing relay
+	 * there are 3 requirements which can be met for debugfs file only in a
+	 * straightforward/clean manner :-
+	 * i)   Need the associated dentry pointer of the file, while opening the
+	 *      relay channel.
+	 * ii)  Should be able to use 'relay_file_operations' fops for the file.
+	 * iii) Set the 'i_private' field of file's inode to the pointer of
+	 *	relay channel buffer.
+	 */
+	if (!log_dir) {
+		DRM_ERROR("Debugfs dir not available yet for GuC log file\n");
+		return -ENODEV;
+	}
+
+	ret = relay_late_setup_files(guc->log.relay_chan, "guc_log", log_dir);
+	if (ret) {
+		DRM_ERROR("Couldn't associate relay chan with file %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void guc_move_to_next_buf(struct intel_guc *guc)
+{
+	/* Make sure the updates made in the sub buffer are visible when
+	 * Consumer sees the following update to offset inside the sub buffer.
+	 */
+	smp_wmb();
+
+	/* All data has been written, so now move the offset of sub buffer. */
+	relay_reserve(guc->log.relay_chan, guc->log.vma->obj->base.size);
+
+	/* Switch to the next sub buffer */
+	relay_flush(guc->log.relay_chan);
+}
+
+static void *guc_get_write_buffer(struct intel_guc *guc)
+{
+	if (!guc->log.relay_chan)
+		return NULL;
+
+	/* Just get the base address of a new sub buffer and copy data into it
+	 * ourselves. NULL will be returned in no-overwrite mode, if all sub
+	 * buffers are full. Could have used the relay_write() to indirectly
+	 * copy the data, but that would have been bit convoluted, as we need to
+	 * write to only certain locations inside a sub buffer which cannot be
+	 * done without using relay_reserve() along with relay_write(). So its
+	 * better to use relay_reserve() alone.
+	 */
+	return relay_reserve(guc->log.relay_chan, 0);
+}
+
+static bool
+guc_check_log_buf_overflow(struct intel_guc *guc,
+			   enum guc_log_buffer_type type, unsigned int full_cnt)
+{
+	unsigned int prev_full_cnt = guc->log.prev_overflow_count[type];
+	bool overflow = false;
+
+	if (full_cnt != prev_full_cnt) {
+		overflow = true;
+
+		guc->log.prev_overflow_count[type] = full_cnt;
+		guc->log.total_overflow_count[type] += full_cnt - prev_full_cnt;
+
+		if (full_cnt < prev_full_cnt) {
+			/* buffer_full_cnt is a 4 bit counter */
+			guc->log.total_overflow_count[type] += 16;
+		}
+		DRM_ERROR_RATELIMITED("GuC log buffer overflow\n");
+	}
+
+	return overflow;
+}
+
+static unsigned int guc_get_log_buffer_size(enum guc_log_buffer_type type)
+{
+	switch (type) {
+	case GUC_ISR_LOG_BUFFER:
+		return (GUC_LOG_ISR_PAGES + 1) * PAGE_SIZE;
+	case GUC_DPC_LOG_BUFFER:
+		return (GUC_LOG_DPC_PAGES + 1) * PAGE_SIZE;
+	case GUC_CRASH_DUMP_LOG_BUFFER:
+		return (GUC_LOG_CRASH_PAGES + 1) * PAGE_SIZE;
+	default:
+		MISSING_CASE(type);
+	}
+
+	return 0;
+}
+
+static void guc_read_update_log_buffer(struct intel_guc *guc)
+{
+	unsigned int buffer_size, read_offset, write_offset, bytes_to_copy, full_cnt;
+	struct guc_log_buffer_state *log_buf_state, *log_buf_snapshot_state;
+	struct guc_log_buffer_state log_buf_state_local;
+	enum guc_log_buffer_type type;
+	void *src_data, *dst_data;
+	bool new_overflow;
+
+	if (WARN_ON(!guc->log.buf_addr))
+		return;
+
+	/* Get the pointer to shared GuC log buffer */
+	log_buf_state = src_data = guc->log.buf_addr;
+
+	/* Get the pointer to local buffer to store the logs */
+	log_buf_snapshot_state = dst_data = guc_get_write_buffer(guc);
+
+	/* Actual logs are present from the 2nd page */
+	src_data += PAGE_SIZE;
+	dst_data += PAGE_SIZE;
+
+	for (type = GUC_ISR_LOG_BUFFER; type < GUC_MAX_LOG_BUFFER; type++) {
+		/* Make a copy of the state structure, inside GuC log buffer
+		 * (which is uncached mapped), on the stack to avoid reading
+		 * from it multiple times.
+		 */
+		memcpy(&log_buf_state_local, log_buf_state,
+		       sizeof(struct guc_log_buffer_state));
+		buffer_size = guc_get_log_buffer_size(type);
+		read_offset = log_buf_state_local.read_ptr;
+		write_offset = log_buf_state_local.sampled_write_ptr;
+		full_cnt = log_buf_state_local.buffer_full_cnt;
+
+		/* Bookkeeping stuff */
+		guc->log.flush_count[type] += log_buf_state_local.flush_to_file;
+		new_overflow = guc_check_log_buf_overflow(guc, type, full_cnt);
+
+		/* Update the state of shared log buffer */
+		log_buf_state->read_ptr = write_offset;
+		log_buf_state->flush_to_file = 0;
+		log_buf_state++;
+
+		if (unlikely(!log_buf_snapshot_state))
+			continue;
+
+		/* First copy the state structure in snapshot buffer */
+		memcpy(log_buf_snapshot_state, &log_buf_state_local,
+		       sizeof(struct guc_log_buffer_state));
+
+		/* The write pointer could have been updated by GuC firmware,
+		 * after sending the flush interrupt to Host, for consistency
+		 * set write pointer value to same value of sampled_write_ptr
+		 * in the snapshot buffer.
+		 */
+		log_buf_snapshot_state->write_ptr = write_offset;
+		log_buf_snapshot_state++;
+
+		/* Now copy the actual logs. */
+		if (unlikely(new_overflow)) {
+			/* copy the whole buffer in case of overflow */
+			read_offset = 0;
+			write_offset = buffer_size;
+		} else if (unlikely((read_offset > buffer_size) ||
+				    (write_offset > buffer_size))) {
+			DRM_ERROR("invalid log buffer state\n");
+			/* copy whole buffer as offsets are unreliable */
+			read_offset = 0;
+			write_offset = buffer_size;
+		}
+
+		/* Just copy the newly written data */
+		if (read_offset > write_offset) {
+			i915_memcpy_from_wc(dst_data, src_data, write_offset);
+			bytes_to_copy = buffer_size - read_offset;
+		} else {
+			bytes_to_copy = write_offset - read_offset;
+		}
+		i915_memcpy_from_wc(dst_data + read_offset,
+				    src_data + read_offset, bytes_to_copy);
+
+		src_data += buffer_size;
+		dst_data += buffer_size;
+	}
+
+	if (log_buf_snapshot_state)
+		guc_move_to_next_buf(guc);
+	else {
+		/* Used rate limited to avoid deluge of messages, logs might be
+		 * getting consumed by User at a slow rate.
+		 */
+		DRM_ERROR_RATELIMITED("no sub-buffer to capture logs\n");
+		guc->log.capture_miss_count++;
+	}
+}
+
+static void guc_capture_logs_work(struct work_struct *work)
+{
+	struct drm_i915_private *dev_priv =
+		container_of(work, struct drm_i915_private, guc.log.flush_work);
+
+	i915_guc_capture_logs(dev_priv);
+}
+
+static void guc_log_cleanup(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+
+	/* First disable the flush interrupt */
+	gen9_disable_guc_interrupts(dev_priv);
+
+	if (guc->log.flush_wq)
+		destroy_workqueue(guc->log.flush_wq);
+
+	guc->log.flush_wq = NULL;
+
+	if (guc->log.relay_chan)
+		guc_log_remove_relay_file(guc);
+
+	guc->log.relay_chan = NULL;
+
+	if (guc->log.buf_addr)
+		i915_gem_object_unpin_map(guc->log.vma->obj);
+
+	guc->log.buf_addr = NULL;
+}
+
+static int guc_log_create_extras(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	void *vaddr;
+	int ret;
+
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+
+	/* Nothing to do */
+	if (i915.guc_log_level < 0)
+		return 0;
+
+	if (!guc->log.buf_addr) {
+		/* Create a WC (Uncached for read) vmalloc mapping of log
+		 * buffer pages, so that we can directly get the data
+		 * (up-to-date) from memory.
+		 */
+		vaddr = i915_gem_object_pin_map(guc->log.vma->obj, I915_MAP_WC);
+		if (IS_ERR(vaddr)) {
+			ret = PTR_ERR(vaddr);
+			DRM_ERROR("Couldn't map log buffer pages %d\n", ret);
+			return ret;
+		}
+
+		guc->log.buf_addr = vaddr;
+	}
+
+	if (!guc->log.relay_chan) {
+		/* Create a relay channel, so that we have buffers for storing
+		 * the GuC firmware logs, the channel will be linked with a file
+		 * later on when debugfs is registered.
+		 */
+		ret = guc_log_create_relay_channel(guc);
+		if (ret)
+			return ret;
+	}
+
+	if (!guc->log.flush_wq) {
+		INIT_WORK(&guc->log.flush_work, guc_capture_logs_work);
+
+		 /*
+		 * GuC log buffer flush work item has to do register access to
+		 * send the ack to GuC and this work item, if not synced before
+		 * suspend, can potentially get executed after the GFX device is
+		 * suspended.
+		 * By marking the WQ as freezable, we don't have to bother about
+		 * flushing of this work item from the suspend hooks, the pending
+		 * work item if any will be either executed before the suspend
+		 * or scheduled later on resume. This way the handling of work
+		 * item can be kept same between system suspend & rpm suspend.
+		 */
+		guc->log.flush_wq = alloc_ordered_workqueue("i915-guc_log",
+							    WQ_HIGHPRI | WQ_FREEZABLE);
+		if (guc->log.flush_wq == NULL) {
+			DRM_ERROR("Couldn't allocate the wq for GuC logging\n");
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
 static void guc_log_create(struct intel_guc *guc)
 {
 	struct i915_vma *vma;
 	unsigned long offset;
 	uint32_t size, flags;
 
-	if (i915.guc_log_level < GUC_LOG_VERBOSITY_MIN)
-		return;
-
 	if (i915.guc_log_level > GUC_LOG_VERBOSITY_MAX)
 		i915.guc_log_level = GUC_LOG_VERBOSITY_MAX;
 
@@ -865,8 +1312,18 @@ static void guc_log_create(struct intel_guc *guc)
 		GUC_LOG_ISR_PAGES + 1 +
 		GUC_LOG_CRASH_PAGES + 1) << PAGE_SHIFT;
 
-	vma = guc->log_vma;
+	vma = guc->log.vma;
 	if (!vma) {
+		/* We require SSE 4.1 for fast reads from the GuC log buffer and
+		 * it should be present on the chipsets supporting GuC based
+		 * submisssions.
+		 */
+		if (WARN_ON(!i915_memcpy_from_wc(NULL, NULL, 0))) {
+			/* logging will not be enabled */
+			i915.guc_log_level = -1;
+			return;
+		}
+
 		vma = guc_allocate_vma(guc, size);
 		if (IS_ERR(vma)) {
 			/* logging will be off */
@@ -874,7 +1331,14 @@ static void guc_log_create(struct intel_guc *guc)
 			return;
 		}
 
-		guc->log_vma = vma;
+		guc->log.vma = vma;
+
+		if (guc_log_create_extras(guc)) {
+			guc_log_cleanup(guc);
+			i915_vma_unpin_and_release(&guc->log.vma);
+			i915.guc_log_level = -1;
+			return;
+		}
 	}
 
 	/* each allocated unit is a page */
@@ -884,7 +1348,37 @@ static void guc_log_create(struct intel_guc *guc)
 		(GUC_LOG_CRASH_PAGES << GUC_LOG_CRASH_SHIFT);
 
 	offset = i915_ggtt_offset(vma) >> PAGE_SHIFT; /* in pages */
-	guc->log_flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags;
+	guc->log.flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags;
+}
+
+static int guc_log_late_setup(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	int ret;
+
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+
+	if (i915.guc_log_level < 0)
+		return -EINVAL;
+
+	/* If log_level was set as -1 at boot time, then setup needed to
+	 * handle log buffer flush interrupts would not have been done yet,
+	 * so do that now.
+	 */
+	ret = guc_log_create_extras(guc);
+	if (ret)
+		goto err;
+
+	ret = guc_log_create_relay_file(guc);
+	if (ret)
+		goto err;
+
+	return 0;
+err:
+	guc_log_cleanup(guc);
+	/* logging will remain off */
+	i915.guc_log_level = -1;
+	return ret;
 }
 
 static void guc_policies_init(struct guc_policies *policies)
@@ -917,6 +1411,7 @@ static void guc_addon_create(struct intel_guc *guc)
 	struct guc_policies *policies;
 	struct guc_mmio_reg_state *reg_state;
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	struct page *page;
 	u32 size;
 
@@ -944,10 +1439,10 @@ static void guc_addon_create(struct intel_guc *guc)
 	 * so its address won't change after we've told the GuC where
 	 * to find it.
 	 */
-	engine = &dev_priv->engine[RCS];
+	engine = dev_priv->engine[RCS];
 	ads->golden_context_lrca = engine->status_page.ggtt_offset;
 
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		ads->eng_state_size[engine->guc_id] = intel_lr_context_size(engine);
 
 	/* GuC scheduling policies */
@@ -960,7 +1455,7 @@ static void guc_addon_create(struct intel_guc *guc)
 	/* MMIO reg state */
 	reg_state = (void *)policies + sizeof(struct guc_policies);
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		reg_state->mmio_white_list[engine->guc_id].mmio_start =
 			engine->mmio_base + GUC_MMIO_WHITE_LIST_START;
 
@@ -1005,6 +1500,7 @@ int i915_guc_submission_init(struct drm_i915_private *dev_priv)
 
 	guc->ctx_pool_vma = vma;
 	ida_init(&guc->ctx_ids);
+	mutex_init(&guc->action_lock);
 	guc_log_create(guc);
 	guc_addon_create(guc);
 
@@ -1014,9 +1510,10 @@ int i915_guc_submission_init(struct drm_i915_private *dev_priv)
 int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc *guc = &dev_priv->guc;
+	struct drm_i915_gem_request *request;
 	struct i915_guc_client *client;
 	struct intel_engine_cs *engine;
-	struct drm_i915_gem_request *request;
+	enum intel_engine_id id;
 
 	/* client for execbuf submission */
 	client = guc_client_alloc(dev_priv,
@@ -1033,11 +1530,13 @@ int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
 	guc_init_doorbell_hw(guc);
 
 	/* Take over from manual control of ELSP (execlists) */
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		engine->submit_request = i915_guc_submit;
+		engine->schedule = NULL;
 
 		/* Replay the current set of previously submitted requests */
-		list_for_each_entry(request, &engine->request_list, link) {
+		list_for_each_entry(request,
+				    &engine->timeline->requests, link) {
 			client->wq_rsvd += sizeof(struct guc_wq_item);
 			if (i915_sw_fence_done(&request->submit))
 				i915_guc_submit(request);
@@ -1066,7 +1565,7 @@ void i915_guc_submission_fini(struct drm_i915_private *dev_priv)
 	struct intel_guc *guc = &dev_priv->guc;
 
 	i915_vma_unpin_and_release(&guc->ads_vma);
-	i915_vma_unpin_and_release(&guc->log_vma);
+	i915_vma_unpin_and_release(&guc->log.vma);
 
 	if (guc->ctx_pool_vma)
 		ida_destroy(&guc->ctx_ids);
@@ -1087,6 +1586,8 @@ int intel_guc_suspend(struct drm_device *dev)
 	if (guc->guc_fw.guc_fw_load_status != GUC_FIRMWARE_SUCCESS)
 		return 0;
 
+	gen9_disable_guc_interrupts(dev_priv);
+
 	ctx = dev_priv->kernel_context;
 
 	data[0] = HOST2GUC_ACTION_ENTER_S_STATE;
@@ -1113,6 +1614,9 @@ int intel_guc_resume(struct drm_device *dev)
 	if (guc->guc_fw.guc_fw_load_status != GUC_FIRMWARE_SUCCESS)
 		return 0;
 
+	if (i915.guc_log_level >= 0)
+		gen9_enable_guc_interrupts(dev_priv);
+
 	ctx = dev_priv->kernel_context;
 
 	data[0] = HOST2GUC_ACTION_EXIT_S_STATE;
@@ -1122,3 +1626,104 @@ int intel_guc_resume(struct drm_device *dev)
 
 	return host2guc_action(guc, data, ARRAY_SIZE(data));
 }
+
+void i915_guc_capture_logs(struct drm_i915_private *dev_priv)
+{
+	guc_read_update_log_buffer(&dev_priv->guc);
+
+	/* Generally device is expected to be active only at this
+	 * time, so get/put should be really quick.
+	 */
+	intel_runtime_pm_get(dev_priv);
+	host2guc_logbuffer_flush_complete(&dev_priv->guc);
+	intel_runtime_pm_put(dev_priv);
+}
+
+void i915_guc_flush_logs(struct drm_i915_private *dev_priv)
+{
+	if (!i915.enable_guc_submission || (i915.guc_log_level < 0))
+		return;
+
+	/* First disable the interrupts, will be renabled afterwards */
+	gen9_disable_guc_interrupts(dev_priv);
+
+	/* Before initiating the forceful flush, wait for any pending/ongoing
+	 * flush to complete otherwise forceful flush may not actually happen.
+	 */
+	flush_work(&dev_priv->guc.log.flush_work);
+
+	/* Ask GuC to update the log buffer state */
+	host2guc_force_logbuffer_flush(&dev_priv->guc);
+
+	/* GuC would have updated log buffer by now, so capture it */
+	i915_guc_capture_logs(dev_priv);
+}
+
+void i915_guc_unregister(struct drm_i915_private *dev_priv)
+{
+	if (!i915.enable_guc_submission)
+		return;
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	guc_log_cleanup(&dev_priv->guc);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+}
+
+void i915_guc_register(struct drm_i915_private *dev_priv)
+{
+	if (!i915.enable_guc_submission)
+		return;
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+	guc_log_late_setup(&dev_priv->guc);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+}
+
+int i915_guc_log_control(struct drm_i915_private *dev_priv, u64 control_val)
+{
+	union guc_log_control log_param;
+	int ret;
+
+	log_param.value = control_val;
+
+	if (log_param.verbosity < GUC_LOG_VERBOSITY_MIN ||
+	    log_param.verbosity > GUC_LOG_VERBOSITY_MAX)
+		return -EINVAL;
+
+	/* This combination doesn't make sense & won't have any effect */
+	if (!log_param.logging_enabled && (i915.guc_log_level < 0))
+		return 0;
+
+	ret = host2guc_logging_control(&dev_priv->guc, log_param.value);
+	if (ret < 0) {
+		DRM_DEBUG_DRIVER("host2guc action failed %d\n", ret);
+		return ret;
+	}
+
+	i915.guc_log_level = log_param.verbosity;
+
+	/* If log_level was set as -1 at boot time, then the relay channel file
+	 * wouldn't have been created by now and interrupts also would not have
+	 * been enabled.
+	 */
+	if (!dev_priv->guc.log.relay_chan) {
+		ret = guc_log_late_setup(&dev_priv->guc);
+		if (!ret)
+			gen9_enable_guc_interrupts(dev_priv);
+	} else if (!log_param.logging_enabled) {
+		/* Once logging is disabled, GuC won't generate logs & send an
+		 * interrupt. But there could be some data in the log buffer
+		 * which is yet to be captured. So request GuC to update the log
+		 * buffer state and then collect the left over logs.
+		 */
+		i915_guc_flush_logs(dev_priv);
+
+		/* As logging is disabled, update log level to reflect that */
+		i915.guc_log_level = -1;
+	} else {
+		/* In case interrupts were disabled, enable them now */
+		gen9_enable_guc_interrupts(dev_priv);
+	}
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 3fc286cd1157..07ca71cabb2b 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -170,6 +170,7 @@ static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv,
 } while (0)
 
 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
+static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
 
 /* For display hotplug interrupt */
 static inline void
@@ -303,18 +304,18 @@ static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
 
 	assert_spin_locked(&dev_priv->irq_lock);
 
-	new_val = dev_priv->pm_irq_mask;
+	new_val = dev_priv->pm_imr;
 	new_val &= ~interrupt_mask;
 	new_val |= (~enabled_irq_mask & interrupt_mask);
 
-	if (new_val != dev_priv->pm_irq_mask) {
-		dev_priv->pm_irq_mask = new_val;
-		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
+	if (new_val != dev_priv->pm_imr) {
+		dev_priv->pm_imr = new_val;
+		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_imr);
 		POSTING_READ(gen6_pm_imr(dev_priv));
 	}
 }
 
-void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
+void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
 {
 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 		return;
@@ -322,28 +323,54 @@ void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 	snb_update_pm_irq(dev_priv, mask, mask);
 }
 
-static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
-				  uint32_t mask)
+static void __gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
 {
 	snb_update_pm_irq(dev_priv, mask, 0);
 }
 
-void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
+void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
 {
 	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
 		return;
 
-	__gen6_disable_pm_irq(dev_priv, mask);
+	__gen6_mask_pm_irq(dev_priv, mask);
 }
 
-void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
+void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 reset_mask)
 {
 	i915_reg_t reg = gen6_pm_iir(dev_priv);
 
-	spin_lock_irq(&dev_priv->irq_lock);
-	I915_WRITE(reg, dev_priv->pm_rps_events);
-	I915_WRITE(reg, dev_priv->pm_rps_events);
+	assert_spin_locked(&dev_priv->irq_lock);
+
+	I915_WRITE(reg, reset_mask);
+	I915_WRITE(reg, reset_mask);
 	POSTING_READ(reg);
+}
+
+void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, u32 enable_mask)
+{
+	assert_spin_locked(&dev_priv->irq_lock);
+
+	dev_priv->pm_ier |= enable_mask;
+	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
+	gen6_unmask_pm_irq(dev_priv, enable_mask);
+	/* unmask_pm_irq provides an implicit barrier (POSTING_READ) */
+}
+
+void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_mask)
+{
+	assert_spin_locked(&dev_priv->irq_lock);
+
+	dev_priv->pm_ier &= ~disable_mask;
+	__gen6_mask_pm_irq(dev_priv, disable_mask);
+	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
+	/* though a barrier is missing here, but don't really need a one */
+}
+
+void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
+{
+	spin_lock_irq(&dev_priv->irq_lock);
+	gen6_reset_pm_iir(dev_priv, dev_priv->pm_rps_events);
 	dev_priv->rps.pm_iir = 0;
 	spin_unlock_irq(&dev_priv->irq_lock);
 }
@@ -357,8 +384,6 @@ void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
 	WARN_ON_ONCE(dev_priv->rps.pm_iir);
 	WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
 	dev_priv->rps.interrupts_enabled = true;
-	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
-				dev_priv->pm_rps_events);
 	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 
 	spin_unlock_irq(&dev_priv->irq_lock);
@@ -379,9 +404,7 @@ void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
 
 	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u));
 
-	__gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
-	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
-				~dev_priv->pm_rps_events);
+	gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 
 	spin_unlock_irq(&dev_priv->irq_lock);
 	synchronize_irq(dev_priv->drm.irq);
@@ -395,6 +418,38 @@ void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
 	gen6_reset_rps_interrupts(dev_priv);
 }
 
+void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv)
+{
+	spin_lock_irq(&dev_priv->irq_lock);
+	gen6_reset_pm_iir(dev_priv, dev_priv->pm_guc_events);
+	spin_unlock_irq(&dev_priv->irq_lock);
+}
+
+void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv)
+{
+	spin_lock_irq(&dev_priv->irq_lock);
+	if (!dev_priv->guc.interrupts_enabled) {
+		WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) &
+				       dev_priv->pm_guc_events);
+		dev_priv->guc.interrupts_enabled = true;
+		gen6_enable_pm_irq(dev_priv, dev_priv->pm_guc_events);
+	}
+	spin_unlock_irq(&dev_priv->irq_lock);
+}
+
+void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv)
+{
+	spin_lock_irq(&dev_priv->irq_lock);
+	dev_priv->guc.interrupts_enabled = false;
+
+	gen6_disable_pm_irq(dev_priv, dev_priv->pm_guc_events);
+
+	spin_unlock_irq(&dev_priv->irq_lock);
+	synchronize_irq(dev_priv->drm.irq);
+
+	gen9_reset_guc_interrupts(dev_priv);
+}
+
 /**
  * bdw_update_port_irq - update DE port interrupt
  * @dev_priv: driver private
@@ -670,8 +725,8 @@ static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	i915_reg_t high_frame, low_frame;
 	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
-	struct intel_crtc *intel_crtc =
-		to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
+	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
+								pipe);
 	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
 
 	htotal = mode->crtc_htotal;
@@ -776,8 +831,8 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
 				    const struct drm_display_mode *mode)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
+								pipe);
 	int position;
 	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
 	bool in_vbl = true;
@@ -912,21 +967,22 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
 			      struct timeval *vblank_time,
 			      unsigned flags)
 {
-	struct drm_crtc *crtc;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_crtc *crtc;
 
-	if (pipe >= INTEL_INFO(dev)->num_pipes) {
+	if (pipe >= INTEL_INFO(dev_priv)->num_pipes) {
 		DRM_ERROR("Invalid crtc %u\n", pipe);
 		return -EINVAL;
 	}
 
 	/* Get drm_crtc to timestamp: */
-	crtc = intel_get_crtc_for_pipe(dev, pipe);
+	crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 	if (crtc == NULL) {
 		DRM_ERROR("Invalid crtc %u\n", pipe);
 		return -EINVAL;
 	}
 
-	if (!crtc->hwmode.crtc_clock) {
+	if (!crtc->base.hwmode.crtc_clock) {
 		DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
 		return -EBUSY;
 	}
@@ -934,7 +990,7 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
 	/* Helper routine in DRM core does all the work: */
 	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
 						     vblank_time, flags,
-						     &crtc->hwmode);
+						     &crtc->base.hwmode);
 }
 
 static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
@@ -1058,8 +1114,9 @@ static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
 static bool any_waiters(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		if (intel_engine_has_waiter(engine))
 			return true;
 
@@ -1084,7 +1141,7 @@ static void gen6_pm_rps_work(struct work_struct *work)
 	pm_iir = dev_priv->rps.pm_iir;
 	dev_priv->rps.pm_iir = 0;
 	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
-	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
+	gen6_unmask_pm_irq(dev_priv, dev_priv->pm_rps_events);
 	client_boost = dev_priv->rps.client_boost;
 	dev_priv->rps.client_boost = false;
 	spin_unlock_irq(&dev_priv->irq_lock);
@@ -1257,20 +1314,20 @@ static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
 			       u32 gt_iir)
 {
 	if (gt_iir & GT_RENDER_USER_INTERRUPT)
-		notify_ring(&dev_priv->engine[RCS]);
+		notify_ring(dev_priv->engine[RCS]);
 	if (gt_iir & ILK_BSD_USER_INTERRUPT)
-		notify_ring(&dev_priv->engine[VCS]);
+		notify_ring(dev_priv->engine[VCS]);
 }
 
 static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
 			       u32 gt_iir)
 {
 	if (gt_iir & GT_RENDER_USER_INTERRUPT)
-		notify_ring(&dev_priv->engine[RCS]);
+		notify_ring(dev_priv->engine[RCS]);
 	if (gt_iir & GT_BSD_USER_INTERRUPT)
-		notify_ring(&dev_priv->engine[VCS]);
+		notify_ring(dev_priv->engine[VCS]);
 	if (gt_iir & GT_BLT_USER_INTERRUPT)
-		notify_ring(&dev_priv->engine[BCS]);
+		notify_ring(dev_priv->engine[BCS]);
 
 	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
 		      GT_BSD_CS_ERROR_INTERRUPT |
@@ -1323,11 +1380,13 @@ static irqreturn_t gen8_gt_irq_ack(struct drm_i915_private *dev_priv,
 			DRM_ERROR("The master control interrupt lied (GT3)!\n");
 	}
 
-	if (master_ctl & GEN8_GT_PM_IRQ) {
+	if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
 		gt_iir[2] = I915_READ_FW(GEN8_GT_IIR(2));
-		if (gt_iir[2] & dev_priv->pm_rps_events) {
+		if (gt_iir[2] & (dev_priv->pm_rps_events |
+				 dev_priv->pm_guc_events)) {
 			I915_WRITE_FW(GEN8_GT_IIR(2),
-				      gt_iir[2] & dev_priv->pm_rps_events);
+				      gt_iir[2] & (dev_priv->pm_rps_events |
+						   dev_priv->pm_guc_events));
 			ret = IRQ_HANDLED;
 		} else
 			DRM_ERROR("The master control interrupt lied (PM)!\n");
@@ -1340,25 +1399,28 @@ static void gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
 				u32 gt_iir[4])
 {
 	if (gt_iir[0]) {
-		gen8_cs_irq_handler(&dev_priv->engine[RCS],
+		gen8_cs_irq_handler(dev_priv->engine[RCS],
 				    gt_iir[0], GEN8_RCS_IRQ_SHIFT);
-		gen8_cs_irq_handler(&dev_priv->engine[BCS],
+		gen8_cs_irq_handler(dev_priv->engine[BCS],
 				    gt_iir[0], GEN8_BCS_IRQ_SHIFT);
 	}
 
 	if (gt_iir[1]) {
-		gen8_cs_irq_handler(&dev_priv->engine[VCS],
+		gen8_cs_irq_handler(dev_priv->engine[VCS],
 				    gt_iir[1], GEN8_VCS1_IRQ_SHIFT);
-		gen8_cs_irq_handler(&dev_priv->engine[VCS2],
+		gen8_cs_irq_handler(dev_priv->engine[VCS2],
 				    gt_iir[1], GEN8_VCS2_IRQ_SHIFT);
 	}
 
 	if (gt_iir[3])
-		gen8_cs_irq_handler(&dev_priv->engine[VECS],
+		gen8_cs_irq_handler(dev_priv->engine[VECS],
 				    gt_iir[3], GEN8_VECS_IRQ_SHIFT);
 
 	if (gt_iir[2] & dev_priv->pm_rps_events)
 		gen6_rps_irq_handler(dev_priv, gt_iir[2]);
+
+	if (gt_iir[2] & dev_priv->pm_guc_events)
+		gen9_guc_irq_handler(dev_priv, gt_iir[2]);
 }
 
 static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
@@ -1585,7 +1647,7 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
 {
 	if (pm_iir & dev_priv->pm_rps_events) {
 		spin_lock(&dev_priv->irq_lock);
-		gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
+		gen6_mask_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
 		if (dev_priv->rps.interrupts_enabled) {
 			dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
 			schedule_work(&dev_priv->rps.work);
@@ -1598,13 +1660,48 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
 
 	if (HAS_VEBOX(dev_priv)) {
 		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
-			notify_ring(&dev_priv->engine[VECS]);
+			notify_ring(dev_priv->engine[VECS]);
 
 		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
 			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
 	}
 }
 
+static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir)
+{
+	if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT) {
+		/* Sample the log buffer flush related bits & clear them out now
+		 * itself from the message identity register to minimize the
+		 * probability of losing a flush interrupt, when there are back
+		 * to back flush interrupts.
+		 * There can be a new flush interrupt, for different log buffer
+		 * type (like for ISR), whilst Host is handling one (for DPC).
+		 * Since same bit is used in message register for ISR & DPC, it
+		 * could happen that GuC sets the bit for 2nd interrupt but Host
+		 * clears out the bit on handling the 1st interrupt.
+		 */
+		u32 msg, flush;
+
+		msg = I915_READ(SOFT_SCRATCH(15));
+		flush = msg & (GUC2HOST_MSG_CRASH_DUMP_POSTED |
+			       GUC2HOST_MSG_FLUSH_LOG_BUFFER);
+		if (flush) {
+			/* Clear the message bits that are handled */
+			I915_WRITE(SOFT_SCRATCH(15), msg & ~flush);
+
+			/* Handle flush interrupt in bottom half */
+			queue_work(dev_priv->guc.log.flush_wq,
+				   &dev_priv->guc.log.flush_work);
+
+			dev_priv->guc.log.flush_interrupt_count++;
+		} else {
+			/* Not clearing of unhandled event bits won't result in
+			 * re-triggering of the interrupt.
+			 */
+		}
+	}
+}
+
 static bool intel_pipe_handle_vblank(struct drm_i915_private *dev_priv,
 				     enum pipe pipe)
 {
@@ -2407,7 +2504,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 			fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
 
 		if (fault_errors)
-			DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
+			DRM_ERROR("Fault errors on pipe %c: 0x%08x\n",
 				  pipe_name(pipe),
 				  fault_errors);
 	}
@@ -2551,92 +2648,52 @@ static void i915_reset_and_wakeup(struct drm_i915_private *dev_priv)
 	wake_up_all(&dev_priv->gpu_error.reset_queue);
 }
 
-static void i915_report_and_clear_eir(struct drm_i915_private *dev_priv)
+static inline void
+i915_err_print_instdone(struct drm_i915_private *dev_priv,
+			struct intel_instdone *instdone)
 {
-	uint32_t instdone[I915_NUM_INSTDONE_REG];
-	u32 eir = I915_READ(EIR);
-	int pipe, i;
+	int slice;
+	int subslice;
+
+	pr_err("  INSTDONE: 0x%08x\n", instdone->instdone);
+
+	if (INTEL_GEN(dev_priv) <= 3)
+		return;
+
+	pr_err("  SC_INSTDONE: 0x%08x\n", instdone->slice_common);
 
-	if (!eir)
+	if (INTEL_GEN(dev_priv) <= 6)
 		return;
 
-	pr_err("render error detected, EIR: 0x%08x\n", eir);
+	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
+		pr_err("  SAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
+		       slice, subslice, instdone->sampler[slice][subslice]);
 
-	i915_get_extra_instdone(dev_priv, instdone);
+	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
+		pr_err("  ROW_INSTDONE[%d][%d]: 0x%08x\n",
+		       slice, subslice, instdone->row[slice][subslice]);
+}
 
-	if (IS_G4X(dev_priv)) {
-		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
-			u32 ipeir = I915_READ(IPEIR_I965);
-
-			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
-			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
-			for (i = 0; i < ARRAY_SIZE(instdone); i++)
-				pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
-			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
-			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
-			I915_WRITE(IPEIR_I965, ipeir);
-			POSTING_READ(IPEIR_I965);
-		}
-		if (eir & GM45_ERROR_PAGE_TABLE) {
-			u32 pgtbl_err = I915_READ(PGTBL_ER);
-			pr_err("page table error\n");
-			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
-			I915_WRITE(PGTBL_ER, pgtbl_err);
-			POSTING_READ(PGTBL_ER);
-		}
-	}
+static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
+{
+	u32 eir;
 
-	if (!IS_GEN2(dev_priv)) {
-		if (eir & I915_ERROR_PAGE_TABLE) {
-			u32 pgtbl_err = I915_READ(PGTBL_ER);
-			pr_err("page table error\n");
-			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
-			I915_WRITE(PGTBL_ER, pgtbl_err);
-			POSTING_READ(PGTBL_ER);
-		}
-	}
+	if (!IS_GEN2(dev_priv))
+		I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER));
 
-	if (eir & I915_ERROR_MEMORY_REFRESH) {
-		pr_err("memory refresh error:\n");
-		for_each_pipe(dev_priv, pipe)
-			pr_err("pipe %c stat: 0x%08x\n",
-			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
-		/* pipestat has already been acked */
-	}
-	if (eir & I915_ERROR_INSTRUCTION) {
-		pr_err("instruction error\n");
-		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
-		for (i = 0; i < ARRAY_SIZE(instdone); i++)
-			pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
-		if (INTEL_GEN(dev_priv) < 4) {
-			u32 ipeir = I915_READ(IPEIR);
-
-			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
-			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
-			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
-			I915_WRITE(IPEIR, ipeir);
-			POSTING_READ(IPEIR);
-		} else {
-			u32 ipeir = I915_READ(IPEIR_I965);
-
-			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
-			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
-			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
-			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
-			I915_WRITE(IPEIR_I965, ipeir);
-			POSTING_READ(IPEIR_I965);
-		}
-	}
+	if (INTEL_GEN(dev_priv) < 4)
+		I915_WRITE(IPEIR, I915_READ(IPEIR));
+	else
+		I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965));
 
-	I915_WRITE(EIR, eir);
-	POSTING_READ(EIR);
+	I915_WRITE(EIR, I915_READ(EIR));
 	eir = I915_READ(EIR);
 	if (eir) {
 		/*
 		 * some errors might have become stuck,
 		 * mask them.
 		 */
-		DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
+		DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
 		I915_WRITE(EMR, I915_READ(EMR) | eir);
 		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
 	}
@@ -2665,7 +2722,7 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
 	va_end(args);
 
 	i915_capture_error_state(dev_priv, engine_mask, error_msg);
-	i915_report_and_clear_eir(dev_priv);
+	i915_clear_error_registers(dev_priv);
 
 	if (!engine_mask)
 		return;
@@ -2694,45 +2751,40 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
 /* Called from drm generic code, passed 'crtc' which
  * we use as a pipe index
  */
-static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
+static int i8xx_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	unsigned long irqflags;
 
 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-	if (INTEL_INFO(dev)->gen >= 4)
-		i915_enable_pipestat(dev_priv, pipe,
-				     PIPE_START_VBLANK_INTERRUPT_STATUS);
-	else
-		i915_enable_pipestat(dev_priv, pipe,
-				     PIPE_VBLANK_INTERRUPT_STATUS);
+	i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
 	return 0;
 }
 
-static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
+static int i965_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	unsigned long irqflags;
-	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
-						     DE_PIPE_VBLANK(pipe);
 
 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-	ilk_enable_display_irq(dev_priv, bit);
+	i915_enable_pipestat(dev_priv, pipe,
+			     PIPE_START_VBLANK_INTERRUPT_STATUS);
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
 	return 0;
 }
 
-static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
+static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	unsigned long irqflags;
+	uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
+		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
 
 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-	i915_enable_pipestat(dev_priv, pipe,
-			     PIPE_START_VBLANK_INTERRUPT_STATUS);
+	ilk_enable_display_irq(dev_priv, bit);
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
 	return 0;
@@ -2753,38 +2805,36 @@ static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
 /* Called from drm generic code, passed 'crtc' which
  * we use as a pipe index
  */
-static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
+static void i8xx_disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	unsigned long irqflags;
 
 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-	i915_disable_pipestat(dev_priv, pipe,
-			      PIPE_VBLANK_INTERRUPT_STATUS |
-			      PIPE_START_VBLANK_INTERRUPT_STATUS);
+	i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
-static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
+static void i965_disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	unsigned long irqflags;
-	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
-						     DE_PIPE_VBLANK(pipe);
 
 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-	ilk_disable_display_irq(dev_priv, bit);
+	i915_disable_pipestat(dev_priv, pipe,
+			      PIPE_START_VBLANK_INTERRUPT_STATUS);
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
-static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
+static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	unsigned long irqflags;
+	uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
+		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
 
 	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-	i915_disable_pipestat(dev_priv, pipe,
-			      PIPE_START_VBLANK_INTERRUPT_STATUS);
+	ilk_disable_display_irq(dev_priv, bit);
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
@@ -2798,411 +2848,14 @@ static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
-static bool
-ipehr_is_semaphore_wait(struct intel_engine_cs *engine, u32 ipehr)
-{
-	if (INTEL_GEN(engine->i915) >= 8) {
-		return (ipehr >> 23) == 0x1c;
-	} else {
-		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
-		return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
-				 MI_SEMAPHORE_REGISTER);
-	}
-}
-
-static struct intel_engine_cs *
-semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr,
-				 u64 offset)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	struct intel_engine_cs *signaller;
-
-	if (INTEL_GEN(dev_priv) >= 8) {
-		for_each_engine(signaller, dev_priv) {
-			if (engine == signaller)
-				continue;
-
-			if (offset == signaller->semaphore.signal_ggtt[engine->hw_id])
-				return signaller;
-		}
-	} else {
-		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
-
-		for_each_engine(signaller, dev_priv) {
-			if(engine == signaller)
-				continue;
-
-			if (sync_bits == signaller->semaphore.mbox.wait[engine->hw_id])
-				return signaller;
-		}
-	}
-
-	DRM_DEBUG_DRIVER("No signaller ring found for %s, ipehr 0x%08x, offset 0x%016llx\n",
-			 engine->name, ipehr, offset);
-
-	return ERR_PTR(-ENODEV);
-}
-
-static struct intel_engine_cs *
-semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
+static void ibx_irq_reset(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-	void __iomem *vaddr;
-	u32 cmd, ipehr, head;
-	u64 offset = 0;
-	int i, backwards;
-
-	/*
-	 * This function does not support execlist mode - any attempt to
-	 * proceed further into this function will result in a kernel panic
-	 * when dereferencing ring->buffer, which is not set up in execlist
-	 * mode.
-	 *
-	 * The correct way of doing it would be to derive the currently
-	 * executing ring buffer from the current context, which is derived
-	 * from the currently running request. Unfortunately, to get the
-	 * current request we would have to grab the struct_mutex before doing
-	 * anything else, which would be ill-advised since some other thread
-	 * might have grabbed it already and managed to hang itself, causing
-	 * the hang checker to deadlock.
-	 *
-	 * Therefore, this function does not support execlist mode in its
-	 * current form. Just return NULL and move on.
-	 */
-	if (engine->buffer == NULL)
-		return NULL;
-
-	ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
-	if (!ipehr_is_semaphore_wait(engine, ipehr))
-		return NULL;
-
-	/*
-	 * HEAD is likely pointing to the dword after the actual command,
-	 * so scan backwards until we find the MBOX. But limit it to just 3
-	 * or 4 dwords depending on the semaphore wait command size.
-	 * Note that we don't care about ACTHD here since that might
-	 * point at at batch, and semaphores are always emitted into the
-	 * ringbuffer itself.
-	 */
-	head = I915_READ_HEAD(engine) & HEAD_ADDR;
-	backwards = (INTEL_GEN(dev_priv) >= 8) ? 5 : 4;
-	vaddr = (void __iomem *)engine->buffer->vaddr;
-
-	for (i = backwards; i; --i) {
-		/*
-		 * Be paranoid and presume the hw has gone off into the wild -
-		 * our ring is smaller than what the hardware (and hence
-		 * HEAD_ADDR) allows. Also handles wrap-around.
-		 */
-		head &= engine->buffer->size - 1;
-
-		/* This here seems to blow up */
-		cmd = ioread32(vaddr + head);
-		if (cmd == ipehr)
-			break;
-
-		head -= 4;
-	}
-
-	if (!i)
-		return NULL;
-
-	*seqno = ioread32(vaddr + head + 4) + 1;
-	if (INTEL_GEN(dev_priv) >= 8) {
-		offset = ioread32(vaddr + head + 12);
-		offset <<= 32;
-		offset |= ioread32(vaddr + head + 8);
-	}
-	return semaphore_wait_to_signaller_ring(engine, ipehr, offset);
-}
-
-static int semaphore_passed(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	struct intel_engine_cs *signaller;
-	u32 seqno;
-
-	engine->hangcheck.deadlock++;
-
-	signaller = semaphore_waits_for(engine, &seqno);
-	if (signaller == NULL)
-		return -1;
-
-	if (IS_ERR(signaller))
-		return 0;
-
-	/* Prevent pathological recursion due to driver bugs */
-	if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES)
-		return -1;
-
-	if (i915_seqno_passed(intel_engine_get_seqno(signaller), seqno))
-		return 1;
-
-	/* cursory check for an unkickable deadlock */
-	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
-	    semaphore_passed(signaller) < 0)
-		return -1;
-
-	return 0;
-}
-
-static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
-{
-	struct intel_engine_cs *engine;
-
-	for_each_engine(engine, dev_priv)
-		engine->hangcheck.deadlock = 0;
-}
-
-static bool subunits_stuck(struct intel_engine_cs *engine)
-{
-	u32 instdone[I915_NUM_INSTDONE_REG];
-	bool stuck;
-	int i;
-
-	if (engine->id != RCS)
-		return true;
-
-	i915_get_extra_instdone(engine->i915, instdone);
-
-	/* There might be unstable subunit states even when
-	 * actual head is not moving. Filter out the unstable ones by
-	 * accumulating the undone -> done transitions and only
-	 * consider those as progress.
-	 */
-	stuck = true;
-	for (i = 0; i < I915_NUM_INSTDONE_REG; i++) {
-		const u32 tmp = instdone[i] | engine->hangcheck.instdone[i];
-
-		if (tmp != engine->hangcheck.instdone[i])
-			stuck = false;
-
-		engine->hangcheck.instdone[i] |= tmp;
-	}
-
-	return stuck;
-}
-
-static enum intel_engine_hangcheck_action
-head_stuck(struct intel_engine_cs *engine, u64 acthd)
-{
-	if (acthd != engine->hangcheck.acthd) {
-
-		/* Clear subunit states on head movement */
-		memset(engine->hangcheck.instdone, 0,
-		       sizeof(engine->hangcheck.instdone));
-
-		return HANGCHECK_ACTIVE;
-	}
-
-	if (!subunits_stuck(engine))
-		return HANGCHECK_ACTIVE;
-
-	return HANGCHECK_HUNG;
-}
-
-static enum intel_engine_hangcheck_action
-engine_stuck(struct intel_engine_cs *engine, u64 acthd)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	enum intel_engine_hangcheck_action ha;
-	u32 tmp;
-
-	ha = head_stuck(engine, acthd);
-	if (ha != HANGCHECK_HUNG)
-		return ha;
-
-	if (IS_GEN2(dev_priv))
-		return HANGCHECK_HUNG;
-
-	/* Is the chip hanging on a WAIT_FOR_EVENT?
-	 * If so we can simply poke the RB_WAIT bit
-	 * and break the hang. This should work on
-	 * all but the second generation chipsets.
-	 */
-	tmp = I915_READ_CTL(engine);
-	if (tmp & RING_WAIT) {
-		i915_handle_error(dev_priv, 0,
-				  "Kicking stuck wait on %s",
-				  engine->name);
-		I915_WRITE_CTL(engine, tmp);
-		return HANGCHECK_KICK;
-	}
-
-	if (INTEL_GEN(dev_priv) >= 6 && tmp & RING_WAIT_SEMAPHORE) {
-		switch (semaphore_passed(engine)) {
-		default:
-			return HANGCHECK_HUNG;
-		case 1:
-			i915_handle_error(dev_priv, 0,
-					  "Kicking stuck semaphore on %s",
-					  engine->name);
-			I915_WRITE_CTL(engine, tmp);
-			return HANGCHECK_KICK;
-		case 0:
-			return HANGCHECK_WAIT;
-		}
-	}
-
-	return HANGCHECK_HUNG;
-}
-
-/*
- * This is called when the chip hasn't reported back with completed
- * batchbuffers in a long time. We keep track per ring seqno progress and
- * if there are no progress, hangcheck score for that ring is increased.
- * Further, acthd is inspected to see if the ring is stuck. On stuck case
- * we kick the ring. If we see no progress on three subsequent calls
- * we assume chip is wedged and try to fix it by resetting the chip.
- */
-static void i915_hangcheck_elapsed(struct work_struct *work)
-{
-	struct drm_i915_private *dev_priv =
-		container_of(work, typeof(*dev_priv),
-			     gpu_error.hangcheck_work.work);
-	struct intel_engine_cs *engine;
-	unsigned int hung = 0, stuck = 0;
-	int busy_count = 0;
-#define BUSY 1
-#define KICK 5
-#define HUNG 20
-#define ACTIVE_DECAY 15
-
-	if (!i915.enable_hangcheck)
-		return;
-
-	if (!READ_ONCE(dev_priv->gt.awake))
-		return;
-
-	/* As enabling the GPU requires fairly extensive mmio access,
-	 * periodically arm the mmio checker to see if we are triggering
-	 * any invalid access.
-	 */
-	intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
-
-	for_each_engine(engine, dev_priv) {
-		bool busy = intel_engine_has_waiter(engine);
-		u64 acthd;
-		u32 seqno;
-		u32 submit;
-
-		semaphore_clear_deadlocks(dev_priv);
-
-		/* We don't strictly need an irq-barrier here, as we are not
-		 * serving an interrupt request, be paranoid in case the
-		 * barrier has side-effects (such as preventing a broken
-		 * cacheline snoop) and so be sure that we can see the seqno
-		 * advance. If the seqno should stick, due to a stale
-		 * cacheline, we would erroneously declare the GPU hung.
-		 */
-		if (engine->irq_seqno_barrier)
-			engine->irq_seqno_barrier(engine);
-
-		acthd = intel_engine_get_active_head(engine);
-		seqno = intel_engine_get_seqno(engine);
-		submit = READ_ONCE(engine->last_submitted_seqno);
-
-		if (engine->hangcheck.seqno == seqno) {
-			if (i915_seqno_passed(seqno, submit)) {
-				engine->hangcheck.action = HANGCHECK_IDLE;
-			} else {
-				/* We always increment the hangcheck score
-				 * if the engine is busy and still processing
-				 * the same request, so that no single request
-				 * can run indefinitely (such as a chain of
-				 * batches). The only time we do not increment
-				 * the hangcheck score on this ring, if this
-				 * engine is in a legitimate wait for another
-				 * engine. In that case the waiting engine is a
-				 * victim and we want to be sure we catch the
-				 * right culprit. Then every time we do kick
-				 * the ring, add a small increment to the
-				 * score so that we can catch a batch that is
-				 * being repeatedly kicked and so responsible
-				 * for stalling the machine.
-				 */
-				engine->hangcheck.action =
-					engine_stuck(engine, acthd);
-
-				switch (engine->hangcheck.action) {
-				case HANGCHECK_IDLE:
-				case HANGCHECK_WAIT:
-					break;
-				case HANGCHECK_ACTIVE:
-					engine->hangcheck.score += BUSY;
-					break;
-				case HANGCHECK_KICK:
-					engine->hangcheck.score += KICK;
-					break;
-				case HANGCHECK_HUNG:
-					engine->hangcheck.score += HUNG;
-					break;
-				}
-			}
-
-			if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
-				hung |= intel_engine_flag(engine);
-				if (engine->hangcheck.action != HANGCHECK_HUNG)
-					stuck |= intel_engine_flag(engine);
-			}
-		} else {
-			engine->hangcheck.action = HANGCHECK_ACTIVE;
-
-			/* Gradually reduce the count so that we catch DoS
-			 * attempts across multiple batches.
-			 */
-			if (engine->hangcheck.score > 0)
-				engine->hangcheck.score -= ACTIVE_DECAY;
-			if (engine->hangcheck.score < 0)
-				engine->hangcheck.score = 0;
-
-			/* Clear head and subunit states on seqno movement */
-			acthd = 0;
-
-			memset(engine->hangcheck.instdone, 0,
-			       sizeof(engine->hangcheck.instdone));
-		}
-
-		engine->hangcheck.seqno = seqno;
-		engine->hangcheck.acthd = acthd;
-		busy_count += busy;
-	}
-
-	if (hung) {
-		char msg[80];
-		unsigned int tmp;
-		int len;
-
-		/* If some rings hung but others were still busy, only
-		 * blame the hanging rings in the synopsis.
-		 */
-		if (stuck != hung)
-			hung &= ~stuck;
-		len = scnprintf(msg, sizeof(msg),
-				"%s on ", stuck == hung ? "No progress" : "Hang");
-		for_each_engine_masked(engine, dev_priv, hung, tmp)
-			len += scnprintf(msg + len, sizeof(msg) - len,
-					 "%s, ", engine->name);
-		msg[len-2] = '\0';
-
-		return i915_handle_error(dev_priv, hung, msg);
-	}
-
-	/* Reset timer in case GPU hangs without another request being added */
-	if (busy_count)
-		i915_queue_hangcheck(dev_priv);
-}
-
-static void ibx_irq_reset(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	if (HAS_PCH_NOP(dev))
+	if (HAS_PCH_NOP(dev_priv))
 		return;
 
 	GEN5_IRQ_RESET(SDE);
 
-	if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
+	if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
 		I915_WRITE(SERR_INT, 0xffffffff);
 }
 
@@ -3218,7 +2871,7 @@ static void ibx_irq_pre_postinstall(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
-	if (HAS_PCH_NOP(dev))
+	if (HAS_PCH_NOP(dev_priv))
 		return;
 
 	WARN_ON(I915_READ(SDEIER) != 0);
@@ -3226,12 +2879,10 @@ static void ibx_irq_pre_postinstall(struct drm_device *dev)
 	POSTING_READ(SDEIER);
 }
 
-static void gen5_gt_irq_reset(struct drm_device *dev)
+static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	GEN5_IRQ_RESET(GT);
-	if (INTEL_INFO(dev)->gen >= 6)
+	if (INTEL_GEN(dev_priv) >= 6)
 		GEN5_IRQ_RESET(GEN6_PM);
 }
 
@@ -3293,12 +2944,12 @@ static void ironlake_irq_reset(struct drm_device *dev)
 	I915_WRITE(HWSTAM, 0xffffffff);
 
 	GEN5_IRQ_RESET(DE);
-	if (IS_GEN7(dev))
+	if (IS_GEN7(dev_priv))
 		I915_WRITE(GEN7_ERR_INT, 0xffffffff);
 
-	gen5_gt_irq_reset(dev);
+	gen5_gt_irq_reset(dev_priv);
 
-	ibx_irq_reset(dev);
+	ibx_irq_reset(dev_priv);
 }
 
 static void valleyview_irq_preinstall(struct drm_device *dev)
@@ -3308,7 +2959,7 @@ static void valleyview_irq_preinstall(struct drm_device *dev)
 	I915_WRITE(VLV_MASTER_IER, 0);
 	POSTING_READ(VLV_MASTER_IER);
 
-	gen5_gt_irq_reset(dev);
+	gen5_gt_irq_reset(dev_priv);
 
 	spin_lock_irq(&dev_priv->irq_lock);
 	if (dev_priv->display_irqs_enabled)
@@ -3343,8 +2994,8 @@ static void gen8_irq_reset(struct drm_device *dev)
 	GEN5_IRQ_RESET(GEN8_DE_MISC_);
 	GEN5_IRQ_RESET(GEN8_PCU_);
 
-	if (HAS_PCH_SPLIT(dev))
-		ibx_irq_reset(dev);
+	if (HAS_PCH_SPLIT(dev_priv))
+		ibx_irq_reset(dev_priv);
 }
 
 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
@@ -3532,10 +3183,10 @@ static void ibx_irq_postinstall(struct drm_device *dev)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 mask;
 
-	if (HAS_PCH_NOP(dev))
+	if (HAS_PCH_NOP(dev_priv))
 		return;
 
-	if (HAS_PCH_IBX(dev))
+	if (HAS_PCH_IBX(dev_priv))
 		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
 	else
 		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
@@ -3552,14 +3203,14 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev)
 	pm_irqs = gt_irqs = 0;
 
 	dev_priv->gt_irq_mask = ~0;
-	if (HAS_L3_DPF(dev)) {
+	if (HAS_L3_DPF(dev_priv)) {
 		/* L3 parity interrupt is always unmasked. */
-		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
-		gt_irqs |= GT_PARITY_ERROR(dev);
+		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev_priv);
+		gt_irqs |= GT_PARITY_ERROR(dev_priv);
 	}
 
 	gt_irqs |= GT_RENDER_USER_INTERRUPT;
-	if (IS_GEN5(dev)) {
+	if (IS_GEN5(dev_priv)) {
 		gt_irqs |= ILK_BSD_USER_INTERRUPT;
 	} else {
 		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
@@ -3567,16 +3218,18 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev)
 
 	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
 
-	if (INTEL_INFO(dev)->gen >= 6) {
+	if (INTEL_GEN(dev_priv) >= 6) {
 		/*
 		 * RPS interrupts will get enabled/disabled on demand when RPS
 		 * itself is enabled/disabled.
 		 */
-		if (HAS_VEBOX(dev))
+		if (HAS_VEBOX(dev_priv)) {
 			pm_irqs |= PM_VEBOX_USER_INTERRUPT;
+			dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT;
+		}
 
-		dev_priv->pm_irq_mask = 0xffffffff;
-		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
+		dev_priv->pm_imr = 0xffffffff;
+		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs);
 	}
 }
 
@@ -3585,7 +3238,7 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 display_mask, extra_mask;
 
-	if (INTEL_INFO(dev)->gen >= 7) {
+	if (INTEL_GEN(dev_priv) >= 7) {
 		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
 				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
 				DE_PLANEB_FLIP_DONE_IVB |
@@ -3616,7 +3269,7 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
 
 	ibx_irq_postinstall(dev);
 
-	if (IS_IRONLAKE_M(dev)) {
+	if (IS_IRONLAKE_M(dev_priv)) {
 		/* Enable PCU event interrupts
 		 *
 		 * spinlocking not required here for correctness since interrupt
@@ -3696,14 +3349,15 @@ static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
 	if (HAS_L3_DPF(dev_priv))
 		gt_interrupts[0] |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
 
-	dev_priv->pm_irq_mask = 0xffffffff;
+	dev_priv->pm_ier = 0x0;
+	dev_priv->pm_imr = ~dev_priv->pm_ier;
 	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
 	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
 	/*
 	 * RPS interrupts will get enabled/disabled on demand when RPS itself
-	 * is enabled/disabled.
+	 * is enabled/disabled. Same wil be the case for GuC interrupts.
 	 */
-	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
+	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_imr, dev_priv->pm_ier);
 	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
 }
 
@@ -3756,13 +3410,13 @@ static int gen8_irq_postinstall(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
-	if (HAS_PCH_SPLIT(dev))
+	if (HAS_PCH_SPLIT(dev_priv))
 		ibx_irq_pre_postinstall(dev);
 
 	gen8_gt_irq_postinstall(dev_priv);
 	gen8_de_irq_postinstall(dev_priv);
 
-	if (HAS_PCH_SPLIT(dev))
+	if (HAS_PCH_SPLIT(dev_priv))
 		ibx_irq_postinstall(dev);
 
 	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
@@ -3808,7 +3462,7 @@ static void valleyview_irq_uninstall(struct drm_device *dev)
 	I915_WRITE(VLV_MASTER_IER, 0);
 	POSTING_READ(VLV_MASTER_IER);
 
-	gen5_gt_irq_reset(dev);
+	gen5_gt_irq_reset(dev_priv);
 
 	I915_WRITE(HWSTAM, 0xffffffff);
 
@@ -3971,7 +3625,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 		new_iir = I915_READ16(IIR); /* Flush posted writes */
 
 		if (iir & I915_USER_INTERRUPT)
-			notify_ring(&dev_priv->engine[RCS]);
+			notify_ring(dev_priv->engine[RCS]);
 
 		for_each_pipe(dev_priv, pipe) {
 			int plane = pipe;
@@ -4020,7 +3674,7 @@ static void i915_irq_preinstall(struct drm_device * dev)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	int pipe;
 
-	if (I915_HAS_HOTPLUG(dev)) {
+	if (I915_HAS_HOTPLUG(dev_priv)) {
 		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
 		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
 	}
@@ -4054,7 +3708,7 @@ static int i915_irq_postinstall(struct drm_device *dev)
 		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
 		I915_USER_INTERRUPT;
 
-	if (I915_HAS_HOTPLUG(dev)) {
+	if (I915_HAS_HOTPLUG(dev_priv)) {
 		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
 		POSTING_READ(PORT_HOTPLUG_EN);
 
@@ -4168,7 +3822,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 		new_iir = I915_READ(IIR); /* Flush posted writes */
 
 		if (iir & I915_USER_INTERRUPT)
-			notify_ring(&dev_priv->engine[RCS]);
+			notify_ring(dev_priv->engine[RCS]);
 
 		for_each_pipe(dev_priv, pipe) {
 			int plane = pipe;
@@ -4222,7 +3876,7 @@ static void i915_irq_uninstall(struct drm_device * dev)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	int pipe;
 
-	if (I915_HAS_HOTPLUG(dev)) {
+	if (I915_HAS_HOTPLUG(dev_priv)) {
 		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
 		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
 	}
@@ -4400,9 +4054,9 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 		new_iir = I915_READ(IIR); /* Flush posted writes */
 
 		if (iir & I915_USER_INTERRUPT)
-			notify_ring(&dev_priv->engine[RCS]);
+			notify_ring(dev_priv->engine[RCS]);
 		if (iir & I915_BSD_USER_INTERRUPT)
-			notify_ring(&dev_priv->engine[VCS]);
+			notify_ring(dev_priv->engine[VCS]);
 
 		for_each_pipe(dev_priv, pipe) {
 			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
@@ -4487,6 +4141,9 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
 	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
 
+	if (HAS_GUC_SCHED(dev_priv))
+		dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT;
+
 	/* Let's track the enabled rps events */
 	if (IS_VALLEYVIEW(dev_priv))
 		/* WaGsvRC0ResidencyMethod:vlv */
@@ -4508,9 +4165,6 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 	if (INTEL_INFO(dev_priv)->gen >= 8)
 		dev_priv->rps.pm_intr_keep |= GEN8_PMINTR_REDIRECT_TO_GUC;
 
-	INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
-			  i915_hangcheck_elapsed);
-
 	if (IS_GEN2(dev_priv)) {
 		/* Gen2 doesn't have a hardware frame counter */
 		dev->max_vblank_count = 0;
@@ -4539,16 +4193,16 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 		dev->driver->irq_preinstall = cherryview_irq_preinstall;
 		dev->driver->irq_postinstall = cherryview_irq_postinstall;
 		dev->driver->irq_uninstall = cherryview_irq_uninstall;
-		dev->driver->enable_vblank = valleyview_enable_vblank;
-		dev->driver->disable_vblank = valleyview_disable_vblank;
+		dev->driver->enable_vblank = i965_enable_vblank;
+		dev->driver->disable_vblank = i965_disable_vblank;
 		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
 	} else if (IS_VALLEYVIEW(dev_priv)) {
 		dev->driver->irq_handler = valleyview_irq_handler;
 		dev->driver->irq_preinstall = valleyview_irq_preinstall;
 		dev->driver->irq_postinstall = valleyview_irq_postinstall;
 		dev->driver->irq_uninstall = valleyview_irq_uninstall;
-		dev->driver->enable_vblank = valleyview_enable_vblank;
-		dev->driver->disable_vblank = valleyview_disable_vblank;
+		dev->driver->enable_vblank = i965_enable_vblank;
+		dev->driver->disable_vblank = i965_disable_vblank;
 		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
 	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
 		dev->driver->irq_handler = gen8_irq_handler;
@@ -4557,13 +4211,13 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 		dev->driver->irq_uninstall = gen8_irq_uninstall;
 		dev->driver->enable_vblank = gen8_enable_vblank;
 		dev->driver->disable_vblank = gen8_disable_vblank;
-		if (IS_BROXTON(dev))
+		if (IS_BROXTON(dev_priv))
 			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
-		else if (HAS_PCH_SPT(dev) || HAS_PCH_KBP(dev))
+		else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv))
 			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
 		else
 			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
-	} else if (HAS_PCH_SPLIT(dev)) {
+	} else if (HAS_PCH_SPLIT(dev_priv)) {
 		dev->driver->irq_handler = ironlake_irq_handler;
 		dev->driver->irq_preinstall = ironlake_irq_reset;
 		dev->driver->irq_postinstall = ironlake_irq_postinstall;
@@ -4577,21 +4231,25 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 			dev->driver->irq_postinstall = i8xx_irq_postinstall;
 			dev->driver->irq_handler = i8xx_irq_handler;
 			dev->driver->irq_uninstall = i8xx_irq_uninstall;
+			dev->driver->enable_vblank = i8xx_enable_vblank;
+			dev->driver->disable_vblank = i8xx_disable_vblank;
 		} else if (IS_GEN3(dev_priv)) {
 			dev->driver->irq_preinstall = i915_irq_preinstall;
 			dev->driver->irq_postinstall = i915_irq_postinstall;
 			dev->driver->irq_uninstall = i915_irq_uninstall;
 			dev->driver->irq_handler = i915_irq_handler;
+			dev->driver->enable_vblank = i8xx_enable_vblank;
+			dev->driver->disable_vblank = i8xx_disable_vblank;
 		} else {
 			dev->driver->irq_preinstall = i965_irq_preinstall;
 			dev->driver->irq_postinstall = i965_irq_postinstall;
 			dev->driver->irq_uninstall = i965_irq_uninstall;
 			dev->driver->irq_handler = i965_irq_handler;
+			dev->driver->enable_vblank = i965_enable_vblank;
+			dev->driver->disable_vblank = i965_disable_vblank;
 		}
 		if (I915_HAS_HOTPLUG(dev_priv))
 			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
-		dev->driver->enable_vblank = i915_enable_vblank;
-		dev->driver->disable_vblank = i915_disable_vblank;
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index 768ad89d9cd4..d46ffe7086bc 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -39,7 +39,7 @@ struct i915_params i915 __read_mostly = {
 	.enable_hangcheck = true,
 	.enable_ppgtt = -1,
 	.enable_psr = -1,
-	.preliminary_hw_support = IS_ENABLED(CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT),
+	.alpha_support = IS_ENABLED(CONFIG_DRM_I915_ALPHA_SUPPORT),
 	.disable_power_well = -1,
 	.enable_ips = 1,
 	.fastboot = 0,
@@ -47,6 +47,7 @@ struct i915_params i915 __read_mostly = {
 	.load_detect_test = 0,
 	.force_reset_modeset_test = 0,
 	.reset = true,
+	.error_capture = true,
 	.invert_brightness = 0,
 	.disable_display = 0,
 	.enable_cmd_parser = 1,
@@ -115,6 +116,14 @@ MODULE_PARM_DESC(vbt_sdvo_panel_type,
 module_param_named_unsafe(reset, i915.reset, bool, 0600);
 MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)");
 
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+module_param_named(error_capture, i915.error_capture, bool, 0600);
+MODULE_PARM_DESC(error_capture,
+	"Record the GPU state following a hang. "
+	"This information in /sys/class/drm/card<N>/error is vital for "
+	"triaging and debugging hangs.");
+#endif
+
 module_param_named_unsafe(enable_hangcheck, i915.enable_hangcheck, bool, 0644);
 MODULE_PARM_DESC(enable_hangcheck,
 	"Periodically check GPU activity for detecting hangs. "
@@ -136,9 +145,10 @@ MODULE_PARM_DESC(enable_psr, "Enable PSR "
 		 "(0=disabled, 1=enabled - link mode chosen per-platform, 2=force link-standby mode, 3=force link-off mode) "
 		 "Default: -1 (use per-chip default)");
 
-module_param_named_unsafe(preliminary_hw_support, i915.preliminary_hw_support, int, 0400);
-MODULE_PARM_DESC(preliminary_hw_support,
-	"Enable preliminary hardware support.");
+module_param_named_unsafe(alpha_support, i915.alpha_support, int, 0400);
+MODULE_PARM_DESC(alpha_support,
+	"Enable alpha quality driver support for latest hardware. "
+	"See also CONFIG_DRM_I915_ALPHA_SUPPORT.");
 
 module_param_named_unsafe(disable_power_well, i915.disable_power_well, int, 0400);
 MODULE_PARM_DESC(disable_power_well,
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index 3a0dd78ddb38..817ad959941e 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -40,7 +40,7 @@ struct i915_params {
 	int enable_ppgtt;
 	int enable_execlists;
 	int enable_psr;
-	unsigned int preliminary_hw_support;
+	unsigned int alpha_support;
 	int disable_power_well;
 	int enable_ips;
 	int invert_brightness;
@@ -59,6 +59,7 @@ struct i915_params {
 	bool load_detect_test;
 	bool force_reset_modeset_test;
 	bool reset;
+	bool error_capture;
 	bool disable_display;
 	bool verbose_state_checks;
 	bool nuclear_pageflip;
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 31e6edd08dd0..fce8e198bc76 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -288,7 +288,8 @@ static const struct intel_device_info intel_haswell_info = {
 #define BDW_FEATURES \
 	HSW_FEATURES, \
 	BDW_COLORS, \
-	.has_logical_ring_contexts = 1
+	.has_logical_ring_contexts = 1, \
+	.has_64bit_reloc = 1
 
 static const struct intel_device_info intel_broadwell_info = {
 	BDW_FEATURES,
@@ -308,6 +309,7 @@ static const struct intel_device_info intel_cherryview_info = {
 	.has_hotplug = 1,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
 	.is_cherryview = 1,
+	.has_64bit_reloc = 1,
 	.has_psr = 1,
 	.has_runtime_pm = 1,
 	.has_resource_streamer = 1,
@@ -347,6 +349,7 @@ static const struct intel_device_info intel_broxton_info = {
 	.has_hotplug = 1,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
 	.num_pipes = 3,
+	.has_64bit_reloc = 1,
 	.has_ddi = 1,
 	.has_fpga_dbg = 1,
 	.has_fbc = 1,
@@ -360,6 +363,7 @@ static const struct intel_device_info intel_broxton_info = {
 	.has_hw_contexts = 1,
 	.has_logical_ring_contexts = 1,
 	.has_guc = 1,
+	.has_decoupled_mmio = 1,
 	.ddb_size = 512,
 	GEN_DEFAULT_PIPEOFFSETS,
 	IVB_CURSOR_OFFSETS,
@@ -436,9 +440,10 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	struct intel_device_info *intel_info =
 		(struct intel_device_info *) ent->driver_data;
 
-	if (IS_PRELIMINARY_HW(intel_info) && !i915.preliminary_hw_support) {
-		DRM_INFO("This hardware requires preliminary hardware support.\n"
-			 "See CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT, and/or modparam preliminary_hw_support\n");
+	if (IS_ALPHA_SUPPORT(intel_info) && !i915.alpha_support) {
+		DRM_INFO("The driver support for your hardware in this kernel version is alpha quality\n"
+			 "See CONFIG_DRM_I915_ALPHA_SUPPORT or i915.alpha_support module parameter\n"
+			 "to enable support in this kernel version, or check for kernel updates.\n");
 		return -ENODEV;
 	}
 
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 70d96162def6..c70c07a7b586 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -86,8 +86,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define DEVEN 0x54
 #define   DEVEN_MCHBAR_EN (1 << 28)
 
-#define BSM 0x5c
-#define   BSM_MASK (0xFFFF << 20)
+/* BSM in include/drm/i915_drm.h */
 
 #define HPLLCC	0xc0 /* 85x only */
 #define   GC_CLOCK_CONTROL_MASK		(0x7 << 0)
@@ -831,96 +830,7 @@ enum skl_disp_power_wells {
 #define  CCK_FREQUENCY_STATUS_SHIFT		8
 #define  CCK_FREQUENCY_VALUES			(0x1f << 0)
 
-/**
- * DOC: DPIO
- *
- * VLV, CHV and BXT have slightly peculiar display PHYs for driving DP/HDMI
- * ports. DPIO is the name given to such a display PHY. These PHYs
- * don't follow the standard programming model using direct MMIO
- * registers, and instead their registers must be accessed trough IOSF
- * sideband. VLV has one such PHY for driving ports B and C, and CHV
- * adds another PHY for driving port D. Each PHY responds to specific
- * IOSF-SB port.
- *
- * Each display PHY is made up of one or two channels. Each channel
- * houses a common lane part which contains the PLL and other common
- * logic. CH0 common lane also contains the IOSF-SB logic for the
- * Common Register Interface (CRI) ie. the DPIO registers. CRI clock
- * must be running when any DPIO registers are accessed.
- *
- * In addition to having their own registers, the PHYs are also
- * controlled through some dedicated signals from the display
- * controller. These include PLL reference clock enable, PLL enable,
- * and CRI clock selection, for example.
- *
- * Eeach channel also has two splines (also called data lanes), and
- * each spline is made up of one Physical Access Coding Sub-Layer
- * (PCS) block and two TX lanes. So each channel has two PCS blocks
- * and four TX lanes. The TX lanes are used as DP lanes or TMDS
- * data/clock pairs depending on the output type.
- *
- * Additionally the PHY also contains an AUX lane with AUX blocks
- * for each channel. This is used for DP AUX communication, but
- * this fact isn't really relevant for the driver since AUX is
- * controlled from the display controller side. No DPIO registers
- * need to be accessed during AUX communication,
- *
- * Generally on VLV/CHV the common lane corresponds to the pipe and
- * the spline (PCS/TX) corresponds to the port.
- *
- * For dual channel PHY (VLV/CHV):
- *
- *  pipe A == CMN/PLL/REF CH0
- *
- *  pipe B == CMN/PLL/REF CH1
- *
- *  port B == PCS/TX CH0
- *
- *  port C == PCS/TX CH1
- *
- * This is especially important when we cross the streams
- * ie. drive port B with pipe B, or port C with pipe A.
- *
- * For single channel PHY (CHV):
- *
- *  pipe C == CMN/PLL/REF CH0
- *
- *  port D == PCS/TX CH0
- *
- * On BXT the entire PHY channel corresponds to the port. That means
- * the PLL is also now associated with the port rather than the pipe,
- * and so the clock needs to be routed to the appropriate transcoder.
- * Port A PLL is directly connected to transcoder EDP and port B/C
- * PLLs can be routed to any transcoder A/B/C.
- *
- * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is
- * digital port D (CHV) or port A (BXT). ::
- *
- *
- *     Dual channel PHY (VLV/CHV/BXT)
- *     ---------------------------------
- *     |      CH0      |      CH1      |
- *     |  CMN/PLL/REF  |  CMN/PLL/REF  |
- *     |---------------|---------------| Display PHY
- *     | PCS01 | PCS23 | PCS01 | PCS23 |
- *     |-------|-------|-------|-------|
- *     |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3|
- *     ---------------------------------
- *     |     DDI0      |     DDI1      | DP/HDMI ports
- *     ---------------------------------
- *
- *     Single channel PHY (CHV/BXT)
- *     -----------------
- *     |      CH0      |
- *     |  CMN/PLL/REF  |
- *     |---------------| Display PHY
- *     | PCS01 | PCS23 |
- *     |-------|-------|
- *     |TX0|TX1|TX2|TX3|
- *     -----------------
- *     |     DDI2      | DP/HDMI port
- *     -----------------
- */
+/* DPIO registers */
 #define DPIO_DEVFN			0
 
 #define DPIO_CTL			_MMIO(VLV_DISPLAY_BASE + 0x2110)
@@ -1276,7 +1186,19 @@ enum skl_disp_power_wells {
 #define   DPIO_UPAR_SHIFT		30
 
 /* BXT PHY registers */
-#define _BXT_PHY(phy, a, b)		_MMIO_PIPE((phy), (a), (b))
+#define _BXT_PHY0_BASE			0x6C000
+#define _BXT_PHY1_BASE			0x162000
+#define BXT_PHY_BASE(phy)		_PIPE((phy), _BXT_PHY0_BASE, \
+						     _BXT_PHY1_BASE)
+
+#define _BXT_PHY(phy, reg)						\
+	_MMIO(BXT_PHY_BASE(phy) - _BXT_PHY0_BASE + (reg))
+
+#define _BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1)		\
+	(BXT_PHY_BASE(phy) + _PIPE((ch), (reg_ch0) - _BXT_PHY0_BASE,	\
+					 (reg_ch1) - _BXT_PHY0_BASE))
+#define _MMIO_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1)		\
+	_MMIO(_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1))
 
 #define BXT_P_CR_GT_DISP_PWRON		_MMIO(0x138090)
 #define   GT_DISPLAY_POWER_ON(phy)	(1 << (phy))
@@ -1293,8 +1215,8 @@ enum skl_disp_power_wells {
 #define _PHY_CTL_FAMILY_EDP		0x64C80
 #define _PHY_CTL_FAMILY_DDI		0x64C90
 #define   COMMON_RESET_DIS		(1 << 31)
-#define BXT_PHY_CTL_FAMILY(phy)		_BXT_PHY((phy), _PHY_CTL_FAMILY_DDI, \
-							_PHY_CTL_FAMILY_EDP)
+#define BXT_PHY_CTL_FAMILY(phy)		_MMIO_PIPE((phy), _PHY_CTL_FAMILY_DDI, \
+							  _PHY_CTL_FAMILY_EDP)
 
 /* BXT PHY PLL registers */
 #define _PORT_PLL_A			0x46074
@@ -1314,18 +1236,18 @@ enum skl_disp_power_wells {
 #define   PORT_PLL_P2_SHIFT		8
 #define   PORT_PLL_P2_MASK		(0x1f << PORT_PLL_P2_SHIFT)
 #define   PORT_PLL_P2(x)		((x)  << PORT_PLL_P2_SHIFT)
-#define BXT_PORT_PLL_EBB_0(port)	_MMIO_PORT3(port, _PORT_PLL_EBB_0_A, \
-						_PORT_PLL_EBB_0_B,	\
-						_PORT_PLL_EBB_0_C)
+#define BXT_PORT_PLL_EBB_0(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PLL_EBB_0_B, \
+							 _PORT_PLL_EBB_0_C)
 
 #define _PORT_PLL_EBB_4_A		0x162038
 #define _PORT_PLL_EBB_4_B		0x6C038
 #define _PORT_PLL_EBB_4_C		0x6C344
 #define   PORT_PLL_10BIT_CLK_ENABLE	(1 << 13)
 #define   PORT_PLL_RECALIBRATE		(1 << 14)
-#define BXT_PORT_PLL_EBB_4(port)	_MMIO_PORT3(port, _PORT_PLL_EBB_4_A, \
-						_PORT_PLL_EBB_4_B,	\
-						_PORT_PLL_EBB_4_C)
+#define BXT_PORT_PLL_EBB_4(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PLL_EBB_4_B, \
+							 _PORT_PLL_EBB_4_C)
 
 #define _PORT_PLL_0_A			0x162100
 #define _PORT_PLL_0_B			0x6C100
@@ -1356,57 +1278,56 @@ enum skl_disp_power_wells {
 #define  PORT_PLL_DCO_AMP_DEFAULT	15
 #define  PORT_PLL_DCO_AMP_MASK		0x3c00
 #define  PORT_PLL_DCO_AMP(x)		((x)<<10)
-#define _PORT_PLL_BASE(port)		_PORT3(port, _PORT_PLL_0_A,	\
-						_PORT_PLL_0_B,		\
-						_PORT_PLL_0_C)
-#define BXT_PORT_PLL(port, idx)		_MMIO(_PORT_PLL_BASE(port) + (idx) * 4)
+#define _PORT_PLL_BASE(phy, ch)		_BXT_PHY_CH(phy, ch, \
+						    _PORT_PLL_0_B, \
+						    _PORT_PLL_0_C)
+#define BXT_PORT_PLL(phy, ch, idx)	_MMIO(_PORT_PLL_BASE(phy, ch) + \
+					      (idx) * 4)
 
 /* BXT PHY common lane registers */
 #define _PORT_CL1CM_DW0_A		0x162000
 #define _PORT_CL1CM_DW0_BC		0x6C000
 #define   PHY_POWER_GOOD		(1 << 16)
 #define   PHY_RESERVED			(1 << 7)
-#define BXT_PORT_CL1CM_DW0(phy)		_BXT_PHY((phy), _PORT_CL1CM_DW0_BC, \
-							_PORT_CL1CM_DW0_A)
+#define BXT_PORT_CL1CM_DW0(phy)		_BXT_PHY((phy), _PORT_CL1CM_DW0_BC)
 
 #define _PORT_CL1CM_DW9_A		0x162024
 #define _PORT_CL1CM_DW9_BC		0x6C024
 #define   IREF0RC_OFFSET_SHIFT		8
 #define   IREF0RC_OFFSET_MASK		(0xFF << IREF0RC_OFFSET_SHIFT)
-#define BXT_PORT_CL1CM_DW9(phy)		_BXT_PHY((phy), _PORT_CL1CM_DW9_BC, \
-							_PORT_CL1CM_DW9_A)
+#define BXT_PORT_CL1CM_DW9(phy)		_BXT_PHY((phy), _PORT_CL1CM_DW9_BC)
 
 #define _PORT_CL1CM_DW10_A		0x162028
 #define _PORT_CL1CM_DW10_BC		0x6C028
 #define   IREF1RC_OFFSET_SHIFT		8
 #define   IREF1RC_OFFSET_MASK		(0xFF << IREF1RC_OFFSET_SHIFT)
-#define BXT_PORT_CL1CM_DW10(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW10_BC, \
-							_PORT_CL1CM_DW10_A)
+#define BXT_PORT_CL1CM_DW10(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW10_BC)
 
 #define _PORT_CL1CM_DW28_A		0x162070
 #define _PORT_CL1CM_DW28_BC		0x6C070
 #define   OCL1_POWER_DOWN_EN		(1 << 23)
 #define   DW28_OLDO_DYN_PWR_DOWN_EN	(1 << 22)
 #define   SUS_CLK_CONFIG		0x3
-#define BXT_PORT_CL1CM_DW28(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW28_BC, \
-							_PORT_CL1CM_DW28_A)
+#define BXT_PORT_CL1CM_DW28(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW28_BC)
 
 #define _PORT_CL1CM_DW30_A		0x162078
 #define _PORT_CL1CM_DW30_BC		0x6C078
 #define   OCL2_LDOFUSE_PWR_DIS		(1 << 6)
-#define BXT_PORT_CL1CM_DW30(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW30_BC, \
-							_PORT_CL1CM_DW30_A)
+#define BXT_PORT_CL1CM_DW30(phy)	_BXT_PHY((phy), _PORT_CL1CM_DW30_BC)
 
-/* Defined for PHY0 only */
-#define BXT_PORT_CL2CM_DW6_BC		_MMIO(0x6C358)
+/* The spec defines this only for BXT PHY0, but lets assume that this
+ * would exist for PHY1 too if it had a second channel.
+ */
+#define _PORT_CL2CM_DW6_A		0x162358
+#define _PORT_CL2CM_DW6_BC		0x6C358
+#define BXT_PORT_CL2CM_DW6(phy)		_BXT_PHY((phy), _PORT_CL2CM_DW6_BC)
 #define   DW6_OLDO_DYN_PWR_DOWN_EN	(1 << 28)
 
 /* BXT PHY Ref registers */
 #define _PORT_REF_DW3_A			0x16218C
 #define _PORT_REF_DW3_BC		0x6C18C
 #define   GRC_DONE			(1 << 22)
-#define BXT_PORT_REF_DW3(phy)		_BXT_PHY((phy), _PORT_REF_DW3_BC, \
-							_PORT_REF_DW3_A)
+#define BXT_PORT_REF_DW3(phy)		_BXT_PHY((phy), _PORT_REF_DW3_BC)
 
 #define _PORT_REF_DW6_A			0x162198
 #define _PORT_REF_DW6_BC		0x6C198
@@ -1417,15 +1338,13 @@ enum skl_disp_power_wells {
 #define   GRC_CODE_SLOW_SHIFT		8
 #define   GRC_CODE_SLOW_MASK		(0xFF << GRC_CODE_SLOW_SHIFT)
 #define   GRC_CODE_NOM_MASK		0xFF
-#define BXT_PORT_REF_DW6(phy)		_BXT_PHY((phy), _PORT_REF_DW6_BC,	\
-						      _PORT_REF_DW6_A)
+#define BXT_PORT_REF_DW6(phy)		_BXT_PHY((phy), _PORT_REF_DW6_BC)
 
 #define _PORT_REF_DW8_A			0x1621A0
 #define _PORT_REF_DW8_BC		0x6C1A0
 #define   GRC_DIS			(1 << 15)
 #define   GRC_RDY_OVRD			(1 << 1)
-#define BXT_PORT_REF_DW8(phy)		_BXT_PHY((phy), _PORT_REF_DW8_BC,	\
-						      _PORT_REF_DW8_A)
+#define BXT_PORT_REF_DW8(phy)		_BXT_PHY((phy), _PORT_REF_DW8_BC)
 
 /* BXT PHY PCS registers */
 #define _PORT_PCS_DW10_LN01_A		0x162428
@@ -1434,12 +1353,13 @@ enum skl_disp_power_wells {
 #define _PORT_PCS_DW10_GRP_A		0x162C28
 #define _PORT_PCS_DW10_GRP_B		0x6CC28
 #define _PORT_PCS_DW10_GRP_C		0x6CE28
-#define BXT_PORT_PCS_DW10_LN01(port)	_MMIO_PORT3(port, _PORT_PCS_DW10_LN01_A, \
-						     _PORT_PCS_DW10_LN01_B, \
-						     _PORT_PCS_DW10_LN01_C)
-#define BXT_PORT_PCS_DW10_GRP(port)	_MMIO_PORT3(port, _PORT_PCS_DW10_GRP_A,  \
-						     _PORT_PCS_DW10_GRP_B,  \
-						     _PORT_PCS_DW10_GRP_C)
+#define BXT_PORT_PCS_DW10_LN01(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PCS_DW10_LN01_B, \
+							 _PORT_PCS_DW10_LN01_C)
+#define BXT_PORT_PCS_DW10_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PCS_DW10_GRP_B, \
+							 _PORT_PCS_DW10_GRP_C)
+
 #define   TX2_SWING_CALC_INIT		(1 << 31)
 #define   TX1_SWING_CALC_INIT		(1 << 30)
 
@@ -1454,15 +1374,15 @@ enum skl_disp_power_wells {
 #define _PORT_PCS_DW12_GRP_C		0x6CE30
 #define   LANESTAGGER_STRAP_OVRD	(1 << 6)
 #define   LANE_STAGGER_MASK		0x1F
-#define BXT_PORT_PCS_DW12_LN01(port)	_MMIO_PORT3(port, _PORT_PCS_DW12_LN01_A, \
-						     _PORT_PCS_DW12_LN01_B, \
-						     _PORT_PCS_DW12_LN01_C)
-#define BXT_PORT_PCS_DW12_LN23(port)	_MMIO_PORT3(port, _PORT_PCS_DW12_LN23_A, \
-						     _PORT_PCS_DW12_LN23_B, \
-						     _PORT_PCS_DW12_LN23_C)
-#define BXT_PORT_PCS_DW12_GRP(port)	_MMIO_PORT3(port, _PORT_PCS_DW12_GRP_A, \
-						     _PORT_PCS_DW12_GRP_B, \
-						     _PORT_PCS_DW12_GRP_C)
+#define BXT_PORT_PCS_DW12_LN01(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PCS_DW12_LN01_B, \
+							 _PORT_PCS_DW12_LN01_C)
+#define BXT_PORT_PCS_DW12_LN23(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PCS_DW12_LN23_B, \
+							 _PORT_PCS_DW12_LN23_C)
+#define BXT_PORT_PCS_DW12_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_PCS_DW12_GRP_B, \
+							 _PORT_PCS_DW12_GRP_C)
 
 /* BXT PHY TX registers */
 #define _BXT_LANE_OFFSET(lane)           (((lane) >> 1) * 0x200 +	\
@@ -1474,12 +1394,12 @@ enum skl_disp_power_wells {
 #define _PORT_TX_DW2_GRP_A		0x162D08
 #define _PORT_TX_DW2_GRP_B		0x6CD08
 #define _PORT_TX_DW2_GRP_C		0x6CF08
-#define BXT_PORT_TX_DW2_GRP(port)	_MMIO_PORT3(port, _PORT_TX_DW2_GRP_A,  \
-						     _PORT_TX_DW2_GRP_B,  \
-						     _PORT_TX_DW2_GRP_C)
-#define BXT_PORT_TX_DW2_LN0(port)	_MMIO_PORT3(port, _PORT_TX_DW2_LN0_A,  \
-						     _PORT_TX_DW2_LN0_B,  \
-						     _PORT_TX_DW2_LN0_C)
+#define BXT_PORT_TX_DW2_LN0(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_TX_DW2_LN0_B, \
+							 _PORT_TX_DW2_LN0_C)
+#define BXT_PORT_TX_DW2_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_TX_DW2_GRP_B, \
+							 _PORT_TX_DW2_GRP_C)
 #define   MARGIN_000_SHIFT		16
 #define   MARGIN_000			(0xFF << MARGIN_000_SHIFT)
 #define   UNIQ_TRANS_SCALE_SHIFT	8
@@ -1491,12 +1411,12 @@ enum skl_disp_power_wells {
 #define _PORT_TX_DW3_GRP_A		0x162D0C
 #define _PORT_TX_DW3_GRP_B		0x6CD0C
 #define _PORT_TX_DW3_GRP_C		0x6CF0C
-#define BXT_PORT_TX_DW3_GRP(port)	_MMIO_PORT3(port, _PORT_TX_DW3_GRP_A,  \
-						     _PORT_TX_DW3_GRP_B,  \
-						     _PORT_TX_DW3_GRP_C)
-#define BXT_PORT_TX_DW3_LN0(port)	_MMIO_PORT3(port, _PORT_TX_DW3_LN0_A,  \
-						     _PORT_TX_DW3_LN0_B,  \
-						     _PORT_TX_DW3_LN0_C)
+#define BXT_PORT_TX_DW3_LN0(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_TX_DW3_LN0_B, \
+							 _PORT_TX_DW3_LN0_C)
+#define BXT_PORT_TX_DW3_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_TX_DW3_GRP_B, \
+							 _PORT_TX_DW3_GRP_C)
 #define   SCALE_DCOMP_METHOD		(1 << 26)
 #define   UNIQUE_TRANGE_EN_METHOD	(1 << 27)
 
@@ -1506,12 +1426,12 @@ enum skl_disp_power_wells {
 #define _PORT_TX_DW4_GRP_A		0x162D10
 #define _PORT_TX_DW4_GRP_B		0x6CD10
 #define _PORT_TX_DW4_GRP_C		0x6CF10
-#define BXT_PORT_TX_DW4_LN0(port)	_MMIO_PORT3(port, _PORT_TX_DW4_LN0_A,  \
-						     _PORT_TX_DW4_LN0_B,  \
-						     _PORT_TX_DW4_LN0_C)
-#define BXT_PORT_TX_DW4_GRP(port)	_MMIO_PORT3(port, _PORT_TX_DW4_GRP_A,  \
-						     _PORT_TX_DW4_GRP_B,  \
-						     _PORT_TX_DW4_GRP_C)
+#define BXT_PORT_TX_DW4_LN0(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_TX_DW4_LN0_B, \
+							 _PORT_TX_DW4_LN0_C)
+#define BXT_PORT_TX_DW4_GRP(phy, ch)	_MMIO_BXT_PHY_CH(phy, ch, \
+							 _PORT_TX_DW4_GRP_B, \
+							 _PORT_TX_DW4_GRP_C)
 #define   DEEMPH_SHIFT			24
 #define   DE_EMPHASIS			(0xFF << DEEMPH_SHIFT)
 
@@ -1520,10 +1440,10 @@ enum skl_disp_power_wells {
 #define _PORT_TX_DW14_LN0_C		0x6C938
 #define   LATENCY_OPTIM_SHIFT		30
 #define   LATENCY_OPTIM			(1 << LATENCY_OPTIM_SHIFT)
-#define BXT_PORT_TX_DW14_LN(port, lane)	_MMIO(_PORT3((port), _PORT_TX_DW14_LN0_A,   \
-							_PORT_TX_DW14_LN0_B,   \
-							_PORT_TX_DW14_LN0_C) + \
-					 _BXT_LANE_OFFSET(lane))
+#define BXT_PORT_TX_DW14_LN(phy, ch, lane)				\
+	_MMIO(_BXT_PHY_CH(phy, ch, _PORT_TX_DW14_LN0_B,			\
+				   _PORT_TX_DW14_LN0_C) +		\
+	      _BXT_LANE_OFFSET(lane))
 
 /* UAIMI scratch pad register 1 */
 #define UAIMI_SPR1			_MMIO(0x4F074)
@@ -1605,6 +1525,7 @@ enum skl_disp_power_wells {
 #define RING_HEAD(base)		_MMIO((base)+0x34)
 #define RING_START(base)	_MMIO((base)+0x38)
 #define RING_CTL(base)		_MMIO((base)+0x3c)
+#define   RING_CTL_SIZE(size)	((size) - PAGE_SIZE) /* in bytes -> pages */
 #define RING_SYNC_0(base)	_MMIO((base)+0x40)
 #define RING_SYNC_1(base)	_MMIO((base)+0x44)
 #define RING_SYNC_2(base)	_MMIO((base)+0x48)
@@ -1708,7 +1629,11 @@ enum skl_disp_power_wells {
 #define GEN7_SC_INSTDONE	_MMIO(0x7100)
 #define GEN7_SAMPLER_INSTDONE	_MMIO(0xe160)
 #define GEN7_ROW_INSTDONE	_MMIO(0xe164)
-#define I915_NUM_INSTDONE_REG	4
+#define GEN8_MCR_SELECTOR		_MMIO(0xfdc)
+#define   GEN8_MCR_SLICE(slice)		(((slice) & 3) << 26)
+#define   GEN8_MCR_SLICE_MASK		GEN8_MCR_SLICE(3)
+#define   GEN8_MCR_SUBSLICE(subslice)	(((subslice) & 3) << 24)
+#define   GEN8_MCR_SUBSLICE_MASK	GEN8_MCR_SUBSLICE(3)
 #define RING_IPEIR(base)	_MMIO((base)+0x64)
 #define RING_IPEHR(base)	_MMIO((base)+0x68)
 /*
@@ -2089,9 +2014,9 @@ enum skl_disp_power_wells {
 #define PM_VEBOX_CS_ERROR_INTERRUPT		(1 << 12) /* hsw+ */
 #define PM_VEBOX_USER_INTERRUPT			(1 << 10) /* hsw+ */
 
-#define GT_PARITY_ERROR(dev) \
+#define GT_PARITY_ERROR(dev_priv) \
 	(GT_RENDER_L3_PARITY_ERROR_INTERRUPT | \
-	 (IS_HASWELL(dev) ? GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1 : 0))
+	 (IS_HASWELL(dev_priv) ? GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1 : 0))
 
 /* These are all the "old" interrupts */
 #define ILK_BSD_USER_INTERRUPT				(1<<5)
@@ -2184,8 +2109,9 @@ enum skl_disp_power_wells {
 #define FBC_FENCE_OFF		_MMIO(0x3218) /* BSpec typo has 321Bh */
 #define FBC_TAG(i)		_MMIO(0x3300 + (i) * 4)
 
-#define FBC_STATUS2		_MMIO(0x43214)
-#define  FBC_COMPRESSION_MASK	0x7ff
+#define FBC_STATUS2			_MMIO(0x43214)
+#define  IVB_FBC_COMPRESSION_MASK	0x7ff
+#define  BDW_FBC_COMPRESSION_MASK	0xfff
 
 #define FBC_LL_SIZE		(1536)
 
@@ -6011,6 +5937,7 @@ enum {
 #define  GEN8_DE_PIPE_A_IRQ		(1<<16)
 #define  GEN8_DE_PIPE_IRQ(pipe)		(1<<(16+(pipe)))
 #define  GEN8_GT_VECS_IRQ		(1<<6)
+#define  GEN8_GT_GUC_IRQ		(1<<5)
 #define  GEN8_GT_PM_IRQ			(1<<4)
 #define  GEN8_GT_VCS2_IRQ		(1<<3)
 #define  GEN8_GT_VCS1_IRQ		(1<<2)
@@ -6022,6 +5949,16 @@ enum {
 #define GEN8_GT_IIR(which) _MMIO(0x44308 + (0x10 * (which)))
 #define GEN8_GT_IER(which) _MMIO(0x4430c + (0x10 * (which)))
 
+#define GEN9_GUC_TO_HOST_INT_EVENT	(1<<31)
+#define GEN9_GUC_EXEC_ERROR_EVENT	(1<<30)
+#define GEN9_GUC_DISPLAY_EVENT		(1<<29)
+#define GEN9_GUC_SEMA_SIGNAL_EVENT	(1<<28)
+#define GEN9_GUC_IOMMU_MSG_EVENT	(1<<27)
+#define GEN9_GUC_DB_RING_EVENT		(1<<26)
+#define GEN9_GUC_DMA_DONE_EVENT		(1<<25)
+#define GEN9_GUC_FATAL_ERROR_EVENT	(1<<24)
+#define GEN9_GUC_NOTIFICATION_EVENT	(1<<23)
+
 #define GEN8_RCS_IRQ_SHIFT 0
 #define GEN8_BCS_IRQ_SHIFT 16
 #define GEN8_VCS1_IRQ_SHIFT 0
@@ -7327,6 +7264,10 @@ enum {
 #define   AUD_CONFIG_UPPER_N_MASK		(0xff << 20)
 #define   AUD_CONFIG_LOWER_N_SHIFT		4
 #define   AUD_CONFIG_LOWER_N_MASK		(0xfff << 4)
+#define   AUD_CONFIG_N_MASK			(AUD_CONFIG_UPPER_N_MASK | AUD_CONFIG_LOWER_N_MASK)
+#define   AUD_CONFIG_N(n) \
+	(((((n) >> 12) & 0xff) << AUD_CONFIG_UPPER_N_SHIFT) |	\
+	 (((n) & 0xfff) << AUD_CONFIG_LOWER_N_SHIFT))
 #define   AUD_CONFIG_PIXEL_CLOCK_HDMI_SHIFT	16
 #define   AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK	(0xf << 16)
 #define   AUD_CONFIG_PIXEL_CLOCK_HDMI_25175	(0 << 16)
@@ -7350,6 +7291,13 @@ enum {
 #define _HSW_AUD_MISC_CTRL_B		0x65110
 #define HSW_AUD_MISC_CTRL(pipe)		_MMIO_PIPE(pipe, _HSW_AUD_MISC_CTRL_A, _HSW_AUD_MISC_CTRL_B)
 
+#define _HSW_AUD_M_CTS_ENABLE_A		0x65028
+#define _HSW_AUD_M_CTS_ENABLE_B		0x65128
+#define HSW_AUD_M_CTS_ENABLE(pipe)	_MMIO_PIPE(pipe, _HSW_AUD_M_CTS_ENABLE_A, _HSW_AUD_M_CTS_ENABLE_B)
+#define   AUD_M_CTS_M_VALUE_INDEX	(1 << 21)
+#define   AUD_M_CTS_M_PROG_ENABLE	(1 << 20)
+#define   AUD_CONFIG_M_MASK		0xfffff
+
 #define _HSW_AUD_DIP_ELD_CTRL_ST_A	0x650b4
 #define _HSW_AUD_DIP_ELD_CTRL_ST_B	0x651b4
 #define HSW_AUD_DIP_ELD_CTRL(pipe)	_MMIO_PIPE(pipe, _HSW_AUD_DIP_ELD_CTRL_ST_A, _HSW_AUD_DIP_ELD_CTRL_ST_B)
@@ -7394,6 +7342,13 @@ enum {
 #define  SKL_FUSE_PG1_DIST_STATUS              (1<<26)
 #define  SKL_FUSE_PG2_DIST_STATUS              (1<<25)
 
+/* Decoupled MMIO register pair for kernel driver */
+#define GEN9_DECOUPLED_REG0_DW0			_MMIO(0xF00)
+#define GEN9_DECOUPLED_REG0_DW1			_MMIO(0xF04)
+#define GEN9_DECOUPLED_DW1_GO			(1<<31)
+#define GEN9_DECOUPLED_PD_SHIFT			28
+#define GEN9_DECOUPLED_OP_SHIFT			24
+
 /* Per-pipe DDI Function Control */
 #define _TRANS_DDI_FUNC_CTL_A		0x60400
 #define _TRANS_DDI_FUNC_CTL_B		0x61400
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index a0af170062b1..b0e1e7ca75da 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -29,35 +29,31 @@
 #include "intel_drv.h"
 #include "i915_reg.h"
 
-static void i915_save_display(struct drm_device *dev)
+static void i915_save_display(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	/* Display arbitration control */
-	if (INTEL_INFO(dev)->gen <= 4)
+	if (INTEL_GEN(dev_priv) <= 4)
 		dev_priv->regfile.saveDSPARB = I915_READ(DSPARB);
 
 	/* save FBC interval */
-	if (HAS_FBC(dev) && INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev))
+	if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv))
 		dev_priv->regfile.saveFBC_CONTROL = I915_READ(FBC_CONTROL);
 }
 
-static void i915_restore_display(struct drm_device *dev)
+static void i915_restore_display(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	/* Display arbitration */
-	if (INTEL_INFO(dev)->gen <= 4)
+	if (INTEL_GEN(dev_priv) <= 4)
 		I915_WRITE(DSPARB, dev_priv->regfile.saveDSPARB);
 
 	/* only restore FBC info on the platform that supports FBC*/
 	intel_fbc_global_disable(dev_priv);
 
 	/* restore FBC interval */
-	if (HAS_FBC(dev) && INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev))
+	if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv))
 		I915_WRITE(FBC_CONTROL, dev_priv->regfile.saveFBC_CONTROL);
 
-	i915_redisable_vga(dev);
+	i915_redisable_vga(dev_priv);
 }
 
 int i915_save_state(struct drm_device *dev)
@@ -68,14 +64,14 @@ int i915_save_state(struct drm_device *dev)
 
 	mutex_lock(&dev->struct_mutex);
 
-	i915_save_display(dev);
+	i915_save_display(dev_priv);
 
-	if (IS_GEN4(dev))
+	if (IS_GEN4(dev_priv))
 		pci_read_config_word(pdev, GCDGMBUS,
 				     &dev_priv->regfile.saveGCDGMBUS);
 
 	/* Cache mode state */
-	if (INTEL_INFO(dev)->gen < 7)
+	if (INTEL_GEN(dev_priv) < 7)
 		dev_priv->regfile.saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0);
 
 	/* Memory Arbitration state */
@@ -114,15 +110,15 @@ int i915_restore_state(struct drm_device *dev)
 
 	mutex_lock(&dev->struct_mutex);
 
-	i915_gem_restore_fences(dev);
+	i915_gem_restore_fences(dev_priv);
 
-	if (IS_GEN4(dev))
+	if (IS_GEN4(dev_priv))
 		pci_write_config_word(pdev, GCDGMBUS,
 				      dev_priv->regfile.saveGCDGMBUS);
-	i915_restore_display(dev);
+	i915_restore_display(dev_priv);
 
 	/* Cache mode state */
-	if (INTEL_INFO(dev)->gen < 7)
+	if (INTEL_GEN(dev_priv) < 7)
 		I915_WRITE(CACHE_MODE_0, dev_priv->regfile.saveCACHE_MODE_0 |
 			   0xffff0000);
 
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c
index 1e5cbc585ca2..147420ccf49c 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence.c
+++ b/drivers/gpu/drm/i915/i915_sw_fence.c
@@ -8,11 +8,13 @@
  */
 
 #include <linux/slab.h>
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 #include <linux/reservation.h>
 
 #include "i915_sw_fence.h"
 
+#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
+
 static DEFINE_SPINLOCK(i915_sw_fence_lock);
 
 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
@@ -114,11 +116,14 @@ static void i915_sw_fence_await(struct i915_sw_fence *fence)
 	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
 }
 
-void i915_sw_fence_init(struct i915_sw_fence *fence, i915_sw_fence_notify_t fn)
+void __i915_sw_fence_init(struct i915_sw_fence *fence,
+			  i915_sw_fence_notify_t fn,
+			  const char *name,
+			  struct lock_class_key *key)
 {
 	BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK);
 
-	init_waitqueue_head(&fence->wait);
+	__init_waitqueue_head(&fence->wait, name, key);
 	kref_init(&fence->kref);
 	atomic_set(&fence->pending, 1);
 	fence->flags = (unsigned long)fn;
@@ -135,6 +140,8 @@ static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void *
 	list_del(&wq->task_list);
 	__i915_sw_fence_complete(wq->private, key);
 	i915_sw_fence_put(wq->private);
+	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
+		kfree(wq);
 	return 0;
 }
 
@@ -192,9 +199,9 @@ static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 	return err;
 }
 
-int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
-				 struct i915_sw_fence *signaler,
-				 wait_queue_t *wq)
+static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
+					  struct i915_sw_fence *signaler,
+					  wait_queue_t *wq, gfp_t gfp)
 {
 	unsigned long flags;
 	int pending;
@@ -206,8 +213,22 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
 		return -EINVAL;
 
+	pending = 0;
+	if (!wq) {
+		wq = kmalloc(sizeof(*wq), gfp);
+		if (!wq) {
+			if (!gfpflags_allow_blocking(gfp))
+				return -ENOMEM;
+
+			i915_sw_fence_wait(signaler);
+			return 0;
+		}
+
+		pending |= I915_SW_FENCE_FLAG_ALLOC;
+	}
+
 	INIT_LIST_HEAD(&wq->task_list);
-	wq->flags = 0;
+	wq->flags = pending;
 	wq->func = i915_sw_fence_wake;
 	wq->private = i915_sw_fence_get(fence);
 
@@ -226,49 +247,64 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 	return pending;
 }
 
-struct dma_fence_cb {
-	struct fence_cb base;
+int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
+				 struct i915_sw_fence *signaler,
+				 wait_queue_t *wq)
+{
+	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
+}
+
+int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
+				     struct i915_sw_fence *signaler,
+				     gfp_t gfp)
+{
+	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
+}
+
+struct i915_sw_dma_fence_cb {
+	struct dma_fence_cb base;
 	struct i915_sw_fence *fence;
-	struct fence *dma;
+	struct dma_fence *dma;
 	struct timer_list timer;
 };
 
 static void timer_i915_sw_fence_wake(unsigned long data)
 {
-	struct dma_fence_cb *cb = (struct dma_fence_cb *)data;
+	struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data;
 
 	printk(KERN_WARNING "asynchronous wait on fence %s:%s:%x timed out\n",
 	       cb->dma->ops->get_driver_name(cb->dma),
 	       cb->dma->ops->get_timeline_name(cb->dma),
 	       cb->dma->seqno);
-	fence_put(cb->dma);
+	dma_fence_put(cb->dma);
 	cb->dma = NULL;
 
 	i915_sw_fence_commit(cb->fence);
 	cb->timer.function = NULL;
 }
 
-static void dma_i915_sw_fence_wake(struct fence *dma, struct fence_cb *data)
+static void dma_i915_sw_fence_wake(struct dma_fence *dma,
+				   struct dma_fence_cb *data)
 {
-	struct dma_fence_cb *cb = container_of(data, typeof(*cb), base);
+	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
 
 	del_timer_sync(&cb->timer);
 	if (cb->timer.function)
 		i915_sw_fence_commit(cb->fence);
-	fence_put(cb->dma);
+	dma_fence_put(cb->dma);
 
 	kfree(cb);
 }
 
 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
-				  struct fence *dma,
+				  struct dma_fence *dma,
 				  unsigned long timeout,
 				  gfp_t gfp)
 {
-	struct dma_fence_cb *cb;
+	struct i915_sw_dma_fence_cb *cb;
 	int ret;
 
-	if (fence_is_signaled(dma))
+	if (dma_fence_is_signaled(dma))
 		return 0;
 
 	cb = kmalloc(sizeof(*cb), gfp);
@@ -276,7 +312,7 @@ int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 		if (!gfpflags_allow_blocking(gfp))
 			return -ENOMEM;
 
-		return fence_wait(dma, false);
+		return dma_fence_wait(dma, false);
 	}
 
 	cb->fence = i915_sw_fence_get(fence);
@@ -287,11 +323,11 @@ int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 		      timer_i915_sw_fence_wake, (unsigned long)cb,
 		      TIMER_IRQSAFE);
 	if (timeout) {
-		cb->dma = fence_get(dma);
+		cb->dma = dma_fence_get(dma);
 		mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
 	}
 
-	ret = fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
+	ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
 	if (ret == 0) {
 		ret = 1;
 	} else {
@@ -305,16 +341,16 @@ int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 
 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 				    struct reservation_object *resv,
-				    const struct fence_ops *exclude,
+				    const struct dma_fence_ops *exclude,
 				    bool write,
 				    unsigned long timeout,
 				    gfp_t gfp)
 {
-	struct fence *excl;
+	struct dma_fence *excl;
 	int ret = 0, pending;
 
 	if (write) {
-		struct fence **shared;
+		struct dma_fence **shared;
 		unsigned int count, i;
 
 		ret = reservation_object_get_fences_rcu(resv,
@@ -339,7 +375,7 @@ int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 		}
 
 		for (i = 0; i < count; i++)
-			fence_put(shared[i]);
+			dma_fence_put(shared[i]);
 		kfree(shared);
 	} else {
 		excl = reservation_object_get_excl_rcu(resv);
@@ -356,7 +392,7 @@ int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 			ret |= pending;
 	}
 
-	fence_put(excl);
+	dma_fence_put(excl);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.h b/drivers/gpu/drm/i915/i915_sw_fence.h
index 373141602ca4..0f3185ef7f4e 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence.h
+++ b/drivers/gpu/drm/i915/i915_sw_fence.h
@@ -16,8 +16,8 @@
 #include <linux/wait.h>
 
 struct completion;
-struct fence;
-struct fence_ops;
+struct dma_fence;
+struct dma_fence_ops;
 struct reservation_object;
 
 struct i915_sw_fence {
@@ -40,26 +40,54 @@ typedef int (*i915_sw_fence_notify_t)(struct i915_sw_fence *,
 				      enum i915_sw_fence_notify state);
 #define __i915_sw_fence_call __aligned(4)
 
-void i915_sw_fence_init(struct i915_sw_fence *fence, i915_sw_fence_notify_t fn);
+void __i915_sw_fence_init(struct i915_sw_fence *fence,
+			  i915_sw_fence_notify_t fn,
+			  const char *name,
+			  struct lock_class_key *key);
+#ifdef CONFIG_LOCKDEP
+#define i915_sw_fence_init(fence, fn)				\
+do {								\
+	static struct lock_class_key __key;			\
+								\
+	__i915_sw_fence_init((fence), (fn), #fence, &__key);	\
+} while (0)
+#else
+#define i915_sw_fence_init(fence, fn)				\
+	__i915_sw_fence_init((fence), (fn), NULL, NULL)
+#endif
+
 void i915_sw_fence_commit(struct i915_sw_fence *fence);
 
 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 				 struct i915_sw_fence *after,
 				 wait_queue_t *wq);
+int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
+				     struct i915_sw_fence *after,
+				     gfp_t gfp);
 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
-				  struct fence *dma,
+				  struct dma_fence *dma,
 				  unsigned long timeout,
 				  gfp_t gfp);
 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 				    struct reservation_object *resv,
-				    const struct fence_ops *exclude,
+				    const struct dma_fence_ops *exclude,
 				    bool write,
 				    unsigned long timeout,
 				    gfp_t gfp);
 
+static inline bool i915_sw_fence_signaled(const struct i915_sw_fence *fence)
+{
+	return atomic_read(&fence->pending) <= 0;
+}
+
 static inline bool i915_sw_fence_done(const struct i915_sw_fence *fence)
 {
 	return atomic_read(&fence->pending) < 0;
 }
 
+static inline void i915_sw_fence_wait(struct i915_sw_fence *fence)
+{
+	wait_event(fence->wait, i915_sw_fence_done(fence));
+}
+
 #endif /* _I915_SW_FENCE_H_ */
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index 1012eeea1324..47590ab08d7e 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -514,6 +514,8 @@ static const struct attribute *vlv_attrs[] = {
 	NULL,
 };
 
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+
 static ssize_t error_state_read(struct file *filp, struct kobject *kobj,
 				struct bin_attribute *attr, char *buf,
 				loff_t off, size_t count)
@@ -571,6 +573,21 @@ static struct bin_attribute error_state_attr = {
 	.write = error_state_write,
 };
 
+static void i915_setup_error_capture(struct device *kdev)
+{
+	if (sysfs_create_bin_file(&kdev->kobj, &error_state_attr))
+		DRM_ERROR("error_state sysfs setup failed\n");
+}
+
+static void i915_teardown_error_capture(struct device *kdev)
+{
+	sysfs_remove_bin_file(&kdev->kobj, &error_state_attr);
+}
+#else
+static void i915_setup_error_capture(struct device *kdev) {}
+static void i915_teardown_error_capture(struct device *kdev) {}
+#endif
+
 void i915_setup_sysfs(struct drm_i915_private *dev_priv)
 {
 	struct device *kdev = dev_priv->drm.primary->kdev;
@@ -617,17 +634,15 @@ void i915_setup_sysfs(struct drm_i915_private *dev_priv)
 	if (ret)
 		DRM_ERROR("RPS sysfs setup failed\n");
 
-	ret = sysfs_create_bin_file(&kdev->kobj,
-				    &error_state_attr);
-	if (ret)
-		DRM_ERROR("error_state sysfs setup failed\n");
+	i915_setup_error_capture(kdev);
 }
 
 void i915_teardown_sysfs(struct drm_i915_private *dev_priv)
 {
 	struct device *kdev = dev_priv->drm.primary->kdev;
 
-	sysfs_remove_bin_file(&kdev->kobj, &error_state_attr);
+	i915_teardown_error_capture(kdev);
+
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		sysfs_remove_files(&kdev->kobj, vlv_attrs);
 	else
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index 178798002a73..c5d210ebaa9a 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -466,7 +466,7 @@ TRACE_EVENT(i915_gem_ring_sync_to,
 			   __entry->dev = from->i915->drm.primary->index;
 			   __entry->sync_from = from->engine->id;
 			   __entry->sync_to = to->engine->id;
-			   __entry->seqno = from->fence.seqno;
+			   __entry->seqno = from->global_seqno;
 			   ),
 
 	    TP_printk("dev=%u, sync-from=%u, sync-to=%u, seqno=%u",
@@ -489,9 +489,9 @@ TRACE_EVENT(i915_gem_ring_dispatch,
 	    TP_fast_assign(
 			   __entry->dev = req->i915->drm.primary->index;
 			   __entry->ring = req->engine->id;
-			   __entry->seqno = req->fence.seqno;
+			   __entry->seqno = req->global_seqno;
 			   __entry->flags = flags;
-			   fence_enable_sw_signaling(&req->fence);
+			   dma_fence_enable_sw_signaling(&req->fence);
 			   ),
 
 	    TP_printk("dev=%u, ring=%u, seqno=%u, flags=%x",
@@ -534,7 +534,7 @@ DECLARE_EVENT_CLASS(i915_gem_request,
 	    TP_fast_assign(
 			   __entry->dev = req->i915->drm.primary->index;
 			   __entry->ring = req->engine->id;
-			   __entry->seqno = req->fence.seqno;
+			   __entry->seqno = req->global_seqno;
 			   ),
 
 	    TP_printk("dev=%u, ring=%u, seqno=%u",
@@ -596,7 +596,7 @@ TRACE_EVENT(i915_gem_request_wait_begin,
 	    TP_fast_assign(
 			   __entry->dev = req->i915->drm.primary->index;
 			   __entry->ring = req->engine->id;
-			   __entry->seqno = req->fence.seqno;
+			   __entry->seqno = req->global_seqno;
 			   __entry->blocking =
 				     mutex_is_locked(&req->i915->drm.struct_mutex);
 			   ),
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
new file mode 100644
index 000000000000..a792dcb902b5
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -0,0 +1,638 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+ 
+#include "i915_vma.h"
+
+#include "i915_drv.h"
+#include "intel_ringbuffer.h"
+#include "intel_frontbuffer.h"
+
+#include <drm/drm_gem.h>
+
+static void
+i915_vma_retire(struct i915_gem_active *active,
+		struct drm_i915_gem_request *rq)
+{
+	const unsigned int idx = rq->engine->id;
+	struct i915_vma *vma =
+		container_of(active, struct i915_vma, last_read[idx]);
+	struct drm_i915_gem_object *obj = vma->obj;
+
+	GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx));
+
+	i915_vma_clear_active(vma, idx);
+	if (i915_vma_is_active(vma))
+		return;
+
+	list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
+	if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma)))
+		WARN_ON(i915_vma_unbind(vma));
+
+	GEM_BUG_ON(!i915_gem_object_is_active(obj));
+	if (--obj->active_count)
+		return;
+
+	/* Bump our place on the bound list to keep it roughly in LRU order
+	 * so that we don't steal from recently used but inactive objects
+	 * (unless we are forced to ofc!)
+	 */
+	if (obj->bind_count)
+		list_move_tail(&obj->global_link, &rq->i915->mm.bound_list);
+
+	obj->mm.dirty = true; /* be paranoid  */
+
+	if (i915_gem_object_has_active_reference(obj)) {
+		i915_gem_object_clear_active_reference(obj);
+		i915_gem_object_put(obj);
+	}
+}
+
+static struct i915_vma *
+__i915_vma_create(struct drm_i915_gem_object *obj,
+		  struct i915_address_space *vm,
+		  const struct i915_ggtt_view *view)
+{
+	struct i915_vma *vma;
+	struct rb_node *rb, **p;
+	int i;
+
+	GEM_BUG_ON(vm->closed);
+
+	vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL);
+	if (vma == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&vma->exec_list);
+	for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
+		init_request_active(&vma->last_read[i], i915_vma_retire);
+	init_request_active(&vma->last_fence, NULL);
+	list_add(&vma->vm_link, &vm->unbound_list);
+	vma->vm = vm;
+	vma->obj = obj;
+	vma->size = obj->base.size;
+
+	if (view) {
+		vma->ggtt_view = *view;
+		if (view->type == I915_GGTT_VIEW_PARTIAL) {
+			vma->size = view->params.partial.size;
+			vma->size <<= PAGE_SHIFT;
+		} else if (view->type == I915_GGTT_VIEW_ROTATED) {
+			vma->size =
+				intel_rotation_info_size(&view->params.rotated);
+			vma->size <<= PAGE_SHIFT;
+		}
+	}
+
+	if (i915_is_ggtt(vm)) {
+		vma->flags |= I915_VMA_GGTT;
+		list_add(&vma->obj_link, &obj->vma_list);
+	} else {
+		i915_ppgtt_get(i915_vm_to_ppgtt(vm));
+		list_add_tail(&vma->obj_link, &obj->vma_list);
+	}
+
+	rb = NULL;
+	p = &obj->vma_tree.rb_node;
+	while (*p) {
+		struct i915_vma *pos;
+
+		rb = *p;
+		pos = rb_entry(rb, struct i915_vma, obj_node);
+		if (i915_vma_compare(pos, vm, view) < 0)
+			p = &rb->rb_right;
+		else
+			p = &rb->rb_left;
+	}
+	rb_link_node(&vma->obj_node, rb, p);
+	rb_insert_color(&vma->obj_node, &obj->vma_tree);
+
+	return vma;
+}
+
+struct i915_vma *
+i915_vma_create(struct drm_i915_gem_object *obj,
+		struct i915_address_space *vm,
+		const struct i915_ggtt_view *view)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	GEM_BUG_ON(view && !i915_is_ggtt(vm));
+	GEM_BUG_ON(i915_gem_obj_to_vma(obj, vm, view));
+
+	return __i915_vma_create(obj, vm, view);
+}
+
+/**
+ * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
+ * @vma: VMA to map
+ * @cache_level: mapping cache level
+ * @flags: flags like global or local mapping
+ *
+ * DMA addresses are taken from the scatter-gather table of this object (or of
+ * this VMA in case of non-default GGTT views) and PTE entries set up.
+ * Note that DMA addresses are also the only part of the SG table we care about.
+ */
+int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
+		  u32 flags)
+{
+	u32 bind_flags;
+	u32 vma_flags;
+	int ret;
+
+	if (WARN_ON(flags == 0))
+		return -EINVAL;
+
+	bind_flags = 0;
+	if (flags & PIN_GLOBAL)
+		bind_flags |= I915_VMA_GLOBAL_BIND;
+	if (flags & PIN_USER)
+		bind_flags |= I915_VMA_LOCAL_BIND;
+
+	vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
+	if (flags & PIN_UPDATE)
+		bind_flags |= vma_flags;
+	else
+		bind_flags &= ~vma_flags;
+	if (bind_flags == 0)
+		return 0;
+
+	if (vma_flags == 0 && vma->vm->allocate_va_range) {
+		trace_i915_va_alloc(vma);
+		ret = vma->vm->allocate_va_range(vma->vm,
+						 vma->node.start,
+						 vma->node.size);
+		if (ret)
+			return ret;
+	}
+
+	ret = vma->vm->bind_vma(vma, cache_level, bind_flags);
+	if (ret)
+		return ret;
+
+	vma->flags |= bind_flags;
+	return 0;
+}
+
+void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
+{
+	void __iomem *ptr;
+
+	/* Access through the GTT requires the device to be awake. */
+	assert_rpm_wakelock_held(to_i915(vma->vm->dev));
+
+	lockdep_assert_held(&vma->vm->dev->struct_mutex);
+	if (WARN_ON(!i915_vma_is_map_and_fenceable(vma)))
+		return IO_ERR_PTR(-ENODEV);
+
+	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
+	GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0);
+
+	ptr = vma->iomap;
+	if (ptr == NULL) {
+		ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->mappable,
+					vma->node.start,
+					vma->node.size);
+		if (ptr == NULL)
+			return IO_ERR_PTR(-ENOMEM);
+
+		vma->iomap = ptr;
+	}
+
+	__i915_vma_pin(vma);
+	return ptr;
+}
+
+void i915_vma_unpin_and_release(struct i915_vma **p_vma)
+{
+	struct i915_vma *vma;
+	struct drm_i915_gem_object *obj;
+
+	vma = fetch_and_zero(p_vma);
+	if (!vma)
+		return;
+
+	obj = vma->obj;
+
+	i915_vma_unpin(vma);
+	i915_vma_close(vma);
+
+	__i915_gem_object_release_unless_active(obj);
+}
+
+bool
+i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
+{
+	if (!drm_mm_node_allocated(&vma->node))
+		return false;
+
+	if (vma->node.size < size)
+		return true;
+
+	if (alignment && vma->node.start & (alignment - 1))
+		return true;
+
+	if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
+		return true;
+
+	if (flags & PIN_OFFSET_BIAS &&
+	    vma->node.start < (flags & PIN_OFFSET_MASK))
+		return true;
+
+	if (flags & PIN_OFFSET_FIXED &&
+	    vma->node.start != (flags & PIN_OFFSET_MASK))
+		return true;
+
+	return false;
+}
+
+void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
+{
+	struct drm_i915_gem_object *obj = vma->obj;
+	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+	bool mappable, fenceable;
+	u32 fence_size, fence_alignment;
+
+	fence_size = i915_gem_get_ggtt_size(dev_priv,
+					    vma->size,
+					    i915_gem_object_get_tiling(obj));
+	fence_alignment = i915_gem_get_ggtt_alignment(dev_priv,
+						      vma->size,
+						      i915_gem_object_get_tiling(obj),
+						      true);
+
+	fenceable = (vma->node.size == fence_size &&
+		     (vma->node.start & (fence_alignment - 1)) == 0);
+
+	mappable = (vma->node.start + fence_size <=
+		    dev_priv->ggtt.mappable_end);
+
+	/*
+	 * Explicitly disable for rotated VMA since the display does not
+	 * need the fence and the VMA is not accessible to other users.
+	 */
+	if (mappable && fenceable &&
+	    vma->ggtt_view.type != I915_GGTT_VIEW_ROTATED)
+		vma->flags |= I915_VMA_CAN_FENCE;
+	else
+		vma->flags &= ~I915_VMA_CAN_FENCE;
+}
+
+bool i915_gem_valid_gtt_space(struct i915_vma *vma,
+			      unsigned long cache_level)
+{
+	struct drm_mm_node *gtt_space = &vma->node;
+	struct drm_mm_node *other;
+
+	/*
+	 * On some machines we have to be careful when putting differing types
+	 * of snoopable memory together to avoid the prefetcher crossing memory
+	 * domains and dying. During vm initialisation, we decide whether or not
+	 * these constraints apply and set the drm_mm.color_adjust
+	 * appropriately.
+	 */
+	if (vma->vm->mm.color_adjust == NULL)
+		return true;
+
+	if (!drm_mm_node_allocated(gtt_space))
+		return true;
+
+	if (list_empty(&gtt_space->node_list))
+		return true;
+
+	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
+	if (other->allocated && !other->hole_follows && other->color != cache_level)
+		return false;
+
+	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
+	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
+		return false;
+
+	return true;
+}
+
+/**
+ * i915_vma_insert - finds a slot for the vma in its address space
+ * @vma: the vma
+ * @size: requested size in bytes (can be larger than the VMA)
+ * @alignment: required alignment
+ * @flags: mask of PIN_* flags to use
+ *
+ * First we try to allocate some free space that meets the requirements for
+ * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
+ * preferrably the oldest idle entry to make room for the new VMA.
+ *
+ * Returns:
+ * 0 on success, negative error code otherwise.
+ */
+static int
+i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
+{
+	struct drm_i915_private *dev_priv = to_i915(vma->vm->dev);
+	struct drm_i915_gem_object *obj = vma->obj;
+	u64 start, end;
+	int ret;
+
+	GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
+	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
+
+	size = max(size, vma->size);
+	if (flags & PIN_MAPPABLE)
+		size = i915_gem_get_ggtt_size(dev_priv, size,
+					      i915_gem_object_get_tiling(obj));
+
+	alignment = max(max(alignment, vma->display_alignment),
+			i915_gem_get_ggtt_alignment(dev_priv, size,
+						    i915_gem_object_get_tiling(obj),
+						    flags & PIN_MAPPABLE));
+
+	start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
+
+	end = vma->vm->total;
+	if (flags & PIN_MAPPABLE)
+		end = min_t(u64, end, dev_priv->ggtt.mappable_end);
+	if (flags & PIN_ZONE_4G)
+		end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
+
+	/* If binding the object/GGTT view requires more space than the entire
+	 * aperture has, reject it early before evicting everything in a vain
+	 * attempt to find space.
+	 */
+	if (size > end) {
+		DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n",
+			  size, obj->base.size,
+			  flags & PIN_MAPPABLE ? "mappable" : "total",
+			  end);
+		return -E2BIG;
+	}
+
+	ret = i915_gem_object_pin_pages(obj);
+	if (ret)
+		return ret;
+
+	if (flags & PIN_OFFSET_FIXED) {
+		u64 offset = flags & PIN_OFFSET_MASK;
+		if (offset & (alignment - 1) || offset > end - size) {
+			ret = -EINVAL;
+			goto err_unpin;
+		}
+
+		vma->node.start = offset;
+		vma->node.size = size;
+		vma->node.color = obj->cache_level;
+		ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
+		if (ret) {
+			ret = i915_gem_evict_for_vma(vma);
+			if (ret == 0)
+				ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
+			if (ret)
+				goto err_unpin;
+		}
+	} else {
+		u32 search_flag, alloc_flag;
+
+		if (flags & PIN_HIGH) {
+			search_flag = DRM_MM_SEARCH_BELOW;
+			alloc_flag = DRM_MM_CREATE_TOP;
+		} else {
+			search_flag = DRM_MM_SEARCH_DEFAULT;
+			alloc_flag = DRM_MM_CREATE_DEFAULT;
+		}
+
+		/* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
+		 * so we know that we always have a minimum alignment of 4096.
+		 * The drm_mm range manager is optimised to return results
+		 * with zero alignment, so where possible use the optimal
+		 * path.
+		 */
+		if (alignment <= 4096)
+			alignment = 0;
+
+search_free:
+		ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm,
+							  &vma->node,
+							  size, alignment,
+							  obj->cache_level,
+							  start, end,
+							  search_flag,
+							  alloc_flag);
+		if (ret) {
+			ret = i915_gem_evict_something(vma->vm, size, alignment,
+						       obj->cache_level,
+						       start, end,
+						       flags);
+			if (ret == 0)
+				goto search_free;
+
+			goto err_unpin;
+		}
+
+		GEM_BUG_ON(vma->node.start < start);
+		GEM_BUG_ON(vma->node.start + vma->node.size > end);
+	}
+	GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level));
+
+	list_move_tail(&obj->global_link, &dev_priv->mm.bound_list);
+	list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
+	obj->bind_count++;
+	GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count);
+
+	return 0;
+
+err_unpin:
+	i915_gem_object_unpin_pages(obj);
+	return ret;
+}
+
+int __i915_vma_do_pin(struct i915_vma *vma,
+		      u64 size, u64 alignment, u64 flags)
+{
+	unsigned int bound = vma->flags;
+	int ret;
+
+	lockdep_assert_held(&vma->vm->dev->struct_mutex);
+	GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
+	GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
+
+	if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
+		ret = -EBUSY;
+		goto err;
+	}
+
+	if ((bound & I915_VMA_BIND_MASK) == 0) {
+		ret = i915_vma_insert(vma, size, alignment, flags);
+		if (ret)
+			goto err;
+	}
+
+	ret = i915_vma_bind(vma, vma->obj->cache_level, flags);
+	if (ret)
+		goto err;
+
+	if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND)
+		__i915_vma_set_map_and_fenceable(vma);
+
+	GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
+	return 0;
+
+err:
+	__i915_vma_unpin(vma);
+	return ret;
+}
+
+void i915_vma_destroy(struct i915_vma *vma)
+{
+	GEM_BUG_ON(vma->node.allocated);
+	GEM_BUG_ON(i915_vma_is_active(vma));
+	GEM_BUG_ON(!i915_vma_is_closed(vma));
+	GEM_BUG_ON(vma->fence);
+
+	list_del(&vma->vm_link);
+	if (!i915_vma_is_ggtt(vma))
+		i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
+
+	kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
+}
+
+void i915_vma_close(struct i915_vma *vma)
+{
+	GEM_BUG_ON(i915_vma_is_closed(vma));
+	vma->flags |= I915_VMA_CLOSED;
+
+	list_del(&vma->obj_link);
+	rb_erase(&vma->obj_node, &vma->obj->vma_tree);
+
+	if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma))
+		WARN_ON(i915_vma_unbind(vma));
+}
+
+static void __i915_vma_iounmap(struct i915_vma *vma)
+{
+	GEM_BUG_ON(i915_vma_is_pinned(vma));
+
+	if (vma->iomap == NULL)
+		return;
+
+	io_mapping_unmap(vma->iomap);
+	vma->iomap = NULL;
+}
+
+int i915_vma_unbind(struct i915_vma *vma)
+{
+	struct drm_i915_gem_object *obj = vma->obj;
+	unsigned long active;
+	int ret;
+
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
+	/* First wait upon any activity as retiring the request may
+	 * have side-effects such as unpinning or even unbinding this vma.
+	 */
+	active = i915_vma_get_active(vma);
+	if (active) {
+		int idx;
+
+		/* When a closed VMA is retired, it is unbound - eek.
+		 * In order to prevent it from being recursively closed,
+		 * take a pin on the vma so that the second unbind is
+		 * aborted.
+		 *
+		 * Even more scary is that the retire callback may free
+		 * the object (last active vma). To prevent the explosion
+		 * we defer the actual object free to a worker that can
+		 * only proceed once it acquires the struct_mutex (which
+		 * we currently hold, therefore it cannot free this object
+		 * before we are finished).
+		 */
+		__i915_vma_pin(vma);
+
+		for_each_active(active, idx) {
+			ret = i915_gem_active_retire(&vma->last_read[idx],
+						   &vma->vm->dev->struct_mutex);
+			if (ret)
+				break;
+		}
+
+		__i915_vma_unpin(vma);
+		if (ret)
+			return ret;
+
+		GEM_BUG_ON(i915_vma_is_active(vma));
+	}
+
+	if (i915_vma_is_pinned(vma))
+		return -EBUSY;
+
+	if (!drm_mm_node_allocated(&vma->node))
+		goto destroy;
+
+	GEM_BUG_ON(obj->bind_count == 0);
+	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
+
+	if (i915_vma_is_map_and_fenceable(vma)) {
+		/* release the fence reg _after_ flushing */
+		ret = i915_vma_put_fence(vma);
+		if (ret)
+			return ret;
+
+		/* Force a pagefault for domain tracking on next user access */
+		i915_gem_release_mmap(obj);
+
+		__i915_vma_iounmap(vma);
+		vma->flags &= ~I915_VMA_CAN_FENCE;
+	}
+
+	if (likely(!vma->vm->closed)) {
+		trace_i915_vma_unbind(vma);
+		vma->vm->unbind_vma(vma);
+	}
+	vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
+
+	drm_mm_remove_node(&vma->node);
+	list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
+
+	if (vma->pages != obj->mm.pages) {
+		GEM_BUG_ON(!vma->pages);
+		sg_free_table(vma->pages);
+		kfree(vma->pages);
+	}
+	vma->pages = NULL;
+
+	/* Since the unbound list is global, only move to that list if
+	 * no more VMAs exist. */
+	if (--obj->bind_count == 0)
+		list_move_tail(&obj->global_link,
+			       &to_i915(obj->base.dev)->mm.unbound_list);
+
+	/* And finally now the object is completely decoupled from this vma,
+	 * we can drop its hold on the backing storage and allow it to be
+	 * reaped by the shrinker.
+	 */
+	i915_gem_object_unpin_pages(obj);
+
+destroy:
+	if (unlikely(i915_vma_is_closed(vma)))
+		i915_vma_destroy(vma);
+
+	return 0;
+}
+
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
new file mode 100644
index 000000000000..85446f0b0b3f
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -0,0 +1,341 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_VMA_H__
+#define __I915_VMA_H__
+
+#include <linux/io-mapping.h>
+
+#include <drm/drm_mm.h>
+
+#include "i915_gem_gtt.h"
+#include "i915_gem_fence_reg.h"
+#include "i915_gem_object.h"
+#include "i915_gem_request.h"
+
+
+enum i915_cache_level;
+
+/**
+ * A VMA represents a GEM BO that is bound into an address space. Therefore, a
+ * VMA's presence cannot be guaranteed before binding, or after unbinding the
+ * object into/from the address space.
+ *
+ * To make things as simple as possible (ie. no refcounting), a VMA's lifetime
+ * will always be <= an objects lifetime. So object refcounting should cover us.
+ */
+struct i915_vma {
+	struct drm_mm_node node;
+	struct drm_i915_gem_object *obj;
+	struct i915_address_space *vm;
+	struct drm_i915_fence_reg *fence;
+	struct sg_table *pages;
+	void __iomem *iomap;
+	u64 size;
+	u64 display_alignment;
+
+	unsigned int flags;
+	/**
+	 * How many users have pinned this object in GTT space. The following
+	 * users can each hold at most one reference: pwrite/pread, execbuffer
+	 * (objects are not allowed multiple times for the same batchbuffer),
+	 * and the framebuffer code. When switching/pageflipping, the
+	 * framebuffer code has at most two buffers pinned per crtc.
+	 *
+	 * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
+	 * bits with absolutely no headroom. So use 4 bits.
+	 */
+#define I915_VMA_PIN_MASK 0xf
+#define I915_VMA_PIN_OVERFLOW	BIT(5)
+
+	/** Flags and address space this VMA is bound to */
+#define I915_VMA_GLOBAL_BIND	BIT(6)
+#define I915_VMA_LOCAL_BIND	BIT(7)
+#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
+
+#define I915_VMA_GGTT		BIT(8)
+#define I915_VMA_CAN_FENCE	BIT(9)
+#define I915_VMA_CLOSED		BIT(10)
+
+	unsigned int active;
+	struct i915_gem_active last_read[I915_NUM_ENGINES];
+	struct i915_gem_active last_fence;
+
+	/**
+	 * Support different GGTT views into the same object.
+	 * This means there can be multiple VMA mappings per object and per VM.
+	 * i915_ggtt_view_type is used to distinguish between those entries.
+	 * The default one of zero (I915_GGTT_VIEW_NORMAL) is default and also
+	 * assumed in GEM functions which take no ggtt view parameter.
+	 */
+	struct i915_ggtt_view ggtt_view;
+
+	/** This object's place on the active/inactive lists */
+	struct list_head vm_link;
+
+	struct list_head obj_link; /* Link in the object's VMA list */
+	struct rb_node obj_node;
+
+	/** This vma's place in the batchbuffer or on the eviction list */
+	struct list_head exec_list;
+
+	/**
+	 * Used for performing relocations during execbuffer insertion.
+	 */
+	struct hlist_node exec_node;
+	unsigned long exec_handle;
+	struct drm_i915_gem_exec_object2 *exec_entry;
+};
+
+struct i915_vma *
+i915_vma_create(struct drm_i915_gem_object *obj,
+		struct i915_address_space *vm,
+		const struct i915_ggtt_view *view);
+
+void i915_vma_unpin_and_release(struct i915_vma **p_vma);
+
+static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
+{
+	return vma->flags & I915_VMA_GGTT;
+}
+
+static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
+{
+	return vma->flags & I915_VMA_CAN_FENCE;
+}
+
+static inline bool i915_vma_is_closed(const struct i915_vma *vma)
+{
+	return vma->flags & I915_VMA_CLOSED;
+}
+
+static inline unsigned int i915_vma_get_active(const struct i915_vma *vma)
+{
+	return vma->active;
+}
+
+static inline bool i915_vma_is_active(const struct i915_vma *vma)
+{
+	return i915_vma_get_active(vma);
+}
+
+static inline void i915_vma_set_active(struct i915_vma *vma,
+				       unsigned int engine)
+{
+	vma->active |= BIT(engine);
+}
+
+static inline void i915_vma_clear_active(struct i915_vma *vma,
+					 unsigned int engine)
+{
+	vma->active &= ~BIT(engine);
+}
+
+static inline bool i915_vma_has_active_engine(const struct i915_vma *vma,
+					      unsigned int engine)
+{
+	return vma->active & BIT(engine);
+}
+
+static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
+{
+	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
+	GEM_BUG_ON(!vma->node.allocated);
+	GEM_BUG_ON(upper_32_bits(vma->node.start));
+	GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1));
+	return lower_32_bits(vma->node.start);
+}
+
+static inline struct i915_vma *i915_vma_get(struct i915_vma *vma)
+{
+	i915_gem_object_get(vma->obj);
+	return vma;
+}
+
+static inline void i915_vma_put(struct i915_vma *vma)
+{
+	i915_gem_object_put(vma->obj);
+}
+
+static inline long
+i915_vma_compare(struct i915_vma *vma,
+		 struct i915_address_space *vm,
+		 const struct i915_ggtt_view *view)
+{
+	GEM_BUG_ON(view && !i915_is_ggtt(vm));
+
+	if (vma->vm != vm)
+		return vma->vm - vm;
+
+	if (!view)
+		return vma->ggtt_view.type;
+
+	if (vma->ggtt_view.type != view->type)
+		return vma->ggtt_view.type - view->type;
+
+	return memcmp(&vma->ggtt_view.params,
+		      &view->params,
+		      sizeof(view->params));
+}
+
+int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
+		  u32 flags);
+bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level);
+bool
+i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags);
+void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
+int __must_check i915_vma_unbind(struct i915_vma *vma);
+void i915_vma_close(struct i915_vma *vma);
+void i915_vma_destroy(struct i915_vma *vma);
+
+int __i915_vma_do_pin(struct i915_vma *vma,
+		      u64 size, u64 alignment, u64 flags);
+static inline int __must_check
+i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
+{
+	BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
+	BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
+	BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
+
+	/* Pin early to prevent the shrinker/eviction logic from destroying
+	 * our vma as we insert and bind.
+	 */
+	if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0))
+		return 0;
+
+	return __i915_vma_do_pin(vma, size, alignment, flags);
+}
+
+static inline int i915_vma_pin_count(const struct i915_vma *vma)
+{
+	return vma->flags & I915_VMA_PIN_MASK;
+}
+
+static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
+{
+	return i915_vma_pin_count(vma);
+}
+
+static inline void __i915_vma_pin(struct i915_vma *vma)
+{
+	vma->flags++;
+	GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
+}
+
+static inline void __i915_vma_unpin(struct i915_vma *vma)
+{
+	GEM_BUG_ON(!i915_vma_is_pinned(vma));
+	vma->flags--;
+}
+
+static inline void i915_vma_unpin(struct i915_vma *vma)
+{
+	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
+	__i915_vma_unpin(vma);
+}
+
+/**
+ * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture
+ * @vma: VMA to iomap
+ *
+ * The passed in VMA has to be pinned in the global GTT mappable region.
+ * An extra pinning of the VMA is acquired for the return iomapping,
+ * the caller must call i915_vma_unpin_iomap to relinquish the pinning
+ * after the iomapping is no longer required.
+ *
+ * Callers must hold the struct_mutex.
+ *
+ * Returns a valid iomapped pointer or ERR_PTR.
+ */
+void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
+#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x))
+
+/**
+ * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap
+ * @vma: VMA to unpin
+ *
+ * Unpins the previously iomapped VMA from i915_vma_pin_iomap().
+ *
+ * Callers must hold the struct_mutex. This function is only valid to be
+ * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap().
+ */
+static inline void i915_vma_unpin_iomap(struct i915_vma *vma)
+{
+	lockdep_assert_held(&vma->vm->dev->struct_mutex);
+	GEM_BUG_ON(vma->iomap == NULL);
+	i915_vma_unpin(vma);
+}
+
+static inline struct page *i915_vma_first_page(struct i915_vma *vma)
+{
+	GEM_BUG_ON(!vma->pages);
+	return sg_page(vma->pages->sgl);
+}
+
+/**
+ * i915_vma_pin_fence - pin fencing state
+ * @vma: vma to pin fencing for
+ *
+ * This pins the fencing state (whether tiled or untiled) to make sure the
+ * vma (and its object) is ready to be used as a scanout target. Fencing
+ * status must be synchronize first by calling i915_vma_get_fence():
+ *
+ * The resulting fence pin reference must be released again with
+ * i915_vma_unpin_fence().
+ *
+ * Returns:
+ *
+ * True if the vma has a fence, false otherwise.
+ */
+static inline bool
+i915_vma_pin_fence(struct i915_vma *vma)
+{
+	lockdep_assert_held(&vma->vm->dev->struct_mutex);
+	if (vma->fence) {
+		vma->fence->pin_count++;
+		return true;
+	} else
+		return false;
+}
+
+/**
+ * i915_vma_unpin_fence - unpin fencing state
+ * @vma: vma to unpin fencing for
+ *
+ * This releases the fence pin reference acquired through
+ * i915_vma_pin_fence. It will handle both objects with and without an
+ * attached fence correctly, callers do not need to distinguish this.
+ */
+static inline void
+i915_vma_unpin_fence(struct i915_vma *vma)
+{
+	lockdep_assert_held(&vma->vm->dev->struct_mutex);
+	if (vma->fence) {
+		GEM_BUG_ON(vma->fence->pin_count <= 0);
+		vma->fence->pin_count--;
+	}
+}
+
+#endif
+
diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c
index b82de3072d4f..dbe9fb41ae53 100644
--- a/drivers/gpu/drm/i915/intel_atomic_plane.c
+++ b/drivers/gpu/drm/i915/intel_atomic_plane.c
@@ -84,7 +84,6 @@ intel_plane_duplicate_state(struct drm_plane *plane)
 	state = &intel_state->base;
 
 	__drm_atomic_helper_plane_duplicate_state(plane, state);
-	intel_state->wait_req = NULL;
 
 	return state;
 }
@@ -101,13 +100,13 @@ void
 intel_plane_destroy_state(struct drm_plane *plane,
 			  struct drm_plane_state *state)
 {
-	WARN_ON(state && to_intel_plane_state(state)->wait_req);
 	drm_atomic_helper_plane_destroy_state(plane, state);
 }
 
 static int intel_plane_atomic_check(struct drm_plane *plane,
 				    struct drm_plane_state *state)
 {
+	struct drm_i915_private *dev_priv = to_i915(plane->dev);
 	struct drm_crtc *crtc = state->crtc;
 	struct intel_crtc *intel_crtc;
 	struct intel_crtc_state *crtc_state;
@@ -142,10 +141,11 @@ static int intel_plane_atomic_check(struct drm_plane *plane,
 	intel_state->clip.y2 =
 		crtc_state->base.enable ? crtc_state->pipe_src_h : 0;
 
-	if (state->fb && intel_rotation_90_or_270(state->rotation)) {
-		char *format_name;
-		if (!(state->fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
-			state->fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED)) {
+	if (state->fb && drm_rotation_90_or_270(state->rotation)) {
+		struct drm_format_name_buf format_name;
+
+		if (state->fb->modifier != I915_FORMAT_MOD_Y_TILED &&
+		    state->fb->modifier != I915_FORMAT_MOD_Yf_TILED) {
 			DRM_DEBUG_KMS("Y/Yf tiling required for 90/270!\n");
 			return -EINVAL;
 		}
@@ -158,9 +158,9 @@ static int intel_plane_atomic_check(struct drm_plane *plane,
 		switch (state->fb->pixel_format) {
 		case DRM_FORMAT_C8:
 		case DRM_FORMAT_RGB565:
-			format_name = drm_get_format_name(state->fb->pixel_format);
-			DRM_DEBUG_KMS("Unsupported pixel format %s for 90/270!\n", format_name);
-			kfree(format_name);
+			DRM_DEBUG_KMS("Unsupported pixel format %s for 90/270!\n",
+			              drm_get_format_name(state->fb->pixel_format,
+			                                  &format_name));
 			return -EINVAL;
 
 		default:
@@ -168,6 +168,14 @@ static int intel_plane_atomic_check(struct drm_plane *plane,
 		}
 	}
 
+	/* CHV ignores the mirror bit when the rotate bit is set :( */
+	if (IS_CHERRYVIEW(dev_priv) &&
+	    state->rotation & DRM_ROTATE_180 &&
+	    state->rotation & DRM_REFLECT_X) {
+		DRM_DEBUG_KMS("Cannot rotate and reflect at the same time\n");
+		return -EINVAL;
+	}
+
 	intel_state->base.visible = false;
 	ret = intel_plane->check_plane(plane, crtc_state, intel_state);
 	if (ret)
diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c
index 6c70a5bfd7d8..49f10538d4aa 100644
--- a/drivers/gpu/drm/i915/intel_audio.c
+++ b/drivers/gpu/drm/i915/intel_audio.c
@@ -57,6 +57,63 @@
  * struct &i915_audio_component_audio_ops @audio_ops is called from i915 driver.
  */
 
+/* DP N/M table */
+#define LC_540M	540000
+#define LC_270M	270000
+#define LC_162M	162000
+
+struct dp_aud_n_m {
+	int sample_rate;
+	int clock;
+	u16 m;
+	u16 n;
+};
+
+/* Values according to DP 1.4 Table 2-104 */
+static const struct dp_aud_n_m dp_aud_n_m[] = {
+	{ 32000, LC_162M, 1024, 10125 },
+	{ 44100, LC_162M, 784, 5625 },
+	{ 48000, LC_162M, 512, 3375 },
+	{ 64000, LC_162M, 2048, 10125 },
+	{ 88200, LC_162M, 1568, 5625 },
+	{ 96000, LC_162M, 1024, 3375 },
+	{ 128000, LC_162M, 4096, 10125 },
+	{ 176400, LC_162M, 3136, 5625 },
+	{ 192000, LC_162M, 2048, 3375 },
+	{ 32000, LC_270M, 1024, 16875 },
+	{ 44100, LC_270M, 784, 9375 },
+	{ 48000, LC_270M, 512, 5625 },
+	{ 64000, LC_270M, 2048, 16875 },
+	{ 88200, LC_270M, 1568, 9375 },
+	{ 96000, LC_270M, 1024, 5625 },
+	{ 128000, LC_270M, 4096, 16875 },
+	{ 176400, LC_270M, 3136, 9375 },
+	{ 192000, LC_270M, 2048, 5625 },
+	{ 32000, LC_540M, 1024, 33750 },
+	{ 44100, LC_540M, 784, 18750 },
+	{ 48000, LC_540M, 512, 11250 },
+	{ 64000, LC_540M, 2048, 33750 },
+	{ 88200, LC_540M, 1568, 18750 },
+	{ 96000, LC_540M, 1024, 11250 },
+	{ 128000, LC_540M, 4096, 33750 },
+	{ 176400, LC_540M, 3136, 18750 },
+	{ 192000, LC_540M, 2048, 11250 },
+};
+
+static const struct dp_aud_n_m *
+audio_config_dp_get_n_m(struct intel_crtc *intel_crtc, int rate)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dp_aud_n_m); i++) {
+		if (rate == dp_aud_n_m[i].sample_rate &&
+		    intel_crtc->config->port_clock == dp_aud_n_m[i].clock)
+			return &dp_aud_n_m[i];
+	}
+
+	return NULL;
+}
+
 static const struct {
 	int clock;
 	u32 config;
@@ -81,7 +138,7 @@ static const struct {
 	int clock;
 	int n;
 	int cts;
-} aud_ncts[] = {
+} hdmi_aud_ncts[] = {
 	{ 44100, TMDS_296M, 4459, 234375 },
 	{ 44100, TMDS_297M, 4704, 247500 },
 	{ 48000, TMDS_296M, 5824, 281250 },
@@ -121,45 +178,20 @@ static u32 audio_config_hdmi_pixel_clock(const struct drm_display_mode *adjusted
 	return hdmi_audio_clock[i].config;
 }
 
-static int audio_config_get_n(const struct drm_display_mode *mode, int rate)
+static int audio_config_hdmi_get_n(const struct drm_display_mode *adjusted_mode,
+				   int rate)
 {
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(aud_ncts); i++) {
-		if ((rate == aud_ncts[i].sample_rate) &&
-			(mode->clock == aud_ncts[i].clock)) {
-			return aud_ncts[i].n;
+	for (i = 0; i < ARRAY_SIZE(hdmi_aud_ncts); i++) {
+		if (rate == hdmi_aud_ncts[i].sample_rate &&
+		    adjusted_mode->crtc_clock == hdmi_aud_ncts[i].clock) {
+			return hdmi_aud_ncts[i].n;
 		}
 	}
 	return 0;
 }
 
-static uint32_t audio_config_setup_n_reg(int n, uint32_t val)
-{
-	int n_low, n_up;
-	uint32_t tmp = val;
-
-	n_low = n & 0xfff;
-	n_up = (n >> 12) & 0xff;
-	tmp &= ~(AUD_CONFIG_UPPER_N_MASK | AUD_CONFIG_LOWER_N_MASK);
-	tmp |= ((n_up << AUD_CONFIG_UPPER_N_SHIFT) |
-			(n_low << AUD_CONFIG_LOWER_N_SHIFT) |
-			AUD_CONFIG_N_PROG_ENABLE);
-	return tmp;
-}
-
-/* check whether N/CTS/M need be set manually */
-static bool audio_rate_need_prog(struct intel_crtc *crtc,
-				 const struct drm_display_mode *mode)
-{
-	if (((mode->clock == TMDS_297M) ||
-		 (mode->clock == TMDS_296M)) &&
-		intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI))
-		return true;
-	else
-		return false;
-}
-
 static bool intel_eld_uptodate(struct drm_connector *connector,
 			       i915_reg_t reg_eldv, uint32_t bits_eldv,
 			       i915_reg_t reg_elda, uint32_t bits_elda,
@@ -245,6 +277,100 @@ static void g4x_audio_codec_enable(struct drm_connector *connector,
 	I915_WRITE(G4X_AUD_CNTL_ST, tmp);
 }
 
+static void
+hsw_dp_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
+			   const struct drm_display_mode *adjusted_mode)
+{
+	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
+	struct i915_audio_component *acomp = dev_priv->audio_component;
+	int rate = acomp ? acomp->aud_sample_rate[port] : 0;
+	const struct dp_aud_n_m *nm = audio_config_dp_get_n_m(intel_crtc, rate);
+	enum pipe pipe = intel_crtc->pipe;
+	u32 tmp;
+
+	if (nm)
+		DRM_DEBUG_KMS("using Maud %u, Naud %u\n", nm->m, nm->n);
+	else
+		DRM_DEBUG_KMS("using automatic Maud, Naud\n");
+
+	tmp = I915_READ(HSW_AUD_CFG(pipe));
+	tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
+	tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
+	tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
+	tmp |= AUD_CONFIG_N_VALUE_INDEX;
+
+	if (nm) {
+		tmp &= ~AUD_CONFIG_N_MASK;
+		tmp |= AUD_CONFIG_N(nm->n);
+		tmp |= AUD_CONFIG_N_PROG_ENABLE;
+	}
+
+	I915_WRITE(HSW_AUD_CFG(pipe), tmp);
+
+	tmp = I915_READ(HSW_AUD_M_CTS_ENABLE(pipe));
+	tmp &= ~AUD_CONFIG_M_MASK;
+	tmp &= ~AUD_M_CTS_M_VALUE_INDEX;
+	tmp &= ~AUD_M_CTS_M_PROG_ENABLE;
+
+	if (nm) {
+		tmp |= nm->m;
+		tmp |= AUD_M_CTS_M_VALUE_INDEX;
+		tmp |= AUD_M_CTS_M_PROG_ENABLE;
+	}
+
+	I915_WRITE(HSW_AUD_M_CTS_ENABLE(pipe), tmp);
+}
+
+static void
+hsw_hdmi_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
+			     const struct drm_display_mode *adjusted_mode)
+{
+	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
+	struct i915_audio_component *acomp = dev_priv->audio_component;
+	int rate = acomp ? acomp->aud_sample_rate[port] : 0;
+	enum pipe pipe = intel_crtc->pipe;
+	int n;
+	u32 tmp;
+
+	tmp = I915_READ(HSW_AUD_CFG(pipe));
+	tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
+	tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
+	tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
+	tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
+
+	n = audio_config_hdmi_get_n(adjusted_mode, rate);
+	if (n != 0) {
+		DRM_DEBUG_KMS("using N %d\n", n);
+
+		tmp &= ~AUD_CONFIG_N_MASK;
+		tmp |= AUD_CONFIG_N(n);
+		tmp |= AUD_CONFIG_N_PROG_ENABLE;
+	} else {
+		DRM_DEBUG_KMS("using automatic N\n");
+	}
+
+	I915_WRITE(HSW_AUD_CFG(pipe), tmp);
+
+	/*
+	 * Let's disable "Enable CTS or M Prog bit"
+	 * and let HW calculate the value
+	 */
+	tmp = I915_READ(HSW_AUD_M_CTS_ENABLE(pipe));
+	tmp &= ~AUD_M_CTS_M_PROG_ENABLE;
+	tmp &= ~AUD_M_CTS_M_VALUE_INDEX;
+	I915_WRITE(HSW_AUD_M_CTS_ENABLE(pipe), tmp);
+}
+
+static void
+hsw_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
+			const struct drm_display_mode *adjusted_mode)
+{
+	if (intel_crtc_has_dp_encoder(intel_crtc->config))
+		hsw_dp_audio_config_update(intel_crtc, port, adjusted_mode);
+	else
+		hsw_hdmi_audio_config_update(intel_crtc, port, adjusted_mode);
+}
+
 static void hsw_audio_codec_disable(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
@@ -276,20 +402,16 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder)
 }
 
 static void hsw_audio_codec_enable(struct drm_connector *connector,
-				   struct intel_encoder *encoder,
+				   struct intel_encoder *intel_encoder,
 				   const struct drm_display_mode *adjusted_mode)
 {
 	struct drm_i915_private *dev_priv = to_i915(connector->dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+	struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
 	enum pipe pipe = intel_crtc->pipe;
-	struct i915_audio_component *acomp = dev_priv->audio_component;
+	enum port port = intel_encoder->port;
 	const uint8_t *eld = connector->eld;
-	struct intel_digital_port *intel_dig_port =
-		enc_to_dig_port(&encoder->base);
-	enum port port = intel_dig_port->port;
 	uint32_t tmp;
 	int len, i;
-	int n, rate;
 
 	DRM_DEBUG_KMS("Enable audio codec on pipe %c, %u bytes ELD\n",
 		      pipe_name(pipe), drm_eld_size(eld));
@@ -325,42 +447,17 @@ static void hsw_audio_codec_enable(struct drm_connector *connector,
 	I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp);
 
 	/* Enable timestamps */
-	tmp = I915_READ(HSW_AUD_CFG(pipe));
-	tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
-	tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
-	if (intel_crtc_has_dp_encoder(intel_crtc->config))
-		tmp |= AUD_CONFIG_N_VALUE_INDEX;
-	else
-		tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
-
-	tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
-	if (audio_rate_need_prog(intel_crtc, adjusted_mode)) {
-		if (!acomp)
-			rate = 0;
-		else if (port >= PORT_A && port <= PORT_E)
-			rate = acomp->aud_sample_rate[port];
-		else {
-			DRM_ERROR("invalid port: %d\n", port);
-			rate = 0;
-		}
-		n = audio_config_get_n(adjusted_mode, rate);
-		if (n != 0)
-			tmp = audio_config_setup_n_reg(n, tmp);
-		else
-			DRM_DEBUG_KMS("no suitable N value is found\n");
-	}
-
-	I915_WRITE(HSW_AUD_CFG(pipe), tmp);
+	hsw_audio_config_update(intel_crtc, port, adjusted_mode);
 
 	mutex_unlock(&dev_priv->av_mutex);
 }
 
-static void ilk_audio_codec_disable(struct intel_encoder *encoder)
+static void ilk_audio_codec_disable(struct intel_encoder *intel_encoder)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
-	enum port port = enc_to_dig_port(&encoder->base)->port;
+	struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
+	struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
 	enum pipe pipe = intel_crtc->pipe;
+	enum port port = intel_encoder->port;
 	uint32_t tmp, eldv;
 	i915_reg_t aud_config, aud_cntrl_st2;
 
@@ -400,13 +497,13 @@ static void ilk_audio_codec_disable(struct intel_encoder *encoder)
 }
 
 static void ilk_audio_codec_enable(struct drm_connector *connector,
-				   struct intel_encoder *encoder,
+				   struct intel_encoder *intel_encoder,
 				   const struct drm_display_mode *adjusted_mode)
 {
 	struct drm_i915_private *dev_priv = to_i915(connector->dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
-	enum port port = enc_to_dig_port(&encoder->base)->port;
+	struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
 	enum pipe pipe = intel_crtc->pipe;
+	enum port port = intel_encoder->port;
 	uint8_t *eld = connector->eld;
 	uint32_t tmp, eldv;
 	int len, i;
@@ -425,13 +522,13 @@ static void ilk_audio_codec_enable(struct drm_connector *connector,
 	 * infrastructure is not there yet.
 	 */
 
-	if (HAS_PCH_IBX(connector->dev)) {
+	if (HAS_PCH_IBX(dev_priv)) {
 		hdmiw_hdmiedid = IBX_HDMIW_HDMIEDID(pipe);
 		aud_config = IBX_AUD_CFG(pipe);
 		aud_cntl_st = IBX_AUD_CNTL_ST(pipe);
 		aud_cntrl_st2 = IBX_AUD_CNTL_ST2;
-	} else if (IS_VALLEYVIEW(connector->dev) ||
-		   IS_CHERRYVIEW(connector->dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv) ||
+		   IS_CHERRYVIEW(dev_priv)) {
 		hdmiw_hdmiedid = VLV_HDMIW_HDMIEDID(pipe);
 		aud_config = VLV_AUD_CFG(pipe);
 		aud_cntl_st = VLV_AUD_CNTL_ST(pipe);
@@ -480,24 +577,26 @@ static void ilk_audio_codec_enable(struct drm_connector *connector,
 /**
  * intel_audio_codec_enable - Enable the audio codec for HD audio
  * @intel_encoder: encoder on which to enable audio
+ * @crtc_state: pointer to the current crtc state.
+ * @conn_state: pointer to the current connector state.
  *
  * The enable sequences may only be performed after enabling the transcoder and
  * port, and after completed link training.
  */
-void intel_audio_codec_enable(struct intel_encoder *intel_encoder)
+void intel_audio_codec_enable(struct intel_encoder *intel_encoder,
+			      const struct intel_crtc_state *crtc_state,
+			      const struct drm_connector_state *conn_state)
 {
 	struct drm_encoder *encoder = &intel_encoder->base;
-	struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
-	const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
+	const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
 	struct drm_connector *connector;
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
 	struct i915_audio_component *acomp = dev_priv->audio_component;
-	struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
-	enum port port = intel_dig_port->port;
+	enum port port = intel_encoder->port;
+	enum pipe pipe = to_intel_crtc(crtc_state->base.crtc)->pipe;
 
-	connector = drm_select_eld(encoder);
-	if (!connector)
+	connector = conn_state->connector;
+	if (!connector || !connector->eld[0])
 		return;
 
 	DRM_DEBUG_DRIVER("ELD on [CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
@@ -508,7 +607,7 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder)
 
 	/* ELD Conn_Type */
 	connector->eld[5] &= ~(3 << 2);
-	if (intel_crtc_has_dp_encoder(crtc->config))
+	if (intel_crtc_has_dp_encoder(crtc_state))
 		connector->eld[5] |= (1 << 2);
 
 	connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2;
@@ -518,13 +617,19 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder)
 						     adjusted_mode);
 
 	mutex_lock(&dev_priv->av_mutex);
-	intel_dig_port->audio_connector = connector;
+	intel_encoder->audio_connector = connector;
+
 	/* referred in audio callbacks */
-	dev_priv->dig_port_map[port] = intel_encoder;
+	dev_priv->av_enc_map[pipe] = intel_encoder;
 	mutex_unlock(&dev_priv->av_mutex);
 
+	/* audio drivers expect pipe = -1 to indicate Non-MST cases */
+	if (intel_encoder->type != INTEL_OUTPUT_DP_MST)
+		pipe = -1;
+
 	if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify)
-		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, (int) port);
+		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
+						 (int) port, (int) pipe);
 }
 
 /**
@@ -537,22 +642,27 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder)
 void intel_audio_codec_disable(struct intel_encoder *intel_encoder)
 {
 	struct drm_encoder *encoder = &intel_encoder->base;
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
 	struct i915_audio_component *acomp = dev_priv->audio_component;
-	struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
-	enum port port = intel_dig_port->port;
+	enum port port = intel_encoder->port;
+	struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
+	enum pipe pipe = crtc->pipe;
 
 	if (dev_priv->display.audio_codec_disable)
 		dev_priv->display.audio_codec_disable(intel_encoder);
 
 	mutex_lock(&dev_priv->av_mutex);
-	intel_dig_port->audio_connector = NULL;
-	dev_priv->dig_port_map[port] = NULL;
+	intel_encoder->audio_connector = NULL;
+	dev_priv->av_enc_map[pipe] = NULL;
 	mutex_unlock(&dev_priv->av_mutex);
 
+	/* audio drivers expect pipe = -1 to indicate Non-MST cases */
+	if (intel_encoder->type != INTEL_OUTPUT_DP_MST)
+		pipe = -1;
+
 	if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify)
-		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, (int) port);
+		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
+						 (int) port, (int) pipe);
 }
 
 /**
@@ -627,74 +737,68 @@ static int i915_audio_component_get_cdclk_freq(struct device *kdev)
 	return dev_priv->cdclk_freq;
 }
 
-static int i915_audio_component_sync_audio_rate(struct device *kdev,
-						int port, int rate)
+static struct intel_encoder *get_saved_enc(struct drm_i915_private *dev_priv,
+					       int port, int pipe)
+{
+
+	if (WARN_ON(pipe >= I915_MAX_PIPES))
+		return NULL;
+
+	/* MST */
+	if (pipe >= 0)
+		return dev_priv->av_enc_map[pipe];
+
+	/* Non-MST */
+	for_each_pipe(dev_priv, pipe) {
+		struct intel_encoder *encoder;
+
+		encoder = dev_priv->av_enc_map[pipe];
+		if (encoder == NULL)
+			continue;
+
+		if (port == encoder->port)
+			return encoder;
+	}
+
+	return NULL;
+}
+
+static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
+						int pipe, int rate)
 {
 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
 	struct intel_encoder *intel_encoder;
 	struct intel_crtc *crtc;
-	struct drm_display_mode *mode;
+	struct drm_display_mode *adjusted_mode;
 	struct i915_audio_component *acomp = dev_priv->audio_component;
-	enum pipe pipe = INVALID_PIPE;
-	u32 tmp;
-	int n;
 	int err = 0;
 
-	/* HSW, BDW, SKL, KBL need this fix */
-	if (!IS_SKYLAKE(dev_priv) &&
-	    !IS_KABYLAKE(dev_priv) &&
-	    !IS_BROADWELL(dev_priv) &&
-	    !IS_HASWELL(dev_priv))
+	if (!HAS_DDI(dev_priv))
 		return 0;
 
 	i915_audio_component_get_power(kdev);
 	mutex_lock(&dev_priv->av_mutex);
+
 	/* 1. get the pipe */
-	intel_encoder = dev_priv->dig_port_map[port];
-	/* intel_encoder might be NULL for DP MST */
+	intel_encoder = get_saved_enc(dev_priv, port, pipe);
 	if (!intel_encoder || !intel_encoder->base.crtc ||
-	    intel_encoder->type != INTEL_OUTPUT_HDMI) {
-		DRM_DEBUG_KMS("no valid port %c\n", port_name(port));
+	    (intel_encoder->type != INTEL_OUTPUT_HDMI &&
+	     intel_encoder->type != INTEL_OUTPUT_DP)) {
+		DRM_DEBUG_KMS("Not valid for port %c\n", port_name(port));
 		err = -ENODEV;
 		goto unlock;
 	}
+
+	/* pipe passed from the audio driver will be -1 for Non-MST case */
 	crtc = to_intel_crtc(intel_encoder->base.crtc);
 	pipe = crtc->pipe;
-	if (pipe == INVALID_PIPE) {
-		DRM_DEBUG_KMS("no pipe for the port %c\n", port_name(port));
-		err = -ENODEV;
-		goto unlock;
-	}
 
-	DRM_DEBUG_KMS("pipe %c connects port %c\n",
-				  pipe_name(pipe), port_name(port));
-	mode = &crtc->config->base.adjusted_mode;
+	adjusted_mode = &crtc->config->base.adjusted_mode;
 
 	/* port must be valid now, otherwise the pipe will be invalid */
 	acomp->aud_sample_rate[port] = rate;
 
-	/* 2. check whether to set the N/CTS/M manually or not */
-	if (!audio_rate_need_prog(crtc, mode)) {
-		tmp = I915_READ(HSW_AUD_CFG(pipe));
-		tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
-		I915_WRITE(HSW_AUD_CFG(pipe), tmp);
-		goto unlock;
-	}
-
-	n = audio_config_get_n(mode, rate);
-	if (n == 0) {
-		DRM_DEBUG_KMS("Using automatic mode for N value on port %c\n",
-					  port_name(port));
-		tmp = I915_READ(HSW_AUD_CFG(pipe));
-		tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
-		I915_WRITE(HSW_AUD_CFG(pipe), tmp);
-		goto unlock;
-	}
-
-	/* 3. set the N/CTS/M */
-	tmp = I915_READ(HSW_AUD_CFG(pipe));
-	tmp = audio_config_setup_n_reg(n, tmp);
-	I915_WRITE(HSW_AUD_CFG(pipe), tmp);
+	hsw_audio_config_update(crtc, port, adjusted_mode);
 
  unlock:
 	mutex_unlock(&dev_priv->av_mutex);
@@ -703,27 +807,29 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev,
 }
 
 static int i915_audio_component_get_eld(struct device *kdev, int port,
-					bool *enabled,
+					int pipe, bool *enabled,
 					unsigned char *buf, int max_bytes)
 {
 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
 	struct intel_encoder *intel_encoder;
-	struct intel_digital_port *intel_dig_port;
 	const u8 *eld;
 	int ret = -EINVAL;
 
 	mutex_lock(&dev_priv->av_mutex);
-	intel_encoder = dev_priv->dig_port_map[port];
-	/* intel_encoder might be NULL for DP MST */
-	if (intel_encoder) {
-		ret = 0;
-		intel_dig_port = enc_to_dig_port(&intel_encoder->base);
-		*enabled = intel_dig_port->audio_connector != NULL;
-		if (*enabled) {
-			eld = intel_dig_port->audio_connector->eld;
-			ret = drm_eld_size(eld);
-			memcpy(buf, eld, min(max_bytes, ret));
-		}
+
+	intel_encoder = get_saved_enc(dev_priv, port, pipe);
+	if (!intel_encoder) {
+		DRM_DEBUG_KMS("Not valid for port %c\n", port_name(port));
+		mutex_unlock(&dev_priv->av_mutex);
+		return ret;
+	}
+
+	ret = 0;
+	*enabled = intel_encoder->audio_connector != NULL;
+	if (*enabled) {
+		eld = intel_encoder->audio_connector->eld;
+		ret = drm_eld_size(eld);
+		memcpy(buf, eld, min(max_bytes, ret));
 	}
 
 	mutex_unlock(&dev_priv->av_mutex);
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index cf2560708e03..7ffab1abc518 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -996,6 +996,10 @@ parse_mipi_sequence(struct drm_i915_private *dev_priv,
 			goto err;
 		}
 
+		/* Log about presence of sequences we won't run. */
+		if (seq_id == MIPI_SEQ_TEAR_ON || seq_id == MIPI_SEQ_TEAR_OFF)
+			DRM_DEBUG_KMS("Unsupported sequence %u\n", seq_id);
+
 		dev_priv->vbt.dsi.sequence[seq_id] = data + index;
 
 		if (sequence->version >= 3)
@@ -1805,3 +1809,52 @@ intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv,
 
 	return false;
 }
+
+/**
+ * intel_bios_is_lspcon_present - if LSPCON is attached on %port
+ * @dev_priv:	i915 device instance
+ * @port:	port to check
+ *
+ * Return true if LSPCON is present on this port
+ */
+bool
+intel_bios_is_lspcon_present(struct drm_i915_private *dev_priv,
+				enum port port)
+{
+	int i;
+
+	if (!HAS_LSPCON(dev_priv))
+		return false;
+
+	for (i = 0; i < dev_priv->vbt.child_dev_num; i++) {
+		if (!dev_priv->vbt.child_dev[i].common.lspcon)
+			continue;
+
+		switch (dev_priv->vbt.child_dev[i].common.dvo_port) {
+		case DVO_PORT_DPA:
+		case DVO_PORT_HDMIA:
+			if (port == PORT_A)
+				return true;
+			break;
+		case DVO_PORT_DPB:
+		case DVO_PORT_HDMIB:
+			if (port == PORT_B)
+				return true;
+			break;
+		case DVO_PORT_DPC:
+		case DVO_PORT_HDMIC:
+			if (port == PORT_C)
+				return true;
+			break;
+		case DVO_PORT_DPD:
+		case DVO_PORT_HDMID:
+			if (port == PORT_D)
+				return true;
+			break;
+		default:
+			break;
+		}
+	}
+
+	return false;
+}
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index 495611b7068d..c9c46a538edb 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -83,16 +83,18 @@ static void irq_enable(struct intel_engine_cs *engine)
 	 */
 	engine->breadcrumbs.irq_posted = true;
 
-	spin_lock_irq(&engine->i915->irq_lock);
+	/* Caller disables interrupts */
+	spin_lock(&engine->i915->irq_lock);
 	engine->irq_enable(engine);
-	spin_unlock_irq(&engine->i915->irq_lock);
+	spin_unlock(&engine->i915->irq_lock);
 }
 
 static void irq_disable(struct intel_engine_cs *engine)
 {
-	spin_lock_irq(&engine->i915->irq_lock);
+	/* Caller disables interrupts */
+	spin_lock(&engine->i915->irq_lock);
 	engine->irq_disable(engine);
-	spin_unlock_irq(&engine->i915->irq_lock);
+	spin_unlock(&engine->i915->irq_lock);
 
 	engine->breadcrumbs.irq_posted = false;
 }
@@ -293,9 +295,9 @@ bool intel_engine_add_wait(struct intel_engine_cs *engine,
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
 	bool first;
 
-	spin_lock(&b->lock);
+	spin_lock_irq(&b->lock);
 	first = __intel_engine_add_wait(engine, wait);
-	spin_unlock(&b->lock);
+	spin_unlock_irq(&b->lock);
 
 	return first;
 }
@@ -326,7 +328,7 @@ void intel_engine_remove_wait(struct intel_engine_cs *engine,
 	if (RB_EMPTY_NODE(&wait->node))
 		return;
 
-	spin_lock(&b->lock);
+	spin_lock_irq(&b->lock);
 
 	if (RB_EMPTY_NODE(&wait->node))
 		goto out_unlock;
@@ -400,7 +402,7 @@ out_unlock:
 	GEM_BUG_ON(rb_first(&b->waiters) !=
 		   (b->first_wait ? &b->first_wait->node : NULL));
 	GEM_BUG_ON(!rcu_access_pointer(b->irq_seqno_bh) ^ RB_EMPTY_ROOT(&b->waiters));
-	spin_unlock(&b->lock);
+	spin_unlock_irq(&b->lock);
 }
 
 static bool signal_complete(struct drm_i915_gem_request *request)
@@ -464,7 +466,7 @@ static int intel_breadcrumbs_signaler(void *arg)
 						 &request->signaling.wait);
 
 			local_bh_disable();
-			fence_signal(&request->fence);
+			dma_fence_signal(&request->fence);
 			local_bh_enable(); /* kick start the tasklets */
 
 			/* Find the next oldest signal. Note that as we have
@@ -473,14 +475,14 @@ static int intel_breadcrumbs_signaler(void *arg)
 			 * we just completed - so double check we are still
 			 * the oldest before picking the next one.
 			 */
-			spin_lock(&b->lock);
+			spin_lock_irq(&b->lock);
 			if (request == b->first_signal) {
 				struct rb_node *rb =
 					rb_next(&request->signaling.node);
 				b->first_signal = rb ? to_signaler(rb) : NULL;
 			}
 			rb_erase(&request->signaling.node, &b->signals);
-			spin_unlock(&b->lock);
+			spin_unlock_irq(&b->lock);
 
 			i915_gem_request_put(request);
 		} else {
@@ -502,11 +504,20 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request)
 	struct rb_node *parent, **p;
 	bool first, wakeup;
 
-	/* locked by fence_enable_sw_signaling() */
+	/* Note that we may be called from an interrupt handler on another
+	 * device (e.g. nouveau signaling a fence completion causing us
+	 * to submit a request, and so enable signaling). As such,
+	 * we need to make sure that all other users of b->lock protect
+	 * against interrupts, i.e. use spin_lock_irqsave.
+	 */
+
+	/* locked by dma_fence_enable_sw_signaling() (irqsafe fence->lock) */
 	assert_spin_locked(&request->lock);
+	if (!request->global_seqno)
+		return;
 
 	request->signaling.wait.tsk = b->signaler;
-	request->signaling.wait.seqno = request->fence.seqno;
+	request->signaling.wait.seqno = request->global_seqno;
 	i915_gem_request_get(request);
 
 	spin_lock(&b->lock);
@@ -530,8 +541,8 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request)
 	p = &b->signals.rb_node;
 	while (*p) {
 		parent = *p;
-		if (i915_seqno_passed(request->fence.seqno,
-				      to_signaler(parent)->fence.seqno)) {
+		if (i915_seqno_passed(request->global_seqno,
+				      to_signaler(parent)->global_seqno)) {
 			p = &parent->rb_right;
 			first = false;
 		} else {
@@ -592,7 +603,7 @@ void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
 
 	cancel_fake_irq(engine);
-	spin_lock(&b->lock);
+	spin_lock_irq(&b->lock);
 
 	__intel_breadcrumbs_disable_irq(b);
 	if (intel_engine_has_waiter(engine)) {
@@ -605,7 +616,7 @@ void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
 		irq_disable(engine);
 	}
 
-	spin_unlock(&b->lock);
+	spin_unlock_irq(&b->lock);
 }
 
 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
@@ -618,33 +629,28 @@ void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
 	cancel_fake_irq(engine);
 }
 
-unsigned int intel_kick_waiters(struct drm_i915_private *i915)
+unsigned int intel_breadcrumbs_busy(struct drm_i915_private *i915)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	unsigned int mask = 0;
 
-	/* To avoid the task_struct disappearing beneath us as we wake up
-	 * the process, we must first inspect the task_struct->state under the
-	 * RCU lock, i.e. as we call wake_up_process() we must be holding the
-	 * rcu_read_lock().
-	 */
-	for_each_engine(engine, i915)
-		if (unlikely(intel_engine_wakeup(engine)))
-			mask |= intel_engine_flag(engine);
+	for_each_engine(engine, i915, id) {
+		struct intel_breadcrumbs *b = &engine->breadcrumbs;
 
-	return mask;
-}
+		spin_lock_irq(&b->lock);
 
-unsigned int intel_kick_signalers(struct drm_i915_private *i915)
-{
-	struct intel_engine_cs *engine;
-	unsigned int mask = 0;
+		if (b->first_wait) {
+			wake_up_process(b->first_wait->tsk);
+			mask |= intel_engine_flag(engine);
+		}
 
-	for_each_engine(engine, i915) {
-		if (unlikely(READ_ONCE(engine->breadcrumbs.first_signal))) {
-			wake_up_process(engine->breadcrumbs.signaler);
+		if (b->first_signal) {
+			wake_up_process(b->signaler);
 			mask |= intel_engine_flag(engine);
 		}
+
+		spin_unlock_irq(&b->lock);
 	}
 
 	return mask;
diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c
index 95a72771eea6..d81232b79f00 100644
--- a/drivers/gpu/drm/i915/intel_color.c
+++ b/drivers/gpu/drm/i915/intel_color.c
@@ -95,8 +95,7 @@ static void ctm_mult_by_limited(uint64_t *result, int64_t *input)
 static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
 {
 	struct drm_crtc *crtc = crtc_state->crtc;
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int i, pipe = intel_crtc->pipe;
 	uint16_t coeffs[9] = { 0, };
@@ -180,7 +179,7 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
 	I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), 0);
 	I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), 0);
 
-	if (INTEL_INFO(dev)->gen > 6) {
+	if (INTEL_GEN(dev_priv) > 6) {
 		uint16_t postoff = 0;
 
 		if (intel_crtc_state->limited_color_range)
@@ -273,7 +272,7 @@ static void i9xx_load_luts_internal(struct drm_crtc *crtc,
 	enum pipe pipe = intel_crtc->pipe;
 	int i;
 
-	if (HAS_GMCH_DISPLAY(dev)) {
+	if (HAS_GMCH_DISPLAY(dev_priv)) {
 		if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI))
 			assert_dsi_pll_enabled(dev_priv);
 		else
@@ -288,7 +287,7 @@ static void i9xx_load_luts_internal(struct drm_crtc *crtc,
 				(drm_color_lut_extract(lut[i].green, 8) << 8) |
 				drm_color_lut_extract(lut[i].blue, 8);
 
-			if (HAS_GMCH_DISPLAY(dev))
+			if (HAS_GMCH_DISPLAY(dev_priv))
 				I915_WRITE(PALETTE(pipe, i), word);
 			else
 				I915_WRITE(LGC_PALETTE(pipe, i), word);
@@ -297,7 +296,7 @@ static void i9xx_load_luts_internal(struct drm_crtc *crtc,
 		for (i = 0; i < 256; i++) {
 			uint32_t word = (i << 16) | (i << 8) | i;
 
-			if (HAS_GMCH_DISPLAY(dev))
+			if (HAS_GMCH_DISPLAY(dev_priv))
 				I915_WRITE(PALETTE(pipe, i), word);
 			else
 				I915_WRITE(LGC_PALETTE(pipe, i), word);
@@ -326,7 +325,7 @@ static void haswell_load_luts(struct drm_crtc_state *crtc_state)
 	 * Workaround : Do not read or write the pipe palette/gamma data while
 	 * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
 	 */
-	if (IS_HASWELL(dev) && intel_crtc_state->ips_enabled &&
+	if (IS_HASWELL(dev_priv) && intel_crtc_state->ips_enabled &&
 	    (intel_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)) {
 		hsw_disable_ips(intel_crtc);
 		reenable_ips = true;
@@ -345,11 +344,10 @@ static void haswell_load_luts(struct drm_crtc_state *crtc_state)
 static void broadwell_load_luts(struct drm_crtc_state *state)
 {
 	struct drm_crtc *crtc = state->crtc;
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	struct intel_crtc_state *intel_state = to_intel_crtc_state(state);
 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
-	uint32_t i, lut_size = INTEL_INFO(dev)->color.degamma_lut_size;
+	uint32_t i, lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size;
 
 	if (crtc_state_is_legacy(state)) {
 		haswell_load_luts(state);
@@ -428,8 +426,7 @@ static void broadwell_load_luts(struct drm_crtc_state *state)
 static void cherryview_load_luts(struct drm_crtc_state *state)
 {
 	struct drm_crtc *crtc = state->crtc;
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	enum pipe pipe = to_intel_crtc(crtc)->pipe;
 	struct drm_color_lut *lut;
 	uint32_t i, lut_size;
@@ -446,7 +443,7 @@ static void cherryview_load_luts(struct drm_crtc_state *state)
 
 	if (state->degamma_lut) {
 		lut = (struct drm_color_lut *) state->degamma_lut->data;
-		lut_size = INTEL_INFO(dev)->color.degamma_lut_size;
+		lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size;
 		for (i = 0; i < lut_size; i++) {
 			/* Write LUT in U0.14 format. */
 			word0 =
@@ -461,7 +458,7 @@ static void cherryview_load_luts(struct drm_crtc_state *state)
 
 	if (state->gamma_lut) {
 		lut = (struct drm_color_lut *) state->gamma_lut->data;
-		lut_size = INTEL_INFO(dev)->color.gamma_lut_size;
+		lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size;
 		for (i = 0; i < lut_size; i++) {
 			/* Write LUT in U0.10 format. */
 			word0 =
@@ -497,12 +494,12 @@ void intel_color_load_luts(struct drm_crtc_state *crtc_state)
 int intel_color_check(struct drm_crtc *crtc,
 		      struct drm_crtc_state *crtc_state)
 {
-	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	size_t gamma_length, degamma_length;
 
-	degamma_length = INTEL_INFO(dev)->color.degamma_lut_size *
+	degamma_length = INTEL_INFO(dev_priv)->color.degamma_lut_size *
 		sizeof(struct drm_color_lut);
-	gamma_length = INTEL_INFO(dev)->color.gamma_lut_size *
+	gamma_length = INTEL_INFO(dev_priv)->color.gamma_lut_size *
 		sizeof(struct drm_color_lut);
 
 	/*
@@ -529,19 +526,18 @@ int intel_color_check(struct drm_crtc *crtc,
 
 void intel_color_init(struct drm_crtc *crtc)
 {
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 
 	drm_mode_crtc_set_gamma_size(crtc, 256);
 
-	if (IS_CHERRYVIEW(dev)) {
+	if (IS_CHERRYVIEW(dev_priv)) {
 		dev_priv->display.load_csc_matrix = cherryview_load_csc_matrix;
 		dev_priv->display.load_luts = cherryview_load_luts;
-	} else if (IS_HASWELL(dev)) {
+	} else if (IS_HASWELL(dev_priv)) {
 		dev_priv->display.load_csc_matrix = i9xx_load_csc_matrix;
 		dev_priv->display.load_luts = haswell_load_luts;
-	} else if (IS_BROADWELL(dev) || IS_SKYLAKE(dev) ||
-		   IS_BROXTON(dev) || IS_KABYLAKE(dev)) {
+	} else if (IS_BROADWELL(dev_priv) || IS_SKYLAKE(dev_priv) ||
+		   IS_BROXTON(dev_priv) || IS_KABYLAKE(dev_priv)) {
 		dev_priv->display.load_csc_matrix = i9xx_load_csc_matrix;
 		dev_priv->display.load_luts = broadwell_load_luts;
 	} else {
@@ -549,10 +545,10 @@ void intel_color_init(struct drm_crtc *crtc)
 	}
 
 	/* Enable color management support when we have degamma & gamma LUTs. */
-	if (INTEL_INFO(dev)->color.degamma_lut_size != 0 &&
-	    INTEL_INFO(dev)->color.gamma_lut_size != 0)
+	if (INTEL_INFO(dev_priv)->color.degamma_lut_size != 0 &&
+	    INTEL_INFO(dev_priv)->color.gamma_lut_size != 0)
 		drm_crtc_enable_color_mgmt(crtc,
-					INTEL_INFO(dev)->color.degamma_lut_size,
-					true,
-					INTEL_INFO(dev)->color.gamma_lut_size);
+					   INTEL_INFO(dev_priv)->color.degamma_lut_size,
+					   true,
+					   INTEL_INFO(dev_priv)->color.gamma_lut_size);
 }
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index dfbcf16b41df..86ecec5601d4 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -84,7 +84,7 @@ static bool intel_crt_get_hw_state(struct intel_encoder *encoder,
 	if (!(tmp & ADPA_DAC_ENABLE))
 		goto out;
 
-	if (HAS_PCH_CPT(dev))
+	if (HAS_PCH_CPT(dev_priv))
 		*pipe = PORT_TO_PIPE_CPT(tmp);
 	else
 		*pipe = PORT_TO_PIPE(tmp);
@@ -147,14 +147,13 @@ static void intel_crt_set_dpms(struct intel_encoder *encoder,
 			       struct intel_crtc_state *crtc_state,
 			       int mode)
 {
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crt *crt = intel_encoder_to_crt(encoder);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
 	u32 adpa;
 
-	if (INTEL_INFO(dev)->gen >= 5)
+	if (INTEL_GEN(dev_priv) >= 5)
 		adpa = ADPA_HOTPLUG_BITS;
 	else
 		adpa = 0;
@@ -165,16 +164,16 @@ static void intel_crt_set_dpms(struct intel_encoder *encoder,
 		adpa |= ADPA_VSYNC_ACTIVE_HIGH;
 
 	/* For CPT allow 3 pipe config, for others just use A or B */
-	if (HAS_PCH_LPT(dev))
+	if (HAS_PCH_LPT(dev_priv))
 		; /* Those bits don't exist here */
-	else if (HAS_PCH_CPT(dev))
+	else if (HAS_PCH_CPT(dev_priv))
 		adpa |= PORT_TRANS_SEL_CPT(crtc->pipe);
 	else if (crtc->pipe == 0)
 		adpa |= ADPA_PIPE_A_SELECT;
 	else
 		adpa |= ADPA_PIPE_B_SELECT;
 
-	if (!HAS_PCH_SPLIT(dev))
+	if (!HAS_PCH_SPLIT(dev_priv))
 		I915_WRITE(BCLRPAT(crtc->pipe), 0);
 
 	switch (mode) {
@@ -241,7 +240,8 @@ intel_crt_mode_valid(struct drm_connector *connector,
 		     struct drm_display_mode *mode)
 {
 	struct drm_device *dev = connector->dev;
-	int max_dotclk = to_i915(dev)->max_dotclk_freq;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	int max_dotclk = dev_priv->max_dotclk_freq;
 	int max_clock;
 
 	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
@@ -250,15 +250,15 @@ intel_crt_mode_valid(struct drm_connector *connector,
 	if (mode->clock < 25000)
 		return MODE_CLOCK_LOW;
 
-	if (HAS_PCH_LPT(dev))
+	if (HAS_PCH_LPT(dev_priv))
 		max_clock = 180000;
-	else if (IS_VALLEYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev_priv))
 		/*
 		 * 270 MHz due to current DPLL limits,
 		 * DAC limit supposedly 355 MHz.
 		 */
 		max_clock = 270000;
-	else if (IS_GEN3(dev) || IS_GEN4(dev))
+	else if (IS_GEN3(dev_priv) || IS_GEN4(dev_priv))
 		max_clock = 400000;
 	else
 		max_clock = 350000;
@@ -269,7 +269,7 @@ intel_crt_mode_valid(struct drm_connector *connector,
 		return MODE_CLOCK_HIGH;
 
 	/* The FDI receiver on LPT only supports 8bpc and only has 2 lanes. */
-	if (HAS_PCH_LPT(dev) &&
+	if (HAS_PCH_LPT(dev_priv) &&
 	    (ironlake_get_lanes_required(mode->clock, 270000, 24) > 2))
 		return MODE_CLOCK_HIGH;
 
@@ -280,13 +280,13 @@ static bool intel_crt_compute_config(struct intel_encoder *encoder,
 				     struct intel_crtc_state *pipe_config,
 				     struct drm_connector_state *conn_state)
 {
-	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	if (HAS_PCH_SPLIT(dev))
+	if (HAS_PCH_SPLIT(dev_priv))
 		pipe_config->has_pch_encoder = true;
 
 	/* LPT FDI RX only supports 8bpc. */
-	if (HAS_PCH_LPT(dev)) {
+	if (HAS_PCH_LPT(dev_priv)) {
 		if (pipe_config->bw_constrained && pipe_config->pipe_bpp < 24) {
 			DRM_DEBUG_KMS("LPT only supports 24bpp\n");
 			return false;
@@ -296,7 +296,7 @@ static bool intel_crt_compute_config(struct intel_encoder *encoder,
 	}
 
 	/* FDI must always be 2.7 GHz */
-	if (HAS_DDI(dev))
+	if (HAS_DDI(dev_priv))
 		pipe_config->port_clock = 135000 * 2;
 
 	return true;
@@ -312,7 +312,7 @@ static bool intel_ironlake_crt_detect_hotplug(struct drm_connector *connector)
 
 	/* The first time through, trigger an explicit detection cycle */
 	if (crt->force_hotplug_required) {
-		bool turn_off_dac = HAS_PCH_SPLIT(dev);
+		bool turn_off_dac = HAS_PCH_SPLIT(dev_priv);
 		u32 save_adpa;
 
 		crt->force_hotplug_required = 0;
@@ -419,10 +419,10 @@ static bool intel_crt_detect_hotplug(struct drm_connector *connector)
 	bool ret = false;
 	int i, tries = 0;
 
-	if (HAS_PCH_SPLIT(dev))
+	if (HAS_PCH_SPLIT(dev_priv))
 		return intel_ironlake_crt_detect_hotplug(connector);
 
-	if (IS_VALLEYVIEW(dev))
+	if (IS_VALLEYVIEW(dev_priv))
 		return valleyview_crt_detect_hotplug(connector);
 
 	/*
@@ -430,7 +430,7 @@ static bool intel_crt_detect_hotplug(struct drm_connector *connector)
 	 * to get a reliable result.
 	 */
 
-	if (IS_G4X(dev) && !IS_GM45(dev))
+	if (IS_G4X(dev_priv) && !IS_GM45(dev_priv))
 		tries = 2;
 	else
 		tries = 1;
@@ -566,13 +566,13 @@ intel_crt_load_detect(struct intel_crt *crt, uint32_t pipe)
 	/* Set the border color to purple. */
 	I915_WRITE(bclrpat_reg, 0x500050);
 
-	if (!IS_GEN2(dev)) {
+	if (!IS_GEN2(dev_priv)) {
 		uint32_t pipeconf = I915_READ(pipeconf_reg);
 		I915_WRITE(pipeconf_reg, pipeconf | PIPECONF_FORCE_BORDER);
 		POSTING_READ(pipeconf_reg);
 		/* Wait for next Vblank to substitue
 		 * border color for Color info */
-		intel_wait_for_vblank(dev, pipe);
+		intel_wait_for_vblank(dev_priv, pipe);
 		st00 = I915_READ8(_VGA_MSR_WRITE);
 		status = ((st00 & (1 << 4)) != 0) ?
 			connector_status_connected :
@@ -643,11 +643,36 @@ intel_crt_load_detect(struct intel_crt *crt, uint32_t pipe)
 	return status;
 }
 
+static int intel_spurious_crt_detect_dmi_callback(const struct dmi_system_id *id)
+{
+	DRM_DEBUG_DRIVER("Skipping CRT detection for %s\n", id->ident);
+	return 1;
+}
+
+static const struct dmi_system_id intel_spurious_crt_detect[] = {
+	{
+		.callback = intel_spurious_crt_detect_dmi_callback,
+		.ident = "ACER ZGB",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ACER"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "ZGB"),
+		},
+	},
+	{
+		.callback = intel_spurious_crt_detect_dmi_callback,
+		.ident = "Intel DZ77BH-55K",
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "Intel Corporation"),
+			DMI_MATCH(DMI_BOARD_NAME, "DZ77BH-55K"),
+		},
+	},
+	{ }
+};
+
 static enum drm_connector_status
 intel_crt_detect(struct drm_connector *connector, bool force)
 {
-	struct drm_device *dev = connector->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(connector->dev);
 	struct intel_crt *crt = intel_attached_crt(connector);
 	struct intel_encoder *intel_encoder = &crt->base;
 	enum intel_display_power_domain power_domain;
@@ -659,10 +684,14 @@ intel_crt_detect(struct drm_connector *connector, bool force)
 		      connector->base.id, connector->name,
 		      force);
 
+	/* Skip machines without VGA that falsely report hotplug events */
+	if (dmi_check_system(intel_spurious_crt_detect))
+		return connector_status_disconnected;
+
 	power_domain = intel_display_port_power_domain(intel_encoder);
 	intel_display_power_get(dev_priv, power_domain);
 
-	if (I915_HAS_HOTPLUG(dev)) {
+	if (I915_HAS_HOTPLUG(dev_priv)) {
 		/* We can not rely on the HPD pin always being correctly wired
 		 * up, for example many KVM do not pass it through, and so
 		 * only trust an assertion that the monitor is connected.
@@ -684,7 +713,7 @@ intel_crt_detect(struct drm_connector *connector, bool force)
 	 * broken monitor (without edid) to work behind a broken kvm (that fails
 	 * to have the right resistors for HP detection) needs to fix this up.
 	 * For now just bail out. */
-	if (I915_HAS_HOTPLUG(dev) && !i915.load_detect_test) {
+	if (I915_HAS_HOTPLUG(dev_priv) && !i915.load_detect_test) {
 		status = connector_status_disconnected;
 		goto out;
 	}
@@ -700,7 +729,7 @@ intel_crt_detect(struct drm_connector *connector, bool force)
 	if (intel_get_load_detect_pipe(connector, NULL, &tmp, &ctx)) {
 		if (intel_crt_detect_ddc(connector))
 			status = connector_status_connected;
-		else if (INTEL_INFO(dev)->gen < 4)
+		else if (INTEL_GEN(dev_priv) < 4)
 			status = intel_crt_load_detect(crt,
 				to_intel_crtc(connector->state->crtc)->pipe);
 		else if (i915.load_detect_test)
@@ -740,7 +769,7 @@ static int intel_crt_get_modes(struct drm_connector *connector)
 
 	i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin);
 	ret = intel_crt_ddc_get_modes(connector, i2c);
-	if (ret || !IS_G4X(dev))
+	if (ret || !IS_G4X(dev_priv))
 		goto out;
 
 	/* Try to probe digital port for output in DVI-I -> VGA mode. */
@@ -762,11 +791,10 @@ static int intel_crt_set_property(struct drm_connector *connector,
 
 void intel_crt_reset(struct drm_encoder *encoder)
 {
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
 	struct intel_crt *crt = intel_encoder_to_crt(to_intel_encoder(encoder));
 
-	if (INTEL_INFO(dev)->gen >= 5) {
+	if (INTEL_GEN(dev_priv) >= 5) {
 		u32 adpa;
 
 		adpa = I915_READ(crt->adpa_reg);
@@ -808,32 +836,6 @@ static const struct drm_encoder_funcs intel_crt_enc_funcs = {
 	.destroy = intel_encoder_destroy,
 };
 
-static int intel_no_crt_dmi_callback(const struct dmi_system_id *id)
-{
-	DRM_INFO("Skipping CRT initialization for %s\n", id->ident);
-	return 1;
-}
-
-static const struct dmi_system_id intel_no_crt[] = {
-	{
-		.callback = intel_no_crt_dmi_callback,
-		.ident = "ACER ZGB",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "ACER"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "ZGB"),
-		},
-	},
-	{
-		.callback = intel_no_crt_dmi_callback,
-		.ident = "DELL XPS 8700",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
-			DMI_MATCH(DMI_PRODUCT_NAME, "XPS 8700"),
-		},
-	},
-	{ }
-};
-
 void intel_crt_init(struct drm_device *dev)
 {
 	struct drm_connector *connector;
@@ -843,13 +845,9 @@ void intel_crt_init(struct drm_device *dev)
 	i915_reg_t adpa_reg;
 	u32 adpa;
 
-	/* Skip machines without VGA that falsely report hotplug events */
-	if (dmi_check_system(intel_no_crt))
-		return;
-
-	if (HAS_PCH_SPLIT(dev))
+	if (HAS_PCH_SPLIT(dev_priv))
 		adpa_reg = PCH_ADPA;
-	else if (IS_VALLEYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev_priv))
 		adpa_reg = VLV_ADPA;
 	else
 		adpa_reg = ADPA;
@@ -893,12 +891,12 @@ void intel_crt_init(struct drm_device *dev)
 
 	crt->base.type = INTEL_OUTPUT_ANALOG;
 	crt->base.cloneable = (1 << INTEL_OUTPUT_DVO) | (1 << INTEL_OUTPUT_HDMI);
-	if (IS_I830(dev))
+	if (IS_I830(dev_priv))
 		crt->base.crtc_mask = (1 << 0);
 	else
 		crt->base.crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
 
-	if (IS_GEN2(dev))
+	if (IS_GEN2(dev_priv))
 		connector->interlace_allowed = 0;
 	else
 		connector->interlace_allowed = 1;
@@ -907,20 +905,23 @@ void intel_crt_init(struct drm_device *dev)
 	crt->adpa_reg = adpa_reg;
 
 	crt->base.compute_config = intel_crt_compute_config;
-	if (HAS_PCH_SPLIT(dev)) {
+	if (HAS_PCH_SPLIT(dev_priv)) {
 		crt->base.disable = pch_disable_crt;
 		crt->base.post_disable = pch_post_disable_crt;
 	} else {
 		crt->base.disable = intel_disable_crt;
 	}
 	crt->base.enable = intel_enable_crt;
-	if (I915_HAS_HOTPLUG(dev))
+	if (I915_HAS_HOTPLUG(dev_priv) &&
+	    !dmi_check_system(intel_spurious_crt_detect))
 		crt->base.hpd_pin = HPD_CRT;
-	if (HAS_DDI(dev)) {
+	if (HAS_DDI(dev_priv)) {
+		crt->base.port = PORT_E;
 		crt->base.get_config = hsw_crt_get_config;
 		crt->base.get_hw_state = intel_ddi_get_hw_state;
 		crt->base.post_disable = hsw_post_disable_crt;
 	} else {
+		crt->base.port = PORT_NONE;
 		crt->base.get_config = intel_crt_get_config;
 		crt->base.get_hw_state = intel_crt_get_hw_state;
 	}
@@ -928,7 +929,7 @@ void intel_crt_init(struct drm_device *dev)
 
 	drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs);
 
-	if (!I915_HAS_HOTPLUG(dev))
+	if (!I915_HAS_HOTPLUG(dev_priv))
 		intel_connector->polled = DRM_CONNECTOR_POLL_CONNECT;
 
 	/*
@@ -941,7 +942,7 @@ void intel_crt_init(struct drm_device *dev)
 	 * polarity and link reversal bits or not, instead of relying on the
 	 * BIOS.
 	 */
-	if (HAS_PCH_LPT(dev)) {
+	if (HAS_PCH_LPT(dev_priv)) {
 		u32 fdi_config = FDI_RX_POLARITY_REVERSED_LPT |
 				 FDI_RX_LINK_REVERSAL_OVERRIDE;
 
diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c
index 1ea0e1f43397..d7a04bca8c28 100644
--- a/drivers/gpu/drm/i915/intel_csr.c
+++ b/drivers/gpu/drm/i915/intel_csr.c
@@ -168,12 +168,6 @@ struct stepping_info {
 	char substepping;
 };
 
-static const struct stepping_info kbl_stepping_info[] = {
-	{'A', '0'}, {'B', '0'}, {'C', '0'},
-	{'D', '0'}, {'E', '0'}, {'F', '0'},
-	{'G', '0'}, {'H', '0'}, {'I', '0'},
-};
-
 static const struct stepping_info skl_stepping_info[] = {
 	{'A', '0'}, {'B', '0'}, {'C', '0'},
 	{'D', '0'}, {'E', '0'}, {'F', '0'},
@@ -194,10 +188,7 @@ intel_get_stepping_info(struct drm_i915_private *dev_priv)
 	const struct stepping_info *si;
 	unsigned int size;
 
-	if (IS_KABYLAKE(dev_priv)) {
-		size = ARRAY_SIZE(kbl_stepping_info);
-		si = kbl_stepping_info;
-	} else if (IS_SKYLAKE(dev_priv)) {
+	if (IS_SKYLAKE(dev_priv)) {
 		size = ARRAY_SIZE(skl_stepping_info);
 		si = skl_stepping_info;
 	} else if (IS_BROXTON(dev_priv)) {
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 15d47c87def6..10ec9d4b7d45 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -167,8 +167,47 @@ static const struct ddi_buf_trans skl_y_ddi_translations_dp[] = {
 	{ 0x80005012, 0x000000C0, 0x3 },
 };
 
+/* Kabylake H and S */
+static const struct ddi_buf_trans kbl_ddi_translations_dp[] = {
+	{ 0x00002016, 0x000000A0, 0x0 },
+	{ 0x00005012, 0x0000009B, 0x0 },
+	{ 0x00007011, 0x00000088, 0x0 },
+	{ 0x80009010, 0x000000C0, 0x1 },
+	{ 0x00002016, 0x0000009B, 0x0 },
+	{ 0x00005012, 0x00000088, 0x0 },
+	{ 0x80007011, 0x000000C0, 0x1 },
+	{ 0x00002016, 0x00000097, 0x0 },
+	{ 0x80005012, 0x000000C0, 0x1 },
+};
+
+/* Kabylake U */
+static const struct ddi_buf_trans kbl_u_ddi_translations_dp[] = {
+	{ 0x0000201B, 0x000000A1, 0x0 },
+	{ 0x00005012, 0x00000088, 0x0 },
+	{ 0x80007011, 0x000000CD, 0x3 },
+	{ 0x80009010, 0x000000C0, 0x3 },
+	{ 0x0000201B, 0x0000009D, 0x0 },
+	{ 0x80005012, 0x000000C0, 0x3 },
+	{ 0x80007011, 0x000000C0, 0x3 },
+	{ 0x00002016, 0x0000004F, 0x0 },
+	{ 0x80005012, 0x000000C0, 0x3 },
+};
+
+/* Kabylake Y */
+static const struct ddi_buf_trans kbl_y_ddi_translations_dp[] = {
+	{ 0x00001017, 0x000000A1, 0x0 },
+	{ 0x00005012, 0x00000088, 0x0 },
+	{ 0x80007011, 0x000000CD, 0x3 },
+	{ 0x8000800F, 0x000000C0, 0x3 },
+	{ 0x00001017, 0x0000009D, 0x0 },
+	{ 0x80005012, 0x000000C0, 0x3 },
+	{ 0x80007011, 0x000000C0, 0x3 },
+	{ 0x00001017, 0x0000004C, 0x0 },
+	{ 0x80005012, 0x000000C0, 0x3 },
+};
+
 /*
- * Skylake H and S
+ * Skylake/Kabylake H and S
  * eDP 1.4 low vswing translation parameters
  */
 static const struct ddi_buf_trans skl_ddi_translations_edp[] = {
@@ -185,7 +224,7 @@ static const struct ddi_buf_trans skl_ddi_translations_edp[] = {
 };
 
 /*
- * Skylake U
+ * Skylake/Kabylake U
  * eDP 1.4 low vswing translation parameters
  */
 static const struct ddi_buf_trans skl_u_ddi_translations_edp[] = {
@@ -202,7 +241,7 @@ static const struct ddi_buf_trans skl_u_ddi_translations_edp[] = {
 };
 
 /*
- * Skylake Y
+ * Skylake/Kabylake Y
  * eDP 1.4 low vswing translation parameters
  */
 static const struct ddi_buf_trans skl_y_ddi_translations_edp[] = {
@@ -218,7 +257,7 @@ static const struct ddi_buf_trans skl_y_ddi_translations_edp[] = {
 	{ 0x00000018, 0x0000008A, 0x0 },
 };
 
-/* Skylake U, H and S */
+/* Skylake/Kabylake U, H and S */
 static const struct ddi_buf_trans skl_ddi_translations_hdmi[] = {
 	{ 0x00000018, 0x000000AC, 0x0 },
 	{ 0x00005012, 0x0000009D, 0x0 },
@@ -233,7 +272,7 @@ static const struct ddi_buf_trans skl_ddi_translations_hdmi[] = {
 	{ 0x80000018, 0x000000C0, 0x1 },
 };
 
-/* Skylake Y */
+/* Skylake/Kabylake Y */
 static const struct ddi_buf_trans skl_y_ddi_translations_hdmi[] = {
 	{ 0x00000018, 0x000000A1, 0x0 },
 	{ 0x00005012, 0x000000DF, 0x0 },
@@ -334,10 +373,10 @@ bdw_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
 static const struct ddi_buf_trans *
 skl_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries)
 {
-	if (IS_SKL_ULX(dev_priv) || IS_KBL_ULX(dev_priv)) {
+	if (IS_SKL_ULX(dev_priv)) {
 		*n_entries = ARRAY_SIZE(skl_y_ddi_translations_dp);
 		return skl_y_ddi_translations_dp;
-	} else if (IS_SKL_ULT(dev_priv) || IS_KBL_ULT(dev_priv)) {
+	} else if (IS_SKL_ULT(dev_priv)) {
 		*n_entries = ARRAY_SIZE(skl_u_ddi_translations_dp);
 		return skl_u_ddi_translations_dp;
 	} else {
@@ -347,6 +386,21 @@ skl_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries)
 }
 
 static const struct ddi_buf_trans *
+kbl_get_buf_trans_dp(struct drm_i915_private *dev_priv, int *n_entries)
+{
+	if (IS_KBL_ULX(dev_priv)) {
+		*n_entries = ARRAY_SIZE(kbl_y_ddi_translations_dp);
+		return kbl_y_ddi_translations_dp;
+	} else if (IS_KBL_ULT(dev_priv)) {
+		*n_entries = ARRAY_SIZE(kbl_u_ddi_translations_dp);
+		return kbl_u_ddi_translations_dp;
+	} else {
+		*n_entries = ARRAY_SIZE(kbl_ddi_translations_dp);
+		return kbl_ddi_translations_dp;
+	}
+}
+
+static const struct ddi_buf_trans *
 skl_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
 {
 	if (dev_priv->vbt.edp.low_vswing) {
@@ -362,7 +416,10 @@ skl_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
 		}
 	}
 
-	return skl_get_buf_trans_dp(dev_priv, n_entries);
+	if (IS_KABYLAKE(dev_priv))
+		return kbl_get_buf_trans_dp(dev_priv, n_entries);
+	else
+		return skl_get_buf_trans_dp(dev_priv, n_entries);
 }
 
 static const struct ddi_buf_trans *
@@ -430,21 +487,18 @@ void intel_prepare_dp_ddi_buffers(struct intel_encoder *encoder)
 	if (IS_BROXTON(dev_priv))
 		return;
 
-	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+	if (IS_KABYLAKE(dev_priv)) {
+		ddi_translations_fdi = NULL;
+		ddi_translations_dp =
+				kbl_get_buf_trans_dp(dev_priv, &n_dp_entries);
+		ddi_translations_edp =
+				skl_get_buf_trans_edp(dev_priv, &n_edp_entries);
+	} else if (IS_SKYLAKE(dev_priv)) {
 		ddi_translations_fdi = NULL;
 		ddi_translations_dp =
 				skl_get_buf_trans_dp(dev_priv, &n_dp_entries);
 		ddi_translations_edp =
 				skl_get_buf_trans_edp(dev_priv, &n_edp_entries);
-
-		/* If we're boosting the current, set bit 31 of trans1 */
-		if (dev_priv->vbt.ddi_port_info[port].dp_boost_level)
-			iboost_bit = DDI_BUF_BALANCE_LEG_ENABLE;
-
-		if (WARN_ON(encoder->type == INTEL_OUTPUT_EDP &&
-			    port != PORT_A && port != PORT_E &&
-			    n_edp_entries > 9))
-			n_edp_entries = 9;
 	} else if (IS_BROADWELL(dev_priv)) {
 		ddi_translations_fdi = bdw_ddi_translations_fdi;
 		ddi_translations_dp = bdw_ddi_translations_dp;
@@ -464,6 +518,17 @@ void intel_prepare_dp_ddi_buffers(struct intel_encoder *encoder)
 		n_dp_entries = ARRAY_SIZE(bdw_ddi_translations_dp);
 	}
 
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+		/* If we're boosting the current, set bit 31 of trans1 */
+		if (dev_priv->vbt.ddi_port_info[port].dp_boost_level)
+			iboost_bit = DDI_BUF_BALANCE_LEG_ENABLE;
+
+		if (WARN_ON(encoder->type == INTEL_OUTPUT_EDP &&
+			    port != PORT_A && port != PORT_E &&
+			    n_edp_entries > 9))
+			n_edp_entries = 9;
+	}
+
 	switch (encoder->type) {
 	case INTEL_OUTPUT_EDP:
 		ddi_translations = ddi_translations_edp;
@@ -1020,13 +1085,13 @@ static void bxt_ddi_clock_get(struct intel_encoder *encoder,
 void intel_ddi_clock_get(struct intel_encoder *encoder,
 			 struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	if (INTEL_INFO(dev)->gen <= 8)
+	if (INTEL_GEN(dev_priv) <= 8)
 		hsw_ddi_clock_get(encoder, pipe_config);
-	else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
+	else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
 		skl_ddi_clock_get(encoder, pipe_config);
-	else if (IS_BROXTON(dev))
+	else if (IS_BROXTON(dev_priv))
 		bxt_ddi_clock_get(encoder, pipe_config);
 }
 
@@ -1081,14 +1146,14 @@ bxt_ddi_pll_select(struct intel_crtc *intel_crtc,
 bool intel_ddi_pll_select(struct intel_crtc *intel_crtc,
 			  struct intel_crtc_state *crtc_state)
 {
-	struct drm_device *dev = intel_crtc->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
 	struct intel_encoder *intel_encoder =
 		intel_ddi_get_crtc_new_encoder(crtc_state);
 
-	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
 		return skl_ddi_pll_select(intel_crtc, crtc_state,
 					  intel_encoder);
-	else if (IS_BROXTON(dev))
+	else if (IS_BROXTON(dev_priv))
 		return bxt_ddi_pll_select(intel_crtc, crtc_state,
 					  intel_encoder);
 	else
@@ -1189,7 +1254,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc)
 			 * eDP when not using the panel fitter, and when not
 			 * using motion blur mitigation (which we don't
 			 * support). */
-			if (IS_HASWELL(dev) &&
+			if (IS_HASWELL(dev_priv) &&
 			    (intel_crtc->config->pch_pfit.enabled ||
 			     intel_crtc->config->pch_pfit.force_thru))
 				temp |= TRANS_DDI_EDP_INPUT_A_ONOFF;
@@ -1434,7 +1499,12 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder, u32 level)
 		if (dp_iboost) {
 			iboost = dp_iboost;
 		} else {
-			ddi_translations = skl_get_buf_trans_dp(dev_priv, &n_entries);
+			if (IS_KABYLAKE(dev_priv))
+				ddi_translations = kbl_get_buf_trans_dp(dev_priv,
+									&n_entries);
+			else
+				ddi_translations = skl_get_buf_trans_dp(dev_priv,
+									&n_entries);
 			iboost = ddi_translations[level].i_boost;
 		}
 	} else if (type == INTEL_OUTPUT_EDP) {
@@ -1477,7 +1547,6 @@ static void bxt_ddi_vswing_sequence(struct drm_i915_private *dev_priv,
 {
 	const struct bxt_ddi_buf_trans *ddi_translations;
 	u32 n_entries, i;
-	uint32_t val;
 
 	if (type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp.low_vswing) {
 		n_entries = ARRAY_SIZE(bxt_ddi_translations_edp);
@@ -1506,38 +1575,11 @@ static void bxt_ddi_vswing_sequence(struct drm_i915_private *dev_priv,
 		}
 	}
 
-	/*
-	 * While we write to the group register to program all lanes at once we
-	 * can read only lane registers and we pick lanes 0/1 for that.
-	 */
-	val = I915_READ(BXT_PORT_PCS_DW10_LN01(port));
-	val &= ~(TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT);
-	I915_WRITE(BXT_PORT_PCS_DW10_GRP(port), val);
-
-	val = I915_READ(BXT_PORT_TX_DW2_LN0(port));
-	val &= ~(MARGIN_000 | UNIQ_TRANS_SCALE);
-	val |= ddi_translations[level].margin << MARGIN_000_SHIFT |
-	       ddi_translations[level].scale << UNIQ_TRANS_SCALE_SHIFT;
-	I915_WRITE(BXT_PORT_TX_DW2_GRP(port), val);
-
-	val = I915_READ(BXT_PORT_TX_DW3_LN0(port));
-	val &= ~SCALE_DCOMP_METHOD;
-	if (ddi_translations[level].enable)
-		val |= SCALE_DCOMP_METHOD;
-
-	if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD))
-		DRM_ERROR("Disabled scaling while ouniqetrangenmethod was set");
-
-	I915_WRITE(BXT_PORT_TX_DW3_GRP(port), val);
-
-	val = I915_READ(BXT_PORT_TX_DW4_LN0(port));
-	val &= ~DE_EMPHASIS;
-	val |= ddi_translations[level].deemphasis << DEEMPH_SHIFT;
-	I915_WRITE(BXT_PORT_TX_DW4_GRP(port), val);
-
-	val = I915_READ(BXT_PORT_PCS_DW10_LN01(port));
-	val |= TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT;
-	I915_WRITE(BXT_PORT_PCS_DW10_GRP(port), val);
+	bxt_ddi_phy_set_signal_level(dev_priv, port,
+				     ddi_translations[level].margin,
+				     ddi_translations[level].scale,
+				     ddi_translations[level].enable,
+				     ddi_translations[level].deemphasis);
 }
 
 static uint32_t translate_signal_level(int signal_levels)
@@ -1711,8 +1753,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder,
 				   struct drm_connector_state *old_conn_state)
 {
 	struct drm_encoder *encoder = &intel_encoder->base;
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
 	enum port port = intel_ddi_get_encoder_port(intel_encoder);
 	int type = intel_encoder->type;
 	uint32_t val;
@@ -1742,10 +1783,10 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder,
 		intel_edp_panel_off(intel_dp);
 	}
 
-	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
 		I915_WRITE(DPLL_CTRL2, (I915_READ(DPLL_CTRL2) |
 					DPLL_CTRL2_DDI_CLK_OFF(port)));
-	else if (INTEL_INFO(dev)->gen < 9)
+	else if (INTEL_GEN(dev_priv) < 9)
 		I915_WRITE(PORT_CLK_SEL(port), PORT_CLK_SEL_NONE);
 
 	if (type == INTEL_OUTPUT_HDMI) {
@@ -1795,8 +1836,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder,
 	struct drm_encoder *encoder = &intel_encoder->base;
 	struct drm_crtc *crtc = encoder->crtc;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
 	enum port port = intel_ddi_get_encoder_port(intel_encoder);
 	int type = intel_encoder->type;
 
@@ -1814,7 +1854,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder,
 	} else if (type == INTEL_OUTPUT_EDP) {
 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 
-		if (port == PORT_A && INTEL_INFO(dev)->gen < 9)
+		if (port == PORT_A && INTEL_GEN(dev_priv) < 9)
 			intel_dp_stop_link_train(intel_dp);
 
 		intel_edp_backlight_on(intel_dp);
@@ -1824,7 +1864,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder,
 
 	if (intel_crtc->config->has_audio) {
 		intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO);
-		intel_audio_codec_enable(intel_encoder);
+		intel_audio_codec_enable(intel_encoder, pipe_config, conn_state);
 	}
 }
 
@@ -1853,332 +1893,14 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder,
 	}
 }
 
-bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv,
-			    enum dpio_phy phy)
-{
-	enum port port;
-
-	if (!(I915_READ(BXT_P_CR_GT_DISP_PWRON) & GT_DISPLAY_POWER_ON(phy)))
-		return false;
-
-	if ((I915_READ(BXT_PORT_CL1CM_DW0(phy)) &
-	     (PHY_POWER_GOOD | PHY_RESERVED)) != PHY_POWER_GOOD) {
-		DRM_DEBUG_DRIVER("DDI PHY %d powered, but power hasn't settled\n",
-				 phy);
-
-		return false;
-	}
-
-	if (phy == DPIO_PHY1 &&
-	    !(I915_READ(BXT_PORT_REF_DW3(DPIO_PHY1)) & GRC_DONE)) {
-		DRM_DEBUG_DRIVER("DDI PHY 1 powered, but GRC isn't done\n");
-
-		return false;
-	}
-
-	if (!(I915_READ(BXT_PHY_CTL_FAMILY(phy)) & COMMON_RESET_DIS)) {
-		DRM_DEBUG_DRIVER("DDI PHY %d powered, but still in reset\n",
-				 phy);
-
-		return false;
-	}
-
-	for_each_port_masked(port,
-			     phy == DPIO_PHY0 ? BIT(PORT_B) | BIT(PORT_C) :
-						BIT(PORT_A)) {
-		u32 tmp = I915_READ(BXT_PHY_CTL(port));
-
-		if (tmp & BXT_PHY_CMNLANE_POWERDOWN_ACK) {
-			DRM_DEBUG_DRIVER("DDI PHY %d powered, but common lane "
-					 "for port %c powered down "
-					 "(PHY_CTL %08x)\n",
-					 phy, port_name(port), tmp);
-
-			return false;
-		}
-	}
-
-	return true;
-}
-
-static u32 bxt_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy)
-{
-	u32 val = I915_READ(BXT_PORT_REF_DW6(phy));
-
-	return (val & GRC_CODE_MASK) >> GRC_CODE_SHIFT;
-}
-
-static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv,
-				  enum dpio_phy phy)
-{
-	if (intel_wait_for_register(dev_priv,
-				    BXT_PORT_REF_DW3(phy),
-				    GRC_DONE, GRC_DONE,
-				    10))
-		DRM_ERROR("timeout waiting for PHY%d GRC\n", phy);
-}
-
-void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy)
-{
-	u32 val;
-
-	if (bxt_ddi_phy_is_enabled(dev_priv, phy)) {
-		/* Still read out the GRC value for state verification */
-		if (phy == DPIO_PHY0)
-			dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, phy);
-
-		if (bxt_ddi_phy_verify_state(dev_priv, phy)) {
-			DRM_DEBUG_DRIVER("DDI PHY %d already enabled, "
-					 "won't reprogram it\n", phy);
-
-			return;
-		}
-
-		DRM_DEBUG_DRIVER("DDI PHY %d enabled with invalid state, "
-				 "force reprogramming it\n", phy);
-	}
-
-	val = I915_READ(BXT_P_CR_GT_DISP_PWRON);
-	val |= GT_DISPLAY_POWER_ON(phy);
-	I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val);
-
-	/*
-	 * The PHY registers start out inaccessible and respond to reads with
-	 * all 1s.  Eventually they become accessible as they power up, then
-	 * the reserved bit will give the default 0.  Poll on the reserved bit
-	 * becoming 0 to find when the PHY is accessible.
-	 * HW team confirmed that the time to reach phypowergood status is
-	 * anywhere between 50 us and 100us.
-	 */
-	if (wait_for_us(((I915_READ(BXT_PORT_CL1CM_DW0(phy)) &
-		(PHY_RESERVED | PHY_POWER_GOOD)) == PHY_POWER_GOOD), 100)) {
-		DRM_ERROR("timeout during PHY%d power on\n", phy);
-	}
-
-	/* Program PLL Rcomp code offset */
-	val = I915_READ(BXT_PORT_CL1CM_DW9(phy));
-	val &= ~IREF0RC_OFFSET_MASK;
-	val |= 0xE4 << IREF0RC_OFFSET_SHIFT;
-	I915_WRITE(BXT_PORT_CL1CM_DW9(phy), val);
-
-	val = I915_READ(BXT_PORT_CL1CM_DW10(phy));
-	val &= ~IREF1RC_OFFSET_MASK;
-	val |= 0xE4 << IREF1RC_OFFSET_SHIFT;
-	I915_WRITE(BXT_PORT_CL1CM_DW10(phy), val);
-
-	/* Program power gating */
-	val = I915_READ(BXT_PORT_CL1CM_DW28(phy));
-	val |= OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN |
-		SUS_CLK_CONFIG;
-	I915_WRITE(BXT_PORT_CL1CM_DW28(phy), val);
-
-	if (phy == DPIO_PHY0) {
-		val = I915_READ(BXT_PORT_CL2CM_DW6_BC);
-		val |= DW6_OLDO_DYN_PWR_DOWN_EN;
-		I915_WRITE(BXT_PORT_CL2CM_DW6_BC, val);
-	}
-
-	val = I915_READ(BXT_PORT_CL1CM_DW30(phy));
-	val &= ~OCL2_LDOFUSE_PWR_DIS;
-	/*
-	 * On PHY1 disable power on the second channel, since no port is
-	 * connected there. On PHY0 both channels have a port, so leave it
-	 * enabled.
-	 * TODO: port C is only connected on BXT-P, so on BXT0/1 we should
-	 * power down the second channel on PHY0 as well.
-	 *
-	 * FIXME: Clarify programming of the following, the register is
-	 * read-only with bit 6 fixed at 0 at least in stepping A.
-	 */
-	if (phy == DPIO_PHY1)
-		val |= OCL2_LDOFUSE_PWR_DIS;
-	I915_WRITE(BXT_PORT_CL1CM_DW30(phy), val);
-
-	if (phy == DPIO_PHY0) {
-		uint32_t grc_code;
-		/*
-		 * PHY0 isn't connected to an RCOMP resistor so copy over
-		 * the corresponding calibrated value from PHY1, and disable
-		 * the automatic calibration on PHY0.
-		 */
-		val = dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, DPIO_PHY1);
-		grc_code = val << GRC_CODE_FAST_SHIFT |
-			   val << GRC_CODE_SLOW_SHIFT |
-			   val;
-		I915_WRITE(BXT_PORT_REF_DW6(DPIO_PHY0), grc_code);
-
-		val = I915_READ(BXT_PORT_REF_DW8(DPIO_PHY0));
-		val |= GRC_DIS | GRC_RDY_OVRD;
-		I915_WRITE(BXT_PORT_REF_DW8(DPIO_PHY0), val);
-	}
-
-	val = I915_READ(BXT_PHY_CTL_FAMILY(phy));
-	val |= COMMON_RESET_DIS;
-	I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val);
-
-	if (phy == DPIO_PHY1)
-		bxt_phy_wait_grc_done(dev_priv, DPIO_PHY1);
-}
-
-void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy)
-{
-	uint32_t val;
-
-	val = I915_READ(BXT_PHY_CTL_FAMILY(phy));
-	val &= ~COMMON_RESET_DIS;
-	I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val);
-
-	val = I915_READ(BXT_P_CR_GT_DISP_PWRON);
-	val &= ~GT_DISPLAY_POWER_ON(phy);
-	I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val);
-}
-
-static bool __printf(6, 7)
-__phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy,
-		       i915_reg_t reg, u32 mask, u32 expected,
-		       const char *reg_fmt, ...)
-{
-	struct va_format vaf;
-	va_list args;
-	u32 val;
-
-	val = I915_READ(reg);
-	if ((val & mask) == expected)
-		return true;
-
-	va_start(args, reg_fmt);
-	vaf.fmt = reg_fmt;
-	vaf.va = &args;
-
-	DRM_DEBUG_DRIVER("DDI PHY %d reg %pV [%08x] state mismatch: "
-			 "current %08x, expected %08x (mask %08x)\n",
-			 phy, &vaf, reg.reg, val, (val & ~mask) | expected,
-			 mask);
-
-	va_end(args);
-
-	return false;
-}
-
-bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
-			      enum dpio_phy phy)
-{
-	uint32_t mask;
-	bool ok;
-
-#define _CHK(reg, mask, exp, fmt, ...)					\
-	__phy_reg_verify_state(dev_priv, phy, reg, mask, exp, fmt,	\
-			       ## __VA_ARGS__)
-
-	if (!bxt_ddi_phy_is_enabled(dev_priv, phy))
-		return false;
-
-	ok = true;
-
-	/* PLL Rcomp code offset */
-	ok &= _CHK(BXT_PORT_CL1CM_DW9(phy),
-		    IREF0RC_OFFSET_MASK, 0xe4 << IREF0RC_OFFSET_SHIFT,
-		    "BXT_PORT_CL1CM_DW9(%d)", phy);
-	ok &= _CHK(BXT_PORT_CL1CM_DW10(phy),
-		    IREF1RC_OFFSET_MASK, 0xe4 << IREF1RC_OFFSET_SHIFT,
-		    "BXT_PORT_CL1CM_DW10(%d)", phy);
-
-	/* Power gating */
-	mask = OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | SUS_CLK_CONFIG;
-	ok &= _CHK(BXT_PORT_CL1CM_DW28(phy), mask, mask,
-		    "BXT_PORT_CL1CM_DW28(%d)", phy);
-
-	if (phy == DPIO_PHY0)
-		ok &= _CHK(BXT_PORT_CL2CM_DW6_BC,
-			   DW6_OLDO_DYN_PWR_DOWN_EN, DW6_OLDO_DYN_PWR_DOWN_EN,
-			   "BXT_PORT_CL2CM_DW6_BC");
-
-	/*
-	 * TODO: Verify BXT_PORT_CL1CM_DW30 bit OCL2_LDOFUSE_PWR_DIS,
-	 * at least on stepping A this bit is read-only and fixed at 0.
-	 */
-
-	if (phy == DPIO_PHY0) {
-		u32 grc_code = dev_priv->bxt_phy_grc;
-
-		grc_code = grc_code << GRC_CODE_FAST_SHIFT |
-			   grc_code << GRC_CODE_SLOW_SHIFT |
-			   grc_code;
-		mask = GRC_CODE_FAST_MASK | GRC_CODE_SLOW_MASK |
-		       GRC_CODE_NOM_MASK;
-		ok &= _CHK(BXT_PORT_REF_DW6(DPIO_PHY0), mask, grc_code,
-			    "BXT_PORT_REF_DW6(%d)", DPIO_PHY0);
-
-		mask = GRC_DIS | GRC_RDY_OVRD;
-		ok &= _CHK(BXT_PORT_REF_DW8(DPIO_PHY0), mask, mask,
-			    "BXT_PORT_REF_DW8(%d)", DPIO_PHY0);
-	}
-
-	return ok;
-#undef _CHK
-}
-
-static uint8_t
-bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder,
-				     struct intel_crtc_state *pipe_config)
-{
-	switch (pipe_config->lane_count) {
-	case 1:
-		return 0;
-	case 2:
-		return BIT(2) | BIT(0);
-	case 4:
-		return BIT(3) | BIT(2) | BIT(0);
-	default:
-		MISSING_CASE(pipe_config->lane_count);
-
-		return 0;
-	}
-}
-
 static void bxt_ddi_pre_pll_enable(struct intel_encoder *encoder,
 				   struct intel_crtc_state *pipe_config,
 				   struct drm_connector_state *conn_state)
 {
-	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-	struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
-	enum port port = dport->port;
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
-	int lane;
-
-	for (lane = 0; lane < 4; lane++) {
-		u32 val = I915_READ(BXT_PORT_TX_DW14_LN(port, lane));
-
-		/*
-		 * Note that on CHV this flag is called UPAR, but has
-		 * the same function.
-		 */
-		val &= ~LATENCY_OPTIM;
-		if (intel_crtc->config->lane_lat_optim_mask & BIT(lane))
-			val |= LATENCY_OPTIM;
-
-		I915_WRITE(BXT_PORT_TX_DW14_LN(port, lane), val);
-	}
-}
-
-static uint8_t
-bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder)
-{
-	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-	struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
-	enum port port = dport->port;
-	int lane;
-	uint8_t mask;
-
-	mask = 0;
-	for (lane = 0; lane < 4; lane++) {
-		u32 val = I915_READ(BXT_PORT_TX_DW14_LN(port, lane));
-
-		if (val & LATENCY_OPTIM)
-			mask |= BIT(lane);
-	}
+	uint8_t mask = intel_crtc->config->lane_lat_optim_mask;
 
-	return mask;
+	bxt_ddi_phy_set_lane_optim_mask(encoder, mask);
 }
 
 void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp)
@@ -2347,7 +2069,7 @@ static bool intel_ddi_compute_config(struct intel_encoder *encoder,
 	if (IS_BROXTON(dev_priv) && ret)
 		pipe_config->lane_lat_optim_mask =
 			bxt_ddi_phy_calc_lane_lat_optim_mask(encoder,
-							     pipe_config);
+							     pipe_config->lane_count);
 
 	return ret;
 
@@ -2438,7 +2160,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
 	struct intel_digital_port *intel_dig_port;
 	struct intel_encoder *intel_encoder;
 	struct drm_encoder *encoder;
-	bool init_hdmi, init_dp;
+	bool init_hdmi, init_dp, init_lspcon = false;
 	int max_lanes;
 
 	if (I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES) {
@@ -2470,6 +2192,19 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
 	init_hdmi = (dev_priv->vbt.ddi_port_info[port].supports_dvi ||
 		     dev_priv->vbt.ddi_port_info[port].supports_hdmi);
 	init_dp = dev_priv->vbt.ddi_port_info[port].supports_dp;
+
+	if (intel_bios_is_lspcon_present(dev_priv, port)) {
+		/*
+		 * Lspcon device needs to be driven with DP connector
+		 * with special detection sequence. So make sure DP
+		 * is initialized before lspcon.
+		 */
+		init_dp = true;
+		init_lspcon = true;
+		init_hdmi = false;
+		DRM_DEBUG_KMS("VBT says port %c has lspcon\n", port_name(port));
+	}
+
 	if (!init_dp && !init_hdmi) {
 		DRM_DEBUG_KMS("VBT says port %c is not DVI/HDMI/DP compatible, respect it\n",
 			      port_name(port));
@@ -2509,7 +2244,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
 	 * configuration so that we use the proper lane count for our
 	 * calculations.
 	 */
-	if (IS_BROXTON(dev) && port == PORT_A) {
+	if (IS_BROXTON(dev_priv) && port == PORT_A) {
 		if (!(intel_dig_port->saved_port_bits & DDI_A_4_LANES)) {
 			DRM_DEBUG_KMS("BXT BIOS forgot to set DDI_A_4_LANES for port A; fixing\n");
 			intel_dig_port->saved_port_bits |= DDI_A_4_LANES;
@@ -2520,6 +2255,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
 	intel_dig_port->max_lanes = max_lanes;
 
 	intel_encoder->type = INTEL_OUTPUT_UNKNOWN;
+	intel_encoder->port = port;
 	intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
 	intel_encoder->cloneable = 0;
 
@@ -2532,7 +2268,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
 		 * On BXT A0/A1, sw needs to activate DDIA HPD logic and
 		 * interrupts to check the external panel connection.
 		 */
-		if (IS_BXT_REVID(dev, 0, BXT_REVID_A1) && port == PORT_B)
+		if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1) && port == PORT_B)
 			dev_priv->hotplug.irq_port[PORT_A] = intel_dig_port;
 		else
 			dev_priv->hotplug.irq_port[port] = intel_dig_port;
@@ -2545,6 +2281,20 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
 			goto err;
 	}
 
+	if (init_lspcon) {
+		if (lspcon_init(intel_dig_port))
+			/* TODO: handle hdmi info frame part */
+			DRM_DEBUG_KMS("LSPCON init success on port %c\n",
+				port_name(port));
+		else
+			/*
+			 * LSPCON init faied, but DP init was success, so
+			 * lets try to drive as DP++ port.
+			 */
+			DRM_ERROR("LSPCON init failed on port %c\n",
+				port_name(port));
+	}
+
 	return;
 
 err:
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 1b20e160bc1f..185e3bbc9ec9 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -28,20 +28,14 @@ void intel_device_info_dump(struct drm_i915_private *dev_priv)
 {
 	const struct intel_device_info *info = &dev_priv->info;
 
-#define PRINT_S(name) "%s"
-#define SEP_EMPTY
-#define PRINT_FLAG(name) info->name ? #name "," : ""
-#define SEP_COMMA ,
-	DRM_DEBUG_DRIVER("i915 device info: gen=%i, pciid=0x%04x rev=0x%02x flags="
-			 DEV_INFO_FOR_EACH_FLAG(PRINT_S, SEP_EMPTY),
+	DRM_DEBUG_DRIVER("i915 device info: gen=%i, pciid=0x%04x rev=0x%02x",
 			 info->gen,
 			 dev_priv->drm.pdev->device,
-			 dev_priv->drm.pdev->revision,
-			 DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_COMMA));
-#undef PRINT_S
-#undef SEP_EMPTY
+			 dev_priv->drm.pdev->revision);
+#define PRINT_FLAG(name) \
+	DRM_DEBUG_DRIVER("i915 device info: " #name ": %s", yesno(info->name))
+	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG);
 #undef PRINT_FLAG
-#undef SEP_COMMA
 }
 
 static void cherryview_sseu_info_init(struct drm_i915_private *dev_priv)
@@ -288,12 +282,13 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
 		info->num_sprites[PIPE_A] = 2;
 		info->num_sprites[PIPE_B] = 2;
 		info->num_sprites[PIPE_C] = 1;
-	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		for_each_pipe(dev_priv, pipe)
 			info->num_sprites[pipe] = 2;
-	else
+	} else if (INTEL_GEN(dev_priv) >= 5) {
 		for_each_pipe(dev_priv, pipe)
 			info->num_sprites[pipe] = 1;
+	}
 
 	if (i915.disable_display) {
 		DRM_INFO("Display disabled (module parameter)\n");
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 3cb70d73239b..cf5cff7b03b8 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -37,7 +37,6 @@
 #include "intel_frontbuffer.h"
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
-#include "i915_gem_dmabuf.h"
 #include "intel_dsi.h"
 #include "i915_trace.h"
 #include <drm/drm_atomic.h>
@@ -116,8 +115,9 @@ static void chv_prepare_pll(struct intel_crtc *crtc,
 			    const struct intel_crtc_state *pipe_config);
 static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
-static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc,
-	struct intel_crtc_state *crtc_state);
+static void skl_init_scalers(struct drm_i915_private *dev_priv,
+			     struct intel_crtc *crtc,
+			     struct intel_crtc_state *crtc_state);
 static void skylake_pfit_enable(struct intel_crtc *crtc);
 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force);
 static void ironlake_pfit_enable(struct intel_crtc *crtc);
@@ -600,7 +600,7 @@ int chv_calc_dpll_params(int refclk, struct dpll *clock)
  * the given connectors.
  */
 
-static bool intel_PLL_is_valid(struct drm_device *dev,
+static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv,
 			       const struct intel_limit *limit,
 			       const struct dpll *clock)
 {
@@ -613,12 +613,13 @@ static bool intel_PLL_is_valid(struct drm_device *dev,
 	if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 		INTELPllInvalid("m1 out of range\n");
 
-	if (!IS_PINEVIEW(dev) && !IS_VALLEYVIEW(dev) &&
-	    !IS_CHERRYVIEW(dev) && !IS_BROXTON(dev))
+	if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) &&
+	    !IS_CHERRYVIEW(dev_priv) && !IS_BROXTON(dev_priv))
 		if (clock->m1 <= clock->m2)
 			INTELPllInvalid("m1 <= m2\n");
 
-	if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && !IS_BROXTON(dev)) {
+	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
+	    !IS_BROXTON(dev_priv)) {
 		if (clock->p < limit->p.min || limit->p.max < clock->p)
 			INTELPllInvalid("p out of range\n");
 		if (clock->m < limit->m.min || limit->m.max < clock->m)
@@ -698,7 +699,8 @@ i9xx_find_best_dpll(const struct intel_limit *limit,
 					int this_err;
 
 					i9xx_calc_dpll_params(refclk, &clock);
-					if (!intel_PLL_is_valid(dev, limit,
+					if (!intel_PLL_is_valid(to_i915(dev),
+								limit,
 								&clock))
 						continue;
 					if (match_clock &&
@@ -753,7 +755,8 @@ pnv_find_best_dpll(const struct intel_limit *limit,
 					int this_err;
 
 					pnv_calc_dpll_params(refclk, &clock);
-					if (!intel_PLL_is_valid(dev, limit,
+					if (!intel_PLL_is_valid(to_i915(dev),
+								limit,
 								&clock))
 						continue;
 					if (match_clock &&
@@ -813,7 +816,8 @@ g4x_find_best_dpll(const struct intel_limit *limit,
 					int this_err;
 
 					i9xx_calc_dpll_params(refclk, &clock);
-					if (!intel_PLL_is_valid(dev, limit,
+					if (!intel_PLL_is_valid(to_i915(dev),
+								limit,
 								&clock))
 						continue;
 
@@ -845,7 +849,7 @@ static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 	 * For CHV ignore the error and consider only the P value.
 	 * Prefer a bigger P value based on HW requirements.
 	 */
-	if (IS_CHERRYVIEW(dev)) {
+	if (IS_CHERRYVIEW(to_i915(dev))) {
 		*error_ppm = 0;
 
 		return calculated_clock->p > best_clock->p;
@@ -909,7 +913,8 @@ vlv_find_best_dpll(const struct intel_limit *limit,
 
 					vlv_calc_dpll_params(refclk, &clock);
 
-					if (!intel_PLL_is_valid(dev, limit,
+					if (!intel_PLL_is_valid(to_i915(dev),
+								limit,
 								&clock))
 						continue;
 
@@ -977,7 +982,7 @@ chv_find_best_dpll(const struct intel_limit *limit,
 
 			chv_calc_dpll_params(refclk, &clock);
 
-			if (!intel_PLL_is_valid(dev, limit, &clock))
+			if (!intel_PLL_is_valid(to_i915(dev), limit, &clock))
 				continue;
 
 			if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
@@ -1003,10 +1008,8 @@ bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
 				  target_clock, refclk, NULL, best_clock);
 }
 
-bool intel_crtc_active(struct drm_crtc *crtc)
+bool intel_crtc_active(struct intel_crtc *crtc)
 {
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-
 	/* Be paranoid as we can arrive here with only partial
 	 * state retrieved from the hardware during setup.
 	 *
@@ -1020,27 +1023,25 @@ bool intel_crtc_active(struct drm_crtc *crtc)
 	 * crtc->state->active once we have proper CRTC states wired up
 	 * for atomic.
 	 */
-	return intel_crtc->active && crtc->primary->state->fb &&
-		intel_crtc->config->base.adjusted_mode.crtc_clock;
+	return crtc->active && crtc->base.primary->state->fb &&
+		crtc->config->base.adjusted_mode.crtc_clock;
 }
 
 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
 					     enum pipe pipe)
 {
-	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 
-	return intel_crtc->config->cpu_transcoder;
+	return crtc->config->cpu_transcoder;
 }
 
-static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
+static bool pipe_dsl_stopped(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	i915_reg_t reg = PIPEDSL(pipe);
 	u32 line1, line2;
 	u32 line_mask;
 
-	if (IS_GEN2(dev))
+	if (IS_GEN2(dev_priv))
 		line_mask = DSL_LINEMASK_GEN2;
 	else
 		line_mask = DSL_LINEMASK_GEN3;
@@ -1070,12 +1071,11 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
  */
 static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
 	enum pipe pipe = crtc->pipe;
 
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_GEN(dev_priv) >= 4) {
 		i915_reg_t reg = PIPECONF(cpu_transcoder);
 
 		/* Wait for the Pipe State to go off */
@@ -1085,7 +1085,7 @@ static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
 			WARN(1, "pipe_off wait timed out\n");
 	} else {
 		/* Wait for the display line to settle */
-		if (wait_for(pipe_dsl_stopped(dev, pipe), 100))
+		if (wait_for(pipe_dsl_stopped(dev_priv, pipe), 100))
 			WARN(1, "pipe_off wait timed out\n");
 	}
 }
@@ -1187,19 +1187,17 @@ void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
 			onoff(state), onoff(cur_state));
 }
 
-void assert_panel_unlocked(struct drm_i915_private *dev_priv,
-			   enum pipe pipe)
+void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	i915_reg_t pp_reg;
 	u32 val;
 	enum pipe panel_pipe = PIPE_A;
 	bool locked = true;
 
-	if (WARN_ON(HAS_DDI(dev)))
+	if (WARN_ON(HAS_DDI(dev_priv)))
 		return;
 
-	if (HAS_PCH_SPLIT(dev)) {
+	if (HAS_PCH_SPLIT(dev_priv)) {
 		u32 port_sel;
 
 		pp_reg = PP_CONTROL(0);
@@ -1209,7 +1207,7 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv,
 		    I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT)
 			panel_pipe = PIPE_B;
 		/* XXX: else fix for eDP */
-	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		/* presumably write lock depends on pipe, not port select */
 		pp_reg = PP_CONTROL(pipe);
 		panel_pipe = pipe;
@@ -1232,10 +1230,9 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv,
 static void assert_cursor(struct drm_i915_private *dev_priv,
 			  enum pipe pipe, bool state)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	bool cur_state;
 
-	if (IS_845G(dev) || IS_I865G(dev))
+	if (IS_845G(dev_priv) || IS_I865G(dev_priv))
 		cur_state = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
 	else
 		cur_state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE;
@@ -1294,11 +1291,10 @@ static void assert_plane(struct drm_i915_private *dev_priv,
 static void assert_planes_disabled(struct drm_i915_private *dev_priv,
 				   enum pipe pipe)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	int i;
 
 	/* Primary planes are fixed to pipes on gen4+ */
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_GEN(dev_priv) >= 4) {
 		u32 val = I915_READ(DSPCNTR(pipe));
 		I915_STATE_WARN(val & DISPLAY_PLANE_ENABLE,
 		     "plane %c assertion failure, should be disabled but not\n",
@@ -1320,29 +1316,28 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv,
 static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
 				    enum pipe pipe)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	int sprite;
 
-	if (INTEL_INFO(dev)->gen >= 9) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		for_each_sprite(dev_priv, pipe, sprite) {
 			u32 val = I915_READ(PLANE_CTL(pipe, sprite));
 			I915_STATE_WARN(val & PLANE_CTL_ENABLE,
 			     "plane %d assertion failure, should be off on pipe %c but is still active\n",
 			     sprite, pipe_name(pipe));
 		}
-	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		for_each_sprite(dev_priv, pipe, sprite) {
 			u32 val = I915_READ(SPCNTR(pipe, sprite));
 			I915_STATE_WARN(val & SP_ENABLE,
 			     "sprite %c assertion failure, should be off on pipe %c but is still active\n",
 			     sprite_name(pipe, sprite), pipe_name(pipe));
 		}
-	} else if (INTEL_INFO(dev)->gen >= 7) {
+	} else if (INTEL_GEN(dev_priv) >= 7) {
 		u32 val = I915_READ(SPRCTL(pipe));
 		I915_STATE_WARN(val & SPRITE_ENABLE,
 		     "sprite %c assertion failure, should be off on pipe %c but is still active\n",
 		     plane_name(pipe), pipe_name(pipe));
-	} else if (INTEL_INFO(dev)->gen >= 5) {
+	} else if (INTEL_GEN(dev_priv) >= 5) {
 		u32 val = I915_READ(DVSCNTR(pipe));
 		I915_STATE_WARN(val & DVS_ENABLE,
 		     "sprite %c assertion failure, should be off on pipe %c but is still active\n",
@@ -1596,12 +1591,12 @@ static void chv_enable_pll(struct intel_crtc *crtc,
 	}
 }
 
-static int intel_num_dvo_pipes(struct drm_device *dev)
+static int intel_num_dvo_pipes(struct drm_i915_private *dev_priv)
 {
 	struct intel_crtc *crtc;
 	int count = 0;
 
-	for_each_intel_crtc(dev, crtc) {
+	for_each_intel_crtc(&dev_priv->drm, crtc) {
 		count += crtc->base.state->active &&
 			intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO);
 	}
@@ -1611,19 +1606,18 @@ static int intel_num_dvo_pipes(struct drm_device *dev)
 
 static void i9xx_enable_pll(struct intel_crtc *crtc)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	i915_reg_t reg = DPLL(crtc->pipe);
 	u32 dpll = crtc->config->dpll_hw_state.dpll;
 
 	assert_pipe_disabled(dev_priv, crtc->pipe);
 
 	/* PLL is protected by panel, make sure we can write it */
-	if (IS_MOBILE(dev) && !IS_I830(dev))
+	if (IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
 		assert_panel_unlocked(dev_priv, crtc->pipe);
 
 	/* Enable DVO 2x clock on both PLLs if necessary */
-	if (IS_I830(dev) && intel_num_dvo_pipes(dev) > 0) {
+	if (IS_I830(dev_priv) && intel_num_dvo_pipes(dev_priv) > 0) {
 		/*
 		 * It appears to be important that we don't enable this
 		 * for the current pipe before otherwise configuring the
@@ -1648,7 +1642,7 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
 	POSTING_READ(reg);
 	udelay(150);
 
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_GEN(dev_priv) >= 4) {
 		I915_WRITE(DPLL_MD(crtc->pipe),
 			   crtc->config->dpll_hw_state.dpll_md);
 	} else {
@@ -1683,14 +1677,13 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
  */
 static void i9xx_disable_pll(struct intel_crtc *crtc)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 
 	/* Disable DVO 2x clock on both PLLs if necessary */
-	if (IS_I830(dev) &&
+	if (IS_I830(dev_priv) &&
 	    intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO) &&
-	    !intel_num_dvo_pipes(dev)) {
+	    !intel_num_dvo_pipes(dev_priv)) {
 		I915_WRITE(DPLL(PIPE_B),
 			   I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
 		I915_WRITE(DPLL(PIPE_A),
@@ -1786,9 +1779,8 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
 					   enum pipe pipe)
 {
-	struct drm_device *dev = &dev_priv->drm;
-	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
+								pipe);
 	i915_reg_t reg;
 	uint32_t val, pipeconf_val;
 
@@ -1799,7 +1791,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
 	assert_fdi_tx_enabled(dev_priv, pipe);
 	assert_fdi_rx_enabled(dev_priv, pipe);
 
-	if (HAS_PCH_CPT(dev)) {
+	if (HAS_PCH_CPT(dev_priv)) {
 		/* Workaround: Set the timing override bit before enabling the
 		 * pch transcoder. */
 		reg = TRANS_CHICKEN2(pipe);
@@ -1877,7 +1869,6 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
 					    enum pipe pipe)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	i915_reg_t reg;
 	uint32_t val;
 
@@ -1898,7 +1889,7 @@ static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
 				    50))
 		DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
 
-	if (HAS_PCH_CPT(dev)) {
+	if (HAS_PCH_CPT(dev_priv)) {
 		/* Workaround: Clear the timing override chicken bit again. */
 		reg = TRANS_CHICKEN2(pipe);
 		val = I915_READ(reg);
@@ -1926,6 +1917,18 @@ void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
 	I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
 }
 
+enum transcoder intel_crtc_pch_transcoder(struct intel_crtc *crtc)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+
+	WARN_ON(!crtc->config->has_pch_encoder);
+
+	if (HAS_PCH_LPT(dev_priv))
+		return TRANSCODER_A;
+	else
+		return (enum transcoder) crtc->pipe;
+}
+
 /**
  * intel_enable_pipe - enable a pipe, asserting requirements
  * @crtc: crtc responsible for the pipe
@@ -1939,7 +1942,6 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum pipe pipe = crtc->pipe;
 	enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
-	enum pipe pch_transcoder;
 	i915_reg_t reg;
 	u32 val;
 
@@ -1949,11 +1951,6 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
 	assert_cursor_disabled(dev_priv, pipe);
 	assert_sprites_disabled(dev_priv, pipe);
 
-	if (HAS_PCH_LPT(dev_priv))
-		pch_transcoder = TRANSCODER_A;
-	else
-		pch_transcoder = pipe;
-
 	/*
 	 * A pipe without a PLL won't actually be able to drive bits from
 	 * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
@@ -1967,7 +1964,8 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
 	} else {
 		if (crtc->config->has_pch_encoder) {
 			/* if driving the PCH, we need FDI enabled */
-			assert_fdi_rx_pll_enabled(dev_priv, pch_transcoder);
+			assert_fdi_rx_pll_enabled(dev_priv,
+						  (enum pipe) intel_crtc_pch_transcoder(crtc));
 			assert_fdi_tx_pll_enabled(dev_priv,
 						  (enum pipe) cpu_transcoder);
 		}
@@ -2139,7 +2137,7 @@ intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
 			const struct drm_framebuffer *fb,
 			unsigned int rotation)
 {
-	if (intel_rotation_90_or_270(rotation)) {
+	if (drm_rotation_90_or_270(rotation)) {
 		*view = i915_ggtt_view_rotated;
 		view->params.rotated = to_intel_framebuffer(fb)->rot_info;
 	} else {
@@ -2191,7 +2189,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
 
 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 
-	alignment = intel_surf_alignment(dev_priv, fb->modifier[0]);
+	alignment = intel_surf_alignment(dev_priv, fb->modifier);
 
 	intel_fill_fb_ggtt_view(&view, fb, rotation);
 
@@ -2260,7 +2258,7 @@ void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
 static int intel_fb_pitch(const struct drm_framebuffer *fb, int plane,
 			  unsigned int rotation)
 {
-	if (intel_rotation_90_or_270(rotation))
+	if (drm_rotation_90_or_270(rotation))
 		return to_intel_framebuffer(fb)->rotated[plane].pitch;
 	else
 		return fb->pitches[plane];
@@ -2296,7 +2294,7 @@ void intel_add_fb_offsets(int *x, int *y,
 	const struct intel_framebuffer *intel_fb = to_intel_framebuffer(state->base.fb);
 	unsigned int rotation = state->base.rotation;
 
-	if (intel_rotation_90_or_270(rotation)) {
+	if (drm_rotation_90_or_270(rotation)) {
 		*x += intel_fb->rotated[plane].x;
 		*y += intel_fb->rotated[plane].y;
 	} else {
@@ -2352,15 +2350,15 @@ static u32 intel_adjust_tile_offset(int *x, int *y,
 
 	WARN_ON(new_offset > old_offset);
 
-	if (fb->modifier[plane] != DRM_FORMAT_MOD_NONE) {
+	if (fb->modifier != DRM_FORMAT_MOD_NONE) {
 		unsigned int tile_size, tile_width, tile_height;
 		unsigned int pitch_tiles;
 
 		tile_size = intel_tile_size(dev_priv);
 		intel_tile_dims(dev_priv, &tile_width, &tile_height,
-				fb->modifier[plane], cpp);
+				fb->modifier, cpp);
 
-		if (intel_rotation_90_or_270(rotation)) {
+		if (drm_rotation_90_or_270(rotation)) {
 			pitch_tiles = pitch / tile_height;
 			swap(tile_width, tile_height);
 		} else {
@@ -2401,7 +2399,7 @@ static u32 _intel_compute_tile_offset(const struct drm_i915_private *dev_priv,
 				      unsigned int rotation,
 				      u32 alignment)
 {
-	uint64_t fb_modifier = fb->modifier[plane];
+	uint64_t fb_modifier = fb->modifier;
 	unsigned int cpp = drm_format_plane_cpp(fb->pixel_format, plane);
 	u32 offset, offset_aligned;
 
@@ -2416,7 +2414,7 @@ static u32 _intel_compute_tile_offset(const struct drm_i915_private *dev_priv,
 		intel_tile_dims(dev_priv, &tile_width, &tile_height,
 				fb_modifier, cpp);
 
-		if (intel_rotation_90_or_270(rotation)) {
+		if (drm_rotation_90_or_270(rotation)) {
 			pitch_tiles = pitch / tile_height;
 			swap(tile_width, tile_height);
 		} else {
@@ -2460,7 +2458,7 @@ u32 intel_compute_tile_offset(int *x, int *y,
 	if (fb->pixel_format == DRM_FORMAT_NV12 && plane == 1)
 		alignment = 4096;
 	else
-		alignment = intel_surf_alignment(dev_priv, fb->modifier[plane]);
+		alignment = intel_surf_alignment(dev_priv, fb->modifier);
 
 	return _intel_compute_tile_offset(dev_priv, x, y, fb, plane, pitch,
 					  rotation, alignment);
@@ -2542,13 +2540,13 @@ intel_fill_fb_info(struct drm_i915_private *dev_priv,
 						    DRM_ROTATE_0, tile_size);
 		offset /= tile_size;
 
-		if (fb->modifier[i] != DRM_FORMAT_MOD_NONE) {
+		if (fb->modifier != DRM_FORMAT_MOD_NONE) {
 			unsigned int tile_width, tile_height;
 			unsigned int pitch_tiles;
 			struct drm_rect r;
 
 			intel_tile_dims(dev_priv, &tile_width, &tile_height,
-					fb->modifier[i], cpp);
+					fb->modifier, cpp);
 
 			rot_info->plane[i].offset = offset;
 			rot_info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp);
@@ -2707,7 +2705,7 @@ intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
 	mode_cmd.width = fb->width;
 	mode_cmd.height = fb->height;
 	mode_cmd.pitches[0] = fb->pitches[0];
-	mode_cmd.modifier[0] = fb->modifier[0];
+	mode_cmd.modifier[0] = fb->modifier;
 	mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
 
 	if (intel_framebuffer_init(dev, to_intel_framebuffer(fb),
@@ -2817,14 +2815,8 @@ valid_fb:
 	plane_state->crtc_w = fb->width;
 	plane_state->crtc_h = fb->height;
 
-	intel_state->base.src.x1 = plane_state->src_x;
-	intel_state->base.src.y1 = plane_state->src_y;
-	intel_state->base.src.x2 = plane_state->src_x + plane_state->src_w;
-	intel_state->base.src.y2 = plane_state->src_y + plane_state->src_h;
-	intel_state->base.dst.x1 = plane_state->crtc_x;
-	intel_state->base.dst.y1 = plane_state->crtc_y;
-	intel_state->base.dst.x2 = plane_state->crtc_x + plane_state->crtc_w;
-	intel_state->base.dst.y2 = plane_state->crtc_y + plane_state->crtc_h;
+	intel_state->base.src = drm_plane_state_src(plane_state);
+	intel_state->base.dst = drm_plane_state_dest(plane_state);
 
 	obj = intel_fb_obj(fb);
 	if (i915_gem_object_is_tiled(obj))
@@ -2843,7 +2835,7 @@ static int skl_max_plane_width(const struct drm_framebuffer *fb, int plane,
 {
 	int cpp = drm_format_plane_cpp(fb->pixel_format, plane);
 
-	switch (fb->modifier[plane]) {
+	switch (fb->modifier) {
 	case DRM_FORMAT_MOD_NONE:
 	case I915_FORMAT_MOD_X_TILED:
 		switch (cpp) {
@@ -2874,7 +2866,7 @@ static int skl_max_plane_width(const struct drm_framebuffer *fb, int plane,
 		}
 		break;
 	default:
-		MISSING_CASE(fb->modifier[plane]);
+		MISSING_CASE(fb->modifier);
 	}
 
 	return 2048;
@@ -2902,7 +2894,7 @@ static int skl_check_main_surface(struct intel_plane_state *plane_state)
 	intel_add_fb_offsets(&x, &y, plane_state, 0);
 	offset = intel_compute_tile_offset(&x, &y, plane_state, 0);
 
-	alignment = intel_surf_alignment(dev_priv, fb->modifier[0]);
+	alignment = intel_surf_alignment(dev_priv, fb->modifier);
 
 	/*
 	 * AUX surface offset is specified as the distance from the
@@ -2919,7 +2911,7 @@ static int skl_check_main_surface(struct intel_plane_state *plane_state)
 	 *
 	 * TODO: linear and Y-tiled seem fine, Yf untested,
 	 */
-	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED) {
+	if (fb->modifier == I915_FORMAT_MOD_X_TILED) {
 		int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 
 		while ((x + w) * cpp > fb->pitches[0]) {
@@ -2976,7 +2968,7 @@ int skl_check_plane_surface(struct intel_plane_state *plane_state)
 	int ret;
 
 	/* Rotate src coordinates to match rotated GTT view */
-	if (intel_rotation_90_or_270(rotation))
+	if (drm_rotation_90_or_270(rotation))
 		drm_rect_rotate(&plane_state->base.src,
 				fb->width << 16, fb->height << 16,
 				DRM_ROTATE_270);
@@ -3006,11 +2998,9 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
 				      const struct intel_crtc_state *crtc_state,
 				      const struct intel_plane_state *plane_state)
 {
-	struct drm_device *dev = primary->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(primary->dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_framebuffer *fb = plane_state->base.fb;
-	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	int plane = intel_crtc->plane;
 	u32 linear_offset;
 	u32 dspcntr;
@@ -3023,7 +3013,7 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
 
 	dspcntr |= DISPLAY_PLANE_ENABLE;
 
-	if (INTEL_INFO(dev)->gen < 4) {
+	if (INTEL_GEN(dev_priv) < 4) {
 		if (intel_crtc->pipe == PIPE_B)
 			dspcntr |= DISPPLANE_SEL_PIPE_B;
 
@@ -3034,7 +3024,7 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
 			   ((crtc_state->pipe_src_h - 1) << 16) |
 			   (crtc_state->pipe_src_w - 1));
 		I915_WRITE(DSPPOS(plane), 0);
-	} else if (IS_CHERRYVIEW(dev) && plane == PLANE_B) {
+	} else if (IS_CHERRYVIEW(dev_priv) && plane == PLANE_B) {
 		I915_WRITE(PRIMSIZE(plane),
 			   ((crtc_state->pipe_src_h - 1) << 16) |
 			   (crtc_state->pipe_src_w - 1));
@@ -3069,28 +3059,34 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
 	}
 
 	if (INTEL_GEN(dev_priv) >= 4 &&
-	    fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
+	    fb->modifier == I915_FORMAT_MOD_X_TILED)
 		dspcntr |= DISPPLANE_TILED;
 
-	if (IS_G4X(dev))
+	if (rotation & DRM_ROTATE_180)
+		dspcntr |= DISPPLANE_ROTATE_180;
+
+	if (rotation & DRM_REFLECT_X)
+		dspcntr |= DISPPLANE_MIRROR;
+
+	if (IS_G4X(dev_priv))
 		dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
 
 	intel_add_fb_offsets(&x, &y, plane_state, 0);
 
-	if (INTEL_INFO(dev)->gen >= 4)
+	if (INTEL_GEN(dev_priv) >= 4)
 		intel_crtc->dspaddr_offset =
 			intel_compute_tile_offset(&x, &y, plane_state, 0);
 
-	if (rotation == DRM_ROTATE_180) {
-		dspcntr |= DISPPLANE_ROTATE_180;
-
-		x += (crtc_state->pipe_src_w - 1);
-		y += (crtc_state->pipe_src_h - 1);
+	if (rotation & DRM_ROTATE_180) {
+		x += crtc_state->pipe_src_w - 1;
+		y += crtc_state->pipe_src_h - 1;
+	} else if (rotation & DRM_REFLECT_X) {
+		x += crtc_state->pipe_src_w - 1;
 	}
 
 	linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
 
-	if (INTEL_INFO(dev)->gen < 4)
+	if (INTEL_GEN(dev_priv) < 4)
 		intel_crtc->dspaddr_offset = linear_offset;
 
 	intel_crtc->adjusted_x = x;
@@ -3099,14 +3095,17 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
 	I915_WRITE(reg, dspcntr);
 
 	I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_GEN(dev_priv) >= 4) {
 		I915_WRITE(DSPSURF(plane),
 			   intel_fb_gtt_offset(fb, rotation) +
 			   intel_crtc->dspaddr_offset);
 		I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
 		I915_WRITE(DSPLINOFF(plane), linear_offset);
-	} else
-		I915_WRITE(DSPADDR(plane), i915_gem_object_ggtt_offset(obj, NULL) + linear_offset);
+	} else {
+		I915_WRITE(DSPADDR(plane),
+			   intel_fb_gtt_offset(fb, rotation) +
+			   intel_crtc->dspaddr_offset);
+	}
 	POSTING_READ(reg);
 }
 
@@ -3145,7 +3144,7 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
 	dspcntr = DISPPLANE_GAMMA_ENABLE;
 	dspcntr |= DISPLAY_PLANE_ENABLE;
 
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
 
 	switch (fb->pixel_format) {
@@ -3171,10 +3170,13 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
 		BUG();
 	}
 
-	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
+	if (fb->modifier == I915_FORMAT_MOD_X_TILED)
 		dspcntr |= DISPPLANE_TILED;
 
-	if (!IS_HASWELL(dev) && !IS_BROADWELL(dev))
+	if (rotation & DRM_ROTATE_180)
+		dspcntr |= DISPPLANE_ROTATE_180;
+
+	if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv))
 		dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
 
 	intel_add_fb_offsets(&x, &y, plane_state, 0);
@@ -3182,13 +3184,11 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
 	intel_crtc->dspaddr_offset =
 		intel_compute_tile_offset(&x, &y, plane_state, 0);
 
-	if (rotation == DRM_ROTATE_180) {
-		dspcntr |= DISPPLANE_ROTATE_180;
-
-		if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
-			x += (crtc_state->pipe_src_w - 1);
-			y += (crtc_state->pipe_src_h - 1);
-		}
+	/* HSW+ does this automagically in hardware */
+	if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv) &&
+	    rotation & DRM_ROTATE_180) {
+		x += crtc_state->pipe_src_w - 1;
+		y += crtc_state->pipe_src_h - 1;
 	}
 
 	linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
@@ -3202,7 +3202,7 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
 	I915_WRITE(DSPSURF(plane),
 		   intel_fb_gtt_offset(fb, rotation) +
 		   intel_crtc->dspaddr_offset);
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
 		I915_WRITE(DSPOFFSET(plane), (y << 16) | x);
 	} else {
 		I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
@@ -3277,12 +3277,12 @@ u32 skl_plane_stride(const struct drm_framebuffer *fb, int plane,
 	 * The stride is either expressed as a multiple of 64 bytes chunks for
 	 * linear buffers or in number of tiles for tiled buffers.
 	 */
-	if (intel_rotation_90_or_270(rotation)) {
+	if (drm_rotation_90_or_270(rotation)) {
 		int cpp = drm_format_plane_cpp(fb->pixel_format, plane);
 
-		stride /= intel_tile_height(dev_priv, fb->modifier[0], cpp);
+		stride /= intel_tile_height(dev_priv, fb->modifier, cpp);
 	} else {
-		stride /= intel_fb_stride_alignment(dev_priv, fb->modifier[0],
+		stride /= intel_fb_stride_alignment(dev_priv, fb->modifier,
 						    fb->pixel_format);
 	}
 
@@ -3378,7 +3378,6 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_framebuffer *fb = plane_state->base.fb;
-	const struct skl_wm_values *wm = &dev_priv->wm.skl_results;
 	int pipe = intel_crtc->pipe;
 	u32 plane_ctl;
 	unsigned int rotation = plane_state->base.rotation;
@@ -3399,7 +3398,7 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
 		    PLANE_CTL_PIPE_CSC_ENABLE;
 
 	plane_ctl |= skl_plane_ctl_format(fb->pixel_format);
-	plane_ctl |= skl_plane_ctl_tiling(fb->modifier[0]);
+	plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
 	plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
 	plane_ctl |= skl_plane_ctl_rotation(rotation);
 
@@ -3414,9 +3413,6 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
 	intel_crtc->adjusted_x = src_x;
 	intel_crtc->adjusted_y = src_y;
 
-	if (wm->dirty_pipes & drm_crtc_mask(&intel_crtc->base))
-		skl_write_plane_wm(intel_crtc, wm, 0);
-
 	I915_WRITE(PLANE_CTL(pipe, 0), plane_ctl);
 	I915_WRITE(PLANE_OFFSET(pipe, 0), (src_y << 16) | src_x);
 	I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
@@ -3451,13 +3447,6 @@ static void skylake_disable_primary_plane(struct drm_plane *primary,
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
 
-	/*
-	 * We only populate skl_results on watermark updates, and if the
-	 * plane's visiblity isn't actually changing neither is its watermarks.
-	 */
-	if (!crtc->primary->state->visible)
-		skl_write_plane_wm(intel_crtc, &dev_priv->wm.skl_results, 0);
-
 	I915_WRITE(PLANE_CTL(pipe, 0), 0);
 	I915_WRITE(PLANE_SURF(pipe, 0), 0);
 	POSTING_READ(PLANE_SURF(pipe, 0));
@@ -3507,7 +3496,7 @@ __intel_display_resume(struct drm_device *dev,
 	int i, ret;
 
 	intel_modeset_setup_hw_state(dev);
-	i915_redisable_vga(dev);
+	i915_redisable_vga(to_i915(dev));
 
 	if (!state)
 		return 0;
@@ -3585,7 +3574,7 @@ void intel_prepare_reset(struct drm_i915_private *dev_priv)
 	return;
 
 err:
-	drm_atomic_state_free(state);
+	drm_atomic_state_put(state);
 }
 
 void intel_finish_reset(struct drm_i915_private *dev_priv)
@@ -3604,8 +3593,6 @@ void intel_finish_reset(struct drm_i915_private *dev_priv)
 
 	dev_priv->modeset_restore_state = NULL;
 
-	dev_priv->modeset_restore_state = NULL;
-
 	/* reset doesn't touch the display */
 	if (!gpu_reset_clobbers_display(dev_priv)) {
 		if (!state) {
@@ -3647,6 +3634,8 @@ void intel_finish_reset(struct drm_i915_private *dev_priv)
 		intel_hpd_init(dev_priv);
 	}
 
+	if (state)
+		drm_atomic_state_put(state);
 	drm_modeset_drop_locks(ctx);
 	drm_modeset_acquire_fini(ctx);
 	mutex_unlock(&dev->mode_config.mutex);
@@ -3684,8 +3673,7 @@ static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
 static void intel_update_pipe_config(struct intel_crtc *crtc,
 				     struct intel_crtc_state *old_crtc_state)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct intel_crtc_state *pipe_config =
 		to_intel_crtc_state(crtc->base.state);
 
@@ -3710,12 +3698,12 @@ static void intel_update_pipe_config(struct intel_crtc *crtc,
 		   (pipe_config->pipe_src_h - 1));
 
 	/* on skylake this is done by detaching scalers */
-	if (INTEL_INFO(dev)->gen >= 9) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		skl_detach_scalers(crtc);
 
 		if (pipe_config->pch_pfit.enabled)
 			skylake_pfit_enable(crtc);
-	} else if (HAS_PCH_SPLIT(dev)) {
+	} else if (HAS_PCH_SPLIT(dev_priv)) {
 		if (pipe_config->pch_pfit.enabled)
 			ironlake_pfit_enable(crtc);
 		else if (old_crtc_state->pch_pfit.enabled)
@@ -3735,7 +3723,7 @@ static void intel_fdi_normal_train(struct drm_crtc *crtc)
 	/* enable normal train */
 	reg = FDI_TX_CTL(pipe);
 	temp = I915_READ(reg);
-	if (IS_IVYBRIDGE(dev)) {
+	if (IS_IVYBRIDGE(dev_priv)) {
 		temp &= ~FDI_LINK_TRAIN_NONE_IVB;
 		temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
 	} else {
@@ -3746,7 +3734,7 @@ static void intel_fdi_normal_train(struct drm_crtc *crtc)
 
 	reg = FDI_RX_CTL(pipe);
 	temp = I915_READ(reg);
-	if (HAS_PCH_CPT(dev)) {
+	if (HAS_PCH_CPT(dev_priv)) {
 		temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
 		temp |= FDI_LINK_TRAIN_NORMAL_CPT;
 	} else {
@@ -3760,7 +3748,7 @@ static void intel_fdi_normal_train(struct drm_crtc *crtc)
 	udelay(1000);
 
 	/* IVB wants error correction enabled */
-	if (IS_IVYBRIDGE(dev))
+	if (IS_IVYBRIDGE(dev_priv))
 		I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
 			   FDI_FE_ERRC_ENABLE);
 }
@@ -3904,7 +3892,7 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
 
 	reg = FDI_RX_CTL(pipe);
 	temp = I915_READ(reg);
-	if (HAS_PCH_CPT(dev)) {
+	if (HAS_PCH_CPT(dev_priv)) {
 		temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
 		temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
 	} else {
@@ -3948,7 +3936,7 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
 	temp = I915_READ(reg);
 	temp &= ~FDI_LINK_TRAIN_NONE;
 	temp |= FDI_LINK_TRAIN_PATTERN_2;
-	if (IS_GEN6(dev)) {
+	if (IS_GEN6(dev_priv)) {
 		temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
 		/* SNB-B */
 		temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
@@ -3957,7 +3945,7 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
 
 	reg = FDI_RX_CTL(pipe);
 	temp = I915_READ(reg);
-	if (HAS_PCH_CPT(dev)) {
+	if (HAS_PCH_CPT(dev_priv)) {
 		temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
 		temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
 	} else {
@@ -4211,7 +4199,7 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc)
 	udelay(100);
 
 	/* Ironlake workaround, disable clock pointer after downing FDI */
-	if (HAS_PCH_IBX(dev))
+	if (HAS_PCH_IBX(dev_priv))
 		I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
 
 	/* still set train pattern 1 */
@@ -4223,7 +4211,7 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc)
 
 	reg = FDI_RX_CTL(pipe);
 	temp = I915_READ(reg);
-	if (HAS_PCH_CPT(dev)) {
+	if (HAS_PCH_CPT(dev_priv)) {
 		temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
 		temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
 	} else {
@@ -4241,6 +4229,7 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc)
 
 bool intel_has_pending_fb_unpin(struct drm_device *dev)
 {
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *crtc;
 
 	/* Note that we don't need to be called with mode_config.lock here
@@ -4255,7 +4244,7 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev)
 			continue;
 
 		if (crtc->flip_work)
-			intel_wait_for_vblank(dev, crtc->pipe);
+			intel_wait_for_vblank(dev_priv, crtc->pipe);
 
 		return true;
 	}
@@ -4546,7 +4535,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 
 	assert_pch_transcoder_disabled(dev_priv, pipe);
 
-	if (IS_IVYBRIDGE(dev))
+	if (IS_IVYBRIDGE(dev_priv))
 		ivybridge_update_fdi_bc_bifurcation(intel_crtc);
 
 	/* Write the TU size bits before fdi link training, so that error
@@ -4559,7 +4548,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 
 	/* We need to program the right clock selection before writing the pixel
 	 * mutliplier into the DPLL. */
-	if (HAS_PCH_CPT(dev)) {
+	if (HAS_PCH_CPT(dev_priv)) {
 		u32 sel;
 
 		temp = I915_READ(PCH_DPLL_SEL);
@@ -4589,7 +4578,8 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 	intel_fdi_normal_train(crtc);
 
 	/* For PCH DP, enable TRANS_DP_CTL */
-	if (HAS_PCH_CPT(dev) && intel_crtc_has_dp_encoder(intel_crtc->config)) {
+	if (HAS_PCH_CPT(dev_priv) &&
+	    intel_crtc_has_dp_encoder(intel_crtc->config)) {
 		const struct drm_display_mode *adjusted_mode =
 			&intel_crtc->config->base.adjusted_mode;
 		u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
@@ -4668,7 +4658,7 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
 		to_intel_crtc(crtc_state->base.crtc);
 	int need_scaling;
 
-	need_scaling = intel_rotation_90_or_270(rotation) ?
+	need_scaling = drm_rotation_90_or_270(rotation) ?
 		(src_h != dst_w || src_w != dst_h):
 		(src_w != dst_w || src_h != dst_h);
 
@@ -4729,13 +4719,8 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
  */
 int skl_update_scaler_crtc(struct intel_crtc_state *state)
 {
-	struct intel_crtc *intel_crtc = to_intel_crtc(state->base.crtc);
 	const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
 
-	DRM_DEBUG_KMS("Updating scaler for [CRTC:%d:%s] scaler_user index %u.%u\n",
-		      intel_crtc->base.base.id, intel_crtc->base.name,
-		      intel_crtc->pipe, SKL_CRTC_INDEX);
-
 	return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
 		&state->scaler_state.scaler_id, DRM_ROTATE_0,
 		state->pipe_src_w, state->pipe_src_h,
@@ -4756,7 +4741,6 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
 				   struct intel_plane_state *plane_state)
 {
 
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct intel_plane *intel_plane =
 		to_intel_plane(plane_state->base.plane);
 	struct drm_framebuffer *fb = plane_state->base.fb;
@@ -4764,10 +4748,6 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
 
 	bool force_detach = !fb || !plane_state->base.visible;
 
-	DRM_DEBUG_KMS("Updating scaler for [PLANE:%d:%s] scaler_user index %u.%u\n",
-		      intel_plane->base.base.id, intel_plane->base.name,
-		      intel_crtc->pipe, drm_plane_index(&intel_plane->base));
-
 	ret = skl_update_scaler(crtc_state, force_detach,
 				drm_plane_index(&intel_plane->base),
 				&plane_state->scaler_id,
@@ -4859,7 +4839,7 @@ static void ironlake_pfit_enable(struct intel_crtc *crtc)
 		 * as some pre-programmed values are broken,
 		 * e.g. x201.
 		 */
-		if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
+		if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
 			I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
 						 PF_PIPE_SEL_IVB(pipe));
 		else
@@ -4884,7 +4864,7 @@ void hsw_enable_ips(struct intel_crtc *crtc)
 	 */
 
 	assert_plane_enabled(dev_priv, crtc->plane);
-	if (IS_BROADWELL(dev)) {
+	if (IS_BROADWELL(dev_priv)) {
 		mutex_lock(&dev_priv->rps.hw_lock);
 		WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0xc0000000));
 		mutex_unlock(&dev_priv->rps.hw_lock);
@@ -4916,7 +4896,7 @@ void hsw_disable_ips(struct intel_crtc *crtc)
 		return;
 
 	assert_plane_enabled(dev_priv, crtc->plane);
-	if (IS_BROADWELL(dev)) {
+	if (IS_BROADWELL(dev_priv)) {
 		mutex_lock(&dev_priv->rps.hw_lock);
 		WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
 		mutex_unlock(&dev_priv->rps.hw_lock);
@@ -4931,7 +4911,7 @@ void hsw_disable_ips(struct intel_crtc *crtc)
 	}
 
 	/* We need to wait for a vblank before we can disable the plane. */
-	intel_wait_for_vblank(dev, crtc->pipe);
+	intel_wait_for_vblank(dev_priv, crtc->pipe);
 }
 
 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
@@ -4985,7 +4965,7 @@ intel_post_enable_primary(struct drm_crtc *crtc)
 	 * FIXME: Need to fix the logic to work when we turn off all planes
 	 * but leave the pipe running.
 	 */
-	if (IS_GEN2(dev))
+	if (IS_GEN2(dev_priv))
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
 
 	/* Underruns don't always raise interrupts, so check manually. */
@@ -5008,7 +4988,7 @@ intel_pre_disable_primary(struct drm_crtc *crtc)
 	 * FIXME: Need to fix the logic to work when we turn off all planes
 	 * but leave the pipe running.
 	 */
-	if (IS_GEN2(dev))
+	if (IS_GEN2(dev_priv))
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
 
 	/*
@@ -5040,10 +5020,10 @@ intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
 	 * event which is after the vblank start event, so we need to have a
 	 * wait-for-vblank between disabling the plane and the pipe.
 	 */
-	if (HAS_GMCH_DISPLAY(dev)) {
+	if (HAS_GMCH_DISPLAY(dev_priv)) {
 		intel_set_memory_cxsr(dev_priv, false);
 		dev_priv->wm.vlv.cxsr = false;
-		intel_wait_for_vblank(dev, pipe);
+		intel_wait_for_vblank(dev_priv, pipe);
 	}
 }
 
@@ -5062,7 +5042,7 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
 	crtc->wm.cxsr_allowed = true;
 
 	if (pipe_config->update_wm_post && pipe_config->base.active)
-		intel_update_watermarks(&crtc->base);
+		intel_update_watermarks(crtc);
 
 	if (old_pri_state) {
 		struct intel_plane_state *primary_state =
@@ -5091,6 +5071,8 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state)
 	struct drm_plane_state *old_pri_state =
 		drm_atomic_get_existing_plane_state(old_state, primary);
 	bool modeset = needs_modeset(&pipe_config->base);
+	struct intel_atomic_state *old_intel_state =
+		to_intel_atomic_state(old_state);
 
 	if (old_pri_state) {
 		struct intel_plane_state *primary_state =
@@ -5105,7 +5087,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state)
 			intel_pre_disable_primary(&crtc->base);
 	}
 
-	if (pipe_config->disable_cxsr && HAS_GMCH_DISPLAY(dev)) {
+	if (pipe_config->disable_cxsr && HAS_GMCH_DISPLAY(dev_priv)) {
 		crtc->wm.cxsr_allowed = false;
 
 		/*
@@ -5120,7 +5102,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state)
 		if (old_crtc_state->base.active) {
 			intel_set_memory_cxsr(dev_priv, false);
 			dev_priv->wm.vlv.cxsr = false;
-			intel_wait_for_vblank(dev, crtc->pipe);
+			intel_wait_for_vblank(dev_priv, crtc->pipe);
 		}
 	}
 
@@ -5133,7 +5115,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state)
 	 */
 	if (pipe_config->disable_lp_wm) {
 		ilk_disable_lp_wm(dev);
-		intel_wait_for_vblank(dev, crtc->pipe);
+		intel_wait_for_vblank(dev_priv, crtc->pipe);
 	}
 
 	/*
@@ -5158,9 +5140,10 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state)
 	 * us to.
 	 */
 	if (dev_priv->display.initial_watermarks != NULL)
-		dev_priv->display.initial_watermarks(pipe_config);
+		dev_priv->display.initial_watermarks(old_intel_state,
+						     pipe_config);
 	else if (pipe_config->update_wm_pre)
-		intel_update_watermarks(&crtc->base);
+		intel_update_watermarks(crtc);
 }
 
 static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask)
@@ -5314,6 +5297,8 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
+	struct intel_atomic_state *old_intel_state =
+		to_intel_atomic_state(old_state);
 
 	if (WARN_ON(intel_crtc->active))
 		return;
@@ -5372,7 +5357,7 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
 	intel_color_load_luts(&pipe_config->base);
 
 	if (dev_priv->display.initial_watermarks != NULL)
-		dev_priv->display.initial_watermarks(intel_crtc->config);
+		dev_priv->display.initial_watermarks(old_intel_state, intel_crtc->config);
 	intel_enable_pipe(intel_crtc);
 
 	if (intel_crtc->config->has_pch_encoder)
@@ -5383,12 +5368,12 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
 
 	intel_encoders_enable(crtc, pipe_config, old_state);
 
-	if (HAS_PCH_CPT(dev))
+	if (HAS_PCH_CPT(dev_priv))
 		cpt_verify_modeset(dev, intel_crtc->pipe);
 
 	/* Must wait for vblank to avoid spurious PCH FIFO underruns */
 	if (intel_crtc->config->has_pch_encoder)
-		intel_wait_for_vblank(dev, pipe);
+		intel_wait_for_vblank(dev_priv, pipe);
 	intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
 	intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
 }
@@ -5396,18 +5381,19 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
 /* IPS only exists on ULT machines and is tied to pipe A. */
 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
 {
-	return HAS_IPS(crtc->base.dev) && crtc->pipe == PIPE_A;
+	return HAS_IPS(to_i915(crtc->base.dev)) && crtc->pipe == PIPE_A;
 }
 
 static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 				struct drm_atomic_state *old_state)
 {
 	struct drm_crtc *crtc = pipe_config->base.crtc;
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe, hsw_workaround_pipe;
 	enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
+	struct intel_atomic_state *old_intel_state =
+		to_intel_atomic_state(old_state);
 
 	if (WARN_ON(intel_crtc->active))
 		return;
@@ -5462,7 +5448,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 	if (!transcoder_is_dsi(cpu_transcoder))
 		intel_ddi_enable_pipe_clock(intel_crtc);
 
-	if (INTEL_INFO(dev)->gen >= 9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		skylake_pfit_enable(intel_crtc);
 	else
 		ironlake_pfit_enable(intel_crtc);
@@ -5478,9 +5464,10 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 		intel_ddi_enable_transcoder_func(crtc);
 
 	if (dev_priv->display.initial_watermarks != NULL)
-		dev_priv->display.initial_watermarks(pipe_config);
+		dev_priv->display.initial_watermarks(old_intel_state,
+						     pipe_config);
 	else
-		intel_update_watermarks(crtc);
+		intel_update_watermarks(intel_crtc);
 
 	/* XXX: Do the pipe assertions at the right place for BXT DSI. */
 	if (!transcoder_is_dsi(cpu_transcoder))
@@ -5489,7 +5476,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 	if (intel_crtc->config->has_pch_encoder)
 		lpt_pch_enable(crtc);
 
-	if (intel_crtc->config->dp_encoder_is_mst)
+	if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DP_MST))
 		intel_ddi_set_vc_payload_alloc(crtc, true);
 
 	assert_vblank_disabled(crtc);
@@ -5498,8 +5485,8 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 	intel_encoders_enable(crtc, pipe_config, old_state);
 
 	if (intel_crtc->config->has_pch_encoder) {
-		intel_wait_for_vblank(dev, pipe);
-		intel_wait_for_vblank(dev, pipe);
+		intel_wait_for_vblank(dev_priv, pipe);
+		intel_wait_for_vblank(dev_priv, pipe);
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
 		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
 						      true);
@@ -5508,9 +5495,9 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 	/* If we change the relative order between pipe/planes enabling, we need
 	 * to change the workaround. */
 	hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
-	if (IS_HASWELL(dev) && hsw_workaround_pipe != INVALID_PIPE) {
-		intel_wait_for_vblank(dev, hsw_workaround_pipe);
-		intel_wait_for_vblank(dev, hsw_workaround_pipe);
+	if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
+		intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
+		intel_wait_for_vblank(dev_priv, hsw_workaround_pipe);
 	}
 }
 
@@ -5565,7 +5552,7 @@ static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state,
 	if (intel_crtc->config->has_pch_encoder) {
 		ironlake_disable_pch_transcoder(dev_priv, pipe);
 
-		if (HAS_PCH_CPT(dev)) {
+		if (HAS_PCH_CPT(dev_priv)) {
 			i915_reg_t reg;
 			u32 temp;
 
@@ -5594,8 +5581,7 @@ static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
 				 struct drm_atomic_state *old_state)
 {
 	struct drm_crtc *crtc = old_crtc_state->base.crtc;
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
 
@@ -5612,13 +5598,13 @@ static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
 	if (!transcoder_is_dsi(cpu_transcoder))
 		intel_disable_pipe(intel_crtc);
 
-	if (intel_crtc->config->dp_encoder_is_mst)
+	if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DP_MST))
 		intel_ddi_set_vc_payload_alloc(crtc, false);
 
 	if (!transcoder_is_dsi(cpu_transcoder))
 		intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
 
-	if (INTEL_INFO(dev)->gen >= 9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		skylake_scaler_disable(intel_crtc);
 	else
 		ironlake_pfit_disable(intel_crtc, false);
@@ -5699,13 +5685,13 @@ static enum intel_display_power_domain port_to_aux_power_domain(enum port port)
 enum intel_display_power_domain
 intel_display_port_power_domain(struct intel_encoder *intel_encoder)
 {
-	struct drm_device *dev = intel_encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
 	struct intel_digital_port *intel_dig_port;
 
 	switch (intel_encoder->type) {
 	case INTEL_OUTPUT_UNKNOWN:
 		/* Only DDI platforms should ever use this output type */
-		WARN_ON_ONCE(!HAS_DDI(dev));
+		WARN_ON_ONCE(!HAS_DDI(dev_priv));
 	case INTEL_OUTPUT_DP:
 	case INTEL_OUTPUT_HDMI:
 	case INTEL_OUTPUT_EDP:
@@ -5726,7 +5712,7 @@ intel_display_port_power_domain(struct intel_encoder *intel_encoder)
 enum intel_display_power_domain
 intel_display_port_aux_power_domain(struct intel_encoder *intel_encoder)
 {
-	struct drm_device *dev = intel_encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
 	struct intel_digital_port *intel_dig_port;
 
 	switch (intel_encoder->type) {
@@ -5739,7 +5725,7 @@ intel_display_port_aux_power_domain(struct intel_encoder *intel_encoder)
 		 * what's the status of the given connectors, play safe and
 		 * run the DP detection too.
 		 */
-		WARN_ON_ONCE(!HAS_DDI(dev));
+		WARN_ON_ONCE(!HAS_DDI(dev_priv));
 	case INTEL_OUTPUT_DP:
 	case INTEL_OUTPUT_EDP:
 		intel_dig_port = enc_to_dig_port(&intel_encoder->base);
@@ -5831,11 +5817,9 @@ static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
 
 static int skl_calc_cdclk(int max_pixclk, int vco);
 
-static void intel_update_max_cdclk(struct drm_device *dev)
+static void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
 		u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
 		int max_cdclk, vco;
 
@@ -5857,9 +5841,9 @@ static void intel_update_max_cdclk(struct drm_device *dev)
 			max_cdclk = 308571;
 
 		dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
-	} else if (IS_BROXTON(dev)) {
+	} else if (IS_BROXTON(dev_priv)) {
 		dev_priv->max_cdclk_freq = 624000;
-	} else if (IS_BROADWELL(dev))  {
+	} else if (IS_BROADWELL(dev_priv))  {
 		/*
 		 * FIXME with extra cooling we can allow
 		 * 540 MHz for ULX and 675 Mhz for ULT.
@@ -5868,15 +5852,15 @@ static void intel_update_max_cdclk(struct drm_device *dev)
 		 */
 		if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
 			dev_priv->max_cdclk_freq = 450000;
-		else if (IS_BDW_ULX(dev))
+		else if (IS_BDW_ULX(dev_priv))
 			dev_priv->max_cdclk_freq = 450000;
-		else if (IS_BDW_ULT(dev))
+		else if (IS_BDW_ULT(dev_priv))
 			dev_priv->max_cdclk_freq = 540000;
 		else
 			dev_priv->max_cdclk_freq = 675000;
-	} else if (IS_CHERRYVIEW(dev)) {
+	} else if (IS_CHERRYVIEW(dev_priv)) {
 		dev_priv->max_cdclk_freq = 320000;
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv)) {
 		dev_priv->max_cdclk_freq = 400000;
 	} else {
 		/* otherwise assume cdclk is fixed */
@@ -5892,11 +5876,9 @@ static void intel_update_max_cdclk(struct drm_device *dev)
 			 dev_priv->max_dotclk_freq);
 }
 
-static void intel_update_cdclk(struct drm_device *dev)
+static void intel_update_cdclk(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev);
+	dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev_priv);
 
 	if (INTEL_GEN(dev_priv) >= 9)
 		DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz, VCO: %d kHz, ref: %d kHz\n",
@@ -6057,14 +6039,14 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, int cdclk)
 		return;
 	}
 
-	intel_update_cdclk(&dev_priv->drm);
+	intel_update_cdclk(dev_priv);
 }
 
 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
 {
 	u32 cdctl, expected;
 
-	intel_update_cdclk(&dev_priv->drm);
+	intel_update_cdclk(dev_priv);
 
 	if (dev_priv->cdclk_pll.vco == 0 ||
 	    dev_priv->cdclk_freq == dev_priv->cdclk_pll.ref)
@@ -6197,7 +6179,7 @@ void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv, int vco)
 	dev_priv->skl_preferred_vco_freq = vco;
 
 	if (changed)
-		intel_update_max_cdclk(&dev_priv->drm);
+		intel_update_max_cdclk(dev_priv);
 }
 
 static void
@@ -6283,7 +6265,6 @@ static bool skl_cdclk_wait_for_pcu_ready(struct drm_i915_private *dev_priv)
 
 static void skl_set_cdclk(struct drm_i915_private *dev_priv, int cdclk, int vco)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	u32 freq_select, pcu_ack;
 
 	WARN_ON((cdclk == 24000) != (vco == 0));
@@ -6334,7 +6315,7 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, int cdclk, int vco)
 	sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack);
 	mutex_unlock(&dev_priv->rps.hw_lock);
 
-	intel_update_cdclk(dev);
+	intel_update_cdclk(dev_priv);
 }
 
 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv);
@@ -6381,7 +6362,7 @@ static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
 	if ((I915_READ(SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
 		goto sanitize;
 
-	intel_update_cdclk(&dev_priv->drm);
+	intel_update_cdclk(dev_priv);
 	/* Is PLL enabled and locked ? */
 	if (dev_priv->cdclk_pll.vco == 0 ||
 	    dev_priv->cdclk_freq == dev_priv->cdclk_pll.ref)
@@ -6415,7 +6396,7 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 val, cmd;
 
-	WARN_ON(dev_priv->display.get_display_clock_speed(dev)
+	WARN_ON(dev_priv->display.get_display_clock_speed(dev_priv)
 					!= dev_priv->cdclk_freq);
 
 	if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
@@ -6472,7 +6453,7 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk)
 
 	mutex_unlock(&dev_priv->sb_lock);
 
-	intel_update_cdclk(dev);
+	intel_update_cdclk(dev_priv);
 }
 
 static void cherryview_set_cdclk(struct drm_device *dev, int cdclk)
@@ -6480,7 +6461,7 @@ static void cherryview_set_cdclk(struct drm_device *dev, int cdclk)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 val, cmd;
 
-	WARN_ON(dev_priv->display.get_display_clock_speed(dev)
+	WARN_ON(dev_priv->display.get_display_clock_speed(dev_priv)
 						!= dev_priv->cdclk_freq);
 
 	switch (cdclk) {
@@ -6513,7 +6494,7 @@ static void cherryview_set_cdclk(struct drm_device *dev, int cdclk)
 	}
 	mutex_unlock(&dev_priv->rps.hw_lock);
 
-	intel_update_cdclk(dev);
+	intel_update_cdclk(dev_priv);
 }
 
 static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv,
@@ -6676,7 +6657,7 @@ static void valleyview_modeset_commit_cdclk(struct drm_atomic_state *old_state)
 	 */
 	intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A);
 
-	if (IS_CHERRYVIEW(dev))
+	if (IS_CHERRYVIEW(dev_priv))
 		cherryview_set_cdclk(dev, req_cdclk);
 	else
 		valleyview_set_cdclk(dev, req_cdclk);
@@ -6704,7 +6685,7 @@ static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
 	intel_set_pipe_timings(intel_crtc);
 	intel_set_pipe_src_size(intel_crtc);
 
-	if (IS_CHERRYVIEW(dev) && pipe == PIPE_B) {
+	if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
 		struct drm_i915_private *dev_priv = to_i915(dev);
 
 		I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
@@ -6719,7 +6700,7 @@ static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
 
 	intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
 
-	if (IS_CHERRYVIEW(dev)) {
+	if (IS_CHERRYVIEW(dev_priv)) {
 		chv_prepare_pll(intel_crtc, intel_crtc->config);
 		chv_enable_pll(intel_crtc, intel_crtc->config);
 	} else {
@@ -6733,7 +6714,7 @@ static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
 
 	intel_color_load_luts(&pipe_config->base);
 
-	intel_update_watermarks(crtc);
+	intel_update_watermarks(intel_crtc);
 	intel_enable_pipe(intel_crtc);
 
 	assert_vblank_disabled(crtc);
@@ -6775,7 +6756,7 @@ static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
 
 	intel_crtc->active = true;
 
-	if (!IS_GEN2(dev))
+	if (!IS_GEN2(dev_priv))
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
 
 	intel_encoders_pre_enable(crtc, pipe_config, old_state);
@@ -6786,7 +6767,7 @@ static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
 
 	intel_color_load_luts(&pipe_config->base);
 
-	intel_update_watermarks(crtc);
+	intel_update_watermarks(intel_crtc);
 	intel_enable_pipe(intel_crtc);
 
 	assert_vblank_disabled(crtc);
@@ -6823,8 +6804,8 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
 	 * On gen2 planes are double buffered but the pipe isn't, so we must
 	 * wait for planes to fully turn off before disabling the pipe.
 	 */
-	if (IS_GEN2(dev))
-		intel_wait_for_vblank(dev, pipe);
+	if (IS_GEN2(dev_priv))
+		intel_wait_for_vblank(dev_priv, pipe);
 
 	intel_encoders_disable(crtc, old_crtc_state, old_state);
 
@@ -6838,9 +6819,9 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
 	intel_encoders_post_disable(crtc, old_crtc_state, old_state);
 
 	if (!intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DSI)) {
-		if (IS_CHERRYVIEW(dev))
+		if (IS_CHERRYVIEW(dev_priv))
 			chv_disable_pll(dev_priv, pipe);
-		else if (IS_VALLEYVIEW(dev))
+		else if (IS_VALLEYVIEW(dev_priv))
 			vlv_disable_pll(dev_priv, pipe);
 		else
 			i9xx_disable_pll(intel_crtc);
@@ -6848,7 +6829,7 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
 
 	intel_encoders_post_pll_disable(crtc, old_crtc_state, old_state);
 
-	if (!IS_GEN2(dev))
+	if (!IS_GEN2(dev_priv))
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
 }
 
@@ -6886,7 +6867,7 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
 
 	dev_priv->display.crtc_disable(crtc_state, state);
 
-	drm_atomic_state_free(state);
+	drm_atomic_state_put(state);
 
 	DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
 		      crtc->base.id, crtc->name);
@@ -6902,7 +6883,7 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
 		encoder->base.crtc = NULL;
 
 	intel_fbc_disable(intel_crtc);
-	intel_update_watermarks(crtc);
+	intel_update_watermarks(intel_crtc);
 	intel_disable_shared_dpll(intel_crtc);
 
 	domains = intel_crtc->enabled_power_domains;
@@ -7028,6 +7009,7 @@ static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
 				     struct intel_crtc_state *pipe_config)
 {
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_atomic_state *state = pipe_config->base.state;
 	struct intel_crtc *other_crtc;
 	struct intel_crtc_state *other_crtc_state;
@@ -7040,7 +7022,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
 		return -EINVAL;
 	}
 
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
 		if (pipe_config->fdi_lanes > 2) {
 			DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
 				      pipe_config->fdi_lanes);
@@ -7050,7 +7032,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
 		}
 	}
 
-	if (INTEL_INFO(dev)->num_pipes == 2)
+	if (INTEL_INFO(dev_priv)->num_pipes == 2)
 		return 0;
 
 	/* Ivybridge 3 pipe is really complicated */
@@ -7061,7 +7043,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
 		if (pipe_config->fdi_lanes <= 2)
 			return 0;
 
-		other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_C));
+		other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C);
 		other_crtc_state =
 			intel_atomic_get_crtc_state(state, other_crtc);
 		if (IS_ERR(other_crtc_state))
@@ -7080,7 +7062,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
 			return -EINVAL;
 		}
 
-		other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_B));
+		other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B);
 		other_crtc_state =
 			intel_atomic_get_crtc_state(state, other_crtc);
 		if (IS_ERR(other_crtc_state))
@@ -7191,7 +7173,7 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
 	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 	int clock_limit = dev_priv->max_dotclk_freq;
 
-	if (INTEL_INFO(dev)->gen < 4) {
+	if (INTEL_GEN(dev_priv) < 4) {
 		clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
 
 		/*
@@ -7225,11 +7207,11 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
 	/* Cantiga+ cannot handle modes with a hsync front porch of 0.
 	 * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
 	 */
-	if ((INTEL_INFO(dev)->gen > 4 || IS_G4X(dev)) &&
+	if ((INTEL_GEN(dev_priv) > 4 || IS_G4X(dev_priv)) &&
 		adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
 		return -EINVAL;
 
-	if (HAS_IPS(dev))
+	if (HAS_IPS(dev_priv))
 		hsw_compute_ips_config(crtc, pipe_config);
 
 	if (pipe_config->has_pch_encoder)
@@ -7238,10 +7220,9 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
 	return 0;
 }
 
-static int skylake_get_display_clock_speed(struct drm_device *dev)
+static int skylake_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	uint32_t cdctl;
+	u32 cdctl;
 
 	skl_dpll0_update(dev_priv);
 
@@ -7300,9 +7281,8 @@ static void bxt_de_pll_update(struct drm_i915_private *dev_priv)
 		dev_priv->cdclk_pll.ref;
 }
 
-static int broxton_get_display_clock_speed(struct drm_device *dev)
+static int broxton_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 divider;
 	int div, vco;
 
@@ -7335,9 +7315,8 @@ static int broxton_get_display_clock_speed(struct drm_device *dev)
 	return DIV_ROUND_CLOSEST(vco, div);
 }
 
-static int broadwell_get_display_clock_speed(struct drm_device *dev)
+static int broadwell_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t lcpll = I915_READ(LCPLL_CTL);
 	uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
 
@@ -7355,9 +7334,8 @@ static int broadwell_get_display_clock_speed(struct drm_device *dev)
 		return 675000;
 }
 
-static int haswell_get_display_clock_speed(struct drm_device *dev)
+static int haswell_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t lcpll = I915_READ(LCPLL_CTL);
 	uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
 
@@ -7367,41 +7345,41 @@ static int haswell_get_display_clock_speed(struct drm_device *dev)
 		return 450000;
 	else if (freq == LCPLL_CLK_FREQ_450)
 		return 450000;
-	else if (IS_HSW_ULT(dev))
+	else if (IS_HSW_ULT(dev_priv))
 		return 337500;
 	else
 		return 540000;
 }
 
-static int valleyview_get_display_clock_speed(struct drm_device *dev)
+static int valleyview_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
-	return vlv_get_cck_clock_hpll(to_i915(dev), "cdclk",
+	return vlv_get_cck_clock_hpll(dev_priv, "cdclk",
 				      CCK_DISPLAY_CLOCK_CONTROL);
 }
 
-static int ilk_get_display_clock_speed(struct drm_device *dev)
+static int ilk_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
 	return 450000;
 }
 
-static int i945_get_display_clock_speed(struct drm_device *dev)
+static int i945_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
 	return 400000;
 }
 
-static int i915_get_display_clock_speed(struct drm_device *dev)
+static int i915_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
 	return 333333;
 }
 
-static int i9xx_misc_get_display_clock_speed(struct drm_device *dev)
+static int i9xx_misc_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
 	return 200000;
 }
 
-static int pnv_get_display_clock_speed(struct drm_device *dev)
+static int pnv_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
-	struct pci_dev *pdev = dev->pdev;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	u16 gcfgc = 0;
 
 	pci_read_config_word(pdev, GCFGC, &gcfgc);
@@ -7424,9 +7402,9 @@ static int pnv_get_display_clock_speed(struct drm_device *dev)
 	}
 }
 
-static int i915gm_get_display_clock_speed(struct drm_device *dev)
+static int i915gm_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
-	struct pci_dev *pdev = dev->pdev;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	u16 gcfgc = 0;
 
 	pci_read_config_word(pdev, GCFGC, &gcfgc);
@@ -7444,14 +7422,14 @@ static int i915gm_get_display_clock_speed(struct drm_device *dev)
 	}
 }
 
-static int i865_get_display_clock_speed(struct drm_device *dev)
+static int i865_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
 	return 266667;
 }
 
-static int i85x_get_display_clock_speed(struct drm_device *dev)
+static int i85x_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
-	struct pci_dev *pdev = dev->pdev;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	u16 hpllcc = 0;
 
 	/*
@@ -7487,14 +7465,13 @@ static int i85x_get_display_clock_speed(struct drm_device *dev)
 	return 0;
 }
 
-static int i830_get_display_clock_speed(struct drm_device *dev)
+static int i830_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
 	return 133333;
 }
 
-static unsigned int intel_hpll_vco(struct drm_device *dev)
+static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	static const unsigned int blb_vco[8] = {
 		[0] = 3200000,
 		[1] = 4000000,
@@ -7537,20 +7514,20 @@ static unsigned int intel_hpll_vco(struct drm_device *dev)
 	uint8_t tmp = 0;
 
 	/* FIXME other chipsets? */
-	if (IS_GM45(dev))
+	if (IS_GM45(dev_priv))
 		vco_table = ctg_vco;
-	else if (IS_G4X(dev))
+	else if (IS_G4X(dev_priv))
 		vco_table = elk_vco;
-	else if (IS_CRESTLINE(dev))
+	else if (IS_CRESTLINE(dev_priv))
 		vco_table = cl_vco;
-	else if (IS_PINEVIEW(dev))
+	else if (IS_PINEVIEW(dev_priv))
 		vco_table = pnv_vco;
-	else if (IS_G33(dev))
+	else if (IS_G33(dev_priv))
 		vco_table = blb_vco;
 	else
 		return 0;
 
-	tmp = I915_READ(IS_MOBILE(dev) ? HPLLVCO_MOBILE : HPLLVCO);
+	tmp = I915_READ(IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
 
 	vco = vco_table[tmp & 0x7];
 	if (vco == 0)
@@ -7561,10 +7538,10 @@ static unsigned int intel_hpll_vco(struct drm_device *dev)
 	return vco;
 }
 
-static int gm45_get_display_clock_speed(struct drm_device *dev)
+static int gm45_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
-	struct pci_dev *pdev = dev->pdev;
-	unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
+	struct pci_dev *pdev = dev_priv->drm.pdev;
+	unsigned int cdclk_sel, vco = intel_hpll_vco(dev_priv);
 	uint16_t tmp = 0;
 
 	pci_read_config_word(pdev, GCFGC, &tmp);
@@ -7584,14 +7561,14 @@ static int gm45_get_display_clock_speed(struct drm_device *dev)
 	}
 }
 
-static int i965gm_get_display_clock_speed(struct drm_device *dev)
+static int i965gm_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
-	struct pci_dev *pdev = dev->pdev;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	static const uint8_t div_3200[] = { 16, 10,  8 };
 	static const uint8_t div_4000[] = { 20, 12, 10 };
 	static const uint8_t div_5333[] = { 24, 16, 14 };
 	const uint8_t *div_table;
-	unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
+	unsigned int cdclk_sel, vco = intel_hpll_vco(dev_priv);
 	uint16_t tmp = 0;
 
 	pci_read_config_word(pdev, GCFGC, &tmp);
@@ -7622,15 +7599,15 @@ fail:
 	return 200000;
 }
 
-static int g33_get_display_clock_speed(struct drm_device *dev)
+static int g33_get_display_clock_speed(struct drm_i915_private *dev_priv)
 {
-	struct pci_dev *pdev = dev->pdev;
+	struct pci_dev *pdev = dev_priv->drm.pdev;
 	static const uint8_t div_3200[] = { 12, 10,  8,  7, 5, 16 };
 	static const uint8_t div_4000[] = { 14, 12, 10,  8, 6, 20 };
 	static const uint8_t div_4800[] = { 20, 14, 12, 10, 8, 24 };
 	static const uint8_t div_5333[] = { 20, 16, 12, 12, 8, 28 };
 	const uint8_t *div_table;
-	unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
+	unsigned int cdclk_sel, vco = intel_hpll_vco(dev_priv);
 	uint16_t tmp = 0;
 
 	pci_read_config_word(pdev, GCFGC, &tmp);
@@ -7719,10 +7696,10 @@ static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
 				     struct intel_crtc_state *crtc_state,
 				     struct dpll *reduced_clock)
 {
-	struct drm_device *dev = crtc->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	u32 fp, fp2 = 0;
 
-	if (IS_PINEVIEW(dev)) {
+	if (IS_PINEVIEW(dev_priv)) {
 		fp = pnv_dpll_compute_fp(&crtc_state->dpll);
 		if (reduced_clock)
 			fp2 = pnv_dpll_compute_fp(reduced_clock);
@@ -7790,12 +7767,11 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
 					 struct intel_link_m_n *m_n,
 					 struct intel_link_m_n *m2_n2)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	int pipe = crtc->pipe;
 	enum transcoder transcoder = crtc->config->cpu_transcoder;
 
-	if (INTEL_INFO(dev)->gen >= 5) {
+	if (INTEL_GEN(dev_priv) >= 5) {
 		I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
 		I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
 		I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
@@ -7804,8 +7780,8 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
 		 * for gen < 8) and if DRRS is supported (to make sure the
 		 * registers are not unnecessarily accessed).
 		 */
-		if (m2_n2 && (IS_CHERRYVIEW(dev) || INTEL_INFO(dev)->gen < 8) &&
-			crtc->config->has_drrs) {
+		if (m2_n2 && (IS_CHERRYVIEW(dev_priv) ||
+		    INTEL_GEN(dev_priv) < 8) && crtc->config->has_drrs) {
 			I915_WRITE(PIPE_DATA_M2(transcoder),
 					TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
 			I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
@@ -8092,11 +8068,10 @@ static void chv_prepare_pll(struct intel_crtc *crtc,
  * in cases where we need the PLL enabled even when @pipe is not going to
  * be enabled.
  */
-int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe,
+int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
 		     const struct dpll *dpll)
 {
-	struct intel_crtc *crtc =
-		to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe));
+	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 	struct intel_crtc_state *pipe_config;
 
 	pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL);
@@ -8107,7 +8082,7 @@ int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe,
 	pipe_config->pixel_multiplier = 1;
 	pipe_config->dpll = *dpll;
 
-	if (IS_CHERRYVIEW(dev)) {
+	if (IS_CHERRYVIEW(dev_priv)) {
 		chv_compute_dpll(crtc, pipe_config);
 		chv_prepare_pll(crtc, pipe_config);
 		chv_enable_pll(crtc, pipe_config);
@@ -8130,20 +8105,19 @@ int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe,
  * Disable the PLL for @pipe. To be used in cases where we need
  * the PLL enabled even when @pipe is not going to be enabled.
  */
-void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe)
+void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
-	if (IS_CHERRYVIEW(dev))
-		chv_disable_pll(to_i915(dev), pipe);
+	if (IS_CHERRYVIEW(dev_priv))
+		chv_disable_pll(dev_priv, pipe);
 	else
-		vlv_disable_pll(to_i915(dev), pipe);
+		vlv_disable_pll(dev_priv, pipe);
 }
 
 static void i9xx_compute_dpll(struct intel_crtc *crtc,
 			      struct intel_crtc_state *crtc_state,
 			      struct dpll *reduced_clock)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	u32 dpll;
 	struct dpll *clock = &crtc_state->dpll;
 
@@ -8156,7 +8130,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc,
 	else
 		dpll |= DPLLB_MODE_DAC_SERIAL;
 
-	if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
+	if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) || IS_G33(dev_priv)) {
 		dpll |= (crtc_state->pixel_multiplier - 1)
 			<< SDVO_MULTIPLIER_SHIFT_HIRES;
 	}
@@ -8169,11 +8143,11 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc,
 		dpll |= DPLL_SDVO_HIGH_SPEED;
 
 	/* compute bitmask from p1 value */
-	if (IS_PINEVIEW(dev))
+	if (IS_PINEVIEW(dev_priv))
 		dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
 	else {
 		dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
-		if (IS_G4X(dev) && reduced_clock)
+		if (IS_G4X(dev_priv) && reduced_clock)
 			dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
 	}
 	switch (clock->p2) {
@@ -8190,7 +8164,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc,
 		dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
 		break;
 	}
-	if (INTEL_INFO(dev)->gen >= 4)
+	if (INTEL_GEN(dev_priv) >= 4)
 		dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
 
 	if (crtc_state->sdvo_tv_clock)
@@ -8204,7 +8178,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc,
 	dpll |= DPLL_VCO_ENABLE;
 	crtc_state->dpll_hw_state.dpll = dpll;
 
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_GEN(dev_priv) >= 4) {
 		u32 dpll_md = (crtc_state->pixel_multiplier - 1)
 			<< DPLL_MD_UDI_MULTIPLIER_SHIFT;
 		crtc_state->dpll_hw_state.dpll_md = dpll_md;
@@ -8235,7 +8209,8 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc,
 			dpll |= PLL_P2_DIVIDE_BY_4;
 	}
 
-	if (!IS_I830(dev) && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
+	if (!IS_I830(dev_priv) &&
+	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
 		dpll |= DPLL_DVO_2X_MODE;
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
@@ -8250,8 +8225,7 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc,
 
 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
 {
-	struct drm_device *dev = intel_crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
 	enum pipe pipe = intel_crtc->pipe;
 	enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
 	const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
@@ -8277,7 +8251,7 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
 			vsyncshift += adjusted_mode->crtc_htotal;
 	}
 
-	if (INTEL_INFO(dev)->gen > 3)
+	if (INTEL_GEN(dev_priv) > 3)
 		I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
 
 	I915_WRITE(HTOTAL(cpu_transcoder),
@@ -8304,7 +8278,7 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
 	 * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
 	 * documented on the DDI_FUNC_CTL register description, EDP Input Select
 	 * bits. */
-	if (IS_HASWELL(dev) && cpu_transcoder == TRANSCODER_EDP &&
+	if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
 	    (pipe == PIPE_B || pipe == PIPE_C))
 		I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
 
@@ -8400,8 +8374,7 @@ void intel_mode_from_pipe_config(struct drm_display_mode *mode,
 
 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
 {
-	struct drm_device *dev = intel_crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
 	uint32_t pipeconf;
 
 	pipeconf = 0;
@@ -8414,7 +8387,8 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
 		pipeconf |= PIPECONF_DOUBLE_WIDE;
 
 	/* only g4x and later have fancy bpc/dither controls */
-	if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
+	    IS_CHERRYVIEW(dev_priv)) {
 		/* Bspec claims that we can't use dithering for 30bpp pipes. */
 		if (intel_crtc->config->dither && intel_crtc->config->pipe_bpp != 30)
 			pipeconf |= PIPECONF_DITHER_EN |
@@ -8436,7 +8410,7 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
 		}
 	}
 
-	if (HAS_PIPE_CXSR(dev)) {
+	if (HAS_PIPE_CXSR(dev_priv)) {
 		if (intel_crtc->lowfreq_avail) {
 			DRM_DEBUG_KMS("enabling CxSR downclocking\n");
 			pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
@@ -8446,7 +8420,7 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
 	}
 
 	if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
-		if (INTEL_INFO(dev)->gen < 4 ||
+		if (INTEL_GEN(dev_priv) < 4 ||
 		    intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO))
 			pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
 		else
@@ -8454,7 +8428,7 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
 	} else
 		pipeconf |= PIPECONF_PROGRESSIVE;
 
-	if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 	     intel_crtc->config->limited_color_range)
 		pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
 
@@ -8654,11 +8628,11 @@ static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
 				 struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	uint32_t tmp;
 
-	if (INTEL_INFO(dev)->gen <= 3 && (IS_I830(dev) || !IS_MOBILE(dev)))
+	if (INTEL_GEN(dev_priv) <= 3 &&
+	    (IS_I830(dev_priv) || !IS_MOBILE(dev_priv)))
 		return;
 
 	tmp = I915_READ(PFIT_CONTROL);
@@ -8666,7 +8640,7 @@ static void i9xx_get_pfit_config(struct intel_crtc *crtc,
 		return;
 
 	/* Check whether the pfit is attached to our pipe. */
-	if (INTEL_INFO(dev)->gen < 4) {
+	if (INTEL_GEN(dev_priv) < 4) {
 		if (crtc->pipe != PIPE_B)
 			return;
 	} else {
@@ -8730,10 +8704,10 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc,
 
 	fb = &intel_fb->base;
 
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_GEN(dev_priv) >= 4) {
 		if (val & DISPPLANE_TILED) {
 			plane_config->tiling = I915_TILING_X;
-			fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
+			fb->modifier = I915_FORMAT_MOD_X_TILED;
 		}
 	}
 
@@ -8742,7 +8716,7 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc,
 	fb->pixel_format = fourcc;
 	fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
 
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_GEN(dev_priv) >= 4) {
 		if (plane_config->tiling)
 			offset = I915_READ(DSPTILEOFF(plane));
 		else
@@ -8762,7 +8736,7 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc,
 
 	aligned_height = intel_fb_align_height(dev, fb->height,
 					       fb->pixel_format,
-					       fb->modifier[0]);
+					       fb->modifier);
 
 	plane_config->size = fb->pitches[0] * aligned_height;
 
@@ -8811,8 +8785,7 @@ static void chv_crtc_clock_get(struct intel_crtc *crtc,
 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 				 struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum intel_display_power_domain power_domain;
 	uint32_t tmp;
 	bool ret;
@@ -8830,7 +8803,8 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 	if (!(tmp & PIPECONF_ENABLE))
 		goto out;
 
-	if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
+	    IS_CHERRYVIEW(dev_priv)) {
 		switch (tmp & PIPECONF_BPC_MASK) {
 		case PIPECONF_6BPC:
 			pipe_config->pipe_bpp = 18;
@@ -8846,11 +8820,11 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 		}
 	}
 
-	if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 	    (tmp & PIPECONF_COLOR_RANGE_SELECT))
 		pipe_config->limited_color_range = true;
 
-	if (INTEL_INFO(dev)->gen < 4)
+	if (INTEL_GEN(dev_priv) < 4)
 		pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
 
 	intel_get_pipe_timings(crtc, pipe_config);
@@ -8858,9 +8832,9 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 
 	i9xx_get_pfit_config(crtc, pipe_config);
 
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_GEN(dev_priv) >= 4) {
 		/* No way to read it out on pipes B and C */
-		if (IS_CHERRYVIEW(dev) && crtc->pipe != PIPE_A)
+		if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
 			tmp = dev_priv->chv_dpll_md[crtc->pipe];
 		else
 			tmp = I915_READ(DPLL_MD(crtc->pipe));
@@ -8868,7 +8842,8 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 			((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
 			 >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
 		pipe_config->dpll_hw_state.dpll_md = tmp;
-	} else if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
+	} else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
+		   IS_G33(dev_priv)) {
 		tmp = I915_READ(DPLL(crtc->pipe));
 		pipe_config->pixel_multiplier =
 			((tmp & SDVO_MULTIPLIER_MASK)
@@ -8880,13 +8855,13 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 		pipe_config->pixel_multiplier = 1;
 	}
 	pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
-	if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
+	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
 		/*
 		 * DPLL_DVO_2X_MODE must be enabled for both DPLLs
 		 * on 830. Filter it out here so that we don't
 		 * report errors due to that.
 		 */
-		if (IS_I830(dev))
+		if (IS_I830(dev_priv))
 			pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
 
 		pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
@@ -8898,9 +8873,9 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 						     DPLL_PORTB_READY_MASK);
 	}
 
-	if (IS_CHERRYVIEW(dev))
+	if (IS_CHERRYVIEW(dev_priv))
 		chv_crtc_clock_get(crtc, pipe_config);
-	else if (IS_VALLEYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev_priv))
 		vlv_crtc_clock_get(crtc, pipe_config);
 	else
 		i9xx_crtc_clock_get(crtc, pipe_config);
@@ -8951,7 +8926,7 @@ static void ironlake_init_pch_refclk(struct drm_device *dev)
 		}
 	}
 
-	if (HAS_PCH_IBX(dev)) {
+	if (HAS_PCH_IBX(dev_priv)) {
 		has_ck505 = dev_priv->vbt.display_clock_mode;
 		can_ssc = has_ck505;
 	} else {
@@ -9199,7 +9174,8 @@ static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread,
 
 	if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
 		with_spread = true;
-	if (WARN(HAS_PCH_LPT_LP(dev) && with_fdi, "LP PCH doesn't have FDI\n"))
+	if (WARN(HAS_PCH_LPT_LP(dev_priv) &&
+	    with_fdi, "LP PCH doesn't have FDI\n"))
 		with_fdi = false;
 
 	mutex_lock(&dev_priv->sb_lock);
@@ -9222,7 +9198,7 @@ static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread,
 		}
 	}
 
-	reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0;
+	reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
 	tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
 	tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
 	intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
@@ -9238,7 +9214,7 @@ static void lpt_disable_clkout_dp(struct drm_device *dev)
 
 	mutex_lock(&dev_priv->sb_lock);
 
-	reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0;
+	reg = HAS_PCH_LPT_LP(dev_priv) ? SBI_GEN0 : SBI_DBUFF0;
 	tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
 	tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
 	intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
@@ -9346,9 +9322,11 @@ static void lpt_init_pch_refclk(struct drm_device *dev)
  */
 void intel_init_pch_refclk(struct drm_device *dev)
 {
-	if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
+	struct drm_i915_private *dev_priv = to_i915(dev);
+
+	if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
 		ironlake_init_pch_refclk(dev);
-	else if (HAS_PCH_LPT(dev))
+	else if (HAS_PCH_LPT(dev_priv))
 		lpt_init_pch_refclk(dev);
 }
 
@@ -9477,7 +9455,7 @@ static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 		if ((intel_panel_use_ssc(dev_priv) &&
 		     dev_priv->vbt.lvds_ssc_freq == 100000) ||
-		    (HAS_PCH_IBX(dev) && intel_is_dual_link_lvds(dev)))
+		    (HAS_PCH_IBX(dev_priv) && intel_is_dual_link_lvds(dev)))
 			factor = 25;
 	} else if (crtc_state->sdvo_tv_clock)
 		factor = 20;
@@ -9651,11 +9629,10 @@ static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
 					 struct intel_link_m_n *m_n,
 					 struct intel_link_m_n *m2_n2)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 
-	if (INTEL_INFO(dev)->gen >= 5) {
+	if (INTEL_GEN(dev_priv) >= 5) {
 		m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
 		m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
 		m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
@@ -9667,7 +9644,7 @@ static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
 		 * gen < 8) and if DRRS is supported (to make sure the
 		 * registers are not unnecessarily read).
 		 */
-		if (m2_n2 && INTEL_INFO(dev)->gen < 8 &&
+		if (m2_n2 && INTEL_GEN(dev_priv) < 8 &&
 			crtc->config->has_drrs) {
 			m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
 			m2_n2->link_n =	I915_READ(PIPE_LINK_N2(transcoder));
@@ -9771,17 +9748,17 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc,
 	tiling = val & PLANE_CTL_TILED_MASK;
 	switch (tiling) {
 	case PLANE_CTL_TILED_LINEAR:
-		fb->modifier[0] = DRM_FORMAT_MOD_NONE;
+		fb->modifier = DRM_FORMAT_MOD_NONE;
 		break;
 	case PLANE_CTL_TILED_X:
 		plane_config->tiling = I915_TILING_X;
-		fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
+		fb->modifier = I915_FORMAT_MOD_X_TILED;
 		break;
 	case PLANE_CTL_TILED_Y:
-		fb->modifier[0] = I915_FORMAT_MOD_Y_TILED;
+		fb->modifier = I915_FORMAT_MOD_Y_TILED;
 		break;
 	case PLANE_CTL_TILED_YF:
-		fb->modifier[0] = I915_FORMAT_MOD_Yf_TILED;
+		fb->modifier = I915_FORMAT_MOD_Yf_TILED;
 		break;
 	default:
 		MISSING_CASE(tiling);
@@ -9798,13 +9775,13 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc,
 	fb->width = ((val >> 0) & 0x1fff) + 1;
 
 	val = I915_READ(PLANE_STRIDE(pipe, 0));
-	stride_mult = intel_fb_stride_alignment(dev_priv, fb->modifier[0],
+	stride_mult = intel_fb_stride_alignment(dev_priv, fb->modifier,
 						fb->pixel_format);
 	fb->pitches[0] = (val & 0x3ff) * stride_mult;
 
 	aligned_height = intel_fb_align_height(dev, fb->height,
 					       fb->pixel_format,
-					       fb->modifier[0]);
+					       fb->modifier);
 
 	plane_config->size = fb->pitches[0] * aligned_height;
 
@@ -9837,7 +9814,7 @@ static void ironlake_get_pfit_config(struct intel_crtc *crtc,
 		/* We currently do not free assignements of panel fitters on
 		 * ivb/hsw (since we don't use the higher upscaling modes which
 		 * differentiates them) so just WARN about this case for now. */
-		if (IS_GEN7(dev)) {
+		if (IS_GEN7(dev_priv)) {
 			WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
 				PF_PIPE_SEL_IVB(crtc->pipe));
 		}
@@ -9869,10 +9846,10 @@ ironlake_get_initial_plane_config(struct intel_crtc *crtc,
 
 	fb = &intel_fb->base;
 
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_GEN(dev_priv) >= 4) {
 		if (val & DISPPLANE_TILED) {
 			plane_config->tiling = I915_TILING_X;
-			fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
+			fb->modifier = I915_FORMAT_MOD_X_TILED;
 		}
 	}
 
@@ -9882,7 +9859,7 @@ ironlake_get_initial_plane_config(struct intel_crtc *crtc,
 	fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
 
 	base = I915_READ(DSPSURF(pipe)) & 0xfffff000;
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
 		offset = I915_READ(DSPOFFSET(pipe));
 	} else {
 		if (plane_config->tiling)
@@ -9901,7 +9878,7 @@ ironlake_get_initial_plane_config(struct intel_crtc *crtc,
 
 	aligned_height = intel_fb_align_height(dev, fb->height,
 					       fb->pixel_format,
-					       fb->modifier[0]);
+					       fb->modifier);
 
 	plane_config->size = fb->pitches[0] * aligned_height;
 
@@ -10026,7 +10003,7 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
 	I915_STATE_WARN(I915_READ(PP_STATUS(0)) & PP_ON, "Panel power on\n");
 	I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
 	     "CPU PWM1 enabled\n");
-	if (IS_HASWELL(dev))
+	if (IS_HASWELL(dev_priv))
 		I915_STATE_WARN(I915_READ(HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
 		     "CPU PWM2 enabled\n");
 	I915_STATE_WARN(I915_READ(BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
@@ -10046,9 +10023,7 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
 
 static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
 {
-	struct drm_device *dev = &dev_priv->drm;
-
-	if (IS_HASWELL(dev))
+	if (IS_HASWELL(dev_priv))
 		return I915_READ(D_COMP_HSW);
 	else
 		return I915_READ(D_COMP_BDW);
@@ -10056,9 +10031,7 @@ static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
 
 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val)
 {
-	struct drm_device *dev = &dev_priv->drm;
-
-	if (IS_HASWELL(dev)) {
+	if (IS_HASWELL(dev_priv)) {
 		mutex_lock(&dev_priv->rps.hw_lock);
 		if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP,
 					    val))
@@ -10173,7 +10146,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
 	}
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
-	intel_update_cdclk(&dev_priv->drm);
+	intel_update_cdclk(dev_priv);
 }
 
 /*
@@ -10206,7 +10179,7 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv)
 
 	DRM_DEBUG_KMS("Enabling package C8+\n");
 
-	if (HAS_PCH_LPT_LP(dev)) {
+	if (HAS_PCH_LPT_LP(dev_priv)) {
 		val = I915_READ(SOUTH_DSPCLK_GATE_D);
 		val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
 		I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
@@ -10226,7 +10199,7 @@ void hsw_disable_pc8(struct drm_i915_private *dev_priv)
 	hsw_restore_lcpll(dev_priv);
 	lpt_init_pch_refclk(dev);
 
-	if (HAS_PCH_LPT_LP(dev)) {
+	if (HAS_PCH_LPT_LP(dev_priv)) {
 		val = I915_READ(SOUTH_DSPCLK_GATE_D);
 		val |= PCH_LP_PARTITION_LEVEL_DISABLE;
 		I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
@@ -10376,7 +10349,7 @@ static void broadwell_set_cdclk(struct drm_device *dev, int cdclk)
 
 	I915_WRITE(CDCLK_FREQ, DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 
-	intel_update_cdclk(dev);
+	intel_update_cdclk(dev_priv);
 
 	WARN(cdclk != dev_priv->cdclk_freq,
 	     "cdclk requested %d kHz but got %d kHz\n",
@@ -10663,8 +10636,7 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
 				       struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct intel_shared_dpll *pll;
 	enum port port;
 	uint32_t tmp;
@@ -10673,9 +10645,9 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
 
 	port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
 
-	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
 		skylake_get_ddi_pll(dev_priv, port, pipe_config);
-	else if (IS_BROXTON(dev))
+	else if (IS_BROXTON(dev_priv))
 		bxt_get_ddi_pll(dev_priv, port, pipe_config);
 	else
 		haswell_get_ddi_pll(dev_priv, port, pipe_config);
@@ -10691,7 +10663,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
 	 * DDI E. So just check whether this pipe is wired to DDI E and whether
 	 * the PCH transcoder is on.
 	 */
-	if (INTEL_INFO(dev)->gen < 9 &&
+	if (INTEL_GEN(dev_priv) < 9 &&
 	    (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
 		pipe_config->has_pch_encoder = true;
 
@@ -10706,8 +10678,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 				    struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum intel_display_power_domain power_domain;
 	unsigned long power_domain_mask;
 	bool active;
@@ -10740,11 +10711,9 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 	pipe_config->gamma_mode =
 		I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK;
 
-	if (INTEL_INFO(dev)->gen >= 9) {
-		skl_init_scalers(dev, crtc, pipe_config);
-	}
+	if (INTEL_GEN(dev_priv) >= 9) {
+		skl_init_scalers(dev_priv, crtc, pipe_config);
 
-	if (INTEL_INFO(dev)->gen >= 9) {
 		pipe_config->scaler_state.scaler_id = -1;
 		pipe_config->scaler_state.scaler_users &= ~(1 << SKL_CRTC_INDEX);
 	}
@@ -10752,13 +10721,13 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 	power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
 	if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
 		power_domain_mask |= BIT(power_domain);
-		if (INTEL_INFO(dev)->gen >= 9)
+		if (INTEL_GEN(dev_priv) >= 9)
 			skylake_get_pfit_config(crtc, pipe_config);
 		else
 			ironlake_get_pfit_config(crtc, pipe_config);
 	}
 
-	if (IS_HASWELL(dev))
+	if (IS_HASWELL(dev_priv))
 		pipe_config->ips_enabled = hsw_crtc_supports_ips(crtc) &&
 			(I915_READ(IPS_CTL) & IPS_ENABLE);
 
@@ -10846,13 +10815,9 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base,
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	const struct skl_wm_values *wm = &dev_priv->wm.skl_results;
 	int pipe = intel_crtc->pipe;
 	uint32_t cntl = 0;
 
-	if (INTEL_GEN(dev_priv) >= 9 && wm->dirty_pipes & drm_crtc_mask(crtc))
-		skl_write_cursor_wm(intel_crtc, wm);
-
 	if (plane_state && plane_state->base.visible) {
 		cntl = MCURSOR_GAMMA_ENABLE;
 		switch (plane_state->base.crtc_w) {
@@ -10871,10 +10836,10 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base,
 		}
 		cntl |= pipe << 28; /* Connect to correct pipe */
 
-		if (HAS_DDI(dev))
+		if (HAS_DDI(dev_priv))
 			cntl |= CURSOR_PIPE_CSC_ENABLE;
 
-		if (plane_state->base.rotation == DRM_ROTATE_180)
+		if (plane_state->base.rotation & DRM_ROTATE_180)
 			cntl |= CURSOR_ROTATE_180;
 	}
 
@@ -10919,8 +10884,8 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
 		pos |= y << CURSOR_Y_SHIFT;
 
 		/* ILK+ do this automagically */
-		if (HAS_GMCH_DISPLAY(dev) &&
-		    plane_state->base.rotation == DRM_ROTATE_180) {
+		if (HAS_GMCH_DISPLAY(dev_priv) &&
+		    plane_state->base.rotation & DRM_ROTATE_180) {
 			base += (plane_state->base.crtc_h *
 				 plane_state->base.crtc_w - 1) * 4;
 		}
@@ -10928,13 +10893,13 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
 
 	I915_WRITE(CURPOS(pipe), pos);
 
-	if (IS_845G(dev) || IS_I865G(dev))
+	if (IS_845G(dev_priv) || IS_I865G(dev_priv))
 		i845_update_cursor(crtc, base, plane_state);
 	else
 		i9xx_update_cursor(crtc, base, plane_state);
 }
 
-static bool cursor_size_ok(struct drm_device *dev,
+static bool cursor_size_ok(struct drm_i915_private *dev_priv,
 			   uint32_t width, uint32_t height)
 {
 	if (width == 0 || height == 0)
@@ -10946,11 +10911,11 @@ static bool cursor_size_ok(struct drm_device *dev,
 	 * the precision of the register. Everything else requires
 	 * square cursors, limited to a few power-of-two sizes.
 	 */
-	if (IS_845G(dev) || IS_I865G(dev)) {
+	if (IS_845G(dev_priv) || IS_I865G(dev_priv)) {
 		if ((width & 63) != 0)
 			return false;
 
-		if (width > (IS_845G(dev) ? 64 : 512))
+		if (width > (IS_845G(dev_priv) ? 64 : 512))
 			return false;
 
 		if (height > 1023)
@@ -10959,7 +10924,7 @@ static bool cursor_size_ok(struct drm_device *dev,
 		switch (width | height) {
 		case 256:
 		case 128:
-			if (IS_GEN2(dev))
+			if (IS_GEN2(dev_priv))
 				return false;
 		case 64:
 			break;
@@ -11053,7 +11018,7 @@ intel_framebuffer_create_for_mode(struct drm_device *dev,
 
 	fb = intel_framebuffer_create(dev, &mode_cmd, obj);
 	if (IS_ERR(fb))
-		i915_gem_object_put_unlocked(obj);
+		i915_gem_object_put(obj);
 
 	return fb;
 }
@@ -11138,6 +11103,7 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector,
 	struct drm_encoder *encoder = &intel_encoder->base;
 	struct drm_crtc *crtc = NULL;
 	struct drm_device *dev = encoder->dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_framebuffer *fb;
 	struct drm_mode_config *config = &dev->mode_config;
 	struct drm_atomic_state *state = NULL, *restore_state = NULL;
@@ -11290,13 +11256,18 @@ found:
 	old->restore_state = restore_state;
 
 	/* let the connector get through one full cycle before testing */
-	intel_wait_for_vblank(dev, intel_crtc->pipe);
+	intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
 	return true;
 
 fail:
-	drm_atomic_state_free(state);
-	drm_atomic_state_free(restore_state);
-	restore_state = state = NULL;
+	if (state) {
+		drm_atomic_state_put(state);
+		state = NULL;
+	}
+	if (restore_state) {
+		drm_atomic_state_put(restore_state);
+		restore_state = NULL;
+	}
 
 	if (ret == -EDEADLK) {
 		drm_modeset_backoff(ctx);
@@ -11324,10 +11295,9 @@ void intel_release_load_detect_pipe(struct drm_connector *connector,
 		return;
 
 	ret = drm_atomic_commit(state);
-	if (ret) {
+	if (ret)
 		DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
-		drm_atomic_state_free(state);
-	}
+	drm_atomic_state_put(state);
 }
 
 static int i9xx_pll_refclk(struct drm_device *dev,
@@ -11338,9 +11308,9 @@ static int i9xx_pll_refclk(struct drm_device *dev,
 
 	if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
 		return dev_priv->vbt.lvds_ssc_freq;
-	else if (HAS_PCH_SPLIT(dev))
+	else if (HAS_PCH_SPLIT(dev_priv))
 		return 120000;
-	else if (!IS_GEN2(dev))
+	else if (!IS_GEN2(dev_priv))
 		return 96000;
 	else
 		return 48000;
@@ -11365,7 +11335,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 		fp = pipe_config->dpll_hw_state.fp1;
 
 	clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
-	if (IS_PINEVIEW(dev)) {
+	if (IS_PINEVIEW(dev_priv)) {
 		clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
 		clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
 	} else {
@@ -11373,8 +11343,8 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 		clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
 	}
 
-	if (!IS_GEN2(dev)) {
-		if (IS_PINEVIEW(dev))
+	if (!IS_GEN2(dev_priv)) {
+		if (IS_PINEVIEW(dev_priv))
 			clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
 				DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
 		else
@@ -11396,12 +11366,12 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 			return;
 		}
 
-		if (IS_PINEVIEW(dev))
+		if (IS_PINEVIEW(dev_priv))
 			port_clock = pnv_calc_dpll_params(refclk, &clock);
 		else
 			port_clock = i9xx_calc_dpll_params(refclk, &clock);
 	} else {
-		u32 lvds = IS_I830(dev) ? 0 : I915_READ(LVDS);
+		u32 lvds = IS_I830(dev_priv) ? 0 : I915_READ(LVDS);
 		bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
 
 		if (is_lvds) {
@@ -11602,7 +11572,7 @@ static bool __pageflip_finished_cs(struct intel_crtc *crtc,
 	 * really needed there. But since ctg has the registers,
 	 * include it in the check anyway.
 	 */
-	if (INTEL_INFO(dev)->gen < 5 && !IS_G4X(dev))
+	if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
 		return true;
 
 	/*
@@ -11665,8 +11635,7 @@ static bool pageflip_finished(struct intel_crtc *crtc,
 void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe)
 {
 	struct drm_device *dev = &dev_priv->drm;
-	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 	struct intel_flip_work *work;
 	unsigned long flags;
 
@@ -11679,12 +11648,12 @@ void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe)
 	 * lost pageflips) so needs the full irqsave spinlocks.
 	 */
 	spin_lock_irqsave(&dev->event_lock, flags);
-	work = intel_crtc->flip_work;
+	work = crtc->flip_work;
 
 	if (work != NULL &&
 	    !is_mmio_work(work) &&
-	    pageflip_finished(intel_crtc, work))
-		page_flip_completed(intel_crtc);
+	    pageflip_finished(crtc, work))
+		page_flip_completed(crtc);
 
 	spin_unlock_irqrestore(&dev->event_lock, flags);
 }
@@ -11692,8 +11661,7 @@ void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe)
 void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe)
 {
 	struct drm_device *dev = &dev_priv->drm;
-	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 	struct intel_flip_work *work;
 	unsigned long flags;
 
@@ -11706,12 +11674,12 @@ void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe)
 	 * lost pageflips) so needs the full irqsave spinlocks.
 	 */
 	spin_lock_irqsave(&dev->event_lock, flags);
-	work = intel_crtc->flip_work;
+	work = crtc->flip_work;
 
 	if (work != NULL &&
 	    is_mmio_work(work) &&
-	    pageflip_finished(intel_crtc, work))
-		page_flip_completed(intel_crtc);
+	    pageflip_finished(crtc, work))
+		page_flip_completed(crtc);
 
 	spin_unlock_irqrestore(&dev->event_lock, flags);
 }
@@ -11816,7 +11784,7 @@ static int intel_gen4_queue_flip(struct drm_device *dev,
 			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
 	intel_ring_emit(ring, fb->pitches[0]);
 	intel_ring_emit(ring, intel_crtc->flip_work->gtt_offset |
-			intel_fb_modifier_to_tiling(fb->modifier[0]));
+			intel_fb_modifier_to_tiling(fb->modifier));
 
 	/* XXX Enabling the panel-fitter across page-flip is so far
 	 * untested on non-native modes, so ignore it for now.
@@ -11849,7 +11817,7 @@ static int intel_gen6_queue_flip(struct drm_device *dev,
 	intel_ring_emit(ring, MI_DISPLAY_FLIP |
 			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
 	intel_ring_emit(ring, fb->pitches[0] |
-			intel_fb_modifier_to_tiling(fb->modifier[0]));
+			intel_fb_modifier_to_tiling(fb->modifier));
 	intel_ring_emit(ring, intel_crtc->flip_work->gtt_offset);
 
 	/* Contrary to the suggestions in the documentation,
@@ -11872,6 +11840,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 				 struct drm_i915_gem_request *req,
 				 uint32_t flags)
 {
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_ring *ring = req->ring;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	uint32_t plane_bit = 0;
@@ -11900,7 +11869,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 		 * 48bits addresses, and we need a NOOP for the batch size to
 		 * stay even.
 		 */
-		if (IS_GEN8(dev))
+		if (IS_GEN8(dev_priv))
 			len += 2;
 	}
 
@@ -11937,7 +11906,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 		intel_ring_emit(ring, ~(DERRMR_PIPEA_PRI_FLIP_DONE |
 					  DERRMR_PIPEB_PRI_FLIP_DONE |
 					  DERRMR_PIPEC_PRI_FLIP_DONE));
-		if (IS_GEN8(dev))
+		if (IS_GEN8(dev_priv))
 			intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8 |
 					      MI_SRM_LRM_GLOBAL_GTT);
 		else
@@ -11946,7 +11915,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 		intel_ring_emit_reg(ring, DERRMR);
 		intel_ring_emit(ring,
 				i915_ggtt_offset(req->engine->scratch) + 256);
-		if (IS_GEN8(dev)) {
+		if (IS_GEN8(dev_priv)) {
 			intel_ring_emit(ring, 0);
 			intel_ring_emit(ring, MI_NOOP);
 		}
@@ -11954,7 +11923,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 
 	intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | plane_bit);
 	intel_ring_emit(ring, fb->pitches[0] |
-			intel_fb_modifier_to_tiling(fb->modifier[0]));
+			intel_fb_modifier_to_tiling(fb->modifier));
 	intel_ring_emit(ring, intel_crtc->flip_work->gtt_offset);
 	intel_ring_emit(ring, (MI_NOOP));
 
@@ -11964,8 +11933,6 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 static bool use_mmio_flip(struct intel_engine_cs *engine,
 			  struct drm_i915_gem_object *obj)
 {
-	struct reservation_object *resv;
-
 	/*
 	 * This is not being used for older platforms, because
 	 * non-availability of flip done interrupt forces us to use
@@ -11987,12 +11954,7 @@ static bool use_mmio_flip(struct intel_engine_cs *engine,
 	else if (i915.enable_execlists)
 		return true;
 
-	resv = i915_gem_object_get_dmabuf_resv(obj);
-	if (resv && !reservation_object_test_signaled_rcu(resv, false))
-		return true;
-
-	return engine != i915_gem_active_get_engine(&obj->last_write,
-						    &obj->base.dev->struct_mutex);
+	return engine != i915_gem_object_last_write_engine(obj);
 }
 
 static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
@@ -12007,7 +11969,7 @@ static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
 
 	ctl = I915_READ(PLANE_CTL(pipe, 0));
 	ctl &= ~PLANE_CTL_TILED_MASK;
-	switch (fb->modifier[0]) {
+	switch (fb->modifier) {
 	case DRM_FORMAT_MOD_NONE:
 		break;
 	case I915_FORMAT_MOD_X_TILED:
@@ -12020,7 +11982,7 @@ static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
 		ctl |= PLANE_CTL_TILED_YF;
 		break;
 	default:
-		MISSING_CASE(fb->modifier[0]);
+		MISSING_CASE(fb->modifier);
 	}
 
 	/*
@@ -12045,7 +12007,7 @@ static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc,
 
 	dspcntr = I915_READ(reg);
 
-	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
+	if (fb->modifier == I915_FORMAT_MOD_X_TILED)
 		dspcntr |= DISPPLANE_TILED;
 	else
 		dspcntr &= ~DISPPLANE_TILED;
@@ -12065,17 +12027,8 @@ static void intel_mmio_flip_work_func(struct work_struct *w)
 	struct intel_framebuffer *intel_fb =
 		to_intel_framebuffer(crtc->base.primary->fb);
 	struct drm_i915_gem_object *obj = intel_fb->obj;
-	struct reservation_object *resv;
 
-	if (work->flip_queued_req)
-		WARN_ON(i915_wait_request(work->flip_queued_req,
-					  0, NULL, NO_WAITBOOST));
-
-	/* For framebuffer backed by dmabuf, wait for fence */
-	resv = i915_gem_object_get_dmabuf_resv(obj);
-	if (resv)
-		WARN_ON(reservation_object_wait_timeout_rcu(resv, false, false,
-							    MAX_SCHEDULE_TIMEOUT) < 0);
+	WARN_ON(i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT, NULL) < 0);
 
 	intel_pipe_update_start(crtc);
 
@@ -12138,8 +12091,7 @@ static bool __pageflip_stall_check_cs(struct drm_i915_private *dev_priv,
 void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe)
 {
 	struct drm_device *dev = &dev_priv->drm;
-	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 	struct intel_flip_work *work;
 
 	WARN_ON(!in_interrupt());
@@ -12148,19 +12100,19 @@ void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe)
 		return;
 
 	spin_lock(&dev->event_lock);
-	work = intel_crtc->flip_work;
+	work = crtc->flip_work;
 
 	if (work != NULL && !is_mmio_work(work) &&
-	    __pageflip_stall_check_cs(dev_priv, intel_crtc, work)) {
+	    __pageflip_stall_check_cs(dev_priv, crtc, work)) {
 		WARN_ONCE(1,
 			  "Kicking stuck page flip: queued at %d, now %d\n",
-			work->flip_queued_vblank, intel_crtc_get_vblank_counter(intel_crtc));
-		page_flip_completed(intel_crtc);
+			work->flip_queued_vblank, intel_crtc_get_vblank_counter(crtc));
+		page_flip_completed(crtc);
 		work = NULL;
 	}
 
 	if (work != NULL && !is_mmio_work(work) &&
-	    intel_crtc_get_vblank_counter(intel_crtc) - work->flip_queued_vblank > 1)
+	    intel_crtc_get_vblank_counter(crtc) - work->flip_queued_vblank > 1)
 		intel_queue_rps_boost_for_request(work->flip_queued_req);
 	spin_unlock(&dev->event_lock);
 }
@@ -12200,7 +12152,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 	 * TILEOFF/LINOFF registers can't be changed via MI display flips.
 	 * Note that pitch changes could also affect these register.
 	 */
-	if (INTEL_INFO(dev)->gen > 3 &&
+	if (INTEL_GEN(dev_priv) > 3 &&
 	    (fb->offsets[0] != crtc->primary->fb->offsets[0] ||
 	     fb->pitches[0] != crtc->primary->fb->pitches[0]))
 		return -EINVAL;
@@ -12265,23 +12217,22 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 
 	atomic_inc(&intel_crtc->unpin_work_count);
 
-	if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
+	if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
 		work->flip_count = I915_READ(PIPE_FLIPCOUNT_G4X(pipe)) + 1;
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
-		engine = &dev_priv->engine[BCS];
-		if (fb->modifier[0] != old_fb->modifier[0])
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
+		engine = dev_priv->engine[BCS];
+		if (fb->modifier != old_fb->modifier)
 			/* vlv: DISPLAY_FLIP fails to change tiling */
 			engine = NULL;
-	} else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
-		engine = &dev_priv->engine[BCS];
-	} else if (INTEL_INFO(dev)->gen >= 7) {
-		engine = i915_gem_active_get_engine(&obj->last_write,
-						    &obj->base.dev->struct_mutex);
+	} else if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
+		engine = dev_priv->engine[BCS];
+	} else if (INTEL_GEN(dev_priv) >= 7) {
+		engine = i915_gem_object_last_write_engine(obj);
 		if (engine == NULL || engine->id != RCS)
-			engine = &dev_priv->engine[BCS];
+			engine = dev_priv->engine[BCS];
 	} else {
-		engine = &dev_priv->engine[RCS];
+		engine = dev_priv->engine[RCS];
 	}
 
 	mmio_flip = use_mmio_flip(engine, obj);
@@ -12309,10 +12260,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 
 	if (mmio_flip) {
 		INIT_WORK(&work->mmio_work, intel_mmio_flip_work_func);
-
-		work->flip_queued_req = i915_gem_active_get(&obj->last_write,
-							    &obj->base.dev->struct_mutex);
-		schedule_work(&work->mmio_work);
+		queue_work(system_unbound_wq, &work->mmio_work);
 	} else {
 		request = i915_gem_request_alloc(engine, engine->last_context);
 		if (IS_ERR(request)) {
@@ -12335,6 +12283,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 		i915_add_request_no_flush(request);
 	}
 
+	i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY);
 	i915_gem_track_fb(intel_fb_obj(old_fb), obj,
 			  to_intel_plane(primary)->frontbuffer_bit);
 	mutex_unlock(&dev->struct_mutex);
@@ -12358,7 +12307,7 @@ cleanup:
 	crtc->primary->fb = old_fb;
 	update_state_fb(crtc->primary);
 
-	i915_gem_object_put_unlocked(obj);
+	i915_gem_object_put(obj);
 	drm_framebuffer_unreference(work->old_fb);
 
 	spin_lock_irq(&dev->event_lock);
@@ -12396,8 +12345,7 @@ retry:
 			goto retry;
 		}
 
-		if (ret)
-			drm_atomic_state_free(state);
+		drm_atomic_state_put(state);
 
 		if (ret == 0 && event) {
 			spin_lock_irq(&dev->event_lock);
@@ -12432,7 +12380,7 @@ static bool intel_wm_need_update(struct drm_plane *plane,
 	if (!cur->base.fb || !new->base.fb)
 		return false;
 
-	if (cur->base.fb->modifier[0] != new->base.fb->modifier[0] ||
+	if (cur->base.fb->modifier != new->base.fb->modifier ||
 	    cur->base.rotation != new->base.rotation ||
 	    drm_rect_width(&new->base.src) != drm_rect_width(&cur->base.src) ||
 	    drm_rect_height(&new->base.src) != drm_rect_height(&cur->base.src) ||
@@ -12471,7 +12419,7 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 	struct drm_framebuffer *fb = plane_state->fb;
 	int ret;
 
-	if (INTEL_GEN(dev) >= 9 && plane->type != DRM_PLANE_TYPE_CURSOR) {
+	if (INTEL_GEN(dev_priv) >= 9 && plane->type != DRM_PLANE_TYPE_CURSOR) {
 		ret = skl_update_scaler_plane(
 			to_intel_crtc_state(crtc_state),
 			to_intel_plane_state(plane_state));
@@ -12538,7 +12486,7 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 
 	/* Pre-gen9 platforms need two-step watermark updates */
 	if ((pipe_config->update_wm_pre || pipe_config->update_wm_post) &&
-	    INTEL_INFO(dev)->gen < 9 && dev_priv->display.optimize_watermarks)
+	    INTEL_GEN(dev_priv) < 9 && dev_priv->display.optimize_watermarks)
 		to_intel_crtc_state(crtc_state)->wm.need_postvbl_update = true;
 
 	if (visible || was_visible)
@@ -12550,7 +12498,7 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 	 * cstate->update_wm was already set above, so this flag will
 	 * take effect when we commit and program watermarks.
 	 */
-	if (plane->type == DRM_PLANE_TYPE_OVERLAY && IS_IVYBRIDGE(dev) &&
+	if (plane->type == DRM_PLANE_TYPE_OVERLAY && IS_IVYBRIDGE(dev_priv) &&
 	    needs_scaling(to_intel_plane_state(plane_state)) &&
 	    !needs_scaling(old_plane_state))
 		pipe_config->disable_lp_wm = true;
@@ -12643,7 +12591,7 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
 		 * old state and the new state.  We can program these
 		 * immediately.
 		 */
-		ret = dev_priv->display.compute_intermediate_wm(crtc->dev,
+		ret = dev_priv->display.compute_intermediate_wm(dev,
 								intel_crtc,
 								pipe_config);
 		if (ret) {
@@ -12655,7 +12603,7 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
 			pipe_config->wm.ilk.intermediate = pipe_config->wm.ilk.optimal;
 	}
 
-	if (INTEL_INFO(dev)->gen >= 9) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		if (mode_changed)
 			ret = skl_update_scaler_crtc(pipe_config);
 
@@ -12726,15 +12674,16 @@ static int
 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
 			  struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = crtc->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct drm_atomic_state *state;
 	struct drm_connector *connector;
 	struct drm_connector_state *connector_state;
 	int bpp, i;
 
-	if ((IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)))
+	if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
+	    IS_CHERRYVIEW(dev_priv)))
 		bpp = 10*3;
-	else if (INTEL_INFO(dev)->gen >= 5)
+	else if (INTEL_GEN(dev_priv) >= 5)
 		bpp = 12*3;
 	else
 		bpp = 8*3;
@@ -12767,73 +12716,81 @@ static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
 		mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags);
 }
 
+static inline void
+intel_dump_m_n_config(struct intel_crtc_state *pipe_config, char *id,
+		      unsigned int lane_count, struct intel_link_m_n *m_n)
+{
+	DRM_DEBUG_KMS("%s: lanes: %i; gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
+		      id, lane_count,
+		      m_n->gmch_m, m_n->gmch_n,
+		      m_n->link_m, m_n->link_n, m_n->tu);
+}
+
 static void intel_dump_pipe_config(struct intel_crtc *crtc,
 				   struct intel_crtc_state *pipe_config,
 				   const char *context)
 {
 	struct drm_device *dev = crtc->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_plane *plane;
 	struct intel_plane *intel_plane;
 	struct intel_plane_state *state;
 	struct drm_framebuffer *fb;
 
-	DRM_DEBUG_KMS("[CRTC:%d:%s]%s config %p for pipe %c\n",
-		      crtc->base.base.id, crtc->base.name,
-		      context, pipe_config, pipe_name(crtc->pipe));
+	DRM_DEBUG_KMS("[CRTC:%d:%s]%s\n",
+		      crtc->base.base.id, crtc->base.name, context);
 
-	DRM_DEBUG_KMS("cpu_transcoder: %s\n", transcoder_name(pipe_config->cpu_transcoder));
-	DRM_DEBUG_KMS("pipe bpp: %i, dithering: %i\n",
+	DRM_DEBUG_KMS("cpu_transcoder: %s, pipe bpp: %i, dithering: %i\n",
+		      transcoder_name(pipe_config->cpu_transcoder),
 		      pipe_config->pipe_bpp, pipe_config->dither);
-	DRM_DEBUG_KMS("fdi/pch: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
-		      pipe_config->has_pch_encoder,
-		      pipe_config->fdi_lanes,
-		      pipe_config->fdi_m_n.gmch_m, pipe_config->fdi_m_n.gmch_n,
-		      pipe_config->fdi_m_n.link_m, pipe_config->fdi_m_n.link_n,
-		      pipe_config->fdi_m_n.tu);
-	DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
-		      intel_crtc_has_dp_encoder(pipe_config),
-		      pipe_config->lane_count,
-		      pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n,
-		      pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n,
-		      pipe_config->dp_m_n.tu);
-
-	DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n",
-		      intel_crtc_has_dp_encoder(pipe_config),
-		      pipe_config->lane_count,
-		      pipe_config->dp_m2_n2.gmch_m,
-		      pipe_config->dp_m2_n2.gmch_n,
-		      pipe_config->dp_m2_n2.link_m,
-		      pipe_config->dp_m2_n2.link_n,
-		      pipe_config->dp_m2_n2.tu);
+
+	if (pipe_config->has_pch_encoder)
+		intel_dump_m_n_config(pipe_config, "fdi",
+				      pipe_config->fdi_lanes,
+				      &pipe_config->fdi_m_n);
+
+	if (intel_crtc_has_dp_encoder(pipe_config)) {
+		intel_dump_m_n_config(pipe_config, "dp m_n",
+				pipe_config->lane_count, &pipe_config->dp_m_n);
+		if (pipe_config->has_drrs)
+			intel_dump_m_n_config(pipe_config, "dp m2_n2",
+					      pipe_config->lane_count,
+					      &pipe_config->dp_m2_n2);
+	}
 
 	DRM_DEBUG_KMS("audio: %i, infoframes: %i\n",
-		      pipe_config->has_audio,
-		      pipe_config->has_infoframe);
+		      pipe_config->has_audio, pipe_config->has_infoframe);
 
 	DRM_DEBUG_KMS("requested mode:\n");
 	drm_mode_debug_printmodeline(&pipe_config->base.mode);
 	DRM_DEBUG_KMS("adjusted mode:\n");
 	drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
 	intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
-	DRM_DEBUG_KMS("port clock: %d\n", pipe_config->port_clock);
-	DRM_DEBUG_KMS("pipe src size: %dx%d\n",
+	DRM_DEBUG_KMS("port clock: %d, pipe src size: %dx%d\n",
+		      pipe_config->port_clock,
 		      pipe_config->pipe_src_w, pipe_config->pipe_src_h);
-	DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
-		      crtc->num_scalers,
-		      pipe_config->scaler_state.scaler_users,
-		      pipe_config->scaler_state.scaler_id);
-	DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
-		      pipe_config->gmch_pfit.control,
-		      pipe_config->gmch_pfit.pgm_ratios,
-		      pipe_config->gmch_pfit.lvds_border_bits);
-	DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n",
-		      pipe_config->pch_pfit.pos,
-		      pipe_config->pch_pfit.size,
-		      pipe_config->pch_pfit.enabled ? "enabled" : "disabled");
-	DRM_DEBUG_KMS("ips: %i\n", pipe_config->ips_enabled);
-	DRM_DEBUG_KMS("double wide: %i\n", pipe_config->double_wide);
-
-	if (IS_BROXTON(dev)) {
+
+	if (INTEL_GEN(dev_priv) >= 9)
+		DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
+			      crtc->num_scalers,
+			      pipe_config->scaler_state.scaler_users,
+		              pipe_config->scaler_state.scaler_id);
+
+	if (HAS_GMCH_DISPLAY(dev_priv))
+		DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
+			      pipe_config->gmch_pfit.control,
+			      pipe_config->gmch_pfit.pgm_ratios,
+			      pipe_config->gmch_pfit.lvds_border_bits);
+	else
+		DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n",
+			      pipe_config->pch_pfit.pos,
+			      pipe_config->pch_pfit.size,
+		              enableddisabled(pipe_config->pch_pfit.enabled));
+
+	DRM_DEBUG_KMS("ips: %i, double wide: %i\n",
+		      pipe_config->ips_enabled, pipe_config->double_wide);
+
+	if (IS_BROXTON(dev_priv)) {
 		DRM_DEBUG_KMS("dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x,"
 			      "pll0: 0x%x, pll1: 0x%x, pll2: 0x%x, pll3: 0x%x, "
 			      "pll6: 0x%x, pll8: 0x%x, pll9: 0x%x, pll10: 0x%x, pcsdw12: 0x%x\n",
@@ -12848,13 +12805,13 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
 			      pipe_config->dpll_hw_state.pll9,
 			      pipe_config->dpll_hw_state.pll10,
 			      pipe_config->dpll_hw_state.pcsdw12);
-	} else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+	} else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
 		DRM_DEBUG_KMS("dpll_hw_state: "
 			      "ctrl1: 0x%x, cfgcr1: 0x%x, cfgcr2: 0x%x\n",
 			      pipe_config->dpll_hw_state.ctrl1,
 			      pipe_config->dpll_hw_state.cfgcr1,
 			      pipe_config->dpll_hw_state.cfgcr2);
-	} else if (HAS_DDI(dev)) {
+	} else if (HAS_DDI(dev_priv)) {
 		DRM_DEBUG_KMS("dpll_hw_state: wrpll: 0x%x spll: 0x%x\n",
 			      pipe_config->dpll_hw_state.wrpll,
 			      pipe_config->dpll_hw_state.spll);
@@ -12869,7 +12826,7 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
 
 	DRM_DEBUG_KMS("planes on this crtc\n");
 	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
-		char *format_name;
+		struct drm_format_name_buf format_name;
 		intel_plane = to_intel_plane(plane);
 		if (intel_plane->pipe != crtc->pipe)
 			continue;
@@ -12882,23 +12839,20 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
 			continue;
 		}
 
-		format_name = drm_get_format_name(fb->pixel_format);
-
-		DRM_DEBUG_KMS("[PLANE:%d:%s] enabled",
-			      plane->base.id, plane->name);
-		DRM_DEBUG_KMS("\tFB:%d, fb = %ux%u format = %s",
-			      fb->base.id, fb->width, fb->height, format_name);
-		DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n",
-			      state->scaler_id,
-			      state->base.src.x1 >> 16,
-			      state->base.src.y1 >> 16,
-			      drm_rect_width(&state->base.src) >> 16,
-			      drm_rect_height(&state->base.src) >> 16,
-			      state->base.dst.x1, state->base.dst.y1,
-			      drm_rect_width(&state->base.dst),
-			      drm_rect_height(&state->base.dst));
-
-		kfree(format_name);
+		DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d, fb = %ux%u format = %s\n",
+			      plane->base.id, plane->name,
+			      fb->base.id, fb->width, fb->height,
+			      drm_get_format_name(fb->pixel_format, &format_name));
+		if (INTEL_GEN(dev_priv) >= 9)
+			DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n",
+				      state->scaler_id,
+				      state->base.src.x1 >> 16,
+				      state->base.src.y1 >> 16,
+				      drm_rect_width(&state->base.src) >> 16,
+				      drm_rect_height(&state->base.src) >> 16,
+				      state->base.dst.x1, state->base.dst.y1,
+				      drm_rect_width(&state->base.dst),
+				      drm_rect_height(&state->base.dst));
 	}
 }
 
@@ -12932,7 +12886,7 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
 		switch (encoder->type) {
 			unsigned int port_mask;
 		case INTEL_OUTPUT_UNKNOWN:
-			if (WARN_ON(!HAS_DDI(dev)))
+			if (WARN_ON(!HAS_DDI(to_i915(dev))))
 				break;
 		case INTEL_OUTPUT_DP:
 		case INTEL_OUTPUT_HDMI:
@@ -13213,7 +13167,7 @@ intel_compare_link_m_n(const struct intel_link_m_n *m_n,
 }
 
 static bool
-intel_pipe_config_compare(struct drm_device *dev,
+intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 			  struct intel_crtc_state *current_config,
 			  struct intel_crtc_state *pipe_config,
 			  bool adjust)
@@ -13337,7 +13291,7 @@ intel_pipe_config_compare(struct drm_device *dev,
 	PIPE_CONF_CHECK_I(lane_count);
 	PIPE_CONF_CHECK_X(lane_lat_optim_mask);
 
-	if (INTEL_INFO(dev)->gen < 8) {
+	if (INTEL_GEN(dev_priv) < 8) {
 		PIPE_CONF_CHECK_M_N(dp_m_n);
 
 		if (current_config->has_drrs)
@@ -13363,8 +13317,8 @@ intel_pipe_config_compare(struct drm_device *dev,
 
 	PIPE_CONF_CHECK_I(pixel_multiplier);
 	PIPE_CONF_CHECK_I(has_hdmi_sink);
-	if ((INTEL_INFO(dev)->gen < 8 && !IS_HASWELL(dev)) ||
-	    IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
+	    IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		PIPE_CONF_CHECK_I(limited_color_range);
 	PIPE_CONF_CHECK_I(has_infoframe);
 
@@ -13386,7 +13340,7 @@ intel_pipe_config_compare(struct drm_device *dev,
 
 	PIPE_CONF_CHECK_X(gmch_pfit.control);
 	/* pfit ratios are autocomputed by the hw on gen4+ */
-	if (INTEL_INFO(dev)->gen < 4)
+	if (INTEL_GEN(dev_priv) < 4)
 		PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
 	PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
 
@@ -13404,7 +13358,7 @@ intel_pipe_config_compare(struct drm_device *dev,
 	}
 
 	/* BDW+ don't expose a synchronous way to read the state */
-	if (IS_HASWELL(dev))
+	if (IS_HASWELL(dev_priv))
 		PIPE_CONF_CHECK_I(ips_enabled);
 
 	PIPE_CONF_CHECK_I(double_wide);
@@ -13423,7 +13377,7 @@ intel_pipe_config_compare(struct drm_device *dev,
 	PIPE_CONF_CHECK_X(dsi_pll.ctrl);
 	PIPE_CONF_CHECK_X(dsi_pll.div);
 
-	if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5)
+	if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5)
 		PIPE_CONF_CHECK_I(pipe_bpp);
 
 	PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
@@ -13461,33 +13415,67 @@ static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
 static void verify_wm_state(struct drm_crtc *crtc,
 			    struct drm_crtc_state *new_state)
 {
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	struct skl_ddb_allocation hw_ddb, *sw_ddb;
-	struct skl_ddb_entry *hw_entry, *sw_entry;
+	struct skl_pipe_wm hw_wm, *sw_wm;
+	struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
+	struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	const enum pipe pipe = intel_crtc->pipe;
-	int plane;
+	int plane, level, max_level = ilk_wm_max_level(dev_priv);
 
-	if (INTEL_INFO(dev)->gen < 9 || !new_state->active)
+	if (INTEL_GEN(dev_priv) < 9 || !new_state->active)
 		return;
 
+	skl_pipe_wm_get_hw_state(crtc, &hw_wm);
+	sw_wm = &to_intel_crtc_state(new_state)->wm.skl.optimal;
+
 	skl_ddb_get_hw_state(dev_priv, &hw_ddb);
 	sw_ddb = &dev_priv->wm.skl_hw.ddb;
 
 	/* planes */
-	for_each_plane(dev_priv, pipe, plane) {
-		hw_entry = &hw_ddb.plane[pipe][plane];
-		sw_entry = &sw_ddb->plane[pipe][plane];
+	for_each_universal_plane(dev_priv, pipe, plane) {
+		hw_plane_wm = &hw_wm.planes[plane];
+		sw_plane_wm = &sw_wm->planes[plane];
+
+		/* Watermarks */
+		for (level = 0; level <= max_level; level++) {
+			if (skl_wm_level_equals(&hw_plane_wm->wm[level],
+						&sw_plane_wm->wm[level]))
+				continue;
 
-		if (skl_ddb_entry_equal(hw_entry, sw_entry))
-			continue;
+			DRM_ERROR("mismatch in WM pipe %c plane %d level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+				  pipe_name(pipe), plane + 1, level,
+				  sw_plane_wm->wm[level].plane_en,
+				  sw_plane_wm->wm[level].plane_res_b,
+				  sw_plane_wm->wm[level].plane_res_l,
+				  hw_plane_wm->wm[level].plane_en,
+				  hw_plane_wm->wm[level].plane_res_b,
+				  hw_plane_wm->wm[level].plane_res_l);
+		}
 
-		DRM_ERROR("mismatch in DDB state pipe %c plane %d "
-			  "(expected (%u,%u), found (%u,%u))\n",
-			  pipe_name(pipe), plane + 1,
-			  sw_entry->start, sw_entry->end,
-			  hw_entry->start, hw_entry->end);
+		if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
+					 &sw_plane_wm->trans_wm)) {
+			DRM_ERROR("mismatch in trans WM pipe %c plane %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+				  pipe_name(pipe), plane + 1,
+				  sw_plane_wm->trans_wm.plane_en,
+				  sw_plane_wm->trans_wm.plane_res_b,
+				  sw_plane_wm->trans_wm.plane_res_l,
+				  hw_plane_wm->trans_wm.plane_en,
+				  hw_plane_wm->trans_wm.plane_res_b,
+				  hw_plane_wm->trans_wm.plane_res_l);
+		}
+
+		/* DDB */
+		hw_ddb_entry = &hw_ddb.plane[pipe][plane];
+		sw_ddb_entry = &sw_ddb->plane[pipe][plane];
+
+		if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
+			DRM_ERROR("mismatch in DDB state pipe %c plane %d (expected (%u,%u), found (%u,%u))\n",
+				  pipe_name(pipe), plane + 1,
+				  sw_ddb_entry->start, sw_ddb_entry->end,
+				  hw_ddb_entry->start, hw_ddb_entry->end);
+		}
 	}
 
 	/*
@@ -13497,25 +13485,60 @@ static void verify_wm_state(struct drm_crtc *crtc,
 	 * once the plane becomes visible, we can skip this check
 	 */
 	if (intel_crtc->cursor_addr) {
-		hw_entry = &hw_ddb.plane[pipe][PLANE_CURSOR];
-		sw_entry = &sw_ddb->plane[pipe][PLANE_CURSOR];
+		hw_plane_wm = &hw_wm.planes[PLANE_CURSOR];
+		sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
 
-		if (!skl_ddb_entry_equal(hw_entry, sw_entry)) {
-			DRM_ERROR("mismatch in DDB state pipe %c cursor "
-				  "(expected (%u,%u), found (%u,%u))\n",
+		/* Watermarks */
+		for (level = 0; level <= max_level; level++) {
+			if (skl_wm_level_equals(&hw_plane_wm->wm[level],
+						&sw_plane_wm->wm[level]))
+				continue;
+
+			DRM_ERROR("mismatch in WM pipe %c cursor level %d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+				  pipe_name(pipe), level,
+				  sw_plane_wm->wm[level].plane_en,
+				  sw_plane_wm->wm[level].plane_res_b,
+				  sw_plane_wm->wm[level].plane_res_l,
+				  hw_plane_wm->wm[level].plane_en,
+				  hw_plane_wm->wm[level].plane_res_b,
+				  hw_plane_wm->wm[level].plane_res_l);
+		}
+
+		if (!skl_wm_level_equals(&hw_plane_wm->trans_wm,
+					 &sw_plane_wm->trans_wm)) {
+			DRM_ERROR("mismatch in trans WM pipe %c cursor (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+				  pipe_name(pipe),
+				  sw_plane_wm->trans_wm.plane_en,
+				  sw_plane_wm->trans_wm.plane_res_b,
+				  sw_plane_wm->trans_wm.plane_res_l,
+				  hw_plane_wm->trans_wm.plane_en,
+				  hw_plane_wm->trans_wm.plane_res_b,
+				  hw_plane_wm->trans_wm.plane_res_l);
+		}
+
+		/* DDB */
+		hw_ddb_entry = &hw_ddb.plane[pipe][PLANE_CURSOR];
+		sw_ddb_entry = &sw_ddb->plane[pipe][PLANE_CURSOR];
+
+		if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
+			DRM_ERROR("mismatch in DDB state pipe %c cursor (expected (%u,%u), found (%u,%u))\n",
 				  pipe_name(pipe),
-				  sw_entry->start, sw_entry->end,
-				  hw_entry->start, hw_entry->end);
+				  sw_ddb_entry->start, sw_ddb_entry->end,
+				  hw_ddb_entry->start, hw_ddb_entry->end);
 		}
 	}
 }
 
 static void
-verify_connector_state(struct drm_device *dev, struct drm_crtc *crtc)
+verify_connector_state(struct drm_device *dev,
+		       struct drm_atomic_state *state,
+		       struct drm_crtc *crtc)
 {
 	struct drm_connector *connector;
+	struct drm_connector_state *old_conn_state;
+	int i;
 
-	drm_for_each_connector(connector, dev) {
+	for_each_connector_in_state(state, connector, old_conn_state, i) {
 		struct drm_encoder *encoder = connector->encoder;
 		struct drm_connector_state *state = connector->state;
 
@@ -13630,7 +13653,7 @@ verify_crtc_state(struct drm_crtc *crtc,
 	intel_pipe_config_sanity_check(dev_priv, pipe_config);
 
 	sw_config = to_intel_crtc_state(crtc->state);
-	if (!intel_pipe_config_compare(dev, sw_config,
+	if (!intel_pipe_config_compare(dev_priv, sw_config,
 				       pipe_config, false)) {
 		I915_STATE_WARN(1, "pipe state doesn't match!\n");
 		intel_dump_pipe_config(intel_crtc, pipe_config,
@@ -13723,15 +13746,16 @@ verify_shared_dpll_state(struct drm_device *dev, struct drm_crtc *crtc,
 
 static void
 intel_modeset_verify_crtc(struct drm_crtc *crtc,
-			 struct drm_crtc_state *old_state,
-			 struct drm_crtc_state *new_state)
+			  struct drm_atomic_state *state,
+			  struct drm_crtc_state *old_state,
+			  struct drm_crtc_state *new_state)
 {
 	if (!needs_modeset(new_state) &&
 	    !to_intel_crtc_state(new_state)->update_pipe)
 		return;
 
 	verify_wm_state(crtc, new_state);
-	verify_connector_state(crtc->dev, crtc);
+	verify_connector_state(crtc->dev, state, crtc);
 	verify_crtc_state(crtc, old_state, new_state);
 	verify_shared_dpll_state(crtc->dev, crtc, old_state, new_state);
 }
@@ -13747,16 +13771,17 @@ verify_disabled_dpll_state(struct drm_device *dev)
 }
 
 static void
-intel_modeset_verify_disabled(struct drm_device *dev)
+intel_modeset_verify_disabled(struct drm_device *dev,
+			      struct drm_atomic_state *state)
 {
 	verify_encoder_state(dev);
-	verify_connector_state(dev, NULL);
+	verify_connector_state(dev, state, NULL);
 	verify_disabled_dpll_state(dev);
 }
 
 static void update_scanline_offset(struct intel_crtc *crtc)
 {
-	struct drm_device *dev = crtc->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
 	/*
 	 * The scanline counter increments at the leading edge of hsync.
@@ -13776,7 +13801,7 @@ static void update_scanline_offset(struct intel_crtc *crtc)
 	 * there's an extra 1 line difference. So we need to add two instead of
 	 * one to the value.
 	 */
-	if (IS_GEN2(dev)) {
+	if (IS_GEN2(dev_priv)) {
 		const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
 		int vtotal;
 
@@ -13785,7 +13810,7 @@ static void update_scanline_offset(struct intel_crtc *crtc)
 			vtotal /= 2;
 
 		crtc->scanline_offset = vtotal - 1;
-	} else if (HAS_DDI(dev) &&
+	} else if (HAS_DDI(dev_priv) &&
 		   intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI)) {
 		crtc->scanline_offset = 2;
 	} else
@@ -13970,8 +13995,9 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
 
 		DRM_DEBUG_KMS("New cdclk calculated to be atomic %u, actual %u\n",
 			      intel_state->cdclk, intel_state->dev_cdclk);
-	} else
+	} else {
 		to_intel_atomic_state(state)->cdclk = dev_priv->atomic_cdclk_freq;
+	}
 
 	intel_modeset_clear_plls(state);
 
@@ -14048,7 +14074,7 @@ static int intel_atomic_check(struct drm_device *dev,
 		}
 
 		if (i915.fastboot &&
-		    intel_pipe_config_compare(dev,
+		    intel_pipe_config_compare(dev_priv,
 					to_intel_crtc_state(crtc->state),
 					pipe_config, true)) {
 			crtc_state->mode_changed = false;
@@ -14072,8 +14098,9 @@ static int intel_atomic_check(struct drm_device *dev,
 
 		if (ret)
 			return ret;
-	} else
-		intel_state->cdclk = dev_priv->cdclk_freq;
+	} else {
+		intel_state->cdclk = dev_priv->atomic_cdclk_freq;
+	}
 
 	ret = drm_atomic_helper_check_planes(dev, state);
 	if (ret)
@@ -14084,13 +14111,10 @@ static int intel_atomic_check(struct drm_device *dev,
 }
 
 static int intel_atomic_prepare_commit(struct drm_device *dev,
-				       struct drm_atomic_state *state,
-				       bool nonblock)
+				       struct drm_atomic_state *state)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_plane_state *plane_state;
 	struct drm_crtc_state *crtc_state;
-	struct drm_plane *plane;
 	struct drm_crtc *crtc;
 	int i, ret;
 
@@ -14113,28 +14137,6 @@ static int intel_atomic_prepare_commit(struct drm_device *dev,
 	ret = drm_atomic_helper_prepare_planes(dev, state);
 	mutex_unlock(&dev->struct_mutex);
 
-	if (!ret && !nonblock) {
-		for_each_plane_in_state(state, plane, plane_state, i) {
-			struct intel_plane_state *intel_plane_state =
-				to_intel_plane_state(plane_state);
-
-			if (!intel_plane_state->wait_req)
-				continue;
-
-			ret = i915_wait_request(intel_plane_state->wait_req,
-						I915_WAIT_INTERRUPTIBLE,
-						NULL, NULL);
-			if (ret) {
-				/* Any hang should be swallowed by the wait */
-				WARN_ON(ret == -EIO);
-				mutex_lock(&dev->struct_mutex);
-				drm_atomic_helper_cleanup_planes(dev, state);
-				mutex_unlock(&dev->struct_mutex);
-				break;
-			}
-		}
-	}
-
 	return ret;
 }
 
@@ -14160,22 +14162,24 @@ static void intel_atomic_wait_for_vblanks(struct drm_device *dev,
 		return;
 
 	for_each_pipe(dev_priv, pipe) {
-		struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+		struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv,
+								  pipe);
 
 		if (!((1 << pipe) & crtc_mask))
 			continue;
 
-		ret = drm_crtc_vblank_get(crtc);
+		ret = drm_crtc_vblank_get(&crtc->base);
 		if (WARN_ON(ret != 0)) {
 			crtc_mask &= ~(1 << pipe);
 			continue;
 		}
 
-		last_vblank_count[pipe] = drm_crtc_vblank_count(crtc);
+		last_vblank_count[pipe] = drm_crtc_vblank_count(&crtc->base);
 	}
 
 	for_each_pipe(dev_priv, pipe) {
-		struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+		struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv,
+								  pipe);
 		long lret;
 
 		if (!((1 << pipe) & crtc_mask))
@@ -14183,12 +14187,12 @@ static void intel_atomic_wait_for_vblanks(struct drm_device *dev,
 
 		lret = wait_event_timeout(dev->vblank[pipe].queue,
 				last_vblank_count[pipe] !=
-					drm_crtc_vblank_count(crtc),
+					drm_crtc_vblank_count(&crtc->base),
 				msecs_to_jiffies(50));
 
 		WARN(!lret, "pipe %c vblank wait timed out\n", pipe_name(pipe));
 
-		drm_crtc_vblank_put(crtc);
+		drm_crtc_vblank_put(&crtc->base);
 	}
 }
 
@@ -14262,16 +14266,23 @@ static void intel_update_crtcs(struct drm_atomic_state *state,
 static void skl_update_crtcs(struct drm_atomic_state *state,
 			     unsigned int *crtc_vblank_mask)
 {
-	struct drm_device *dev = state->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(state->dev);
 	struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
 	struct drm_crtc *crtc;
+	struct intel_crtc *intel_crtc;
 	struct drm_crtc_state *old_crtc_state;
-	struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
-	struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
+	struct intel_crtc_state *cstate;
 	unsigned int updated = 0;
 	bool progress;
 	enum pipe pipe;
+	int i;
+
+	const struct skl_ddb_entry *entries[I915_MAX_PIPES] = {};
+
+	for_each_crtc_in_state(state, crtc, old_crtc_state, i)
+		/* ignore allocations for crtc's that have been turned off. */
+		if (crtc->state->active)
+			entries[i] = &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb;
 
 	/*
 	 * Whenever the number of active pipes changes, we need to make sure we
@@ -14280,21 +14291,24 @@ static void skl_update_crtcs(struct drm_atomic_state *state,
 	 * cause pipe underruns and other bad stuff.
 	 */
 	do {
-		int i;
 		progress = false;
 
 		for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
 			bool vbl_wait = false;
 			unsigned int cmask = drm_crtc_mask(crtc);
-			pipe = to_intel_crtc(crtc)->pipe;
 
-			if (updated & cmask || !crtc->state->active)
+			intel_crtc = to_intel_crtc(crtc);
+			cstate = to_intel_crtc_state(crtc->state);
+			pipe = intel_crtc->pipe;
+
+			if (updated & cmask || !cstate->base.active)
 				continue;
-			if (skl_ddb_allocation_overlaps(state, cur_ddb, new_ddb,
-							pipe))
+
+			if (skl_ddb_allocation_overlaps(entries, &cstate->wm.skl.ddb, i))
 				continue;
 
 			updated |= cmask;
+			entries[i] = &cstate->wm.skl.ddb;
 
 			/*
 			 * If this is an already active pipe, it's DDB changed,
@@ -14302,7 +14316,8 @@ static void skl_update_crtcs(struct drm_atomic_state *state,
 			 * then we need to wait for a vblank to pass for the
 			 * new ddb allocation to take effect.
 			 */
-			if (!skl_ddb_allocation_equals(cur_ddb, new_ddb, pipe) &&
+			if (!skl_ddb_entry_equal(&cstate->wm.skl.ddb,
+						 &to_intel_crtc_state(old_crtc_state)->wm.skl.ddb) &&
 			    !crtc->state->active_changed &&
 			    intel_state->wm_results.dirty_pipes != updated)
 				vbl_wait = true;
@@ -14311,7 +14326,7 @@ static void skl_update_crtcs(struct drm_atomic_state *state,
 					  crtc_vblank_mask);
 
 			if (vbl_wait)
-				intel_wait_for_vblank(dev, pipe);
+				intel_wait_for_vblank(dev_priv, pipe);
 
 			progress = true;
 		}
@@ -14326,37 +14341,15 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 	struct drm_crtc_state *old_crtc_state;
 	struct drm_crtc *crtc;
 	struct intel_crtc_state *intel_cstate;
-	struct drm_plane *plane;
-	struct drm_plane_state *plane_state;
 	bool hw_check = intel_state->modeset;
 	unsigned long put_domains[I915_MAX_PIPES] = {};
 	unsigned crtc_vblank_mask = 0;
-	int i, ret;
-
-	for_each_plane_in_state(state, plane, plane_state, i) {
-		struct intel_plane_state *intel_plane_state =
-			to_intel_plane_state(plane->state);
-
-		if (!intel_plane_state->wait_req)
-			continue;
-
-		ret = i915_wait_request(intel_plane_state->wait_req,
-					0, NULL, NULL);
-		/* EIO should be eaten, and we can't get interrupted in the
-		 * worker, and blocking commits have waited already. */
-		WARN_ON(ret);
-	}
+	int i;
 
 	drm_atomic_helper_wait_for_dependencies(state);
 
-	if (intel_state->modeset) {
-		memcpy(dev_priv->min_pixclk, intel_state->min_pixclk,
-		       sizeof(intel_state->min_pixclk));
-		dev_priv->active_crtcs = intel_state->active_crtcs;
-		dev_priv->atomic_cdclk_freq = intel_state->cdclk;
-
+	if (intel_state->modeset)
 		intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
-	}
 
 	for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
 		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
@@ -14389,8 +14382,17 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 			intel_check_cpu_fifo_underruns(dev_priv);
 			intel_check_pch_fifo_underruns(dev_priv);
 
-			if (!crtc->state->active)
-				intel_update_watermarks(crtc);
+			if (!crtc->state->active) {
+				/*
+				 * Make sure we don't call initial_watermarks
+				 * for ILK-style watermark updates.
+				 */
+				if (dev_priv->display.atomic_update_watermarks)
+					dev_priv->display.initial_watermarks(intel_state,
+									     to_intel_crtc_state(crtc->state));
+				else
+					intel_update_watermarks(intel_crtc);
+			}
 		}
 	}
 
@@ -14413,7 +14415,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 		if (!intel_can_enable_sagv(state))
 			intel_disable_sagv(dev_priv);
 
-		intel_modeset_verify_disabled(dev);
+		intel_modeset_verify_disabled(dev, state);
 	}
 
 	/* Complete the events for pipes that have now been disabled */
@@ -14456,7 +14458,8 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 		intel_cstate = to_intel_crtc_state(crtc->state);
 
 		if (dev_priv->display.optimize_watermarks)
-			dev_priv->display.optimize_watermarks(intel_cstate);
+			dev_priv->display.optimize_watermarks(intel_state,
+							      intel_cstate);
 	}
 
 	for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
@@ -14465,7 +14468,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 		if (put_domains[i])
 			modeset_put_power_domains(dev_priv, put_domains[i]);
 
-		intel_modeset_verify_crtc(crtc, old_crtc_state, crtc->state);
+		intel_modeset_verify_crtc(crtc, state, old_crtc_state, crtc->state);
 	}
 
 	if (intel_state->modeset && intel_can_enable_sagv(state))
@@ -14482,7 +14485,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 
 	drm_atomic_helper_commit_cleanup_done(state);
 
-	drm_atomic_state_free(state);
+	drm_atomic_state_put(state);
 
 	/* As one of the primary mmio accessors, KMS has a high likelihood
 	 * of triggering bugs in unclaimed access. After we finish
@@ -14500,12 +14503,33 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 
 static void intel_atomic_commit_work(struct work_struct *work)
 {
-	struct drm_atomic_state *state = container_of(work,
-						      struct drm_atomic_state,
-						      commit_work);
+	struct drm_atomic_state *state =
+		container_of(work, struct drm_atomic_state, commit_work);
+
 	intel_atomic_commit_tail(state);
 }
 
+static int __i915_sw_fence_call
+intel_atomic_commit_ready(struct i915_sw_fence *fence,
+			  enum i915_sw_fence_notify notify)
+{
+	struct intel_atomic_state *state =
+		container_of(fence, struct intel_atomic_state, commit_ready);
+
+	switch (notify) {
+	case FENCE_COMPLETE:
+		if (state->base.commit_work.func)
+			queue_work(system_unbound_wq, &state->base.commit_work);
+		break;
+
+	case FENCE_FREE:
+		drm_atomic_state_put(&state->base);
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
 static void intel_atomic_track_fbs(struct drm_atomic_state *state)
 {
 	struct drm_plane_state *old_plane_state;
@@ -14527,10 +14551,6 @@ static void intel_atomic_track_fbs(struct drm_atomic_state *state)
  * This function commits a top-level state object that has been validated
  * with drm_atomic_helper_check().
  *
- * FIXME:  Atomic modeset support for i915 is not yet complete.  At the moment
- * nonblocking commits are only safe for pure plane updates. Everything else
- * should work though.
- *
  * RETURNS
  * Zero for success or -errno.
  */
@@ -14542,33 +14562,42 @@ static int intel_atomic_commit(struct drm_device *dev,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	int ret = 0;
 
-	if (intel_state->modeset && nonblock) {
-		DRM_DEBUG_KMS("nonblocking commit for modeset not yet implemented.\n");
-		return -EINVAL;
-	}
-
 	ret = drm_atomic_helper_setup_commit(state, nonblock);
 	if (ret)
 		return ret;
 
-	INIT_WORK(&state->commit_work, intel_atomic_commit_work);
+	drm_atomic_state_get(state);
+	i915_sw_fence_init(&intel_state->commit_ready,
+			   intel_atomic_commit_ready);
 
-	ret = intel_atomic_prepare_commit(dev, state, nonblock);
+	ret = intel_atomic_prepare_commit(dev, state);
 	if (ret) {
 		DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
+		i915_sw_fence_commit(&intel_state->commit_ready);
 		return ret;
 	}
 
 	drm_atomic_helper_swap_state(state, true);
 	dev_priv->wm.distrust_bios_wm = false;
-	dev_priv->wm.skl_results = intel_state->wm_results;
 	intel_shared_dpll_commit(state);
 	intel_atomic_track_fbs(state);
 
-	if (nonblock)
-		queue_work(system_unbound_wq, &state->commit_work);
-	else
+	if (intel_state->modeset) {
+		memcpy(dev_priv->min_pixclk, intel_state->min_pixclk,
+		       sizeof(intel_state->min_pixclk));
+		dev_priv->active_crtcs = intel_state->active_crtcs;
+		dev_priv->atomic_cdclk_freq = intel_state->cdclk;
+	}
+
+	drm_atomic_state_get(state);
+	INIT_WORK(&state->commit_work,
+		  nonblock ? intel_atomic_commit_work : NULL);
+
+	i915_sw_fence_commit(&intel_state->commit_ready);
+	if (!nonblock) {
+		i915_sw_fence_wait(&intel_state->commit_ready);
 		intel_atomic_commit_tail(state);
+	}
 
 	return 0;
 }
@@ -14606,9 +14635,8 @@ retry:
 		goto retry;
 	}
 
-	if (ret)
 out:
-		drm_atomic_state_free(state);
+	drm_atomic_state_put(state);
 }
 
 /*
@@ -14681,19 +14709,21 @@ int
 intel_prepare_plane_fb(struct drm_plane *plane,
 		       struct drm_plane_state *new_state)
 {
-	struct drm_device *dev = plane->dev;
+	struct intel_atomic_state *intel_state =
+		to_intel_atomic_state(new_state->state);
+	struct drm_i915_private *dev_priv = to_i915(plane->dev);
 	struct drm_framebuffer *fb = new_state->fb;
 	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
-	struct reservation_object *resv;
-	int ret = 0;
+	int ret;
 
 	if (!obj && !old_obj)
 		return 0;
 
 	if (old_obj) {
 		struct drm_crtc_state *crtc_state =
-			drm_atomic_get_existing_crtc_state(new_state->state, plane->state->crtc);
+			drm_atomic_get_existing_crtc_state(new_state->state,
+							   plane->state->crtc);
 
 		/* Big Hammer, we also need to ensure that any pending
 		 * MI_WAIT_FOR_EVENT inside a user batch buffer on the
@@ -14706,52 +14736,58 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 		 * This should only fail upon a hung GPU, in which case we
 		 * can safely continue.
 		 */
-		if (needs_modeset(crtc_state))
-			ret = i915_gem_object_wait_rendering(old_obj, true);
-		if (ret) {
-			/* GPU hangs should have been swallowed by the wait */
-			WARN_ON(ret == -EIO);
-			return ret;
+		if (needs_modeset(crtc_state)) {
+			ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
+							      old_obj->resv, NULL,
+							      false, 0,
+							      GFP_KERNEL);
+			if (ret < 0)
+				return ret;
 		}
 	}
 
+	if (new_state->fence) { /* explicit fencing */
+		ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready,
+						    new_state->fence,
+						    I915_FENCE_TIMEOUT,
+						    GFP_KERNEL);
+		if (ret < 0)
+			return ret;
+	}
+
 	if (!obj)
 		return 0;
 
-	/* For framebuffer backed by dmabuf, wait for fence */
-	resv = i915_gem_object_get_dmabuf_resv(obj);
-	if (resv) {
-		long lret;
-
-		lret = reservation_object_wait_timeout_rcu(resv, false, true,
-							   MAX_SCHEDULE_TIMEOUT);
-		if (lret == -ERESTARTSYS)
-			return lret;
+	if (!new_state->fence) { /* implicit fencing */
+		ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
+						      obj->resv, NULL,
+						      false, I915_FENCE_TIMEOUT,
+						      GFP_KERNEL);
+		if (ret < 0)
+			return ret;
 
-		WARN(lret < 0, "waiting returns %li\n", lret);
+		i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY);
 	}
 
 	if (plane->type == DRM_PLANE_TYPE_CURSOR &&
-	    INTEL_INFO(dev)->cursor_needs_physical) {
-		int align = IS_I830(dev) ? 16 * 1024 : 256;
+	    INTEL_INFO(dev_priv)->cursor_needs_physical) {
+		int align = IS_I830(dev_priv) ? 16 * 1024 : 256;
 		ret = i915_gem_object_attach_phys(obj, align);
-		if (ret)
+		if (ret) {
 			DRM_DEBUG_KMS("failed to attach phys object\n");
+			return ret;
+		}
 	} else {
 		struct i915_vma *vma;
 
 		vma = intel_pin_and_fence_fb_obj(fb, new_state->rotation);
-		if (IS_ERR(vma))
-			ret = PTR_ERR(vma);
-	}
-
-	if (ret == 0) {
-		to_intel_plane_state(new_state)->wait_req =
-			i915_gem_active_get(&obj->last_write,
-					    &obj->base.dev->struct_mutex);
+		if (IS_ERR(vma)) {
+			DRM_DEBUG_KMS("failed to pin object\n");
+			return PTR_ERR(vma);
+		}
 	}
 
-	return ret;
+	return 0;
 }
 
 /**
@@ -14767,9 +14803,8 @@ void
 intel_cleanup_plane_fb(struct drm_plane *plane,
 		       struct drm_plane_state *old_state)
 {
-	struct drm_device *dev = plane->dev;
+	struct drm_i915_private *dev_priv = to_i915(plane->dev);
 	struct intel_plane_state *old_intel_state;
-	struct intel_plane_state *intel_state = to_intel_plane_state(plane->state);
 	struct drm_i915_gem_object *old_obj = intel_fb_obj(old_state->fb);
 	struct drm_i915_gem_object *obj = intel_fb_obj(plane->state->fb);
 
@@ -14779,11 +14814,8 @@ intel_cleanup_plane_fb(struct drm_plane *plane,
 		return;
 
 	if (old_obj && (plane->type != DRM_PLANE_TYPE_CURSOR ||
-	    !INTEL_INFO(dev)->cursor_needs_physical))
+	    !INTEL_INFO(dev_priv)->cursor_needs_physical))
 		intel_unpin_fb_obj(old_state->fb, old_state->rotation);
-
-	i915_gem_request_assign(&intel_state->wait_req, NULL);
-	i915_gem_request_assign(&old_intel_state->wait_req, NULL);
 }
 
 int
@@ -14858,30 +14890,34 @@ static void intel_begin_crtc_commit(struct drm_crtc *crtc,
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_crtc_state *old_intel_state =
+	struct intel_crtc_state *intel_cstate =
+		to_intel_crtc_state(crtc->state);
+	struct intel_crtc_state *old_intel_cstate =
 		to_intel_crtc_state(old_crtc_state);
+	struct intel_atomic_state *old_intel_state =
+		to_intel_atomic_state(old_crtc_state->state);
 	bool modeset = needs_modeset(crtc->state);
-	enum pipe pipe = intel_crtc->pipe;
 
 	/* Perform vblank evasion around commit operation */
 	intel_pipe_update_start(intel_crtc);
 
 	if (modeset)
-		return;
+		goto out;
 
 	if (crtc->state->color_mgmt_changed || to_intel_crtc_state(crtc->state)->update_pipe) {
 		intel_color_set_csc(crtc->state);
 		intel_color_load_luts(crtc->state);
 	}
 
-	if (to_intel_crtc_state(crtc->state)->update_pipe)
-		intel_update_pipe_config(intel_crtc, old_intel_state);
-	else if (INTEL_GEN(dev_priv) >= 9) {
+	if (intel_cstate->update_pipe)
+		intel_update_pipe_config(intel_crtc, old_intel_cstate);
+	else if (INTEL_GEN(dev_priv) >= 9)
 		skl_detach_scalers(intel_crtc);
 
-		I915_WRITE(PIPE_WM_LINETIME(pipe),
-			   dev_priv->wm.skl_hw.wm_linetime[pipe]);
-	}
+out:
+	if (dev_priv->display.atomic_update_watermarks)
+		dev_priv->display.atomic_update_watermarks(old_intel_state,
+							   intel_cstate);
 }
 
 static void intel_finish_crtc_commit(struct drm_crtc *crtc,
@@ -14901,9 +14937,6 @@ static void intel_finish_crtc_commit(struct drm_crtc *crtc,
  */
 void intel_plane_destroy(struct drm_plane *plane)
 {
-	if (!plane)
-		return;
-
 	drm_plane_cleanup(plane);
 	kfree(to_intel_plane(plane));
 }
@@ -14917,53 +14950,63 @@ const struct drm_plane_funcs intel_plane_funcs = {
 	.atomic_set_property = intel_plane_atomic_set_property,
 	.atomic_duplicate_state = intel_plane_duplicate_state,
 	.atomic_destroy_state = intel_plane_destroy_state,
-
 };
 
-static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
-						    int pipe)
+static struct intel_plane *
+intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
 	struct intel_plane *primary = NULL;
 	struct intel_plane_state *state = NULL;
 	const uint32_t *intel_primary_formats;
+	unsigned int supported_rotations;
 	unsigned int num_formats;
 	int ret;
 
 	primary = kzalloc(sizeof(*primary), GFP_KERNEL);
-	if (!primary)
+	if (!primary) {
+		ret = -ENOMEM;
 		goto fail;
+	}
 
 	state = intel_create_plane_state(&primary->base);
-	if (!state)
+	if (!state) {
+		ret = -ENOMEM;
 		goto fail;
+	}
+
 	primary->base.state = &state->base;
 
 	primary->can_scale = false;
 	primary->max_downscale = 1;
-	if (INTEL_INFO(dev)->gen >= 9) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		primary->can_scale = true;
 		state->scaler_id = -1;
 	}
 	primary->pipe = pipe;
-	primary->plane = pipe;
+	/*
+	 * On gen2/3 only plane A can do FBC, but the panel fitter and LVDS
+	 * port is hooked to pipe B. Hence we want plane A feeding pipe B.
+	 */
+	if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
+		primary->plane = (enum plane) !pipe;
+	else
+		primary->plane = (enum plane) pipe;
 	primary->frontbuffer_bit = INTEL_FRONTBUFFER_PRIMARY(pipe);
 	primary->check_plane = intel_check_primary_plane;
-	if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4)
-		primary->plane = !pipe;
 
-	if (INTEL_INFO(dev)->gen >= 9) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		intel_primary_formats = skl_primary_formats;
 		num_formats = ARRAY_SIZE(skl_primary_formats);
 
 		primary->update_plane = skylake_update_primary_plane;
 		primary->disable_plane = skylake_disable_primary_plane;
-	} else if (HAS_PCH_SPLIT(dev)) {
+	} else if (HAS_PCH_SPLIT(dev_priv)) {
 		intel_primary_formats = i965_primary_formats;
 		num_formats = ARRAY_SIZE(i965_primary_formats);
 
 		primary->update_plane = ironlake_update_primary_plane;
 		primary->disable_plane = i9xx_disable_primary_plane;
-	} else if (INTEL_INFO(dev)->gen >= 4) {
+	} else if (INTEL_GEN(dev_priv) >= 4) {
 		intel_primary_formats = i965_primary_formats;
 		num_formats = ARRAY_SIZE(i965_primary_formats);
 
@@ -14977,57 +15020,56 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
 		primary->disable_plane = i9xx_disable_primary_plane;
 	}
 
-	if (INTEL_INFO(dev)->gen >= 9)
-		ret = drm_universal_plane_init(dev, &primary->base, 0,
-					       &intel_plane_funcs,
+	if (INTEL_GEN(dev_priv) >= 9)
+		ret = drm_universal_plane_init(&dev_priv->drm, &primary->base,
+					       0, &intel_plane_funcs,
 					       intel_primary_formats, num_formats,
 					       DRM_PLANE_TYPE_PRIMARY,
 					       "plane 1%c", pipe_name(pipe));
-	else if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
-		ret = drm_universal_plane_init(dev, &primary->base, 0,
-					       &intel_plane_funcs,
+	else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
+		ret = drm_universal_plane_init(&dev_priv->drm, &primary->base,
+					       0, &intel_plane_funcs,
 					       intel_primary_formats, num_formats,
 					       DRM_PLANE_TYPE_PRIMARY,
 					       "primary %c", pipe_name(pipe));
 	else
-		ret = drm_universal_plane_init(dev, &primary->base, 0,
-					       &intel_plane_funcs,
+		ret = drm_universal_plane_init(&dev_priv->drm, &primary->base,
+					       0, &intel_plane_funcs,
 					       intel_primary_formats, num_formats,
 					       DRM_PLANE_TYPE_PRIMARY,
 					       "plane %c", plane_name(primary->plane));
 	if (ret)
 		goto fail;
 
-	if (INTEL_INFO(dev)->gen >= 4)
-		intel_create_rotation_property(dev, primary);
+	if (INTEL_GEN(dev_priv) >= 9) {
+		supported_rotations =
+			DRM_ROTATE_0 | DRM_ROTATE_90 |
+			DRM_ROTATE_180 | DRM_ROTATE_270;
+	} else if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
+		supported_rotations =
+			DRM_ROTATE_0 | DRM_ROTATE_180 |
+			DRM_REFLECT_X;
+	} else if (INTEL_GEN(dev_priv) >= 4) {
+		supported_rotations =
+			DRM_ROTATE_0 | DRM_ROTATE_180;
+	} else {
+		supported_rotations = DRM_ROTATE_0;
+	}
+
+	if (INTEL_GEN(dev_priv) >= 4)
+		drm_plane_create_rotation_property(&primary->base,
+						   DRM_ROTATE_0,
+						   supported_rotations);
 
 	drm_plane_helper_add(&primary->base, &intel_plane_helper_funcs);
 
-	return &primary->base;
+	return primary;
 
 fail:
 	kfree(state);
 	kfree(primary);
 
-	return NULL;
-}
-
-void intel_create_rotation_property(struct drm_device *dev, struct intel_plane *plane)
-{
-	if (!dev->mode_config.rotation_property) {
-		unsigned long flags = DRM_ROTATE_0 |
-			DRM_ROTATE_180;
-
-		if (INTEL_INFO(dev)->gen >= 9)
-			flags |= DRM_ROTATE_90 | DRM_ROTATE_270;
-
-		dev->mode_config.rotation_property =
-			drm_mode_create_rotation_property(dev, flags);
-	}
-	if (dev->mode_config.rotation_property)
-		drm_object_attach_property(&plane->base.base,
-				dev->mode_config.rotation_property,
-				plane->base.state->rotation);
+	return ERR_PTR(ret);
 }
 
 static int
@@ -15054,7 +15096,8 @@ intel_check_cursor_plane(struct drm_plane *plane,
 		return 0;
 
 	/* Check for which cursor types we support */
-	if (!cursor_size_ok(plane->dev, state->base.crtc_w, state->base.crtc_h)) {
+	if (!cursor_size_ok(to_i915(plane->dev), state->base.crtc_w,
+			    state->base.crtc_h)) {
 		DRM_DEBUG("Cursor dimension %dx%d not supported\n",
 			  state->base.crtc_w, state->base.crtc_h);
 		return -EINVAL;
@@ -15066,7 +15109,7 @@ intel_check_cursor_plane(struct drm_plane *plane,
 		return -ENOMEM;
 	}
 
-	if (fb->modifier[0] != DRM_FORMAT_MOD_NONE) {
+	if (fb->modifier != DRM_FORMAT_MOD_NONE) {
 		DRM_DEBUG_KMS("cursor cannot be tiled\n");
 		return -EINVAL;
 	}
@@ -15081,7 +15124,7 @@ intel_check_cursor_plane(struct drm_plane *plane,
 	 * display power well must be turned off and on again.
 	 * Refuse the put the cursor into that compromised position.
 	 */
-	if (IS_CHERRYVIEW(plane->dev) && pipe == PIPE_C &&
+	if (IS_CHERRYVIEW(to_i915(plane->dev)) && pipe == PIPE_C &&
 	    state->base.visible && state->base.crtc_x < 0) {
 		DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
 		return -EINVAL;
@@ -15107,13 +15150,13 @@ intel_update_cursor_plane(struct drm_plane *plane,
 {
 	struct drm_crtc *crtc = crtc_state->base.crtc;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct drm_device *dev = plane->dev;
+	struct drm_i915_private *dev_priv = to_i915(plane->dev);
 	struct drm_i915_gem_object *obj = intel_fb_obj(state->base.fb);
 	uint32_t addr;
 
 	if (!obj)
 		addr = 0;
-	else if (!INTEL_INFO(dev)->cursor_needs_physical)
+	else if (!INTEL_INFO(dev_priv)->cursor_needs_physical)
 		addr = i915_gem_object_ggtt_offset(obj, NULL);
 	else
 		addr = obj->phys_handle->busaddr;
@@ -15122,20 +15165,25 @@ intel_update_cursor_plane(struct drm_plane *plane,
 	intel_crtc_update_cursor(crtc, state);
 }
 
-static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
-						   int pipe)
+static struct intel_plane *
+intel_cursor_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
 	struct intel_plane *cursor = NULL;
 	struct intel_plane_state *state = NULL;
 	int ret;
 
 	cursor = kzalloc(sizeof(*cursor), GFP_KERNEL);
-	if (!cursor)
+	if (!cursor) {
+		ret = -ENOMEM;
 		goto fail;
+	}
 
 	state = intel_create_plane_state(&cursor->base);
-	if (!state)
+	if (!state) {
+		ret = -ENOMEM;
 		goto fail;
+	}
+
 	cursor->base.state = &state->base;
 
 	cursor->can_scale = false;
@@ -15147,8 +15195,8 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
 	cursor->update_plane = intel_update_cursor_plane;
 	cursor->disable_plane = intel_disable_cursor_plane;
 
-	ret = drm_universal_plane_init(dev, &cursor->base, 0,
-				       &intel_plane_funcs,
+	ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base,
+				       0, &intel_plane_funcs,
 				       intel_cursor_formats,
 				       ARRAY_SIZE(intel_cursor_formats),
 				       DRM_PLANE_TYPE_CURSOR,
@@ -15156,102 +15204,106 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
 	if (ret)
 		goto fail;
 
-	if (INTEL_INFO(dev)->gen >= 4) {
-		if (!dev->mode_config.rotation_property)
-			dev->mode_config.rotation_property =
-				drm_mode_create_rotation_property(dev,
-							DRM_ROTATE_0 |
-							DRM_ROTATE_180);
-		if (dev->mode_config.rotation_property)
-			drm_object_attach_property(&cursor->base.base,
-				dev->mode_config.rotation_property,
-				state->base.rotation);
-	}
+	if (INTEL_GEN(dev_priv) >= 4)
+		drm_plane_create_rotation_property(&cursor->base,
+						   DRM_ROTATE_0,
+						   DRM_ROTATE_0 |
+						   DRM_ROTATE_180);
 
-	if (INTEL_INFO(dev)->gen >=9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		state->scaler_id = -1;
 
 	drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
 
-	return &cursor->base;
+	return cursor;
 
 fail:
 	kfree(state);
 	kfree(cursor);
 
-	return NULL;
+	return ERR_PTR(ret);
 }
 
-static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc,
-	struct intel_crtc_state *crtc_state)
+static void skl_init_scalers(struct drm_i915_private *dev_priv,
+			     struct intel_crtc *crtc,
+			     struct intel_crtc_state *crtc_state)
 {
+	struct intel_crtc_scaler_state *scaler_state =
+		&crtc_state->scaler_state;
 	int i;
-	struct intel_scaler *intel_scaler;
-	struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state;
 
-	for (i = 0; i < intel_crtc->num_scalers; i++) {
-		intel_scaler = &scaler_state->scalers[i];
-		intel_scaler->in_use = 0;
-		intel_scaler->mode = PS_SCALER_MODE_DYN;
+	for (i = 0; i < crtc->num_scalers; i++) {
+		struct intel_scaler *scaler = &scaler_state->scalers[i];
+
+		scaler->in_use = 0;
+		scaler->mode = PS_SCALER_MODE_DYN;
 	}
 
 	scaler_state->scaler_id = -1;
 }
 
-static void intel_crtc_init(struct drm_device *dev, int pipe)
+static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc;
 	struct intel_crtc_state *crtc_state = NULL;
-	struct drm_plane *primary = NULL;
-	struct drm_plane *cursor = NULL;
-	int ret;
+	struct intel_plane *primary = NULL;
+	struct intel_plane *cursor = NULL;
+	int sprite, ret;
 
 	intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
-	if (intel_crtc == NULL)
-		return;
+	if (!intel_crtc)
+		return -ENOMEM;
 
 	crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
-	if (!crtc_state)
+	if (!crtc_state) {
+		ret = -ENOMEM;
 		goto fail;
+	}
 	intel_crtc->config = crtc_state;
 	intel_crtc->base.state = &crtc_state->base;
 	crtc_state->base.crtc = &intel_crtc->base;
 
 	/* initialize shared scalers */
-	if (INTEL_INFO(dev)->gen >= 9) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		if (pipe == PIPE_C)
 			intel_crtc->num_scalers = 1;
 		else
 			intel_crtc->num_scalers = SKL_NUM_SCALERS;
 
-		skl_init_scalers(dev, intel_crtc, crtc_state);
+		skl_init_scalers(dev_priv, intel_crtc, crtc_state);
 	}
 
-	primary = intel_primary_plane_create(dev, pipe);
-	if (!primary)
+	primary = intel_primary_plane_create(dev_priv, pipe);
+	if (IS_ERR(primary)) {
+		ret = PTR_ERR(primary);
 		goto fail;
+	}
+
+	for_each_sprite(dev_priv, pipe, sprite) {
+		struct intel_plane *plane;
 
-	cursor = intel_cursor_plane_create(dev, pipe);
-	if (!cursor)
+		plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
+		if (IS_ERR(plane)) {
+			ret = PTR_ERR(plane);
+			goto fail;
+		}
+	}
+
+	cursor = intel_cursor_plane_create(dev_priv, pipe);
+	if (IS_ERR(cursor)) {
+		ret = PTR_ERR(cursor);
 		goto fail;
+	}
 
-	ret = drm_crtc_init_with_planes(dev, &intel_crtc->base, primary,
-					cursor, &intel_crtc_funcs,
+	ret = drm_crtc_init_with_planes(&dev_priv->drm, &intel_crtc->base,
+					&primary->base, &cursor->base,
+					&intel_crtc_funcs,
 					"pipe %c", pipe_name(pipe));
 	if (ret)
 		goto fail;
 
-	/*
-	 * On gen2/3 only plane A can do fbc, but the panel fitter and lvds port
-	 * is hooked to pipe B. Hence we want plane A feeding pipe B.
-	 */
 	intel_crtc->pipe = pipe;
-	intel_crtc->plane = pipe;
-	if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4) {
-		DRM_DEBUG_KMS("swapping pipes & planes for FBC\n");
-		intel_crtc->plane = !pipe;
-	}
+	intel_crtc->plane = primary->plane;
 
 	intel_crtc->cursor_base = ~0;
 	intel_crtc->cursor_cntl = ~0;
@@ -15261,21 +15313,26 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
 
 	BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
 	       dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL);
-	dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base;
-	dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base;
+	dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = intel_crtc;
+	dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = intel_crtc;
 
 	drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
 
 	intel_color_init(&intel_crtc->base);
 
 	WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
-	return;
+
+	return 0;
 
 fail:
-	intel_plane_destroy(primary);
-	intel_plane_destroy(cursor);
+	/*
+	 * drm_mode_config_cleanup() will free up any
+	 * crtcs/planes already initialized.
+	 */
 	kfree(crtc_state);
 	kfree(intel_crtc);
+
+	return ret;
 }
 
 enum pipe intel_get_pipe_from_connector(struct intel_connector *connector)
@@ -15325,40 +15382,37 @@ static int intel_encoder_clones(struct intel_encoder *encoder)
 	return index_mask;
 }
 
-static bool has_edp_a(struct drm_device *dev)
+static bool has_edp_a(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	if (!IS_MOBILE(dev))
+	if (!IS_MOBILE(dev_priv))
 		return false;
 
 	if ((I915_READ(DP_A) & DP_DETECTED) == 0)
 		return false;
 
-	if (IS_GEN5(dev) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
+	if (IS_GEN5(dev_priv) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
 		return false;
 
 	return true;
 }
 
-static bool intel_crt_present(struct drm_device *dev)
+static bool intel_crt_present(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	if (INTEL_INFO(dev)->gen >= 9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		return false;
 
-	if (IS_HSW_ULT(dev) || IS_BDW_ULT(dev))
+	if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
 		return false;
 
-	if (IS_CHERRYVIEW(dev))
+	if (IS_CHERRYVIEW(dev_priv))
 		return false;
 
-	if (HAS_PCH_LPT_H(dev) && I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
+	if (HAS_PCH_LPT_H(dev_priv) &&
+	    I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
 		return false;
 
 	/* DDI E can't be used if DDI A requires 4 lanes */
-	if (HAS_DDI(dev) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
+	if (HAS_DDI(dev_priv) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
 		return false;
 
 	if (!dev_priv->vbt.int_crt_support)
@@ -15418,10 +15472,10 @@ static void intel_setup_outputs(struct drm_device *dev)
 	 */
 	intel_lvds_init(dev);
 
-	if (intel_crt_present(dev))
+	if (intel_crt_present(dev_priv))
 		intel_crt_init(dev);
 
-	if (IS_BROXTON(dev)) {
+	if (IS_BROXTON(dev_priv)) {
 		/*
 		 * FIXME: Broxton doesn't support port detection via the
 		 * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
@@ -15432,7 +15486,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 		intel_ddi_init(dev, PORT_C);
 
 		intel_dsi_init(dev);
-	} else if (HAS_DDI(dev)) {
+	} else if (HAS_DDI(dev_priv)) {
 		int found;
 
 		/*
@@ -15442,7 +15496,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 		 */
 		found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
 		/* WaIgnoreDDIAStrap: skl */
-		if (found || IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
+		if (found || IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
 			intel_ddi_init(dev, PORT_A);
 
 		/* DDI B, C and D detection is indicated by the SFUSE_STRAP
@@ -15458,17 +15512,17 @@ static void intel_setup_outputs(struct drm_device *dev)
 		/*
 		 * On SKL we don't have a way to detect DDI-E so we rely on VBT.
 		 */
-		if ((IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) &&
+		if ((IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) &&
 		    (dev_priv->vbt.ddi_port_info[PORT_E].supports_dp ||
 		     dev_priv->vbt.ddi_port_info[PORT_E].supports_dvi ||
 		     dev_priv->vbt.ddi_port_info[PORT_E].supports_hdmi))
 			intel_ddi_init(dev, PORT_E);
 
-	} else if (HAS_PCH_SPLIT(dev)) {
+	} else if (HAS_PCH_SPLIT(dev_priv)) {
 		int found;
-		dpd_is_edp = intel_dp_is_edp(dev, PORT_D);
+		dpd_is_edp = intel_dp_is_edp(dev_priv, PORT_D);
 
-		if (has_edp_a(dev))
+		if (has_edp_a(dev_priv))
 			intel_dp_init(dev, DP_A, PORT_A);
 
 		if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
@@ -15491,7 +15545,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 
 		if (I915_READ(PCH_DP_D) & DP_DETECTED)
 			intel_dp_init(dev, PCH_DP_D, PORT_D);
-	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		bool has_edp, has_port;
 
 		/*
@@ -15509,21 +15563,21 @@ static void intel_setup_outputs(struct drm_device *dev)
 		 * trust the port type the VBT declares as we've seen at least
 		 * HDMI ports that the VBT claim are DP or eDP.
 		 */
-		has_edp = intel_dp_is_edp(dev, PORT_B);
+		has_edp = intel_dp_is_edp(dev_priv, PORT_B);
 		has_port = intel_bios_is_port_present(dev_priv, PORT_B);
 		if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
 			has_edp &= intel_dp_init(dev, VLV_DP_B, PORT_B);
 		if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
 			intel_hdmi_init(dev, VLV_HDMIB, PORT_B);
 
-		has_edp = intel_dp_is_edp(dev, PORT_C);
+		has_edp = intel_dp_is_edp(dev_priv, PORT_C);
 		has_port = intel_bios_is_port_present(dev_priv, PORT_C);
 		if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
 			has_edp &= intel_dp_init(dev, VLV_DP_C, PORT_C);
 		if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
 			intel_hdmi_init(dev, VLV_HDMIC, PORT_C);
 
-		if (IS_CHERRYVIEW(dev)) {
+		if (IS_CHERRYVIEW(dev_priv)) {
 			/*
 			 * eDP not supported on port D,
 			 * so no need to worry about it
@@ -15536,18 +15590,18 @@ static void intel_setup_outputs(struct drm_device *dev)
 		}
 
 		intel_dsi_init(dev);
-	} else if (!IS_GEN2(dev) && !IS_PINEVIEW(dev)) {
+	} else if (!IS_GEN2(dev_priv) && !IS_PINEVIEW(dev_priv)) {
 		bool found = false;
 
 		if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
 			DRM_DEBUG_KMS("probing SDVOB\n");
 			found = intel_sdvo_init(dev, GEN3_SDVOB, PORT_B);
-			if (!found && IS_G4X(dev)) {
+			if (!found && IS_G4X(dev_priv)) {
 				DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
 				intel_hdmi_init(dev, GEN4_HDMIB, PORT_B);
 			}
 
-			if (!found && IS_G4X(dev))
+			if (!found && IS_G4X(dev_priv))
 				intel_dp_init(dev, DP_B, PORT_B);
 		}
 
@@ -15560,21 +15614,20 @@ static void intel_setup_outputs(struct drm_device *dev)
 
 		if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
 
-			if (IS_G4X(dev)) {
+			if (IS_G4X(dev_priv)) {
 				DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
 				intel_hdmi_init(dev, GEN4_HDMIC, PORT_C);
 			}
-			if (IS_G4X(dev))
+			if (IS_G4X(dev_priv))
 				intel_dp_init(dev, DP_C, PORT_C);
 		}
 
-		if (IS_G4X(dev) &&
-		    (I915_READ(DP_D) & DP_DETECTED))
+		if (IS_G4X(dev_priv) && (I915_READ(DP_D) & DP_DETECTED))
 			intel_dp_init(dev, DP_D, PORT_D);
-	} else if (IS_GEN2(dev))
+	} else if (IS_GEN2(dev_priv))
 		intel_dvo_init(dev);
 
-	if (SUPPORTS_TV(dev))
+	if (SUPPORTS_TV(dev_priv))
 		intel_tv_init(dev);
 
 	intel_psr_init(dev);
@@ -15629,6 +15682,8 @@ static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
 	struct drm_i915_gem_object *obj = intel_fb->obj;
 
 	mutex_lock(&dev->struct_mutex);
+	if (obj->pin_display && obj->cache_dirty)
+		i915_gem_clflush_object(obj, true);
 	intel_fb_obj_flush(obj, false, ORIGIN_DIRTYFB);
 	mutex_unlock(&dev->struct_mutex);
 
@@ -15642,10 +15697,10 @@ static const struct drm_framebuffer_funcs intel_fb_funcs = {
 };
 
 static
-u32 intel_fb_pitch_limit(struct drm_device *dev, uint64_t fb_modifier,
-			 uint32_t pixel_format)
+u32 intel_fb_pitch_limit(struct drm_i915_private *dev_priv,
+			 uint64_t fb_modifier, uint32_t pixel_format)
 {
-	u32 gen = INTEL_INFO(dev)->gen;
+	u32 gen = INTEL_INFO(dev_priv)->gen;
 
 	if (gen >= 9) {
 		int cpp = drm_format_plane_cpp(pixel_format, 0);
@@ -15654,7 +15709,8 @@ u32 intel_fb_pitch_limit(struct drm_device *dev, uint64_t fb_modifier,
 		 *  pixels and 32K bytes."
 		 */
 		return min(8192 * cpp, 32768);
-	} else if (gen >= 5 && !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
+	} else if (gen >= 5 && !IS_VALLEYVIEW(dev_priv) &&
+		   !IS_CHERRYVIEW(dev_priv)) {
 		return 32*1024;
 	} else if (gen >= 4) {
 		if (fb_modifier == I915_FORMAT_MOD_X_TILED)
@@ -15681,7 +15737,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
 	unsigned int tiling = i915_gem_object_get_tiling(obj);
 	int ret;
 	u32 pitch_limit, stride_alignment;
-	char *format_name;
+	struct drm_format_name_buf format_name;
 
 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 
@@ -15708,7 +15764,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
 	switch (mode_cmd->modifier[0]) {
 	case I915_FORMAT_MOD_Y_TILED:
 	case I915_FORMAT_MOD_Yf_TILED:
-		if (INTEL_INFO(dev)->gen < 9) {
+		if (INTEL_GEN(dev_priv) < 9) {
 			DRM_DEBUG("Unsupported tiling 0x%llx!\n",
 				  mode_cmd->modifier[0]);
 			return -EINVAL;
@@ -15741,7 +15797,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
 		return -EINVAL;
 	}
 
-	pitch_limit = intel_fb_pitch_limit(dev, mode_cmd->modifier[0],
+	pitch_limit = intel_fb_pitch_limit(dev_priv, mode_cmd->modifier[0],
 					   mode_cmd->pixel_format);
 	if (mode_cmd->pitches[0] > pitch_limit) {
 		DRM_DEBUG("%s pitch (%u) must be at less than %d\n",
@@ -15771,37 +15827,33 @@ static int intel_framebuffer_init(struct drm_device *dev,
 	case DRM_FORMAT_ARGB8888:
 		break;
 	case DRM_FORMAT_XRGB1555:
-		if (INTEL_INFO(dev)->gen > 3) {
-			format_name = drm_get_format_name(mode_cmd->pixel_format);
-			DRM_DEBUG("unsupported pixel format: %s\n", format_name);
-			kfree(format_name);
+		if (INTEL_GEN(dev_priv) > 3) {
+			DRM_DEBUG("unsupported pixel format: %s\n",
+			          drm_get_format_name(mode_cmd->pixel_format, &format_name));
 			return -EINVAL;
 		}
 		break;
 	case DRM_FORMAT_ABGR8888:
-		if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
-		    INTEL_INFO(dev)->gen < 9) {
-			format_name = drm_get_format_name(mode_cmd->pixel_format);
-			DRM_DEBUG("unsupported pixel format: %s\n", format_name);
-			kfree(format_name);
+		if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
+		    INTEL_GEN(dev_priv) < 9) {
+			DRM_DEBUG("unsupported pixel format: %s\n",
+			          drm_get_format_name(mode_cmd->pixel_format, &format_name));
 			return -EINVAL;
 		}
 		break;
 	case DRM_FORMAT_XBGR8888:
 	case DRM_FORMAT_XRGB2101010:
 	case DRM_FORMAT_XBGR2101010:
-		if (INTEL_INFO(dev)->gen < 4) {
-			format_name = drm_get_format_name(mode_cmd->pixel_format);
-			DRM_DEBUG("unsupported pixel format: %s\n", format_name);
-			kfree(format_name);
+		if (INTEL_GEN(dev_priv) < 4) {
+			DRM_DEBUG("unsupported pixel format: %s\n",
+			          drm_get_format_name(mode_cmd->pixel_format, &format_name));
 			return -EINVAL;
 		}
 		break;
 	case DRM_FORMAT_ABGR2101010:
-		if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
-			format_name = drm_get_format_name(mode_cmd->pixel_format);
-			DRM_DEBUG("unsupported pixel format: %s\n", format_name);
-			kfree(format_name);
+		if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
+			DRM_DEBUG("unsupported pixel format: %s\n",
+			          drm_get_format_name(mode_cmd->pixel_format, &format_name));
 			return -EINVAL;
 		}
 		break;
@@ -15809,17 +15861,15 @@ static int intel_framebuffer_init(struct drm_device *dev,
 	case DRM_FORMAT_UYVY:
 	case DRM_FORMAT_YVYU:
 	case DRM_FORMAT_VYUY:
-		if (INTEL_INFO(dev)->gen < 5) {
-			format_name = drm_get_format_name(mode_cmd->pixel_format);
-			DRM_DEBUG("unsupported pixel format: %s\n", format_name);
-			kfree(format_name);
+		if (INTEL_GEN(dev_priv) < 5) {
+			DRM_DEBUG("unsupported pixel format: %s\n",
+			          drm_get_format_name(mode_cmd->pixel_format, &format_name));
 			return -EINVAL;
 		}
 		break;
 	default:
-		format_name = drm_get_format_name(mode_cmd->pixel_format);
-		DRM_DEBUG("unsupported pixel format: %s\n", format_name);
-		kfree(format_name);
+		DRM_DEBUG("unsupported pixel format: %s\n",
+		          drm_get_format_name(mode_cmd->pixel_format, &format_name));
 		return -EINVAL;
 	}
 
@@ -15860,17 +15910,11 @@ intel_user_framebuffer_create(struct drm_device *dev,
 
 	fb = intel_framebuffer_create(dev, &mode_cmd, obj);
 	if (IS_ERR(fb))
-		i915_gem_object_put_unlocked(obj);
+		i915_gem_object_put(obj);
 
 	return fb;
 }
 
-#ifndef CONFIG_DRM_FBDEV_EMULATION
-static inline void intel_fbdev_output_poll_changed(struct drm_device *dev)
-{
-}
-#endif
-
 static const struct drm_mode_config_funcs intel_mode_funcs = {
 	.fb_create = intel_user_framebuffer_create,
 	.output_poll_changed = intel_fbdev_output_poll_changed,
@@ -16246,12 +16290,11 @@ static void intel_init_quirks(struct drm_device *dev)
 }
 
 /* Disable the VGA plane that we never use */
-static void i915_disable_vga(struct drm_device *dev)
+static void i915_disable_vga(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct pci_dev *pdev = dev_priv->drm.pdev;
 	u8 sr1;
-	i915_reg_t vga_reg = i915_vgacntrl_reg(dev);
+	i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
 
 	/* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
 	vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
@@ -16269,11 +16312,11 @@ void intel_modeset_init_hw(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
-	intel_update_cdclk(dev);
+	intel_update_cdclk(dev_priv);
 
 	dev_priv->atomic_cdclk_freq = dev_priv->cdclk_freq;
 
-	intel_init_clock_gating(dev);
+	intel_init_clock_gating(dev_priv);
 }
 
 /*
@@ -16290,6 +16333,7 @@ static void sanitize_watermarks(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_atomic_state *state;
+	struct intel_atomic_state *intel_state;
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *cstate;
 	struct drm_modeset_acquire_ctx ctx;
@@ -16318,12 +16362,14 @@ retry:
 	if (WARN_ON(IS_ERR(state)))
 		goto fail;
 
+	intel_state = to_intel_atomic_state(state);
+
 	/*
 	 * Hardware readout is the only time we don't want to calculate
 	 * intermediate watermarks (since we don't trust the current
 	 * watermarks).
 	 */
-	to_intel_atomic_state(state)->skip_intermediate_wm = true;
+	intel_state->skip_intermediate_wm = true;
 
 	ret = intel_atomic_check(dev, state);
 	if (ret) {
@@ -16339,7 +16385,7 @@ retry:
 		 * BIOS-programmed watermarks untouched and hope for the best.
 		 */
 		WARN(true, "Could not determine valid watermarks for inherited state\n");
-		goto fail;
+		goto put_state;
 	}
 
 	/* Write calculated watermark values back */
@@ -16347,20 +16393,20 @@ retry:
 		struct intel_crtc_state *cs = to_intel_crtc_state(cstate);
 
 		cs->wm.need_postvbl_update = true;
-		dev_priv->display.optimize_watermarks(cs);
+		dev_priv->display.optimize_watermarks(intel_state, cs);
 	}
 
-	drm_atomic_state_free(state);
+put_state:
+	drm_atomic_state_put(state);
 fail:
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
 }
 
-void intel_modeset_init(struct drm_device *dev)
+int intel_modeset_init(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	int sprite, ret;
 	enum pipe pipe;
 	struct intel_crtc *crtc;
 
@@ -16378,10 +16424,10 @@ void intel_modeset_init(struct drm_device *dev)
 
 	intel_init_quirks(dev);
 
-	intel_init_pm(dev);
+	intel_init_pm(dev_priv);
 
-	if (INTEL_INFO(dev)->num_pipes == 0)
-		return;
+	if (INTEL_INFO(dev_priv)->num_pipes == 0)
+		return 0;
 
 	/*
 	 * There may be no VBT; and if the BIOS enabled SSC we can
@@ -16389,7 +16435,7 @@ void intel_modeset_init(struct drm_device *dev)
 	 * BIOS isn't using it, don't assume it will work even if the VBT
 	 * indicates as much.
 	 */
-	if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) {
+	if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
 		bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
 					    DREF_SSC1_ENABLE);
 
@@ -16401,10 +16447,10 @@ void intel_modeset_init(struct drm_device *dev)
 		}
 	}
 
-	if (IS_GEN2(dev)) {
+	if (IS_GEN2(dev_priv)) {
 		dev->mode_config.max_width = 2048;
 		dev->mode_config.max_height = 2048;
-	} else if (IS_GEN3(dev)) {
+	} else if (IS_GEN3(dev_priv)) {
 		dev->mode_config.max_width = 4096;
 		dev->mode_config.max_height = 4096;
 	} else {
@@ -16412,10 +16458,10 @@ void intel_modeset_init(struct drm_device *dev)
 		dev->mode_config.max_height = 8192;
 	}
 
-	if (IS_845G(dev) || IS_I865G(dev)) {
-		dev->mode_config.cursor_width = IS_845G(dev) ? 64 : 512;
+	if (IS_845G(dev_priv) || IS_I865G(dev_priv)) {
+		dev->mode_config.cursor_width = IS_845G(dev_priv) ? 64 : 512;
 		dev->mode_config.cursor_height = 1023;
-	} else if (IS_GEN2(dev)) {
+	} else if (IS_GEN2(dev_priv)) {
 		dev->mode_config.cursor_width = GEN2_CURSOR_WIDTH;
 		dev->mode_config.cursor_height = GEN2_CURSOR_HEIGHT;
 	} else {
@@ -16426,29 +16472,30 @@ void intel_modeset_init(struct drm_device *dev)
 	dev->mode_config.fb_base = ggtt->mappable_base;
 
 	DRM_DEBUG_KMS("%d display pipe%s available.\n",
-		      INTEL_INFO(dev)->num_pipes,
-		      INTEL_INFO(dev)->num_pipes > 1 ? "s" : "");
+		      INTEL_INFO(dev_priv)->num_pipes,
+		      INTEL_INFO(dev_priv)->num_pipes > 1 ? "s" : "");
 
 	for_each_pipe(dev_priv, pipe) {
-		intel_crtc_init(dev, pipe);
-		for_each_sprite(dev_priv, pipe, sprite) {
-			ret = intel_plane_init(dev, pipe, sprite);
-			if (ret)
-				DRM_DEBUG_KMS("pipe %c sprite %c init failed: %d\n",
-					      pipe_name(pipe), sprite_name(pipe, sprite), ret);
+		int ret;
+
+		ret = intel_crtc_init(dev_priv, pipe);
+		if (ret) {
+			drm_mode_config_cleanup(dev);
+			return ret;
 		}
 	}
 
 	intel_update_czclk(dev_priv);
-	intel_update_cdclk(dev);
+	intel_update_cdclk(dev_priv);
+	dev_priv->atomic_cdclk_freq = dev_priv->cdclk_freq;
 
 	intel_shared_dpll_init(dev);
 
 	if (dev_priv->max_cdclk_freq == 0)
-		intel_update_max_cdclk(dev);
+		intel_update_max_cdclk(dev_priv);
 
 	/* Just disable it once at startup */
-	i915_disable_vga(dev);
+	i915_disable_vga(dev_priv);
 	intel_setup_outputs(dev);
 
 	drm_modeset_lock_all(dev);
@@ -16484,6 +16531,8 @@ void intel_modeset_init(struct drm_device *dev)
 	 * since the watermark calculation done here will use pstate->fb.
 	 */
 	sanitize_watermarks(dev);
+
+	return 0;
 }
 
 static void intel_enable_pipe_a(struct drm_device *dev)
@@ -16513,11 +16562,10 @@ static void intel_enable_pipe_a(struct drm_device *dev)
 static bool
 intel_check_plane_mapping(struct intel_crtc *crtc)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	u32 val;
 
-	if (INTEL_INFO(dev)->num_pipes == 1)
+	if (INTEL_INFO(dev_priv)->num_pipes == 1)
 		return true;
 
 	val = I915_READ(DSPCNTR(!crtc->plane));
@@ -16591,7 +16639,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
 	/* We need to sanitize the plane -> pipe mapping first because this will
 	 * disable the crtc (and hence change the state) if it is wrong. Note
 	 * that gen4+ has a fixed plane -> pipe mapping.  */
-	if (INTEL_INFO(dev)->gen < 4 && !intel_check_plane_mapping(crtc)) {
+	if (INTEL_GEN(dev_priv) < 4 && !intel_check_plane_mapping(crtc)) {
 		bool plane;
 
 		DRM_DEBUG_KMS("[CRTC:%d:%s] wrong plane connection detected!\n",
@@ -16621,7 +16669,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
 	if (crtc->active && !intel_crtc_has_encoders(crtc))
 		intel_crtc_disable_noatomic(&crtc->base);
 
-	if (crtc->active || HAS_GMCH_DISPLAY(dev)) {
+	if (crtc->active || HAS_GMCH_DISPLAY(dev_priv)) {
 		/*
 		 * We start out with underrun reporting disabled to avoid races.
 		 * For correct bookkeeping mark this on active crtcs.
@@ -16693,21 +16741,18 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder)
 	 * the crtc fixup. */
 }
 
-void i915_redisable_vga_power_on(struct drm_device *dev)
+void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	i915_reg_t vga_reg = i915_vgacntrl_reg(dev);
+	i915_reg_t vga_reg = i915_vgacntrl_reg(dev_priv);
 
 	if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
 		DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
-		i915_disable_vga(dev);
+		i915_disable_vga(dev_priv);
 	}
 }
 
-void i915_redisable_vga(struct drm_device *dev)
+void i915_redisable_vga(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	/* This function can be called both from intel_modeset_setup_hw_state or
 	 * at a very early point in our resume sequence, where the power well
 	 * structures are not yet restored. Since this function is at a very
@@ -16718,7 +16763,7 @@ void i915_redisable_vga(struct drm_device *dev)
 	if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_VGA))
 		return;
 
-	i915_redisable_vga_power_on(dev);
+	i915_redisable_vga_power_on(dev_priv);
 
 	intel_display_power_put(dev_priv, POWER_DOMAIN_VGA);
 }
@@ -16790,7 +16835,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
 
 		DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
 			      crtc->base.base.id, crtc->base.name,
-			      crtc->active ? "enabled" : "disabled");
+			      enableddisabled(crtc->active));
 	}
 
 	for (i = 0; i < dev_priv->num_shared_dpll; i++) {
@@ -16813,7 +16858,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
 		pipe = 0;
 
 		if (encoder->get_hw_state(encoder, &pipe)) {
-			crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
+			crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
+
 			encoder->base.crtc = &crtc->base;
 			crtc->config->output_types |= 1 << encoder->type;
 			encoder->get_config(encoder, crtc->config);
@@ -16822,9 +16868,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
 		}
 
 		DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
-			      encoder->base.base.id,
-			      encoder->base.name,
-			      encoder->base.crtc ? "enabled" : "disabled",
+			      encoder->base.base.id, encoder->base.name,
+			      enableddisabled(encoder->base.crtc),
 			      pipe_name(pipe));
 	}
 
@@ -16853,9 +16898,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
 			connector->base.encoder = NULL;
 		}
 		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
-			      connector->base.base.id,
-			      connector->base.name,
-			      connector->base.encoder ? "enabled" : "disabled");
+			      connector->base.base.id, connector->base.name,
+			      enableddisabled(connector->base.encoder));
 	}
 
 	for_each_intel_crtc(dev, crtc) {
@@ -16914,7 +16958,8 @@ intel_modeset_setup_hw_state(struct drm_device *dev)
 	}
 
 	for_each_pipe(dev_priv, pipe) {
-		crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
+		crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
+
 		intel_sanitize_crtc(crtc);
 		intel_dump_pipe_config(crtc, crtc->config,
 				       "[setup_hw_state]");
@@ -16934,11 +16979,11 @@ intel_modeset_setup_hw_state(struct drm_device *dev)
 		pll->on = false;
 	}
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		vlv_wm_get_hw_state(dev);
-	else if (IS_GEN9(dev))
+	else if (IS_GEN9(dev_priv))
 		skl_wm_get_hw_state(dev);
-	else if (HAS_PCH_SPLIT(dev))
+	else if (HAS_PCH_SPLIT(dev_priv))
 		ilk_wm_get_hw_state(dev);
 
 	for_each_intel_crtc(dev, crtc) {
@@ -16988,10 +17033,9 @@ void intel_display_resume(struct drm_device *dev)
 	drm_modeset_acquire_fini(&ctx);
 	mutex_unlock(&dev->mode_config.mutex);
 
-	if (ret) {
+	if (ret)
 		DRM_ERROR("Restoring old state failed with %i\n", ret);
-		drm_atomic_state_free(state);
-	}
+	drm_atomic_state_put(state);
 }
 
 void intel_modeset_gem_init(struct drm_device *dev)
@@ -17103,10 +17147,9 @@ void intel_connector_attach_encoder(struct intel_connector *connector,
 /*
  * set vga decode state - true == enable VGA decode
  */
-int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
+int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, bool state)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	unsigned reg = INTEL_INFO(dev)->gen >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
+	unsigned reg = INTEL_GEN(dev_priv) >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
 	u16 gmch_ctrl;
 
 	if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
@@ -17130,6 +17173,8 @@ int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
 	return 0;
 }
 
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+
 struct intel_display_error_state {
 
 	u32 power_well_driver;
@@ -17258,17 +17303,16 @@ intel_display_capture_error_state(struct drm_i915_private *dev_priv)
 
 void
 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
-				struct drm_device *dev,
+				struct drm_i915_private *dev_priv,
 				struct intel_display_error_state *error)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	int i;
 
 	if (!error)
 		return;
 
-	err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev)->num_pipes);
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+	err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev_priv)->num_pipes);
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		err_printf(m, "PWR_WELL_CTL2: %08x\n",
 			   error->power_well_driver);
 	for_each_pipe(dev_priv, i) {
@@ -17281,13 +17325,13 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
 		err_printf(m, "Plane [%d]:\n", i);
 		err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
 		err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
-		if (INTEL_INFO(dev)->gen <= 3) {
+		if (INTEL_GEN(dev_priv) <= 3) {
 			err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
 			err_printf(m, "  POS: %08x\n", error->plane[i].pos);
 		}
-		if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
+		if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
 			err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
-		if (INTEL_INFO(dev)->gen >= 4) {
+		if (INTEL_GEN(dev_priv) >= 4) {
 			err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
 			err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
 		}
@@ -17312,3 +17356,5 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
 		err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
 	}
 }
+
+#endif
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index bf344d08356a..90283edcafba 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -213,6 +213,81 @@ intel_dp_downstream_max_dotclock(struct intel_dp *intel_dp)
 	return max_dotclk;
 }
 
+static int
+intel_dp_sink_rates(struct intel_dp *intel_dp, const int **sink_rates)
+{
+	if (intel_dp->num_sink_rates) {
+		*sink_rates = intel_dp->sink_rates;
+		return intel_dp->num_sink_rates;
+	}
+
+	*sink_rates = default_rates;
+
+	return (intel_dp_max_link_bw(intel_dp) >> 3) + 1;
+}
+
+static int
+intel_dp_source_rates(struct intel_dp *intel_dp, const int **source_rates)
+{
+	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
+	int size;
+
+	if (IS_BROXTON(dev_priv)) {
+		*source_rates = bxt_rates;
+		size = ARRAY_SIZE(bxt_rates);
+	} else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+		*source_rates = skl_rates;
+		size = ARRAY_SIZE(skl_rates);
+	} else {
+		*source_rates = default_rates;
+		size = ARRAY_SIZE(default_rates);
+	}
+
+	/* This depends on the fact that 5.4 is last value in the array */
+	if (!intel_dp_source_supports_hbr2(intel_dp))
+		size--;
+
+	return size;
+}
+
+static int intersect_rates(const int *source_rates, int source_len,
+			   const int *sink_rates, int sink_len,
+			   int *common_rates)
+{
+	int i = 0, j = 0, k = 0;
+
+	while (i < source_len && j < sink_len) {
+		if (source_rates[i] == sink_rates[j]) {
+			if (WARN_ON(k >= DP_MAX_SUPPORTED_RATES))
+				return k;
+			common_rates[k] = source_rates[i];
+			++k;
+			++i;
+			++j;
+		} else if (source_rates[i] < sink_rates[j]) {
+			++i;
+		} else {
+			++j;
+		}
+	}
+	return k;
+}
+
+static int intel_dp_common_rates(struct intel_dp *intel_dp,
+				 int *common_rates)
+{
+	const int *source_rates, *sink_rates;
+	int source_len, sink_len;
+
+	sink_len = intel_dp_sink_rates(intel_dp, &sink_rates);
+	source_len = intel_dp_source_rates(intel_dp, &source_rates);
+
+	return intersect_rates(source_rates, source_len,
+			       sink_rates, sink_len,
+			       common_rates);
+}
+
 static enum drm_mode_status
 intel_dp_mode_valid(struct drm_connector *connector,
 		    struct drm_display_mode *mode)
@@ -320,8 +395,7 @@ static void
 vlv_power_sequencer_kick(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
 	enum pipe pipe = intel_dp->pps_pipe;
 	bool pll_enabled, release_cl_override = false;
 	enum dpio_phy phy = DPIO_PHY(pipe);
@@ -344,7 +418,7 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp)
 	DP |= DP_PORT_WIDTH(1);
 	DP |= DP_LINK_TRAIN_PAT_1;
 
-	if (IS_CHERRYVIEW(dev))
+	if (IS_CHERRYVIEW(dev_priv))
 		DP |= DP_PIPE_SELECT_CHV(pipe);
 	else if (pipe == PIPE_B)
 		DP |= DP_PIPEB_SELECT;
@@ -356,10 +430,10 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp)
 	 * So enable temporarily it if it's not already enabled.
 	 */
 	if (!pll_enabled) {
-		release_cl_override = IS_CHERRYVIEW(dev) &&
+		release_cl_override = IS_CHERRYVIEW(dev_priv) &&
 			!chv_phy_powergate_ch(dev_priv, phy, ch, true);
 
-		if (vlv_force_pll_on(dev, pipe, IS_CHERRYVIEW(dev) ?
+		if (vlv_force_pll_on(dev_priv, pipe, IS_CHERRYVIEW(dev_priv) ?
 				     &chv_dpll[0].dpll : &vlv_dpll[0].dpll)) {
 			DRM_ERROR("Failed to force on pll for pipe %c!\n",
 				  pipe_name(pipe));
@@ -383,7 +457,7 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp)
 	POSTING_READ(intel_dp->output_reg);
 
 	if (!pll_enabled) {
-		vlv_force_pll_off(dev, pipe);
+		vlv_force_pll_off(dev_priv, pipe);
 
 		if (release_cl_override)
 			chv_phy_powergate_ch(dev_priv, phy, ch, false);
@@ -570,8 +644,8 @@ void intel_power_sequencer_reset(struct drm_i915_private *dev_priv)
 	struct drm_device *dev = &dev_priv->drm;
 	struct intel_encoder *encoder;
 
-	if (WARN_ON(!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
-		    !IS_BROXTON(dev)))
+	if (WARN_ON(!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
+		    !IS_BROXTON(dev_priv)))
 		return;
 
 	/*
@@ -591,7 +665,7 @@ void intel_power_sequencer_reset(struct drm_i915_private *dev_priv)
 			continue;
 
 		intel_dp = enc_to_intel_dp(&encoder->base);
-		if (IS_BROXTON(dev))
+		if (IS_BROXTON(dev_priv))
 			intel_dp->pps_reset = true;
 		else
 			intel_dp->pps_pipe = INVALID_PIPE;
@@ -664,7 +738,7 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
 
 	pps_lock(intel_dp);
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		enum pipe pipe = vlv_power_sequencer_pipe(intel_dp);
 		i915_reg_t pp_ctrl_reg, pp_div_reg;
 		u32 pp_div;
@@ -692,7 +766,7 @@ static bool edp_have_panel_power(struct intel_dp *intel_dp)
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
-	if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 	    intel_dp->pps_pipe == INVALID_PIPE)
 		return false;
 
@@ -706,7 +780,7 @@ static bool edp_have_panel_vdd(struct intel_dp *intel_dp)
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
-	if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 	    intel_dp->pps_pipe == INVALID_PIPE)
 		return false;
 
@@ -821,15 +895,16 @@ static uint32_t g4x_get_aux_send_ctl(struct intel_dp *intel_dp,
 				     uint32_t aux_clock_divider)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = intel_dig_port->base.base.dev;
+	struct drm_i915_private *dev_priv =
+			to_i915(intel_dig_port->base.base.dev);
 	uint32_t precharge, timeout;
 
-	if (IS_GEN6(dev))
+	if (IS_GEN6(dev_priv))
 		precharge = 3;
 	else
 		precharge = 5;
 
-	if (IS_BROADWELL(dev) && intel_dig_port->port == PORT_A)
+	if (IS_BROADWELL(dev_priv) && intel_dig_port->port == PORT_A)
 		timeout = DP_AUX_CH_CTL_TIME_OUT_600us;
 	else
 		timeout = DP_AUX_CH_CTL_TIME_OUT_400us;
@@ -867,14 +942,14 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
 		uint8_t *recv, int recv_size)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv =
+			to_i915(intel_dig_port->base.base.dev);
 	i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg;
 	uint32_t aux_clock_divider;
 	int i, ret, recv_bytes;
 	uint32_t status;
 	int try, clock = 0;
-	bool has_aux_irq = HAS_AUX_IRQ(dev);
+	bool has_aux_irq = HAS_AUX_IRQ(dev_priv);
 	bool vdd;
 
 	pps_lock(intel_dp);
@@ -1147,7 +1222,7 @@ static enum port intel_aux_port(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t g4x_aux_ctl_reg(struct drm_i915_private *dev_priv,
-				       enum port port)
+				  enum port port)
 {
 	switch (port) {
 	case PORT_B:
@@ -1161,7 +1236,7 @@ static i915_reg_t g4x_aux_ctl_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t g4x_aux_data_reg(struct drm_i915_private *dev_priv,
-					enum port port, int index)
+				   enum port port, int index)
 {
 	switch (port) {
 	case PORT_B:
@@ -1175,7 +1250,7 @@ static i915_reg_t g4x_aux_data_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t ilk_aux_ctl_reg(struct drm_i915_private *dev_priv,
-				       enum port port)
+				  enum port port)
 {
 	switch (port) {
 	case PORT_A:
@@ -1191,7 +1266,7 @@ static i915_reg_t ilk_aux_ctl_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t ilk_aux_data_reg(struct drm_i915_private *dev_priv,
-					enum port port, int index)
+				   enum port port, int index)
 {
 	switch (port) {
 	case PORT_A:
@@ -1207,7 +1282,7 @@ static i915_reg_t ilk_aux_data_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t skl_aux_ctl_reg(struct drm_i915_private *dev_priv,
-				       enum port port)
+				  enum port port)
 {
 	switch (port) {
 	case PORT_A:
@@ -1222,7 +1297,7 @@ static i915_reg_t skl_aux_ctl_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t skl_aux_data_reg(struct drm_i915_private *dev_priv,
-					enum port port, int index)
+				   enum port port, int index)
 {
 	switch (port) {
 	case PORT_A:
@@ -1237,7 +1312,7 @@ static i915_reg_t skl_aux_data_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t intel_aux_ctl_reg(struct drm_i915_private *dev_priv,
-					 enum port port)
+				    enum port port)
 {
 	if (INTEL_INFO(dev_priv)->gen >= 9)
 		return skl_aux_ctl_reg(dev_priv, port);
@@ -1248,7 +1323,7 @@ static i915_reg_t intel_aux_ctl_reg(struct drm_i915_private *dev_priv,
 }
 
 static i915_reg_t intel_aux_data_reg(struct drm_i915_private *dev_priv,
-					  enum port port, int index)
+				     enum port port, int index)
 {
 	if (INTEL_INFO(dev_priv)->gen >= 9)
 		return skl_aux_data_reg(dev_priv, port, index);
@@ -1290,78 +1365,37 @@ intel_dp_aux_init(struct intel_dp *intel_dp)
 	intel_dp->aux.transfer = intel_dp_aux_transfer;
 }
 
-static int
-intel_dp_sink_rates(struct intel_dp *intel_dp, const int **sink_rates)
-{
-	if (intel_dp->num_sink_rates) {
-		*sink_rates = intel_dp->sink_rates;
-		return intel_dp->num_sink_rates;
-	}
-
-	*sink_rates = default_rates;
-
-	return (intel_dp_max_link_bw(intel_dp) >> 3) + 1;
-}
-
 bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = dig_port->base.base.dev;
-
-	/* WaDisableHBR2:skl */
-	if (IS_SKL_REVID(dev, 0, SKL_REVID_B0))
-		return false;
+	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
 
-	if ((IS_HASWELL(dev) && !IS_HSW_ULX(dev)) || IS_BROADWELL(dev) ||
-	    (INTEL_INFO(dev)->gen >= 9))
+	if ((IS_HASWELL(dev_priv) && !IS_HSW_ULX(dev_priv)) ||
+	    IS_BROADWELL(dev_priv) || (INTEL_GEN(dev_priv) >= 9))
 		return true;
 	else
 		return false;
 }
 
-static int
-intel_dp_source_rates(struct intel_dp *intel_dp, const int **source_rates)
-{
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = dig_port->base.base.dev;
-	int size;
-
-	if (IS_BROXTON(dev)) {
-		*source_rates = bxt_rates;
-		size = ARRAY_SIZE(bxt_rates);
-	} else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
-		*source_rates = skl_rates;
-		size = ARRAY_SIZE(skl_rates);
-	} else {
-		*source_rates = default_rates;
-		size = ARRAY_SIZE(default_rates);
-	}
-
-	/* This depends on the fact that 5.4 is last value in the array */
-	if (!intel_dp_source_supports_hbr2(intel_dp))
-		size--;
-
-	return size;
-}
-
 static void
 intel_dp_set_clock(struct intel_encoder *encoder,
 		   struct intel_crtc_state *pipe_config)
 {
 	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	const struct dp_link_dpll *divisor = NULL;
 	int i, count = 0;
 
-	if (IS_G4X(dev)) {
+	if (IS_G4X(dev_priv)) {
 		divisor = gen4_dpll;
 		count = ARRAY_SIZE(gen4_dpll);
-	} else if (HAS_PCH_SPLIT(dev)) {
+	} else if (HAS_PCH_SPLIT(dev_priv)) {
 		divisor = pch_dpll;
 		count = ARRAY_SIZE(pch_dpll);
-	} else if (IS_CHERRYVIEW(dev)) {
+	} else if (IS_CHERRYVIEW(dev_priv)) {
 		divisor = chv_dpll;
 		count = ARRAY_SIZE(chv_dpll);
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv)) {
 		divisor = vlv_dpll;
 		count = ARRAY_SIZE(vlv_dpll);
 	}
@@ -1377,43 +1411,6 @@ intel_dp_set_clock(struct intel_encoder *encoder,
 	}
 }
 
-static int intersect_rates(const int *source_rates, int source_len,
-			   const int *sink_rates, int sink_len,
-			   int *common_rates)
-{
-	int i = 0, j = 0, k = 0;
-
-	while (i < source_len && j < sink_len) {
-		if (source_rates[i] == sink_rates[j]) {
-			if (WARN_ON(k >= DP_MAX_SUPPORTED_RATES))
-				return k;
-			common_rates[k] = source_rates[i];
-			++k;
-			++i;
-			++j;
-		} else if (source_rates[i] < sink_rates[j]) {
-			++i;
-		} else {
-			++j;
-		}
-	}
-	return k;
-}
-
-static int intel_dp_common_rates(struct intel_dp *intel_dp,
-				 int *common_rates)
-{
-	const int *source_rates, *sink_rates;
-	int source_len, sink_len;
-
-	sink_len = intel_dp_sink_rates(intel_dp, &sink_rates);
-	source_len = intel_dp_source_rates(intel_dp, &source_rates);
-
-	return intersect_rates(source_rates, source_len,
-			       sink_rates, sink_len,
-			       common_rates);
-}
-
 static void snprintf_int_array(char *str, size_t len,
 			       const int *array, int nelem)
 {
@@ -1453,42 +1450,35 @@ static void intel_dp_print_rates(struct intel_dp *intel_dp)
 	DRM_DEBUG_KMS("common rates: %s\n", str);
 }
 
-static void intel_dp_print_hw_revision(struct intel_dp *intel_dp)
+bool
+__intel_dp_read_desc(struct intel_dp *intel_dp, struct intel_dp_desc *desc)
 {
-	uint8_t rev;
-	int len;
-
-	if ((drm_debug & DRM_UT_KMS) == 0)
-		return;
-
-	if (!(intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] &
-	      DP_DWN_STRM_PORT_PRESENT))
-		return;
-
-	len = drm_dp_dpcd_read(&intel_dp->aux, DP_BRANCH_HW_REV, &rev, 1);
-	if (len < 0)
-		return;
+	u32 base = drm_dp_is_branch(intel_dp->dpcd) ? DP_BRANCH_OUI :
+						      DP_SINK_OUI;
 
-	DRM_DEBUG_KMS("sink hw revision: %d.%d\n", (rev & 0xf0) >> 4, rev & 0xf);
+	return drm_dp_dpcd_read(&intel_dp->aux, base, desc, sizeof(*desc)) ==
+	       sizeof(*desc);
 }
 
-static void intel_dp_print_sw_revision(struct intel_dp *intel_dp)
+bool intel_dp_read_desc(struct intel_dp *intel_dp)
 {
-	uint8_t rev[2];
-	int len;
-
-	if ((drm_debug & DRM_UT_KMS) == 0)
-		return;
+	struct intel_dp_desc *desc = &intel_dp->desc;
+	bool oui_sup = intel_dp->dpcd[DP_DOWN_STREAM_PORT_COUNT] &
+		       DP_OUI_SUPPORT;
+	int dev_id_len;
 
-	if (!(intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] &
-	      DP_DWN_STRM_PORT_PRESENT))
-		return;
+	if (!__intel_dp_read_desc(intel_dp, desc))
+		return false;
 
-	len = drm_dp_dpcd_read(&intel_dp->aux, DP_BRANCH_SW_REV, &rev, 2);
-	if (len < 0)
-		return;
+	dev_id_len = strnlen(desc->device_id, sizeof(desc->device_id));
+	DRM_DEBUG_KMS("DP %s: OUI %*phD%s dev-ID %*pE HW-rev %d.%d SW-rev %d.%d\n",
+		      drm_dp_is_branch(intel_dp->dpcd) ? "branch" : "sink",
+		      (int)sizeof(desc->oui), desc->oui, oui_sup ? "" : "(NS)",
+		      dev_id_len, desc->device_id,
+		      desc->hw_rev >> 4, desc->hw_rev & 0xf,
+		      desc->sw_major_rev, desc->sw_minor_rev);
 
-	DRM_DEBUG_KMS("sink sw revision: %d.%d\n", rev[0], rev[1]);
+	return true;
 }
 
 static int rate_to_index(int find, const int *rates)
@@ -1552,8 +1542,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 			struct intel_crtc_state *pipe_config,
 			struct drm_connector_state *conn_state)
 {
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 	enum port port = dp_to_dig_port(intel_dp)->port;
@@ -1578,7 +1567,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 
 	max_clock = common_len - 1;
 
-	if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev) && port != PORT_A)
+	if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv) && port != PORT_A)
 		pipe_config->has_pch_encoder = true;
 
 	pipe_config->has_drrs = false;
@@ -1588,14 +1577,14 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 		intel_fixed_panel_mode(intel_connector->panel.fixed_mode,
 				       adjusted_mode);
 
-		if (INTEL_INFO(dev)->gen >= 9) {
+		if (INTEL_GEN(dev_priv) >= 9) {
 			int ret;
 			ret = skl_update_scaler_crtc(pipe_config);
 			if (ret)
 				return ret;
 		}
 
-		if (HAS_GMCH_DISPLAY(dev))
+		if (HAS_GMCH_DISPLAY(dev_priv))
 			intel_gmch_panel_fitting(intel_crtc, pipe_config,
 						 intel_connector->panel.fitting_mode);
 		else
@@ -1720,7 +1709,7 @@ found:
 		to_intel_atomic_state(pipe_config->base.state)->cdclk_pll_vco = vco;
 	}
 
-	if (!HAS_DDI(dev))
+	if (!HAS_DDI(dev_priv))
 		intel_dp_set_clock(encoder, pipe_config);
 
 	return true;
@@ -1778,7 +1767,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder,
 
 	/* Split out the IBX/CPU vs CPT settings */
 
-	if (IS_GEN7(dev) && port == PORT_A) {
+	if (IS_GEN7(dev_priv) && port == PORT_A) {
 		if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
 			intel_dp->DP |= DP_SYNC_HS_HIGH;
 		if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
@@ -1789,7 +1778,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder,
 			intel_dp->DP |= DP_ENHANCED_FRAMING;
 
 		intel_dp->DP |= crtc->pipe << 29;
-	} else if (HAS_PCH_CPT(dev) && port != PORT_A) {
+	} else if (HAS_PCH_CPT(dev_priv) && port != PORT_A) {
 		u32 trans_dp;
 
 		intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT;
@@ -1801,8 +1790,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder,
 			trans_dp &= ~TRANS_DP_ENH_FRAMING;
 		I915_WRITE(TRANS_DP_CTL(crtc->pipe), trans_dp);
 	} else {
-		if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev) &&
-		    !IS_CHERRYVIEW(dev) && pipe_config->limited_color_range)
+		if (IS_G4X(dev_priv) && pipe_config->limited_color_range)
 			intel_dp->DP |= DP_COLOR_RANGE_16_235;
 
 		if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
@@ -1814,7 +1802,7 @@ static void intel_dp_prepare(struct intel_encoder *encoder,
 		if (drm_dp_enhanced_frame_cap(intel_dp->dpcd))
 			intel_dp->DP |= DP_ENHANCED_FRAMING;
 
-		if (IS_CHERRYVIEW(dev))
+		if (IS_CHERRYVIEW(dev_priv))
 			intel_dp->DP |= DP_PIPE_SELECT_CHV(crtc->pipe);
 		else if (crtc->pipe == PIPE_B)
 			intel_dp->DP |= DP_PIPEB_SELECT;
@@ -2123,7 +2111,7 @@ static void edp_panel_on(struct intel_dp *intel_dp)
 
 	pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
 	pp = ironlake_get_pp_control(intel_dp);
-	if (IS_GEN5(dev)) {
+	if (IS_GEN5(dev_priv)) {
 		/* ILK workaround: disable reset around power sequence */
 		pp &= ~PANEL_POWER_RESET;
 		I915_WRITE(pp_ctrl_reg, pp);
@@ -2131,7 +2119,7 @@ static void edp_panel_on(struct intel_dp *intel_dp)
 	}
 
 	pp |= PANEL_POWER_ON;
-	if (!IS_GEN5(dev))
+	if (!IS_GEN5(dev_priv))
 		pp |= PANEL_POWER_RESET;
 
 	I915_WRITE(pp_ctrl_reg, pp);
@@ -2140,7 +2128,7 @@ static void edp_panel_on(struct intel_dp *intel_dp)
 	wait_panel_on(intel_dp);
 	intel_dp->last_power_on = jiffies;
 
-	if (IS_GEN5(dev)) {
+	if (IS_GEN5(dev_priv)) {
 		pp |= PANEL_POWER_RESET; /* restore panel reset bit */
 		I915_WRITE(pp_ctrl_reg, pp);
 		POSTING_READ(pp_ctrl_reg);
@@ -2372,7 +2360,7 @@ static void ironlake_edp_pll_on(struct intel_dp *intel_dp,
 	 * 2. Program DP PLL enable
 	 */
 	if (IS_GEN5(dev_priv))
-		intel_wait_for_vblank_if_active(&dev_priv->drm, !crtc->pipe);
+		intel_wait_for_vblank_if_active(dev_priv, !crtc->pipe);
 
 	intel_dp->DP |= DP_PLL_ENABLE;
 
@@ -2453,9 +2441,9 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
 	if (!(tmp & DP_PORT_EN))
 		goto out;
 
-	if (IS_GEN7(dev) && port == PORT_A) {
+	if (IS_GEN7(dev_priv) && port == PORT_A) {
 		*pipe = PORT_TO_PIPE_CPT(tmp);
-	} else if (HAS_PCH_CPT(dev) && port != PORT_A) {
+	} else if (HAS_PCH_CPT(dev_priv) && port != PORT_A) {
 		enum pipe p;
 
 		for_each_pipe(dev_priv, p) {
@@ -2470,7 +2458,7 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
 
 		DRM_DEBUG_KMS("No pipe for dp port 0x%x found\n",
 			      i915_mmio_reg_offset(intel_dp->output_reg));
-	} else if (IS_CHERRYVIEW(dev)) {
+	} else if (IS_CHERRYVIEW(dev_priv)) {
 		*pipe = DP_PORT_TO_PIPE_CHV(tmp);
 	} else {
 		*pipe = PORT_TO_PIPE(tmp);
@@ -2498,7 +2486,7 @@ static void intel_dp_get_config(struct intel_encoder *encoder,
 
 	pipe_config->has_audio = tmp & DP_AUDIO_OUTPUT_ENABLE && port != PORT_A;
 
-	if (HAS_PCH_CPT(dev) && port != PORT_A) {
+	if (HAS_PCH_CPT(dev_priv) && port != PORT_A) {
 		u32 trans_dp = I915_READ(TRANS_DP_CTL(crtc->pipe));
 
 		if (trans_dp & TRANS_DP_HSYNC_ACTIVE_HIGH)
@@ -2524,8 +2512,7 @@ static void intel_dp_get_config(struct intel_encoder *encoder,
 
 	pipe_config->base.adjusted_mode.flags |= flags;
 
-	if (!HAS_PCH_SPLIT(dev) && !IS_VALLEYVIEW(dev) &&
-	    !IS_CHERRYVIEW(dev) && tmp & DP_COLOR_RANGE_16_235)
+	if (IS_G4X(dev_priv) && tmp & DP_COLOR_RANGE_16_235)
 		pipe_config->limited_color_range = true;
 
 	pipe_config->lane_count =
@@ -2645,7 +2632,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 		DRM_DEBUG_KMS("Using DP training pattern TPS%d\n",
 			      dp_train_pat & DP_TRAINING_PATTERN_MASK);
 
-	if (HAS_DDI(dev)) {
+	if (HAS_DDI(dev_priv)) {
 		uint32_t temp = I915_READ(DP_TP_CTL(port));
 
 		if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE)
@@ -2671,8 +2658,8 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 		}
 		I915_WRITE(DP_TP_CTL(port), temp);
 
-	} else if ((IS_GEN7(dev) && port == PORT_A) ||
-		   (HAS_PCH_CPT(dev) && port != PORT_A)) {
+	} else if ((IS_GEN7(dev_priv) && port == PORT_A) ||
+		   (HAS_PCH_CPT(dev_priv) && port != PORT_A)) {
 		*DP &= ~DP_LINK_TRAIN_MASK_CPT;
 
 		switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
@@ -2692,7 +2679,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 		}
 
 	} else {
-		if (IS_CHERRYVIEW(dev))
+		if (IS_CHERRYVIEW(dev_priv))
 			*DP &= ~DP_LINK_TRAIN_MASK_CHV;
 		else
 			*DP &= ~DP_LINK_TRAIN_MASK;
@@ -2708,7 +2695,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 			*DP |= DP_LINK_TRAIN_PAT_2;
 			break;
 		case DP_TRAINING_PATTERN_3:
-			if (IS_CHERRYVIEW(dev)) {
+			if (IS_CHERRYVIEW(dev_priv)) {
 				*DP |= DP_LINK_TRAIN_PAT_3_CHV;
 			} else {
 				DRM_DEBUG_KMS("TPS3 not supported, using TPS2 instead\n");
@@ -2744,7 +2731,8 @@ static void intel_dp_enable_port(struct intel_dp *intel_dp,
 }
 
 static void intel_enable_dp(struct intel_encoder *encoder,
-			    struct intel_crtc_state *pipe_config)
+			    struct intel_crtc_state *pipe_config,
+			    struct drm_connector_state *conn_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 	struct drm_device *dev = encoder->base.dev;
@@ -2758,7 +2746,7 @@ static void intel_enable_dp(struct intel_encoder *encoder,
 
 	pps_lock(intel_dp);
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		vlv_init_panel_power_sequencer(intel_dp);
 
 	intel_dp_enable_port(intel_dp, pipe_config);
@@ -2769,10 +2757,10 @@ static void intel_enable_dp(struct intel_encoder *encoder,
 
 	pps_unlock(intel_dp);
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		unsigned int lane_mask = 0x0;
 
-		if (IS_CHERRYVIEW(dev))
+		if (IS_CHERRYVIEW(dev_priv))
 			lane_mask = intel_dp_unused_lane_mask(pipe_config->lane_count);
 
 		vlv_wait_port_ready(dev_priv, dp_to_dig_port(intel_dp),
@@ -2786,7 +2774,7 @@ static void intel_enable_dp(struct intel_encoder *encoder,
 	if (pipe_config->has_audio) {
 		DRM_DEBUG_DRIVER("Enabling DP audio on pipe %c\n",
 				 pipe_name(pipe));
-		intel_audio_codec_enable(encoder);
+		intel_audio_codec_enable(encoder, pipe_config, conn_state);
 	}
 }
 
@@ -2796,7 +2784,7 @@ static void g4x_enable_dp(struct intel_encoder *encoder,
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
-	intel_enable_dp(encoder, pipe_config);
+	intel_enable_dp(encoder, pipe_config, conn_state);
 	intel_edp_backlight_on(intel_dp);
 }
 
@@ -2933,7 +2921,7 @@ static void vlv_pre_enable_dp(struct intel_encoder *encoder,
 {
 	vlv_phy_pre_encoder_enable(encoder);
 
-	intel_enable_dp(encoder, pipe_config);
+	intel_enable_dp(encoder, pipe_config, conn_state);
 }
 
 static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder,
@@ -2951,7 +2939,7 @@ static void chv_pre_enable_dp(struct intel_encoder *encoder,
 {
 	chv_phy_pre_encoder_enable(encoder);
 
-	intel_enable_dp(encoder, pipe_config);
+	intel_enable_dp(encoder, pipe_config, conn_state);
 
 	/* Second common lane will stay alive on its own now */
 	chv_phy_release_cl2_override(encoder);
@@ -2988,21 +2976,20 @@ intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_
 uint8_t
 intel_dp_voltage_max(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	enum port port = dp_to_dig_port(intel_dp)->port;
 
-	if (IS_BROXTON(dev))
+	if (IS_BROXTON(dev_priv))
 		return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
-	else if (INTEL_INFO(dev)->gen >= 9) {
+	else if (INTEL_GEN(dev_priv) >= 9) {
 		if (dev_priv->vbt.edp.low_vswing && port == PORT_A)
 			return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
 		return DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
-	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
-	else if (IS_GEN7(dev) && port == PORT_A)
+	else if (IS_GEN7(dev_priv) && port == PORT_A)
 		return DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
-	else if (HAS_PCH_CPT(dev) && port != PORT_A)
+	else if (HAS_PCH_CPT(dev_priv) && port != PORT_A)
 		return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
 	else
 		return DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
@@ -3011,10 +2998,10 @@ intel_dp_voltage_max(struct intel_dp *intel_dp)
 uint8_t
 intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	enum port port = dp_to_dig_port(intel_dp)->port;
 
-	if (INTEL_INFO(dev)->gen >= 9) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
 		case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
 			return DP_TRAIN_PRE_EMPH_LEVEL_3;
@@ -3027,7 +3014,7 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
 		default:
 			return DP_TRAIN_PRE_EMPH_LEVEL_0;
 		}
-	} else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
 		switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
 		case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
 			return DP_TRAIN_PRE_EMPH_LEVEL_3;
@@ -3039,7 +3026,7 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
 		default:
 			return DP_TRAIN_PRE_EMPH_LEVEL_0;
 		}
-	} else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
 		case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
 			return DP_TRAIN_PRE_EMPH_LEVEL_3;
@@ -3051,7 +3038,7 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
 		default:
 			return DP_TRAIN_PRE_EMPH_LEVEL_0;
 		}
-	} else if (IS_GEN7(dev) && port == PORT_A) {
+	} else if (IS_GEN7(dev_priv) && port == PORT_A) {
 		switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
 		case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
 			return DP_TRAIN_PRE_EMPH_LEVEL_2;
@@ -3352,21 +3339,21 @@ intel_dp_set_signal_levels(struct intel_dp *intel_dp)
 	uint32_t signal_levels, mask = 0;
 	uint8_t train_set = intel_dp->train_set[0];
 
-	if (HAS_DDI(dev)) {
+	if (HAS_DDI(dev_priv)) {
 		signal_levels = ddi_signal_levels(intel_dp);
 
-		if (IS_BROXTON(dev))
+		if (IS_BROXTON(dev_priv))
 			signal_levels = 0;
 		else
 			mask = DDI_BUF_EMP_MASK;
-	} else if (IS_CHERRYVIEW(dev)) {
+	} else if (IS_CHERRYVIEW(dev_priv)) {
 		signal_levels = chv_signal_levels(intel_dp);
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv)) {
 		signal_levels = vlv_signal_levels(intel_dp);
-	} else if (IS_GEN7(dev) && port == PORT_A) {
+	} else if (IS_GEN7(dev_priv) && port == PORT_A) {
 		signal_levels = gen7_edp_signal_levels(train_set);
 		mask = EDP_LINK_TRAIN_VOL_EMP_MASK_IVB;
-	} else if (IS_GEN6(dev) && port == PORT_A) {
+	} else if (IS_GEN6(dev_priv) && port == PORT_A) {
 		signal_levels = gen6_edp_signal_levels(train_set);
 		mask = EDP_LINK_TRAIN_VOL_EMP_MASK_SNB;
 	} else {
@@ -3411,7 +3398,7 @@ void intel_dp_set_idle_link_train(struct intel_dp *intel_dp)
 	enum port port = intel_dig_port->port;
 	uint32_t val;
 
-	if (!HAS_DDI(dev))
+	if (!HAS_DDI(dev_priv))
 		return;
 
 	val = I915_READ(DP_TP_CTL(port));
@@ -3446,7 +3433,7 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t DP = intel_dp->DP;
 
-	if (WARN_ON(HAS_DDI(dev)))
+	if (WARN_ON(HAS_DDI(dev_priv)))
 		return;
 
 	if (WARN_ON((I915_READ(intel_dp->output_reg) & DP_PORT_EN) == 0))
@@ -3454,12 +3441,12 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 
 	DRM_DEBUG_KMS("\n");
 
-	if ((IS_GEN7(dev) && port == PORT_A) ||
-	    (HAS_PCH_CPT(dev) && port != PORT_A)) {
+	if ((IS_GEN7(dev_priv) && port == PORT_A) ||
+	    (HAS_PCH_CPT(dev_priv) && port != PORT_A)) {
 		DP &= ~DP_LINK_TRAIN_MASK_CPT;
 		DP |= DP_LINK_TRAIN_PAT_IDLE_CPT;
 	} else {
-		if (IS_CHERRYVIEW(dev))
+		if (IS_CHERRYVIEW(dev_priv))
 			DP &= ~DP_LINK_TRAIN_MASK_CHV;
 		else
 			DP &= ~DP_LINK_TRAIN_MASK;
@@ -3477,7 +3464,7 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 	 * to transcoder A after disabling it to allow the
 	 * matching HDMI port to be enabled on transcoder A.
 	 */
-	if (HAS_PCH_IBX(dev) && crtc->pipe == PIPE_B && port != PORT_A) {
+	if (HAS_PCH_IBX(dev_priv) && crtc->pipe == PIPE_B && port != PORT_A) {
 		/*
 		 * We get CPU/PCH FIFO underruns on the other pipe when
 		 * doing the workaround. Sweep them under the rug.
@@ -3495,7 +3482,7 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 		I915_WRITE(intel_dp->output_reg, DP);
 		POSTING_READ(intel_dp->output_reg);
 
-		intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A);
+		intel_wait_for_vblank_if_active(dev_priv, PIPE_A);
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true);
 		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true);
 	}
@@ -3505,7 +3492,7 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 	intel_dp->DP = DP;
 }
 
-static bool
+bool
 intel_dp_read_dpcd(struct intel_dp *intel_dp)
 {
 	if (drm_dp_dpcd_read(&intel_dp->aux, 0x000, intel_dp->dpcd,
@@ -3529,6 +3516,8 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp)
 	if (!intel_dp_read_dpcd(intel_dp))
 		return false;
 
+	intel_dp_read_desc(intel_dp);
+
 	if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11)
 		dev_priv->no_aux_handshake = intel_dp->dpcd[DP_MAX_DOWNSPREAD] &
 			DP_NO_AUX_HANDSHAKE_LINK_TRAINING;
@@ -3616,8 +3605,7 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
 	if (!is_edp(intel_dp) && !intel_dp->sink_count)
 		return false;
 
-	if (!(intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] &
-	      DP_DWN_STRM_PORT_PRESENT))
+	if (!drm_dp_is_branch(intel_dp->dpcd))
 		return true; /* native DP sink */
 
 	if (intel_dp->dpcd[DP_DPCD_REV] == 0x10)
@@ -3631,23 +3619,6 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
 	return true;
 }
 
-static void
-intel_dp_probe_oui(struct intel_dp *intel_dp)
-{
-	u8 buf[3];
-
-	if (!(intel_dp->dpcd[DP_DOWN_STREAM_PORT_COUNT] & DP_OUI_SUPPORT))
-		return;
-
-	if (drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_OUI, buf, 3) == 3)
-		DRM_DEBUG_KMS("Sink OUI: %02hx%02hx%02hx\n",
-			      buf[0], buf[1], buf[2]);
-
-	if (drm_dp_dpcd_read(&intel_dp->aux, DP_BRANCH_OUI, buf, 3) == 3)
-		DRM_DEBUG_KMS("Branch OUI: %02hx%02hx%02hx\n",
-			      buf[0], buf[1], buf[2]);
-}
-
 static bool
 intel_dp_can_mst(struct intel_dp *intel_dp)
 {
@@ -3691,7 +3662,7 @@ intel_dp_configure_mst(struct intel_dp *intel_dp)
 static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = dig_port->base.base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
 	u8 buf;
 	int ret = 0;
@@ -3712,7 +3683,7 @@ static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
 	}
 
 	do {
-		intel_wait_for_vblank(dev, intel_crtc->pipe);
+		intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
 
 		if (drm_dp_dpcd_readb(&intel_dp->aux,
 				      DP_TEST_SINK_MISC, &buf) < 0) {
@@ -3735,7 +3706,7 @@ static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
 static int intel_dp_sink_crc_start(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = dig_port->base.base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
 	u8 buf;
 	int ret;
@@ -3763,14 +3734,14 @@ static int intel_dp_sink_crc_start(struct intel_dp *intel_dp)
 		return -EIO;
 	}
 
-	intel_wait_for_vblank(dev, intel_crtc->pipe);
+	intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
 	return 0;
 }
 
 int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = dig_port->base.base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
 	u8 buf;
 	int count, ret;
@@ -3781,7 +3752,7 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc)
 		return ret;
 
 	do {
-		intel_wait_for_vblank(dev, intel_crtc->pipe);
+		intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
 
 		if (drm_dp_dpcd_readb(&intel_dp->aux,
 				      DP_TEST_SINK_MISC, &buf) < 0) {
@@ -3998,6 +3969,31 @@ go_again:
 }
 
 static void
+intel_dp_retrain_link(struct intel_dp *intel_dp)
+{
+	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+
+	/* Suppress underruns caused by re-training */
+	intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
+	if (crtc->config->has_pch_encoder)
+		intel_set_pch_fifo_underrun_reporting(dev_priv,
+						      intel_crtc_pch_transcoder(crtc), false);
+
+	intel_dp_start_link_train(intel_dp);
+	intel_dp_stop_link_train(intel_dp);
+
+	/* Keep underrun reporting disabled until things are stable */
+	intel_wait_for_vblank(dev_priv, crtc->pipe);
+
+	intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
+	if (crtc->config->has_pch_encoder)
+		intel_set_pch_fifo_underrun_reporting(dev_priv,
+						      intel_crtc_pch_transcoder(crtc), true);
+}
+
+static void
 intel_dp_check_link_status(struct intel_dp *intel_dp)
 {
 	struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base;
@@ -4017,13 +4013,18 @@ intel_dp_check_link_status(struct intel_dp *intel_dp)
 	if (!to_intel_crtc(intel_encoder->base.crtc)->active)
 		return;
 
+	/* FIXME: we need to synchronize this sort of stuff with hardware
+	 * readout */
+	if (WARN_ON_ONCE(!intel_dp->lane_count))
+		return;
+
 	/* if link training is requested we should perform it always */
 	if ((intel_dp->compliance_test_type == DP_TEST_LINK_TRAINING) ||
 	    (!drm_dp_channel_eq_ok(link_status, intel_dp->lane_count))) {
 		DRM_DEBUG_KMS("%s: channel EQ not ok, retraining\n",
 			      intel_encoder->base.name);
-		intel_dp_start_link_train(intel_dp);
-		intel_dp_stop_link_train(intel_dp);
+
+		intel_dp_retrain_link(intel_dp);
 	}
 }
 
@@ -4105,7 +4106,7 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp)
 		return connector_status_connected;
 
 	/* if there's no downstream port, we're done */
-	if (!(dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT))
+	if (!drm_dp_is_branch(dpcd))
 		return connector_status_connected;
 
 	/* If we're HPD-aware, SINK_COUNT changes dynamically */
@@ -4396,10 +4397,7 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
 
 	intel_dp_print_rates(intel_dp);
 
-	intel_dp_probe_oui(intel_dp);
-
-	intel_dp_print_hw_revision(intel_dp);
-	intel_dp_print_sw_revision(intel_dp);
+	intel_dp_read_desc(intel_dp);
 
 	intel_dp_configure_mst(intel_dp);
 
@@ -4755,11 +4753,16 @@ static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp)
 void intel_dp_encoder_reset(struct drm_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
-	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+	struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
+	struct intel_lspcon *lspcon = &intel_dig_port->lspcon;
+	struct intel_dp *intel_dp = &intel_dig_port->dp;
 
 	if (!HAS_DDI(dev_priv))
 		intel_dp->DP = I915_READ(intel_dp->output_reg);
 
+	if (IS_GEN9(dev_priv) && lspcon->active)
+		lspcon_resume(lspcon);
+
 	if (to_intel_encoder(encoder)->type != INTEL_OUTPUT_EDP)
 		return;
 
@@ -4866,15 +4869,13 @@ put_power:
 }
 
 /* check the VBT to see whether the eDP is on another port */
-bool intel_dp_is_edp(struct drm_device *dev, enum port port)
+bool intel_dp_is_edp(struct drm_i915_private *dev_priv, enum port port)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	/*
 	 * eDP not supported on g4x. so bail out early just
 	 * for a bit extra safety in case the VBT is bonkers.
 	 */
-	if (INTEL_INFO(dev)->gen < 5)
+	if (INTEL_GEN(dev_priv) < 5)
 		return false;
 
 	if (port == PORT_A)
@@ -5073,7 +5074,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 		 (seq->t10 << PANEL_POWER_DOWN_DELAY_SHIFT);
 	/* Compute the divisor for the pp clock, simply match the Bspec
 	 * formula. */
-	if (IS_BROXTON(dev)) {
+	if (IS_BROXTON(dev_priv)) {
 		pp_div = I915_READ(regs.pp_ctrl);
 		pp_div &= ~BXT_POWER_CYCLE_DELAY_MASK;
 		pp_div |= (DIV_ROUND_UP((seq->t11_t12 + 1), 1000)
@@ -5086,9 +5087,9 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 
 	/* Haswell doesn't have any port selection bits for the panel
 	 * power sequencer any more. */
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		port_sel = PANEL_PORT_SELECT_VLV(port);
-	} else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) {
+	} else if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
 		if (port == PORT_A)
 			port_sel = PANEL_PORT_SELECT_DPA;
 		else
@@ -5099,7 +5100,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 
 	I915_WRITE(regs.pp_on, pp_on);
 	I915_WRITE(regs.pp_off, pp_off);
-	if (IS_BROXTON(dev))
+	if (IS_BROXTON(dev_priv))
 		I915_WRITE(regs.pp_ctrl, pp_div);
 	else
 		I915_WRITE(regs.pp_div, pp_div);
@@ -5107,7 +5108,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 	DRM_DEBUG_KMS("panel power sequencer register settings: PP_ON %#x, PP_OFF %#x, PP_DIV %#x\n",
 		      I915_READ(regs.pp_on),
 		      I915_READ(regs.pp_off),
-		      IS_BROXTON(dev) ?
+		      IS_BROXTON(dev_priv) ?
 		      (I915_READ(regs.pp_ctrl) & BXT_POWER_CYCLE_DELAY_MASK) :
 		      I915_READ(regs.pp_div));
 }
@@ -5115,7 +5116,9 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 static void intel_dp_pps_init(struct drm_device *dev,
 			      struct intel_dp *intel_dp)
 {
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	struct drm_i915_private *dev_priv = to_i915(dev);
+
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		vlv_initial_power_sequencer_setup(intel_dp);
 	} else {
 		intel_dp_init_panel_power_sequencer(dev, intel_dp);
@@ -5474,7 +5477,7 @@ intel_dp_drrs_init(struct intel_connector *intel_connector,
 	INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_edp_drrs_downclock_work);
 	mutex_init(&dev_priv->drrs.mutex);
 
-	if (INTEL_INFO(dev)->gen <= 6) {
+	if (INTEL_GEN(dev_priv) <= 6) {
 		DRM_DEBUG_KMS("DRRS supported for Gen7 and above\n");
 		return NULL;
 	}
@@ -5585,7 +5588,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 	}
 	mutex_unlock(&dev->mode_config.mutex);
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		intel_dp->edp_notifier.notifier_call = edp_notify_handler;
 		register_reboot_notifier(&intel_dp->edp_notifier);
 
@@ -5594,7 +5597,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 		 * If the current pipe isn't valid, try the PPS pipe, and if that
 		 * fails just assume pipe A.
 		 */
-		if (IS_CHERRYVIEW(dev))
+		if (IS_CHERRYVIEW(dev_priv))
 			pipe = DP_PORT_TO_PIPE_CHV(intel_dp->DP);
 		else
 			pipe = PORT_TO_PIPE(intel_dp->DP);
@@ -5648,28 +5651,28 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 	intel_dp->pps_pipe = INVALID_PIPE;
 
 	/* intel_dp vfuncs */
-	if (INTEL_INFO(dev)->gen >= 9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		intel_dp->get_aux_clock_divider = skl_get_aux_clock_divider;
-	else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		intel_dp->get_aux_clock_divider = hsw_get_aux_clock_divider;
-	else if (HAS_PCH_SPLIT(dev))
+	else if (HAS_PCH_SPLIT(dev_priv))
 		intel_dp->get_aux_clock_divider = ilk_get_aux_clock_divider;
 	else
 		intel_dp->get_aux_clock_divider = g4x_get_aux_clock_divider;
 
-	if (INTEL_INFO(dev)->gen >= 9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		intel_dp->get_aux_send_ctl = skl_get_aux_send_ctl;
 	else
 		intel_dp->get_aux_send_ctl = g4x_get_aux_send_ctl;
 
-	if (HAS_DDI(dev))
+	if (HAS_DDI(dev_priv))
 		intel_dp->prepare_link_retrain = intel_ddi_prepare_link_retrain;
 
 	/* Preserve the current hw state. */
 	intel_dp->DP = I915_READ(intel_dp->output_reg);
 	intel_dp->attached_connector = intel_connector;
 
-	if (intel_dp_is_edp(dev, port))
+	if (intel_dp_is_edp(dev_priv, port))
 		type = DRM_MODE_CONNECTOR_eDP;
 	else
 		type = DRM_MODE_CONNECTOR_DisplayPort;
@@ -5683,7 +5686,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 		intel_encoder->type = INTEL_OUTPUT_EDP;
 
 	/* eDP only on port B and/or C on vlv/chv */
-	if (WARN_ON((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+	if (WARN_ON((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 		    is_edp(intel_dp) && port != PORT_B && port != PORT_C))
 		return false;
 
@@ -5704,7 +5707,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 
 	intel_connector_attach_encoder(intel_connector, intel_encoder);
 
-	if (HAS_DDI(dev))
+	if (HAS_DDI(dev_priv))
 		intel_connector->get_hw_state = intel_ddi_connector_get_hw_state;
 	else
 		intel_connector->get_hw_state = intel_connector_get_hw_state;
@@ -5716,7 +5719,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 		break;
 	case PORT_B:
 		intel_encoder->hpd_pin = HPD_PORT_B;
-		if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
+		if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
 			intel_encoder->hpd_pin = HPD_PORT_A;
 		break;
 	case PORT_C:
@@ -5733,7 +5736,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 	}
 
 	/* init MST on ports that can support it */
-	if (HAS_DP_MST(dev) && !is_edp(intel_dp) &&
+	if (HAS_DP_MST(dev_priv) && !is_edp(intel_dp) &&
 	    (port == PORT_B || port == PORT_C || port == PORT_D))
 		intel_dp_mst_encoder_init(intel_dig_port,
 					  intel_connector->base.base.id);
@@ -5750,7 +5753,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 	 * 0xd.  Failure to do so will result in spurious interrupts being
 	 * generated on the port when a cable is not attached.
 	 */
-	if (IS_G4X(dev) && !IS_GM45(dev)) {
+	if (IS_G4X(dev_priv) && !IS_GM45(dev_priv)) {
 		u32 temp = I915_READ(PEG_BAND_GAP_DATA);
 		I915_WRITE(PEG_BAND_GAP_DATA, (temp & ~0xf) | 0xd);
 	}
@@ -5793,13 +5796,13 @@ bool intel_dp_init(struct drm_device *dev,
 	intel_encoder->get_hw_state = intel_dp_get_hw_state;
 	intel_encoder->get_config = intel_dp_get_config;
 	intel_encoder->suspend = intel_dp_encoder_suspend;
-	if (IS_CHERRYVIEW(dev)) {
+	if (IS_CHERRYVIEW(dev_priv)) {
 		intel_encoder->pre_pll_enable = chv_dp_pre_pll_enable;
 		intel_encoder->pre_enable = chv_pre_enable_dp;
 		intel_encoder->enable = vlv_enable_dp;
 		intel_encoder->post_disable = chv_post_disable_dp;
 		intel_encoder->post_pll_disable = chv_dp_post_pll_disable;
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv)) {
 		intel_encoder->pre_pll_enable = vlv_dp_pre_pll_enable;
 		intel_encoder->pre_enable = vlv_pre_enable_dp;
 		intel_encoder->enable = vlv_enable_dp;
@@ -5807,7 +5810,7 @@ bool intel_dp_init(struct drm_device *dev,
 	} else {
 		intel_encoder->pre_enable = g4x_pre_enable_dp;
 		intel_encoder->enable = g4x_enable_dp;
-		if (INTEL_INFO(dev)->gen >= 5)
+		if (INTEL_GEN(dev_priv) >= 5)
 			intel_encoder->post_disable = ilk_post_disable_dp;
 	}
 
@@ -5816,7 +5819,7 @@ bool intel_dp_init(struct drm_device *dev,
 	intel_dig_port->max_lanes = 4;
 
 	intel_encoder->type = INTEL_OUTPUT_DP;
-	if (IS_CHERRYVIEW(dev)) {
+	if (IS_CHERRYVIEW(dev_priv)) {
 		if (port == PORT_D)
 			intel_encoder->crtc_mask = 1 << 2;
 		else
@@ -5825,6 +5828,7 @@ bool intel_dp_init(struct drm_device *dev,
 		intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
 	}
 	intel_encoder->cloneable = 0;
+	intel_encoder->port = port;
 
 	intel_dig_port->hpd_pulse = intel_dp_hpd_pulse;
 	dev_priv->hotplug.irq_port[port] = intel_dig_port;
diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c
index c438b02184cb..0048b520baf7 100644
--- a/drivers/gpu/drm/i915/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/intel_dp_link_training.c
@@ -225,9 +225,6 @@ static u32 intel_dp_training_pattern(struct intel_dp *intel_dp)
 	 * Intel platforms that support HBR2 also support TPS3. TPS3 support is
 	 * also mandatory for downstream devices that support HBR2. However, not
 	 * all sinks follow the spec.
-	 *
-	 * Due to WaDisableHBR2 SKL < B0 is the only exception where TPS3 is
-	 * supported in source but still not enabled.
 	 */
 	source_tps3 = intel_dp_source_supports_hbr2(intel_dp);
 	sink_tps3 = drm_dp_tps3_supported(intel_dp->dpcd);
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index 54a9d7610d8f..b029d1026a28 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -43,7 +43,6 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
 	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 	int mst_pbn;
 
-	pipe_config->dp_encoder_is_mst = true;
 	pipe_config->has_pch_encoder = false;
 	bpp = 24;
 	/*
@@ -523,6 +522,7 @@ intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum
 			 DRM_MODE_ENCODER_DPMST, "DP-MST %c", pipe_name(pipe));
 
 	intel_encoder->type = INTEL_OUTPUT_DP_MST;
+	intel_encoder->port = intel_dig_port->port;
 	intel_encoder->crtc_mask = 0x7;
 	intel_encoder->cloneable = 0;
 
diff --git a/drivers/gpu/drm/i915/intel_dpio_phy.c b/drivers/gpu/drm/i915/intel_dpio_phy.c
index 047f48748944..7a8e82dabbf2 100644
--- a/drivers/gpu/drm/i915/intel_dpio_phy.c
+++ b/drivers/gpu/drm/i915/intel_dpio_phy.c
@@ -23,6 +23,565 @@
 
 #include "intel_drv.h"
 
+/**
+ * DOC: DPIO
+ *
+ * VLV, CHV and BXT have slightly peculiar display PHYs for driving DP/HDMI
+ * ports. DPIO is the name given to such a display PHY. These PHYs
+ * don't follow the standard programming model using direct MMIO
+ * registers, and instead their registers must be accessed trough IOSF
+ * sideband. VLV has one such PHY for driving ports B and C, and CHV
+ * adds another PHY for driving port D. Each PHY responds to specific
+ * IOSF-SB port.
+ *
+ * Each display PHY is made up of one or two channels. Each channel
+ * houses a common lane part which contains the PLL and other common
+ * logic. CH0 common lane also contains the IOSF-SB logic for the
+ * Common Register Interface (CRI) ie. the DPIO registers. CRI clock
+ * must be running when any DPIO registers are accessed.
+ *
+ * In addition to having their own registers, the PHYs are also
+ * controlled through some dedicated signals from the display
+ * controller. These include PLL reference clock enable, PLL enable,
+ * and CRI clock selection, for example.
+ *
+ * Eeach channel also has two splines (also called data lanes), and
+ * each spline is made up of one Physical Access Coding Sub-Layer
+ * (PCS) block and two TX lanes. So each channel has two PCS blocks
+ * and four TX lanes. The TX lanes are used as DP lanes or TMDS
+ * data/clock pairs depending on the output type.
+ *
+ * Additionally the PHY also contains an AUX lane with AUX blocks
+ * for each channel. This is used for DP AUX communication, but
+ * this fact isn't really relevant for the driver since AUX is
+ * controlled from the display controller side. No DPIO registers
+ * need to be accessed during AUX communication,
+ *
+ * Generally on VLV/CHV the common lane corresponds to the pipe and
+ * the spline (PCS/TX) corresponds to the port.
+ *
+ * For dual channel PHY (VLV/CHV):
+ *
+ *  pipe A == CMN/PLL/REF CH0
+ *
+ *  pipe B == CMN/PLL/REF CH1
+ *
+ *  port B == PCS/TX CH0
+ *
+ *  port C == PCS/TX CH1
+ *
+ * This is especially important when we cross the streams
+ * ie. drive port B with pipe B, or port C with pipe A.
+ *
+ * For single channel PHY (CHV):
+ *
+ *  pipe C == CMN/PLL/REF CH0
+ *
+ *  port D == PCS/TX CH0
+ *
+ * On BXT the entire PHY channel corresponds to the port. That means
+ * the PLL is also now associated with the port rather than the pipe,
+ * and so the clock needs to be routed to the appropriate transcoder.
+ * Port A PLL is directly connected to transcoder EDP and port B/C
+ * PLLs can be routed to any transcoder A/B/C.
+ *
+ * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is
+ * digital port D (CHV) or port A (BXT). ::
+ *
+ *
+ *     Dual channel PHY (VLV/CHV/BXT)
+ *     ---------------------------------
+ *     |      CH0      |      CH1      |
+ *     |  CMN/PLL/REF  |  CMN/PLL/REF  |
+ *     |---------------|---------------| Display PHY
+ *     | PCS01 | PCS23 | PCS01 | PCS23 |
+ *     |-------|-------|-------|-------|
+ *     |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3|
+ *     ---------------------------------
+ *     |     DDI0      |     DDI1      | DP/HDMI ports
+ *     ---------------------------------
+ *
+ *     Single channel PHY (CHV/BXT)
+ *     -----------------
+ *     |      CH0      |
+ *     |  CMN/PLL/REF  |
+ *     |---------------| Display PHY
+ *     | PCS01 | PCS23 |
+ *     |-------|-------|
+ *     |TX0|TX1|TX2|TX3|
+ *     -----------------
+ *     |     DDI2      | DP/HDMI port
+ *     -----------------
+ */
+
+/**
+ * struct bxt_ddi_phy_info - Hold info for a broxton DDI phy
+ */
+struct bxt_ddi_phy_info {
+	/**
+	 * @dual_channel: true if this phy has a second channel.
+	 */
+	bool dual_channel;
+
+	/**
+	 * @rcomp_phy: If -1, indicates this phy has its own rcomp resistor.
+	 * Otherwise the GRC value will be copied from the phy indicated by
+	 * this field.
+	 */
+	enum dpio_phy rcomp_phy;
+
+	/**
+	 * @channel: struct containing per channel information.
+	 */
+	struct {
+		/**
+		 * @port: which port maps to this channel.
+		 */
+		enum port port;
+	} channel[2];
+};
+
+static const struct bxt_ddi_phy_info bxt_ddi_phy_info[] = {
+	[DPIO_PHY0] = {
+		.dual_channel = true,
+		.rcomp_phy = DPIO_PHY1,
+
+		.channel = {
+			[DPIO_CH0] = { .port = PORT_B },
+			[DPIO_CH1] = { .port = PORT_C },
+		}
+	},
+	[DPIO_PHY1] = {
+		.dual_channel = false,
+		.rcomp_phy = -1,
+
+		.channel = {
+			[DPIO_CH0] = { .port = PORT_A },
+		}
+	},
+};
+
+static u32 bxt_phy_port_mask(const struct bxt_ddi_phy_info *phy_info)
+{
+	return (phy_info->dual_channel * BIT(phy_info->channel[DPIO_CH1].port)) |
+		BIT(phy_info->channel[DPIO_CH0].port);
+}
+
+void bxt_port_to_phy_channel(enum port port,
+			     enum dpio_phy *phy, enum dpio_channel *ch)
+{
+	const struct bxt_ddi_phy_info *phy_info;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(bxt_ddi_phy_info); i++) {
+		phy_info = &bxt_ddi_phy_info[i];
+
+		if (port == phy_info->channel[DPIO_CH0].port) {
+			*phy = i;
+			*ch = DPIO_CH0;
+			return;
+		}
+
+		if (phy_info->dual_channel &&
+		    port == phy_info->channel[DPIO_CH1].port) {
+			*phy = i;
+			*ch = DPIO_CH1;
+			return;
+		}
+	}
+
+	WARN(1, "PHY not found for PORT %c", port_name(port));
+	*phy = DPIO_PHY0;
+	*ch = DPIO_CH0;
+}
+
+void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv,
+				  enum port port, u32 margin, u32 scale,
+				  u32 enable, u32 deemphasis)
+{
+	u32 val;
+	enum dpio_phy phy;
+	enum dpio_channel ch;
+
+	bxt_port_to_phy_channel(port, &phy, &ch);
+
+	/*
+	 * While we write to the group register to program all lanes at once we
+	 * can read only lane registers and we pick lanes 0/1 for that.
+	 */
+	val = I915_READ(BXT_PORT_PCS_DW10_LN01(phy, ch));
+	val &= ~(TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT);
+	I915_WRITE(BXT_PORT_PCS_DW10_GRP(phy, ch), val);
+
+	val = I915_READ(BXT_PORT_TX_DW2_LN0(phy, ch));
+	val &= ~(MARGIN_000 | UNIQ_TRANS_SCALE);
+	val |= margin << MARGIN_000_SHIFT | scale << UNIQ_TRANS_SCALE_SHIFT;
+	I915_WRITE(BXT_PORT_TX_DW2_GRP(phy, ch), val);
+
+	val = I915_READ(BXT_PORT_TX_DW3_LN0(phy, ch));
+	val &= ~SCALE_DCOMP_METHOD;
+	if (enable)
+		val |= SCALE_DCOMP_METHOD;
+
+	if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD))
+		DRM_ERROR("Disabled scaling while ouniqetrangenmethod was set");
+
+	I915_WRITE(BXT_PORT_TX_DW3_GRP(phy, ch), val);
+
+	val = I915_READ(BXT_PORT_TX_DW4_LN0(phy, ch));
+	val &= ~DE_EMPHASIS;
+	val |= deemphasis << DEEMPH_SHIFT;
+	I915_WRITE(BXT_PORT_TX_DW4_GRP(phy, ch), val);
+
+	val = I915_READ(BXT_PORT_PCS_DW10_LN01(phy, ch));
+	val |= TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT;
+	I915_WRITE(BXT_PORT_PCS_DW10_GRP(phy, ch), val);
+}
+
+bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv,
+			    enum dpio_phy phy)
+{
+	const struct bxt_ddi_phy_info *phy_info = &bxt_ddi_phy_info[phy];
+	enum port port;
+
+	if (!(I915_READ(BXT_P_CR_GT_DISP_PWRON) & GT_DISPLAY_POWER_ON(phy)))
+		return false;
+
+	if ((I915_READ(BXT_PORT_CL1CM_DW0(phy)) &
+	     (PHY_POWER_GOOD | PHY_RESERVED)) != PHY_POWER_GOOD) {
+		DRM_DEBUG_DRIVER("DDI PHY %d powered, but power hasn't settled\n",
+				 phy);
+
+		return false;
+	}
+
+	if (phy_info->rcomp_phy == -1 &&
+	    !(I915_READ(BXT_PORT_REF_DW3(phy)) & GRC_DONE)) {
+		DRM_DEBUG_DRIVER("DDI PHY %d powered, but GRC isn't done\n",
+				 phy);
+
+		return false;
+	}
+
+	if (!(I915_READ(BXT_PHY_CTL_FAMILY(phy)) & COMMON_RESET_DIS)) {
+		DRM_DEBUG_DRIVER("DDI PHY %d powered, but still in reset\n",
+				 phy);
+
+		return false;
+	}
+
+	for_each_port_masked(port, bxt_phy_port_mask(phy_info)) {
+		u32 tmp = I915_READ(BXT_PHY_CTL(port));
+
+		if (tmp & BXT_PHY_CMNLANE_POWERDOWN_ACK) {
+			DRM_DEBUG_DRIVER("DDI PHY %d powered, but common lane "
+					 "for port %c powered down "
+					 "(PHY_CTL %08x)\n",
+					 phy, port_name(port), tmp);
+
+			return false;
+		}
+	}
+
+	return true;
+}
+
+static u32 bxt_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy)
+{
+	u32 val = I915_READ(BXT_PORT_REF_DW6(phy));
+
+	return (val & GRC_CODE_MASK) >> GRC_CODE_SHIFT;
+}
+
+static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv,
+				  enum dpio_phy phy)
+{
+	if (intel_wait_for_register(dev_priv,
+				    BXT_PORT_REF_DW3(phy),
+				    GRC_DONE, GRC_DONE,
+				    10))
+		DRM_ERROR("timeout waiting for PHY%d GRC\n", phy);
+}
+
+static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv,
+			      enum dpio_phy phy)
+{
+	const struct bxt_ddi_phy_info *phy_info = &bxt_ddi_phy_info[phy];
+	u32 val;
+
+	if (bxt_ddi_phy_is_enabled(dev_priv, phy)) {
+		/* Still read out the GRC value for state verification */
+		if (phy_info->rcomp_phy != -1)
+			dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, phy);
+
+		if (bxt_ddi_phy_verify_state(dev_priv, phy)) {
+			DRM_DEBUG_DRIVER("DDI PHY %d already enabled, "
+					 "won't reprogram it\n", phy);
+
+			return;
+		}
+
+		DRM_DEBUG_DRIVER("DDI PHY %d enabled with invalid state, "
+				 "force reprogramming it\n", phy);
+	}
+
+	val = I915_READ(BXT_P_CR_GT_DISP_PWRON);
+	val |= GT_DISPLAY_POWER_ON(phy);
+	I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val);
+
+	/*
+	 * The PHY registers start out inaccessible and respond to reads with
+	 * all 1s.  Eventually they become accessible as they power up, then
+	 * the reserved bit will give the default 0.  Poll on the reserved bit
+	 * becoming 0 to find when the PHY is accessible.
+	 * HW team confirmed that the time to reach phypowergood status is
+	 * anywhere between 50 us and 100us.
+	 */
+	if (wait_for_us(((I915_READ(BXT_PORT_CL1CM_DW0(phy)) &
+		(PHY_RESERVED | PHY_POWER_GOOD)) == PHY_POWER_GOOD), 100)) {
+		DRM_ERROR("timeout during PHY%d power on\n", phy);
+	}
+
+	/* Program PLL Rcomp code offset */
+	val = I915_READ(BXT_PORT_CL1CM_DW9(phy));
+	val &= ~IREF0RC_OFFSET_MASK;
+	val |= 0xE4 << IREF0RC_OFFSET_SHIFT;
+	I915_WRITE(BXT_PORT_CL1CM_DW9(phy), val);
+
+	val = I915_READ(BXT_PORT_CL1CM_DW10(phy));
+	val &= ~IREF1RC_OFFSET_MASK;
+	val |= 0xE4 << IREF1RC_OFFSET_SHIFT;
+	I915_WRITE(BXT_PORT_CL1CM_DW10(phy), val);
+
+	/* Program power gating */
+	val = I915_READ(BXT_PORT_CL1CM_DW28(phy));
+	val |= OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN |
+		SUS_CLK_CONFIG;
+	I915_WRITE(BXT_PORT_CL1CM_DW28(phy), val);
+
+	if (phy_info->dual_channel) {
+		val = I915_READ(BXT_PORT_CL2CM_DW6(phy));
+		val |= DW6_OLDO_DYN_PWR_DOWN_EN;
+		I915_WRITE(BXT_PORT_CL2CM_DW6(phy), val);
+	}
+
+	if (phy_info->rcomp_phy != -1) {
+		uint32_t grc_code;
+		/*
+		 * PHY0 isn't connected to an RCOMP resistor so copy over
+		 * the corresponding calibrated value from PHY1, and disable
+		 * the automatic calibration on PHY0.
+		 */
+		val = dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv,
+							  phy_info->rcomp_phy);
+		grc_code = val << GRC_CODE_FAST_SHIFT |
+			   val << GRC_CODE_SLOW_SHIFT |
+			   val;
+		I915_WRITE(BXT_PORT_REF_DW6(phy), grc_code);
+
+		val = I915_READ(BXT_PORT_REF_DW8(phy));
+		val |= GRC_DIS | GRC_RDY_OVRD;
+		I915_WRITE(BXT_PORT_REF_DW8(phy), val);
+	}
+
+	val = I915_READ(BXT_PHY_CTL_FAMILY(phy));
+	val |= COMMON_RESET_DIS;
+	I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val);
+
+	if (phy_info->rcomp_phy == -1)
+		bxt_phy_wait_grc_done(dev_priv, phy);
+
+}
+
+void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy)
+{
+	uint32_t val;
+
+	val = I915_READ(BXT_PHY_CTL_FAMILY(phy));
+	val &= ~COMMON_RESET_DIS;
+	I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val);
+
+	val = I915_READ(BXT_P_CR_GT_DISP_PWRON);
+	val &= ~GT_DISPLAY_POWER_ON(phy);
+	I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val);
+}
+
+void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy)
+{
+	const struct bxt_ddi_phy_info *phy_info = &bxt_ddi_phy_info[phy];
+	enum dpio_phy rcomp_phy = phy_info->rcomp_phy;
+	bool was_enabled;
+
+	lockdep_assert_held(&dev_priv->power_domains.lock);
+
+	if (rcomp_phy != -1) {
+		was_enabled = bxt_ddi_phy_is_enabled(dev_priv, rcomp_phy);
+
+		/*
+		 * We need to copy the GRC calibration value from rcomp_phy,
+		 * so make sure it's powered up.
+		 */
+		if (!was_enabled)
+			_bxt_ddi_phy_init(dev_priv, rcomp_phy);
+	}
+
+	_bxt_ddi_phy_init(dev_priv, phy);
+
+	if (rcomp_phy != -1 && !was_enabled)
+		bxt_ddi_phy_uninit(dev_priv, phy_info->rcomp_phy);
+}
+
+static bool __printf(6, 7)
+__phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy,
+		       i915_reg_t reg, u32 mask, u32 expected,
+		       const char *reg_fmt, ...)
+{
+	struct va_format vaf;
+	va_list args;
+	u32 val;
+
+	val = I915_READ(reg);
+	if ((val & mask) == expected)
+		return true;
+
+	va_start(args, reg_fmt);
+	vaf.fmt = reg_fmt;
+	vaf.va = &args;
+
+	DRM_DEBUG_DRIVER("DDI PHY %d reg %pV [%08x] state mismatch: "
+			 "current %08x, expected %08x (mask %08x)\n",
+			 phy, &vaf, reg.reg, val, (val & ~mask) | expected,
+			 mask);
+
+	va_end(args);
+
+	return false;
+}
+
+bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
+			      enum dpio_phy phy)
+{
+	const struct bxt_ddi_phy_info *phy_info = &bxt_ddi_phy_info[phy];
+	uint32_t mask;
+	bool ok;
+
+#define _CHK(reg, mask, exp, fmt, ...)					\
+	__phy_reg_verify_state(dev_priv, phy, reg, mask, exp, fmt,	\
+			       ## __VA_ARGS__)
+
+	if (!bxt_ddi_phy_is_enabled(dev_priv, phy))
+		return false;
+
+	ok = true;
+
+	/* PLL Rcomp code offset */
+	ok &= _CHK(BXT_PORT_CL1CM_DW9(phy),
+		    IREF0RC_OFFSET_MASK, 0xe4 << IREF0RC_OFFSET_SHIFT,
+		    "BXT_PORT_CL1CM_DW9(%d)", phy);
+	ok &= _CHK(BXT_PORT_CL1CM_DW10(phy),
+		    IREF1RC_OFFSET_MASK, 0xe4 << IREF1RC_OFFSET_SHIFT,
+		    "BXT_PORT_CL1CM_DW10(%d)", phy);
+
+	/* Power gating */
+	mask = OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | SUS_CLK_CONFIG;
+	ok &= _CHK(BXT_PORT_CL1CM_DW28(phy), mask, mask,
+		    "BXT_PORT_CL1CM_DW28(%d)", phy);
+
+	if (phy_info->dual_channel)
+		ok &= _CHK(BXT_PORT_CL2CM_DW6(phy),
+			   DW6_OLDO_DYN_PWR_DOWN_EN, DW6_OLDO_DYN_PWR_DOWN_EN,
+			   "BXT_PORT_CL2CM_DW6(%d)", phy);
+
+	if (phy_info->rcomp_phy != -1) {
+		u32 grc_code = dev_priv->bxt_phy_grc;
+
+		grc_code = grc_code << GRC_CODE_FAST_SHIFT |
+			   grc_code << GRC_CODE_SLOW_SHIFT |
+			   grc_code;
+		mask = GRC_CODE_FAST_MASK | GRC_CODE_SLOW_MASK |
+		       GRC_CODE_NOM_MASK;
+		ok &= _CHK(BXT_PORT_REF_DW6(phy), mask, grc_code,
+			   "BXT_PORT_REF_DW6(%d)", phy);
+
+		mask = GRC_DIS | GRC_RDY_OVRD;
+		ok &= _CHK(BXT_PORT_REF_DW8(phy), mask, mask,
+			    "BXT_PORT_REF_DW8(%d)", phy);
+	}
+
+	return ok;
+#undef _CHK
+}
+
+uint8_t
+bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder,
+				     uint8_t lane_count)
+{
+	switch (lane_count) {
+	case 1:
+		return 0;
+	case 2:
+		return BIT(2) | BIT(0);
+	case 4:
+		return BIT(3) | BIT(2) | BIT(0);
+	default:
+		MISSING_CASE(lane_count);
+
+		return 0;
+	}
+}
+
+void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder,
+				     uint8_t lane_lat_optim_mask)
+{
+	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+	struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
+	enum port port = dport->port;
+	enum dpio_phy phy;
+	enum dpio_channel ch;
+	int lane;
+
+	bxt_port_to_phy_channel(port, &phy, &ch);
+
+	for (lane = 0; lane < 4; lane++) {
+		u32 val = I915_READ(BXT_PORT_TX_DW14_LN(phy, ch, lane));
+
+		/*
+		 * Note that on CHV this flag is called UPAR, but has
+		 * the same function.
+		 */
+		val &= ~LATENCY_OPTIM;
+		if (lane_lat_optim_mask & BIT(lane))
+			val |= LATENCY_OPTIM;
+
+		I915_WRITE(BXT_PORT_TX_DW14_LN(phy, ch, lane), val);
+	}
+}
+
+uint8_t
+bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder)
+{
+	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+	struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
+	enum port port = dport->port;
+	enum dpio_phy phy;
+	enum dpio_channel ch;
+	int lane;
+	uint8_t mask;
+
+	bxt_port_to_phy_channel(port, &phy, &ch);
+
+	mask = 0;
+	for (lane = 0; lane < 4; lane++) {
+		u32 val = I915_READ(BXT_PORT_TX_DW14_LN(phy, ch, lane));
+
+		if (val & LATENCY_OPTIM)
+			mask |= BIT(lane);
+	}
+
+	return mask;
+}
+
+
 void chv_set_phy_signal_level(struct intel_encoder *encoder,
 			      u32 deemph_reg_value, u32 margin_reg_value,
 			      bool uniq_trans_scale)
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index 1c59ca50c430..58a756f2f224 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -188,13 +188,12 @@ out:
 
 void intel_disable_shared_dpll(struct intel_crtc *crtc)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct intel_shared_dpll *pll = crtc->config->shared_dpll;
 	unsigned crtc_mask = 1 << drm_crtc_index(&crtc->base);
 
 	/* PCH only available on ILK+ */
-	if (INTEL_INFO(dev)->gen < 5)
+	if (INTEL_GEN(dev_priv) < 5)
 		return;
 
 	if (pll == NULL)
@@ -1371,6 +1370,10 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv,
 {
 	uint32_t temp;
 	enum port port = (enum port)pll->id;	/* 1:1 port->PLL mapping */
+	enum dpio_phy phy;
+	enum dpio_channel ch;
+
+	bxt_port_to_phy_channel(port, &phy, &ch);
 
 	/* Non-SSC reference */
 	temp = I915_READ(BXT_PORT_PLL_ENABLE(port));
@@ -1378,72 +1381,72 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	I915_WRITE(BXT_PORT_PLL_ENABLE(port), temp);
 
 	/* Disable 10 bit clock */
-	temp = I915_READ(BXT_PORT_PLL_EBB_4(port));
+	temp = I915_READ(BXT_PORT_PLL_EBB_4(phy, ch));
 	temp &= ~PORT_PLL_10BIT_CLK_ENABLE;
-	I915_WRITE(BXT_PORT_PLL_EBB_4(port), temp);
+	I915_WRITE(BXT_PORT_PLL_EBB_4(phy, ch), temp);
 
 	/* Write P1 & P2 */
-	temp = I915_READ(BXT_PORT_PLL_EBB_0(port));
+	temp = I915_READ(BXT_PORT_PLL_EBB_0(phy, ch));
 	temp &= ~(PORT_PLL_P1_MASK | PORT_PLL_P2_MASK);
 	temp |= pll->config.hw_state.ebb0;
-	I915_WRITE(BXT_PORT_PLL_EBB_0(port), temp);
+	I915_WRITE(BXT_PORT_PLL_EBB_0(phy, ch), temp);
 
 	/* Write M2 integer */
-	temp = I915_READ(BXT_PORT_PLL(port, 0));
+	temp = I915_READ(BXT_PORT_PLL(phy, ch, 0));
 	temp &= ~PORT_PLL_M2_MASK;
 	temp |= pll->config.hw_state.pll0;
-	I915_WRITE(BXT_PORT_PLL(port, 0), temp);
+	I915_WRITE(BXT_PORT_PLL(phy, ch, 0), temp);
 
 	/* Write N */
-	temp = I915_READ(BXT_PORT_PLL(port, 1));
+	temp = I915_READ(BXT_PORT_PLL(phy, ch, 1));
 	temp &= ~PORT_PLL_N_MASK;
 	temp |= pll->config.hw_state.pll1;
-	I915_WRITE(BXT_PORT_PLL(port, 1), temp);
+	I915_WRITE(BXT_PORT_PLL(phy, ch, 1), temp);
 
 	/* Write M2 fraction */
-	temp = I915_READ(BXT_PORT_PLL(port, 2));
+	temp = I915_READ(BXT_PORT_PLL(phy, ch, 2));
 	temp &= ~PORT_PLL_M2_FRAC_MASK;
 	temp |= pll->config.hw_state.pll2;
-	I915_WRITE(BXT_PORT_PLL(port, 2), temp);
+	I915_WRITE(BXT_PORT_PLL(phy, ch, 2), temp);
 
 	/* Write M2 fraction enable */
-	temp = I915_READ(BXT_PORT_PLL(port, 3));
+	temp = I915_READ(BXT_PORT_PLL(phy, ch, 3));
 	temp &= ~PORT_PLL_M2_FRAC_ENABLE;
 	temp |= pll->config.hw_state.pll3;
-	I915_WRITE(BXT_PORT_PLL(port, 3), temp);
+	I915_WRITE(BXT_PORT_PLL(phy, ch, 3), temp);
 
 	/* Write coeff */
-	temp = I915_READ(BXT_PORT_PLL(port, 6));
+	temp = I915_READ(BXT_PORT_PLL(phy, ch, 6));
 	temp &= ~PORT_PLL_PROP_COEFF_MASK;
 	temp &= ~PORT_PLL_INT_COEFF_MASK;
 	temp &= ~PORT_PLL_GAIN_CTL_MASK;
 	temp |= pll->config.hw_state.pll6;
-	I915_WRITE(BXT_PORT_PLL(port, 6), temp);
+	I915_WRITE(BXT_PORT_PLL(phy, ch, 6), temp);
 
 	/* Write calibration val */
-	temp = I915_READ(BXT_PORT_PLL(port, 8));
+	temp = I915_READ(BXT_PORT_PLL(phy, ch, 8));
 	temp &= ~PORT_PLL_TARGET_CNT_MASK;
 	temp |= pll->config.hw_state.pll8;
-	I915_WRITE(BXT_PORT_PLL(port, 8), temp);
+	I915_WRITE(BXT_PORT_PLL(phy, ch, 8), temp);
 
-	temp = I915_READ(BXT_PORT_PLL(port, 9));
+	temp = I915_READ(BXT_PORT_PLL(phy, ch, 9));
 	temp &= ~PORT_PLL_LOCK_THRESHOLD_MASK;
 	temp |= pll->config.hw_state.pll9;
-	I915_WRITE(BXT_PORT_PLL(port, 9), temp);
+	I915_WRITE(BXT_PORT_PLL(phy, ch, 9), temp);
 
-	temp = I915_READ(BXT_PORT_PLL(port, 10));
+	temp = I915_READ(BXT_PORT_PLL(phy, ch, 10));
 	temp &= ~PORT_PLL_DCO_AMP_OVR_EN_H;
 	temp &= ~PORT_PLL_DCO_AMP_MASK;
 	temp |= pll->config.hw_state.pll10;
-	I915_WRITE(BXT_PORT_PLL(port, 10), temp);
+	I915_WRITE(BXT_PORT_PLL(phy, ch, 10), temp);
 
 	/* Recalibrate with new settings */
-	temp = I915_READ(BXT_PORT_PLL_EBB_4(port));
+	temp = I915_READ(BXT_PORT_PLL_EBB_4(phy, ch));
 	temp |= PORT_PLL_RECALIBRATE;
-	I915_WRITE(BXT_PORT_PLL_EBB_4(port), temp);
+	I915_WRITE(BXT_PORT_PLL_EBB_4(phy, ch), temp);
 	temp &= ~PORT_PLL_10BIT_CLK_ENABLE;
 	temp |= pll->config.hw_state.ebb4;
-	I915_WRITE(BXT_PORT_PLL_EBB_4(port), temp);
+	I915_WRITE(BXT_PORT_PLL_EBB_4(phy, ch), temp);
 
 	/* Enable PLL */
 	temp = I915_READ(BXT_PORT_PLL_ENABLE(port));
@@ -1459,11 +1462,11 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	 * While we write to the group register to program all lanes at once we
 	 * can read only lane registers and we pick lanes 0/1 for that.
 	 */
-	temp = I915_READ(BXT_PORT_PCS_DW12_LN01(port));
+	temp = I915_READ(BXT_PORT_PCS_DW12_LN01(phy, ch));
 	temp &= ~LANE_STAGGER_MASK;
 	temp &= ~LANESTAGGER_STRAP_OVRD;
 	temp |= pll->config.hw_state.pcsdw12;
-	I915_WRITE(BXT_PORT_PCS_DW12_GRP(port), temp);
+	I915_WRITE(BXT_PORT_PCS_DW12_GRP(phy, ch), temp);
 }
 
 static void bxt_ddi_pll_disable(struct drm_i915_private *dev_priv,
@@ -1485,6 +1488,10 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv,
 	enum port port = (enum port)pll->id;	/* 1:1 port->PLL mapping */
 	uint32_t val;
 	bool ret;
+	enum dpio_phy phy;
+	enum dpio_channel ch;
+
+	bxt_port_to_phy_channel(port, &phy, &ch);
 
 	if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS))
 		return false;
@@ -1495,36 +1502,36 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv,
 	if (!(val & PORT_PLL_ENABLE))
 		goto out;
 
-	hw_state->ebb0 = I915_READ(BXT_PORT_PLL_EBB_0(port));
+	hw_state->ebb0 = I915_READ(BXT_PORT_PLL_EBB_0(phy, ch));
 	hw_state->ebb0 &= PORT_PLL_P1_MASK | PORT_PLL_P2_MASK;
 
-	hw_state->ebb4 = I915_READ(BXT_PORT_PLL_EBB_4(port));
+	hw_state->ebb4 = I915_READ(BXT_PORT_PLL_EBB_4(phy, ch));
 	hw_state->ebb4 &= PORT_PLL_10BIT_CLK_ENABLE;
 
-	hw_state->pll0 = I915_READ(BXT_PORT_PLL(port, 0));
+	hw_state->pll0 = I915_READ(BXT_PORT_PLL(phy, ch, 0));
 	hw_state->pll0 &= PORT_PLL_M2_MASK;
 
-	hw_state->pll1 = I915_READ(BXT_PORT_PLL(port, 1));
+	hw_state->pll1 = I915_READ(BXT_PORT_PLL(phy, ch, 1));
 	hw_state->pll1 &= PORT_PLL_N_MASK;
 
-	hw_state->pll2 = I915_READ(BXT_PORT_PLL(port, 2));
+	hw_state->pll2 = I915_READ(BXT_PORT_PLL(phy, ch, 2));
 	hw_state->pll2 &= PORT_PLL_M2_FRAC_MASK;
 
-	hw_state->pll3 = I915_READ(BXT_PORT_PLL(port, 3));
+	hw_state->pll3 = I915_READ(BXT_PORT_PLL(phy, ch, 3));
 	hw_state->pll3 &= PORT_PLL_M2_FRAC_ENABLE;
 
-	hw_state->pll6 = I915_READ(BXT_PORT_PLL(port, 6));
+	hw_state->pll6 = I915_READ(BXT_PORT_PLL(phy, ch, 6));
 	hw_state->pll6 &= PORT_PLL_PROP_COEFF_MASK |
 			  PORT_PLL_INT_COEFF_MASK |
 			  PORT_PLL_GAIN_CTL_MASK;
 
-	hw_state->pll8 = I915_READ(BXT_PORT_PLL(port, 8));
+	hw_state->pll8 = I915_READ(BXT_PORT_PLL(phy, ch, 8));
 	hw_state->pll8 &= PORT_PLL_TARGET_CNT_MASK;
 
-	hw_state->pll9 = I915_READ(BXT_PORT_PLL(port, 9));
+	hw_state->pll9 = I915_READ(BXT_PORT_PLL(phy, ch, 9));
 	hw_state->pll9 &= PORT_PLL_LOCK_THRESHOLD_MASK;
 
-	hw_state->pll10 = I915_READ(BXT_PORT_PLL(port, 10));
+	hw_state->pll10 = I915_READ(BXT_PORT_PLL(phy, ch, 10));
 	hw_state->pll10 &= PORT_PLL_DCO_AMP_OVR_EN_H |
 			   PORT_PLL_DCO_AMP_MASK;
 
@@ -1533,11 +1540,11 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv,
 	 * can read only lane registers. We configure all lanes the same way, so
 	 * here just read out lanes 0/1 and output a note if lanes 2/3 differ.
 	 */
-	hw_state->pcsdw12 = I915_READ(BXT_PORT_PCS_DW12_LN01(port));
-	if (I915_READ(BXT_PORT_PCS_DW12_LN23(port)) != hw_state->pcsdw12)
+	hw_state->pcsdw12 = I915_READ(BXT_PORT_PCS_DW12_LN01(phy, ch));
+	if (I915_READ(BXT_PORT_PCS_DW12_LN23(phy, ch)) != hw_state->pcsdw12)
 		DRM_DEBUG_DRIVER("lane stagger config different for lane 01 (%08x) and 23 (%08x)\n",
 				 hw_state->pcsdw12,
-				 I915_READ(BXT_PORT_PCS_DW12_LN23(port)));
+				 I915_READ(BXT_PORT_PCS_DW12_LN23(phy, ch)));
 	hw_state->pcsdw12 &= LANE_STAGGER_MASK | LANESTAGGER_STRAP_OVRD;
 
 	ret = true;
@@ -1851,13 +1858,13 @@ void intel_shared_dpll_init(struct drm_device *dev)
 	const struct dpll_info *dpll_info;
 	int i;
 
-	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
 		dpll_mgr = &skl_pll_mgr;
-	else if (IS_BROXTON(dev))
+	else if (IS_BROXTON(dev_priv))
 		dpll_mgr = &bxt_pll_mgr;
-	else if (HAS_DDI(dev))
+	else if (HAS_DDI(dev_priv))
 		dpll_mgr = &hsw_pll_mgr;
-	else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
+	else if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
 		dpll_mgr = &pch_pll_mgr;
 
 	if (!dpll_mgr) {
@@ -1883,7 +1890,7 @@ void intel_shared_dpll_init(struct drm_device *dev)
 	BUG_ON(dev_priv->num_shared_dpll > I915_NUM_PLLS);
 
 	/* FIXME: Move this to a more suitable place */
-	if (HAS_DDI(dev))
+	if (HAS_DDI(dev_priv))
 		intel_ddi_pll_init(dev);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index a19ec06f9e42..cd132c216a67 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -206,6 +206,7 @@ struct intel_encoder {
 	struct drm_encoder base;
 
 	enum intel_output_type type;
+	enum port port;
 	unsigned int cloneable;
 	void (*hot_plug)(struct intel_encoder *);
 	bool (*compute_config)(struct intel_encoder *,
@@ -247,6 +248,8 @@ struct intel_encoder {
 	void (*suspend)(struct intel_encoder *);
 	int crtc_mask;
 	enum hpd_pin hpd_pin;
+	/* for communication with audio component; protected by av_mutex */
+	const struct drm_connector *audio_connector;
 };
 
 struct intel_panel {
@@ -291,6 +294,9 @@ struct intel_connector {
 	 */
 	struct intel_encoder *encoder;
 
+	/* ACPI device id for ACPI and driver cooperation */
+	u32 acpi_device_id;
+
 	/* Reads out the current hw, returning true if the connector is enabled
 	 * and active (i.e. dpms ON state). */
 	bool (*get_hw_state)(struct intel_connector *);
@@ -362,6 +368,8 @@ struct intel_atomic_state {
 
 	/* Gen9+ only */
 	struct skl_wm_values wm_results;
+
+	struct i915_sw_fence commit_ready;
 };
 
 struct intel_plane_state {
@@ -398,9 +406,6 @@ struct intel_plane_state {
 	int scaler_id;
 
 	struct drm_intel_sprite_colorkey ckey;
-
-	/* async flip related structures */
-	struct drm_i915_gem_request *wait_req;
 };
 
 struct intel_initial_plane_config {
@@ -465,9 +470,13 @@ struct intel_pipe_wm {
 	bool sprites_scaled;
 };
 
-struct skl_pipe_wm {
+struct skl_plane_wm {
 	struct skl_wm_level wm[8];
 	struct skl_wm_level trans_wm;
+};
+
+struct skl_pipe_wm {
+	struct skl_plane_wm planes[I915_MAX_PLANES];
 	uint32_t linetime;
 };
 
@@ -493,14 +502,7 @@ struct intel_crtc_wm_state {
 		struct {
 			/* gen9+ only needs 1-step wm programming */
 			struct skl_pipe_wm optimal;
-
-			/* cached plane data rate */
-			unsigned plane_data_rate[I915_MAX_PLANES];
-			unsigned plane_y_data_rate[I915_MAX_PLANES];
-
-			/* minimum block allocation */
-			uint16_t minimum_blocks[I915_MAX_PLANES];
-			uint16_t minimum_y_blocks[I915_MAX_PLANES];
+			struct skl_ddb_entry ddb;
 		} skl;
 	};
 
@@ -653,7 +655,6 @@ struct intel_crtc_state {
 
 	bool double_wide;
 
-	bool dp_encoder_is_mst;
 	int pbn;
 
 	struct intel_crtc_scaler_state scaler_state;
@@ -723,7 +724,6 @@ struct intel_crtc {
 		/* watermarks currently being used  */
 		union {
 			struct intel_pipe_wm ilk;
-			struct skl_pipe_wm skl;
 		} active;
 
 		/* allow CxSR on this pipe */
@@ -796,22 +796,22 @@ struct intel_plane {
 };
 
 struct intel_watermark_params {
-	unsigned long fifo_size;
-	unsigned long max_wm;
-	unsigned long default_wm;
-	unsigned long guard_size;
-	unsigned long cacheline_size;
+	u16 fifo_size;
+	u16 max_wm;
+	u8 default_wm;
+	u8 guard_size;
+	u8 cacheline_size;
 };
 
 struct cxsr_latency {
-	int is_desktop;
-	int is_ddr3;
-	unsigned long fsb_freq;
-	unsigned long mem_freq;
-	unsigned long display_sr;
-	unsigned long display_hpll_disable;
-	unsigned long cursor_sr;
-	unsigned long cursor_hpll_disable;
+	bool is_desktop : 1;
+	bool is_ddr3 : 1;
+	u16 fsb_freq;
+	u16 mem_freq;
+	u16 display_sr;
+	u16 display_hpll_disable;
+	u16 cursor_sr;
+	u16 cursor_hpll_disable;
 };
 
 #define to_intel_atomic_state(x) container_of(x, struct intel_atomic_state, base)
@@ -872,6 +872,14 @@ enum link_m_n_set {
 	M2_N2
 };
 
+struct intel_dp_desc {
+	u8 oui[3];
+	u8 device_id[6];
+	u8 hw_rev;
+	u8 sw_major_rev;
+	u8 sw_minor_rev;
+} __packed;
+
 struct intel_dp {
 	i915_reg_t output_reg;
 	i915_reg_t aux_ch_ctl_reg;
@@ -894,6 +902,8 @@ struct intel_dp {
 	/* sink rates as reported by DP_SUPPORTED_LINK_RATES */
 	uint8_t num_sink_rates;
 	int sink_rates[DP_MAX_SUPPORTED_RATES];
+	/* sink or branch descriptor */
+	struct intel_dp_desc desc;
 	struct drm_dp_aux aux;
 	uint8_t train_set[4];
 	int panel_power_up_delay;
@@ -950,17 +960,22 @@ struct intel_dp {
 	bool compliance_test_active;
 };
 
+struct intel_lspcon {
+	bool active;
+	enum drm_lspcon_mode mode;
+	bool desc_valid;
+};
+
 struct intel_digital_port {
 	struct intel_encoder base;
 	enum port port;
 	u32 saved_port_bits;
 	struct intel_dp dp;
 	struct intel_hdmi hdmi;
+	struct intel_lspcon lspcon;
 	enum irqreturn (*hpd_pulse)(struct intel_digital_port *, bool);
 	bool release_cl2_override;
 	uint8_t max_lanes;
-	/* for communication with audio component; protected by av_mutex */
-	const struct drm_connector *audio_connector;
 };
 
 struct intel_dp_mst_encoder {
@@ -1012,17 +1027,15 @@ vlv_pipe_to_channel(enum pipe pipe)
 	}
 }
 
-static inline struct drm_crtc *
-intel_get_crtc_for_pipe(struct drm_device *dev, int pipe)
+static inline struct intel_crtc *
+intel_get_crtc_for_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	return dev_priv->pipe_to_crtc_mapping[pipe];
 }
 
-static inline struct drm_crtc *
-intel_get_crtc_for_plane(struct drm_device *dev, int plane)
+static inline struct intel_crtc *
+intel_get_crtc_for_plane(struct drm_i915_private *dev_priv, enum plane plane)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	return dev_priv->plane_to_crtc_mapping[plane];
 }
 
@@ -1082,15 +1095,6 @@ hdmi_to_dig_port(struct intel_hdmi *intel_hdmi)
 	return container_of(intel_hdmi, struct intel_digital_port, hdmi);
 }
 
-/*
- * Returns the number of planes for this pipe, ie the number of sprites + 1
- * (primary plane). This doesn't count the cursor plane then.
- */
-static inline unsigned int intel_num_planes(struct intel_crtc *crtc)
-{
-	return INTEL_INFO(crtc->base.dev)->num_sprites[crtc->pipe] + 1;
-}
-
 /* intel_fifo_underrun.c */
 bool intel_set_cpu_fifo_underrun_reporting(struct drm_i915_private *dev_priv,
 					   enum pipe pipe, bool enable);
@@ -1107,6 +1111,9 @@ void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv);
 /* i915_irq.c */
 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask);
+void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 mask);
+void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask);
+void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask);
 void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv);
@@ -1129,6 +1136,9 @@ void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
 				     unsigned int pipe_mask);
 void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
 				     unsigned int pipe_mask);
+void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv);
+void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv);
+void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv);
 
 /* intel_crt.c */
 void intel_crt_init(struct drm_device *dev);
@@ -1176,12 +1186,15 @@ u32 intel_fb_stride_alignment(const struct drm_i915_private *dev_priv,
 
 /* intel_audio.c */
 void intel_init_audio_hooks(struct drm_i915_private *dev_priv);
-void intel_audio_codec_enable(struct intel_encoder *encoder);
+void intel_audio_codec_enable(struct intel_encoder *encoder,
+			      const struct intel_crtc_state *crtc_state,
+			      const struct drm_connector_state *conn_state);
 void intel_audio_codec_disable(struct intel_encoder *encoder);
 void i915_audio_component_init(struct drm_i915_private *dev_priv);
 void i915_audio_component_cleanup(struct drm_i915_private *dev_priv);
 
 /* intel_display.c */
+enum transcoder intel_crtc_pch_transcoder(struct intel_crtc *crtc);
 void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv, int vco);
 void intel_update_rawclk(struct drm_i915_private *dev_priv);
 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
@@ -1230,18 +1243,17 @@ intel_crtc_has_dp_encoder(const struct intel_crtc_state *crtc_state)
 		 (1 << INTEL_OUTPUT_EDP));
 }
 static inline void
-intel_wait_for_vblank(struct drm_device *dev, int pipe)
+intel_wait_for_vblank(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
-	drm_wait_one_vblank(dev, pipe);
+	drm_wait_one_vblank(&dev_priv->drm, pipe);
 }
 static inline void
-intel_wait_for_vblank_if_active(struct drm_device *dev, int pipe)
+intel_wait_for_vblank_if_active(struct drm_i915_private *dev_priv, int pipe)
 {
-	const struct intel_crtc *crtc =
-		to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe));
+	const struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 
 	if (crtc->active)
-		intel_wait_for_vblank(dev, pipe);
+		intel_wait_for_vblank(dev_priv, pipe);
 }
 
 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc);
@@ -1285,21 +1297,12 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
 unsigned int intel_tile_height(const struct drm_i915_private *dev_priv,
 			       uint64_t fb_modifier, unsigned int cpp);
 
-static inline bool
-intel_rotation_90_or_270(unsigned int rotation)
-{
-	return rotation & (DRM_ROTATE_90 | DRM_ROTATE_270);
-}
-
-void intel_create_rotation_property(struct drm_device *dev,
-					struct intel_plane *plane);
-
 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
 				    enum pipe pipe);
 
-int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe,
+int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe,
 		     const struct dpll *dpll);
-void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe);
+void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe);
 int lpt_get_iclkip(struct drm_i915_private *dev_priv);
 
 /* modesetting asserts */
@@ -1327,12 +1330,6 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv);
 void hsw_disable_pc8(struct drm_i915_private *dev_priv);
 void bxt_init_cdclk(struct drm_i915_private *dev_priv);
 void bxt_uninit_cdclk(struct drm_i915_private *dev_priv);
-void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy);
-void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy);
-bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv,
-			    enum dpio_phy phy);
-bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
-			      enum dpio_phy phy);
 void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv);
 void bxt_enable_dc9(struct drm_i915_private *dev_priv);
 void bxt_disable_dc9(struct drm_i915_private *dev_priv);
@@ -1350,7 +1347,7 @@ bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
 			struct dpll *best_clock);
 int chv_calc_dpll_params(int refclk, struct dpll *pll_clock);
 
-bool intel_crtc_active(struct drm_crtc *crtc);
+bool intel_crtc_active(struct intel_crtc *crtc);
 void hsw_enable_ips(struct intel_crtc *crtc);
 void hsw_disable_ips(struct intel_crtc *crtc);
 enum intel_display_power_domain
@@ -1396,7 +1393,7 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc);
 bool intel_dp_compute_config(struct intel_encoder *encoder,
 			     struct intel_crtc_state *pipe_config,
 			     struct drm_connector_state *conn_state);
-bool intel_dp_is_edp(struct drm_device *dev, enum port port);
+bool intel_dp_is_edp(struct drm_i915_private *dev_priv, enum port port);
 enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port,
 				  bool long_hpd);
 void intel_edp_backlight_on(struct intel_dp *intel_dp);
@@ -1443,6 +1440,11 @@ static inline unsigned int intel_dp_unused_lane_mask(int lane_count)
 	return ~((1 << lane_count) - 1) & 0xf;
 }
 
+bool intel_dp_read_dpcd(struct intel_dp *intel_dp);
+bool __intel_dp_read_desc(struct intel_dp *intel_dp,
+			  struct intel_dp_desc *desc);
+bool intel_dp_read_desc(struct intel_dp *intel_dp);
+
 /* intel_dp_aux_backlight.c */
 int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector);
 
@@ -1487,6 +1489,10 @@ static inline void intel_fbdev_set_suspend(struct drm_device *dev, int state, bo
 {
 }
 
+static inline void intel_fbdev_output_poll_changed(struct drm_device *dev)
+{
+}
+
 static inline void intel_fbdev_restore_mode(struct drm_device *dev)
 {
 }
@@ -1513,6 +1519,7 @@ void intel_fbc_invalidate(struct drm_i915_private *dev_priv,
 void intel_fbc_flush(struct drm_i915_private *dev_priv,
 		     unsigned int frontbuffer_bits, enum fb_op_origin origin);
 void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv);
+void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *dev_priv);
 
 /* intel_hdmi.c */
 void intel_hdmi_init(struct drm_device *dev, i915_reg_t hdmi_reg, enum port port);
@@ -1642,23 +1649,6 @@ assert_rpm_wakelock_held(struct drm_i915_private *dev_priv)
 		DRM_DEBUG_DRIVER("RPM wakelock ref not held during HW access");
 }
 
-static inline int
-assert_rpm_atomic_begin(struct drm_i915_private *dev_priv)
-{
-	int seq = atomic_read(&dev_priv->pm.atomic_seq);
-
-	assert_rpm_wakelock_held(dev_priv);
-
-	return seq;
-}
-
-static inline void
-assert_rpm_atomic_end(struct drm_i915_private *dev_priv, int begin_seq)
-{
-	WARN_ONCE(atomic_read(&dev_priv->pm.atomic_seq) != begin_seq,
-		  "HW access outside of RPM atomic section\n");
-}
-
 /**
  * disable_rpm_wakeref_asserts - disable the RPM assert checks
  * @dev_priv: i915 device instance
@@ -1714,11 +1704,11 @@ bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
 
 
 /* intel_pm.c */
-void intel_init_clock_gating(struct drm_device *dev);
-void intel_suspend_hw(struct drm_device *dev);
-int ilk_wm_max_level(const struct drm_device *dev);
-void intel_update_watermarks(struct drm_crtc *crtc);
-void intel_init_pm(struct drm_device *dev);
+void intel_init_clock_gating(struct drm_i915_private *dev_priv);
+void intel_suspend_hw(struct drm_i915_private *dev_priv);
+int ilk_wm_max_level(const struct drm_i915_private *dev_priv);
+void intel_update_watermarks(struct intel_crtc *crtc);
+void intel_init_pm(struct drm_i915_private *dev_priv);
 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv);
 void intel_pm_setup(struct drm_device *dev);
 void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
@@ -1742,21 +1732,16 @@ void ilk_wm_get_hw_state(struct drm_device *dev);
 void skl_wm_get_hw_state(struct drm_device *dev);
 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
 			  struct skl_ddb_allocation *ddb /* out */);
+void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc,
+			      struct skl_pipe_wm *out);
 bool intel_can_enable_sagv(struct drm_atomic_state *state);
 int intel_enable_sagv(struct drm_i915_private *dev_priv);
 int intel_disable_sagv(struct drm_i915_private *dev_priv);
-bool skl_ddb_allocation_equals(const struct skl_ddb_allocation *old,
-			       const struct skl_ddb_allocation *new,
-			       enum pipe pipe);
-bool skl_ddb_allocation_overlaps(struct drm_atomic_state *state,
-				 const struct skl_ddb_allocation *old,
-				 const struct skl_ddb_allocation *new,
-				 enum pipe pipe);
-void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
-			 const struct skl_wm_values *wm);
-void skl_write_plane_wm(struct intel_crtc *intel_crtc,
-			const struct skl_wm_values *wm,
-			int plane);
+bool skl_wm_level_equals(const struct skl_wm_level *l1,
+			 const struct skl_wm_level *l2);
+bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry **entries,
+				 const struct skl_ddb_entry *ddb,
+				 int ignore);
 uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config);
 bool ilk_disable_lp_wm(struct drm_device *dev);
 int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6);
@@ -1773,7 +1758,8 @@ bool intel_sdvo_init(struct drm_device *dev,
 /* intel_sprite.c */
 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
 			     int usecs);
-int intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane);
+struct intel_plane *intel_sprite_plane_create(struct drm_i915_private *dev_priv,
+					      enum pipe pipe, int plane);
 int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv);
 void intel_pipe_update_start(struct intel_crtc *crtc);
@@ -1835,4 +1821,7 @@ int intel_color_check(struct drm_crtc *crtc, struct drm_crtc_state *state);
 void intel_color_set_csc(struct drm_crtc_state *crtc_state);
 void intel_color_load_luts(struct drm_crtc_state *crtc_state);
 
+/* intel_lspcon.c */
+bool lspcon_init(struct intel_digital_port *intel_dig_port);
+void lspcon_resume(struct intel_lspcon *lspcon);
 #endif /* __INTEL_DRV_H__ */
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c
index b2e3d3a334f7..5b72c50d6f76 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/intel_dsi.c
@@ -437,11 +437,11 @@ static void vlv_dsi_device_ready(struct intel_encoder *encoder)
 
 static void intel_dsi_device_ready(struct intel_encoder *encoder)
 {
-	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		vlv_dsi_device_ready(encoder);
-	else if (IS_BROXTON(dev))
+	else if (IS_BROXTON(dev_priv))
 		bxt_dsi_device_ready(encoder);
 }
 
@@ -464,7 +464,7 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder)
 	}
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		i915_reg_t port_ctrl = IS_BROXTON(dev) ?
+		i915_reg_t port_ctrl = IS_BROXTON(dev_priv) ?
 			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
 		u32 temp;
 
@@ -494,7 +494,7 @@ static void intel_dsi_port_disable(struct intel_encoder *encoder)
 	enum port port;
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		i915_reg_t port_ctrl = IS_BROXTON(dev) ?
+		i915_reg_t port_ctrl = IS_BROXTON(dev_priv) ?
 			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
 		u32 temp;
 
@@ -656,7 +656,6 @@ static void intel_dsi_disable(struct intel_encoder *encoder)
 
 static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
 {
-	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 	enum port port;
@@ -664,7 +663,7 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
 	DRM_DEBUG_KMS("\n");
 	for_each_dsi_port(port, intel_dsi->ports) {
 		/* Common bit for both MIPI Port A & MIPI Port C on VLV/CHV */
-		i915_reg_t port_ctrl = IS_BROXTON(dev) ?
+		i915_reg_t port_ctrl = IS_BROXTON(dev_priv) ?
 			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(PORT_A);
 		u32 val;
 
@@ -741,7 +740,6 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
-	struct drm_device *dev = encoder->base.dev;
 	enum intel_display_power_domain power_domain;
 	enum port port;
 	bool active = false;
@@ -762,7 +760,7 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 
 	/* XXX: this only works for one DSI output */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		i915_reg_t ctrl_reg = IS_BROXTON(dev) ?
+		i915_reg_t ctrl_reg = IS_BROXTON(dev_priv) ?
 			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
 		bool enabled = I915_READ(ctrl_reg) & DPI_ENABLE;
 
@@ -771,7 +769,8 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 		 * bit in port C control register does not get set. As a
 		 * workaround, check pipe B conf instead.
 		 */
-		if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) && port == PORT_C)
+		if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
+		    port == PORT_C)
 			enabled = I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
 
 		/* Try command mode if video mode not enabled */
@@ -970,11 +969,11 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
 static void intel_dsi_get_config(struct intel_encoder *encoder,
 				 struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	u32 pclk;
 	DRM_DEBUG_KMS("\n");
 
-	if (IS_BROXTON(dev))
+	if (IS_BROXTON(dev_priv))
 		bxt_dsi_get_pipe_config(encoder, pipe_config);
 
 	pclk = intel_dsi_get_pclk(encoder, pipe_config->pipe_bpp,
@@ -1066,7 +1065,7 @@ static void set_dsi_timings(struct drm_encoder *encoder,
 	hbp = txbyteclkhs(hbp, bpp, lane_count, intel_dsi->burst_mode_ratio);
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (IS_BROXTON(dev)) {
+		if (IS_BROXTON(dev_priv)) {
 			/*
 			 * Program hdisplay and vdisplay on MIPI transcoder.
 			 * This is different from calculated hactive and
@@ -1138,7 +1137,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 	}
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+		if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 			/*
 			 * escape clock divider, 20MHz, shared for A and C.
 			 * device ready must be off when doing this! txclkesc?
@@ -1153,7 +1152,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 			tmp &= ~READ_REQUEST_PRIORITY_MASK;
 			I915_WRITE(MIPI_CTRL(port), tmp |
 					READ_REQUEST_PRIORITY_HIGH);
-		} else if (IS_BROXTON(dev)) {
+		} else if (IS_BROXTON(dev_priv)) {
 			enum pipe pipe = intel_crtc->pipe;
 
 			tmp = I915_READ(MIPI_CTRL(port));
@@ -1242,7 +1241,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 		I915_WRITE(MIPI_INIT_COUNT(port),
 				txclkesc(intel_dsi->escape_clk_div, 100));
 
-		if (IS_BROXTON(dev) && (!intel_dsi->dual_link)) {
+		if (IS_BROXTON(dev_priv) && (!intel_dsi->dual_link)) {
 			/*
 			 * BXT spec says write MIPI_INIT_COUNT for
 			 * both the ports, even if only one is
@@ -1299,12 +1298,6 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder,
 	}
 }
 
-static enum drm_connector_status
-intel_dsi_detect(struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static int intel_dsi_get_modes(struct drm_connector *connector)
 {
 	struct intel_connector *intel_connector = to_intel_connector(connector);
@@ -1346,7 +1339,7 @@ static int intel_dsi_set_property(struct drm_connector *connector,
 			DRM_DEBUG_KMS("no scaling not supported\n");
 			return -EINVAL;
 		}
-		if (HAS_GMCH_DISPLAY(dev) &&
+		if (HAS_GMCH_DISPLAY(to_i915(dev)) &&
 		    val == DRM_MODE_SCALE_CENTER) {
 			DRM_DEBUG_KMS("centering not supported\n");
 			return -EINVAL;
@@ -1408,7 +1401,6 @@ static const struct drm_connector_helper_funcs intel_dsi_connector_helper_funcs
 
 static const struct drm_connector_funcs intel_dsi_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
-	.detect = intel_dsi_detect,
 	.late_register = intel_connector_register,
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_dsi_connector_destroy,
@@ -1450,9 +1442,9 @@ void intel_dsi_init(struct drm_device *dev)
 	if (!intel_bios_is_dsi_present(dev_priv, &port))
 		return;
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		dev_priv->mipi_mmio_base = VLV_MIPI_BASE;
-	} else if (IS_BROXTON(dev)) {
+	} else if (IS_BROXTON(dev_priv)) {
 		dev_priv->mipi_mmio_base = BXT_MIPI_BASE;
 	} else {
 		DRM_ERROR("Unsupported Mipi device to reg base");
@@ -1488,6 +1480,7 @@ void intel_dsi_init(struct drm_device *dev)
 
 	intel_connector->get_hw_state = intel_connector_get_hw_state;
 
+	intel_encoder->port = port;
 	/*
 	 * On BYT/CHV, pipe A maps to MIPI DSI port A, pipe B maps to MIPI DSI
 	 * port C. BXT isn't limited like this.
diff --git a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
index cd154ce6b6c1..0d8ff0034b88 100644
--- a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
+++ b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
@@ -126,6 +126,8 @@ static const u8 *mipi_exec_send_packet(struct intel_dsi *intel_dsi,
 	u16 len;
 	enum port port;
 
+	DRM_DEBUG_KMS("\n");
+
 	flags = *data++;
 	type = *data++;
 
@@ -199,6 +201,8 @@ static const u8 *mipi_exec_delay(struct intel_dsi *intel_dsi, const u8 *data)
 {
 	u32 delay = *((const u32 *) data);
 
+	DRM_DEBUG_KMS("\n");
+
 	usleep_range(delay, delay + 10);
 	data += 4;
 
@@ -307,6 +311,8 @@ static const u8 *mipi_exec_gpio(struct intel_dsi *intel_dsi, const u8 *data)
 	u8 gpio_source, gpio_index;
 	bool value;
 
+	DRM_DEBUG_KMS("\n");
+
 	if (dev_priv->vbt.dsi.seq_version >= 3)
 		data++;
 
@@ -331,18 +337,36 @@ static const u8 *mipi_exec_gpio(struct intel_dsi *intel_dsi, const u8 *data)
 	return data;
 }
 
-static const u8 *mipi_exec_i2c_skip(struct intel_dsi *intel_dsi, const u8 *data)
+static const u8 *mipi_exec_i2c(struct intel_dsi *intel_dsi, const u8 *data)
 {
+	DRM_DEBUG_KMS("Skipping I2C element execution\n");
+
 	return data + *(data + 6) + 7;
 }
 
+static const u8 *mipi_exec_spi(struct intel_dsi *intel_dsi, const u8 *data)
+{
+	DRM_DEBUG_KMS("Skipping SPI element execution\n");
+
+	return data + *(data + 5) + 6;
+}
+
+static const u8 *mipi_exec_pmic(struct intel_dsi *intel_dsi, const u8 *data)
+{
+	DRM_DEBUG_KMS("Skipping PMIC element execution\n");
+
+	return data + 15;
+}
+
 typedef const u8 * (*fn_mipi_elem_exec)(struct intel_dsi *intel_dsi,
 					const u8 *data);
 static const fn_mipi_elem_exec exec_elem[] = {
 	[MIPI_SEQ_ELEM_SEND_PKT] = mipi_exec_send_packet,
 	[MIPI_SEQ_ELEM_DELAY] = mipi_exec_delay,
 	[MIPI_SEQ_ELEM_GPIO] = mipi_exec_gpio,
-	[MIPI_SEQ_ELEM_I2C] = mipi_exec_i2c_skip,
+	[MIPI_SEQ_ELEM_I2C] = mipi_exec_i2c,
+	[MIPI_SEQ_ELEM_SPI] = mipi_exec_spi,
+	[MIPI_SEQ_ELEM_PMIC] = mipi_exec_pmic,
 };
 
 /*
@@ -385,11 +409,8 @@ static void generic_exec_sequence(struct drm_panel *panel, enum mipi_seq seq_id)
 		return;
 
 	data = dev_priv->vbt.dsi.sequence[seq_id];
-	if (!data) {
-		DRM_DEBUG_KMS("MIPI sequence %d - %s not available\n",
-			      seq_id, sequence_name(seq_id));
+	if (!data)
 		return;
-	}
 
 	WARN_ON(*data != seq_id);
 
@@ -420,7 +441,15 @@ static void generic_exec_sequence(struct drm_panel *panel, enum mipi_seq seq_id)
 			operation_size = *data++;
 
 		if (mipi_elem_exec) {
+			const u8 *next = data + operation_size;
+
 			data = mipi_elem_exec(intel_dsi, data);
+
+			/* Consistency check if we have size. */
+			if (operation_size && data != next) {
+				DRM_ERROR("Inconsistent operation size\n");
+				return;
+			}
 		} else if (operation_size) {
 			/* We have size, skip. */
 			DRM_DEBUG_KMS("Unsupported MIPI operation byte %u\n",
@@ -438,6 +467,8 @@ static void generic_exec_sequence(struct drm_panel *panel, enum mipi_seq seq_id)
 static int vbt_panel_prepare(struct drm_panel *panel)
 {
 	generic_exec_sequence(panel, MIPI_SEQ_ASSERT_RESET);
+	generic_exec_sequence(panel, MIPI_SEQ_POWER_ON);
+	generic_exec_sequence(panel, MIPI_SEQ_DEASSERT_RESET);
 	generic_exec_sequence(panel, MIPI_SEQ_INIT_OTP);
 
 	return 0;
@@ -445,7 +476,8 @@ static int vbt_panel_prepare(struct drm_panel *panel)
 
 static int vbt_panel_unprepare(struct drm_panel *panel)
 {
-	generic_exec_sequence(panel, MIPI_SEQ_DEASSERT_RESET);
+	generic_exec_sequence(panel, MIPI_SEQ_ASSERT_RESET);
+	generic_exec_sequence(panel, MIPI_SEQ_POWER_OFF);
 
 	return 0;
 }
@@ -453,12 +485,14 @@ static int vbt_panel_unprepare(struct drm_panel *panel)
 static int vbt_panel_enable(struct drm_panel *panel)
 {
 	generic_exec_sequence(panel, MIPI_SEQ_DISPLAY_ON);
+	generic_exec_sequence(panel, MIPI_SEQ_BACKLIGHT_ON);
 
 	return 0;
 }
 
 static int vbt_panel_disable(struct drm_panel *panel)
 {
+	generic_exec_sequence(panel, MIPI_SEQ_BACKLIGHT_OFF);
 	generic_exec_sequence(panel, MIPI_SEQ_DISPLAY_OFF);
 
 	return 0;
@@ -740,9 +774,8 @@ struct drm_panel *vbt_panel_init(struct intel_dsi *intel_dsi, u16 panel_id)
 			8);
 	intel_dsi->clk_hs_to_lp_count += extra_byte_count;
 
-	DRM_DEBUG_KMS("Eot %s\n", intel_dsi->eotp_pkt ? "enabled" : "disabled");
-	DRM_DEBUG_KMS("Clockstop %s\n", intel_dsi->clock_stop ?
-						"disabled" : "enabled");
+	DRM_DEBUG_KMS("Eot %s\n", enableddisabled(intel_dsi->eotp_pkt));
+	DRM_DEBUG_KMS("Clockstop %s\n", enableddisabled(!intel_dsi->clock_stop));
 	DRM_DEBUG_KMS("Mode %s\n", intel_dsi->operation_mode ? "command" : "video");
 	if (intel_dsi->dual_link == DSI_DUAL_LINK_FRONT_BACK)
 		DRM_DEBUG_KMS("Dual link: DSI_DUAL_LINK_FRONT_BACK\n");
@@ -761,8 +794,7 @@ struct drm_panel *vbt_panel_init(struct intel_dsi *intel_dsi, u16 panel_id)
 	DRM_DEBUG_KMS("LP to HS Clock Count 0x%x\n", intel_dsi->clk_lp_to_hs_count);
 	DRM_DEBUG_KMS("HS to LP Clock Count 0x%x\n", intel_dsi->clk_hs_to_lp_count);
 	DRM_DEBUG_KMS("BTA %s\n",
-			intel_dsi->video_frmt_cfg_bits & DISABLE_VIDEO_BTA ?
-			"disabled" : "enabled");
+			enableddisabled(!(intel_dsi->video_frmt_cfg_bits & DISABLE_VIDEO_BTA)));
 
 	/* delays in VBT are in unit of 100us, so need to convert
 	 * here in ms
diff --git a/drivers/gpu/drm/i915/intel_dsi_pll.c b/drivers/gpu/drm/i915/intel_dsi_pll.c
index 6ab58a01b18e..56eff6004bc0 100644
--- a/drivers/gpu/drm/i915/intel_dsi_pll.c
+++ b/drivers/gpu/drm/i915/intel_dsi_pll.c
@@ -351,7 +351,7 @@ static u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
 u32 intel_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
 		       struct intel_crtc_state *config)
 {
-	if (IS_BROXTON(encoder->base.dev))
+	if (IS_BROXTON(to_i915(encoder->base.dev)))
 		return bxt_dsi_get_pclk(encoder, pipe_bpp, config);
 	else
 		return vlv_dsi_get_pclk(encoder, pipe_bpp, config);
@@ -515,11 +515,11 @@ bool intel_dsi_pll_is_enabled(struct drm_i915_private *dev_priv)
 int intel_compute_dsi_pll(struct intel_encoder *encoder,
 			  struct intel_crtc_state *config)
 {
-	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		return vlv_compute_dsi_pll(encoder, config);
-	else if (IS_BROXTON(dev))
+	else if (IS_BROXTON(dev_priv))
 		return bxt_compute_dsi_pll(encoder, config);
 
 	return -ENODEV;
@@ -528,21 +528,21 @@ int intel_compute_dsi_pll(struct intel_encoder *encoder,
 void intel_enable_dsi_pll(struct intel_encoder *encoder,
 			  const struct intel_crtc_state *config)
 {
-	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		vlv_enable_dsi_pll(encoder, config);
-	else if (IS_BROXTON(dev))
+	else if (IS_BROXTON(dev_priv))
 		bxt_enable_dsi_pll(encoder, config);
 }
 
 void intel_disable_dsi_pll(struct intel_encoder *encoder)
 {
-	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		vlv_disable_dsi_pll(encoder);
-	else if (IS_BROXTON(dev))
+	else if (IS_BROXTON(dev_priv))
 		bxt_disable_dsi_pll(encoder);
 }
 
@@ -564,10 +564,10 @@ static void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 
 void intel_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 {
-	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	if (IS_BROXTON(dev))
+	if (IS_BROXTON(dev_priv))
 		bxt_dsi_reset_clocks(encoder, port);
-	else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		vlv_dsi_reset_clocks(encoder, port);
 }
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index 2e452c505e7e..708645443046 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -393,12 +393,12 @@ intel_dvo_get_current_mode(struct drm_connector *connector)
 	 * its timings to get how the BIOS set up the panel.
 	 */
 	if (dvo_val & DVO_ENABLE) {
-		struct drm_crtc *crtc;
+		struct intel_crtc *crtc;
 		int pipe = (dvo_val & DVO_PIPE_B_SELECT) ? 1 : 0;
 
-		crtc = intel_get_crtc_for_pipe(dev, pipe);
+		crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 		if (crtc) {
-			mode = intel_crtc_mode_get(dev, crtc);
+			mode = intel_crtc_mode_get(dev, &crtc->base);
 			if (mode) {
 				mode->type |= DRM_MODE_TYPE_PREFERRED;
 				if (dvo_val & DVO_HSYNC_ACTIVE_HIGH)
@@ -412,16 +412,14 @@ intel_dvo_get_current_mode(struct drm_connector *connector)
 	return mode;
 }
 
-static char intel_dvo_port_name(i915_reg_t dvo_reg)
+static enum port intel_dvo_port(i915_reg_t dvo_reg)
 {
 	if (i915_mmio_reg_equal(dvo_reg, DVOA))
-		return 'A';
+		return PORT_A;
 	else if (i915_mmio_reg_equal(dvo_reg, DVOB))
-		return 'B';
-	else if (i915_mmio_reg_equal(dvo_reg, DVOC))
-		return 'C';
+		return PORT_B;
 	else
-		return '?';
+		return PORT_C;
 }
 
 void intel_dvo_init(struct drm_device *dev)
@@ -464,6 +462,7 @@ void intel_dvo_init(struct drm_device *dev)
 		bool dvoinit;
 		enum pipe pipe;
 		uint32_t dpll[I915_MAX_PIPES];
+		enum port port;
 
 		/* Allow the I2C driver info to specify the GPIO to be used in
 		 * special cases, but otherwise default to what's defined
@@ -511,12 +510,15 @@ void intel_dvo_init(struct drm_device *dev)
 		if (!dvoinit)
 			continue;
 
+		port = intel_dvo_port(dvo->dvo_reg);
 		drm_encoder_init(dev, &intel_encoder->base,
 				 &intel_dvo_enc_funcs, encoder_type,
-				 "DVO %c", intel_dvo_port_name(dvo->dvo_reg));
+				 "DVO %c", port_name(port));
 
 		intel_encoder->type = INTEL_OUTPUT_DVO;
+		intel_encoder->port = port;
 		intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
+
 		switch (dvo->type) {
 		case INTEL_DVO_CHIP_TMDS:
 			intel_encoder->cloneable = (1 << INTEL_OUTPUT_ANALOG) |
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 025e232a4205..3da4d466e332 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -82,12 +82,17 @@ static const struct engine_info {
 	},
 };
 
-static struct intel_engine_cs *
+static int
 intel_engine_setup(struct drm_i915_private *dev_priv,
 		   enum intel_engine_id id)
 {
 	const struct engine_info *info = &intel_engines[id];
-	struct intel_engine_cs *engine = &dev_priv->engine[id];
+	struct intel_engine_cs *engine;
+
+	GEM_BUG_ON(dev_priv->engine[id]);
+	engine = kzalloc(sizeof(*engine), GFP_KERNEL);
+	if (!engine)
+		return -ENOMEM;
 
 	engine->id = id;
 	engine->i915 = dev_priv;
@@ -97,7 +102,11 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
 	engine->mmio_base = info->mmio_base;
 	engine->irq_shift = info->irq_shift;
 
-	return engine;
+	/* Nothing to do here, execute in order of dependencies */
+	engine->schedule = NULL;
+
+	dev_priv->engine[id] = engine;
+	return 0;
 }
 
 /**
@@ -110,13 +119,16 @@ int intel_engines_init(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
+	unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
 	unsigned int mask = 0;
 	int (*init)(struct intel_engine_cs *engine);
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	unsigned int i;
 	int ret;
 
-	WARN_ON(INTEL_INFO(dev_priv)->ring_mask == 0);
-	WARN_ON(INTEL_INFO(dev_priv)->ring_mask &
+	WARN_ON(ring_mask == 0);
+	WARN_ON(ring_mask &
 		GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));
 
 	for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
@@ -131,7 +143,11 @@ int intel_engines_init(struct drm_device *dev)
 		if (!init)
 			continue;
 
-		ret = init(intel_engine_setup(dev_priv, i));
+		ret = intel_engine_setup(dev_priv, i);
+		if (ret)
+			goto cleanup;
+
+		ret = init(dev_priv->engine[i]);
 		if (ret)
 			goto cleanup;
 
@@ -143,7 +159,7 @@ int intel_engines_init(struct drm_device *dev)
 	 * are added to the driver by a warning and disabling the forgotten
 	 * engines.
 	 */
-	if (WARN_ON(mask != INTEL_INFO(dev_priv)->ring_mask))
+	if (WARN_ON(mask != ring_mask))
 		device_info->ring_mask = mask;
 
 	device_info->num_rings = hweight32(mask);
@@ -151,17 +167,17 @@ int intel_engines_init(struct drm_device *dev)
 	return 0;
 
 cleanup:
-	for (i = 0; i < I915_NUM_ENGINES; i++) {
+	for_each_engine(engine, dev_priv, id) {
 		if (i915.enable_execlists)
-			intel_logical_ring_cleanup(&dev_priv->engine[i]);
+			intel_logical_ring_cleanup(engine);
 		else
-			intel_engine_cleanup(&dev_priv->engine[i]);
+			intel_engine_cleanup(engine);
 	}
 
 	return ret;
 }
 
-void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno)
+void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 
@@ -191,13 +207,13 @@ void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno)
 				       I915_NUM_ENGINES * gen8_semaphore_seqno_size);
 		kunmap(page);
 	}
-	memset(engine->semaphore.sync_seqno, 0,
-	       sizeof(engine->semaphore.sync_seqno));
 
 	intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
 	if (engine->irq_seqno_barrier)
 		engine->irq_seqno_barrier(engine);
-	engine->last_submitted_seqno = seqno;
+
+	GEM_BUG_ON(i915_gem_active_isset(&engine->timeline->last_request));
+	engine->timeline->last_submitted_seqno = seqno;
 
 	engine->hangcheck.seqno = seqno;
 
@@ -207,15 +223,9 @@ void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno)
 	intel_engine_wakeup(engine);
 }
 
-void intel_engine_init_hangcheck(struct intel_engine_cs *engine)
-{
-	memset(&engine->hangcheck, 0, sizeof(engine->hangcheck));
-}
-
-static void intel_engine_init_requests(struct intel_engine_cs *engine)
+static void intel_engine_init_timeline(struct intel_engine_cs *engine)
 {
-	init_request_active(&engine->last_request, NULL);
-	INIT_LIST_HEAD(&engine->request_list);
+	engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id];
 }
 
 /**
@@ -229,12 +239,10 @@ static void intel_engine_init_requests(struct intel_engine_cs *engine)
  */
 void intel_engine_setup_common(struct intel_engine_cs *engine)
 {
-	INIT_LIST_HEAD(&engine->execlist_queue);
-	spin_lock_init(&engine->execlist_lock);
-
-	engine->fence_context = fence_context_alloc(1);
+	engine->execlist_queue = RB_ROOT;
+	engine->execlist_first = NULL;
 
-	intel_engine_init_requests(engine);
+	intel_engine_init_timeline(engine);
 	intel_engine_init_hangcheck(engine);
 	i915_gem_batch_pool_init(engine, &engine->batch_pool);
 
@@ -251,7 +259,7 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
 
 	obj = i915_gem_object_create_stolen(&engine->i915->drm, size);
 	if (!obj)
-		obj = i915_gem_object_create(&engine->i915->drm, size);
+		obj = i915_gem_object_create_internal(engine->i915, size);
 	if (IS_ERR(obj)) {
 		DRM_ERROR("Failed to allocate scratch page\n");
 		return PTR_ERR(obj);
@@ -301,6 +309,10 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
 	if (ret)
 		return ret;
 
+	ret = i915_gem_render_state_init(engine);
+	if (ret)
+		return ret;
+
 	return 0;
 }
 
@@ -315,7 +327,142 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine)
 {
 	intel_engine_cleanup_scratch(engine);
 
+	i915_gem_render_state_fini(engine);
 	intel_engine_fini_breadcrumbs(engine);
 	intel_engine_cleanup_cmd_parser(engine);
 	i915_gem_batch_pool_fini(&engine->batch_pool);
 }
+
+u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	u64 acthd;
+
+	if (INTEL_GEN(dev_priv) >= 8)
+		acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
+					 RING_ACTHD_UDW(engine->mmio_base));
+	else if (INTEL_GEN(dev_priv) >= 4)
+		acthd = I915_READ(RING_ACTHD(engine->mmio_base));
+	else
+		acthd = I915_READ(ACTHD);
+
+	return acthd;
+}
+
+u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	u64 bbaddr;
+
+	if (INTEL_GEN(dev_priv) >= 8)
+		bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
+					  RING_BBADDR_UDW(engine->mmio_base));
+	else
+		bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
+
+	return bbaddr;
+}
+
+const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
+{
+	switch (type) {
+	case I915_CACHE_NONE: return " uncached";
+	case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
+	case I915_CACHE_L3_LLC: return " L3+LLC";
+	case I915_CACHE_WT: return " WT";
+	default: return "";
+	}
+}
+
+static inline uint32_t
+read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
+		  int subslice, i915_reg_t reg)
+{
+	uint32_t mcr;
+	uint32_t ret;
+	enum forcewake_domains fw_domains;
+
+	fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
+						    FW_REG_READ);
+	fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
+						     GEN8_MCR_SELECTOR,
+						     FW_REG_READ | FW_REG_WRITE);
+
+	spin_lock_irq(&dev_priv->uncore.lock);
+	intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
+
+	mcr = I915_READ_FW(GEN8_MCR_SELECTOR);
+	/*
+	 * The HW expects the slice and sublice selectors to be reset to 0
+	 * after reading out the registers.
+	 */
+	WARN_ON_ONCE(mcr & (GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK));
+	mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
+	mcr |= GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
+	I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
+
+	ret = I915_READ_FW(reg);
+
+	mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
+	I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
+
+	intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
+	spin_unlock_irq(&dev_priv->uncore.lock);
+
+	return ret;
+}
+
+/* NB: please notice the memset */
+void intel_engine_get_instdone(struct intel_engine_cs *engine,
+			       struct intel_instdone *instdone)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	u32 mmio_base = engine->mmio_base;
+	int slice;
+	int subslice;
+
+	memset(instdone, 0, sizeof(*instdone));
+
+	switch (INTEL_GEN(dev_priv)) {
+	default:
+		instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
+
+		if (engine->id != RCS)
+			break;
+
+		instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
+		for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
+			instdone->sampler[slice][subslice] =
+				read_subslice_reg(dev_priv, slice, subslice,
+						  GEN7_SAMPLER_INSTDONE);
+			instdone->row[slice][subslice] =
+				read_subslice_reg(dev_priv, slice, subslice,
+						  GEN7_ROW_INSTDONE);
+		}
+		break;
+	case 7:
+		instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
+
+		if (engine->id != RCS)
+			break;
+
+		instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
+		instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
+		instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);
+
+		break;
+	case 6:
+	case 5:
+	case 4:
+		instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
+
+		if (engine->id == RCS)
+			/* HACK: Using the wrong struct member */
+			instdone->slice_common = I915_READ(GEN4_INSTDONE1);
+		break;
+	case 3:
+	case 2:
+		instdone->instdone = I915_READ(GEN2_INSTDONE);
+		break;
+	}
+}
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index c43dd9abce79..62f215b12eb5 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -48,17 +48,17 @@ static inline bool fbc_supported(struct drm_i915_private *dev_priv)
 
 static inline bool fbc_on_pipe_a_only(struct drm_i915_private *dev_priv)
 {
-	return IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8;
+	return IS_HASWELL(dev_priv) || INTEL_GEN(dev_priv) >= 8;
 }
 
 static inline bool fbc_on_plane_a_only(struct drm_i915_private *dev_priv)
 {
-	return INTEL_INFO(dev_priv)->gen < 4;
+	return INTEL_GEN(dev_priv) < 4;
 }
 
 static inline bool no_fbc_on_multiple_pipes(struct drm_i915_private *dev_priv)
 {
-	return INTEL_INFO(dev_priv)->gen <= 3;
+	return INTEL_GEN(dev_priv) <= 3;
 }
 
 /*
@@ -84,7 +84,7 @@ static void intel_fbc_get_plane_source_size(struct intel_fbc_state_cache *cache,
 {
 	int w, h;
 
-	if (intel_rotation_90_or_270(cache->plane.rotation)) {
+	if (drm_rotation_90_or_270(cache->plane.rotation)) {
 		w = cache->plane.src_h;
 		h = cache->plane.src_w;
 	} else {
@@ -351,7 +351,7 @@ static void gen7_fbc_activate(struct drm_i915_private *dev_priv)
 
 static bool intel_fbc_hw_is_active(struct drm_i915_private *dev_priv)
 {
-	if (INTEL_INFO(dev_priv)->gen >= 5)
+	if (INTEL_GEN(dev_priv) >= 5)
 		return ilk_fbc_is_active(dev_priv);
 	else if (IS_GM45(dev_priv))
 		return g4x_fbc_is_active(dev_priv);
@@ -365,9 +365,9 @@ static void intel_fbc_hw_activate(struct drm_i915_private *dev_priv)
 
 	fbc->active = true;
 
-	if (INTEL_INFO(dev_priv)->gen >= 7)
+	if (INTEL_GEN(dev_priv) >= 7)
 		gen7_fbc_activate(dev_priv);
-	else if (INTEL_INFO(dev_priv)->gen >= 5)
+	else if (INTEL_GEN(dev_priv) >= 5)
 		ilk_fbc_activate(dev_priv);
 	else if (IS_GM45(dev_priv))
 		g4x_fbc_activate(dev_priv);
@@ -381,7 +381,7 @@ static void intel_fbc_hw_deactivate(struct drm_i915_private *dev_priv)
 
 	fbc->active = false;
 
-	if (INTEL_INFO(dev_priv)->gen >= 5)
+	if (INTEL_GEN(dev_priv) >= 5)
 		ilk_fbc_deactivate(dev_priv);
 	else if (IS_GM45(dev_priv))
 		g4x_fbc_deactivate(dev_priv);
@@ -561,7 +561,7 @@ again:
 
 	ret = i915_gem_stolen_insert_node_in_range(dev_priv, node, size >>= 1,
 						   4096, 0, end);
-	if (ret && INTEL_INFO(dev_priv)->gen <= 4) {
+	if (ret && INTEL_GEN(dev_priv) <= 4) {
 		return 0;
 	} else if (ret) {
 		compression_threshold <<= 1;
@@ -594,7 +594,7 @@ static int intel_fbc_alloc_cfb(struct intel_crtc *crtc)
 
 	fbc->threshold = ret;
 
-	if (INTEL_INFO(dev_priv)->gen >= 5)
+	if (INTEL_GEN(dev_priv) >= 5)
 		I915_WRITE(ILK_DPFC_CB_BASE, fbc->compressed_fb.start);
 	else if (IS_GM45(dev_priv)) {
 		I915_WRITE(DPFC_CB_BASE, fbc->compressed_fb.start);
@@ -708,10 +708,10 @@ static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc)
 	struct intel_fbc *fbc = &dev_priv->fbc;
 	unsigned int effective_w, effective_h, max_w, max_h;
 
-	if (INTEL_INFO(dev_priv)->gen >= 8 || IS_HASWELL(dev_priv)) {
+	if (INTEL_GEN(dev_priv) >= 8 || IS_HASWELL(dev_priv)) {
 		max_w = 4096;
 		max_h = 4096;
-	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
+	} else if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
 		max_w = 4096;
 		max_h = 2048;
 	} else {
@@ -776,6 +776,14 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
 	struct intel_fbc *fbc = &dev_priv->fbc;
 	struct intel_fbc_state_cache *cache = &fbc->state_cache;
 
+	/* We don't need to use a state cache here since this information is
+	 * global for all CRTC.
+	 */
+	if (fbc->underrun_detected) {
+		fbc->no_fbc_reason = "underrun detected";
+		return false;
+	}
+
 	if (!cache->plane.visible) {
 		fbc->no_fbc_reason = "primary plane not visible";
 		return false;
@@ -804,7 +812,7 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
 		fbc->no_fbc_reason = "framebuffer not tiled or fenced";
 		return false;
 	}
-	if (INTEL_INFO(dev_priv)->gen <= 4 && !IS_G4X(dev_priv) &&
+	if (INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv) &&
 	    cache->plane.rotation != DRM_ROTATE_0) {
 		fbc->no_fbc_reason = "rotation unsupported";
 		return false;
@@ -846,9 +854,8 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
 	return true;
 }
 
-static bool intel_fbc_can_choose(struct intel_crtc *crtc)
+static bool intel_fbc_can_enable(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct intel_fbc *fbc = &dev_priv->fbc;
 
 	if (intel_vgpu_active(dev_priv)) {
@@ -861,13 +868,8 @@ static bool intel_fbc_can_choose(struct intel_crtc *crtc)
 		return false;
 	}
 
-	if (fbc_on_pipe_a_only(dev_priv) && crtc->pipe != PIPE_A) {
-		fbc->no_fbc_reason = "no enabled pipes can have FBC";
-		return false;
-	}
-
-	if (fbc_on_plane_a_only(dev_priv) && crtc->plane != PLANE_A) {
-		fbc->no_fbc_reason = "no enabled planes can have FBC";
+	if (fbc->underrun_detected) {
+		fbc->no_fbc_reason = "underrun detected";
 		return false;
 	}
 
@@ -1053,23 +1055,19 @@ void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
 			   struct drm_atomic_state *state)
 {
 	struct intel_fbc *fbc = &dev_priv->fbc;
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *crtc_state;
 	struct drm_plane *plane;
 	struct drm_plane_state *plane_state;
-	bool fbc_crtc_present = false;
-	int i, j;
+	bool crtc_chosen = false;
+	int i;
 
 	mutex_lock(&fbc->lock);
 
-	for_each_crtc_in_state(state, crtc, crtc_state, i) {
-		if (fbc->crtc == to_intel_crtc(crtc)) {
-			fbc_crtc_present = true;
-			break;
-		}
-	}
-	/* This atomic commit doesn't involve the CRTC currently tied to FBC. */
-	if (!fbc_crtc_present && fbc->crtc != NULL)
+	/* Does this atomic commit involve the CRTC currently tied to FBC? */
+	if (fbc->crtc &&
+	    !drm_atomic_get_existing_crtc_state(state, &fbc->crtc->base))
+		goto out;
+
+	if (!intel_fbc_can_enable(dev_priv))
 		goto out;
 
 	/* Simply choose the first CRTC that is compatible and has a visible
@@ -1079,25 +1077,29 @@ void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
 	for_each_plane_in_state(state, plane, plane_state, i) {
 		struct intel_plane_state *intel_plane_state =
 			to_intel_plane_state(plane_state);
+		struct intel_crtc_state *intel_crtc_state;
+		struct intel_crtc *crtc = to_intel_crtc(plane_state->crtc);
 
 		if (!intel_plane_state->base.visible)
 			continue;
 
-		for_each_crtc_in_state(state, crtc, crtc_state, j) {
-			struct intel_crtc_state *intel_crtc_state =
-				to_intel_crtc_state(crtc_state);
+		if (fbc_on_pipe_a_only(dev_priv) && crtc->pipe != PIPE_A)
+			continue;
 
-			if (plane_state->crtc != crtc)
-				continue;
+		if (fbc_on_plane_a_only(dev_priv) && crtc->plane != PLANE_A)
+			continue;
 
-			if (!intel_fbc_can_choose(to_intel_crtc(crtc)))
-				break;
+		intel_crtc_state = to_intel_crtc_state(
+			drm_atomic_get_existing_crtc_state(state, &crtc->base));
 
-			intel_crtc_state->enable_fbc = true;
-			goto out;
-		}
+		intel_crtc_state->enable_fbc = true;
+		crtc_chosen = true;
+		break;
 	}
 
+	if (!crtc_chosen)
+		fbc->no_fbc_reason = "no suitable CRTC for FBC";
+
 out:
 	mutex_unlock(&fbc->lock);
 }
@@ -1223,6 +1225,59 @@ void intel_fbc_global_disable(struct drm_i915_private *dev_priv)
 	cancel_work_sync(&fbc->work.work);
 }
 
+static void intel_fbc_underrun_work_fn(struct work_struct *work)
+{
+	struct drm_i915_private *dev_priv =
+		container_of(work, struct drm_i915_private, fbc.underrun_work);
+	struct intel_fbc *fbc = &dev_priv->fbc;
+
+	mutex_lock(&fbc->lock);
+
+	/* Maybe we were scheduled twice. */
+	if (fbc->underrun_detected)
+		goto out;
+
+	DRM_DEBUG_KMS("Disabling FBC due to FIFO underrun.\n");
+	fbc->underrun_detected = true;
+
+	intel_fbc_deactivate(dev_priv);
+out:
+	mutex_unlock(&fbc->lock);
+}
+
+/**
+ * intel_fbc_handle_fifo_underrun_irq - disable FBC when we get a FIFO underrun
+ * @dev_priv: i915 device instance
+ *
+ * Without FBC, most underruns are harmless and don't really cause too many
+ * problems, except for an annoying message on dmesg. With FBC, underruns can
+ * become black screens or even worse, especially when paired with bad
+ * watermarks. So in order for us to be on the safe side, completely disable FBC
+ * in case we ever detect a FIFO underrun on any pipe. An underrun on any pipe
+ * already suggests that watermarks may be bad, so try to be as safe as
+ * possible.
+ *
+ * This function is called from the IRQ handler.
+ */
+void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *dev_priv)
+{
+	struct intel_fbc *fbc = &dev_priv->fbc;
+
+	if (!fbc_supported(dev_priv))
+		return;
+
+	/* There's no guarantee that underrun_detected won't be set to true
+	 * right after this check and before the work is scheduled, but that's
+	 * not a problem since we'll check it again under the work function
+	 * while FBC is locked. This check here is just to prevent us from
+	 * unnecessarily scheduling the work, and it relies on the fact that we
+	 * never switch underrun_detect back to false after it's true. */
+	if (READ_ONCE(fbc->underrun_detected))
+		return;
+
+	schedule_work(&fbc->underrun_work);
+}
+
 /**
  * intel_fbc_init_pipe_state - initialize FBC's CRTC visibility tracking
  * @dev_priv: i915 device instance
@@ -1240,7 +1295,7 @@ void intel_fbc_init_pipe_state(struct drm_i915_private *dev_priv)
 		return;
 
 	for_each_intel_crtc(&dev_priv->drm, crtc)
-		if (intel_crtc_active(&crtc->base) &&
+		if (intel_crtc_active(crtc) &&
 		    to_intel_plane_state(crtc->base.primary->state)->base.visible)
 			dev_priv->fbc.visible_pipes_mask |= (1 << crtc->pipe);
 }
@@ -1294,6 +1349,7 @@ void intel_fbc_init(struct drm_i915_private *dev_priv)
 	enum pipe pipe;
 
 	INIT_WORK(&fbc->work.work, intel_fbc_work_fn);
+	INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);
 	mutex_init(&fbc->lock);
 	fbc->enabled = false;
 	fbc->active = false;
@@ -1319,7 +1375,7 @@ void intel_fbc_init(struct drm_i915_private *dev_priv)
 	}
 
 	/* This value was pulled out of someone's hat */
-	if (INTEL_INFO(dev_priv)->gen <= 4 && !IS_GM45(dev_priv))
+	if (INTEL_GEN(dev_priv) <= 4 && !IS_GM45(dev_priv))
 		I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT);
 
 	/* We still don't have any sort of hardware state readout for FBC, so
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index b7098f98bb67..beb08982dc0b 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -102,16 +102,13 @@ static int intel_fbdev_pan_display(struct fb_var_screeninfo *var,
 
 static struct fb_ops intelfb_ops = {
 	.owner = THIS_MODULE,
-	.fb_check_var = drm_fb_helper_check_var,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_set_par = intel_fbdev_set_par,
 	.fb_fillrect = drm_fb_helper_cfb_fillrect,
 	.fb_copyarea = drm_fb_helper_cfb_copyarea,
 	.fb_imageblit = drm_fb_helper_cfb_imageblit,
 	.fb_pan_display = intel_fbdev_pan_display,
 	.fb_blank = intel_fbdev_blank,
-	.fb_setcmap = drm_fb_helper_setcmap,
-	.fb_debug_enter = drm_fb_helper_debug_enter,
-	.fb_debug_leave = drm_fb_helper_debug_leave,
 };
 
 static int intelfb_alloc(struct drm_fb_helper *helper,
@@ -359,7 +356,7 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
 				    struct drm_fb_offset *offsets,
 				    bool *enabled, int width, int height)
 {
-	struct drm_device *dev = fb_helper->dev;
+	struct drm_i915_private *dev_priv = to_i915(fb_helper->dev);
 	unsigned long conn_configured, mask;
 	unsigned int count = min(fb_helper->connector_count, BITS_PER_LONG);
 	int i, j;
@@ -512,7 +509,7 @@ retry:
 	 * fbdev helper library.
 	 */
 	if (num_connectors_enabled != num_connectors_detected &&
-	    num_connectors_enabled < INTEL_INFO(dev)->num_pipes) {
+	    num_connectors_enabled < INTEL_INFO(dev_priv)->num_pipes) {
 		DRM_DEBUG_KMS("fallback: Not all outputs enabled\n");
 		DRM_DEBUG_KMS("Enabled: %i, detected: %i\n", num_connectors_enabled,
 			      num_connectors_detected);
@@ -636,7 +633,7 @@ static bool intel_fbdev_init_bios(struct drm_device *dev,
 		cur_size = intel_crtc->config->base.adjusted_mode.crtc_vdisplay;
 		cur_size = intel_fb_align_height(dev, cur_size,
 						 fb->base.pixel_format,
-						 fb->base.modifier[0]);
+						 fb->base.modifier);
 		cur_size *= fb->base.pitches[0];
 		DRM_DEBUG_KMS("pipe %c area: %dx%d, bpp: %d, size: %d\n",
 			      pipe_name(intel_crtc->pipe),
@@ -700,11 +697,11 @@ static void intel_fbdev_suspend_worker(struct work_struct *work)
 
 int intel_fbdev_init(struct drm_device *dev)
 {
-	struct intel_fbdev *ifbdev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_fbdev *ifbdev;
 	int ret;
 
-	if (WARN_ON(INTEL_INFO(dev)->num_pipes == 0))
+	if (WARN_ON(INTEL_INFO(dev_priv)->num_pipes == 0))
 		return -ENODEV;
 
 	ifbdev = kzalloc(sizeof(struct intel_fbdev), GFP_KERNEL);
@@ -717,7 +714,7 @@ int intel_fbdev_init(struct drm_device *dev)
 		ifbdev->preferred_bpp = 32;
 
 	ret = drm_fb_helper_init(dev, &ifbdev->helper,
-				 INTEL_INFO(dev)->num_pipes, 4);
+				 INTEL_INFO(dev_priv)->num_pipes, 4);
 	if (ret) {
 		kfree(ifbdev);
 		return ret;
diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c
index 2aa744081f09..e660d8b4bbc3 100644
--- a/drivers/gpu/drm/i915/intel_fifo_underrun.c
+++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c
@@ -57,7 +57,7 @@ static bool ivb_can_enable_err_int(struct drm_device *dev)
 	assert_spin_locked(&dev_priv->irq_lock);
 
 	for_each_pipe(dev_priv, pipe) {
-		crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
+		crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 
 		if (crtc->cpu_fifo_underrun_disabled)
 			return false;
@@ -75,7 +75,7 @@ static bool cpt_can_enable_serr_int(struct drm_device *dev)
 	assert_spin_locked(&dev_priv->irq_lock);
 
 	for_each_pipe(dev_priv, pipe) {
-		crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
+		crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 
 		if (crtc->pch_fifo_underrun_disabled)
 			return false;
@@ -245,22 +245,21 @@ static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
 						    enum pipe pipe, bool enable)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 	bool old;
 
 	assert_spin_locked(&dev_priv->irq_lock);
 
-	old = !intel_crtc->cpu_fifo_underrun_disabled;
-	intel_crtc->cpu_fifo_underrun_disabled = !enable;
+	old = !crtc->cpu_fifo_underrun_disabled;
+	crtc->cpu_fifo_underrun_disabled = !enable;
 
-	if (HAS_GMCH_DISPLAY(dev))
+	if (HAS_GMCH_DISPLAY(dev_priv))
 		i9xx_set_fifo_underrun_reporting(dev, pipe, enable, old);
-	else if (IS_GEN5(dev) || IS_GEN6(dev))
+	else if (IS_GEN5(dev_priv) || IS_GEN6(dev_priv))
 		ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
-	else if (IS_GEN7(dev))
+	else if (IS_GEN7(dev_priv))
 		ivybridge_set_fifo_underrun_reporting(dev, pipe, enable, old);
-	else if (IS_GEN8(dev) || IS_GEN9(dev))
+	else if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv))
 		broadwell_set_fifo_underrun_reporting(dev, pipe, enable);
 
 	return old;
@@ -314,8 +313,8 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv,
 					   enum transcoder pch_transcoder,
 					   bool enable)
 {
-	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder];
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc *crtc =
+		intel_get_crtc_for_pipe(dev_priv, (enum pipe) pch_transcoder);
 	unsigned long flags;
 	bool old;
 
@@ -330,8 +329,8 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv,
 
 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
 
-	old = !intel_crtc->pch_fifo_underrun_disabled;
-	intel_crtc->pch_fifo_underrun_disabled = !enable;
+	old = !crtc->pch_fifo_underrun_disabled;
+	crtc->pch_fifo_underrun_disabled = !enable;
 
 	if (HAS_PCH_IBX(dev_priv))
 		ibx_set_fifo_underrun_reporting(&dev_priv->drm,
@@ -358,7 +357,7 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv,
 void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv,
 					 enum pipe pipe)
 {
-	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 
 	/* We may be called too early in init, thanks BIOS! */
 	if (crtc == NULL)
@@ -366,12 +365,14 @@ void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv,
 
 	/* GMCH can't disable fifo underruns, filter them. */
 	if (HAS_GMCH_DISPLAY(dev_priv) &&
-	    to_intel_crtc(crtc)->cpu_fifo_underrun_disabled)
+	    crtc->cpu_fifo_underrun_disabled)
 		return;
 
 	if (intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false))
 		DRM_ERROR("CPU pipe %c FIFO underrun\n",
 			  pipe_name(pipe));
+
+	intel_fbc_handle_fifo_underrun_irq(dev_priv);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_frontbuffer.h b/drivers/gpu/drm/i915/intel_frontbuffer.h
index 76ceb539f9f0..7bab41218cf7 100644
--- a/drivers/gpu/drm/i915/intel_frontbuffer.h
+++ b/drivers/gpu/drm/i915/intel_frontbuffer.h
@@ -53,16 +53,17 @@ void __intel_fb_obj_flush(struct drm_i915_gem_object *obj,
  * until the rendering completes or a flip on this frontbuffer plane is
  * scheduled.
  */
-static inline void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
+static inline bool intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
 					   enum fb_op_origin origin)
 {
 	unsigned int frontbuffer_bits;
 
 	frontbuffer_bits = atomic_read(&obj->frontbuffer_bits);
 	if (!frontbuffer_bits)
-		return;
+		return false;
 
 	__intel_fb_obj_invalidate(obj, origin, frontbuffer_bits);
+	return true;
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index 5cdf7aa75be5..0053258e03d3 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -64,7 +64,7 @@ struct drm_i915_gem_request;
  */
 struct i915_guc_client {
 	struct i915_vma *vma;
-	void *client_base;		/* first page (only) of above	*/
+	void *vaddr;
 	struct i915_gem_context *owner;
 	struct intel_guc *guc;
 
@@ -123,10 +123,28 @@ struct intel_guc_fw {
 	uint32_t ucode_offset;
 };
 
+struct intel_guc_log {
+	uint32_t flags;
+	struct i915_vma *vma;
+	void *buf_addr;
+	struct workqueue_struct *flush_wq;
+	struct work_struct flush_work;
+	struct rchan *relay_chan;
+
+	/* logging related stats */
+	u32 capture_miss_count;
+	u32 flush_interrupt_count;
+	u32 prev_overflow_count[GUC_MAX_LOG_BUFFER];
+	u32 total_overflow_count[GUC_MAX_LOG_BUFFER];
+	u32 flush_count[GUC_MAX_LOG_BUFFER];
+};
+
 struct intel_guc {
 	struct intel_guc_fw guc_fw;
-	uint32_t log_flags;
-	struct i915_vma *log_vma;
+	struct intel_guc_log log;
+
+	/* GuC2Host interrupt related state */
+	bool interrupts_enabled;
 
 	struct i915_vma *ads_vma;
 	struct i915_vma *ctx_pool_vma;
@@ -146,6 +164,9 @@ struct intel_guc {
 
 	uint64_t submissions[I915_NUM_ENGINES];
 	uint32_t last_seqno[I915_NUM_ENGINES];
+
+	/* To serialize the Host2GuC actions */
+	struct mutex action_lock;
 };
 
 /* intel_guc_loader.c */
@@ -163,5 +184,10 @@ int i915_guc_wq_reserve(struct drm_i915_gem_request *rq);
 void i915_guc_wq_unreserve(struct drm_i915_gem_request *request);
 void i915_guc_submission_disable(struct drm_i915_private *dev_priv);
 void i915_guc_submission_fini(struct drm_i915_private *dev_priv);
+void i915_guc_capture_logs(struct drm_i915_private *dev_priv);
+void i915_guc_flush_logs(struct drm_i915_private *dev_priv);
+void i915_guc_register(struct drm_i915_private *dev_priv);
+void i915_guc_unregister(struct drm_i915_private *dev_priv);
+int i915_guc_log_control(struct drm_i915_private *dev_priv, u64 control_val);
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h
index e40db2d2ae99..324ea902558b 100644
--- a/drivers/gpu/drm/i915/intel_guc_fwif.h
+++ b/drivers/gpu/drm/i915/intel_guc_fwif.h
@@ -104,9 +104,9 @@
 #define   GUC_LOG_ALLOC_IN_MEGABYTE	(1 << 3)
 #define   GUC_LOG_CRASH_PAGES		1
 #define   GUC_LOG_CRASH_SHIFT		4
-#define   GUC_LOG_DPC_PAGES		3
+#define   GUC_LOG_DPC_PAGES		7
 #define   GUC_LOG_DPC_SHIFT		6
-#define   GUC_LOG_ISR_PAGES		3
+#define   GUC_LOG_ISR_PAGES		7
 #define   GUC_LOG_ISR_SHIFT		9
 #define   GUC_LOG_BUF_ADDR_SHIFT	12
 
@@ -419,15 +419,87 @@ struct guc_ads {
 	u32 reserved2[4];
 } __packed;
 
+/* GuC logging structures */
+
+enum guc_log_buffer_type {
+	GUC_ISR_LOG_BUFFER,
+	GUC_DPC_LOG_BUFFER,
+	GUC_CRASH_DUMP_LOG_BUFFER,
+	GUC_MAX_LOG_BUFFER
+};
+
+/**
+ * DOC: GuC Log buffer Layout
+ *
+ * Page0  +-------------------------------+
+ *        |   ISR state header (32 bytes) |
+ *        |      DPC state header         |
+ *        |   Crash dump state header     |
+ * Page1  +-------------------------------+
+ *        |           ISR logs            |
+ * Page9  +-------------------------------+
+ *        |           DPC logs            |
+ * Page17 +-------------------------------+
+ *        |         Crash Dump logs       |
+ *        +-------------------------------+
+ *
+ * Below state structure is used for coordination of retrieval of GuC firmware
+ * logs. Separate state is maintained for each log buffer type.
+ * read_ptr points to the location where i915 read last in log buffer and
+ * is read only for GuC firmware. write_ptr is incremented by GuC with number
+ * of bytes written for each log entry and is read only for i915.
+ * When any type of log buffer becomes half full, GuC sends a flush interrupt.
+ * GuC firmware expects that while it is writing to 2nd half of the buffer,
+ * first half would get consumed by Host and then get a flush completed
+ * acknowledgment from Host, so that it does not end up doing any overwrite
+ * causing loss of logs. So when buffer gets half filled & i915 has requested
+ * for interrupt, GuC will set flush_to_file field, set the sampled_write_ptr
+ * to the value of write_ptr and raise the interrupt.
+ * On receiving the interrupt i915 should read the buffer, clear flush_to_file
+ * field and also update read_ptr with the value of sample_write_ptr, before
+ * sending an acknowledgment to GuC. marker & version fields are for internal
+ * usage of GuC and opaque to i915. buffer_full_cnt field is incremented every
+ * time GuC detects the log buffer overflow.
+ */
+struct guc_log_buffer_state {
+	u32 marker[2];
+	u32 read_ptr;
+	u32 write_ptr;
+	u32 size;
+	u32 sampled_write_ptr;
+	union {
+		struct {
+			u32 flush_to_file:1;
+			u32 buffer_full_cnt:4;
+			u32 reserved:27;
+		};
+		u32 flags;
+	};
+	u32 version;
+} __packed;
+
+union guc_log_control {
+	struct {
+		u32 logging_enabled:1;
+		u32 reserved1:3;
+		u32 verbosity:4;
+		u32 reserved2:24;
+	};
+	u32 value;
+} __packed;
+
 /* This Action will be programmed in C180 - SOFT_SCRATCH_O_REG */
 enum host2guc_action {
 	HOST2GUC_ACTION_DEFAULT = 0x0,
 	HOST2GUC_ACTION_SAMPLE_FORCEWAKE = 0x6,
 	HOST2GUC_ACTION_ALLOCATE_DOORBELL = 0x10,
 	HOST2GUC_ACTION_DEALLOCATE_DOORBELL = 0x20,
+	HOST2GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE = 0x30,
+	HOST2GUC_ACTION_FORCE_LOG_BUFFER_FLUSH = 0x302,
 	HOST2GUC_ACTION_ENTER_S_STATE = 0x501,
 	HOST2GUC_ACTION_EXIT_S_STATE = 0x502,
 	HOST2GUC_ACTION_SLPC_REQUEST = 0x3003,
+	HOST2GUC_ACTION_UK_LOG_ENABLE_LOGGING = 0x0E000,
 	HOST2GUC_ACTION_LIMIT
 };
 
@@ -449,4 +521,10 @@ enum guc2host_status {
 	GUC2HOST_STATUS_GENERIC_FAIL = GUC2HOST_STATUS(0x0000F000)
 };
 
+/* This action will be programmed in C1BC - SOFT_SCRATCH_15_REG */
+enum guc2host_message {
+	GUC2HOST_MSG_CRASH_DUMP_POSTED = (1 << 1),
+	GUC2HOST_MSG_FLUSH_LOG_BUFFER = (1 << 3)
+};
+
 #endif
diff --git a/drivers/gpu/drm/i915/intel_guc_loader.c b/drivers/gpu/drm/i915/intel_guc_loader.c
index 6fd39efb7894..34d6ad2cf7c1 100644
--- a/drivers/gpu/drm/i915/intel_guc_loader.c
+++ b/drivers/gpu/drm/i915/intel_guc_loader.c
@@ -100,12 +100,13 @@ const char *intel_guc_fw_status_repr(enum intel_guc_fw_status status)
 static void guc_interrupts_release(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	int irqs;
 
 	/* tell all command streamers NOT to forward interrupts or vblank to GuC */
 	irqs = _MASKED_FIELD(GFX_FORWARD_VBLANK_MASK, GFX_FORWARD_VBLANK_NEVER);
 	irqs |= _MASKED_BIT_DISABLE(GFX_INTERRUPT_STEERING);
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		I915_WRITE(RING_MODE_GEN7(engine), irqs);
 
 	/* route all GT interrupts to the host */
@@ -117,12 +118,13 @@ static void guc_interrupts_release(struct drm_i915_private *dev_priv)
 static void guc_interrupts_capture(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	int irqs;
 	u32 tmp;
 
 	/* tell all command streamers to forward interrupts (but not vblank) to GuC */
 	irqs = _MASKED_BIT_ENABLE(GFX_INTERRUPT_STEERING);
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		I915_WRITE(RING_MODE_GEN7(engine), irqs);
 
 	/* route USER_INTERRUPT to Host, all others are sent to GuC. */
@@ -209,11 +211,13 @@ static void guc_params_init(struct drm_i915_private *dev_priv)
 	params[GUC_CTL_FEATURE] |= GUC_CTL_DISABLE_SCHEDULER |
 			GUC_CTL_VCS2_ENABLED;
 
+	params[GUC_CTL_LOG_PARAMS] = guc->log.flags;
+
 	if (i915.guc_log_level >= 0) {
-		params[GUC_CTL_LOG_PARAMS] = guc->log_flags;
 		params[GUC_CTL_DEBUG] =
 			i915.guc_log_level << GUC_LOG_VERBOSITY_SHIFT;
-	}
+	} else
+		params[GUC_CTL_DEBUG] = GUC_LOG_DISABLED;
 
 	if (guc->ads_vma) {
 		u32 ads = i915_ggtt_offset(guc->ads_vma) >> PAGE_SHIFT;
@@ -347,7 +351,6 @@ static u32 guc_wopcm_size(struct drm_i915_private *dev_priv)
 static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
-	struct drm_device *dev = &dev_priv->drm;
 	struct i915_vma *vma;
 	int ret;
 
@@ -375,24 +378,22 @@ static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
 	/* Enable MIA caching. GuC clock gating is disabled. */
 	I915_WRITE(GUC_SHIM_CONTROL, GUC_SHIM_CONTROL_VALUE);
 
-	/* WaDisableMinuteIaClockGating:skl,bxt */
-	if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
-	    IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
+	/* WaDisableMinuteIaClockGating:bxt */
+	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
 		I915_WRITE(GUC_SHIM_CONTROL, (I915_READ(GUC_SHIM_CONTROL) &
 					      ~GUC_ENABLE_MIA_CLOCK_GATING));
 	}
 
-	/* WaC6DisallowByGfxPause*/
-	if (IS_SKL_REVID(dev, 0, SKL_REVID_C0) ||
-	    IS_BXT_REVID(dev, 0, BXT_REVID_B0))
+	/* WaC6DisallowByGfxPause:bxt */
+	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
 		I915_WRITE(GEN6_GFXPAUSE, 0x30FFF);
 
-	if (IS_BROXTON(dev))
+	if (IS_BROXTON(dev_priv))
 		I915_WRITE(GEN9LP_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
 	else
 		I915_WRITE(GEN9_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
 
-	if (IS_GEN9(dev)) {
+	if (IS_GEN9(dev_priv)) {
 		/* DOP Clock Gating Enable for GuC clocks */
 		I915_WRITE(GEN7_MISCCPCTL, (GEN8_DOP_CLOCK_GATE_GUC_ENABLE |
 					    I915_READ(GEN7_MISCCPCTL)));
@@ -484,6 +485,7 @@ int intel_guc_setup(struct drm_device *dev)
 	}
 
 	guc_interrupts_release(dev_priv);
+	gen9_reset_guc_interrupts(dev_priv);
 
 	guc_fw->guc_fw_load_status = GUC_FIRMWARE_PENDING;
 
@@ -528,6 +530,9 @@ int intel_guc_setup(struct drm_device *dev)
 		intel_guc_fw_status_repr(guc_fw->guc_fw_load_status));
 
 	if (i915.enable_guc_submission) {
+		if (i915.guc_log_level >= 0)
+			gen9_enable_guc_interrupts(dev_priv);
+
 		err = i915_guc_submission_enable(dev_priv);
 		if (err)
 			goto fail;
@@ -561,7 +566,7 @@ fail:
 		ret = 0;
 	}
 
-	if (err == 0 && !HAS_GUC_UCODE(dev))
+	if (err == 0 && !HAS_GUC_UCODE(dev_priv))
 		;	/* Don't mention the GuC! */
 	else if (err == 0)
 		DRM_INFO("GuC firmware load skipped\n");
@@ -720,23 +725,28 @@ void intel_guc_init(struct drm_device *dev)
 	struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
 	const char *fw_path;
 
-	/* A negative value means "use platform default" */
-	if (i915.enable_guc_loading < 0)
-		i915.enable_guc_loading = HAS_GUC_UCODE(dev);
-	if (i915.enable_guc_submission < 0)
-		i915.enable_guc_submission = HAS_GUC_SCHED(dev);
+	if (!HAS_GUC(dev_priv)) {
+		i915.enable_guc_loading = 0;
+		i915.enable_guc_submission = 0;
+	} else {
+		/* A negative value means "use platform default" */
+		if (i915.enable_guc_loading < 0)
+			i915.enable_guc_loading = HAS_GUC_UCODE(dev_priv);
+		if (i915.enable_guc_submission < 0)
+			i915.enable_guc_submission = HAS_GUC_SCHED(dev_priv);
+	}
 
-	if (!HAS_GUC_UCODE(dev)) {
+	if (!HAS_GUC_UCODE(dev_priv)) {
 		fw_path = NULL;
-	} else if (IS_SKYLAKE(dev)) {
+	} else if (IS_SKYLAKE(dev_priv)) {
 		fw_path = I915_SKL_GUC_UCODE;
 		guc_fw->guc_fw_major_wanted = SKL_FW_MAJOR;
 		guc_fw->guc_fw_minor_wanted = SKL_FW_MINOR;
-	} else if (IS_BROXTON(dev)) {
+	} else if (IS_BROXTON(dev_priv)) {
 		fw_path = I915_BXT_GUC_UCODE;
 		guc_fw->guc_fw_major_wanted = BXT_FW_MAJOR;
 		guc_fw->guc_fw_minor_wanted = BXT_FW_MINOR;
-	} else if (IS_KABYLAKE(dev)) {
+	} else if (IS_KABYLAKE(dev_priv)) {
 		fw_path = I915_KBL_GUC_UCODE;
 		guc_fw->guc_fw_major_wanted = KBL_FW_MAJOR;
 		guc_fw->guc_fw_minor_wanted = KBL_FW_MINOR;
diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c
index 434f4d5c553d..290384e86c63 100644
--- a/drivers/gpu/drm/i915/intel_gvt.c
+++ b/drivers/gpu/drm/i915/intel_gvt.c
@@ -31,14 +31,20 @@
  * GPU among multiple virtual machines on a time-sharing basis. Each
  * virtual machine is presented a virtual GPU (vGPU), which has equivalent
  * features as the underlying physical GPU (pGPU), so i915 driver can run
- * seamlessly in a virtual machine. This file provides the englightments
- * of GVT and the necessary components used by GVT in i915 driver.
+ * seamlessly in a virtual machine.
+ *
+ * To virtualize GPU resources GVT-g driver depends on hypervisor technology
+ * e.g KVM/VFIO/mdev, Xen, etc. to provide resource access trapping capability
+ * and be virtualized within GVT-g device module. More architectural design
+ * doc is available on https://01.org/group/2230/documentation-list.
  */
 
 static bool is_supported_device(struct drm_i915_private *dev_priv)
 {
 	if (IS_BROADWELL(dev_priv))
 		return true;
+	if (IS_SKYLAKE(dev_priv))
+		return true;
 	return false;
 }
 
diff --git a/drivers/gpu/drm/i915/intel_gvt.h b/drivers/gpu/drm/i915/intel_gvt.h
index 960211df74db..25df2d65b985 100644
--- a/drivers/gpu/drm/i915/intel_gvt.h
+++ b/drivers/gpu/drm/i915/intel_gvt.h
@@ -24,7 +24,7 @@
 #ifndef _INTEL_GVT_H_
 #define _INTEL_GVT_H_
 
-#include "gvt/gvt.h"
+struct intel_gvt;
 
 #ifdef CONFIG_DRM_I915_GVT
 int intel_gvt_init(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/intel_hangcheck.c b/drivers/gpu/drm/i915/intel_hangcheck.c
new file mode 100644
index 000000000000..53df5b11bff4
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_hangcheck.c
@@ -0,0 +1,450 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "i915_drv.h"
+
+static bool
+ipehr_is_semaphore_wait(struct intel_engine_cs *engine, u32 ipehr)
+{
+	if (INTEL_GEN(engine->i915) >= 8) {
+		return (ipehr >> 23) == 0x1c;
+	} else {
+		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
+		return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
+				 MI_SEMAPHORE_REGISTER);
+	}
+}
+
+static struct intel_engine_cs *
+semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr,
+				 u64 offset)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	struct intel_engine_cs *signaller;
+	enum intel_engine_id id;
+
+	if (INTEL_GEN(dev_priv) >= 8) {
+		for_each_engine(signaller, dev_priv, id) {
+			if (engine == signaller)
+				continue;
+
+			if (offset == signaller->semaphore.signal_ggtt[engine->hw_id])
+				return signaller;
+		}
+	} else {
+		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
+
+		for_each_engine(signaller, dev_priv, id) {
+			if(engine == signaller)
+				continue;
+
+			if (sync_bits == signaller->semaphore.mbox.wait[engine->hw_id])
+				return signaller;
+		}
+	}
+
+	DRM_DEBUG_DRIVER("No signaller ring found for %s, ipehr 0x%08x, offset 0x%016llx\n",
+			 engine->name, ipehr, offset);
+
+	return ERR_PTR(-ENODEV);
+}
+
+static struct intel_engine_cs *
+semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	void __iomem *vaddr;
+	u32 cmd, ipehr, head;
+	u64 offset = 0;
+	int i, backwards;
+
+	/*
+	 * This function does not support execlist mode - any attempt to
+	 * proceed further into this function will result in a kernel panic
+	 * when dereferencing ring->buffer, which is not set up in execlist
+	 * mode.
+	 *
+	 * The correct way of doing it would be to derive the currently
+	 * executing ring buffer from the current context, which is derived
+	 * from the currently running request. Unfortunately, to get the
+	 * current request we would have to grab the struct_mutex before doing
+	 * anything else, which would be ill-advised since some other thread
+	 * might have grabbed it already and managed to hang itself, causing
+	 * the hang checker to deadlock.
+	 *
+	 * Therefore, this function does not support execlist mode in its
+	 * current form. Just return NULL and move on.
+	 */
+	if (engine->buffer == NULL)
+		return NULL;
+
+	ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
+	if (!ipehr_is_semaphore_wait(engine, ipehr))
+		return NULL;
+
+	/*
+	 * HEAD is likely pointing to the dword after the actual command,
+	 * so scan backwards until we find the MBOX. But limit it to just 3
+	 * or 4 dwords depending on the semaphore wait command size.
+	 * Note that we don't care about ACTHD here since that might
+	 * point at at batch, and semaphores are always emitted into the
+	 * ringbuffer itself.
+	 */
+	head = I915_READ_HEAD(engine) & HEAD_ADDR;
+	backwards = (INTEL_GEN(dev_priv) >= 8) ? 5 : 4;
+	vaddr = (void __iomem *)engine->buffer->vaddr;
+
+	for (i = backwards; i; --i) {
+		/*
+		 * Be paranoid and presume the hw has gone off into the wild -
+		 * our ring is smaller than what the hardware (and hence
+		 * HEAD_ADDR) allows. Also handles wrap-around.
+		 */
+		head &= engine->buffer->size - 1;
+
+		/* This here seems to blow up */
+		cmd = ioread32(vaddr + head);
+		if (cmd == ipehr)
+			break;
+
+		head -= 4;
+	}
+
+	if (!i)
+		return NULL;
+
+	*seqno = ioread32(vaddr + head + 4) + 1;
+	if (INTEL_GEN(dev_priv) >= 8) {
+		offset = ioread32(vaddr + head + 12);
+		offset <<= 32;
+		offset |= ioread32(vaddr + head + 8);
+	}
+	return semaphore_wait_to_signaller_ring(engine, ipehr, offset);
+}
+
+static int semaphore_passed(struct intel_engine_cs *engine)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	struct intel_engine_cs *signaller;
+	u32 seqno;
+
+	engine->hangcheck.deadlock++;
+
+	signaller = semaphore_waits_for(engine, &seqno);
+	if (signaller == NULL)
+		return -1;
+
+	if (IS_ERR(signaller))
+		return 0;
+
+	/* Prevent pathological recursion due to driver bugs */
+	if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES)
+		return -1;
+
+	if (i915_seqno_passed(intel_engine_get_seqno(signaller), seqno))
+		return 1;
+
+	/* cursory check for an unkickable deadlock */
+	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
+	    semaphore_passed(signaller) < 0)
+		return -1;
+
+	return 0;
+}
+
+static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	for_each_engine(engine, dev_priv, id)
+		engine->hangcheck.deadlock = 0;
+}
+
+static bool instdone_unchanged(u32 current_instdone, u32 *old_instdone)
+{
+	u32 tmp = current_instdone | *old_instdone;
+	bool unchanged;
+
+	unchanged = tmp == *old_instdone;
+	*old_instdone |= tmp;
+
+	return unchanged;
+}
+
+static bool subunits_stuck(struct intel_engine_cs *engine)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	struct intel_instdone instdone;
+	struct intel_instdone *accu_instdone = &engine->hangcheck.instdone;
+	bool stuck;
+	int slice;
+	int subslice;
+
+	if (engine->id != RCS)
+		return true;
+
+	intel_engine_get_instdone(engine, &instdone);
+
+	/* There might be unstable subunit states even when
+	 * actual head is not moving. Filter out the unstable ones by
+	 * accumulating the undone -> done transitions and only
+	 * consider those as progress.
+	 */
+	stuck = instdone_unchanged(instdone.instdone,
+				   &accu_instdone->instdone);
+	stuck &= instdone_unchanged(instdone.slice_common,
+				    &accu_instdone->slice_common);
+
+	for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
+		stuck &= instdone_unchanged(instdone.sampler[slice][subslice],
+					    &accu_instdone->sampler[slice][subslice]);
+		stuck &= instdone_unchanged(instdone.row[slice][subslice],
+					    &accu_instdone->row[slice][subslice]);
+	}
+
+	return stuck;
+}
+
+static enum intel_engine_hangcheck_action
+head_stuck(struct intel_engine_cs *engine, u64 acthd)
+{
+	if (acthd != engine->hangcheck.acthd) {
+
+		/* Clear subunit states on head movement */
+		memset(&engine->hangcheck.instdone, 0,
+		       sizeof(engine->hangcheck.instdone));
+
+		return HANGCHECK_ACTIVE;
+	}
+
+	if (!subunits_stuck(engine))
+		return HANGCHECK_ACTIVE;
+
+	return HANGCHECK_HUNG;
+}
+
+static enum intel_engine_hangcheck_action
+engine_stuck(struct intel_engine_cs *engine, u64 acthd)
+{
+	struct drm_i915_private *dev_priv = engine->i915;
+	enum intel_engine_hangcheck_action ha;
+	u32 tmp;
+
+	ha = head_stuck(engine, acthd);
+	if (ha != HANGCHECK_HUNG)
+		return ha;
+
+	if (IS_GEN2(dev_priv))
+		return HANGCHECK_HUNG;
+
+	/* Is the chip hanging on a WAIT_FOR_EVENT?
+	 * If so we can simply poke the RB_WAIT bit
+	 * and break the hang. This should work on
+	 * all but the second generation chipsets.
+	 */
+	tmp = I915_READ_CTL(engine);
+	if (tmp & RING_WAIT) {
+		i915_handle_error(dev_priv, 0,
+				  "Kicking stuck wait on %s",
+				  engine->name);
+		I915_WRITE_CTL(engine, tmp);
+		return HANGCHECK_KICK;
+	}
+
+	if (INTEL_GEN(dev_priv) >= 6 && tmp & RING_WAIT_SEMAPHORE) {
+		switch (semaphore_passed(engine)) {
+		default:
+			return HANGCHECK_HUNG;
+		case 1:
+			i915_handle_error(dev_priv, 0,
+					  "Kicking stuck semaphore on %s",
+					  engine->name);
+			I915_WRITE_CTL(engine, tmp);
+			return HANGCHECK_KICK;
+		case 0:
+			return HANGCHECK_WAIT;
+		}
+	}
+
+	return HANGCHECK_HUNG;
+}
+
+/*
+ * This is called when the chip hasn't reported back with completed
+ * batchbuffers in a long time. We keep track per ring seqno progress and
+ * if there are no progress, hangcheck score for that ring is increased.
+ * Further, acthd is inspected to see if the ring is stuck. On stuck case
+ * we kick the ring. If we see no progress on three subsequent calls
+ * we assume chip is wedged and try to fix it by resetting the chip.
+ */
+static void i915_hangcheck_elapsed(struct work_struct *work)
+{
+	struct drm_i915_private *dev_priv =
+		container_of(work, typeof(*dev_priv),
+			     gpu_error.hangcheck_work.work);
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+	unsigned int hung = 0, stuck = 0;
+	int busy_count = 0;
+#define BUSY 1
+#define KICK 5
+#define HUNG 20
+#define ACTIVE_DECAY 15
+
+	if (!i915.enable_hangcheck)
+		return;
+
+	if (!READ_ONCE(dev_priv->gt.awake))
+		return;
+
+	/* As enabling the GPU requires fairly extensive mmio access,
+	 * periodically arm the mmio checker to see if we are triggering
+	 * any invalid access.
+	 */
+	intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
+
+	for_each_engine(engine, dev_priv, id) {
+		bool busy = intel_engine_has_waiter(engine);
+		u64 acthd;
+		u32 seqno;
+		u32 submit;
+
+		semaphore_clear_deadlocks(dev_priv);
+
+		/* We don't strictly need an irq-barrier here, as we are not
+		 * serving an interrupt request, be paranoid in case the
+		 * barrier has side-effects (such as preventing a broken
+		 * cacheline snoop) and so be sure that we can see the seqno
+		 * advance. If the seqno should stick, due to a stale
+		 * cacheline, we would erroneously declare the GPU hung.
+		 */
+		if (engine->irq_seqno_barrier)
+			engine->irq_seqno_barrier(engine);
+
+		acthd = intel_engine_get_active_head(engine);
+		seqno = intel_engine_get_seqno(engine);
+		submit = intel_engine_last_submit(engine);
+
+		if (engine->hangcheck.seqno == seqno) {
+			if (i915_seqno_passed(seqno, submit)) {
+				engine->hangcheck.action = HANGCHECK_IDLE;
+			} else {
+				/* We always increment the hangcheck score
+				 * if the engine is busy and still processing
+				 * the same request, so that no single request
+				 * can run indefinitely (such as a chain of
+				 * batches). The only time we do not increment
+				 * the hangcheck score on this ring, if this
+				 * engine is in a legitimate wait for another
+				 * engine. In that case the waiting engine is a
+				 * victim and we want to be sure we catch the
+				 * right culprit. Then every time we do kick
+				 * the ring, add a small increment to the
+				 * score so that we can catch a batch that is
+				 * being repeatedly kicked and so responsible
+				 * for stalling the machine.
+				 */
+				engine->hangcheck.action =
+					engine_stuck(engine, acthd);
+
+				switch (engine->hangcheck.action) {
+				case HANGCHECK_IDLE:
+				case HANGCHECK_WAIT:
+					break;
+				case HANGCHECK_ACTIVE:
+					engine->hangcheck.score += BUSY;
+					break;
+				case HANGCHECK_KICK:
+					engine->hangcheck.score += KICK;
+					break;
+				case HANGCHECK_HUNG:
+					engine->hangcheck.score += HUNG;
+					break;
+				}
+			}
+
+			if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
+				hung |= intel_engine_flag(engine);
+				if (engine->hangcheck.action != HANGCHECK_HUNG)
+					stuck |= intel_engine_flag(engine);
+			}
+		} else {
+			engine->hangcheck.action = HANGCHECK_ACTIVE;
+
+			/* Gradually reduce the count so that we catch DoS
+			 * attempts across multiple batches.
+			 */
+			if (engine->hangcheck.score > 0)
+				engine->hangcheck.score -= ACTIVE_DECAY;
+			if (engine->hangcheck.score < 0)
+				engine->hangcheck.score = 0;
+
+			/* Clear head and subunit states on seqno movement */
+			acthd = 0;
+
+			memset(&engine->hangcheck.instdone, 0,
+			       sizeof(engine->hangcheck.instdone));
+		}
+
+		engine->hangcheck.seqno = seqno;
+		engine->hangcheck.acthd = acthd;
+		busy_count += busy;
+	}
+
+	if (hung) {
+		char msg[80];
+		unsigned int tmp;
+		int len;
+
+		/* If some rings hung but others were still busy, only
+		 * blame the hanging rings in the synopsis.
+		 */
+		if (stuck != hung)
+			hung &= ~stuck;
+		len = scnprintf(msg, sizeof(msg),
+				"%s on ", stuck == hung ? "No progress" : "Hang");
+		for_each_engine_masked(engine, dev_priv, hung, tmp)
+			len += scnprintf(msg + len, sizeof(msg) - len,
+					 "%s, ", engine->name);
+		msg[len-2] = '\0';
+
+		return i915_handle_error(dev_priv, hung, msg);
+	}
+
+	/* Reset timer in case GPU hangs without another request being added */
+	if (busy_count)
+		i915_queue_hangcheck(dev_priv);
+}
+
+void intel_engine_init_hangcheck(struct intel_engine_cs *engine)
+{
+	memset(&engine->hangcheck, 0, sizeof(engine->hangcheck));
+}
+
+void intel_hangcheck_init(struct drm_i915_private *i915)
+{
+	INIT_DELAYED_WORK(&i915->gpu_error.hangcheck_work,
+			  i915_hangcheck_elapsed);
+}
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index 13c306173f27..fb88e32e25a3 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -50,7 +50,7 @@ assert_hdmi_port_disabled(struct intel_hdmi *intel_hdmi)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t enabled_bits;
 
-	enabled_bits = HAS_DDI(dev) ? DDI_BUF_CTL_ENABLE : SDVO_ENABLE;
+	enabled_bits = HAS_DDI(dev_priv) ? DDI_BUF_CTL_ENABLE : SDVO_ENABLE;
 
 	WARN(I915_READ(intel_hdmi->hdmi_reg) & enabled_bits,
 	     "HDMI port enabled, expecting disabled\n");
@@ -864,7 +864,7 @@ static void intel_hdmi_prepare(struct intel_encoder *encoder)
 	intel_dp_dual_mode_set_tmds_output(intel_hdmi, true);
 
 	hdmi_val = SDVO_ENCODING_HDMI;
-	if (!HAS_PCH_SPLIT(dev) && crtc->config->limited_color_range)
+	if (!HAS_PCH_SPLIT(dev_priv) && crtc->config->limited_color_range)
 		hdmi_val |= HDMI_COLOR_RANGE_16_235;
 	if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
 		hdmi_val |= SDVO_VSYNC_ACTIVE_HIGH;
@@ -879,9 +879,9 @@ static void intel_hdmi_prepare(struct intel_encoder *encoder)
 	if (crtc->config->has_hdmi_sink)
 		hdmi_val |= HDMI_MODE_SELECT_HDMI;
 
-	if (HAS_PCH_CPT(dev))
+	if (HAS_PCH_CPT(dev_priv))
 		hdmi_val |= SDVO_PIPE_SEL_CPT(crtc->pipe);
-	else if (IS_CHERRYVIEW(dev))
+	else if (IS_CHERRYVIEW(dev_priv))
 		hdmi_val |= SDVO_PIPE_SEL_CHV(crtc->pipe);
 	else
 		hdmi_val |= SDVO_PIPE_SEL(crtc->pipe);
@@ -911,9 +911,9 @@ static bool intel_hdmi_get_hw_state(struct intel_encoder *encoder,
 	if (!(tmp & SDVO_ENABLE))
 		goto out;
 
-	if (HAS_PCH_CPT(dev))
+	if (HAS_PCH_CPT(dev_priv))
 		*pipe = PORT_TO_PIPE_CPT(tmp);
-	else if (IS_CHERRYVIEW(dev))
+	else if (IS_CHERRYVIEW(dev_priv))
 		*pipe = SDVO_PORT_TO_PIPE_CHV(tmp);
 	else
 		*pipe = PORT_TO_PIPE(tmp);
@@ -956,7 +956,7 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder,
 	if (tmp & SDVO_AUDIO_ENABLE)
 		pipe_config->has_audio = true;
 
-	if (!HAS_PCH_SPLIT(dev) &&
+	if (!HAS_PCH_SPLIT(dev_priv) &&
 	    tmp & HDMI_COLOR_RANGE_16_235)
 		pipe_config->limited_color_range = true;
 
@@ -975,14 +975,16 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder,
 	pipe_config->lane_count = 4;
 }
 
-static void intel_enable_hdmi_audio(struct intel_encoder *encoder)
+static void intel_enable_hdmi_audio(struct intel_encoder *encoder,
+				    struct intel_crtc_state *pipe_config,
+				    struct drm_connector_state *conn_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
 
 	WARN_ON(!crtc->config->has_hdmi_sink);
 	DRM_DEBUG_DRIVER("Enabling HDMI audio on pipe %c\n",
 			 pipe_name(crtc->pipe));
-	intel_audio_codec_enable(encoder);
+	intel_audio_codec_enable(encoder, pipe_config, conn_state);
 }
 
 static void g4x_enable_hdmi(struct intel_encoder *encoder,
@@ -991,21 +993,20 @@ static void g4x_enable_hdmi(struct intel_encoder *encoder,
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
 	u32 temp;
 
 	temp = I915_READ(intel_hdmi->hdmi_reg);
 
 	temp |= SDVO_ENABLE;
-	if (crtc->config->has_audio)
+	if (pipe_config->has_audio)
 		temp |= SDVO_AUDIO_ENABLE;
 
 	I915_WRITE(intel_hdmi->hdmi_reg, temp);
 	POSTING_READ(intel_hdmi->hdmi_reg);
 
-	if (crtc->config->has_audio)
-		intel_enable_hdmi_audio(encoder);
+	if (pipe_config->has_audio)
+		intel_enable_hdmi_audio(encoder, pipe_config, conn_state);
 }
 
 static void ibx_enable_hdmi(struct intel_encoder *encoder,
@@ -1040,8 +1041,8 @@ static void ibx_enable_hdmi(struct intel_encoder *encoder,
 	 * FIXME: BSpec says this should be done at the end of
 	 * of the modeset sequence, so not sure if this isn't too soon.
 	 */
-	if (crtc->config->pipe_bpp > 24 &&
-	    crtc->config->pixel_multiplier > 1) {
+	if (pipe_config->pipe_bpp > 24 &&
+	    pipe_config->pixel_multiplier > 1) {
 		I915_WRITE(intel_hdmi->hdmi_reg, temp & ~SDVO_ENABLE);
 		POSTING_READ(intel_hdmi->hdmi_reg);
 
@@ -1055,8 +1056,8 @@ static void ibx_enable_hdmi(struct intel_encoder *encoder,
 		POSTING_READ(intel_hdmi->hdmi_reg);
 	}
 
-	if (crtc->config->has_audio)
-		intel_enable_hdmi_audio(encoder);
+	if (pipe_config->has_audio)
+		intel_enable_hdmi_audio(encoder, pipe_config, conn_state);
 }
 
 static void cpt_enable_hdmi(struct intel_encoder *encoder,
@@ -1073,7 +1074,7 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder,
 	temp = I915_READ(intel_hdmi->hdmi_reg);
 
 	temp |= SDVO_ENABLE;
-	if (crtc->config->has_audio)
+	if (pipe_config->has_audio)
 		temp |= SDVO_AUDIO_ENABLE;
 
 	/*
@@ -1086,7 +1087,7 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder,
 	 * 4. enable HDMI clock gating
 	 */
 
-	if (crtc->config->pipe_bpp > 24) {
+	if (pipe_config->pipe_bpp > 24) {
 		I915_WRITE(TRANS_CHICKEN1(pipe),
 			   I915_READ(TRANS_CHICKEN1(pipe)) |
 			   TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE);
@@ -1098,7 +1099,7 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder,
 	I915_WRITE(intel_hdmi->hdmi_reg, temp);
 	POSTING_READ(intel_hdmi->hdmi_reg);
 
-	if (crtc->config->pipe_bpp > 24) {
+	if (pipe_config->pipe_bpp > 24) {
 		temp &= ~SDVO_COLOR_FORMAT_MASK;
 		temp |= HDMI_COLOR_FORMAT_12bpc;
 
@@ -1110,8 +1111,8 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder,
 			   ~TRANS_CHICKEN1_HDMIUNIT_GC_DISABLE);
 	}
 
-	if (crtc->config->has_audio)
-		intel_enable_hdmi_audio(encoder);
+	if (pipe_config->has_audio)
+		intel_enable_hdmi_audio(encoder, pipe_config, conn_state);
 }
 
 static void vlv_enable_hdmi(struct intel_encoder *encoder,
@@ -1141,7 +1142,7 @@ static void intel_disable_hdmi(struct intel_encoder *encoder,
 	 * to transcoder A after disabling it to allow the
 	 * matching DP port to be enabled on transcoder A.
 	 */
-	if (HAS_PCH_IBX(dev) && crtc->pipe == PIPE_B) {
+	if (HAS_PCH_IBX(dev_priv) && crtc->pipe == PIPE_B) {
 		/*
 		 * We get CPU/PCH FIFO underruns on the other pipe when
 		 * doing the workaround. Sweep them under the rug.
@@ -1164,7 +1165,7 @@ static void intel_disable_hdmi(struct intel_encoder *encoder,
 		I915_WRITE(intel_hdmi->hdmi_reg, temp);
 		POSTING_READ(intel_hdmi->hdmi_reg);
 
-		intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A);
+		intel_wait_for_vblank_if_active(dev_priv, PIPE_A);
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true);
 		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true);
 	}
@@ -1178,9 +1179,7 @@ static void g4x_disable_hdmi(struct intel_encoder *encoder,
 			     struct intel_crtc_state *old_crtc_state,
 			     struct drm_connector_state *old_conn_state)
 {
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
-
-	if (crtc->config->has_audio)
+	if (old_crtc_state->has_audio)
 		intel_audio_codec_disable(encoder);
 
 	intel_disable_hdmi(encoder, old_crtc_state, old_conn_state);
@@ -1190,9 +1189,7 @@ static void pch_disable_hdmi(struct intel_encoder *encoder,
 			     struct intel_crtc_state *old_crtc_state,
 			     struct drm_connector_state *old_conn_state)
 {
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
-
-	if (crtc->config->has_audio)
+	if (old_crtc_state->has_audio)
 		intel_audio_codec_disable(encoder);
 }
 
@@ -1241,7 +1238,7 @@ static enum drm_mode_status
 hdmi_port_clock_valid(struct intel_hdmi *hdmi,
 		      int clock, bool respect_downstream_limits)
 {
-	struct drm_device *dev = intel_hdmi_to_dev(hdmi);
+	struct drm_i915_private *dev_priv = to_i915(intel_hdmi_to_dev(hdmi));
 
 	if (clock < 25000)
 		return MODE_CLOCK_LOW;
@@ -1249,11 +1246,11 @@ hdmi_port_clock_valid(struct intel_hdmi *hdmi,
 		return MODE_CLOCK_HIGH;
 
 	/* BXT DPLL can't generate 223-240 MHz */
-	if (IS_BROXTON(dev) && clock > 223333 && clock < 240000)
+	if (IS_BROXTON(dev_priv) && clock > 223333 && clock < 240000)
 		return MODE_CLOCK_RANGE;
 
 	/* CHV DPLL can't generate 216-240 MHz */
-	if (IS_CHERRYVIEW(dev) && clock > 216000 && clock < 240000)
+	if (IS_CHERRYVIEW(dev_priv) && clock > 216000 && clock < 240000)
 		return MODE_CLOCK_RANGE;
 
 	return MODE_OK;
@@ -1265,6 +1262,7 @@ intel_hdmi_mode_valid(struct drm_connector *connector,
 {
 	struct intel_hdmi *hdmi = intel_attached_hdmi(connector);
 	struct drm_device *dev = intel_hdmi_to_dev(hdmi);
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum drm_mode_status status;
 	int clock;
 	int max_dotclk = to_i915(connector->dev)->max_dotclk_freq;
@@ -1287,7 +1285,7 @@ intel_hdmi_mode_valid(struct drm_connector *connector,
 	status = hdmi_port_clock_valid(hdmi, clock, true);
 
 	/* if we can't do 8bpc we may still be able to do 12bpc */
-	if (!HAS_GMCH_DISPLAY(dev) && status != MODE_OK)
+	if (!HAS_GMCH_DISPLAY(dev_priv) && status != MODE_OK)
 		status = hdmi_port_clock_valid(hdmi, clock * 3 / 2, true);
 
 	return status;
@@ -1297,7 +1295,7 @@ static bool hdmi_12bpc_possible(struct intel_crtc_state *crtc_state)
 {
 	struct drm_device *dev = crtc_state->base.crtc->dev;
 
-	if (HAS_GMCH_DISPLAY(dev))
+	if (HAS_GMCH_DISPLAY(to_i915(dev)))
 		return false;
 
 	/*
@@ -1312,7 +1310,7 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 			       struct drm_connector_state *conn_state)
 {
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
-	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 	int clock_8bpc = pipe_config->base.adjusted_mode.crtc_clock;
 	int clock_12bpc = clock_8bpc * 3 / 2;
@@ -1339,7 +1337,7 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 		clock_12bpc *= 2;
 	}
 
-	if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev))
+	if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv))
 		pipe_config->has_pch_encoder = true;
 
 	if (pipe_config->has_hdmi_sink && intel_hdmi->has_audio)
@@ -1644,13 +1642,12 @@ static void intel_hdmi_pre_enable(struct intel_encoder *encoder,
 				  struct drm_connector_state *conn_state)
 {
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
-	const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
+	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 
 	intel_hdmi_prepare(encoder);
 
 	intel_hdmi->set_infoframes(&encoder->base,
-				   intel_crtc->config->has_hdmi_sink,
+				   pipe_config->has_hdmi_sink,
 				   adjusted_mode);
 }
 
@@ -1662,9 +1659,7 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder,
 	struct intel_hdmi *intel_hdmi = &dport->hdmi;
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc =
-		to_intel_crtc(encoder->base.crtc);
-	const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
+	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 
 	vlv_phy_pre_encoder_enable(encoder);
 
@@ -1673,7 +1668,7 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder,
 				 0x2b247878);
 
 	intel_hdmi->set_infoframes(&encoder->base,
-				   intel_crtc->config->has_hdmi_sink,
+				   pipe_config->has_hdmi_sink,
 				   adjusted_mode);
 
 	g4x_enable_hdmi(encoder, pipe_config, conn_state);
@@ -1896,19 +1891,19 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 		return;
 	}
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		intel_hdmi->write_infoframe = vlv_write_infoframe;
 		intel_hdmi->set_infoframes = vlv_set_infoframes;
 		intel_hdmi->infoframe_enabled = vlv_infoframe_enabled;
-	} else if (IS_G4X(dev)) {
+	} else if (IS_G4X(dev_priv)) {
 		intel_hdmi->write_infoframe = g4x_write_infoframe;
 		intel_hdmi->set_infoframes = g4x_set_infoframes;
 		intel_hdmi->infoframe_enabled = g4x_infoframe_enabled;
-	} else if (HAS_DDI(dev)) {
+	} else if (HAS_DDI(dev_priv)) {
 		intel_hdmi->write_infoframe = hsw_write_infoframe;
 		intel_hdmi->set_infoframes = hsw_set_infoframes;
 		intel_hdmi->infoframe_enabled = hsw_infoframe_enabled;
-	} else if (HAS_PCH_IBX(dev)) {
+	} else if (HAS_PCH_IBX(dev_priv)) {
 		intel_hdmi->write_infoframe = ibx_write_infoframe;
 		intel_hdmi->set_infoframes = ibx_set_infoframes;
 		intel_hdmi->infoframe_enabled = ibx_infoframe_enabled;
@@ -1918,7 +1913,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 		intel_hdmi->infoframe_enabled = cpt_infoframe_enabled;
 	}
 
-	if (HAS_DDI(dev))
+	if (HAS_DDI(dev_priv))
 		intel_connector->get_hw_state = intel_ddi_connector_get_hw_state;
 	else
 		intel_connector->get_hw_state = intel_connector_get_hw_state;
@@ -1932,7 +1927,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 	 * 0xd.  Failure to do so will result in spurious interrupts being
 	 * generated on the port when a cable is not attached.
 	 */
-	if (IS_G4X(dev) && !IS_GM45(dev)) {
+	if (IS_G4X(dev_priv) && !IS_GM45(dev_priv)) {
 		u32 temp = I915_READ(PEG_BAND_GAP_DATA);
 		I915_WRITE(PEG_BAND_GAP_DATA, (temp & ~0xf) | 0xd);
 	}
@@ -1941,6 +1936,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 void intel_hdmi_init(struct drm_device *dev,
 		     i915_reg_t hdmi_reg, enum port port)
 {
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_digital_port *intel_dig_port;
 	struct intel_encoder *intel_encoder;
 	struct intel_connector *intel_connector;
@@ -1961,7 +1957,7 @@ void intel_hdmi_init(struct drm_device *dev,
 			 DRM_MODE_ENCODER_TMDS, "HDMI %c", port_name(port));
 
 	intel_encoder->compute_config = intel_hdmi_compute_config;
-	if (HAS_PCH_SPLIT(dev)) {
+	if (HAS_PCH_SPLIT(dev_priv)) {
 		intel_encoder->disable = pch_disable_hdmi;
 		intel_encoder->post_disable = pch_post_disable_hdmi;
 	} else {
@@ -1969,29 +1965,30 @@ void intel_hdmi_init(struct drm_device *dev,
 	}
 	intel_encoder->get_hw_state = intel_hdmi_get_hw_state;
 	intel_encoder->get_config = intel_hdmi_get_config;
-	if (IS_CHERRYVIEW(dev)) {
+	if (IS_CHERRYVIEW(dev_priv)) {
 		intel_encoder->pre_pll_enable = chv_hdmi_pre_pll_enable;
 		intel_encoder->pre_enable = chv_hdmi_pre_enable;
 		intel_encoder->enable = vlv_enable_hdmi;
 		intel_encoder->post_disable = chv_hdmi_post_disable;
 		intel_encoder->post_pll_disable = chv_hdmi_post_pll_disable;
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv)) {
 		intel_encoder->pre_pll_enable = vlv_hdmi_pre_pll_enable;
 		intel_encoder->pre_enable = vlv_hdmi_pre_enable;
 		intel_encoder->enable = vlv_enable_hdmi;
 		intel_encoder->post_disable = vlv_hdmi_post_disable;
 	} else {
 		intel_encoder->pre_enable = intel_hdmi_pre_enable;
-		if (HAS_PCH_CPT(dev))
+		if (HAS_PCH_CPT(dev_priv))
 			intel_encoder->enable = cpt_enable_hdmi;
-		else if (HAS_PCH_IBX(dev))
+		else if (HAS_PCH_IBX(dev_priv))
 			intel_encoder->enable = ibx_enable_hdmi;
 		else
 			intel_encoder->enable = g4x_enable_hdmi;
 	}
 
 	intel_encoder->type = INTEL_OUTPUT_HDMI;
-	if (IS_CHERRYVIEW(dev)) {
+	intel_encoder->port = port;
+	if (IS_CHERRYVIEW(dev_priv)) {
 		if (port == PORT_D)
 			intel_encoder->crtc_mask = 1 << 2;
 		else
@@ -2005,7 +2002,7 @@ void intel_hdmi_init(struct drm_device *dev,
 	 * to work on real hardware. And since g4x can send infoframes to
 	 * only one port anyway, nothing is lost by allowing it.
 	 */
-	if (IS_G4X(dev))
+	if (IS_G4X(dev_priv))
 		intel_encoder->cloneable |= 1 << INTEL_OUTPUT_HDMI;
 
 	intel_dig_port->port = port;
diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c
index 334d47b5811a..3d546c019de0 100644
--- a/drivers/gpu/drm/i915/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/intel_hotplug.c
@@ -501,7 +501,7 @@ static void i915_hpd_poll_init_work(struct work_struct *work)
 		if (intel_connector->mst_port)
 			continue;
 
-		if (!connector->polled && I915_HAS_HOTPLUG(dev) &&
+		if (!connector->polled && I915_HAS_HOTPLUG(dev_priv) &&
 		    intel_connector->encoder->hpd_pin > HPD_NONE) {
 			connector->polled = enabled ?
 				DRM_CONNECTOR_POLL_CONNECT |
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index 79aab9ad6faa..83f260bb4eef 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -138,11 +138,10 @@ static void intel_i2c_quirk_set(struct drm_i915_private *dev_priv, bool enable)
 static u32 get_reserved(struct intel_gmbus *bus)
 {
 	struct drm_i915_private *dev_priv = bus->dev_priv;
-	struct drm_device *dev = &dev_priv->drm;
 	u32 reserved = 0;
 
 	/* On most chips, these bits must be preserved in software. */
-	if (!IS_I830(dev) && !IS_845G(dev))
+	if (!IS_I830(dev_priv) && !IS_845G(dev_priv))
 		reserved = I915_READ_NOTRACE(bus->gpio_reg) &
 					     (GPIO_DATA_PULLUP_DISABLE |
 					      GPIO_CLOCK_PULLUP_DISABLE);
@@ -468,13 +467,9 @@ do_gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
 					       struct intel_gmbus,
 					       adapter);
 	struct drm_i915_private *dev_priv = bus->dev_priv;
-	const unsigned int fw =
-		intel_uncore_forcewake_for_reg(dev_priv, GMBUS0,
-					       FW_REG_READ | FW_REG_WRITE);
 	int i = 0, inc, try = 0;
 	int ret = 0;
 
-	intel_uncore_forcewake_get(dev_priv, fw);
 retry:
 	I915_WRITE_FW(GMBUS0, bus->reg0);
 
@@ -576,7 +571,6 @@ timeout:
 	ret = -EAGAIN;
 
 out:
-	intel_uncore_forcewake_put(dev_priv, fw);
 	return ret;
 }
 
@@ -633,10 +627,10 @@ int intel_setup_gmbus(struct drm_device *dev)
 	unsigned int pin;
 	int ret;
 
-	if (HAS_PCH_NOP(dev))
+	if (HAS_PCH_NOP(dev_priv))
 		return 0;
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		dev_priv->gpio_mmio_base = VLV_DISPLAY_BASE;
 	else if (!HAS_GMCH_DISPLAY(dev_priv))
 		dev_priv->gpio_mmio_base =
@@ -674,7 +668,7 @@ int intel_setup_gmbus(struct drm_device *dev)
 		bus->reg0 = pin | GMBUS_RATE_100KHZ;
 
 		/* gmbus seems to be broken on i830 */
-		if (IS_I830(dev))
+		if (IS_I830(dev_priv))
 			bus->force_bit = 1;
 
 		intel_gpio_setup(bus, pin);
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 0adb879833ff..0a09024d6ca3 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -275,8 +275,7 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine)
 	struct drm_i915_private *dev_priv = engine->i915;
 
 	engine->disable_lite_restore_wa =
-		(IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
-		 IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) &&
+		IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1) &&
 		(engine->id == VCS || engine->id == VCS2);
 
 	engine->ctx_desc_template = GEN8_CTX_VALID;
@@ -366,7 +365,7 @@ static u64 execlists_update_context(struct drm_i915_gem_request *rq)
 	struct i915_hw_ppgtt *ppgtt = rq->ctx->ppgtt;
 	u32 *reg_state = ce->lrc_reg_state;
 
-	reg_state[CTX_RING_TAIL+1] = intel_ring_offset(rq->ring, rq->tail);
+	reg_state[CTX_RING_TAIL+1] = rq->tail;
 
 	/* True 32b PPGTT with dynamic page allocation: update PDP
 	 * registers and point the unallocated PDPs to scratch page.
@@ -433,15 +432,17 @@ static bool can_merge_ctx(const struct i915_gem_context *prev,
 
 static void execlists_dequeue(struct intel_engine_cs *engine)
 {
-	struct drm_i915_gem_request *cursor, *last;
+	struct drm_i915_gem_request *last;
 	struct execlist_port *port = engine->execlist_port;
+	unsigned long flags;
+	struct rb_node *rb;
 	bool submit = false;
 
 	last = port->request;
 	if (last)
 		/* WaIdleLiteRestore:bdw,skl
 		 * Apply the wa NOOPs to prevent ring:HEAD == req:TAIL
-		 * as we resubmit the request. See gen8_emit_request()
+		 * as we resubmit the request. See gen8_emit_breadcrumb()
 		 * for where we prepare the padding after the end of the
 		 * request.
 		 */
@@ -470,8 +471,12 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 	 * and context switches) submission.
 	 */
 
-	spin_lock(&engine->execlist_lock);
-	list_for_each_entry(cursor, &engine->execlist_queue, execlist_link) {
+	spin_lock_irqsave(&engine->timeline->lock, flags);
+	rb = engine->execlist_first;
+	while (rb) {
+		struct drm_i915_gem_request *cursor =
+			rb_entry(rb, typeof(*cursor), priotree.node);
+
 		/* Can we combine this request with the current port? It has to
 		 * be the same context/ringbuffer and not have any exceptions
 		 * (e.g. GVT saying never to combine contexts).
@@ -494,7 +499,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 			 * context (even though a different request) to
 			 * the second port.
 			 */
-			if (ctx_single_port_submission(cursor->ctx))
+			if (ctx_single_port_submission(last->ctx) ||
+			    ctx_single_port_submission(cursor->ctx))
 				break;
 
 			GEM_BUG_ON(last->ctx == cursor->ctx);
@@ -502,17 +508,30 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 			i915_gem_request_assign(&port->request, last);
 			port++;
 		}
+
+		rb = rb_next(rb);
+		rb_erase(&cursor->priotree.node, &engine->execlist_queue);
+		RB_CLEAR_NODE(&cursor->priotree.node);
+		cursor->priotree.priority = INT_MAX;
+
+		/* We keep the previous context alive until we retire the
+		 * following request. This ensures that any the context object
+		 * is still pinned for any residual writes the HW makes into it
+		 * on the context switch into the next object following the
+		 * breadcrumb. Otherwise, we may retire the context too early.
+		 */
+		cursor->previous_context = engine->last_context;
+		engine->last_context = cursor->ctx;
+
+		__i915_gem_request_submit(cursor);
 		last = cursor;
 		submit = true;
 	}
 	if (submit) {
-		/* Decouple all the requests submitted from the queue */
-		engine->execlist_queue.next = &cursor->execlist_link;
-		cursor->execlist_link.prev = &engine->execlist_queue;
-
 		i915_gem_request_assign(&port->request, last);
+		engine->execlist_first = rb;
 	}
-	spin_unlock(&engine->execlist_lock);
+	spin_unlock_irqrestore(&engine->timeline->lock, flags);
 
 	if (submit)
 		execlists_submit_ports(engine);
@@ -523,6 +542,28 @@ static bool execlists_elsp_idle(struct intel_engine_cs *engine)
 	return !engine->execlist_port[0].request;
 }
 
+/**
+ * intel_execlists_idle() - Determine if all engine submission ports are idle
+ * @dev_priv: i915 device private
+ *
+ * Return true if there are no requests pending on any of the submission ports
+ * of any engines.
+ */
+bool intel_execlists_idle(struct drm_i915_private *dev_priv)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	if (!i915.enable_execlists)
+		return true;
+
+	for_each_engine(engine, dev_priv, id)
+		if (!execlists_elsp_idle(engine))
+			return false;
+
+	return true;
+}
+
 static bool execlists_elsp_ready(struct intel_engine_cs *engine)
 {
 	int port;
@@ -593,18 +634,147 @@ static void intel_lrc_irq_handler(unsigned long data)
 	intel_uncore_forcewake_put(dev_priv, engine->fw_domains);
 }
 
+static bool insert_request(struct i915_priotree *pt, struct rb_root *root)
+{
+	struct rb_node **p, *rb;
+	bool first = true;
+
+	/* most positive priority is scheduled first, equal priorities fifo */
+	rb = NULL;
+	p = &root->rb_node;
+	while (*p) {
+		struct i915_priotree *pos;
+
+		rb = *p;
+		pos = rb_entry(rb, typeof(*pos), node);
+		if (pt->priority > pos->priority) {
+			p = &rb->rb_left;
+		} else {
+			p = &rb->rb_right;
+			first = false;
+		}
+	}
+	rb_link_node(&pt->node, rb, p);
+	rb_insert_color(&pt->node, root);
+
+	return first;
+}
+
 static void execlists_submit_request(struct drm_i915_gem_request *request)
 {
 	struct intel_engine_cs *engine = request->engine;
 	unsigned long flags;
 
-	spin_lock_irqsave(&engine->execlist_lock, flags);
+	/* Will be called from irq-context when using foreign fences. */
+	spin_lock_irqsave(&engine->timeline->lock, flags);
 
-	list_add_tail(&request->execlist_link, &engine->execlist_queue);
+	if (insert_request(&request->priotree, &engine->execlist_queue))
+		engine->execlist_first = &request->priotree.node;
 	if (execlists_elsp_idle(engine))
 		tasklet_hi_schedule(&engine->irq_tasklet);
 
-	spin_unlock_irqrestore(&engine->execlist_lock, flags);
+	spin_unlock_irqrestore(&engine->timeline->lock, flags);
+}
+
+static struct intel_engine_cs *
+pt_lock_engine(struct i915_priotree *pt, struct intel_engine_cs *locked)
+{
+	struct intel_engine_cs *engine;
+
+	engine = container_of(pt,
+			      struct drm_i915_gem_request,
+			      priotree)->engine;
+	if (engine != locked) {
+		if (locked)
+			spin_unlock_irq(&locked->timeline->lock);
+		spin_lock_irq(&engine->timeline->lock);
+	}
+
+	return engine;
+}
+
+static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
+{
+	static DEFINE_MUTEX(lock);
+	struct intel_engine_cs *engine = NULL;
+	struct i915_dependency *dep, *p;
+	struct i915_dependency stack;
+	LIST_HEAD(dfs);
+
+	if (prio <= READ_ONCE(request->priotree.priority))
+		return;
+
+	/* Need global lock to use the temporary link inside i915_dependency */
+	mutex_lock(&lock);
+
+	stack.signaler = &request->priotree;
+	list_add(&stack.dfs_link, &dfs);
+
+	/* Recursively bump all dependent priorities to match the new request.
+	 *
+	 * A naive approach would be to use recursion:
+	 * static void update_priorities(struct i915_priotree *pt, prio) {
+	 *	list_for_each_entry(dep, &pt->signalers_list, signal_link)
+	 *		update_priorities(dep->signal, prio)
+	 *	insert_request(pt);
+	 * }
+	 * but that may have unlimited recursion depth and so runs a very
+	 * real risk of overunning the kernel stack. Instead, we build
+	 * a flat list of all dependencies starting with the current request.
+	 * As we walk the list of dependencies, we add all of its dependencies
+	 * to the end of the list (this may include an already visited
+	 * request) and continue to walk onwards onto the new dependencies. The
+	 * end result is a topological list of requests in reverse order, the
+	 * last element in the list is the request we must execute first.
+	 */
+	list_for_each_entry_safe(dep, p, &dfs, dfs_link) {
+		struct i915_priotree *pt = dep->signaler;
+
+		list_for_each_entry(p, &pt->signalers_list, signal_link)
+			if (prio > READ_ONCE(p->signaler->priority))
+				list_move_tail(&p->dfs_link, &dfs);
+
+		p = list_next_entry(dep, dfs_link);
+		if (!RB_EMPTY_NODE(&pt->node))
+			continue;
+
+		engine = pt_lock_engine(pt, engine);
+
+		/* If it is not already in the rbtree, we can update the
+		 * priority inplace and skip over it (and its dependencies)
+		 * if it is referenced *again* as we descend the dfs.
+		 */
+		if (prio > pt->priority && RB_EMPTY_NODE(&pt->node)) {
+			pt->priority = prio;
+			list_del_init(&dep->dfs_link);
+		}
+	}
+
+	/* Fifo and depth-first replacement ensure our deps execute before us */
+	list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) {
+		struct i915_priotree *pt = dep->signaler;
+
+		INIT_LIST_HEAD(&dep->dfs_link);
+
+		engine = pt_lock_engine(pt, engine);
+
+		if (prio <= pt->priority)
+			continue;
+
+		GEM_BUG_ON(RB_EMPTY_NODE(&pt->node));
+
+		pt->priority = prio;
+		rb_erase(&pt->node, &engine->execlist_queue);
+		if (insert_request(pt, &engine->execlist_queue))
+			engine->execlist_first = &pt->node;
+	}
+
+	if (engine)
+		spin_unlock_irq(&engine->timeline->lock);
+
+	mutex_unlock(&lock);
+
+	/* XXX Do we need to preempt to make room for us and our deps? */
 }
 
 int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request)
@@ -672,46 +842,6 @@ err_unpin:
 	return ret;
 }
 
-/*
- * intel_logical_ring_advance() - advance the tail and prepare for submission
- * @request: Request to advance the logical ringbuffer of.
- *
- * The tail is updated in our logical ringbuffer struct, not in the actual context. What
- * really happens during submission is that the context and current tail will be placed
- * on a queue waiting for the ELSP to be ready to accept a new context submission. At that
- * point, the tail *inside* the context is updated and the ELSP written to.
- */
-static int
-intel_logical_ring_advance(struct drm_i915_gem_request *request)
-{
-	struct intel_ring *ring = request->ring;
-	struct intel_engine_cs *engine = request->engine;
-
-	intel_ring_advance(ring);
-	request->tail = ring->tail;
-
-	/*
-	 * Here we add two extra NOOPs as padding to avoid
-	 * lite restore of a context with HEAD==TAIL.
-	 *
-	 * Caller must reserve WA_TAIL_DWORDS for us!
-	 */
-	intel_ring_emit(ring, MI_NOOP);
-	intel_ring_emit(ring, MI_NOOP);
-	intel_ring_advance(ring);
-	request->wa_tail = ring->tail;
-
-	/* We keep the previous context alive until we retire the following
-	 * request. This ensures that any the context object is still pinned
-	 * for any residual writes the HW makes into it on the context switch
-	 * into the next object following the breadcrumb. Otherwise, we may
-	 * retire the context too early.
-	 */
-	request->previous_context = engine->last_context;
-	engine->last_context = request->ctx;
-	return 0;
-}
-
 static int intel_lr_context_pin(struct i915_gem_context *ctx,
 				struct intel_engine_cs *engine)
 {
@@ -745,7 +875,7 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx,
 	ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
 		i915_ggtt_offset(ce->ring->vma);
 
-	ce->state->obj->dirty = true;
+	ce->state->obj->mm.dirty = true;
 
 	/* Invalidate GuC TLB. */
 	if (i915.enable_guc_submission) {
@@ -853,13 +983,12 @@ static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine,
 	uint32_t l3sqc4_flush = (0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES);
 
 	/*
-	 * WaDisableLSQCROPERFforOCL:skl,kbl
+	 * WaDisableLSQCROPERFforOCL:kbl
 	 * This WA is implemented in skl_init_clock_gating() but since
 	 * this batch updates GEN8_L3SQCREG4 with default value we need to
 	 * set this bit here to retain the WA during flush.
 	 */
-	if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0) ||
-	    IS_KBL_REVID(dev_priv, 0, KBL_REVID_E0))
+	if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_E0))
 		l3sqc4_flush |= GEN8_LQSC_RO_PERF_DIS;
 
 	wa_ctx_emit(batch, index, (MI_STORE_REGISTER_MEM_GEN8 |
@@ -1002,9 +1131,8 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine,
 	struct drm_i915_private *dev_priv = engine->i915;
 	uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
 
-	/* WaDisableCtxRestoreArbitration:skl,bxt */
-	if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_D0) ||
-	    IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
+	/* WaDisableCtxRestoreArbitration:bxt */
+	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
 		wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE);
 
 	/* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt */
@@ -1075,9 +1203,8 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
 {
 	uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
 
-	/* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
-	if (IS_SKL_REVID(engine->i915, 0, SKL_REVID_B0) ||
-	    IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) {
+	/* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
+	if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) {
 		wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
 		wa_ctx_emit_reg(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0);
 		wa_ctx_emit(batch, index,
@@ -1104,9 +1231,8 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
 		wa_ctx_emit(batch, index, MI_NOOP);
 	}
 
-	/* WaDisableCtxRestoreArbitration:skl,bxt */
-	if (IS_SKL_REVID(engine->i915, 0, SKL_REVID_D0) ||
-	    IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1))
+	/* WaDisableCtxRestoreArbitration:bxt */
+	if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1))
 		wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE);
 
 	wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END);
@@ -1250,8 +1376,12 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine)
 
 	intel_engine_init_hangcheck(engine);
 
-	if (!execlists_elsp_idle(engine))
+	/* After a GPU reset, we may have requests to replay */
+	if (!execlists_elsp_idle(engine)) {
+		engine->execlist_port[0].count = 0;
+		engine->execlist_port[1].count = 0;
 		execlists_submit_ports(engine);
+	}
 
 	return 0;
 }
@@ -1326,10 +1456,7 @@ static void reset_common_ring(struct intel_engine_cs *engine,
 		memset(&port[1], 0, sizeof(port[1]));
 	}
 
-	/* CS is stopped, and we will resubmit both ports on resume */
 	GEM_BUG_ON(request->ctx != port[0].request->ctx);
-	port[0].count = 0;
-	port[1].count = 0;
 
 	/* Reset WaIdleLiteRestore:bdw,skl as well */
 	request->tail = request->wa_tail - WA_TAIL_DWORDS * sizeof(u32);
@@ -1570,39 +1697,35 @@ static void bxt_a_seqno_barrier(struct intel_engine_cs *engine)
  * used as a workaround for not being allowed to do lite
  * restore with HEAD==TAIL (WaIdleLiteRestore).
  */
-
-static int gen8_emit_request(struct drm_i915_gem_request *request)
+static void gen8_emit_wa_tail(struct drm_i915_gem_request *request, u32 *out)
 {
-	struct intel_ring *ring = request->ring;
-	int ret;
-
-	ret = intel_ring_begin(request, 6 + WA_TAIL_DWORDS);
-	if (ret)
-		return ret;
+	*out++ = MI_NOOP;
+	*out++ = MI_NOOP;
+	request->wa_tail = intel_ring_offset(request->ring, out);
+}
 
+static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request,
+				 u32 *out)
+{
 	/* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
 	BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
 
-	intel_ring_emit(ring, (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW);
-	intel_ring_emit(ring,
-			intel_hws_seqno_address(request->engine) |
-			MI_FLUSH_DW_USE_GTT);
-	intel_ring_emit(ring, 0);
-	intel_ring_emit(ring, request->fence.seqno);
-	intel_ring_emit(ring, MI_USER_INTERRUPT);
-	intel_ring_emit(ring, MI_NOOP);
-	return intel_logical_ring_advance(request);
-}
+	*out++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
+	*out++ = intel_hws_seqno_address(request->engine) | MI_FLUSH_DW_USE_GTT;
+	*out++ = 0;
+	*out++ = request->global_seqno;
+	*out++ = MI_USER_INTERRUPT;
+	*out++ = MI_NOOP;
+	request->tail = intel_ring_offset(request->ring, out);
 
-static int gen8_emit_request_render(struct drm_i915_gem_request *request)
-{
-	struct intel_ring *ring = request->ring;
-	int ret;
+	gen8_emit_wa_tail(request, out);
+}
 
-	ret = intel_ring_begin(request, 8 + WA_TAIL_DWORDS);
-	if (ret)
-		return ret;
+static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS;
 
+static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request,
+					u32 *out)
+{
 	/* We're using qword write, seqno should be aligned to 8 bytes. */
 	BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1);
 
@@ -1610,21 +1733,24 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request)
 	 * need a prior CS_STALL, which is emitted by the flush
 	 * following the batch.
 	 */
-	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
-	intel_ring_emit(ring,
-			(PIPE_CONTROL_GLOBAL_GTT_IVB |
-			 PIPE_CONTROL_CS_STALL |
-			 PIPE_CONTROL_QW_WRITE));
-	intel_ring_emit(ring, intel_hws_seqno_address(request->engine));
-	intel_ring_emit(ring, 0);
-	intel_ring_emit(ring, i915_gem_request_get_seqno(request));
+	*out++ = GFX_OP_PIPE_CONTROL(6);
+	*out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB |
+		  PIPE_CONTROL_CS_STALL |
+		  PIPE_CONTROL_QW_WRITE);
+	*out++ = intel_hws_seqno_address(request->engine);
+	*out++ = 0;
+	*out++ = request->global_seqno;
 	/* We're thrashing one dword of HWS. */
-	intel_ring_emit(ring, 0);
-	intel_ring_emit(ring, MI_USER_INTERRUPT);
-	intel_ring_emit(ring, MI_NOOP);
-	return intel_logical_ring_advance(request);
+	*out++ = 0;
+	*out++ = MI_USER_INTERRUPT;
+	*out++ = MI_NOOP;
+	request->tail = intel_ring_offset(request->ring, out);
+
+	gen8_emit_wa_tail(request, out);
 }
 
+static const int gen8_emit_breadcrumb_render_sz = 8 + WA_TAIL_DWORDS;
+
 static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
 {
 	int ret;
@@ -1641,7 +1767,7 @@ static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
 	if (ret)
 		DRM_ERROR("MOCS failed to program: expect performance issues.\n");
 
-	return i915_gem_render_state_init(req);
+	return i915_gem_render_state_emit(req);
 }
 
 /**
@@ -1652,9 +1778,6 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv;
 
-	if (!intel_engine_initialized(engine))
-		return;
-
 	/*
 	 * Tasklet cannot be active at this point due intel_mark_active/idle
 	 * so this is just for documentation.
@@ -1681,14 +1804,19 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
 
 	lrc_destroy_wa_ctx_obj(engine);
 	engine->i915 = NULL;
+	dev_priv->engine[engine->id] = NULL;
+	kfree(engine);
 }
 
 void intel_execlists_enable_submission(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id) {
 		engine->submit_request = execlists_submit_request;
+		engine->schedule = execlists_schedule;
+	}
 }
 
 static void
@@ -1698,8 +1826,10 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine)
 	engine->init_hw = gen8_init_common_ring;
 	engine->reset_hw = reset_common_ring;
 	engine->emit_flush = gen8_emit_flush;
-	engine->emit_request = gen8_emit_request;
+	engine->emit_breadcrumb = gen8_emit_breadcrumb;
+	engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz;
 	engine->submit_request = execlists_submit_request;
+	engine->schedule = execlists_schedule;
 
 	engine->irq_enable = gen8_logical_ring_enable_irq;
 	engine->irq_disable = gen8_logical_ring_disable_irq;
@@ -1820,7 +1950,8 @@ int logical_render_ring_init(struct intel_engine_cs *engine)
 		engine->init_hw = gen8_init_render_ring;
 	engine->init_context = gen8_init_rcs_context;
 	engine->emit_flush = gen8_emit_flush_render;
-	engine->emit_request = gen8_emit_request_render;
+	engine->emit_breadcrumb = gen8_emit_breadcrumb_render;
+	engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_render_sz;
 
 	ret = intel_engine_create_scratch(engine, 4096);
 	if (ret)
@@ -1945,7 +2076,7 @@ static void execlists_init_reg_state(u32 *reg_state,
 		       RING_START(engine->mmio_base), 0);
 	ASSIGN_CTX_REG(reg_state, CTX_RING_BUFFER_CONTROL,
 		       RING_CTL(engine->mmio_base),
-		       ((ring->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID);
+		       RING_CTL_SIZE(ring->size) | RING_VALID);
 	ASSIGN_CTX_REG(reg_state, CTX_BB_HEAD_U,
 		       RING_BBADDR_UDW(engine->mmio_base), 0);
 	ASSIGN_CTX_REG(reg_state, CTX_BB_HEAD_L,
@@ -2046,7 +2177,7 @@ populate_lr_context(struct i915_gem_context *ctx,
 		DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
 		return ret;
 	}
-	ctx_obj->dirty = true;
+	ctx_obj->mm.dirty = true;
 
 	/* The second page of the context object contains some fields which must
 	 * be set up prior to the first execution. */
@@ -2153,30 +2284,43 @@ error_deref_obj:
 
 void intel_lr_context_resume(struct drm_i915_private *dev_priv)
 {
-	struct i915_gem_context *ctx = dev_priv->kernel_context;
 	struct intel_engine_cs *engine;
+	struct i915_gem_context *ctx;
+	enum intel_engine_id id;
+
+	/* Because we emit WA_TAIL_DWORDS there may be a disparity
+	 * between our bookkeeping in ce->ring->head and ce->ring->tail and
+	 * that stored in context. As we only write new commands from
+	 * ce->ring->tail onwards, everything before that is junk. If the GPU
+	 * starts reading from its RING_HEAD from the context, it may try to
+	 * execute that junk and die.
+	 *
+	 * So to avoid that we reset the context images upon resume. For
+	 * simplicity, we just zero everything out.
+	 */
+	list_for_each_entry(ctx, &dev_priv->context_list, link) {
+		for_each_engine(engine, dev_priv, id) {
+			struct intel_context *ce = &ctx->engine[engine->id];
+			u32 *reg;
 
-	for_each_engine(engine, dev_priv) {
-		struct intel_context *ce = &ctx->engine[engine->id];
-		void *vaddr;
-		uint32_t *reg_state;
-
-		if (!ce->state)
-			continue;
-
-		vaddr = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB);
-		if (WARN_ON(IS_ERR(vaddr)))
-			continue;
+			if (!ce->state)
+				continue;
 
-		reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
+			reg = i915_gem_object_pin_map(ce->state->obj,
+						      I915_MAP_WB);
+			if (WARN_ON(IS_ERR(reg)))
+				continue;
 
-		reg_state[CTX_RING_HEAD+1] = 0;
-		reg_state[CTX_RING_TAIL+1] = 0;
+			reg += LRC_STATE_PN * PAGE_SIZE / sizeof(*reg);
+			reg[CTX_RING_HEAD+1] = 0;
+			reg[CTX_RING_TAIL+1] = 0;
 
-		ce->state->obj->dirty = true;
-		i915_gem_object_unpin_map(ce->state->obj);
+			ce->state->obj->mm.dirty = true;
+			i915_gem_object_unpin_map(ce->state->obj);
 
-		ce->ring->head = 0;
-		ce->ring->tail = 0;
+			ce->ring->head = ce->ring->tail = 0;
+			ce->ring->last_retired_head = -1;
+			intel_ring_update_space(ce->ring);
+		}
 	}
 }
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index 4fed8165f98a..c1f546180ba2 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -95,5 +95,6 @@ uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx,
 int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv,
 				    int enable_execlists);
 void intel_execlists_enable_submission(struct drm_i915_private *dev_priv);
+bool intel_execlists_idle(struct drm_i915_private *dev_priv);
 
 #endif /* _INTEL_LRC_H_ */
diff --git a/drivers/gpu/drm/i915/intel_lspcon.c b/drivers/gpu/drm/i915/intel_lspcon.c
new file mode 100644
index 000000000000..daa523410953
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_lspcon.c
@@ -0,0 +1,185 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ *
+ */
+#include <drm/drm_edid.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_dp_dual_mode_helper.h>
+#include "intel_drv.h"
+
+static struct intel_dp *lspcon_to_intel_dp(struct intel_lspcon *lspcon)
+{
+	struct intel_digital_port *dig_port =
+		container_of(lspcon, struct intel_digital_port, lspcon);
+
+	return &dig_port->dp;
+}
+
+static enum drm_lspcon_mode lspcon_get_current_mode(struct intel_lspcon *lspcon)
+{
+	enum drm_lspcon_mode current_mode = DRM_LSPCON_MODE_INVALID;
+	struct i2c_adapter *adapter = &lspcon_to_intel_dp(lspcon)->aux.ddc;
+
+	if (drm_lspcon_get_mode(adapter, &current_mode))
+		DRM_ERROR("Error reading LSPCON mode\n");
+	else
+		DRM_DEBUG_KMS("Current LSPCON mode %s\n",
+			current_mode == DRM_LSPCON_MODE_PCON ? "PCON" : "LS");
+	return current_mode;
+}
+
+static int lspcon_change_mode(struct intel_lspcon *lspcon,
+	enum drm_lspcon_mode mode, bool force)
+{
+	int err;
+	enum drm_lspcon_mode current_mode;
+	struct i2c_adapter *adapter = &lspcon_to_intel_dp(lspcon)->aux.ddc;
+
+	err = drm_lspcon_get_mode(adapter, &current_mode);
+	if (err) {
+		DRM_ERROR("Error reading LSPCON mode\n");
+		return err;
+	}
+
+	if (current_mode == mode) {
+		DRM_DEBUG_KMS("Current mode = desired LSPCON mode\n");
+		return 0;
+	}
+
+	err = drm_lspcon_set_mode(adapter, mode);
+	if (err < 0) {
+		DRM_ERROR("LSPCON mode change failed\n");
+		return err;
+	}
+
+	lspcon->mode = mode;
+	DRM_DEBUG_KMS("LSPCON mode changed done\n");
+	return 0;
+}
+
+static bool lspcon_probe(struct intel_lspcon *lspcon)
+{
+	enum drm_dp_dual_mode_type adaptor_type;
+	struct i2c_adapter *adapter = &lspcon_to_intel_dp(lspcon)->aux.ddc;
+
+	/* Lets probe the adaptor and check its type */
+	adaptor_type = drm_dp_dual_mode_detect(adapter);
+	if (adaptor_type != DRM_DP_DUAL_MODE_LSPCON) {
+		DRM_DEBUG_KMS("No LSPCON detected, found %s\n",
+			drm_dp_get_dual_mode_type_name(adaptor_type));
+		return false;
+	}
+
+	/* Yay ... got a LSPCON device */
+	DRM_DEBUG_KMS("LSPCON detected\n");
+	lspcon->mode = lspcon_get_current_mode(lspcon);
+	lspcon->active = true;
+	return true;
+}
+
+static void lspcon_resume_in_pcon_wa(struct intel_lspcon *lspcon)
+{
+	struct intel_dp *intel_dp = lspcon_to_intel_dp(lspcon);
+	unsigned long start = jiffies;
+
+	if (!lspcon->desc_valid)
+		return;
+
+	while (1) {
+		struct intel_dp_desc desc;
+
+		/*
+		 * The w/a only applies in PCON mode and we don't expect any
+		 * AUX errors.
+		 */
+		if (!__intel_dp_read_desc(intel_dp, &desc))
+			return;
+
+		if (!memcmp(&intel_dp->desc, &desc, sizeof(desc))) {
+			DRM_DEBUG_KMS("LSPCON recovering in PCON mode after %u ms\n",
+				      jiffies_to_msecs(jiffies - start));
+			return;
+		}
+
+		if (time_after(jiffies, start + msecs_to_jiffies(1000)))
+			break;
+
+		usleep_range(10000, 15000);
+	}
+
+	DRM_DEBUG_KMS("LSPCON DP descriptor mismatch after resume\n");
+}
+
+void lspcon_resume(struct intel_lspcon *lspcon)
+{
+	lspcon_resume_in_pcon_wa(lspcon);
+
+	if (lspcon_change_mode(lspcon, DRM_LSPCON_MODE_PCON, true))
+		DRM_ERROR("LSPCON resume failed\n");
+	else
+		DRM_DEBUG_KMS("LSPCON resume success\n");
+}
+
+bool lspcon_init(struct intel_digital_port *intel_dig_port)
+{
+	struct intel_dp *dp = &intel_dig_port->dp;
+	struct intel_lspcon *lspcon = &intel_dig_port->lspcon;
+	struct drm_device *dev = intel_dig_port->base.base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+
+	if (!IS_GEN9(dev_priv)) {
+		DRM_ERROR("LSPCON is supported on GEN9 only\n");
+		return false;
+	}
+
+	lspcon->active = false;
+	lspcon->mode = DRM_LSPCON_MODE_INVALID;
+
+	if (!lspcon_probe(lspcon)) {
+		DRM_ERROR("Failed to probe lspcon\n");
+		return false;
+	}
+
+	/*
+	* In the SW state machine, lets Put LSPCON in PCON mode only.
+	* In this way, it will work with both HDMI 1.4 sinks as well as HDMI
+	* 2.0 sinks.
+	*/
+	if (lspcon->active && lspcon->mode != DRM_LSPCON_MODE_PCON) {
+		if (lspcon_change_mode(lspcon, DRM_LSPCON_MODE_PCON,
+			true) < 0) {
+			DRM_ERROR("LSPCON mode change to PCON failed\n");
+			return false;
+		}
+	}
+
+	if (!intel_dp_read_dpcd(dp)) {
+		DRM_ERROR("LSPCON DPCD read failed\n");
+		return false;
+	}
+
+	lspcon->desc_valid = intel_dp_read_desc(dp);
+
+	DRM_DEBUG_KMS("Success: LSPCON init\n");
+	return true;
+}
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index e1d47d51ea47..d12ef0047d49 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -106,7 +106,7 @@ static bool intel_lvds_get_hw_state(struct intel_encoder *encoder,
 	if (!(tmp & LVDS_PORT_EN))
 		goto out;
 
-	if (HAS_PCH_CPT(dev))
+	if (HAS_PCH_CPT(dev_priv))
 		*pipe = PORT_TO_PIPE_CPT(tmp);
 	else
 		*pipe = PORT_TO_PIPE(tmp);
@@ -122,8 +122,7 @@ out:
 static void intel_lvds_get_config(struct intel_encoder *encoder,
 				  struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
 	u32 tmp, flags = 0;
 
@@ -139,12 +138,12 @@ static void intel_lvds_get_config(struct intel_encoder *encoder,
 
 	pipe_config->base.adjusted_mode.flags |= flags;
 
-	if (INTEL_INFO(dev)->gen < 5)
+	if (INTEL_GEN(dev_priv) < 5)
 		pipe_config->gmch_pfit.lvds_border_bits =
 			tmp & LVDS_BORDER_ENABLE;
 
 	/* gen2/3 store dither state in pfit control, needs to match */
-	if (INTEL_INFO(dev)->gen < 4) {
+	if (INTEL_GEN(dev_priv) < 4) {
 		tmp = I915_READ(PFIT_CONTROL);
 
 		pipe_config->gmch_pfit.control |= tmp & PANEL_8TO6_DITHER_ENABLE;
@@ -396,7 +395,7 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
 				      struct intel_crtc_state *pipe_config,
 				      struct drm_connector_state *conn_state)
 {
-	struct drm_device *dev = intel_encoder->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
 	struct intel_lvds_encoder *lvds_encoder =
 		to_lvds_encoder(&intel_encoder->base);
 	struct intel_connector *intel_connector =
@@ -406,7 +405,7 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
 	unsigned int lvds_bpp;
 
 	/* Should never happen!! */
-	if (INTEL_INFO(dev)->gen < 4 && intel_crtc->pipe == 0) {
+	if (INTEL_GEN(dev_priv) < 4 && intel_crtc->pipe == 0) {
 		DRM_ERROR("Can't support LVDS on pipe A\n");
 		return false;
 	}
@@ -431,7 +430,7 @@ static bool intel_lvds_compute_config(struct intel_encoder *intel_encoder,
 	intel_fixed_panel_mode(intel_connector->panel.fixed_mode,
 			       adjusted_mode);
 
-	if (HAS_PCH_SPLIT(dev)) {
+	if (HAS_PCH_SPLIT(dev_priv)) {
 		pipe_config->has_pch_encoder = true;
 
 		intel_pch_panel_fitting(intel_crtc, pipe_config,
@@ -566,7 +565,7 @@ static int intel_lid_notify(struct notifier_block *nb, unsigned long val,
 	 * and as part of the cleanup in the hw state restore we also redisable
 	 * the vga plane.
 	 */
-	if (!HAS_PCH_SPLIT(dev))
+	if (!HAS_PCH_SPLIT(dev_priv))
 		intel_display_resume(dev);
 
 	dev_priv->modeset_restore = MODESET_DONE;
@@ -949,16 +948,17 @@ static bool compute_is_dual_link_lvds(struct intel_lvds_encoder *lvds_encoder)
 	return (val & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP;
 }
 
-static bool intel_lvds_supported(struct drm_device *dev)
+static bool intel_lvds_supported(struct drm_i915_private *dev_priv)
 {
 	/* With the introduction of the PCH we gained a dedicated
 	 * LVDS presence pin, use it. */
-	if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
+	if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))
 		return true;
 
 	/* Otherwise LVDS was only attached to mobile products,
 	 * except for the inglorious 830gm */
-	if (INTEL_INFO(dev)->gen <= 4 && IS_MOBILE(dev) && !IS_I830(dev))
+	if (INTEL_GEN(dev_priv) <= 4 &&
+	    IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
 		return true;
 
 	return false;
@@ -984,27 +984,27 @@ void intel_lvds_init(struct drm_device *dev)
 	struct drm_display_mode *fixed_mode = NULL;
 	struct drm_display_mode *downclock_mode = NULL;
 	struct edid *edid;
-	struct drm_crtc *crtc;
+	struct intel_crtc *crtc;
 	i915_reg_t lvds_reg;
 	u32 lvds;
 	int pipe;
 	u8 pin;
 
-	if (!intel_lvds_supported(dev))
+	if (!intel_lvds_supported(dev_priv))
 		return;
 
 	/* Skip init on machines we know falsely report LVDS */
 	if (dmi_check_system(intel_no_lvds))
 		return;
 
-	if (HAS_PCH_SPLIT(dev))
+	if (HAS_PCH_SPLIT(dev_priv))
 		lvds_reg = PCH_LVDS;
 	else
 		lvds_reg = LVDS;
 
 	lvds = I915_READ(lvds_reg);
 
-	if (HAS_PCH_SPLIT(dev)) {
+	if (HAS_PCH_SPLIT(dev_priv)) {
 		if ((lvds & LVDS_DETECTED) == 0)
 			return;
 		if (dev_priv->vbt.edp.support) {
@@ -1064,12 +1064,13 @@ void intel_lvds_init(struct drm_device *dev)
 	intel_connector->get_hw_state = intel_connector_get_hw_state;
 
 	intel_connector_attach_encoder(intel_connector, intel_encoder);
-	intel_encoder->type = INTEL_OUTPUT_LVDS;
 
+	intel_encoder->type = INTEL_OUTPUT_LVDS;
+	intel_encoder->port = PORT_NONE;
 	intel_encoder->cloneable = 0;
-	if (HAS_PCH_SPLIT(dev))
+	if (HAS_PCH_SPLIT(dev_priv))
 		intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
-	else if (IS_GEN4(dev))
+	else if (IS_GEN4(dev_priv))
 		intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
 	else
 		intel_encoder->crtc_mask = (1 << 1);
@@ -1157,14 +1158,14 @@ void intel_lvds_init(struct drm_device *dev)
 	 */
 
 	/* Ironlake: FIXME if still fail, not try pipe mode now */
-	if (HAS_PCH_SPLIT(dev))
+	if (HAS_PCH_SPLIT(dev_priv))
 		goto failed;
 
 	pipe = (lvds & LVDS_PIPEB_SELECT) ? 1 : 0;
-	crtc = intel_get_crtc_for_pipe(dev, pipe);
+	crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 
 	if (crtc && (lvds & LVDS_PORT_EN)) {
-		fixed_mode = intel_crtc_mode_get(dev, crtc);
+		fixed_mode = intel_crtc_mode_get(dev, &crtc->base);
 		if (fixed_mode) {
 			DRM_DEBUG_KMS("using current (BIOS) mode: ");
 			drm_mode_debug_printmodeline(fixed_mode);
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index 7acbbbf97833..f4429f67a4e3 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -642,24 +642,6 @@ static struct notifier_block intel_opregion_notifier = {
  * (version 3)
  */
 
-static u32 get_did(struct intel_opregion *opregion, int i)
-{
-	u32 did;
-
-	if (i < ARRAY_SIZE(opregion->acpi->didl)) {
-		did = opregion->acpi->didl[i];
-	} else {
-		i -= ARRAY_SIZE(opregion->acpi->didl);
-
-		if (WARN_ON(i >= ARRAY_SIZE(opregion->acpi->did2)))
-			return 0;
-
-		did = opregion->acpi->did2[i];
-	}
-
-	return did;
-}
-
 static void set_did(struct intel_opregion *opregion, int i, u32 val)
 {
 	if (i < ARRAY_SIZE(opregion->acpi->didl)) {
@@ -674,11 +656,11 @@ static void set_did(struct intel_opregion *opregion, int i, u32 val)
 	}
 }
 
-static u32 acpi_display_type(struct drm_connector *connector)
+static u32 acpi_display_type(struct intel_connector *connector)
 {
 	u32 display_type;
 
-	switch (connector->connector_type) {
+	switch (connector->base.connector_type) {
 	case DRM_MODE_CONNECTOR_VGA:
 	case DRM_MODE_CONNECTOR_DVIA:
 		display_type = ACPI_DISPLAY_TYPE_VGA;
@@ -707,7 +689,7 @@ static u32 acpi_display_type(struct drm_connector *connector)
 		display_type = ACPI_DISPLAY_TYPE_OTHER;
 		break;
 	default:
-		MISSING_CASE(connector->connector_type);
+		MISSING_CASE(connector->base.connector_type);
 		display_type = ACPI_DISPLAY_TYPE_OTHER;
 		break;
 	}
@@ -718,34 +700,9 @@ static u32 acpi_display_type(struct drm_connector *connector)
 static void intel_didl_outputs(struct drm_i915_private *dev_priv)
 {
 	struct intel_opregion *opregion = &dev_priv->opregion;
-	struct pci_dev *pdev = dev_priv->drm.pdev;
-	struct drm_connector *connector;
-	acpi_handle handle;
-	struct acpi_device *acpi_dev, *acpi_cdev, *acpi_video_bus = NULL;
-	unsigned long long device_id;
-	acpi_status status;
-	u32 temp, max_outputs;
-	int i = 0;
-
-	handle = ACPI_HANDLE(&pdev->dev);
-	if (!handle || acpi_bus_get_device(handle, &acpi_dev))
-		return;
-
-	if (acpi_is_video_device(handle))
-		acpi_video_bus = acpi_dev;
-	else {
-		list_for_each_entry(acpi_cdev, &acpi_dev->children, node) {
-			if (acpi_is_video_device(acpi_cdev->handle)) {
-				acpi_video_bus = acpi_cdev;
-				break;
-			}
-		}
-	}
-
-	if (!acpi_video_bus) {
-		DRM_DEBUG_KMS("No ACPI video bus found\n");
-		return;
-	}
+	struct intel_connector *connector;
+	int i = 0, max_outputs;
+	int display_index[16] = {};
 
 	/*
 	 * In theory, did2, the extended didl, gets added at opregion version
@@ -757,64 +714,58 @@ static void intel_didl_outputs(struct drm_i915_private *dev_priv)
 	max_outputs = ARRAY_SIZE(opregion->acpi->didl) +
 		ARRAY_SIZE(opregion->acpi->did2);
 
-	list_for_each_entry(acpi_cdev, &acpi_video_bus->children, node) {
-		if (i >= max_outputs) {
-			DRM_DEBUG_KMS("More than %u outputs detected via ACPI\n",
-				      max_outputs);
-			return;
-		}
-		status = acpi_evaluate_integer(acpi_cdev->handle, "_ADR",
-					       NULL, &device_id);
-		if (ACPI_SUCCESS(status)) {
-			if (!device_id)
-				goto blind_set;
-			set_did(opregion, i++, (u32)(device_id & 0x0f0f));
-		}
+	for_each_intel_connector(&dev_priv->drm, connector) {
+		u32 device_id, type;
+
+		device_id = acpi_display_type(connector);
+
+		/* Use display type specific display index. */
+		type = (device_id & ACPI_DISPLAY_TYPE_MASK)
+			>> ACPI_DISPLAY_TYPE_SHIFT;
+		device_id |= display_index[type]++ << ACPI_DISPLAY_INDEX_SHIFT;
+
+		connector->acpi_device_id = device_id;
+		if (i < max_outputs)
+			set_did(opregion, i, device_id);
+		i++;
 	}
 
-end:
 	DRM_DEBUG_KMS("%d outputs detected\n", i);
 
+	if (i > max_outputs)
+		DRM_ERROR("More than %d outputs in connector list\n",
+			  max_outputs);
+
 	/* If fewer than max outputs, the list must be null terminated */
 	if (i < max_outputs)
 		set_did(opregion, i, 0);
-	return;
-
-blind_set:
-	i = 0;
-	list_for_each_entry(connector,
-			    &dev_priv->drm.mode_config.connector_list, head) {
-		int display_type = acpi_display_type(connector);
-
-		if (i >= max_outputs) {
-			DRM_DEBUG_KMS("More than %u outputs in connector list\n",
-				      max_outputs);
-			return;
-		}
-
-		temp = get_did(opregion, i);
-		set_did(opregion, i, temp | (1 << 31) | display_type | i);
-		i++;
-	}
-	goto end;
 }
 
 static void intel_setup_cadls(struct drm_i915_private *dev_priv)
 {
 	struct intel_opregion *opregion = &dev_priv->opregion;
+	struct intel_connector *connector;
 	int i = 0;
-	u32 disp_id;
-
-	/* Initialize the CADL field by duplicating the DIDL values.
-	 * Technically, this is not always correct as display outputs may exist,
-	 * but not active. This initialization is necessary for some Clevo
-	 * laptops that check this field before processing the brightness and
-	 * display switching hotkeys. Just like DIDL, CADL is NULL-terminated if
-	 * there are less than eight devices. */
-	do {
-		disp_id = get_did(opregion, i);
-		opregion->acpi->cadl[i] = disp_id;
-	} while (++i < 8 && disp_id != 0);
+
+	/*
+	 * Initialize the CADL field from the connector device ids. This is
+	 * essentially the same as copying from the DIDL. Technically, this is
+	 * not always correct as display outputs may exist, but not active. This
+	 * initialization is necessary for some Clevo laptops that check this
+	 * field before processing the brightness and display switching hotkeys.
+	 *
+	 * Note that internal panels should be at the front of the connector
+	 * list already, ensuring they're not left out.
+	 */
+	for_each_intel_connector(&dev_priv->drm, connector) {
+		if (i >= ARRAY_SIZE(opregion->acpi->cadl))
+			break;
+		opregion->acpi->cadl[i++] = connector->acpi_device_id;
+	}
+
+	/* If fewer than 8 active devices, the list must be null terminated */
+	if (i < ARRAY_SIZE(opregion->acpi->cadl))
+		opregion->acpi->cadl[i] = 0;
 }
 
 void intel_opregion_register(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index a24bc8c7889f..fd0e4dac7cc1 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -233,7 +233,7 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
 static struct drm_i915_gem_request *alloc_request(struct intel_overlay *overlay)
 {
 	struct drm_i915_private *dev_priv = overlay->i915;
-	struct intel_engine_cs *engine = &dev_priv->engine[RCS];
+	struct intel_engine_cs *engine = dev_priv->engine[RCS];
 
 	return i915_gem_request_alloc(engine, dev_priv->kernel_context);
 }
@@ -1222,7 +1222,7 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
 out_unlock:
 	mutex_unlock(&dev->struct_mutex);
 	drm_modeset_unlock_all(dev);
-	i915_gem_object_put_unlocked(new_bo);
+	i915_gem_object_put(new_bo);
 out_free:
 	kfree(params);
 
@@ -1466,10 +1466,12 @@ void intel_cleanup_overlay(struct drm_i915_private *dev_priv)
 	 * hardware should be off already */
 	WARN_ON(dev_priv->overlay->active);
 
-	i915_gem_object_put_unlocked(dev_priv->overlay->reg_bo);
+	i915_gem_object_put(dev_priv->overlay->reg_bo);
 	kfree(dev_priv->overlay);
 }
 
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+
 struct intel_overlay_error_state {
 	struct overlay_registers regs;
 	unsigned long base;
@@ -1587,3 +1589,5 @@ intel_overlay_print_error_state(struct drm_i915_error_state_buf *m,
 	P(UVSCALEV);
 #undef P
 }
+
+#endif
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index be4b4d546fd9..08ab6d762ca4 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -304,7 +304,7 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc,
 			      struct intel_crtc_state *pipe_config,
 			      int fitting_mode)
 {
-	struct drm_device *dev = intel_crtc->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
 	u32 pfit_control = 0, pfit_pgm_ratios = 0, border = 0;
 	struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 
@@ -325,7 +325,7 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc,
 		break;
 	case DRM_MODE_SCALE_ASPECT:
 		/* Scale but preserve the aspect ratio */
-		if (INTEL_INFO(dev)->gen >= 4)
+		if (INTEL_GEN(dev_priv) >= 4)
 			i965_scale_aspect(pipe_config, &pfit_control);
 		else
 			i9xx_scale_aspect(pipe_config, &pfit_control,
@@ -339,7 +339,7 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc,
 		if (pipe_config->pipe_src_h != adjusted_mode->crtc_vdisplay ||
 		    pipe_config->pipe_src_w != adjusted_mode->crtc_hdisplay) {
 			pfit_control |= PFIT_ENABLE;
-			if (INTEL_INFO(dev)->gen >= 4)
+			if (INTEL_GEN(dev_priv) >= 4)
 				pfit_control |= PFIT_SCALING_AUTO;
 			else
 				pfit_control |= (VERT_AUTO_SCALE |
@@ -355,7 +355,7 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc,
 
 	/* 965+ wants fuzzy fitting */
 	/* FIXME: handle multiple panels by failing gracefully */
-	if (INTEL_INFO(dev)->gen >= 4)
+	if (INTEL_GEN(dev_priv) >= 4)
 		pfit_control |= ((intel_crtc->pipe << PFIT_PIPE_SHIFT) |
 				 PFIT_FILTER_FUZZY);
 
@@ -366,7 +366,7 @@ out:
 	}
 
 	/* Make sure pre-965 set dither correctly for 18bpp panels. */
-	if (INTEL_INFO(dev)->gen < 4 && pipe_config->pipe_bpp == 18)
+	if (INTEL_GEN(dev_priv) < 4 && pipe_config->pipe_bpp == 18)
 		pfit_control |= PANEL_8TO6_DITHER_ENABLE;
 
 	pipe_config->gmch_pfit.control = pfit_control;
@@ -1722,7 +1722,7 @@ int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe)
 
 	DRM_DEBUG_KMS("Connector %s backlight initialized, %s, brightness %u/%u\n",
 		      connector->name,
-		      panel->backlight.enabled ? "enabled" : "disabled",
+		      enableddisabled(panel->backlight.enabled),
 		      panel->backlight.level, panel->backlight.max);
 
 	return 0;
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index db24f898853c..d67974eb127a 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -31,6 +31,7 @@
 #include "intel_drv.h"
 #include "../../../platform/x86/intel_ips.h"
 #include <linux/module.h>
+#include <drm/drm_atomic_helper.h>
 
 /**
  * DOC: RC6
@@ -55,10 +56,8 @@
 #define INTEL_RC6p_ENABLE			(1<<1)
 #define INTEL_RC6pp_ENABLE			(1<<2)
 
-static void gen9_init_clock_gating(struct drm_device *dev)
+static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
 	/* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl */
 	I915_WRITE(CHICKEN_PAR1_1,
 		   I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
@@ -81,11 +80,9 @@ static void gen9_init_clock_gating(struct drm_device *dev)
 		   ILK_DPFC_DISABLE_DUMMY0);
 }
 
-static void bxt_init_clock_gating(struct drm_device *dev)
+static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	gen9_init_clock_gating(dev);
+	gen9_init_clock_gating(dev_priv);
 
 	/* WaDisableSDEUnitClockGating:bxt */
 	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
@@ -107,9 +104,8 @@ static void bxt_init_clock_gating(struct drm_device *dev)
 			   PWM1_GATING_DIS | PWM2_GATING_DIS);
 }
 
-static void i915_pineview_get_mem_freq(struct drm_device *dev)
+static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 tmp;
 
 	tmp = I915_READ(CLKCFG);
@@ -146,9 +142,8 @@ static void i915_pineview_get_mem_freq(struct drm_device *dev)
 	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 }
 
-static void i915_ironlake_get_mem_freq(struct drm_device *dev)
+static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	u16 ddrpll, csipll;
 
 	ddrpll = I915_READ16(DDRMPLL1);
@@ -252,8 +247,8 @@ static const struct cxsr_latency cxsr_latency_table[] = {
 	{0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 };
 
-static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
-							 int is_ddr3,
+static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
+							 bool is_ddr3,
 							 int fsb,
 							 int mem)
 {
@@ -319,27 +314,26 @@ static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 
 void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	u32 val;
 
-	if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 		POSTING_READ(FW_BLC_SELF_VLV);
 		dev_priv->wm.vlv.cxsr = enable;
-	} else if (IS_G4X(dev) || IS_CRESTLINE(dev)) {
+	} else if (IS_G4X(dev_priv) || IS_CRESTLINE(dev_priv)) {
 		I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 		POSTING_READ(FW_BLC_SELF);
-	} else if (IS_PINEVIEW(dev)) {
+	} else if (IS_PINEVIEW(dev_priv)) {
 		val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
 		val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
 		I915_WRITE(DSPFW3, val);
 		POSTING_READ(DSPFW3);
-	} else if (IS_I945G(dev) || IS_I945GM(dev)) {
+	} else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 		val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 			       _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 		I915_WRITE(FW_BLC_SELF, val);
 		POSTING_READ(FW_BLC_SELF);
-	} else if (IS_I915GM(dev)) {
+	} else if (IS_I915GM(dev_priv)) {
 		/*
 		 * FIXME can't find a bit like this for 915G, and
 		 * and yet it does have the related watermark in
@@ -353,8 +347,7 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 		return;
 	}
 
-	DRM_DEBUG_KMS("memory self-refresh is %s\n",
-		      enable ? "enabled" : "disabled");
+	DRM_DEBUG_KMS("memory self-refresh is %s\n", enableddisabled(enable));
 }
 
 
@@ -377,10 +370,9 @@ static const int pessimal_latency_ns = 5000;
 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 	((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 
-static int vlv_get_fifo_size(struct drm_device *dev,
+static int vlv_get_fifo_size(struct drm_i915_private *dev_priv,
 			      enum pipe pipe, int plane)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	int sprite0_start, sprite1_start, size;
 
 	switch (pipe) {
@@ -429,9 +421,8 @@ static int vlv_get_fifo_size(struct drm_device *dev,
 	return size;
 }
 
-static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
+static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, int plane)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t dsparb = I915_READ(DSPARB);
 	int size;
 
@@ -445,9 +436,8 @@ static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
 	return size;
 }
 
-static int i830_get_fifo_size(struct drm_device *dev, int plane)
+static int i830_get_fifo_size(struct drm_i915_private *dev_priv, int plane)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t dsparb = I915_READ(DSPARB);
 	int size;
 
@@ -462,9 +452,8 @@ static int i830_get_fifo_size(struct drm_device *dev, int plane)
 	return size;
 }
 
-static int i845_get_fifo_size(struct drm_device *dev, int plane)
+static int i845_get_fifo_size(struct drm_i915_private *dev_priv, int plane)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t dsparb = I915_READ(DSPARB);
 	int size;
 
@@ -624,11 +613,11 @@ static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
 	return wm_size;
 }
 
-static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
+static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 {
-	struct drm_crtc *crtc, *enabled = NULL;
+	struct intel_crtc *crtc, *enabled = NULL;
 
-	for_each_crtc(dev, crtc) {
+	for_each_intel_crtc(&dev_priv->drm, crtc) {
 		if (intel_crtc_active(crtc)) {
 			if (enabled)
 				return NULL;
@@ -639,27 +628,31 @@ static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
 	return enabled;
 }
 
-static void pineview_update_wm(struct drm_crtc *unused_crtc)
+static void pineview_update_wm(struct intel_crtc *unused_crtc)
 {
-	struct drm_device *dev = unused_crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_crtc *crtc;
+	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
+	struct intel_crtc *crtc;
 	const struct cxsr_latency *latency;
 	u32 reg;
 	unsigned long wm;
 
-	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
-					 dev_priv->fsb_freq, dev_priv->mem_freq);
+	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
+					 dev_priv->is_ddr3,
+					 dev_priv->fsb_freq,
+					 dev_priv->mem_freq);
 	if (!latency) {
 		DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 		intel_set_memory_cxsr(dev_priv, false);
 		return;
 	}
 
-	crtc = single_enabled_crtc(dev);
+	crtc = single_enabled_crtc(dev_priv);
 	if (crtc) {
-		const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
-		int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
+		const struct drm_display_mode *adjusted_mode =
+			&crtc->config->base.adjusted_mode;
+		const struct drm_framebuffer *fb =
+			crtc->base.primary->state->fb;
+		int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 		int clock = adjusted_mode->crtc_clock;
 
 		/* Display SR */
@@ -706,7 +699,7 @@ static void pineview_update_wm(struct drm_crtc *unused_crtc)
 	}
 }
 
-static bool g4x_compute_wm0(struct drm_device *dev,
+static bool g4x_compute_wm0(struct drm_i915_private *dev_priv,
 			    int plane,
 			    const struct intel_watermark_params *display,
 			    int display_latency_ns,
@@ -715,24 +708,26 @@ static bool g4x_compute_wm0(struct drm_device *dev,
 			    int *plane_wm,
 			    int *cursor_wm)
 {
-	struct drm_crtc *crtc;
+	struct intel_crtc *crtc;
 	const struct drm_display_mode *adjusted_mode;
+	const struct drm_framebuffer *fb;
 	int htotal, hdisplay, clock, cpp;
 	int line_time_us, line_count;
 	int entries, tlb_miss;
 
-	crtc = intel_get_crtc_for_plane(dev, plane);
+	crtc = intel_get_crtc_for_plane(dev_priv, plane);
 	if (!intel_crtc_active(crtc)) {
 		*cursor_wm = cursor->guard_size;
 		*plane_wm = display->guard_size;
 		return false;
 	}
 
-	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
+	adjusted_mode = &crtc->config->base.adjusted_mode;
+	fb = crtc->base.primary->state->fb;
 	clock = adjusted_mode->crtc_clock;
 	htotal = adjusted_mode->crtc_htotal;
-	hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
-	cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
+	hdisplay = crtc->config->pipe_src_w;
+	cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 
 	/* Use the small buffer method to calculate plane watermark */
 	entries = ((clock * cpp / 1000) * display_latency_ns) / 1000;
@@ -747,7 +742,7 @@ static bool g4x_compute_wm0(struct drm_device *dev,
 	/* Use the large buffer method to calculate cursor watermark */
 	line_time_us = max(htotal * 1000 / clock, 1);
 	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
-	entries = line_count * crtc->cursor->state->crtc_w * cpp;
+	entries = line_count * crtc->base.cursor->state->crtc_w * cpp;
 	tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
 	if (tlb_miss > 0)
 		entries += tlb_miss;
@@ -766,7 +761,7 @@ static bool g4x_compute_wm0(struct drm_device *dev,
  * can be programmed into the associated watermark register, that watermark
  * must be disabled.
  */
-static bool g4x_check_srwm(struct drm_device *dev,
+static bool g4x_check_srwm(struct drm_i915_private *dev_priv,
 			   int display_wm, int cursor_wm,
 			   const struct intel_watermark_params *display,
 			   const struct intel_watermark_params *cursor)
@@ -775,13 +770,13 @@ static bool g4x_check_srwm(struct drm_device *dev,
 		      display_wm, cursor_wm);
 
 	if (display_wm > display->max_wm) {
-		DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
+		DRM_DEBUG_KMS("display watermark is too large(%d/%u), disabling\n",
 			      display_wm, display->max_wm);
 		return false;
 	}
 
 	if (cursor_wm > cursor->max_wm) {
-		DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
+		DRM_DEBUG_KMS("cursor watermark is too large(%d/%u), disabling\n",
 			      cursor_wm, cursor->max_wm);
 		return false;
 	}
@@ -794,15 +789,16 @@ static bool g4x_check_srwm(struct drm_device *dev,
 	return true;
 }
 
-static bool g4x_compute_srwm(struct drm_device *dev,
+static bool g4x_compute_srwm(struct drm_i915_private *dev_priv,
 			     int plane,
 			     int latency_ns,
 			     const struct intel_watermark_params *display,
 			     const struct intel_watermark_params *cursor,
 			     int *display_wm, int *cursor_wm)
 {
-	struct drm_crtc *crtc;
+	struct intel_crtc *crtc;
 	const struct drm_display_mode *adjusted_mode;
+	const struct drm_framebuffer *fb;
 	int hdisplay, htotal, cpp, clock;
 	unsigned long line_time_us;
 	int line_count, line_size;
@@ -814,12 +810,13 @@ static bool g4x_compute_srwm(struct drm_device *dev,
 		return false;
 	}
 
-	crtc = intel_get_crtc_for_plane(dev, plane);
-	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
+	crtc = intel_get_crtc_for_plane(dev_priv, plane);
+	adjusted_mode = &crtc->config->base.adjusted_mode;
+	fb = crtc->base.primary->state->fb;
 	clock = adjusted_mode->crtc_clock;
 	htotal = adjusted_mode->crtc_htotal;
-	hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
-	cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
+	hdisplay = crtc->config->pipe_src_w;
+	cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 
 	line_time_us = max(htotal * 1000 / clock, 1);
 	line_count = (latency_ns / line_time_us + 1000) / 1000;
@@ -833,11 +830,11 @@ static bool g4x_compute_srwm(struct drm_device *dev,
 	*display_wm = entries + display->guard_size;
 
 	/* calculate the self-refresh watermark for display cursor */
-	entries = line_count * cpp * crtc->cursor->state->crtc_w;
+	entries = line_count * cpp * crtc->base.cursor->state->crtc_w;
 	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
 	*cursor_wm = entries + cursor->guard_size;
 
-	return g4x_check_srwm(dev,
+	return g4x_check_srwm(dev_priv,
 			      *display_wm, *cursor_wm,
 			      display, cursor);
 }
@@ -937,10 +934,8 @@ static unsigned int vlv_wm_method2(unsigned int pixel_rate,
 	return ret;
 }
 
-static void vlv_setup_wm_latency(struct drm_device *dev)
+static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	/* all latencies in usec */
 	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
 
@@ -1065,7 +1060,8 @@ static void vlv_invert_wms(struct intel_crtc *crtc)
 
 	for (level = 0; level < wm_state->num_levels; level++) {
 		struct drm_device *dev = crtc->base.dev;
-		const int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
+		const int sr_fifo_size =
+			INTEL_INFO(to_i915(dev))->num_pipes * 512 - 1;
 		struct intel_plane *plane;
 
 		wm_state->sr[level].plane = sr_fifo_size - wm_state->sr[level].plane;
@@ -1095,15 +1091,16 @@ static void vlv_invert_wms(struct intel_crtc *crtc)
 static void vlv_compute_wm(struct intel_crtc *crtc)
 {
 	struct drm_device *dev = crtc->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct vlv_wm_state *wm_state = &crtc->wm_state;
 	struct intel_plane *plane;
-	int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
+	int sr_fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
 	int level;
 
 	memset(wm_state, 0, sizeof(*wm_state));
 
 	wm_state->cxsr = crtc->pipe != PIPE_C && crtc->wm.cxsr_allowed;
-	wm_state->num_levels = to_i915(dev)->wm.max_level + 1;
+	wm_state->num_levels = dev_priv->wm.max_level + 1;
 
 	wm_state->num_active_planes = 0;
 
@@ -1183,7 +1180,7 @@ static void vlv_compute_wm(struct intel_crtc *crtc)
 	}
 
 	/* clear any (partially) filled invalid levels */
-	for (level = wm_state->num_levels; level < to_i915(dev)->wm.max_level + 1; level++) {
+	for (level = wm_state->num_levels; level < dev_priv->wm.max_level + 1; level++) {
 		memset(&wm_state->wm[level], 0, sizeof(wm_state->wm[level]));
 		memset(&wm_state->sr[level], 0, sizeof(wm_state->sr[level]));
 	}
@@ -1327,20 +1324,19 @@ static void vlv_merge_wm(struct drm_device *dev,
 	}
 }
 
-static void vlv_update_wm(struct drm_crtc *crtc)
+static void vlv_update_wm(struct intel_crtc *crtc)
 {
-	struct drm_device *dev = crtc->dev;
+	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	enum pipe pipe = intel_crtc->pipe;
+	enum pipe pipe = crtc->pipe;
 	struct vlv_wm_values wm = {};
 
-	vlv_compute_wm(intel_crtc);
+	vlv_compute_wm(crtc);
 	vlv_merge_wm(dev, &wm);
 
 	if (memcmp(&dev_priv->wm.vlv, &wm, sizeof(wm)) == 0) {
 		/* FIXME should be part of crtc atomic commit */
-		vlv_pipe_set_fifo_size(intel_crtc);
+		vlv_pipe_set_fifo_size(crtc);
 		return;
 	}
 
@@ -1356,9 +1352,9 @@ static void vlv_update_wm(struct drm_crtc *crtc)
 		intel_set_memory_cxsr(dev_priv, false);
 
 	/* FIXME should be part of crtc atomic commit */
-	vlv_pipe_set_fifo_size(intel_crtc);
+	vlv_pipe_set_fifo_size(crtc);
 
-	vlv_write_wm_values(intel_crtc, &wm);
+	vlv_write_wm_values(crtc, &wm);
 
 	DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, "
 		      "sprite0=%d, sprite1=%d, SR: plane=%d, cursor=%d level=%d cxsr=%d\n",
@@ -1382,30 +1378,29 @@ static void vlv_update_wm(struct drm_crtc *crtc)
 
 #define single_plane_enabled(mask) is_power_of_2(mask)
 
-static void g4x_update_wm(struct drm_crtc *crtc)
+static void g4x_update_wm(struct intel_crtc *crtc)
 {
-	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	static const int sr_latency_ns = 12000;
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
 	int plane_sr, cursor_sr;
 	unsigned int enabled = 0;
 	bool cxsr_enabled;
 
-	if (g4x_compute_wm0(dev, PIPE_A,
+	if (g4x_compute_wm0(dev_priv, PIPE_A,
 			    &g4x_wm_info, pessimal_latency_ns,
 			    &g4x_cursor_wm_info, pessimal_latency_ns,
 			    &planea_wm, &cursora_wm))
 		enabled |= 1 << PIPE_A;
 
-	if (g4x_compute_wm0(dev, PIPE_B,
+	if (g4x_compute_wm0(dev_priv, PIPE_B,
 			    &g4x_wm_info, pessimal_latency_ns,
 			    &g4x_cursor_wm_info, pessimal_latency_ns,
 			    &planeb_wm, &cursorb_wm))
 		enabled |= 1 << PIPE_B;
 
 	if (single_plane_enabled(enabled) &&
-	    g4x_compute_srwm(dev, ffs(enabled) - 1,
+	    g4x_compute_srwm(dev_priv, ffs(enabled) - 1,
 			     sr_latency_ns,
 			     &g4x_wm_info,
 			     &g4x_cursor_wm_info,
@@ -1440,25 +1435,27 @@ static void g4x_update_wm(struct drm_crtc *crtc)
 		intel_set_memory_cxsr(dev_priv, true);
 }
 
-static void i965_update_wm(struct drm_crtc *unused_crtc)
+static void i965_update_wm(struct intel_crtc *unused_crtc)
 {
-	struct drm_device *dev = unused_crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_crtc *crtc;
+	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
+	struct intel_crtc *crtc;
 	int srwm = 1;
 	int cursor_sr = 16;
 	bool cxsr_enabled;
 
 	/* Calc sr entries for one plane configs */
-	crtc = single_enabled_crtc(dev);
+	crtc = single_enabled_crtc(dev_priv);
 	if (crtc) {
 		/* self-refresh has much higher latency */
 		static const int sr_latency_ns = 12000;
-		const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
+		const struct drm_display_mode *adjusted_mode =
+			&crtc->config->base.adjusted_mode;
+		const struct drm_framebuffer *fb =
+			crtc->base.primary->state->fb;
 		int clock = adjusted_mode->crtc_clock;
 		int htotal = adjusted_mode->crtc_htotal;
-		int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
-		int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
+		int hdisplay = crtc->config->pipe_src_w;
+		int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 		unsigned long line_time_us;
 		int entries;
 
@@ -1476,7 +1473,7 @@ static void i965_update_wm(struct drm_crtc *unused_crtc)
 			      entries, srwm);
 
 		entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
-			cpp * crtc->cursor->state->crtc_w;
+			cpp * crtc->base.cursor->state->crtc_w;
 		entries = DIV_ROUND_UP(entries,
 					  i965_cursor_wm_info.cacheline_size);
 		cursor_sr = i965_cursor_wm_info.fifo_size -
@@ -1514,34 +1511,38 @@ static void i965_update_wm(struct drm_crtc *unused_crtc)
 
 #undef FW_WM
 
-static void i9xx_update_wm(struct drm_crtc *unused_crtc)
+static void i9xx_update_wm(struct intel_crtc *unused_crtc)
 {
-	struct drm_device *dev = unused_crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
 	const struct intel_watermark_params *wm_info;
 	uint32_t fwater_lo;
 	uint32_t fwater_hi;
 	int cwm, srwm = 1;
 	int fifo_size;
 	int planea_wm, planeb_wm;
-	struct drm_crtc *crtc, *enabled = NULL;
+	struct intel_crtc *crtc, *enabled = NULL;
 
-	if (IS_I945GM(dev))
+	if (IS_I945GM(dev_priv))
 		wm_info = &i945_wm_info;
-	else if (!IS_GEN2(dev))
+	else if (!IS_GEN2(dev_priv))
 		wm_info = &i915_wm_info;
 	else
 		wm_info = &i830_a_wm_info;
 
-	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
-	crtc = intel_get_crtc_for_plane(dev, 0);
+	fifo_size = dev_priv->display.get_fifo_size(dev_priv, 0);
+	crtc = intel_get_crtc_for_plane(dev_priv, 0);
 	if (intel_crtc_active(crtc)) {
-		const struct drm_display_mode *adjusted_mode;
-		int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
-		if (IS_GEN2(dev))
+		const struct drm_display_mode *adjusted_mode =
+			&crtc->config->base.adjusted_mode;
+		const struct drm_framebuffer *fb =
+			crtc->base.primary->state->fb;
+		int cpp;
+
+		if (IS_GEN2(dev_priv))
 			cpp = 4;
+		else
+			cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 
-		adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
 		planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
 					       wm_info, fifo_size, cpp,
 					       pessimal_latency_ns);
@@ -1552,18 +1553,23 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
 			planea_wm = wm_info->max_wm;
 	}
 
-	if (IS_GEN2(dev))
+	if (IS_GEN2(dev_priv))
 		wm_info = &i830_bc_wm_info;
 
-	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
-	crtc = intel_get_crtc_for_plane(dev, 1);
+	fifo_size = dev_priv->display.get_fifo_size(dev_priv, 1);
+	crtc = intel_get_crtc_for_plane(dev_priv, 1);
 	if (intel_crtc_active(crtc)) {
-		const struct drm_display_mode *adjusted_mode;
-		int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0);
-		if (IS_GEN2(dev))
+		const struct drm_display_mode *adjusted_mode =
+			&crtc->config->base.adjusted_mode;
+		const struct drm_framebuffer *fb =
+			crtc->base.primary->state->fb;
+		int cpp;
+
+		if (IS_GEN2(dev_priv))
 			cpp = 4;
+		else
+			cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 
-		adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
 		planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
 					       wm_info, fifo_size, cpp,
 					       pessimal_latency_ns);
@@ -1579,10 +1585,10 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
 
 	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
 
-	if (IS_I915GM(dev) && enabled) {
+	if (IS_I915GM(dev_priv) && enabled) {
 		struct drm_i915_gem_object *obj;
 
-		obj = intel_fb_obj(enabled->primary->state->fb);
+		obj = intel_fb_obj(enabled->base.primary->state->fb);
 
 		/* self-refresh seems busted with untiled */
 		if (!i915_gem_object_is_tiled(obj))
@@ -1598,19 +1604,24 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
 	intel_set_memory_cxsr(dev_priv, false);
 
 	/* Calc sr entries for one plane configs */
-	if (HAS_FW_BLC(dev) && enabled) {
+	if (HAS_FW_BLC(dev_priv) && enabled) {
 		/* self-refresh has much higher latency */
 		static const int sr_latency_ns = 6000;
-		const struct drm_display_mode *adjusted_mode = &to_intel_crtc(enabled)->config->base.adjusted_mode;
+		const struct drm_display_mode *adjusted_mode =
+			&enabled->config->base.adjusted_mode;
+		const struct drm_framebuffer *fb =
+			enabled->base.primary->state->fb;
 		int clock = adjusted_mode->crtc_clock;
 		int htotal = adjusted_mode->crtc_htotal;
-		int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
-		int cpp = drm_format_plane_cpp(enabled->primary->state->fb->pixel_format, 0);
+		int hdisplay = enabled->config->pipe_src_w;
+		int cpp;
 		unsigned long line_time_us;
 		int entries;
 
-		if (IS_I915GM(dev) || IS_I945GM(dev))
+		if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
 			cpp = 4;
+		else
+			cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 
 		line_time_us = max(htotal * 1000 / clock, 1);
 
@@ -1623,7 +1634,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
 		if (srwm < 0)
 			srwm = 1;
 
-		if (IS_I945G(dev) || IS_I945GM(dev))
+		if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
 			I915_WRITE(FW_BLC_SELF,
 				   FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
 		else
@@ -1647,23 +1658,22 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
 		intel_set_memory_cxsr(dev_priv, true);
 }
 
-static void i845_update_wm(struct drm_crtc *unused_crtc)
+static void i845_update_wm(struct intel_crtc *unused_crtc)
 {
-	struct drm_device *dev = unused_crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_crtc *crtc;
+	struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
+	struct intel_crtc *crtc;
 	const struct drm_display_mode *adjusted_mode;
 	uint32_t fwater_lo;
 	int planea_wm;
 
-	crtc = single_enabled_crtc(dev);
+	crtc = single_enabled_crtc(dev_priv);
 	if (crtc == NULL)
 		return;
 
-	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
+	adjusted_mode = &crtc->config->base.adjusted_mode;
 	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
 				       &i845_wm_info,
-				       dev_priv->display.get_fifo_size(dev, 0),
+				       dev_priv->display.get_fifo_size(dev_priv, 0),
 				       4, pessimal_latency_ns);
 	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
 	fwater_lo |= (3<<8) | planea_wm;
@@ -1852,23 +1862,25 @@ static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
 	return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->base.dst), cpp);
 }
 
-static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
+static unsigned int
+ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
 {
-	if (INTEL_INFO(dev)->gen >= 8)
+	if (INTEL_GEN(dev_priv) >= 8)
 		return 3072;
-	else if (INTEL_INFO(dev)->gen >= 7)
+	else if (INTEL_GEN(dev_priv) >= 7)
 		return 768;
 	else
 		return 512;
 }
 
-static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
-					 int level, bool is_sprite)
+static unsigned int
+ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
+		     int level, bool is_sprite)
 {
-	if (INTEL_INFO(dev)->gen >= 8)
+	if (INTEL_GEN(dev_priv) >= 8)
 		/* BDW primary/sprite plane watermarks */
 		return level == 0 ? 255 : 2047;
-	else if (INTEL_INFO(dev)->gen >= 7)
+	else if (INTEL_GEN(dev_priv) >= 7)
 		/* IVB/HSW primary/sprite plane watermarks */
 		return level == 0 ? 127 : 1023;
 	else if (!is_sprite)
@@ -1879,18 +1891,18 @@ static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
 		return level == 0 ? 63 : 255;
 }
 
-static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
-					  int level)
+static unsigned int
+ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
 {
-	if (INTEL_INFO(dev)->gen >= 7)
+	if (INTEL_GEN(dev_priv) >= 7)
 		return level == 0 ? 63 : 255;
 	else
 		return level == 0 ? 31 : 63;
 }
 
-static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
+static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
 {
-	if (INTEL_INFO(dev)->gen >= 8)
+	if (INTEL_GEN(dev_priv) >= 8)
 		return 31;
 	else
 		return 15;
@@ -1903,7 +1915,8 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
 				     enum intel_ddb_partitioning ddb_partitioning,
 				     bool is_sprite)
 {
-	unsigned int fifo_size = ilk_display_fifo_size(dev);
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
 
 	/* if sprites aren't enabled, sprites get nothing */
 	if (is_sprite && !config->sprites_enabled)
@@ -1911,14 +1924,14 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
 
 	/* HSW allows LP1+ watermarks even with multiple pipes */
 	if (level == 0 || config->num_pipes_active > 1) {
-		fifo_size /= INTEL_INFO(dev)->num_pipes;
+		fifo_size /= INTEL_INFO(dev_priv)->num_pipes;
 
 		/*
 		 * For some reason the non self refresh
 		 * FIFO size is only half of the self
 		 * refresh FIFO size on ILK/SNB.
 		 */
-		if (INTEL_INFO(dev)->gen <= 6)
+		if (INTEL_GEN(dev_priv) <= 6)
 			fifo_size /= 2;
 	}
 
@@ -1934,7 +1947,7 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
 	}
 
 	/* clamp to max that the registers can hold */
-	return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
+	return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
 }
 
 /* Calculate the maximum cursor plane watermark */
@@ -1947,7 +1960,7 @@ static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
 		return 64;
 
 	/* otherwise just report max that registers can hold */
-	return ilk_cursor_wm_reg_max(dev, level);
+	return ilk_cursor_wm_reg_max(to_i915(dev), level);
 }
 
 static void ilk_compute_wm_maximums(const struct drm_device *dev,
@@ -1959,17 +1972,17 @@ static void ilk_compute_wm_maximums(const struct drm_device *dev,
 	max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
 	max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
 	max->cur = ilk_cursor_wm_max(dev, level, config);
-	max->fbc = ilk_fbc_wm_reg_max(dev);
+	max->fbc = ilk_fbc_wm_reg_max(to_i915(dev));
 }
 
-static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
+static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
 					int level,
 					struct ilk_wm_maximums *max)
 {
-	max->pri = ilk_plane_wm_reg_max(dev, level, false);
-	max->spr = ilk_plane_wm_reg_max(dev, level, true);
-	max->cur = ilk_cursor_wm_reg_max(dev, level);
-	max->fbc = ilk_fbc_wm_reg_max(dev);
+	max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
+	max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
+	max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
+	max->fbc = ilk_fbc_wm_reg_max(dev_priv);
 }
 
 static bool ilk_validate_wm_level(int level,
@@ -2076,14 +2089,13 @@ hsw_compute_linetime_wm(const struct intel_crtc_state *cstate)
 	       PIPE_WM_LINETIME_TIME(linetime);
 }
 
-static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
+static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
+				  uint16_t wm[8])
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	if (IS_GEN9(dev)) {
+	if (IS_GEN9(dev_priv)) {
 		uint32_t val;
 		int ret, i;
-		int level, max_level = ilk_wm_max_level(dev);
+		int level, max_level = ilk_wm_max_level(dev_priv);
 
 		/* read the first set of memory latencies[0:3] */
 		val = 0; /* data0 to be programmed to 0 for first set */
@@ -2155,7 +2167,7 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
 			}
 		}
 
-	} else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
 		uint64_t sskpd = I915_READ64(MCH_SSKPD);
 
 		wm[0] = (sskpd >> 56) & 0xFF;
@@ -2165,14 +2177,14 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
 		wm[2] = (sskpd >> 12) & 0xFF;
 		wm[3] = (sskpd >> 20) & 0x1FF;
 		wm[4] = (sskpd >> 32) & 0x1FF;
-	} else if (INTEL_INFO(dev)->gen >= 6) {
+	} else if (INTEL_GEN(dev_priv) >= 6) {
 		uint32_t sskpd = I915_READ(MCH_SSKPD);
 
 		wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
 		wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
 		wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
 		wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
-	} else if (INTEL_INFO(dev)->gen >= 5) {
+	} else if (INTEL_GEN(dev_priv) >= 5) {
 		uint32_t mltr = I915_READ(MLTR_ILK);
 
 		/* ILK primary LP0 latency is 700 ns */
@@ -2182,42 +2194,44 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
 	}
 }
 
-static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
+static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
+				       uint16_t wm[5])
 {
 	/* ILK sprite LP0 latency is 1300 ns */
-	if (IS_GEN5(dev))
+	if (IS_GEN5(dev_priv))
 		wm[0] = 13;
 }
 
-static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
+static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
+				       uint16_t wm[5])
 {
 	/* ILK cursor LP0 latency is 1300 ns */
-	if (IS_GEN5(dev))
+	if (IS_GEN5(dev_priv))
 		wm[0] = 13;
 
 	/* WaDoubleCursorLP3Latency:ivb */
-	if (IS_IVYBRIDGE(dev))
+	if (IS_IVYBRIDGE(dev_priv))
 		wm[3] *= 2;
 }
 
-int ilk_wm_max_level(const struct drm_device *dev)
+int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
 {
 	/* how many WM levels are we expecting */
-	if (INTEL_INFO(dev)->gen >= 9)
+	if (INTEL_GEN(dev_priv) >= 9)
 		return 7;
-	else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		return 4;
-	else if (INTEL_INFO(dev)->gen >= 6)
+	else if (INTEL_GEN(dev_priv) >= 6)
 		return 3;
 	else
 		return 2;
 }
 
-static void intel_print_wm_latency(struct drm_device *dev,
+static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
 				   const char *name,
 				   const uint16_t wm[8])
 {
-	int level, max_level = ilk_wm_max_level(dev);
+	int level, max_level = ilk_wm_max_level(dev_priv);
 
 	for (level = 0; level <= max_level; level++) {
 		unsigned int latency = wm[level];
@@ -2232,7 +2246,7 @@ static void intel_print_wm_latency(struct drm_device *dev,
 		 * - latencies are in us on gen9.
 		 * - before then, WM1+ latency values are in 0.5us units
 		 */
-		if (IS_GEN9(dev))
+		if (IS_GEN9(dev_priv))
 			latency *= 10;
 		else if (level > 0)
 			latency *= 5;
@@ -2246,7 +2260,7 @@ static void intel_print_wm_latency(struct drm_device *dev,
 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
 				    uint16_t wm[5], uint16_t min)
 {
-	int level, max_level = ilk_wm_max_level(&dev_priv->drm);
+	int level, max_level = ilk_wm_max_level(dev_priv);
 
 	if (wm[0] >= min)
 		return false;
@@ -2258,9 +2272,8 @@ static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
 	return true;
 }
 
-static void snb_wm_latency_quirk(struct drm_device *dev)
+static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	bool changed;
 
 	/*
@@ -2275,39 +2288,35 @@ static void snb_wm_latency_quirk(struct drm_device *dev)
 		return;
 
 	DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
-	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
-	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
-	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
+	intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
+	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
+	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
 }
 
-static void ilk_setup_wm_latency(struct drm_device *dev)
+static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
+	intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
 
 	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
 	       sizeof(dev_priv->wm.pri_latency));
 	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
 	       sizeof(dev_priv->wm.pri_latency));
 
-	intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
-	intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
+	intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
+	intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
 
-	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
-	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
-	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
+	intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
+	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
+	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
 
-	if (IS_GEN6(dev))
-		snb_wm_latency_quirk(dev);
+	if (IS_GEN6(dev_priv))
+		snb_wm_latency_quirk(dev_priv);
 }
 
-static void skl_setup_wm_latency(struct drm_device *dev)
+static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
-	intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
+	intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
+	intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
 }
 
 static bool ilk_validate_pipe_wm(struct drm_device *dev,
@@ -2345,7 +2354,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
 	struct intel_plane_state *pristate = NULL;
 	struct intel_plane_state *sprstate = NULL;
 	struct intel_plane_state *curstate = NULL;
-	int level, max_level = ilk_wm_max_level(dev), usable_level;
+	int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
 	struct ilk_wm_maximums max;
 
 	pipe_wm = &cstate->wm.ilk.optimal;
@@ -2377,7 +2386,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
 	usable_level = max_level;
 
 	/* ILK/SNB: LP2+ watermarks only w/o sprites */
-	if (INTEL_INFO(dev)->gen <= 6 && pipe_wm->sprites_enabled)
+	if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
 		usable_level = 1;
 
 	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
@@ -2390,13 +2399,13 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
 	memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
 	pipe_wm->wm[0] = pipe_wm->raw_wm[0];
 
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		pipe_wm->linetime = hsw_compute_linetime_wm(cstate);
 
 	if (!ilk_validate_pipe_wm(dev, pipe_wm))
 		return -EINVAL;
 
-	ilk_compute_wm_reg_maximums(dev, 1, &max);
+	ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
 
 	for (level = 1; level <= max_level; level++) {
 		struct intel_wm_level *wm = &pipe_wm->raw_wm[level];
@@ -2432,7 +2441,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev,
 {
 	struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
 	struct intel_pipe_wm *b = &intel_crtc->wm.active.ilk;
-	int level, max_level = ilk_wm_max_level(dev);
+	int level, max_level = ilk_wm_max_level(to_i915(dev));
 
 	/*
 	 * Start with the final, target watermarks, then combine with the
@@ -2516,16 +2525,16 @@ static void ilk_wm_merge(struct drm_device *dev,
 			 struct intel_pipe_wm *merged)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	int level, max_level = ilk_wm_max_level(dev);
+	int level, max_level = ilk_wm_max_level(dev_priv);
 	int last_enabled_level = max_level;
 
 	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
-	if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) &&
+	if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
 	    config->num_pipes_active > 1)
 		last_enabled_level = 0;
 
 	/* ILK: FBC WM must be disabled always */
-	merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
+	merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
 
 	/* merge each WM1+ level */
 	for (level = 1; level <= max_level; level++) {
@@ -2556,7 +2565,7 @@ static void ilk_wm_merge(struct drm_device *dev,
 	 * What we should check here is whether FBC can be
 	 * enabled sometime later.
 	 */
-	if (IS_GEN5(dev) && !merged->fbc_wm_enabled &&
+	if (IS_GEN5(dev_priv) && !merged->fbc_wm_enabled &&
 	    intel_fbc_is_active(dev_priv)) {
 		for (level = 2; level <= max_level; level++) {
 			struct intel_wm_level *wm = &merged->wm[level];
@@ -2577,7 +2586,7 @@ static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		return 2 * level;
 	else
 		return dev_priv->wm.pri_latency[level];
@@ -2588,6 +2597,7 @@ static void ilk_compute_wm_results(struct drm_device *dev,
 				   enum intel_ddb_partitioning partitioning,
 				   struct ilk_wm_values *results)
 {
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc;
 	int level, wm_lp;
 
@@ -2614,7 +2624,7 @@ static void ilk_compute_wm_results(struct drm_device *dev,
 		if (r->enable)
 			results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
 
-		if (INTEL_INFO(dev)->gen >= 8)
+		if (INTEL_GEN(dev_priv) >= 8)
 			results->wm_lp[wm_lp - 1] |=
 				r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
 		else
@@ -2625,7 +2635,7 @@ static void ilk_compute_wm_results(struct drm_device *dev,
 		 * Always set WM1S_LP_EN when spr_val != 0, even if the
 		 * level is disabled. Doing otherwise could cause underruns.
 		 */
-		if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
+		if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
 			WARN_ON(wm_lp != 1);
 			results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
 		} else
@@ -2656,7 +2666,7 @@ static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
 						  struct intel_pipe_wm *r1,
 						  struct intel_pipe_wm *r2)
 {
-	int level, max_level = ilk_wm_max_level(dev);
+	int level, max_level = ilk_wm_max_level(to_i915(dev));
 	int level1 = 0, level2 = 0;
 
 	for (level = 1; level <= max_level; level++) {
@@ -2775,7 +2785,6 @@ static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
 				struct ilk_wm_values *results)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	struct ilk_wm_values *previous = &dev_priv->wm.hw;
 	unsigned int dirty;
 	uint32_t val;
@@ -2801,7 +2810,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
 		I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
 
 	if (dirty & WM_DIRTY_DDB) {
-		if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+		if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
 			val = I915_READ(WM_MISC);
 			if (results->partitioning == INTEL_DDB_PART_1_2)
 				val &= ~WM_MISC_DATA_PARTITION_5_6;
@@ -2831,7 +2840,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
 	    previous->wm_lp_spr[0] != results->wm_lp_spr[0])
 		I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
 
-	if (INTEL_INFO(dev)->gen >= 7) {
+	if (INTEL_GEN(dev_priv) >= 7) {
 		if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
 			I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
 		if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
@@ -2879,6 +2888,21 @@ skl_wm_plane_id(const struct intel_plane *plane)
 	}
 }
 
+/*
+ * FIXME: We still don't have the proper code detect if we need to apply the WA,
+ * so assume we'll always need it in order to avoid underruns.
+ */
+static bool skl_needs_memory_bw_wa(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
+
+	if (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv) ||
+	    IS_KABYLAKE(dev_priv))
+		return true;
+
+	return false;
+}
+
 static bool
 intel_has_sagv(struct drm_i915_private *dev_priv)
 {
@@ -2999,9 +3023,12 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state)
 	struct drm_device *dev = state->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
-	struct drm_crtc *crtc;
+	struct intel_crtc *crtc;
+	struct intel_plane *plane;
+	struct intel_crtc_state *cstate;
+	struct skl_plane_wm *wm;
 	enum pipe pipe;
-	int level, plane;
+	int level, latency;
 
 	if (!intel_has_sagv(dev_priv))
 		return false;
@@ -3019,27 +3046,37 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state)
 
 	/* Since we're now guaranteed to only have one active CRTC... */
 	pipe = ffs(intel_state->active_crtcs) - 1;
-	crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+	crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
+	cstate = to_intel_crtc_state(crtc->base.state);
 
-	if (crtc->state->mode.flags & DRM_MODE_FLAG_INTERLACE)
+	if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
 		return false;
 
-	for_each_plane(dev_priv, pipe, plane) {
+	for_each_intel_plane_on_crtc(dev, crtc, plane) {
+		wm = &cstate->wm.skl.optimal.planes[skl_wm_plane_id(plane)];
+
 		/* Skip this plane if it's not enabled */
-		if (intel_state->wm_results.plane[pipe][plane][0] == 0)
+		if (!wm->wm[0].plane_en)
 			continue;
 
 		/* Find the highest enabled wm level for this plane */
-		for (level = ilk_wm_max_level(dev);
-		     intel_state->wm_results.plane[pipe][plane][level] == 0; --level)
+		for (level = ilk_wm_max_level(dev_priv);
+		     !wm->wm[level].plane_en; --level)
 		     { }
 
+		latency = dev_priv->wm.skl_latency[level];
+
+		if (skl_needs_memory_bw_wa(intel_state) &&
+		    plane->base.state->fb->modifier ==
+		    I915_FORMAT_MOD_X_TILED)
+			latency += 15;
+
 		/*
 		 * If any of the planes on this pipe don't enable wm levels
 		 * that incur memory latencies higher then 30µs we can't enable
 		 * the SAGV
 		 */
-		if (dev_priv->wm.skl_latency[level] < SKL_SAGV_BLOCK_TIME)
+		if (latency < SKL_SAGV_BLOCK_TIME)
 			return false;
 	}
 
@@ -3058,7 +3095,6 @@ skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
 	struct drm_crtc *for_crtc = cstate->base.crtc;
 	unsigned int pipe_size, ddb_size;
 	int nth_active_pipe;
-	int pipe = to_intel_crtc(for_crtc)->pipe;
 
 	if (WARN_ON(!state) || !cstate->base.active) {
 		alloc->start = 0;
@@ -3086,7 +3122,11 @@ skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
 	 * we currently hold.
 	 */
 	if (!intel_state->active_pipe_changes) {
-		*alloc = dev_priv->wm.skl_hw.ddb.pipe[pipe];
+		/*
+		 * alloc may be cleared by clear_intel_crtc_state,
+		 * copy from old state to be sure
+		 */
+		*alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
 		return;
 	}
 
@@ -3129,7 +3169,7 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
 		if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
 			continue;
 
-		for_each_plane(dev_priv, pipe, plane) {
+		for_each_universal_plane(dev_priv, pipe, plane) {
 			val = I915_READ(PLANE_BUF_CFG(pipe, plane));
 			skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
 						   val);
@@ -3173,7 +3213,7 @@ skl_plane_downscale_amount(const struct intel_plane_state *pstate)
 	src_h = drm_rect_height(&pstate->base.src);
 	dst_w = drm_rect_width(&pstate->base.dst);
 	dst_h = drm_rect_height(&pstate->base.dst);
-	if (intel_rotation_90_or_270(pstate->base.rotation))
+	if (drm_rotation_90_or_270(pstate->base.rotation))
 		swap(dst_w, dst_h);
 
 	downscale_h = max(src_h / dst_h, (uint32_t)DRM_PLANE_HELPER_NO_SCALING);
@@ -3204,7 +3244,7 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
 	width = drm_rect_width(&intel_pstate->base.src) >> 16;
 	height = drm_rect_height(&intel_pstate->base.src) >> 16;
 
-	if (intel_rotation_90_or_270(pstate->rotation))
+	if (drm_rotation_90_or_270(pstate->rotation))
 		swap(width, height);
 
 	/* for planar format */
@@ -3231,49 +3271,39 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
  *   3 * 4096 * 8192  * 4 < 2^32
  */
 static unsigned int
-skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate)
+skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate,
+				 unsigned *plane_data_rate,
+				 unsigned *plane_y_data_rate)
 {
 	struct drm_crtc_state *cstate = &intel_cstate->base;
 	struct drm_atomic_state *state = cstate->state;
-	struct drm_crtc *crtc = cstate->crtc;
-	struct drm_device *dev = crtc->dev;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	const struct drm_plane *plane;
+	struct drm_plane *plane;
 	const struct intel_plane *intel_plane;
-	struct drm_plane_state *pstate;
+	const struct drm_plane_state *pstate;
 	unsigned int rate, total_data_rate = 0;
 	int id;
-	int i;
 
 	if (WARN_ON(!state))
 		return 0;
 
 	/* Calculate and cache data rate for each plane */
-	for_each_plane_in_state(state, plane, pstate, i) {
+	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) {
 		id = skl_wm_plane_id(to_intel_plane(plane));
 		intel_plane = to_intel_plane(plane);
 
-		if (intel_plane->pipe != intel_crtc->pipe)
-			continue;
-
 		/* packed/uv */
 		rate = skl_plane_relative_data_rate(intel_cstate,
 						    pstate, 0);
-		intel_cstate->wm.skl.plane_data_rate[id] = rate;
+		plane_data_rate[id] = rate;
+
+		total_data_rate += rate;
 
 		/* y-plane */
 		rate = skl_plane_relative_data_rate(intel_cstate,
 						    pstate, 1);
-		intel_cstate->wm.skl.plane_y_data_rate[id] = rate;
-	}
-
-	/* Calculate CRTC's total data rate from cached values */
-	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
-		int id = skl_wm_plane_id(intel_plane);
+		plane_y_data_rate[id] = rate;
 
-		/* packed/uv */
-		total_data_rate += intel_cstate->wm.skl.plane_data_rate[id];
-		total_data_rate += intel_cstate->wm.skl.plane_y_data_rate[id];
+		total_data_rate += rate;
 	}
 
 	return total_data_rate;
@@ -3297,14 +3327,14 @@ skl_ddb_min_alloc(const struct drm_plane_state *pstate,
 		return 0;
 
 	/* For Non Y-tile return 8-blocks */
-	if (fb->modifier[0] != I915_FORMAT_MOD_Y_TILED &&
-	    fb->modifier[0] != I915_FORMAT_MOD_Yf_TILED)
+	if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
+	    fb->modifier != I915_FORMAT_MOD_Yf_TILED)
 		return 8;
 
 	src_w = drm_rect_width(&intel_pstate->base.src) >> 16;
 	src_h = drm_rect_height(&intel_pstate->base.src) >> 16;
 
-	if (intel_rotation_90_or_270(pstate->rotation))
+	if (drm_rotation_90_or_270(pstate->rotation))
 		swap(src_w, src_h);
 
 	/* Halve UV plane width and height for NV12 */
@@ -3318,7 +3348,7 @@ skl_ddb_min_alloc(const struct drm_plane_state *pstate,
 	else
 		plane_bpp = drm_format_plane_cpp(fb->pixel_format, 0);
 
-	if (intel_rotation_90_or_270(pstate->rotation)) {
+	if (drm_rotation_90_or_270(pstate->rotation)) {
 		switch (plane_bpp) {
 		case 1:
 			min_scanlines = 32;
@@ -3342,6 +3372,30 @@ skl_ddb_min_alloc(const struct drm_plane_state *pstate,
 	return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3;
 }
 
+static void
+skl_ddb_calc_min(const struct intel_crtc_state *cstate, int num_active,
+		 uint16_t *minimum, uint16_t *y_minimum)
+{
+	const struct drm_plane_state *pstate;
+	struct drm_plane *plane;
+
+	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, &cstate->base) {
+		struct intel_plane *intel_plane = to_intel_plane(plane);
+		int id = skl_wm_plane_id(intel_plane);
+
+		if (id == PLANE_CURSOR)
+			continue;
+
+		if (!pstate->visible)
+			continue;
+
+		minimum[id] = skl_ddb_min_alloc(pstate, 0);
+		y_minimum[id] = skl_ddb_min_alloc(pstate, 1);
+	}
+
+	minimum[PLANE_CURSOR] = skl_cursor_allocation(num_active);
+}
+
 static int
 skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 		      struct skl_ddb_allocation *ddb /* out */)
@@ -3350,17 +3404,16 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 	struct drm_crtc *crtc = cstate->base.crtc;
 	struct drm_device *dev = crtc->dev;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_plane *intel_plane;
-	struct drm_plane *plane;
-	struct drm_plane_state *pstate;
 	enum pipe pipe = intel_crtc->pipe;
-	struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
-	uint16_t alloc_size, start, cursor_blocks;
-	uint16_t *minimum = cstate->wm.skl.minimum_blocks;
-	uint16_t *y_minimum = cstate->wm.skl.minimum_y_blocks;
+	struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb;
+	uint16_t alloc_size, start;
+	uint16_t minimum[I915_MAX_PLANES] = {};
+	uint16_t y_minimum[I915_MAX_PLANES] = {};
 	unsigned int total_data_rate;
 	int num_active;
 	int id, i;
+	unsigned plane_data_rate[I915_MAX_PLANES] = {};
+	unsigned plane_y_data_rate[I915_MAX_PLANES] = {};
 
 	/* Clear the partitioning for disabled planes. */
 	memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
@@ -3370,7 +3423,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 		return 0;
 
 	if (!cstate->base.active) {
-		ddb->pipe[pipe].start = ddb->pipe[pipe].end = 0;
+		alloc->start = alloc->end = 0;
 		return 0;
 	}
 
@@ -3381,57 +3434,43 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 		return 0;
 	}
 
-	cursor_blocks = skl_cursor_allocation(num_active);
-	ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks;
-	ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
-
-	alloc_size -= cursor_blocks;
-
-	/* 1. Allocate the mininum required blocks for each active plane */
-	for_each_plane_in_state(state, plane, pstate, i) {
-		intel_plane = to_intel_plane(plane);
-		id = skl_wm_plane_id(intel_plane);
-
-		if (intel_plane->pipe != pipe)
-			continue;
-
-		if (!to_intel_plane_state(pstate)->base.visible) {
-			minimum[id] = 0;
-			y_minimum[id] = 0;
-			continue;
-		}
-		if (plane->type == DRM_PLANE_TYPE_CURSOR) {
-			minimum[id] = 0;
-			y_minimum[id] = 0;
-			continue;
-		}
+	skl_ddb_calc_min(cstate, num_active, minimum, y_minimum);
 
-		minimum[id] = skl_ddb_min_alloc(pstate, 0);
-		y_minimum[id] = skl_ddb_min_alloc(pstate, 1);
-	}
+	/*
+	 * 1. Allocate the mininum required blocks for each active plane
+	 * and allocate the cursor, it doesn't require extra allocation
+	 * proportional to the data rate.
+	 */
 
-	for (i = 0; i < PLANE_CURSOR; i++) {
+	for (i = 0; i < I915_MAX_PLANES; i++) {
 		alloc_size -= minimum[i];
 		alloc_size -= y_minimum[i];
 	}
 
+	ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - minimum[PLANE_CURSOR];
+	ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
+
 	/*
 	 * 2. Distribute the remaining space in proportion to the amount of
 	 * data each plane needs to fetch from memory.
 	 *
 	 * FIXME: we may not allocate every single block here.
 	 */
-	total_data_rate = skl_get_total_relative_data_rate(cstate);
+	total_data_rate = skl_get_total_relative_data_rate(cstate,
+							   plane_data_rate,
+							   plane_y_data_rate);
 	if (total_data_rate == 0)
 		return 0;
 
 	start = alloc->start;
-	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
+	for (id = 0; id < I915_MAX_PLANES; id++) {
 		unsigned int data_rate, y_data_rate;
 		uint16_t plane_blocks, y_plane_blocks = 0;
-		int id = skl_wm_plane_id(intel_plane);
 
-		data_rate = cstate->wm.skl.plane_data_rate[id];
+		if (id == PLANE_CURSOR)
+			continue;
+
+		data_rate = plane_data_rate[id];
 
 		/*
 		 * allocation for (packed formats) or (uv-plane part of planar format):
@@ -3453,7 +3492,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 		/*
 		 * allocation for y_plane part of planar format:
 		 */
-		y_data_rate = cstate->wm.skl.plane_y_data_rate[id];
+		y_data_rate = plane_y_data_rate[id];
 
 		y_plane_blocks = y_minimum[id];
 		y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
@@ -3549,22 +3588,28 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 	uint32_t width = 0, height = 0;
 	uint32_t plane_pixel_rate;
 	uint32_t y_tile_minimum, y_min_scanlines;
+	struct intel_atomic_state *state =
+		to_intel_atomic_state(cstate->base.state);
+	bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
 
 	if (latency == 0 || !cstate->base.active || !intel_pstate->base.visible) {
 		*enabled = false;
 		return 0;
 	}
 
+	if (apply_memory_bw_wa && fb->modifier == I915_FORMAT_MOD_X_TILED)
+		latency += 15;
+
 	width = drm_rect_width(&intel_pstate->base.src) >> 16;
 	height = drm_rect_height(&intel_pstate->base.src) >> 16;
 
-	if (intel_rotation_90_or_270(pstate->rotation))
+	if (drm_rotation_90_or_270(pstate->rotation))
 		swap(width, height);
 
 	cpp = drm_format_plane_cpp(fb->pixel_format, 0);
 	plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate, intel_pstate);
 
-	if (intel_rotation_90_or_270(pstate->rotation)) {
+	if (drm_rotation_90_or_270(pstate->rotation)) {
 		int cpp = (fb->pixel_format == DRM_FORMAT_NV12) ?
 			drm_format_plane_cpp(fb->pixel_format, 1) :
 			drm_format_plane_cpp(fb->pixel_format, 0);
@@ -3576,23 +3621,27 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 		case 2:
 			y_min_scanlines = 8;
 			break;
-		default:
-			WARN(1, "Unsupported pixel depth for rotation");
 		case 4:
 			y_min_scanlines = 4;
 			break;
+		default:
+			MISSING_CASE(cpp);
+			return -EINVAL;
 		}
 	} else {
 		y_min_scanlines = 4;
 	}
 
+	if (apply_memory_bw_wa)
+		y_min_scanlines *= 2;
+
 	plane_bytes_per_line = width * cpp;
-	if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
-	    fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
+	if (fb->modifier == I915_FORMAT_MOD_Y_TILED ||
+	    fb->modifier == I915_FORMAT_MOD_Yf_TILED) {
 		plane_blocks_per_line =
 		      DIV_ROUND_UP(plane_bytes_per_line * y_min_scanlines, 512);
 		plane_blocks_per_line /= y_min_scanlines;
-	} else if (fb->modifier[0] == DRM_FORMAT_MOD_NONE) {
+	} else if (fb->modifier == DRM_FORMAT_MOD_NONE) {
 		plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512)
 					+ 1;
 	} else {
@@ -3607,11 +3656,14 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 
 	y_tile_minimum = plane_blocks_per_line * y_min_scanlines;
 
-	if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
-	    fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
+	if (fb->modifier == I915_FORMAT_MOD_Y_TILED ||
+	    fb->modifier == I915_FORMAT_MOD_Yf_TILED) {
 		selected_result = max(method2, y_tile_minimum);
 	} else {
-		if ((ddb_allocation / plane_blocks_per_line) >= 1)
+		if ((cpp * cstate->base.adjusted_mode.crtc_htotal / 512 < 1) &&
+		    (plane_bytes_per_line / 512 < 1))
+			selected_result = method2;
+		else if ((ddb_allocation / plane_blocks_per_line) >= 1)
 			selected_result = min(method1, method2);
 		else
 			selected_result = method1;
@@ -3621,8 +3673,8 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 	res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line);
 
 	if (level >= 1 && level <= 7) {
-		if (fb->modifier[0] == I915_FORMAT_MOD_Y_TILED ||
-		    fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
+		if (fb->modifier == I915_FORMAT_MOD_Y_TILED ||
+		    fb->modifier == I915_FORMAT_MOD_Yf_TILED) {
 			res_blocks += y_tile_minimum;
 			res_lines += y_min_scanlines;
 		} else {
@@ -3661,67 +3713,52 @@ static int
 skl_compute_wm_level(const struct drm_i915_private *dev_priv,
 		     struct skl_ddb_allocation *ddb,
 		     struct intel_crtc_state *cstate,
+		     struct intel_plane *intel_plane,
 		     int level,
 		     struct skl_wm_level *result)
 {
 	struct drm_atomic_state *state = cstate->base.state;
 	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
-	struct drm_plane *plane;
-	struct intel_plane *intel_plane;
-	struct intel_plane_state *intel_pstate;
+	struct drm_plane *plane = &intel_plane->base;
+	struct intel_plane_state *intel_pstate = NULL;
 	uint16_t ddb_blocks;
 	enum pipe pipe = intel_crtc->pipe;
 	int ret;
+	int i = skl_wm_plane_id(intel_plane);
+
+	if (state)
+		intel_pstate =
+			intel_atomic_get_existing_plane_state(state,
+							      intel_plane);
 
 	/*
-	 * We'll only calculate watermarks for planes that are actually
-	 * enabled, so make sure all other planes are set as disabled.
+	 * Note: If we start supporting multiple pending atomic commits against
+	 * the same planes/CRTC's in the future, plane->state will no longer be
+	 * the correct pre-state to use for the calculations here and we'll
+	 * need to change where we get the 'unchanged' plane data from.
+	 *
+	 * For now this is fine because we only allow one queued commit against
+	 * a CRTC.  Even if the plane isn't modified by this transaction and we
+	 * don't have a plane lock, we still have the CRTC's lock, so we know
+	 * that no other transactions are racing with us to update it.
 	 */
-	memset(result, 0, sizeof(*result));
-
-	for_each_intel_plane_mask(&dev_priv->drm,
-				  intel_plane,
-				  cstate->base.plane_mask) {
-		int i = skl_wm_plane_id(intel_plane);
-
-		plane = &intel_plane->base;
-		intel_pstate = NULL;
-		if (state)
-			intel_pstate =
-				intel_atomic_get_existing_plane_state(state,
-								      intel_plane);
-
-		/*
-		 * Note: If we start supporting multiple pending atomic commits
-		 * against the same planes/CRTC's in the future, plane->state
-		 * will no longer be the correct pre-state to use for the
-		 * calculations here and we'll need to change where we get the
-		 * 'unchanged' plane data from.
-		 *
-		 * For now this is fine because we only allow one queued commit
-		 * against a CRTC.  Even if the plane isn't modified by this
-		 * transaction and we don't have a plane lock, we still have
-		 * the CRTC's lock, so we know that no other transactions are
-		 * racing with us to update it.
-		 */
-		if (!intel_pstate)
-			intel_pstate = to_intel_plane_state(plane->state);
+	if (!intel_pstate)
+		intel_pstate = to_intel_plane_state(plane->state);
 
-		WARN_ON(!intel_pstate->base.fb);
+	WARN_ON(!intel_pstate->base.fb);
 
-		ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
+	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
 
-		ret = skl_compute_plane_wm(dev_priv,
-					   cstate,
-					   intel_pstate,
-					   ddb_blocks,
-					   level,
-					   &result->plane_res_b[i],
-					   &result->plane_res_l[i],
-					   &result->plane_en[i]);
-		if (ret)
-			return ret;
-	}
+	ret = skl_compute_plane_wm(dev_priv,
+				   cstate,
+				   intel_pstate,
+				   ddb_blocks,
+				   level,
+				   &result->plane_res_b,
+				   &result->plane_res_l,
+				   &result->plane_en);
+	if (ret)
+		return ret;
 
 	return 0;
 }
@@ -3729,32 +3766,28 @@ skl_compute_wm_level(const struct drm_i915_private *dev_priv,
 static uint32_t
 skl_compute_linetime_wm(struct intel_crtc_state *cstate)
 {
+	uint32_t pixel_rate;
+
 	if (!cstate->base.active)
 		return 0;
 
-	if (WARN_ON(ilk_pipe_pixel_rate(cstate) == 0))
+	pixel_rate = ilk_pipe_pixel_rate(cstate);
+
+	if (WARN_ON(pixel_rate == 0))
 		return 0;
 
 	return DIV_ROUND_UP(8 * cstate->base.adjusted_mode.crtc_htotal * 1000,
-			    ilk_pipe_pixel_rate(cstate));
+			    pixel_rate);
 }
 
 static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
 				      struct skl_wm_level *trans_wm /* out */)
 {
-	struct drm_crtc *crtc = cstate->base.crtc;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_plane *intel_plane;
-
 	if (!cstate->base.active)
 		return;
 
 	/* Until we know more, just disable transition WMs */
-	for_each_intel_plane_on_crtc(crtc->dev, intel_crtc, intel_plane) {
-		int i = skl_wm_plane_id(intel_plane);
-
-		trans_wm->plane_en[i] = false;
-	}
+	trans_wm->plane_en = false;
 }
 
 static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
@@ -3763,77 +3796,34 @@ static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
 {
 	struct drm_device *dev = cstate->base.crtc->dev;
 	const struct drm_i915_private *dev_priv = to_i915(dev);
-	int level, max_level = ilk_wm_max_level(dev);
+	struct intel_plane *intel_plane;
+	struct skl_plane_wm *wm;
+	int level, max_level = ilk_wm_max_level(dev_priv);
 	int ret;
 
-	for (level = 0; level <= max_level; level++) {
-		ret = skl_compute_wm_level(dev_priv, ddb, cstate,
-					   level, &pipe_wm->wm[level]);
-		if (ret)
-			return ret;
-	}
-	pipe_wm->linetime = skl_compute_linetime_wm(cstate);
-
-	skl_compute_transition_wm(cstate, &pipe_wm->trans_wm);
-
-	return 0;
-}
-
-static void skl_compute_wm_results(struct drm_device *dev,
-				   struct skl_pipe_wm *p_wm,
-				   struct skl_wm_values *r,
-				   struct intel_crtc *intel_crtc)
-{
-	int level, max_level = ilk_wm_max_level(dev);
-	enum pipe pipe = intel_crtc->pipe;
-	uint32_t temp;
-	int i;
-
-	for (level = 0; level <= max_level; level++) {
-		for (i = 0; i < intel_num_planes(intel_crtc); i++) {
-			temp = 0;
-
-			temp |= p_wm->wm[level].plane_res_l[i] <<
-					PLANE_WM_LINES_SHIFT;
-			temp |= p_wm->wm[level].plane_res_b[i];
-			if (p_wm->wm[level].plane_en[i])
-				temp |= PLANE_WM_EN;
+	/*
+	 * We'll only calculate watermarks for planes that are actually
+	 * enabled, so make sure all other planes are set as disabled.
+	 */
+	memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
 
-			r->plane[pipe][i][level] = temp;
+	for_each_intel_plane_mask(&dev_priv->drm,
+				  intel_plane,
+				  cstate->base.plane_mask) {
+		wm = &pipe_wm->planes[skl_wm_plane_id(intel_plane)];
+
+		for (level = 0; level <= max_level; level++) {
+			ret = skl_compute_wm_level(dev_priv, ddb, cstate,
+						   intel_plane, level,
+						   &wm->wm[level]);
+			if (ret)
+				return ret;
 		}
-
-		temp = 0;
-
-		temp |= p_wm->wm[level].plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
-		temp |= p_wm->wm[level].plane_res_b[PLANE_CURSOR];
-
-		if (p_wm->wm[level].plane_en[PLANE_CURSOR])
-			temp |= PLANE_WM_EN;
-
-		r->plane[pipe][PLANE_CURSOR][level] = temp;
-
+		skl_compute_transition_wm(cstate, &wm->trans_wm);
 	}
+	pipe_wm->linetime = skl_compute_linetime_wm(cstate);
 
-	/* transition WMs */
-	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
-		temp = 0;
-		temp |= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
-		temp |= p_wm->trans_wm.plane_res_b[i];
-		if (p_wm->trans_wm.plane_en[i])
-			temp |= PLANE_WM_EN;
-
-		r->plane_trans[pipe][i] = temp;
-	}
-
-	temp = 0;
-	temp |= p_wm->trans_wm.plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
-	temp |= p_wm->trans_wm.plane_res_b[PLANE_CURSOR];
-	if (p_wm->trans_wm.plane_en[PLANE_CURSOR])
-		temp |= PLANE_WM_EN;
-
-	r->plane_trans[pipe][PLANE_CURSOR] = temp;
-
-	r->wm_linetime[pipe] = p_wm->linetime;
+	return 0;
 }
 
 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
@@ -3846,53 +3836,77 @@ static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
 		I915_WRITE(reg, 0);
 }
 
-void skl_write_plane_wm(struct intel_crtc *intel_crtc,
-			const struct skl_wm_values *wm,
-			int plane)
+static void skl_write_wm_level(struct drm_i915_private *dev_priv,
+			       i915_reg_t reg,
+			       const struct skl_wm_level *level)
+{
+	uint32_t val = 0;
+
+	if (level->plane_en) {
+		val |= PLANE_WM_EN;
+		val |= level->plane_res_b;
+		val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
+	}
+
+	I915_WRITE(reg, val);
+}
+
+static void skl_write_plane_wm(struct intel_crtc *intel_crtc,
+			       const struct skl_plane_wm *wm,
+			       const struct skl_ddb_allocation *ddb,
+			       int plane)
 {
 	struct drm_crtc *crtc = &intel_crtc->base;
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	int level, max_level = ilk_wm_max_level(dev);
+	int level, max_level = ilk_wm_max_level(dev_priv);
 	enum pipe pipe = intel_crtc->pipe;
 
 	for (level = 0; level <= max_level; level++) {
-		I915_WRITE(PLANE_WM(pipe, plane, level),
-			   wm->plane[pipe][plane][level]);
+		skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane, level),
+				   &wm->wm[level]);
 	}
-	I915_WRITE(PLANE_WM_TRANS(pipe, plane), wm->plane_trans[pipe][plane]);
+	skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane),
+			   &wm->trans_wm);
 
 	skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane),
-			    &wm->ddb.plane[pipe][plane]);
+			    &ddb->plane[pipe][plane]);
 	skl_ddb_entry_write(dev_priv, PLANE_NV12_BUF_CFG(pipe, plane),
-			    &wm->ddb.y_plane[pipe][plane]);
+			    &ddb->y_plane[pipe][plane]);
 }
 
-void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
-			 const struct skl_wm_values *wm)
+static void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
+				const struct skl_plane_wm *wm,
+				const struct skl_ddb_allocation *ddb)
 {
 	struct drm_crtc *crtc = &intel_crtc->base;
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	int level, max_level = ilk_wm_max_level(dev);
+	int level, max_level = ilk_wm_max_level(dev_priv);
 	enum pipe pipe = intel_crtc->pipe;
 
 	for (level = 0; level <= max_level; level++) {
-		I915_WRITE(CUR_WM(pipe, level),
-			   wm->plane[pipe][PLANE_CURSOR][level]);
+		skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
+				   &wm->wm[level]);
 	}
-	I915_WRITE(CUR_WM_TRANS(pipe), wm->plane_trans[pipe][PLANE_CURSOR]);
+	skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
 
 	skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
-			    &wm->ddb.plane[pipe][PLANE_CURSOR]);
+			    &ddb->plane[pipe][PLANE_CURSOR]);
 }
 
-bool skl_ddb_allocation_equals(const struct skl_ddb_allocation *old,
-			       const struct skl_ddb_allocation *new,
-			       enum pipe pipe)
+bool skl_wm_level_equals(const struct skl_wm_level *l1,
+			 const struct skl_wm_level *l2)
 {
-	return new->pipe[pipe].start == old->pipe[pipe].start &&
-	       new->pipe[pipe].end == old->pipe[pipe].end;
+	if (l1->plane_en != l2->plane_en)
+		return false;
+
+	/* If both planes aren't enabled, the rest shouldn't matter */
+	if (!l1->plane_en)
+		return true;
+
+	return (l1->plane_res_l == l2->plane_res_l &&
+		l1->plane_res_b == l2->plane_res_b);
 }
 
 static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
@@ -3901,35 +3915,26 @@ static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
 	return a->start < b->end && b->start < a->end;
 }
 
-bool skl_ddb_allocation_overlaps(struct drm_atomic_state *state,
-				 const struct skl_ddb_allocation *old,
-				 const struct skl_ddb_allocation *new,
-				 enum pipe pipe)
+bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry **entries,
+				 const struct skl_ddb_entry *ddb,
+				 int ignore)
 {
-	struct drm_device *dev = state->dev;
-	struct intel_crtc *intel_crtc;
-	enum pipe otherp;
-
-	for_each_intel_crtc(dev, intel_crtc) {
-		otherp = intel_crtc->pipe;
-
-		if (otherp == pipe)
-			continue;
+	int i;
 
-		if (skl_ddb_entries_overlap(&new->pipe[pipe],
-					    &old->pipe[otherp]))
+	for (i = 0; i < I915_MAX_PIPES; i++)
+		if (i != ignore && entries[i] &&
+		    skl_ddb_entries_overlap(ddb, entries[i]))
 			return true;
-	}
 
 	return false;
 }
 
 static int skl_update_pipe_wm(struct drm_crtc_state *cstate,
-			      struct skl_ddb_allocation *ddb, /* out */
+			      const struct skl_pipe_wm *old_pipe_wm,
 			      struct skl_pipe_wm *pipe_wm, /* out */
+			      struct skl_ddb_allocation *ddb, /* out */
 			      bool *changed /* out */)
 {
-	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->crtc);
 	struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate);
 	int ret;
 
@@ -3937,7 +3942,7 @@ static int skl_update_pipe_wm(struct drm_crtc_state *cstate,
 	if (ret)
 		return ret;
 
-	if (!memcmp(&intel_crtc->wm.active.skl, pipe_wm, sizeof(*pipe_wm)))
+	if (!memcmp(old_pipe_wm, pipe_wm, sizeof(*pipe_wm)))
 		*changed = false;
 	else
 		*changed = true;
@@ -3958,7 +3963,7 @@ pipes_modified(struct drm_atomic_state *state)
 	return ret;
 }
 
-int
+static int
 skl_ddb_add_affected_planes(struct intel_crtc_state *cstate)
 {
 	struct drm_atomic_state *state = cstate->base.state;
@@ -3976,7 +3981,7 @@ skl_ddb_add_affected_planes(struct intel_crtc_state *cstate)
 
 	WARN_ON(!drm_atomic_get_existing_crtc_state(state, crtc));
 
-	drm_for_each_plane_mask(plane, dev, crtc->state->plane_mask) {
+	drm_for_each_plane_mask(plane, dev, cstate->base.plane_mask) {
 		id = skl_wm_plane_id(to_intel_plane(plane));
 
 		if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][id],
@@ -4076,19 +4081,50 @@ skl_copy_wm_for_pipe(struct skl_wm_values *dst,
 		     struct skl_wm_values *src,
 		     enum pipe pipe)
 {
-	dst->wm_linetime[pipe] = src->wm_linetime[pipe];
-	memcpy(dst->plane[pipe], src->plane[pipe],
-	       sizeof(dst->plane[pipe]));
-	memcpy(dst->plane_trans[pipe], src->plane_trans[pipe],
-	       sizeof(dst->plane_trans[pipe]));
-
-	dst->ddb.pipe[pipe] = src->ddb.pipe[pipe];
 	memcpy(dst->ddb.y_plane[pipe], src->ddb.y_plane[pipe],
 	       sizeof(dst->ddb.y_plane[pipe]));
 	memcpy(dst->ddb.plane[pipe], src->ddb.plane[pipe],
 	       sizeof(dst->ddb.plane[pipe]));
 }
 
+static void
+skl_print_wm_changes(const struct drm_atomic_state *state)
+{
+	const struct drm_device *dev = state->dev;
+	const struct drm_i915_private *dev_priv = to_i915(dev);
+	const struct intel_atomic_state *intel_state =
+		to_intel_atomic_state(state);
+	const struct drm_crtc *crtc;
+	const struct drm_crtc_state *cstate;
+	const struct intel_plane *intel_plane;
+	const struct skl_ddb_allocation *old_ddb = &dev_priv->wm.skl_hw.ddb;
+	const struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
+	int id;
+	int i;
+
+	for_each_crtc_in_state(state, crtc, cstate, i) {
+		const struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+		enum pipe pipe = intel_crtc->pipe;
+
+		for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
+			const struct skl_ddb_entry *old, *new;
+
+			id = skl_wm_plane_id(intel_plane);
+			old = &old_ddb->plane[pipe][id];
+			new = &new_ddb->plane[pipe][id];
+
+			if (skl_ddb_entry_equal(old, new))
+				continue;
+
+			DRM_DEBUG_ATOMIC("[PLANE:%d:%s] ddb (%d - %d) -> (%d - %d)\n",
+					 intel_plane->base.base.id,
+					 intel_plane->base.name,
+					 old->start, old->end,
+					 new->start, new->end);
+		}
+	}
+}
+
 static int
 skl_compute_wm(struct drm_atomic_state *state)
 {
@@ -4131,13 +4167,14 @@ skl_compute_wm(struct drm_atomic_state *state)
 	 * no suitable watermark values can be found.
 	 */
 	for_each_crtc_in_state(state, crtc, cstate, i) {
-		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 		struct intel_crtc_state *intel_cstate =
 			to_intel_crtc_state(cstate);
+		const struct skl_pipe_wm *old_pipe_wm =
+			&to_intel_crtc_state(crtc->state)->wm.skl.optimal;
 
 		pipe_wm = &intel_cstate->wm.skl.optimal;
-		ret = skl_update_pipe_wm(cstate, &results->ddb, pipe_wm,
-					 &changed);
+		ret = skl_update_pipe_wm(cstate, old_pipe_wm, pipe_wm,
+					 &results->ddb, &changed);
 		if (ret)
 			return ret;
 
@@ -4149,44 +4186,51 @@ skl_compute_wm(struct drm_atomic_state *state)
 			continue;
 
 		intel_cstate->update_wm_pre = true;
-		skl_compute_wm_results(crtc->dev, pipe_wm, results, intel_crtc);
 	}
 
+	skl_print_wm_changes(state);
+
 	return 0;
 }
 
-static void skl_update_wm(struct drm_crtc *crtc)
+static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state,
+				      struct intel_crtc_state *cstate)
 {
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct skl_wm_values *results = &dev_priv->wm.skl_results;
-	struct skl_wm_values *hw_vals = &dev_priv->wm.skl_hw;
-	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
+	struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 	struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
-	enum pipe pipe = intel_crtc->pipe;
+	const struct skl_ddb_allocation *ddb = &state->wm_results.ddb;
+	enum pipe pipe = crtc->pipe;
+	int plane;
 
-	if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0)
+	if (!(state->wm_results.dirty_pipes & drm_crtc_mask(&crtc->base)))
 		return;
 
-	intel_crtc->wm.active.skl = *pipe_wm;
+	I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime);
 
-	mutex_lock(&dev_priv->wm.wm_mutex);
+	for_each_universal_plane(dev_priv, pipe, plane)
+		skl_write_plane_wm(crtc, &pipe_wm->planes[plane], ddb, plane);
 
-	/*
-	 * If this pipe isn't active already, we're going to be enabling it
-	 * very soon. Since it's safe to update a pipe's ddb allocation while
-	 * the pipe's shut off, just do so here. Already active pipes will have
-	 * their watermarks updated once we update their planes.
-	 */
-	if (crtc->state->active_changed) {
-		int plane;
+	skl_write_cursor_wm(crtc, &pipe_wm->planes[PLANE_CURSOR], ddb);
+}
+
+static void skl_initial_wm(struct intel_atomic_state *state,
+			   struct intel_crtc_state *cstate)
+{
+	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
+	struct drm_device *dev = intel_crtc->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct skl_wm_values *results = &state->wm_results;
+	struct skl_wm_values *hw_vals = &dev_priv->wm.skl_hw;
+	enum pipe pipe = intel_crtc->pipe;
 
-		for (plane = 0; plane < intel_num_planes(intel_crtc); plane++)
-			skl_write_plane_wm(intel_crtc, results, plane);
+	if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0)
+		return;
 
-		skl_write_cursor_wm(intel_crtc, results);
-	}
+	mutex_lock(&dev_priv->wm.wm_mutex);
+
+	if (cstate->base.active_changed)
+		skl_atomic_update_crtc_wm(state, cstate);
 
 	skl_copy_wm_for_pipe(hw_vals, results, pipe);
 
@@ -4226,7 +4270,7 @@ static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
 	ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
 
 	/* 5/6 split only in single pipe config on IVB+ */
-	if (INTEL_INFO(dev)->gen >= 7 &&
+	if (INTEL_GEN(dev_priv) >= 7 &&
 	    config.num_pipes_active == 1 && config.sprites_enabled) {
 		ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
 		ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
@@ -4244,7 +4288,8 @@ static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
 	ilk_write_wm_values(dev_priv, &results);
 }
 
-static void ilk_initial_watermarks(struct intel_crtc_state *cstate)
+static void ilk_initial_watermarks(struct intel_atomic_state *state,
+				   struct intel_crtc_state *cstate)
 {
 	struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
@@ -4255,7 +4300,8 @@ static void ilk_initial_watermarks(struct intel_crtc_state *cstate)
 	mutex_unlock(&dev_priv->wm.wm_mutex);
 }
 
-static void ilk_optimize_watermarks(struct intel_crtc_state *cstate)
+static void ilk_optimize_watermarks(struct intel_atomic_state *state,
+				    struct intel_crtc_state *cstate)
 {
 	struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
@@ -4268,114 +4314,75 @@ static void ilk_optimize_watermarks(struct intel_crtc_state *cstate)
 	mutex_unlock(&dev_priv->wm.wm_mutex);
 }
 
-static void skl_pipe_wm_active_state(uint32_t val,
-				     struct skl_pipe_wm *active,
-				     bool is_transwm,
-				     bool is_cursor,
-				     int i,
-				     int level)
+static inline void skl_wm_level_from_reg_val(uint32_t val,
+					     struct skl_wm_level *level)
 {
-	bool is_enabled = (val & PLANE_WM_EN) != 0;
-
-	if (!is_transwm) {
-		if (!is_cursor) {
-			active->wm[level].plane_en[i] = is_enabled;
-			active->wm[level].plane_res_b[i] =
-					val & PLANE_WM_BLOCKS_MASK;
-			active->wm[level].plane_res_l[i] =
-					(val >> PLANE_WM_LINES_SHIFT) &
-						PLANE_WM_LINES_MASK;
-		} else {
-			active->wm[level].plane_en[PLANE_CURSOR] = is_enabled;
-			active->wm[level].plane_res_b[PLANE_CURSOR] =
-					val & PLANE_WM_BLOCKS_MASK;
-			active->wm[level].plane_res_l[PLANE_CURSOR] =
-					(val >> PLANE_WM_LINES_SHIFT) &
-						PLANE_WM_LINES_MASK;
-		}
-	} else {
-		if (!is_cursor) {
-			active->trans_wm.plane_en[i] = is_enabled;
-			active->trans_wm.plane_res_b[i] =
-					val & PLANE_WM_BLOCKS_MASK;
-			active->trans_wm.plane_res_l[i] =
-					(val >> PLANE_WM_LINES_SHIFT) &
-						PLANE_WM_LINES_MASK;
-		} else {
-			active->trans_wm.plane_en[PLANE_CURSOR] = is_enabled;
-			active->trans_wm.plane_res_b[PLANE_CURSOR] =
-					val & PLANE_WM_BLOCKS_MASK;
-			active->trans_wm.plane_res_l[PLANE_CURSOR] =
-					(val >> PLANE_WM_LINES_SHIFT) &
-						PLANE_WM_LINES_MASK;
-		}
-	}
+	level->plane_en = val & PLANE_WM_EN;
+	level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
+	level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
+		PLANE_WM_LINES_MASK;
 }
 
-static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
+void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc,
+			      struct skl_pipe_wm *out)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
-	struct skl_pipe_wm *active = &cstate->wm.skl.optimal;
+	struct intel_plane *intel_plane;
+	struct skl_plane_wm *wm;
 	enum pipe pipe = intel_crtc->pipe;
-	int level, i, max_level;
-	uint32_t temp;
-
-	max_level = ilk_wm_max_level(dev);
-
-	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
-
-	for (level = 0; level <= max_level; level++) {
-		for (i = 0; i < intel_num_planes(intel_crtc); i++)
-			hw->plane[pipe][i][level] =
-					I915_READ(PLANE_WM(pipe, i, level));
-		hw->plane[pipe][PLANE_CURSOR][level] = I915_READ(CUR_WM(pipe, level));
-	}
-
-	for (i = 0; i < intel_num_planes(intel_crtc); i++)
-		hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
-	hw->plane_trans[pipe][PLANE_CURSOR] = I915_READ(CUR_WM_TRANS(pipe));
+	int level, id, max_level;
+	uint32_t val;
 
-	if (!intel_crtc->active)
-		return;
+	max_level = ilk_wm_max_level(dev_priv);
 
-	hw->dirty_pipes |= drm_crtc_mask(crtc);
+	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
+		id = skl_wm_plane_id(intel_plane);
+		wm = &out->planes[id];
 
-	active->linetime = hw->wm_linetime[pipe];
+		for (level = 0; level <= max_level; level++) {
+			if (id != PLANE_CURSOR)
+				val = I915_READ(PLANE_WM(pipe, id, level));
+			else
+				val = I915_READ(CUR_WM(pipe, level));
 
-	for (level = 0; level <= max_level; level++) {
-		for (i = 0; i < intel_num_planes(intel_crtc); i++) {
-			temp = hw->plane[pipe][i][level];
-			skl_pipe_wm_active_state(temp, active, false,
-						false, i, level);
+			skl_wm_level_from_reg_val(val, &wm->wm[level]);
 		}
-		temp = hw->plane[pipe][PLANE_CURSOR][level];
-		skl_pipe_wm_active_state(temp, active, false, true, i, level);
-	}
 
-	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
-		temp = hw->plane_trans[pipe][i];
-		skl_pipe_wm_active_state(temp, active, true, false, i, 0);
+		if (id != PLANE_CURSOR)
+			val = I915_READ(PLANE_WM_TRANS(pipe, id));
+		else
+			val = I915_READ(CUR_WM_TRANS(pipe));
+
+		skl_wm_level_from_reg_val(val, &wm->trans_wm);
 	}
 
-	temp = hw->plane_trans[pipe][PLANE_CURSOR];
-	skl_pipe_wm_active_state(temp, active, true, true, i, 0);
+	if (!intel_crtc->active)
+		return;
 
-	intel_crtc->wm.active.skl = *active;
+	out->linetime = I915_READ(PIPE_WM_LINETIME(pipe));
 }
 
 void skl_wm_get_hw_state(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
 	struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
 	struct drm_crtc *crtc;
+	struct intel_crtc *intel_crtc;
+	struct intel_crtc_state *cstate;
 
 	skl_ddb_get_hw_state(dev_priv, ddb);
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
-		skl_pipe_wm_get_hw_state(crtc);
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		intel_crtc = to_intel_crtc(crtc);
+		cstate = to_intel_crtc_state(crtc->state);
+
+		skl_pipe_wm_get_hw_state(crtc, &cstate->wm.skl.optimal);
+
+		if (intel_crtc->active)
+			hw->dirty_pipes |= drm_crtc_mask(crtc);
+	}
 
 	if (dev_priv->active_crtcs) {
 		/* Fully recompute DDB on first atomic commit */
@@ -4402,7 +4409,7 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
 	};
 
 	hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
 
 	memset(active, 0, sizeof(*active));
@@ -4424,7 +4431,7 @@ static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
 		active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
 		active->linetime = hw->wm_linetime[pipe];
 	} else {
-		int level, max_level = ilk_wm_max_level(dev);
+		int level, max_level = ilk_wm_max_level(dev_priv);
 
 		/*
 		 * For inactive pipes, all watermark levels
@@ -4536,11 +4543,11 @@ void vlv_wm_get_hw_state(struct drm_device *dev)
 			plane->wm.fifo_size = 63;
 			break;
 		case DRM_PLANE_TYPE_PRIMARY:
-			plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, 0);
+			plane->wm.fifo_size = vlv_get_fifo_size(dev_priv, plane->pipe, 0);
 			break;
 		case DRM_PLANE_TYPE_OVERLAY:
 			sprite = plane->plane;
-			plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, sprite + 1);
+			plane->wm.fifo_size = vlv_get_fifo_size(dev_priv, plane->pipe, sprite + 1);
 			break;
 		}
 	}
@@ -4605,15 +4612,15 @@ void ilk_wm_get_hw_state(struct drm_device *dev)
 	hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
 
 	hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
-	if (INTEL_INFO(dev)->gen >= 7) {
+	if (INTEL_GEN(dev_priv) >= 7) {
 		hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
 		hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
 	}
 
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
 			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
-	else if (IS_IVYBRIDGE(dev))
+	else if (IS_IVYBRIDGE(dev_priv))
 		hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
 			INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
 
@@ -4653,9 +4660,9 @@ void ilk_wm_get_hw_state(struct drm_device *dev)
  * We don't use the sprite, so we can ignore that.  And on Crestline we have
  * to set the non-SR watermarks to 8.
  */
-void intel_update_watermarks(struct drm_crtc *crtc)
+void intel_update_watermarks(struct intel_crtc *crtc)
 {
-	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
 	if (dev_priv->display.update_wm)
 		dev_priv->display.update_wm(crtc);
@@ -5357,6 +5364,7 @@ static void gen9_enable_rps(struct drm_i915_private *dev_priv)
 static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	uint32_t rc6_mask = 0;
 
 	/* 1a: Software RC state - RC0 */
@@ -5378,7 +5386,7 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 		I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
 	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
 	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
 
 	if (HAS_GUC(dev_priv))
@@ -5394,9 +5402,8 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 	if (intel_enable_rc6() & INTEL_RC6_ENABLE)
 		rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
 	DRM_INFO("RC6 %s\n", onoff(rc6_mask & GEN6_RC_CTL_RC6_ENABLE));
-	/* WaRsUseTimeoutMode */
-	if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_D0) ||
-	    IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
+	/* WaRsUseTimeoutMode:bxt */
+	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
 		I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */
 		I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
 			   GEN7_RC_CTL_TO_MODE |
@@ -5424,6 +5431,7 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 static void gen8_enable_rps(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	uint32_t rc6_mask = 0;
 
 	/* 1a: Software RC state - RC0 */
@@ -5440,7 +5448,7 @@ static void gen8_enable_rps(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
 	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
 	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
 	I915_WRITE(GEN6_RC_SLEEP, 0);
 	if (IS_BROADWELL(dev_priv))
@@ -5500,6 +5508,7 @@ static void gen8_enable_rps(struct drm_i915_private *dev_priv)
 static void gen6_enable_rps(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	u32 rc6vids, rc6_mask = 0;
 	u32 gtfifodbg;
 	int rc6_mode;
@@ -5533,7 +5542,7 @@ static void gen6_enable_rps(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
 	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
 
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
 
 	I915_WRITE(GEN6_RC_SLEEP, 0);
@@ -5570,10 +5579,6 @@ static void gen6_enable_rps(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
 	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
 
-	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
-	if (ret)
-		DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
-
 	reset_rps(dev_priv, gen6_set_rps);
 
 	rc6vids = 0;
@@ -5863,7 +5868,7 @@ static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
 	if (WARN_ON(!dev_priv->vlv_pctx))
 		return;
 
-	i915_gem_object_put_unlocked(dev_priv->vlv_pctx);
+	i915_gem_object_put(dev_priv->vlv_pctx);
 	dev_priv->vlv_pctx = NULL;
 }
 
@@ -5982,6 +5987,7 @@ static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
 static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	u32 gtfifodbg, val, rc6_mode = 0, pcbr;
 
 	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
@@ -6008,7 +6014,7 @@ static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
 	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
 
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
 	I915_WRITE(GEN6_RC_SLEEP, 0);
 
@@ -6070,6 +6076,7 @@ static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
 static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	u32 gtfifodbg, val, rc6_mode = 0;
 
 	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
@@ -6109,7 +6116,7 @@ static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
 	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
 
-	for_each_engine(engine, dev_priv)
+	for_each_engine(engine, dev_priv, id)
 		I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
 
 	I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
@@ -6792,7 +6799,7 @@ static void __intel_autoenable_gt_powersave(struct work_struct *work)
 	if (READ_ONCE(dev_priv->rps.enabled))
 		goto out;
 
-	rcs = &dev_priv->engine[RCS];
+	rcs = dev_priv->engine[RCS];
 	if (rcs->last_context)
 		goto out;
 
@@ -6845,10 +6852,8 @@ void intel_autoenable_gt_powersave(struct drm_i915_private *dev_priv)
 	}
 }
 
-static void ibx_init_clock_gating(struct drm_device *dev)
+static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	/*
 	 * On Ibex Peak and Cougar Point, we need to disable clock
 	 * gating for the panel power sequencer or it will fail to
@@ -6857,9 +6862,8 @@ static void ibx_init_clock_gating(struct drm_device *dev)
 	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
 }
 
-static void g4x_disable_trickle_feed(struct drm_device *dev)
+static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum pipe pipe;
 
 	for_each_pipe(dev_priv, pipe) {
@@ -6872,10 +6876,8 @@ static void g4x_disable_trickle_feed(struct drm_device *dev)
 	}
 }
 
-static void ilk_init_lp_watermarks(struct drm_device *dev)
+static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
 	I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
 	I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
@@ -6886,9 +6888,8 @@ static void ilk_init_lp_watermarks(struct drm_device *dev)
 	 */
 }
 
-static void ironlake_init_clock_gating(struct drm_device *dev)
+static void ironlake_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 
 	/*
@@ -6920,7 +6921,7 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
 		   (I915_READ(DISP_ARB_CTL) |
 		    DISP_FBC_WM_DIS));
 
-	ilk_init_lp_watermarks(dev);
+	ilk_init_lp_watermarks(dev_priv);
 
 	/*
 	 * Based on the document from hardware guys the following bits
@@ -6929,7 +6930,7 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
 	 * The bit 22 of 0x42004
 	 * The bit 7,8,9 of 0x42020.
 	 */
-	if (IS_IRONLAKE_M(dev)) {
+	if (IS_IRONLAKE_M(dev_priv)) {
 		/* WaFbcAsynchFlipDisableFbcQueue:ilk */
 		I915_WRITE(ILK_DISPLAY_CHICKEN1,
 			   I915_READ(ILK_DISPLAY_CHICKEN1) |
@@ -6955,14 +6956,13 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
 	/* WaDisable_RenderCache_OperationalFlush:ilk */
 	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
 
-	g4x_disable_trickle_feed(dev);
+	g4x_disable_trickle_feed(dev_priv);
 
-	ibx_init_clock_gating(dev);
+	ibx_init_clock_gating(dev_priv);
 }
 
-static void cpt_init_clock_gating(struct drm_device *dev)
+static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	int pipe;
 	uint32_t val;
 
@@ -6997,9 +6997,8 @@ static void cpt_init_clock_gating(struct drm_device *dev)
 	}
 }
 
-static void gen6_check_mch_setup(struct drm_device *dev)
+static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t tmp;
 
 	tmp = I915_READ(MCH_SSKPD);
@@ -7008,9 +7007,8 @@ static void gen6_check_mch_setup(struct drm_device *dev)
 			      tmp);
 }
 
-static void gen6_init_clock_gating(struct drm_device *dev)
+static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 
 	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
@@ -7037,7 +7035,7 @@ static void gen6_init_clock_gating(struct drm_device *dev)
 	I915_WRITE(GEN6_GT_MODE,
 		   _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
 
-	ilk_init_lp_watermarks(dev);
+	ilk_init_lp_watermarks(dev_priv);
 
 	I915_WRITE(CACHE_MODE_0,
 		   _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
@@ -7098,11 +7096,11 @@ static void gen6_init_clock_gating(struct drm_device *dev)
 		   ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
 		   ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
 
-	g4x_disable_trickle_feed(dev);
+	g4x_disable_trickle_feed(dev_priv);
 
-	cpt_init_clock_gating(dev);
+	cpt_init_clock_gating(dev_priv);
 
-	gen6_check_mch_setup(dev);
+	gen6_check_mch_setup(dev_priv);
 }
 
 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
@@ -7123,15 +7121,13 @@ static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
 }
 
-static void lpt_init_clock_gating(struct drm_device *dev)
+static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	/*
 	 * TODO: this bit should only be enabled when really needed, then
 	 * disabled when not needed anymore in order to save power.
 	 */
-	if (HAS_PCH_LPT_LP(dev))
+	if (HAS_PCH_LPT_LP(dev_priv))
 		I915_WRITE(SOUTH_DSPCLK_GATE_D,
 			   I915_READ(SOUTH_DSPCLK_GATE_D) |
 			   PCH_LP_PARTITION_LEVEL_DISABLE);
@@ -7142,11 +7138,9 @@ static void lpt_init_clock_gating(struct drm_device *dev)
 		   TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 }
 
-static void lpt_suspend_hw(struct drm_device *dev)
+static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	if (HAS_PCH_LPT_LP(dev)) {
+	if (HAS_PCH_LPT_LP(dev_priv)) {
 		uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
 
 		val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
@@ -7177,11 +7171,9 @@ static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
 	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
 }
 
-static void kabylake_init_clock_gating(struct drm_device *dev)
+static void kabylake_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	gen9_init_clock_gating(dev);
+	gen9_init_clock_gating(dev_priv);
 
 	/* WaDisableSDEUnitClockGating:kbl */
 	if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
@@ -7198,11 +7190,9 @@ static void kabylake_init_clock_gating(struct drm_device *dev)
 		   ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
 }
 
-static void skylake_init_clock_gating(struct drm_device *dev)
+static void skylake_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	gen9_init_clock_gating(dev);
+	gen9_init_clock_gating(dev_priv);
 
 	/* WAC6entrylatency:skl */
 	I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
@@ -7213,12 +7203,11 @@ static void skylake_init_clock_gating(struct drm_device *dev)
 		   ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
 }
 
-static void broadwell_init_clock_gating(struct drm_device *dev)
+static void broadwell_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum pipe pipe;
 
-	ilk_init_lp_watermarks(dev);
+	ilk_init_lp_watermarks(dev_priv);
 
 	/* WaSwitchSolVfFArbitrationPriority:bdw */
 	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
@@ -7261,14 +7250,12 @@ static void broadwell_init_clock_gating(struct drm_device *dev)
 	I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
 		   | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
 
-	lpt_init_clock_gating(dev);
+	lpt_init_clock_gating(dev_priv);
 }
 
-static void haswell_init_clock_gating(struct drm_device *dev)
+static void haswell_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	ilk_init_lp_watermarks(dev);
+	ilk_init_lp_watermarks(dev_priv);
 
 	/* L3 caching of data atomics doesn't work -- disable it. */
 	I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
@@ -7317,15 +7304,14 @@ static void haswell_init_clock_gating(struct drm_device *dev)
 	I915_WRITE(CHICKEN_PAR1_1,
 		   I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
 
-	lpt_init_clock_gating(dev);
+	lpt_init_clock_gating(dev_priv);
 }
 
-static void ivybridge_init_clock_gating(struct drm_device *dev)
+static void ivybridge_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t snpcr;
 
-	ilk_init_lp_watermarks(dev);
+	ilk_init_lp_watermarks(dev_priv);
 
 	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
 
@@ -7339,7 +7325,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
 		   CHICKEN3_DGMG_DONE_FIX_DISABLE);
 
 	/* WaDisablePSDDualDispatchEnable:ivb */
-	if (IS_IVB_GT1(dev))
+	if (IS_IVB_GT1(dev_priv))
 		I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
 			   _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
 
@@ -7355,7 +7341,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
 			GEN7_WA_FOR_GEN7_L3_CONTROL);
 	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
 		   GEN7_WA_L3_CHICKEN_MODE);
-	if (IS_IVB_GT1(dev))
+	if (IS_IVB_GT1(dev_priv))
 		I915_WRITE(GEN7_ROW_CHICKEN2,
 			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 	else {
@@ -7382,7 +7368,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
 			I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
 			GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
-	g4x_disable_trickle_feed(dev);
+	g4x_disable_trickle_feed(dev_priv);
 
 	gen7_setup_fixed_func_scheduler(dev_priv);
 
@@ -7412,16 +7398,14 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
 	snpcr |= GEN6_MBC_SNPCR_MED;
 	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
 
-	if (!HAS_PCH_NOP(dev))
-		cpt_init_clock_gating(dev);
+	if (!HAS_PCH_NOP(dev_priv))
+		cpt_init_clock_gating(dev_priv);
 
-	gen6_check_mch_setup(dev);
+	gen6_check_mch_setup(dev_priv);
 }
 
-static void valleyview_init_clock_gating(struct drm_device *dev)
+static void valleyview_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	/* WaDisableEarlyCull:vlv */
 	I915_WRITE(_3D_CHICKEN3,
 		   _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
@@ -7500,10 +7484,8 @@ static void valleyview_init_clock_gating(struct drm_device *dev)
 	I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
 }
 
-static void cherryview_init_clock_gating(struct drm_device *dev)
+static void cherryview_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	/* WaVSRefCountFullforceMissDisable:chv */
 	/* WaDSRefCountFullforceMissDisable:chv */
 	I915_WRITE(GEN7_FF_THREAD_MODE,
@@ -7536,9 +7518,8 @@ static void cherryview_init_clock_gating(struct drm_device *dev)
 	I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
 }
 
-static void g4x_init_clock_gating(struct drm_device *dev)
+static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	uint32_t dspclk_gate;
 
 	I915_WRITE(RENCLK_GATE_D1, 0);
@@ -7549,7 +7530,7 @@ static void g4x_init_clock_gating(struct drm_device *dev)
 	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
 		OVRUNIT_CLOCK_GATE_DISABLE |
 		OVCUNIT_CLOCK_GATE_DISABLE;
-	if (IS_GM45(dev))
+	if (IS_GM45(dev_priv))
 		dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
 	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
 
@@ -7560,13 +7541,11 @@ static void g4x_init_clock_gating(struct drm_device *dev)
 	/* WaDisable_RenderCache_OperationalFlush:g4x */
 	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
 
-	g4x_disable_trickle_feed(dev);
+	g4x_disable_trickle_feed(dev_priv);
 }
 
-static void crestline_init_clock_gating(struct drm_device *dev)
+static void crestline_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
 	I915_WRITE(RENCLK_GATE_D2, 0);
 	I915_WRITE(DSPCLK_GATE_D, 0);
@@ -7579,10 +7558,8 @@ static void crestline_init_clock_gating(struct drm_device *dev)
 	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
 }
 
-static void broadwater_init_clock_gating(struct drm_device *dev)
+static void broadwater_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
 		   I965_RCC_CLOCK_GATE_DISABLE |
 		   I965_RCPB_CLOCK_GATE_DISABLE |
@@ -7596,16 +7573,15 @@ static void broadwater_init_clock_gating(struct drm_device *dev)
 	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
 }
 
-static void gen3_init_clock_gating(struct drm_device *dev)
+static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 dstate = I915_READ(D_STATE);
 
 	dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
 		DSTATE_DOT_CLOCK_GATING;
 	I915_WRITE(D_STATE, dstate);
 
-	if (IS_PINEVIEW(dev))
+	if (IS_PINEVIEW(dev_priv))
 		I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
 
 	/* IIR "flip pending" means done if this bit is set */
@@ -7621,10 +7597,8 @@ static void gen3_init_clock_gating(struct drm_device *dev)
 		   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 }
 
-static void i85x_init_clock_gating(struct drm_device *dev)
+static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
 
 	/* interrupts should cause a wake up from C3 */
@@ -7635,10 +7609,8 @@ static void i85x_init_clock_gating(struct drm_device *dev)
 		   _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
 }
 
-static void i830_init_clock_gating(struct drm_device *dev)
+static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
 
 	I915_WRITE(MEM_MODE,
@@ -7646,20 +7618,18 @@ static void i830_init_clock_gating(struct drm_device *dev)
 		   _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
 }
 
-void intel_init_clock_gating(struct drm_device *dev)
+void intel_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	dev_priv->display.init_clock_gating(dev);
+	dev_priv->display.init_clock_gating(dev_priv);
 }
 
-void intel_suspend_hw(struct drm_device *dev)
+void intel_suspend_hw(struct drm_i915_private *dev_priv)
 {
-	if (HAS_PCH_LPT(dev))
-		lpt_suspend_hw(dev);
+	if (HAS_PCH_LPT(dev_priv))
+		lpt_suspend_hw(dev_priv);
 }
 
-static void nop_init_clock_gating(struct drm_device *dev)
+static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
 {
 	DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n");
 }
@@ -7714,29 +7684,28 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
 }
 
 /* Set up chip specific power management-related functions */
-void intel_init_pm(struct drm_device *dev)
+void intel_init_pm(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
 	intel_fbc_init(dev_priv);
 
 	/* For cxsr */
-	if (IS_PINEVIEW(dev))
-		i915_pineview_get_mem_freq(dev);
-	else if (IS_GEN5(dev))
-		i915_ironlake_get_mem_freq(dev);
+	if (IS_PINEVIEW(dev_priv))
+		i915_pineview_get_mem_freq(dev_priv);
+	else if (IS_GEN5(dev_priv))
+		i915_ironlake_get_mem_freq(dev_priv);
 
 	/* For FIFO watermark updates */
-	if (INTEL_INFO(dev)->gen >= 9) {
-		skl_setup_wm_latency(dev);
-		dev_priv->display.update_wm = skl_update_wm;
+	if (INTEL_GEN(dev_priv) >= 9) {
+		skl_setup_wm_latency(dev_priv);
+		dev_priv->display.initial_watermarks = skl_initial_wm;
+		dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm;
 		dev_priv->display.compute_global_watermarks = skl_compute_wm;
-	} else if (HAS_PCH_SPLIT(dev)) {
-		ilk_setup_wm_latency(dev);
+	} else if (HAS_PCH_SPLIT(dev_priv)) {
+		ilk_setup_wm_latency(dev_priv);
 
-		if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
+		if ((IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[1] &&
 		     dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
-		    (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
+		    (!IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[0] &&
 		     dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
 			dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
 			dev_priv->display.compute_intermediate_wm =
@@ -7749,14 +7718,14 @@ void intel_init_pm(struct drm_device *dev)
 			DRM_DEBUG_KMS("Failed to read display plane latency. "
 				      "Disable CxSR\n");
 		}
-	} else if (IS_CHERRYVIEW(dev)) {
-		vlv_setup_wm_latency(dev);
+	} else if (IS_CHERRYVIEW(dev_priv)) {
+		vlv_setup_wm_latency(dev_priv);
 		dev_priv->display.update_wm = vlv_update_wm;
-	} else if (IS_VALLEYVIEW(dev)) {
-		vlv_setup_wm_latency(dev);
+	} else if (IS_VALLEYVIEW(dev_priv)) {
+		vlv_setup_wm_latency(dev_priv);
 		dev_priv->display.update_wm = vlv_update_wm;
-	} else if (IS_PINEVIEW(dev)) {
-		if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
+	} else if (IS_PINEVIEW(dev_priv)) {
+		if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
 					    dev_priv->is_ddr3,
 					    dev_priv->fsb_freq,
 					    dev_priv->mem_freq)) {
@@ -7770,15 +7739,15 @@ void intel_init_pm(struct drm_device *dev)
 			dev_priv->display.update_wm = NULL;
 		} else
 			dev_priv->display.update_wm = pineview_update_wm;
-	} else if (IS_G4X(dev)) {
+	} else if (IS_G4X(dev_priv)) {
 		dev_priv->display.update_wm = g4x_update_wm;
-	} else if (IS_GEN4(dev)) {
+	} else if (IS_GEN4(dev_priv)) {
 		dev_priv->display.update_wm = i965_update_wm;
-	} else if (IS_GEN3(dev)) {
+	} else if (IS_GEN3(dev_priv)) {
 		dev_priv->display.update_wm = i9xx_update_wm;
 		dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
-	} else if (IS_GEN2(dev)) {
-		if (INTEL_INFO(dev)->num_pipes == 1) {
+	} else if (IS_GEN2(dev_priv)) {
+		if (INTEL_INFO(dev_priv)->num_pipes == 1) {
 			dev_priv->display.update_wm = i845_update_wm;
 			dev_priv->display.get_fifo_size = i845_get_fifo_size;
 		} else {
@@ -8026,5 +7995,4 @@ void intel_pm_setup(struct drm_device *dev)
 
 	dev_priv->pm.suspended = false;
 	atomic_set(&dev_priv->pm.wakeref_count, 0);
-	atomic_set(&dev_priv->pm.atomic_seq, 0);
 }
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index 108ba1e5d658..7b488e2793d9 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -268,7 +268,7 @@ static void hsw_psr_enable_source(struct intel_dp *intel_dp)
 	val |= max_sleep_time << EDP_PSR_MAX_SLEEP_TIME_SHIFT;
 	val |= idle_frames << EDP_PSR_IDLE_FRAME_SHIFT;
 
-	if (IS_HASWELL(dev))
+	if (IS_HASWELL(dev_priv))
 		val |= EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES;
 
 	if (dev_priv->psr.link_standby)
@@ -344,7 +344,7 @@ static bool intel_psr_match_conditions(struct intel_dp *intel_dp)
 	 * ones. Since by Display design transcoder EDP is tied to port A
 	 * we can safely escape based on the port A.
 	 */
-	if (HAS_DDI(dev) && dig_port->port != PORT_A) {
+	if (HAS_DDI(dev_priv) && dig_port->port != PORT_A) {
 		DRM_DEBUG_KMS("PSR condition failed: Port not supported\n");
 		return false;
 	}
@@ -354,20 +354,20 @@ static bool intel_psr_match_conditions(struct intel_dp *intel_dp)
 		return false;
 	}
 
-	if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 	    !dev_priv->psr.link_standby) {
 		DRM_ERROR("PSR condition failed: Link off requested but not supported on this platform\n");
 		return false;
 	}
 
-	if (IS_HASWELL(dev) &&
+	if (IS_HASWELL(dev_priv) &&
 	    I915_READ(HSW_STEREO_3D_CTL(intel_crtc->config->cpu_transcoder)) &
 		      S3D_ENABLE) {
 		DRM_DEBUG_KMS("PSR condition failed: Stereo 3D is Enabled\n");
 		return false;
 	}
 
-	if (IS_HASWELL(dev) &&
+	if (IS_HASWELL(dev_priv) &&
 	    adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
 		DRM_DEBUG_KMS("PSR condition failed: Interlaced is Enabled\n");
 		return false;
@@ -402,7 +402,7 @@ static void intel_psr_activate(struct intel_dp *intel_dp)
 	lockdep_assert_held(&dev_priv->psr.lock);
 
 	/* Enable/Re-enable PSR on the host */
-	if (HAS_DDI(dev))
+	if (HAS_DDI(dev_priv))
 		/* On HSW+ after we enable PSR on source it will activate it
 		 * as soon as it match configure idle_frame count. So
 		 * we just actually enable it here on activation time.
@@ -427,7 +427,7 @@ void intel_psr_enable(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc);
 
-	if (!HAS_PSR(dev)) {
+	if (!HAS_PSR(dev_priv)) {
 		DRM_DEBUG_KMS("PSR not supported on this platform\n");
 		return;
 	}
@@ -448,7 +448,7 @@ void intel_psr_enable(struct intel_dp *intel_dp)
 
 	dev_priv->psr.busy_frontbuffer_bits = 0;
 
-	if (HAS_DDI(dev)) {
+	if (HAS_DDI(dev_priv)) {
 		hsw_psr_setup_vsc(intel_dp);
 
 		if (dev_priv->psr.psr2_support) {
@@ -472,7 +472,7 @@ void intel_psr_enable(struct intel_dp *intel_dp)
 		/* Enable PSR on the panel */
 		hsw_psr_enable_sink(intel_dp);
 
-		if (INTEL_INFO(dev)->gen >= 9)
+		if (INTEL_GEN(dev_priv) >= 9)
 			intel_psr_activate(intel_dp);
 	} else {
 		vlv_psr_setup_vsc(intel_dp);
@@ -498,7 +498,7 @@ void intel_psr_enable(struct intel_dp *intel_dp)
 	 *     - On HSW/BDW we get a recoverable frozen screen until next
 	 *       exit-activate sequence.
 	 */
-	if (INTEL_INFO(dev)->gen < 9)
+	if (INTEL_GEN(dev_priv) < 9)
 		schedule_delayed_work(&dev_priv->psr.work,
 				      msecs_to_jiffies(intel_dp->panel_power_cycle_delay * 5));
 
@@ -580,7 +580,7 @@ void intel_psr_disable(struct intel_dp *intel_dp)
 	}
 
 	/* Disable PSR on Source */
-	if (HAS_DDI(dev))
+	if (HAS_DDI(dev_priv))
 		hsw_psr_disable(intel_dp);
 	else
 		vlv_psr_disable(intel_dp);
@@ -827,17 +827,17 @@ void intel_psr_init(struct drm_device *dev)
 
 	/* Per platform default */
 	if (i915.enable_psr == -1) {
-		if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+		if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 			i915.enable_psr = 1;
 		else
 			i915.enable_psr = 0;
 	}
 
 	/* Set link_standby x link_off defaults */
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		/* HSW and BDW require workarounds that we don't implement. */
 		dev_priv->psr.link_standby = false;
-	else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		/* On VLV and CHV only standby mode is supported. */
 		dev_priv->psr.link_standby = true;
 	else
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index ed9955dce156..aeb637dc1fdf 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -405,22 +405,6 @@ gen8_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 	return gen8_emit_pipe_control(req, flags, scratch_addr);
 }
 
-u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	u64 acthd;
-
-	if (INTEL_GEN(dev_priv) >= 8)
-		acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
-					 RING_ACTHD_UDW(engine->mmio_base));
-	else if (INTEL_GEN(dev_priv) >= 4)
-		acthd = I915_READ(RING_ACTHD(engine->mmio_base));
-	else
-		acthd = I915_READ(ACTHD);
-
-	return acthd;
-}
-
 static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
@@ -585,9 +569,7 @@ static int init_ring_common(struct intel_engine_cs *engine)
 	I915_WRITE_TAIL(engine, ring->tail);
 	(void)I915_READ_TAIL(engine);
 
-	I915_WRITE_CTL(engine,
-			((ring->size - PAGE_SIZE) & RING_NR_PAGES)
-			| RING_VALID);
+	I915_WRITE_CTL(engine, RING_CTL_SIZE(ring->size) | RING_VALID);
 
 	/* If the head is still not zero, the ring is dead */
 	if (intel_wait_for_register_fw(dev_priv, RING_CTL(engine->mmio_base),
@@ -666,7 +648,7 @@ static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
 	if (ret != 0)
 		return ret;
 
-	ret = i915_gem_render_state_init(req);
+	ret = i915_gem_render_state_emit(req);
 	if (ret)
 		return ret;
 
@@ -851,15 +833,13 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
 	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 			  GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
 
-	/* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */
-	if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
-	    IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
+	/* WaDisableDgMirrorFixInHalfSliceChicken5:bxt */
+	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
 		WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 				  GEN9_DG_MIRROR_FIX_ENABLE);
 
-	/* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
-	if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
-	    IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
+	/* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
+	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
 		WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
 				  GEN9_RHWO_OPTIMIZATION_DISABLE);
 		/*
@@ -869,10 +849,8 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
 		 */
 	}
 
-	/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl */
 	/* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl */
 	WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
-			  GEN9_ENABLE_YV12_BUGFIX |
 			  GEN9_ENABLE_GPGPU_PREEMPTION);
 
 	/* Wa4x4STCOptimizationDisable:skl,bxt,kbl */
@@ -884,9 +862,8 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
 	WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 			  GEN9_CCS_TLB_PREFETCH_ENABLE);
 
-	/* WaDisableMaskBasedCammingInRCC:skl,bxt */
-	if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, SKL_REVID_C0) ||
-	    IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
+	/* WaDisableMaskBasedCammingInRCC:bxt */
+	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
 		WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
 				  PIXEL_MASK_CAMMING_DISABLE);
 
@@ -1003,47 +980,12 @@ static int skl_init_workarounds(struct intel_engine_cs *engine)
 	 * until D0 which is the default case so this is equivalent to
 	 * !WaDisablePerCtxtPreemptionGranularityControl:skl
 	 */
-	if (IS_SKL_REVID(dev_priv, SKL_REVID_E0, REVID_FOREVER)) {
-		I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
-			   _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
-	}
-
-	if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0)) {
-		/* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
-		I915_WRITE(FF_SLICE_CS_CHICKEN2,
-			   _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
-	}
-
-	/* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
-	 * involving this register should also be added to WA batch as required.
-	 */
-	if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0))
-		/* WaDisableLSQCROPERFforOCL:skl */
-		I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
-			   GEN8_LQSC_RO_PERF_DIS);
+	I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
+		   _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
 
 	/* WaEnableGapsTsvCreditFix:skl */
-	if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, REVID_FOREVER)) {
-		I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
-					   GEN9_GAPS_TSV_CREDIT_DISABLE));
-	}
-
-	/* WaDisablePowerCompilerClockGating:skl */
-	if (IS_SKL_REVID(dev_priv, SKL_REVID_B0, SKL_REVID_B0))
-		WA_SET_BIT_MASKED(HIZ_CHICKEN,
-				  BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE);
-
-	/* WaBarrierPerformanceFixDisable:skl */
-	if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, SKL_REVID_D0))
-		WA_SET_BIT_MASKED(HDC_CHICKEN0,
-				  HDC_FENCE_DEST_SLM_DISABLE |
-				  HDC_BARRIER_PERFORMANCE_DISABLE);
-
-	/* WaDisableSbeCacheDispatchPortSharing:skl */
-	if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0))
-		WA_SET_BIT_MASKED(
-			GEN7_HALF_SLICE_CHICKEN1,
-			GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
+	I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
+				   GEN9_GAPS_TSV_CREDIT_DISABLE));
 
 	/* WaDisableGafsUnitClkGating:skl */
 	WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
@@ -1271,91 +1213,64 @@ static void render_ring_cleanup(struct intel_engine_cs *engine)
 	i915_vma_unpin_and_release(&dev_priv->semaphore);
 }
 
-static int gen8_rcs_signal(struct drm_i915_gem_request *req)
+static u32 *gen8_rcs_signal(struct drm_i915_gem_request *req, u32 *out)
 {
-	struct intel_ring *ring = req->ring;
 	struct drm_i915_private *dev_priv = req->i915;
 	struct intel_engine_cs *waiter;
 	enum intel_engine_id id;
-	int ret, num_rings;
-
-	num_rings = INTEL_INFO(dev_priv)->num_rings;
-	ret = intel_ring_begin(req, (num_rings-1) * 8);
-	if (ret)
-		return ret;
 
-	for_each_engine_id(waiter, dev_priv, id) {
+	for_each_engine(waiter, dev_priv, id) {
 		u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
 		if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
 			continue;
 
-		intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
-		intel_ring_emit(ring,
-				PIPE_CONTROL_GLOBAL_GTT_IVB |
-				PIPE_CONTROL_QW_WRITE |
-				PIPE_CONTROL_CS_STALL);
-		intel_ring_emit(ring, lower_32_bits(gtt_offset));
-		intel_ring_emit(ring, upper_32_bits(gtt_offset));
-		intel_ring_emit(ring, req->fence.seqno);
-		intel_ring_emit(ring, 0);
-		intel_ring_emit(ring,
-				MI_SEMAPHORE_SIGNAL |
-				MI_SEMAPHORE_TARGET(waiter->hw_id));
-		intel_ring_emit(ring, 0);
+		*out++ = GFX_OP_PIPE_CONTROL(6);
+		*out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB |
+			  PIPE_CONTROL_QW_WRITE |
+			  PIPE_CONTROL_CS_STALL);
+		*out++ = lower_32_bits(gtt_offset);
+		*out++ = upper_32_bits(gtt_offset);
+		*out++ = req->global_seqno;
+		*out++ = 0;
+		*out++ = (MI_SEMAPHORE_SIGNAL |
+			  MI_SEMAPHORE_TARGET(waiter->hw_id));
+		*out++ = 0;
 	}
-	intel_ring_advance(ring);
 
-	return 0;
+	return out;
 }
 
-static int gen8_xcs_signal(struct drm_i915_gem_request *req)
+static u32 *gen8_xcs_signal(struct drm_i915_gem_request *req, u32 *out)
 {
-	struct intel_ring *ring = req->ring;
 	struct drm_i915_private *dev_priv = req->i915;
 	struct intel_engine_cs *waiter;
 	enum intel_engine_id id;
-	int ret, num_rings;
 
-	num_rings = INTEL_INFO(dev_priv)->num_rings;
-	ret = intel_ring_begin(req, (num_rings-1) * 6);
-	if (ret)
-		return ret;
-
-	for_each_engine_id(waiter, dev_priv, id) {
+	for_each_engine(waiter, dev_priv, id) {
 		u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
 		if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
 			continue;
 
-		intel_ring_emit(ring,
-				(MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW);
-		intel_ring_emit(ring,
-				lower_32_bits(gtt_offset) |
-				MI_FLUSH_DW_USE_GTT);
-		intel_ring_emit(ring, upper_32_bits(gtt_offset));
-		intel_ring_emit(ring, req->fence.seqno);
-		intel_ring_emit(ring,
-				MI_SEMAPHORE_SIGNAL |
-				MI_SEMAPHORE_TARGET(waiter->hw_id));
-		intel_ring_emit(ring, 0);
+		*out++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
+		*out++ = lower_32_bits(gtt_offset) | MI_FLUSH_DW_USE_GTT;
+		*out++ = upper_32_bits(gtt_offset);
+		*out++ = req->global_seqno;
+		*out++ = (MI_SEMAPHORE_SIGNAL |
+			  MI_SEMAPHORE_TARGET(waiter->hw_id));
+		*out++ = 0;
 	}
-	intel_ring_advance(ring);
 
-	return 0;
+	return out;
 }
 
-static int gen6_signal(struct drm_i915_gem_request *req)
+static u32 *gen6_signal(struct drm_i915_gem_request *req, u32 *out)
 {
-	struct intel_ring *ring = req->ring;
 	struct drm_i915_private *dev_priv = req->i915;
 	struct intel_engine_cs *engine;
-	int ret, num_rings;
-
-	num_rings = INTEL_INFO(dev_priv)->num_rings;
-	ret = intel_ring_begin(req, round_up((num_rings-1) * 3, 2));
-	if (ret)
-		return ret;
+	enum intel_engine_id id;
+	int num_rings = 0;
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		i915_reg_t mbox_reg;
 
 		if (!(BIT(engine->hw_id) & GEN6_SEMAPHORES_MASK))
@@ -1363,101 +1278,80 @@ static int gen6_signal(struct drm_i915_gem_request *req)
 
 		mbox_reg = req->engine->semaphore.mbox.signal[engine->hw_id];
 		if (i915_mmio_reg_valid(mbox_reg)) {
-			intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
-			intel_ring_emit_reg(ring, mbox_reg);
-			intel_ring_emit(ring, req->fence.seqno);
+			*out++ = MI_LOAD_REGISTER_IMM(1);
+			*out++ = i915_mmio_reg_offset(mbox_reg);
+			*out++ = req->global_seqno;
+			num_rings++;
 		}
 	}
+	if (num_rings & 1)
+		*out++ = MI_NOOP;
 
-	/* If num_dwords was rounded, make sure the tail pointer is correct */
-	if (num_rings % 2 == 0)
-		intel_ring_emit(ring, MI_NOOP);
-	intel_ring_advance(ring);
-
-	return 0;
+	return out;
 }
 
 static void i9xx_submit_request(struct drm_i915_gem_request *request)
 {
 	struct drm_i915_private *dev_priv = request->i915;
 
-	I915_WRITE_TAIL(request->engine,
-			intel_ring_offset(request->ring, request->tail));
+	i915_gem_request_submit(request);
+
+	I915_WRITE_TAIL(request->engine, request->tail);
 }
 
-static int i9xx_emit_request(struct drm_i915_gem_request *req)
+static void i9xx_emit_breadcrumb(struct drm_i915_gem_request *req,
+				 u32 *out)
 {
-	struct intel_ring *ring = req->ring;
-	int ret;
-
-	ret = intel_ring_begin(req, 4);
-	if (ret)
-		return ret;
-
-	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
-	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-	intel_ring_emit(ring, req->fence.seqno);
-	intel_ring_emit(ring, MI_USER_INTERRUPT);
-	intel_ring_advance(ring);
+	*out++ = MI_STORE_DWORD_INDEX;
+	*out++ = I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT;
+	*out++ = req->global_seqno;
+	*out++ = MI_USER_INTERRUPT;
 
-	req->tail = ring->tail;
-
-	return 0;
+	req->tail = intel_ring_offset(req->ring, out);
 }
 
+static const int i9xx_emit_breadcrumb_sz = 4;
+
 /**
- * gen6_sema_emit_request - Update the semaphore mailbox registers
+ * gen6_sema_emit_breadcrumb - Update the semaphore mailbox registers
  *
  * @request - request to write to the ring
  *
  * Update the mailbox registers in the *other* rings with the current seqno.
  * This acts like a signal in the canonical semaphore.
  */
-static int gen6_sema_emit_request(struct drm_i915_gem_request *req)
+static void gen6_sema_emit_breadcrumb(struct drm_i915_gem_request *req,
+				      u32 *out)
 {
-	int ret;
-
-	ret = req->engine->semaphore.signal(req);
-	if (ret)
-		return ret;
-
-	return i9xx_emit_request(req);
+	return i9xx_emit_breadcrumb(req,
+				    req->engine->semaphore.signal(req, out));
 }
 
-static int gen8_render_emit_request(struct drm_i915_gem_request *req)
+static void gen8_render_emit_breadcrumb(struct drm_i915_gem_request *req,
+					u32 *out)
 {
 	struct intel_engine_cs *engine = req->engine;
-	struct intel_ring *ring = req->ring;
-	int ret;
 
-	if (engine->semaphore.signal) {
-		ret = engine->semaphore.signal(req);
-		if (ret)
-			return ret;
-	}
+	if (engine->semaphore.signal)
+		out = engine->semaphore.signal(req, out);
 
-	ret = intel_ring_begin(req, 8);
-	if (ret)
-		return ret;
-
-	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
-	intel_ring_emit(ring, (PIPE_CONTROL_GLOBAL_GTT_IVB |
+	*out++ = GFX_OP_PIPE_CONTROL(6);
+	*out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB |
 			       PIPE_CONTROL_CS_STALL |
-			       PIPE_CONTROL_QW_WRITE));
-	intel_ring_emit(ring, intel_hws_seqno_address(engine));
-	intel_ring_emit(ring, 0);
-	intel_ring_emit(ring, i915_gem_request_get_seqno(req));
+			       PIPE_CONTROL_QW_WRITE);
+	*out++ = intel_hws_seqno_address(engine);
+	*out++ = 0;
+	*out++ = req->global_seqno;
 	/* We're thrashing one dword of HWS. */
-	intel_ring_emit(ring, 0);
-	intel_ring_emit(ring, MI_USER_INTERRUPT);
-	intel_ring_emit(ring, MI_NOOP);
-	intel_ring_advance(ring);
-
-	req->tail = ring->tail;
+	*out++ = 0;
+	*out++ = MI_USER_INTERRUPT;
+	*out++ = MI_NOOP;
 
-	return 0;
+	req->tail = intel_ring_offset(req->ring, out);
 }
 
+static const int gen8_render_emit_breadcrumb_sz = 8;
+
 /**
  * intel_ring_sync - sync the waiter to the signaller on seqno
  *
@@ -1484,7 +1378,7 @@ gen8_ring_sync_to(struct drm_i915_gem_request *req,
 			MI_SEMAPHORE_WAIT |
 			MI_SEMAPHORE_GLOBAL_GTT |
 			MI_SEMAPHORE_SAD_GTE_SDD);
-	intel_ring_emit(ring, signal->fence.seqno);
+	intel_ring_emit(ring, signal->global_seqno);
 	intel_ring_emit(ring, lower_32_bits(offset));
 	intel_ring_emit(ring, upper_32_bits(offset));
 	intel_ring_advance(ring);
@@ -1522,7 +1416,7 @@ gen6_ring_sync_to(struct drm_i915_gem_request *req,
 	 * seqno is >= the last seqno executed. However for hardware the
 	 * comparison is strictly greater than.
 	 */
-	intel_ring_emit(ring, signal->fence.seqno - 1);
+	intel_ring_emit(ring, signal->global_seqno - 1);
 	intel_ring_emit(ring, 0);
 	intel_ring_emit(ring, MI_NOOP);
 	intel_ring_advance(ring);
@@ -1665,7 +1559,7 @@ hsw_vebox_irq_enable(struct intel_engine_cs *engine)
 	struct drm_i915_private *dev_priv = engine->i915;
 
 	I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
-	gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask);
+	gen6_unmask_pm_irq(dev_priv, engine->irq_enable_mask);
 }
 
 static void
@@ -1674,7 +1568,7 @@ hsw_vebox_irq_disable(struct intel_engine_cs *engine)
 	struct drm_i915_private *dev_priv = engine->i915;
 
 	I915_WRITE_IMR(engine, ~0);
-	gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask);
+	gen6_mask_pm_irq(dev_priv, engine->irq_enable_mask);
 }
 
 static void
@@ -1819,14 +1713,19 @@ static void cleanup_phys_status_page(struct intel_engine_cs *engine)
 static void cleanup_status_page(struct intel_engine_cs *engine)
 {
 	struct i915_vma *vma;
+	struct drm_i915_gem_object *obj;
 
 	vma = fetch_and_zero(&engine->status_page.vma);
 	if (!vma)
 		return;
 
+	obj = vma->obj;
+
 	i915_vma_unpin(vma);
-	i915_gem_object_unpin_map(vma->obj);
-	i915_vma_put(vma);
+	i915_vma_close(vma);
+
+	i915_gem_object_unpin_map(obj);
+	__i915_gem_object_release_unless_active(obj);
 }
 
 static int init_status_page(struct intel_engine_cs *engine)
@@ -1834,9 +1733,10 @@ static int init_status_page(struct intel_engine_cs *engine)
 	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
 	unsigned int flags;
+	void *vaddr;
 	int ret;
 
-	obj = i915_gem_object_create(&engine->i915->drm, 4096);
+	obj = i915_gem_object_create_internal(engine->i915, 4096);
 	if (IS_ERR(obj)) {
 		DRM_ERROR("Failed to allocate status page\n");
 		return PTR_ERR(obj);
@@ -1869,15 +1769,22 @@ static int init_status_page(struct intel_engine_cs *engine)
 	if (ret)
 		goto err;
 
+	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
+	if (IS_ERR(vaddr)) {
+		ret = PTR_ERR(vaddr);
+		goto err_unpin;
+	}
+
 	engine->status_page.vma = vma;
 	engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
-	engine->status_page.page_addr =
-		i915_gem_object_pin_map(obj, I915_MAP_WB);
+	engine->status_page.page_addr = memset(vaddr, 0, 4096);
 
 	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 			 engine->name, i915_ggtt_offset(vma));
 	return 0;
 
+err_unpin:
+	i915_vma_unpin(vma);
 err:
 	i915_gem_object_put(obj);
 	return ret;
@@ -1989,6 +1896,7 @@ intel_engine_create_ring(struct intel_engine_cs *engine, int size)
 	struct i915_vma *vma;
 
 	GEM_BUG_ON(!is_power_of_2(size));
+	GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
 
 	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
 	if (!ring)
@@ -2023,7 +1931,11 @@ intel_engine_create_ring(struct intel_engine_cs *engine, int size)
 void
 intel_ring_free(struct intel_ring *ring)
 {
-	i915_vma_put(ring->vma);
+	struct drm_i915_gem_object *obj = ring->vma->obj;
+
+	i915_vma_close(ring->vma);
+	__i915_gem_object_release_unless_active(obj);
+
 	kfree(ring);
 }
 
@@ -2039,14 +1951,13 @@ static int intel_ring_context_pin(struct i915_gem_context *ctx,
 		return 0;
 
 	if (ce->state) {
-		ret = i915_gem_object_set_to_gtt_domain(ce->state->obj, false);
-		if (ret)
-			goto error;
+		struct i915_vma *vma;
 
-		ret = i915_vma_pin(ce->state, 0, ctx->ggtt_alignment,
-				   PIN_GLOBAL | PIN_HIGH);
-		if (ret)
+		vma = i915_gem_context_pin_legacy(ctx, PIN_HIGH);
+		if (IS_ERR(vma)) {
+			ret = PTR_ERR(vma);
 			goto error;
+		}
 	}
 
 	/* The kernel context is only used as a placeholder for flushing the
@@ -2093,9 +2004,6 @@ static int intel_init_ring_buffer(struct intel_engine_cs *engine)
 
 	intel_engine_setup_common(engine);
 
-	memset(engine->semaphore.sync_seqno, 0,
-	       sizeof(engine->semaphore.sync_seqno));
-
 	ret = intel_engine_init_common(engine);
 	if (ret)
 		goto error;
@@ -2146,9 +2054,6 @@ void intel_engine_cleanup(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv;
 
-	if (!intel_engine_initialized(engine))
-		return;
-
 	dev_priv = engine->i915;
 
 	if (engine->buffer) {
@@ -2175,13 +2080,16 @@ void intel_engine_cleanup(struct intel_engine_cs *engine)
 	intel_ring_context_unpin(dev_priv->kernel_context, engine);
 
 	engine->i915 = NULL;
+	dev_priv->engine[engine->id] = NULL;
+	kfree(engine);
 }
 
 void intel_legacy_submission_resume(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 
-	for_each_engine(engine, dev_priv) {
+	for_each_engine(engine, dev_priv, id) {
 		engine->buffer->head = engine->buffer->tail;
 		engine->buffer->last_retired_head = -1;
 	}
@@ -2211,7 +2119,9 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
 {
 	struct intel_ring *ring = req->ring;
 	struct drm_i915_gem_request *target;
-	int ret;
+	long timeout;
+
+	lockdep_assert_held(&req->i915->drm.struct_mutex);
 
 	intel_ring_update_space(ring);
 	if (ring->space >= bytes)
@@ -2241,11 +2151,11 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
 	if (WARN_ON(&target->ring_link == &ring->request_list))
 		return -ENOSPC;
 
-	ret = i915_wait_request(target,
-				I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
-				NULL, NO_WAITBOOST);
-	if (ret)
-		return ret;
+	timeout = i915_wait_request(target,
+				    I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
+				    MAX_SCHEDULE_TIMEOUT);
+	if (timeout < 0)
+		return timeout;
 
 	i915_gem_request_retire_upto(target);
 
@@ -2674,9 +2584,22 @@ static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
 	engine->init_hw = init_ring_common;
 	engine->reset_hw = reset_ring_common;
 
-	engine->emit_request = i9xx_emit_request;
-	if (i915.semaphores)
-		engine->emit_request = gen6_sema_emit_request;
+	engine->emit_breadcrumb = i9xx_emit_breadcrumb;
+	engine->emit_breadcrumb_sz = i9xx_emit_breadcrumb_sz;
+	if (i915.semaphores) {
+		int num_rings;
+
+		engine->emit_breadcrumb = gen6_sema_emit_breadcrumb;
+
+		num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1;
+		if (INTEL_GEN(dev_priv) >= 8) {
+			engine->emit_breadcrumb_sz += num_rings * 6;
+		} else {
+			engine->emit_breadcrumb_sz += num_rings * 3;
+			if (num_rings & 1)
+				engine->emit_breadcrumb_sz++;
+		}
+	}
 	engine->submit_request = i9xx_submit_request;
 
 	if (INTEL_GEN(dev_priv) >= 8)
@@ -2703,10 +2626,18 @@ int intel_init_render_ring_buffer(struct intel_engine_cs *engine)
 
 	if (INTEL_GEN(dev_priv) >= 8) {
 		engine->init_context = intel_rcs_ctx_init;
-		engine->emit_request = gen8_render_emit_request;
+		engine->emit_breadcrumb = gen8_render_emit_breadcrumb;
+		engine->emit_breadcrumb_sz = gen8_render_emit_breadcrumb_sz;
 		engine->emit_flush = gen8_render_ring_flush;
-		if (i915.semaphores)
+		if (i915.semaphores) {
+			int num_rings;
+
 			engine->semaphore.signal = gen8_rcs_signal;
+
+			num_rings =
+				hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1;
+			engine->emit_breadcrumb_sz += num_rings * 6;
+		}
 	} else if (INTEL_GEN(dev_priv) >= 6) {
 		engine->init_context = intel_rcs_ctx_init;
 		engine->emit_flush = gen7_render_ring_flush;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index ec0b4a0c605d..3466b4e77e7c 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -4,6 +4,7 @@
 #include <linux/hashtable.h>
 #include "i915_gem_batch_pool.h"
 #include "i915_gem_request.h"
+#include "i915_gem_timeline.h"
 
 #define I915_CMD_HASH_ORDER 9
 
@@ -73,13 +74,40 @@ enum intel_engine_hangcheck_action {
 
 #define HANGCHECK_SCORE_RING_HUNG 31
 
+#define I915_MAX_SLICES	3
+#define I915_MAX_SUBSLICES 3
+
+#define instdone_slice_mask(dev_priv__) \
+	(INTEL_GEN(dev_priv__) == 7 ? \
+	 1 : INTEL_INFO(dev_priv__)->sseu.slice_mask)
+
+#define instdone_subslice_mask(dev_priv__) \
+	(INTEL_GEN(dev_priv__) == 7 ? \
+	 1 : INTEL_INFO(dev_priv__)->sseu.subslice_mask)
+
+#define for_each_instdone_slice_subslice(dev_priv__, slice__, subslice__) \
+	for ((slice__) = 0, (subslice__) = 0; \
+	     (slice__) < I915_MAX_SLICES; \
+	     (subslice__) = ((subslice__) + 1) < I915_MAX_SUBSLICES ? (subslice__) + 1 : 0, \
+	       (slice__) += ((subslice__) == 0)) \
+		for_each_if((BIT(slice__) & instdone_slice_mask(dev_priv__)) && \
+			    (BIT(subslice__) & instdone_subslice_mask(dev_priv__)))
+
+struct intel_instdone {
+	u32 instdone;
+	/* The following exist only in the RCS engine */
+	u32 slice_common;
+	u32 sampler[I915_MAX_SLICES][I915_MAX_SUBSLICES];
+	u32 row[I915_MAX_SLICES][I915_MAX_SUBSLICES];
+};
+
 struct intel_engine_hangcheck {
 	u64 acthd;
 	u32 seqno;
 	int score;
 	enum intel_engine_hangcheck_action action;
 	int deadlock;
-	u32 instdone[I915_NUM_INSTDONE_REG];
+	struct intel_instdone instdone;
 };
 
 struct intel_ring {
@@ -130,6 +158,7 @@ struct i915_ctx_workarounds {
 };
 
 struct drm_i915_gem_request;
+struct intel_render_state;
 
 struct intel_engine_cs {
 	struct drm_i915_private *i915;
@@ -141,7 +170,6 @@ struct intel_engine_cs {
 		VCS2,	/* Keep instances of the same type engine together. */
 		VECS
 	} id;
-#define I915_NUM_ENGINES 5
 #define _VCS(n) (VCS + (n))
 	unsigned int exec_id;
 	enum intel_engine_hw_id {
@@ -152,10 +180,12 @@ struct intel_engine_cs {
 		VCS2_HW
 	} hw_id;
 	enum intel_engine_hw_id guc_id; /* XXX same as hw_id? */
-	u64 fence_context;
 	u32		mmio_base;
 	unsigned int irq_shift;
 	struct intel_ring *buffer;
+	struct intel_timeline *timeline;
+
+	struct intel_render_state *render_state;
 
 	/* Rather than have every client wait upon all user interrupts,
 	 * with the herd waking after every interrupt and each doing the
@@ -177,7 +207,7 @@ struct intel_engine_cs {
 		struct task_struct __rcu *irq_seqno_bh; /* bh for interrupts */
 		bool irq_posted;
 
-		spinlock_t lock; /* protects the lists of requests */
+		spinlock_t lock; /* protects the lists of requests; irqsafe */
 		struct rb_root waiters; /* sorted by retirement, priority */
 		struct rb_root signals; /* sorted by retirement */
 		struct intel_wait *first_wait; /* oldest waiter by retirement */
@@ -225,7 +255,9 @@ struct intel_engine_cs {
 #define I915_DISPATCH_SECURE BIT(0)
 #define I915_DISPATCH_PINNED BIT(1)
 #define I915_DISPATCH_RS     BIT(2)
-	int		(*emit_request)(struct drm_i915_gem_request *req);
+	void		(*emit_breadcrumb)(struct drm_i915_gem_request *req,
+					   u32 *out);
+	int		emit_breadcrumb_sz;
 
 	/* Pass the request to the hardware queue (e.g. directly into
 	 * the legacy ringbuffer or to the end of an execlist).
@@ -235,6 +267,15 @@ struct intel_engine_cs {
 	 */
 	void		(*submit_request)(struct drm_i915_gem_request *req);
 
+	/* Call when the priority on a request has changed and it and its
+	 * dependencies may need rescheduling. Note the request itself may
+	 * not be ready to run!
+	 *
+	 * Called under the struct_mutex.
+	 */
+	void		(*schedule)(struct drm_i915_gem_request *request,
+				    int priority);
+
 	/* Some chipsets are not quite as coherent as advertised and need
 	 * an expensive kick to force a true read of the up-to-date seqno.
 	 * However, the up-to-date seqno is not always required and the last
@@ -282,8 +323,6 @@ struct intel_engine_cs {
 	 *  ie. transpose of f(x, y)
 	 */
 	struct {
-		u32	sync_seqno[I915_NUM_ENGINES-1];
-
 		union {
 #define GEN6_SEMAPHORE_LAST	VECS_HW
 #define GEN6_NUM_SEMAPHORES	(GEN6_SEMAPHORE_LAST + 1)
@@ -300,43 +339,22 @@ struct intel_engine_cs {
 		/* AKA wait() */
 		int	(*sync_to)(struct drm_i915_gem_request *req,
 				   struct drm_i915_gem_request *signal);
-		int	(*signal)(struct drm_i915_gem_request *req);
+		u32	*(*signal)(struct drm_i915_gem_request *req, u32 *out);
 	} semaphore;
 
 	/* Execlists */
 	struct tasklet_struct irq_tasklet;
-	spinlock_t execlist_lock; /* used inside tasklet, use spin_lock_bh */
 	struct execlist_port {
 		struct drm_i915_gem_request *request;
 		unsigned int count;
 	} execlist_port[2];
-	struct list_head execlist_queue;
+	struct rb_root execlist_queue;
+	struct rb_node *execlist_first;
 	unsigned int fw_domains;
 	bool disable_lite_restore_wa;
 	bool preempt_wa;
 	u32 ctx_desc_template;
 
-	/**
-	 * List of breadcrumbs associated with GPU requests currently
-	 * outstanding.
-	 */
-	struct list_head request_list;
-
-	/**
-	 * Seqno of request most recently submitted to request_list.
-	 * Used exclusively by hang checker to avoid grabbing lock while
-	 * inspecting request list.
-	 */
-	u32 last_submitted_seqno;
-	u32 last_pending_seqno;
-
-	/* An RCU guarded pointer to the last request. No reference is
-	 * held to the request, users must carefully acquire a reference to
-	 * the request using i915_gem_active_get_rcu(), or hold the
-	 * struct_mutex.
-	 */
-	struct i915_gem_active last_request;
-
 	struct i915_gem_context *last_context;
 
 	struct intel_engine_hangcheck hangcheck;
@@ -368,39 +386,12 @@ struct intel_engine_cs {
 	u32 (*get_cmd_length_mask)(u32 cmd_header);
 };
 
-static inline bool
-intel_engine_initialized(const struct intel_engine_cs *engine)
-{
-	return engine->i915 != NULL;
-}
-
 static inline unsigned
 intel_engine_flag(const struct intel_engine_cs *engine)
 {
 	return 1 << engine->id;
 }
 
-static inline u32
-intel_engine_sync_index(struct intel_engine_cs *engine,
-			struct intel_engine_cs *other)
-{
-	int idx;
-
-	/*
-	 * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2;
-	 * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs;
-	 * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs;
-	 * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs;
-	 * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
-	 */
-
-	idx = (other - engine) - 1;
-	if (idx < 0)
-		idx += I915_NUM_ENGINES;
-
-	return idx;
-}
-
 static inline void
 intel_flush_status_page(struct intel_engine_cs *engine, int reg)
 {
@@ -483,30 +474,23 @@ static inline void intel_ring_advance(struct intel_ring *ring)
 	 */
 }
 
-static inline u32 intel_ring_offset(struct intel_ring *ring, u32 value)
+static inline u32 intel_ring_offset(struct intel_ring *ring, void *addr)
 {
 	/* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */
-	return value & (ring->size - 1);
+	u32 offset = addr - ring->vaddr;
+	return offset & (ring->size - 1);
 }
 
 int __intel_ring_space(int head, int tail, int size);
 void intel_ring_update_space(struct intel_ring *ring);
 
-void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno);
+void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno);
 
 void intel_engine_setup_common(struct intel_engine_cs *engine);
 int intel_engine_init_common(struct intel_engine_cs *engine);
 int intel_engine_create_scratch(struct intel_engine_cs *engine, int size);
 void intel_engine_cleanup_common(struct intel_engine_cs *engine);
 
-static inline int intel_engine_idle(struct intel_engine_cs *engine,
-				    unsigned int flags)
-{
-	/* Wait upon the last request to be completed */
-	return i915_gem_active_wait_unlocked(&engine->last_request,
-					     flags, NULL, NULL);
-}
-
 int intel_init_render_ring_buffer(struct intel_engine_cs *engine);
 int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine);
 int intel_init_bsd2_ring_buffer(struct intel_engine_cs *engine);
@@ -514,13 +498,30 @@ int intel_init_blt_ring_buffer(struct intel_engine_cs *engine);
 int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine);
 
 u64 intel_engine_get_active_head(struct intel_engine_cs *engine);
+u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine);
+
 static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine)
 {
 	return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
 }
 
+static inline u32 intel_engine_last_submit(struct intel_engine_cs *engine)
+{
+	/* We are only peeking at the tail of the submit queue (and not the
+	 * queue itself) in order to gain a hint as to the current active
+	 * state of the engine. Callers are not expected to be taking
+	 * engine->timeline->lock, nor are they expected to be concerned
+	 * wtih serialising this hint with anything, so document it as
+	 * a hint and nothing more.
+	 */
+	return READ_ONCE(engine->timeline->last_submitted_seqno);
+}
+
 int init_workarounds_ring(struct intel_engine_cs *engine);
 
+void intel_engine_get_instdone(struct intel_engine_cs *engine,
+			       struct intel_instdone *instdone);
+
 /*
  * Arbitrary size for largest possible 'add request' sequence. The code paths
  * are complex and variable. Empirical measurement shows that the worst case
@@ -586,12 +587,6 @@ static inline bool intel_engine_wakeup(const struct intel_engine_cs *engine)
 
 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine);
 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
-unsigned int intel_kick_waiters(struct drm_i915_private *i915);
-unsigned int intel_kick_signalers(struct drm_i915_private *i915);
-
-static inline bool intel_engine_is_active(struct intel_engine_cs *engine)
-{
-	return i915_gem_active_isset(&engine->last_request);
-}
+unsigned int intel_breadcrumbs_busy(struct drm_i915_private *i915);
 
 #endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index a38c2fefe85a..356c662ad453 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -288,7 +288,6 @@ void intel_display_set_init_power(struct drm_i915_private *dev_priv,
 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
 {
 	struct pci_dev *pdev = dev_priv->drm.pdev;
-	struct drm_device *dev = &dev_priv->drm;
 
 	/*
 	 * After we re-enable the power well, if we touch VGA register 0x3d5
@@ -304,7 +303,7 @@ static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
 	outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
 	vga_put(pdev, VGA_RSRC_LEGACY_IO);
 
-	if (IS_BROADWELL(dev))
+	if (IS_BROADWELL(dev_priv))
 		gen8_irq_power_well_post_enable(dev_priv,
 						1 << PIPE_C | 1 << PIPE_B);
 }
@@ -331,7 +330,7 @@ static void skl_power_well_post_enable(struct drm_i915_private *dev_priv,
 	 * sure vgacon can keep working normally without triggering interrupts
 	 * and error messages.
 	 */
-	if (power_well->data == SKL_DISP_PW_2) {
+	if (power_well->id == SKL_DISP_PW_2) {
 		vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
 		outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
 		vga_put(pdev, VGA_RSRC_LEGACY_IO);
@@ -344,7 +343,7 @@ static void skl_power_well_post_enable(struct drm_i915_private *dev_priv,
 static void skl_power_well_pre_disable(struct drm_i915_private *dev_priv,
 				       struct i915_power_well *power_well)
 {
-	if (power_well->data == SKL_DISP_PW_2)
+	if (power_well->id == SKL_DISP_PW_2)
 		gen8_irq_power_well_pre_disable(dev_priv,
 						1 << PIPE_C | 1 << PIPE_B);
 }
@@ -659,7 +658,7 @@ static void
 gen9_sanitize_power_well_requests(struct drm_i915_private *dev_priv,
 				  struct i915_power_well *power_well)
 {
-	enum skl_disp_power_wells power_well_id = power_well->data;
+	enum skl_disp_power_wells power_well_id = power_well->id;
 	u32 val;
 	u32 mask;
 
@@ -704,7 +703,7 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv,
 	tmp = I915_READ(HSW_PWR_WELL_DRIVER);
 	fuse_status = I915_READ(SKL_FUSE_STATUS);
 
-	switch (power_well->data) {
+	switch (power_well->id) {
 	case SKL_DISP_PW_1:
 		if (intel_wait_for_register(dev_priv,
 					    SKL_FUSE_STATUS,
@@ -728,13 +727,13 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv,
 	case SKL_DISP_PW_MISC_IO:
 		break;
 	default:
-		WARN(1, "Unknown power well %lu\n", power_well->data);
+		WARN(1, "Unknown power well %lu\n", power_well->id);
 		return;
 	}
 
-	req_mask = SKL_POWER_WELL_REQ(power_well->data);
+	req_mask = SKL_POWER_WELL_REQ(power_well->id);
 	enable_requested = tmp & req_mask;
-	state_mask = SKL_POWER_WELL_STATE(power_well->data);
+	state_mask = SKL_POWER_WELL_STATE(power_well->id);
 	is_enabled = tmp & state_mask;
 
 	if (!enable && enable_requested)
@@ -770,14 +769,14 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv,
 			  power_well->name, enable ? "enable" : "disable");
 
 	if (check_fuse_status) {
-		if (power_well->data == SKL_DISP_PW_1) {
+		if (power_well->id == SKL_DISP_PW_1) {
 			if (intel_wait_for_register(dev_priv,
 						    SKL_FUSE_STATUS,
 						    SKL_FUSE_PG1_DIST_STATUS,
 						    SKL_FUSE_PG1_DIST_STATUS,
 						    1))
 				DRM_ERROR("PG1 distributing status timeout\n");
-		} else if (power_well->data == SKL_DISP_PW_2) {
+		} else if (power_well->id == SKL_DISP_PW_2) {
 			if (intel_wait_for_register(dev_priv,
 						    SKL_FUSE_STATUS,
 						    SKL_FUSE_PG2_DIST_STATUS,
@@ -819,8 +818,8 @@ static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 static bool skl_power_well_enabled(struct drm_i915_private *dev_priv,
 					struct i915_power_well *power_well)
 {
-	uint32_t mask = SKL_POWER_WELL_REQ(power_well->data) |
-		SKL_POWER_WELL_STATE(power_well->data);
+	uint32_t mask = SKL_POWER_WELL_REQ(power_well->id) |
+		SKL_POWER_WELL_STATE(power_well->id);
 
 	return (I915_READ(HSW_PWR_WELL_DRIVER) & mask) == mask;
 }
@@ -846,45 +845,22 @@ static void skl_power_well_disable(struct drm_i915_private *dev_priv,
 	skl_set_power_well(dev_priv, power_well, false);
 }
 
-static enum dpio_phy bxt_power_well_to_phy(struct i915_power_well *power_well)
-{
-	enum skl_disp_power_wells power_well_id = power_well->data;
-
-	return power_well_id == BXT_DPIO_CMN_A ? DPIO_PHY1 : DPIO_PHY0;
-}
-
 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 					   struct i915_power_well *power_well)
 {
-	enum skl_disp_power_wells power_well_id = power_well->data;
-	struct i915_power_well *cmn_a_well = NULL;
-
-	if (power_well_id == BXT_DPIO_CMN_BC) {
-		/*
-		 * We need to copy the GRC calibration value from the eDP PHY,
-		 * so make sure it's powered up.
-		 */
-		cmn_a_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
-		intel_power_well_get(dev_priv, cmn_a_well);
-	}
-
-	bxt_ddi_phy_init(dev_priv, bxt_power_well_to_phy(power_well));
-
-	if (cmn_a_well)
-		intel_power_well_put(dev_priv, cmn_a_well);
+	bxt_ddi_phy_init(dev_priv, power_well->data);
 }
 
 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 					    struct i915_power_well *power_well)
 {
-	bxt_ddi_phy_uninit(dev_priv, bxt_power_well_to_phy(power_well));
+	bxt_ddi_phy_uninit(dev_priv, power_well->data);
 }
 
 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
 					    struct i915_power_well *power_well)
 {
-	return bxt_ddi_phy_is_enabled(dev_priv,
-				      bxt_power_well_to_phy(power_well));
+	return bxt_ddi_phy_is_enabled(dev_priv, power_well->data);
 }
 
 static void bxt_dpio_cmn_power_well_sync_hw(struct drm_i915_private *dev_priv,
@@ -903,13 +879,11 @@ static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
 
 	power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
 	if (power_well->count > 0)
-		bxt_ddi_phy_verify_state(dev_priv,
-					 bxt_power_well_to_phy(power_well));
+		bxt_ddi_phy_verify_state(dev_priv, power_well->data);
 
 	power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC);
 	if (power_well->count > 0)
-		bxt_ddi_phy_verify_state(dev_priv,
-					 bxt_power_well_to_phy(power_well));
+		bxt_ddi_phy_verify_state(dev_priv, power_well->data);
 }
 
 static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
@@ -933,7 +907,7 @@ static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
 	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 
 	WARN_ON(dev_priv->cdclk_freq !=
-		dev_priv->display.get_display_clock_speed(&dev_priv->drm));
+		dev_priv->display.get_display_clock_speed(dev_priv));
 
 	gen9_assert_dbuf_enabled(dev_priv);
 
@@ -976,7 +950,7 @@ static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
 			       struct i915_power_well *power_well, bool enable)
 {
-	enum punit_power_well power_well_id = power_well->data;
+	enum punit_power_well power_well_id = power_well->id;
 	u32 mask;
 	u32 state;
 	u32 ctrl;
@@ -1030,7 +1004,7 @@ static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
 				   struct i915_power_well *power_well)
 {
-	int power_well_id = power_well->data;
+	int power_well_id = power_well->id;
 	bool enabled = false;
 	u32 mask;
 	u32 state;
@@ -1092,7 +1066,7 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
 	 *
 	 * CHV DPLL B/C have some issues if VGA mode is enabled.
 	 */
-	for_each_pipe(&dev_priv->drm, pipe) {
+	for_each_pipe(dev_priv, pipe) {
 		u32 val = I915_READ(DPLL(pipe));
 
 		val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
@@ -1123,7 +1097,7 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
 			intel_crt_reset(&encoder->base);
 	}
 
-	i915_redisable_vga_power_on(&dev_priv->drm);
+	i915_redisable_vga_power_on(dev_priv);
 
 	intel_pps_unlock_regs_wa(dev_priv);
 }
@@ -1147,7 +1121,7 @@ static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
 					  struct i915_power_well *power_well)
 {
-	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
+	WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
 
 	vlv_set_power_well(dev_priv, power_well, true);
 
@@ -1157,7 +1131,7 @@ static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
 					   struct i915_power_well *power_well)
 {
-	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
+	WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
 
 	vlv_display_power_well_deinit(dev_priv);
 
@@ -1167,7 +1141,7 @@ static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 					   struct i915_power_well *power_well)
 {
-	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
+	WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
 
 	/* since ref/cri clock was enabled */
 	udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
@@ -1193,7 +1167,7 @@ static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 {
 	enum pipe pipe;
 
-	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
+	WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
 
 	for_each_pipe(dev_priv, pipe)
 		assert_pll_disabled(dev_priv, pipe);
@@ -1216,7 +1190,7 @@ static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_pr
 		struct i915_power_well *power_well;
 
 		power_well = &power_domains->power_wells[i];
-		if (power_well->data == power_well_id)
+		if (power_well->id == power_well_id)
 			return power_well;
 	}
 
@@ -1340,10 +1314,10 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 	enum pipe pipe;
 	uint32_t tmp;
 
-	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC &&
-		     power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D);
+	WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
+		     power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
 
-	if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
+	if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
 		pipe = PIPE_A;
 		phy = DPIO_PHY0;
 	} else {
@@ -1371,7 +1345,7 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 		DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
 	vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
 
-	if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
+	if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
 		tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
 		tmp |= DPIO_DYNPWRDOWNEN_CH1;
 		vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
@@ -1402,10 +1376,10 @@ static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 {
 	enum dpio_phy phy;
 
-	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC &&
-		     power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D);
+	WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
+		     power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
 
-	if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
+	if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
 		phy = DPIO_PHY0;
 		assert_pll_disabled(dev_priv, PIPE_A);
 		assert_pll_disabled(dev_priv, PIPE_B);
@@ -1554,7 +1528,7 @@ void chv_phy_powergate_lanes(struct intel_encoder *encoder,
 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
 					struct i915_power_well *power_well)
 {
-	enum pipe pipe = power_well->data;
+	enum pipe pipe = power_well->id;
 	bool enabled;
 	u32 state, ctrl;
 
@@ -1584,7 +1558,7 @@ static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
 				    struct i915_power_well *power_well,
 				    bool enable)
 {
-	enum pipe pipe = power_well->data;
+	enum pipe pipe = power_well->id;
 	u32 state;
 	u32 ctrl;
 
@@ -1617,7 +1591,7 @@ out:
 static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
 					struct i915_power_well *power_well)
 {
-	WARN_ON_ONCE(power_well->data != PIPE_A);
+	WARN_ON_ONCE(power_well->id != PIPE_A);
 
 	chv_set_pipe_power_well(dev_priv, power_well, power_well->count > 0);
 }
@@ -1625,7 +1599,7 @@ static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
 				       struct i915_power_well *power_well)
 {
-	WARN_ON_ONCE(power_well->data != PIPE_A);
+	WARN_ON_ONCE(power_well->id != PIPE_A);
 
 	chv_set_pipe_power_well(dev_priv, power_well, true);
 
@@ -1635,7 +1609,7 @@ static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
 					struct i915_power_well *power_well)
 {
-	WARN_ON_ONCE(power_well->data != PIPE_A);
+	WARN_ON_ONCE(power_well->id != PIPE_A);
 
 	vlv_display_power_well_deinit(dev_priv);
 
@@ -1979,12 +1953,12 @@ static struct i915_power_well vlv_power_wells[] = {
 		.always_on = 1,
 		.domains = POWER_DOMAIN_MASK,
 		.ops = &i9xx_always_on_power_well_ops,
-		.data = PUNIT_POWER_WELL_ALWAYS_ON,
+		.id = PUNIT_POWER_WELL_ALWAYS_ON,
 	},
 	{
 		.name = "display",
 		.domains = VLV_DISPLAY_POWER_DOMAINS,
-		.data = PUNIT_POWER_WELL_DISP2D,
+		.id = PUNIT_POWER_WELL_DISP2D,
 		.ops = &vlv_display_power_well_ops,
 	},
 	{
@@ -1994,7 +1968,7 @@ static struct i915_power_well vlv_power_wells[] = {
 			   VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
 			   VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
 		.ops = &vlv_dpio_power_well_ops,
-		.data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
+		.id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
 	},
 	{
 		.name = "dpio-tx-b-23",
@@ -2003,7 +1977,7 @@ static struct i915_power_well vlv_power_wells[] = {
 			   VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
 			   VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
 		.ops = &vlv_dpio_power_well_ops,
-		.data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
+		.id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
 	},
 	{
 		.name = "dpio-tx-c-01",
@@ -2012,7 +1986,7 @@ static struct i915_power_well vlv_power_wells[] = {
 			   VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
 			   VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
 		.ops = &vlv_dpio_power_well_ops,
-		.data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
+		.id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
 	},
 	{
 		.name = "dpio-tx-c-23",
@@ -2021,12 +1995,12 @@ static struct i915_power_well vlv_power_wells[] = {
 			   VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
 			   VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
 		.ops = &vlv_dpio_power_well_ops,
-		.data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
+		.id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
 	},
 	{
 		.name = "dpio-common",
 		.domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
-		.data = PUNIT_POWER_WELL_DPIO_CMN_BC,
+		.id = PUNIT_POWER_WELL_DPIO_CMN_BC,
 		.ops = &vlv_dpio_cmn_power_well_ops,
 	},
 };
@@ -2046,19 +2020,19 @@ static struct i915_power_well chv_power_wells[] = {
 		 * required for any pipe to work.
 		 */
 		.domains = CHV_DISPLAY_POWER_DOMAINS,
-		.data = PIPE_A,
+		.id = PIPE_A,
 		.ops = &chv_pipe_power_well_ops,
 	},
 	{
 		.name = "dpio-common-bc",
 		.domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
-		.data = PUNIT_POWER_WELL_DPIO_CMN_BC,
+		.id = PUNIT_POWER_WELL_DPIO_CMN_BC,
 		.ops = &chv_dpio_cmn_power_well_ops,
 	},
 	{
 		.name = "dpio-common-d",
 		.domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
-		.data = PUNIT_POWER_WELL_DPIO_CMN_D,
+		.id = PUNIT_POWER_WELL_DPIO_CMN_D,
 		.ops = &chv_dpio_cmn_power_well_ops,
 	},
 };
@@ -2081,57 +2055,57 @@ static struct i915_power_well skl_power_wells[] = {
 		.always_on = 1,
 		.domains = POWER_DOMAIN_MASK,
 		.ops = &i9xx_always_on_power_well_ops,
-		.data = SKL_DISP_PW_ALWAYS_ON,
+		.id = SKL_DISP_PW_ALWAYS_ON,
 	},
 	{
 		.name = "power well 1",
 		/* Handled by the DMC firmware */
 		.domains = 0,
 		.ops = &skl_power_well_ops,
-		.data = SKL_DISP_PW_1,
+		.id = SKL_DISP_PW_1,
 	},
 	{
 		.name = "MISC IO power well",
 		/* Handled by the DMC firmware */
 		.domains = 0,
 		.ops = &skl_power_well_ops,
-		.data = SKL_DISP_PW_MISC_IO,
+		.id = SKL_DISP_PW_MISC_IO,
 	},
 	{
 		.name = "DC off",
 		.domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
 		.ops = &gen9_dc_off_power_well_ops,
-		.data = SKL_DISP_PW_DC_OFF,
+		.id = SKL_DISP_PW_DC_OFF,
 	},
 	{
 		.name = "power well 2",
 		.domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
 		.ops = &skl_power_well_ops,
-		.data = SKL_DISP_PW_2,
+		.id = SKL_DISP_PW_2,
 	},
 	{
 		.name = "DDI A/E power well",
 		.domains = SKL_DISPLAY_DDI_A_E_POWER_DOMAINS,
 		.ops = &skl_power_well_ops,
-		.data = SKL_DISP_PW_DDI_A_E,
+		.id = SKL_DISP_PW_DDI_A_E,
 	},
 	{
 		.name = "DDI B power well",
 		.domains = SKL_DISPLAY_DDI_B_POWER_DOMAINS,
 		.ops = &skl_power_well_ops,
-		.data = SKL_DISP_PW_DDI_B,
+		.id = SKL_DISP_PW_DDI_B,
 	},
 	{
 		.name = "DDI C power well",
 		.domains = SKL_DISPLAY_DDI_C_POWER_DOMAINS,
 		.ops = &skl_power_well_ops,
-		.data = SKL_DISP_PW_DDI_C,
+		.id = SKL_DISP_PW_DDI_C,
 	},
 	{
 		.name = "DDI D power well",
 		.domains = SKL_DISPLAY_DDI_D_POWER_DOMAINS,
 		.ops = &skl_power_well_ops,
-		.data = SKL_DISP_PW_DDI_D,
+		.id = SKL_DISP_PW_DDI_D,
 	},
 };
 
@@ -2146,31 +2120,33 @@ static struct i915_power_well bxt_power_wells[] = {
 		.name = "power well 1",
 		.domains = 0,
 		.ops = &skl_power_well_ops,
-		.data = SKL_DISP_PW_1,
+		.id = SKL_DISP_PW_1,
 	},
 	{
 		.name = "DC off",
 		.domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
 		.ops = &gen9_dc_off_power_well_ops,
-		.data = SKL_DISP_PW_DC_OFF,
+		.id = SKL_DISP_PW_DC_OFF,
 	},
 	{
 		.name = "power well 2",
 		.domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
 		.ops = &skl_power_well_ops,
-		.data = SKL_DISP_PW_2,
+		.id = SKL_DISP_PW_2,
 	},
 	{
 		.name = "dpio-common-a",
 		.domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
 		.ops = &bxt_dpio_cmn_power_well_ops,
-		.data = BXT_DPIO_CMN_A,
+		.id = BXT_DPIO_CMN_A,
+		.data = DPIO_PHY1,
 	},
 	{
 		.name = "dpio-common-bc",
 		.domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
 		.ops = &bxt_dpio_cmn_power_well_ops,
-		.data = BXT_DPIO_CMN_BC,
+		.id = BXT_DPIO_CMN_BC,
+		.data = DPIO_PHY0,
 	},
 };
 
@@ -2592,20 +2568,19 @@ static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
  */
 void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	struct i915_power_domains *power_domains = &dev_priv->power_domains;
 
 	power_domains->initializing = true;
 
-	if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
 		skl_display_core_init(dev_priv, resume);
-	} else if (IS_BROXTON(dev)) {
+	} else if (IS_BROXTON(dev_priv)) {
 		bxt_display_core_init(dev_priv, resume);
-	} else if (IS_CHERRYVIEW(dev)) {
+	} else if (IS_CHERRYVIEW(dev_priv)) {
 		mutex_lock(&power_domains->lock);
 		chv_phy_control_init(dev_priv);
 		mutex_unlock(&power_domains->lock);
-	} else if (IS_VALLEYVIEW(dev)) {
+	} else if (IS_VALLEYVIEW(dev_priv)) {
 		mutex_lock(&power_domains->lock);
 		vlv_cmnlane_wa(dev_priv);
 		mutex_unlock(&power_domains->lock);
@@ -2740,8 +2715,7 @@ void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
 	struct device *kdev = &pdev->dev;
 
 	assert_rpm_wakelock_held(dev_priv);
-	if (atomic_dec_and_test(&dev_priv->pm.wakeref_count))
-		atomic_inc(&dev_priv->pm.atomic_seq);
+	atomic_dec(&dev_priv->pm.wakeref_count);
 
 	pm_runtime_mark_last_busy(kdev);
 	pm_runtime_put_autosuspend(kdev);
@@ -2760,7 +2734,6 @@ void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
 {
 	struct pci_dev *pdev = dev_priv->drm.pdev;
-	struct drm_device *dev = &dev_priv->drm;
 	struct device *kdev = &pdev->dev;
 
 	pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */
@@ -2772,7 +2745,7 @@ void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
 	 * so the driver's own RPM reference tracking asserts also work on
 	 * platforms without RPM support.
 	 */
-	if (!HAS_RUNTIME_PM(dev)) {
+	if (!HAS_RUNTIME_PM(dev_priv)) {
 		pm_runtime_dont_use_autosuspend(kdev);
 		pm_runtime_get_sync(kdev);
 	} else {
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index c551024d4871..27808e91cb5a 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -251,7 +251,7 @@ static void intel_sdvo_write_sdvox(struct intel_sdvo *intel_sdvo, u32 val)
 		 * HW workaround, need to write this twice for issue
 		 * that may result in first write getting masked.
 		 */
-		if (HAS_PCH_IBX(dev)) {
+		if (HAS_PCH_IBX(dev_priv)) {
 			I915_WRITE(intel_sdvo->sdvo_reg, val);
 			POSTING_READ(intel_sdvo->sdvo_reg);
 		}
@@ -307,7 +307,7 @@ static bool intel_sdvo_read_byte(struct intel_sdvo *intel_sdvo, u8 addr, u8 *ch)
 static const struct _sdvo_cmd_name {
 	u8 cmd;
 	const char *name;
-} sdvo_cmd_names[] = {
+} __attribute__ ((packed)) sdvo_cmd_names[] = {
 	SDVO_CMD_NAME_ENTRY(SDVO_CMD_RESET),
 	SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_DEVICE_CAPS),
 	SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_FIRMWARE_REV),
@@ -1133,7 +1133,7 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder,
 	DRM_DEBUG_KMS("forcing bpc to 8 for SDVO\n");
 	pipe_config->pipe_bpp = 8*3;
 
-	if (HAS_PCH_SPLIT(encoder->base.dev))
+	if (HAS_PCH_SPLIT(to_i915(encoder->base.dev)))
 		pipe_config->has_pch_encoder = true;
 
 	/* We need to construct preferred input timings based on our
@@ -1195,8 +1195,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 				  struct intel_crtc_state *crtc_state,
 				  struct drm_connector_state *conn_state)
 {
-	struct drm_device *dev = intel_encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
 	struct drm_display_mode *mode = &crtc_state->base.mode;
@@ -1269,13 +1268,13 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 		return;
 
 	/* Set the SDVO control regs. */
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_GEN(dev_priv) >= 4) {
 		/* The real mode polarity is set by the SDVO commands, using
 		 * struct intel_sdvo_dtd. */
 		sdvox = SDVO_VSYNC_ACTIVE_HIGH | SDVO_HSYNC_ACTIVE_HIGH;
-		if (!HAS_PCH_SPLIT(dev) && crtc_state->limited_color_range)
+		if (!HAS_PCH_SPLIT(dev_priv) && crtc_state->limited_color_range)
 			sdvox |= HDMI_COLOR_RANGE_16_235;
-		if (INTEL_INFO(dev)->gen < 5)
+		if (INTEL_GEN(dev_priv) < 5)
 			sdvox |= SDVO_BORDER_ENABLE;
 	} else {
 		sdvox = I915_READ(intel_sdvo->sdvo_reg);
@@ -1286,7 +1285,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 		sdvox |= (9 << 19) | SDVO_BORDER_ENABLE;
 	}
 
-	if (INTEL_PCH_TYPE(dev) >= PCH_CPT)
+	if (INTEL_PCH_TYPE(dev_priv) >= PCH_CPT)
 		sdvox |= SDVO_PIPE_SEL_CPT(crtc->pipe);
 	else
 		sdvox |= SDVO_PIPE_SEL(crtc->pipe);
@@ -1294,9 +1293,10 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 	if (intel_sdvo->has_hdmi_audio)
 		sdvox |= SDVO_AUDIO_ENABLE;
 
-	if (INTEL_INFO(dev)->gen >= 4) {
+	if (INTEL_GEN(dev_priv) >= 4) {
 		/* done in crtc_mode_set as the dpll_md reg must be written early */
-	} else if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
+	} else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
+		   IS_G33(dev_priv)) {
 		/* done in crtc_mode_set as it lives inside the dpll register */
 	} else {
 		sdvox |= (crtc_state->pixel_multiplier - 1)
@@ -1304,7 +1304,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 	}
 
 	if (input_dtd.part2.sdvo_flags & SDVO_NEED_TO_STALL &&
-	    INTEL_INFO(dev)->gen < 5)
+	    INTEL_GEN(dev_priv) < 5)
 		sdvox |= SDVO_STALL_SELECT;
 	intel_sdvo_write_sdvox(intel_sdvo, sdvox);
 }
@@ -1339,7 +1339,7 @@ static bool intel_sdvo_get_hw_state(struct intel_encoder *encoder,
 	if (!(tmp & SDVO_ENABLE) && (active_outputs == 0))
 		return false;
 
-	if (HAS_PCH_CPT(dev))
+	if (HAS_PCH_CPT(dev_priv))
 		*pipe = PORT_TO_PIPE_CPT(tmp);
 	else
 		*pipe = PORT_TO_PIPE(tmp);
@@ -1389,7 +1389,7 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder,
 	 * encoder->get_config we so already have a valid pixel multplier on all
 	 * other platfroms.
 	 */
-	if (IS_I915G(dev) || IS_I915GM(dev)) {
+	if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
 		pipe_config->pixel_multiplier =
 			((sdvox & SDVO_PORT_MULTIPLY_MASK)
 			 >> SDVO_PORT_MULTIPLY_SHIFT) + 1;
@@ -1471,7 +1471,7 @@ static void intel_disable_sdvo(struct intel_encoder *encoder,
 		temp &= ~SDVO_ENABLE;
 		intel_sdvo_write_sdvox(intel_sdvo, temp);
 
-		intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A);
+		intel_wait_for_vblank_if_active(dev_priv, PIPE_A);
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true);
 		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true);
 	}
@@ -1508,7 +1508,7 @@ static void intel_enable_sdvo(struct intel_encoder *encoder,
 	intel_sdvo_write_sdvox(intel_sdvo, temp);
 
 	for (i = 0; i < 2; i++)
-		intel_wait_for_vblank(dev, intel_crtc->pipe);
+		intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
 
 	success = intel_sdvo_get_trained_inputs(intel_sdvo, &input1, &input2);
 	/* Warn if the device reported failure to sync.
@@ -1595,15 +1595,15 @@ static bool intel_sdvo_get_capabilities(struct intel_sdvo *intel_sdvo, struct in
 
 static uint16_t intel_sdvo_get_hotplug_support(struct intel_sdvo *intel_sdvo)
 {
-	struct drm_device *dev = intel_sdvo->base.base.dev;
+	struct drm_i915_private *dev_priv = to_i915(intel_sdvo->base.base.dev);
 	uint16_t hotplug;
 
-	if (!I915_HAS_HOTPLUG(dev))
+	if (!I915_HAS_HOTPLUG(dev_priv))
 		return 0;
 
 	/* HW Erratum: SDVO Hotplug is broken on all i945G chips, there's noise
 	 * on the line. */
-	if (IS_I945G(dev) || IS_I945GM(dev))
+	if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
 		return 0;
 
 	if (!intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_HOT_PLUG_SUPPORT,
@@ -2410,10 +2410,10 @@ static void
 intel_sdvo_add_hdmi_properties(struct intel_sdvo *intel_sdvo,
 			       struct intel_sdvo_connector *connector)
 {
-	struct drm_device *dev = connector->base.base.dev;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.base.dev);
 
 	intel_attach_force_audio_property(&connector->base.base);
-	if (INTEL_INFO(dev)->gen >= 4 && IS_MOBILE(dev)) {
+	if (INTEL_GEN(dev_priv) >= 4 && IS_MOBILE(dev_priv)) {
 		intel_attach_broadcast_rgb_property(&connector->base.base);
 		intel_sdvo->color_range_auto = true;
 	}
@@ -2981,6 +2981,7 @@ bool intel_sdvo_init(struct drm_device *dev,
 	/* encoder type will be decided later */
 	intel_encoder = &intel_sdvo->base;
 	intel_encoder->type = INTEL_OUTPUT_SDVO;
+	intel_encoder->port = port;
 	drm_encoder_init(dev, &intel_encoder->base, &intel_sdvo_enc_funcs, 0,
 			 "SDVO %c", port_name(port));
 
@@ -2996,7 +2997,7 @@ bool intel_sdvo_init(struct drm_device *dev,
 	}
 
 	intel_encoder->compute_config = intel_sdvo_compute_config;
-	if (HAS_PCH_SPLIT(dev)) {
+	if (HAS_PCH_SPLIT(dev_priv)) {
 		intel_encoder->disable = pch_disable_sdvo;
 		intel_encoder->post_disable = pch_post_disable_sdvo;
 	} else {
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index dbed12c484c9..8f131a08d440 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -203,9 +203,6 @@ skl_update_plane(struct drm_plane *drm_plane,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_plane *intel_plane = to_intel_plane(drm_plane);
 	struct drm_framebuffer *fb = plane_state->base.fb;
-	const struct skl_wm_values *wm = &dev_priv->wm.skl_results;
-	struct drm_crtc *crtc = crtc_state->base.crtc;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	const int pipe = intel_plane->pipe;
 	const int plane = intel_plane->plane + 1;
 	u32 plane_ctl;
@@ -227,13 +224,10 @@ skl_update_plane(struct drm_plane *drm_plane,
 		PLANE_CTL_PIPE_CSC_ENABLE;
 
 	plane_ctl |= skl_plane_ctl_format(fb->pixel_format);
-	plane_ctl |= skl_plane_ctl_tiling(fb->modifier[0]);
+	plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
 
 	plane_ctl |= skl_plane_ctl_rotation(rotation);
 
-	if (wm->dirty_pipes & drm_crtc_mask(crtc))
-		skl_write_plane_wm(intel_crtc, wm, plane);
-
 	if (key->flags) {
 		I915_WRITE(PLANE_KEYVAL(pipe, plane), key->min_value);
 		I915_WRITE(PLANE_KEYMAX(pipe, plane), key->max_value);
@@ -292,14 +286,6 @@ skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc)
 	const int pipe = intel_plane->pipe;
 	const int plane = intel_plane->plane + 1;
 
-	/*
-	 * We only populate skl_results on watermark updates, and if the
-	 * plane's visiblity isn't actually changing neither is its watermarks.
-	 */
-	if (!dplane->state->visible)
-		skl_write_plane_wm(to_intel_crtc(crtc),
-				   &dev_priv->wm.skl_results, plane);
-
 	I915_WRITE(PLANE_CTL(pipe, plane), 0);
 
 	I915_WRITE(PLANE_SURF(pipe, plane), 0);
@@ -420,9 +406,15 @@ vlv_update_plane(struct drm_plane *dplane,
 	 */
 	sprctl |= SP_GAMMA_ENABLE;
 
-	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
+	if (fb->modifier == I915_FORMAT_MOD_X_TILED)
 		sprctl |= SP_TILED;
 
+	if (rotation & DRM_ROTATE_180)
+		sprctl |= SP_ROTATE_180;
+
+	if (rotation & DRM_REFLECT_X)
+		sprctl |= SP_MIRROR;
+
 	/* Sizes are 0 based */
 	src_w--;
 	src_h--;
@@ -432,11 +424,11 @@ vlv_update_plane(struct drm_plane *dplane,
 	intel_add_fb_offsets(&x, &y, plane_state, 0);
 	sprsurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0);
 
-	if (rotation == DRM_ROTATE_180) {
-		sprctl |= SP_ROTATE_180;
-
+	if (rotation & DRM_ROTATE_180) {
 		x += src_w;
 		y += src_h;
+	} else if (rotation & DRM_REFLECT_X) {
+		x += src_w;
 	}
 
 	linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
@@ -450,13 +442,13 @@ vlv_update_plane(struct drm_plane *dplane,
 	if (key->flags & I915_SET_COLORKEY_SOURCE)
 		sprctl |= SP_SOURCE_KEY;
 
-	if (IS_CHERRYVIEW(dev) && pipe == PIPE_B)
+	if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B)
 		chv_update_csc(intel_plane, fb->pixel_format);
 
 	I915_WRITE(SPSTRIDE(pipe, plane), fb->pitches[0]);
 	I915_WRITE(SPPOS(pipe, plane), (crtc_y << 16) | crtc_x);
 
-	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
+	if (fb->modifier == I915_FORMAT_MOD_X_TILED)
 		I915_WRITE(SPTILEOFF(pipe, plane), (y << 16) | x);
 	else
 		I915_WRITE(SPLINOFF(pipe, plane), linear_offset);
@@ -539,15 +531,18 @@ ivb_update_plane(struct drm_plane *plane,
 	 */
 	sprctl |= SPRITE_GAMMA_ENABLE;
 
-	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
+	if (fb->modifier == I915_FORMAT_MOD_X_TILED)
 		sprctl |= SPRITE_TILED;
 
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+	if (rotation & DRM_ROTATE_180)
+		sprctl |= SPRITE_ROTATE_180;
+
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		sprctl &= ~SPRITE_TRICKLE_FEED_DISABLE;
 	else
 		sprctl |= SPRITE_TRICKLE_FEED_DISABLE;
 
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		sprctl |= SPRITE_PIPE_CSC_ENABLE;
 
 	/* Sizes are 0 based */
@@ -562,14 +557,11 @@ ivb_update_plane(struct drm_plane *plane,
 	intel_add_fb_offsets(&x, &y, plane_state, 0);
 	sprsurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0);
 
-	if (rotation == DRM_ROTATE_180) {
-		sprctl |= SPRITE_ROTATE_180;
-
-		/* HSW and BDW does this automagically in hardware */
-		if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
-			x += src_w;
-			y += src_h;
-		}
+	/* HSW+ does this automagically in hardware */
+	if (!IS_HASWELL(dev_priv) && !IS_BROADWELL(dev_priv) &&
+	    rotation & DRM_ROTATE_180) {
+		x += src_w;
+		y += src_h;
 	}
 
 	linear_offset = intel_fb_xy_to_linear(x, y, plane_state, 0);
@@ -590,9 +582,9 @@ ivb_update_plane(struct drm_plane *plane,
 
 	/* HSW consolidates SPRTILEOFF and SPRLINOFF into a single SPROFFSET
 	 * register */
-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		I915_WRITE(SPROFFSET(pipe), (y << 16) | x);
-	else if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
+	else if (fb->modifier == I915_FORMAT_MOD_X_TILED)
 		I915_WRITE(SPRTILEOFF(pipe), (y << 16) | x);
 	else
 		I915_WRITE(SPRLINOFF(pipe), linear_offset);
@@ -677,10 +669,13 @@ ilk_update_plane(struct drm_plane *plane,
 	 */
 	dvscntr |= DVS_GAMMA_ENABLE;
 
-	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
+	if (fb->modifier == I915_FORMAT_MOD_X_TILED)
 		dvscntr |= DVS_TILED;
 
-	if (IS_GEN6(dev))
+	if (rotation & DRM_ROTATE_180)
+		dvscntr |= DVS_ROTATE_180;
+
+	if (IS_GEN6(dev_priv))
 		dvscntr |= DVS_TRICKLE_FEED_DISABLE; /* must disable */
 
 	/* Sizes are 0 based */
@@ -696,9 +691,7 @@ ilk_update_plane(struct drm_plane *plane,
 	intel_add_fb_offsets(&x, &y, plane_state, 0);
 	dvssurf_offset = intel_compute_tile_offset(&x, &y, plane_state, 0);
 
-	if (rotation == DRM_ROTATE_180) {
-		dvscntr |= DVS_ROTATE_180;
-
+	if (rotation & DRM_ROTATE_180) {
 		x += src_w;
 		y += src_h;
 	}
@@ -719,7 +712,7 @@ ilk_update_plane(struct drm_plane *plane,
 	I915_WRITE(DVSSTRIDE(pipe), fb->pitches[0]);
 	I915_WRITE(DVSPOS(pipe), (crtc_y << 16) | crtc_x);
 
-	if (fb->modifier[0] == I915_FORMAT_MOD_X_TILED)
+	if (fb->modifier == I915_FORMAT_MOD_X_TILED)
 		I915_WRITE(DVSTILEOFF(pipe), (y << 16) | x);
 	else
 		I915_WRITE(DVSLINOFF(pipe), linear_offset);
@@ -753,7 +746,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
 			 struct intel_crtc_state *crtc_state,
 			 struct intel_plane_state *state)
 {
-	struct drm_device *dev = plane->dev;
+	struct drm_i915_private *dev_priv = to_i915(plane->dev);
 	struct drm_crtc *crtc = state->base.crtc;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct intel_plane *intel_plane = to_intel_plane(plane);
@@ -769,15 +762,8 @@ intel_check_sprite_plane(struct drm_plane *plane,
 	bool can_scale;
 	int ret;
 
-	src->x1 = state->base.src_x;
-	src->y1 = state->base.src_y;
-	src->x2 = state->base.src_x + state->base.src_w;
-	src->y2 = state->base.src_y + state->base.src_h;
-
-	dst->x1 = state->base.crtc_x;
-	dst->y1 = state->base.crtc_y;
-	dst->x2 = state->base.crtc_x + state->base.crtc_w;
-	dst->y2 = state->base.crtc_y + state->base.crtc_h;
+	*src = drm_plane_state_src(&state->base);
+	*dst = drm_plane_state_dest(&state->base);
 
 	if (!fb) {
 		state->base.visible = false;
@@ -797,7 +783,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
 	}
 
 	/* setup can_scale, min_scale, max_scale */
-	if (INTEL_INFO(dev)->gen >= 9) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		/* use scaler when colorkey is not required */
 		if (state->ckey.flags == I915_SET_COLORKEY_NONE) {
 			can_scale = 1;
@@ -913,7 +899,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
 
 		width_bytes = ((src_x * cpp) & 63) + src_w * cpp;
 
-		if (INTEL_INFO(dev)->gen < 9 && (src_w > 2048 || src_h > 2048 ||
+		if (INTEL_GEN(dev_priv) < 9 && (src_w > 2048 || src_h > 2048 ||
 		    width_bytes > 4096 || fb->pitches[0] > 4096)) {
 			DRM_DEBUG_KMS("Source dimensions exceed hardware limits\n");
 			return -EINVAL;
@@ -932,7 +918,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
 	dst->y1 = crtc_y;
 	dst->y2 = crtc_y + crtc_h;
 
-	if (INTEL_GEN(dev) >= 9) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		ret = skl_check_plane_surface(state);
 		if (ret)
 			return ret;
@@ -944,6 +930,7 @@ intel_check_sprite_plane(struct drm_plane *plane,
 int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv)
 {
+	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_intel_sprite_colorkey *set = data;
 	struct drm_plane *plane;
 	struct drm_plane_state *plane_state;
@@ -955,7 +942,7 @@ int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
 	if ((set->flags & (I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE)) == (I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE))
 		return -EINVAL;
 
-	if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 	    set->flags & I915_SET_COLORKEY_DESTINATION)
 		return -EINVAL;
 
@@ -987,9 +974,7 @@ int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
 		drm_modeset_backoff(&ctx);
 	}
 
-	if (ret)
-		drm_atomic_state_free(state);
-
+	drm_atomic_state_put(state);
 out:
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
@@ -1039,19 +1024,18 @@ static uint32_t skl_plane_formats[] = {
 	DRM_FORMAT_VYUY,
 };
 
-int
-intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
+struct intel_plane *
+intel_sprite_plane_create(struct drm_i915_private *dev_priv,
+			  enum pipe pipe, int plane)
 {
 	struct intel_plane *intel_plane = NULL;
 	struct intel_plane_state *state = NULL;
 	unsigned long possible_crtcs;
 	const uint32_t *plane_formats;
+	unsigned int supported_rotations;
 	int num_plane_formats;
 	int ret;
 
-	if (INTEL_INFO(dev)->gen < 5)
-		return -ENODEV;
-
 	intel_plane = kzalloc(sizeof(*intel_plane), GFP_KERNEL);
 	if (!intel_plane) {
 		ret = -ENOMEM;
@@ -1065,26 +1049,26 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
 	}
 	intel_plane->base.state = &state->base;
 
-	switch (INTEL_INFO(dev)->gen) {
-	case 5:
-	case 6:
+	if (INTEL_GEN(dev_priv) >= 9) {
 		intel_plane->can_scale = true;
-		intel_plane->max_downscale = 16;
-		intel_plane->update_plane = ilk_update_plane;
-		intel_plane->disable_plane = ilk_disable_plane;
+		state->scaler_id = -1;
 
-		if (IS_GEN6(dev)) {
-			plane_formats = snb_plane_formats;
-			num_plane_formats = ARRAY_SIZE(snb_plane_formats);
-		} else {
-			plane_formats = ilk_plane_formats;
-			num_plane_formats = ARRAY_SIZE(ilk_plane_formats);
-		}
-		break;
+		intel_plane->update_plane = skl_update_plane;
+		intel_plane->disable_plane = skl_disable_plane;
+
+		plane_formats = skl_plane_formats;
+		num_plane_formats = ARRAY_SIZE(skl_plane_formats);
+	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
+		intel_plane->can_scale = false;
+		intel_plane->max_downscale = 1;
 
-	case 7:
-	case 8:
-		if (IS_IVYBRIDGE(dev)) {
+		intel_plane->update_plane = vlv_update_plane;
+		intel_plane->disable_plane = vlv_disable_plane;
+
+		plane_formats = vlv_plane_formats;
+		num_plane_formats = ARRAY_SIZE(vlv_plane_formats);
+	} else if (INTEL_GEN(dev_priv) >= 7) {
+		if (IS_IVYBRIDGE(dev_priv)) {
 			intel_plane->can_scale = true;
 			intel_plane->max_downscale = 2;
 		} else {
@@ -1092,33 +1076,38 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
 			intel_plane->max_downscale = 1;
 		}
 
-		if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
-			intel_plane->update_plane = vlv_update_plane;
-			intel_plane->disable_plane = vlv_disable_plane;
+		intel_plane->update_plane = ivb_update_plane;
+		intel_plane->disable_plane = ivb_disable_plane;
 
-			plane_formats = vlv_plane_formats;
-			num_plane_formats = ARRAY_SIZE(vlv_plane_formats);
-		} else {
-			intel_plane->update_plane = ivb_update_plane;
-			intel_plane->disable_plane = ivb_disable_plane;
+		plane_formats = snb_plane_formats;
+		num_plane_formats = ARRAY_SIZE(snb_plane_formats);
+	} else {
+		intel_plane->can_scale = true;
+		intel_plane->max_downscale = 16;
+
+		intel_plane->update_plane = ilk_update_plane;
+		intel_plane->disable_plane = ilk_disable_plane;
 
+		if (IS_GEN6(dev_priv)) {
 			plane_formats = snb_plane_formats;
 			num_plane_formats = ARRAY_SIZE(snb_plane_formats);
+		} else {
+			plane_formats = ilk_plane_formats;
+			num_plane_formats = ARRAY_SIZE(ilk_plane_formats);
 		}
-		break;
-	case 9:
-		intel_plane->can_scale = true;
-		intel_plane->update_plane = skl_update_plane;
-		intel_plane->disable_plane = skl_disable_plane;
-		state->scaler_id = -1;
+	}
 
-		plane_formats = skl_plane_formats;
-		num_plane_formats = ARRAY_SIZE(skl_plane_formats);
-		break;
-	default:
-		MISSING_CASE(INTEL_INFO(dev)->gen);
-		ret = -ENODEV;
-		goto fail;
+	if (INTEL_GEN(dev_priv) >= 9) {
+		supported_rotations =
+			DRM_ROTATE_0 | DRM_ROTATE_90 |
+			DRM_ROTATE_180 | DRM_ROTATE_270;
+	} else if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
+		supported_rotations =
+			DRM_ROTATE_0 | DRM_ROTATE_180 |
+			DRM_REFLECT_X;
+	} else {
+		supported_rotations =
+			DRM_ROTATE_0 | DRM_ROTATE_180;
 	}
 
 	intel_plane->pipe = pipe;
@@ -1128,30 +1117,32 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
 
 	possible_crtcs = (1 << pipe);
 
-	if (INTEL_INFO(dev)->gen >= 9)
-		ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs,
-					       &intel_plane_funcs,
+	if (INTEL_GEN(dev_priv) >= 9)
+		ret = drm_universal_plane_init(&dev_priv->drm, &intel_plane->base,
+					       possible_crtcs, &intel_plane_funcs,
 					       plane_formats, num_plane_formats,
 					       DRM_PLANE_TYPE_OVERLAY,
 					       "plane %d%c", plane + 2, pipe_name(pipe));
 	else
-		ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs,
-					       &intel_plane_funcs,
+		ret = drm_universal_plane_init(&dev_priv->drm, &intel_plane->base,
+					       possible_crtcs, &intel_plane_funcs,
 					       plane_formats, num_plane_formats,
 					       DRM_PLANE_TYPE_OVERLAY,
 					       "sprite %c", sprite_name(pipe, plane));
 	if (ret)
 		goto fail;
 
-	intel_create_rotation_property(dev, intel_plane);
+	drm_plane_create_rotation_property(&intel_plane->base,
+					   DRM_ROTATE_0,
+					   supported_rotations);
 
 	drm_plane_helper_add(&intel_plane->base, &intel_plane_helper_funcs);
 
-	return 0;
+	return intel_plane;
 
 fail:
 	kfree(state);
 	kfree(intel_plane);
 
-	return ret;
+	return ERR_PTR(ret);
 }
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index d960e4866595..78cdfc6833d6 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -86,7 +86,8 @@ struct intel_tv {
 };
 
 struct video_levels {
-	int blank, black, burst;
+	u16 blank, black;
+	u8 burst;
 };
 
 struct color_conversion {
@@ -339,34 +340,43 @@ static const struct video_levels component_levels = {
 
 struct tv_mode {
 	const char *name;
-	int clock;
-	int refresh; /* in millihertz (for precision) */
+
+	u32 clock;
+	u16 refresh; /* in millihertz (for precision) */
 	u32 oversample;
-	int hsync_end, hblank_start, hblank_end, htotal;
-	bool progressive, trilevel_sync, component_only;
-	int vsync_start_f1, vsync_start_f2, vsync_len;
-	bool veq_ena;
-	int veq_start_f1, veq_start_f2, veq_len;
-	int vi_end_f1, vi_end_f2, nbr_end;
-	bool burst_ena;
-	int hburst_start, hburst_len;
-	int vburst_start_f1, vburst_end_f1;
-	int vburst_start_f2, vburst_end_f2;
-	int vburst_start_f3, vburst_end_f3;
-	int vburst_start_f4, vburst_end_f4;
+	u8 hsync_end;
+	u16 hblank_start, hblank_end, htotal;
+	bool progressive : 1, trilevel_sync : 1, component_only : 1;
+	u8 vsync_start_f1, vsync_start_f2, vsync_len;
+	bool veq_ena : 1;
+	u8 veq_start_f1, veq_start_f2, veq_len;
+	u8 vi_end_f1, vi_end_f2;
+	u16 nbr_end;
+	bool burst_ena : 1;
+	u8 hburst_start, hburst_len;
+	u8 vburst_start_f1;
+	u16 vburst_end_f1;
+	u8 vburst_start_f2;
+	u16 vburst_end_f2;
+	u8 vburst_start_f3;
+	u16 vburst_end_f3;
+	u8 vburst_start_f4;
+	u16 vburst_end_f4;
 	/*
 	 * subcarrier programming
 	 */
-	int dda2_size, dda3_size, dda1_inc, dda2_inc, dda3_inc;
+	u16 dda2_size, dda3_size;
+	u8 dda1_inc;
+	u16 dda2_inc, dda3_inc;
 	u32 sc_reset;
-	bool pal_burst;
+	bool pal_burst : 1;
 	/*
 	 * blank/black levels
 	 */
 	const struct video_levels *composite_levels, *svideo_levels;
 	const struct color_conversion *composite_color, *svideo_color;
 	const u32 *filter_table;
-	int max_srcw;
+	u16 max_srcw;
 };
 
 
@@ -846,7 +856,7 @@ intel_enable_tv(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
 	/* Prevents vblank waits from timing out in intel_tv_detect_type() */
-	intel_wait_for_vblank(encoder->base.dev,
+	intel_wait_for_vblank(dev_priv,
 			      to_intel_crtc(encoder->base.crtc)->pipe);
 
 	I915_WRITE(TV_CTL, I915_READ(TV_CTL) | TV_ENC_ENABLE);
@@ -1019,8 +1029,7 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder,
 				struct intel_crtc_state *pipe_config,
 				struct drm_connector_state *conn_state)
 {
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
 	struct intel_tv *intel_tv = enc_to_tv(encoder);
 	const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
@@ -1095,7 +1104,7 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder,
 		tv_mode->dda3_inc << TV_SCDDA3_INC_SHIFT;
 
 	/* Enable two fixes for the chips that need them. */
-	if (IS_I915GM(dev))
+	if (IS_I915GM(dev_priv))
 		tv_ctl |= TV_ENC_C0_FIX | TV_ENC_SDP_FIX;
 
 	set_tv_mode_timings(dev_priv, tv_mode, burst_ena);
@@ -1106,7 +1115,7 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder,
 
 	set_color_conversion(dev_priv, color_conversion);
 
-	if (INTEL_INFO(dev)->gen >= 4)
+	if (INTEL_GEN(dev_priv) >= 4)
 		I915_WRITE(TV_CLR_KNOBS, 0x00404000);
 	else
 		I915_WRITE(TV_CLR_KNOBS, 0x00606000);
@@ -1220,7 +1229,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv,
 	 * The TV sense state should be cleared to zero on cantiga platform. Otherwise
 	 * the TV is misdetected. This is hardware requirement.
 	 */
-	if (IS_GM45(dev))
+	if (IS_GM45(dev_priv))
 		tv_dac &= ~(TVDAC_STATE_CHG_EN | TVDAC_A_SENSE_CTL |
 			    TVDAC_B_SENSE_CTL | TVDAC_C_SENSE_CTL);
 
@@ -1228,7 +1237,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv,
 	I915_WRITE(TV_DAC, tv_dac);
 	POSTING_READ(TV_DAC);
 
-	intel_wait_for_vblank(dev, intel_crtc->pipe);
+	intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
 
 	type = -1;
 	tv_dac = I915_READ(TV_DAC);
@@ -1258,7 +1267,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv,
 	POSTING_READ(TV_CTL);
 
 	/* For unknown reasons the hw barfs if we don't do this vblank wait. */
-	intel_wait_for_vblank(dev, intel_crtc->pipe);
+	intel_wait_for_vblank(dev_priv, intel_crtc->pipe);
 
 	/* Restore interrupt config */
 	if (connector->polled & DRM_CONNECTOR_POLL_HPD) {
@@ -1610,7 +1619,9 @@ intel_tv_init(struct drm_device *dev)
 	intel_connector->get_hw_state = intel_connector_get_hw_state;
 
 	intel_connector_attach_encoder(intel_connector, intel_encoder);
+
 	intel_encoder->type = INTEL_OUTPUT_TVOUT;
+	intel_encoder->port = PORT_NONE;
 	intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
 	intel_encoder->cloneable = 0;
 	intel_encoder->base.possible_crtcs = ((1 << 0) | (1 << 1));
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index ee2306a79747..d7be0d94ba4d 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -231,19 +231,21 @@ intel_uncore_fw_release_timer(struct hrtimer *timer)
 {
 	struct intel_uncore_forcewake_domain *domain =
 	       container_of(timer, struct intel_uncore_forcewake_domain, timer);
+	struct drm_i915_private *dev_priv = domain->i915;
 	unsigned long irqflags;
 
-	assert_rpm_device_not_suspended(domain->i915);
+	assert_rpm_device_not_suspended(dev_priv);
 
-	spin_lock_irqsave(&domain->i915->uncore.lock, irqflags);
+	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 	if (WARN_ON(domain->wake_count == 0))
 		domain->wake_count++;
 
-	if (--domain->wake_count == 0)
-		domain->i915->uncore.funcs.force_wake_put(domain->i915,
-							  1 << domain->id);
+	if (--domain->wake_count == 0) {
+		dev_priv->uncore.funcs.force_wake_put(dev_priv, domain->mask);
+		dev_priv->uncore.fw_domains_active &= ~domain->mask;
+	}
 
-	spin_unlock_irqrestore(&domain->i915->uncore.lock, irqflags);
+	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 
 	return HRTIMER_NORESTART;
 }
@@ -254,7 +256,7 @@ void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
 	unsigned long irqflags;
 	struct intel_uncore_forcewake_domain *domain;
 	int retry_count = 100;
-	enum forcewake_domains fw = 0, active_domains;
+	enum forcewake_domains fw, active_domains;
 
 	/* Hold uncore.lock across reset to prevent any register access
 	 * with forcewake not set correctly. Wait until all pending
@@ -291,10 +293,7 @@ void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
 
 	WARN_ON(active_domains);
 
-	for_each_fw_domain(domain, dev_priv)
-		if (domain->wake_count)
-			fw |= domain->mask;
-
+	fw = dev_priv->uncore.fw_domains_active;
 	if (fw)
 		dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
 
@@ -403,6 +402,8 @@ check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
 					  bool restore_forcewake)
 {
+	struct intel_device_info *info = mkwrite_device_info(dev_priv);
+
 	/* clear out unclaimed reg detection bit */
 	if (check_for_unclaimed_mmio(dev_priv))
 		DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
@@ -420,6 +421,10 @@ static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
 				   GT_FIFO_CTL_RC6_POLICY_STALL);
 	}
 
+	/* Enable Decoupled MMIO only on BXT C stepping onwards */
+	if (!IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
+		info->has_decoupled_mmio = false;
+
 	intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
 }
 
@@ -443,9 +448,6 @@ static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
 {
 	struct intel_uncore_forcewake_domain *domain;
 
-	if (!dev_priv->uncore.funcs.force_wake_get)
-		return;
-
 	fw_domains &= dev_priv->uncore.fw_domains;
 
 	for_each_fw_domain_masked(domain, fw_domains, dev_priv) {
@@ -453,8 +455,10 @@ static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
 			fw_domains &= ~domain->mask;
 	}
 
-	if (fw_domains)
+	if (fw_domains) {
 		dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
+		dev_priv->uncore.fw_domains_active |= fw_domains;
+	}
 }
 
 /**
@@ -509,9 +513,6 @@ static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
 {
 	struct intel_uncore_forcewake_domain *domain;
 
-	if (!dev_priv->uncore.funcs.force_wake_put)
-		return;
-
 	fw_domains &= dev_priv->uncore.fw_domains;
 
 	for_each_fw_domain_masked(domain, fw_domains, dev_priv) {
@@ -567,13 +568,10 @@ void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
 
 void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
 {
-	struct intel_uncore_forcewake_domain *domain;
-
 	if (!dev_priv->uncore.funcs.force_wake_get)
 		return;
 
-	for_each_fw_domain(domain, dev_priv)
-		WARN_ON(domain->wake_count);
+	WARN_ON(dev_priv->uncore.fw_domains_active);
 }
 
 /* We give fast paths for the really cool registers */
@@ -589,49 +587,148 @@ void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
 	__fwd; \
 })
 
-#define REG_RANGE(reg, start, end) ((reg) >= (start) && (reg) < (end))
+static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
+{
+	if (offset < entry->start)
+		return -1;
+	else if (offset > entry->end)
+		return 1;
+	else
+		return 0;
+}
 
-#define FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg) \
-	(REG_RANGE((reg), 0x2000, 0x4000) || \
-	 REG_RANGE((reg), 0x5000, 0x8000) || \
-	 REG_RANGE((reg), 0xB000, 0x12000) || \
-	 REG_RANGE((reg), 0x2E000, 0x30000))
+/* Copied and "macroized" from lib/bsearch.c */
+#define BSEARCH(key, base, num, cmp) ({                                 \
+	unsigned int start__ = 0, end__ = (num);                        \
+	typeof(base) result__ = NULL;                                   \
+	while (start__ < end__) {                                       \
+		unsigned int mid__ = start__ + (end__ - start__) / 2;   \
+		int ret__ = (cmp)((key), (base) + mid__);               \
+		if (ret__ < 0) {                                        \
+			end__ = mid__;                                  \
+		} else if (ret__ > 0) {                                 \
+			start__ = mid__ + 1;                            \
+		} else {                                                \
+			result__ = (base) + mid__;                      \
+			break;                                          \
+		}                                                       \
+	}                                                               \
+	result__;                                                       \
+})
+
+static enum forcewake_domains
+find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
+{
+	const struct intel_forcewake_range *entry;
+
+	entry = BSEARCH(offset,
+			dev_priv->uncore.fw_domains_table,
+			dev_priv->uncore.fw_domains_table_entries,
+			fw_range_cmp);
 
-#define FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg) \
-	(REG_RANGE((reg), 0x12000, 0x14000) || \
-	 REG_RANGE((reg), 0x22000, 0x24000) || \
-	 REG_RANGE((reg), 0x30000, 0x40000))
+	return entry ? entry->domains : 0;
+}
 
-#define __vlv_reg_read_fw_domains(offset) \
+static void
+intel_fw_table_check(struct drm_i915_private *dev_priv)
+{
+	const struct intel_forcewake_range *ranges;
+	unsigned int num_ranges;
+	s32 prev;
+	unsigned int i;
+
+	if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG))
+		return;
+
+	ranges = dev_priv->uncore.fw_domains_table;
+	if (!ranges)
+		return;
+
+	num_ranges = dev_priv->uncore.fw_domains_table_entries;
+
+	for (i = 0, prev = -1; i < num_ranges; i++, ranges++) {
+		WARN_ON_ONCE(IS_GEN9(dev_priv) &&
+			     (prev + 1) != (s32)ranges->start);
+		WARN_ON_ONCE(prev >= (s32)ranges->start);
+		prev = ranges->start;
+		WARN_ON_ONCE(prev >= (s32)ranges->end);
+		prev = ranges->end;
+	}
+}
+
+#define GEN_FW_RANGE(s, e, d) \
+	{ .start = (s), .end = (e), .domains = (d) }
+
+#define HAS_FWTABLE(dev_priv) \
+	(IS_GEN9(dev_priv) || \
+	 IS_CHERRYVIEW(dev_priv) || \
+	 IS_VALLEYVIEW(dev_priv))
+
+/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
+static const struct intel_forcewake_range __vlv_fw_ranges[] = {
+	GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
+};
+
+#define __fwtable_reg_read_fw_domains(offset) \
 ({ \
 	enum forcewake_domains __fwd = 0; \
-	if (!NEEDS_FORCE_WAKE(offset)) \
-		__fwd = 0; \
-	else if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_RENDER; \
-	else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_MEDIA; \
+	if (NEEDS_FORCE_WAKE((offset))) \
+		__fwd = find_fw_domain(dev_priv, offset); \
 	__fwd; \
 })
 
+/* *Must* be sorted by offset! See intel_shadow_table_check(). */
 static const i915_reg_t gen8_shadowed_regs[] = {
-	GEN6_RPNSWREQ,
-	GEN6_RC_VIDEO_FREQ,
-	RING_TAIL(RENDER_RING_BASE),
-	RING_TAIL(GEN6_BSD_RING_BASE),
-	RING_TAIL(VEBOX_RING_BASE),
-	RING_TAIL(BLT_RING_BASE),
+	RING_TAIL(RENDER_RING_BASE),	/* 0x2000 (base) */
+	GEN6_RPNSWREQ,			/* 0xA008 */
+	GEN6_RC_VIDEO_FREQ,		/* 0xA00C */
+	RING_TAIL(GEN6_BSD_RING_BASE),	/* 0x12000 (base) */
+	RING_TAIL(VEBOX_RING_BASE),	/* 0x1a000 (base) */
+	RING_TAIL(BLT_RING_BASE),	/* 0x22000 (base) */
 	/* TODO: Other registers are not yet used */
 };
 
+static void intel_shadow_table_check(void)
+{
+	const i915_reg_t *reg = gen8_shadowed_regs;
+	s32 prev;
+	u32 offset;
+	unsigned int i;
+
+	if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG))
+		return;
+
+	for (i = 0, prev = -1; i < ARRAY_SIZE(gen8_shadowed_regs); i++, reg++) {
+		offset = i915_mmio_reg_offset(*reg);
+		WARN_ON_ONCE(prev >= (s32)offset);
+		prev = offset;
+	}
+}
+
+static int mmio_reg_cmp(u32 key, const i915_reg_t *reg)
+{
+	u32 offset = i915_mmio_reg_offset(*reg);
+
+	if (key < offset)
+		return -1;
+	else if (key > offset)
+		return 1;
+	else
+		return 0;
+}
+
 static bool is_gen8_shadowed(u32 offset)
 {
-	int i;
-	for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++)
-		if (offset == gen8_shadowed_regs[i].reg)
-			return true;
+	const i915_reg_t *regs = gen8_shadowed_regs;
 
-	return false;
+	return BSEARCH(offset, regs, ARRAY_SIZE(gen8_shadowed_regs),
+		       mmio_reg_cmp);
 }
 
 #define __gen8_reg_write_fw_domains(offset) \
@@ -644,143 +741,70 @@ static bool is_gen8_shadowed(u32 offset)
 	__fwd; \
 })
 
-#define FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg) \
-	(REG_RANGE((reg), 0x2000, 0x4000) || \
-	 REG_RANGE((reg), 0x5200, 0x8000) || \
-	 REG_RANGE((reg), 0x8300, 0x8500) || \
-	 REG_RANGE((reg), 0xB000, 0xB480) || \
-	 REG_RANGE((reg), 0xE000, 0xE800))
-
-#define FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg) \
-	(REG_RANGE((reg), 0x8800, 0x8900) || \
-	 REG_RANGE((reg), 0xD000, 0xD800) || \
-	 REG_RANGE((reg), 0x12000, 0x14000) || \
-	 REG_RANGE((reg), 0x1A000, 0x1C000) || \
-	 REG_RANGE((reg), 0x1E800, 0x1EA00) || \
-	 REG_RANGE((reg), 0x30000, 0x38000))
-
-#define FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg) \
-	(REG_RANGE((reg), 0x4000, 0x5000) || \
-	 REG_RANGE((reg), 0x8000, 0x8300) || \
-	 REG_RANGE((reg), 0x8500, 0x8600) || \
-	 REG_RANGE((reg), 0x9000, 0xB000) || \
-	 REG_RANGE((reg), 0xF000, 0x10000))
-
-#define __chv_reg_read_fw_domains(offset) \
-({ \
-	enum forcewake_domains __fwd = 0; \
-	if (!NEEDS_FORCE_WAKE(offset)) \
-		__fwd = 0; \
-	else if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_RENDER; \
-	else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_MEDIA; \
-	else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
-	__fwd; \
-})
+/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
+static const struct intel_forcewake_range __chv_fw_ranges[] = {
+	GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
+};
 
-#define __chv_reg_write_fw_domains(offset) \
+#define __fwtable_reg_write_fw_domains(offset) \
 ({ \
 	enum forcewake_domains __fwd = 0; \
-	if (!NEEDS_FORCE_WAKE(offset) || is_gen8_shadowed(offset)) \
-		__fwd = 0; \
-	else if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_RENDER; \
-	else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_MEDIA; \
-	else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
+	if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \
+		__fwd = find_fw_domain(dev_priv, offset); \
 	__fwd; \
 })
 
-#define FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg) \
-	REG_RANGE((reg), 0xB00,  0x2000)
-
-#define FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg) \
-	(REG_RANGE((reg), 0x2000, 0x2700) || \
-	 REG_RANGE((reg), 0x3000, 0x4000) || \
-	 REG_RANGE((reg), 0x5200, 0x8000) || \
-	 REG_RANGE((reg), 0x8140, 0x8160) || \
-	 REG_RANGE((reg), 0x8300, 0x8500) || \
-	 REG_RANGE((reg), 0x8C00, 0x8D00) || \
-	 REG_RANGE((reg), 0xB000, 0xB480) || \
-	 REG_RANGE((reg), 0xE000, 0xE900) || \
-	 REG_RANGE((reg), 0x24400, 0x24800))
-
-#define FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg) \
-	(REG_RANGE((reg), 0x8130, 0x8140) || \
-	 REG_RANGE((reg), 0x8800, 0x8A00) || \
-	 REG_RANGE((reg), 0xD000, 0xD800) || \
-	 REG_RANGE((reg), 0x12000, 0x14000) || \
-	 REG_RANGE((reg), 0x1A000, 0x1EA00) || \
-	 REG_RANGE((reg), 0x30000, 0x40000))
-
-#define FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg) \
-	REG_RANGE((reg), 0x9400, 0x9800)
-
-#define FORCEWAKE_GEN9_BLITTER_RANGE_OFFSET(reg) \
-	((reg) < 0x40000 && \
-	 !FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg) && \
-	 !FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg) && \
-	 !FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg) && \
-	 !FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg))
-
-#define SKL_NEEDS_FORCE_WAKE(reg) \
-	((reg) < 0x40000 && !FORCEWAKE_GEN9_UNCORE_RANGE_OFFSET(reg))
-
-#define __gen9_reg_read_fw_domains(offset) \
-({ \
-	enum forcewake_domains __fwd; \
-	if (!SKL_NEEDS_FORCE_WAKE(offset)) \
-		__fwd = 0; \
-	else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_RENDER; \
-	else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_MEDIA; \
-	else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
-	else \
-		__fwd = FORCEWAKE_BLITTER; \
-	__fwd; \
-})
-
-static const i915_reg_t gen9_shadowed_regs[] = {
-	RING_TAIL(RENDER_RING_BASE),
-	RING_TAIL(GEN6_BSD_RING_BASE),
-	RING_TAIL(VEBOX_RING_BASE),
-	RING_TAIL(BLT_RING_BASE),
-	GEN6_RPNSWREQ,
-	GEN6_RC_VIDEO_FREQ,
-	/* TODO: Other registers are not yet used */
+/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
+static const struct intel_forcewake_range __gen9_fw_ranges[] = {
+	GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
+	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
+	GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
+	GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER),
+	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
 };
 
-static bool is_gen9_shadowed(u32 offset)
-{
-	int i;
-	for (i = 0; i < ARRAY_SIZE(gen9_shadowed_regs); i++)
-		if (offset == gen9_shadowed_regs[i].reg)
-			return true;
-
-	return false;
-}
-
-#define __gen9_reg_write_fw_domains(offset) \
-({ \
-	enum forcewake_domains __fwd; \
-	if (!SKL_NEEDS_FORCE_WAKE(offset) || is_gen9_shadowed(offset)) \
-		__fwd = 0; \
-	else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_RENDER; \
-	else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_MEDIA; \
-	else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(offset)) \
-		__fwd = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
-	else \
-		__fwd = FORCEWAKE_BLITTER; \
-	__fwd; \
-})
-
 static void
 ilk_dummy_write(struct drm_i915_private *dev_priv)
 {
@@ -815,6 +839,66 @@ unclaimed_reg_debug(struct drm_i915_private *dev_priv,
 	__unclaimed_reg_debug(dev_priv, reg, read, before);
 }
 
+static const enum decoupled_power_domain fw2dpd_domain[] = {
+	GEN9_DECOUPLED_PD_RENDER,
+	GEN9_DECOUPLED_PD_BLITTER,
+	GEN9_DECOUPLED_PD_ALL,
+	GEN9_DECOUPLED_PD_MEDIA,
+	GEN9_DECOUPLED_PD_ALL,
+	GEN9_DECOUPLED_PD_ALL,
+	GEN9_DECOUPLED_PD_ALL
+};
+
+/*
+ * Decoupled MMIO access for only 1 DWORD
+ */
+static void __gen9_decoupled_mmio_access(struct drm_i915_private *dev_priv,
+					 u32 reg,
+					 enum forcewake_domains fw_domain,
+					 enum decoupled_ops operation)
+{
+	enum decoupled_power_domain dp_domain;
+	u32 ctrl_reg_data = 0;
+
+	dp_domain = fw2dpd_domain[fw_domain - 1];
+
+	ctrl_reg_data |= reg;
+	ctrl_reg_data |= (operation << GEN9_DECOUPLED_OP_SHIFT);
+	ctrl_reg_data |= (dp_domain << GEN9_DECOUPLED_PD_SHIFT);
+	ctrl_reg_data |= GEN9_DECOUPLED_DW1_GO;
+	__raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW1, ctrl_reg_data);
+
+	if (wait_for_atomic((__raw_i915_read32(dev_priv,
+			    GEN9_DECOUPLED_REG0_DW1) &
+			    GEN9_DECOUPLED_DW1_GO) == 0,
+			    FORCEWAKE_ACK_TIMEOUT_MS))
+		DRM_ERROR("Decoupled MMIO wait timed out\n");
+}
+
+static inline u32
+__gen9_decoupled_mmio_read32(struct drm_i915_private *dev_priv,
+			     u32 reg,
+			     enum forcewake_domains fw_domain)
+{
+	__gen9_decoupled_mmio_access(dev_priv, reg, fw_domain,
+				     GEN9_DECOUPLED_OP_READ);
+
+	return __raw_i915_read32(dev_priv, GEN9_DECOUPLED_REG0_DW0);
+}
+
+static inline void
+__gen9_decoupled_mmio_write(struct drm_i915_private *dev_priv,
+			    u32 reg, u32 data,
+			    enum forcewake_domains fw_domain)
+{
+
+	__raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW0, data);
+
+	__gen9_decoupled_mmio_access(dev_priv, reg, fw_domain,
+				     GEN9_DECOUPLED_OP_WRITE);
+}
+
+
 #define GEN2_READ_HEADER(x) \
 	u##x val = 0; \
 	assert_rpm_wakelock_held(dev_priv);
@@ -869,26 +953,30 @@ __gen2_read(64)
 	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
 	return val
 
-static inline void __force_wake_auto(struct drm_i915_private *dev_priv,
-				     enum forcewake_domains fw_domains)
+static noinline void ___force_wake_auto(struct drm_i915_private *dev_priv,
+					enum forcewake_domains fw_domains)
 {
 	struct intel_uncore_forcewake_domain *domain;
 
+	for_each_fw_domain_masked(domain, fw_domains, dev_priv)
+		fw_domain_arm_timer(domain);
+
+	dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
+	dev_priv->uncore.fw_domains_active |= fw_domains;
+}
+
+static inline void __force_wake_auto(struct drm_i915_private *dev_priv,
+				     enum forcewake_domains fw_domains)
+{
 	if (WARN_ON(!fw_domains))
 		return;
 
-	/* Ideally GCC would be constant-fold and eliminate this loop */
-	for_each_fw_domain_masked(domain, fw_domains, dev_priv) {
-		if (domain->wake_count) {
-			fw_domains &= ~domain->mask;
-			continue;
-		}
-
-		fw_domain_arm_timer(domain);
-	}
+	/* Turn on all requested but inactive supported forcewake domains. */
+	fw_domains &= dev_priv->uncore.fw_domains;
+	fw_domains &= ~dev_priv->uncore.fw_domains_active;
 
 	if (fw_domains)
-		dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
+		___force_wake_auto(dev_priv, fw_domains);
 }
 
 #define __gen6_read(x) \
@@ -903,62 +991,50 @@ gen6_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
 	GEN6_READ_FOOTER; \
 }
 
-#define __vlv_read(x) \
+#define __fwtable_read(x) \
 static u##x \
-vlv_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
+fwtable_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
 	enum forcewake_domains fw_engine; \
 	GEN6_READ_HEADER(x); \
-	fw_engine = __vlv_reg_read_fw_domains(offset); \
+	fw_engine = __fwtable_reg_read_fw_domains(offset); \
 	if (fw_engine) \
 		__force_wake_auto(dev_priv, fw_engine); \
 	val = __raw_i915_read##x(dev_priv, reg); \
 	GEN6_READ_FOOTER; \
 }
 
-#define __chv_read(x) \
+#define __gen9_decoupled_read(x) \
 static u##x \
-chv_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
+gen9_decoupled_read##x(struct drm_i915_private *dev_priv, \
+		       i915_reg_t reg, bool trace) { \
 	enum forcewake_domains fw_engine; \
 	GEN6_READ_HEADER(x); \
-	fw_engine = __chv_reg_read_fw_domains(offset); \
-	if (fw_engine) \
-		__force_wake_auto(dev_priv, fw_engine); \
-	val = __raw_i915_read##x(dev_priv, reg); \
-	GEN6_READ_FOOTER; \
-}
-
-#define __gen9_read(x) \
-static u##x \
-gen9_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
-	enum forcewake_domains fw_engine; \
-	GEN6_READ_HEADER(x); \
-	fw_engine = __gen9_reg_read_fw_domains(offset); \
-	if (fw_engine) \
-		__force_wake_auto(dev_priv, fw_engine); \
-	val = __raw_i915_read##x(dev_priv, reg); \
+	fw_engine = __fwtable_reg_read_fw_domains(offset); \
+	if (fw_engine & ~dev_priv->uncore.fw_domains_active) { \
+		unsigned i; \
+		u32 *ptr_data = (u32 *) &val; \
+		for (i = 0; i < x/32; i++, offset += sizeof(u32), ptr_data++) \
+			*ptr_data = __gen9_decoupled_mmio_read32(dev_priv, \
+								 offset, \
+								 fw_engine); \
+	} else { \
+		val = __raw_i915_read##x(dev_priv, reg); \
+	} \
 	GEN6_READ_FOOTER; \
 }
 
-__gen9_read(8)
-__gen9_read(16)
-__gen9_read(32)
-__gen9_read(64)
-__chv_read(8)
-__chv_read(16)
-__chv_read(32)
-__chv_read(64)
-__vlv_read(8)
-__vlv_read(16)
-__vlv_read(32)
-__vlv_read(64)
+__gen9_decoupled_read(32)
+__gen9_decoupled_read(64)
+__fwtable_read(8)
+__fwtable_read(16)
+__fwtable_read(32)
+__fwtable_read(64)
 __gen6_read(8)
 __gen6_read(16)
 __gen6_read(32)
 __gen6_read(64)
 
-#undef __gen9_read
-#undef __chv_read
-#undef __vlv_read
+#undef __fwtable_read
 #undef __gen6_read
 #undef GEN6_READ_FOOTER
 #undef GEN6_READ_HEADER
@@ -1054,21 +1130,6 @@ gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool
 	GEN6_WRITE_FOOTER; \
 }
 
-#define __hsw_write(x) \
-static void \
-hsw_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
-	u32 __fifo_ret = 0; \
-	GEN6_WRITE_HEADER; \
-	if (NEEDS_FORCE_WAKE(offset)) { \
-		__fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
-	} \
-	__raw_i915_write##x(dev_priv, reg, val); \
-	if (unlikely(__fifo_ret)) { \
-		gen6_gt_check_fifodbg(dev_priv); \
-	} \
-	GEN6_WRITE_FOOTER; \
-}
-
 #define __gen8_write(x) \
 static void \
 gen8_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
@@ -1081,51 +1142,49 @@ gen8_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool
 	GEN6_WRITE_FOOTER; \
 }
 
-#define __chv_write(x) \
+#define __fwtable_write(x) \
 static void \
-chv_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
+fwtable_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
 	enum forcewake_domains fw_engine; \
 	GEN6_WRITE_HEADER; \
-	fw_engine = __chv_reg_write_fw_domains(offset); \
+	fw_engine = __fwtable_reg_write_fw_domains(offset); \
 	if (fw_engine) \
 		__force_wake_auto(dev_priv, fw_engine); \
 	__raw_i915_write##x(dev_priv, reg, val); \
 	GEN6_WRITE_FOOTER; \
 }
 
-#define __gen9_write(x) \
+#define __gen9_decoupled_write(x) \
 static void \
-gen9_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, \
+gen9_decoupled_write##x(struct drm_i915_private *dev_priv, \
+			i915_reg_t reg, u##x val, \
 		bool trace) { \
 	enum forcewake_domains fw_engine; \
 	GEN6_WRITE_HEADER; \
-	fw_engine = __gen9_reg_write_fw_domains(offset); \
-	if (fw_engine) \
-		__force_wake_auto(dev_priv, fw_engine); \
-	__raw_i915_write##x(dev_priv, reg, val); \
+	fw_engine = __fwtable_reg_write_fw_domains(offset); \
+	if (fw_engine & ~dev_priv->uncore.fw_domains_active) \
+		__gen9_decoupled_mmio_write(dev_priv, \
+					    offset, \
+					    val, \
+					    fw_engine); \
+	else \
+		__raw_i915_write##x(dev_priv, reg, val); \
 	GEN6_WRITE_FOOTER; \
 }
 
-__gen9_write(8)
-__gen9_write(16)
-__gen9_write(32)
-__chv_write(8)
-__chv_write(16)
-__chv_write(32)
+__gen9_decoupled_write(32)
+__fwtable_write(8)
+__fwtable_write(16)
+__fwtable_write(32)
 __gen8_write(8)
 __gen8_write(16)
 __gen8_write(32)
-__hsw_write(8)
-__hsw_write(16)
-__hsw_write(32)
 __gen6_write(8)
 __gen6_write(16)
 __gen6_write(32)
 
-#undef __gen9_write
-#undef __chv_write
+#undef __fwtable_write
 #undef __gen8_write
-#undef __hsw_write
 #undef __gen6_write
 #undef GEN6_WRITE_FOOTER
 #undef GEN6_WRITE_HEADER
@@ -1314,6 +1373,13 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 	WARN_ON(dev_priv->uncore.fw_domains == 0);
 }
 
+#define ASSIGN_FW_DOMAINS_TABLE(d) \
+{ \
+	dev_priv->uncore.fw_domains_table = \
+			(struct intel_forcewake_range *)(d); \
+	dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \
+}
+
 void intel_uncore_init(struct drm_i915_private *dev_priv)
 {
 	i915_check_vgpu(dev_priv);
@@ -1327,13 +1393,23 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
 	switch (INTEL_INFO(dev_priv)->gen) {
 	default:
 	case 9:
-		ASSIGN_WRITE_MMIO_VFUNCS(gen9);
-		ASSIGN_READ_MMIO_VFUNCS(gen9);
+		ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
+		ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
+		ASSIGN_READ_MMIO_VFUNCS(fwtable);
+		if (HAS_DECOUPLED_MMIO(dev_priv)) {
+			dev_priv->uncore.funcs.mmio_readl =
+						gen9_decoupled_read32;
+			dev_priv->uncore.funcs.mmio_readq =
+						gen9_decoupled_read64;
+			dev_priv->uncore.funcs.mmio_writel =
+						gen9_decoupled_write32;
+		}
 		break;
 	case 8:
 		if (IS_CHERRYVIEW(dev_priv)) {
-			ASSIGN_WRITE_MMIO_VFUNCS(chv);
-			ASSIGN_READ_MMIO_VFUNCS(chv);
+			ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges);
+			ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
+			ASSIGN_READ_MMIO_VFUNCS(fwtable);
 
 		} else {
 			ASSIGN_WRITE_MMIO_VFUNCS(gen8);
@@ -1342,14 +1418,11 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
 		break;
 	case 7:
 	case 6:
-		if (IS_HASWELL(dev_priv)) {
-			ASSIGN_WRITE_MMIO_VFUNCS(hsw);
-		} else {
-			ASSIGN_WRITE_MMIO_VFUNCS(gen6);
-		}
+		ASSIGN_WRITE_MMIO_VFUNCS(gen6);
 
 		if (IS_VALLEYVIEW(dev_priv)) {
-			ASSIGN_READ_MMIO_VFUNCS(vlv);
+			ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges);
+			ASSIGN_READ_MMIO_VFUNCS(fwtable);
 		} else {
 			ASSIGN_READ_MMIO_VFUNCS(gen6);
 		}
@@ -1366,6 +1439,10 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
 		break;
 	}
 
+	intel_fw_table_check(dev_priv);
+	if (INTEL_GEN(dev_priv) >= 8)
+		intel_shadow_table_check();
+
 	if (intel_vgpu_active(dev_priv)) {
 		ASSIGN_WRITE_MMIO_VFUNCS(vgpu);
 		ASSIGN_READ_MMIO_VFUNCS(vgpu);
@@ -1408,7 +1485,7 @@ int i915_reg_read_ioctl(struct drm_device *dev,
 
 	for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
 		if (i915_mmio_reg_offset(entry->offset_ldw) == (reg->offset & -entry->size) &&
-		    (INTEL_INFO(dev)->gen_mask & entry->gen_bitmask))
+		    (INTEL_INFO(dev_priv)->gen_mask & entry->gen_bitmask))
 			break;
 	}
 
@@ -1815,35 +1892,16 @@ static enum forcewake_domains
 intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv,
 				i915_reg_t reg)
 {
+	u32 offset = i915_mmio_reg_offset(reg);
 	enum forcewake_domains fw_domains;
 
-	if (intel_vgpu_active(dev_priv))
-		return 0;
-
-	switch (INTEL_GEN(dev_priv)) {
-	case 9:
-		fw_domains = __gen9_reg_read_fw_domains(i915_mmio_reg_offset(reg));
-		break;
-	case 8:
-		if (IS_CHERRYVIEW(dev_priv))
-			fw_domains = __chv_reg_read_fw_domains(i915_mmio_reg_offset(reg));
-		else
-			fw_domains = __gen6_reg_read_fw_domains(i915_mmio_reg_offset(reg));
-		break;
-	case 7:
-	case 6:
-		if (IS_VALLEYVIEW(dev_priv))
-			fw_domains = __vlv_reg_read_fw_domains(i915_mmio_reg_offset(reg));
-		else
-			fw_domains = __gen6_reg_read_fw_domains(i915_mmio_reg_offset(reg));
-		break;
-	default:
-		MISSING_CASE(INTEL_INFO(dev_priv)->gen);
-	case 5: /* forcewake was introduced with gen6 */
-	case 4:
-	case 3:
-	case 2:
-		return 0;
+	if (HAS_FWTABLE(dev_priv)) {
+		fw_domains = __fwtable_reg_read_fw_domains(offset);
+	} else if (INTEL_GEN(dev_priv) >= 6) {
+		fw_domains = __gen6_reg_read_fw_domains(offset);
+	} else {
+		WARN_ON(!IS_GEN(dev_priv, 2, 5));
+		fw_domains = 0;
 	}
 
 	WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
@@ -1855,32 +1913,18 @@ static enum forcewake_domains
 intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
 				 i915_reg_t reg)
 {
+	u32 offset = i915_mmio_reg_offset(reg);
 	enum forcewake_domains fw_domains;
 
-	if (intel_vgpu_active(dev_priv))
-		return 0;
-
-	switch (INTEL_GEN(dev_priv)) {
-	case 9:
-		fw_domains = __gen9_reg_write_fw_domains(i915_mmio_reg_offset(reg));
-		break;
-	case 8:
-		if (IS_CHERRYVIEW(dev_priv))
-			fw_domains = __chv_reg_write_fw_domains(i915_mmio_reg_offset(reg));
-		else
-			fw_domains = __gen8_reg_write_fw_domains(i915_mmio_reg_offset(reg));
-		break;
-	case 7:
-	case 6:
+	if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) {
+		fw_domains = __fwtable_reg_write_fw_domains(offset);
+	} else if (IS_GEN8(dev_priv)) {
+		fw_domains = __gen8_reg_write_fw_domains(offset);
+	} else if (IS_GEN(dev_priv, 6, 7)) {
 		fw_domains = FORCEWAKE_RENDER;
-		break;
-	default:
-		MISSING_CASE(INTEL_INFO(dev_priv)->gen);
-	case 5:
-	case 4:
-	case 3:
-	case 2:
-		return 0;
+	} else {
+		WARN_ON(!IS_GEN(dev_priv, 2, 5));
+		fw_domains = 0;
 	}
 
 	WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
@@ -1910,6 +1954,9 @@ intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
 
 	WARN_ON(!op);
 
+	if (intel_vgpu_active(dev_priv))
+		return 0;
+
 	if (op & FW_REG_READ)
 		fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg);
 
diff --git a/drivers/gpu/drm/imx/imx-drm-core.c b/drivers/gpu/drm/imx/imx-drm-core.c
index 9672b579f950..33404295b447 100644
--- a/drivers/gpu/drm/imx/imx-drm-core.c
+++ b/drivers/gpu/drm/imx/imx-drm-core.c
@@ -18,7 +18,6 @@
 #include <linux/dma-buf.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/reservation.h>
 #include <drm/drmP.h>
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
@@ -151,38 +150,11 @@ static int imx_drm_atomic_check(struct drm_device *dev,
 	return ret;
 }
 
-static int imx_drm_atomic_commit(struct drm_device *dev,
-				 struct drm_atomic_state *state,
-				 bool nonblock)
-{
-	struct drm_plane_state *plane_state;
-	struct drm_plane *plane;
-	struct dma_buf *dma_buf;
-	int i;
-
-	/*
-	 * If the plane fb has an dma-buf attached, fish out the exclusive
-	 * fence for the atomic helper to wait on.
-	 */
-	for_each_plane_in_state(state, plane, plane_state, i) {
-		if ((plane->state->fb != plane_state->fb) && plane_state->fb) {
-			dma_buf = drm_fb_cma_get_gem_obj(plane_state->fb,
-							 0)->base.dma_buf;
-			if (!dma_buf)
-				continue;
-			plane_state->fence =
-				reservation_object_get_excl_rcu(dma_buf->resv);
-		}
-	}
-
-	return drm_atomic_helper_commit(dev, state, nonblock);
-}
-
 static const struct drm_mode_config_funcs imx_drm_mode_config_funcs = {
 	.fb_create = drm_fb_cma_create,
 	.output_poll_changed = imx_drm_output_poll_changed,
 	.atomic_check = imx_drm_atomic_check,
-	.atomic_commit = imx_drm_atomic_commit,
+	.atomic_commit = drm_atomic_helper_commit,
 };
 
 static void imx_drm_atomic_commit_tail(struct drm_atomic_state *state)
diff --git a/drivers/gpu/drm/imx/imx-ldb.c b/drivers/gpu/drm/imx/imx-ldb.c
index 3ce391c239b0..516d06490465 100644
--- a/drivers/gpu/drm/imx/imx-ldb.c
+++ b/drivers/gpu/drm/imx/imx-ldb.c
@@ -101,12 +101,6 @@ struct imx_ldb {
 	const struct bus_mux *lvds_mux;
 };
 
-static enum drm_connector_status imx_ldb_connector_detect(
-		struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static void imx_ldb_ch_set_bus_format(struct imx_ldb_channel *imx_ldb_ch,
 				      u32 bus_format)
 {
@@ -319,18 +313,6 @@ static void imx_ldb_encoder_disable(struct drm_encoder *encoder)
 	struct imx_ldb *ldb = imx_ldb_ch->ldb;
 	int mux, ret;
 
-	/*
-	 * imx_ldb_encoder_disable is called by
-	 * drm_helper_disable_unused_functions without
-	 * the encoder being enabled before.
-	 */
-	if (imx_ldb_ch == &ldb->channel[0] &&
-	    (ldb->ldb_ctrl & LDB_CH0_MODE_EN_MASK) == 0)
-		return;
-	else if (imx_ldb_ch == &ldb->channel[1] &&
-		 (ldb->ldb_ctrl & LDB_CH1_MODE_EN_MASK) == 0)
-		return;
-
 	drm_panel_disable(imx_ldb_ch->panel);
 
 	if (imx_ldb_ch == &ldb->channel[0])
@@ -409,7 +391,6 @@ static int imx_ldb_encoder_atomic_check(struct drm_encoder *encoder,
 static const struct drm_connector_funcs imx_ldb_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.detect = imx_ldb_connector_detect,
 	.destroy = imx_drm_connector_destroy,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/imx/imx-tve.c b/drivers/gpu/drm/imx/imx-tve.c
index 8fc088843e55..3b602ee33c44 100644
--- a/drivers/gpu/drm/imx/imx-tve.c
+++ b/drivers/gpu/drm/imx/imx-tve.c
@@ -227,12 +227,6 @@ static int tve_setup_vga(struct imx_tve *tve)
 				 TVE_TVDAC_TEST_MODE_MASK, 1);
 }
 
-static enum drm_connector_status imx_tve_connector_detect(
-				struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static int imx_tve_connector_get_modes(struct drm_connector *connector)
 {
 	struct imx_tve *tve = con_to_tve(connector);
@@ -352,7 +346,6 @@ static int imx_tve_atomic_check(struct drm_encoder *encoder,
 static const struct drm_connector_funcs imx_tve_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.detect = imx_tve_connector_detect,
 	.destroy = imx_drm_connector_destroy,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/imx/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3-plane.c
index d5864ed4d772..e74a0ad52950 100644
--- a/drivers/gpu/drm/imx/ipuv3-plane.c
+++ b/drivers/gpu/drm/imx/ipuv3-plane.c
@@ -50,6 +50,12 @@ static const uint32_t ipu_plane_formats[] = {
 	DRM_FORMAT_YVYU,
 	DRM_FORMAT_YUV420,
 	DRM_FORMAT_YVU420,
+	DRM_FORMAT_YUV422,
+	DRM_FORMAT_YVU422,
+	DRM_FORMAT_YUV444,
+	DRM_FORMAT_YVU444,
+	DRM_FORMAT_NV12,
+	DRM_FORMAT_NV16,
 	DRM_FORMAT_RGB565,
 };
 
@@ -64,13 +70,14 @@ drm_plane_state_to_eba(struct drm_plane_state *state)
 {
 	struct drm_framebuffer *fb = state->fb;
 	struct drm_gem_cma_object *cma_obj;
+	int x = state->src_x >> 16;
+	int y = state->src_y >> 16;
 
 	cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
 	BUG_ON(!cma_obj);
 
-	return cma_obj->paddr + fb->offsets[0] +
-	       fb->pitches[0] * (state->src_y >> 16) +
-	       (fb->bits_per_pixel >> 3) * (state->src_x >> 16);
+	return cma_obj->paddr + fb->offsets[0] + fb->pitches[0] * y +
+	       drm_format_plane_cpp(fb->pixel_format, 0) * x;
 }
 
 static inline unsigned long
@@ -79,13 +86,17 @@ drm_plane_state_to_ubo(struct drm_plane_state *state)
 	struct drm_framebuffer *fb = state->fb;
 	struct drm_gem_cma_object *cma_obj;
 	unsigned long eba = drm_plane_state_to_eba(state);
+	int x = state->src_x >> 16;
+	int y = state->src_y >> 16;
 
 	cma_obj = drm_fb_cma_get_gem_obj(fb, 1);
 	BUG_ON(!cma_obj);
 
-	return cma_obj->paddr + fb->offsets[1] +
-	       fb->pitches[1] * (state->src_y >> 16) / 2 +
-	       (state->src_x >> 16) / 2 - eba;
+	x /= drm_format_horz_chroma_subsampling(fb->pixel_format);
+	y /= drm_format_vert_chroma_subsampling(fb->pixel_format);
+
+	return cma_obj->paddr + fb->offsets[1] + fb->pitches[1] * y +
+	       drm_format_plane_cpp(fb->pixel_format, 1) * x - eba;
 }
 
 static inline unsigned long
@@ -94,69 +105,17 @@ drm_plane_state_to_vbo(struct drm_plane_state *state)
 	struct drm_framebuffer *fb = state->fb;
 	struct drm_gem_cma_object *cma_obj;
 	unsigned long eba = drm_plane_state_to_eba(state);
+	int x = state->src_x >> 16;
+	int y = state->src_y >> 16;
 
 	cma_obj = drm_fb_cma_get_gem_obj(fb, 2);
 	BUG_ON(!cma_obj);
 
-	return cma_obj->paddr + fb->offsets[2] +
-	       fb->pitches[2] * (state->src_y >> 16) / 2 +
-	       (state->src_x >> 16) / 2 - eba;
-}
-
-static void ipu_plane_atomic_set_base(struct ipu_plane *ipu_plane)
-{
-	struct drm_plane *plane = &ipu_plane->base;
-	struct drm_plane_state *state = plane->state;
-	struct drm_crtc_state *crtc_state = state->crtc->state;
-	struct drm_framebuffer *fb = state->fb;
-	unsigned long eba, ubo, vbo;
-	int active;
-
-	eba = drm_plane_state_to_eba(state);
-
-	switch (fb->pixel_format) {
-	case DRM_FORMAT_YUV420:
-	case DRM_FORMAT_YVU420:
-		if (!drm_atomic_crtc_needs_modeset(crtc_state))
-			break;
+	x /= drm_format_horz_chroma_subsampling(fb->pixel_format);
+	y /= drm_format_vert_chroma_subsampling(fb->pixel_format);
 
-		/*
-		 * Multiplanar formats have to meet the following restrictions:
-		 * - The (up to) three plane addresses are EBA, EBA+UBO, EBA+VBO
-		 * - EBA, UBO and VBO are a multiple of 8
-		 * - UBO and VBO are unsigned and not larger than 0xfffff8
-		 * - Only EBA may be changed while scanout is active
-		 * - The strides of U and V planes must be identical.
-		 */
-		ubo = drm_plane_state_to_ubo(state);
-		vbo = drm_plane_state_to_vbo(state);
-
-		if (fb->pixel_format == DRM_FORMAT_YUV420)
-			ipu_cpmem_set_yuv_planar_full(ipu_plane->ipu_ch,
-						      fb->pitches[1], ubo, vbo);
-		else
-			ipu_cpmem_set_yuv_planar_full(ipu_plane->ipu_ch,
-						      fb->pitches[1], vbo, ubo);
-
-		dev_dbg(ipu_plane->base.dev->dev,
-			"phy = %lu %lu %lu, x = %d, y = %d", eba, ubo, vbo,
-			state->src_x >> 16, state->src_y >> 16);
-		break;
-	default:
-		dev_dbg(ipu_plane->base.dev->dev, "phys = %lu, x = %d, y = %d",
-			eba, state->src_x >> 16, state->src_y >> 16);
-
-		break;
-	}
-
-	if (!drm_atomic_crtc_needs_modeset(crtc_state)) {
-		active = ipu_idmac_get_current_buffer(ipu_plane->ipu_ch);
-		ipu_cpmem_set_buffer(ipu_plane->ipu_ch, !active, eba);
-		ipu_idmac_select_buffer(ipu_plane->ipu_ch, !active);
-	} else {
-		ipu_cpmem_set_buffer(ipu_plane->ipu_ch, 0, eba);
-		ipu_cpmem_set_buffer(ipu_plane->ipu_ch, 1, eba);
-	}
+	return cma_obj->paddr + fb->offsets[2] + fb->pitches[2] * y +
+	       drm_format_plane_cpp(fb->pixel_format, 2) * x - eba;
 }
 
 void ipu_plane_put_resources(struct ipu_plane *ipu_plane)
@@ -339,6 +298,10 @@ static int ipu_plane_atomic_check(struct drm_plane *plane,
 	switch (fb->pixel_format) {
 	case DRM_FORMAT_YUV420:
 	case DRM_FORMAT_YVU420:
+	case DRM_FORMAT_YUV422:
+	case DRM_FORMAT_YVU422:
+	case DRM_FORMAT_YUV444:
+	case DRM_FORMAT_YVU444:
 		/*
 		 * Multiplanar formats have to meet the following restrictions:
 		 * - The (up to) three plane addresses are EBA, EBA+UBO, EBA+VBO
@@ -347,27 +310,34 @@ static int ipu_plane_atomic_check(struct drm_plane *plane,
 		 * - Only EBA may be changed while scanout is active
 		 * - The strides of U and V planes must be identical.
 		 */
-		ubo = drm_plane_state_to_ubo(state);
 		vbo = drm_plane_state_to_vbo(state);
 
-		if ((ubo & 0x7) || (vbo & 0x7))
-			return -EINVAL;
-
-		if ((ubo > 0xfffff8) || (vbo > 0xfffff8))
+		if (vbo & 0x7 || vbo > 0xfffff8)
 			return -EINVAL;
 
-		if (old_fb &&
-		    (old_fb->pixel_format == DRM_FORMAT_YUV420 ||
-		     old_fb->pixel_format == DRM_FORMAT_YVU420)) {
-			old_ubo = drm_plane_state_to_ubo(old_state);
+		if (old_fb && (fb->pixel_format == old_fb->pixel_format)) {
 			old_vbo = drm_plane_state_to_vbo(old_state);
-			if (ubo != old_ubo || vbo != old_vbo)
-				return -EINVAL;
+			if (vbo != old_vbo)
+				crtc_state->mode_changed = true;
 		}
 
 		if (fb->pitches[1] != fb->pitches[2])
 			return -EINVAL;
 
+		/* fall-through */
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV16:
+		ubo = drm_plane_state_to_ubo(state);
+
+		if (ubo & 0x7 || ubo > 0xfffff8)
+			return -EINVAL;
+
+		if (old_fb && (fb->pixel_format == old_fb->pixel_format)) {
+			old_ubo = drm_plane_state_to_ubo(old_state);
+			if (ubo != old_ubo)
+				crtc_state->mode_changed = true;
+		}
+
 		if (fb->pitches[1] < 1 || fb->pitches[1] > 16384)
 			return -EINVAL;
 
@@ -399,15 +369,19 @@ static void ipu_plane_atomic_update(struct drm_plane *plane,
 {
 	struct ipu_plane *ipu_plane = to_ipu_plane(plane);
 	struct drm_plane_state *state = plane->state;
+	struct drm_crtc_state *crtc_state = state->crtc->state;
+	struct drm_framebuffer *fb = state->fb;
+	unsigned long eba, ubo, vbo;
 	enum ipu_color_space ics;
+	int active;
 
-	if (old_state->fb) {
-		struct drm_crtc_state *crtc_state = state->crtc->state;
+	eba = drm_plane_state_to_eba(state);
 
-		if (!drm_atomic_crtc_needs_modeset(crtc_state)) {
-			ipu_plane_atomic_set_base(ipu_plane);
-			return;
-		}
+	if (old_state->fb && !drm_atomic_crtc_needs_modeset(crtc_state)) {
+		active = ipu_idmac_get_current_buffer(ipu_plane->ipu_ch);
+		ipu_cpmem_set_buffer(ipu_plane->ipu_ch, !active, eba);
+		ipu_idmac_select_buffer(ipu_plane->ipu_ch, !active);
+		return;
 	}
 
 	switch (ipu_plane->dp_flow) {
@@ -451,11 +425,50 @@ static void ipu_plane_atomic_update(struct drm_plane *plane,
 	ipu_cpmem_set_high_priority(ipu_plane->ipu_ch);
 	ipu_idmac_set_double_buffer(ipu_plane->ipu_ch, 1);
 	ipu_cpmem_set_stride(ipu_plane->ipu_ch, state->fb->pitches[0]);
-	ipu_plane_atomic_set_base(ipu_plane);
+	switch (fb->pixel_format) {
+	case DRM_FORMAT_YUV420:
+	case DRM_FORMAT_YVU420:
+	case DRM_FORMAT_YUV422:
+	case DRM_FORMAT_YVU422:
+	case DRM_FORMAT_YUV444:
+	case DRM_FORMAT_YVU444:
+		ubo = drm_plane_state_to_ubo(state);
+		vbo = drm_plane_state_to_vbo(state);
+		if (fb->pixel_format == DRM_FORMAT_YVU420 ||
+		    fb->pixel_format == DRM_FORMAT_YVU422 ||
+		    fb->pixel_format == DRM_FORMAT_YVU444)
+			swap(ubo, vbo);
+
+		ipu_cpmem_set_yuv_planar_full(ipu_plane->ipu_ch,
+					      fb->pitches[1], ubo, vbo);
+
+		dev_dbg(ipu_plane->base.dev->dev,
+			"phy = %lu %lu %lu, x = %d, y = %d", eba, ubo, vbo,
+			state->src_x >> 16, state->src_y >> 16);
+		break;
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV16:
+		ubo = drm_plane_state_to_ubo(state);
+
+		ipu_cpmem_set_yuv_planar_full(ipu_plane->ipu_ch,
+					      fb->pitches[1], ubo, ubo);
+
+		dev_dbg(ipu_plane->base.dev->dev,
+			"phy = %lu %lu, x = %d, y = %d", eba, ubo,
+			state->src_x >> 16, state->src_y >> 16);
+		break;
+	default:
+		dev_dbg(ipu_plane->base.dev->dev, "phys = %lu, x = %d, y = %d",
+			eba, state->src_x >> 16, state->src_y >> 16);
+		break;
+	}
+	ipu_cpmem_set_buffer(ipu_plane->ipu_ch, 0, eba);
+	ipu_cpmem_set_buffer(ipu_plane->ipu_ch, 1, eba);
 	ipu_plane_enable(ipu_plane);
 }
 
 static const struct drm_plane_helper_funcs ipu_plane_helper_funcs = {
+	.prepare_fb = drm_fb_cma_prepare_fb,
 	.atomic_check = ipu_plane_atomic_check,
 	.atomic_disable = ipu_plane_atomic_disable,
 	.atomic_update = ipu_plane_atomic_update,
diff --git a/drivers/gpu/drm/imx/parallel-display.c b/drivers/gpu/drm/imx/parallel-display.c
index d796ada2a47a..8582a83c0d9b 100644
--- a/drivers/gpu/drm/imx/parallel-display.c
+++ b/drivers/gpu/drm/imx/parallel-display.c
@@ -49,12 +49,6 @@ static inline struct imx_parallel_display *enc_to_imxpd(struct drm_encoder *e)
 	return container_of(e, struct imx_parallel_display, encoder);
 }
 
-static enum drm_connector_status imx_pd_connector_detect(
-		struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static int imx_pd_connector_get_modes(struct drm_connector *connector)
 {
 	struct imx_parallel_display *imxpd = con_to_imxpd(connector);
@@ -143,7 +137,6 @@ static int imx_pd_encoder_atomic_check(struct drm_encoder *encoder,
 static const struct drm_connector_funcs imx_pd_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.detect = imx_pd_connector_detect,
 	.destroy = imx_drm_connector_destroy,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
index cf83f6507ec8..4b7fe7eaec01 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
@@ -18,6 +18,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_gem.h>
 #include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_of.h>
 #include <linux/component.h>
 #include <linux/iommu.h>
 #include <linux/of_address.h>
@@ -83,7 +84,7 @@ static void mtk_atomic_complete(struct mtk_drm_private *private,
 	drm_atomic_helper_wait_for_vblanks(drm, state);
 
 	drm_atomic_helper_cleanup_planes(drm, state);
-	drm_atomic_state_free(state);
+	drm_atomic_state_put(state);
 }
 
 static void mtk_atomic_work(struct work_struct *work)
@@ -110,6 +111,7 @@ static int mtk_atomic_commit(struct drm_device *drm,
 
 	drm_atomic_helper_swap_state(state, true);
 
+	drm_atomic_state_get(state);
 	if (async)
 		mtk_atomic_schedule(private, state);
 	else
@@ -247,16 +249,14 @@ static const struct file_operations mtk_drm_fops = {
 	.mmap = mtk_drm_gem_mmap,
 	.poll = drm_poll,
 	.read = drm_read,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 };
 
 static struct drm_driver mtk_drm_driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME |
 			   DRIVER_ATOMIC,
 
-	.get_vblank_counter = drm_vblank_count,
+	.get_vblank_counter = drm_vblank_no_hw_counter,
 	.enable_vblank = mtk_drm_crtc_enable_vblank,
 	.disable_vblank = mtk_drm_crtc_disable_vblank,
 
@@ -415,7 +415,8 @@ static int mtk_drm_probe(struct platform_device *pdev)
 		    comp_type == MTK_DPI) {
 			dev_info(dev, "Adding component match for %s\n",
 				 node->full_name);
-			component_match_add(dev, &match, compare_of, node);
+			drm_of_component_match_add(dev, &match, compare_of,
+						   node);
 		} else {
 			struct mtk_ddp_comp *comp;
 
diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c
index eaa5a2240c0c..2c42f90809d8 100644
--- a/drivers/gpu/drm/mediatek/mtk_dsi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dsi.c
@@ -594,12 +594,6 @@ static void mtk_dsi_encoder_enable(struct drm_encoder *encoder)
 	mtk_output_dsi_enable(dsi);
 }
 
-static enum drm_connector_status mtk_dsi_connector_detect(
-	struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static int mtk_dsi_connector_get_modes(struct drm_connector *connector)
 {
 	struct mtk_dsi *dsi = connector_to_dsi(connector);
@@ -616,7 +610,6 @@ static const struct drm_encoder_helper_funcs mtk_dsi_encoder_helper_funcs = {
 
 static const struct drm_connector_funcs mtk_dsi_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
-	.detect = mtk_dsi_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
diff --git a/drivers/gpu/drm/meson/Kconfig b/drivers/gpu/drm/meson/Kconfig
new file mode 100644
index 000000000000..99719afcc77f
--- /dev/null
+++ b/drivers/gpu/drm/meson/Kconfig
@@ -0,0 +1,9 @@
+config DRM_MESON
+	tristate "DRM Support for Amlogic Meson Display Controller"
+	depends on DRM && OF && (ARM || ARM64)
+	depends on ARCH_MESON || COMPILE_TEST
+	select DRM_KMS_HELPER
+	select DRM_KMS_CMA_HELPER
+	select DRM_GEM_CMA_HELPER
+	select VIDEOMODE_HELPERS
+	select REGMAP_MMIO
diff --git a/drivers/gpu/drm/meson/Makefile b/drivers/gpu/drm/meson/Makefile
new file mode 100644
index 000000000000..2591978b8aad
--- /dev/null
+++ b/drivers/gpu/drm/meson/Makefile
@@ -0,0 +1,4 @@
+meson-y := meson_drv.o meson_plane.o meson_crtc.o meson_venc_cvbs.o
+meson-y += meson_viu.o meson_vpp.o meson_venc.o meson_vclk.o meson_canvas.o
+
+obj-$(CONFIG_DRM_MESON) += meson.o
diff --git a/drivers/gpu/drm/meson/meson_canvas.c b/drivers/gpu/drm/meson/meson_canvas.c
new file mode 100644
index 000000000000..4109e36c297f
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_canvas.c
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2015 Amlogic, Inc. All rights reserved.
+ * Copyright (C) 2014 Endless Mobile
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include "meson_drv.h"
+#include "meson_canvas.h"
+#include "meson_registers.h"
+
+/*
+ * CANVAS is a memory zone where physical memory frames information
+ * are stored for the VIU to scanout.
+ */
+
+/* DMC Registers */
+#define DMC_CAV_LUT_DATAL	0x48 /* 0x12 offset in data sheet */
+#define CANVAS_WIDTH_LBIT	29
+#define CANVAS_WIDTH_LWID       3
+#define DMC_CAV_LUT_DATAH	0x4c /* 0x13 offset in data sheet */
+#define CANVAS_WIDTH_HBIT       0
+#define CANVAS_HEIGHT_BIT       9
+#define CANVAS_BLKMODE_BIT      24
+#define DMC_CAV_LUT_ADDR	0x50 /* 0x14 offset in data sheet */
+#define CANVAS_LUT_WR_EN        (0x2 << 8)
+#define CANVAS_LUT_RD_EN        (0x1 << 8)
+
+void meson_canvas_setup(struct meson_drm *priv,
+			uint32_t canvas_index, uint32_t addr,
+			uint32_t stride, uint32_t height,
+			unsigned int wrap,
+			unsigned int blkmode)
+{
+	unsigned int val;
+
+	regmap_write(priv->dmc, DMC_CAV_LUT_DATAL,
+		(((addr + 7) >> 3)) |
+		(((stride + 7) >> 3) << CANVAS_WIDTH_LBIT));
+
+	regmap_write(priv->dmc, DMC_CAV_LUT_DATAH,
+		((((stride + 7) >> 3) >> CANVAS_WIDTH_LWID) <<
+						CANVAS_WIDTH_HBIT) |
+		(height << CANVAS_HEIGHT_BIT) |
+		(wrap << 22) |
+		(blkmode << CANVAS_BLKMODE_BIT));
+
+	regmap_write(priv->dmc, DMC_CAV_LUT_ADDR,
+			CANVAS_LUT_WR_EN | canvas_index);
+
+	/* Force a read-back to make sure everything is flushed. */
+	regmap_read(priv->dmc, DMC_CAV_LUT_DATAH, &val);
+}
diff --git a/drivers/gpu/drm/meson/meson_canvas.h b/drivers/gpu/drm/meson/meson_canvas.h
new file mode 100644
index 000000000000..af1759da4b27
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_canvas.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2014 Endless Mobile
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* Canvas LUT Memory */
+
+#ifndef __MESON_CANVAS_H
+#define __MESON_CANVAS_H
+
+#define MESON_CANVAS_ID_OSD1	0x4e
+
+/* Canvas configuration. */
+#define MESON_CANVAS_WRAP_NONE	0x00
+#define	MESON_CANVAS_WRAP_X	0x01
+#define	MESON_CANVAS_WRAP_Y	0x02
+
+#define	MESON_CANVAS_BLKMODE_LINEAR	0x00
+#define	MESON_CANVAS_BLKMODE_32x32	0x01
+#define	MESON_CANVAS_BLKMODE_64x64	0x02
+
+void meson_canvas_setup(struct meson_drm *priv,
+			uint32_t canvas_index, uint32_t addr,
+			uint32_t stride, uint32_t height,
+			unsigned int wrap,
+			unsigned int blkmode);
+
+#endif /* __MESON_CANVAS_H */
diff --git a/drivers/gpu/drm/meson/meson_crtc.c b/drivers/gpu/drm/meson/meson_crtc.c
new file mode 100644
index 000000000000..749770e5c65f
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_crtc.c
@@ -0,0 +1,208 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2015 Amlogic, Inc. All rights reserved.
+ * Copyright (C) 2014 Endless Mobile
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * Written by:
+ *     Jasper St. Pierre <jstpierre@mecheye.net>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_flip_work.h>
+#include <drm/drm_crtc_helper.h>
+
+#include "meson_crtc.h"
+#include "meson_plane.h"
+#include "meson_vpp.h"
+#include "meson_viu.h"
+#include "meson_registers.h"
+
+/* CRTC definition */
+
+struct meson_crtc {
+	struct drm_crtc base;
+	struct drm_pending_vblank_event *event;
+	struct meson_drm *priv;
+};
+#define to_meson_crtc(x) container_of(x, struct meson_crtc, base)
+
+/* CRTC */
+
+static const struct drm_crtc_funcs meson_crtc_funcs = {
+	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.destroy		= drm_crtc_cleanup,
+	.page_flip		= drm_atomic_helper_page_flip,
+	.reset			= drm_atomic_helper_crtc_reset,
+	.set_config             = drm_atomic_helper_set_config,
+};
+
+static void meson_crtc_enable(struct drm_crtc *crtc)
+{
+	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
+	struct drm_plane *plane = meson_crtc->priv->primary_plane;
+	struct meson_drm *priv = meson_crtc->priv;
+
+	/* Enable VPP Postblend */
+	writel(plane->state->crtc_w,
+	       priv->io_base + _REG(VPP_POSTBLEND_H_SIZE));
+
+	writel_bits_relaxed(VPP_POSTBLEND_ENABLE, VPP_POSTBLEND_ENABLE,
+			    priv->io_base + _REG(VPP_MISC));
+
+	priv->viu.osd1_enabled = true;
+}
+
+static void meson_crtc_disable(struct drm_crtc *crtc)
+{
+	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
+	struct meson_drm *priv = meson_crtc->priv;
+
+	priv->viu.osd1_enabled = false;
+
+	/* Disable VPP Postblend */
+	writel_bits_relaxed(VPP_POSTBLEND_ENABLE, 0,
+			    priv->io_base + _REG(VPP_MISC));
+
+	if (crtc->state->event && !crtc->state->active) {
+		spin_lock_irq(&crtc->dev->event_lock);
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		spin_unlock_irq(&crtc->dev->event_lock);
+
+		crtc->state->event = NULL;
+	}
+}
+
+static void meson_crtc_atomic_begin(struct drm_crtc *crtc,
+				    struct drm_crtc_state *state)
+{
+	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
+	unsigned long flags;
+
+	if (crtc->state->event) {
+		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+
+		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+		meson_crtc->event = crtc->state->event;
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+		crtc->state->event = NULL;
+	}
+}
+
+static void meson_crtc_atomic_flush(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_crtc_state)
+{
+	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
+	struct meson_drm *priv = meson_crtc->priv;
+
+	if (priv->viu.osd1_enabled)
+		priv->viu.osd1_commit = true;
+}
+
+static const struct drm_crtc_helper_funcs meson_crtc_helper_funcs = {
+	.enable		= meson_crtc_enable,
+	.disable	= meson_crtc_disable,
+	.atomic_begin	= meson_crtc_atomic_begin,
+	.atomic_flush	= meson_crtc_atomic_flush,
+};
+
+void meson_crtc_irq(struct meson_drm *priv)
+{
+	struct meson_crtc *meson_crtc = to_meson_crtc(priv->crtc);
+	unsigned long flags;
+
+	/* Update the OSD registers */
+	if (priv->viu.osd1_enabled && priv->viu.osd1_commit) {
+		writel_relaxed(priv->viu.osd1_ctrl_stat,
+				priv->io_base + _REG(VIU_OSD1_CTRL_STAT));
+		writel_relaxed(priv->viu.osd1_blk0_cfg[0],
+				priv->io_base + _REG(VIU_OSD1_BLK0_CFG_W0));
+		writel_relaxed(priv->viu.osd1_blk0_cfg[1],
+				priv->io_base + _REG(VIU_OSD1_BLK0_CFG_W1));
+		writel_relaxed(priv->viu.osd1_blk0_cfg[2],
+				priv->io_base + _REG(VIU_OSD1_BLK0_CFG_W2));
+		writel_relaxed(priv->viu.osd1_blk0_cfg[3],
+				priv->io_base + _REG(VIU_OSD1_BLK0_CFG_W3));
+		writel_relaxed(priv->viu.osd1_blk0_cfg[4],
+				priv->io_base + _REG(VIU_OSD1_BLK0_CFG_W4));
+
+		/* If output is interlace, make use of the Scaler */
+		if (priv->viu.osd1_interlace) {
+			struct drm_plane *plane = priv->primary_plane;
+			struct drm_plane_state *state = plane->state;
+			struct drm_rect dest = {
+				.x1 = state->crtc_x,
+				.y1 = state->crtc_y,
+				.x2 = state->crtc_x + state->crtc_w,
+				.y2 = state->crtc_y + state->crtc_h,
+			};
+
+			meson_vpp_setup_interlace_vscaler_osd1(priv, &dest);
+		} else
+			meson_vpp_disable_interlace_vscaler_osd1(priv);
+
+		/* Enable OSD1 */
+		writel_bits_relaxed(VPP_OSD1_POSTBLEND, VPP_OSD1_POSTBLEND,
+				    priv->io_base + _REG(VPP_MISC));
+
+		priv->viu.osd1_commit = false;
+	}
+
+	drm_crtc_handle_vblank(priv->crtc);
+
+	spin_lock_irqsave(&priv->drm->event_lock, flags);
+	if (meson_crtc->event) {
+		drm_crtc_send_vblank_event(priv->crtc, meson_crtc->event);
+		drm_crtc_vblank_put(priv->crtc);
+		meson_crtc->event = NULL;
+	}
+	spin_unlock_irqrestore(&priv->drm->event_lock, flags);
+}
+
+int meson_crtc_create(struct meson_drm *priv)
+{
+	struct meson_crtc *meson_crtc;
+	struct drm_crtc *crtc;
+	int ret;
+
+	meson_crtc = devm_kzalloc(priv->drm->dev, sizeof(*meson_crtc),
+				  GFP_KERNEL);
+	if (!meson_crtc)
+		return -ENOMEM;
+
+	meson_crtc->priv = priv;
+	crtc = &meson_crtc->base;
+	ret = drm_crtc_init_with_planes(priv->drm, crtc,
+					priv->primary_plane, NULL,
+					&meson_crtc_funcs, "meson_crtc");
+	if (ret) {
+		dev_err(priv->drm->dev, "Failed to init CRTC\n");
+		return ret;
+	}
+
+	drm_crtc_helper_add(crtc, &meson_crtc_helper_funcs);
+
+	priv->crtc = crtc;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/meson/meson_crtc.h b/drivers/gpu/drm/meson/meson_crtc.h
new file mode 100644
index 000000000000..b62b9e51764d
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_crtc.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2014 Endless Mobile
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * Written by:
+ *     Jasper St. Pierre <jstpierre@mecheye.net>
+ */
+
+#ifndef __MESON_CRTC_H
+#define __MESON_CRTC_H
+
+#include "meson_drv.h"
+
+int meson_crtc_create(struct meson_drm *priv);
+
+void meson_crtc_irq(struct meson_drm *priv);
+
+#endif /* __MESON_CRTC_H */
diff --git a/drivers/gpu/drm/meson/meson_drv.c b/drivers/gpu/drm/meson/meson_drv.c
new file mode 100644
index 000000000000..ff1f6019b97b
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_drv.c
@@ -0,0 +1,343 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2014 Endless Mobile
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * Written by:
+ *     Jasper St. Pierre <jstpierre@mecheye.net>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/of_graph.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_flip_work.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_rect.h>
+#include <drm/drm_fb_helper.h>
+
+#include "meson_drv.h"
+#include "meson_plane.h"
+#include "meson_crtc.h"
+#include "meson_venc_cvbs.h"
+
+#include "meson_vpp.h"
+#include "meson_viu.h"
+#include "meson_venc.h"
+#include "meson_canvas.h"
+#include "meson_registers.h"
+
+#define DRIVER_NAME "meson"
+#define DRIVER_DESC "Amlogic Meson DRM driver"
+
+/*
+ * Video Processing Unit
+ *
+ * VPU Handles the Global Video Processing, it includes management of the
+ * clocks gates, blocks reset lines and power domains.
+ *
+ * What is missing :
+ * - Full reset of entire video processing HW blocks
+ * - Scaling and setup of the VPU clock
+ * - Bus clock gates
+ * - Powering up video processing HW blocks
+ * - Powering Up HDMI controller and PHY
+ */
+
+static void meson_fb_output_poll_changed(struct drm_device *dev)
+{
+	struct meson_drm *priv = dev->dev_private;
+
+	drm_fbdev_cma_hotplug_event(priv->fbdev);
+}
+
+static const struct drm_mode_config_funcs meson_mode_config_funcs = {
+	.output_poll_changed = meson_fb_output_poll_changed,
+	.atomic_check        = drm_atomic_helper_check,
+	.atomic_commit       = drm_atomic_helper_commit,
+	.fb_create           = drm_fb_cma_create,
+};
+
+static int meson_enable_vblank(struct drm_device *dev, unsigned int crtc)
+{
+	struct meson_drm *priv = dev->dev_private;
+
+	meson_venc_enable_vsync(priv);
+
+	return 0;
+}
+
+static void meson_disable_vblank(struct drm_device *dev, unsigned int crtc)
+{
+	struct meson_drm *priv = dev->dev_private;
+
+	meson_venc_disable_vsync(priv);
+}
+
+static irqreturn_t meson_irq(int irq, void *arg)
+{
+	struct drm_device *dev = arg;
+	struct meson_drm *priv = dev->dev_private;
+
+	(void)readl_relaxed(priv->io_base + _REG(VENC_INTFLAG));
+
+	meson_crtc_irq(priv);
+
+	return IRQ_HANDLED;
+}
+
+static const struct file_operations fops = {
+	.owner		= THIS_MODULE,
+	.open		= drm_open,
+	.release	= drm_release,
+	.unlocked_ioctl	= drm_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= drm_compat_ioctl,
+#endif
+	.poll		= drm_poll,
+	.read		= drm_read,
+	.llseek		= no_llseek,
+	.mmap		= drm_gem_cma_mmap,
+};
+
+static struct drm_driver meson_driver = {
+	.driver_features	= DRIVER_HAVE_IRQ | DRIVER_GEM |
+				  DRIVER_MODESET | DRIVER_PRIME |
+				  DRIVER_ATOMIC,
+
+	/* Vblank */
+	.enable_vblank		= meson_enable_vblank,
+	.disable_vblank		= meson_disable_vblank,
+	.get_vblank_counter	= drm_vblank_no_hw_counter,
+
+	/* IRQ */
+	.irq_handler		= meson_irq,
+
+	/* PRIME Ops */
+	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
+	.gem_prime_import	= drm_gem_prime_import,
+	.gem_prime_export	= drm_gem_prime_export,
+	.gem_prime_get_sg_table	= drm_gem_cma_prime_get_sg_table,
+	.gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table,
+	.gem_prime_vmap		= drm_gem_cma_prime_vmap,
+	.gem_prime_vunmap	= drm_gem_cma_prime_vunmap,
+	.gem_prime_mmap		= drm_gem_cma_prime_mmap,
+
+	/* GEM Ops */
+	.dumb_create		= drm_gem_cma_dumb_create,
+	.dumb_destroy		= drm_gem_dumb_destroy,
+	.dumb_map_offset	= drm_gem_cma_dumb_map_offset,
+	.gem_free_object_unlocked = drm_gem_cma_free_object,
+	.gem_vm_ops		= &drm_gem_cma_vm_ops,
+
+	/* Misc */
+	.fops			= &fops,
+	.name			= DRIVER_NAME,
+	.desc			= DRIVER_DESC,
+	.date			= "20161109",
+	.major			= 1,
+	.minor			= 0,
+};
+
+static bool meson_vpu_has_available_connectors(struct device *dev)
+{
+	struct device_node *ep, *remote;
+
+	/* Parses each endpoint and check if remote exists */
+	for_each_endpoint_of_node(dev->of_node, ep) {
+		/* If the endpoint node exists, consider it enabled */
+		remote = of_graph_get_remote_port(ep);
+		if (remote)
+			return true;
+	}
+
+	return false;
+}
+
+static struct regmap_config meson_regmap_config = {
+	.reg_bits       = 32,
+	.val_bits       = 32,
+	.reg_stride     = 4,
+	.max_register   = 0x1000,
+};
+
+static int meson_drv_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct meson_drm *priv;
+	struct drm_device *drm;
+	struct resource *res;
+	void __iomem *regs;
+	int ret;
+
+	/* Checks if an output connector is available */
+	if (!meson_vpu_has_available_connectors(dev)) {
+		dev_err(dev, "No output connector available\n");
+		return -ENODEV;
+	}
+
+	drm = drm_dev_alloc(&meson_driver, dev);
+	if (IS_ERR(drm))
+		return PTR_ERR(drm);
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv) {
+		ret = -ENOMEM;
+		goto free_drm;
+	}
+	drm->dev_private = priv;
+	priv->drm = drm;
+	priv->dev = dev;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "vpu");
+	regs = devm_ioremap_resource(dev, res);
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
+
+	priv->io_base = regs;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "hhi");
+	/* Simply ioremap since it may be a shared register zone */
+	regs = devm_ioremap(dev, res->start, resource_size(res));
+	if (!regs)
+		return -EADDRNOTAVAIL;
+
+	priv->hhi = devm_regmap_init_mmio(dev, regs,
+					  &meson_regmap_config);
+	if (IS_ERR(priv->hhi)) {
+		dev_err(&pdev->dev, "Couldn't create the HHI regmap\n");
+		return PTR_ERR(priv->hhi);
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dmc");
+	/* Simply ioremap since it may be a shared register zone */
+	regs = devm_ioremap(dev, res->start, resource_size(res));
+	if (!regs)
+		return -EADDRNOTAVAIL;
+
+	priv->dmc = devm_regmap_init_mmio(dev, regs,
+					  &meson_regmap_config);
+	if (IS_ERR(priv->dmc)) {
+		dev_err(&pdev->dev, "Couldn't create the DMC regmap\n");
+		return PTR_ERR(priv->dmc);
+	}
+
+	priv->vsync_irq = platform_get_irq(pdev, 0);
+
+	drm_vblank_init(drm, 1);
+	drm_mode_config_init(drm);
+
+	/* Encoder Initialization */
+
+	ret = meson_venc_cvbs_create(priv);
+	if (ret)
+		goto free_drm;
+
+	/* Hardware Initialization */
+
+	meson_venc_init(priv);
+	meson_vpp_init(priv);
+	meson_viu_init(priv);
+
+	ret = meson_plane_create(priv);
+	if (ret)
+		goto free_drm;
+
+	ret = meson_crtc_create(priv);
+	if (ret)
+		goto free_drm;
+
+	ret = drm_irq_install(drm, priv->vsync_irq);
+	if (ret)
+		goto free_drm;
+
+	drm_mode_config_reset(drm);
+	drm->mode_config.max_width = 8192;
+	drm->mode_config.max_height = 8192;
+	drm->mode_config.funcs = &meson_mode_config_funcs;
+
+	priv->fbdev = drm_fbdev_cma_init(drm, 32,
+					 drm->mode_config.num_crtc,
+					 drm->mode_config.num_connector);
+	if (IS_ERR(priv->fbdev)) {
+		ret = PTR_ERR(priv->fbdev);
+		goto free_drm;
+	}
+
+	drm_kms_helper_poll_init(drm);
+
+	platform_set_drvdata(pdev, priv);
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		goto free_drm;
+
+	return 0;
+
+free_drm:
+	drm_dev_unref(drm);
+
+	return ret;
+}
+
+static int meson_drv_remove(struct platform_device *pdev)
+{
+	struct drm_device *drm = dev_get_drvdata(&pdev->dev);
+	struct meson_drm *priv = drm->dev_private;
+
+	drm_dev_unregister(drm);
+	drm_kms_helper_poll_fini(drm);
+	drm_fbdev_cma_fini(priv->fbdev);
+	drm_mode_config_cleanup(drm);
+	drm_vblank_cleanup(drm);
+	drm_dev_unref(drm);
+
+	return 0;
+}
+
+static const struct of_device_id dt_match[] = {
+	{ .compatible = "amlogic,meson-gxbb-vpu" },
+	{ .compatible = "amlogic,meson-gxl-vpu" },
+	{ .compatible = "amlogic,meson-gxm-vpu" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, dt_match);
+
+static struct platform_driver meson_drm_platform_driver = {
+	.probe      = meson_drv_probe,
+	.remove     = meson_drv_remove,
+	.driver     = {
+		.owner  = THIS_MODULE,
+		.name   = DRIVER_NAME,
+		.of_match_table = dt_match,
+	},
+};
+
+module_platform_driver(meson_drm_platform_driver);
+
+MODULE_AUTHOR("Jasper St. Pierre <jstpierre@mecheye.net>");
+MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/meson/meson_drv.h b/drivers/gpu/drm/meson/meson_drv.h
new file mode 100644
index 000000000000..6195327c51ca
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_drv.h
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __MESON_DRV_H
+#define __MESON_DRV_H
+
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/of.h>
+#include <drm/drmP.h>
+
+struct meson_drm {
+	struct device *dev;
+	void __iomem *io_base;
+	struct regmap *hhi;
+	struct regmap *dmc;
+	int vsync_irq;
+
+	struct drm_device *drm;
+	struct drm_crtc *crtc;
+	struct drm_fbdev_cma *fbdev;
+	struct drm_plane *primary_plane;
+
+	/* Components Data */
+	struct {
+		bool osd1_enabled;
+		bool osd1_interlace;
+		bool osd1_commit;
+		uint32_t osd1_ctrl_stat;
+		uint32_t osd1_blk0_cfg[5];
+	} viu;
+
+	struct {
+		unsigned int current_mode;
+	} venc;
+};
+
+static inline int meson_vpu_is_compatible(struct meson_drm *priv,
+					  const char *compat)
+{
+	return of_device_is_compatible(priv->dev->of_node, compat);
+}
+
+#endif /* __MESON_DRV_H */
diff --git a/drivers/gpu/drm/meson/meson_plane.c b/drivers/gpu/drm/meson/meson_plane.c
new file mode 100644
index 000000000000..4942ca090b46
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_plane.c
@@ -0,0 +1,230 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2015 Amlogic, Inc. All rights reserved.
+ * Copyright (C) 2014 Endless Mobile
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * Written by:
+ *     Jasper St. Pierre <jstpierre@mecheye.net>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_rect.h>
+
+#include "meson_plane.h"
+#include "meson_vpp.h"
+#include "meson_viu.h"
+#include "meson_canvas.h"
+#include "meson_registers.h"
+
+struct meson_plane {
+	struct drm_plane base;
+	struct meson_drm *priv;
+};
+#define to_meson_plane(x) container_of(x, struct meson_plane, base)
+
+static int meson_plane_atomic_check(struct drm_plane *plane,
+				    struct drm_plane_state *state)
+{
+	struct drm_crtc_state *crtc_state;
+	struct drm_rect clip = { 0, };
+
+	crtc_state = drm_atomic_get_crtc_state(state->state, state->crtc);
+	if (IS_ERR(crtc_state))
+		return PTR_ERR(crtc_state);
+
+	clip.x2 = crtc_state->mode.hdisplay;
+	clip.y2 = crtc_state->mode.vdisplay;
+
+	return drm_plane_helper_check_state(state, &clip,
+					    DRM_PLANE_HELPER_NO_SCALING,
+					    DRM_PLANE_HELPER_NO_SCALING,
+					    true, true);
+}
+
+/* Takes a fixed 16.16 number and converts it to integer. */
+static inline int64_t fixed16_to_int(int64_t value)
+{
+	return value >> 16;
+}
+
+static void meson_plane_atomic_update(struct drm_plane *plane,
+				      struct drm_plane_state *old_state)
+{
+	struct meson_plane *meson_plane = to_meson_plane(plane);
+	struct drm_plane_state *state = plane->state;
+	struct drm_framebuffer *fb = state->fb;
+	struct meson_drm *priv = meson_plane->priv;
+	struct drm_gem_cma_object *gem;
+	struct drm_rect src = {
+		.x1 = (state->src_x),
+		.y1 = (state->src_y),
+		.x2 = (state->src_x + state->src_w),
+		.y2 = (state->src_y + state->src_h),
+	};
+	struct drm_rect dest = {
+		.x1 = state->crtc_x,
+		.y1 = state->crtc_y,
+		.x2 = state->crtc_x + state->crtc_w,
+		.y2 = state->crtc_y + state->crtc_h,
+	};
+	unsigned long flags;
+
+	/*
+	 * Update Coordinates
+	 * Update Formats
+	 * Update Buffer
+	 * Enable Plane
+	 */
+	spin_lock_irqsave(&priv->drm->event_lock, flags);
+
+	/* Enable OSD and BLK0, set max global alpha */
+	priv->viu.osd1_ctrl_stat = OSD_ENABLE |
+				   (0xFF << OSD_GLOBAL_ALPHA_SHIFT) |
+				   OSD_BLK0_ENABLE;
+
+	/* Set up BLK0 to point to the right canvas */
+	priv->viu.osd1_blk0_cfg[0] = ((MESON_CANVAS_ID_OSD1 << OSD_CANVAS_SEL) |
+				      OSD_ENDIANNESS_LE);
+
+	/* On GXBB, Use the old non-HDR RGB2YUV converter */
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-gxbb-vpu"))
+		priv->viu.osd1_blk0_cfg[0] |= OSD_OUTPUT_COLOR_RGB;
+
+	switch (fb->pixel_format) {
+	case DRM_FORMAT_XRGB8888:
+		/* For XRGB, replace the pixel's alpha by 0xFF */
+		writel_bits_relaxed(OSD_REPLACE_EN, OSD_REPLACE_EN,
+				    priv->io_base + _REG(VIU_OSD1_CTRL_STAT2));
+		priv->viu.osd1_blk0_cfg[0] |= OSD_BLK_MODE_32 |
+					      OSD_COLOR_MATRIX_32_ARGB;
+		break;
+	case DRM_FORMAT_ARGB8888:
+		/* For ARGB, use the pixel's alpha */
+		writel_bits_relaxed(OSD_REPLACE_EN, 0,
+				    priv->io_base + _REG(VIU_OSD1_CTRL_STAT2));
+		priv->viu.osd1_blk0_cfg[0] |= OSD_BLK_MODE_32 |
+					      OSD_COLOR_MATRIX_32_ARGB;
+		break;
+	case DRM_FORMAT_RGB888:
+		priv->viu.osd1_blk0_cfg[0] |= OSD_BLK_MODE_24 |
+					      OSD_COLOR_MATRIX_24_RGB;
+		break;
+	case DRM_FORMAT_RGB565:
+		priv->viu.osd1_blk0_cfg[0] |= OSD_BLK_MODE_16 |
+					      OSD_COLOR_MATRIX_16_RGB565;
+		break;
+	};
+
+	if (state->crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) {
+		priv->viu.osd1_interlace = true;
+
+		dest.y1 /= 2;
+		dest.y2 /= 2;
+	} else
+		priv->viu.osd1_interlace = false;
+
+	/*
+	 * The format of these registers is (x2 << 16 | x1),
+	 * where x2 is exclusive.
+	 * e.g. +30x1920 would be (1919 << 16) | 30
+	 */
+	priv->viu.osd1_blk0_cfg[1] = ((fixed16_to_int(src.x2) - 1) << 16) |
+					fixed16_to_int(src.x1);
+	priv->viu.osd1_blk0_cfg[2] = ((fixed16_to_int(src.y2) - 1) << 16) |
+					fixed16_to_int(src.y1);
+	priv->viu.osd1_blk0_cfg[3] = ((dest.x2 - 1) << 16) | dest.x1;
+	priv->viu.osd1_blk0_cfg[4] = ((dest.y2 - 1) << 16) | dest.y1;
+
+	/* Update Canvas with buffer address */
+	gem = drm_fb_cma_get_gem_obj(fb, 0);
+
+	meson_canvas_setup(priv, MESON_CANVAS_ID_OSD1,
+			   gem->paddr, fb->pitches[0],
+			   fb->height, MESON_CANVAS_WRAP_NONE,
+			   MESON_CANVAS_BLKMODE_LINEAR);
+
+	spin_unlock_irqrestore(&priv->drm->event_lock, flags);
+}
+
+static void meson_plane_atomic_disable(struct drm_plane *plane,
+				       struct drm_plane_state *old_state)
+{
+	struct meson_plane *meson_plane = to_meson_plane(plane);
+	struct meson_drm *priv = meson_plane->priv;
+
+	/* Disable OSD1 */
+	writel_bits_relaxed(VPP_OSD1_POSTBLEND, 0,
+			    priv->io_base + _REG(VPP_MISC));
+
+}
+
+static const struct drm_plane_helper_funcs meson_plane_helper_funcs = {
+	.atomic_check	= meson_plane_atomic_check,
+	.atomic_disable	= meson_plane_atomic_disable,
+	.atomic_update	= meson_plane_atomic_update,
+};
+
+static const struct drm_plane_funcs meson_plane_funcs = {
+	.update_plane		= drm_atomic_helper_update_plane,
+	.disable_plane		= drm_atomic_helper_disable_plane,
+	.destroy		= drm_plane_cleanup,
+	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
+};
+
+static const uint32_t supported_drm_formats[] = {
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_RGB565,
+};
+
+int meson_plane_create(struct meson_drm *priv)
+{
+	struct meson_plane *meson_plane;
+	struct drm_plane *plane;
+
+	meson_plane = devm_kzalloc(priv->drm->dev, sizeof(*meson_plane),
+				   GFP_KERNEL);
+	if (!meson_plane)
+		return -ENOMEM;
+
+	meson_plane->priv = priv;
+	plane = &meson_plane->base;
+
+	drm_universal_plane_init(priv->drm, plane, 0xFF,
+				 &meson_plane_funcs,
+				 supported_drm_formats,
+				 ARRAY_SIZE(supported_drm_formats),
+				 DRM_PLANE_TYPE_PRIMARY, "meson_primary_plane");
+
+	drm_plane_helper_add(plane, &meson_plane_helper_funcs);
+
+	priv->primary_plane = plane;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/meson/meson_plane.h b/drivers/gpu/drm/meson/meson_plane.h
new file mode 100644
index 000000000000..e26b8b0aa1fa
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_plane.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2014 Endless Mobile
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * Written by:
+ *     Jasper St. Pierre <jstpierre@mecheye.net>
+ */
+
+#ifndef __MESON_PLANE_H
+#define __MESON_PLANE_H
+
+#include "meson_drv.h"
+
+int meson_plane_create(struct meson_drm *priv);
+
+#endif /* __MESON_PLANE_H */
diff --git a/drivers/gpu/drm/meson/meson_registers.h b/drivers/gpu/drm/meson/meson_registers.h
new file mode 100644
index 000000000000..6adf9c13fafa
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_registers.h
@@ -0,0 +1,1395 @@
+/*
+ * Copyright (C) 2015 Amlogic, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ */
+
+#ifndef __MESON_REGISTERS_H
+#define __MESON_REGISTERS_H
+
+/* Shift all registers by 2 */
+#define _REG(reg)	((reg) << 2)
+
+#define writel_bits_relaxed(mask, val, addr) \
+	writel_relaxed((readl_relaxed(addr) & ~(mask)) | (val), addr)
+
+/* vpp2 */
+#define VPP2_DUMMY_DATA 0x1900
+#define VPP2_LINE_IN_LENGTH 0x1901
+#define VPP2_PIC_IN_HEIGHT 0x1902
+#define VPP2_SCALE_COEF_IDX 0x1903
+#define VPP2_SCALE_COEF 0x1904
+#define VPP2_VSC_REGION12_STARTP 0x1905
+#define VPP2_VSC_REGION34_STARTP 0x1906
+#define VPP2_VSC_REGION4_ENDP 0x1907
+#define VPP2_VSC_START_PHASE_STEP 0x1908
+#define VPP2_VSC_REGION0_PHASE_SLOPE 0x1909
+#define VPP2_VSC_REGION1_PHASE_SLOPE 0x190a
+#define VPP2_VSC_REGION3_PHASE_SLOPE 0x190b
+#define VPP2_VSC_REGION4_PHASE_SLOPE 0x190c
+#define VPP2_VSC_PHASE_CTRL 0x190d
+#define VPP2_VSC_INI_PHASE 0x190e
+#define VPP2_HSC_REGION12_STARTP 0x1910
+#define VPP2_HSC_REGION34_STARTP 0x1911
+#define VPP2_HSC_REGION4_ENDP 0x1912
+#define VPP2_HSC_START_PHASE_STEP 0x1913
+#define VPP2_HSC_REGION0_PHASE_SLOPE 0x1914
+#define VPP2_HSC_REGION1_PHASE_SLOPE 0x1915
+#define VPP2_HSC_REGION3_PHASE_SLOPE 0x1916
+#define VPP2_HSC_REGION4_PHASE_SLOPE 0x1917
+#define VPP2_HSC_PHASE_CTRL 0x1918
+#define VPP2_SC_MISC 0x1919
+#define VPP2_PREBLEND_VD1_H_START_END 0x191a
+#define VPP2_PREBLEND_VD1_V_START_END 0x191b
+#define VPP2_POSTBLEND_VD1_H_START_END 0x191c
+#define VPP2_POSTBLEND_VD1_V_START_END 0x191d
+#define VPP2_PREBLEND_H_SIZE 0x1920
+#define VPP2_POSTBLEND_H_SIZE 0x1921
+#define VPP2_HOLD_LINES 0x1922
+#define VPP2_BLEND_ONECOLOR_CTRL 0x1923
+#define VPP2_PREBLEND_CURRENT_XY 0x1924
+#define VPP2_POSTBLEND_CURRENT_XY 0x1925
+#define VPP2_MISC 0x1926
+#define VPP2_OFIFO_SIZE 0x1927
+#define VPP2_FIFO_STATUS 0x1928
+#define VPP2_SMOKE_CTRL 0x1929
+#define VPP2_SMOKE1_VAL 0x192a
+#define VPP2_SMOKE2_VAL 0x192b
+#define VPP2_SMOKE1_H_START_END 0x192d
+#define VPP2_SMOKE1_V_START_END 0x192e
+#define VPP2_SMOKE2_H_START_END 0x192f
+#define VPP2_SMOKE2_V_START_END 0x1930
+#define VPP2_SCO_FIFO_CTRL 0x1933
+#define VPP2_HSC_PHASE_CTRL1 0x1934
+#define VPP2_HSC_INI_PAT_CTRL 0x1935
+#define VPP2_VADJ_CTRL 0x1940
+#define VPP2_VADJ1_Y 0x1941
+#define VPP2_VADJ1_MA_MB 0x1942
+#define VPP2_VADJ1_MC_MD 0x1943
+#define VPP2_VADJ2_Y 0x1944
+#define VPP2_VADJ2_MA_MB 0x1945
+#define VPP2_VADJ2_MC_MD 0x1946
+#define VPP2_MATRIX_PROBE_COLOR 0x195c
+#define VPP2_MATRIX_HL_COLOR 0x195d
+#define VPP2_MATRIX_PROBE_POS 0x195e
+#define VPP2_MATRIX_CTRL 0x195f
+#define VPP2_MATRIX_COEF00_01 0x1960
+#define VPP2_MATRIX_COEF02_10 0x1961
+#define VPP2_MATRIX_COEF11_12 0x1962
+#define VPP2_MATRIX_COEF20_21 0x1963
+#define VPP2_MATRIX_COEF22 0x1964
+#define VPP2_MATRIX_OFFSET0_1 0x1965
+#define VPP2_MATRIX_OFFSET2 0x1966
+#define VPP2_MATRIX_PRE_OFFSET0_1 0x1967
+#define VPP2_MATRIX_PRE_OFFSET2 0x1968
+#define VPP2_DUMMY_DATA1 0x1969
+#define VPP2_GAINOFF_CTRL0 0x196a
+#define VPP2_GAINOFF_CTRL1 0x196b
+#define VPP2_GAINOFF_CTRL2 0x196c
+#define VPP2_GAINOFF_CTRL3 0x196d
+#define VPP2_GAINOFF_CTRL4 0x196e
+#define VPP2_CHROMA_ADDR_PORT 0x1970
+#define VPP2_CHROMA_DATA_PORT 0x1971
+#define VPP2_GCLK_CTRL0 0x1972
+#define VPP2_GCLK_CTRL1 0x1973
+#define VPP2_SC_GCLK_CTRL 0x1974
+#define VPP2_MISC1 0x1976
+#define VPP2_DNLP_CTRL_00 0x1981
+#define VPP2_DNLP_CTRL_01 0x1982
+#define VPP2_DNLP_CTRL_02 0x1983
+#define VPP2_DNLP_CTRL_03 0x1984
+#define VPP2_DNLP_CTRL_04 0x1985
+#define VPP2_DNLP_CTRL_05 0x1986
+#define VPP2_DNLP_CTRL_06 0x1987
+#define VPP2_DNLP_CTRL_07 0x1988
+#define VPP2_DNLP_CTRL_08 0x1989
+#define VPP2_DNLP_CTRL_09 0x198a
+#define VPP2_DNLP_CTRL_10 0x198b
+#define VPP2_DNLP_CTRL_11 0x198c
+#define VPP2_DNLP_CTRL_12 0x198d
+#define VPP2_DNLP_CTRL_13 0x198e
+#define VPP2_DNLP_CTRL_14 0x198f
+#define VPP2_DNLP_CTRL_15 0x1990
+#define VPP2_VE_ENABLE_CTRL 0x19a1
+#define VPP2_VE_DEMO_LEFT_TOP_SCREEN_WIDTH 0x19a2
+#define VPP2_VE_DEMO_CENTER_BAR 0x19a3
+#define VPP2_VE_H_V_SIZE 0x19a4
+#define VPP2_VDO_MEAS_CTRL 0x19a8
+#define VPP2_VDO_MEAS_VS_COUNT_HI 0x19a9
+#define VPP2_VDO_MEAS_VS_COUNT_LO 0x19aa
+#define VPP2_OSD_VSC_PHASE_STEP 0x19c0
+#define VPP2_OSD_VSC_INI_PHASE 0x19c1
+#define VPP2_OSD_VSC_CTRL0 0x19c2
+#define VPP2_OSD_HSC_PHASE_STEP 0x19c3
+#define VPP2_OSD_HSC_INI_PHASE 0x19c4
+#define VPP2_OSD_HSC_CTRL0 0x19c5
+#define VPP2_OSD_HSC_INI_PAT_CTRL 0x19c6
+#define VPP2_OSD_SC_DUMMY_DATA 0x19c7
+#define VPP2_OSD_SC_CTRL0 0x19c8
+#define VPP2_OSD_SCI_WH_M1 0x19c9
+#define VPP2_OSD_SCO_H_START_END 0x19ca
+#define VPP2_OSD_SCO_V_START_END 0x19cb
+#define VPP2_OSD_SCALE_COEF_IDX 0x19cc
+#define VPP2_OSD_SCALE_COEF 0x19cd
+#define VPP2_INT_LINE_NUM 0x19ce
+
+/* viu */
+#define VIU_ADDR_START 0x1a00
+#define VIU_ADDR_END 0x1aff
+#define VIU_SW_RESET 0x1a01
+#define VIU_MISC_CTRL0 0x1a06
+#define VIU_MISC_CTRL1 0x1a07
+#define D2D3_INTF_LENGTH 0x1a08
+#define D2D3_INTF_CTRL0 0x1a09
+#define VIU_OSD1_CTRL_STAT 0x1a10
+#define VIU_OSD1_CTRL_STAT2 0x1a2d
+#define VIU_OSD1_COLOR_ADDR 0x1a11
+#define VIU_OSD1_COLOR 0x1a12
+#define VIU_OSD1_TCOLOR_AG0 0x1a17
+#define VIU_OSD1_TCOLOR_AG1 0x1a18
+#define VIU_OSD1_TCOLOR_AG2 0x1a19
+#define VIU_OSD1_TCOLOR_AG3 0x1a1a
+#define VIU_OSD1_BLK0_CFG_W0 0x1a1b
+#define VIU_OSD1_BLK1_CFG_W0 0x1a1f
+#define VIU_OSD1_BLK2_CFG_W0 0x1a23
+#define VIU_OSD1_BLK3_CFG_W0 0x1a27
+#define VIU_OSD1_BLK0_CFG_W1 0x1a1c
+#define VIU_OSD1_BLK1_CFG_W1 0x1a20
+#define VIU_OSD1_BLK2_CFG_W1 0x1a24
+#define VIU_OSD1_BLK3_CFG_W1 0x1a28
+#define VIU_OSD1_BLK0_CFG_W2 0x1a1d
+#define VIU_OSD1_BLK1_CFG_W2 0x1a21
+#define VIU_OSD1_BLK2_CFG_W2 0x1a25
+#define VIU_OSD1_BLK3_CFG_W2 0x1a29
+#define VIU_OSD1_BLK0_CFG_W3 0x1a1e
+#define VIU_OSD1_BLK1_CFG_W3 0x1a22
+#define VIU_OSD1_BLK2_CFG_W3 0x1a26
+#define VIU_OSD1_BLK3_CFG_W3 0x1a2a
+#define VIU_OSD1_BLK0_CFG_W4 0x1a13
+#define VIU_OSD1_BLK1_CFG_W4 0x1a14
+#define VIU_OSD1_BLK2_CFG_W4 0x1a15
+#define VIU_OSD1_BLK3_CFG_W4 0x1a16
+#define VIU_OSD1_FIFO_CTRL_STAT 0x1a2b
+#define VIU_OSD1_TEST_RDDATA 0x1a2c
+#define VIU_OSD1_PROT_CTRL 0x1a2e
+#define VIU_OSD2_CTRL_STAT 0x1a30
+#define VIU_OSD2_CTRL_STAT2 0x1a4d
+#define VIU_OSD2_COLOR_ADDR 0x1a31
+#define VIU_OSD2_COLOR 0x1a32
+#define VIU_OSD2_HL1_H_START_END 0x1a33
+#define VIU_OSD2_HL1_V_START_END 0x1a34
+#define VIU_OSD2_HL2_H_START_END 0x1a35
+#define VIU_OSD2_HL2_V_START_END 0x1a36
+#define VIU_OSD2_TCOLOR_AG0 0x1a37
+#define VIU_OSD2_TCOLOR_AG1 0x1a38
+#define VIU_OSD2_TCOLOR_AG2 0x1a39
+#define VIU_OSD2_TCOLOR_AG3 0x1a3a
+#define VIU_OSD2_BLK0_CFG_W0 0x1a3b
+#define VIU_OSD2_BLK1_CFG_W0 0x1a3f
+#define VIU_OSD2_BLK2_CFG_W0 0x1a43
+#define VIU_OSD2_BLK3_CFG_W0 0x1a47
+#define VIU_OSD2_BLK0_CFG_W1 0x1a3c
+#define VIU_OSD2_BLK1_CFG_W1 0x1a40
+#define VIU_OSD2_BLK2_CFG_W1 0x1a44
+#define VIU_OSD2_BLK3_CFG_W1 0x1a48
+#define VIU_OSD2_BLK0_CFG_W2 0x1a3d
+#define VIU_OSD2_BLK1_CFG_W2 0x1a41
+#define VIU_OSD2_BLK2_CFG_W2 0x1a45
+#define VIU_OSD2_BLK3_CFG_W2 0x1a49
+#define VIU_OSD2_BLK0_CFG_W3 0x1a3e
+#define VIU_OSD2_BLK1_CFG_W3 0x1a42
+#define VIU_OSD2_BLK2_CFG_W3 0x1a46
+#define VIU_OSD2_BLK3_CFG_W3 0x1a4a
+#define VIU_OSD2_BLK0_CFG_W4 0x1a64
+#define VIU_OSD2_BLK1_CFG_W4 0x1a65
+#define VIU_OSD2_BLK2_CFG_W4 0x1a66
+#define VIU_OSD2_BLK3_CFG_W4 0x1a67
+#define VIU_OSD2_FIFO_CTRL_STAT 0x1a4b
+#define VIU_OSD2_TEST_RDDATA 0x1a4c
+#define VIU_OSD2_PROT_CTRL 0x1a4e
+
+#define VD1_IF0_GEN_REG 0x1a50
+#define VD1_IF0_CANVAS0 0x1a51
+#define VD1_IF0_CANVAS1 0x1a52
+#define VD1_IF0_LUMA_X0 0x1a53
+#define VD1_IF0_LUMA_Y0 0x1a54
+#define VD1_IF0_CHROMA_X0 0x1a55
+#define VD1_IF0_CHROMA_Y0 0x1a56
+#define VD1_IF0_LUMA_X1 0x1a57
+#define VD1_IF0_LUMA_Y1 0x1a58
+#define VD1_IF0_CHROMA_X1 0x1a59
+#define VD1_IF0_CHROMA_Y1 0x1a5a
+#define VD1_IF0_RPT_LOOP 0x1a5b
+#define VD1_IF0_LUMA0_RPT_PAT 0x1a5c
+#define VD1_IF0_CHROMA0_RPT_PAT 0x1a5d
+#define VD1_IF0_LUMA1_RPT_PAT 0x1a5e
+#define VD1_IF0_CHROMA1_RPT_PAT 0x1a5f
+#define VD1_IF0_LUMA_PSEL 0x1a60
+#define VD1_IF0_CHROMA_PSEL 0x1a61
+#define VD1_IF0_DUMMY_PIXEL 0x1a62
+#define VD1_IF0_LUMA_FIFO_SIZE 0x1a63
+#define VD1_IF0_RANGE_MAP_Y 0x1a6a
+#define VD1_IF0_RANGE_MAP_CB 0x1a6b
+#define VD1_IF0_RANGE_MAP_CR 0x1a6c
+#define VD1_IF0_GEN_REG2 0x1a6d
+#define VD1_IF0_PROT_CNTL 0x1a6e
+#define VIU_VD1_FMT_CTRL 0x1a68
+#define VIU_VD1_FMT_W 0x1a69
+#define VD2_IF0_GEN_REG 0x1a70
+#define VD2_IF0_CANVAS0 0x1a71
+#define VD2_IF0_CANVAS1 0x1a72
+#define VD2_IF0_LUMA_X0 0x1a73
+#define VD2_IF0_LUMA_Y0 0x1a74
+#define VD2_IF0_CHROMA_X0 0x1a75
+#define VD2_IF0_CHROMA_Y0 0x1a76
+#define VD2_IF0_LUMA_X1 0x1a77
+#define VD2_IF0_LUMA_Y1 0x1a78
+#define VD2_IF0_CHROMA_X1 0x1a79
+#define VD2_IF0_CHROMA_Y1 0x1a7a
+#define VD2_IF0_RPT_LOOP 0x1a7b
+#define VD2_IF0_LUMA0_RPT_PAT 0x1a7c
+#define VD2_IF0_CHROMA0_RPT_PAT 0x1a7d
+#define VD2_IF0_LUMA1_RPT_PAT 0x1a7e
+#define VD2_IF0_CHROMA1_RPT_PAT 0x1a7f
+#define VD2_IF0_LUMA_PSEL 0x1a80
+#define VD2_IF0_CHROMA_PSEL 0x1a81
+#define VD2_IF0_DUMMY_PIXEL 0x1a82
+#define VD2_IF0_LUMA_FIFO_SIZE 0x1a83
+#define VD2_IF0_RANGE_MAP_Y 0x1a8a
+#define VD2_IF0_RANGE_MAP_CB 0x1a8b
+#define VD2_IF0_RANGE_MAP_CR 0x1a8c
+#define VD2_IF0_GEN_REG2 0x1a8d
+#define VD2_IF0_PROT_CNTL 0x1a8e
+#define VIU_VD2_FMT_CTRL 0x1a88
+#define VIU_VD2_FMT_W 0x1a89
+
+/* VIU Matrix Registers */
+#define VIU_OSD1_MATRIX_CTRL 0x1a90
+#define VIU_OSD1_MATRIX_COEF00_01 0x1a91
+#define VIU_OSD1_MATRIX_COEF02_10 0x1a92
+#define VIU_OSD1_MATRIX_COEF11_12 0x1a93
+#define VIU_OSD1_MATRIX_COEF20_21 0x1a94
+#define VIU_OSD1_MATRIX_COLMOD_COEF42 0x1a95
+#define VIU_OSD1_MATRIX_OFFSET0_1 0x1a96
+#define VIU_OSD1_MATRIX_OFFSET2 0x1a97
+#define VIU_OSD1_MATRIX_PRE_OFFSET0_1 0x1a98
+#define VIU_OSD1_MATRIX_PRE_OFFSET2 0x1a99
+#define VIU_OSD1_MATRIX_COEF22_30 0x1a9d
+#define VIU_OSD1_MATRIX_COEF31_32 0x1a9e
+#define VIU_OSD1_MATRIX_COEF40_41 0x1a9f
+#define VIU_OSD1_EOTF_CTL 0x1ad4
+#define VIU_OSD1_EOTF_COEF00_01 0x1ad5
+#define VIU_OSD1_EOTF_COEF02_10 0x1ad6
+#define VIU_OSD1_EOTF_COEF11_12 0x1ad7
+#define VIU_OSD1_EOTF_COEF20_21 0x1ad8
+#define VIU_OSD1_EOTF_COEF22_RS 0x1ad9
+#define VIU_OSD1_EOTF_LUT_ADDR_PORT 0x1ada
+#define VIU_OSD1_EOTF_LUT_DATA_PORT 0x1adb
+#define VIU_OSD1_OETF_CTL 0x1adc
+#define VIU_OSD1_OETF_LUT_ADDR_PORT 0x1add
+#define VIU_OSD1_OETF_LUT_DATA_PORT 0x1ade
+
+/* vpp */
+#define VPP_DUMMY_DATA 0x1d00
+#define VPP_LINE_IN_LENGTH 0x1d01
+#define VPP_PIC_IN_HEIGHT 0x1d02
+#define VPP_SCALE_COEF_IDX 0x1d03
+#define VPP_SCALE_COEF 0x1d04
+#define VPP_VSC_REGION12_STARTP 0x1d05
+#define VPP_VSC_REGION34_STARTP 0x1d06
+#define VPP_VSC_REGION4_ENDP 0x1d07
+#define VPP_VSC_START_PHASE_STEP 0x1d08
+#define VPP_VSC_REGION0_PHASE_SLOPE 0x1d09
+#define VPP_VSC_REGION1_PHASE_SLOPE 0x1d0a
+#define VPP_VSC_REGION3_PHASE_SLOPE 0x1d0b
+#define VPP_VSC_REGION4_PHASE_SLOPE 0x1d0c
+#define VPP_VSC_PHASE_CTRL 0x1d0d
+#define VPP_VSC_INI_PHASE 0x1d0e
+#define VPP_HSC_REGION12_STARTP 0x1d10
+#define VPP_HSC_REGION34_STARTP 0x1d11
+#define VPP_HSC_REGION4_ENDP 0x1d12
+#define VPP_HSC_START_PHASE_STEP 0x1d13
+#define VPP_HSC_REGION0_PHASE_SLOPE 0x1d14
+#define VPP_HSC_REGION1_PHASE_SLOPE 0x1d15
+#define VPP_HSC_REGION3_PHASE_SLOPE 0x1d16
+#define VPP_HSC_REGION4_PHASE_SLOPE 0x1d17
+#define VPP_HSC_PHASE_CTRL 0x1d18
+#define VPP_SC_MISC 0x1d19
+#define VPP_PREBLEND_VD1_H_START_END 0x1d1a
+#define VPP_PREBLEND_VD1_V_START_END 0x1d1b
+#define VPP_POSTBLEND_VD1_H_START_END 0x1d1c
+#define VPP_POSTBLEND_VD1_V_START_END 0x1d1d
+#define VPP_BLEND_VD2_H_START_END 0x1d1e
+#define VPP_BLEND_VD2_V_START_END 0x1d1f
+#define VPP_PREBLEND_H_SIZE 0x1d20
+#define VPP_POSTBLEND_H_SIZE 0x1d21
+#define VPP_HOLD_LINES 0x1d22
+#define VPP_BLEND_ONECOLOR_CTRL 0x1d23
+#define VPP_PREBLEND_CURRENT_XY 0x1d24
+#define VPP_POSTBLEND_CURRENT_XY 0x1d25
+#define VPP_MISC 0x1d26
+#define		VPP_PREBLEND_ENABLE	BIT(6)
+#define		VPP_POSTBLEND_ENABLE	BIT(7)
+#define		VPP_OSD2_ALPHA_PREMULT	BIT(8)
+#define		VPP_OSD1_ALPHA_PREMULT	BIT(9)
+#define		VPP_VD1_POSTBLEND	BIT(10)
+#define		VPP_VD2_POSTBLEND	BIT(11)
+#define		VPP_OSD1_POSTBLEND	BIT(12)
+#define		VPP_OSD2_POSTBLEND	BIT(13)
+#define		VPP_VD1_PREBLEND	BIT(14)
+#define		VPP_VD2_PREBLEND	BIT(15)
+#define		VPP_OSD1_PREBLEND	BIT(16)
+#define		VPP_OSD2_PREBLEND	BIT(17)
+#define VPP_OFIFO_SIZE 0x1d27
+#define VPP_FIFO_STATUS 0x1d28
+#define VPP_SMOKE_CTRL 0x1d29
+#define VPP_SMOKE1_VAL 0x1d2a
+#define VPP_SMOKE2_VAL 0x1d2b
+#define VPP_SMOKE3_VAL 0x1d2c
+#define VPP_SMOKE1_H_START_END 0x1d2d
+#define VPP_SMOKE1_V_START_END 0x1d2e
+#define VPP_SMOKE2_H_START_END 0x1d2f
+#define VPP_SMOKE2_V_START_END 0x1d30
+#define VPP_SMOKE3_H_START_END 0x1d31
+#define VPP_SMOKE3_V_START_END 0x1d32
+#define VPP_SCO_FIFO_CTRL 0x1d33
+#define VPP_HSC_PHASE_CTRL1 0x1d34
+#define VPP_HSC_INI_PAT_CTRL 0x1d35
+#define VPP_VADJ_CTRL 0x1d40
+#define VPP_VADJ1_Y 0x1d41
+#define VPP_VADJ1_MA_MB 0x1d42
+#define VPP_VADJ1_MC_MD 0x1d43
+#define VPP_VADJ2_Y 0x1d44
+#define VPP_VADJ2_MA_MB 0x1d45
+#define VPP_VADJ2_MC_MD 0x1d46
+#define VPP_HSHARP_CTRL 0x1d50
+#define VPP_HSHARP_LUMA_THRESH01 0x1d51
+#define VPP_HSHARP_LUMA_THRESH23 0x1d52
+#define VPP_HSHARP_CHROMA_THRESH01 0x1d53
+#define VPP_HSHARP_CHROMA_THRESH23 0x1d54
+#define VPP_HSHARP_LUMA_GAIN 0x1d55
+#define VPP_HSHARP_CHROMA_GAIN 0x1d56
+#define VPP_MATRIX_PROBE_COLOR 0x1d5c
+#define VPP_MATRIX_HL_COLOR 0x1d5d
+#define VPP_MATRIX_PROBE_POS 0x1d5e
+#define VPP_MATRIX_CTRL 0x1d5f
+#define VPP_MATRIX_COEF00_01 0x1d60
+#define VPP_MATRIX_COEF02_10 0x1d61
+#define VPP_MATRIX_COEF11_12 0x1d62
+#define VPP_MATRIX_COEF20_21 0x1d63
+#define VPP_MATRIX_COEF22 0x1d64
+#define VPP_MATRIX_OFFSET0_1 0x1d65
+#define VPP_MATRIX_OFFSET2 0x1d66
+#define VPP_MATRIX_PRE_OFFSET0_1 0x1d67
+#define VPP_MATRIX_PRE_OFFSET2 0x1d68
+#define VPP_DUMMY_DATA1 0x1d69
+#define VPP_GAINOFF_CTRL0 0x1d6a
+#define VPP_GAINOFF_CTRL1 0x1d6b
+#define VPP_GAINOFF_CTRL2 0x1d6c
+#define VPP_GAINOFF_CTRL3 0x1d6d
+#define VPP_GAINOFF_CTRL4 0x1d6e
+#define VPP_CHROMA_ADDR_PORT 0x1d70
+#define VPP_CHROMA_DATA_PORT 0x1d71
+#define VPP_GCLK_CTRL0 0x1d72
+#define VPP_GCLK_CTRL1 0x1d73
+#define VPP_SC_GCLK_CTRL 0x1d74
+#define VPP_MISC1 0x1d76
+#define VPP_BLACKEXT_CTRL 0x1d80
+#define VPP_DNLP_CTRL_00 0x1d81
+#define VPP_DNLP_CTRL_01 0x1d82
+#define VPP_DNLP_CTRL_02 0x1d83
+#define VPP_DNLP_CTRL_03 0x1d84
+#define VPP_DNLP_CTRL_04 0x1d85
+#define VPP_DNLP_CTRL_05 0x1d86
+#define VPP_DNLP_CTRL_06 0x1d87
+#define VPP_DNLP_CTRL_07 0x1d88
+#define VPP_DNLP_CTRL_08 0x1d89
+#define VPP_DNLP_CTRL_09 0x1d8a
+#define VPP_DNLP_CTRL_10 0x1d8b
+#define VPP_DNLP_CTRL_11 0x1d8c
+#define VPP_DNLP_CTRL_12 0x1d8d
+#define VPP_DNLP_CTRL_13 0x1d8e
+#define VPP_DNLP_CTRL_14 0x1d8f
+#define VPP_DNLP_CTRL_15 0x1d90
+#define VPP_PEAKING_HGAIN 0x1d91
+#define VPP_PEAKING_VGAIN 0x1d92
+#define VPP_PEAKING_NLP_1 0x1d93
+#define VPP_DOLBY_CTRL 0x1d93
+#define VPP_PEAKING_NLP_2 0x1d94
+#define VPP_PEAKING_NLP_3 0x1d95
+#define VPP_PEAKING_NLP_4 0x1d96
+#define VPP_PEAKING_NLP_5 0x1d97
+#define VPP_SHARP_LIMIT 0x1d98
+#define VPP_VLTI_CTRL 0x1d99
+#define VPP_HLTI_CTRL 0x1d9a
+#define VPP_CTI_CTRL 0x1d9b
+#define VPP_BLUE_STRETCH_1 0x1d9c
+#define VPP_BLUE_STRETCH_2 0x1d9d
+#define VPP_BLUE_STRETCH_3 0x1d9e
+#define VPP_CCORING_CTRL 0x1da0
+#define VPP_VE_ENABLE_CTRL 0x1da1
+#define VPP_VE_DEMO_LEFT_TOP_SCREEN_WIDTH 0x1da2
+#define VPP_VE_DEMO_CENTER_BAR 0x1da3
+#define VPP_VE_H_V_SIZE 0x1da4
+#define VPP_VDO_MEAS_CTRL 0x1da8
+#define VPP_VDO_MEAS_VS_COUNT_HI 0x1da9
+#define VPP_VDO_MEAS_VS_COUNT_LO 0x1daa
+#define VPP_INPUT_CTRL 0x1dab
+#define VPP_CTI_CTRL2 0x1dac
+#define VPP_PEAKING_SAT_THD1 0x1dad
+#define VPP_PEAKING_SAT_THD2 0x1dae
+#define VPP_PEAKING_SAT_THD3 0x1daf
+#define VPP_PEAKING_SAT_THD4 0x1db0
+#define VPP_PEAKING_SAT_THD5 0x1db1
+#define VPP_PEAKING_SAT_THD6 0x1db2
+#define VPP_PEAKING_SAT_THD7 0x1db3
+#define VPP_PEAKING_SAT_THD8 0x1db4
+#define VPP_PEAKING_SAT_THD9 0x1db5
+#define VPP_PEAKING_GAIN_ADD1 0x1db6
+#define VPP_PEAKING_GAIN_ADD2 0x1db7
+#define VPP_PEAKING_DNLP 0x1db8
+#define VPP_SHARP_DEMO_WIN_CTRL1 0x1db9
+#define VPP_SHARP_DEMO_WIN_CTRL2 0x1dba
+#define VPP_FRONT_HLTI_CTRL 0x1dbb
+#define VPP_FRONT_CTI_CTRL 0x1dbc
+#define VPP_FRONT_CTI_CTRL2 0x1dbd
+#define VPP_OSD_VSC_PHASE_STEP 0x1dc0
+#define VPP_OSD_VSC_INI_PHASE 0x1dc1
+#define VPP_OSD_VSC_CTRL0 0x1dc2
+#define VPP_OSD_HSC_PHASE_STEP 0x1dc3
+#define VPP_OSD_HSC_INI_PHASE 0x1dc4
+#define VPP_OSD_HSC_CTRL0 0x1dc5
+#define VPP_OSD_HSC_INI_PAT_CTRL 0x1dc6
+#define VPP_OSD_SC_DUMMY_DATA 0x1dc7
+#define VPP_OSD_SC_CTRL0 0x1dc8
+#define VPP_OSD_SCI_WH_M1 0x1dc9
+#define VPP_OSD_SCO_H_START_END 0x1dca
+#define VPP_OSD_SCO_V_START_END 0x1dcb
+#define VPP_OSD_SCALE_COEF_IDX 0x1dcc
+#define VPP_OSD_SCALE_COEF 0x1dcd
+#define VPP_INT_LINE_NUM 0x1dce
+
+/* viu2 */
+#define VIU2_ADDR_START 0x1e00
+#define VIU2_ADDR_END 0x1eff
+#define VIU2_SW_RESET 0x1e01
+#define VIU2_OSD1_CTRL_STAT 0x1e10
+#define VIU2_OSD1_CTRL_STAT2 0x1e2d
+#define VIU2_OSD1_COLOR_ADDR 0x1e11
+#define VIU2_OSD1_COLOR 0x1e12
+#define VIU2_OSD1_TCOLOR_AG0 0x1e17
+#define VIU2_OSD1_TCOLOR_AG1 0x1e18
+#define VIU2_OSD1_TCOLOR_AG2 0x1e19
+#define VIU2_OSD1_TCOLOR_AG3 0x1e1a
+#define VIU2_OSD1_BLK0_CFG_W0 0x1e1b
+#define VIU2_OSD1_BLK1_CFG_W0 0x1e1f
+#define VIU2_OSD1_BLK2_CFG_W0 0x1e23
+#define VIU2_OSD1_BLK3_CFG_W0 0x1e27
+#define VIU2_OSD1_BLK0_CFG_W1 0x1e1c
+#define VIU2_OSD1_BLK1_CFG_W1 0x1e20
+#define VIU2_OSD1_BLK2_CFG_W1 0x1e24
+#define VIU2_OSD1_BLK3_CFG_W1 0x1e28
+#define VIU2_OSD1_BLK0_CFG_W2 0x1e1d
+#define VIU2_OSD1_BLK1_CFG_W2 0x1e21
+#define VIU2_OSD1_BLK2_CFG_W2 0x1e25
+#define VIU2_OSD1_BLK3_CFG_W2 0x1e29
+#define VIU2_OSD1_BLK0_CFG_W3 0x1e1e
+#define VIU2_OSD1_BLK1_CFG_W3 0x1e22
+#define VIU2_OSD1_BLK2_CFG_W3 0x1e26
+#define VIU2_OSD1_BLK3_CFG_W3 0x1e2a
+#define VIU2_OSD1_BLK0_CFG_W4 0x1e13
+#define VIU2_OSD1_BLK1_CFG_W4 0x1e14
+#define VIU2_OSD1_BLK2_CFG_W4 0x1e15
+#define VIU2_OSD1_BLK3_CFG_W4 0x1e16
+#define VIU2_OSD1_FIFO_CTRL_STAT 0x1e2b
+#define VIU2_OSD1_TEST_RDDATA 0x1e2c
+#define VIU2_OSD1_PROT_CTRL 0x1e2e
+#define VIU2_OSD2_CTRL_STAT 0x1e30
+#define VIU2_OSD2_CTRL_STAT2 0x1e4d
+#define VIU2_OSD2_COLOR_ADDR 0x1e31
+#define VIU2_OSD2_COLOR 0x1e32
+#define VIU2_OSD2_HL1_H_START_END 0x1e33
+#define VIU2_OSD2_HL1_V_START_END 0x1e34
+#define VIU2_OSD2_HL2_H_START_END 0x1e35
+#define VIU2_OSD2_HL2_V_START_END 0x1e36
+#define VIU2_OSD2_TCOLOR_AG0 0x1e37
+#define VIU2_OSD2_TCOLOR_AG1 0x1e38
+#define VIU2_OSD2_TCOLOR_AG2 0x1e39
+#define VIU2_OSD2_TCOLOR_AG3 0x1e3a
+#define VIU2_OSD2_BLK0_CFG_W0 0x1e3b
+#define VIU2_OSD2_BLK1_CFG_W0 0x1e3f
+#define VIU2_OSD2_BLK2_CFG_W0 0x1e43
+#define VIU2_OSD2_BLK3_CFG_W0 0x1e47
+#define VIU2_OSD2_BLK0_CFG_W1 0x1e3c
+#define VIU2_OSD2_BLK1_CFG_W1 0x1e40
+#define VIU2_OSD2_BLK2_CFG_W1 0x1e44
+#define VIU2_OSD2_BLK3_CFG_W1 0x1e48
+#define VIU2_OSD2_BLK0_CFG_W2 0x1e3d
+#define VIU2_OSD2_BLK1_CFG_W2 0x1e41
+#define VIU2_OSD2_BLK2_CFG_W2 0x1e45
+#define VIU2_OSD2_BLK3_CFG_W2 0x1e49
+#define VIU2_OSD2_BLK0_CFG_W3 0x1e3e
+#define VIU2_OSD2_BLK1_CFG_W3 0x1e42
+#define VIU2_OSD2_BLK2_CFG_W3 0x1e46
+#define VIU2_OSD2_BLK3_CFG_W3 0x1e4a
+#define VIU2_OSD2_BLK0_CFG_W4 0x1e64
+#define VIU2_OSD2_BLK1_CFG_W4 0x1e65
+#define VIU2_OSD2_BLK2_CFG_W4 0x1e66
+#define VIU2_OSD2_BLK3_CFG_W4 0x1e67
+#define VIU2_OSD2_FIFO_CTRL_STAT 0x1e4b
+#define VIU2_OSD2_TEST_RDDATA 0x1e4c
+#define VIU2_OSD2_PROT_CTRL 0x1e4e
+#define VIU2_VD1_IF0_GEN_REG 0x1e50
+#define VIU2_VD1_IF0_CANVAS0 0x1e51
+#define VIU2_VD1_IF0_CANVAS1 0x1e52
+#define VIU2_VD1_IF0_LUMA_X0 0x1e53
+#define VIU2_VD1_IF0_LUMA_Y0 0x1e54
+#define VIU2_VD1_IF0_CHROMA_X0 0x1e55
+#define VIU2_VD1_IF0_CHROMA_Y0 0x1e56
+#define VIU2_VD1_IF0_LUMA_X1 0x1e57
+#define VIU2_VD1_IF0_LUMA_Y1 0x1e58
+#define VIU2_VD1_IF0_CHROMA_X1 0x1e59
+#define VIU2_VD1_IF0_CHROMA_Y1 0x1e5a
+#define VIU2_VD1_IF0_RPT_LOOP 0x1e5b
+#define VIU2_VD1_IF0_LUMA0_RPT_PAT 0x1e5c
+#define VIU2_VD1_IF0_CHROMA0_RPT_PAT 0x1e5d
+#define VIU2_VD1_IF0_LUMA1_RPT_PAT 0x1e5e
+#define VIU2_VD1_IF0_CHROMA1_RPT_PAT 0x1e5f
+#define VIU2_VD1_IF0_LUMA_PSEL 0x1e60
+#define VIU2_VD1_IF0_CHROMA_PSEL 0x1e61
+#define VIU2_VD1_IF0_DUMMY_PIXEL 0x1e62
+#define VIU2_VD1_IF0_LUMA_FIFO_SIZE 0x1e63
+#define VIU2_VD1_IF0_RANGE_MAP_Y 0x1e6a
+#define VIU2_VD1_IF0_RANGE_MAP_CB 0x1e6b
+#define VIU2_VD1_IF0_RANGE_MAP_CR 0x1e6c
+#define VIU2_VD1_IF0_GEN_REG2 0x1e6d
+#define VIU2_VD1_IF0_PROT_CNTL 0x1e6e
+#define VIU2_VD1_FMT_CTRL 0x1e68
+#define VIU2_VD1_FMT_W 0x1e69
+
+/* encode */
+#define ENCP_VFIFO2VD_CTL 0x1b58
+#define ENCP_VFIFO2VD_PIXEL_START 0x1b59
+#define ENCP_VFIFO2VD_PIXEL_END 0x1b5a
+#define ENCP_VFIFO2VD_LINE_TOP_START 0x1b5b
+#define ENCP_VFIFO2VD_LINE_TOP_END 0x1b5c
+#define ENCP_VFIFO2VD_LINE_BOT_START 0x1b5d
+#define ENCP_VFIFO2VD_LINE_BOT_END 0x1b5e
+#define VENC_SYNC_ROUTE 0x1b60
+#define VENC_VIDEO_EXSRC 0x1b61
+#define VENC_DVI_SETTING 0x1b62
+#define VENC_C656_CTRL 0x1b63
+#define VENC_UPSAMPLE_CTRL0 0x1b64
+#define VENC_UPSAMPLE_CTRL1 0x1b65
+#define VENC_UPSAMPLE_CTRL2 0x1b66
+#define TCON_INVERT_CTL 0x1b67
+#define VENC_VIDEO_PROG_MODE 0x1b68
+#define VENC_ENCI_LINE 0x1b69
+#define VENC_ENCI_PIXEL 0x1b6a
+#define VENC_ENCP_LINE 0x1b6b
+#define VENC_ENCP_PIXEL 0x1b6c
+#define VENC_STATA 0x1b6d
+#define VENC_INTCTRL 0x1b6e
+#define VENC_INTFLAG 0x1b6f
+#define VENC_VIDEO_TST_EN 0x1b70
+#define VENC_VIDEO_TST_MDSEL 0x1b71
+#define VENC_VIDEO_TST_Y 0x1b72
+#define VENC_VIDEO_TST_CB 0x1b73
+#define VENC_VIDEO_TST_CR 0x1b74
+#define VENC_VIDEO_TST_CLRBAR_STRT 0x1b75
+#define VENC_VIDEO_TST_CLRBAR_WIDTH 0x1b76
+#define VENC_VIDEO_TST_VDCNT_STSET 0x1b77
+#define VENC_VDAC_DACSEL0 0x1b78
+#define VENC_VDAC_DACSEL1 0x1b79
+#define VENC_VDAC_DACSEL2 0x1b7a
+#define VENC_VDAC_DACSEL3 0x1b7b
+#define VENC_VDAC_DACSEL4 0x1b7c
+#define VENC_VDAC_DACSEL5 0x1b7d
+#define VENC_VDAC_SETTING 0x1b7e
+#define VENC_VDAC_TST_VAL 0x1b7f
+#define VENC_VDAC_DAC0_GAINCTRL 0x1bf0
+#define VENC_VDAC_DAC0_OFFSET 0x1bf1
+#define VENC_VDAC_DAC1_GAINCTRL 0x1bf2
+#define VENC_VDAC_DAC1_OFFSET 0x1bf3
+#define VENC_VDAC_DAC2_GAINCTRL 0x1bf4
+#define VENC_VDAC_DAC2_OFFSET 0x1bf5
+#define VENC_VDAC_DAC3_GAINCTRL 0x1bf6
+#define VENC_VDAC_DAC3_OFFSET 0x1bf7
+#define VENC_VDAC_DAC4_GAINCTRL 0x1bf8
+#define VENC_VDAC_DAC4_OFFSET 0x1bf9
+#define VENC_VDAC_DAC5_GAINCTRL 0x1bfa
+#define VENC_VDAC_DAC5_OFFSET 0x1bfb
+#define VENC_VDAC_FIFO_CTRL 0x1bfc
+#define ENCL_TCON_INVERT_CTL 0x1bfd
+#define ENCP_VIDEO_EN 0x1b80
+#define ENCP_VIDEO_SYNC_MODE 0x1b81
+#define ENCP_MACV_EN 0x1b82
+#define ENCP_VIDEO_Y_SCL 0x1b83
+#define ENCP_VIDEO_PB_SCL 0x1b84
+#define ENCP_VIDEO_PR_SCL 0x1b85
+#define ENCP_VIDEO_SYNC_SCL 0x1b86
+#define ENCP_VIDEO_MACV_SCL 0x1b87
+#define ENCP_VIDEO_Y_OFFST 0x1b88
+#define ENCP_VIDEO_PB_OFFST 0x1b89
+#define ENCP_VIDEO_PR_OFFST 0x1b8a
+#define ENCP_VIDEO_SYNC_OFFST 0x1b8b
+#define ENCP_VIDEO_MACV_OFFST 0x1b8c
+#define ENCP_VIDEO_MODE 0x1b8d
+#define ENCP_VIDEO_MODE_ADV 0x1b8e
+#define ENCP_DBG_PX_RST 0x1b90
+#define ENCP_DBG_LN_RST 0x1b91
+#define ENCP_DBG_PX_INT 0x1b92
+#define ENCP_DBG_LN_INT 0x1b93
+#define ENCP_VIDEO_YFP1_HTIME 0x1b94
+#define ENCP_VIDEO_YFP2_HTIME 0x1b95
+#define ENCP_VIDEO_YC_DLY 0x1b96
+#define ENCP_VIDEO_MAX_PXCNT 0x1b97
+#define ENCP_VIDEO_HSPULS_BEGIN 0x1b98
+#define ENCP_VIDEO_HSPULS_END 0x1b99
+#define ENCP_VIDEO_HSPULS_SWITCH 0x1b9a
+#define ENCP_VIDEO_VSPULS_BEGIN 0x1b9b
+#define ENCP_VIDEO_VSPULS_END 0x1b9c
+#define ENCP_VIDEO_VSPULS_BLINE 0x1b9d
+#define ENCP_VIDEO_VSPULS_ELINE 0x1b9e
+#define ENCP_VIDEO_EQPULS_BEGIN 0x1b9f
+#define ENCP_VIDEO_EQPULS_END 0x1ba0
+#define ENCP_VIDEO_EQPULS_BLINE 0x1ba1
+#define ENCP_VIDEO_EQPULS_ELINE 0x1ba2
+#define ENCP_VIDEO_HAVON_END 0x1ba3
+#define ENCP_VIDEO_HAVON_BEGIN 0x1ba4
+#define ENCP_VIDEO_VAVON_ELINE 0x1baf
+#define ENCP_VIDEO_VAVON_BLINE 0x1ba6
+#define ENCP_VIDEO_HSO_BEGIN 0x1ba7
+#define ENCP_VIDEO_HSO_END 0x1ba8
+#define ENCP_VIDEO_VSO_BEGIN 0x1ba9
+#define ENCP_VIDEO_VSO_END 0x1baa
+#define ENCP_VIDEO_VSO_BLINE 0x1bab
+#define ENCP_VIDEO_VSO_ELINE 0x1bac
+#define ENCP_VIDEO_SYNC_WAVE_CURVE 0x1bad
+#define ENCP_VIDEO_MAX_LNCNT 0x1bae
+#define ENCP_VIDEO_SY_VAL 0x1bb0
+#define ENCP_VIDEO_SY2_VAL 0x1bb1
+#define ENCP_VIDEO_BLANKY_VAL 0x1bb2
+#define ENCP_VIDEO_BLANKPB_VAL 0x1bb3
+#define ENCP_VIDEO_BLANKPR_VAL 0x1bb4
+#define ENCP_VIDEO_HOFFST 0x1bb5
+#define ENCP_VIDEO_VOFFST 0x1bb6
+#define ENCP_VIDEO_RGB_CTRL 0x1bb7
+#define ENCP_VIDEO_FILT_CTRL 0x1bb8
+#define ENCP_VIDEO_OFLD_VPEQ_OFST 0x1bb9
+#define ENCP_VIDEO_OFLD_VOAV_OFST 0x1bba
+#define ENCP_VIDEO_MATRIX_CB 0x1bbb
+#define ENCP_VIDEO_MATRIX_CR 0x1bbc
+#define ENCP_VIDEO_RGBIN_CTRL 0x1bbd
+#define ENCP_MACV_BLANKY_VAL 0x1bc0
+#define ENCP_MACV_MAXY_VAL 0x1bc1
+#define ENCP_MACV_1ST_PSSYNC_STRT 0x1bc2
+#define ENCP_MACV_PSSYNC_STRT 0x1bc3
+#define ENCP_MACV_AGC_STRT 0x1bc4
+#define ENCP_MACV_AGC_END 0x1bc5
+#define ENCP_MACV_WAVE_END 0x1bc6
+#define ENCP_MACV_STRTLINE 0x1bc7
+#define ENCP_MACV_ENDLINE 0x1bc8
+#define ENCP_MACV_TS_CNT_MAX_L 0x1bc9
+#define ENCP_MACV_TS_CNT_MAX_H 0x1bca
+#define ENCP_MACV_TIME_DOWN 0x1bcb
+#define ENCP_MACV_TIME_LO 0x1bcc
+#define ENCP_MACV_TIME_UP 0x1bcd
+#define ENCP_MACV_TIME_RST 0x1bce
+#define ENCP_VBI_CTRL 0x1bd0
+#define ENCP_VBI_SETTING 0x1bd1
+#define ENCP_VBI_BEGIN 0x1bd2
+#define ENCP_VBI_WIDTH 0x1bd3
+#define ENCP_VBI_HVAL 0x1bd4
+#define ENCP_VBI_DATA0 0x1bd5
+#define ENCP_VBI_DATA1 0x1bd6
+#define C656_HS_ST 0x1be0
+#define C656_HS_ED 0x1be1
+#define C656_VS_LNST_E 0x1be2
+#define C656_VS_LNST_O 0x1be3
+#define C656_VS_LNED_E 0x1be4
+#define C656_VS_LNED_O 0x1be5
+#define C656_FS_LNST 0x1be6
+#define C656_FS_LNED 0x1be7
+#define ENCI_VIDEO_MODE 0x1b00
+#define ENCI_VIDEO_MODE_ADV 0x1b01
+#define ENCI_VIDEO_FSC_ADJ 0x1b02
+#define ENCI_VIDEO_BRIGHT 0x1b03
+#define ENCI_VIDEO_CONT 0x1b04
+#define ENCI_VIDEO_SAT 0x1b05
+#define ENCI_VIDEO_HUE 0x1b06
+#define ENCI_VIDEO_SCH 0x1b07
+#define ENCI_SYNC_MODE 0x1b08
+#define ENCI_SYNC_CTRL 0x1b09
+#define ENCI_SYNC_HSO_BEGIN 0x1b0a
+#define ENCI_SYNC_HSO_END 0x1b0b
+#define ENCI_SYNC_VSO_EVN 0x1b0c
+#define ENCI_SYNC_VSO_ODD 0x1b0d
+#define ENCI_SYNC_VSO_EVNLN 0x1b0e
+#define ENCI_SYNC_VSO_ODDLN 0x1b0f
+#define ENCI_SYNC_HOFFST 0x1b10
+#define ENCI_SYNC_VOFFST 0x1b11
+#define ENCI_SYNC_ADJ 0x1b12
+#define ENCI_RGB_SETTING 0x1b13
+#define ENCI_DE_H_BEGIN 0x1b16
+#define ENCI_DE_H_END 0x1b17
+#define ENCI_DE_V_BEGIN_EVEN 0x1b18
+#define ENCI_DE_V_END_EVEN 0x1b19
+#define ENCI_DE_V_BEGIN_ODD 0x1b1a
+#define ENCI_DE_V_END_ODD 0x1b1b
+#define ENCI_VBI_SETTING 0x1b20
+#define ENCI_VBI_CCDT_EVN 0x1b21
+#define ENCI_VBI_CCDT_ODD 0x1b22
+#define ENCI_VBI_CC525_LN 0x1b23
+#define ENCI_VBI_CC625_LN 0x1b24
+#define ENCI_VBI_WSSDT 0x1b25
+#define ENCI_VBI_WSS_LN 0x1b26
+#define ENCI_VBI_CGMSDT_L 0x1b27
+#define ENCI_VBI_CGMSDT_H 0x1b28
+#define ENCI_VBI_CGMS_LN 0x1b29
+#define ENCI_VBI_TTX_HTIME 0x1b2a
+#define ENCI_VBI_TTX_LN 0x1b2b
+#define ENCI_VBI_TTXDT0 0x1b2c
+#define ENCI_VBI_TTXDT1 0x1b2d
+#define ENCI_VBI_TTXDT2 0x1b2e
+#define ENCI_VBI_TTXDT3 0x1b2f
+#define ENCI_MACV_N0 0x1b30
+#define ENCI_MACV_N1 0x1b31
+#define ENCI_MACV_N2 0x1b32
+#define ENCI_MACV_N3 0x1b33
+#define ENCI_MACV_N4 0x1b34
+#define ENCI_MACV_N5 0x1b35
+#define ENCI_MACV_N6 0x1b36
+#define ENCI_MACV_N7 0x1b37
+#define ENCI_MACV_N8 0x1b38
+#define ENCI_MACV_N9 0x1b39
+#define ENCI_MACV_N10 0x1b3a
+#define ENCI_MACV_N11 0x1b3b
+#define ENCI_MACV_N12 0x1b3c
+#define ENCI_MACV_N13 0x1b3d
+#define ENCI_MACV_N14 0x1b3e
+#define ENCI_MACV_N15 0x1b3f
+#define ENCI_MACV_N16 0x1b40
+#define ENCI_MACV_N17 0x1b41
+#define ENCI_MACV_N18 0x1b42
+#define ENCI_MACV_N19 0x1b43
+#define ENCI_MACV_N20 0x1b44
+#define ENCI_MACV_N21 0x1b45
+#define ENCI_MACV_N22 0x1b46
+#define ENCI_DBG_PX_RST 0x1b48
+#define ENCI_DBG_FLDLN_RST 0x1b49
+#define ENCI_DBG_PX_INT 0x1b4a
+#define ENCI_DBG_FLDLN_INT 0x1b4b
+#define ENCI_DBG_MAXPX 0x1b4c
+#define ENCI_DBG_MAXLN 0x1b4d
+#define ENCI_MACV_MAX_AMP 0x1b50
+#define ENCI_MACV_PULSE_LO 0x1b51
+#define ENCI_MACV_PULSE_HI 0x1b52
+#define ENCI_MACV_BKP_MAX 0x1b53
+#define ENCI_CFILT_CTRL 0x1b54
+#define ENCI_CFILT7 0x1b55
+#define ENCI_YC_DELAY 0x1b56
+#define ENCI_VIDEO_EN 0x1b57
+#define ENCI_DVI_HSO_BEGIN 0x1c00
+#define ENCI_DVI_HSO_END 0x1c01
+#define ENCI_DVI_VSO_BLINE_EVN 0x1c02
+#define ENCI_DVI_VSO_BLINE_ODD 0x1c03
+#define ENCI_DVI_VSO_ELINE_EVN 0x1c04
+#define ENCI_DVI_VSO_ELINE_ODD 0x1c05
+#define ENCI_DVI_VSO_BEGIN_EVN 0x1c06
+#define ENCI_DVI_VSO_BEGIN_ODD 0x1c07
+#define ENCI_DVI_VSO_END_EVN 0x1c08
+#define ENCI_DVI_VSO_END_ODD 0x1c09
+#define ENCI_CFILT_CTRL2 0x1c0a
+#define ENCI_DACSEL_0 0x1c0b
+#define ENCI_DACSEL_1 0x1c0c
+#define ENCP_DACSEL_0 0x1c0d
+#define ENCP_DACSEL_1 0x1c0e
+#define ENCP_MAX_LINE_SWITCH_POINT 0x1c0f
+#define ENCI_TST_EN 0x1c10
+#define ENCI_TST_MDSEL 0x1c11
+#define ENCI_TST_Y 0x1c12
+#define ENCI_TST_CB 0x1c13
+#define ENCI_TST_CR 0x1c14
+#define ENCI_TST_CLRBAR_STRT 0x1c15
+#define ENCI_TST_CLRBAR_WIDTH 0x1c16
+#define ENCI_TST_VDCNT_STSET 0x1c17
+#define ENCI_VFIFO2VD_CTL 0x1c18
+#define ENCI_VFIFO2VD_PIXEL_START 0x1c19
+#define ENCI_VFIFO2VD_PIXEL_END 0x1c1a
+#define ENCI_VFIFO2VD_LINE_TOP_START 0x1c1b
+#define ENCI_VFIFO2VD_LINE_TOP_END 0x1c1c
+#define ENCI_VFIFO2VD_LINE_BOT_START 0x1c1d
+#define ENCI_VFIFO2VD_LINE_BOT_END 0x1c1e
+#define ENCI_VFIFO2VD_CTL2 0x1c1f
+#define ENCT_VFIFO2VD_CTL 0x1c20
+#define ENCT_VFIFO2VD_PIXEL_START 0x1c21
+#define ENCT_VFIFO2VD_PIXEL_END 0x1c22
+#define ENCT_VFIFO2VD_LINE_TOP_START 0x1c23
+#define ENCT_VFIFO2VD_LINE_TOP_END 0x1c24
+#define ENCT_VFIFO2VD_LINE_BOT_START 0x1c25
+#define ENCT_VFIFO2VD_LINE_BOT_END 0x1c26
+#define ENCT_VFIFO2VD_CTL2 0x1c27
+#define ENCT_TST_EN 0x1c28
+#define ENCT_TST_MDSEL 0x1c29
+#define ENCT_TST_Y 0x1c2a
+#define ENCT_TST_CB 0x1c2b
+#define ENCT_TST_CR 0x1c2c
+#define ENCT_TST_CLRBAR_STRT 0x1c2d
+#define ENCT_TST_CLRBAR_WIDTH 0x1c2e
+#define ENCT_TST_VDCNT_STSET 0x1c2f
+#define ENCP_DVI_HSO_BEGIN 0x1c30
+#define ENCP_DVI_HSO_END 0x1c31
+#define ENCP_DVI_VSO_BLINE_EVN 0x1c32
+#define ENCP_DVI_VSO_BLINE_ODD 0x1c33
+#define ENCP_DVI_VSO_ELINE_EVN 0x1c34
+#define ENCP_DVI_VSO_ELINE_ODD 0x1c35
+#define ENCP_DVI_VSO_BEGIN_EVN 0x1c36
+#define ENCP_DVI_VSO_BEGIN_ODD 0x1c37
+#define ENCP_DVI_VSO_END_EVN 0x1c38
+#define ENCP_DVI_VSO_END_ODD 0x1c39
+#define ENCP_DE_H_BEGIN 0x1c3a
+#define ENCP_DE_H_END 0x1c3b
+#define ENCP_DE_V_BEGIN_EVEN 0x1c3c
+#define ENCP_DE_V_END_EVEN 0x1c3d
+#define ENCP_DE_V_BEGIN_ODD 0x1c3e
+#define ENCP_DE_V_END_ODD 0x1c3f
+#define ENCI_SYNC_LINE_LENGTH 0x1c40
+#define ENCI_SYNC_PIXEL_EN 0x1c41
+#define ENCI_SYNC_TO_LINE_EN 0x1c42
+#define ENCI_SYNC_TO_PIXEL 0x1c43
+#define ENCP_SYNC_LINE_LENGTH 0x1c44
+#define ENCP_SYNC_PIXEL_EN 0x1c45
+#define ENCP_SYNC_TO_LINE_EN 0x1c46
+#define ENCP_SYNC_TO_PIXEL 0x1c47
+#define ENCT_SYNC_LINE_LENGTH 0x1c48
+#define ENCT_SYNC_PIXEL_EN 0x1c49
+#define ENCT_SYNC_TO_LINE_EN 0x1c4a
+#define ENCT_SYNC_TO_PIXEL 0x1c4b
+#define ENCL_SYNC_LINE_LENGTH 0x1c4c
+#define ENCL_SYNC_PIXEL_EN 0x1c4d
+#define ENCL_SYNC_TO_LINE_EN 0x1c4e
+#define ENCL_SYNC_TO_PIXEL 0x1c4f
+#define ENCP_VFIFO2VD_CTL2 0x1c50
+#define VENC_DVI_SETTING_MORE 0x1c51
+#define VENC_VDAC_DAC4_FILT_CTRL0 0x1c54
+#define VENC_VDAC_DAC4_FILT_CTRL1 0x1c55
+#define VENC_VDAC_DAC5_FILT_CTRL0 0x1c56
+#define VENC_VDAC_DAC5_FILT_CTRL1 0x1c57
+#define VENC_VDAC_DAC0_FILT_CTRL0 0x1c58
+#define VENC_VDAC_DAC0_FILT_CTRL1 0x1c59
+#define VENC_VDAC_DAC1_FILT_CTRL0 0x1c5a
+#define VENC_VDAC_DAC1_FILT_CTRL1 0x1c5b
+#define VENC_VDAC_DAC2_FILT_CTRL0 0x1c5c
+#define VENC_VDAC_DAC2_FILT_CTRL1 0x1c5d
+#define VENC_VDAC_DAC3_FILT_CTRL0 0x1c5e
+#define VENC_VDAC_DAC3_FILT_CTRL1 0x1c5f
+#define ENCT_VIDEO_EN 0x1c60
+#define ENCT_VIDEO_Y_SCL 0x1c61
+#define ENCT_VIDEO_PB_SCL 0x1c62
+#define ENCT_VIDEO_PR_SCL 0x1c63
+#define ENCT_VIDEO_Y_OFFST 0x1c64
+#define ENCT_VIDEO_PB_OFFST 0x1c65
+#define ENCT_VIDEO_PR_OFFST 0x1c66
+#define ENCT_VIDEO_MODE 0x1c67
+#define ENCT_VIDEO_MODE_ADV 0x1c68
+#define ENCT_DBG_PX_RST 0x1c69
+#define ENCT_DBG_LN_RST 0x1c6a
+#define ENCT_DBG_PX_INT 0x1c6b
+#define ENCT_DBG_LN_INT 0x1c6c
+#define ENCT_VIDEO_YFP1_HTIME 0x1c6d
+#define ENCT_VIDEO_YFP2_HTIME 0x1c6e
+#define ENCT_VIDEO_YC_DLY 0x1c6f
+#define ENCT_VIDEO_MAX_PXCNT 0x1c70
+#define ENCT_VIDEO_HAVON_END 0x1c71
+#define ENCT_VIDEO_HAVON_BEGIN 0x1c72
+#define ENCT_VIDEO_VAVON_ELINE 0x1c73
+#define ENCT_VIDEO_VAVON_BLINE 0x1c74
+#define ENCT_VIDEO_HSO_BEGIN 0x1c75
+#define ENCT_VIDEO_HSO_END 0x1c76
+#define ENCT_VIDEO_VSO_BEGIN 0x1c77
+#define ENCT_VIDEO_VSO_END 0x1c78
+#define ENCT_VIDEO_VSO_BLINE 0x1c79
+#define ENCT_VIDEO_VSO_ELINE 0x1c7a
+#define ENCT_VIDEO_MAX_LNCNT 0x1c7b
+#define ENCT_VIDEO_BLANKY_VAL 0x1c7c
+#define ENCT_VIDEO_BLANKPB_VAL 0x1c7d
+#define ENCT_VIDEO_BLANKPR_VAL 0x1c7e
+#define ENCT_VIDEO_HOFFST 0x1c7f
+#define ENCT_VIDEO_VOFFST 0x1c80
+#define ENCT_VIDEO_RGB_CTRL 0x1c81
+#define ENCT_VIDEO_FILT_CTRL 0x1c82
+#define ENCT_VIDEO_OFLD_VPEQ_OFST 0x1c83
+#define ENCT_VIDEO_OFLD_VOAV_OFST 0x1c84
+#define ENCT_VIDEO_MATRIX_CB 0x1c85
+#define ENCT_VIDEO_MATRIX_CR 0x1c86
+#define ENCT_VIDEO_RGBIN_CTRL 0x1c87
+#define ENCT_MAX_LINE_SWITCH_POINT 0x1c88
+#define ENCT_DACSEL_0 0x1c89
+#define ENCT_DACSEL_1 0x1c8a
+#define ENCL_VFIFO2VD_CTL 0x1c90
+#define ENCL_VFIFO2VD_PIXEL_START 0x1c91
+#define ENCL_VFIFO2VD_PIXEL_END 0x1c92
+#define ENCL_VFIFO2VD_LINE_TOP_START 0x1c93
+#define ENCL_VFIFO2VD_LINE_TOP_END 0x1c94
+#define ENCL_VFIFO2VD_LINE_BOT_START 0x1c95
+#define ENCL_VFIFO2VD_LINE_BOT_END 0x1c96
+#define ENCL_VFIFO2VD_CTL2 0x1c97
+#define ENCL_TST_EN 0x1c98
+#define ENCL_TST_MDSEL 0x1c99
+#define ENCL_TST_Y 0x1c9a
+#define ENCL_TST_CB 0x1c9b
+#define ENCL_TST_CR 0x1c9c
+#define ENCL_TST_CLRBAR_STRT 0x1c9d
+#define ENCL_TST_CLRBAR_WIDTH 0x1c9e
+#define ENCL_TST_VDCNT_STSET 0x1c9f
+#define ENCL_VIDEO_EN 0x1ca0
+#define ENCL_VIDEO_Y_SCL 0x1ca1
+#define ENCL_VIDEO_PB_SCL 0x1ca2
+#define ENCL_VIDEO_PR_SCL 0x1ca3
+#define ENCL_VIDEO_Y_OFFST 0x1ca4
+#define ENCL_VIDEO_PB_OFFST 0x1ca5
+#define ENCL_VIDEO_PR_OFFST 0x1ca6
+#define ENCL_VIDEO_MODE 0x1ca7
+#define ENCL_VIDEO_MODE_ADV 0x1ca8
+#define ENCL_DBG_PX_RST 0x1ca9
+#define ENCL_DBG_LN_RST 0x1caa
+#define ENCL_DBG_PX_INT 0x1cab
+#define ENCL_DBG_LN_INT 0x1cac
+#define ENCL_VIDEO_YFP1_HTIME 0x1cad
+#define ENCL_VIDEO_YFP2_HTIME 0x1cae
+#define ENCL_VIDEO_YC_DLY 0x1caf
+#define ENCL_VIDEO_MAX_PXCNT 0x1cb0
+#define ENCL_VIDEO_HAVON_END 0x1cb1
+#define ENCL_VIDEO_HAVON_BEGIN 0x1cb2
+#define ENCL_VIDEO_VAVON_ELINE 0x1cb3
+#define ENCL_VIDEO_VAVON_BLINE 0x1cb4
+#define ENCL_VIDEO_HSO_BEGIN 0x1cb5
+#define ENCL_VIDEO_HSO_END 0x1cb6
+#define ENCL_VIDEO_VSO_BEGIN 0x1cb7
+#define ENCL_VIDEO_VSO_END 0x1cb8
+#define ENCL_VIDEO_VSO_BLINE 0x1cb9
+#define ENCL_VIDEO_VSO_ELINE 0x1cba
+#define ENCL_VIDEO_MAX_LNCNT 0x1cbb
+#define ENCL_VIDEO_BLANKY_VAL 0x1cbc
+#define ENCL_VIDEO_BLANKPB_VAL 0x1cbd
+#define ENCL_VIDEO_BLANKPR_VAL 0x1cbe
+#define ENCL_VIDEO_HOFFST 0x1cbf
+#define ENCL_VIDEO_VOFFST 0x1cc0
+#define ENCL_VIDEO_RGB_CTRL 0x1cc1
+#define ENCL_VIDEO_FILT_CTRL 0x1cc2
+#define ENCL_VIDEO_OFLD_VPEQ_OFST 0x1cc3
+#define ENCL_VIDEO_OFLD_VOAV_OFST 0x1cc4
+#define ENCL_VIDEO_MATRIX_CB 0x1cc5
+#define ENCL_VIDEO_MATRIX_CR 0x1cc6
+#define ENCL_VIDEO_RGBIN_CTRL 0x1cc7
+#define ENCL_MAX_LINE_SWITCH_POINT 0x1cc8
+#define ENCL_DACSEL_0 0x1cc9
+#define ENCL_DACSEL_1 0x1cca
+#define RDMA_AHB_START_ADDR_MAN 0x1100
+#define RDMA_AHB_END_ADDR_MAN 0x1101
+#define RDMA_AHB_START_ADDR_1 0x1102
+#define RDMA_AHB_END_ADDR_1 0x1103
+#define RDMA_AHB_START_ADDR_2 0x1104
+#define RDMA_AHB_END_ADDR_2 0x1105
+#define RDMA_AHB_START_ADDR_3 0x1106
+#define RDMA_AHB_END_ADDR_3 0x1107
+#define RDMA_AHB_START_ADDR_4 0x1108
+#define RDMA_AHB_END_ADDR_4 0x1109
+#define RDMA_AHB_START_ADDR_5 0x110a
+#define RDMA_AHB_END_ADDR_5 0x110b
+#define RDMA_AHB_START_ADDR_6 0x110c
+#define RDMA_AHB_END_ADDR_6 0x110d
+#define RDMA_AHB_START_ADDR_7 0x110e
+#define RDMA_AHB_END_ADDR_7 0x110f
+#define RDMA_ACCESS_AUTO 0x1110
+#define RDMA_ACCESS_AUTO2 0x1111
+#define RDMA_ACCESS_AUTO3 0x1112
+#define RDMA_ACCESS_MAN 0x1113
+#define RDMA_CTRL 0x1114
+#define RDMA_STATUS 0x1115
+#define RDMA_STATUS2 0x1116
+#define RDMA_STATUS3 0x1117
+#define L_GAMMA_CNTL_PORT 0x1400
+#define L_GAMMA_DATA_PORT 0x1401
+#define L_GAMMA_ADDR_PORT 0x1402
+#define L_GAMMA_VCOM_HSWITCH_ADDR 0x1403
+#define L_RGB_BASE_ADDR 0x1405
+#define L_RGB_COEFF_ADDR 0x1406
+#define L_POL_CNTL_ADDR 0x1407
+#define L_DITH_CNTL_ADDR 0x1408
+#define L_GAMMA_PROBE_CTRL 0x1409
+#define L_GAMMA_PROBE_COLOR_L 0x140a
+#define L_GAMMA_PROBE_COLOR_H 0x140b
+#define L_GAMMA_PROBE_HL_COLOR 0x140c
+#define L_GAMMA_PROBE_POS_X 0x140d
+#define L_GAMMA_PROBE_POS_Y 0x140e
+#define L_STH1_HS_ADDR 0x1410
+#define L_STH1_HE_ADDR 0x1411
+#define L_STH1_VS_ADDR 0x1412
+#define L_STH1_VE_ADDR 0x1413
+#define L_STH2_HS_ADDR 0x1414
+#define L_STH2_HE_ADDR 0x1415
+#define L_STH2_VS_ADDR 0x1416
+#define L_STH2_VE_ADDR 0x1417
+#define L_OEH_HS_ADDR 0x1418
+#define L_OEH_HE_ADDR 0x1419
+#define L_OEH_VS_ADDR 0x141a
+#define L_OEH_VE_ADDR 0x141b
+#define L_VCOM_HSWITCH_ADDR 0x141c
+#define L_VCOM_VS_ADDR 0x141d
+#define L_VCOM_VE_ADDR 0x141e
+#define L_CPV1_HS_ADDR 0x141f
+#define L_CPV1_HE_ADDR 0x1420
+#define L_CPV1_VS_ADDR 0x1421
+#define L_CPV1_VE_ADDR 0x1422
+#define L_CPV2_HS_ADDR 0x1423
+#define L_CPV2_HE_ADDR 0x1424
+#define L_CPV2_VS_ADDR 0x1425
+#define L_CPV2_VE_ADDR 0x1426
+#define L_STV1_HS_ADDR 0x1427
+#define L_STV1_HE_ADDR 0x1428
+#define L_STV1_VS_ADDR 0x1429
+#define L_STV1_VE_ADDR 0x142a
+#define L_STV2_HS_ADDR 0x142b
+#define L_STV2_HE_ADDR 0x142c
+#define L_STV2_VS_ADDR 0x142d
+#define L_STV2_VE_ADDR 0x142e
+#define L_OEV1_HS_ADDR 0x142f
+#define L_OEV1_HE_ADDR 0x1430
+#define L_OEV1_VS_ADDR 0x1431
+#define L_OEV1_VE_ADDR 0x1432
+#define L_OEV2_HS_ADDR 0x1433
+#define L_OEV2_HE_ADDR 0x1434
+#define L_OEV2_VS_ADDR 0x1435
+#define L_OEV2_VE_ADDR 0x1436
+#define L_OEV3_HS_ADDR 0x1437
+#define L_OEV3_HE_ADDR 0x1438
+#define L_OEV3_VS_ADDR 0x1439
+#define L_OEV3_VE_ADDR 0x143a
+#define L_LCD_PWR_ADDR 0x143b
+#define L_LCD_PWM0_LO_ADDR 0x143c
+#define L_LCD_PWM0_HI_ADDR 0x143d
+#define L_LCD_PWM1_LO_ADDR 0x143e
+#define L_LCD_PWM1_HI_ADDR 0x143f
+#define L_INV_CNT_ADDR 0x1440
+#define L_TCON_MISC_SEL_ADDR 0x1441
+#define L_DUAL_PORT_CNTL_ADDR 0x1442
+#define MLVDS_CLK_CTL1_HI 0x1443
+#define MLVDS_CLK_CTL1_LO 0x1444
+#define L_TCON_DOUBLE_CTL 0x1449
+#define L_TCON_PATTERN_HI 0x144a
+#define L_TCON_PATTERN_LO 0x144b
+#define LDIM_BL_ADDR_PORT 0x144e
+#define LDIM_BL_DATA_PORT 0x144f
+#define L_DE_HS_ADDR 0x1451
+#define L_DE_HE_ADDR 0x1452
+#define L_DE_VS_ADDR 0x1453
+#define L_DE_VE_ADDR 0x1454
+#define L_HSYNC_HS_ADDR 0x1455
+#define L_HSYNC_HE_ADDR 0x1456
+#define L_HSYNC_VS_ADDR 0x1457
+#define L_HSYNC_VE_ADDR 0x1458
+#define L_VSYNC_HS_ADDR 0x1459
+#define L_VSYNC_HE_ADDR 0x145a
+#define L_VSYNC_VS_ADDR 0x145b
+#define L_VSYNC_VE_ADDR 0x145c
+#define L_LCD_MCU_CTL 0x145d
+#define DUAL_MLVDS_CTL 0x1460
+#define DUAL_MLVDS_LINE_START 0x1461
+#define DUAL_MLVDS_LINE_END 0x1462
+#define DUAL_MLVDS_PIXEL_W_START_L 0x1463
+#define DUAL_MLVDS_PIXEL_W_END_L 0x1464
+#define DUAL_MLVDS_PIXEL_W_START_R 0x1465
+#define DUAL_MLVDS_PIXEL_W_END_R 0x1466
+#define DUAL_MLVDS_PIXEL_R_START_L 0x1467
+#define DUAL_MLVDS_PIXEL_R_CNT_L 0x1468
+#define DUAL_MLVDS_PIXEL_R_START_R 0x1469
+#define DUAL_MLVDS_PIXEL_R_CNT_R 0x146a
+#define V_INVERSION_PIXEL 0x1470
+#define V_INVERSION_LINE 0x1471
+#define V_INVERSION_CONTROL 0x1472
+#define MLVDS2_CONTROL 0x1474
+#define MLVDS2_CONFIG_HI 0x1475
+#define MLVDS2_CONFIG_LO 0x1476
+#define MLVDS2_DUAL_GATE_WR_START 0x1477
+#define MLVDS2_DUAL_GATE_WR_END 0x1478
+#define MLVDS2_DUAL_GATE_RD_START 0x1479
+#define MLVDS2_DUAL_GATE_RD_END 0x147a
+#define MLVDS2_SECOND_RESET_CTL 0x147b
+#define MLVDS2_DUAL_GATE_CTL_HI 0x147c
+#define MLVDS2_DUAL_GATE_CTL_LO 0x147d
+#define MLVDS2_RESET_CONFIG_HI 0x147e
+#define MLVDS2_RESET_CONFIG_LO 0x147f
+#define GAMMA_CNTL_PORT 0x1480
+#define GAMMA_DATA_PORT 0x1481
+#define GAMMA_ADDR_PORT 0x1482
+#define GAMMA_VCOM_HSWITCH_ADDR 0x1483
+#define RGB_BASE_ADDR 0x1485
+#define RGB_COEFF_ADDR 0x1486
+#define POL_CNTL_ADDR 0x1487
+#define DITH_CNTL_ADDR 0x1488
+#define GAMMA_PROBE_CTRL 0x1489
+#define GAMMA_PROBE_COLOR_L 0x148a
+#define GAMMA_PROBE_COLOR_H 0x148b
+#define GAMMA_PROBE_HL_COLOR 0x148c
+#define GAMMA_PROBE_POS_X 0x148d
+#define GAMMA_PROBE_POS_Y 0x148e
+#define STH1_HS_ADDR 0x1490
+#define STH1_HE_ADDR 0x1491
+#define STH1_VS_ADDR 0x1492
+#define STH1_VE_ADDR 0x1493
+#define STH2_HS_ADDR 0x1494
+#define STH2_HE_ADDR 0x1495
+#define STH2_VS_ADDR 0x1496
+#define STH2_VE_ADDR 0x1497
+#define OEH_HS_ADDR 0x1498
+#define OEH_HE_ADDR 0x1499
+#define OEH_VS_ADDR 0x149a
+#define OEH_VE_ADDR 0x149b
+#define VCOM_HSWITCH_ADDR 0x149c
+#define VCOM_VS_ADDR 0x149d
+#define VCOM_VE_ADDR 0x149e
+#define CPV1_HS_ADDR 0x149f
+#define CPV1_HE_ADDR 0x14a0
+#define CPV1_VS_ADDR 0x14a1
+#define CPV1_VE_ADDR 0x14a2
+#define CPV2_HS_ADDR 0x14a3
+#define CPV2_HE_ADDR 0x14a4
+#define CPV2_VS_ADDR 0x14a5
+#define CPV2_VE_ADDR 0x14a6
+#define STV1_HS_ADDR 0x14a7
+#define STV1_HE_ADDR 0x14a8
+#define STV1_VS_ADDR 0x14a9
+#define STV1_VE_ADDR 0x14aa
+#define STV2_HS_ADDR 0x14ab
+#define STV2_HE_ADDR 0x14ac
+#define STV2_VS_ADDR 0x14ad
+#define STV2_VE_ADDR 0x14ae
+#define OEV1_HS_ADDR 0x14af
+#define OEV1_HE_ADDR 0x14b0
+#define OEV1_VS_ADDR 0x14b1
+#define OEV1_VE_ADDR 0x14b2
+#define OEV2_HS_ADDR 0x14b3
+#define OEV2_HE_ADDR 0x14b4
+#define OEV2_VS_ADDR 0x14b5
+#define OEV2_VE_ADDR 0x14b6
+#define OEV3_HS_ADDR 0x14b7
+#define OEV3_HE_ADDR 0x14b8
+#define OEV3_VS_ADDR 0x14b9
+#define OEV3_VE_ADDR 0x14ba
+#define LCD_PWR_ADDR 0x14bb
+#define LCD_PWM0_LO_ADDR 0x14bc
+#define LCD_PWM0_HI_ADDR 0x14bd
+#define LCD_PWM1_LO_ADDR 0x14be
+#define LCD_PWM1_HI_ADDR 0x14bf
+#define INV_CNT_ADDR 0x14c0
+#define TCON_MISC_SEL_ADDR 0x14c1
+#define DUAL_PORT_CNTL_ADDR 0x14c2
+#define MLVDS_CONTROL 0x14c3
+#define MLVDS_RESET_PATTERN_HI 0x14c4
+#define MLVDS_RESET_PATTERN_LO 0x14c5
+#define MLVDS_RESET_PATTERN_EXT 0x14c6
+#define MLVDS_CONFIG_HI 0x14c7
+#define MLVDS_CONFIG_LO 0x14c8
+#define TCON_DOUBLE_CTL 0x14c9
+#define TCON_PATTERN_HI 0x14ca
+#define TCON_PATTERN_LO 0x14cb
+#define TCON_CONTROL_HI 0x14cc
+#define TCON_CONTROL_LO 0x14cd
+#define LVDS_BLANK_DATA_HI 0x14ce
+#define LVDS_BLANK_DATA_LO 0x14cf
+#define LVDS_PACK_CNTL_ADDR 0x14d0
+#define DE_HS_ADDR 0x14d1
+#define DE_HE_ADDR 0x14d2
+#define DE_VS_ADDR 0x14d3
+#define DE_VE_ADDR 0x14d4
+#define HSYNC_HS_ADDR 0x14d5
+#define HSYNC_HE_ADDR 0x14d6
+#define HSYNC_VS_ADDR 0x14d7
+#define HSYNC_VE_ADDR 0x14d8
+#define VSYNC_HS_ADDR 0x14d9
+#define VSYNC_HE_ADDR 0x14da
+#define VSYNC_VS_ADDR 0x14db
+#define VSYNC_VE_ADDR 0x14dc
+#define LCD_MCU_CTL 0x14dd
+#define LCD_MCU_DATA_0 0x14de
+#define LCD_MCU_DATA_1 0x14df
+#define LVDS_GEN_CNTL 0x14e0
+#define LVDS_PHY_CNTL0 0x14e1
+#define LVDS_PHY_CNTL1 0x14e2
+#define LVDS_PHY_CNTL2 0x14e3
+#define LVDS_PHY_CNTL3 0x14e4
+#define LVDS_PHY_CNTL4 0x14e5
+#define LVDS_PHY_CNTL5 0x14e6
+#define LVDS_SRG_TEST 0x14e8
+#define LVDS_BIST_MUX0 0x14e9
+#define LVDS_BIST_MUX1 0x14ea
+#define LVDS_BIST_FIXED0 0x14eb
+#define LVDS_BIST_FIXED1 0x14ec
+#define LVDS_BIST_CNTL0 0x14ed
+#define LVDS_CLKB_CLKA 0x14ee
+#define LVDS_PHY_CLK_CNTL 0x14ef
+#define LVDS_SER_EN 0x14f0
+#define LVDS_PHY_CNTL6 0x14f1
+#define LVDS_PHY_CNTL7 0x14f2
+#define LVDS_PHY_CNTL8 0x14f3
+#define MLVDS_CLK_CTL0_HI 0x14f4
+#define MLVDS_CLK_CTL0_LO 0x14f5
+#define MLVDS_DUAL_GATE_WR_START 0x14f6
+#define MLVDS_DUAL_GATE_WR_END 0x14f7
+#define MLVDS_DUAL_GATE_RD_START 0x14f8
+#define MLVDS_DUAL_GATE_RD_END 0x14f9
+#define MLVDS_SECOND_RESET_CTL 0x14fa
+#define MLVDS_DUAL_GATE_CTL_HI 0x14fb
+#define MLVDS_DUAL_GATE_CTL_LO 0x14fc
+#define MLVDS_RESET_CONFIG_HI 0x14fd
+#define MLVDS_RESET_CONFIG_LO 0x14fe
+#define VPU_OSD1_MMC_CTRL 0x2701
+#define VPU_OSD2_MMC_CTRL 0x2702
+#define VPU_VD1_MMC_CTRL 0x2703
+#define VPU_VD2_MMC_CTRL 0x2704
+#define VPU_DI_IF1_MMC_CTRL 0x2705
+#define VPU_DI_MEM_MMC_CTRL 0x2706
+#define VPU_DI_INP_MMC_CTRL 0x2707
+#define VPU_DI_MTNRD_MMC_CTRL 0x2708
+#define VPU_DI_CHAN2_MMC_CTRL 0x2709
+#define VPU_DI_MTNWR_MMC_CTRL 0x270a
+#define VPU_DI_NRWR_MMC_CTRL 0x270b
+#define VPU_DI_DIWR_MMC_CTRL 0x270c
+#define VPU_VDIN0_MMC_CTRL 0x270d
+#define VPU_VDIN1_MMC_CTRL 0x270e
+#define VPU_BT656_MMC_CTRL 0x270f
+#define VPU_TVD3D_MMC_CTRL 0x2710
+#define VPU_TVDVBI_MMC_CTRL 0x2711
+#define VPU_TVDVBI_VSLATCH_ADDR 0x2712
+#define VPU_TVDVBI_WRRSP_ADDR 0x2713
+#define VPU_VDIN_PRE_ARB_CTRL 0x2714
+#define VPU_VDISP_PRE_ARB_CTRL 0x2715
+#define VPU_VPUARB2_PRE_ARB_CTRL 0x2716
+#define VPU_OSD3_MMC_CTRL 0x2717
+#define VPU_OSD4_MMC_CTRL 0x2718
+#define VPU_VD3_MMC_CTRL 0x2719
+#define VPU_VIU_VENC_MUX_CTRL 0x271a
+#define		VIU1_SEL_VENC_MASK	0x3
+#define		VIU1_SEL_VENC_ENCL	0
+#define		VIU1_SEL_VENC_ENCI	1
+#define		VIU1_SEL_VENC_ENCP	2
+#define		VIU1_SEL_VENC_ENCT	3
+#define		VIU2_SEL_VENC_MASK	0xc
+#define		VIU2_SEL_VENC_ENCL	0
+#define		VIU2_SEL_VENC_ENCI	(1 << 2)
+#define		VIU2_SEL_VENC_ENCP	(2 << 2)
+#define		VIU2_SEL_VENC_ENCT	(3 << 2)
+#define VPU_HDMI_SETTING 0x271b
+#define ENCI_INFO_READ 0x271c
+#define ENCP_INFO_READ 0x271d
+#define ENCT_INFO_READ 0x271e
+#define ENCL_INFO_READ 0x271f
+#define VPU_SW_RESET 0x2720
+#define VPU_D2D3_MMC_CTRL 0x2721
+#define VPU_CONT_MMC_CTRL 0x2722
+#define VPU_CLK_GATE 0x2723
+#define VPU_RDMA_MMC_CTRL 0x2724
+#define VPU_MEM_PD_REG0 0x2725
+#define VPU_MEM_PD_REG1 0x2726
+#define VPU_HDMI_DATA_OVR 0x2727
+#define VPU_PROT1_MMC_CTRL 0x2728
+#define VPU_PROT2_MMC_CTRL 0x2729
+#define VPU_PROT3_MMC_CTRL 0x272a
+#define VPU_ARB4_V1_MMC_CTRL 0x272b
+#define VPU_ARB4_V2_MMC_CTRL 0x272c
+#define VPU_VPU_PWM_V0 0x2730
+#define VPU_VPU_PWM_V1 0x2731
+#define VPU_VPU_PWM_V2 0x2732
+#define VPU_VPU_PWM_V3 0x2733
+#define VPU_VPU_PWM_H0 0x2734
+#define VPU_VPU_PWM_H1 0x2735
+#define VPU_VPU_PWM_H2 0x2736
+#define VPU_VPU_PWM_H3 0x2737
+#define VPU_MISC_CTRL 0x2740
+#define VPU_ISP_GCLK_CTRL0 0x2741
+#define VPU_ISP_GCLK_CTRL1 0x2742
+#define VPU_VDIN_ASYNC_HOLD_CTRL 0x2743
+#define VPU_VDISP_ASYNC_HOLD_CTRL 0x2744
+#define VPU_VPUARB2_ASYNC_HOLD_CTRL 0x2745
+
+#define VPU_PROT1_CLK_GATE 0x2750
+#define VPU_PROT1_GEN_CNTL 0x2751
+#define VPU_PROT1_X_START_END 0x2752
+#define VPU_PROT1_Y_START_END 0x2753
+#define VPU_PROT1_Y_LEN_STEP 0x2754
+#define VPU_PROT1_RPT_LOOP 0x2755
+#define VPU_PROT1_RPT_PAT 0x2756
+#define VPU_PROT1_DDR 0x2757
+#define VPU_PROT1_RBUF_ROOM 0x2758
+#define VPU_PROT1_STAT_0 0x2759
+#define VPU_PROT1_STAT_1 0x275a
+#define VPU_PROT1_STAT_2 0x275b
+#define VPU_PROT1_REQ_ONOFF 0x275c
+#define VPU_PROT2_CLK_GATE 0x2760
+#define VPU_PROT2_GEN_CNTL 0x2761
+#define VPU_PROT2_X_START_END 0x2762
+#define VPU_PROT2_Y_START_END 0x2763
+#define VPU_PROT2_Y_LEN_STEP 0x2764
+#define VPU_PROT2_RPT_LOOP 0x2765
+#define VPU_PROT2_RPT_PAT 0x2766
+#define VPU_PROT2_DDR 0x2767
+#define VPU_PROT2_RBUF_ROOM 0x2768
+#define VPU_PROT2_STAT_0 0x2769
+#define VPU_PROT2_STAT_1 0x276a
+#define VPU_PROT2_STAT_2 0x276b
+#define VPU_PROT2_REQ_ONOFF 0x276c
+#define VPU_PROT3_CLK_GATE 0x2770
+#define VPU_PROT3_GEN_CNTL 0x2771
+#define VPU_PROT3_X_START_END 0x2772
+#define VPU_PROT3_Y_START_END 0x2773
+#define VPU_PROT3_Y_LEN_STEP 0x2774
+#define VPU_PROT3_RPT_LOOP 0x2775
+#define VPU_PROT3_RPT_PAT 0x2776
+#define VPU_PROT3_DDR 0x2777
+#define VPU_PROT3_RBUF_ROOM 0x2778
+#define VPU_PROT3_STAT_0 0x2779
+#define VPU_PROT3_STAT_1 0x277a
+#define VPU_PROT3_STAT_2 0x277b
+#define VPU_PROT3_REQ_ONOFF 0x277c
+
+/* osd super scale */
+#define OSDSR_HV_SIZEIN 0x3130
+#define OSDSR_CTRL_MODE 0x3131
+#define OSDSR_ABIC_HCOEF 0x3132
+#define OSDSR_YBIC_HCOEF 0x3133
+#define OSDSR_CBIC_HCOEF 0x3134
+#define OSDSR_ABIC_VCOEF 0x3135
+#define OSDSR_YBIC_VCOEF 0x3136
+#define OSDSR_CBIC_VCOEF 0x3137
+#define OSDSR_VAR_PARA 0x3138
+#define OSDSR_CONST_PARA 0x3139
+#define OSDSR_RKE_EXTWIN 0x313a
+#define OSDSR_UK_GRAD2DDIAG_TH_RATE 0x313b
+#define OSDSR_UK_GRAD2DDIAG_LIMIT 0x313c
+#define OSDSR_UK_GRAD2DADJA_TH_RATE 0x313d
+#define OSDSR_UK_GRAD2DADJA_LIMIT 0x313e
+#define OSDSR_UK_BST_GAIN 0x313f
+#define OSDSR_HVBLEND_TH 0x3140
+#define OSDSR_DEMO_WIND_TB 0x3141
+#define OSDSR_DEMO_WIND_LR 0x3142
+#define OSDSR_INT_BLANK_NUM 0x3143
+#define OSDSR_FRM_END_STAT 0x3144
+#define OSDSR_ABIC_HCOEF0 0x3145
+#define OSDSR_YBIC_HCOEF0 0x3146
+#define OSDSR_CBIC_HCOEF0 0x3147
+#define OSDSR_ABIC_VCOEF0 0x3148
+#define OSDSR_YBIC_VCOEF0 0x3149
+#define OSDSR_CBIC_VCOEF0 0x314a
+
+#endif /* __MESON_REGISTERS_H */
diff --git a/drivers/gpu/drm/meson/meson_vclk.c b/drivers/gpu/drm/meson/meson_vclk.c
new file mode 100644
index 000000000000..252cfd4b19b1
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_vclk.c
@@ -0,0 +1,167 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2015 Amlogic, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <drm/drmP.h>
+#include "meson_drv.h"
+#include "meson_vclk.h"
+
+/*
+ * VCLK is the "Pixel Clock" frequency generator from a dedicated PLL.
+ * We handle the following encodings :
+ * - CVBS 27MHz generator via the VCLK2 to the VENCI and VDAC blocks
+ *
+ * What is missing :
+ * - HDMI Pixel Clocks generation
+ */
+
+/* HHI Registers */
+#define HHI_VID_PLL_CLK_DIV	0x1a0 /* 0x68 offset in data sheet */
+#define VID_PLL_EN		BIT(19)
+#define VID_PLL_BYPASS		BIT(18)
+#define VID_PLL_PRESET		BIT(15)
+#define HHI_VIID_CLK_DIV	0x128 /* 0x4a offset in data sheet */
+#define VCLK2_DIV_MASK		0xff
+#define VCLK2_DIV_EN		BIT(16)
+#define VCLK2_DIV_RESET		BIT(17)
+#define CTS_VDAC_SEL_MASK	(0xf << 28)
+#define CTS_VDAC_SEL_SHIFT	28
+#define HHI_VIID_CLK_CNTL	0x12c /* 0x4b offset in data sheet */
+#define VCLK2_EN		BIT(19)
+#define VCLK2_SEL_MASK		(0x7 << 16)
+#define VCLK2_SEL_SHIFT		16
+#define VCLK2_SOFT_RESET	BIT(15)
+#define VCLK2_DIV1_EN		BIT(0)
+#define HHI_VID_CLK_DIV		0x164 /* 0x59 offset in data sheet */
+#define CTS_ENCI_SEL_MASK	(0xf << 28)
+#define CTS_ENCI_SEL_SHIFT	28
+#define HHI_VID_CLK_CNTL2	0x194 /* 0x65 offset in data sheet */
+#define CTS_ENCI_EN		BIT(0)
+#define CTS_VDAC_EN		BIT(4)
+
+#define HHI_VDAC_CNTL0		0x2F4 /* 0xbd offset in data sheet */
+#define HHI_VDAC_CNTL1		0x2F8 /* 0xbe offset in data sheet */
+
+#define HHI_HDMI_PLL_CNTL	0x320 /* 0xc8 offset in data sheet */
+#define HHI_HDMI_PLL_CNTL2	0x324 /* 0xc9 offset in data sheet */
+#define HHI_HDMI_PLL_CNTL3	0x328 /* 0xca offset in data sheet */
+#define HHI_HDMI_PLL_CNTL4	0x32C /* 0xcb offset in data sheet */
+#define HHI_HDMI_PLL_CNTL5	0x330 /* 0xcc offset in data sheet */
+#define HHI_HDMI_PLL_CNTL6	0x334 /* 0xcd offset in data sheet */
+
+#define HDMI_PLL_RESET		BIT(28)
+#define HDMI_PLL_LOCK		BIT(31)
+
+/*
+ * Setup VCLK2 for 27MHz, and enable clocks for ENCI and VDAC
+ *
+ * TOFIX: Refactor into table to also handle HDMI frequency and paths
+ */
+static void meson_venci_cvbs_clock_config(struct meson_drm *priv)
+{
+	unsigned int val;
+
+	/* Setup PLL to output 1.485GHz */
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-gxbb-vpu")) {
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL, 0x5800023d);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL2, 0x00404e00);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL3, 0x0d5c5091);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL4, 0x801da72c);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL5, 0x71486980);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL6, 0x00000e55);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL, 0x4800023d);
+	} else if (meson_vpu_is_compatible(priv, "amlogic,meson-gxm-vpu") ||
+		   meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu")) {
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL, 0x4000027b);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL2, 0x800cb300);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL3, 0xa6212844);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL4, 0x0c4d000c);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL5, 0x001fa729);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL6, 0x01a31500);
+
+		/* Reset PLL */
+		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL,
+					HDMI_PLL_RESET, HDMI_PLL_RESET);
+		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL,
+					HDMI_PLL_RESET, 0);
+	}
+
+	/* Poll for lock bit */
+	regmap_read_poll_timeout(priv->hhi, HHI_HDMI_PLL_CNTL, val,
+				 (val & HDMI_PLL_LOCK), 10, 0);
+
+	/* Disable VCLK2 */
+	regmap_update_bits(priv->hhi, HHI_VIID_CLK_CNTL, VCLK2_EN, 0);
+
+	/* Disable vid_pll output clock */
+	regmap_update_bits(priv->hhi, HHI_VID_PLL_CLK_DIV, VID_PLL_EN, 0);
+	regmap_update_bits(priv->hhi, HHI_VID_PLL_CLK_DIV, VID_PLL_PRESET, 0);
+	/* Enable vid_pll bypass to HDMI pll */
+	regmap_update_bits(priv->hhi, HHI_VID_PLL_CLK_DIV,
+				VID_PLL_BYPASS, VID_PLL_BYPASS);
+	/* Enable the vid_pll output clock */
+	regmap_update_bits(priv->hhi, HHI_VID_PLL_CLK_DIV,
+				VID_PLL_EN, VID_PLL_EN);
+
+	/* Setup the VCLK2 divider value to achieve 27MHz */
+	regmap_update_bits(priv->hhi, HHI_VIID_CLK_DIV,
+				VCLK2_DIV_MASK, (55 - 1));
+
+	/* select vid_pll for vclk2 */
+	regmap_update_bits(priv->hhi, HHI_VIID_CLK_CNTL,
+				VCLK2_SEL_MASK, (4 << VCLK2_SEL_SHIFT));
+	/* enable vclk2 gate */
+	regmap_update_bits(priv->hhi, HHI_VIID_CLK_CNTL, VCLK2_EN, VCLK2_EN);
+
+	/* select vclk_div1 for enci */
+	regmap_update_bits(priv->hhi, HHI_VID_CLK_DIV,
+				CTS_ENCI_SEL_MASK, (8 << CTS_ENCI_SEL_SHIFT));
+	/* select vclk_div1 for vdac */
+	regmap_update_bits(priv->hhi, HHI_VIID_CLK_DIV,
+				CTS_VDAC_SEL_MASK, (8 << CTS_VDAC_SEL_SHIFT));
+
+	/* release vclk2_div_reset and enable vclk2_div */
+	regmap_update_bits(priv->hhi, HHI_VIID_CLK_DIV,
+				VCLK2_DIV_EN | VCLK2_DIV_RESET, VCLK2_DIV_EN);
+
+	/* enable vclk2_div1 gate */
+	regmap_update_bits(priv->hhi, HHI_VIID_CLK_CNTL,
+				VCLK2_DIV1_EN, VCLK2_DIV1_EN);
+
+	/* reset vclk2 */
+	regmap_update_bits(priv->hhi, HHI_VIID_CLK_CNTL,
+				VCLK2_SOFT_RESET, VCLK2_SOFT_RESET);
+	regmap_update_bits(priv->hhi, HHI_VIID_CLK_CNTL,
+				VCLK2_SOFT_RESET, 0);
+
+	/* enable enci_clk */
+	regmap_update_bits(priv->hhi, HHI_VID_CLK_CNTL2,
+				CTS_ENCI_EN, CTS_ENCI_EN);
+	/* enable vdac_clk */
+	regmap_update_bits(priv->hhi, HHI_VID_CLK_CNTL2,
+				CTS_VDAC_EN, CTS_VDAC_EN);
+}
+
+void meson_vclk_setup(struct meson_drm *priv, unsigned int target,
+		      unsigned int freq)
+{
+	if (target == MESON_VCLK_TARGET_CVBS && freq == MESON_VCLK_CVBS)
+		meson_venci_cvbs_clock_config(priv);
+}
diff --git a/drivers/gpu/drm/meson/meson_vclk.h b/drivers/gpu/drm/meson/meson_vclk.h
new file mode 100644
index 000000000000..ec62735996de
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_vclk.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* Video Clock */
+
+#ifndef __MESON_VCLK_H
+#define __MESON_VCLK_H
+
+enum {
+	MESON_VCLK_TARGET_CVBS = 0,
+};
+
+/* 27MHz is the CVBS Pixel Clock */
+#define MESON_VCLK_CVBS	27000
+
+void meson_vclk_setup(struct meson_drm *priv, unsigned int target,
+		      unsigned int freq);
+
+#endif /* __MESON_VCLK_H */
diff --git a/drivers/gpu/drm/meson/meson_venc.c b/drivers/gpu/drm/meson/meson_venc.c
new file mode 100644
index 000000000000..d836b2274531
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_venc.c
@@ -0,0 +1,254 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2015 Amlogic, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <drm/drmP.h>
+#include "meson_drv.h"
+#include "meson_venc.h"
+#include "meson_vpp.h"
+#include "meson_vclk.h"
+#include "meson_registers.h"
+
+/*
+ * VENC Handle the pixels encoding to the output formats.
+ * We handle the following encodings :
+ * - CVBS Encoding via the ENCI encoder and VDAC digital to analog converter
+ *
+ * What is missing :
+ * - TMDS/HDMI Encoding via ENCI_DIV and ENCP
+ * - Setup of more clock rates for HDMI modes
+ * - LCD Panel encoding via ENCL
+ * - TV Panel encoding via ENCT
+ */
+
+struct meson_cvbs_enci_mode meson_cvbs_enci_pal = {
+	.mode_tag = MESON_VENC_MODE_CVBS_PAL,
+	.hso_begin = 3,
+	.hso_end = 129,
+	.vso_even = 3,
+	.vso_odd = 260,
+	.macv_max_amp = 7,
+	.video_prog_mode = 0xff,
+	.video_mode = 0x13,
+	.sch_adjust = 0x28,
+	.yc_delay = 0x343,
+	.pixel_start = 251,
+	.pixel_end = 1691,
+	.top_field_line_start = 22,
+	.top_field_line_end = 310,
+	.bottom_field_line_start = 23,
+	.bottom_field_line_end = 311,
+	.video_saturation = 9,
+	.video_contrast = 0,
+	.video_brightness = 0,
+	.video_hue = 0,
+	.analog_sync_adj = 0x8080,
+};
+
+struct meson_cvbs_enci_mode meson_cvbs_enci_ntsc = {
+	.mode_tag = MESON_VENC_MODE_CVBS_NTSC,
+	.hso_begin = 5,
+	.hso_end = 129,
+	.vso_even = 3,
+	.vso_odd = 260,
+	.macv_max_amp = 0xb,
+	.video_prog_mode = 0xf0,
+	.video_mode = 0x8,
+	.sch_adjust = 0x20,
+	.yc_delay = 0x333,
+	.pixel_start = 227,
+	.pixel_end = 1667,
+	.top_field_line_start = 18,
+	.top_field_line_end = 258,
+	.bottom_field_line_start = 19,
+	.bottom_field_line_end = 259,
+	.video_saturation = 18,
+	.video_contrast = 3,
+	.video_brightness = 0,
+	.video_hue = 0,
+	.analog_sync_adj = 0x9c00,
+};
+
+void meson_venci_cvbs_mode_set(struct meson_drm *priv,
+			       struct meson_cvbs_enci_mode *mode)
+{
+	if (mode->mode_tag == priv->venc.current_mode)
+		return;
+
+	/* CVBS Filter settings */
+	writel_relaxed(0x12, priv->io_base + _REG(ENCI_CFILT_CTRL));
+	writel_relaxed(0x12, priv->io_base + _REG(ENCI_CFILT_CTRL2));
+
+	/* Digital Video Select : Interlace, clk27 clk, external */
+	writel_relaxed(0, priv->io_base + _REG(VENC_DVI_SETTING));
+
+	/* Reset Video Mode */
+	writel_relaxed(0, priv->io_base + _REG(ENCI_VIDEO_MODE));
+	writel_relaxed(0, priv->io_base + _REG(ENCI_VIDEO_MODE_ADV));
+
+	/* Horizontal sync signal output */
+	writel_relaxed(mode->hso_begin,
+			priv->io_base + _REG(ENCI_SYNC_HSO_BEGIN));
+	writel_relaxed(mode->hso_end,
+			priv->io_base + _REG(ENCI_SYNC_HSO_END));
+
+	/* Vertical Sync lines */
+	writel_relaxed(mode->vso_even,
+			priv->io_base + _REG(ENCI_SYNC_VSO_EVNLN));
+	writel_relaxed(mode->vso_odd,
+			priv->io_base + _REG(ENCI_SYNC_VSO_ODDLN));
+
+	/* Macrovision max amplitude change */
+	writel_relaxed(0x8100 + mode->macv_max_amp,
+			priv->io_base + _REG(ENCI_MACV_MAX_AMP));
+
+	/* Video mode */
+	writel_relaxed(mode->video_prog_mode,
+			priv->io_base + _REG(VENC_VIDEO_PROG_MODE));
+	writel_relaxed(mode->video_mode,
+			priv->io_base + _REG(ENCI_VIDEO_MODE));
+
+	/* Advanced Video Mode :
+	 * Demux shifting 0x2
+	 * Blank line end at line17/22
+	 * High bandwidth Luma Filter
+	 * Low bandwidth Chroma Filter
+	 * Bypass luma low pass filter
+	 * No macrovision on CSYNC
+	 */
+	writel_relaxed(0x26, priv->io_base + _REG(ENCI_VIDEO_MODE_ADV));
+
+	writel(mode->sch_adjust, priv->io_base + _REG(ENCI_VIDEO_SCH));
+
+	/* Sync mode : MASTER Master mode, free run, send HSO/VSO out */
+	writel_relaxed(0x07, priv->io_base + _REG(ENCI_SYNC_MODE));
+
+	/* 0x3 Y, C, and Component Y delay */
+	writel_relaxed(mode->yc_delay, priv->io_base + _REG(ENCI_YC_DELAY));
+
+	/* Timings */
+	writel_relaxed(mode->pixel_start,
+			priv->io_base + _REG(ENCI_VFIFO2VD_PIXEL_START));
+	writel_relaxed(mode->pixel_end,
+			priv->io_base + _REG(ENCI_VFIFO2VD_PIXEL_END));
+
+	writel_relaxed(mode->top_field_line_start,
+			priv->io_base + _REG(ENCI_VFIFO2VD_LINE_TOP_START));
+	writel_relaxed(mode->top_field_line_end,
+			priv->io_base + _REG(ENCI_VFIFO2VD_LINE_TOP_END));
+
+	writel_relaxed(mode->bottom_field_line_start,
+			priv->io_base + _REG(ENCI_VFIFO2VD_LINE_BOT_START));
+	writel_relaxed(mode->bottom_field_line_end,
+			priv->io_base + _REG(ENCI_VFIFO2VD_LINE_BOT_END));
+
+	/* Internal Venc, Internal VIU Sync, Internal Vencoder */
+	writel_relaxed(0, priv->io_base + _REG(VENC_SYNC_ROUTE));
+
+	/* UNreset Interlaced TV Encoder */
+	writel_relaxed(0, priv->io_base + _REG(ENCI_DBG_PX_RST));
+
+	/* Enable Vfifo2vd, Y_Cb_Y_Cr select */
+	writel_relaxed(0x4e01, priv->io_base + _REG(ENCI_VFIFO2VD_CTL));
+
+	/* Power UP Dacs */
+	writel_relaxed(0, priv->io_base + _REG(VENC_VDAC_SETTING));
+
+	/* Video Upsampling */
+	writel_relaxed(0x0061, priv->io_base + _REG(VENC_UPSAMPLE_CTRL0));
+	writel_relaxed(0x4061, priv->io_base + _REG(VENC_UPSAMPLE_CTRL1));
+	writel_relaxed(0x5061, priv->io_base + _REG(VENC_UPSAMPLE_CTRL2));
+
+	/* Select Interlace Y DACs */
+	writel_relaxed(0, priv->io_base + _REG(VENC_VDAC_DACSEL0));
+	writel_relaxed(0, priv->io_base + _REG(VENC_VDAC_DACSEL1));
+	writel_relaxed(0, priv->io_base + _REG(VENC_VDAC_DACSEL2));
+	writel_relaxed(0, priv->io_base + _REG(VENC_VDAC_DACSEL3));
+	writel_relaxed(0, priv->io_base + _REG(VENC_VDAC_DACSEL4));
+	writel_relaxed(0, priv->io_base + _REG(VENC_VDAC_DACSEL5));
+
+	/* Select ENCI for VIU */
+	meson_vpp_setup_mux(priv, MESON_VIU_VPP_MUX_ENCI);
+
+	/* Enable ENCI FIFO */
+	writel_relaxed(0x2000, priv->io_base + _REG(VENC_VDAC_FIFO_CTRL));
+
+	/* Select ENCI DACs 0, 1, 4, and 5 */
+	writel_relaxed(0x11, priv->io_base + _REG(ENCI_DACSEL_0));
+	writel_relaxed(0x11, priv->io_base + _REG(ENCI_DACSEL_1));
+
+	/* Interlace video enable */
+	writel_relaxed(1, priv->io_base + _REG(ENCI_VIDEO_EN));
+
+	/* Configure Video Saturation / Contrast / Brightness / Hue */
+	writel_relaxed(mode->video_saturation,
+			priv->io_base + _REG(ENCI_VIDEO_SAT));
+	writel_relaxed(mode->video_contrast,
+			priv->io_base + _REG(ENCI_VIDEO_CONT));
+	writel_relaxed(mode->video_brightness,
+			priv->io_base + _REG(ENCI_VIDEO_BRIGHT));
+	writel_relaxed(mode->video_hue,
+			priv->io_base + _REG(ENCI_VIDEO_HUE));
+
+	/* Enable DAC0 Filter */
+	writel_relaxed(0x1, priv->io_base + _REG(VENC_VDAC_DAC0_FILT_CTRL0));
+	writel_relaxed(0xfc48, priv->io_base + _REG(VENC_VDAC_DAC0_FILT_CTRL1));
+
+	/* 0 in Macrovision register 0 */
+	writel_relaxed(0, priv->io_base + _REG(ENCI_MACV_N0));
+
+	/* Analog Synchronization and color burst value adjust */
+	writel_relaxed(mode->analog_sync_adj,
+			priv->io_base + _REG(ENCI_SYNC_ADJ));
+
+	/* Setup 27MHz vclk2 for ENCI and VDAC */
+	meson_vclk_setup(priv, MESON_VCLK_TARGET_CVBS, MESON_VCLK_CVBS);
+
+	priv->venc.current_mode = mode->mode_tag;
+}
+
+/* Returns the current ENCI field polarity */
+unsigned int meson_venci_get_field(struct meson_drm *priv)
+{
+	return readl_relaxed(priv->io_base + _REG(ENCI_INFO_READ)) & BIT(29);
+}
+
+void meson_venc_enable_vsync(struct meson_drm *priv)
+{
+	writel_relaxed(2, priv->io_base + _REG(VENC_INTCTRL));
+}
+
+void meson_venc_disable_vsync(struct meson_drm *priv)
+{
+	writel_relaxed(0, priv->io_base + _REG(VENC_INTCTRL));
+}
+
+void meson_venc_init(struct meson_drm *priv)
+{
+	/* Disable all encoders */
+	writel_relaxed(0, priv->io_base + _REG(ENCI_VIDEO_EN));
+	writel_relaxed(0, priv->io_base + _REG(ENCP_VIDEO_EN));
+	writel_relaxed(0, priv->io_base + _REG(ENCL_VIDEO_EN));
+
+	/* Disable VSync IRQ */
+	meson_venc_disable_vsync(priv);
+
+	priv->venc.current_mode = MESON_VENC_MODE_NONE;
+}
diff --git a/drivers/gpu/drm/meson/meson_venc.h b/drivers/gpu/drm/meson/meson_venc.h
new file mode 100644
index 000000000000..77d4a7d82c44
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_venc.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * Video Encoders
+ * - ENCI : Interlace Video Encoder
+ * - ENCI_DVI : Interlace Video Encoder for DVI/HDMI
+ * - ENCP : Progressive Video Encoder
+ */
+
+#ifndef __MESON_VENC_H
+#define __MESON_VENC_H
+
+enum {
+	MESON_VENC_MODE_NONE = 0,
+	MESON_VENC_MODE_CVBS_PAL,
+	MESON_VENC_MODE_CVBS_NTSC,
+};
+
+struct meson_cvbs_enci_mode {
+	unsigned int mode_tag;
+	unsigned int hso_begin; /* HSO begin position */
+	unsigned int hso_end; /* HSO end position */
+	unsigned int vso_even; /* VSO even line */
+	unsigned int vso_odd; /* VSO odd line */
+	unsigned int macv_max_amp; /* Macrovision max amplitude */
+	unsigned int video_prog_mode;
+	unsigned int video_mode;
+	unsigned int sch_adjust;
+	unsigned int yc_delay;
+	unsigned int pixel_start;
+	unsigned int pixel_end;
+	unsigned int top_field_line_start;
+	unsigned int top_field_line_end;
+	unsigned int bottom_field_line_start;
+	unsigned int bottom_field_line_end;
+	unsigned int video_saturation;
+	unsigned int video_contrast;
+	unsigned int video_brightness;
+	unsigned int video_hue;
+	unsigned int analog_sync_adj;
+};
+
+/* CVBS Timings and Parameters */
+extern struct meson_cvbs_enci_mode meson_cvbs_enci_pal;
+extern struct meson_cvbs_enci_mode meson_cvbs_enci_ntsc;
+
+void meson_venci_cvbs_mode_set(struct meson_drm *priv,
+			       struct meson_cvbs_enci_mode *mode);
+unsigned int meson_venci_get_field(struct meson_drm *priv);
+
+void meson_venc_enable_vsync(struct meson_drm *priv);
+void meson_venc_disable_vsync(struct meson_drm *priv);
+
+void meson_venc_init(struct meson_drm *priv);
+
+#endif /* __MESON_VENC_H */
diff --git a/drivers/gpu/drm/meson/meson_venc_cvbs.c b/drivers/gpu/drm/meson/meson_venc_cvbs.c
new file mode 100644
index 000000000000..c809c085fd78
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_venc_cvbs.c
@@ -0,0 +1,293 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2015 Amlogic, Inc. All rights reserved.
+ * Copyright (C) 2014 Endless Mobile
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * Written by:
+ *     Jasper St. Pierre <jstpierre@mecheye.net>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_graph.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_atomic_helper.h>
+
+#include "meson_venc_cvbs.h"
+#include "meson_venc.h"
+#include "meson_registers.h"
+
+/* HHI VDAC Registers */
+#define HHI_VDAC_CNTL0		0x2F4 /* 0xbd offset in data sheet */
+#define HHI_VDAC_CNTL1		0x2F8 /* 0xbe offset in data sheet */
+
+struct meson_venc_cvbs {
+	struct drm_encoder	encoder;
+	struct drm_connector	connector;
+	struct meson_drm	*priv;
+};
+#define encoder_to_meson_venc_cvbs(x) \
+	container_of(x, struct meson_venc_cvbs, encoder)
+
+#define connector_to_meson_venc_cvbs(x) \
+	container_of(x, struct meson_venc_cvbs, connector)
+
+/* Supported Modes */
+
+struct meson_cvbs_mode meson_cvbs_modes[MESON_CVBS_MODES_COUNT] = {
+	{ /* PAL */
+		.enci = &meson_cvbs_enci_pal,
+		.mode = {
+			DRM_MODE("720x576i", DRM_MODE_TYPE_DRIVER, 13500,
+				 720, 732, 795, 864, 0, 576, 580, 586, 625, 0,
+				 DRM_MODE_FLAG_INTERLACE),
+			.vrefresh = 50,
+			.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+		},
+	},
+	{ /* NTSC */
+		.enci = &meson_cvbs_enci_ntsc,
+		.mode = {
+			DRM_MODE("720x480i", DRM_MODE_TYPE_DRIVER, 13500,
+				720, 739, 801, 858, 0, 480, 488, 494, 525, 0,
+				DRM_MODE_FLAG_INTERLACE),
+			.vrefresh = 60,
+			.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3,
+		},
+	},
+};
+
+/* Connector */
+
+static void meson_cvbs_connector_destroy(struct drm_connector *connector)
+{
+	drm_connector_cleanup(connector);
+}
+
+static enum drm_connector_status
+meson_cvbs_connector_detect(struct drm_connector *connector, bool force)
+{
+	/* FIXME: Add load-detect or jack-detect if possible */
+	return connector_status_connected;
+}
+
+static int meson_cvbs_connector_get_modes(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_display_mode *mode;
+	int i;
+
+	for (i = 0; i < MESON_CVBS_MODES_COUNT; ++i) {
+		struct meson_cvbs_mode *meson_mode = &meson_cvbs_modes[i];
+
+		mode = drm_mode_duplicate(dev, &meson_mode->mode);
+		if (!mode) {
+			DRM_ERROR("Failed to create a new display mode\n");
+			return 0;
+		}
+
+		drm_mode_probed_add(connector, mode);
+	}
+
+	return i;
+}
+
+static int meson_cvbs_connector_mode_valid(struct drm_connector *connector,
+					   struct drm_display_mode *mode)
+{
+	/* Validate the modes added in get_modes */
+	return MODE_OK;
+}
+
+static const struct drm_connector_funcs meson_cvbs_connector_funcs = {
+	.dpms			= drm_atomic_helper_connector_dpms,
+	.detect			= meson_cvbs_connector_detect,
+	.fill_modes		= drm_helper_probe_single_connector_modes,
+	.destroy		= meson_cvbs_connector_destroy,
+	.reset			= drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state	= drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
+};
+
+static const
+struct drm_connector_helper_funcs meson_cvbs_connector_helper_funcs = {
+	.get_modes	= meson_cvbs_connector_get_modes,
+	.mode_valid	= meson_cvbs_connector_mode_valid,
+};
+
+/* Encoder */
+
+static void meson_venc_cvbs_encoder_destroy(struct drm_encoder *encoder)
+{
+	drm_encoder_cleanup(encoder);
+}
+
+static const struct drm_encoder_funcs meson_venc_cvbs_encoder_funcs = {
+	.destroy        = meson_venc_cvbs_encoder_destroy,
+};
+
+static int meson_venc_cvbs_encoder_atomic_check(struct drm_encoder *encoder,
+					struct drm_crtc_state *crtc_state,
+					struct drm_connector_state *conn_state)
+{
+	int i;
+
+	for (i = 0; i < MESON_CVBS_MODES_COUNT; ++i) {
+		struct meson_cvbs_mode *meson_mode = &meson_cvbs_modes[i];
+
+		if (drm_mode_equal(&crtc_state->mode, &meson_mode->mode))
+			return 0;
+	}
+
+	return -EINVAL;
+}
+
+static void meson_venc_cvbs_encoder_disable(struct drm_encoder *encoder)
+{
+	struct meson_venc_cvbs *meson_venc_cvbs =
+					encoder_to_meson_venc_cvbs(encoder);
+	struct meson_drm *priv = meson_venc_cvbs->priv;
+
+	/* Disable CVBS VDAC */
+	regmap_write(priv->hhi, HHI_VDAC_CNTL0, 0);
+	regmap_write(priv->hhi, HHI_VDAC_CNTL1, 0);
+}
+
+static void meson_venc_cvbs_encoder_enable(struct drm_encoder *encoder)
+{
+	struct meson_venc_cvbs *meson_venc_cvbs =
+					encoder_to_meson_venc_cvbs(encoder);
+	struct meson_drm *priv = meson_venc_cvbs->priv;
+
+	/* VDAC0 source is not from ATV */
+	writel_bits_relaxed(BIT(5), 0, priv->io_base + _REG(VENC_VDAC_DACSEL0));
+
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-gxbb-vpu"))
+		regmap_write(priv->hhi, HHI_VDAC_CNTL0, 1);
+	else if (meson_vpu_is_compatible(priv, "amlogic,meson-gxm-vpu") ||
+		 meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu"))
+		regmap_write(priv->hhi, HHI_VDAC_CNTL0, 0xf0001);
+
+	regmap_write(priv->hhi, HHI_VDAC_CNTL1, 0);
+}
+
+static void meson_venc_cvbs_encoder_mode_set(struct drm_encoder *encoder,
+				   struct drm_display_mode *mode,
+				   struct drm_display_mode *adjusted_mode)
+{
+	struct meson_venc_cvbs *meson_venc_cvbs =
+					encoder_to_meson_venc_cvbs(encoder);
+	int i;
+
+	for (i = 0; i < MESON_CVBS_MODES_COUNT; ++i) {
+		struct meson_cvbs_mode *meson_mode = &meson_cvbs_modes[i];
+
+		if (drm_mode_equal(mode, &meson_mode->mode)) {
+			meson_venci_cvbs_mode_set(meson_venc_cvbs->priv,
+						  meson_mode->enci);
+			break;
+		}
+	}
+}
+
+static const struct drm_encoder_helper_funcs
+				meson_venc_cvbs_encoder_helper_funcs = {
+	.atomic_check	= meson_venc_cvbs_encoder_atomic_check,
+	.disable	= meson_venc_cvbs_encoder_disable,
+	.enable		= meson_venc_cvbs_encoder_enable,
+	.mode_set	= meson_venc_cvbs_encoder_mode_set,
+};
+
+static bool meson_venc_cvbs_connector_is_available(struct meson_drm *priv)
+{
+	struct device_node *ep, *remote;
+
+	/* CVBS VDAC output is on the first port, first endpoint */
+	ep = of_graph_get_endpoint_by_regs(priv->dev->of_node, 0, 0);
+	if (!ep)
+		return false;
+
+
+	/* If the endpoint node exists, consider it enabled */
+	remote = of_graph_get_remote_port(ep);
+	if (remote) {
+		of_node_put(ep);
+		return true;
+	}
+
+	of_node_put(ep);
+	of_node_put(remote);
+
+	return false;
+}
+
+int meson_venc_cvbs_create(struct meson_drm *priv)
+{
+	struct drm_device *drm = priv->drm;
+	struct meson_venc_cvbs *meson_venc_cvbs;
+	struct drm_connector *connector;
+	struct drm_encoder *encoder;
+	int ret;
+
+	if (!meson_venc_cvbs_connector_is_available(priv)) {
+		dev_info(drm->dev, "CVBS Output connector not available\n");
+		return -ENODEV;
+	}
+
+	meson_venc_cvbs = devm_kzalloc(priv->dev, sizeof(*meson_venc_cvbs),
+				       GFP_KERNEL);
+	if (!meson_venc_cvbs)
+		return -ENOMEM;
+
+	meson_venc_cvbs->priv = priv;
+	encoder = &meson_venc_cvbs->encoder;
+	connector = &meson_venc_cvbs->connector;
+
+	/* Connector */
+
+	drm_connector_helper_add(connector,
+				 &meson_cvbs_connector_helper_funcs);
+
+	ret = drm_connector_init(drm, connector, &meson_cvbs_connector_funcs,
+				 DRM_MODE_CONNECTOR_Composite);
+	if (ret) {
+		dev_err(priv->dev, "Failed to init CVBS connector\n");
+		return ret;
+	}
+
+	connector->interlace_allowed = 1;
+
+	/* Encoder */
+
+	drm_encoder_helper_add(encoder, &meson_venc_cvbs_encoder_helper_funcs);
+
+	ret = drm_encoder_init(drm, encoder, &meson_venc_cvbs_encoder_funcs,
+			       DRM_MODE_ENCODER_TVDAC, "meson_venc_cvbs");
+	if (ret) {
+		dev_err(priv->dev, "Failed to init CVBS encoder\n");
+		return ret;
+	}
+
+	encoder->possible_crtcs = BIT(0);
+
+	drm_mode_connector_attach_encoder(connector, encoder);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/meson/meson_venc_cvbs.h b/drivers/gpu/drm/meson/meson_venc_cvbs.h
new file mode 100644
index 000000000000..9256ccf9d931
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_venc_cvbs.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2014 Endless Mobile
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * Written by:
+ *     Jasper St. Pierre <jstpierre@mecheye.net>
+ */
+
+#ifndef __MESON_VENC_CVBS_H
+#define __MESON_VENC_CVBS_H
+
+#include "meson_drv.h"
+#include "meson_venc.h"
+
+struct meson_cvbs_mode {
+	struct meson_cvbs_enci_mode *enci;
+	struct drm_display_mode mode;
+};
+
+#define MESON_CVBS_MODES_COUNT	2
+
+/* Modes supported by the CVBS output */
+extern struct meson_cvbs_mode meson_cvbs_modes[MESON_CVBS_MODES_COUNT];
+
+int meson_venc_cvbs_create(struct meson_drm *priv);
+
+#endif /* __MESON_VENC_CVBS_H */
diff --git a/drivers/gpu/drm/meson/meson_viu.c b/drivers/gpu/drm/meson/meson_viu.c
new file mode 100644
index 000000000000..a6de8ba7af19
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_viu.c
@@ -0,0 +1,331 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2015 Amlogic, Inc. All rights reserved.
+ * Copyright (C) 2014 Endless Mobile
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <drm/drmP.h>
+#include "meson_drv.h"
+#include "meson_viu.h"
+#include "meson_vpp.h"
+#include "meson_venc.h"
+#include "meson_canvas.h"
+#include "meson_registers.h"
+
+/*
+ * VIU Handles the Pixel scanout and the basic Colorspace conversions
+ * We handle the following features :
+ * - OSD1 RGB565/RGB888/xRGB8888 scanout
+ * - RGB conversion to x/cb/cr
+ * - Progressive or Interlace buffer scanout
+ * - OSD1 Commit on Vsync
+ * - HDR OSD matrix for GXL/GXM
+ *
+ * What is missing :
+ * - BGR888/xBGR8888/BGRx8888/BGRx8888 modes
+ * - YUV4:2:2 Y0CbY1Cr scanout
+ * - Conversion to YUV 4:4:4 from 4:2:2 input
+ * - Colorkey Alpha matching
+ * - Big endian scanout
+ * - X/Y reverse scanout
+ * - Global alpha setup
+ * - OSD2 support, would need interlace switching on vsync
+ * - OSD1 full scaling to support TV overscan
+ */
+
+/* OSD csc defines */
+
+enum viu_matrix_sel_e {
+	VIU_MATRIX_OSD_EOTF = 0,
+	VIU_MATRIX_OSD,
+};
+
+enum viu_lut_sel_e {
+	VIU_LUT_OSD_EOTF = 0,
+	VIU_LUT_OSD_OETF,
+};
+
+#define COEFF_NORM(a) ((int)((((a) * 2048.0) + 1) / 2))
+#define MATRIX_5X3_COEF_SIZE 24
+
+#define EOTF_COEFF_NORM(a) ((int)((((a) * 4096.0) + 1) / 2))
+#define EOTF_COEFF_SIZE 10
+#define EOTF_COEFF_RIGHTSHIFT 1
+
+static int RGB709_to_YUV709l_coeff[MATRIX_5X3_COEF_SIZE] = {
+	0, 0, 0, /* pre offset */
+	COEFF_NORM(0.181873),	COEFF_NORM(0.611831),	COEFF_NORM(0.061765),
+	COEFF_NORM(-0.100251),	COEFF_NORM(-0.337249),	COEFF_NORM(0.437500),
+	COEFF_NORM(0.437500),	COEFF_NORM(-0.397384),	COEFF_NORM(-0.040116),
+	0, 0, 0, /* 10'/11'/12' */
+	0, 0, 0, /* 20'/21'/22' */
+	64, 512, 512, /* offset */
+	0, 0, 0 /* mode, right_shift, clip_en */
+};
+
+/*  eotf matrix: bypass */
+static int eotf_bypass_coeff[EOTF_COEFF_SIZE] = {
+	EOTF_COEFF_NORM(1.0),	EOTF_COEFF_NORM(0.0),	EOTF_COEFF_NORM(0.0),
+	EOTF_COEFF_NORM(0.0),	EOTF_COEFF_NORM(1.0),	EOTF_COEFF_NORM(0.0),
+	EOTF_COEFF_NORM(0.0),	EOTF_COEFF_NORM(0.0),	EOTF_COEFF_NORM(1.0),
+	EOTF_COEFF_RIGHTSHIFT /* right shift */
+};
+
+void meson_viu_set_osd_matrix(struct meson_drm *priv,
+			      enum viu_matrix_sel_e m_select,
+			      int *m, bool csc_on)
+{
+	if (m_select == VIU_MATRIX_OSD) {
+		/* osd matrix, VIU_MATRIX_0 */
+		writel(((m[0] & 0xfff) << 16) | (m[1] & 0xfff),
+			priv->io_base + _REG(VIU_OSD1_MATRIX_PRE_OFFSET0_1));
+		writel(m[2] & 0xfff,
+			priv->io_base + _REG(VIU_OSD1_MATRIX_PRE_OFFSET2));
+		writel(((m[3] & 0x1fff) << 16) | (m[4] & 0x1fff),
+			priv->io_base + _REG(VIU_OSD1_MATRIX_COEF00_01));
+		writel(((m[5] & 0x1fff) << 16) | (m[6] & 0x1fff),
+			priv->io_base + _REG(VIU_OSD1_MATRIX_COEF02_10));
+		writel(((m[7] & 0x1fff) << 16) | (m[8] & 0x1fff),
+			priv->io_base + _REG(VIU_OSD1_MATRIX_COEF11_12));
+		writel(((m[9] & 0x1fff) << 16) | (m[10] & 0x1fff),
+			priv->io_base + _REG(VIU_OSD1_MATRIX_COEF20_21));
+
+		if (m[21]) {
+			writel(((m[11] & 0x1fff) << 16) | (m[12] & 0x1fff),
+				priv->io_base +
+					_REG(VIU_OSD1_MATRIX_COEF22_30));
+			writel(((m[13] & 0x1fff) << 16) | (m[14] & 0x1fff),
+				priv->io_base +
+					_REG(VIU_OSD1_MATRIX_COEF31_32));
+			writel(((m[15] & 0x1fff) << 16) | (m[16] & 0x1fff),
+				priv->io_base +
+					_REG(VIU_OSD1_MATRIX_COEF40_41));
+			writel(m[17] & 0x1fff, priv->io_base +
+				_REG(VIU_OSD1_MATRIX_COLMOD_COEF42));
+		} else
+			writel((m[11] & 0x1fff) << 16, priv->io_base +
+				_REG(VIU_OSD1_MATRIX_COEF22_30));
+
+		writel(((m[18] & 0xfff) << 16) | (m[19] & 0xfff),
+			priv->io_base + _REG(VIU_OSD1_MATRIX_OFFSET0_1));
+		writel(m[20] & 0xfff,
+			priv->io_base + _REG(VIU_OSD1_MATRIX_OFFSET2));
+
+		writel_bits_relaxed(3 << 30, m[21] << 30,
+			priv->io_base + _REG(VIU_OSD1_MATRIX_COLMOD_COEF42));
+		writel_bits_relaxed(7 << 16, m[22] << 16,
+			priv->io_base + _REG(VIU_OSD1_MATRIX_COLMOD_COEF42));
+
+		/* 23 reserved for clipping control */
+		writel_bits_relaxed(BIT(0), csc_on ? BIT(0) : 0,
+			priv->io_base + _REG(VIU_OSD1_MATRIX_CTRL));
+		writel_bits_relaxed(BIT(1), 0,
+			priv->io_base + _REG(VIU_OSD1_MATRIX_CTRL));
+	} else if (m_select == VIU_MATRIX_OSD_EOTF) {
+		int i;
+
+		/* osd eotf matrix, VIU_MATRIX_OSD_EOTF */
+		for (i = 0; i < 5; i++)
+			writel(((m[i * 2] & 0x1fff) << 16) |
+				(m[i * 2 + 1] & 0x1fff), priv->io_base +
+				_REG(VIU_OSD1_EOTF_CTL + i + 1));
+
+		writel_bits_relaxed(BIT(30), csc_on ? BIT(30) : 0,
+			priv->io_base + _REG(VIU_OSD1_EOTF_CTL));
+		writel_bits_relaxed(BIT(31), csc_on ? BIT(31) : 0,
+			priv->io_base + _REG(VIU_OSD1_EOTF_CTL));
+	}
+}
+
+#define OSD_EOTF_LUT_SIZE 33
+#define OSD_OETF_LUT_SIZE 41
+
+void meson_viu_set_osd_lut(struct meson_drm *priv, enum viu_lut_sel_e lut_sel,
+			   unsigned int *r_map, unsigned int *g_map,
+			   unsigned int *b_map,
+			   bool csc_on)
+{
+	unsigned int addr_port;
+	unsigned int data_port;
+	unsigned int ctrl_port;
+	int i;
+
+	if (lut_sel == VIU_LUT_OSD_EOTF) {
+		addr_port = VIU_OSD1_EOTF_LUT_ADDR_PORT;
+		data_port = VIU_OSD1_EOTF_LUT_DATA_PORT;
+		ctrl_port = VIU_OSD1_EOTF_CTL;
+	} else if (lut_sel == VIU_LUT_OSD_OETF) {
+		addr_port = VIU_OSD1_OETF_LUT_ADDR_PORT;
+		data_port = VIU_OSD1_OETF_LUT_DATA_PORT;
+		ctrl_port = VIU_OSD1_OETF_CTL;
+	} else
+		return;
+
+	if (lut_sel == VIU_LUT_OSD_OETF) {
+		writel(0, priv->io_base + _REG(addr_port));
+
+		for (i = 0; i < 20; i++)
+			writel(r_map[i * 2] | (r_map[i * 2 + 1] << 16),
+				priv->io_base + _REG(data_port));
+
+		writel(r_map[OSD_OETF_LUT_SIZE - 1] | (g_map[0] << 16),
+			priv->io_base + _REG(data_port));
+
+		for (i = 0; i < 20; i++)
+			writel(g_map[i * 2 + 1] | (g_map[i * 2 + 2] << 16),
+				priv->io_base + _REG(data_port));
+
+		for (i = 0; i < 20; i++)
+			writel(b_map[i * 2] | (b_map[i * 2 + 1] << 16),
+				priv->io_base + _REG(data_port));
+
+		writel(b_map[OSD_OETF_LUT_SIZE - 1],
+			priv->io_base + _REG(data_port));
+
+		if (csc_on)
+			writel_bits_relaxed(0x7 << 29, 7 << 29,
+					    priv->io_base + _REG(ctrl_port));
+		else
+			writel_bits_relaxed(0x7 << 29, 0,
+					    priv->io_base + _REG(ctrl_port));
+	} else if (lut_sel == VIU_LUT_OSD_EOTF) {
+		writel(0, priv->io_base + _REG(addr_port));
+
+		for (i = 0; i < 20; i++)
+			writel(r_map[i * 2] | (r_map[i * 2 + 1] << 16),
+				priv->io_base + _REG(data_port));
+
+		writel(r_map[OSD_EOTF_LUT_SIZE - 1] | (g_map[0] << 16),
+			priv->io_base + _REG(data_port));
+
+		for (i = 0; i < 20; i++)
+			writel(g_map[i * 2 + 1] | (g_map[i * 2 + 2] << 16),
+				priv->io_base + _REG(data_port));
+
+		for (i = 0; i < 20; i++)
+			writel(b_map[i * 2] | (b_map[i * 2 + 1] << 16),
+				priv->io_base + _REG(data_port));
+
+		writel(b_map[OSD_EOTF_LUT_SIZE - 1],
+			priv->io_base + _REG(data_port));
+
+		if (csc_on)
+			writel_bits_relaxed(7 << 27, 7 << 27,
+					    priv->io_base + _REG(ctrl_port));
+		else
+			writel_bits_relaxed(7 << 27, 0,
+					    priv->io_base + _REG(ctrl_port));
+
+		writel_bits_relaxed(BIT(31), BIT(31),
+				    priv->io_base + _REG(ctrl_port));
+	}
+}
+
+/* eotf lut: linear */
+static unsigned int eotf_33_linear_mapping[OSD_EOTF_LUT_SIZE] = {
+	0x0000,	0x0200,	0x0400, 0x0600,
+	0x0800, 0x0a00, 0x0c00, 0x0e00,
+	0x1000, 0x1200, 0x1400, 0x1600,
+	0x1800, 0x1a00, 0x1c00, 0x1e00,
+	0x2000, 0x2200, 0x2400, 0x2600,
+	0x2800, 0x2a00, 0x2c00, 0x2e00,
+	0x3000, 0x3200, 0x3400, 0x3600,
+	0x3800, 0x3a00, 0x3c00, 0x3e00,
+	0x4000
+};
+
+/* osd oetf lut: linear */
+static unsigned int oetf_41_linear_mapping[OSD_OETF_LUT_SIZE] = {
+	0, 0, 0, 0,
+	0, 32, 64, 96,
+	128, 160, 196, 224,
+	256, 288, 320, 352,
+	384, 416, 448, 480,
+	512, 544, 576, 608,
+	640, 672, 704, 736,
+	768, 800, 832, 864,
+	896, 928, 960, 992,
+	1023, 1023, 1023, 1023,
+	1023
+};
+
+static void meson_viu_load_matrix(struct meson_drm *priv)
+{
+	/* eotf lut bypass */
+	meson_viu_set_osd_lut(priv, VIU_LUT_OSD_EOTF,
+			      eotf_33_linear_mapping, /* R */
+			      eotf_33_linear_mapping, /* G */
+			      eotf_33_linear_mapping, /* B */
+			      false);
+
+	/* eotf matrix bypass */
+	meson_viu_set_osd_matrix(priv, VIU_MATRIX_OSD_EOTF,
+				 eotf_bypass_coeff,
+				 false);
+
+	/* oetf lut bypass */
+	meson_viu_set_osd_lut(priv, VIU_LUT_OSD_OETF,
+			      oetf_41_linear_mapping, /* R */
+			      oetf_41_linear_mapping, /* G */
+			      oetf_41_linear_mapping, /* B */
+			      false);
+
+	/* osd matrix RGB709 to YUV709 limit */
+	meson_viu_set_osd_matrix(priv, VIU_MATRIX_OSD,
+				 RGB709_to_YUV709l_coeff,
+				 true);
+}
+
+void meson_viu_init(struct meson_drm *priv)
+{
+	uint32_t reg;
+
+	/* Disable OSDs */
+	writel_bits_relaxed(BIT(0) | BIT(21), 0,
+			priv->io_base + _REG(VIU_OSD1_CTRL_STAT));
+	writel_bits_relaxed(BIT(0) | BIT(21), 0,
+			priv->io_base + _REG(VIU_OSD2_CTRL_STAT));
+
+	/* On GXL/GXM, Use the 10bit HDR conversion matrix */
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-gxm-vpu") ||
+	    meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu"))
+		meson_viu_load_matrix(priv);
+
+	/* Initialize OSD1 fifo control register */
+	reg = BIT(0) |	/* Urgent DDR request priority */
+	      (4 << 5) | /* hold_fifo_lines */
+	      (3 << 10) | /* burst length 64 */
+	      (32 << 12) | /* fifo_depth_val: 32*8=256 */
+	      (2 << 22) | /* 4 words in 1 burst */
+	      (2 << 24);
+	writel_relaxed(reg, priv->io_base + _REG(VIU_OSD1_FIFO_CTRL_STAT));
+	writel_relaxed(reg, priv->io_base + _REG(VIU_OSD2_FIFO_CTRL_STAT));
+
+	/* Set OSD alpha replace value */
+	writel_bits_relaxed(0xff << OSD_REPLACE_SHIFT,
+			    0xff << OSD_REPLACE_SHIFT,
+			    priv->io_base + _REG(VIU_OSD1_CTRL_STAT2));
+	writel_bits_relaxed(0xff << OSD_REPLACE_SHIFT,
+			    0xff << OSD_REPLACE_SHIFT,
+			    priv->io_base + _REG(VIU_OSD2_CTRL_STAT2));
+
+	priv->viu.osd1_enabled = false;
+	priv->viu.osd1_commit = false;
+	priv->viu.osd1_interlace = false;
+}
diff --git a/drivers/gpu/drm/meson/meson_viu.h b/drivers/gpu/drm/meson/meson_viu.h
new file mode 100644
index 000000000000..073b1910bd1b
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_viu.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* Video Input Unit */
+
+#ifndef __MESON_VIU_H
+#define __MESON_VIU_H
+
+/* OSDx_BLKx_CFG */
+#define OSD_CANVAS_SEL		16
+
+#define OSD_ENDIANNESS_LE	BIT(15)
+#define OSD_ENDIANNESS_BE	(0)
+
+#define OSD_BLK_MODE_422	(0x03 << 8)
+#define OSD_BLK_MODE_16		(0x04 << 8)
+#define OSD_BLK_MODE_32		(0x05 << 8)
+#define OSD_BLK_MODE_24		(0x07 << 8)
+
+#define OSD_OUTPUT_COLOR_RGB	BIT(7)
+#define OSD_OUTPUT_COLOR_YUV	(0)
+
+#define OSD_COLOR_MATRIX_32_RGBA	(0x00 << 2)
+#define OSD_COLOR_MATRIX_32_ARGB	(0x01 << 2)
+#define OSD_COLOR_MATRIX_32_ABGR	(0x02 << 2)
+#define OSD_COLOR_MATRIX_32_BGRA	(0x03 << 2)
+
+#define OSD_COLOR_MATRIX_24_RGB		(0x00 << 2)
+
+#define OSD_COLOR_MATRIX_16_RGB655	(0x00 << 2)
+#define OSD_COLOR_MATRIX_16_RGB565	(0x04 << 2)
+
+#define OSD_INTERLACE_ENABLED	BIT(1)
+#define OSD_INTERLACE_ODD	BIT(0)
+#define OSD_INTERLACE_EVEN	(0)
+
+/* OSDx_CTRL_STAT */
+#define OSD_ENABLE		BIT(21)
+#define OSD_BLK0_ENABLE		BIT(0)
+
+#define OSD_GLOBAL_ALPHA_SHIFT	12
+
+/* OSDx_CTRL_STAT2 */
+#define OSD_REPLACE_EN		BIT(14)
+#define OSD_REPLACE_SHIFT	6
+
+void meson_viu_init(struct meson_drm *priv);
+
+#endif /* __MESON_VIU_H */
diff --git a/drivers/gpu/drm/meson/meson_vpp.c b/drivers/gpu/drm/meson/meson_vpp.c
new file mode 100644
index 000000000000..671909d8672e
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_vpp.c
@@ -0,0 +1,162 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ * Copyright (C) 2015 Amlogic, Inc. All rights reserved.
+ * Copyright (C) 2014 Endless Mobile
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <drm/drmP.h>
+#include "meson_drv.h"
+#include "meson_vpp.h"
+#include "meson_registers.h"
+
+/*
+ * VPP Handles all the Post Processing after the Scanout from the VIU
+ * We handle the following post processings :
+ * - Postblend : Blends the OSD1 only
+ *	We exclude OSD2, VS1, VS1 and Preblend output
+ * - Vertical OSD Scaler for OSD1 only, we disable vertical scaler and
+ *	use it only for interlace scanout
+ * - Intermediate FIFO with default Amlogic values
+ *
+ * What is missing :
+ * - Preblend for video overlay pre-scaling
+ * - OSD2 support for cursor framebuffer
+ * - Video pre-scaling before postblend
+ * - Full Vertical/Horizontal OSD scaling to support TV overscan
+ * - HDR conversion
+ */
+
+void meson_vpp_setup_mux(struct meson_drm *priv, unsigned int mux)
+{
+	writel(mux, priv->io_base + _REG(VPU_VIU_VENC_MUX_CTRL));
+}
+
+/*
+ * When the output is interlaced, the OSD must switch between
+ * each field using the INTERLACE_SEL_ODD (0) of VIU_OSD1_BLK0_CFG_W0
+ * at each vsync.
+ * But the vertical scaler can provide such funtionnality if
+ * is configured for 2:1 scaling with interlace options enabled.
+ */
+void meson_vpp_setup_interlace_vscaler_osd1(struct meson_drm *priv,
+					    struct drm_rect *input)
+{
+	writel_relaxed(BIT(3) /* Enable scaler */ |
+		       BIT(2), /* Select OSD1 */
+			priv->io_base + _REG(VPP_OSD_SC_CTRL0));
+
+	writel_relaxed(((drm_rect_width(input) - 1) << 16) |
+		       (drm_rect_height(input) - 1),
+			priv->io_base + _REG(VPP_OSD_SCI_WH_M1));
+	/* 2:1 scaling */
+	writel_relaxed(((input->x1) << 16) | (input->x2),
+			priv->io_base + _REG(VPP_OSD_SCO_H_START_END));
+	writel_relaxed(((input->y1 >> 1) << 16) | (input->y2 >> 1),
+			priv->io_base + _REG(VPP_OSD_SCO_V_START_END));
+
+	/* 2:1 scaling values */
+	writel_relaxed(BIT(16), priv->io_base + _REG(VPP_OSD_VSC_INI_PHASE));
+	writel_relaxed(BIT(25), priv->io_base + _REG(VPP_OSD_VSC_PHASE_STEP));
+
+	writel_relaxed(0, priv->io_base + _REG(VPP_OSD_HSC_CTRL0));
+
+	writel_relaxed((4 << 0) /* osd_vsc_bank_length */ |
+		       (4 << 3) /* osd_vsc_top_ini_rcv_num0 */ |
+		       (1 << 8) /* osd_vsc_top_rpt_p0_num0 */ |
+		       (6 << 11) /* osd_vsc_bot_ini_rcv_num0 */ |
+		       (2 << 16) /* osd_vsc_bot_rpt_p0_num0 */ |
+		       BIT(23)	/* osd_prog_interlace */ |
+		       BIT(24), /* Enable vertical scaler */
+			priv->io_base + _REG(VPP_OSD_VSC_CTRL0));
+}
+
+void meson_vpp_disable_interlace_vscaler_osd1(struct meson_drm *priv)
+{
+	writel_relaxed(0, priv->io_base + _REG(VPP_OSD_SC_CTRL0));
+	writel_relaxed(0, priv->io_base + _REG(VPP_OSD_VSC_CTRL0));
+	writel_relaxed(0, priv->io_base + _REG(VPP_OSD_HSC_CTRL0));
+}
+
+static unsigned int vpp_filter_coefs_4point_bspline[] = {
+	0x15561500, 0x14561600, 0x13561700, 0x12561800,
+	0x11551a00, 0x11541b00, 0x10541c00, 0x0f541d00,
+	0x0f531e00, 0x0e531f00, 0x0d522100, 0x0c522200,
+	0x0b522300, 0x0b512400, 0x0a502600, 0x0a4f2700,
+	0x094e2900, 0x084e2a00, 0x084d2b00, 0x074c2c01,
+	0x074b2d01, 0x064a2f01, 0x06493001, 0x05483201,
+	0x05473301, 0x05463401, 0x04453601, 0x04433702,
+	0x04423802, 0x03413a02, 0x03403b02, 0x033f3c02,
+	0x033d3d03
+};
+
+static void meson_vpp_write_scaling_filter_coefs(struct meson_drm *priv,
+						 const unsigned int *coefs,
+						 bool is_horizontal)
+{
+	int i;
+
+	writel_relaxed(is_horizontal ? BIT(8) : 0,
+			priv->io_base + _REG(VPP_OSD_SCALE_COEF_IDX));
+	for (i = 0; i < 33; i++)
+		writel_relaxed(coefs[i],
+				priv->io_base + _REG(VPP_OSD_SCALE_COEF));
+}
+
+void meson_vpp_init(struct meson_drm *priv)
+{
+	/* set dummy data default YUV black */
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu"))
+		writel_relaxed(0x108080, priv->io_base + _REG(VPP_DUMMY_DATA1));
+	else if (meson_vpu_is_compatible(priv, "amlogic,meson-gxm-vpu")) {
+		writel_bits_relaxed(0xff << 16, 0xff << 16,
+				    priv->io_base + _REG(VIU_MISC_CTRL1));
+		writel_relaxed(0x20000, priv->io_base + _REG(VPP_DOLBY_CTRL));
+		writel_relaxed(0x1020080,
+				priv->io_base + _REG(VPP_DUMMY_DATA1));
+	}
+
+	/* Initialize vpu fifo control registers */
+	writel_relaxed(readl_relaxed(priv->io_base + _REG(VPP_OFIFO_SIZE)) |
+			0x77f, priv->io_base + _REG(VPP_OFIFO_SIZE));
+	writel_relaxed(0x08080808, priv->io_base + _REG(VPP_HOLD_LINES));
+
+	/* Turn off preblend */
+	writel_bits_relaxed(VPP_PREBLEND_ENABLE, 0,
+			    priv->io_base + _REG(VPP_MISC));
+
+	/* Turn off POSTBLEND */
+	writel_bits_relaxed(VPP_POSTBLEND_ENABLE, 0,
+			    priv->io_base + _REG(VPP_MISC));
+
+	/* Force all planes off */
+	writel_bits_relaxed(VPP_OSD1_POSTBLEND | VPP_OSD2_POSTBLEND |
+			    VPP_VD1_POSTBLEND | VPP_VD2_POSTBLEND, 0,
+			    priv->io_base + _REG(VPP_MISC));
+
+	/* Disable Scalers */
+	writel_relaxed(0, priv->io_base + _REG(VPP_OSD_SC_CTRL0));
+	writel_relaxed(0, priv->io_base + _REG(VPP_OSD_VSC_CTRL0));
+	writel_relaxed(0, priv->io_base + _REG(VPP_OSD_HSC_CTRL0));
+
+	/* Write in the proper filter coefficients. */
+	meson_vpp_write_scaling_filter_coefs(priv,
+				vpp_filter_coefs_4point_bspline, false);
+	meson_vpp_write_scaling_filter_coefs(priv,
+				vpp_filter_coefs_4point_bspline, true);
+}
diff --git a/drivers/gpu/drm/meson/meson_vpp.h b/drivers/gpu/drm/meson/meson_vpp.h
new file mode 100644
index 000000000000..ede3b26e0f22
--- /dev/null
+++ b/drivers/gpu/drm/meson/meson_vpp.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2016 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* Video Post Process */
+
+#ifndef __MESON_VPP_H
+#define __MESON_VPP_H
+
+/* Mux VIU/VPP to ENCI */
+#define MESON_VIU_VPP_MUX_ENCI	0x5
+
+void meson_vpp_setup_mux(struct meson_drm *priv, unsigned int mux);
+
+void meson_vpp_setup_interlace_vscaler_osd1(struct meson_drm *priv,
+					    struct drm_rect *input);
+void meson_vpp_disable_interlace_vscaler_osd1(struct meson_drm *priv);
+
+void meson_vpp_init(struct meson_drm *priv);
+
+#endif /* __MESON_VPP_H */
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.c b/drivers/gpu/drm/mgag200/mgag200_drv.c
index 1443b3a34775..b0b874264f9d 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.c
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.c
@@ -82,9 +82,7 @@ static const struct file_operations mgag200_driver_fops = {
 	.unlocked_ioctl = drm_ioctl,
 	.mmap = mgag200_mmap,
 	.poll = drm_poll,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.read = drm_read,
 };
 
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index 6b21cb27e1cc..3a03ac4045d8 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -1658,12 +1658,6 @@ static struct drm_encoder *mga_connector_best_encoder(struct drm_connector
 	return NULL;
 }
 
-static enum drm_connector_status mga_vga_detect(struct drm_connector
-						   *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static void mga_connector_destroy(struct drm_connector *connector)
 {
 	struct mga_connector *mga_connector = to_mga_connector(connector);
@@ -1680,7 +1674,6 @@ static const struct drm_connector_helper_funcs mga_vga_connector_helper_funcs =
 
 static const struct drm_connector_funcs mga_vga_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.detect = mga_vga_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = mga_connector_destroy,
 };
diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c
index dcf7d11ac380..5e20220ef4c6 100644
--- a/drivers/gpu/drm/mgag200/mgag200_ttm.c
+++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c
@@ -230,6 +230,7 @@ struct ttm_bo_driver mgag200_bo_driver = {
 	.ttm_tt_populate = mgag200_ttm_tt_populate,
 	.ttm_tt_unpopulate = mgag200_ttm_tt_unpopulate,
 	.init_mem_type = mgag200_bo_init_mem_type,
+	.eviction_valuable = ttm_bo_eviction_valuable,
 	.evict_flags = mgag200_bo_evict_flags,
 	.move = NULL,
 	.verify_access = mgag200_bo_verify_access,
diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index 4e2806cf778c..028c24df2291 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -6,6 +6,8 @@ msm-y := \
 	adreno/adreno_gpu.o \
 	adreno/a3xx_gpu.o \
 	adreno/a4xx_gpu.o \
+	adreno/a5xx_gpu.o \
+	adreno/a5xx_power.o \
 	hdmi/hdmi.o \
 	hdmi/hdmi_audio.o \
 	hdmi/hdmi_bridge.o \
@@ -37,6 +39,7 @@ msm-y := \
 	mdp/mdp5/mdp5_irq.o \
 	mdp/mdp5/mdp5_mdss.o \
 	mdp/mdp5/mdp5_kms.o \
+	mdp/mdp5/mdp5_pipe.o \
 	mdp/mdp5/mdp5_plane.o \
 	mdp/mdp5/mdp5_smp.o \
 	msm_atomic.o \
@@ -48,6 +51,7 @@ msm-y := \
 	msm_gem_prime.o \
 	msm_gem_shrinker.o \
 	msm_gem_submit.o \
+	msm_gem_vma.o \
 	msm_gpu.o \
 	msm_iommu.o \
 	msm_perf.o \
diff --git a/drivers/gpu/drm/msm/adreno/a2xx.xml.h b/drivers/gpu/drm/msm/adreno/a2xx.xml.h
index fee24297fb92..4be092f911f9 100644
--- a/drivers/gpu/drm/msm/adreno/a2xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a2xx.xml.h
@@ -8,16 +8,17 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    398 bytes, from 2015-09-24 17:25:31)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    431 bytes, from 2016-04-26 17:56:44)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2016-02-10 17:07:21)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  32901 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  11518 bytes, from 2016-02-10 21:03:25)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  16166 bytes, from 2016-02-11 21:20:31)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83967 bytes, from 2016-02-10 17:07:21)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 109916 bytes, from 2016-02-20 18:44:48)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  32907 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  12025 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  22544 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 110765 bytes, from 2016-11-26 23:01:48)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml          (  90321 bytes, from 2016-11-28 16:50:05)
 - /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2015-09-24 17:30:00)
 
-Copyright (C) 2013-2015 by the following authors:
+Copyright (C) 2013-2016 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -206,12 +207,12 @@ enum a2xx_rb_copy_sample_select {
 };
 
 enum a2xx_rb_blend_opcode {
-	BLEND_DST_PLUS_SRC = 0,
-	BLEND_SRC_MINUS_DST = 1,
-	BLEND_MIN_DST_SRC = 2,
-	BLEND_MAX_DST_SRC = 3,
-	BLEND_DST_MINUS_SRC = 4,
-	BLEND_DST_PLUS_SRC_BIAS = 5,
+	BLEND2_DST_PLUS_SRC = 0,
+	BLEND2_SRC_MINUS_DST = 1,
+	BLEND2_MIN_DST_SRC = 2,
+	BLEND2_MAX_DST_SRC = 3,
+	BLEND2_DST_MINUS_SRC = 4,
+	BLEND2_DST_PLUS_SRC_BIAS = 5,
 };
 
 enum adreno_mmu_clnt_beh {
diff --git a/drivers/gpu/drm/msm/adreno/a3xx.xml.h b/drivers/gpu/drm/msm/adreno/a3xx.xml.h
index 27dabd5e57fb..a066c8b9eccd 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a3xx.xml.h
@@ -8,13 +8,14 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    398 bytes, from 2015-09-24 17:25:31)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    431 bytes, from 2016-04-26 17:56:44)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2016-02-10 17:07:21)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  32901 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  11518 bytes, from 2016-02-10 21:03:25)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  16166 bytes, from 2016-02-11 21:20:31)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83967 bytes, from 2016-02-10 17:07:21)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 109916 bytes, from 2016-02-20 18:44:48)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  32907 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  12025 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  22544 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 110765 bytes, from 2016-11-26 23:01:48)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml          (  90321 bytes, from 2016-11-28 16:50:05)
 - /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2015-09-24 17:30:00)
 
 Copyright (C) 2013-2016 by the following authors:
@@ -129,10 +130,14 @@ enum a3xx_tex_fmt {
 	TFMT_Z16_UNORM = 9,
 	TFMT_X8Z24_UNORM = 10,
 	TFMT_Z32_FLOAT = 11,
-	TFMT_NV12_UV_TILED = 17,
-	TFMT_NV12_Y_TILED = 19,
-	TFMT_NV12_UV = 21,
-	TFMT_NV12_Y = 23,
+	TFMT_UV_64X32 = 16,
+	TFMT_VU_64X32 = 17,
+	TFMT_Y_64X32 = 18,
+	TFMT_NV12_64X32 = 19,
+	TFMT_UV_LINEAR = 20,
+	TFMT_VU_LINEAR = 21,
+	TFMT_Y_LINEAR = 22,
+	TFMT_NV12_LINEAR = 23,
 	TFMT_I420_Y = 24,
 	TFMT_I420_U = 26,
 	TFMT_I420_V = 27,
@@ -525,14 +530,6 @@ enum a3xx_uche_perfcounter_select {
 	UCHE_UCHEPERF_ACTIVE_CYCLES = 20,
 };
 
-enum a3xx_rb_blend_opcode {
-	BLEND_DST_PLUS_SRC = 0,
-	BLEND_SRC_MINUS_DST = 1,
-	BLEND_DST_MINUS_SRC = 2,
-	BLEND_MIN_DST_SRC = 3,
-	BLEND_MAX_DST_SRC = 4,
-};
-
 enum a3xx_intp_mode {
 	SMOOTH = 0,
 	FLAT = 1,
@@ -1393,13 +1390,14 @@ static inline uint32_t A3XX_RB_COPY_CONTROL_MODE(enum adreno_rb_copy_control_mod
 {
 	return ((val) << A3XX_RB_COPY_CONTROL_MODE__SHIFT) & A3XX_RB_COPY_CONTROL_MODE__MASK;
 }
+#define A3XX_RB_COPY_CONTROL_MSAA_SRGB_DOWNSAMPLE		0x00000080
 #define A3XX_RB_COPY_CONTROL_FASTCLEAR__MASK			0x00000f00
 #define A3XX_RB_COPY_CONTROL_FASTCLEAR__SHIFT			8
 static inline uint32_t A3XX_RB_COPY_CONTROL_FASTCLEAR(uint32_t val)
 {
 	return ((val) << A3XX_RB_COPY_CONTROL_FASTCLEAR__SHIFT) & A3XX_RB_COPY_CONTROL_FASTCLEAR__MASK;
 }
-#define A3XX_RB_COPY_CONTROL_UNK12				0x00001000
+#define A3XX_RB_COPY_CONTROL_DEPTH32_RESOLVE			0x00001000
 #define A3XX_RB_COPY_CONTROL_GMEM_BASE__MASK			0xffffc000
 #define A3XX_RB_COPY_CONTROL_GMEM_BASE__SHIFT			14
 static inline uint32_t A3XX_RB_COPY_CONTROL_GMEM_BASE(uint32_t val)
@@ -1472,7 +1470,7 @@ static inline uint32_t A3XX_RB_DEPTH_CONTROL_ZFUNC(enum adreno_compare_func val)
 {
 	return ((val) << A3XX_RB_DEPTH_CONTROL_ZFUNC__SHIFT) & A3XX_RB_DEPTH_CONTROL_ZFUNC__MASK;
 }
-#define A3XX_RB_DEPTH_CONTROL_BF_ENABLE				0x00000080
+#define A3XX_RB_DEPTH_CONTROL_Z_CLAMP_ENABLE			0x00000080
 #define A3XX_RB_DEPTH_CONTROL_Z_TEST_ENABLE			0x80000000
 
 #define REG_A3XX_RB_DEPTH_CLEAR					0x00002101
diff --git a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
index fd266ed963b6..b999349b7d2d 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
@@ -41,7 +41,7 @@ extern bool hang_debug;
 
 static void a3xx_dump(struct msm_gpu *gpu);
 
-static void a3xx_me_init(struct msm_gpu *gpu)
+static bool a3xx_me_init(struct msm_gpu *gpu)
 {
 	struct msm_ringbuffer *ring = gpu->rb;
 
@@ -65,7 +65,7 @@ static void a3xx_me_init(struct msm_gpu *gpu)
 	OUT_RING(ring, 0x00000000);
 
 	gpu->funcs->flush(gpu);
-	gpu->funcs->idle(gpu);
+	return gpu->funcs->idle(gpu);
 }
 
 static int a3xx_hw_init(struct msm_gpu *gpu)
@@ -294,15 +294,20 @@ static int a3xx_hw_init(struct msm_gpu *gpu)
 	/* clear ME_HALT to start micro engine */
 	gpu_write(gpu, REG_AXXX_CP_ME_CNTL, 0);
 
-	a3xx_me_init(gpu);
-
-	return 0;
+	return a3xx_me_init(gpu) ? 0 : -EINVAL;
 }
 
 static void a3xx_recover(struct msm_gpu *gpu)
 {
+	int i;
+
 	adreno_dump_info(gpu);
 
+	for (i = 0; i < 8; i++) {
+		printk("CP_SCRATCH_REG%d: %u\n", i,
+			gpu_read(gpu, REG_AXXX_CP_SCRATCH_REG0 + i));
+	}
+
 	/* dump registers before resetting gpu, if enabled: */
 	if (hang_debug)
 		a3xx_dump(gpu);
@@ -330,17 +335,22 @@ static void a3xx_destroy(struct msm_gpu *gpu)
 	kfree(a3xx_gpu);
 }
 
-static void a3xx_idle(struct msm_gpu *gpu)
+static bool a3xx_idle(struct msm_gpu *gpu)
 {
 	/* wait for ringbuffer to drain: */
-	adreno_idle(gpu);
+	if (!adreno_idle(gpu))
+		return false;
 
 	/* then wait for GPU to finish: */
 	if (spin_until(!(gpu_read(gpu, REG_A3XX_RBBM_STATUS) &
-			A3XX_RBBM_STATUS_GPU_BUSY)))
+			A3XX_RBBM_STATUS_GPU_BUSY))) {
 		DRM_ERROR("%s: timeout waiting for GPU to idle!\n", gpu->name);
 
-	/* TODO maybe we need to reset GPU here to recover from hang? */
+		/* TODO maybe we need to reset GPU here to recover from hang? */
+		return false;
+	}
+
+	return true;
 }
 
 static irqreturn_t a3xx_irq(struct msm_gpu *gpu)
@@ -419,91 +429,13 @@ static void a3xx_dump(struct msm_gpu *gpu)
 }
 /* Register offset defines for A3XX */
 static const unsigned int a3xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_DEBUG, REG_AXXX_CP_DEBUG),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_RAM_WADDR, REG_AXXX_CP_ME_RAM_WADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_RAM_DATA, REG_AXXX_CP_ME_RAM_DATA),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_PFP_UCODE_DATA,
-			REG_A3XX_CP_PFP_UCODE_DATA),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_PFP_UCODE_ADDR,
-			REG_A3XX_CP_PFP_UCODE_ADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_WFI_PEND_CTR, REG_A3XX_CP_WFI_PEND_CTR),
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_AXXX_CP_RB_BASE),
+	REG_ADRENO_SKIP(REG_ADRENO_CP_RB_BASE_HI),
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, REG_AXXX_CP_RB_RPTR_ADDR),
+	REG_ADRENO_SKIP(REG_ADRENO_CP_RB_RPTR_ADDR_HI),
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_AXXX_CP_RB_RPTR),
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_AXXX_CP_RB_WPTR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_PROTECT_CTRL, REG_A3XX_CP_PROTECT_CTRL),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_CNTL, REG_AXXX_CP_ME_CNTL),
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_AXXX_CP_RB_CNTL),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_IB1_BASE, REG_AXXX_CP_IB1_BASE),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_IB1_BUFSZ, REG_AXXX_CP_IB1_BUFSZ),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_IB2_BASE, REG_AXXX_CP_IB2_BASE),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_IB2_BUFSZ, REG_AXXX_CP_IB2_BUFSZ),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_TIMESTAMP, REG_AXXX_CP_SCRATCH_REG0),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_RAM_RADDR, REG_AXXX_CP_ME_RAM_RADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_SCRATCH_ADDR, REG_AXXX_SCRATCH_ADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_SCRATCH_UMSK, REG_AXXX_SCRATCH_UMSK),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_ROQ_ADDR, REG_A3XX_CP_ROQ_ADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_ROQ_DATA, REG_A3XX_CP_ROQ_DATA),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_MERCIU_ADDR, REG_A3XX_CP_MERCIU_ADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_MERCIU_DATA, REG_A3XX_CP_MERCIU_DATA),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_MERCIU_DATA2, REG_A3XX_CP_MERCIU_DATA2),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_MEQ_ADDR, REG_A3XX_CP_MEQ_ADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_MEQ_DATA, REG_A3XX_CP_MEQ_DATA),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_HW_FAULT, REG_A3XX_CP_HW_FAULT),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_PROTECT_STATUS,
-			REG_A3XX_CP_PROTECT_STATUS),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_STATUS, REG_A3XX_RBBM_STATUS),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_CTL,
-			REG_A3XX_RBBM_PERFCTR_CTL),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_CMD0,
-			REG_A3XX_RBBM_PERFCTR_LOAD_CMD0),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_CMD1,
-			REG_A3XX_RBBM_PERFCTR_LOAD_CMD1),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_PWR_1_LO,
-			REG_A3XX_RBBM_PERFCTR_PWR_1_LO),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_INT_0_MASK, REG_A3XX_RBBM_INT_0_MASK),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_INT_0_STATUS,
-			REG_A3XX_RBBM_INT_0_STATUS),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_AHB_ERROR_STATUS,
-			REG_A3XX_RBBM_AHB_ERROR_STATUS),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_AHB_CMD, REG_A3XX_RBBM_AHB_CMD),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_INT_CLEAR_CMD,
-			REG_A3XX_RBBM_INT_CLEAR_CMD),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_CLOCK_CTL, REG_A3XX_RBBM_CLOCK_CTL),
-	REG_ADRENO_DEFINE(REG_ADRENO_VPC_DEBUG_RAM_SEL,
-			REG_A3XX_VPC_VPC_DEBUG_RAM_SEL),
-	REG_ADRENO_DEFINE(REG_ADRENO_VPC_DEBUG_RAM_READ,
-			REG_A3XX_VPC_VPC_DEBUG_RAM_READ),
-	REG_ADRENO_DEFINE(REG_ADRENO_VSC_SIZE_ADDRESS,
-			REG_A3XX_VSC_SIZE_ADDRESS),
-	REG_ADRENO_DEFINE(REG_ADRENO_VFD_CONTROL_0, REG_A3XX_VFD_CONTROL_0),
-	REG_ADRENO_DEFINE(REG_ADRENO_VFD_INDEX_MAX, REG_A3XX_VFD_INDEX_MAX),
-	REG_ADRENO_DEFINE(REG_ADRENO_SP_VS_PVT_MEM_ADDR_REG,
-			REG_A3XX_SP_VS_PVT_MEM_ADDR_REG),
-	REG_ADRENO_DEFINE(REG_ADRENO_SP_FS_PVT_MEM_ADDR_REG,
-			REG_A3XX_SP_FS_PVT_MEM_ADDR_REG),
-	REG_ADRENO_DEFINE(REG_ADRENO_SP_VS_OBJ_START_REG,
-			REG_A3XX_SP_VS_OBJ_START_REG),
-	REG_ADRENO_DEFINE(REG_ADRENO_SP_FS_OBJ_START_REG,
-			REG_A3XX_SP_FS_OBJ_START_REG),
-	REG_ADRENO_DEFINE(REG_ADRENO_PA_SC_AA_CONFIG, REG_A3XX_PA_SC_AA_CONFIG),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PM_OVERRIDE2,
-			REG_A3XX_RBBM_PM_OVERRIDE2),
-	REG_ADRENO_DEFINE(REG_ADRENO_SCRATCH_REG2, REG_AXXX_CP_SCRATCH_REG2),
-	REG_ADRENO_DEFINE(REG_ADRENO_SQ_GPR_MANAGEMENT,
-			REG_A3XX_SQ_GPR_MANAGEMENT),
-	REG_ADRENO_DEFINE(REG_ADRENO_SQ_INST_STORE_MANAGMENT,
-			REG_A3XX_SQ_INST_STORE_MANAGMENT),
-	REG_ADRENO_DEFINE(REG_ADRENO_TP0_CHICKEN, REG_A3XX_TP0_CHICKEN),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_RBBM_CTL, REG_A3XX_RBBM_RBBM_CTL),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_SW_RESET_CMD,
-			REG_A3XX_RBBM_SW_RESET_CMD),
-	REG_ADRENO_DEFINE(REG_ADRENO_UCHE_INVALIDATE0,
-			REG_A3XX_UCHE_CACHE_INVALIDATE0_REG),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_VALUE_LO,
-			REG_A3XX_RBBM_PERFCTR_LOAD_VALUE_LO),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_VALUE_HI,
-			REG_A3XX_RBBM_PERFCTR_LOAD_VALUE_HI),
 };
 
 static const struct adreno_gpu_funcs funcs = {
@@ -583,7 +515,7 @@ struct msm_gpu *a3xx_gpu_init(struct drm_device *dev)
 #endif
 	}
 
-	if (!gpu->mmu) {
+	if (!gpu->aspace) {
 		/* TODO we think it is possible to configure the GPU to
 		 * restrict access to VRAM carveout.  But the required
 		 * registers are unknown.  For now just bail out and
diff --git a/drivers/gpu/drm/msm/adreno/a4xx.xml.h b/drivers/gpu/drm/msm/adreno/a4xx.xml.h
index 3220b91f559a..4ce21b902779 100644
--- a/drivers/gpu/drm/msm/adreno/a4xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a4xx.xml.h
@@ -8,13 +8,14 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    398 bytes, from 2015-09-24 17:25:31)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    431 bytes, from 2016-04-26 17:56:44)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2016-02-10 17:07:21)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  32901 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  11518 bytes, from 2016-02-10 21:03:25)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  16166 bytes, from 2016-02-11 21:20:31)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83967 bytes, from 2016-02-10 17:07:21)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 109916 bytes, from 2016-02-20 18:44:48)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  32907 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  12025 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  22544 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 110765 bytes, from 2016-11-26 23:01:48)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml          (  90321 bytes, from 2016-11-28 16:50:05)
 - /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2015-09-24 17:30:00)
 
 Copyright (C) 2013-2016 by the following authors:
@@ -46,6 +47,9 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 enum a4xx_color_fmt {
 	RB4_A8_UNORM = 1,
 	RB4_R8_UNORM = 2,
+	RB4_R8_SNORM = 3,
+	RB4_R8_UINT = 4,
+	RB4_R8_SINT = 5,
 	RB4_R4G4B4A4_UNORM = 8,
 	RB4_R5G5B5A1_UNORM = 10,
 	RB4_R5G6B5_UNORM = 14,
@@ -89,17 +93,10 @@ enum a4xx_color_fmt {
 
 enum a4xx_tile_mode {
 	TILE4_LINEAR = 0,
+	TILE4_2 = 2,
 	TILE4_3 = 3,
 };
 
-enum a4xx_rb_blend_opcode {
-	BLEND_DST_PLUS_SRC = 0,
-	BLEND_SRC_MINUS_DST = 1,
-	BLEND_DST_MINUS_SRC = 2,
-	BLEND_MIN_DST_SRC = 3,
-	BLEND_MAX_DST_SRC = 4,
-};
-
 enum a4xx_vtx_fmt {
 	VFMT4_32_FLOAT = 1,
 	VFMT4_32_32_FLOAT = 2,
@@ -940,6 +937,7 @@ static inline uint32_t A4XX_RB_MODE_CONTROL_HEIGHT(uint32_t val)
 {
 	return ((val >> 5) << A4XX_RB_MODE_CONTROL_HEIGHT__SHIFT) & A4XX_RB_MODE_CONTROL_HEIGHT__MASK;
 }
+#define A4XX_RB_MODE_CONTROL_ENABLE_GMEM			0x00010000
 
 #define REG_A4XX_RB_RENDER_CONTROL				0x000020a1
 #define A4XX_RB_RENDER_CONTROL_BINNING_PASS			0x00000001
@@ -1043,7 +1041,7 @@ static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR(enum adreno_rb_b
 }
 #define A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK	0x000000e0
 #define A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT	5
-static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE(enum a4xx_rb_blend_opcode val)
+static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
 {
 	return ((val) << A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT) & A4XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK;
 }
@@ -1061,7 +1059,7 @@ static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR(enum adreno_rb
 }
 #define A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK	0x00e00000
 #define A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT	21
-static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE(enum a4xx_rb_blend_opcode val)
+static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
 {
 	return ((val) << A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT) & A4XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK;
 }
@@ -1073,12 +1071,18 @@ static inline uint32_t A4XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR(enum adreno_r
 }
 
 #define REG_A4XX_RB_BLEND_RED					0x000020f0
-#define A4XX_RB_BLEND_RED_UINT__MASK				0x0000ffff
+#define A4XX_RB_BLEND_RED_UINT__MASK				0x000000ff
 #define A4XX_RB_BLEND_RED_UINT__SHIFT				0
 static inline uint32_t A4XX_RB_BLEND_RED_UINT(uint32_t val)
 {
 	return ((val) << A4XX_RB_BLEND_RED_UINT__SHIFT) & A4XX_RB_BLEND_RED_UINT__MASK;
 }
+#define A4XX_RB_BLEND_RED_SINT__MASK				0x0000ff00
+#define A4XX_RB_BLEND_RED_SINT__SHIFT				8
+static inline uint32_t A4XX_RB_BLEND_RED_SINT(uint32_t val)
+{
+	return ((val) << A4XX_RB_BLEND_RED_SINT__SHIFT) & A4XX_RB_BLEND_RED_SINT__MASK;
+}
 #define A4XX_RB_BLEND_RED_FLOAT__MASK				0xffff0000
 #define A4XX_RB_BLEND_RED_FLOAT__SHIFT				16
 static inline uint32_t A4XX_RB_BLEND_RED_FLOAT(float val)
@@ -1095,12 +1099,18 @@ static inline uint32_t A4XX_RB_BLEND_RED_F32(float val)
 }
 
 #define REG_A4XX_RB_BLEND_GREEN					0x000020f2
-#define A4XX_RB_BLEND_GREEN_UINT__MASK				0x0000ffff
+#define A4XX_RB_BLEND_GREEN_UINT__MASK				0x000000ff
 #define A4XX_RB_BLEND_GREEN_UINT__SHIFT				0
 static inline uint32_t A4XX_RB_BLEND_GREEN_UINT(uint32_t val)
 {
 	return ((val) << A4XX_RB_BLEND_GREEN_UINT__SHIFT) & A4XX_RB_BLEND_GREEN_UINT__MASK;
 }
+#define A4XX_RB_BLEND_GREEN_SINT__MASK				0x0000ff00
+#define A4XX_RB_BLEND_GREEN_SINT__SHIFT				8
+static inline uint32_t A4XX_RB_BLEND_GREEN_SINT(uint32_t val)
+{
+	return ((val) << A4XX_RB_BLEND_GREEN_SINT__SHIFT) & A4XX_RB_BLEND_GREEN_SINT__MASK;
+}
 #define A4XX_RB_BLEND_GREEN_FLOAT__MASK				0xffff0000
 #define A4XX_RB_BLEND_GREEN_FLOAT__SHIFT			16
 static inline uint32_t A4XX_RB_BLEND_GREEN_FLOAT(float val)
@@ -1117,12 +1127,18 @@ static inline uint32_t A4XX_RB_BLEND_GREEN_F32(float val)
 }
 
 #define REG_A4XX_RB_BLEND_BLUE					0x000020f4
-#define A4XX_RB_BLEND_BLUE_UINT__MASK				0x0000ffff
+#define A4XX_RB_BLEND_BLUE_UINT__MASK				0x000000ff
 #define A4XX_RB_BLEND_BLUE_UINT__SHIFT				0
 static inline uint32_t A4XX_RB_BLEND_BLUE_UINT(uint32_t val)
 {
 	return ((val) << A4XX_RB_BLEND_BLUE_UINT__SHIFT) & A4XX_RB_BLEND_BLUE_UINT__MASK;
 }
+#define A4XX_RB_BLEND_BLUE_SINT__MASK				0x0000ff00
+#define A4XX_RB_BLEND_BLUE_SINT__SHIFT				8
+static inline uint32_t A4XX_RB_BLEND_BLUE_SINT(uint32_t val)
+{
+	return ((val) << A4XX_RB_BLEND_BLUE_SINT__SHIFT) & A4XX_RB_BLEND_BLUE_SINT__MASK;
+}
 #define A4XX_RB_BLEND_BLUE_FLOAT__MASK				0xffff0000
 #define A4XX_RB_BLEND_BLUE_FLOAT__SHIFT				16
 static inline uint32_t A4XX_RB_BLEND_BLUE_FLOAT(float val)
@@ -1139,12 +1155,18 @@ static inline uint32_t A4XX_RB_BLEND_BLUE_F32(float val)
 }
 
 #define REG_A4XX_RB_BLEND_ALPHA					0x000020f6
-#define A4XX_RB_BLEND_ALPHA_UINT__MASK				0x0000ffff
+#define A4XX_RB_BLEND_ALPHA_UINT__MASK				0x000000ff
 #define A4XX_RB_BLEND_ALPHA_UINT__SHIFT				0
 static inline uint32_t A4XX_RB_BLEND_ALPHA_UINT(uint32_t val)
 {
 	return ((val) << A4XX_RB_BLEND_ALPHA_UINT__SHIFT) & A4XX_RB_BLEND_ALPHA_UINT__MASK;
 }
+#define A4XX_RB_BLEND_ALPHA_SINT__MASK				0x0000ff00
+#define A4XX_RB_BLEND_ALPHA_SINT__SHIFT				8
+static inline uint32_t A4XX_RB_BLEND_ALPHA_SINT(uint32_t val)
+{
+	return ((val) << A4XX_RB_BLEND_ALPHA_SINT__SHIFT) & A4XX_RB_BLEND_ALPHA_SINT__MASK;
+}
 #define A4XX_RB_BLEND_ALPHA_FLOAT__MASK				0xffff0000
 #define A4XX_RB_BLEND_ALPHA_FLOAT__SHIFT			16
 static inline uint32_t A4XX_RB_BLEND_ALPHA_FLOAT(float val)
@@ -1348,7 +1370,7 @@ static inline uint32_t A4XX_RB_DEPTH_CONTROL_ZFUNC(enum adreno_compare_func val)
 {
 	return ((val) << A4XX_RB_DEPTH_CONTROL_ZFUNC__SHIFT) & A4XX_RB_DEPTH_CONTROL_ZFUNC__MASK;
 }
-#define A4XX_RB_DEPTH_CONTROL_BF_ENABLE				0x00000080
+#define A4XX_RB_DEPTH_CONTROL_Z_CLAMP_ENABLE			0x00000080
 #define A4XX_RB_DEPTH_CONTROL_EARLY_Z_DISABLE			0x00010000
 #define A4XX_RB_DEPTH_CONTROL_FORCE_FRAGZ_TO_FS			0x00020000
 #define A4XX_RB_DEPTH_CONTROL_Z_TEST_ENABLE			0x80000000
@@ -2177,11 +2199,23 @@ static inline uint32_t REG_A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1_REG(uint32_t i0)
 
 #define REG_A4XX_CP_DRAW_STATE_ADDR				0x00000232
 
-#define REG_A4XX_CP_PROTECT_REG_0				0x00000240
-
 static inline uint32_t REG_A4XX_CP_PROTECT(uint32_t i0) { return 0x00000240 + 0x1*i0; }
 
 static inline uint32_t REG_A4XX_CP_PROTECT_REG(uint32_t i0) { return 0x00000240 + 0x1*i0; }
+#define A4XX_CP_PROTECT_REG_BASE_ADDR__MASK			0x0001ffff
+#define A4XX_CP_PROTECT_REG_BASE_ADDR__SHIFT			0
+static inline uint32_t A4XX_CP_PROTECT_REG_BASE_ADDR(uint32_t val)
+{
+	return ((val) << A4XX_CP_PROTECT_REG_BASE_ADDR__SHIFT) & A4XX_CP_PROTECT_REG_BASE_ADDR__MASK;
+}
+#define A4XX_CP_PROTECT_REG_MASK_LEN__MASK			0x1f000000
+#define A4XX_CP_PROTECT_REG_MASK_LEN__SHIFT			24
+static inline uint32_t A4XX_CP_PROTECT_REG_MASK_LEN(uint32_t val)
+{
+	return ((val) << A4XX_CP_PROTECT_REG_MASK_LEN__SHIFT) & A4XX_CP_PROTECT_REG_MASK_LEN__MASK;
+}
+#define A4XX_CP_PROTECT_REG_TRAP_WRITE				0x20000000
+#define A4XX_CP_PROTECT_REG_TRAP_READ				0x40000000
 
 #define REG_A4XX_CP_PROTECT_CTRL				0x00000250
 
@@ -2272,7 +2306,7 @@ static inline uint32_t A4XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
 {
 	return ((val) << A4XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A4XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
 }
-#define A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0003fc00
+#define A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
 #define A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
 static inline uint32_t A4XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
 {
@@ -2420,7 +2454,7 @@ static inline uint32_t A4XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
 {
 	return ((val) << A4XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A4XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
 }
-#define A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0003fc00
+#define A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
 #define A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
 static inline uint32_t A4XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
 {
@@ -3117,6 +3151,8 @@ static inline uint32_t A4XX_TPL1_TP_TEX_COUNT_GS(uint32_t val)
 
 #define REG_A4XX_GRAS_CL_CLIP_CNTL				0x00002000
 #define A4XX_GRAS_CL_CLIP_CNTL_CLIP_DISABLE			0x00008000
+#define A4XX_GRAS_CL_CLIP_CNTL_ZNEAR_CLIP_DISABLE		0x00010000
+#define A4XX_GRAS_CL_CLIP_CNTL_ZFAR_CLIP_DISABLE		0x00020000
 #define A4XX_GRAS_CL_CLIP_CNTL_ZERO_GB_SCALE_Z			0x00400000
 
 #define REG_A4XX_GRAS_CLEAR_CNTL				0x00002003
@@ -3253,6 +3289,7 @@ static inline uint32_t A4XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH(float val)
 	return ((((int32_t)(val * 4.0))) << A4XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__SHIFT) & A4XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__MASK;
 }
 #define A4XX_GRAS_SU_MODE_CONTROL_POLY_OFFSET			0x00000800
+#define A4XX_GRAS_SU_MODE_CONTROL_MSAA_ENABLE			0x00002000
 #define A4XX_GRAS_SU_MODE_CONTROL_RENDERING_PASS		0x00100000
 
 #define REG_A4XX_GRAS_SC_CONTROL				0x0000207b
@@ -3670,6 +3707,8 @@ static inline uint32_t A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH(uint32_t val)
 #define REG_A4XX_PC_BINNING_COMMAND				0x00000d00
 #define A4XX_PC_BINNING_COMMAND_BINNING_ENABLE			0x00000001
 
+#define REG_A4XX_PC_TESSFACTOR_ADDR				0x00000d08
+
 #define REG_A4XX_PC_DRAWCALL_SETUP_OVERRIDE			0x00000d0c
 
 #define REG_A4XX_PC_PERFCTR_PC_SEL_0				0x00000d10
@@ -3690,6 +3729,20 @@ static inline uint32_t A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH(uint32_t val)
 
 #define REG_A4XX_PC_BIN_BASE					0x000021c0
 
+#define REG_A4XX_PC_VSTREAM_CONTROL				0x000021c2
+#define A4XX_PC_VSTREAM_CONTROL_SIZE__MASK			0x003f0000
+#define A4XX_PC_VSTREAM_CONTROL_SIZE__SHIFT			16
+static inline uint32_t A4XX_PC_VSTREAM_CONTROL_SIZE(uint32_t val)
+{
+	return ((val) << A4XX_PC_VSTREAM_CONTROL_SIZE__SHIFT) & A4XX_PC_VSTREAM_CONTROL_SIZE__MASK;
+}
+#define A4XX_PC_VSTREAM_CONTROL_N__MASK				0x07c00000
+#define A4XX_PC_VSTREAM_CONTROL_N__SHIFT			22
+static inline uint32_t A4XX_PC_VSTREAM_CONTROL_N(uint32_t val)
+{
+	return ((val) << A4XX_PC_VSTREAM_CONTROL_N__SHIFT) & A4XX_PC_VSTREAM_CONTROL_N__MASK;
+}
+
 #define REG_A4XX_PC_PRIM_VTX_CNTL				0x000021c4
 #define A4XX_PC_PRIM_VTX_CNTL_VAROUT__MASK			0x0000000f
 #define A4XX_PC_PRIM_VTX_CNTL_VAROUT__SHIFT			0
@@ -3752,12 +3805,8 @@ static inline uint32_t A4XX_PC_HS_PARAM_SPACING(enum a4xx_tess_spacing val)
 {
 	return ((val) << A4XX_PC_HS_PARAM_SPACING__SHIFT) & A4XX_PC_HS_PARAM_SPACING__MASK;
 }
-#define A4XX_PC_HS_PARAM_PRIMTYPE__MASK				0x01800000
-#define A4XX_PC_HS_PARAM_PRIMTYPE__SHIFT			23
-static inline uint32_t A4XX_PC_HS_PARAM_PRIMTYPE(enum adreno_pa_su_sc_draw val)
-{
-	return ((val) << A4XX_PC_HS_PARAM_PRIMTYPE__SHIFT) & A4XX_PC_HS_PARAM_PRIMTYPE__MASK;
-}
+#define A4XX_PC_HS_PARAM_CW					0x00800000
+#define A4XX_PC_HS_PARAM_CONNECTED				0x01000000
 
 #define REG_A4XX_VBIF_VERSION					0x00003000
 
diff --git a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
index d0d3c7baa8fe..511bc855cc7f 100644
--- a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
@@ -113,7 +113,7 @@ static void a4xx_enable_hwcg(struct msm_gpu *gpu)
 }
 
 
-static void a4xx_me_init(struct msm_gpu *gpu)
+static bool a4xx_me_init(struct msm_gpu *gpu)
 {
 	struct msm_ringbuffer *ring = gpu->rb;
 
@@ -137,7 +137,7 @@ static void a4xx_me_init(struct msm_gpu *gpu)
 	OUT_RING(ring, 0x00000000);
 
 	gpu->funcs->flush(gpu);
-	gpu->funcs->idle(gpu);
+	return gpu->funcs->idle(gpu);
 }
 
 static int a4xx_hw_init(struct msm_gpu *gpu)
@@ -292,15 +292,20 @@ static int a4xx_hw_init(struct msm_gpu *gpu)
 	/* clear ME_HALT to start micro engine */
 	gpu_write(gpu, REG_A4XX_CP_ME_CNTL, 0);
 
-	a4xx_me_init(gpu);
-
-	return 0;
+	return a4xx_me_init(gpu) ? 0 : -EINVAL;
 }
 
 static void a4xx_recover(struct msm_gpu *gpu)
 {
+	int i;
+
 	adreno_dump_info(gpu);
 
+	for (i = 0; i < 8; i++) {
+		printk("CP_SCRATCH_REG%d: %u\n", i,
+			gpu_read(gpu, REG_AXXX_CP_SCRATCH_REG0 + i));
+	}
+
 	/* dump registers before resetting gpu, if enabled: */
 	if (hang_debug)
 		a4xx_dump(gpu);
@@ -328,17 +333,21 @@ static void a4xx_destroy(struct msm_gpu *gpu)
 	kfree(a4xx_gpu);
 }
 
-static void a4xx_idle(struct msm_gpu *gpu)
+static bool a4xx_idle(struct msm_gpu *gpu)
 {
 	/* wait for ringbuffer to drain: */
-	adreno_idle(gpu);
+	if (!adreno_idle(gpu))
+		return false;
 
 	/* then wait for GPU to finish: */
 	if (spin_until(!(gpu_read(gpu, REG_A4XX_RBBM_STATUS) &
-					A4XX_RBBM_STATUS_GPU_BUSY)))
+					A4XX_RBBM_STATUS_GPU_BUSY))) {
 		DRM_ERROR("%s: timeout waiting for GPU to idle!\n", gpu->name);
+		/* TODO maybe we need to reset GPU here to recover from hang? */
+		return false;
+	}
 
-	/* TODO maybe we need to reset GPU here to recover from hang? */
+	return true;
 }
 
 static irqreturn_t a4xx_irq(struct msm_gpu *gpu)
@@ -460,87 +469,13 @@ static void a4xx_show(struct msm_gpu *gpu, struct seq_file *m)
 
 /* Register offset defines for A4XX, in order of enum adreno_regs */
 static const unsigned int a4xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_DEBUG, REG_A4XX_CP_DEBUG),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_RAM_WADDR, REG_A4XX_CP_ME_RAM_WADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_RAM_DATA, REG_A4XX_CP_ME_RAM_DATA),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_PFP_UCODE_DATA,
-			REG_A4XX_CP_PFP_UCODE_DATA),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_PFP_UCODE_ADDR,
-			REG_A4XX_CP_PFP_UCODE_ADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_WFI_PEND_CTR, REG_A4XX_CP_WFI_PEND_CTR),
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A4XX_CP_RB_BASE),
+	REG_ADRENO_SKIP(REG_ADRENO_CP_RB_BASE_HI),
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, REG_A4XX_CP_RB_RPTR_ADDR),
+	REG_ADRENO_SKIP(REG_ADRENO_CP_RB_RPTR_ADDR_HI),
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A4XX_CP_RB_RPTR),
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A4XX_CP_RB_WPTR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_PROTECT_CTRL, REG_A4XX_CP_PROTECT_CTRL),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_CNTL, REG_A4XX_CP_ME_CNTL),
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A4XX_CP_RB_CNTL),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_IB1_BASE, REG_A4XX_CP_IB1_BASE),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_IB1_BUFSZ, REG_A4XX_CP_IB1_BUFSZ),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_IB2_BASE, REG_A4XX_CP_IB2_BASE),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_IB2_BUFSZ, REG_A4XX_CP_IB2_BUFSZ),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_TIMESTAMP, REG_AXXX_CP_SCRATCH_REG0),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_ME_RAM_RADDR, REG_A4XX_CP_ME_RAM_RADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_ROQ_ADDR, REG_A4XX_CP_ROQ_ADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_ROQ_DATA, REG_A4XX_CP_ROQ_DATA),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_MERCIU_ADDR, REG_A4XX_CP_MERCIU_ADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_MERCIU_DATA, REG_A4XX_CP_MERCIU_DATA),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_MERCIU_DATA2, REG_A4XX_CP_MERCIU_DATA2),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_MEQ_ADDR, REG_A4XX_CP_MEQ_ADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_MEQ_DATA, REG_A4XX_CP_MEQ_DATA),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_HW_FAULT, REG_A4XX_CP_HW_FAULT),
-	REG_ADRENO_DEFINE(REG_ADRENO_CP_PROTECT_STATUS,
-			REG_A4XX_CP_PROTECT_STATUS),
-	REG_ADRENO_DEFINE(REG_ADRENO_SCRATCH_ADDR, REG_A4XX_CP_SCRATCH_ADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_SCRATCH_UMSK, REG_A4XX_CP_SCRATCH_UMASK),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_STATUS, REG_A4XX_RBBM_STATUS),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_CTL,
-			REG_A4XX_RBBM_PERFCTR_CTL),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_CMD0,
-			REG_A4XX_RBBM_PERFCTR_LOAD_CMD0),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_CMD1,
-			REG_A4XX_RBBM_PERFCTR_LOAD_CMD1),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_CMD2,
-			REG_A4XX_RBBM_PERFCTR_LOAD_CMD2),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_PWR_1_LO,
-			REG_A4XX_RBBM_PERFCTR_PWR_1_LO),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_INT_0_MASK, REG_A4XX_RBBM_INT_0_MASK),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_INT_0_STATUS,
-			REG_A4XX_RBBM_INT_0_STATUS),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_AHB_ERROR_STATUS,
-			REG_A4XX_RBBM_AHB_ERROR_STATUS),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_AHB_CMD, REG_A4XX_RBBM_AHB_CMD),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_CLOCK_CTL, REG_A4XX_RBBM_CLOCK_CTL),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_AHB_ME_SPLIT_STATUS,
-			REG_A4XX_RBBM_AHB_ME_SPLIT_STATUS),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_AHB_PFP_SPLIT_STATUS,
-			REG_A4XX_RBBM_AHB_PFP_SPLIT_STATUS),
-	REG_ADRENO_DEFINE(REG_ADRENO_VPC_DEBUG_RAM_SEL,
-			REG_A4XX_VPC_DEBUG_RAM_SEL),
-	REG_ADRENO_DEFINE(REG_ADRENO_VPC_DEBUG_RAM_READ,
-			REG_A4XX_VPC_DEBUG_RAM_READ),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_INT_CLEAR_CMD,
-			REG_A4XX_RBBM_INT_CLEAR_CMD),
-	REG_ADRENO_DEFINE(REG_ADRENO_VSC_SIZE_ADDRESS,
-			REG_A4XX_VSC_SIZE_ADDRESS),
-	REG_ADRENO_DEFINE(REG_ADRENO_VFD_CONTROL_0, REG_A4XX_VFD_CONTROL_0),
-	REG_ADRENO_DEFINE(REG_ADRENO_SP_VS_PVT_MEM_ADDR_REG,
-			REG_A4XX_SP_VS_PVT_MEM_ADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_SP_FS_PVT_MEM_ADDR_REG,
-			REG_A4XX_SP_FS_PVT_MEM_ADDR),
-	REG_ADRENO_DEFINE(REG_ADRENO_SP_VS_OBJ_START_REG,
-			REG_A4XX_SP_VS_OBJ_START),
-	REG_ADRENO_DEFINE(REG_ADRENO_SP_FS_OBJ_START_REG,
-			REG_A4XX_SP_FS_OBJ_START),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_RBBM_CTL, REG_A4XX_RBBM_RBBM_CTL),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_SW_RESET_CMD,
-			REG_A4XX_RBBM_SW_RESET_CMD),
-	REG_ADRENO_DEFINE(REG_ADRENO_UCHE_INVALIDATE0,
-			REG_A4XX_UCHE_INVALIDATE0),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_VALUE_LO,
-			REG_A4XX_RBBM_PERFCTR_LOAD_VALUE_LO),
-	REG_ADRENO_DEFINE(REG_ADRENO_RBBM_PERFCTR_LOAD_VALUE_HI,
-			REG_A4XX_RBBM_PERFCTR_LOAD_VALUE_HI),
 };
 
 static void a4xx_dump(struct msm_gpu *gpu)
@@ -587,16 +522,8 @@ static int a4xx_pm_suspend(struct msm_gpu *gpu) {
 
 static int a4xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
 {
-	uint32_t hi, lo, tmp;
-
-	tmp = gpu_read(gpu, REG_A4XX_RBBM_PERFCTR_CP_0_HI);
-	do {
-		hi = tmp;
-		lo = gpu_read(gpu, REG_A4XX_RBBM_PERFCTR_CP_0_LO);
-		tmp = gpu_read(gpu, REG_A4XX_RBBM_PERFCTR_CP_0_HI);
-	} while (tmp != hi);
-
-	*value = (((uint64_t)hi) << 32) | lo;
+	*value = gpu_read64(gpu, REG_A4XX_RBBM_PERFCTR_CP_0_LO,
+		REG_A4XX_RBBM_PERFCTR_CP_0_HI);
 
 	return 0;
 }
@@ -672,7 +599,7 @@ struct msm_gpu *a4xx_gpu_init(struct drm_device *dev)
 #endif
 	}
 
-	if (!gpu->mmu) {
+	if (!gpu->aspace) {
 		/* TODO we think it is possible to configure the GPU to
 		 * restrict access to VRAM carveout.  But the required
 		 * registers are unknown.  For now just bail out and
diff --git a/drivers/gpu/drm/msm/adreno/a5xx.xml.h b/drivers/gpu/drm/msm/adreno/a5xx.xml.h
new file mode 100644
index 000000000000..b6fe763ddf34
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a5xx.xml.h
@@ -0,0 +1,3757 @@
+#ifndef A5XX_XML
+#define A5XX_XML
+
+/* Autogenerated file, DO NOT EDIT manually!
+
+This file was generated by the rules-ng-ng headergen tool in this git repository:
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
+
+The rules-ng-ng source files this header was generated from are:
+- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    431 bytes, from 2016-04-26 17:56:44)
+- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2016-02-10 17:07:21)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  32907 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  12025 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  22544 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 110765 bytes, from 2016-11-26 23:01:48)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml          (  90321 bytes, from 2016-11-28 16:50:05)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2015-09-24 17:30:00)
+
+Copyright (C) 2013-2016 by the following authors:
+- Rob Clark <robdclark@gmail.com> (robclark)
+- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice (including the
+next paragraph) shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+
+enum a5xx_color_fmt {
+	RB5_R8_UNORM = 3,
+	RB5_R4G4B4A4_UNORM = 8,
+	RB5_R5G5B5A1_UNORM = 10,
+	RB5_R5G6B5_UNORM = 14,
+	RB5_R16_FLOAT = 23,
+	RB5_R8G8B8A8_UNORM = 48,
+	RB5_R8G8B8_UNORM = 49,
+	RB5_R8G8B8A8_UINT = 51,
+	RB5_R10G10B10A2_UINT = 58,
+	RB5_R16G16_FLOAT = 69,
+	RB5_R32_FLOAT = 74,
+	RB5_R16G16B16A16_FLOAT = 98,
+	RB5_R32G32_FLOAT = 103,
+	RB5_R32G32B32A32_FLOAT = 130,
+};
+
+enum a5xx_tile_mode {
+	TILE5_LINEAR = 0,
+	TILE5_2 = 2,
+	TILE5_3 = 3,
+};
+
+enum a5xx_vtx_fmt {
+	VFMT5_8_UNORM = 3,
+	VFMT5_8_SNORM = 4,
+	VFMT5_8_UINT = 5,
+	VFMT5_8_SINT = 6,
+	VFMT5_8_8_UNORM = 15,
+	VFMT5_8_8_SNORM = 16,
+	VFMT5_8_8_UINT = 17,
+	VFMT5_8_8_SINT = 18,
+	VFMT5_16_UNORM = 21,
+	VFMT5_16_SNORM = 22,
+	VFMT5_16_FLOAT = 23,
+	VFMT5_16_UINT = 24,
+	VFMT5_16_SINT = 25,
+	VFMT5_8_8_8_UNORM = 33,
+	VFMT5_8_8_8_SNORM = 34,
+	VFMT5_8_8_8_UINT = 35,
+	VFMT5_8_8_8_SINT = 36,
+	VFMT5_8_8_8_8_UNORM = 48,
+	VFMT5_8_8_8_8_SNORM = 50,
+	VFMT5_8_8_8_8_UINT = 51,
+	VFMT5_8_8_8_8_SINT = 52,
+	VFMT5_16_16_UNORM = 67,
+	VFMT5_16_16_SNORM = 68,
+	VFMT5_16_16_FLOAT = 69,
+	VFMT5_16_16_UINT = 70,
+	VFMT5_16_16_SINT = 71,
+	VFMT5_32_UNORM = 72,
+	VFMT5_32_SNORM = 73,
+	VFMT5_32_FLOAT = 74,
+	VFMT5_32_UINT = 75,
+	VFMT5_32_SINT = 76,
+	VFMT5_32_FIXED = 77,
+	VFMT5_16_16_16_UNORM = 88,
+	VFMT5_16_16_16_SNORM = 89,
+	VFMT5_16_16_16_FLOAT = 90,
+	VFMT5_16_16_16_UINT = 91,
+	VFMT5_16_16_16_SINT = 92,
+	VFMT5_16_16_16_16_UNORM = 96,
+	VFMT5_16_16_16_16_SNORM = 97,
+	VFMT5_16_16_16_16_FLOAT = 98,
+	VFMT5_16_16_16_16_UINT = 99,
+	VFMT5_16_16_16_16_SINT = 100,
+	VFMT5_32_32_UNORM = 101,
+	VFMT5_32_32_SNORM = 102,
+	VFMT5_32_32_FLOAT = 103,
+	VFMT5_32_32_UINT = 104,
+	VFMT5_32_32_SINT = 105,
+	VFMT5_32_32_FIXED = 106,
+	VFMT5_32_32_32_UNORM = 112,
+	VFMT5_32_32_32_SNORM = 113,
+	VFMT5_32_32_32_UINT = 114,
+	VFMT5_32_32_32_SINT = 115,
+	VFMT5_32_32_32_FLOAT = 116,
+	VFMT5_32_32_32_FIXED = 117,
+	VFMT5_32_32_32_32_UNORM = 128,
+	VFMT5_32_32_32_32_SNORM = 129,
+	VFMT5_32_32_32_32_FLOAT = 130,
+	VFMT5_32_32_32_32_UINT = 131,
+	VFMT5_32_32_32_32_SINT = 132,
+	VFMT5_32_32_32_32_FIXED = 133,
+};
+
+enum a5xx_tex_fmt {
+	TFMT5_A8_UNORM = 2,
+	TFMT5_8_UNORM = 3,
+	TFMT5_4_4_4_4_UNORM = 8,
+	TFMT5_5_5_5_1_UNORM = 10,
+	TFMT5_5_6_5_UNORM = 14,
+	TFMT5_8_8_UNORM = 15,
+	TFMT5_8_8_SNORM = 16,
+	TFMT5_L8_A8_UNORM = 19,
+	TFMT5_16_FLOAT = 23,
+	TFMT5_8_8_8_8_UNORM = 48,
+	TFMT5_8_8_8_UNORM = 49,
+	TFMT5_8_8_8_SNORM = 50,
+	TFMT5_9_9_9_E5_FLOAT = 53,
+	TFMT5_10_10_10_2_UNORM = 54,
+	TFMT5_11_11_10_FLOAT = 66,
+	TFMT5_16_16_FLOAT = 69,
+	TFMT5_32_FLOAT = 74,
+	TFMT5_16_16_16_16_FLOAT = 98,
+	TFMT5_32_32_FLOAT = 103,
+	TFMT5_32_32_32_32_FLOAT = 130,
+	TFMT5_X8Z24_UNORM = 160,
+};
+
+enum a5xx_tex_fetchsize {
+	TFETCH5_1_BYTE = 0,
+	TFETCH5_2_BYTE = 1,
+	TFETCH5_4_BYTE = 2,
+	TFETCH5_8_BYTE = 3,
+	TFETCH5_16_BYTE = 4,
+};
+
+enum a5xx_depth_format {
+	DEPTH5_NONE = 0,
+	DEPTH5_16 = 1,
+	DEPTH5_24_8 = 2,
+	DEPTH5_32 = 4,
+};
+
+enum a5xx_blit_buf {
+	BLIT_MRT0 = 0,
+	BLIT_MRT1 = 1,
+	BLIT_MRT2 = 2,
+	BLIT_MRT3 = 3,
+	BLIT_MRT4 = 4,
+	BLIT_MRT5 = 5,
+	BLIT_MRT6 = 6,
+	BLIT_MRT7 = 7,
+	BLIT_ZS = 8,
+	BLIT_Z32 = 9,
+};
+
+enum a5xx_tex_filter {
+	A5XX_TEX_NEAREST = 0,
+	A5XX_TEX_LINEAR = 1,
+	A5XX_TEX_ANISO = 2,
+};
+
+enum a5xx_tex_clamp {
+	A5XX_TEX_REPEAT = 0,
+	A5XX_TEX_CLAMP_TO_EDGE = 1,
+	A5XX_TEX_MIRROR_REPEAT = 2,
+	A5XX_TEX_CLAMP_TO_BORDER = 3,
+	A5XX_TEX_MIRROR_CLAMP = 4,
+};
+
+enum a5xx_tex_aniso {
+	A5XX_TEX_ANISO_1 = 0,
+	A5XX_TEX_ANISO_2 = 1,
+	A5XX_TEX_ANISO_4 = 2,
+	A5XX_TEX_ANISO_8 = 3,
+	A5XX_TEX_ANISO_16 = 4,
+};
+
+enum a5xx_tex_swiz {
+	A5XX_TEX_X = 0,
+	A5XX_TEX_Y = 1,
+	A5XX_TEX_Z = 2,
+	A5XX_TEX_W = 3,
+	A5XX_TEX_ZERO = 4,
+	A5XX_TEX_ONE = 5,
+};
+
+enum a5xx_tex_type {
+	A5XX_TEX_1D = 0,
+	A5XX_TEX_2D = 1,
+	A5XX_TEX_CUBE = 2,
+	A5XX_TEX_3D = 3,
+};
+
+#define A5XX_INT0_RBBM_GPU_IDLE					0x00000001
+#define A5XX_INT0_RBBM_AHB_ERROR				0x00000002
+#define A5XX_INT0_RBBM_TRANSFER_TIMEOUT				0x00000004
+#define A5XX_INT0_RBBM_ME_MS_TIMEOUT				0x00000008
+#define A5XX_INT0_RBBM_PFP_MS_TIMEOUT				0x00000010
+#define A5XX_INT0_RBBM_ETS_MS_TIMEOUT				0x00000020
+#define A5XX_INT0_RBBM_ATB_ASYNC_OVERFLOW			0x00000040
+#define A5XX_INT0_RBBM_GPC_ERROR				0x00000080
+#define A5XX_INT0_CP_SW						0x00000100
+#define A5XX_INT0_CP_HW_ERROR					0x00000200
+#define A5XX_INT0_CP_CCU_FLUSH_DEPTH_TS				0x00000400
+#define A5XX_INT0_CP_CCU_FLUSH_COLOR_TS				0x00000800
+#define A5XX_INT0_CP_CCU_RESOLVE_TS				0x00001000
+#define A5XX_INT0_CP_IB2					0x00002000
+#define A5XX_INT0_CP_IB1					0x00004000
+#define A5XX_INT0_CP_RB						0x00008000
+#define A5XX_INT0_CP_UNUSED_1					0x00010000
+#define A5XX_INT0_CP_RB_DONE_TS					0x00020000
+#define A5XX_INT0_CP_WT_DONE_TS					0x00040000
+#define A5XX_INT0_UNKNOWN_1					0x00080000
+#define A5XX_INT0_CP_CACHE_FLUSH_TS				0x00100000
+#define A5XX_INT0_UNUSED_2					0x00200000
+#define A5XX_INT0_RBBM_ATB_BUS_OVERFLOW				0x00400000
+#define A5XX_INT0_MISC_HANG_DETECT				0x00800000
+#define A5XX_INT0_UCHE_OOB_ACCESS				0x01000000
+#define A5XX_INT0_UCHE_TRAP_INTR				0x02000000
+#define A5XX_INT0_DEBBUS_INTR_0					0x04000000
+#define A5XX_INT0_DEBBUS_INTR_1					0x08000000
+#define A5XX_INT0_GPMU_VOLTAGE_DROOP				0x10000000
+#define A5XX_INT0_GPMU_FIRMWARE					0x20000000
+#define A5XX_INT0_ISDB_CPU_IRQ					0x40000000
+#define A5XX_INT0_ISDB_UNDER_DEBUG				0x80000000
+#define A5XX_CP_INT_CP_OPCODE_ERROR				0x00000001
+#define A5XX_CP_INT_CP_RESERVED_BIT_ERROR			0x00000002
+#define A5XX_CP_INT_CP_HW_FAULT_ERROR				0x00000004
+#define A5XX_CP_INT_CP_DMA_ERROR				0x00000008
+#define A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR		0x00000010
+#define A5XX_CP_INT_CP_AHB_ERROR				0x00000020
+#define REG_A5XX_CP_RB_BASE					0x00000800
+
+#define REG_A5XX_CP_RB_BASE_HI					0x00000801
+
+#define REG_A5XX_CP_RB_CNTL					0x00000802
+
+#define REG_A5XX_CP_RB_RPTR_ADDR				0x00000804
+
+#define REG_A5XX_CP_RB_RPTR_ADDR_HI				0x00000805
+
+#define REG_A5XX_CP_RB_RPTR					0x00000806
+
+#define REG_A5XX_CP_RB_WPTR					0x00000807
+
+#define REG_A5XX_CP_PFP_STAT_ADDR				0x00000808
+
+#define REG_A5XX_CP_PFP_STAT_DATA				0x00000809
+
+#define REG_A5XX_CP_DRAW_STATE_ADDR				0x0000080b
+
+#define REG_A5XX_CP_DRAW_STATE_DATA				0x0000080c
+
+#define REG_A5XX_CP_CRASH_SCRIPT_BASE_LO			0x00000817
+
+#define REG_A5XX_CP_CRASH_SCRIPT_BASE_HI			0x00000818
+
+#define REG_A5XX_CP_CRASH_DUMP_CNTL				0x00000819
+
+#define REG_A5XX_CP_ME_STAT_ADDR				0x0000081a
+
+#define REG_A5XX_CP_ROQ_THRESHOLDS_1				0x0000081f
+
+#define REG_A5XX_CP_ROQ_THRESHOLDS_2				0x00000820
+
+#define REG_A5XX_CP_ROQ_DBG_ADDR				0x00000821
+
+#define REG_A5XX_CP_ROQ_DBG_DATA				0x00000822
+
+#define REG_A5XX_CP_MEQ_DBG_ADDR				0x00000823
+
+#define REG_A5XX_CP_MEQ_DBG_DATA				0x00000824
+
+#define REG_A5XX_CP_MEQ_THRESHOLDS				0x00000825
+
+#define REG_A5XX_CP_MERCIU_SIZE					0x00000826
+
+#define REG_A5XX_CP_MERCIU_DBG_ADDR				0x00000827
+
+#define REG_A5XX_CP_MERCIU_DBG_DATA_1				0x00000828
+
+#define REG_A5XX_CP_MERCIU_DBG_DATA_2				0x00000829
+
+#define REG_A5XX_CP_PFP_UCODE_DBG_ADDR				0x0000082a
+
+#define REG_A5XX_CP_PFP_UCODE_DBG_DATA				0x0000082b
+
+#define REG_A5XX_CP_ME_UCODE_DBG_ADDR				0x0000082f
+
+#define REG_A5XX_CP_ME_UCODE_DBG_DATA				0x00000830
+
+#define REG_A5XX_CP_CNTL					0x00000831
+
+#define REG_A5XX_CP_PFP_ME_CNTL					0x00000832
+
+#define REG_A5XX_CP_CHICKEN_DBG					0x00000833
+
+#define REG_A5XX_CP_PFP_INSTR_BASE_LO				0x00000835
+
+#define REG_A5XX_CP_PFP_INSTR_BASE_HI				0x00000836
+
+#define REG_A5XX_CP_ME_INSTR_BASE_LO				0x00000838
+
+#define REG_A5XX_CP_ME_INSTR_BASE_HI				0x00000839
+
+#define REG_A5XX_CP_CONTEXT_SWITCH_CNTL				0x0000083b
+
+#define REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO		0x0000083c
+
+#define REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI		0x0000083d
+
+#define REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO			0x0000083e
+
+#define REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_HI			0x0000083f
+
+#define REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO			0x00000840
+
+#define REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI			0x00000841
+
+#define REG_A5XX_CP_ADDR_MODE_CNTL				0x00000860
+
+#define REG_A5XX_CP_ME_STAT_DATA				0x00000b14
+
+#define REG_A5XX_CP_WFI_PEND_CTR				0x00000b15
+
+#define REG_A5XX_CP_INTERRUPT_STATUS				0x00000b18
+
+#define REG_A5XX_CP_HW_FAULT					0x00000b1a
+
+#define REG_A5XX_CP_PROTECT_STATUS				0x00000b1c
+
+#define REG_A5XX_CP_IB1_BASE					0x00000b1f
+
+#define REG_A5XX_CP_IB1_BASE_HI					0x00000b20
+
+#define REG_A5XX_CP_IB1_BUFSZ					0x00000b21
+
+#define REG_A5XX_CP_IB2_BASE					0x00000b22
+
+#define REG_A5XX_CP_IB2_BASE_HI					0x00000b23
+
+#define REG_A5XX_CP_IB2_BUFSZ					0x00000b24
+
+static inline uint32_t REG_A5XX_CP_SCRATCH(uint32_t i0) { return 0x00000b78 + 0x1*i0; }
+
+static inline uint32_t REG_A5XX_CP_SCRATCH_REG(uint32_t i0) { return 0x00000b78 + 0x1*i0; }
+
+static inline uint32_t REG_A5XX_CP_PROTECT(uint32_t i0) { return 0x00000880 + 0x1*i0; }
+
+static inline uint32_t REG_A5XX_CP_PROTECT_REG(uint32_t i0) { return 0x00000880 + 0x1*i0; }
+#define A5XX_CP_PROTECT_REG_BASE_ADDR__MASK			0x0001ffff
+#define A5XX_CP_PROTECT_REG_BASE_ADDR__SHIFT			0
+static inline uint32_t A5XX_CP_PROTECT_REG_BASE_ADDR(uint32_t val)
+{
+	return ((val) << A5XX_CP_PROTECT_REG_BASE_ADDR__SHIFT) & A5XX_CP_PROTECT_REG_BASE_ADDR__MASK;
+}
+#define A5XX_CP_PROTECT_REG_MASK_LEN__MASK			0x1f000000
+#define A5XX_CP_PROTECT_REG_MASK_LEN__SHIFT			24
+static inline uint32_t A5XX_CP_PROTECT_REG_MASK_LEN(uint32_t val)
+{
+	return ((val) << A5XX_CP_PROTECT_REG_MASK_LEN__SHIFT) & A5XX_CP_PROTECT_REG_MASK_LEN__MASK;
+}
+#define A5XX_CP_PROTECT_REG_TRAP_WRITE				0x20000000
+#define A5XX_CP_PROTECT_REG_TRAP_READ				0x40000000
+
+#define REG_A5XX_CP_PROTECT_CNTL				0x000008a0
+
+#define REG_A5XX_CP_AHB_FAULT					0x00000b1b
+
+#define REG_A5XX_CP_PERFCTR_CP_SEL_0				0x00000bb0
+
+#define REG_A5XX_CP_PERFCTR_CP_SEL_1				0x00000bb1
+
+#define REG_A5XX_CP_PERFCTR_CP_SEL_2				0x00000bb2
+
+#define REG_A5XX_CP_PERFCTR_CP_SEL_3				0x00000bb3
+
+#define REG_A5XX_CP_PERFCTR_CP_SEL_4				0x00000bb4
+
+#define REG_A5XX_CP_PERFCTR_CP_SEL_5				0x00000bb5
+
+#define REG_A5XX_CP_PERFCTR_CP_SEL_6				0x00000bb6
+
+#define REG_A5XX_CP_PERFCTR_CP_SEL_7				0x00000bb7
+
+#define REG_A5XX_VSC_ADDR_MODE_CNTL				0x00000bc1
+
+#define REG_A5XX_CP_POWERCTR_CP_SEL_0				0x00000bba
+
+#define REG_A5XX_CP_POWERCTR_CP_SEL_1				0x00000bbb
+
+#define REG_A5XX_CP_POWERCTR_CP_SEL_2				0x00000bbc
+
+#define REG_A5XX_CP_POWERCTR_CP_SEL_3				0x00000bbd
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_SEL_A				0x00000004
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_SEL_B				0x00000005
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_SEL_C				0x00000006
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_SEL_D				0x00000007
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_CNTLT				0x00000008
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_CNTLM				0x00000009
+
+#define REG_A5XX_RBBM_CFG_DEBBUS_CTLTM_ENABLE_SHIFT		0x00000018
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_OPL				0x0000000a
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_OPE				0x0000000b
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_IVTL_0				0x0000000c
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_IVTL_1				0x0000000d
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_IVTL_2				0x0000000e
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_IVTL_3				0x0000000f
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_MASKL_0			0x00000010
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_MASKL_1			0x00000011
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_MASKL_2			0x00000012
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_MASKL_3			0x00000013
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_BYTEL_0			0x00000014
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_BYTEL_1			0x00000015
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_IVTE_0				0x00000016
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_IVTE_1				0x00000017
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_IVTE_2				0x00000018
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_IVTE_3				0x00000019
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_MASKE_0			0x0000001a
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_MASKE_1			0x0000001b
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_MASKE_2			0x0000001c
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_MASKE_3			0x0000001d
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_NIBBLEE			0x0000001e
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_PTRC0				0x0000001f
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_PTRC1				0x00000020
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_LOADREG			0x00000021
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_IDX				0x00000022
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_CLRC				0x00000023
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_LOADIVT			0x00000024
+
+#define REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL			0x0000002f
+
+#define REG_A5XX_RBBM_INT_CLEAR_CMD				0x00000037
+
+#define REG_A5XX_RBBM_INT_0_MASK				0x00000038
+#define A5XX_RBBM_INT_0_MASK_RBBM_GPU_IDLE			0x00000001
+#define A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR			0x00000002
+#define A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT		0x00000004
+#define A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT			0x00000008
+#define A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT		0x00000010
+#define A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT		0x00000020
+#define A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW		0x00000040
+#define A5XX_RBBM_INT_0_MASK_RBBM_GPC_ERROR			0x00000080
+#define A5XX_RBBM_INT_0_MASK_CP_SW				0x00000100
+#define A5XX_RBBM_INT_0_MASK_CP_HW_ERROR			0x00000200
+#define A5XX_RBBM_INT_0_MASK_CP_CCU_FLUSH_DEPTH_TS		0x00000400
+#define A5XX_RBBM_INT_0_MASK_CP_CCU_FLUSH_COLOR_TS		0x00000800
+#define A5XX_RBBM_INT_0_MASK_CP_CCU_RESOLVE_TS			0x00001000
+#define A5XX_RBBM_INT_0_MASK_CP_IB2				0x00002000
+#define A5XX_RBBM_INT_0_MASK_CP_IB1				0x00004000
+#define A5XX_RBBM_INT_0_MASK_CP_RB				0x00008000
+#define A5XX_RBBM_INT_0_MASK_CP_RB_DONE_TS			0x00020000
+#define A5XX_RBBM_INT_0_MASK_CP_WT_DONE_TS			0x00040000
+#define A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS			0x00100000
+#define A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW		0x00400000
+#define A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT			0x00800000
+#define A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS			0x01000000
+#define A5XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR			0x02000000
+#define A5XX_RBBM_INT_0_MASK_DEBBUS_INTR_0			0x04000000
+#define A5XX_RBBM_INT_0_MASK_DEBBUS_INTR_1			0x08000000
+#define A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP			0x10000000
+#define A5XX_RBBM_INT_0_MASK_GPMU_FIRMWARE			0x20000000
+#define A5XX_RBBM_INT_0_MASK_ISDB_CPU_IRQ			0x40000000
+#define A5XX_RBBM_INT_0_MASK_ISDB_UNDER_DEBUG			0x80000000
+
+#define REG_A5XX_RBBM_AHB_DBG_CNTL				0x0000003f
+
+#define REG_A5XX_RBBM_EXT_VBIF_DBG_CNTL				0x00000041
+
+#define REG_A5XX_RBBM_SW_RESET_CMD				0x00000043
+
+#define REG_A5XX_RBBM_BLOCK_SW_RESET_CMD			0x00000045
+
+#define REG_A5XX_RBBM_BLOCK_SW_RESET_CMD2			0x00000046
+
+#define REG_A5XX_RBBM_DBG_LO_HI_GPIO				0x00000048
+
+#define REG_A5XX_RBBM_EXT_TRACE_BUS_CNTL			0x00000049
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_TP0				0x0000004a
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_TP1				0x0000004b
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_TP2				0x0000004c
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_TP3				0x0000004d
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_TP0				0x0000004e
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_TP1				0x0000004f
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_TP2				0x00000050
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_TP3				0x00000051
+
+#define REG_A5XX_RBBM_CLOCK_CNTL3_TP0				0x00000052
+
+#define REG_A5XX_RBBM_CLOCK_CNTL3_TP1				0x00000053
+
+#define REG_A5XX_RBBM_CLOCK_CNTL3_TP2				0x00000054
+
+#define REG_A5XX_RBBM_CLOCK_CNTL3_TP3				0x00000055
+
+#define REG_A5XX_RBBM_READ_AHB_THROUGH_DBG			0x00000059
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_UCHE				0x0000005a
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_UCHE				0x0000005b
+
+#define REG_A5XX_RBBM_CLOCK_CNTL3_UCHE				0x0000005c
+
+#define REG_A5XX_RBBM_CLOCK_CNTL4_UCHE				0x0000005d
+
+#define REG_A5XX_RBBM_CLOCK_HYST_UCHE				0x0000005e
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_UCHE				0x0000005f
+
+#define REG_A5XX_RBBM_CLOCK_MODE_GPC				0x00000060
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_GPC				0x00000061
+
+#define REG_A5XX_RBBM_CLOCK_HYST_GPC				0x00000062
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM			0x00000063
+
+#define REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM			0x00000064
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM			0x00000065
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_HLSQ				0x00000066
+
+#define REG_A5XX_RBBM_CLOCK_CNTL				0x00000067
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_SP0				0x00000068
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_SP1				0x00000069
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_SP2				0x0000006a
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_SP3				0x0000006b
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_SP0				0x0000006c
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_SP1				0x0000006d
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_SP2				0x0000006e
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_SP3				0x0000006f
+
+#define REG_A5XX_RBBM_CLOCK_HYST_SP0				0x00000070
+
+#define REG_A5XX_RBBM_CLOCK_HYST_SP1				0x00000071
+
+#define REG_A5XX_RBBM_CLOCK_HYST_SP2				0x00000072
+
+#define REG_A5XX_RBBM_CLOCK_HYST_SP3				0x00000073
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_SP0				0x00000074
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_SP1				0x00000075
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_SP2				0x00000076
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_SP3				0x00000077
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_RB0				0x00000078
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_RB1				0x00000079
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_RB2				0x0000007a
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_RB3				0x0000007b
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_RB0				0x0000007c
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_RB1				0x0000007d
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_RB2				0x0000007e
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_RB3				0x0000007f
+
+#define REG_A5XX_RBBM_CLOCK_HYST_RAC				0x00000080
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_RAC				0x00000081
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_CCU0				0x00000082
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_CCU1				0x00000083
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_CCU2				0x00000084
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_CCU3				0x00000085
+
+#define REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0			0x00000086
+
+#define REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1			0x00000087
+
+#define REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2			0x00000088
+
+#define REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3			0x00000089
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_RAC				0x0000008a
+
+#define REG_A5XX_RBBM_CLOCK_CNTL2_RAC				0x0000008b
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0			0x0000008c
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1			0x0000008d
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2			0x0000008e
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3			0x0000008f
+
+#define REG_A5XX_RBBM_CLOCK_HYST_VFD				0x00000090
+
+#define REG_A5XX_RBBM_CLOCK_MODE_VFD				0x00000091
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_VFD				0x00000092
+
+#define REG_A5XX_RBBM_AHB_CNTL0					0x00000093
+
+#define REG_A5XX_RBBM_AHB_CNTL1					0x00000094
+
+#define REG_A5XX_RBBM_AHB_CNTL2					0x00000095
+
+#define REG_A5XX_RBBM_AHB_CMD					0x00000096
+
+#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11		0x0000009c
+
+#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12		0x0000009d
+
+#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13		0x0000009e
+
+#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14		0x0000009f
+
+#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15		0x000000a0
+
+#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16		0x000000a1
+
+#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17		0x000000a2
+
+#define REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18		0x000000a3
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_TP0				0x000000a4
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_TP1				0x000000a5
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_TP2				0x000000a6
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_TP3				0x000000a7
+
+#define REG_A5XX_RBBM_CLOCK_DELAY2_TP0				0x000000a8
+
+#define REG_A5XX_RBBM_CLOCK_DELAY2_TP1				0x000000a9
+
+#define REG_A5XX_RBBM_CLOCK_DELAY2_TP2				0x000000aa
+
+#define REG_A5XX_RBBM_CLOCK_DELAY2_TP3				0x000000ab
+
+#define REG_A5XX_RBBM_CLOCK_DELAY3_TP0				0x000000ac
+
+#define REG_A5XX_RBBM_CLOCK_DELAY3_TP1				0x000000ad
+
+#define REG_A5XX_RBBM_CLOCK_DELAY3_TP2				0x000000ae
+
+#define REG_A5XX_RBBM_CLOCK_DELAY3_TP3				0x000000af
+
+#define REG_A5XX_RBBM_CLOCK_HYST_TP0				0x000000b0
+
+#define REG_A5XX_RBBM_CLOCK_HYST_TP1				0x000000b1
+
+#define REG_A5XX_RBBM_CLOCK_HYST_TP2				0x000000b2
+
+#define REG_A5XX_RBBM_CLOCK_HYST_TP3				0x000000b3
+
+#define REG_A5XX_RBBM_CLOCK_HYST2_TP0				0x000000b4
+
+#define REG_A5XX_RBBM_CLOCK_HYST2_TP1				0x000000b5
+
+#define REG_A5XX_RBBM_CLOCK_HYST2_TP2				0x000000b6
+
+#define REG_A5XX_RBBM_CLOCK_HYST2_TP3				0x000000b7
+
+#define REG_A5XX_RBBM_CLOCK_HYST3_TP0				0x000000b8
+
+#define REG_A5XX_RBBM_CLOCK_HYST3_TP1				0x000000b9
+
+#define REG_A5XX_RBBM_CLOCK_HYST3_TP2				0x000000ba
+
+#define REG_A5XX_RBBM_CLOCK_HYST3_TP3				0x000000bb
+
+#define REG_A5XX_RBBM_CLOCK_CNTL_GPMU				0x000000c8
+
+#define REG_A5XX_RBBM_CLOCK_DELAY_GPMU				0x000000c9
+
+#define REG_A5XX_RBBM_CLOCK_HYST_GPMU				0x000000ca
+
+#define REG_A5XX_RBBM_PERFCTR_CP_0_LO				0x000003a0
+
+#define REG_A5XX_RBBM_PERFCTR_CP_0_HI				0x000003a1
+
+#define REG_A5XX_RBBM_PERFCTR_CP_1_LO				0x000003a2
+
+#define REG_A5XX_RBBM_PERFCTR_CP_1_HI				0x000003a3
+
+#define REG_A5XX_RBBM_PERFCTR_CP_2_LO				0x000003a4
+
+#define REG_A5XX_RBBM_PERFCTR_CP_2_HI				0x000003a5
+
+#define REG_A5XX_RBBM_PERFCTR_CP_3_LO				0x000003a6
+
+#define REG_A5XX_RBBM_PERFCTR_CP_3_HI				0x000003a7
+
+#define REG_A5XX_RBBM_PERFCTR_CP_4_LO				0x000003a8
+
+#define REG_A5XX_RBBM_PERFCTR_CP_4_HI				0x000003a9
+
+#define REG_A5XX_RBBM_PERFCTR_CP_5_LO				0x000003aa
+
+#define REG_A5XX_RBBM_PERFCTR_CP_5_HI				0x000003ab
+
+#define REG_A5XX_RBBM_PERFCTR_CP_6_LO				0x000003ac
+
+#define REG_A5XX_RBBM_PERFCTR_CP_6_HI				0x000003ad
+
+#define REG_A5XX_RBBM_PERFCTR_CP_7_LO				0x000003ae
+
+#define REG_A5XX_RBBM_PERFCTR_CP_7_HI				0x000003af
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_0_LO				0x000003b0
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_0_HI				0x000003b1
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_1_LO				0x000003b2
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_1_HI				0x000003b3
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_2_LO				0x000003b4
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_2_HI				0x000003b5
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_3_LO				0x000003b6
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_3_HI				0x000003b7
+
+#define REG_A5XX_RBBM_PERFCTR_PC_0_LO				0x000003b8
+
+#define REG_A5XX_RBBM_PERFCTR_PC_0_HI				0x000003b9
+
+#define REG_A5XX_RBBM_PERFCTR_PC_1_LO				0x000003ba
+
+#define REG_A5XX_RBBM_PERFCTR_PC_1_HI				0x000003bb
+
+#define REG_A5XX_RBBM_PERFCTR_PC_2_LO				0x000003bc
+
+#define REG_A5XX_RBBM_PERFCTR_PC_2_HI				0x000003bd
+
+#define REG_A5XX_RBBM_PERFCTR_PC_3_LO				0x000003be
+
+#define REG_A5XX_RBBM_PERFCTR_PC_3_HI				0x000003bf
+
+#define REG_A5XX_RBBM_PERFCTR_PC_4_LO				0x000003c0
+
+#define REG_A5XX_RBBM_PERFCTR_PC_4_HI				0x000003c1
+
+#define REG_A5XX_RBBM_PERFCTR_PC_5_LO				0x000003c2
+
+#define REG_A5XX_RBBM_PERFCTR_PC_5_HI				0x000003c3
+
+#define REG_A5XX_RBBM_PERFCTR_PC_6_LO				0x000003c4
+
+#define REG_A5XX_RBBM_PERFCTR_PC_6_HI				0x000003c5
+
+#define REG_A5XX_RBBM_PERFCTR_PC_7_LO				0x000003c6
+
+#define REG_A5XX_RBBM_PERFCTR_PC_7_HI				0x000003c7
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_0_LO				0x000003c8
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_0_HI				0x000003c9
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_1_LO				0x000003ca
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_1_HI				0x000003cb
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_2_LO				0x000003cc
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_2_HI				0x000003cd
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_3_LO				0x000003ce
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_3_HI				0x000003cf
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_4_LO				0x000003d0
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_4_HI				0x000003d1
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_5_LO				0x000003d2
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_5_HI				0x000003d3
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_6_LO				0x000003d4
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_6_HI				0x000003d5
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_7_LO				0x000003d6
+
+#define REG_A5XX_RBBM_PERFCTR_VFD_7_HI				0x000003d7
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_0_LO				0x000003d8
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_0_HI				0x000003d9
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_1_LO				0x000003da
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_1_HI				0x000003db
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_2_LO				0x000003dc
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_2_HI				0x000003dd
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_3_LO				0x000003de
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_3_HI				0x000003df
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_4_LO				0x000003e0
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_4_HI				0x000003e1
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_5_LO				0x000003e2
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_5_HI				0x000003e3
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_6_LO				0x000003e4
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_6_HI				0x000003e5
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_7_LO				0x000003e6
+
+#define REG_A5XX_RBBM_PERFCTR_HLSQ_7_HI				0x000003e7
+
+#define REG_A5XX_RBBM_PERFCTR_VPC_0_LO				0x000003e8
+
+#define REG_A5XX_RBBM_PERFCTR_VPC_0_HI				0x000003e9
+
+#define REG_A5XX_RBBM_PERFCTR_VPC_1_LO				0x000003ea
+
+#define REG_A5XX_RBBM_PERFCTR_VPC_1_HI				0x000003eb
+
+#define REG_A5XX_RBBM_PERFCTR_VPC_2_LO				0x000003ec
+
+#define REG_A5XX_RBBM_PERFCTR_VPC_2_HI				0x000003ed
+
+#define REG_A5XX_RBBM_PERFCTR_VPC_3_LO				0x000003ee
+
+#define REG_A5XX_RBBM_PERFCTR_VPC_3_HI				0x000003ef
+
+#define REG_A5XX_RBBM_PERFCTR_CCU_0_LO				0x000003f0
+
+#define REG_A5XX_RBBM_PERFCTR_CCU_0_HI				0x000003f1
+
+#define REG_A5XX_RBBM_PERFCTR_CCU_1_LO				0x000003f2
+
+#define REG_A5XX_RBBM_PERFCTR_CCU_1_HI				0x000003f3
+
+#define REG_A5XX_RBBM_PERFCTR_CCU_2_LO				0x000003f4
+
+#define REG_A5XX_RBBM_PERFCTR_CCU_2_HI				0x000003f5
+
+#define REG_A5XX_RBBM_PERFCTR_CCU_3_LO				0x000003f6
+
+#define REG_A5XX_RBBM_PERFCTR_CCU_3_HI				0x000003f7
+
+#define REG_A5XX_RBBM_PERFCTR_TSE_0_LO				0x000003f8
+
+#define REG_A5XX_RBBM_PERFCTR_TSE_0_HI				0x000003f9
+
+#define REG_A5XX_RBBM_PERFCTR_TSE_1_LO				0x000003fa
+
+#define REG_A5XX_RBBM_PERFCTR_TSE_1_HI				0x000003fb
+
+#define REG_A5XX_RBBM_PERFCTR_TSE_2_LO				0x000003fc
+
+#define REG_A5XX_RBBM_PERFCTR_TSE_2_HI				0x000003fd
+
+#define REG_A5XX_RBBM_PERFCTR_TSE_3_LO				0x000003fe
+
+#define REG_A5XX_RBBM_PERFCTR_TSE_3_HI				0x000003ff
+
+#define REG_A5XX_RBBM_PERFCTR_RAS_0_LO				0x00000400
+
+#define REG_A5XX_RBBM_PERFCTR_RAS_0_HI				0x00000401
+
+#define REG_A5XX_RBBM_PERFCTR_RAS_1_LO				0x00000402
+
+#define REG_A5XX_RBBM_PERFCTR_RAS_1_HI				0x00000403
+
+#define REG_A5XX_RBBM_PERFCTR_RAS_2_LO				0x00000404
+
+#define REG_A5XX_RBBM_PERFCTR_RAS_2_HI				0x00000405
+
+#define REG_A5XX_RBBM_PERFCTR_RAS_3_LO				0x00000406
+
+#define REG_A5XX_RBBM_PERFCTR_RAS_3_HI				0x00000407
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_0_LO				0x00000408
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_0_HI				0x00000409
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_1_LO				0x0000040a
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_1_HI				0x0000040b
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_2_LO				0x0000040c
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_2_HI				0x0000040d
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_3_LO				0x0000040e
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_3_HI				0x0000040f
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_4_LO				0x00000410
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_4_HI				0x00000411
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_5_LO				0x00000412
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_5_HI				0x00000413
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_6_LO				0x00000414
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_6_HI				0x00000415
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_7_LO				0x00000416
+
+#define REG_A5XX_RBBM_PERFCTR_UCHE_7_HI				0x00000417
+
+#define REG_A5XX_RBBM_PERFCTR_TP_0_LO				0x00000418
+
+#define REG_A5XX_RBBM_PERFCTR_TP_0_HI				0x00000419
+
+#define REG_A5XX_RBBM_PERFCTR_TP_1_LO				0x0000041a
+
+#define REG_A5XX_RBBM_PERFCTR_TP_1_HI				0x0000041b
+
+#define REG_A5XX_RBBM_PERFCTR_TP_2_LO				0x0000041c
+
+#define REG_A5XX_RBBM_PERFCTR_TP_2_HI				0x0000041d
+
+#define REG_A5XX_RBBM_PERFCTR_TP_3_LO				0x0000041e
+
+#define REG_A5XX_RBBM_PERFCTR_TP_3_HI				0x0000041f
+
+#define REG_A5XX_RBBM_PERFCTR_TP_4_LO				0x00000420
+
+#define REG_A5XX_RBBM_PERFCTR_TP_4_HI				0x00000421
+
+#define REG_A5XX_RBBM_PERFCTR_TP_5_LO				0x00000422
+
+#define REG_A5XX_RBBM_PERFCTR_TP_5_HI				0x00000423
+
+#define REG_A5XX_RBBM_PERFCTR_TP_6_LO				0x00000424
+
+#define REG_A5XX_RBBM_PERFCTR_TP_6_HI				0x00000425
+
+#define REG_A5XX_RBBM_PERFCTR_TP_7_LO				0x00000426
+
+#define REG_A5XX_RBBM_PERFCTR_TP_7_HI				0x00000427
+
+#define REG_A5XX_RBBM_PERFCTR_SP_0_LO				0x00000428
+
+#define REG_A5XX_RBBM_PERFCTR_SP_0_HI				0x00000429
+
+#define REG_A5XX_RBBM_PERFCTR_SP_1_LO				0x0000042a
+
+#define REG_A5XX_RBBM_PERFCTR_SP_1_HI				0x0000042b
+
+#define REG_A5XX_RBBM_PERFCTR_SP_2_LO				0x0000042c
+
+#define REG_A5XX_RBBM_PERFCTR_SP_2_HI				0x0000042d
+
+#define REG_A5XX_RBBM_PERFCTR_SP_3_LO				0x0000042e
+
+#define REG_A5XX_RBBM_PERFCTR_SP_3_HI				0x0000042f
+
+#define REG_A5XX_RBBM_PERFCTR_SP_4_LO				0x00000430
+
+#define REG_A5XX_RBBM_PERFCTR_SP_4_HI				0x00000431
+
+#define REG_A5XX_RBBM_PERFCTR_SP_5_LO				0x00000432
+
+#define REG_A5XX_RBBM_PERFCTR_SP_5_HI				0x00000433
+
+#define REG_A5XX_RBBM_PERFCTR_SP_6_LO				0x00000434
+
+#define REG_A5XX_RBBM_PERFCTR_SP_6_HI				0x00000435
+
+#define REG_A5XX_RBBM_PERFCTR_SP_7_LO				0x00000436
+
+#define REG_A5XX_RBBM_PERFCTR_SP_7_HI				0x00000437
+
+#define REG_A5XX_RBBM_PERFCTR_SP_8_LO				0x00000438
+
+#define REG_A5XX_RBBM_PERFCTR_SP_8_HI				0x00000439
+
+#define REG_A5XX_RBBM_PERFCTR_SP_9_LO				0x0000043a
+
+#define REG_A5XX_RBBM_PERFCTR_SP_9_HI				0x0000043b
+
+#define REG_A5XX_RBBM_PERFCTR_SP_10_LO				0x0000043c
+
+#define REG_A5XX_RBBM_PERFCTR_SP_10_HI				0x0000043d
+
+#define REG_A5XX_RBBM_PERFCTR_SP_11_LO				0x0000043e
+
+#define REG_A5XX_RBBM_PERFCTR_SP_11_HI				0x0000043f
+
+#define REG_A5XX_RBBM_PERFCTR_RB_0_LO				0x00000440
+
+#define REG_A5XX_RBBM_PERFCTR_RB_0_HI				0x00000441
+
+#define REG_A5XX_RBBM_PERFCTR_RB_1_LO				0x00000442
+
+#define REG_A5XX_RBBM_PERFCTR_RB_1_HI				0x00000443
+
+#define REG_A5XX_RBBM_PERFCTR_RB_2_LO				0x00000444
+
+#define REG_A5XX_RBBM_PERFCTR_RB_2_HI				0x00000445
+
+#define REG_A5XX_RBBM_PERFCTR_RB_3_LO				0x00000446
+
+#define REG_A5XX_RBBM_PERFCTR_RB_3_HI				0x00000447
+
+#define REG_A5XX_RBBM_PERFCTR_RB_4_LO				0x00000448
+
+#define REG_A5XX_RBBM_PERFCTR_RB_4_HI				0x00000449
+
+#define REG_A5XX_RBBM_PERFCTR_RB_5_LO				0x0000044a
+
+#define REG_A5XX_RBBM_PERFCTR_RB_5_HI				0x0000044b
+
+#define REG_A5XX_RBBM_PERFCTR_RB_6_LO				0x0000044c
+
+#define REG_A5XX_RBBM_PERFCTR_RB_6_HI				0x0000044d
+
+#define REG_A5XX_RBBM_PERFCTR_RB_7_LO				0x0000044e
+
+#define REG_A5XX_RBBM_PERFCTR_RB_7_HI				0x0000044f
+
+#define REG_A5XX_RBBM_PERFCTR_VSC_0_LO				0x00000450
+
+#define REG_A5XX_RBBM_PERFCTR_VSC_0_HI				0x00000451
+
+#define REG_A5XX_RBBM_PERFCTR_VSC_1_LO				0x00000452
+
+#define REG_A5XX_RBBM_PERFCTR_VSC_1_HI				0x00000453
+
+#define REG_A5XX_RBBM_PERFCTR_LRZ_0_LO				0x00000454
+
+#define REG_A5XX_RBBM_PERFCTR_LRZ_0_HI				0x00000455
+
+#define REG_A5XX_RBBM_PERFCTR_LRZ_1_LO				0x00000456
+
+#define REG_A5XX_RBBM_PERFCTR_LRZ_1_HI				0x00000457
+
+#define REG_A5XX_RBBM_PERFCTR_LRZ_2_LO				0x00000458
+
+#define REG_A5XX_RBBM_PERFCTR_LRZ_2_HI				0x00000459
+
+#define REG_A5XX_RBBM_PERFCTR_LRZ_3_LO				0x0000045a
+
+#define REG_A5XX_RBBM_PERFCTR_LRZ_3_HI				0x0000045b
+
+#define REG_A5XX_RBBM_PERFCTR_CMP_0_LO				0x0000045c
+
+#define REG_A5XX_RBBM_PERFCTR_CMP_0_HI				0x0000045d
+
+#define REG_A5XX_RBBM_PERFCTR_CMP_1_LO				0x0000045e
+
+#define REG_A5XX_RBBM_PERFCTR_CMP_1_HI				0x0000045f
+
+#define REG_A5XX_RBBM_PERFCTR_CMP_2_LO				0x00000460
+
+#define REG_A5XX_RBBM_PERFCTR_CMP_2_HI				0x00000461
+
+#define REG_A5XX_RBBM_PERFCTR_CMP_3_LO				0x00000462
+
+#define REG_A5XX_RBBM_PERFCTR_CMP_3_HI				0x00000463
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0			0x0000046b
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_SEL_1			0x0000046c
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_SEL_2			0x0000046d
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_SEL_3			0x0000046e
+
+#define REG_A5XX_RBBM_ALWAYSON_COUNTER_LO			0x000004d2
+
+#define REG_A5XX_RBBM_ALWAYSON_COUNTER_HI			0x000004d3
+
+#define REG_A5XX_RBBM_STATUS					0x000004f5
+#define A5XX_RBBM_STATUS_GPU_BUSY_IGN_AHB			0x80000000
+#define A5XX_RBBM_STATUS_GPU_BUSY_IGN_AHB_CP			0x40000000
+#define A5XX_RBBM_STATUS_HLSQ_BUSY				0x20000000
+#define A5XX_RBBM_STATUS_VSC_BUSY				0x10000000
+#define A5XX_RBBM_STATUS_TPL1_BUSY				0x08000000
+#define A5XX_RBBM_STATUS_SP_BUSY				0x04000000
+#define A5XX_RBBM_STATUS_UCHE_BUSY				0x02000000
+#define A5XX_RBBM_STATUS_VPC_BUSY				0x01000000
+#define A5XX_RBBM_STATUS_VFDP_BUSY				0x00800000
+#define A5XX_RBBM_STATUS_VFD_BUSY				0x00400000
+#define A5XX_RBBM_STATUS_TESS_BUSY				0x00200000
+#define A5XX_RBBM_STATUS_PC_VSD_BUSY				0x00100000
+#define A5XX_RBBM_STATUS_PC_DCALL_BUSY				0x00080000
+#define A5XX_RBBM_STATUS_GPMU_SLAVE_BUSY			0x00040000
+#define A5XX_RBBM_STATUS_DCOM_BUSY				0x00020000
+#define A5XX_RBBM_STATUS_COM_BUSY				0x00010000
+#define A5XX_RBBM_STATUS_LRZ_BUZY				0x00008000
+#define A5XX_RBBM_STATUS_A2D_DSP_BUSY				0x00004000
+#define A5XX_RBBM_STATUS_CCUFCHE_BUSY				0x00002000
+#define A5XX_RBBM_STATUS_RB_BUSY				0x00001000
+#define A5XX_RBBM_STATUS_RAS_BUSY				0x00000800
+#define A5XX_RBBM_STATUS_TSE_BUSY				0x00000400
+#define A5XX_RBBM_STATUS_VBIF_BUSY				0x00000200
+#define A5XX_RBBM_STATUS_GPU_BUSY_IGN_AHB_HYST			0x00000100
+#define A5XX_RBBM_STATUS_CP_BUSY_IGN_HYST			0x00000080
+#define A5XX_RBBM_STATUS_CP_BUSY				0x00000040
+#define A5XX_RBBM_STATUS_GPMU_MASTER_BUSY			0x00000020
+#define A5XX_RBBM_STATUS_CP_CRASH_BUSY				0x00000010
+#define A5XX_RBBM_STATUS_CP_ETS_BUSY				0x00000008
+#define A5XX_RBBM_STATUS_CP_PFP_BUSY				0x00000004
+#define A5XX_RBBM_STATUS_CP_ME_BUSY				0x00000002
+#define A5XX_RBBM_STATUS_HI_BUSY				0x00000001
+
+#define REG_A5XX_RBBM_STATUS3					0x00000530
+
+#define REG_A5XX_RBBM_INT_0_STATUS				0x000004e1
+
+#define REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS			0x000004f0
+
+#define REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS			0x000004f1
+
+#define REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS			0x000004f3
+
+#define REG_A5XX_RBBM_AHB_ERROR_STATUS				0x000004f4
+
+#define REG_A5XX_RBBM_PERFCTR_CNTL				0x00000464
+
+#define REG_A5XX_RBBM_PERFCTR_LOAD_CMD0				0x00000465
+
+#define REG_A5XX_RBBM_PERFCTR_LOAD_CMD1				0x00000466
+
+#define REG_A5XX_RBBM_PERFCTR_LOAD_CMD2				0x00000467
+
+#define REG_A5XX_RBBM_PERFCTR_LOAD_CMD3				0x00000468
+
+#define REG_A5XX_RBBM_PERFCTR_LOAD_VALUE_LO			0x00000469
+
+#define REG_A5XX_RBBM_PERFCTR_LOAD_VALUE_HI			0x0000046a
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0			0x0000046b
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_SEL_1			0x0000046c
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_SEL_2			0x0000046d
+
+#define REG_A5XX_RBBM_PERFCTR_RBBM_SEL_3			0x0000046e
+
+#define REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED			0x0000046f
+
+#define REG_A5XX_RBBM_AHB_ERROR					0x000004ed
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_EVENT_LOGIC			0x00000504
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_OVER				0x00000505
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_COUNT0				0x00000506
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_COUNT1				0x00000507
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_COUNT2				0x00000508
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_COUNT3				0x00000509
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_COUNT4				0x0000050a
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_COUNT5				0x0000050b
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_TRACE_ADDR			0x0000050c
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_TRACE_BUF0			0x0000050d
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_TRACE_BUF1			0x0000050e
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_TRACE_BUF2			0x0000050f
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_TRACE_BUF3			0x00000510
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_TRACE_BUF4			0x00000511
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_MISR0				0x00000512
+
+#define REG_A5XX_RBBM_CFG_DBGBUS_MISR1				0x00000513
+
+#define REG_A5XX_RBBM_ISDB_CNT					0x00000533
+
+#define REG_A5XX_RBBM_SECVID_TRUST_CONFIG			0x0000f000
+
+#define REG_A5XX_RBBM_SECVID_TRUST_CNTL				0x0000f400
+
+#define REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO		0x0000f800
+
+#define REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI		0x0000f801
+
+#define REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE			0x0000f802
+
+#define REG_A5XX_RBBM_SECVID_TSB_CNTL				0x0000f803
+
+#define REG_A5XX_RBBM_SECVID_TSB_COMP_STATUS_LO			0x0000f804
+
+#define REG_A5XX_RBBM_SECVID_TSB_COMP_STATUS_HI			0x0000f805
+
+#define REG_A5XX_RBBM_SECVID_TSB_UCHE_STATUS_LO			0x0000f806
+
+#define REG_A5XX_RBBM_SECVID_TSB_UCHE_STATUS_HI			0x0000f807
+
+#define REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL			0x0000f810
+
+#define REG_A5XX_VSC_PIPE_DATA_LENGTH_0				0x00000c00
+
+#define REG_A5XX_VSC_PERFCTR_VSC_SEL_0				0x00000c60
+
+#define REG_A5XX_VSC_PERFCTR_VSC_SEL_1				0x00000c61
+
+#define REG_A5XX_VSC_BIN_SIZE					0x00000cdd
+#define A5XX_VSC_BIN_SIZE_WINDOW_OFFSET_DISABLE			0x80000000
+#define A5XX_VSC_BIN_SIZE_X__MASK				0x00007fff
+#define A5XX_VSC_BIN_SIZE_X__SHIFT				0
+static inline uint32_t A5XX_VSC_BIN_SIZE_X(uint32_t val)
+{
+	return ((val) << A5XX_VSC_BIN_SIZE_X__SHIFT) & A5XX_VSC_BIN_SIZE_X__MASK;
+}
+#define A5XX_VSC_BIN_SIZE_Y__MASK				0x7fff0000
+#define A5XX_VSC_BIN_SIZE_Y__SHIFT				16
+static inline uint32_t A5XX_VSC_BIN_SIZE_Y(uint32_t val)
+{
+	return ((val) << A5XX_VSC_BIN_SIZE_Y__SHIFT) & A5XX_VSC_BIN_SIZE_Y__MASK;
+}
+
+#define REG_A5XX_GRAS_ADDR_MODE_CNTL				0x00000c81
+
+#define REG_A5XX_GRAS_PERFCTR_TSE_SEL_0				0x00000c90
+
+#define REG_A5XX_GRAS_PERFCTR_TSE_SEL_1				0x00000c91
+
+#define REG_A5XX_GRAS_PERFCTR_TSE_SEL_2				0x00000c92
+
+#define REG_A5XX_GRAS_PERFCTR_TSE_SEL_3				0x00000c93
+
+#define REG_A5XX_GRAS_PERFCTR_RAS_SEL_0				0x00000c94
+
+#define REG_A5XX_GRAS_PERFCTR_RAS_SEL_1				0x00000c95
+
+#define REG_A5XX_GRAS_PERFCTR_RAS_SEL_2				0x00000c96
+
+#define REG_A5XX_GRAS_PERFCTR_RAS_SEL_3				0x00000c97
+
+#define REG_A5XX_GRAS_PERFCTR_LRZ_SEL_0				0x00000c98
+
+#define REG_A5XX_GRAS_PERFCTR_LRZ_SEL_1				0x00000c99
+
+#define REG_A5XX_GRAS_PERFCTR_LRZ_SEL_2				0x00000c9a
+
+#define REG_A5XX_GRAS_PERFCTR_LRZ_SEL_3				0x00000c9b
+
+#define REG_A5XX_RB_DBG_ECO_CNTL				0x00000cc4
+
+#define REG_A5XX_RB_ADDR_MODE_CNTL				0x00000cc5
+
+#define REG_A5XX_RB_MODE_CNTL					0x00000cc6
+
+#define REG_A5XX_RB_CCU_CNTL					0x00000cc7
+
+#define REG_A5XX_RB_PERFCTR_RB_SEL_0				0x00000cd0
+
+#define REG_A5XX_RB_PERFCTR_RB_SEL_1				0x00000cd1
+
+#define REG_A5XX_RB_PERFCTR_RB_SEL_2				0x00000cd2
+
+#define REG_A5XX_RB_PERFCTR_RB_SEL_3				0x00000cd3
+
+#define REG_A5XX_RB_PERFCTR_RB_SEL_4				0x00000cd4
+
+#define REG_A5XX_RB_PERFCTR_RB_SEL_5				0x00000cd5
+
+#define REG_A5XX_RB_PERFCTR_RB_SEL_6				0x00000cd6
+
+#define REG_A5XX_RB_PERFCTR_RB_SEL_7				0x00000cd7
+
+#define REG_A5XX_RB_PERFCTR_CCU_SEL_0				0x00000cd8
+
+#define REG_A5XX_RB_PERFCTR_CCU_SEL_1				0x00000cd9
+
+#define REG_A5XX_RB_PERFCTR_CCU_SEL_2				0x00000cda
+
+#define REG_A5XX_RB_PERFCTR_CCU_SEL_3				0x00000cdb
+
+#define REG_A5XX_RB_POWERCTR_RB_SEL_0				0x00000ce0
+
+#define REG_A5XX_RB_POWERCTR_RB_SEL_1				0x00000ce1
+
+#define REG_A5XX_RB_POWERCTR_RB_SEL_2				0x00000ce2
+
+#define REG_A5XX_RB_POWERCTR_RB_SEL_3				0x00000ce3
+
+#define REG_A5XX_RB_POWERCTR_CCU_SEL_0				0x00000ce4
+
+#define REG_A5XX_RB_POWERCTR_CCU_SEL_1				0x00000ce5
+
+#define REG_A5XX_RB_PERFCTR_CMP_SEL_0				0x00000cec
+
+#define REG_A5XX_RB_PERFCTR_CMP_SEL_1				0x00000ced
+
+#define REG_A5XX_RB_PERFCTR_CMP_SEL_2				0x00000cee
+
+#define REG_A5XX_RB_PERFCTR_CMP_SEL_3				0x00000cef
+
+#define REG_A5XX_PC_DBG_ECO_CNTL				0x00000d00
+#define A5XX_PC_DBG_ECO_CNTL_TWOPASSUSEWFI			0x00000100
+
+#define REG_A5XX_PC_ADDR_MODE_CNTL				0x00000d01
+
+#define REG_A5XX_PC_MODE_CNTL					0x00000d02
+
+#define REG_A5XX_UNKNOWN_0D08					0x00000d08
+
+#define REG_A5XX_UNKNOWN_0D09					0x00000d09
+
+#define REG_A5XX_PC_PERFCTR_PC_SEL_0				0x00000d10
+
+#define REG_A5XX_PC_PERFCTR_PC_SEL_1				0x00000d11
+
+#define REG_A5XX_PC_PERFCTR_PC_SEL_2				0x00000d12
+
+#define REG_A5XX_PC_PERFCTR_PC_SEL_3				0x00000d13
+
+#define REG_A5XX_PC_PERFCTR_PC_SEL_4				0x00000d14
+
+#define REG_A5XX_PC_PERFCTR_PC_SEL_5				0x00000d15
+
+#define REG_A5XX_PC_PERFCTR_PC_SEL_6				0x00000d16
+
+#define REG_A5XX_PC_PERFCTR_PC_SEL_7				0x00000d17
+
+#define REG_A5XX_HLSQ_TIMEOUT_THRESHOLD_0			0x00000e00
+
+#define REG_A5XX_HLSQ_TIMEOUT_THRESHOLD_1			0x00000e01
+
+#define REG_A5XX_HLSQ_ADDR_MODE_CNTL				0x00000e05
+
+#define REG_A5XX_HLSQ_MODE_CNTL					0x00000e06
+
+#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_0			0x00000e10
+
+#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_1			0x00000e11
+
+#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_2			0x00000e12
+
+#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_3			0x00000e13
+
+#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_4			0x00000e14
+
+#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_5			0x00000e15
+
+#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_6			0x00000e16
+
+#define REG_A5XX_HLSQ_PERFCTR_HLSQ_SEL_7			0x00000e17
+
+#define REG_A5XX_HLSQ_SPTP_RDSEL				0x00000f08
+
+#define REG_A5XX_HLSQ_DBG_READ_SEL				0x0000bc00
+
+#define REG_A5XX_HLSQ_DBG_AHB_READ_APERTURE			0x0000a000
+
+#define REG_A5XX_VFD_ADDR_MODE_CNTL				0x00000e41
+
+#define REG_A5XX_VFD_MODE_CNTL					0x00000e42
+
+#define REG_A5XX_VFD_PERFCTR_VFD_SEL_0				0x00000e50
+
+#define REG_A5XX_VFD_PERFCTR_VFD_SEL_1				0x00000e51
+
+#define REG_A5XX_VFD_PERFCTR_VFD_SEL_2				0x00000e52
+
+#define REG_A5XX_VFD_PERFCTR_VFD_SEL_3				0x00000e53
+
+#define REG_A5XX_VFD_PERFCTR_VFD_SEL_4				0x00000e54
+
+#define REG_A5XX_VFD_PERFCTR_VFD_SEL_5				0x00000e55
+
+#define REG_A5XX_VFD_PERFCTR_VFD_SEL_6				0x00000e56
+
+#define REG_A5XX_VFD_PERFCTR_VFD_SEL_7				0x00000e57
+
+#define REG_A5XX_VPC_DBG_ECO_CNTL				0x00000e60
+
+#define REG_A5XX_VPC_ADDR_MODE_CNTL				0x00000e61
+
+#define REG_A5XX_VPC_MODE_CNTL					0x00000e62
+
+#define REG_A5XX_VPC_PERFCTR_VPC_SEL_0				0x00000e64
+
+#define REG_A5XX_VPC_PERFCTR_VPC_SEL_1				0x00000e65
+
+#define REG_A5XX_VPC_PERFCTR_VPC_SEL_2				0x00000e66
+
+#define REG_A5XX_VPC_PERFCTR_VPC_SEL_3				0x00000e67
+
+#define REG_A5XX_UCHE_ADDR_MODE_CNTL				0x00000e80
+
+#define REG_A5XX_UCHE_SVM_CNTL					0x00000e82
+
+#define REG_A5XX_UCHE_WRITE_THRU_BASE_LO			0x00000e87
+
+#define REG_A5XX_UCHE_WRITE_THRU_BASE_HI			0x00000e88
+
+#define REG_A5XX_UCHE_TRAP_BASE_LO				0x00000e89
+
+#define REG_A5XX_UCHE_TRAP_BASE_HI				0x00000e8a
+
+#define REG_A5XX_UCHE_GMEM_RANGE_MIN_LO				0x00000e8b
+
+#define REG_A5XX_UCHE_GMEM_RANGE_MIN_HI				0x00000e8c
+
+#define REG_A5XX_UCHE_GMEM_RANGE_MAX_LO				0x00000e8d
+
+#define REG_A5XX_UCHE_GMEM_RANGE_MAX_HI				0x00000e8e
+
+#define REG_A5XX_UCHE_DBG_ECO_CNTL_2				0x00000e8f
+
+#define REG_A5XX_UCHE_DBG_ECO_CNTL				0x00000e90
+
+#define REG_A5XX_UCHE_CACHE_INVALIDATE_MIN_LO			0x00000e91
+
+#define REG_A5XX_UCHE_CACHE_INVALIDATE_MIN_HI			0x00000e92
+
+#define REG_A5XX_UCHE_CACHE_INVALIDATE_MAX_LO			0x00000e93
+
+#define REG_A5XX_UCHE_CACHE_INVALIDATE_MAX_HI			0x00000e94
+
+#define REG_A5XX_UCHE_CACHE_INVALIDATE				0x00000e95
+
+#define REG_A5XX_UCHE_CACHE_WAYS				0x00000e96
+
+#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_0			0x00000ea0
+
+#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_1			0x00000ea1
+
+#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_2			0x00000ea2
+
+#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_3			0x00000ea3
+
+#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_4			0x00000ea4
+
+#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_5			0x00000ea5
+
+#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_6			0x00000ea6
+
+#define REG_A5XX_UCHE_PERFCTR_UCHE_SEL_7			0x00000ea7
+
+#define REG_A5XX_UCHE_POWERCTR_UCHE_SEL_0			0x00000ea8
+
+#define REG_A5XX_UCHE_POWERCTR_UCHE_SEL_1			0x00000ea9
+
+#define REG_A5XX_UCHE_POWERCTR_UCHE_SEL_2			0x00000eaa
+
+#define REG_A5XX_UCHE_POWERCTR_UCHE_SEL_3			0x00000eab
+
+#define REG_A5XX_UCHE_TRAP_LOG_LO				0x00000eb1
+
+#define REG_A5XX_UCHE_TRAP_LOG_HI				0x00000eb2
+
+#define REG_A5XX_SP_DBG_ECO_CNTL				0x00000ec0
+
+#define REG_A5XX_SP_ADDR_MODE_CNTL				0x00000ec1
+
+#define REG_A5XX_SP_MODE_CNTL					0x00000ec2
+
+#define REG_A5XX_SP_PERFCTR_SP_SEL_0				0x00000ed0
+
+#define REG_A5XX_SP_PERFCTR_SP_SEL_1				0x00000ed1
+
+#define REG_A5XX_SP_PERFCTR_SP_SEL_2				0x00000ed2
+
+#define REG_A5XX_SP_PERFCTR_SP_SEL_3				0x00000ed3
+
+#define REG_A5XX_SP_PERFCTR_SP_SEL_4				0x00000ed4
+
+#define REG_A5XX_SP_PERFCTR_SP_SEL_5				0x00000ed5
+
+#define REG_A5XX_SP_PERFCTR_SP_SEL_6				0x00000ed6
+
+#define REG_A5XX_SP_PERFCTR_SP_SEL_7				0x00000ed7
+
+#define REG_A5XX_SP_PERFCTR_SP_SEL_8				0x00000ed8
+
+#define REG_A5XX_SP_PERFCTR_SP_SEL_9				0x00000ed9
+
+#define REG_A5XX_SP_PERFCTR_SP_SEL_10				0x00000eda
+
+#define REG_A5XX_SP_PERFCTR_SP_SEL_11				0x00000edb
+
+#define REG_A5XX_SP_POWERCTR_SP_SEL_0				0x00000edc
+
+#define REG_A5XX_SP_POWERCTR_SP_SEL_1				0x00000edd
+
+#define REG_A5XX_SP_POWERCTR_SP_SEL_2				0x00000ede
+
+#define REG_A5XX_SP_POWERCTR_SP_SEL_3				0x00000edf
+
+#define REG_A5XX_TPL1_ADDR_MODE_CNTL				0x00000f01
+
+#define REG_A5XX_TPL1_MODE_CNTL					0x00000f02
+
+#define REG_A5XX_TPL1_PERFCTR_TP_SEL_0				0x00000f10
+
+#define REG_A5XX_TPL1_PERFCTR_TP_SEL_1				0x00000f11
+
+#define REG_A5XX_TPL1_PERFCTR_TP_SEL_2				0x00000f12
+
+#define REG_A5XX_TPL1_PERFCTR_TP_SEL_3				0x00000f13
+
+#define REG_A5XX_TPL1_PERFCTR_TP_SEL_4				0x00000f14
+
+#define REG_A5XX_TPL1_PERFCTR_TP_SEL_5				0x00000f15
+
+#define REG_A5XX_TPL1_PERFCTR_TP_SEL_6				0x00000f16
+
+#define REG_A5XX_TPL1_PERFCTR_TP_SEL_7				0x00000f17
+
+#define REG_A5XX_TPL1_POWERCTR_TP_SEL_0				0x00000f18
+
+#define REG_A5XX_TPL1_POWERCTR_TP_SEL_1				0x00000f19
+
+#define REG_A5XX_TPL1_POWERCTR_TP_SEL_2				0x00000f1a
+
+#define REG_A5XX_TPL1_POWERCTR_TP_SEL_3				0x00000f1b
+
+#define REG_A5XX_VBIF_VERSION					0x00003000
+
+#define REG_A5XX_VBIF_CLKON					0x00003001
+
+#define REG_A5XX_VBIF_ABIT_SORT					0x00003028
+
+#define REG_A5XX_VBIF_ABIT_SORT_CONF				0x00003029
+
+#define REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB			0x00003049
+
+#define REG_A5XX_VBIF_GATE_OFF_WRREQ_EN				0x0000302a
+
+#define REG_A5XX_VBIF_IN_RD_LIM_CONF0				0x0000302c
+
+#define REG_A5XX_VBIF_IN_RD_LIM_CONF1				0x0000302d
+
+#define REG_A5XX_VBIF_XIN_HALT_CTRL0				0x00003080
+
+#define REG_A5XX_VBIF_XIN_HALT_CTRL1				0x00003081
+
+#define REG_A5XX_VBIF_TEST_BUS_OUT_CTRL				0x00003084
+
+#define REG_A5XX_VBIF_TEST_BUS1_CTRL0				0x00003085
+
+#define REG_A5XX_VBIF_TEST_BUS1_CTRL1				0x00003086
+
+#define REG_A5XX_VBIF_TEST_BUS2_CTRL0				0x00003087
+
+#define REG_A5XX_VBIF_TEST_BUS2_CTRL1				0x00003088
+
+#define REG_A5XX_VBIF_TEST_BUS_OUT				0x0000308c
+
+#define REG_A5XX_VBIF_PERF_CNT_SEL0				0x000030d0
+
+#define REG_A5XX_VBIF_PERF_CNT_SEL1				0x000030d1
+
+#define REG_A5XX_VBIF_PERF_CNT_SEL2				0x000030d2
+
+#define REG_A5XX_VBIF_PERF_CNT_SEL3				0x000030d3
+
+#define REG_A5XX_VBIF_PERF_CNT_LOW0				0x000030d8
+
+#define REG_A5XX_VBIF_PERF_CNT_LOW1				0x000030d9
+
+#define REG_A5XX_VBIF_PERF_CNT_LOW2				0x000030da
+
+#define REG_A5XX_VBIF_PERF_CNT_LOW3				0x000030db
+
+#define REG_A5XX_VBIF_PERF_CNT_HIGH0				0x000030e0
+
+#define REG_A5XX_VBIF_PERF_CNT_HIGH1				0x000030e1
+
+#define REG_A5XX_VBIF_PERF_CNT_HIGH2				0x000030e2
+
+#define REG_A5XX_VBIF_PERF_CNT_HIGH3				0x000030e3
+
+#define REG_A5XX_VBIF_PERF_PWR_CNT_EN0				0x00003100
+
+#define REG_A5XX_VBIF_PERF_PWR_CNT_EN1				0x00003101
+
+#define REG_A5XX_VBIF_PERF_PWR_CNT_EN2				0x00003102
+
+#define REG_A5XX_VBIF_PERF_PWR_CNT_LOW0				0x00003110
+
+#define REG_A5XX_VBIF_PERF_PWR_CNT_LOW1				0x00003111
+
+#define REG_A5XX_VBIF_PERF_PWR_CNT_LOW2				0x00003112
+
+#define REG_A5XX_VBIF_PERF_PWR_CNT_HIGH0			0x00003118
+
+#define REG_A5XX_VBIF_PERF_PWR_CNT_HIGH1			0x00003119
+
+#define REG_A5XX_VBIF_PERF_PWR_CNT_HIGH2			0x0000311a
+
+#define REG_A5XX_GPMU_INST_RAM_BASE				0x00008800
+
+#define REG_A5XX_GPMU_DATA_RAM_BASE				0x00009800
+
+#define REG_A5XX_GPMU_SP_POWER_CNTL				0x0000a881
+
+#define REG_A5XX_GPMU_RBCCU_CLOCK_CNTL				0x0000a886
+
+#define REG_A5XX_GPMU_RBCCU_POWER_CNTL				0x0000a887
+
+#define REG_A5XX_GPMU_SP_PWR_CLK_STATUS				0x0000a88b
+#define A5XX_GPMU_SP_PWR_CLK_STATUS_PWR_ON			0x00100000
+
+#define REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS			0x0000a88d
+#define A5XX_GPMU_RBCCU_PWR_CLK_STATUS_PWR_ON			0x00100000
+
+#define REG_A5XX_GPMU_PWR_COL_STAGGER_DELAY			0x0000a891
+
+#define REG_A5XX_GPMU_PWR_COL_INTER_FRAME_CTRL			0x0000a892
+
+#define REG_A5XX_GPMU_PWR_COL_INTER_FRAME_HYST			0x0000a893
+
+#define REG_A5XX_GPMU_PWR_COL_BINNING_CTRL			0x0000a894
+
+#define REG_A5XX_GPMU_CLOCK_THROTTLE_CTRL			0x0000a8a3
+
+#define REG_A5XX_GPMU_WFI_CONFIG				0x0000a8c1
+
+#define REG_A5XX_GPMU_RBBM_INTR_INFO				0x0000a8d6
+
+#define REG_A5XX_GPMU_CM3_SYSRESET				0x0000a8d8
+
+#define REG_A5XX_GPMU_GENERAL_0					0x0000a8e0
+
+#define REG_A5XX_GPMU_GENERAL_1					0x0000a8e1
+
+#define REG_A5XX_SP_POWER_COUNTER_0_LO				0x0000a840
+
+#define REG_A5XX_SP_POWER_COUNTER_0_HI				0x0000a841
+
+#define REG_A5XX_SP_POWER_COUNTER_1_LO				0x0000a842
+
+#define REG_A5XX_SP_POWER_COUNTER_1_HI				0x0000a843
+
+#define REG_A5XX_SP_POWER_COUNTER_2_LO				0x0000a844
+
+#define REG_A5XX_SP_POWER_COUNTER_2_HI				0x0000a845
+
+#define REG_A5XX_SP_POWER_COUNTER_3_LO				0x0000a846
+
+#define REG_A5XX_SP_POWER_COUNTER_3_HI				0x0000a847
+
+#define REG_A5XX_TP_POWER_COUNTER_0_LO				0x0000a848
+
+#define REG_A5XX_TP_POWER_COUNTER_0_HI				0x0000a849
+
+#define REG_A5XX_TP_POWER_COUNTER_1_LO				0x0000a84a
+
+#define REG_A5XX_TP_POWER_COUNTER_1_HI				0x0000a84b
+
+#define REG_A5XX_TP_POWER_COUNTER_2_LO				0x0000a84c
+
+#define REG_A5XX_TP_POWER_COUNTER_2_HI				0x0000a84d
+
+#define REG_A5XX_TP_POWER_COUNTER_3_LO				0x0000a84e
+
+#define REG_A5XX_TP_POWER_COUNTER_3_HI				0x0000a84f
+
+#define REG_A5XX_RB_POWER_COUNTER_0_LO				0x0000a850
+
+#define REG_A5XX_RB_POWER_COUNTER_0_HI				0x0000a851
+
+#define REG_A5XX_RB_POWER_COUNTER_1_LO				0x0000a852
+
+#define REG_A5XX_RB_POWER_COUNTER_1_HI				0x0000a853
+
+#define REG_A5XX_RB_POWER_COUNTER_2_LO				0x0000a854
+
+#define REG_A5XX_RB_POWER_COUNTER_2_HI				0x0000a855
+
+#define REG_A5XX_RB_POWER_COUNTER_3_LO				0x0000a856
+
+#define REG_A5XX_RB_POWER_COUNTER_3_HI				0x0000a857
+
+#define REG_A5XX_CCU_POWER_COUNTER_0_LO				0x0000a858
+
+#define REG_A5XX_CCU_POWER_COUNTER_0_HI				0x0000a859
+
+#define REG_A5XX_CCU_POWER_COUNTER_1_LO				0x0000a85a
+
+#define REG_A5XX_CCU_POWER_COUNTER_1_HI				0x0000a85b
+
+#define REG_A5XX_UCHE_POWER_COUNTER_0_LO			0x0000a85c
+
+#define REG_A5XX_UCHE_POWER_COUNTER_0_HI			0x0000a85d
+
+#define REG_A5XX_UCHE_POWER_COUNTER_1_LO			0x0000a85e
+
+#define REG_A5XX_UCHE_POWER_COUNTER_1_HI			0x0000a85f
+
+#define REG_A5XX_UCHE_POWER_COUNTER_2_LO			0x0000a860
+
+#define REG_A5XX_UCHE_POWER_COUNTER_2_HI			0x0000a861
+
+#define REG_A5XX_UCHE_POWER_COUNTER_3_LO			0x0000a862
+
+#define REG_A5XX_UCHE_POWER_COUNTER_3_HI			0x0000a863
+
+#define REG_A5XX_CP_POWER_COUNTER_0_LO				0x0000a864
+
+#define REG_A5XX_CP_POWER_COUNTER_0_HI				0x0000a865
+
+#define REG_A5XX_CP_POWER_COUNTER_1_LO				0x0000a866
+
+#define REG_A5XX_CP_POWER_COUNTER_1_HI				0x0000a867
+
+#define REG_A5XX_CP_POWER_COUNTER_2_LO				0x0000a868
+
+#define REG_A5XX_CP_POWER_COUNTER_2_HI				0x0000a869
+
+#define REG_A5XX_CP_POWER_COUNTER_3_LO				0x0000a86a
+
+#define REG_A5XX_CP_POWER_COUNTER_3_HI				0x0000a86b
+
+#define REG_A5XX_GPMU_POWER_COUNTER_0_LO			0x0000a86c
+
+#define REG_A5XX_GPMU_POWER_COUNTER_0_HI			0x0000a86d
+
+#define REG_A5XX_GPMU_POWER_COUNTER_1_LO			0x0000a86e
+
+#define REG_A5XX_GPMU_POWER_COUNTER_1_HI			0x0000a86f
+
+#define REG_A5XX_GPMU_POWER_COUNTER_2_LO			0x0000a870
+
+#define REG_A5XX_GPMU_POWER_COUNTER_2_HI			0x0000a871
+
+#define REG_A5XX_GPMU_POWER_COUNTER_3_LO			0x0000a872
+
+#define REG_A5XX_GPMU_POWER_COUNTER_3_HI			0x0000a873
+
+#define REG_A5XX_GPMU_POWER_COUNTER_4_LO			0x0000a874
+
+#define REG_A5XX_GPMU_POWER_COUNTER_4_HI			0x0000a875
+
+#define REG_A5XX_GPMU_POWER_COUNTER_5_LO			0x0000a876
+
+#define REG_A5XX_GPMU_POWER_COUNTER_5_HI			0x0000a877
+
+#define REG_A5XX_GPMU_POWER_COUNTER_ENABLE			0x0000a878
+
+#define REG_A5XX_GPMU_ALWAYS_ON_COUNTER_LO			0x0000a879
+
+#define REG_A5XX_GPMU_ALWAYS_ON_COUNTER_HI			0x0000a87a
+
+#define REG_A5XX_GPMU_ALWAYS_ON_COUNTER_RESET			0x0000a87b
+
+#define REG_A5XX_GPMU_POWER_COUNTER_SELECT_0			0x0000a87c
+
+#define REG_A5XX_GPMU_POWER_COUNTER_SELECT_1			0x0000a87d
+
+#define REG_A5XX_GPMU_CLOCK_THROTTLE_CTRL			0x0000a8a3
+
+#define REG_A5XX_GPMU_THROTTLE_UNMASK_FORCE_CTRL		0x0000a8a8
+
+#define REG_A5XX_GPMU_TEMP_SENSOR_ID				0x0000ac00
+
+#define REG_A5XX_GPMU_TEMP_SENSOR_CONFIG			0x0000ac01
+
+#define REG_A5XX_GPMU_TEMP_VAL					0x0000ac02
+
+#define REG_A5XX_GPMU_DELTA_TEMP_THRESHOLD			0x0000ac03
+
+#define REG_A5XX_GPMU_TEMP_THRESHOLD_INTR_STATUS		0x0000ac05
+
+#define REG_A5XX_GPMU_TEMP_THRESHOLD_INTR_EN_MASK		0x0000ac06
+
+#define REG_A5XX_GPMU_LEAKAGE_TEMP_COEFF_0_1			0x0000ac40
+
+#define REG_A5XX_GPMU_LEAKAGE_TEMP_COEFF_2_3			0x0000ac41
+
+#define REG_A5XX_GPMU_LEAKAGE_VTG_COEFF_0_1			0x0000ac42
+
+#define REG_A5XX_GPMU_LEAKAGE_VTG_COEFF_2_3			0x0000ac43
+
+#define REG_A5XX_GPMU_BASE_LEAKAGE				0x0000ac46
+
+#define REG_A5XX_GPMU_GPMU_VOLTAGE				0x0000ac60
+
+#define REG_A5XX_GPMU_GPMU_VOLTAGE_INTR_STATUS			0x0000ac61
+
+#define REG_A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK			0x0000ac62
+
+#define REG_A5XX_GPMU_GPMU_PWR_THRESHOLD			0x0000ac80
+
+#define REG_A5XX_GPMU_GPMU_LLM_GLM_SLEEP_CTRL			0x0000acc4
+
+#define REG_A5XX_GPMU_GPMU_LLM_GLM_SLEEP_STATUS			0x0000acc5
+
+#define REG_A5XX_GDPM_CONFIG1					0x0000b80c
+
+#define REG_A5XX_GDPM_CONFIG2					0x0000b80d
+
+#define REG_A5XX_GDPM_INT_EN					0x0000b80f
+
+#define REG_A5XX_GDPM_INT_MASK					0x0000b811
+
+#define REG_A5XX_GPMU_BEC_ENABLE				0x0000b9a0
+
+#define REG_A5XX_GPU_CS_SENSOR_GENERAL_STATUS			0x0000c41a
+
+#define REG_A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_0		0x0000c41d
+
+#define REG_A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_2		0x0000c41f
+
+#define REG_A5XX_GPU_CS_AMP_CALIBRATION_STATUS1_4		0x0000c421
+
+#define REG_A5XX_GPU_CS_ENABLE_REG				0x0000c520
+
+#define REG_A5XX_GPU_CS_AMP_CALIBRATION_CONTROL1		0x0000c557
+
+#define REG_A5XX_GRAS_CL_CNTL					0x0000e000
+
+#define REG_A5XX_UNKNOWN_E001					0x0000e001
+
+#define REG_A5XX_UNKNOWN_E004					0x0000e004
+
+#define REG_A5XX_GRAS_CNTL					0x0000e005
+#define A5XX_GRAS_CNTL_VARYING					0x00000001
+
+#define REG_A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ			0x0000e006
+#define A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__MASK		0x000003ff
+#define A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__SHIFT		0
+static inline uint32_t A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__SHIFT) & A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__MASK;
+}
+#define A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__MASK		0x000ffc00
+#define A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__SHIFT		10
+static inline uint32_t A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__SHIFT) & A5XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__MASK;
+}
+
+#define REG_A5XX_GRAS_CL_VPORT_XOFFSET_0			0x0000e010
+#define A5XX_GRAS_CL_VPORT_XOFFSET_0__MASK			0xffffffff
+#define A5XX_GRAS_CL_VPORT_XOFFSET_0__SHIFT			0
+static inline uint32_t A5XX_GRAS_CL_VPORT_XOFFSET_0(float val)
+{
+	return ((fui(val)) << A5XX_GRAS_CL_VPORT_XOFFSET_0__SHIFT) & A5XX_GRAS_CL_VPORT_XOFFSET_0__MASK;
+}
+
+#define REG_A5XX_GRAS_CL_VPORT_XSCALE_0				0x0000e011
+#define A5XX_GRAS_CL_VPORT_XSCALE_0__MASK			0xffffffff
+#define A5XX_GRAS_CL_VPORT_XSCALE_0__SHIFT			0
+static inline uint32_t A5XX_GRAS_CL_VPORT_XSCALE_0(float val)
+{
+	return ((fui(val)) << A5XX_GRAS_CL_VPORT_XSCALE_0__SHIFT) & A5XX_GRAS_CL_VPORT_XSCALE_0__MASK;
+}
+
+#define REG_A5XX_GRAS_CL_VPORT_YOFFSET_0			0x0000e012
+#define A5XX_GRAS_CL_VPORT_YOFFSET_0__MASK			0xffffffff
+#define A5XX_GRAS_CL_VPORT_YOFFSET_0__SHIFT			0
+static inline uint32_t A5XX_GRAS_CL_VPORT_YOFFSET_0(float val)
+{
+	return ((fui(val)) << A5XX_GRAS_CL_VPORT_YOFFSET_0__SHIFT) & A5XX_GRAS_CL_VPORT_YOFFSET_0__MASK;
+}
+
+#define REG_A5XX_GRAS_CL_VPORT_YSCALE_0				0x0000e013
+#define A5XX_GRAS_CL_VPORT_YSCALE_0__MASK			0xffffffff
+#define A5XX_GRAS_CL_VPORT_YSCALE_0__SHIFT			0
+static inline uint32_t A5XX_GRAS_CL_VPORT_YSCALE_0(float val)
+{
+	return ((fui(val)) << A5XX_GRAS_CL_VPORT_YSCALE_0__SHIFT) & A5XX_GRAS_CL_VPORT_YSCALE_0__MASK;
+}
+
+#define REG_A5XX_GRAS_CL_VPORT_ZOFFSET_0			0x0000e014
+#define A5XX_GRAS_CL_VPORT_ZOFFSET_0__MASK			0xffffffff
+#define A5XX_GRAS_CL_VPORT_ZOFFSET_0__SHIFT			0
+static inline uint32_t A5XX_GRAS_CL_VPORT_ZOFFSET_0(float val)
+{
+	return ((fui(val)) << A5XX_GRAS_CL_VPORT_ZOFFSET_0__SHIFT) & A5XX_GRAS_CL_VPORT_ZOFFSET_0__MASK;
+}
+
+#define REG_A5XX_GRAS_CL_VPORT_ZSCALE_0				0x0000e015
+#define A5XX_GRAS_CL_VPORT_ZSCALE_0__MASK			0xffffffff
+#define A5XX_GRAS_CL_VPORT_ZSCALE_0__SHIFT			0
+static inline uint32_t A5XX_GRAS_CL_VPORT_ZSCALE_0(float val)
+{
+	return ((fui(val)) << A5XX_GRAS_CL_VPORT_ZSCALE_0__SHIFT) & A5XX_GRAS_CL_VPORT_ZSCALE_0__MASK;
+}
+
+#define REG_A5XX_GRAS_SU_CNTL					0x0000e090
+#define A5XX_GRAS_SU_CNTL_FRONT_CW				0x00000004
+#define A5XX_GRAS_SU_CNTL_LINEHALFWIDTH__MASK			0x000007f8
+#define A5XX_GRAS_SU_CNTL_LINEHALFWIDTH__SHIFT			3
+static inline uint32_t A5XX_GRAS_SU_CNTL_LINEHALFWIDTH(float val)
+{
+	return ((((int32_t)(val * 4.0))) << A5XX_GRAS_SU_CNTL_LINEHALFWIDTH__SHIFT) & A5XX_GRAS_SU_CNTL_LINEHALFWIDTH__MASK;
+}
+#define A5XX_GRAS_SU_CNTL_POLY_OFFSET				0x00000800
+#define A5XX_GRAS_SU_CNTL_MSAA_ENABLE				0x00002000
+
+#define REG_A5XX_GRAS_SU_POINT_MINMAX				0x0000e091
+#define A5XX_GRAS_SU_POINT_MINMAX_MIN__MASK			0x0000ffff
+#define A5XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT			0
+static inline uint32_t A5XX_GRAS_SU_POINT_MINMAX_MIN(float val)
+{
+	return ((((uint32_t)(val * 16.0))) << A5XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT) & A5XX_GRAS_SU_POINT_MINMAX_MIN__MASK;
+}
+#define A5XX_GRAS_SU_POINT_MINMAX_MAX__MASK			0xffff0000
+#define A5XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT			16
+static inline uint32_t A5XX_GRAS_SU_POINT_MINMAX_MAX(float val)
+{
+	return ((((uint32_t)(val * 16.0))) << A5XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT) & A5XX_GRAS_SU_POINT_MINMAX_MAX__MASK;
+}
+
+#define REG_A5XX_GRAS_SU_POINT_SIZE				0x0000e092
+#define A5XX_GRAS_SU_POINT_SIZE__MASK				0xffffffff
+#define A5XX_GRAS_SU_POINT_SIZE__SHIFT				0
+static inline uint32_t A5XX_GRAS_SU_POINT_SIZE(float val)
+{
+	return ((((int32_t)(val * 16.0))) << A5XX_GRAS_SU_POINT_SIZE__SHIFT) & A5XX_GRAS_SU_POINT_SIZE__MASK;
+}
+
+#define REG_A5XX_UNKNOWN_E093					0x0000e093
+
+#define REG_A5XX_GRAS_SU_DEPTH_PLANE_CNTL			0x0000e094
+#define A5XX_GRAS_SU_DEPTH_PLANE_CNTL_ALPHA_TEST_ENABLE		0x00000001
+
+#define REG_A5XX_GRAS_SU_POLY_OFFSET_SCALE			0x0000e095
+#define A5XX_GRAS_SU_POLY_OFFSET_SCALE__MASK			0xffffffff
+#define A5XX_GRAS_SU_POLY_OFFSET_SCALE__SHIFT			0
+static inline uint32_t A5XX_GRAS_SU_POLY_OFFSET_SCALE(float val)
+{
+	return ((fui(val)) << A5XX_GRAS_SU_POLY_OFFSET_SCALE__SHIFT) & A5XX_GRAS_SU_POLY_OFFSET_SCALE__MASK;
+}
+
+#define REG_A5XX_GRAS_SU_POLY_OFFSET_OFFSET			0x0000e096
+#define A5XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK			0xffffffff
+#define A5XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT			0
+static inline uint32_t A5XX_GRAS_SU_POLY_OFFSET_OFFSET(float val)
+{
+	return ((fui(val)) << A5XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT) & A5XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK;
+}
+
+#define REG_A5XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP		0x0000e097
+#define A5XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__MASK		0xffffffff
+#define A5XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__SHIFT		0
+static inline uint32_t A5XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP(float val)
+{
+	return ((fui(val)) << A5XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__SHIFT) & A5XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__MASK;
+}
+
+#define REG_A5XX_GRAS_SU_DEPTH_BUFFER_INFO			0x0000e098
+#define A5XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK	0x00000007
+#define A5XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT	0
+static inline uint32_t A5XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT(enum a5xx_depth_format val)
+{
+	return ((val) << A5XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT) & A5XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK;
+}
+
+#define REG_A5XX_GRAS_SU_CONSERVATIVE_RAS_CNTL			0x0000e099
+
+#define REG_A5XX_GRAS_SC_CNTL					0x0000e0a0
+#define A5XX_GRAS_SC_CNTL_SAMPLES_PASSED			0x00008000
+
+#define REG_A5XX_GRAS_SC_BIN_CNTL				0x0000e0a1
+
+#define REG_A5XX_GRAS_SC_RAS_MSAA_CNTL				0x0000e0a2
+#define A5XX_GRAS_SC_RAS_MSAA_CNTL_SAMPLES__MASK		0x00000003
+#define A5XX_GRAS_SC_RAS_MSAA_CNTL_SAMPLES__SHIFT		0
+static inline uint32_t A5XX_GRAS_SC_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A5XX_GRAS_SC_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A5XX_GRAS_SC_RAS_MSAA_CNTL_SAMPLES__MASK;
+}
+
+#define REG_A5XX_GRAS_SC_DEST_MSAA_CNTL				0x0000e0a3
+#define A5XX_GRAS_SC_DEST_MSAA_CNTL_SAMPLES__MASK		0x00000003
+#define A5XX_GRAS_SC_DEST_MSAA_CNTL_SAMPLES__SHIFT		0
+static inline uint32_t A5XX_GRAS_SC_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A5XX_GRAS_SC_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A5XX_GRAS_SC_DEST_MSAA_CNTL_SAMPLES__MASK;
+}
+#define A5XX_GRAS_SC_DEST_MSAA_CNTL_MSAA_DISABLE		0x00000004
+
+#define REG_A5XX_GRAS_SC_SCREEN_SCISSOR_CNTL			0x0000e0a4
+
+#define REG_A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0			0x0000e0aa
+#define A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_WINDOW_OFFSET_DISABLE	0x80000000
+#define A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__MASK		0x00007fff
+#define A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__SHIFT		0
+static inline uint32_t A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__SHIFT) & A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__MASK;
+}
+#define A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__MASK		0x7fff0000
+#define A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__SHIFT		16
+static inline uint32_t A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__SHIFT) & A5XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__MASK;
+}
+
+#define REG_A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0			0x0000e0ab
+#define A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_WINDOW_OFFSET_DISABLE	0x80000000
+#define A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__MASK		0x00007fff
+#define A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__SHIFT		0
+static inline uint32_t A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__SHIFT) & A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__MASK;
+}
+#define A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__MASK		0x7fff0000
+#define A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__SHIFT		16
+static inline uint32_t A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__SHIFT) & A5XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__MASK;
+}
+
+#define REG_A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0			0x0000e0ca
+#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_WINDOW_OFFSET_DISABLE	0x80000000
+#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__MASK		0x00007fff
+#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__SHIFT		0
+static inline uint32_t A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__SHIFT) & A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__MASK;
+}
+#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__MASK		0x7fff0000
+#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__SHIFT		16
+static inline uint32_t A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__SHIFT) & A5XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__MASK;
+}
+
+#define REG_A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0			0x0000e0cb
+#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_WINDOW_OFFSET_DISABLE	0x80000000
+#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__MASK		0x00007fff
+#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__SHIFT		0
+static inline uint32_t A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__SHIFT) & A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__MASK;
+}
+#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__MASK		0x7fff0000
+#define A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__SHIFT		16
+static inline uint32_t A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__SHIFT) & A5XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__MASK;
+}
+
+#define REG_A5XX_GRAS_SC_WINDOW_SCISSOR_TL			0x0000e0ea
+#define A5XX_GRAS_SC_WINDOW_SCISSOR_TL_WINDOW_OFFSET_DISABLE	0x80000000
+#define A5XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK			0x00007fff
+#define A5XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT			0
+static inline uint32_t A5XX_GRAS_SC_WINDOW_SCISSOR_TL_X(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT) & A5XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK;
+}
+#define A5XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK			0x7fff0000
+#define A5XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT			16
+static inline uint32_t A5XX_GRAS_SC_WINDOW_SCISSOR_TL_Y(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT) & A5XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK;
+}
+
+#define REG_A5XX_GRAS_SC_WINDOW_SCISSOR_BR			0x0000e0eb
+#define A5XX_GRAS_SC_WINDOW_SCISSOR_BR_WINDOW_OFFSET_DISABLE	0x80000000
+#define A5XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK			0x00007fff
+#define A5XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT			0
+static inline uint32_t A5XX_GRAS_SC_WINDOW_SCISSOR_BR_X(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT) & A5XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK;
+}
+#define A5XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK			0x7fff0000
+#define A5XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT			16
+static inline uint32_t A5XX_GRAS_SC_WINDOW_SCISSOR_BR_Y(uint32_t val)
+{
+	return ((val) << A5XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT) & A5XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK;
+}
+
+#define REG_A5XX_GRAS_LRZ_CNTL					0x0000e100
+
+#define REG_A5XX_GRAS_LRZ_BUFFER_BASE_LO			0x0000e101
+
+#define REG_A5XX_GRAS_LRZ_BUFFER_BASE_HI			0x0000e102
+
+#define REG_A5XX_GRAS_LRZ_BUFFER_PITCH				0x0000e103
+
+#define REG_A5XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_LO		0x0000e104
+
+#define REG_A5XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_HI		0x0000e105
+
+#define REG_A5XX_RB_CNTL					0x0000e140
+#define A5XX_RB_CNTL_WIDTH__MASK				0x000000ff
+#define A5XX_RB_CNTL_WIDTH__SHIFT				0
+static inline uint32_t A5XX_RB_CNTL_WIDTH(uint32_t val)
+{
+	return ((val >> 5) << A5XX_RB_CNTL_WIDTH__SHIFT) & A5XX_RB_CNTL_WIDTH__MASK;
+}
+#define A5XX_RB_CNTL_HEIGHT__MASK				0x0001fe00
+#define A5XX_RB_CNTL_HEIGHT__SHIFT				9
+static inline uint32_t A5XX_RB_CNTL_HEIGHT(uint32_t val)
+{
+	return ((val >> 5) << A5XX_RB_CNTL_HEIGHT__SHIFT) & A5XX_RB_CNTL_HEIGHT__MASK;
+}
+#define A5XX_RB_CNTL_BYPASS					0x00020000
+
+#define REG_A5XX_RB_RENDER_CNTL					0x0000e141
+#define A5XX_RB_RENDER_CNTL_SAMPLES_PASSED			0x00000040
+#define A5XX_RB_RENDER_CNTL_FLAG_DEPTH				0x00004000
+#define A5XX_RB_RENDER_CNTL_FLAG_DEPTH2				0x00008000
+#define A5XX_RB_RENDER_CNTL_FLAG_MRTS__MASK			0x00ff0000
+#define A5XX_RB_RENDER_CNTL_FLAG_MRTS__SHIFT			16
+static inline uint32_t A5XX_RB_RENDER_CNTL_FLAG_MRTS(uint32_t val)
+{
+	return ((val) << A5XX_RB_RENDER_CNTL_FLAG_MRTS__SHIFT) & A5XX_RB_RENDER_CNTL_FLAG_MRTS__MASK;
+}
+#define A5XX_RB_RENDER_CNTL_FLAG_MRTS2__MASK			0xff000000
+#define A5XX_RB_RENDER_CNTL_FLAG_MRTS2__SHIFT			24
+static inline uint32_t A5XX_RB_RENDER_CNTL_FLAG_MRTS2(uint32_t val)
+{
+	return ((val) << A5XX_RB_RENDER_CNTL_FLAG_MRTS2__SHIFT) & A5XX_RB_RENDER_CNTL_FLAG_MRTS2__MASK;
+}
+
+#define REG_A5XX_RB_RAS_MSAA_CNTL				0x0000e142
+#define A5XX_RB_RAS_MSAA_CNTL_SAMPLES__MASK			0x00000003
+#define A5XX_RB_RAS_MSAA_CNTL_SAMPLES__SHIFT			0
+static inline uint32_t A5XX_RB_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A5XX_RB_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A5XX_RB_RAS_MSAA_CNTL_SAMPLES__MASK;
+}
+
+#define REG_A5XX_RB_DEST_MSAA_CNTL				0x0000e143
+#define A5XX_RB_DEST_MSAA_CNTL_SAMPLES__MASK			0x00000003
+#define A5XX_RB_DEST_MSAA_CNTL_SAMPLES__SHIFT			0
+static inline uint32_t A5XX_RB_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A5XX_RB_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A5XX_RB_DEST_MSAA_CNTL_SAMPLES__MASK;
+}
+#define A5XX_RB_DEST_MSAA_CNTL_MSAA_DISABLE			0x00000004
+
+#define REG_A5XX_RB_RENDER_CONTROL0				0x0000e144
+#define A5XX_RB_RENDER_CONTROL0_VARYING				0x00000001
+#define A5XX_RB_RENDER_CONTROL0_XCOORD				0x00000040
+#define A5XX_RB_RENDER_CONTROL0_YCOORD				0x00000080
+#define A5XX_RB_RENDER_CONTROL0_ZCOORD				0x00000100
+#define A5XX_RB_RENDER_CONTROL0_WCOORD				0x00000200
+
+#define REG_A5XX_RB_RENDER_CONTROL1				0x0000e145
+#define A5XX_RB_RENDER_CONTROL1_FACENESS			0x00000002
+
+#define REG_A5XX_RB_FS_OUTPUT_CNTL				0x0000e146
+#define A5XX_RB_FS_OUTPUT_CNTL_MRT__MASK			0x0000000f
+#define A5XX_RB_FS_OUTPUT_CNTL_MRT__SHIFT			0
+static inline uint32_t A5XX_RB_FS_OUTPUT_CNTL_MRT(uint32_t val)
+{
+	return ((val) << A5XX_RB_FS_OUTPUT_CNTL_MRT__SHIFT) & A5XX_RB_FS_OUTPUT_CNTL_MRT__MASK;
+}
+#define A5XX_RB_FS_OUTPUT_CNTL_FRAG_WRITES_Z			0x00000020
+
+#define REG_A5XX_RB_RENDER_COMPONENTS				0x0000e147
+#define A5XX_RB_RENDER_COMPONENTS_RT0__MASK			0x0000000f
+#define A5XX_RB_RENDER_COMPONENTS_RT0__SHIFT			0
+static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT0(uint32_t val)
+{
+	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT0__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT0__MASK;
+}
+#define A5XX_RB_RENDER_COMPONENTS_RT1__MASK			0x000000f0
+#define A5XX_RB_RENDER_COMPONENTS_RT1__SHIFT			4
+static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT1(uint32_t val)
+{
+	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT1__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT1__MASK;
+}
+#define A5XX_RB_RENDER_COMPONENTS_RT2__MASK			0x00000f00
+#define A5XX_RB_RENDER_COMPONENTS_RT2__SHIFT			8
+static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT2(uint32_t val)
+{
+	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT2__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT2__MASK;
+}
+#define A5XX_RB_RENDER_COMPONENTS_RT3__MASK			0x0000f000
+#define A5XX_RB_RENDER_COMPONENTS_RT3__SHIFT			12
+static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT3(uint32_t val)
+{
+	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT3__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT3__MASK;
+}
+#define A5XX_RB_RENDER_COMPONENTS_RT4__MASK			0x000f0000
+#define A5XX_RB_RENDER_COMPONENTS_RT4__SHIFT			16
+static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT4(uint32_t val)
+{
+	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT4__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT4__MASK;
+}
+#define A5XX_RB_RENDER_COMPONENTS_RT5__MASK			0x00f00000
+#define A5XX_RB_RENDER_COMPONENTS_RT5__SHIFT			20
+static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT5(uint32_t val)
+{
+	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT5__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT5__MASK;
+}
+#define A5XX_RB_RENDER_COMPONENTS_RT6__MASK			0x0f000000
+#define A5XX_RB_RENDER_COMPONENTS_RT6__SHIFT			24
+static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT6(uint32_t val)
+{
+	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT6__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT6__MASK;
+}
+#define A5XX_RB_RENDER_COMPONENTS_RT7__MASK			0xf0000000
+#define A5XX_RB_RENDER_COMPONENTS_RT7__SHIFT			28
+static inline uint32_t A5XX_RB_RENDER_COMPONENTS_RT7(uint32_t val)
+{
+	return ((val) << A5XX_RB_RENDER_COMPONENTS_RT7__SHIFT) & A5XX_RB_RENDER_COMPONENTS_RT7__MASK;
+}
+
+static inline uint32_t REG_A5XX_RB_MRT(uint32_t i0) { return 0x0000e150 + 0x7*i0; }
+
+static inline uint32_t REG_A5XX_RB_MRT_CONTROL(uint32_t i0) { return 0x0000e150 + 0x7*i0; }
+#define A5XX_RB_MRT_CONTROL_BLEND				0x00000001
+#define A5XX_RB_MRT_CONTROL_BLEND2				0x00000002
+#define A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK		0x00000780
+#define A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT		7
+static inline uint32_t A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE(uint32_t val)
+{
+	return ((val) << A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT) & A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK;
+}
+
+static inline uint32_t REG_A5XX_RB_MRT_BLEND_CONTROL(uint32_t i0) { return 0x0000e151 + 0x7*i0; }
+#define A5XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK		0x0000001f
+#define A5XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT		0
+static inline uint32_t A5XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR(enum adreno_rb_blend_factor val)
+{
+	return ((val) << A5XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT) & A5XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK;
+}
+#define A5XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK	0x000000e0
+#define A5XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT	5
+static inline uint32_t A5XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
+{
+	return ((val) << A5XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT) & A5XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK;
+}
+#define A5XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK		0x00001f00
+#define A5XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT	8
+static inline uint32_t A5XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR(enum adreno_rb_blend_factor val)
+{
+	return ((val) << A5XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT) & A5XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK;
+}
+#define A5XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK	0x001f0000
+#define A5XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT	16
+static inline uint32_t A5XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR(enum adreno_rb_blend_factor val)
+{
+	return ((val) << A5XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT) & A5XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK;
+}
+#define A5XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK	0x00e00000
+#define A5XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT	21
+static inline uint32_t A5XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE(enum a3xx_rb_blend_opcode val)
+{
+	return ((val) << A5XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT) & A5XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK;
+}
+#define A5XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK	0x1f000000
+#define A5XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT	24
+static inline uint32_t A5XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR(enum adreno_rb_blend_factor val)
+{
+	return ((val) << A5XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT) & A5XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK;
+}
+
+static inline uint32_t REG_A5XX_RB_MRT_BUF_INFO(uint32_t i0) { return 0x0000e152 + 0x7*i0; }
+#define A5XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK			0x000000ff
+#define A5XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT		0
+static inline uint32_t A5XX_RB_MRT_BUF_INFO_COLOR_FORMAT(enum a5xx_color_fmt val)
+{
+	return ((val) << A5XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT) & A5XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK;
+}
+#define A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK		0x00000300
+#define A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT		8
+static inline uint32_t A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE(enum a5xx_tile_mode val)
+{
+	return ((val) << A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT) & A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK;
+}
+#define A5XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK			0x00006000
+#define A5XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT			13
+static inline uint32_t A5XX_RB_MRT_BUF_INFO_COLOR_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A5XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT) & A5XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK;
+}
+#define A5XX_RB_MRT_BUF_INFO_COLOR_SRGB				0x00008000
+
+static inline uint32_t REG_A5XX_RB_MRT_PITCH(uint32_t i0) { return 0x0000e153 + 0x7*i0; }
+#define A5XX_RB_MRT_PITCH__MASK					0xffffffff
+#define A5XX_RB_MRT_PITCH__SHIFT				0
+static inline uint32_t A5XX_RB_MRT_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A5XX_RB_MRT_PITCH__SHIFT) & A5XX_RB_MRT_PITCH__MASK;
+}
+
+static inline uint32_t REG_A5XX_RB_MRT_ARRAY_PITCH(uint32_t i0) { return 0x0000e154 + 0x7*i0; }
+#define A5XX_RB_MRT_ARRAY_PITCH__MASK				0xffffffff
+#define A5XX_RB_MRT_ARRAY_PITCH__SHIFT				0
+static inline uint32_t A5XX_RB_MRT_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A5XX_RB_MRT_ARRAY_PITCH__SHIFT) & A5XX_RB_MRT_ARRAY_PITCH__MASK;
+}
+
+static inline uint32_t REG_A5XX_RB_MRT_BASE_LO(uint32_t i0) { return 0x0000e155 + 0x7*i0; }
+
+static inline uint32_t REG_A5XX_RB_MRT_BASE_HI(uint32_t i0) { return 0x0000e156 + 0x7*i0; }
+
+#define REG_A5XX_RB_BLEND_RED					0x0000e1a0
+#define A5XX_RB_BLEND_RED_UINT__MASK				0x000000ff
+#define A5XX_RB_BLEND_RED_UINT__SHIFT				0
+static inline uint32_t A5XX_RB_BLEND_RED_UINT(uint32_t val)
+{
+	return ((val) << A5XX_RB_BLEND_RED_UINT__SHIFT) & A5XX_RB_BLEND_RED_UINT__MASK;
+}
+#define A5XX_RB_BLEND_RED_SINT__MASK				0x0000ff00
+#define A5XX_RB_BLEND_RED_SINT__SHIFT				8
+static inline uint32_t A5XX_RB_BLEND_RED_SINT(uint32_t val)
+{
+	return ((val) << A5XX_RB_BLEND_RED_SINT__SHIFT) & A5XX_RB_BLEND_RED_SINT__MASK;
+}
+#define A5XX_RB_BLEND_RED_FLOAT__MASK				0xffff0000
+#define A5XX_RB_BLEND_RED_FLOAT__SHIFT				16
+static inline uint32_t A5XX_RB_BLEND_RED_FLOAT(float val)
+{
+	return ((util_float_to_half(val)) << A5XX_RB_BLEND_RED_FLOAT__SHIFT) & A5XX_RB_BLEND_RED_FLOAT__MASK;
+}
+
+#define REG_A5XX_RB_BLEND_RED_F32				0x0000e1a1
+#define A5XX_RB_BLEND_RED_F32__MASK				0xffffffff
+#define A5XX_RB_BLEND_RED_F32__SHIFT				0
+static inline uint32_t A5XX_RB_BLEND_RED_F32(float val)
+{
+	return ((fui(val)) << A5XX_RB_BLEND_RED_F32__SHIFT) & A5XX_RB_BLEND_RED_F32__MASK;
+}
+
+#define REG_A5XX_RB_BLEND_GREEN					0x0000e1a2
+#define A5XX_RB_BLEND_GREEN_UINT__MASK				0x000000ff
+#define A5XX_RB_BLEND_GREEN_UINT__SHIFT				0
+static inline uint32_t A5XX_RB_BLEND_GREEN_UINT(uint32_t val)
+{
+	return ((val) << A5XX_RB_BLEND_GREEN_UINT__SHIFT) & A5XX_RB_BLEND_GREEN_UINT__MASK;
+}
+#define A5XX_RB_BLEND_GREEN_SINT__MASK				0x0000ff00
+#define A5XX_RB_BLEND_GREEN_SINT__SHIFT				8
+static inline uint32_t A5XX_RB_BLEND_GREEN_SINT(uint32_t val)
+{
+	return ((val) << A5XX_RB_BLEND_GREEN_SINT__SHIFT) & A5XX_RB_BLEND_GREEN_SINT__MASK;
+}
+#define A5XX_RB_BLEND_GREEN_FLOAT__MASK				0xffff0000
+#define A5XX_RB_BLEND_GREEN_FLOAT__SHIFT			16
+static inline uint32_t A5XX_RB_BLEND_GREEN_FLOAT(float val)
+{
+	return ((util_float_to_half(val)) << A5XX_RB_BLEND_GREEN_FLOAT__SHIFT) & A5XX_RB_BLEND_GREEN_FLOAT__MASK;
+}
+
+#define REG_A5XX_RB_BLEND_GREEN_F32				0x0000e1a3
+#define A5XX_RB_BLEND_GREEN_F32__MASK				0xffffffff
+#define A5XX_RB_BLEND_GREEN_F32__SHIFT				0
+static inline uint32_t A5XX_RB_BLEND_GREEN_F32(float val)
+{
+	return ((fui(val)) << A5XX_RB_BLEND_GREEN_F32__SHIFT) & A5XX_RB_BLEND_GREEN_F32__MASK;
+}
+
+#define REG_A5XX_RB_BLEND_BLUE					0x0000e1a4
+#define A5XX_RB_BLEND_BLUE_UINT__MASK				0x000000ff
+#define A5XX_RB_BLEND_BLUE_UINT__SHIFT				0
+static inline uint32_t A5XX_RB_BLEND_BLUE_UINT(uint32_t val)
+{
+	return ((val) << A5XX_RB_BLEND_BLUE_UINT__SHIFT) & A5XX_RB_BLEND_BLUE_UINT__MASK;
+}
+#define A5XX_RB_BLEND_BLUE_SINT__MASK				0x0000ff00
+#define A5XX_RB_BLEND_BLUE_SINT__SHIFT				8
+static inline uint32_t A5XX_RB_BLEND_BLUE_SINT(uint32_t val)
+{
+	return ((val) << A5XX_RB_BLEND_BLUE_SINT__SHIFT) & A5XX_RB_BLEND_BLUE_SINT__MASK;
+}
+#define A5XX_RB_BLEND_BLUE_FLOAT__MASK				0xffff0000
+#define A5XX_RB_BLEND_BLUE_FLOAT__SHIFT				16
+static inline uint32_t A5XX_RB_BLEND_BLUE_FLOAT(float val)
+{
+	return ((util_float_to_half(val)) << A5XX_RB_BLEND_BLUE_FLOAT__SHIFT) & A5XX_RB_BLEND_BLUE_FLOAT__MASK;
+}
+
+#define REG_A5XX_RB_BLEND_BLUE_F32				0x0000e1a5
+#define A5XX_RB_BLEND_BLUE_F32__MASK				0xffffffff
+#define A5XX_RB_BLEND_BLUE_F32__SHIFT				0
+static inline uint32_t A5XX_RB_BLEND_BLUE_F32(float val)
+{
+	return ((fui(val)) << A5XX_RB_BLEND_BLUE_F32__SHIFT) & A5XX_RB_BLEND_BLUE_F32__MASK;
+}
+
+#define REG_A5XX_RB_BLEND_ALPHA					0x0000e1a6
+#define A5XX_RB_BLEND_ALPHA_UINT__MASK				0x000000ff
+#define A5XX_RB_BLEND_ALPHA_UINT__SHIFT				0
+static inline uint32_t A5XX_RB_BLEND_ALPHA_UINT(uint32_t val)
+{
+	return ((val) << A5XX_RB_BLEND_ALPHA_UINT__SHIFT) & A5XX_RB_BLEND_ALPHA_UINT__MASK;
+}
+#define A5XX_RB_BLEND_ALPHA_SINT__MASK				0x0000ff00
+#define A5XX_RB_BLEND_ALPHA_SINT__SHIFT				8
+static inline uint32_t A5XX_RB_BLEND_ALPHA_SINT(uint32_t val)
+{
+	return ((val) << A5XX_RB_BLEND_ALPHA_SINT__SHIFT) & A5XX_RB_BLEND_ALPHA_SINT__MASK;
+}
+#define A5XX_RB_BLEND_ALPHA_FLOAT__MASK				0xffff0000
+#define A5XX_RB_BLEND_ALPHA_FLOAT__SHIFT			16
+static inline uint32_t A5XX_RB_BLEND_ALPHA_FLOAT(float val)
+{
+	return ((util_float_to_half(val)) << A5XX_RB_BLEND_ALPHA_FLOAT__SHIFT) & A5XX_RB_BLEND_ALPHA_FLOAT__MASK;
+}
+
+#define REG_A5XX_RB_BLEND_ALPHA_F32				0x0000e1a7
+#define A5XX_RB_BLEND_ALPHA_F32__MASK				0xffffffff
+#define A5XX_RB_BLEND_ALPHA_F32__SHIFT				0
+static inline uint32_t A5XX_RB_BLEND_ALPHA_F32(float val)
+{
+	return ((fui(val)) << A5XX_RB_BLEND_ALPHA_F32__SHIFT) & A5XX_RB_BLEND_ALPHA_F32__MASK;
+}
+
+#define REG_A5XX_RB_ALPHA_CONTROL				0x0000e1a8
+#define A5XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK			0x000000ff
+#define A5XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT			0
+static inline uint32_t A5XX_RB_ALPHA_CONTROL_ALPHA_REF(uint32_t val)
+{
+	return ((val) << A5XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT) & A5XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK;
+}
+#define A5XX_RB_ALPHA_CONTROL_ALPHA_TEST			0x00000100
+#define A5XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__MASK		0x00000e00
+#define A5XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__SHIFT		9
+static inline uint32_t A5XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC(enum adreno_compare_func val)
+{
+	return ((val) << A5XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__SHIFT) & A5XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__MASK;
+}
+
+#define REG_A5XX_RB_BLEND_CNTL					0x0000e1a9
+#define A5XX_RB_BLEND_CNTL_ENABLE_BLEND__MASK			0x000000ff
+#define A5XX_RB_BLEND_CNTL_ENABLE_BLEND__SHIFT			0
+static inline uint32_t A5XX_RB_BLEND_CNTL_ENABLE_BLEND(uint32_t val)
+{
+	return ((val) << A5XX_RB_BLEND_CNTL_ENABLE_BLEND__SHIFT) & A5XX_RB_BLEND_CNTL_ENABLE_BLEND__MASK;
+}
+#define A5XX_RB_BLEND_CNTL_INDEPENDENT_BLEND			0x00000100
+#define A5XX_RB_BLEND_CNTL_SAMPLE_MASK__MASK			0xffff0000
+#define A5XX_RB_BLEND_CNTL_SAMPLE_MASK__SHIFT			16
+static inline uint32_t A5XX_RB_BLEND_CNTL_SAMPLE_MASK(uint32_t val)
+{
+	return ((val) << A5XX_RB_BLEND_CNTL_SAMPLE_MASK__SHIFT) & A5XX_RB_BLEND_CNTL_SAMPLE_MASK__MASK;
+}
+
+#define REG_A5XX_RB_DEPTH_PLANE_CNTL				0x0000e1b0
+#define A5XX_RB_DEPTH_PLANE_CNTL_FRAG_WRITES_Z			0x00000001
+
+#define REG_A5XX_RB_DEPTH_CNTL					0x0000e1b1
+#define A5XX_RB_DEPTH_CNTL_Z_ENABLE				0x00000001
+#define A5XX_RB_DEPTH_CNTL_Z_WRITE_ENABLE			0x00000002
+#define A5XX_RB_DEPTH_CNTL_ZFUNC__MASK				0x0000001c
+#define A5XX_RB_DEPTH_CNTL_ZFUNC__SHIFT				2
+static inline uint32_t A5XX_RB_DEPTH_CNTL_ZFUNC(enum adreno_compare_func val)
+{
+	return ((val) << A5XX_RB_DEPTH_CNTL_ZFUNC__SHIFT) & A5XX_RB_DEPTH_CNTL_ZFUNC__MASK;
+}
+#define A5XX_RB_DEPTH_CNTL_Z_TEST_ENABLE			0x00000040
+
+#define REG_A5XX_RB_DEPTH_BUFFER_INFO				0x0000e1b2
+#define A5XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK		0x00000007
+#define A5XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT		0
+static inline uint32_t A5XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT(enum a5xx_depth_format val)
+{
+	return ((val) << A5XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT) & A5XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK;
+}
+
+#define REG_A5XX_RB_DEPTH_BUFFER_BASE_LO			0x0000e1b3
+
+#define REG_A5XX_RB_DEPTH_BUFFER_BASE_HI			0x0000e1b4
+
+#define REG_A5XX_RB_DEPTH_BUFFER_PITCH				0x0000e1b5
+#define A5XX_RB_DEPTH_BUFFER_PITCH__MASK			0xffffffff
+#define A5XX_RB_DEPTH_BUFFER_PITCH__SHIFT			0
+static inline uint32_t A5XX_RB_DEPTH_BUFFER_PITCH(uint32_t val)
+{
+	return ((val >> 5) << A5XX_RB_DEPTH_BUFFER_PITCH__SHIFT) & A5XX_RB_DEPTH_BUFFER_PITCH__MASK;
+}
+
+#define REG_A5XX_RB_DEPTH_BUFFER_ARRAY_PITCH			0x0000e1b6
+#define A5XX_RB_DEPTH_BUFFER_ARRAY_PITCH__MASK			0xffffffff
+#define A5XX_RB_DEPTH_BUFFER_ARRAY_PITCH__SHIFT			0
+static inline uint32_t A5XX_RB_DEPTH_BUFFER_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 5) << A5XX_RB_DEPTH_BUFFER_ARRAY_PITCH__SHIFT) & A5XX_RB_DEPTH_BUFFER_ARRAY_PITCH__MASK;
+}
+
+#define REG_A5XX_RB_STENCIL_CONTROL				0x0000e1c0
+#define A5XX_RB_STENCIL_CONTROL_STENCIL_ENABLE			0x00000001
+#define A5XX_RB_STENCIL_CONTROL_STENCIL_ENABLE_BF		0x00000002
+#define A5XX_RB_STENCIL_CONTROL_STENCIL_READ			0x00000004
+#define A5XX_RB_STENCIL_CONTROL_FUNC__MASK			0x00000700
+#define A5XX_RB_STENCIL_CONTROL_FUNC__SHIFT			8
+static inline uint32_t A5XX_RB_STENCIL_CONTROL_FUNC(enum adreno_compare_func val)
+{
+	return ((val) << A5XX_RB_STENCIL_CONTROL_FUNC__SHIFT) & A5XX_RB_STENCIL_CONTROL_FUNC__MASK;
+}
+#define A5XX_RB_STENCIL_CONTROL_FAIL__MASK			0x00003800
+#define A5XX_RB_STENCIL_CONTROL_FAIL__SHIFT			11
+static inline uint32_t A5XX_RB_STENCIL_CONTROL_FAIL(enum adreno_stencil_op val)
+{
+	return ((val) << A5XX_RB_STENCIL_CONTROL_FAIL__SHIFT) & A5XX_RB_STENCIL_CONTROL_FAIL__MASK;
+}
+#define A5XX_RB_STENCIL_CONTROL_ZPASS__MASK			0x0001c000
+#define A5XX_RB_STENCIL_CONTROL_ZPASS__SHIFT			14
+static inline uint32_t A5XX_RB_STENCIL_CONTROL_ZPASS(enum adreno_stencil_op val)
+{
+	return ((val) << A5XX_RB_STENCIL_CONTROL_ZPASS__SHIFT) & A5XX_RB_STENCIL_CONTROL_ZPASS__MASK;
+}
+#define A5XX_RB_STENCIL_CONTROL_ZFAIL__MASK			0x000e0000
+#define A5XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT			17
+static inline uint32_t A5XX_RB_STENCIL_CONTROL_ZFAIL(enum adreno_stencil_op val)
+{
+	return ((val) << A5XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT) & A5XX_RB_STENCIL_CONTROL_ZFAIL__MASK;
+}
+#define A5XX_RB_STENCIL_CONTROL_FUNC_BF__MASK			0x00700000
+#define A5XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT			20
+static inline uint32_t A5XX_RB_STENCIL_CONTROL_FUNC_BF(enum adreno_compare_func val)
+{
+	return ((val) << A5XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT) & A5XX_RB_STENCIL_CONTROL_FUNC_BF__MASK;
+}
+#define A5XX_RB_STENCIL_CONTROL_FAIL_BF__MASK			0x03800000
+#define A5XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT			23
+static inline uint32_t A5XX_RB_STENCIL_CONTROL_FAIL_BF(enum adreno_stencil_op val)
+{
+	return ((val) << A5XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT) & A5XX_RB_STENCIL_CONTROL_FAIL_BF__MASK;
+}
+#define A5XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK			0x1c000000
+#define A5XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT			26
+static inline uint32_t A5XX_RB_STENCIL_CONTROL_ZPASS_BF(enum adreno_stencil_op val)
+{
+	return ((val) << A5XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT) & A5XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK;
+}
+#define A5XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK			0xe0000000
+#define A5XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT			29
+static inline uint32_t A5XX_RB_STENCIL_CONTROL_ZFAIL_BF(enum adreno_stencil_op val)
+{
+	return ((val) << A5XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT) & A5XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK;
+}
+
+#define REG_A5XX_RB_STENCIL_INFO				0x0000e1c1
+#define A5XX_RB_STENCIL_INFO_SEPARATE_STENCIL			0x00000001
+
+#define REG_A5XX_RB_STENCIL_BASE_LO				0x0000e1c2
+
+#define REG_A5XX_RB_STENCIL_BASE_HI				0x0000e1c3
+
+#define REG_A5XX_RB_STENCIL_PITCH				0x0000e1c4
+#define A5XX_RB_STENCIL_PITCH__MASK				0xffffffff
+#define A5XX_RB_STENCIL_PITCH__SHIFT				0
+static inline uint32_t A5XX_RB_STENCIL_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A5XX_RB_STENCIL_PITCH__SHIFT) & A5XX_RB_STENCIL_PITCH__MASK;
+}
+
+#define REG_A5XX_RB_STENCIL_ARRAY_PITCH				0x0000e1c5
+#define A5XX_RB_STENCIL_ARRAY_PITCH__MASK			0xffffffff
+#define A5XX_RB_STENCIL_ARRAY_PITCH__SHIFT			0
+static inline uint32_t A5XX_RB_STENCIL_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A5XX_RB_STENCIL_ARRAY_PITCH__SHIFT) & A5XX_RB_STENCIL_ARRAY_PITCH__MASK;
+}
+
+#define REG_A5XX_RB_STENCILREFMASK				0x0000e1c6
+#define A5XX_RB_STENCILREFMASK_STENCILREF__MASK			0x000000ff
+#define A5XX_RB_STENCILREFMASK_STENCILREF__SHIFT		0
+static inline uint32_t A5XX_RB_STENCILREFMASK_STENCILREF(uint32_t val)
+{
+	return ((val) << A5XX_RB_STENCILREFMASK_STENCILREF__SHIFT) & A5XX_RB_STENCILREFMASK_STENCILREF__MASK;
+}
+#define A5XX_RB_STENCILREFMASK_STENCILMASK__MASK		0x0000ff00
+#define A5XX_RB_STENCILREFMASK_STENCILMASK__SHIFT		8
+static inline uint32_t A5XX_RB_STENCILREFMASK_STENCILMASK(uint32_t val)
+{
+	return ((val) << A5XX_RB_STENCILREFMASK_STENCILMASK__SHIFT) & A5XX_RB_STENCILREFMASK_STENCILMASK__MASK;
+}
+#define A5XX_RB_STENCILREFMASK_STENCILWRITEMASK__MASK		0x00ff0000
+#define A5XX_RB_STENCILREFMASK_STENCILWRITEMASK__SHIFT		16
+static inline uint32_t A5XX_RB_STENCILREFMASK_STENCILWRITEMASK(uint32_t val)
+{
+	return ((val) << A5XX_RB_STENCILREFMASK_STENCILWRITEMASK__SHIFT) & A5XX_RB_STENCILREFMASK_STENCILWRITEMASK__MASK;
+}
+
+#define REG_A5XX_UNKNOWN_E1C7					0x0000e1c7
+
+#define REG_A5XX_RB_WINDOW_OFFSET				0x0000e1d0
+#define A5XX_RB_WINDOW_OFFSET_WINDOW_OFFSET_DISABLE		0x80000000
+#define A5XX_RB_WINDOW_OFFSET_X__MASK				0x00007fff
+#define A5XX_RB_WINDOW_OFFSET_X__SHIFT				0
+static inline uint32_t A5XX_RB_WINDOW_OFFSET_X(uint32_t val)
+{
+	return ((val) << A5XX_RB_WINDOW_OFFSET_X__SHIFT) & A5XX_RB_WINDOW_OFFSET_X__MASK;
+}
+#define A5XX_RB_WINDOW_OFFSET_Y__MASK				0x7fff0000
+#define A5XX_RB_WINDOW_OFFSET_Y__SHIFT				16
+static inline uint32_t A5XX_RB_WINDOW_OFFSET_Y(uint32_t val)
+{
+	return ((val) << A5XX_RB_WINDOW_OFFSET_Y__SHIFT) & A5XX_RB_WINDOW_OFFSET_Y__MASK;
+}
+
+#define REG_A5XX_RB_BLIT_CNTL					0x0000e210
+#define A5XX_RB_BLIT_CNTL_BUF__MASK				0x0000003f
+#define A5XX_RB_BLIT_CNTL_BUF__SHIFT				0
+static inline uint32_t A5XX_RB_BLIT_CNTL_BUF(enum a5xx_blit_buf val)
+{
+	return ((val) << A5XX_RB_BLIT_CNTL_BUF__SHIFT) & A5XX_RB_BLIT_CNTL_BUF__MASK;
+}
+
+#define REG_A5XX_RB_RESOLVE_CNTL_1				0x0000e211
+#define A5XX_RB_RESOLVE_CNTL_1_WINDOW_OFFSET_DISABLE		0x80000000
+#define A5XX_RB_RESOLVE_CNTL_1_X__MASK				0x00007fff
+#define A5XX_RB_RESOLVE_CNTL_1_X__SHIFT				0
+static inline uint32_t A5XX_RB_RESOLVE_CNTL_1_X(uint32_t val)
+{
+	return ((val) << A5XX_RB_RESOLVE_CNTL_1_X__SHIFT) & A5XX_RB_RESOLVE_CNTL_1_X__MASK;
+}
+#define A5XX_RB_RESOLVE_CNTL_1_Y__MASK				0x7fff0000
+#define A5XX_RB_RESOLVE_CNTL_1_Y__SHIFT				16
+static inline uint32_t A5XX_RB_RESOLVE_CNTL_1_Y(uint32_t val)
+{
+	return ((val) << A5XX_RB_RESOLVE_CNTL_1_Y__SHIFT) & A5XX_RB_RESOLVE_CNTL_1_Y__MASK;
+}
+
+#define REG_A5XX_RB_RESOLVE_CNTL_2				0x0000e212
+#define A5XX_RB_RESOLVE_CNTL_2_WINDOW_OFFSET_DISABLE		0x80000000
+#define A5XX_RB_RESOLVE_CNTL_2_X__MASK				0x00007fff
+#define A5XX_RB_RESOLVE_CNTL_2_X__SHIFT				0
+static inline uint32_t A5XX_RB_RESOLVE_CNTL_2_X(uint32_t val)
+{
+	return ((val) << A5XX_RB_RESOLVE_CNTL_2_X__SHIFT) & A5XX_RB_RESOLVE_CNTL_2_X__MASK;
+}
+#define A5XX_RB_RESOLVE_CNTL_2_Y__MASK				0x7fff0000
+#define A5XX_RB_RESOLVE_CNTL_2_Y__SHIFT				16
+static inline uint32_t A5XX_RB_RESOLVE_CNTL_2_Y(uint32_t val)
+{
+	return ((val) << A5XX_RB_RESOLVE_CNTL_2_Y__SHIFT) & A5XX_RB_RESOLVE_CNTL_2_Y__MASK;
+}
+
+#define REG_A5XX_RB_RESOLVE_CNTL_3				0x0000e213
+
+#define REG_A5XX_RB_BLIT_DST_LO					0x0000e214
+
+#define REG_A5XX_RB_BLIT_DST_HI					0x0000e215
+
+#define REG_A5XX_RB_BLIT_DST_PITCH				0x0000e216
+#define A5XX_RB_BLIT_DST_PITCH__MASK				0xffffffff
+#define A5XX_RB_BLIT_DST_PITCH__SHIFT				0
+static inline uint32_t A5XX_RB_BLIT_DST_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A5XX_RB_BLIT_DST_PITCH__SHIFT) & A5XX_RB_BLIT_DST_PITCH__MASK;
+}
+
+#define REG_A5XX_RB_BLIT_DST_ARRAY_PITCH			0x0000e217
+#define A5XX_RB_BLIT_DST_ARRAY_PITCH__MASK			0xffffffff
+#define A5XX_RB_BLIT_DST_ARRAY_PITCH__SHIFT			0
+static inline uint32_t A5XX_RB_BLIT_DST_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A5XX_RB_BLIT_DST_ARRAY_PITCH__SHIFT) & A5XX_RB_BLIT_DST_ARRAY_PITCH__MASK;
+}
+
+#define REG_A5XX_RB_CLEAR_COLOR_DW0				0x0000e218
+
+#define REG_A5XX_RB_CLEAR_COLOR_DW1				0x0000e219
+
+#define REG_A5XX_RB_CLEAR_COLOR_DW2				0x0000e21a
+
+#define REG_A5XX_RB_CLEAR_COLOR_DW3				0x0000e21b
+
+#define REG_A5XX_RB_CLEAR_CNTL					0x0000e21c
+#define A5XX_RB_CLEAR_CNTL_FAST_CLEAR				0x00000002
+#define A5XX_RB_CLEAR_CNTL_MASK__MASK				0x000000f0
+#define A5XX_RB_CLEAR_CNTL_MASK__SHIFT				4
+static inline uint32_t A5XX_RB_CLEAR_CNTL_MASK(uint32_t val)
+{
+	return ((val) << A5XX_RB_CLEAR_CNTL_MASK__SHIFT) & A5XX_RB_CLEAR_CNTL_MASK__MASK;
+}
+
+#define REG_A5XX_RB_DEPTH_FLAG_BUFFER_BASE_LO			0x0000e240
+
+#define REG_A5XX_RB_DEPTH_FLAG_BUFFER_BASE_HI			0x0000e241
+
+#define REG_A5XX_RB_DEPTH_FLAG_BUFFER_PITCH			0x0000e242
+
+static inline uint32_t REG_A5XX_RB_MRT_FLAG_BUFFER(uint32_t i0) { return 0x0000e243 + 0x4*i0; }
+
+static inline uint32_t REG_A5XX_RB_MRT_FLAG_BUFFER_ADDR_LO(uint32_t i0) { return 0x0000e243 + 0x4*i0; }
+
+static inline uint32_t REG_A5XX_RB_MRT_FLAG_BUFFER_ADDR_HI(uint32_t i0) { return 0x0000e244 + 0x4*i0; }
+
+static inline uint32_t REG_A5XX_RB_MRT_FLAG_BUFFER_PITCH(uint32_t i0) { return 0x0000e245 + 0x4*i0; }
+#define A5XX_RB_MRT_FLAG_BUFFER_PITCH__MASK			0xffffffff
+#define A5XX_RB_MRT_FLAG_BUFFER_PITCH__SHIFT			0
+static inline uint32_t A5XX_RB_MRT_FLAG_BUFFER_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A5XX_RB_MRT_FLAG_BUFFER_PITCH__SHIFT) & A5XX_RB_MRT_FLAG_BUFFER_PITCH__MASK;
+}
+
+static inline uint32_t REG_A5XX_RB_MRT_FLAG_BUFFER_ARRAY_PITCH(uint32_t i0) { return 0x0000e246 + 0x4*i0; }
+#define A5XX_RB_MRT_FLAG_BUFFER_ARRAY_PITCH__MASK		0xffffffff
+#define A5XX_RB_MRT_FLAG_BUFFER_ARRAY_PITCH__SHIFT		0
+static inline uint32_t A5XX_RB_MRT_FLAG_BUFFER_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A5XX_RB_MRT_FLAG_BUFFER_ARRAY_PITCH__SHIFT) & A5XX_RB_MRT_FLAG_BUFFER_ARRAY_PITCH__MASK;
+}
+
+#define REG_A5XX_RB_BLIT_FLAG_DST_LO				0x0000e263
+
+#define REG_A5XX_RB_BLIT_FLAG_DST_HI				0x0000e264
+
+#define REG_A5XX_RB_BLIT_FLAG_DST_PITCH				0x0000e265
+#define A5XX_RB_BLIT_FLAG_DST_PITCH__MASK			0xffffffff
+#define A5XX_RB_BLIT_FLAG_DST_PITCH__SHIFT			0
+static inline uint32_t A5XX_RB_BLIT_FLAG_DST_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A5XX_RB_BLIT_FLAG_DST_PITCH__SHIFT) & A5XX_RB_BLIT_FLAG_DST_PITCH__MASK;
+}
+
+#define REG_A5XX_RB_BLIT_FLAG_DST_ARRAY_PITCH			0x0000e266
+#define A5XX_RB_BLIT_FLAG_DST_ARRAY_PITCH__MASK			0xffffffff
+#define A5XX_RB_BLIT_FLAG_DST_ARRAY_PITCH__SHIFT		0
+static inline uint32_t A5XX_RB_BLIT_FLAG_DST_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 6) << A5XX_RB_BLIT_FLAG_DST_ARRAY_PITCH__SHIFT) & A5XX_RB_BLIT_FLAG_DST_ARRAY_PITCH__MASK;
+}
+
+#define REG_A5XX_VPC_CNTL_0					0x0000e280
+#define A5XX_VPC_CNTL_0_STRIDE_IN_VPC__MASK			0x0000007f
+#define A5XX_VPC_CNTL_0_STRIDE_IN_VPC__SHIFT			0
+static inline uint32_t A5XX_VPC_CNTL_0_STRIDE_IN_VPC(uint32_t val)
+{
+	return ((val) << A5XX_VPC_CNTL_0_STRIDE_IN_VPC__SHIFT) & A5XX_VPC_CNTL_0_STRIDE_IN_VPC__MASK;
+}
+#define A5XX_VPC_CNTL_0_VARYING					0x00000800
+
+static inline uint32_t REG_A5XX_VPC_VARYING_INTERP(uint32_t i0) { return 0x0000e282 + 0x1*i0; }
+
+static inline uint32_t REG_A5XX_VPC_VARYING_INTERP_MODE(uint32_t i0) { return 0x0000e282 + 0x1*i0; }
+
+static inline uint32_t REG_A5XX_VPC_VARYING_PS_REPL(uint32_t i0) { return 0x0000e28a + 0x1*i0; }
+
+static inline uint32_t REG_A5XX_VPC_VARYING_PS_REPL_MODE(uint32_t i0) { return 0x0000e28a + 0x1*i0; }
+
+#define REG_A5XX_UNKNOWN_E292					0x0000e292
+
+#define REG_A5XX_UNKNOWN_E293					0x0000e293
+
+static inline uint32_t REG_A5XX_VPC_VAR(uint32_t i0) { return 0x0000e294 + 0x1*i0; }
+
+static inline uint32_t REG_A5XX_VPC_VAR_DISABLE(uint32_t i0) { return 0x0000e294 + 0x1*i0; }
+
+#define REG_A5XX_VPC_GS_SIV_CNTL				0x0000e298
+
+#define REG_A5XX_UNKNOWN_E29A					0x0000e29a
+
+#define REG_A5XX_VPC_PACK					0x0000e29d
+#define A5XX_VPC_PACK_NUMNONPOSVAR__MASK			0x000000ff
+#define A5XX_VPC_PACK_NUMNONPOSVAR__SHIFT			0
+static inline uint32_t A5XX_VPC_PACK_NUMNONPOSVAR(uint32_t val)
+{
+	return ((val) << A5XX_VPC_PACK_NUMNONPOSVAR__SHIFT) & A5XX_VPC_PACK_NUMNONPOSVAR__MASK;
+}
+
+#define REG_A5XX_VPC_FS_PRIMITIVEID_CNTL			0x0000e2a0
+
+#define REG_A5XX_UNKNOWN_E2A1					0x0000e2a1
+
+#define REG_A5XX_VPC_SO_OVERRIDE				0x0000e2a2
+
+#define REG_A5XX_VPC_SO_BUFFER_BASE_LO_0			0x0000e2a7
+
+#define REG_A5XX_VPC_SO_BUFFER_BASE_HI_0			0x0000e2a8
+
+#define REG_A5XX_VPC_SO_BUFFER_SIZE_0				0x0000e2a9
+
+#define REG_A5XX_UNKNOWN_E2AB					0x0000e2ab
+
+#define REG_A5XX_VPC_SO_FLUSH_BASE_LO_0				0x0000e2ac
+
+#define REG_A5XX_VPC_SO_FLUSH_BASE_HI_0				0x0000e2ad
+
+#define REG_A5XX_UNKNOWN_E2AE					0x0000e2ae
+
+#define REG_A5XX_UNKNOWN_E2B2					0x0000e2b2
+
+#define REG_A5XX_UNKNOWN_E2B9					0x0000e2b9
+
+#define REG_A5XX_UNKNOWN_E2C0					0x0000e2c0
+
+#define REG_A5XX_PC_PRIMITIVE_CNTL				0x0000e384
+#define A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC__MASK		0x0000007f
+#define A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC__SHIFT		0
+static inline uint32_t A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC(uint32_t val)
+{
+	return ((val) << A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC__SHIFT) & A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC__MASK;
+}
+
+#define REG_A5XX_PC_PRIM_VTX_CNTL				0x0000e385
+#define A5XX_PC_PRIM_VTX_CNTL_PSIZE				0x00000800
+
+#define REG_A5XX_PC_RASTER_CNTL					0x0000e388
+
+#define REG_A5XX_UNKNOWN_E389					0x0000e389
+
+#define REG_A5XX_PC_RESTART_INDEX				0x0000e38c
+
+#define REG_A5XX_UNKNOWN_E38D					0x0000e38d
+
+#define REG_A5XX_PC_GS_PARAM					0x0000e38e
+
+#define REG_A5XX_PC_HS_PARAM					0x0000e38f
+
+#define REG_A5XX_PC_POWER_CNTL					0x0000e3b0
+
+#define REG_A5XX_VFD_CONTROL_0					0x0000e400
+#define A5XX_VFD_CONTROL_0_VTXCNT__MASK				0x0000003f
+#define A5XX_VFD_CONTROL_0_VTXCNT__SHIFT			0
+static inline uint32_t A5XX_VFD_CONTROL_0_VTXCNT(uint32_t val)
+{
+	return ((val) << A5XX_VFD_CONTROL_0_VTXCNT__SHIFT) & A5XX_VFD_CONTROL_0_VTXCNT__MASK;
+}
+
+#define REG_A5XX_VFD_CONTROL_1					0x0000e401
+#define A5XX_VFD_CONTROL_1_REGID4INST__MASK			0x0000ff00
+#define A5XX_VFD_CONTROL_1_REGID4INST__SHIFT			8
+static inline uint32_t A5XX_VFD_CONTROL_1_REGID4INST(uint32_t val)
+{
+	return ((val) << A5XX_VFD_CONTROL_1_REGID4INST__SHIFT) & A5XX_VFD_CONTROL_1_REGID4INST__MASK;
+}
+#define A5XX_VFD_CONTROL_1_REGID4VTX__MASK			0x00ff0000
+#define A5XX_VFD_CONTROL_1_REGID4VTX__SHIFT			16
+static inline uint32_t A5XX_VFD_CONTROL_1_REGID4VTX(uint32_t val)
+{
+	return ((val) << A5XX_VFD_CONTROL_1_REGID4VTX__SHIFT) & A5XX_VFD_CONTROL_1_REGID4VTX__MASK;
+}
+
+#define REG_A5XX_VFD_CONTROL_2					0x0000e402
+
+#define REG_A5XX_VFD_CONTROL_3					0x0000e403
+
+#define REG_A5XX_VFD_CONTROL_4					0x0000e404
+
+#define REG_A5XX_VFD_CONTROL_5					0x0000e405
+
+#define REG_A5XX_VFD_INDEX_OFFSET				0x0000e408
+
+#define REG_A5XX_VFD_INSTANCE_START_OFFSET			0x0000e409
+
+static inline uint32_t REG_A5XX_VFD_FETCH(uint32_t i0) { return 0x0000e40a + 0x4*i0; }
+
+static inline uint32_t REG_A5XX_VFD_FETCH_BASE_LO(uint32_t i0) { return 0x0000e40a + 0x4*i0; }
+
+static inline uint32_t REG_A5XX_VFD_FETCH_BASE_HI(uint32_t i0) { return 0x0000e40b + 0x4*i0; }
+
+static inline uint32_t REG_A5XX_VFD_FETCH_SIZE(uint32_t i0) { return 0x0000e40c + 0x4*i0; }
+
+static inline uint32_t REG_A5XX_VFD_FETCH_STRIDE(uint32_t i0) { return 0x0000e40d + 0x4*i0; }
+
+static inline uint32_t REG_A5XX_VFD_DECODE(uint32_t i0) { return 0x0000e48a + 0x2*i0; }
+
+static inline uint32_t REG_A5XX_VFD_DECODE_INSTR(uint32_t i0) { return 0x0000e48a + 0x2*i0; }
+#define A5XX_VFD_DECODE_INSTR_IDX__MASK				0x0000001f
+#define A5XX_VFD_DECODE_INSTR_IDX__SHIFT			0
+static inline uint32_t A5XX_VFD_DECODE_INSTR_IDX(uint32_t val)
+{
+	return ((val) << A5XX_VFD_DECODE_INSTR_IDX__SHIFT) & A5XX_VFD_DECODE_INSTR_IDX__MASK;
+}
+#define A5XX_VFD_DECODE_INSTR_FORMAT__MASK			0x3ff00000
+#define A5XX_VFD_DECODE_INSTR_FORMAT__SHIFT			20
+static inline uint32_t A5XX_VFD_DECODE_INSTR_FORMAT(enum a5xx_vtx_fmt val)
+{
+	return ((val) << A5XX_VFD_DECODE_INSTR_FORMAT__SHIFT) & A5XX_VFD_DECODE_INSTR_FORMAT__MASK;
+}
+#define A5XX_VFD_DECODE_INSTR_SWAP__MASK			0xc0000000
+#define A5XX_VFD_DECODE_INSTR_SWAP__SHIFT			30
+static inline uint32_t A5XX_VFD_DECODE_INSTR_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A5XX_VFD_DECODE_INSTR_SWAP__SHIFT) & A5XX_VFD_DECODE_INSTR_SWAP__MASK;
+}
+
+static inline uint32_t REG_A5XX_VFD_DECODE_STEP_RATE(uint32_t i0) { return 0x0000e48b + 0x2*i0; }
+
+static inline uint32_t REG_A5XX_VFD_DEST_CNTL(uint32_t i0) { return 0x0000e4ca + 0x1*i0; }
+
+static inline uint32_t REG_A5XX_VFD_DEST_CNTL_INSTR(uint32_t i0) { return 0x0000e4ca + 0x1*i0; }
+#define A5XX_VFD_DEST_CNTL_INSTR_WRITEMASK__MASK		0x0000000f
+#define A5XX_VFD_DEST_CNTL_INSTR_WRITEMASK__SHIFT		0
+static inline uint32_t A5XX_VFD_DEST_CNTL_INSTR_WRITEMASK(uint32_t val)
+{
+	return ((val) << A5XX_VFD_DEST_CNTL_INSTR_WRITEMASK__SHIFT) & A5XX_VFD_DEST_CNTL_INSTR_WRITEMASK__MASK;
+}
+#define A5XX_VFD_DEST_CNTL_INSTR_REGID__MASK			0x00000ff0
+#define A5XX_VFD_DEST_CNTL_INSTR_REGID__SHIFT			4
+static inline uint32_t A5XX_VFD_DEST_CNTL_INSTR_REGID(uint32_t val)
+{
+	return ((val) << A5XX_VFD_DEST_CNTL_INSTR_REGID__SHIFT) & A5XX_VFD_DEST_CNTL_INSTR_REGID__MASK;
+}
+
+#define REG_A5XX_VFD_POWER_CNTL					0x0000e4f0
+
+#define REG_A5XX_SP_SP_CNTL					0x0000e580
+
+#define REG_A5XX_SP_VS_CONTROL_REG				0x0000e584
+#define A5XX_SP_VS_CONTROL_REG_ENABLED				0x00000001
+#define A5XX_SP_VS_CONTROL_REG_CONSTOBJECTOFFSET__MASK		0x000000fe
+#define A5XX_SP_VS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT		1
+static inline uint32_t A5XX_SP_VS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_SP_VS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A5XX_SP_VS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
+}
+#define A5XX_SP_VS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00007f00
+#define A5XX_SP_VS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		8
+static inline uint32_t A5XX_SP_VS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_SP_VS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A5XX_SP_VS_CONTROL_REG_SHADEROBJOFFSET__MASK;
+}
+
+#define REG_A5XX_SP_FS_CONTROL_REG				0x0000e585
+#define A5XX_SP_FS_CONTROL_REG_ENABLED				0x00000001
+#define A5XX_SP_FS_CONTROL_REG_CONSTOBJECTOFFSET__MASK		0x000000fe
+#define A5XX_SP_FS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT		1
+static inline uint32_t A5XX_SP_FS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_SP_FS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A5XX_SP_FS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
+}
+#define A5XX_SP_FS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00007f00
+#define A5XX_SP_FS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		8
+static inline uint32_t A5XX_SP_FS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_SP_FS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A5XX_SP_FS_CONTROL_REG_SHADEROBJOFFSET__MASK;
+}
+
+#define REG_A5XX_SP_HS_CONTROL_REG				0x0000e586
+#define A5XX_SP_HS_CONTROL_REG_ENABLED				0x00000001
+#define A5XX_SP_HS_CONTROL_REG_CONSTOBJECTOFFSET__MASK		0x000000fe
+#define A5XX_SP_HS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT		1
+static inline uint32_t A5XX_SP_HS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_SP_HS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A5XX_SP_HS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
+}
+#define A5XX_SP_HS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00007f00
+#define A5XX_SP_HS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		8
+static inline uint32_t A5XX_SP_HS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_SP_HS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A5XX_SP_HS_CONTROL_REG_SHADEROBJOFFSET__MASK;
+}
+
+#define REG_A5XX_SP_DS_CONTROL_REG				0x0000e587
+#define A5XX_SP_DS_CONTROL_REG_ENABLED				0x00000001
+#define A5XX_SP_DS_CONTROL_REG_CONSTOBJECTOFFSET__MASK		0x000000fe
+#define A5XX_SP_DS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT		1
+static inline uint32_t A5XX_SP_DS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_SP_DS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A5XX_SP_DS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
+}
+#define A5XX_SP_DS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00007f00
+#define A5XX_SP_DS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		8
+static inline uint32_t A5XX_SP_DS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_SP_DS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A5XX_SP_DS_CONTROL_REG_SHADEROBJOFFSET__MASK;
+}
+
+#define REG_A5XX_SP_GS_CONTROL_REG				0x0000e588
+#define A5XX_SP_GS_CONTROL_REG_ENABLED				0x00000001
+#define A5XX_SP_GS_CONTROL_REG_CONSTOBJECTOFFSET__MASK		0x000000fe
+#define A5XX_SP_GS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT		1
+static inline uint32_t A5XX_SP_GS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_SP_GS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A5XX_SP_GS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
+}
+#define A5XX_SP_GS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00007f00
+#define A5XX_SP_GS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		8
+static inline uint32_t A5XX_SP_GS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_SP_GS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A5XX_SP_GS_CONTROL_REG_SHADEROBJOFFSET__MASK;
+}
+
+#define REG_A5XX_SP_CS_CONFIG					0x0000e589
+
+#define REG_A5XX_SP_VS_CONFIG_MAX_CONST				0x0000e58a
+
+#define REG_A5XX_SP_FS_CONFIG_MAX_CONST				0x0000e58b
+
+#define REG_A5XX_SP_VS_CTRL_REG0				0x0000e590
+#define A5XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
+#define A5XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
+static inline uint32_t A5XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A5XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
+}
+#define A5XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
+#define A5XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
+static inline uint32_t A5XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A5XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
+}
+#define A5XX_SP_VS_CTRL_REG0_VARYING				0x00010000
+#define A5XX_SP_VS_CTRL_REG0_PIXLODENABLE			0x00100000
+
+#define REG_A5XX_SP_PRIMITIVE_CNTL				0x0000e592
+#define A5XX_SP_PRIMITIVE_CNTL_STRIDE_IN_VPC__MASK		0x0000001f
+#define A5XX_SP_PRIMITIVE_CNTL_STRIDE_IN_VPC__SHIFT		0
+static inline uint32_t A5XX_SP_PRIMITIVE_CNTL_STRIDE_IN_VPC(uint32_t val)
+{
+	return ((val >> 2) << A5XX_SP_PRIMITIVE_CNTL_STRIDE_IN_VPC__SHIFT) & A5XX_SP_PRIMITIVE_CNTL_STRIDE_IN_VPC__MASK;
+}
+
+static inline uint32_t REG_A5XX_SP_VS_OUT(uint32_t i0) { return 0x0000e593 + 0x1*i0; }
+
+static inline uint32_t REG_A5XX_SP_VS_OUT_REG(uint32_t i0) { return 0x0000e593 + 0x1*i0; }
+#define A5XX_SP_VS_OUT_REG_A_REGID__MASK			0x000000ff
+#define A5XX_SP_VS_OUT_REG_A_REGID__SHIFT			0
+static inline uint32_t A5XX_SP_VS_OUT_REG_A_REGID(uint32_t val)
+{
+	return ((val) << A5XX_SP_VS_OUT_REG_A_REGID__SHIFT) & A5XX_SP_VS_OUT_REG_A_REGID__MASK;
+}
+#define A5XX_SP_VS_OUT_REG_A_COMPMASK__MASK			0x00000f00
+#define A5XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT			8
+static inline uint32_t A5XX_SP_VS_OUT_REG_A_COMPMASK(uint32_t val)
+{
+	return ((val) << A5XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT) & A5XX_SP_VS_OUT_REG_A_COMPMASK__MASK;
+}
+#define A5XX_SP_VS_OUT_REG_B_REGID__MASK			0x00ff0000
+#define A5XX_SP_VS_OUT_REG_B_REGID__SHIFT			16
+static inline uint32_t A5XX_SP_VS_OUT_REG_B_REGID(uint32_t val)
+{
+	return ((val) << A5XX_SP_VS_OUT_REG_B_REGID__SHIFT) & A5XX_SP_VS_OUT_REG_B_REGID__MASK;
+}
+#define A5XX_SP_VS_OUT_REG_B_COMPMASK__MASK			0x0f000000
+#define A5XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT			24
+static inline uint32_t A5XX_SP_VS_OUT_REG_B_COMPMASK(uint32_t val)
+{
+	return ((val) << A5XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT) & A5XX_SP_VS_OUT_REG_B_COMPMASK__MASK;
+}
+
+static inline uint32_t REG_A5XX_SP_VS_VPC_DST(uint32_t i0) { return 0x0000e5a3 + 0x1*i0; }
+
+static inline uint32_t REG_A5XX_SP_VS_VPC_DST_REG(uint32_t i0) { return 0x0000e5a3 + 0x1*i0; }
+#define A5XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK			0x000000ff
+#define A5XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT			0
+static inline uint32_t A5XX_SP_VS_VPC_DST_REG_OUTLOC0(uint32_t val)
+{
+	return ((val) << A5XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT) & A5XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK;
+}
+#define A5XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK			0x0000ff00
+#define A5XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT			8
+static inline uint32_t A5XX_SP_VS_VPC_DST_REG_OUTLOC1(uint32_t val)
+{
+	return ((val) << A5XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT) & A5XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK;
+}
+#define A5XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK			0x00ff0000
+#define A5XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT			16
+static inline uint32_t A5XX_SP_VS_VPC_DST_REG_OUTLOC2(uint32_t val)
+{
+	return ((val) << A5XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT) & A5XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK;
+}
+#define A5XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK			0xff000000
+#define A5XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT			24
+static inline uint32_t A5XX_SP_VS_VPC_DST_REG_OUTLOC3(uint32_t val)
+{
+	return ((val) << A5XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT) & A5XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK;
+}
+
+#define REG_A5XX_UNKNOWN_E5AB					0x0000e5ab
+
+#define REG_A5XX_SP_VS_OBJ_START_LO				0x0000e5ac
+
+#define REG_A5XX_SP_VS_OBJ_START_HI				0x0000e5ad
+
+#define REG_A5XX_SP_FS_CTRL_REG0				0x0000e5c0
+#define A5XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK		0x000003f0
+#define A5XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT		4
+static inline uint32_t A5XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A5XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK;
+}
+#define A5XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK		0x0000fc00
+#define A5XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT		10
+static inline uint32_t A5XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val)
+{
+	return ((val) << A5XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK;
+}
+#define A5XX_SP_FS_CTRL_REG0_VARYING				0x00010000
+#define A5XX_SP_FS_CTRL_REG0_PIXLODENABLE			0x00100000
+
+#define REG_A5XX_UNKNOWN_E5C2					0x0000e5c2
+
+#define REG_A5XX_SP_FS_OBJ_START_LO				0x0000e5c3
+
+#define REG_A5XX_SP_FS_OBJ_START_HI				0x0000e5c4
+
+#define REG_A5XX_SP_BLEND_CNTL					0x0000e5c9
+
+#define REG_A5XX_SP_FS_OUTPUT_CNTL				0x0000e5ca
+#define A5XX_SP_FS_OUTPUT_CNTL_MRT__MASK			0x0000000f
+#define A5XX_SP_FS_OUTPUT_CNTL_MRT__SHIFT			0
+static inline uint32_t A5XX_SP_FS_OUTPUT_CNTL_MRT(uint32_t val)
+{
+	return ((val) << A5XX_SP_FS_OUTPUT_CNTL_MRT__SHIFT) & A5XX_SP_FS_OUTPUT_CNTL_MRT__MASK;
+}
+#define A5XX_SP_FS_OUTPUT_CNTL_DEPTH_REGID__MASK		0x00001fe0
+#define A5XX_SP_FS_OUTPUT_CNTL_DEPTH_REGID__SHIFT		5
+static inline uint32_t A5XX_SP_FS_OUTPUT_CNTL_DEPTH_REGID(uint32_t val)
+{
+	return ((val) << A5XX_SP_FS_OUTPUT_CNTL_DEPTH_REGID__SHIFT) & A5XX_SP_FS_OUTPUT_CNTL_DEPTH_REGID__MASK;
+}
+#define A5XX_SP_FS_OUTPUT_CNTL_SAMPLEMASK_REGID__MASK		0x001fe000
+#define A5XX_SP_FS_OUTPUT_CNTL_SAMPLEMASK_REGID__SHIFT		13
+static inline uint32_t A5XX_SP_FS_OUTPUT_CNTL_SAMPLEMASK_REGID(uint32_t val)
+{
+	return ((val) << A5XX_SP_FS_OUTPUT_CNTL_SAMPLEMASK_REGID__SHIFT) & A5XX_SP_FS_OUTPUT_CNTL_SAMPLEMASK_REGID__MASK;
+}
+
+static inline uint32_t REG_A5XX_SP_FS_OUTPUT(uint32_t i0) { return 0x0000e5cb + 0x1*i0; }
+
+static inline uint32_t REG_A5XX_SP_FS_OUTPUT_REG(uint32_t i0) { return 0x0000e5cb + 0x1*i0; }
+#define A5XX_SP_FS_OUTPUT_REG_REGID__MASK			0x000000ff
+#define A5XX_SP_FS_OUTPUT_REG_REGID__SHIFT			0
+static inline uint32_t A5XX_SP_FS_OUTPUT_REG_REGID(uint32_t val)
+{
+	return ((val) << A5XX_SP_FS_OUTPUT_REG_REGID__SHIFT) & A5XX_SP_FS_OUTPUT_REG_REGID__MASK;
+}
+#define A5XX_SP_FS_OUTPUT_REG_HALF_PRECISION			0x00000100
+
+static inline uint32_t REG_A5XX_SP_FS_MRT(uint32_t i0) { return 0x0000e5d3 + 0x1*i0; }
+
+static inline uint32_t REG_A5XX_SP_FS_MRT_REG(uint32_t i0) { return 0x0000e5d3 + 0x1*i0; }
+#define A5XX_SP_FS_MRT_REG_COLOR_FORMAT__MASK			0x000000ff
+#define A5XX_SP_FS_MRT_REG_COLOR_FORMAT__SHIFT			0
+static inline uint32_t A5XX_SP_FS_MRT_REG_COLOR_FORMAT(enum a5xx_color_fmt val)
+{
+	return ((val) << A5XX_SP_FS_MRT_REG_COLOR_FORMAT__SHIFT) & A5XX_SP_FS_MRT_REG_COLOR_FORMAT__MASK;
+}
+
+#define REG_A5XX_UNKNOWN_E5DB					0x0000e5db
+
+#define REG_A5XX_SP_CS_CNTL_0					0x0000e5f0
+
+#define REG_A5XX_UNKNOWN_E600					0x0000e600
+
+#define REG_A5XX_UNKNOWN_E640					0x0000e640
+
+#define REG_A5XX_TPL1_TP_RAS_MSAA_CNTL				0x0000e704
+#define A5XX_TPL1_TP_RAS_MSAA_CNTL_SAMPLES__MASK		0x00000003
+#define A5XX_TPL1_TP_RAS_MSAA_CNTL_SAMPLES__SHIFT		0
+static inline uint32_t A5XX_TPL1_TP_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A5XX_TPL1_TP_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A5XX_TPL1_TP_RAS_MSAA_CNTL_SAMPLES__MASK;
+}
+
+#define REG_A5XX_TPL1_TP_DEST_MSAA_CNTL				0x0000e705
+#define A5XX_TPL1_TP_DEST_MSAA_CNTL_SAMPLES__MASK		0x00000003
+#define A5XX_TPL1_TP_DEST_MSAA_CNTL_SAMPLES__SHIFT		0
+static inline uint32_t A5XX_TPL1_TP_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val)
+{
+	return ((val) << A5XX_TPL1_TP_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A5XX_TPL1_TP_DEST_MSAA_CNTL_SAMPLES__MASK;
+}
+#define A5XX_TPL1_TP_DEST_MSAA_CNTL_MSAA_DISABLE		0x00000004
+
+#define REG_A5XX_TPL1_VS_TEX_COUNT				0x0000e700
+
+#define REG_A5XX_TPL1_VS_TEX_SAMP_LO				0x0000e722
+
+#define REG_A5XX_TPL1_VS_TEX_SAMP_HI				0x0000e723
+
+#define REG_A5XX_TPL1_VS_TEX_CONST_LO				0x0000e72a
+
+#define REG_A5XX_TPL1_VS_TEX_CONST_HI				0x0000e72b
+
+#define REG_A5XX_TPL1_FS_TEX_COUNT				0x0000e750
+
+#define REG_A5XX_TPL1_FS_TEX_SAMP_LO				0x0000e75a
+
+#define REG_A5XX_TPL1_FS_TEX_SAMP_HI				0x0000e75b
+
+#define REG_A5XX_TPL1_FS_TEX_CONST_LO				0x0000e75e
+
+#define REG_A5XX_TPL1_FS_TEX_CONST_HI				0x0000e75f
+
+#define REG_A5XX_TPL1_TP_FS_ROTATION_CNTL			0x0000e764
+
+#define REG_A5XX_HLSQ_CONTROL_0_REG				0x0000e784
+
+#define REG_A5XX_HLSQ_CONTROL_1_REG				0x0000e785
+#define A5XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__MASK	0x0000003f
+#define A5XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__SHIFT	0
+static inline uint32_t A5XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__SHIFT) & A5XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD__MASK;
+}
+
+#define REG_A5XX_HLSQ_CONTROL_2_REG				0x0000e786
+#define A5XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK			0x000000ff
+#define A5XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT		0
+static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_FACEREGID(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK;
+}
+
+#define REG_A5XX_HLSQ_CONTROL_3_REG				0x0000e787
+#define A5XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__MASK		0x000000ff
+#define A5XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__SHIFT		0
+static inline uint32_t A5XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__SHIFT) & A5XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__MASK;
+}
+
+#define REG_A5XX_HLSQ_CONTROL_4_REG				0x0000e788
+#define A5XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK		0x00ff0000
+#define A5XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT		16
+static inline uint32_t A5XX_HLSQ_CONTROL_4_REG_XYCOORDREGID(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT) & A5XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK;
+}
+#define A5XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK		0xff000000
+#define A5XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT		24
+static inline uint32_t A5XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT) & A5XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK;
+}
+
+#define REG_A5XX_HLSQ_UPDATE_CNTL				0x0000e78a
+
+#define REG_A5XX_HLSQ_VS_CONTROL_REG				0x0000e78b
+#define A5XX_HLSQ_VS_CONTROL_REG_ENABLED			0x00000001
+#define A5XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x000000fe
+#define A5XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	1
+static inline uint32_t A5XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A5XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
+}
+#define A5XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00007f00
+#define A5XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		8
+static inline uint32_t A5XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A5XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET__MASK;
+}
+
+#define REG_A5XX_HLSQ_FS_CONTROL_REG				0x0000e78c
+#define A5XX_HLSQ_FS_CONTROL_REG_ENABLED			0x00000001
+#define A5XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x000000fe
+#define A5XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	1
+static inline uint32_t A5XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A5XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
+}
+#define A5XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00007f00
+#define A5XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		8
+static inline uint32_t A5XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A5XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET__MASK;
+}
+
+#define REG_A5XX_HLSQ_HS_CONTROL_REG				0x0000e78d
+#define A5XX_HLSQ_HS_CONTROL_REG_ENABLED			0x00000001
+#define A5XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x000000fe
+#define A5XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	1
+static inline uint32_t A5XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A5XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
+}
+#define A5XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00007f00
+#define A5XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		8
+static inline uint32_t A5XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A5XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET__MASK;
+}
+
+#define REG_A5XX_HLSQ_DS_CONTROL_REG				0x0000e78e
+#define A5XX_HLSQ_DS_CONTROL_REG_ENABLED			0x00000001
+#define A5XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x000000fe
+#define A5XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	1
+static inline uint32_t A5XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A5XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
+}
+#define A5XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00007f00
+#define A5XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		8
+static inline uint32_t A5XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A5XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET__MASK;
+}
+
+#define REG_A5XX_HLSQ_GS_CONTROL_REG				0x0000e78f
+#define A5XX_HLSQ_GS_CONTROL_REG_ENABLED			0x00000001
+#define A5XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__MASK	0x000000fe
+#define A5XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT	1
+static inline uint32_t A5XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A5XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__MASK;
+}
+#define A5XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET__MASK		0x00007f00
+#define A5XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET__SHIFT		8
+static inline uint32_t A5XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A5XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET__MASK;
+}
+
+#define REG_A5XX_HLSQ_CS_CONFIG					0x0000e790
+
+#define REG_A5XX_HLSQ_VS_CNTL					0x0000e791
+#define A5XX_HLSQ_VS_CNTL_INSTRLEN__MASK			0xfffffffe
+#define A5XX_HLSQ_VS_CNTL_INSTRLEN__SHIFT			1
+static inline uint32_t A5XX_HLSQ_VS_CNTL_INSTRLEN(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_VS_CNTL_INSTRLEN__SHIFT) & A5XX_HLSQ_VS_CNTL_INSTRLEN__MASK;
+}
+
+#define REG_A5XX_HLSQ_FS_CNTL					0x0000e792
+#define A5XX_HLSQ_FS_CNTL_INSTRLEN__MASK			0xfffffffe
+#define A5XX_HLSQ_FS_CNTL_INSTRLEN__SHIFT			1
+static inline uint32_t A5XX_HLSQ_FS_CNTL_INSTRLEN(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_FS_CNTL_INSTRLEN__SHIFT) & A5XX_HLSQ_FS_CNTL_INSTRLEN__MASK;
+}
+
+#define REG_A5XX_HLSQ_HS_CNTL					0x0000e793
+#define A5XX_HLSQ_HS_CNTL_INSTRLEN__MASK			0xfffffffe
+#define A5XX_HLSQ_HS_CNTL_INSTRLEN__SHIFT			1
+static inline uint32_t A5XX_HLSQ_HS_CNTL_INSTRLEN(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_HS_CNTL_INSTRLEN__SHIFT) & A5XX_HLSQ_HS_CNTL_INSTRLEN__MASK;
+}
+
+#define REG_A5XX_HLSQ_DS_CNTL					0x0000e794
+#define A5XX_HLSQ_DS_CNTL_INSTRLEN__MASK			0xfffffffe
+#define A5XX_HLSQ_DS_CNTL_INSTRLEN__SHIFT			1
+static inline uint32_t A5XX_HLSQ_DS_CNTL_INSTRLEN(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_DS_CNTL_INSTRLEN__SHIFT) & A5XX_HLSQ_DS_CNTL_INSTRLEN__MASK;
+}
+
+#define REG_A5XX_HLSQ_GS_CNTL					0x0000e795
+#define A5XX_HLSQ_GS_CNTL_INSTRLEN__MASK			0xfffffffe
+#define A5XX_HLSQ_GS_CNTL_INSTRLEN__SHIFT			1
+static inline uint32_t A5XX_HLSQ_GS_CNTL_INSTRLEN(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_GS_CNTL_INSTRLEN__SHIFT) & A5XX_HLSQ_GS_CNTL_INSTRLEN__MASK;
+}
+
+#define REG_A5XX_HLSQ_CS_CNTL					0x0000e796
+#define A5XX_HLSQ_CS_CNTL_INSTRLEN__MASK			0xfffffffe
+#define A5XX_HLSQ_CS_CNTL_INSTRLEN__SHIFT			1
+static inline uint32_t A5XX_HLSQ_CS_CNTL_INSTRLEN(uint32_t val)
+{
+	return ((val) << A5XX_HLSQ_CS_CNTL_INSTRLEN__SHIFT) & A5XX_HLSQ_CS_CNTL_INSTRLEN__MASK;
+}
+
+#define REG_A5XX_HLSQ_CS_KERNEL_GROUP_X				0x0000e7b9
+
+#define REG_A5XX_HLSQ_CS_KERNEL_GROUP_Y				0x0000e7ba
+
+#define REG_A5XX_HLSQ_CS_KERNEL_GROUP_Z				0x0000e7bb
+
+#define REG_A5XX_HLSQ_CS_NDRANGE_0				0x0000e7b0
+
+#define REG_A5XX_HLSQ_CS_NDRANGE_1				0x0000e7b1
+
+#define REG_A5XX_HLSQ_CS_NDRANGE_2				0x0000e7b2
+
+#define REG_A5XX_HLSQ_CS_NDRANGE_3				0x0000e7b3
+
+#define REG_A5XX_HLSQ_CS_NDRANGE_4				0x0000e7b4
+
+#define REG_A5XX_HLSQ_CS_NDRANGE_5				0x0000e7b5
+
+#define REG_A5XX_HLSQ_CS_NDRANGE_6				0x0000e7b6
+
+#define REG_A5XX_HLSQ_CS_CNTL_0					0x0000e7b7
+
+#define REG_A5XX_HLSQ_CS_CNTL_1					0x0000e7b8
+
+#define REG_A5XX_UNKNOWN_E7C0					0x0000e7c0
+
+#define REG_A5XX_HLSQ_VS_CONSTLEN				0x0000e7c3
+
+#define REG_A5XX_HLSQ_VS_INSTRLEN				0x0000e7c4
+
+#define REG_A5XX_UNKNOWN_E7C5					0x0000e7c5
+
+#define REG_A5XX_UNKNOWN_E7CA					0x0000e7ca
+
+#define REG_A5XX_HLSQ_FS_CONSTLEN				0x0000e7d7
+
+#define REG_A5XX_HLSQ_FS_INSTRLEN				0x0000e7d8
+
+#define REG_A5XX_HLSQ_HS_CONSTLEN				0x0000e7c8
+
+#define REG_A5XX_HLSQ_HS_INSTRLEN				0x0000e7c9
+
+#define REG_A5XX_HLSQ_DS_CONSTLEN				0x0000e7cd
+
+#define REG_A5XX_HLSQ_DS_INSTRLEN				0x0000e7ce
+
+#define REG_A5XX_UNKNOWN_E7CF					0x0000e7cf
+
+#define REG_A5XX_HLSQ_GS_CONSTLEN				0x0000e7d2
+
+#define REG_A5XX_HLSQ_GS_INSTRLEN				0x0000e7d3
+
+#define REG_A5XX_UNKNOWN_E7D4					0x0000e7d4
+
+#define REG_A5XX_UNKNOWN_E7D9					0x0000e7d9
+
+#define REG_A5XX_HLSQ_CONTEXT_SWITCH_CS_SW_3			0x0000e7dc
+
+#define REG_A5XX_HLSQ_CONTEXT_SWITCH_CS_SW_4			0x0000e7dd
+
+#define REG_A5XX_RB_2D_DST_FILL					0x00002101
+
+#define REG_A5XX_RB_2D_SRC_INFO					0x00002107
+#define A5XX_RB_2D_SRC_INFO_COLOR_FORMAT__MASK			0x000000ff
+#define A5XX_RB_2D_SRC_INFO_COLOR_FORMAT__SHIFT			0
+static inline uint32_t A5XX_RB_2D_SRC_INFO_COLOR_FORMAT(enum a5xx_color_fmt val)
+{
+	return ((val) << A5XX_RB_2D_SRC_INFO_COLOR_FORMAT__SHIFT) & A5XX_RB_2D_SRC_INFO_COLOR_FORMAT__MASK;
+}
+#define A5XX_RB_2D_SRC_INFO_COLOR_SWAP__MASK			0x00000c00
+#define A5XX_RB_2D_SRC_INFO_COLOR_SWAP__SHIFT			10
+static inline uint32_t A5XX_RB_2D_SRC_INFO_COLOR_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A5XX_RB_2D_SRC_INFO_COLOR_SWAP__SHIFT) & A5XX_RB_2D_SRC_INFO_COLOR_SWAP__MASK;
+}
+
+#define REG_A5XX_RB_2D_SRC_LO					0x00002108
+
+#define REG_A5XX_RB_2D_SRC_HI					0x00002109
+
+#define REG_A5XX_RB_2D_DST_INFO					0x00002110
+#define A5XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK			0x000000ff
+#define A5XX_RB_2D_DST_INFO_COLOR_FORMAT__SHIFT			0
+static inline uint32_t A5XX_RB_2D_DST_INFO_COLOR_FORMAT(enum a5xx_color_fmt val)
+{
+	return ((val) << A5XX_RB_2D_DST_INFO_COLOR_FORMAT__SHIFT) & A5XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK;
+}
+#define A5XX_RB_2D_DST_INFO_COLOR_SWAP__MASK			0x00000c00
+#define A5XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT			10
+static inline uint32_t A5XX_RB_2D_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A5XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT) & A5XX_RB_2D_DST_INFO_COLOR_SWAP__MASK;
+}
+
+#define REG_A5XX_RB_2D_SRC_FLAGS_LO				0x00002140
+
+#define REG_A5XX_RB_2D_SRC_FLAGS_HI				0x00002141
+
+#define REG_A5XX_RB_2D_DST_LO					0x00002111
+
+#define REG_A5XX_RB_2D_DST_HI					0x00002112
+
+#define REG_A5XX_RB_2D_DST_FLAGS_LO				0x00002143
+
+#define REG_A5XX_RB_2D_DST_FLAGS_HI				0x00002144
+
+#define REG_A5XX_GRAS_2D_SRC_INFO				0x00002181
+#define A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__MASK		0x000000ff
+#define A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__SHIFT		0
+static inline uint32_t A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT(enum a5xx_color_fmt val)
+{
+	return ((val) << A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__SHIFT) & A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__MASK;
+}
+#define A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__MASK			0x00000c00
+#define A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__SHIFT			10
+static inline uint32_t A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__SHIFT) & A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__MASK;
+}
+
+#define REG_A5XX_GRAS_2D_DST_INFO				0x00002182
+#define A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__MASK		0x000000ff
+#define A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__SHIFT		0
+static inline uint32_t A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT(enum a5xx_color_fmt val)
+{
+	return ((val) << A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__SHIFT) & A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__MASK;
+}
+#define A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__MASK			0x00000c00
+#define A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__SHIFT			10
+static inline uint32_t A5XX_GRAS_2D_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__SHIFT) & A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__MASK;
+}
+
+#define REG_A5XX_TEX_SAMP_0					0x00000000
+#define A5XX_TEX_SAMP_0_MIPFILTER_LINEAR_NEAR			0x00000001
+#define A5XX_TEX_SAMP_0_XY_MAG__MASK				0x00000006
+#define A5XX_TEX_SAMP_0_XY_MAG__SHIFT				1
+static inline uint32_t A5XX_TEX_SAMP_0_XY_MAG(enum a5xx_tex_filter val)
+{
+	return ((val) << A5XX_TEX_SAMP_0_XY_MAG__SHIFT) & A5XX_TEX_SAMP_0_XY_MAG__MASK;
+}
+#define A5XX_TEX_SAMP_0_XY_MIN__MASK				0x00000018
+#define A5XX_TEX_SAMP_0_XY_MIN__SHIFT				3
+static inline uint32_t A5XX_TEX_SAMP_0_XY_MIN(enum a5xx_tex_filter val)
+{
+	return ((val) << A5XX_TEX_SAMP_0_XY_MIN__SHIFT) & A5XX_TEX_SAMP_0_XY_MIN__MASK;
+}
+#define A5XX_TEX_SAMP_0_WRAP_S__MASK				0x000000e0
+#define A5XX_TEX_SAMP_0_WRAP_S__SHIFT				5
+static inline uint32_t A5XX_TEX_SAMP_0_WRAP_S(enum a5xx_tex_clamp val)
+{
+	return ((val) << A5XX_TEX_SAMP_0_WRAP_S__SHIFT) & A5XX_TEX_SAMP_0_WRAP_S__MASK;
+}
+#define A5XX_TEX_SAMP_0_WRAP_T__MASK				0x00000700
+#define A5XX_TEX_SAMP_0_WRAP_T__SHIFT				8
+static inline uint32_t A5XX_TEX_SAMP_0_WRAP_T(enum a5xx_tex_clamp val)
+{
+	return ((val) << A5XX_TEX_SAMP_0_WRAP_T__SHIFT) & A5XX_TEX_SAMP_0_WRAP_T__MASK;
+}
+#define A5XX_TEX_SAMP_0_WRAP_R__MASK				0x00003800
+#define A5XX_TEX_SAMP_0_WRAP_R__SHIFT				11
+static inline uint32_t A5XX_TEX_SAMP_0_WRAP_R(enum a5xx_tex_clamp val)
+{
+	return ((val) << A5XX_TEX_SAMP_0_WRAP_R__SHIFT) & A5XX_TEX_SAMP_0_WRAP_R__MASK;
+}
+#define A5XX_TEX_SAMP_0_ANISO__MASK				0x0001c000
+#define A5XX_TEX_SAMP_0_ANISO__SHIFT				14
+static inline uint32_t A5XX_TEX_SAMP_0_ANISO(enum a5xx_tex_aniso val)
+{
+	return ((val) << A5XX_TEX_SAMP_0_ANISO__SHIFT) & A5XX_TEX_SAMP_0_ANISO__MASK;
+}
+#define A5XX_TEX_SAMP_0_LOD_BIAS__MASK				0xfff80000
+#define A5XX_TEX_SAMP_0_LOD_BIAS__SHIFT				19
+static inline uint32_t A5XX_TEX_SAMP_0_LOD_BIAS(float val)
+{
+	return ((((int32_t)(val * 256.0))) << A5XX_TEX_SAMP_0_LOD_BIAS__SHIFT) & A5XX_TEX_SAMP_0_LOD_BIAS__MASK;
+}
+
+#define REG_A5XX_TEX_SAMP_1					0x00000001
+#define A5XX_TEX_SAMP_1_COMPARE_FUNC__MASK			0x0000000e
+#define A5XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT			1
+static inline uint32_t A5XX_TEX_SAMP_1_COMPARE_FUNC(enum adreno_compare_func val)
+{
+	return ((val) << A5XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT) & A5XX_TEX_SAMP_1_COMPARE_FUNC__MASK;
+}
+#define A5XX_TEX_SAMP_1_CUBEMAPSEAMLESSFILTOFF			0x00000010
+#define A5XX_TEX_SAMP_1_UNNORM_COORDS				0x00000020
+#define A5XX_TEX_SAMP_1_MIPFILTER_LINEAR_FAR			0x00000040
+#define A5XX_TEX_SAMP_1_MAX_LOD__MASK				0x000fff00
+#define A5XX_TEX_SAMP_1_MAX_LOD__SHIFT				8
+static inline uint32_t A5XX_TEX_SAMP_1_MAX_LOD(float val)
+{
+	return ((((uint32_t)(val * 256.0))) << A5XX_TEX_SAMP_1_MAX_LOD__SHIFT) & A5XX_TEX_SAMP_1_MAX_LOD__MASK;
+}
+#define A5XX_TEX_SAMP_1_MIN_LOD__MASK				0xfff00000
+#define A5XX_TEX_SAMP_1_MIN_LOD__SHIFT				20
+static inline uint32_t A5XX_TEX_SAMP_1_MIN_LOD(float val)
+{
+	return ((((uint32_t)(val * 256.0))) << A5XX_TEX_SAMP_1_MIN_LOD__SHIFT) & A5XX_TEX_SAMP_1_MIN_LOD__MASK;
+}
+
+#define REG_A5XX_TEX_SAMP_2					0x00000002
+
+#define REG_A5XX_TEX_SAMP_3					0x00000003
+
+#define REG_A5XX_TEX_CONST_0					0x00000000
+#define A5XX_TEX_CONST_0_TILE_MODE__MASK			0x00000003
+#define A5XX_TEX_CONST_0_TILE_MODE__SHIFT			0
+static inline uint32_t A5XX_TEX_CONST_0_TILE_MODE(enum a5xx_tile_mode val)
+{
+	return ((val) << A5XX_TEX_CONST_0_TILE_MODE__SHIFT) & A5XX_TEX_CONST_0_TILE_MODE__MASK;
+}
+#define A5XX_TEX_CONST_0_SRGB					0x00000004
+#define A5XX_TEX_CONST_0_SWIZ_X__MASK				0x00000070
+#define A5XX_TEX_CONST_0_SWIZ_X__SHIFT				4
+static inline uint32_t A5XX_TEX_CONST_0_SWIZ_X(enum a5xx_tex_swiz val)
+{
+	return ((val) << A5XX_TEX_CONST_0_SWIZ_X__SHIFT) & A5XX_TEX_CONST_0_SWIZ_X__MASK;
+}
+#define A5XX_TEX_CONST_0_SWIZ_Y__MASK				0x00000380
+#define A5XX_TEX_CONST_0_SWIZ_Y__SHIFT				7
+static inline uint32_t A5XX_TEX_CONST_0_SWIZ_Y(enum a5xx_tex_swiz val)
+{
+	return ((val) << A5XX_TEX_CONST_0_SWIZ_Y__SHIFT) & A5XX_TEX_CONST_0_SWIZ_Y__MASK;
+}
+#define A5XX_TEX_CONST_0_SWIZ_Z__MASK				0x00001c00
+#define A5XX_TEX_CONST_0_SWIZ_Z__SHIFT				10
+static inline uint32_t A5XX_TEX_CONST_0_SWIZ_Z(enum a5xx_tex_swiz val)
+{
+	return ((val) << A5XX_TEX_CONST_0_SWIZ_Z__SHIFT) & A5XX_TEX_CONST_0_SWIZ_Z__MASK;
+}
+#define A5XX_TEX_CONST_0_SWIZ_W__MASK				0x0000e000
+#define A5XX_TEX_CONST_0_SWIZ_W__SHIFT				13
+static inline uint32_t A5XX_TEX_CONST_0_SWIZ_W(enum a5xx_tex_swiz val)
+{
+	return ((val) << A5XX_TEX_CONST_0_SWIZ_W__SHIFT) & A5XX_TEX_CONST_0_SWIZ_W__MASK;
+}
+#define A5XX_TEX_CONST_0_FMT__MASK				0x3fc00000
+#define A5XX_TEX_CONST_0_FMT__SHIFT				22
+static inline uint32_t A5XX_TEX_CONST_0_FMT(enum a5xx_tex_fmt val)
+{
+	return ((val) << A5XX_TEX_CONST_0_FMT__SHIFT) & A5XX_TEX_CONST_0_FMT__MASK;
+}
+#define A5XX_TEX_CONST_0_SWAP__MASK				0xc0000000
+#define A5XX_TEX_CONST_0_SWAP__SHIFT				30
+static inline uint32_t A5XX_TEX_CONST_0_SWAP(enum a3xx_color_swap val)
+{
+	return ((val) << A5XX_TEX_CONST_0_SWAP__SHIFT) & A5XX_TEX_CONST_0_SWAP__MASK;
+}
+
+#define REG_A5XX_TEX_CONST_1					0x00000001
+#define A5XX_TEX_CONST_1_WIDTH__MASK				0x00007fff
+#define A5XX_TEX_CONST_1_WIDTH__SHIFT				0
+static inline uint32_t A5XX_TEX_CONST_1_WIDTH(uint32_t val)
+{
+	return ((val) << A5XX_TEX_CONST_1_WIDTH__SHIFT) & A5XX_TEX_CONST_1_WIDTH__MASK;
+}
+#define A5XX_TEX_CONST_1_HEIGHT__MASK				0x3fff8000
+#define A5XX_TEX_CONST_1_HEIGHT__SHIFT				15
+static inline uint32_t A5XX_TEX_CONST_1_HEIGHT(uint32_t val)
+{
+	return ((val) << A5XX_TEX_CONST_1_HEIGHT__SHIFT) & A5XX_TEX_CONST_1_HEIGHT__MASK;
+}
+
+#define REG_A5XX_TEX_CONST_2					0x00000002
+#define A5XX_TEX_CONST_2_FETCHSIZE__MASK			0x0000000f
+#define A5XX_TEX_CONST_2_FETCHSIZE__SHIFT			0
+static inline uint32_t A5XX_TEX_CONST_2_FETCHSIZE(enum a5xx_tex_fetchsize val)
+{
+	return ((val) << A5XX_TEX_CONST_2_FETCHSIZE__SHIFT) & A5XX_TEX_CONST_2_FETCHSIZE__MASK;
+}
+#define A5XX_TEX_CONST_2_PITCH__MASK				0x1fffff80
+#define A5XX_TEX_CONST_2_PITCH__SHIFT				7
+static inline uint32_t A5XX_TEX_CONST_2_PITCH(uint32_t val)
+{
+	return ((val) << A5XX_TEX_CONST_2_PITCH__SHIFT) & A5XX_TEX_CONST_2_PITCH__MASK;
+}
+#define A5XX_TEX_CONST_2_TYPE__MASK				0x60000000
+#define A5XX_TEX_CONST_2_TYPE__SHIFT				29
+static inline uint32_t A5XX_TEX_CONST_2_TYPE(enum a5xx_tex_type val)
+{
+	return ((val) << A5XX_TEX_CONST_2_TYPE__SHIFT) & A5XX_TEX_CONST_2_TYPE__MASK;
+}
+
+#define REG_A5XX_TEX_CONST_3					0x00000003
+#define A5XX_TEX_CONST_3_ARRAY_PITCH__MASK			0x00003fff
+#define A5XX_TEX_CONST_3_ARRAY_PITCH__SHIFT			0
+static inline uint32_t A5XX_TEX_CONST_3_ARRAY_PITCH(uint32_t val)
+{
+	return ((val >> 12) << A5XX_TEX_CONST_3_ARRAY_PITCH__SHIFT) & A5XX_TEX_CONST_3_ARRAY_PITCH__MASK;
+}
+#define A5XX_TEX_CONST_3_FLAG					0x10000000
+
+#define REG_A5XX_TEX_CONST_4					0x00000004
+#define A5XX_TEX_CONST_4_BASE_LO__MASK				0xffffffe0
+#define A5XX_TEX_CONST_4_BASE_LO__SHIFT				5
+static inline uint32_t A5XX_TEX_CONST_4_BASE_LO(uint32_t val)
+{
+	return ((val >> 5) << A5XX_TEX_CONST_4_BASE_LO__SHIFT) & A5XX_TEX_CONST_4_BASE_LO__MASK;
+}
+
+#define REG_A5XX_TEX_CONST_5					0x00000005
+#define A5XX_TEX_CONST_5_BASE_HI__MASK				0x0001ffff
+#define A5XX_TEX_CONST_5_BASE_HI__SHIFT				0
+static inline uint32_t A5XX_TEX_CONST_5_BASE_HI(uint32_t val)
+{
+	return ((val) << A5XX_TEX_CONST_5_BASE_HI__SHIFT) & A5XX_TEX_CONST_5_BASE_HI__MASK;
+}
+#define A5XX_TEX_CONST_5_DEPTH__MASK				0x3ffe0000
+#define A5XX_TEX_CONST_5_DEPTH__SHIFT				17
+static inline uint32_t A5XX_TEX_CONST_5_DEPTH(uint32_t val)
+{
+	return ((val) << A5XX_TEX_CONST_5_DEPTH__SHIFT) & A5XX_TEX_CONST_5_DEPTH__MASK;
+}
+
+#define REG_A5XX_TEX_CONST_6					0x00000006
+
+#define REG_A5XX_TEX_CONST_7					0x00000007
+
+#define REG_A5XX_TEX_CONST_8					0x00000008
+
+#define REG_A5XX_TEX_CONST_9					0x00000009
+
+#define REG_A5XX_TEX_CONST_10					0x0000000a
+
+#define REG_A5XX_TEX_CONST_11					0x0000000b
+
+
+#endif /* A5XX_XML */
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
new file mode 100644
index 000000000000..b8647198c11c
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
@@ -0,0 +1,888 @@
+/* Copyright (c) 2016 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include "msm_gem.h"
+#include "a5xx_gpu.h"
+
+extern bool hang_debug;
+static void a5xx_dump(struct msm_gpu *gpu);
+
+static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
+	struct msm_file_private *ctx)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct msm_drm_private *priv = gpu->dev->dev_private;
+	struct msm_ringbuffer *ring = gpu->rb;
+	unsigned int i, ibs = 0;
+
+	for (i = 0; i < submit->nr_cmds; i++) {
+		switch (submit->cmd[i].type) {
+		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
+			break;
+		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
+			if (priv->lastctx == ctx)
+				break;
+		case MSM_SUBMIT_CMD_BUF:
+			OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
+			OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
+			OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
+			OUT_RING(ring, submit->cmd[i].size);
+			ibs++;
+			break;
+		}
+	}
+
+	OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1);
+	OUT_RING(ring, submit->fence->seqno);
+
+	OUT_PKT7(ring, CP_EVENT_WRITE, 4);
+	OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31));
+	OUT_RING(ring, lower_32_bits(rbmemptr(adreno_gpu, fence)));
+	OUT_RING(ring, upper_32_bits(rbmemptr(adreno_gpu, fence)));
+	OUT_RING(ring, submit->fence->seqno);
+
+	gpu->funcs->flush(gpu);
+}
+
+struct a5xx_hwcg {
+	u32 offset;
+	u32 value;
+};
+
+static const struct a5xx_hwcg a530_hwcg[] = {
+	{REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220},
+	{REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
+	{REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
+	{REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF},
+	{REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF},
+	{REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
+	{REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
+	{REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
+	{REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
+	{REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
+	{REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
+	{REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222},
+	{REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222},
+	{REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
+	{REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
+	{REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
+	{REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
+	{REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
+	{REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
+	{REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
+	{REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
+	{REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
+	{REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
+	{REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777},
+	{REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777},
+	{REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
+	{REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
+	{REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
+	{REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
+	{REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
+	{REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
+	{REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
+	{REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
+	{REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
+	{REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
+	{REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111},
+	{REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111},
+	{REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
+	{REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
+	{REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
+	{REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222},
+	{REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
+	{REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
+	{REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220},
+	{REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220},
+	{REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
+	{REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
+	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
+	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
+	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404},
+	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404},
+	{REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
+	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
+	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
+	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002},
+	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002},
+	{REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
+	{REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
+	{REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
+	{REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
+	{REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
+	{REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
+	{REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
+	{REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
+	{REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
+	{REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
+	{REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}
+};
+
+static const struct {
+	int (*test)(struct adreno_gpu *gpu);
+	const struct a5xx_hwcg *regs;
+	unsigned int count;
+} a5xx_hwcg_regs[] = {
+	{ adreno_is_a530, a530_hwcg, ARRAY_SIZE(a530_hwcg), },
+};
+
+static void _a5xx_enable_hwcg(struct msm_gpu *gpu,
+		const struct a5xx_hwcg *regs, unsigned int count)
+{
+	unsigned int i;
+
+	for (i = 0; i < count; i++)
+		gpu_write(gpu, regs[i].offset, regs[i].value);
+
+	gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xAAA8AA00);
+	gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, 0x182);
+}
+
+static void a5xx_enable_hwcg(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(a5xx_hwcg_regs); i++) {
+		if (a5xx_hwcg_regs[i].test(adreno_gpu)) {
+			_a5xx_enable_hwcg(gpu, a5xx_hwcg_regs[i].regs,
+				a5xx_hwcg_regs[i].count);
+			return;
+		}
+	}
+}
+
+static int a5xx_me_init(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct msm_ringbuffer *ring = gpu->rb;
+
+	OUT_PKT7(ring, CP_ME_INIT, 8);
+
+	OUT_RING(ring, 0x0000002F);
+
+	/* Enable multiple hardware contexts */
+	OUT_RING(ring, 0x00000003);
+
+	/* Enable error detection */
+	OUT_RING(ring, 0x20000000);
+
+	/* Don't enable header dump */
+	OUT_RING(ring, 0x00000000);
+	OUT_RING(ring, 0x00000000);
+
+	/* Specify workarounds for various microcode issues */
+	if (adreno_is_a530(adreno_gpu)) {
+		/* Workaround for token end syncs
+		 * Force a WFI after every direct-render 3D mode draw and every
+		 * 2D mode 3 draw
+		 */
+		OUT_RING(ring, 0x0000000B);
+	} else {
+		/* No workarounds enabled */
+		OUT_RING(ring, 0x00000000);
+	}
+
+	OUT_RING(ring, 0x00000000);
+	OUT_RING(ring, 0x00000000);
+
+	gpu->funcs->flush(gpu);
+
+	return gpu->funcs->idle(gpu) ? 0 : -EINVAL;
+}
+
+static struct drm_gem_object *a5xx_ucode_load_bo(struct msm_gpu *gpu,
+		const struct firmware *fw, u64 *iova)
+{
+	struct drm_device *drm = gpu->dev;
+	struct drm_gem_object *bo;
+	void *ptr;
+
+	mutex_lock(&drm->struct_mutex);
+	bo = msm_gem_new(drm, fw->size - 4, MSM_BO_UNCACHED);
+	mutex_unlock(&drm->struct_mutex);
+
+	if (IS_ERR(bo))
+		return bo;
+
+	ptr = msm_gem_get_vaddr(bo);
+	if (!ptr) {
+		drm_gem_object_unreference_unlocked(bo);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	if (iova) {
+		int ret = msm_gem_get_iova(bo, gpu->id, iova);
+
+		if (ret) {
+			drm_gem_object_unreference_unlocked(bo);
+			return ERR_PTR(ret);
+		}
+	}
+
+	memcpy(ptr, &fw->data[4], fw->size - 4);
+
+	msm_gem_put_vaddr(bo);
+	return bo;
+}
+
+static int a5xx_ucode_init(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
+	int ret;
+
+	if (!a5xx_gpu->pm4_bo) {
+		a5xx_gpu->pm4_bo = a5xx_ucode_load_bo(gpu, adreno_gpu->pm4,
+			&a5xx_gpu->pm4_iova);
+
+		if (IS_ERR(a5xx_gpu->pm4_bo)) {
+			ret = PTR_ERR(a5xx_gpu->pm4_bo);
+			a5xx_gpu->pm4_bo = NULL;
+			dev_err(gpu->dev->dev, "could not allocate PM4: %d\n",
+				ret);
+			return ret;
+		}
+	}
+
+	if (!a5xx_gpu->pfp_bo) {
+		a5xx_gpu->pfp_bo = a5xx_ucode_load_bo(gpu, adreno_gpu->pfp,
+			&a5xx_gpu->pfp_iova);
+
+		if (IS_ERR(a5xx_gpu->pfp_bo)) {
+			ret = PTR_ERR(a5xx_gpu->pfp_bo);
+			a5xx_gpu->pfp_bo = NULL;
+			dev_err(gpu->dev->dev, "could not allocate PFP: %d\n",
+				ret);
+			return ret;
+		}
+	}
+
+	gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO,
+		REG_A5XX_CP_ME_INSTR_BASE_HI, a5xx_gpu->pm4_iova);
+
+	gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO,
+		REG_A5XX_CP_PFP_INSTR_BASE_HI, a5xx_gpu->pfp_iova);
+
+	return 0;
+}
+
+#define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
+	  A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
+	  A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
+	  A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
+	  A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
+	  A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \
+	  A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
+	  A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
+	  A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
+	  A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
+
+static int a5xx_hw_init(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	int ret;
+
+	gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);
+
+	/* Make all blocks contribute to the GPU BUSY perf counter */
+	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);
+
+	/* Enable RBBM error reporting bits */
+	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);
+
+	if (adreno_gpu->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) {
+		/*
+		 * Mask out the activity signals from RB1-3 to avoid false
+		 * positives
+		 */
+
+		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,
+			0xF0000000);
+		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,
+			0xFFFFFFFF);
+		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,
+			0xFFFFFFFF);
+		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,
+			0xFFFFFFFF);
+		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,
+			0xFFFFFFFF);
+		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,
+			0xFFFFFFFF);
+		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,
+			0xFFFFFFFF);
+		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,
+			0xFFFFFFFF);
+	}
+
+	/* Enable fault detection */
+	gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,
+		(1 << 30) | 0xFFFF);
+
+	/* Turn on performance counters */
+	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);
+
+	/* Increase VFD cache access so LRZ and other data gets evicted less */
+	gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);
+
+	/* Disable L2 bypass in the UCHE */
+	gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000);
+	gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF);
+	gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000);
+	gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF);
+
+	/* Set the GMEM VA range (0 to gpu->gmem) */
+	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);
+	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);
+	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,
+		0x00100000 + adreno_gpu->gmem - 1);
+	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);
+
+	gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
+	gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
+	gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
+	gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
+
+	gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, (0x400 << 11 | 0x300 << 22));
+
+	if (adreno_gpu->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)
+		gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
+
+	gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0xc0200100);
+
+	/* Enable USE_RETENTION_FLOPS */
+	gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);
+
+	/* Enable ME/PFP split notification */
+	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
+
+	/* Enable HWCG */
+	a5xx_enable_hwcg(gpu);
+
+	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);
+
+	/* Set the highest bank bit */
+	gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, 2 << 7);
+	gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, 2 << 1);
+
+	/* Protect registers from the CP */
+	gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);
+
+	/* RBBM */
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));
+
+	/* Content protect */
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(6),
+		ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
+			16));
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(7),
+		ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2));
+
+	/* CP */
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));
+
+	/* RB */
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));
+
+	/* VPC */
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 4));
+
+	/* UCHE */
+	gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));
+
+	if (adreno_is_a530(adreno_gpu))
+		gpu_write(gpu, REG_A5XX_CP_PROTECT(17),
+			ADRENO_PROTECT_RW(0x10000, 0x8000));
+
+	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);
+	/*
+	 * Disable the trusted memory range - we don't actually supported secure
+	 * memory rendering at this point in time and we don't want to block off
+	 * part of the virtual memory space.
+	 */
+	gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
+		REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
+	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
+
+	/* Load the GPMU firmware before starting the HW init */
+	a5xx_gpmu_ucode_init(gpu);
+
+	ret = adreno_hw_init(gpu);
+	if (ret)
+		return ret;
+
+	ret = a5xx_ucode_init(gpu);
+	if (ret)
+		return ret;
+
+	/* Disable the interrupts through the initial bringup stage */
+	gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);
+
+	/* Clear ME_HALT to start the micro engine */
+	gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);
+	ret = a5xx_me_init(gpu);
+	if (ret)
+		return ret;
+
+	ret = a5xx_power_init(gpu);
+	if (ret)
+		return ret;
+
+	/*
+	 * Send a pipeline event stat to get misbehaving counters to start
+	 * ticking correctly
+	 */
+	if (adreno_is_a530(adreno_gpu)) {
+		OUT_PKT7(gpu->rb, CP_EVENT_WRITE, 1);
+		OUT_RING(gpu->rb, 0x0F);
+
+		gpu->funcs->flush(gpu);
+		if (!gpu->funcs->idle(gpu))
+			return -EINVAL;
+	}
+
+	/* Put the GPU into unsecure mode */
+	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);
+
+	return 0;
+}
+
+static void a5xx_recover(struct msm_gpu *gpu)
+{
+	int i;
+
+	adreno_dump_info(gpu);
+
+	for (i = 0; i < 8; i++) {
+		printk("CP_SCRATCH_REG%d: %u\n", i,
+			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));
+	}
+
+	if (hang_debug)
+		a5xx_dump(gpu);
+
+	gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);
+	gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);
+	gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);
+	adreno_recover(gpu);
+}
+
+static void a5xx_destroy(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
+
+	DBG("%s", gpu->name);
+
+	if (a5xx_gpu->pm4_bo) {
+		if (a5xx_gpu->pm4_iova)
+			msm_gem_put_iova(a5xx_gpu->pm4_bo, gpu->id);
+		drm_gem_object_unreference_unlocked(a5xx_gpu->pm4_bo);
+	}
+
+	if (a5xx_gpu->pfp_bo) {
+		if (a5xx_gpu->pfp_iova)
+			msm_gem_put_iova(a5xx_gpu->pfp_bo, gpu->id);
+		drm_gem_object_unreference_unlocked(a5xx_gpu->pfp_bo);
+	}
+
+	if (a5xx_gpu->gpmu_bo) {
+		if (a5xx_gpu->gpmu_bo)
+			msm_gem_put_iova(a5xx_gpu->gpmu_bo, gpu->id);
+		drm_gem_object_unreference_unlocked(a5xx_gpu->gpmu_bo);
+	}
+
+	adreno_gpu_cleanup(adreno_gpu);
+	kfree(a5xx_gpu);
+}
+
+static inline bool _a5xx_check_idle(struct msm_gpu *gpu)
+{
+	if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)
+		return false;
+
+	/*
+	 * Nearly every abnormality ends up pausing the GPU and triggering a
+	 * fault so we can safely just watch for this one interrupt to fire
+	 */
+	return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &
+		A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT);
+}
+
+static bool a5xx_idle(struct msm_gpu *gpu)
+{
+	/* wait for CP to drain ringbuffer: */
+	if (!adreno_idle(gpu))
+		return false;
+
+	if (spin_until(_a5xx_check_idle(gpu))) {
+		DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X\n",
+			gpu->name, __builtin_return_address(0),
+			gpu_read(gpu, REG_A5XX_RBBM_STATUS),
+			gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS));
+
+		return false;
+	}
+
+	return true;
+}
+
+static void a5xx_cp_err_irq(struct msm_gpu *gpu)
+{
+	u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);
+
+	if (status & A5XX_CP_INT_CP_OPCODE_ERROR) {
+		u32 val;
+
+		gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);
+
+		/*
+		 * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so
+		 * read it twice
+		 */
+
+		gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
+		val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
+
+		dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",
+			val);
+	}
+
+	if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR)
+		dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",
+			gpu_read(gpu, REG_A5XX_CP_HW_FAULT));
+
+	if (status & A5XX_CP_INT_CP_DMA_ERROR)
+		dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");
+
+	if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
+		u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);
+
+		dev_err_ratelimited(gpu->dev->dev,
+			"CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
+			val & (1 << 24) ? "WRITE" : "READ",
+			(val & 0xFFFFF) >> 2, val);
+	}
+
+	if (status & A5XX_CP_INT_CP_AHB_ERROR) {
+		u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);
+		const char *access[16] = { "reserved", "reserved",
+			"timestamp lo", "timestamp hi", "pfp read", "pfp write",
+			"", "", "me read", "me write", "", "", "crashdump read",
+			"crashdump write" };
+
+		dev_err_ratelimited(gpu->dev->dev,
+			"CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n",
+			status & 0xFFFFF, access[(status >> 24) & 0xF],
+			(status & (1 << 31)), status);
+	}
+}
+
+static void a5xx_rbbm_err_irq(struct msm_gpu *gpu)
+{
+	u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
+
+	if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) {
+		u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);
+
+		dev_err_ratelimited(gpu->dev->dev,
+			"RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n",
+			val & (1 << 28) ? "WRITE" : "READ",
+			(val & 0xFFFFF) >> 2, (val >> 20) & 0x3,
+			(val >> 24) & 0xF);
+
+		/* Clear the error */
+		gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));
+	}
+
+	if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT)
+		dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");
+
+	if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT)
+		dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",
+			gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));
+
+	if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT)
+		dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",
+			gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));
+
+	if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT)
+		dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",
+			gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));
+
+	if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
+		dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");
+
+	if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
+		dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");
+}
+
+static void a5xx_uche_err_irq(struct msm_gpu *gpu)
+{
+	uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);
+
+	addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);
+
+	dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",
+		addr);
+}
+
+static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)
+{
+	dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");
+}
+
+#define RBBM_ERROR_MASK \
+	(A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
+	A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
+	A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
+	A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
+	A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
+	A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
+
+static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
+{
+	u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
+
+	gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD, status);
+
+	if (status & RBBM_ERROR_MASK)
+		a5xx_rbbm_err_irq(gpu);
+
+	if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR)
+		a5xx_cp_err_irq(gpu);
+
+	if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
+		a5xx_uche_err_irq(gpu);
+
+	if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
+		a5xx_gpmu_err_irq(gpu);
+
+	if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
+		msm_gpu_retire(gpu);
+
+	return IRQ_HANDLED;
+}
+
+static const u32 a5xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A5XX_CP_RB_BASE),
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A5XX_CP_RB_BASE_HI),
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, REG_A5XX_CP_RB_RPTR_ADDR),
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI,
+		REG_A5XX_CP_RB_RPTR_ADDR_HI),
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A5XX_CP_RB_RPTR),
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A5XX_CP_RB_WPTR),
+	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A5XX_CP_RB_CNTL),
+};
+
+static const u32 a5xx_registers[] = {
+	0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B,
+	0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095,
+	0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3,
+	0x04E0, 0x0533, 0x0540, 0x0555, 0xF400, 0xF400, 0xF800, 0xF807,
+	0x0800, 0x081A, 0x081F, 0x0841, 0x0860, 0x0860, 0x0880, 0x08A0,
+	0x0B00, 0x0B12, 0x0B15, 0x0B28, 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD,
+	0x0BC0, 0x0BC6, 0x0BD0, 0x0C53, 0x0C60, 0x0C61, 0x0C80, 0x0C82,
+	0x0C84, 0x0C85, 0x0C90, 0x0C98, 0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2,
+	0x2180, 0x2185, 0x2580, 0x2585, 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7,
+	0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8, 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8,
+	0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E, 0x2100, 0x211E, 0x2140, 0x2145,
+	0x2500, 0x251E, 0x2540, 0x2545, 0x0D10, 0x0D17, 0x0D20, 0x0D23,
+	0x0D30, 0x0D30, 0x20C0, 0x20C0, 0x24C0, 0x24C0, 0x0E40, 0x0E43,
+	0x0E4A, 0x0E4A, 0x0E50, 0x0E57, 0x0E60, 0x0E7C, 0x0E80, 0x0E8E,
+	0x0E90, 0x0E96, 0x0EA0, 0x0EA8, 0x0EB0, 0x0EB2, 0xE140, 0xE147,
+	0xE150, 0xE187, 0xE1A0, 0xE1A9, 0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7,
+	0xE1D0, 0xE1D1, 0xE200, 0xE201, 0xE210, 0xE21C, 0xE240, 0xE268,
+	0xE000, 0xE006, 0xE010, 0xE09A, 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB,
+	0xE100, 0xE105, 0xE380, 0xE38F, 0xE3B0, 0xE3B0, 0xE400, 0xE405,
+	0xE408, 0xE4E9, 0xE4F0, 0xE4F0, 0xE280, 0xE280, 0xE282, 0xE2A3,
+	0xE2A5, 0xE2C2, 0xE940, 0xE947, 0xE950, 0xE987, 0xE9A0, 0xE9A9,
+	0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7, 0xE9D0, 0xE9D1, 0xEA00, 0xEA01,
+	0xEA10, 0xEA1C, 0xEA40, 0xEA68, 0xE800, 0xE806, 0xE810, 0xE89A,
+	0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB, 0xE900, 0xE905, 0xEB80, 0xEB8F,
+	0xEBB0, 0xEBB0, 0xEC00, 0xEC05, 0xEC08, 0xECE9, 0xECF0, 0xECF0,
+	0xEA80, 0xEA80, 0xEA82, 0xEAA3, 0xEAA5, 0xEAC2, 0xA800, 0xA8FF,
+	0xAC60, 0xAC60, 0xB000, 0xB97F, 0xB9A0, 0xB9BF,
+	~0
+};
+
+static void a5xx_dump(struct msm_gpu *gpu)
+{
+	dev_info(gpu->dev->dev, "status:   %08x\n",
+		gpu_read(gpu, REG_A5XX_RBBM_STATUS));
+	adreno_dump(gpu);
+}
+
+static int a5xx_pm_resume(struct msm_gpu *gpu)
+{
+	int ret;
+
+	/* Turn on the core power */
+	ret = msm_gpu_pm_resume(gpu);
+	if (ret)
+		return ret;
+
+	/* Turn the RBCCU domain first to limit the chances of voltage droop */
+	gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);
+
+	/* Wait 3 usecs before polling */
+	udelay(3);
+
+	ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS,
+		(1 << 20), (1 << 20));
+	if (ret) {
+		DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n",
+			gpu->name,
+			gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS));
+		return ret;
+	}
+
+	/* Turn on the SP domain */
+	gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000);
+	ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS,
+		(1 << 20), (1 << 20));
+	if (ret)
+		DRM_ERROR("%s: timeout waiting for SP GDSC enable\n",
+			gpu->name);
+
+	return ret;
+}
+
+static int a5xx_pm_suspend(struct msm_gpu *gpu)
+{
+	/* Clear the VBIF pipe before shutting down */
+	gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0xF);
+	spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) & 0xF) == 0xF);
+
+	gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);
+
+	/*
+	 * Reset the VBIF before power collapse to avoid issue with FIFO
+	 * entries
+	 */
+	gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000);
+	gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000);
+
+	return msm_gpu_pm_suspend(gpu);
+}
+
+static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
+{
+	*value = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_CP_0_LO,
+		REG_A5XX_RBBM_PERFCTR_CP_0_HI);
+
+	return 0;
+}
+
+#ifdef CONFIG_DEBUG_FS
+static void a5xx_show(struct msm_gpu *gpu, struct seq_file *m)
+{
+	gpu->funcs->pm_resume(gpu);
+
+	seq_printf(m, "status:   %08x\n",
+			gpu_read(gpu, REG_A5XX_RBBM_STATUS));
+	gpu->funcs->pm_suspend(gpu);
+
+	adreno_show(gpu, m);
+}
+#endif
+
+static const struct adreno_gpu_funcs funcs = {
+	.base = {
+		.get_param = adreno_get_param,
+		.hw_init = a5xx_hw_init,
+		.pm_suspend = a5xx_pm_suspend,
+		.pm_resume = a5xx_pm_resume,
+		.recover = a5xx_recover,
+		.last_fence = adreno_last_fence,
+		.submit = a5xx_submit,
+		.flush = adreno_flush,
+		.idle = a5xx_idle,
+		.irq = a5xx_irq,
+		.destroy = a5xx_destroy,
+		.show = a5xx_show,
+	},
+	.get_timestamp = a5xx_get_timestamp,
+};
+
+struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
+{
+	struct msm_drm_private *priv = dev->dev_private;
+	struct platform_device *pdev = priv->gpu_pdev;
+	struct a5xx_gpu *a5xx_gpu = NULL;
+	struct adreno_gpu *adreno_gpu;
+	struct msm_gpu *gpu;
+	int ret;
+
+	if (!pdev) {
+		dev_err(dev->dev, "No A5XX device is defined\n");
+		return ERR_PTR(-ENXIO);
+	}
+
+	a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL);
+	if (!a5xx_gpu)
+		return ERR_PTR(-ENOMEM);
+
+	adreno_gpu = &a5xx_gpu->base;
+	gpu = &adreno_gpu->base;
+
+	a5xx_gpu->pdev = pdev;
+	adreno_gpu->registers = a5xx_registers;
+	adreno_gpu->reg_offsets = a5xx_register_offsets;
+
+	a5xx_gpu->lm_leakage = 0x4E001A;
+
+	ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs);
+	if (ret) {
+		a5xx_destroy(&(a5xx_gpu->base.base));
+		return ERR_PTR(ret);
+	}
+
+	return gpu;
+}
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.h b/drivers/gpu/drm/msm/adreno/a5xx_gpu.h
new file mode 100644
index 000000000000..1590f845d554
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.h
@@ -0,0 +1,60 @@
+/* Copyright (c) 2016 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+#ifndef __A5XX_GPU_H__
+#define __A5XX_GPU_H__
+
+#include "adreno_gpu.h"
+
+/* Bringing over the hack from the previous targets */
+#undef ROP_COPY
+#undef ROP_XOR
+
+#include "a5xx.xml.h"
+
+struct a5xx_gpu {
+	struct adreno_gpu base;
+	struct platform_device *pdev;
+
+	struct drm_gem_object *pm4_bo;
+	uint64_t pm4_iova;
+
+	struct drm_gem_object *pfp_bo;
+	uint64_t pfp_iova;
+
+	struct drm_gem_object *gpmu_bo;
+	uint64_t gpmu_iova;
+	uint32_t gpmu_dwords;
+
+	uint32_t lm_leakage;
+};
+
+#define to_a5xx_gpu(x) container_of(x, struct a5xx_gpu, base)
+
+int a5xx_power_init(struct msm_gpu *gpu);
+void a5xx_gpmu_ucode_init(struct msm_gpu *gpu);
+
+static inline int spin_usecs(struct msm_gpu *gpu, uint32_t usecs,
+		uint32_t reg, uint32_t mask, uint32_t value)
+{
+	while (usecs--) {
+		udelay(1);
+		if ((gpu_read(gpu, reg) & mask) == value)
+			return 0;
+		cpu_relax();
+	}
+
+	return -ETIMEDOUT;
+}
+
+
+#endif /* __A5XX_GPU_H__ */
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_power.c b/drivers/gpu/drm/msm/adreno/a5xx_power.c
new file mode 100644
index 000000000000..72d52c71f769
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a5xx_power.c
@@ -0,0 +1,344 @@
+/* Copyright (c) 2016 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/pm_opp.h>
+#include "a5xx_gpu.h"
+
+/*
+ * The GPMU data block is a block of shared registers that can be used to
+ * communicate back and forth. These "registers" are by convention with the GPMU
+ * firwmare and not bound to any specific hardware design
+ */
+
+#define AGC_INIT_BASE REG_A5XX_GPMU_DATA_RAM_BASE
+#define AGC_INIT_MSG_MAGIC (AGC_INIT_BASE + 5)
+#define AGC_MSG_BASE (AGC_INIT_BASE + 7)
+
+#define AGC_MSG_STATE (AGC_MSG_BASE + 0)
+#define AGC_MSG_COMMAND (AGC_MSG_BASE + 1)
+#define AGC_MSG_PAYLOAD_SIZE (AGC_MSG_BASE + 3)
+#define AGC_MSG_PAYLOAD(_o) ((AGC_MSG_BASE + 5) + (_o))
+
+#define AGC_POWER_CONFIG_PRODUCTION_ID 1
+#define AGC_INIT_MSG_VALUE 0xBABEFACE
+
+static struct {
+	uint32_t reg;
+	uint32_t value;
+} a5xx_sequence_regs[] = {
+	{ 0xB9A1, 0x00010303 },
+	{ 0xB9A2, 0x13000000 },
+	{ 0xB9A3, 0x00460020 },
+	{ 0xB9A4, 0x10000000 },
+	{ 0xB9A5, 0x040A1707 },
+	{ 0xB9A6, 0x00010000 },
+	{ 0xB9A7, 0x0E000904 },
+	{ 0xB9A8, 0x10000000 },
+	{ 0xB9A9, 0x01165000 },
+	{ 0xB9AA, 0x000E0002 },
+	{ 0xB9AB, 0x03884141 },
+	{ 0xB9AC, 0x10000840 },
+	{ 0xB9AD, 0x572A5000 },
+	{ 0xB9AE, 0x00000003 },
+	{ 0xB9AF, 0x00000000 },
+	{ 0xB9B0, 0x10000000 },
+	{ 0xB828, 0x6C204010 },
+	{ 0xB829, 0x6C204011 },
+	{ 0xB82A, 0x6C204012 },
+	{ 0xB82B, 0x6C204013 },
+	{ 0xB82C, 0x6C204014 },
+	{ 0xB90F, 0x00000004 },
+	{ 0xB910, 0x00000002 },
+	{ 0xB911, 0x00000002 },
+	{ 0xB912, 0x00000002 },
+	{ 0xB913, 0x00000002 },
+	{ 0xB92F, 0x00000004 },
+	{ 0xB930, 0x00000005 },
+	{ 0xB931, 0x00000005 },
+	{ 0xB932, 0x00000005 },
+	{ 0xB933, 0x00000005 },
+	{ 0xB96F, 0x00000001 },
+	{ 0xB970, 0x00000003 },
+	{ 0xB94F, 0x00000004 },
+	{ 0xB950, 0x0000000B },
+	{ 0xB951, 0x0000000B },
+	{ 0xB952, 0x0000000B },
+	{ 0xB953, 0x0000000B },
+	{ 0xB907, 0x00000019 },
+	{ 0xB927, 0x00000019 },
+	{ 0xB947, 0x00000019 },
+	{ 0xB967, 0x00000019 },
+	{ 0xB987, 0x00000019 },
+	{ 0xB906, 0x00220001 },
+	{ 0xB926, 0x00220001 },
+	{ 0xB946, 0x00220001 },
+	{ 0xB966, 0x00220001 },
+	{ 0xB986, 0x00300000 },
+	{ 0xAC40, 0x0340FF41 },
+	{ 0xAC41, 0x03BEFED0 },
+	{ 0xAC42, 0x00331FED },
+	{ 0xAC43, 0x021FFDD3 },
+	{ 0xAC44, 0x5555AAAA },
+	{ 0xAC45, 0x5555AAAA },
+	{ 0xB9BA, 0x00000008 },
+};
+
+/*
+ * Get the actual voltage value for the operating point at the specified
+ * frequency
+ */
+static inline uint32_t _get_mvolts(struct msm_gpu *gpu, uint32_t freq)
+{
+	struct drm_device *dev = gpu->dev;
+	struct msm_drm_private *priv = dev->dev_private;
+	struct platform_device *pdev = priv->gpu_pdev;
+	struct dev_pm_opp *opp;
+
+	opp = dev_pm_opp_find_freq_exact(&pdev->dev, freq, true);
+
+	return (!IS_ERR(opp)) ? dev_pm_opp_get_voltage(opp) / 1000 : 0;
+}
+
+/* Setup thermal limit management */
+static void a5xx_lm_setup(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
+	unsigned int i;
+
+	/* Write the block of sequence registers */
+	for (i = 0; i < ARRAY_SIZE(a5xx_sequence_regs); i++)
+		gpu_write(gpu, a5xx_sequence_regs[i].reg,
+			a5xx_sequence_regs[i].value);
+
+	/* Hard code the A530 GPU thermal sensor ID for the GPMU */
+	gpu_write(gpu, REG_A5XX_GPMU_TEMP_SENSOR_ID, 0x60007);
+	gpu_write(gpu, REG_A5XX_GPMU_DELTA_TEMP_THRESHOLD, 0x01);
+	gpu_write(gpu, REG_A5XX_GPMU_TEMP_SENSOR_CONFIG, 0x01);
+
+	/* Until we get clock scaling 0 is always the active power level */
+	gpu_write(gpu, REG_A5XX_GPMU_GPMU_VOLTAGE, 0x80000000 | 0);
+
+	gpu_write(gpu, REG_A5XX_GPMU_BASE_LEAKAGE, a5xx_gpu->lm_leakage);
+
+	/* The threshold is fixed at 6000 for A530 */
+	gpu_write(gpu, REG_A5XX_GPMU_GPMU_PWR_THRESHOLD, 0x80000000 | 6000);
+
+	gpu_write(gpu, REG_A5XX_GPMU_BEC_ENABLE, 0x10001FFF);
+	gpu_write(gpu, REG_A5XX_GDPM_CONFIG1, 0x00201FF1);
+
+	/* Write the voltage table */
+	gpu_write(gpu, REG_A5XX_GPMU_BEC_ENABLE, 0x10001FFF);
+	gpu_write(gpu, REG_A5XX_GDPM_CONFIG1, 0x201FF1);
+
+	gpu_write(gpu, AGC_MSG_STATE, 1);
+	gpu_write(gpu, AGC_MSG_COMMAND, AGC_POWER_CONFIG_PRODUCTION_ID);
+
+	/* Write the max power - hard coded to 5448 for A530 */
+	gpu_write(gpu, AGC_MSG_PAYLOAD(0), 5448);
+	gpu_write(gpu, AGC_MSG_PAYLOAD(1), 1);
+
+	/*
+	 * For now just write the one voltage level - we will do more when we
+	 * can do scaling
+	 */
+	gpu_write(gpu, AGC_MSG_PAYLOAD(2), _get_mvolts(gpu, gpu->fast_rate));
+	gpu_write(gpu, AGC_MSG_PAYLOAD(3), gpu->fast_rate / 1000000);
+
+	gpu_write(gpu, AGC_MSG_PAYLOAD_SIZE, 4 * sizeof(uint32_t));
+	gpu_write(gpu, AGC_INIT_MSG_MAGIC, AGC_INIT_MSG_VALUE);
+}
+
+/* Enable SP/TP cpower collapse */
+static void a5xx_pc_init(struct msm_gpu *gpu)
+{
+	gpu_write(gpu, REG_A5XX_GPMU_PWR_COL_INTER_FRAME_CTRL, 0x7F);
+	gpu_write(gpu, REG_A5XX_GPMU_PWR_COL_BINNING_CTRL, 0);
+	gpu_write(gpu, REG_A5XX_GPMU_PWR_COL_INTER_FRAME_HYST, 0xA0080);
+	gpu_write(gpu, REG_A5XX_GPMU_PWR_COL_STAGGER_DELAY, 0x600040);
+}
+
+/* Enable the GPMU microcontroller */
+static int a5xx_gpmu_init(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
+	struct msm_ringbuffer *ring = gpu->rb;
+
+	if (!a5xx_gpu->gpmu_dwords)
+		return 0;
+
+	/* Turn off protected mode for this operation */
+	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
+	OUT_RING(ring, 0);
+
+	/* Kick off the IB to load the GPMU microcode */
+	OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
+	OUT_RING(ring, lower_32_bits(a5xx_gpu->gpmu_iova));
+	OUT_RING(ring, upper_32_bits(a5xx_gpu->gpmu_iova));
+	OUT_RING(ring, a5xx_gpu->gpmu_dwords);
+
+	/* Turn back on protected mode */
+	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
+	OUT_RING(ring, 1);
+
+	gpu->funcs->flush(gpu);
+
+	if (!gpu->funcs->idle(gpu)) {
+		DRM_ERROR("%s: Unable to load GPMU firmware. GPMU will not be active\n",
+			gpu->name);
+		return -EINVAL;
+	}
+
+	gpu_write(gpu, REG_A5XX_GPMU_WFI_CONFIG, 0x4014);
+
+	/* Kick off the GPMU */
+	gpu_write(gpu, REG_A5XX_GPMU_CM3_SYSRESET, 0x0);
+
+	/*
+	 * Wait for the GPMU to respond. It isn't fatal if it doesn't, we just
+	 * won't have advanced power collapse.
+	 */
+	if (spin_usecs(gpu, 25, REG_A5XX_GPMU_GENERAL_0, 0xFFFFFFFF,
+		0xBABEFACE))
+		DRM_ERROR("%s: GPMU firmware initialization timed out\n",
+			gpu->name);
+
+	return 0;
+}
+
+/* Enable limits management */
+static void a5xx_lm_enable(struct msm_gpu *gpu)
+{
+	gpu_write(gpu, REG_A5XX_GDPM_INT_MASK, 0x0);
+	gpu_write(gpu, REG_A5XX_GDPM_INT_EN, 0x0A);
+	gpu_write(gpu, REG_A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK, 0x01);
+	gpu_write(gpu, REG_A5XX_GPMU_TEMP_THRESHOLD_INTR_EN_MASK, 0x50000);
+	gpu_write(gpu, REG_A5XX_GPMU_THROTTLE_UNMASK_FORCE_CTRL, 0x30000);
+
+	gpu_write(gpu, REG_A5XX_GPMU_CLOCK_THROTTLE_CTRL, 0x011);
+}
+
+int a5xx_power_init(struct msm_gpu *gpu)
+{
+	int ret;
+
+	/* Set up the limits management */
+	a5xx_lm_setup(gpu);
+
+	/* Set up SP/TP power collpase */
+	a5xx_pc_init(gpu);
+
+	/* Start the GPMU */
+	ret = a5xx_gpmu_init(gpu);
+	if (ret)
+		return ret;
+
+	/* Start the limits management */
+	a5xx_lm_enable(gpu);
+
+	return 0;
+}
+
+void a5xx_gpmu_ucode_init(struct msm_gpu *gpu)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
+	struct drm_device *drm = gpu->dev;
+	const struct firmware *fw;
+	uint32_t dwords = 0, offset = 0, bosize;
+	unsigned int *data, *ptr, *cmds;
+	unsigned int cmds_size;
+
+	if (a5xx_gpu->gpmu_bo)
+		return;
+
+	/* Get the firmware */
+	if (request_firmware(&fw, adreno_gpu->info->gpmufw, drm->dev)) {
+		DRM_ERROR("%s: Could not get GPMU firmware. GPMU will not be active\n",
+			gpu->name);
+		return;
+	}
+
+	data = (unsigned int *) fw->data;
+
+	/*
+	 * The first dword is the size of the remaining data in dwords. Use it
+	 * as a checksum of sorts and make sure it matches the actual size of
+	 * the firmware that we read
+	 */
+
+	if (fw->size < 8 || (data[0] < 2) || (data[0] >= (fw->size >> 2)))
+		goto out;
+
+	/* The second dword is an ID - look for 2 (GPMU_FIRMWARE_ID) */
+	if (data[1] != 2)
+		goto out;
+
+	cmds = data + data[2] + 3;
+	cmds_size = data[0] - data[2] - 2;
+
+	/*
+	 * A single type4 opcode can only have so many values attached so
+	 * add enough opcodes to load the all the commands
+	 */
+	bosize = (cmds_size + (cmds_size / TYPE4_MAX_PAYLOAD) + 1) << 2;
+
+	mutex_lock(&drm->struct_mutex);
+	a5xx_gpu->gpmu_bo = msm_gem_new(drm, bosize, MSM_BO_UNCACHED);
+	mutex_unlock(&drm->struct_mutex);
+
+	if (IS_ERR(a5xx_gpu->gpmu_bo))
+		goto err;
+
+	if (msm_gem_get_iova(a5xx_gpu->gpmu_bo, gpu->id, &a5xx_gpu->gpmu_iova))
+		goto err;
+
+	ptr = msm_gem_get_vaddr(a5xx_gpu->gpmu_bo);
+	if (!ptr)
+		goto err;
+
+	while (cmds_size > 0) {
+		int i;
+		uint32_t _size = cmds_size > TYPE4_MAX_PAYLOAD ?
+			TYPE4_MAX_PAYLOAD : cmds_size;
+
+		ptr[dwords++] = PKT4(REG_A5XX_GPMU_INST_RAM_BASE + offset,
+			_size);
+
+		for (i = 0; i < _size; i++)
+			ptr[dwords++] = *cmds++;
+
+		offset += _size;
+		cmds_size -= _size;
+	}
+
+	msm_gem_put_vaddr(a5xx_gpu->gpmu_bo);
+	a5xx_gpu->gpmu_dwords = dwords;
+
+	goto out;
+
+err:
+	if (a5xx_gpu->gpmu_iova)
+		msm_gem_put_iova(a5xx_gpu->gpmu_bo, gpu->id);
+	if (a5xx_gpu->gpmu_bo)
+		drm_gem_object_unreference_unlocked(a5xx_gpu->gpmu_bo);
+
+	a5xx_gpu->gpmu_bo = NULL;
+	a5xx_gpu->gpmu_iova = 0;
+	a5xx_gpu->gpmu_dwords = 0;
+
+out:
+	/* No need to keep that firmware laying around anymore */
+	release_firmware(fw);
+}
diff --git a/drivers/gpu/drm/msm/adreno/adreno_common.xml.h b/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
index e81481d1b7df..4a33ba6f1244 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
@@ -8,13 +8,14 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    398 bytes, from 2015-09-24 17:25:31)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    431 bytes, from 2016-04-26 17:56:44)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2016-02-10 17:07:21)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  32901 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  11518 bytes, from 2016-02-10 21:03:25)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  16166 bytes, from 2016-02-11 21:20:31)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83967 bytes, from 2016-02-10 17:07:21)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 109916 bytes, from 2016-02-20 18:44:48)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  32907 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  12025 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  22544 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 110765 bytes, from 2016-11-26 23:01:48)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml          (  90321 bytes, from 2016-11-28 16:50:05)
 - /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2015-09-24 17:30:00)
 
 Copyright (C) 2013-2016 by the following authors:
@@ -172,6 +173,14 @@ enum a3xx_color_swap {
 	XYZW = 3,
 };
 
+enum a3xx_rb_blend_opcode {
+	BLEND_DST_PLUS_SRC = 0,
+	BLEND_SRC_MINUS_DST = 1,
+	BLEND_DST_MINUS_SRC = 2,
+	BLEND_MIN_DST_SRC = 3,
+	BLEND_MAX_DST_SRC = 4,
+};
+
 #define REG_AXXX_CP_RB_BASE					0x000001c0
 
 #define REG_AXXX_CP_RB_CNTL					0x000001c1
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index 5127b75dbf40..893eb2b2531b 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -25,9 +25,6 @@ bool hang_debug = false;
 MODULE_PARM_DESC(hang_debug, "Dump registers when hang is detected (can be slow!)");
 module_param_named(hang_debug, hang_debug, bool, 0600);
 
-struct msm_gpu *a3xx_gpu_init(struct drm_device *dev);
-struct msm_gpu *a4xx_gpu_init(struct drm_device *dev);
-
 static const struct adreno_info gpulist[] = {
 	{
 		.rev   = ADRENO_REV(3, 0, 5, ANY_ID),
@@ -77,6 +74,15 @@ static const struct adreno_info gpulist[] = {
 		.pfpfw = "a420_pfp.fw",
 		.gmem  = (SZ_1M + SZ_512K),
 		.init  = a4xx_gpu_init,
+	}, {
+		.rev = ADRENO_REV(5, 3, 0, ANY_ID),
+		.revn = 530,
+		.name = "A530",
+		.pm4fw = "a530_pm4.fw",
+		.pfpfw = "a530_pfp.fw",
+		.gmem = SZ_1M,
+		.init = a5xx_gpu_init,
+		.gpmufw = "a530v3_gpmu.fw2",
 	},
 };
 
@@ -86,6 +92,8 @@ MODULE_FIRMWARE("a330_pm4.fw");
 MODULE_FIRMWARE("a330_pfp.fw");
 MODULE_FIRMWARE("a420_pm4.fw");
 MODULE_FIRMWARE("a420_pfp.fw");
+MODULE_FIRMWARE("a530_fm4.fw");
+MODULE_FIRMWARE("a530_pfp.fw");
 
 static inline bool _rev_match(uint8_t entry, uint8_t id)
 {
@@ -148,12 +156,16 @@ struct msm_gpu *adreno_load_gpu(struct drm_device *dev)
 		mutex_lock(&dev->struct_mutex);
 		gpu->funcs->pm_resume(gpu);
 		mutex_unlock(&dev->struct_mutex);
+
+		disable_irq(gpu->irq);
+
 		ret = gpu->funcs->hw_init(gpu);
 		if (ret) {
 			dev_err(dev->dev, "gpu hw init failed: %d\n", ret);
 			gpu->funcs->destroy(gpu);
 			gpu = NULL;
 		} else {
+			enable_irq(gpu->irq);
 			/* give inactive pm a chance to kick in: */
 			msm_gpu_retire(gpu);
 		}
@@ -169,12 +181,20 @@ static void set_gpu_pdev(struct drm_device *dev,
 	priv->gpu_pdev = pdev;
 }
 
+static const struct {
+	const char *str;
+	uint32_t flag;
+} quirks[] = {
+	{ "qcom,gpu-quirk-two-pass-use-wfi", ADRENO_QUIRK_TWO_PASS_USE_WFI },
+	{ "qcom,gpu-quirk-fault-detect-mask", ADRENO_QUIRK_FAULT_DETECT_MASK },
+};
+
 static int adreno_bind(struct device *dev, struct device *master, void *data)
 {
 	static struct adreno_platform_config config = {};
 	struct device_node *child, *node = dev->of_node;
 	u32 val;
-	int ret;
+	int ret, i;
 
 	ret = of_property_read_u32(node, "qcom,chipid", &val);
 	if (ret) {
@@ -208,6 +228,10 @@ static int adreno_bind(struct device *dev, struct device *master, void *data)
 		return -ENXIO;
 	}
 
+	for (i = 0; i < ARRAY_SIZE(quirks); i++)
+		if (of_property_read_bool(node, quirks[i].str))
+			config.quirks |= quirks[i].flag;
+
 	dev->platform_data = &config;
 	set_gpu_pdev(dev_get_drvdata(master), to_platform_device(dev));
 	return 0;
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index f386f463278d..a18126150e11 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -22,7 +22,7 @@
 #include "msm_mmu.h"
 
 #define RB_SIZE    SZ_32K
-#define RB_BLKSIZE 16
+#define RB_BLKSIZE 32
 
 int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value)
 {
@@ -54,9 +54,6 @@ int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value)
 	}
 }
 
-#define rbmemptr(adreno_gpu, member)  \
-	((adreno_gpu)->memptrs_iova + offsetof(struct adreno_rbmemptrs, member))
-
 int adreno_hw_init(struct msm_gpu *gpu)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
@@ -79,11 +76,14 @@ int adreno_hw_init(struct msm_gpu *gpu)
 			(adreno_is_a430(adreno_gpu) ? AXXX_CP_RB_CNTL_NO_UPDATE : 0));
 
 	/* Setup ringbuffer address: */
-	adreno_gpu_write(adreno_gpu, REG_ADRENO_CP_RB_BASE, gpu->rb_iova);
+	adreno_gpu_write64(adreno_gpu, REG_ADRENO_CP_RB_BASE,
+		REG_ADRENO_CP_RB_BASE_HI, gpu->rb_iova);
 
-	if (!adreno_is_a430(adreno_gpu))
-		adreno_gpu_write(adreno_gpu, REG_ADRENO_CP_RB_RPTR_ADDR,
-						rbmemptr(adreno_gpu, rptr));
+	if (!adreno_is_a430(adreno_gpu)) {
+		adreno_gpu_write64(adreno_gpu, REG_ADRENO_CP_RB_RPTR_ADDR,
+			REG_ADRENO_CP_RB_RPTR_ADDR_HI,
+			rbmemptr(adreno_gpu, rptr));
+	}
 
 	return 0;
 }
@@ -126,11 +126,14 @@ void adreno_recover(struct msm_gpu *gpu)
 	adreno_gpu->memptrs->wptr  = 0;
 
 	gpu->funcs->pm_resume(gpu);
+
+	disable_irq(gpu->irq);
 	ret = gpu->funcs->hw_init(gpu);
 	if (ret) {
 		dev_err(dev->dev, "gpu hw init failed: %d\n", ret);
 		/* hmm, oh well? */
 	}
+	enable_irq(gpu->irq);
 }
 
 void adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
@@ -218,19 +221,18 @@ void adreno_flush(struct msm_gpu *gpu)
 	adreno_gpu_write(adreno_gpu, REG_ADRENO_CP_RB_WPTR, wptr);
 }
 
-void adreno_idle(struct msm_gpu *gpu)
+bool adreno_idle(struct msm_gpu *gpu)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 	uint32_t wptr = get_wptr(gpu->rb);
-	int ret;
 
 	/* wait for CP to drain ringbuffer: */
-	ret = spin_until(get_rptr(adreno_gpu) == wptr);
-
-	if (ret)
-		DRM_ERROR("%s: timeout waiting to drain ringbuffer!\n", gpu->name);
+	if (!spin_until(get_rptr(adreno_gpu) == wptr))
+		return true;
 
 	/* TODO maybe we need to reset GPU here to recover from hang? */
+	DRM_ERROR("%s: timeout waiting to drain ringbuffer!\n", gpu->name);
+	return false;
 }
 
 #ifdef CONFIG_DEBUG_FS
@@ -278,7 +280,6 @@ void adreno_show(struct msm_gpu *gpu, struct seq_file *m)
 void adreno_dump_info(struct msm_gpu *gpu)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
-	int i;
 
 	printk("revision: %d (%d.%d.%d.%d)\n",
 			adreno_gpu->info->revn, adreno_gpu->rev.core,
@@ -290,11 +291,6 @@ void adreno_dump_info(struct msm_gpu *gpu)
 	printk("rptr:     %d\n", get_rptr(adreno_gpu));
 	printk("wptr:     %d\n", adreno_gpu->memptrs->wptr);
 	printk("rb wptr:  %d\n", get_wptr(gpu->rb));
-
-	for (i = 0; i < 8; i++) {
-		printk("CP_SCRATCH_REG%d: %u\n", i,
-			gpu_read(gpu, REG_AXXX_CP_SCRATCH_REG0 + i));
-	}
 }
 
 /* would be nice to not have to duplicate the _show() stuff with printk(): */
@@ -350,6 +346,7 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 	adreno_gpu->gmem = adreno_gpu->info->gmem;
 	adreno_gpu->revn = adreno_gpu->info->revn;
 	adreno_gpu->rev = config->rev;
+	adreno_gpu->quirks = config->quirks;
 
 	gpu->fast_rate = config->fast_rate;
 	gpu->slow_rate = config->slow_rate;
@@ -381,7 +378,7 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 		return ret;
 	}
 
-	mmu = gpu->mmu;
+	mmu = gpu->aspace->mmu;
 	if (mmu) {
 		ret = mmu->funcs->attach(mmu, iommu_ports,
 				ARRAY_SIZE(iommu_ports));
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
index a54f6e036b4a..e8d55b0306ed 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
@@ -28,6 +28,9 @@
 #include "adreno_pm4.xml.h"
 
 #define REG_ADRENO_DEFINE(_offset, _reg) [_offset] = (_reg) + 1
+#define REG_SKIP ~0
+#define REG_ADRENO_SKIP(_offset) [_offset] = REG_SKIP
+
 /**
  * adreno_regs: List of registers that are used in across all
  * 3D devices. Each device type has different offset value for the same
@@ -35,73 +38,21 @@
  * and are indexed by the enumeration values defined in this enum
  */
 enum adreno_regs {
-	REG_ADRENO_CP_DEBUG,
-	REG_ADRENO_CP_ME_RAM_WADDR,
-	REG_ADRENO_CP_ME_RAM_DATA,
-	REG_ADRENO_CP_PFP_UCODE_DATA,
-	REG_ADRENO_CP_PFP_UCODE_ADDR,
-	REG_ADRENO_CP_WFI_PEND_CTR,
 	REG_ADRENO_CP_RB_BASE,
+	REG_ADRENO_CP_RB_BASE_HI,
 	REG_ADRENO_CP_RB_RPTR_ADDR,
+	REG_ADRENO_CP_RB_RPTR_ADDR_HI,
 	REG_ADRENO_CP_RB_RPTR,
 	REG_ADRENO_CP_RB_WPTR,
-	REG_ADRENO_CP_PROTECT_CTRL,
-	REG_ADRENO_CP_ME_CNTL,
 	REG_ADRENO_CP_RB_CNTL,
-	REG_ADRENO_CP_IB1_BASE,
-	REG_ADRENO_CP_IB1_BUFSZ,
-	REG_ADRENO_CP_IB2_BASE,
-	REG_ADRENO_CP_IB2_BUFSZ,
-	REG_ADRENO_CP_TIMESTAMP,
-	REG_ADRENO_CP_ME_RAM_RADDR,
-	REG_ADRENO_CP_ROQ_ADDR,
-	REG_ADRENO_CP_ROQ_DATA,
-	REG_ADRENO_CP_MERCIU_ADDR,
-	REG_ADRENO_CP_MERCIU_DATA,
-	REG_ADRENO_CP_MERCIU_DATA2,
-	REG_ADRENO_CP_MEQ_ADDR,
-	REG_ADRENO_CP_MEQ_DATA,
-	REG_ADRENO_CP_HW_FAULT,
-	REG_ADRENO_CP_PROTECT_STATUS,
-	REG_ADRENO_SCRATCH_ADDR,
-	REG_ADRENO_SCRATCH_UMSK,
-	REG_ADRENO_SCRATCH_REG2,
-	REG_ADRENO_RBBM_STATUS,
-	REG_ADRENO_RBBM_PERFCTR_CTL,
-	REG_ADRENO_RBBM_PERFCTR_LOAD_CMD0,
-	REG_ADRENO_RBBM_PERFCTR_LOAD_CMD1,
-	REG_ADRENO_RBBM_PERFCTR_LOAD_CMD2,
-	REG_ADRENO_RBBM_PERFCTR_PWR_1_LO,
-	REG_ADRENO_RBBM_INT_0_MASK,
-	REG_ADRENO_RBBM_INT_0_STATUS,
-	REG_ADRENO_RBBM_AHB_ERROR_STATUS,
-	REG_ADRENO_RBBM_PM_OVERRIDE2,
-	REG_ADRENO_RBBM_AHB_CMD,
-	REG_ADRENO_RBBM_INT_CLEAR_CMD,
-	REG_ADRENO_RBBM_SW_RESET_CMD,
-	REG_ADRENO_RBBM_CLOCK_CTL,
-	REG_ADRENO_RBBM_AHB_ME_SPLIT_STATUS,
-	REG_ADRENO_RBBM_AHB_PFP_SPLIT_STATUS,
-	REG_ADRENO_VPC_DEBUG_RAM_SEL,
-	REG_ADRENO_VPC_DEBUG_RAM_READ,
-	REG_ADRENO_VSC_SIZE_ADDRESS,
-	REG_ADRENO_VFD_CONTROL_0,
-	REG_ADRENO_VFD_INDEX_MAX,
-	REG_ADRENO_SP_VS_PVT_MEM_ADDR_REG,
-	REG_ADRENO_SP_FS_PVT_MEM_ADDR_REG,
-	REG_ADRENO_SP_VS_OBJ_START_REG,
-	REG_ADRENO_SP_FS_OBJ_START_REG,
-	REG_ADRENO_PA_SC_AA_CONFIG,
-	REG_ADRENO_SQ_GPR_MANAGEMENT,
-	REG_ADRENO_SQ_INST_STORE_MANAGMENT,
-	REG_ADRENO_TP0_CHICKEN,
-	REG_ADRENO_RBBM_RBBM_CTL,
-	REG_ADRENO_UCHE_INVALIDATE0,
-	REG_ADRENO_RBBM_PERFCTR_LOAD_VALUE_LO,
-	REG_ADRENO_RBBM_PERFCTR_LOAD_VALUE_HI,
 	REG_ADRENO_REGISTER_MAX,
 };
 
+enum adreno_quirks {
+	ADRENO_QUIRK_TWO_PASS_USE_WFI = 1,
+	ADRENO_QUIRK_FAULT_DETECT_MASK = 2,
+};
+
 struct adreno_rev {
 	uint8_t  core;
 	uint8_t  major;
@@ -122,12 +73,16 @@ struct adreno_info {
 	uint32_t revn;
 	const char *name;
 	const char *pm4fw, *pfpfw;
+	const char *gpmufw;
 	uint32_t gmem;
 	struct msm_gpu *(*init)(struct drm_device *dev);
 };
 
 const struct adreno_info *adreno_info(struct adreno_rev rev);
 
+#define rbmemptr(adreno_gpu, member)  \
+	((adreno_gpu)->memptrs_iova + offsetof(struct adreno_rbmemptrs, member))
+
 struct adreno_rbmemptrs {
 	volatile uint32_t rptr;
 	volatile uint32_t wptr;
@@ -153,7 +108,7 @@ struct adreno_gpu {
 	// different for z180..
 	struct adreno_rbmemptrs *memptrs;
 	struct drm_gem_object *memptrs_bo;
-	uint32_t memptrs_iova;
+	uint64_t memptrs_iova;
 
 	/*
 	 * Register offsets are different between some GPUs.
@@ -161,6 +116,8 @@ struct adreno_gpu {
 	 * code (a3xx_gpu.c) and stored in this common location.
 	 */
 	const unsigned int *reg_offsets;
+
+	uint32_t quirks;
 };
 #define to_adreno_gpu(x) container_of(x, struct adreno_gpu, base)
 
@@ -171,6 +128,7 @@ struct adreno_platform_config {
 #ifdef DOWNSTREAM_CONFIG_MSM_BUS_SCALING
 	struct msm_bus_scale_pdata *bus_scale_table;
 #endif
+	uint32_t quirks;
 };
 
 #define ADRENO_IDLE_TIMEOUT msecs_to_jiffies(1000)
@@ -234,6 +192,11 @@ static inline int adreno_is_a430(struct adreno_gpu *gpu)
        return gpu->revn == 430;
 }
 
+static inline int adreno_is_a530(struct adreno_gpu *gpu)
+{
+	return gpu->revn == 530;
+}
+
 int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value);
 int adreno_hw_init(struct msm_gpu *gpu);
 uint32_t adreno_last_fence(struct msm_gpu *gpu);
@@ -241,7 +204,7 @@ void adreno_recover(struct msm_gpu *gpu);
 void adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 		struct msm_file_private *ctx);
 void adreno_flush(struct msm_gpu *gpu);
-void adreno_idle(struct msm_gpu *gpu);
+bool adreno_idle(struct msm_gpu *gpu);
 #ifdef CONFIG_DEBUG_FS
 void adreno_show(struct msm_gpu *gpu, struct seq_file *m);
 #endif
@@ -278,8 +241,38 @@ OUT_PKT3(struct msm_ringbuffer *ring, uint8_t opcode, uint16_t cnt)
 	OUT_RING(ring, CP_TYPE3_PKT | ((cnt-1) << 16) | ((opcode & 0xFF) << 8));
 }
 
+static inline u32 PM4_PARITY(u32 val)
+{
+	return (0x9669 >> (0xF & (val ^
+		(val >> 4) ^ (val >> 8) ^ (val >> 12) ^
+		(val >> 16) ^ ((val) >> 20) ^ (val >> 24) ^
+		(val >> 28)))) & 1;
+}
+
+/* Maximum number of values that can be executed for one opcode */
+#define TYPE4_MAX_PAYLOAD 127
+
+#define PKT4(_reg, _cnt) \
+	(CP_TYPE4_PKT | ((_cnt) << 0) | (PM4_PARITY((_cnt)) << 7) | \
+	 (((_reg) & 0x3FFFF) << 8) | (PM4_PARITY((_reg)) << 27))
+
+static inline void
+OUT_PKT4(struct msm_ringbuffer *ring, uint16_t regindx, uint16_t cnt)
+{
+	adreno_wait_ring(ring->gpu, cnt + 1);
+	OUT_RING(ring, PKT4(regindx, cnt));
+}
+
+static inline void
+OUT_PKT7(struct msm_ringbuffer *ring, uint8_t opcode, uint16_t cnt)
+{
+	adreno_wait_ring(ring->gpu, cnt + 1);
+	OUT_RING(ring, CP_TYPE7_PKT | (cnt << 0) | (PM4_PARITY(cnt) << 15) |
+		((opcode & 0x7F) << 16) | (PM4_PARITY(opcode) << 23));
+}
+
 /*
- * adreno_checkreg_off() - Checks the validity of a register enum
+ * adreno_reg_check() - Checks the validity of a register enum
  * @gpu:		Pointer to struct adreno_gpu
  * @offset_name:	The register enum that is checked
  */
@@ -290,6 +283,16 @@ static inline bool adreno_reg_check(struct adreno_gpu *gpu,
 			!gpu->reg_offsets[offset_name]) {
 		BUG();
 	}
+
+	/*
+	 * REG_SKIP is a special value that tell us that the register in
+	 * question isn't implemented on target but don't trigger a BUG(). This
+	 * is used to cleanly implement adreno_gpu_write64() and
+	 * adreno_gpu_read64() in a generic fashion
+	 */
+	if (gpu->reg_offsets[offset_name] == REG_SKIP)
+		return false;
+
 	return true;
 }
 
@@ -311,4 +314,39 @@ static inline void adreno_gpu_write(struct adreno_gpu *gpu,
 		gpu_write(&gpu->base, reg - 1, data);
 }
 
+struct msm_gpu *a3xx_gpu_init(struct drm_device *dev);
+struct msm_gpu *a4xx_gpu_init(struct drm_device *dev);
+struct msm_gpu *a5xx_gpu_init(struct drm_device *dev);
+
+static inline void adreno_gpu_write64(struct adreno_gpu *gpu,
+		enum adreno_regs lo, enum adreno_regs hi, u64 data)
+{
+	adreno_gpu_write(gpu, lo, lower_32_bits(data));
+	adreno_gpu_write(gpu, hi, upper_32_bits(data));
+}
+
+/*
+ * Given a register and a count, return a value to program into
+ * REG_CP_PROTECT_REG(n) - this will block both reads and writes for _len
+ * registers starting at _reg.
+ *
+ * The register base needs to be a multiple of the length. If it is not, the
+ * hardware will quietly mask off the bits for you and shift the size. For
+ * example, if you intend the protection to start at 0x07 for a length of 4
+ * (0x07-0x0A) the hardware will actually protect (0x04-0x07) which might
+ * expose registers you intended to protect!
+ */
+#define ADRENO_PROTECT_RW(_reg, _len) \
+	((1 << 30) | (1 << 29) | \
+	((ilog2((_len)) & 0x1F) << 24) | (((_reg) << 2) & 0xFFFFF))
+
+/*
+ * Same as above, but allow reads over the range. For areas of mixed use (such
+ * as performance counters) this allows us to protect a much larger range with a
+ * single register
+ */
+#define ADRENO_PROTECT_RDONLY(_reg, _len) \
+	((1 << 29) \
+	((ilog2((_len)) & 0x1F) << 24) | (((_reg) << 2) & 0xFFFFF))
+
 #endif /* __ADRENO_GPU_H__ */
diff --git a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
index d7477ff867c9..6a2930e75503 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
@@ -8,13 +8,14 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    398 bytes, from 2015-09-24 17:25:31)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml               (    431 bytes, from 2016-04-26 17:56:44)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml  (   1572 bytes, from 2016-02-10 17:07:21)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  32901 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  11518 bytes, from 2016-02-10 21:03:25)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  16166 bytes, from 2016-02-11 21:20:31)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83967 bytes, from 2016-02-10 17:07:21)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 109916 bytes, from 2016-02-20 18:44:48)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml          (  32907 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml (  12025 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml    (  22544 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml          (  83840 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml          ( 110765 bytes, from 2016-11-26 23:01:48)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml          (  90321 bytes, from 2016-11-28 16:50:05)
 - /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml         (   1773 bytes, from 2015-09-24 17:30:00)
 
 Copyright (C) 2013-2016 by the following authors:
@@ -58,6 +59,7 @@ enum vgt_event_type {
 	RST_PIX_CNT = 13,
 	RST_VTX_CNT = 14,
 	TILE_FLUSH = 15,
+	STAT_EVENT = 16,
 	CACHE_FLUSH_AND_INV_TS_EVENT = 20,
 	ZPASS_DONE = 21,
 	CACHE_FLUSH_AND_INV_EVENT = 22,
@@ -65,6 +67,10 @@ enum vgt_event_type {
 	PERFCOUNTER_STOP = 24,
 	VS_FETCH_DONE = 27,
 	FACENESS_FLUSH = 28,
+	UNK_1C = 28,
+	UNK_1D = 29,
+	BLIT = 30,
+	UNK_26 = 38,
 };
 
 enum pc_di_primtype {
@@ -82,7 +88,6 @@ enum pc_di_primtype {
 	DI_PT_LINESTRIP_ADJ = 11,
 	DI_PT_TRI_ADJ = 12,
 	DI_PT_TRISTRIP_ADJ = 13,
-	DI_PT_PATCHES = 34,
 };
 
 enum pc_di_src_sel {
@@ -110,11 +115,15 @@ enum adreno_pm4_packet_type {
 	CP_TYPE1_PKT = 0x40000000,
 	CP_TYPE2_PKT = 0x80000000,
 	CP_TYPE3_PKT = 0xc0000000,
+	CP_TYPE4_PKT = 0x40000000,
+	CP_TYPE7_PKT = 0x70000000,
 };
 
 enum adreno_pm4_type3_packets {
 	CP_ME_INIT = 72,
 	CP_NOP = 16,
+	CP_PREEMPT_ENABLE = 28,
+	CP_PREEMPT_TOKEN = 30,
 	CP_INDIRECT_BUFFER = 63,
 	CP_INDIRECT_BUFFER_PFD = 55,
 	CP_WAIT_FOR_IDLE = 38,
@@ -163,6 +172,7 @@ enum adreno_pm4_type3_packets {
 	CP_TEST_TWO_MEMS = 113,
 	CP_REG_WR_NO_CTXT = 120,
 	CP_RECORD_PFP_TIMESTAMP = 17,
+	CP_SET_SECURE_MODE = 102,
 	CP_WAIT_FOR_ME = 19,
 	CP_SET_DRAW_STATE = 67,
 	CP_DRAW_INDX_OFFSET = 56,
@@ -178,6 +188,22 @@ enum adreno_pm4_type3_packets {
 	CP_WAIT_MEM_WRITES = 18,
 	CP_COND_REG_EXEC = 71,
 	CP_MEM_TO_REG = 66,
+	CP_EXEC_CS = 51,
+	CP_PERFCOUNTER_ACTION = 80,
+	CP_SMMU_TABLE_UPDATE = 83,
+	CP_CONTEXT_REG_BUNCH = 92,
+	CP_YIELD_ENABLE = 28,
+	CP_SKIP_IB2_ENABLE_GLOBAL = 29,
+	CP_SKIP_IB2_ENABLE_LOCAL = 35,
+	CP_SET_SUBDRAW_SIZE = 53,
+	CP_SET_VISIBILITY_OVERRIDE = 100,
+	CP_PREEMPT_ENABLE_GLOBAL = 105,
+	CP_PREEMPT_ENABLE_LOCAL = 106,
+	CP_CONTEXT_SWITCH_YIELD = 107,
+	CP_SET_RENDER_MODE = 108,
+	CP_COMPUTE_CHECKPOINT = 110,
+	CP_MEM_TO_MEM = 115,
+	CP_BLIT = 44,
 	IN_IB_PREFETCH_END = 23,
 	IN_SUBBLK_PREFETCH = 31,
 	IN_INSTR_PREFETCH = 32,
@@ -196,6 +222,7 @@ enum adreno_state_block {
 	SB_VERT_SHADER = 4,
 	SB_GEOM_SHADER = 5,
 	SB_FRAG_SHADER = 6,
+	SB_COMPUTE_SHADER = 7,
 };
 
 enum adreno_state_type {
@@ -218,6 +245,17 @@ enum a4xx_index_size {
 	INDEX4_SIZE_32_BIT = 2,
 };
 
+enum render_mode_cmd {
+	BYPASS = 1,
+	GMEM = 3,
+	BLIT2D = 5,
+};
+
+enum cp_blit_cmd {
+	BLIT_OP_FILL = 0,
+	BLIT_OP_BLIT = 1,
+};
+
 #define REG_CP_LOAD_STATE_0					0x00000000
 #define CP_LOAD_STATE_0_DST_OFF__MASK				0x0000ffff
 #define CP_LOAD_STATE_0_DST_OFF__SHIFT				0
@@ -258,6 +296,14 @@ static inline uint32_t CP_LOAD_STATE_1_EXT_SRC_ADDR(uint32_t val)
 	return ((val >> 2) << CP_LOAD_STATE_1_EXT_SRC_ADDR__SHIFT) & CP_LOAD_STATE_1_EXT_SRC_ADDR__MASK;
 }
 
+#define REG_CP_LOAD_STATE_2					0x00000002
+#define CP_LOAD_STATE_2_EXT_SRC_ADDR_HI__MASK			0xffffffff
+#define CP_LOAD_STATE_2_EXT_SRC_ADDR_HI__SHIFT			0
+static inline uint32_t CP_LOAD_STATE_2_EXT_SRC_ADDR_HI(uint32_t val)
+{
+	return ((val) << CP_LOAD_STATE_2_EXT_SRC_ADDR_HI__SHIFT) & CP_LOAD_STATE_2_EXT_SRC_ADDR_HI__MASK;
+}
+
 #define REG_CP_DRAW_INDX_0					0x00000000
 #define CP_DRAW_INDX_0_VIZ_QUERY__MASK				0xffffffff
 #define CP_DRAW_INDX_0_VIZ_QUERY__SHIFT				0
@@ -389,7 +435,12 @@ static inline uint32_t CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT(enum pc_di_src_sel va
 {
 	return ((val) << CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT__SHIFT) & CP_DRAW_INDX_OFFSET_0_SOURCE_SELECT__MASK;
 }
-#define CP_DRAW_INDX_OFFSET_0_TESSELLATE			0x00000100
+#define CP_DRAW_INDX_OFFSET_0_VIS_CULL__MASK			0x00000300
+#define CP_DRAW_INDX_OFFSET_0_VIS_CULL__SHIFT			8
+static inline uint32_t CP_DRAW_INDX_OFFSET_0_VIS_CULL(enum pc_di_vis_cull_mode val)
+{
+	return ((val) << CP_DRAW_INDX_OFFSET_0_VIS_CULL__SHIFT) & CP_DRAW_INDX_OFFSET_0_VIS_CULL__MASK;
+}
 #define CP_DRAW_INDX_OFFSET_0_INDEX_SIZE__MASK			0x00000c00
 #define CP_DRAW_INDX_OFFSET_0_INDEX_SIZE__SHIFT			10
 static inline uint32_t CP_DRAW_INDX_OFFSET_0_INDEX_SIZE(enum a4xx_index_size val)
@@ -437,30 +488,40 @@ static inline uint32_t CP_DRAW_INDX_OFFSET_5_INDX_SIZE(uint32_t val)
 	return ((val) << CP_DRAW_INDX_OFFSET_5_INDX_SIZE__SHIFT) & CP_DRAW_INDX_OFFSET_5_INDX_SIZE__MASK;
 }
 
-#define REG_CP_SET_DRAW_STATE_0					0x00000000
-#define CP_SET_DRAW_STATE_0_COUNT__MASK				0x0000ffff
-#define CP_SET_DRAW_STATE_0_COUNT__SHIFT			0
-static inline uint32_t CP_SET_DRAW_STATE_0_COUNT(uint32_t val)
+static inline uint32_t REG_CP_SET_DRAW_STATE_(uint32_t i0) { return 0x00000000 + 0x3*i0; }
+
+static inline uint32_t REG_CP_SET_DRAW_STATE__0(uint32_t i0) { return 0x00000000 + 0x3*i0; }
+#define CP_SET_DRAW_STATE__0_COUNT__MASK			0x0000ffff
+#define CP_SET_DRAW_STATE__0_COUNT__SHIFT			0
+static inline uint32_t CP_SET_DRAW_STATE__0_COUNT(uint32_t val)
 {
-	return ((val) << CP_SET_DRAW_STATE_0_COUNT__SHIFT) & CP_SET_DRAW_STATE_0_COUNT__MASK;
+	return ((val) << CP_SET_DRAW_STATE__0_COUNT__SHIFT) & CP_SET_DRAW_STATE__0_COUNT__MASK;
 }
-#define CP_SET_DRAW_STATE_0_DIRTY				0x00010000
-#define CP_SET_DRAW_STATE_0_DISABLE				0x00020000
-#define CP_SET_DRAW_STATE_0_DISABLE_ALL_GROUPS			0x00040000
-#define CP_SET_DRAW_STATE_0_LOAD_IMMED				0x00080000
-#define CP_SET_DRAW_STATE_0_GROUP_ID__MASK			0x1f000000
-#define CP_SET_DRAW_STATE_0_GROUP_ID__SHIFT			24
-static inline uint32_t CP_SET_DRAW_STATE_0_GROUP_ID(uint32_t val)
+#define CP_SET_DRAW_STATE__0_DIRTY				0x00010000
+#define CP_SET_DRAW_STATE__0_DISABLE				0x00020000
+#define CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS			0x00040000
+#define CP_SET_DRAW_STATE__0_LOAD_IMMED				0x00080000
+#define CP_SET_DRAW_STATE__0_GROUP_ID__MASK			0x1f000000
+#define CP_SET_DRAW_STATE__0_GROUP_ID__SHIFT			24
+static inline uint32_t CP_SET_DRAW_STATE__0_GROUP_ID(uint32_t val)
 {
-	return ((val) << CP_SET_DRAW_STATE_0_GROUP_ID__SHIFT) & CP_SET_DRAW_STATE_0_GROUP_ID__MASK;
+	return ((val) << CP_SET_DRAW_STATE__0_GROUP_ID__SHIFT) & CP_SET_DRAW_STATE__0_GROUP_ID__MASK;
 }
 
-#define REG_CP_SET_DRAW_STATE_1					0x00000001
-#define CP_SET_DRAW_STATE_1_ADDR__MASK				0xffffffff
-#define CP_SET_DRAW_STATE_1_ADDR__SHIFT				0
-static inline uint32_t CP_SET_DRAW_STATE_1_ADDR(uint32_t val)
+static inline uint32_t REG_CP_SET_DRAW_STATE__1(uint32_t i0) { return 0x00000001 + 0x3*i0; }
+#define CP_SET_DRAW_STATE__1_ADDR_LO__MASK			0xffffffff
+#define CP_SET_DRAW_STATE__1_ADDR_LO__SHIFT			0
+static inline uint32_t CP_SET_DRAW_STATE__1_ADDR_LO(uint32_t val)
 {
-	return ((val) << CP_SET_DRAW_STATE_1_ADDR__SHIFT) & CP_SET_DRAW_STATE_1_ADDR__MASK;
+	return ((val) << CP_SET_DRAW_STATE__1_ADDR_LO__SHIFT) & CP_SET_DRAW_STATE__1_ADDR_LO__MASK;
+}
+
+static inline uint32_t REG_CP_SET_DRAW_STATE__2(uint32_t i0) { return 0x00000002 + 0x3*i0; }
+#define CP_SET_DRAW_STATE__2_ADDR_HI__MASK			0xffffffff
+#define CP_SET_DRAW_STATE__2_ADDR_HI__SHIFT			0
+static inline uint32_t CP_SET_DRAW_STATE__2_ADDR_HI(uint32_t val)
+{
+	return ((val) << CP_SET_DRAW_STATE__2_ADDR_HI__SHIFT) & CP_SET_DRAW_STATE__2_ADDR_HI__MASK;
 }
 
 #define REG_CP_SET_BIN_0					0x00000000
@@ -533,5 +594,192 @@ static inline uint32_t CP_REG_TO_MEM_1_DEST(uint32_t val)
 	return ((val) << CP_REG_TO_MEM_1_DEST__SHIFT) & CP_REG_TO_MEM_1_DEST__MASK;
 }
 
+#define REG_CP_DISPATCH_COMPUTE_0				0x00000000
+
+#define REG_CP_DISPATCH_COMPUTE_1				0x00000001
+#define CP_DISPATCH_COMPUTE_1_X__MASK				0xffffffff
+#define CP_DISPATCH_COMPUTE_1_X__SHIFT				0
+static inline uint32_t CP_DISPATCH_COMPUTE_1_X(uint32_t val)
+{
+	return ((val) << CP_DISPATCH_COMPUTE_1_X__SHIFT) & CP_DISPATCH_COMPUTE_1_X__MASK;
+}
+
+#define REG_CP_DISPATCH_COMPUTE_2				0x00000002
+#define CP_DISPATCH_COMPUTE_2_Y__MASK				0xffffffff
+#define CP_DISPATCH_COMPUTE_2_Y__SHIFT				0
+static inline uint32_t CP_DISPATCH_COMPUTE_2_Y(uint32_t val)
+{
+	return ((val) << CP_DISPATCH_COMPUTE_2_Y__SHIFT) & CP_DISPATCH_COMPUTE_2_Y__MASK;
+}
+
+#define REG_CP_DISPATCH_COMPUTE_3				0x00000003
+#define CP_DISPATCH_COMPUTE_3_Z__MASK				0xffffffff
+#define CP_DISPATCH_COMPUTE_3_Z__SHIFT				0
+static inline uint32_t CP_DISPATCH_COMPUTE_3_Z(uint32_t val)
+{
+	return ((val) << CP_DISPATCH_COMPUTE_3_Z__SHIFT) & CP_DISPATCH_COMPUTE_3_Z__MASK;
+}
+
+#define REG_CP_SET_RENDER_MODE_0				0x00000000
+#define CP_SET_RENDER_MODE_0_MODE__MASK				0x000001ff
+#define CP_SET_RENDER_MODE_0_MODE__SHIFT			0
+static inline uint32_t CP_SET_RENDER_MODE_0_MODE(enum render_mode_cmd val)
+{
+	return ((val) << CP_SET_RENDER_MODE_0_MODE__SHIFT) & CP_SET_RENDER_MODE_0_MODE__MASK;
+}
+
+#define REG_CP_SET_RENDER_MODE_1				0x00000001
+#define CP_SET_RENDER_MODE_1_ADDR_0_LO__MASK			0xffffffff
+#define CP_SET_RENDER_MODE_1_ADDR_0_LO__SHIFT			0
+static inline uint32_t CP_SET_RENDER_MODE_1_ADDR_0_LO(uint32_t val)
+{
+	return ((val) << CP_SET_RENDER_MODE_1_ADDR_0_LO__SHIFT) & CP_SET_RENDER_MODE_1_ADDR_0_LO__MASK;
+}
+
+#define REG_CP_SET_RENDER_MODE_2				0x00000002
+#define CP_SET_RENDER_MODE_2_ADDR_0_HI__MASK			0xffffffff
+#define CP_SET_RENDER_MODE_2_ADDR_0_HI__SHIFT			0
+static inline uint32_t CP_SET_RENDER_MODE_2_ADDR_0_HI(uint32_t val)
+{
+	return ((val) << CP_SET_RENDER_MODE_2_ADDR_0_HI__SHIFT) & CP_SET_RENDER_MODE_2_ADDR_0_HI__MASK;
+}
+
+#define REG_CP_SET_RENDER_MODE_3				0x00000003
+#define CP_SET_RENDER_MODE_3_GMEM_ENABLE			0x00000010
+
+#define REG_CP_SET_RENDER_MODE_4				0x00000004
+
+#define REG_CP_SET_RENDER_MODE_5				0x00000005
+#define CP_SET_RENDER_MODE_5_ADDR_1_LEN__MASK			0xffffffff
+#define CP_SET_RENDER_MODE_5_ADDR_1_LEN__SHIFT			0
+static inline uint32_t CP_SET_RENDER_MODE_5_ADDR_1_LEN(uint32_t val)
+{
+	return ((val) << CP_SET_RENDER_MODE_5_ADDR_1_LEN__SHIFT) & CP_SET_RENDER_MODE_5_ADDR_1_LEN__MASK;
+}
+
+#define REG_CP_SET_RENDER_MODE_6				0x00000006
+#define CP_SET_RENDER_MODE_6_ADDR_1_LO__MASK			0xffffffff
+#define CP_SET_RENDER_MODE_6_ADDR_1_LO__SHIFT			0
+static inline uint32_t CP_SET_RENDER_MODE_6_ADDR_1_LO(uint32_t val)
+{
+	return ((val) << CP_SET_RENDER_MODE_6_ADDR_1_LO__SHIFT) & CP_SET_RENDER_MODE_6_ADDR_1_LO__MASK;
+}
+
+#define REG_CP_SET_RENDER_MODE_7				0x00000007
+#define CP_SET_RENDER_MODE_7_ADDR_1_HI__MASK			0xffffffff
+#define CP_SET_RENDER_MODE_7_ADDR_1_HI__SHIFT			0
+static inline uint32_t CP_SET_RENDER_MODE_7_ADDR_1_HI(uint32_t val)
+{
+	return ((val) << CP_SET_RENDER_MODE_7_ADDR_1_HI__SHIFT) & CP_SET_RENDER_MODE_7_ADDR_1_HI__MASK;
+}
+
+#define REG_CP_PERFCOUNTER_ACTION_0				0x00000000
+
+#define REG_CP_PERFCOUNTER_ACTION_1				0x00000001
+#define CP_PERFCOUNTER_ACTION_1_ADDR_0_LO__MASK			0xffffffff
+#define CP_PERFCOUNTER_ACTION_1_ADDR_0_LO__SHIFT		0
+static inline uint32_t CP_PERFCOUNTER_ACTION_1_ADDR_0_LO(uint32_t val)
+{
+	return ((val) << CP_PERFCOUNTER_ACTION_1_ADDR_0_LO__SHIFT) & CP_PERFCOUNTER_ACTION_1_ADDR_0_LO__MASK;
+}
+
+#define REG_CP_PERFCOUNTER_ACTION_2				0x00000002
+#define CP_PERFCOUNTER_ACTION_2_ADDR_0_HI__MASK			0xffffffff
+#define CP_PERFCOUNTER_ACTION_2_ADDR_0_HI__SHIFT		0
+static inline uint32_t CP_PERFCOUNTER_ACTION_2_ADDR_0_HI(uint32_t val)
+{
+	return ((val) << CP_PERFCOUNTER_ACTION_2_ADDR_0_HI__SHIFT) & CP_PERFCOUNTER_ACTION_2_ADDR_0_HI__MASK;
+}
+
+#define REG_CP_EVENT_WRITE_0					0x00000000
+#define CP_EVENT_WRITE_0_EVENT__MASK				0x000000ff
+#define CP_EVENT_WRITE_0_EVENT__SHIFT				0
+static inline uint32_t CP_EVENT_WRITE_0_EVENT(enum vgt_event_type val)
+{
+	return ((val) << CP_EVENT_WRITE_0_EVENT__SHIFT) & CP_EVENT_WRITE_0_EVENT__MASK;
+}
+
+#define REG_CP_EVENT_WRITE_1					0x00000001
+#define CP_EVENT_WRITE_1_ADDR_0_LO__MASK			0xffffffff
+#define CP_EVENT_WRITE_1_ADDR_0_LO__SHIFT			0
+static inline uint32_t CP_EVENT_WRITE_1_ADDR_0_LO(uint32_t val)
+{
+	return ((val) << CP_EVENT_WRITE_1_ADDR_0_LO__SHIFT) & CP_EVENT_WRITE_1_ADDR_0_LO__MASK;
+}
+
+#define REG_CP_EVENT_WRITE_2					0x00000002
+#define CP_EVENT_WRITE_2_ADDR_0_HI__MASK			0xffffffff
+#define CP_EVENT_WRITE_2_ADDR_0_HI__SHIFT			0
+static inline uint32_t CP_EVENT_WRITE_2_ADDR_0_HI(uint32_t val)
+{
+	return ((val) << CP_EVENT_WRITE_2_ADDR_0_HI__SHIFT) & CP_EVENT_WRITE_2_ADDR_0_HI__MASK;
+}
+
+#define REG_CP_EVENT_WRITE_3					0x00000003
+
+#define REG_CP_BLIT_0						0x00000000
+#define CP_BLIT_0_OP__MASK					0x0000000f
+#define CP_BLIT_0_OP__SHIFT					0
+static inline uint32_t CP_BLIT_0_OP(enum cp_blit_cmd val)
+{
+	return ((val) << CP_BLIT_0_OP__SHIFT) & CP_BLIT_0_OP__MASK;
+}
+
+#define REG_CP_BLIT_1						0x00000001
+#define CP_BLIT_1_SRC_X1__MASK					0x0000ffff
+#define CP_BLIT_1_SRC_X1__SHIFT					0
+static inline uint32_t CP_BLIT_1_SRC_X1(uint32_t val)
+{
+	return ((val) << CP_BLIT_1_SRC_X1__SHIFT) & CP_BLIT_1_SRC_X1__MASK;
+}
+#define CP_BLIT_1_SRC_Y1__MASK					0xffff0000
+#define CP_BLIT_1_SRC_Y1__SHIFT					16
+static inline uint32_t CP_BLIT_1_SRC_Y1(uint32_t val)
+{
+	return ((val) << CP_BLIT_1_SRC_Y1__SHIFT) & CP_BLIT_1_SRC_Y1__MASK;
+}
+
+#define REG_CP_BLIT_2						0x00000002
+#define CP_BLIT_2_SRC_X2__MASK					0x0000ffff
+#define CP_BLIT_2_SRC_X2__SHIFT					0
+static inline uint32_t CP_BLIT_2_SRC_X2(uint32_t val)
+{
+	return ((val) << CP_BLIT_2_SRC_X2__SHIFT) & CP_BLIT_2_SRC_X2__MASK;
+}
+#define CP_BLIT_2_SRC_Y2__MASK					0xffff0000
+#define CP_BLIT_2_SRC_Y2__SHIFT					16
+static inline uint32_t CP_BLIT_2_SRC_Y2(uint32_t val)
+{
+	return ((val) << CP_BLIT_2_SRC_Y2__SHIFT) & CP_BLIT_2_SRC_Y2__MASK;
+}
+
+#define REG_CP_BLIT_3						0x00000003
+#define CP_BLIT_3_DST_X1__MASK					0x0000ffff
+#define CP_BLIT_3_DST_X1__SHIFT					0
+static inline uint32_t CP_BLIT_3_DST_X1(uint32_t val)
+{
+	return ((val) << CP_BLIT_3_DST_X1__SHIFT) & CP_BLIT_3_DST_X1__MASK;
+}
+#define CP_BLIT_3_DST_Y1__MASK					0xffff0000
+#define CP_BLIT_3_DST_Y1__SHIFT					16
+static inline uint32_t CP_BLIT_3_DST_Y1(uint32_t val)
+{
+	return ((val) << CP_BLIT_3_DST_Y1__SHIFT) & CP_BLIT_3_DST_Y1__MASK;
+}
+
+#define REG_CP_BLIT_4						0x00000004
+#define CP_BLIT_4_DST_X2__MASK					0x0000ffff
+#define CP_BLIT_4_DST_X2__SHIFT					0
+static inline uint32_t CP_BLIT_4_DST_X2(uint32_t val)
+{
+	return ((val) << CP_BLIT_4_DST_X2__SHIFT) & CP_BLIT_4_DST_X2__MASK;
+}
+#define CP_BLIT_4_DST_Y2__MASK					0xffff0000
+#define CP_BLIT_4_DST_Y2__SHIFT					16
+static inline uint32_t CP_BLIT_4_DST_Y2(uint32_t val)
+{
+	return ((val) << CP_BLIT_4_DST_Y2__SHIFT) & CP_BLIT_4_DST_Y2__MASK;
+}
+
 
 #endif /* ADRENO_PM4_XML */
diff --git a/drivers/gpu/drm/msm/dsi/dsi.xml.h b/drivers/gpu/drm/msm/dsi/dsi.xml.h
index 4958594d5266..39dff7d5e89b 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.xml.h
+++ b/drivers/gpu/drm/msm/dsi/dsi.xml.h
@@ -12,7 +12,7 @@ The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-02-10 17:07:21)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2015-05-20 20:03:14)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2015-09-18 12:07:28)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  36965 bytes, from 2016-11-26 23:01:08)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  27887 bytes, from 2015-10-22 16:34:52)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2015-10-22 16:35:02)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2015-05-20 20:03:14)
diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
index 6f240021705b..3819fdefcae2 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
@@ -982,7 +982,7 @@ static int dsi_tx_buf_alloc(struct msm_dsi_host *msm_host, int size)
 	struct drm_device *dev = msm_host->dev;
 	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	int ret;
-	u32 iova;
+	uint64_t iova;
 
 	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G) {
 		mutex_lock(&dev->struct_mutex);
@@ -1147,7 +1147,7 @@ static int dsi_cmd_dma_tx(struct msm_dsi_host *msm_host, int len)
 {
 	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
 	int ret;
-	u32 dma_base;
+	uint64_t dma_base;
 	bool triggered;
 
 	if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G) {
diff --git a/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h b/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
index 2d999494cdea..8b9f3ebaeba7 100644
--- a/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
+++ b/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
@@ -12,7 +12,7 @@ The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-02-10 17:07:21)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2015-05-20 20:03:14)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2015-09-18 12:07:28)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  36965 bytes, from 2016-11-26 23:01:08)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  27887 bytes, from 2015-10-22 16:34:52)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2015-10-22 16:35:02)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2015-05-20 20:03:14)
diff --git a/drivers/gpu/drm/msm/dsi/sfpb.xml.h b/drivers/gpu/drm/msm/dsi/sfpb.xml.h
index 506434fac993..3fcbb30dc241 100644
--- a/drivers/gpu/drm/msm/dsi/sfpb.xml.h
+++ b/drivers/gpu/drm/msm/dsi/sfpb.xml.h
@@ -12,7 +12,7 @@ The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-02-10 17:07:21)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2015-05-20 20:03:14)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2015-09-18 12:07:28)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  36965 bytes, from 2016-11-26 23:01:08)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  27887 bytes, from 2015-10-22 16:34:52)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2015-10-22 16:35:02)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2015-05-20 20:03:14)
diff --git a/drivers/gpu/drm/msm/edp/edp.xml.h b/drivers/gpu/drm/msm/edp/edp.xml.h
index f1072c18c81e..d7bf3232dc88 100644
--- a/drivers/gpu/drm/msm/edp/edp.xml.h
+++ b/drivers/gpu/drm/msm/edp/edp.xml.h
@@ -12,7 +12,7 @@ The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-02-10 17:07:21)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2015-05-20 20:03:14)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2015-09-18 12:07:28)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  36965 bytes, from 2016-11-26 23:01:08)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  27887 bytes, from 2015-10-22 16:34:52)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2015-10-22 16:35:02)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2015-05-20 20:03:14)
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.xml.h b/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
index 34c7df6549c1..0a97ff75ed6f 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
@@ -12,7 +12,7 @@ The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-02-10 17:07:21)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2015-05-20 20:03:14)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2015-09-18 12:07:28)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  36965 bytes, from 2016-11-26 23:01:08)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  27887 bytes, from 2015-10-22 16:34:52)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2015-10-22 16:35:02)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2015-05-20 20:03:14)
diff --git a/drivers/gpu/drm/msm/hdmi/qfprom.xml.h b/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
index 6eab7d0cf6b5..1b996ede7a65 100644
--- a/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
+++ b/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
@@ -12,7 +12,7 @@ The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-02-10 17:07:21)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2015-05-20 20:03:14)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2015-09-18 12:07:28)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  36965 bytes, from 2016-11-26 23:01:08)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  27887 bytes, from 2015-10-22 16:34:52)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2015-10-22 16:35:02)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2015-05-20 20:03:14)
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4.xml.h b/drivers/gpu/drm/msm/mdp/mdp4/mdp4.xml.h
index 6688e79cc88e..88037889589b 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4.xml.h
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4.xml.h
@@ -12,7 +12,7 @@ The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-02-10 17:07:21)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2015-05-20 20:03:14)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2015-09-18 12:07:28)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  36965 bytes, from 2016-11-26 23:01:08)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  27887 bytes, from 2015-10-22 16:34:52)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2015-10-22 16:35:02)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2015-05-20 20:03:14)
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c
index 9527dafc3e69..1c29618f4ddb 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c
@@ -373,7 +373,7 @@ static void update_cursor(struct drm_crtc *crtc)
 	if (mdp4_crtc->cursor.stale) {
 		struct drm_gem_object *next_bo = mdp4_crtc->cursor.next_bo;
 		struct drm_gem_object *prev_bo = mdp4_crtc->cursor.scanout_bo;
-		uint32_t iova = mdp4_crtc->cursor.next_iova;
+		uint64_t iova = mdp4_crtc->cursor.next_iova;
 
 		if (next_bo) {
 			/* take a obj ref + iova ref when we start scanning out: */
@@ -418,7 +418,7 @@ static int mdp4_crtc_cursor_set(struct drm_crtc *crtc,
 	struct drm_device *dev = crtc->dev;
 	struct drm_gem_object *cursor_bo, *old_bo;
 	unsigned long flags;
-	uint32_t iova;
+	uint64_t iova;
 	int ret;
 
 	if ((width > CURSOR_WIDTH) || (height > CURSOR_HEIGHT)) {
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_irq.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_irq.c
index a521207db8a1..b764d7f10312 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_irq.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_irq.c
@@ -15,6 +15,7 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <drm/drm_print.h>
 
 #include "msm_drv.h"
 #include "mdp4_kms.h"
@@ -29,7 +30,16 @@ void mdp4_set_irqmask(struct mdp_kms *mdp_kms, uint32_t irqmask,
 
 static void mdp4_irq_error_handler(struct mdp_irq *irq, uint32_t irqstatus)
 {
+	struct mdp4_kms *mdp4_kms = container_of(irq, struct mdp4_kms, error_handler);
+	static DEFINE_RATELIMIT_STATE(rs, 5*HZ, 1);
+	extern bool dumpstate;
+
 	DRM_ERROR_RATELIMITED("errors: %08x\n", irqstatus);
+
+	if (dumpstate && __ratelimit(&rs)) {
+		struct drm_printer p = drm_info_printer(mdp4_kms->dev->dev);
+		drm_state_dump(mdp4_kms->dev, &p);
+	}
 }
 
 void mdp4_irq_preinstall(struct msm_kms *kms)
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
index 571a91ee9607..b782efd4b95f 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
@@ -17,6 +17,7 @@
 
 
 #include "msm_drv.h"
+#include "msm_gem.h"
 #include "msm_mmu.h"
 #include "mdp4_kms.h"
 
@@ -159,17 +160,18 @@ static void mdp4_destroy(struct msm_kms *kms)
 {
 	struct mdp4_kms *mdp4_kms = to_mdp4_kms(to_mdp_kms(kms));
 	struct device *dev = mdp4_kms->dev->dev;
-	struct msm_mmu *mmu = mdp4_kms->mmu;
-
-	if (mmu) {
-		mmu->funcs->detach(mmu, iommu_ports, ARRAY_SIZE(iommu_ports));
-		mmu->funcs->destroy(mmu);
-	}
+	struct msm_gem_address_space *aspace = mdp4_kms->aspace;
 
 	if (mdp4_kms->blank_cursor_iova)
 		msm_gem_put_iova(mdp4_kms->blank_cursor_bo, mdp4_kms->id);
 	drm_gem_object_unreference_unlocked(mdp4_kms->blank_cursor_bo);
 
+	if (aspace) {
+		aspace->mmu->funcs->detach(aspace->mmu,
+				iommu_ports, ARRAY_SIZE(iommu_ports));
+		msm_gem_address_space_destroy(aspace);
+	}
+
 	if (mdp4_kms->rpm_enabled)
 		pm_runtime_disable(dev);
 
@@ -440,7 +442,7 @@ struct msm_kms *mdp4_kms_init(struct drm_device *dev)
 	struct mdp4_platform_config *config = mdp4_get_config(pdev);
 	struct mdp4_kms *mdp4_kms;
 	struct msm_kms *kms = NULL;
-	struct msm_mmu *mmu;
+	struct msm_gem_address_space *aspace;
 	int irq, ret;
 
 	mdp4_kms = kzalloc(sizeof(*mdp4_kms), GFP_KERNEL);
@@ -531,24 +533,26 @@ struct msm_kms *mdp4_kms_init(struct drm_device *dev)
 	mdelay(16);
 
 	if (config->iommu) {
-		mmu = msm_iommu_new(&pdev->dev, config->iommu);
-		if (IS_ERR(mmu)) {
-			ret = PTR_ERR(mmu);
+		aspace = msm_gem_address_space_create(&pdev->dev,
+				config->iommu, "mdp4");
+		if (IS_ERR(aspace)) {
+			ret = PTR_ERR(aspace);
 			goto fail;
 		}
-		ret = mmu->funcs->attach(mmu, iommu_ports,
+
+		mdp4_kms->aspace = aspace;
+
+		ret = aspace->mmu->funcs->attach(aspace->mmu, iommu_ports,
 				ARRAY_SIZE(iommu_ports));
 		if (ret)
 			goto fail;
-
-		mdp4_kms->mmu = mmu;
 	} else {
 		dev_info(dev->dev, "no iommu, fallback to phys "
 				"contig buffers for scanout\n");
-		mmu = NULL;
+		aspace = NULL;
 	}
 
-	mdp4_kms->id = msm_register_mmu(dev, mmu);
+	mdp4_kms->id = msm_register_address_space(dev, aspace);
 	if (mdp4_kms->id < 0) {
 		ret = mdp4_kms->id;
 		dev_err(dev->dev, "failed to register mdp4 iommu: %d\n", ret);
@@ -598,6 +602,10 @@ static struct mdp4_platform_config *mdp4_get_config(struct platform_device *dev)
 	/* TODO: Chips that aren't apq8064 have a 200 Mhz max_clk */
 	config.max_clk = 266667000;
 	config.iommu = iommu_domain_alloc(&platform_bus_type);
+	if (config.iommu) {
+		config.iommu->geometry.aperture_start = 0x1000;
+		config.iommu->geometry.aperture_end = 0xffffffff;
+	}
 
 	return &config;
 }
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h
index 25fb83997119..62712ca164ee 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h
@@ -43,7 +43,7 @@ struct mdp4_kms {
 	struct clk *pclk;
 	struct clk *lut_clk;
 	struct clk *axi_clk;
-	struct msm_mmu *mmu;
+	struct msm_gem_address_space *aspace;
 
 	struct mdp_irq error_handler;
 
@@ -51,7 +51,7 @@ struct mdp4_kms {
 
 	/* empty/blank cursor bo to use when cursor is "disabled" */
 	struct drm_gem_object *blank_cursor_bo;
-	uint32_t blank_cursor_iova;
+	uint64_t blank_cursor_iova;
 };
 #define to_mdp4_kms(x) container_of(x, struct mdp4_kms, base)
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
index 3903dbcda763..911e4690d36a 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
@@ -40,7 +40,7 @@ enum mdp4_frame_format mdp4_get_frame_format(struct drm_framebuffer *fb)
 {
 	bool is_tile = false;
 
-	if (fb->modifier[1] == DRM_FORMAT_MOD_SAMSUNG_64_32_TILE)
+	if (fb->modifier == DRM_FORMAT_MOD_SAMSUNG_64_32_TILE)
 		is_tile = true;
 
 	if (fb->pixel_format == DRM_FORMAT_NV12 && is_tile)
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5.xml.h b/drivers/gpu/drm/msm/mdp/mdp5/mdp5.xml.h
index ca6ca30650a0..27d5371acee0 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5.xml.h
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5.xml.h
@@ -8,9 +8,17 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /local/mnt/workspace/source_trees/envytools/rnndb/../rnndb/mdp/mdp5.xml   (  36965 bytes, from 2016-05-10 05:06:30)
-- /local/mnt/workspace/source_trees/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-05-09 06:32:54)
-- /local/mnt/workspace/source_trees/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2016-01-07 08:45:55)
+- /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    676 bytes, from 2015-05-20 20:03:14)
+- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-02-10 17:07:21)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2015-05-20 20:03:14)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  36965 bytes, from 2016-11-26 23:01:08)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  27887 bytes, from 2015-10-22 16:34:52)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2015-10-22 16:35:02)
+- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2015-05-20 20:03:14)
+- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2015-05-20 20:03:07)
+- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml           (  41472 bytes, from 2016-01-22 18:18:18)
+- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2015-05-20 20:03:14)
 
 Copyright (C) 2013-2016 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c
index 8b4e3004f451..618b2ffed9b4 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c
@@ -550,6 +550,10 @@ static struct mdp5_cfg_platform *mdp5_get_config(struct platform_device *dev)
 	static struct mdp5_cfg_platform config = {};
 
 	config.iommu = iommu_domain_alloc(&platform_bus_type);
+	if (config.iommu) {
+		config.iommu->geometry.aperture_start = 0x1000;
+		config.iommu->geometry.aperture_end = 0xffffffff;
+	}
 
 	return &config;
 }
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
index c205c360e16d..1ce8a01a5a28 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
@@ -27,11 +27,8 @@
 #define CURSOR_WIDTH	64
 #define CURSOR_HEIGHT	64
 
-#define SSPP_MAX	(SSPP_RGB3 + 1) /* TODO: Add SSPP_MAX in mdp5.xml.h */
-
 struct mdp5_crtc {
 	struct drm_crtc base;
-	char name[8];
 	int id;
 	bool enabled;
 
@@ -102,7 +99,7 @@ static u32 crtc_flush(struct drm_crtc *crtc, u32 flush_mask)
 {
 	struct mdp5_crtc *mdp5_crtc = to_mdp5_crtc(crtc);
 
-	DBG("%s: flush=%08x", mdp5_crtc->name, flush_mask);
+	DBG("%s: flush=%08x", crtc->name, flush_mask);
 	return mdp5_ctl_commit(mdp5_crtc->ctl, flush_mask);
 }
 
@@ -136,7 +133,6 @@ static void complete_flip(struct drm_crtc *crtc, struct drm_file *file)
 	struct mdp5_crtc *mdp5_crtc = to_mdp5_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct drm_pending_vblank_event *event;
-	struct drm_plane *plane;
 	unsigned long flags;
 
 	spin_lock_irqsave(&dev->event_lock, flags);
@@ -148,16 +144,12 @@ static void complete_flip(struct drm_crtc *crtc, struct drm_file *file)
 		 */
 		if (!file || (event->base.file_priv == file)) {
 			mdp5_crtc->event = NULL;
-			DBG("%s: send event: %p", mdp5_crtc->name, event);
+			DBG("%s: send event: %p", crtc->name, event);
 			drm_crtc_send_vblank_event(crtc, event);
 		}
 	}
 	spin_unlock_irqrestore(&dev->event_lock, flags);
 
-	drm_atomic_crtc_for_each_plane(plane, crtc) {
-		mdp5_plane_complete_flip(plane);
-	}
-
 	if (mdp5_crtc->ctl && !crtc->state->enable) {
 		/* set STAGE_UNUSED for all layers */
 		mdp5_ctl_blend(mdp5_crtc->ctl, NULL, 0, 0);
@@ -295,7 +287,7 @@ static void mdp5_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	mode = &crtc->state->adjusted_mode;
 
 	DBG("%s: set mode: %d:\"%s\" %d %d %d %d %d %d %d %d %d %d 0x%x 0x%x",
-			mdp5_crtc->name, mode->base.id, mode->name,
+			crtc->name, mode->base.id, mode->name,
 			mode->vrefresh, mode->clock,
 			mode->hdisplay, mode->hsync_start,
 			mode->hsync_end, mode->htotal,
@@ -315,7 +307,7 @@ static void mdp5_crtc_disable(struct drm_crtc *crtc)
 	struct mdp5_crtc *mdp5_crtc = to_mdp5_crtc(crtc);
 	struct mdp5_kms *mdp5_kms = get_kms(crtc);
 
-	DBG("%s", mdp5_crtc->name);
+	DBG("%s", crtc->name);
 
 	if (WARN_ON(!mdp5_crtc->enabled))
 		return;
@@ -334,7 +326,7 @@ static void mdp5_crtc_enable(struct drm_crtc *crtc)
 	struct mdp5_crtc *mdp5_crtc = to_mdp5_crtc(crtc);
 	struct mdp5_kms *mdp5_kms = get_kms(crtc);
 
-	DBG("%s", mdp5_crtc->name);
+	DBG("%s", crtc->name);
 
 	if (WARN_ON(mdp5_crtc->enabled))
 		return;
@@ -372,7 +364,6 @@ static bool is_fullscreen(struct drm_crtc_state *cstate,
 static int mdp5_crtc_atomic_check(struct drm_crtc *crtc,
 		struct drm_crtc_state *state)
 {
-	struct mdp5_crtc *mdp5_crtc = to_mdp5_crtc(crtc);
 	struct mdp5_kms *mdp5_kms = get_kms(crtc);
 	struct drm_plane *plane;
 	struct drm_device *dev = crtc->dev;
@@ -381,7 +372,7 @@ static int mdp5_crtc_atomic_check(struct drm_crtc *crtc,
 	const struct drm_plane_state *pstate;
 	int cnt = 0, base = 0, i;
 
-	DBG("%s: check", mdp5_crtc->name);
+	DBG("%s: check", crtc->name);
 
 	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, state) {
 		pstates[cnt].plane = plane;
@@ -405,14 +396,14 @@ static int mdp5_crtc_atomic_check(struct drm_crtc *crtc,
 	hw_cfg = mdp5_cfg_get_hw_config(mdp5_kms->cfg);
 
 	if ((cnt + base) >= hw_cfg->lm.nb_stages) {
-		dev_err(dev->dev, "too many planes!\n");
+		dev_err(dev->dev, "too many planes! cnt=%d, base=%d\n", cnt, base);
 		return -EINVAL;
 	}
 
 	for (i = 0; i < cnt; i++) {
 		pstates[i].state->stage = STAGE_BASE + i + base;
-		DBG("%s: assign pipe %s on stage=%d", mdp5_crtc->name,
-				pipe2name(mdp5_plane_pipe(pstates[i].plane)),
+		DBG("%s: assign pipe %s on stage=%d", crtc->name,
+				pstates[i].plane->name,
 				pstates[i].state->stage);
 	}
 
@@ -422,8 +413,7 @@ static int mdp5_crtc_atomic_check(struct drm_crtc *crtc,
 static void mdp5_crtc_atomic_begin(struct drm_crtc *crtc,
 				   struct drm_crtc_state *old_crtc_state)
 {
-	struct mdp5_crtc *mdp5_crtc = to_mdp5_crtc(crtc);
-	DBG("%s: begin", mdp5_crtc->name);
+	DBG("%s: begin", crtc->name);
 }
 
 static void mdp5_crtc_atomic_flush(struct drm_crtc *crtc,
@@ -433,7 +423,7 @@ static void mdp5_crtc_atomic_flush(struct drm_crtc *crtc,
 	struct drm_device *dev = crtc->dev;
 	unsigned long flags;
 
-	DBG("%s: event: %p", mdp5_crtc->name, crtc->state->event);
+	DBG("%s: event: %p", crtc->name, crtc->state->event);
 
 	WARN_ON(mdp5_crtc->event);
 
@@ -499,7 +489,8 @@ static int mdp5_crtc_cursor_set(struct drm_crtc *crtc,
 	struct drm_device *dev = crtc->dev;
 	struct mdp5_kms *mdp5_kms = get_kms(crtc);
 	struct drm_gem_object *cursor_bo, *old_bo = NULL;
-	uint32_t blendcfg, cursor_addr, stride;
+	uint32_t blendcfg, stride;
+	uint64_t cursor_addr;
 	int ret, lm;
 	enum mdp5_cursor_alpha cur_alpha = CURSOR_ALPHA_PER_PIXEL;
 	uint32_t flush_mask = mdp_ctl_flush_mask_cursor(0);
@@ -653,7 +644,7 @@ static void mdp5_crtc_err_irq(struct mdp_irq *irq, uint32_t irqstatus)
 {
 	struct mdp5_crtc *mdp5_crtc = container_of(irq, struct mdp5_crtc, err);
 
-	DBG("%s: error: %08x", mdp5_crtc->name, irqstatus);
+	DBG("%s: error: %08x", mdp5_crtc->base.name, irqstatus);
 }
 
 static void mdp5_crtc_pp_done_irq(struct mdp_irq *irq, uint32_t irqstatus)
@@ -775,9 +766,6 @@ struct drm_crtc *mdp5_crtc_init(struct drm_device *dev,
 	mdp5_crtc->vblank.irq = mdp5_crtc_vblank_irq;
 	mdp5_crtc->err.irq = mdp5_crtc_err_irq;
 
-	snprintf(mdp5_crtc->name, sizeof(mdp5_crtc->name), "%s:%d",
-			pipe2name(mdp5_plane_pipe(plane)), id);
-
 	drm_crtc_init_with_planes(dev, crtc, plane, NULL, &mdp5_crtc_funcs,
 				  NULL);
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c
index d53e5510fd7c..3ce8b9dec9c1 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c
@@ -17,6 +17,8 @@
 
 #include <linux/irq.h>
 
+#include <drm/drm_print.h>
+
 #include "msm_drv.h"
 #include "mdp5_kms.h"
 
@@ -30,7 +32,18 @@ void mdp5_set_irqmask(struct mdp_kms *mdp_kms, uint32_t irqmask,
 
 static void mdp5_irq_error_handler(struct mdp_irq *irq, uint32_t irqstatus)
 {
+	struct mdp5_kms *mdp5_kms = container_of(irq, struct mdp5_kms, error_handler);
+	static DEFINE_RATELIMIT_STATE(rs, 5*HZ, 1);
+	extern bool dumpstate;
+
 	DRM_ERROR_RATELIMITED("errors: %08x\n", irqstatus);
+
+	if (dumpstate && __ratelimit(&rs)) {
+		struct drm_printer p = drm_info_printer(mdp5_kms->dev->dev);
+		drm_state_dump(mdp5_kms->dev, &p);
+		if (mdp5_kms->smp)
+			mdp5_smp_dump(mdp5_kms->smp, &p);
+	}
 }
 
 void mdp5_irq_preinstall(struct msm_kms *kms)
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
index ed7143d35b25..5f6cd8745dbc 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
@@ -19,6 +19,7 @@
 #include <linux/of_irq.h>
 
 #include "msm_drv.h"
+#include "msm_gem.h"
 #include "msm_mmu.h"
 #include "mdp5_kms.h"
 
@@ -71,10 +72,49 @@ static int mdp5_hw_init(struct msm_kms *kms)
 	return 0;
 }
 
+struct mdp5_state *mdp5_get_state(struct drm_atomic_state *s)
+{
+	struct msm_drm_private *priv = s->dev->dev_private;
+	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(priv->kms));
+	struct msm_kms_state *state = to_kms_state(s);
+	struct mdp5_state *new_state;
+	int ret;
+
+	if (state->state)
+		return state->state;
+
+	ret = drm_modeset_lock(&mdp5_kms->state_lock, s->acquire_ctx);
+	if (ret)
+		return ERR_PTR(ret);
+
+	new_state = kmalloc(sizeof(*mdp5_kms->state), GFP_KERNEL);
+	if (!new_state)
+		return ERR_PTR(-ENOMEM);
+
+	/* Copy state: */
+	new_state->hwpipe = mdp5_kms->state->hwpipe;
+	if (mdp5_kms->smp)
+		new_state->smp = mdp5_kms->state->smp;
+
+	state->state = new_state;
+
+	return new_state;
+}
+
+static void mdp5_swap_state(struct msm_kms *kms, struct drm_atomic_state *state)
+{
+	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
+	swap(to_kms_state(state)->state, mdp5_kms->state);
+}
+
 static void mdp5_prepare_commit(struct msm_kms *kms, struct drm_atomic_state *state)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
+
 	mdp5_enable(mdp5_kms);
+
+	if (mdp5_kms->smp)
+		mdp5_smp_prepare_commit(mdp5_kms->smp, &mdp5_kms->state->smp);
 }
 
 static void mdp5_complete_commit(struct msm_kms *kms, struct drm_atomic_state *state)
@@ -87,6 +127,9 @@ static void mdp5_complete_commit(struct msm_kms *kms, struct drm_atomic_state *s
 	for_each_plane_in_state(state, plane, plane_state, i)
 		mdp5_plane_complete_commit(plane, plane_state);
 
+	if (mdp5_kms->smp)
+		mdp5_smp_complete_commit(mdp5_kms->smp, &mdp5_kms->state->smp);
+
 	mdp5_disable(mdp5_kms);
 }
 
@@ -117,14 +160,66 @@ static int mdp5_set_split_display(struct msm_kms *kms,
 static void mdp5_kms_destroy(struct msm_kms *kms)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
-	struct msm_mmu *mmu = mdp5_kms->mmu;
+	struct msm_gem_address_space *aspace = mdp5_kms->aspace;
+	int i;
+
+	for (i = 0; i < mdp5_kms->num_hwpipes; i++)
+		mdp5_pipe_destroy(mdp5_kms->hwpipes[i]);
 
-	if (mmu) {
-		mmu->funcs->detach(mmu, iommu_ports, ARRAY_SIZE(iommu_ports));
-		mmu->funcs->destroy(mmu);
+	if (aspace) {
+		aspace->mmu->funcs->detach(aspace->mmu,
+				iommu_ports, ARRAY_SIZE(iommu_ports));
+		msm_gem_address_space_destroy(aspace);
 	}
 }
 
+#ifdef CONFIG_DEBUG_FS
+static int smp_show(struct seq_file *m, void *arg)
+{
+	struct drm_info_node *node = (struct drm_info_node *) m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct msm_drm_private *priv = dev->dev_private;
+	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(priv->kms));
+	struct drm_printer p = drm_seq_file_printer(m);
+
+	if (!mdp5_kms->smp) {
+		drm_printf(&p, "no SMP pool\n");
+		return 0;
+	}
+
+	mdp5_smp_dump(mdp5_kms->smp, &p);
+
+	return 0;
+}
+
+static struct drm_info_list mdp5_debugfs_list[] = {
+		{"smp", smp_show },
+};
+
+static int mdp5_kms_debugfs_init(struct msm_kms *kms, struct drm_minor *minor)
+{
+	struct drm_device *dev = minor->dev;
+	int ret;
+
+	ret = drm_debugfs_create_files(mdp5_debugfs_list,
+			ARRAY_SIZE(mdp5_debugfs_list),
+			minor->debugfs_root, minor);
+
+	if (ret) {
+		dev_err(dev->dev, "could not install mdp5_debugfs_list\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static void mdp5_kms_debugfs_cleanup(struct msm_kms *kms, struct drm_minor *minor)
+{
+	drm_debugfs_remove_files(mdp5_debugfs_list,
+			ARRAY_SIZE(mdp5_debugfs_list), minor);
+}
+#endif
+
 static const struct mdp_kms_funcs kms_funcs = {
 	.base = {
 		.hw_init         = mdp5_hw_init,
@@ -134,6 +229,7 @@ static const struct mdp_kms_funcs kms_funcs = {
 		.irq             = mdp5_irq,
 		.enable_vblank   = mdp5_enable_vblank,
 		.disable_vblank  = mdp5_disable_vblank,
+		.swap_state      = mdp5_swap_state,
 		.prepare_commit  = mdp5_prepare_commit,
 		.complete_commit = mdp5_complete_commit,
 		.wait_for_crtc_commit_done = mdp5_wait_for_crtc_commit_done,
@@ -141,6 +237,10 @@ static const struct mdp_kms_funcs kms_funcs = {
 		.round_pixclk    = mdp5_round_pixclk,
 		.set_split_display = mdp5_set_split_display,
 		.destroy         = mdp5_kms_destroy,
+#ifdef CONFIG_DEBUG_FS
+		.debugfs_init    = mdp5_kms_debugfs_init,
+		.debugfs_cleanup = mdp5_kms_debugfs_cleanup,
+#endif
 	},
 	.set_irqmask         = mdp5_set_irqmask,
 };
@@ -321,15 +421,6 @@ static int modeset_init_intf(struct mdp5_kms *mdp5_kms, int intf_num)
 
 static int modeset_init(struct mdp5_kms *mdp5_kms)
 {
-	static const enum mdp5_pipe crtcs[] = {
-			SSPP_RGB0, SSPP_RGB1, SSPP_RGB2, SSPP_RGB3,
-	};
-	static const enum mdp5_pipe vig_planes[] = {
-			SSPP_VIG0, SSPP_VIG1, SSPP_VIG2, SSPP_VIG3,
-	};
-	static const enum mdp5_pipe dma_planes[] = {
-			SSPP_DMA0, SSPP_DMA1,
-	};
 	struct drm_device *dev = mdp5_kms->dev;
 	struct msm_drm_private *priv = dev->dev_private;
 	const struct mdp5_cfg_hw *hw_cfg;
@@ -337,58 +428,35 @@ static int modeset_init(struct mdp5_kms *mdp5_kms)
 
 	hw_cfg = mdp5_cfg_get_hw_config(mdp5_kms->cfg);
 
-	/* construct CRTCs and their private planes: */
-	for (i = 0; i < hw_cfg->pipe_rgb.count; i++) {
+	/* Construct planes equaling the number of hw pipes, and CRTCs
+	 * for the N layer-mixers (LM).  The first N planes become primary
+	 * planes for the CRTCs, with the remainder as overlay planes:
+	 */
+	for (i = 0; i < mdp5_kms->num_hwpipes; i++) {
+		bool primary = i < mdp5_cfg->lm.count;
 		struct drm_plane *plane;
 		struct drm_crtc *crtc;
 
-		plane = mdp5_plane_init(dev, crtcs[i], true,
-			hw_cfg->pipe_rgb.base[i], hw_cfg->pipe_rgb.caps);
+		plane = mdp5_plane_init(dev, primary);
 		if (IS_ERR(plane)) {
 			ret = PTR_ERR(plane);
-			dev_err(dev->dev, "failed to construct plane for %s (%d)\n",
-					pipe2name(crtcs[i]), ret);
+			dev_err(dev->dev, "failed to construct plane %d (%d)\n", i, ret);
 			goto fail;
 		}
+		priv->planes[priv->num_planes++] = plane;
+
+		if (!primary)
+			continue;
 
 		crtc  = mdp5_crtc_init(dev, plane, i);
 		if (IS_ERR(crtc)) {
 			ret = PTR_ERR(crtc);
-			dev_err(dev->dev, "failed to construct crtc for %s (%d)\n",
-					pipe2name(crtcs[i]), ret);
+			dev_err(dev->dev, "failed to construct crtc %d (%d)\n", i, ret);
 			goto fail;
 		}
 		priv->crtcs[priv->num_crtcs++] = crtc;
 	}
 
-	/* Construct video planes: */
-	for (i = 0; i < hw_cfg->pipe_vig.count; i++) {
-		struct drm_plane *plane;
-
-		plane = mdp5_plane_init(dev, vig_planes[i], false,
-			hw_cfg->pipe_vig.base[i], hw_cfg->pipe_vig.caps);
-		if (IS_ERR(plane)) {
-			ret = PTR_ERR(plane);
-			dev_err(dev->dev, "failed to construct %s plane: %d\n",
-					pipe2name(vig_planes[i]), ret);
-			goto fail;
-		}
-	}
-
-	/* DMA planes */
-	for (i = 0; i < hw_cfg->pipe_dma.count; i++) {
-		struct drm_plane *plane;
-
-		plane = mdp5_plane_init(dev, dma_planes[i], false,
-				hw_cfg->pipe_dma.base[i], hw_cfg->pipe_dma.caps);
-		if (IS_ERR(plane)) {
-			ret = PTR_ERR(plane);
-			dev_err(dev->dev, "failed to construct %s plane: %d\n",
-					pipe2name(dma_planes[i]), ret);
-			goto fail;
-		}
-	}
-
 	/* Construct encoders and modeset initialize connector devices
 	 * for each external display interface.
 	 */
@@ -564,7 +632,7 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 	struct mdp5_kms *mdp5_kms;
 	struct mdp5_cfg *config;
 	struct msm_kms *kms;
-	struct msm_mmu *mmu;
+	struct msm_gem_address_space *aspace;
 	int irq, i, ret;
 
 	/* priv->kms would have been populated by the MDP5 driver */
@@ -606,30 +674,29 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 	mdelay(16);
 
 	if (config->platform.iommu) {
-		mmu = msm_iommu_new(&pdev->dev, config->platform.iommu);
-		if (IS_ERR(mmu)) {
-			ret = PTR_ERR(mmu);
-			dev_err(&pdev->dev, "failed to init iommu: %d\n", ret);
-			iommu_domain_free(config->platform.iommu);
+		aspace = msm_gem_address_space_create(&pdev->dev,
+				config->platform.iommu, "mdp5");
+		if (IS_ERR(aspace)) {
+			ret = PTR_ERR(aspace);
 			goto fail;
 		}
 
-		ret = mmu->funcs->attach(mmu, iommu_ports,
+		mdp5_kms->aspace = aspace;
+
+		ret = aspace->mmu->funcs->attach(aspace->mmu, iommu_ports,
 				ARRAY_SIZE(iommu_ports));
 		if (ret) {
 			dev_err(&pdev->dev, "failed to attach iommu: %d\n",
 				ret);
-			mmu->funcs->destroy(mmu);
 			goto fail;
 		}
 	} else {
 		dev_info(&pdev->dev,
 			 "no iommu, fallback to phys contig buffers for scanout\n");
-		mmu = NULL;
+		aspace = NULL;;
 	}
-	mdp5_kms->mmu = mmu;
 
-	mdp5_kms->id = msm_register_mmu(dev, mmu);
+	mdp5_kms->id = msm_register_address_space(dev, aspace);
 	if (mdp5_kms->id < 0) {
 		ret = mdp5_kms->id;
 		dev_err(&pdev->dev, "failed to register mdp5 iommu: %d\n", ret);
@@ -644,8 +711,8 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 
 	dev->mode_config.min_width = 0;
 	dev->mode_config.min_height = 0;
-	dev->mode_config.max_width = config->hw->lm.max_width;
-	dev->mode_config.max_height = config->hw->lm.max_height;
+	dev->mode_config.max_width = 0xffff;
+	dev->mode_config.max_height = 0xffff;
 
 	dev->driver->get_vblank_timestamp = mdp5_get_vblank_timestamp;
 	dev->driver->get_scanout_position = mdp5_get_scanoutpos;
@@ -673,6 +740,69 @@ static void mdp5_destroy(struct platform_device *pdev)
 
 	if (mdp5_kms->rpm_enabled)
 		pm_runtime_disable(&pdev->dev);
+
+	kfree(mdp5_kms->state);
+}
+
+static int construct_pipes(struct mdp5_kms *mdp5_kms, int cnt,
+		const enum mdp5_pipe *pipes, const uint32_t *offsets,
+		uint32_t caps)
+{
+	struct drm_device *dev = mdp5_kms->dev;
+	int i, ret;
+
+	for (i = 0; i < cnt; i++) {
+		struct mdp5_hw_pipe *hwpipe;
+
+		hwpipe = mdp5_pipe_init(pipes[i], offsets[i], caps);
+		if (IS_ERR(hwpipe)) {
+			ret = PTR_ERR(hwpipe);
+			dev_err(dev->dev, "failed to construct pipe for %s (%d)\n",
+					pipe2name(pipes[i]), ret);
+			return ret;
+		}
+		hwpipe->idx = mdp5_kms->num_hwpipes;
+		mdp5_kms->hwpipes[mdp5_kms->num_hwpipes++] = hwpipe;
+	}
+
+	return 0;
+}
+
+static int hwpipe_init(struct mdp5_kms *mdp5_kms)
+{
+	static const enum mdp5_pipe rgb_planes[] = {
+			SSPP_RGB0, SSPP_RGB1, SSPP_RGB2, SSPP_RGB3,
+	};
+	static const enum mdp5_pipe vig_planes[] = {
+			SSPP_VIG0, SSPP_VIG1, SSPP_VIG2, SSPP_VIG3,
+	};
+	static const enum mdp5_pipe dma_planes[] = {
+			SSPP_DMA0, SSPP_DMA1,
+	};
+	const struct mdp5_cfg_hw *hw_cfg;
+	int ret;
+
+	hw_cfg = mdp5_cfg_get_hw_config(mdp5_kms->cfg);
+
+	/* Construct RGB pipes: */
+	ret = construct_pipes(mdp5_kms, hw_cfg->pipe_rgb.count, rgb_planes,
+			hw_cfg->pipe_rgb.base, hw_cfg->pipe_rgb.caps);
+	if (ret)
+		return ret;
+
+	/* Construct video (VIG) pipes: */
+	ret = construct_pipes(mdp5_kms, hw_cfg->pipe_vig.count, vig_planes,
+			hw_cfg->pipe_vig.base, hw_cfg->pipe_vig.caps);
+	if (ret)
+		return ret;
+
+	/* Construct DMA pipes: */
+	ret = construct_pipes(mdp5_kms, hw_cfg->pipe_dma.count, dma_planes,
+			hw_cfg->pipe_dma.base, hw_cfg->pipe_dma.caps);
+	if (ret)
+		return ret;
+
+	return 0;
 }
 
 static int mdp5_init(struct platform_device *pdev, struct drm_device *dev)
@@ -696,6 +826,13 @@ static int mdp5_init(struct platform_device *pdev, struct drm_device *dev)
 	mdp5_kms->dev = dev;
 	mdp5_kms->pdev = pdev;
 
+	drm_modeset_lock_init(&mdp5_kms->state_lock);
+	mdp5_kms->state = kzalloc(sizeof(*mdp5_kms->state), GFP_KERNEL);
+	if (!mdp5_kms->state) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
 	mdp5_kms->mmio = msm_ioremap(pdev, "mdp_phys", "MDP5");
 	if (IS_ERR(mdp5_kms->mmio)) {
 		ret = PTR_ERR(mdp5_kms->mmio);
@@ -749,7 +886,7 @@ static int mdp5_init(struct platform_device *pdev, struct drm_device *dev)
 	 * this section initializes the SMP:
 	 */
 	if (mdp5_kms->caps & MDP_CAP_SMP) {
-		mdp5_kms->smp = mdp5_smp_init(mdp5_kms->dev, &config->hw->smp);
+		mdp5_kms->smp = mdp5_smp_init(mdp5_kms, &config->hw->smp);
 		if (IS_ERR(mdp5_kms->smp)) {
 			ret = PTR_ERR(mdp5_kms->smp);
 			mdp5_kms->smp = NULL;
@@ -764,6 +901,10 @@ static int mdp5_init(struct platform_device *pdev, struct drm_device *dev)
 		goto fail;
 	}
 
+	ret = hwpipe_init(mdp5_kms);
+	if (ret)
+		goto fail;
+
 	/* set uninit-ed kms */
 	priv->kms = &mdp5_kms->base.base;
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h
index 03738927be10..17b0cc101171 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h
@@ -24,8 +24,11 @@
 #include "mdp5_cfg.h"	/* must be included before mdp5.xml.h */
 #include "mdp5.xml.h"
 #include "mdp5_ctl.h"
+#include "mdp5_pipe.h"
 #include "mdp5_smp.h"
 
+struct mdp5_state;
+
 struct mdp5_kms {
 	struct mdp_kms base;
 
@@ -33,13 +36,21 @@ struct mdp5_kms {
 
 	struct platform_device *pdev;
 
+	unsigned num_hwpipes;
+	struct mdp5_hw_pipe *hwpipes[SSPP_MAX];
+
 	struct mdp5_cfg_handler *cfg;
 	uint32_t caps;	/* MDP capabilities (MDP_CAP_XXX bits) */
 
+	/**
+	 * Global atomic state.  Do not access directly, use mdp5_get_state()
+	 */
+	struct mdp5_state *state;
+	struct drm_modeset_lock state_lock;
 
 	/* mapper-id used to request GEM buffer mapped for scanout: */
 	int id;
-	struct msm_mmu *mmu;
+	struct msm_gem_address_space *aspace;
 
 	struct mdp5_smp *smp;
 	struct mdp5_ctl_manager *ctlm;
@@ -65,9 +76,27 @@ struct mdp5_kms {
 };
 #define to_mdp5_kms(x) container_of(x, struct mdp5_kms, base)
 
+/* Global atomic state for tracking resources that are shared across
+ * multiple kms objects (planes/crtcs/etc).
+ *
+ * For atomic updates which require modifying global state,
+ */
+struct mdp5_state {
+	struct mdp5_hw_pipe_state hwpipe;
+	struct mdp5_smp_state smp;
+};
+
+struct mdp5_state *__must_check
+mdp5_get_state(struct drm_atomic_state *s);
+
+/* Atomic plane state.  Subclasses the base drm_plane_state in order to
+ * track assigned hwpipe and hw specific state.
+ */
 struct mdp5_plane_state {
 	struct drm_plane_state base;
 
+	struct mdp5_hw_pipe *hwpipe;
+
 	/* aligned with property */
 	uint8_t premultiplied;
 	uint8_t zpos;
@@ -76,11 +105,6 @@ struct mdp5_plane_state {
 	/* assigned by crtc blender */
 	enum mdp_mixer_stage_id stage;
 
-	/* some additional transactional status to help us know in the
-	 * apply path whether we need to update SMP allocation, and
-	 * whether current update is still pending:
-	 */
-	bool mode_changed : 1;
 	bool pending : 1;
 };
 #define to_mdp5_plane_state(x) \
@@ -114,6 +138,18 @@ static inline u32 mdp5_read(struct mdp5_kms *mdp5_kms, u32 reg)
 	return msm_readl(mdp5_kms->mmio + reg);
 }
 
+static inline const char *stage2name(enum mdp_mixer_stage_id stage)
+{
+	static const char *names[] = {
+#define NAME(n) [n] = #n
+		NAME(STAGE_UNUSED), NAME(STAGE_BASE),
+		NAME(STAGE0), NAME(STAGE1), NAME(STAGE2),
+		NAME(STAGE3), NAME(STAGE4), NAME(STAGE6),
+#undef NAME
+	};
+	return names[stage];
+}
+
 static inline const char *pipe2name(enum mdp5_pipe pipe)
 {
 	static const char *names[] = {
@@ -196,13 +232,10 @@ int mdp5_irq_domain_init(struct mdp5_kms *mdp5_kms);
 void mdp5_irq_domain_fini(struct mdp5_kms *mdp5_kms);
 
 uint32_t mdp5_plane_get_flush(struct drm_plane *plane);
-void mdp5_plane_complete_flip(struct drm_plane *plane);
 void mdp5_plane_complete_commit(struct drm_plane *plane,
 	struct drm_plane_state *state);
 enum mdp5_pipe mdp5_plane_pipe(struct drm_plane *plane);
-struct drm_plane *mdp5_plane_init(struct drm_device *dev,
-		enum mdp5_pipe pipe, bool private_plane,
-		uint32_t reg_offset, uint32_t caps);
+struct drm_plane *mdp5_plane_init(struct drm_device *dev, bool primary);
 
 uint32_t mdp5_crtc_vblank(struct drm_crtc *crtc);
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_pipe.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_pipe.c
new file mode 100644
index 000000000000..1ae9dc8d260d
--- /dev/null
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_pipe.c
@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2016 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mdp5_kms.h"
+
+struct mdp5_hw_pipe *mdp5_pipe_assign(struct drm_atomic_state *s,
+		struct drm_plane *plane, uint32_t caps, uint32_t blkcfg)
+{
+	struct msm_drm_private *priv = s->dev->dev_private;
+	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(priv->kms));
+	struct mdp5_state *state;
+	struct mdp5_hw_pipe_state *old_state, *new_state;
+	struct mdp5_hw_pipe *hwpipe = NULL;
+	int i;
+
+	state = mdp5_get_state(s);
+	if (IS_ERR(state))
+		return ERR_CAST(state);
+
+	/* grab old_state after mdp5_get_state(), since now we hold lock: */
+	old_state = &mdp5_kms->state->hwpipe;
+	new_state = &state->hwpipe;
+
+	for (i = 0; i < mdp5_kms->num_hwpipes; i++) {
+		struct mdp5_hw_pipe *cur = mdp5_kms->hwpipes[i];
+
+		/* skip if already in-use.. check both new and old state,
+		 * since we cannot immediately re-use a pipe that is
+		 * released in the current update in some cases:
+		 *  (1) mdp5 can have SMP (non-double-buffered)
+		 *  (2) hw pipe previously assigned to different CRTC
+		 *      (vblanks might not be aligned)
+		 */
+		if (new_state->hwpipe_to_plane[cur->idx] ||
+				old_state->hwpipe_to_plane[cur->idx])
+			continue;
+
+		/* skip if doesn't support some required caps: */
+		if (caps & ~cur->caps)
+			continue;
+
+		/* possible candidate, take the one with the
+		 * fewest unneeded caps bits set:
+		 */
+		if (!hwpipe || (hweight_long(cur->caps & ~caps) <
+				hweight_long(hwpipe->caps & ~caps)))
+			hwpipe = cur;
+	}
+
+	if (!hwpipe)
+		return ERR_PTR(-ENOMEM);
+
+	if (mdp5_kms->smp) {
+		int ret;
+
+		DBG("%s: alloc SMP blocks", hwpipe->name);
+		ret = mdp5_smp_assign(mdp5_kms->smp, &state->smp,
+				hwpipe->pipe, blkcfg);
+		if (ret)
+			return ERR_PTR(-ENOMEM);
+
+		hwpipe->blkcfg = blkcfg;
+	}
+
+	DBG("%s: assign to plane %s for caps %x",
+			hwpipe->name, plane->name, caps);
+	new_state->hwpipe_to_plane[hwpipe->idx] = plane;
+
+	return hwpipe;
+}
+
+void mdp5_pipe_release(struct drm_atomic_state *s, struct mdp5_hw_pipe *hwpipe)
+{
+	struct msm_drm_private *priv = s->dev->dev_private;
+	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(priv->kms));
+	struct mdp5_state *state = mdp5_get_state(s);
+	struct mdp5_hw_pipe_state *new_state = &state->hwpipe;
+
+	if (!hwpipe)
+		return;
+
+	if (WARN_ON(!new_state->hwpipe_to_plane[hwpipe->idx]))
+		return;
+
+	DBG("%s: release from plane %s", hwpipe->name,
+		new_state->hwpipe_to_plane[hwpipe->idx]->name);
+
+	if (mdp5_kms->smp) {
+		DBG("%s: free SMP blocks", hwpipe->name);
+		mdp5_smp_release(mdp5_kms->smp, &state->smp, hwpipe->pipe);
+	}
+
+	new_state->hwpipe_to_plane[hwpipe->idx] = NULL;
+}
+
+void mdp5_pipe_destroy(struct mdp5_hw_pipe *hwpipe)
+{
+	kfree(hwpipe);
+}
+
+struct mdp5_hw_pipe *mdp5_pipe_init(enum mdp5_pipe pipe,
+		uint32_t reg_offset, uint32_t caps)
+{
+	struct mdp5_hw_pipe *hwpipe;
+
+	hwpipe = kzalloc(sizeof(*hwpipe), GFP_KERNEL);
+	if (!hwpipe)
+		return ERR_PTR(-ENOMEM);
+
+	hwpipe->name = pipe2name(pipe);
+	hwpipe->pipe = pipe;
+	hwpipe->reg_offset = reg_offset;
+	hwpipe->caps = caps;
+	hwpipe->flush_mask = mdp_ctl_flush_mask_pipe(pipe);
+
+	spin_lock_init(&hwpipe->pipe_lock);
+
+	return hwpipe;
+}
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_pipe.h b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_pipe.h
new file mode 100644
index 000000000000..611da7a660c9
--- /dev/null
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_pipe.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2016 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __MDP5_PIPE_H__
+#define __MDP5_PIPE_H__
+
+#define SSPP_MAX	(SSPP_RGB3 + 1) /* TODO: Add SSPP_MAX in mdp5.xml.h */
+
+/* represents a hw pipe, which is dynamically assigned to a plane */
+struct mdp5_hw_pipe {
+	int idx;
+
+	const char *name;
+	enum mdp5_pipe pipe;
+
+	spinlock_t pipe_lock;     /* protect REG_MDP5_PIPE_* registers */
+	uint32_t reg_offset;
+	uint32_t caps;
+
+	uint32_t flush_mask;      /* used to commit pipe registers */
+
+	/* number of smp blocks per plane, ie:
+	 *   nblks_y | (nblks_u << 8) | (nblks_v << 16)
+	 */
+	uint32_t blkcfg;
+};
+
+/* global atomic state of assignment between pipes and planes: */
+struct mdp5_hw_pipe_state {
+	struct drm_plane *hwpipe_to_plane[SSPP_MAX];
+};
+
+struct mdp5_hw_pipe *__must_check
+mdp5_pipe_assign(struct drm_atomic_state *s, struct drm_plane *plane,
+		uint32_t caps, uint32_t blkcfg);
+void mdp5_pipe_release(struct drm_atomic_state *s, struct mdp5_hw_pipe *hwpipe);
+
+struct mdp5_hw_pipe *mdp5_pipe_init(enum mdp5_pipe pipe,
+		uint32_t reg_offset, uint32_t caps);
+void mdp5_pipe_destroy(struct mdp5_hw_pipe *hwpipe);
+
+#endif /* __MDP5_PIPE_H__ */
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
index 83bf997dda03..c099da7bc212 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
@@ -16,19 +16,11 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <drm/drm_print.h>
 #include "mdp5_kms.h"
 
 struct mdp5_plane {
 	struct drm_plane base;
-	const char *name;
-
-	enum mdp5_pipe pipe;
-
-	spinlock_t pipe_lock;	/* protect REG_MDP5_PIPE_* registers */
-	uint32_t reg_offset;
-	uint32_t caps;
-
-	uint32_t flush_mask;	/* used to commit pipe registers */
 
 	uint32_t nformats;
 	uint32_t formats[32];
@@ -69,21 +61,12 @@ static void mdp5_plane_destroy(struct drm_plane *plane)
 static void mdp5_plane_install_rotation_property(struct drm_device *dev,
 		struct drm_plane *plane)
 {
-	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
-
-	if (!(mdp5_plane->caps & MDP_PIPE_CAP_HFLIP) &&
-		!(mdp5_plane->caps & MDP_PIPE_CAP_VFLIP))
-		return;
-
-	if (!dev->mode_config.rotation_property)
-		dev->mode_config.rotation_property =
-			drm_mode_create_rotation_property(dev,
-				DRM_ROTATE_0 | DRM_REFLECT_X | DRM_REFLECT_Y);
-
-	if (dev->mode_config.rotation_property)
-		drm_object_attach_property(&plane->base,
-			dev->mode_config.rotation_property,
-			DRM_ROTATE_0);
+	drm_plane_create_rotation_property(plane,
+					   DRM_ROTATE_0,
+					   DRM_ROTATE_0 |
+					   DRM_ROTATE_180 |
+					   DRM_REFLECT_X |
+					   DRM_REFLECT_Y);
 }
 
 /* helper to install properties which are common to planes and crtcs */
@@ -184,6 +167,21 @@ done:
 #undef SET_PROPERTY
 }
 
+static void
+mdp5_plane_atomic_print_state(struct drm_printer *p,
+		const struct drm_plane_state *state)
+{
+	struct mdp5_plane_state *pstate = to_mdp5_plane_state(state);
+
+	drm_printf(p, "\thwpipe=%s\n", pstate->hwpipe ?
+			pstate->hwpipe->name : "(null)");
+	drm_printf(p, "\tpremultiplied=%u\n", pstate->premultiplied);
+	drm_printf(p, "\tzpos=%u\n", pstate->zpos);
+	drm_printf(p, "\talpha=%u\n", pstate->alpha);
+	drm_printf(p, "\tstage=%s\n", stage2name(pstate->stage));
+	drm_printf(p, "\tpending=%u\n", pstate->pending);
+}
+
 static void mdp5_plane_reset(struct drm_plane *plane)
 {
 	struct mdp5_plane_state *mdp5_state;
@@ -222,7 +220,6 @@ mdp5_plane_duplicate_state(struct drm_plane *plane)
 	if (mdp5_state && mdp5_state->base.fb)
 		drm_framebuffer_reference(mdp5_state->base.fb);
 
-	mdp5_state->mode_changed = false;
 	mdp5_state->pending = false;
 
 	return &mdp5_state->base;
@@ -231,10 +228,12 @@ mdp5_plane_duplicate_state(struct drm_plane *plane)
 static void mdp5_plane_destroy_state(struct drm_plane *plane,
 		struct drm_plane_state *state)
 {
+	struct mdp5_plane_state *pstate = to_mdp5_plane_state(state);
+
 	if (state->fb)
 		drm_framebuffer_unreference(state->fb);
 
-	kfree(to_mdp5_plane_state(state));
+	kfree(pstate);
 }
 
 static const struct drm_plane_funcs mdp5_plane_funcs = {
@@ -247,99 +246,121 @@ static const struct drm_plane_funcs mdp5_plane_funcs = {
 		.reset = mdp5_plane_reset,
 		.atomic_duplicate_state = mdp5_plane_duplicate_state,
 		.atomic_destroy_state = mdp5_plane_destroy_state,
+		.atomic_print_state = mdp5_plane_atomic_print_state,
 };
 
 static int mdp5_plane_prepare_fb(struct drm_plane *plane,
 				 struct drm_plane_state *new_state)
 {
-	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
 	struct mdp5_kms *mdp5_kms = get_kms(plane);
 	struct drm_framebuffer *fb = new_state->fb;
 
 	if (!new_state->fb)
 		return 0;
 
-	DBG("%s: prepare: FB[%u]", mdp5_plane->name, fb->base.id);
+	DBG("%s: prepare: FB[%u]", plane->name, fb->base.id);
 	return msm_framebuffer_prepare(fb, mdp5_kms->id);
 }
 
 static void mdp5_plane_cleanup_fb(struct drm_plane *plane,
 				  struct drm_plane_state *old_state)
 {
-	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
 	struct mdp5_kms *mdp5_kms = get_kms(plane);
 	struct drm_framebuffer *fb = old_state->fb;
 
 	if (!fb)
 		return;
 
-	DBG("%s: cleanup: FB[%u]", mdp5_plane->name, fb->base.id);
+	DBG("%s: cleanup: FB[%u]", plane->name, fb->base.id);
 	msm_framebuffer_cleanup(fb, mdp5_kms->id);
 }
 
 static int mdp5_plane_atomic_check(struct drm_plane *plane,
 		struct drm_plane_state *state)
 {
-	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
+	struct mdp5_plane_state *mdp5_state = to_mdp5_plane_state(state);
 	struct drm_plane_state *old_state = plane->state;
-	const struct mdp_format *format;
-	bool vflip, hflip;
+	struct mdp5_cfg *config = mdp5_cfg_get_config(get_kms(plane)->cfg);
+	bool new_hwpipe = false;
+	uint32_t max_width, max_height;
+	uint32_t caps = 0;
 
-	DBG("%s: check (%d -> %d)", mdp5_plane->name,
+	DBG("%s: check (%d -> %d)", plane->name,
 			plane_enabled(old_state), plane_enabled(state));
 
+	/* We don't allow faster-than-vblank updates.. if we did add this
+	 * some day, we would need to disallow in cases where hwpipe
+	 * changes
+	 */
+	if (WARN_ON(to_mdp5_plane_state(old_state)->pending))
+		return -EBUSY;
+
+	max_width = config->hw->lm.max_width << 16;
+	max_height = config->hw->lm.max_height << 16;
+
+	/* Make sure source dimensions are within bounds. */
+	if ((state->src_w > max_width) || (state->src_h > max_height)) {
+		struct drm_rect src = drm_plane_state_src(state);
+		DBG("Invalid source size "DRM_RECT_FP_FMT,
+				DRM_RECT_FP_ARG(&src));
+		return -ERANGE;
+	}
+
 	if (plane_enabled(state)) {
+		unsigned int rotation;
+		const struct mdp_format *format;
+		struct mdp5_kms *mdp5_kms = get_kms(plane);
+		uint32_t blkcfg = 0;
+
 		format = to_mdp_format(msm_framebuffer_format(state->fb));
-		if (MDP_FORMAT_IS_YUV(format) &&
-			!pipe_supports_yuv(mdp5_plane->caps)) {
-			DBG("Pipe doesn't support YUV\n");
+		if (MDP_FORMAT_IS_YUV(format))
+			caps |= MDP_PIPE_CAP_SCALE | MDP_PIPE_CAP_CSC;
 
-			return -EINVAL;
-		}
+		if (((state->src_w >> 16) != state->crtc_w) ||
+				((state->src_h >> 16) != state->crtc_h))
+			caps |= MDP_PIPE_CAP_SCALE;
 
-		if (!(mdp5_plane->caps & MDP_PIPE_CAP_SCALE) &&
-			(((state->src_w >> 16) != state->crtc_w) ||
-			((state->src_h >> 16) != state->crtc_h))) {
-			DBG("Pipe doesn't support scaling (%dx%d -> %dx%d)\n",
-				state->src_w >> 16, state->src_h >> 16,
-				state->crtc_w, state->crtc_h);
+		rotation = drm_rotation_simplify(state->rotation,
+						 DRM_ROTATE_0 |
+						 DRM_REFLECT_X |
+						 DRM_REFLECT_Y);
 
-			return -EINVAL;
-		}
+		if (rotation & DRM_REFLECT_X)
+			caps |= MDP_PIPE_CAP_HFLIP;
 
-		hflip = !!(state->rotation & DRM_REFLECT_X);
-		vflip = !!(state->rotation & DRM_REFLECT_Y);
-		if ((vflip && !(mdp5_plane->caps & MDP_PIPE_CAP_VFLIP)) ||
-			(hflip && !(mdp5_plane->caps & MDP_PIPE_CAP_HFLIP))) {
-			DBG("Pipe doesn't support flip\n");
+		if (rotation & DRM_REFLECT_Y)
+			caps |= MDP_PIPE_CAP_VFLIP;
 
-			return -EINVAL;
-		}
-	}
+		/* (re)allocate hw pipe if we don't have one or caps-mismatch: */
+		if (!mdp5_state->hwpipe || (caps & ~mdp5_state->hwpipe->caps))
+			new_hwpipe = true;
 
-	if (plane_enabled(state) && plane_enabled(old_state)) {
-		/* we cannot change SMP block configuration during scanout: */
-		bool full_modeset = false;
-		if (state->fb->pixel_format != old_state->fb->pixel_format) {
-			DBG("%s: pixel_format change!", mdp5_plane->name);
-			full_modeset = true;
-		}
-		if (state->src_w != old_state->src_w) {
-			DBG("%s: src_w change!", mdp5_plane->name);
-			full_modeset = true;
-		}
-		if (to_mdp5_plane_state(old_state)->pending) {
-			DBG("%s: still pending!", mdp5_plane->name);
-			full_modeset = true;
+		if (mdp5_kms->smp) {
+			const struct mdp_format *format =
+				to_mdp_format(msm_framebuffer_format(state->fb));
+
+			blkcfg = mdp5_smp_calculate(mdp5_kms->smp, format,
+					state->src_w >> 16, false);
+
+			if (mdp5_state->hwpipe && (mdp5_state->hwpipe->blkcfg != blkcfg))
+				new_hwpipe = true;
 		}
-		if (full_modeset) {
-			struct drm_crtc_state *crtc_state =
-					drm_atomic_get_crtc_state(state->state, state->crtc);
-			crtc_state->mode_changed = true;
-			to_mdp5_plane_state(state)->mode_changed = true;
+
+		/* (re)assign hwpipe if needed, otherwise keep old one: */
+		if (new_hwpipe) {
+			/* TODO maybe we want to re-assign hwpipe sometimes
+			 * in cases when we no-longer need some caps to make
+			 * it available for other planes?
+			 */
+			struct mdp5_hw_pipe *old_hwpipe = mdp5_state->hwpipe;
+			mdp5_state->hwpipe = mdp5_pipe_assign(state->state,
+					plane, caps, blkcfg);
+			if (IS_ERR(mdp5_state->hwpipe)) {
+				DBG("%s: failed to assign hwpipe!", plane->name);
+				return PTR_ERR(mdp5_state->hwpipe);
+			}
+			mdp5_pipe_release(state->state, old_hwpipe);
 		}
-	} else {
-		to_mdp5_plane_state(state)->mode_changed = true;
 	}
 
 	return 0;
@@ -348,16 +369,16 @@ static int mdp5_plane_atomic_check(struct drm_plane *plane,
 static void mdp5_plane_atomic_update(struct drm_plane *plane,
 				     struct drm_plane_state *old_state)
 {
-	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
 	struct drm_plane_state *state = plane->state;
+	struct mdp5_plane_state *mdp5_state = to_mdp5_plane_state(state);
 
-	DBG("%s: update", mdp5_plane->name);
+	DBG("%s: update", plane->name);
 
-	if (!plane_enabled(state)) {
-		to_mdp5_plane_state(state)->pending = true;
-	} else if (to_mdp5_plane_state(state)->mode_changed) {
+	mdp5_state->pending = true;
+
+	if (plane_enabled(state)) {
 		int ret;
-		to_mdp5_plane_state(state)->pending = true;
+
 		ret = mdp5_plane_mode_set(plane,
 				state->crtc, state->fb,
 				state->crtc_x, state->crtc_y,
@@ -366,11 +387,6 @@ static void mdp5_plane_atomic_update(struct drm_plane *plane,
 				state->src_w, state->src_h);
 		/* atomic_check should have ensured that this doesn't fail */
 		WARN_ON(ret < 0);
-	} else {
-		unsigned long flags;
-		spin_lock_irqsave(&mdp5_plane->pipe_lock, flags);
-		set_scanout_locked(plane, state->fb);
-		spin_unlock_irqrestore(&mdp5_plane->pipe_lock, flags);
 	}
 }
 
@@ -384,9 +400,9 @@ static const struct drm_plane_helper_funcs mdp5_plane_helper_funcs = {
 static void set_scanout_locked(struct drm_plane *plane,
 		struct drm_framebuffer *fb)
 {
-	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
 	struct mdp5_kms *mdp5_kms = get_kms(plane);
-	enum mdp5_pipe pipe = mdp5_plane->pipe;
+	struct mdp5_hw_pipe *hwpipe = to_mdp5_plane_state(plane->state)->hwpipe;
+	enum mdp5_pipe pipe = hwpipe->pipe;
 
 	mdp5_write(mdp5_kms, REG_MDP5_PIPE_SRC_STRIDE_A(pipe),
 			MDP5_PIPE_SRC_STRIDE_A_P0(fb->pitches[0]) |
@@ -666,18 +682,19 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 		uint32_t src_x, uint32_t src_y,
 		uint32_t src_w, uint32_t src_h)
 {
-	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
 	struct drm_plane_state *pstate = plane->state;
+	struct mdp5_hw_pipe *hwpipe = to_mdp5_plane_state(pstate)->hwpipe;
 	struct mdp5_kms *mdp5_kms = get_kms(plane);
-	enum mdp5_pipe pipe = mdp5_plane->pipe;
+	enum mdp5_pipe pipe = hwpipe->pipe;
 	const struct mdp_format *format;
 	uint32_t nplanes, config = 0;
 	uint32_t phasex_step[COMP_MAX] = {0,}, phasey_step[COMP_MAX] = {0,};
-	bool pe = mdp5_plane->caps & MDP_PIPE_CAP_SW_PIX_EXT;
+	bool pe = hwpipe->caps & MDP_PIPE_CAP_SW_PIX_EXT;
 	int pe_left[COMP_MAX], pe_right[COMP_MAX];
 	int pe_top[COMP_MAX], pe_bottom[COMP_MAX];
 	uint32_t hdecm = 0, vdecm = 0;
 	uint32_t pix_format;
+	unsigned int rotation;
 	bool vflip, hflip;
 	unsigned long flags;
 	int ret;
@@ -697,27 +714,10 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 	src_w = src_w >> 16;
 	src_h = src_h >> 16;
 
-	DBG("%s: FB[%u] %u,%u,%u,%u -> CRTC[%u] %d,%d,%u,%u", mdp5_plane->name,
+	DBG("%s: FB[%u] %u,%u,%u,%u -> CRTC[%u] %d,%d,%u,%u", plane->name,
 			fb->base.id, src_x, src_y, src_w, src_h,
 			crtc->base.id, crtc_x, crtc_y, crtc_w, crtc_h);
 
-	/* Request some memory from the SMP: */
-	if (mdp5_kms->smp) {
-		ret = mdp5_smp_request(mdp5_kms->smp,
-				mdp5_plane->pipe, format, src_w, false);
-		if (ret)
-			return ret;
-	}
-
-	/*
-	 * Currently we update the hw for allocations/requests immediately,
-	 * but once atomic modeset/pageflip is in place, the allocation
-	 * would move into atomic->check_plane_state(), while updating the
-	 * hw would remain here:
-	 */
-	if (mdp5_kms->smp)
-		mdp5_smp_configure(mdp5_kms->smp, pipe);
-
 	ret = calc_scalex_steps(plane, pix_format, src_w, crtc_w, phasex_step);
 	if (ret)
 		return ret;
@@ -726,7 +726,7 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 	if (ret)
 		return ret;
 
-	if (mdp5_plane->caps & MDP_PIPE_CAP_SW_PIX_EXT) {
+	if (hwpipe->caps & MDP_PIPE_CAP_SW_PIX_EXT) {
 		calc_pixel_ext(format, src_w, crtc_w, phasex_step,
 					 pe_left, pe_right, true);
 		calc_pixel_ext(format, src_h, crtc_h, phasey_step,
@@ -740,14 +740,18 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 	config |= get_scale_config(format, src_h, crtc_h, false);
 	DBG("scale config = %x", config);
 
-	hflip = !!(pstate->rotation & DRM_REFLECT_X);
-	vflip = !!(pstate->rotation & DRM_REFLECT_Y);
+	rotation = drm_rotation_simplify(pstate->rotation,
+					 DRM_ROTATE_0 |
+					 DRM_REFLECT_X |
+					 DRM_REFLECT_Y);
+	hflip = !!(rotation & DRM_REFLECT_X);
+	vflip = !!(rotation & DRM_REFLECT_Y);
 
-	spin_lock_irqsave(&mdp5_plane->pipe_lock, flags);
+	spin_lock_irqsave(&hwpipe->pipe_lock, flags);
 
 	mdp5_write(mdp5_kms, REG_MDP5_PIPE_SRC_IMG_SIZE(pipe),
-			MDP5_PIPE_SRC_IMG_SIZE_WIDTH(fb->width) |
-			MDP5_PIPE_SRC_IMG_SIZE_HEIGHT(fb->height));
+			MDP5_PIPE_SRC_IMG_SIZE_WIDTH(min(fb->width, src_w)) |
+			MDP5_PIPE_SRC_IMG_SIZE_HEIGHT(min(fb->height, src_h)));
 
 	mdp5_write(mdp5_kms, REG_MDP5_PIPE_SRC_SIZE(pipe),
 			MDP5_PIPE_SRC_SIZE_WIDTH(src_w) |
@@ -792,12 +796,12 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 	/* not using secure mode: */
 	mdp5_write(mdp5_kms, REG_MDP5_PIPE_SRC_ADDR_SW_STATUS(pipe), 0);
 
-	if (mdp5_plane->caps & MDP_PIPE_CAP_SW_PIX_EXT)
+	if (hwpipe->caps & MDP_PIPE_CAP_SW_PIX_EXT)
 		mdp5_write_pixel_ext(mdp5_kms, pipe, format,
 				src_w, pe_left, pe_right,
 				src_h, pe_top, pe_bottom);
 
-	if (mdp5_plane->caps & MDP_PIPE_CAP_SCALE) {
+	if (hwpipe->caps & MDP_PIPE_CAP_SCALE) {
 		mdp5_write(mdp5_kms, REG_MDP5_PIPE_SCALE_PHASE_STEP_X(pipe),
 				phasex_step[COMP_0]);
 		mdp5_write(mdp5_kms, REG_MDP5_PIPE_SCALE_PHASE_STEP_Y(pipe),
@@ -812,7 +816,7 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 		mdp5_write(mdp5_kms, REG_MDP5_PIPE_SCALE_CONFIG(pipe), config);
 	}
 
-	if (mdp5_plane->caps & MDP_PIPE_CAP_CSC) {
+	if (hwpipe->caps & MDP_PIPE_CAP_CSC) {
 		if (MDP_FORMAT_IS_YUV(format))
 			csc_enable(mdp5_kms, pipe,
 					mdp_get_default_csc_cfg(CSC_YUV2RGB));
@@ -822,56 +826,42 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 
 	set_scanout_locked(plane, fb);
 
-	spin_unlock_irqrestore(&mdp5_plane->pipe_lock, flags);
+	spin_unlock_irqrestore(&hwpipe->pipe_lock, flags);
 
 	return ret;
 }
 
-void mdp5_plane_complete_flip(struct drm_plane *plane)
+enum mdp5_pipe mdp5_plane_pipe(struct drm_plane *plane)
 {
-	struct mdp5_kms *mdp5_kms = get_kms(plane);
-	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
-	enum mdp5_pipe pipe = mdp5_plane->pipe;
+	struct mdp5_plane_state *pstate = to_mdp5_plane_state(plane->state);
 
-	DBG("%s: complete flip", mdp5_plane->name);
-
-	if (mdp5_kms->smp)
-		mdp5_smp_commit(mdp5_kms->smp, pipe);
-
-	to_mdp5_plane_state(plane->state)->pending = false;
-}
+	if (WARN_ON(!pstate->hwpipe))
+		return 0;
 
-enum mdp5_pipe mdp5_plane_pipe(struct drm_plane *plane)
-{
-	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
-	return mdp5_plane->pipe;
+	return pstate->hwpipe->pipe;
 }
 
 uint32_t mdp5_plane_get_flush(struct drm_plane *plane)
 {
-	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
+	struct mdp5_plane_state *pstate = to_mdp5_plane_state(plane->state);
 
-	return mdp5_plane->flush_mask;
+	if (WARN_ON(!pstate->hwpipe))
+		return 0;
+
+	return pstate->hwpipe->flush_mask;
 }
 
 /* called after vsync in thread context */
 void mdp5_plane_complete_commit(struct drm_plane *plane,
 	struct drm_plane_state *state)
 {
-	struct mdp5_kms *mdp5_kms = get_kms(plane);
-	struct mdp5_plane *mdp5_plane = to_mdp5_plane(plane);
-	enum mdp5_pipe pipe = mdp5_plane->pipe;
+	struct mdp5_plane_state *pstate = to_mdp5_plane_state(plane->state);
 
-	if (!plane_enabled(plane->state) && mdp5_kms->smp) {
-		DBG("%s: free SMP", mdp5_plane->name);
-		mdp5_smp_release(mdp5_kms->smp, pipe);
-	}
+	pstate->pending = false;
 }
 
 /* initialize plane */
-struct drm_plane *mdp5_plane_init(struct drm_device *dev,
-		enum mdp5_pipe pipe, bool private_plane, uint32_t reg_offset,
-		uint32_t caps)
+struct drm_plane *mdp5_plane_init(struct drm_device *dev, bool primary)
 {
 	struct drm_plane *plane = NULL;
 	struct mdp5_plane *mdp5_plane;
@@ -886,19 +876,10 @@ struct drm_plane *mdp5_plane_init(struct drm_device *dev,
 
 	plane = &mdp5_plane->base;
 
-	mdp5_plane->pipe = pipe;
-	mdp5_plane->name = pipe2name(pipe);
-	mdp5_plane->caps = caps;
-
 	mdp5_plane->nformats = mdp_get_formats(mdp5_plane->formats,
-		ARRAY_SIZE(mdp5_plane->formats),
-		!pipe_supports_yuv(mdp5_plane->caps));
-
-	mdp5_plane->flush_mask = mdp_ctl_flush_mask_pipe(pipe);
-	mdp5_plane->reg_offset = reg_offset;
-	spin_lock_init(&mdp5_plane->pipe_lock);
+		ARRAY_SIZE(mdp5_plane->formats), false);
 
-	type = private_plane ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
+	type = primary ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
 	ret = drm_universal_plane_init(dev, plane, 0xff, &mdp5_plane_funcs,
 				 mdp5_plane->formats, mdp5_plane->nformats,
 				 type, NULL);
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c
index 27d7b55b52c9..58f712d37e7f 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c
@@ -21,72 +21,6 @@
 #include "mdp5_smp.h"
 
 
-/* SMP - Shared Memory Pool
- *
- * These are shared between all the clients, where each plane in a
- * scanout buffer is a SMP client.  Ie. scanout of 3 plane I420 on
- * pipe VIG0 => 3 clients: VIG0_Y, VIG0_CB, VIG0_CR.
- *
- * Based on the size of the attached scanout buffer, a certain # of
- * blocks must be allocated to that client out of the shared pool.
- *
- * In some hw, some blocks are statically allocated for certain pipes
- * and CANNOT be re-allocated (eg: MMB0 and MMB1 both tied to RGB0).
- *
- * For each block that can be dynamically allocated, it can be either
- *     free:
- *     The block is free.
- *
- *     pending:
- *     The block is allocated to some client and not free.
- *
- *     configured:
- *     The block is allocated to some client, and assigned to that
- *     client in MDP5_SMP_ALLOC registers.
- *
- *     inuse:
- *     The block is being actively used by a client.
- *
- * The updates happen in the following steps:
- *
- *  1) mdp5_smp_request():
- *     When plane scanout is setup, calculate required number of
- *     blocks needed per client, and request. Blocks neither inuse nor
- *     configured nor pending by any other client are added to client's
- *     pending set.
- *     For shrinking, blocks in pending but not in configured can be freed
- *     directly, but those already in configured will be freed later by
- *     mdp5_smp_commit.
- *
- *  2) mdp5_smp_configure():
- *     As hw is programmed, before FLUSH, MDP5_SMP_ALLOC registers
- *     are configured for the union(pending, inuse)
- *     Current pending is copied to configured.
- *     It is assumed that mdp5_smp_request and mdp5_smp_configure not run
- *     concurrently for the same pipe.
- *
- *  3) mdp5_smp_commit():
- *     After next vblank, copy configured -> inuse.  Optionally update
- *     MDP5_SMP_ALLOC registers if there are newly unused blocks
- *
- *  4) mdp5_smp_release():
- *     Must be called after the pipe is disabled and no longer uses any SMB
- *
- * On the next vblank after changes have been committed to hw, the
- * client's pending blocks become it's in-use blocks (and no-longer
- * in-use blocks become available to other clients).
- *
- * btw, hurray for confusing overloaded acronyms!  :-/
- *
- * NOTE: for atomic modeset/pageflip NONBLOCK operations, step #1
- * should happen at (or before)? atomic->check().  And we'd need
- * an API to discard previous requests if update is aborted or
- * (test-only).
- *
- * TODO would perhaps be nice to have debugfs to dump out kernel
- * inuse and pending state of all clients..
- */
-
 struct mdp5_smp {
 	struct drm_device *dev;
 
@@ -94,16 +28,8 @@ struct mdp5_smp {
 
 	int blk_cnt;
 	int blk_size;
-
-	spinlock_t state_lock;
-	mdp5_smp_state_t state; /* to track smp allocation amongst pipes: */
-
-	struct mdp5_client_smp_state client_state[MAX_CLIENTS];
 };
 
-static void update_smp_state(struct mdp5_smp *smp,
-		u32 cid, mdp5_smp_state_t *assigned);
-
 static inline
 struct mdp5_kms *get_kms(struct mdp5_smp *smp)
 {
@@ -134,57 +60,38 @@ static inline u32 pipe2client(enum mdp5_pipe pipe, int plane)
 	return mdp5_cfg->smp.clients[pipe] + plane;
 }
 
-/* step #1: update # of blocks pending for the client: */
+/* allocate blocks for the specified request: */
 static int smp_request_block(struct mdp5_smp *smp,
+		struct mdp5_smp_state *state,
 		u32 cid, int nblks)
 {
-	struct mdp5_kms *mdp5_kms = get_kms(smp);
-	struct mdp5_client_smp_state *ps = &smp->client_state[cid];
-	int i, ret, avail, cur_nblks, cnt = smp->blk_cnt;
+	void *cs = state->client_state[cid];
+	int i, avail, cnt = smp->blk_cnt;
 	uint8_t reserved;
-	unsigned long flags;
 
-	reserved = smp->reserved[cid];
+	/* we shouldn't be requesting blocks for an in-use client: */
+	WARN_ON(bitmap_weight(cs, cnt) > 0);
 
-	spin_lock_irqsave(&smp->state_lock, flags);
+	reserved = smp->reserved[cid];
 
 	if (reserved) {
 		nblks = max(0, nblks - reserved);
 		DBG("%d MMBs allocated (%d reserved)", nblks, reserved);
 	}
 
-	avail = cnt - bitmap_weight(smp->state, cnt);
+	avail = cnt - bitmap_weight(state->state, cnt);
 	if (nblks > avail) {
-		dev_err(mdp5_kms->dev->dev, "out of blks (req=%d > avail=%d)\n",
+		dev_err(smp->dev->dev, "out of blks (req=%d > avail=%d)\n",
 				nblks, avail);
-		ret = -ENOSPC;
-		goto fail;
+		return -ENOSPC;
 	}
 
-	cur_nblks = bitmap_weight(ps->pending, cnt);
-	if (nblks > cur_nblks) {
-		/* grow the existing pending reservation: */
-		for (i = cur_nblks; i < nblks; i++) {
-			int blk = find_first_zero_bit(smp->state, cnt);
-			set_bit(blk, ps->pending);
-			set_bit(blk, smp->state);
-		}
-	} else {
-		/* shrink the existing pending reservation: */
-		for (i = cur_nblks; i > nblks; i--) {
-			int blk = find_first_bit(ps->pending, cnt);
-			clear_bit(blk, ps->pending);
-
-			/* clear in global smp_state if not in configured
-			 * otherwise until _commit()
-			 */
-			if (!test_bit(blk, ps->configured))
-				clear_bit(blk, smp->state);
-		}
+	for (i = 0; i < nblks; i++) {
+		int blk = find_first_zero_bit(state->state, cnt);
+		set_bit(blk, cs);
+		set_bit(blk, state->state);
 	}
 
-fail:
-	spin_unlock_irqrestore(&smp->state_lock, flags);
 	return 0;
 }
 
@@ -209,14 +116,15 @@ static void set_fifo_thresholds(struct mdp5_smp *smp,
  * decimated width.  Ie. SMP buffering sits downstream of decimation (which
  * presumably happens during the dma from scanout buffer).
  */
-int mdp5_smp_request(struct mdp5_smp *smp, enum mdp5_pipe pipe,
-		const struct mdp_format *format, u32 width, bool hdecim)
+uint32_t mdp5_smp_calculate(struct mdp5_smp *smp,
+		const struct mdp_format *format,
+		u32 width, bool hdecim)
 {
 	struct mdp5_kms *mdp5_kms = get_kms(smp);
-	struct drm_device *dev = mdp5_kms->dev;
 	int rev = mdp5_cfg_get_hw_rev(mdp5_kms->cfg);
-	int i, hsub, nplanes, nlines, nblks, ret;
+	int i, hsub, nplanes, nlines;
 	u32 fmt = format->base.pixel_format;
+	uint32_t blkcfg = 0;
 
 	nplanes = drm_format_num_planes(fmt);
 	hsub = drm_format_horz_chroma_subsampling(fmt);
@@ -239,7 +147,7 @@ int mdp5_smp_request(struct mdp5_smp *smp, enum mdp5_pipe pipe,
 			hsub = 1;
 	}
 
-	for (i = 0, nblks = 0; i < nplanes; i++) {
+	for (i = 0; i < nplanes; i++) {
 		int n, fetch_stride, cpp;
 
 		cpp = drm_format_plane_cpp(fmt, i);
@@ -251,60 +159,72 @@ int mdp5_smp_request(struct mdp5_smp *smp, enum mdp5_pipe pipe,
 		if (rev == 0)
 			n = roundup_pow_of_two(n);
 
+		blkcfg |= (n << (8 * i));
+	}
+
+	return blkcfg;
+}
+
+int mdp5_smp_assign(struct mdp5_smp *smp, struct mdp5_smp_state *state,
+		enum mdp5_pipe pipe, uint32_t blkcfg)
+{
+	struct mdp5_kms *mdp5_kms = get_kms(smp);
+	struct drm_device *dev = mdp5_kms->dev;
+	int i, ret;
+
+	for (i = 0; i < pipe2nclients(pipe); i++) {
+		u32 cid = pipe2client(pipe, i);
+		int n = blkcfg & 0xff;
+
+		if (!n)
+			continue;
+
 		DBG("%s[%d]: request %d SMP blocks", pipe2name(pipe), i, n);
-		ret = smp_request_block(smp, pipe2client(pipe, i), n);
+		ret = smp_request_block(smp, state, cid, n);
 		if (ret) {
 			dev_err(dev->dev, "Cannot allocate %d SMP blocks: %d\n",
 					n, ret);
 			return ret;
 		}
 
-		nblks += n;
+		blkcfg >>= 8;
 	}
 
-	set_fifo_thresholds(smp, pipe, nblks);
+	state->assigned |= (1 << pipe);
 
 	return 0;
 }
 
 /* Release SMP blocks for all clients of the pipe */
-void mdp5_smp_release(struct mdp5_smp *smp, enum mdp5_pipe pipe)
+void mdp5_smp_release(struct mdp5_smp *smp, struct mdp5_smp_state *state,
+		enum mdp5_pipe pipe)
 {
 	int i;
-	unsigned long flags;
 	int cnt = smp->blk_cnt;
 
 	for (i = 0; i < pipe2nclients(pipe); i++) {
-		mdp5_smp_state_t assigned;
 		u32 cid = pipe2client(pipe, i);
-		struct mdp5_client_smp_state *ps = &smp->client_state[cid];
-
-		spin_lock_irqsave(&smp->state_lock, flags);
-
-		/* clear hw assignment */
-		bitmap_or(assigned, ps->inuse, ps->configured, cnt);
-		update_smp_state(smp, CID_UNUSED, &assigned);
-
-		/* free to global pool */
-		bitmap_andnot(smp->state, smp->state, ps->pending, cnt);
-		bitmap_andnot(smp->state, smp->state, assigned, cnt);
+		void *cs = state->client_state[cid];
 
-		/* clear client's infor */
-		bitmap_zero(ps->pending, cnt);
-		bitmap_zero(ps->configured, cnt);
-		bitmap_zero(ps->inuse, cnt);
+		/* update global state: */
+		bitmap_andnot(state->state, state->state, cs, cnt);
 
-		spin_unlock_irqrestore(&smp->state_lock, flags);
+		/* clear client's state */
+		bitmap_zero(cs, cnt);
 	}
 
-	set_fifo_thresholds(smp, pipe, 0);
+	state->released |= (1 << pipe);
 }
 
-static void update_smp_state(struct mdp5_smp *smp,
+/* NOTE: SMP_ALLOC_* regs are *not* double buffered, so release has to
+ * happen after scanout completes.
+ */
+static unsigned update_smp_state(struct mdp5_smp *smp,
 		u32 cid, mdp5_smp_state_t *assigned)
 {
 	struct mdp5_kms *mdp5_kms = get_kms(smp);
 	int cnt = smp->blk_cnt;
+	unsigned nblks = 0;
 	u32 blk, val;
 
 	for_each_set_bit(blk, *assigned, cnt) {
@@ -330,62 +250,88 @@ static void update_smp_state(struct mdp5_smp *smp,
 
 		mdp5_write(mdp5_kms, REG_MDP5_SMP_ALLOC_W_REG(idx), val);
 		mdp5_write(mdp5_kms, REG_MDP5_SMP_ALLOC_R_REG(idx), val);
+
+		nblks++;
 	}
+
+	return nblks;
 }
 
-/* step #2: configure hw for union(pending, inuse): */
-void mdp5_smp_configure(struct mdp5_smp *smp, enum mdp5_pipe pipe)
+void mdp5_smp_prepare_commit(struct mdp5_smp *smp, struct mdp5_smp_state *state)
 {
-	int cnt = smp->blk_cnt;
-	mdp5_smp_state_t assigned;
-	int i;
+	enum mdp5_pipe pipe;
 
-	for (i = 0; i < pipe2nclients(pipe); i++) {
-		u32 cid = pipe2client(pipe, i);
-		struct mdp5_client_smp_state *ps = &smp->client_state[cid];
+	for_each_set_bit(pipe, &state->assigned, sizeof(state->assigned) * 8) {
+		unsigned i, nblks = 0;
 
-		/*
-		 * if vblank has not happened since last smp_configure
-		 * skip the configure for now
-		 */
-		if (!bitmap_equal(ps->inuse, ps->configured, cnt))
-			continue;
+		for (i = 0; i < pipe2nclients(pipe); i++) {
+			u32 cid = pipe2client(pipe, i);
+			void *cs = state->client_state[cid];
 
-		bitmap_copy(ps->configured, ps->pending, cnt);
-		bitmap_or(assigned, ps->inuse, ps->configured, cnt);
-		update_smp_state(smp, cid, &assigned);
+			nblks += update_smp_state(smp, cid, cs);
+
+			DBG("assign %s:%u, %u blks",
+				pipe2name(pipe), i, nblks);
+		}
+
+		set_fifo_thresholds(smp, pipe, nblks);
 	}
+
+	state->assigned = 0;
 }
 
-/* step #3: after vblank, copy configured -> inuse: */
-void mdp5_smp_commit(struct mdp5_smp *smp, enum mdp5_pipe pipe)
+void mdp5_smp_complete_commit(struct mdp5_smp *smp, struct mdp5_smp_state *state)
 {
-	int cnt = smp->blk_cnt;
-	mdp5_smp_state_t released;
-	int i;
-
-	for (i = 0; i < pipe2nclients(pipe); i++) {
-		u32 cid = pipe2client(pipe, i);
-		struct mdp5_client_smp_state *ps = &smp->client_state[cid];
+	enum mdp5_pipe pipe;
 
-		/*
-		 * Figure out if there are any blocks we where previously
-		 * using, which can be released and made available to other
-		 * clients:
-		 */
-		if (bitmap_andnot(released, ps->inuse, ps->configured, cnt)) {
-			unsigned long flags;
+	for_each_set_bit(pipe, &state->released, sizeof(state->released) * 8) {
+		DBG("release %s", pipe2name(pipe));
+		set_fifo_thresholds(smp, pipe, 0);
+	}
 
-			spin_lock_irqsave(&smp->state_lock, flags);
-			/* clear released blocks: */
-			bitmap_andnot(smp->state, smp->state, released, cnt);
-			spin_unlock_irqrestore(&smp->state_lock, flags);
+	state->released = 0;
+}
 
-			update_smp_state(smp, CID_UNUSED, &released);
+void mdp5_smp_dump(struct mdp5_smp *smp, struct drm_printer *p)
+{
+	struct mdp5_kms *mdp5_kms = get_kms(smp);
+	struct mdp5_hw_pipe_state *hwpstate;
+	struct mdp5_smp_state *state;
+	int total = 0, i, j;
+
+	drm_printf(p, "name\tinuse\tplane\n");
+	drm_printf(p, "----\t-----\t-----\n");
+
+	if (drm_can_sleep())
+		drm_modeset_lock(&mdp5_kms->state_lock, NULL);
+
+	/* grab these *after* we hold the state_lock */
+	hwpstate = &mdp5_kms->state->hwpipe;
+	state = &mdp5_kms->state->smp;
+
+	for (i = 0; i < mdp5_kms->num_hwpipes; i++) {
+		struct mdp5_hw_pipe *hwpipe = mdp5_kms->hwpipes[i];
+		struct drm_plane *plane = hwpstate->hwpipe_to_plane[hwpipe->idx];
+		enum mdp5_pipe pipe = hwpipe->pipe;
+		for (j = 0; j < pipe2nclients(pipe); j++) {
+			u32 cid = pipe2client(pipe, j);
+			void *cs = state->client_state[cid];
+			int inuse = bitmap_weight(cs, smp->blk_cnt);
+
+			drm_printf(p, "%s:%d\t%d\t%s\n",
+				pipe2name(pipe), j, inuse,
+				plane ? plane->name : NULL);
+
+			total += inuse;
 		}
-
-		bitmap_copy(ps->inuse, ps->configured, cnt);
 	}
+
+	drm_printf(p, "TOTAL:\t%d\t(of %d)\n", total, smp->blk_cnt);
+	drm_printf(p, "AVAIL:\t%d\n", smp->blk_cnt -
+			bitmap_weight(state->state, smp->blk_cnt));
+
+	if (drm_can_sleep())
+		drm_modeset_unlock(&mdp5_kms->state_lock);
 }
 
 void mdp5_smp_destroy(struct mdp5_smp *smp)
@@ -393,8 +339,9 @@ void mdp5_smp_destroy(struct mdp5_smp *smp)
 	kfree(smp);
 }
 
-struct mdp5_smp *mdp5_smp_init(struct drm_device *dev, const struct mdp5_smp_block *cfg)
+struct mdp5_smp *mdp5_smp_init(struct mdp5_kms *mdp5_kms, const struct mdp5_smp_block *cfg)
 {
+	struct mdp5_smp_state *state = &mdp5_kms->state->smp;
 	struct mdp5_smp *smp = NULL;
 	int ret;
 
@@ -404,14 +351,13 @@ struct mdp5_smp *mdp5_smp_init(struct drm_device *dev, const struct mdp5_smp_blo
 		goto fail;
 	}
 
-	smp->dev = dev;
+	smp->dev = mdp5_kms->dev;
 	smp->blk_cnt = cfg->mmb_count;
 	smp->blk_size = cfg->mmb_size;
 
 	/* statically tied MMBs cannot be re-allocated: */
-	bitmap_copy(smp->state, cfg->reserved_state, smp->blk_cnt);
+	bitmap_copy(state->state, cfg->reserved_state, smp->blk_cnt);
 	memcpy(smp->reserved, cfg->reserved, sizeof(smp->reserved));
-	spin_lock_init(&smp->state_lock);
 
 	return smp;
 fail:
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.h b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.h
index 20b87e800ea3..b41d0448fbe8 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.h
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.h
@@ -19,12 +19,53 @@
 #ifndef __MDP5_SMP_H__
 #define __MDP5_SMP_H__
 
+#include <drm/drm_print.h>
+
 #include "msm_drv.h"
 
-struct mdp5_client_smp_state {
-	mdp5_smp_state_t inuse;
-	mdp5_smp_state_t configured;
-	mdp5_smp_state_t pending;
+/*
+ * SMP - Shared Memory Pool:
+ *
+ * SMP blocks are shared between all the clients, where each plane in
+ * a scanout buffer is a SMP client.  Ie. scanout of 3 plane I420 on
+ * pipe VIG0 => 3 clients: VIG0_Y, VIG0_CB, VIG0_CR.
+ *
+ * Based on the size of the attached scanout buffer, a certain # of
+ * blocks must be allocated to that client out of the shared pool.
+ *
+ * In some hw, some blocks are statically allocated for certain pipes
+ * and CANNOT be re-allocated (eg: MMB0 and MMB1 both tied to RGB0).
+ *
+ *
+ * Atomic SMP State:
+ *
+ * On atomic updates that modify SMP configuration, the state is cloned
+ * (copied) and modified.  For test-only, or in cases where atomic
+ * update fails (or if we hit ww_mutex deadlock/backoff condition) the
+ * new state is simply thrown away.
+ *
+ * Because the SMP registers are not double buffered, updates are a
+ * two step process:
+ *
+ * 1) in _prepare_commit() we configure things (via read-modify-write)
+ *    for the newly assigned pipes, so we don't take away blocks
+ *    assigned to pipes that are still scanning out
+ * 2) in _complete_commit(), after vblank/etc, we clear things for the
+ *    released clients, since at that point old pipes are no longer
+ *    scanning out.
+ */
+struct mdp5_smp_state {
+	/* global state of what blocks are in use: */
+	mdp5_smp_state_t state;
+
+	/* per client state of what blocks they are using: */
+	mdp5_smp_state_t client_state[MAX_CLIENTS];
+
+	/* assigned pipes (hw updated at _prepare_commit()): */
+	unsigned long assigned;
+
+	/* released pipes (hw updated at _complete_commit()): */
+	unsigned long released;
 };
 
 struct mdp5_kms;
@@ -36,13 +77,22 @@ struct mdp5_smp;
  * which is then used to call the other mdp5_smp_*(handler, ...) functions.
  */
 
-struct mdp5_smp *mdp5_smp_init(struct drm_device *dev, const struct mdp5_smp_block *cfg);
+struct mdp5_smp *mdp5_smp_init(struct mdp5_kms *mdp5_kms,
+		const struct mdp5_smp_block *cfg);
 void  mdp5_smp_destroy(struct mdp5_smp *smp);
 
-int  mdp5_smp_request(struct mdp5_smp *smp, enum mdp5_pipe pipe,
-		const struct mdp_format *format, u32 width, bool hdecim);
-void mdp5_smp_configure(struct mdp5_smp *smp, enum mdp5_pipe pipe);
-void mdp5_smp_commit(struct mdp5_smp *smp, enum mdp5_pipe pipe);
-void mdp5_smp_release(struct mdp5_smp *smp, enum mdp5_pipe pipe);
+void mdp5_smp_dump(struct mdp5_smp *smp, struct drm_printer *p);
+
+uint32_t mdp5_smp_calculate(struct mdp5_smp *smp,
+		const struct mdp_format *format,
+		u32 width, bool hdecim);
+
+int mdp5_smp_assign(struct mdp5_smp *smp, struct mdp5_smp_state *state,
+		enum mdp5_pipe pipe, uint32_t blkcfg);
+void mdp5_smp_release(struct mdp5_smp *smp, struct mdp5_smp_state *state,
+		enum mdp5_pipe pipe);
+
+void mdp5_smp_prepare_commit(struct mdp5_smp *smp, struct mdp5_smp_state *state);
+void mdp5_smp_complete_commit(struct mdp5_smp *smp, struct mdp5_smp_state *state);
 
 #endif /* __MDP5_SMP_H__ */
diff --git a/drivers/gpu/drm/msm/mdp/mdp_common.xml.h b/drivers/gpu/drm/msm/mdp/mdp_common.xml.h
index 452e3518f98b..8994c365e218 100644
--- a/drivers/gpu/drm/msm/mdp/mdp_common.xml.h
+++ b/drivers/gpu/drm/msm/mdp/mdp_common.xml.h
@@ -12,7 +12,7 @@ The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-02-10 17:07:21)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2015-05-20 20:03:14)
 - /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2015-09-18 12:07:28)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37194 bytes, from 2015-09-18 12:07:28)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  36965 bytes, from 2016-11-26 23:01:08)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  27887 bytes, from 2015-10-22 16:34:52)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2015-10-22 16:35:02)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2015-05-20 20:03:14)
diff --git a/drivers/gpu/drm/msm/msm_atomic.c b/drivers/gpu/drm/msm/msm_atomic.c
index 73bae382eac3..30b5d23e53b4 100644
--- a/drivers/gpu/drm/msm/msm_atomic.c
+++ b/drivers/gpu/drm/msm/msm_atomic.c
@@ -141,7 +141,7 @@ static void complete_commit(struct msm_commit *c, bool async)
 
 	kms->funcs->complete_commit(kms, state);
 
-	drm_atomic_state_free(state);
+	drm_atomic_state_put(state);
 
 	commit_destroy(c);
 }
@@ -217,8 +217,9 @@ int msm_atomic_commit(struct drm_device *dev,
 		if ((plane->state->fb != plane_state->fb) && plane_state->fb) {
 			struct drm_gem_object *obj = msm_framebuffer_bo(plane_state->fb, 0);
 			struct msm_gem_object *msm_obj = to_msm_bo(obj);
+			struct dma_fence *fence = reservation_object_get_excl_rcu(msm_obj->resv);
 
-			plane_state->fence = reservation_object_get_excl_rcu(msm_obj->resv);
+			drm_atomic_set_fence_for_plane(plane_state, fence);
 		}
 	}
 
@@ -240,6 +241,10 @@ int msm_atomic_commit(struct drm_device *dev,
 
 	drm_atomic_helper_swap_state(state, true);
 
+	/* swap driver private state while still holding state_lock */
+	if (to_kms_state(state)->state)
+		priv->kms->funcs->swap_state(priv->kms, state);
+
 	/*
 	 * Everything below can be run asynchronously without the need to grab
 	 * any modeset locks at all under one conditions: It must be guaranteed
@@ -256,6 +261,7 @@ int msm_atomic_commit(struct drm_device *dev,
 	 * current layout.
 	 */
 
+	drm_atomic_state_get(state);
 	if (nonblock) {
 		queue_work(priv->atomic_wq, &c->work);
 		return 0;
@@ -269,3 +275,30 @@ error:
 	drm_atomic_helper_cleanup_planes(dev, state);
 	return ret;
 }
+
+struct drm_atomic_state *msm_atomic_state_alloc(struct drm_device *dev)
+{
+	struct msm_kms_state *state = kzalloc(sizeof(*state), GFP_KERNEL);
+
+	if (!state || drm_atomic_state_init(dev, &state->base) < 0) {
+		kfree(state);
+		return NULL;
+	}
+
+	return &state->base;
+}
+
+void msm_atomic_state_clear(struct drm_atomic_state *s)
+{
+	struct msm_kms_state *state = to_kms_state(s);
+	drm_atomic_state_default_clear(&state->base);
+	kfree(state->state);
+	state->state = NULL;
+}
+
+void msm_atomic_state_free(struct drm_atomic_state *state)
+{
+	kfree(to_kms_state(state)->state);
+	drm_atomic_state_default_release(state);
+	kfree(state);
+}
diff --git a/drivers/gpu/drm/msm/msm_debugfs.c b/drivers/gpu/drm/msm/msm_debugfs.c
index 663f2b6ef091..c1b40f5adb60 100644
--- a/drivers/gpu/drm/msm/msm_debugfs.c
+++ b/drivers/gpu/drm/msm/msm_debugfs.c
@@ -18,6 +18,8 @@
 #ifdef CONFIG_DEBUG_FS
 #include "msm_drv.h"
 #include "msm_gpu.h"
+#include "msm_kms.h"
+#include "msm_debugfs.h"
 
 static int msm_gpu_show(struct drm_device *dev, struct seq_file *m)
 {
@@ -141,6 +143,7 @@ int msm_debugfs_late_init(struct drm_device *dev)
 int msm_debugfs_init(struct drm_minor *minor)
 {
 	struct drm_device *dev = minor->dev;
+	struct msm_drm_private *priv = dev->dev_private;
 	int ret;
 
 	ret = drm_debugfs_create_files(msm_debugfs_list,
@@ -152,15 +155,25 @@ int msm_debugfs_init(struct drm_minor *minor)
 		return ret;
 	}
 
-	return 0;
+	if (priv->kms->funcs->debugfs_init)
+		ret = priv->kms->funcs->debugfs_init(priv->kms, minor);
+
+	return ret;
 }
 
 void msm_debugfs_cleanup(struct drm_minor *minor)
 {
+	struct drm_device *dev = minor->dev;
+	struct msm_drm_private *priv = dev->dev_private;
+
 	drm_debugfs_remove_files(msm_debugfs_list,
 			ARRAY_SIZE(msm_debugfs_list), minor);
-	if (!minor->dev->dev_private)
+	if (!priv)
 		return;
+
+	if (priv->kms->funcs->debugfs_cleanup)
+		priv->kms->funcs->debugfs_cleanup(priv->kms, minor);
+
 	msm_rd_debugfs_cleanup(minor);
 	msm_perf_debugfs_cleanup(minor);
 }
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 46568fc80848..e29bb66f55b1 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -15,6 +15,8 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <drm/drm_of.h>
+
 #include "msm_drv.h"
 #include "msm_debugfs.h"
 #include "msm_fence.h"
@@ -44,17 +46,21 @@ static const struct drm_mode_config_funcs mode_config_funcs = {
 	.output_poll_changed = msm_fb_output_poll_changed,
 	.atomic_check = msm_atomic_check,
 	.atomic_commit = msm_atomic_commit,
+	.atomic_state_alloc = msm_atomic_state_alloc,
+	.atomic_state_clear = msm_atomic_state_clear,
+	.atomic_state_free = msm_atomic_state_free,
 };
 
-int msm_register_mmu(struct drm_device *dev, struct msm_mmu *mmu)
+int msm_register_address_space(struct drm_device *dev,
+		struct msm_gem_address_space *aspace)
 {
 	struct msm_drm_private *priv = dev->dev_private;
-	int idx = priv->num_mmus++;
+	int idx = priv->num_aspaces++;
 
-	if (WARN_ON(idx >= ARRAY_SIZE(priv->mmus)))
+	if (WARN_ON(idx >= ARRAY_SIZE(priv->aspace)))
 		return -EINVAL;
 
-	priv->mmus[idx] = mmu;
+	priv->aspace[idx] = aspace;
 
 	return idx;
 }
@@ -77,6 +83,10 @@ static char *vram = "16m";
 MODULE_PARM_DESC(vram, "Configure VRAM size (for devices without IOMMU/GPUMMU)");
 module_param(vram, charp, 0);
 
+bool dumpstate = false;
+MODULE_PARM_DESC(dumpstate, "Dump KMS state on errors");
+module_param(dumpstate, bool, 0600);
+
 /*
  * Util/helpers:
  */
@@ -766,9 +776,7 @@ static const struct file_operations fops = {
 	.open               = drm_open,
 	.release            = drm_release,
 	.unlocked_ioctl     = drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl       = drm_compat_ioctl,
-#endif
 	.poll               = drm_poll,
 	.read               = drm_read,
 	.llseek             = no_llseek,
@@ -903,10 +911,8 @@ static int add_components_mdp(struct device *mdp_dev,
 		 * remote-endpoint isn't a component that we need to add
 		 */
 		if (of_device_is_compatible(np, "qcom,mdp4") &&
-		    ep.port == 0) {
-			of_node_put(ep_node);
+		    ep.port == 0)
 			continue;
-		}
 
 		/*
 		 * It's okay if some of the ports don't have a remote endpoint
@@ -914,15 +920,12 @@ static int add_components_mdp(struct device *mdp_dev,
 		 * any external interface.
 		 */
 		intf = of_graph_get_remote_port_parent(ep_node);
-		if (!intf) {
-			of_node_put(ep_node);
+		if (!intf)
 			continue;
-		}
-
-		component_match_add(master_dev, matchptr, compare_of, intf);
 
+		drm_of_component_match_add(master_dev, matchptr, compare_of,
+					   intf);
 		of_node_put(intf);
-		of_node_put(ep_node);
 	}
 
 	return 0;
@@ -962,8 +965,8 @@ static int add_display_components(struct device *dev,
 		put_device(mdp_dev);
 
 		/* add the MDP component itself */
-		component_match_add(dev, matchptr, compare_of,
-				    mdp_dev->of_node);
+		drm_of_component_match_add(dev, matchptr, compare_of,
+					   mdp_dev->of_node);
 	} else {
 		/* MDP4 */
 		mdp_dev = dev;
@@ -996,7 +999,7 @@ static int add_gpu_components(struct device *dev,
 	if (!np)
 		return 0;
 
-	component_match_add(dev, matchptr, compare_of, np);
+	drm_of_component_match_add(dev, matchptr, compare_of, np);
 
 	of_node_put(np);
 
@@ -1035,7 +1038,13 @@ static int msm_pdev_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
+	/* on all devices that I am aware of, iommu's which can map
+	 * any address the cpu can see are used:
+	 */
+	ret = dma_set_mask_and_coherent(&pdev->dev, ~0);
+	if (ret)
+		return ret;
+
 	return component_master_add_with_match(&pdev->dev, &msm_drm_ops, match);
 }
 
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index d0da52f2a806..ed4dad3ca133 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -52,6 +52,8 @@ struct msm_perf_state;
 struct msm_gem_submit;
 struct msm_fence_context;
 struct msm_fence_cb;
+struct msm_gem_address_space;
+struct msm_gem_vma;
 
 #define NUM_DOMAINS 2    /* one for KMS, then one per gpu core (?) */
 
@@ -121,12 +123,16 @@ struct msm_drm_private {
 	uint32_t pending_crtcs;
 	wait_queue_head_t pending_crtcs_event;
 
-	/* registered MMUs: */
-	unsigned int num_mmus;
-	struct msm_mmu *mmus[NUM_DOMAINS];
+	/* Registered address spaces.. currently this is fixed per # of
+	 * iommu's.  Ie. one for display block and one for gpu block.
+	 * Eventually, to do per-process gpu pagetables, we'll want one
+	 * of these per-process.
+	 */
+	unsigned int num_aspaces;
+	struct msm_gem_address_space *aspace[NUM_DOMAINS];
 
 	unsigned int num_planes;
-	struct drm_plane *planes[8];
+	struct drm_plane *planes[16];
 
 	unsigned int num_crtcs;
 	struct drm_crtc *crtcs[8];
@@ -173,8 +179,22 @@ int msm_atomic_check(struct drm_device *dev,
 		     struct drm_atomic_state *state);
 int msm_atomic_commit(struct drm_device *dev,
 		struct drm_atomic_state *state, bool nonblock);
+struct drm_atomic_state *msm_atomic_state_alloc(struct drm_device *dev);
+void msm_atomic_state_clear(struct drm_atomic_state *state);
+void msm_atomic_state_free(struct drm_atomic_state *state);
+
+int msm_register_address_space(struct drm_device *dev,
+		struct msm_gem_address_space *aspace);
+
+void msm_gem_unmap_vma(struct msm_gem_address_space *aspace,
+		struct msm_gem_vma *vma, struct sg_table *sgt);
+int msm_gem_map_vma(struct msm_gem_address_space *aspace,
+		struct msm_gem_vma *vma, struct sg_table *sgt, int npages);
 
-int msm_register_mmu(struct drm_device *dev, struct msm_mmu *mmu);
+void msm_gem_address_space_destroy(struct msm_gem_address_space *aspace);
+struct msm_gem_address_space *
+msm_gem_address_space_create(struct device *dev, struct iommu_domain *domain,
+		const char *name);
 
 void msm_gem_submit_free(struct msm_gem_submit *submit);
 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
@@ -189,9 +209,9 @@ int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma);
 int msm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
 uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj);
 int msm_gem_get_iova_locked(struct drm_gem_object *obj, int id,
-		uint32_t *iova);
-int msm_gem_get_iova(struct drm_gem_object *obj, int id, uint32_t *iova);
-uint32_t msm_gem_iova(struct drm_gem_object *obj, int id);
+		uint64_t *iova);
+int msm_gem_get_iova(struct drm_gem_object *obj, int id, uint64_t *iova);
+uint64_t msm_gem_iova(struct drm_gem_object *obj, int id);
 struct page **msm_gem_get_pages(struct drm_gem_object *obj);
 void msm_gem_put_pages(struct drm_gem_object *obj);
 void msm_gem_put_iova(struct drm_gem_object *obj, int id);
@@ -217,7 +237,7 @@ void msm_gem_vunmap(struct drm_gem_object *obj);
 int msm_gem_sync_object(struct drm_gem_object *obj,
 		struct msm_fence_context *fctx, bool exclusive);
 void msm_gem_move_to_active(struct drm_gem_object *obj,
-		struct msm_gpu *gpu, bool exclusive, struct fence *fence);
+		struct msm_gpu *gpu, bool exclusive, struct dma_fence *fence);
 void msm_gem_move_to_inactive(struct drm_gem_object *obj);
 int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout);
 int msm_gem_cpu_fini(struct drm_gem_object *obj);
@@ -303,8 +323,8 @@ void __iomem *msm_ioremap(struct platform_device *pdev, const char *name,
 void msm_writel(u32 data, void __iomem *addr);
 u32 msm_readl(const void __iomem *addr);
 
-#define DBG(fmt, ...) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
-#define VERB(fmt, ...) if (0) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
+#define DBG(fmt, ...) DRM_DEBUG_DRIVER(fmt"\n", ##__VA_ARGS__)
+#define VERB(fmt, ...) if (0) DRM_DEBUG_DRIVER(fmt"\n", ##__VA_ARGS__)
 
 static inline int align_pitch(int width, int bpp)
 {
diff --git a/drivers/gpu/drm/msm/msm_fb.c b/drivers/gpu/drm/msm/msm_fb.c
index 95cf8fe72ee5..9acf544e7a8f 100644
--- a/drivers/gpu/drm/msm/msm_fb.c
+++ b/drivers/gpu/drm/msm/msm_fb.c
@@ -88,11 +88,11 @@ int msm_framebuffer_prepare(struct drm_framebuffer *fb, int id)
 {
 	struct msm_framebuffer *msm_fb = to_msm_framebuffer(fb);
 	int ret, i, n = drm_format_num_planes(fb->pixel_format);
-	uint32_t iova;
+	uint64_t iova;
 
 	for (i = 0; i < n; i++) {
 		ret = msm_gem_get_iova(msm_fb->planes[i], id, &iova);
-		DBG("FB[%u]: iova[%d]: %08x (%d)", fb->base.id, i, iova, ret);
+		DBG("FB[%u]: iova[%d]: %08llx (%d)", fb->base.id, i, iova, ret);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/gpu/drm/msm/msm_fbdev.c b/drivers/gpu/drm/msm/msm_fbdev.c
index ffd4a338ca12..bffe93498512 100644
--- a/drivers/gpu/drm/msm/msm_fbdev.c
+++ b/drivers/gpu/drm/msm/msm_fbdev.c
@@ -39,6 +39,7 @@ struct msm_fbdev {
 
 static struct fb_ops msm_fb_ops = {
 	.owner = THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
 
 	/* Note: to properly handle manual update displays, we wrap the
 	 * basic fbdev ops which write to the framebuffer
@@ -49,12 +50,6 @@ static struct fb_ops msm_fb_ops = {
 	.fb_copyarea = drm_fb_helper_sys_copyarea,
 	.fb_imageblit = drm_fb_helper_sys_imageblit,
 	.fb_mmap = msm_fbdev_mmap,
-
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
-	.fb_pan_display = drm_fb_helper_pan_display,
-	.fb_blank = drm_fb_helper_blank,
-	.fb_setcmap = drm_fb_helper_setcmap,
 };
 
 static int msm_fbdev_mmap(struct fb_info *info, struct vm_area_struct *vma)
@@ -81,7 +76,7 @@ static int msm_fbdev_create(struct drm_fb_helper *helper,
 	struct drm_framebuffer *fb = NULL;
 	struct fb_info *fbi = NULL;
 	struct drm_mode_fb_cmd2 mode_cmd = {0};
-	uint32_t paddr;
+	uint64_t paddr;
 	int ret, size;
 
 	DBG("create fbdev: %dx%d@%d (%dx%d)", sizes->surface_width,
diff --git a/drivers/gpu/drm/msm/msm_fence.c b/drivers/gpu/drm/msm/msm_fence.c
index a9b9b1c95a2e..3f299c537b77 100644
--- a/drivers/gpu/drm/msm/msm_fence.c
+++ b/drivers/gpu/drm/msm/msm_fence.c
@@ -15,7 +15,7 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 
 #include "msm_drv.h"
 #include "msm_fence.h"
@@ -32,7 +32,7 @@ msm_fence_context_alloc(struct drm_device *dev, const char *name)
 
 	fctx->dev = dev;
 	fctx->name = name;
-	fctx->context = fence_context_alloc(1);
+	fctx->context = dma_fence_context_alloc(1);
 	init_waitqueue_head(&fctx->event);
 	spin_lock_init(&fctx->spinlock);
 
@@ -100,52 +100,52 @@ void msm_update_fence(struct msm_fence_context *fctx, uint32_t fence)
 
 struct msm_fence {
 	struct msm_fence_context *fctx;
-	struct fence base;
+	struct dma_fence base;
 };
 
-static inline struct msm_fence *to_msm_fence(struct fence *fence)
+static inline struct msm_fence *to_msm_fence(struct dma_fence *fence)
 {
 	return container_of(fence, struct msm_fence, base);
 }
 
-static const char *msm_fence_get_driver_name(struct fence *fence)
+static const char *msm_fence_get_driver_name(struct dma_fence *fence)
 {
 	return "msm";
 }
 
-static const char *msm_fence_get_timeline_name(struct fence *fence)
+static const char *msm_fence_get_timeline_name(struct dma_fence *fence)
 {
 	struct msm_fence *f = to_msm_fence(fence);
 	return f->fctx->name;
 }
 
-static bool msm_fence_enable_signaling(struct fence *fence)
+static bool msm_fence_enable_signaling(struct dma_fence *fence)
 {
 	return true;
 }
 
-static bool msm_fence_signaled(struct fence *fence)
+static bool msm_fence_signaled(struct dma_fence *fence)
 {
 	struct msm_fence *f = to_msm_fence(fence);
 	return fence_completed(f->fctx, f->base.seqno);
 }
 
-static void msm_fence_release(struct fence *fence)
+static void msm_fence_release(struct dma_fence *fence)
 {
 	struct msm_fence *f = to_msm_fence(fence);
 	kfree_rcu(f, base.rcu);
 }
 
-static const struct fence_ops msm_fence_ops = {
+static const struct dma_fence_ops msm_fence_ops = {
 	.get_driver_name = msm_fence_get_driver_name,
 	.get_timeline_name = msm_fence_get_timeline_name,
 	.enable_signaling = msm_fence_enable_signaling,
 	.signaled = msm_fence_signaled,
-	.wait = fence_default_wait,
+	.wait = dma_fence_default_wait,
 	.release = msm_fence_release,
 };
 
-struct fence *
+struct dma_fence *
 msm_fence_alloc(struct msm_fence_context *fctx)
 {
 	struct msm_fence *f;
@@ -156,8 +156,8 @@ msm_fence_alloc(struct msm_fence_context *fctx)
 
 	f->fctx = fctx;
 
-	fence_init(&f->base, &msm_fence_ops, &fctx->spinlock,
-			fctx->context, ++fctx->last_fence);
+	dma_fence_init(&f->base, &msm_fence_ops, &fctx->spinlock,
+		       fctx->context, ++fctx->last_fence);
 
 	return &f->base;
 }
diff --git a/drivers/gpu/drm/msm/msm_fence.h b/drivers/gpu/drm/msm/msm_fence.h
index ceb5b3d314b4..56061aa1959d 100644
--- a/drivers/gpu/drm/msm/msm_fence.h
+++ b/drivers/gpu/drm/msm/msm_fence.h
@@ -41,6 +41,6 @@ int msm_queue_fence_cb(struct msm_fence_context *fctx,
 		struct msm_fence_cb *cb, uint32_t fence);
 void msm_update_fence(struct msm_fence_context *fctx, uint32_t fence);
 
-struct fence * msm_fence_alloc(struct msm_fence_context *fctx);
+struct dma_fence * msm_fence_alloc(struct msm_fence_context *fctx);
 
 #endif
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index b6ac27e31929..d8bc59c7e261 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -225,16 +225,14 @@ int msm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	}
 
 	/* We don't use vmf->pgoff since that has the fake offset: */
-	pgoff = ((unsigned long)vmf->virtual_address -
-			vma->vm_start) >> PAGE_SHIFT;
+	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
 
 	pfn = page_to_pfn(pages[pgoff]);
 
-	VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
+	VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address,
 			pfn, pfn << PAGE_SHIFT);
 
-	ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address,
-			__pfn_to_pfn_t(pfn, PFN_DEV));
+	ret = vm_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV));
 
 out_unlock:
 	mutex_unlock(&dev->struct_mutex);
@@ -296,12 +294,8 @@ put_iova(struct drm_gem_object *obj)
 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 
 	for (id = 0; id < ARRAY_SIZE(msm_obj->domain); id++) {
-		struct msm_mmu *mmu = priv->mmus[id];
-		if (mmu && msm_obj->domain[id].iova) {
-			uint32_t offset = msm_obj->domain[id].iova;
-			mmu->funcs->unmap(mmu, offset, msm_obj->sgt, obj->size);
-			msm_obj->domain[id].iova = 0;
-		}
+		msm_gem_unmap_vma(priv->aspace[id],
+				&msm_obj->domain[id], msm_obj->sgt);
 	}
 }
 
@@ -313,7 +307,7 @@ put_iova(struct drm_gem_object *obj)
  * the refcnt counter needs to be atomic_t.
  */
 int msm_gem_get_iova_locked(struct drm_gem_object *obj, int id,
-		uint32_t *iova)
+		uint64_t *iova)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	int ret = 0;
@@ -326,16 +320,8 @@ int msm_gem_get_iova_locked(struct drm_gem_object *obj, int id,
 			return PTR_ERR(pages);
 
 		if (iommu_present(&platform_bus_type)) {
-			struct msm_mmu *mmu = priv->mmus[id];
-			uint32_t offset;
-
-			if (WARN_ON(!mmu))
-				return -EINVAL;
-
-			offset = (uint32_t)mmap_offset(obj);
-			ret = mmu->funcs->map(mmu, offset, msm_obj->sgt,
-					obj->size, IOMMU_READ | IOMMU_WRITE);
-			msm_obj->domain[id].iova = offset;
+			ret = msm_gem_map_vma(priv->aspace[id], &msm_obj->domain[id],
+					msm_obj->sgt, obj->size >> PAGE_SHIFT);
 		} else {
 			msm_obj->domain[id].iova = physaddr(obj);
 		}
@@ -348,7 +334,7 @@ int msm_gem_get_iova_locked(struct drm_gem_object *obj, int id,
 }
 
 /* get iova, taking a reference.  Should have a matching put */
-int msm_gem_get_iova(struct drm_gem_object *obj, int id, uint32_t *iova)
+int msm_gem_get_iova(struct drm_gem_object *obj, int id, uint64_t *iova)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	int ret;
@@ -370,7 +356,7 @@ int msm_gem_get_iova(struct drm_gem_object *obj, int id, uint32_t *iova)
 /* get iova without taking a reference, used in places where you have
  * already done a 'msm_gem_get_iova()'.
  */
-uint32_t msm_gem_iova(struct drm_gem_object *obj, int id)
+uint64_t msm_gem_iova(struct drm_gem_object *obj, int id)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	WARN_ON(!msm_obj->domain[id].iova);
@@ -521,7 +507,7 @@ int msm_gem_sync_object(struct drm_gem_object *obj,
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	struct reservation_object_list *fobj;
-	struct fence *fence;
+	struct dma_fence *fence;
 	int i, ret;
 
 	if (!exclusive) {
@@ -540,7 +526,7 @@ int msm_gem_sync_object(struct drm_gem_object *obj,
 		fence = reservation_object_get_excl(msm_obj->resv);
 		/* don't need to wait on our own fences, since ring is fifo */
 		if (fence && (fence->context != fctx->context)) {
-			ret = fence_wait(fence, true);
+			ret = dma_fence_wait(fence, true);
 			if (ret)
 				return ret;
 		}
@@ -553,7 +539,7 @@ int msm_gem_sync_object(struct drm_gem_object *obj,
 		fence = rcu_dereference_protected(fobj->shared[i],
 						reservation_object_held(msm_obj->resv));
 		if (fence->context != fctx->context) {
-			ret = fence_wait(fence, true);
+			ret = dma_fence_wait(fence, true);
 			if (ret)
 				return ret;
 		}
@@ -563,7 +549,7 @@ int msm_gem_sync_object(struct drm_gem_object *obj,
 }
 
 void msm_gem_move_to_active(struct drm_gem_object *obj,
-		struct msm_gpu *gpu, bool exclusive, struct fence *fence)
+		struct msm_gpu *gpu, bool exclusive, struct dma_fence *fence)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED);
@@ -616,10 +602,10 @@ int msm_gem_cpu_fini(struct drm_gem_object *obj)
 }
 
 #ifdef CONFIG_DEBUG_FS
-static void describe_fence(struct fence *fence, const char *type,
+static void describe_fence(struct dma_fence *fence, const char *type,
 		struct seq_file *m)
 {
-	if (!fence_is_signaled(fence))
+	if (!dma_fence_is_signaled(fence))
 		seq_printf(m, "\t%9s: %s %s seq %u\n", type,
 				fence->ops->get_driver_name(fence),
 				fence->ops->get_timeline_name(fence),
@@ -631,9 +617,11 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	struct reservation_object *robj = msm_obj->resv;
 	struct reservation_object_list *fobj;
-	struct fence *fence;
+	struct msm_drm_private *priv = obj->dev->dev_private;
+	struct dma_fence *fence;
 	uint64_t off = drm_vma_node_start(&obj->vma_node);
 	const char *madv;
+	unsigned id;
 
 	WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
 
@@ -650,10 +638,15 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
 		break;
 	}
 
-	seq_printf(m, "%08x: %c %2d (%2d) %08llx %p %zu%s\n",
+	seq_printf(m, "%08x: %c %2d (%2d) %08llx %p\t",
 			msm_obj->flags, is_active(msm_obj) ? 'A' : 'I',
 			obj->name, obj->refcount.refcount.counter,
-			off, msm_obj->vaddr, obj->size, madv);
+			off, msm_obj->vaddr);
+
+	for (id = 0; id < priv->num_aspaces; id++)
+		seq_printf(m, " %08llx", msm_obj->domain[id].iova);
+
+	seq_printf(m, " %zu%s\n", obj->size, madv);
 
 	rcu_read_lock();
 	fobj = rcu_dereference(robj->fence);
@@ -761,7 +754,6 @@ static int msm_gem_new_impl(struct drm_device *dev,
 {
 	struct msm_drm_private *priv = dev->dev_private;
 	struct msm_gem_object *msm_obj;
-	unsigned sz;
 	bool use_vram = false;
 
 	switch (flags & MSM_BO_CACHE_MASK) {
@@ -783,16 +775,12 @@ static int msm_gem_new_impl(struct drm_device *dev,
 	if (WARN_ON(use_vram && !priv->vram.size))
 		return -EINVAL;
 
-	sz = sizeof(*msm_obj);
-	if (use_vram)
-		sz += sizeof(struct drm_mm_node);
-
-	msm_obj = kzalloc(sz, GFP_KERNEL);
+	msm_obj = kzalloc(sizeof(*msm_obj), GFP_KERNEL);
 	if (!msm_obj)
 		return -ENOMEM;
 
 	if (use_vram)
-		msm_obj->vram_node = (void *)&msm_obj[1];
+		msm_obj->vram_node = &msm_obj->domain[0].node;
 
 	msm_obj->flags = flags;
 	msm_obj->madv = MSM_MADV_WILLNEED;
diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h
index b2f13cfe945e..7d529516b332 100644
--- a/drivers/gpu/drm/msm/msm_gem.h
+++ b/drivers/gpu/drm/msm/msm_gem.h
@@ -24,6 +24,20 @@
 /* Additional internal-use only BO flags: */
 #define MSM_BO_STOLEN        0x10000000    /* try to use stolen/splash memory */
 
+struct msm_gem_address_space {
+	const char *name;
+	/* NOTE: mm managed at the page level, size is in # of pages
+	 * and position mm_node->start is in # of pages:
+	 */
+	struct drm_mm mm;
+	struct msm_mmu *mmu;
+};
+
+struct msm_gem_vma {
+	struct drm_mm_node node;
+	uint64_t iova;
+};
+
 struct msm_gem_object {
 	struct drm_gem_object base;
 
@@ -61,10 +75,7 @@ struct msm_gem_object {
 	struct sg_table *sgt;
 	void *vaddr;
 
-	struct {
-		// XXX
-		uint32_t iova;
-	} domain[NUM_DOMAINS];
+	struct msm_gem_vma domain[NUM_DOMAINS];
 
 	/* normally (resv == &_resv) except for imported bo's */
 	struct reservation_object *resv;
@@ -104,7 +115,7 @@ struct msm_gem_submit {
 	struct list_head node;   /* node in gpu submit_list */
 	struct list_head bo_list;
 	struct ww_acquire_ctx ticket;
-	struct fence *fence;
+	struct dma_fence *fence;
 	struct pid *pid;    /* submitting process */
 	bool valid;         /* true if no cmdstream patching needed */
 	unsigned int nr_cmds;
@@ -112,13 +123,13 @@ struct msm_gem_submit {
 	struct {
 		uint32_t type;
 		uint32_t size;  /* in dwords */
-		uint32_t iova;
+		uint64_t iova;
 		uint32_t idx;   /* cmdstream buffer idx in bos[] */
 	} *cmd;  /* array of size nr_cmds */
 	struct {
 		uint32_t flags;
 		struct msm_gem_object *obj;
-		uint32_t iova;
+		uint64_t iova;
 	} bos[0];
 };
 
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index b6a0f37a65f3..166e84e4f0d4 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -60,7 +60,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev,
 
 void msm_gem_submit_free(struct msm_gem_submit *submit)
 {
-	fence_put(submit->fence);
+	dma_fence_put(submit->fence);
 	list_del(&submit->node);
 	put_pid(submit->pid);
 	kfree(submit);
@@ -241,7 +241,7 @@ static int submit_pin_objects(struct msm_gem_submit *submit)
 
 	for (i = 0; i < submit->nr_bos; i++) {
 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
-		uint32_t iova;
+		uint64_t iova;
 
 		/* if locking succeeded, pin bo: */
 		ret = msm_gem_get_iova_locked(&msm_obj->base,
@@ -266,7 +266,7 @@ static int submit_pin_objects(struct msm_gem_submit *submit)
 }
 
 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
-		struct msm_gem_object **obj, uint32_t *iova, bool *valid)
+		struct msm_gem_object **obj, uint64_t *iova, bool *valid)
 {
 	if (idx >= submit->nr_bos) {
 		DRM_ERROR("invalid buffer index: %u (out of %u)\n",
@@ -312,7 +312,8 @@ static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *ob
 		struct drm_msm_gem_submit_reloc submit_reloc;
 		void __user *userptr =
 			u64_to_user_ptr(relocs + (i * sizeof(submit_reloc)));
-		uint32_t iova, off;
+		uint32_t off;
+		uint64_t iova;
 		bool valid;
 
 		ret = copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc));
@@ -380,7 +381,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 	struct msm_file_private *ctx = file->driver_priv;
 	struct msm_gem_submit *submit;
 	struct msm_gpu *gpu = priv->gpu;
-	struct fence *in_fence = NULL;
+	struct dma_fence *in_fence = NULL;
 	struct sync_file *sync_file = NULL;
 	int out_fence_fd = -1;
 	unsigned i;
@@ -439,7 +440,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 		 */
 
 		if (in_fence->context != gpu->fctx->context) {
-			ret = fence_wait(in_fence, true);
+			ret = dma_fence_wait(in_fence, true);
 			if (ret)
 				goto out;
 		}
@@ -461,7 +462,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 		void __user *userptr =
 			u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
 		struct msm_gem_object *msm_obj;
-		uint32_t iova;
+		uint64_t iova;
 
 		ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
 		if (ret) {
@@ -542,7 +543,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 
 out:
 	if (in_fence)
-		fence_put(in_fence);
+		dma_fence_put(in_fence);
 	submit_cleanup(submit);
 	if (ret)
 		msm_gem_submit_free(submit);
diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c
new file mode 100644
index 000000000000..a311d26ccb21
--- /dev/null
+++ b/drivers/gpu/drm/msm/msm_gem_vma.c
@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) 2016 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "msm_drv.h"
+#include "msm_gem.h"
+#include "msm_mmu.h"
+
+void
+msm_gem_unmap_vma(struct msm_gem_address_space *aspace,
+		struct msm_gem_vma *vma, struct sg_table *sgt)
+{
+	if (!vma->iova)
+		return;
+
+	if (aspace->mmu) {
+		unsigned size = vma->node.size << PAGE_SHIFT;
+		aspace->mmu->funcs->unmap(aspace->mmu, vma->iova, sgt, size);
+	}
+
+	drm_mm_remove_node(&vma->node);
+
+	vma->iova = 0;
+}
+
+int
+msm_gem_map_vma(struct msm_gem_address_space *aspace,
+		struct msm_gem_vma *vma, struct sg_table *sgt, int npages)
+{
+	int ret;
+
+	if (WARN_ON(drm_mm_node_allocated(&vma->node)))
+		return 0;
+
+	ret = drm_mm_insert_node(&aspace->mm, &vma->node, npages,
+			0, DRM_MM_SEARCH_DEFAULT);
+	if (ret)
+		return ret;
+
+	vma->iova = vma->node.start << PAGE_SHIFT;
+
+	if (aspace->mmu) {
+		unsigned size = npages << PAGE_SHIFT;
+		ret = aspace->mmu->funcs->map(aspace->mmu, vma->iova, sgt,
+				size, IOMMU_READ | IOMMU_WRITE);
+	}
+
+	return ret;
+}
+
+void
+msm_gem_address_space_destroy(struct msm_gem_address_space *aspace)
+{
+	drm_mm_takedown(&aspace->mm);
+	if (aspace->mmu)
+		aspace->mmu->funcs->destroy(aspace->mmu);
+	kfree(aspace);
+}
+
+struct msm_gem_address_space *
+msm_gem_address_space_create(struct device *dev, struct iommu_domain *domain,
+		const char *name)
+{
+	struct msm_gem_address_space *aspace;
+
+	aspace = kzalloc(sizeof(*aspace), GFP_KERNEL);
+	if (!aspace)
+		return ERR_PTR(-ENOMEM);
+
+	aspace->name = name;
+	aspace->mmu = msm_iommu_new(dev, domain);
+
+	drm_mm_init(&aspace->mm, (domain->geometry.aperture_start >> PAGE_SHIFT),
+			(domain->geometry.aperture_end >> PAGE_SHIFT) - 1);
+
+	return aspace;
+}
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 5bb09838b5ae..b28527a65d09 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -91,21 +91,20 @@ static int disable_pwrrail(struct msm_gpu *gpu)
 
 static int enable_clk(struct msm_gpu *gpu)
 {
-	struct clk *rate_clk = NULL;
 	int i;
 
-	/* NOTE: kgsl_pwrctrl_clk() ignores grp_clks[0].. */
-	for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--) {
-		if (gpu->grp_clks[i]) {
-			clk_prepare(gpu->grp_clks[i]);
-			rate_clk = gpu->grp_clks[i];
-		}
-	}
+	if (gpu->grp_clks[0] && gpu->fast_rate)
+		clk_set_rate(gpu->grp_clks[0], gpu->fast_rate);
 
-	if (rate_clk && gpu->fast_rate)
-		clk_set_rate(rate_clk, gpu->fast_rate);
+	/* Set the RBBM timer rate to 19.2Mhz */
+	if (gpu->grp_clks[2])
+		clk_set_rate(gpu->grp_clks[2], 19200000);
 
-	for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--)
+	for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i >= 0; i--)
+		if (gpu->grp_clks[i])
+			clk_prepare(gpu->grp_clks[i]);
+
+	for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i >= 0; i--)
 		if (gpu->grp_clks[i])
 			clk_enable(gpu->grp_clks[i]);
 
@@ -114,24 +113,22 @@ static int enable_clk(struct msm_gpu *gpu)
 
 static int disable_clk(struct msm_gpu *gpu)
 {
-	struct clk *rate_clk = NULL;
 	int i;
 
-	/* NOTE: kgsl_pwrctrl_clk() ignores grp_clks[0].. */
-	for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--) {
-		if (gpu->grp_clks[i]) {
+	for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i >= 0; i--)
+		if (gpu->grp_clks[i])
 			clk_disable(gpu->grp_clks[i]);
-			rate_clk = gpu->grp_clks[i];
-		}
-	}
-
-	if (rate_clk && gpu->slow_rate)
-		clk_set_rate(rate_clk, gpu->slow_rate);
 
-	for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i > 0; i--)
+	for (i = ARRAY_SIZE(gpu->grp_clks) - 1; i >= 0; i--)
 		if (gpu->grp_clks[i])
 			clk_unprepare(gpu->grp_clks[i]);
 
+	if (gpu->grp_clks[0] && gpu->slow_rate)
+		clk_set_rate(gpu->grp_clks[0], gpu->slow_rate);
+
+	if (gpu->grp_clks[2])
+		clk_set_rate(gpu->grp_clks[2], 0);
+
 	return 0;
 }
 
@@ -476,7 +473,7 @@ static void retire_submits(struct msm_gpu *gpu)
 		submit = list_first_entry(&gpu->submit_list,
 				struct msm_gem_submit, node);
 
-		if (fence_is_signaled(submit->fence)) {
+		if (dma_fence_is_signaled(submit->fence)) {
 			retire_submit(gpu, submit);
 		} else {
 			break;
@@ -528,7 +525,7 @@ void msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 
 	for (i = 0; i < submit->nr_bos; i++) {
 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
-		uint32_t iova;
+		uint64_t iova;
 
 		/* can't happen yet.. but when we add 2d support we'll have
 		 * to deal w/ cross-ring synchronization:
@@ -563,8 +560,8 @@ static irqreturn_t irq_handler(int irq, void *data)
 }
 
 static const char *clk_names[] = {
-		"src_clk", "core_clk", "iface_clk", "mem_clk", "mem_iface_clk",
-		"alt_mem_iface_clk",
+		"core_clk", "iface_clk", "rbbmtimer_clk", "mem_clk",
+		"mem_iface_clk", "alt_mem_iface_clk",
 };
 
 int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
@@ -656,12 +653,17 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 	 */
 	iommu = iommu_domain_alloc(&platform_bus_type);
 	if (iommu) {
+		/* TODO 32b vs 64b address space.. */
+		iommu->geometry.aperture_start = SZ_16M;
+		iommu->geometry.aperture_end = 0xffffffff;
+
 		dev_info(drm->dev, "%s: using IOMMU\n", name);
-		gpu->mmu = msm_iommu_new(&pdev->dev, iommu);
-		if (IS_ERR(gpu->mmu)) {
-			ret = PTR_ERR(gpu->mmu);
+		gpu->aspace = msm_gem_address_space_create(&pdev->dev,
+				iommu, "gpu");
+		if (IS_ERR(gpu->aspace)) {
+			ret = PTR_ERR(gpu->aspace);
 			dev_err(drm->dev, "failed to init iommu: %d\n", ret);
-			gpu->mmu = NULL;
+			gpu->aspace = NULL;
 			iommu_domain_free(iommu);
 			goto fail;
 		}
@@ -669,7 +671,7 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 	} else {
 		dev_info(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name);
 	}
-	gpu->id = msm_register_mmu(drm, gpu->mmu);
+	gpu->id = msm_register_address_space(drm, gpu->aspace);
 
 
 	/* Create ringbuffer: */
@@ -705,8 +707,8 @@ void msm_gpu_cleanup(struct msm_gpu *gpu)
 		msm_ringbuffer_destroy(gpu->rb);
 	}
 
-	if (gpu->mmu)
-		gpu->mmu->funcs->destroy(gpu->mmu);
+	if (gpu->aspace)
+		msm_gem_address_space_destroy(gpu->aspace);
 
 	if (gpu->fctx)
 		msm_fence_context_free(gpu->fctx);
diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index d61d98a6e047..c4c39d3272c7 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -50,7 +50,7 @@ struct msm_gpu_funcs {
 	void (*submit)(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 			struct msm_file_private *ctx);
 	void (*flush)(struct msm_gpu *gpu);
-	void (*idle)(struct msm_gpu *gpu);
+	bool (*idle)(struct msm_gpu *gpu);
 	irqreturn_t (*irq)(struct msm_gpu *irq);
 	uint32_t (*last_fence)(struct msm_gpu *gpu);
 	void (*recover)(struct msm_gpu *gpu);
@@ -80,7 +80,7 @@ struct msm_gpu {
 
 	/* ringbuffer: */
 	struct msm_ringbuffer *rb;
-	uint32_t rb_iova;
+	uint64_t rb_iova;
 
 	/* list of GEM active objects: */
 	struct list_head active_list;
@@ -98,7 +98,7 @@ struct msm_gpu {
 	void __iomem *mmio;
 	int irq;
 
-	struct msm_mmu *mmu;
+	struct msm_gem_address_space *aspace;
 	int id;
 
 	/* Power Control: */
@@ -154,6 +154,45 @@ static inline u32 gpu_read(struct msm_gpu *gpu, u32 reg)
 	return msm_readl(gpu->mmio + (reg << 2));
 }
 
+static inline void gpu_rmw(struct msm_gpu *gpu, u32 reg, u32 mask, u32 or)
+{
+	uint32_t val = gpu_read(gpu, reg);
+
+	val &= ~mask;
+	gpu_write(gpu, reg, val | or);
+}
+
+static inline u64 gpu_read64(struct msm_gpu *gpu, u32 lo, u32 hi)
+{
+	u64 val;
+
+	/*
+	 * Why not a readq here? Two reasons: 1) many of the LO registers are
+	 * not quad word aligned and 2) the GPU hardware designers have a bit
+	 * of a history of putting registers where they fit, especially in
+	 * spins. The longer a GPU family goes the higher the chance that
+	 * we'll get burned.  We could do a series of validity checks if we
+	 * wanted to, but really is a readq() that much better? Nah.
+	 */
+
+	/*
+	 * For some lo/hi registers (like perfcounters), the hi value is latched
+	 * when the lo is read, so make sure to read the lo first to trigger
+	 * that
+	 */
+	val = (u64) msm_readl(gpu->mmio + (lo << 2));
+	val |= ((u64) msm_readl(gpu->mmio + (hi << 2)) << 32);
+
+	return val;
+}
+
+static inline void gpu_write64(struct msm_gpu *gpu, u32 lo, u32 hi, u64 val)
+{
+	/* Why not a writeq here? Read the screed above */
+	msm_writel(lower_32_bits(val), gpu->mmio + (lo << 2));
+	msm_writel(upper_32_bits(val), gpu->mmio + (hi << 2));
+}
+
 int msm_gpu_pm_suspend(struct msm_gpu *gpu);
 int msm_gpu_pm_resume(struct msm_gpu *gpu);
 
diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c
index 3a294d0da3a0..61aaaa1de6eb 100644
--- a/drivers/gpu/drm/msm/msm_iommu.c
+++ b/drivers/gpu/drm/msm/msm_iommu.c
@@ -45,13 +45,13 @@ static void msm_iommu_detach(struct msm_mmu *mmu, const char * const *names,
 	iommu_detach_device(iommu->domain, mmu->dev);
 }
 
-static int msm_iommu_map(struct msm_mmu *mmu, uint32_t iova,
+static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
 		struct sg_table *sgt, unsigned len, int prot)
 {
 	struct msm_iommu *iommu = to_msm_iommu(mmu);
 	struct iommu_domain *domain = iommu->domain;
 	struct scatterlist *sg;
-	unsigned int da = iova;
+	unsigned long da = iova;
 	unsigned int i, j;
 	int ret;
 
@@ -62,7 +62,7 @@ static int msm_iommu_map(struct msm_mmu *mmu, uint32_t iova,
 		dma_addr_t pa = sg_phys(sg) - sg->offset;
 		size_t bytes = sg->length + sg->offset;
 
-		VERB("map[%d]: %08x %08lx(%zx)", i, da, (unsigned long)pa, bytes);
+		VERB("map[%d]: %08lx %08lx(%zx)", i, da, (unsigned long)pa, bytes);
 
 		ret = iommu_map(domain, da, pa, bytes, prot);
 		if (ret)
@@ -84,13 +84,13 @@ fail:
 	return ret;
 }
 
-static int msm_iommu_unmap(struct msm_mmu *mmu, uint32_t iova,
+static int msm_iommu_unmap(struct msm_mmu *mmu, uint64_t iova,
 		struct sg_table *sgt, unsigned len)
 {
 	struct msm_iommu *iommu = to_msm_iommu(mmu);
 	struct iommu_domain *domain = iommu->domain;
 	struct scatterlist *sg;
-	unsigned int da = iova;
+	unsigned long da = iova;
 	int i;
 
 	for_each_sg(sgt->sgl, sg, sgt->nents, i) {
@@ -101,7 +101,7 @@ static int msm_iommu_unmap(struct msm_mmu *mmu, uint32_t iova,
 		if (unmapped < bytes)
 			return unmapped;
 
-		VERB("unmap[%d]: %08x(%zx)", i, da, bytes);
+		VERB("unmap[%d]: %08lx(%zx)", i, da, bytes);
 
 		BUG_ON(!PAGE_ALIGNED(bytes));
 
diff --git a/drivers/gpu/drm/msm/msm_kms.h b/drivers/gpu/drm/msm/msm_kms.h
index 40e41e5cdbc6..e470f4cf8f76 100644
--- a/drivers/gpu/drm/msm/msm_kms.h
+++ b/drivers/gpu/drm/msm/msm_kms.h
@@ -40,6 +40,8 @@ struct msm_kms_funcs {
 	irqreturn_t (*irq)(struct msm_kms *kms);
 	int (*enable_vblank)(struct msm_kms *kms, struct drm_crtc *crtc);
 	void (*disable_vblank)(struct msm_kms *kms, struct drm_crtc *crtc);
+	/* swap global atomic state: */
+	void (*swap_state)(struct msm_kms *kms, struct drm_atomic_state *state);
 	/* modeset, bracketing atomic_commit(): */
 	void (*prepare_commit)(struct msm_kms *kms, struct drm_atomic_state *state);
 	void (*complete_commit)(struct msm_kms *kms, struct drm_atomic_state *state);
@@ -56,6 +58,11 @@ struct msm_kms_funcs {
 			bool is_cmd_mode);
 	/* cleanup: */
 	void (*destroy)(struct msm_kms *kms);
+#ifdef CONFIG_DEBUG_FS
+	/* debugfs: */
+	int (*debugfs_init)(struct msm_kms *kms, struct drm_minor *minor);
+	void (*debugfs_cleanup)(struct msm_kms *kms, struct drm_minor *minor);
+#endif
 };
 
 struct msm_kms {
@@ -65,6 +72,18 @@ struct msm_kms {
 	int irq;
 };
 
+/**
+ * Subclass of drm_atomic_state, to allow kms backend to have driver
+ * private global state.  The kms backend can do whatever it wants
+ * with the ->state ptr.  On ->atomic_state_clear() the ->state ptr
+ * is kfree'd and set back to NULL.
+ */
+struct msm_kms_state {
+	struct drm_atomic_state base;
+	void *state;
+};
+#define to_kms_state(x) container_of(x, struct msm_kms_state, base)
+
 static inline void msm_kms_init(struct msm_kms *kms,
 		const struct msm_kms_funcs *funcs)
 {
diff --git a/drivers/gpu/drm/msm/msm_mmu.h b/drivers/gpu/drm/msm/msm_mmu.h
index b8ca9a0e9170..f85c879e68d2 100644
--- a/drivers/gpu/drm/msm/msm_mmu.h
+++ b/drivers/gpu/drm/msm/msm_mmu.h
@@ -23,9 +23,9 @@
 struct msm_mmu_funcs {
 	int (*attach)(struct msm_mmu *mmu, const char * const *names, int cnt);
 	void (*detach)(struct msm_mmu *mmu, const char * const *names, int cnt);
-	int (*map)(struct msm_mmu *mmu, uint32_t iova, struct sg_table *sgt,
+	int (*map)(struct msm_mmu *mmu, uint64_t iova, struct sg_table *sgt,
 			unsigned len, int prot);
-	int (*unmap)(struct msm_mmu *mmu, uint32_t iova, struct sg_table *sgt,
+	int (*unmap)(struct msm_mmu *mmu, uint64_t iova, struct sg_table *sgt,
 			unsigned len);
 	void (*destroy)(struct msm_mmu *mmu);
 };
diff --git a/drivers/gpu/drm/msm/msm_rd.c b/drivers/gpu/drm/msm/msm_rd.c
index 8487f461f05f..6607456dc626 100644
--- a/drivers/gpu/drm/msm/msm_rd.c
+++ b/drivers/gpu/drm/msm/msm_rd.c
@@ -289,7 +289,7 @@ void msm_rd_debugfs_cleanup(struct drm_minor *minor)
 
 static void snapshot_buf(struct msm_rd_state *rd,
 		struct msm_gem_submit *submit, int idx,
-		uint32_t iova, uint32_t size)
+		uint64_t iova, uint32_t size)
 {
 	struct msm_gem_object *obj = submit->bos[idx].obj;
 	const char *buf;
@@ -306,7 +306,7 @@ static void snapshot_buf(struct msm_rd_state *rd,
 	}
 
 	rd_write_section(rd, RD_GPUADDR,
-			(uint32_t[2]){ iova, size }, 8);
+			(uint32_t[3]){ iova, size, iova >> 32 }, 12);
 	rd_write_section(rd, RD_BUFFER_CONTENTS, buf, size);
 
 	msm_gem_put_vaddr_locked(&obj->base);
diff --git a/drivers/gpu/drm/mxsfb/Kconfig b/drivers/gpu/drm/mxsfb/Kconfig
new file mode 100644
index 000000000000..e9a8d90e6723
--- /dev/null
+++ b/drivers/gpu/drm/mxsfb/Kconfig
@@ -0,0 +1,19 @@
+config DRM_MXS
+	bool
+	help
+	  Choose this option to select drivers for MXS FB devices
+
+config DRM_MXSFB
+	tristate "i.MX23/i.MX28/i.MX6SX MXSFB LCD controller"
+	depends on DRM && OF
+	depends on COMMON_CLK
+	select DRM_MXS
+	select DRM_KMS_HELPER
+	select DRM_KMS_FB_HELPER
+	select DRM_KMS_CMA_HELPER
+	select DRM_PANEL
+	help
+	  Choose this option if you have an i.MX23/i.MX28/i.MX6SX MXSFB
+	  LCD controller.
+
+	  If M is selected the module will be called mxsfb.
diff --git a/drivers/gpu/drm/mxsfb/Makefile b/drivers/gpu/drm/mxsfb/Makefile
new file mode 100644
index 000000000000..857f3a4545ff
--- /dev/null
+++ b/drivers/gpu/drm/mxsfb/Makefile
@@ -0,0 +1,2 @@
+mxsfb-y := mxsfb_drv.o mxsfb_crtc.o mxsfb_out.o
+obj-$(CONFIG_DRM_MXSFB)	+= mxsfb.o
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_crtc.c b/drivers/gpu/drm/mxsfb/mxsfb_crtc.c
new file mode 100644
index 000000000000..081890336ce7
--- /dev/null
+++ b/drivers/gpu/drm/mxsfb/mxsfb_crtc.c
@@ -0,0 +1,241 @@
+/*
+ * Copyright (C) 2016 Marek Vasut <marex@denx.de>
+ *
+ * This code is based on drivers/video/fbdev/mxsfb.c :
+ * Copyright (C) 2010 Juergen Beisert, Pengutronix
+ * Copyright (C) 2008-2009 Freescale Semiconductor, Inc. All Rights Reserved.
+ * Copyright (C) 2008 Embedded Alley Solutions, Inc All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_of.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_simple_kms_helper.h>
+#include <linux/clk.h>
+#include <linux/iopoll.h>
+#include <linux/of_graph.h>
+#include <linux/platform_data/simplefb.h>
+#include <video/videomode.h>
+
+#include "mxsfb_drv.h"
+#include "mxsfb_regs.h"
+
+static u32 set_hsync_pulse_width(struct mxsfb_drm_private *mxsfb, u32 val)
+{
+	return (val & mxsfb->devdata->hs_wdth_mask) <<
+		mxsfb->devdata->hs_wdth_shift;
+}
+
+/* Setup the MXSFB registers for decoding the pixels out of the framebuffer */
+static int mxsfb_set_pixel_fmt(struct mxsfb_drm_private *mxsfb)
+{
+	struct drm_crtc *crtc = &mxsfb->pipe.crtc;
+	struct drm_device *drm = crtc->dev;
+	const u32 format = crtc->primary->state->fb->pixel_format;
+	u32 ctrl, ctrl1;
+
+	ctrl = CTRL_BYPASS_COUNT | CTRL_MASTER;
+
+	/*
+	 * WARNING: The bus width, CTRL_SET_BUS_WIDTH(), is configured to
+	 * match the selected mode here. This differs from the original
+	 * MXSFB driver, which had the option to configure the bus width
+	 * to arbitrary value. This limitation should not pose an issue.
+	 */
+
+	/* CTRL1 contains IRQ config and status bits, preserve those. */
+	ctrl1 = readl(mxsfb->base + LCDC_CTRL1);
+	ctrl1 &= CTRL1_CUR_FRAME_DONE_IRQ_EN | CTRL1_CUR_FRAME_DONE_IRQ;
+
+	switch (format) {
+	case DRM_FORMAT_RGB565:
+		dev_dbg(drm->dev, "Setting up RGB565 mode\n");
+		ctrl |= CTRL_SET_BUS_WIDTH(STMLCDIF_16BIT);
+		ctrl |= CTRL_SET_WORD_LENGTH(0);
+		ctrl1 |= CTRL1_SET_BYTE_PACKAGING(0xf);
+		break;
+	case DRM_FORMAT_XRGB8888:
+		dev_dbg(drm->dev, "Setting up XRGB8888 mode\n");
+		ctrl |= CTRL_SET_BUS_WIDTH(STMLCDIF_24BIT);
+		ctrl |= CTRL_SET_WORD_LENGTH(3);
+		/* Do not use packed pixels = one pixel per word instead. */
+		ctrl1 |= CTRL1_SET_BYTE_PACKAGING(0x7);
+		break;
+	default:
+		dev_err(drm->dev, "Unhandled pixel format %08x\n", format);
+		return -EINVAL;
+	}
+
+	writel(ctrl1, mxsfb->base + LCDC_CTRL1);
+	writel(ctrl, mxsfb->base + LCDC_CTRL);
+
+	return 0;
+}
+
+static void mxsfb_enable_controller(struct mxsfb_drm_private *mxsfb)
+{
+	u32 reg;
+
+	if (mxsfb->clk_disp_axi)
+		clk_prepare_enable(mxsfb->clk_disp_axi);
+	clk_prepare_enable(mxsfb->clk);
+	mxsfb_enable_axi_clk(mxsfb);
+
+	/* If it was disabled, re-enable the mode again */
+	writel(CTRL_DOTCLK_MODE, mxsfb->base + LCDC_CTRL + REG_SET);
+
+	/* Enable the SYNC signals first, then the DMA engine */
+	reg = readl(mxsfb->base + LCDC_VDCTRL4);
+	reg |= VDCTRL4_SYNC_SIGNALS_ON;
+	writel(reg, mxsfb->base + LCDC_VDCTRL4);
+
+	writel(CTRL_RUN, mxsfb->base + LCDC_CTRL + REG_SET);
+}
+
+static void mxsfb_disable_controller(struct mxsfb_drm_private *mxsfb)
+{
+	u32 reg;
+
+	/*
+	 * Even if we disable the controller here, it will still continue
+	 * until its FIFOs are running out of data
+	 */
+	writel(CTRL_DOTCLK_MODE, mxsfb->base + LCDC_CTRL + REG_CLR);
+
+	readl_poll_timeout(mxsfb->base + LCDC_CTRL, reg, !(reg & CTRL_RUN),
+			   0, 1000);
+
+	reg = readl(mxsfb->base + LCDC_VDCTRL4);
+	reg &= ~VDCTRL4_SYNC_SIGNALS_ON;
+	writel(reg, mxsfb->base + LCDC_VDCTRL4);
+
+	mxsfb_disable_axi_clk(mxsfb);
+
+	clk_disable_unprepare(mxsfb->clk);
+	if (mxsfb->clk_disp_axi)
+		clk_disable_unprepare(mxsfb->clk_disp_axi);
+}
+
+static void mxsfb_crtc_mode_set_nofb(struct mxsfb_drm_private *mxsfb)
+{
+	struct drm_display_mode *m = &mxsfb->pipe.crtc.state->adjusted_mode;
+	const u32 bus_flags = mxsfb->connector.display_info.bus_flags;
+	u32 vdctrl0, vsync_pulse_len, hsync_pulse_len;
+	int err;
+
+	/*
+	 * It seems, you can't re-program the controller if it is still
+	 * running. This may lead to shifted pictures (FIFO issue?), so
+	 * first stop the controller and drain its FIFOs.
+	 */
+	mxsfb_enable_axi_clk(mxsfb);
+
+	/* Clear the FIFOs */
+	writel(CTRL1_FIFO_CLEAR, mxsfb->base + LCDC_CTRL1 + REG_SET);
+
+	err = mxsfb_set_pixel_fmt(mxsfb);
+	if (err)
+		return;
+
+	clk_set_rate(mxsfb->clk, m->crtc_clock * 1000);
+
+	writel(TRANSFER_COUNT_SET_VCOUNT(m->crtc_vdisplay) |
+	       TRANSFER_COUNT_SET_HCOUNT(m->crtc_hdisplay),
+	       mxsfb->base + mxsfb->devdata->transfer_count);
+
+	vsync_pulse_len = m->crtc_vsync_end - m->crtc_vsync_start;
+
+	vdctrl0 = VDCTRL0_ENABLE_PRESENT |	/* Always in DOTCLOCK mode */
+		  VDCTRL0_VSYNC_PERIOD_UNIT |
+		  VDCTRL0_VSYNC_PULSE_WIDTH_UNIT |
+		  VDCTRL0_SET_VSYNC_PULSE_WIDTH(vsync_pulse_len);
+	if (m->flags & DRM_MODE_FLAG_PHSYNC)
+		vdctrl0 |= VDCTRL0_HSYNC_ACT_HIGH;
+	if (m->flags & DRM_MODE_FLAG_PVSYNC)
+		vdctrl0 |= VDCTRL0_VSYNC_ACT_HIGH;
+	if (bus_flags & DRM_BUS_FLAG_DE_HIGH)
+		vdctrl0 |= VDCTRL0_ENABLE_ACT_HIGH;
+	if (bus_flags & DRM_BUS_FLAG_PIXDATA_NEGEDGE)
+		vdctrl0 |= VDCTRL0_DOTCLK_ACT_FALLING;
+
+	writel(vdctrl0, mxsfb->base + LCDC_VDCTRL0);
+
+	/* Frame length in lines. */
+	writel(m->crtc_vtotal, mxsfb->base + LCDC_VDCTRL1);
+
+	/* Line length in units of clocks or pixels. */
+	hsync_pulse_len = m->crtc_hsync_end - m->crtc_hsync_start;
+	writel(set_hsync_pulse_width(mxsfb, hsync_pulse_len) |
+	       VDCTRL2_SET_HSYNC_PERIOD(m->crtc_htotal),
+	       mxsfb->base + LCDC_VDCTRL2);
+
+	writel(SET_HOR_WAIT_CNT(m->crtc_hblank_end - m->crtc_hsync_end) |
+	       SET_VERT_WAIT_CNT(m->crtc_vblank_end - m->crtc_vsync_end),
+	       mxsfb->base + LCDC_VDCTRL3);
+
+	writel(SET_DOTCLK_H_VALID_DATA_CNT(m->hdisplay),
+	       mxsfb->base + LCDC_VDCTRL4);
+
+	mxsfb_disable_axi_clk(mxsfb);
+}
+
+void mxsfb_crtc_enable(struct mxsfb_drm_private *mxsfb)
+{
+	mxsfb_crtc_mode_set_nofb(mxsfb);
+	mxsfb_enable_controller(mxsfb);
+}
+
+void mxsfb_crtc_disable(struct mxsfb_drm_private *mxsfb)
+{
+	mxsfb_disable_controller(mxsfb);
+}
+
+void mxsfb_plane_atomic_update(struct mxsfb_drm_private *mxsfb,
+			       struct drm_plane_state *state)
+{
+	struct drm_simple_display_pipe *pipe = &mxsfb->pipe;
+	struct drm_crtc *crtc = &pipe->crtc;
+	struct drm_framebuffer *fb = pipe->plane.state->fb;
+	struct drm_pending_vblank_event *event;
+	struct drm_gem_cma_object *gem;
+
+	if (!crtc)
+		return;
+
+	spin_lock_irq(&crtc->dev->event_lock);
+	event = crtc->state->event;
+	if (event) {
+		crtc->state->event = NULL;
+
+		if (drm_crtc_vblank_get(crtc) == 0) {
+			drm_crtc_arm_vblank_event(crtc, event);
+		} else {
+			drm_crtc_send_vblank_event(crtc, event);
+		}
+	}
+	spin_unlock_irq(&crtc->dev->event_lock);
+
+	if (!fb)
+		return;
+
+	gem = drm_fb_cma_get_gem_obj(fb, 0);
+
+	mxsfb_enable_axi_clk(mxsfb);
+	writel(gem->paddr, mxsfb->base + mxsfb->devdata->next_buf);
+	mxsfb_disable_axi_clk(mxsfb);
+}
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_drv.c b/drivers/gpu/drm/mxsfb/mxsfb_drv.c
new file mode 100644
index 000000000000..79a18bf48b54
--- /dev/null
+++ b/drivers/gpu/drm/mxsfb/mxsfb_drv.c
@@ -0,0 +1,444 @@
+/*
+ * Copyright (C) 2016 Marek Vasut <marex@denx.de>
+ *
+ * This code is based on drivers/video/fbdev/mxsfb.c :
+ * Copyright (C) 2010 Juergen Beisert, Pengutronix
+ * Copyright (C) 2008-2009 Freescale Semiconductor, Inc. All Rights Reserved.
+ * Copyright (C) 2008 Embedded Alley Solutions, Inc All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <linux/spinlock.h>
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/list.h>
+#include <linux/of_device.h>
+#include <linux/of_graph.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/pm_runtime.h>
+#include <linux/reservation.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_simple_kms_helper.h>
+
+#include "mxsfb_drv.h"
+#include "mxsfb_regs.h"
+
+enum mxsfb_devtype {
+	MXSFB_V3,
+	MXSFB_V4,
+};
+
+static const struct mxsfb_devdata mxsfb_devdata[] = {
+	[MXSFB_V3] = {
+		.transfer_count	= LCDC_V3_TRANSFER_COUNT,
+		.cur_buf	= LCDC_V3_CUR_BUF,
+		.next_buf	= LCDC_V3_NEXT_BUF,
+		.debug0		= LCDC_V3_DEBUG0,
+		.hs_wdth_mask	= 0xff,
+		.hs_wdth_shift	= 24,
+		.ipversion	= 3,
+	},
+	[MXSFB_V4] = {
+		.transfer_count	= LCDC_V4_TRANSFER_COUNT,
+		.cur_buf	= LCDC_V4_CUR_BUF,
+		.next_buf	= LCDC_V4_NEXT_BUF,
+		.debug0		= LCDC_V4_DEBUG0,
+		.hs_wdth_mask	= 0x3fff,
+		.hs_wdth_shift	= 18,
+		.ipversion	= 4,
+	},
+};
+
+static const uint32_t mxsfb_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB565
+};
+
+static struct mxsfb_drm_private *
+drm_pipe_to_mxsfb_drm_private(struct drm_simple_display_pipe *pipe)
+{
+	return container_of(pipe, struct mxsfb_drm_private, pipe);
+}
+
+void mxsfb_enable_axi_clk(struct mxsfb_drm_private *mxsfb)
+{
+	if (mxsfb->clk_axi)
+		clk_prepare_enable(mxsfb->clk_axi);
+}
+
+void mxsfb_disable_axi_clk(struct mxsfb_drm_private *mxsfb)
+{
+	if (mxsfb->clk_axi)
+		clk_disable_unprepare(mxsfb->clk_axi);
+}
+
+static const struct drm_mode_config_funcs mxsfb_mode_config_funcs = {
+	.fb_create		= drm_fb_cma_create,
+	.atomic_check		= drm_atomic_helper_check,
+	.atomic_commit		= drm_atomic_helper_commit,
+};
+
+static void mxsfb_pipe_enable(struct drm_simple_display_pipe *pipe,
+			      struct drm_crtc_state *crtc_state)
+{
+	struct mxsfb_drm_private *mxsfb = drm_pipe_to_mxsfb_drm_private(pipe);
+
+	mxsfb_crtc_enable(mxsfb);
+}
+
+static void mxsfb_pipe_disable(struct drm_simple_display_pipe *pipe)
+{
+	struct mxsfb_drm_private *mxsfb = drm_pipe_to_mxsfb_drm_private(pipe);
+
+	mxsfb_crtc_disable(mxsfb);
+}
+
+static void mxsfb_pipe_update(struct drm_simple_display_pipe *pipe,
+			      struct drm_plane_state *plane_state)
+{
+	struct mxsfb_drm_private *mxsfb = drm_pipe_to_mxsfb_drm_private(pipe);
+
+	mxsfb_plane_atomic_update(mxsfb, plane_state);
+}
+
+static int mxsfb_pipe_prepare_fb(struct drm_simple_display_pipe *pipe,
+				 struct drm_plane_state *plane_state)
+{
+	return drm_fb_cma_prepare_fb(&pipe->plane, plane_state);
+}
+
+struct drm_simple_display_pipe_funcs mxsfb_funcs = {
+	.enable		= mxsfb_pipe_enable,
+	.disable	= mxsfb_pipe_disable,
+	.update		= mxsfb_pipe_update,
+	.prepare_fb	= mxsfb_pipe_prepare_fb,
+};
+
+static int mxsfb_load(struct drm_device *drm, unsigned long flags)
+{
+	struct platform_device *pdev = to_platform_device(drm->dev);
+	struct mxsfb_drm_private *mxsfb;
+	struct resource *res;
+	int ret;
+
+	mxsfb = devm_kzalloc(&pdev->dev, sizeof(*mxsfb), GFP_KERNEL);
+	if (!mxsfb)
+		return -ENOMEM;
+
+	drm->dev_private = mxsfb;
+	mxsfb->devdata = &mxsfb_devdata[pdev->id_entry->driver_data];
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	mxsfb->base = devm_ioremap_resource(drm->dev, res);
+	if (IS_ERR(mxsfb->base))
+		return PTR_ERR(mxsfb->base);
+
+	mxsfb->clk = devm_clk_get(drm->dev, NULL);
+	if (IS_ERR(mxsfb->clk))
+		return PTR_ERR(mxsfb->clk);
+
+	mxsfb->clk_axi = devm_clk_get(drm->dev, "axi");
+	if (IS_ERR(mxsfb->clk_axi))
+		mxsfb->clk_axi = NULL;
+
+	mxsfb->clk_disp_axi = devm_clk_get(drm->dev, "disp_axi");
+	if (IS_ERR(mxsfb->clk_disp_axi))
+		mxsfb->clk_disp_axi = NULL;
+
+	ret = dma_set_mask_and_coherent(drm->dev, DMA_BIT_MASK(32));
+	if (ret)
+		return ret;
+
+	pm_runtime_enable(drm->dev);
+
+	ret = drm_vblank_init(drm, drm->mode_config.num_crtc);
+	if (ret < 0) {
+		dev_err(drm->dev, "Failed to initialise vblank\n");
+		goto err_vblank;
+	}
+
+	/* Modeset init */
+	drm_mode_config_init(drm);
+
+	ret = mxsfb_create_output(drm);
+	if (ret < 0) {
+		dev_err(drm->dev, "Failed to create outputs\n");
+		goto err_vblank;
+	}
+
+	ret = drm_simple_display_pipe_init(drm, &mxsfb->pipe, &mxsfb_funcs,
+			mxsfb_formats, ARRAY_SIZE(mxsfb_formats),
+			&mxsfb->connector);
+	if (ret < 0) {
+		dev_err(drm->dev, "Cannot setup simple display pipe\n");
+		goto err_vblank;
+	}
+
+	ret = drm_panel_attach(mxsfb->panel, &mxsfb->connector);
+	if (ret) {
+		dev_err(drm->dev, "Cannot connect panel\n");
+		goto err_vblank;
+	}
+
+	drm->mode_config.min_width	= MXSFB_MIN_XRES;
+	drm->mode_config.min_height	= MXSFB_MIN_YRES;
+	drm->mode_config.max_width	= MXSFB_MAX_XRES;
+	drm->mode_config.max_height	= MXSFB_MAX_YRES;
+	drm->mode_config.funcs		= &mxsfb_mode_config_funcs;
+
+	drm_mode_config_reset(drm);
+
+	pm_runtime_get_sync(drm->dev);
+	ret = drm_irq_install(drm, platform_get_irq(pdev, 0));
+	pm_runtime_put_sync(drm->dev);
+
+	if (ret < 0) {
+		dev_err(drm->dev, "Failed to install IRQ handler\n");
+		goto err_irq;
+	}
+
+	drm_kms_helper_poll_init(drm);
+
+	mxsfb->fbdev = drm_fbdev_cma_init(drm, 32, drm->mode_config.num_crtc,
+					  drm->mode_config.num_connector);
+	if (IS_ERR(mxsfb->fbdev)) {
+		mxsfb->fbdev = NULL;
+		dev_err(drm->dev, "Failed to init FB CMA area\n");
+		goto err_cma;
+	}
+
+	platform_set_drvdata(pdev, drm);
+
+	drm_helper_hpd_irq_event(drm);
+
+	return 0;
+
+err_cma:
+	drm_irq_uninstall(drm);
+err_irq:
+	drm_panel_detach(mxsfb->panel);
+err_vblank:
+	pm_runtime_disable(drm->dev);
+
+	return ret;
+}
+
+static void mxsfb_unload(struct drm_device *drm)
+{
+	struct mxsfb_drm_private *mxsfb = drm->dev_private;
+
+	if (mxsfb->fbdev)
+		drm_fbdev_cma_fini(mxsfb->fbdev);
+
+	drm_kms_helper_poll_fini(drm);
+	drm_mode_config_cleanup(drm);
+	drm_vblank_cleanup(drm);
+
+	pm_runtime_get_sync(drm->dev);
+	drm_irq_uninstall(drm);
+	pm_runtime_put_sync(drm->dev);
+
+	drm->dev_private = NULL;
+
+	pm_runtime_disable(drm->dev);
+}
+
+static void mxsfb_lastclose(struct drm_device *drm)
+{
+	struct mxsfb_drm_private *mxsfb = drm->dev_private;
+
+	drm_fbdev_cma_restore_mode(mxsfb->fbdev);
+}
+
+static int mxsfb_enable_vblank(struct drm_device *drm, unsigned int crtc)
+{
+	struct mxsfb_drm_private *mxsfb = drm->dev_private;
+
+	/* Clear and enable VBLANK IRQ */
+	mxsfb_enable_axi_clk(mxsfb);
+	writel(CTRL1_CUR_FRAME_DONE_IRQ, mxsfb->base + LCDC_CTRL1 + REG_CLR);
+	writel(CTRL1_CUR_FRAME_DONE_IRQ_EN, mxsfb->base + LCDC_CTRL1 + REG_SET);
+	mxsfb_disable_axi_clk(mxsfb);
+
+	return 0;
+}
+
+static void mxsfb_disable_vblank(struct drm_device *drm, unsigned int crtc)
+{
+	struct mxsfb_drm_private *mxsfb = drm->dev_private;
+
+	/* Disable and clear VBLANK IRQ */
+	mxsfb_enable_axi_clk(mxsfb);
+	writel(CTRL1_CUR_FRAME_DONE_IRQ_EN, mxsfb->base + LCDC_CTRL1 + REG_CLR);
+	writel(CTRL1_CUR_FRAME_DONE_IRQ, mxsfb->base + LCDC_CTRL1 + REG_CLR);
+	mxsfb_disable_axi_clk(mxsfb);
+}
+
+static void mxsfb_irq_preinstall(struct drm_device *drm)
+{
+	mxsfb_disable_vblank(drm, 0);
+}
+
+static irqreturn_t mxsfb_irq_handler(int irq, void *data)
+{
+	struct drm_device *drm = data;
+	struct mxsfb_drm_private *mxsfb = drm->dev_private;
+	u32 reg;
+
+	mxsfb_enable_axi_clk(mxsfb);
+
+	reg = readl(mxsfb->base + LCDC_CTRL1);
+
+	if (reg & CTRL1_CUR_FRAME_DONE_IRQ)
+		drm_crtc_handle_vblank(&mxsfb->pipe.crtc);
+
+	writel(CTRL1_CUR_FRAME_DONE_IRQ, mxsfb->base + LCDC_CTRL1 + REG_CLR);
+
+	mxsfb_disable_axi_clk(mxsfb);
+
+	return IRQ_HANDLED;
+}
+
+static const struct file_operations fops = {
+	.owner		= THIS_MODULE,
+	.open		= drm_open,
+	.release	= drm_release,
+	.unlocked_ioctl	= drm_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= drm_compat_ioctl,
+#endif
+	.poll		= drm_poll,
+	.read		= drm_read,
+	.llseek		= noop_llseek,
+	.mmap		= drm_gem_cma_mmap,
+};
+
+static struct drm_driver mxsfb_driver = {
+	.driver_features	= DRIVER_GEM | DRIVER_MODESET |
+				  DRIVER_PRIME | DRIVER_ATOMIC |
+				  DRIVER_HAVE_IRQ,
+	.lastclose		= mxsfb_lastclose,
+	.irq_handler		= mxsfb_irq_handler,
+	.irq_preinstall		= mxsfb_irq_preinstall,
+	.irq_uninstall		= mxsfb_irq_preinstall,
+	.get_vblank_counter	= drm_vblank_no_hw_counter,
+	.enable_vblank		= mxsfb_enable_vblank,
+	.disable_vblank		= mxsfb_disable_vblank,
+	.gem_free_object	= drm_gem_cma_free_object,
+	.gem_vm_ops		= &drm_gem_cma_vm_ops,
+	.dumb_create		= drm_gem_cma_dumb_create,
+	.dumb_map_offset	= drm_gem_cma_dumb_map_offset,
+	.dumb_destroy		= drm_gem_dumb_destroy,
+	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
+	.gem_prime_export	= drm_gem_prime_export,
+	.gem_prime_import	= drm_gem_prime_import,
+	.gem_prime_get_sg_table	= drm_gem_cma_prime_get_sg_table,
+	.gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table,
+	.gem_prime_vmap		= drm_gem_cma_prime_vmap,
+	.gem_prime_vunmap	= drm_gem_cma_prime_vunmap,
+	.gem_prime_mmap		= drm_gem_cma_prime_mmap,
+	.fops	= &fops,
+	.name	= "mxsfb-drm",
+	.desc	= "MXSFB Controller DRM",
+	.date	= "20160824",
+	.major	= 1,
+	.minor	= 0,
+};
+
+static const struct platform_device_id mxsfb_devtype[] = {
+	{ .name = "imx23-fb", .driver_data = MXSFB_V3, },
+	{ .name = "imx28-fb", .driver_data = MXSFB_V4, },
+	{ .name = "imx6sx-fb", .driver_data = MXSFB_V4, },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(platform, mxsfb_devtype);
+
+static const struct of_device_id mxsfb_dt_ids[] = {
+	{ .compatible = "fsl,imx23-lcdif", .data = &mxsfb_devtype[0], },
+	{ .compatible = "fsl,imx28-lcdif", .data = &mxsfb_devtype[1], },
+	{ .compatible = "fsl,imx6sx-lcdif", .data = &mxsfb_devtype[2], },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, mxsfb_dt_ids);
+
+static int mxsfb_probe(struct platform_device *pdev)
+{
+	struct drm_device *drm;
+	const struct of_device_id *of_id =
+			of_match_device(mxsfb_dt_ids, &pdev->dev);
+	int ret;
+
+	if (!pdev->dev.of_node)
+		return -ENODEV;
+
+	if (of_id)
+		pdev->id_entry = of_id->data;
+
+	drm = drm_dev_alloc(&mxsfb_driver, &pdev->dev);
+	if (!drm)
+		return -ENOMEM;
+
+	ret = mxsfb_load(drm, 0);
+	if (ret)
+		goto err_free;
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		goto err_unload;
+
+	return 0;
+
+err_unload:
+	mxsfb_unload(drm);
+err_free:
+	drm_dev_unref(drm);
+
+	return ret;
+}
+
+static int mxsfb_remove(struct platform_device *pdev)
+{
+	struct drm_device *drm = platform_get_drvdata(pdev);
+
+	drm_dev_unregister(drm);
+	mxsfb_unload(drm);
+	drm_dev_unref(drm);
+
+	return 0;
+}
+
+static struct platform_driver mxsfb_platform_driver = {
+	.probe		= mxsfb_probe,
+	.remove		= mxsfb_remove,
+	.id_table	= mxsfb_devtype,
+	.driver	= {
+		.name		= "mxsfb",
+		.of_match_table	= mxsfb_dt_ids,
+	},
+};
+
+module_platform_driver(mxsfb_platform_driver);
+
+MODULE_AUTHOR("Marek Vasut <marex@denx.de>");
+MODULE_DESCRIPTION("Freescale MXS DRM/KMS driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_drv.h b/drivers/gpu/drm/mxsfb/mxsfb_drv.h
new file mode 100644
index 000000000000..5d0883fc805b
--- /dev/null
+++ b/drivers/gpu/drm/mxsfb/mxsfb_drv.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2016 Marek Vasut <marex@denx.de>
+ *
+ * i.MX23/i.MX28/i.MX6SX MXSFB LCD controller driver.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __MXSFB_DRV_H__
+#define __MXSFB_DRV_H__
+
+struct mxsfb_devdata {
+	unsigned int	 transfer_count;
+	unsigned int	 cur_buf;
+	unsigned int	 next_buf;
+	unsigned int	 debug0;
+	unsigned int	 hs_wdth_mask;
+	unsigned int	 hs_wdth_shift;
+	unsigned int	 ipversion;
+};
+
+struct mxsfb_drm_private {
+	const struct mxsfb_devdata	*devdata;
+
+	void __iomem			*base;	/* registers */
+	struct clk			*clk;
+	struct clk			*clk_axi;
+	struct clk			*clk_disp_axi;
+
+	struct drm_simple_display_pipe	pipe;
+	struct drm_connector		connector;
+	struct drm_panel		*panel;
+	struct drm_fbdev_cma		*fbdev;
+};
+
+int mxsfb_setup_crtc(struct drm_device *dev);
+int mxsfb_create_output(struct drm_device *dev);
+
+void mxsfb_enable_axi_clk(struct mxsfb_drm_private *mxsfb);
+void mxsfb_disable_axi_clk(struct mxsfb_drm_private *mxsfb);
+
+void mxsfb_crtc_enable(struct mxsfb_drm_private *mxsfb);
+void mxsfb_crtc_disable(struct mxsfb_drm_private *mxsfb);
+void mxsfb_plane_atomic_update(struct mxsfb_drm_private *mxsfb,
+			       struct drm_plane_state *state);
+
+#endif /* __MXSFB_DRV_H__ */
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_out.c b/drivers/gpu/drm/mxsfb/mxsfb_out.c
new file mode 100644
index 000000000000..fa8d17399407
--- /dev/null
+++ b/drivers/gpu/drm/mxsfb/mxsfb_out.c
@@ -0,0 +1,131 @@
+/*
+ * Copyright (C) 2016 Marek Vasut <marex@denx.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/of_graph.h>
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_simple_kms_helper.h>
+#include <drm/drmP.h>
+
+#include "mxsfb_drv.h"
+
+static struct mxsfb_drm_private *
+drm_connector_to_mxsfb_drm_private(struct drm_connector *connector)
+{
+	return container_of(connector, struct mxsfb_drm_private, connector);
+}
+
+static int mxsfb_panel_get_modes(struct drm_connector *connector)
+{
+	struct mxsfb_drm_private *mxsfb =
+			drm_connector_to_mxsfb_drm_private(connector);
+
+	if (mxsfb->panel)
+		return mxsfb->panel->funcs->get_modes(mxsfb->panel);
+
+	return 0;
+}
+
+static const struct
+drm_connector_helper_funcs mxsfb_panel_connector_helper_funcs = {
+	.get_modes = mxsfb_panel_get_modes,
+};
+
+static enum drm_connector_status
+mxsfb_panel_connector_detect(struct drm_connector *connector, bool force)
+{
+	struct mxsfb_drm_private *mxsfb =
+			drm_connector_to_mxsfb_drm_private(connector);
+
+	if (mxsfb->panel)
+		return connector_status_connected;
+
+	return connector_status_disconnected;
+}
+
+static void mxsfb_panel_connector_destroy(struct drm_connector *connector)
+{
+	struct mxsfb_drm_private *mxsfb =
+			drm_connector_to_mxsfb_drm_private(connector);
+
+	if (mxsfb->panel)
+		drm_panel_detach(mxsfb->panel);
+
+	drm_connector_unregister(connector);
+	drm_connector_cleanup(connector);
+}
+
+static const struct drm_connector_funcs mxsfb_panel_connector_funcs = {
+	.dpms			= drm_atomic_helper_connector_dpms,
+	.detect			= mxsfb_panel_connector_detect,
+	.fill_modes		= drm_helper_probe_single_connector_modes,
+	.destroy		= mxsfb_panel_connector_destroy,
+	.reset			= drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state	= drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
+};
+
+static int mxsfb_attach_endpoint(struct drm_device *drm,
+				 const struct of_endpoint *ep)
+{
+	struct mxsfb_drm_private *mxsfb = drm->dev_private;
+	struct device_node *np;
+	struct drm_panel *panel;
+	int ret = -EPROBE_DEFER;
+
+	np = of_graph_get_remote_port_parent(ep->local_node);
+	panel = of_drm_find_panel(np);
+	of_node_put(np);
+
+	if (!panel)
+		return -EPROBE_DEFER;
+
+	mxsfb->connector.dpms = DRM_MODE_DPMS_OFF;
+	mxsfb->connector.polled = 0;
+	drm_connector_helper_add(&mxsfb->connector,
+			&mxsfb_panel_connector_helper_funcs);
+	ret = drm_connector_init(drm, &mxsfb->connector,
+				 &mxsfb_panel_connector_funcs,
+				 DRM_MODE_CONNECTOR_Unknown);
+	if (!ret)
+		mxsfb->panel = panel;
+
+	return ret;
+}
+
+int mxsfb_create_output(struct drm_device *drm)
+{
+	struct device_node *ep_np = NULL;
+	struct of_endpoint ep;
+	int ret;
+
+	for_each_endpoint_of_node(drm->dev->of_node, ep_np) {
+		ret = of_graph_parse_endpoint(ep_np, &ep);
+		if (!ret)
+			ret = mxsfb_attach_endpoint(drm, &ep);
+
+		if (ret) {
+			of_node_put(ep_np);
+			return ret;
+		}
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_regs.h b/drivers/gpu/drm/mxsfb/mxsfb_regs.h
new file mode 100644
index 000000000000..31d62cd0d3d7
--- /dev/null
+++ b/drivers/gpu/drm/mxsfb/mxsfb_regs.h
@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) 2010 Juergen Beisert, Pengutronix
+ * Copyright (C) 2016 Marek Vasut <marex@denx.de>
+ *
+ * i.MX23/i.MX28/i.MX6SX MXSFB LCD controller driver.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __MXSFB_REGS_H__
+#define __MXSFB_REGS_H__
+
+#define REG_SET	4
+#define REG_CLR	8
+
+#define LCDC_CTRL			0x00
+#define LCDC_CTRL1			0x10
+#define LCDC_V3_TRANSFER_COUNT		0x20
+#define LCDC_V4_TRANSFER_COUNT		0x30
+#define LCDC_V4_CUR_BUF			0x40
+#define LCDC_V4_NEXT_BUF		0x50
+#define LCDC_V3_CUR_BUF			0x30
+#define LCDC_V3_NEXT_BUF		0x40
+#define LCDC_VDCTRL0			0x70
+#define LCDC_VDCTRL1			0x80
+#define LCDC_VDCTRL2			0x90
+#define LCDC_VDCTRL3			0xa0
+#define LCDC_VDCTRL4			0xb0
+#define LCDC_V4_DEBUG0			0x1d0
+#define LCDC_V3_DEBUG0			0x1f0
+
+#define CTRL_SFTRST			(1 << 31)
+#define CTRL_CLKGATE			(1 << 30)
+#define CTRL_BYPASS_COUNT		(1 << 19)
+#define CTRL_VSYNC_MODE			(1 << 18)
+#define CTRL_DOTCLK_MODE		(1 << 17)
+#define CTRL_DATA_SELECT		(1 << 16)
+#define CTRL_SET_BUS_WIDTH(x)		(((x) & 0x3) << 10)
+#define CTRL_GET_BUS_WIDTH(x)		(((x) >> 10) & 0x3)
+#define CTRL_SET_WORD_LENGTH(x)		(((x) & 0x3) << 8)
+#define CTRL_GET_WORD_LENGTH(x)		(((x) >> 8) & 0x3)
+#define CTRL_MASTER			(1 << 5)
+#define CTRL_DF16			(1 << 3)
+#define CTRL_DF18			(1 << 2)
+#define CTRL_DF24			(1 << 1)
+#define CTRL_RUN			(1 << 0)
+
+#define CTRL1_FIFO_CLEAR		(1 << 21)
+#define CTRL1_SET_BYTE_PACKAGING(x)	(((x) & 0xf) << 16)
+#define CTRL1_GET_BYTE_PACKAGING(x)	(((x) >> 16) & 0xf)
+#define CTRL1_CUR_FRAME_DONE_IRQ_EN	(1 << 13)
+#define CTRL1_CUR_FRAME_DONE_IRQ	(1 << 9)
+
+#define TRANSFER_COUNT_SET_VCOUNT(x)	(((x) & 0xffff) << 16)
+#define TRANSFER_COUNT_GET_VCOUNT(x)	(((x) >> 16) & 0xffff)
+#define TRANSFER_COUNT_SET_HCOUNT(x)	((x) & 0xffff)
+#define TRANSFER_COUNT_GET_HCOUNT(x)	((x) & 0xffff)
+
+#define VDCTRL0_ENABLE_PRESENT		(1 << 28)
+#define VDCTRL0_VSYNC_ACT_HIGH		(1 << 27)
+#define VDCTRL0_HSYNC_ACT_HIGH		(1 << 26)
+#define VDCTRL0_DOTCLK_ACT_FALLING	(1 << 25)
+#define VDCTRL0_ENABLE_ACT_HIGH		(1 << 24)
+#define VDCTRL0_VSYNC_PERIOD_UNIT	(1 << 21)
+#define VDCTRL0_VSYNC_PULSE_WIDTH_UNIT	(1 << 20)
+#define VDCTRL0_HALF_LINE		(1 << 19)
+#define VDCTRL0_HALF_LINE_MODE		(1 << 18)
+#define VDCTRL0_SET_VSYNC_PULSE_WIDTH(x) ((x) & 0x3ffff)
+#define VDCTRL0_GET_VSYNC_PULSE_WIDTH(x) ((x) & 0x3ffff)
+
+#define VDCTRL2_SET_HSYNC_PERIOD(x)	((x) & 0x3ffff)
+#define VDCTRL2_GET_HSYNC_PERIOD(x)	((x) & 0x3ffff)
+
+#define VDCTRL3_MUX_SYNC_SIGNALS	(1 << 29)
+#define VDCTRL3_VSYNC_ONLY		(1 << 28)
+#define SET_HOR_WAIT_CNT(x)		(((x) & 0xfff) << 16)
+#define GET_HOR_WAIT_CNT(x)		(((x) >> 16) & 0xfff)
+#define SET_VERT_WAIT_CNT(x)		((x) & 0xffff)
+#define GET_VERT_WAIT_CNT(x)		((x) & 0xffff)
+
+#define VDCTRL4_SET_DOTCLK_DLY(x)	(((x) & 0x7) << 29) /* v4 only */
+#define VDCTRL4_GET_DOTCLK_DLY(x)	(((x) >> 29) & 0x7) /* v4 only */
+#define VDCTRL4_SYNC_SIGNALS_ON		(1 << 18)
+#define SET_DOTCLK_H_VALID_DATA_CNT(x)	((x) & 0x3ffff)
+
+#define DEBUG0_HSYNC			(1 < 26)
+#define DEBUG0_VSYNC			(1 < 25)
+
+#define MXSFB_MIN_XRES			120
+#define MXSFB_MIN_YRES			120
+#define MXSFB_MAX_XRES			0xffff
+#define MXSFB_MAX_YRES			0xffff
+
+#define RED 0
+#define GREEN 1
+#define BLUE 2
+#define TRANSP 3
+
+#define STMLCDIF_8BIT  1 /* pixel data bus to the display is of 8 bit width */
+#define STMLCDIF_16BIT 0 /* pixel data bus to the display is of 16 bit width */
+#define STMLCDIF_18BIT 2 /* pixel data bus to the display is of 18 bit width */
+#define STMLCDIF_24BIT 3 /* pixel data bus to the display is of 24 bit width */
+
+#define MXSFB_SYNC_DATA_ENABLE_HIGH_ACT	(1 << 6)
+#define MXSFB_SYNC_DOTCLK_FALLING_ACT	(1 << 7) /* negative edge sampling */
+
+#endif /* __MXSFB_REGS_H__ */
diff --git a/drivers/gpu/drm/nouveau/Kbuild b/drivers/gpu/drm/nouveau/Kbuild
index 2527bf4ca5d9..fde6e3656636 100644
--- a/drivers/gpu/drm/nouveau/Kbuild
+++ b/drivers/gpu/drm/nouveau/Kbuild
@@ -22,6 +22,7 @@ nouveau-$(CONFIG_DEBUG_FS) += nouveau_debugfs.o
 nouveau-y += nouveau_drm.o
 nouveau-y += nouveau_hwmon.o
 nouveau-$(CONFIG_COMPAT) += nouveau_ioc32.o
+nouveau-$(CONFIG_LEDS_CLASS) += nouveau_led.o
 nouveau-y += nouveau_nvif.o
 nouveau-$(CONFIG_NOUVEAU_PLATFORM_DRIVER) += nouveau_platform.o
 nouveau-y += nouveau_usif.o # userspace <-> nvif
diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
index 0cb7a18cde26..59d1d1c5de5f 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
@@ -702,7 +702,7 @@ static void nv_crtc_prepare(struct drm_crtc *crtc)
 	if (nv_two_heads(dev))
 		NVSetOwner(dev, nv_crtc->index);
 
-	drm_vblank_pre_modeset(dev, nv_crtc->index);
+	drm_crtc_vblank_off(crtc);
 	funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
 
 	NVBlankScreen(dev, nv_crtc->index, true);
@@ -734,7 +734,7 @@ static void nv_crtc_commit(struct drm_crtc *crtc)
 #endif
 
 	funcs->dpms(crtc, DRM_MODE_DPMS_ON);
-	drm_vblank_post_modeset(dev, nv_crtc->index);
+	drm_crtc_vblank_on(crtc);
 }
 
 static void nv_crtc_destroy(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/nouveau/dispnv04/overlay.c b/drivers/gpu/drm/nouveau/dispnv04/overlay.c
index ec444eac6258..a79514d440b3 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/overlay.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/overlay.c
@@ -33,7 +33,7 @@
 #include "nouveau_connector.h"
 #include "nouveau_display.h"
 #include "nvreg.h"
-
+#include "disp.h"
 
 struct nouveau_plane {
 	struct drm_plane base;
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h
index d15c296b5f33..ae49dfd1f97b 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h
@@ -34,6 +34,8 @@ struct nv50_disp_mthd_v1 {
 #define NV50_DISP_MTHD_V1_SOR_HDMI_PWR                                     0x22
 #define NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT                                  0x23
 #define NV50_DISP_MTHD_V1_SOR_DP_PWR                                       0x24
+#define NV50_DISP_MTHD_V1_SOR_DP_MST_LINK                                  0x25
+#define NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI                                  0x26
 #define NV50_DISP_MTHD_V1_PIOR_PWR                                         0x30
 	__u8  method;
 	__u16 hasht;
@@ -90,6 +92,21 @@ struct nv50_disp_sor_dp_pwr_v0 {
 	__u8  pad02[6];
 };
 
+struct nv50_disp_sor_dp_mst_link_v0 {
+	__u8  version;
+	__u8  state;
+	__u8  pad02[6];
+};
+
+struct nv50_disp_sor_dp_mst_vcpi_v0 {
+	__u8  version;
+	__u8  pad01[1];
+	__u8  start_slot;
+	__u8  num_slots;
+	__u16 pbn;
+	__u16 aligned_pbn;
+};
+
 struct nv50_disp_pior_pwr_v0 {
 	__u8  version;
 	__u8  state;
diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h
index e6e9537537cf..82235f30277c 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/class.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/class.h
@@ -52,7 +52,7 @@
 #define GM107_DISP                                    /* cl5070.h */ 0x00009470
 #define GM200_DISP                                    /* cl5070.h */ 0x00009570
 #define GP100_DISP                                    /* cl5070.h */ 0x00009770
-#define GP104_DISP                                    /* cl5070.h */ 0x00009870
+#define GP102_DISP                                    /* cl5070.h */ 0x00009870
 
 #define NV31_MPEG                                                    0x00003174
 #define G82_MPEG                                                     0x00008274
@@ -90,7 +90,7 @@
 #define GM107_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000947d
 #define GM200_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000957d
 #define GP100_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000977d
-#define GP104_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000987d
+#define GP102_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000987d
 
 #define NV50_DISP_OVERLAY_CHANNEL_DMA                 /* cl507e.h */ 0x0000507e
 #define G82_DISP_OVERLAY_CHANNEL_DMA                  /* cl507e.h */ 0x0000827e
diff --git a/drivers/gpu/drm/nouveau/include/nvif/object.h b/drivers/gpu/drm/nouveau/include/nvif/object.h
index 8d815967767f..9e58b305b020 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/object.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/object.h
@@ -66,6 +66,35 @@ void nvif_object_unmap(struct nvif_object *);
 
 #define nvif_mthd(a,b,c,d) nvif_object_mthd((a), (b), (c), (d))
 
+struct nvif_mclass {
+	s32 oclass;
+	int version;
+};
+
+#define nvif_mclass(o,m) ({                                                    \
+	struct nvif_object *object = (o);                                      \
+	struct nvif_sclass *sclass;                                            \
+	const typeof(m[0]) *mclass = (m);                                      \
+	int ret = -ENODEV;                                                     \
+	int cnt, i, j;                                                         \
+                                                                               \
+	cnt = nvif_object_sclass_get(object, &sclass);                         \
+	if (cnt >= 0) {                                                        \
+		for (i = 0; ret < 0 && mclass[i].oclass; i++) {                \
+			for (j = 0; j < cnt; j++) {                            \
+				if (mclass[i].oclass  == sclass[j].oclass &&   \
+				    mclass[i].version >= sclass[j].minver &&   \
+				    mclass[i].version <= sclass[j].maxver) {   \
+					ret = i;                               \
+					break;                                 \
+				}                                              \
+			}                                                      \
+		}                                                              \
+		nvif_object_sclass_put(&sclass);                               \
+	}                                                                      \
+	ret;                                                                   \
+})
+
 /*XXX*/
 #include <core/object.h>
 #define nvxx_object(a) ({                                                      \
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h
index d3d26a1e215d..b93f4c1a95e5 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h
@@ -8,5 +8,5 @@ int gk104_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
 int gm107_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
 int gm200_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
 int gp100_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
-int gp104_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
+int gp102_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h
index e82049667ce4..970ae753968a 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h
@@ -33,5 +33,5 @@ int gk110_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
 int gm107_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
 int gm200_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
 int gp100_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
-int gp104_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
+int gp102_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/boost.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/boost.h
index 934b0ae5521d..2ff64a20c0ec 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/boost.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/boost.h
@@ -1,6 +1,6 @@
 #ifndef __NVBIOS_BOOST_H__
 #define __NVBIOS_BOOST_H__
-u16 nvbios_boostTe(struct nvkm_bios *, u8 *, u8 *, u8 *, u8 *, u8 *, u8 *);
+u32 nvbios_boostTe(struct nvkm_bios *, u8 *, u8 *, u8 *, u8 *, u8 *, u8 *);
 
 struct nvbios_boostE {
 	u8  pstate;
@@ -8,10 +8,10 @@ struct nvbios_boostE {
 	u32 max;
 };
 
-u16 nvbios_boostEe(struct nvkm_bios *, int idx, u8 *, u8 *, u8 *, u8 *);
-u16 nvbios_boostEp(struct nvkm_bios *, int idx, u8 *, u8 *, u8 *, u8 *,
+u32 nvbios_boostEe(struct nvkm_bios *, int idx, u8 *, u8 *, u8 *, u8 *);
+u32 nvbios_boostEp(struct nvkm_bios *, int idx, u8 *, u8 *, u8 *, u8 *,
 		   struct nvbios_boostE *);
-u16 nvbios_boostEm(struct nvkm_bios *, u8, u8 *, u8 *, u8 *, u8 *,
+u32 nvbios_boostEm(struct nvkm_bios *, u8, u8 *, u8 *, u8 *, u8 *,
 		   struct nvbios_boostE *);
 
 struct nvbios_boostS {
@@ -21,7 +21,7 @@ struct nvbios_boostS {
 	u32 max;
 };
 
-u16 nvbios_boostSe(struct nvkm_bios *, int, u16, u8 *, u8 *, u8, u8);
-u16 nvbios_boostSp(struct nvkm_bios *, int, u16, u8 *, u8 *, u8, u8,
+u32 nvbios_boostSe(struct nvkm_bios *, int, u32, u8 *, u8 *, u8, u8);
+u32 nvbios_boostSp(struct nvkm_bios *, int, u32, u8 *, u8 *, u8, u8,
 		   struct nvbios_boostS *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/cstep.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/cstep.h
index 2f0e0c8e83be..76fe7d50a1ce 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/cstep.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/cstep.h
@@ -1,6 +1,6 @@
 #ifndef __NVBIOS_CSTEP_H__
 #define __NVBIOS_CSTEP_H__
-u16 nvbios_cstepTe(struct nvkm_bios *,
+u32 nvbios_cstepTe(struct nvkm_bios *,
 		   u8 *ver, u8 *hdr, u8 *cnt, u8 *len, u8 *xnr, u8 *xsz);
 
 struct nvbios_cstepE {
@@ -8,10 +8,10 @@ struct nvbios_cstepE {
 	u8  index;
 };
 
-u16 nvbios_cstepEe(struct nvkm_bios *, int idx, u8 *ver, u8 *hdr);
-u16 nvbios_cstepEp(struct nvkm_bios *, int idx, u8 *ver, u8 *hdr,
+u32 nvbios_cstepEe(struct nvkm_bios *, int idx, u8 *ver, u8 *hdr);
+u32 nvbios_cstepEp(struct nvkm_bios *, int idx, u8 *ver, u8 *hdr,
 		   struct nvbios_cstepE *);
-u16 nvbios_cstepEm(struct nvkm_bios *, u8 pstate, u8 *ver, u8 *hdr,
+u32 nvbios_cstepEm(struct nvkm_bios *, u8 pstate, u8 *ver, u8 *hdr,
 		   struct nvbios_cstepE *);
 
 struct nvbios_cstepX {
@@ -20,7 +20,7 @@ struct nvbios_cstepX {
 	u8  voltage;
 };
 
-u16 nvbios_cstepXe(struct nvkm_bios *, int idx, u8 *ver, u8 *hdr);
-u16 nvbios_cstepXp(struct nvkm_bios *, int idx, u8 *ver, u8 *hdr,
+u32 nvbios_cstepXe(struct nvkm_bios *, int idx, u8 *ver, u8 *hdr);
+u32 nvbios_cstepXp(struct nvkm_bios *, int idx, u8 *ver, u8 *hdr,
 		   struct nvbios_cstepX *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/fan.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/fan.h
index 693ea7d9ec43..a7513e8406a3 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/fan.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/fan.h
@@ -2,5 +2,5 @@
 #define __NVBIOS_FAN_H__
 #include <subdev/bios/therm.h>
 
-u16 nvbios_fan_parse(struct nvkm_bios *bios, struct nvbios_therm_fan *fan);
+u32 nvbios_fan_parse(struct nvkm_bios *bios, struct nvbios_therm_fan *fan);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/gpio.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/gpio.h
index a47d46dda704..b7a54e605469 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/gpio.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/gpio.h
@@ -6,6 +6,7 @@ enum dcb_gpio_func_name {
 	DCB_GPIO_TVDAC1 = 0x2d,
 	DCB_GPIO_FAN = 0x09,
 	DCB_GPIO_FAN_SENSE = 0x3d,
+	DCB_GPIO_LOGO_LED_PWM = 0x84,
 	DCB_GPIO_UNUSED = 0xff,
 	DCB_GPIO_VID0 = 0x04,
 	DCB_GPIO_VID1 = 0x05,
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/iccsense.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/iccsense.h
index 9cb97477248b..e933d3eede70 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/iccsense.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/iccsense.h
@@ -1,10 +1,16 @@
 #ifndef __NVBIOS_ICCSENSE_H__
 #define __NVBIOS_ICCSENSE_H__
+struct pwr_rail_resistor_t {
+	u8 mohm;
+	bool enabled;
+};
+
 struct pwr_rail_t {
 	u8 mode;
 	u8 extdev_id;
-	u8 resistor_mohm;
-	u8 rail;
+	u8 resistor_count;
+	struct pwr_rail_resistor_t resistors[3];
+	u16 config;
 };
 
 struct nvbios_iccsense {
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/perf.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/perf.h
index d3bd250103d5..478b1c0d2089 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/perf.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/perf.h
@@ -1,6 +1,6 @@
 #ifndef __NVBIOS_PERF_H__
 #define __NVBIOS_PERF_H__
-u16 nvbios_perf_table(struct nvkm_bios *, u8 *ver, u8 *hdr,
+u32 nvbios_perf_table(struct nvkm_bios *, u8 *ver, u8 *hdr,
 		      u8 *cnt, u8 *len, u8 *snr, u8 *ssz);
 
 struct nvbios_perfE {
@@ -17,9 +17,9 @@ struct nvbios_perfE {
 	u8  pcie_width;
 };
 
-u16 nvbios_perf_entry(struct nvkm_bios *, int idx,
+u32 nvbios_perf_entry(struct nvkm_bios *, int idx,
 		      u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
-u16 nvbios_perfEp(struct nvkm_bios *, int idx,
+u32 nvbios_perfEp(struct nvkm_bios *, int idx,
 		  u8 *ver, u8 *hdr, u8 *cnt, u8 *len, struct nvbios_perfE *);
 
 struct nvbios_perfS {
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/timing.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/timing.h
index 339a826aa176..38188d4c9ab5 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/timing.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/timing.h
@@ -2,10 +2,10 @@
 #define __NVBIOS_TIMING_H__
 #include <subdev/bios/ramcfg.h>
 
-u16 nvbios_timingTe(struct nvkm_bios *,
+u32 nvbios_timingTe(struct nvkm_bios *,
 		    u8 *ver, u8 *hdr, u8 *cnt, u8 *len, u8 *snr, u8 *ssz);
-u16 nvbios_timingEe(struct nvkm_bios *, int idx,
+u32 nvbios_timingEe(struct nvkm_bios *, int idx,
 		    u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
-u16 nvbios_timingEp(struct nvkm_bios *, int idx,
+u32 nvbios_timingEp(struct nvkm_bios *, int idx,
 		    u8 *ver, u8 *hdr, u8 *cnt, u8 *len, struct nvbios_ramcfg *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/vmap.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/vmap.h
index 6633c6db9281..bea31cdd1dd1 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/vmap.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/vmap.h
@@ -1,21 +1,24 @@
 #ifndef __NVBIOS_VMAP_H__
 #define __NVBIOS_VMAP_H__
 struct nvbios_vmap {
+	u8  max0;
+	u8  max1;
+	u8  max2;
 };
 
-u16 nvbios_vmap_table(struct nvkm_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
-u16 nvbios_vmap_parse(struct nvkm_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+u32 nvbios_vmap_table(struct nvkm_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u32 nvbios_vmap_parse(struct nvkm_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
 		      struct nvbios_vmap *);
 
 struct nvbios_vmap_entry {
-	u8  unk0;
+	u8  mode;
 	u8  link;
 	u32 min;
 	u32 max;
 	s32 arg[6];
 };
 
-u16 nvbios_vmap_entry(struct nvkm_bios *, int idx, u8 *ver, u8 *len);
-u16 nvbios_vmap_entry_parse(struct nvkm_bios *, int idx, u8 *ver, u8 *len,
+u32 nvbios_vmap_entry(struct nvkm_bios *, int idx, u8 *ver, u8 *len);
+u32 nvbios_vmap_entry_parse(struct nvkm_bios *, int idx, u8 *ver, u8 *len,
 			    struct nvbios_vmap_entry *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/volt.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/volt.h
index b0df610cec2b..f0baa2c7de09 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/volt.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/volt.h
@@ -13,16 +13,17 @@ struct nvbios_volt {
 	u32 base;
 
 	/* GPIO mode */
-	u8  vidmask;
-	s16 step;
+	bool ranged;
+	u8   vidmask;
+	s16  step;
 
 	/* PWM mode */
 	u32 pwm_freq;
 	u32 pwm_range;
 };
 
-u16 nvbios_volt_table(struct nvkm_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
-u16 nvbios_volt_parse(struct nvkm_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+u32 nvbios_volt_table(struct nvkm_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u32 nvbios_volt_parse(struct nvkm_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
 		      struct nvbios_volt *);
 
 struct nvbios_volt_entry {
@@ -30,7 +31,7 @@ struct nvbios_volt_entry {
 	u8  vid;
 };
 
-u16 nvbios_volt_entry(struct nvkm_bios *, int idx, u8 *ver, u8 *len);
-u16 nvbios_volt_entry_parse(struct nvkm_bios *, int idx, u8 *ver, u8 *len,
+u32 nvbios_volt_entry(struct nvkm_bios *, int idx, u8 *ver, u8 *len);
+u32 nvbios_volt_entry_parse(struct nvkm_bios *, int idx, u8 *ver, u8 *len,
 			    struct nvbios_volt_entry *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/vpstate.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/vpstate.h
new file mode 100644
index 000000000000..87f804fc3a88
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/vpstate.h
@@ -0,0 +1,24 @@
+#ifndef __NVBIOS_VPSTATE_H__
+#define __NVBIOS_VPSTATE_H__
+struct nvbios_vpstate_header {
+	u32 offset;
+
+	u8 version;
+	u8 hlen;
+	u8 ecount;
+	u8 elen;
+	u8 scount;
+	u8 slen;
+
+	u8 base_id;
+	u8 boost_id;
+	u8 tdp_id;
+};
+struct nvbios_vpstate_entry {
+	u8  pstate;
+	u16 clock_mhz;
+};
+int nvbios_vpstate_parse(struct nvkm_bios *, struct nvbios_vpstate_header *);
+int nvbios_vpstate_entry(struct nvkm_bios *, struct nvbios_vpstate_header *,
+			 u8 idx, struct nvbios_vpstate_entry *);
+#endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/clk.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/clk.h
index fb54417bc458..e5275f742977 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/clk.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/clk.h
@@ -6,6 +6,10 @@
 struct nvbios_pll;
 struct nvkm_pll_vals;
 
+#define NVKM_CLK_CSTATE_DEFAULT -1 /* POSTed default */
+#define NVKM_CLK_CSTATE_BASE    -2 /* pstate base */
+#define NVKM_CLK_CSTATE_HIGHEST -3 /* highest possible */
+
 enum nv_clk_src {
 	nv_clk_src_crystal,
 	nv_clk_src_href,
@@ -52,6 +56,7 @@ struct nvkm_cstate {
 	struct list_head head;
 	u8  voltage;
 	u32 domain[nv_clk_src_max];
+	u8  id;
 };
 
 struct nvkm_pstate {
@@ -67,7 +72,8 @@ struct nvkm_pstate {
 struct nvkm_domain {
 	enum nv_clk_src name;
 	u8 bios; /* 0xff for none */
-#define NVKM_CLK_DOM_FLAG_CORE 0x01
+#define NVKM_CLK_DOM_FLAG_CORE    0x01
+#define NVKM_CLK_DOM_FLAG_VPSTATE 0x02
 	u8 flags;
 	const char *mname;
 	int mdiv;
@@ -93,10 +99,16 @@ struct nvkm_clk {
 	int ustate_ac; /* user-requested (-1 disabled, -2 perfmon) */
 	int ustate_dc; /* user-requested (-1 disabled, -2 perfmon) */
 	int astate; /* perfmon adjustment (base) */
-	int tstate; /* thermal adjustment (max-) */
 	int dstate; /* display adjustment (min+) */
+	u8  temp;
 
 	bool allow_reclock;
+#define NVKM_CLK_BOOST_NONE 0x0
+#define NVKM_CLK_BOOST_BIOS 0x1
+#define NVKM_CLK_BOOST_FULL 0x2
+	u8  boost_mode;
+	u32 base_khz;
+	u32 boost_khz;
 
 	/*XXX: die, these are here *only* to support the completely
 	 *     bat-shit insane what-was-nouveau_hw.c code
@@ -110,7 +122,7 @@ int nvkm_clk_read(struct nvkm_clk *, enum nv_clk_src);
 int nvkm_clk_ustate(struct nvkm_clk *, int req, int pwr);
 int nvkm_clk_astate(struct nvkm_clk *, int req, int rel, bool wait);
 int nvkm_clk_dstate(struct nvkm_clk *, int req, int rel);
-int nvkm_clk_tstate(struct nvkm_clk *, int req, int rel);
+int nvkm_clk_tstate(struct nvkm_clk *, u8 temperature);
 
 int nv04_clk_new(struct nvkm_device *, int, struct nvkm_clk **);
 int nv40_clk_new(struct nvkm_device *, int, struct nvkm_clk **);
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h
index 3a410275fa71..794e432578b2 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h
@@ -93,8 +93,9 @@ int gk104_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gk20a_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gm107_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gm200_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
+int gm20b_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gp100_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
-int gp104_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
+int gp102_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 
 #include <subdev/bios.h>
 #include <subdev/bios/ramcfg.h>
@@ -156,4 +157,6 @@ struct nvkm_ram_func {
 	int (*prog)(struct nvkm_ram *);
 	void (*tidy)(struct nvkm_ram *);
 };
+
+extern const u8 gf100_pte_storage_type_map[256];
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/pmu.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/pmu.h
index e61923d5e49c..f37538eb1fe5 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/pmu.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/pmu.h
@@ -35,6 +35,8 @@ int gk110_pmu_new(struct nvkm_device *, int, struct nvkm_pmu **);
 int gk208_pmu_new(struct nvkm_device *, int, struct nvkm_pmu **);
 int gk20a_pmu_new(struct nvkm_device *, int, struct nvkm_pmu **);
 int gm107_pmu_new(struct nvkm_device *, int, struct nvkm_pmu **);
+int gp100_pmu_new(struct nvkm_device *, int, struct nvkm_pmu **);
+int gp102_pmu_new(struct nvkm_device *, int, struct nvkm_pmu **);
 
 /* interface to MEMX process running on PMU */
 struct nvkm_memx;
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h
index b765f4ffcde6..08ef9983c643 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h
@@ -15,12 +15,28 @@ struct nvkm_volt {
 
 	u32 max_uv;
 	u32 min_uv;
+
+	/*
+	 * These are fully functional map entries creating a sw ceiling for
+	 * the voltage. These all can describe different kind of curves, so
+	 * that for any given temperature a different one can return the lowest
+	 * value of all three.
+	 */
+	u8 max0_id;
+	u8 max1_id;
+	u8 max2_id;
+
+	int speedo;
 };
 
+int nvkm_volt_map(struct nvkm_volt *volt, u8 id, u8 temperature);
+int nvkm_volt_map_min(struct nvkm_volt *volt, u8 id);
 int nvkm_volt_get(struct nvkm_volt *);
-int nvkm_volt_set_id(struct nvkm_volt *, u8 id, int condition);
+int nvkm_volt_set_id(struct nvkm_volt *, u8 id, u8 min_id, u8 temp,
+		     int condition);
 
 int nv40_volt_new(struct nvkm_device *, int, struct nvkm_volt **);
+int gf100_volt_new(struct nvkm_device *, int, struct nvkm_volt **);
 int gk104_volt_new(struct nvkm_device *, int, struct nvkm_volt **);
 int gk20a_volt_new(struct nvkm_device *, int, struct nvkm_volt **);
 int gm20b_volt_new(struct nvkm_device *, int, struct nvkm_volt **);
diff --git a/drivers/gpu/drm/nouveau/nouveau_backlight.c b/drivers/gpu/drm/nouveau/nouveau_backlight.c
index f5101be806cb..8b1ca4add2ed 100644
--- a/drivers/gpu/drm/nouveau/nouveau_backlight.c
+++ b/drivers/gpu/drm/nouveau/nouveau_backlight.c
@@ -30,12 +30,37 @@
  * Register locations derived from NVClock by Roderick Colenbrander
  */
 
+#include <linux/apple-gmux.h>
 #include <linux/backlight.h>
+#include <linux/idr.h>
 
 #include "nouveau_drv.h"
 #include "nouveau_reg.h"
 #include "nouveau_encoder.h"
 
+static struct ida bl_ida;
+#define BL_NAME_SIZE 15 // 12 for name + 2 for digits + 1 for '\0'
+
+struct backlight_connector {
+	struct list_head head;
+	int id;
+};
+
+static bool
+nouveau_get_backlight_name(char backlight_name[BL_NAME_SIZE], struct backlight_connector
+		*connector)
+{
+	const int nb = ida_simple_get(&bl_ida, 0, 0, GFP_KERNEL);
+	if (nb < 0 || nb >= 100)
+		return false;
+	if (nb > 0)
+		snprintf(backlight_name, BL_NAME_SIZE, "nv_backlight%d", nb);
+	else
+		snprintf(backlight_name, BL_NAME_SIZE, "nv_backlight");
+	connector->id = nb;
+	return true;
+}
+
 static int
 nv40_get_intensity(struct backlight_device *bd)
 {
@@ -74,6 +99,8 @@ nv40_backlight_init(struct drm_connector *connector)
 	struct nvif_object *device = &drm->device.object;
 	struct backlight_properties props;
 	struct backlight_device *bd;
+	struct backlight_connector bl_connector;
+	char backlight_name[BL_NAME_SIZE];
 
 	if (!(nvif_rd32(device, NV40_PMC_BACKLIGHT) & NV40_PMC_BACKLIGHT_MASK))
 		return 0;
@@ -81,10 +108,19 @@ nv40_backlight_init(struct drm_connector *connector)
 	memset(&props, 0, sizeof(struct backlight_properties));
 	props.type = BACKLIGHT_RAW;
 	props.max_brightness = 31;
-	bd = backlight_device_register("nv_backlight", connector->kdev, drm,
+	if (!nouveau_get_backlight_name(backlight_name, &bl_connector)) {
+		NV_ERROR(drm, "Failed to retrieve a unique name for the backlight interface\n");
+		return 0;
+	}
+	bd = backlight_device_register(backlight_name , connector->kdev, drm,
 				       &nv40_bl_ops, &props);
-	if (IS_ERR(bd))
+
+	if (IS_ERR(bd)) {
+		if (bl_connector.id > 0)
+			ida_simple_remove(&bl_ida, bl_connector.id);
 		return PTR_ERR(bd);
+	}
+	list_add(&bl_connector.head, &drm->bl_connectors);
 	drm->backlight = bd;
 	bd->props.brightness = nv40_get_intensity(bd);
 	backlight_update_status(bd);
@@ -182,6 +218,8 @@ nv50_backlight_init(struct drm_connector *connector)
 	struct backlight_properties props;
 	struct backlight_device *bd;
 	const struct backlight_ops *ops;
+	struct backlight_connector bl_connector;
+	char backlight_name[BL_NAME_SIZE];
 
 	nv_encoder = find_encoder(connector, DCB_OUTPUT_LVDS);
 	if (!nv_encoder) {
@@ -203,11 +241,20 @@ nv50_backlight_init(struct drm_connector *connector)
 	memset(&props, 0, sizeof(struct backlight_properties));
 	props.type = BACKLIGHT_RAW;
 	props.max_brightness = 100;
-	bd = backlight_device_register("nv_backlight", connector->kdev,
+	if (!nouveau_get_backlight_name(backlight_name, &bl_connector)) {
+		NV_ERROR(drm, "Failed to retrieve a unique name for the backlight interface\n");
+		return 0;
+	}
+	bd = backlight_device_register(backlight_name , connector->kdev,
 				       nv_encoder, ops, &props);
-	if (IS_ERR(bd))
+
+	if (IS_ERR(bd)) {
+		if (bl_connector.id > 0)
+			ida_simple_remove(&bl_ida, bl_connector.id);
 		return PTR_ERR(bd);
+	}
 
+	list_add(&bl_connector.head, &drm->bl_connectors);
 	drm->backlight = bd;
 	bd->props.brightness = bd->ops->get_brightness(bd);
 	backlight_update_status(bd);
@@ -221,6 +268,13 @@ nouveau_backlight_init(struct drm_device *dev)
 	struct nvif_device *device = &drm->device;
 	struct drm_connector *connector;
 
+	if (apple_gmux_present()) {
+		NV_INFO(drm, "Apple GMUX detected: not registering Nouveau backlight interface\n");
+		return 0;
+	}
+
+	INIT_LIST_HEAD(&drm->bl_connectors);
+
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		if (connector->connector_type != DRM_MODE_CONNECTOR_LVDS &&
 		    connector->connector_type != DRM_MODE_CONNECTOR_eDP)
@@ -232,6 +286,7 @@ nouveau_backlight_init(struct drm_device *dev)
 		case NV_DEVICE_INFO_V0_TESLA:
 		case NV_DEVICE_INFO_V0_FERMI:
 		case NV_DEVICE_INFO_V0_KEPLER:
+		case NV_DEVICE_INFO_V0_MAXWELL:
 			return nv50_backlight_init(connector);
 		default:
 			break;
@@ -246,9 +301,27 @@ void
 nouveau_backlight_exit(struct drm_device *dev)
 {
 	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct backlight_connector *connector;
+
+	list_for_each_entry(connector, &drm->bl_connectors, head) {
+		if (connector->id >= 0)
+			ida_simple_remove(&bl_ida, connector->id);
+	}
 
 	if (drm->backlight) {
 		backlight_device_unregister(drm->backlight);
 		drm->backlight = NULL;
 	}
 }
+
+void
+nouveau_backlight_ctor(void)
+{
+	ida_init(&bl_ida);
+}
+
+void
+nouveau_backlight_dtor(void)
+{
+	ida_destroy(&bl_ida);
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index a1570b109434..23ffe8571a99 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -333,6 +333,9 @@ get_fp_strap(struct drm_device *dev, struct nvbios *bios)
 	if (bios->major_version < 5 && bios->data[0x48] & 0x4)
 		return NVReadVgaCrtc5758(dev, 0, 0xf) & 0xf;
 
+	if (drm->device.info.family >= NV_DEVICE_INFO_V0_MAXWELL)
+		return nvif_rd32(device, 0x001800) & 0x0000000f;
+	else
 	if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
 		return (nvif_rd32(device, NV_PEXTDEV_BOOT_0) >> 24) & 0xf;
 	else
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.h b/drivers/gpu/drm/nouveau/nouveau_bios.h
index 0067586eb015..18eb061ccafb 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.h
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.h
@@ -31,10 +31,8 @@
 
 #define DCB_LOC_ON_CHIP 0
 
-#define ROM16(x) le16_to_cpu(*(u16 *)&(x))
-#define ROM32(x) le32_to_cpu(*(u32 *)&(x))
-#define ROM48(x) ({ u8 *p = &(x); (u64)ROM16(p[4]) << 32 | ROM32(p[0]); })
-#define ROM64(x) le64_to_cpu(*(u64 *)&(x))
+#define ROM16(x) get_unaligned_le16(&(x))
+#define ROM32(x) get_unaligned_le32(&(x))
 #define ROMPTR(d,x) ({            \
 	struct nouveau_drm *drm = nouveau_drm((d)); \
 	ROM16(x) ? &drm->vbios.data[ROM16(x)] : NULL; \
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 343b8659472c..dd07ca140d12 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -83,13 +83,13 @@ nv10_bo_get_tile_region(struct drm_device *dev, int i)
 
 static void
 nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile,
-			struct fence *fence)
+			struct dma_fence *fence)
 {
 	struct nouveau_drm *drm = nouveau_drm(dev);
 
 	if (tile) {
 		spin_lock(&drm->tile.lock);
-		tile->fence = (struct nouveau_fence *)fence_get(fence);
+		tile->fence = (struct nouveau_fence *)dma_fence_get(fence);
 		tile->used = false;
 		spin_unlock(&drm->tile.lock);
 	}
@@ -1209,6 +1209,7 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
 			       nvbo->page_shift != vma->vm->mmu->lpg_shift)) {
 			nvkm_vm_map(vma, new_mem->mm_node);
 		} else {
+			WARN_ON(ttm_bo_wait(bo, false, false));
 			nvkm_vm_unmap(vma);
 		}
 	}
@@ -1243,7 +1244,7 @@ nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
 {
 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
 	struct drm_device *dev = drm->dev;
-	struct fence *fence = reservation_object_get_excl(bo->resv);
+	struct dma_fence *fence = reservation_object_get_excl(bo->resv);
 
 	nv10_bo_put_tile_region(dev, *old_tile, fence);
 	*old_tile = new_tile;
@@ -1561,6 +1562,7 @@ struct ttm_bo_driver nouveau_bo_driver = {
 	.ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
 	.invalidate_caches = nouveau_bo_invalidate_caches,
 	.init_mem_type = nouveau_bo_init_mem_type,
+	.eviction_valuable = ttm_bo_eviction_valuable,
 	.evict_flags = nouveau_bo_evict_flags,
 	.move_notify = nouveau_bo_move_ntfy,
 	.move = nouveau_bo_move,
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index c1084088f9e4..947c200655b4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -30,6 +30,7 @@
 #include <linux/vga_switcheroo.h>
 
 #include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_crtc_helper.h>
 
@@ -47,6 +48,301 @@
 #include <nvif/cl0046.h>
 #include <nvif/event.h>
 
+struct drm_display_mode *
+nouveau_conn_native_mode(struct drm_connector *connector)
+{
+	const struct drm_connector_helper_funcs *helper = connector->helper_private;
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
+	struct drm_device *dev = connector->dev;
+	struct drm_display_mode *mode, *largest = NULL;
+	int high_w = 0, high_h = 0, high_v = 0;
+
+	list_for_each_entry(mode, &connector->probed_modes, head) {
+		mode->vrefresh = drm_mode_vrefresh(mode);
+		if (helper->mode_valid(connector, mode) != MODE_OK ||
+		    (mode->flags & DRM_MODE_FLAG_INTERLACE))
+			continue;
+
+		/* Use preferred mode if there is one.. */
+		if (mode->type & DRM_MODE_TYPE_PREFERRED) {
+			NV_DEBUG(drm, "native mode from preferred\n");
+			return drm_mode_duplicate(dev, mode);
+		}
+
+		/* Otherwise, take the resolution with the largest width, then
+		 * height, then vertical refresh
+		 */
+		if (mode->hdisplay < high_w)
+			continue;
+
+		if (mode->hdisplay == high_w && mode->vdisplay < high_h)
+			continue;
+
+		if (mode->hdisplay == high_w && mode->vdisplay == high_h &&
+		    mode->vrefresh < high_v)
+			continue;
+
+		high_w = mode->hdisplay;
+		high_h = mode->vdisplay;
+		high_v = mode->vrefresh;
+		largest = mode;
+	}
+
+	NV_DEBUG(drm, "native mode from largest: %dx%d@%d\n",
+		      high_w, high_h, high_v);
+	return largest ? drm_mode_duplicate(dev, largest) : NULL;
+}
+
+int
+nouveau_conn_atomic_get_property(struct drm_connector *connector,
+				 const struct drm_connector_state *state,
+				 struct drm_property *property, u64 *val)
+{
+	struct nouveau_conn_atom *asyc = nouveau_conn_atom(state);
+	struct nouveau_display *disp = nouveau_display(connector->dev);
+	struct drm_device *dev = connector->dev;
+
+	if (property == dev->mode_config.scaling_mode_property)
+		*val = asyc->scaler.mode;
+	else if (property == disp->underscan_property)
+		*val = asyc->scaler.underscan.mode;
+	else if (property == disp->underscan_hborder_property)
+		*val = asyc->scaler.underscan.hborder;
+	else if (property == disp->underscan_vborder_property)
+		*val = asyc->scaler.underscan.vborder;
+	else if (property == disp->dithering_mode)
+		*val = asyc->dither.mode;
+	else if (property == disp->dithering_depth)
+		*val = asyc->dither.depth;
+	else if (property == disp->vibrant_hue_property)
+		*val = asyc->procamp.vibrant_hue;
+	else if (property == disp->color_vibrance_property)
+		*val = asyc->procamp.color_vibrance;
+	else
+		return -EINVAL;
+
+	return 0;
+}
+
+int
+nouveau_conn_atomic_set_property(struct drm_connector *connector,
+				 struct drm_connector_state *state,
+				 struct drm_property *property, u64 val)
+{
+	struct drm_device *dev = connector->dev;
+	struct nouveau_conn_atom *asyc = nouveau_conn_atom(state);
+	struct nouveau_display *disp = nouveau_display(dev);
+
+	if (property == dev->mode_config.scaling_mode_property) {
+		switch (val) {
+		case DRM_MODE_SCALE_NONE:
+			/* We allow 'None' for EDID modes, even on a fixed
+			 * panel (some exist with support for lower refresh
+			 * rates, which people might want to use for power-
+			 * saving purposes).
+			 *
+			 * Non-EDID modes will force the use of GPU scaling
+			 * to the native mode regardless of this setting.
+			 */
+			switch (connector->connector_type) {
+			case DRM_MODE_CONNECTOR_LVDS:
+			case DRM_MODE_CONNECTOR_eDP:
+				/* ... except prior to G80, where the code
+				 * doesn't support such things.
+				 */
+				if (disp->disp.oclass < NV50_DISP)
+					return -EINVAL;
+				break;
+			default:
+				break;
+			}
+		case DRM_MODE_SCALE_FULLSCREEN:
+		case DRM_MODE_SCALE_CENTER:
+		case DRM_MODE_SCALE_ASPECT:
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		if (asyc->scaler.mode != val) {
+			asyc->scaler.mode = val;
+			asyc->set.scaler = true;
+		}
+	} else
+	if (property == disp->underscan_property) {
+		if (asyc->scaler.underscan.mode != val) {
+			asyc->scaler.underscan.mode = val;
+			asyc->set.scaler = true;
+		}
+	} else
+	if (property == disp->underscan_hborder_property) {
+		if (asyc->scaler.underscan.hborder != val) {
+			asyc->scaler.underscan.hborder = val;
+			asyc->set.scaler = true;
+		}
+	} else
+	if (property == disp->underscan_vborder_property) {
+		if (asyc->scaler.underscan.vborder != val) {
+			asyc->scaler.underscan.vborder = val;
+			asyc->set.scaler = true;
+		}
+	} else
+	if (property == disp->dithering_mode) {
+		if (asyc->dither.mode != val) {
+			asyc->dither.mode = val;
+			asyc->set.dither = true;
+		}
+	} else
+	if (property == disp->dithering_depth) {
+		if (asyc->dither.mode != val) {
+			asyc->dither.depth = val;
+			asyc->set.dither = true;
+		}
+	} else
+	if (property == disp->vibrant_hue_property) {
+		if (asyc->procamp.vibrant_hue != val) {
+			asyc->procamp.vibrant_hue = val;
+			asyc->set.procamp = true;
+		}
+	} else
+	if (property == disp->color_vibrance_property) {
+		if (asyc->procamp.color_vibrance != val) {
+			asyc->procamp.color_vibrance = val;
+			asyc->set.procamp = true;
+		}
+	} else {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+nouveau_conn_atomic_destroy_state(struct drm_connector *connector,
+				  struct drm_connector_state *state)
+{
+	struct nouveau_conn_atom *asyc = nouveau_conn_atom(state);
+	__drm_atomic_helper_connector_destroy_state(&asyc->state);
+	kfree(asyc);
+}
+
+struct drm_connector_state *
+nouveau_conn_atomic_duplicate_state(struct drm_connector *connector)
+{
+	struct nouveau_conn_atom *armc = nouveau_conn_atom(connector->state);
+	struct nouveau_conn_atom *asyc;
+	if (!(asyc = kmalloc(sizeof(*asyc), GFP_KERNEL)))
+		return NULL;
+	__drm_atomic_helper_connector_duplicate_state(connector, &asyc->state);
+	asyc->dither = armc->dither;
+	asyc->scaler = armc->scaler;
+	asyc->procamp = armc->procamp;
+	asyc->set.mask = 0;
+	return &asyc->state;
+}
+
+void
+nouveau_conn_reset(struct drm_connector *connector)
+{
+	struct nouveau_conn_atom *asyc;
+
+	if (WARN_ON(!(asyc = kzalloc(sizeof(*asyc), GFP_KERNEL))))
+		return;
+
+	if (connector->state)
+		__drm_atomic_helper_connector_destroy_state(connector->state);
+	__drm_atomic_helper_connector_reset(connector, &asyc->state);
+	asyc->dither.mode = DITHERING_MODE_AUTO;
+	asyc->dither.depth = DITHERING_DEPTH_AUTO;
+	asyc->scaler.mode = DRM_MODE_SCALE_NONE;
+	asyc->scaler.underscan.mode = UNDERSCAN_OFF;
+	asyc->procamp.color_vibrance = 150;
+	asyc->procamp.vibrant_hue = 90;
+
+	if (nouveau_display(connector->dev)->disp.oclass < NV50_DISP) {
+		switch (connector->connector_type) {
+		case DRM_MODE_CONNECTOR_LVDS:
+			/* See note in nouveau_conn_atomic_set_property(). */
+			asyc->scaler.mode = DRM_MODE_SCALE_FULLSCREEN;
+			break;
+		default:
+			break;
+		}
+	}
+}
+
+void
+nouveau_conn_attach_properties(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	struct nouveau_conn_atom *armc = nouveau_conn_atom(connector->state);
+	struct nouveau_display *disp = nouveau_display(dev);
+
+	/* Init DVI-I specific properties. */
+	if (connector->connector_type == DRM_MODE_CONNECTOR_DVII)
+		drm_object_attach_property(&connector->base, dev->mode_config.
+					   dvi_i_subconnector_property, 0);
+
+	/* Add overscan compensation options to digital outputs. */
+	if (disp->underscan_property &&
+	    (connector->connector_type == DRM_MODE_CONNECTOR_DVID ||
+	     connector->connector_type == DRM_MODE_CONNECTOR_DVII ||
+	     connector->connector_type == DRM_MODE_CONNECTOR_HDMIA ||
+	     connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort)) {
+		drm_object_attach_property(&connector->base,
+					   disp->underscan_property,
+					   UNDERSCAN_OFF);
+		drm_object_attach_property(&connector->base,
+					   disp->underscan_hborder_property, 0);
+		drm_object_attach_property(&connector->base,
+					   disp->underscan_vborder_property, 0);
+	}
+
+	/* Add hue and saturation options. */
+	if (disp->vibrant_hue_property)
+		drm_object_attach_property(&connector->base,
+					   disp->vibrant_hue_property,
+					   armc->procamp.vibrant_hue);
+	if (disp->color_vibrance_property)
+		drm_object_attach_property(&connector->base,
+					   disp->color_vibrance_property,
+					   armc->procamp.color_vibrance);
+
+	/* Scaling mode property. */
+	switch (connector->connector_type) {
+	case DRM_MODE_CONNECTOR_TV:
+		break;
+	case DRM_MODE_CONNECTOR_VGA:
+		if (disp->disp.oclass < NV50_DISP)
+			break; /* Can only scale on DFPs. */
+		/* Fall-through. */
+	default:
+		drm_object_attach_property(&connector->base, dev->mode_config.
+					   scaling_mode_property,
+					   armc->scaler.mode);
+		break;
+	}
+
+	/* Dithering properties. */
+	switch (connector->connector_type) {
+	case DRM_MODE_CONNECTOR_TV:
+	case DRM_MODE_CONNECTOR_VGA:
+		break;
+	default:
+		if (disp->dithering_mode) {
+			drm_object_attach_property(&connector->base,
+						   disp->dithering_mode,
+						   armc->dither.mode);
+		}
+		if (disp->dithering_depth) {
+			drm_object_attach_property(&connector->base,
+						   disp->dithering_depth,
+						   armc->dither.depth);
+		}
+		break;
+	}
+}
+
 MODULE_PARM_DESC(tv_disable, "Disable TV-out detection");
 int nouveau_tv_disable = 0;
 module_param_named(tv_disable, nouveau_tv_disable, int, 0400);
@@ -151,7 +447,9 @@ nouveau_connector_ddc_detect(struct drm_connector *connector)
 
 		if (nv_encoder->dcb->type == DCB_OUTPUT_DP) {
 			int ret = nouveau_dp_detect(nv_encoder);
-			if (ret == 0)
+			if (ret == NOUVEAU_DP_MST)
+				return NULL;
+			if (ret == NOUVEAU_DP_SST)
 				break;
 		} else
 		if ((vga_switcheroo_handler_flags() &
@@ -465,199 +763,39 @@ static int
 nouveau_connector_set_property(struct drm_connector *connector,
 			       struct drm_property *property, uint64_t value)
 {
-	struct nouveau_display *disp = nouveau_display(connector->dev);
+	struct nouveau_conn_atom *asyc = nouveau_conn_atom(connector->state);
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	struct nouveau_encoder *nv_encoder = nv_connector->detected_encoder;
 	struct drm_encoder *encoder = to_drm_encoder(nv_encoder);
-	struct drm_device *dev = connector->dev;
-	struct nouveau_crtc *nv_crtc;
 	int ret;
 
-	nv_crtc = NULL;
-	if (connector->encoder && connector->encoder->crtc)
-		nv_crtc = nouveau_crtc(connector->encoder->crtc);
-
-	/* Scaling mode */
-	if (property == dev->mode_config.scaling_mode_property) {
-		bool modeset = false;
-
-		switch (value) {
-		case DRM_MODE_SCALE_NONE:
-			/* We allow 'None' for EDID modes, even on a fixed
-			 * panel (some exist with support for lower refresh
-			 * rates, which people might want to use for power
-			 * saving purposes).
-			 *
-			 * Non-EDID modes will force the use of GPU scaling
-			 * to the native mode regardless of this setting.
-			 */
-			switch (nv_connector->type) {
-			case DCB_CONNECTOR_LVDS:
-			case DCB_CONNECTOR_LVDS_SPWG:
-			case DCB_CONNECTOR_eDP:
-				/* ... except prior to G80, where the code
-				 * doesn't support such things.
-				 */
-				if (disp->disp.oclass < NV50_DISP)
-					return -EINVAL;
-				break;
-			default:
-				break;
-			}
-			break;
-		case DRM_MODE_SCALE_FULLSCREEN:
-		case DRM_MODE_SCALE_CENTER:
-		case DRM_MODE_SCALE_ASPECT:
-			break;
-		default:
-			return -EINVAL;
-		}
-
-		/* Changing between GPU and panel scaling requires a full
-		 * modeset
-		 */
-		if ((nv_connector->scaling_mode == DRM_MODE_SCALE_NONE) ||
-		    (value == DRM_MODE_SCALE_NONE))
-			modeset = true;
-		nv_connector->scaling_mode = value;
-
-		if (!nv_crtc)
-			return 0;
-
-		if (modeset || !nv_crtc->set_scale) {
-			ret = drm_crtc_helper_set_mode(&nv_crtc->base,
-							&nv_crtc->base.mode,
-							nv_crtc->base.x,
-							nv_crtc->base.y, NULL);
-			if (!ret)
-				return -EINVAL;
-		} else {
-			ret = nv_crtc->set_scale(nv_crtc, true);
-			if (ret)
-				return ret;
-		}
-
-		return 0;
-	}
-
-	/* Underscan */
-	if (property == disp->underscan_property) {
-		if (nv_connector->underscan != value) {
-			nv_connector->underscan = value;
-			if (!nv_crtc || !nv_crtc->set_scale)
-				return 0;
-
-			return nv_crtc->set_scale(nv_crtc, true);
-		}
-
-		return 0;
-	}
-
-	if (property == disp->underscan_hborder_property) {
-		if (nv_connector->underscan_hborder != value) {
-			nv_connector->underscan_hborder = value;
-			if (!nv_crtc || !nv_crtc->set_scale)
-				return 0;
-
-			return nv_crtc->set_scale(nv_crtc, true);
-		}
-
-		return 0;
-	}
-
-	if (property == disp->underscan_vborder_property) {
-		if (nv_connector->underscan_vborder != value) {
-			nv_connector->underscan_vborder = value;
-			if (!nv_crtc || !nv_crtc->set_scale)
-				return 0;
-
-			return nv_crtc->set_scale(nv_crtc, true);
-		}
-
-		return 0;
-	}
+	if (connector->dev->mode_config.funcs->atomic_commit)
+		return drm_atomic_helper_connector_set_property(connector, property, value);
 
-	/* Dithering */
-	if (property == disp->dithering_mode) {
-		nv_connector->dithering_mode = value;
-		if (!nv_crtc || !nv_crtc->set_dither)
-			return 0;
-
-		return nv_crtc->set_dither(nv_crtc, true);
-	}
-
-	if (property == disp->dithering_depth) {
-		nv_connector->dithering_depth = value;
-		if (!nv_crtc || !nv_crtc->set_dither)
-			return 0;
-
-		return nv_crtc->set_dither(nv_crtc, true);
-	}
-
-	if (nv_crtc && nv_crtc->set_color_vibrance) {
-		/* Hue */
-		if (property == disp->vibrant_hue_property) {
-			nv_crtc->vibrant_hue = value - 90;
-			return nv_crtc->set_color_vibrance(nv_crtc, true);
-		}
-		/* Saturation */
-		if (property == disp->color_vibrance_property) {
-			nv_crtc->color_vibrance = value - 100;
-			return nv_crtc->set_color_vibrance(nv_crtc, true);
-		}
+	ret = connector->funcs->atomic_set_property(&nv_connector->base,
+						    &asyc->state,
+						    property, value);
+	if (ret) {
+		if (nv_encoder && nv_encoder->dcb->type == DCB_OUTPUT_TV)
+			return get_slave_funcs(encoder)->set_property(
+				encoder, connector, property, value);
+		return ret;
 	}
 
-	if (nv_encoder && nv_encoder->dcb->type == DCB_OUTPUT_TV)
-		return get_slave_funcs(encoder)->set_property(
-			encoder, connector, property, value);
-
-	return -EINVAL;
-}
-
-static struct drm_display_mode *
-nouveau_connector_native_mode(struct drm_connector *connector)
-{
-	const struct drm_connector_helper_funcs *helper = connector->helper_private;
-	struct nouveau_drm *drm = nouveau_drm(connector->dev);
-	struct nouveau_connector *nv_connector = nouveau_connector(connector);
-	struct drm_device *dev = connector->dev;
-	struct drm_display_mode *mode, *largest = NULL;
-	int high_w = 0, high_h = 0, high_v = 0;
+	nv_connector->scaling_mode = asyc->scaler.mode;
+	nv_connector->dithering_mode = asyc->dither.mode;
 
-	list_for_each_entry(mode, &nv_connector->base.probed_modes, head) {
-		mode->vrefresh = drm_mode_vrefresh(mode);
-		if (helper->mode_valid(connector, mode) != MODE_OK ||
-		    (mode->flags & DRM_MODE_FLAG_INTERLACE))
-			continue;
-
-		/* Use preferred mode if there is one.. */
-		if (mode->type & DRM_MODE_TYPE_PREFERRED) {
-			NV_DEBUG(drm, "native mode from preferred\n");
-			return drm_mode_duplicate(dev, mode);
-		}
-
-		/* Otherwise, take the resolution with the largest width, then
-		 * height, then vertical refresh
-		 */
-		if (mode->hdisplay < high_w)
-			continue;
-
-		if (mode->hdisplay == high_w && mode->vdisplay < high_h)
-			continue;
-
-		if (mode->hdisplay == high_w && mode->vdisplay == high_h &&
-		    mode->vrefresh < high_v)
-			continue;
-
-		high_w = mode->hdisplay;
-		high_h = mode->vdisplay;
-		high_v = mode->vrefresh;
-		largest = mode;
+	if (connector->encoder && connector->encoder->crtc) {
+		ret = drm_crtc_helper_set_mode(connector->encoder->crtc,
+					      &connector->encoder->crtc->mode,
+					       connector->encoder->crtc->x,
+					       connector->encoder->crtc->y,
+					       NULL);
+		if (!ret)
+			return -EINVAL;
 	}
 
-	NV_DEBUG(drm, "native mode from largest: %dx%d@%d\n",
-		      high_w, high_h, high_v);
-	return largest ? drm_mode_duplicate(dev, largest) : NULL;
+	return 0;
 }
 
 struct moderec {
@@ -805,8 +943,7 @@ nouveau_connector_get_modes(struct drm_connector *connector)
 	 * the list of modes.
 	 */
 	if (!nv_connector->native_mode)
-		nv_connector->native_mode =
-			nouveau_connector_native_mode(connector);
+		nv_connector->native_mode = nouveau_conn_native_mode(connector);
 	if (ret == 0 && nv_connector->native_mode) {
 		struct drm_display_mode *mode;
 
@@ -934,56 +1071,42 @@ nouveau_connector_helper_funcs = {
 	.best_encoder = nouveau_connector_best_encoder,
 };
 
+static int
+nouveau_connector_dpms(struct drm_connector *connector, int mode)
+{
+	if (connector->dev->mode_config.funcs->atomic_commit)
+		return drm_atomic_helper_connector_dpms(connector, mode);
+	return drm_helper_connector_dpms(connector, mode);
+}
+
 static const struct drm_connector_funcs
 nouveau_connector_funcs = {
-	.dpms = drm_helper_connector_dpms,
+	.dpms = nouveau_connector_dpms,
+	.reset = nouveau_conn_reset,
 	.detect = nouveau_connector_detect,
-	.destroy = nouveau_connector_destroy,
+	.force = nouveau_connector_force,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = nouveau_connector_set_property,
-	.force = nouveau_connector_force
+	.destroy = nouveau_connector_destroy,
+	.atomic_duplicate_state = nouveau_conn_atomic_duplicate_state,
+	.atomic_destroy_state = nouveau_conn_atomic_destroy_state,
+	.atomic_set_property = nouveau_conn_atomic_set_property,
+	.atomic_get_property = nouveau_conn_atomic_get_property,
 };
 
 static const struct drm_connector_funcs
 nouveau_connector_funcs_lvds = {
-	.dpms = drm_helper_connector_dpms,
+	.dpms = nouveau_connector_dpms,
+	.reset = nouveau_conn_reset,
 	.detect = nouveau_connector_detect_lvds,
-	.destroy = nouveau_connector_destroy,
+	.force = nouveau_connector_force,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = nouveau_connector_set_property,
-	.force = nouveau_connector_force
-};
-
-static int
-nouveau_connector_dp_dpms(struct drm_connector *connector, int mode)
-{
-	struct nouveau_encoder *nv_encoder = NULL;
-
-	if (connector->encoder)
-		nv_encoder = nouveau_encoder(connector->encoder);
-	if (nv_encoder && nv_encoder->dcb &&
-	    nv_encoder->dcb->type == DCB_OUTPUT_DP) {
-		if (mode == DRM_MODE_DPMS_ON) {
-			u8 data = DP_SET_POWER_D0;
-			nvkm_wraux(nv_encoder->aux, DP_SET_POWER, &data, 1);
-			usleep_range(1000, 2000);
-		} else {
-			u8 data = DP_SET_POWER_D3;
-			nvkm_wraux(nv_encoder->aux, DP_SET_POWER, &data, 1);
-		}
-	}
-
-	return drm_helper_connector_dpms(connector, mode);
-}
-
-static const struct drm_connector_funcs
-nouveau_connector_funcs_dp = {
-	.dpms = nouveau_connector_dp_dpms,
-	.detect = nouveau_connector_detect,
 	.destroy = nouveau_connector_destroy,
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = nouveau_connector_set_property,
-	.force = nouveau_connector_force
+	.atomic_duplicate_state = nouveau_conn_atomic_duplicate_state,
+	.atomic_destroy_state = nouveau_conn_atomic_destroy_state,
+	.atomic_set_property = nouveau_conn_atomic_set_property,
+	.atomic_get_property = nouveau_conn_atomic_get_property,
 };
 
 static int
@@ -995,19 +1118,20 @@ nouveau_connector_hotplug(struct nvif_notify *notify)
 	struct nouveau_drm *drm = nouveau_drm(connector->dev);
 	const struct nvif_notify_conn_rep_v0 *rep = notify->data;
 	const char *name = connector->name;
+	struct nouveau_encoder *nv_encoder;
 
 	if (rep->mask & NVIF_NOTIFY_CONN_V0_IRQ) {
+		NV_DEBUG(drm, "service %s\n", name);
+		if ((nv_encoder = find_encoder(connector, DCB_OUTPUT_DP)))
+			nv50_mstm_service(nv_encoder->dp.mstm);
 	} else {
 		bool plugged = (rep->mask != NVIF_NOTIFY_CONN_V0_UNPLUG);
 
 		NV_DEBUG(drm, "%splugged %s\n", plugged ? "" : "un", name);
-
-		mutex_lock(&drm->dev->mode_config.mutex);
-		if (plugged)
-			drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
-		else
-			drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
-		mutex_unlock(&drm->dev->mode_config.mutex);
+		if ((nv_encoder = find_encoder(connector, DCB_OUTPUT_DP))) {
+			if (!plugged)
+				nv50_mstm_remove(nv_encoder->dp.mstm);
+		}
 
 		drm_helper_hpd_irq_event(connector->dev);
 	}
@@ -1188,7 +1312,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
 			return ERR_PTR(ret);
 		}
 
-		funcs = &nouveau_connector_funcs_dp;
+		funcs = &nouveau_connector_funcs;
 		break;
 	default:
 		funcs = &nouveau_connector_funcs;
@@ -1202,38 +1326,10 @@ nouveau_connector_create(struct drm_device *dev, int index)
 	drm_connector_init(dev, connector, funcs, type);
 	drm_connector_helper_add(connector, &nouveau_connector_helper_funcs);
 
-	/* Init DVI-I specific properties */
-	if (nv_connector->type == DCB_CONNECTOR_DVI_I)
-		drm_object_attach_property(&connector->base, dev->mode_config.dvi_i_subconnector_property, 0);
+	connector->funcs->reset(connector);
+	nouveau_conn_attach_properties(connector);
 
-	/* Add overscan compensation options to digital outputs */
-	if (disp->underscan_property &&
-	    (type == DRM_MODE_CONNECTOR_DVID ||
-	     type == DRM_MODE_CONNECTOR_DVII ||
-	     type == DRM_MODE_CONNECTOR_HDMIA ||
-	     type == DRM_MODE_CONNECTOR_DisplayPort)) {
-		drm_object_attach_property(&connector->base,
-					      disp->underscan_property,
-					      UNDERSCAN_OFF);
-		drm_object_attach_property(&connector->base,
-					      disp->underscan_hborder_property,
-					      0);
-		drm_object_attach_property(&connector->base,
-					      disp->underscan_vborder_property,
-					      0);
-	}
-
-	/* Add hue and saturation options */
-	if (disp->vibrant_hue_property)
-		drm_object_attach_property(&connector->base,
-					      disp->vibrant_hue_property,
-					      90);
-	if (disp->color_vibrance_property)
-		drm_object_attach_property(&connector->base,
-					      disp->color_vibrance_property,
-					      150);
-
-	/* default scaling mode */
+	/* Default scaling mode */
 	switch (nv_connector->type) {
 	case DCB_CONNECTOR_LVDS:
 	case DCB_CONNECTOR_LVDS_SPWG:
@@ -1250,23 +1346,6 @@ nouveau_connector_create(struct drm_device *dev, int index)
 		break;
 	}
 
-	/* scaling mode property */
-	switch (nv_connector->type) {
-	case DCB_CONNECTOR_TV_0:
-	case DCB_CONNECTOR_TV_1:
-	case DCB_CONNECTOR_TV_3:
-		break;
-	case DCB_CONNECTOR_VGA:
-		if (disp->disp.oclass < NV50_DISP)
-			break; /* can only scale on DFPs */
-		/* fall-through */
-	default:
-		drm_object_attach_property(&connector->base, dev->mode_config.
-					   scaling_mode_property,
-					   nv_connector->scaling_mode);
-		break;
-	}
-
 	/* dithering properties */
 	switch (nv_connector->type) {
 	case DCB_CONNECTOR_TV_0:
@@ -1275,20 +1354,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
 	case DCB_CONNECTOR_VGA:
 		break;
 	default:
-		if (disp->dithering_mode) {
-			nv_connector->dithering_mode = DITHERING_MODE_AUTO;
-			drm_object_attach_property(&connector->base,
-						   disp->dithering_mode,
-						   nv_connector->
-						   dithering_mode);
-		}
-		if (disp->dithering_depth) {
-			nv_connector->dithering_depth = DITHERING_DEPTH_AUTO;
-			drm_object_attach_property(&connector->base,
-						   disp->dithering_depth,
-						   nv_connector->
-						   dithering_depth);
-		}
+		nv_connector->dithering_mode = DITHERING_MODE_AUTO;
 		break;
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.h b/drivers/gpu/drm/nouveau/nouveau_connector.h
index 7446ee66ea04..096983c42a1f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.h
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.h
@@ -35,30 +35,6 @@
 
 struct nvkm_i2c_port;
 
-enum nouveau_underscan_type {
-	UNDERSCAN_OFF,
-	UNDERSCAN_ON,
-	UNDERSCAN_AUTO,
-};
-
-/* the enum values specifically defined here match nv50/nvd0 hw values, and
- * the code relies on this
- */
-enum nouveau_dithering_mode {
-	DITHERING_MODE_OFF = 0x00,
-	DITHERING_MODE_ON = 0x01,
-	DITHERING_MODE_DYNAMIC2X2 = 0x10 | DITHERING_MODE_ON,
-	DITHERING_MODE_STATIC2X2 = 0x18 | DITHERING_MODE_ON,
-	DITHERING_MODE_TEMPORAL = 0x20 | DITHERING_MODE_ON,
-	DITHERING_MODE_AUTO
-};
-
-enum nouveau_dithering_depth {
-	DITHERING_DEPTH_6BPC = 0x00,
-	DITHERING_DEPTH_8BPC = 0x02,
-	DITHERING_DEPTH_AUTO
-};
-
 struct nouveau_connector {
 	struct drm_connector base;
 	enum dcb_connector_type type;
@@ -70,12 +46,7 @@ struct nouveau_connector {
 	struct drm_dp_aux aux;
 
 	int dithering_mode;
-	int dithering_depth;
 	int scaling_mode;
-	bool scaling_full;
-	enum nouveau_underscan_type underscan;
-	u32 underscan_hborder;
-	u32 underscan_vborder;
 
 	struct nouveau_encoder *detected_encoder;
 	struct edid *edid;
@@ -109,5 +80,74 @@ nouveau_connector_create(struct drm_device *, int index);
 extern int nouveau_tv_disable;
 extern int nouveau_ignorelid;
 extern int nouveau_duallink;
+extern int nouveau_hdmimhz;
+
+#include <drm/drm_crtc.h>
+#define nouveau_conn_atom(p)                                                   \
+	container_of((p), struct nouveau_conn_atom, state)
+
+struct nouveau_conn_atom {
+	struct drm_connector_state state;
+
+	struct {
+		/* The enum values specifically defined here match nv50/gf119
+		 * hw values, and the code relies on this.
+		 */
+		enum {
+			DITHERING_MODE_OFF = 0x00,
+			DITHERING_MODE_ON = 0x01,
+			DITHERING_MODE_DYNAMIC2X2 = 0x10 | DITHERING_MODE_ON,
+			DITHERING_MODE_STATIC2X2 = 0x18 | DITHERING_MODE_ON,
+			DITHERING_MODE_TEMPORAL = 0x20 | DITHERING_MODE_ON,
+			DITHERING_MODE_AUTO
+		} mode;
+		enum {
+			DITHERING_DEPTH_6BPC = 0x00,
+			DITHERING_DEPTH_8BPC = 0x02,
+			DITHERING_DEPTH_AUTO
+		} depth;
+	} dither;
+
+	struct {
+		int mode;	/* DRM_MODE_SCALE_* */
+		struct {
+			enum {
+				UNDERSCAN_OFF,
+				UNDERSCAN_ON,
+				UNDERSCAN_AUTO,
+			} mode;
+			u32 hborder;
+			u32 vborder;
+		} underscan;
+		bool full;
+	} scaler;
+
+	struct {
+		int color_vibrance;
+		int vibrant_hue;
+	} procamp;
+
+	union {
+		struct {
+			bool dither:1;
+			bool scaler:1;
+			bool procamp:1;
+		};
+		u8 mask;
+	} set;
+};
 
+void nouveau_conn_attach_properties(struct drm_connector *);
+void nouveau_conn_reset(struct drm_connector *);
+struct drm_connector_state *
+nouveau_conn_atomic_duplicate_state(struct drm_connector *);
+void nouveau_conn_atomic_destroy_state(struct drm_connector *,
+				       struct drm_connector_state *);
+int nouveau_conn_atomic_set_property(struct drm_connector *,
+				     struct drm_connector_state *,
+				     struct drm_property *, u64);
+int nouveau_conn_atomic_get_property(struct drm_connector *,
+				     const struct drm_connector_state *,
+				     struct drm_property *, u64 *);
+struct drm_display_mode *nouveau_conn_native_mode(struct drm_connector *);
 #endif /* __NOUVEAU_CONNECTOR_H__ */
diff --git a/drivers/gpu/drm/nouveau/nouveau_crtc.h b/drivers/gpu/drm/nouveau/nouveau_crtc.h
index 863f10b8d818..050fcf30a0d2 100644
--- a/drivers/gpu/drm/nouveau/nouveau_crtc.h
+++ b/drivers/gpu/drm/nouveau/nouveau_crtc.h
@@ -38,8 +38,6 @@ struct nouveau_crtc {
 	uint32_t dpms_saved_fp_control;
 	uint32_t fp_users;
 	int saturation;
-	int color_vibrance;
-	int vibrant_hue;
 	int sharpness;
 	int last_dpms;
 
@@ -54,7 +52,6 @@ struct nouveau_crtc {
 
 	struct {
 		struct nouveau_bo *nvbo;
-		bool visible;
 		uint32_t offset;
 		void (*set_offset)(struct nouveau_crtc *, uint32_t offset);
 		void (*set_pos)(struct nouveau_crtc *, int x, int y);
@@ -70,10 +67,6 @@ struct nouveau_crtc {
 		int depth;
 	} lut;
 
-	int (*set_dither)(struct nouveau_crtc *crtc, bool update);
-	int (*set_scale)(struct nouveau_crtc *crtc, bool update);
-	int (*set_color_vibrance)(struct nouveau_crtc *crtc, bool update);
-
 	void (*save)(struct drm_crtc *crtc);
 	void (*restore)(struct drm_crtc *crtc);
 };
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index afbf557b23d4..cef08da1da4e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -24,7 +24,10 @@
  *
  */
 
+#include <acpi/video.h>
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 
 #include <nvif/class.h>
@@ -92,7 +95,7 @@ calc(int blanks, int blanke, int total, int line)
 	return line;
 }
 
-int
+static int
 nouveau_display_scanoutpos_head(struct drm_crtc *crtc, int *vpos, int *hpos,
 				ktime_t *stime, ktime_t *etime)
 {
@@ -158,9 +161,13 @@ nouveau_display_vblstamp(struct drm_device *dev, unsigned int pipe,
 
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		if (nouveau_crtc(crtc)->index == pipe) {
+			struct drm_display_mode *mode;
+			if (dev->mode_config.funcs->atomic_commit)
+				mode = &crtc->state->adjusted_mode;
+			else
+				mode = &crtc->hwmode;
 			return drm_calc_vbltimestamp_from_scanoutpos(dev,
-					pipe, max_error, time, flags,
-					&crtc->hwmode);
+					pipe, max_error, time, flags, mode);
 		}
 	}
 
@@ -217,10 +224,6 @@ static void
 nouveau_user_framebuffer_destroy(struct drm_framebuffer *drm_fb)
 {
 	struct nouveau_framebuffer *fb = nouveau_framebuffer(drm_fb);
-	struct nouveau_display *disp = nouveau_display(drm_fb->dev);
-
-	if (disp->fb_dtor)
-		disp->fb_dtor(drm_fb);
 
 	if (fb->nvbo)
 		drm_gem_object_unreference_unlocked(&fb->nvbo->gem);
@@ -245,57 +248,45 @@ static const struct drm_framebuffer_funcs nouveau_framebuffer_funcs = {
 };
 
 int
-nouveau_framebuffer_init(struct drm_device *dev,
-			 struct nouveau_framebuffer *nv_fb,
-			 const struct drm_mode_fb_cmd2 *mode_cmd,
-			 struct nouveau_bo *nvbo)
+nouveau_framebuffer_new(struct drm_device *dev,
+			const struct drm_mode_fb_cmd2 *mode_cmd,
+			struct nouveau_bo *nvbo,
+			struct nouveau_framebuffer **pfb)
 {
-	struct nouveau_display *disp = nouveau_display(dev);
-	struct drm_framebuffer *fb = &nv_fb->base;
+	struct nouveau_framebuffer *fb;
 	int ret;
 
-	drm_helper_mode_fill_fb_struct(fb, mode_cmd);
-	nv_fb->nvbo = nvbo;
-
-	ret = drm_framebuffer_init(dev, fb, &nouveau_framebuffer_funcs);
-	if (ret)
-		return ret;
+	if (!(fb = *pfb = kzalloc(sizeof(*fb), GFP_KERNEL)))
+		return -ENOMEM;
 
-	if (disp->fb_ctor) {
-		ret = disp->fb_ctor(fb);
-		if (ret)
-			disp->fb_dtor(fb);
-	}
+	drm_helper_mode_fill_fb_struct(&fb->base, mode_cmd);
+	fb->nvbo = nvbo;
 
+	ret = drm_framebuffer_init(dev, &fb->base, &nouveau_framebuffer_funcs);
+	if (ret)
+		kfree(fb);
 	return ret;
 }
 
-static struct drm_framebuffer *
+struct drm_framebuffer *
 nouveau_user_framebuffer_create(struct drm_device *dev,
 				struct drm_file *file_priv,
 				const struct drm_mode_fb_cmd2 *mode_cmd)
 {
-	struct nouveau_framebuffer *nouveau_fb;
+	struct nouveau_framebuffer *fb;
+	struct nouveau_bo *nvbo;
 	struct drm_gem_object *gem;
-	int ret = -ENOMEM;
+	int ret;
 
 	gem = drm_gem_object_lookup(file_priv, mode_cmd->handles[0]);
 	if (!gem)
 		return ERR_PTR(-ENOENT);
+	nvbo = nouveau_gem_object(gem);
 
-	nouveau_fb = kzalloc(sizeof(struct nouveau_framebuffer), GFP_KERNEL);
-	if (!nouveau_fb)
-		goto err_unref;
-
-	ret = nouveau_framebuffer_init(dev, nouveau_fb, mode_cmd, nouveau_gem_object(gem));
-	if (ret)
-		goto err;
+	ret = nouveau_framebuffer_new(dev, mode_cmd, nvbo, &fb);
+	if (ret == 0)
+		return &fb->base;
 
-	return &nouveau_fb->base;
-
-err:
-	kfree(nouveau_fb);
-err_unref:
 	drm_gem_object_unreference_unlocked(gem);
 	return ERR_PTR(ret);
 }
@@ -358,6 +349,55 @@ static struct nouveau_drm_prop_enum_list dither_depth[] = {
 	}                                                                      \
 } while(0)
 
+static void
+nouveau_display_hpd_work(struct work_struct *work)
+{
+	struct nouveau_drm *drm = container_of(work, typeof(*drm), hpd_work);
+
+	pm_runtime_get_sync(drm->dev->dev);
+
+	drm_helper_hpd_irq_event(drm->dev);
+
+	pm_runtime_mark_last_busy(drm->dev->dev);
+	pm_runtime_put_sync(drm->dev->dev);
+}
+
+#ifdef CONFIG_ACPI
+
+/*
+ * Hans de Goede: This define belongs in acpi/video.h, I've submitted a patch
+ * to the acpi subsys to move it there from drivers/acpi/acpi_video.c .
+ * This should be dropped once that is merged.
+ */
+#ifndef ACPI_VIDEO_NOTIFY_PROBE
+#define ACPI_VIDEO_NOTIFY_PROBE			0x81
+#endif
+
+static int
+nouveau_display_acpi_ntfy(struct notifier_block *nb, unsigned long val,
+			  void *data)
+{
+	struct nouveau_drm *drm = container_of(nb, typeof(*drm), acpi_nb);
+	struct acpi_bus_event *info = data;
+
+	if (!strcmp(info->device_class, ACPI_VIDEO_CLASS)) {
+		if (info->type == ACPI_VIDEO_NOTIFY_PROBE) {
+			/*
+			 * This may be the only indication we receive of a
+			 * connector hotplug on a runtime suspended GPU,
+			 * schedule hpd_work to check.
+			 */
+			schedule_work(&drm->hpd_work);
+
+			/* acpi-video should not generate keypresses for this */
+			return NOTIFY_BAD;
+		}
+	}
+
+	return NOTIFY_DONE;
+}
+#endif
+
 int
 nouveau_display_init(struct drm_device *dev)
 {
@@ -385,16 +425,19 @@ nouveau_display_init(struct drm_device *dev)
 }
 
 void
-nouveau_display_fini(struct drm_device *dev)
+nouveau_display_fini(struct drm_device *dev, bool suspend)
 {
 	struct nouveau_display *disp = nouveau_display(dev);
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct drm_connector *connector;
-	int head;
+	struct drm_crtc *crtc;
+
+	if (!suspend)
+		drm_crtc_force_disable_all(dev);
 
 	/* Make sure that drm and hw vblank irqs get properly disabled. */
-	for (head = 0; head < dev->mode_config.num_crtc; head++)
-		drm_vblank_off(dev, head);
+	drm_for_each_crtc(crtc, dev)
+		drm_crtc_vblank_off(crtc);
 
 	/* disable flip completion events */
 	nvif_notify_put(&drm->flip);
@@ -495,7 +538,7 @@ nouveau_display_create(struct drm_device *dev)
 
 	if (nouveau_modeset != 2 && drm->vbios.dcb.entries) {
 		static const u16 oclass[] = {
-			GP104_DISP,
+			GP102_DISP,
 			GP100_DISP,
 			GM200_DISP,
 			GM107_DISP,
@@ -530,6 +573,8 @@ nouveau_display_create(struct drm_device *dev)
 	if (ret)
 		goto disp_create_err;
 
+	drm_mode_config_reset(dev);
+
 	if (dev->mode_config.num_crtc) {
 		ret = nouveau_display_vblank_init(dev);
 		if (ret)
@@ -537,6 +582,12 @@ nouveau_display_create(struct drm_device *dev)
 	}
 
 	nouveau_backlight_init(dev);
+	INIT_WORK(&drm->hpd_work, nouveau_display_hpd_work);
+#ifdef CONFIG_ACPI
+	drm->acpi_nb.notifier_call = nouveau_display_acpi_ntfy;
+	register_acpi_notifier(&drm->acpi_nb);
+#endif
+
 	return 0;
 
 vblank_err:
@@ -552,11 +603,13 @@ nouveau_display_destroy(struct drm_device *dev)
 {
 	struct nouveau_display *disp = nouveau_display(dev);
 
+#ifdef CONFIG_ACPI
+	unregister_acpi_notifier(&nouveau_drm(dev)->acpi_nb);
+#endif
 	nouveau_backlight_exit(dev);
 	nouveau_display_vblank_fini(dev);
 
 	drm_kms_helper_poll_fini(dev);
-	drm_crtc_force_disable_all(dev);
 	drm_mode_config_cleanup(dev);
 
 	if (disp->dtor)
@@ -568,12 +621,138 @@ nouveau_display_destroy(struct drm_device *dev)
 	kfree(disp);
 }
 
+static int
+nouveau_atomic_disable_connector(struct drm_atomic_state *state,
+				 struct drm_connector *connector)
+{
+	struct drm_connector_state *connector_state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *crtc_state;
+	struct drm_plane_state *plane_state;
+	struct drm_plane *plane;
+	int ret;
+
+	if (!(crtc = connector->state->crtc))
+		return 0;
+
+	connector_state = drm_atomic_get_connector_state(state, connector);
+	if (IS_ERR(connector_state))
+		return PTR_ERR(connector_state);
+
+	ret = drm_atomic_set_crtc_for_connector(connector_state, NULL);
+	if (ret)
+		return ret;
+
+	crtc_state = drm_atomic_get_crtc_state(state, crtc);
+	if (IS_ERR(crtc_state))
+		return PTR_ERR(crtc_state);
+
+	ret = drm_atomic_set_mode_for_crtc(crtc_state, NULL);
+	if (ret)
+		return ret;
+
+	crtc_state->active = false;
+
+	drm_for_each_plane_mask(plane, connector->dev, crtc_state->plane_mask) {
+		plane_state = drm_atomic_get_plane_state(state, plane);
+		if (IS_ERR(plane_state))
+			return PTR_ERR(plane_state);
+
+		ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
+		if (ret)
+			return ret;
+
+		drm_atomic_set_fb_for_plane(plane_state, NULL);
+	}
+
+	return 0;
+}
+
+static int
+nouveau_atomic_disable(struct drm_device *dev,
+		       struct drm_modeset_acquire_ctx *ctx)
+{
+	struct drm_atomic_state *state;
+	struct drm_connector *connector;
+	int ret;
+
+	state = drm_atomic_state_alloc(dev);
+	if (!state)
+		return -ENOMEM;
+
+	state->acquire_ctx = ctx;
+
+	drm_for_each_connector(connector, dev) {
+		ret = nouveau_atomic_disable_connector(state, connector);
+		if (ret)
+			break;
+	}
+
+	if (ret == 0)
+		ret = drm_atomic_commit(state);
+	drm_atomic_state_put(state);
+	return ret;
+}
+
+static struct drm_atomic_state *
+nouveau_atomic_suspend(struct drm_device *dev)
+{
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_atomic_state *state;
+	int ret;
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+retry:
+	ret = drm_modeset_lock_all_ctx(dev, &ctx);
+	if (ret < 0) {
+		state = ERR_PTR(ret);
+		goto unlock;
+	}
+
+	state = drm_atomic_helper_duplicate_state(dev, &ctx);
+	if (IS_ERR(state))
+		goto unlock;
+
+	ret = nouveau_atomic_disable(dev, &ctx);
+	if (ret < 0) {
+		drm_atomic_state_put(state);
+		state = ERR_PTR(ret);
+		goto unlock;
+	}
+
+unlock:
+	if (PTR_ERR(state) == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry;
+	}
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+	return state;
+}
+
 int
 nouveau_display_suspend(struct drm_device *dev, bool runtime)
 {
+	struct nouveau_display *disp = nouveau_display(dev);
 	struct drm_crtc *crtc;
 
-	nouveau_display_fini(dev);
+	if (dev->mode_config.funcs->atomic_commit) {
+		if (!runtime) {
+			disp->suspend = nouveau_atomic_suspend(dev);
+			if (IS_ERR(disp->suspend)) {
+				int ret = PTR_ERR(disp->suspend);
+				disp->suspend = NULL;
+				return ret;
+			}
+		}
+
+		nouveau_display_fini(dev, true);
+		return 0;
+	}
+
+	nouveau_display_fini(dev, true);
 
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		struct nouveau_framebuffer *nouveau_fb;
@@ -600,9 +779,19 @@ nouveau_display_suspend(struct drm_device *dev, bool runtime)
 void
 nouveau_display_resume(struct drm_device *dev, bool runtime)
 {
+	struct nouveau_display *disp = nouveau_display(dev);
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct drm_crtc *crtc;
-	int ret, head;
+	int ret;
+
+	if (dev->mode_config.funcs->atomic_commit) {
+		nouveau_display_init(dev);
+		if (disp->suspend) {
+			drm_atomic_helper_resume(dev, disp->suspend);
+			disp->suspend = NULL;
+		}
+		return;
+	}
 
 	/* re-pin fb/cursors */
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
@@ -647,10 +836,6 @@ nouveau_display_resume(struct drm_device *dev, bool runtime)
 
 	drm_helper_resume_force_mode(dev);
 
-	/* Make sure that drm and hw vblank irqs get resumed if needed. */
-	for (head = 0; head < dev->mode_config.num_crtc; head++)
-		drm_vblank_on(dev, head);
-
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 
@@ -692,10 +877,7 @@ nouveau_page_flip_emit(struct nouveau_channel *chan,
 	if (ret)
 		goto fail;
 
-	if (drm->device.info.family < NV_DEVICE_INFO_V0_FERMI)
-		BEGIN_NV04(chan, NvSubSw, NV_SW_PAGE_FLIP, 1);
-	else
-		BEGIN_NVC0(chan, FermiSw, NV_SW_PAGE_FLIP, 1);
+	BEGIN_NV04(chan, NvSubSw, NV_SW_PAGE_FLIP, 1);
 	OUT_RING  (chan, 0x00000000);
 	FIRE_RING (chan);
 
@@ -724,6 +906,8 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 	struct nouveau_channel *chan;
 	struct nouveau_cli *cli;
 	struct nouveau_fence *fence;
+	struct nv04_display *dispnv04 = nv04_display(dev);
+	int head = nouveau_crtc(crtc)->index;
 	int ret;
 
 	chan = drm->channel;
@@ -770,32 +954,23 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 	drm_crtc_vblank_get(crtc);
 
 	/* Emit a page flip */
-	if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
-		ret = nv50_display_flip_next(crtc, fb, chan, swap_interval);
+	if (swap_interval) {
+		ret = RING_SPACE(chan, 8);
 		if (ret)
 			goto fail_unreserve;
-	} else {
-		struct nv04_display *dispnv04 = nv04_display(dev);
-		int head = nouveau_crtc(crtc)->index;
-
-		if (swap_interval) {
-			ret = RING_SPACE(chan, 8);
-			if (ret)
-				goto fail_unreserve;
-
-			BEGIN_NV04(chan, NvSubImageBlit, 0x012c, 1);
-			OUT_RING  (chan, 0);
-			BEGIN_NV04(chan, NvSubImageBlit, 0x0134, 1);
-			OUT_RING  (chan, head);
-			BEGIN_NV04(chan, NvSubImageBlit, 0x0100, 1);
-			OUT_RING  (chan, 0);
-			BEGIN_NV04(chan, NvSubImageBlit, 0x0130, 1);
-			OUT_RING  (chan, 0);
-		}
 
-		nouveau_bo_ref(new_bo, &dispnv04->image[head]);
+		BEGIN_NV04(chan, NvSubImageBlit, 0x012c, 1);
+		OUT_RING  (chan, 0);
+		BEGIN_NV04(chan, NvSubImageBlit, 0x0134, 1);
+		OUT_RING  (chan, head);
+		BEGIN_NV04(chan, NvSubImageBlit, 0x0100, 1);
+		OUT_RING  (chan, 0);
+		BEGIN_NV04(chan, NvSubImageBlit, 0x0130, 1);
+		OUT_RING  (chan, 0);
 	}
 
+	nouveau_bo_ref(new_bo, &dispnv04->image[head]);
+
 	ret = nouveau_page_flip_emit(chan, old_bo, new_bo, s, &fence);
 	if (ret)
 		goto fail_unreserve;
@@ -843,16 +1018,8 @@ nouveau_finish_page_flip(struct nouveau_channel *chan,
 
 	s = list_first_entry(&fctx->flip, struct nouveau_page_flip_state, head);
 	if (s->event) {
-		if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
-			drm_crtc_arm_vblank_event(s->crtc, s->event);
-		} else {
-			drm_crtc_send_vblank_event(s->crtc, s->event);
-
-			/* Give up ownership of vblank for page-flipped crtc */
-			drm_crtc_vblank_put(s->crtc);
-		}
-	}
-	else {
+		drm_crtc_arm_vblank_event(s->crtc, s->event);
+	} else {
 		/* Give up ownership of vblank for page-flipped crtc */
 		drm_crtc_vblank_put(s->crtc);
 	}
@@ -874,12 +1041,10 @@ nouveau_flip_complete(struct nvif_notify *notify)
 	struct nouveau_page_flip_state state;
 
 	if (!nouveau_finish_page_flip(chan, &state)) {
-		if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
-			nv_set_crtc_base(drm->dev, drm_crtc_index(state.crtc),
-					 state.offset + state.crtc->y *
-					 state.pitch + state.crtc->x *
-					 state.bpp / 8);
-		}
+		nv_set_crtc_base(drm->dev, drm_crtc_index(state.crtc),
+				 state.offset + state.crtc->y *
+				 state.pitch + state.crtc->x *
+				 state.bpp / 8);
 	}
 
 	return NVIF_NOTIFY_KEEP;
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.h b/drivers/gpu/drm/nouveau/nouveau_display.h
index 0420ee861ea4..4a75df06c139 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.h
+++ b/drivers/gpu/drm/nouveau/nouveau_display.h
@@ -22,8 +22,9 @@ nouveau_framebuffer(struct drm_framebuffer *fb)
 	return container_of(fb, struct nouveau_framebuffer, base);
 }
 
-int nouveau_framebuffer_init(struct drm_device *, struct nouveau_framebuffer *,
-			     const struct drm_mode_fb_cmd2 *, struct nouveau_bo *);
+int nouveau_framebuffer_new(struct drm_device *,
+			    const struct drm_mode_fb_cmd2 *,
+			    struct nouveau_bo *, struct nouveau_framebuffer **);
 
 struct nouveau_page_flip_state {
 	struct list_head head;
@@ -39,9 +40,6 @@ struct nouveau_display {
 	int  (*init)(struct drm_device *);
 	void (*fini)(struct drm_device *);
 
-	int  (*fb_ctor)(struct drm_framebuffer *);
-	void (*fb_dtor)(struct drm_framebuffer *);
-
 	struct nvif_object disp;
 
 	struct drm_property *dithering_mode;
@@ -52,6 +50,8 @@ struct nouveau_display {
 	/* not really hue and saturation: */
 	struct drm_property *vibrant_hue_property;
 	struct drm_property *color_vibrance_property;
+
+	struct drm_atomic_state *suspend;
 };
 
 static inline struct nouveau_display *
@@ -63,7 +63,7 @@ nouveau_display(struct drm_device *dev)
 int  nouveau_display_create(struct drm_device *dev);
 void nouveau_display_destroy(struct drm_device *dev);
 int  nouveau_display_init(struct drm_device *dev);
-void nouveau_display_fini(struct drm_device *dev);
+void nouveau_display_fini(struct drm_device *dev, bool suspend);
 int  nouveau_display_suspend(struct drm_device *dev, bool runtime);
 void nouveau_display_resume(struct drm_device *dev, bool runtime);
 int  nouveau_display_vblank_enable(struct drm_device *, unsigned int);
@@ -91,6 +91,8 @@ int nouveau_crtc_set_config(struct drm_mode_set *set);
 #ifdef CONFIG_DRM_NOUVEAU_BACKLIGHT
 extern int nouveau_backlight_init(struct drm_device *);
 extern void nouveau_backlight_exit(struct drm_device *);
+extern void nouveau_backlight_ctor(void);
+extern void nouveau_backlight_dtor(void);
 #else
 static inline int
 nouveau_backlight_init(struct drm_device *dev)
@@ -101,6 +103,17 @@ nouveau_backlight_init(struct drm_device *dev)
 static inline void
 nouveau_backlight_exit(struct drm_device *dev) {
 }
+
+static inline void
+nouveau_backlight_ctor(void) {
+}
+
+static inline void
+nouveau_backlight_dtor(void) {
+}
 #endif
 
+struct drm_framebuffer *
+nouveau_user_framebuffer_create(struct drm_device *, struct drm_file *,
+				const struct drm_mode_fb_cmd2 *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c
index 87d52d36f4fc..0d052e1660f8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dp.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dp.c
@@ -30,6 +30,13 @@
 #include "nouveau_encoder.h"
 #include "nouveau_crtc.h"
 
+#include <nvif/class.h>
+#include <nvif/cl5070.h>
+
+MODULE_PARM_DESC(mst, "Enable DisplayPort multi-stream (default: enabled)");
+static int nouveau_mst = 1;
+module_param_named(mst, nouveau_mst, int, 0400);
+
 static void
 nouveau_dp_probe_oui(struct drm_device *dev, struct nvkm_i2c_aux *aux, u8 *dpcd)
 {
@@ -55,14 +62,14 @@ nouveau_dp_detect(struct nouveau_encoder *nv_encoder)
 	struct drm_device *dev = nv_encoder->base.base.dev;
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nvkm_i2c_aux *aux;
-	u8 *dpcd = nv_encoder->dp.dpcd;
+	u8 dpcd[8];
 	int ret;
 
 	aux = nv_encoder->aux;
 	if (!aux)
 		return -ENODEV;
 
-	ret = nvkm_rdaux(aux, DP_DPCD_REV, dpcd, 8);
+	ret = nvkm_rdaux(aux, DP_DPCD_REV, dpcd, sizeof(dpcd));
 	if (ret)
 		return ret;
 
@@ -84,5 +91,11 @@ nouveau_dp_detect(struct nouveau_encoder *nv_encoder)
 		     nv_encoder->dp.link_nr, nv_encoder->dp.link_bw);
 
 	nouveau_dp_probe_oui(dev, aux, dpcd);
-	return 0;
+
+	ret = nv50_mstm_detect(nv_encoder->dp.mstm, dpcd, nouveau_mst);
+	if (ret == 1)
+		return NOUVEAU_DP_MST;
+	if (ret == 0)
+		return NOUVEAU_DP_SST;
+	return ret;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 3100fd88a015..59348fc41c77 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -47,6 +47,7 @@
 #include "nouveau_ttm.h"
 #include "nouveau_gem.h"
 #include "nouveau_vga.h"
+#include "nouveau_led.h"
 #include "nouveau_hwmon.h"
 #include "nouveau_acpi.h"
 #include "nouveau_bios.h"
@@ -475,6 +476,7 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags)
 	nouveau_hwmon_init(dev);
 	nouveau_accel_init(drm);
 	nouveau_fbcon_init(dev);
+	nouveau_led_init(dev);
 
 	if (nouveau_runtime_pm != 0) {
 		pm_runtime_use_autosuspend(dev->dev);
@@ -510,13 +512,14 @@ nouveau_drm_unload(struct drm_device *dev)
 		pm_runtime_forbid(dev->dev);
 	}
 
+	nouveau_led_fini(dev);
 	nouveau_fbcon_fini(dev);
 	nouveau_accel_fini(drm);
 	nouveau_hwmon_fini(dev);
 	nouveau_debugfs_fini(drm);
 
 	if (dev->mode_config.num_crtc)
-		nouveau_display_fini(dev);
+		nouveau_display_fini(dev, false);
 	nouveau_display_destroy(dev);
 
 	nouveau_bios_takedown(dev);
@@ -561,6 +564,8 @@ nouveau_do_suspend(struct drm_device *dev, bool runtime)
 	struct nouveau_cli *cli;
 	int ret;
 
+	nouveau_led_suspend(dev);
+
 	if (dev->mode_config.num_crtc) {
 		NV_INFO(drm, "suspending console...\n");
 		nouveau_fbcon_set_suspend(dev, 1);
@@ -649,6 +654,8 @@ nouveau_do_resume(struct drm_device *dev, bool runtime)
 		nouveau_fbcon_set_suspend(dev, 0);
 	}
 
+	nouveau_led_resume(dev);
+
 	return 0;
 }
 
@@ -692,7 +699,12 @@ nouveau_pmops_resume(struct device *dev)
 		return ret;
 	pci_set_master(pdev);
 
-	return nouveau_do_resume(drm_dev, false);
+	ret = nouveau_do_resume(drm_dev, false);
+
+	/* Monitors may have been connected / disconnected during suspend */
+	schedule_work(&nouveau_drm(drm_dev)->hpd_work);
+
+	return ret;
 }
 
 static int
@@ -766,6 +778,10 @@ nouveau_pmops_runtime_resume(struct device *dev)
 	nvif_mask(&device->object, 0x088488, (1 << 25), (1 << 25));
 	vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_ON);
 	drm_dev->switch_power_state = DRM_SWITCH_POWER_ON;
+
+	/* Monitors may have been connected / disconnected during suspend */
+	schedule_work(&nouveau_drm(drm_dev)->hpd_work);
+
 	return ret;
 }
 
@@ -1030,6 +1046,7 @@ static void nouveau_display_options(void)
 	DRM_DEBUG_DRIVER("... modeset      : %d\n", nouveau_modeset);
 	DRM_DEBUG_DRIVER("... runpm        : %d\n", nouveau_runtime_pm);
 	DRM_DEBUG_DRIVER("... vram_pushbuf : %d\n", nouveau_vram_pushbuf);
+	DRM_DEBUG_DRIVER("... hdmimhz      : %d\n", nouveau_hdmimhz);
 }
 
 static const struct dev_pm_ops nouveau_pm_ops = {
@@ -1105,6 +1122,7 @@ nouveau_drm_init(void)
 #endif
 
 	nouveau_register_dsm_handler();
+	nouveau_backlight_ctor();
 	return drm_pci_init(&driver_pci, &nouveau_drm_pci_driver);
 }
 
@@ -1115,6 +1133,7 @@ nouveau_drm_exit(void)
 		return;
 
 	drm_pci_exit(&driver_pci, &nouveau_drm_pci_driver);
+	nouveau_backlight_dtor();
 	nouveau_unregister_dsm_handler();
 
 #ifdef CONFIG_NOUVEAU_PLATFORM_DRIVER
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h
index 822a0212cd48..8d5ed5bfdacb 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.h
@@ -37,6 +37,8 @@
  *      - implemented limited ABI16/NVIF interop
  */
 
+#include <linux/notifier.h>
+
 #include <nvif/client.h>
 #include <nvif/device.h>
 #include <nvif/ioctl.h>
@@ -161,11 +163,19 @@ struct nouveau_drm {
 	struct nvbios vbios;
 	struct nouveau_display *display;
 	struct backlight_device *backlight;
+	struct list_head bl_connectors;
+	struct work_struct hpd_work;
+#ifdef CONFIG_ACPI
+	struct notifier_block acpi_nb;
+#endif
 
 	/* power management */
 	struct nouveau_hwmon *hwmon;
 	struct nouveau_debugfs *debugfs;
 
+	/* led management */
+	struct nouveau_led *led;
+
 	/* display power reference */
 	bool have_disp_power_ref;
 
@@ -201,6 +211,10 @@ void nouveau_drm_device_remove(struct drm_device *dev);
 	if (unlikely(drm_debug & DRM_UT_DRIVER))                               \
 		NV_PRINTK(info, &(drm)->client, f, ##a);                       \
 } while(0)
+#define NV_ATOMIC(drm,f,a...) do {                                             \
+	if (unlikely(drm_debug & DRM_UT_ATOMIC))                               \
+		NV_PRINTK(info, &(drm)->client, f, ##a);                       \
+} while(0)
 
 extern int nouveau_modeset;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h
index ee6a6d3fc80f..198e5f27682f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_encoder.h
+++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h
@@ -30,6 +30,7 @@
 #include <subdev/bios/dcb.h>
 
 #include <drm/drm_encoder_slave.h>
+#include <drm/drm_dp_mst_helper.h>
 #include "dispnv04/disp.h"
 
 #define NV_DPMS_CLEARED 0x80
@@ -57,15 +58,16 @@ struct nouveau_encoder {
 
 	union {
 		struct {
-			u8  dpcd[8];
+			struct nv50_mstm *mstm;
 			int link_nr;
 			int link_bw;
-			u32 datarate;
 		} dp;
 	};
 
 	void (*enc_save)(struct drm_encoder *encoder);
 	void (*enc_restore)(struct drm_encoder *encoder);
+	void (*update)(struct nouveau_encoder *, u8 head,
+		       struct drm_display_mode *, u8 proto, u8 depth);
 };
 
 struct nouveau_encoder *
@@ -90,9 +92,17 @@ get_slave_funcs(struct drm_encoder *enc)
 }
 
 /* nouveau_dp.c */
+enum nouveau_dp_status {
+	NOUVEAU_DP_SST,
+	NOUVEAU_DP_MST,
+};
+
 int nouveau_dp_detect(struct nouveau_encoder *);
 
 struct nouveau_connector *
 nouveau_encoder_connector_get(struct nouveau_encoder *encoder);
 
+int nv50_mstm_detect(struct nv50_mstm *, u8 dpcd[8], int allow);
+void nv50_mstm_remove(struct nv50_mstm *);
+void nv50_mstm_service(struct nv50_mstm *);
 #endif /* __NOUVEAU_ENCODER_H__ */
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index 9f5692726c16..2f2a3dcd4ad7 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -58,7 +58,7 @@ static void
 nouveau_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
 	struct nouveau_fbdev *fbcon = info->par;
-	struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+	struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev);
 	struct nvif_device *device = &drm->device;
 	int ret;
 
@@ -90,7 +90,7 @@ static void
 nouveau_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *image)
 {
 	struct nouveau_fbdev *fbcon = info->par;
-	struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+	struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev);
 	struct nvif_device *device = &drm->device;
 	int ret;
 
@@ -122,7 +122,7 @@ static void
 nouveau_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 {
 	struct nouveau_fbdev *fbcon = info->par;
-	struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+	struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev);
 	struct nvif_device *device = &drm->device;
 	int ret;
 
@@ -154,7 +154,7 @@ static int
 nouveau_fbcon_sync(struct fb_info *info)
 {
 	struct nouveau_fbdev *fbcon = info->par;
-	struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+	struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev);
 	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
@@ -181,7 +181,7 @@ static int
 nouveau_fbcon_open(struct fb_info *info, int user)
 {
 	struct nouveau_fbdev *fbcon = info->par;
-	struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+	struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev);
 	int ret = pm_runtime_get_sync(drm->dev->dev);
 	if (ret < 0 && ret != -EACCES)
 		return ret;
@@ -192,42 +192,30 @@ static int
 nouveau_fbcon_release(struct fb_info *info, int user)
 {
 	struct nouveau_fbdev *fbcon = info->par;
-	struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+	struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev);
 	pm_runtime_put(drm->dev->dev);
 	return 0;
 }
 
 static struct fb_ops nouveau_fbcon_ops = {
 	.owner = THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_open = nouveau_fbcon_open,
 	.fb_release = nouveau_fbcon_release,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
 	.fb_fillrect = nouveau_fbcon_fillrect,
 	.fb_copyarea = nouveau_fbcon_copyarea,
 	.fb_imageblit = nouveau_fbcon_imageblit,
 	.fb_sync = nouveau_fbcon_sync,
-	.fb_pan_display = drm_fb_helper_pan_display,
-	.fb_blank = drm_fb_helper_blank,
-	.fb_setcmap = drm_fb_helper_setcmap,
-	.fb_debug_enter = drm_fb_helper_debug_enter,
-	.fb_debug_leave = drm_fb_helper_debug_leave,
 };
 
 static struct fb_ops nouveau_fbcon_sw_ops = {
 	.owner = THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_open = nouveau_fbcon_open,
 	.fb_release = nouveau_fbcon_release,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
 	.fb_fillrect = drm_fb_helper_cfb_fillrect,
 	.fb_copyarea = drm_fb_helper_cfb_copyarea,
 	.fb_imageblit = drm_fb_helper_cfb_imageblit,
-	.fb_pan_display = drm_fb_helper_pan_display,
-	.fb_blank = drm_fb_helper_blank,
-	.fb_setcmap = drm_fb_helper_setcmap,
-	.fb_debug_enter = drm_fb_helper_debug_enter,
-	.fb_debug_leave = drm_fb_helper_debug_leave,
 };
 
 void
@@ -333,16 +321,15 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
 {
 	struct nouveau_fbdev *fbcon =
 		container_of(helper, struct nouveau_fbdev, helper);
-	struct drm_device *dev = fbcon->dev;
+	struct drm_device *dev = fbcon->helper.dev;
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nvif_device *device = &drm->device;
 	struct fb_info *info;
-	struct drm_framebuffer *fb;
-	struct nouveau_framebuffer *nouveau_fb;
+	struct nouveau_framebuffer *fb;
 	struct nouveau_channel *chan;
 	struct nouveau_bo *nvbo;
 	struct drm_mode_fb_cmd2 mode_cmd;
-	int size, ret;
+	int ret;
 
 	mode_cmd.width = sizes->surface_width;
 	mode_cmd.height = sizes->surface_height;
@@ -353,16 +340,17 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
 	mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
 							  sizes->surface_depth);
 
-	size = mode_cmd.pitches[0] * mode_cmd.height;
-	size = roundup(size, PAGE_SIZE);
-
-	ret = nouveau_gem_new(dev, size, 0, NOUVEAU_GEM_DOMAIN_VRAM,
-			      0, 0x0000, &nvbo);
+	ret = nouveau_gem_new(dev, mode_cmd.pitches[0] * mode_cmd.height,
+			      0, NOUVEAU_GEM_DOMAIN_VRAM, 0, 0x0000, &nvbo);
 	if (ret) {
 		NV_ERROR(drm, "failed to allocate framebuffer\n");
 		goto out;
 	}
 
+	ret = nouveau_framebuffer_new(dev, &mode_cmd, nvbo, &fb);
+	if (ret)
+		goto out_unref;
+
 	ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_VRAM, false);
 	if (ret) {
 		NV_ERROR(drm, "failed to pin fb: %d\n", ret);
@@ -377,8 +365,7 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
 
 	chan = nouveau_nofbaccel ? NULL : drm->channel;
 	if (chan && device->info.family >= NV_DEVICE_INFO_V0_TESLA) {
-		ret = nouveau_bo_vma_add(nvbo, drm->client.vm,
-					&fbcon->nouveau_fb.vma);
+		ret = nouveau_bo_vma_add(nvbo, drm->client.vm, &fb->vma);
 		if (ret) {
 			NV_ERROR(drm, "failed to map fb into chan: %d\n", ret);
 			chan = NULL;
@@ -394,13 +381,8 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
 
 	info->par = fbcon;
 
-	nouveau_framebuffer_init(dev, &fbcon->nouveau_fb, &mode_cmd, nvbo);
-
-	nouveau_fb = &fbcon->nouveau_fb;
-	fb = &nouveau_fb->base;
-
 	/* setup helper */
-	fbcon->helper.fb = fb;
+	fbcon->helper.fb = &fb->base;
 
 	strcpy(info->fix.id, "nouveaufb");
 	if (!chan)
@@ -411,14 +393,14 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
 			      FBINFO_HWACCEL_IMAGEBLIT;
 	info->flags |= FBINFO_CAN_FORCE_OUTPUT;
 	info->fbops = &nouveau_fbcon_sw_ops;
-	info->fix.smem_start = nvbo->bo.mem.bus.base +
-			       nvbo->bo.mem.bus.offset;
-	info->fix.smem_len = size;
+	info->fix.smem_start = fb->nvbo->bo.mem.bus.base +
+			       fb->nvbo->bo.mem.bus.offset;
+	info->fix.smem_len = fb->nvbo->bo.mem.num_pages << PAGE_SHIFT;
 
-	info->screen_base = nvbo_kmap_obj_iovirtual(nouveau_fb->nvbo);
-	info->screen_size = size;
+	info->screen_base = nvbo_kmap_obj_iovirtual(fb->nvbo);
+	info->screen_size = fb->nvbo->bo.mem.num_pages << PAGE_SHIFT;
 
-	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->depth);
+	drm_fb_helper_fill_fix(info, fb->base.pitches[0], fb->base.depth);
 	drm_fb_helper_fill_var(info, &fbcon->helper, sizes->fb_width, sizes->fb_height);
 
 	/* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
@@ -429,20 +411,19 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
 
 	/* To allow resizeing without swapping buffers */
 	NV_INFO(drm, "allocated %dx%d fb: 0x%llx, bo %p\n",
-		nouveau_fb->base.width, nouveau_fb->base.height,
-		nvbo->bo.offset, nvbo);
+		fb->base.width, fb->base.height, fb->nvbo->bo.offset, nvbo);
 
 	vga_switcheroo_client_fb_set(dev->pdev, info);
 	return 0;
 
 out_unlock:
 	if (chan)
-		nouveau_bo_vma_del(nvbo, &fbcon->nouveau_fb.vma);
-	nouveau_bo_unmap(nvbo);
+		nouveau_bo_vma_del(fb->nvbo, &fb->vma);
+	nouveau_bo_unmap(fb->nvbo);
 out_unpin:
-	nouveau_bo_unpin(nvbo);
+	nouveau_bo_unpin(fb->nvbo);
 out_unref:
-	nouveau_bo_ref(NULL, &nvbo);
+	nouveau_bo_ref(NULL, &fb->nvbo);
 out:
 	return ret;
 }
@@ -458,28 +439,26 @@ nouveau_fbcon_output_poll_changed(struct drm_device *dev)
 static int
 nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *fbcon)
 {
-	struct nouveau_framebuffer *nouveau_fb = &fbcon->nouveau_fb;
+	struct nouveau_framebuffer *nouveau_fb = nouveau_framebuffer(fbcon->helper.fb);
 
 	drm_fb_helper_unregister_fbi(&fbcon->helper);
 	drm_fb_helper_release_fbi(&fbcon->helper);
+	drm_fb_helper_fini(&fbcon->helper);
 
 	if (nouveau_fb->nvbo) {
-		nouveau_bo_unmap(nouveau_fb->nvbo);
 		nouveau_bo_vma_del(nouveau_fb->nvbo, &nouveau_fb->vma);
+		nouveau_bo_unmap(nouveau_fb->nvbo);
 		nouveau_bo_unpin(nouveau_fb->nvbo);
-		drm_gem_object_unreference_unlocked(&nouveau_fb->nvbo->gem);
-		nouveau_fb->nvbo = NULL;
+		drm_framebuffer_unreference(&nouveau_fb->base);
 	}
-	drm_fb_helper_fini(&fbcon->helper);
-	drm_framebuffer_unregister_private(&nouveau_fb->base);
-	drm_framebuffer_cleanup(&nouveau_fb->base);
+
 	return 0;
 }
 
 void nouveau_fbcon_gpu_lockup(struct fb_info *info)
 {
 	struct nouveau_fbdev *fbcon = info->par;
-	struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+	struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev);
 
 	NV_ERROR(drm, "GPU lockup - switching to software fbcon\n");
 	info->flags |= FBINFO_HWACCEL_DISABLED;
@@ -522,7 +501,6 @@ nouveau_fbcon_init(struct drm_device *dev)
 	if (!fbcon)
 		return -ENOMEM;
 
-	fbcon->dev = dev;
 	drm->fbcon = fbcon;
 
 	drm_fb_helper_prepare(dev, &fbcon->helper, &nouveau_fbcon_helper_funcs);
@@ -545,7 +523,8 @@ nouveau_fbcon_init(struct drm_device *dev)
 		preferred_bpp = 32;
 
 	/* disable all the possible outputs/crtcs before entering KMS mode */
-	drm_helper_disable_unused_functions(dev);
+	if (!dev->mode_config.funcs->atomic_commit)
+		drm_helper_disable_unused_functions(dev);
 
 	ret = drm_fb_helper_initial_config(&fbcon->helper, preferred_bpp);
 	if (ret)
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.h b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
index ca77ad001978..e2bca729721e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
@@ -33,8 +33,6 @@
 
 struct nouveau_fbdev {
 	struct drm_fb_helper helper;
-	struct nouveau_framebuffer nouveau_fb;
-	struct drm_device *dev;
 	unsigned int saved_flags;
 	struct nvif_object surf2d;
 	struct nvif_object clip;
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index 4bb9ab892ae1..f2f348f0160c 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -28,7 +28,7 @@
 
 #include <linux/ktime.h>
 #include <linux/hrtimer.h>
-#include <trace/events/fence.h>
+#include <trace/events/dma_fence.h>
 
 #include <nvif/cl826e.h>
 #include <nvif/notify.h>
@@ -38,11 +38,11 @@
 #include "nouveau_dma.h"
 #include "nouveau_fence.h"
 
-static const struct fence_ops nouveau_fence_ops_uevent;
-static const struct fence_ops nouveau_fence_ops_legacy;
+static const struct dma_fence_ops nouveau_fence_ops_uevent;
+static const struct dma_fence_ops nouveau_fence_ops_legacy;
 
 static inline struct nouveau_fence *
-from_fence(struct fence *fence)
+from_fence(struct dma_fence *fence)
 {
 	return container_of(fence, struct nouveau_fence, base);
 }
@@ -58,23 +58,23 @@ nouveau_fence_signal(struct nouveau_fence *fence)
 {
 	int drop = 0;
 
-	fence_signal_locked(&fence->base);
+	dma_fence_signal_locked(&fence->base);
 	list_del(&fence->head);
 	rcu_assign_pointer(fence->channel, NULL);
 
-	if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) {
+	if (test_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags)) {
 		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 
 		if (!--fctx->notify_ref)
 			drop = 1;
 	}
 
-	fence_put(&fence->base);
+	dma_fence_put(&fence->base);
 	return drop;
 }
 
 static struct nouveau_fence *
-nouveau_local_fence(struct fence *fence, struct nouveau_drm *drm) {
+nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm) {
 	struct nouveau_fence_priv *priv = (void*)drm->fence;
 
 	if (fence->ops != &nouveau_fence_ops_legacy &&
@@ -201,7 +201,7 @@ nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_cha
 
 struct nouveau_fence_work {
 	struct work_struct work;
-	struct fence_cb cb;
+	struct dma_fence_cb cb;
 	void (*func)(void *);
 	void *data;
 };
@@ -214,7 +214,7 @@ nouveau_fence_work_handler(struct work_struct *kwork)
 	kfree(work);
 }
 
-static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb)
+static void nouveau_fence_work_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 {
 	struct nouveau_fence_work *work = container_of(cb, typeof(*work), cb);
 
@@ -222,12 +222,12 @@ static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb)
 }
 
 void
-nouveau_fence_work(struct fence *fence,
+nouveau_fence_work(struct dma_fence *fence,
 		   void (*func)(void *), void *data)
 {
 	struct nouveau_fence_work *work;
 
-	if (fence_is_signaled(fence))
+	if (dma_fence_is_signaled(fence))
 		goto err;
 
 	work = kmalloc(sizeof(*work), GFP_KERNEL);
@@ -245,7 +245,7 @@ nouveau_fence_work(struct fence *fence,
 	work->func = func;
 	work->data = data;
 
-	if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
+	if (dma_fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0)
 		goto err_free;
 	return;
 
@@ -266,17 +266,17 @@ nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
 	fence->timeout  = jiffies + (15 * HZ);
 
 	if (priv->uevent)
-		fence_init(&fence->base, &nouveau_fence_ops_uevent,
-			   &fctx->lock, fctx->context, ++fctx->sequence);
+		dma_fence_init(&fence->base, &nouveau_fence_ops_uevent,
+			       &fctx->lock, fctx->context, ++fctx->sequence);
 	else
-		fence_init(&fence->base, &nouveau_fence_ops_legacy,
-			   &fctx->lock, fctx->context, ++fctx->sequence);
+		dma_fence_init(&fence->base, &nouveau_fence_ops_legacy,
+			       &fctx->lock, fctx->context, ++fctx->sequence);
 	kref_get(&fctx->fence_ref);
 
-	trace_fence_emit(&fence->base);
+	trace_dma_fence_emit(&fence->base);
 	ret = fctx->emit(fence);
 	if (!ret) {
-		fence_get(&fence->base);
+		dma_fence_get(&fence->base);
 		spin_lock_irq(&fctx->lock);
 
 		if (nouveau_fence_update(chan, fctx))
@@ -298,7 +298,7 @@ nouveau_fence_done(struct nouveau_fence *fence)
 		struct nouveau_channel *chan;
 		unsigned long flags;
 
-		if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
+		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
 			return true;
 
 		spin_lock_irqsave(&fctx->lock, flags);
@@ -307,11 +307,11 @@ nouveau_fence_done(struct nouveau_fence *fence)
 			nvif_notify_put(&fctx->notify);
 		spin_unlock_irqrestore(&fctx->lock, flags);
 	}
-	return fence_is_signaled(&fence->base);
+	return dma_fence_is_signaled(&fence->base);
 }
 
 static long
-nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait)
+nouveau_fence_wait_legacy(struct dma_fence *f, bool intr, long wait)
 {
 	struct nouveau_fence *fence = from_fence(f);
 	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
@@ -378,7 +378,7 @@ nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
 	if (!lazy)
 		return nouveau_fence_wait_busy(fence, intr);
 
-	ret = fence_wait_timeout(&fence->base, intr, 15 * HZ);
+	ret = dma_fence_wait_timeout(&fence->base, intr, 15 * HZ);
 	if (ret < 0)
 		return ret;
 	else if (!ret)
@@ -391,7 +391,7 @@ int
 nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool exclusive, bool intr)
 {
 	struct nouveau_fence_chan *fctx = chan->fence;
-	struct fence *fence;
+	struct dma_fence *fence;
 	struct reservation_object *resv = nvbo->bo.resv;
 	struct reservation_object_list *fobj;
 	struct nouveau_fence *f;
@@ -421,7 +421,7 @@ nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool e
 		}
 
 		if (must_wait)
-			ret = fence_wait(fence, intr);
+			ret = dma_fence_wait(fence, intr);
 
 		return ret;
 	}
@@ -446,7 +446,7 @@ nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool e
 		}
 
 		if (must_wait)
-			ret = fence_wait(fence, intr);
+			ret = dma_fence_wait(fence, intr);
 	}
 
 	return ret;
@@ -456,7 +456,7 @@ void
 nouveau_fence_unref(struct nouveau_fence **pfence)
 {
 	if (*pfence)
-		fence_put(&(*pfence)->base);
+		dma_fence_put(&(*pfence)->base);
 	*pfence = NULL;
 }
 
@@ -484,12 +484,12 @@ nouveau_fence_new(struct nouveau_channel *chan, bool sysmem,
 	return ret;
 }
 
-static const char *nouveau_fence_get_get_driver_name(struct fence *fence)
+static const char *nouveau_fence_get_get_driver_name(struct dma_fence *fence)
 {
 	return "nouveau";
 }
 
-static const char *nouveau_fence_get_timeline_name(struct fence *f)
+static const char *nouveau_fence_get_timeline_name(struct dma_fence *f)
 {
 	struct nouveau_fence *fence = from_fence(f);
 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
@@ -503,7 +503,7 @@ static const char *nouveau_fence_get_timeline_name(struct fence *f)
  * result. The drm node should still be there, so we can derive the index from
  * the fence context.
  */
-static bool nouveau_fence_is_signaled(struct fence *f)
+static bool nouveau_fence_is_signaled(struct dma_fence *f)
 {
 	struct nouveau_fence *fence = from_fence(f);
 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
@@ -519,7 +519,7 @@ static bool nouveau_fence_is_signaled(struct fence *f)
 	return ret;
 }
 
-static bool nouveau_fence_no_signaling(struct fence *f)
+static bool nouveau_fence_no_signaling(struct dma_fence *f)
 {
 	struct nouveau_fence *fence = from_fence(f);
 
@@ -530,30 +530,30 @@ static bool nouveau_fence_no_signaling(struct fence *f)
 	WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1);
 
 	/*
-	 * This needs uevents to work correctly, but fence_add_callback relies on
+	 * This needs uevents to work correctly, but dma_fence_add_callback relies on
 	 * being able to enable signaling. It will still get signaled eventually,
 	 * just not right away.
 	 */
 	if (nouveau_fence_is_signaled(f)) {
 		list_del(&fence->head);
 
-		fence_put(&fence->base);
+		dma_fence_put(&fence->base);
 		return false;
 	}
 
 	return true;
 }
 
-static void nouveau_fence_release(struct fence *f)
+static void nouveau_fence_release(struct dma_fence *f)
 {
 	struct nouveau_fence *fence = from_fence(f);
 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 
 	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
-	fence_free(&fence->base);
+	dma_fence_free(&fence->base);
 }
 
-static const struct fence_ops nouveau_fence_ops_legacy = {
+static const struct dma_fence_ops nouveau_fence_ops_legacy = {
 	.get_driver_name = nouveau_fence_get_get_driver_name,
 	.get_timeline_name = nouveau_fence_get_timeline_name,
 	.enable_signaling = nouveau_fence_no_signaling,
@@ -562,7 +562,7 @@ static const struct fence_ops nouveau_fence_ops_legacy = {
 	.release = nouveau_fence_release
 };
 
-static bool nouveau_fence_enable_signaling(struct fence *f)
+static bool nouveau_fence_enable_signaling(struct dma_fence *f)
 {
 	struct nouveau_fence *fence = from_fence(f);
 	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
@@ -573,18 +573,18 @@ static bool nouveau_fence_enable_signaling(struct fence *f)
 
 	ret = nouveau_fence_no_signaling(f);
 	if (ret)
-		set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags);
+		set_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags);
 	else if (!--fctx->notify_ref)
 		nvif_notify_put(&fctx->notify);
 
 	return ret;
 }
 
-static const struct fence_ops nouveau_fence_ops_uevent = {
+static const struct dma_fence_ops nouveau_fence_ops_uevent = {
 	.get_driver_name = nouveau_fence_get_get_driver_name,
 	.get_timeline_name = nouveau_fence_get_timeline_name,
 	.enable_signaling = nouveau_fence_enable_signaling,
 	.signaled = nouveau_fence_is_signaled,
-	.wait = fence_default_wait,
-	.release = NULL
+	.wait = dma_fence_default_wait,
+	.release = nouveau_fence_release
 };
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.h b/drivers/gpu/drm/nouveau/nouveau_fence.h
index 64c4ce7115ad..ccdce1b4eec4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.h
@@ -1,14 +1,14 @@
 #ifndef __NOUVEAU_FENCE_H__
 #define __NOUVEAU_FENCE_H__
 
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 #include <nvif/notify.h>
 
 struct nouveau_drm;
 struct nouveau_bo;
 
 struct nouveau_fence {
-	struct fence base;
+	struct dma_fence base;
 
 	struct list_head head;
 
@@ -24,7 +24,7 @@ void nouveau_fence_unref(struct nouveau_fence **);
 
 int  nouveau_fence_emit(struct nouveau_fence *, struct nouveau_channel *);
 bool nouveau_fence_done(struct nouveau_fence *);
-void nouveau_fence_work(struct fence *, void (*)(void *), void *);
+void nouveau_fence_work(struct dma_fence *, void (*)(void *), void *);
 int  nouveau_fence_wait(struct nouveau_fence *, bool lazy, bool intr);
 int  nouveau_fence_sync(struct nouveau_bo *, struct nouveau_channel *, bool exclusive, bool intr);
 
@@ -92,7 +92,6 @@ struct nv84_fence_chan {
 	struct nouveau_fence_chan base;
 	struct nvkm_vma vma;
 	struct nvkm_vma vma_gart;
-	struct nvkm_vma dispc_vma[4];
 };
 
 struct nv84_fence_priv {
@@ -102,7 +101,6 @@ struct nv84_fence_priv {
 	u32 *suspend;
 };
 
-u64  nv84_fence_crtc(struct nouveau_channel *, int);
 int  nv84_fence_context_new(struct nouveau_channel *);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index 72e2399bce39..201b52b750dd 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -119,7 +119,7 @@ nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nvkm_vma *vma)
 	const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM;
 	struct reservation_object *resv = nvbo->bo.resv;
 	struct reservation_object_list *fobj;
-	struct fence *fence = NULL;
+	struct dma_fence *fence = NULL;
 
 	fobj = reservation_object_get_list(resv);
 
@@ -369,7 +369,7 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
 {
 	struct nouveau_cli *cli = nouveau_cli(file_priv);
 	int trycnt = 0;
-	int ret, i;
+	int ret = -EINVAL, i;
 	struct nouveau_bo *res_bo = NULL;
 	LIST_HEAD(gart_list);
 	LIST_HEAD(vram_list);
@@ -861,6 +861,7 @@ nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
 	struct nouveau_bo *nvbo;
 	bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT);
 	bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE);
+	long lret;
 	int ret;
 
 	gem = drm_gem_object_lookup(file_priv, req->handle);
@@ -868,19 +869,15 @@ nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
 		return -ENOENT;
 	nvbo = nouveau_gem_object(gem);
 
-	if (no_wait)
-		ret = reservation_object_test_signaled_rcu(nvbo->bo.resv, write) ? 0 : -EBUSY;
-	else {
-		long lret;
+	lret = reservation_object_wait_timeout_rcu(nvbo->bo.resv, write, true,
+						   no_wait ? 0 : 30 * HZ);
+	if (!lret)
+		ret = -EBUSY;
+	else if (lret > 0)
+		ret = 0;
+	else
+		ret = lret;
 
-		lret = reservation_object_wait_timeout_rcu(nvbo->bo.resv, write, true, 30 * HZ);
-		if (!lret)
-			ret = -EBUSY;
-		else if (lret > 0)
-			ret = 0;
-		else
-			ret = lret;
-	}
 	nouveau_bo_sync_for_cpu(nvbo);
 	drm_gem_object_unreference_unlocked(gem);
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_led.c b/drivers/gpu/drm/nouveau/nouveau_led.c
new file mode 100644
index 000000000000..3e2f1b6cd4df
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_led.c
@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) 2016 Martin Peres
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+/*
+ * Authors:
+ *  Martin Peres <martin.peres@free.fr>
+ */
+
+#include <linux/leds.h>
+
+#include "nouveau_led.h"
+#include <nvkm/subdev/gpio.h>
+
+static enum led_brightness
+nouveau_led_get_brightness(struct led_classdev *led)
+{
+	struct drm_device *drm_dev = container_of(led, struct nouveau_led, led)->dev;
+	struct nouveau_drm *drm = nouveau_drm(drm_dev);
+	struct nvif_object *device = &drm->device.object;
+	u32 div, duty;
+
+	div =  nvif_rd32(device, 0x61c880) & 0x00ffffff;
+	duty = nvif_rd32(device, 0x61c884) & 0x00ffffff;
+
+	if (div > 0)
+		return duty * LED_FULL / div;
+	else
+		return 0;
+}
+
+static void
+nouveau_led_set_brightness(struct led_classdev *led, enum led_brightness value)
+{
+	struct drm_device *drm_dev = container_of(led, struct nouveau_led, led)->dev;
+	struct nouveau_drm *drm = nouveau_drm(drm_dev);
+	struct nvif_object *device = &drm->device.object;
+
+	u32 input_clk = 27e6; /* PDISPLAY.SOR[1].PWM is connected to the crystal */
+	u32 freq = 100; /* this is what nvidia uses and it should be good-enough */
+	u32 div, duty;
+
+	div = input_clk / freq;
+	duty = value * div / LED_FULL;
+
+	/* for now, this is safe to directly poke those registers because:
+	 *  - A: nvidia never puts the logo led to any other PWM controler
+	 *       than PDISPLAY.SOR[1].PWM.
+	 *  - B: nouveau does not touch these registers anywhere else
+	 */
+	nvif_wr32(device, 0x61c880, div);
+	nvif_wr32(device, 0x61c884, 0xc0000000 | duty);
+}
+
+
+int
+nouveau_led_init(struct drm_device *dev)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvkm_gpio *gpio = nvxx_gpio(&drm->device);
+	struct dcb_gpio_func logo_led;
+	int ret;
+
+	if (!gpio)
+		return 0;
+
+	/* check that there is a GPIO controlling the logo LED */
+	if (nvkm_gpio_find(gpio, 0, DCB_GPIO_LOGO_LED_PWM, 0xff, &logo_led))
+		return 0;
+
+	drm->led = kzalloc(sizeof(*drm->led), GFP_KERNEL);
+	if (!drm->led)
+		return -ENOMEM;
+	drm->led->dev = dev;
+
+	drm->led->led.name = "nvidia-logo";
+	drm->led->led.max_brightness = 255;
+	drm->led->led.brightness_get = nouveau_led_get_brightness;
+	drm->led->led.brightness_set = nouveau_led_set_brightness;
+
+	ret = led_classdev_register(dev->dev, &drm->led->led);
+	if (ret) {
+		kfree(drm->led);
+		return ret;
+	}
+
+	return 0;
+}
+
+void
+nouveau_led_suspend(struct drm_device *dev)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
+	if (drm->led)
+		led_classdev_suspend(&drm->led->led);
+}
+
+void
+nouveau_led_resume(struct drm_device *dev)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
+	if (drm->led)
+		led_classdev_resume(&drm->led->led);
+}
+
+void
+nouveau_led_fini(struct drm_device *dev)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
+	if (drm->led) {
+		led_classdev_unregister(&drm->led->led);
+		kfree(drm->led);
+		drm->led = NULL;
+	}
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_led.h b/drivers/gpu/drm/nouveau/nouveau_led.h
new file mode 100644
index 000000000000..187ecdb82002
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_led.h
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2015 Martin Peres
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres <martin.peres@free.fr>
+ */
+
+#ifndef __NOUVEAU_LED_H__
+#define __NOUVEAU_LED_H__
+
+#include "nouveau_drv.h"
+
+struct led_classdev;
+
+struct nouveau_led {
+	struct drm_device *dev;
+
+	struct led_classdev led;
+};
+
+static inline struct nouveau_led *
+nouveau_led(struct drm_device *dev)
+{
+	return nouveau_drm(dev)->led;
+}
+
+/* nouveau_led.c */
+#if IS_ENABLED(CONFIG_LEDS_CLASS)
+int  nouveau_led_init(struct drm_device *dev);
+void nouveau_led_suspend(struct drm_device *dev);
+void nouveau_led_resume(struct drm_device *dev);
+void nouveau_led_fini(struct drm_device *dev);
+#else
+static inline int  nouveau_led_init(struct drm_device *dev) { return 0; };
+static inline void nouveau_led_suspend(struct drm_device *dev) { };
+static inline void nouveau_led_resume(struct drm_device *dev) { };
+static inline void nouveau_led_fini(struct drm_device *dev) { };
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nv04_fbcon.c b/drivers/gpu/drm/nouveau/nv04_fbcon.c
index da8fd5ff9d0f..6a2b187e3c3b 100644
--- a/drivers/gpu/drm/nouveau/nv04_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv04_fbcon.c
@@ -30,7 +30,7 @@ int
 nv04_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev);
 	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
@@ -50,7 +50,7 @@ int
 nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev);
 	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
@@ -77,7 +77,7 @@ int
 nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev);
 	struct nouveau_channel *chan = drm->channel;
 	uint32_t fg;
 	uint32_t bg;
@@ -133,7 +133,7 @@ int
 nv04_fbcon_accel_init(struct fb_info *info)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
+	struct drm_device *dev = nfbdev->helper.dev;
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_channel *chan = drm->channel;
 	struct nvif_device *device = &drm->device;
diff --git a/drivers/gpu/drm/nouveau/nv04_fence.c b/drivers/gpu/drm/nouveau/nv04_fence.c
index 1915b7b82a59..fa8f2375c398 100644
--- a/drivers/gpu/drm/nouveau/nv04_fence.c
+++ b/drivers/gpu/drm/nouveau/nv04_fence.c
@@ -110,6 +110,6 @@ nv04_fence_create(struct nouveau_drm *drm)
 	priv->base.context_new = nv04_fence_context_new;
 	priv->base.context_del = nv04_fence_context_del;
 	priv->base.contexts = 15;
-	priv->base.context_base = fence_context_alloc(priv->base.contexts);
+	priv->base.context_base = dma_fence_context_alloc(priv->base.contexts);
 	return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nv10_fence.c b/drivers/gpu/drm/nouveau/nv10_fence.c
index 4e3de34ff6f4..2998bde29211 100644
--- a/drivers/gpu/drm/nouveau/nv10_fence.c
+++ b/drivers/gpu/drm/nouveau/nv10_fence.c
@@ -57,16 +57,13 @@ void
 nv10_fence_context_del(struct nouveau_channel *chan)
 {
 	struct nv10_fence_chan *fctx = chan->fence;
-	int i;
 	nouveau_fence_context_del(&fctx->base);
-	for (i = 0; i < ARRAY_SIZE(fctx->head); i++)
-		nvif_object_fini(&fctx->head[i]);
 	nvif_object_fini(&fctx->sema);
 	chan->fence = NULL;
 	nouveau_fence_context_free(&fctx->base);
 }
 
-int
+static int
 nv10_fence_context_new(struct nouveau_channel *chan)
 {
 	struct nv10_fence_chan *fctx;
@@ -107,7 +104,7 @@ nv10_fence_create(struct nouveau_drm *drm)
 	priv->base.context_new = nv10_fence_context_new;
 	priv->base.context_del = nv10_fence_context_del;
 	priv->base.contexts = 31;
-	priv->base.context_base = fence_context_alloc(priv->base.contexts);
+	priv->base.context_base = dma_fence_context_alloc(priv->base.contexts);
 	spin_lock_init(&priv->lock);
 	return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nv10_fence.h b/drivers/gpu/drm/nouveau/nv10_fence.h
index a87259f3983a..b7a508585304 100644
--- a/drivers/gpu/drm/nouveau/nv10_fence.h
+++ b/drivers/gpu/drm/nouveau/nv10_fence.h
@@ -7,7 +7,6 @@
 struct nv10_fence_chan {
 	struct nouveau_fence_chan base;
 	struct nvif_object sema;
-	struct nvif_object head[4];
 };
 
 struct nv10_fence_priv {
diff --git a/drivers/gpu/drm/nouveau/nv17_fence.c b/drivers/gpu/drm/nouveau/nv17_fence.c
index 7d5e562a55c5..79bc01111351 100644
--- a/drivers/gpu/drm/nouveau/nv17_fence.c
+++ b/drivers/gpu/drm/nouveau/nv17_fence.c
@@ -126,7 +126,7 @@ nv17_fence_create(struct nouveau_drm *drm)
 	priv->base.context_new = nv17_fence_context_new;
 	priv->base.context_del = nv10_fence_context_del;
 	priv->base.contexts = 31;
-	priv->base.context_base = fence_context_alloc(priv->base.contexts);
+	priv->base.context_base = dma_fence_context_alloc(priv->base.contexts);
 	spin_lock_init(&priv->lock);
 
 	ret = nouveau_bo_new(drm->dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index 7d0edcbcfca7..2c2c64507661 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -25,10 +25,12 @@
 #include <linux/dma-mapping.h>
 
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_dp_helper.h>
 #include <drm/drm_fb_helper.h>
+#include <drm/drm_plane_helper.h>
 
 #include <nvif/class.h>
 #include <nvif/cl0002.h>
@@ -38,6 +40,7 @@
 #include <nvif/cl507c.h>
 #include <nvif/cl507d.h>
 #include <nvif/cl507e.h>
+#include <nvif/event.h>
 
 #include "nouveau_drv.h"
 #include "nouveau_dma.h"
@@ -46,6 +49,7 @@
 #include "nouveau_encoder.h"
 #include "nouveau_crtc.h"
 #include "nouveau_fence.h"
+#include "nouveau_fbcon.h"
 #include "nv50_display.h"
 
 #define EVO_DMA_NR 9
@@ -61,6 +65,227 @@
 #define EVO_MAST_NTFY     EVO_SYNC(      0, 0x00)
 #define EVO_FLIP_SEM0(c)  EVO_SYNC((c) + 1, 0x00)
 #define EVO_FLIP_SEM1(c)  EVO_SYNC((c) + 1, 0x10)
+#define EVO_FLIP_NTFY0(c) EVO_SYNC((c) + 1, 0x20)
+#define EVO_FLIP_NTFY1(c) EVO_SYNC((c) + 1, 0x30)
+
+/******************************************************************************
+ * Atomic state
+ *****************************************************************************/
+#define nv50_atom(p) container_of((p), struct nv50_atom, state)
+
+struct nv50_atom {
+	struct drm_atomic_state state;
+
+	struct list_head outp;
+	bool lock_core;
+	bool flush_disable;
+};
+
+struct nv50_outp_atom {
+	struct list_head head;
+
+	struct drm_encoder *encoder;
+	bool flush_disable;
+
+	union {
+		struct {
+			bool ctrl:1;
+		};
+		u8 mask;
+	} clr;
+
+	union {
+		struct {
+			bool ctrl:1;
+		};
+		u8 mask;
+	} set;
+};
+
+#define nv50_head_atom(p) container_of((p), struct nv50_head_atom, state)
+
+struct nv50_head_atom {
+	struct drm_crtc_state state;
+
+	struct {
+		u16 iW;
+		u16 iH;
+		u16 oW;
+		u16 oH;
+	} view;
+
+	struct nv50_head_mode {
+		bool interlace;
+		u32 clock;
+		struct {
+			u16 active;
+			u16 synce;
+			u16 blanke;
+			u16 blanks;
+		} h;
+		struct {
+			u32 active;
+			u16 synce;
+			u16 blanke;
+			u16 blanks;
+			u16 blank2s;
+			u16 blank2e;
+			u16 blankus;
+		} v;
+	} mode;
+
+	struct {
+		u32 handle;
+		u64 offset:40;
+	} lut;
+
+	struct {
+		bool visible;
+		u32 handle;
+		u64 offset:40;
+		u8  format;
+		u8  kind:7;
+		u8  layout:1;
+		u8  block:4;
+		u32 pitch:20;
+		u16 x;
+		u16 y;
+		u16 w;
+		u16 h;
+	} core;
+
+	struct {
+		bool visible;
+		u32 handle;
+		u64 offset:40;
+		u8  layout:1;
+		u8  format:1;
+	} curs;
+
+	struct {
+		u8  depth;
+		u8  cpp;
+		u16 x;
+		u16 y;
+		u16 w;
+		u16 h;
+	} base;
+
+	struct {
+		u8 cpp;
+	} ovly;
+
+	struct {
+		bool enable:1;
+		u8 bits:2;
+		u8 mode:4;
+	} dither;
+
+	struct {
+		struct {
+			u16 cos:12;
+			u16 sin:12;
+		} sat;
+	} procamp;
+
+	union {
+		struct {
+			bool core:1;
+			bool curs:1;
+		};
+		u8 mask;
+	} clr;
+
+	union {
+		struct {
+			bool core:1;
+			bool curs:1;
+			bool view:1;
+			bool mode:1;
+			bool base:1;
+			bool ovly:1;
+			bool dither:1;
+			bool procamp:1;
+		};
+		u16 mask;
+	} set;
+};
+
+static inline struct nv50_head_atom *
+nv50_head_atom_get(struct drm_atomic_state *state, struct drm_crtc *crtc)
+{
+	struct drm_crtc_state *statec = drm_atomic_get_crtc_state(state, crtc);
+	if (IS_ERR(statec))
+		return (void *)statec;
+	return nv50_head_atom(statec);
+}
+
+#define nv50_wndw_atom(p) container_of((p), struct nv50_wndw_atom, state)
+
+struct nv50_wndw_atom {
+	struct drm_plane_state state;
+	u8 interval;
+
+	struct drm_rect clip;
+
+	struct {
+		u32  handle;
+		u16  offset:12;
+		bool awaken:1;
+	} ntfy;
+
+	struct {
+		u32 handle;
+		u16 offset:12;
+		u32 acquire;
+		u32 release;
+	} sema;
+
+	struct {
+		u8 enable:2;
+	} lut;
+
+	struct {
+		u8  mode:2;
+		u8  interval:4;
+
+		u8  format;
+		u8  kind:7;
+		u8  layout:1;
+		u8  block:4;
+		u32 pitch:20;
+		u16 w;
+		u16 h;
+
+		u32 handle;
+		u64 offset;
+	} image;
+
+	struct {
+		u16 x;
+		u16 y;
+	} point;
+
+	union {
+		struct {
+			bool ntfy:1;
+			bool sema:1;
+			bool image:1;
+		};
+		u8 mask;
+	} clr;
+
+	union {
+		struct {
+			bool ntfy:1;
+			bool sema:1;
+			bool image:1;
+			bool lut:1;
+			bool point:1;
+		};
+		u8 mask;
+	} set;
+};
 
 /******************************************************************************
  * EVO channel
@@ -133,34 +358,6 @@ nv50_pioc_create(struct nvif_device *device, struct nvif_object *disp,
 }
 
 /******************************************************************************
- * Cursor Immediate
- *****************************************************************************/
-
-struct nv50_curs {
-	struct nv50_pioc base;
-};
-
-static int
-nv50_curs_create(struct nvif_device *device, struct nvif_object *disp,
-		 int head, struct nv50_curs *curs)
-{
-	struct nv50_disp_cursor_v0 args = {
-		.head = head,
-	};
-	static const s32 oclass[] = {
-		GK104_DISP_CURSOR,
-		GF110_DISP_CURSOR,
-		GT214_DISP_CURSOR,
-		G82_DISP_CURSOR,
-		NV50_DISP_CURSOR,
-		0
-	};
-
-	return nv50_pioc_create(device, disp, oclass, head, &args, sizeof(args),
-				&curs->base);
-}
-
-/******************************************************************************
  * Overlay Immediate
  *****************************************************************************/
 
@@ -192,6 +389,11 @@ nv50_oimm_create(struct nvif_device *device, struct nvif_object *disp,
  * DMA EVO channel
  *****************************************************************************/
 
+struct nv50_dmac_ctxdma {
+	struct list_head head;
+	struct nvif_object object;
+};
+
 struct nv50_dmac {
 	struct nv50_chan base;
 	dma_addr_t handle;
@@ -199,6 +401,7 @@ struct nv50_dmac {
 
 	struct nvif_object sync;
 	struct nvif_object vram;
+	struct list_head ctxdma;
 
 	/* Protects against concurrent pushbuf access to this channel, lock is
 	 * grabbed by evo_wait (if the pushbuf reservation is successful) and
@@ -207,9 +410,82 @@ struct nv50_dmac {
 };
 
 static void
+nv50_dmac_ctxdma_del(struct nv50_dmac_ctxdma *ctxdma)
+{
+	nvif_object_fini(&ctxdma->object);
+	list_del(&ctxdma->head);
+	kfree(ctxdma);
+}
+
+static struct nv50_dmac_ctxdma *
+nv50_dmac_ctxdma_new(struct nv50_dmac *dmac, struct nouveau_framebuffer *fb)
+{
+	struct nouveau_drm *drm = nouveau_drm(fb->base.dev);
+	struct nv50_dmac_ctxdma *ctxdma;
+	const u8    kind = (fb->nvbo->tile_flags & 0x0000ff00) >> 8;
+	const u32 handle = 0xfb000000 | kind;
+	struct {
+		struct nv_dma_v0 base;
+		union {
+			struct nv50_dma_v0 nv50;
+			struct gf100_dma_v0 gf100;
+			struct gf119_dma_v0 gf119;
+		};
+	} args = {};
+	u32 argc = sizeof(args.base);
+	int ret;
+
+	list_for_each_entry(ctxdma, &dmac->ctxdma, head) {
+		if (ctxdma->object.handle == handle)
+			return ctxdma;
+	}
+
+	if (!(ctxdma = kzalloc(sizeof(*ctxdma), GFP_KERNEL)))
+		return ERR_PTR(-ENOMEM);
+	list_add(&ctxdma->head, &dmac->ctxdma);
+
+	args.base.target = NV_DMA_V0_TARGET_VRAM;
+	args.base.access = NV_DMA_V0_ACCESS_RDWR;
+	args.base.start  = 0;
+	args.base.limit  = drm->device.info.ram_user - 1;
+
+	if (drm->device.info.chipset < 0x80) {
+		args.nv50.part = NV50_DMA_V0_PART_256;
+		argc += sizeof(args.nv50);
+	} else
+	if (drm->device.info.chipset < 0xc0) {
+		args.nv50.part = NV50_DMA_V0_PART_256;
+		args.nv50.kind = kind;
+		argc += sizeof(args.nv50);
+	} else
+	if (drm->device.info.chipset < 0xd0) {
+		args.gf100.kind = kind;
+		argc += sizeof(args.gf100);
+	} else {
+		args.gf119.page = GF119_DMA_V0_PAGE_LP;
+		args.gf119.kind = kind;
+		argc += sizeof(args.gf119);
+	}
+
+	ret = nvif_object_init(&dmac->base.user, handle, NV_DMA_IN_MEMORY,
+			       &args, argc, &ctxdma->object);
+	if (ret) {
+		nv50_dmac_ctxdma_del(ctxdma);
+		return ERR_PTR(ret);
+	}
+
+	return ctxdma;
+}
+
+static void
 nv50_dmac_destroy(struct nv50_dmac *dmac, struct nvif_object *disp)
 {
 	struct nvif_device *device = dmac->base.device;
+	struct nv50_dmac_ctxdma *ctxdma, *ctxtmp;
+
+	list_for_each_entry_safe(ctxdma, ctxtmp, &dmac->ctxdma, head) {
+		nv50_dmac_ctxdma_del(ctxdma);
+	}
 
 	nvif_object_fini(&dmac->vram);
 	nvif_object_fini(&dmac->sync);
@@ -278,6 +554,7 @@ nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp,
 	if (ret)
 		return ret;
 
+	INIT_LIST_HEAD(&dmac->ctxdma);
 	return ret;
 }
 
@@ -297,7 +574,7 @@ nv50_core_create(struct nvif_device *device, struct nvif_object *disp,
 		.pushbuf = 0xb0007d00,
 	};
 	static const s32 oclass[] = {
-		GP104_DISP_CORE_CHANNEL_DMA,
+		GP102_DISP_CORE_CHANNEL_DMA,
 		GP100_DISP_CORE_CHANNEL_DMA,
 		GM200_DISP_CORE_CHANNEL_DMA,
 		GM107_DISP_CORE_CHANNEL_DMA,
@@ -381,34 +658,23 @@ nv50_ovly_create(struct nvif_device *device, struct nvif_object *disp,
 
 struct nv50_head {
 	struct nouveau_crtc base;
-	struct nouveau_bo *image;
-	struct nv50_curs curs;
-	struct nv50_sync sync;
 	struct nv50_ovly ovly;
 	struct nv50_oimm oimm;
 };
 
 #define nv50_head(c) ((struct nv50_head *)nouveau_crtc(c))
-#define nv50_curs(c) (&nv50_head(c)->curs)
-#define nv50_sync(c) (&nv50_head(c)->sync)
 #define nv50_ovly(c) (&nv50_head(c)->ovly)
 #define nv50_oimm(c) (&nv50_head(c)->oimm)
 #define nv50_chan(c) (&(c)->base.base)
 #define nv50_vers(c) nv50_chan(c)->user.oclass
 
-struct nv50_fbdma {
-	struct list_head head;
-	struct nvif_object core;
-	struct nvif_object base[4];
-};
-
 struct nv50_disp {
 	struct nvif_object *disp;
 	struct nv50_mast mast;
 
-	struct list_head fbdma;
-
 	struct nouveau_bo *sync;
+
+	struct mutex mutex;
 };
 
 static struct nv50_disp *
@@ -419,12 +685,6 @@ nv50_disp(struct drm_device *dev)
 
 #define nv50_mast(d) (&nv50_disp(d)->mast)
 
-static struct drm_crtc *
-nv50_display_crtc_get(struct drm_encoder *encoder)
-{
-	return nouveau_encoder(encoder)->crtc;
-}
-
 /******************************************************************************
  * EVO channel helpers
  *****************************************************************************/
@@ -463,812 +723,1465 @@ evo_kick(u32 *push, void *evoc)
 	mutex_unlock(&dmac->lock);
 }
 
-#if 1
-#define evo_mthd(p,m,s) *((p)++) = (((s) << 18) | (m))
-#define evo_data(p,d)   *((p)++) = (d)
-#else
 #define evo_mthd(p,m,s) do {                                                   \
 	const u32 _m = (m), _s = (s);                                          \
-	printk(KERN_ERR "%04x %d %s\n", _m, _s, __func__);                     \
+	if (drm_debug & DRM_UT_KMS)                                            \
+		printk(KERN_ERR "%04x %d %s\n", _m, _s, __func__);             \
 	*((p)++) = ((_s << 18) | _m);                                          \
 } while(0)
+
 #define evo_data(p,d) do {                                                     \
 	const u32 _d = (d);                                                    \
-	printk(KERN_ERR "\t%08x\n", _d);                                       \
+	if (drm_debug & DRM_UT_KMS)                                            \
+		printk(KERN_ERR "\t%08x\n", _d);                               \
 	*((p)++) = _d;                                                         \
 } while(0)
-#endif
 
-static bool
-evo_sync_wait(void *data)
+/******************************************************************************
+ * Plane
+ *****************************************************************************/
+#define nv50_wndw(p) container_of((p), struct nv50_wndw, plane)
+
+struct nv50_wndw {
+	const struct nv50_wndw_func *func;
+	struct nv50_dmac *dmac;
+
+	struct drm_plane plane;
+
+	struct nvif_notify notify;
+	u16 ntfy;
+	u16 sema;
+	u32 data;
+};
+
+struct nv50_wndw_func {
+	void *(*dtor)(struct nv50_wndw *);
+	int (*acquire)(struct nv50_wndw *, struct nv50_wndw_atom *asyw,
+		       struct nv50_head_atom *asyh);
+	void (*release)(struct nv50_wndw *, struct nv50_wndw_atom *asyw,
+			struct nv50_head_atom *asyh);
+	void (*prepare)(struct nv50_wndw *, struct nv50_head_atom *asyh,
+			struct nv50_wndw_atom *asyw);
+
+	void (*sema_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
+	void (*sema_clr)(struct nv50_wndw *);
+	void (*ntfy_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
+	void (*ntfy_clr)(struct nv50_wndw *);
+	int (*ntfy_wait_begun)(struct nv50_wndw *, struct nv50_wndw_atom *);
+	void (*image_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
+	void (*image_clr)(struct nv50_wndw *);
+	void (*lut)(struct nv50_wndw *, struct nv50_wndw_atom *);
+	void (*point)(struct nv50_wndw *, struct nv50_wndw_atom *);
+
+	u32 (*update)(struct nv50_wndw *, u32 interlock);
+};
+
+static int
+nv50_wndw_wait_armed(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
 {
-	if (nouveau_bo_rd32(data, EVO_MAST_NTFY) != 0x00000000)
-		return true;
-	usleep_range(1, 2);
-	return false;
+	if (asyw->set.ntfy)
+		return wndw->func->ntfy_wait_begun(wndw, asyw);
+	return 0;
 }
 
-static int
-evo_sync(struct drm_device *dev)
+static u32
+nv50_wndw_flush_clr(struct nv50_wndw *wndw, u32 interlock, bool flush,
+		    struct nv50_wndw_atom *asyw)
 {
-	struct nvif_device *device = &nouveau_drm(dev)->device;
-	struct nv50_disp *disp = nv50_disp(dev);
-	struct nv50_mast *mast = nv50_mast(dev);
-	u32 *push = evo_wait(mast, 8);
-	if (push) {
-		nouveau_bo_wr32(disp->sync, EVO_MAST_NTFY, 0x00000000);
-		evo_mthd(push, 0x0084, 1);
-		evo_data(push, 0x80000000 | EVO_MAST_NTFY);
-		evo_mthd(push, 0x0080, 2);
-		evo_data(push, 0x00000000);
-		evo_data(push, 0x00000000);
-		evo_kick(push, mast);
-		if (nvif_msec(device, 2000,
-			if (evo_sync_wait(disp->sync))
-				break;
-		) >= 0)
-			return 0;
+	if (asyw->clr.sema && (!asyw->set.sema || flush))
+		wndw->func->sema_clr(wndw);
+	if (asyw->clr.ntfy && (!asyw->set.ntfy || flush))
+		wndw->func->ntfy_clr(wndw);
+	if (asyw->clr.image && (!asyw->set.image || flush))
+		wndw->func->image_clr(wndw);
+
+	return flush ? wndw->func->update(wndw, interlock) : 0;
+}
+
+static u32
+nv50_wndw_flush_set(struct nv50_wndw *wndw, u32 interlock,
+		    struct nv50_wndw_atom *asyw)
+{
+	if (interlock) {
+		asyw->image.mode = 0;
+		asyw->image.interval = 1;
 	}
 
-	return -EBUSY;
+	if (asyw->set.sema ) wndw->func->sema_set (wndw, asyw);
+	if (asyw->set.ntfy ) wndw->func->ntfy_set (wndw, asyw);
+	if (asyw->set.image) wndw->func->image_set(wndw, asyw);
+	if (asyw->set.lut  ) wndw->func->lut      (wndw, asyw);
+	if (asyw->set.point) wndw->func->point    (wndw, asyw);
+
+	return wndw->func->update(wndw, interlock);
 }
 
-/******************************************************************************
- * Page flipping channel
- *****************************************************************************/
-struct nouveau_bo *
-nv50_display_crtc_sema(struct drm_device *dev, int crtc)
+static void
+nv50_wndw_atomic_check_release(struct nv50_wndw *wndw,
+			       struct nv50_wndw_atom *asyw,
+			       struct nv50_head_atom *asyh)
 {
-	return nv50_disp(dev)->sync;
+	struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev);
+	NV_ATOMIC(drm, "%s release\n", wndw->plane.name);
+	wndw->func->release(wndw, asyw, asyh);
+	asyw->ntfy.handle = 0;
+	asyw->sema.handle = 0;
 }
 
-struct nv50_display_flip {
-	struct nv50_disp *disp;
-	struct nv50_sync *chan;
-};
-
-static bool
-nv50_display_flip_wait(void *data)
+static int
+nv50_wndw_atomic_check_acquire(struct nv50_wndw *wndw,
+			       struct nv50_wndw_atom *asyw,
+			       struct nv50_head_atom *asyh)
 {
-	struct nv50_display_flip *flip = data;
-	if (nouveau_bo_rd32(flip->disp->sync, flip->chan->addr / 4) ==
-					      flip->chan->data)
-		return true;
-	usleep_range(1, 2);
-	return false;
+	struct nouveau_framebuffer *fb = nouveau_framebuffer(asyw->state.fb);
+	struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev);
+	int ret;
+
+	NV_ATOMIC(drm, "%s acquire\n", wndw->plane.name);
+	asyw->clip.x1 = 0;
+	asyw->clip.y1 = 0;
+	asyw->clip.x2 = asyh->state.mode.hdisplay;
+	asyw->clip.y2 = asyh->state.mode.vdisplay;
+
+	asyw->image.w = fb->base.width;
+	asyw->image.h = fb->base.height;
+	asyw->image.kind = (fb->nvbo->tile_flags & 0x0000ff00) >> 8;
+	if (asyw->image.kind) {
+		asyw->image.layout = 0;
+		if (drm->device.info.chipset >= 0xc0)
+			asyw->image.block = fb->nvbo->tile_mode >> 4;
+		else
+			asyw->image.block = fb->nvbo->tile_mode;
+		asyw->image.pitch = (fb->base.pitches[0] / 4) << 4;
+	} else {
+		asyw->image.layout = 1;
+		asyw->image.block  = 0;
+		asyw->image.pitch  = fb->base.pitches[0];
+	}
+
+	ret = wndw->func->acquire(wndw, asyw, asyh);
+	if (ret)
+		return ret;
+
+	if (asyw->set.image) {
+		if (!(asyw->image.mode = asyw->interval ? 0 : 1))
+			asyw->image.interval = asyw->interval;
+		else
+			asyw->image.interval = 0;
+	}
+
+	return 0;
 }
 
-void
-nv50_display_flip_stop(struct drm_crtc *crtc)
+static int
+nv50_wndw_atomic_check(struct drm_plane *plane, struct drm_plane_state *state)
 {
-	struct nvif_device *device = &nouveau_drm(crtc->dev)->device;
-	struct nv50_display_flip flip = {
-		.disp = nv50_disp(crtc->dev),
-		.chan = nv50_sync(crtc),
-	};
-	u32 *push;
+	struct nouveau_drm *drm = nouveau_drm(plane->dev);
+	struct nv50_wndw *wndw = nv50_wndw(plane);
+	struct nv50_wndw_atom *armw = nv50_wndw_atom(wndw->plane.state);
+	struct nv50_wndw_atom *asyw = nv50_wndw_atom(state);
+	struct nv50_head_atom *harm = NULL, *asyh = NULL;
+	bool varm = false, asyv = false, asym = false;
+	int ret;
 
-	push = evo_wait(flip.chan, 8);
-	if (push) {
-		evo_mthd(push, 0x0084, 1);
-		evo_data(push, 0x00000000);
-		evo_mthd(push, 0x0094, 1);
-		evo_data(push, 0x00000000);
-		evo_mthd(push, 0x00c0, 1);
-		evo_data(push, 0x00000000);
-		evo_mthd(push, 0x0080, 1);
-		evo_data(push, 0x00000000);
-		evo_kick(push, flip.chan);
+	NV_ATOMIC(drm, "%s atomic_check\n", plane->name);
+	if (asyw->state.crtc) {
+		asyh = nv50_head_atom_get(asyw->state.state, asyw->state.crtc);
+		if (IS_ERR(asyh))
+			return PTR_ERR(asyh);
+		asym = drm_atomic_crtc_needs_modeset(&asyh->state);
+		asyv = asyh->state.active;
 	}
 
-	nvif_msec(device, 2000,
-		if (nv50_display_flip_wait(&flip))
-			break;
-	);
+	if (armw->state.crtc) {
+		harm = nv50_head_atom_get(asyw->state.state, armw->state.crtc);
+		if (IS_ERR(harm))
+			return PTR_ERR(harm);
+		varm = harm->state.crtc->state->active;
+	}
+
+	if (asyv) {
+		asyw->point.x = asyw->state.crtc_x;
+		asyw->point.y = asyw->state.crtc_y;
+		if (memcmp(&armw->point, &asyw->point, sizeof(asyw->point)))
+			asyw->set.point = true;
+
+		if (!varm || asym || armw->state.fb != asyw->state.fb) {
+			ret = nv50_wndw_atomic_check_acquire(wndw, asyw, asyh);
+			if (ret)
+				return ret;
+		}
+	} else
+	if (varm) {
+		nv50_wndw_atomic_check_release(wndw, asyw, harm);
+	} else {
+		return 0;
+	}
+
+	if (!asyv || asym) {
+		asyw->clr.ntfy = armw->ntfy.handle != 0;
+		asyw->clr.sema = armw->sema.handle != 0;
+		if (wndw->func->image_clr)
+			asyw->clr.image = armw->image.handle != 0;
+		asyw->set.lut = wndw->func->lut && asyv;
+	}
+
+	return 0;
 }
 
-int
-nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
-		       struct nouveau_channel *chan, u32 swap_interval)
+static void
+nv50_wndw_cleanup_fb(struct drm_plane *plane, struct drm_plane_state *old_state)
 {
-	struct nouveau_framebuffer *nv_fb = nouveau_framebuffer(fb);
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct nv50_head *head = nv50_head(crtc);
-	struct nv50_sync *sync = nv50_sync(crtc);
-	u32 *push;
-	int ret;
+	struct nouveau_framebuffer *fb = nouveau_framebuffer(old_state->fb);
+	struct nouveau_drm *drm = nouveau_drm(plane->dev);
 
-	if (crtc->primary->fb->width != fb->width ||
-	    crtc->primary->fb->height != fb->height)
-		return -EINVAL;
+	NV_ATOMIC(drm, "%s cleanup: %p\n", plane->name, old_state->fb);
+	if (!old_state->fb)
+		return;
+
+	nouveau_bo_unpin(fb->nvbo);
+}
 
-	swap_interval <<= 4;
-	if (swap_interval == 0)
-		swap_interval |= 0x100;
-	if (chan == NULL)
-		evo_sync(crtc->dev);
+static int
+nv50_wndw_prepare_fb(struct drm_plane *plane, struct drm_plane_state *state)
+{
+	struct nouveau_framebuffer *fb = nouveau_framebuffer(state->fb);
+	struct nouveau_drm *drm = nouveau_drm(plane->dev);
+	struct nv50_wndw *wndw = nv50_wndw(plane);
+	struct nv50_wndw_atom *asyw = nv50_wndw_atom(state);
+	struct nv50_head_atom *asyh;
+	struct nv50_dmac_ctxdma *ctxdma;
+	int ret;
 
-	push = evo_wait(sync, 128);
-	if (unlikely(push == NULL))
-		return -EBUSY;
+	NV_ATOMIC(drm, "%s prepare: %p\n", plane->name, state->fb);
+	if (!asyw->state.fb)
+		return 0;
 
-	if (chan && chan->user.oclass < G82_CHANNEL_GPFIFO) {
-		ret = RING_SPACE(chan, 8);
-		if (ret)
-			return ret;
+	ret = nouveau_bo_pin(fb->nvbo, TTM_PL_FLAG_VRAM, true);
+	if (ret)
+		return ret;
 
-		BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 2);
-		OUT_RING  (chan, NvEvoSema0 + nv_crtc->index);
-		OUT_RING  (chan, sync->addr ^ 0x10);
-		BEGIN_NV04(chan, 0, NV11_SUBCHAN_SEMAPHORE_RELEASE, 1);
-		OUT_RING  (chan, sync->data + 1);
-		BEGIN_NV04(chan, 0, NV11_SUBCHAN_SEMAPHORE_OFFSET, 2);
-		OUT_RING  (chan, sync->addr);
-		OUT_RING  (chan, sync->data);
-	} else
-	if (chan && chan->user.oclass < FERMI_CHANNEL_GPFIFO) {
-		u64 addr = nv84_fence_crtc(chan, nv_crtc->index) + sync->addr;
-		ret = RING_SPACE(chan, 12);
-		if (ret)
-			return ret;
+	ctxdma = nv50_dmac_ctxdma_new(wndw->dmac, fb);
+	if (IS_ERR(ctxdma)) {
+		nouveau_bo_unpin(fb->nvbo);
+		return PTR_ERR(ctxdma);
+	}
 
-		BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1);
-		OUT_RING  (chan, chan->vram.handle);
-		BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
-		OUT_RING  (chan, upper_32_bits(addr ^ 0x10));
-		OUT_RING  (chan, lower_32_bits(addr ^ 0x10));
-		OUT_RING  (chan, sync->data + 1);
-		OUT_RING  (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_WRITE_LONG);
-		BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
-		OUT_RING  (chan, upper_32_bits(addr));
-		OUT_RING  (chan, lower_32_bits(addr));
-		OUT_RING  (chan, sync->data);
-		OUT_RING  (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL);
-	} else
-	if (chan) {
-		u64 addr = nv84_fence_crtc(chan, nv_crtc->index) + sync->addr;
-		ret = RING_SPACE(chan, 10);
-		if (ret)
-			return ret;
+	asyw->state.fence = reservation_object_get_excl_rcu(fb->nvbo->bo.resv);
+	asyw->image.handle = ctxdma->object.handle;
+	asyw->image.offset = fb->nvbo->bo.offset;
 
-		BEGIN_NVC0(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
-		OUT_RING  (chan, upper_32_bits(addr ^ 0x10));
-		OUT_RING  (chan, lower_32_bits(addr ^ 0x10));
-		OUT_RING  (chan, sync->data + 1);
-		OUT_RING  (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_WRITE_LONG |
-				 NVC0_SUBCHAN_SEMAPHORE_TRIGGER_YIELD);
-		BEGIN_NVC0(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
-		OUT_RING  (chan, upper_32_bits(addr));
-		OUT_RING  (chan, lower_32_bits(addr));
-		OUT_RING  (chan, sync->data);
-		OUT_RING  (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL |
-				 NVC0_SUBCHAN_SEMAPHORE_TRIGGER_YIELD);
-	}
-
-	if (chan) {
-		sync->addr ^= 0x10;
-		sync->data++;
-		FIRE_RING (chan);
-	}
-
-	/* queue the flip */
-	evo_mthd(push, 0x0100, 1);
-	evo_data(push, 0xfffe0000);
-	evo_mthd(push, 0x0084, 1);
-	evo_data(push, swap_interval);
-	if (!(swap_interval & 0x00000100)) {
-		evo_mthd(push, 0x00e0, 1);
-		evo_data(push, 0x40000000);
-	}
-	evo_mthd(push, 0x0088, 4);
-	evo_data(push, sync->addr);
-	evo_data(push, sync->data++);
-	evo_data(push, sync->data);
-	evo_data(push, sync->base.sync.handle);
-	evo_mthd(push, 0x00a0, 2);
-	evo_data(push, 0x00000000);
-	evo_data(push, 0x00000000);
-	evo_mthd(push, 0x00c0, 1);
-	evo_data(push, nv_fb->r_handle);
-	evo_mthd(push, 0x0110, 2);
-	evo_data(push, 0x00000000);
-	evo_data(push, 0x00000000);
-	if (nv50_vers(sync) < GF110_DISP_BASE_CHANNEL_DMA) {
-		evo_mthd(push, 0x0800, 5);
-		evo_data(push, nv_fb->nvbo->bo.offset >> 8);
-		evo_data(push, 0);
-		evo_data(push, (fb->height << 16) | fb->width);
-		evo_data(push, nv_fb->r_pitch);
-		evo_data(push, nv_fb->r_format);
-	} else {
-		evo_mthd(push, 0x0400, 5);
-		evo_data(push, nv_fb->nvbo->bo.offset >> 8);
-		evo_data(push, 0);
-		evo_data(push, (fb->height << 16) | fb->width);
-		evo_data(push, nv_fb->r_pitch);
-		evo_data(push, nv_fb->r_format);
+	if (wndw->func->prepare) {
+		asyh = nv50_head_atom_get(asyw->state.state, asyw->state.crtc);
+		if (IS_ERR(asyh))
+			return PTR_ERR(asyh);
+
+		wndw->func->prepare(wndw, asyh, asyw);
 	}
-	evo_mthd(push, 0x0080, 1);
-	evo_data(push, 0x00000000);
-	evo_kick(push, sync);
 
-	nouveau_bo_ref(nv_fb->nvbo, &head->image);
+	return 0;
+}
+
+static const struct drm_plane_helper_funcs
+nv50_wndw_helper = {
+	.prepare_fb = nv50_wndw_prepare_fb,
+	.cleanup_fb = nv50_wndw_cleanup_fb,
+	.atomic_check = nv50_wndw_atomic_check,
+};
+
+static void
+nv50_wndw_atomic_destroy_state(struct drm_plane *plane,
+			       struct drm_plane_state *state)
+{
+	struct nv50_wndw_atom *asyw = nv50_wndw_atom(state);
+	__drm_atomic_helper_plane_destroy_state(&asyw->state);
+	dma_fence_put(asyw->state.fence);
+	kfree(asyw);
+}
+
+static struct drm_plane_state *
+nv50_wndw_atomic_duplicate_state(struct drm_plane *plane)
+{
+	struct nv50_wndw_atom *armw = nv50_wndw_atom(plane->state);
+	struct nv50_wndw_atom *asyw;
+	if (!(asyw = kmalloc(sizeof(*asyw), GFP_KERNEL)))
+		return NULL;
+	__drm_atomic_helper_plane_duplicate_state(plane, &asyw->state);
+	asyw->state.fence = NULL;
+	asyw->interval = 1;
+	asyw->sema = armw->sema;
+	asyw->ntfy = armw->ntfy;
+	asyw->image = armw->image;
+	asyw->point = armw->point;
+	asyw->lut = armw->lut;
+	asyw->clr.mask = 0;
+	asyw->set.mask = 0;
+	return &asyw->state;
+}
+
+static void
+nv50_wndw_reset(struct drm_plane *plane)
+{
+	struct nv50_wndw_atom *asyw;
+
+	if (WARN_ON(!(asyw = kzalloc(sizeof(*asyw), GFP_KERNEL))))
+		return;
+
+	if (plane->state)
+		plane->funcs->atomic_destroy_state(plane, plane->state);
+	plane->state = &asyw->state;
+	plane->state->plane = plane;
+	plane->state->rotation = DRM_ROTATE_0;
+}
+
+static void
+nv50_wndw_destroy(struct drm_plane *plane)
+{
+	struct nv50_wndw *wndw = nv50_wndw(plane);
+	void *data;
+	nvif_notify_fini(&wndw->notify);
+	data = wndw->func->dtor(wndw);
+	drm_plane_cleanup(&wndw->plane);
+	kfree(data);
+}
+
+static const struct drm_plane_funcs
+nv50_wndw = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.destroy = nv50_wndw_destroy,
+	.reset = nv50_wndw_reset,
+	.set_property = drm_atomic_helper_plane_set_property,
+	.atomic_duplicate_state = nv50_wndw_atomic_duplicate_state,
+	.atomic_destroy_state = nv50_wndw_atomic_destroy_state,
+};
+
+static void
+nv50_wndw_fini(struct nv50_wndw *wndw)
+{
+	nvif_notify_put(&wndw->notify);
+}
+
+static void
+nv50_wndw_init(struct nv50_wndw *wndw)
+{
+	nvif_notify_get(&wndw->notify);
+}
+
+static int
+nv50_wndw_ctor(const struct nv50_wndw_func *func, struct drm_device *dev,
+	       enum drm_plane_type type, const char *name, int index,
+	       struct nv50_dmac *dmac, const u32 *format, int nformat,
+	       struct nv50_wndw *wndw)
+{
+	int ret;
+
+	wndw->func = func;
+	wndw->dmac = dmac;
+
+	ret = drm_universal_plane_init(dev, &wndw->plane, 0, &nv50_wndw, format,
+				       nformat, type, "%s-%d", name, index);
+	if (ret)
+		return ret;
+
+	drm_plane_helper_add(&wndw->plane, &nv50_wndw_helper);
 	return 0;
 }
 
 /******************************************************************************
- * CRTC
+ * Cursor plane
  *****************************************************************************/
+#define nv50_curs(p) container_of((p), struct nv50_curs, wndw)
+
+struct nv50_curs {
+	struct nv50_wndw wndw;
+	struct nvif_object chan;
+};
+
+static u32
+nv50_curs_update(struct nv50_wndw *wndw, u32 interlock)
+{
+	struct nv50_curs *curs = nv50_curs(wndw);
+	nvif_wr32(&curs->chan, 0x0080, 0x00000000);
+	return 0;
+}
+
+static void
+nv50_curs_point(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	struct nv50_curs *curs = nv50_curs(wndw);
+	nvif_wr32(&curs->chan, 0x0084, (asyw->point.y << 16) | asyw->point.x);
+}
+
+static void
+nv50_curs_prepare(struct nv50_wndw *wndw, struct nv50_head_atom *asyh,
+		  struct nv50_wndw_atom *asyw)
+{
+	asyh->curs.handle = nv50_disp(wndw->plane.dev)->mast.base.vram.handle;
+	asyh->curs.offset = asyw->image.offset;
+	asyh->set.curs = asyh->curs.visible;
+}
+
+static void
+nv50_curs_release(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
+		  struct nv50_head_atom *asyh)
+{
+	asyh->curs.visible = false;
+}
+
 static int
-nv50_crtc_set_dither(struct nouveau_crtc *nv_crtc, bool update)
+nv50_curs_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
+		  struct nv50_head_atom *asyh)
 {
-	struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
-	struct nouveau_connector *nv_connector;
-	struct drm_connector *connector;
-	u32 *push, mode = 0x00;
+	int ret;
 
-	nv_connector = nouveau_crtc_connector_get(nv_crtc);
-	connector = &nv_connector->base;
-	if (nv_connector->dithering_mode == DITHERING_MODE_AUTO) {
-		if (nv_crtc->base.primary->fb->depth > connector->display_info.bpc * 3)
-			mode = DITHERING_MODE_DYNAMIC2X2;
-	} else {
-		mode = nv_connector->dithering_mode;
-	}
+	ret = drm_plane_helper_check_state(&asyw->state, &asyw->clip,
+					   DRM_PLANE_HELPER_NO_SCALING,
+					   DRM_PLANE_HELPER_NO_SCALING,
+					   true, true);
+	asyh->curs.visible = asyw->state.visible;
+	if (ret || !asyh->curs.visible)
+		return ret;
 
-	if (nv_connector->dithering_depth == DITHERING_DEPTH_AUTO) {
-		if (connector->display_info.bpc >= 8)
-			mode |= DITHERING_DEPTH_8BPC;
-	} else {
-		mode |= nv_connector->dithering_depth;
+	switch (asyw->state.fb->width) {
+	case 32: asyh->curs.layout = 0; break;
+	case 64: asyh->curs.layout = 1; break;
+	default:
+		return -EINVAL;
 	}
 
-	push = evo_wait(mast, 4);
-	if (push) {
-		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x08a0 + (nv_crtc->index * 0x0400), 1);
-			evo_data(push, mode);
-		} else
-		if (nv50_vers(mast) < GK104_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0490 + (nv_crtc->index * 0x0300), 1);
-			evo_data(push, mode);
-		} else {
-			evo_mthd(push, 0x04a0 + (nv_crtc->index * 0x0300), 1);
-			evo_data(push, mode);
-		}
+	if (asyw->state.fb->width != asyw->state.fb->height)
+		return -EINVAL;
 
-		if (update) {
-			evo_mthd(push, 0x0080, 1);
-			evo_data(push, 0x00000000);
-		}
-		evo_kick(push, mast);
+	switch (asyw->state.fb->pixel_format) {
+	case DRM_FORMAT_ARGB8888: asyh->curs.format = 1; break;
+	default:
+		WARN_ON(1);
+		return -EINVAL;
 	}
 
 	return 0;
 }
 
+static void *
+nv50_curs_dtor(struct nv50_wndw *wndw)
+{
+	struct nv50_curs *curs = nv50_curs(wndw);
+	nvif_object_fini(&curs->chan);
+	return curs;
+}
+
+static const u32
+nv50_curs_format[] = {
+	DRM_FORMAT_ARGB8888,
+};
+
+static const struct nv50_wndw_func
+nv50_curs = {
+	.dtor = nv50_curs_dtor,
+	.acquire = nv50_curs_acquire,
+	.release = nv50_curs_release,
+	.prepare = nv50_curs_prepare,
+	.point = nv50_curs_point,
+	.update = nv50_curs_update,
+};
+
 static int
-nv50_crtc_set_scale(struct nouveau_crtc *nv_crtc, bool update)
+nv50_curs_new(struct nouveau_drm *drm, struct nv50_head *head,
+	      struct nv50_curs **pcurs)
 {
-	struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
-	struct drm_display_mode *omode, *umode = &nv_crtc->base.mode;
-	struct drm_crtc *crtc = &nv_crtc->base;
-	struct nouveau_connector *nv_connector;
-	int mode = DRM_MODE_SCALE_NONE;
-	u32 oX, oY, *push;
+	static const struct nvif_mclass curses[] = {
+		{ GK104_DISP_CURSOR, 0 },
+		{ GF110_DISP_CURSOR, 0 },
+		{ GT214_DISP_CURSOR, 0 },
+		{   G82_DISP_CURSOR, 0 },
+		{  NV50_DISP_CURSOR, 0 },
+		{}
+	};
+	struct nv50_disp_cursor_v0 args = {
+		.head = head->base.index,
+	};
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	struct nv50_curs *curs;
+	int cid, ret;
+
+	cid = nvif_mclass(disp->disp, curses);
+	if (cid < 0) {
+		NV_ERROR(drm, "No supported cursor immediate class\n");
+		return cid;
+	}
 
-	/* start off at the resolution we programmed the crtc for, this
-	 * effectively handles NONE/FULL scaling
-	 */
-	nv_connector = nouveau_crtc_connector_get(nv_crtc);
-	if (nv_connector && nv_connector->native_mode) {
-		mode = nv_connector->scaling_mode;
-		if (nv_connector->scaling_full) /* non-EDID LVDS/eDP mode */
-			mode = DRM_MODE_SCALE_FULLSCREEN;
+	if (!(curs = *pcurs = kzalloc(sizeof(*curs), GFP_KERNEL)))
+		return -ENOMEM;
+
+	ret = nv50_wndw_ctor(&nv50_curs, drm->dev, DRM_PLANE_TYPE_CURSOR,
+			     "curs", head->base.index, &disp->mast.base,
+			     nv50_curs_format, ARRAY_SIZE(nv50_curs_format),
+			     &curs->wndw);
+	if (ret) {
+		kfree(curs);
+		return ret;
 	}
 
-	if (mode != DRM_MODE_SCALE_NONE)
-		omode = nv_connector->native_mode;
-	else
-		omode = umode;
+	ret = nvif_object_init(disp->disp, 0, curses[cid].oclass, &args,
+			       sizeof(args), &curs->chan);
+	if (ret) {
+		NV_ERROR(drm, "curs%04x allocation failed: %d\n",
+			 curses[cid].oclass, ret);
+		return ret;
+	}
 
-	oX = omode->hdisplay;
-	oY = omode->vdisplay;
-	if (omode->flags & DRM_MODE_FLAG_DBLSCAN)
-		oY *= 2;
+	return 0;
+}
 
-	/* add overscan compensation if necessary, will keep the aspect
-	 * ratio the same as the backend mode unless overridden by the
-	 * user setting both hborder and vborder properties.
-	 */
-	if (nv_connector && ( nv_connector->underscan == UNDERSCAN_ON ||
-			     (nv_connector->underscan == UNDERSCAN_AUTO &&
-			      drm_detect_hdmi_monitor(nv_connector->edid)))) {
-		u32 bX = nv_connector->underscan_hborder;
-		u32 bY = nv_connector->underscan_vborder;
-		u32 aspect = (oY << 19) / oX;
+/******************************************************************************
+ * Primary plane
+ *****************************************************************************/
+#define nv50_base(p) container_of((p), struct nv50_base, wndw)
 
-		if (bX) {
-			oX -= (bX * 2);
-			if (bY) oY -= (bY * 2);
-			else    oY  = ((oX * aspect) + (aspect / 2)) >> 19;
-		} else {
-			oX -= (oX >> 4) + 32;
-			if (bY) oY -= (bY * 2);
-			else    oY  = ((oX * aspect) + (aspect / 2)) >> 19;
-		}
+struct nv50_base {
+	struct nv50_wndw wndw;
+	struct nv50_sync chan;
+	int id;
+};
+
+static int
+nv50_base_notify(struct nvif_notify *notify)
+{
+	return NVIF_NOTIFY_KEEP;
+}
+
+static void
+nv50_base_lut(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	struct nv50_base *base = nv50_base(wndw);
+	u32 *push;
+	if ((push = evo_wait(&base->chan, 2))) {
+		evo_mthd(push, 0x00e0, 1);
+		evo_data(push, asyw->lut.enable << 30);
+		evo_kick(push, &base->chan);
 	}
+}
 
-	/* handle CENTER/ASPECT scaling, taking into account the areas
-	 * removed already for overscan compensation
-	 */
-	switch (mode) {
-	case DRM_MODE_SCALE_CENTER:
-		oX = min((u32)umode->hdisplay, oX);
-		oY = min((u32)umode->vdisplay, oY);
-		/* fall-through */
-	case DRM_MODE_SCALE_ASPECT:
-		if (oY < oX) {
-			u32 aspect = (umode->hdisplay << 19) / umode->vdisplay;
-			oX = ((oY * aspect) + (aspect / 2)) >> 19;
-		} else {
-			u32 aspect = (umode->vdisplay << 19) / umode->hdisplay;
-			oY = ((oX * aspect) + (aspect / 2)) >> 19;
-		}
-		break;
-	default:
-		break;
+static void
+nv50_base_image_clr(struct nv50_wndw *wndw)
+{
+	struct nv50_base *base = nv50_base(wndw);
+	u32 *push;
+	if ((push = evo_wait(&base->chan, 4))) {
+		evo_mthd(push, 0x0084, 1);
+		evo_data(push, 0x00000000);
+		evo_mthd(push, 0x00c0, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &base->chan);
 	}
+}
 
-	push = evo_wait(mast, 8);
-	if (push) {
-		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-			/*XXX: SCALE_CTRL_ACTIVE??? */
-			evo_mthd(push, 0x08d8 + (nv_crtc->index * 0x400), 2);
-			evo_data(push, (oY << 16) | oX);
-			evo_data(push, (oY << 16) | oX);
-			evo_mthd(push, 0x08a4 + (nv_crtc->index * 0x400), 1);
+static void
+nv50_base_image_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	struct nv50_base *base = nv50_base(wndw);
+	const s32 oclass = base->chan.base.base.user.oclass;
+	u32 *push;
+	if ((push = evo_wait(&base->chan, 10))) {
+		evo_mthd(push, 0x0084, 1);
+		evo_data(push, (asyw->image.mode << 8) |
+			       (asyw->image.interval << 4));
+		evo_mthd(push, 0x00c0, 1);
+		evo_data(push, asyw->image.handle);
+		if (oclass < G82_DISP_BASE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0800, 5);
+			evo_data(push, asyw->image.offset >> 8);
+			evo_data(push, 0x00000000);
+			evo_data(push, (asyw->image.h << 16) | asyw->image.w);
+			evo_data(push, (asyw->image.layout << 20) |
+					asyw->image.pitch |
+					asyw->image.block);
+			evo_data(push, (asyw->image.kind << 16) |
+				       (asyw->image.format << 8));
+		} else
+		if (oclass < GF110_DISP_BASE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0800, 5);
+			evo_data(push, asyw->image.offset >> 8);
 			evo_data(push, 0x00000000);
-			evo_mthd(push, 0x08c8 + (nv_crtc->index * 0x400), 1);
-			evo_data(push, umode->vdisplay << 16 | umode->hdisplay);
+			evo_data(push, (asyw->image.h << 16) | asyw->image.w);
+			evo_data(push, (asyw->image.layout << 20) |
+					asyw->image.pitch |
+					asyw->image.block);
+			evo_data(push, asyw->image.format << 8);
 		} else {
-			evo_mthd(push, 0x04c0 + (nv_crtc->index * 0x300), 3);
-			evo_data(push, (oY << 16) | oX);
-			evo_data(push, (oY << 16) | oX);
-			evo_data(push, (oY << 16) | oX);
-			evo_mthd(push, 0x0494 + (nv_crtc->index * 0x300), 1);
+			evo_mthd(push, 0x0400, 5);
+			evo_data(push, asyw->image.offset >> 8);
 			evo_data(push, 0x00000000);
-			evo_mthd(push, 0x04b8 + (nv_crtc->index * 0x300), 1);
-			evo_data(push, umode->vdisplay << 16 | umode->hdisplay);
+			evo_data(push, (asyw->image.h << 16) | asyw->image.w);
+			evo_data(push, (asyw->image.layout << 24) |
+					asyw->image.pitch |
+					asyw->image.block);
+			evo_data(push, asyw->image.format << 8);
 		}
+		evo_kick(push, &base->chan);
+	}
+}
 
-		evo_kick(push, mast);
+static void
+nv50_base_ntfy_clr(struct nv50_wndw *wndw)
+{
+	struct nv50_base *base = nv50_base(wndw);
+	u32 *push;
+	if ((push = evo_wait(&base->chan, 2))) {
+		evo_mthd(push, 0x00a4, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &base->chan);
+	}
+}
 
-		if (update) {
-			nv50_display_flip_stop(crtc);
-			nv50_display_flip_next(crtc, crtc->primary->fb,
-					       NULL, 1);
-		}
+static void
+nv50_base_ntfy_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	struct nv50_base *base = nv50_base(wndw);
+	u32 *push;
+	if ((push = evo_wait(&base->chan, 3))) {
+		evo_mthd(push, 0x00a0, 2);
+		evo_data(push, (asyw->ntfy.awaken << 30) | asyw->ntfy.offset);
+		evo_data(push, asyw->ntfy.handle);
+		evo_kick(push, &base->chan);
 	}
+}
 
-	return 0;
+static void
+nv50_base_sema_clr(struct nv50_wndw *wndw)
+{
+	struct nv50_base *base = nv50_base(wndw);
+	u32 *push;
+	if ((push = evo_wait(&base->chan, 2))) {
+		evo_mthd(push, 0x0094, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, &base->chan);
+	}
 }
 
-static int
-nv50_crtc_set_raster_vblank_dmi(struct nouveau_crtc *nv_crtc, u32 usec)
+static void
+nv50_base_sema_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
 {
-	struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
+	struct nv50_base *base = nv50_base(wndw);
 	u32 *push;
+	if ((push = evo_wait(&base->chan, 5))) {
+		evo_mthd(push, 0x0088, 4);
+		evo_data(push, asyw->sema.offset);
+		evo_data(push, asyw->sema.acquire);
+		evo_data(push, asyw->sema.release);
+		evo_data(push, asyw->sema.handle);
+		evo_kick(push, &base->chan);
+	}
+}
 
-	push = evo_wait(mast, 8);
-	if (!push)
-		return -ENOMEM;
+static u32
+nv50_base_update(struct nv50_wndw *wndw, u32 interlock)
+{
+	struct nv50_base *base = nv50_base(wndw);
+	u32 *push;
 
-	evo_mthd(push, 0x0828 + (nv_crtc->index * 0x400), 1);
-	evo_data(push, usec);
-	evo_kick(push, mast);
+	if (!(push = evo_wait(&base->chan, 2)))
+		return 0;
+	evo_mthd(push, 0x0080, 1);
+	evo_data(push, interlock);
+	evo_kick(push, &base->chan);
+
+	if (base->chan.base.base.user.oclass < GF110_DISP_BASE_CHANNEL_DMA)
+		return interlock ? 2 << (base->id * 8) : 0;
+	return interlock ? 2 << (base->id * 4) : 0;
+}
+
+static int
+nv50_base_ntfy_wait_begun(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
+{
+	struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev);
+	struct nv50_disp *disp = nv50_disp(wndw->plane.dev);
+	if (nvif_msec(&drm->device, 2000ULL,
+		u32 data = nouveau_bo_rd32(disp->sync, asyw->ntfy.offset / 4);
+		if ((data & 0xc0000000) == 0x40000000)
+			break;
+		usleep_range(1, 2);
+	) < 0)
+		return -ETIMEDOUT;
 	return 0;
 }
 
+static void
+nv50_base_release(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
+		  struct nv50_head_atom *asyh)
+{
+	asyh->base.cpp = 0;
+}
+
 static int
-nv50_crtc_set_color_vibrance(struct nouveau_crtc *nv_crtc, bool update)
+nv50_base_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
+		  struct nv50_head_atom *asyh)
 {
-	struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
-	u32 *push, hue, vib;
-	int adj;
+	const u32 format = asyw->state.fb->pixel_format;
+	const struct drm_format_info *info;
+	int ret;
 
-	adj = (nv_crtc->color_vibrance > 0) ? 50 : 0;
-	vib = ((nv_crtc->color_vibrance * 2047 + adj) / 100) & 0xfff;
-	hue = ((nv_crtc->vibrant_hue * 2047) / 100) & 0xfff;
+	info = drm_format_info(format);
+	if (!info || !info->depth)
+		return -EINVAL;
 
-	push = evo_wait(mast, 16);
-	if (push) {
-		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x08a8 + (nv_crtc->index * 0x400), 1);
-			evo_data(push, (hue << 20) | (vib << 8));
-		} else {
-			evo_mthd(push, 0x0498 + (nv_crtc->index * 0x300), 1);
-			evo_data(push, (hue << 20) | (vib << 8));
-		}
+	ret = drm_plane_helper_check_state(&asyw->state, &asyw->clip,
+					   DRM_PLANE_HELPER_NO_SCALING,
+					   DRM_PLANE_HELPER_NO_SCALING,
+					   false, true);
+	if (ret)
+		return ret;
 
-		if (update) {
-			evo_mthd(push, 0x0080, 1);
-			evo_data(push, 0x00000000);
-		}
-		evo_kick(push, mast);
+	asyh->base.depth = info->depth;
+	asyh->base.cpp = info->cpp[0];
+	asyh->base.x = asyw->state.src.x1 >> 16;
+	asyh->base.y = asyw->state.src.y1 >> 16;
+	asyh->base.w = asyw->state.fb->width;
+	asyh->base.h = asyw->state.fb->height;
+
+	switch (format) {
+	case DRM_FORMAT_C8         : asyw->image.format = 0x1e; break;
+	case DRM_FORMAT_RGB565     : asyw->image.format = 0xe8; break;
+	case DRM_FORMAT_XRGB1555   :
+	case DRM_FORMAT_ARGB1555   : asyw->image.format = 0xe9; break;
+	case DRM_FORMAT_XRGB8888   :
+	case DRM_FORMAT_ARGB8888   : asyw->image.format = 0xcf; break;
+	case DRM_FORMAT_XBGR2101010:
+	case DRM_FORMAT_ABGR2101010: asyw->image.format = 0xd1; break;
+	case DRM_FORMAT_XBGR8888   :
+	case DRM_FORMAT_ABGR8888   : asyw->image.format = 0xd5; break;
+	default:
+		WARN_ON(1);
+		return -EINVAL;
 	}
 
+	asyw->lut.enable = 1;
+	asyw->set.image = true;
 	return 0;
 }
 
+static void *
+nv50_base_dtor(struct nv50_wndw *wndw)
+{
+	struct nv50_disp *disp = nv50_disp(wndw->plane.dev);
+	struct nv50_base *base = nv50_base(wndw);
+	nv50_dmac_destroy(&base->chan.base, disp->disp);
+	return base;
+}
+
+static const u32
+nv50_base_format[] = {
+	DRM_FORMAT_C8,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_ABGR2101010,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ABGR8888,
+};
+
+static const struct nv50_wndw_func
+nv50_base = {
+	.dtor = nv50_base_dtor,
+	.acquire = nv50_base_acquire,
+	.release = nv50_base_release,
+	.sema_set = nv50_base_sema_set,
+	.sema_clr = nv50_base_sema_clr,
+	.ntfy_set = nv50_base_ntfy_set,
+	.ntfy_clr = nv50_base_ntfy_clr,
+	.ntfy_wait_begun = nv50_base_ntfy_wait_begun,
+	.image_set = nv50_base_image_set,
+	.image_clr = nv50_base_image_clr,
+	.lut = nv50_base_lut,
+	.update = nv50_base_update,
+};
+
 static int
-nv50_crtc_set_image(struct nouveau_crtc *nv_crtc, struct drm_framebuffer *fb,
-		    int x, int y, bool update)
+nv50_base_new(struct nouveau_drm *drm, struct nv50_head *head,
+	      struct nv50_base **pbase)
+{
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	struct nv50_base *base;
+	int ret;
+
+	if (!(base = *pbase = kzalloc(sizeof(*base), GFP_KERNEL)))
+		return -ENOMEM;
+	base->id = head->base.index;
+	base->wndw.ntfy = EVO_FLIP_NTFY0(base->id);
+	base->wndw.sema = EVO_FLIP_SEM0(base->id);
+	base->wndw.data = 0x00000000;
+
+	ret = nv50_wndw_ctor(&nv50_base, drm->dev, DRM_PLANE_TYPE_PRIMARY,
+			     "base", base->id, &base->chan.base,
+			     nv50_base_format, ARRAY_SIZE(nv50_base_format),
+			     &base->wndw);
+	if (ret) {
+		kfree(base);
+		return ret;
+	}
+
+	ret = nv50_base_create(&drm->device, disp->disp, base->id,
+			       disp->sync->bo.offset, &base->chan);
+	if (ret)
+		return ret;
+
+	return nvif_notify_init(&base->chan.base.base.user, nv50_base_notify,
+				false,
+				NV50_DISP_BASE_CHANNEL_DMA_V0_NTFY_UEVENT,
+				&(struct nvif_notify_uevent_req) {},
+				sizeof(struct nvif_notify_uevent_req),
+				sizeof(struct nvif_notify_uevent_rep),
+				&base->wndw.notify);
+}
+
+/******************************************************************************
+ * Head
+ *****************************************************************************/
+static void
+nv50_head_procamp(struct nv50_head *head, struct nv50_head_atom *asyh)
 {
-	struct nouveau_framebuffer *nvfb = nouveau_framebuffer(fb);
-	struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
 	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
+			evo_mthd(push, 0x08a8 + (head->base.index * 0x400), 1);
+		else
+			evo_mthd(push, 0x0498 + (head->base.index * 0x300), 1);
+		evo_data(push, (asyh->procamp.sat.sin << 20) |
+			       (asyh->procamp.sat.cos << 8));
+		evo_kick(push, core);
+	}
+}
 
-	push = evo_wait(mast, 16);
-	if (push) {
-		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0860 + (nv_crtc->index * 0x400), 1);
-			evo_data(push, nvfb->nvbo->bo.offset >> 8);
-			evo_mthd(push, 0x0868 + (nv_crtc->index * 0x400), 3);
-			evo_data(push, (fb->height << 16) | fb->width);
-			evo_data(push, nvfb->r_pitch);
-			evo_data(push, nvfb->r_format);
-			evo_mthd(push, 0x08c0 + (nv_crtc->index * 0x400), 1);
-			evo_data(push, (y << 16) | x);
-			if (nv50_vers(mast) > NV50_DISP_CORE_CHANNEL_DMA) {
-				evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1);
-				evo_data(push, nvfb->r_handle);
-			}
-		} else {
-			evo_mthd(push, 0x0460 + (nv_crtc->index * 0x300), 1);
-			evo_data(push, nvfb->nvbo->bo.offset >> 8);
-			evo_mthd(push, 0x0468 + (nv_crtc->index * 0x300), 4);
-			evo_data(push, (fb->height << 16) | fb->width);
-			evo_data(push, nvfb->r_pitch);
-			evo_data(push, nvfb->r_format);
-			evo_data(push, nvfb->r_handle);
-			evo_mthd(push, 0x04b0 + (nv_crtc->index * 0x300), 1);
-			evo_data(push, (y << 16) | x);
-		}
+static void
+nv50_head_dither(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
+			evo_mthd(push, 0x08a0 + (head->base.index * 0x0400), 1);
+		else
+		if (core->base.user.oclass < GK104_DISP_CORE_CHANNEL_DMA)
+			evo_mthd(push, 0x0490 + (head->base.index * 0x0300), 1);
+		else
+			evo_mthd(push, 0x04a0 + (head->base.index * 0x0300), 1);
+		evo_data(push, (asyh->dither.mode << 3) |
+			       (asyh->dither.bits << 1) |
+			        asyh->dither.enable);
+		evo_kick(push, core);
+	}
+}
 
-		if (update) {
-			evo_mthd(push, 0x0080, 1);
-			evo_data(push, 0x00000000);
+static void
+nv50_head_ovly(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
+	u32 bounds = 0;
+	u32 *push;
+
+	if (asyh->base.cpp) {
+		switch (asyh->base.cpp) {
+		case 8: bounds |= 0x00000500; break;
+		case 4: bounds |= 0x00000300; break;
+		case 2: bounds |= 0x00000100; break;
+		default:
+			WARN_ON(1);
+			break;
 		}
-		evo_kick(push, mast);
+		bounds |= 0x00000001;
 	}
 
-	nv_crtc->fb.handle = nvfb->r_handle;
-	return 0;
+	if ((push = evo_wait(core, 2))) {
+		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
+			evo_mthd(push, 0x0904 + head->base.index * 0x400, 1);
+		else
+			evo_mthd(push, 0x04d4 + head->base.index * 0x300, 1);
+		evo_data(push, bounds);
+		evo_kick(push, core);
+	}
 }
 
 static void
-nv50_crtc_cursor_show(struct nouveau_crtc *nv_crtc)
+nv50_head_base(struct nv50_head *head, struct nv50_head_atom *asyh)
 {
-	struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
-	u32 *push = evo_wait(mast, 16);
-	if (push) {
-		if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 2);
-			evo_data(push, 0x85000000);
-			evo_data(push, nv_crtc->cursor.nvbo->bo.offset >> 8);
-		} else
-		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 2);
-			evo_data(push, 0x85000000);
-			evo_data(push, nv_crtc->cursor.nvbo->bo.offset >> 8);
-			evo_mthd(push, 0x089c + (nv_crtc->index * 0x400), 1);
-			evo_data(push, mast->base.vram.handle);
-		} else {
-			evo_mthd(push, 0x0480 + (nv_crtc->index * 0x300), 2);
-			evo_data(push, 0x85000000);
-			evo_data(push, nv_crtc->cursor.nvbo->bo.offset >> 8);
-			evo_mthd(push, 0x048c + (nv_crtc->index * 0x300), 1);
-			evo_data(push, mast->base.vram.handle);
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
+	u32 bounds = 0;
+	u32 *push;
+
+	if (asyh->base.cpp) {
+		switch (asyh->base.cpp) {
+		case 8: bounds |= 0x00000500; break;
+		case 4: bounds |= 0x00000300; break;
+		case 2: bounds |= 0x00000100; break;
+		case 1: bounds |= 0x00000000; break;
+		default:
+			WARN_ON(1);
+			break;
 		}
-		evo_kick(push, mast);
+		bounds |= 0x00000001;
+	}
+
+	if ((push = evo_wait(core, 2))) {
+		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
+			evo_mthd(push, 0x0900 + head->base.index * 0x400, 1);
+		else
+			evo_mthd(push, 0x04d0 + head->base.index * 0x300, 1);
+		evo_data(push, bounds);
+		evo_kick(push, core);
 	}
-	nv_crtc->cursor.visible = true;
 }
 
 static void
-nv50_crtc_cursor_hide(struct nouveau_crtc *nv_crtc)
+nv50_head_curs_clr(struct nv50_head *head)
 {
-	struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
-	u32 *push = evo_wait(mast, 16);
-	if (push) {
-		if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 1);
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
+	u32 *push;
+	if ((push = evo_wait(core, 4))) {
+		if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0880 + head->base.index * 0x400, 1);
 			evo_data(push, 0x05000000);
 		} else
-		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 1);
+		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0880 + head->base.index * 0x400, 1);
 			evo_data(push, 0x05000000);
-			evo_mthd(push, 0x089c + (nv_crtc->index * 0x400), 1);
+			evo_mthd(push, 0x089c + head->base.index * 0x400, 1);
 			evo_data(push, 0x00000000);
 		} else {
-			evo_mthd(push, 0x0480 + (nv_crtc->index * 0x300), 1);
+			evo_mthd(push, 0x0480 + head->base.index * 0x300, 1);
 			evo_data(push, 0x05000000);
-			evo_mthd(push, 0x048c + (nv_crtc->index * 0x300), 1);
+			evo_mthd(push, 0x048c + head->base.index * 0x300, 1);
 			evo_data(push, 0x00000000);
 		}
-		evo_kick(push, mast);
+		evo_kick(push, core);
 	}
-	nv_crtc->cursor.visible = false;
 }
 
 static void
-nv50_crtc_cursor_show_hide(struct nouveau_crtc *nv_crtc, bool show, bool update)
+nv50_head_curs_set(struct nv50_head *head, struct nv50_head_atom *asyh)
 {
-	struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
-
-	if (show && nv_crtc->cursor.nvbo && nv_crtc->base.enabled)
-		nv50_crtc_cursor_show(nv_crtc);
-	else
-		nv50_crtc_cursor_hide(nv_crtc);
-
-	if (update) {
-		u32 *push = evo_wait(mast, 2);
-		if (push) {
-			evo_mthd(push, 0x0080, 1);
-			evo_data(push, 0x00000000);
-			evo_kick(push, mast);
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
+	u32 *push;
+	if ((push = evo_wait(core, 5))) {
+		if (core->base.user.oclass < G82_DISP_BASE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0880 + head->base.index * 0x400, 2);
+			evo_data(push, 0x80000000 | (asyh->curs.layout << 26) |
+						    (asyh->curs.format << 24));
+			evo_data(push, asyh->curs.offset >> 8);
+		} else
+		if (core->base.user.oclass < GF110_DISP_BASE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0880 + head->base.index * 0x400, 2);
+			evo_data(push, 0x80000000 | (asyh->curs.layout << 26) |
+						    (asyh->curs.format << 24));
+			evo_data(push, asyh->curs.offset >> 8);
+			evo_mthd(push, 0x089c + head->base.index * 0x400, 1);
+			evo_data(push, asyh->curs.handle);
+		} else {
+			evo_mthd(push, 0x0480 + head->base.index * 0x300, 2);
+			evo_data(push, 0x80000000 | (asyh->curs.layout << 26) |
+						    (asyh->curs.format << 24));
+			evo_data(push, asyh->curs.offset >> 8);
+			evo_mthd(push, 0x048c + head->base.index * 0x300, 1);
+			evo_data(push, asyh->curs.handle);
 		}
+		evo_kick(push, core);
 	}
 }
 
 static void
-nv50_crtc_dpms(struct drm_crtc *crtc, int mode)
+nv50_head_core_clr(struct nv50_head *head)
 {
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
+	u32 *push;
+	if ((push = evo_wait(core, 2))) {
+		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
+			evo_mthd(push, 0x0874 + head->base.index * 0x400, 1);
+		else
+			evo_mthd(push, 0x0474 + head->base.index * 0x300, 1);
+		evo_data(push, 0x00000000);
+		evo_kick(push, core);
+	}
 }
 
 static void
-nv50_crtc_prepare(struct drm_crtc *crtc)
+nv50_head_core_set(struct nv50_head *head, struct nv50_head_atom *asyh)
 {
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct nv50_mast *mast = nv50_mast(crtc->dev);
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
 	u32 *push;
+	if ((push = evo_wait(core, 9))) {
+		if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0860 + head->base.index * 0x400, 1);
+			evo_data(push, asyh->core.offset >> 8);
+			evo_mthd(push, 0x0868 + head->base.index * 0x400, 4);
+			evo_data(push, (asyh->core.h << 16) | asyh->core.w);
+			evo_data(push, asyh->core.layout << 20 |
+				       (asyh->core.pitch >> 8) << 8 |
+				       asyh->core.block);
+			evo_data(push, asyh->core.kind << 16 |
+				       asyh->core.format << 8);
+			evo_data(push, asyh->core.handle);
+			evo_mthd(push, 0x08c0 + head->base.index * 0x400, 1);
+			evo_data(push, (asyh->core.y << 16) | asyh->core.x);
+			/* EVO will complain with INVALID_STATE if we have an
+			 * active cursor and (re)specify HeadSetContextDmaIso
+			 * without also updating HeadSetOffsetCursor.
+			 */
+			asyh->set.curs = asyh->curs.visible;
+		} else
+		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0860 + head->base.index * 0x400, 1);
+			evo_data(push, asyh->core.offset >> 8);
+			evo_mthd(push, 0x0868 + head->base.index * 0x400, 4);
+			evo_data(push, (asyh->core.h << 16) | asyh->core.w);
+			evo_data(push, asyh->core.layout << 20 |
+				       (asyh->core.pitch >> 8) << 8 |
+				       asyh->core.block);
+			evo_data(push, asyh->core.format << 8);
+			evo_data(push, asyh->core.handle);
+			evo_mthd(push, 0x08c0 + head->base.index * 0x400, 1);
+			evo_data(push, (asyh->core.y << 16) | asyh->core.x);
+		} else {
+			evo_mthd(push, 0x0460 + head->base.index * 0x300, 1);
+			evo_data(push, asyh->core.offset >> 8);
+			evo_mthd(push, 0x0468 + head->base.index * 0x300, 4);
+			evo_data(push, (asyh->core.h << 16) | asyh->core.w);
+			evo_data(push, asyh->core.layout << 24 |
+				       (asyh->core.pitch >> 8) << 8 |
+				       asyh->core.block);
+			evo_data(push, asyh->core.format << 8);
+			evo_data(push, asyh->core.handle);
+			evo_mthd(push, 0x04b0 + head->base.index * 0x300, 1);
+			evo_data(push, (asyh->core.y << 16) | asyh->core.x);
+		}
+		evo_kick(push, core);
+	}
+}
 
-	nv50_display_flip_stop(crtc);
-
-	push = evo_wait(mast, 6);
-	if (push) {
-		if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1);
-			evo_data(push, 0x00000000);
-			evo_mthd(push, 0x0840 + (nv_crtc->index * 0x400), 1);
+static void
+nv50_head_lut_clr(struct nv50_head *head)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
+	u32 *push;
+	if ((push = evo_wait(core, 4))) {
+		if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0840 + (head->base.index * 0x400), 1);
 			evo_data(push, 0x40000000);
 		} else
-		if (nv50_vers(mast) <  GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1);
-			evo_data(push, 0x00000000);
-			evo_mthd(push, 0x0840 + (nv_crtc->index * 0x400), 1);
+		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0840 + (head->base.index * 0x400), 1);
 			evo_data(push, 0x40000000);
-			evo_mthd(push, 0x085c + (nv_crtc->index * 0x400), 1);
+			evo_mthd(push, 0x085c + (head->base.index * 0x400), 1);
 			evo_data(push, 0x00000000);
 		} else {
-			evo_mthd(push, 0x0474 + (nv_crtc->index * 0x300), 1);
-			evo_data(push, 0x00000000);
-			evo_mthd(push, 0x0440 + (nv_crtc->index * 0x300), 1);
+			evo_mthd(push, 0x0440 + (head->base.index * 0x300), 1);
 			evo_data(push, 0x03000000);
-			evo_mthd(push, 0x045c + (nv_crtc->index * 0x300), 1);
+			evo_mthd(push, 0x045c + (head->base.index * 0x300), 1);
 			evo_data(push, 0x00000000);
 		}
-
-		evo_kick(push, mast);
+		evo_kick(push, core);
 	}
-
-	nv50_crtc_cursor_show_hide(nv_crtc, false, false);
 }
 
 static void
-nv50_crtc_commit(struct drm_crtc *crtc)
+nv50_head_lut_set(struct nv50_head *head, struct nv50_head_atom *asyh)
 {
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct nv50_mast *mast = nv50_mast(crtc->dev);
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
 	u32 *push;
-
-	push = evo_wait(mast, 32);
-	if (push) {
-		if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1);
-			evo_data(push, nv_crtc->fb.handle);
-			evo_mthd(push, 0x0840 + (nv_crtc->index * 0x400), 2);
+	if ((push = evo_wait(core, 7))) {
+		if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0840 + (head->base.index * 0x400), 2);
 			evo_data(push, 0xc0000000);
-			evo_data(push, nv_crtc->lut.nvbo->bo.offset >> 8);
+			evo_data(push, asyh->lut.offset >> 8);
 		} else
-		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1);
-			evo_data(push, nv_crtc->fb.handle);
-			evo_mthd(push, 0x0840 + (nv_crtc->index * 0x400), 2);
+		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0840 + (head->base.index * 0x400), 2);
 			evo_data(push, 0xc0000000);
-			evo_data(push, nv_crtc->lut.nvbo->bo.offset >> 8);
-			evo_mthd(push, 0x085c + (nv_crtc->index * 0x400), 1);
-			evo_data(push, mast->base.vram.handle);
+			evo_data(push, asyh->lut.offset >> 8);
+			evo_mthd(push, 0x085c + (head->base.index * 0x400), 1);
+			evo_data(push, asyh->lut.handle);
 		} else {
-			evo_mthd(push, 0x0474 + (nv_crtc->index * 0x300), 1);
-			evo_data(push, nv_crtc->fb.handle);
-			evo_mthd(push, 0x0440 + (nv_crtc->index * 0x300), 4);
+			evo_mthd(push, 0x0440 + (head->base.index * 0x300), 4);
 			evo_data(push, 0x83000000);
-			evo_data(push, nv_crtc->lut.nvbo->bo.offset >> 8);
+			evo_data(push, asyh->lut.offset >> 8);
 			evo_data(push, 0x00000000);
 			evo_data(push, 0x00000000);
-			evo_mthd(push, 0x045c + (nv_crtc->index * 0x300), 1);
-			evo_data(push, mast->base.vram.handle);
-			evo_mthd(push, 0x0430 + (nv_crtc->index * 0x300), 1);
+			evo_mthd(push, 0x045c + (head->base.index * 0x300), 1);
+			evo_data(push, asyh->lut.handle);
+		}
+		evo_kick(push, core);
+	}
+}
+
+static void
+nv50_head_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
+	struct nv50_head_mode *m = &asyh->mode;
+	u32 *push;
+	if ((push = evo_wait(core, 14))) {
+		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
+			evo_mthd(push, 0x0804 + (head->base.index * 0x400), 2);
+			evo_data(push, 0x00800000 | m->clock);
+			evo_data(push, m->interlace ? 0x00000002 : 0x00000000);
+			evo_mthd(push, 0x0810 + (head->base.index * 0x400), 7);
+			evo_data(push, 0x00000000);
+			evo_data(push, (m->v.active  << 16) | m->h.active );
+			evo_data(push, (m->v.synce   << 16) | m->h.synce  );
+			evo_data(push, (m->v.blanke  << 16) | m->h.blanke );
+			evo_data(push, (m->v.blanks  << 16) | m->h.blanks );
+			evo_data(push, (m->v.blank2e << 16) | m->v.blank2s);
+			evo_data(push, asyh->mode.v.blankus);
+			evo_mthd(push, 0x082c + (head->base.index * 0x400), 1);
+			evo_data(push, 0x00000000);
+		} else {
+			evo_mthd(push, 0x0410 + (head->base.index * 0x300), 6);
+			evo_data(push, 0x00000000);
+			evo_data(push, (m->v.active  << 16) | m->h.active );
+			evo_data(push, (m->v.synce   << 16) | m->h.synce  );
+			evo_data(push, (m->v.blanke  << 16) | m->h.blanke );
+			evo_data(push, (m->v.blanks  << 16) | m->h.blanks );
+			evo_data(push, (m->v.blank2e << 16) | m->v.blank2s);
+			evo_mthd(push, 0x042c + (head->base.index * 0x300), 2);
+			evo_data(push, 0x00000000); /* ??? */
 			evo_data(push, 0xffffff00);
+			evo_mthd(push, 0x0450 + (head->base.index * 0x300), 3);
+			evo_data(push, m->clock * 1000);
+			evo_data(push, 0x00200000); /* ??? */
+			evo_data(push, m->clock * 1000);
 		}
+		evo_kick(push, core);
+	}
+}
 
-		evo_kick(push, mast);
+static void
+nv50_head_view(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
+	u32 *push;
+	if ((push = evo_wait(core, 10))) {
+		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
+			evo_mthd(push, 0x08a4 + (head->base.index * 0x400), 1);
+			evo_data(push, 0x00000000);
+			evo_mthd(push, 0x08c8 + (head->base.index * 0x400), 1);
+			evo_data(push, (asyh->view.iH << 16) | asyh->view.iW);
+			evo_mthd(push, 0x08d8 + (head->base.index * 0x400), 2);
+			evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
+			evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
+		} else {
+			evo_mthd(push, 0x0494 + (head->base.index * 0x300), 1);
+			evo_data(push, 0x00000000);
+			evo_mthd(push, 0x04b8 + (head->base.index * 0x300), 1);
+			evo_data(push, (asyh->view.iH << 16) | asyh->view.iW);
+			evo_mthd(push, 0x04c0 + (head->base.index * 0x300), 3);
+			evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
+			evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
+			evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
+		}
+		evo_kick(push, core);
 	}
+}
 
-	nv50_crtc_cursor_show_hide(nv_crtc, true, true);
-	nv50_display_flip_next(crtc, crtc->primary->fb, NULL, 1);
+static void
+nv50_head_flush_clr(struct nv50_head *head, struct nv50_head_atom *asyh, bool y)
+{
+	if (asyh->clr.core && (!asyh->set.core || y))
+		nv50_head_lut_clr(head);
+	if (asyh->clr.core && (!asyh->set.core || y))
+		nv50_head_core_clr(head);
+	if (asyh->clr.curs && (!asyh->set.curs || y))
+		nv50_head_curs_clr(head);
 }
 
-static bool
-nv50_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode,
-		     struct drm_display_mode *adjusted_mode)
+static void
+nv50_head_flush_set(struct nv50_head *head, struct nv50_head_atom *asyh)
 {
-	drm_mode_set_crtcinfo(adjusted_mode, CRTC_INTERLACE_HALVE_V);
-	return true;
+	if (asyh->set.view   ) nv50_head_view    (head, asyh);
+	if (asyh->set.mode   ) nv50_head_mode    (head, asyh);
+	if (asyh->set.core   ) nv50_head_lut_set (head, asyh);
+	if (asyh->set.core   ) nv50_head_core_set(head, asyh);
+	if (asyh->set.curs   ) nv50_head_curs_set(head, asyh);
+	if (asyh->set.base   ) nv50_head_base    (head, asyh);
+	if (asyh->set.ovly   ) nv50_head_ovly    (head, asyh);
+	if (asyh->set.dither ) nv50_head_dither  (head, asyh);
+	if (asyh->set.procamp) nv50_head_procamp (head, asyh);
 }
 
-static int
-nv50_crtc_swap_fbs(struct drm_crtc *crtc, struct drm_framebuffer *old_fb)
+static void
+nv50_head_atomic_check_procamp(struct nv50_head_atom *armh,
+			       struct nv50_head_atom *asyh,
+			       struct nouveau_conn_atom *asyc)
 {
-	struct nouveau_framebuffer *nvfb = nouveau_framebuffer(crtc->primary->fb);
-	struct nv50_head *head = nv50_head(crtc);
-	int ret;
+	const int vib = asyc->procamp.color_vibrance - 100;
+	const int hue = asyc->procamp.vibrant_hue - 90;
+	const int adj = (vib > 0) ? 50 : 0;
+	asyh->procamp.sat.cos = ((vib * 2047 + adj) / 100) & 0xfff;
+	asyh->procamp.sat.sin = ((hue * 2047) / 100) & 0xfff;
+	asyh->set.procamp = true;
+}
+
+static void
+nv50_head_atomic_check_dither(struct nv50_head_atom *armh,
+			      struct nv50_head_atom *asyh,
+			      struct nouveau_conn_atom *asyc)
+{
+	struct drm_connector *connector = asyc->state.connector;
+	u32 mode = 0x00;
+
+	if (asyc->dither.mode == DITHERING_MODE_AUTO) {
+		if (asyh->base.depth > connector->display_info.bpc * 3)
+			mode = DITHERING_MODE_DYNAMIC2X2;
+	} else {
+		mode = asyc->dither.mode;
+	}
 
-	ret = nouveau_bo_pin(nvfb->nvbo, TTM_PL_FLAG_VRAM, true);
-	if (ret == 0) {
-		if (head->image)
-			nouveau_bo_unpin(head->image);
-		nouveau_bo_ref(nvfb->nvbo, &head->image);
+	if (asyc->dither.depth == DITHERING_DEPTH_AUTO) {
+		if (connector->display_info.bpc >= 8)
+			mode |= DITHERING_DEPTH_8BPC;
+	} else {
+		mode |= asyc->dither.depth;
 	}
 
-	return ret;
+	asyh->dither.enable = mode;
+	asyh->dither.bits = mode >> 1;
+	asyh->dither.mode = mode >> 3;
+	asyh->set.dither = true;
 }
 
-static int
-nv50_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *umode,
-		   struct drm_display_mode *mode, int x, int y,
-		   struct drm_framebuffer *old_fb)
+static void
+nv50_head_atomic_check_view(struct nv50_head_atom *armh,
+			    struct nv50_head_atom *asyh,
+			    struct nouveau_conn_atom *asyc)
 {
-	struct nv50_mast *mast = nv50_mast(crtc->dev);
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct nouveau_connector *nv_connector;
-	u32 ilace = (mode->flags & DRM_MODE_FLAG_INTERLACE) ? 2 : 1;
-	u32 vscan = (mode->flags & DRM_MODE_FLAG_DBLSCAN) ? 2 : 1;
-	u32 hactive, hsynce, hbackp, hfrontp, hblanke, hblanks;
-	u32 vactive, vsynce, vbackp, vfrontp, vblanke, vblanks;
-	u32 vblan2e = 0, vblan2s = 1, vblankus = 0;
-	u32 *push;
-	int ret;
-
-	hactive = mode->htotal;
-	hsynce  = mode->hsync_end - mode->hsync_start - 1;
-	hbackp  = mode->htotal - mode->hsync_end;
-	hblanke = hsynce + hbackp;
-	hfrontp = mode->hsync_start - mode->hdisplay;
-	hblanks = mode->htotal - hfrontp - 1;
-
-	vactive = mode->vtotal * vscan / ilace;
-	vsynce  = ((mode->vsync_end - mode->vsync_start) * vscan / ilace) - 1;
-	vbackp  = (mode->vtotal - mode->vsync_end) * vscan / ilace;
-	vblanke = vsynce + vbackp;
-	vfrontp = (mode->vsync_start - mode->vdisplay) * vscan / ilace;
-	vblanks = vactive - vfrontp - 1;
-	/* XXX: Safe underestimate, even "0" works */
-	vblankus = (vactive - mode->vdisplay - 2) * hactive;
-	vblankus *= 1000;
-	vblankus /= mode->clock;
+	struct drm_connector *connector = asyc->state.connector;
+	struct drm_display_mode *omode = &asyh->state.adjusted_mode;
+	struct drm_display_mode *umode = &asyh->state.mode;
+	int mode = asyc->scaler.mode;
+	struct edid *edid;
+
+	if (connector->edid_blob_ptr)
+		edid = (struct edid *)connector->edid_blob_ptr->data;
+	else
+		edid = NULL;
 
-	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
-		vblan2e = vactive + vsynce + vbackp;
-		vblan2s = vblan2e + (mode->vdisplay * vscan / ilace);
-		vactive = (vactive * 2) + 1;
+	if (!asyc->scaler.full) {
+		if (mode == DRM_MODE_SCALE_NONE)
+			omode = umode;
+	} else {
+		/* Non-EDID LVDS/eDP mode. */
+		mode = DRM_MODE_SCALE_FULLSCREEN;
 	}
 
-	ret = nv50_crtc_swap_fbs(crtc, old_fb);
-	if (ret)
-		return ret;
+	asyh->view.iW = umode->hdisplay;
+	asyh->view.iH = umode->vdisplay;
+	asyh->view.oW = omode->hdisplay;
+	asyh->view.oH = omode->vdisplay;
+	if (omode->flags & DRM_MODE_FLAG_DBLSCAN)
+		asyh->view.oH *= 2;
 
-	push = evo_wait(mast, 64);
-	if (push) {
-		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-			evo_mthd(push, 0x0804 + (nv_crtc->index * 0x400), 2);
-			evo_data(push, 0x00800000 | mode->clock);
-			evo_data(push, (ilace == 2) ? 2 : 0);
-			evo_mthd(push, 0x0810 + (nv_crtc->index * 0x400), 6);
-			evo_data(push, 0x00000000);
-			evo_data(push, (vactive << 16) | hactive);
-			evo_data(push, ( vsynce << 16) | hsynce);
-			evo_data(push, (vblanke << 16) | hblanke);
-			evo_data(push, (vblanks << 16) | hblanks);
-			evo_data(push, (vblan2e << 16) | vblan2s);
-			evo_mthd(push, 0x082c + (nv_crtc->index * 0x400), 1);
-			evo_data(push, 0x00000000);
-			evo_mthd(push, 0x0900 + (nv_crtc->index * 0x400), 2);
-			evo_data(push, 0x00000311);
-			evo_data(push, 0x00000100);
+	/* Add overscan compensation if necessary, will keep the aspect
+	 * ratio the same as the backend mode unless overridden by the
+	 * user setting both hborder and vborder properties.
+	 */
+	if ((asyc->scaler.underscan.mode == UNDERSCAN_ON ||
+	    (asyc->scaler.underscan.mode == UNDERSCAN_AUTO &&
+	     drm_detect_hdmi_monitor(edid)))) {
+		u32 bX = asyc->scaler.underscan.hborder;
+		u32 bY = asyc->scaler.underscan.vborder;
+		u32 r = (asyh->view.oH << 19) / asyh->view.oW;
+
+		if (bX) {
+			asyh->view.oW -= (bX * 2);
+			if (bY) asyh->view.oH -= (bY * 2);
+			else    asyh->view.oH  = ((asyh->view.oW * r) + (r / 2)) >> 19;
 		} else {
-			evo_mthd(push, 0x0410 + (nv_crtc->index * 0x300), 6);
-			evo_data(push, 0x00000000);
-			evo_data(push, (vactive << 16) | hactive);
-			evo_data(push, ( vsynce << 16) | hsynce);
-			evo_data(push, (vblanke << 16) | hblanke);
-			evo_data(push, (vblanks << 16) | hblanks);
-			evo_data(push, (vblan2e << 16) | vblan2s);
-			evo_mthd(push, 0x042c + (nv_crtc->index * 0x300), 1);
-			evo_data(push, 0x00000000); /* ??? */
-			evo_mthd(push, 0x0450 + (nv_crtc->index * 0x300), 3);
-			evo_data(push, mode->clock * 1000);
-			evo_data(push, 0x00200000); /* ??? */
-			evo_data(push, mode->clock * 1000);
-			evo_mthd(push, 0x04d0 + (nv_crtc->index * 0x300), 2);
-			evo_data(push, 0x00000311);
-			evo_data(push, 0x00000100);
+			asyh->view.oW -= (asyh->view.oW >> 4) + 32;
+			if (bY) asyh->view.oH -= (bY * 2);
+			else    asyh->view.oH  = ((asyh->view.oW * r) + (r / 2)) >> 19;
 		}
+	}
 
-		evo_kick(push, mast);
+	/* Handle CENTER/ASPECT scaling, taking into account the areas
+	 * removed already for overscan compensation.
+	 */
+	switch (mode) {
+	case DRM_MODE_SCALE_CENTER:
+		asyh->view.oW = min((u16)umode->hdisplay, asyh->view.oW);
+		asyh->view.oH = min((u16)umode->vdisplay, asyh->view.oH);
+		/* fall-through */
+	case DRM_MODE_SCALE_ASPECT:
+		if (asyh->view.oH < asyh->view.oW) {
+			u32 r = (asyh->view.iW << 19) / asyh->view.iH;
+			asyh->view.oW = ((asyh->view.oH * r) + (r / 2)) >> 19;
+		} else {
+			u32 r = (asyh->view.iH << 19) / asyh->view.iW;
+			asyh->view.oH = ((asyh->view.oW * r) + (r / 2)) >> 19;
+		}
+		break;
+	default:
+		break;
 	}
 
-	nv_connector = nouveau_crtc_connector_get(nv_crtc);
-	nv50_crtc_set_dither(nv_crtc, false);
-	nv50_crtc_set_scale(nv_crtc, false);
+	asyh->set.view = true;
+}
 
-	/* G94 only accepts this after setting scale */
-	if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA)
-		nv50_crtc_set_raster_vblank_dmi(nv_crtc, vblankus);
+static void
+nv50_head_atomic_check_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
+{
+	struct drm_display_mode *mode = &asyh->state.adjusted_mode;
+	u32 ilace   = (mode->flags & DRM_MODE_FLAG_INTERLACE) ? 2 : 1;
+	u32 vscan   = (mode->flags & DRM_MODE_FLAG_DBLSCAN) ? 2 : 1;
+	u32 hbackp  =  mode->htotal - mode->hsync_end;
+	u32 vbackp  = (mode->vtotal - mode->vsync_end) * vscan / ilace;
+	u32 hfrontp =  mode->hsync_start - mode->hdisplay;
+	u32 vfrontp = (mode->vsync_start - mode->vdisplay) * vscan / ilace;
+	struct nv50_head_mode *m = &asyh->mode;
+
+	m->h.active = mode->htotal;
+	m->h.synce  = mode->hsync_end - mode->hsync_start - 1;
+	m->h.blanke = m->h.synce + hbackp;
+	m->h.blanks = mode->htotal - hfrontp - 1;
+
+	m->v.active = mode->vtotal * vscan / ilace;
+	m->v.synce  = ((mode->vsync_end - mode->vsync_start) * vscan / ilace) - 1;
+	m->v.blanke = m->v.synce + vbackp;
+	m->v.blanks = m->v.active - vfrontp - 1;
+
+	/*XXX: Safe underestimate, even "0" works */
+	m->v.blankus = (m->v.active - mode->vdisplay - 2) * m->h.active;
+	m->v.blankus *= 1000;
+	m->v.blankus /= mode->clock;
 
-	nv50_crtc_set_color_vibrance(nv_crtc, false);
-	nv50_crtc_set_image(nv_crtc, crtc->primary->fb, x, y, false);
-	return 0;
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
+		m->v.blank2e =  m->v.active + m->v.synce + vbackp;
+		m->v.blank2s =  m->v.blank2e + (mode->vdisplay * vscan / ilace);
+		m->v.active  = (m->v.active * 2) + 1;
+		m->interlace = true;
+	} else {
+		m->v.blank2e = 0;
+		m->v.blank2s = 1;
+		m->interlace = false;
+	}
+	m->clock = mode->clock;
+
+	drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V);
+	asyh->set.mode = true;
 }
 
 static int
-nv50_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
-			struct drm_framebuffer *old_fb)
+nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state)
 {
 	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	int ret;
+	struct nv50_disp *disp = nv50_disp(crtc->dev);
+	struct nv50_head *head = nv50_head(crtc);
+	struct nv50_head_atom *armh = nv50_head_atom(crtc->state);
+	struct nv50_head_atom *asyh = nv50_head_atom(state);
+	struct nouveau_conn_atom *asyc = NULL;
+	struct drm_connector_state *conns;
+	struct drm_connector *conn;
+	int i;
 
-	if (!crtc->primary->fb) {
-		NV_DEBUG(drm, "No FB bound\n");
-		return 0;
+	NV_ATOMIC(drm, "%s atomic_check %d\n", crtc->name, asyh->state.active);
+	if (asyh->state.active) {
+		for_each_connector_in_state(asyh->state.state, conn, conns, i) {
+			if (conns->crtc == crtc) {
+				asyc = nouveau_conn_atom(conns);
+				break;
+			}
+		}
+
+		if (armh->state.active) {
+			if (asyc) {
+				if (asyh->state.mode_changed)
+					asyc->set.scaler = true;
+				if (armh->base.depth != asyh->base.depth)
+					asyc->set.dither = true;
+			}
+		} else {
+			asyc->set.mask = ~0;
+			asyh->set.mask = ~0;
+		}
+
+		if (asyh->state.mode_changed)
+			nv50_head_atomic_check_mode(head, asyh);
+
+		if (asyc) {
+			if (asyc->set.scaler)
+				nv50_head_atomic_check_view(armh, asyh, asyc);
+			if (asyc->set.dither)
+				nv50_head_atomic_check_dither(armh, asyh, asyc);
+			if (asyc->set.procamp)
+				nv50_head_atomic_check_procamp(armh, asyh, asyc);
+		}
+
+		if ((asyh->core.visible = (asyh->base.cpp != 0))) {
+			asyh->core.x = asyh->base.x;
+			asyh->core.y = asyh->base.y;
+			asyh->core.w = asyh->base.w;
+			asyh->core.h = asyh->base.h;
+		} else
+		if ((asyh->core.visible = asyh->curs.visible)) {
+			/*XXX: We need to either find some way of having the
+			 *     primary base layer appear black, while still
+			 *     being able to display the other layers, or we
+			 *     need to allocate a dummy black surface here.
+			 */
+			asyh->core.x = 0;
+			asyh->core.y = 0;
+			asyh->core.w = asyh->state.mode.hdisplay;
+			asyh->core.h = asyh->state.mode.vdisplay;
+		}
+		asyh->core.handle = disp->mast.base.vram.handle;
+		asyh->core.offset = 0;
+		asyh->core.format = 0xcf;
+		asyh->core.kind = 0;
+		asyh->core.layout = 1;
+		asyh->core.block = 0;
+		asyh->core.pitch = ALIGN(asyh->core.w, 64) * 4;
+		asyh->lut.handle = disp->mast.base.vram.handle;
+		asyh->lut.offset = head->base.lut.nvbo->bo.offset;
+		asyh->set.base = armh->base.cpp != asyh->base.cpp;
+		asyh->set.ovly = armh->ovly.cpp != asyh->ovly.cpp;
+	} else {
+		asyh->core.visible = false;
+		asyh->curs.visible = false;
+		asyh->base.cpp = 0;
+		asyh->ovly.cpp = 0;
 	}
 
-	ret = nv50_crtc_swap_fbs(crtc, old_fb);
-	if (ret)
-		return ret;
+	if (!drm_atomic_crtc_needs_modeset(&asyh->state)) {
+		if (asyh->core.visible) {
+			if (memcmp(&armh->core, &asyh->core, sizeof(asyh->core)))
+				asyh->set.core = true;
+		} else
+		if (armh->core.visible) {
+			asyh->clr.core = true;
+		}
 
-	nv50_display_flip_stop(crtc);
-	nv50_crtc_set_image(nv_crtc, crtc->primary->fb, x, y, true);
-	nv50_display_flip_next(crtc, crtc->primary->fb, NULL, 1);
-	return 0;
-}
+		if (asyh->curs.visible) {
+			if (memcmp(&armh->curs, &asyh->curs, sizeof(asyh->curs)))
+				asyh->set.curs = true;
+		} else
+		if (armh->curs.visible) {
+			asyh->clr.curs = true;
+		}
+	} else {
+		asyh->clr.core = armh->core.visible;
+		asyh->clr.curs = armh->curs.visible;
+		asyh->set.core = asyh->core.visible;
+		asyh->set.curs = asyh->curs.visible;
+	}
 
-static int
-nv50_crtc_mode_set_base_atomic(struct drm_crtc *crtc,
-			       struct drm_framebuffer *fb, int x, int y,
-			       enum mode_set_atomic state)
-{
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	nv50_display_flip_stop(crtc);
-	nv50_crtc_set_image(nv_crtc, fb, x, y, true);
+	if (asyh->clr.mask || asyh->set.mask)
+		nv50_atom(asyh->state.state)->lock_core = true;
 	return 0;
 }
 
 static void
-nv50_crtc_lut_load(struct drm_crtc *crtc)
+nv50_head_lut_load(struct drm_crtc *crtc)
 {
 	struct nv50_disp *disp = nv50_disp(crtc->dev);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
@@ -1292,64 +2205,95 @@ nv50_crtc_lut_load(struct drm_crtc *crtc)
 	}
 }
 
-static void
-nv50_crtc_disable(struct drm_crtc *crtc)
+static int
+nv50_head_mode_set_base_atomic(struct drm_crtc *crtc,
+			       struct drm_framebuffer *fb, int x, int y,
+			       enum mode_set_atomic state)
 {
-	struct nv50_head *head = nv50_head(crtc);
-	evo_sync(crtc->dev);
-	if (head->image)
-		nouveau_bo_unpin(head->image);
-	nouveau_bo_ref(NULL, &head->image);
+	WARN_ON(1);
+	return 0;
 }
 
+static const struct drm_crtc_helper_funcs
+nv50_head_help = {
+	.mode_set_base_atomic = nv50_head_mode_set_base_atomic,
+	.load_lut = nv50_head_lut_load,
+	.atomic_check = nv50_head_atomic_check,
+};
+
+/* This is identical to the version in the atomic helpers, except that
+ * it supports non-vblanked ("async") page flips.
+ */
 static int
-nv50_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
-		     uint32_t handle, uint32_t width, uint32_t height)
+nv50_head_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+		    struct drm_pending_vblank_event *event, u32 flags)
 {
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct drm_gem_object *gem = NULL;
-	struct nouveau_bo *nvbo = NULL;
+	struct drm_plane *plane = crtc->primary;
+	struct drm_atomic_state *state;
+	struct drm_plane_state *plane_state;
+	struct drm_crtc_state *crtc_state;
 	int ret = 0;
 
-	if (handle) {
-		if (width != 64 || height != 64)
-			return -EINVAL;
+	state = drm_atomic_state_alloc(plane->dev);
+	if (!state)
+		return -ENOMEM;
 
-		gem = drm_gem_object_lookup(file_priv, handle);
-		if (unlikely(!gem))
-			return -ENOENT;
-		nvbo = nouveau_gem_object(gem);
+	state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc);
+retry:
+	crtc_state = drm_atomic_get_crtc_state(state, crtc);
+	if (IS_ERR(crtc_state)) {
+		ret = PTR_ERR(crtc_state);
+		goto fail;
+	}
+	crtc_state->event = event;
 
-		ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_VRAM, true);
+	plane_state = drm_atomic_get_plane_state(state, plane);
+	if (IS_ERR(plane_state)) {
+		ret = PTR_ERR(plane_state);
+		goto fail;
 	}
 
-	if (ret == 0) {
-		if (nv_crtc->cursor.nvbo)
-			nouveau_bo_unpin(nv_crtc->cursor.nvbo);
-		nouveau_bo_ref(nvbo, &nv_crtc->cursor.nvbo);
+	ret = drm_atomic_set_crtc_for_plane(plane_state, crtc);
+	if (ret != 0)
+		goto fail;
+	drm_atomic_set_fb_for_plane(plane_state, fb);
+
+	/* Make sure we don't accidentally do a full modeset. */
+	state->allow_modeset = false;
+	if (!crtc_state->active) {
+		DRM_DEBUG_ATOMIC("[CRTC:%d] disabled, rejecting legacy flip\n",
+				 crtc->base.id);
+		ret = -EINVAL;
+		goto fail;
 	}
-	drm_gem_object_unreference_unlocked(gem);
 
-	nv50_crtc_cursor_show_hide(nv_crtc, true, true);
+	if (flags & DRM_MODE_PAGE_FLIP_ASYNC)
+		nv50_wndw_atom(plane_state)->interval = 0;
+
+	ret = drm_atomic_nonblocking_commit(state);
+fail:
+	if (ret == -EDEADLK)
+		goto backoff;
+
+	drm_atomic_state_put(state);
 	return ret;
-}
 
-static int
-nv50_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
-{
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct nv50_curs *curs = nv50_curs(crtc);
-	struct nv50_chan *chan = nv50_chan(curs);
-	nvif_wr32(&chan->user, 0x0084, (y << 16) | (x & 0xffff));
-	nvif_wr32(&chan->user, 0x0080, 0x00000000);
+backoff:
+	drm_atomic_state_clear(state);
+	drm_atomic_legacy_backoff(state);
 
-	nv_crtc->cursor_saved_x = x;
-	nv_crtc->cursor_saved_y = y;
-	return 0;
+	/*
+	 * Someone might have exchanged the framebuffer while we dropped locks
+	 * in the backoff code. We need to fix up the fb refcount tracking the
+	 * core does for us.
+	 */
+	plane->old_fb = plane->fb;
+
+	goto retry;
 }
 
 static int
-nv50_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
+nv50_head_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
 		    uint32_t size)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
@@ -1361,47 +2305,71 @@ nv50_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
 		nv_crtc->lut.b[i] = b[i];
 	}
 
-	nv50_crtc_lut_load(crtc);
-
+	nv50_head_lut_load(crtc);
 	return 0;
 }
 
 static void
-nv50_crtc_cursor_restore(struct nouveau_crtc *nv_crtc, int x, int y)
+nv50_head_atomic_destroy_state(struct drm_crtc *crtc,
+			       struct drm_crtc_state *state)
 {
-	nv50_crtc_cursor_move(&nv_crtc->base, x, y);
+	struct nv50_head_atom *asyh = nv50_head_atom(state);
+	__drm_atomic_helper_crtc_destroy_state(&asyh->state);
+	kfree(asyh);
+}
 
-	nv50_crtc_cursor_show_hide(nv_crtc, true, true);
+static struct drm_crtc_state *
+nv50_head_atomic_duplicate_state(struct drm_crtc *crtc)
+{
+	struct nv50_head_atom *armh = nv50_head_atom(crtc->state);
+	struct nv50_head_atom *asyh;
+	if (!(asyh = kmalloc(sizeof(*asyh), GFP_KERNEL)))
+		return NULL;
+	__drm_atomic_helper_crtc_duplicate_state(crtc, &asyh->state);
+	asyh->view = armh->view;
+	asyh->mode = armh->mode;
+	asyh->lut  = armh->lut;
+	asyh->core = armh->core;
+	asyh->curs = armh->curs;
+	asyh->base = armh->base;
+	asyh->ovly = armh->ovly;
+	asyh->dither = armh->dither;
+	asyh->procamp = armh->procamp;
+	asyh->clr.mask = 0;
+	asyh->set.mask = 0;
+	return &asyh->state;
+}
+
+static void
+__drm_atomic_helper_crtc_reset(struct drm_crtc *crtc,
+			       struct drm_crtc_state *state)
+{
+	if (crtc->state)
+		crtc->funcs->atomic_destroy_state(crtc, crtc->state);
+	crtc->state = state;
+	crtc->state->crtc = crtc;
 }
 
 static void
-nv50_crtc_destroy(struct drm_crtc *crtc)
+nv50_head_reset(struct drm_crtc *crtc)
+{
+	struct nv50_head_atom *asyh;
+
+	if (WARN_ON(!(asyh = kzalloc(sizeof(*asyh), GFP_KERNEL))))
+		return;
+
+	__drm_atomic_helper_crtc_reset(crtc, &asyh->state);
+}
+
+static void
+nv50_head_destroy(struct drm_crtc *crtc)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	struct nv50_disp *disp = nv50_disp(crtc->dev);
 	struct nv50_head *head = nv50_head(crtc);
-	struct nv50_fbdma *fbdma;
-
-	list_for_each_entry(fbdma, &disp->fbdma, head) {
-		nvif_object_fini(&fbdma->base[nv_crtc->index]);
-	}
 
 	nv50_dmac_destroy(&head->ovly.base, disp->disp);
 	nv50_pioc_destroy(&head->oimm.base);
-	nv50_dmac_destroy(&head->sync.base, disp->disp);
-	nv50_pioc_destroy(&head->curs.base);
-
-	/*XXX: this shouldn't be necessary, but the core doesn't call
-	 *     disconnect() during the cleanup paths
-	 */
-	if (head->image)
-		nouveau_bo_unpin(head->image);
-	nouveau_bo_ref(NULL, &head->image);
-
-	/*XXX: ditto */
-	if (nv_crtc->cursor.nvbo)
-		nouveau_bo_unpin(nv_crtc->cursor.nvbo);
-	nouveau_bo_ref(NULL, &nv_crtc->cursor.nvbo);
 
 	nouveau_bo_unmap(nv_crtc->lut.nvbo);
 	if (nv_crtc->lut.nvbo)
@@ -1412,34 +2380,27 @@ nv50_crtc_destroy(struct drm_crtc *crtc)
 	kfree(crtc);
 }
 
-static const struct drm_crtc_helper_funcs nv50_crtc_hfunc = {
-	.dpms = nv50_crtc_dpms,
-	.prepare = nv50_crtc_prepare,
-	.commit = nv50_crtc_commit,
-	.mode_fixup = nv50_crtc_mode_fixup,
-	.mode_set = nv50_crtc_mode_set,
-	.mode_set_base = nv50_crtc_mode_set_base,
-	.mode_set_base_atomic = nv50_crtc_mode_set_base_atomic,
-	.load_lut = nv50_crtc_lut_load,
-	.disable = nv50_crtc_disable,
-};
-
-static const struct drm_crtc_funcs nv50_crtc_func = {
-	.cursor_set = nv50_crtc_cursor_set,
-	.cursor_move = nv50_crtc_cursor_move,
-	.gamma_set = nv50_crtc_gamma_set,
-	.set_config = nouveau_crtc_set_config,
-	.destroy = nv50_crtc_destroy,
-	.page_flip = nouveau_crtc_page_flip,
+static const struct drm_crtc_funcs
+nv50_head_func = {
+	.reset = nv50_head_reset,
+	.gamma_set = nv50_head_gamma_set,
+	.destroy = nv50_head_destroy,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = nv50_head_page_flip,
+	.set_property = drm_atomic_helper_crtc_set_property,
+	.atomic_duplicate_state = nv50_head_atomic_duplicate_state,
+	.atomic_destroy_state = nv50_head_atomic_destroy_state,
 };
 
 static int
-nv50_crtc_create(struct drm_device *dev, int index)
+nv50_head_create(struct drm_device *dev, int index)
 {
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nvif_device *device = &drm->device;
 	struct nv50_disp *disp = nv50_disp(dev);
 	struct nv50_head *head;
+	struct nv50_base *base;
+	struct nv50_curs *curs;
 	struct drm_crtc *crtc;
 	int ret, i;
 
@@ -1448,21 +2409,25 @@ nv50_crtc_create(struct drm_device *dev, int index)
 		return -ENOMEM;
 
 	head->base.index = index;
-	head->base.set_dither = nv50_crtc_set_dither;
-	head->base.set_scale = nv50_crtc_set_scale;
-	head->base.set_color_vibrance = nv50_crtc_set_color_vibrance;
-	head->base.color_vibrance = 50;
-	head->base.vibrant_hue = 0;
-	head->base.cursor.set_pos = nv50_crtc_cursor_restore;
 	for (i = 0; i < 256; i++) {
 		head->base.lut.r[i] = i << 8;
 		head->base.lut.g[i] = i << 8;
 		head->base.lut.b[i] = i << 8;
 	}
 
+	ret = nv50_base_new(drm, head, &base);
+	if (ret == 0)
+		ret = nv50_curs_new(drm, head, &curs);
+	if (ret) {
+		kfree(head);
+		return ret;
+	}
+
 	crtc = &head->base.base;
-	drm_crtc_init(dev, crtc, &nv50_crtc_func);
-	drm_crtc_helper_add(crtc, &nv50_crtc_hfunc);
+	drm_crtc_init_with_planes(dev, crtc, &base->wndw.plane,
+				  &curs->wndw.plane, &nv50_head_func,
+				  "head-%d", head->base.index);
+	drm_crtc_helper_add(crtc, &nv50_head_help);
 	drm_mode_crtc_set_gamma_size(crtc, 256);
 
 	ret = nouveau_bo_new(dev, 8192, 0x100, TTM_PL_FLAG_VRAM,
@@ -1481,20 +2446,6 @@ nv50_crtc_create(struct drm_device *dev, int index)
 	if (ret)
 		goto out;
 
-	/* allocate cursor resources */
-	ret = nv50_curs_create(device, disp->disp, index, &head->curs);
-	if (ret)
-		goto out;
-
-	/* allocate page flip / sync resources */
-	ret = nv50_base_create(device, disp->disp, index, disp->sync->bo.offset,
-			       &head->sync);
-	if (ret)
-		goto out;
-
-	head->sync.addr = EVO_FLIP_SEM0(index);
-	head->sync.data = 0x00000000;
-
 	/* allocate overlay resources */
 	ret = nv50_oimm_create(device, disp->disp, index, &head->oimm);
 	if (ret)
@@ -1507,43 +2458,64 @@ nv50_crtc_create(struct drm_device *dev, int index)
 
 out:
 	if (ret)
-		nv50_crtc_destroy(crtc);
+		nv50_head_destroy(crtc);
 	return ret;
 }
 
 /******************************************************************************
- * Encoder helpers
+ * Output path helpers
  *****************************************************************************/
-static bool
-nv50_encoder_mode_fixup(struct drm_encoder *encoder,
-			const struct drm_display_mode *mode,
-			struct drm_display_mode *adjusted_mode)
+static int
+nv50_outp_atomic_check_view(struct drm_encoder *encoder,
+			    struct drm_crtc_state *crtc_state,
+			    struct drm_connector_state *conn_state,
+			    struct drm_display_mode *native_mode)
 {
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nouveau_connector *nv_connector;
+	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
+	struct drm_display_mode *mode = &crtc_state->mode;
+	struct drm_connector *connector = conn_state->connector;
+	struct nouveau_conn_atom *asyc = nouveau_conn_atom(conn_state);
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
+
+	NV_ATOMIC(drm, "%s atomic_check\n", encoder->name);
+	asyc->scaler.full = false;
+	if (!native_mode)
+		return 0;
 
-	nv_connector = nouveau_encoder_connector_get(nv_encoder);
-	if (nv_connector && nv_connector->native_mode) {
-		nv_connector->scaling_full = false;
-		if (nv_connector->scaling_mode == DRM_MODE_SCALE_NONE) {
-			switch (nv_connector->type) {
-			case DCB_CONNECTOR_LVDS:
-			case DCB_CONNECTOR_LVDS_SPWG:
-			case DCB_CONNECTOR_eDP:
-				/* force use of scaler for non-edid modes */
-				if (adjusted_mode->type & DRM_MODE_TYPE_DRIVER)
-					return true;
-				nv_connector->scaling_full = true;
+	if (asyc->scaler.mode == DRM_MODE_SCALE_NONE) {
+		switch (connector->connector_type) {
+		case DRM_MODE_CONNECTOR_LVDS:
+		case DRM_MODE_CONNECTOR_eDP:
+			/* Force use of scaler for non-EDID modes. */
+			if (adjusted_mode->type & DRM_MODE_TYPE_DRIVER)
 				break;
-			default:
-				return true;
-			}
+			mode = native_mode;
+			asyc->scaler.full = true;
+			break;
+		default:
+			break;
 		}
+	} else {
+		mode = native_mode;
+	}
 
-		drm_mode_copy(adjusted_mode, nv_connector->native_mode);
+	if (!drm_mode_equal(adjusted_mode, mode)) {
+		drm_mode_copy(adjusted_mode, mode);
+		crtc_state->mode_changed = true;
 	}
 
-	return true;
+	return 0;
+}
+
+static int
+nv50_outp_atomic_check(struct drm_encoder *encoder,
+		       struct drm_crtc_state *crtc_state,
+		       struct drm_connector_state *conn_state)
+{
+	struct nouveau_connector *nv_connector =
+		nouveau_connector(conn_state->connector);
+	return nv50_outp_atomic_check_view(encoder, crtc_state, conn_state,
+					   nv_connector->native_mode);
 }
 
 /******************************************************************************
@@ -1574,21 +2546,39 @@ nv50_dac_dpms(struct drm_encoder *encoder, int mode)
 }
 
 static void
-nv50_dac_commit(struct drm_encoder *encoder)
+nv50_dac_disable(struct drm_encoder *encoder)
 {
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nv50_mast *mast = nv50_mast(encoder->dev);
+	const int or = nv_encoder->or;
+	u32 *push;
+
+	if (nv_encoder->crtc) {
+		push = evo_wait(mast, 4);
+		if (push) {
+			if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
+				evo_mthd(push, 0x0400 + (or * 0x080), 1);
+				evo_data(push, 0x00000000);
+			} else {
+				evo_mthd(push, 0x0180 + (or * 0x020), 1);
+				evo_data(push, 0x00000000);
+			}
+			evo_kick(push, mast);
+		}
+	}
+
+	nv_encoder->crtc = NULL;
 }
 
 static void
-nv50_dac_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
-		  struct drm_display_mode *adjusted_mode)
+nv50_dac_enable(struct drm_encoder *encoder)
 {
 	struct nv50_mast *mast = nv50_mast(encoder->dev);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
+	struct drm_display_mode *mode = &nv_crtc->base.state->adjusted_mode;
 	u32 *push;
 
-	nv50_dac_dpms(encoder, DRM_MODE_DPMS_ON);
-
 	push = evo_wait(mast, 8);
 	if (push) {
 		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
@@ -1627,33 +2617,6 @@ nv50_dac_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 	nv_encoder->crtc = encoder->crtc;
 }
 
-static void
-nv50_dac_disconnect(struct drm_encoder *encoder)
-{
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nv50_mast *mast = nv50_mast(encoder->dev);
-	const int or = nv_encoder->or;
-	u32 *push;
-
-	if (nv_encoder->crtc) {
-		nv50_crtc_prepare(nv_encoder->crtc);
-
-		push = evo_wait(mast, 4);
-		if (push) {
-			if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-				evo_mthd(push, 0x0400 + (or * 0x080), 1);
-				evo_data(push, 0x00000000);
-			} else {
-				evo_mthd(push, 0x0180 + (or * 0x020), 1);
-				evo_data(push, 0x00000000);
-			}
-			evo_kick(push, mast);
-		}
-	}
-
-	nv_encoder->crtc = NULL;
-}
-
 static enum drm_connector_status
 nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
 {
@@ -1681,6 +2644,15 @@ nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
 	return connector_status_connected;
 }
 
+static const struct drm_encoder_helper_funcs
+nv50_dac_help = {
+	.dpms = nv50_dac_dpms,
+	.atomic_check = nv50_outp_atomic_check,
+	.enable = nv50_dac_enable,
+	.disable = nv50_dac_disable,
+	.detect = nv50_dac_detect
+};
+
 static void
 nv50_dac_destroy(struct drm_encoder *encoder)
 {
@@ -1688,18 +2660,8 @@ nv50_dac_destroy(struct drm_encoder *encoder)
 	kfree(encoder);
 }
 
-static const struct drm_encoder_helper_funcs nv50_dac_hfunc = {
-	.dpms = nv50_dac_dpms,
-	.mode_fixup = nv50_encoder_mode_fixup,
-	.prepare = nv50_dac_disconnect,
-	.commit = nv50_dac_commit,
-	.mode_set = nv50_dac_mode_set,
-	.disable = nv50_dac_disconnect,
-	.get_crtc = nv50_display_crtc_get,
-	.detect = nv50_dac_detect
-};
-
-static const struct drm_encoder_funcs nv50_dac_func = {
+static const struct drm_encoder_funcs
+nv50_dac_func = {
 	.destroy = nv50_dac_destroy,
 };
 
@@ -1726,8 +2688,9 @@ nv50_dac_create(struct drm_connector *connector, struct dcb_output *dcbe)
 	encoder = to_drm_encoder(nv_encoder);
 	encoder->possible_crtcs = dcbe->heads;
 	encoder->possible_clones = 0;
-	drm_encoder_init(connector->dev, encoder, &nv50_dac_func, type, NULL);
-	drm_encoder_helper_add(encoder, &nv50_dac_hfunc);
+	drm_encoder_init(connector->dev, encoder, &nv50_dac_func, type,
+			 "dac-%04x-%04x", dcbe->hasht, dcbe->hashm);
+	drm_encoder_helper_add(encoder, &nv50_dac_help);
 
 	drm_mode_connector_attach_encoder(connector, encoder);
 	return 0;
@@ -1737,7 +2700,26 @@ nv50_dac_create(struct drm_connector *connector, struct dcb_output *dcbe)
  * Audio
  *****************************************************************************/
 static void
-nv50_audio_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode)
+nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
+{
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nv50_disp *disp = nv50_disp(encoder->dev);
+	struct {
+		struct nv50_disp_mthd_v1 base;
+		struct nv50_disp_sor_hda_eld_v0 eld;
+	} args = {
+		.base.version = 1,
+		.base.method  = NV50_DISP_MTHD_V1_SOR_HDA_ELD,
+		.base.hasht   = nv_encoder->dcb->hasht,
+		.base.hashm   = (0xf0ff & nv_encoder->dcb->hashm) |
+				(0x0100 << nv_crtc->index),
+	};
+
+	nvif_mthd(disp->disp, 0, &args, sizeof(args));
+}
+
+static void
+nv50_audio_enable(struct drm_encoder *encoder, struct drm_display_mode *mode)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
@@ -1768,30 +2750,30 @@ nv50_audio_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode)
 		  sizeof(args.base) + drm_eld_size(args.data));
 }
 
+/******************************************************************************
+ * HDMI
+ *****************************************************************************/
 static void
-nv50_audio_disconnect(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
+nv50_hdmi_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nv50_disp *disp = nv50_disp(encoder->dev);
 	struct {
 		struct nv50_disp_mthd_v1 base;
-		struct nv50_disp_sor_hda_eld_v0 eld;
+		struct nv50_disp_sor_hdmi_pwr_v0 pwr;
 	} args = {
 		.base.version = 1,
-		.base.method  = NV50_DISP_MTHD_V1_SOR_HDA_ELD,
-		.base.hasht   = nv_encoder->dcb->hasht,
-		.base.hashm   = (0xf0ff & nv_encoder->dcb->hashm) |
-				(0x0100 << nv_crtc->index),
+		.base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR,
+		.base.hasht  = nv_encoder->dcb->hasht,
+		.base.hashm  = (0xf0ff & nv_encoder->dcb->hashm) |
+			       (0x0100 << nv_crtc->index),
 	};
 
 	nvif_mthd(disp->disp, 0, &args, sizeof(args));
 }
 
-/******************************************************************************
- * HDMI
- *****************************************************************************/
 static void
-nv50_hdmi_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode)
+nv50_hdmi_enable(struct drm_encoder *encoder, struct drm_display_mode *mode)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
@@ -1821,26 +2803,635 @@ nv50_hdmi_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode)
 	args.pwr.max_ac_packet = max_ac_packet / 32;
 
 	nvif_mthd(disp->disp, 0, &args, sizeof(args));
-	nv50_audio_mode_set(encoder, mode);
+	nv50_audio_enable(encoder, mode);
+}
+
+/******************************************************************************
+ * MST
+ *****************************************************************************/
+#define nv50_mstm(p) container_of((p), struct nv50_mstm, mgr)
+#define nv50_mstc(p) container_of((p), struct nv50_mstc, connector)
+#define nv50_msto(p) container_of((p), struct nv50_msto, encoder)
+
+struct nv50_mstm {
+	struct nouveau_encoder *outp;
+
+	struct drm_dp_mst_topology_mgr mgr;
+	struct nv50_msto *msto[4];
+
+	bool modified;
+};
+
+struct nv50_mstc {
+	struct nv50_mstm *mstm;
+	struct drm_dp_mst_port *port;
+	struct drm_connector connector;
+
+	struct drm_display_mode *native;
+	struct edid *edid;
+
+	int pbn;
+};
+
+struct nv50_msto {
+	struct drm_encoder encoder;
+
+	struct nv50_head *head;
+	struct nv50_mstc *mstc;
+	bool disabled;
+};
+
+static struct drm_dp_payload *
+nv50_msto_payload(struct nv50_msto *msto)
+{
+	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
+	struct nv50_mstc *mstc = msto->mstc;
+	struct nv50_mstm *mstm = mstc->mstm;
+	int vcpi = mstc->port->vcpi.vcpi, i;
+
+	NV_ATOMIC(drm, "%s: vcpi %d\n", msto->encoder.name, vcpi);
+	for (i = 0; i < mstm->mgr.max_payloads; i++) {
+		struct drm_dp_payload *payload = &mstm->mgr.payloads[i];
+		NV_ATOMIC(drm, "%s: %d: vcpi %d start 0x%02x slots 0x%02x\n",
+			  mstm->outp->base.base.name, i, payload->vcpi,
+			  payload->start_slot, payload->num_slots);
+	}
+
+	for (i = 0; i < mstm->mgr.max_payloads; i++) {
+		struct drm_dp_payload *payload = &mstm->mgr.payloads[i];
+		if (payload->vcpi == vcpi)
+			return payload;
+	}
+
+	return NULL;
 }
 
 static void
-nv50_hdmi_disconnect(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
+nv50_msto_cleanup(struct nv50_msto *msto)
 {
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nv50_disp *disp = nv50_disp(encoder->dev);
+	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
+	struct nv50_mstc *mstc = msto->mstc;
+	struct nv50_mstm *mstm = mstc->mstm;
+
+	NV_ATOMIC(drm, "%s: msto cleanup\n", msto->encoder.name);
+	if (mstc->port && mstc->port->vcpi.vcpi > 0 && !nv50_msto_payload(msto))
+		drm_dp_mst_deallocate_vcpi(&mstm->mgr, mstc->port);
+	if (msto->disabled) {
+		msto->mstc = NULL;
+		msto->head = NULL;
+		msto->disabled = false;
+	}
+}
+
+static void
+nv50_msto_prepare(struct nv50_msto *msto)
+{
+	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
+	struct nv50_mstc *mstc = msto->mstc;
+	struct nv50_mstm *mstm = mstc->mstm;
 	struct {
 		struct nv50_disp_mthd_v1 base;
-		struct nv50_disp_sor_hdmi_pwr_v0 pwr;
+		struct nv50_disp_sor_dp_mst_vcpi_v0 vcpi;
 	} args = {
 		.base.version = 1,
-		.base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR,
-		.base.hasht  = nv_encoder->dcb->hasht,
-		.base.hashm  = (0xf0ff & nv_encoder->dcb->hashm) |
-			       (0x0100 << nv_crtc->index),
+		.base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI,
+		.base.hasht  = mstm->outp->dcb->hasht,
+		.base.hashm  = (0xf0ff & mstm->outp->dcb->hashm) |
+			       (0x0100 << msto->head->base.index),
 	};
 
-	nvif_mthd(disp->disp, 0, &args, sizeof(args));
+	NV_ATOMIC(drm, "%s: msto prepare\n", msto->encoder.name);
+	if (mstc->port && mstc->port->vcpi.vcpi > 0) {
+		struct drm_dp_payload *payload = nv50_msto_payload(msto);
+		if (payload) {
+			args.vcpi.start_slot = payload->start_slot;
+			args.vcpi.num_slots = payload->num_slots;
+			args.vcpi.pbn = mstc->port->vcpi.pbn;
+			args.vcpi.aligned_pbn = mstc->port->vcpi.aligned_pbn;
+		}
+	}
+
+	NV_ATOMIC(drm, "%s: %s: %02x %02x %04x %04x\n",
+		  msto->encoder.name, msto->head->base.base.name,
+		  args.vcpi.start_slot, args.vcpi.num_slots,
+		  args.vcpi.pbn, args.vcpi.aligned_pbn);
+	nvif_mthd(&drm->display->disp, 0, &args, sizeof(args));
+}
+
+static int
+nv50_msto_atomic_check(struct drm_encoder *encoder,
+		       struct drm_crtc_state *crtc_state,
+		       struct drm_connector_state *conn_state)
+{
+	struct nv50_mstc *mstc = nv50_mstc(conn_state->connector);
+	struct nv50_mstm *mstm = mstc->mstm;
+	int bpp = conn_state->connector->display_info.bpc * 3;
+	int slots;
+
+	mstc->pbn = drm_dp_calc_pbn_mode(crtc_state->adjusted_mode.clock, bpp);
+
+	slots = drm_dp_find_vcpi_slots(&mstm->mgr, mstc->pbn);
+	if (slots < 0)
+		return slots;
+
+	return nv50_outp_atomic_check_view(encoder, crtc_state, conn_state,
+					   mstc->native);
+}
+
+static void
+nv50_msto_enable(struct drm_encoder *encoder)
+{
+	struct nv50_head *head = nv50_head(encoder->crtc);
+	struct nv50_msto *msto = nv50_msto(encoder);
+	struct nv50_mstc *mstc = NULL;
+	struct nv50_mstm *mstm = NULL;
+	struct drm_connector *connector;
+	u8 proto, depth;
+	int slots;
+	bool r;
+
+	drm_for_each_connector(connector, encoder->dev) {
+		if (connector->state->best_encoder == &msto->encoder) {
+			mstc = nv50_mstc(connector);
+			mstm = mstc->mstm;
+			break;
+		}
+	}
+
+	if (WARN_ON(!mstc))
+		return;
+
+	r = drm_dp_mst_allocate_vcpi(&mstm->mgr, mstc->port, mstc->pbn, &slots);
+	WARN_ON(!r);
+
+	if (mstm->outp->dcb->sorconf.link & 1)
+		proto = 0x8;
+	else
+		proto = 0x9;
+
+	switch (mstc->connector.display_info.bpc) {
+	case  6: depth = 0x2; break;
+	case  8: depth = 0x5; break;
+	case 10:
+	default: depth = 0x6; break;
+	}
+
+	mstm->outp->update(mstm->outp, head->base.index,
+			   &head->base.base.state->adjusted_mode, proto, depth);
+
+	msto->head = head;
+	msto->mstc = mstc;
+	mstm->modified = true;
+}
+
+static void
+nv50_msto_disable(struct drm_encoder *encoder)
+{
+	struct nv50_msto *msto = nv50_msto(encoder);
+	struct nv50_mstc *mstc = msto->mstc;
+	struct nv50_mstm *mstm = mstc->mstm;
+
+	if (mstc->port)
+		drm_dp_mst_reset_vcpi_slots(&mstm->mgr, mstc->port);
+
+	mstm->outp->update(mstm->outp, msto->head->base.index, NULL, 0, 0);
+	mstm->modified = true;
+	msto->disabled = true;
+}
+
+static const struct drm_encoder_helper_funcs
+nv50_msto_help = {
+	.disable = nv50_msto_disable,
+	.enable = nv50_msto_enable,
+	.atomic_check = nv50_msto_atomic_check,
+};
+
+static void
+nv50_msto_destroy(struct drm_encoder *encoder)
+{
+	struct nv50_msto *msto = nv50_msto(encoder);
+	drm_encoder_cleanup(&msto->encoder);
+	kfree(msto);
+}
+
+static const struct drm_encoder_funcs
+nv50_msto = {
+	.destroy = nv50_msto_destroy,
+};
+
+static int
+nv50_msto_new(struct drm_device *dev, u32 heads, const char *name, int id,
+	      struct nv50_msto **pmsto)
+{
+	struct nv50_msto *msto;
+	int ret;
+
+	if (!(msto = *pmsto = kzalloc(sizeof(*msto), GFP_KERNEL)))
+		return -ENOMEM;
+
+	ret = drm_encoder_init(dev, &msto->encoder, &nv50_msto,
+			       DRM_MODE_ENCODER_DPMST, "%s-mst-%d", name, id);
+	if (ret) {
+		kfree(*pmsto);
+		*pmsto = NULL;
+		return ret;
+	}
+
+	drm_encoder_helper_add(&msto->encoder, &nv50_msto_help);
+	msto->encoder.possible_crtcs = heads;
+	return 0;
+}
+
+static struct drm_encoder *
+nv50_mstc_atomic_best_encoder(struct drm_connector *connector,
+			      struct drm_connector_state *connector_state)
+{
+	struct nv50_head *head = nv50_head(connector_state->crtc);
+	struct nv50_mstc *mstc = nv50_mstc(connector);
+	if (mstc->port) {
+		struct nv50_mstm *mstm = mstc->mstm;
+		return &mstm->msto[head->base.index]->encoder;
+	}
+	return NULL;
+}
+
+static struct drm_encoder *
+nv50_mstc_best_encoder(struct drm_connector *connector)
+{
+	struct nv50_mstc *mstc = nv50_mstc(connector);
+	if (mstc->port) {
+		struct nv50_mstm *mstm = mstc->mstm;
+		return &mstm->msto[0]->encoder;
+	}
+	return NULL;
+}
+
+static enum drm_mode_status
+nv50_mstc_mode_valid(struct drm_connector *connector,
+		     struct drm_display_mode *mode)
+{
+	return MODE_OK;
+}
+
+static int
+nv50_mstc_get_modes(struct drm_connector *connector)
+{
+	struct nv50_mstc *mstc = nv50_mstc(connector);
+	int ret = 0;
+
+	mstc->edid = drm_dp_mst_get_edid(&mstc->connector, mstc->port->mgr, mstc->port);
+	drm_mode_connector_update_edid_property(&mstc->connector, mstc->edid);
+	if (mstc->edid) {
+		ret = drm_add_edid_modes(&mstc->connector, mstc->edid);
+		drm_edid_to_eld(&mstc->connector, mstc->edid);
+	}
+
+	if (!mstc->connector.display_info.bpc)
+		mstc->connector.display_info.bpc = 8;
+
+	if (mstc->native)
+		drm_mode_destroy(mstc->connector.dev, mstc->native);
+	mstc->native = nouveau_conn_native_mode(&mstc->connector);
+	return ret;
+}
+
+static const struct drm_connector_helper_funcs
+nv50_mstc_help = {
+	.get_modes = nv50_mstc_get_modes,
+	.mode_valid = nv50_mstc_mode_valid,
+	.best_encoder = nv50_mstc_best_encoder,
+	.atomic_best_encoder = nv50_mstc_atomic_best_encoder,
+};
+
+static enum drm_connector_status
+nv50_mstc_detect(struct drm_connector *connector, bool force)
+{
+	struct nv50_mstc *mstc = nv50_mstc(connector);
+	if (!mstc->port)
+		return connector_status_disconnected;
+	return drm_dp_mst_detect_port(connector, mstc->port->mgr, mstc->port);
+}
+
+static void
+nv50_mstc_destroy(struct drm_connector *connector)
+{
+	struct nv50_mstc *mstc = nv50_mstc(connector);
+	drm_connector_cleanup(&mstc->connector);
+	kfree(mstc);
+}
+
+static const struct drm_connector_funcs
+nv50_mstc = {
+	.dpms = drm_atomic_helper_connector_dpms,
+	.reset = nouveau_conn_reset,
+	.detect = nv50_mstc_detect,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.set_property = drm_atomic_helper_connector_set_property,
+	.destroy = nv50_mstc_destroy,
+	.atomic_duplicate_state = nouveau_conn_atomic_duplicate_state,
+	.atomic_destroy_state = nouveau_conn_atomic_destroy_state,
+	.atomic_set_property = nouveau_conn_atomic_set_property,
+	.atomic_get_property = nouveau_conn_atomic_get_property,
+};
+
+static int
+nv50_mstc_new(struct nv50_mstm *mstm, struct drm_dp_mst_port *port,
+	      const char *path, struct nv50_mstc **pmstc)
+{
+	struct drm_device *dev = mstm->outp->base.base.dev;
+	struct nv50_mstc *mstc;
+	int ret, i;
+
+	if (!(mstc = *pmstc = kzalloc(sizeof(*mstc), GFP_KERNEL)))
+		return -ENOMEM;
+	mstc->mstm = mstm;
+	mstc->port = port;
+
+	ret = drm_connector_init(dev, &mstc->connector, &nv50_mstc,
+				 DRM_MODE_CONNECTOR_DisplayPort);
+	if (ret) {
+		kfree(*pmstc);
+		*pmstc = NULL;
+		return ret;
+	}
+
+	drm_connector_helper_add(&mstc->connector, &nv50_mstc_help);
+
+	mstc->connector.funcs->reset(&mstc->connector);
+	nouveau_conn_attach_properties(&mstc->connector);
+
+	for (i = 0; i < ARRAY_SIZE(mstm->msto) && mstm->msto; i++)
+		drm_mode_connector_attach_encoder(&mstc->connector, &mstm->msto[i]->encoder);
+
+	drm_object_attach_property(&mstc->connector.base, dev->mode_config.path_property, 0);
+	drm_object_attach_property(&mstc->connector.base, dev->mode_config.tile_property, 0);
+	drm_mode_connector_set_path_property(&mstc->connector, path);
+	return 0;
+}
+
+static void
+nv50_mstm_cleanup(struct nv50_mstm *mstm)
+{
+	struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev);
+	struct drm_encoder *encoder;
+	int ret;
+
+	NV_ATOMIC(drm, "%s: mstm cleanup\n", mstm->outp->base.base.name);
+	ret = drm_dp_check_act_status(&mstm->mgr);
+
+	ret = drm_dp_update_payload_part2(&mstm->mgr);
+
+	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
+		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
+			struct nv50_msto *msto = nv50_msto(encoder);
+			struct nv50_mstc *mstc = msto->mstc;
+			if (mstc && mstc->mstm == mstm)
+				nv50_msto_cleanup(msto);
+		}
+	}
+
+	mstm->modified = false;
+}
+
+static void
+nv50_mstm_prepare(struct nv50_mstm *mstm)
+{
+	struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev);
+	struct drm_encoder *encoder;
+	int ret;
+
+	NV_ATOMIC(drm, "%s: mstm prepare\n", mstm->outp->base.base.name);
+	ret = drm_dp_update_payload_part1(&mstm->mgr);
+
+	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
+		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
+			struct nv50_msto *msto = nv50_msto(encoder);
+			struct nv50_mstc *mstc = msto->mstc;
+			if (mstc && mstc->mstm == mstm)
+				nv50_msto_prepare(msto);
+		}
+	}
+}
+
+static void
+nv50_mstm_hotplug(struct drm_dp_mst_topology_mgr *mgr)
+{
+	struct nv50_mstm *mstm = nv50_mstm(mgr);
+	drm_kms_helper_hotplug_event(mstm->outp->base.base.dev);
+}
+
+static void
+nv50_mstm_destroy_connector(struct drm_dp_mst_topology_mgr *mgr,
+			    struct drm_connector *connector)
+{
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
+	struct nv50_mstc *mstc = nv50_mstc(connector);
+
+	drm_connector_unregister(&mstc->connector);
+
+	drm_modeset_lock_all(drm->dev);
+	drm_fb_helper_remove_one_connector(&drm->fbcon->helper, &mstc->connector);
+	mstc->port = NULL;
+	drm_modeset_unlock_all(drm->dev);
+
+	drm_connector_unreference(&mstc->connector);
+}
+
+static void
+nv50_mstm_register_connector(struct drm_connector *connector)
+{
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
+
+	drm_modeset_lock_all(drm->dev);
+	drm_fb_helper_add_one_connector(&drm->fbcon->helper, connector);
+	drm_modeset_unlock_all(drm->dev);
+
+	drm_connector_register(connector);
+}
+
+static struct drm_connector *
+nv50_mstm_add_connector(struct drm_dp_mst_topology_mgr *mgr,
+			struct drm_dp_mst_port *port, const char *path)
+{
+	struct nv50_mstm *mstm = nv50_mstm(mgr);
+	struct nv50_mstc *mstc;
+	int ret;
+
+	ret = nv50_mstc_new(mstm, port, path, &mstc);
+	if (ret) {
+		if (mstc)
+			mstc->connector.funcs->destroy(&mstc->connector);
+		return NULL;
+	}
+
+	return &mstc->connector;
+}
+
+static const struct drm_dp_mst_topology_cbs
+nv50_mstm = {
+	.add_connector = nv50_mstm_add_connector,
+	.register_connector = nv50_mstm_register_connector,
+	.destroy_connector = nv50_mstm_destroy_connector,
+	.hotplug = nv50_mstm_hotplug,
+};
+
+void
+nv50_mstm_service(struct nv50_mstm *mstm)
+{
+	struct drm_dp_aux *aux = mstm->mgr.aux;
+	bool handled = true;
+	int ret;
+	u8 esi[8] = {};
+
+	while (handled) {
+		ret = drm_dp_dpcd_read(aux, DP_SINK_COUNT_ESI, esi, 8);
+		if (ret != 8) {
+			drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false);
+			return;
+		}
+
+		drm_dp_mst_hpd_irq(&mstm->mgr, esi, &handled);
+		if (!handled)
+			break;
+
+		drm_dp_dpcd_write(aux, DP_SINK_COUNT_ESI + 1, &esi[1], 3);
+	}
+}
+
+void
+nv50_mstm_remove(struct nv50_mstm *mstm)
+{
+	if (mstm)
+		drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false);
+}
+
+static int
+nv50_mstm_enable(struct nv50_mstm *mstm, u8 dpcd, int state)
+{
+	struct nouveau_encoder *outp = mstm->outp;
+	struct {
+		struct nv50_disp_mthd_v1 base;
+		struct nv50_disp_sor_dp_mst_link_v0 mst;
+	} args = {
+		.base.version = 1,
+		.base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_LINK,
+		.base.hasht = outp->dcb->hasht,
+		.base.hashm = outp->dcb->hashm,
+		.mst.state = state,
+	};
+	struct nouveau_drm *drm = nouveau_drm(outp->base.base.dev);
+	struct nvif_object *disp = &drm->display->disp;
+	int ret;
+
+	if (dpcd >= 0x12) {
+		ret = drm_dp_dpcd_readb(mstm->mgr.aux, DP_MSTM_CTRL, &dpcd);
+		if (ret < 0)
+			return ret;
+
+		dpcd &= ~DP_MST_EN;
+		if (state)
+			dpcd |= DP_MST_EN;
+
+		ret = drm_dp_dpcd_writeb(mstm->mgr.aux, DP_MSTM_CTRL, dpcd);
+		if (ret < 0)
+			return ret;
+	}
+
+	return nvif_mthd(disp, 0, &args, sizeof(args));
+}
+
+int
+nv50_mstm_detect(struct nv50_mstm *mstm, u8 dpcd[8], int allow)
+{
+	int ret, state = 0;
+
+	if (!mstm)
+		return 0;
+
+	if (dpcd[0] >= 0x12) {
+		ret = drm_dp_dpcd_readb(mstm->mgr.aux, DP_MSTM_CAP, &dpcd[1]);
+		if (ret < 0)
+			return ret;
+
+		if (!(dpcd[1] & DP_MST_CAP))
+			dpcd[0] = 0x11;
+		else
+			state = allow;
+	}
+
+	ret = nv50_mstm_enable(mstm, dpcd[0], state);
+	if (ret)
+		return ret;
+
+	ret = drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, state);
+	if (ret)
+		return nv50_mstm_enable(mstm, dpcd[0], 0);
+
+	return mstm->mgr.mst_state;
+}
+
+static void
+nv50_mstm_fini(struct nv50_mstm *mstm)
+{
+	if (mstm && mstm->mgr.mst_state)
+		drm_dp_mst_topology_mgr_suspend(&mstm->mgr);
+}
+
+static void
+nv50_mstm_init(struct nv50_mstm *mstm)
+{
+	if (mstm && mstm->mgr.mst_state)
+		drm_dp_mst_topology_mgr_resume(&mstm->mgr);
+}
+
+static void
+nv50_mstm_del(struct nv50_mstm **pmstm)
+{
+	struct nv50_mstm *mstm = *pmstm;
+	if (mstm) {
+		kfree(*pmstm);
+		*pmstm = NULL;
+	}
+}
+
+static int
+nv50_mstm_new(struct nouveau_encoder *outp, struct drm_dp_aux *aux, int aux_max,
+	      int conn_base_id, struct nv50_mstm **pmstm)
+{
+	const int max_payloads = hweight8(outp->dcb->heads);
+	struct drm_device *dev = outp->base.base.dev;
+	struct nv50_mstm *mstm;
+	int ret, i;
+	u8 dpcd;
+
+	/* This is a workaround for some monitors not functioning
+	 * correctly in MST mode on initial module load.  I think
+	 * some bad interaction with the VBIOS may be responsible.
+	 *
+	 * A good ol' off and on again seems to work here ;)
+	 */
+	ret = drm_dp_dpcd_readb(aux, DP_DPCD_REV, &dpcd);
+	if (ret >= 0 && dpcd >= 0x12)
+		drm_dp_dpcd_writeb(aux, DP_MSTM_CTRL, 0);
+
+	if (!(mstm = *pmstm = kzalloc(sizeof(*mstm), GFP_KERNEL)))
+		return -ENOMEM;
+	mstm->outp = outp;
+	mstm->mgr.cbs = &nv50_mstm;
+
+	ret = drm_dp_mst_topology_mgr_init(&mstm->mgr, dev->dev, aux, aux_max,
+					   max_payloads, conn_base_id);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < max_payloads; i++) {
+		ret = nv50_msto_new(dev, outp->dcb->heads, outp->base.base.name,
+				    i, &mstm->msto[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
 }
 
 /******************************************************************************
@@ -1861,89 +3452,91 @@ nv50_sor_dpms(struct drm_encoder *encoder, int mode)
 		.base.hashm  = nv_encoder->dcb->hashm,
 		.pwr.state = mode == DRM_MODE_DPMS_ON,
 	};
-	struct {
-		struct nv50_disp_mthd_v1 base;
-		struct nv50_disp_sor_dp_pwr_v0 pwr;
-	} link = {
-		.base.version = 1,
-		.base.method = NV50_DISP_MTHD_V1_SOR_DP_PWR,
-		.base.hasht  = nv_encoder->dcb->hasht,
-		.base.hashm  = nv_encoder->dcb->hashm,
-		.pwr.state = mode == DRM_MODE_DPMS_ON,
-	};
-	struct drm_device *dev = encoder->dev;
-	struct drm_encoder *partner;
 
-	nv_encoder->last_dpms = mode;
-
-	list_for_each_entry(partner, &dev->mode_config.encoder_list, head) {
-		struct nouveau_encoder *nv_partner = nouveau_encoder(partner);
-
-		if (partner->encoder_type != DRM_MODE_ENCODER_TMDS)
-			continue;
+	nvif_mthd(disp->disp, 0, &args, sizeof(args));
+}
 
-		if (nv_partner != nv_encoder &&
-		    nv_partner->dcb->or == nv_encoder->dcb->or) {
-			if (nv_partner->last_dpms == DRM_MODE_DPMS_ON)
-				return;
-			break;
-		}
-	}
+static void
+nv50_sor_update(struct nouveau_encoder *nv_encoder, u8 head,
+		struct drm_display_mode *mode, u8 proto, u8 depth)
+{
+	struct nv50_dmac *core = &nv50_mast(nv_encoder->base.base.dev)->base;
+	u32 *push;
 
-	if (nv_encoder->dcb->type == DCB_OUTPUT_DP) {
-		args.pwr.state = 1;
-		nvif_mthd(disp->disp, 0, &args, sizeof(args));
-		nvif_mthd(disp->disp, 0, &link, sizeof(link));
+	if (!mode) {
+		nv_encoder->ctrl &= ~BIT(head);
+		if (!(nv_encoder->ctrl & 0x0000000f))
+			nv_encoder->ctrl = 0;
 	} else {
-		nvif_mthd(disp->disp, 0, &args, sizeof(args));
+		nv_encoder->ctrl |= proto << 8;
+		nv_encoder->ctrl |= BIT(head);
 	}
-}
 
-static void
-nv50_sor_ctrl(struct nouveau_encoder *nv_encoder, u32 mask, u32 data)
-{
-	struct nv50_mast *mast = nv50_mast(nv_encoder->base.base.dev);
-	u32 temp = (nv_encoder->ctrl & ~mask) | (data & mask), *push;
-	if (temp != nv_encoder->ctrl && (push = evo_wait(mast, 2))) {
-		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
+	if ((push = evo_wait(core, 6))) {
+		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
+			if (mode) {
+				if (mode->flags & DRM_MODE_FLAG_NHSYNC)
+					nv_encoder->ctrl |= 0x00001000;
+				if (mode->flags & DRM_MODE_FLAG_NVSYNC)
+					nv_encoder->ctrl |= 0x00002000;
+				nv_encoder->ctrl |= depth << 16;
+			}
 			evo_mthd(push, 0x0600 + (nv_encoder->or * 0x40), 1);
-			evo_data(push, (nv_encoder->ctrl = temp));
 		} else {
+			if (mode) {
+				u32 magic = 0x31ec6000 | (head << 25);
+				u32 syncs = 0x00000001;
+				if (mode->flags & DRM_MODE_FLAG_NHSYNC)
+					syncs |= 0x00000008;
+				if (mode->flags & DRM_MODE_FLAG_NVSYNC)
+					syncs |= 0x00000010;
+				if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+					magic |= 0x00000001;
+
+				evo_mthd(push, 0x0404 + (head * 0x300), 2);
+				evo_data(push, syncs | (depth << 6));
+				evo_data(push, magic);
+			}
 			evo_mthd(push, 0x0200 + (nv_encoder->or * 0x20), 1);
-			evo_data(push, (nv_encoder->ctrl = temp));
 		}
-		evo_kick(push, mast);
+		evo_data(push, nv_encoder->ctrl);
+		evo_kick(push, core);
 	}
 }
 
 static void
-nv50_sor_disconnect(struct drm_encoder *encoder)
+nv50_sor_disable(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(nv_encoder->crtc);
 
-	nv_encoder->last_dpms = DRM_MODE_DPMS_OFF;
 	nv_encoder->crtc = NULL;
 
 	if (nv_crtc) {
-		nv50_crtc_prepare(&nv_crtc->base);
-		nv50_sor_ctrl(nv_encoder, 1 << nv_crtc->index, 0);
-		nv50_audio_disconnect(encoder, nv_crtc);
-		nv50_hdmi_disconnect(&nv_encoder->base.base, nv_crtc);
-	}
-}
+		struct nvkm_i2c_aux *aux = nv_encoder->aux;
+		u8 pwr;
 
-static void
-nv50_sor_commit(struct drm_encoder *encoder)
-{
+		if (aux) {
+			int ret = nvkm_rdaux(aux, DP_SET_POWER, &pwr, 1);
+			if (ret == 0) {
+				pwr &= ~DP_SET_POWER_MASK;
+				pwr |=  DP_SET_POWER_D3;
+				nvkm_wraux(aux, DP_SET_POWER, &pwr, 1);
+			}
+		}
+
+		nv_encoder->update(nv_encoder, nv_crtc->index, NULL, 0, 0);
+		nv50_audio_disable(encoder, nv_crtc);
+		nv50_hdmi_disable(&nv_encoder->base.base, nv_crtc);
+	}
 }
 
 static void
-nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
-		  struct drm_display_mode *mode)
+nv50_sor_enable(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
+	struct drm_display_mode *mode = &nv_crtc->base.state->adjusted_mode;
 	struct {
 		struct nv50_disp_mthd_v1 base;
 		struct nv50_disp_sor_lvds_script_v0 lvds;
@@ -1954,13 +3547,10 @@ nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 		.base.hashm   = nv_encoder->dcb->hashm,
 	};
 	struct nv50_disp *disp = nv50_disp(encoder->dev);
-	struct nv50_mast *mast = nv50_mast(encoder->dev);
 	struct drm_device *dev = encoder->dev;
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_connector *nv_connector;
 	struct nvbios *bios = &drm->vbios;
-	u32 mask, ctrl;
-	u8 owner = 1 << nv_crtc->index;
 	u8 proto = 0xf;
 	u8 depth = 0x0;
 
@@ -1985,7 +3575,7 @@ nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 			proto = 0x2;
 		}
 
-		nv50_hdmi_mode_set(&nv_encoder->base.base, mode);
+		nv50_hdmi_enable(&nv_encoder->base.base, mode);
 		break;
 	case DCB_OUTPUT_LVDS:
 		proto = 0x0;
@@ -2019,94 +3609,60 @@ nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 		nvif_mthd(disp->disp, 0, &lvds, sizeof(lvds));
 		break;
 	case DCB_OUTPUT_DP:
-		if (nv_connector->base.display_info.bpc == 6) {
-			nv_encoder->dp.datarate = mode->clock * 18 / 8;
+		if (nv_connector->base.display_info.bpc == 6)
 			depth = 0x2;
-		} else
-		if (nv_connector->base.display_info.bpc == 8) {
-			nv_encoder->dp.datarate = mode->clock * 24 / 8;
+		else
+		if (nv_connector->base.display_info.bpc == 8)
 			depth = 0x5;
-		} else {
-			nv_encoder->dp.datarate = mode->clock * 30 / 8;
+		else
 			depth = 0x6;
-		}
 
 		if (nv_encoder->dcb->sorconf.link & 1)
 			proto = 0x8;
 		else
 			proto = 0x9;
-		nv50_audio_mode_set(encoder, mode);
+
+		nv50_audio_enable(encoder, mode);
 		break;
 	default:
 		BUG_ON(1);
 		break;
 	}
 
-	nv50_sor_dpms(&nv_encoder->base.base, DRM_MODE_DPMS_ON);
-
-	if (nv50_vers(mast) >= GF110_DISP) {
-		u32 *push = evo_wait(mast, 3);
-		if (push) {
-			u32 magic = 0x31ec6000 | (nv_crtc->index << 25);
-			u32 syncs = 0x00000001;
-
-			if (mode->flags & DRM_MODE_FLAG_NHSYNC)
-				syncs |= 0x00000008;
-			if (mode->flags & DRM_MODE_FLAG_NVSYNC)
-				syncs |= 0x00000010;
-
-			if (mode->flags & DRM_MODE_FLAG_INTERLACE)
-				magic |= 0x00000001;
-
-			evo_mthd(push, 0x0404 + (nv_crtc->index * 0x300), 2);
-			evo_data(push, syncs | (depth << 6));
-			evo_data(push, magic);
-			evo_kick(push, mast);
-		}
-
-		ctrl = proto << 8;
-		mask = 0x00000f00;
-	} else {
-		ctrl = (depth << 16) | (proto << 8);
-		if (mode->flags & DRM_MODE_FLAG_NHSYNC)
-			ctrl |= 0x00001000;
-		if (mode->flags & DRM_MODE_FLAG_NVSYNC)
-			ctrl |= 0x00002000;
-		mask = 0x000f3f00;
-	}
-
-	nv50_sor_ctrl(nv_encoder, mask | owner, ctrl | owner);
+	nv_encoder->update(nv_encoder, nv_crtc->index, mode, proto, depth);
 }
 
+static const struct drm_encoder_helper_funcs
+nv50_sor_help = {
+	.dpms = nv50_sor_dpms,
+	.atomic_check = nv50_outp_atomic_check,
+	.enable = nv50_sor_enable,
+	.disable = nv50_sor_disable,
+};
+
 static void
 nv50_sor_destroy(struct drm_encoder *encoder)
 {
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	nv50_mstm_del(&nv_encoder->dp.mstm);
 	drm_encoder_cleanup(encoder);
 	kfree(encoder);
 }
 
-static const struct drm_encoder_helper_funcs nv50_sor_hfunc = {
-	.dpms = nv50_sor_dpms,
-	.mode_fixup = nv50_encoder_mode_fixup,
-	.prepare = nv50_sor_disconnect,
-	.commit = nv50_sor_commit,
-	.mode_set = nv50_sor_mode_set,
-	.disable = nv50_sor_disconnect,
-	.get_crtc = nv50_display_crtc_get,
-};
-
-static const struct drm_encoder_funcs nv50_sor_func = {
+static const struct drm_encoder_funcs
+nv50_sor_func = {
 	.destroy = nv50_sor_destroy,
 };
 
 static int
 nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe)
 {
+	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	struct nouveau_drm *drm = nouveau_drm(connector->dev);
 	struct nvkm_i2c *i2c = nvxx_i2c(&drm->device);
 	struct nouveau_encoder *nv_encoder;
 	struct drm_encoder *encoder;
-	int type;
+	int type, ret;
 
 	switch (dcbe->type) {
 	case DCB_OUTPUT_LVDS: type = DRM_MODE_ENCODER_LVDS; break;
@@ -2122,7 +3678,16 @@ nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe)
 		return -ENOMEM;
 	nv_encoder->dcb = dcbe;
 	nv_encoder->or = ffs(dcbe->or) - 1;
-	nv_encoder->last_dpms = DRM_MODE_DPMS_OFF;
+	nv_encoder->update = nv50_sor_update;
+
+	encoder = to_drm_encoder(nv_encoder);
+	encoder->possible_crtcs = dcbe->heads;
+	encoder->possible_clones = 0;
+	drm_encoder_init(connector->dev, encoder, &nv50_sor_func, type,
+			 "sor-%04x-%04x", dcbe->hasht, dcbe->hashm);
+	drm_encoder_helper_add(encoder, &nv50_sor_help);
+
+	drm_mode_connector_attach_encoder(connector, encoder);
 
 	if (dcbe->type == DCB_OUTPUT_DP) {
 		struct nvkm_i2c_aux *aux =
@@ -2131,6 +3696,15 @@ nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe)
 			nv_encoder->i2c = &aux->i2c;
 			nv_encoder->aux = aux;
 		}
+
+		/*TODO: Use DP Info Table to check for support. */
+		if (nv50_disp(encoder->dev)->disp->oclass >= GF110_DISP) {
+			ret = nv50_mstm_new(nv_encoder, &nv_connector->aux, 16,
+					    nv_connector->base.base.id,
+					    &nv_encoder->dp.mstm);
+			if (ret)
+				return ret;
+		}
 	} else {
 		struct nvkm_i2c_bus *bus =
 			nvkm_i2c_bus_find(i2c, dcbe->i2c_index);
@@ -2138,20 +3712,12 @@ nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe)
 			nv_encoder->i2c = &bus->i2c;
 	}
 
-	encoder = to_drm_encoder(nv_encoder);
-	encoder->possible_crtcs = dcbe->heads;
-	encoder->possible_clones = 0;
-	drm_encoder_init(connector->dev, encoder, &nv50_sor_func, type, NULL);
-	drm_encoder_helper_add(encoder, &nv50_sor_hfunc);
-
-	drm_mode_connector_attach_encoder(connector, encoder);
 	return 0;
 }
 
 /******************************************************************************
  * PIOR
  *****************************************************************************/
-
 static void
 nv50_pior_dpms(struct drm_encoder *encoder, int mode)
 {
@@ -2172,30 +3738,48 @@ nv50_pior_dpms(struct drm_encoder *encoder, int mode)
 	nvif_mthd(disp->disp, 0, &args, sizeof(args));
 }
 
-static bool
-nv50_pior_mode_fixup(struct drm_encoder *encoder,
-		     const struct drm_display_mode *mode,
-		     struct drm_display_mode *adjusted_mode)
+static int
+nv50_pior_atomic_check(struct drm_encoder *encoder,
+		       struct drm_crtc_state *crtc_state,
+		       struct drm_connector_state *conn_state)
 {
-	if (!nv50_encoder_mode_fixup(encoder, mode, adjusted_mode))
-		return false;
-	adjusted_mode->clock *= 2;
-	return true;
+	int ret = nv50_outp_atomic_check(encoder, crtc_state, conn_state);
+	if (ret)
+		return ret;
+	crtc_state->adjusted_mode.clock *= 2;
+	return 0;
 }
 
 static void
-nv50_pior_commit(struct drm_encoder *encoder)
+nv50_pior_disable(struct drm_encoder *encoder)
 {
+	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nv50_mast *mast = nv50_mast(encoder->dev);
+	const int or = nv_encoder->or;
+	u32 *push;
+
+	if (nv_encoder->crtc) {
+		push = evo_wait(mast, 4);
+		if (push) {
+			if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
+				evo_mthd(push, 0x0700 + (or * 0x040), 1);
+				evo_data(push, 0x00000000);
+			}
+			evo_kick(push, mast);
+		}
+	}
+
+	nv_encoder->crtc = NULL;
 }
 
 static void
-nv50_pior_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
-		   struct drm_display_mode *adjusted_mode)
+nv50_pior_enable(struct drm_encoder *encoder)
 {
 	struct nv50_mast *mast = nv50_mast(encoder->dev);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
 	struct nouveau_connector *nv_connector;
+	struct drm_display_mode *mode = &nv_crtc->base.state->adjusted_mode;
 	u8 owner = 1 << nv_crtc->index;
 	u8 proto, depth;
 	u32 *push;
@@ -2218,8 +3802,6 @@ nv50_pior_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 		break;
 	}
 
-	nv50_pior_dpms(encoder, DRM_MODE_DPMS_ON);
-
 	push = evo_wait(mast, 8);
 	if (push) {
 		if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
@@ -2238,29 +3820,13 @@ nv50_pior_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 	nv_encoder->crtc = encoder->crtc;
 }
 
-static void
-nv50_pior_disconnect(struct drm_encoder *encoder)
-{
-	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nv50_mast *mast = nv50_mast(encoder->dev);
-	const int or = nv_encoder->or;
-	u32 *push;
-
-	if (nv_encoder->crtc) {
-		nv50_crtc_prepare(nv_encoder->crtc);
-
-		push = evo_wait(mast, 4);
-		if (push) {
-			if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
-				evo_mthd(push, 0x0700 + (or * 0x040), 1);
-				evo_data(push, 0x00000000);
-			}
-			evo_kick(push, mast);
-		}
-	}
-
-	nv_encoder->crtc = NULL;
-}
+static const struct drm_encoder_helper_funcs
+nv50_pior_help = {
+	.dpms = nv50_pior_dpms,
+	.atomic_check = nv50_pior_atomic_check,
+	.enable = nv50_pior_enable,
+	.disable = nv50_pior_disable,
+};
 
 static void
 nv50_pior_destroy(struct drm_encoder *encoder)
@@ -2269,17 +3835,8 @@ nv50_pior_destroy(struct drm_encoder *encoder)
 	kfree(encoder);
 }
 
-static const struct drm_encoder_helper_funcs nv50_pior_hfunc = {
-	.dpms = nv50_pior_dpms,
-	.mode_fixup = nv50_pior_mode_fixup,
-	.prepare = nv50_pior_disconnect,
-	.commit = nv50_pior_commit,
-	.mode_set = nv50_pior_mode_set,
-	.disable = nv50_pior_disconnect,
-	.get_crtc = nv50_display_crtc_get,
-};
-
-static const struct drm_encoder_funcs nv50_pior_func = {
+static const struct drm_encoder_funcs
+nv50_pior_func = {
 	.destroy = nv50_pior_destroy,
 };
 
@@ -2321,149 +3878,464 @@ nv50_pior_create(struct drm_connector *connector, struct dcb_output *dcbe)
 	encoder = to_drm_encoder(nv_encoder);
 	encoder->possible_crtcs = dcbe->heads;
 	encoder->possible_clones = 0;
-	drm_encoder_init(connector->dev, encoder, &nv50_pior_func, type, NULL);
-	drm_encoder_helper_add(encoder, &nv50_pior_hfunc);
+	drm_encoder_init(connector->dev, encoder, &nv50_pior_func, type,
+			 "pior-%04x-%04x", dcbe->hasht, dcbe->hashm);
+	drm_encoder_helper_add(encoder, &nv50_pior_help);
 
 	drm_mode_connector_attach_encoder(connector, encoder);
 	return 0;
 }
 
 /******************************************************************************
- * Framebuffer
+ * Atomic
  *****************************************************************************/
 
 static void
-nv50_fbdma_fini(struct nv50_fbdma *fbdma)
+nv50_disp_atomic_commit_core(struct nouveau_drm *drm, u32 interlock)
 {
-	int i;
-	for (i = 0; i < ARRAY_SIZE(fbdma->base); i++)
-		nvif_object_fini(&fbdma->base[i]);
-	nvif_object_fini(&fbdma->core);
-	list_del(&fbdma->head);
-	kfree(fbdma);
+	struct nv50_disp *disp = nv50_disp(drm->dev);
+	struct nv50_dmac *core = &disp->mast.base;
+	struct nv50_mstm *mstm;
+	struct drm_encoder *encoder;
+	u32 *push;
+
+	NV_ATOMIC(drm, "commit core %08x\n", interlock);
+
+	drm_for_each_encoder(encoder, drm->dev) {
+		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
+			mstm = nouveau_encoder(encoder)->dp.mstm;
+			if (mstm && mstm->modified)
+				nv50_mstm_prepare(mstm);
+		}
+	}
+
+	if ((push = evo_wait(core, 5))) {
+		evo_mthd(push, 0x0084, 1);
+		evo_data(push, 0x80000000);
+		evo_mthd(push, 0x0080, 2);
+		evo_data(push, interlock);
+		evo_data(push, 0x00000000);
+		nouveau_bo_wr32(disp->sync, 0, 0x00000000);
+		evo_kick(push, core);
+		if (nvif_msec(&drm->device, 2000ULL,
+			if (nouveau_bo_rd32(disp->sync, 0))
+				break;
+			usleep_range(1, 2);
+		) < 0)
+			NV_ERROR(drm, "EVO timeout\n");
+	}
+
+	drm_for_each_encoder(encoder, drm->dev) {
+		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
+			mstm = nouveau_encoder(encoder)->dp.mstm;
+			if (mstm && mstm->modified)
+				nv50_mstm_cleanup(mstm);
+		}
+	}
 }
 
-static int
-nv50_fbdma_init(struct drm_device *dev, u32 name, u64 offset, u64 length, u8 kind)
+static void
+nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 {
+	struct drm_device *dev = state->dev;
+	struct drm_crtc_state *crtc_state;
+	struct drm_crtc *crtc;
+	struct drm_plane_state *plane_state;
+	struct drm_plane *plane;
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nv50_disp *disp = nv50_disp(dev);
-	struct nv50_mast *mast = nv50_mast(dev);
-	struct __attribute__ ((packed)) {
-		struct nv_dma_v0 base;
-		union {
-			struct nv50_dma_v0 nv50;
-			struct gf100_dma_v0 gf100;
-			struct gf119_dma_v0 gf119;
-		};
-	} args = {};
-	struct nv50_fbdma *fbdma;
-	struct drm_crtc *crtc;
-	u32 size = sizeof(args.base);
-	int ret;
+	struct nv50_atom *atom = nv50_atom(state);
+	struct nv50_outp_atom *outp, *outt;
+	u32 interlock_core = 0;
+	u32 interlock_chan = 0;
+	int i;
+
+	NV_ATOMIC(drm, "commit %d %d\n", atom->lock_core, atom->flush_disable);
+	drm_atomic_helper_wait_for_fences(dev, state, false);
+	drm_atomic_helper_wait_for_dependencies(state);
+	drm_atomic_helper_update_legacy_modeset_state(dev, state);
 
-	list_for_each_entry(fbdma, &disp->fbdma, head) {
-		if (fbdma->core.handle == name)
-			return 0;
+	if (atom->lock_core)
+		mutex_lock(&disp->mutex);
+
+	/* Disable head(s). */
+	for_each_crtc_in_state(state, crtc, crtc_state, i) {
+		struct nv50_head_atom *asyh = nv50_head_atom(crtc->state);
+		struct nv50_head *head = nv50_head(crtc);
+
+		NV_ATOMIC(drm, "%s: clr %04x (set %04x)\n", crtc->name,
+			  asyh->clr.mask, asyh->set.mask);
+
+		if (asyh->clr.mask) {
+			nv50_head_flush_clr(head, asyh, atom->flush_disable);
+			interlock_core |= 1;
+		}
 	}
 
-	fbdma = kzalloc(sizeof(*fbdma), GFP_KERNEL);
-	if (!fbdma)
-		return -ENOMEM;
-	list_add(&fbdma->head, &disp->fbdma);
+	/* Disable plane(s). */
+	for_each_plane_in_state(state, plane, plane_state, i) {
+		struct nv50_wndw_atom *asyw = nv50_wndw_atom(plane->state);
+		struct nv50_wndw *wndw = nv50_wndw(plane);
 
-	args.base.target = NV_DMA_V0_TARGET_VRAM;
-	args.base.access = NV_DMA_V0_ACCESS_RDWR;
-	args.base.start = offset;
-	args.base.limit = offset + length - 1;
+		NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", plane->name,
+			  asyw->clr.mask, asyw->set.mask);
+		if (!asyw->clr.mask)
+			continue;
 
-	if (drm->device.info.chipset < 0x80) {
-		args.nv50.part = NV50_DMA_V0_PART_256;
-		size += sizeof(args.nv50);
-	} else
-	if (drm->device.info.chipset < 0xc0) {
-		args.nv50.part = NV50_DMA_V0_PART_256;
-		args.nv50.kind = kind;
-		size += sizeof(args.nv50);
-	} else
-	if (drm->device.info.chipset < 0xd0) {
-		args.gf100.kind = kind;
-		size += sizeof(args.gf100);
-	} else {
-		args.gf119.page = GF119_DMA_V0_PAGE_LP;
-		args.gf119.kind = kind;
-		size += sizeof(args.gf119);
+		interlock_chan |= nv50_wndw_flush_clr(wndw, interlock_core,
+						      atom->flush_disable,
+						      asyw);
+	}
+
+	/* Disable output path(s). */
+	list_for_each_entry(outp, &atom->outp, head) {
+		const struct drm_encoder_helper_funcs *help;
+		struct drm_encoder *encoder;
+
+		encoder = outp->encoder;
+		help = encoder->helper_private;
+
+		NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", encoder->name,
+			  outp->clr.mask, outp->set.mask);
+
+		if (outp->clr.mask) {
+			help->disable(encoder);
+			interlock_core |= 1;
+			if (outp->flush_disable) {
+				nv50_disp_atomic_commit_core(drm, interlock_chan);
+				interlock_core = 0;
+				interlock_chan = 0;
+			}
+		}
+	}
+
+	/* Flush disable. */
+	if (interlock_core) {
+		if (atom->flush_disable) {
+			nv50_disp_atomic_commit_core(drm, interlock_chan);
+			interlock_core = 0;
+			interlock_chan = 0;
+		}
 	}
 
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+	/* Update output path(s). */
+	list_for_each_entry_safe(outp, outt, &atom->outp, head) {
+		const struct drm_encoder_helper_funcs *help;
+		struct drm_encoder *encoder;
+
+		encoder = outp->encoder;
+		help = encoder->helper_private;
+
+		NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", encoder->name,
+			  outp->set.mask, outp->clr.mask);
+
+		if (outp->set.mask) {
+			help->enable(encoder);
+			interlock_core = 1;
+		}
+
+		list_del(&outp->head);
+		kfree(outp);
+	}
+
+	/* Update head(s). */
+	for_each_crtc_in_state(state, crtc, crtc_state, i) {
+		struct nv50_head_atom *asyh = nv50_head_atom(crtc->state);
 		struct nv50_head *head = nv50_head(crtc);
-		int ret = nvif_object_init(&head->sync.base.base.user, name,
-					   NV_DMA_IN_MEMORY, &args, size,
-					   &fbdma->base[head->base.index]);
-		if (ret) {
-			nv50_fbdma_fini(fbdma);
-			return ret;
+
+		NV_ATOMIC(drm, "%s: set %04x (clr %04x)\n", crtc->name,
+			  asyh->set.mask, asyh->clr.mask);
+
+		if (asyh->set.mask) {
+			nv50_head_flush_set(head, asyh);
+			interlock_core = 1;
 		}
 	}
 
-	ret = nvif_object_init(&mast->base.base.user, name, NV_DMA_IN_MEMORY,
-			       &args, size, &fbdma->core);
-	if (ret) {
-		nv50_fbdma_fini(fbdma);
+	/* Update plane(s). */
+	for_each_plane_in_state(state, plane, plane_state, i) {
+		struct nv50_wndw_atom *asyw = nv50_wndw_atom(plane->state);
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+
+		NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", plane->name,
+			  asyw->set.mask, asyw->clr.mask);
+		if ( !asyw->set.mask &&
+		    (!asyw->clr.mask || atom->flush_disable))
+			continue;
+
+		interlock_chan |= nv50_wndw_flush_set(wndw, interlock_core, asyw);
+	}
+
+	/* Flush update. */
+	if (interlock_core) {
+		if (!interlock_chan && atom->state.legacy_cursor_update) {
+			u32 *push = evo_wait(&disp->mast, 2);
+			if (push) {
+				evo_mthd(push, 0x0080, 1);
+				evo_data(push, 0x00000000);
+				evo_kick(push, &disp->mast);
+			}
+		} else {
+			nv50_disp_atomic_commit_core(drm, interlock_chan);
+		}
+	}
+
+	if (atom->lock_core)
+		mutex_unlock(&disp->mutex);
+
+	/* Wait for HW to signal completion. */
+	for_each_plane_in_state(state, plane, plane_state, i) {
+		struct nv50_wndw_atom *asyw = nv50_wndw_atom(plane->state);
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+		int ret = nv50_wndw_wait_armed(wndw, asyw);
+		if (ret)
+			NV_ERROR(drm, "%s: timeout\n", plane->name);
+	}
+
+	for_each_crtc_in_state(state, crtc, crtc_state, i) {
+		if (crtc->state->event) {
+			unsigned long flags;
+			/* Get correct count/ts if racing with vblank irq */
+			drm_accurate_vblank_count(crtc);
+			spin_lock_irqsave(&crtc->dev->event_lock, flags);
+			drm_crtc_send_vblank_event(crtc, crtc->state->event);
+			spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+			crtc->state->event = NULL;
+		}
+	}
+
+	drm_atomic_helper_commit_hw_done(state);
+	drm_atomic_helper_cleanup_planes(dev, state);
+	drm_atomic_helper_commit_cleanup_done(state);
+	drm_atomic_state_put(state);
+}
+
+static void
+nv50_disp_atomic_commit_work(struct work_struct *work)
+{
+	struct drm_atomic_state *state =
+		container_of(work, typeof(*state), commit_work);
+	nv50_disp_atomic_commit_tail(state);
+}
+
+static int
+nv50_disp_atomic_commit(struct drm_device *dev,
+			struct drm_atomic_state *state, bool nonblock)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nv50_disp *disp = nv50_disp(dev);
+	struct drm_plane_state *plane_state;
+	struct drm_plane *plane;
+	struct drm_crtc *crtc;
+	bool active = false;
+	int ret, i;
+
+	ret = pm_runtime_get_sync(dev->dev);
+	if (ret < 0 && ret != -EACCES)
 		return ret;
+
+	ret = drm_atomic_helper_setup_commit(state, nonblock);
+	if (ret)
+		goto done;
+
+	INIT_WORK(&state->commit_work, nv50_disp_atomic_commit_work);
+
+	ret = drm_atomic_helper_prepare_planes(dev, state);
+	if (ret)
+		goto done;
+
+	if (!nonblock) {
+		ret = drm_atomic_helper_wait_for_fences(dev, state, true);
+		if (ret)
+			goto done;
+	}
+
+	for_each_plane_in_state(state, plane, plane_state, i) {
+		struct nv50_wndw_atom *asyw = nv50_wndw_atom(plane_state);
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+		if (asyw->set.image) {
+			asyw->ntfy.handle = wndw->dmac->sync.handle;
+			asyw->ntfy.offset = wndw->ntfy;
+			asyw->ntfy.awaken = false;
+			asyw->set.ntfy = true;
+			nouveau_bo_wr32(disp->sync, wndw->ntfy / 4, 0x00000000);
+			wndw->ntfy ^= 0x10;
+		}
+	}
+
+	drm_atomic_helper_swap_state(state, true);
+	drm_atomic_state_get(state);
+
+	if (nonblock)
+		queue_work(system_unbound_wq, &state->commit_work);
+	else
+		nv50_disp_atomic_commit_tail(state);
+
+	drm_for_each_crtc(crtc, dev) {
+		if (crtc->state->enable) {
+			if (!drm->have_disp_power_ref) {
+				drm->have_disp_power_ref = true;
+				return ret;
+			}
+			active = true;
+			break;
+		}
+	}
+
+	if (!active && drm->have_disp_power_ref) {
+		pm_runtime_put_autosuspend(dev->dev);
+		drm->have_disp_power_ref = false;
+	}
+
+done:
+	pm_runtime_put_autosuspend(dev->dev);
+	return ret;
+}
+
+static struct nv50_outp_atom *
+nv50_disp_outp_atomic_add(struct nv50_atom *atom, struct drm_encoder *encoder)
+{
+	struct nv50_outp_atom *outp;
+
+	list_for_each_entry(outp, &atom->outp, head) {
+		if (outp->encoder == encoder)
+			return outp;
+	}
+
+	outp = kzalloc(sizeof(*outp), GFP_KERNEL);
+	if (!outp)
+		return ERR_PTR(-ENOMEM);
+
+	list_add(&outp->head, &atom->outp);
+	outp->encoder = encoder;
+	return outp;
+}
+
+static int
+nv50_disp_outp_atomic_check_clr(struct nv50_atom *atom,
+				struct drm_connector *connector)
+{
+	struct drm_encoder *encoder = connector->state->best_encoder;
+	struct drm_crtc_state *crtc_state;
+	struct drm_crtc *crtc;
+	struct nv50_outp_atom *outp;
+
+	if (!(crtc = connector->state->crtc))
+		return 0;
+
+	crtc_state = drm_atomic_get_existing_crtc_state(&atom->state, crtc);
+	if (crtc->state->active && drm_atomic_crtc_needs_modeset(crtc_state)) {
+		outp = nv50_disp_outp_atomic_add(atom, encoder);
+		if (IS_ERR(outp))
+			return PTR_ERR(outp);
+
+		if (outp->encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
+			outp->flush_disable = true;
+			atom->flush_disable = true;
+		}
+		outp->clr.ctrl = true;
+		atom->lock_core = true;
 	}
 
 	return 0;
 }
 
-static void
-nv50_fb_dtor(struct drm_framebuffer *fb)
+static int
+nv50_disp_outp_atomic_check_set(struct nv50_atom *atom,
+				struct drm_connector_state *connector_state)
 {
+	struct drm_encoder *encoder = connector_state->best_encoder;
+	struct drm_crtc_state *crtc_state;
+	struct drm_crtc *crtc;
+	struct nv50_outp_atom *outp;
+
+	if (!(crtc = connector_state->crtc))
+		return 0;
+
+	crtc_state = drm_atomic_get_existing_crtc_state(&atom->state, crtc);
+	if (crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state)) {
+		outp = nv50_disp_outp_atomic_add(atom, encoder);
+		if (IS_ERR(outp))
+			return PTR_ERR(outp);
+
+		outp->set.ctrl = true;
+		atom->lock_core = true;
+	}
+
+	return 0;
 }
 
 static int
-nv50_fb_ctor(struct drm_framebuffer *fb)
-{
-	struct nouveau_framebuffer *nv_fb = nouveau_framebuffer(fb);
-	struct nouveau_drm *drm = nouveau_drm(fb->dev);
-	struct nouveau_bo *nvbo = nv_fb->nvbo;
-	struct nv50_disp *disp = nv50_disp(fb->dev);
-	u8 kind = nouveau_bo_tile_layout(nvbo) >> 8;
-	u8 tile = nvbo->tile_mode;
-
-	if (drm->device.info.chipset >= 0xc0)
-		tile >>= 4; /* yep.. */
-
-	switch (fb->depth) {
-	case  8: nv_fb->r_format = 0x1e00; break;
-	case 15: nv_fb->r_format = 0xe900; break;
-	case 16: nv_fb->r_format = 0xe800; break;
-	case 24:
-	case 32: nv_fb->r_format = 0xcf00; break;
-	case 30: nv_fb->r_format = 0xd100; break;
-	default:
-		 NV_ERROR(drm, "unknown depth %d\n", fb->depth);
-		 return -EINVAL;
+nv50_disp_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
+{
+	struct nv50_atom *atom = nv50_atom(state);
+	struct drm_connector_state *connector_state;
+	struct drm_connector *connector;
+	int ret, i;
+
+	ret = drm_atomic_helper_check(dev, state);
+	if (ret)
+		return ret;
+
+	for_each_connector_in_state(state, connector, connector_state, i) {
+		ret = nv50_disp_outp_atomic_check_clr(atom, connector);
+		if (ret)
+			return ret;
+
+		ret = nv50_disp_outp_atomic_check_set(atom, connector_state);
+		if (ret)
+			return ret;
 	}
 
-	if (disp->disp->oclass < G82_DISP) {
-		nv_fb->r_pitch   = kind ? (((fb->pitches[0] / 4) << 4) | tile) :
-					    (fb->pitches[0] | 0x00100000);
-		nv_fb->r_format |= kind << 16;
-	} else
-	if (disp->disp->oclass < GF110_DISP) {
-		nv_fb->r_pitch  = kind ? (((fb->pitches[0] / 4) << 4) | tile) :
-					   (fb->pitches[0] | 0x00100000);
-	} else {
-		nv_fb->r_pitch  = kind ? (((fb->pitches[0] / 4) << 4) | tile) :
-					   (fb->pitches[0] | 0x01000000);
+	return 0;
+}
+
+static void
+nv50_disp_atomic_state_clear(struct drm_atomic_state *state)
+{
+	struct nv50_atom *atom = nv50_atom(state);
+	struct nv50_outp_atom *outp, *outt;
+
+	list_for_each_entry_safe(outp, outt, &atom->outp, head) {
+		list_del(&outp->head);
+		kfree(outp);
 	}
-	nv_fb->r_handle = 0xffff0000 | kind;
 
-	return nv50_fbdma_init(fb->dev, nv_fb->r_handle, 0,
-			       drm->device.info.ram_user, kind);
+	drm_atomic_state_default_clear(state);
+}
+
+static void
+nv50_disp_atomic_state_free(struct drm_atomic_state *state)
+{
+	struct nv50_atom *atom = nv50_atom(state);
+	drm_atomic_state_default_release(&atom->state);
+	kfree(atom);
 }
 
+static struct drm_atomic_state *
+nv50_disp_atomic_state_alloc(struct drm_device *dev)
+{
+	struct nv50_atom *atom;
+	if (!(atom = kzalloc(sizeof(*atom), GFP_KERNEL)) ||
+	    drm_atomic_state_init(dev, &atom->state) < 0) {
+		kfree(atom);
+		return NULL;
+	}
+	INIT_LIST_HEAD(&atom->outp);
+	return &atom->state;
+}
+
+static const struct drm_mode_config_funcs
+nv50_disp_func = {
+	.fb_create = nouveau_user_framebuffer_create,
+	.output_poll_changed = nouveau_fbcon_output_poll_changed,
+	.atomic_check = nv50_disp_atomic_check,
+	.atomic_commit = nv50_disp_atomic_commit,
+	.atomic_state_alloc = nv50_disp_atomic_state_alloc,
+	.atomic_state_clear = nv50_disp_atomic_state_clear,
+	.atomic_state_free = nv50_disp_atomic_state_free,
+};
+
 /******************************************************************************
  * Init
  *****************************************************************************/
@@ -2471,12 +4343,30 @@ nv50_fb_ctor(struct drm_framebuffer *fb)
 void
 nv50_display_fini(struct drm_device *dev)
 {
+	struct nouveau_encoder *nv_encoder;
+	struct drm_encoder *encoder;
+	struct drm_plane *plane;
+
+	drm_for_each_plane(plane, dev) {
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+		if (plane->funcs != &nv50_wndw)
+			continue;
+		nv50_wndw_fini(wndw);
+	}
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
+			nv_encoder = nouveau_encoder(encoder);
+			nv50_mstm_fini(nv_encoder->dp.mstm);
+		}
+	}
 }
 
 int
 nv50_display_init(struct drm_device *dev)
 {
-	struct nv50_disp *disp = nv50_disp(dev);
+	struct drm_encoder *encoder;
+	struct drm_plane *plane;
 	struct drm_crtc *crtc;
 	u32 *push;
 
@@ -2484,16 +4374,35 @@ nv50_display_init(struct drm_device *dev)
 	if (!push)
 		return -EBUSY;
 
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		struct nv50_sync *sync = nv50_sync(crtc);
-
-		nv50_crtc_lut_load(crtc);
-		nouveau_bo_wr32(disp->sync, sync->addr / 4, sync->data);
-	}
-
 	evo_mthd(push, 0x0088, 1);
 	evo_data(push, nv50_mast(dev)->base.sync.handle);
 	evo_kick(push, nv50_mast(dev));
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
+			const struct drm_encoder_helper_funcs *help;
+			struct nouveau_encoder *nv_encoder;
+
+			nv_encoder = nouveau_encoder(encoder);
+			help = encoder->helper_private;
+			if (help && help->dpms)
+				help->dpms(encoder, DRM_MODE_DPMS_ON);
+
+			nv50_mstm_init(nv_encoder->dp.mstm);
+		}
+	}
+
+	drm_for_each_crtc(crtc, dev) {
+		nv50_head_lut_load(crtc);
+	}
+
+	drm_for_each_plane(plane, dev) {
+		struct nv50_wndw *wndw = nv50_wndw(plane);
+		if (plane->funcs != &nv50_wndw)
+			continue;
+		nv50_wndw_init(wndw);
+	}
+
 	return 0;
 }
 
@@ -2501,11 +4410,6 @@ void
 nv50_display_destroy(struct drm_device *dev)
 {
 	struct nv50_disp *disp = nv50_disp(dev);
-	struct nv50_fbdma *fbdma, *fbtmp;
-
-	list_for_each_entry_safe(fbdma, fbtmp, &disp->fbdma, head) {
-		nv50_fbdma_fini(fbdma);
-	}
 
 	nv50_dmac_destroy(&disp->mast.base, disp->disp);
 
@@ -2518,6 +4422,10 @@ nv50_display_destroy(struct drm_device *dev)
 	kfree(disp);
 }
 
+MODULE_PARM_DESC(atomic, "Expose atomic ioctl (default: disabled)");
+static int nouveau_atomic = 0;
+module_param_named(atomic, nouveau_atomic, int, 0400);
+
 int
 nv50_display_create(struct drm_device *dev)
 {
@@ -2532,15 +4440,17 @@ nv50_display_create(struct drm_device *dev)
 	disp = kzalloc(sizeof(*disp), GFP_KERNEL);
 	if (!disp)
 		return -ENOMEM;
-	INIT_LIST_HEAD(&disp->fbdma);
+
+	mutex_init(&disp->mutex);
 
 	nouveau_display(dev)->priv = disp;
 	nouveau_display(dev)->dtor = nv50_display_destroy;
 	nouveau_display(dev)->init = nv50_display_init;
 	nouveau_display(dev)->fini = nv50_display_fini;
-	nouveau_display(dev)->fb_ctor = nv50_fb_ctor;
-	nouveau_display(dev)->fb_dtor = nv50_fb_dtor;
 	disp->disp = &nouveau_display(dev)->disp;
+	dev->mode_config.funcs = &nv50_disp_func;
+	if (nouveau_atomic)
+		dev->driver->driver_features |= DRIVER_ATOMIC;
 
 	/* small shared memory area we use for notifiers and semaphores */
 	ret = nouveau_bo_new(dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
@@ -2572,7 +4482,7 @@ nv50_display_create(struct drm_device *dev)
 		crtcs = 2;
 
 	for (i = 0; i < crtcs; i++) {
-		ret = nv50_crtc_create(dev, i);
+		ret = nv50_head_create(dev, i);
 		if (ret)
 			goto out;
 	}
diff --git a/drivers/gpu/drm/nouveau/nv50_display.h b/drivers/gpu/drm/nouveau/nv50_display.h
index 70da347aa8c5..918187cee84b 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.h
+++ b/drivers/gpu/drm/nouveau/nv50_display.h
@@ -35,11 +35,4 @@ int  nv50_display_create(struct drm_device *);
 void nv50_display_destroy(struct drm_device *);
 int  nv50_display_init(struct drm_device *);
 void nv50_display_fini(struct drm_device *);
-
-void nv50_display_flip_stop(struct drm_crtc *);
-int  nv50_display_flip_next(struct drm_crtc *, struct drm_framebuffer *,
-			    struct nouveau_channel *, u32 swap_interval);
-
-struct nouveau_bo *nv50_display_crtc_sema(struct drm_device *, int head);
-
 #endif /* __NV50_DISPLAY_H__ */
diff --git a/drivers/gpu/drm/nouveau/nv50_fbcon.c b/drivers/gpu/drm/nouveau/nv50_fbcon.c
index af3d3c49411a..327dcd7901ed 100644
--- a/drivers/gpu/drm/nouveau/nv50_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv50_fbcon.c
@@ -30,7 +30,7 @@ int
 nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev);
 	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
@@ -65,7 +65,7 @@ int
 nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev);
 	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
@@ -93,7 +93,7 @@ int
 nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev);
 	struct nouveau_channel *chan = drm->channel;
 	uint32_t dwords, *data = (uint32_t *)image->data;
 	uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel));
@@ -148,8 +148,8 @@ int
 nv50_fbcon_accel_init(struct fb_info *info)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct nouveau_framebuffer *fb = &nfbdev->nouveau_fb;
-	struct drm_device *dev = nfbdev->dev;
+	struct nouveau_framebuffer *fb = nouveau_framebuffer(nfbdev->helper.fb);
+	struct drm_device *dev = nfbdev->helper.dev;
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_channel *chan = drm->channel;
 	int ret, format;
diff --git a/drivers/gpu/drm/nouveau/nv50_fence.c b/drivers/gpu/drm/nouveau/nv50_fence.c
index 4d6f202b7770..f68c7054fd53 100644
--- a/drivers/gpu/drm/nouveau/nv50_fence.c
+++ b/drivers/gpu/drm/nouveau/nv50_fence.c
@@ -35,13 +35,12 @@
 static int
 nv50_fence_context_new(struct nouveau_channel *chan)
 {
-	struct drm_device *dev = chan->drm->dev;
 	struct nv10_fence_priv *priv = chan->drm->fence;
 	struct nv10_fence_chan *fctx;
 	struct ttm_mem_reg *mem = &priv->bo->bo.mem;
 	u32 start = mem->start * PAGE_SIZE;
 	u32 limit = start + mem->size - 1;
-	int ret, i;
+	int ret;
 
 	fctx = chan->fence = kzalloc(sizeof(*fctx), GFP_KERNEL);
 	if (!fctx)
@@ -60,23 +59,6 @@ nv50_fence_context_new(struct nouveau_channel *chan)
 					.limit = limit,
 			       }, sizeof(struct nv_dma_v0),
 			       &fctx->sema);
-
-	/* dma objects for display sync channel semaphore blocks */
-	for (i = 0; !ret && i < dev->mode_config.num_crtc; i++) {
-		struct nouveau_bo *bo = nv50_display_crtc_sema(dev, i);
-		u32 start = bo->bo.mem.start * PAGE_SIZE;
-		u32 limit = start + bo->bo.mem.size - 1;
-
-		ret = nvif_object_init(&chan->user, NvEvoSema0 + i,
-				       NV_DMA_IN_MEMORY, &(struct nv_dma_v0) {
-						.target = NV_DMA_V0_TARGET_VRAM,
-						.access = NV_DMA_V0_ACCESS_RDWR,
-						.start = start,
-						.limit = limit,
-				       }, sizeof(struct nv_dma_v0),
-				       &fctx->head[i]);
-	}
-
 	if (ret)
 		nv10_fence_context_del(chan);
 	return ret;
@@ -97,7 +79,7 @@ nv50_fence_create(struct nouveau_drm *drm)
 	priv->base.context_new = nv50_fence_context_new;
 	priv->base.context_del = nv10_fence_context_del;
 	priv->base.contexts = 127;
-	priv->base.context_base = fence_context_alloc(priv->base.contexts);
+	priv->base.context_base = dma_fence_context_alloc(priv->base.contexts);
 	spin_lock_init(&priv->lock);
 
 	ret = nouveau_bo_new(drm->dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c
index 18bde9d8e6d6..52b87ae83e7b 100644
--- a/drivers/gpu/drm/nouveau/nv84_fence.c
+++ b/drivers/gpu/drm/nouveau/nv84_fence.c
@@ -28,13 +28,6 @@
 
 #include "nv50_display.h"
 
-u64
-nv84_fence_crtc(struct nouveau_channel *chan, int crtc)
-{
-	struct nv84_fence_chan *fctx = chan->fence;
-	return fctx->dispc_vma[crtc].offset;
-}
-
 static int
 nv84_fence_emit32(struct nouveau_channel *chan, u64 virtual, u32 sequence)
 {
@@ -110,15 +103,8 @@ nv84_fence_read(struct nouveau_channel *chan)
 static void
 nv84_fence_context_del(struct nouveau_channel *chan)
 {
-	struct drm_device *dev = chan->drm->dev;
 	struct nv84_fence_priv *priv = chan->drm->fence;
 	struct nv84_fence_chan *fctx = chan->fence;
-	int i;
-
-	for (i = 0; i < dev->mode_config.num_crtc; i++) {
-		struct nouveau_bo *bo = nv50_display_crtc_sema(dev, i);
-		nouveau_bo_vma_del(bo, &fctx->dispc_vma[i]);
-	}
 
 	nouveau_bo_wr32(priv->bo, chan->chid * 16 / 4, fctx->base.sequence);
 	nouveau_bo_vma_del(priv->bo, &fctx->vma_gart);
@@ -134,7 +120,7 @@ nv84_fence_context_new(struct nouveau_channel *chan)
 	struct nouveau_cli *cli = (void *)chan->user.client;
 	struct nv84_fence_priv *priv = chan->drm->fence;
 	struct nv84_fence_chan *fctx;
-	int ret, i;
+	int ret;
 
 	fctx = chan->fence = kzalloc(sizeof(*fctx), GFP_KERNEL);
 	if (!fctx)
@@ -154,12 +140,6 @@ nv84_fence_context_new(struct nouveau_channel *chan)
 					&fctx->vma_gart);
 	}
 
-	/* map display semaphore buffers into channel's vm */
-	for (i = 0; !ret && i < chan->drm->dev->mode_config.num_crtc; i++) {
-		struct nouveau_bo *bo = nv50_display_crtc_sema(chan->drm->dev, i);
-		ret = nouveau_bo_vma_add(bo, cli->vm, &fctx->dispc_vma[i]);
-	}
-
 	if (ret)
 		nv84_fence_context_del(chan);
 	return ret;
@@ -229,7 +209,7 @@ nv84_fence_create(struct nouveau_drm *drm)
 	priv->base.context_del = nv84_fence_context_del;
 
 	priv->base.contexts = fifo->nr;
-	priv->base.context_base = fence_context_alloc(priv->base.contexts);
+	priv->base.context_base = dma_fence_context_alloc(priv->base.contexts);
 	priv->base.uevent = true;
 
 	/* Use VRAM if there is any ; otherwise fallback to system memory */
diff --git a/drivers/gpu/drm/nouveau/nvc0_fbcon.c b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
index 054b6a056d99..90f27bfa381f 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
@@ -30,7 +30,7 @@ int
 nvc0_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev);
 	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
@@ -65,7 +65,7 @@ int
 nvc0_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev);
 	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
@@ -93,7 +93,7 @@ int
 nvc0_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev);
 	struct nouveau_channel *chan = drm->channel;
 	uint32_t dwords, *data = (uint32_t *)image->data;
 	uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel));
@@ -148,8 +148,8 @@ int
 nvc0_fbcon_accel_init(struct fb_info *info)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct nouveau_framebuffer *fb = &nfbdev->nouveau_fb;
+	struct drm_device *dev = nfbdev->helper.dev;
+	struct nouveau_framebuffer *fb = nouveau_framebuffer(nfbdev->helper.fb);
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_channel *chan = drm->channel;
 	int ret, format;
diff --git a/drivers/gpu/drm/nouveau/nvif/client.c b/drivers/gpu/drm/nouveau/nvif/client.c
index 1ee9294eca2e..29c20dfd894d 100644
--- a/drivers/gpu/drm/nouveau/nvif/client.c
+++ b/drivers/gpu/drm/nouveau/nvif/client.c
@@ -55,7 +55,7 @@ nvif_client_fini(struct nvif_client *client)
 	}
 }
 
-const struct nvif_driver *
+static const struct nvif_driver *
 nvif_drivers[] = {
 #ifdef __KERNEL__
 	&nvif_driver_nvkm,
diff --git a/drivers/gpu/drm/nouveau/nvif/notify.c b/drivers/gpu/drm/nouveau/nvif/notify.c
index b0787ff833ef..278b3933dc96 100644
--- a/drivers/gpu/drm/nouveau/nvif/notify.c
+++ b/drivers/gpu/drm/nouveau/nvif/notify.c
@@ -155,10 +155,8 @@ nvif_notify_fini(struct nvif_notify *notify)
 	int ret = nvif_notify_put(notify);
 	if (ret >= 0 && object) {
 		ret = nvif_object_ioctl(object, &args, sizeof(args), NULL);
-		if (ret == 0) {
-			notify->object = NULL;
-			kfree((void *)notify->data);
-		}
+		notify->object = NULL;
+		kfree((void *)notify->data);
 	}
 	return ret;
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c
index 34ecd4a7e0c1..058ff46b5f16 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c
@@ -20,6 +20,7 @@
  * DEALINGS IN THE SOFTWARE.
  */
 #include <core/device.h>
+#include <core/firmware.h>
 
 /**
  * nvkm_firmware_get - load firmware from the official nvidia/chip/ directory
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild
index a4458a8eb30a..255d81ccf916 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild
@@ -4,4 +4,4 @@ nvkm-y += nvkm/engine/ce/gk104.o
 nvkm-y += nvkm/engine/ce/gm107.o
 nvkm-y += nvkm/engine/ce/gm200.o
 nvkm-y += nvkm/engine/ce/gp100.o
-nvkm-y += nvkm/engine/ce/gp104.o
+nvkm-y += nvkm/engine/ce/gp102.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gf100.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gf100.fuc3.h
index 05bb65608dfe..d9ca9636a3e3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gf100.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gf100.fuc3.h
@@ -1,4 +1,4 @@
-uint32_t gf100_ce_data[] = {
+static uint32_t gf100_ce_data[] = {
 /* 0x0000: ctx_object */
 	0x00000000,
 /* 0x0004: ctx_query_address_high */
@@ -171,7 +171,7 @@ uint32_t gf100_ce_data[] = {
 	0x00000800,
 };
 
-uint32_t gf100_ce_code[] = {
+static uint32_t gf100_ce_code[] = {
 /* 0x0000: main */
 	0x04fe04bd,
 	0x3517f000,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gt215.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gt215.fuc3.h
index 972281d10f38..f0a1cf31c7ca 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gt215.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gt215.fuc3.h
@@ -1,4 +1,4 @@
-uint32_t gt215_ce_data[] = {
+static uint32_t gt215_ce_data[] = {
 /* 0x0000: ctx_object */
 	0x00000000,
 /* 0x0004: ctx_dma */
@@ -183,7 +183,7 @@ uint32_t gt215_ce_data[] = {
 	0x00000800,
 };
 
-uint32_t gt215_ce_code[] = {
+static uint32_t gt215_ce_code[] = {
 /* 0x0000: main */
 	0x04fe04bd,
 	0x3517f000,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/gp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gp102.c
index 20e019788a53..985c8f653874 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/gp104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gp102.c
@@ -27,7 +27,7 @@
 #include <nvif/class.h>
 
 static const struct nvkm_engine_func
-gp104_ce = {
+gp102_ce = {
 	.intr = gp100_ce_intr,
 	.sclass = {
 		{ -1, -1, PASCAL_DMA_COPY_B },
@@ -37,8 +37,8 @@ gp104_ce = {
 };
 
 int
-gp104_ce_new(struct nvkm_device *device, int index,
+gp102_ce_new(struct nvkm_device *device, int index,
 	     struct nvkm_engine **pengine)
 {
-	return nvkm_engine_new_(&gp104_ce, device, index, true, pengine);
+	return nvkm_engine_new_(&gp102_ce, device, index, true, pengine);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
index 7218a067a6c5..cceda959b47c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
@@ -1357,7 +1357,7 @@ nvc0_chipset = {
 	.pmu = gf100_pmu_new,
 	.therm = gt215_therm_new,
 	.timer = nv41_timer_new,
-	.volt = nv40_volt_new,
+	.volt = gf100_volt_new,
 	.ce[0] = gf100_ce_new,
 	.ce[1] = gf100_ce_new,
 	.disp = gt215_disp_new,
@@ -1394,7 +1394,7 @@ nvc1_chipset = {
 	.pmu = gf100_pmu_new,
 	.therm = gt215_therm_new,
 	.timer = nv41_timer_new,
-	.volt = nv40_volt_new,
+	.volt = gf100_volt_new,
 	.ce[0] = gf100_ce_new,
 	.disp = gt215_disp_new,
 	.dma = gf100_dma_new,
@@ -1430,7 +1430,7 @@ nvc3_chipset = {
 	.pmu = gf100_pmu_new,
 	.therm = gt215_therm_new,
 	.timer = nv41_timer_new,
-	.volt = nv40_volt_new,
+	.volt = gf100_volt_new,
 	.ce[0] = gf100_ce_new,
 	.disp = gt215_disp_new,
 	.dma = gf100_dma_new,
@@ -1466,7 +1466,7 @@ nvc4_chipset = {
 	.pmu = gf100_pmu_new,
 	.therm = gt215_therm_new,
 	.timer = nv41_timer_new,
-	.volt = nv40_volt_new,
+	.volt = gf100_volt_new,
 	.ce[0] = gf100_ce_new,
 	.ce[1] = gf100_ce_new,
 	.disp = gt215_disp_new,
@@ -1503,7 +1503,7 @@ nvc8_chipset = {
 	.pmu = gf100_pmu_new,
 	.therm = gt215_therm_new,
 	.timer = nv41_timer_new,
-	.volt = nv40_volt_new,
+	.volt = gf100_volt_new,
 	.ce[0] = gf100_ce_new,
 	.ce[1] = gf100_ce_new,
 	.disp = gt215_disp_new,
@@ -1540,7 +1540,7 @@ nvce_chipset = {
 	.pmu = gf100_pmu_new,
 	.therm = gt215_therm_new,
 	.timer = nv41_timer_new,
-	.volt = nv40_volt_new,
+	.volt = gf100_volt_new,
 	.ce[0] = gf100_ce_new,
 	.ce[1] = gf100_ce_new,
 	.disp = gt215_disp_new,
@@ -1577,7 +1577,7 @@ nvcf_chipset = {
 	.pmu = gf100_pmu_new,
 	.therm = gt215_therm_new,
 	.timer = nv41_timer_new,
-	.volt = nv40_volt_new,
+	.volt = gf100_volt_new,
 	.ce[0] = gf100_ce_new,
 	.disp = gt215_disp_new,
 	.dma = gf100_dma_new,
@@ -1612,6 +1612,7 @@ nvd7_chipset = {
 	.pci = gf106_pci_new,
 	.therm = gf119_therm_new,
 	.timer = nv41_timer_new,
+	.volt = gf100_volt_new,
 	.ce[0] = gf100_ce_new,
 	.disp = gf119_disp_new,
 	.dma = gf119_dma_new,
@@ -1647,7 +1648,7 @@ nvd9_chipset = {
 	.pmu = gf119_pmu_new,
 	.therm = gf119_therm_new,
 	.timer = nv41_timer_new,
-	.volt = nv40_volt_new,
+	.volt = gf100_volt_new,
 	.ce[0] = gf100_ce_new,
 	.disp = gf119_disp_new,
 	.dma = gf119_dma_new,
@@ -1851,7 +1852,7 @@ nvf1_chipset = {
 	.fb = gk104_fb_new,
 	.fuse = gf100_fuse_new,
 	.gpio = gk104_gpio_new,
-	.i2c = gf119_i2c_new,
+	.i2c = gk104_i2c_new,
 	.ibus = gk104_ibus_new,
 	.iccsense = gf100_iccsense_new,
 	.imem = nv50_instmem_new,
@@ -1965,7 +1966,7 @@ nv117_chipset = {
 	.fb = gm107_fb_new,
 	.fuse = gm107_fuse_new,
 	.gpio = gk104_gpio_new,
-	.i2c = gf119_i2c_new,
+	.i2c = gk104_i2c_new,
 	.ibus = gk104_ibus_new,
 	.iccsense = gf100_iccsense_new,
 	.imem = nv50_instmem_new,
@@ -1999,7 +2000,7 @@ nv118_chipset = {
 	.fb = gm107_fb_new,
 	.fuse = gm107_fuse_new,
 	.gpio = gk104_gpio_new,
-	.i2c = gf119_i2c_new,
+	.i2c = gk104_i2c_new,
 	.ibus = gk104_ibus_new,
 	.iccsense = gf100_iccsense_new,
 	.imem = nv50_instmem_new,
@@ -2130,7 +2131,7 @@ nv12b_chipset = {
 	.bar = gk20a_bar_new,
 	.bus = gf100_bus_new,
 	.clk = gm20b_clk_new,
-	.fb = gk20a_fb_new,
+	.fb = gm20b_fb_new,
 	.fuse = gm107_fuse_new,
 	.ibus = gk20a_ibus_new,
 	.imem = gk20a_instmem_new,
@@ -2166,6 +2167,7 @@ nv130_chipset = {
 	.mmu = gf100_mmu_new,
 	.secboot = gm200_secboot_new,
 	.pci = gp100_pci_new,
+	.pmu = gp100_pmu_new,
 	.timer = gk20a_timer_new,
 	.top = gk104_top_new,
 	.ce[0] = gp100_ce_new,
@@ -2182,13 +2184,71 @@ nv130_chipset = {
 };
 
 static const struct nvkm_device_chip
+nv132_chipset = {
+	.name = "GP102",
+	.bar = gf100_bar_new,
+	.bios = nvkm_bios_new,
+	.bus = gf100_bus_new,
+	.devinit = gm200_devinit_new,
+	.fb = gp102_fb_new,
+	.fuse = gm107_fuse_new,
+	.gpio = gk104_gpio_new,
+	.i2c = gm200_i2c_new,
+	.ibus = gm200_ibus_new,
+	.imem = nv50_instmem_new,
+	.ltc = gp100_ltc_new,
+	.mc = gp100_mc_new,
+	.mmu = gf100_mmu_new,
+	.pci = gp100_pci_new,
+	.pmu = gp102_pmu_new,
+	.timer = gk20a_timer_new,
+	.top = gk104_top_new,
+	.ce[0] = gp102_ce_new,
+	.ce[1] = gp102_ce_new,
+	.ce[2] = gp102_ce_new,
+	.ce[3] = gp102_ce_new,
+	.disp = gp102_disp_new,
+	.dma = gf119_dma_new,
+	.fifo = gp100_fifo_new,
+};
+
+static const struct nvkm_device_chip
 nv134_chipset = {
 	.name = "GP104",
 	.bar = gf100_bar_new,
 	.bios = nvkm_bios_new,
 	.bus = gf100_bus_new,
 	.devinit = gm200_devinit_new,
-	.fb = gp104_fb_new,
+	.fb = gp102_fb_new,
+	.fuse = gm107_fuse_new,
+	.gpio = gk104_gpio_new,
+	.i2c = gm200_i2c_new,
+	.ibus = gm200_ibus_new,
+	.imem = nv50_instmem_new,
+	.ltc = gp100_ltc_new,
+	.mc = gp100_mc_new,
+	.mmu = gf100_mmu_new,
+	.pci = gp100_pci_new,
+	.pmu = gp102_pmu_new,
+	.timer = gk20a_timer_new,
+	.top = gk104_top_new,
+	.ce[0] = gp102_ce_new,
+	.ce[1] = gp102_ce_new,
+	.ce[2] = gp102_ce_new,
+	.ce[3] = gp102_ce_new,
+	.disp = gp102_disp_new,
+	.dma = gf119_dma_new,
+	.fifo = gp100_fifo_new,
+};
+
+static const struct nvkm_device_chip
+nv136_chipset = {
+	.name = "GP106",
+	.bar = gf100_bar_new,
+	.bios = nvkm_bios_new,
+	.bus = gf100_bus_new,
+	.devinit = gm200_devinit_new,
+	.fb = gp102_fb_new,
 	.fuse = gm107_fuse_new,
 	.gpio = gk104_gpio_new,
 	.i2c = gm200_i2c_new,
@@ -2198,13 +2258,14 @@ nv134_chipset = {
 	.mc = gp100_mc_new,
 	.mmu = gf100_mmu_new,
 	.pci = gp100_pci_new,
+	.pmu = gp102_pmu_new,
 	.timer = gk20a_timer_new,
 	.top = gk104_top_new,
-	.ce[0] = gp104_ce_new,
-	.ce[1] = gp104_ce_new,
-	.ce[2] = gp104_ce_new,
-	.ce[3] = gp104_ce_new,
-	.disp = gp104_disp_new,
+	.ce[0] = gp102_ce_new,
+	.ce[1] = gp102_ce_new,
+	.ce[2] = gp102_ce_new,
+	.ce[3] = gp102_ce_new,
+	.disp = gp102_disp_new,
 	.dma = gf119_dma_new,
 	.fifo = gp100_fifo_new,
 };
@@ -2643,7 +2704,9 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
 		case 0x126: device->chip = &nv126_chipset; break;
 		case 0x12b: device->chip = &nv12b_chipset; break;
 		case 0x130: device->chip = &nv130_chipset; break;
+		case 0x132: device->chip = &nv132_chipset; break;
 		case 0x134: device->chip = &nv134_chipset; break;
+		case 0x136: device->chip = &nv136_chipset; break;
 		default:
 			nvdev_error(device, "unknown chipset (%08x)\n", boot0);
 			goto done;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
index 62ad0300cfa5..74a1ffa425f7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
@@ -1665,14 +1665,31 @@ nvkm_device_pci_new(struct pci_dev *pci_dev, const char *cfg, const char *dbg,
 	*pdevice = &pdev->device;
 	pdev->pdev = pci_dev;
 
-	return nvkm_device_ctor(&nvkm_device_pci_func, quirk, &pci_dev->dev,
-				pci_is_pcie(pci_dev) ? NVKM_DEVICE_PCIE :
-				pci_find_capability(pci_dev, PCI_CAP_ID_AGP) ?
-				NVKM_DEVICE_AGP : NVKM_DEVICE_PCI,
-				(u64)pci_domain_nr(pci_dev->bus) << 32 |
-				     pci_dev->bus->number << 16 |
-				     PCI_SLOT(pci_dev->devfn) << 8 |
-				     PCI_FUNC(pci_dev->devfn), name,
-				cfg, dbg, detect, mmio, subdev_mask,
-				&pdev->device);
+	ret = nvkm_device_ctor(&nvkm_device_pci_func, quirk, &pci_dev->dev,
+			       pci_is_pcie(pci_dev) ? NVKM_DEVICE_PCIE :
+			       pci_find_capability(pci_dev, PCI_CAP_ID_AGP) ?
+			       NVKM_DEVICE_AGP : NVKM_DEVICE_PCI,
+			       (u64)pci_domain_nr(pci_dev->bus) << 32 |
+				    pci_dev->bus->number << 16 |
+				    PCI_SLOT(pci_dev->devfn) << 8 |
+				    PCI_FUNC(pci_dev->devfn), name,
+			       cfg, dbg, detect, mmio, subdev_mask,
+			       &pdev->device);
+
+	if (ret)
+		return ret;
+
+	/*
+	 * Set a preliminary DMA mask based on the .dma_bits member of the
+	 * MMU subdevice. This allows other subdevices to create DMA mappings
+	 * in their init() or oneinit() methods, which may be called before the
+	 * TTM layer sets the DMA mask definitively.
+	 * This is necessary for platforms where the default DMA mask of 32
+	 * does not cover any system memory, i.e., when all RAM is > 4 GB.
+	 */
+	if (pdev->device.mmu)
+		dma_set_mask_and_coherent(&pci_dev->dev,
+				DMA_BIT_MASK(pdev->device.mmu->dma_bits));
+
+	return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
index 9b638bd905ff..f2bc0b7d9b93 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
@@ -102,7 +102,7 @@ nvkm_device_tegra_probe_iommu(struct nvkm_device_tegra *tdev)
 
 	if (iommu_present(&platform_bus_type)) {
 		tdev->iommu.domain = iommu_domain_alloc(&platform_bus_type);
-		if (IS_ERR(tdev->iommu.domain))
+		if (!tdev->iommu.domain)
 			goto error;
 
 		/*
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
index 79a8f71cf788..513ee6b79553 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
@@ -326,7 +326,7 @@ nvkm_udevice = {
 	.sclass = nvkm_udevice_child_get,
 };
 
-int
+static int
 nvkm_udevice_new(const struct nvkm_oclass *oclass, void *data, u32 size,
 		 struct nvkm_object **pobject)
 {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild
index 77a52b54a31e..fa05d16ae948 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild
@@ -11,7 +11,7 @@ nvkm-y += nvkm/engine/disp/gk110.o
 nvkm-y += nvkm/engine/disp/gm107.o
 nvkm-y += nvkm/engine/disp/gm200.o
 nvkm-y += nvkm/engine/disp/gp100.o
-nvkm-y += nvkm/engine/disp/gp104.o
+nvkm-y += nvkm/engine/disp/gp102.o
 
 nvkm-y += nvkm/engine/disp/outp.o
 nvkm-y += nvkm/engine/disp/outpdp.o
@@ -48,14 +48,14 @@ nvkm-y += nvkm/engine/disp/rootgk110.o
 nvkm-y += nvkm/engine/disp/rootgm107.o
 nvkm-y += nvkm/engine/disp/rootgm200.o
 nvkm-y += nvkm/engine/disp/rootgp100.o
-nvkm-y += nvkm/engine/disp/rootgp104.o
+nvkm-y += nvkm/engine/disp/rootgp102.o
 
 nvkm-y += nvkm/engine/disp/channv50.o
 nvkm-y += nvkm/engine/disp/changf119.o
 
 nvkm-y += nvkm/engine/disp/dmacnv50.o
 nvkm-y += nvkm/engine/disp/dmacgf119.o
-nvkm-y += nvkm/engine/disp/dmacgp104.o
+nvkm-y += nvkm/engine/disp/dmacgp102.o
 
 nvkm-y += nvkm/engine/disp/basenv50.o
 nvkm-y += nvkm/engine/disp/baseg84.o
@@ -64,7 +64,7 @@ nvkm-y += nvkm/engine/disp/basegt215.o
 nvkm-y += nvkm/engine/disp/basegf119.o
 nvkm-y += nvkm/engine/disp/basegk104.o
 nvkm-y += nvkm/engine/disp/basegk110.o
-nvkm-y += nvkm/engine/disp/basegp104.o
+nvkm-y += nvkm/engine/disp/basegp102.o
 
 nvkm-y += nvkm/engine/disp/corenv50.o
 nvkm-y += nvkm/engine/disp/coreg84.o
@@ -77,7 +77,7 @@ nvkm-y += nvkm/engine/disp/coregk110.o
 nvkm-y += nvkm/engine/disp/coregm107.o
 nvkm-y += nvkm/engine/disp/coregm200.o
 nvkm-y += nvkm/engine/disp/coregp100.o
-nvkm-y += nvkm/engine/disp/coregp104.o
+nvkm-y += nvkm/engine/disp/coregp102.o
 
 nvkm-y += nvkm/engine/disp/ovlynv50.o
 nvkm-y += nvkm/engine/disp/ovlyg84.o
@@ -85,7 +85,7 @@ nvkm-y += nvkm/engine/disp/ovlygt200.o
 nvkm-y += nvkm/engine/disp/ovlygt215.o
 nvkm-y += nvkm/engine/disp/ovlygf119.o
 nvkm-y += nvkm/engine/disp/ovlygk104.o
-nvkm-y += nvkm/engine/disp/ovlygp104.o
+nvkm-y += nvkm/engine/disp/ovlygp102.o
 
 nvkm-y += nvkm/engine/disp/piocnv50.o
 nvkm-y += nvkm/engine/disp/piocgf119.o
@@ -95,9 +95,11 @@ nvkm-y += nvkm/engine/disp/cursg84.o
 nvkm-y += nvkm/engine/disp/cursgt215.o
 nvkm-y += nvkm/engine/disp/cursgf119.o
 nvkm-y += nvkm/engine/disp/cursgk104.o
+nvkm-y += nvkm/engine/disp/cursgp102.o
 
 nvkm-y += nvkm/engine/disp/oimmnv50.o
 nvkm-y += nvkm/engine/disp/oimmg84.o
 nvkm-y += nvkm/engine/disp/oimmgt215.o
 nvkm-y += nvkm/engine/disp/oimmgf119.o
 nvkm-y += nvkm/engine/disp/oimmgk104.o
+nvkm-y += nvkm/engine/disp/oimmgp102.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegp102.c
index 51688e37c54e..8a3cdeef8d2c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegp104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegp102.c
@@ -27,12 +27,12 @@
 #include <nvif/class.h>
 
 const struct nv50_disp_dmac_oclass
-gp104_disp_base_oclass = {
+gp102_disp_base_oclass = {
 	.base.oclass = GK110_DISP_BASE_CHANNEL_DMA,
 	.base.minver = 0,
 	.base.maxver = 0,
 	.ctor = nv50_disp_base_new,
-	.func = &gp104_disp_dmac_func,
+	.func = &gp102_disp_dmac_func,
 	.mthd = &gf119_disp_base_chan_mthd,
 	.chid = 1,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
index dd2953bc9264..524a24eae1a0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c
@@ -82,7 +82,7 @@ nv50_disp_chan_mthd(struct nv50_disp_chan *chan, int debug)
 
 			if (mthd->addr) {
 				snprintf(cname_, sizeof(cname_), "%s %d",
-					 mthd->name, chan->chid);
+					 mthd->name, chan->chid.user);
 				cname = cname_;
 			}
 
@@ -139,7 +139,7 @@ nv50_disp_chan_uevent_ctor(struct nvkm_object *object, void *data, u32 size,
 	if (!(ret = nvif_unvers(ret, &data, &size, args->none))) {
 		notify->size  = sizeof(struct nvif_notify_uevent_rep);
 		notify->types = 1;
-		notify->index = chan->chid;
+		notify->index = chan->chid.user;
 		return 0;
 	}
 
@@ -153,27 +153,27 @@ nv50_disp_chan_uevent = {
 	.fini = nv50_disp_chan_uevent_fini,
 };
 
-int
+static int
 nv50_disp_chan_rd32(struct nvkm_object *object, u64 addr, u32 *data)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
 	struct nv50_disp *disp = chan->root->disp;
 	struct nvkm_device *device = disp->base.engine.subdev.device;
-	*data = nvkm_rd32(device, 0x640000 + (chan->chid * 0x1000) + addr);
+	*data = nvkm_rd32(device, 0x640000 + (chan->chid.user * 0x1000) + addr);
 	return 0;
 }
 
-int
+static int
 nv50_disp_chan_wr32(struct nvkm_object *object, u64 addr, u32 data)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
 	struct nv50_disp *disp = chan->root->disp;
 	struct nvkm_device *device = disp->base.engine.subdev.device;
-	nvkm_wr32(device, 0x640000 + (chan->chid * 0x1000) + addr, data);
+	nvkm_wr32(device, 0x640000 + (chan->chid.user * 0x1000) + addr, data);
 	return 0;
 }
 
-int
+static int
 nv50_disp_chan_ntfy(struct nvkm_object *object, u32 type,
 		    struct nvkm_event **pevent)
 {
@@ -189,14 +189,14 @@ nv50_disp_chan_ntfy(struct nvkm_object *object, u32 type,
 	return -EINVAL;
 }
 
-int
+static int
 nv50_disp_chan_map(struct nvkm_object *object, u64 *addr, u32 *size)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
 	struct nv50_disp *disp = chan->root->disp;
 	struct nvkm_device *device = disp->base.engine.subdev.device;
 	*addr = device->func->resource_addr(device, 0) +
-		0x640000 + (chan->chid * 0x1000);
+		0x640000 + (chan->chid.user * 0x1000);
 	*size = 0x001000;
 	return 0;
 }
@@ -243,8 +243,8 @@ nv50_disp_chan_dtor(struct nvkm_object *object)
 {
 	struct nv50_disp_chan *chan = nv50_disp_chan(object);
 	struct nv50_disp *disp = chan->root->disp;
-	if (chan->chid >= 0)
-		disp->chan[chan->chid] = NULL;
+	if (chan->chid.user >= 0)
+		disp->chan[chan->chid.user] = NULL;
 	return chan->func->dtor ? chan->func->dtor(chan) : chan;
 }
 
@@ -263,7 +263,7 @@ nv50_disp_chan = {
 int
 nv50_disp_chan_ctor(const struct nv50_disp_chan_func *func,
 		    const struct nv50_disp_chan_mthd *mthd,
-		    struct nv50_disp_root *root, int chid, int head,
+		    struct nv50_disp_root *root, int ctrl, int user, int head,
 		    const struct nvkm_oclass *oclass,
 		    struct nv50_disp_chan *chan)
 {
@@ -273,21 +273,22 @@ nv50_disp_chan_ctor(const struct nv50_disp_chan_func *func,
 	chan->func = func;
 	chan->mthd = mthd;
 	chan->root = root;
-	chan->chid = chid;
+	chan->chid.ctrl = ctrl;
+	chan->chid.user = user;
 	chan->head = head;
 
-	if (disp->chan[chan->chid]) {
-		chan->chid = -1;
+	if (disp->chan[chan->chid.user]) {
+		chan->chid.user = -1;
 		return -EBUSY;
 	}
-	disp->chan[chan->chid] = chan;
+	disp->chan[chan->chid.user] = chan;
 	return 0;
 }
 
 int
 nv50_disp_chan_new_(const struct nv50_disp_chan_func *func,
 		    const struct nv50_disp_chan_mthd *mthd,
-		    struct nv50_disp_root *root, int chid, int head,
+		    struct nv50_disp_root *root, int ctrl, int user, int head,
 		    const struct nvkm_oclass *oclass,
 		    struct nvkm_object **pobject)
 {
@@ -297,5 +298,6 @@ nv50_disp_chan_new_(const struct nv50_disp_chan_func *func,
 		return -ENOMEM;
 	*pobject = &chan->object;
 
-	return nv50_disp_chan_ctor(func, mthd, root, chid, head, oclass, chan);
+	return nv50_disp_chan_ctor(func, mthd, root, ctrl, user,
+				   head, oclass, chan);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
index f5f683d9fd20..737b38f6fbd2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
@@ -7,7 +7,11 @@ struct nv50_disp_chan {
 	const struct nv50_disp_chan_func *func;
 	const struct nv50_disp_chan_mthd *mthd;
 	struct nv50_disp_root *root;
-	int chid;
+
+	struct {
+		int ctrl;
+		int user;
+	} chid;
 	int head;
 
 	struct nvkm_object object;
@@ -25,11 +29,11 @@ struct nv50_disp_chan_func {
 
 int nv50_disp_chan_ctor(const struct nv50_disp_chan_func *,
 			const struct nv50_disp_chan_mthd *,
-			struct nv50_disp_root *, int chid, int head,
+			struct nv50_disp_root *, int ctrl, int user, int head,
 			const struct nvkm_oclass *, struct nv50_disp_chan *);
 int nv50_disp_chan_new_(const struct nv50_disp_chan_func *,
 			const struct nv50_disp_chan_mthd *,
-			struct nv50_disp_root *, int chid, int head,
+			struct nv50_disp_root *, int ctrl, int user, int head,
 			const struct nvkm_oclass *, struct nvkm_object **);
 
 extern const struct nv50_disp_chan_func nv50_disp_pioc_func;
@@ -90,13 +94,16 @@ extern const struct nv50_disp_chan_mthd gk104_disp_ovly_chan_mthd;
 struct nv50_disp_pioc_oclass {
 	int (*ctor)(const struct nv50_disp_chan_func *,
 		    const struct nv50_disp_chan_mthd *,
-		    struct nv50_disp_root *, int chid,
+		    struct nv50_disp_root *, int ctrl, int user,
 		    const struct nvkm_oclass *, void *data, u32 size,
 		    struct nvkm_object **);
 	struct nvkm_sclass base;
 	const struct nv50_disp_chan_func *func;
 	const struct nv50_disp_chan_mthd *mthd;
-	int chid;
+	struct {
+		int ctrl;
+		int user;
+	} chid;
 };
 
 extern const struct nv50_disp_pioc_oclass nv50_disp_oimm_oclass;
@@ -114,15 +121,17 @@ extern const struct nv50_disp_pioc_oclass gf119_disp_curs_oclass;
 extern const struct nv50_disp_pioc_oclass gk104_disp_oimm_oclass;
 extern const struct nv50_disp_pioc_oclass gk104_disp_curs_oclass;
 
+extern const struct nv50_disp_pioc_oclass gp102_disp_oimm_oclass;
+extern const struct nv50_disp_pioc_oclass gp102_disp_curs_oclass;
 
 int nv50_disp_curs_new(const struct nv50_disp_chan_func *,
 		       const struct nv50_disp_chan_mthd *,
-		       struct nv50_disp_root *, int chid,
+		       struct nv50_disp_root *, int ctrl, int user,
 		       const struct nvkm_oclass *, void *data, u32 size,
 		       struct nvkm_object **);
 int nv50_disp_oimm_new(const struct nv50_disp_chan_func *,
 		       const struct nv50_disp_chan_mthd *,
-		       struct nv50_disp_root *, int chid,
+		       struct nv50_disp_root *, int ctrl, int user,
 		       const struct nvkm_oclass *, void *data, u32 size,
 		       struct nvkm_object **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c
index 019379a3a01c..c65c9f3ff69f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c
@@ -26,7 +26,7 @@
 
 #include <nvif/class.h>
 
-const struct nv50_disp_mthd_list
+static const struct nv50_disp_mthd_list
 g94_disp_core_mthd_sor = {
 	.mthd = 0x0040,
 	.addr = 0x000008,
@@ -43,8 +43,8 @@ g94_disp_core_chan_mthd = {
 	.prev = 0x000004,
 	.data = {
 		{ "Global", 1, &nv50_disp_core_mthd_base },
-		{    "DAC", 3, &g84_disp_core_mthd_dac  },
-		{    "SOR", 4, &g94_disp_core_mthd_sor  },
+		{    "DAC", 3, &g84_disp_core_mthd_dac },
+		{    "SOR", 4, &g94_disp_core_mthd_sor },
 		{   "PIOR", 3, &nv50_disp_core_mthd_pior },
 		{   "HEAD", 2, &g84_disp_core_mthd_head },
 		{}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c
index 6922f4007b61..b0df4b752b8c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c
@@ -29,7 +29,7 @@
 #include <nvif/class.h>
 
 static int
-gp104_disp_core_init(struct nv50_disp_dmac *chan)
+gp102_disp_core_init(struct nv50_disp_dmac *chan)
 {
 	struct nv50_disp *disp = chan->base.root->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
@@ -59,20 +59,20 @@ gp104_disp_core_init(struct nv50_disp_dmac *chan)
 	return 0;
 }
 
-const struct nv50_disp_dmac_func
-gp104_disp_core_func = {
-	.init = gp104_disp_core_init,
+static const struct nv50_disp_dmac_func
+gp102_disp_core_func = {
+	.init = gp102_disp_core_init,
 	.fini = gf119_disp_core_fini,
 	.bind = gf119_disp_dmac_bind,
 };
 
 const struct nv50_disp_dmac_oclass
-gp104_disp_core_oclass = {
-	.base.oclass = GP104_DISP_CORE_CHANNEL_DMA,
+gp102_disp_core_oclass = {
+	.base.oclass = GP102_DISP_CORE_CHANNEL_DMA,
 	.base.minver = 0,
 	.base.maxver = 0,
 	.ctor = nv50_disp_core_new,
-	.func = &gp104_disp_core_func,
+	.func = &gp102_disp_core_func,
 	.mthd = &gk104_disp_core_chan_mthd,
 	.chid = 0,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursg84.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursg84.c
index dd99fc7060b1..fa781b5a7e07 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursg84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursg84.c
@@ -33,5 +33,5 @@ g84_disp_curs_oclass = {
 	.base.maxver = 0,
 	.ctor = nv50_disp_curs_new,
 	.func = &nv50_disp_pioc_func,
-	.chid = 7,
+	.chid = { 7, 7 },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgf119.c
index 2a1574e06ad6..2be6fb052c65 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgf119.c
@@ -33,5 +33,5 @@ gf119_disp_curs_oclass = {
 	.base.maxver = 0,
 	.ctor = nv50_disp_curs_new,
 	.func = &gf119_disp_pioc_func,
-	.chid = 13,
+	.chid = { 13, 13 },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgk104.c
index 28e8f06c9472..2a99db4bf8f8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgk104.c
@@ -33,5 +33,5 @@ gk104_disp_curs_oclass = {
 	.base.maxver = 0,
 	.ctor = nv50_disp_curs_new,
 	.func = &gf119_disp_pioc_func,
-	.chid = 13,
+	.chid = { 13, 13 },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgp102.c
new file mode 100644
index 000000000000..e958210d8105
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgp102.c
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "channv50.h"
+#include "rootnv50.h"
+
+#include <nvif/class.h>
+
+const struct nv50_disp_pioc_oclass
+gp102_disp_curs_oclass = {
+	.base.oclass = GK104_DISP_CURSOR,
+	.base.minver = 0,
+	.base.maxver = 0,
+	.ctor = nv50_disp_curs_new,
+	.func = &gf119_disp_pioc_func,
+	.chid = { 13, 17 },
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgt215.c
index d8a4b9ca139c..00a7f3564450 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursgt215.c
@@ -33,5 +33,5 @@ gt215_disp_curs_oclass = {
 	.base.maxver = 0,
 	.ctor = nv50_disp_curs_new,
 	.func = &nv50_disp_pioc_func,
-	.chid = 7,
+	.chid = { 7, 7 },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c
index 8b1320499a0f..82ff82d8c1ab 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/cursnv50.c
@@ -33,7 +33,7 @@
 int
 nv50_disp_curs_new(const struct nv50_disp_chan_func *func,
 		   const struct nv50_disp_chan_mthd *mthd,
-		   struct nv50_disp_root *root, int chid,
+		   struct nv50_disp_root *root, int ctrl, int user,
 		   const struct nvkm_oclass *oclass, void *data, u32 size,
 		   struct nvkm_object **pobject)
 {
@@ -54,7 +54,7 @@ nv50_disp_curs_new(const struct nv50_disp_chan_func *func,
 	} else
 		return ret;
 
-	return nv50_disp_chan_new_(func, mthd, root, chid + head,
+	return nv50_disp_chan_new_(func, mthd, root, ctrl + head, user + head,
 				   head, oclass, pobject);
 }
 
@@ -65,5 +65,5 @@ nv50_disp_curs_oclass = {
 	.base.maxver = 0,
 	.ctor = nv50_disp_curs_new,
 	.func = &nv50_disp_pioc_func,
-	.chid = 7,
+	.chid = { 7, 7 },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c
index a57f7cef307a..ce7cd74fbd5d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c
@@ -32,8 +32,8 @@ gf119_disp_dmac_bind(struct nv50_disp_dmac *chan,
 		     struct nvkm_object *object, u32 handle)
 {
 	return nvkm_ramht_insert(chan->base.root->ramht, object,
-				 chan->base.chid, -9, handle,
-				 chan->base.chid << 27 | 0x00000001);
+				 chan->base.chid.user, -9, handle,
+				 chan->base.chid.user << 27 | 0x00000001);
 }
 
 void
@@ -42,22 +42,23 @@ gf119_disp_dmac_fini(struct nv50_disp_dmac *chan)
 	struct nv50_disp *disp = chan->base.root->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int chid = chan->base.chid;
+	int ctrl = chan->base.chid.ctrl;
+	int user = chan->base.chid.user;
 
 	/* deactivate channel */
-	nvkm_mask(device, 0x610490 + (chid * 0x0010), 0x00001010, 0x00001000);
-	nvkm_mask(device, 0x610490 + (chid * 0x0010), 0x00000003, 0x00000000);
+	nvkm_mask(device, 0x610490 + (ctrl * 0x0010), 0x00001010, 0x00001000);
+	nvkm_mask(device, 0x610490 + (ctrl * 0x0010), 0x00000003, 0x00000000);
 	if (nvkm_msec(device, 2000,
-		if (!(nvkm_rd32(device, 0x610490 + (chid * 0x10)) & 0x001e0000))
+		if (!(nvkm_rd32(device, 0x610490 + (ctrl * 0x10)) & 0x001e0000))
 			break;
 	) < 0) {
-		nvkm_error(subdev, "ch %d fini: %08x\n", chid,
-			   nvkm_rd32(device, 0x610490 + (chid * 0x10)));
+		nvkm_error(subdev, "ch %d fini: %08x\n", user,
+			   nvkm_rd32(device, 0x610490 + (ctrl * 0x10)));
 	}
 
 	/* disable error reporting and completion notification */
-	nvkm_mask(device, 0x610090, 0x00000001 << chid, 0x00000000);
-	nvkm_mask(device, 0x6100a0, 0x00000001 << chid, 0x00000000);
+	nvkm_mask(device, 0x610090, 0x00000001 << user, 0x00000000);
+	nvkm_mask(device, 0x6100a0, 0x00000001 << user, 0x00000000);
 }
 
 static int
@@ -66,26 +67,27 @@ gf119_disp_dmac_init(struct nv50_disp_dmac *chan)
 	struct nv50_disp *disp = chan->base.root->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int chid = chan->base.chid;
+	int ctrl = chan->base.chid.ctrl;
+	int user = chan->base.chid.user;
 
 	/* enable error reporting */
-	nvkm_mask(device, 0x6100a0, 0x00000001 << chid, 0x00000001 << chid);
+	nvkm_mask(device, 0x6100a0, 0x00000001 << user, 0x00000001 << user);
 
 	/* initialise channel for dma command submission */
-	nvkm_wr32(device, 0x610494 + (chid * 0x0010), chan->push);
-	nvkm_wr32(device, 0x610498 + (chid * 0x0010), 0x00010000);
-	nvkm_wr32(device, 0x61049c + (chid * 0x0010), 0x00000001);
-	nvkm_mask(device, 0x610490 + (chid * 0x0010), 0x00000010, 0x00000010);
-	nvkm_wr32(device, 0x640000 + (chid * 0x1000), 0x00000000);
-	nvkm_wr32(device, 0x610490 + (chid * 0x0010), 0x00000013);
+	nvkm_wr32(device, 0x610494 + (ctrl * 0x0010), chan->push);
+	nvkm_wr32(device, 0x610498 + (ctrl * 0x0010), 0x00010000);
+	nvkm_wr32(device, 0x61049c + (ctrl * 0x0010), 0x00000001);
+	nvkm_mask(device, 0x610490 + (ctrl * 0x0010), 0x00000010, 0x00000010);
+	nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), 0x00000000);
+	nvkm_wr32(device, 0x610490 + (ctrl * 0x0010), 0x00000013);
 
 	/* wait for it to go inactive */
 	if (nvkm_msec(device, 2000,
-		if (!(nvkm_rd32(device, 0x610490 + (chid * 0x10)) & 0x80000000))
+		if (!(nvkm_rd32(device, 0x610490 + (ctrl * 0x10)) & 0x80000000))
 			break;
 	) < 0) {
-		nvkm_error(subdev, "ch %d init: %08x\n", chid,
-			   nvkm_rd32(device, 0x610490 + (chid * 0x10)));
+		nvkm_error(subdev, "ch %d init: %08x\n", user,
+			   nvkm_rd32(device, 0x610490 + (ctrl * 0x10)));
 		return -EBUSY;
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c
index ad24c2c57696..cdead9500343 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c
@@ -27,31 +27,32 @@
 #include <subdev/timer.h>
 
 static int
-gp104_disp_dmac_init(struct nv50_disp_dmac *chan)
+gp102_disp_dmac_init(struct nv50_disp_dmac *chan)
 {
 	struct nv50_disp *disp = chan->base.root->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int chid = chan->base.chid;
+	int ctrl = chan->base.chid.ctrl;
+	int user = chan->base.chid.user;
 
 	/* enable error reporting */
-	nvkm_mask(device, 0x6100a0, 0x00000001 << chid, 0x00000001 << chid);
+	nvkm_mask(device, 0x6100a0, 0x00000001 << user, 0x00000001 << user);
 
 	/* initialise channel for dma command submission */
-	nvkm_wr32(device, 0x611494 + (chid * 0x0010), chan->push);
-	nvkm_wr32(device, 0x611498 + (chid * 0x0010), 0x00010000);
-	nvkm_wr32(device, 0x61149c + (chid * 0x0010), 0x00000001);
-	nvkm_mask(device, 0x610490 + (chid * 0x0010), 0x00000010, 0x00000010);
-	nvkm_wr32(device, 0x640000 + (chid * 0x1000), 0x00000000);
-	nvkm_wr32(device, 0x610490 + (chid * 0x0010), 0x00000013);
+	nvkm_wr32(device, 0x611494 + (ctrl * 0x0010), chan->push);
+	nvkm_wr32(device, 0x611498 + (ctrl * 0x0010), 0x00010000);
+	nvkm_wr32(device, 0x61149c + (ctrl * 0x0010), 0x00000001);
+	nvkm_mask(device, 0x610490 + (ctrl * 0x0010), 0x00000010, 0x00000010);
+	nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), 0x00000000);
+	nvkm_wr32(device, 0x610490 + (ctrl * 0x0010), 0x00000013);
 
 	/* wait for it to go inactive */
 	if (nvkm_msec(device, 2000,
-		if (!(nvkm_rd32(device, 0x610490 + (chid * 0x10)) & 0x80000000))
+		if (!(nvkm_rd32(device, 0x610490 + (ctrl * 0x10)) & 0x80000000))
 			break;
 	) < 0) {
-		nvkm_error(subdev, "ch %d init: %08x\n", chid,
-			   nvkm_rd32(device, 0x610490 + (chid * 0x10)));
+		nvkm_error(subdev, "ch %d init: %08x\n", user,
+			   nvkm_rd32(device, 0x610490 + (ctrl * 0x10)));
 		return -EBUSY;
 	}
 
@@ -59,8 +60,8 @@ gp104_disp_dmac_init(struct nv50_disp_dmac *chan)
 }
 
 const struct nv50_disp_dmac_func
-gp104_disp_dmac_func = {
-	.init = gp104_disp_dmac_init,
+gp102_disp_dmac_func = {
+	.init = gp102_disp_dmac_init,
 	.fini = gf119_disp_dmac_fini,
 	.bind = gf119_disp_dmac_bind,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c
index 9c6645a357b9..0a1381a84552 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c
@@ -149,7 +149,7 @@ nv50_disp_dmac_new_(const struct nv50_disp_dmac_func *func,
 	chan->func = func;
 
 	ret = nv50_disp_chan_ctor(&nv50_disp_dmac_func_, mthd, root,
-				  chid, head, oclass, &chan->base);
+				  chid, chid, head, oclass, &chan->base);
 	if (ret)
 		return ret;
 
@@ -179,9 +179,9 @@ nv50_disp_dmac_bind(struct nv50_disp_dmac *chan,
 		    struct nvkm_object *object, u32 handle)
 {
 	return nvkm_ramht_insert(chan->base.root->ramht, object,
-				 chan->base.chid, -10, handle,
-				 chan->base.chid << 28 |
-				 chan->base.chid);
+				 chan->base.chid.user, -10, handle,
+				 chan->base.chid.user << 28 |
+				 chan->base.chid.user);
 }
 
 static void
@@ -190,21 +190,22 @@ nv50_disp_dmac_fini(struct nv50_disp_dmac *chan)
 	struct nv50_disp *disp = chan->base.root->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int chid = chan->base.chid;
+	int ctrl = chan->base.chid.ctrl;
+	int user = chan->base.chid.user;
 
 	/* deactivate channel */
-	nvkm_mask(device, 0x610200 + (chid * 0x0010), 0x00001010, 0x00001000);
-	nvkm_mask(device, 0x610200 + (chid * 0x0010), 0x00000003, 0x00000000);
+	nvkm_mask(device, 0x610200 + (ctrl * 0x0010), 0x00001010, 0x00001000);
+	nvkm_mask(device, 0x610200 + (ctrl * 0x0010), 0x00000003, 0x00000000);
 	if (nvkm_msec(device, 2000,
-		if (!(nvkm_rd32(device, 0x610200 + (chid * 0x10)) & 0x001e0000))
+		if (!(nvkm_rd32(device, 0x610200 + (ctrl * 0x10)) & 0x001e0000))
 			break;
 	) < 0) {
-		nvkm_error(subdev, "ch %d fini timeout, %08x\n", chid,
-			   nvkm_rd32(device, 0x610200 + (chid * 0x10)));
+		nvkm_error(subdev, "ch %d fini timeout, %08x\n", user,
+			   nvkm_rd32(device, 0x610200 + (ctrl * 0x10)));
 	}
 
 	/* disable error reporting and completion notifications */
-	nvkm_mask(device, 0x610028, 0x00010001 << chid, 0x00000000 << chid);
+	nvkm_mask(device, 0x610028, 0x00010001 << user, 0x00000000 << user);
 }
 
 static int
@@ -213,26 +214,27 @@ nv50_disp_dmac_init(struct nv50_disp_dmac *chan)
 	struct nv50_disp *disp = chan->base.root->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int chid = chan->base.chid;
+	int ctrl = chan->base.chid.ctrl;
+	int user = chan->base.chid.user;
 
 	/* enable error reporting */
-	nvkm_mask(device, 0x610028, 0x00010000 << chid, 0x00010000 << chid);
+	nvkm_mask(device, 0x610028, 0x00010000 << user, 0x00010000 << user);
 
 	/* initialise channel for dma command submission */
-	nvkm_wr32(device, 0x610204 + (chid * 0x0010), chan->push);
-	nvkm_wr32(device, 0x610208 + (chid * 0x0010), 0x00010000);
-	nvkm_wr32(device, 0x61020c + (chid * 0x0010), chid);
-	nvkm_mask(device, 0x610200 + (chid * 0x0010), 0x00000010, 0x00000010);
-	nvkm_wr32(device, 0x640000 + (chid * 0x1000), 0x00000000);
-	nvkm_wr32(device, 0x610200 + (chid * 0x0010), 0x00000013);
+	nvkm_wr32(device, 0x610204 + (ctrl * 0x0010), chan->push);
+	nvkm_wr32(device, 0x610208 + (ctrl * 0x0010), 0x00010000);
+	nvkm_wr32(device, 0x61020c + (ctrl * 0x0010), ctrl);
+	nvkm_mask(device, 0x610200 + (ctrl * 0x0010), 0x00000010, 0x00000010);
+	nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), 0x00000000);
+	nvkm_wr32(device, 0x610200 + (ctrl * 0x0010), 0x00000013);
 
 	/* wait for it to go inactive */
 	if (nvkm_msec(device, 2000,
-		if (!(nvkm_rd32(device, 0x610200 + (chid * 0x10)) & 0x80000000))
+		if (!(nvkm_rd32(device, 0x610200 + (ctrl * 0x10)) & 0x80000000))
 			break;
 	) < 0) {
-		nvkm_error(subdev, "ch %d init timeout, %08x\n", chid,
-			   nvkm_rd32(device, 0x610200 + (chid * 0x10)));
+		nvkm_error(subdev, "ch %d init timeout, %08x\n", user,
+			   nvkm_rd32(device, 0x610200 + (ctrl * 0x10)));
 		return -EBUSY;
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h
index 43ac05857853..ea4a0d062e31 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h
@@ -30,7 +30,7 @@ int gf119_disp_dmac_bind(struct nv50_disp_dmac *, struct nvkm_object *, u32);
 extern const struct nv50_disp_dmac_func gf119_disp_core_func;
 void gf119_disp_core_fini(struct nv50_disp_dmac *);
 
-extern const struct nv50_disp_dmac_func gp104_disp_dmac_func;
+extern const struct nv50_disp_dmac_func gp102_disp_dmac_func;
 
 struct nv50_disp_dmac_oclass {
 	int (*ctor)(const struct nv50_disp_dmac_func *,
@@ -95,7 +95,7 @@ extern const struct nv50_disp_dmac_oclass gm200_disp_core_oclass;
 
 extern const struct nv50_disp_dmac_oclass gp100_disp_core_oclass;
 
-extern const struct nv50_disp_dmac_oclass gp104_disp_core_oclass;
-extern const struct nv50_disp_dmac_oclass gp104_disp_base_oclass;
-extern const struct nv50_disp_dmac_oclass gp104_disp_ovly_oclass;
+extern const struct nv50_disp_dmac_oclass gp102_disp_core_oclass;
+extern const struct nv50_disp_dmac_oclass gp102_disp_base_oclass;
+extern const struct nv50_disp_dmac_oclass gp102_disp_ovly_oclass;
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.c
index 9688970eca47..4a93ceb850ac 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.c
@@ -319,9 +319,8 @@ static const struct dp_rates {
 };
 
 void
-nvkm_dp_train(struct work_struct *w)
+nvkm_dp_train(struct nvkm_output_dp *outp)
 {
-	struct nvkm_output_dp *outp = container_of(w, typeof(*outp), lt.work);
 	struct nv50_disp *disp = nv50_disp(outp->base.disp);
 	const struct dp_rates *cfg = nvkm_dp_rates;
 	struct dp_state _dp = {
@@ -353,9 +352,6 @@ nvkm_dp_train(struct work_struct *w)
 	}
 	cfg--;
 
-	/* disable link interrupt handling during link training */
-	nvkm_notify_put(&outp->irq);
-
 	/* ensure sink is not in a low-power state */
 	if (!nvkm_rdaux(outp->aux, DPCD_SC00, &pwr, 1)) {
 		if ((pwr & DPCD_SC00_SET_POWER) != DPCD_SC00_SET_POWER_D0) {
@@ -400,9 +396,6 @@ nvkm_dp_train(struct work_struct *w)
 
 	dp_link_train_fini(dp);
 
-	/* signal completion and enable link interrupt handling */
 	OUTP_DBG(&outp->base, "training complete");
 	atomic_set(&outp->lt.done, 1);
-	wake_up(&outp->lt.wait);
-	nvkm_notify_get(&outp->irq);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.h
index 6e10c5e0ef11..baf1dd9ff975 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.h
@@ -1,6 +1,6 @@
 #ifndef __NVKM_DISP_DPORT_H__
 #define __NVKM_DISP_DPORT_H__
-#include <core/os.h>
+struct nvkm_output_dp;
 
 /* DPCD Receiver Capabilities */
 #define DPCD_RC00_DPCD_REV                                              0x00000
@@ -77,5 +77,5 @@
 #define DPCD_SC00_SET_POWER_D0                                             0x01
 #define DPCD_SC00_SET_POWER_D3                                             0x03
 
-void nvkm_dp_train(struct work_struct *);
+void nvkm_dp_train(struct nvkm_output_dp *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c
index 29e84b241cca..7b346ccc38b7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c
@@ -203,17 +203,20 @@ gf119_disp_intr_unk2_0(struct nv50_disp *disp, int head)
 	/* see note in nv50_disp_intr_unk20_0() */
 	if (outp && outp->info.type == DCB_OUTPUT_DP) {
 		struct nvkm_output_dp *outpdp = nvkm_output_dp(outp);
-		struct nvbios_init init = {
-			.subdev = subdev,
-			.bios = subdev->device->bios,
-			.outp = &outp->info,
-			.crtc = head,
-			.offset = outpdp->info.script[4],
-			.execute = 1,
-		};
+		if (!outpdp->lt.mst) {
+			struct nvbios_init init = {
+				.subdev = subdev,
+				.bios = subdev->device->bios,
+				.outp = &outp->info,
+				.crtc = head,
+				.offset = outpdp->info.script[4],
+				.execute = 1,
+			};
 
-		nvbios_exec(&init);
-		atomic_set(&outpdp->lt.done, 0);
+			nvkm_notify_put(&outpdp->irq);
+			nvbios_exec(&init);
+			atomic_set(&outpdp->lt.done, 0);
+		}
 	}
 }
 
@@ -314,7 +317,7 @@ gf119_disp_intr_unk2_2(struct nv50_disp *disp, int head)
 			break;
 		}
 
-		if (nvkm_output_dp_train(outp, pclk, true))
+		if (nvkm_output_dp_train(outp, pclk))
 			OUTP_ERR(outp, "link not trained before attach");
 	} else {
 		if (disp->func->sor.magic)
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c
index 3bf3380336e4..f5d613f82709 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c
@@ -25,7 +25,7 @@
 #include "rootnv50.h"
 
 static void
-gp104_disp_intr_error(struct nv50_disp *disp, int chid)
+gp102_disp_intr_error(struct nv50_disp *disp, int chid)
 {
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
@@ -51,12 +51,12 @@ gp104_disp_intr_error(struct nv50_disp *disp, int chid)
 }
 
 static const struct nv50_disp_func
-gp104_disp = {
+gp102_disp = {
 	.intr = gf119_disp_intr,
-	.intr_error = gp104_disp_intr_error,
+	.intr_error = gp102_disp_intr_error,
 	.uevent = &gf119_disp_chan_uevent,
 	.super = gf119_disp_intr_supervisor,
-	.root = &gp104_disp_root_oclass,
+	.root = &gp102_disp_root_oclass,
 	.head.vblank_init = gf119_disp_vblank_init,
 	.head.vblank_fini = gf119_disp_vblank_fini,
 	.head.scanoutpos = gf119_disp_root_scanoutpos,
@@ -75,7 +75,7 @@ gp104_disp = {
 };
 
 int
-gp104_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
+gp102_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
 {
-	return gf119_disp_new_(&gp104_disp, device, index, pdisp);
+	return gf119_disp_new_(&gp102_disp, device, index, pdisp);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c
index fbb8c7dc18fd..567466f93cd5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c
@@ -590,6 +590,7 @@ nv50_disp_intr_unk20_0(struct nv50_disp *disp, int head)
 			.execute = 1,
 		};
 
+		nvkm_notify_put(&outpdp->irq);
 		nvbios_exec(&init);
 		atomic_set(&outpdp->lt.done, 0);
 	}
@@ -779,7 +780,7 @@ nv50_disp_intr_unk20_2(struct nv50_disp *disp, int head)
 			break;
 		}
 
-		if (nvkm_output_dp_train(outp, datarate / soff, true))
+		if (nvkm_output_dp_train(outp, datarate / soff))
 			OUTP_ERR(outp, "link not trained before attach");
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmg84.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmg84.c
index 54a4ae8d66c6..5ad5d0f5db05 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmg84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmg84.c
@@ -33,5 +33,5 @@ g84_disp_oimm_oclass = {
 	.base.maxver = 0,
 	.ctor = nv50_disp_oimm_new,
 	.func = &nv50_disp_pioc_func,
-	.chid = 5,
+	.chid = { 5, 5 },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgf119.c
index c658db54afc5..1f9fd3403f07 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgf119.c
@@ -33,5 +33,5 @@ gf119_disp_oimm_oclass = {
 	.base.maxver = 0,
 	.ctor = nv50_disp_oimm_new,
 	.func = &gf119_disp_pioc_func,
-	.chid = 9,
+	.chid = { 9, 9 },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgk104.c
index b1fde8c125d6..0c09fe85e952 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgk104.c
@@ -33,5 +33,5 @@ gk104_disp_oimm_oclass = {
 	.base.maxver = 0,
 	.ctor = nv50_disp_oimm_new,
 	.func = &gf119_disp_pioc_func,
-	.chid = 9,
+	.chid = { 9, 9 },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgp102.c
new file mode 100644
index 000000000000..abf82365c671
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgp102.c
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "channv50.h"
+#include "rootnv50.h"
+
+#include <nvif/class.h>
+
+const struct nv50_disp_pioc_oclass
+gp102_disp_oimm_oclass = {
+	.base.oclass = GK104_DISP_OVERLAY,
+	.base.minver = 0,
+	.base.maxver = 0,
+	.ctor = nv50_disp_oimm_new,
+	.func = &gf119_disp_pioc_func,
+	.chid = { 9, 13 },
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgt215.c
index f4e7eb3d1177..1281db28aebd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmgt215.c
@@ -33,5 +33,5 @@ gt215_disp_oimm_oclass = {
 	.base.maxver = 0,
 	.ctor = nv50_disp_oimm_new,
 	.func = &nv50_disp_pioc_func,
-	.chid = 5,
+	.chid = { 5, 5 },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c
index 3940b9c966ec..07540f3d32dc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/oimmnv50.c
@@ -33,7 +33,7 @@
 int
 nv50_disp_oimm_new(const struct nv50_disp_chan_func *func,
 		   const struct nv50_disp_chan_mthd *mthd,
-		   struct nv50_disp_root *root, int chid,
+		   struct nv50_disp_root *root, int ctrl, int user,
 		   const struct nvkm_oclass *oclass, void *data, u32 size,
 		   struct nvkm_object **pobject)
 {
@@ -54,7 +54,7 @@ nv50_disp_oimm_new(const struct nv50_disp_chan_func *func,
 	} else
 		return ret;
 
-	return nv50_disp_chan_new_(func, mthd, root, chid + head,
+	return nv50_disp_chan_new_(func, mthd, root, ctrl + head, user + head,
 				   head, oclass, pobject);
 }
 
@@ -65,5 +65,5 @@ nv50_disp_oimm_oclass = {
 	.base.maxver = 0,
 	.ctor = nv50_disp_oimm_new,
 	.func = &nv50_disp_pioc_func,
-	.chid = 5,
+	.chid = { 5, 5 },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.c
index 3b7a9e7a1ea8..de36f73b14dc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.c
@@ -31,7 +31,7 @@
 #include <nvif/event.h>
 
 int
-nvkm_output_dp_train(struct nvkm_output *base, u32 datarate, bool wait)
+nvkm_output_dp_train(struct nvkm_output *base, u32 datarate)
 {
 	struct nvkm_output_dp *outp = nvkm_output_dp(base);
 	bool retrain = true;
@@ -39,6 +39,8 @@ nvkm_output_dp_train(struct nvkm_output *base, u32 datarate, bool wait)
 	u32 linkrate;
 	int ret, i;
 
+	mutex_lock(&outp->mutex);
+
 	/* check that the link is trained at a high enough rate */
 	ret = nvkm_rdaux(outp->aux, DPCD_LC00_LINK_BW_SET, link, 2);
 	if (ret) {
@@ -88,19 +90,10 @@ done:
 			outp->dpcd[DPCD_RC02] =
 				outp->base.info.dpconf.link_nr;
 		}
-		atomic_set(&outp->lt.done, 0);
-		schedule_work(&outp->lt.work);
-	} else {
-		nvkm_notify_get(&outp->irq);
-	}
-
-	if (wait) {
-		if (!wait_event_timeout(outp->lt.wait,
-					atomic_read(&outp->lt.done),
-					msecs_to_jiffies(2000)))
-			ret = -ETIMEDOUT;
+		nvkm_dp_train(outp);
 	}
 
+	mutex_unlock(&outp->mutex);
 	return ret;
 }
 
@@ -118,7 +111,7 @@ nvkm_output_dp_enable(struct nvkm_output_dp *outp, bool enable)
 
 		if (!nvkm_rdaux(aux, DPCD_RC00_DPCD_REV, outp->dpcd,
 				sizeof(outp->dpcd))) {
-			nvkm_output_dp_train(&outp->base, 0, true);
+			nvkm_output_dp_train(&outp->base, 0);
 			return;
 		}
 	}
@@ -165,10 +158,10 @@ nvkm_output_dp_irq(struct nvkm_notify *notify)
 	};
 
 	OUTP_DBG(&outp->base, "IRQ: %d", line->mask);
-	nvkm_output_dp_train(&outp->base, 0, true);
+	nvkm_output_dp_train(&outp->base, 0);
 
 	nvkm_event_send(&disp->hpd, rep.mask, conn->index, &rep, sizeof(rep));
-	return NVKM_NOTIFY_DROP;
+	return NVKM_NOTIFY_KEEP;
 }
 
 static void
@@ -177,7 +170,6 @@ nvkm_output_dp_fini(struct nvkm_output *base)
 	struct nvkm_output_dp *outp = nvkm_output_dp(base);
 	nvkm_notify_put(&outp->hpd);
 	nvkm_notify_put(&outp->irq);
-	flush_work(&outp->lt.work);
 	nvkm_output_dp_enable(outp, false);
 }
 
@@ -187,6 +179,7 @@ nvkm_output_dp_init(struct nvkm_output *base)
 	struct nvkm_output_dp *outp = nvkm_output_dp(base);
 	nvkm_notify_put(&outp->base.conn->hpd);
 	nvkm_output_dp_enable(outp, true);
+	nvkm_notify_get(&outp->irq);
 	nvkm_notify_get(&outp->hpd);
 }
 
@@ -238,11 +231,6 @@ nvkm_output_dp_ctor(const struct nvkm_output_dp_func *func,
 	OUTP_DBG(&outp->base, "bios dp %02x %02x %02x %02x",
 		 outp->version, hdr, cnt, len);
 
-	/* link training */
-	INIT_WORK(&outp->lt.work, nvkm_dp_train);
-	init_waitqueue_head(&outp->lt.wait);
-	atomic_set(&outp->lt.done, 0);
-
 	/* link maintenance */
 	ret = nvkm_notify_init(NULL, &i2c->event, nvkm_output_dp_irq, true,
 			       &(struct nvkm_i2c_ntfy_req) {
@@ -257,6 +245,9 @@ nvkm_output_dp_ctor(const struct nvkm_output_dp_func *func,
 		return ret;
 	}
 
+	mutex_init(&outp->mutex);
+	atomic_set(&outp->lt.done, 0);
+
 	/* hotplug detect, replaces gpio-based mechanism with aux events */
 	ret = nvkm_notify_init(NULL, &i2c->event, nvkm_output_dp_hpd, true,
 			       &(struct nvkm_i2c_ntfy_req) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.h
index 4e983f6d7032..3c83a561cd88 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.h
@@ -29,10 +29,10 @@ struct nvkm_output_dp {
 	bool present;
 	u8 dpcd[16];
 
+	struct mutex mutex;
 	struct {
-		struct work_struct work;
-		wait_queue_head_t wait;
 		atomic_t done;
+		bool mst;
 	} lt;
 };
 
@@ -41,9 +41,11 @@ struct nvkm_output_dp_func {
 	int (*lnk_pwr)(struct nvkm_output_dp *, int nr);
 	int (*lnk_ctl)(struct nvkm_output_dp *, int nr, int bw, bool ef);
 	int (*drv_ctl)(struct nvkm_output_dp *, int ln, int vs, int pe, int pc);
+	void (*vcpi)(struct nvkm_output_dp *, int head, u8 start_slot,
+		     u8 num_slots, u16 pbn, u16 aligned_pbn);
 };
 
-int nvkm_output_dp_train(struct nvkm_output *, u32 rate, bool wait);
+int nvkm_output_dp_train(struct nvkm_output *, u32 rate);
 
 int nvkm_output_dp_ctor(const struct nvkm_output_dp_func *, struct nvkm_disp *,
 			int index, struct dcb_output *, struct nvkm_i2c_aux *,
@@ -63,6 +65,7 @@ int gf119_sor_dp_new(struct nvkm_disp *, int, struct dcb_output *,
 		     struct nvkm_output **);
 int gf119_sor_dp_lnk_ctl(struct nvkm_output_dp *, int, int, bool);
 int gf119_sor_dp_drv_ctl(struct nvkm_output_dp *, int, int, int, int);
+void gf119_sor_dp_vcpi(struct nvkm_output_dp *, int, u8, u8, u16, u16);
 
 int gm107_sor_dp_new(struct nvkm_disp *, int, struct dcb_output *,
 		     struct nvkm_output **);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygp102.c
index 97e2dd2d908e..589bd2f12b41 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygp104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygp102.c
@@ -27,12 +27,12 @@
 #include <nvif/class.h>
 
 const struct nv50_disp_dmac_oclass
-gp104_disp_ovly_oclass = {
+gp102_disp_ovly_oclass = {
 	.base.oclass = GK104_DISP_OVERLAY_CONTROL_DMA,
 	.base.minver = 0,
 	.base.maxver = 0,
 	.ctor = nv50_disp_ovly_new,
-	.func = &gp104_disp_dmac_func,
+	.func = &gp102_disp_dmac_func,
 	.mthd = &gk104_disp_ovly_chan_mthd,
 	.chid = 5,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocgf119.c
index a625a9876e34..0abaa6431943 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocgf119.c
@@ -32,20 +32,21 @@ gf119_disp_pioc_fini(struct nv50_disp_chan *chan)
 	struct nv50_disp *disp = chan->root->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int chid = chan->chid;
+	int ctrl = chan->chid.ctrl;
+	int user = chan->chid.user;
 
-	nvkm_mask(device, 0x610490 + (chid * 0x10), 0x00000001, 0x00000000);
+	nvkm_mask(device, 0x610490 + (ctrl * 0x10), 0x00000001, 0x00000000);
 	if (nvkm_msec(device, 2000,
-		if (!(nvkm_rd32(device, 0x610490 + (chid * 0x10)) & 0x00030000))
+		if (!(nvkm_rd32(device, 0x610490 + (ctrl * 0x10)) & 0x00030000))
 			break;
 	) < 0) {
-		nvkm_error(subdev, "ch %d fini: %08x\n", chid,
-			   nvkm_rd32(device, 0x610490 + (chid * 0x10)));
+		nvkm_error(subdev, "ch %d fini: %08x\n", user,
+			   nvkm_rd32(device, 0x610490 + (ctrl * 0x10)));
 	}
 
 	/* disable error reporting and completion notification */
-	nvkm_mask(device, 0x610090, 0x00000001 << chid, 0x00000000);
-	nvkm_mask(device, 0x6100a0, 0x00000001 << chid, 0x00000000);
+	nvkm_mask(device, 0x610090, 0x00000001 << user, 0x00000000);
+	nvkm_mask(device, 0x6100a0, 0x00000001 << user, 0x00000000);
 }
 
 static int
@@ -54,20 +55,21 @@ gf119_disp_pioc_init(struct nv50_disp_chan *chan)
 	struct nv50_disp *disp = chan->root->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int chid = chan->chid;
+	int ctrl = chan->chid.ctrl;
+	int user = chan->chid.user;
 
 	/* enable error reporting */
-	nvkm_mask(device, 0x6100a0, 0x00000001 << chid, 0x00000001 << chid);
+	nvkm_mask(device, 0x6100a0, 0x00000001 << user, 0x00000001 << user);
 
 	/* activate channel */
-	nvkm_wr32(device, 0x610490 + (chid * 0x10), 0x00000001);
+	nvkm_wr32(device, 0x610490 + (ctrl * 0x10), 0x00000001);
 	if (nvkm_msec(device, 2000,
-		u32 tmp = nvkm_rd32(device, 0x610490 + (chid * 0x10));
+		u32 tmp = nvkm_rd32(device, 0x610490 + (ctrl * 0x10));
 		if ((tmp & 0x00030000) == 0x00010000)
 			break;
 	) < 0) {
-		nvkm_error(subdev, "ch %d init: %08x\n", chid,
-			   nvkm_rd32(device, 0x610490 + (chid * 0x10)));
+		nvkm_error(subdev, "ch %d init: %08x\n", user,
+			   nvkm_rd32(device, 0x610490 + (ctrl * 0x10)));
 		return -EBUSY;
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocnv50.c
index 9d2618dacf20..0211e0e8a35f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/piocnv50.c
@@ -32,15 +32,16 @@ nv50_disp_pioc_fini(struct nv50_disp_chan *chan)
 	struct nv50_disp *disp = chan->root->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int chid = chan->chid;
+	int ctrl = chan->chid.ctrl;
+	int user = chan->chid.user;
 
-	nvkm_mask(device, 0x610200 + (chid * 0x10), 0x00000001, 0x00000000);
+	nvkm_mask(device, 0x610200 + (ctrl * 0x10), 0x00000001, 0x00000000);
 	if (nvkm_msec(device, 2000,
-		if (!(nvkm_rd32(device, 0x610200 + (chid * 0x10)) & 0x00030000))
+		if (!(nvkm_rd32(device, 0x610200 + (ctrl * 0x10)) & 0x00030000))
 			break;
 	) < 0) {
-		nvkm_error(subdev, "ch %d timeout: %08x\n", chid,
-			   nvkm_rd32(device, 0x610200 + (chid * 0x10)));
+		nvkm_error(subdev, "ch %d timeout: %08x\n", user,
+			   nvkm_rd32(device, 0x610200 + (ctrl * 0x10)));
 	}
 }
 
@@ -50,26 +51,27 @@ nv50_disp_pioc_init(struct nv50_disp_chan *chan)
 	struct nv50_disp *disp = chan->root->disp;
 	struct nvkm_subdev *subdev = &disp->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	int chid = chan->chid;
+	int ctrl = chan->chid.ctrl;
+	int user = chan->chid.user;
 
-	nvkm_wr32(device, 0x610200 + (chid * 0x10), 0x00002000);
+	nvkm_wr32(device, 0x610200 + (ctrl * 0x10), 0x00002000);
 	if (nvkm_msec(device, 2000,
-		if (!(nvkm_rd32(device, 0x610200 + (chid * 0x10)) & 0x00030000))
+		if (!(nvkm_rd32(device, 0x610200 + (ctrl * 0x10)) & 0x00030000))
 			break;
 	) < 0) {
-		nvkm_error(subdev, "ch %d timeout0: %08x\n", chid,
-			   nvkm_rd32(device, 0x610200 + (chid * 0x10)));
+		nvkm_error(subdev, "ch %d timeout0: %08x\n", user,
+			   nvkm_rd32(device, 0x610200 + (ctrl * 0x10)));
 		return -EBUSY;
 	}
 
-	nvkm_wr32(device, 0x610200 + (chid * 0x10), 0x00000001);
+	nvkm_wr32(device, 0x610200 + (ctrl * 0x10), 0x00000001);
 	if (nvkm_msec(device, 2000,
-		u32 tmp = nvkm_rd32(device, 0x610200 + (chid * 0x10));
+		u32 tmp = nvkm_rd32(device, 0x610200 + (ctrl * 0x10));
 		if ((tmp & 0x00030000) == 0x00010000)
 			break;
 	) < 0) {
-		nvkm_error(subdev, "ch %d timeout1: %08x\n", chid,
-			   nvkm_rd32(device, 0x610200 + (chid * 0x10)));
+		nvkm_error(subdev, "ch %d timeout1: %08x\n", user,
+			   nvkm_rd32(device, 0x610200 + (ctrl * 0x10)));
 		return -EBUSY;
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp102.c
index 8443e04dc626..37122ca579ad 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp102.c
@@ -27,32 +27,32 @@
 #include <nvif/class.h>
 
 static const struct nv50_disp_root_func
-gp104_disp_root = {
+gp102_disp_root = {
 	.init = gf119_disp_root_init,
 	.fini = gf119_disp_root_fini,
 	.dmac = {
-		&gp104_disp_core_oclass,
-		&gp104_disp_base_oclass,
-		&gp104_disp_ovly_oclass,
+		&gp102_disp_core_oclass,
+		&gp102_disp_base_oclass,
+		&gp102_disp_ovly_oclass,
 	},
 	.pioc = {
-		&gk104_disp_oimm_oclass,
-		&gk104_disp_curs_oclass,
+		&gp102_disp_oimm_oclass,
+		&gp102_disp_curs_oclass,
 	},
 };
 
 static int
-gp104_disp_root_new(struct nvkm_disp *disp, const struct nvkm_oclass *oclass,
+gp102_disp_root_new(struct nvkm_disp *disp, const struct nvkm_oclass *oclass,
 		    void *data, u32 size, struct nvkm_object **pobject)
 {
-	return nv50_disp_root_new_(&gp104_disp_root, disp, oclass,
+	return nv50_disp_root_new_(&gp102_disp_root, disp, oclass,
 				   data, size, pobject);
 }
 
 const struct nvkm_disp_oclass
-gp104_disp_root_oclass = {
-	.base.oclass = GP104_DISP,
+gp102_disp_root_oclass = {
+	.base.oclass = GP102_DISP,
 	.base.minver = -1,
 	.base.maxver = -1,
-	.ctor = gp104_disp_root_new,
+	.ctor = gp102_disp_root_new,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c
index 2f9cecd81d04..e70dc6a9ff7d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c
@@ -66,7 +66,7 @@ nv50_disp_root_scanoutpos(NV50_DISP_MTHD_V0)
 	return 0;
 }
 
-int
+static int
 nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 {
 	union {
@@ -173,13 +173,56 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 				return 0;
 			} else
 			if (args->v0.state != 0) {
-				nvkm_output_dp_train(&outpdp->base, 0, true);
+				nvkm_output_dp_train(&outpdp->base, 0);
 				return 0;
 			}
 		} else
 			return ret;
 	}
 		break;
+	case NV50_DISP_MTHD_V1_SOR_DP_MST_LINK: {
+		struct nvkm_output_dp *outpdp = nvkm_output_dp(outp);
+		union {
+			struct nv50_disp_sor_dp_mst_link_v0 v0;
+		} *args = data;
+		int ret = -ENOSYS;
+		nvif_ioctl(object, "disp sor dp mst link size %d\n", size);
+		if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
+			nvif_ioctl(object, "disp sor dp mst link vers %d state %d\n",
+				   args->v0.version, args->v0.state);
+			if (outpdp->lt.mst != !!args->v0.state) {
+				outpdp->lt.mst = !!args->v0.state;
+				atomic_set(&outpdp->lt.done, 0);
+				nvkm_output_dp_train(&outpdp->base, 0);
+			}
+			return 0;
+		} else
+			return ret;
+	}
+		break;
+	case NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI: {
+		struct nvkm_output_dp *outpdp = nvkm_output_dp(outp);
+		union {
+			struct nv50_disp_sor_dp_mst_vcpi_v0 v0;
+		} *args = data;
+		int ret = -ENOSYS;
+		nvif_ioctl(object, "disp sor dp mst vcpi size %d\n", size);
+		if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
+			nvif_ioctl(object, "disp sor dp mst vcpi vers %d "
+					   "slot %02x/%02x pbn %04x/%04x\n",
+				   args->v0.version, args->v0.start_slot,
+				   args->v0.num_slots, args->v0.pbn,
+				   args->v0.aligned_pbn);
+			if (!outpdp->func->vcpi)
+				return -ENODEV;
+			outpdp->func->vcpi(outpdp, head, args->v0.start_slot,
+					   args->v0.num_slots, args->v0.pbn,
+					   args->v0.aligned_pbn);
+			return 0;
+		} else
+			return ret;
+	}
+		break;
 	case NV50_DISP_MTHD_V1_PIOR_PWR:
 		if (!func->pior.power)
 			return -ENODEV;
@@ -207,8 +250,8 @@ nv50_disp_root_pioc_new_(const struct nvkm_oclass *oclass,
 {
 	const struct nv50_disp_pioc_oclass *sclass = oclass->priv;
 	struct nv50_disp_root *root = nv50_disp_root(oclass->parent);
-	return sclass->ctor(sclass->func, sclass->mthd, root, sclass->chid,
-			    oclass, data, size, pobject);
+	return sclass->ctor(sclass->func, sclass->mthd, root, sclass->chid.ctrl,
+			    sclass->chid.user, oclass, data, size, pobject);
 }
 
 static int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h
index ad00f1724b72..b147cf5b3518 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h
@@ -41,5 +41,5 @@ extern const struct nvkm_disp_oclass gk110_disp_root_oclass;
 extern const struct nvkm_disp_oclass gm107_disp_root_oclass;
 extern const struct nvkm_disp_oclass gm200_disp_root_oclass;
 extern const struct nvkm_disp_oclass gp100_disp_root_oclass;
-extern const struct nvkm_disp_oclass gp104_disp_root_oclass;
+extern const struct nvkm_disp_oclass gp102_disp_root_oclass;
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg94.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg94.c
index 1bb9d661e9b3..4510cb6e10a8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg94.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorg94.c
@@ -45,14 +45,6 @@ static const struct nvkm_output_func
 g94_sor_output_func = {
 };
 
-int
-g94_sor_output_new(struct nvkm_disp *disp, int index,
-		   struct dcb_output *dcbE, struct nvkm_output **poutp)
-{
-	return nvkm_output_new_(&g94_sor_output_func, disp,
-				index, dcbE, poutp);
-}
-
 /*******************************************************************************
  * DisplayPort
  ******************************************************************************/
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c
index 49bd5da194e1..6ffdaa65aa77 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c
@@ -56,11 +56,13 @@ gf119_sor_dp_lnk_ctl(struct nvkm_output_dp *outp, int nr, int bw, bool ef)
 
 	clksor |= bw << 18;
 	dpctrl |= ((1 << nr) - 1) << 16;
+	if (outp->lt.mst)
+		dpctrl |= 0x40000000;
 	if (ef)
 		dpctrl |= 0x00004000;
 
 	nvkm_mask(device, 0x612300 + soff, 0x007c0000, clksor);
-	nvkm_mask(device, 0x61c10c + loff, 0x001f4000, dpctrl);
+	nvkm_mask(device, 0x61c10c + loff, 0x401f4000, dpctrl);
 	return 0;
 }
 
@@ -101,12 +103,24 @@ gf119_sor_dp_drv_ctl(struct nvkm_output_dp *outp,
 	return 0;
 }
 
+void
+gf119_sor_dp_vcpi(struct nvkm_output_dp *outp, int head, u8 slot,
+		  u8 slot_nr, u16 pbn, u16 aligned)
+{
+	struct nvkm_device *device = outp->base.disp->engine.subdev.device;
+	const u32 hoff = head * 0x800;
+
+	nvkm_mask(device, 0x616588 + hoff, 0x00003f3f, (slot_nr << 8) | slot);
+	nvkm_mask(device, 0x61658c + hoff, 0xffffffff, (aligned << 16) | pbn);
+}
+
 static const struct nvkm_output_dp_func
 gf119_sor_dp_func = {
 	.pattern = gf119_sor_dp_pattern,
 	.lnk_pwr = g94_sor_dp_lnk_pwr,
 	.lnk_ctl = gf119_sor_dp_lnk_ctl,
 	.drv_ctl = gf119_sor_dp_drv_ctl,
+	.vcpi = gf119_sor_dp_vcpi,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c
index 37790b2617c5..4cf8ad4d18ab 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c
@@ -43,6 +43,7 @@ gm107_sor_dp_func = {
 	.lnk_pwr = g94_sor_dp_lnk_pwr,
 	.lnk_ctl = gf119_sor_dp_lnk_ctl,
 	.drv_ctl = gf119_sor_dp_drv_ctl,
+	.vcpi = gf119_sor_dp_vcpi,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c
index c44fa7ea672a..81b788fa61be 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c
@@ -120,6 +120,7 @@ gm200_sor_dp_func = {
 	.lnk_pwr = gm200_sor_dp_lnk_pwr,
 	.lnk_ctl = gf119_sor_dp_lnk_ctl,
 	.drv_ctl = gm200_sor_dp_drv_ctl,
+	.vcpi = gf119_sor_dp_vcpi,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c
index aeb3387a3fb0..15a992b3580a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c
@@ -129,7 +129,7 @@ g84_fifo_chan_engine_fini(struct nvkm_fifo_chan *base,
 }
 
 
-int
+static int
 g84_fifo_chan_engine_init(struct nvkm_fifo_chan *base,
 			  struct nvkm_engine *engine)
 {
@@ -170,7 +170,7 @@ g84_fifo_chan_engine_ctor(struct nvkm_fifo_chan *base,
 	return nvkm_object_bind(object, NULL, 0, &chan->engn[engn]);
 }
 
-int
+static int
 g84_fifo_chan_object_ctor(struct nvkm_fifo_chan *base,
 			  struct nvkm_object *object)
 {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
index 352a0baec84d..ec68ea9747d5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
@@ -180,7 +180,8 @@ gf100_fifo_recover(struct gf100_fifo *fifo, struct nvkm_engine *engine,
 	list_del_init(&chan->head);
 	chan->killed = true;
 
-	fifo->recover.mask |= 1ULL << engine->subdev.index;
+	if (engine != &fifo->base.engine)
+		fifo->recover.mask |= 1ULL << engine->subdev.index;
 	schedule_work(&fifo->recover.work);
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
index 103c0afaaa6d..38c0910722c0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
@@ -743,14 +743,14 @@ gk104_fifo_fault_engine[] = {
 	{ 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR },
 	{ 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM },
 	{ 0x06, "SCHED" },
-	{ 0x07, "HOST0" },
-	{ 0x08, "HOST1" },
-	{ 0x09, "HOST2" },
-	{ 0x0a, "HOST3" },
-	{ 0x0b, "HOST4" },
-	{ 0x0c, "HOST5" },
-	{ 0x0d, "HOST6" },
-	{ 0x0e, "HOST7" },
+	{ 0x07, "HOST0", NULL, NVKM_ENGINE_FIFO },
+	{ 0x08, "HOST1", NULL, NVKM_ENGINE_FIFO },
+	{ 0x09, "HOST2", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0a, "HOST3", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0b, "HOST4", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0c, "HOST5", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0d, "HOST6", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0e, "HOST7", NULL, NVKM_ENGINE_FIFO },
 	{ 0x0f, "HOSTSR" },
 	{ 0x10, "MSVLD", NULL, NVKM_ENGINE_MSVLD },
 	{ 0x11, "MSPPP", NULL, NVKM_ENGINE_MSPPP },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
index bd1ff877aa06..29c080683b32 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
@@ -32,14 +32,14 @@ gm107_fifo_fault_engine[] = {
 	{ 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR },
 	{ 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM },
 	{ 0x06, "SCHED" },
-	{ 0x07, "HOST0" },
-	{ 0x08, "HOST1" },
-	{ 0x09, "HOST2" },
-	{ 0x0a, "HOST3" },
-	{ 0x0b, "HOST4" },
-	{ 0x0c, "HOST5" },
-	{ 0x0d, "HOST6" },
-	{ 0x0e, "HOST7" },
+	{ 0x07, "HOST0", NULL, NVKM_ENGINE_FIFO },
+	{ 0x08, "HOST1", NULL, NVKM_ENGINE_FIFO },
+	{ 0x09, "HOST2", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0a, "HOST3", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0b, "HOST4", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0c, "HOST5", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0d, "HOST6", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0e, "HOST7", NULL, NVKM_ENGINE_FIFO },
 	{ 0x0f, "HOSTSR" },
 	{ 0x13, "PERF" },
 	{ 0x17, "PMU" },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
index eff83f7fb705..b2635aea9f6e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
@@ -30,17 +30,17 @@ gp100_fifo_fault_engine[] = {
 	{ 0x03, "IFB", NULL, NVKM_ENGINE_IFB },
 	{ 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR },
 	{ 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM },
-	{ 0x06, "HOST0" },
-	{ 0x07, "HOST1" },
-	{ 0x08, "HOST2" },
-	{ 0x09, "HOST3" },
-	{ 0x0a, "HOST4" },
-	{ 0x0b, "HOST5" },
-	{ 0x0c, "HOST6" },
-	{ 0x0d, "HOST7" },
-	{ 0x0e, "HOST8" },
-	{ 0x0f, "HOST9" },
-	{ 0x10, "HOST10" },
+	{ 0x06, "HOST0", NULL, NVKM_ENGINE_FIFO },
+	{ 0x07, "HOST1", NULL, NVKM_ENGINE_FIFO },
+	{ 0x08, "HOST2", NULL, NVKM_ENGINE_FIFO },
+	{ 0x09, "HOST3", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0a, "HOST4", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0b, "HOST5", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0c, "HOST6", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0d, "HOST7", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0e, "HOST8", NULL, NVKM_ENGINE_FIFO },
+	{ 0x0f, "HOST9", NULL, NVKM_ENGINE_FIFO },
+	{ 0x10, "HOST10", NULL, NVKM_ENGINE_FIFO },
 	{ 0x13, "PERF" },
 	{ 0x17, "PMU" },
 	{ 0x18, "PTP" },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c
index cbc67f262322..12d964260a29 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c
@@ -60,6 +60,7 @@ gf100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base,
 	struct nvkm_gpuobj *inst = chan->base.inst;
 	int ret = 0;
 
+	mutex_lock(&subdev->mutex);
 	nvkm_wr32(device, 0x002634, chan->base.chid);
 	if (nvkm_msec(device, 2000,
 		if (nvkm_rd32(device, 0x002634) == chan->base.chid)
@@ -67,10 +68,12 @@ gf100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base,
 	) < 0) {
 		nvkm_error(subdev, "channel %d [%s] kick timeout\n",
 			   chan->base.chid, chan->base.object.client->name);
-		ret = -EBUSY;
-		if (suspend)
-			return ret;
+		ret = -ETIMEDOUT;
 	}
+	mutex_unlock(&subdev->mutex);
+
+	if (ret && suspend)
+		return ret;
 
 	if (offset) {
 		nvkm_kmap(inst);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
index ed4351032ed6..a2df4f3e7763 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c
@@ -40,7 +40,9 @@ gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *chan)
 	struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 	struct nvkm_client *client = chan->base.object.client;
+	int ret = 0;
 
+	mutex_lock(&subdev->mutex);
 	nvkm_wr32(device, 0x002634, chan->base.chid);
 	if (nvkm_msec(device, 2000,
 		if (!(nvkm_rd32(device, 0x002634) & 0x00100000))
@@ -48,10 +50,10 @@ gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *chan)
 	) < 0) {
 		nvkm_error(subdev, "channel %d [%s] kick timeout\n",
 			   chan->base.chid, client->name);
-		return -EBUSY;
+		ret = -ETIMEDOUT;
 	}
-
-	return 0;
+	mutex_unlock(&subdev->mutex);
+	return ret;
 }
 
 static u32
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c
index c925ade5880e..74a64e3fd59a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c
@@ -218,7 +218,7 @@ gf117_grctx_generate_attrib(struct gf100_grctx *info)
 	}
 }
 
-void
+static void
 gf117_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c
index 6d3c5011e18c..4c4b5ab6e46d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c
@@ -933,7 +933,7 @@ gm107_grctx_generate_attrib(struct gf100_grctx *info)
 	}
 }
 
-void
+static void
 gm107_grctx_generate_tpcid(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxnv50.c
index 1e13278cf306..c8bb9191f9a2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxnv50.c
@@ -106,6 +106,7 @@
 #define CP_SEEK_2      0x00c800ff
 
 #include "ctxnv40.h"
+#include "nv50.h"
 
 #include <subdev/fb.h>
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf100.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf100.fuc3.h
index 8cb240b65ec2..12a703fe355d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf100.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf100.fuc3.h
@@ -1,4 +1,4 @@
-uint32_t gf100_grgpc_data[] = {
+static uint32_t gf100_grgpc_data[] = {
 /* 0x0000: gpc_mmio_list_head */
 	0x00000064,
 /* 0x0004: gpc_mmio_list_tail */
@@ -36,7 +36,7 @@ uint32_t gf100_grgpc_data[] = {
 	0x00000000,
 };
 
-uint32_t gf100_grgpc_code[] = {
+static uint32_t gf100_grgpc_code[] = {
 	0x03a10ef5,
 /* 0x0004: queue_put */
 	0x9800d898,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf117.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf117.fuc3.h
index 550d6ba0933b..ffbfc51200f1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf117.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf117.fuc3.h
@@ -1,4 +1,4 @@
-uint32_t gf117_grgpc_data[] = {
+static uint32_t gf117_grgpc_data[] = {
 /* 0x0000: gpc_mmio_list_head */
 	0x0000006c,
 /* 0x0004: gpc_mmio_list_tail */
@@ -40,7 +40,7 @@ uint32_t gf117_grgpc_data[] = {
 	0x00000000,
 };
 
-uint32_t gf117_grgpc_code[] = {
+static uint32_t gf117_grgpc_code[] = {
 	0x03a10ef5,
 /* 0x0004: queue_put */
 	0x9800d898,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk104.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk104.fuc3.h
index 271b59d365e5..357f662de571 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk104.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk104.fuc3.h
@@ -1,4 +1,4 @@
-uint32_t gk104_grgpc_data[] = {
+static uint32_t gk104_grgpc_data[] = {
 /* 0x0000: gpc_mmio_list_head */
 	0x0000006c,
 /* 0x0004: gpc_mmio_list_tail */
@@ -40,7 +40,7 @@ uint32_t gk104_grgpc_data[] = {
 	0x00000000,
 };
 
-uint32_t gk104_grgpc_code[] = {
+static uint32_t gk104_grgpc_code[] = {
 	0x03a10ef5,
 /* 0x0004: queue_put */
 	0x9800d898,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk110.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk110.fuc3.h
index 73b4a32c5d29..4ffc8212a85c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk110.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk110.fuc3.h
@@ -1,4 +1,4 @@
-uint32_t gk110_grgpc_data[] = {
+static uint32_t gk110_grgpc_data[] = {
 /* 0x0000: gpc_mmio_list_head */
 	0x0000006c,
 /* 0x0004: gpc_mmio_list_tail */
@@ -40,7 +40,7 @@ uint32_t gk110_grgpc_data[] = {
 	0x00000000,
 };
 
-uint32_t gk110_grgpc_code[] = {
+static uint32_t gk110_grgpc_code[] = {
 	0x03a10ef5,
 /* 0x0004: queue_put */
 	0x9800d898,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk208.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk208.fuc5.h
index 018169818317..09196206c9bc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk208.fuc5.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk208.fuc5.h
@@ -1,4 +1,4 @@
-uint32_t gk208_grgpc_data[] = {
+static uint32_t gk208_grgpc_data[] = {
 /* 0x0000: gpc_mmio_list_head */
 	0x0000006c,
 /* 0x0004: gpc_mmio_list_tail */
@@ -40,7 +40,7 @@ uint32_t gk208_grgpc_data[] = {
 	0x00000000,
 };
 
-uint32_t gk208_grgpc_code[] = {
+static uint32_t gk208_grgpc_code[] = {
 	0x03140ef5,
 /* 0x0004: queue_put */
 	0x9800d898,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h
index eca007f03fa9..6d7d004363d9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h
@@ -1,4 +1,4 @@
-uint32_t gm107_grgpc_data[] = {
+static uint32_t gm107_grgpc_data[] = {
 /* 0x0000: gpc_mmio_list_head */
 	0x0000006c,
 /* 0x0004: gpc_mmio_list_tail */
@@ -40,7 +40,7 @@ uint32_t gm107_grgpc_data[] = {
 	0x00000000,
 };
 
-uint32_t gm107_grgpc_code[] = {
+static uint32_t gm107_grgpc_code[] = {
 	0x03410ef5,
 /* 0x0004: queue_put */
 	0x9800d898,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf100.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf100.fuc3.h
index 8015b40a61d6..7538404b8b13 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf100.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf100.fuc3.h
@@ -1,4 +1,4 @@
-uint32_t gf100_grhub_data[] = {
+static uint32_t gf100_grhub_data[] = {
 /* 0x0000: hub_mmio_list_head */
 	0x00000300,
 /* 0x0004: hub_mmio_list_tail */
@@ -205,7 +205,7 @@ uint32_t gf100_grhub_data[] = {
 	0x0417e91c,
 };
 
-uint32_t gf100_grhub_code[] = {
+static uint32_t gf100_grhub_code[] = {
 	0x039b0ef5,
 /* 0x0004: queue_put */
 	0x9800d898,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf117.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf117.fuc3.h
index 2af90ec6852a..ce000a47ec6d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf117.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf117.fuc3.h
@@ -1,4 +1,4 @@
-uint32_t gf117_grhub_data[] = {
+static uint32_t gf117_grhub_data[] = {
 /* 0x0000: hub_mmio_list_head */
 	0x00000300,
 /* 0x0004: hub_mmio_list_tail */
@@ -205,7 +205,7 @@ uint32_t gf117_grhub_data[] = {
 	0x0417e91c,
 };
 
-uint32_t gf117_grhub_code[] = {
+static uint32_t gf117_grhub_code[] = {
 	0x039b0ef5,
 /* 0x0004: queue_put */
 	0x9800d898,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk104.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk104.fuc3.h
index e8b8c1c94700..1f26cb6a233c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk104.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk104.fuc3.h
@@ -1,4 +1,4 @@
-uint32_t gk104_grhub_data[] = {
+static uint32_t gk104_grhub_data[] = {
 /* 0x0000: hub_mmio_list_head */
 	0x00000300,
 /* 0x0004: hub_mmio_list_tail */
@@ -205,7 +205,7 @@ uint32_t gk104_grhub_data[] = {
 	0x0417e91c,
 };
 
-uint32_t gk104_grhub_code[] = {
+static uint32_t gk104_grhub_code[] = {
 	0x039b0ef5,
 /* 0x0004: queue_put */
 	0x9800d898,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk110.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk110.fuc3.h
index f4ed2fb6f714..70436d93efe3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk110.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk110.fuc3.h
@@ -1,4 +1,4 @@
-uint32_t gk110_grhub_data[] = {
+static uint32_t gk110_grhub_data[] = {
 /* 0x0000: hub_mmio_list_head */
 	0x00000300,
 /* 0x0004: hub_mmio_list_tail */
@@ -205,7 +205,7 @@ uint32_t gk110_grhub_data[] = {
 	0x0417e91c,
 };
 
-uint32_t gk110_grhub_code[] = {
+static uint32_t gk110_grhub_code[] = {
 	0x039b0ef5,
 /* 0x0004: queue_put */
 	0x9800d898,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk208.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk208.fuc5.h
index ed488973c117..e0933a07426a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk208.fuc5.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk208.fuc5.h
@@ -1,4 +1,4 @@
-uint32_t gk208_grhub_data[] = {
+static uint32_t gk208_grhub_data[] = {
 /* 0x0000: hub_mmio_list_head */
 	0x00000300,
 /* 0x0004: hub_mmio_list_tail */
@@ -205,7 +205,7 @@ uint32_t gk208_grhub_data[] = {
 	0x0417e91c,
 };
 
-uint32_t gk208_grhub_code[] = {
+static uint32_t gk208_grhub_code[] = {
 	0x030e0ef5,
 /* 0x0004: queue_put */
 	0x9800d898,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgm107.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgm107.fuc5.h
index 5c9051839557..9b432823bcbe 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgm107.fuc5.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgm107.fuc5.h
@@ -1,4 +1,4 @@
-uint32_t gm107_grhub_data[] = {
+static uint32_t gm107_grhub_data[] = {
 /* 0x0000: hub_mmio_list_head */
 	0x00000300,
 /* 0x0004: hub_mmio_list_tail */
@@ -205,7 +205,7 @@ uint32_t gm107_grhub_data[] = {
 	0x0417e91c,
 };
 
-uint32_t gm107_grhub_code[] = {
+static uint32_t gm107_grhub_code[] = {
 	0x030e0ef5,
 /* 0x0004: queue_put */
 	0x9800d898,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
index 157919c788e6..f65a5b0a1a4d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
@@ -1041,6 +1041,13 @@ gf100_gr_trap_tpc(struct gf100_gr *gr, int gpc, int tpc)
 		stat &= ~0x00000008;
 	}
 
+	if (stat & 0x00000010) {
+		u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0430));
+		nvkm_error(subdev, "GPC%d/TPC%d/MPC: %08x\n", gpc, tpc, trap);
+		nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x0430), 0xc0000000);
+		stat &= ~0x00000010;
+	}
+
 	if (stat) {
 		nvkm_error(subdev, "GPC%d/TPC%d/%08x: unknown\n", gpc, tpc, stat);
 	}
@@ -1258,7 +1265,7 @@ gf100_gr_ctxctl_isr(struct gf100_gr *gr)
 	struct nvkm_device *device = subdev->device;
 	u32 stat = nvkm_rd32(device, 0x409c18);
 
-	if (stat & 0x00000001) {
+	if (!gr->firmware && (stat & 0x00000001)) {
 		u32 code = nvkm_rd32(device, 0x409814);
 		if (code == E_BAD_FWMTHD) {
 			u32 class = nvkm_rd32(device, 0x409808);
@@ -1270,15 +1277,14 @@ gf100_gr_ctxctl_isr(struct gf100_gr *gr)
 			nvkm_error(subdev, "FECS MTHD subc %d class %04x "
 					   "mthd %04x data %08x\n",
 				   subc, class, mthd, data);
-
-			nvkm_wr32(device, 0x409c20, 0x00000001);
-			stat &= ~0x00000001;
 		} else {
 			nvkm_error(subdev, "FECS ucode error %d\n", code);
 		}
+		nvkm_wr32(device, 0x409c20, 0x00000001);
+		stat &= ~0x00000001;
 	}
 
-	if (stat & 0x00080000) {
+	if (!gr->firmware && (stat & 0x00080000)) {
 		nvkm_error(subdev, "FECS watchdog timeout\n");
 		gf100_gr_ctxctl_debug(gr);
 		nvkm_wr32(device, 0x409c20, 0x00080000);
@@ -1384,7 +1390,7 @@ gf100_gr_intr(struct nvkm_gr *base)
 	nvkm_fifo_chan_put(device->fifo, flags, &chan);
 }
 
-void
+static void
 gf100_gr_init_fw(struct gf100_gr *gr, u32 fuc_base,
 		 struct gf100_gr_fuc *code, struct gf100_gr_fuc *data)
 {
@@ -1701,7 +1707,7 @@ gf100_gr_oneinit(struct nvkm_gr *base)
 	return 0;
 }
 
-int
+static int
 gf100_gr_init_(struct nvkm_gr *base)
 {
 	struct gf100_gr *gr = gf100_gr(base);
@@ -1756,6 +1762,50 @@ gf100_gr_ = {
 };
 
 int
+gf100_gr_ctor_fw_legacy(struct gf100_gr *gr, const char *fwname,
+			struct gf100_gr_fuc *fuc, int ret)
+{
+	struct nvkm_subdev *subdev = &gr->base.engine.subdev;
+	struct nvkm_device *device = subdev->device;
+	const struct firmware *fw;
+	char f[32];
+
+	/* see if this firmware has a legacy path */
+	if (!strcmp(fwname, "fecs_inst"))
+		fwname = "fuc409c";
+	else if (!strcmp(fwname, "fecs_data"))
+		fwname = "fuc409d";
+	else if (!strcmp(fwname, "gpccs_inst"))
+		fwname = "fuc41ac";
+	else if (!strcmp(fwname, "gpccs_data"))
+		fwname = "fuc41ad";
+	else {
+		/* nope, let's just return the error we got */
+		nvkm_error(subdev, "failed to load %s\n", fwname);
+		return ret;
+	}
+
+	/* yes, try to load from the legacy path */
+	nvkm_debug(subdev, "%s: falling back to legacy path\n", fwname);
+
+	snprintf(f, sizeof(f), "nouveau/nv%02x_%s", device->chipset, fwname);
+	ret = request_firmware(&fw, f, device->dev);
+	if (ret) {
+		snprintf(f, sizeof(f), "nouveau/%s", fwname);
+		ret = request_firmware(&fw, f, device->dev);
+		if (ret) {
+			nvkm_error(subdev, "failed to load %s\n", fwname);
+			return ret;
+		}
+	}
+
+	fuc->size = fw->size;
+	fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
+	release_firmware(fw);
+	return (fuc->data != NULL) ? 0 : -ENOMEM;
+}
+
+int
 gf100_gr_ctor_fw(struct gf100_gr *gr, const char *fwname,
 		 struct gf100_gr_fuc *fuc)
 {
@@ -1765,10 +1815,8 @@ gf100_gr_ctor_fw(struct gf100_gr *gr, const char *fwname,
 	int ret;
 
 	ret = nvkm_firmware_get(device, fwname, &fw);
-	if (ret) {
-		nvkm_error(subdev, "failed to load %s\n", fwname);
-		return ret;
-	}
+	if (ret)
+		return gf100_gr_ctor_fw_legacy(gr, fwname, fuc, ret);
 
 	fuc->size = fw->size;
 	fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c
index 70335f65c51e..0124e468086e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c
@@ -102,7 +102,7 @@ gf117_gr_pack_mmio[] = {
 
 #include "fuc/hubgf117.fuc3.h"
 
-struct gf100_gr_ucode
+static struct gf100_gr_ucode
 gf117_gr_fecs_ucode = {
 	.code.data = gf117_grhub_code,
 	.code.size = sizeof(gf117_grhub_code),
@@ -112,7 +112,7 @@ gf117_gr_fecs_ucode = {
 
 #include "fuc/gpcgf117.fuc3.h"
 
-struct gf100_gr_ucode
+static struct gf100_gr_ucode
 gf117_gr_gpccs_ucode = {
 	.code.data = gf117_grgpc_code,
 	.code.size = sizeof(gf117_grgpc_code),
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c
index 45f965f608a7..2c67fac576d1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c
@@ -308,7 +308,7 @@ gm107_gr_init_bios(struct gf100_gr *gr)
 	}
 }
 
-int
+static int
 gm107_gr_init(struct gf100_gr *gr)
 {
 	struct nvkm_device *device = gr->base.engine.subdev.device;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c
index f1e15a4d4f64..b4e3c50badc7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c
@@ -187,6 +187,7 @@ nv30_gr = {
 		{ -1, -1, 0x038a, &nv04_gr_object }, /* ifc (nv30) */
 		{ -1, -1, 0x039e, &nv04_gr_object }, /* swzsurf (nv30) */
 		{ -1, -1, 0x0397, &nv04_gr_object }, /* rankine */
+		{ -1, -1, 0x0597, &nv04_gr_object }, /* kelvin */
 		{}
 	}
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c
index 300f5ed5de0b..e7ed04b935cd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c
@@ -123,6 +123,7 @@ nv34_gr = {
 		{ -1, -1, 0x0389, &nv04_gr_object }, /* sifm (nv30) */
 		{ -1, -1, 0x038a, &nv04_gr_object }, /* ifc (nv30) */
 		{ -1, -1, 0x039e, &nv04_gr_object }, /* swzsurf (nv30) */
+		{ -1, -1, 0x0597, &nv04_gr_object }, /* kelvin */
 		{ -1, -1, 0x0697, &nv04_gr_object }, /* rankine */
 		{}
 	}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv35.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv35.c
index 740df0f52c38..5e8abacbacc6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv35.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv35.c
@@ -124,6 +124,7 @@ nv35_gr = {
 		{ -1, -1, 0x038a, &nv04_gr_object }, /* ifc (nv30) */
 		{ -1, -1, 0x039e, &nv04_gr_object }, /* swzsurf (nv30) */
 		{ -1, -1, 0x0497, &nv04_gr_object }, /* rankine */
+		{ -1, -1, 0x0597, &nv04_gr_object }, /* kelvin */
 		{}
 	}
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c
index 8616636ad7b4..dde89a4a0f5b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c
@@ -71,7 +71,7 @@ nvkm_perfdom_find(struct nvkm_pm *pm, int di)
 	return NULL;
 }
 
-struct nvkm_perfsig *
+static struct nvkm_perfsig *
 nvkm_perfsig_find(struct nvkm_pm *pm, u8 di, u8 si, struct nvkm_perfdom **pdom)
 {
 	struct nvkm_perfdom *dom = *pdom;
@@ -699,7 +699,7 @@ nvkm_pm_oclass_get(struct nvkm_oclass *oclass, int index,
 	return 1;
 }
 
-int
+static int
 nvkm_perfsrc_new(struct nvkm_pm *pm, struct nvkm_perfsig *sig,
 		 const struct nvkm_specsrc *spec)
 {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/pm/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/pm/gf100.c
index d2901e9a7808..fe2532ee4145 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/pm/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/pm/gf100.c
@@ -102,7 +102,7 @@ gf100_pm_gpc[] = {
 	{}
 };
 
-const struct nvkm_specdom
+static const struct nvkm_specdom
 gf100_pm_part[] = {
 	{ 0xe0, (const struct nvkm_specsig[]) {
 			{ 0x0f, "part00_pbfb_00", gf100_pbfb_sources },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec/fuc/g98.fuc0s.h b/drivers/gpu/drm/nouveau/nvkm/engine/sec/fuc/g98.fuc0s.h
index eca62221f299..4b57f8814560 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/sec/fuc/g98.fuc0s.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec/fuc/g98.fuc0s.h
@@ -1,4 +1,4 @@
-uint32_t g98_sec_data[] = {
+static uint32_t g98_sec_data[] = {
 /* 0x0000: ctx_dma */
 /* 0x0000: ctx_dma_query */
 	0x00000000,
@@ -150,7 +150,7 @@ uint32_t g98_sec_data[] = {
 	0x00000000,
 };
 
-uint32_t g98_sec_code[] = {
+static uint32_t g98_sec_code[] = {
 	0x17f004bd,
 	0x0010fe35,
 	0xf10004fe,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
index 370dcd8ff7b5..6eff637ac301 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
@@ -84,7 +84,7 @@ nv50_bar_oneinit(struct nvkm_bar *base)
 	start = 0x0100000000ULL;
 	limit = start + device->func->resource_size(device, 3);
 
-	ret = nvkm_vm_new(device, start, limit, start, &bar3_lock, &vm);
+	ret = nvkm_vm_new(device, start, limit - start, start, &bar3_lock, &vm);
 	if (ret)
 		return ret;
 
@@ -117,7 +117,7 @@ nv50_bar_oneinit(struct nvkm_bar *base)
 	start = 0x0000000000ULL;
 	limit = start + device->func->resource_size(device, 1);
 
-	ret = nvkm_vm_new(device, start, limit--, start, &bar1_lock, &vm);
+	ret = nvkm_vm_new(device, start, limit-- - start, start, &bar1_lock, &vm);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/Kbuild
index dbcb0ef21587..be57220a2e01 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/Kbuild
@@ -31,6 +31,7 @@ nvkm-y += nvkm/subdev/bios/timing.o
 nvkm-y += nvkm/subdev/bios/therm.o
 nvkm-y += nvkm/subdev/bios/vmap.o
 nvkm-y += nvkm/subdev/bios/volt.o
+nvkm-y += nvkm/subdev/bios/vpstate.o
 nvkm-y += nvkm/subdev/bios/xpio.o
 nvkm-y += nvkm/subdev/bios/M0203.o
 nvkm-y += nvkm/subdev/bios/M0205.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/boost.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/boost.c
index 3756ec91a88d..eaf74eb72983 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/boost.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/boost.c
@@ -25,16 +25,16 @@
 #include <subdev/bios/bit.h>
 #include <subdev/bios/boost.h>
 
-u16
+u32
 nvbios_boostTe(struct nvkm_bios *bios,
 	       u8 *ver, u8 *hdr, u8 *cnt, u8 *len, u8 *snr, u8 *ssz)
 {
 	struct bit_entry bit_P;
-	u16 boost = 0x0000;
+	u32 boost = 0;
 
 	if (!bit_entry(bios, 'P', &bit_P)) {
 		if (bit_P.version == 2)
-			boost = nvbios_rd16(bios, bit_P.offset + 0x30);
+			boost = nvbios_rd32(bios, bit_P.offset + 0x30);
 
 		if (boost) {
 			*ver = nvbios_rd08(bios, boost + 0);
@@ -52,15 +52,15 @@ nvbios_boostTe(struct nvkm_bios *bios,
 		}
 	}
 
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_boostEe(struct nvkm_bios *bios, int idx,
 	       u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 {
 	u8  snr, ssz;
-	u16 data = nvbios_boostTe(bios, ver, hdr, cnt, len, &snr, &ssz);
+	u32 data = nvbios_boostTe(bios, ver, hdr, cnt, len, &snr, &ssz);
 	if (data && idx < *cnt) {
 		data = data + *hdr + (idx * (*len + (snr * ssz)));
 		*hdr = *len;
@@ -68,14 +68,14 @@ nvbios_boostEe(struct nvkm_bios *bios, int idx,
 		*len = ssz;
 		return data;
 	}
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_boostEp(struct nvkm_bios *bios, int idx,
 	       u8 *ver, u8 *hdr, u8 *cnt, u8 *len, struct nvbios_boostE *info)
 {
-	u16 data = nvbios_boostEe(bios, idx, ver, hdr, cnt, len);
+	u32 data = nvbios_boostEe(bios, idx, ver, hdr, cnt, len);
 	memset(info, 0x00, sizeof(*info));
 	if (data) {
 		info->pstate = (nvbios_rd16(bios, data + 0x00) & 0x01e0) >> 5;
@@ -85,7 +85,7 @@ nvbios_boostEp(struct nvkm_bios *bios, int idx,
 	return data;
 }
 
-u16
+u32
 nvbios_boostEm(struct nvkm_bios *bios, u8 pstate,
 	       u8 *ver, u8 *hdr, u8 *cnt, u8 *len, struct nvbios_boostE *info)
 {
@@ -97,21 +97,21 @@ nvbios_boostEm(struct nvkm_bios *bios, u8 pstate,
 	return data;
 }
 
-u16
+u32
 nvbios_boostSe(struct nvkm_bios *bios, int idx,
-	       u16 data, u8 *ver, u8 *hdr, u8 cnt, u8 len)
+	       u32 data, u8 *ver, u8 *hdr, u8 cnt, u8 len)
 {
 	if (data && idx < cnt) {
 		data = data + *hdr + (idx * len);
 		*hdr = len;
 		return data;
 	}
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_boostSp(struct nvkm_bios *bios, int idx,
-	       u16 data, u8 *ver, u8 *hdr, u8 cnt, u8 len,
+	       u32 data, u8 *ver, u8 *hdr, u8 cnt, u8 len,
 	       struct nvbios_boostS *info)
 {
 	data = nvbios_boostSe(bios, idx, data, ver, hdr, cnt, len);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/cstep.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/cstep.c
index 32e01624a162..5063382d8a6c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/cstep.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/cstep.c
@@ -25,16 +25,16 @@
 #include <subdev/bios/bit.h>
 #include <subdev/bios/cstep.h>
 
-u16
+u32
 nvbios_cstepTe(struct nvkm_bios *bios,
 	       u8 *ver, u8 *hdr, u8 *cnt, u8 *len, u8 *xnr, u8 *xsz)
 {
 	struct bit_entry bit_P;
-	u16 cstep = 0x0000;
+	u32 cstep = 0;
 
 	if (!bit_entry(bios, 'P', &bit_P)) {
 		if (bit_P.version == 2)
-			cstep = nvbios_rd16(bios, bit_P.offset + 0x34);
+			cstep = nvbios_rd32(bios, bit_P.offset + 0x34);
 
 		if (cstep) {
 			*ver = nvbios_rd08(bios, cstep + 0);
@@ -52,27 +52,27 @@ nvbios_cstepTe(struct nvkm_bios *bios,
 		}
 	}
 
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_cstepEe(struct nvkm_bios *bios, int idx, u8 *ver, u8 *hdr)
 {
 	u8  cnt, len, xnr, xsz;
-	u16 data = nvbios_cstepTe(bios, ver, hdr, &cnt, &len, &xnr, &xsz);
+	u32 data = nvbios_cstepTe(bios, ver, hdr, &cnt, &len, &xnr, &xsz);
 	if (data && idx < cnt) {
 		data = data + *hdr + (idx * len);
 		*hdr = len;
 		return data;
 	}
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_cstepEp(struct nvkm_bios *bios, int idx, u8 *ver, u8 *hdr,
 	       struct nvbios_cstepE *info)
 {
-	u16 data = nvbios_cstepEe(bios, idx, ver, hdr);
+	u32 data = nvbios_cstepEe(bios, idx, ver, hdr);
 	memset(info, 0x00, sizeof(*info));
 	if (data) {
 		info->pstate = (nvbios_rd16(bios, data + 0x00) & 0x01e0) >> 5;
@@ -81,7 +81,7 @@ nvbios_cstepEp(struct nvkm_bios *bios, int idx, u8 *ver, u8 *hdr,
 	return data;
 }
 
-u16
+u32
 nvbios_cstepEm(struct nvkm_bios *bios, u8 pstate, u8 *ver, u8 *hdr,
 	       struct nvbios_cstepE *info)
 {
@@ -93,24 +93,24 @@ nvbios_cstepEm(struct nvkm_bios *bios, u8 pstate, u8 *ver, u8 *hdr,
 	return data;
 }
 
-u16
+u32
 nvbios_cstepXe(struct nvkm_bios *bios, int idx, u8 *ver, u8 *hdr)
 {
 	u8  cnt, len, xnr, xsz;
-	u16 data = nvbios_cstepTe(bios, ver, hdr, &cnt, &len, &xnr, &xsz);
+	u32 data = nvbios_cstepTe(bios, ver, hdr, &cnt, &len, &xnr, &xsz);
 	if (data && idx < xnr) {
 		data = data + *hdr + (cnt * len) + (idx * xsz);
 		*hdr = xsz;
 		return data;
 	}
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_cstepXp(struct nvkm_bios *bios, int idx, u8 *ver, u8 *hdr,
 	       struct nvbios_cstepX *info)
 {
-	u16 data = nvbios_cstepXe(bios, idx, ver, hdr);
+	u32 data = nvbios_cstepXe(bios, idx, ver, hdr);
 	memset(info, 0x00, sizeof(*info));
 	if (data) {
 		info->freq    = nvbios_rd16(bios, data + 0x00) * 1000;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c
index d89e78c4e689..972370ed36f0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c
@@ -207,8 +207,11 @@ nvbios_dpcfg_match(struct nvkm_bios *bios, u16 outp, u8 pc, u8 vs, u8 pe,
 	if (*ver >= 0x30) {
 		const u8 vsoff[] = { 0, 4, 7, 9 };
 		idx = (pc * 10) + vsoff[vs] + pe;
-		if (*ver >= 0x40 && *hdr >= 0x12)
+		if (*ver >= 0x40 && *ver <= 0x41 && *hdr >= 0x12)
 			idx += nvbios_rd08(bios, outp + 0x11) * 40;
+		else
+		if (*ver >= 0x42)
+			idx += nvbios_rd08(bios, outp + 0x11) * 10;
 	} else {
 		while ((data = nvbios_dpcfg_entry(bios, outp, ++idx,
 						  ver, hdr, cnt, len))) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/fan.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/fan.c
index 80fed7e78dcb..456f9ea920dc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/fan.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/fan.c
@@ -25,15 +25,15 @@
 #include <subdev/bios/bit.h>
 #include <subdev/bios/fan.h>
 
-u16
+static u32
 nvbios_fan_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 {
 	struct bit_entry bit_P;
-	u16 fan = 0x0000;
+	u32 fan = 0;
 
 	if (!bit_entry(bios, 'P', &bit_P)) {
 		if (bit_P.version == 2 && bit_P.length >= 0x5a)
-			fan = nvbios_rd16(bios, bit_P.offset + 0x58);
+			fan = nvbios_rd32(bios, bit_P.offset + 0x58);
 
 		if (fan) {
 			*ver = nvbios_rd08(bios, fan + 0);
@@ -49,25 +49,25 @@ nvbios_fan_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 		}
 	}
 
-	return 0x0000;
+	return 0;
 }
 
-u16
+static u32
 nvbios_fan_entry(struct nvkm_bios *bios, int idx, u8 *ver, u8 *hdr,
 		 u8 *cnt, u8 *len)
 {
-	u16 data = nvbios_fan_table(bios, ver, hdr, cnt, len);
+	u32 data = nvbios_fan_table(bios, ver, hdr, cnt, len);
 	if (data && idx < *cnt)
 		return data + *hdr + (idx * (*len));
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_fan_parse(struct nvkm_bios *bios, struct nvbios_therm_fan *fan)
 {
 	u8 ver, hdr, cnt, len;
 
-	u16 data = nvbios_fan_entry(bios, 0, &ver, &hdr, &cnt, &len);
+	u32 data = nvbios_fan_entry(bios, 0, &ver, &hdr, &cnt, &len);
 	if (data) {
 		u8 type = nvbios_rd08(bios, data + 0x00);
 		switch (type) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c
index 084328028af1..3953d11844ea 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/iccsense.c
@@ -23,20 +23,21 @@
  */
 #include <subdev/bios.h>
 #include <subdev/bios/bit.h>
+#include <subdev/bios/extdev.h>
 #include <subdev/bios/iccsense.h>
 
-static u16
+static u32
 nvbios_iccsense_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt,
 		      u8 *len)
 {
 	struct bit_entry bit_P;
-	u16 iccsense;
+	u32 iccsense;
 
 	if (bit_entry(bios, 'P', &bit_P) || bit_P.version != 2 ||
 	    bit_P.length < 0x2c)
 		return 0;
 
-	iccsense = nvbios_rd16(bios, bit_P.offset + 0x28);
+	iccsense = nvbios_rd32(bios, bit_P.offset + 0x28);
 	if (!iccsense)
 		return 0;
 
@@ -60,7 +61,7 @@ nvbios_iccsense_parse(struct nvkm_bios *bios, struct nvbios_iccsense *iccsense)
 {
 	struct nvkm_subdev *subdev = &bios->subdev;
 	u8 ver, hdr, cnt, len, i;
-	u16 table, entry;
+	u32 table, entry;
 
 	table = nvbios_iccsense_table(bios, &ver, &hdr, &cnt, &len);
 	if (!table || !cnt)
@@ -77,23 +78,47 @@ nvbios_iccsense_parse(struct nvkm_bios *bios, struct nvbios_iccsense *iccsense)
 		return -ENOMEM;
 
 	for (i = 0; i < cnt; ++i) {
+		struct nvbios_extdev_func extdev;
 		struct pwr_rail_t *rail = &iccsense->rail[i];
+		u8 res_start = 0;
+		int r;
+
 		entry = table + hdr + i * len;
 
 		switch(ver) {
 		case 0x10:
 			rail->mode = nvbios_rd08(bios, entry + 0x1);
 			rail->extdev_id = nvbios_rd08(bios, entry + 0x2);
-			rail->resistor_mohm = nvbios_rd08(bios, entry + 0x3);
-			rail->rail = nvbios_rd08(bios, entry + 0x4);
+			res_start = 0x3;
 			break;
 		case 0x20:
 			rail->mode = nvbios_rd08(bios, entry);
 			rail->extdev_id = nvbios_rd08(bios, entry + 0x1);
-			rail->resistor_mohm = nvbios_rd08(bios, entry + 0x5);
-			rail->rail = nvbios_rd08(bios, entry + 0x6);
+			res_start = 0x5;
+			break;
+		};
+
+		if (nvbios_extdev_parse(bios, rail->extdev_id, &extdev))
+			continue;
+
+		switch (extdev.type) {
+		case NVBIOS_EXTDEV_INA209:
+		case NVBIOS_EXTDEV_INA219:
+			rail->resistor_count = 1;
+			break;
+		case NVBIOS_EXTDEV_INA3221:
+			rail->resistor_count = 3;
+			break;
+		default:
+			rail->resistor_count = 0;
 			break;
 		};
+
+		for (r = 0; r < rail->resistor_count; ++r) {
+			rail->resistors[r].mohm = nvbios_rd08(bios, entry + res_start + r * 2);
+			rail->resistors[r].enabled = !(nvbios_rd08(bios, entry + res_start + r * 2 + 1) & 0x40);
+		}
+		rail->config = nvbios_rd16(bios, entry + res_start + rail->resistor_count * 2);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/mxm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/mxm.c
index 3ddf0939ded3..994cc2d7759b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/mxm.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/mxm.c
@@ -81,7 +81,7 @@ mxm_sor_map(struct nvkm_bios *bios, u8 conn)
 		u16 map = nvbios_rd16(bios, mxm + 4);
 		if (map) {
 			ver = nvbios_rd08(bios, map);
-			if (ver == 0x10) {
+			if (ver == 0x10 || ver == 0x11) {
 				if (conn < nvbios_rd08(bios, map + 3)) {
 					map += nvbios_rd08(bios, map + 1);
 					map += conn;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/perf.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/perf.c
index 636bfb665bb9..c3068358f695 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/perf.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/perf.c
@@ -26,16 +26,16 @@
 #include <subdev/bios/perf.h>
 #include <subdev/pci.h>
 
-u16
+u32
 nvbios_perf_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr,
 		  u8 *cnt, u8 *len, u8 *snr, u8 *ssz)
 {
 	struct bit_entry bit_P;
-	u16 perf = 0x0000;
+	u32 perf = 0;
 
 	if (!bit_entry(bios, 'P', &bit_P)) {
 		if (bit_P.version <= 2) {
-			perf = nvbios_rd16(bios, bit_P.offset + 0);
+			perf = nvbios_rd32(bios, bit_P.offset + 0);
 			if (perf) {
 				*ver = nvbios_rd08(bios, perf + 0);
 				*hdr = nvbios_rd08(bios, perf + 1);
@@ -72,15 +72,15 @@ nvbios_perf_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr,
 		}
 	}
 
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_perf_entry(struct nvkm_bios *bios, int idx,
 		  u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 {
 	u8  snr, ssz;
-	u16 perf = nvbios_perf_table(bios, ver, hdr, cnt, len, &snr, &ssz);
+	u32 perf = nvbios_perf_table(bios, ver, hdr, cnt, len, &snr, &ssz);
 	if (perf && idx < *cnt) {
 		perf = perf + *hdr + (idx * (*len + (snr * ssz)));
 		*hdr = *len;
@@ -88,14 +88,14 @@ nvbios_perf_entry(struct nvkm_bios *bios, int idx,
 		*len = ssz;
 		return perf;
 	}
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_perfEp(struct nvkm_bios *bios, int idx,
 	      u8 *ver, u8 *hdr, u8 *cnt, u8 *len, struct nvbios_perfE *info)
 {
-	u16 perf = nvbios_perf_entry(bios, idx, ver, hdr, cnt, len);
+	u32 perf = nvbios_perf_entry(bios, idx, ver, hdr, cnt, len);
 	memset(info, 0x00, sizeof(*info));
 	info->pstate = nvbios_rd08(bios, perf + 0x00);
 	switch (!!perf * *ver) {
@@ -163,7 +163,7 @@ nvbios_perfEp(struct nvkm_bios *bios, int idx,
 		info->pcie_width = 0xff;
 		break;
 	default:
-		return 0x0000;
+		return 0;
 	}
 	return perf;
 }
@@ -202,7 +202,7 @@ nvbios_perf_fan_parse(struct nvkm_bios *bios,
 		      struct nvbios_perf_fan *fan)
 {
 	u8  ver, hdr, cnt, len, snr, ssz;
-	u16 perf = nvbios_perf_table(bios, &ver, &hdr, &cnt, &len, &snr, &ssz);
+	u32 perf = nvbios_perf_table(bios, &ver, &hdr, &cnt, &len, &snr, &ssz);
 	if (!perf)
 		return -ENODEV;
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h
index 212800ecdce9..7d1d3c6b4b72 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h
@@ -12,6 +12,7 @@ struct nvbios_source {
 	bool rw;
 	bool ignore_checksum;
 	bool no_pcir;
+	bool require_checksum;
 };
 
 int nvbios_extend(struct nvkm_bios *, u32 length);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c
index b2557e87afdd..7deb81b6dbac 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c
@@ -86,9 +86,12 @@ shadow_image(struct nvkm_bios *bios, int idx, u32 offset, struct shadow *mthd)
 		    nvbios_checksum(&bios->data[image.base], image.size)) {
 			nvkm_debug(subdev, "%08x: checksum failed\n",
 				   image.base);
-			if (mthd->func->rw)
+			if (!mthd->func->require_checksum) {
+				if (mthd->func->rw)
+					score += 1;
 				score += 1;
-			score += 1;
+			} else
+				return 0;
 		} else {
 			score += 3;
 		}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c
index 8fecb5ff22a0..06572f8ce914 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c
@@ -99,6 +99,7 @@ nvbios_acpi_fast = {
 	.init = acpi_init,
 	.read = acpi_read_fast,
 	.rw = false,
+	.require_checksum = true,
 };
 
 const struct nvbios_source
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/therm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/therm.c
index a54cfec0550d..5babc5a7c7d5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/therm.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/therm.c
@@ -25,17 +25,17 @@
 #include <subdev/bios/bit.h>
 #include <subdev/bios/therm.h>
 
-static u16
+static u32
 therm_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *len, u8 *cnt)
 {
 	struct bit_entry bit_P;
-	u16 therm = 0;
+	u32 therm = 0;
 
 	if (!bit_entry(bios, 'P', &bit_P)) {
 		if (bit_P.version == 1)
-			therm = nvbios_rd16(bios, bit_P.offset + 12);
+			therm = nvbios_rd32(bios, bit_P.offset + 12);
 		else if (bit_P.version == 2)
-			therm = nvbios_rd16(bios, bit_P.offset + 16);
+			therm = nvbios_rd32(bios, bit_P.offset + 16);
 		else
 			nvkm_error(&bios->subdev,
 				   "unknown offset for thermal in BIT P %d\n",
@@ -44,7 +44,7 @@ therm_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *len, u8 *cnt)
 
 	/* exit now if we haven't found the thermal table */
 	if (!therm)
-		return 0x0000;
+		return 0;
 
 	*ver = nvbios_rd08(bios, therm + 0);
 	*hdr = nvbios_rd08(bios, therm + 1);
@@ -53,14 +53,14 @@ therm_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *len, u8 *cnt)
 	return therm + nvbios_rd08(bios, therm + 1);
 }
 
-static u16
+static u32
 nvbios_therm_entry(struct nvkm_bios *bios, int idx, u8 *ver, u8 *len)
 {
 	u8 hdr, cnt;
-	u16 therm = therm_table(bios, ver, &hdr, len, &cnt);
+	u32 therm = therm_table(bios, ver, &hdr, len, &cnt);
 	if (therm && idx < cnt)
 		return therm + idx * *len;
-	return 0x0000;
+	return 0;
 }
 
 int
@@ -70,7 +70,7 @@ nvbios_therm_sensor_parse(struct nvkm_bios *bios,
 {
 	s8 thrs_section, sensor_section, offset;
 	u8 ver, len, i;
-	u16 entry;
+	u32 entry;
 
 	/* we only support the core domain for now */
 	if (domain != NVBIOS_THERM_DOMAIN_CORE)
@@ -154,7 +154,7 @@ nvbios_therm_fan_parse(struct nvkm_bios *bios, struct nvbios_therm_fan *fan)
 {
 	struct nvbios_therm_trip_point *cur_trip = NULL;
 	u8 ver, len, i;
-	u16 entry;
+	u32 entry;
 
 	uint8_t duty_lut[] = { 0, 0, 25, 0, 40, 0, 50, 0,
 				75, 0, 85, 0, 100, 0, 100, 0 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/timing.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/timing.c
index 99f6432ac0af..7e83c3985020 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/timing.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/timing.c
@@ -25,19 +25,19 @@
 #include <subdev/bios/bit.h>
 #include <subdev/bios/timing.h>
 
-u16
+u32
 nvbios_timingTe(struct nvkm_bios *bios,
 		u8 *ver, u8 *hdr, u8 *cnt, u8 *len, u8 *snr, u8 *ssz)
 {
 	struct bit_entry bit_P;
-	u16 timing = 0x0000;
+	u32 timing = 0;
 
 	if (!bit_entry(bios, 'P', &bit_P)) {
 		if (bit_P.version == 1)
-			timing = nvbios_rd16(bios, bit_P.offset + 4);
+			timing = nvbios_rd32(bios, bit_P.offset + 4);
 		else
 		if (bit_P.version == 2)
-			timing = nvbios_rd16(bios, bit_P.offset + 8);
+			timing = nvbios_rd32(bios, bit_P.offset + 8);
 
 		if (timing) {
 			*ver = nvbios_rd08(bios, timing + 0);
@@ -62,15 +62,15 @@ nvbios_timingTe(struct nvkm_bios *bios,
 		}
 	}
 
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_timingEe(struct nvkm_bios *bios, int idx,
 		u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 {
 	u8  snr, ssz;
-	u16 timing = nvbios_timingTe(bios, ver, hdr, cnt, len, &snr, &ssz);
+	u32 timing = nvbios_timingTe(bios, ver, hdr, cnt, len, &snr, &ssz);
 	if (timing && idx < *cnt) {
 		timing += *hdr + idx * (*len + (snr * ssz));
 		*hdr = *len;
@@ -78,14 +78,14 @@ nvbios_timingEe(struct nvkm_bios *bios, int idx,
 		*len = ssz;
 		return timing;
 	}
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_timingEp(struct nvkm_bios *bios, int idx,
 		u8 *ver, u8 *hdr, u8 *cnt, u8 *len, struct nvbios_ramcfg *p)
 {
-	u16 data = nvbios_timingEe(bios, idx, ver, hdr, cnt, len), temp;
+	u32 data = nvbios_timingEe(bios, idx, ver, hdr, cnt, len), temp;
 	p->timing_ver = *ver;
 	p->timing_hdr = *hdr;
 	switch (!!data * *ver) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/vmap.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/vmap.c
index 2f13db745948..c228ca15fa3b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/vmap.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/vmap.c
@@ -25,15 +25,15 @@
 #include <subdev/bios/bit.h>
 #include <subdev/bios/vmap.h>
 
-u16
+u32
 nvbios_vmap_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 {
 	struct bit_entry bit_P;
-	u16 vmap = 0x0000;
+	u32 vmap = 0;
 
 	if (!bit_entry(bios, 'P', &bit_P)) {
 		if (bit_P.version == 2) {
-			vmap = nvbios_rd16(bios, bit_P.offset + 0x20);
+			vmap = nvbios_rd32(bios, bit_P.offset + 0x20);
 			if (vmap) {
 				*ver = nvbios_rd08(bios, vmap + 0);
 				switch (*ver) {
@@ -50,40 +50,50 @@ nvbios_vmap_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 		}
 	}
 
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_vmap_parse(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
 		  struct nvbios_vmap *info)
 {
-	u16 vmap = nvbios_vmap_table(bios, ver, hdr, cnt, len);
+	u32 vmap = nvbios_vmap_table(bios, ver, hdr, cnt, len);
 	memset(info, 0x00, sizeof(*info));
 	switch (!!vmap * *ver) {
 	case 0x10:
+		info->max0 = 0xff;
+		info->max1 = 0xff;
+		info->max2 = 0xff;
+		break;
 	case 0x20:
+		info->max0 = nvbios_rd08(bios, vmap + 0x7);
+		info->max1 = nvbios_rd08(bios, vmap + 0x8);
+		if (*len >= 0xc)
+			info->max2 = nvbios_rd08(bios, vmap + 0xc);
+		else
+			info->max2 = 0xff;
 		break;
 	}
 	return vmap;
 }
 
-u16
+u32
 nvbios_vmap_entry(struct nvkm_bios *bios, int idx, u8 *ver, u8 *len)
 {
 	u8  hdr, cnt;
-	u16 vmap = nvbios_vmap_table(bios, ver, &hdr, &cnt, len);
+	u32 vmap = nvbios_vmap_table(bios, ver, &hdr, &cnt, len);
 	if (vmap && idx < cnt) {
 		vmap = vmap + hdr + (idx * *len);
 		return vmap;
 	}
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_vmap_entry_parse(struct nvkm_bios *bios, int idx, u8 *ver, u8 *len,
 			struct nvbios_vmap_entry *info)
 {
-	u16 vmap = nvbios_vmap_entry(bios, idx, ver, len);
+	u32 vmap = nvbios_vmap_entry(bios, idx, ver, len);
 	memset(info, 0x00, sizeof(*info));
 	switch (!!vmap * *ver) {
 	case 0x10:
@@ -95,7 +105,7 @@ nvbios_vmap_entry_parse(struct nvkm_bios *bios, int idx, u8 *ver, u8 *len,
 		info->arg[2] = nvbios_rd32(bios, vmap + 0x10);
 		break;
 	case 0x20:
-		info->unk0   = nvbios_rd08(bios, vmap + 0x00);
+		info->mode   = nvbios_rd08(bios, vmap + 0x00);
 		info->link   = nvbios_rd08(bios, vmap + 0x01);
 		info->min    = nvbios_rd32(bios, vmap + 0x02);
 		info->max    = nvbios_rd32(bios, vmap + 0x06);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/volt.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/volt.c
index 6e0a33648be9..a7797a9e9cbc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/volt.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/volt.c
@@ -25,18 +25,18 @@
 #include <subdev/bios/bit.h>
 #include <subdev/bios/volt.h>
 
-u16
+u32
 nvbios_volt_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 {
 	struct bit_entry bit_P;
-	u16 volt = 0x0000;
+	u32 volt = 0;
 
 	if (!bit_entry(bios, 'P', &bit_P)) {
 		if (bit_P.version == 2)
-			volt = nvbios_rd16(bios, bit_P.offset + 0x0c);
+			volt = nvbios_rd32(bios, bit_P.offset + 0x0c);
 		else
 		if (bit_P.version == 1)
-			volt = nvbios_rd16(bios, bit_P.offset + 0x10);
+			volt = nvbios_rd32(bios, bit_P.offset + 0x10);
 
 		if (volt) {
 			*ver = nvbios_rd08(bios, volt + 0);
@@ -62,33 +62,37 @@ nvbios_volt_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 		}
 	}
 
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_volt_parse(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
 		  struct nvbios_volt *info)
 {
-	u16 volt = nvbios_volt_table(bios, ver, hdr, cnt, len);
+	u32 volt = nvbios_volt_table(bios, ver, hdr, cnt, len);
 	memset(info, 0x00, sizeof(*info));
 	switch (!!volt * *ver) {
 	case 0x12:
 		info->type    = NVBIOS_VOLT_GPIO;
 		info->vidmask = nvbios_rd08(bios, volt + 0x04);
+		info->ranged  = false;
 		break;
 	case 0x20:
 		info->type    = NVBIOS_VOLT_GPIO;
 		info->vidmask = nvbios_rd08(bios, volt + 0x05);
+		info->ranged  = false;
 		break;
 	case 0x30:
 		info->type    = NVBIOS_VOLT_GPIO;
 		info->vidmask = nvbios_rd08(bios, volt + 0x04);
+		info->ranged  = false;
 		break;
 	case 0x40:
 		info->type    = NVBIOS_VOLT_GPIO;
 		info->base    = nvbios_rd32(bios, volt + 0x04);
 		info->step    = nvbios_rd16(bios, volt + 0x08);
 		info->vidmask = nvbios_rd08(bios, volt + 0x0b);
+		info->ranged  = true; /* XXX: find the flag byte */
 		/*XXX*/
 		info->min     = 0;
 		info->max     = info->base;
@@ -104,32 +108,34 @@ nvbios_volt_parse(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
 			info->pwm_freq  = nvbios_rd32(bios, volt + 0x5) / 1000;
 			info->pwm_range = nvbios_rd32(bios, volt + 0x16);
 		} else {
-			info->type      = NVBIOS_VOLT_GPIO;
-			info->vidmask   = nvbios_rd08(bios, volt + 0x06);
-			info->step      = nvbios_rd16(bios, volt + 0x16);
+			info->type    = NVBIOS_VOLT_GPIO;
+			info->vidmask = nvbios_rd08(bios, volt + 0x06);
+			info->step    = nvbios_rd16(bios, volt + 0x16);
+			info->ranged  =
+				!!(nvbios_rd08(bios, volt + 0x4) & 0x2);
 		}
 		break;
 	}
 	return volt;
 }
 
-u16
+u32
 nvbios_volt_entry(struct nvkm_bios *bios, int idx, u8 *ver, u8 *len)
 {
 	u8  hdr, cnt;
-	u16 volt = nvbios_volt_table(bios, ver, &hdr, &cnt, len);
+	u32 volt = nvbios_volt_table(bios, ver, &hdr, &cnt, len);
 	if (volt && idx < cnt) {
 		volt = volt + hdr + (idx * *len);
 		return volt;
 	}
-	return 0x0000;
+	return 0;
 }
 
-u16
+u32
 nvbios_volt_entry_parse(struct nvkm_bios *bios, int idx, u8 *ver, u8 *len,
 			struct nvbios_volt_entry *info)
 {
-	u16 volt = nvbios_volt_entry(bios, idx, ver, len);
+	u32 volt = nvbios_volt_entry(bios, idx, ver, len);
 	memset(info, 0x00, sizeof(*info));
 	switch (!!volt * *ver) {
 	case 0x12:
@@ -142,7 +148,10 @@ nvbios_volt_entry_parse(struct nvkm_bios *bios, int idx, u8 *ver, u8 *len,
 		info->vid     = nvbios_rd08(bios, volt + 0x01) >> 2;
 		break;
 	case 0x40:
+		break;
 	case 0x50:
+		info->voltage = nvbios_rd32(bios, volt) & 0x001fffff;
+		info->vid     = (nvbios_rd32(bios, volt) >> 23) & 0xff;
 		break;
 	}
 	return volt;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/vpstate.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/vpstate.c
new file mode 100644
index 000000000000..f199270163d2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/vpstate.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2016 Karol Herbst
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Karol Herbst
+ */
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/vpstate.h>
+
+static u32
+nvbios_vpstate_offset(struct nvkm_bios *b)
+{
+	struct bit_entry bit_P;
+
+	if (!bit_entry(b, 'P', &bit_P)) {
+		if (bit_P.version == 2)
+			return nvbios_rd32(b, bit_P.offset + 0x38);
+	}
+
+	return 0x0000;
+}
+
+int
+nvbios_vpstate_parse(struct nvkm_bios *b, struct nvbios_vpstate_header *h)
+{
+	if (!h)
+		return -EINVAL;
+
+	h->offset = nvbios_vpstate_offset(b);
+	if (!h->offset)
+		return -ENODEV;
+
+	h->version = nvbios_rd08(b, h->offset);
+	switch (h->version) {
+	case 0x10:
+		h->hlen     = nvbios_rd08(b, h->offset + 0x1);
+		h->elen     = nvbios_rd08(b, h->offset + 0x2);
+		h->slen     = nvbios_rd08(b, h->offset + 0x3);
+		h->scount   = nvbios_rd08(b, h->offset + 0x4);
+		h->ecount   = nvbios_rd08(b, h->offset + 0x5);
+
+		h->base_id  = nvbios_rd08(b, h->offset + 0x0f);
+		h->boost_id = nvbios_rd08(b, h->offset + 0x10);
+		h->tdp_id   = nvbios_rd08(b, h->offset + 0x11);
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+int
+nvbios_vpstate_entry(struct nvkm_bios *b, struct nvbios_vpstate_header *h,
+		     u8 idx, struct nvbios_vpstate_entry *e)
+{
+	u32 offset;
+
+	if (!e || !h || idx > h->ecount)
+		return -EINVAL;
+
+	offset = h->offset + h->hlen + idx * (h->elen + (h->slen * h->scount));
+	e->pstate    = nvbios_rd08(b, offset);
+	e->clock_mhz = nvbios_rd16(b, offset + 0x5);
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c
index 7102c25320fc..e4c8d310d870 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c
@@ -27,6 +27,7 @@
 #include <subdev/bios/boost.h>
 #include <subdev/bios/cstep.h>
 #include <subdev/bios/perf.h>
+#include <subdev/bios/vpstate.h>
 #include <subdev/fb.h>
 #include <subdev/therm.h>
 #include <subdev/volt.h>
@@ -43,13 +44,13 @@ nvkm_clk_adjust(struct nvkm_clk *clk, bool adjust,
 	struct nvkm_bios *bios = clk->subdev.device->bios;
 	struct nvbios_boostE boostE;
 	u8  ver, hdr, cnt, len;
-	u16 data;
+	u32 data;
 
 	data = nvbios_boostEm(bios, pstate, &ver, &hdr, &cnt, &len, &boostE);
 	if (data) {
 		struct nvbios_boostS boostS;
 		u8  idx = 0, sver, shdr;
-		u16 subd;
+		u32 subd;
 
 		input = max(boostE.min, input);
 		input = min(boostE.max, input);
@@ -74,6 +75,88 @@ nvkm_clk_adjust(struct nvkm_clk *clk, bool adjust,
 /******************************************************************************
  * C-States
  *****************************************************************************/
+static bool
+nvkm_cstate_valid(struct nvkm_clk *clk, struct nvkm_cstate *cstate,
+		  u32 max_volt, int temp)
+{
+	const struct nvkm_domain *domain = clk->domains;
+	struct nvkm_volt *volt = clk->subdev.device->volt;
+	int voltage;
+
+	while (domain && domain->name != nv_clk_src_max) {
+		if (domain->flags & NVKM_CLK_DOM_FLAG_VPSTATE) {
+			u32 freq = cstate->domain[domain->name];
+			switch (clk->boost_mode) {
+			case NVKM_CLK_BOOST_NONE:
+				if (clk->base_khz && freq > clk->base_khz)
+					return false;
+			case NVKM_CLK_BOOST_BIOS:
+				if (clk->boost_khz && freq > clk->boost_khz)
+					return false;
+			}
+		}
+		domain++;
+	}
+
+	if (!volt)
+		return true;
+
+	voltage = nvkm_volt_map(volt, cstate->voltage, temp);
+	if (voltage < 0)
+		return false;
+	return voltage <= min(max_volt, volt->max_uv);
+}
+
+static struct nvkm_cstate *
+nvkm_cstate_find_best(struct nvkm_clk *clk, struct nvkm_pstate *pstate,
+		      struct nvkm_cstate *start)
+{
+	struct nvkm_device *device = clk->subdev.device;
+	struct nvkm_volt *volt = device->volt;
+	struct nvkm_cstate *cstate;
+	int max_volt;
+
+	if (!pstate || !start)
+		return NULL;
+
+	if (!volt)
+		return start;
+
+	max_volt = volt->max_uv;
+	if (volt->max0_id != 0xff)
+		max_volt = min(max_volt,
+			       nvkm_volt_map(volt, volt->max0_id, clk->temp));
+	if (volt->max1_id != 0xff)
+		max_volt = min(max_volt,
+			       nvkm_volt_map(volt, volt->max1_id, clk->temp));
+	if (volt->max2_id != 0xff)
+		max_volt = min(max_volt,
+			       nvkm_volt_map(volt, volt->max2_id, clk->temp));
+
+	for (cstate = start; &cstate->head != &pstate->list;
+	     cstate = list_entry(cstate->head.prev, typeof(*cstate), head)) {
+		if (nvkm_cstate_valid(clk, cstate, max_volt, clk->temp))
+			break;
+	}
+
+	return cstate;
+}
+
+static struct nvkm_cstate *
+nvkm_cstate_get(struct nvkm_clk *clk, struct nvkm_pstate *pstate, int cstatei)
+{
+	struct nvkm_cstate *cstate;
+	if (cstatei == NVKM_CLK_CSTATE_HIGHEST)
+		return list_last_entry(&pstate->list, typeof(*cstate), head);
+	else {
+		list_for_each_entry(cstate, &pstate->list, head) {
+			if (cstate->id == cstatei)
+				return cstate;
+		}
+	}
+	return NULL;
+}
+
 static int
 nvkm_cstate_prog(struct nvkm_clk *clk, struct nvkm_pstate *pstate, int cstatei)
 {
@@ -85,7 +168,8 @@ nvkm_cstate_prog(struct nvkm_clk *clk, struct nvkm_pstate *pstate, int cstatei)
 	int ret;
 
 	if (!list_empty(&pstate->list)) {
-		cstate = list_entry(pstate->list.prev, typeof(*cstate), head);
+		cstate = nvkm_cstate_get(clk, pstate, cstatei);
+		cstate = nvkm_cstate_find_best(clk, pstate, cstate);
 	} else {
 		cstate = &pstate->base;
 	}
@@ -99,7 +183,8 @@ nvkm_cstate_prog(struct nvkm_clk *clk, struct nvkm_pstate *pstate, int cstatei)
 	}
 
 	if (volt) {
-		ret = nvkm_volt_set_id(volt, cstate->voltage, +1);
+		ret = nvkm_volt_set_id(volt, cstate->voltage,
+				       pstate->base.voltage, clk->temp, +1);
 		if (ret && ret != -ENODEV) {
 			nvkm_error(subdev, "failed to raise voltage: %d\n", ret);
 			return ret;
@@ -113,7 +198,8 @@ nvkm_cstate_prog(struct nvkm_clk *clk, struct nvkm_pstate *pstate, int cstatei)
 	}
 
 	if (volt) {
-		ret = nvkm_volt_set_id(volt, cstate->voltage, -1);
+		ret = nvkm_volt_set_id(volt, cstate->voltage,
+				       pstate->base.voltage, clk->temp, -1);
 		if (ret && ret != -ENODEV)
 			nvkm_error(subdev, "failed to lower voltage: %d\n", ret);
 	}
@@ -138,22 +224,27 @@ static int
 nvkm_cstate_new(struct nvkm_clk *clk, int idx, struct nvkm_pstate *pstate)
 {
 	struct nvkm_bios *bios = clk->subdev.device->bios;
+	struct nvkm_volt *volt = clk->subdev.device->volt;
 	const struct nvkm_domain *domain = clk->domains;
 	struct nvkm_cstate *cstate = NULL;
 	struct nvbios_cstepX cstepX;
 	u8  ver, hdr;
-	u16 data;
+	u32 data;
 
 	data = nvbios_cstepXp(bios, idx, &ver, &hdr, &cstepX);
 	if (!data)
 		return -ENOENT;
 
+	if (volt && nvkm_volt_map_min(volt, cstepX.voltage) > volt->max_uv)
+		return -EINVAL;
+
 	cstate = kzalloc(sizeof(*cstate), GFP_KERNEL);
 	if (!cstate)
 		return -ENOMEM;
 
 	*cstate = pstate->base;
 	cstate->voltage = cstepX.voltage;
+	cstate->id = idx;
 
 	while (domain && domain->name != nv_clk_src_max) {
 		if (domain->flags & NVKM_CLK_DOM_FLAG_CORE) {
@@ -175,7 +266,7 @@ static int
 nvkm_pstate_prog(struct nvkm_clk *clk, int pstatei)
 {
 	struct nvkm_subdev *subdev = &clk->subdev;
-	struct nvkm_ram *ram = subdev->device->fb->ram;
+	struct nvkm_fb *fb = subdev->device->fb;
 	struct nvkm_pci *pci = subdev->device->pci;
 	struct nvkm_pstate *pstate;
 	int ret, idx = 0;
@@ -190,7 +281,8 @@ nvkm_pstate_prog(struct nvkm_clk *clk, int pstatei)
 
 	nvkm_pcie_set_link(pci, pstate->pcie_speed, pstate->pcie_width);
 
-	if (ram && ram->func->calc) {
+	if (fb && fb->ram && fb->ram->func->calc) {
+		struct nvkm_ram *ram = fb->ram;
 		int khz = pstate->base.domain[nv_clk_src_mem];
 		do {
 			ret = ram->func->calc(ram, khz);
@@ -200,7 +292,7 @@ nvkm_pstate_prog(struct nvkm_clk *clk, int pstatei)
 		ram->func->tidy(ram);
 	}
 
-	return nvkm_cstate_prog(clk, pstate, 0);
+	return nvkm_cstate_prog(clk, pstate, NVKM_CLK_CSTATE_HIGHEST);
 }
 
 static void
@@ -214,14 +306,14 @@ nvkm_pstate_work(struct work_struct *work)
 		return;
 	clk->pwrsrc = power_supply_is_system_supplied();
 
-	nvkm_trace(subdev, "P %d PWR %d U(AC) %d U(DC) %d A %d T %d D %d\n",
+	nvkm_trace(subdev, "P %d PWR %d U(AC) %d U(DC) %d A %d T %d°C D %d\n",
 		   clk->pstate, clk->pwrsrc, clk->ustate_ac, clk->ustate_dc,
-		   clk->astate, clk->tstate, clk->dstate);
+		   clk->astate, clk->temp, clk->dstate);
 
 	pstate = clk->pwrsrc ? clk->ustate_ac : clk->ustate_dc;
 	if (clk->state_nr && pstate != -1) {
 		pstate = (pstate < 0) ? clk->astate : pstate;
-		pstate = min(pstate, clk->state_nr - 1 + clk->tstate);
+		pstate = min(pstate, clk->state_nr - 1);
 		pstate = max(pstate, clk->dstate);
 	} else {
 		pstate = clk->pstate = -1;
@@ -316,7 +408,7 @@ nvkm_pstate_new(struct nvkm_clk *clk, int idx)
 	struct nvbios_cstepE cstepE;
 	struct nvbios_perfE perfE;
 	u8  ver, hdr, cnt, len;
-	u16 data;
+	u32 data;
 
 	data = nvbios_perfEp(bios, idx, &ver, &hdr, &cnt, &len, &perfE);
 	if (!data)
@@ -448,13 +540,12 @@ nvkm_clk_astate(struct nvkm_clk *clk, int req, int rel, bool wait)
 }
 
 int
-nvkm_clk_tstate(struct nvkm_clk *clk, int req, int rel)
+nvkm_clk_tstate(struct nvkm_clk *clk, u8 temp)
 {
-	if (!rel) clk->tstate  = req;
-	if ( rel) clk->tstate += rel;
-	clk->tstate = min(clk->tstate, 0);
-	clk->tstate = max(clk->tstate, -(clk->state_nr - 1));
-	return nvkm_pstate_calc(clk, true);
+	if (clk->temp == temp)
+		return 0;
+	clk->temp = temp;
+	return nvkm_pstate_calc(clk, false);
 }
 
 int
@@ -524,9 +615,9 @@ nvkm_clk_init(struct nvkm_subdev *subdev)
 		return clk->func->init(clk);
 
 	clk->astate = clk->state_nr - 1;
-	clk->tstate = 0;
 	clk->dstate = 0;
 	clk->pstate = -1;
+	clk->temp = 90; /* reasonable default value */
 	nvkm_pstate_calc(clk, true);
 	return 0;
 }
@@ -561,10 +652,22 @@ int
 nvkm_clk_ctor(const struct nvkm_clk_func *func, struct nvkm_device *device,
 	      int index, bool allow_reclock, struct nvkm_clk *clk)
 {
+	struct nvkm_subdev *subdev = &clk->subdev;
+	struct nvkm_bios *bios = device->bios;
 	int ret, idx, arglen;
 	const char *mode;
+	struct nvbios_vpstate_header h;
+
+	nvkm_subdev_ctor(&nvkm_clk, device, index, subdev);
+
+	if (bios && !nvbios_vpstate_parse(bios, &h)) {
+		struct nvbios_vpstate_entry base, boost;
+		if (!nvbios_vpstate_entry(bios, &h, h.boost_id, &boost))
+			clk->boost_khz = boost.clock_mhz * 1000;
+		if (!nvbios_vpstate_entry(bios, &h, h.base_id, &base))
+			clk->base_khz = base.clock_mhz * 1000;
+	}
 
-	nvkm_subdev_ctor(&nvkm_clk, device, index, &clk->subdev);
 	clk->func = func;
 	INIT_LIST_HEAD(&clk->states);
 	clk->domains = func->domains;
@@ -607,6 +710,8 @@ nvkm_clk_ctor(const struct nvkm_clk_func *func, struct nvkm_device *device,
 	if (mode)
 		clk->ustate_dc = nvkm_clk_nstate(clk, mode, arglen);
 
+	clk->boost_mode = nvkm_longopt(device->cfgopt, "NvBoost",
+				       NVKM_CLK_BOOST_NONE);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c
index 89d5543118cf..7f67f9f5a550 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c
@@ -457,7 +457,7 @@ gf100_clk = {
 		{ nv_clk_src_hubk06 , 0x00 },
 		{ nv_clk_src_hubk01 , 0x01 },
 		{ nv_clk_src_copy   , 0x02 },
-		{ nv_clk_src_gpc    , 0x03, 0, "core", 2000 },
+		{ nv_clk_src_gpc    , 0x03, NVKM_CLK_DOM_FLAG_VPSTATE, "core", 2000 },
 		{ nv_clk_src_rop    , 0x04 },
 		{ nv_clk_src_mem    , 0x05, 0, "memory", 1000 },
 		{ nv_clk_src_vdec   , 0x06 },
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c
index 06bc0d2d6ae1..0b37e3da7feb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c
@@ -491,7 +491,7 @@ gk104_clk = {
 	.domains = {
 		{ nv_clk_src_crystal, 0xff },
 		{ nv_clk_src_href   , 0xff },
-		{ nv_clk_src_gpc    , 0x00, NVKM_CLK_DOM_FLAG_CORE, "core", 2000 },
+		{ nv_clk_src_gpc    , 0x00, NVKM_CLK_DOM_FLAG_CORE | NVKM_CLK_DOM_FLAG_VPSTATE, "core", 2000 },
 		{ nv_clk_src_hubk07 , 0x01, NVKM_CLK_DOM_FLAG_CORE },
 		{ nv_clk_src_rop    , 0x02, NVKM_CLK_DOM_FLAG_CORE },
 		{ nv_clk_src_mem    , 0x03, 0, "memory", 500 },
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c
index 056702ef69aa..96e0941c8edd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c
@@ -180,7 +180,7 @@ gt215_clk_read(struct nvkm_clk *base, enum nv_clk_src src)
 	return 0;
 }
 
-int
+static int
 gt215_clk_info(struct nvkm_clk *base, int idx, u32 khz,
 	       struct gt215_clk_info *info)
 {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c
index a410c0db8a08..1730371933df 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c
@@ -26,6 +26,7 @@
 #include <subdev/bios.h>
 #include <subdev/bios/bit.h>
 #include <subdev/bios/pmu.h>
+#include <subdev/timer.h>
 
 static void
 pmu_code(struct nv50_devinit *init, u32 pmu, u32 img, u32 len, bool sec)
@@ -123,21 +124,13 @@ gm200_devinit_post(struct nvkm_devinit *base, bool post)
 		return -EINVAL;
 	}
 
-	/* reset PMU and load init table parser ucode */
-	if (post) {
-		nvkm_mask(device, 0x000200, 0x00002000, 0x00000000);
-		nvkm_mask(device, 0x000200, 0x00002000, 0x00002000);
-		nvkm_rd32(device, 0x000200);
-		while (nvkm_rd32(device, 0x10a10c) & 0x00000006) {
-		}
-	}
-
 	ret = pmu_load(init, 0x04, post, &exec, &args);
 	if (ret)
 		return ret;
 
 	/* upload first chunk of init data */
 	if (post) {
+		// devinit tables
 		u32 pmu = pmu_args(init, args + 0x08, 0x08);
 		u32 img = nvbios_rd16(bios, bit_I.offset + 0x14);
 		u32 len = nvbios_rd16(bios, bit_I.offset + 0x16);
@@ -146,6 +139,7 @@ gm200_devinit_post(struct nvkm_devinit *base, bool post)
 
 	/* upload second chunk of init data */
 	if (post) {
+		// devinit boot scripts
 		u32 pmu = pmu_args(init, args + 0x08, 0x10);
 		u32 img = nvbios_rd16(bios, bit_I.offset + 0x18);
 		u32 len = nvbios_rd16(bios, bit_I.offset + 0x1a);
@@ -156,8 +150,11 @@ gm200_devinit_post(struct nvkm_devinit *base, bool post)
 	if (post) {
 		nvkm_wr32(device, 0x10a040, 0x00005000);
 		pmu_exec(init, exec);
-		while (!(nvkm_rd32(device, 0x10a040) & 0x00002000)) {
-		}
+		if (nvkm_msec(device, 2000,
+			if (nvkm_rd32(device, 0x10a040) & 0x00002000)
+				break;
+		) < 0)
+			return -ETIMEDOUT;
 	}
 
 	/* load and execute some other ucode image (bios therm?) */
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild
index edcc157e6ac8..63566ba12fbb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild
@@ -24,8 +24,9 @@ nvkm-y += nvkm/subdev/fb/gk104.o
 nvkm-y += nvkm/subdev/fb/gk20a.o
 nvkm-y += nvkm/subdev/fb/gm107.o
 nvkm-y += nvkm/subdev/fb/gm200.o
+nvkm-y += nvkm/subdev/fb/gm20b.o
 nvkm-y += nvkm/subdev/fb/gp100.o
-nvkm-y += nvkm/subdev/fb/gp104.o
+nvkm-y += nvkm/subdev/fb/gp102.o
 
 nvkm-y += nvkm/subdev/fb/ram.o
 nvkm-y += nvkm/subdev/fb/ramnv04.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
index 76433cc66fff..3841ad6be99e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
@@ -50,24 +50,33 @@ gf100_fb_intr(struct nvkm_fb *base)
 }
 
 int
-gf100_fb_oneinit(struct nvkm_fb *fb)
+gf100_fb_oneinit(struct nvkm_fb *base)
 {
-	struct nvkm_device *device = fb->subdev.device;
+	struct gf100_fb *fb = gf100_fb(base);
+	struct nvkm_device *device = fb->base.subdev.device;
 	int ret, size = 0x1000;
 
 	size = nvkm_longopt(device->cfgopt, "MmuDebugBufferSize", size);
 	size = min(size, 0x1000);
 
 	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, size, 0x1000,
-			      false, &fb->mmu_rd);
+			      false, &fb->base.mmu_rd);
 	if (ret)
 		return ret;
 
 	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, size, 0x1000,
-			      false, &fb->mmu_wr);
+			      false, &fb->base.mmu_wr);
 	if (ret)
 		return ret;
 
+	fb->r100c10_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+	if (fb->r100c10_page) {
+		fb->r100c10 = dma_map_page(device->dev, fb->r100c10_page, 0,
+					   PAGE_SIZE, DMA_BIDIRECTIONAL);
+		if (dma_mapping_error(device->dev, fb->r100c10))
+			return -EFAULT;
+	}
+
 	return 0;
 }
 
@@ -123,14 +132,6 @@ gf100_fb_new_(const struct nvkm_fb_func *func, struct nvkm_device *device,
 	nvkm_fb_ctor(func, device, index, &fb->base);
 	*pfb = &fb->base;
 
-	fb->r100c10_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-	if (fb->r100c10_page) {
-		fb->r100c10 = dma_map_page(device->dev, fb->r100c10_page, 0,
-					   PAGE_SIZE, DMA_BIDIRECTIONAL);
-		if (dma_mapping_error(device->dev, fb->r100c10))
-			return -EFAULT;
-	}
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h
index 449f431644b3..412eb89834e8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h
@@ -16,4 +16,8 @@ void gf100_fb_init(struct nvkm_fb *);
 void gf100_fb_intr(struct nvkm_fb *);
 
 void gp100_fb_init(struct nvkm_fb *);
+
+void gm200_fb_init_page(struct nvkm_fb *fb);
+void gm200_fb_init(struct nvkm_fb *base);
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c
index f815fe2bbf08..5d34d6136616 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2014-2016, NVIDIA CORPORATION. All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -20,27 +20,21 @@
  * DEALINGS IN THE SOFTWARE.
  */
 #include "priv.h"
+#include "gf100.h"
 
-#include <core/memory.h>
-
-static void
-gk20a_fb_init(struct nvkm_fb *fb)
-{
-	struct nvkm_device *device = fb->subdev.device;
-	nvkm_wr32(device, 0x100cc8, nvkm_memory_addr(fb->mmu_wr) >> 8);
-	nvkm_wr32(device, 0x100ccc, nvkm_memory_addr(fb->mmu_rd) >> 8);
-}
-
+/* GK20A's FB is similar to GF100's, but without the ability to allocate VRAM */
 static const struct nvkm_fb_func
 gk20a_fb = {
+	.dtor = gf100_fb_dtor,
 	.oneinit = gf100_fb_oneinit,
-	.init = gk20a_fb_init,
+	.init = gf100_fb_init,
 	.init_page = gf100_fb_init_page,
+	.intr = gf100_fb_intr,
 	.memtype_valid = gf100_fb_memtype_valid,
 };
 
 int
 gk20a_fb_new(struct nvkm_device *device, int index, struct nvkm_fb **pfb)
 {
-	return nvkm_fb_new_(&gk20a_fb, device, index, pfb);
+	return gf100_fb_new_(&gk20a_fb, device, index, pfb);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c
index 62f653240be3..fe5886013ac0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c
@@ -44,7 +44,7 @@ gm200_fb_init_page(struct nvkm_fb *fb)
 	}
 }
 
-static void
+void
 gm200_fb_init(struct nvkm_fb *base)
 {
 	struct gf100_fb *fb = gf100_fb(base);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c
new file mode 100644
index 000000000000..b87c233bcd6d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#include "priv.h"
+#include "gf100.h"
+
+/* GM20B's FB is similar to GM200, but without the ability to allocate VRAM */
+static const struct nvkm_fb_func
+gm20b_fb = {
+	.dtor = gf100_fb_dtor,
+	.oneinit = gf100_fb_oneinit,
+	.init = gm200_fb_init,
+	.init_page = gm200_fb_init_page,
+	.intr = gf100_fb_intr,
+	.memtype_valid = gf100_fb_memtype_valid,
+};
+
+int
+gm20b_fb_new(struct nvkm_device *device, int index, struct nvkm_fb **pfb)
+{
+	return gf100_fb_new_(&gm20b_fb, device, index, pfb);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c
index 92cb71861bec..73b4ae1c73dc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c
@@ -27,7 +27,7 @@
 #include <core/memory.h>
 
 static const struct nvkm_fb_func
-gp104_fb = {
+gp102_fb = {
 	.dtor = gf100_fb_dtor,
 	.oneinit = gf100_fb_oneinit,
 	.init = gp100_fb_init,
@@ -37,7 +37,7 @@ gp104_fb = {
 };
 
 int
-gp104_fb_new(struct nvkm_device *device, int index, struct nvkm_fb **pfb)
+gp102_fb_new(struct nvkm_device *device, int index, struct nvkm_fb **pfb)
 {
-	return gf100_fb_new_(&gp104_fb, device, index, pfb);
+	return gf100_fb_new_(&gp102_fb, device, index, pfb);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c
index 1b5fb02eab2a..0595e0722bfc 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c
@@ -210,6 +210,23 @@ nv50_fb_intr(struct nvkm_fb *base)
 	nvkm_fifo_chan_put(fifo, flags, &chan);
 }
 
+static int
+nv50_fb_oneinit(struct nvkm_fb *base)
+{
+	struct nv50_fb *fb = nv50_fb(base);
+	struct nvkm_device *device = fb->base.subdev.device;
+
+	fb->r100c08_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+	if (fb->r100c08_page) {
+		fb->r100c08 = dma_map_page(device->dev, fb->r100c08_page, 0,
+					   PAGE_SIZE, DMA_BIDIRECTIONAL);
+		if (dma_mapping_error(device->dev, fb->r100c08))
+			return -EFAULT;
+	}
+
+	return 0;
+}
+
 static void
 nv50_fb_init(struct nvkm_fb *base)
 {
@@ -245,6 +262,7 @@ nv50_fb_dtor(struct nvkm_fb *base)
 static const struct nvkm_fb_func
 nv50_fb_ = {
 	.dtor = nv50_fb_dtor,
+	.oneinit = nv50_fb_oneinit,
 	.init = nv50_fb_init,
 	.intr = nv50_fb_intr,
 	.ram_new = nv50_fb_ram_new,
@@ -263,16 +281,6 @@ nv50_fb_new_(const struct nv50_fb_func *func, struct nvkm_device *device,
 	fb->func = func;
 	*pfb = &fb->base;
 
-	fb->r100c08_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-	if (fb->r100c08_page) {
-		fb->r100c08 = dma_map_page(device->dev, fb->r100c08_page, 0,
-					   PAGE_SIZE, DMA_BIDIRECTIONAL);
-		if (dma_mapping_error(device->dev, fb->r100c08))
-			return -EFAULT;
-	} else {
-		nvkm_warn(&fb->base.subdev, "failed 100c08 page alloc\n");
-	}
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h
index b9ec0ae6723a..b60068b7d8f9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h
@@ -24,6 +24,7 @@ int  gf100_ram_ctor(const struct nvkm_ram_func *, struct nvkm_fb *,
 int  gf100_ram_get(struct nvkm_ram *, u64, u32, u32, u32, struct nvkm_mem **);
 void gf100_ram_put(struct nvkm_ram *, struct nvkm_mem **);
 
+int  gk104_ram_ctor(struct nvkm_fb *, struct nvkm_ram **, u32);
 int  gk104_ram_init(struct nvkm_ram *ram);
 
 /* RAM type-specific MR calculation routines */
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c
index 772425ca5a9e..093223d1df4f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c
@@ -420,8 +420,6 @@ gf100_ram_tidy(struct nvkm_ram *base)
 	ram_exec(&ram->fuc, false);
 }
 
-extern const u8 gf100_pte_storage_type_map[256];
-
 void
 gf100_ram_put(struct nvkm_ram *ram, struct nvkm_mem **pmem)
 {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c
index 1fa3ade468ae..7904fa41acef 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c
@@ -259,7 +259,9 @@ gk104_ram_calc_gddr5(struct gk104_ram *ram, u32 freq)
 
 	ram_mask(fuc, 0x10f808, 0x40000000, 0x40000000);
 	ram_block(fuc);
-	ram_wr32(fuc, 0x62c000, 0x0f0f0000);
+
+	if (nvkm_device_engine(ram->base.fb->subdev.device, NVKM_ENGINE_DISP))
+		ram_wr32(fuc, 0x62c000, 0x0f0f0000);
 
 	/* MR1: turn termination on early, for some reason.. */
 	if ((ram->base.mr[1] & 0x03c) != 0x030) {
@@ -658,7 +660,9 @@ gk104_ram_calc_gddr5(struct gk104_ram *ram, u32 freq)
 		gk104_ram_train(fuc, 0x80020000, 0x01000000);
 
 	ram_unblock(fuc);
-	ram_wr32(fuc, 0x62c000, 0x0f0f0f00);
+
+	if (nvkm_device_engine(ram->base.fb->subdev.device, NVKM_ENGINE_DISP))
+		ram_wr32(fuc, 0x62c000, 0x0f0f0f00);
 
 	if (next->bios.rammap_11_08_01)
 		data = 0x00000800;
@@ -706,7 +710,9 @@ gk104_ram_calc_sddr3(struct gk104_ram *ram, u32 freq)
 
 	ram_mask(fuc, 0x10f808, 0x40000000, 0x40000000);
 	ram_block(fuc);
-	ram_wr32(fuc, 0x62c000, 0x0f0f0000);
+
+	if (nvkm_device_engine(ram->base.fb->subdev.device, NVKM_ENGINE_DISP))
+		ram_wr32(fuc, 0x62c000, 0x0f0f0000);
 
 	if (vc == 1 && ram_have(fuc, gpio2E)) {
 		u32 temp  = ram_mask(fuc, gpio2E, 0x3000, fuc->r_func2E[1]);
@@ -936,7 +942,9 @@ gk104_ram_calc_sddr3(struct gk104_ram *ram, u32 freq)
 	ram_nsec(fuc, 1000);
 
 	ram_unblock(fuc);
-	ram_wr32(fuc, 0x62c000, 0x0f0f0f00);
+
+	if (nvkm_device_engine(ram->base.fb->subdev.device, NVKM_ENGINE_DISP))
+		ram_wr32(fuc, 0x62c000, 0x0f0f0f00);
 
 	if (next->bios.rammap_11_08_01)
 		data = 0x00000800;
@@ -1530,6 +1538,12 @@ gk104_ram_func = {
 int
 gk104_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
 {
+	return gk104_ram_ctor(fb, pram, 0x022554);
+}
+
+int
+gk104_ram_ctor(struct nvkm_fb *fb, struct nvkm_ram **pram, u32 maskaddr)
+{
 	struct nvkm_subdev *subdev = &fb->subdev;
 	struct nvkm_device *device = subdev->device;
 	struct nvkm_bios *bios = device->bios;
@@ -1544,7 +1558,7 @@ gk104_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
 		return -ENOMEM;
 	*pram = &ram->base;
 
-	ret = gf100_ram_ctor(&gk104_ram_func, fb, 0x022554, &ram->base);
+	ret = gf100_ram_ctor(&gk104_ram_func, fb, maskaddr, &ram->base);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c
index 43d807f6ca71..ac862d1d77bd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgm107.c
@@ -23,18 +23,8 @@
  */
 #include "ram.h"
 
-static const struct nvkm_ram_func
-gm107_ram_func = {
-	.init = gk104_ram_init,
-	.get = gf100_ram_get,
-	.put = gf100_ram_put,
-};
-
 int
 gm107_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
 {
-	if (!(*pram = kzalloc(sizeof(**pram), GFP_KERNEL)))
-		return -ENOMEM;
-
-	return gf100_ram_ctor(&gm107_ram_func, fb, 0x021c14, *pram);
+	return gk104_ram_ctor(fb, pram, 0x021c14);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c
index f3be408b5e5e..405faabe8dcd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c
@@ -92,13 +92,13 @@ gp100_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
 	enum nvkm_ram_type type = nvkm_fb_bios_memtype(device->bios);
 	const u32 rsvd_head = ( 256 * 1024); /* vga memory */
 	const u32 rsvd_tail = (1024 * 1024); /* vbios etc */
-	u32 fbpa_num = nvkm_rd32(device, 0x022438), fbpa;
+	u32 fbpa_num = nvkm_rd32(device, 0x02243c), fbpa;
 	u32 fbio_opt = nvkm_rd32(device, 0x021c14);
 	u64 part, size = 0, comm = ~0ULL;
 	bool mixed = false;
 	int ret;
 
-	nvkm_debug(subdev, "022438: %08x\n", fbpa_num);
+	nvkm_debug(subdev, "02243c: %08x\n", fbpa_num);
 	nvkm_debug(subdev, "021c14: %08x\n", fbio_opt);
 	for (fbpa = 0; fbpa < fbpa_num; fbpa++) {
 		if (!(fbio_opt & (1 << fbpa))) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c
index d15ea886df27..f10664372161 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c
@@ -95,7 +95,7 @@ struct gt215_ram {
 	struct gt215_ltrain ltrain;
 };
 
-void
+static void
 gt215_link_train_calc(u32 *vals, struct gt215_ltrain *train)
 {
 	int i, lo, hi;
@@ -149,7 +149,7 @@ gt215_link_train_calc(u32 *vals, struct gt215_ltrain *train)
 /*
  * Link training for (at least) DDR3
  */
-int
+static int
 gt215_link_train(struct gt215_ram *ram)
 {
 	struct gt215_ltrain *train = &ram->ltrain;
@@ -267,7 +267,7 @@ out:
 	return ret;
 }
 
-int
+static int
 gt215_link_train_init(struct gt215_ram *ram)
 {
 	static const u32 pattern[16] = {
@@ -333,7 +333,7 @@ gt215_link_train_init(struct gt215_ram *ram)
 	return 0;
 }
 
-void
+static void
 gt215_link_train_fini(struct gt215_ram *ram)
 {
 	if (ram->ltrain.mem)
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr2.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr2.c
index b9f1ffdfc602..4dcd8742f2da 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr2.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr2.c
@@ -23,6 +23,7 @@
  *          Ben Skeggs
  */
 #include "priv.h"
+#include "ram.h"
 
 struct ramxlat {
 	int id;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr3.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr3.c
index 26900333b1d6..eca8a445eab3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr3.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr3.c
@@ -23,6 +23,7 @@
  * 	    Roy Spliet <rspliet@eclipso.eu>
  */
 #include "priv.h"
+#include "ram.h"
 
 struct ramxlat {
 	int id;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/gk104.c
index 3f45afd17d5a..2ead515b8530 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/gk104.c
@@ -37,7 +37,7 @@ gk104_gpio_intr_stat(struct nvkm_gpio *gpio, u32 *hi, u32 *lo)
 	nvkm_wr32(device, 0x00dc80, intr1);
 }
 
-void
+static void
 gk104_gpio_intr_mask(struct nvkm_gpio *gpio, u32 type, u32 mask, u32 data)
 {
 	struct nvkm_device *device = gpio->subdev.device;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c
index f0851d57df2f..01d5c5a56e2e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c
@@ -74,7 +74,7 @@ nvkm_i2c_aux_i2c_func(struct i2c_adapter *adap)
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
 }
 
-const struct i2c_algorithm
+static const struct i2c_algorithm
 nvkm_i2c_aux_i2c_algo = {
 	.master_xfer = nvkm_i2c_aux_i2c_xfer,
 	.functionality = nvkm_i2c_aux_i2c_func
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxg94.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxg94.c
index 954f5b76bfcf..b80236a4eeac 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxg94.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxg94.c
@@ -79,7 +79,7 @@ g94_i2c_aux_xfer(struct nvkm_i2c_aux *obj, bool retry,
 	struct g94_i2c_aux *aux = g94_i2c_aux(obj);
 	struct nvkm_device *device = aux->base.pad->i2c->subdev.device;
 	const u32 base = aux->ch * 0x50;
-	u32 ctrl, stat, timeout, retries;
+	u32 ctrl, stat, timeout, retries = 0;
 	u32 xbuf[4] = {};
 	int ret, i;
 
@@ -111,7 +111,7 @@ g94_i2c_aux_xfer(struct nvkm_i2c_aux *obj, bool retry,
 	nvkm_wr32(device, 0x00e4e0 + base, addr);
 
 	/* (maybe) retry transaction a number of times on failure... */
-	for (retries = 0; !ret && retries < 32; retries++) {
+	do {
 		/* reset, and delay a while if this is a retry */
 		nvkm_wr32(device, 0x00e4e4 + base, 0x80000000 | ctrl);
 		nvkm_wr32(device, 0x00e4e4 + base, 0x00000000 | ctrl);
@@ -131,20 +131,20 @@ g94_i2c_aux_xfer(struct nvkm_i2c_aux *obj, bool retry,
 				goto out;
 			}
 		} while (ctrl & 0x00010000);
-		ret = 1;
+		ret = 0;
 
 		/* read status, and check if transaction completed ok */
 		stat = nvkm_mask(device, 0x00e4e8 + base, 0, 0);
 		if ((stat & 0x000f0000) == 0x00080000 ||
 		    (stat & 0x000f0000) == 0x00020000)
-			ret = retry ? 0 : 1;
+			ret = 1;
 		if ((stat & 0x00000100))
 			ret = -ETIMEDOUT;
 		if ((stat & 0x00000e00))
 			ret = -EIO;
 
 		AUX_TRACE(&aux->base, "%02d %08x %08x", retries, ctrl, stat);
-	}
+	} while (ret && retry && retries++ < 32);
 
 	if (type & 1) {
 		for (i = 0; i < 16; i += 4) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxgm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxgm200.c
index 61d729b82c69..ed458c7f056b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxgm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxgm200.c
@@ -79,7 +79,7 @@ gm200_i2c_aux_xfer(struct nvkm_i2c_aux *obj, bool retry,
 	struct gm200_i2c_aux *aux = gm200_i2c_aux(obj);
 	struct nvkm_device *device = aux->base.pad->i2c->subdev.device;
 	const u32 base = aux->ch * 0x50;
-	u32 ctrl, stat, timeout, retries;
+	u32 ctrl, stat, timeout, retries = 0;
 	u32 xbuf[4] = {};
 	int ret, i;
 
@@ -111,7 +111,7 @@ gm200_i2c_aux_xfer(struct nvkm_i2c_aux *obj, bool retry,
 	nvkm_wr32(device, 0x00d950 + base, addr);
 
 	/* (maybe) retry transaction a number of times on failure... */
-	for (retries = 0; !ret && retries < 32; retries++) {
+	do {
 		/* reset, and delay a while if this is a retry */
 		nvkm_wr32(device, 0x00d954 + base, 0x80000000 | ctrl);
 		nvkm_wr32(device, 0x00d954 + base, 0x00000000 | ctrl);
@@ -131,20 +131,20 @@ gm200_i2c_aux_xfer(struct nvkm_i2c_aux *obj, bool retry,
 				goto out;
 			}
 		} while (ctrl & 0x00010000);
-		ret = 1;
+		ret = 0;
 
 		/* read status, and check if transaction completed ok */
 		stat = nvkm_mask(device, 0x00d958 + base, 0, 0);
 		if ((stat & 0x000f0000) == 0x00080000 ||
 		    (stat & 0x000f0000) == 0x00020000)
-			ret = retry ? 0 : 1;
+			ret = 1;
 		if ((stat & 0x00000100))
 			ret = -ETIMEDOUT;
 		if ((stat & 0x00000e00))
 			ret = -EIO;
 
 		AUX_TRACE(&aux->base, "%02d %08x %08x", retries, ctrl, stat);
-	}
+	} while (ret && retry && retries++ < 32);
 
 	if (type & 1) {
 		for (i = 0; i < 16; i += 4) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gk20a.c
index b7159b338fac..1a4ab825852c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ibus/gk20a.c
@@ -29,7 +29,7 @@ gk20a_ibus_init_ibus_ring(struct nvkm_subdev *ibus)
 	nvkm_mask(device, 0x137250, 0x3f, 0);
 
 	nvkm_mask(device, 0x000200, 0x20, 0);
-	usleep_range(20, 30);
+	udelay(20);
 	nvkm_mask(device, 0x000200, 0x20, 0x20);
 
 	nvkm_wr32(device, 0x12004c, 0x4);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/base.c
index 41bd5d0f7692..f0af2a381eea 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/base.c
@@ -96,60 +96,12 @@ nvkm_iccsense_ina3221_read(struct nvkm_iccsense *iccsense,
 }
 
 static void
-nvkm_iccsense_ina209_config(struct nvkm_iccsense *iccsense,
-			    struct nvkm_iccsense_sensor *sensor)
-{
-	struct nvkm_subdev *subdev = &iccsense->subdev;
-	/* configuration:
-	 * 0x0007: 0x0007 shunt and bus continous
-	 * 0x0078: 0x0078 128 samples shunt
-	 * 0x0780: 0x0780 128 samples bus
-	 * 0x1800: 0x0000 +-40 mV shunt range
-	 * 0x2000: 0x0000 16V FSR
-         */
-	u16 value = 0x07ff;
-	nvkm_debug(subdev, "config for sensor id %i: 0x%x\n", sensor->id, value);
-	nv_wr16i2cr(sensor->i2c, sensor->addr, 0x00, value);
-}
-
-static void
-nvkm_iccsense_ina3221_config(struct nvkm_iccsense *iccsense,
-			     struct nvkm_iccsense_sensor *sensor)
-{
-	struct nvkm_subdev *subdev = &iccsense->subdev;
-	/* configuration:
-	 * 0x0007: 0x0007 shunt and bus continous
-	 * 0x0031: 0x0000 140 us conversion time shunt
-	 * 0x01c0: 0x0000 140 us conversion time bus
-	 * 0x0f00: 0x0f00 1024 samples
-	 * 0x7000: 0x?000 channels
-         */
-	u16 value = 0x0e07;
-	if (sensor->rail_mask & 0x1)
-		value |= 0x1 << 14;
-	if (sensor->rail_mask & 0x2)
-		value |= 0x1 << 13;
-	if (sensor->rail_mask & 0x4)
-		value |= 0x1 << 12;
-	nvkm_debug(subdev, "config for sensor id %i: 0x%x\n", sensor->id, value);
-	nv_wr16i2cr(sensor->i2c, sensor->addr, 0x00, value);
-}
-
-static void
 nvkm_iccsense_sensor_config(struct nvkm_iccsense *iccsense,
 		            struct nvkm_iccsense_sensor *sensor)
 {
-	switch (sensor->type) {
-	case NVBIOS_EXTDEV_INA209:
-	case NVBIOS_EXTDEV_INA219:
-		nvkm_iccsense_ina209_config(iccsense, sensor);
-		break;
-	case NVBIOS_EXTDEV_INA3221:
-		nvkm_iccsense_ina3221_config(iccsense, sensor);
-		break;
-	default:
-		break;
-	}
+	struct nvkm_subdev *subdev = &iccsense->subdev;
+	nvkm_trace(subdev, "write config of extdev %i: 0x%04x\n", sensor->id, sensor->config);
+	nv_wr16i2cr(sensor->i2c, sensor->addr, 0x00, sensor->config);
 }
 
 int
@@ -196,7 +148,6 @@ nvkm_iccsense_dtor(struct nvkm_subdev *subdev)
 static struct nvkm_iccsense_sensor*
 nvkm_iccsense_create_sensor(struct nvkm_iccsense *iccsense, u8 id)
 {
-
 	struct nvkm_subdev *subdev = &iccsense->subdev;
 	struct nvkm_bios *bios = subdev->device->bios;
 	struct nvkm_i2c *i2c = subdev->device->i2c;
@@ -245,7 +196,7 @@ nvkm_iccsense_create_sensor(struct nvkm_iccsense *iccsense, u8 id)
 	sensor->type = extdev.type;
 	sensor->i2c = &i2c_bus->i2c;
 	sensor->addr = addr;
-	sensor->rail_mask = 0x0;
+	sensor->config = 0x0;
 	return sensor;
 }
 
@@ -273,48 +224,56 @@ nvkm_iccsense_oneinit(struct nvkm_subdev *subdev)
 
 	iccsense->data_valid = true;
 	for (i = 0; i < stbl.nr_entry; ++i) {
-		struct pwr_rail_t *r = &stbl.rail[i];
-		struct nvkm_iccsense_rail *rail;
+		struct pwr_rail_t *pwr_rail = &stbl.rail[i];
 		struct nvkm_iccsense_sensor *sensor;
-		int (*read)(struct nvkm_iccsense *,
-			    struct nvkm_iccsense_rail *);
+		int r;
 
-		if (!r->mode || r->resistor_mohm == 0)
+		if (pwr_rail->mode != 1 || !pwr_rail->resistor_count)
 			continue;
 
-		sensor = nvkm_iccsense_get_sensor(iccsense, r->extdev_id);
+		sensor = nvkm_iccsense_get_sensor(iccsense, pwr_rail->extdev_id);
 		if (!sensor)
 			continue;
 
-		switch (sensor->type) {
-		case NVBIOS_EXTDEV_INA209:
-			if (r->rail != 0)
-				continue;
-			read = nvkm_iccsense_ina209_read;
-			break;
-		case NVBIOS_EXTDEV_INA219:
-			if (r->rail != 0)
+		if (!sensor->config)
+			sensor->config = pwr_rail->config;
+		else if (sensor->config != pwr_rail->config)
+			nvkm_error(subdev, "config mismatch found for extdev %i\n", pwr_rail->extdev_id);
+
+		for (r = 0; r < pwr_rail->resistor_count; ++r) {
+			struct nvkm_iccsense_rail *rail;
+			struct pwr_rail_resistor_t *res = &pwr_rail->resistors[r];
+			int (*read)(struct nvkm_iccsense *,
+				    struct nvkm_iccsense_rail *);
+
+			if (!res->mohm || !res->enabled)
 				continue;
-			read = nvkm_iccsense_ina219_read;
-			break;
-		case NVBIOS_EXTDEV_INA3221:
-			if (r->rail >= 3)
+
+			switch (sensor->type) {
+			case NVBIOS_EXTDEV_INA209:
+				read = nvkm_iccsense_ina209_read;
+				break;
+			case NVBIOS_EXTDEV_INA219:
+				read = nvkm_iccsense_ina219_read;
+				break;
+			case NVBIOS_EXTDEV_INA3221:
+				read = nvkm_iccsense_ina3221_read;
+				break;
+			default:
 				continue;
-			read = nvkm_iccsense_ina3221_read;
-			break;
-		default:
-			continue;
+			}
+
+			rail = kmalloc(sizeof(*rail), GFP_KERNEL);
+			if (!rail)
+				return -ENOMEM;
+
+			rail->read = read;
+			rail->sensor = sensor;
+			rail->idx = r;
+			rail->mohm = res->mohm;
+			nvkm_debug(subdev, "create rail for extdev %i: { idx: %i, mohm: %i }\n", pwr_rail->extdev_id, r, rail->mohm);
+			list_add_tail(&rail->head, &iccsense->rails);
 		}
-
-		rail = kmalloc(sizeof(*rail), GFP_KERNEL);
-		if (!rail)
-			return -ENOMEM;
-		sensor->rail_mask |= 1 << r->rail;
-		rail->read = read;
-		rail->sensor = sensor;
-		rail->idx = r->rail;
-		rail->mohm = r->resistor_mohm;
-		list_add_tail(&rail->head, &iccsense->rails);
 	}
 	return 0;
 }
@@ -329,7 +288,8 @@ nvkm_iccsense_init(struct nvkm_subdev *subdev)
 	return 0;
 }
 
-struct nvkm_subdev_func iccsense_func = {
+static const struct nvkm_subdev_func
+iccsense_func = {
 	.oneinit = nvkm_iccsense_oneinit,
 	.init = nvkm_iccsense_init,
 	.dtor = nvkm_iccsense_dtor,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/priv.h
index b72c31d2f908..e90e0f6ed008 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/priv.h
@@ -10,7 +10,7 @@ struct nvkm_iccsense_sensor {
 	enum nvbios_extdev_type type;
 	struct i2c_adapter *i2c;
 	u8 addr;
-	u8 rail_mask;
+	u16 config;
 };
 
 struct nvkm_iccsense_rail {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c
index 8ed8f65ff664..10c987a654ec 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c
@@ -104,7 +104,7 @@ nvkm_instobj_dtor(struct nvkm_memory *memory)
 	return iobj;
 }
 
-const struct nvkm_memory_func
+static const struct nvkm_memory_func
 nvkm_instobj_func = {
 	.dtor = nvkm_instobj_dtor,
 	.target = nvkm_instobj_target,
@@ -156,7 +156,7 @@ nvkm_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data)
 	return nvkm_wo32(iobj->parent, offset, data);
 }
 
-const struct nvkm_memory_func
+static const struct nvkm_memory_func
 nvkm_instobj_func_slow = {
 	.dtor = nvkm_instobj_dtor,
 	.target = nvkm_instobj_target,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c
index 39c2a38e54f7..0c7ef250dcaf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c
@@ -47,8 +47,10 @@ nvkm_ltc_tags_clear(struct nvkm_ltc *ltc, u32 first, u32 count)
 
 	BUG_ON((first > limit) || (limit >= ltc->num_tags));
 
+	mutex_lock(&ltc->subdev.mutex);
 	ltc->func->cbc_clear(ltc, first, limit);
 	ltc->func->cbc_wait(ltc);
+	mutex_unlock(&ltc->subdev.mutex);
 }
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c
index c3d66ef5dc12..430a61c3df44 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c
@@ -34,7 +34,7 @@ g84_mc_reset[] = {
 	{}
 };
 
-const struct nvkm_mc_map
+static const struct nvkm_mc_map
 g84_mc_intr[] = {
 	{ 0x04000000, NVKM_ENGINE_DISP },
 	{ 0x00020000, NVKM_ENGINE_VP },
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/base.c
index 21b65ee254e4..e3e2f5e83815 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/base.c
@@ -250,6 +250,10 @@ nvkm_mxm_new_(struct nvkm_device *device, int index, struct nvkm_mxm **pmxm)
 	}
 
 	nvkm_info(&mxm->subdev, "BIOS version %d.%d\n", ver >> 4, ver & 0x0f);
+	nvkm_debug(&mxm->subdev, "module flags: %02x\n",
+		   nvbios_rd08(bios, data + 0x01));
+	nvkm_debug(&mxm->subdev, "config flags: %02x\n",
+		   nvbios_rd08(bios, data + 0x02));
 
 	if (mxm_shadow(mxm, ver)) {
 		nvkm_warn(&mxm->subdev, "failed to locate valid SIS\n");
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/mxms.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/mxms.c
index 45a2f8e784f9..9abfa5e2fe9f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/mxms.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/mxms.c
@@ -23,8 +23,8 @@
  */
 #include "mxms.h"
 
-#define ROM16(x) le16_to_cpu(*(u16 *)&(x))
-#define ROM32(x) le32_to_cpu(*(u32 *)&(x))
+#define ROM16(x) get_unaligned_le16(&(x))
+#define ROM32(x) get_unaligned_le32(&(x))
 
 static u8 *
 mxms_data(struct nvkm_mxm *mxm)
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/nv50.c
index db14fad2ddfc..844971e5e874 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/nv50.c
@@ -190,8 +190,8 @@ mxm_dcb_sanitise(struct nvkm_mxm *mxm)
 	struct nvkm_bios *bios = subdev->device->bios;
 	u8  ver, hdr, cnt, len;
 	u16 dcb = dcb_table(bios, &ver, &hdr, &cnt, &len);
-	if (dcb == 0x0000 || ver != 0x40) {
-		nvkm_debug(subdev, "unsupported DCB version\n");
+	if (dcb == 0x0000 || (ver != 0x40 && ver != 0x41)) {
+		nvkm_warn(subdev, "unsupported DCB version\n");
 		return;
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/Kbuild
index 88b643b8664e..51fb4bf94a44 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/Kbuild
@@ -8,3 +8,5 @@ nvkm-y += nvkm/subdev/pmu/gk110.o
 nvkm-y += nvkm/subdev/pmu/gk208.o
 nvkm-y += nvkm/subdev/pmu/gk20a.o
 nvkm-y += nvkm/subdev/pmu/gm107.o
+nvkm-y += nvkm/subdev/pmu/gp100.o
+nvkm-y += nvkm/subdev/pmu/gp102.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c
index 8dd164d13043..e611ce80f8ef 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c
@@ -32,225 +32,85 @@ nvkm_pmu_pgob(struct nvkm_pmu *pmu, bool enable)
 		pmu->func->pgob(pmu, enable);
 }
 
-int
-nvkm_pmu_send(struct nvkm_pmu *pmu, u32 reply[2],
-	      u32 process, u32 message, u32 data0, u32 data1)
-{
-	struct nvkm_subdev *subdev = &pmu->subdev;
-	struct nvkm_device *device = subdev->device;
-	u32 addr;
-
-	mutex_lock(&subdev->mutex);
-	/* wait for a free slot in the fifo */
-	addr  = nvkm_rd32(device, 0x10a4a0);
-	if (nvkm_msec(device, 2000,
-		u32 tmp = nvkm_rd32(device, 0x10a4b0);
-		if (tmp != (addr ^ 8))
-			break;
-	) < 0) {
-		mutex_unlock(&subdev->mutex);
-		return -EBUSY;
-	}
-
-	/* we currently only support a single process at a time waiting
-	 * on a synchronous reply, take the PMU mutex and tell the
-	 * receive handler what we're waiting for
-	 */
-	if (reply) {
-		pmu->recv.message = message;
-		pmu->recv.process = process;
-	}
-
-	/* acquire data segment access */
-	do {
-		nvkm_wr32(device, 0x10a580, 0x00000001);
-	} while (nvkm_rd32(device, 0x10a580) != 0x00000001);
-
-	/* write the packet */
-	nvkm_wr32(device, 0x10a1c0, 0x01000000 | (((addr & 0x07) << 4) +
-				pmu->send.base));
-	nvkm_wr32(device, 0x10a1c4, process);
-	nvkm_wr32(device, 0x10a1c4, message);
-	nvkm_wr32(device, 0x10a1c4, data0);
-	nvkm_wr32(device, 0x10a1c4, data1);
-	nvkm_wr32(device, 0x10a4a0, (addr + 1) & 0x0f);
-
-	/* release data segment access */
-	nvkm_wr32(device, 0x10a580, 0x00000000);
-
-	/* wait for reply, if requested */
-	if (reply) {
-		wait_event(pmu->recv.wait, (pmu->recv.process == 0));
-		reply[0] = pmu->recv.data[0];
-		reply[1] = pmu->recv.data[1];
-	}
-
-	mutex_unlock(&subdev->mutex);
-	return 0;
-}
-
 static void
 nvkm_pmu_recv(struct work_struct *work)
 {
-	struct nvkm_pmu *pmu = container_of(work, struct nvkm_pmu, recv.work);
-	struct nvkm_subdev *subdev = &pmu->subdev;
-	struct nvkm_device *device = subdev->device;
-	u32 process, message, data0, data1;
-
-	/* nothing to do if GET == PUT */
-	u32 addr =  nvkm_rd32(device, 0x10a4cc);
-	if (addr == nvkm_rd32(device, 0x10a4c8))
-		return;
-
-	/* acquire data segment access */
-	do {
-		nvkm_wr32(device, 0x10a580, 0x00000002);
-	} while (nvkm_rd32(device, 0x10a580) != 0x00000002);
-
-	/* read the packet */
-	nvkm_wr32(device, 0x10a1c0, 0x02000000 | (((addr & 0x07) << 4) +
-				pmu->recv.base));
-	process = nvkm_rd32(device, 0x10a1c4);
-	message = nvkm_rd32(device, 0x10a1c4);
-	data0   = nvkm_rd32(device, 0x10a1c4);
-	data1   = nvkm_rd32(device, 0x10a1c4);
-	nvkm_wr32(device, 0x10a4cc, (addr + 1) & 0x0f);
-
-	/* release data segment access */
-	nvkm_wr32(device, 0x10a580, 0x00000000);
-
-	/* wake process if it's waiting on a synchronous reply */
-	if (pmu->recv.process) {
-		if (process == pmu->recv.process &&
-		    message == pmu->recv.message) {
-			pmu->recv.data[0] = data0;
-			pmu->recv.data[1] = data1;
-			pmu->recv.process = 0;
-			wake_up(&pmu->recv.wait);
-			return;
-		}
-	}
+	struct nvkm_pmu *pmu = container_of(work, typeof(*pmu), recv.work);
+	return pmu->func->recv(pmu);
+}
 
-	/* right now there's no other expected responses from the engine,
-	 * so assume that any unexpected message is an error.
-	 */
-	nvkm_warn(subdev, "%c%c%c%c %08x %08x %08x %08x\n",
-		  (char)((process & 0x000000ff) >>  0),
-		  (char)((process & 0x0000ff00) >>  8),
-		  (char)((process & 0x00ff0000) >> 16),
-		  (char)((process & 0xff000000) >> 24),
-		  process, message, data0, data1);
+int
+nvkm_pmu_send(struct nvkm_pmu *pmu, u32 reply[2],
+	      u32 process, u32 message, u32 data0, u32 data1)
+{
+	if (!pmu || !pmu->func->send)
+		return -ENODEV;
+	return pmu->func->send(pmu, reply, process, message, data0, data1);
 }
 
 static void
 nvkm_pmu_intr(struct nvkm_subdev *subdev)
 {
 	struct nvkm_pmu *pmu = nvkm_pmu(subdev);
-	struct nvkm_device *device = pmu->subdev.device;
-	u32 disp = nvkm_rd32(device, 0x10a01c);
-	u32 intr = nvkm_rd32(device, 0x10a008) & disp & ~(disp >> 16);
-
-	if (intr & 0x00000020) {
-		u32 stat = nvkm_rd32(device, 0x10a16c);
-		if (stat & 0x80000000) {
-			nvkm_error(subdev, "UAS fault at %06x addr %08x\n",
-				   stat & 0x00ffffff,
-				   nvkm_rd32(device, 0x10a168));
-			nvkm_wr32(device, 0x10a16c, 0x00000000);
-			intr &= ~0x00000020;
-		}
-	}
-
-	if (intr & 0x00000040) {
-		schedule_work(&pmu->recv.work);
-		nvkm_wr32(device, 0x10a004, 0x00000040);
-		intr &= ~0x00000040;
-	}
-
-	if (intr & 0x00000080) {
-		nvkm_info(subdev, "wr32 %06x %08x\n",
-			  nvkm_rd32(device, 0x10a7a0),
-			  nvkm_rd32(device, 0x10a7a4));
-		nvkm_wr32(device, 0x10a004, 0x00000080);
-		intr &= ~0x00000080;
-	}
-
-	if (intr) {
-		nvkm_error(subdev, "intr %08x\n", intr);
-		nvkm_wr32(device, 0x10a004, intr);
-	}
+	if (!pmu->func->intr)
+		return;
+	pmu->func->intr(pmu);
 }
 
 static int
 nvkm_pmu_fini(struct nvkm_subdev *subdev, bool suspend)
 {
 	struct nvkm_pmu *pmu = nvkm_pmu(subdev);
-	struct nvkm_device *device = pmu->subdev.device;
 
-	nvkm_wr32(device, 0x10a014, 0x00000060);
+	if (pmu->func->fini)
+		pmu->func->fini(pmu);
+
 	flush_work(&pmu->recv.work);
 	return 0;
 }
 
 static int
-nvkm_pmu_init(struct nvkm_subdev *subdev)
+nvkm_pmu_reset(struct nvkm_pmu *pmu)
 {
-	struct nvkm_pmu *pmu = nvkm_pmu(subdev);
 	struct nvkm_device *device = pmu->subdev.device;
-	int i;
 
-	/* prevent previous ucode from running, wait for idle, reset */
-	nvkm_wr32(device, 0x10a014, 0x0000ffff); /* INTR_EN_CLR = ALL */
+	if (!(nvkm_rd32(device, 0x000200) & 0x00002000))
+		return 0;
+
+	/* Inhibit interrupts, and wait for idle. */
+	nvkm_wr32(device, 0x10a014, 0x0000ffff);
 	nvkm_msec(device, 2000,
 		if (!nvkm_rd32(device, 0x10a04c))
 			break;
 	);
-	nvkm_mask(device, 0x000200, 0x00002000, 0x00000000);
-	nvkm_mask(device, 0x000200, 0x00002000, 0x00002000);
-	nvkm_rd32(device, 0x000200);
+
+	/* Reset. */
+	pmu->func->reset(pmu);
+
+	/* Wait for IMEM/DMEM scrubbing to be complete. */
 	nvkm_msec(device, 2000,
 		if (!(nvkm_rd32(device, 0x10a10c) & 0x00000006))
 			break;
 	);
 
-	/* upload data segment */
-	nvkm_wr32(device, 0x10a1c0, 0x01000000);
-	for (i = 0; i < pmu->func->data.size / 4; i++)
-		nvkm_wr32(device, 0x10a1c4, pmu->func->data.data[i]);
-
-	/* upload code segment */
-	nvkm_wr32(device, 0x10a180, 0x01000000);
-	for (i = 0; i < pmu->func->code.size / 4; i++) {
-		if ((i & 0x3f) == 0)
-			nvkm_wr32(device, 0x10a188, i >> 6);
-		nvkm_wr32(device, 0x10a184, pmu->func->code.data[i]);
-	}
-
-	/* start it running */
-	nvkm_wr32(device, 0x10a10c, 0x00000000);
-	nvkm_wr32(device, 0x10a104, 0x00000000);
-	nvkm_wr32(device, 0x10a100, 0x00000002);
-
-	/* wait for valid host->pmu ring configuration */
-	if (nvkm_msec(device, 2000,
-		if (nvkm_rd32(device, 0x10a4d0))
-			break;
-	) < 0)
-		return -EBUSY;
-	pmu->send.base = nvkm_rd32(device, 0x10a4d0) & 0x0000ffff;
-	pmu->send.size = nvkm_rd32(device, 0x10a4d0) >> 16;
+	return 0;
+}
 
-	/* wait for valid pmu->host ring configuration */
-	if (nvkm_msec(device, 2000,
-		if (nvkm_rd32(device, 0x10a4dc))
-			break;
-	) < 0)
-		return -EBUSY;
-	pmu->recv.base = nvkm_rd32(device, 0x10a4dc) & 0x0000ffff;
-	pmu->recv.size = nvkm_rd32(device, 0x10a4dc) >> 16;
+static int
+nvkm_pmu_preinit(struct nvkm_subdev *subdev)
+{
+	struct nvkm_pmu *pmu = nvkm_pmu(subdev);
+	return nvkm_pmu_reset(pmu);
+}
 
-	nvkm_wr32(device, 0x10a010, 0x000000e0);
-	return 0;
+static int
+nvkm_pmu_init(struct nvkm_subdev *subdev)
+{
+	struct nvkm_pmu *pmu = nvkm_pmu(subdev);
+	int ret = nvkm_pmu_reset(pmu);
+	if (ret == 0 && pmu->func->init)
+		ret = pmu->func->init(pmu);
+	return ret;
 }
 
 static void *
@@ -262,6 +122,7 @@ nvkm_pmu_dtor(struct nvkm_subdev *subdev)
 static const struct nvkm_subdev_func
 nvkm_pmu = {
 	.dtor = nvkm_pmu_dtor,
+	.preinit = nvkm_pmu_preinit,
 	.init = nvkm_pmu_init,
 	.fini = nvkm_pmu_fini,
 	.intr = nvkm_pmu_intr,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h
index e2faccffee6f..0bcf0b307a61 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h
@@ -1,4 +1,4 @@
-uint32_t gf100_pmu_data[] = {
+static uint32_t gf100_pmu_data[] = {
 /* 0x0000: proc_kern */
 	0x52544e49,
 	0x00000000,
@@ -916,7 +916,7 @@ uint32_t gf100_pmu_data[] = {
 	0x00000000,
 };
 
-uint32_t gf100_pmu_code[] = {
+static uint32_t gf100_pmu_code[] = {
 	0x03920ef5,
 /* 0x0004: rd32 */
 	0x07a007f1,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h
index 2d5bdc539697..fe8905666c67 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h
@@ -1,4 +1,4 @@
-uint32_t gf119_pmu_data[] = {
+static uint32_t gf119_pmu_data[] = {
 /* 0x0000: proc_kern */
 	0x52544e49,
 	0x00000000,
@@ -915,7 +915,7 @@ uint32_t gf119_pmu_data[] = {
 	0x00000000,
 };
 
-uint32_t gf119_pmu_code[] = {
+static uint32_t gf119_pmu_code[] = {
 	0x03410ef5,
 /* 0x0004: rd32 */
 	0x07a007f1,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h
index 3c731ff12871..9cf4e6fc724e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h
@@ -1,4 +1,4 @@
-uint32_t gk208_pmu_data[] = {
+static uint32_t gk208_pmu_data[] = {
 /* 0x0000: proc_kern */
 	0x52544e49,
 	0x00000000,
@@ -915,7 +915,7 @@ uint32_t gk208_pmu_data[] = {
 	0x00000000,
 };
 
-uint32_t gk208_pmu_code[] = {
+static uint32_t gk208_pmu_code[] = {
 	0x02f90ef5,
 /* 0x0004: rd32 */
 	0xf607a040,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h
index e83341815ec6..5d692425b190 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h
@@ -1,4 +1,4 @@
-uint32_t gt215_pmu_data[] = {
+static uint32_t gt215_pmu_data[] = {
 /* 0x0000: proc_kern */
 	0x52544e49,
 	0x00000000,
@@ -916,7 +916,7 @@ uint32_t gt215_pmu_data[] = {
 	0x00000000,
 };
 
-uint32_t gt215_pmu_code[] = {
+static uint32_t gt215_pmu_code[] = {
 	0x03920ef5,
 /* 0x0004: rd32 */
 	0x07a007f1,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf100.c
index aeb8ccd891fc..0e36d4cb7201 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf100.c
@@ -30,6 +30,12 @@ gf100_pmu = {
 	.code.size = sizeof(gf100_pmu_code),
 	.data.data = gf100_pmu_data,
 	.data.size = sizeof(gf100_pmu_data),
+	.reset = gt215_pmu_reset,
+	.init = gt215_pmu_init,
+	.fini = gt215_pmu_fini,
+	.intr = gt215_pmu_intr,
+	.send = gt215_pmu_send,
+	.recv = gt215_pmu_recv,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf119.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf119.c
index fbc88d8ecd4d..0e4ba4248b15 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf119.c
@@ -30,6 +30,12 @@ gf119_pmu = {
 	.code.size = sizeof(gf119_pmu_code),
 	.data.data = gf119_pmu_data,
 	.data.size = sizeof(gf119_pmu_data),
+	.reset = gt215_pmu_reset,
+	.init = gt215_pmu_init,
+	.fini = gt215_pmu_fini,
+	.intr = gt215_pmu_intr,
+	.send = gt215_pmu_send,
+	.recv = gt215_pmu_recv,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk104.c
index 86f9f3b13f71..2ad858d825ac 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk104.c
@@ -109,6 +109,12 @@ gk104_pmu = {
 	.code.size = sizeof(gk104_pmu_code),
 	.data.data = gk104_pmu_data,
 	.data.size = sizeof(gk104_pmu_data),
+	.reset = gt215_pmu_reset,
+	.init = gt215_pmu_init,
+	.fini = gt215_pmu_fini,
+	.intr = gt215_pmu_intr,
+	.send = gt215_pmu_send,
+	.recv = gt215_pmu_recv,
 	.pgob = gk104_pmu_pgob,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk110.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk110.c
index ae255247c9d1..fc4b8ecfdaeb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk110.c
@@ -88,6 +88,12 @@ gk110_pmu = {
 	.code.size = sizeof(gk110_pmu_code),
 	.data.data = gk110_pmu_data,
 	.data.size = sizeof(gk110_pmu_data),
+	.reset = gt215_pmu_reset,
+	.init = gt215_pmu_init,
+	.fini = gt215_pmu_fini,
+	.intr = gt215_pmu_intr,
+	.send = gt215_pmu_send,
+	.recv = gt215_pmu_recv,
 	.pgob = gk110_pmu_pgob,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk208.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk208.c
index 3b4917637902..e9a91277683a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk208.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk208.c
@@ -30,6 +30,12 @@ gk208_pmu = {
 	.code.size = sizeof(gk208_pmu_code),
 	.data.data = gk208_pmu_data,
 	.data.size = sizeof(gk208_pmu_data),
+	.reset = gt215_pmu_reset,
+	.init = gt215_pmu_init,
+	.fini = gt215_pmu_fini,
+	.intr = gt215_pmu_intr,
+	.send = gt215_pmu_send,
+	.recv = gt215_pmu_recv,
 	.pgob = gk110_pmu_pgob,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm107.c
index 31b8692b4641..9a248ed75f09 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm107.c
@@ -32,6 +32,12 @@ gm107_pmu = {
 	.code.size = sizeof(gm107_pmu_code),
 	.data.data = gm107_pmu_data,
 	.data.size = sizeof(gm107_pmu_data),
+	.reset = gt215_pmu_reset,
+	.init = gt215_pmu_init,
+	.fini = gt215_pmu_fini,
+	.intr = gt215_pmu_intr,
+	.send = gt215_pmu_send,
+	.recv = gt215_pmu_recv,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp100.c
new file mode 100644
index 000000000000..6c41c20c85a7
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp100.c
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "priv.h"
+
+static const struct nvkm_pmu_func
+gp100_pmu = {
+	.reset = gt215_pmu_reset,
+};
+
+int
+gp100_pmu_new(struct nvkm_device *device, int index, struct nvkm_pmu **ppmu)
+{
+	return nvkm_pmu_new_(&gp100_pmu, device, index, ppmu);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c
new file mode 100644
index 000000000000..f017352206c9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "priv.h"
+
+static void
+gp102_pmu_reset(struct nvkm_pmu *pmu)
+{
+	struct nvkm_device *device = pmu->subdev.device;
+	nvkm_mask(device, 0x10a3c0, 0x00000001, 0x00000001);
+	nvkm_mask(device, 0x10a3c0, 0x00000001, 0x00000000);
+}
+
+static const struct nvkm_pmu_func
+gp102_pmu = {
+	.reset = gp102_pmu_reset,
+};
+
+int
+gp102_pmu_new(struct nvkm_device *device, int index, struct nvkm_pmu **ppmu)
+{
+	return nvkm_pmu_new_(&gp102_pmu, device, index, ppmu);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c
index 8ba7fa4ca75b..90d428b3be97 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c
@@ -24,21 +24,229 @@
 #include "priv.h"
 #include "fuc/gt215.fuc3.h"
 
-static void
+#include <subdev/timer.h>
+
+int
+gt215_pmu_send(struct nvkm_pmu *pmu, u32 reply[2],
+	       u32 process, u32 message, u32 data0, u32 data1)
+{
+	struct nvkm_subdev *subdev = &pmu->subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 addr;
+
+	mutex_lock(&subdev->mutex);
+	/* wait for a free slot in the fifo */
+	addr  = nvkm_rd32(device, 0x10a4a0);
+	if (nvkm_msec(device, 2000,
+		u32 tmp = nvkm_rd32(device, 0x10a4b0);
+		if (tmp != (addr ^ 8))
+			break;
+	) < 0) {
+		mutex_unlock(&subdev->mutex);
+		return -EBUSY;
+	}
+
+	/* we currently only support a single process at a time waiting
+	 * on a synchronous reply, take the PMU mutex and tell the
+	 * receive handler what we're waiting for
+	 */
+	if (reply) {
+		pmu->recv.message = message;
+		pmu->recv.process = process;
+	}
+
+	/* acquire data segment access */
+	do {
+		nvkm_wr32(device, 0x10a580, 0x00000001);
+	} while (nvkm_rd32(device, 0x10a580) != 0x00000001);
+
+	/* write the packet */
+	nvkm_wr32(device, 0x10a1c0, 0x01000000 | (((addr & 0x07) << 4) +
+				pmu->send.base));
+	nvkm_wr32(device, 0x10a1c4, process);
+	nvkm_wr32(device, 0x10a1c4, message);
+	nvkm_wr32(device, 0x10a1c4, data0);
+	nvkm_wr32(device, 0x10a1c4, data1);
+	nvkm_wr32(device, 0x10a4a0, (addr + 1) & 0x0f);
+
+	/* release data segment access */
+	nvkm_wr32(device, 0x10a580, 0x00000000);
+
+	/* wait for reply, if requested */
+	if (reply) {
+		wait_event(pmu->recv.wait, (pmu->recv.process == 0));
+		reply[0] = pmu->recv.data[0];
+		reply[1] = pmu->recv.data[1];
+	}
+
+	mutex_unlock(&subdev->mutex);
+	return 0;
+}
+
+void
+gt215_pmu_recv(struct nvkm_pmu *pmu)
+{
+	struct nvkm_subdev *subdev = &pmu->subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 process, message, data0, data1;
+
+	/* nothing to do if GET == PUT */
+	u32 addr =  nvkm_rd32(device, 0x10a4cc);
+	if (addr == nvkm_rd32(device, 0x10a4c8))
+		return;
+
+	/* acquire data segment access */
+	do {
+		nvkm_wr32(device, 0x10a580, 0x00000002);
+	} while (nvkm_rd32(device, 0x10a580) != 0x00000002);
+
+	/* read the packet */
+	nvkm_wr32(device, 0x10a1c0, 0x02000000 | (((addr & 0x07) << 4) +
+				pmu->recv.base));
+	process = nvkm_rd32(device, 0x10a1c4);
+	message = nvkm_rd32(device, 0x10a1c4);
+	data0   = nvkm_rd32(device, 0x10a1c4);
+	data1   = nvkm_rd32(device, 0x10a1c4);
+	nvkm_wr32(device, 0x10a4cc, (addr + 1) & 0x0f);
+
+	/* release data segment access */
+	nvkm_wr32(device, 0x10a580, 0x00000000);
+
+	/* wake process if it's waiting on a synchronous reply */
+	if (pmu->recv.process) {
+		if (process == pmu->recv.process &&
+		    message == pmu->recv.message) {
+			pmu->recv.data[0] = data0;
+			pmu->recv.data[1] = data1;
+			pmu->recv.process = 0;
+			wake_up(&pmu->recv.wait);
+			return;
+		}
+	}
+
+	/* right now there's no other expected responses from the engine,
+	 * so assume that any unexpected message is an error.
+	 */
+	nvkm_warn(subdev, "%c%c%c%c %08x %08x %08x %08x\n",
+		  (char)((process & 0x000000ff) >>  0),
+		  (char)((process & 0x0000ff00) >>  8),
+		  (char)((process & 0x00ff0000) >> 16),
+		  (char)((process & 0xff000000) >> 24),
+		  process, message, data0, data1);
+}
+
+void
+gt215_pmu_intr(struct nvkm_pmu *pmu)
+{
+	struct nvkm_subdev *subdev = &pmu->subdev;
+	struct nvkm_device *device = subdev->device;
+	u32 disp = nvkm_rd32(device, 0x10a01c);
+	u32 intr = nvkm_rd32(device, 0x10a008) & disp & ~(disp >> 16);
+
+	if (intr & 0x00000020) {
+		u32 stat = nvkm_rd32(device, 0x10a16c);
+		if (stat & 0x80000000) {
+			nvkm_error(subdev, "UAS fault at %06x addr %08x\n",
+				   stat & 0x00ffffff,
+				   nvkm_rd32(device, 0x10a168));
+			nvkm_wr32(device, 0x10a16c, 0x00000000);
+			intr &= ~0x00000020;
+		}
+	}
+
+	if (intr & 0x00000040) {
+		schedule_work(&pmu->recv.work);
+		nvkm_wr32(device, 0x10a004, 0x00000040);
+		intr &= ~0x00000040;
+	}
+
+	if (intr & 0x00000080) {
+		nvkm_info(subdev, "wr32 %06x %08x\n",
+			  nvkm_rd32(device, 0x10a7a0),
+			  nvkm_rd32(device, 0x10a7a4));
+		nvkm_wr32(device, 0x10a004, 0x00000080);
+		intr &= ~0x00000080;
+	}
+
+	if (intr) {
+		nvkm_error(subdev, "intr %08x\n", intr);
+		nvkm_wr32(device, 0x10a004, intr);
+	}
+}
+
+void
+gt215_pmu_fini(struct nvkm_pmu *pmu)
+{
+	nvkm_wr32(pmu->subdev.device, 0x10a014, 0x00000060);
+}
+
+void
 gt215_pmu_reset(struct nvkm_pmu *pmu)
 {
 	struct nvkm_device *device = pmu->subdev.device;
-	nvkm_mask(device, 0x022210, 0x00000001, 0x00000000);
-	nvkm_mask(device, 0x022210, 0x00000001, 0x00000001);
+	nvkm_mask(device, 0x000200, 0x00002000, 0x00000000);
+	nvkm_mask(device, 0x000200, 0x00002000, 0x00002000);
+	nvkm_rd32(device, 0x000200);
+}
+
+int
+gt215_pmu_init(struct nvkm_pmu *pmu)
+{
+	struct nvkm_device *device = pmu->subdev.device;
+	int i;
+
+	/* upload data segment */
+	nvkm_wr32(device, 0x10a1c0, 0x01000000);
+	for (i = 0; i < pmu->func->data.size / 4; i++)
+		nvkm_wr32(device, 0x10a1c4, pmu->func->data.data[i]);
+
+	/* upload code segment */
+	nvkm_wr32(device, 0x10a180, 0x01000000);
+	for (i = 0; i < pmu->func->code.size / 4; i++) {
+		if ((i & 0x3f) == 0)
+			nvkm_wr32(device, 0x10a188, i >> 6);
+		nvkm_wr32(device, 0x10a184, pmu->func->code.data[i]);
+	}
+
+	/* start it running */
+	nvkm_wr32(device, 0x10a10c, 0x00000000);
+	nvkm_wr32(device, 0x10a104, 0x00000000);
+	nvkm_wr32(device, 0x10a100, 0x00000002);
+
+	/* wait for valid host->pmu ring configuration */
+	if (nvkm_msec(device, 2000,
+		if (nvkm_rd32(device, 0x10a4d0))
+			break;
+	) < 0)
+		return -EBUSY;
+	pmu->send.base = nvkm_rd32(device, 0x10a4d0) & 0x0000ffff;
+	pmu->send.size = nvkm_rd32(device, 0x10a4d0) >> 16;
+
+	/* wait for valid pmu->host ring configuration */
+	if (nvkm_msec(device, 2000,
+		if (nvkm_rd32(device, 0x10a4dc))
+			break;
+	) < 0)
+		return -EBUSY;
+	pmu->recv.base = nvkm_rd32(device, 0x10a4dc) & 0x0000ffff;
+	pmu->recv.size = nvkm_rd32(device, 0x10a4dc) >> 16;
+
+	nvkm_wr32(device, 0x10a010, 0x000000e0);
+	return 0;
 }
 
 static const struct nvkm_pmu_func
 gt215_pmu = {
-	.reset = gt215_pmu_reset,
 	.code.data = gt215_pmu_code,
 	.code.size = sizeof(gt215_pmu_code),
 	.data.data = gt215_pmu_data,
 	.data.size = sizeof(gt215_pmu_data),
+	.reset = gt215_pmu_reset,
+	.init = gt215_pmu_init,
+	.fini = gt215_pmu_fini,
+	.intr = gt215_pmu_intr,
+	.send = gt215_pmu_send,
+	.recv = gt215_pmu_recv,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h
index f38c88fae3d6..2e2179a4ad17 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h
@@ -8,8 +8,6 @@ int nvkm_pmu_new_(const struct nvkm_pmu_func *, struct nvkm_device *,
 		  int index, struct nvkm_pmu **);
 
 struct nvkm_pmu_func {
-	void (*reset)(struct nvkm_pmu *);
-
 	struct {
 		u32 *data;
 		u32  size;
@@ -20,8 +18,22 @@ struct nvkm_pmu_func {
 		u32  size;
 	} data;
 
+	void (*reset)(struct nvkm_pmu *);
+	int (*init)(struct nvkm_pmu *);
+	void (*fini)(struct nvkm_pmu *);
+	void (*intr)(struct nvkm_pmu *);
+	int (*send)(struct nvkm_pmu *, u32 reply[2], u32 process,
+		    u32 message, u32 data0, u32 data1);
+	void (*recv)(struct nvkm_pmu *);
 	void (*pgob)(struct nvkm_pmu *, bool);
 };
 
+void gt215_pmu_reset(struct nvkm_pmu *);
+int gt215_pmu_init(struct nvkm_pmu *);
+void gt215_pmu_fini(struct nvkm_pmu *);
+void gt215_pmu_intr(struct nvkm_pmu *);
+void gt215_pmu_recv(struct nvkm_pmu *);
+int gt215_pmu_send(struct nvkm_pmu *, u32[2], u32, u32, u32, u32);
+
 void gk110_pmu_pgob(struct nvkm_pmu *, bool);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c
index f1e2dc914366..ec48e4ace37a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c
@@ -1364,7 +1364,7 @@ gm200_secboot_init(struct nvkm_secboot *sb)
 	return 0;
 }
 
-int
+static int
 gm200_secboot_fini(struct nvkm_secboot *sb, bool suspend)
 {
 	struct gm200_secboot *gsb = gm200_secboot(sb);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild
index c34076223b7b..bcd179ba11d0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/Kbuild
@@ -1,6 +1,7 @@
 nvkm-y += nvkm/subdev/volt/base.o
 nvkm-y += nvkm/subdev/volt/gpio.o
 nvkm-y += nvkm/subdev/volt/nv40.o
+nvkm-y += nvkm/subdev/volt/gf100.o
 nvkm-y += nvkm/subdev/volt/gk104.o
 nvkm-y += nvkm/subdev/volt/gk20a.o
 nvkm-y += nvkm/subdev/volt/gm20b.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/base.c
index 1c3d23b0e84a..e344901cfdc7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/base.c
@@ -26,6 +26,7 @@
 #include <subdev/bios.h>
 #include <subdev/bios/vmap.h>
 #include <subdev/bios/volt.h>
+#include <subdev/therm.h>
 
 int
 nvkm_volt_get(struct nvkm_volt *volt)
@@ -50,33 +51,45 @@ static int
 nvkm_volt_set(struct nvkm_volt *volt, u32 uv)
 {
 	struct nvkm_subdev *subdev = &volt->subdev;
-	int i, ret = -EINVAL;
+	int i, ret = -EINVAL, best_err = volt->max_uv, best = -1;
 
 	if (volt->func->volt_set)
 		return volt->func->volt_set(volt, uv);
 
 	for (i = 0; i < volt->vid_nr; i++) {
-		if (volt->vid[i].uv == uv) {
-			ret = volt->func->vid_set(volt, volt->vid[i].vid);
-			nvkm_debug(subdev, "set %duv: %d\n", uv, ret);
+		int err = volt->vid[i].uv - uv;
+		if (err < 0 || err > best_err)
+			continue;
+
+		best_err = err;
+		best = i;
+		if (best_err == 0)
 			break;
-		}
 	}
+
+	if (best == -1) {
+		nvkm_error(subdev, "couldn't set %iuv\n", uv);
+		return ret;
+	}
+
+	ret = volt->func->vid_set(volt, volt->vid[best].vid);
+	nvkm_debug(subdev, "set req %duv to %duv: %d\n", uv,
+		   volt->vid[best].uv, ret);
 	return ret;
 }
 
-static int
-nvkm_volt_map(struct nvkm_volt *volt, u8 id)
+int
+nvkm_volt_map_min(struct nvkm_volt *volt, u8 id)
 {
 	struct nvkm_bios *bios = volt->subdev.device->bios;
 	struct nvbios_vmap_entry info;
 	u8  ver, len;
-	u16 vmap;
+	u32 vmap;
 
 	vmap = nvbios_vmap_entry_parse(bios, id, &ver, &len, &info);
 	if (vmap) {
 		if (info.link != 0xff) {
-			int ret = nvkm_volt_map(volt, info.link);
+			int ret = nvkm_volt_map_min(volt, info.link);
 			if (ret < 0)
 				return ret;
 			info.min += ret;
@@ -88,19 +101,79 @@ nvkm_volt_map(struct nvkm_volt *volt, u8 id)
 }
 
 int
-nvkm_volt_set_id(struct nvkm_volt *volt, u8 id, int condition)
+nvkm_volt_map(struct nvkm_volt *volt, u8 id, u8 temp)
+{
+	struct nvkm_bios *bios = volt->subdev.device->bios;
+	struct nvbios_vmap_entry info;
+	u8  ver, len;
+	u32 vmap;
+
+	vmap = nvbios_vmap_entry_parse(bios, id, &ver, &len, &info);
+	if (vmap) {
+		s64 result;
+
+		if (volt->speedo < 0)
+			return volt->speedo;
+
+		if (ver == 0x10 || (ver == 0x20 && info.mode == 0)) {
+			result  = div64_s64((s64)info.arg[0], 10);
+			result += div64_s64((s64)info.arg[1] * volt->speedo, 10);
+			result += div64_s64((s64)info.arg[2] * volt->speedo * volt->speedo, 100000);
+		} else if (ver == 0x20) {
+			switch (info.mode) {
+			/* 0x0 handled above! */
+			case 0x1:
+				result =  ((s64)info.arg[0] * 15625) >> 18;
+				result += ((s64)info.arg[1] * volt->speedo * 15625) >> 18;
+				result += ((s64)info.arg[2] * temp * 15625) >> 10;
+				result += ((s64)info.arg[3] * volt->speedo * temp * 15625) >> 18;
+				result += ((s64)info.arg[4] * volt->speedo * volt->speedo * 15625) >> 30;
+				result += ((s64)info.arg[5] * temp * temp * 15625) >> 18;
+				break;
+			case 0x3:
+				result = (info.min + info.max) / 2;
+				break;
+			case 0x2:
+			default:
+				result = info.min;
+				break;
+			}
+		} else {
+			return -ENODEV;
+		}
+
+		result = min(max(result, (s64)info.min), (s64)info.max);
+
+		if (info.link != 0xff) {
+			int ret = nvkm_volt_map(volt, info.link, temp);
+			if (ret < 0)
+				return ret;
+			result += ret;
+		}
+		return result;
+	}
+
+	return id ? id * 10000 : -ENODEV;
+}
+
+int
+nvkm_volt_set_id(struct nvkm_volt *volt, u8 id, u8 min_id, u8 temp,
+		 int condition)
 {
 	int ret;
 
 	if (volt->func->set_id)
 		return volt->func->set_id(volt, id, condition);
 
-	ret = nvkm_volt_map(volt, id);
+	ret = nvkm_volt_map(volt, id, temp);
 	if (ret >= 0) {
 		int prev = nvkm_volt_get(volt);
 		if (!condition || prev < 0 ||
 		    (condition < 0 && ret < prev) ||
 		    (condition > 0 && ret > prev)) {
+			int min = nvkm_volt_map(volt, min_id, temp);
+			if (min >= 0)
+				ret = max(min, ret);
 			ret = nvkm_volt_set(volt, ret);
 		} else {
 			ret = 0;
@@ -112,14 +185,16 @@ nvkm_volt_set_id(struct nvkm_volt *volt, u8 id, int condition)
 static void
 nvkm_volt_parse_bios(struct nvkm_bios *bios, struct nvkm_volt *volt)
 {
+	struct nvkm_subdev *subdev = &bios->subdev;
 	struct nvbios_volt_entry ivid;
 	struct nvbios_volt info;
 	u8  ver, hdr, cnt, len;
-	u16 data;
+	u32 data;
 	int i;
 
 	data = nvbios_volt_parse(bios, &ver, &hdr, &cnt, &len, &info);
-	if (data && info.vidmask && info.base && info.step) {
+	if (data && info.vidmask && info.base && info.step && info.ranged) {
+		nvkm_debug(subdev, "found ranged based VIDs\n");
 		volt->min_uv = info.min;
 		volt->max_uv = info.max;
 		for (i = 0; i < info.vidmask + 1; i++) {
@@ -132,7 +207,8 @@ nvkm_volt_parse_bios(struct nvkm_bios *bios, struct nvkm_volt *volt)
 			info.base += info.step;
 		}
 		volt->vid_mask = info.vidmask;
-	} else if (data && info.vidmask) {
+	} else if (data && info.vidmask && !info.ranged) {
+		nvkm_debug(subdev, "found entry based VIDs\n");
 		volt->min_uv = 0xffffffff;
 		volt->max_uv = 0;
 		for (i = 0; i < cnt; i++) {
@@ -154,6 +230,14 @@ nvkm_volt_parse_bios(struct nvkm_bios *bios, struct nvkm_volt *volt)
 }
 
 static int
+nvkm_volt_speedo_read(struct nvkm_volt *volt)
+{
+	if (volt->func->speedo_read)
+		return volt->func->speedo_read(volt);
+	return -EINVAL;
+}
+
+static int
 nvkm_volt_init(struct nvkm_subdev *subdev)
 {
 	struct nvkm_volt *volt = nvkm_volt(subdev);
@@ -167,6 +251,21 @@ nvkm_volt_init(struct nvkm_subdev *subdev)
 	return 0;
 }
 
+static int
+nvkm_volt_oneinit(struct nvkm_subdev *subdev)
+{
+	struct nvkm_volt *volt = nvkm_volt(subdev);
+
+	volt->speedo = nvkm_volt_speedo_read(volt);
+	if (volt->speedo > 0)
+		nvkm_debug(&volt->subdev, "speedo %x\n", volt->speedo);
+
+	if (volt->func->oneinit)
+		return volt->func->oneinit(volt);
+
+	return 0;
+}
+
 static void *
 nvkm_volt_dtor(struct nvkm_subdev *subdev)
 {
@@ -177,6 +276,7 @@ static const struct nvkm_subdev_func
 nvkm_volt = {
 	.dtor = nvkm_volt_dtor,
 	.init = nvkm_volt_init,
+	.oneinit = nvkm_volt_oneinit,
 };
 
 void
@@ -191,9 +291,22 @@ nvkm_volt_ctor(const struct nvkm_volt_func *func, struct nvkm_device *device,
 
 	/* Assuming the non-bios device should build the voltage table later */
 	if (bios) {
+		u8 ver, hdr, cnt, len;
+		struct nvbios_vmap vmap;
+
 		nvkm_volt_parse_bios(bios, volt);
 		nvkm_debug(&volt->subdev, "min: %iuv max: %iuv\n",
 			   volt->min_uv, volt->max_uv);
+
+		if (nvbios_vmap_parse(bios, &ver, &hdr, &cnt, &len, &vmap)) {
+			volt->max0_id = vmap.max0;
+			volt->max1_id = vmap.max1;
+			volt->max2_id = vmap.max2;
+		} else {
+			volt->max0_id = 0xff;
+			volt->max1_id = 0xff;
+			volt->max2_id = 0xff;
+		}
 	}
 
 	if (volt->vid_nr) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gf100.c
new file mode 100644
index 000000000000..d9ed6925ca64
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gf100.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2016 Karol Herbst
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Karol Herbst
+ */
+#include "priv.h"
+
+#include <subdev/fuse.h>
+
+static int
+gf100_volt_speedo_read(struct nvkm_volt *volt)
+{
+	struct nvkm_device *device = volt->subdev.device;
+	struct nvkm_fuse *fuse = device->fuse;
+
+	if (!fuse)
+		return -EINVAL;
+
+	return nvkm_fuse_read(fuse, 0x1cc);
+}
+
+int
+gf100_volt_oneinit(struct nvkm_volt *volt)
+{
+	struct nvkm_subdev *subdev = &volt->subdev;
+	if (volt->speedo <= 0)
+		nvkm_error(subdev, "couldn't find speedo value, volting not "
+			   "possible\n");
+	return 0;
+}
+
+static const struct nvkm_volt_func
+gf100_volt = {
+	.oneinit = gf100_volt_oneinit,
+	.vid_get = nvkm_voltgpio_get,
+	.vid_set = nvkm_voltgpio_set,
+	.speedo_read = gf100_volt_speedo_read,
+};
+
+int
+gf100_volt_new(struct nvkm_device *device, int index, struct nvkm_volt **pvolt)
+{
+	struct nvkm_volt *volt;
+	int ret;
+
+	ret = nvkm_volt_new_(&gf100_volt, device, index, &volt);
+	*pvolt = volt;
+	if (ret)
+		return ret;
+
+	return nvkm_voltgpio_init(volt);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk104.c
index 420bd84d8483..1c744e029454 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk104.c
@@ -27,6 +27,7 @@
 #include <subdev/gpio.h>
 #include <subdev/bios.h>
 #include <subdev/bios/volt.h>
+#include <subdev/fuse.h>
 
 #define gk104_volt(p) container_of((p), struct gk104_volt, base)
 struct gk104_volt {
@@ -34,7 +35,7 @@ struct gk104_volt {
 	struct nvbios_volt bios;
 };
 
-int
+static int
 gk104_volt_get(struct nvkm_volt *base)
 {
 	struct nvbios_volt *bios = &gk104_volt(base)->bios;
@@ -47,7 +48,7 @@ gk104_volt_get(struct nvkm_volt *base)
 	return bios->base + bios->pwm_range * duty / div;
 }
 
-int
+static int
 gk104_volt_set(struct nvkm_volt *base, u32 uv)
 {
 	struct nvbios_volt *bios = &gk104_volt(base)->bios;
@@ -64,13 +65,33 @@ gk104_volt_set(struct nvkm_volt *base, u32 uv)
 	return 0;
 }
 
+static int
+gk104_volt_speedo_read(struct nvkm_volt *volt)
+{
+	struct nvkm_device *device = volt->subdev.device;
+	struct nvkm_fuse *fuse = device->fuse;
+	int ret;
+
+	if (!fuse)
+		return -EINVAL;
+
+	nvkm_wr32(device, 0x122634, 0x0);
+	ret = nvkm_fuse_read(fuse, 0x3a8);
+	nvkm_wr32(device, 0x122634, 0x41);
+	return ret;
+}
+
 static const struct nvkm_volt_func
 gk104_volt_pwm = {
+	.oneinit = gf100_volt_oneinit,
 	.volt_get = gk104_volt_get,
 	.volt_set = gk104_volt_set,
+	.speedo_read = gk104_volt_speedo_read,
 }, gk104_volt_gpio = {
+	.oneinit = gf100_volt_oneinit,
 	.vid_get = nvkm_voltgpio_get,
 	.vid_set = nvkm_voltgpio_set,
+	.speedo_read = gk104_volt_speedo_read,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c
index 74db4d28930f..2925b9cae681 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c
@@ -25,7 +25,7 @@
 
 #include <core/tegra.h>
 
-const struct cvb_coef gm20b_cvb_coef[] = {
+static const struct cvb_coef gm20b_cvb_coef[] = {
 	/* KHz,             c0,      c1,   c2 */
 	/*  76800 */ { 1786666,  -85625, 1632 },
 	/* 153600 */ { 1846729,  -87525, 1632 },
@@ -58,7 +58,7 @@ static const struct cvb_coef gm20b_na_cvb_coef[] = {
 	/* 998400 */ { 1316991, 8144, -940, 808, -21583, 226 },
 };
 
-const u32 speedo_to_vmin[] = {
+static const u32 speedo_to_vmin[] = {
 	/*   0,      1,      2,      3,      4, */
 	950000, 840000, 818750, 840000, 810000,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gpio.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gpio.c
index d2bac1d77819..443c031b966b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gpio.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gpio.c
@@ -25,6 +25,7 @@
 #include <subdev/bios.h>
 #include <subdev/bios/gpio.h>
 #include <subdev/gpio.h>
+#include "priv.h"
 
 static const u8 tags[] = {
 	DCB_GPIO_VID0, DCB_GPIO_VID1, DCB_GPIO_VID2, DCB_GPIO_VID3,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/priv.h
index d5140d991161..354bafe4b4e2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/priv.h
@@ -9,11 +9,13 @@ int nvkm_volt_new_(const struct nvkm_volt_func *, struct nvkm_device *,
 		   int index, struct nvkm_volt **);
 
 struct nvkm_volt_func {
+	int (*oneinit)(struct nvkm_volt *);
 	int (*volt_get)(struct nvkm_volt *);
 	int (*volt_set)(struct nvkm_volt *, u32 uv);
 	int (*vid_get)(struct nvkm_volt *);
 	int (*vid_set)(struct nvkm_volt *, u8 vid);
 	int (*set_id)(struct nvkm_volt *, u8 id, int condition);
+	int (*speedo_read)(struct nvkm_volt *);
 };
 
 int nvkm_voltgpio_init(struct nvkm_volt *);
@@ -23,4 +25,6 @@ int nvkm_voltgpio_set(struct nvkm_volt *, u8);
 int nvkm_voltpwm_init(struct nvkm_volt *volt);
 int nvkm_voltpwm_get(struct nvkm_volt *volt);
 int nvkm_voltpwm_set(struct nvkm_volt *volt, u32 uv);
+
+int gf100_volt_oneinit(struct nvkm_volt *);
 #endif
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
index 3485d1ecd655..aaa8a58390f1 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
@@ -24,23 +24,24 @@ struct panel_drv_data {
 
 	struct device *dev;
 
-	struct omap_video_timings timings;
+	struct videomode vm;
 
 	bool invert_polarity;
 };
 
-static const struct omap_video_timings tvc_pal_timings = {
-	.x_res		= 720,
-	.y_res		= 574,
+static const struct videomode tvc_pal_vm = {
+	.hactive	= 720,
+	.vactive	= 574,
 	.pixelclock	= 13500000,
-	.hsw		= 64,
-	.hfp		= 12,
-	.hbp		= 68,
-	.vsw		= 5,
-	.vfp		= 5,
-	.vbp		= 41,
-
-	.interlace	= true,
+	.hsync_len	= 64,
+	.hfront_porch	= 12,
+	.hback_porch	= 68,
+	.vsync_len	= 5,
+	.vfront_porch	= 5,
+	.vback_porch	= 41,
+
+	.flags		= DISPLAY_FLAGS_INTERLACED | DISPLAY_FLAGS_HSYNC_LOW |
+			  DISPLAY_FLAGS_VSYNC_LOW,
 };
 
 static const struct of_device_id tvc_of_match[];
@@ -92,7 +93,7 @@ static int tvc_enable(struct omap_dss_device *dssdev)
 	if (omapdss_device_is_enabled(dssdev))
 		return 0;
 
-	in->ops.atv->set_timings(in, &ddata->timings);
+	in->ops.atv->set_timings(in, &ddata->vm);
 
 	if (!ddata->dev->of_node) {
 		in->ops.atv->set_type(in, OMAP_DSS_VENC_TYPE_COMPOSITE);
@@ -126,32 +127,32 @@ static void tvc_disable(struct omap_dss_device *dssdev)
 }
 
 static void tvc_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			    struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	ddata->timings = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.atv->set_timings(in, timings);
+	in->ops.atv->set_timings(in, vm);
 }
 
 static void tvc_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			    struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*timings = ddata->timings;
+	*vm = ddata->vm;
 }
 
 static int tvc_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			     struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.atv->check_timings(in, timings);
+	return in->ops.atv->check_timings(in, vm);
 }
 
 static u32 tvc_get_wss(struct omap_dss_device *dssdev)
@@ -253,14 +254,14 @@ static int tvc_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
-	ddata->timings = tvc_pal_timings;
+	ddata->vm = tvc_pal_vm;
 
 	dssdev = &ddata->dssdev;
 	dssdev->driver = &tvc_driver;
 	dssdev->dev = &pdev->dev;
 	dssdev->type = OMAP_DISPLAY_TYPE_VENC;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.timings = tvc_pal_timings;
+	dssdev->panel.vm = tvc_pal_vm;
 
 	r = omapdss_register_display(dssdev);
 	if (r) {
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-dvi.c b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
index 684b7aeda411..d6875d9fcefa 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
@@ -19,32 +19,30 @@
 
 #include "../dss/omapdss.h"
 
-static const struct omap_video_timings dvic_default_timings = {
-	.x_res		= 640,
-	.y_res		= 480,
+static const struct videomode dvic_default_vm = {
+	.hactive	= 640,
+	.vactive	= 480,
 
 	.pixelclock	= 23500000,
 
-	.hfp		= 48,
-	.hsw		= 32,
-	.hbp		= 80,
+	.hfront_porch	= 48,
+	.hsync_len	= 32,
+	.hback_porch	= 80,
 
-	.vfp		= 3,
-	.vsw		= 4,
-	.vbp		= 7,
+	.vfront_porch	= 3,
+	.vsync_len	= 4,
+	.vback_porch	= 7,
 
-	.vsync_level	= OMAPDSS_SIG_ACTIVE_HIGH,
-	.hsync_level	= OMAPDSS_SIG_ACTIVE_HIGH,
-	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
-	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
-	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+	.flags		= DISPLAY_FLAGS_HSYNC_HIGH | DISPLAY_FLAGS_VSYNC_HIGH |
+			  DISPLAY_FLAGS_SYNC_NEGEDGE | DISPLAY_FLAGS_DE_HIGH |
+			  DISPLAY_FLAGS_PIXDATA_POSEDGE,
 };
 
 struct panel_drv_data {
 	struct omap_dss_device dssdev;
 	struct omap_dss_device *in;
 
-	struct omap_video_timings timings;
+	struct videomode vm;
 
 	struct i2c_adapter *i2c_adapter;
 };
@@ -90,7 +88,7 @@ static int dvic_enable(struct omap_dss_device *dssdev)
 	if (omapdss_device_is_enabled(dssdev))
 		return 0;
 
-	in->ops.dvi->set_timings(in, &ddata->timings);
+	in->ops.dvi->set_timings(in, &ddata->vm);
 
 	r = in->ops.dvi->enable(in);
 	if (r)
@@ -115,32 +113,32 @@ static void dvic_disable(struct omap_dss_device *dssdev)
 }
 
 static void dvic_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			     struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	ddata->timings = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.dvi->set_timings(in, timings);
+	in->ops.dvi->set_timings(in, vm);
 }
 
 static void dvic_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			     struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*timings = ddata->timings;
+	*vm = ddata->vm;
 }
 
 static int dvic_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			      struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.dvi->check_timings(in, timings);
+	return in->ops.dvi->check_timings(in, vm);
 }
 
 static int dvic_ddc_read(struct i2c_adapter *adapter,
@@ -287,14 +285,14 @@ static int dvic_probe(struct platform_device *pdev)
 	if (r)
 		return r;
 
-	ddata->timings = dvic_default_timings;
+	ddata->vm = dvic_default_vm;
 
 	dssdev = &ddata->dssdev;
 	dssdev->driver = &dvic_driver;
 	dssdev->dev = &pdev->dev;
 	dssdev->type = OMAP_DISPLAY_TYPE_DVI;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.timings = dvic_default_timings;
+	dssdev->panel.vm = dvic_default_vm;
 
 	r = omapdss_register_display(dssdev);
 	if (r) {
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
index 7bdf83af9797..1ef130641bae 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
@@ -21,21 +21,18 @@
 
 #include "../dss/omapdss.h"
 
-static const struct omap_video_timings hdmic_default_timings = {
-	.x_res		= 640,
-	.y_res		= 480,
+static const struct videomode hdmic_default_vm = {
+	.hactive	= 640,
+	.vactive	= 480,
 	.pixelclock	= 25175000,
-	.hsw		= 96,
-	.hfp		= 16,
-	.hbp		= 48,
-	.vsw		= 2,
-	.vfp		= 11,
-	.vbp		= 31,
-
-	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-
-	.interlace	= false,
+	.hsync_len	= 96,
+	.hfront_porch	= 16,
+	.hback_porch	= 48,
+	.vsync_len	= 2,
+	.vfront_porch	= 11,
+	.vback_porch	= 31,
+
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW,
 };
 
 struct panel_drv_data {
@@ -44,7 +41,7 @@ struct panel_drv_data {
 
 	struct device *dev;
 
-	struct omap_video_timings timings;
+	struct videomode vm;
 
 	int hpd_gpio;
 };
@@ -96,7 +93,7 @@ static int hdmic_enable(struct omap_dss_device *dssdev)
 	if (omapdss_device_is_enabled(dssdev))
 		return 0;
 
-	in->ops.hdmi->set_timings(in, &ddata->timings);
+	in->ops.hdmi->set_timings(in, &ddata->vm);
 
 	r = in->ops.hdmi->enable(in);
 	if (r)
@@ -123,32 +120,32 @@ static void hdmic_disable(struct omap_dss_device *dssdev)
 }
 
 static void hdmic_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			      struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	ddata->timings = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.hdmi->set_timings(in, timings);
+	in->ops.hdmi->set_timings(in, vm);
 }
 
 static void hdmic_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			      struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*timings = ddata->timings;
+	*vm = ddata->vm;
 }
 
 static int hdmic_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			       struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.hdmi->check_timings(in, timings);
+	return in->ops.hdmi->check_timings(in, vm);
 }
 
 static int hdmic_read_edid(struct omap_dss_device *dssdev,
@@ -259,14 +256,14 @@ static int hdmic_probe(struct platform_device *pdev)
 			goto err_reg;
 	}
 
-	ddata->timings = hdmic_default_timings;
+	ddata->vm = hdmic_default_vm;
 
 	dssdev = &ddata->dssdev;
 	dssdev->driver = &hdmic_driver;
 	dssdev->dev = &pdev->dev;
 	dssdev->type = OMAP_DISPLAY_TYPE_HDMI;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.timings = hdmic_default_timings;
+	dssdev->panel.vm = hdmic_default_vm;
 
 	r = omapdss_register_display(dssdev);
 	if (r) {
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
index fe4e7ec3bab0..f7a5731492d0 100644
--- a/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
@@ -27,7 +27,7 @@ struct panel_drv_data {
 
 	struct gpio_desc *enable_gpio;
 
-	struct omap_video_timings timings;
+	struct videomode vm;
 };
 
 #define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
@@ -90,7 +90,7 @@ static int opa362_enable(struct omap_dss_device *dssdev)
 	if (omapdss_device_is_enabled(dssdev))
 		return 0;
 
-	in->ops.atv->set_timings(in, &ddata->timings);
+	in->ops.atv->set_timings(in, &ddata->vm);
 
 	r = in->ops.atv->enable(in);
 	if (r)
@@ -123,38 +123,38 @@ static void opa362_disable(struct omap_dss_device *dssdev)
 }
 
 static void opa362_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			       struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
 	dev_dbg(dssdev->dev, "set_timings\n");
 
-	ddata->timings = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.atv->set_timings(in, timings);
+	in->ops.atv->set_timings(in, vm);
 }
 
 static void opa362_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			       struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
 	dev_dbg(dssdev->dev, "get_timings\n");
 
-	*timings = ddata->timings;
+	*vm = ddata->vm;
 }
 
 static int opa362_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
 	dev_dbg(dssdev->dev, "check_timings\n");
 
-	return in->ops.atv->check_timings(in, timings);
+	return in->ops.atv->check_timings(in, vm);
 }
 
 static void opa362_set_type(struct omap_dss_device *dssdev,
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
index d768217cefe0..13e32d02c884 100644
--- a/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
@@ -24,7 +24,7 @@ struct panel_drv_data {
 	int pd_gpio;
 	int data_lines;
 
-	struct omap_video_timings timings;
+	struct videomode vm;
 };
 
 #define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
@@ -81,7 +81,7 @@ static int tfp410_enable(struct omap_dss_device *dssdev)
 	if (omapdss_device_is_enabled(dssdev))
 		return 0;
 
-	in->ops.dpi->set_timings(in, &ddata->timings);
+	in->ops.dpi->set_timings(in, &ddata->vm);
 	if (ddata->data_lines)
 		in->ops.dpi->set_data_lines(in, ddata->data_lines);
 
@@ -113,44 +113,43 @@ static void tfp410_disable(struct omap_dss_device *dssdev)
 	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
 }
 
-static void tfp410_fix_timings(struct omap_video_timings *timings)
+static void tfp410_fix_timings(struct videomode *vm)
 {
-	timings->data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
-	timings->sync_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
-	timings->de_level = OMAPDSS_SIG_ACTIVE_HIGH;
+	vm->flags |= DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_PIXDATA_POSEDGE |
+		     DISPLAY_FLAGS_SYNC_POSEDGE;
 }
 
 static void tfp410_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			       struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	tfp410_fix_timings(timings);
+	tfp410_fix_timings(vm);
 
-	ddata->timings = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.dpi->set_timings(in, timings);
+	in->ops.dpi->set_timings(in, vm);
 }
 
 static void tfp410_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			       struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*timings = ddata->timings;
+	*vm = ddata->vm;
 }
 
 static int tfp410_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	tfp410_fix_timings(timings);
+	tfp410_fix_timings(vm);
 
-	return in->ops.dpi->check_timings(in, timings);
+	return in->ops.dpi->check_timings(in, vm);
 }
 
 static const struct omapdss_dvi_ops tfp410_dvi_ops = {
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
index 46855c8f5cbf..58276a48112e 100644
--- a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
@@ -26,7 +26,7 @@ struct panel_drv_data {
 	struct gpio_desc *ls_oe_gpio;
 	struct gpio_desc *hpd_gpio;
 
-	struct omap_video_timings timings;
+	struct videomode vm;
 };
 
 #define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
@@ -80,7 +80,7 @@ static int tpd_enable(struct omap_dss_device *dssdev)
 	if (dssdev->state == OMAP_DSS_DISPLAY_ACTIVE)
 		return 0;
 
-	in->ops.hdmi->set_timings(in, &ddata->timings);
+	in->ops.hdmi->set_timings(in, &ddata->vm);
 
 	r = in->ops.hdmi->enable(in);
 	if (r)
@@ -105,33 +105,33 @@ static void tpd_disable(struct omap_dss_device *dssdev)
 }
 
 static void tpd_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			    struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	ddata->timings = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.hdmi->set_timings(in, timings);
+	in->ops.hdmi->set_timings(in, vm);
 }
 
 static void tpd_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			    struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*timings = ddata->timings;
+	*vm = ddata->vm;
 }
 
 static int tpd_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			     struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 	int r;
 
-	r = in->ops.hdmi->check_timings(in, timings);
+	r = in->ops.hdmi->check_timings(in, vm);
 
 	return r;
 }
@@ -234,25 +234,30 @@ static int tpd_probe(struct platform_device *pdev)
 	if (r)
 		return r;
 
-
 	gpio = devm_gpiod_get_index_optional(&pdev->dev, NULL, 0,
 		 GPIOD_OUT_LOW);
-	if (IS_ERR(gpio))
+	if (IS_ERR(gpio)) {
+		r = PTR_ERR(gpio);
 		goto err_gpio;
+	}
 
 	ddata->ct_cp_hpd_gpio = gpio;
 
 	gpio = devm_gpiod_get_index_optional(&pdev->dev, NULL, 1,
 		 GPIOD_OUT_LOW);
-	if (IS_ERR(gpio))
+	if (IS_ERR(gpio)) {
+		r = PTR_ERR(gpio);
 		goto err_gpio;
+	}
 
 	ddata->ls_oe_gpio = gpio;
 
 	gpio = devm_gpiod_get_index(&pdev->dev, NULL, 2,
 		GPIOD_IN);
-	if (IS_ERR(gpio))
+	if (IS_ERR(gpio)) {
+		r = PTR_ERR(gpio);
 		goto err_gpio;
+	}
 
 	ddata->hpd_gpio = gpio;
 
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
index 7f16f985ab22..38003208d9ca 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
@@ -28,7 +28,7 @@ struct panel_drv_data {
 
 	int data_lines;
 
-	struct omap_video_timings videomode;
+	struct videomode vm;
 
 	/* used for non-DT boot, to be removed */
 	int backlight_gpio;
@@ -80,7 +80,7 @@ static int panel_dpi_enable(struct omap_dss_device *dssdev)
 
 	if (ddata->data_lines)
 		in->ops.dpi->set_data_lines(in, ddata->data_lines);
-	in->ops.dpi->set_timings(in, &ddata->videomode);
+	in->ops.dpi->set_timings(in, &ddata->vm);
 
 	r = in->ops.dpi->enable(in);
 	if (r)
@@ -122,32 +122,32 @@ static void panel_dpi_disable(struct omap_dss_device *dssdev)
 }
 
 static void panel_dpi_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				  struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	ddata->videomode = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.dpi->set_timings(in, timings);
+	in->ops.dpi->set_timings(in, vm);
 }
 
 static void panel_dpi_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				  struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*timings = ddata->videomode;
+	*vm = ddata->vm;
 }
 
 static int panel_dpi_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				   struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.dpi->check_timings(in, timings);
+	return in->ops.dpi->check_timings(in, vm);
 }
 
 static struct omap_dss_driver panel_dpi_ops = {
@@ -169,7 +169,6 @@ static int panel_dpi_probe_pdata(struct platform_device *pdev)
 	const struct panel_dpi_platform_data *pdata;
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
 	struct omap_dss_device *dssdev, *in;
-	struct videomode vm;
 	int r;
 
 	pdata = dev_get_platdata(&pdev->dev);
@@ -185,8 +184,7 @@ static int panel_dpi_probe_pdata(struct platform_device *pdev)
 
 	ddata->data_lines = pdata->data_lines;
 
-	videomode_from_timing(pdata->display_timing, &vm);
-	videomode_to_omap_video_timings(&vm, &ddata->videomode);
+	videomode_from_timing(pdata->display_timing, &ddata->vm);
 
 	dssdev = &ddata->dssdev;
 	dssdev->name = pdata->name;
@@ -214,7 +212,6 @@ static int panel_dpi_probe_of(struct platform_device *pdev)
 	struct omap_dss_device *in;
 	int r;
 	struct display_timing timing;
-	struct videomode vm;
 	struct gpio_desc *gpio;
 
 	gpio = devm_gpiod_get_optional(&pdev->dev, "enable", GPIOD_OUT_LOW);
@@ -245,8 +242,7 @@ static int panel_dpi_probe_of(struct platform_device *pdev)
 		return r;
 	}
 
-	videomode_from_timing(&timing, &vm);
-	videomode_to_omap_video_timings(&vm, &ddata->videomode);
+	videomode_from_timing(&timing, &ddata->vm);
 
 	in = omapdss_of_find_source_for_first_ep(node);
 	if (IS_ERR(in)) {
@@ -295,7 +291,7 @@ static int panel_dpi_probe(struct platform_device *pdev)
 	dssdev->driver = &panel_dpi_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.timings = ddata->videomode;
+	dssdev->panel.vm = ddata->vm;
 	dssdev->phy.dpi.data_lines = ddata->data_lines;
 
 	r = omapdss_register_display(dssdev);
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
index b1f3b818edf4..dc026a843712 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
@@ -42,7 +42,7 @@ struct panel_drv_data {
 	struct omap_dss_device dssdev;
 	struct omap_dss_device *in;
 
-	struct omap_video_timings timings;
+	struct videomode vm;
 
 	struct platform_device *pdev;
 
@@ -382,8 +382,8 @@ static const struct backlight_ops dsicm_bl_ops = {
 static void dsicm_get_resolution(struct omap_dss_device *dssdev,
 		u16 *xres, u16 *yres)
 {
-	*xres = dssdev->panel.timings.x_res;
-	*yres = dssdev->panel.timings.y_res;
+	*xres = dssdev->panel.vm.hactive;
+	*yres = dssdev->panel.vm.vactive;
 }
 
 static ssize_t dsicm_num_errors_show(struct device *dev,
@@ -589,7 +589,7 @@ static int dsicm_power_on(struct panel_drv_data *ddata)
 	struct omap_dss_dsi_config dsi_config = {
 		.mode = OMAP_DSS_DSI_CMD_MODE,
 		.pixel_format = OMAP_DSS_DSI_FMT_RGB888,
-		.timings = &ddata->timings,
+		.vm = &ddata->vm,
 		.hs_clk_min = 150000000,
 		.hs_clk_max = 300000000,
 		.lp_clk_min = 7000000,
@@ -892,8 +892,8 @@ static int dsicm_update(struct omap_dss_device *dssdev,
 
 	/* XXX no need to send this every frame, but dsi break if not done */
 	r = dsicm_set_update_window(ddata, 0, 0,
-			dssdev->panel.timings.x_res,
-			dssdev->panel.timings.y_res);
+			dssdev->panel.vm.hactive,
+			dssdev->panel.vm.vactive);
 	if (r)
 		goto err;
 
@@ -1023,9 +1023,8 @@ static int dsicm_memory_read(struct omap_dss_device *dssdev,
 		goto err1;
 	}
 
-	size = min(w * h * 3,
-			dssdev->panel.timings.x_res *
-			dssdev->panel.timings.y_res * 3);
+	size = min((u32)w * h * 3,
+		   dssdev->panel.vm.hactive * dssdev->panel.vm.vactive * 3);
 
 	in->ops.dsi->bus_lock(in);
 
@@ -1186,14 +1185,14 @@ static int dsicm_probe(struct platform_device *pdev)
 	if (r)
 		return r;
 
-	ddata->timings.x_res = 864;
-	ddata->timings.y_res = 480;
-	ddata->timings.pixelclock = 864 * 480 * 60;
+	ddata->vm.hactive = 864;
+	ddata->vm.vactive = 480;
+	ddata->vm.pixelclock = 864 * 480 * 60;
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = dev;
 	dssdev->driver = &dsicm_ops;
-	dssdev->panel.timings = ddata->timings;
+	dssdev->panel.vm = ddata->vm;
 	dssdev->type = OMAP_DISPLAY_TYPE_DSI;
 	dssdev->owner = THIS_MODULE;
 
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
index 6dfb96cea293..43d21edb51f5 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
@@ -19,25 +19,28 @@
 
 #include "../dss/omapdss.h"
 
-static struct omap_video_timings lb035q02_timings = {
-	.x_res = 320,
-	.y_res = 240,
+static struct videomode lb035q02_vm = {
+	.hactive = 320,
+	.vactive = 240,
 
 	.pixelclock	= 6500000,
 
-	.hsw		= 2,
-	.hfp		= 20,
-	.hbp		= 68,
-
-	.vsw		= 2,
-	.vfp		= 4,
-	.vbp		= 18,
-
-	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
-	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
-	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+	.hsync_len	= 2,
+	.hfront_porch	= 20,
+	.hback_porch	= 68,
+
+	.vsync_len	= 2,
+	.vfront_porch	= 4,
+	.vback_porch	= 18,
+
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
+			  DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_NEGEDGE |
+			  DISPLAY_FLAGS_PIXDATA_POSEDGE,
+	/*
+	 * Note: According to the panel documentation:
+	 * DE is active LOW
+	 * DATA needs to be driven on the FALLING edge
+	 */
 };
 
 struct panel_drv_data {
@@ -48,7 +51,7 @@ struct panel_drv_data {
 
 	int data_lines;
 
-	struct omap_video_timings videomode;
+	struct videomode vm;
 
 	struct gpio_desc *enable_gpio;
 };
@@ -158,7 +161,7 @@ static int lb035q02_enable(struct omap_dss_device *dssdev)
 
 	if (ddata->data_lines)
 		in->ops.dpi->set_data_lines(in, ddata->data_lines);
-	in->ops.dpi->set_timings(in, &ddata->videomode);
+	in->ops.dpi->set_timings(in, &ddata->vm);
 
 	r = in->ops.dpi->enable(in);
 	if (r)
@@ -189,32 +192,32 @@ static void lb035q02_disable(struct omap_dss_device *dssdev)
 }
 
 static void lb035q02_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				 struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	ddata->videomode = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.dpi->set_timings(in, timings);
+	in->ops.dpi->set_timings(in, vm);
 }
 
 static void lb035q02_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				 struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*timings = ddata->videomode;
+	*vm = ddata->vm;
 }
 
 static int lb035q02_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				  struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.dpi->check_timings(in, timings);
+	return in->ops.dpi->check_timings(in, vm);
 }
 
 static struct omap_dss_driver lb035q02_ops = {
@@ -278,14 +281,14 @@ static int lb035q02_panel_spi_probe(struct spi_device *spi)
 	if (r)
 		return r;
 
-	ddata->videomode = lb035q02_timings;
+	ddata->vm = lb035q02_vm;
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = &spi->dev;
 	dssdev->driver = &lb035q02_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.timings = ddata->videomode;
+	dssdev->panel.vm = ddata->vm;
 	dssdev->phy.dpi.data_lines = ddata->data_lines;
 
 	r = omapdss_register_display(dssdev);
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
index 9f3d6f48f3e1..2de27ba01552 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
@@ -23,7 +23,7 @@ struct panel_drv_data {
 	struct omap_dss_device	dssdev;
 	struct omap_dss_device *in;
 
-	struct omap_video_timings videomode;
+	struct videomode vm;
 
 	int data_lines;
 
@@ -65,22 +65,20 @@ static const struct {
 	{ 156, 0x00 }, { 157, 0x00 }, { 2, 0x00 },
 };
 
-static const struct omap_video_timings nec_8048_panel_timings = {
-	.x_res		= LCD_XRES,
-	.y_res		= LCD_YRES,
+static const struct videomode nec_8048_panel_vm = {
+	.hactive	= LCD_XRES,
+	.vactive	= LCD_YRES,
 	.pixelclock	= LCD_PIXEL_CLOCK,
-	.hfp		= 6,
-	.hsw		= 1,
-	.hbp		= 4,
-	.vfp		= 3,
-	.vsw		= 1,
-	.vbp		= 4,
-
-	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
-	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
-	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
+	.hfront_porch	= 6,
+	.hsync_len	= 1,
+	.hback_porch	= 4,
+	.vfront_porch	= 3,
+	.vsync_len	= 1,
+	.vback_porch	= 4,
+
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
+			  DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_POSEDGE |
+			  DISPLAY_FLAGS_PIXDATA_POSEDGE,
 };
 
 #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
@@ -157,7 +155,7 @@ static int nec_8048_enable(struct omap_dss_device *dssdev)
 
 	if (ddata->data_lines)
 		in->ops.dpi->set_data_lines(in, ddata->data_lines);
-	in->ops.dpi->set_timings(in, &ddata->videomode);
+	in->ops.dpi->set_timings(in, &ddata->vm);
 
 	r = in->ops.dpi->enable(in);
 	if (r)
@@ -188,32 +186,32 @@ static void nec_8048_disable(struct omap_dss_device *dssdev)
 }
 
 static void nec_8048_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				 struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	ddata->videomode = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.dpi->set_timings(in, timings);
+	in->ops.dpi->set_timings(in, vm);
 }
 
 static void nec_8048_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				 struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*timings = ddata->videomode;
+	*vm = ddata->vm;
 }
 
 static int nec_8048_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				  struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.dpi->check_timings(in, timings);
+	return in->ops.dpi->check_timings(in, vm);
 }
 
 static struct omap_dss_driver nec_8048_ops = {
@@ -306,14 +304,14 @@ static int nec_8048_probe(struct spi_device *spi)
 			goto err_gpio;
 	}
 
-	ddata->videomode = nec_8048_panel_timings;
+	ddata->vm = nec_8048_panel_vm;
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = &spi->dev;
 	dssdev->driver = &nec_8048_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.timings = ddata->videomode;
+	dssdev->panel.vm = ddata->vm;
 
 	r = omapdss_register_display(dssdev);
 	if (r) {
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
index 3d3efc561ea9..04fe235b7cac 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
@@ -26,7 +26,7 @@ struct panel_drv_data {
 
 	int data_lines;
 
-	struct omap_video_timings videomode;
+	struct videomode vm;
 
 	struct gpio_desc *resb_gpio;	/* low = reset active min 20 us */
 	struct gpio_desc *ini_gpio;	/* high = power on */
@@ -35,25 +35,27 @@ struct panel_drv_data {
 	struct gpio_desc *ud_gpio;	/* high = conventional vertical scanning */
 };
 
-static const struct omap_video_timings sharp_ls_timings = {
-	.x_res = 480,
-	.y_res = 640,
+static const struct videomode sharp_ls_vm = {
+	.hactive = 480,
+	.vactive = 640,
 
 	.pixelclock	= 19200000,
 
-	.hsw		= 2,
-	.hfp		= 1,
-	.hbp		= 28,
-
-	.vsw		= 1,
-	.vfp		= 1,
-	.vbp		= 1,
-
-	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
-	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
-	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+	.hsync_len	= 2,
+	.hfront_porch	= 1,
+	.hback_porch	= 28,
+
+	.vsync_len	= 1,
+	.vfront_porch	= 1,
+	.vback_porch	= 1,
+
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
+			  DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_NEGEDGE |
+			  DISPLAY_FLAGS_PIXDATA_POSEDGE,
+	/*
+	 * Note: According to the panel documentation:
+	 * DATA needs to be driven on the FALLING edge
+	 */
 };
 
 #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
@@ -99,7 +101,7 @@ static int sharp_ls_enable(struct omap_dss_device *dssdev)
 
 	if (ddata->data_lines)
 		in->ops.dpi->set_data_lines(in, ddata->data_lines);
-	in->ops.dpi->set_timings(in, &ddata->videomode);
+	in->ops.dpi->set_timings(in, &ddata->vm);
 
 	if (ddata->vcc) {
 		r = regulator_enable(ddata->vcc);
@@ -154,32 +156,32 @@ static void sharp_ls_disable(struct omap_dss_device *dssdev)
 }
 
 static void sharp_ls_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				 struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	ddata->videomode = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.dpi->set_timings(in, timings);
+	in->ops.dpi->set_timings(in, vm);
 }
 
 static void sharp_ls_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				 struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*timings = ddata->videomode;
+	*vm = ddata->vm;
 }
 
 static int sharp_ls_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				  struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.dpi->check_timings(in, timings);
+	return in->ops.dpi->check_timings(in, vm);
 }
 
 static struct omap_dss_driver sharp_ls_ops = {
@@ -279,14 +281,14 @@ static int sharp_ls_probe(struct platform_device *pdev)
 	if (r)
 		return r;
 
-	ddata->videomode = sharp_ls_timings;
+	ddata->vm = sharp_ls_vm;
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = &pdev->dev;
 	dssdev->driver = &sharp_ls_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.timings = ddata->videomode;
+	dssdev->panel.vm = ddata->vm;
 	dssdev->phy.dpi.data_lines = ddata->data_lines;
 
 	r = omapdss_register_display(dssdev);
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
index 3557a4c7dd7b..746cb8d9cba1 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
@@ -71,7 +71,7 @@ struct panel_drv_data {
 	int reset_gpio;
 	int datapairs;
 
-	struct omap_video_timings videomode;
+	struct videomode vm;
 
 	char		*name;
 	int		enabled;
@@ -92,23 +92,20 @@ struct panel_drv_data {
 	struct backlight_device *bl_dev;
 };
 
-static const struct omap_video_timings acx565akm_panel_timings = {
-	.x_res		= 800,
-	.y_res		= 480,
+static const struct videomode acx565akm_panel_vm = {
+	.hactive	= 800,
+	.vactive	= 480,
 	.pixelclock	= 24000000,
-	.hfp		= 28,
-	.hsw		= 4,
-	.hbp		= 24,
-	.vfp		= 3,
-	.vsw		= 3,
-	.vbp		= 4,
-
-	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-
-	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
-	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
-	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+	.hfront_porch	= 28,
+	.hsync_len	= 4,
+	.hback_porch	= 24,
+	.vfront_porch	= 3,
+	.vsync_len	= 3,
+	.vback_porch	= 4,
+
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
+			  DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_NEGEDGE |
+			  DISPLAY_FLAGS_PIXDATA_POSEDGE,
 };
 
 #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
@@ -548,7 +545,7 @@ static int acx565akm_panel_power_on(struct omap_dss_device *dssdev)
 
 	dev_dbg(&ddata->spi->dev, "%s\n", __func__);
 
-	in->ops.sdi->set_timings(in, &ddata->videomode);
+	in->ops.sdi->set_timings(in, &ddata->vm);
 
 	if (ddata->datapairs > 0)
 		in->ops.sdi->set_datapairs(in, ddata->datapairs);
@@ -662,32 +659,32 @@ static void acx565akm_disable(struct omap_dss_device *dssdev)
 }
 
 static void acx565akm_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				  struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	ddata->videomode = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.sdi->set_timings(in, timings);
+	in->ops.sdi->set_timings(in, vm);
 }
 
 static void acx565akm_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				  struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*timings = ddata->videomode;
+	*vm = ddata->vm;
 }
 
 static int acx565akm_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				   struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.sdi->check_timings(in, timings);
+	return in->ops.sdi->check_timings(in, vm);
 }
 
 static struct omap_dss_driver acx565akm_ops = {
@@ -845,14 +842,14 @@ static int acx565akm_probe(struct spi_device *spi)
 	acx565akm_bl_update_status(bldev);
 
 
-	ddata->videomode = acx565akm_panel_timings;
+	ddata->vm = acx565akm_panel_vm;
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = &spi->dev;
 	dssdev->driver = &acx565akm_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_SDI;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.timings = ddata->videomode;
+	dssdev->panel.vm = ddata->vm;
 
 	r = omapdss_register_display(dssdev);
 	if (r) {
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
index e859b3f893f7..f313dbfcbacb 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
@@ -37,28 +37,29 @@ struct panel_drv_data {
 
 	int data_lines;
 
-	struct omap_video_timings videomode;
+	struct videomode vm;
 
 	struct spi_device *spi_dev;
 };
 
-static struct omap_video_timings td028ttec1_panel_timings = {
-	.x_res		= 480,
-	.y_res		= 640,
+static struct videomode td028ttec1_panel_vm = {
+	.hactive	= 480,
+	.vactive	= 640,
 	.pixelclock	= 22153000,
-	.hfp		= 24,
-	.hsw		= 8,
-	.hbp		= 8,
-	.vfp		= 4,
-	.vsw		= 2,
-	.vbp		= 2,
-
-	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-
-	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_FALLING_EDGE,
-	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
-	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
+	.hfront_porch	= 24,
+	.hsync_len	= 8,
+	.hback_porch	= 8,
+	.vfront_porch	= 4,
+	.vsync_len	= 2,
+	.vback_porch	= 2,
+
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
+			  DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_POSEDGE |
+			  DISPLAY_FLAGS_PIXDATA_NEGEDGE,
+	/*
+	 * Note: According to the panel documentation:
+	 * SYNC needs to be driven on the FALLING edge
+	 */
 };
 
 #define JBT_COMMAND	0x000
@@ -208,7 +209,7 @@ static int td028ttec1_panel_enable(struct omap_dss_device *dssdev)
 
 	if (ddata->data_lines)
 		in->ops.dpi->set_data_lines(in, ddata->data_lines);
-	in->ops.dpi->set_timings(in, &ddata->videomode);
+	in->ops.dpi->set_timings(in, &ddata->vm);
 
 	r = in->ops.dpi->enable(in);
 	if (r)
@@ -325,32 +326,32 @@ static void td028ttec1_panel_disable(struct omap_dss_device *dssdev)
 }
 
 static void td028ttec1_panel_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+					 struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	ddata->videomode = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.dpi->set_timings(in, timings);
+	in->ops.dpi->set_timings(in, vm);
 }
 
 static void td028ttec1_panel_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+					 struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*timings = ddata->videomode;
+	*vm = ddata->vm;
 }
 
 static int td028ttec1_panel_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+					  struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.dpi->check_timings(in, timings);
+	return in->ops.dpi->check_timings(in, vm);
 }
 
 static struct omap_dss_driver td028ttec1_ops = {
@@ -414,14 +415,14 @@ static int td028ttec1_panel_probe(struct spi_device *spi)
 	if (r)
 		return r;
 
-	ddata->videomode = td028ttec1_panel_timings;
+	ddata->vm = td028ttec1_panel_vm;
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = &spi->dev;
 	dssdev->driver = &td028ttec1_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.timings = ddata->videomode;
+	dssdev->panel.vm = ddata->vm;
 	dssdev->phy.dpi.data_lines = ddata->data_lines;
 
 	r = omapdss_register_display(dssdev);
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
index 66c6bbe6472b..0787dba44faa 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
@@ -56,7 +56,7 @@ struct panel_drv_data {
 	struct omap_dss_device	dssdev;
 	struct omap_dss_device *in;
 
-	struct omap_video_timings videomode;
+	struct videomode vm;
 
 	int data_lines;
 
@@ -72,25 +72,27 @@ struct panel_drv_data {
 	u32 power_on_resume:1;
 };
 
-static const struct omap_video_timings tpo_td043_timings = {
-	.x_res		= 800,
-	.y_res		= 480,
+static const struct videomode tpo_td043_vm = {
+	.hactive	= 800,
+	.vactive	= 480,
 
 	.pixelclock	= 36000000,
 
-	.hsw		= 1,
-	.hfp		= 68,
-	.hbp		= 214,
+	.hsync_len	= 1,
+	.hfront_porch	= 68,
+	.hback_porch	= 214,
 
-	.vsw		= 1,
-	.vfp		= 39,
-	.vbp		= 34,
+	.vsync_len	= 1,
+	.vfront_porch	= 39,
+	.vback_porch	= 34,
 
-	.vsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-	.hsync_level	= OMAPDSS_SIG_ACTIVE_LOW,
-	.data_pclk_edge	= OMAPDSS_DRIVE_SIG_FALLING_EDGE,
-	.de_level	= OMAPDSS_SIG_ACTIVE_HIGH,
-	.sync_pclk_edge	= OMAPDSS_DRIVE_SIG_RISING_EDGE,
+	.flags		= DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
+			  DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_POSEDGE |
+			  DISPLAY_FLAGS_PIXDATA_NEGEDGE,
+	/*
+	 * Note: According to the panel documentation:
+	 * SYNC needs to be driven on the FALLING edge
+	 */
 };
 
 #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
@@ -378,7 +380,7 @@ static int tpo_td043_enable(struct omap_dss_device *dssdev)
 
 	if (ddata->data_lines)
 		in->ops.dpi->set_data_lines(in, ddata->data_lines);
-	in->ops.dpi->set_timings(in, &ddata->videomode);
+	in->ops.dpi->set_timings(in, &ddata->vm);
 
 	r = in->ops.dpi->enable(in);
 	if (r)
@@ -418,32 +420,32 @@ static void tpo_td043_disable(struct omap_dss_device *dssdev)
 }
 
 static void tpo_td043_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				  struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	ddata->videomode = *timings;
-	dssdev->panel.timings = *timings;
+	ddata->vm = *vm;
+	dssdev->panel.vm = *vm;
 
-	in->ops.dpi->set_timings(in, timings);
+	in->ops.dpi->set_timings(in, vm);
 }
 
 static void tpo_td043_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				  struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*timings = ddata->videomode;
+	*vm = ddata->vm;
 }
 
 static int tpo_td043_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				   struct videomode *vm)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	struct omap_dss_device *in = ddata->in;
 
-	return in->ops.dpi->check_timings(in, timings);
+	return in->ops.dpi->check_timings(in, vm);
 }
 
 static struct omap_dss_driver tpo_td043_ops = {
@@ -546,14 +548,14 @@ static int tpo_td043_probe(struct spi_device *spi)
 		goto err_sysfs;
 	}
 
-	ddata->videomode = tpo_td043_timings;
+	ddata->vm = tpo_td043_vm;
 
 	dssdev = &ddata->dssdev;
 	dssdev->dev = &spi->dev;
 	dssdev->driver = &tpo_td043_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
 	dssdev->owner = THIS_MODULE;
-	dssdev->panel.timings = ddata->videomode;
+	dssdev->panel.vm = ddata->vm;
 
 	r = omapdss_register_display(dssdev);
 	if (r) {
diff --git a/drivers/gpu/drm/omapdrm/dss/dispc.c b/drivers/gpu/drm/omapdrm/dss/dispc.c
index 535240fba671..c839f6456db2 100644
--- a/drivers/gpu/drm/omapdrm/dss/dispc.c
+++ b/drivers/gpu/drm/omapdrm/dss/dispc.c
@@ -75,7 +75,7 @@ struct dispc_features {
 	unsigned long max_lcd_pclk;
 	unsigned long max_tv_pclk;
 	int (*calc_scaling) (unsigned long pclk, unsigned long lclk,
-		const struct omap_video_timings *mgr_timings,
+		const struct videomode *vm,
 		u16 width, u16 height, u16 out_width, u16 out_height,
 		enum omap_color_mode color_mode, bool *five_taps,
 		int *x_predecim, int *y_predecim, int *decim_x, int *decim_y,
@@ -1679,7 +1679,7 @@ static void dispc_ovl_set_scaling_uv(enum omap_plane plane,
 {
 	int scale_x = out_width != orig_width;
 	int scale_y = out_height != orig_height;
-	bool chroma_upscale = plane != OMAP_DSS_WB ? true : false;
+	bool chroma_upscale = plane != OMAP_DSS_WB;
 
 	if (!dss_has_feature(FEAT_HANDLE_UV_SEPARATE))
 		return;
@@ -2179,7 +2179,7 @@ static void calc_tiler_rotation_offset(u16 screen_width, u16 width,
  * undocumented horizontal position and timing related limitations.
  */
 static int check_horiz_timing_omap3(unsigned long pclk, unsigned long lclk,
-		const struct omap_video_timings *t, u16 pos_x,
+		const struct videomode *vm, u16 pos_x,
 		u16 width, u16 height, u16 out_width, u16 out_height,
 		bool five_taps)
 {
@@ -2189,14 +2189,16 @@ static int check_horiz_timing_omap3(unsigned long pclk, unsigned long lclk,
 	u64 val, blank;
 	int i;
 
-	nonactive = t->x_res + t->hfp + t->hsw + t->hbp - out_width;
+	nonactive = vm->hactive + vm->hfront_porch + vm->hsync_len +
+		    vm->hback_porch - out_width;
 
 	i = 0;
 	if (out_height < height)
 		i++;
 	if (out_width < width)
 		i++;
-	blank = div_u64((u64)(t->hbp + t->hsw + t->hfp) * lclk, pclk);
+	blank = div_u64((u64)(vm->hback_porch + vm->hsync_len + vm->hfront_porch) *
+			lclk, pclk);
 	DSSDBG("blanking period + ppl = %llu (limit = %u)\n", blank, limits[i]);
 	if (blank <= limits[i])
 		return -EINVAL;
@@ -2231,7 +2233,7 @@ static int check_horiz_timing_omap3(unsigned long pclk, unsigned long lclk,
 }
 
 static unsigned long calc_core_clk_five_taps(unsigned long pclk,
-		const struct omap_video_timings *mgr_timings, u16 width,
+		const struct videomode *vm, u16 width,
 		u16 height, u16 out_width, u16 out_height,
 		enum omap_color_mode color_mode)
 {
@@ -2242,7 +2244,7 @@ static unsigned long calc_core_clk_five_taps(unsigned long pclk,
 		return (unsigned long) pclk;
 
 	if (height > out_height) {
-		unsigned int ppl = mgr_timings->x_res;
+		unsigned int ppl = vm->hactive;
 
 		tmp = (u64)pclk * height * out_width;
 		do_div(tmp, 2 * out_height * ppl);
@@ -2324,7 +2326,7 @@ static unsigned long calc_core_clk_44xx(unsigned long pclk, u16 width,
 }
 
 static int dispc_ovl_calc_scaling_24xx(unsigned long pclk, unsigned long lclk,
-		const struct omap_video_timings *mgr_timings,
+		const struct videomode *vm,
 		u16 width, u16 height, u16 out_width, u16 out_height,
 		enum omap_color_mode color_mode, bool *five_taps,
 		int *x_predecim, int *y_predecim, int *decim_x, int *decim_y,
@@ -2370,7 +2372,7 @@ static int dispc_ovl_calc_scaling_24xx(unsigned long pclk, unsigned long lclk,
 }
 
 static int dispc_ovl_calc_scaling_34xx(unsigned long pclk, unsigned long lclk,
-		const struct omap_video_timings *mgr_timings,
+		const struct videomode *vm,
 		u16 width, u16 height, u16 out_width, u16 out_height,
 		enum omap_color_mode color_mode, bool *five_taps,
 		int *x_predecim, int *y_predecim, int *decim_x, int *decim_y,
@@ -2392,7 +2394,7 @@ static int dispc_ovl_calc_scaling_34xx(unsigned long pclk, unsigned long lclk,
 				*five_taps = false;
 again:
 		if (*five_taps)
-			*core_clk = calc_core_clk_five_taps(pclk, mgr_timings,
+			*core_clk = calc_core_clk_five_taps(pclk, vm,
 						in_width, in_height, out_width,
 						out_height, color_mode);
 		else
@@ -2400,7 +2402,7 @@ again:
 					in_height, out_width, out_height,
 					mem_to_mem);
 
-		error = check_horiz_timing_omap3(pclk, lclk, mgr_timings,
+		error = check_horiz_timing_omap3(pclk, lclk, vm,
 				pos_x, in_width, in_height, out_width,
 				out_height, *five_taps);
 		if (error && *five_taps) {
@@ -2435,7 +2437,7 @@ again:
 		return -EINVAL;
 	}
 
-	if (check_horiz_timing_omap3(pclk, lclk, mgr_timings, pos_x, in_width,
+	if (check_horiz_timing_omap3(pclk, lclk, vm, pos_x, in_width,
 				in_height, out_width, out_height, *five_taps)) {
 			DSSERR("horizontal timing too tight\n");
 			return -EINVAL;
@@ -2455,7 +2457,7 @@ again:
 }
 
 static int dispc_ovl_calc_scaling_44xx(unsigned long pclk, unsigned long lclk,
-		const struct omap_video_timings *mgr_timings,
+		const struct videomode *vm,
 		u16 width, u16 height, u16 out_width, u16 out_height,
 		enum omap_color_mode color_mode, bool *five_taps,
 		int *x_predecim, int *y_predecim, int *decim_x, int *decim_y,
@@ -2501,7 +2503,7 @@ static int dispc_ovl_calc_scaling_44xx(unsigned long pclk, unsigned long lclk,
 
 static int dispc_ovl_calc_scaling(unsigned long pclk, unsigned long lclk,
 		enum omap_overlay_caps caps,
-		const struct omap_video_timings *mgr_timings,
+		const struct videomode *vm,
 		u16 width, u16 height, u16 out_width, u16 out_height,
 		enum omap_color_mode color_mode, bool *five_taps,
 		int *x_predecim, int *y_predecim, u16 pos_x,
@@ -2515,7 +2517,7 @@ static int dispc_ovl_calc_scaling(unsigned long pclk, unsigned long lclk,
 	if (width == out_width && height == out_height)
 		return 0;
 
-	if (!mem_to_mem && (pclk == 0 || mgr_timings->pixelclock == 0)) {
+	if (!mem_to_mem && (pclk == 0 || vm->pixelclock == 0)) {
 		DSSERR("cannot calculate scaling settings: pclk is zero\n");
 		return -EINVAL;
 	}
@@ -2551,7 +2553,7 @@ static int dispc_ovl_calc_scaling(unsigned long pclk, unsigned long lclk,
 	if (decim_y > *y_predecim || out_height > height * 8)
 		return -EINVAL;
 
-	ret = dispc.feat->calc_scaling(pclk, lclk, mgr_timings, width, height,
+	ret = dispc.feat->calc_scaling(pclk, lclk, vm, width, height,
 		out_width, out_height, color_mode, five_taps,
 		x_predecim, y_predecim, &decim_x, &decim_y, pos_x, &core_clk,
 		mem_to_mem);
@@ -2591,7 +2593,7 @@ static int dispc_ovl_setup_common(enum omap_plane plane,
 		u16 out_width, u16 out_height, enum omap_color_mode color_mode,
 		u8 rotation, bool mirror, u8 zorder, u8 pre_mult_alpha,
 		u8 global_alpha, enum omap_dss_rotation_type rotation_type,
-		bool replication, const struct omap_video_timings *mgr_timings,
+		bool replication, const struct videomode *vm,
 		bool mem_to_mem)
 {
 	bool five_taps = true;
@@ -2605,7 +2607,7 @@ static int dispc_ovl_setup_common(enum omap_plane plane,
 	u16 in_height = height;
 	u16 in_width = width;
 	int x_predecim = 1, y_predecim = 1;
-	bool ilace = mgr_timings->interlace;
+	bool ilace = !!(vm->flags & DISPLAY_FLAGS_INTERLACED);
 	unsigned long pclk = dispc_plane_pclk_rate(plane);
 	unsigned long lclk = dispc_plane_lclk_rate(plane);
 
@@ -2647,7 +2649,7 @@ static int dispc_ovl_setup_common(enum omap_plane plane,
 	if (!dss_feat_color_mode_supported(plane, color_mode))
 		return -EINVAL;
 
-	r = dispc_ovl_calc_scaling(pclk, lclk, caps, mgr_timings, in_width,
+	r = dispc_ovl_calc_scaling(pclk, lclk, caps, vm, in_width,
 			in_height, out_width, out_height, color_mode,
 			&five_taps, &x_predecim, &y_predecim, pos_x,
 			rotation_type, mem_to_mem);
@@ -2784,7 +2786,7 @@ static int dispc_ovl_setup_common(enum omap_plane plane,
 }
 
 int dispc_ovl_setup(enum omap_plane plane, const struct omap_overlay_info *oi,
-		bool replication, const struct omap_video_timings *mgr_timings,
+		bool replication, const struct videomode *vm,
 		bool mem_to_mem)
 {
 	int r;
@@ -2803,14 +2805,14 @@ int dispc_ovl_setup(enum omap_plane plane, const struct omap_overlay_info *oi,
 		oi->screen_width, oi->pos_x, oi->pos_y, oi->width, oi->height,
 		oi->out_width, oi->out_height, oi->color_mode, oi->rotation,
 		oi->mirror, oi->zorder, oi->pre_mult_alpha, oi->global_alpha,
-		oi->rotation_type, replication, mgr_timings, mem_to_mem);
+		oi->rotation_type, replication, vm, mem_to_mem);
 
 	return r;
 }
 EXPORT_SYMBOL(dispc_ovl_setup);
 
 int dispc_wb_setup(const struct omap_dss_writeback_info *wi,
-		bool mem_to_mem, const struct omap_video_timings *mgr_timings)
+		bool mem_to_mem, const struct videomode *vm)
 {
 	int r;
 	u32 l;
@@ -2819,8 +2821,8 @@ int dispc_wb_setup(const struct omap_dss_writeback_info *wi,
 	const u8 zorder = 0, global_alpha = 0;
 	const bool replication = false;
 	bool truncation;
-	int in_width = mgr_timings->x_res;
-	int in_height = mgr_timings->y_res;
+	int in_width = vm->hactive;
+	int in_height = vm->vactive;
 	enum omap_overlay_caps caps =
 		OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA;
 
@@ -2833,7 +2835,7 @@ int dispc_wb_setup(const struct omap_dss_writeback_info *wi,
 		wi->buf_width, pos_x, pos_y, in_width, in_height, wi->width,
 		wi->height, wi->color_mode, wi->rotation, wi->mirror, zorder,
 		wi->pre_mult_alpha, global_alpha, wi->rotation_type,
-		replication, mgr_timings, mem_to_mem);
+		replication, vm, mem_to_mem);
 
 	switch (wi->color_mode) {
 	case OMAP_DSS_COLOR_RGB16:
@@ -2867,8 +2869,8 @@ int dispc_wb_setup(const struct omap_dss_writeback_info *wi,
 	} else {
 		int wbdelay;
 
-		wbdelay = min(mgr_timings->vfp + mgr_timings->vsw +
-			mgr_timings->vbp, 255);
+		wbdelay = min(vm->vfront_porch +
+			      vm->vsync_len + vm->vback_porch, (u32)255);
 
 		/* WBDELAYCOUNT */
 		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES2(plane), wbdelay, 7, 0);
@@ -3093,10 +3095,10 @@ static bool _dispc_mgr_size_ok(u16 width, u16 height)
 		height <= dispc.feat->mgr_height_max;
 }
 
-static bool _dispc_lcd_timings_ok(int hsw, int hfp, int hbp,
+static bool _dispc_lcd_timings_ok(int hsync_len, int hfp, int hbp,
 		int vsw, int vfp, int vbp)
 {
-	if (hsw < 1 || hsw > dispc.feat->sw_max ||
+	if (hsync_len < 1 || hsync_len > dispc.feat->sw_max ||
 			hfp < 1 || hfp > dispc.feat->hp_max ||
 			hbp < 1 || hbp > dispc.feat->hp_max ||
 			vsw < 1 || vsw > dispc.feat->sw_max ||
@@ -3110,113 +3112,77 @@ static bool _dispc_mgr_pclk_ok(enum omap_channel channel,
 		unsigned long pclk)
 {
 	if (dss_mgr_is_lcd(channel))
-		return pclk <= dispc.feat->max_lcd_pclk ? true : false;
+		return pclk <= dispc.feat->max_lcd_pclk;
 	else
-		return pclk <= dispc.feat->max_tv_pclk ? true : false;
+		return pclk <= dispc.feat->max_tv_pclk;
 }
 
-bool dispc_mgr_timings_ok(enum omap_channel channel,
-		const struct omap_video_timings *timings)
+bool dispc_mgr_timings_ok(enum omap_channel channel, const struct videomode *vm)
 {
-	if (!_dispc_mgr_size_ok(timings->x_res, timings->y_res))
+	if (!_dispc_mgr_size_ok(vm->hactive, vm->vactive))
 		return false;
 
-	if (!_dispc_mgr_pclk_ok(channel, timings->pixelclock))
+	if (!_dispc_mgr_pclk_ok(channel, vm->pixelclock))
 		return false;
 
 	if (dss_mgr_is_lcd(channel)) {
 		/* TODO: OMAP4+ supports interlace for LCD outputs */
-		if (timings->interlace)
+		if (vm->flags & DISPLAY_FLAGS_INTERLACED)
 			return false;
 
-		if (!_dispc_lcd_timings_ok(timings->hsw, timings->hfp,
-				timings->hbp, timings->vsw, timings->vfp,
-				timings->vbp))
+		if (!_dispc_lcd_timings_ok(vm->hsync_len,
+				vm->hfront_porch, vm->hback_porch,
+				vm->vsync_len, vm->vfront_porch,
+				vm->vback_porch))
 			return false;
 	}
 
 	return true;
 }
 
-static void _dispc_mgr_set_lcd_timings(enum omap_channel channel, int hsw,
-		int hfp, int hbp, int vsw, int vfp, int vbp,
-		enum omap_dss_signal_level vsync_level,
-		enum omap_dss_signal_level hsync_level,
-		enum omap_dss_signal_edge data_pclk_edge,
-		enum omap_dss_signal_level de_level,
-		enum omap_dss_signal_edge sync_pclk_edge)
-
+static void _dispc_mgr_set_lcd_timings(enum omap_channel channel,
+				       const struct videomode *vm)
 {
 	u32 timing_h, timing_v, l;
 	bool onoff, rf, ipc, vs, hs, de;
 
-	timing_h = FLD_VAL(hsw-1, dispc.feat->sw_start, 0) |
-			FLD_VAL(hfp-1, dispc.feat->fp_start, 8) |
-			FLD_VAL(hbp-1, dispc.feat->bp_start, 20);
-	timing_v = FLD_VAL(vsw-1, dispc.feat->sw_start, 0) |
-			FLD_VAL(vfp, dispc.feat->fp_start, 8) |
-			FLD_VAL(vbp, dispc.feat->bp_start, 20);
+	timing_h = FLD_VAL(vm->hsync_len - 1, dispc.feat->sw_start, 0) |
+		   FLD_VAL(vm->hfront_porch - 1, dispc.feat->fp_start, 8) |
+		   FLD_VAL(vm->hback_porch - 1, dispc.feat->bp_start, 20);
+	timing_v = FLD_VAL(vm->vsync_len - 1, dispc.feat->sw_start, 0) |
+		   FLD_VAL(vm->vfront_porch, dispc.feat->fp_start, 8) |
+		   FLD_VAL(vm->vback_porch, dispc.feat->bp_start, 20);
 
 	dispc_write_reg(DISPC_TIMING_H(channel), timing_h);
 	dispc_write_reg(DISPC_TIMING_V(channel), timing_v);
 
-	switch (vsync_level) {
-	case OMAPDSS_SIG_ACTIVE_LOW:
-		vs = true;
-		break;
-	case OMAPDSS_SIG_ACTIVE_HIGH:
+	if (vm->flags & DISPLAY_FLAGS_VSYNC_HIGH)
 		vs = false;
-		break;
-	default:
-		BUG();
-	}
+	else
+		vs = true;
 
-	switch (hsync_level) {
-	case OMAPDSS_SIG_ACTIVE_LOW:
-		hs = true;
-		break;
-	case OMAPDSS_SIG_ACTIVE_HIGH:
+	if (vm->flags & DISPLAY_FLAGS_HSYNC_HIGH)
 		hs = false;
-		break;
-	default:
-		BUG();
-	}
+	else
+		hs = true;
 
-	switch (de_level) {
-	case OMAPDSS_SIG_ACTIVE_LOW:
-		de = true;
-		break;
-	case OMAPDSS_SIG_ACTIVE_HIGH:
+	if (vm->flags & DISPLAY_FLAGS_DE_HIGH)
 		de = false;
-		break;
-	default:
-		BUG();
-	}
+	else
+		de = true;
 
-	switch (data_pclk_edge) {
-	case OMAPDSS_DRIVE_SIG_RISING_EDGE:
+	if (vm->flags & DISPLAY_FLAGS_PIXDATA_POSEDGE)
 		ipc = false;
-		break;
-	case OMAPDSS_DRIVE_SIG_FALLING_EDGE:
+	else
 		ipc = true;
-		break;
-	default:
-		BUG();
-	}
 
 	/* always use the 'rf' setting */
 	onoff = true;
 
-	switch (sync_pclk_edge) {
-	case OMAPDSS_DRIVE_SIG_FALLING_EDGE:
-		rf = false;
-		break;
-	case OMAPDSS_DRIVE_SIG_RISING_EDGE:
+	if (vm->flags & DISPLAY_FLAGS_SYNC_POSEDGE)
 		rf = true;
-		break;
-	default:
-		BUG();
-	}
+	else
+		rf = false;
 
 	l = FLD_VAL(onoff, 17, 17) |
 		FLD_VAL(rf, 16, 16) |
@@ -3253,13 +3219,13 @@ static void _dispc_mgr_set_lcd_timings(enum omap_channel channel, int hsw,
 
 /* change name to mode? */
 void dispc_mgr_set_timings(enum omap_channel channel,
-		const struct omap_video_timings *timings)
+			   const struct videomode *vm)
 {
 	unsigned xtot, ytot;
 	unsigned long ht, vt;
-	struct omap_video_timings t = *timings;
+	struct videomode t = *vm;
 
-	DSSDBG("channel %d xres %u yres %u\n", channel, t.x_res, t.y_res);
+	DSSDBG("channel %d xres %u yres %u\n", channel, t.hactive, t.vactive);
 
 	if (!dispc_mgr_timings_ok(channel, &t)) {
 		BUG();
@@ -3267,34 +3233,37 @@ void dispc_mgr_set_timings(enum omap_channel channel,
 	}
 
 	if (dss_mgr_is_lcd(channel)) {
-		_dispc_mgr_set_lcd_timings(channel, t.hsw, t.hfp, t.hbp, t.vsw,
-				t.vfp, t.vbp, t.vsync_level, t.hsync_level,
-				t.data_pclk_edge, t.de_level, t.sync_pclk_edge);
+		_dispc_mgr_set_lcd_timings(channel, &t);
 
-		xtot = t.x_res + t.hfp + t.hsw + t.hbp;
-		ytot = t.y_res + t.vfp + t.vsw + t.vbp;
+		xtot = t.hactive + t.hfront_porch + t.hsync_len + t.hback_porch;
+		ytot = t.vactive + t.vfront_porch + t.vsync_len + t.vback_porch;
 
-		ht = timings->pixelclock / xtot;
-		vt = timings->pixelclock / xtot / ytot;
+		ht = vm->pixelclock / xtot;
+		vt = vm->pixelclock / xtot / ytot;
 
-		DSSDBG("pck %u\n", timings->pixelclock);
-		DSSDBG("hsw %d hfp %d hbp %d vsw %d vfp %d vbp %d\n",
-			t.hsw, t.hfp, t.hbp, t.vsw, t.vfp, t.vbp);
+		DSSDBG("pck %lu\n", vm->pixelclock);
+		DSSDBG("hsync_len %d hfp %d hbp %d vsw %d vfp %d vbp %d\n",
+			t.hsync_len, t.hfront_porch, t.hback_porch,
+			t.vsync_len, t.vfront_porch, t.vback_porch);
 		DSSDBG("vsync_level %d hsync_level %d data_pclk_edge %d de_level %d sync_pclk_edge %d\n",
-			t.vsync_level, t.hsync_level, t.data_pclk_edge,
-			t.de_level, t.sync_pclk_edge);
+			!!(t.flags & DISPLAY_FLAGS_VSYNC_HIGH),
+			!!(t.flags & DISPLAY_FLAGS_HSYNC_HIGH),
+			!!(t.flags & DISPLAY_FLAGS_PIXDATA_POSEDGE),
+			!!(t.flags & DISPLAY_FLAGS_DE_HIGH),
+			!!(t.flags & DISPLAY_FLAGS_SYNC_POSEDGE));
 
 		DSSDBG("hsync %luHz, vsync %luHz\n", ht, vt);
 	} else {
-		if (t.interlace)
-			t.y_res /= 2;
+		if (t.flags & DISPLAY_FLAGS_INTERLACED)
+			t.vactive /= 2;
 
 		if (dispc.feat->supports_double_pixel)
-			REG_FLD_MOD(DISPC_CONTROL, t.double_pixel ? 1 : 0,
-				19, 17);
+			REG_FLD_MOD(DISPC_CONTROL,
+				    !!(t.flags & DISPLAY_FLAGS_DOUBLECLK),
+				    19, 17);
 	}
 
-	dispc_mgr_set_size(channel, t.x_res, t.y_res);
+	dispc_mgr_set_size(channel, t.hactive, t.vactive);
 }
 EXPORT_SYMBOL(dispc_mgr_set_timings);
 
@@ -4214,23 +4183,20 @@ EXPORT_SYMBOL(dispc_free_irq);
  */
 
 static const struct dispc_errata_i734_data {
-	struct omap_video_timings timings;
+	struct videomode vm;
 	struct omap_overlay_info ovli;
 	struct omap_overlay_manager_info mgri;
 	struct dss_lcd_mgr_config lcd_conf;
 } i734 = {
-	.timings = {
-		.x_res = 8, .y_res = 1,
+	.vm = {
+		.hactive = 8, .vactive = 1,
 		.pixelclock = 16000000,
-		.hsw = 8, .hfp = 4, .hbp = 4,
-		.vsw = 1, .vfp = 1, .vbp = 1,
-		.vsync_level = OMAPDSS_SIG_ACTIVE_LOW,
-		.hsync_level = OMAPDSS_SIG_ACTIVE_LOW,
-		.interlace = false,
-		.data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE,
-		.de_level = OMAPDSS_SIG_ACTIVE_HIGH,
-		.sync_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE,
-		.double_pixel = false,
+		.hsync_len = 8, .hfront_porch = 4, .hback_porch = 4,
+		.vsync_len = 1, .vfront_porch = 1, .vback_porch = 1,
+
+		.flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
+			 DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_POSEDGE |
+			 DISPLAY_FLAGS_PIXDATA_POSEDGE,
 	},
 	.ovli = {
 		.screen_width = 1,
@@ -4320,7 +4286,7 @@ static void dispc_errata_i734_wa(void)
 
 	/* Setup and enable GFX plane */
 	dispc_ovl_set_channel_out(OMAP_DSS_GFX, OMAP_DSS_CHANNEL_LCD);
-	dispc_ovl_setup(OMAP_DSS_GFX, &ovli, false, &i734.timings, false);
+	dispc_ovl_setup(OMAP_DSS_GFX, &ovli, false, &i734.vm, false);
 	dispc_ovl_enable(OMAP_DSS_GFX, true);
 
 	/* Set up and enable display manager for LCD1 */
@@ -4328,7 +4294,7 @@ static void dispc_errata_i734_wa(void)
 	dispc_calc_clock_rates(dss_get_dispc_clk_rate(),
 			       &lcd_conf.clock_info);
 	dispc_mgr_set_lcd_config(OMAP_DSS_CHANNEL_LCD, &lcd_conf);
-	dispc_mgr_set_timings(OMAP_DSS_CHANNEL_LCD, &i734.timings);
+	dispc_mgr_set_timings(OMAP_DSS_CHANNEL_LCD, &i734.vm);
 
 	dispc_clear_irqstatus(framedone_irq);
 
diff --git a/drivers/gpu/drm/omapdrm/dss/display.c b/drivers/gpu/drm/omapdrm/dss/display.c
index 8dcdd7cf9937..425a5a8dff8b 100644
--- a/drivers/gpu/drm/omapdrm/dss/display.c
+++ b/drivers/gpu/drm/omapdrm/dss/display.c
@@ -35,8 +35,8 @@
 void omapdss_default_get_resolution(struct omap_dss_device *dssdev,
 			u16 *xres, u16 *yres)
 {
-	*xres = dssdev->panel.timings.x_res;
-	*yres = dssdev->panel.timings.y_res;
+	*xres = dssdev->panel.vm.hactive;
+	*yres = dssdev->panel.vm.vactive;
 }
 EXPORT_SYMBOL(omapdss_default_get_resolution);
 
@@ -72,9 +72,9 @@ int omapdss_default_get_recommended_bpp(struct omap_dss_device *dssdev)
 EXPORT_SYMBOL(omapdss_default_get_recommended_bpp);
 
 void omapdss_default_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				 struct videomode *vm)
 {
-	*timings = dssdev->panel.timings;
+	*vm = dssdev->panel.vm;
 }
 EXPORT_SYMBOL(omapdss_default_get_timings);
 
@@ -217,73 +217,3 @@ struct omap_dss_device *omap_dss_find_device(void *data,
 	return NULL;
 }
 EXPORT_SYMBOL(omap_dss_find_device);
-
-void videomode_to_omap_video_timings(const struct videomode *vm,
-		struct omap_video_timings *ovt)
-{
-	memset(ovt, 0, sizeof(*ovt));
-
-	ovt->pixelclock = vm->pixelclock;
-	ovt->x_res = vm->hactive;
-	ovt->hbp = vm->hback_porch;
-	ovt->hfp = vm->hfront_porch;
-	ovt->hsw = vm->hsync_len;
-	ovt->y_res = vm->vactive;
-	ovt->vbp = vm->vback_porch;
-	ovt->vfp = vm->vfront_porch;
-	ovt->vsw = vm->vsync_len;
-
-	ovt->vsync_level = vm->flags & DISPLAY_FLAGS_VSYNC_HIGH ?
-		OMAPDSS_SIG_ACTIVE_HIGH :
-		OMAPDSS_SIG_ACTIVE_LOW;
-	ovt->hsync_level = vm->flags & DISPLAY_FLAGS_HSYNC_HIGH ?
-		OMAPDSS_SIG_ACTIVE_HIGH :
-		OMAPDSS_SIG_ACTIVE_LOW;
-	ovt->de_level = vm->flags & DISPLAY_FLAGS_DE_HIGH ?
-		OMAPDSS_SIG_ACTIVE_HIGH :
-		OMAPDSS_SIG_ACTIVE_LOW;
-	ovt->data_pclk_edge = vm->flags & DISPLAY_FLAGS_PIXDATA_POSEDGE ?
-		OMAPDSS_DRIVE_SIG_RISING_EDGE :
-		OMAPDSS_DRIVE_SIG_FALLING_EDGE;
-
-	ovt->sync_pclk_edge = ovt->data_pclk_edge;
-}
-EXPORT_SYMBOL(videomode_to_omap_video_timings);
-
-void omap_video_timings_to_videomode(const struct omap_video_timings *ovt,
-		struct videomode *vm)
-{
-	memset(vm, 0, sizeof(*vm));
-
-	vm->pixelclock = ovt->pixelclock;
-
-	vm->hactive = ovt->x_res;
-	vm->hback_porch = ovt->hbp;
-	vm->hfront_porch = ovt->hfp;
-	vm->hsync_len = ovt->hsw;
-	vm->vactive = ovt->y_res;
-	vm->vback_porch = ovt->vbp;
-	vm->vfront_porch = ovt->vfp;
-	vm->vsync_len = ovt->vsw;
-
-	if (ovt->hsync_level == OMAPDSS_SIG_ACTIVE_HIGH)
-		vm->flags |= DISPLAY_FLAGS_HSYNC_HIGH;
-	else
-		vm->flags |= DISPLAY_FLAGS_HSYNC_LOW;
-
-	if (ovt->vsync_level == OMAPDSS_SIG_ACTIVE_HIGH)
-		vm->flags |= DISPLAY_FLAGS_VSYNC_HIGH;
-	else
-		vm->flags |= DISPLAY_FLAGS_VSYNC_LOW;
-
-	if (ovt->de_level == OMAPDSS_SIG_ACTIVE_HIGH)
-		vm->flags |= DISPLAY_FLAGS_DE_HIGH;
-	else
-		vm->flags |= DISPLAY_FLAGS_DE_LOW;
-
-	if (ovt->data_pclk_edge == OMAPDSS_DRIVE_SIG_RISING_EDGE)
-		vm->flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE;
-	else
-		vm->flags |= DISPLAY_FLAGS_PIXDATA_NEGEDGE;
-}
-EXPORT_SYMBOL(omap_video_timings_to_videomode);
diff --git a/drivers/gpu/drm/omapdrm/dss/dpi.c b/drivers/gpu/drm/omapdrm/dss/dpi.c
index b268295b76cf..e75162d26ac0 100644
--- a/drivers/gpu/drm/omapdrm/dss/dpi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dpi.c
@@ -47,7 +47,7 @@ struct dpi_data {
 
 	struct mutex lock;
 
-	struct omap_video_timings timings;
+	struct videomode vm;
 	struct dss_lcd_mgr_config mgr_config;
 	int data_lines;
 
@@ -333,31 +333,31 @@ static int dpi_set_mode(struct dpi_data *dpi)
 {
 	struct omap_dss_device *out = &dpi->output;
 	enum omap_channel channel = out->dispc_channel;
-	struct omap_video_timings *t = &dpi->timings;
+	struct videomode *vm = &dpi->vm;
 	int lck_div = 0, pck_div = 0;
 	unsigned long fck = 0;
 	unsigned long pck;
 	int r = 0;
 
 	if (dpi->pll)
-		r = dpi_set_pll_clk(dpi, channel, t->pixelclock, &fck,
+		r = dpi_set_pll_clk(dpi, channel, vm->pixelclock, &fck,
 				&lck_div, &pck_div);
 	else
-		r = dpi_set_dispc_clk(dpi, t->pixelclock, &fck,
+		r = dpi_set_dispc_clk(dpi, vm->pixelclock, &fck,
 				&lck_div, &pck_div);
 	if (r)
 		return r;
 
 	pck = fck / lck_div / pck_div;
 
-	if (pck != t->pixelclock) {
-		DSSWARN("Could not find exact pixel clock. Requested %d Hz, got %lu Hz\n",
-			t->pixelclock, pck);
+	if (pck != vm->pixelclock) {
+		DSSWARN("Could not find exact pixel clock. Requested %lu Hz, got %lu Hz\n",
+			vm->pixelclock, pck);
 
-		t->pixelclock = pck;
+		vm->pixelclock = pck;
 	}
 
-	dss_mgr_set_timings(channel, t);
+	dss_mgr_set_timings(channel, vm);
 
 	return 0;
 }
@@ -476,7 +476,7 @@ static void dpi_display_disable(struct omap_dss_device *dssdev)
 }
 
 static void dpi_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			    struct videomode *vm)
 {
 	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
 
@@ -484,25 +484,25 @@ static void dpi_set_timings(struct omap_dss_device *dssdev,
 
 	mutex_lock(&dpi->lock);
 
-	dpi->timings = *timings;
+	dpi->vm = *vm;
 
 	mutex_unlock(&dpi->lock);
 }
 
 static void dpi_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			    struct videomode *vm)
 {
 	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
 
 	mutex_lock(&dpi->lock);
 
-	*timings = dpi->timings;
+	*vm = dpi->vm;
 
 	mutex_unlock(&dpi->lock);
 }
 
 static int dpi_check_timings(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings)
+			     struct videomode *vm)
 {
 	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
 	enum omap_channel channel = dpi->output.dispc_channel;
@@ -512,23 +512,23 @@ static int dpi_check_timings(struct omap_dss_device *dssdev,
 	struct dpi_clk_calc_ctx ctx;
 	bool ok;
 
-	if (timings->x_res % 8 != 0)
+	if (vm->hactive % 8 != 0)
 		return -EINVAL;
 
-	if (!dispc_mgr_timings_ok(channel, timings))
+	if (!dispc_mgr_timings_ok(channel, vm))
 		return -EINVAL;
 
-	if (timings->pixelclock == 0)
+	if (vm->pixelclock == 0)
 		return -EINVAL;
 
 	if (dpi->pll) {
-		ok = dpi_pll_clk_calc(dpi, timings->pixelclock, &ctx);
+		ok = dpi_pll_clk_calc(dpi, vm->pixelclock, &ctx);
 		if (!ok)
 			return -EINVAL;
 
 		fck = ctx.pll_cinfo.clkout[ctx.clkout_idx];
 	} else {
-		ok = dpi_dss_clk_calc(timings->pixelclock, &ctx);
+		ok = dpi_dss_clk_calc(vm->pixelclock, &ctx);
 		if (!ok)
 			return -EINVAL;
 
@@ -540,7 +540,7 @@ static int dpi_check_timings(struct omap_dss_device *dssdev,
 
 	pck = fck / lck_div / pck_div;
 
-	timings->pixelclock = pck;
+	vm->pixelclock = pck;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c
index e1be5e795cd8..f060bda31235 100644
--- a/drivers/gpu/drm/omapdrm/dss/dsi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dsi.c
@@ -289,7 +289,7 @@ struct dsi_clk_calc_ctx {
 	struct dss_pll_clock_info dsi_cinfo;
 	struct dispc_clock_info dispc_cinfo;
 
-	struct omap_video_timings dispc_vm;
+	struct videomode vm;
 	struct omap_dss_dsi_videomode_timings dsi_vm;
 };
 
@@ -383,7 +383,7 @@ struct dsi_data {
 	unsigned scp_clk_refcount;
 
 	struct dss_lcd_mgr_config mgr_config;
-	struct omap_video_timings timings;
+	struct videomode vm;
 	enum omap_dss_dsi_pixel_format pix_fmt;
 	enum omap_dss_dsi_mode mode;
 	struct omap_dss_dsi_videomode_timings vm_timings;
@@ -3321,12 +3321,12 @@ static void dsi_config_vp_num_line_buffers(struct platform_device *dsidev)
 
 	if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) {
 		int bpp = dsi_get_pixel_size(dsi->pix_fmt);
-		struct omap_video_timings *timings = &dsi->timings;
+		struct videomode *vm = &dsi->vm;
 		/*
 		 * Don't use line buffers if width is greater than the video
 		 * port's line buffer size
 		 */
-		if (dsi->line_buffer_size <= timings->x_res * bpp / 8)
+		if (dsi->line_buffer_size <= vm->hactive * bpp / 8)
 			num_line_buffers = 0;
 		else
 			num_line_buffers = 2;
@@ -3453,7 +3453,7 @@ static void dsi_config_cmd_mode_interleaving(struct platform_device *dsidev)
 	int ddr_clk_pre, ddr_clk_post, enter_hs_mode_lat, exit_hs_mode_lat;
 	int tclk_trail, ths_exit, exiths_clk;
 	bool ddr_alwon;
-	struct omap_video_timings *timings = &dsi->timings;
+	struct videomode *vm = &dsi->vm;
 	int bpp = dsi_get_pixel_size(dsi->pix_fmt);
 	int ndl = dsi->num_lanes_used - 1;
 	int dsi_fclk_hsdiv = dsi->user_dsi_cinfo.mX[HSDIV_DSI] + 1;
@@ -3494,7 +3494,7 @@ static void dsi_config_cmd_mode_interleaving(struct platform_device *dsidev)
 
 	exiths_clk = ths_exit + tclk_trail;
 
-	width_bytes = DIV_ROUND_UP(timings->x_res * bpp, 8);
+	width_bytes = DIV_ROUND_UP(vm->hactive * bpp, 8);
 	bllp = hbp + hfp + hsa + DIV_ROUND_UP(width_bytes + 6, ndl);
 
 	if (!hsa_blanking_mode) {
@@ -3705,7 +3705,7 @@ static void dsi_proto_timings(struct platform_device *dsidev)
 		int vbp = dsi->vm_timings.vbp;
 		int window_sync = dsi->vm_timings.window_sync;
 		bool hsync_end;
-		struct omap_video_timings *timings = &dsi->timings;
+		struct videomode *vm = &dsi->vm;
 		int bpp = dsi_get_pixel_size(dsi->pix_fmt);
 		int tl, t_he, width_bytes;
 
@@ -3713,7 +3713,7 @@ static void dsi_proto_timings(struct platform_device *dsidev)
 		t_he = hsync_end ?
 			((hsa == 0 && ndl == 3) ? 1 : DIV_ROUND_UP(4, ndl)) : 0;
 
-		width_bytes = DIV_ROUND_UP(timings->x_res * bpp, 8);
+		width_bytes = DIV_ROUND_UP(vm->hactive * bpp, 8);
 
 		/* TL = t_HS + HSA + t_HE + HFP + ceil((WC + 6) / NDL) + HBP */
 		tl = DIV_ROUND_UP(4, ndl) + (hsync_end ? hsa : 0) + t_he + hfp +
@@ -3722,7 +3722,7 @@ static void dsi_proto_timings(struct platform_device *dsidev)
 		DSSDBG("HBP: %d, HFP: %d, HSA: %d, TL: %d TXBYTECLKHS\n", hbp,
 			hfp, hsync_end ? hsa : 0, tl);
 		DSSDBG("VBP: %d, VFP: %d, VSA: %d, VACT: %d lines\n", vbp, vfp,
-			vsa, timings->y_res);
+			vsa, vm->vactive);
 
 		r = dsi_read_reg(dsidev, DSI_VM_TIMING1);
 		r = FLD_MOD(r, hbp, 11, 0);	/* HBP */
@@ -3738,7 +3738,7 @@ static void dsi_proto_timings(struct platform_device *dsidev)
 		dsi_write_reg(dsidev, DSI_VM_TIMING2, r);
 
 		r = dsi_read_reg(dsidev, DSI_VM_TIMING3);
-		r = FLD_MOD(r, timings->y_res, 14, 0);	/* VACT */
+		r = FLD_MOD(r, vm->vactive, 14, 0);	/* VACT */
 		r = FLD_MOD(r, tl, 31, 16);		/* TL */
 		dsi_write_reg(dsidev, DSI_VM_TIMING3, r);
 	}
@@ -3856,7 +3856,7 @@ static int dsi_enable_video_output(struct omap_dss_device *dssdev, int channel)
 		/* MODE, 1 = video mode */
 		REG_FLD_MOD(dsidev, DSI_VC_CTRL(channel), 1, 4, 4);
 
-		word_count = DIV_ROUND_UP(dsi->timings.x_res * bpp, 8);
+		word_count = DIV_ROUND_UP(dsi->vm.hactive * bpp, 8);
 
 		dsi_vc_write_long_header(dsidev, channel, data_type,
 				word_count, 0);
@@ -3918,8 +3918,8 @@ static void dsi_update_screen_dispc(struct platform_device *dsidev)
 	int r;
 	const unsigned channel = dsi->update_channel;
 	const unsigned line_buf_size = dsi->line_buffer_size;
-	u16 w = dsi->timings.x_res;
-	u16 h = dsi->timings.y_res;
+	u16 w = dsi->vm.hactive;
+	u16 h = dsi->vm.vactive;
 
 	DSSDBG("dsi_update_screen_dispc(%dx%d)\n", w, h);
 
@@ -3969,7 +3969,7 @@ static void dsi_update_screen_dispc(struct platform_device *dsidev)
 		msecs_to_jiffies(250));
 	BUG_ON(r == 0);
 
-	dss_mgr_set_timings(dispc_channel, &dsi->timings);
+	dss_mgr_set_timings(dispc_channel, &dsi->vm);
 
 	dss_mgr_start_update(dispc_channel);
 
@@ -4056,8 +4056,8 @@ static int dsi_update(struct omap_dss_device *dssdev, int channel,
 	dsi->framedone_callback = callback;
 	dsi->framedone_data = data;
 
-	dw = dsi->timings.x_res;
-	dh = dsi->timings.y_res;
+	dw = dsi->vm.hactive;
+	dh = dsi->vm.vactive;
 
 #ifdef DSI_PERF_MEASURE
 	dsi->update_bytes = dw * dh *
@@ -4120,16 +4120,21 @@ static int dsi_display_init_dispc(struct platform_device *dsidev,
 
 	/*
 	 * override interlace, logic level and edge related parameters in
-	 * omap_video_timings with default values
+	 * videomode with default values
 	 */
-	dsi->timings.interlace = false;
-	dsi->timings.hsync_level = OMAPDSS_SIG_ACTIVE_HIGH;
-	dsi->timings.vsync_level = OMAPDSS_SIG_ACTIVE_HIGH;
-	dsi->timings.data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
-	dsi->timings.de_level = OMAPDSS_SIG_ACTIVE_HIGH;
-	dsi->timings.sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE;
-
-	dss_mgr_set_timings(channel, &dsi->timings);
+	dsi->vm.flags &= ~DISPLAY_FLAGS_INTERLACED;
+	dsi->vm.flags &= ~DISPLAY_FLAGS_HSYNC_LOW;
+	dsi->vm.flags |= DISPLAY_FLAGS_HSYNC_HIGH;
+	dsi->vm.flags &= ~DISPLAY_FLAGS_VSYNC_LOW;
+	dsi->vm.flags |= DISPLAY_FLAGS_VSYNC_HIGH;
+	dsi->vm.flags &= ~DISPLAY_FLAGS_PIXDATA_NEGEDGE;
+	dsi->vm.flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE;
+	dsi->vm.flags &= ~DISPLAY_FLAGS_DE_LOW;
+	dsi->vm.flags |= DISPLAY_FLAGS_DE_HIGH;
+	dsi->vm.flags &= ~DISPLAY_FLAGS_SYNC_POSEDGE;
+	dsi->vm.flags |= DISPLAY_FLAGS_SYNC_NEGEDGE;
+
+	dss_mgr_set_timings(channel, &dsi->vm);
 
 	r = dsi_configure_dispc_clocks(dsidev);
 	if (r)
@@ -4331,7 +4336,7 @@ static void print_dsi_vm(const char *str,
 
 	wc = DIV_ROUND_UP(t->hact * t->bitspp, 8);
 	pps = DIV_ROUND_UP(wc + 6, t->ndl); /* pixel packet size */
-	bl = t->hss + t->hsa + t->hse + t->hbp + t->hfp;
+	bl = t->hss + t->hsa + t->hse + t->hbp + t->hfront_porch;
 	tot = bl + pps;
 
 #define TO_DSI_T(x) ((u32)div64_u64((u64)x * 1000000000llu, byteclk))
@@ -4340,14 +4345,14 @@ static void print_dsi_vm(const char *str,
 			"%u/%u/%u/%u/%u/%u = %u + %u = %u\n",
 			str,
 			byteclk,
-			t->hss, t->hsa, t->hse, t->hbp, pps, t->hfp,
+			t->hss, t->hsa, t->hse, t->hbp, pps, t->hfront_porch,
 			bl, pps, tot,
 			TO_DSI_T(t->hss),
 			TO_DSI_T(t->hsa),
 			TO_DSI_T(t->hse),
 			TO_DSI_T(t->hbp),
 			TO_DSI_T(pps),
-			TO_DSI_T(t->hfp),
+			TO_DSI_T(t->hfront_porch),
 
 			TO_DSI_T(bl),
 			TO_DSI_T(pps),
@@ -4356,13 +4361,13 @@ static void print_dsi_vm(const char *str,
 #undef TO_DSI_T
 }
 
-static void print_dispc_vm(const char *str, const struct omap_video_timings *t)
+static void print_dispc_vm(const char *str, const struct videomode *vm)
 {
-	unsigned long pck = t->pixelclock;
+	unsigned long pck = vm->pixelclock;
 	int hact, bl, tot;
 
-	hact = t->x_res;
-	bl = t->hsw + t->hbp + t->hfp;
+	hact = vm->hactive;
+	bl = vm->hsync_len + vm->hbp + vm->hfront_porch;
 	tot = hact + bl;
 
 #define TO_DISPC_T(x) ((u32)div64_u64((u64)x * 1000000000llu, pck))
@@ -4371,12 +4376,12 @@ static void print_dispc_vm(const char *str, const struct omap_video_timings *t)
 			"%u/%u/%u/%u = %u + %u = %u\n",
 			str,
 			pck,
-			t->hsw, t->hbp, hact, t->hfp,
+			vm->hsync_len, vm->hbp, hact, vm->hfront_porch,
 			bl, hact, tot,
-			TO_DISPC_T(t->hsw),
-			TO_DISPC_T(t->hbp),
+			TO_DISPC_T(vm->hsync_len),
+			TO_DISPC_T(vm->hbp),
 			TO_DISPC_T(hact),
-			TO_DISPC_T(t->hfp),
+			TO_DISPC_T(vm->hfront_porch),
 			TO_DISPC_T(bl),
 			TO_DISPC_T(hact),
 			TO_DISPC_T(tot));
@@ -4387,7 +4392,7 @@ static void print_dispc_vm(const char *str, const struct omap_video_timings *t)
 static void print_dsi_dispc_vm(const char *str,
 		const struct omap_dss_dsi_videomode_timings *t)
 {
-	struct omap_video_timings vm = { 0 };
+	struct videomode vm = { 0 };
 	unsigned long byteclk = t->hsclk / 4;
 	unsigned long pck;
 	u64 dsi_tput;
@@ -4396,13 +4401,13 @@ static void print_dsi_dispc_vm(const char *str,
 	dsi_tput = (u64)byteclk * t->ndl * 8;
 	pck = (u32)div64_u64(dsi_tput, t->bitspp);
 	dsi_hact = DIV_ROUND_UP(DIV_ROUND_UP(t->hact * t->bitspp, 8) + 6, t->ndl);
-	dsi_htot = t->hss + t->hsa + t->hse + t->hbp + dsi_hact + t->hfp;
+	dsi_htot = t->hss + t->hsa + t->hse + t->hbp + dsi_hact + t->hfront_porch;
 
 	vm.pixelclock = pck;
-	vm.hsw = div64_u64((u64)(t->hsa + t->hse) * pck, byteclk);
+	vm.hsync_len = div64_u64((u64)(t->hsa + t->hse) * pck, byteclk);
 	vm.hbp = div64_u64((u64)t->hbp * pck, byteclk);
-	vm.hfp = div64_u64((u64)t->hfp * pck, byteclk);
-	vm.x_res = t->hact;
+	vm.hfront_porch = div64_u64((u64)t->hfront_porch * pck, byteclk);
+	vm.hactive = t->hact;
 
 	print_dispc_vm(str, &vm);
 }
@@ -4412,19 +4417,19 @@ static bool dsi_cm_calc_dispc_cb(int lckd, int pckd, unsigned long lck,
 		unsigned long pck, void *data)
 {
 	struct dsi_clk_calc_ctx *ctx = data;
-	struct omap_video_timings *t = &ctx->dispc_vm;
+	struct videomode *vm = &ctx->vm;
 
 	ctx->dispc_cinfo.lck_div = lckd;
 	ctx->dispc_cinfo.pck_div = pckd;
 	ctx->dispc_cinfo.lck = lck;
 	ctx->dispc_cinfo.pck = pck;
 
-	*t = *ctx->config->timings;
-	t->pixelclock = pck;
-	t->x_res = ctx->config->timings->x_res;
-	t->y_res = ctx->config->timings->y_res;
-	t->hsw = t->hfp = t->hbp = t->vsw = 1;
-	t->vfp = t->vbp = 0;
+	*vm = *ctx->config->vm;
+	vm->pixelclock = pck;
+	vm->hactive = ctx->config->vm->hactive;
+	vm->vactive = ctx->config->vm->vactive;
+	vm->hsync_len = vm->hfront_porch = vm->hback_porch = vm->vsync_len = 1;
+	vm->vfront_porch = vm->vback_porch = 0;
 
 	return true;
 }
@@ -4475,7 +4480,7 @@ static bool dsi_cm_calc(struct dsi_data *dsi,
 	 * especially as we go to LP between each pixel packet due to HW
 	 * "feature". So let's just estimate very roughly and multiply by 1.5.
 	 */
-	pck = cfg->timings->pixelclock;
+	pck = cfg->vm->pixelclock;
 	pck = pck * 3 / 2;
 	txbyteclk = pck * bitspp / 8 / ndl;
 
@@ -4510,14 +4515,14 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx)
 	int dispc_htot, dispc_hbl; /* pixels */
 	int dsi_htot, dsi_hact, dsi_hbl, hss, hse; /* byteclks */
 	int hfp, hsa, hbp;
-	const struct omap_video_timings *req_vm;
-	struct omap_video_timings *dispc_vm;
+	const struct videomode *req_vm;
+	struct videomode *dispc_vm;
 	struct omap_dss_dsi_videomode_timings *dsi_vm;
 	u64 dsi_tput, dispc_tput;
 
 	dsi_tput = (u64)byteclk * ndl * 8;
 
-	req_vm = cfg->timings;
+	req_vm = cfg->vm;
 	req_pck_min = ctx->req_pck_min;
 	req_pck_max = ctx->req_pck_max;
 	req_pck_nom = ctx->req_pck_nom;
@@ -4525,9 +4530,10 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx)
 	dispc_pck = ctx->dispc_cinfo.pck;
 	dispc_tput = (u64)dispc_pck * bitspp;
 
-	xres = req_vm->x_res;
+	xres = req_vm->hactive;
 
-	panel_hbl = req_vm->hfp + req_vm->hbp + req_vm->hsw;
+	panel_hbl = req_vm->hfront_porch + req_vm->hback_porch +
+		    req_vm->hsync_len;
 	panel_htot = xres + panel_hbl;
 
 	dsi_hact = DIV_ROUND_UP(DIV_ROUND_UP(xres * bitspp, 8) + 6, ndl);
@@ -4557,7 +4563,7 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx)
 	hss = DIV_ROUND_UP(4, ndl);
 
 	if (cfg->trans_mode == OMAP_DSS_DSI_PULSE_MODE) {
-		if (ndl == 3 && req_vm->hsw == 0)
+		if (ndl == 3 && req_vm->hsync_len == 0)
 			hse = 1;
 		else
 			hse = DIV_ROUND_UP(4, ndl);
@@ -4596,14 +4602,14 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx)
 
 	if (cfg->trans_mode != OMAP_DSS_DSI_PULSE_MODE) {
 		hsa = 0;
-	} else if (ndl == 3 && req_vm->hsw == 0) {
+	} else if (ndl == 3 && req_vm->hsync_len == 0) {
 		hsa = 0;
 	} else {
-		hsa = div64_u64((u64)req_vm->hsw * byteclk, req_pck_nom);
+		hsa = div64_u64((u64)req_vm->hsync_len * byteclk, req_pck_nom);
 		hsa = max(hsa - hse, 1);
 	}
 
-	hbp = div64_u64((u64)req_vm->hbp * byteclk, req_pck_nom);
+	hbp = div64_u64((u64)req_vm->hback_porch * byteclk, req_pck_nom);
 	hbp = max(hbp, 1);
 
 	hfp = dsi_hbl - (hss + hsa + hse + hbp);
@@ -4633,10 +4639,10 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx)
 	dsi_vm->hact = xres;
 	dsi_vm->hfp = hfp;
 
-	dsi_vm->vsa = req_vm->vsw;
-	dsi_vm->vbp = req_vm->vbp;
-	dsi_vm->vact = req_vm->y_res;
-	dsi_vm->vfp = req_vm->vfp;
+	dsi_vm->vsa = req_vm->vsync_len;
+	dsi_vm->vbp = req_vm->vback_porch;
+	dsi_vm->vact = req_vm->vactive;
+	dsi_vm->vfp = req_vm->vfront_porch;
 
 	dsi_vm->trans_mode = cfg->trans_mode;
 
@@ -4650,19 +4656,19 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx)
 
 	/* setup DISPC videomode */
 
-	dispc_vm = &ctx->dispc_vm;
+	dispc_vm = &ctx->vm;
 	*dispc_vm = *req_vm;
 	dispc_vm->pixelclock = dispc_pck;
 
 	if (cfg->trans_mode == OMAP_DSS_DSI_PULSE_MODE) {
-		hsa = div64_u64((u64)req_vm->hsw * dispc_pck,
+		hsa = div64_u64((u64)req_vm->hsync_len * dispc_pck,
 				req_pck_nom);
 		hsa = max(hsa, 1);
 	} else {
 		hsa = 1;
 	}
 
-	hbp = div64_u64((u64)req_vm->hbp * dispc_pck, req_pck_nom);
+	hbp = div64_u64((u64)req_vm->hback_porch * dispc_pck, req_pck_nom);
 	hbp = max(hbp, 1);
 
 	hfp = dispc_hbl - hsa - hbp;
@@ -4685,9 +4691,9 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx)
 	if (hfp < 1)
 		return false;
 
-	dispc_vm->hfp = hfp;
-	dispc_vm->hsw = hsa;
-	dispc_vm->hbp = hbp;
+	dispc_vm->hfront_porch = hfp;
+	dispc_vm->hsync_len = hsa;
+	dispc_vm->hback_porch = hbp;
 
 	return true;
 }
@@ -4707,9 +4713,9 @@ static bool dsi_vm_calc_dispc_cb(int lckd, int pckd, unsigned long lck,
 		return false;
 
 #ifdef PRINT_VERBOSE_VM_TIMINGS
-	print_dispc_vm("dispc", &ctx->dispc_vm);
+	print_dispc_vm("dispc", &ctx->vm);
 	print_dsi_vm("dsi  ", &ctx->dsi_vm);
-	print_dispc_vm("req  ", ctx->config->timings);
+	print_dispc_vm("req  ", ctx->config->vm);
 	print_dsi_dispc_vm("act  ", &ctx->dsi_vm);
 #endif
 
@@ -4758,7 +4764,7 @@ static bool dsi_vm_calc(struct dsi_data *dsi,
 		const struct omap_dss_dsi_config *cfg,
 		struct dsi_clk_calc_ctx *ctx)
 {
-	const struct omap_video_timings *t = cfg->timings;
+	const struct videomode *vm = cfg->vm;
 	unsigned long clkin;
 	unsigned long pll_min;
 	unsigned long pll_max;
@@ -4774,9 +4780,9 @@ static bool dsi_vm_calc(struct dsi_data *dsi,
 	ctx->config = cfg;
 
 	/* these limits should come from the panel driver */
-	ctx->req_pck_min = t->pixelclock - 1000;
-	ctx->req_pck_nom = t->pixelclock;
-	ctx->req_pck_max = t->pixelclock + 1000;
+	ctx->req_pck_min = vm->pixelclock - 1000;
+	ctx->req_pck_nom = vm->pixelclock;
+	ctx->req_pck_max = vm->pixelclock + 1000;
 
 	byteclk_min = div64_u64((u64)ctx->req_pck_min * bitspp, ndl * 8);
 	pll_min = max(cfg->hs_clk_min * 4, byteclk_min * 4 * 4);
@@ -4833,7 +4839,7 @@ static int dsi_set_config(struct omap_dss_device *dssdev,
 	dsi->user_dsi_cinfo = ctx.dsi_cinfo;
 	dsi->user_dispc_cinfo = ctx.dispc_cinfo;
 
-	dsi->timings = ctx.dispc_vm;
+	dsi->vm = ctx.vm;
 	dsi->vm_timings = ctx.dsi_vm;
 
 	mutex_unlock(&dsi->lock);
@@ -5342,7 +5348,7 @@ static int dsi_bind(struct device *dev, struct device *master, void *data)
 
 	dsi->phy_base = devm_ioremap(&dsidev->dev, res->start,
 		resource_size(res));
-	if (!dsi->proto_base) {
+	if (!dsi->phy_base) {
 		DSSERR("can't ioremap DSI PHY\n");
 		return -ENOMEM;
 	}
@@ -5362,7 +5368,7 @@ static int dsi_bind(struct device *dev, struct device *master, void *data)
 
 	dsi->pll_base = devm_ioremap(&dsidev->dev, res->start,
 		resource_size(res));
-	if (!dsi->proto_base) {
+	if (!dsi->pll_base) {
 		DSSERR("can't ioremap DSI PLL\n");
 		return -ENOMEM;
 	}
diff --git a/drivers/gpu/drm/omapdrm/dss/dss.h b/drivers/gpu/drm/omapdrm/dss/dss.h
index 4fd06dc41cb3..56493b290731 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss.h
+++ b/drivers/gpu/drm/omapdrm/dss/dss.h
@@ -366,8 +366,7 @@ bool dispc_div_calc(unsigned long dispc,
 		unsigned long pck_min, unsigned long pck_max,
 		dispc_div_calc_func func, void *data);
 
-bool dispc_mgr_timings_ok(enum omap_channel channel,
-		const struct omap_video_timings *timings);
+bool dispc_mgr_timings_ok(enum omap_channel channel, const struct videomode *vm);
 int dispc_calc_clock_rates(unsigned long dispc_fclk_rate,
 		struct dispc_clock_info *cinfo);
 
@@ -390,7 +389,7 @@ void dispc_wb_enable(bool enable);
 bool dispc_wb_is_enabled(void);
 void dispc_wb_set_channel_in(enum dss_writeback_channel channel);
 int dispc_wb_setup(const struct omap_dss_writeback_info *wi,
-		bool mem_to_mem, const struct omap_video_timings *timings);
+		bool mem_to_mem, const struct videomode *vm);
 
 /* VENC */
 int venc_init_platform_driver(void) __init;
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi.h b/drivers/gpu/drm/omapdrm/dss/hdmi.h
index 63e711545865..fb6cccd02374 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi.h
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi.h
@@ -181,7 +181,7 @@ struct hdmi_video_format {
 };
 
 struct hdmi_config {
-	struct omap_video_timings timings;
+	struct videomode vm;
 	struct hdmi_avi_infoframe infoframe;
 	enum hdmi_core_hdmi_dvi hdmi_dvi_mode;
 };
@@ -298,11 +298,11 @@ int hdmi_wp_set_pll_pwr(struct hdmi_wp_data *wp, enum hdmi_pll_pwr val);
 void hdmi_wp_video_config_format(struct hdmi_wp_data *wp,
 		struct hdmi_video_format *video_fmt);
 void hdmi_wp_video_config_interface(struct hdmi_wp_data *wp,
-		struct omap_video_timings *timings);
+		struct videomode *vm);
 void hdmi_wp_video_config_timing(struct hdmi_wp_data *wp,
-		struct omap_video_timings *timings);
+		struct videomode *vm);
 void hdmi_wp_init_vid_fmt_timings(struct hdmi_video_format *video_fmt,
-		struct omap_video_timings *timings, struct hdmi_config *param);
+		struct videomode *vm, struct hdmi_config *param);
 int hdmi_wp_init(struct platform_device *pdev, struct hdmi_wp_data *wp);
 phys_addr_t hdmi_wp_get_audio_dma_addr(struct hdmi_wp_data *wp);
 
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4.c b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
index cbd28dfdb86a..e7162c16de2e 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
@@ -155,7 +155,7 @@ static void hdmi_power_off_core(struct omap_dss_device *dssdev)
 static int hdmi_power_on_full(struct omap_dss_device *dssdev)
 {
 	int r;
-	struct omap_video_timings *p;
+	struct videomode *vm;
 	enum omap_channel channel = dssdev->dispc_channel;
 	struct hdmi_wp_data *wp = &hdmi.wp;
 	struct dss_pll_clock_info hdmi_cinfo = { 0 };
@@ -169,12 +169,13 @@ static int hdmi_power_on_full(struct omap_dss_device *dssdev)
 	hdmi_wp_clear_irqenable(wp, 0xffffffff);
 	hdmi_wp_set_irqstatus(wp, 0xffffffff);
 
-	p = &hdmi.cfg.timings;
+	vm = &hdmi.cfg.vm;
 
-	DSSDBG("hdmi_power_on x_res= %d y_res = %d\n", p->x_res, p->y_res);
+	DSSDBG("hdmi_power_on hactive= %d vactive = %d\n", vm->hactive,
+	       vm->vactive);
 
-	pc = p->pixelclock;
-	if (p->double_pixel)
+	pc = vm->pixelclock;
+	if (vm->flags & DISPLAY_FLAGS_DOUBLECLK)
 		pc *= 2;
 
 	/* DSS_HDMI_TCLK is bitclk / 10 */
@@ -209,7 +210,7 @@ static int hdmi_power_on_full(struct omap_dss_device *dssdev)
 	hdmi4_configure(&hdmi.core, &hdmi.wp, &hdmi.cfg);
 
 	/* tv size */
-	dss_mgr_set_timings(channel, p);
+	dss_mgr_set_timings(channel, vm);
 
 	r = dss_mgr_enable(channel);
 	if (r)
@@ -255,30 +256,30 @@ static void hdmi_power_off_full(struct omap_dss_device *dssdev)
 }
 
 static int hdmi_display_check_timing(struct omap_dss_device *dssdev,
-					struct omap_video_timings *timings)
+				     struct videomode *vm)
 {
-	if (!dispc_mgr_timings_ok(dssdev->dispc_channel, timings))
+	if (!dispc_mgr_timings_ok(dssdev->dispc_channel, vm))
 		return -EINVAL;
 
 	return 0;
 }
 
 static void hdmi_display_set_timing(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				    struct videomode *vm)
 {
 	mutex_lock(&hdmi.lock);
 
-	hdmi.cfg.timings = *timings;
+	hdmi.cfg.vm = *vm;
 
-	dispc_set_tv_pclk(timings->pixelclock);
+	dispc_set_tv_pclk(vm->pixelclock);
 
 	mutex_unlock(&hdmi.lock);
 }
 
 static void hdmi_display_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				     struct videomode *vm)
 {
-	*timings = hdmi.cfg.timings;
+	*vm = hdmi.cfg.vm;
 }
 
 static void hdmi_dump_regs(struct seq_file *s)
@@ -352,7 +353,7 @@ static int hdmi_display_enable(struct omap_dss_device *dssdev)
 
 	if (hdmi.audio_configured) {
 		r = hdmi4_audio_config(&hdmi.core, &hdmi.wp, &hdmi.audio_config,
-				       hdmi.cfg.timings.pixelclock);
+				       hdmi.cfg.vm.pixelclock);
 		if (r) {
 			DSSERR("Error restoring audio configuration: %d", r);
 			hdmi.audio_abort_cb(&hdmi.pdev->dev);
@@ -643,7 +644,7 @@ static int hdmi_audio_config(struct device *dev,
 	}
 
 	ret = hdmi4_audio_config(&hd->core, &hd->wp, dss_audio,
-				 hd->cfg.timings.pixelclock);
+				 hd->cfg.vm.pixelclock);
 	if (!ret) {
 		hd->audio_configured = true;
 		hd->audio_config = *dss_audio;
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c
index ef3afe99e487..e05b7ac4f7dd 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c
@@ -310,7 +310,7 @@ void hdmi4_configure(struct hdmi_core_data *core,
 	struct hdmi_wp_data *wp, struct hdmi_config *cfg)
 {
 	/* HDMI */
-	struct omap_video_timings video_timing;
+	struct videomode vm;
 	struct hdmi_video_format video_format;
 	/* HDMI core */
 	struct hdmi_core_video_config v_core_cfg;
@@ -318,16 +318,16 @@ void hdmi4_configure(struct hdmi_core_data *core,
 
 	hdmi_core_init(&v_core_cfg);
 
-	hdmi_wp_init_vid_fmt_timings(&video_format, &video_timing, cfg);
+	hdmi_wp_init_vid_fmt_timings(&video_format, &vm, cfg);
 
-	hdmi_wp_video_config_timing(wp, &video_timing);
+	hdmi_wp_video_config_timing(wp, &vm);
 
 	/* video config */
 	video_format.packing_mode = HDMI_PACK_24b_RGB_YUV444_YUV422;
 
 	hdmi_wp_video_config_format(wp, &video_format);
 
-	hdmi_wp_video_config_interface(wp, &video_timing);
+	hdmi_wp_video_config_interface(wp, &vm);
 
 	/*
 	 * configure core video part
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi5.c b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
index 0c0a5139a301..678dfb02764a 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi5.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
@@ -172,7 +172,7 @@ static void hdmi_power_off_core(struct omap_dss_device *dssdev)
 static int hdmi_power_on_full(struct omap_dss_device *dssdev)
 {
 	int r;
-	struct omap_video_timings *p;
+	struct videomode *vm;
 	enum omap_channel channel = dssdev->dispc_channel;
 	struct dss_pll_clock_info hdmi_cinfo = { 0 };
 	unsigned pc;
@@ -181,12 +181,13 @@ static int hdmi_power_on_full(struct omap_dss_device *dssdev)
 	if (r)
 		return r;
 
-	p = &hdmi.cfg.timings;
+	vm = &hdmi.cfg.vm;
 
-	DSSDBG("hdmi_power_on x_res= %d y_res = %d\n", p->x_res, p->y_res);
+	DSSDBG("hdmi_power_on hactive= %d vactive = %d\n", vm->hactive,
+	       vm->vactive);
 
-	pc = p->pixelclock;
-	if (p->double_pixel)
+	pc = vm->pixelclock;
+	if (vm->flags & DISPLAY_FLAGS_DOUBLECLK)
 		pc *= 2;
 
 	/* DSS_HDMI_TCLK is bitclk / 10 */
@@ -226,7 +227,7 @@ static int hdmi_power_on_full(struct omap_dss_device *dssdev)
 	hdmi5_configure(&hdmi.core, &hdmi.wp, &hdmi.cfg);
 
 	/* tv size */
-	dss_mgr_set_timings(channel, p);
+	dss_mgr_set_timings(channel, vm);
 
 	r = dss_mgr_enable(channel);
 	if (r)
@@ -272,30 +273,30 @@ static void hdmi_power_off_full(struct omap_dss_device *dssdev)
 }
 
 static int hdmi_display_check_timing(struct omap_dss_device *dssdev,
-					struct omap_video_timings *timings)
+				     struct videomode *vm)
 {
-	if (!dispc_mgr_timings_ok(dssdev->dispc_channel, timings))
+	if (!dispc_mgr_timings_ok(dssdev->dispc_channel, vm))
 		return -EINVAL;
 
 	return 0;
 }
 
 static void hdmi_display_set_timing(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				    struct videomode *vm)
 {
 	mutex_lock(&hdmi.lock);
 
-	hdmi.cfg.timings = *timings;
+	hdmi.cfg.vm = *vm;
 
-	dispc_set_tv_pclk(timings->pixelclock);
+	dispc_set_tv_pclk(vm->pixelclock);
 
 	mutex_unlock(&hdmi.lock);
 }
 
 static void hdmi_display_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+				     struct videomode *vm)
 {
-	*timings = hdmi.cfg.timings;
+	*vm = hdmi.cfg.vm;
 }
 
 static void hdmi_dump_regs(struct seq_file *s)
@@ -378,7 +379,7 @@ static int hdmi_display_enable(struct omap_dss_device *dssdev)
 
 	if (hdmi.audio_configured) {
 		r = hdmi5_audio_config(&hdmi.core, &hdmi.wp, &hdmi.audio_config,
-				       hdmi.cfg.timings.pixelclock);
+				       hdmi.cfg.vm.pixelclock);
 		if (r) {
 			DSSERR("Error restoring audio configuration: %d", r);
 			hdmi.audio_abort_cb(&hdmi.pdev->dev);
@@ -669,7 +670,7 @@ static int hdmi_audio_config(struct device *dev,
 	}
 
 	ret = hdmi5_audio_config(&hd->core, &hd->wp, dss_audio,
-				 hd->cfg.timings.pixelclock);
+				 hd->cfg.vm.pixelclock);
 
 	if (!ret) {
 		hd->audio_configured = true;
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c b/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c
index 8ab2093daa12..8de1d7b2ae55 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c
@@ -292,35 +292,35 @@ static void hdmi_core_init(struct hdmi_core_vid_config *video_cfg,
 {
 	DSSDBG("hdmi_core_init\n");
 
-	video_cfg->v_fc_config.timings = cfg->timings;
+	video_cfg->v_fc_config.vm = cfg->vm;
 
 	/* video core */
 	video_cfg->data_enable_pol = 1; /* It is always 1*/
-	video_cfg->hblank = cfg->timings.hfp +
-				cfg->timings.hbp + cfg->timings.hsw;
+	video_cfg->hblank = cfg->vm.hfront_porch +
+			    cfg->vm.hback_porch + cfg->vm.hsync_len;
 	video_cfg->vblank_osc = 0;
-	video_cfg->vblank = cfg->timings.vsw +
-				cfg->timings.vfp + cfg->timings.vbp;
+	video_cfg->vblank = cfg->vm.vsync_len + cfg->vm.vfront_porch +
+			    cfg->vm.vback_porch;
 	video_cfg->v_fc_config.hdmi_dvi_mode = cfg->hdmi_dvi_mode;
 
-	if (cfg->timings.interlace) {
+	if (cfg->vm.flags & DISPLAY_FLAGS_INTERLACED) {
 		/* set vblank_osc if vblank is fractional */
 		if (video_cfg->vblank % 2 != 0)
 			video_cfg->vblank_osc = 1;
 
-		video_cfg->v_fc_config.timings.y_res /= 2;
+		video_cfg->v_fc_config.vm.vactive /= 2;
 		video_cfg->vblank /= 2;
-		video_cfg->v_fc_config.timings.vfp /= 2;
-		video_cfg->v_fc_config.timings.vsw /= 2;
-		video_cfg->v_fc_config.timings.vbp /= 2;
+		video_cfg->v_fc_config.vm.vfront_porch /= 2;
+		video_cfg->v_fc_config.vm.vsync_len /= 2;
+		video_cfg->v_fc_config.vm.vback_porch /= 2;
 	}
 
-	if (cfg->timings.double_pixel) {
-		video_cfg->v_fc_config.timings.x_res *= 2;
+	if (cfg->vm.flags & DISPLAY_FLAGS_DOUBLECLK) {
+		video_cfg->v_fc_config.vm.hactive *= 2;
 		video_cfg->hblank *= 2;
-		video_cfg->v_fc_config.timings.hfp *= 2;
-		video_cfg->v_fc_config.timings.hsw *= 2;
-		video_cfg->v_fc_config.timings.hbp *= 2;
+		video_cfg->v_fc_config.vm.hfront_porch *= 2;
+		video_cfg->v_fc_config.vm.hsync_len *= 2;
+		video_cfg->v_fc_config.vm.hback_porch *= 2;
 	}
 }
 
@@ -329,13 +329,12 @@ static void hdmi_core_video_config(struct hdmi_core_data *core,
 			struct hdmi_core_vid_config *cfg)
 {
 	void __iomem *base = core->base;
+	struct videomode *vm = &cfg->v_fc_config.vm;
 	unsigned char r = 0;
 	bool vsync_pol, hsync_pol;
 
-	vsync_pol =
-		cfg->v_fc_config.timings.vsync_level == OMAPDSS_SIG_ACTIVE_HIGH;
-	hsync_pol =
-		cfg->v_fc_config.timings.hsync_level == OMAPDSS_SIG_ACTIVE_HIGH;
+	vsync_pol = !!(vm->flags & DISPLAY_FLAGS_VSYNC_HIGH);
+	hsync_pol = !!(vm->flags & DISPLAY_FLAGS_HSYNC_HIGH);
 
 	/* Set hsync, vsync and data-enable polarity  */
 	r = hdmi_read_reg(base, HDMI_CORE_FC_INVIDCONF);
@@ -343,20 +342,16 @@ static void hdmi_core_video_config(struct hdmi_core_data *core,
 	r = FLD_MOD(r, hsync_pol, 5, 5);
 	r = FLD_MOD(r, cfg->data_enable_pol, 4, 4);
 	r = FLD_MOD(r, cfg->vblank_osc, 1, 1);
-	r = FLD_MOD(r, cfg->v_fc_config.timings.interlace, 0, 0);
+	r = FLD_MOD(r, !!(vm->flags & DISPLAY_FLAGS_INTERLACED), 0, 0);
 	hdmi_write_reg(base, HDMI_CORE_FC_INVIDCONF, r);
 
 	/* set x resolution */
-	REG_FLD_MOD(base, HDMI_CORE_FC_INHACTIV1,
-			cfg->v_fc_config.timings.x_res >> 8, 4, 0);
-	REG_FLD_MOD(base, HDMI_CORE_FC_INHACTIV0,
-			cfg->v_fc_config.timings.x_res & 0xFF, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_INHACTIV1, vm->hactive >> 8, 4, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_INHACTIV0, vm->hactive & 0xFF, 7, 0);
 
 	/* set y resolution */
-	REG_FLD_MOD(base, HDMI_CORE_FC_INVACTIV1,
-			cfg->v_fc_config.timings.y_res >> 8, 4, 0);
-	REG_FLD_MOD(base, HDMI_CORE_FC_INVACTIV0,
-			cfg->v_fc_config.timings.y_res & 0xFF, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_INVACTIV1, vm->vactive >> 8, 4, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_INVACTIV0, vm->vactive & 0xFF, 7, 0);
 
 	/* set horizontal blanking pixels */
 	REG_FLD_MOD(base, HDMI_CORE_FC_INHBLANK1, cfg->hblank >> 8, 4, 0);
@@ -366,30 +361,28 @@ static void hdmi_core_video_config(struct hdmi_core_data *core,
 	REG_FLD_MOD(base, HDMI_CORE_FC_INVBLANK, cfg->vblank, 7, 0);
 
 	/* set horizontal sync offset */
-	REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINDELAY1,
-			cfg->v_fc_config.timings.hfp >> 8, 4, 0);
-	REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINDELAY0,
-			cfg->v_fc_config.timings.hfp & 0xFF, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINDELAY1, vm->hfront_porch >> 8,
+		    4, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINDELAY0, vm->hfront_porch & 0xFF,
+		    7, 0);
 
 	/* set vertical sync offset */
-	REG_FLD_MOD(base, HDMI_CORE_FC_VSYNCINDELAY,
-			cfg->v_fc_config.timings.vfp, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_VSYNCINDELAY, vm->vfront_porch, 7, 0);
 
 	/* set horizontal sync pulse width */
-	REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINWIDTH1,
-			(cfg->v_fc_config.timings.hsw >> 8), 1, 0);
-	REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINWIDTH0,
-			cfg->v_fc_config.timings.hsw & 0xFF, 7, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINWIDTH1, (vm->hsync_len >> 8),
+		    1, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINWIDTH0, vm->hsync_len & 0xFF,
+		    7, 0);
 
 	/*  set vertical sync pulse width */
-	REG_FLD_MOD(base, HDMI_CORE_FC_VSYNCINWIDTH,
-			cfg->v_fc_config.timings.vsw, 5, 0);
+	REG_FLD_MOD(base, HDMI_CORE_FC_VSYNCINWIDTH, vm->vsync_len, 5, 0);
 
 	/* select DVI mode */
 	REG_FLD_MOD(base, HDMI_CORE_FC_INVIDCONF,
-			cfg->v_fc_config.hdmi_dvi_mode, 3, 3);
+		    cfg->v_fc_config.hdmi_dvi_mode, 3, 3);
 
-	if (cfg->v_fc_config.timings.double_pixel)
+	if (vm->flags & DISPLAY_FLAGS_DOUBLECLK)
 		REG_FLD_MOD(base, HDMI_CORE_FC_PRCONF, 2, 7, 4);
 	else
 		REG_FLD_MOD(base, HDMI_CORE_FC_PRCONF, 1, 7, 4);
@@ -616,7 +609,7 @@ int hdmi5_core_handle_irqs(struct hdmi_core_data *core)
 void hdmi5_configure(struct hdmi_core_data *core, struct hdmi_wp_data *wp,
 		struct hdmi_config *cfg)
 {
-	struct omap_video_timings video_timing;
+	struct videomode vm;
 	struct hdmi_video_format video_format;
 	struct hdmi_core_vid_config v_core_cfg;
 
@@ -624,16 +617,16 @@ void hdmi5_configure(struct hdmi_core_data *core, struct hdmi_wp_data *wp,
 
 	hdmi_core_init(&v_core_cfg, cfg);
 
-	hdmi_wp_init_vid_fmt_timings(&video_format, &video_timing, cfg);
+	hdmi_wp_init_vid_fmt_timings(&video_format, &vm, cfg);
 
-	hdmi_wp_video_config_timing(wp, &video_timing);
+	hdmi_wp_video_config_timing(wp, &vm);
 
 	/* video config */
 	video_format.packing_mode = HDMI_PACK_24b_RGB_YUV444_YUV422;
 
 	hdmi_wp_video_config_format(wp, &video_format);
 
-	hdmi_wp_video_config_interface(wp, &video_timing);
+	hdmi_wp_video_config_interface(wp, &vm);
 
 	/* support limited range with 24 bit color depth for now */
 	hdmi_core_configure_range(core);
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
index 203694a52d18..b783d5a0750e 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
@@ -144,87 +144,84 @@ void hdmi_wp_video_config_format(struct hdmi_wp_data *wp,
 }
 
 void hdmi_wp_video_config_interface(struct hdmi_wp_data *wp,
-		struct omap_video_timings *timings)
+				    struct videomode *vm)
 {
 	u32 r;
 	bool vsync_pol, hsync_pol;
 	DSSDBG("Enter hdmi_wp_video_config_interface\n");
 
-	vsync_pol = timings->vsync_level == OMAPDSS_SIG_ACTIVE_HIGH;
-	hsync_pol = timings->hsync_level == OMAPDSS_SIG_ACTIVE_HIGH;
+	vsync_pol = !!(vm->flags & DISPLAY_FLAGS_VSYNC_HIGH);
+	hsync_pol = !!(vm->flags & DISPLAY_FLAGS_HSYNC_HIGH);
 
 	r = hdmi_read_reg(wp->base, HDMI_WP_VIDEO_CFG);
 	r = FLD_MOD(r, vsync_pol, 7, 7);
 	r = FLD_MOD(r, hsync_pol, 6, 6);
-	r = FLD_MOD(r, timings->interlace, 3, 3);
+	r = FLD_MOD(r, !!(vm->flags & DISPLAY_FLAGS_INTERLACED), 3, 3);
 	r = FLD_MOD(r, 1, 1, 0); /* HDMI_TIMING_MASTER_24BIT */
 	hdmi_write_reg(wp->base, HDMI_WP_VIDEO_CFG, r);
 }
 
 void hdmi_wp_video_config_timing(struct hdmi_wp_data *wp,
-		struct omap_video_timings *timings)
+				 struct videomode *vm)
 {
 	u32 timing_h = 0;
 	u32 timing_v = 0;
-	unsigned hsw_offset = 1;
+	unsigned hsync_len_offset = 1;
 
 	DSSDBG("Enter hdmi_wp_video_config_timing\n");
 
 	/*
 	 * On OMAP4 and OMAP5 ES1 the HSW field is programmed as is. On OMAP5
-	 * ES2+ (including DRA7/AM5 SoCs) HSW field is programmed to hsw-1.
+	 * ES2+ (including DRA7/AM5 SoCs) HSW field is programmed to hsync_len-1.
 	 * However, we don't support OMAP5 ES1 at all, so we can just check for
 	 * OMAP4 here.
 	 */
 	if (omapdss_get_version() == OMAPDSS_VER_OMAP4430_ES1 ||
 	    omapdss_get_version() == OMAPDSS_VER_OMAP4430_ES2 ||
 	    omapdss_get_version() == OMAPDSS_VER_OMAP4)
-		hsw_offset = 0;
+		hsync_len_offset = 0;
 
-	timing_h |= FLD_VAL(timings->hbp, 31, 20);
-	timing_h |= FLD_VAL(timings->hfp, 19, 8);
-	timing_h |= FLD_VAL(timings->hsw - hsw_offset, 7, 0);
+	timing_h |= FLD_VAL(vm->hback_porch, 31, 20);
+	timing_h |= FLD_VAL(vm->hfront_porch, 19, 8);
+	timing_h |= FLD_VAL(vm->hsync_len - hsync_len_offset, 7, 0);
 	hdmi_write_reg(wp->base, HDMI_WP_VIDEO_TIMING_H, timing_h);
 
-	timing_v |= FLD_VAL(timings->vbp, 31, 20);
-	timing_v |= FLD_VAL(timings->vfp, 19, 8);
-	timing_v |= FLD_VAL(timings->vsw, 7, 0);
+	timing_v |= FLD_VAL(vm->vback_porch, 31, 20);
+	timing_v |= FLD_VAL(vm->vfront_porch, 19, 8);
+	timing_v |= FLD_VAL(vm->vsync_len, 7, 0);
 	hdmi_write_reg(wp->base, HDMI_WP_VIDEO_TIMING_V, timing_v);
 }
 
 void hdmi_wp_init_vid_fmt_timings(struct hdmi_video_format *video_fmt,
-		struct omap_video_timings *timings, struct hdmi_config *param)
+		struct videomode *vm, struct hdmi_config *param)
 {
 	DSSDBG("Enter hdmi_wp_video_init_format\n");
 
 	video_fmt->packing_mode = HDMI_PACK_10b_RGB_YUV444;
-	video_fmt->y_res = param->timings.y_res;
-	video_fmt->x_res = param->timings.x_res;
-
-	timings->hbp = param->timings.hbp;
-	timings->hfp = param->timings.hfp;
-	timings->hsw = param->timings.hsw;
-	timings->vbp = param->timings.vbp;
-	timings->vfp = param->timings.vfp;
-	timings->vsw = param->timings.vsw;
-
-	timings->vsync_level = param->timings.vsync_level;
-	timings->hsync_level = param->timings.hsync_level;
-	timings->interlace = param->timings.interlace;
-	timings->double_pixel = param->timings.double_pixel;
-
-	if (param->timings.interlace) {
+	video_fmt->y_res = param->vm.vactive;
+	video_fmt->x_res = param->vm.hactive;
+
+	vm->hback_porch = param->vm.hback_porch;
+	vm->hfront_porch = param->vm.hfront_porch;
+	vm->hsync_len = param->vm.hsync_len;
+	vm->vback_porch = param->vm.vback_porch;
+	vm->vfront_porch = param->vm.vfront_porch;
+	vm->vsync_len = param->vm.vsync_len;
+
+	vm->flags = param->vm.flags;
+
+	if (param->vm.flags & DISPLAY_FLAGS_INTERLACED) {
 		video_fmt->y_res /= 2;
-		timings->vbp /= 2;
-		timings->vfp /= 2;
-		timings->vsw /= 2;
+		vm->vback_porch /= 2;
+		vm->vfront_porch /= 2;
+		vm->vsync_len /= 2;
 	}
 
-	if (param->timings.double_pixel) {
+	if (param->vm.flags & DISPLAY_FLAGS_DOUBLECLK) {
 		video_fmt->x_res *= 2;
-		timings->hfp *= 2;
-		timings->hsw *= 2;
-		timings->hbp *= 2;
+		vm->hfront_porch *= 2;
+		vm->hsync_len *= 2;
+		vm->hback_porch *= 2;
 	}
 }
 
diff --git a/drivers/gpu/drm/omapdrm/dss/omapdss.h b/drivers/gpu/drm/omapdrm/dss/omapdss.h
index 6eaf1adbd606..b420dde8c0fb 100644
--- a/drivers/gpu/drm/omapdrm/dss/omapdss.h
+++ b/drivers/gpu/drm/omapdrm/dss/omapdss.h
@@ -290,7 +290,7 @@ struct omap_dss_dsi_videomode_timings {
 struct omap_dss_dsi_config {
 	enum omap_dss_dsi_mode mode;
 	enum omap_dss_dsi_pixel_format pixel_format;
-	const struct omap_video_timings *timings;
+	const struct videomode *vm;
 
 	unsigned long hs_clk_min, hs_clk_max;
 	unsigned long lp_clk_min, lp_clk_max;
@@ -299,48 +299,12 @@ struct omap_dss_dsi_config {
 	enum omap_dss_dsi_trans_mode trans_mode;
 };
 
-struct omap_video_timings {
-	/* Unit: pixels */
-	u16 x_res;
-	/* Unit: pixels */
-	u16 y_res;
-	/* Unit: Hz */
-	u32 pixelclock;
-	/* Unit: pixel clocks */
-	u16 hsw;	/* Horizontal synchronization pulse width */
-	/* Unit: pixel clocks */
-	u16 hfp;	/* Horizontal front porch */
-	/* Unit: pixel clocks */
-	u16 hbp;	/* Horizontal back porch */
-	/* Unit: line clocks */
-	u16 vsw;	/* Vertical synchronization pulse width */
-	/* Unit: line clocks */
-	u16 vfp;	/* Vertical front porch */
-	/* Unit: line clocks */
-	u16 vbp;	/* Vertical back porch */
-
-	/* Vsync logic level */
-	enum omap_dss_signal_level vsync_level;
-	/* Hsync logic level */
-	enum omap_dss_signal_level hsync_level;
-	/* Interlaced or Progressive timings */
-	bool interlace;
-	/* Pixel clock edge to drive LCD data */
-	enum omap_dss_signal_edge data_pclk_edge;
-	/* Data enable logic level */
-	enum omap_dss_signal_level de_level;
-	/* Pixel clock edges to drive HSYNC and VSYNC signals */
-	enum omap_dss_signal_edge sync_pclk_edge;
-
-	bool double_pixel;
-};
-
-/* Hardcoded timings for tv modes. Venc only uses these to
+/* Hardcoded videomodes for tv. Venc only uses these to
  * identify the mode, and does not actually use the configs
  * itself. However, the configs should be something that
  * a normal monitor can also show */
-extern const struct omap_video_timings omap_dss_pal_timings;
-extern const struct omap_video_timings omap_dss_ntsc_timings;
+extern const struct videomode omap_dss_pal_vm;
+extern const struct videomode omap_dss_ntsc_vm;
 
 struct omap_dss_cpr_coefs {
 	s16 rr, rg, rb;
@@ -502,11 +466,11 @@ struct omapdss_dpi_ops {
 	void (*disable)(struct omap_dss_device *dssdev);
 
 	int (*check_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			     struct videomode *vm);
 	void (*set_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			    struct videomode *vm);
 	void (*get_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			    struct videomode *vm);
 
 	void (*set_data_lines)(struct omap_dss_device *dssdev, int data_lines);
 };
@@ -521,11 +485,11 @@ struct omapdss_sdi_ops {
 	void (*disable)(struct omap_dss_device *dssdev);
 
 	int (*check_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			     struct videomode *vm);
 	void (*set_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			    struct videomode *vm);
 	void (*get_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			    struct videomode *vm);
 
 	void (*set_datapairs)(struct omap_dss_device *dssdev, int datapairs);
 };
@@ -540,11 +504,11 @@ struct omapdss_dvi_ops {
 	void (*disable)(struct omap_dss_device *dssdev);
 
 	int (*check_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			     struct videomode *vm);
 	void (*set_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			    struct videomode *vm);
 	void (*get_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			    struct videomode *vm);
 };
 
 struct omapdss_atv_ops {
@@ -557,11 +521,11 @@ struct omapdss_atv_ops {
 	void (*disable)(struct omap_dss_device *dssdev);
 
 	int (*check_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			     struct videomode *vm);
 	void (*set_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			    struct videomode *vm);
 	void (*get_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			    struct videomode *vm);
 
 	void (*set_type)(struct omap_dss_device *dssdev,
 		enum omap_dss_venc_type type);
@@ -582,11 +546,11 @@ struct omapdss_hdmi_ops {
 	void (*disable)(struct omap_dss_device *dssdev);
 
 	int (*check_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			     struct videomode *vm);
 	void (*set_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			    struct videomode *vm);
 	void (*get_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			    struct videomode *vm);
 
 	int (*read_edid)(struct omap_dss_device *dssdev, u8 *buf, int len);
 	bool (*detect)(struct omap_dss_device *dssdev);
@@ -692,7 +656,7 @@ struct omap_dss_device {
 	} phy;
 
 	struct {
-		struct omap_video_timings timings;
+		struct videomode vm;
 
 		enum omap_dss_dsi_pixel_format dsi_pix_fmt;
 		enum omap_dss_dsi_mode dsi_mode;
@@ -785,11 +749,11 @@ struct omap_dss_driver {
 	int (*get_recommended_bpp)(struct omap_dss_device *dssdev);
 
 	int (*check_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			     struct videomode *vm);
 	void (*set_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			    struct videomode *vm);
 	void (*get_timings)(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings);
+			    struct videomode *vm);
 
 	int (*set_wss)(struct omap_dss_device *dssdev, u32 wss);
 	u32 (*get_wss)(struct omap_dss_device *dssdev);
@@ -819,11 +783,6 @@ struct omap_dss_device *omap_dss_find_device(void *data,
 		int (*match)(struct omap_dss_device *dssdev, void *data));
 const char *omapdss_get_default_display_name(void);
 
-void videomode_to_omap_video_timings(const struct videomode *vm,
-		struct omap_video_timings *ovt);
-void omap_video_timings_to_videomode(const struct omap_video_timings *ovt,
-		struct videomode *vm);
-
 int dss_feat_get_num_mgrs(void);
 int dss_feat_get_num_ovls(void);
 enum omap_color_mode dss_feat_get_supported_color_modes(enum omap_plane plane);
@@ -852,7 +811,7 @@ void omapdss_default_get_resolution(struct omap_dss_device *dssdev,
 		u16 *xres, u16 *yres);
 int omapdss_default_get_recommended_bpp(struct omap_dss_device *dssdev);
 void omapdss_default_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings);
+				 struct videomode *vm);
 
 typedef void (*omap_dispc_isr_t) (void *arg, u32 mask);
 int omap_dispc_register_isr(omap_dispc_isr_t isr, void *arg, u32 mask);
@@ -906,7 +865,7 @@ void dispc_mgr_go(enum omap_channel channel);
 void dispc_mgr_set_lcd_config(enum omap_channel channel,
 		const struct dss_lcd_mgr_config *config);
 void dispc_mgr_set_timings(enum omap_channel channel,
-		const struct omap_video_timings *timings);
+		const struct videomode *vm);
 void dispc_mgr_setup(enum omap_channel channel,
 		const struct omap_overlay_manager_info *info);
 u32 dispc_mgr_gamma_size(enum omap_channel channel);
@@ -919,8 +878,7 @@ bool dispc_ovl_enabled(enum omap_plane plane);
 void dispc_ovl_set_channel_out(enum omap_plane plane,
 		enum omap_channel channel);
 int dispc_ovl_setup(enum omap_plane plane, const struct omap_overlay_info *oi,
-		bool replication, const struct omap_video_timings *mgr_timings,
-		bool mem_to_mem);
+		bool replication, const struct videomode *vm, bool mem_to_mem);
 
 enum omap_dss_output_id dispc_mgr_get_supported_outputs(enum omap_channel channel);
 
@@ -934,7 +892,7 @@ struct dss_mgr_ops {
 	int (*enable)(enum omap_channel channel);
 	void (*disable)(enum omap_channel channel);
 	void (*set_timings)(enum omap_channel channel,
-			const struct omap_video_timings *timings);
+			const struct videomode *vm);
 	void (*set_lcd_config)(enum omap_channel channel,
 			const struct dss_lcd_mgr_config *config);
 	int (*register_framedone_handler)(enum omap_channel channel,
@@ -951,7 +909,7 @@ int dss_mgr_connect(enum omap_channel channel,
 void dss_mgr_disconnect(enum omap_channel channel,
 		struct omap_dss_device *dst);
 void dss_mgr_set_timings(enum omap_channel channel,
-		const struct omap_video_timings *timings);
+		const struct videomode *vm);
 void dss_mgr_set_lcd_config(enum omap_channel channel,
 		const struct dss_lcd_mgr_config *config);
 int dss_mgr_enable(enum omap_channel channel);
diff --git a/drivers/gpu/drm/omapdrm/dss/output.c b/drivers/gpu/drm/omapdrm/dss/output.c
index 24f859488201..a901af5a9bc3 100644
--- a/drivers/gpu/drm/omapdrm/dss/output.c
+++ b/drivers/gpu/drm/omapdrm/dss/output.c
@@ -201,10 +201,9 @@ void dss_mgr_disconnect(enum omap_channel channel,
 }
 EXPORT_SYMBOL(dss_mgr_disconnect);
 
-void dss_mgr_set_timings(enum omap_channel channel,
-		const struct omap_video_timings *timings)
+void dss_mgr_set_timings(enum omap_channel channel, const struct videomode *vm)
 {
-	dss_mgr_ops->set_timings(channel, timings);
+	dss_mgr_ops->set_timings(channel, vm);
 }
 EXPORT_SYMBOL(dss_mgr_set_timings);
 
diff --git a/drivers/gpu/drm/omapdrm/dss/rfbi.c b/drivers/gpu/drm/omapdrm/dss/rfbi.c
index cd53566d75eb..09724757366a 100644
--- a/drivers/gpu/drm/omapdrm/dss/rfbi.c
+++ b/drivers/gpu/drm/omapdrm/dss/rfbi.c
@@ -113,7 +113,7 @@ static struct {
 
 	struct semaphore bus_lock;
 
-	struct omap_video_timings timings;
+	struct videomode vm;
 	int pixel_size;
 	int data_lines;
 	struct rfbi_timings intf_timings;
@@ -308,15 +308,15 @@ static int rfbi_transfer_area(struct omap_dss_device *dssdev,
 	u32 l;
 	int r;
 	struct omap_overlay_manager *mgr = rfbi.output.manager;
-	u16 width = rfbi.timings.x_res;
-	u16 height = rfbi.timings.y_res;
+	u16 width = rfbi.vm.hactive;
+	u16 height = rfbi.vm.vactive;
 
 	/*BUG_ON(callback == 0);*/
 	BUG_ON(rfbi.framedone_callback != NULL);
 
 	DSSDBG("rfbi_transfer_area %dx%d\n", width, height);
 
-	dss_mgr_set_timings(mgr, &rfbi.timings);
+	dss_mgr_set_timings(mgr, &rfbi.vm);
 
 	r = dss_mgr_enable(mgr);
 	if (r)
@@ -777,8 +777,8 @@ static int rfbi_update(struct omap_dss_device *dssdev, void (*callback)(void *),
 
 static void rfbi_set_size(struct omap_dss_device *dssdev, u16 w, u16 h)
 {
-	rfbi.timings.x_res = w;
-	rfbi.timings.y_res = h;
+	rfbi.vm.hactive = w;
+	rfbi.vm.vactive = h;
 }
 
 static void rfbi_set_pixel_size(struct omap_dss_device *dssdev, int pixel_size)
@@ -854,25 +854,30 @@ static void rfbi_config_lcd_manager(struct omap_dss_device *dssdev)
 	dss_mgr_set_lcd_config(mgr, &mgr_config);
 
 	/*
-	 * Set rfbi.timings with default values, the x_res and y_res fields
+	 * Set rfbi.timings with default values, the hactive and vactive fields
 	 * are expected to be already configured by the panel driver via
 	 * omapdss_rfbi_set_size()
 	 */
-	rfbi.timings.hsw = 1;
-	rfbi.timings.hfp = 1;
-	rfbi.timings.hbp = 1;
-	rfbi.timings.vsw = 1;
-	rfbi.timings.vfp = 0;
-	rfbi.timings.vbp = 0;
-
-	rfbi.timings.interlace = false;
-	rfbi.timings.hsync_level = OMAPDSS_SIG_ACTIVE_HIGH;
-	rfbi.timings.vsync_level = OMAPDSS_SIG_ACTIVE_HIGH;
-	rfbi.timings.data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
-	rfbi.timings.de_level = OMAPDSS_SIG_ACTIVE_HIGH;
-	rfbi.timings.sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE;
-
-	dss_mgr_set_timings(mgr, &rfbi.timings);
+	rfbi.vm.hsync_len = 1;
+	rfbi.vm.hfront_porch = 1;
+	rfbi.vm.hback_porch = 1;
+	rfbi.vm.vsync_len = 1;
+	rfbi.vm.vfront_porch = 0;
+	rfbi.vm.vback_porch = 0;
+
+	rfbi.vm.flags &= ~DISPLAY_FLAGS_INTERLACED;
+	rfbi.vm.flags &= ~DISPLAY_FLAGS_HSYNC_LOW;
+	rfbi.vm.flags |= DISPLAY_FLAGS_HSYNC_HIGH;
+	rfbi.vm.flags &= ~DISPLAY_FLAGS_VSYNC_LOW;
+	rfbi.vm.flags |= DISPLAY_FLAGS_VSYNC_HIGH;
+	rfbi.vm.flags &= ~DISPLAY_FLAGS_PIXDATA_NEGEDGE;
+	rfbi.vm.flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE;
+	rfbi.vm.flags &= ~DISPLAY_FLAGS_DE_LOW;
+	rfbi.vm.flags |= DISPLAY_FLAGS_DE_HIGH;
+	rfbi.vm.flags &= ~DISPLAY_FLAGS_SYNC_POSEDGE;
+	rfbi.vm.flags |= DISPLAY_FLAGS_SYNC_NEGEDGE;
+
+	dss_mgr_set_timings(mgr, &rfbi.vm);
 }
 
 static int rfbi_display_enable(struct omap_dss_device *dssdev)
diff --git a/drivers/gpu/drm/omapdrm/dss/sdi.c b/drivers/gpu/drm/omapdrm/dss/sdi.c
index 0a96c321ce62..b3bda2d3c08d 100644
--- a/drivers/gpu/drm/omapdrm/dss/sdi.c
+++ b/drivers/gpu/drm/omapdrm/dss/sdi.c
@@ -39,7 +39,7 @@ static struct {
 	struct regulator *vdds_sdi_reg;
 
 	struct dss_lcd_mgr_config mgr_config;
-	struct omap_video_timings timings;
+	struct videomode vm;
 	int datapairs;
 
 	struct omap_dss_device output;
@@ -131,7 +131,7 @@ static int sdi_display_enable(struct omap_dss_device *dssdev)
 {
 	struct omap_dss_device *out = &sdi.output;
 	enum omap_channel channel = dssdev->dispc_channel;
-	struct omap_video_timings *t = &sdi.timings;
+	struct videomode *vm = &sdi.vm;
 	unsigned long fck;
 	struct dispc_clock_info dispc_cinfo;
 	unsigned long pck;
@@ -151,10 +151,9 @@ static int sdi_display_enable(struct omap_dss_device *dssdev)
 		goto err_get_dispc;
 
 	/* 15.5.9.1.2 */
-	t->data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
-	t->sync_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
+	vm->flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE | DISPLAY_FLAGS_SYNC_POSEDGE;
 
-	r = sdi_calc_clock_div(t->pixelclock, &fck, &dispc_cinfo);
+	r = sdi_calc_clock_div(vm->pixelclock, &fck, &dispc_cinfo);
 	if (r)
 		goto err_calc_clock_div;
 
@@ -162,15 +161,15 @@ static int sdi_display_enable(struct omap_dss_device *dssdev)
 
 	pck = fck / dispc_cinfo.lck_div / dispc_cinfo.pck_div;
 
-	if (pck != t->pixelclock) {
-		DSSWARN("Could not find exact pixel clock. Requested %d Hz, got %lu Hz\n",
-			t->pixelclock, pck);
+	if (pck != vm->pixelclock) {
+		DSSWARN("Could not find exact pixel clock. Requested %lu Hz, got %lu Hz\n",
+			vm->pixelclock, pck);
 
-		t->pixelclock = pck;
+		vm->pixelclock = pck;
 	}
 
 
-	dss_mgr_set_timings(channel, t);
+	dss_mgr_set_timings(channel, vm);
 
 	r = dss_set_fck_rate(fck);
 	if (r)
@@ -229,26 +228,26 @@ static void sdi_display_disable(struct omap_dss_device *dssdev)
 }
 
 static void sdi_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			    struct videomode *vm)
 {
-	sdi.timings = *timings;
+	sdi.vm = *vm;
 }
 
 static void sdi_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			    struct videomode *vm)
 {
-	*timings = sdi.timings;
+	*vm = sdi.vm;
 }
 
 static int sdi_check_timings(struct omap_dss_device *dssdev,
-			struct omap_video_timings *timings)
+			     struct videomode *vm)
 {
 	enum omap_channel channel = dssdev->dispc_channel;
 
-	if (!dispc_mgr_timings_ok(channel, timings))
+	if (!dispc_mgr_timings_ok(channel, vm))
 		return -EINVAL;
 
-	if (timings->pixelclock == 0)
+	if (vm->pixelclock == 0)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/gpu/drm/omapdrm/dss/venc.c b/drivers/gpu/drm/omapdrm/dss/venc.c
index 6eedf2118708..d74f7fcc2e46 100644
--- a/drivers/gpu/drm/omapdrm/dss/venc.c
+++ b/drivers/gpu/drm/omapdrm/dss/venc.c
@@ -262,47 +262,41 @@ static const struct venc_config venc_config_pal_bdghi = {
 	.fid_ext_start_y__fid_ext_offset_y	= 0x01380005,
 };
 
-const struct omap_video_timings omap_dss_pal_timings = {
-	.x_res		= 720,
-	.y_res		= 574,
+const struct videomode omap_dss_pal_vm = {
+	.hactive	= 720,
+	.vactive	= 574,
 	.pixelclock	= 13500000,
-	.hsw		= 64,
-	.hfp		= 12,
-	.hbp		= 68,
-	.vsw		= 5,
-	.vfp		= 5,
-	.vbp		= 41,
-
-	.interlace	= true,
-
-	.hsync_level = OMAPDSS_SIG_ACTIVE_LOW,
-	.vsync_level = OMAPDSS_SIG_ACTIVE_LOW,
-	.data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE,
-	.de_level = OMAPDSS_SIG_ACTIVE_HIGH,
-	.sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+	.hsync_len	= 64,
+	.hfront_porch	= 12,
+	.hback_porch	= 68,
+	.vsync_len	= 5,
+	.vfront_porch	= 5,
+	.vback_porch	= 41,
+
+	.flags		= DISPLAY_FLAGS_INTERLACED | DISPLAY_FLAGS_HSYNC_LOW |
+			  DISPLAY_FLAGS_VSYNC_LOW | DISPLAY_FLAGS_DE_HIGH |
+			  DISPLAY_FLAGS_PIXDATA_POSEDGE |
+			  DISPLAY_FLAGS_SYNC_NEGEDGE,
 };
-EXPORT_SYMBOL(omap_dss_pal_timings);
+EXPORT_SYMBOL(omap_dss_pal_vm);
 
-const struct omap_video_timings omap_dss_ntsc_timings = {
-	.x_res		= 720,
-	.y_res		= 482,
+const struct videomode omap_dss_ntsc_vm = {
+	.hactive	= 720,
+	.vactive	= 482,
 	.pixelclock	= 13500000,
-	.hsw		= 64,
-	.hfp		= 16,
-	.hbp		= 58,
-	.vsw		= 6,
-	.vfp		= 6,
-	.vbp		= 31,
-
-	.interlace	= true,
-
-	.hsync_level = OMAPDSS_SIG_ACTIVE_LOW,
-	.vsync_level = OMAPDSS_SIG_ACTIVE_LOW,
-	.data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE,
-	.de_level = OMAPDSS_SIG_ACTIVE_HIGH,
-	.sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+	.hsync_len	= 64,
+	.hfront_porch	= 16,
+	.hback_porch	= 58,
+	.vsync_len	= 6,
+	.vfront_porch	= 6,
+	.vback_porch	= 31,
+
+	.flags		= DISPLAY_FLAGS_INTERLACED | DISPLAY_FLAGS_HSYNC_LOW |
+			  DISPLAY_FLAGS_VSYNC_LOW | DISPLAY_FLAGS_DE_HIGH |
+			  DISPLAY_FLAGS_PIXDATA_POSEDGE |
+			  DISPLAY_FLAGS_SYNC_NEGEDGE,
 };
-EXPORT_SYMBOL(omap_dss_ntsc_timings);
+EXPORT_SYMBOL(omap_dss_ntsc_vm);
 
 static struct {
 	struct platform_device *pdev;
@@ -313,7 +307,7 @@ static struct {
 
 	struct clk	*tv_dac_clk;
 
-	struct omap_video_timings timings;
+	struct videomode vm;
 	enum omap_dss_venc_type type;
 	bool invert_polarity;
 
@@ -427,13 +421,12 @@ static void venc_runtime_put(void)
 	WARN_ON(r < 0 && r != -ENOSYS);
 }
 
-static const struct venc_config *venc_timings_to_config(
-		struct omap_video_timings *timings)
+static const struct venc_config *venc_timings_to_config(struct videomode *vm)
 {
-	if (memcmp(&omap_dss_pal_timings, timings, sizeof(*timings)) == 0)
+	if (memcmp(&omap_dss_pal_vm, vm, sizeof(*vm)) == 0)
 		return &venc_config_pal_trm;
 
-	if (memcmp(&omap_dss_ntsc_timings, timings, sizeof(*timings)) == 0)
+	if (memcmp(&omap_dss_ntsc_vm, vm, sizeof(*vm)) == 0)
 		return &venc_config_ntsc_trm;
 
 	BUG();
@@ -451,7 +444,7 @@ static int venc_power_on(struct omap_dss_device *dssdev)
 		goto err0;
 
 	venc_reset();
-	venc_write_config(venc_timings_to_config(&venc.timings));
+	venc_write_config(venc_timings_to_config(&venc.vm));
 
 	dss_set_venc_output(venc.type);
 	dss_set_dac_pwrdn_bgz(1);
@@ -468,7 +461,7 @@ static int venc_power_on(struct omap_dss_device *dssdev)
 
 	venc_write_reg(VENC_OUTPUT_CONTROL, l);
 
-	dss_mgr_set_timings(channel, &venc.timings);
+	dss_mgr_set_timings(channel, &venc.vm);
 
 	r = regulator_enable(venc.vdda_dac_reg);
 	if (r)
@@ -546,17 +539,17 @@ static void venc_display_disable(struct omap_dss_device *dssdev)
 }
 
 static void venc_set_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			     struct videomode *vm)
 {
 	DSSDBG("venc_set_timings\n");
 
 	mutex_lock(&venc.venc_lock);
 
 	/* Reset WSS data when the TV standard changes. */
-	if (memcmp(&venc.timings, timings, sizeof(*timings)))
+	if (memcmp(&venc.vm, vm, sizeof(*vm)))
 		venc.wss_data = 0;
 
-	venc.timings = *timings;
+	venc.vm = *vm;
 
 	dispc_set_tv_pclk(13500000);
 
@@ -564,25 +557,25 @@ static void venc_set_timings(struct omap_dss_device *dssdev,
 }
 
 static int venc_check_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			      struct videomode *vm)
 {
 	DSSDBG("venc_check_timings\n");
 
-	if (memcmp(&omap_dss_pal_timings, timings, sizeof(*timings)) == 0)
+	if (memcmp(&omap_dss_pal_vm, vm, sizeof(*vm)) == 0)
 		return 0;
 
-	if (memcmp(&omap_dss_ntsc_timings, timings, sizeof(*timings)) == 0)
+	if (memcmp(&omap_dss_ntsc_vm, vm, sizeof(*vm)) == 0)
 		return 0;
 
 	return -EINVAL;
 }
 
 static void venc_get_timings(struct omap_dss_device *dssdev,
-		struct omap_video_timings *timings)
+			     struct videomode *vm)
 {
 	mutex_lock(&venc.venc_lock);
 
-	*timings = venc.timings;
+	*vm = venc.vm;
 
 	mutex_unlock(&venc.venc_lock);
 }
@@ -602,7 +595,7 @@ static int venc_set_wss(struct omap_dss_device *dssdev, u32 wss)
 
 	mutex_lock(&venc.venc_lock);
 
-	config = venc_timings_to_config(&venc.timings);
+	config = venc_timings_to_config(&venc.vm);
 
 	/* Invert due to VENC_L21_WC_CTL:INV=1 */
 	venc.wss_data = (wss ^ 0xfffff) << 8;
diff --git a/drivers/gpu/drm/omapdrm/omap_connector.c b/drivers/gpu/drm/omapdrm/omap_connector.c
index 137fe690a0da..2580e8673908 100644
--- a/drivers/gpu/drm/omapdrm/omap_connector.c
+++ b/drivers/gpu/drm/omapdrm/omap_connector.c
@@ -42,73 +42,6 @@ bool omap_connector_get_hdmi_mode(struct drm_connector *connector)
 	return omap_connector->hdmi_mode;
 }
 
-void copy_timings_omap_to_drm(struct drm_display_mode *mode,
-		struct omap_video_timings *timings)
-{
-	mode->clock = timings->pixelclock / 1000;
-
-	mode->hdisplay = timings->x_res;
-	mode->hsync_start = mode->hdisplay + timings->hfp;
-	mode->hsync_end = mode->hsync_start + timings->hsw;
-	mode->htotal = mode->hsync_end + timings->hbp;
-
-	mode->vdisplay = timings->y_res;
-	mode->vsync_start = mode->vdisplay + timings->vfp;
-	mode->vsync_end = mode->vsync_start + timings->vsw;
-	mode->vtotal = mode->vsync_end + timings->vbp;
-
-	mode->flags = 0;
-
-	if (timings->interlace)
-		mode->flags |= DRM_MODE_FLAG_INTERLACE;
-
-	if (timings->double_pixel)
-		mode->flags |= DRM_MODE_FLAG_DBLCLK;
-
-	if (timings->hsync_level == OMAPDSS_SIG_ACTIVE_HIGH)
-		mode->flags |= DRM_MODE_FLAG_PHSYNC;
-	else
-		mode->flags |= DRM_MODE_FLAG_NHSYNC;
-
-	if (timings->vsync_level == OMAPDSS_SIG_ACTIVE_HIGH)
-		mode->flags |= DRM_MODE_FLAG_PVSYNC;
-	else
-		mode->flags |= DRM_MODE_FLAG_NVSYNC;
-}
-
-void copy_timings_drm_to_omap(struct omap_video_timings *timings,
-		struct drm_display_mode *mode)
-{
-	timings->pixelclock = mode->clock * 1000;
-
-	timings->x_res = mode->hdisplay;
-	timings->hfp = mode->hsync_start - mode->hdisplay;
-	timings->hsw = mode->hsync_end - mode->hsync_start;
-	timings->hbp = mode->htotal - mode->hsync_end;
-
-	timings->y_res = mode->vdisplay;
-	timings->vfp = mode->vsync_start - mode->vdisplay;
-	timings->vsw = mode->vsync_end - mode->vsync_start;
-	timings->vbp = mode->vtotal - mode->vsync_end;
-
-	timings->interlace = !!(mode->flags & DRM_MODE_FLAG_INTERLACE);
-	timings->double_pixel = !!(mode->flags & DRM_MODE_FLAG_DBLCLK);
-
-	if (mode->flags & DRM_MODE_FLAG_PHSYNC)
-		timings->hsync_level = OMAPDSS_SIG_ACTIVE_HIGH;
-	else
-		timings->hsync_level = OMAPDSS_SIG_ACTIVE_LOW;
-
-	if (mode->flags & DRM_MODE_FLAG_PVSYNC)
-		timings->vsync_level = OMAPDSS_SIG_ACTIVE_HIGH;
-	else
-		timings->vsync_level = OMAPDSS_SIG_ACTIVE_LOW;
-
-	timings->data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE;
-	timings->de_level = OMAPDSS_SIG_ACTIVE_HIGH;
-	timings->sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE;
-}
-
 static enum drm_connector_status omap_connector_detect(
 		struct drm_connector *connector, bool force)
 {
@@ -185,11 +118,11 @@ static int omap_connector_get_modes(struct drm_connector *connector)
 		kfree(edid);
 	} else {
 		struct drm_display_mode *mode = drm_mode_create(dev);
-		struct omap_video_timings timings = {0};
+		struct videomode vm = {0};
 
-		dssdrv->get_timings(dssdev, &timings);
+		dssdrv->get_timings(dssdev, &vm);
 
-		copy_timings_omap_to_drm(mode, &timings);
+		drm_display_mode_from_videomode(&vm, mode);
 
 		mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
 		drm_mode_set_name(mode);
@@ -207,12 +140,14 @@ static int omap_connector_mode_valid(struct drm_connector *connector,
 	struct omap_connector *omap_connector = to_omap_connector(connector);
 	struct omap_dss_device *dssdev = omap_connector->dssdev;
 	struct omap_dss_driver *dssdrv = dssdev->driver;
-	struct omap_video_timings timings = {0};
+	struct videomode vm = {0};
 	struct drm_device *dev = connector->dev;
 	struct drm_display_mode *new_mode;
 	int r, ret = MODE_BAD;
 
-	copy_timings_drm_to_omap(&timings, mode);
+	drm_display_mode_to_videomode(mode, &vm);
+	vm.flags |= DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_PIXDATA_POSEDGE |
+		    DISPLAY_FLAGS_SYNC_NEGEDGE;
 	mode->vrefresh = drm_mode_vrefresh(mode);
 
 	/*
@@ -221,13 +156,13 @@ static int omap_connector_mode_valid(struct drm_connector *connector,
 	 * panel's timings
 	 */
 	if (dssdrv->check_timings) {
-		r = dssdrv->check_timings(dssdev, &timings);
+		r = dssdrv->check_timings(dssdev, &vm);
 	} else {
-		struct omap_video_timings t = {0};
+		struct videomode t = {0};
 
 		dssdrv->get_timings(dssdev, &t);
 
-		if (memcmp(&timings, &t, sizeof(struct omap_video_timings)))
+		if (memcmp(&vm, &t, sizeof(struct videomode)))
 			r = -EINVAL;
 		else
 			r = 0;
@@ -236,7 +171,7 @@ static int omap_connector_mode_valid(struct drm_connector *connector,
 	if (!r) {
 		/* check if vrefresh is still valid */
 		new_mode = drm_mode_duplicate(dev, mode);
-		new_mode->clock = timings.pixelclock / 1000;
+		new_mode->clock = vm.pixelclock / 1000;
 		new_mode->vrefresh = 0;
 		if (mode->vrefresh == drm_mode_vrefresh(new_mode))
 			ret = MODE_OK;
diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c
index 180f644e861e..8dea89030e66 100644
--- a/drivers/gpu/drm/omapdrm/omap_crtc.c
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.c
@@ -34,7 +34,7 @@ struct omap_crtc {
 	const char *name;
 	enum omap_channel channel;
 
-	struct omap_video_timings timings;
+	struct videomode vm;
 
 	struct omap_drm_irq vblank_irq;
 	struct omap_drm_irq error_irq;
@@ -56,10 +56,10 @@ uint32_t pipe2vbl(struct drm_crtc *crtc)
 	return dispc_mgr_get_vsync_irq(omap_crtc->channel);
 }
 
-struct omap_video_timings *omap_crtc_timings(struct drm_crtc *crtc)
+struct videomode *omap_crtc_timings(struct drm_crtc *crtc)
 {
 	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
-	return &omap_crtc->timings;
+	return &omap_crtc->vm;
 }
 
 enum omap_channel omap_crtc_channel(struct drm_crtc *crtc)
@@ -201,7 +201,7 @@ static int omap_crtc_dss_enable(enum omap_channel channel)
 
 	dispc_mgr_setup(omap_crtc->channel, &info);
 	dispc_mgr_set_timings(omap_crtc->channel,
-			&omap_crtc->timings);
+			&omap_crtc->vm);
 	omap_crtc_set_enabled(&omap_crtc->base, true);
 
 	return 0;
@@ -215,11 +215,11 @@ static void omap_crtc_dss_disable(enum omap_channel channel)
 }
 
 static void omap_crtc_dss_set_timings(enum omap_channel channel,
-		const struct omap_video_timings *timings)
+		const struct videomode *vm)
 {
 	struct omap_crtc *omap_crtc = omap_crtcs[channel];
 	DBG("%s", omap_crtc->name);
-	omap_crtc->timings = *timings;
+	omap_crtc->vm = *vm;
 }
 
 static void omap_crtc_dss_set_lcd_config(enum omap_channel channel,
@@ -369,7 +369,10 @@ static void omap_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	    mode->vdisplay, mode->vsync_start, mode->vsync_end, mode->vtotal,
 	    mode->type, mode->flags);
 
-	copy_timings_drm_to_omap(&omap_crtc->timings, mode);
+	drm_display_mode_to_videomode(mode, &omap_crtc->vm);
+	omap_crtc->vm.flags |= DISPLAY_FLAGS_DE_HIGH |
+			       DISPLAY_FLAGS_PIXDATA_POSEDGE |
+			       DISPLAY_FLAGS_SYNC_NEGEDGE;
 }
 
 static int omap_crtc_atomic_check(struct drm_crtc *crtc,
@@ -411,19 +414,6 @@ static void omap_crtc_atomic_flush(struct drm_crtc *crtc,
 		dispc_mgr_set_gamma(omap_crtc->channel, lut, length);
 	}
 
-	if (crtc->state->color_mgmt_changed) {
-		struct drm_color_lut *lut = NULL;
-		uint length = 0;
-
-		if (crtc->state->gamma_lut) {
-			lut = (struct drm_color_lut *)
-				crtc->state->gamma_lut->data;
-			length = crtc->state->gamma_lut->length /
-				sizeof(*lut);
-		}
-		dispc_mgr_set_gamma(omap_crtc->channel, lut, length);
-	}
-
 	if (dispc_mgr_is_enabled(omap_crtc->channel)) {
 
 		DBG("%s: GO", omap_crtc->name);
@@ -438,13 +428,14 @@ static void omap_crtc_atomic_flush(struct drm_crtc *crtc,
 	}
 }
 
-static bool omap_crtc_is_plane_prop(struct drm_device *dev,
+static bool omap_crtc_is_plane_prop(struct drm_crtc *crtc,
 	struct drm_property *property)
 {
+	struct drm_device *dev = crtc->dev;
 	struct omap_drm_private *priv = dev->dev_private;
 
 	return property == priv->zorder_prop ||
-		property == dev->mode_config.rotation_property;
+		property == crtc->primary->rotation_property;
 }
 
 static int omap_crtc_atomic_set_property(struct drm_crtc *crtc,
@@ -452,9 +443,7 @@ static int omap_crtc_atomic_set_property(struct drm_crtc *crtc,
 					 struct drm_property *property,
 					 uint64_t val)
 {
-	struct drm_device *dev = crtc->dev;
-
-	if (omap_crtc_is_plane_prop(dev, property)) {
+	if (omap_crtc_is_plane_prop(crtc, property)) {
 		struct drm_plane_state *plane_state;
 		struct drm_plane *plane = crtc->primary;
 
@@ -479,9 +468,7 @@ static int omap_crtc_atomic_get_property(struct drm_crtc *crtc,
 					 struct drm_property *property,
 					 uint64_t *val)
 {
-	struct drm_device *dev = crtc->dev;
-
-	if (omap_crtc_is_plane_prop(dev, property)) {
+	if (omap_crtc_is_plane_prop(crtc, property)) {
 		/*
 		 * Delegate property get to the primary plane. The
 		 * drm_atomic_plane_get_property() function isn't exported, but
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index e1cfba51cff6..fdc83cbcde61 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -105,7 +105,7 @@ static void omap_atomic_complete(struct omap_atomic_state_commit *commit)
 
 	dispc_runtime_put();
 
-	drm_atomic_state_free(old_state);
+	drm_atomic_state_put(old_state);
 
 	/* Complete the commit, wake up any waiter. */
 	spin_lock(&priv->commit.lock);
@@ -176,6 +176,7 @@ static int omap_atomic_commit(struct drm_device *dev,
 	/* Swap the state, this is the point of no return. */
 	drm_atomic_helper_swap_state(state, true);
 
+	drm_atomic_state_get(state);
 	if (nonblock)
 		schedule_work(&commit->work);
 	else
@@ -266,13 +267,15 @@ cleanup:
 }
 
 static int omap_modeset_create_crtc(struct drm_device *dev, int id,
-				    enum omap_channel channel)
+				    enum omap_channel channel,
+				    u32 possible_crtcs)
 {
 	struct omap_drm_private *priv = dev->dev_private;
 	struct drm_plane *plane;
 	struct drm_crtc *crtc;
 
-	plane = omap_plane_init(dev, id, DRM_PLANE_TYPE_PRIMARY);
+	plane = omap_plane_init(dev, id, DRM_PLANE_TYPE_PRIMARY,
+		possible_crtcs);
 	if (IS_ERR(plane))
 		return PTR_ERR(plane);
 
@@ -292,16 +295,6 @@ static int omap_modeset_init_properties(struct drm_device *dev)
 {
 	struct omap_drm_private *priv = dev->dev_private;
 
-	if (priv->has_dmm) {
-		dev->mode_config.rotation_property =
-			drm_mode_create_rotation_property(dev,
-				DRM_ROTATE_0 | DRM_ROTATE_90 |
-				DRM_ROTATE_180 | DRM_ROTATE_270 |
-				DRM_REFLECT_X | DRM_REFLECT_Y);
-		if (!dev->mode_config.rotation_property)
-			return -ENOMEM;
-	}
-
 	priv->zorder_prop = drm_property_create_range(dev, 0, "zorder", 0, 3);
 	if (!priv->zorder_prop)
 		return -ENOMEM;
@@ -318,6 +311,7 @@ static int omap_modeset_init(struct drm_device *dev)
 	int num_crtcs;
 	int i, id = 0;
 	int ret;
+	u32 possible_crtcs;
 
 	drm_mode_config_init(dev);
 
@@ -334,6 +328,7 @@ static int omap_modeset_init(struct drm_device *dev)
 	 * We use the num_crtc argument to limit the number of crtcs we create.
 	 */
 	num_crtcs = min3(num_crtc, num_mgrs, num_ovls);
+	possible_crtcs = (1 << num_crtcs) - 1;
 
 	dssdev = NULL;
 
@@ -397,7 +392,8 @@ static int omap_modeset_init(struct drm_device *dev)
 		 * allocated crtc, we create a new crtc for it
 		 */
 		if (!channel_used(dev, channel)) {
-			ret = omap_modeset_create_crtc(dev, id, channel);
+			ret = omap_modeset_create_crtc(dev, id, channel,
+				possible_crtcs);
 			if (ret < 0) {
 				dev_err(dev->dev,
 					"could not create CRTC (channel %u)\n",
@@ -427,7 +423,8 @@ static int omap_modeset_init(struct drm_device *dev)
 			return -ENOMEM;
 		}
 
-		ret = omap_modeset_create_crtc(dev, id, i);
+		ret = omap_modeset_create_crtc(dev, id, i,
+			possible_crtcs);
 		if (ret < 0) {
 			dev_err(dev->dev,
 				"could not create CRTC (channel %u)\n", i);
@@ -441,7 +438,8 @@ static int omap_modeset_init(struct drm_device *dev)
 	for (; id < num_ovls; id++) {
 		struct drm_plane *plane;
 
-		plane = omap_plane_init(dev, id, DRM_PLANE_TYPE_OVERLAY);
+		plane = omap_plane_init(dev, id, DRM_PLANE_TYPE_OVERLAY,
+			possible_crtcs);
 		if (IS_ERR(plane))
 			return PTR_ERR(plane);
 
@@ -752,22 +750,32 @@ static void dev_lastclose(struct drm_device *dev)
 
 	DBG("lastclose: dev=%p", dev);
 
-	if (dev->mode_config.rotation_property) {
-		/* need to restore default rotation state.. not sure
-		 * if there is a cleaner way to restore properties to
-		 * default state?  Maybe a flag that properties should
-		 * automatically be restored to default state on
-		 * lastclose?
-		 */
-		for (i = 0; i < priv->num_crtcs; i++) {
-			drm_object_property_set_value(&priv->crtcs[i]->base,
-					dev->mode_config.rotation_property, 0);
-		}
+	/* need to restore default rotation state.. not sure
+	 * if there is a cleaner way to restore properties to
+	 * default state?  Maybe a flag that properties should
+	 * automatically be restored to default state on
+	 * lastclose?
+	 */
+	for (i = 0; i < priv->num_crtcs; i++) {
+		struct drm_crtc *crtc = priv->crtcs[i];
 
-		for (i = 0; i < priv->num_planes; i++) {
-			drm_object_property_set_value(&priv->planes[i]->base,
-					dev->mode_config.rotation_property, 0);
-		}
+		if (!crtc->primary->rotation_property)
+			continue;
+
+		drm_object_property_set_value(&crtc->base,
+					      crtc->primary->rotation_property,
+					      DRM_ROTATE_0);
+	}
+
+	for (i = 0; i < priv->num_planes; i++) {
+		struct drm_plane *plane = priv->planes[i];
+
+		if (!plane->rotation_property)
+			continue;
+
+		drm_object_property_set_value(&plane->base,
+					      plane->rotation_property,
+					      DRM_ROTATE_0);
 	}
 
 	if (priv->fbdev) {
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.h b/drivers/gpu/drm/omapdrm/omap_drv.h
index dcc30a98b9d4..7d9dd5400cef 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.h
+++ b/drivers/gpu/drm/omapdrm/omap_drv.h
@@ -148,7 +148,7 @@ static inline void omap_fbdev_free(struct drm_device *dev)
 }
 #endif
 
-struct omap_video_timings *omap_crtc_timings(struct drm_crtc *crtc);
+struct videomode *omap_crtc_timings(struct drm_crtc *crtc);
 enum omap_channel omap_crtc_channel(struct drm_crtc *crtc);
 void omap_crtc_pre_init(void);
 void omap_crtc_pre_uninit(void);
@@ -157,7 +157,8 @@ struct drm_crtc *omap_crtc_init(struct drm_device *dev,
 int omap_crtc_wait_pending(struct drm_crtc *crtc);
 
 struct drm_plane *omap_plane_init(struct drm_device *dev,
-		int id, enum drm_plane_type type);
+		int id, enum drm_plane_type type,
+		u32 possible_crtcs);
 void omap_plane_install_properties(struct drm_plane *plane,
 		struct drm_mode_object *obj);
 
@@ -171,11 +172,6 @@ struct drm_encoder *omap_connector_attached_encoder(
 		struct drm_connector *connector);
 bool omap_connector_get_hdmi_mode(struct drm_connector *connector);
 
-void copy_timings_omap_to_drm(struct drm_display_mode *mode,
-		struct omap_video_timings *timings);
-void copy_timings_drm_to_omap(struct omap_video_timings *timings,
-		struct drm_display_mode *mode);
-
 uint32_t omap_framebuffer_get_formats(uint32_t *pixel_formats,
 		uint32_t max_formats, enum omap_color_mode supported_modes);
 struct drm_framebuffer *omap_framebuffer_create(struct drm_device *dev,
diff --git a/drivers/gpu/drm/omapdrm/omap_encoder.c b/drivers/gpu/drm/omapdrm/omap_encoder.c
index 0bbb9c59622e..a20f30039aee 100644
--- a/drivers/gpu/drm/omapdrm/omap_encoder.c
+++ b/drivers/gpu/drm/omapdrm/omap_encoder.c
@@ -102,7 +102,7 @@ static void omap_encoder_disable(struct drm_encoder *encoder)
 
 static int omap_encoder_update(struct drm_encoder *encoder,
 			       enum omap_channel channel,
-			       struct omap_video_timings *timings)
+			       struct videomode *vm)
 {
 	struct drm_device *dev = encoder->dev;
 	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
@@ -111,13 +111,13 @@ static int omap_encoder_update(struct drm_encoder *encoder,
 	int ret;
 
 	if (dssdrv->check_timings) {
-		ret = dssdrv->check_timings(dssdev, timings);
+		ret = dssdrv->check_timings(dssdev, vm);
 	} else {
-		struct omap_video_timings t = {0};
+		struct videomode t = {0};
 
 		dssdrv->get_timings(dssdev, &t);
 
-		if (memcmp(timings, &t, sizeof(struct omap_video_timings)))
+		if (memcmp(vm, &t, sizeof(struct videomode)))
 			ret = -EINVAL;
 		else
 			ret = 0;
@@ -129,7 +129,7 @@ static int omap_encoder_update(struct drm_encoder *encoder,
 	}
 
 	if (dssdrv->set_timings)
-		dssdrv->set_timings(dssdev, timings);
+		dssdrv->set_timings(dssdev, vm);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c
index adb10fbe918d..8d8ac173f55d 100644
--- a/drivers/gpu/drm/omapdrm/omap_fbdev.c
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c
@@ -82,6 +82,7 @@ fallback:
 
 static struct fb_ops omap_fb_ops = {
 	.owner = THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
 
 	/* Note: to properly handle manual update displays, we wrap the
 	 * basic fbdev ops which write to the framebuffer
@@ -92,11 +93,7 @@ static struct fb_ops omap_fb_ops = {
 	.fb_copyarea = drm_fb_helper_sys_copyarea,
 	.fb_imageblit = drm_fb_helper_sys_imageblit,
 
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
 	.fb_pan_display = omap_fbdev_pan_display,
-	.fb_blank = drm_fb_helper_blank,
-	.fb_setcmap = drm_fb_helper_setcmap,
 };
 
 static int omap_fbdev_create(struct drm_fb_helper *helper,
diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c
index 505dee0db973..4a90c690f09e 100644
--- a/drivers/gpu/drm/omapdrm/omap_gem.c
+++ b/drivers/gpu/drm/omapdrm/omap_gem.c
@@ -336,8 +336,10 @@ static void omap_gem_detach_pages(struct drm_gem_object *obj)
 	if (omap_obj->flags & (OMAP_BO_WC|OMAP_BO_UNCACHED)) {
 		int i, npages = obj->size >> PAGE_SHIFT;
 		for (i = 0; i < npages; i++) {
-			dma_unmap_page(obj->dev->dev, omap_obj->addrs[i],
-					PAGE_SIZE, DMA_BIDIRECTIONAL);
+			if (omap_obj->addrs[i])
+				dma_unmap_page(obj->dev->dev,
+					       omap_obj->addrs[i],
+					       PAGE_SIZE, DMA_BIDIRECTIONAL);
 		}
 	}
 
@@ -396,8 +398,7 @@ static int fault_1d(struct drm_gem_object *obj,
 	pgoff_t pgoff;
 
 	/* We don't use vmf->pgoff since that has the fake offset: */
-	pgoff = ((unsigned long)vmf->virtual_address -
-			vma->vm_start) >> PAGE_SHIFT;
+	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
 
 	if (omap_obj->pages) {
 		omap_gem_cpu_sync(obj, pgoff);
@@ -407,11 +408,10 @@ static int fault_1d(struct drm_gem_object *obj,
 		pfn = (omap_obj->paddr >> PAGE_SHIFT) + pgoff;
 	}
 
-	VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
+	VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address,
 			pfn, pfn << PAGE_SHIFT);
 
-	return vm_insert_mixed(vma, (unsigned long)vmf->virtual_address,
-			__pfn_to_pfn_t(pfn, PFN_DEV));
+	return vm_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV));
 }
 
 /* Special handling for the case of faulting in 2d tiled buffers */
@@ -425,7 +425,7 @@ static int fault_2d(struct drm_gem_object *obj,
 	struct page *pages[64];  /* XXX is this too much to have on stack? */
 	unsigned long pfn;
 	pgoff_t pgoff, base_pgoff;
-	void __user *vaddr;
+	unsigned long vaddr;
 	int i, ret, slots;
 
 	/*
@@ -445,8 +445,7 @@ static int fault_2d(struct drm_gem_object *obj,
 	const int m = 1 + ((omap_obj->width << fmt) / PAGE_SIZE);
 
 	/* We don't use vmf->pgoff since that has the fake offset: */
-	pgoff = ((unsigned long)vmf->virtual_address -
-			vma->vm_start) >> PAGE_SHIFT;
+	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
 
 	/*
 	 * Actual address we start mapping at is rounded down to previous slot
@@ -457,7 +456,7 @@ static int fault_2d(struct drm_gem_object *obj,
 	/* figure out buffer width in slots */
 	slots = omap_obj->width >> priv->usergart[fmt].slot_shift;
 
-	vaddr = vmf->virtual_address - ((pgoff - base_pgoff) << PAGE_SHIFT);
+	vaddr = vmf->address - ((pgoff - base_pgoff) << PAGE_SHIFT);
 
 	entry = &priv->usergart[fmt].entry[priv->usergart[fmt].last];
 
@@ -501,12 +500,11 @@ static int fault_2d(struct drm_gem_object *obj,
 
 	pfn = entry->paddr >> PAGE_SHIFT;
 
-	VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
+	VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address,
 			pfn, pfn << PAGE_SHIFT);
 
 	for (i = n; i > 0; i--) {
-		vm_insert_mixed(vma, (unsigned long)vaddr,
-				__pfn_to_pfn_t(pfn, PFN_DEV));
+		vm_insert_mixed(vma, vaddr, __pfn_to_pfn_t(pfn, PFN_DEV));
 		pfn += priv->usergart[fmt].stride_pfn;
 		vaddr += PAGE_SIZE * m;
 	}
diff --git a/drivers/gpu/drm/omapdrm/omap_plane.c b/drivers/gpu/drm/omapdrm/omap_plane.c
index 66ac8c40db26..82b2c23d6769 100644
--- a/drivers/gpu/drm/omapdrm/omap_plane.c
+++ b/drivers/gpu/drm/omapdrm/omap_plane.c
@@ -108,16 +108,12 @@ static void omap_plane_atomic_update(struct drm_plane *plane,
 	win.src_x = state->src_x >> 16;
 	win.src_y = state->src_y >> 16;
 
-	switch (state->rotation & DRM_ROTATE_MASK) {
-	case DRM_ROTATE_90:
-	case DRM_ROTATE_270:
+	if (drm_rotation_90_or_270(state->rotation)) {
 		win.src_w = state->src_h >> 16;
 		win.src_h = state->src_w >> 16;
-		break;
-	default:
+	} else {
 		win.src_w = state->src_w >> 16;
 		win.src_h = state->src_h >> 16;
-		break;
 	}
 
 	/* update scanout: */
@@ -135,7 +131,9 @@ static void omap_plane_atomic_update(struct drm_plane *plane,
 	/* and finally, update omapdss: */
 	ret = dispc_ovl_setup(omap_plane->id, &info, false,
 			      omap_crtc_timings(state->crtc), false);
-	if (WARN_ON(ret)) {
+	if (ret) {
+		dev_err(plane->dev->dev, "Failed to setup plane %s\n",
+			omap_plane->name);
 		dispc_ovl_enable(omap_plane->id, false);
 		return;
 	}
@@ -161,12 +159,20 @@ static int omap_plane_atomic_check(struct drm_plane *plane,
 {
 	struct drm_crtc_state *crtc_state;
 
-	if (!state->crtc)
+	if (!state->fb)
 		return 0;
 
-	crtc_state = drm_atomic_get_crtc_state(state->state, state->crtc);
-	if (IS_ERR(crtc_state))
-		return PTR_ERR(crtc_state);
+	/* crtc should only be NULL when disabling (i.e., !state->fb) */
+	if (WARN_ON(!state->crtc))
+		return 0;
+
+	crtc_state = drm_atomic_get_existing_crtc_state(state->state, state->crtc);
+	/* we should have a crtc state if the plane is attached to a crtc */
+	if (WARN_ON(!crtc_state))
+		return 0;
+
+	if (!crtc_state->enable)
+		return 0;
 
 	if (state->crtc_x < 0 || state->crtc_y < 0)
 		return -EINVAL;
@@ -177,11 +183,9 @@ static int omap_plane_atomic_check(struct drm_plane *plane,
 	if (state->crtc_y + state->crtc_h > crtc_state->adjusted_mode.vdisplay)
 		return -EINVAL;
 
-	if (state->fb) {
-		if (state->rotation != DRM_ROTATE_0 &&
-		    !omap_framebuffer_supports_rotation(state->fb))
-			return -EINVAL;
-	}
+	if (state->rotation != DRM_ROTATE_0 &&
+	    !omap_framebuffer_supports_rotation(state->fb))
+		return -EINVAL;
 
 	return 0;
 }
@@ -215,9 +219,17 @@ void omap_plane_install_properties(struct drm_plane *plane,
 	struct omap_drm_private *priv = dev->dev_private;
 
 	if (priv->has_dmm) {
-		struct drm_property *prop = dev->mode_config.rotation_property;
-
-		drm_object_attach_property(obj, prop, 0);
+		if (!plane->rotation_property)
+			drm_plane_create_rotation_property(plane,
+							   DRM_ROTATE_0,
+							   DRM_ROTATE_0 | DRM_ROTATE_90 |
+							   DRM_ROTATE_180 | DRM_ROTATE_270 |
+							   DRM_REFLECT_X | DRM_REFLECT_Y);
+
+		/* Attach the rotation property also to the crtc object */
+		if (plane->rotation_property && obj != &plane->base)
+			drm_object_attach_property(obj, plane->rotation_property,
+						   DRM_ROTATE_0);
 	}
 
 	drm_object_attach_property(obj, priv->zorder_prop, 0);
@@ -344,9 +356,9 @@ static const uint32_t error_irqs[] = {
 
 /* initialize plane */
 struct drm_plane *omap_plane_init(struct drm_device *dev,
-		int id, enum drm_plane_type type)
+		int id, enum drm_plane_type type,
+		u32 possible_crtcs)
 {
-	struct omap_drm_private *priv = dev->dev_private;
 	struct drm_plane *plane;
 	struct omap_plane *omap_plane;
 	int ret;
@@ -369,7 +381,7 @@ struct drm_plane *omap_plane_init(struct drm_device *dev,
 	omap_plane->error_irq.irq = omap_plane_error_irq;
 	omap_irq_register(dev, &omap_plane->error_irq);
 
-	ret = drm_universal_plane_init(dev, plane, (1 << priv->num_crtcs) - 1,
+	ret = drm_universal_plane_init(dev, plane, possible_crtcs,
 				       &omap_plane_funcs, omap_plane->formats,
 				       omap_plane->nformats, type, NULL);
 	if (ret < 0)
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 113db3c4a633..06aaf79de8c8 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -120,7 +120,7 @@ static int panel_simple_get_fixed_modes(struct panel_simple *panel)
 
 		mode->type |= DRM_MODE_TYPE_DRIVER;
 
-		if (panel->desc->num_modes == 1)
+		if (panel->desc->num_timings == 1)
 			mode->type |= DRM_MODE_TYPE_PREFERRED;
 
 		drm_mode_probed_add(connector, mode);
@@ -555,6 +555,91 @@ static const struct panel_desc auo_b133htn01 = {
 	},
 };
 
+static const struct display_timing auo_g133han01_timings = {
+	.pixelclock = { 134000000, 141200000, 149000000 },
+	.hactive = { 1920, 1920, 1920 },
+	.hfront_porch = { 39, 58, 77 },
+	.hback_porch = { 59, 88, 117 },
+	.hsync_len = { 28, 42, 56 },
+	.vactive = { 1080, 1080, 1080 },
+	.vfront_porch = { 3, 8, 11 },
+	.vback_porch = { 5, 14, 19 },
+	.vsync_len = { 4, 14, 19 },
+};
+
+static const struct panel_desc auo_g133han01 = {
+	.timings = &auo_g133han01_timings,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 293,
+		.height = 165,
+	},
+	.delay = {
+		.prepare = 200,
+		.enable = 50,
+		.disable = 50,
+		.unprepare = 1000,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_JEIDA,
+};
+
+static const struct display_timing auo_g185han01_timings = {
+	.pixelclock = { 120000000, 144000000, 175000000 },
+	.hactive = { 1920, 1920, 1920 },
+	.hfront_porch = { 18, 60, 74 },
+	.hback_porch = { 12, 44, 54 },
+	.hsync_len = { 10, 24, 32 },
+	.vactive = { 1080, 1080, 1080 },
+	.vfront_porch = { 6, 10, 40 },
+	.vback_porch = { 2, 5, 20 },
+	.vsync_len = { 2, 5, 20 },
+};
+
+static const struct panel_desc auo_g185han01 = {
+	.timings = &auo_g185han01_timings,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 409,
+		.height = 230,
+	},
+	.delay = {
+		.prepare = 50,
+		.enable = 200,
+		.disable = 110,
+		.unprepare = 1000,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+};
+
+static const struct drm_display_mode auo_t215hvn01_mode = {
+	.clock = 148800,
+	.hdisplay = 1920,
+	.hsync_start = 1920 + 88,
+	.hsync_end = 1920 + 88 + 44,
+	.htotal = 1920 + 88 + 44 + 148,
+	.vdisplay = 1080,
+	.vsync_start = 1080 + 4,
+	.vsync_end = 1080 + 4 + 5,
+	.vtotal = 1080 + 4 + 5 + 36,
+	.vrefresh = 60,
+};
+
+static const struct panel_desc auo_t215hvn01 = {
+	.modes = &auo_t215hvn01_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 430,
+		.height = 270,
+	},
+	.delay = {
+		.disable = 5,
+		.unprepare = 1000,
+	}
+};
+
 static const struct drm_display_mode avic_tm070ddh03_mode = {
 	.clock = 51200,
 	.hdisplay = 1024,
@@ -583,6 +668,30 @@ static const struct panel_desc avic_tm070ddh03 = {
 	},
 };
 
+static const struct drm_display_mode chunghwa_claa070wp03xg_mode = {
+	.clock = 66770,
+	.hdisplay = 800,
+	.hsync_start = 800 + 49,
+	.hsync_end = 800 + 49 + 33,
+	.htotal = 800 + 49 + 33 + 17,
+	.vdisplay = 1280,
+	.vsync_start = 1280 + 1,
+	.vsync_end = 1280 + 1 + 7,
+	.vtotal = 1280 + 1 + 7 + 15,
+	.vrefresh = 60,
+	.flags = DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
+};
+
+static const struct panel_desc chunghwa_claa070wp03xg = {
+	.modes = &chunghwa_claa070wp03xg_mode,
+	.num_modes = 1,
+	.bpc = 6,
+	.size = {
+		.width = 94,
+		.height = 150,
+	},
+};
+
 static const struct drm_display_mode chunghwa_claa101wa01a_mode = {
 	.clock = 72070,
 	.hdisplay = 1366,
@@ -877,27 +986,31 @@ static const struct panel_desc innolux_g101ice_l01 = {
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
 };
 
-static const struct drm_display_mode innolux_g121i1_l01_mode = {
-	.clock = 71000,
-	.hdisplay = 1280,
-	.hsync_start = 1280 + 64,
-	.hsync_end = 1280 + 64 + 32,
-	.htotal = 1280 + 64 + 32 + 64,
-	.vdisplay = 800,
-	.vsync_start = 800 + 9,
-	.vsync_end = 800 + 9 + 6,
-	.vtotal = 800 + 9 + 6 + 9,
-	.vrefresh = 60,
+static const struct display_timing innolux_g121i1_l01_timing = {
+	.pixelclock = { 67450000, 71000000, 74550000 },
+	.hactive = { 1280, 1280, 1280 },
+	.hfront_porch = { 40, 80, 160 },
+	.hback_porch = { 39, 79, 159 },
+	.hsync_len = { 1, 1, 1 },
+	.vactive = { 800, 800, 800 },
+	.vfront_porch = { 5, 11, 100 },
+	.vback_porch = { 4, 11, 99 },
+	.vsync_len = { 1, 1, 1 },
 };
 
 static const struct panel_desc innolux_g121i1_l01 = {
-	.modes = &innolux_g121i1_l01_mode,
-	.num_modes = 1,
+	.timings = &innolux_g121i1_l01_timing,
+	.num_timings = 1,
 	.bpc = 6,
 	.size = {
 		.width = 261,
 		.height = 163,
 	},
+	.delay = {
+		.enable = 200,
+		.disable = 20,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
 };
 
 static const struct drm_display_mode innolux_g121x1_l03_mode = {
@@ -1164,6 +1277,29 @@ static const struct panel_desc nec_nl4827hc19_05b = {
 	.bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE,
 };
 
+static const struct drm_display_mode nvd_9128_mode = {
+	.clock = 29500,
+	.hdisplay = 800,
+	.hsync_start = 800 + 130,
+	.hsync_end = 800 + 130 + 98,
+	.htotal = 800 + 0 + 130 + 98,
+	.vdisplay = 480,
+	.vsync_start = 480 + 10,
+	.vsync_end = 480 + 10 + 50,
+	.vtotal = 480 + 0 + 10 + 50,
+};
+
+static const struct panel_desc nvd_9128 = {
+	.modes = &nvd_9128_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 156,
+		.height = 88,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+};
+
 static const struct display_timing okaya_rs800480t_7x0gp_timing = {
 	.pixelclock = { 30000000, 30000000, 40000000 },
 	.hactive = { 800, 800, 800 },
@@ -1409,6 +1545,7 @@ static const struct drm_display_mode sharp_lq123p1jx31_mode = {
 static const struct panel_desc sharp_lq123p1jx31 = {
 	.modes = &sharp_lq123p1jx31_mode,
 	.num_modes = 1,
+	.bpc = 8,
 	.size = {
 		.width = 259,
 		.height = 173,
@@ -1420,6 +1557,30 @@ static const struct panel_desc sharp_lq123p1jx31 = {
 	},
 };
 
+static const struct drm_display_mode sharp_lq150x1lg11_mode = {
+	.clock = 71100,
+	.hdisplay = 1024,
+	.hsync_start = 1024 + 168,
+	.hsync_end = 1024 + 168 + 64,
+	.htotal = 1024 + 168 + 64 + 88,
+	.vdisplay = 768,
+	.vsync_start = 768 + 37,
+	.vsync_end = 768 + 37 + 2,
+	.vtotal = 768 + 37 + 2 + 8,
+	.vrefresh = 60,
+};
+
+static const struct panel_desc sharp_lq150x1lg11 = {
+	.modes = &sharp_lq150x1lg11_mode,
+	.num_modes = 1,
+	.bpc = 6,
+	.size = {
+		.width = 304,
+		.height = 228,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB565_1X16,
+};
+
 static const struct drm_display_mode shelly_sca07010_bfn_lnn_mode = {
 	.clock = 33300,
 	.hdisplay = 800,
@@ -1575,9 +1736,21 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible = "auo,b133xtn01",
 		.data = &auo_b133xtn01,
 	}, {
+		.compatible = "auo,g133han01",
+		.data = &auo_g133han01,
+	}, {
+		.compatible = "auo,g185han01",
+		.data = &auo_g185han01,
+	}, {
+		.compatible = "auo,t215hvn01",
+		.data = &auo_t215hvn01,
+	}, {
 		.compatible = "avic,tm070ddh03",
 		.data = &avic_tm070ddh03,
 	}, {
+		.compatible = "chunghwa,claa070wp03xg",
+		.data = &chunghwa_claa070wp03xg,
+	}, {
 		.compatible = "chunghwa,claa101wa01a",
 		.data = &chunghwa_claa101wa01a
 	}, {
@@ -1653,6 +1826,9 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible = "nec,nl4827hc19-05b",
 		.data = &nec_nl4827hc19_05b,
 	}, {
+		.compatible = "nvd,9128",
+		.data = &nvd_9128,
+	}, {
 		.compatible = "okaya,rs800480t-7x0gp",
 		.data = &okaya_rs800480t_7x0gp,
 	}, {
@@ -1683,6 +1859,9 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible = "sharp,lq123p1jx31",
 		.data = &sharp_lq123p1jx31,
 	}, {
+		.compatible = "sharp,lq150x1lg11",
+		.data = &sharp_lq150x1lg11,
+	}, {
 		.compatible = "shelly,sca07010-bfn-lnn",
 		.data = &shelly_sca07010_bfn_lnn,
 	}, {
diff --git a/drivers/gpu/drm/qxl/qxl_cmd.c b/drivers/gpu/drm/qxl/qxl_cmd.c
index 04270f5d110c..74fc9362ecf9 100644
--- a/drivers/gpu/drm/qxl/qxl_cmd.c
+++ b/drivers/gpu/drm/qxl/qxl_cmd.c
@@ -578,7 +578,7 @@ int qxl_hw_surface_dealloc(struct qxl_device *qdev,
 	return 0;
 }
 
-int qxl_update_surface(struct qxl_device *qdev, struct qxl_bo *surf)
+static int qxl_update_surface(struct qxl_device *qdev, struct qxl_bo *surf)
 {
 	struct qxl_rect rect;
 	int ret;
diff --git a/drivers/gpu/drm/qxl/qxl_debugfs.c b/drivers/gpu/drm/qxl/qxl_debugfs.c
index 6911b8c44492..241af9131dc8 100644
--- a/drivers/gpu/drm/qxl/qxl_debugfs.c
+++ b/drivers/gpu/drm/qxl/qxl_debugfs.c
@@ -123,9 +123,6 @@ int qxl_debugfs_add_files(struct qxl_device *qdev,
 	qdev->debugfs_count = i;
 #if defined(CONFIG_DEBUG_FS)
 	drm_debugfs_create_files(files, nfiles,
-				 qdev->ddev->control->debugfs_root,
-				 qdev->ddev->control);
-	drm_debugfs_create_files(files, nfiles,
 				 qdev->ddev->primary->debugfs_root,
 				 qdev->ddev->primary);
 #endif
@@ -140,9 +137,6 @@ void qxl_debugfs_remove_files(struct qxl_device *qdev)
 	for (i = 0; i < qdev->debugfs_count; i++) {
 		drm_debugfs_remove_files(qdev->debugfs[i].files,
 					 qdev->debugfs[i].num_files,
-					 qdev->ddev->control);
-		drm_debugfs_remove_files(qdev->debugfs[i].files,
-					 qdev->debugfs[i].num_files,
 					 qdev->ddev->primary);
 	}
 #endif
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index a61c0d460ec2..4b5eab8a47b3 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -36,7 +36,7 @@ static bool qxl_head_enabled(struct qxl_head *head)
 	return head->width && head->height;
 }
 
-void qxl_alloc_client_monitors_config(struct qxl_device *qdev, unsigned count)
+static void qxl_alloc_client_monitors_config(struct qxl_device *qdev, unsigned count)
 {
 	if (qdev->client_monitors_config &&
 	    count > qdev->client_monitors_config->count) {
@@ -57,11 +57,18 @@ void qxl_alloc_client_monitors_config(struct qxl_device *qdev, unsigned count)
 	qdev->client_monitors_config->count = count;
 }
 
+enum {
+	MONITORS_CONFIG_MODIFIED,
+	MONITORS_CONFIG_UNCHANGED,
+	MONITORS_CONFIG_BAD_CRC,
+};
+
 static int qxl_display_copy_rom_client_monitors_config(struct qxl_device *qdev)
 {
 	int i;
 	int num_monitors;
 	uint32_t crc;
+	int status = MONITORS_CONFIG_UNCHANGED;
 
 	num_monitors = qdev->rom->client_monitors_config.count;
 	crc = crc32(0, (const uint8_t *)&qdev->rom->client_monitors_config,
@@ -70,7 +77,7 @@ static int qxl_display_copy_rom_client_monitors_config(struct qxl_device *qdev)
 		qxl_io_log(qdev, "crc mismatch: have %X (%zd) != %X\n", crc,
 			   sizeof(qdev->rom->client_monitors_config),
 			   qdev->rom->client_monitors_config_crc);
-		return 1;
+		return MONITORS_CONFIG_BAD_CRC;
 	}
 	if (num_monitors > qdev->monitors_config->max_allowed) {
 		DRM_DEBUG_KMS("client monitors list will be truncated: %d < %d\n",
@@ -79,6 +86,10 @@ static int qxl_display_copy_rom_client_monitors_config(struct qxl_device *qdev)
 	} else {
 		num_monitors = qdev->rom->client_monitors_config.count;
 	}
+	if (qdev->client_monitors_config
+	      && (num_monitors != qdev->client_monitors_config->count)) {
+		status = MONITORS_CONFIG_MODIFIED;
+	}
 	qxl_alloc_client_monitors_config(qdev, num_monitors);
 	/* we copy max from the client but it isn't used */
 	qdev->client_monitors_config->max_allowed =
@@ -88,17 +99,39 @@ static int qxl_display_copy_rom_client_monitors_config(struct qxl_device *qdev)
 			&qdev->rom->client_monitors_config.heads[i];
 		struct qxl_head *client_head =
 			&qdev->client_monitors_config->heads[i];
-		client_head->x = c_rect->left;
-		client_head->y = c_rect->top;
-		client_head->width = c_rect->right - c_rect->left;
-		client_head->height = c_rect->bottom - c_rect->top;
-		client_head->surface_id = 0;
-		client_head->id = i;
-		client_head->flags = 0;
+		if (client_head->x != c_rect->left) {
+			client_head->x = c_rect->left;
+			status = MONITORS_CONFIG_MODIFIED;
+		}
+		if (client_head->y != c_rect->top) {
+			client_head->y = c_rect->top;
+			status = MONITORS_CONFIG_MODIFIED;
+		}
+		if (client_head->width != c_rect->right - c_rect->left) {
+			client_head->width = c_rect->right - c_rect->left;
+			status = MONITORS_CONFIG_MODIFIED;
+		}
+		if (client_head->height != c_rect->bottom - c_rect->top) {
+			client_head->height = c_rect->bottom - c_rect->top;
+			status = MONITORS_CONFIG_MODIFIED;
+		}
+		if (client_head->surface_id != 0) {
+			client_head->surface_id = 0;
+			status = MONITORS_CONFIG_MODIFIED;
+		}
+		if (client_head->id != i) {
+			client_head->id = i;
+			status = MONITORS_CONFIG_MODIFIED;
+		}
+		if (client_head->flags != 0) {
+			client_head->flags = 0;
+			status = MONITORS_CONFIG_MODIFIED;
+		}
 		DRM_DEBUG_KMS("read %dx%d+%d+%d\n", client_head->width, client_head->height,
 			  client_head->x, client_head->y);
 	}
-	return 0;
+
+	return status;
 }
 
 static void qxl_update_offset_props(struct qxl_device *qdev)
@@ -124,9 +157,18 @@ void qxl_display_read_client_monitors_config(struct qxl_device *qdev)
 {
 
 	struct drm_device *dev = qdev->ddev;
-	while (qxl_display_copy_rom_client_monitors_config(qdev)) {
+	int status;
+
+	status = qxl_display_copy_rom_client_monitors_config(qdev);
+	while (status == MONITORS_CONFIG_BAD_CRC) {
 		qxl_io_log(qdev, "failed crc check for client_monitors_config,"
 				 " retrying\n");
+		status = qxl_display_copy_rom_client_monitors_config(qdev);
+	}
+	if (status == MONITORS_CONFIG_UNCHANGED) {
+		qxl_io_log(qdev, "config unchanged\n");
+		DRM_DEBUG("ignoring unchanged client monitors config");
+		return;
 	}
 
 	drm_modeset_lock_all(dev);
@@ -157,6 +199,9 @@ static int qxl_add_monitors_config_modes(struct drm_connector *connector,
 	mode = drm_cvt_mode(dev, head->width, head->height, 60, false, false,
 			    false);
 	mode->type |= DRM_MODE_TYPE_PREFERRED;
+	mode->hdisplay = head->width;
+	mode->vdisplay = head->height;
+	drm_mode_set_name(mode);
 	*pwidth = head->width;
 	*pheight = head->height;
 	drm_mode_probed_add(connector, mode);
@@ -607,7 +652,7 @@ static bool qxl_crtc_mode_fixup(struct drm_crtc *crtc,
 	return true;
 }
 
-void
+static void
 qxl_send_monitors_config(struct qxl_device *qdev)
 {
 	int i;
diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h
index 5f3e5ad99de7..785aad42e9bb 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.h
+++ b/drivers/gpu/drm/qxl/qxl_drv.h
@@ -31,7 +31,7 @@
  * Definitions taken from spice-protocol, plus kernel driver specific bits.
  */
 
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 #include <linux/workqueue.h>
 #include <linux/firmware.h>
 #include <linux/platform_device.h>
@@ -190,7 +190,7 @@ enum {
  * spice-protocol/qxl_dev.h */
 #define QXL_MAX_RES 96
 struct qxl_release {
-	struct fence base;
+	struct dma_fence base;
 
 	int id;
 	int type;
@@ -395,16 +395,11 @@ qxl_framebuffer_init(struct drm_device *dev,
 		     struct drm_gem_object *obj,
 		     const struct drm_framebuffer_funcs *funcs);
 void qxl_display_read_client_monitors_config(struct qxl_device *qdev);
-void qxl_send_monitors_config(struct qxl_device *qdev);
 int qxl_create_monitors_object(struct qxl_device *qdev);
 int qxl_destroy_monitors_object(struct qxl_device *qdev);
 
-/* used by qxl_debugfs only */
-void qxl_crtc_set_from_monitors_config(struct qxl_device *qdev);
-void qxl_alloc_client_monitors_config(struct qxl_device *qdev, unsigned count);
-
 /* qxl_gem.c */
-int qxl_gem_init(struct qxl_device *qdev);
+void qxl_gem_init(struct qxl_device *qdev);
 void qxl_gem_fini(struct qxl_device *qdev);
 int qxl_gem_object_create(struct qxl_device *qdev, int size,
 			  int alignment, int initial_domain,
@@ -574,6 +569,5 @@ int qxl_bo_check_id(struct qxl_device *qdev, struct qxl_bo *bo);
 struct qxl_drv_surface *
 qxl_surface_lookup(struct drm_device *dev, int surface_id);
 void qxl_surface_evict(struct qxl_device *qdev, struct qxl_bo *surf, bool freeing);
-int qxl_update_surface(struct qxl_device *qdev, struct qxl_bo *surf);
 
 #endif
diff --git a/drivers/gpu/drm/qxl/qxl_fb.c b/drivers/gpu/drm/qxl/qxl_fb.c
index 2cd879a4ae15..fd7e5e94be5b 100644
--- a/drivers/gpu/drm/qxl/qxl_fb.c
+++ b/drivers/gpu/drm/qxl/qxl_fb.c
@@ -81,16 +81,10 @@ static struct fb_deferred_io qxl_defio = {
 
 static struct fb_ops qxlfb_ops = {
 	.owner = THIS_MODULE,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par, /* TODO: copy vmwgfx */
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_fillrect = drm_fb_helper_sys_fillrect,
 	.fb_copyarea = drm_fb_helper_sys_copyarea,
 	.fb_imageblit = drm_fb_helper_sys_imageblit,
-	.fb_pan_display = drm_fb_helper_pan_display,
-	.fb_blank = drm_fb_helper_blank,
-	.fb_setcmap = drm_fb_helper_setcmap,
-	.fb_debug_enter = drm_fb_helper_debug_enter,
-	.fb_debug_leave = drm_fb_helper_debug_leave,
 };
 
 static void qxlfb_destroy_pinned_object(struct drm_gem_object *gobj)
@@ -197,7 +191,7 @@ static int qxlfb_framebuffer_dirty(struct drm_framebuffer *fb,
 	/*
 	 * we are using a shadow draw buffer, at qdev->surface0_shadow
 	 */
-	qxl_io_log(qdev, "dirty x[%d, %d], y[%d, %d]", clips->x1, clips->x2,
+	qxl_io_log(qdev, "dirty x[%d, %d], y[%d, %d]\n", clips->x1, clips->x2,
 		   clips->y1, clips->y2);
 	image->dx = clips->x1;
 	image->dy = clips->y1;
diff --git a/drivers/gpu/drm/qxl/qxl_gem.c b/drivers/gpu/drm/qxl/qxl_gem.c
index d9746e904ef1..3f185c4da5b7 100644
--- a/drivers/gpu/drm/qxl/qxl_gem.c
+++ b/drivers/gpu/drm/qxl/qxl_gem.c
@@ -111,10 +111,9 @@ void qxl_gem_object_close(struct drm_gem_object *obj,
 {
 }
 
-int qxl_gem_init(struct qxl_device *qdev)
+void qxl_gem_init(struct qxl_device *qdev)
 {
 	INIT_LIST_HEAD(&qdev->gem.objects);
-	return 0;
 }
 
 void qxl_gem_fini(struct qxl_device *qdev)
diff --git a/drivers/gpu/drm/qxl/qxl_kms.c b/drivers/gpu/drm/qxl/qxl_kms.c
index e642242728c0..af685f1d91f8 100644
--- a/drivers/gpu/drm/qxl/qxl_kms.c
+++ b/drivers/gpu/drm/qxl/qxl_kms.c
@@ -131,7 +131,7 @@ static int qxl_device_init(struct qxl_device *qdev,
 	mutex_init(&qdev->update_area_mutex);
 	mutex_init(&qdev->release_mutex);
 	mutex_init(&qdev->surf_evict_mutex);
-	INIT_LIST_HEAD(&qdev->gem.objects);
+	qxl_gem_init(qdev);
 
 	qdev->rom_base = pci_resource_start(pdev, 2);
 	qdev->rom_size = pci_resource_len(pdev, 2);
@@ -273,6 +273,7 @@ static void qxl_device_fini(struct qxl_device *qdev)
 	qxl_ring_free(qdev->command_ring);
 	qxl_ring_free(qdev->cursor_ring);
 	qxl_ring_free(qdev->release_ring);
+	qxl_gem_fini(qdev);
 	qxl_bo_fini(qdev);
 	io_mapping_free(qdev->surface_mapping);
 	io_mapping_free(qdev->vram_mapping);
diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c
index cd83f050cf3e..e6ec845b5be0 100644
--- a/drivers/gpu/drm/qxl/qxl_release.c
+++ b/drivers/gpu/drm/qxl/qxl_release.c
@@ -21,7 +21,7 @@
  */
 #include "qxl_drv.h"
 #include "qxl_object.h"
-#include <trace/events/fence.h>
+#include <trace/events/dma_fence.h>
 
 /*
  * drawable cmd cache - allocate a bunch of VRAM pages, suballocate
@@ -40,23 +40,24 @@
 static const int release_size_per_bo[] = { RELEASE_SIZE, SURFACE_RELEASE_SIZE, RELEASE_SIZE };
 static const int releases_per_bo[] = { RELEASES_PER_BO, SURFACE_RELEASES_PER_BO, RELEASES_PER_BO };
 
-static const char *qxl_get_driver_name(struct fence *fence)
+static const char *qxl_get_driver_name(struct dma_fence *fence)
 {
 	return "qxl";
 }
 
-static const char *qxl_get_timeline_name(struct fence *fence)
+static const char *qxl_get_timeline_name(struct dma_fence *fence)
 {
 	return "release";
 }
 
-static bool qxl_nop_signaling(struct fence *fence)
+static bool qxl_nop_signaling(struct dma_fence *fence)
 {
 	/* fences are always automatically signaled, so just pretend we did this.. */
 	return true;
 }
 
-static long qxl_fence_wait(struct fence *fence, bool intr, signed long timeout)
+static long qxl_fence_wait(struct dma_fence *fence, bool intr,
+			   signed long timeout)
 {
 	struct qxl_device *qdev;
 	struct qxl_release *release;
@@ -71,7 +72,7 @@ static long qxl_fence_wait(struct fence *fence, bool intr, signed long timeout)
 retry:
 	sc++;
 
-	if (fence_is_signaled(fence))
+	if (dma_fence_is_signaled(fence))
 		goto signaled;
 
 	qxl_io_notify_oom(qdev);
@@ -80,11 +81,11 @@ retry:
 		if (!qxl_queue_garbage_collect(qdev, true))
 			break;
 
-		if (fence_is_signaled(fence))
+		if (dma_fence_is_signaled(fence))
 			goto signaled;
 	}
 
-	if (fence_is_signaled(fence))
+	if (dma_fence_is_signaled(fence))
 		goto signaled;
 
 	if (have_drawable_releases || sc < 4) {
@@ -96,9 +97,9 @@ retry:
 			return 0;
 
 		if (have_drawable_releases && sc > 300) {
-			FENCE_WARN(fence, "failed to wait on release %llu "
-					  "after spincount %d\n",
-					  fence->context & ~0xf0000000, sc);
+			DMA_FENCE_WARN(fence, "failed to wait on release %llu "
+				       "after spincount %d\n",
+				       fence->context & ~0xf0000000, sc);
 			goto signaled;
 		}
 		goto retry;
@@ -115,7 +116,7 @@ signaled:
 	return end - cur;
 }
 
-static const struct fence_ops qxl_fence_ops = {
+static const struct dma_fence_ops qxl_fence_ops = {
 	.get_driver_name = qxl_get_driver_name,
 	.get_timeline_name = qxl_get_timeline_name,
 	.enable_signaling = qxl_nop_signaling,
@@ -133,7 +134,7 @@ qxl_release_alloc(struct qxl_device *qdev, int type,
 	release = kmalloc(size, GFP_KERNEL);
 	if (!release) {
 		DRM_ERROR("Out of memory\n");
-		return 0;
+		return -ENOMEM;
 	}
 	release->base.ops = NULL;
 	release->type = type;
@@ -192,8 +193,8 @@ qxl_release_free(struct qxl_device *qdev,
 		WARN_ON(list_empty(&release->bos));
 		qxl_release_free_list(release);
 
-		fence_signal(&release->base);
-		fence_put(&release->base);
+		dma_fence_signal(&release->base);
+		dma_fence_put(&release->base);
 	} else {
 		qxl_release_free_list(release);
 		kfree(release);
@@ -453,9 +454,9 @@ void qxl_release_fence_buffer_objects(struct qxl_release *release)
 	 * Since we never really allocated a context and we don't want to conflict,
 	 * set the highest bits. This will break if we really allow exporting of dma-bufs.
 	 */
-	fence_init(&release->base, &qxl_fence_ops, &qdev->release_lock,
-		   release->id | 0xf0000000, release->base.seqno);
-	trace_fence_emit(&release->base);
+	dma_fence_init(&release->base, &qxl_fence_ops, &qdev->release_lock,
+		       release->id | 0xf0000000, release->base.seqno);
+	trace_dma_fence_emit(&release->base);
 
 	driver = bdev->driver;
 	glob = bo->glob;
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index e26c82db948b..11761330a6b8 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -387,6 +387,7 @@ static struct ttm_bo_driver qxl_bo_driver = {
 	.ttm_tt_unpopulate = &qxl_ttm_tt_unpopulate,
 	.invalidate_caches = &qxl_invalidate_caches,
 	.init_mem_type = &qxl_init_mem_type,
+	.eviction_valuable = ttm_bo_eviction_valuable,
 	.evict_flags = &qxl_evict_flags,
 	.move = &qxl_bo_move,
 	.verify_access = &qxl_verify_access,
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index 74f99bac08b1..05f4ebe31ce2 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -1156,7 +1156,7 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
 	u32 tmp, viewport_w, viewport_h;
 	int r;
 	bool bypass_lut = false;
-	char *format_name;
+	struct drm_format_name_buf format_name;
 
 	/* no fb bound */
 	if (!atomic && !crtc->primary->fb) {
@@ -1260,9 +1260,8 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
 		bypass_lut = true;
 		break;
 	default:
-		format_name = drm_get_format_name(target_fb->pixel_format);
-		DRM_ERROR("Unsupported screen format %s\n", format_name);
-		kfree(format_name);
+		DRM_ERROR("Unsupported screen format %s\n",
+		          drm_get_format_name(target_fb->pixel_format, &format_name));
 		return -EINVAL;
 	}
 
@@ -1473,7 +1472,7 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
 	u32 viewport_w, viewport_h;
 	int r;
 	bool bypass_lut = false;
-	char *format_name;
+	struct drm_format_name_buf format_name;
 
 	/* no fb bound */
 	if (!atomic && !crtc->primary->fb) {
@@ -1563,9 +1562,8 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
 		bypass_lut = true;
 		break;
 	default:
-		format_name = drm_get_format_name(target_fb->pixel_format);
-		DRM_ERROR("Unsupported screen format %s\n", format_name);
-		kfree(format_name);
+		DRM_ERROR("Unsupported screen format %s\n",
+		          drm_get_format_name(target_fb->pixel_format, &format_name));
 		return -EINVAL;
 	}
 
diff --git a/drivers/gpu/drm/radeon/atombios_encoders.c b/drivers/gpu/drm/radeon/atombios_encoders.c
index 56bb758f4e33..fa4f8f008e4d 100644
--- a/drivers/gpu/drm/radeon/atombios_encoders.c
+++ b/drivers/gpu/drm/radeon/atombios_encoders.c
@@ -28,6 +28,7 @@
 #include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "radeon_audio.h"
+#include "radeon_asic.h"
 #include "atom.h"
 #include <linux/backlight.h>
 
diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c
index d960d3915408..f8b05090232a 100644
--- a/drivers/gpu/drm/radeon/evergreen_cs.c
+++ b/drivers/gpu/drm/radeon/evergreen_cs.c
@@ -27,6 +27,7 @@
  */
 #include <drm/drmP.h>
 #include "radeon.h"
+#include "radeon_asic.h"
 #include "evergreend.h"
 #include "evergreen_reg_safe.h"
 #include "cayman_reg_safe.h"
diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c
index b69c8de35bd3..595a19736458 100644
--- a/drivers/gpu/drm/radeon/r600_cs.c
+++ b/drivers/gpu/drm/radeon/r600_cs.c
@@ -28,6 +28,7 @@
 #include <linux/kernel.h>
 #include <drm/drmP.h>
 #include "radeon.h"
+#include "radeon_asic.h"
 #include "r600d.h"
 #include "r600_reg_safe.h"
 
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 1b0dcad916b0..44e0c5ed6418 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -66,7 +66,7 @@
 #include <linux/kref.h>
 #include <linux/interval_tree.h>
 #include <linux/hashtable.h>
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 
 #include <ttm/ttm_bo_api.h>
 #include <ttm/ttm_bo_driver.h>
@@ -367,7 +367,7 @@ struct radeon_fence_driver {
 };
 
 struct radeon_fence {
-	struct fence		base;
+	struct dma_fence		base;
 
 	struct radeon_device	*rdev;
 	uint64_t		seq;
@@ -746,7 +746,7 @@ struct radeon_flip_work {
 	uint64_t			base;
 	struct drm_pending_vblank_event *event;
 	struct radeon_bo		*old_rbo;
-	struct fence			*fence;
+	struct dma_fence		*fence;
 	bool				async;
 };
 
@@ -2514,9 +2514,9 @@ void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v);
 /*
  * Cast helper
  */
-extern const struct fence_ops radeon_fence_ops;
+extern const struct dma_fence_ops radeon_fence_ops;
 
-static inline struct radeon_fence *to_radeon_fence(struct fence *f)
+static inline struct radeon_fence *to_radeon_fence(struct dma_fence *f)
 {
 	struct radeon_fence *__f = container_of(f, struct radeon_fence, base);
 
diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c
index 5df3ec73021b..4134759a6823 100644
--- a/drivers/gpu/drm/radeon/radeon_atombios.c
+++ b/drivers/gpu/drm/radeon/radeon_atombios.c
@@ -29,6 +29,7 @@
 
 #include "atom.h"
 #include "atom-bits.h"
+#include "radeon_asic.h"
 
 extern void
 radeon_add_atom_encoder(struct drm_device *dev, uint32_t encoder_enum,
diff --git a/drivers/gpu/drm/radeon/radeon_clocks.c b/drivers/gpu/drm/radeon/radeon_clocks.c
index 38e396dae0a9..c1135feb93c1 100644
--- a/drivers/gpu/drm/radeon/radeon_clocks.c
+++ b/drivers/gpu/drm/radeon/radeon_clocks.c
@@ -29,6 +29,7 @@
 #include <drm/radeon_drm.h>
 #include "radeon_reg.h"
 #include "radeon.h"
+#include "radeon_asic.h"
 #include "atom.h"
 
 /* 10 khz */
diff --git a/drivers/gpu/drm/radeon/radeon_cursor.c b/drivers/gpu/drm/radeon/radeon_cursor.c
index 2a10e24b34b1..fb16070b266e 100644
--- a/drivers/gpu/drm/radeon/radeon_cursor.c
+++ b/drivers/gpu/drm/radeon/radeon_cursor.c
@@ -90,6 +90,9 @@ static void radeon_show_cursor(struct drm_crtc *crtc)
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct radeon_device *rdev = crtc->dev->dev_private;
 
+	if (radeon_crtc->cursor_out_of_bounds)
+		return;
+
 	if (ASIC_IS_DCE4(rdev)) {
 		WREG32(EVERGREEN_CUR_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
 		       upper_32_bits(radeon_crtc->cursor_addr));
@@ -143,21 +146,25 @@ static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y)
 	int xorigin = 0, yorigin = 0;
 	int w = radeon_crtc->cursor_width;
 
+	radeon_crtc->cursor_x = x;
+	radeon_crtc->cursor_y = y;
+
 	if (ASIC_IS_AVIVO(rdev)) {
 		/* avivo cursor are offset into the total surface */
 		x += crtc->x;
 		y += crtc->y;
 	}
-	DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y);
 
-	if (x < 0) {
+	if (x < 0)
 		xorigin = min(-x, radeon_crtc->max_cursor_width - 1);
-		x = 0;
-	}
-	if (y < 0) {
+	if (y < 0)
 		yorigin = min(-y, radeon_crtc->max_cursor_height - 1);
-		y = 0;
+
+	if (!ASIC_IS_AVIVO(rdev)) {
+		x += crtc->x;
+		y += crtc->y;
 	}
+	DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y);
 
 	/* fixed on DCE6 and newer */
 	if (ASIC_IS_AVIVO(rdev) && !ASIC_IS_DCE6(rdev)) {
@@ -180,27 +187,31 @@ static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y)
 		if (i > 1) {
 			int cursor_end, frame_end;
 
-			cursor_end = x - xorigin + w;
+			cursor_end = x + w;
 			frame_end = crtc->x + crtc->mode.crtc_hdisplay;
 			if (cursor_end >= frame_end) {
 				w = w - (cursor_end - frame_end);
 				if (!(frame_end & 0x7f))
 					w--;
-			} else {
-				if (!(cursor_end & 0x7f))
-					w--;
+			} else if (cursor_end <= 0) {
+				goto out_of_bounds;
+			} else if (!(cursor_end & 0x7f)) {
+				w--;
 			}
 			if (w <= 0) {
-				w = 1;
-				cursor_end = x - xorigin + w;
-				if (!(cursor_end & 0x7f)) {
-					x--;
-					WARN_ON_ONCE(x < 0);
-				}
+				goto out_of_bounds;
 			}
 		}
 	}
 
+	if (x <= (crtc->x - w) || y <= (crtc->y - radeon_crtc->cursor_height) ||
+	    x >= (crtc->x + crtc->mode.crtc_hdisplay) ||
+	    y >= (crtc->y + crtc->mode.crtc_vdisplay))
+		goto out_of_bounds;
+
+	x += xorigin;
+	y += yorigin;
+
 	if (ASIC_IS_DCE4(rdev)) {
 		WREG32(EVERGREEN_CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y);
 		WREG32(EVERGREEN_CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin);
@@ -212,6 +223,9 @@ static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y)
 		WREG32(AVIVO_D1CUR_SIZE + radeon_crtc->crtc_offset,
 		       ((w - 1) << 16) | (radeon_crtc->cursor_height - 1));
 	} else {
+		x -= crtc->x;
+		y -= crtc->y;
+
 		if (crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)
 			y *= 2;
 
@@ -229,10 +243,20 @@ static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y)
 		       yorigin * 256);
 	}
 
-	radeon_crtc->cursor_x = x;
-	radeon_crtc->cursor_y = y;
+	if (radeon_crtc->cursor_out_of_bounds) {
+		radeon_crtc->cursor_out_of_bounds = false;
+		if (radeon_crtc->cursor_bo)
+			radeon_show_cursor(crtc);
+	}
 
 	return 0;
+
+ out_of_bounds:
+	if (!radeon_crtc->cursor_out_of_bounds) {
+		radeon_hide_cursor(crtc);
+		radeon_crtc->cursor_out_of_bounds = true;
+	}
+	return 0;
 }
 
 int radeon_crtc_cursor_move(struct drm_crtc *crtc,
@@ -297,22 +321,23 @@ int radeon_crtc_cursor_set2(struct drm_crtc *crtc,
 		return ret;
 	}
 
-	radeon_crtc->cursor_width = width;
-	radeon_crtc->cursor_height = height;
-
 	radeon_lock_cursor(crtc, true);
 
-	if (hot_x != radeon_crtc->cursor_hot_x ||
+	if (width != radeon_crtc->cursor_width ||
+	    height != radeon_crtc->cursor_height ||
+	    hot_x != radeon_crtc->cursor_hot_x ||
 	    hot_y != radeon_crtc->cursor_hot_y) {
 		int x, y;
 
 		x = radeon_crtc->cursor_x + radeon_crtc->cursor_hot_x - hot_x;
 		y = radeon_crtc->cursor_y + radeon_crtc->cursor_hot_y - hot_y;
 
-		radeon_cursor_move_locked(crtc, x, y);
-
+		radeon_crtc->cursor_width = width;
+		radeon_crtc->cursor_height = height;
 		radeon_crtc->cursor_hot_x = hot_x;
 		radeon_crtc->cursor_hot_y = hot_y;
+
+		radeon_cursor_move_locked(crtc, x, y);
 	}
 
 	radeon_show_cursor(crtc);
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 621af069a3d2..8a1df2a1afbd 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -1333,7 +1333,7 @@ int radeon_device_init(struct radeon_device *rdev,
 	for (i = 0; i < RADEON_NUM_RINGS; i++) {
 		rdev->ring[i].idx = i;
 	}
-	rdev->fence_context = fence_context_alloc(RADEON_NUM_RINGS);
+	rdev->fence_context = dma_fence_context_alloc(RADEON_NUM_RINGS);
 
 	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
 		 radeon_family_name[rdev->family], pdev->vendor, pdev->device,
@@ -1664,7 +1664,10 @@ int radeon_suspend_kms(struct drm_device *dev, bool suspend,
 
 	radeon_suspend(rdev);
 	radeon_hpd_fini(rdev);
-	/* evict remaining vram memory */
+	/* evict remaining vram memory
+	 * This second call to evict vram is to evict the gart page table
+	 * using the CPU.
+	 */
 	radeon_bo_evict_vram(rdev);
 
 	radeon_agp_suspend(rdev);
@@ -1946,9 +1949,6 @@ int radeon_debugfs_add_files(struct radeon_device *rdev,
 	rdev->debugfs_count = i;
 #if defined(CONFIG_DEBUG_FS)
 	drm_debugfs_create_files(files, nfiles,
-				 rdev->ddev->control->debugfs_root,
-				 rdev->ddev->control);
-	drm_debugfs_create_files(files, nfiles,
 				 rdev->ddev->primary->debugfs_root,
 				 rdev->ddev->primary);
 #endif
@@ -1963,9 +1963,6 @@ static void radeon_debugfs_remove_files(struct radeon_device *rdev)
 	for (i = 0; i < rdev->debugfs_count; i++) {
 		drm_debugfs_remove_files(rdev->debugfs[i].files,
 					 rdev->debugfs[i].num_files,
-					 rdev->ddev->control);
-		drm_debugfs_remove_files(rdev->debugfs[i].files,
-					 rdev->debugfs[i].num_files,
 					 rdev->ddev->primary);
 	}
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index cdb8cb568c15..e7409e8a9f87 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -437,7 +437,7 @@ static void radeon_flip_work_func(struct work_struct *__work)
 				down_read(&rdev->exclusive_lock);
 			}
 		} else
-			r = fence_wait(work->fence, false);
+			r = dma_fence_wait(work->fence, false);
 
 		if (r)
 			DRM_ERROR("failed to wait on page flip fence (%d)!\n", r);
@@ -447,7 +447,7 @@ static void radeon_flip_work_func(struct work_struct *__work)
 		 * confused about which BO the CRTC is scanning out
 		 */
 
-		fence_put(work->fence);
+		dma_fence_put(work->fence);
 		work->fence = NULL;
 	}
 
@@ -542,7 +542,7 @@ static int radeon_crtc_page_flip_target(struct drm_crtc *crtc,
 		DRM_ERROR("failed to pin new rbo buffer before flip\n");
 		goto cleanup;
 	}
-	work->fence = fence_get(reservation_object_get_excl(new_rbo->tbo.resv));
+	work->fence = dma_fence_get(reservation_object_get_excl(new_rbo->tbo.resv));
 	radeon_bo_get_tiling_flags(new_rbo, &tiling_flags, NULL);
 	radeon_bo_unreserve(new_rbo);
 
@@ -617,7 +617,7 @@ pflip_cleanup:
 
 cleanup:
 	drm_gem_object_unreference_unlocked(&work->old_rbo->gem_base);
-	fence_put(work->fence);
+	dma_fence_put(work->fence);
 	kfree(work);
 	return r;
 }
diff --git a/drivers/gpu/drm/radeon/radeon_dp_mst.c b/drivers/gpu/drm/radeon/radeon_dp_mst.c
index de504ea29c06..6d1237d6e1b8 100644
--- a/drivers/gpu/drm/radeon/radeon_dp_mst.c
+++ b/drivers/gpu/drm/radeon/radeon_dp_mst.c
@@ -223,7 +223,8 @@ radeon_dp_mst_mode_valid(struct drm_connector *connector,
 	return MODE_OK;
 }
 
-struct drm_encoder *radeon_mst_best_encoder(struct drm_connector *connector)
+static struct
+drm_encoder *radeon_mst_best_encoder(struct drm_connector *connector)
 {
 	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 
@@ -341,7 +342,8 @@ const struct drm_dp_mst_topology_cbs mst_cbs = {
 	.hotplug = radeon_dp_mst_hotplug,
 };
 
-struct radeon_connector *radeon_mst_find_connector(struct drm_encoder *encoder)
+static struct
+radeon_connector *radeon_mst_find_connector(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
 	struct drm_connector *connector;
@@ -597,7 +599,7 @@ static const struct drm_encoder_helper_funcs radeon_mst_helper_funcs = {
 	.commit = radeon_mst_encoder_commit,
 };
 
-void radeon_dp_mst_encoder_destroy(struct drm_encoder *encoder)
+static void radeon_dp_mst_encoder_destroy(struct drm_encoder *encoder)
 {
 	drm_encoder_cleanup(encoder);
 	kfree(encoder);
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
index 0daad446d2c7..899b6a1644bd 100644
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ b/drivers/gpu/drm/radeon/radeon_fb.c
@@ -74,28 +74,22 @@ radeonfb_release(struct fb_info *info, int user)
 
 static struct fb_ops radeonfb_ops = {
 	.owner = THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_open = radeonfb_open,
 	.fb_release = radeonfb_release,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
 	.fb_fillrect = drm_fb_helper_cfb_fillrect,
 	.fb_copyarea = drm_fb_helper_cfb_copyarea,
 	.fb_imageblit = drm_fb_helper_cfb_imageblit,
-	.fb_pan_display = drm_fb_helper_pan_display,
-	.fb_blank = drm_fb_helper_blank,
-	.fb_setcmap = drm_fb_helper_setcmap,
-	.fb_debug_enter = drm_fb_helper_debug_enter,
-	.fb_debug_leave = drm_fb_helper_debug_leave,
 };
 
 
-int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tiled)
+int radeon_align_pitch(struct radeon_device *rdev, int width, int cpp, bool tiled)
 {
 	int aligned = width;
 	int align_large = (ASIC_IS_AVIVO(rdev)) || tiled;
 	int pitch_mask = 0;
 
-	switch (bpp / 8) {
+	switch (cpp) {
 	case 1:
 		pitch_mask = align_large ? 255 : 127;
 		break;
@@ -110,7 +104,7 @@ int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tile
 
 	aligned += pitch_mask;
 	aligned &= ~pitch_mask;
-	return aligned;
+	return aligned * cpp;
 }
 
 static void radeonfb_destroy_pinned_object(struct drm_gem_object *gobj)
@@ -139,13 +133,13 @@ static int radeonfb_create_pinned_object(struct radeon_fbdev *rfbdev,
 	int ret;
 	int aligned_size, size;
 	int height = mode_cmd->height;
-	u32 bpp, depth;
+	u32 cpp;
 
-	drm_fb_get_bpp_depth(mode_cmd->pixel_format, &depth, &bpp);
+	cpp = drm_format_plane_cpp(mode_cmd->pixel_format, 0);
 
 	/* need to align pitch with crtc limits */
-	mode_cmd->pitches[0] = radeon_align_pitch(rdev, mode_cmd->width, bpp,
-						  fb_tiled) * ((bpp + 1) / 8);
+	mode_cmd->pitches[0] = radeon_align_pitch(rdev, mode_cmd->width, cpp,
+						  fb_tiled);
 
 	if (rdev->family >= CHIP_R600)
 		height = ALIGN(mode_cmd->height, 8);
@@ -165,11 +159,11 @@ static int radeonfb_create_pinned_object(struct radeon_fbdev *rfbdev,
 		tiling_flags = RADEON_TILING_MACRO;
 
 #ifdef __BIG_ENDIAN
-	switch (bpp) {
-	case 32:
+	switch (cpp) {
+	case 4:
 		tiling_flags |= RADEON_TILING_SWAP_32BIT;
 		break;
-	case 16:
+	case 2:
 		tiling_flags |= RADEON_TILING_SWAP_16BIT;
 	default:
 		break;
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index 7ef075acde9c..ef09f0a63754 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -141,8 +141,10 @@ int radeon_fence_emit(struct radeon_device *rdev,
 	(*fence)->seq = seq = ++rdev->fence_drv[ring].sync_seq[ring];
 	(*fence)->ring = ring;
 	(*fence)->is_vm_update = false;
-	fence_init(&(*fence)->base, &radeon_fence_ops,
-		   &rdev->fence_queue.lock, rdev->fence_context + ring, seq);
+	dma_fence_init(&(*fence)->base, &radeon_fence_ops,
+		       &rdev->fence_queue.lock,
+		       rdev->fence_context + ring,
+		       seq);
 	radeon_fence_ring_emit(rdev, ring, *fence);
 	trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq);
 	radeon_fence_schedule_check(rdev, ring);
@@ -169,18 +171,18 @@ static int radeon_fence_check_signaled(wait_queue_t *wait, unsigned mode, int fl
 	 */
 	seq = atomic64_read(&fence->rdev->fence_drv[fence->ring].last_seq);
 	if (seq >= fence->seq) {
-		int ret = fence_signal_locked(&fence->base);
+		int ret = dma_fence_signal_locked(&fence->base);
 
 		if (!ret)
-			FENCE_TRACE(&fence->base, "signaled from irq context\n");
+			DMA_FENCE_TRACE(&fence->base, "signaled from irq context\n");
 		else
-			FENCE_TRACE(&fence->base, "was already signaled\n");
+			DMA_FENCE_TRACE(&fence->base, "was already signaled\n");
 
 		radeon_irq_kms_sw_irq_put(fence->rdev, fence->ring);
 		__remove_wait_queue(&fence->rdev->fence_queue, &fence->fence_wake);
-		fence_put(&fence->base);
+		dma_fence_put(&fence->base);
 	} else
-		FENCE_TRACE(&fence->base, "pending\n");
+		DMA_FENCE_TRACE(&fence->base, "pending\n");
 	return 0;
 }
 
@@ -351,7 +353,7 @@ static bool radeon_fence_seq_signaled(struct radeon_device *rdev,
 	return false;
 }
 
-static bool radeon_fence_is_signaled(struct fence *f)
+static bool radeon_fence_is_signaled(struct dma_fence *f)
 {
 	struct radeon_fence *fence = to_radeon_fence(f);
 	struct radeon_device *rdev = fence->rdev;
@@ -381,7 +383,7 @@ static bool radeon_fence_is_signaled(struct fence *f)
  * to fence_queue that checks if this fence is signaled, and if so it
  * signals the fence and removes itself.
  */
-static bool radeon_fence_enable_signaling(struct fence *f)
+static bool radeon_fence_enable_signaling(struct dma_fence *f)
 {
 	struct radeon_fence *fence = to_radeon_fence(f);
 	struct radeon_device *rdev = fence->rdev;
@@ -414,9 +416,9 @@ static bool radeon_fence_enable_signaling(struct fence *f)
 	fence->fence_wake.private = NULL;
 	fence->fence_wake.func = radeon_fence_check_signaled;
 	__add_wait_queue(&rdev->fence_queue, &fence->fence_wake);
-	fence_get(f);
+	dma_fence_get(f);
 
-	FENCE_TRACE(&fence->base, "armed on ring %i!\n", fence->ring);
+	DMA_FENCE_TRACE(&fence->base, "armed on ring %i!\n", fence->ring);
 	return true;
 }
 
@@ -436,9 +438,9 @@ bool radeon_fence_signaled(struct radeon_fence *fence)
 	if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) {
 		int ret;
 
-		ret = fence_signal(&fence->base);
+		ret = dma_fence_signal(&fence->base);
 		if (!ret)
-			FENCE_TRACE(&fence->base, "signaled from radeon_fence_signaled\n");
+			DMA_FENCE_TRACE(&fence->base, "signaled from radeon_fence_signaled\n");
 		return true;
 	}
 	return false;
@@ -552,7 +554,7 @@ long radeon_fence_wait_timeout(struct radeon_fence *fence, bool intr, long timeo
 	 * exclusive_lock is not held in that case.
 	 */
 	if (WARN_ON_ONCE(!to_radeon_fence(&fence->base)))
-		return fence_wait(&fence->base, intr);
+		return dma_fence_wait(&fence->base, intr);
 
 	seq[fence->ring] = fence->seq;
 	r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, timeout);
@@ -560,9 +562,9 @@ long radeon_fence_wait_timeout(struct radeon_fence *fence, bool intr, long timeo
 		return r;
 	}
 
-	r_sig = fence_signal(&fence->base);
+	r_sig = dma_fence_signal(&fence->base);
 	if (!r_sig)
-		FENCE_TRACE(&fence->base, "signaled from fence_wait\n");
+		DMA_FENCE_TRACE(&fence->base, "signaled from fence_wait\n");
 	return r;
 }
 
@@ -697,7 +699,7 @@ int radeon_fence_wait_empty(struct radeon_device *rdev, int ring)
  */
 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
 {
-	fence_get(&fence->base);
+	dma_fence_get(&fence->base);
 	return fence;
 }
 
@@ -714,7 +716,7 @@ void radeon_fence_unref(struct radeon_fence **fence)
 
 	*fence = NULL;
 	if (tmp) {
-		fence_put(&tmp->base);
+		dma_fence_put(&tmp->base);
 	}
 }
 
@@ -1028,12 +1030,12 @@ int radeon_debugfs_fence_init(struct radeon_device *rdev)
 #endif
 }
 
-static const char *radeon_fence_get_driver_name(struct fence *fence)
+static const char *radeon_fence_get_driver_name(struct dma_fence *fence)
 {
 	return "radeon";
 }
 
-static const char *radeon_fence_get_timeline_name(struct fence *f)
+static const char *radeon_fence_get_timeline_name(struct dma_fence *f)
 {
 	struct radeon_fence *fence = to_radeon_fence(f);
 	switch (fence->ring) {
@@ -1051,16 +1053,16 @@ static const char *radeon_fence_get_timeline_name(struct fence *f)
 
 static inline bool radeon_test_signaled(struct radeon_fence *fence)
 {
-	return test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags);
+	return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags);
 }
 
 struct radeon_wait_cb {
-	struct fence_cb base;
+	struct dma_fence_cb base;
 	struct task_struct *task;
 };
 
 static void
-radeon_fence_wait_cb(struct fence *fence, struct fence_cb *cb)
+radeon_fence_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 {
 	struct radeon_wait_cb *wait =
 		container_of(cb, struct radeon_wait_cb, base);
@@ -1068,7 +1070,7 @@ radeon_fence_wait_cb(struct fence *fence, struct fence_cb *cb)
 	wake_up_process(wait->task);
 }
 
-static signed long radeon_fence_default_wait(struct fence *f, bool intr,
+static signed long radeon_fence_default_wait(struct dma_fence *f, bool intr,
 					     signed long t)
 {
 	struct radeon_fence *fence = to_radeon_fence(f);
@@ -1077,7 +1079,7 @@ static signed long radeon_fence_default_wait(struct fence *f, bool intr,
 
 	cb.task = current;
 
-	if (fence_add_callback(f, &cb.base, radeon_fence_wait_cb))
+	if (dma_fence_add_callback(f, &cb.base, radeon_fence_wait_cb))
 		return t;
 
 	while (t > 0) {
@@ -1105,12 +1107,12 @@ static signed long radeon_fence_default_wait(struct fence *f, bool intr,
 	}
 
 	__set_current_state(TASK_RUNNING);
-	fence_remove_callback(f, &cb.base);
+	dma_fence_remove_callback(f, &cb.base);
 
 	return t;
 }
 
-const struct fence_ops radeon_fence_ops = {
+const struct dma_fence_ops radeon_fence_ops = {
 	.get_driver_name = radeon_fence_get_driver_name,
 	.get_timeline_name = radeon_fence_get_timeline_name,
 	.enable_signaling = radeon_fence_enable_signaling,
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index deb9511725c9..0bcffd8a7bd3 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -745,7 +745,8 @@ int radeon_mode_dumb_create(struct drm_file *file_priv,
 	uint32_t handle;
 	int r;
 
-	args->pitch = radeon_align_pitch(rdev, args->width, args->bpp, 0) * ((args->bpp + 1) / 8);
+	args->pitch = radeon_align_pitch(rdev, args->width,
+					 DIV_ROUND_UP(args->bpp, 8), 0);
 	args->size = args->pitch * args->height;
 	args->size = ALIGN(args->size, PAGE_SIZE);
 
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_encoders.c b/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
index 868c3ba2efaa..222a1fa41d7c 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
@@ -27,6 +27,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/radeon_drm.h>
 #include "radeon.h"
+#include "radeon_asic.h"
 #include "atom.h"
 #include <linux/backlight.h>
 #ifdef CONFIG_PMAC_BACKLIGHT
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index bb75201a24ba..f1da484864a9 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -330,6 +330,7 @@ struct radeon_crtc {
 	u16 lut_r[256], lut_g[256], lut_b[256];
 	bool enabled;
 	bool can_tile;
+	bool cursor_out_of_bounds;
 	uint32_t crtc_offset;
 	struct drm_gem_object *cursor_bo;
 	uint64_t cursor_addr;
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 4b6542538ff9..326ad068c15a 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -47,6 +47,7 @@ static bool radeon_pm_in_vbl(struct radeon_device *rdev);
 static bool radeon_pm_debug_check_in_vbl(struct radeon_device *rdev, bool finish);
 static void radeon_pm_update_profile(struct radeon_device *rdev);
 static void radeon_pm_set_clocks(struct radeon_device *rdev);
+static void radeon_pm_compute_clocks_dpm(struct radeon_device *rdev);
 
 int radeon_pm_get_type_index(struct radeon_device *rdev,
 			     enum radeon_pm_state_type ps_type,
@@ -79,6 +80,8 @@ void radeon_pm_acpi_event_handler(struct radeon_device *rdev)
 				radeon_dpm_enable_bapm(rdev, rdev->pm.dpm.ac_power);
 		}
 		mutex_unlock(&rdev->pm.mutex);
+		/* allow new DPM state to be picked */
+		radeon_pm_compute_clocks_dpm(rdev);
 	} else if (rdev->pm.pm_method == PM_METHOD_PROFILE) {
 		if (rdev->pm.profile == PM_PROFILE_AUTO) {
 			mutex_lock(&rdev->pm.mutex);
@@ -882,7 +885,8 @@ static struct radeon_ps *radeon_dpm_pick_power_state(struct radeon_device *rdev,
 		dpm_state = POWER_STATE_TYPE_INTERNAL_3DPERF;
 	/* balanced states don't exist at the moment */
 	if (dpm_state == POWER_STATE_TYPE_BALANCED)
-		dpm_state = POWER_STATE_TYPE_PERFORMANCE;
+		dpm_state = rdev->pm.dpm.ac_power ?
+			POWER_STATE_TYPE_PERFORMANCE : POWER_STATE_TYPE_BATTERY;
 
 restart_search:
 	/* Pick the best power state based on current conditions */
diff --git a/drivers/gpu/drm/radeon/radeon_sync.c b/drivers/gpu/drm/radeon/radeon_sync.c
index 02ac8a1de4ff..be5d7a38d3aa 100644
--- a/drivers/gpu/drm/radeon/radeon_sync.c
+++ b/drivers/gpu/drm/radeon/radeon_sync.c
@@ -92,7 +92,7 @@ int radeon_sync_resv(struct radeon_device *rdev,
 		     bool shared)
 {
 	struct reservation_object_list *flist;
-	struct fence *f;
+	struct dma_fence *f;
 	struct radeon_fence *fence;
 	unsigned i;
 	int r = 0;
@@ -103,7 +103,7 @@ int radeon_sync_resv(struct radeon_device *rdev,
 	if (fence && fence->rdev == rdev)
 		radeon_sync_fence(sync, fence);
 	else if (f)
-		r = fence_wait(f, true);
+		r = dma_fence_wait(f, true);
 
 	flist = reservation_object_get_list(resv);
 	if (shared || !flist || r)
@@ -116,7 +116,7 @@ int radeon_sync_resv(struct radeon_device *rdev,
 		if (fence && fence->rdev == rdev)
 			radeon_sync_fence(sync, fence);
 		else
-			r = fence_wait(f, true);
+			r = dma_fence_wait(f, true);
 
 		if (r)
 			break;
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 3de5e6e21662..0cf03ccbf0a7 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -863,6 +863,7 @@ static struct ttm_bo_driver radeon_bo_driver = {
 	.ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate,
 	.invalidate_caches = &radeon_invalidate_caches,
 	.init_mem_type = &radeon_init_mem_type,
+	.eviction_valuable = ttm_bo_eviction_valuable,
 	.evict_flags = &radeon_evict_flags,
 	.move = &radeon_bo_move,
 	.verify_access = &radeon_verify_access,
diff --git a/drivers/gpu/drm/radeon/radeon_uvd.c b/drivers/gpu/drm/radeon/radeon_uvd.c
index 0cd0e7bdee55..d34d1cf33895 100644
--- a/drivers/gpu/drm/radeon/radeon_uvd.c
+++ b/drivers/gpu/drm/radeon/radeon_uvd.c
@@ -467,7 +467,7 @@ static int radeon_uvd_cs_msg(struct radeon_cs_parser *p, struct radeon_bo *bo,
 {
 	int32_t *msg, msg_type, handle;
 	unsigned img_size = 0;
-	struct fence *f;
+	struct dma_fence *f;
 	void *ptr;
 
 	int i, r;
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index e402be8821c4..ad4d7b8b8322 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -1714,6 +1714,7 @@ static int si_init_microcode(struct radeon_device *rdev)
 		    (rdev->pdev->revision == 0x80) ||
 		    (rdev->pdev->revision == 0x81) ||
 		    (rdev->pdev->revision == 0x83) ||
+		    (rdev->pdev->revision == 0x87) ||
 		    (rdev->pdev->device == 0x6604) ||
 		    (rdev->pdev->device == 0x6605))
 			new_smc = true;
@@ -7858,7 +7859,7 @@ static void si_program_aspm(struct radeon_device *rdev)
 	}
 }
 
-int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
+static int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
 {
 	unsigned i;
 
diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c
index c49934527a87..8b5e697f2549 100644
--- a/drivers/gpu/drm/radeon/si_dpm.c
+++ b/drivers/gpu/drm/radeon/si_dpm.c
@@ -3026,6 +3026,7 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev,
 		    (rdev->pdev->revision == 0x80) ||
 		    (rdev->pdev->revision == 0x81) ||
 		    (rdev->pdev->revision == 0x83) ||
+		    (rdev->pdev->revision == 0x87) ||
 		    (rdev->pdev->device == 0x6604) ||
 		    (rdev->pdev->device == 0x6605)) {
 			max_sclk = 75000;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
index 7316fc7fa0bd..a2ec6d8796a0 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
@@ -149,8 +149,8 @@ static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
 	rcar_du_group_write(rcrtc->group, rcrtc->index % 2 ? OTAR2 : OTAR, 0);
 
 	/* Signal polarities */
-	value = ((mode->flags & DRM_MODE_FLAG_PVSYNC) ? 0 : DSMR_VSL)
-	      | ((mode->flags & DRM_MODE_FLAG_PHSYNC) ? 0 : DSMR_HSL)
+	value = ((mode->flags & DRM_MODE_FLAG_PVSYNC) ? DSMR_VSL : 0)
+	      | ((mode->flags & DRM_MODE_FLAG_PHSYNC) ? DSMR_HSL : 0)
 	      | DSMR_DIPM_DISP | DSMR_CSPM;
 	rcar_du_crtc_write(rcrtc, DSMR, value);
 
@@ -172,7 +172,7 @@ static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
 					mode->crtc_vsync_start - 1);
 	rcar_du_crtc_write(rcrtc, VCR,  mode->crtc_vtotal - 1);
 
-	rcar_du_crtc_write(rcrtc, DESR,  mode->htotal - mode->hsync_start);
+	rcar_du_crtc_write(rcrtc, DESR,  mode->htotal - mode->hsync_start - 1);
 	rcar_du_crtc_write(rcrtc, DEWR,  mode->hdisplay);
 }
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.c b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
index 73c971e39b1c..c05e00872778 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_drv.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
@@ -110,6 +110,27 @@ static const struct rcar_du_device_info rcar_du_r8a7791_info = {
 	.num_lvds = 1,
 };
 
+static const struct rcar_du_device_info rcar_du_r8a7792_info = {
+	.gen = 2,
+	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
+		  | RCAR_DU_FEATURE_EXT_CTRL_REGS,
+	.num_crtcs = 2,
+	.routes = {
+		/* R8A7792 has two RGB outputs. */
+		[RCAR_DU_OUTPUT_DPAD0] = {
+			.possible_crtcs = BIT(0),
+			.encoder_type = DRM_MODE_ENCODER_NONE,
+			.port = 0,
+		},
+		[RCAR_DU_OUTPUT_DPAD1] = {
+			.possible_crtcs = BIT(1),
+			.encoder_type = DRM_MODE_ENCODER_NONE,
+			.port = 1,
+		},
+	},
+	.num_lvds = 0,
+};
+
 static const struct rcar_du_device_info rcar_du_r8a7794_info = {
 	.gen = 2,
 	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
@@ -157,13 +178,39 @@ static const struct rcar_du_device_info rcar_du_r8a7795_info = {
 	.num_lvds = 1,
 };
 
+static const struct rcar_du_device_info rcar_du_r8a7796_info = {
+	.gen = 3,
+	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
+		  | RCAR_DU_FEATURE_EXT_CTRL_REGS
+		  | RCAR_DU_FEATURE_VSP1_SOURCE,
+	.num_crtcs = 3,
+	.routes = {
+		/* R8A7796 has one RGB output, one LVDS output and one
+		 * (currently unsupported) HDMI output.
+		 */
+		[RCAR_DU_OUTPUT_DPAD0] = {
+			.possible_crtcs = BIT(2),
+			.encoder_type = DRM_MODE_ENCODER_NONE,
+			.port = 0,
+		},
+		[RCAR_DU_OUTPUT_LVDS0] = {
+			.possible_crtcs = BIT(0),
+			.encoder_type = DRM_MODE_ENCODER_LVDS,
+			.port = 2,
+		},
+	},
+	.num_lvds = 1,
+};
+
 static const struct of_device_id rcar_du_of_table[] = {
 	{ .compatible = "renesas,du-r8a7779", .data = &rcar_du_r8a7779_info },
 	{ .compatible = "renesas,du-r8a7790", .data = &rcar_du_r8a7790_info },
 	{ .compatible = "renesas,du-r8a7791", .data = &rcar_du_r8a7791_info },
+	{ .compatible = "renesas,du-r8a7792", .data = &rcar_du_r8a7792_info },
 	{ .compatible = "renesas,du-r8a7793", .data = &rcar_du_r8a7791_info },
 	{ .compatible = "renesas,du-r8a7794", .data = &rcar_du_r8a7794_info },
 	{ .compatible = "renesas,du-r8a7795", .data = &rcar_du_r8a7795_info },
+	{ .compatible = "renesas,du-r8a7796", .data = &rcar_du_r8a7796_info },
 	{ }
 };
 
@@ -201,9 +248,7 @@ static const struct file_operations rcar_du_fops = {
 	.open		= drm_open,
 	.release	= drm_release,
 	.unlocked_ioctl	= drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl	= drm_compat_ioctl,
-#endif
 	.poll		= drm_poll,
 	.read		= drm_read,
 	.llseek		= no_llseek,
@@ -285,7 +330,6 @@ static int rcar_du_remove(struct platform_device *pdev)
 
 	drm_kms_helper_poll_fini(ddev);
 	drm_mode_config_cleanup(ddev);
-	drm_vblank_cleanup(ddev);
 
 	drm_dev_unref(ddev);
 
@@ -294,18 +338,12 @@ static int rcar_du_remove(struct platform_device *pdev)
 
 static int rcar_du_probe(struct platform_device *pdev)
 {
-	struct device_node *np = pdev->dev.of_node;
 	struct rcar_du_device *rcdu;
 	struct drm_device *ddev;
 	struct resource *mem;
 	int ret;
 
-	if (np == NULL) {
-		dev_err(&pdev->dev, "no device tree node\n");
-		return -ENODEV;
-	}
-
-	/* Allocate and initialize the DRM and R-Car device structures. */
+	/* Allocate and initialize the R-Car device structure. */
 	rcdu = devm_kzalloc(&pdev->dev, sizeof(*rcdu), GFP_KERNEL);
 	if (rcdu == NULL)
 		return -ENOMEM;
@@ -315,31 +353,22 @@ static int rcar_du_probe(struct platform_device *pdev)
 	rcdu->dev = &pdev->dev;
 	rcdu->info = of_match_device(rcar_du_of_table, rcdu->dev)->data;
 
-	ddev = drm_dev_alloc(&rcar_du_driver, &pdev->dev);
-	if (IS_ERR(ddev))
-		return PTR_ERR(ddev);
-
-	rcdu->ddev = ddev;
-	ddev->dev_private = rcdu;
-
 	platform_set_drvdata(pdev, rcdu);
 
 	/* I/O resources */
 	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	rcdu->mmio = devm_ioremap_resource(&pdev->dev, mem);
-	if (IS_ERR(rcdu->mmio)) {
-		ret = PTR_ERR(rcdu->mmio);
-		goto error;
-	}
-
-	/* Initialize vertical blanking interrupts handling. Start with vblank
-	 * disabled for all CRTCs.
-	 */
-	ret = drm_vblank_init(ddev, (1 << rcdu->info->num_crtcs) - 1);
-	if (ret < 0)
-		goto error;
+	if (IS_ERR(rcdu->mmio))
+		return PTR_ERR(rcdu->mmio);
 
 	/* DRM/KMS objects */
+	ddev = drm_dev_alloc(&rcar_du_driver, &pdev->dev);
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
+
+	rcdu->ddev = ddev;
+	ddev->dev_private = rcdu;
+
 	ret = rcar_du_modeset_init(rcdu);
 	if (ret < 0) {
 		if (ret != -EPROBE_DEFER)
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_group.c b/drivers/gpu/drm/rcar-du/rcar_du_group.c
index 33b2fc53da3e..64738fca96d0 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_group.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_group.c
@@ -105,16 +105,20 @@ static void rcar_du_group_setup(struct rcar_du_group *rgrp)
 	if (rcar_du_has(rgrp->dev, RCAR_DU_FEATURE_EXT_CTRL_REGS)) {
 		rcar_du_group_setup_defr8(rgrp);
 
-		/* Configure input dot clock routing. We currently hardcode the
-		 * configuration to routing DOTCLKINn to DUn.
+		/*
+		 * Configure input dot clock routing. We currently hardcode the
+		 * configuration to routing DOTCLKINn to DUn. Register fields
+		 * depend on the DU generation, but the resulting value is 0 in
+		 * all cases.
+		 *
+		 * On Gen2 a single register in the first group controls dot
+		 * clock selection for all channels, while on Gen3 dot clocks
+		 * are setup through per-group registers, only available when
+		 * the group has two channels.
 		 */
-		rcar_du_group_write(rgrp, DIDSR, DIDSR_CODE |
-				    DIDSR_LCDS_DCLKIN(2) |
-				    DIDSR_LCDS_DCLKIN(1) |
-				    DIDSR_LCDS_DCLKIN(0) |
-				    DIDSR_PDCS_CLK(2, 0) |
-				    DIDSR_PDCS_CLK(1, 0) |
-				    DIDSR_PDCS_CLK(0, 0));
+		if ((rcdu->info->gen < 3 && rgrp->index == 0) ||
+		    (rcdu->info->gen == 3 &&  rgrp->num_crtcs > 1))
+			rcar_du_group_write(rgrp, DIDSR, DIDSR_CODE);
 	}
 
 	if (rcdu->info->gen >= 3)
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c b/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c
index e03004f4588d..f9515f53cc5b 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c
@@ -108,7 +108,7 @@ int rcar_du_hdmienc_init(struct rcar_du_device *rcdu,
 	if (hdmienc == NULL)
 		return -ENOMEM;
 
-	/* Locate drm bridge from the hdmi encoder DT node */
+	/* Locate the DRM bridge from the HDMI encoder DT node. */
 	bridge = of_drm_find_bridge(np);
 	if (!bridge)
 		return -EPROBE_DEFER;
@@ -123,7 +123,7 @@ int rcar_du_hdmienc_init(struct rcar_du_device *rcdu,
 	renc->hdmi = hdmienc;
 	hdmienc->renc = renc;
 
-	/* Link drm_bridge to encoder */
+	/* Link the bridge to the encoder. */
 	bridge->encoder = encoder;
 	encoder->bridge = bridge;
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
index 392c7e6de042..b5d3f16cfa12 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_kms.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
@@ -272,7 +272,7 @@ static void rcar_du_atomic_complete(struct rcar_du_commit *commit)
 
 	drm_atomic_helper_cleanup_planes(dev, old_state);
 
-	drm_atomic_state_free(old_state);
+	drm_atomic_state_put(old_state);
 
 	/* Complete the commit, wake up any waiter. */
 	spin_lock(&rcdu->commit.wait.lock);
@@ -338,6 +338,7 @@ static int rcar_du_atomic_commit(struct drm_device *dev,
 	/* Swap the state, this is the point of no return. */
 	drm_atomic_helper_swap_state(state, true);
 
+	drm_atomic_state_get(state);
 	if (nonblock)
 		schedule_work(&commit->work);
 	else
@@ -453,13 +454,13 @@ static int rcar_du_encoders_init_one(struct rcar_du_device *rcdu,
 	}
 
 	ret = rcar_du_encoder_init(rcdu, enc_type, output, encoder, connector);
-	of_node_put(encoder);
-	of_node_put(connector);
-
 	if (ret && ret != -EPROBE_DEFER)
 		dev_warn(rcdu->dev,
-			 "failed to initialize encoder %s (%d), skipping\n",
-			 encoder->full_name, ret);
+			 "failed to initialize encoder %s on output %u (%d), skipping\n",
+			 of_node_full_name(encoder), output, ret);
+
+	of_node_put(encoder);
+	of_node_put(connector);
 
 	return ret;
 }
@@ -567,6 +568,13 @@ int rcar_du_modeset_init(struct rcar_du_device *rcdu)
 	if (ret < 0)
 		return ret;
 
+	/* Initialize vertical blanking interrupts handling. Start with vblank
+	 * disabled for all CRTCs.
+	 */
+	ret = drm_vblank_init(dev, (1 << rcdu->info->num_crtcs) - 1);
+	if (ret < 0)
+		return ret;
+
 	/* Initialize the groups. */
 	num_groups = DIV_ROUND_UP(rcdu->num_crtcs, 2);
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c b/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c
index 6afd0af312ba..3bcfd161c53f 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c
@@ -61,16 +61,9 @@ static const struct drm_connector_helper_funcs connector_helper_funcs = {
 	.get_modes = rcar_du_lvds_connector_get_modes,
 };
 
-static enum drm_connector_status
-rcar_du_lvds_connector_detect(struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static const struct drm_connector_funcs connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.reset = drm_atomic_helper_connector_reset,
-	.detect = rcar_du_lvds_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -79,7 +72,7 @@ static const struct drm_connector_funcs connector_funcs = {
 
 int rcar_du_lvds_connector_init(struct rcar_du_device *rcdu,
 				struct rcar_du_encoder *renc,
-				/* TODO const */ struct device_node *np)
+				const struct device_node *np)
 {
 	struct drm_encoder *encoder = rcar_encoder_to_drm_encoder(renc);
 	struct rcar_du_lvds_connector *lvdscon;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.h b/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.h
index d4881ee0be7e..639071dd235c 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.h
@@ -19,6 +19,6 @@ struct rcar_du_encoder;
 
 int rcar_du_lvds_connector_init(struct rcar_du_device *rcdu,
 				struct rcar_du_encoder *renc,
-				struct device_node *np);
+				const struct device_node *np);
 
 #endif /* __RCAR_DU_LVDSCON_H__ */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_lvdsenc.c b/drivers/gpu/drm/rcar-du/rcar_du_lvdsenc.c
index ef3a50321ecc..e3a4985f6f3f 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_lvdsenc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_lvdsenc.c
@@ -104,7 +104,14 @@ static void rcar_du_lvdsenc_start_gen3(struct rcar_du_lvdsenc *lvds,
 
 	rcar_lvds_write(lvds, LVDPLLCR, pllcr);
 
-	/* Turn the PLL on, set it to LVDS normal mode, wait for the startup
+	/* Turn all the channels on. */
+	rcar_lvds_write(lvds, LVDCR1,
+			LVDCR1_CHSTBY_GEN3(3) | LVDCR1_CHSTBY_GEN3(2) |
+			LVDCR1_CHSTBY_GEN3(1) | LVDCR1_CHSTBY_GEN3(0) |
+			LVDCR1_CLKSTBY_GEN3);
+
+	/*
+	 * Turn the PLL on, set it to LVDS normal mode, wait for the startup
 	 * delay and turn the output on.
 	 */
 	lvdcr0 = LVDCR0_PLLON;
@@ -117,12 +124,6 @@ static void rcar_du_lvdsenc_start_gen3(struct rcar_du_lvdsenc *lvds,
 
 	lvdcr0 |= LVDCR0_LVRES;
 	rcar_lvds_write(lvds, LVDCR0, lvdcr0);
-
-	/* Turn all the channels on. */
-	rcar_lvds_write(lvds, LVDCR1,
-			LVDCR1_CHSTBY_GEN3(3) | LVDCR1_CHSTBY_GEN3(2) |
-			LVDCR1_CHSTBY_GEN3(1) | LVDCR1_CHSTBY_GEN3(0) |
-			LVDCR1_CLKSTBY_GEN3);
 }
 
 static int rcar_du_lvdsenc_start(struct rcar_du_lvdsenc *lvds,
@@ -241,10 +242,8 @@ int rcar_du_lvdsenc_init(struct rcar_du_device *rcdu)
 
 	for (i = 0; i < rcdu->info->num_lvds; ++i) {
 		lvds = devm_kzalloc(&pdev->dev, sizeof(*lvds), GFP_KERNEL);
-		if (lvds == NULL) {
-			dev_err(&pdev->dev, "failed to allocate private data\n");
+		if (lvds == NULL)
 			return -ENOMEM;
-		}
 
 		lvds->dev = rcdu;
 		lvds->index = i;
diff --git a/drivers/gpu/drm/rockchip/Kconfig b/drivers/gpu/drm/rockchip/Kconfig
index 3c58669a06ce..6f7f9c59f05b 100644
--- a/drivers/gpu/drm/rockchip/Kconfig
+++ b/drivers/gpu/drm/rockchip/Kconfig
@@ -1,7 +1,6 @@
 config DRM_ROCKCHIP
 	tristate "DRM Support for Rockchip"
 	depends on DRM && ROCKCHIP_IOMMU
-	depends on RESET_CONTROLLER
 	select DRM_GEM_CMA_HELPER
 	select DRM_KMS_HELPER
 	select DRM_PANEL
diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi.c
index ca22e5ee89ca..d9aa382bb629 100644
--- a/drivers/gpu/drm/rockchip/dw-mipi-dsi.c
+++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi.c
@@ -969,12 +969,6 @@ static struct drm_connector_helper_funcs dw_mipi_dsi_connector_helper_funcs = {
 	.mode_valid = dw_mipi_dsi_mode_valid,
 };
 
-static enum drm_connector_status
-dw_mipi_dsi_detect(struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static void dw_mipi_dsi_drm_connector_destroy(struct drm_connector *connector)
 {
 	drm_connector_unregister(connector);
@@ -984,7 +978,6 @@ static void dw_mipi_dsi_drm_connector_destroy(struct drm_connector *connector)
 static struct drm_connector_funcs dw_mipi_dsi_atomic_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.detect = dw_mipi_dsi_detect,
 	.destroy = dw_mipi_dsi_drm_connector_destroy,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
index 8c8cbe837e61..2390c8577617 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
@@ -20,6 +20,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_of.h>
 #include <linux/dma-mapping.h>
 #include <linux/pm_runtime.h>
 #include <linux/module.h>
@@ -274,9 +275,7 @@ static const struct file_operations rockchip_drm_driver_fops = {
 	.poll = drm_poll,
 	.read = drm_read,
 	.unlocked_ioctl = drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.release = drm_release,
 };
 
@@ -388,7 +387,7 @@ static void rockchip_add_endpoints(struct device *dev,
 			continue;
 		}
 
-		component_match_add(dev, match, compare_of, remote);
+		drm_of_component_match_add(dev, match, compare_of, remote);
 		of_node_put(remote);
 	}
 }
@@ -437,7 +436,8 @@ static int rockchip_drm_platform_probe(struct platform_device *pdev)
 		}
 
 		of_node_put(iommu);
-		component_match_add(dev, &match, compare_of, port->parent);
+		drm_of_component_match_add(dev, &match, compare_of,
+					   port->parent);
 		of_node_put(port);
 	}
 
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c b/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
index a16c69f96ed5..8f639c8597a5 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
@@ -37,15 +37,11 @@ static int rockchip_fbdev_mmap(struct fb_info *info,
 
 static struct fb_ops rockchip_drm_fbdev_ops = {
 	.owner		= THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_mmap	= rockchip_fbdev_mmap,
 	.fb_fillrect	= drm_fb_helper_cfb_fillrect,
 	.fb_copyarea	= drm_fb_helper_cfb_copyarea,
 	.fb_imageblit	= drm_fb_helper_cfb_imageblit,
-	.fb_check_var	= drm_fb_helper_check_var,
-	.fb_set_par	= drm_fb_helper_set_par,
-	.fb_blank	= drm_fb_helper_blank,
-	.fb_pan_display	= drm_fb_helper_pan_display,
-	.fb_setcmap	= drm_fb_helper_setcmap,
 };
 
 static int rockchip_drm_fbdev_create(struct drm_fb_helper *helper,
diff --git a/drivers/gpu/drm/savage/savage_drv.c b/drivers/gpu/drm/savage/savage_drv.c
index 3b807135a5cd..78c6d8e9b42c 100644
--- a/drivers/gpu/drm/savage/savage_drv.c
+++ b/drivers/gpu/drm/savage/savage_drv.c
@@ -42,9 +42,7 @@ static const struct file_operations savage_driver_fops = {
 	.unlocked_ioctl = drm_ioctl,
 	.mmap = drm_legacy_mmap,
 	.poll = drm_poll,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.llseek = noop_llseek,
 };
 
diff --git a/drivers/gpu/drm/savage/savage_state.c b/drivers/gpu/drm/savage/savage_state.c
index 3dc0d8ff95ec..2db89bed52e8 100644
--- a/drivers/gpu/drm/savage/savage_state.c
+++ b/drivers/gpu/drm/savage/savage_state.c
@@ -1004,6 +1004,7 @@ int savage_bci_cmdbuf(struct drm_device *dev, void *data, struct drm_file *file_
 		kvb_addr = memdup_user(cmdbuf->vb_addr, cmdbuf->vb_size);
 		if (IS_ERR(kvb_addr)) {
 			ret = PTR_ERR(kvb_addr);
+			kvb_addr = NULL;
 			goto done;
 		}
 		cmdbuf->vb_addr = kvb_addr;
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
index 6547b1db460a..dddbdd62bed0 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
@@ -669,15 +669,8 @@ static void shmob_drm_connector_destroy(struct drm_connector *connector)
 	drm_connector_cleanup(connector);
 }
 
-static enum drm_connector_status
-shmob_drm_connector_detect(struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static const struct drm_connector_funcs connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
-	.detect = shmob_drm_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = shmob_drm_connector_destroy,
 };
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_drv.c b/drivers/gpu/drm/shmobile/shmob_drm_drv.c
index f0492603ea88..38dd55f4af81 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_drv.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_drv.c
@@ -245,9 +245,7 @@ static const struct file_operations shmob_drm_fops = {
 	.open		= drm_open,
 	.release	= drm_release,
 	.unlocked_ioctl	= drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl	= drm_compat_ioctl,
-#endif
 	.poll		= drm_poll,
 	.read		= drm_read,
 	.llseek		= no_llseek,
diff --git a/drivers/gpu/drm/sis/sis_drv.c b/drivers/gpu/drm/sis/sis_drv.c
index ae9839886c4d..a836451920f0 100644
--- a/drivers/gpu/drm/sis/sis_drv.c
+++ b/drivers/gpu/drm/sis/sis_drv.c
@@ -72,9 +72,7 @@ static const struct file_operations sis_driver_fops = {
 	.unlocked_ioctl = drm_ioctl,
 	.mmap = drm_legacy_mmap,
 	.poll = drm_poll,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.llseek = noop_llseek,
 };
 
diff --git a/drivers/gpu/drm/sti/sti_drv.c b/drivers/gpu/drm/sti/sti_drv.c
index 9df308565f6c..ff71e25ab5bf 100644
--- a/drivers/gpu/drm/sti/sti_drv.c
+++ b/drivers/gpu/drm/sti/sti_drv.c
@@ -17,6 +17,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_of.h>
 
 #include "sti_crtc.h"
 #include "sti_drv.h"
@@ -184,7 +185,7 @@ static void sti_atomic_complete(struct sti_private *private,
 	drm_atomic_helper_wait_for_vblanks(drm, state);
 
 	drm_atomic_helper_cleanup_planes(drm, state);
-	drm_atomic_state_free(state);
+	drm_atomic_state_put(state);
 }
 
 static void sti_atomic_work(struct work_struct *work)
@@ -237,6 +238,7 @@ static int sti_atomic_commit(struct drm_device *drm,
 
 	drm_atomic_helper_swap_state(state, true);
 
+	drm_atomic_state_get(state);
 	if (nonblock)
 		sti_atomic_schedule(private, state);
 	else
@@ -295,9 +297,7 @@ static const struct file_operations sti_driver_fops = {
 	.poll = drm_poll,
 	.read = drm_read,
 	.unlocked_ioctl = drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.release = drm_release,
 };
 
@@ -443,8 +443,8 @@ static int sti_platform_probe(struct platform_device *pdev)
 	child_np = of_get_next_available_child(node, NULL);
 
 	while (child_np) {
-		component_match_add(dev, &match, compare_of, child_np);
-		of_node_put(child_np);
+		drm_of_component_match_add(dev, &match, compare_of,
+					   child_np);
 		child_np = of_get_next_available_child(node, child_np);
 	}
 
diff --git a/drivers/gpu/drm/sti/sti_hda.c b/drivers/gpu/drm/sti/sti_hda.c
index e7c243f70870..96f336dd0e29 100644
--- a/drivers/gpu/drm/sti/sti_hda.c
+++ b/drivers/gpu/drm/sti/sti_hda.c
@@ -642,12 +642,6 @@ struct drm_connector_helper_funcs sti_hda_connector_helper_funcs = {
 	.mode_valid = sti_hda_connector_mode_valid,
 };
 
-static enum drm_connector_status
-sti_hda_connector_detect(struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static int sti_hda_late_register(struct drm_connector *connector)
 {
 	struct sti_hda_connector *hda_connector
@@ -665,7 +659,6 @@ static int sti_hda_late_register(struct drm_connector *connector)
 static const struct drm_connector_funcs sti_hda_connector_funcs = {
 	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.detect = sti_hda_connector_detect,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c
index 32c0584e3c35..2e08f969bb64 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.c
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.c
@@ -95,6 +95,22 @@ static int sun4i_backend_drm_format_to_layer(struct drm_plane *plane,
 		*mode = SUN4I_BACKEND_LAY_FBFMT_ARGB8888;
 		break;
 
+	case DRM_FORMAT_ARGB4444:
+		*mode = SUN4I_BACKEND_LAY_FBFMT_ARGB4444;
+		break;
+
+	case DRM_FORMAT_ARGB1555:
+		*mode = SUN4I_BACKEND_LAY_FBFMT_ARGB1555;
+		break;
+
+	case DRM_FORMAT_RGBA5551:
+		*mode = SUN4I_BACKEND_LAY_FBFMT_RGBA5551;
+		break;
+
+	case DRM_FORMAT_RGBA4444:
+		*mode = SUN4I_BACKEND_LAY_FBFMT_RGBA4444;
+		break;
+
 	case DRM_FORMAT_XRGB8888:
 		*mode = SUN4I_BACKEND_LAY_FBFMT_XRGB8888;
 		break;
@@ -103,6 +119,10 @@ static int sun4i_backend_drm_format_to_layer(struct drm_plane *plane,
 		*mode = SUN4I_BACKEND_LAY_FBFMT_RGB888;
 		break;
 
+	case DRM_FORMAT_RGB565:
+		*mode = SUN4I_BACKEND_LAY_FBFMT_RGB565;
+		break;
+
 	default:
 		return -EINVAL;
 	}
@@ -389,7 +409,7 @@ static void sun4i_backend_unbind(struct device *dev, struct device *master,
 	reset_control_assert(backend->reset);
 }
 
-static struct component_ops sun4i_backend_ops = {
+static const struct component_ops sun4i_backend_ops = {
 	.bind	= sun4i_backend_bind,
 	.unbind	= sun4i_backend_unbind,
 };
@@ -408,6 +428,7 @@ static int sun4i_backend_remove(struct platform_device *pdev)
 
 static const struct of_device_id sun4i_backend_of_table[] = {
 	{ .compatible = "allwinner,sun5i-a13-display-backend" },
+	{ .compatible = "allwinner,sun6i-a31-display-backend" },
 	{ .compatible = "allwinner,sun8i-a33-display-backend" },
 	{ }
 };
diff --git a/drivers/gpu/drm/sun4i/sun4i_drv.c b/drivers/gpu/drm/sun4i/sun4i_drv.c
index 70e9fd59c5a2..4ce665349f6b 100644
--- a/drivers/gpu/drm/sun4i/sun4i_drv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_drv.c
@@ -18,6 +18,7 @@
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_fb_helper.h>
+#include <drm/drm_of.h>
 
 #include "sun4i_crtc.h"
 #include "sun4i_drv.h"
@@ -52,9 +53,7 @@ static const struct file_operations sun4i_drv_fops = {
 	.open		= drm_open,
 	.release	= drm_release,
 	.unlocked_ioctl	= drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl	= drm_compat_ioctl,
-#endif
 	.poll		= drm_poll,
 	.read		= drm_read,
 	.llseek		= no_llseek,
@@ -201,12 +200,15 @@ static const struct component_master_ops sun4i_drv_master_ops = {
 static bool sun4i_drv_node_is_frontend(struct device_node *node)
 {
 	return of_device_is_compatible(node, "allwinner,sun5i-a13-display-frontend") ||
+		of_device_is_compatible(node, "allwinner,sun6i-a31-display-frontend") ||
 		of_device_is_compatible(node, "allwinner,sun8i-a33-display-frontend");
 }
 
 static bool sun4i_drv_node_is_tcon(struct device_node *node)
 {
 	return of_device_is_compatible(node, "allwinner,sun5i-a13-tcon") ||
+		of_device_is_compatible(node, "allwinner,sun6i-a31-tcon") ||
+		of_device_is_compatible(node, "allwinner,sun6i-a31s-tcon") ||
 		of_device_is_compatible(node, "allwinner,sun8i-a33-tcon");
 }
 
@@ -239,7 +241,7 @@ static int sun4i_drv_add_endpoints(struct device *dev,
 		/* Add current component */
 		DRM_DEBUG_DRIVER("Adding component %s\n",
 				 of_node_full_name(node));
-		component_match_add(dev, match, compare_of, node);
+		drm_of_component_match_add(dev, match, compare_of, node);
 		count++;
 	}
 
@@ -322,6 +324,8 @@ static int sun4i_drv_remove(struct platform_device *pdev)
 
 static const struct of_device_id sun4i_drv_of_table[] = {
 	{ .compatible = "allwinner,sun5i-a13-display-engine" },
+	{ .compatible = "allwinner,sun6i-a31-display-engine" },
+	{ .compatible = "allwinner,sun6i-a31s-display-engine" },
 	{ .compatible = "allwinner,sun8i-a33-display-engine" },
 	{ }
 };
diff --git a/drivers/gpu/drm/sun4i/sun4i_layer.c b/drivers/gpu/drm/sun4i/sun4i_layer.c
index f0035bf5efea..5d53c977bca5 100644
--- a/drivers/gpu/drm/sun4i/sun4i_layer.c
+++ b/drivers/gpu/drm/sun4i/sun4i_layer.c
@@ -73,12 +73,18 @@ static const struct drm_plane_funcs sun4i_backend_layer_funcs = {
 static const uint32_t sun4i_backend_layer_formats_primary[] = {
 	DRM_FORMAT_ARGB8888,
 	DRM_FORMAT_RGB888,
+	DRM_FORMAT_RGB565,
 	DRM_FORMAT_XRGB8888,
 };
 
 static const uint32_t sun4i_backend_layer_formats_overlay[] = {
 	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_RGBA5551,
+	DRM_FORMAT_RGBA4444,
 	DRM_FORMAT_RGB888,
+	DRM_FORMAT_RGB565,
 	DRM_FORMAT_XRGB8888,
 };
 
diff --git a/drivers/gpu/drm/sun4i/sun4i_rgb.c b/drivers/gpu/drm/sun4i/sun4i_rgb.c
index d198ad7e5323..f5e86fe7750e 100644
--- a/drivers/gpu/drm/sun4i/sun4i_rgb.c
+++ b/drivers/gpu/drm/sun4i/sun4i_rgb.c
@@ -110,12 +110,6 @@ static struct drm_connector_helper_funcs sun4i_rgb_con_helper_funcs = {
 	.mode_valid	= sun4i_rgb_mode_valid,
 };
 
-static enum drm_connector_status
-sun4i_rgb_connector_detect(struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static void
 sun4i_rgb_connector_destroy(struct drm_connector *connector)
 {
@@ -129,7 +123,6 @@ sun4i_rgb_connector_destroy(struct drm_connector *connector)
 
 static struct drm_connector_funcs sun4i_rgb_con_funcs = {
 	.dpms			= drm_atomic_helper_connector_dpms,
-	.detect			= sun4i_rgb_connector_detect,
 	.fill_modes		= drm_helper_probe_single_connector_modes,
 	.destroy		= sun4i_rgb_connector_destroy,
 	.reset			= drm_atomic_helper_connector_reset,
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c
index cadacb517f95..ea2906f87cb9 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c
@@ -20,6 +20,7 @@
 #include <linux/component.h>
 #include <linux/ioport.h>
 #include <linux/of_address.h>
+#include <linux/of_device.h>
 #include <linux/of_graph.h>
 #include <linux/of_irq.h>
 #include <linux/regmap.h>
@@ -62,7 +63,7 @@ void sun4i_tcon_channel_disable(struct sun4i_tcon *tcon, int channel)
 		return;
 	}
 
-	WARN_ON(!tcon->has_channel_1);
+	WARN_ON(!tcon->quirks->has_channel_1);
 	regmap_update_bits(tcon->regs, SUN4I_TCON1_CTL_REG,
 			   SUN4I_TCON1_CTL_TCON_ENABLE, 0);
 	clk_disable_unprepare(tcon->sclk1);
@@ -80,7 +81,7 @@ void sun4i_tcon_channel_enable(struct sun4i_tcon *tcon, int channel)
 		return;
 	}
 
-	WARN_ON(!tcon->has_channel_1);
+	WARN_ON(!tcon->quirks->has_channel_1);
 	regmap_update_bits(tcon->regs, SUN4I_TCON1_CTL_REG,
 			   SUN4I_TCON1_CTL_TCON_ENABLE,
 			   SUN4I_TCON1_CTL_TCON_ENABLE);
@@ -202,7 +203,7 @@ void sun4i_tcon1_mode_set(struct sun4i_tcon *tcon,
 	u8 clk_delay;
 	u32 val;
 
-	WARN_ON(!tcon->has_channel_1);
+	WARN_ON(!tcon->quirks->has_channel_1);
 
 	/* Adjust clock delay */
 	clk_delay = sun4i_tcon_get_clk_delay(mode, 1);
@@ -266,7 +267,7 @@ void sun4i_tcon1_mode_set(struct sun4i_tcon *tcon,
 	/*
 	 * FIXME: Undocumented bits
 	 */
-	if (tcon->has_mux)
+	if (tcon->quirks->has_unknown_mux)
 		regmap_write(tcon->regs, SUN4I_TCON_MUX_CTRL_REG, 1);
 }
 EXPORT_SYMBOL(sun4i_tcon1_mode_set);
@@ -327,7 +328,7 @@ static int sun4i_tcon_init_clocks(struct device *dev,
 		return PTR_ERR(tcon->sclk0);
 	}
 
-	if (tcon->has_channel_1) {
+	if (tcon->quirks->has_channel_1) {
 		tcon->sclk1 = devm_clk_get(dev, "tcon-ch1");
 		if (IS_ERR(tcon->sclk1)) {
 			dev_err(dev, "Couldn't get the TCON channel 1 clock\n");
@@ -487,14 +488,7 @@ static int sun4i_tcon_bind(struct device *dev, struct device *master,
 	drv->tcon = tcon;
 	tcon->drm = drm;
 	tcon->dev = dev;
-
-	if (of_device_is_compatible(dev->of_node, "allwinner,sun5i-a13-tcon")) {
-		tcon->has_mux = true;
-		tcon->has_channel_1 = true;
-	} else {
-		tcon->has_mux = false;
-		tcon->has_channel_1 = false;
-	}
+	tcon->quirks = of_device_get_match_data(dev);
 
 	tcon->lcd_rst = devm_reset_control_get(dev, "lcd");
 	if (IS_ERR(tcon->lcd_rst)) {
@@ -551,7 +545,7 @@ static void sun4i_tcon_unbind(struct device *dev, struct device *master,
 	sun4i_tcon_free_clocks(tcon);
 }
 
-static struct component_ops sun4i_tcon_ops = {
+static const struct component_ops sun4i_tcon_ops = {
 	.bind	= sun4i_tcon_bind,
 	.unbind	= sun4i_tcon_unbind,
 };
@@ -588,9 +582,28 @@ static int sun4i_tcon_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct sun4i_tcon_quirks sun5i_a13_quirks = {
+	.has_unknown_mux = true,
+	.has_channel_1	= true,
+};
+
+static const struct sun4i_tcon_quirks sun6i_a31_quirks = {
+	.has_channel_1	= true,
+};
+
+static const struct sun4i_tcon_quirks sun6i_a31s_quirks = {
+	.has_channel_1	= true,
+};
+
+static const struct sun4i_tcon_quirks sun8i_a33_quirks = {
+	/* nothing is supported */
+};
+
 static const struct of_device_id sun4i_tcon_of_table[] = {
-	{ .compatible = "allwinner,sun5i-a13-tcon" },
-	{ .compatible = "allwinner,sun8i-a33-tcon" },
+	{ .compatible = "allwinner,sun5i-a13-tcon", .data = &sun5i_a13_quirks },
+	{ .compatible = "allwinner,sun6i-a31-tcon", .data = &sun6i_a31_quirks },
+	{ .compatible = "allwinner,sun6i-a31s-tcon", .data = &sun6i_a31s_quirks },
+	{ .compatible = "allwinner,sun8i-a33-tcon", .data = &sun8i_a33_quirks },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, sun4i_tcon_of_table);
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.h b/drivers/gpu/drm/sun4i/sun4i_tcon.h
index 12bd48925f4d..166064bafe2e 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.h
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.h
@@ -142,6 +142,11 @@
 
 #define SUN4I_TCON_MAX_CHANNELS		2
 
+struct sun4i_tcon_quirks {
+	bool	has_unknown_mux; /* sun5i has undocumented mux */
+	bool	has_channel_1;	/* a33 does not have channel 1 */
+};
+
 struct sun4i_tcon {
 	struct device			*dev;
 	struct drm_device		*drm;
@@ -160,12 +165,10 @@ struct sun4i_tcon {
 	/* Reset control */
 	struct reset_control		*lcd_rst;
 
-	/* Platform adjustments */
-	bool				has_mux;
-
 	struct drm_panel		*panel;
 
-	bool				has_channel_1;
+	/* Platform adjustments */
+	const struct sun4i_tcon_quirks	*quirks;
 };
 
 struct drm_bridge *sun4i_tcon_find_bridge(struct device_node *node);
diff --git a/drivers/gpu/drm/sun4i/sun4i_tv.c b/drivers/gpu/drm/sun4i/sun4i_tv.c
index 1dd3d9eabf2e..c6f47222e8fc 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tv.c
@@ -537,12 +537,6 @@ static struct drm_connector_helper_funcs sun4i_tv_comp_connector_helper_funcs =
 	.mode_valid	= sun4i_tv_comp_mode_valid,
 };
 
-static enum drm_connector_status
-sun4i_tv_comp_connector_detect(struct drm_connector *connector, bool force)
-{
-	return connector_status_connected;
-}
-
 static void
 sun4i_tv_comp_connector_destroy(struct drm_connector *connector)
 {
@@ -551,7 +545,6 @@ sun4i_tv_comp_connector_destroy(struct drm_connector *connector)
 
 static struct drm_connector_funcs sun4i_tv_comp_connector_funcs = {
 	.dpms			= drm_atomic_helper_connector_dpms,
-	.detect			= sun4i_tv_comp_connector_detect,
 	.fill_modes		= drm_helper_probe_single_connector_modes,
 	.destroy		= sun4i_tv_comp_connector_destroy,
 	.reset			= drm_atomic_helper_connector_reset,
@@ -667,7 +660,7 @@ static void sun4i_tv_unbind(struct device *dev, struct device *master,
 	clk_disable_unprepare(tv->clk);
 }
 
-static struct component_ops sun4i_tv_ops = {
+static const struct component_ops sun4i_tv_ops = {
 	.bind	= sun4i_tv_bind,
 	.unbind	= sun4i_tv_unbind,
 };
diff --git a/drivers/gpu/drm/sun4i/sun6i_drc.c b/drivers/gpu/drm/sun4i/sun6i_drc.c
index bf6d846d8132..09bba853e2a4 100644
--- a/drivers/gpu/drm/sun4i/sun6i_drc.c
+++ b/drivers/gpu/drm/sun4i/sun6i_drc.c
@@ -80,7 +80,7 @@ static void sun6i_drc_unbind(struct device *dev, struct device *master,
 	reset_control_assert(drc->reset);
 }
 
-static struct component_ops sun6i_drc_ops = {
+static const struct component_ops sun6i_drc_ops = {
 	.bind	= sun6i_drc_bind,
 	.unbind	= sun6i_drc_unbind,
 };
@@ -98,6 +98,8 @@ static int sun6i_drc_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id sun6i_drc_of_table[] = {
+	{ .compatible = "allwinner,sun6i-a31-drc" },
+	{ .compatible = "allwinner,sun6i-a31s-drc" },
 	{ .compatible = "allwinner,sun8i-a33-drc" },
 	{ }
 };
diff --git a/drivers/gpu/drm/tdfx/tdfx_drv.c b/drivers/gpu/drm/tdfx/tdfx_drv.c
index f418892b0c71..c54138c3a376 100644
--- a/drivers/gpu/drm/tdfx/tdfx_drv.c
+++ b/drivers/gpu/drm/tdfx/tdfx_drv.c
@@ -49,9 +49,7 @@ static const struct file_operations tdfx_driver_fops = {
 	.unlocked_ioctl = drm_ioctl,
 	.mmap = drm_legacy_mmap,
 	.poll = drm_poll,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.llseek = noop_llseek,
 };
 
diff --git a/drivers/gpu/drm/tegra/Kconfig b/drivers/gpu/drm/tegra/Kconfig
index 63ebb154b9b5..bbf5a4b7e0b6 100644
--- a/drivers/gpu/drm/tegra/Kconfig
+++ b/drivers/gpu/drm/tegra/Kconfig
@@ -3,7 +3,6 @@ config DRM_TEGRA
 	depends on ARCH_TEGRA || (ARM && COMPILE_TEST)
 	depends on COMMON_CLK
 	depends on DRM
-	depends on RESET_CONTROLLER
 	select DRM_KMS_HELPER
 	select DRM_MIPI_DSI
 	select DRM_PANEL
diff --git a/drivers/gpu/drm/tegra/dpaux.c b/drivers/gpu/drm/tegra/dpaux.c
index 059f409556d5..2fde44c3a1b3 100644
--- a/drivers/gpu/drm/tegra/dpaux.c
+++ b/drivers/gpu/drm/tegra/dpaux.c
@@ -539,9 +539,9 @@ static int tegra_dpaux_probe(struct platform_device *pdev)
 	dpaux->desc.owner = THIS_MODULE;
 
 	dpaux->pinctrl = devm_pinctrl_register(&pdev->dev, &dpaux->desc, dpaux);
-	if (!dpaux->pinctrl) {
+	if (IS_ERR(dpaux->pinctrl)) {
 		dev_err(&pdev->dev, "failed to register pincontrol\n");
-		return -ENODEV;
+		return PTR_ERR(dpaux->pinctrl);
 	}
 #endif
 	/* enable and clear all interrupts */
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index 8ab47b502d83..b8be3ee4d3b8 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -63,7 +63,7 @@ static void tegra_atomic_complete(struct tegra_drm *tegra,
 	drm_atomic_helper_wait_for_vblanks(drm, state);
 
 	drm_atomic_helper_cleanup_planes(drm, state);
-	drm_atomic_state_free(state);
+	drm_atomic_state_put(state);
 }
 
 static void tegra_atomic_work(struct work_struct *work)
@@ -96,6 +96,7 @@ static int tegra_atomic_commit(struct drm_device *drm,
 
 	drm_atomic_helper_swap_state(state, true);
 
+	drm_atomic_state_get(state);
 	if (nonblock)
 		tegra_atomic_schedule(tegra, state);
 	else
@@ -801,9 +802,7 @@ static const struct file_operations tegra_drm_fops = {
 	.mmap = tegra_drm_mmap,
 	.poll = drm_poll,
 	.read = drm_read,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.llseek = noop_llseek,
 };
 
diff --git a/drivers/gpu/drm/tegra/fb.c b/drivers/gpu/drm/tegra/fb.c
index e6d71fa4028e..e4a5ab0a9677 100644
--- a/drivers/gpu/drm/tegra/fb.c
+++ b/drivers/gpu/drm/tegra/fb.c
@@ -186,14 +186,10 @@ unreference:
 #ifdef CONFIG_DRM_FBDEV_EMULATION
 static struct fb_ops tegra_fb_ops = {
 	.owner = THIS_MODULE,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_fillrect = drm_fb_helper_sys_fillrect,
 	.fb_copyarea = drm_fb_helper_sys_copyarea,
 	.fb_imageblit = drm_fb_helper_sys_imageblit,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
-	.fb_blank = drm_fb_helper_blank,
-	.fb_pan_display = drm_fb_helper_pan_display,
-	.fb_setcmap = drm_fb_helper_setcmap,
 };
 
 static int tegra_fbdev_probe(struct drm_fb_helper *helper,
diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c
index 95e622e31931..7d853e6b5ff0 100644
--- a/drivers/gpu/drm/tegra/gem.c
+++ b/drivers/gpu/drm/tegra/gem.c
@@ -2,7 +2,7 @@
  * NVIDIA Tegra DRM GEM helper functions
  *
  * Copyright (C) 2012 Sascha Hauer, Pengutronix
- * Copyright (C) 2013 NVIDIA CORPORATION, All rights reserved.
+ * Copyright (C) 2013-2015 NVIDIA CORPORATION, All rights reserved.
  *
  * Based on the GEM/CMA helpers
  *
@@ -36,6 +36,8 @@ static dma_addr_t tegra_bo_pin(struct host1x_bo *bo, struct sg_table **sgt)
 {
 	struct tegra_bo *obj = host1x_to_tegra_bo(bo);
 
+	*sgt = obj->sgt;
+
 	return obj->paddr;
 }
 
@@ -47,23 +49,51 @@ static void *tegra_bo_mmap(struct host1x_bo *bo)
 {
 	struct tegra_bo *obj = host1x_to_tegra_bo(bo);
 
-	return obj->vaddr;
+	if (obj->vaddr)
+		return obj->vaddr;
+	else if (obj->gem.import_attach)
+		return dma_buf_vmap(obj->gem.import_attach->dmabuf);
+	else
+		return vmap(obj->pages, obj->num_pages, VM_MAP,
+			    pgprot_writecombine(PAGE_KERNEL));
 }
 
 static void tegra_bo_munmap(struct host1x_bo *bo, void *addr)
 {
+	struct tegra_bo *obj = host1x_to_tegra_bo(bo);
+
+	if (obj->vaddr)
+		return;
+	else if (obj->gem.import_attach)
+		dma_buf_vunmap(obj->gem.import_attach->dmabuf, addr);
+	else
+		vunmap(addr);
 }
 
 static void *tegra_bo_kmap(struct host1x_bo *bo, unsigned int page)
 {
 	struct tegra_bo *obj = host1x_to_tegra_bo(bo);
 
-	return obj->vaddr + page * PAGE_SIZE;
+	if (obj->vaddr)
+		return obj->vaddr + page * PAGE_SIZE;
+	else if (obj->gem.import_attach)
+		return dma_buf_kmap(obj->gem.import_attach->dmabuf, page);
+	else
+		return vmap(obj->pages + page, 1, VM_MAP,
+			    pgprot_writecombine(PAGE_KERNEL));
 }
 
 static void tegra_bo_kunmap(struct host1x_bo *bo, unsigned int page,
 			    void *addr)
 {
+	struct tegra_bo *obj = host1x_to_tegra_bo(bo);
+
+	if (obj->vaddr)
+		return;
+	else if (obj->gem.import_attach)
+		dma_buf_kunmap(obj->gem.import_attach->dmabuf, page, addr);
+	else
+		vunmap(addr);
 }
 
 static struct host1x_bo *tegra_bo_get(struct host1x_bo *bo)
@@ -318,11 +348,6 @@ static struct tegra_bo *tegra_bo_import(struct drm_device *drm,
 	get_dma_buf(buf);
 
 	bo->sgt = dma_buf_map_attachment(attach, DMA_TO_DEVICE);
-	if (!bo->sgt) {
-		err = -ENOMEM;
-		goto detach;
-	}
-
 	if (IS_ERR(bo->sgt)) {
 		err = PTR_ERR(bo->sgt);
 		goto detach;
@@ -427,10 +452,10 @@ static int tegra_bo_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	if (!bo->pages)
 		return VM_FAULT_SIGBUS;
 
-	offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >> PAGE_SHIFT;
+	offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
 	page = bo->pages[offset];
 
-	err = vm_insert_page(vma, (unsigned long)vmf->virtual_address, page);
+	err = vm_insert_page(vma, vmf->address, page);
 	switch (err) {
 	case -EAGAIN:
 	case 0:
diff --git a/drivers/gpu/drm/tegra/gr3d.c b/drivers/gpu/drm/tegra/gr3d.c
index 0b3f2b977ba0..13f0d1b7cd98 100644
--- a/drivers/gpu/drm/tegra/gr3d.c
+++ b/drivers/gpu/drm/tegra/gr3d.c
@@ -268,9 +268,9 @@ static int gr3d_probe(struct platform_device *pdev)
 
 	if (of_device_is_compatible(np, "nvidia,tegra30-gr3d")) {
 		gr3d->clk_secondary = devm_clk_get(&pdev->dev, "3d2");
-		if (IS_ERR(gr3d->clk)) {
+		if (IS_ERR(gr3d->clk_secondary)) {
 			dev_err(&pdev->dev, "cannot get secondary clock\n");
-			return PTR_ERR(gr3d->clk);
+			return PTR_ERR(gr3d->clk_secondary);
 		}
 
 		gr3d->rst_secondary = devm_reset_control_get(&pdev->dev,
diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c
index 74d0540b8d4c..a8f528925009 100644
--- a/drivers/gpu/drm/tegra/sor.c
+++ b/drivers/gpu/drm/tegra/sor.c
@@ -349,8 +349,6 @@ static struct clk *tegra_clk_sor_brick_register(struct tegra_sor *sor,
 	brick->hw.init = &init;
 
 	clk = devm_clk_register(sor->dev, &brick->hw);
-	if (IS_ERR(clk))
-		kfree(brick);
 
 	return clk;
 }
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
index 52ebe8fc1784..9942b0577d6e 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
@@ -21,11 +21,15 @@
 #include <drm/drm_flip_work.h>
 #include <drm/drm_plane_helper.h>
 #include <linux/workqueue.h>
+#include <linux/completion.h>
+#include <linux/dma-mapping.h>
 
 #include "tilcdc_drv.h"
 #include "tilcdc_regs.h"
 
-#define TILCDC_VBLANK_SAFETY_THRESHOLD_US 1000
+#define TILCDC_VBLANK_SAFETY_THRESHOLD_US	1000
+#define TILCDC_PALETTE_SIZE			32
+#define TILCDC_PALETTE_FIRST_ENTRY		0x4000
 
 struct tilcdc_crtc {
 	struct drm_crtc base;
@@ -33,7 +37,9 @@ struct tilcdc_crtc {
 	struct drm_plane primary;
 	const struct tilcdc_panel_info *info;
 	struct drm_pending_vblank_event *event;
+	struct mutex enable_lock;
 	bool enabled;
+	bool shutdown;
 	wait_queue_head_t frame_done_wq;
 	bool frame_done;
 	spinlock_t irq_lock;
@@ -53,6 +59,11 @@ struct tilcdc_crtc {
 
 	int sync_lost_count;
 	bool frame_intact;
+	struct work_struct recover_work;
+
+	dma_addr_t palette_dma_handle;
+	u16 *palette_base;
+	struct completion palette_loaded;
 };
 #define to_tilcdc_crtc(x) container_of(x, struct tilcdc_crtc, base)
 
@@ -71,17 +82,16 @@ static void set_scanout(struct drm_crtc *crtc, struct drm_framebuffer *fb)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
+	struct tilcdc_drm_private *priv = dev->dev_private;
 	struct drm_gem_cma_object *gem;
-	unsigned int depth, bpp;
 	dma_addr_t start, end;
 	u64 dma_base_and_ceiling;
 
-	drm_fb_get_bpp_depth(fb->pixel_format, &depth, &bpp);
 	gem = drm_fb_cma_get_gem_obj(fb, 0);
 
 	start = gem->paddr + fb->offsets[0] +
 		crtc->y * fb->pitches[0] +
-		crtc->x * bpp / 8;
+		crtc->x * drm_format_plane_cpp(fb->pixel_format, 0);
 
 	end = start + (crtc->mode.vdisplay * fb->pitches[0]);
 
@@ -90,7 +100,10 @@ static void set_scanout(struct drm_crtc *crtc, struct drm_framebuffer *fb)
 	 * unlikely that LCDC would fetch the DMA addresses in the middle of
 	 * an update.
 	 */
-	dma_base_and_ceiling = (u64)(end - 1) << 32 | start;
+	if (priv->rev == 1)
+		end -= 1;
+
+	dma_base_and_ceiling = (u64)end << 32 | start;
 	tilcdc_write64(dev, LCDC_DMA_FB_BASE_ADDR_0_REG, dma_base_and_ceiling);
 
 	if (tilcdc_crtc->curr_fb)
@@ -100,6 +113,56 @@ static void set_scanout(struct drm_crtc *crtc, struct drm_framebuffer *fb)
 	tilcdc_crtc->curr_fb = fb;
 }
 
+/*
+ * The driver currently only supports only true color formats. For
+ * true color the palette block is bypassed, but a 32 byte palette
+ * should still be loaded. The first 16-bit entry must be 0x4000 while
+ * all other entries must be zeroed.
+ */
+static void tilcdc_crtc_load_palette(struct drm_crtc *crtc)
+{
+	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct tilcdc_drm_private *priv = dev->dev_private;
+	int ret;
+
+	reinit_completion(&tilcdc_crtc->palette_loaded);
+
+	/* Tell the LCDC where the palette is located. */
+	tilcdc_write(dev, LCDC_DMA_FB_BASE_ADDR_0_REG,
+		     tilcdc_crtc->palette_dma_handle);
+	tilcdc_write(dev, LCDC_DMA_FB_CEILING_ADDR_0_REG,
+		     (u32) tilcdc_crtc->palette_dma_handle +
+		     TILCDC_PALETTE_SIZE - 1);
+
+	/* Set dma load mode for palette loading only. */
+	tilcdc_write_mask(dev, LCDC_RASTER_CTRL_REG,
+			  LCDC_PALETTE_LOAD_MODE(PALETTE_ONLY),
+			  LCDC_PALETTE_LOAD_MODE_MASK);
+
+	/* Enable DMA Palette Loaded Interrupt */
+	if (priv->rev == 1)
+		tilcdc_set(dev, LCDC_RASTER_CTRL_REG, LCDC_V1_PL_INT_ENA);
+	else
+		tilcdc_write(dev, LCDC_INT_ENABLE_SET_REG, LCDC_V2_PL_INT_ENA);
+
+	/* Enable LCDC DMA and wait for palette to be loaded. */
+	tilcdc_clear_irqstatus(dev, 0xffffffff);
+	tilcdc_set(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ENABLE);
+
+	ret = wait_for_completion_timeout(&tilcdc_crtc->palette_loaded,
+					  msecs_to_jiffies(50));
+	if (ret == 0)
+		dev_err(dev->dev, "%s: Palette loading timeout", __func__);
+
+	/* Disable LCDC DMA and DMA Palette Loaded Interrupt. */
+	tilcdc_clear(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ENABLE);
+	if (priv->rev == 1)
+		tilcdc_clear(dev, LCDC_RASTER_CTRL_REG, LCDC_V1_PL_INT_ENA);
+	else
+		tilcdc_write(dev, LCDC_INT_ENABLE_CLR_REG, LCDC_V2_PL_INT_ENA);
+}
+
 static void tilcdc_crtc_enable_irqs(struct drm_device *dev)
 {
 	struct tilcdc_drm_private *priv = dev->dev_private;
@@ -108,6 +171,7 @@ static void tilcdc_crtc_enable_irqs(struct drm_device *dev)
 
 	if (priv->rev == 1) {
 		tilcdc_set(dev, LCDC_RASTER_CTRL_REG,
+			LCDC_V1_SYNC_LOST_INT_ENA | LCDC_V1_FRAME_DONE_INT_ENA |
 			LCDC_V1_UNDERFLOW_INT_ENA);
 		tilcdc_set(dev, LCDC_DMA_CTRL_REG,
 			LCDC_V1_END_OF_FRAME_INT_ENA);
@@ -126,6 +190,7 @@ static void tilcdc_crtc_disable_irqs(struct drm_device *dev)
 	/* disable irqs that we might have enabled: */
 	if (priv->rev == 1) {
 		tilcdc_clear(dev, LCDC_RASTER_CTRL_REG,
+			LCDC_V1_SYNC_LOST_INT_ENA | LCDC_V1_FRAME_DONE_INT_ENA |
 			LCDC_V1_UNDERFLOW_INT_ENA | LCDC_V1_PL_INT_ENA);
 		tilcdc_clear(dev, LCDC_DMA_CTRL_REG,
 			LCDC_V1_END_OF_FRAME_INT_ENA);
@@ -150,193 +215,68 @@ static void reset(struct drm_crtc *crtc)
 	tilcdc_clear(dev, LCDC_CLK_RESET_REG, LCDC_CLK_MAIN_RESET);
 }
 
-static void tilcdc_crtc_enable(struct drm_crtc *crtc)
+/*
+ * Calculate the percentage difference between the requested pixel clock rate
+ * and the effective rate resulting from calculating the clock divider value.
+ */
+static unsigned int tilcdc_pclk_diff(unsigned long rate,
+				     unsigned long real_rate)
 {
-	struct drm_device *dev = crtc->dev;
-	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
-
-	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
-
-	if (tilcdc_crtc->enabled)
-		return;
-
-	pm_runtime_get_sync(dev->dev);
-
-	reset(crtc);
-
-	tilcdc_crtc_enable_irqs(dev);
-
-	tilcdc_clear(dev, LCDC_DMA_CTRL_REG, LCDC_DUAL_FRAME_BUFFER_ENABLE);
-	tilcdc_set(dev, LCDC_RASTER_CTRL_REG, LCDC_PALETTE_LOAD_MODE(DATA_ONLY));
-	tilcdc_set(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ENABLE);
-
-	drm_crtc_vblank_on(crtc);
+	int r = rate / 100, rr = real_rate / 100;
 
-	tilcdc_crtc->enabled = true;
+	return (unsigned int)(abs(((rr - r) * 100) / r));
 }
 
-void tilcdc_crtc_disable(struct drm_crtc *crtc)
+static void tilcdc_crtc_set_clk(struct drm_crtc *crtc)
 {
-	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct tilcdc_drm_private *priv = dev->dev_private;
-
-	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
-
-	if (!tilcdc_crtc->enabled)
-		return;
-
-	tilcdc_crtc->frame_done = false;
-	tilcdc_clear(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ENABLE);
-
-	/*
-	 * if necessary wait for framedone irq which will still come
-	 * before putting things to sleep..
-	 */
-	if (priv->rev == 2) {
-		int ret = wait_event_timeout(tilcdc_crtc->frame_done_wq,
-					     tilcdc_crtc->frame_done,
-					     msecs_to_jiffies(500));
-		if (ret == 0)
-			dev_err(dev->dev, "%s: timeout waiting for framedone\n",
-				__func__);
-	}
-
-	drm_crtc_vblank_off(crtc);
-
-	tilcdc_crtc_disable_irqs(dev);
-
-	pm_runtime_put_sync(dev->dev);
-
-	if (tilcdc_crtc->next_fb) {
-		drm_flip_work_queue(&tilcdc_crtc->unref_work,
-				    tilcdc_crtc->next_fb);
-		tilcdc_crtc->next_fb = NULL;
-	}
-
-	if (tilcdc_crtc->curr_fb) {
-		drm_flip_work_queue(&tilcdc_crtc->unref_work,
-				    tilcdc_crtc->curr_fb);
-		tilcdc_crtc->curr_fb = NULL;
-	}
-
-	drm_flip_work_commit(&tilcdc_crtc->unref_work, priv->wq);
-	tilcdc_crtc->last_vblank = ktime_set(0, 0);
-
-	tilcdc_crtc->enabled = false;
-}
-
-static bool tilcdc_crtc_is_on(struct drm_crtc *crtc)
-{
-	return crtc->state && crtc->state->enable && crtc->state->active;
-}
-
-static void tilcdc_crtc_destroy(struct drm_crtc *crtc)
-{
-	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
-	struct tilcdc_drm_private *priv = crtc->dev->dev_private;
-
-	drm_modeset_lock_crtc(crtc, NULL);
-	tilcdc_crtc_disable(crtc);
-	drm_modeset_unlock_crtc(crtc);
-
-	flush_workqueue(priv->wq);
-
-	of_node_put(crtc->port);
-	drm_crtc_cleanup(crtc);
-	drm_flip_work_cleanup(&tilcdc_crtc->unref_work);
-}
-
-int tilcdc_crtc_update_fb(struct drm_crtc *crtc,
-		struct drm_framebuffer *fb,
-		struct drm_pending_vblank_event *event)
-{
-	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
-	struct drm_device *dev = crtc->dev;
-	unsigned long flags;
-
-	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
-
-	if (tilcdc_crtc->event) {
-		dev_err(dev->dev, "already pending page flip!\n");
-		return -EBUSY;
-	}
-
-	drm_framebuffer_reference(fb);
-
-	crtc->primary->fb = fb;
-
-	spin_lock_irqsave(&tilcdc_crtc->irq_lock, flags);
-
-	if (crtc->hwmode.vrefresh && ktime_to_ns(tilcdc_crtc->last_vblank)) {
-		ktime_t next_vblank;
-		s64 tdiff;
-
-		next_vblank = ktime_add_us(tilcdc_crtc->last_vblank,
-			1000000 / crtc->hwmode.vrefresh);
-
-		tdiff = ktime_to_us(ktime_sub(next_vblank, ktime_get()));
-
-		if (tdiff < TILCDC_VBLANK_SAFETY_THRESHOLD_US)
-			tilcdc_crtc->next_fb = fb;
-	}
-
-	if (tilcdc_crtc->next_fb != fb)
-		set_scanout(crtc, fb);
-
-	tilcdc_crtc->event = event;
-
-	spin_unlock_irqrestore(&tilcdc_crtc->irq_lock, flags);
-
-	return 0;
-}
-
-static bool tilcdc_crtc_mode_fixup(struct drm_crtc *crtc,
-		const struct drm_display_mode *mode,
-		struct drm_display_mode *adjusted_mode)
-{
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
+	unsigned long clk_rate, real_rate, req_rate;
+	unsigned int clkdiv;
+	int ret;
 
-	if (!tilcdc_crtc->simulate_vesa_sync)
-		return true;
+	clkdiv = 2; /* first try using a standard divider of 2 */
 
-	/*
-	 * tilcdc does not generate VESA-compliant sync but aligns
-	 * VS on the second edge of HS instead of first edge.
-	 * We use adjusted_mode, to fixup sync by aligning both rising
-	 * edges and add HSKEW offset to fix the sync.
-	 */
-	adjusted_mode->hskew = mode->hsync_end - mode->hsync_start;
-	adjusted_mode->flags |= DRM_MODE_FLAG_HSKEW;
+	/* mode.clock is in KHz, set_rate wants parameter in Hz */
+	req_rate = crtc->mode.clock * 1000;
 
-	if (mode->flags & DRM_MODE_FLAG_NHSYNC) {
-		adjusted_mode->flags |= DRM_MODE_FLAG_PHSYNC;
-		adjusted_mode->flags &= ~DRM_MODE_FLAG_NHSYNC;
-	} else {
-		adjusted_mode->flags |= DRM_MODE_FLAG_NHSYNC;
-		adjusted_mode->flags &= ~DRM_MODE_FLAG_PHSYNC;
-	}
+	ret = clk_set_rate(priv->clk, req_rate * clkdiv);
+	clk_rate = clk_get_rate(priv->clk);
+	if (ret < 0) {
+		/*
+		 * If we fail to set the clock rate (some architectures don't
+		 * use the common clock framework yet and may not implement
+		 * all the clk API calls for every clock), try the next best
+		 * thing: adjusting the clock divider, unless clk_get_rate()
+		 * failed as well.
+		 */
+		if (!clk_rate) {
+			/* Nothing more we can do. Just bail out. */
+			dev_err(dev->dev,
+				"failed to set the pixel clock - unable to read current lcdc clock rate\n");
+			return;
+		}
 
-	return true;
-}
+		clkdiv = DIV_ROUND_CLOSEST(clk_rate, req_rate);
 
-static void tilcdc_crtc_set_clk(struct drm_crtc *crtc)
-{
-	struct drm_device *dev = crtc->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
-	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
-	const unsigned clkdiv = 2; /* using a fixed divider of 2 */
-	int ret;
+		/*
+		 * Emit a warning if the real clock rate resulting from the
+		 * calculated divider differs much from the requested rate.
+		 *
+		 * 5% is an arbitrary value - LCDs are usually quite tolerant
+		 * about pixel clock rates.
+		 */
+		real_rate = clkdiv * req_rate;
 
-	/* mode.clock is in KHz, set_rate wants parameter in Hz */
-	ret = clk_set_rate(priv->clk, crtc->mode.clock * 1000 * clkdiv);
-	if (ret < 0) {
-		dev_err(dev->dev, "failed to set display clock rate to: %d\n",
-			crtc->mode.clock);
-		return;
+		if (tilcdc_pclk_diff(clk_rate, real_rate) > 5) {
+			dev_warn(dev->dev,
+				 "effective pixel clock rate (%luHz) differs from the calculated rate (%luHz)\n",
+				 clk_rate, real_rate);
+		}
 	}
 
-	tilcdc_crtc->lcd_fck_rate = clk_get_rate(priv->clk);
+	tilcdc_crtc->lcd_fck_rate = clk_rate;
 
 	DBG("lcd_clk=%u, mode clock=%d, div=%u",
 	    tilcdc_crtc->lcd_fck_rate, crtc->mode.clock, clkdiv);
@@ -351,7 +291,7 @@ static void tilcdc_crtc_set_clk(struct drm_crtc *crtc)
 				LCDC_V2_CORE_CLK_EN);
 }
 
-static void tilcdc_crtc_mode_set_nofb(struct drm_crtc *crtc)
+static void tilcdc_crtc_set_mode(struct drm_crtc *crtc)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
@@ -361,8 +301,6 @@ static void tilcdc_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
 	struct drm_framebuffer *fb = crtc->primary->state->fb;
 
-	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
-
 	if (WARN_ON(!info))
 		return;
 
@@ -461,16 +399,16 @@ static void tilcdc_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	if (info->tft_alt_mode)
 		reg |= LCDC_TFT_ALT_ENABLE;
 	if (priv->rev == 2) {
-		unsigned int depth, bpp;
-
-		drm_fb_get_bpp_depth(fb->pixel_format, &depth, &bpp);
-		switch (bpp) {
-		case 16:
+		switch (fb->pixel_format) {
+		case DRM_FORMAT_BGR565:
+		case DRM_FORMAT_RGB565:
 			break;
-		case 32:
+		case DRM_FORMAT_XBGR8888:
+		case DRM_FORMAT_XRGB8888:
 			reg |= LCDC_V2_TFT_24BPP_UNPACK;
 			/* fallthrough */
-		case 24:
+		case DRM_FORMAT_BGR888:
+		case DRM_FORMAT_RGB888:
 			reg |= LCDC_V2_TFT_24BPP_MODE;
 			break;
 		default:
@@ -511,15 +449,226 @@ static void tilcdc_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	else
 		tilcdc_clear(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ORDER);
 
-	drm_framebuffer_reference(fb);
+	tilcdc_crtc_set_clk(crtc);
+
+	tilcdc_crtc_load_palette(crtc);
 
 	set_scanout(crtc, fb);
 
-	tilcdc_crtc_set_clk(crtc);
+	drm_framebuffer_reference(fb);
 
 	crtc->hwmode = crtc->state->adjusted_mode;
 }
 
+static void tilcdc_crtc_enable(struct drm_crtc *crtc)
+{
+	struct drm_device *dev = crtc->dev;
+	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
+
+	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
+	mutex_lock(&tilcdc_crtc->enable_lock);
+	if (tilcdc_crtc->enabled || tilcdc_crtc->shutdown) {
+		mutex_unlock(&tilcdc_crtc->enable_lock);
+		return;
+	}
+
+	pm_runtime_get_sync(dev->dev);
+
+	reset(crtc);
+
+	tilcdc_crtc_set_mode(crtc);
+
+	tilcdc_crtc_enable_irqs(dev);
+
+	tilcdc_clear(dev, LCDC_DMA_CTRL_REG, LCDC_DUAL_FRAME_BUFFER_ENABLE);
+	tilcdc_write_mask(dev, LCDC_RASTER_CTRL_REG,
+			  LCDC_PALETTE_LOAD_MODE(DATA_ONLY),
+			  LCDC_PALETTE_LOAD_MODE_MASK);
+	tilcdc_set(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ENABLE);
+
+	drm_crtc_vblank_on(crtc);
+
+	tilcdc_crtc->enabled = true;
+	mutex_unlock(&tilcdc_crtc->enable_lock);
+}
+
+static void tilcdc_crtc_off(struct drm_crtc *crtc, bool shutdown)
+{
+	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct tilcdc_drm_private *priv = dev->dev_private;
+	int ret;
+
+	mutex_lock(&tilcdc_crtc->enable_lock);
+	if (shutdown)
+		tilcdc_crtc->shutdown = true;
+	if (!tilcdc_crtc->enabled) {
+		mutex_unlock(&tilcdc_crtc->enable_lock);
+		return;
+	}
+	tilcdc_crtc->frame_done = false;
+	tilcdc_clear(dev, LCDC_RASTER_CTRL_REG, LCDC_RASTER_ENABLE);
+
+	/*
+	 * Wait for framedone irq which will still come before putting
+	 * things to sleep..
+	 */
+	ret = wait_event_timeout(tilcdc_crtc->frame_done_wq,
+				 tilcdc_crtc->frame_done,
+				 msecs_to_jiffies(500));
+	if (ret == 0)
+		dev_err(dev->dev, "%s: timeout waiting for framedone\n",
+			__func__);
+
+	drm_crtc_vblank_off(crtc);
+
+	tilcdc_crtc_disable_irqs(dev);
+
+	pm_runtime_put_sync(dev->dev);
+
+	if (tilcdc_crtc->next_fb) {
+		drm_flip_work_queue(&tilcdc_crtc->unref_work,
+				    tilcdc_crtc->next_fb);
+		tilcdc_crtc->next_fb = NULL;
+	}
+
+	if (tilcdc_crtc->curr_fb) {
+		drm_flip_work_queue(&tilcdc_crtc->unref_work,
+				    tilcdc_crtc->curr_fb);
+		tilcdc_crtc->curr_fb = NULL;
+	}
+
+	drm_flip_work_commit(&tilcdc_crtc->unref_work, priv->wq);
+	tilcdc_crtc->last_vblank = ktime_set(0, 0);
+
+	tilcdc_crtc->enabled = false;
+	mutex_unlock(&tilcdc_crtc->enable_lock);
+}
+
+static void tilcdc_crtc_disable(struct drm_crtc *crtc)
+{
+	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
+	tilcdc_crtc_off(crtc, false);
+}
+
+void tilcdc_crtc_shutdown(struct drm_crtc *crtc)
+{
+	tilcdc_crtc_off(crtc, true);
+}
+
+static bool tilcdc_crtc_is_on(struct drm_crtc *crtc)
+{
+	return crtc->state && crtc->state->enable && crtc->state->active;
+}
+
+static void tilcdc_crtc_recover_work(struct work_struct *work)
+{
+	struct tilcdc_crtc *tilcdc_crtc =
+		container_of(work, struct tilcdc_crtc, recover_work);
+	struct drm_crtc *crtc = &tilcdc_crtc->base;
+
+	dev_info(crtc->dev->dev, "%s: Reset CRTC", __func__);
+
+	drm_modeset_lock_crtc(crtc, NULL);
+
+	if (!tilcdc_crtc_is_on(crtc))
+		goto out;
+
+	tilcdc_crtc_disable(crtc);
+	tilcdc_crtc_enable(crtc);
+out:
+	drm_modeset_unlock_crtc(crtc);
+}
+
+static void tilcdc_crtc_destroy(struct drm_crtc *crtc)
+{
+	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
+	struct tilcdc_drm_private *priv = crtc->dev->dev_private;
+
+	drm_modeset_lock_crtc(crtc, NULL);
+	tilcdc_crtc_disable(crtc);
+	drm_modeset_unlock_crtc(crtc);
+
+	flush_workqueue(priv->wq);
+
+	of_node_put(crtc->port);
+	drm_crtc_cleanup(crtc);
+	drm_flip_work_cleanup(&tilcdc_crtc->unref_work);
+}
+
+int tilcdc_crtc_update_fb(struct drm_crtc *crtc,
+		struct drm_framebuffer *fb,
+		struct drm_pending_vblank_event *event)
+{
+	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	unsigned long flags;
+
+	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
+
+	if (tilcdc_crtc->event) {
+		dev_err(dev->dev, "already pending page flip!\n");
+		return -EBUSY;
+	}
+
+	drm_framebuffer_reference(fb);
+
+	crtc->primary->fb = fb;
+
+	spin_lock_irqsave(&tilcdc_crtc->irq_lock, flags);
+
+	if (crtc->hwmode.vrefresh && ktime_to_ns(tilcdc_crtc->last_vblank)) {
+		ktime_t next_vblank;
+		s64 tdiff;
+
+		next_vblank = ktime_add_us(tilcdc_crtc->last_vblank,
+			1000000 / crtc->hwmode.vrefresh);
+
+		tdiff = ktime_to_us(ktime_sub(next_vblank, ktime_get()));
+
+		if (tdiff < TILCDC_VBLANK_SAFETY_THRESHOLD_US)
+			tilcdc_crtc->next_fb = fb;
+	}
+
+	if (tilcdc_crtc->next_fb != fb)
+		set_scanout(crtc, fb);
+
+	tilcdc_crtc->event = event;
+
+	spin_unlock_irqrestore(&tilcdc_crtc->irq_lock, flags);
+
+	return 0;
+}
+
+static bool tilcdc_crtc_mode_fixup(struct drm_crtc *crtc,
+		const struct drm_display_mode *mode,
+		struct drm_display_mode *adjusted_mode)
+{
+	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
+
+	if (!tilcdc_crtc->simulate_vesa_sync)
+		return true;
+
+	/*
+	 * tilcdc does not generate VESA-compliant sync but aligns
+	 * VS on the second edge of HS instead of first edge.
+	 * We use adjusted_mode, to fixup sync by aligning both rising
+	 * edges and add HSKEW offset to fix the sync.
+	 */
+	adjusted_mode->hskew = mode->hsync_end - mode->hsync_start;
+	adjusted_mode->flags |= DRM_MODE_FLAG_HSKEW;
+
+	if (mode->flags & DRM_MODE_FLAG_NHSYNC) {
+		adjusted_mode->flags |= DRM_MODE_FLAG_PHSYNC;
+		adjusted_mode->flags &= ~DRM_MODE_FLAG_NHSYNC;
+	} else {
+		adjusted_mode->flags |= DRM_MODE_FLAG_NHSYNC;
+		adjusted_mode->flags &= ~DRM_MODE_FLAG_PHSYNC;
+	}
+
+	return true;
+}
+
 static int tilcdc_crtc_atomic_check(struct drm_crtc *crtc,
 				    struct drm_crtc_state *state)
 {
@@ -560,7 +709,6 @@ static const struct drm_crtc_helper_funcs tilcdc_crtc_helper_funcs = {
 		.enable		= tilcdc_crtc_enable,
 		.disable	= tilcdc_crtc_disable,
 		.atomic_check	= tilcdc_crtc_atomic_check,
-		.mode_set_nofb	= tilcdc_crtc_mode_set_nofb,
 };
 
 int tilcdc_crtc_max_width(struct drm_crtc *crtc)
@@ -756,28 +904,48 @@ irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc)
 	}
 
 	if (stat & LCDC_FIFO_UNDERFLOW)
-		dev_err_ratelimited(dev->dev, "%s(0x%08x): FIFO underfow",
+		dev_err_ratelimited(dev->dev, "%s(0x%08x): FIFO underflow",
 				    __func__, stat);
 
-	/* For revision 2 only */
-	if (priv->rev == 2) {
-		if (stat & LCDC_FRAME_DONE) {
-			tilcdc_crtc->frame_done = true;
-			wake_up(&tilcdc_crtc->frame_done_wq);
-		}
+	if (stat & LCDC_PL_LOAD_DONE) {
+		complete(&tilcdc_crtc->palette_loaded);
+		if (priv->rev == 1)
+			tilcdc_clear(dev, LCDC_RASTER_CTRL_REG,
+				     LCDC_V1_PL_INT_ENA);
+		else
+			tilcdc_write(dev, LCDC_INT_ENABLE_CLR_REG,
+				     LCDC_V2_PL_INT_ENA);
+	}
 
-		if (stat & LCDC_SYNC_LOST) {
-			dev_err_ratelimited(dev->dev, "%s(0x%08x): Sync lost",
-					    __func__, stat);
-			tilcdc_crtc->frame_intact = false;
-			if (tilcdc_crtc->sync_lost_count++ >
-			    SYNC_LOST_COUNT_LIMIT) {
-				dev_err(dev->dev, "%s(0x%08x): Sync lost flood detected, disabling the interrupt", __func__, stat);
+	if (stat & LCDC_SYNC_LOST) {
+		dev_err_ratelimited(dev->dev, "%s(0x%08x): Sync lost",
+				    __func__, stat);
+		tilcdc_crtc->frame_intact = false;
+		if (tilcdc_crtc->sync_lost_count++ >
+		    SYNC_LOST_COUNT_LIMIT) {
+			dev_err(dev->dev, "%s(0x%08x): Sync lost flood detected, recovering", __func__, stat);
+			queue_work(system_wq, &tilcdc_crtc->recover_work);
+			if (priv->rev == 1)
+				tilcdc_clear(dev, LCDC_RASTER_CTRL_REG,
+					     LCDC_V1_SYNC_LOST_INT_ENA);
+			else
 				tilcdc_write(dev, LCDC_INT_ENABLE_CLR_REG,
 					     LCDC_SYNC_LOST);
-			}
+			tilcdc_crtc->sync_lost_count = 0;
 		}
+	}
+
+	if (stat & LCDC_FRAME_DONE) {
+		tilcdc_crtc->frame_done = true;
+		wake_up(&tilcdc_crtc->frame_done_wq);
+		/* rev 1 lcdc appears to hang if irq is not disbaled here */
+		if (priv->rev == 1)
+			tilcdc_clear(dev, LCDC_RASTER_CTRL_REG,
+				     LCDC_V1_FRAME_DONE_INT_ENA);
+	}
 
+	/* For revision 2 only */
+	if (priv->rev == 2) {
 		/* Indicate to LCDC that the interrupt service routine has
 		 * completed, see 13.3.6.1.6 in AM335x TRM.
 		 */
@@ -787,7 +955,7 @@ irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc)
 	return IRQ_HANDLED;
 }
 
-struct drm_crtc *tilcdc_crtc_create(struct drm_device *dev)
+int tilcdc_crtc_create(struct drm_device *dev)
 {
 	struct tilcdc_drm_private *priv = dev->dev_private;
 	struct tilcdc_crtc *tilcdc_crtc;
@@ -797,21 +965,33 @@ struct drm_crtc *tilcdc_crtc_create(struct drm_device *dev)
 	tilcdc_crtc = devm_kzalloc(dev->dev, sizeof(*tilcdc_crtc), GFP_KERNEL);
 	if (!tilcdc_crtc) {
 		dev_err(dev->dev, "allocation failed\n");
-		return NULL;
+		return -ENOMEM;
 	}
 
+	init_completion(&tilcdc_crtc->palette_loaded);
+	tilcdc_crtc->palette_base = dmam_alloc_coherent(dev->dev,
+					TILCDC_PALETTE_SIZE,
+					&tilcdc_crtc->palette_dma_handle,
+					GFP_KERNEL | __GFP_ZERO);
+	if (!tilcdc_crtc->palette_base)
+		return -ENOMEM;
+	*tilcdc_crtc->palette_base = TILCDC_PALETTE_FIRST_ENTRY;
+
 	crtc = &tilcdc_crtc->base;
 
 	ret = tilcdc_plane_init(dev, &tilcdc_crtc->primary);
 	if (ret < 0)
 		goto fail;
 
+	mutex_init(&tilcdc_crtc->enable_lock);
+
 	init_waitqueue_head(&tilcdc_crtc->frame_done_wq);
 
 	drm_flip_work_init(&tilcdc_crtc->unref_work,
 			"unref", unref_worker);
 
 	spin_lock_init(&tilcdc_crtc->irq_lock);
+	INIT_WORK(&tilcdc_crtc->recover_work, tilcdc_crtc_recover_work);
 
 	ret = drm_crtc_init_with_planes(dev, crtc,
 					&tilcdc_crtc->primary,
@@ -837,13 +1017,15 @@ struct drm_crtc *tilcdc_crtc_create(struct drm_device *dev)
 		if (!crtc->port) { /* This should never happen */
 			dev_err(dev->dev, "Port node not found in %s\n",
 				dev->dev->of_node->full_name);
+			ret = -EINVAL;
 			goto fail;
 		}
 	}
 
-	return crtc;
+	priv->crtc = crtc;
+	return 0;
 
 fail:
 	tilcdc_crtc_destroy(crtc);
-	return NULL;
+	return -ENOMEM;
 }
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index a694977c32f4..bd0a3bd07167 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -127,24 +127,16 @@ static int tilcdc_commit(struct drm_device *dev,
 	 * current layout.
 	 */
 
-	/* Keep HW on while we commit the state. */
-	pm_runtime_get_sync(dev->dev);
-
 	drm_atomic_helper_commit_modeset_disables(dev, state);
 
 	drm_atomic_helper_commit_planes(dev, state, 0);
 
 	drm_atomic_helper_commit_modeset_enables(dev, state);
 
-	/* Now HW should remain on if need becase the crtc is enabled */
-	pm_runtime_put_sync(dev->dev);
-
 	drm_atomic_helper_wait_for_vblanks(dev, state);
 
 	drm_atomic_helper_cleanup_planes(dev, state);
 
-	drm_atomic_state_free(state);
-
 	return 0;
 }
 
@@ -155,15 +147,11 @@ static const struct drm_mode_config_funcs mode_config_funcs = {
 	.atomic_commit = tilcdc_commit,
 };
 
-static int modeset_init(struct drm_device *dev)
+static void modeset_init(struct drm_device *dev)
 {
 	struct tilcdc_drm_private *priv = dev->dev_private;
 	struct tilcdc_module *mod;
 
-	drm_mode_config_init(dev);
-
-	priv->crtc = tilcdc_crtc_create(dev);
-
 	list_for_each_entry(mod, &module_list, list) {
 		DBG("loading module: %s", mod->name);
 		mod->funcs->modeset_init(mod, dev);
@@ -174,8 +162,6 @@ static int modeset_init(struct drm_device *dev)
 	dev->mode_config.max_width = tilcdc_crtc_max_width(priv->crtc);
 	dev->mode_config.max_height = 2048;
 	dev->mode_config.funcs = &mode_config_funcs;
-
-	return 0;
 }
 
 #ifdef CONFIG_CPU_FREQ
@@ -196,22 +182,29 @@ static int cpufreq_transition(struct notifier_block *nb,
  * DRM operations:
  */
 
-static int tilcdc_unload(struct drm_device *dev)
+static void tilcdc_fini(struct drm_device *dev)
 {
 	struct tilcdc_drm_private *priv = dev->dev_private;
 
-	tilcdc_remove_external_encoders(dev);
+	if (priv->crtc)
+		tilcdc_crtc_shutdown(priv->crtc);
+
+	if (priv->is_registered)
+		drm_dev_unregister(dev);
 
-	drm_fbdev_cma_fini(priv->fbdev);
 	drm_kms_helper_poll_fini(dev);
-	drm_mode_config_cleanup(dev);
-	drm_vblank_cleanup(dev);
+
+	if (priv->fbdev)
+		drm_fbdev_cma_fini(priv->fbdev);
 
 	drm_irq_uninstall(dev);
+	drm_mode_config_cleanup(dev);
+	tilcdc_remove_external_device(dev);
 
 #ifdef CONFIG_CPU_FREQ
-	cpufreq_unregister_notifier(&priv->freq_transition,
-			CPUFREQ_TRANSITION_NOTIFIER);
+	if (priv->freq_transition.notifier_call)
+		cpufreq_unregister_notifier(&priv->freq_transition,
+					    CPUFREQ_TRANSITION_NOTIFIER);
 #endif
 
 	if (priv->clk)
@@ -220,61 +213,71 @@ static int tilcdc_unload(struct drm_device *dev)
 	if (priv->mmio)
 		iounmap(priv->mmio);
 
-	flush_workqueue(priv->wq);
-	destroy_workqueue(priv->wq);
+	if (priv->wq) {
+		flush_workqueue(priv->wq);
+		destroy_workqueue(priv->wq);
+	}
 
 	dev->dev_private = NULL;
 
 	pm_runtime_disable(dev->dev);
 
-	return 0;
+	drm_dev_unref(dev);
 }
 
-static int tilcdc_load(struct drm_device *dev, unsigned long flags)
+static int tilcdc_init(struct drm_driver *ddrv, struct device *dev)
 {
-	struct platform_device *pdev = dev->platformdev;
-	struct device_node *node = pdev->dev.of_node;
+	struct drm_device *ddev;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct device_node *node = dev->of_node;
 	struct tilcdc_drm_private *priv;
 	struct resource *res;
 	u32 bpp = 0;
 	int ret;
 
-	priv = devm_kzalloc(dev->dev, sizeof(*priv), GFP_KERNEL);
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv) {
-		dev_err(dev->dev, "failed to allocate private data\n");
+		dev_err(dev, "failed to allocate private data\n");
 		return -ENOMEM;
 	}
 
-	dev->dev_private = priv;
+	ddev = drm_dev_alloc(ddrv, dev);
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
+
+	ddev->platformdev = pdev;
+	ddev->dev_private = priv;
+	platform_set_drvdata(pdev, ddev);
+	drm_mode_config_init(ddev);
 
 	priv->is_componentized =
-		tilcdc_get_external_components(dev->dev, NULL) > 0;
+		tilcdc_get_external_components(dev, NULL) > 0;
 
 	priv->wq = alloc_ordered_workqueue("tilcdc", 0);
 	if (!priv->wq) {
 		ret = -ENOMEM;
-		goto fail_unset_priv;
+		goto init_failed;
 	}
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!res) {
-		dev_err(dev->dev, "failed to get memory resource\n");
+		dev_err(dev, "failed to get memory resource\n");
 		ret = -EINVAL;
-		goto fail_free_wq;
+		goto init_failed;
 	}
 
 	priv->mmio = ioremap_nocache(res->start, resource_size(res));
 	if (!priv->mmio) {
-		dev_err(dev->dev, "failed to ioremap\n");
+		dev_err(dev, "failed to ioremap\n");
 		ret = -ENOMEM;
-		goto fail_free_wq;
+		goto init_failed;
 	}
 
-	priv->clk = clk_get(dev->dev, "fck");
+	priv->clk = clk_get(dev, "fck");
 	if (IS_ERR(priv->clk)) {
-		dev_err(dev->dev, "failed to get functional clock\n");
+		dev_err(dev, "failed to get functional clock\n");
 		ret = -ENODEV;
-		goto fail_iounmap;
+		goto init_failed;
 	}
 
 #ifdef CONFIG_CPU_FREQ
@@ -282,8 +285,9 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 	ret = cpufreq_register_notifier(&priv->freq_transition,
 			CPUFREQ_TRANSITION_NOTIFIER);
 	if (ret) {
-		dev_err(dev->dev, "failed to register cpufreq notifier\n");
-		goto fail_put_clk;
+		dev_err(dev, "failed to register cpufreq notifier\n");
+		priv->freq_transition.notifier_call = NULL;
+		goto init_failed;
 	}
 #endif
 
@@ -292,22 +296,22 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 
 	DBG("Maximum Bandwidth Value %d", priv->max_bandwidth);
 
-	if (of_property_read_u32(node, "ti,max-width", &priv->max_width))
+	if (of_property_read_u32(node, "max-width", &priv->max_width))
 		priv->max_width = TILCDC_DEFAULT_MAX_WIDTH;
 
 	DBG("Maximum Horizontal Pixel Width Value %dpixels", priv->max_width);
 
-	if (of_property_read_u32(node, "ti,max-pixelclock",
+	if (of_property_read_u32(node, "max-pixelclock",
 					&priv->max_pixelclock))
 		priv->max_pixelclock = TILCDC_DEFAULT_MAX_PIXELCLOCK;
 
 	DBG("Maximum Pixel Clock Value %dKHz", priv->max_pixelclock);
 
-	pm_runtime_enable(dev->dev);
+	pm_runtime_enable(dev);
 
 	/* Determine LCD IP Version */
-	pm_runtime_get_sync(dev->dev);
-	switch (tilcdc_read(dev, LCDC_PID_REG)) {
+	pm_runtime_get_sync(dev);
+	switch (tilcdc_read(ddev, LCDC_PID_REG)) {
 	case 0x4c100102:
 		priv->rev = 1;
 		break;
@@ -316,14 +320,14 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 		priv->rev = 2;
 		break;
 	default:
-		dev_warn(dev->dev, "Unknown PID Reg value 0x%08x, "
-				"defaulting to LCD revision 1\n",
-				tilcdc_read(dev, LCDC_PID_REG));
+		dev_warn(dev, "Unknown PID Reg value 0x%08x, "
+			"defaulting to LCD revision 1\n",
+			tilcdc_read(ddev, LCDC_PID_REG));
 		priv->rev = 1;
 		break;
 	}
 
-	pm_runtime_put_sync(dev->dev);
+	pm_runtime_put_sync(dev);
 
 	if (priv->rev == 1) {
 		DBG("Revision 1 LCDC supports only RGB565 format");
@@ -356,91 +360,67 @@ static int tilcdc_load(struct drm_device *dev, unsigned long flags)
 		}
 	}
 
-	ret = modeset_init(dev);
+	ret = tilcdc_crtc_create(ddev);
 	if (ret < 0) {
-		dev_err(dev->dev, "failed to initialize mode setting\n");
-		goto fail_cpufreq_unregister;
+		dev_err(dev, "failed to create crtc\n");
+		goto init_failed;
 	}
-
-	platform_set_drvdata(pdev, dev);
+	modeset_init(ddev);
 
 	if (priv->is_componentized) {
-		ret = component_bind_all(dev->dev, dev);
+		ret = component_bind_all(dev, ddev);
 		if (ret < 0)
-			goto fail_mode_config_cleanup;
+			goto init_failed;
 
-		ret = tilcdc_add_external_encoders(dev);
+		ret = tilcdc_add_component_encoder(ddev);
 		if (ret < 0)
-			goto fail_component_cleanup;
+			goto init_failed;
+	} else {
+		ret = tilcdc_attach_external_device(ddev);
+		if (ret)
+			goto init_failed;
 	}
 
-	if ((priv->num_encoders == 0) || (priv->num_connectors == 0)) {
-		dev_err(dev->dev, "no encoders/connectors found\n");
+	if (!priv->external_connector &&
+	    ((priv->num_encoders == 0) || (priv->num_connectors == 0))) {
+		dev_err(dev, "no encoders/connectors found\n");
 		ret = -ENXIO;
-		goto fail_external_cleanup;
+		goto init_failed;
 	}
 
-	ret = drm_vblank_init(dev, 1);
+	ret = drm_vblank_init(ddev, 1);
 	if (ret < 0) {
-		dev_err(dev->dev, "failed to initialize vblank\n");
-		goto fail_external_cleanup;
+		dev_err(dev, "failed to initialize vblank\n");
+		goto init_failed;
 	}
 
-	ret = drm_irq_install(dev, platform_get_irq(dev->platformdev, 0));
+	ret = drm_irq_install(ddev, platform_get_irq(pdev, 0));
 	if (ret < 0) {
-		dev_err(dev->dev, "failed to install IRQ handler\n");
-		goto fail_vblank_cleanup;
+		dev_err(dev, "failed to install IRQ handler\n");
+		goto init_failed;
 	}
 
-	drm_mode_config_reset(dev);
+	drm_mode_config_reset(ddev);
 
-	priv->fbdev = drm_fbdev_cma_init(dev, bpp,
-			dev->mode_config.num_crtc,
-			dev->mode_config.num_connector);
+	priv->fbdev = drm_fbdev_cma_init(ddev, bpp,
+			ddev->mode_config.num_crtc,
+			ddev->mode_config.num_connector);
 	if (IS_ERR(priv->fbdev)) {
 		ret = PTR_ERR(priv->fbdev);
-		goto fail_irq_uninstall;
+		goto init_failed;
 	}
 
-	drm_kms_helper_poll_init(dev);
+	drm_kms_helper_poll_init(ddev);
 
-	return 0;
-
-fail_irq_uninstall:
-	drm_irq_uninstall(dev);
-
-fail_vblank_cleanup:
-	drm_vblank_cleanup(dev);
-
-fail_component_cleanup:
-	if (priv->is_componentized)
-		component_unbind_all(dev->dev, dev);
-
-fail_mode_config_cleanup:
-	drm_mode_config_cleanup(dev);
-
-fail_external_cleanup:
-	tilcdc_remove_external_encoders(dev);
-
-fail_cpufreq_unregister:
-	pm_runtime_disable(dev->dev);
-#ifdef CONFIG_CPU_FREQ
-	cpufreq_unregister_notifier(&priv->freq_transition,
-			CPUFREQ_TRANSITION_NOTIFIER);
-
-fail_put_clk:
-#endif
-	clk_put(priv->clk);
-
-fail_iounmap:
-	iounmap(priv->mmio);
+	ret = drm_dev_register(ddev, 0);
+	if (ret)
+		goto init_failed;
 
-fail_free_wq:
-	flush_workqueue(priv->wq);
-	destroy_workqueue(priv->wq);
+	priv->is_registered = true;
+	return 0;
 
-fail_unset_priv:
-	dev->dev_private = NULL;
+init_failed:
+	tilcdc_fini(ddev);
 
 	return ret;
 }
@@ -575,9 +555,7 @@ static const struct file_operations fops = {
 	.open               = drm_open,
 	.release            = drm_release,
 	.unlocked_ioctl     = drm_ioctl,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl       = drm_compat_ioctl,
-#endif
 	.poll               = drm_poll,
 	.read               = drm_read,
 	.llseek             = no_llseek,
@@ -587,8 +565,6 @@ static const struct file_operations fops = {
 static struct drm_driver tilcdc_driver = {
 	.driver_features    = (DRIVER_HAVE_IRQ | DRIVER_GEM | DRIVER_MODESET |
 			       DRIVER_PRIME | DRIVER_ATOMIC),
-	.load               = tilcdc_load,
-	.unload             = tilcdc_unload,
 	.lastclose          = tilcdc_lastclose,
 	.irq_handler        = tilcdc_irq,
 	.get_vblank_counter = drm_vblank_no_hw_counter,
@@ -662,10 +638,9 @@ static const struct dev_pm_ops tilcdc_pm_ops = {
 /*
  * Platform driver:
  */
-
 static int tilcdc_bind(struct device *dev)
 {
-	return drm_platform_init(&tilcdc_driver, to_platform_device(dev));
+	return tilcdc_init(&tilcdc_driver, dev);
 }
 
 static void tilcdc_unbind(struct device *dev)
@@ -676,7 +651,7 @@ static void tilcdc_unbind(struct device *dev)
 	if (!ddev->dev_private)
 		return;
 
-	drm_put_dev(dev_get_drvdata(dev));
+	tilcdc_fini(dev_get_drvdata(dev));
 }
 
 static const struct component_master_ops tilcdc_comp_ops = {
@@ -699,7 +674,7 @@ static int tilcdc_pdev_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 	else if (ret == 0)
-		return drm_platform_init(&tilcdc_driver, pdev);
+		return tilcdc_init(&tilcdc_driver, &pdev->dev);
 	else
 		return component_master_add_with_match(&pdev->dev,
 						       &tilcdc_comp_ops,
@@ -714,7 +689,7 @@ static int tilcdc_pdev_remove(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 	else if (ret == 0)
-		drm_put_dev(platform_get_drvdata(pdev));
+		tilcdc_fini(platform_get_drvdata(pdev));
 	else
 		component_master_del(&pdev->dev, &tilcdc_comp_ops);
 
@@ -723,6 +698,7 @@ static int tilcdc_pdev_remove(struct platform_device *pdev)
 
 static struct of_device_id tilcdc_of_match[] = {
 		{ .compatible = "ti,am33xx-tilcdc", },
+		{ .compatible = "ti,da850-tilcdc", },
 		{ },
 };
 MODULE_DEVICE_TABLE(of, tilcdc_of_match);
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.h b/drivers/gpu/drm/tilcdc/tilcdc_drv.h
index 9780c37ec4cd..0e71daf5b5cb 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.h
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.h
@@ -33,6 +33,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_bridge.h>
 
 /* Defaulting to pixel clock defined on AM335x */
 #define TILCDC_DEFAULT_MAX_PIXELCLOCK  126000
@@ -87,8 +88,12 @@ struct tilcdc_drm_private {
 
 	unsigned int num_connectors;
 	struct drm_connector *connectors[8];
-	const struct drm_connector_helper_funcs *connector_funcs[8];
 
+	struct drm_encoder *external_encoder;
+	struct drm_connector *external_connector;
+	const struct drm_connector_helper_funcs *connector_funcs;
+
+	bool is_registered;
 	bool is_componentized;
 };
 
@@ -163,7 +168,7 @@ struct tilcdc_panel_info {
 
 #define DBG(fmt, ...) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
 
-struct drm_crtc *tilcdc_crtc_create(struct drm_device *dev);
+int tilcdc_crtc_create(struct drm_device *dev);
 irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc);
 void tilcdc_crtc_update_clk(struct drm_crtc *crtc);
 void tilcdc_crtc_set_panel_info(struct drm_crtc *crtc,
@@ -172,7 +177,7 @@ void tilcdc_crtc_set_simulate_vesa_sync(struct drm_crtc *crtc,
 					bool simulate_vesa_sync);
 int tilcdc_crtc_mode_valid(struct drm_crtc *crtc, struct drm_display_mode *mode);
 int tilcdc_crtc_max_width(struct drm_crtc *crtc);
-void tilcdc_crtc_disable(struct drm_crtc *crtc);
+void tilcdc_crtc_shutdown(struct drm_crtc *crtc);
 int tilcdc_crtc_update_fb(struct drm_crtc *crtc,
 		struct drm_framebuffer *fb,
 		struct drm_pending_vblank_event *event);
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_external.c b/drivers/gpu/drm/tilcdc/tilcdc_external.c
index 68e895021005..c67d7cd7d57e 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_external.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_external.c
@@ -10,6 +10,7 @@
 
 #include <linux/component.h>
 #include <linux/of_graph.h>
+#include <drm/drm_of.h>
 
 #include "tilcdc_drv.h"
 #include "tilcdc_external.h"
@@ -27,44 +28,50 @@ static const struct tilcdc_panel_info panel_info_tda998x = {
 		.raster_order           = 0,
 };
 
+static const struct tilcdc_panel_info panel_info_default = {
+		.ac_bias                = 255,
+		.ac_bias_intrpt         = 0,
+		.dma_burst_sz           = 16,
+		.bpp                    = 16,
+		.fdd                    = 0x80,
+		.tft_alt_mode           = 0,
+		.sync_edge              = 0,
+		.sync_ctrl              = 1,
+		.raster_order           = 0,
+};
+
 static int tilcdc_external_mode_valid(struct drm_connector *connector,
 				      struct drm_display_mode *mode)
 {
 	struct tilcdc_drm_private *priv = connector->dev->dev_private;
-	int ret, i;
+	int ret;
 
 	ret = tilcdc_crtc_mode_valid(priv->crtc, mode);
 	if (ret != MODE_OK)
 		return ret;
 
-	for (i = 0; i < priv->num_connectors &&
-		     priv->connectors[i] != connector; i++)
-		;
-
-	BUG_ON(priv->connectors[i] != connector);
-	BUG_ON(!priv->connector_funcs[i]);
+	BUG_ON(priv->external_connector != connector);
+	BUG_ON(!priv->connector_funcs);
 
 	/* If the connector has its own mode_valid call it. */
-	if (!IS_ERR(priv->connector_funcs[i]) &&
-	    priv->connector_funcs[i]->mode_valid)
-		return priv->connector_funcs[i]->mode_valid(connector, mode);
+	if (!IS_ERR(priv->connector_funcs) &&
+	    priv->connector_funcs->mode_valid)
+		return priv->connector_funcs->mode_valid(connector, mode);
 
 	return MODE_OK;
 }
 
-static int tilcdc_add_external_encoder(struct drm_device *dev,
-				       struct drm_connector *connector)
+static int tilcdc_add_external_connector(struct drm_device *dev,
+					 struct drm_connector *connector)
 {
 	struct tilcdc_drm_private *priv = dev->dev_private;
 	struct drm_connector_helper_funcs *connector_funcs;
 
-	priv->connectors[priv->num_connectors] = connector;
-	priv->encoders[priv->num_encoders++] = connector->encoder;
-
-	/* Only tda998x is supported at the moment. */
-	tilcdc_crtc_set_simulate_vesa_sync(priv->crtc, true);
-	tilcdc_crtc_set_panel_info(priv->crtc, &panel_info_tda998x);
+	/* There should never be more than one connector */
+	if (WARN_ON(priv->external_connector))
+		return -EINVAL;
 
+	priv->external_connector = connector;
 	connector_funcs = devm_kzalloc(dev->dev, sizeof(*connector_funcs),
 				       GFP_KERNEL);
 	if (!connector_funcs)
@@ -77,56 +84,177 @@ static int tilcdc_add_external_encoder(struct drm_device *dev,
 	 * everything else but use our own mode_valid() (above).
 	 */
 	if (connector->helper_private) {
-		priv->connector_funcs[priv->num_connectors] =
-			connector->helper_private;
-		*connector_funcs = *priv->connector_funcs[priv->num_connectors];
+		priv->connector_funcs =	connector->helper_private;
+		*connector_funcs = *priv->connector_funcs;
 	} else {
-		priv->connector_funcs[priv->num_connectors] = ERR_PTR(-ENOENT);
+		priv->connector_funcs = ERR_PTR(-ENOENT);
 	}
 	connector_funcs->mode_valid = tilcdc_external_mode_valid;
 	drm_connector_helper_add(connector, connector_funcs);
-	priv->num_connectors++;
 
-	dev_dbg(dev->dev, "External encoder '%s' connected\n",
-		connector->encoder->name);
+	dev_dbg(dev->dev, "External connector '%s' connected\n",
+		connector->name);
 
 	return 0;
 }
 
-int tilcdc_add_external_encoders(struct drm_device *dev)
+static
+struct drm_connector *tilcdc_encoder_find_connector(struct drm_device *ddev,
+						    struct drm_encoder *encoder)
 {
-	struct tilcdc_drm_private *priv = dev->dev_private;
 	struct drm_connector *connector;
-	int num_internal_connectors = priv->num_connectors;
-
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-		bool found = false;
-		int i, ret;
-
-		for (i = 0; i < num_internal_connectors; i++)
-			if (connector == priv->connectors[i])
-				found = true;
-		if (!found) {
-			ret = tilcdc_add_external_encoder(dev, connector);
-			if (ret)
-				return ret;
-		}
+	int i;
+
+	list_for_each_entry(connector, &ddev->mode_config.connector_list, head)
+		for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++)
+			if (connector->encoder_ids[i] == encoder->base.id)
+				return connector;
+
+	dev_err(ddev->dev, "No connector found for %s encoder (id %d)\n",
+		encoder->name, encoder->base.id);
+
+	return NULL;
+}
+
+int tilcdc_add_component_encoder(struct drm_device *ddev)
+{
+	struct tilcdc_drm_private *priv = ddev->dev_private;
+	struct drm_connector *connector;
+	struct drm_encoder *encoder;
+
+	list_for_each_entry(encoder, &ddev->mode_config.encoder_list, head)
+		if (encoder->possible_crtcs & (1 << priv->crtc->index))
+			break;
+
+	if (!encoder) {
+		dev_err(ddev->dev, "%s: No suitable encoder found\n", __func__);
+		return -ENODEV;
 	}
-	return 0;
+
+	connector = tilcdc_encoder_find_connector(ddev, encoder);
+
+	if (!connector)
+		return -ENODEV;
+
+	/* Only tda998x is supported at the moment. */
+	tilcdc_crtc_set_simulate_vesa_sync(priv->crtc, true);
+	tilcdc_crtc_set_panel_info(priv->crtc, &panel_info_tda998x);
+
+	return tilcdc_add_external_connector(ddev, connector);
 }
 
-void tilcdc_remove_external_encoders(struct drm_device *dev)
+void tilcdc_remove_external_device(struct drm_device *dev)
 {
 	struct tilcdc_drm_private *priv = dev->dev_private;
-	int i;
 
 	/* Restore the original helper functions, if any. */
-	for (i = 0; i < priv->num_connectors; i++)
-		if (IS_ERR(priv->connector_funcs[i]))
-			drm_connector_helper_add(priv->connectors[i], NULL);
-		else if (priv->connector_funcs[i])
-			drm_connector_helper_add(priv->connectors[i],
-						 priv->connector_funcs[i]);
+	if (IS_ERR(priv->connector_funcs))
+		drm_connector_helper_add(priv->external_connector, NULL);
+	else if (priv->connector_funcs)
+		drm_connector_helper_add(priv->external_connector,
+					 priv->connector_funcs);
+}
+
+static const struct drm_encoder_funcs tilcdc_external_encoder_funcs = {
+	.destroy	= drm_encoder_cleanup,
+};
+
+static
+int tilcdc_attach_bridge(struct drm_device *ddev, struct drm_bridge *bridge)
+{
+	struct tilcdc_drm_private *priv = ddev->dev_private;
+	struct drm_connector *connector;
+	int ret;
+
+	priv->external_encoder->possible_crtcs = BIT(0);
+	priv->external_encoder->bridge = bridge;
+	bridge->encoder = priv->external_encoder;
+
+	ret = drm_bridge_attach(ddev, bridge);
+	if (ret) {
+		dev_err(ddev->dev, "drm_bridge_attach() failed %d\n", ret);
+		return ret;
+	}
+
+	tilcdc_crtc_set_panel_info(priv->crtc, &panel_info_default);
+
+	connector = tilcdc_encoder_find_connector(ddev, priv->external_encoder);
+	if (!connector)
+		return -ENODEV;
+
+	ret = tilcdc_add_external_connector(ddev, connector);
+
+	return ret;
+}
+
+static int tilcdc_node_has_port(struct device_node *dev_node)
+{
+	struct device_node *node;
+
+	node = of_get_child_by_name(dev_node, "ports");
+	if (!node)
+		node = of_get_child_by_name(dev_node, "port");
+	if (!node)
+		return 0;
+	of_node_put(node);
+
+	return 1;
+}
+
+static
+struct device_node *tilcdc_get_remote_node(struct device_node *node)
+{
+	struct device_node *ep;
+	struct device_node *parent;
+
+	if (!tilcdc_node_has_port(node))
+		return NULL;
+
+	ep = of_graph_get_next_endpoint(node, NULL);
+	if (!ep)
+		return NULL;
+
+	parent = of_graph_get_remote_port_parent(ep);
+	of_node_put(ep);
+
+	return parent;
+}
+
+int tilcdc_attach_external_device(struct drm_device *ddev)
+{
+	struct tilcdc_drm_private *priv = ddev->dev_private;
+	struct device_node *remote_node;
+	struct drm_bridge *bridge;
+	int ret;
+
+	remote_node = tilcdc_get_remote_node(ddev->dev->of_node);
+	if (!remote_node)
+		return 0;
+
+	bridge = of_drm_find_bridge(remote_node);
+	of_node_put(remote_node);
+	if (!bridge)
+		return -EPROBE_DEFER;
+
+	priv->external_encoder = devm_kzalloc(ddev->dev,
+					      sizeof(*priv->external_encoder),
+					      GFP_KERNEL);
+	if (!priv->external_encoder)
+		return -ENOMEM;
+
+	ret = drm_encoder_init(ddev, priv->external_encoder,
+			       &tilcdc_external_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
+	if (ret) {
+		dev_err(ddev->dev, "drm_encoder_init() failed %d\n", ret);
+		return ret;
+	}
+
+	ret = tilcdc_attach_bridge(ddev, bridge);
+	if (ret)
+		drm_encoder_cleanup(priv->external_encoder);
+
+	return ret;
 }
 
 static int dev_match_of(struct device *dev, void *data)
@@ -140,16 +268,10 @@ int tilcdc_get_external_components(struct device *dev,
 	struct device_node *node;
 	struct device_node *ep = NULL;
 	int count = 0;
+	int ret = 0;
 
-	/* Avoid error print by of_graph_get_next_endpoint() if there
-	 * is no ports present.
-	 */
-	node = of_get_child_by_name(dev->of_node, "ports");
-	if (!node)
-		node = of_get_child_by_name(dev->of_node, "port");
-	if (!node)
+	if (!tilcdc_node_has_port(dev->of_node))
 		return 0;
-	of_node_put(node);
 
 	while ((ep = of_graph_get_next_endpoint(dev->of_node, ep))) {
 		node = of_graph_get_remote_port_parent(ep);
@@ -159,16 +281,20 @@ int tilcdc_get_external_components(struct device *dev,
 		}
 
 		dev_dbg(dev, "Subdevice node '%s' found\n", node->name);
-		if (match)
-			component_match_add(dev, match, dev_match_of, node);
-		of_node_put(node);
-		count++;
-	}
 
-	if (count > 1) {
-		dev_err(dev, "Only one external encoder is supported\n");
-		return -EINVAL;
+		if (of_device_is_compatible(node, "nxp,tda998x")) {
+			if (match)
+				drm_of_component_match_add(dev, match,
+							   dev_match_of, node);
+			ret = 1;
+		}
+
+		of_node_put(node);
+		if (count++ > 1) {
+			dev_err(dev, "Only one port is supported\n");
+			return -EINVAL;
+		}
 	}
 
-	return count;
+	return ret;
 }
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_external.h b/drivers/gpu/drm/tilcdc/tilcdc_external.h
index c700e0c1623e..763d18f006c7 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_external.h
+++ b/drivers/gpu/drm/tilcdc/tilcdc_external.h
@@ -18,8 +18,9 @@
 #ifndef __TILCDC_EXTERNAL_H__
 #define __TILCDC_EXTERNAL_H__
 
-int tilcdc_add_external_encoders(struct drm_device *dev);
-void tilcdc_remove_external_encoders(struct drm_device *dev);
+int tilcdc_add_component_encoder(struct drm_device *dev);
+void tilcdc_remove_external_device(struct drm_device *dev);
 int tilcdc_get_external_components(struct device *dev,
 				   struct component_match **match);
+int tilcdc_attach_external_device(struct drm_device *ddev);
 #endif /* __TILCDC_SLAVE_H__ */
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_panel.c b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
index 2134bb20fbe9..28c3e2f44f64 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_panel.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
@@ -144,13 +144,6 @@ static void panel_connector_destroy(struct drm_connector *connector)
 	drm_connector_cleanup(connector);
 }
 
-static enum drm_connector_status panel_connector_detect(
-		struct drm_connector *connector,
-		bool force)
-{
-	return connector_status_connected;
-}
-
 static int panel_connector_get_modes(struct drm_connector *connector)
 {
 	struct drm_device *dev = connector->dev;
@@ -197,7 +190,6 @@ static struct drm_encoder *panel_connector_best_encoder(
 static const struct drm_connector_funcs panel_connector_funcs = {
 	.destroy            = panel_connector_destroy,
 	.dpms               = drm_atomic_helper_connector_dpms,
-	.detect             = panel_connector_detect,
 	.fill_modes         = drm_helper_probe_single_connector_modes,
 	.reset              = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -240,8 +232,6 @@ static struct drm_connector *panel_connector_create(struct drm_device *dev,
 	if (ret)
 		goto fail;
 
-	drm_connector_register(connector);
-
 	return connector;
 
 fail:
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_plane.c b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
index 74c65fa859b2..8a6a50d74aff 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_plane.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
@@ -39,7 +39,7 @@ static int tilcdc_plane_atomic_check(struct drm_plane *plane,
 {
 	struct drm_crtc_state *crtc_state;
 	struct drm_plane_state *old_state = plane->state;
-	unsigned int depth, bpp;
+	unsigned int pitch;
 
 	if (!state->crtc)
 		return 0;
@@ -68,8 +68,9 @@ static int tilcdc_plane_atomic_check(struct drm_plane *plane,
 		return -EINVAL;
 	}
 
-	drm_fb_get_bpp_depth(state->fb->pixel_format, &depth, &bpp);
-	if (state->fb->pitches[0] != crtc_state->mode.hdisplay * bpp / 8) {
+	pitch = crtc_state->mode.hdisplay *
+		drm_format_plane_cpp(state->fb->pixel_format, 0);
+	if (state->fb->pitches[0] != pitch) {
 		dev_err(plane->dev->dev,
 			"Invalid pitch: fb and crtc widths must be the same");
 		return -EINVAL;
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_regs.h b/drivers/gpu/drm/tilcdc/tilcdc_regs.h
index f57c0d62c76a..9d528c0a67a4 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_regs.h
+++ b/drivers/gpu/drm/tilcdc/tilcdc_regs.h
@@ -34,11 +34,14 @@
 
 /* LCDC DMA Control Register */
 #define LCDC_DMA_BURST_SIZE(x)                   ((x) << 4)
+#define LCDC_DMA_BURST_SIZE_MASK                 ((0x7) << 4)
 #define LCDC_DMA_BURST_1                         0x0
 #define LCDC_DMA_BURST_2                         0x1
 #define LCDC_DMA_BURST_4                         0x2
 #define LCDC_DMA_BURST_8                         0x3
 #define LCDC_DMA_BURST_16                        0x4
+#define LCDC_DMA_FIFO_THRESHOLD(x)               ((x) << 8)
+#define LCDC_DMA_FIFO_THRESHOLD_MASK             ((0x3) << 8)
 #define LCDC_V1_END_OF_FRAME_INT_ENA             BIT(2)
 #define LCDC_V2_END_OF_FRAME0_INT_ENA            BIT(8)
 #define LCDC_V2_END_OF_FRAME1_INT_ENA            BIT(9)
@@ -46,10 +49,12 @@
 
 /* LCDC Control Register */
 #define LCDC_CLK_DIVISOR(x)                      ((x) << 8)
+#define LCDC_CLK_DIVISOR_MASK                    ((0xFF) << 8)
 #define LCDC_RASTER_MODE                         0x01
 
 /* LCDC Raster Control Register */
 #define LCDC_PALETTE_LOAD_MODE(x)                ((x) << 20)
+#define LCDC_PALETTE_LOAD_MODE_MASK              ((0x3) << 20)
 #define PALETTE_AND_DATA                         0x00
 #define PALETTE_ONLY                             0x01
 #define DATA_ONLY                                0x02
@@ -61,6 +66,8 @@
 #define LCDC_V2_UNDERFLOW_INT_ENA                BIT(5)
 #define LCDC_V1_PL_INT_ENA                       BIT(4)
 #define LCDC_V2_PL_INT_ENA                       BIT(6)
+#define LCDC_V1_SYNC_LOST_INT_ENA                BIT(5)
+#define LCDC_V1_FRAME_DONE_INT_ENA               BIT(3)
 #define LCDC_MONOCHROME_MODE                     BIT(1)
 #define LCDC_RASTER_ENABLE                       BIT(0)
 #define LCDC_TFT_ALT_ENABLE                      BIT(23)
@@ -74,7 +81,9 @@
 
 /* LCDC Raster Timing 2 Register */
 #define LCDC_AC_BIAS_TRANSITIONS_PER_INT(x)      ((x) << 16)
+#define LCDC_AC_BIAS_TRANSITIONS_PER_INT_MASK    ((0xF) << 16)
 #define LCDC_AC_BIAS_FREQUENCY(x)                ((x) << 8)
+#define LCDC_AC_BIAS_FREQUENCY_MASK              ((0xFF) << 8)
 #define LCDC_SYNC_CTRL                           BIT(25)
 #define LCDC_SYNC_EDGE                           BIT(24)
 #define LCDC_INVERT_PIXEL_CLOCK                  BIT(22)
@@ -139,6 +148,12 @@ static inline u32 tilcdc_read(struct drm_device *dev, u32 reg)
 	return ioread32(priv->mmio + reg);
 }
 
+static inline void tilcdc_write_mask(struct drm_device *dev, u32 reg,
+				     u32 val, u32 mask)
+{
+	tilcdc_write(dev, reg, (tilcdc_read(dev, reg) & ~mask) | (val & mask));
+}
+
 static inline void tilcdc_set(struct drm_device *dev, u32 reg, u32 mask)
 {
 	tilcdc_write(dev, reg, tilcdc_read(dev, reg) | mask);
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
index 458043a53995..aabfad882e23 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
@@ -249,8 +249,6 @@ static struct drm_connector *tfp410_connector_create(struct drm_device *dev,
 	if (ret)
 		goto fail;
 
-	drm_connector_register(connector);
-
 	return connector;
 
 fail:
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index fc6217dfe401..d5063618efa7 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -148,7 +148,7 @@ static void ttm_bo_release_list(struct kref *list_kref)
 	BUG_ON(!list_empty(&bo->ddestroy));
 	ttm_tt_destroy(bo->ttm);
 	atomic_dec(&bo->glob->bo_count);
-	fence_put(bo->moving);
+	dma_fence_put(bo->moving);
 	if (bo->resv == &bo->ttm_resv)
 		reservation_object_fini(&bo->ttm_resv);
 	mutex_destroy(&bo->wu_mutex);
@@ -426,20 +426,20 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 {
 	struct reservation_object_list *fobj;
-	struct fence *fence;
+	struct dma_fence *fence;
 	int i;
 
 	fobj = reservation_object_get_list(bo->resv);
 	fence = reservation_object_get_excl(bo->resv);
 	if (fence && !fence->ops->signaled)
-		fence_enable_sw_signaling(fence);
+		dma_fence_enable_sw_signaling(fence);
 
 	for (i = 0; fobj && i < fobj->shared_count; ++i) {
 		fence = rcu_dereference_protected(fobj->shared[i],
 					reservation_object_held(bo->resv));
 
 		if (!fence->ops->signaled)
-			fence_enable_sw_signaling(fence);
+			dma_fence_enable_sw_signaling(fence);
 	}
 }
 
@@ -717,6 +717,20 @@ out:
 	return ret;
 }
 
+bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
+			      const struct ttm_place *place)
+{
+	/* Don't evict this BO if it's outside of the
+	 * requested placement range
+	 */
+	if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
+	    (place->lpfn && place->lpfn <= bo->mem.start))
+		return false;
+
+	return true;
+}
+EXPORT_SYMBOL(ttm_bo_eviction_valuable);
+
 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 				uint32_t mem_type,
 				const struct ttm_place *place,
@@ -731,21 +745,16 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 	spin_lock(&glob->lru_lock);
 	list_for_each_entry(bo, &man->lru, lru) {
 		ret = __ttm_bo_reserve(bo, false, true, NULL);
-		if (!ret) {
-			if (place && (place->fpfn || place->lpfn)) {
-				/* Don't evict this BO if it's outside of the
-				 * requested placement range
-				 */
-				if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
-				    (place->lpfn && place->lpfn <= bo->mem.start)) {
-					__ttm_bo_unreserve(bo);
-					ret = -EBUSY;
-					continue;
-				}
-			}
+		if (ret)
+			continue;
 
-			break;
+		if (place && !bdev->driver->eviction_valuable(bo, place)) {
+			__ttm_bo_unreserve(bo);
+			ret = -EBUSY;
+			continue;
 		}
+
+		break;
 	}
 
 	if (ret) {
@@ -792,11 +801,11 @@ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 				 struct ttm_mem_type_manager *man,
 				 struct ttm_mem_reg *mem)
 {
-	struct fence *fence;
+	struct dma_fence *fence;
 	int ret;
 
 	spin_lock(&man->move_lock);
-	fence = fence_get(man->move);
+	fence = dma_fence_get(man->move);
 	spin_unlock(&man->move_lock);
 
 	if (fence) {
@@ -806,7 +815,7 @@ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 		if (unlikely(ret))
 			return ret;
 
-		fence_put(bo->moving);
+		dma_fence_put(bo->moving);
 		bo->moving = fence;
 	}
 
@@ -1286,7 +1295,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
 {
 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 	struct ttm_bo_global *glob = bdev->glob;
-	struct fence *fence;
+	struct dma_fence *fence;
 	int ret;
 
 	/*
@@ -1309,12 +1318,12 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
 	spin_unlock(&glob->lru_lock);
 
 	spin_lock(&man->move_lock);
-	fence = fence_get(man->move);
+	fence = dma_fence_get(man->move);
 	spin_unlock(&man->move_lock);
 
 	if (fence) {
-		ret = fence_wait(fence, false);
-		fence_put(fence);
+		ret = dma_fence_wait(fence, false);
+		dma_fence_put(fence);
 		if (ret) {
 			if (allow_errors) {
 				return ret;
@@ -1343,7 +1352,7 @@ int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
 		       mem_type);
 		return ret;
 	}
-	fence_put(man->move);
+	dma_fence_put(man->move);
 
 	man->use_type = false;
 	man->has_type = false;
@@ -1602,7 +1611,14 @@ EXPORT_SYMBOL(ttm_bo_unmap_virtual);
 int ttm_bo_wait(struct ttm_buffer_object *bo,
 		bool interruptible, bool no_wait)
 {
-	long timeout = no_wait ? 0 : 15 * HZ;
+	long timeout = 15 * HZ;
+
+	if (no_wait) {
+		if (reservation_object_test_signaled_rcu(bo->resv, true))
+			return 0;
+		else
+			return -EBUSY;
+	}
 
 	timeout = reservation_object_wait_timeout_rcu(bo->resv, true,
 						      interruptible, timeout);
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index bf6e21655c57..d0459b392e5e 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -644,7 +644,7 @@ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 EXPORT_SYMBOL(ttm_bo_kunmap);
 
 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
-			      struct fence *fence,
+			      struct dma_fence *fence,
 			      bool evict,
 			      struct ttm_mem_reg *new_mem)
 {
@@ -674,8 +674,8 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 		 * operation has completed.
 		 */
 
-		fence_put(bo->moving);
-		bo->moving = fence_get(fence);
+		dma_fence_put(bo->moving);
+		bo->moving = dma_fence_get(fence);
 
 		ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 		if (ret)
@@ -706,7 +706,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
 
 int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
-			 struct fence *fence, bool evict,
+			 struct dma_fence *fence, bool evict,
 			 struct ttm_mem_reg *new_mem)
 {
 	struct ttm_bo_device *bdev = bo->bdev;
@@ -730,8 +730,8 @@ int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
 		 * operation has completed.
 		 */
 
-		fence_put(bo->moving);
-		bo->moving = fence_get(fence);
+		dma_fence_put(bo->moving);
+		bo->moving = dma_fence_get(fence);
 
 		ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 		if (ret)
@@ -761,16 +761,16 @@ int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
 		 */
 
 		spin_lock(&from->move_lock);
-		if (!from->move || fence_is_later(fence, from->move)) {
-			fence_put(from->move);
-			from->move = fence_get(fence);
+		if (!from->move || dma_fence_is_later(fence, from->move)) {
+			dma_fence_put(from->move);
+			from->move = dma_fence_get(fence);
 		}
 		spin_unlock(&from->move_lock);
 
 		ttm_bo_free_old_node(bo);
 
-		fence_put(bo->moving);
-		bo->moving = fence_get(fence);
+		dma_fence_put(bo->moving);
+		bo->moving = dma_fence_get(fence);
 
 	} else {
 		/**
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index a6ed9d5e5167..68ef993ab431 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -54,7 +54,7 @@ static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,
 	/*
 	 * Quick non-stalling check for idle.
 	 */
-	if (fence_is_signaled(bo->moving))
+	if (dma_fence_is_signaled(bo->moving))
 		goto out_clear;
 
 	/*
@@ -67,14 +67,14 @@ static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,
 			goto out_unlock;
 
 		up_read(&vma->vm_mm->mmap_sem);
-		(void) fence_wait(bo->moving, true);
+		(void) dma_fence_wait(bo->moving, true);
 		goto out_unlock;
 	}
 
 	/*
 	 * Ordinary wait.
 	 */
-	ret = fence_wait(bo->moving, true);
+	ret = dma_fence_wait(bo->moving, true);
 	if (unlikely(ret != 0)) {
 		ret = (ret != -ERESTARTSYS) ? VM_FAULT_SIGBUS :
 			VM_FAULT_NOPAGE;
@@ -82,7 +82,7 @@ static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,
 	}
 
 out_clear:
-	fence_put(bo->moving);
+	dma_fence_put(bo->moving);
 	bo->moving = NULL;
 
 out_unlock:
@@ -101,7 +101,7 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	struct page *page;
 	int ret;
 	int i;
-	unsigned long address = (unsigned long)vmf->virtual_address;
+	unsigned long address = vmf->address;
 	int retval = VM_FAULT_NOPAGE;
 	struct ttm_mem_type_manager *man =
 		&bdev->man[bo->mem.mem_type];
diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
index a80717b35dc6..d35bc491e8de 100644
--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c
+++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
@@ -179,7 +179,8 @@ int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket,
 EXPORT_SYMBOL(ttm_eu_reserve_buffers);
 
 void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket,
-				 struct list_head *list, struct fence *fence)
+				 struct list_head *list,
+				 struct dma_fence *fence)
 {
 	struct ttm_validate_buffer *entry;
 	struct ttm_buffer_object *bo;
diff --git a/drivers/gpu/drm/udl/udl_drv.c b/drivers/gpu/drm/udl/udl_drv.c
index cc45d98f9bb5..cd8b01727734 100644
--- a/drivers/gpu/drm/udl/udl_drv.c
+++ b/drivers/gpu/drm/udl/udl_drv.c
@@ -44,9 +44,7 @@ static const struct file_operations udl_driver_fops = {
 	.read = drm_read,
 	.unlocked_ioctl	= drm_ioctl,
 	.release = drm_release,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.llseek = noop_llseek,
 };
 
diff --git a/drivers/gpu/drm/udl/udl_fb.c b/drivers/gpu/drm/udl/udl_fb.c
index 611b6b9bb3cb..167f42c67c7c 100644
--- a/drivers/gpu/drm/udl/udl_fb.c
+++ b/drivers/gpu/drm/udl/udl_fb.c
@@ -254,16 +254,10 @@ static int udl_fb_release(struct fb_info *info, int user)
 
 static struct fb_ops udlfb_ops = {
 	.owner = THIS_MODULE,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_fillrect = drm_fb_helper_sys_fillrect,
 	.fb_copyarea = drm_fb_helper_sys_copyarea,
 	.fb_imageblit = drm_fb_helper_sys_imageblit,
-	.fb_pan_display = drm_fb_helper_pan_display,
-	.fb_blank = drm_fb_helper_blank,
-	.fb_setcmap = drm_fb_helper_setcmap,
-	.fb_debug_enter = drm_fb_helper_debug_enter,
-	.fb_debug_leave = drm_fb_helper_debug_leave,
 	.fb_mmap = udl_fb_mmap,
 	.fb_open = udl_fb_open,
 	.fb_release = udl_fb_release,
diff --git a/drivers/gpu/drm/udl/udl_gem.c b/drivers/gpu/drm/udl/udl_gem.c
index 818e70712b18..3c0c4bd3f750 100644
--- a/drivers/gpu/drm/udl/udl_gem.c
+++ b/drivers/gpu/drm/udl/udl_gem.c
@@ -107,14 +107,13 @@ int udl_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	unsigned int page_offset;
 	int ret = 0;
 
-	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
-		PAGE_SHIFT;
+	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
 
 	if (!obj->pages)
 		return VM_FAULT_SIGBUS;
 
 	page = obj->pages[page_offset];
-	ret = vm_insert_page(vma, (unsigned long)vmf->virtual_address, page);
+	ret = vm_insert_page(vma, vmf->address, page);
 	switch (ret) {
 	case -EAGAIN:
 	case 0:
diff --git a/drivers/gpu/drm/vc4/Makefile b/drivers/gpu/drm/vc4/Makefile
index fb77db755e0a..7757f69a8a77 100644
--- a/drivers/gpu/drm/vc4/Makefile
+++ b/drivers/gpu/drm/vc4/Makefile
@@ -11,6 +11,7 @@ vc4-y := \
 	vc4_kms.o \
 	vc4_gem.o \
 	vc4_hdmi.o \
+	vc4_vec.o \
 	vc4_hvs.o \
 	vc4_irq.o \
 	vc4_plane.o \
diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c
index 7f08d681a74b..a0fd3e66bc4b 100644
--- a/drivers/gpu/drm/vc4/vc4_crtc.c
+++ b/drivers/gpu/drm/vc4/vc4_crtc.c
@@ -83,8 +83,7 @@ struct vc4_crtc_data {
 	/* Which channel of the HVS this pixelvalve sources from. */
 	int hvs_channel;
 
-	enum vc4_encoder_type encoder0_type;
-	enum vc4_encoder_type encoder1_type;
+	enum vc4_encoder_type encoder_types[4];
 };
 
 #define CRTC_WRITE(offset, val) writel(val, vc4_crtc->regs + (offset))
@@ -669,6 +668,14 @@ void vc4_disable_vblank(struct drm_device *dev, unsigned int crtc_id)
 	CRTC_WRITE(PV_INTEN, 0);
 }
 
+/* Must be called with the event lock held */
+bool vc4_event_pending(struct drm_crtc *crtc)
+{
+	struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+
+	return !!vc4_crtc->event;
+}
+
 static void vc4_crtc_handle_page_flip(struct vc4_crtc *vc4_crtc)
 {
 	struct drm_crtc *crtc = &vc4_crtc->base;
@@ -859,20 +866,26 @@ static const struct drm_crtc_helper_funcs vc4_crtc_helper_funcs = {
 
 static const struct vc4_crtc_data pv0_data = {
 	.hvs_channel = 0,
-	.encoder0_type = VC4_ENCODER_TYPE_DSI0,
-	.encoder1_type = VC4_ENCODER_TYPE_DPI,
+	.encoder_types = {
+		[PV_CONTROL_CLK_SELECT_DSI] = VC4_ENCODER_TYPE_DSI0,
+		[PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI] = VC4_ENCODER_TYPE_DPI,
+	},
 };
 
 static const struct vc4_crtc_data pv1_data = {
 	.hvs_channel = 2,
-	.encoder0_type = VC4_ENCODER_TYPE_DSI1,
-	.encoder1_type = VC4_ENCODER_TYPE_SMI,
+	.encoder_types = {
+		[PV_CONTROL_CLK_SELECT_DSI] = VC4_ENCODER_TYPE_DSI1,
+		[PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI] = VC4_ENCODER_TYPE_SMI,
+	},
 };
 
 static const struct vc4_crtc_data pv2_data = {
 	.hvs_channel = 1,
-	.encoder0_type = VC4_ENCODER_TYPE_VEC,
-	.encoder1_type = VC4_ENCODER_TYPE_HDMI,
+	.encoder_types = {
+		[PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI] = VC4_ENCODER_TYPE_HDMI,
+		[PV_CONTROL_CLK_SELECT_VEC] = VC4_ENCODER_TYPE_VEC,
+	},
 };
 
 static const struct of_device_id vc4_crtc_dt_match[] = {
@@ -886,17 +899,20 @@ static void vc4_set_crtc_possible_masks(struct drm_device *drm,
 					struct drm_crtc *crtc)
 {
 	struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+	const struct vc4_crtc_data *crtc_data = vc4_crtc->data;
+	const enum vc4_encoder_type *encoder_types = crtc_data->encoder_types;
 	struct drm_encoder *encoder;
 
 	drm_for_each_encoder(encoder, drm) {
 		struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder);
-
-		if (vc4_encoder->type == vc4_crtc->data->encoder0_type) {
-			vc4_encoder->clock_select = 0;
-			encoder->possible_crtcs |= drm_crtc_mask(crtc);
-		} else if (vc4_encoder->type == vc4_crtc->data->encoder1_type) {
-			vc4_encoder->clock_select = 1;
-			encoder->possible_crtcs |= drm_crtc_mask(crtc);
+		int i;
+
+		for (i = 0; i < ARRAY_SIZE(crtc_data->encoder_types); i++) {
+			if (vc4_encoder->type == encoder_types[i]) {
+				vc4_encoder->clock_select = i;
+				encoder->possible_crtcs |= drm_crtc_mask(crtc);
+				break;
+			}
 		}
 	}
 }
diff --git a/drivers/gpu/drm/vc4/vc4_debugfs.c b/drivers/gpu/drm/vc4/vc4_debugfs.c
index 245115d49c46..caf817bac885 100644
--- a/drivers/gpu/drm/vc4/vc4_debugfs.c
+++ b/drivers/gpu/drm/vc4/vc4_debugfs.c
@@ -19,6 +19,7 @@ static const struct drm_info_list vc4_debugfs_list[] = {
 	{"bo_stats", vc4_bo_stats_debugfs, 0},
 	{"dpi_regs", vc4_dpi_debugfs_regs, 0},
 	{"hdmi_regs", vc4_hdmi_debugfs_regs, 0},
+	{"vec_regs", vc4_vec_debugfs_regs, 0},
 	{"hvs_regs", vc4_hvs_debugfs_regs, 0},
 	{"crtc0_regs", vc4_crtc_debugfs_regs, 0, (void *)(uintptr_t)0},
 	{"crtc1_regs", vc4_crtc_debugfs_regs, 0, (void *)(uintptr_t)1},
diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
index 8703f56b7947..ac09ca7ff430 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.c
+++ b/drivers/gpu/drm/vc4/vc4_drv.c
@@ -61,23 +61,28 @@ static int vc4_get_param_ioctl(struct drm_device *dev, void *data,
 		if (ret < 0)
 			return ret;
 		args->value = V3D_READ(V3D_IDENT0);
-		pm_runtime_put(&vc4->v3d->pdev->dev);
+		pm_runtime_mark_last_busy(&vc4->v3d->pdev->dev);
+		pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev);
 		break;
 	case DRM_VC4_PARAM_V3D_IDENT1:
 		ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
 		if (ret < 0)
 			return ret;
 		args->value = V3D_READ(V3D_IDENT1);
-		pm_runtime_put(&vc4->v3d->pdev->dev);
+		pm_runtime_mark_last_busy(&vc4->v3d->pdev->dev);
+		pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev);
 		break;
 	case DRM_VC4_PARAM_V3D_IDENT2:
 		ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
 		if (ret < 0)
 			return ret;
 		args->value = V3D_READ(V3D_IDENT2);
-		pm_runtime_put(&vc4->v3d->pdev->dev);
+		pm_runtime_mark_last_busy(&vc4->v3d->pdev->dev);
+		pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev);
 		break;
 	case DRM_VC4_PARAM_SUPPORTS_BRANCHES:
+	case DRM_VC4_PARAM_SUPPORTS_ETC1:
+	case DRM_VC4_PARAM_SUPPORTS_THREADED_FS:
 		args->value = true;
 		break;
 	default:
@@ -103,9 +108,7 @@ static const struct file_operations vc4_drm_fops = {
 	.mmap = vc4_mmap,
 	.poll = drm_poll,
 	.read = drm_read,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.llseek = noop_llseek,
 };
 
@@ -274,12 +277,14 @@ static void vc4_drm_unbind(struct device *dev)
 	struct drm_device *drm = platform_get_drvdata(pdev);
 	struct vc4_dev *vc4 = to_vc4_dev(drm);
 
+	drm_dev_unregister(drm);
+
 	if (vc4->fbdev)
 		drm_fbdev_cma_fini(vc4->fbdev);
 
 	drm_mode_config_cleanup(drm);
 
-	drm_put_dev(drm);
+	drm_dev_unref(drm);
 }
 
 static const struct component_master_ops vc4_drm_ops = {
@@ -289,6 +294,7 @@ static const struct component_master_ops vc4_drm_ops = {
 
 static struct platform_driver *const component_drivers[] = {
 	&vc4_hdmi_driver,
+	&vc4_vec_driver,
 	&vc4_dpi_driver,
 	&vc4_hvs_driver,
 	&vc4_crtc_driver,
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index 7c1e4d97486f..b5c4bb14d0d1 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -17,6 +17,7 @@ struct vc4_dev {
 	struct vc4_crtc *crtc[3];
 	struct vc4_v3d *v3d;
 	struct vc4_dpi *dpi;
+	struct vc4_vec *vec;
 
 	struct drm_fbdev_cma *fbdev;
 
@@ -194,6 +195,7 @@ to_vc4_plane(struct drm_plane *plane)
 }
 
 enum vc4_encoder_type {
+	VC4_ENCODER_TYPE_NONE,
 	VC4_ENCODER_TYPE_HDMI,
 	VC4_ENCODER_TYPE_VEC,
 	VC4_ENCODER_TYPE_DSI0,
@@ -381,6 +383,8 @@ struct vc4_validated_shader_info {
 
 	uint32_t num_uniform_addr_offsets;
 	uint32_t *uniform_addr_offsets;
+
+	bool is_threaded;
 };
 
 /**
@@ -440,6 +444,7 @@ int vc4_bo_stats_debugfs(struct seq_file *m, void *arg);
 extern struct platform_driver vc4_crtc_driver;
 int vc4_enable_vblank(struct drm_device *dev, unsigned int crtc_id);
 void vc4_disable_vblank(struct drm_device *dev, unsigned int crtc_id);
+bool vc4_event_pending(struct drm_crtc *crtc);
 int vc4_crtc_debugfs_regs(struct seq_file *m, void *arg);
 int vc4_crtc_get_scanoutpos(struct drm_device *dev, unsigned int crtc_id,
 			    unsigned int flags, int *vpos, int *hpos,
@@ -483,6 +488,10 @@ int vc4_queue_seqno_cb(struct drm_device *dev,
 extern struct platform_driver vc4_hdmi_driver;
 int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused);
 
+/* vc4_hdmi.c */
+extern struct platform_driver vc4_vec_driver;
+int vc4_vec_debugfs_regs(struct seq_file *m, void *unused);
+
 /* vc4_irq.c */
 irqreturn_t vc4_irq(int irq, void *arg);
 void vc4_irq_preinstall(struct drm_device *dev);
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index 47a095f392f8..db920771bfb5 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -544,14 +544,15 @@ vc4_cl_lookup_bos(struct drm_device *dev,
 
 	handles = drm_malloc_ab(exec->bo_count, sizeof(uint32_t));
 	if (!handles) {
+		ret = -ENOMEM;
 		DRM_ERROR("Failed to allocate incoming GEM handles\n");
 		goto fail;
 	}
 
-	ret = copy_from_user(handles,
-			     (void __user *)(uintptr_t)args->bo_handles,
-			     exec->bo_count * sizeof(uint32_t));
-	if (ret) {
+	if (copy_from_user(handles,
+			   (void __user *)(uintptr_t)args->bo_handles,
+			   exec->bo_count * sizeof(uint32_t))) {
+		ret = -EFAULT;
 		DRM_ERROR("Failed to copy in GEM handles\n");
 		goto fail;
 	}
@@ -708,8 +709,10 @@ vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec)
 	}
 
 	mutex_lock(&vc4->power_lock);
-	if (--vc4->power_refcount == 0)
-		pm_runtime_put(&vc4->v3d->pdev->dev);
+	if (--vc4->power_refcount == 0) {
+		pm_runtime_mark_last_busy(&vc4->v3d->pdev->dev);
+		pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev);
+	}
 	mutex_unlock(&vc4->power_lock);
 
 	kfree(exec);
diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
index c1f65c6c8e60..be8dd8262f27 100644
--- a/drivers/gpu/drm/vc4/vc4_kms.c
+++ b/drivers/gpu/drm/vc4/vc4_kms.c
@@ -61,7 +61,7 @@ vc4_atomic_complete_commit(struct vc4_commit *c)
 
 	drm_atomic_helper_cleanup_planes(dev, state);
 
-	drm_atomic_state_free(state);
+	drm_atomic_state_put(state);
 
 	up(&vc4->async_modeset);
 
@@ -119,17 +119,34 @@ static int vc4_atomic_commit(struct drm_device *dev,
 
 	/* Make sure that any outstanding modesets have finished. */
 	if (nonblock) {
-		ret = down_trylock(&vc4->async_modeset);
-		if (ret) {
+		struct drm_crtc *crtc;
+		struct drm_crtc_state *crtc_state;
+		unsigned long flags;
+		bool busy = false;
+
+		/*
+		 * If there's an undispatched event to send then we're
+		 * obviously still busy.  If there isn't, then we can
+		 * unconditionally wait for the semaphore because it
+		 * shouldn't be contended (for long).
+		 *
+		 * This is to prevent a race where queuing a new flip
+		 * from userspace immediately on receipt of an event
+		 * beats our clean-up and returns EBUSY.
+		 */
+		spin_lock_irqsave(&dev->event_lock, flags);
+		for_each_crtc_in_state(state, crtc, crtc_state, i)
+			busy |= vc4_event_pending(crtc);
+		spin_unlock_irqrestore(&dev->event_lock, flags);
+		if (busy) {
 			kfree(c);
 			return -EBUSY;
 		}
-	} else {
-		ret = down_interruptible(&vc4->async_modeset);
-		if (ret) {
-			kfree(c);
-			return ret;
-		}
+	}
+	ret = down_interruptible(&vc4->async_modeset);
+	if (ret) {
+		kfree(c);
+		return ret;
 	}
 
 	ret = drm_atomic_helper_prepare_planes(dev, state);
@@ -173,6 +190,7 @@ static int vc4_atomic_commit(struct drm_device *dev,
 	 * current layout.
 	 */
 
+	drm_atomic_state_get(state);
 	if (nonblock) {
 		vc4_queue_seqno_cb(dev, &c->cb, wait_seqno,
 				   vc4_atomic_complete_commit_seqno_cb);
diff --git a/drivers/gpu/drm/vc4/vc4_regs.h b/drivers/gpu/drm/vc4/vc4_regs.h
index 1aa44c2db556..39f6886b2410 100644
--- a/drivers/gpu/drm/vc4/vc4_regs.h
+++ b/drivers/gpu/drm/vc4/vc4_regs.h
@@ -177,8 +177,9 @@
 # define PV_CONTROL_WAIT_HSTART			BIT(12)
 # define PV_CONTROL_PIXEL_REP_MASK		VC4_MASK(5, 4)
 # define PV_CONTROL_PIXEL_REP_SHIFT		4
-# define PV_CONTROL_CLK_SELECT_DSI_VEC		0
+# define PV_CONTROL_CLK_SELECT_DSI		0
 # define PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI	1
+# define PV_CONTROL_CLK_SELECT_VEC		2
 # define PV_CONTROL_CLK_SELECT_MASK		VC4_MASK(3, 2)
 # define PV_CONTROL_CLK_SELECT_SHIFT		2
 # define PV_CONTROL_FIFO_CLR			BIT(1)
diff --git a/drivers/gpu/drm/vc4/vc4_v3d.c b/drivers/gpu/drm/vc4/vc4_v3d.c
index e6d3c6028341..7cc346ad9b0b 100644
--- a/drivers/gpu/drm/vc4/vc4_v3d.c
+++ b/drivers/gpu/drm/vc4/vc4_v3d.c
@@ -222,6 +222,8 @@ static int vc4_v3d_bind(struct device *dev, struct device *master, void *data)
 		return ret;
 	}
 
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_autosuspend_delay(dev, 40); /* a little over 2 frames. */
 	pm_runtime_enable(dev);
 
 	return 0;
diff --git a/drivers/gpu/drm/vc4/vc4_validate.c b/drivers/gpu/drm/vc4/vc4_validate.c
index 26503e307438..9fd171c361c2 100644
--- a/drivers/gpu/drm/vc4/vc4_validate.c
+++ b/drivers/gpu/drm/vc4/vc4_validate.c
@@ -644,6 +644,13 @@ reloc_tex(struct vc4_exec_info *exec,
 		cpp = 1;
 		break;
 	case VC4_TEXTURE_TYPE_ETC1:
+		/* ETC1 is arranged as 64-bit blocks, where each block is 4x4
+		 * pixels.
+		 */
+		cpp = 8;
+		width = (width + 3) >> 2;
+		height = (height + 3) >> 2;
+		break;
 	case VC4_TEXTURE_TYPE_BW1:
 	case VC4_TEXTURE_TYPE_A4:
 	case VC4_TEXTURE_TYPE_A1:
@@ -782,11 +789,6 @@ validate_gl_shader_rec(struct drm_device *dev,
 	exec->shader_rec_v += roundup(packet_size, 16);
 	exec->shader_rec_size -= packet_size;
 
-	if (!(*(uint16_t *)pkt_u & VC4_SHADER_FLAG_FS_SINGLE_THREAD)) {
-		DRM_ERROR("Multi-threaded fragment shaders not supported.\n");
-		return -EINVAL;
-	}
-
 	for (i = 0; i < shader_reloc_count; i++) {
 		if (src_handles[i] > exec->bo_count) {
 			DRM_ERROR("Shader handle %d too big\n", src_handles[i]);
@@ -803,6 +805,18 @@ validate_gl_shader_rec(struct drm_device *dev,
 			return -EINVAL;
 	}
 
+	if (((*(uint16_t *)pkt_u & VC4_SHADER_FLAG_FS_SINGLE_THREAD) == 0) !=
+	    to_vc4_bo(&bo[0]->base)->validated_shader->is_threaded) {
+		DRM_ERROR("Thread mode of CL and FS do not match\n");
+		return -EINVAL;
+	}
+
+	if (to_vc4_bo(&bo[1]->base)->validated_shader->is_threaded ||
+	    to_vc4_bo(&bo[2]->base)->validated_shader->is_threaded) {
+		DRM_ERROR("cs and vs cannot be threaded\n");
+		return -EINVAL;
+	}
+
 	for (i = 0; i < shader_reloc_count; i++) {
 		struct vc4_validated_shader_info *validated_shader;
 		uint32_t o = shader_reloc_offsets[i];
diff --git a/drivers/gpu/drm/vc4/vc4_validate_shaders.c b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
index 2543cf5b8b51..5dba13dd1e9b 100644
--- a/drivers/gpu/drm/vc4/vc4_validate_shaders.c
+++ b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
@@ -83,6 +83,13 @@ struct vc4_shader_validation_state {
 	 * basic blocks.
 	 */
 	bool needs_uniform_address_for_loop;
+
+	/* Set when we find an instruction writing the top half of the
+	 * register files.  If we allowed writing the unusable regs in
+	 * a threaded shader, then the other shader running on our
+	 * QPU's clamp validation would be invalid.
+	 */
+	bool all_registers_used;
 };
 
 static uint32_t
@@ -119,6 +126,13 @@ raddr_add_a_to_live_reg_index(uint64_t inst)
 }
 
 static bool
+live_reg_is_upper_half(uint32_t lri)
+{
+	return	(lri >= 16 && lri < 32) ||
+		(lri >= 32 + 16 && lri < 32 + 32);
+}
+
+static bool
 is_tmu_submit(uint32_t waddr)
 {
 	return (waddr == QPU_W_TMU0_S ||
@@ -390,6 +404,9 @@ check_reg_write(struct vc4_validated_shader_info *validated_shader,
 		} else {
 			validation_state->live_immediates[lri] = ~0;
 		}
+
+		if (live_reg_is_upper_half(lri))
+			validation_state->all_registers_used = true;
 	}
 
 	switch (waddr) {
@@ -598,6 +615,11 @@ check_instruction_reads(struct vc4_validated_shader_info *validated_shader,
 		}
 	}
 
+	if ((raddr_a >= 16 && raddr_a < 32) ||
+	    (raddr_b >= 16 && raddr_b < 32 && sig != QPU_SIG_SMALL_IMM)) {
+		validation_state->all_registers_used = true;
+	}
+
 	return true;
 }
 
@@ -608,9 +630,7 @@ static bool
 vc4_validate_branches(struct vc4_shader_validation_state *validation_state)
 {
 	uint32_t max_branch_target = 0;
-	bool found_shader_end = false;
 	int ip;
-	int shader_end_ip = 0;
 	int last_branch = -2;
 
 	for (ip = 0; ip < validation_state->max_ip; ip++) {
@@ -621,8 +641,13 @@ vc4_validate_branches(struct vc4_shader_validation_state *validation_state)
 		uint32_t branch_target_ip;
 
 		if (sig == QPU_SIG_PROG_END) {
-			shader_end_ip = ip;
-			found_shader_end = true;
+			/* There are two delay slots after program end is
+			 * signaled that are still executed, then we're
+			 * finished.  validation_state->max_ip is the
+			 * instruction after the last valid instruction in the
+			 * program.
+			 */
+			validation_state->max_ip = ip + 3;
 			continue;
 		}
 
@@ -676,15 +701,9 @@ vc4_validate_branches(struct vc4_shader_validation_state *validation_state)
 		}
 		set_bit(after_delay_ip, validation_state->branch_targets);
 		max_branch_target = max(max_branch_target, after_delay_ip);
-
-		/* There are two delay slots after program end is signaled
-		 * that are still executed, then we're finished.
-		 */
-		if (found_shader_end && ip == shader_end_ip + 2)
-			break;
 	}
 
-	if (max_branch_target > shader_end_ip) {
+	if (max_branch_target > validation_state->max_ip - 3) {
 		DRM_ERROR("Branch landed after QPU_SIG_PROG_END");
 		return false;
 	}
@@ -756,6 +775,7 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 {
 	bool found_shader_end = false;
 	int shader_end_ip = 0;
+	uint32_t last_thread_switch_ip = -3;
 	uint32_t ip;
 	struct vc4_validated_shader_info *validated_shader = NULL;
 	struct vc4_shader_validation_state validation_state;
@@ -788,6 +808,17 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 		if (!vc4_handle_branch_target(&validation_state))
 			goto fail;
 
+		if (ip == last_thread_switch_ip + 3) {
+			/* Reset r0-r3 live clamp data */
+			int i;
+
+			for (i = 64; i < LIVE_REG_COUNT; i++) {
+				validation_state.live_min_clamp_offsets[i] = ~0;
+				validation_state.live_max_clamp_regs[i] = false;
+				validation_state.live_immediates[i] = ~0;
+			}
+		}
+
 		switch (sig) {
 		case QPU_SIG_NONE:
 		case QPU_SIG_WAIT_FOR_SCOREBOARD:
@@ -797,6 +828,8 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 		case QPU_SIG_LOAD_TMU1:
 		case QPU_SIG_PROG_END:
 		case QPU_SIG_SMALL_IMM:
+		case QPU_SIG_THREAD_SWITCH:
+		case QPU_SIG_LAST_THREAD_SWITCH:
 			if (!check_instruction_writes(validated_shader,
 						      &validation_state)) {
 				DRM_ERROR("Bad write at ip %d\n", ip);
@@ -812,6 +845,18 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 				shader_end_ip = ip;
 			}
 
+			if (sig == QPU_SIG_THREAD_SWITCH ||
+			    sig == QPU_SIG_LAST_THREAD_SWITCH) {
+				validated_shader->is_threaded = true;
+
+				if (ip < last_thread_switch_ip + 3) {
+					DRM_ERROR("Thread switch too soon after "
+						  "last switch at ip %d\n", ip);
+					goto fail;
+				}
+				last_thread_switch_ip = ip;
+			}
+
 			break;
 
 		case QPU_SIG_LOAD_IMM:
@@ -826,6 +871,13 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 			if (!check_branch(inst, validated_shader,
 					  &validation_state, ip))
 				goto fail;
+
+			if (ip < last_thread_switch_ip + 3) {
+				DRM_ERROR("Branch in thread switch at ip %d",
+					  ip);
+				goto fail;
+			}
+
 			break;
 		default:
 			DRM_ERROR("Unsupported QPU signal %d at "
@@ -847,6 +899,14 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 		goto fail;
 	}
 
+	/* Might corrupt other thread */
+	if (validated_shader->is_threaded &&
+	    validation_state.all_registers_used) {
+		DRM_ERROR("Shader uses threading, but uses the upper "
+			  "half of the registers, too\n");
+		goto fail;
+	}
+
 	/* If we did a backwards branch and we haven't emitted a uniforms
 	 * reset since then, we still need the uniforms stream to have the
 	 * uniforms address available so that the backwards branch can do its
diff --git a/drivers/gpu/drm/vc4/vc4_vec.c b/drivers/gpu/drm/vc4/vc4_vec.c
new file mode 100644
index 000000000000..32bb8ef985fb
--- /dev/null
+++ b/drivers/gpu/drm/vc4/vc4_vec.c
@@ -0,0 +1,657 @@
+/*
+ * Copyright (C) 2016 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * DOC: VC4 SDTV module
+ */
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_panel.h>
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/of_graph.h>
+#include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
+
+#include "vc4_drv.h"
+#include "vc4_regs.h"
+
+/* WSE Registers */
+#define VEC_WSE_RESET			0xc0
+
+#define VEC_WSE_CONTROL			0xc4
+#define VEC_WSE_WSS_ENABLE		BIT(7)
+
+#define VEC_WSE_WSS_DATA		0xc8
+#define VEC_WSE_VPS_DATA1		0xcc
+#define VEC_WSE_VPS_CONTROL		0xd0
+
+/* VEC Registers */
+#define VEC_REVID			0x100
+
+#define VEC_CONFIG0			0x104
+#define VEC_CONFIG0_YDEL_MASK		GENMASK(28, 26)
+#define VEC_CONFIG0_YDEL(x)		((x) << 26)
+#define VEC_CONFIG0_CDEL_MASK		GENMASK(25, 24)
+#define VEC_CONFIG0_CDEL(x)		((x) << 24)
+#define VEC_CONFIG0_PBPR_FIL		BIT(18)
+#define VEC_CONFIG0_CHROMA_GAIN_MASK	GENMASK(17, 16)
+#define VEC_CONFIG0_CHROMA_GAIN_UNITY	(0 << 16)
+#define VEC_CONFIG0_CHROMA_GAIN_1_32	(1 << 16)
+#define VEC_CONFIG0_CHROMA_GAIN_1_16	(2 << 16)
+#define VEC_CONFIG0_CHROMA_GAIN_1_8	(3 << 16)
+#define VEC_CONFIG0_CBURST_GAIN_MASK	GENMASK(14, 13)
+#define VEC_CONFIG0_CBURST_GAIN_UNITY	(0 << 13)
+#define VEC_CONFIG0_CBURST_GAIN_1_128	(1 << 13)
+#define VEC_CONFIG0_CBURST_GAIN_1_64	(2 << 13)
+#define VEC_CONFIG0_CBURST_GAIN_1_32	(3 << 13)
+#define VEC_CONFIG0_CHRBW1		BIT(11)
+#define VEC_CONFIG0_CHRBW0		BIT(10)
+#define VEC_CONFIG0_SYNCDIS		BIT(9)
+#define VEC_CONFIG0_BURDIS		BIT(8)
+#define VEC_CONFIG0_CHRDIS		BIT(7)
+#define VEC_CONFIG0_PDEN		BIT(6)
+#define VEC_CONFIG0_YCDELAY		BIT(4)
+#define VEC_CONFIG0_RAMPEN		BIT(2)
+#define VEC_CONFIG0_YCDIS		BIT(2)
+#define VEC_CONFIG0_STD_MASK		GENMASK(1, 0)
+#define VEC_CONFIG0_NTSC_STD		0
+#define VEC_CONFIG0_PAL_BDGHI_STD	1
+#define VEC_CONFIG0_PAL_N_STD		3
+
+#define VEC_SCHPH			0x108
+#define VEC_SOFT_RESET			0x10c
+#define VEC_CLMP0_START			0x144
+#define VEC_CLMP0_END			0x148
+#define VEC_FREQ3_2			0x180
+#define VEC_FREQ1_0			0x184
+
+#define VEC_CONFIG1			0x188
+#define VEC_CONFIG_VEC_RESYNC_OFF	BIT(18)
+#define VEC_CONFIG_RGB219		BIT(17)
+#define VEC_CONFIG_CBAR_EN		BIT(16)
+#define VEC_CONFIG_TC_OBB		BIT(15)
+#define VEC_CONFIG1_OUTPUT_MODE_MASK	GENMASK(12, 10)
+#define VEC_CONFIG1_C_Y_CVBS		(0 << 10)
+#define VEC_CONFIG1_CVBS_Y_C		(1 << 10)
+#define VEC_CONFIG1_PR_Y_PB		(2 << 10)
+#define VEC_CONFIG1_RGB			(4 << 10)
+#define VEC_CONFIG1_Y_C_CVBS		(5 << 10)
+#define VEC_CONFIG1_C_CVBS_Y		(6 << 10)
+#define VEC_CONFIG1_C_CVBS_CVBS		(7 << 10)
+#define VEC_CONFIG1_DIS_CHR		BIT(9)
+#define VEC_CONFIG1_DIS_LUMA		BIT(8)
+#define VEC_CONFIG1_YCBCR_IN		BIT(6)
+#define VEC_CONFIG1_DITHER_TYPE_LFSR	0
+#define VEC_CONFIG1_DITHER_TYPE_COUNTER	BIT(5)
+#define VEC_CONFIG1_DITHER_EN		BIT(4)
+#define VEC_CONFIG1_CYDELAY		BIT(3)
+#define VEC_CONFIG1_LUMADIS		BIT(2)
+#define VEC_CONFIG1_COMPDIS		BIT(1)
+#define VEC_CONFIG1_CUSTOM_FREQ		BIT(0)
+
+#define VEC_CONFIG2			0x18c
+#define VEC_CONFIG2_PROG_SCAN		BIT(15)
+#define VEC_CONFIG2_SYNC_ADJ_MASK	GENMASK(14, 12)
+#define VEC_CONFIG2_SYNC_ADJ(x)		(((x) / 2) << 12)
+#define VEC_CONFIG2_PBPR_EN		BIT(10)
+#define VEC_CONFIG2_UV_DIG_DIS		BIT(6)
+#define VEC_CONFIG2_RGB_DIG_DIS		BIT(5)
+#define VEC_CONFIG2_TMUX_MASK		GENMASK(3, 2)
+#define VEC_CONFIG2_TMUX_DRIVE0		(0 << 2)
+#define VEC_CONFIG2_TMUX_RG_COMP	(1 << 2)
+#define VEC_CONFIG2_TMUX_UV_YC		(2 << 2)
+#define VEC_CONFIG2_TMUX_SYNC_YC	(3 << 2)
+
+#define VEC_INTERRUPT_CONTROL		0x190
+#define VEC_INTERRUPT_STATUS		0x194
+#define VEC_FCW_SECAM_B			0x198
+#define VEC_SECAM_GAIN_VAL		0x19c
+
+#define VEC_CONFIG3			0x1a0
+#define VEC_CONFIG3_HORIZ_LEN_STD	(0 << 0)
+#define VEC_CONFIG3_HORIZ_LEN_MPEG1_SIF	(1 << 0)
+#define VEC_CONFIG3_SHAPE_NON_LINEAR	BIT(1)
+
+#define VEC_STATUS0			0x200
+#define VEC_MASK0			0x204
+
+#define VEC_CFG				0x208
+#define VEC_CFG_SG_MODE_MASK		GENMASK(6, 5)
+#define VEC_CFG_SG_MODE(x)		((x) << 5)
+#define VEC_CFG_SG_EN			BIT(4)
+#define VEC_CFG_VEC_EN			BIT(3)
+#define VEC_CFG_MB_EN			BIT(2)
+#define VEC_CFG_ENABLE			BIT(1)
+#define VEC_CFG_TB_EN			BIT(0)
+
+#define VEC_DAC_TEST			0x20c
+
+#define VEC_DAC_CONFIG			0x210
+#define VEC_DAC_CONFIG_LDO_BIAS_CTRL(x)	((x) << 24)
+#define VEC_DAC_CONFIG_DRIVER_CTRL(x)	((x) << 16)
+#define VEC_DAC_CONFIG_DAC_CTRL(x)	(x)
+
+#define VEC_DAC_MISC			0x214
+#define VEC_DAC_MISC_VCD_CTRL_MASK	GENMASK(31, 16)
+#define VEC_DAC_MISC_VCD_CTRL(x)	((x) << 16)
+#define VEC_DAC_MISC_VID_ACT		BIT(8)
+#define VEC_DAC_MISC_VCD_PWRDN		BIT(6)
+#define VEC_DAC_MISC_BIAS_PWRDN		BIT(5)
+#define VEC_DAC_MISC_DAC_PWRDN		BIT(2)
+#define VEC_DAC_MISC_LDO_PWRDN		BIT(1)
+#define VEC_DAC_MISC_DAC_RST_N		BIT(0)
+
+
+/* General VEC hardware state. */
+struct vc4_vec {
+	struct platform_device *pdev;
+
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
+
+	void __iomem *regs;
+
+	struct clk *clock;
+
+	const struct vc4_vec_tv_mode *tv_mode;
+};
+
+#define VEC_READ(offset) readl(vec->regs + (offset))
+#define VEC_WRITE(offset, val) writel(val, vec->regs + (offset))
+
+/* VC4 VEC encoder KMS struct */
+struct vc4_vec_encoder {
+	struct vc4_encoder base;
+	struct vc4_vec *vec;
+};
+
+static inline struct vc4_vec_encoder *
+to_vc4_vec_encoder(struct drm_encoder *encoder)
+{
+	return container_of(encoder, struct vc4_vec_encoder, base.base);
+}
+
+/* VC4 VEC connector KMS struct */
+struct vc4_vec_connector {
+	struct drm_connector base;
+	struct vc4_vec *vec;
+
+	/* Since the connector is attached to just the one encoder,
+	 * this is the reference to it so we can do the best_encoder()
+	 * hook.
+	 */
+	struct drm_encoder *encoder;
+};
+
+static inline struct vc4_vec_connector *
+to_vc4_vec_connector(struct drm_connector *connector)
+{
+	return container_of(connector, struct vc4_vec_connector, base);
+}
+
+enum vc4_vec_tv_mode_id {
+	VC4_VEC_TV_MODE_NTSC,
+	VC4_VEC_TV_MODE_NTSC_J,
+	VC4_VEC_TV_MODE_PAL,
+	VC4_VEC_TV_MODE_PAL_M,
+};
+
+struct vc4_vec_tv_mode {
+	const struct drm_display_mode *mode;
+	void (*mode_set)(struct vc4_vec *vec);
+};
+
+#define VEC_REG(reg) { reg, #reg }
+static const struct {
+	u32 reg;
+	const char *name;
+} vec_regs[] = {
+	VEC_REG(VEC_WSE_CONTROL),
+	VEC_REG(VEC_WSE_WSS_DATA),
+	VEC_REG(VEC_WSE_VPS_DATA1),
+	VEC_REG(VEC_WSE_VPS_CONTROL),
+	VEC_REG(VEC_REVID),
+	VEC_REG(VEC_CONFIG0),
+	VEC_REG(VEC_SCHPH),
+	VEC_REG(VEC_CLMP0_START),
+	VEC_REG(VEC_CLMP0_END),
+	VEC_REG(VEC_FREQ3_2),
+	VEC_REG(VEC_FREQ1_0),
+	VEC_REG(VEC_CONFIG1),
+	VEC_REG(VEC_CONFIG2),
+	VEC_REG(VEC_INTERRUPT_CONTROL),
+	VEC_REG(VEC_INTERRUPT_STATUS),
+	VEC_REG(VEC_FCW_SECAM_B),
+	VEC_REG(VEC_SECAM_GAIN_VAL),
+	VEC_REG(VEC_CONFIG3),
+	VEC_REG(VEC_STATUS0),
+	VEC_REG(VEC_MASK0),
+	VEC_REG(VEC_CFG),
+	VEC_REG(VEC_DAC_TEST),
+	VEC_REG(VEC_DAC_CONFIG),
+	VEC_REG(VEC_DAC_MISC),
+};
+
+#ifdef CONFIG_DEBUG_FS
+int vc4_vec_debugfs_regs(struct seq_file *m, void *unused)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct vc4_vec *vec = vc4->vec;
+	int i;
+
+	if (!vec)
+		return 0;
+
+	for (i = 0; i < ARRAY_SIZE(vec_regs); i++) {
+		seq_printf(m, "%s (0x%04x): 0x%08x\n",
+			   vec_regs[i].name, vec_regs[i].reg,
+			   VEC_READ(vec_regs[i].reg));
+	}
+
+	return 0;
+}
+#endif
+
+static void vc4_vec_ntsc_mode_set(struct vc4_vec *vec)
+{
+	VEC_WRITE(VEC_CONFIG0, VEC_CONFIG0_NTSC_STD | VEC_CONFIG0_PDEN);
+	VEC_WRITE(VEC_CONFIG1, VEC_CONFIG1_C_CVBS_CVBS);
+}
+
+static void vc4_vec_ntsc_j_mode_set(struct vc4_vec *vec)
+{
+	VEC_WRITE(VEC_CONFIG0, VEC_CONFIG0_NTSC_STD);
+	VEC_WRITE(VEC_CONFIG1, VEC_CONFIG1_C_CVBS_CVBS);
+}
+
+static const struct drm_display_mode ntsc_mode = {
+	DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 13500,
+		 720, 720 + 14, 720 + 14 + 64, 720 + 14 + 64 + 60, 0,
+		 480, 480 + 3, 480 + 3 + 3, 480 + 3 + 3 + 16, 0,
+		 DRM_MODE_FLAG_INTERLACE)
+};
+
+static void vc4_vec_pal_mode_set(struct vc4_vec *vec)
+{
+	VEC_WRITE(VEC_CONFIG0, VEC_CONFIG0_PAL_BDGHI_STD);
+	VEC_WRITE(VEC_CONFIG1, VEC_CONFIG1_C_CVBS_CVBS);
+}
+
+static void vc4_vec_pal_m_mode_set(struct vc4_vec *vec)
+{
+	VEC_WRITE(VEC_CONFIG0, VEC_CONFIG0_PAL_BDGHI_STD);
+	VEC_WRITE(VEC_CONFIG1,
+		  VEC_CONFIG1_C_CVBS_CVBS | VEC_CONFIG1_CUSTOM_FREQ);
+	VEC_WRITE(VEC_FREQ3_2, 0x223b);
+	VEC_WRITE(VEC_FREQ1_0, 0x61d1);
+}
+
+static const struct drm_display_mode pal_mode = {
+	DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 13500,
+		 720, 720 + 20, 720 + 20 + 64, 720 + 20 + 64 + 60, 0,
+		 576, 576 + 2, 576 + 2 + 3, 576 + 2 + 3 + 20, 0,
+		 DRM_MODE_FLAG_INTERLACE)
+};
+
+static const struct vc4_vec_tv_mode vc4_vec_tv_modes[] = {
+	[VC4_VEC_TV_MODE_NTSC] = {
+		.mode = &ntsc_mode,
+		.mode_set = vc4_vec_ntsc_mode_set,
+	},
+	[VC4_VEC_TV_MODE_NTSC_J] = {
+		.mode = &ntsc_mode,
+		.mode_set = vc4_vec_ntsc_j_mode_set,
+	},
+	[VC4_VEC_TV_MODE_PAL] = {
+		.mode = &pal_mode,
+		.mode_set = vc4_vec_pal_mode_set,
+	},
+	[VC4_VEC_TV_MODE_PAL_M] = {
+		.mode = &pal_mode,
+		.mode_set = vc4_vec_pal_m_mode_set,
+	},
+};
+
+static enum drm_connector_status
+vc4_vec_connector_detect(struct drm_connector *connector, bool force)
+{
+	return connector_status_unknown;
+}
+
+static void vc4_vec_connector_destroy(struct drm_connector *connector)
+{
+	drm_connector_unregister(connector);
+	drm_connector_cleanup(connector);
+}
+
+static int vc4_vec_connector_get_modes(struct drm_connector *connector)
+{
+	struct drm_connector_state *state = connector->state;
+	struct drm_display_mode *mode;
+
+	mode = drm_mode_duplicate(connector->dev,
+				  vc4_vec_tv_modes[state->tv.mode].mode);
+	if (!mode) {
+		DRM_ERROR("Failed to create a new display mode\n");
+		return -ENOMEM;
+	}
+
+	drm_mode_probed_add(connector, mode);
+
+	return 1;
+}
+
+static const struct drm_connector_funcs vc4_vec_connector_funcs = {
+	.dpms = drm_atomic_helper_connector_dpms,
+	.detect = vc4_vec_connector_detect,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.set_property = drm_atomic_helper_connector_set_property,
+	.destroy = vc4_vec_connector_destroy,
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static const struct drm_connector_helper_funcs vc4_vec_connector_helper_funcs = {
+	.get_modes = vc4_vec_connector_get_modes,
+};
+
+static struct drm_connector *vc4_vec_connector_init(struct drm_device *dev,
+						    struct vc4_vec *vec)
+{
+	struct drm_connector *connector = NULL;
+	struct vc4_vec_connector *vec_connector;
+
+	vec_connector = devm_kzalloc(dev->dev, sizeof(*vec_connector),
+				     GFP_KERNEL);
+	if (!vec_connector)
+		return ERR_PTR(-ENOMEM);
+
+	connector = &vec_connector->base;
+	connector->interlace_allowed = true;
+
+	vec_connector->encoder = vec->encoder;
+	vec_connector->vec = vec;
+
+	drm_connector_init(dev, connector, &vc4_vec_connector_funcs,
+			   DRM_MODE_CONNECTOR_Composite);
+	drm_connector_helper_add(connector, &vc4_vec_connector_helper_funcs);
+
+	drm_object_attach_property(&connector->base,
+				   dev->mode_config.tv_mode_property,
+				   VC4_VEC_TV_MODE_NTSC);
+	vec->tv_mode = &vc4_vec_tv_modes[VC4_VEC_TV_MODE_NTSC];
+
+	drm_mode_connector_attach_encoder(connector, vec->encoder);
+
+	return connector;
+}
+
+static const struct drm_encoder_funcs vc4_vec_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
+static void vc4_vec_encoder_disable(struct drm_encoder *encoder)
+{
+	struct vc4_vec_encoder *vc4_vec_encoder = to_vc4_vec_encoder(encoder);
+	struct vc4_vec *vec = vc4_vec_encoder->vec;
+	int ret;
+
+	VEC_WRITE(VEC_CFG, 0);
+	VEC_WRITE(VEC_DAC_MISC,
+		  VEC_DAC_MISC_VCD_PWRDN |
+		  VEC_DAC_MISC_BIAS_PWRDN |
+		  VEC_DAC_MISC_DAC_PWRDN |
+		  VEC_DAC_MISC_LDO_PWRDN);
+
+	clk_disable_unprepare(vec->clock);
+
+	ret = pm_runtime_put(&vec->pdev->dev);
+	if (ret < 0) {
+		DRM_ERROR("Failed to release power domain: %d\n", ret);
+		return;
+	}
+}
+
+static void vc4_vec_encoder_enable(struct drm_encoder *encoder)
+{
+	struct vc4_vec_encoder *vc4_vec_encoder = to_vc4_vec_encoder(encoder);
+	struct vc4_vec *vec = vc4_vec_encoder->vec;
+	int ret;
+
+	ret = pm_runtime_get_sync(&vec->pdev->dev);
+	if (ret < 0) {
+		DRM_ERROR("Failed to retain power domain: %d\n", ret);
+		return;
+	}
+
+	/*
+	 * We need to set the clock rate each time we enable the encoder
+	 * because there's a chance we share the same parent with the HDMI
+	 * clock, and both drivers are requesting different rates.
+	 * The good news is, these 2 encoders cannot be enabled at the same
+	 * time, thus preventing incompatible rate requests.
+	 */
+	ret = clk_set_rate(vec->clock, 108000000);
+	if (ret) {
+		DRM_ERROR("Failed to set clock rate: %d\n", ret);
+		return;
+	}
+
+	ret = clk_prepare_enable(vec->clock);
+	if (ret) {
+		DRM_ERROR("Failed to turn on core clock: %d\n", ret);
+		return;
+	}
+
+	/* Reset the different blocks */
+	VEC_WRITE(VEC_WSE_RESET, 1);
+	VEC_WRITE(VEC_SOFT_RESET, 1);
+
+	/* Disable the CGSM-A and WSE blocks */
+	VEC_WRITE(VEC_WSE_CONTROL, 0);
+
+	/* Write config common to all modes. */
+
+	/*
+	 * Color subcarrier phase: phase = 360 * SCHPH / 256.
+	 * 0x28 <=> 39.375 deg.
+	 */
+	VEC_WRITE(VEC_SCHPH, 0x28);
+
+	/*
+	 * Reset to default values.
+	 */
+	VEC_WRITE(VEC_CLMP0_START, 0xac);
+	VEC_WRITE(VEC_CLMP0_END, 0xec);
+	VEC_WRITE(VEC_CONFIG2,
+		  VEC_CONFIG2_UV_DIG_DIS | VEC_CONFIG2_RGB_DIG_DIS);
+	VEC_WRITE(VEC_CONFIG3, VEC_CONFIG3_HORIZ_LEN_STD);
+	VEC_WRITE(VEC_DAC_CONFIG,
+		  VEC_DAC_CONFIG_DAC_CTRL(0xc) |
+		  VEC_DAC_CONFIG_DRIVER_CTRL(0xc) |
+		  VEC_DAC_CONFIG_LDO_BIAS_CTRL(0x46));
+
+	/* Mask all interrupts. */
+	VEC_WRITE(VEC_MASK0, 0);
+
+	vec->tv_mode->mode_set(vec);
+
+	VEC_WRITE(VEC_DAC_MISC,
+		  VEC_DAC_MISC_VID_ACT | VEC_DAC_MISC_DAC_RST_N);
+	VEC_WRITE(VEC_CFG, VEC_CFG_VEC_EN);
+}
+
+
+static bool vc4_vec_encoder_mode_fixup(struct drm_encoder *encoder,
+				       const struct drm_display_mode *mode,
+				       struct drm_display_mode *adjusted_mode)
+{
+	return true;
+}
+
+static void vc4_vec_encoder_atomic_mode_set(struct drm_encoder *encoder,
+					struct drm_crtc_state *crtc_state,
+					struct drm_connector_state *conn_state)
+{
+	struct vc4_vec_encoder *vc4_vec_encoder = to_vc4_vec_encoder(encoder);
+	struct vc4_vec *vec = vc4_vec_encoder->vec;
+
+	vec->tv_mode = &vc4_vec_tv_modes[conn_state->tv.mode];
+}
+
+static int vc4_vec_encoder_atomic_check(struct drm_encoder *encoder,
+					struct drm_crtc_state *crtc_state,
+					struct drm_connector_state *conn_state)
+{
+	const struct vc4_vec_tv_mode *vec_mode;
+
+	vec_mode = &vc4_vec_tv_modes[conn_state->tv.mode];
+
+	if (conn_state->crtc &&
+	    !drm_mode_equal(vec_mode->mode, &crtc_state->adjusted_mode))
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct drm_encoder_helper_funcs vc4_vec_encoder_helper_funcs = {
+	.disable = vc4_vec_encoder_disable,
+	.enable = vc4_vec_encoder_enable,
+	.mode_fixup = vc4_vec_encoder_mode_fixup,
+	.atomic_check = vc4_vec_encoder_atomic_check,
+	.atomic_mode_set = vc4_vec_encoder_atomic_mode_set,
+};
+
+static const struct of_device_id vc4_vec_dt_match[] = {
+	{ .compatible = "brcm,bcm2835-vec", .data = NULL },
+	{ /* sentinel */ },
+};
+
+static const char * const tv_mode_names[] = {
+	[VC4_VEC_TV_MODE_NTSC] = "NTSC",
+	[VC4_VEC_TV_MODE_NTSC_J] = "NTSC-J",
+	[VC4_VEC_TV_MODE_PAL] = "PAL",
+	[VC4_VEC_TV_MODE_PAL_M] = "PAL-M",
+};
+
+static int vc4_vec_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct drm_device *drm = dev_get_drvdata(master);
+	struct vc4_dev *vc4 = to_vc4_dev(drm);
+	struct vc4_vec *vec;
+	struct vc4_vec_encoder *vc4_vec_encoder;
+	int ret;
+
+	ret = drm_mode_create_tv_properties(drm, ARRAY_SIZE(tv_mode_names),
+					    tv_mode_names);
+	if (ret)
+		return ret;
+
+	vec = devm_kzalloc(dev, sizeof(*vec), GFP_KERNEL);
+	if (!vec)
+		return -ENOMEM;
+
+	vc4_vec_encoder = devm_kzalloc(dev, sizeof(*vc4_vec_encoder),
+				       GFP_KERNEL);
+	if (!vc4_vec_encoder)
+		return -ENOMEM;
+	vc4_vec_encoder->base.type = VC4_ENCODER_TYPE_VEC;
+	vc4_vec_encoder->vec = vec;
+	vec->encoder = &vc4_vec_encoder->base.base;
+
+	vec->pdev = pdev;
+	vec->regs = vc4_ioremap_regs(pdev, 0);
+	if (IS_ERR(vec->regs))
+		return PTR_ERR(vec->regs);
+
+	vec->clock = devm_clk_get(dev, NULL);
+	if (IS_ERR(vec->clock)) {
+		ret = PTR_ERR(vec->clock);
+		if (ret != -EPROBE_DEFER)
+			DRM_ERROR("Failed to get clock: %d\n", ret);
+		return ret;
+	}
+
+	pm_runtime_enable(dev);
+
+	drm_encoder_init(drm, vec->encoder, &vc4_vec_encoder_funcs,
+			 DRM_MODE_ENCODER_TVDAC, NULL);
+	drm_encoder_helper_add(vec->encoder, &vc4_vec_encoder_helper_funcs);
+
+	vec->connector = vc4_vec_connector_init(drm, vec);
+	if (IS_ERR(vec->connector)) {
+		ret = PTR_ERR(vec->connector);
+		goto err_destroy_encoder;
+	}
+
+	dev_set_drvdata(dev, vec);
+
+	vc4->vec = vec;
+
+	return 0;
+
+err_destroy_encoder:
+	drm_encoder_cleanup(vec->encoder);
+	pm_runtime_disable(dev);
+
+	return ret;
+}
+
+static void vc4_vec_unbind(struct device *dev, struct device *master,
+			   void *data)
+{
+	struct drm_device *drm = dev_get_drvdata(master);
+	struct vc4_dev *vc4 = to_vc4_dev(drm);
+	struct vc4_vec *vec = dev_get_drvdata(dev);
+
+	vc4_vec_connector_destroy(vec->connector);
+	drm_encoder_cleanup(vec->encoder);
+	pm_runtime_disable(dev);
+
+	vc4->vec = NULL;
+}
+
+static const struct component_ops vc4_vec_ops = {
+	.bind   = vc4_vec_bind,
+	.unbind = vc4_vec_unbind,
+};
+
+static int vc4_vec_dev_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &vc4_vec_ops);
+}
+
+static int vc4_vec_dev_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &vc4_vec_ops);
+	return 0;
+}
+
+struct platform_driver vc4_vec_driver = {
+	.probe = vc4_vec_dev_probe,
+	.remove = vc4_vec_dev_remove,
+	.driver = {
+		.name = "vc4_vec",
+		.of_match_table = vc4_vec_dt_match,
+	},
+};
diff --git a/drivers/gpu/drm/vgem/vgem_drv.c b/drivers/gpu/drm/vgem/vgem_drv.c
index f36c14729b55..477e07f0ecb6 100644
--- a/drivers/gpu/drm/vgem/vgem_drv.c
+++ b/drivers/gpu/drm/vgem/vgem_drv.c
@@ -54,7 +54,7 @@ static int vgem_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct drm_vgem_gem_object *obj = vma->vm_private_data;
 	/* We don't use vmf->pgoff since that has the fake offset */
-	unsigned long vaddr = (unsigned long)vmf->virtual_address;
+	unsigned long vaddr = vmf->address;
 	struct page *page;
 
 	page = shmem_read_mapping_page(file_inode(obj->base.filp)->i_mapping,
diff --git a/drivers/gpu/drm/vgem/vgem_fence.c b/drivers/gpu/drm/vgem/vgem_fence.c
index 5c57c1ffa1f9..da25dfe7b80e 100644
--- a/drivers/gpu/drm/vgem/vgem_fence.c
+++ b/drivers/gpu/drm/vgem/vgem_fence.c
@@ -28,56 +28,57 @@
 #define VGEM_FENCE_TIMEOUT (10*HZ)
 
 struct vgem_fence {
-	struct fence base;
+	struct dma_fence base;
 	struct spinlock lock;
 	struct timer_list timer;
 };
 
-static const char *vgem_fence_get_driver_name(struct fence *fence)
+static const char *vgem_fence_get_driver_name(struct dma_fence *fence)
 {
 	return "vgem";
 }
 
-static const char *vgem_fence_get_timeline_name(struct fence *fence)
+static const char *vgem_fence_get_timeline_name(struct dma_fence *fence)
 {
 	return "unbound";
 }
 
-static bool vgem_fence_signaled(struct fence *fence)
+static bool vgem_fence_signaled(struct dma_fence *fence)
 {
 	return false;
 }
 
-static bool vgem_fence_enable_signaling(struct fence *fence)
+static bool vgem_fence_enable_signaling(struct dma_fence *fence)
 {
 	return true;
 }
 
-static void vgem_fence_release(struct fence *base)
+static void vgem_fence_release(struct dma_fence *base)
 {
 	struct vgem_fence *fence = container_of(base, typeof(*fence), base);
 
 	del_timer_sync(&fence->timer);
-	fence_free(&fence->base);
+	dma_fence_free(&fence->base);
 }
 
-static void vgem_fence_value_str(struct fence *fence, char *str, int size)
+static void vgem_fence_value_str(struct dma_fence *fence, char *str, int size)
 {
 	snprintf(str, size, "%u", fence->seqno);
 }
 
-static void vgem_fence_timeline_value_str(struct fence *fence, char *str,
+static void vgem_fence_timeline_value_str(struct dma_fence *fence, char *str,
 					  int size)
 {
-	snprintf(str, size, "%u", fence_is_signaled(fence) ? fence->seqno : 0);
+	snprintf(str, size, "%u",
+		 dma_fence_is_signaled(fence) ? fence->seqno : 0);
 }
 
-static const struct fence_ops vgem_fence_ops = {
+static const struct dma_fence_ops vgem_fence_ops = {
 	.get_driver_name = vgem_fence_get_driver_name,
 	.get_timeline_name = vgem_fence_get_timeline_name,
 	.enable_signaling = vgem_fence_enable_signaling,
 	.signaled = vgem_fence_signaled,
-	.wait = fence_default_wait,
+	.wait = dma_fence_default_wait,
 	.release = vgem_fence_release,
 
 	.fence_value_str = vgem_fence_value_str,
@@ -88,11 +89,11 @@ static void vgem_fence_timeout(unsigned long data)
 {
 	struct vgem_fence *fence = (struct vgem_fence *)data;
 
-	fence_signal(&fence->base);
+	dma_fence_signal(&fence->base);
 }
 
-static struct fence *vgem_fence_create(struct vgem_file *vfile,
-				       unsigned int flags)
+static struct dma_fence *vgem_fence_create(struct vgem_file *vfile,
+					   unsigned int flags)
 {
 	struct vgem_fence *fence;
 
@@ -101,8 +102,8 @@ static struct fence *vgem_fence_create(struct vgem_file *vfile,
 		return NULL;
 
 	spin_lock_init(&fence->lock);
-	fence_init(&fence->base, &vgem_fence_ops, &fence->lock,
-		   fence_context_alloc(1), 1);
+	dma_fence_init(&fence->base, &vgem_fence_ops, &fence->lock,
+		       dma_fence_context_alloc(1), 1);
 
 	setup_timer(&fence->timer, vgem_fence_timeout, (unsigned long)fence);
 
@@ -125,7 +126,6 @@ static int attach_dmabuf(struct drm_device *dev,
 		return PTR_ERR(dmabuf);
 
 	obj->dma_buf = dmabuf;
-	drm_gem_object_reference(obj);
 	return 0;
 }
 
@@ -157,7 +157,7 @@ int vgem_fence_attach_ioctl(struct drm_device *dev,
 	struct vgem_file *vfile = file->driver_priv;
 	struct reservation_object *resv;
 	struct drm_gem_object *obj;
-	struct fence *fence;
+	struct dma_fence *fence;
 	int ret;
 
 	if (arg->flags & ~VGEM_FENCE_WRITE)
@@ -190,12 +190,12 @@ int vgem_fence_attach_ioctl(struct drm_device *dev,
 
 	/* Expose the fence via the dma-buf */
 	ret = 0;
-	mutex_lock(&resv->lock.base);
+	ww_mutex_lock(&resv->lock, NULL);
 	if (arg->flags & VGEM_FENCE_WRITE)
 		reservation_object_add_excl_fence(resv, fence);
 	else if ((ret = reservation_object_reserve_shared(resv)) == 0)
 		reservation_object_add_shared_fence(resv, fence);
-	mutex_unlock(&resv->lock.base);
+	ww_mutex_unlock(&resv->lock);
 
 	/* Record the fence in our idr for later signaling */
 	if (ret == 0) {
@@ -209,8 +209,8 @@ int vgem_fence_attach_ioctl(struct drm_device *dev,
 	}
 err_fence:
 	if (ret) {
-		fence_signal(fence);
-		fence_put(fence);
+		dma_fence_signal(fence);
+		dma_fence_put(fence);
 	}
 err:
 	drm_gem_object_unreference_unlocked(obj);
@@ -239,7 +239,7 @@ int vgem_fence_signal_ioctl(struct drm_device *dev,
 {
 	struct vgem_file *vfile = file->driver_priv;
 	struct drm_vgem_fence_signal *arg = data;
-	struct fence *fence;
+	struct dma_fence *fence;
 	int ret = 0;
 
 	if (arg->flags)
@@ -253,11 +253,11 @@ int vgem_fence_signal_ioctl(struct drm_device *dev,
 	if (IS_ERR(fence))
 		return PTR_ERR(fence);
 
-	if (fence_is_signaled(fence))
+	if (dma_fence_is_signaled(fence))
 		ret = -ETIMEDOUT;
 
-	fence_signal(fence);
-	fence_put(fence);
+	dma_fence_signal(fence);
+	dma_fence_put(fence);
 	return ret;
 }
 
@@ -271,8 +271,8 @@ int vgem_fence_open(struct vgem_file *vfile)
 
 static int __vgem_fence_idr_fini(int id, void *p, void *data)
 {
-	fence_signal(p);
-	fence_put(p);
+	dma_fence_signal(p);
+	dma_fence_put(p);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/via/via_drv.c b/drivers/gpu/drm/via/via_drv.c
index e5582bab7e3c..9e0e5392b6ec 100644
--- a/drivers/gpu/drm/via/via_drv.c
+++ b/drivers/gpu/drm/via/via_drv.c
@@ -64,9 +64,7 @@ static const struct file_operations via_driver_fops = {
 	.unlocked_ioctl = drm_ioctl,
 	.mmap = drm_legacy_mmap,
 	.poll = drm_poll,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.llseek = noop_llseek,
 };
 
diff --git a/drivers/gpu/drm/virtio/Kconfig b/drivers/gpu/drm/virtio/Kconfig
index e1afc3d3f8d9..81d1807ac228 100644
--- a/drivers/gpu/drm/virtio/Kconfig
+++ b/drivers/gpu/drm/virtio/Kconfig
@@ -1,10 +1,10 @@
 config DRM_VIRTIO_GPU
 	tristate "Virtio GPU driver"
 	depends on DRM && VIRTIO
-        select DRM_KMS_HELPER
-        select DRM_TTM
+	select DRM_KMS_HELPER
+	select DRM_TTM
 	help
 	   This is the virtual GPU driver for virtio.  It can be used with
-           QEMU based VMMs (like KVM or Xen).
+	   QEMU based VMMs (like KVM or Xen).
 
 	   If unsure say M.
diff --git a/drivers/gpu/drm/virtio/virtgpu_drm_bus.c b/drivers/gpu/drm/virtio/virtgpu_drm_bus.c
index 49e5996cb9f2..3b97d50fd392 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drm_bus.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drm_bus.c
@@ -28,16 +28,6 @@
 
 #include "virtgpu_drv.h"
 
-int drm_virtio_set_busid(struct drm_device *dev, struct drm_master *master)
-{
-	struct pci_dev *pdev = dev->pdev;
-
-	if (pdev) {
-		return drm_pci_set_busid(dev, master);
-	}
-	return 0;
-}
-
 static void virtio_pci_kick_out_firmware_fb(struct pci_dev *pci_dev)
 {
 	struct apertures_struct *ap;
@@ -71,13 +61,22 @@ int drm_virtio_init(struct drm_driver *driver, struct virtio_device *vdev)
 
 	if (strcmp(vdev->dev.parent->bus->name, "pci") == 0) {
 		struct pci_dev *pdev = to_pci_dev(vdev->dev.parent);
+		const char *pname = dev_name(&pdev->dev);
 		bool vga = (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;
+		char unique[20];
 
-		DRM_INFO("pci: %s detected\n",
-			 vga ? "virtio-vga" : "virtio-gpu-pci");
+		DRM_INFO("pci: %s detected at %s\n",
+			 vga ? "virtio-vga" : "virtio-gpu-pci",
+			 pname);
 		dev->pdev = pdev;
 		if (vga)
 			virtio_pci_kick_out_firmware_fb(pdev);
+
+		snprintf(unique, sizeof(unique), "pci:%s", pname);
+		ret = drm_dev_set_unique(dev, unique);
+		if (ret)
+			goto err_free;
+
 	}
 
 	ret = drm_dev_register(dev, 0);
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c
index 5820b7020ae5..d82489815096 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.c
@@ -108,16 +108,13 @@ static const struct file_operations virtio_gpu_driver_fops = {
 	.read = drm_read,
 	.unlocked_ioctl	= drm_ioctl,
 	.release = drm_release,
-#ifdef CONFIG_COMPAT
 	.compat_ioctl = drm_compat_ioctl,
-#endif
 	.llseek = noop_llseek,
 };
 
 
 static struct drm_driver driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER | DRIVER_ATOMIC,
-	.set_busid = drm_virtio_set_busid,
 	.load = virtio_gpu_driver_load,
 	.unload = virtio_gpu_driver_unload,
 	.open = virtio_gpu_driver_open,
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index ae59080d63d1..08906c8ce3fa 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -49,7 +49,6 @@
 #define DRIVER_PATCHLEVEL 1
 
 /* virtgpu_drm_bus.c */
-int drm_virtio_set_busid(struct drm_device *dev, struct drm_master *master);
 int drm_virtio_init(struct drm_driver *driver, struct virtio_device *vdev);
 
 struct virtio_gpu_object {
@@ -82,7 +81,7 @@ struct virtio_gpu_fence_driver {
 };
 
 struct virtio_gpu_fence {
-	struct fence f;
+	struct dma_fence f;
 	struct virtio_gpu_fence_driver *drv;
 	struct list_head node;
 	uint64_t seq;
diff --git a/drivers/gpu/drm/virtio/virtgpu_fb.c b/drivers/gpu/drm/virtio/virtgpu_fb.c
index 2242a80866a9..dd21f950e129 100644
--- a/drivers/gpu/drm/virtio/virtgpu_fb.c
+++ b/drivers/gpu/drm/virtio/virtgpu_fb.c
@@ -200,16 +200,10 @@ static void virtio_gpu_3d_imageblit(struct fb_info *info,
 
 static struct fb_ops virtio_gpufb_ops = {
 	.owner = THIS_MODULE,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par, /* TODO: copy vmwgfx */
+	DRM_FB_HELPER_DEFAULT_OPS,
 	.fb_fillrect = virtio_gpu_3d_fillrect,
 	.fb_copyarea = virtio_gpu_3d_copyarea,
 	.fb_imageblit = virtio_gpu_3d_imageblit,
-	.fb_pan_display = drm_fb_helper_pan_display,
-	.fb_blank = drm_fb_helper_blank,
-	.fb_setcmap = drm_fb_helper_setcmap,
-	.fb_debug_enter = drm_fb_helper_debug_enter,
-	.fb_debug_leave = drm_fb_helper_debug_leave,
 };
 
 static int virtio_gpu_vmap_fb(struct virtio_gpu_device *vgdev,
diff --git a/drivers/gpu/drm/virtio/virtgpu_fence.c b/drivers/gpu/drm/virtio/virtgpu_fence.c
index f3f70fa8a4c7..23353521f903 100644
--- a/drivers/gpu/drm/virtio/virtgpu_fence.c
+++ b/drivers/gpu/drm/virtio/virtgpu_fence.c
@@ -26,22 +26,22 @@
 #include <drm/drmP.h>
 #include "virtgpu_drv.h"
 
-static const char *virtio_get_driver_name(struct fence *f)
+static const char *virtio_get_driver_name(struct dma_fence *f)
 {
 	return "virtio_gpu";
 }
 
-static const char *virtio_get_timeline_name(struct fence *f)
+static const char *virtio_get_timeline_name(struct dma_fence *f)
 {
 	return "controlq";
 }
 
-static bool virtio_enable_signaling(struct fence *f)
+static bool virtio_enable_signaling(struct dma_fence *f)
 {
 	return true;
 }
 
-static bool virtio_signaled(struct fence *f)
+static bool virtio_signaled(struct dma_fence *f)
 {
 	struct virtio_gpu_fence *fence = to_virtio_fence(f);
 
@@ -50,26 +50,26 @@ static bool virtio_signaled(struct fence *f)
 	return false;
 }
 
-static void virtio_fence_value_str(struct fence *f, char *str, int size)
+static void virtio_fence_value_str(struct dma_fence *f, char *str, int size)
 {
 	struct virtio_gpu_fence *fence = to_virtio_fence(f);
 
 	snprintf(str, size, "%llu", fence->seq);
 }
 
-static void virtio_timeline_value_str(struct fence *f, char *str, int size)
+static void virtio_timeline_value_str(struct dma_fence *f, char *str, int size)
 {
 	struct virtio_gpu_fence *fence = to_virtio_fence(f);
 
 	snprintf(str, size, "%llu", (u64)atomic64_read(&fence->drv->last_seq));
 }
 
-static const struct fence_ops virtio_fence_ops = {
+static const struct dma_fence_ops virtio_fence_ops = {
 	.get_driver_name     = virtio_get_driver_name,
 	.get_timeline_name   = virtio_get_timeline_name,
 	.enable_signaling    = virtio_enable_signaling,
 	.signaled            = virtio_signaled,
-	.wait                = fence_default_wait,
+	.wait                = dma_fence_default_wait,
 	.fence_value_str     = virtio_fence_value_str,
 	.timeline_value_str  = virtio_timeline_value_str,
 };
@@ -88,9 +88,9 @@ int virtio_gpu_fence_emit(struct virtio_gpu_device *vgdev,
 	spin_lock_irqsave(&drv->lock, irq_flags);
 	(*fence)->drv = drv;
 	(*fence)->seq = ++drv->sync_seq;
-	fence_init(&(*fence)->f, &virtio_fence_ops, &drv->lock,
-		   drv->context, (*fence)->seq);
-	fence_get(&(*fence)->f);
+	dma_fence_init(&(*fence)->f, &virtio_fence_ops, &drv->lock,
+		       drv->context, (*fence)->seq);
+	dma_fence_get(&(*fence)->f);
 	list_add_tail(&(*fence)->node, &drv->fences);
 	spin_unlock_irqrestore(&drv->lock, irq_flags);
 
@@ -111,9 +111,9 @@ void virtio_gpu_fence_event_process(struct virtio_gpu_device *vgdev,
 	list_for_each_entry_safe(fence, tmp, &drv->fences, node) {
 		if (last_seq < fence->seq)
 			continue;
-		fence_signal_locked(&fence->f);
+		dma_fence_signal_locked(&fence->f);
 		list_del(&fence->node);
-		fence_put(&fence->f);
+		dma_fence_put(&fence->f);
 	}
 	spin_unlock_irqrestore(&drv->lock, irq_flags);
 }
diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
index 818478b4c4f0..61f3a963af95 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
@@ -172,7 +172,7 @@ static int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
 	/* fence the command bo */
 	virtio_gpu_unref_list(&validate_list);
 	drm_free_large(buflist);
-	fence_put(&fence->f);
+	dma_fence_put(&fence->f);
 	return 0;
 
 out_unresv:
@@ -298,7 +298,7 @@ static int virtio_gpu_resource_create_ioctl(struct drm_device *dev, void *data,
 		drm_gem_object_release(obj);
 		if (vgdev->has_virgl_3d) {
 			virtio_gpu_unref_list(&validate_list);
-			fence_put(&fence->f);
+			dma_fence_put(&fence->f);
 		}
 		return ret;
 	}
@@ -309,13 +309,13 @@ static int virtio_gpu_resource_create_ioctl(struct drm_device *dev, void *data,
 
 	if (vgdev->has_virgl_3d) {
 		virtio_gpu_unref_list(&validate_list);
-		fence_put(&fence->f);
+		dma_fence_put(&fence->f);
 	}
 	return 0;
 fail_unref:
 	if (vgdev->has_virgl_3d) {
 		virtio_gpu_unref_list(&validate_list);
-		fence_put(&fence->f);
+		dma_fence_put(&fence->f);
 	}
 //fail_obj:
 //	drm_gem_object_handle_unreference_unlocked(obj);
@@ -383,7 +383,7 @@ static int virtio_gpu_transfer_from_host_ioctl(struct drm_device *dev,
 	reservation_object_add_excl_fence(qobj->tbo.resv,
 					  &fence->f);
 
-	fence_put(&fence->f);
+	dma_fence_put(&fence->f);
 out_unres:
 	virtio_gpu_object_unreserve(qobj);
 out:
@@ -431,7 +431,7 @@ static int virtio_gpu_transfer_to_host_ioctl(struct drm_device *dev, void *data,
 			 args->level, &box, &fence);
 		reservation_object_add_excl_fence(qobj->tbo.resv,
 						  &fence->f);
-		fence_put(&fence->f);
+		dma_fence_put(&fence->f);
 	}
 
 out_unres:
diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c
index 036b0fbae0fb..1235519853f4 100644
--- a/drivers/gpu/drm/virtio/virtgpu_kms.c
+++ b/drivers/gpu/drm/virtio/virtgpu_kms.c
@@ -159,7 +159,7 @@ int virtio_gpu_driver_load(struct drm_device *dev, unsigned long flags)
 	virtio_gpu_init_vq(&vgdev->ctrlq, virtio_gpu_dequeue_ctrl_func);
 	virtio_gpu_init_vq(&vgdev->cursorq, virtio_gpu_dequeue_cursor_func);
 
-	vgdev->fence_drv.context = fence_context_alloc(1);
+	vgdev->fence_drv.context = dma_fence_context_alloc(1);
 	spin_lock_init(&vgdev->fence_drv.lock);
 	INIT_LIST_HEAD(&vgdev->fence_drv.fences);
 	INIT_LIST_HEAD(&vgdev->cap_cache);
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c b/drivers/gpu/drm/virtio/virtgpu_plane.c
index ba28c0f6f28a..11288ffa4af6 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -88,8 +88,8 @@ static void virtio_gpu_primary_plane_update(struct drm_plane *plane,
 				(vgdev, handle, 0,
 				 cpu_to_le32(plane->state->src_w >> 16),
 				 cpu_to_le32(plane->state->src_h >> 16),
-				 plane->state->src_x >> 16,
-				 plane->state->src_y >> 16, NULL);
+				 cpu_to_le32(plane->state->src_x >> 16),
+				 cpu_to_le32(plane->state->src_y >> 16), NULL);
 		}
 	} else {
 		handle = 0;
@@ -152,7 +152,7 @@ static void virtio_gpu_cursor_plane_update(struct drm_plane *plane,
 		if (!ret) {
 			reservation_object_add_excl_fence(bo->tbo.resv,
 							  &fence->f);
-			fence_put(&fence->f);
+			dma_fence_put(&fence->f);
 			fence = NULL;
 			virtio_gpu_object_unreserve(bo);
 			virtio_gpu_object_wait(bo, false);
diff --git a/drivers/gpu/drm/virtio/virtgpu_ttm.c b/drivers/gpu/drm/virtio/virtgpu_ttm.c
index 80482ac5f95d..4a1de9f81193 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ttm.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ttm.c
@@ -425,6 +425,7 @@ static struct ttm_bo_driver virtio_gpu_bo_driver = {
 	.ttm_tt_unpopulate = &virtio_gpu_ttm_tt_unpopulate,
 	.invalidate_caches = &virtio_gpu_invalidate_caches,
 	.init_mem_type = &virtio_gpu_init_mem_type,
+	.eviction_valuable = ttm_bo_eviction_valuable,
 	.evict_flags = &virtio_gpu_evict_flags,
 	.move = &virtio_gpu_bo_move,
 	.verify_access = &virtio_gpu_verify_access,
diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c
index 5a0f8a745b9d..43ea0dc957d2 100644
--- a/drivers/gpu/drm/virtio/virtgpu_vq.c
+++ b/drivers/gpu/drm/virtio/virtgpu_vq.c
@@ -75,7 +75,7 @@ void virtio_gpu_cursor_ack(struct virtqueue *vq)
 int virtio_gpu_alloc_vbufs(struct virtio_gpu_device *vgdev)
 {
 	struct virtio_gpu_vbuffer *vbuf;
-	int i, size, count = 0;
+	int i, size, count = 16;
 	void *ptr;
 
 	INIT_LIST_HEAD(&vgdev->free_vbufs);
@@ -109,8 +109,10 @@ void virtio_gpu_free_vbufs(struct virtio_gpu_device *vgdev)
 
 	spin_lock(&vgdev->free_vbufs_lock);
 	for (i = 0; i < count; i++) {
-		if (WARN_ON(list_empty(&vgdev->free_vbufs)))
+		if (WARN_ON(list_empty(&vgdev->free_vbufs))) {
+			spin_unlock(&vgdev->free_vbufs_lock);
 			return;
+		}
 		vbuf = list_first_entry(&vgdev->free_vbufs,
 					struct virtio_gpu_vbuffer, list);
 		list_del(&vbuf->list);
@@ -295,6 +297,8 @@ void virtio_gpu_dequeue_cursor_func(struct work_struct *work)
 
 static int virtio_gpu_queue_ctrl_buffer_locked(struct virtio_gpu_device *vgdev,
 					       struct virtio_gpu_vbuffer *vbuf)
+		__releases(&vgdev->ctrlq.qlock)
+		__acquires(&vgdev->ctrlq.qlock)
 {
 	struct virtqueue *vq = vgdev->ctrlq.vq;
 	struct scatterlist *sgs[3], vcmd, vout, vresp;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
index 78b75ee3c931..c894a48a74a6 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
@@ -849,6 +849,7 @@ struct ttm_bo_driver vmw_bo_driver = {
 	.ttm_tt_unpopulate = &vmw_ttm_unpopulate,
 	.invalidate_caches = vmw_invalidate_caches,
 	.init_mem_type = vmw_init_mem_type,
+	.eviction_valuable = ttm_bo_eviction_valuable,
 	.evict_flags = vmw_evict_flags,
 	.move = NULL,
 	.verify_access = vmw_verify_access,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
index d2d93959b119..723fd763da8e 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
@@ -465,33 +465,34 @@ static int vmw_fb_kms_detach(struct vmw_fb_par *par,
 
 static int vmw_fb_kms_framebuffer(struct fb_info *info)
 {
-	struct drm_mode_fb_cmd mode_cmd;
+	struct drm_mode_fb_cmd2 mode_cmd;
 	struct vmw_fb_par *par = info->par;
 	struct fb_var_screeninfo *var = &info->var;
 	struct drm_framebuffer *cur_fb;
 	struct vmw_framebuffer *vfb;
-	int ret = 0;
+	int ret = 0, depth;
 	size_t new_bo_size;
 
-	ret = vmw_fb_compute_depth(var, &mode_cmd.depth);
+	ret = vmw_fb_compute_depth(var, &depth);
 	if (ret)
 		return ret;
 
 	mode_cmd.width = var->xres;
 	mode_cmd.height = var->yres;
-	mode_cmd.bpp = var->bits_per_pixel;
-	mode_cmd.pitch = ((mode_cmd.bpp + 7) / 8) * mode_cmd.width;
+	mode_cmd.pitches[0] = ((var->bits_per_pixel + 7) / 8) * mode_cmd.width;
+	mode_cmd.pixel_format =
+		drm_mode_legacy_fb_format(var->bits_per_pixel,
+			((var->bits_per_pixel + 7) / 8) * mode_cmd.width);
 
 	cur_fb = par->set_fb;
 	if (cur_fb && cur_fb->width == mode_cmd.width &&
 	    cur_fb->height == mode_cmd.height &&
-	    cur_fb->bits_per_pixel == mode_cmd.bpp &&
-	    cur_fb->depth == mode_cmd.depth &&
-	    cur_fb->pitches[0] == mode_cmd.pitch)
+	    cur_fb->pixel_format == mode_cmd.pixel_format &&
+	    cur_fb->pitches[0] == mode_cmd.pitches[0])
 		return 0;
 
 	/* Need new buffer object ? */
-	new_bo_size = (size_t) mode_cmd.pitch * (size_t) mode_cmd.height;
+	new_bo_size = (size_t) mode_cmd.pitches[0] * (size_t) mode_cmd.height;
 	ret = vmw_fb_kms_detach(par,
 				par->bo_size < new_bo_size ||
 				par->bo_size > 2*new_bo_size,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
index 26ac8e80a478..6541dd8b82dc 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
@@ -108,7 +108,7 @@ fman_from_fence(struct vmw_fence_obj *fence)
  * objects with actions attached to them.
  */
 
-static void vmw_fence_obj_destroy(struct fence *f)
+static void vmw_fence_obj_destroy(struct dma_fence *f)
 {
 	struct vmw_fence_obj *fence =
 		container_of(f, struct vmw_fence_obj, base);
@@ -123,17 +123,17 @@ static void vmw_fence_obj_destroy(struct fence *f)
 	fence->destroy(fence);
 }
 
-static const char *vmw_fence_get_driver_name(struct fence *f)
+static const char *vmw_fence_get_driver_name(struct dma_fence *f)
 {
 	return "vmwgfx";
 }
 
-static const char *vmw_fence_get_timeline_name(struct fence *f)
+static const char *vmw_fence_get_timeline_name(struct dma_fence *f)
 {
 	return "svga";
 }
 
-static bool vmw_fence_enable_signaling(struct fence *f)
+static bool vmw_fence_enable_signaling(struct dma_fence *f)
 {
 	struct vmw_fence_obj *fence =
 		container_of(f, struct vmw_fence_obj, base);
@@ -152,12 +152,12 @@ static bool vmw_fence_enable_signaling(struct fence *f)
 }
 
 struct vmwgfx_wait_cb {
-	struct fence_cb base;
+	struct dma_fence_cb base;
 	struct task_struct *task;
 };
 
 static void
-vmwgfx_wait_cb(struct fence *fence, struct fence_cb *cb)
+vmwgfx_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 {
 	struct vmwgfx_wait_cb *wait =
 		container_of(cb, struct vmwgfx_wait_cb, base);
@@ -167,7 +167,7 @@ vmwgfx_wait_cb(struct fence *fence, struct fence_cb *cb)
 
 static void __vmw_fences_update(struct vmw_fence_manager *fman);
 
-static long vmw_fence_wait(struct fence *f, bool intr, signed long timeout)
+static long vmw_fence_wait(struct dma_fence *f, bool intr, signed long timeout)
 {
 	struct vmw_fence_obj *fence =
 		container_of(f, struct vmw_fence_obj, base);
@@ -197,7 +197,7 @@ static long vmw_fence_wait(struct fence *f, bool intr, signed long timeout)
 
 	while (ret > 0) {
 		__vmw_fences_update(fman);
-		if (test_bit(FENCE_FLAG_SIGNALED_BIT, &f->flags))
+		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags))
 			break;
 
 		if (intr)
@@ -225,7 +225,7 @@ out:
 	return ret;
 }
 
-static struct fence_ops vmw_fence_ops = {
+static struct dma_fence_ops vmw_fence_ops = {
 	.get_driver_name = vmw_fence_get_driver_name,
 	.get_timeline_name = vmw_fence_get_timeline_name,
 	.enable_signaling = vmw_fence_enable_signaling,
@@ -298,7 +298,7 @@ struct vmw_fence_manager *vmw_fence_manager_init(struct vmw_private *dev_priv)
 	fman->event_fence_action_size =
 		ttm_round_pot(sizeof(struct vmw_event_fence_action));
 	mutex_init(&fman->goal_irq_mutex);
-	fman->ctx = fence_context_alloc(1);
+	fman->ctx = dma_fence_context_alloc(1);
 
 	return fman;
 }
@@ -326,8 +326,8 @@ static int vmw_fence_obj_init(struct vmw_fence_manager *fman,
 	unsigned long irq_flags;
 	int ret = 0;
 
-	fence_init(&fence->base, &vmw_fence_ops, &fman->lock,
-		   fman->ctx, seqno);
+	dma_fence_init(&fence->base, &vmw_fence_ops, &fman->lock,
+		       fman->ctx, seqno);
 	INIT_LIST_HEAD(&fence->seq_passed_actions);
 	fence->destroy = destroy;
 
@@ -431,7 +431,7 @@ static bool vmw_fence_goal_check_locked(struct vmw_fence_obj *fence)
 	u32 goal_seqno;
 	u32 *fifo_mem;
 
-	if (fence_is_signaled_locked(&fence->base))
+	if (dma_fence_is_signaled_locked(&fence->base))
 		return false;
 
 	fifo_mem = fman->dev_priv->mmio_virt;
@@ -459,7 +459,7 @@ rerun:
 	list_for_each_entry_safe(fence, next_fence, &fman->fence_list, head) {
 		if (seqno - fence->base.seqno < VMW_FENCE_WRAP) {
 			list_del_init(&fence->head);
-			fence_signal_locked(&fence->base);
+			dma_fence_signal_locked(&fence->base);
 			INIT_LIST_HEAD(&action_list);
 			list_splice_init(&fence->seq_passed_actions,
 					 &action_list);
@@ -500,18 +500,18 @@ bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence)
 {
 	struct vmw_fence_manager *fman = fman_from_fence(fence);
 
-	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
 		return 1;
 
 	vmw_fences_update(fman);
 
-	return fence_is_signaled(&fence->base);
+	return dma_fence_is_signaled(&fence->base);
 }
 
 int vmw_fence_obj_wait(struct vmw_fence_obj *fence, bool lazy,
 		       bool interruptible, unsigned long timeout)
 {
-	long ret = fence_wait_timeout(&fence->base, interruptible, timeout);
+	long ret = dma_fence_wait_timeout(&fence->base, interruptible, timeout);
 
 	if (likely(ret > 0))
 		return 0;
@@ -530,7 +530,7 @@ void vmw_fence_obj_flush(struct vmw_fence_obj *fence)
 
 static void vmw_fence_destroy(struct vmw_fence_obj *fence)
 {
-	fence_free(&fence->base);
+	dma_fence_free(&fence->base);
 }
 
 int vmw_fence_create(struct vmw_fence_manager *fman,
@@ -669,7 +669,7 @@ void vmw_fence_fifo_down(struct vmw_fence_manager *fman)
 		struct vmw_fence_obj *fence =
 			list_entry(fman->fence_list.prev, struct vmw_fence_obj,
 				   head);
-		fence_get(&fence->base);
+		dma_fence_get(&fence->base);
 		spin_unlock_irq(&fman->lock);
 
 		ret = vmw_fence_obj_wait(fence, false, false,
@@ -677,7 +677,7 @@ void vmw_fence_fifo_down(struct vmw_fence_manager *fman)
 
 		if (unlikely(ret != 0)) {
 			list_del_init(&fence->head);
-			fence_signal(&fence->base);
+			dma_fence_signal(&fence->base);
 			INIT_LIST_HEAD(&action_list);
 			list_splice_init(&fence->seq_passed_actions,
 					 &action_list);
@@ -685,7 +685,7 @@ void vmw_fence_fifo_down(struct vmw_fence_manager *fman)
 		}
 
 		BUG_ON(!list_empty(&fence->head));
-		fence_put(&fence->base);
+		dma_fence_put(&fence->base);
 		spin_lock_irq(&fman->lock);
 	}
 	spin_unlock_irq(&fman->lock);
@@ -884,7 +884,7 @@ static void vmw_fence_obj_add_action(struct vmw_fence_obj *fence,
 	spin_lock_irqsave(&fman->lock, irq_flags);
 
 	fman->pending_actions[action->type]++;
-	if (fence_is_signaled_locked(&fence->base)) {
+	if (dma_fence_is_signaled_locked(&fence->base)) {
 		struct list_head action_list;
 
 		INIT_LIST_HEAD(&action_list);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.h b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.h
index 83ae301ee141..d9d85aa6ed20 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.h
@@ -27,7 +27,7 @@
 
 #ifndef _VMWGFX_FENCE_H_
 
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 
 #define VMW_FENCE_WAIT_TIMEOUT (5*HZ)
 
@@ -52,7 +52,7 @@ struct vmw_fence_action {
 };
 
 struct vmw_fence_obj {
-	struct fence base;
+	struct dma_fence base;
 
 	struct list_head head;
 	struct list_head seq_passed_actions;
@@ -71,14 +71,14 @@ vmw_fence_obj_unreference(struct vmw_fence_obj **fence_p)
 
 	*fence_p = NULL;
 	if (fence)
-		fence_put(&fence->base);
+		dma_fence_put(&fence->base);
 }
 
 static inline struct vmw_fence_obj *
 vmw_fence_obj_reference(struct vmw_fence_obj *fence)
 {
 	if (fence)
-		fence_get(&fence->base);
+		dma_fence_get(&fence->base);
 	return fence;
 }
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index bf28ccc150df..e7daf59bac80 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -516,7 +516,7 @@ static const struct drm_framebuffer_funcs vmw_framebuffer_surface_funcs = {
 static int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv,
 					   struct vmw_surface *surface,
 					   struct vmw_framebuffer **out,
-					   const struct drm_mode_fb_cmd
+					   const struct drm_mode_fb_cmd2
 					   *mode_cmd,
 					   bool is_dmabuf_proxy)
 
@@ -525,6 +525,7 @@ static int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv,
 	struct vmw_framebuffer_surface *vfbs;
 	enum SVGA3dSurfaceFormat format;
 	int ret;
+	struct drm_format_name_buf format_name;
 
 	/* 3D is only supported on HWv8 and newer hosts */
 	if (dev_priv->active_display_unit == vmw_du_legacy)
@@ -548,21 +549,22 @@ static int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv,
 		return -EINVAL;
 	}
 
-	switch (mode_cmd->depth) {
-	case 32:
+	switch (mode_cmd->pixel_format) {
+	case DRM_FORMAT_ARGB8888:
 		format = SVGA3D_A8R8G8B8;
 		break;
-	case 24:
+	case DRM_FORMAT_XRGB8888:
 		format = SVGA3D_X8R8G8B8;
 		break;
-	case 16:
+	case DRM_FORMAT_RGB565:
 		format = SVGA3D_R5G6B5;
 		break;
-	case 15:
+	case DRM_FORMAT_XRGB1555:
 		format = SVGA3D_A1R5G5B5;
 		break;
 	default:
-		DRM_ERROR("Invalid color depth: %d\n", mode_cmd->depth);
+		DRM_ERROR("Invalid pixel format: %s\n",
+			  drm_get_format_name(mode_cmd->pixel_format, &format_name));
 		return -EINVAL;
 	}
 
@@ -581,14 +583,9 @@ static int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv,
 		goto out_err1;
 	}
 
-	/* XXX get the first 3 from the surface info */
-	vfbs->base.base.bits_per_pixel = mode_cmd->bpp;
-	vfbs->base.base.pitches[0] = mode_cmd->pitch;
-	vfbs->base.base.depth = mode_cmd->depth;
-	vfbs->base.base.width = mode_cmd->width;
-	vfbs->base.base.height = mode_cmd->height;
+	drm_helper_mode_fill_fb_struct(&vfbs->base.base, mode_cmd);
 	vfbs->surface = vmw_surface_reference(surface);
-	vfbs->base.user_handle = mode_cmd->handle;
+	vfbs->base.user_handle = mode_cmd->handles[0];
 	vfbs->is_dmabuf_proxy = is_dmabuf_proxy;
 
 	*out = &vfbs->base;
@@ -755,7 +752,7 @@ static int vmw_framebuffer_unpin(struct vmw_framebuffer *vfb)
  * 0 on success, error code otherwise
  */
 static int vmw_create_dmabuf_proxy(struct drm_device *dev,
-				   const struct drm_mode_fb_cmd *mode_cmd,
+				   const struct drm_mode_fb_cmd2 *mode_cmd,
 				   struct vmw_dma_buffer *dmabuf_mob,
 				   struct vmw_surface **srf_out)
 {
@@ -763,17 +760,18 @@ static int vmw_create_dmabuf_proxy(struct drm_device *dev,
 	struct drm_vmw_size content_base_size;
 	struct vmw_resource *res;
 	unsigned int bytes_pp;
+	struct drm_format_name_buf format_name;
 	int ret;
 
-	switch (mode_cmd->depth) {
-	case 32:
-	case 24:
+	switch (mode_cmd->pixel_format) {
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_XRGB8888:
 		format = SVGA3D_X8R8G8B8;
 		bytes_pp = 4;
 		break;
 
-	case 16:
-	case 15:
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_XRGB1555:
 		format = SVGA3D_R5G6B5;
 		bytes_pp = 2;
 		break;
@@ -784,11 +782,12 @@ static int vmw_create_dmabuf_proxy(struct drm_device *dev,
 		break;
 
 	default:
-		DRM_ERROR("Invalid framebuffer format %d\n", mode_cmd->depth);
+		DRM_ERROR("Invalid framebuffer format %s\n",
+			  drm_get_format_name(mode_cmd->pixel_format, &format_name));
 		return -EINVAL;
 	}
 
-	content_base_size.width  = mode_cmd->pitch / bytes_pp;
+	content_base_size.width  = mode_cmd->pitches[0] / bytes_pp;
 	content_base_size.height = mode_cmd->height;
 	content_base_size.depth  = 1;
 
@@ -826,16 +825,17 @@ static int vmw_create_dmabuf_proxy(struct drm_device *dev,
 static int vmw_kms_new_framebuffer_dmabuf(struct vmw_private *dev_priv,
 					  struct vmw_dma_buffer *dmabuf,
 					  struct vmw_framebuffer **out,
-					  const struct drm_mode_fb_cmd
+					  const struct drm_mode_fb_cmd2
 					  *mode_cmd)
 
 {
 	struct drm_device *dev = dev_priv->dev;
 	struct vmw_framebuffer_dmabuf *vfbd;
 	unsigned int requested_size;
+	struct drm_format_name_buf format_name;
 	int ret;
 
-	requested_size = mode_cmd->height * mode_cmd->pitch;
+	requested_size = mode_cmd->height * mode_cmd->pitches[0];
 	if (unlikely(requested_size > dmabuf->base.num_pages * PAGE_SIZE)) {
 		DRM_ERROR("Screen buffer object size is too small "
 			  "for requested mode.\n");
@@ -844,27 +844,16 @@ static int vmw_kms_new_framebuffer_dmabuf(struct vmw_private *dev_priv,
 
 	/* Limited framebuffer color depth support for screen objects */
 	if (dev_priv->active_display_unit == vmw_du_screen_object) {
-		switch (mode_cmd->depth) {
-		case 32:
-		case 24:
-			/* Only support 32 bpp for 32 and 24 depth fbs */
-			if (mode_cmd->bpp == 32)
-				break;
-
-			DRM_ERROR("Invalid color depth/bbp: %d %d\n",
-				  mode_cmd->depth, mode_cmd->bpp);
-			return -EINVAL;
-		case 16:
-		case 15:
-			/* Only support 16 bpp for 16 and 15 depth fbs */
-			if (mode_cmd->bpp == 16)
-				break;
-
-			DRM_ERROR("Invalid color depth/bbp: %d %d\n",
-				  mode_cmd->depth, mode_cmd->bpp);
-			return -EINVAL;
+		switch (mode_cmd->pixel_format) {
+		case DRM_FORMAT_XRGB8888:
+		case DRM_FORMAT_ARGB8888:
+			break;
+		case DRM_FORMAT_XRGB1555:
+		case DRM_FORMAT_RGB565:
+			break;
 		default:
-			DRM_ERROR("Invalid color depth: %d\n", mode_cmd->depth);
+			DRM_ERROR("Invalid pixel format: %s\n",
+				  drm_get_format_name(mode_cmd->pixel_format, &format_name));
 			return -EINVAL;
 		}
 	}
@@ -875,14 +864,10 @@ static int vmw_kms_new_framebuffer_dmabuf(struct vmw_private *dev_priv,
 		goto out_err1;
 	}
 
-	vfbd->base.base.bits_per_pixel = mode_cmd->bpp;
-	vfbd->base.base.pitches[0] = mode_cmd->pitch;
-	vfbd->base.base.depth = mode_cmd->depth;
-	vfbd->base.base.width = mode_cmd->width;
-	vfbd->base.base.height = mode_cmd->height;
+	drm_helper_mode_fill_fb_struct(&vfbd->base.base, mode_cmd);
 	vfbd->base.dmabuf = true;
 	vfbd->buffer = vmw_dmabuf_reference(dmabuf);
-	vfbd->base.user_handle = mode_cmd->handle;
+	vfbd->base.user_handle = mode_cmd->handles[0];
 	*out = &vfbd->base;
 
 	ret = drm_framebuffer_init(dev, &vfbd->base.base,
@@ -916,7 +901,7 @@ vmw_kms_new_framebuffer(struct vmw_private *dev_priv,
 			struct vmw_dma_buffer *dmabuf,
 			struct vmw_surface *surface,
 			bool only_2d,
-			const struct drm_mode_fb_cmd *mode_cmd)
+			const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct vmw_framebuffer *vfb = NULL;
 	bool is_dmabuf_proxy = false;
@@ -971,7 +956,7 @@ vmw_kms_new_framebuffer(struct vmw_private *dev_priv,
 
 static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 						 struct drm_file *file_priv,
-						 const struct drm_mode_fb_cmd2 *mode_cmd2)
+						 const struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct vmw_private *dev_priv = vmw_priv(dev);
 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
@@ -979,16 +964,8 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 	struct vmw_surface *surface = NULL;
 	struct vmw_dma_buffer *bo = NULL;
 	struct ttm_base_object *user_obj;
-	struct drm_mode_fb_cmd mode_cmd;
 	int ret;
 
-	mode_cmd.width = mode_cmd2->width;
-	mode_cmd.height = mode_cmd2->height;
-	mode_cmd.pitch = mode_cmd2->pitches[0];
-	mode_cmd.handle = mode_cmd2->handles[0];
-	drm_fb_get_bpp_depth(mode_cmd2->pixel_format, &mode_cmd.depth,
-				    &mode_cmd.bpp);
-
 	/**
 	 * This code should be conditioned on Screen Objects not being used.
 	 * If screen objects are used, we can allocate a GMR to hold the
@@ -996,8 +973,8 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 	 */
 
 	if (!vmw_kms_validate_mode_vram(dev_priv,
-					mode_cmd.pitch,
-					mode_cmd.height)) {
+					mode_cmd->pitches[0],
+					mode_cmd->height)) {
 		DRM_ERROR("Requested mode exceed bounding box limit.\n");
 		return ERR_PTR(-ENOMEM);
 	}
@@ -1011,7 +988,7 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 	 * command stream using user-space handles.
 	 */
 
-	user_obj = ttm_base_object_lookup(tfile, mode_cmd.handle);
+	user_obj = ttm_base_object_lookup(tfile, mode_cmd->handles[0]);
 	if (unlikely(user_obj == NULL)) {
 		DRM_ERROR("Could not locate requested kms frame buffer.\n");
 		return ERR_PTR(-ENOENT);
@@ -1023,14 +1000,14 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 
 	/* returns either a dmabuf or surface */
 	ret = vmw_user_lookup_handle(dev_priv, tfile,
-				     mode_cmd.handle,
+				     mode_cmd->handles[0],
 				     &surface, &bo);
 	if (ret)
 		goto err_out;
 
 	vfb = vmw_kms_new_framebuffer(dev_priv, bo, surface,
 				      !(dev_priv->capabilities & SVGA_CAP_3D),
-				      &mode_cmd);
+				      mode_cmd);
 	if (IS_ERR(vfb)) {
 		ret = PTR_ERR(vfb);
 		goto err_out;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index ff4803c107bc..f42ce9a1c3ac 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -248,7 +248,7 @@ vmw_kms_new_framebuffer(struct vmw_private *dev_priv,
 			struct vmw_dma_buffer *dmabuf,
 			struct vmw_surface *surface,
 			bool only_2d,
-			const struct drm_mode_fb_cmd *mode_cmd);
+			const struct drm_mode_fb_cmd2 *mode_cmd);
 int vmw_kms_fbdev_init_data(struct vmw_private *dev_priv,
 			    unsigned unit,
 			    u32 max_width,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index 52ca1c9d070e..8e86d6d4141b 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -575,7 +575,7 @@ static int vmw_user_dmabuf_synccpu_grab(struct vmw_user_dma_buffer *user_bo,
 		long lret;
 
 		lret = reservation_object_wait_timeout_rcu(bo->resv, true, true,
-					nonblock ? 0 : MAX_SCHEDULE_TIMEOUT);
+							   nonblock ? 0 : MAX_SCHEDULE_TIMEOUT);
 		if (!lret)
 			return -EBUSY;
 		else if (lret < 0)
@@ -1454,7 +1454,7 @@ void vmw_fence_single_bo(struct ttm_buffer_object *bo,
 	if (fence == NULL) {
 		vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
 		reservation_object_add_excl_fence(bo->resv, &fence->base);
-		fence_put(&fence->base);
+		dma_fence_put(&fence->base);
 	} else
 		reservation_object_add_excl_fence(bo->resv, &fence->base);
 }
diff --git a/drivers/gpu/drm/zte/Kconfig b/drivers/gpu/drm/zte/Kconfig
new file mode 100644
index 000000000000..4065b2840f1c
--- /dev/null
+++ b/drivers/gpu/drm/zte/Kconfig
@@ -0,0 +1,8 @@
+config DRM_ZTE
+	tristate "DRM Support for ZTE SoCs"
+	depends on DRM && ARCH_ZX
+	select DRM_KMS_CMA_HELPER
+	select DRM_KMS_FB_HELPER
+	select DRM_KMS_HELPER
+	help
+	  Choose this option to enable DRM on ZTE ZX SoCs.
diff --git a/drivers/gpu/drm/zte/Makefile b/drivers/gpu/drm/zte/Makefile
new file mode 100644
index 000000000000..699180bfd57c
--- /dev/null
+++ b/drivers/gpu/drm/zte/Makefile
@@ -0,0 +1,7 @@
+zxdrm-y := \
+	zx_drm_drv.o \
+	zx_hdmi.o \
+	zx_plane.o \
+	zx_vou.o
+
+obj-$(CONFIG_DRM_ZTE) += zxdrm.o
diff --git a/drivers/gpu/drm/zte/zx_drm_drv.c b/drivers/gpu/drm/zte/zx_drm_drv.c
new file mode 100644
index 000000000000..3e76f72c92ff
--- /dev/null
+++ b/drivers/gpu/drm/zte/zx_drm_drv.c
@@ -0,0 +1,267 @@
+/*
+ * Copyright 2016 Linaro Ltd.
+ * Copyright 2016 ZTE Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of_graph.h>
+#include <linux/of_platform.h>
+#include <linux/spinlock.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_of.h>
+#include <drm/drmP.h>
+
+#include "zx_drm_drv.h"
+#include "zx_vou.h"
+
+struct zx_drm_private {
+	struct drm_fbdev_cma *fbdev;
+};
+
+static void zx_drm_fb_output_poll_changed(struct drm_device *drm)
+{
+	struct zx_drm_private *priv = drm->dev_private;
+
+	drm_fbdev_cma_hotplug_event(priv->fbdev);
+}
+
+static const struct drm_mode_config_funcs zx_drm_mode_config_funcs = {
+	.fb_create = drm_fb_cma_create,
+	.output_poll_changed = zx_drm_fb_output_poll_changed,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
+static void zx_drm_lastclose(struct drm_device *drm)
+{
+	struct zx_drm_private *priv = drm->dev_private;
+
+	drm_fbdev_cma_restore_mode(priv->fbdev);
+}
+
+static const struct file_operations zx_drm_fops = {
+	.owner = THIS_MODULE,
+	.open = drm_open,
+	.release = drm_release,
+	.unlocked_ioctl = drm_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl = drm_compat_ioctl,
+#endif
+	.poll = drm_poll,
+	.read = drm_read,
+	.llseek = noop_llseek,
+	.mmap = drm_gem_cma_mmap,
+};
+
+static struct drm_driver zx_drm_driver = {
+	.driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
+			   DRIVER_ATOMIC,
+	.lastclose = zx_drm_lastclose,
+	.get_vblank_counter = drm_vblank_no_hw_counter,
+	.enable_vblank = zx_vou_enable_vblank,
+	.disable_vblank = zx_vou_disable_vblank,
+	.gem_free_object = drm_gem_cma_free_object,
+	.gem_vm_ops = &drm_gem_cma_vm_ops,
+	.dumb_create = drm_gem_cma_dumb_create,
+	.dumb_map_offset = drm_gem_cma_dumb_map_offset,
+	.dumb_destroy = drm_gem_dumb_destroy,
+	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
+	.gem_prime_export = drm_gem_prime_export,
+	.gem_prime_import = drm_gem_prime_import,
+	.gem_prime_get_sg_table = drm_gem_cma_prime_get_sg_table,
+	.gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table,
+	.gem_prime_vmap = drm_gem_cma_prime_vmap,
+	.gem_prime_vunmap = drm_gem_cma_prime_vunmap,
+	.gem_prime_mmap = drm_gem_cma_prime_mmap,
+	.fops = &zx_drm_fops,
+	.name = "zx-vou",
+	.desc = "ZTE VOU Controller DRM",
+	.date = "20160811",
+	.major = 1,
+	.minor = 0,
+};
+
+static int zx_drm_bind(struct device *dev)
+{
+	struct drm_device *drm;
+	struct zx_drm_private *priv;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	drm = drm_dev_alloc(&zx_drm_driver, dev);
+	if (IS_ERR(drm))
+		return PTR_ERR(drm);
+
+	drm->dev_private = priv;
+	dev_set_drvdata(dev, drm);
+
+	drm_mode_config_init(drm);
+	drm->mode_config.min_width = 16;
+	drm->mode_config.min_height = 16;
+	drm->mode_config.max_width = 4096;
+	drm->mode_config.max_height = 4096;
+	drm->mode_config.funcs = &zx_drm_mode_config_funcs;
+
+	ret = component_bind_all(dev, drm);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "failed to bind all components: %d\n", ret);
+		goto out_unregister;
+	}
+
+	ret = drm_vblank_init(drm, drm->mode_config.num_crtc);
+	if (ret < 0) {
+		DRM_DEV_ERROR(dev, "failed to init vblank: %d\n", ret);
+		goto out_unbind;
+	}
+
+	/*
+	 * We will manage irq handler on our own.  In this case, irq_enabled
+	 * need to be true for using vblank core support.
+	 */
+	drm->irq_enabled = true;
+
+	drm_mode_config_reset(drm);
+	drm_kms_helper_poll_init(drm);
+
+	priv->fbdev = drm_fbdev_cma_init(drm, 32, drm->mode_config.num_crtc,
+					 drm->mode_config.num_connector);
+	if (IS_ERR(priv->fbdev)) {
+		ret = PTR_ERR(priv->fbdev);
+		DRM_DEV_ERROR(dev, "failed to init cma fbdev: %d\n", ret);
+		priv->fbdev = NULL;
+		goto out_poll_fini;
+	}
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		goto out_fbdev_fini;
+
+	return 0;
+
+out_fbdev_fini:
+	if (priv->fbdev) {
+		drm_fbdev_cma_fini(priv->fbdev);
+		priv->fbdev = NULL;
+	}
+out_poll_fini:
+	drm_kms_helper_poll_fini(drm);
+	drm_mode_config_cleanup(drm);
+	drm_vblank_cleanup(drm);
+out_unbind:
+	component_unbind_all(dev, drm);
+out_unregister:
+	dev_set_drvdata(dev, NULL);
+	drm->dev_private = NULL;
+	drm_dev_unref(drm);
+	return ret;
+}
+
+static void zx_drm_unbind(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct zx_drm_private *priv = drm->dev_private;
+
+	drm_dev_unregister(drm);
+	if (priv->fbdev) {
+		drm_fbdev_cma_fini(priv->fbdev);
+		priv->fbdev = NULL;
+	}
+	drm_kms_helper_poll_fini(drm);
+	drm_mode_config_cleanup(drm);
+	drm_vblank_cleanup(drm);
+	component_unbind_all(dev, drm);
+	dev_set_drvdata(dev, NULL);
+	drm->dev_private = NULL;
+	drm_dev_unref(drm);
+}
+
+static const struct component_master_ops zx_drm_master_ops = {
+	.bind = zx_drm_bind,
+	.unbind = zx_drm_unbind,
+};
+
+static int compare_of(struct device *dev, void *data)
+{
+	return dev->of_node == data;
+}
+
+static int zx_drm_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *parent = dev->of_node;
+	struct device_node *child;
+	struct component_match *match = NULL;
+	int ret;
+
+	ret = of_platform_populate(parent, NULL, NULL, dev);
+	if (ret)
+		return ret;
+
+	for_each_available_child_of_node(parent, child) {
+		component_match_add(dev, &match, compare_of, child);
+		of_node_put(child);
+	}
+
+	return component_master_add_with_match(dev, &zx_drm_master_ops, match);
+}
+
+static int zx_drm_remove(struct platform_device *pdev)
+{
+	component_master_del(&pdev->dev, &zx_drm_master_ops);
+	return 0;
+}
+
+static const struct of_device_id zx_drm_of_match[] = {
+	{ .compatible = "zte,zx296718-vou", },
+	{ /* end */ },
+};
+MODULE_DEVICE_TABLE(of, zx_drm_of_match);
+
+static struct platform_driver zx_drm_platform_driver = {
+	.probe = zx_drm_probe,
+	.remove = zx_drm_remove,
+	.driver	= {
+		.name = "zx-drm",
+		.of_match_table	= zx_drm_of_match,
+	},
+};
+
+static struct platform_driver *drivers[] = {
+	&zx_crtc_driver,
+	&zx_hdmi_driver,
+	&zx_drm_platform_driver,
+};
+
+static int zx_drm_init(void)
+{
+	return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
+}
+module_init(zx_drm_init);
+
+static void zx_drm_exit(void)
+{
+	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
+}
+module_exit(zx_drm_exit);
+
+MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>");
+MODULE_DESCRIPTION("ZTE ZX VOU DRM driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/zte/zx_drm_drv.h b/drivers/gpu/drm/zte/zx_drm_drv.h
new file mode 100644
index 000000000000..e65cd18a6cba
--- /dev/null
+++ b/drivers/gpu/drm/zte/zx_drm_drv.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2016 Linaro Ltd.
+ * Copyright 2016 ZTE Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __ZX_DRM_DRV_H__
+#define __ZX_DRM_DRV_H__
+
+extern struct platform_driver zx_crtc_driver;
+extern struct platform_driver zx_hdmi_driver;
+
+static inline u32 zx_readl(void __iomem *reg)
+{
+	return readl_relaxed(reg);
+}
+
+static inline void zx_writel(void __iomem *reg, u32 val)
+{
+	writel_relaxed(val, reg);
+}
+
+static inline void zx_writel_mask(void __iomem *reg, u32 mask, u32 val)
+{
+	u32 tmp;
+
+	tmp = zx_readl(reg);
+	tmp = (tmp & ~mask) | (val & mask);
+	zx_writel(reg, tmp);
+}
+
+#endif /* __ZX_DRM_DRV_H__ */
diff --git a/drivers/gpu/drm/zte/zx_hdmi.c b/drivers/gpu/drm/zte/zx_hdmi.c
new file mode 100644
index 000000000000..6bf6c364811e
--- /dev/null
+++ b/drivers/gpu/drm/zte/zx_hdmi.c
@@ -0,0 +1,624 @@
+/*
+ * Copyright 2016 Linaro Ltd.
+ * Copyright 2016 ZTE Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/hdmi.h>
+#include <linux/irq.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_device.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_of.h>
+#include <drm/drmP.h>
+
+#include "zx_hdmi_regs.h"
+#include "zx_vou.h"
+
+#define ZX_HDMI_INFOFRAME_SIZE		31
+#define DDC_SEGMENT_ADDR		0x30
+
+struct zx_hdmi_i2c {
+	struct i2c_adapter adap;
+	struct mutex lock;
+};
+
+struct zx_hdmi {
+	struct drm_connector connector;
+	struct drm_encoder encoder;
+	struct zx_hdmi_i2c *ddc;
+	struct device *dev;
+	struct drm_device *drm;
+	void __iomem *mmio;
+	struct clk *cec_clk;
+	struct clk *osc_clk;
+	struct clk *xclk;
+	bool sink_is_hdmi;
+	bool sink_has_audio;
+	const struct vou_inf *inf;
+};
+
+#define to_zx_hdmi(x) container_of(x, struct zx_hdmi, x)
+
+static const struct vou_inf vou_inf_hdmi = {
+	.id = VOU_HDMI,
+	.data_sel = VOU_YUV444,
+	.clocks_en_bits = BIT(24) | BIT(18) | BIT(6),
+	.clocks_sel_bits = BIT(13) | BIT(2),
+};
+
+static inline u8 hdmi_readb(struct zx_hdmi *hdmi, u16 offset)
+{
+	return readl_relaxed(hdmi->mmio + offset * 4);
+}
+
+static inline void hdmi_writeb(struct zx_hdmi *hdmi, u16 offset, u8 val)
+{
+	writel_relaxed(val, hdmi->mmio + offset * 4);
+}
+
+static inline void hdmi_writeb_mask(struct zx_hdmi *hdmi, u16 offset,
+				    u8 mask, u8 val)
+{
+	u8 tmp;
+
+	tmp = hdmi_readb(hdmi, offset);
+	tmp = (tmp & ~mask) | (val & mask);
+	hdmi_writeb(hdmi, offset, tmp);
+}
+
+static int zx_hdmi_infoframe_trans(struct zx_hdmi *hdmi,
+				   union hdmi_infoframe *frame, u8 fsel)
+{
+	u8 buffer[ZX_HDMI_INFOFRAME_SIZE];
+	int num;
+	int i;
+
+	hdmi_writeb(hdmi, TPI_INFO_FSEL, fsel);
+
+	num = hdmi_infoframe_pack(frame, buffer, ZX_HDMI_INFOFRAME_SIZE);
+	if (num < 0) {
+		DRM_DEV_ERROR(hdmi->dev, "failed to pack infoframe: %d\n", num);
+		return num;
+	}
+
+	for (i = 0; i < num; i++)
+		hdmi_writeb(hdmi, TPI_INFO_B0 + i, buffer[i]);
+
+	hdmi_writeb_mask(hdmi, TPI_INFO_EN, TPI_INFO_TRANS_RPT,
+			 TPI_INFO_TRANS_RPT);
+	hdmi_writeb_mask(hdmi, TPI_INFO_EN, TPI_INFO_TRANS_EN,
+			 TPI_INFO_TRANS_EN);
+
+	return num;
+}
+
+static int zx_hdmi_config_video_vsi(struct zx_hdmi *hdmi,
+				    struct drm_display_mode *mode)
+{
+	union hdmi_infoframe frame;
+	int ret;
+
+	ret = drm_hdmi_vendor_infoframe_from_display_mode(&frame.vendor.hdmi,
+							  mode);
+	if (ret) {
+		DRM_DEV_ERROR(hdmi->dev, "failed to get vendor infoframe: %d\n",
+			      ret);
+		return ret;
+	}
+
+	return zx_hdmi_infoframe_trans(hdmi, &frame, FSEL_VSIF);
+}
+
+static int zx_hdmi_config_video_avi(struct zx_hdmi *hdmi,
+				    struct drm_display_mode *mode)
+{
+	union hdmi_infoframe frame;
+	int ret;
+
+	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode);
+	if (ret) {
+		DRM_DEV_ERROR(hdmi->dev, "failed to get avi infoframe: %d\n",
+			      ret);
+		return ret;
+	}
+
+	/* We always use YUV444 for HDMI output. */
+	frame.avi.colorspace = HDMI_COLORSPACE_YUV444;
+
+	return zx_hdmi_infoframe_trans(hdmi, &frame, FSEL_AVI);
+}
+
+static void zx_hdmi_encoder_mode_set(struct drm_encoder *encoder,
+				     struct drm_display_mode *mode,
+				     struct drm_display_mode *adj_mode)
+{
+	struct zx_hdmi *hdmi = to_zx_hdmi(encoder);
+
+	if (hdmi->sink_is_hdmi) {
+		zx_hdmi_config_video_avi(hdmi, mode);
+		zx_hdmi_config_video_vsi(hdmi, mode);
+	}
+}
+
+static void zx_hdmi_phy_start(struct zx_hdmi *hdmi)
+{
+	/* Copy from ZTE BSP code */
+	hdmi_writeb(hdmi, 0x222, 0x0);
+	hdmi_writeb(hdmi, 0x224, 0x4);
+	hdmi_writeb(hdmi, 0x909, 0x0);
+	hdmi_writeb(hdmi, 0x7b0, 0x90);
+	hdmi_writeb(hdmi, 0x7b1, 0x00);
+	hdmi_writeb(hdmi, 0x7b2, 0xa7);
+	hdmi_writeb(hdmi, 0x7b8, 0xaa);
+	hdmi_writeb(hdmi, 0x7b2, 0xa7);
+	hdmi_writeb(hdmi, 0x7b3, 0x0f);
+	hdmi_writeb(hdmi, 0x7b4, 0x0f);
+	hdmi_writeb(hdmi, 0x7b5, 0x55);
+	hdmi_writeb(hdmi, 0x7b7, 0x03);
+	hdmi_writeb(hdmi, 0x7b9, 0x12);
+	hdmi_writeb(hdmi, 0x7ba, 0x32);
+	hdmi_writeb(hdmi, 0x7bc, 0x68);
+	hdmi_writeb(hdmi, 0x7be, 0x40);
+	hdmi_writeb(hdmi, 0x7bf, 0x84);
+	hdmi_writeb(hdmi, 0x7c1, 0x0f);
+	hdmi_writeb(hdmi, 0x7c8, 0x02);
+	hdmi_writeb(hdmi, 0x7c9, 0x03);
+	hdmi_writeb(hdmi, 0x7ca, 0x40);
+	hdmi_writeb(hdmi, 0x7dc, 0x31);
+	hdmi_writeb(hdmi, 0x7e2, 0x04);
+	hdmi_writeb(hdmi, 0x7e0, 0x06);
+	hdmi_writeb(hdmi, 0x7cb, 0x68);
+	hdmi_writeb(hdmi, 0x7f9, 0x02);
+	hdmi_writeb(hdmi, 0x7b6, 0x02);
+	hdmi_writeb(hdmi, 0x7f3, 0x0);
+}
+
+static void zx_hdmi_hw_enable(struct zx_hdmi *hdmi)
+{
+	/* Enable pclk */
+	hdmi_writeb_mask(hdmi, CLKPWD, CLKPWD_PDIDCK, CLKPWD_PDIDCK);
+
+	/* Enable HDMI for TX */
+	hdmi_writeb_mask(hdmi, FUNC_SEL, FUNC_HDMI_EN, FUNC_HDMI_EN);
+
+	/* Enable deep color packet */
+	hdmi_writeb_mask(hdmi, P2T_CTRL, P2T_DC_PKT_EN, P2T_DC_PKT_EN);
+
+	/* Enable HDMI/MHL mode for output */
+	hdmi_writeb_mask(hdmi, TEST_TXCTRL, TEST_TXCTRL_HDMI_MODE,
+			 TEST_TXCTRL_HDMI_MODE);
+
+	/* Configure reg_qc_sel */
+	hdmi_writeb(hdmi, HDMICTL4, 0x3);
+
+	/* Enable interrupt */
+	hdmi_writeb_mask(hdmi, INTR1_MASK, INTR1_MONITOR_DETECT,
+			 INTR1_MONITOR_DETECT);
+
+	/* Start up phy */
+	zx_hdmi_phy_start(hdmi);
+}
+
+static void zx_hdmi_hw_disable(struct zx_hdmi *hdmi)
+{
+	/* Disable interrupt */
+	hdmi_writeb_mask(hdmi, INTR1_MASK, INTR1_MONITOR_DETECT, 0);
+
+	/* Disable deep color packet */
+	hdmi_writeb_mask(hdmi, P2T_CTRL, P2T_DC_PKT_EN, P2T_DC_PKT_EN);
+
+	/* Disable HDMI for TX */
+	hdmi_writeb_mask(hdmi, FUNC_SEL, FUNC_HDMI_EN, 0);
+
+	/* Disable pclk */
+	hdmi_writeb_mask(hdmi, CLKPWD, CLKPWD_PDIDCK, 0);
+}
+
+static void zx_hdmi_encoder_enable(struct drm_encoder *encoder)
+{
+	struct zx_hdmi *hdmi = to_zx_hdmi(encoder);
+
+	clk_prepare_enable(hdmi->cec_clk);
+	clk_prepare_enable(hdmi->osc_clk);
+	clk_prepare_enable(hdmi->xclk);
+
+	zx_hdmi_hw_enable(hdmi);
+
+	vou_inf_enable(hdmi->inf, encoder->crtc);
+}
+
+static void zx_hdmi_encoder_disable(struct drm_encoder *encoder)
+{
+	struct zx_hdmi *hdmi = to_zx_hdmi(encoder);
+
+	vou_inf_disable(hdmi->inf, encoder->crtc);
+
+	zx_hdmi_hw_disable(hdmi);
+
+	clk_disable_unprepare(hdmi->xclk);
+	clk_disable_unprepare(hdmi->osc_clk);
+	clk_disable_unprepare(hdmi->cec_clk);
+}
+
+static const struct drm_encoder_helper_funcs zx_hdmi_encoder_helper_funcs = {
+	.enable	= zx_hdmi_encoder_enable,
+	.disable = zx_hdmi_encoder_disable,
+	.mode_set = zx_hdmi_encoder_mode_set,
+};
+
+static const struct drm_encoder_funcs zx_hdmi_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
+static int zx_hdmi_connector_get_modes(struct drm_connector *connector)
+{
+	struct zx_hdmi *hdmi = to_zx_hdmi(connector);
+	struct edid *edid;
+	int ret;
+
+	edid = drm_get_edid(connector, &hdmi->ddc->adap);
+	if (!edid)
+		return 0;
+
+	hdmi->sink_is_hdmi = drm_detect_hdmi_monitor(edid);
+	hdmi->sink_has_audio = drm_detect_monitor_audio(edid);
+	drm_mode_connector_update_edid_property(connector, edid);
+	ret = drm_add_edid_modes(connector, edid);
+	kfree(edid);
+
+	return ret;
+}
+
+static enum drm_mode_status
+zx_hdmi_connector_mode_valid(struct drm_connector *connector,
+			     struct drm_display_mode *mode)
+{
+	return MODE_OK;
+}
+
+static struct drm_connector_helper_funcs zx_hdmi_connector_helper_funcs = {
+	.get_modes = zx_hdmi_connector_get_modes,
+	.mode_valid = zx_hdmi_connector_mode_valid,
+};
+
+static enum drm_connector_status
+zx_hdmi_connector_detect(struct drm_connector *connector, bool force)
+{
+	struct zx_hdmi *hdmi = to_zx_hdmi(connector);
+
+	return (hdmi_readb(hdmi, TPI_HPD_RSEN) & TPI_HPD_CONNECTION) ?
+		connector_status_connected : connector_status_disconnected;
+}
+
+static const struct drm_connector_funcs zx_hdmi_connector_funcs = {
+	.dpms = drm_atomic_helper_connector_dpms,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.detect = zx_hdmi_connector_detect,
+	.destroy = drm_connector_cleanup,
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static int zx_hdmi_register(struct drm_device *drm, struct zx_hdmi *hdmi)
+{
+	struct drm_encoder *encoder = &hdmi->encoder;
+
+	encoder->possible_crtcs = VOU_CRTC_MASK;
+
+	drm_encoder_init(drm, encoder, &zx_hdmi_encoder_funcs,
+			 DRM_MODE_ENCODER_TMDS, NULL);
+	drm_encoder_helper_add(encoder, &zx_hdmi_encoder_helper_funcs);
+
+	hdmi->connector.polled = DRM_CONNECTOR_POLL_HPD;
+
+	drm_connector_init(drm, &hdmi->connector, &zx_hdmi_connector_funcs,
+			   DRM_MODE_CONNECTOR_HDMIA);
+	drm_connector_helper_add(&hdmi->connector,
+				 &zx_hdmi_connector_helper_funcs);
+
+	drm_mode_connector_attach_encoder(&hdmi->connector, encoder);
+
+	return 0;
+}
+
+static irqreturn_t zx_hdmi_irq_thread(int irq, void *dev_id)
+{
+	struct zx_hdmi *hdmi = dev_id;
+
+	drm_helper_hpd_irq_event(hdmi->connector.dev);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t zx_hdmi_irq_handler(int irq, void *dev_id)
+{
+	struct zx_hdmi *hdmi = dev_id;
+	u8 lstat;
+
+	lstat = hdmi_readb(hdmi, L1_INTR_STAT);
+
+	/* Monitor detect/HPD interrupt */
+	if (lstat & L1_INTR_STAT_INTR1) {
+		u8 stat;
+
+		stat = hdmi_readb(hdmi, INTR1_STAT);
+		hdmi_writeb(hdmi, INTR1_STAT, stat);
+
+		if (stat & INTR1_MONITOR_DETECT)
+			return IRQ_WAKE_THREAD;
+	}
+
+	return IRQ_NONE;
+}
+
+static int zx_hdmi_i2c_read(struct zx_hdmi *hdmi, struct i2c_msg *msg)
+{
+	int len = msg->len;
+	u8 *buf = msg->buf;
+	int retry = 0;
+	int ret = 0;
+
+	/* Bits [9:8] of bytes */
+	hdmi_writeb(hdmi, ZX_DDC_DIN_CNT2, (len >> 8) & 0xff);
+	/* Bits [7:0] of bytes */
+	hdmi_writeb(hdmi, ZX_DDC_DIN_CNT1, len & 0xff);
+
+	/* Clear FIFO */
+	hdmi_writeb_mask(hdmi, ZX_DDC_CMD, DDC_CMD_MASK, DDC_CMD_CLEAR_FIFO);
+
+	/* Kick off the read */
+	hdmi_writeb_mask(hdmi, ZX_DDC_CMD, DDC_CMD_MASK,
+			 DDC_CMD_SEQUENTIAL_READ);
+
+	while (len > 0) {
+		int cnt, i;
+
+		/* FIFO needs some time to get ready */
+		usleep_range(500, 1000);
+
+		cnt = hdmi_readb(hdmi, ZX_DDC_DOUT_CNT) & DDC_DOUT_CNT_MASK;
+		if (cnt == 0) {
+			if (++retry > 5) {
+				DRM_DEV_ERROR(hdmi->dev,
+					      "DDC FIFO read timed out!");
+				return -ETIMEDOUT;
+			}
+			continue;
+		}
+
+		for (i = 0; i < cnt; i++)
+			*buf++ = hdmi_readb(hdmi, ZX_DDC_DATA);
+		len -= cnt;
+	}
+
+	return ret;
+}
+
+static int zx_hdmi_i2c_write(struct zx_hdmi *hdmi, struct i2c_msg *msg)
+{
+	/*
+	 * The DDC I2C adapter is only for reading EDID data, so we assume
+	 * that the write to this adapter must be the EDID data offset.
+	 */
+	if ((msg->len != 1) ||
+	    ((msg->addr != DDC_ADDR) && (msg->addr != DDC_SEGMENT_ADDR)))
+		return -EINVAL;
+
+	if (msg->addr == DDC_SEGMENT_ADDR)
+		hdmi_writeb(hdmi, ZX_DDC_SEGM, msg->addr << 1);
+	else if (msg->addr == DDC_ADDR)
+		hdmi_writeb(hdmi, ZX_DDC_ADDR, msg->addr << 1);
+
+	hdmi_writeb(hdmi, ZX_DDC_OFFSET, msg->buf[0]);
+
+	return 0;
+}
+
+static int zx_hdmi_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
+			    int num)
+{
+	struct zx_hdmi *hdmi = i2c_get_adapdata(adap);
+	struct zx_hdmi_i2c *ddc = hdmi->ddc;
+	int i, ret = 0;
+
+	mutex_lock(&ddc->lock);
+
+	/* Enable DDC master access */
+	hdmi_writeb_mask(hdmi, TPI_DDC_MASTER_EN, HW_DDC_MASTER, HW_DDC_MASTER);
+
+	for (i = 0; i < num; i++) {
+		DRM_DEV_DEBUG(hdmi->dev,
+			      "xfer: num: %d/%d, len: %d, flags: %#x\n",
+			      i + 1, num, msgs[i].len, msgs[i].flags);
+
+		if (msgs[i].flags & I2C_M_RD)
+			ret = zx_hdmi_i2c_read(hdmi, &msgs[i]);
+		else
+			ret = zx_hdmi_i2c_write(hdmi, &msgs[i]);
+
+		if (ret < 0)
+			break;
+	}
+
+	if (!ret)
+		ret = num;
+
+	/* Disable DDC master access */
+	hdmi_writeb_mask(hdmi, TPI_DDC_MASTER_EN, HW_DDC_MASTER, 0);
+
+	mutex_unlock(&ddc->lock);
+
+	return ret;
+}
+
+static u32 zx_hdmi_i2c_func(struct i2c_adapter *adapter)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm zx_hdmi_algorithm = {
+	.master_xfer	= zx_hdmi_i2c_xfer,
+	.functionality	= zx_hdmi_i2c_func,
+};
+
+static int zx_hdmi_ddc_register(struct zx_hdmi *hdmi)
+{
+	struct i2c_adapter *adap;
+	struct zx_hdmi_i2c *ddc;
+	int ret;
+
+	ddc = devm_kzalloc(hdmi->dev, sizeof(*ddc), GFP_KERNEL);
+	if (!ddc)
+		return -ENOMEM;
+
+	hdmi->ddc = ddc;
+	mutex_init(&ddc->lock);
+
+	adap = &ddc->adap;
+	adap->owner = THIS_MODULE;
+	adap->class = I2C_CLASS_DDC;
+	adap->dev.parent = hdmi->dev;
+	adap->algo = &zx_hdmi_algorithm;
+	snprintf(adap->name, sizeof(adap->name), "zx hdmi i2c");
+
+	ret = i2c_add_adapter(adap);
+	if (ret) {
+		DRM_DEV_ERROR(hdmi->dev, "failed to add I2C adapter: %d\n",
+			      ret);
+		return ret;
+	}
+
+	i2c_set_adapdata(adap, hdmi);
+
+	return 0;
+}
+
+static int zx_hdmi_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct drm_device *drm = data;
+	struct resource *res;
+	struct zx_hdmi *hdmi;
+	int irq;
+	int ret;
+
+	hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL);
+	if (!hdmi)
+		return -ENOMEM;
+
+	hdmi->dev = dev;
+	hdmi->drm = drm;
+	hdmi->inf = &vou_inf_hdmi;
+
+	dev_set_drvdata(dev, hdmi);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	hdmi->mmio = devm_ioremap_resource(dev, res);
+	if (IS_ERR(hdmi->mmio)) {
+		ret = PTR_ERR(hdmi->mmio);
+		DRM_DEV_ERROR(dev, "failed to remap hdmi region: %d\n", ret);
+		return ret;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	hdmi->cec_clk = devm_clk_get(hdmi->dev, "osc_cec");
+	if (IS_ERR(hdmi->cec_clk)) {
+		ret = PTR_ERR(hdmi->cec_clk);
+		DRM_DEV_ERROR(dev, "failed to get cec_clk: %d\n", ret);
+		return ret;
+	}
+
+	hdmi->osc_clk = devm_clk_get(hdmi->dev, "osc_clk");
+	if (IS_ERR(hdmi->osc_clk)) {
+		ret = PTR_ERR(hdmi->osc_clk);
+		DRM_DEV_ERROR(dev, "failed to get osc_clk: %d\n", ret);
+		return ret;
+	}
+
+	hdmi->xclk = devm_clk_get(hdmi->dev, "xclk");
+	if (IS_ERR(hdmi->xclk)) {
+		ret = PTR_ERR(hdmi->xclk);
+		DRM_DEV_ERROR(dev, "failed to get xclk: %d\n", ret);
+		return ret;
+	}
+
+	ret = zx_hdmi_ddc_register(hdmi);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "failed to register ddc: %d\n", ret);
+		return ret;
+	}
+
+	ret = zx_hdmi_register(drm, hdmi);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "failed to register hdmi: %d\n", ret);
+		return ret;
+	}
+
+	ret = devm_request_threaded_irq(dev, irq, zx_hdmi_irq_handler,
+					zx_hdmi_irq_thread, IRQF_SHARED,
+					dev_name(dev), hdmi);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "failed to request threaded irq: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void zx_hdmi_unbind(struct device *dev, struct device *master,
+			   void *data)
+{
+	struct zx_hdmi *hdmi = dev_get_drvdata(dev);
+
+	hdmi->connector.funcs->destroy(&hdmi->connector);
+	hdmi->encoder.funcs->destroy(&hdmi->encoder);
+}
+
+static const struct component_ops zx_hdmi_component_ops = {
+	.bind = zx_hdmi_bind,
+	.unbind = zx_hdmi_unbind,
+};
+
+static int zx_hdmi_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &zx_hdmi_component_ops);
+}
+
+static int zx_hdmi_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &zx_hdmi_component_ops);
+	return 0;
+}
+
+static const struct of_device_id zx_hdmi_of_match[] = {
+	{ .compatible = "zte,zx296718-hdmi", },
+	{ /* end */ },
+};
+MODULE_DEVICE_TABLE(of, zx_hdmi_of_match);
+
+struct platform_driver zx_hdmi_driver = {
+	.probe = zx_hdmi_probe,
+	.remove = zx_hdmi_remove,
+	.driver	= {
+		.name = "zx-hdmi",
+		.of_match_table	= zx_hdmi_of_match,
+	},
+};
diff --git a/drivers/gpu/drm/zte/zx_hdmi_regs.h b/drivers/gpu/drm/zte/zx_hdmi_regs.h
new file mode 100644
index 000000000000..de911f66b658
--- /dev/null
+++ b/drivers/gpu/drm/zte/zx_hdmi_regs.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2016 Linaro Ltd.
+ * Copyright 2016 ZTE Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __ZX_HDMI_REGS_H__
+#define __ZX_HDMI_REGS_H__
+
+#define FUNC_SEL			0x000b
+#define FUNC_HDMI_EN			BIT(0)
+#define CLKPWD				0x000d
+#define CLKPWD_PDIDCK			BIT(2)
+#define P2T_CTRL			0x0066
+#define P2T_DC_PKT_EN			BIT(7)
+#define L1_INTR_STAT			0x007e
+#define L1_INTR_STAT_INTR1		BIT(0)
+#define INTR1_STAT			0x008f
+#define INTR1_MASK			0x0095
+#define INTR1_MONITOR_DETECT		(BIT(5) | BIT(6))
+#define ZX_DDC_ADDR			0x00ed
+#define ZX_DDC_SEGM			0x00ee
+#define ZX_DDC_OFFSET			0x00ef
+#define ZX_DDC_DIN_CNT1			0x00f0
+#define ZX_DDC_DIN_CNT2			0x00f1
+#define ZX_DDC_CMD			0x00f3
+#define DDC_CMD_MASK			0xf
+#define DDC_CMD_CLEAR_FIFO		0x9
+#define DDC_CMD_SEQUENTIAL_READ		0x2
+#define ZX_DDC_DATA			0x00f4
+#define ZX_DDC_DOUT_CNT			0x00f5
+#define DDC_DOUT_CNT_MASK		0x1f
+#define TEST_TXCTRL			0x00f7
+#define TEST_TXCTRL_HDMI_MODE		BIT(1)
+#define HDMICTL4			0x0235
+#define TPI_HPD_RSEN			0x063b
+#define TPI_HPD_CONNECTION		(BIT(1) | BIT(2))
+#define TPI_INFO_FSEL			0x06bf
+#define FSEL_AVI			0
+#define FSEL_GBD			1
+#define FSEL_AUDIO			2
+#define FSEL_SPD			3
+#define FSEL_MPEG			4
+#define FSEL_VSIF			5
+#define TPI_INFO_B0			0x06c0
+#define TPI_INFO_EN			0x06df
+#define TPI_INFO_TRANS_EN		BIT(7)
+#define TPI_INFO_TRANS_RPT		BIT(6)
+#define TPI_DDC_MASTER_EN		0x06f8
+#define HW_DDC_MASTER			BIT(7)
+
+#endif /* __ZX_HDMI_REGS_H__ */
diff --git a/drivers/gpu/drm/zte/zx_plane.c b/drivers/gpu/drm/zte/zx_plane.c
new file mode 100644
index 000000000000..546eb92a94e8
--- /dev/null
+++ b/drivers/gpu/drm/zte/zx_plane.c
@@ -0,0 +1,299 @@
+/*
+ * Copyright 2016 Linaro Ltd.
+ * Copyright 2016 ZTE Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drmP.h>
+
+#include "zx_drm_drv.h"
+#include "zx_plane.h"
+#include "zx_plane_regs.h"
+#include "zx_vou.h"
+
+struct zx_plane {
+	struct drm_plane plane;
+	void __iomem *layer;
+	void __iomem *csc;
+	void __iomem *hbsc;
+	void __iomem *rsz;
+};
+
+#define to_zx_plane(plane)	container_of(plane, struct zx_plane, plane)
+
+static const uint32_t gl_formats[] = {
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_ARGB4444,
+};
+
+static int zx_gl_plane_atomic_check(struct drm_plane *plane,
+				    struct drm_plane_state *plane_state)
+{
+	struct drm_framebuffer *fb = plane_state->fb;
+	struct drm_crtc *crtc = plane_state->crtc;
+	struct drm_crtc_state *crtc_state;
+	struct drm_rect clip;
+
+	if (!crtc || !fb)
+		return 0;
+
+	crtc_state = drm_atomic_get_existing_crtc_state(plane_state->state,
+							crtc);
+	if (WARN_ON(!crtc_state))
+		return -EINVAL;
+
+	/* nothing to check when disabling or disabled */
+	if (!crtc_state->enable)
+		return 0;
+
+	/* plane must be enabled */
+	if (!plane_state->crtc)
+		return -EINVAL;
+
+	clip.x1 = 0;
+	clip.y1 = 0;
+	clip.x2 = crtc_state->adjusted_mode.hdisplay;
+	clip.y2 = crtc_state->adjusted_mode.vdisplay;
+
+	return drm_plane_helper_check_state(plane_state, &clip,
+					    DRM_PLANE_HELPER_NO_SCALING,
+					    DRM_PLANE_HELPER_NO_SCALING,
+					    false, true);
+}
+
+static int zx_gl_get_fmt(uint32_t format)
+{
+	switch (format) {
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_XRGB8888:
+		return GL_FMT_ARGB8888;
+	case DRM_FORMAT_RGB888:
+		return GL_FMT_RGB888;
+	case DRM_FORMAT_RGB565:
+		return GL_FMT_RGB565;
+	case DRM_FORMAT_ARGB1555:
+		return GL_FMT_ARGB1555;
+	case DRM_FORMAT_ARGB4444:
+		return GL_FMT_ARGB4444;
+	default:
+		WARN_ONCE(1, "invalid pixel format %d\n", format);
+		return -EINVAL;
+	}
+}
+
+static inline void zx_gl_set_update(struct zx_plane *zplane)
+{
+	void __iomem *layer = zplane->layer;
+
+	zx_writel_mask(layer + GL_CTRL0, GL_UPDATE, GL_UPDATE);
+}
+
+static inline void zx_gl_rsz_set_update(struct zx_plane *zplane)
+{
+	zx_writel(zplane->rsz + RSZ_ENABLE_CFG, 1);
+}
+
+void zx_plane_set_update(struct drm_plane *plane)
+{
+	struct zx_plane *zplane = to_zx_plane(plane);
+
+	zx_gl_rsz_set_update(zplane);
+	zx_gl_set_update(zplane);
+}
+
+static void zx_gl_rsz_setup(struct zx_plane *zplane, u32 src_w, u32 src_h,
+			    u32 dst_w, u32 dst_h)
+{
+	void __iomem *rsz = zplane->rsz;
+
+	zx_writel(rsz + RSZ_SRC_CFG, RSZ_VER(src_h - 1) | RSZ_HOR(src_w - 1));
+	zx_writel(rsz + RSZ_DEST_CFG, RSZ_VER(dst_h - 1) | RSZ_HOR(dst_w - 1));
+
+	zx_gl_rsz_set_update(zplane);
+}
+
+static void zx_gl_plane_atomic_update(struct drm_plane *plane,
+				      struct drm_plane_state *old_state)
+{
+	struct zx_plane *zplane = to_zx_plane(plane);
+	struct drm_framebuffer *fb = plane->state->fb;
+	struct drm_gem_cma_object *cma_obj;
+	void __iomem *layer = zplane->layer;
+	void __iomem *csc = zplane->csc;
+	void __iomem *hbsc = zplane->hbsc;
+	u32 src_x, src_y, src_w, src_h;
+	u32 dst_x, dst_y, dst_w, dst_h;
+	unsigned int bpp;
+	uint32_t format;
+	dma_addr_t paddr;
+	u32 stride;
+	int fmt;
+
+	if (!fb)
+		return;
+
+	format = fb->pixel_format;
+	stride = fb->pitches[0];
+
+	src_x = plane->state->src_x >> 16;
+	src_y = plane->state->src_y >> 16;
+	src_w = plane->state->src_w >> 16;
+	src_h = plane->state->src_h >> 16;
+
+	dst_x = plane->state->crtc_x;
+	dst_y = plane->state->crtc_y;
+	dst_w = plane->state->crtc_w;
+	dst_h = plane->state->crtc_h;
+
+	bpp = drm_format_plane_cpp(format, 0);
+
+	cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
+	paddr = cma_obj->paddr + fb->offsets[0];
+	paddr += src_y * stride + src_x * bpp / 8;
+	zx_writel(layer + GL_ADDR, paddr);
+
+	/* Set up source height/width register */
+	zx_writel(layer + GL_SRC_SIZE, GL_SRC_W(src_w) | GL_SRC_H(src_h));
+
+	/* Set up start position register */
+	zx_writel(layer + GL_POS_START, GL_POS_X(dst_x) | GL_POS_Y(dst_y));
+
+	/* Set up end position register */
+	zx_writel(layer + GL_POS_END,
+		  GL_POS_X(dst_x + dst_w) | GL_POS_Y(dst_y + dst_h));
+
+	/* Set up stride register */
+	zx_writel(layer + GL_STRIDE, stride & 0xffff);
+
+	/* Set up graphic layer data format */
+	fmt = zx_gl_get_fmt(format);
+	if (fmt >= 0)
+		zx_writel_mask(layer + GL_CTRL1, GL_DATA_FMT_MASK,
+			       fmt << GL_DATA_FMT_SHIFT);
+
+	/* Initialize global alpha with a sane value */
+	zx_writel_mask(layer + GL_CTRL2, GL_GLOBAL_ALPHA_MASK,
+		       0xff << GL_GLOBAL_ALPHA_SHIFT);
+
+	/* Setup CSC for the GL */
+	if (dst_h > 720)
+		zx_writel_mask(csc + CSC_CTRL0, CSC_COV_MODE_MASK,
+			       CSC_BT709_IMAGE_RGB2YCBCR << CSC_COV_MODE_SHIFT);
+	else
+		zx_writel_mask(csc + CSC_CTRL0, CSC_COV_MODE_MASK,
+			       CSC_BT601_IMAGE_RGB2YCBCR << CSC_COV_MODE_SHIFT);
+	zx_writel_mask(csc + CSC_CTRL0, CSC_WORK_ENABLE, CSC_WORK_ENABLE);
+
+	/* Always use scaler since it exists (set for not bypass) */
+	zx_writel_mask(layer + GL_CTRL3, GL_SCALER_BYPASS_MODE,
+		       GL_SCALER_BYPASS_MODE);
+
+	zx_gl_rsz_setup(zplane, src_w, src_h, dst_w, dst_h);
+
+	/* Enable HBSC block */
+	zx_writel_mask(hbsc + HBSC_CTRL0, HBSC_CTRL_EN, HBSC_CTRL_EN);
+
+	zx_gl_set_update(zplane);
+}
+
+static const struct drm_plane_helper_funcs zx_gl_plane_helper_funcs = {
+	.atomic_check = zx_gl_plane_atomic_check,
+	.atomic_update = zx_gl_plane_atomic_update,
+};
+
+static void zx_plane_destroy(struct drm_plane *plane)
+{
+	drm_plane_helper_disable(plane);
+	drm_plane_cleanup(plane);
+}
+
+static const struct drm_plane_funcs zx_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.destroy = zx_plane_destroy,
+	.reset = drm_atomic_helper_plane_reset,
+	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+};
+
+static void zx_plane_hbsc_init(struct zx_plane *zplane)
+{
+	void __iomem *hbsc = zplane->hbsc;
+
+	/*
+	 *  Initialize HBSC block with a sane configuration per recommedation
+	 *  from ZTE BSP code.
+	 */
+	zx_writel(hbsc + HBSC_SATURATION, 0x200);
+	zx_writel(hbsc + HBSC_HUE, 0x0);
+	zx_writel(hbsc + HBSC_BRIGHT, 0x0);
+	zx_writel(hbsc + HBSC_CONTRAST, 0x200);
+
+	zx_writel(hbsc + HBSC_THRESHOLD_COL1, (0x3ac << 16) | 0x40);
+	zx_writel(hbsc + HBSC_THRESHOLD_COL2, (0x3c0 << 16) | 0x40);
+	zx_writel(hbsc + HBSC_THRESHOLD_COL3, (0x3c0 << 16) | 0x40);
+}
+
+struct drm_plane *zx_plane_init(struct drm_device *drm, struct device *dev,
+				struct zx_layer_data *data,
+				enum drm_plane_type type)
+{
+	const struct drm_plane_helper_funcs *helper;
+	struct zx_plane *zplane;
+	struct drm_plane *plane;
+	const uint32_t *formats;
+	unsigned int format_count;
+	int ret;
+
+	zplane = devm_kzalloc(dev, sizeof(*zplane), GFP_KERNEL);
+	if (!zplane)
+		return ERR_PTR(-ENOMEM);
+
+	plane = &zplane->plane;
+
+	zplane->layer = data->layer;
+	zplane->hbsc = data->hbsc;
+	zplane->csc = data->csc;
+	zplane->rsz = data->rsz;
+
+	zx_plane_hbsc_init(zplane);
+
+	switch (type) {
+	case DRM_PLANE_TYPE_PRIMARY:
+		helper = &zx_gl_plane_helper_funcs;
+		formats = gl_formats;
+		format_count = ARRAY_SIZE(gl_formats);
+		break;
+	case DRM_PLANE_TYPE_OVERLAY:
+		/* TODO: add video layer (vl) support */
+		break;
+	default:
+		return ERR_PTR(-ENODEV);
+	}
+
+	ret = drm_universal_plane_init(drm, plane, VOU_CRTC_MASK,
+				       &zx_plane_funcs, formats, format_count,
+				       type, NULL);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "failed to init universal plane: %d\n", ret);
+		return ERR_PTR(ret);
+	}
+
+	drm_plane_helper_add(plane, helper);
+
+	return plane;
+}
diff --git a/drivers/gpu/drm/zte/zx_plane.h b/drivers/gpu/drm/zte/zx_plane.h
new file mode 100644
index 000000000000..2b82cd558d9d
--- /dev/null
+++ b/drivers/gpu/drm/zte/zx_plane.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2016 Linaro Ltd.
+ * Copyright 2016 ZTE Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __ZX_PLANE_H__
+#define __ZX_PLANE_H__
+
+struct zx_layer_data {
+	void __iomem *layer;
+	void __iomem *csc;
+	void __iomem *hbsc;
+	void __iomem *rsz;
+};
+
+struct drm_plane *zx_plane_init(struct drm_device *drm, struct device *dev,
+				struct zx_layer_data *data,
+				enum drm_plane_type type);
+void zx_plane_set_update(struct drm_plane *plane);
+
+#endif /* __ZX_PLANE_H__ */
diff --git a/drivers/gpu/drm/zte/zx_plane_regs.h b/drivers/gpu/drm/zte/zx_plane_regs.h
new file mode 100644
index 000000000000..3dde6716a558
--- /dev/null
+++ b/drivers/gpu/drm/zte/zx_plane_regs.h
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2016 Linaro Ltd.
+ * Copyright 2016 ZTE Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __ZX_PLANE_REGS_H__
+#define __ZX_PLANE_REGS_H__
+
+/* GL registers */
+#define GL_CTRL0			0x00
+#define GL_UPDATE			BIT(5)
+#define GL_CTRL1			0x04
+#define GL_DATA_FMT_SHIFT		0
+#define GL_DATA_FMT_MASK		(0xf << GL_DATA_FMT_SHIFT)
+#define GL_FMT_ARGB8888			0
+#define GL_FMT_RGB888			1
+#define GL_FMT_RGB565			2
+#define GL_FMT_ARGB1555			3
+#define GL_FMT_ARGB4444			4
+#define GL_CTRL2			0x08
+#define GL_GLOBAL_ALPHA_SHIFT		8
+#define GL_GLOBAL_ALPHA_MASK		(0xff << GL_GLOBAL_ALPHA_SHIFT)
+#define GL_CTRL3			0x0c
+#define GL_SCALER_BYPASS_MODE		BIT(0)
+#define GL_STRIDE			0x18
+#define GL_ADDR				0x1c
+#define GL_SRC_SIZE			0x38
+#define GL_SRC_W_SHIFT			16
+#define GL_SRC_W_MASK			(0x3fff << GL_SRC_W_SHIFT)
+#define GL_SRC_H_SHIFT			0
+#define GL_SRC_H_MASK			(0x3fff << GL_SRC_H_SHIFT)
+#define GL_POS_START			0x9c
+#define GL_POS_END			0xa0
+#define GL_POS_X_SHIFT			16
+#define GL_POS_X_MASK			(0x1fff << GL_POS_X_SHIFT)
+#define GL_POS_Y_SHIFT			0
+#define GL_POS_Y_MASK			(0x1fff << GL_POS_Y_SHIFT)
+
+#define GL_SRC_W(x)	(((x) << GL_SRC_W_SHIFT) & GL_SRC_W_MASK)
+#define GL_SRC_H(x)	(((x) << GL_SRC_H_SHIFT) & GL_SRC_H_MASK)
+#define GL_POS_X(x)	(((x) << GL_POS_X_SHIFT) & GL_POS_X_MASK)
+#define GL_POS_Y(x)	(((x) << GL_POS_Y_SHIFT) & GL_POS_Y_MASK)
+
+/* CSC registers */
+#define CSC_CTRL0			0x30
+#define CSC_COV_MODE_SHIFT		16
+#define CSC_COV_MODE_MASK		(0xffff << CSC_COV_MODE_SHIFT)
+#define CSC_BT601_IMAGE_RGB2YCBCR	0
+#define CSC_BT601_IMAGE_YCBCR2RGB	1
+#define CSC_BT601_VIDEO_RGB2YCBCR	2
+#define CSC_BT601_VIDEO_YCBCR2RGB	3
+#define CSC_BT709_IMAGE_RGB2YCBCR	4
+#define CSC_BT709_IMAGE_YCBCR2RGB	5
+#define CSC_BT709_VIDEO_RGB2YCBCR	6
+#define CSC_BT709_VIDEO_YCBCR2RGB	7
+#define CSC_BT2020_IMAGE_RGB2YCBCR	8
+#define CSC_BT2020_IMAGE_YCBCR2RGB	9
+#define CSC_BT2020_VIDEO_RGB2YCBCR	10
+#define CSC_BT2020_VIDEO_YCBCR2RGB	11
+#define CSC_WORK_ENABLE			BIT(0)
+
+/* RSZ registers */
+#define RSZ_SRC_CFG			0x00
+#define RSZ_DEST_CFG			0x04
+#define RSZ_ENABLE_CFG			0x14
+
+#define RSZ_VER_SHIFT			16
+#define RSZ_VER_MASK			(0xffff << RSZ_VER_SHIFT)
+#define RSZ_HOR_SHIFT			0
+#define RSZ_HOR_MASK			(0xffff << RSZ_HOR_SHIFT)
+
+#define RSZ_VER(x)	(((x) << RSZ_VER_SHIFT) & RSZ_VER_MASK)
+#define RSZ_HOR(x)	(((x) << RSZ_HOR_SHIFT) & RSZ_HOR_MASK)
+
+/* HBSC registers */
+#define HBSC_SATURATION			0x00
+#define HBSC_HUE			0x04
+#define HBSC_BRIGHT			0x08
+#define HBSC_CONTRAST			0x0c
+#define HBSC_THRESHOLD_COL1		0x10
+#define HBSC_THRESHOLD_COL2		0x14
+#define HBSC_THRESHOLD_COL3		0x18
+#define HBSC_CTRL0			0x28
+#define HBSC_CTRL_EN			BIT(2)
+
+#endif /* __ZX_PLANE_REGS_H__ */
diff --git a/drivers/gpu/drm/zte/zx_vou.c b/drivers/gpu/drm/zte/zx_vou.c
new file mode 100644
index 000000000000..73fe15c17c32
--- /dev/null
+++ b/drivers/gpu/drm/zte/zx_vou.c
@@ -0,0 +1,661 @@
+/*
+ * Copyright 2016 Linaro Ltd.
+ * Copyright 2016 ZTE Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/of_address.h>
+#include <video/videomode.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_of.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drmP.h>
+
+#include "zx_drm_drv.h"
+#include "zx_plane.h"
+#include "zx_vou.h"
+#include "zx_vou_regs.h"
+
+#define GL_NUM	2
+#define VL_NUM	3
+
+enum vou_chn_type {
+	VOU_CHN_MAIN,
+	VOU_CHN_AUX,
+};
+
+struct zx_crtc_regs {
+	u32 fir_active;
+	u32 fir_htiming;
+	u32 fir_vtiming;
+	u32 timing_shift;
+	u32 timing_pi_shift;
+};
+
+static const struct zx_crtc_regs main_crtc_regs = {
+	.fir_active = FIR_MAIN_ACTIVE,
+	.fir_htiming = FIR_MAIN_H_TIMING,
+	.fir_vtiming = FIR_MAIN_V_TIMING,
+	.timing_shift = TIMING_MAIN_SHIFT,
+	.timing_pi_shift = TIMING_MAIN_PI_SHIFT,
+};
+
+static const struct zx_crtc_regs aux_crtc_regs = {
+	.fir_active = FIR_AUX_ACTIVE,
+	.fir_htiming = FIR_AUX_H_TIMING,
+	.fir_vtiming = FIR_AUX_V_TIMING,
+	.timing_shift = TIMING_AUX_SHIFT,
+	.timing_pi_shift = TIMING_AUX_PI_SHIFT,
+};
+
+struct zx_crtc_bits {
+	u32 polarity_mask;
+	u32 polarity_shift;
+	u32 int_frame_mask;
+	u32 tc_enable;
+	u32 gl_enable;
+};
+
+static const struct zx_crtc_bits main_crtc_bits = {
+	.polarity_mask = MAIN_POL_MASK,
+	.polarity_shift = MAIN_POL_SHIFT,
+	.int_frame_mask = TIMING_INT_MAIN_FRAME,
+	.tc_enable = MAIN_TC_EN,
+	.gl_enable = OSD_CTRL0_GL0_EN,
+};
+
+static const struct zx_crtc_bits aux_crtc_bits = {
+	.polarity_mask = AUX_POL_MASK,
+	.polarity_shift = AUX_POL_SHIFT,
+	.int_frame_mask = TIMING_INT_AUX_FRAME,
+	.tc_enable = AUX_TC_EN,
+	.gl_enable = OSD_CTRL0_GL1_EN,
+};
+
+struct zx_crtc {
+	struct drm_crtc crtc;
+	struct drm_plane *primary;
+	struct zx_vou_hw *vou;
+	void __iomem *chnreg;
+	const struct zx_crtc_regs *regs;
+	const struct zx_crtc_bits *bits;
+	enum vou_chn_type chn_type;
+	struct clk *pixclk;
+};
+
+#define to_zx_crtc(x) container_of(x, struct zx_crtc, crtc)
+
+struct zx_vou_hw {
+	struct device *dev;
+	void __iomem *osd;
+	void __iomem *timing;
+	void __iomem *vouctl;
+	void __iomem *otfppu;
+	void __iomem *dtrc;
+	struct clk *axi_clk;
+	struct clk *ppu_clk;
+	struct clk *main_clk;
+	struct clk *aux_clk;
+	struct zx_crtc *main_crtc;
+	struct zx_crtc *aux_crtc;
+};
+
+static inline struct zx_vou_hw *crtc_to_vou(struct drm_crtc *crtc)
+{
+	struct zx_crtc *zcrtc = to_zx_crtc(crtc);
+
+	return zcrtc->vou;
+}
+
+void vou_inf_enable(const struct vou_inf *inf, struct drm_crtc *crtc)
+{
+	struct zx_crtc *zcrtc = to_zx_crtc(crtc);
+	struct zx_vou_hw *vou = zcrtc->vou;
+	bool is_main = zcrtc->chn_type == VOU_CHN_MAIN;
+	u32 data_sel_shift = inf->id << 1;
+
+	/* Select data format */
+	zx_writel_mask(vou->vouctl + VOU_INF_DATA_SEL, 0x3 << data_sel_shift,
+		       inf->data_sel << data_sel_shift);
+
+	/* Select channel */
+	zx_writel_mask(vou->vouctl + VOU_INF_CH_SEL, 0x1 << inf->id,
+		       zcrtc->chn_type << inf->id);
+
+	/* Select interface clocks */
+	zx_writel_mask(vou->vouctl + VOU_CLK_SEL, inf->clocks_sel_bits,
+		       is_main ? 0 : inf->clocks_sel_bits);
+
+	/* Enable interface clocks */
+	zx_writel_mask(vou->vouctl + VOU_CLK_EN, inf->clocks_en_bits,
+		       inf->clocks_en_bits);
+
+	/* Enable the device */
+	zx_writel_mask(vou->vouctl + VOU_INF_EN, 1 << inf->id, 1 << inf->id);
+}
+
+void vou_inf_disable(const struct vou_inf *inf, struct drm_crtc *crtc)
+{
+	struct zx_vou_hw *vou = crtc_to_vou(crtc);
+
+	/* Disable the device */
+	zx_writel_mask(vou->vouctl + VOU_INF_EN, 1 << inf->id, 0);
+
+	/* Disable interface clocks */
+	zx_writel_mask(vou->vouctl + VOU_CLK_EN, inf->clocks_en_bits, 0);
+}
+
+static inline void vou_chn_set_update(struct zx_crtc *zcrtc)
+{
+	zx_writel(zcrtc->chnreg + CHN_UPDATE, 1);
+}
+
+static void zx_crtc_enable(struct drm_crtc *crtc)
+{
+	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+	struct zx_crtc *zcrtc = to_zx_crtc(crtc);
+	struct zx_vou_hw *vou = zcrtc->vou;
+	const struct zx_crtc_regs *regs = zcrtc->regs;
+	const struct zx_crtc_bits *bits = zcrtc->bits;
+	struct videomode vm;
+	u32 pol = 0;
+	u32 val;
+	int ret;
+
+	drm_display_mode_to_videomode(mode, &vm);
+
+	/* Set up timing parameters */
+	val = V_ACTIVE(vm.vactive - 1);
+	val |= H_ACTIVE(vm.hactive - 1);
+	zx_writel(vou->timing + regs->fir_active, val);
+
+	val = SYNC_WIDE(vm.hsync_len - 1);
+	val |= BACK_PORCH(vm.hback_porch - 1);
+	val |= FRONT_PORCH(vm.hfront_porch - 1);
+	zx_writel(vou->timing + regs->fir_htiming, val);
+
+	val = SYNC_WIDE(vm.vsync_len - 1);
+	val |= BACK_PORCH(vm.vback_porch - 1);
+	val |= FRONT_PORCH(vm.vfront_porch - 1);
+	zx_writel(vou->timing + regs->fir_vtiming, val);
+
+	/* Set up polarities */
+	if (vm.flags & DISPLAY_FLAGS_VSYNC_LOW)
+		pol |= 1 << POL_VSYNC_SHIFT;
+	if (vm.flags & DISPLAY_FLAGS_HSYNC_LOW)
+		pol |= 1 << POL_HSYNC_SHIFT;
+
+	zx_writel_mask(vou->timing + TIMING_CTRL, bits->polarity_mask,
+		       pol << bits->polarity_shift);
+
+	/* Setup SHIFT register by following what ZTE BSP does */
+	zx_writel(vou->timing + regs->timing_shift, H_SHIFT_VAL);
+	zx_writel(vou->timing + regs->timing_pi_shift, H_PI_SHIFT_VAL);
+
+	/* Enable TIMING_CTRL */
+	zx_writel_mask(vou->timing + TIMING_TC_ENABLE, bits->tc_enable,
+		       bits->tc_enable);
+
+	/* Configure channel screen size */
+	zx_writel_mask(zcrtc->chnreg + CHN_CTRL1, CHN_SCREEN_W_MASK,
+		       vm.hactive << CHN_SCREEN_W_SHIFT);
+	zx_writel_mask(zcrtc->chnreg + CHN_CTRL1, CHN_SCREEN_H_MASK,
+		       vm.vactive << CHN_SCREEN_H_SHIFT);
+
+	/* Update channel */
+	vou_chn_set_update(zcrtc);
+
+	/* Enable channel */
+	zx_writel_mask(zcrtc->chnreg + CHN_CTRL0, CHN_ENABLE, CHN_ENABLE);
+
+	/* Enable Graphic Layer */
+	zx_writel_mask(vou->osd + OSD_CTRL0, bits->gl_enable,
+		       bits->gl_enable);
+
+	drm_crtc_vblank_on(crtc);
+
+	ret = clk_set_rate(zcrtc->pixclk, mode->clock * 1000);
+	if (ret) {
+		DRM_DEV_ERROR(vou->dev, "failed to set pixclk rate: %d\n", ret);
+		return;
+	}
+
+	ret = clk_prepare_enable(zcrtc->pixclk);
+	if (ret)
+		DRM_DEV_ERROR(vou->dev, "failed to enable pixclk: %d\n", ret);
+}
+
+static void zx_crtc_disable(struct drm_crtc *crtc)
+{
+	struct zx_crtc *zcrtc = to_zx_crtc(crtc);
+	const struct zx_crtc_bits *bits = zcrtc->bits;
+	struct zx_vou_hw *vou = zcrtc->vou;
+
+	clk_disable_unprepare(zcrtc->pixclk);
+
+	drm_crtc_vblank_off(crtc);
+
+	/* Disable Graphic Layer */
+	zx_writel_mask(vou->osd + OSD_CTRL0, bits->gl_enable, 0);
+
+	/* Disable channel */
+	zx_writel_mask(zcrtc->chnreg + CHN_CTRL0, CHN_ENABLE, 0);
+
+	/* Disable TIMING_CTRL */
+	zx_writel_mask(vou->timing + TIMING_TC_ENABLE, bits->tc_enable, 0);
+}
+
+static void zx_crtc_atomic_flush(struct drm_crtc *crtc,
+				  struct drm_crtc_state *old_state)
+{
+	struct drm_pending_vblank_event *event = crtc->state->event;
+
+	if (!event)
+		return;
+
+	crtc->state->event = NULL;
+
+	spin_lock_irq(&crtc->dev->event_lock);
+	if (drm_crtc_vblank_get(crtc) == 0)
+		drm_crtc_arm_vblank_event(crtc, event);
+	else
+		drm_crtc_send_vblank_event(crtc, event);
+	spin_unlock_irq(&crtc->dev->event_lock);
+}
+
+static const struct drm_crtc_helper_funcs zx_crtc_helper_funcs = {
+	.enable = zx_crtc_enable,
+	.disable = zx_crtc_disable,
+	.atomic_flush = zx_crtc_atomic_flush,
+};
+
+static const struct drm_crtc_funcs zx_crtc_funcs = {
+	.destroy = drm_crtc_cleanup,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+};
+
+static int zx_crtc_init(struct drm_device *drm, struct zx_vou_hw *vou,
+			enum vou_chn_type chn_type)
+{
+	struct device *dev = vou->dev;
+	struct zx_layer_data data;
+	struct zx_crtc *zcrtc;
+	int ret;
+
+	zcrtc = devm_kzalloc(dev, sizeof(*zcrtc), GFP_KERNEL);
+	if (!zcrtc)
+		return -ENOMEM;
+
+	zcrtc->vou = vou;
+	zcrtc->chn_type = chn_type;
+
+	if (chn_type == VOU_CHN_MAIN) {
+		data.layer = vou->osd + MAIN_GL_OFFSET;
+		data.csc = vou->osd + MAIN_CSC_OFFSET;
+		data.hbsc = vou->osd + MAIN_HBSC_OFFSET;
+		data.rsz = vou->otfppu + MAIN_RSZ_OFFSET;
+		zcrtc->chnreg = vou->osd + OSD_MAIN_CHN;
+		zcrtc->regs = &main_crtc_regs;
+		zcrtc->bits = &main_crtc_bits;
+	} else {
+		data.layer = vou->osd + AUX_GL_OFFSET;
+		data.csc = vou->osd + AUX_CSC_OFFSET;
+		data.hbsc = vou->osd + AUX_HBSC_OFFSET;
+		data.rsz = vou->otfppu + AUX_RSZ_OFFSET;
+		zcrtc->chnreg = vou->osd + OSD_AUX_CHN;
+		zcrtc->regs = &aux_crtc_regs;
+		zcrtc->bits = &aux_crtc_bits;
+	}
+
+	zcrtc->pixclk = devm_clk_get(dev, (chn_type == VOU_CHN_MAIN) ?
+					  "main_wclk" : "aux_wclk");
+	if (IS_ERR(zcrtc->pixclk)) {
+		ret = PTR_ERR(zcrtc->pixclk);
+		DRM_DEV_ERROR(dev, "failed to get pix clk: %d\n", ret);
+		return ret;
+	}
+
+	zcrtc->primary = zx_plane_init(drm, dev, &data, DRM_PLANE_TYPE_PRIMARY);
+	if (IS_ERR(zcrtc->primary)) {
+		ret = PTR_ERR(zcrtc->primary);
+		DRM_DEV_ERROR(dev, "failed to init primary plane: %d\n", ret);
+		return ret;
+	}
+
+	ret = drm_crtc_init_with_planes(drm, &zcrtc->crtc, zcrtc->primary, NULL,
+					&zx_crtc_funcs, NULL);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "failed to init drm crtc: %d\n", ret);
+		return ret;
+	}
+
+	drm_crtc_helper_add(&zcrtc->crtc, &zx_crtc_helper_funcs);
+
+	if (chn_type == VOU_CHN_MAIN)
+		vou->main_crtc = zcrtc;
+	else
+		vou->aux_crtc = zcrtc;
+
+	return 0;
+}
+
+static inline struct drm_crtc *zx_find_crtc(struct drm_device *drm, int pipe)
+{
+	struct drm_crtc *crtc;
+
+	list_for_each_entry(crtc, &drm->mode_config.crtc_list, head)
+		if (crtc->index == pipe)
+			return crtc;
+
+	return NULL;
+}
+
+int zx_vou_enable_vblank(struct drm_device *drm, unsigned int pipe)
+{
+	struct drm_crtc *crtc;
+	struct zx_crtc *zcrtc;
+	struct zx_vou_hw *vou;
+	u32 int_frame_mask;
+
+	crtc = zx_find_crtc(drm, pipe);
+	if (!crtc)
+		return 0;
+
+	vou = crtc_to_vou(crtc);
+	zcrtc = to_zx_crtc(crtc);
+	int_frame_mask = zcrtc->bits->int_frame_mask;
+
+	zx_writel_mask(vou->timing + TIMING_INT_CTRL, int_frame_mask,
+		       int_frame_mask);
+
+	return 0;
+}
+
+void zx_vou_disable_vblank(struct drm_device *drm, unsigned int pipe)
+{
+	struct drm_crtc *crtc;
+	struct zx_crtc *zcrtc;
+	struct zx_vou_hw *vou;
+
+	crtc = zx_find_crtc(drm, pipe);
+	if (!crtc)
+		return;
+
+	vou = crtc_to_vou(crtc);
+	zcrtc = to_zx_crtc(crtc);
+
+	zx_writel_mask(vou->timing + TIMING_INT_CTRL,
+		       zcrtc->bits->int_frame_mask, 0);
+}
+
+static irqreturn_t vou_irq_handler(int irq, void *dev_id)
+{
+	struct zx_vou_hw *vou = dev_id;
+	u32 state;
+
+	/* Handle TIMING_CTRL frame interrupts */
+	state = zx_readl(vou->timing + TIMING_INT_STATE);
+	zx_writel(vou->timing + TIMING_INT_STATE, state);
+
+	if (state & TIMING_INT_MAIN_FRAME)
+		drm_crtc_handle_vblank(&vou->main_crtc->crtc);
+
+	if (state & TIMING_INT_AUX_FRAME)
+		drm_crtc_handle_vblank(&vou->aux_crtc->crtc);
+
+	/* Handle OSD interrupts */
+	state = zx_readl(vou->osd + OSD_INT_STA);
+	zx_writel(vou->osd + OSD_INT_CLRSTA, state);
+
+	if (state & OSD_INT_MAIN_UPT) {
+		vou_chn_set_update(vou->main_crtc);
+		zx_plane_set_update(vou->main_crtc->primary);
+	}
+
+	if (state & OSD_INT_AUX_UPT) {
+		vou_chn_set_update(vou->aux_crtc);
+		zx_plane_set_update(vou->aux_crtc->primary);
+	}
+
+	if (state & OSD_INT_ERROR)
+		DRM_DEV_ERROR(vou->dev, "OSD ERROR: 0x%08x!\n", state);
+
+	return IRQ_HANDLED;
+}
+
+static void vou_dtrc_init(struct zx_vou_hw *vou)
+{
+	/* Clear bit for bypass by ID */
+	zx_writel_mask(vou->dtrc + DTRC_DETILE_CTRL,
+		       TILE2RASTESCAN_BYPASS_MODE, 0);
+
+	/* Select ARIDR mode */
+	zx_writel_mask(vou->dtrc + DTRC_DETILE_CTRL, DETILE_ARIDR_MODE_MASK,
+		       DETILE_ARID_IN_ARIDR);
+
+	/* Bypass decompression for both frames */
+	zx_writel_mask(vou->dtrc + DTRC_F0_CTRL, DTRC_DECOMPRESS_BYPASS,
+		       DTRC_DECOMPRESS_BYPASS);
+	zx_writel_mask(vou->dtrc + DTRC_F1_CTRL, DTRC_DECOMPRESS_BYPASS,
+		       DTRC_DECOMPRESS_BYPASS);
+
+	/* Set up ARID register */
+	zx_writel(vou->dtrc + DTRC_ARID, DTRC_ARID3(0xf) | DTRC_ARID2(0xe) |
+		  DTRC_ARID1(0xf) | DTRC_ARID0(0xe));
+}
+
+static void vou_hw_init(struct zx_vou_hw *vou)
+{
+	/* Set GL0 to main channel and GL1 to aux channel */
+	zx_writel_mask(vou->osd + OSD_CTRL0, OSD_CTRL0_GL0_SEL, 0);
+	zx_writel_mask(vou->osd + OSD_CTRL0, OSD_CTRL0_GL1_SEL,
+		       OSD_CTRL0_GL1_SEL);
+
+	/* Release reset for all VOU modules */
+	zx_writel(vou->vouctl + VOU_SOFT_RST, ~0);
+
+	/* Select main clock for GL0 and aux clock for GL1 module */
+	zx_writel_mask(vou->vouctl + VOU_CLK_SEL, VOU_CLK_GL0_SEL, 0);
+	zx_writel_mask(vou->vouctl + VOU_CLK_SEL, VOU_CLK_GL1_SEL,
+		       VOU_CLK_GL1_SEL);
+
+	/* Enable clock auto-gating for all VOU modules */
+	zx_writel(vou->vouctl + VOU_CLK_REQEN, ~0);
+
+	/* Enable all VOU module clocks */
+	zx_writel(vou->vouctl + VOU_CLK_EN, ~0);
+
+	/* Clear both OSD and TIMING_CTRL interrupt state */
+	zx_writel(vou->osd + OSD_INT_CLRSTA, ~0);
+	zx_writel(vou->timing + TIMING_INT_STATE, ~0);
+
+	/* Enable OSD and TIMING_CTRL interrrupts */
+	zx_writel(vou->osd + OSD_INT_MSK, OSD_INT_ENABLE);
+	zx_writel(vou->timing + TIMING_INT_CTRL, TIMING_INT_ENABLE);
+
+	/* Select GPC as input to gl/vl scaler as a sane default setting */
+	zx_writel(vou->otfppu + OTFPPU_RSZ_DATA_SOURCE, 0x2a);
+
+	/*
+	 * Needs to reset channel and layer logic per frame when frame starts
+	 * to get VOU work properly.
+	 */
+	zx_writel_mask(vou->osd + OSD_RST_CLR, RST_PER_FRAME, RST_PER_FRAME);
+
+	vou_dtrc_init(vou);
+}
+
+static int zx_crtc_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct drm_device *drm = data;
+	struct zx_vou_hw *vou;
+	struct resource *res;
+	int irq;
+	int ret;
+
+	vou = devm_kzalloc(dev, sizeof(*vou), GFP_KERNEL);
+	if (!vou)
+		return -ENOMEM;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "osd");
+	vou->osd = devm_ioremap_resource(dev, res);
+	if (IS_ERR(vou->osd)) {
+		ret = PTR_ERR(vou->osd);
+		DRM_DEV_ERROR(dev, "failed to remap osd region: %d\n", ret);
+		return ret;
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "timing_ctrl");
+	vou->timing = devm_ioremap_resource(dev, res);
+	if (IS_ERR(vou->timing)) {
+		ret = PTR_ERR(vou->timing);
+		DRM_DEV_ERROR(dev, "failed to remap timing_ctrl region: %d\n",
+			      ret);
+		return ret;
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dtrc");
+	vou->dtrc = devm_ioremap_resource(dev, res);
+	if (IS_ERR(vou->dtrc)) {
+		ret = PTR_ERR(vou->dtrc);
+		DRM_DEV_ERROR(dev, "failed to remap dtrc region: %d\n", ret);
+		return ret;
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "vou_ctrl");
+	vou->vouctl = devm_ioremap_resource(dev, res);
+	if (IS_ERR(vou->vouctl)) {
+		ret = PTR_ERR(vou->vouctl);
+		DRM_DEV_ERROR(dev, "failed to remap vou_ctrl region: %d\n",
+			      ret);
+		return ret;
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "otfppu");
+	vou->otfppu = devm_ioremap_resource(dev, res);
+	if (IS_ERR(vou->otfppu)) {
+		ret = PTR_ERR(vou->otfppu);
+		DRM_DEV_ERROR(dev, "failed to remap otfppu region: %d\n", ret);
+		return ret;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	vou->axi_clk = devm_clk_get(dev, "aclk");
+	if (IS_ERR(vou->axi_clk)) {
+		ret = PTR_ERR(vou->axi_clk);
+		DRM_DEV_ERROR(dev, "failed to get axi_clk: %d\n", ret);
+		return ret;
+	}
+
+	vou->ppu_clk = devm_clk_get(dev, "ppu_wclk");
+	if (IS_ERR(vou->ppu_clk)) {
+		ret = PTR_ERR(vou->ppu_clk);
+		DRM_DEV_ERROR(dev, "failed to get ppu_clk: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(vou->axi_clk);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "failed to enable axi_clk: %d\n", ret);
+		return ret;
+	}
+
+	clk_prepare_enable(vou->ppu_clk);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "failed to enable ppu_clk: %d\n", ret);
+		goto disable_axi_clk;
+	}
+
+	vou->dev = dev;
+	dev_set_drvdata(dev, vou);
+
+	vou_hw_init(vou);
+
+	ret = devm_request_irq(dev, irq, vou_irq_handler, 0, "zx_vou", vou);
+	if (ret < 0) {
+		DRM_DEV_ERROR(dev, "failed to request vou irq: %d\n", ret);
+		goto disable_ppu_clk;
+	}
+
+	ret = zx_crtc_init(drm, vou, VOU_CHN_MAIN);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "failed to init main channel crtc: %d\n",
+			      ret);
+		goto disable_ppu_clk;
+	}
+
+	ret = zx_crtc_init(drm, vou, VOU_CHN_AUX);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "failed to init aux channel crtc: %d\n",
+			      ret);
+		goto disable_ppu_clk;
+	}
+
+	return 0;
+
+disable_ppu_clk:
+	clk_disable_unprepare(vou->ppu_clk);
+disable_axi_clk:
+	clk_disable_unprepare(vou->axi_clk);
+	return ret;
+}
+
+static void zx_crtc_unbind(struct device *dev, struct device *master,
+			   void *data)
+{
+	struct zx_vou_hw *vou = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(vou->axi_clk);
+	clk_disable_unprepare(vou->ppu_clk);
+}
+
+static const struct component_ops zx_crtc_component_ops = {
+	.bind = zx_crtc_bind,
+	.unbind = zx_crtc_unbind,
+};
+
+static int zx_crtc_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &zx_crtc_component_ops);
+}
+
+static int zx_crtc_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &zx_crtc_component_ops);
+	return 0;
+}
+
+static const struct of_device_id zx_crtc_of_match[] = {
+	{ .compatible = "zte,zx296718-dpc", },
+	{ /* end */ },
+};
+MODULE_DEVICE_TABLE(of, zx_crtc_of_match);
+
+struct platform_driver zx_crtc_driver = {
+	.probe = zx_crtc_probe,
+	.remove = zx_crtc_remove,
+	.driver	= {
+		.name = "zx-crtc",
+		.of_match_table	= zx_crtc_of_match,
+	},
+};
diff --git a/drivers/gpu/drm/zte/zx_vou.h b/drivers/gpu/drm/zte/zx_vou.h
new file mode 100644
index 000000000000..349e06cd86f4
--- /dev/null
+++ b/drivers/gpu/drm/zte/zx_vou.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2016 Linaro Ltd.
+ * Copyright 2016 ZTE Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __ZX_VOU_H__
+#define __ZX_VOU_H__
+
+#define VOU_CRTC_MASK		0x3
+
+/* VOU output interfaces */
+enum vou_inf_id {
+	VOU_HDMI	= 0,
+	VOU_RGB_LCD	= 1,
+	VOU_TV_ENC	= 2,
+	VOU_MIPI_DSI	= 3,
+	VOU_LVDS	= 4,
+	VOU_VGA		= 5,
+};
+
+enum vou_inf_data_sel {
+	VOU_YUV444	= 0,
+	VOU_RGB_101010	= 1,
+	VOU_RGB_888	= 2,
+	VOU_RGB_666	= 3,
+};
+
+struct vou_inf {
+	enum vou_inf_id id;
+	enum vou_inf_data_sel data_sel;
+	u32 clocks_en_bits;
+	u32 clocks_sel_bits;
+};
+
+void vou_inf_enable(const struct vou_inf *inf, struct drm_crtc *crtc);
+void vou_inf_disable(const struct vou_inf *inf, struct drm_crtc *crtc);
+
+int zx_vou_enable_vblank(struct drm_device *drm, unsigned int pipe);
+void zx_vou_disable_vblank(struct drm_device *drm, unsigned int pipe);
+
+#endif /* __ZX_VOU_H__ */
diff --git a/drivers/gpu/drm/zte/zx_vou_regs.h b/drivers/gpu/drm/zte/zx_vou_regs.h
new file mode 100644
index 000000000000..f44e7a4ae441
--- /dev/null
+++ b/drivers/gpu/drm/zte/zx_vou_regs.h
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2016 Linaro Ltd.
+ * Copyright 2016 ZTE Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __ZX_VOU_REGS_H__
+#define __ZX_VOU_REGS_H__
+
+/* Sub-module offset */
+#define MAIN_GL_OFFSET			0x130
+#define MAIN_CSC_OFFSET			0x580
+#define MAIN_HBSC_OFFSET		0x820
+#define MAIN_RSZ_OFFSET			0x600 /* OTFPPU sub-module */
+
+#define AUX_GL_OFFSET			0x200
+#define AUX_CSC_OFFSET			0x5d0
+#define AUX_HBSC_OFFSET			0x860
+#define AUX_RSZ_OFFSET			0x800
+
+/* OSD (GPC_GLOBAL) registers */
+#define OSD_INT_STA			0x04
+#define OSD_INT_CLRSTA			0x08
+#define OSD_INT_MSK			0x0c
+#define OSD_INT_AUX_UPT			BIT(14)
+#define OSD_INT_MAIN_UPT		BIT(13)
+#define OSD_INT_GL1_LBW			BIT(10)
+#define OSD_INT_GL0_LBW			BIT(9)
+#define OSD_INT_VL2_LBW			BIT(8)
+#define OSD_INT_VL1_LBW			BIT(7)
+#define OSD_INT_VL0_LBW			BIT(6)
+#define OSD_INT_BUS_ERR			BIT(3)
+#define OSD_INT_CFG_ERR			BIT(2)
+#define OSD_INT_ERROR (\
+	OSD_INT_GL1_LBW | OSD_INT_GL0_LBW | \
+	OSD_INT_VL2_LBW | OSD_INT_VL1_LBW | OSD_INT_VL0_LBW | \
+	OSD_INT_BUS_ERR | OSD_INT_CFG_ERR \
+)
+#define OSD_INT_ENABLE (OSD_INT_ERROR | OSD_INT_AUX_UPT | OSD_INT_MAIN_UPT)
+#define OSD_CTRL0			0x10
+#define OSD_CTRL0_GL0_EN		BIT(7)
+#define OSD_CTRL0_GL0_SEL		BIT(6)
+#define OSD_CTRL0_GL1_EN		BIT(5)
+#define OSD_CTRL0_GL1_SEL		BIT(4)
+#define OSD_RST_CLR			0x1c
+#define RST_PER_FRAME			BIT(19)
+
+/* Main/Aux channel registers */
+#define OSD_MAIN_CHN			0x470
+#define OSD_AUX_CHN			0x4d0
+#define CHN_CTRL0			0x00
+#define CHN_ENABLE			BIT(0)
+#define CHN_CTRL1			0x04
+#define CHN_SCREEN_W_SHIFT		18
+#define CHN_SCREEN_W_MASK		(0x1fff << CHN_SCREEN_W_SHIFT)
+#define CHN_SCREEN_H_SHIFT		5
+#define CHN_SCREEN_H_MASK		(0x1fff << CHN_SCREEN_H_SHIFT)
+#define CHN_UPDATE			0x08
+
+/* TIMING_CTRL registers */
+#define TIMING_TC_ENABLE		0x04
+#define AUX_TC_EN			BIT(1)
+#define MAIN_TC_EN			BIT(0)
+#define FIR_MAIN_ACTIVE			0x08
+#define FIR_AUX_ACTIVE			0x0c
+#define V_ACTIVE_SHIFT			16
+#define V_ACTIVE_MASK			(0xffff << V_ACTIVE_SHIFT)
+#define H_ACTIVE_SHIFT			0
+#define H_ACTIVE_MASK			(0xffff << H_ACTIVE_SHIFT)
+#define FIR_MAIN_H_TIMING		0x10
+#define FIR_MAIN_V_TIMING		0x14
+#define FIR_AUX_H_TIMING		0x18
+#define FIR_AUX_V_TIMING		0x1c
+#define SYNC_WIDE_SHIFT			22
+#define SYNC_WIDE_MASK			(0x3ff << SYNC_WIDE_SHIFT)
+#define BACK_PORCH_SHIFT		11
+#define BACK_PORCH_MASK			(0x7ff << BACK_PORCH_SHIFT)
+#define FRONT_PORCH_SHIFT		0
+#define FRONT_PORCH_MASK		(0x7ff << FRONT_PORCH_SHIFT)
+#define TIMING_CTRL			0x20
+#define AUX_POL_SHIFT			3
+#define AUX_POL_MASK			(0x7 << AUX_POL_SHIFT)
+#define MAIN_POL_SHIFT			0
+#define MAIN_POL_MASK			(0x7 << MAIN_POL_SHIFT)
+#define POL_DE_SHIFT			2
+#define POL_VSYNC_SHIFT			1
+#define POL_HSYNC_SHIFT			0
+#define TIMING_INT_CTRL			0x24
+#define TIMING_INT_STATE		0x28
+#define TIMING_INT_AUX_FRAME		BIT(3)
+#define TIMING_INT_MAIN_FRAME		BIT(1)
+#define TIMING_INT_AUX_FRAME_SEL_VSW	(0x2 << 10)
+#define TIMING_INT_MAIN_FRAME_SEL_VSW	(0x2 << 6)
+#define TIMING_INT_ENABLE (\
+	TIMING_INT_MAIN_FRAME_SEL_VSW | TIMING_INT_AUX_FRAME_SEL_VSW | \
+	TIMING_INT_MAIN_FRAME | TIMING_INT_AUX_FRAME \
+)
+#define TIMING_MAIN_SHIFT		0x2c
+#define TIMING_AUX_SHIFT		0x30
+#define H_SHIFT_VAL			0x0048
+#define TIMING_MAIN_PI_SHIFT		0x68
+#define TIMING_AUX_PI_SHIFT		0x6c
+#define H_PI_SHIFT_VAL			0x000f
+
+#define V_ACTIVE(x)	(((x) << V_ACTIVE_SHIFT) & V_ACTIVE_MASK)
+#define H_ACTIVE(x)	(((x) << H_ACTIVE_SHIFT) & H_ACTIVE_MASK)
+
+#define SYNC_WIDE(x)	(((x) << SYNC_WIDE_SHIFT) & SYNC_WIDE_MASK)
+#define BACK_PORCH(x)	(((x) << BACK_PORCH_SHIFT) & BACK_PORCH_MASK)
+#define FRONT_PORCH(x)	(((x) << FRONT_PORCH_SHIFT) & FRONT_PORCH_MASK)
+
+/* DTRC registers */
+#define DTRC_F0_CTRL			0x2c
+#define DTRC_F1_CTRL			0x5c
+#define DTRC_DECOMPRESS_BYPASS		BIT(17)
+#define DTRC_DETILE_CTRL		0x68
+#define TILE2RASTESCAN_BYPASS_MODE	BIT(30)
+#define DETILE_ARIDR_MODE_MASK		(0x3 << 0)
+#define DETILE_ARID_ALL			0
+#define DETILE_ARID_IN_ARIDR		1
+#define DETILE_ARID_BYP_BUT_ARIDR	2
+#define DETILE_ARID_IN_ARIDR2		3
+#define DTRC_ARID			0x6c
+#define DTRC_ARID3_SHIFT		24
+#define DTRC_ARID3_MASK			(0xff << DTRC_ARID3_SHIFT)
+#define DTRC_ARID2_SHIFT		16
+#define DTRC_ARID2_MASK			(0xff << DTRC_ARID2_SHIFT)
+#define DTRC_ARID1_SHIFT		8
+#define DTRC_ARID1_MASK			(0xff << DTRC_ARID1_SHIFT)
+#define DTRC_ARID0_SHIFT		0
+#define DTRC_ARID0_MASK			(0xff << DTRC_ARID0_SHIFT)
+#define DTRC_DEC2DDR_ARID		0x70
+
+#define DTRC_ARID3(x)	(((x) << DTRC_ARID3_SHIFT) & DTRC_ARID3_MASK)
+#define DTRC_ARID2(x)	(((x) << DTRC_ARID2_SHIFT) & DTRC_ARID2_MASK)
+#define DTRC_ARID1(x)	(((x) << DTRC_ARID1_SHIFT) & DTRC_ARID1_MASK)
+#define DTRC_ARID0(x)	(((x) << DTRC_ARID0_SHIFT) & DTRC_ARID0_MASK)
+
+/* VOU_CTRL registers */
+#define VOU_INF_EN			0x00
+#define VOU_INF_CH_SEL			0x04
+#define VOU_INF_DATA_SEL		0x08
+#define VOU_SOFT_RST			0x14
+#define VOU_CLK_SEL			0x18
+#define VOU_CLK_GL1_SEL			BIT(5)
+#define VOU_CLK_GL0_SEL			BIT(4)
+#define VOU_CLK_REQEN			0x20
+#define VOU_CLK_EN			0x24
+
+/* OTFPPU_CTRL registers */
+#define OTFPPU_RSZ_DATA_SOURCE		0x04
+
+#endif /* __ZX_VOU_REGS_H__ */
diff --git a/drivers/gpu/host1x/dev.h b/drivers/gpu/host1x/dev.h
index 5220510f39da..06dd4f85125f 100644
--- a/drivers/gpu/host1x/dev.h
+++ b/drivers/gpu/host1x/dev.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012-2013, NVIDIA Corporation.
+ * Copyright (c) 2012-2015, NVIDIA Corporation.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -120,6 +120,7 @@ struct host1x {
 
 	struct host1x_syncpt *nop_sp;
 
+	struct mutex syncpt_mutex;
 	struct mutex chlist_mutex;
 	struct host1x_channel chlist;
 	unsigned long allocated_channels;
diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c
index a91b7c4a6110..92c3df933303 100644
--- a/drivers/gpu/host1x/job.c
+++ b/drivers/gpu/host1x/job.c
@@ -1,7 +1,7 @@
 /*
  * Tegra host1x Job
  *
- * Copyright (c) 2010-2013, NVIDIA Corporation.
+ * Copyright (c) 2010-2015, NVIDIA Corporation.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -539,9 +539,12 @@ int host1x_job_pin(struct host1x_job *job, struct device *dev)
 
 		g->base = job->gather_addr_phys[i];
 
-		for (j = i + 1; j < job->num_gathers; j++)
-			if (job->gathers[j].bo == g->bo)
+		for (j = i + 1; j < job->num_gathers; j++) {
+			if (job->gathers[j].bo == g->bo) {
 				job->gathers[j].handled = true;
+				job->gathers[j].base = g->base;
+			}
+		}
 
 		err = do_relocs(job, g->bo);
 		if (err)
diff --git a/drivers/gpu/host1x/syncpt.c b/drivers/gpu/host1x/syncpt.c
index 95589328ad52..25c11a85050b 100644
--- a/drivers/gpu/host1x/syncpt.c
+++ b/drivers/gpu/host1x/syncpt.c
@@ -1,7 +1,7 @@
 /*
  * Tegra host1x Syncpoints
  *
- * Copyright (c) 2010-2013, NVIDIA Corporation.
+ * Copyright (c) 2010-2015, NVIDIA Corporation.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -61,22 +61,24 @@ static struct host1x_syncpt *host1x_syncpt_alloc(struct host1x *host,
 	struct host1x_syncpt *sp = host->syncpt;
 	char *name;
 
+	mutex_lock(&host->syncpt_mutex);
+
 	for (i = 0; i < host->info->nb_pts && sp->name; i++, sp++)
 		;
 
 	if (i >= host->info->nb_pts)
-		return NULL;
+		goto unlock;
 
 	if (flags & HOST1X_SYNCPT_HAS_BASE) {
 		sp->base = host1x_syncpt_base_request(host);
 		if (!sp->base)
-			return NULL;
+			goto unlock;
 	}
 
 	name = kasprintf(GFP_KERNEL, "%02u-%s", sp->id,
 			dev ? dev_name(dev) : NULL);
 	if (!name)
-		return NULL;
+		goto free_base;
 
 	sp->dev = dev;
 	sp->name = name;
@@ -86,7 +88,15 @@ static struct host1x_syncpt *host1x_syncpt_alloc(struct host1x *host,
 	else
 		sp->client_managed = false;
 
+	mutex_unlock(&host->syncpt_mutex);
 	return sp;
+
+free_base:
+	host1x_syncpt_base_free(sp->base);
+	sp->base = NULL;
+unlock:
+	mutex_unlock(&host->syncpt_mutex);
+	return NULL;
 }
 
 u32 host1x_syncpt_id(struct host1x_syncpt *sp)
@@ -378,6 +388,7 @@ int host1x_syncpt_init(struct host1x *host)
 	for (i = 0; i < host->info->nb_bases; i++)
 		bases[i].id = i;
 
+	mutex_init(&host->syncpt_mutex);
 	host->syncpt = syncpt;
 	host->bases = bases;
 
@@ -405,12 +416,16 @@ void host1x_syncpt_free(struct host1x_syncpt *sp)
 	if (!sp)
 		return;
 
+	mutex_lock(&sp->host->syncpt_mutex);
+
 	host1x_syncpt_base_free(sp->base);
 	kfree(sp->name);
 	sp->base = NULL;
 	sp->dev = NULL;
 	sp->name = NULL;
 	sp->client_managed = false;
+
+	mutex_unlock(&sp->host->syncpt_mutex);
 }
 EXPORT_SYMBOL(host1x_syncpt_free);
 
diff --git a/drivers/gpu/ipu-v3/Kconfig b/drivers/gpu/ipu-v3/Kconfig
index aefdff95356d..08766c6e7856 100644
--- a/drivers/gpu/ipu-v3/Kconfig
+++ b/drivers/gpu/ipu-v3/Kconfig
@@ -1,7 +1,6 @@
 config IMX_IPUV3_CORE
 	tristate "IPUv3 core support"
 	depends on SOC_IMX5 || SOC_IMX6Q || ARCH_MULTIPLATFORM
-	depends on RESET_CONTROLLER
 	select GENERIC_IRQ_CHIP
 	help
 	  Choose this if you have a i.MX5/6 system and want to use the Image
diff --git a/drivers/gpu/ipu-v3/ipu-common.c b/drivers/gpu/ipu-v3/ipu-common.c
index b9539f7c5e9a..97218af4fe75 100644
--- a/drivers/gpu/ipu-v3/ipu-common.c
+++ b/drivers/gpu/ipu-v3/ipu-common.c
@@ -88,6 +88,8 @@ enum ipu_color_space ipu_drm_fourcc_to_colorspace(u32 drm_fourcc)
 	case DRM_FORMAT_YVU420:
 	case DRM_FORMAT_YUV422:
 	case DRM_FORMAT_YVU422:
+	case DRM_FORMAT_YUV444:
+	case DRM_FORMAT_YVU444:
 	case DRM_FORMAT_NV12:
 	case DRM_FORMAT_NV21:
 	case DRM_FORMAT_NV16:
@@ -1284,8 +1286,11 @@ static int ipu_irq_init(struct ipu_soc *ipu)
 		return ret;
 	}
 
-	for (i = 0; i < IPU_NUM_IRQS; i += 32)
+	/* Mask and clear all interrupts */
+	for (i = 0; i < IPU_NUM_IRQS; i += 32) {
 		ipu_cm_write(ipu, 0, IPU_INT_CTRL(i / 32));
+		ipu_cm_write(ipu, ~unused[i / 32], IPU_INT_STAT(i / 32));
+	}
 
 	for (i = 0; i < IPU_NUM_IRQS; i += 32) {
 		gc = irq_get_domain_generic_chip(ipu->domain, i);
diff --git a/drivers/gpu/ipu-v3/ipu-cpmem.c b/drivers/gpu/ipu-v3/ipu-cpmem.c
index fcb7dc86167b..4b2b67113d92 100644
--- a/drivers/gpu/ipu-v3/ipu-cpmem.c
+++ b/drivers/gpu/ipu-v3/ipu-cpmem.c
@@ -417,42 +417,6 @@ void ipu_cpmem_set_yuv_planar_full(struct ipuv3_channel *ch,
 }
 EXPORT_SYMBOL_GPL(ipu_cpmem_set_yuv_planar_full);
 
-void ipu_cpmem_set_yuv_planar(struct ipuv3_channel *ch,
-			      u32 pixel_format, int stride, int height)
-{
-	int fourcc, u_offset, v_offset;
-	int uv_stride = 0;
-
-	fourcc = v4l2_pix_fmt_to_drm_fourcc(pixel_format);
-	switch (fourcc) {
-	case DRM_FORMAT_YUV420:
-		uv_stride = stride / 2;
-		u_offset = stride * height;
-		v_offset = u_offset + (uv_stride * height / 2);
-		break;
-	case DRM_FORMAT_YVU420:
-		uv_stride = stride / 2;
-		v_offset = stride * height;
-		u_offset = v_offset + (uv_stride * height / 2);
-		break;
-	case DRM_FORMAT_YUV422:
-		uv_stride = stride / 2;
-		u_offset = stride * height;
-		v_offset = u_offset + (uv_stride * height);
-		break;
-	case DRM_FORMAT_NV12:
-	case DRM_FORMAT_NV16:
-		uv_stride = stride;
-		u_offset = stride * height;
-		v_offset = 0;
-		break;
-	default:
-		return;
-	}
-	ipu_cpmem_set_yuv_planar_full(ch, uv_stride, u_offset, v_offset);
-}
-EXPORT_SYMBOL_GPL(ipu_cpmem_set_yuv_planar);
-
 static const struct ipu_rgb def_xrgb_32 = {
 	.red	= { .offset = 16, .length = 8, },
 	.green	= { .offset =  8, .length = 8, },
@@ -590,6 +554,13 @@ int ipu_cpmem_set_fmt(struct ipuv3_channel *ch, u32 drm_fourcc)
 		/* burst size */
 		ipu_ch_param_write_field(ch, IPU_FIELD_NPB, 31);
 		break;
+	case DRM_FORMAT_YUV444:
+	case DRM_FORMAT_YVU444:
+		/* pix format */
+		ipu_ch_param_write_field(ch, IPU_FIELD_PFS, 0);
+		/* burst size */
+		ipu_ch_param_write_field(ch, IPU_FIELD_NPB, 31);
+		break;
 	case DRM_FORMAT_NV12:
 		/* pix format */
 		ipu_ch_param_write_field(ch, IPU_FIELD_PFS, 4);
diff --git a/drivers/gpu/ipu-v3/ipu-csi.c b/drivers/gpu/ipu-v3/ipu-csi.c
index d6e5ded24418..63c7292f427a 100644
--- a/drivers/gpu/ipu-v3/ipu-csi.c
+++ b/drivers/gpu/ipu-v3/ipu-csi.c
@@ -529,6 +529,22 @@ void ipu_csi_set_window(struct ipu_csi *csi, struct v4l2_rect *w)
 }
 EXPORT_SYMBOL_GPL(ipu_csi_set_window);
 
+void ipu_csi_set_downsize(struct ipu_csi *csi, bool horiz, bool vert)
+{
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(&csi->lock, flags);
+
+	reg = ipu_csi_read(csi, CSI_OUT_FRM_CTRL);
+	reg &= ~(CSI_HORI_DOWNSIZE_EN | CSI_VERT_DOWNSIZE_EN);
+	reg |= (horiz ? CSI_HORI_DOWNSIZE_EN : 0) |
+	       (vert ? CSI_VERT_DOWNSIZE_EN : 0);
+	ipu_csi_write(csi, reg, CSI_OUT_FRM_CTRL);
+
+	spin_unlock_irqrestore(&csi->lock, flags);
+}
+
 void ipu_csi_set_test_generator(struct ipu_csi *csi, bool active,
 				u32 r_value, u32 g_value, u32 b_value,
 				u32 pix_clk)
diff --git a/drivers/gpu/ipu-v3/ipu-di.c b/drivers/gpu/ipu-v3/ipu-di.c
index a8d87ddd8a17..d2f1bd9d3deb 100644
--- a/drivers/gpu/ipu-v3/ipu-di.c
+++ b/drivers/gpu/ipu-v3/ipu-di.c
@@ -535,7 +535,7 @@ int ipu_di_adjust_videomode(struct ipu_di *di, struct videomode *mode)
 		return -EINVAL;
 	}
 
-	dev_warn(di->ipu->dev, "videomode adapted for IPU restrictions\n");
+	dev_dbg(di->ipu->dev, "videomode adapted for IPU restrictions\n");
 	return 0;
 }
 EXPORT_SYMBOL_GPL(ipu_di_adjust_videomode);
diff --git a/drivers/gpu/vga/vgaarb.c b/drivers/gpu/vga/vgaarb.c
index 1887f199ccb7..0f5b2dd24507 100644
--- a/drivers/gpu/vga/vgaarb.c
+++ b/drivers/gpu/vga/vgaarb.c
@@ -31,6 +31,10 @@
 
 #define pr_fmt(fmt) "vgaarb: " fmt
 
+#define vgaarb_dbg(dev, fmt, arg...)	dev_dbg(dev, "vgaarb: " fmt, ##arg)
+#define vgaarb_info(dev, fmt, arg...)	dev_info(dev, "vgaarb: " fmt, ##arg)
+#define vgaarb_err(dev, fmt, arg...)	dev_err(dev, "vgaarb: " fmt, ##arg)
+
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/pci.h>
@@ -188,6 +192,7 @@ static void vga_check_first_use(void)
 static struct vga_device *__vga_tryget(struct vga_device *vgadev,
 				       unsigned int rsrc)
 {
+	struct device *dev = &vgadev->pdev->dev;
 	unsigned int wants, legacy_wants, match;
 	struct vga_device *conflict;
 	unsigned int pci_bits;
@@ -203,8 +208,8 @@ static struct vga_device *__vga_tryget(struct vga_device *vgadev,
 	    (vgadev->decodes & VGA_RSRC_LEGACY_MEM))
 		rsrc |= VGA_RSRC_LEGACY_MEM;
 
-	pr_debug("%s: %d\n", __func__, rsrc);
-	pr_debug("%s: owns: %d\n", __func__, vgadev->owns);
+	vgaarb_dbg(dev, "%s: %d\n", __func__, rsrc);
+	vgaarb_dbg(dev, "%s: owns: %d\n", __func__, vgadev->owns);
 
 	/* Check what resources we need to acquire */
 	wants = rsrc & ~vgadev->owns;
@@ -336,9 +341,10 @@ lock_them:
 
 static void __vga_put(struct vga_device *vgadev, unsigned int rsrc)
 {
+	struct device *dev = &vgadev->pdev->dev;
 	unsigned int old_locks = vgadev->locks;
 
-	pr_debug("%s\n", __func__);
+	vgaarb_dbg(dev, "%s\n", __func__);
 
 	/* Update our counters, and account for equivalent legacy resources
 	 * if we decode them
@@ -611,7 +617,7 @@ static bool vga_arbiter_add_pci_device(struct pci_dev *pdev)
 	/* Allocate structure */
 	vgadev = kzalloc(sizeof(struct vga_device), GFP_KERNEL);
 	if (vgadev == NULL) {
-		pr_err("failed to allocate pci device\n");
+		vgaarb_err(&pdev->dev, "failed to allocate VGA arbiter data\n");
 		/*
 		 * What to do on allocation failure ? For now, let's just do
 		 * nothing, I'm not sure there is anything saner to be done.
@@ -663,7 +669,7 @@ static bool vga_arbiter_add_pci_device(struct pci_dev *pdev)
 	 */
 	if (vga_default == NULL &&
 	    ((vgadev->owns & VGA_RSRC_LEGACY_MASK) == VGA_RSRC_LEGACY_MASK)) {
-		pr_info("setting as boot device: PCI:%s\n", pci_name(pdev));
+		vgaarb_info(&pdev->dev, "setting as boot VGA device\n");
 		vga_set_default_device(pdev);
 	}
 
@@ -672,8 +678,7 @@ static bool vga_arbiter_add_pci_device(struct pci_dev *pdev)
 	/* Add to the list */
 	list_add(&vgadev->list, &vga_list);
 	vga_count++;
-	pr_info("device added: PCI:%s,decodes=%s,owns=%s,locks=%s\n",
-		pci_name(pdev),
+	vgaarb_info(&pdev->dev, "VGA device added: decodes=%s,owns=%s,locks=%s\n",
 		vga_iostate_to_str(vgadev->decodes),
 		vga_iostate_to_str(vgadev->owns),
 		vga_iostate_to_str(vgadev->locks));
@@ -725,6 +730,7 @@ bail:
 static inline void vga_update_device_decodes(struct vga_device *vgadev,
 					     int new_decodes)
 {
+	struct device *dev = &vgadev->pdev->dev;
 	int old_decodes, decodes_removed, decodes_unlocked;
 
 	old_decodes = vgadev->decodes;
@@ -732,8 +738,7 @@ static inline void vga_update_device_decodes(struct vga_device *vgadev,
 	decodes_unlocked = vgadev->locks & decodes_removed;
 	vgadev->decodes = new_decodes;
 
-	pr_info("device changed decodes: PCI:%s,olddecodes=%s,decodes=%s:owns=%s\n",
-		pci_name(vgadev->pdev),
+	vgaarb_info(dev, "changed VGA decodes: olddecodes=%s,decodes=%s:owns=%s\n",
 		vga_iostate_to_str(old_decodes),
 		vga_iostate_to_str(vgadev->decodes),
 		vga_iostate_to_str(vgadev->owns));
@@ -754,7 +759,7 @@ static inline void vga_update_device_decodes(struct vga_device *vgadev,
 	if (!(old_decodes & VGA_RSRC_LEGACY_MASK) &&
 	    new_decodes & VGA_RSRC_LEGACY_MASK)
 		vga_decode_count++;
-	pr_debug("decoding count now is: %d\n", vga_decode_count);
+	vgaarb_dbg(dev, "decoding count now is: %d\n", vga_decode_count);
 }
 
 static void __vga_set_legacy_decoding(struct pci_dev *pdev,
@@ -1022,21 +1027,16 @@ static ssize_t vga_arb_write(struct file *file, const char __user *buf,
 
 	unsigned int io_state;
 
-	char *kbuf, *curr_pos;
+	char kbuf[64], *curr_pos;
 	size_t remaining = count;
 
 	int ret_val;
 	int i;
 
-
-	kbuf = kmalloc(count + 1, GFP_KERNEL);
-	if (!kbuf)
-		return -ENOMEM;
-
-	if (copy_from_user(kbuf, buf, count)) {
-		kfree(kbuf);
+	if (count >= sizeof(kbuf))
+		return -EINVAL;
+	if (copy_from_user(kbuf, buf, count))
 		return -EFAULT;
-	}
 	curr_pos = kbuf;
 	kbuf[count] = '\0';	/* Just to make sure... */
 
@@ -1189,24 +1189,25 @@ static ssize_t vga_arb_write(struct file *file, const char __user *buf,
 				ret_val = -EPROTO;
 				goto done;
 			}
-			pr_debug("%s ==> %x:%x:%x.%x\n", curr_pos,
-				domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
-
 			pdev = pci_get_domain_bus_and_slot(domain, bus, devfn);
-			pr_debug("pdev %p\n", pdev);
 			if (!pdev) {
-				pr_err("invalid PCI address %x:%x:%x\n",
-					domain, bus, devfn);
+				pr_debug("invalid PCI address %04x:%02x:%02x.%x\n",
+					 domain, bus, PCI_SLOT(devfn),
+					 PCI_FUNC(devfn));
 				ret_val = -ENODEV;
 				goto done;
 			}
+
+			pr_debug("%s ==> %04x:%02x:%02x.%x pdev %p\n", curr_pos,
+				domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
+				pdev);
 		}
 
 		vgadev = vgadev_find(pdev);
 		pr_debug("vgadev %p\n", vgadev);
 		if (vgadev == NULL) {
 			if (pdev) {
-				pr_err("this pci device is not a vga device\n");
+				vgaarb_dbg(&pdev->dev, "not a VGA device\n");
 				pci_dev_put(pdev);
 			}
 
@@ -1226,7 +1227,7 @@ static ssize_t vga_arb_write(struct file *file, const char __user *buf,
 			}
 		}
 		if (i == MAX_USER_CARDS) {
-			pr_err("maximum user cards (%d) number reached!\n",
+			vgaarb_dbg(&pdev->dev, "maximum user cards (%d) number reached, ignoring this one!\n",
 				MAX_USER_CARDS);
 			pci_dev_put(pdev);
 			/* XXX: which value to return? */
@@ -1259,11 +1260,9 @@ static ssize_t vga_arb_write(struct file *file, const char __user *buf,
 		goto done;
 	}
 	/* If we got here, the message written is not part of the protocol! */
-	kfree(kbuf);
 	return -EPROTO;
 
 done:
-	kfree(kbuf);
 	return ret_val;
 }
 
@@ -1317,8 +1316,8 @@ static int vga_arb_release(struct inode *inode, struct file *file)
 		uc = &priv->cards[i];
 		if (uc->pdev == NULL)
 			continue;
-		pr_debug("uc->io_cnt == %d, uc->mem_cnt == %d\n",
-			 uc->io_cnt, uc->mem_cnt);
+		vgaarb_dbg(&uc->pdev->dev, "uc->io_cnt == %d, uc->mem_cnt == %d\n",
+			uc->io_cnt, uc->mem_cnt);
 		while (uc->io_cnt--)
 			vga_put(uc->pdev, VGA_RSRC_LEGACY_IO);
 		while (uc->mem_cnt--)
@@ -1371,7 +1370,7 @@ static int pci_notify(struct notifier_block *nb, unsigned long action,
 	struct pci_dev *pdev = to_pci_dev(dev);
 	bool notify = false;
 
-	pr_debug("%s\n", __func__);
+	vgaarb_dbg(dev, "%s\n", __func__);
 
 	/* For now we're only intereted in devices added and removed. I didn't
 	 * test this thing here, so someone needs to double check for the
@@ -1423,9 +1422,8 @@ static int __init vga_arb_device_init(void)
 			       PCI_ANY_ID, pdev)) != NULL)
 		vga_arbiter_add_pci_device(pdev);
 
-	pr_info("loaded\n");
-
 	list_for_each_entry(vgadev, &vga_list, list) {
+		struct device *dev = &vgadev->pdev->dev;
 #if defined(CONFIG_X86) || defined(CONFIG_IA64)
 		/*
 		 * Override vga_arbiter_add_pci_device()'s I/O based detection
@@ -1458,21 +1456,19 @@ static int __init vga_arb_device_init(void)
 				continue;
 
 			if (!vga_default_device())
-				pr_info("setting as boot device: PCI:%s\n",
-					pci_name(vgadev->pdev));
+				vgaarb_info(dev, "setting as boot device\n");
 			else if (vgadev->pdev != vga_default_device())
-				pr_info("overriding boot device: PCI:%s\n",
-					pci_name(vgadev->pdev));
+				vgaarb_info(dev, "overriding boot device\n");
 			vga_set_default_device(vgadev->pdev);
 		}
 #endif
 		if (vgadev->bridge_has_one_vga)
-			pr_info("bridge control possible %s\n",
-				pci_name(vgadev->pdev));
+			vgaarb_info(dev, "bridge control possible\n");
 		else
-			pr_info("no bridge control possible %s\n",
-				pci_name(vgadev->pdev));
+			vgaarb_info(dev, "no bridge control possible\n");
 	}
+
+	pr_info("loaded\n");
 	return rc;
 }
 subsys_initcall(vga_arb_device_init);
diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig
index cd4599c0523b..4070b7386e9d 100644
--- a/drivers/hid/Kconfig
+++ b/drivers/hid/Kconfig
@@ -138,7 +138,7 @@ config HID_ASUS
 	tristate "Asus"
 	depends on I2C_HID
 	---help---
-	Support for Asus notebook built-in keyboard via i2c.
+	Support for Asus notebook built-in keyboard and touchpad via i2c.
 
 	Supported devices:
 	- EeeBook X205TA
@@ -214,7 +214,7 @@ config HID_CMEDIA
 
 config HID_CP2112
 	tristate "Silicon Labs CP2112 HID USB-to-SMBus Bridge support"
-	depends on USB_HID && I2C && GPIOLIB
+	depends on USB_HID && I2C && GPIOLIB && GPIOLIB_IRQCHIP
 	---help---
 	Support for Silicon Labs CP2112 HID USB to SMBus Master Bridge.
 	This is a HID device driver which registers as an i2c adapter
@@ -512,6 +512,14 @@ config HID_MAGICMOUSE
 	Say Y here if you want support for the multi-touch features of the
 	Apple Wireless "Magic" Mouse and the Apple Wireless "Magic" Trackpad.
 
+config HID_MAYFLASH
+	tristate "Mayflash game controller adapter force feedback"
+	depends on HID
+	select INPUT_FF_MEMLESS
+	---help---
+	Say Y here if you have HJZ Mayflash PS3 game controller adapters
+	and want to enable force feedback support.
+
 config HID_MICROSOFT
 	tristate "Microsoft non-fully HID-compliant devices"
 	depends on HID
@@ -861,6 +869,13 @@ config THRUSTMASTER_FF
 	  a THRUSTMASTER Dual Trigger 3-in-1 or a THRUSTMASTER Ferrari GT
 	  Rumble Force or Force Feedback Wheel.
 
+config HID_UDRAW_PS3
+	tristate "THQ PS3 uDraw tablet"
+	depends on HID
+	---help---
+	  Say Y here if you want to use the THQ uDraw gaming tablet for
+	  the PS3.
+
 config HID_WACOM
 	tristate "Wacom Intuos/Graphire tablet support (USB)"
 	depends on HID
diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile
index 86b2b5785fd2..4d111f23e801 100644
--- a/drivers/hid/Makefile
+++ b/drivers/hid/Makefile
@@ -58,6 +58,7 @@ obj-$(CONFIG_HID_LOGITECH)	+= hid-logitech.o
 obj-$(CONFIG_HID_LOGITECH_DJ)	+= hid-logitech-dj.o
 obj-$(CONFIG_HID_LOGITECH_HIDPP)	+= hid-logitech-hidpp.o
 obj-$(CONFIG_HID_MAGICMOUSE)	+= hid-magicmouse.o
+obj-$(CONFIG_HID_MAYFLASH)	+= hid-mf.o
 obj-$(CONFIG_HID_MICROSOFT)	+= hid-microsoft.o
 obj-$(CONFIG_HID_MONTEREY)	+= hid-monterey.o
 obj-$(CONFIG_HID_MULTITOUCH)	+= hid-multitouch.o
@@ -96,6 +97,7 @@ obj-$(CONFIG_HID_TIVO)		+= hid-tivo.o
 obj-$(CONFIG_HID_TOPSEED)	+= hid-topseed.o
 obj-$(CONFIG_HID_TWINHAN)	+= hid-twinhan.o
 obj-$(CONFIG_HID_UCLOGIC)	+= hid-uclogic.o
+obj-$(CONFIG_HID_UDRAW_PS3)	+= hid-udraw-ps3.o
 obj-$(CONFIG_HID_LED)		+= hid-led.o
 obj-$(CONFIG_HID_XINMO)		+= hid-xinmo.o
 obj-$(CONFIG_HID_ZEROPLUS)	+= hid-zpff.o
diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c
index 7a811ec4f2e1..d40ed9fdf68d 100644
--- a/drivers/hid/hid-asus.c
+++ b/drivers/hid/hid-asus.c
@@ -11,6 +11,12 @@
  *  This module based on hid-ortek by
  *  Copyright (c) 2010 Johnathon Harris <jmharris@gmail.com>
  *  Copyright (c) 2011 Jiri Kosina
+ *
+ *  This module has been updated to add support for Asus i2c touchpad.
+ *
+ *  Copyright (c) 2016 Brendan McGrath <redmcg@redmandi.dyndns.org>
+ *  Copyright (c) 2016 Victor Vlasenko <victor.vlasenko@sysgears.com>
+ *  Copyright (c) 2016 Frederik Wenigwieser <frederik.wenigwieser@gmail.com>
  */
 
 /*
@@ -20,16 +26,287 @@
  * any later version.
  */
 
-#include <linux/device.h>
 #include <linux/hid.h>
 #include <linux/module.h>
+#include <linux/input/mt.h>
 
 #include "hid-ids.h"
 
+MODULE_AUTHOR("Yusuke Fujimaki <usk.fujimaki@gmail.com>");
+MODULE_AUTHOR("Brendan McGrath <redmcg@redmandi.dyndns.org>");
+MODULE_AUTHOR("Victor Vlasenko <victor.vlasenko@sysgears.com>");
+MODULE_AUTHOR("Frederik Wenigwieser <frederik.wenigwieser@gmail.com>");
+MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad");
+
+#define FEATURE_REPORT_ID 0x0d
+#define INPUT_REPORT_ID 0x5d
+
+#define INPUT_REPORT_SIZE 28
+
+#define MAX_CONTACTS 5
+
+#define MAX_X 2794
+#define MAX_Y 1758
+#define MAX_TOUCH_MAJOR 8
+#define MAX_PRESSURE 128
+
+#define CONTACT_DATA_SIZE 5
+
+#define BTN_LEFT_MASK 0x01
+#define CONTACT_TOOL_TYPE_MASK 0x80
+#define CONTACT_X_MSB_MASK 0xf0
+#define CONTACT_Y_MSB_MASK 0x0f
+#define CONTACT_TOUCH_MAJOR_MASK 0x07
+#define CONTACT_PRESSURE_MASK 0x7f
+
+#define QUIRK_FIX_NOTEBOOK_REPORT	BIT(0)
+#define QUIRK_NO_INIT_REPORTS		BIT(1)
+#define QUIRK_SKIP_INPUT_MAPPING	BIT(2)
+#define QUIRK_IS_MULTITOUCH		BIT(3)
+
+#define NOTEBOOK_QUIRKS			QUIRK_FIX_NOTEBOOK_REPORT
+#define TOUCHPAD_QUIRKS			(QUIRK_NO_INIT_REPORTS | \
+						 QUIRK_SKIP_INPUT_MAPPING | \
+						 QUIRK_IS_MULTITOUCH)
+
+#define TRKID_SGN       ((TRKID_MAX + 1) >> 1)
+
+struct asus_drvdata {
+	unsigned long quirks;
+	struct input_dev *input;
+};
+
+static void asus_report_contact_down(struct input_dev *input,
+		int toolType, u8 *data)
+{
+	int touch_major, pressure;
+	int x = (data[0] & CONTACT_X_MSB_MASK) << 4 | data[1];
+	int y = MAX_Y - ((data[0] & CONTACT_Y_MSB_MASK) << 8 | data[2]);
+
+	if (toolType == MT_TOOL_PALM) {
+		touch_major = MAX_TOUCH_MAJOR;
+		pressure = MAX_PRESSURE;
+	} else {
+		touch_major = (data[3] >> 4) & CONTACT_TOUCH_MAJOR_MASK;
+		pressure = data[4] & CONTACT_PRESSURE_MASK;
+	}
+
+	input_report_abs(input, ABS_MT_POSITION_X, x);
+	input_report_abs(input, ABS_MT_POSITION_Y, y);
+	input_report_abs(input, ABS_MT_TOUCH_MAJOR, touch_major);
+	input_report_abs(input, ABS_MT_PRESSURE, pressure);
+}
+
+/* Required for Synaptics Palm Detection */
+static void asus_report_tool_width(struct input_dev *input)
+{
+	struct input_mt *mt = input->mt;
+	struct input_mt_slot *oldest;
+	int oldid, count, i;
+
+	oldest = NULL;
+	oldid = mt->trkid;
+	count = 0;
+
+	for (i = 0; i < mt->num_slots; ++i) {
+		struct input_mt_slot *ps = &mt->slots[i];
+		int id = input_mt_get_value(ps, ABS_MT_TRACKING_ID);
+
+		if (id < 0)
+			continue;
+		if ((id - oldid) & TRKID_SGN) {
+			oldest = ps;
+			oldid = id;
+		}
+		count++;
+	}
+
+	if (oldest) {
+		input_report_abs(input, ABS_TOOL_WIDTH,
+			input_mt_get_value(oldest, ABS_MT_TOUCH_MAJOR));
+	}
+}
+
+static void asus_report_input(struct input_dev *input, u8 *data)
+{
+	int i;
+	u8 *contactData = data + 2;
+
+	for (i = 0; i < MAX_CONTACTS; i++) {
+		bool down = !!(data[1] & BIT(i+3));
+		int toolType = contactData[3] & CONTACT_TOOL_TYPE_MASK ?
+						MT_TOOL_PALM : MT_TOOL_FINGER;
+
+		input_mt_slot(input, i);
+		input_mt_report_slot_state(input, toolType, down);
+
+		if (down) {
+			asus_report_contact_down(input, toolType, contactData);
+			contactData += CONTACT_DATA_SIZE;
+		}
+	}
+
+	input_report_key(input, BTN_LEFT, data[1] & BTN_LEFT_MASK);
+	asus_report_tool_width(input);
+
+	input_mt_sync_frame(input);
+	input_sync(input);
+}
+
+static int asus_raw_event(struct hid_device *hdev,
+		struct hid_report *report, u8 *data, int size)
+{
+	struct asus_drvdata *drvdata = hid_get_drvdata(hdev);
+
+	if (drvdata->quirks & QUIRK_IS_MULTITOUCH &&
+					 data[0] == INPUT_REPORT_ID &&
+						size == INPUT_REPORT_SIZE) {
+		asus_report_input(drvdata->input, data);
+		return 1;
+	}
+
+	return 0;
+}
+
+static int asus_input_configured(struct hid_device *hdev, struct hid_input *hi)
+{
+	struct asus_drvdata *drvdata = hid_get_drvdata(hdev);
+
+	if (drvdata->quirks & QUIRK_IS_MULTITOUCH) {
+		int ret;
+		struct input_dev *input = hi->input;
+
+		input_set_abs_params(input, ABS_MT_POSITION_X, 0, MAX_X, 0, 0);
+		input_set_abs_params(input, ABS_MT_POSITION_Y, 0, MAX_Y, 0, 0);
+		input_set_abs_params(input, ABS_TOOL_WIDTH, 0, MAX_TOUCH_MAJOR, 0, 0);
+		input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, MAX_TOUCH_MAJOR, 0, 0);
+		input_set_abs_params(input, ABS_MT_PRESSURE, 0, MAX_PRESSURE, 0, 0);
+
+		__set_bit(BTN_LEFT, input->keybit);
+		__set_bit(INPUT_PROP_BUTTONPAD, input->propbit);
+
+		ret = input_mt_init_slots(input, MAX_CONTACTS, INPUT_MT_POINTER);
+
+		if (ret) {
+			hid_err(hdev, "Asus input mt init slots failed: %d\n", ret);
+			return ret;
+		}
+
+		drvdata->input = input;
+	}
+
+	return 0;
+}
+
+static int asus_input_mapping(struct hid_device *hdev,
+		struct hid_input *hi, struct hid_field *field,
+		struct hid_usage *usage, unsigned long **bit,
+		int *max)
+{
+	struct asus_drvdata *drvdata = hid_get_drvdata(hdev);
+
+	if (drvdata->quirks & QUIRK_SKIP_INPUT_MAPPING) {
+		/* Don't map anything from the HID report.
+		 * We do it all manually in asus_input_configured
+		 */
+		return -1;
+	}
+
+	return 0;
+}
+
+static int asus_start_multitouch(struct hid_device *hdev)
+{
+	int ret;
+	const unsigned char buf[] = { FEATURE_REPORT_ID, 0x00, 0x03, 0x01, 0x00 };
+	unsigned char *dmabuf = kmemdup(buf, sizeof(buf), GFP_KERNEL);
+
+	if (!dmabuf) {
+		ret = -ENOMEM;
+		hid_err(hdev, "Asus failed to alloc dma buf: %d\n", ret);
+		return ret;
+	}
+
+	ret = hid_hw_raw_request(hdev, dmabuf[0], dmabuf, sizeof(buf),
+					HID_FEATURE_REPORT, HID_REQ_SET_REPORT);
+
+	kfree(dmabuf);
+
+	if (ret != sizeof(buf)) {
+		hid_err(hdev, "Asus failed to start multitouch: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int __maybe_unused asus_reset_resume(struct hid_device *hdev)
+{
+	struct asus_drvdata *drvdata = hid_get_drvdata(hdev);
+
+	if (drvdata->quirks & QUIRK_IS_MULTITOUCH)
+		return asus_start_multitouch(hdev);
+
+	return 0;
+}
+
+static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id)
+{
+	int ret;
+	struct asus_drvdata *drvdata;
+
+	drvdata = devm_kzalloc(&hdev->dev, sizeof(*drvdata), GFP_KERNEL);
+	if (drvdata == NULL) {
+		hid_err(hdev, "Can't alloc Asus descriptor\n");
+		return -ENOMEM;
+	}
+
+	hid_set_drvdata(hdev, drvdata);
+
+	drvdata->quirks = id->driver_data;
+
+	if (drvdata->quirks & QUIRK_NO_INIT_REPORTS)
+		hdev->quirks |= HID_QUIRK_NO_INIT_REPORTS;
+
+	ret = hid_parse(hdev);
+	if (ret) {
+		hid_err(hdev, "Asus hid parse failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT);
+	if (ret) {
+		hid_err(hdev, "Asus hw start failed: %d\n", ret);
+		return ret;
+	}
+
+	if (!drvdata->input) {
+		hid_err(hdev, "Asus input not registered\n");
+		ret = -ENOMEM;
+		goto err_stop_hw;
+	}
+
+	drvdata->input->name = "Asus TouchPad";
+
+	if (drvdata->quirks & QUIRK_IS_MULTITOUCH) {
+		ret = asus_start_multitouch(hdev);
+		if (ret)
+			goto err_stop_hw;
+	}
+
+	return 0;
+err_stop_hw:
+	hid_hw_stop(hdev);
+	return ret;
+}
+
 static __u8 *asus_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 		unsigned int *rsize)
 {
-	if (*rsize >= 56 && rdesc[54] == 0x25 && rdesc[55] == 0x65) {
+	struct asus_drvdata *drvdata = hid_get_drvdata(hdev);
+
+	if (drvdata->quirks & QUIRK_FIX_NOTEBOOK_REPORT &&
+			*rsize >= 56 && rdesc[54] == 0x25 && rdesc[55] == 0x65) {
 		hid_info(hdev, "Fixing up Asus notebook report descriptor\n");
 		rdesc[55] = 0xdd;
 	}
@@ -37,15 +314,25 @@ static __u8 *asus_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 }
 
 static const struct hid_device_id asus_devices[] = {
-	{ HID_I2C_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_NOTEBOOK_KEYBOARD) },
+	{ HID_I2C_DEVICE(USB_VENDOR_ID_ASUSTEK,
+		 USB_DEVICE_ID_ASUSTEK_NOTEBOOK_KEYBOARD), NOTEBOOK_QUIRKS},
+	{ HID_I2C_DEVICE(USB_VENDOR_ID_ASUSTEK,
+			 USB_DEVICE_ID_ASUSTEK_TOUCHPAD), TOUCHPAD_QUIRKS },
 	{ }
 };
 MODULE_DEVICE_TABLE(hid, asus_devices);
 
 static struct hid_driver asus_driver = {
-	.name = "asus",
-	.id_table = asus_devices,
-	.report_fixup = asus_report_fixup
+	.name			= "asus",
+	.id_table		= asus_devices,
+	.report_fixup		= asus_report_fixup,
+	.probe                  = asus_probe,
+	.input_mapping          = asus_input_mapping,
+	.input_configured       = asus_input_configured,
+#ifdef CONFIG_PM
+	.reset_resume           = asus_reset_resume,
+#endif
+	.raw_event		= asus_raw_event
 };
 module_hid_driver(asus_driver);
 
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index 2b89c701076f..cff060b56da9 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -727,8 +727,9 @@ static void hid_scan_collection(struct hid_parser *parser, unsigned type)
 	    (hid->product == USB_DEVICE_ID_MS_TYPE_COVER_PRO_3 ||
 	     hid->product == USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_2 ||
 	     hid->product == USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_JP ||
+	     hid->product == USB_DEVICE_ID_MS_TYPE_COVER_PRO_4 ||
+	     hid->product == USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_2 ||
 	     hid->product == USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_JP ||
-	     hid->product == USB_DEVICE_ID_MS_TYPE_COVER_3 ||
 	     hid->product == USB_DEVICE_ID_MS_POWER_COVER) &&
 	    hid->group == HID_GROUP_MULTITOUCH)
 		hid->group = HID_GROUP_GENERIC;
@@ -1857,6 +1858,7 @@ static const struct hid_device_id hid_have_special_driver[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_TP_ONLY) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY) },
 	{ HID_I2C_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_NOTEBOOK_KEYBOARD) },
+	{ HID_I2C_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_TOUCHPAD) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_AUREAL, USB_DEVICE_ID_AUREAL_W01RN) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_BELKIN, USB_DEVICE_ID_FLIP_KVM) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_BETOP_2185BFM, 0x2208) },
@@ -1883,6 +1885,9 @@ static const struct hid_device_id hid_have_special_driver[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_DELCOM, USB_DEVICE_ID_DELCOM_VISUAL_IND) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, 0x0006) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, 0x0011) },
+#if IS_ENABLED(CONFIG_HID_MAYFLASH)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_PS3) },
+#endif
 	{ HID_USB_DEVICE(USB_VENDOR_ID_DREAM_CHEEKY, USB_DEVICE_ID_DREAM_CHEEKY_WN) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_DREAM_CHEEKY, USB_DEVICE_ID_DREAM_CHEEKY_FA) },
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) },
@@ -1983,8 +1988,9 @@ static const struct hid_device_id hid_have_special_driver[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_2) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_JP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_2) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_JP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_3) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_7K) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_600) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_3KV1) },
@@ -2059,6 +2065,9 @@ static const struct hid_device_id hid_have_special_driver[] = {
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS3_CONTROLLER) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER) },
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER_2) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER_2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER_DONGLE) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_VAIO_VGX_MOUSE) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_VAIO_VGP_MOUSE) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SINO_LITE, USB_DEVICE_ID_SINO_LITE_CONTROLLER) },
@@ -2086,6 +2095,7 @@ static const struct hid_device_id hid_have_special_driver[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_WP1062) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_WIRELESS_TABLET_TWHL850) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_TWHA60) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THQ, USB_DEVICE_ID_THQ_PS3_UDRAW) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_YIYNOVA_TABLET) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UGEE_TABLET_81) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UGEE_TABLET_45) },
diff --git a/drivers/hid/hid-cp2112.c b/drivers/hid/hid-cp2112.c
index 60d30203a5fa..f31a778b0851 100644
--- a/drivers/hid/hid-cp2112.c
+++ b/drivers/hid/hid-cp2112.c
@@ -24,6 +24,7 @@
  *   http://www.silabs.com/Support%20Documents/TechnicalDocs/AN495.pdf
  */
 
+#include <linux/gpio.h>
 #include <linux/gpio/driver.h>
 #include <linux/hid.h>
 #include <linux/i2c.h>
@@ -168,6 +169,12 @@ struct cp2112_device {
 	struct gpio_chip gc;
 	u8 *in_out_buffer;
 	spinlock_t lock;
+
+	struct gpio_desc *desc[8];
+	bool gpio_poll;
+	struct delayed_work gpio_poll_worker;
+	unsigned long irq_mask;
+	u8 gpio_prev_state;
 };
 
 static int gpio_push_pull = 0xFF;
@@ -233,7 +240,7 @@ static void cp2112_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 	spin_unlock_irqrestore(&dev->lock, flags);
 }
 
-static int cp2112_gpio_get(struct gpio_chip *chip, unsigned offset)
+static int cp2112_gpio_get_all(struct gpio_chip *chip)
 {
 	struct cp2112_device *dev = gpiochip_get_data(chip);
 	struct hid_device *hdev = dev->hdev;
@@ -252,7 +259,7 @@ static int cp2112_gpio_get(struct gpio_chip *chip, unsigned offset)
 		goto exit;
 	}
 
-	ret = (buf[1] >> offset) & 1;
+	ret = buf[1];
 
 exit:
 	spin_unlock_irqrestore(&dev->lock, flags);
@@ -260,6 +267,17 @@ exit:
 	return ret;
 }
 
+static int cp2112_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+	int ret;
+
+	ret = cp2112_gpio_get_all(chip);
+	if (ret < 0)
+		return ret;
+
+	return (ret >> offset) & 1;
+}
+
 static int cp2112_gpio_direction_output(struct gpio_chip *chip,
 					unsigned offset, int value)
 {
@@ -1041,6 +1059,166 @@ static void chmod_sysfs_attrs(struct hid_device *hdev)
 	}
 }
 
+static void cp2112_gpio_irq_ack(struct irq_data *d)
+{
+}
+
+static void cp2112_gpio_irq_mask(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct cp2112_device *dev = gpiochip_get_data(gc);
+
+	__clear_bit(d->hwirq, &dev->irq_mask);
+}
+
+static void cp2112_gpio_irq_unmask(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct cp2112_device *dev = gpiochip_get_data(gc);
+
+	__set_bit(d->hwirq, &dev->irq_mask);
+}
+
+static void cp2112_gpio_poll_callback(struct work_struct *work)
+{
+	struct cp2112_device *dev = container_of(work, struct cp2112_device,
+						 gpio_poll_worker.work);
+	struct irq_data *d;
+	u8 gpio_mask;
+	u8 virqs = (u8)dev->irq_mask;
+	u32 irq_type;
+	int irq, virq, ret;
+
+	ret = cp2112_gpio_get_all(&dev->gc);
+	if (ret == -ENODEV) /* the hardware has been disconnected */
+		return;
+	if (ret < 0)
+		goto exit;
+
+	gpio_mask = ret;
+
+	while (virqs) {
+		virq = ffs(virqs) - 1;
+		virqs &= ~BIT(virq);
+
+		if (!dev->gc.to_irq)
+			break;
+
+		irq = dev->gc.to_irq(&dev->gc, virq);
+
+		d = irq_get_irq_data(irq);
+		if (!d)
+			continue;
+
+		irq_type = irqd_get_trigger_type(d);
+
+		if (gpio_mask & BIT(virq)) {
+			/* Level High */
+
+			if (irq_type & IRQ_TYPE_LEVEL_HIGH)
+				handle_nested_irq(irq);
+
+			if ((irq_type & IRQ_TYPE_EDGE_RISING) &&
+			    !(dev->gpio_prev_state & BIT(virq)))
+				handle_nested_irq(irq);
+		} else {
+			/* Level Low */
+
+			if (irq_type & IRQ_TYPE_LEVEL_LOW)
+				handle_nested_irq(irq);
+
+			if ((irq_type & IRQ_TYPE_EDGE_FALLING) &&
+			    (dev->gpio_prev_state & BIT(virq)))
+				handle_nested_irq(irq);
+		}
+	}
+
+	dev->gpio_prev_state = gpio_mask;
+
+exit:
+	if (dev->gpio_poll)
+		schedule_delayed_work(&dev->gpio_poll_worker, 10);
+}
+
+
+static unsigned int cp2112_gpio_irq_startup(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct cp2112_device *dev = gpiochip_get_data(gc);
+
+	INIT_DELAYED_WORK(&dev->gpio_poll_worker, cp2112_gpio_poll_callback);
+
+	cp2112_gpio_direction_input(gc, d->hwirq);
+
+	if (!dev->gpio_poll) {
+		dev->gpio_poll = true;
+		schedule_delayed_work(&dev->gpio_poll_worker, 0);
+	}
+
+	cp2112_gpio_irq_unmask(d);
+	return 0;
+}
+
+static void cp2112_gpio_irq_shutdown(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct cp2112_device *dev = gpiochip_get_data(gc);
+
+	cancel_delayed_work_sync(&dev->gpio_poll_worker);
+}
+
+static int cp2112_gpio_irq_type(struct irq_data *d, unsigned int type)
+{
+	return 0;
+}
+
+static struct irq_chip cp2112_gpio_irqchip = {
+	.name = "cp2112-gpio",
+	.irq_startup = cp2112_gpio_irq_startup,
+	.irq_shutdown = cp2112_gpio_irq_shutdown,
+	.irq_ack = cp2112_gpio_irq_ack,
+	.irq_mask = cp2112_gpio_irq_mask,
+	.irq_unmask = cp2112_gpio_irq_unmask,
+	.irq_set_type = cp2112_gpio_irq_type,
+};
+
+static int __maybe_unused cp2112_allocate_irq(struct cp2112_device *dev,
+					      int pin)
+{
+	int ret;
+
+	if (dev->desc[pin])
+		return -EINVAL;
+
+	dev->desc[pin] = gpiochip_request_own_desc(&dev->gc, pin,
+						   "HID/I2C:Event");
+	if (IS_ERR(dev->desc[pin])) {
+		dev_err(dev->gc.parent, "Failed to request GPIO\n");
+		return PTR_ERR(dev->desc[pin]);
+	}
+
+	ret = gpiochip_lock_as_irq(&dev->gc, pin);
+	if (ret) {
+		dev_err(dev->gc.parent, "Failed to lock GPIO as interrupt\n");
+		goto err_desc;
+	}
+
+	ret = gpiod_to_irq(dev->desc[pin]);
+	if (ret < 0) {
+		dev_err(dev->gc.parent, "Failed to translate GPIO to IRQ\n");
+		goto err_lock;
+	}
+
+	return ret;
+
+err_lock:
+	gpiochip_unlock_as_irq(&dev->gc, pin);
+err_desc:
+	gpiochip_free_own_desc(dev->desc[pin]);
+	dev->desc[pin] = NULL;
+	return ret;
+}
+
 static int cp2112_probe(struct hid_device *hdev, const struct hid_device_id *id)
 {
 	struct cp2112_device *dev;
@@ -1163,8 +1341,17 @@ static int cp2112_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	chmod_sysfs_attrs(hdev);
 	hid_hw_power(hdev, PM_HINT_NORMAL);
 
+	ret = gpiochip_irqchip_add(&dev->gc, &cp2112_gpio_irqchip, 0,
+				   handle_simple_irq, IRQ_TYPE_NONE);
+	if (ret) {
+		dev_err(dev->gc.parent, "failed to add IRQ chip\n");
+		goto err_sysfs_remove;
+	}
+
 	return ret;
 
+err_sysfs_remove:
+	sysfs_remove_group(&hdev->dev.kobj, &cp2112_attr_group);
 err_gpiochip_remove:
 	gpiochip_remove(&dev->gc);
 err_free_i2c:
@@ -1181,10 +1368,22 @@ err_hid_stop:
 static void cp2112_remove(struct hid_device *hdev)
 {
 	struct cp2112_device *dev = hid_get_drvdata(hdev);
+	int i;
 
 	sysfs_remove_group(&hdev->dev.kobj, &cp2112_attr_group);
-	gpiochip_remove(&dev->gc);
 	i2c_del_adapter(&dev->adap);
+
+	if (dev->gpio_poll) {
+		dev->gpio_poll = false;
+		cancel_delayed_work_sync(&dev->gpio_poll_worker);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(dev->desc); i++) {
+		gpiochip_unlock_as_irq(&dev->gc, i);
+		gpiochip_free_own_desc(dev->desc[i]);
+	}
+
+	gpiochip_remove(&dev->gc);
 	/* i2c_del_adapter has finished removing all i2c devices from our
 	 * adapter. Well behaved devices should no longer call our cp2112_xfer
 	 * and should have waited for any pending calls to finish. It has also
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index 575aa65436d1..ec277b96eaa1 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -171,6 +171,7 @@
 #define USB_DEVICE_ID_ASUSTEK_LCM	0x1726
 #define USB_DEVICE_ID_ASUSTEK_LCM2	0x175b
 #define USB_DEVICE_ID_ASUSTEK_NOTEBOOK_KEYBOARD	0x8585
+#define USB_DEVICE_ID_ASUSTEK_TOUCHPAD	0x0101
 
 #define USB_VENDOR_ID_ATEN		0x0557
 #define USB_DEVICE_ID_ATEN_UC100KM	0x2004
@@ -315,8 +316,10 @@
 #define USB_VENDOR_ID_DMI		0x0c0b
 #define USB_DEVICE_ID_DMI_ENC		0x5fab
 
-#define USB_VENDOR_ID_DRAGONRISE	0x0079
-#define USB_DEVICE_ID_DRAGONRISE_WIIU	0x1800
+#define USB_VENDOR_ID_DRAGONRISE		0x0079
+#define USB_DEVICE_ID_DRAGONRISE_WIIU		0x1800
+#define USB_DEVICE_ID_DRAGONRISE_PS3		0x1801
+#define USB_DEVICE_ID_DRAGONRISE_GAMECUBE	0x1843
 
 #define USB_VENDOR_ID_DWAV		0x0eef
 #define USB_DEVICE_ID_EGALAX_TOUCHCONTROLLER	0x0001
@@ -718,8 +721,9 @@
 #define USB_DEVICE_ID_MS_TYPE_COVER_PRO_3    0x07dc
 #define USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_2  0x07e2
 #define USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_JP 0x07dd
+#define USB_DEVICE_ID_MS_TYPE_COVER_PRO_4 0x07e4
+#define USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_2 0x07e8
 #define USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_JP 0x07e9
-#define USB_DEVICE_ID_MS_TYPE_COVER_3    0x07de
 #define USB_DEVICE_ID_MS_POWER_COVER     0x07da
 
 #define USB_VENDOR_ID_MOJO		0x8282
@@ -903,6 +907,8 @@
 #define USB_DEVICE_ID_SONY_PS3_BDREMOTE		0x0306
 #define USB_DEVICE_ID_SONY_PS3_CONTROLLER	0x0268
 #define USB_DEVICE_ID_SONY_PS4_CONTROLLER	0x05c4
+#define USB_DEVICE_ID_SONY_PS4_CONTROLLER_2	0x09cc
+#define USB_DEVICE_ID_SONY_PS4_CONTROLLER_DONGLE	0x0ba0
 #define USB_DEVICE_ID_SONY_MOTION_CONTROLLER	0x03d5
 #define USB_DEVICE_ID_SONY_NAVIGATION_CONTROLLER	0x042f
 #define USB_DEVICE_ID_SONY_BUZZ_CONTROLLER		0x0002
@@ -959,6 +965,9 @@
 #define USB_VENDOR_ID_THINGM		0x27b8
 #define USB_DEVICE_ID_BLINK1		0x01ed
 
+#define USB_VENDOR_ID_THQ		0x20d6
+#define USB_DEVICE_ID_THQ_PS3_UDRAW	0xcb17
+
 #define USB_VENDOR_ID_THRUSTMASTER	0x044f
 
 #define USB_VENDOR_ID_TIVO		0x150a
@@ -1034,6 +1043,10 @@
 #define USB_DEVICE_ID_WALTOP_MEDIA_TABLET_14_1_INCH	0x0500
 #define USB_DEVICE_ID_WALTOP_SIRIUS_BATTERY_FREE_TABLET	0x0502
 
+#define	USB_VENDOR_ID_WEIDA		0x2575
+#define	USB_DEVICE_ID_WEIDA_8752	0xC300
+#define	USB_DEVICE_ID_WEIDA_8755	0xC301
+
 #define USB_VENDOR_ID_WISEGROUP		0x0925
 #define USB_DEVICE_ID_SMARTJOY_PLUS	0x0005
 #define USB_DEVICE_ID_SUPER_JOY_BOX_3	0x8888
diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c
index fb9ace1cef8b..d05f903c7614 100644
--- a/drivers/hid/hid-input.c
+++ b/drivers/hid/hid-input.c
@@ -253,6 +253,7 @@ __s32 hidinput_calc_abs_res(const struct hid_field *field, __u16 code)
 	case ABS_RX:
 	case ABS_RY:
 	case ABS_RZ:
+	case ABS_WHEEL:
 	case ABS_TILT_X:
 	case ABS_TILT_Y:
 		if (field->unit == 0x14) {		/* If degrees */
@@ -1468,6 +1469,31 @@ static void hidinput_cleanup_hidinput(struct hid_device *hid,
 	kfree(hidinput);
 }
 
+static struct hid_input *hidinput_match(struct hid_report *report)
+{
+	struct hid_device *hid = report->device;
+	struct hid_input *hidinput;
+
+	list_for_each_entry(hidinput, &hid->inputs, list) {
+		if (hidinput->report &&
+		    hidinput->report->id == report->id)
+			return hidinput;
+	}
+
+	return NULL;
+}
+
+static inline void hidinput_configure_usages(struct hid_input *hidinput,
+					     struct hid_report *report)
+{
+	int i, j;
+
+	for (i = 0; i < report->maxfield; i++)
+		for (j = 0; j < report->field[i]->maxusage; j++)
+			hidinput_configure_usage(hidinput, report->field[i],
+						 report->field[i]->usage + j);
+}
+
 /*
  * Register the input device; print a message.
  * Configure the input layer interface
@@ -1478,8 +1504,8 @@ int hidinput_connect(struct hid_device *hid, unsigned int force)
 {
 	struct hid_driver *drv = hid->driver;
 	struct hid_report *report;
-	struct hid_input *hidinput = NULL;
-	int i, j, k;
+	struct hid_input *next, *hidinput = NULL;
+	int i, k;
 
 	INIT_LIST_HEAD(&hid->inputs);
 	INIT_WORK(&hid->led_work, hidinput_led_worker);
@@ -1509,43 +1535,40 @@ int hidinput_connect(struct hid_device *hid, unsigned int force)
 			if (!report->maxfield)
 				continue;
 
+			/*
+			 * Find the previous hidinput report attached
+			 * to this report id.
+			 */
+			if (hid->quirks & HID_QUIRK_MULTI_INPUT)
+				hidinput = hidinput_match(report);
+
 			if (!hidinput) {
 				hidinput = hidinput_allocate(hid);
 				if (!hidinput)
 					goto out_unwind;
 			}
 
-			for (i = 0; i < report->maxfield; i++)
-				for (j = 0; j < report->field[i]->maxusage; j++)
-					hidinput_configure_usage(hidinput, report->field[i],
-								 report->field[i]->usage + j);
-
-			if ((hid->quirks & HID_QUIRK_NO_EMPTY_INPUT) &&
-			    !hidinput_has_been_populated(hidinput))
-				continue;
+			hidinput_configure_usages(hidinput, report);
 
-			if (hid->quirks & HID_QUIRK_MULTI_INPUT) {
-				/* This will leave hidinput NULL, so that it
-				 * allocates another one if we have more inputs on
-				 * the same interface. Some devices (e.g. Happ's
-				 * UGCI) cram a lot of unrelated inputs into the
-				 * same interface. */
+			if (hid->quirks & HID_QUIRK_MULTI_INPUT)
 				hidinput->report = report;
-				if (drv->input_configured &&
-				    drv->input_configured(hid, hidinput))
-					goto out_cleanup;
-				if (input_register_device(hidinput->input))
-					goto out_cleanup;
-				hidinput = NULL;
-			}
 		}
 	}
 
-	if (hidinput && (hid->quirks & HID_QUIRK_NO_EMPTY_INPUT) &&
-	    !hidinput_has_been_populated(hidinput)) {
-		/* no need to register an input device not populated */
-		hidinput_cleanup_hidinput(hid, hidinput);
-		hidinput = NULL;
+	list_for_each_entry_safe(hidinput, next, &hid->inputs, list) {
+		if ((hid->quirks & HID_QUIRK_NO_EMPTY_INPUT) &&
+		    !hidinput_has_been_populated(hidinput)) {
+			/* no need to register an input device not populated */
+			hidinput_cleanup_hidinput(hid, hidinput);
+			continue;
+		}
+
+		if (drv->input_configured &&
+		    drv->input_configured(hid, hidinput))
+			goto out_unwind;
+		if (input_register_device(hidinput->input))
+			goto out_unwind;
+		hidinput->registered = true;
 	}
 
 	if (list_empty(&hid->inputs)) {
@@ -1553,20 +1576,8 @@ int hidinput_connect(struct hid_device *hid, unsigned int force)
 		goto out_unwind;
 	}
 
-	if (hidinput) {
-		if (drv->input_configured &&
-		    drv->input_configured(hid, hidinput))
-			goto out_cleanup;
-		if (input_register_device(hidinput->input))
-			goto out_cleanup;
-	}
-
 	return 0;
 
-out_cleanup:
-	list_del(&hidinput->list);
-	input_free_device(hidinput->input);
-	kfree(hidinput);
 out_unwind:
 	/* unwind the ones we already registered */
 	hidinput_disconnect(hid);
@@ -1583,7 +1594,10 @@ void hidinput_disconnect(struct hid_device *hid)
 
 	list_for_each_entry_safe(hidinput, next, &hid->inputs, list) {
 		list_del(&hidinput->list);
-		input_unregister_device(hidinput->input);
+		if (hidinput->registered)
+			input_unregister_device(hidinput->input);
+		else
+			input_free_device(hidinput->input);
 		kfree(hidinput);
 	}
 
diff --git a/drivers/hid/hid-mf.c b/drivers/hid/hid-mf.c
new file mode 100644
index 000000000000..d9090765a6e5
--- /dev/null
+++ b/drivers/hid/hid-mf.c
@@ -0,0 +1,166 @@
+/*
+ * Force feedback support for Mayflash game controller adapters.
+ *
+ * These devices are manufactured by Mayflash but identify themselves
+ * using the vendor ID of DragonRise Inc.
+ *
+ * Tested with:
+ * 0079:1801 "DragonRise Inc. Mayflash PS3 Game Controller Adapter"
+ *
+ * The following adapters probably work too, but need to be tested:
+ * 0079:1800 "DragonRise Inc. Mayflash WIIU Game Controller Adapter"
+ * 0079:1843 "DragonRise Inc. Mayflash GameCube Game Controller Adapter"
+ *
+ * Copyright (c) 2016 Marcel Hasler <mahasler@gmail.com>
+ */
+
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/input.h>
+#include <linux/slab.h>
+#include <linux/hid.h>
+#include <linux/module.h>
+
+#include "hid-ids.h"
+
+struct mf_device {
+	struct hid_report *report;
+};
+
+static int mf_play(struct input_dev *dev, void *data, struct ff_effect *effect)
+{
+	struct hid_device *hid = input_get_drvdata(dev);
+	struct mf_device *mf = data;
+	int strong, weak;
+
+	strong = effect->u.rumble.strong_magnitude;
+	weak = effect->u.rumble.weak_magnitude;
+
+	dbg_hid("Called with 0x%04x 0x%04x.\n", strong, weak);
+
+	strong = strong * 0xff / 0xffff;
+	weak = weak * 0xff / 0xffff;
+
+	dbg_hid("Running with 0x%02x 0x%02x.\n", strong, weak);
+
+	mf->report->field[0]->value[0] = weak;
+	mf->report->field[0]->value[1] = strong;
+	hid_hw_request(hid, mf->report, HID_REQ_SET_REPORT);
+
+	return 0;
+}
+
+static int mf_init(struct hid_device *hid)
+{
+	struct mf_device *mf;
+
+	struct list_head *report_list =
+			&hid->report_enum[HID_OUTPUT_REPORT].report_list;
+
+	struct list_head *report_ptr;
+	struct hid_report *report;
+
+	struct list_head *input_ptr = &hid->inputs;
+	struct hid_input *input;
+
+	struct input_dev *dev;
+
+	int error;
+
+	/* Setup each of the four inputs */
+	list_for_each(report_ptr, report_list) {
+		report = list_entry(report_ptr, struct hid_report, list);
+
+		if (report->maxfield < 1 || report->field[0]->report_count < 2) {
+			hid_err(hid, "Invalid report, this should never happen!\n");
+			return -ENODEV;
+		}
+
+		if (list_is_last(input_ptr, &hid->inputs)) {
+			hid_err(hid, "Missing input, this should never happen!\n");
+			return -ENODEV;
+		}
+
+		input_ptr = input_ptr->next;
+		input = list_entry(input_ptr, struct hid_input, list);
+
+		mf = kzalloc(sizeof(struct mf_device), GFP_KERNEL);
+		if (!mf)
+			return -ENOMEM;
+
+		dev = input->input;
+		set_bit(FF_RUMBLE, dev->ffbit);
+
+		error = input_ff_create_memless(dev, mf, mf_play);
+		if (error) {
+			kfree(mf);
+			return error;
+		}
+
+		mf->report = report;
+		mf->report->field[0]->value[0] = 0x00;
+		mf->report->field[0]->value[1] = 0x00;
+		hid_hw_request(hid, mf->report, HID_REQ_SET_REPORT);
+	}
+
+	hid_info(hid, "Force feedback for HJZ Mayflash game controller "
+		      "adapters by Marcel Hasler <mahasler@gmail.com>\n");
+
+	return 0;
+}
+
+static int mf_probe(struct hid_device *hid, const struct hid_device_id *id)
+{
+	int error;
+
+	dev_dbg(&hid->dev, "Mayflash HID hardware probe...\n");
+
+	/* Split device into four inputs */
+	hid->quirks |= HID_QUIRK_MULTI_INPUT;
+
+	error = hid_parse(hid);
+	if (error) {
+		hid_err(hid, "HID parse failed.\n");
+		return error;
+	}
+
+	error = hid_hw_start(hid, HID_CONNECT_DEFAULT & ~HID_CONNECT_FF);
+	if (error) {
+		hid_err(hid, "HID hw start failed\n");
+		return error;
+	}
+
+	error = mf_init(hid);
+	if (error) {
+		hid_err(hid, "Force feedback init failed.\n");
+		hid_hw_stop(hid);
+		return error;
+	}
+
+	return 0;
+}
+
+static const struct hid_device_id mf_devices[] = {
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_PS3),  },
+	{ }
+};
+MODULE_DEVICE_TABLE(hid, mf_devices);
+
+static struct hid_driver mf_driver = {
+	.name = "hid_mf",
+	.id_table = mf_devices,
+	.probe = mf_probe,
+};
+module_hid_driver(mf_driver);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/hid/hid-microsoft.c b/drivers/hid/hid-microsoft.c
index c6cd392e9f99..74b7b84a0420 100644
--- a/drivers/hid/hid-microsoft.c
+++ b/drivers/hid/hid-microsoft.c
@@ -280,9 +280,11 @@ static const struct hid_device_id ms_devices[] = {
 		.driver_data = MS_HIDINPUT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_JP),
 		.driver_data = MS_HIDINPUT },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_JP),
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4),
+		.driver_data = MS_HIDINPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_2),
 		.driver_data = MS_HIDINPUT },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_3),
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_JP),
 		.driver_data = MS_HIDINPUT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_POWER_COVER),
 		.driver_data = MS_HIDINPUT },
diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c
index fb6f1f447279..6dca66806844 100644
--- a/drivers/hid/hid-multitouch.c
+++ b/drivers/hid/hid-multitouch.c
@@ -108,6 +108,7 @@ struct mt_device {
 	int cc_value_index;	/* contact count value index in the field */
 	unsigned last_slot_field;	/* the last field of a slot */
 	unsigned mt_report_id;	/* the report ID of the multitouch device */
+	unsigned long initial_quirks;	/* initial quirks state */
 	__s16 inputmode;	/* InputMode HID feature, -1 if non-existent */
 	__s16 inputmode_index;	/* InputMode HID feature index in the report */
 	__s16 maxcontact_report_id;	/* Maximum Contact Number HID feature,
@@ -318,13 +319,10 @@ static void mt_get_feature(struct hid_device *hdev, struct hid_report *report)
 	u8 *buf;
 
 	/*
-	 * Only fetch the feature report if initial reports are not already
-	 * been retrieved. Currently this is only done for Windows 8 touch
-	 * devices.
+	 * Do not fetch the feature report if the device has been explicitly
+	 * marked as non-capable.
 	 */
-	if (!(hdev->quirks & HID_QUIRK_NO_INIT_REPORTS))
-		return;
-	if (td->mtclass.name != MT_CLS_WIN_8)
+	if (td->initial_quirks & HID_QUIRK_NO_INIT_REPORTS)
 		return;
 
 	buf = hid_alloc_report_buf(report, GFP_KERNEL);
@@ -567,6 +565,14 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi,
 
 	case HID_UP_BUTTON:
 		code = BTN_MOUSE + ((usage->hid - 1) & HID_USAGE);
+		/*
+		 * MS PTP spec says that external buttons left and right have
+		 * usages 2 and 3.
+		 */
+		if (cls->name == MT_CLS_WIN_8 &&
+		    field->application == HID_DG_TOUCHPAD &&
+		    (usage->hid & HID_USAGE) > 1)
+			code--;
 		hid_map_usage(hi, usage, bit, max, EV_KEY, code);
 		input_set_capability(hi->input, EV_KEY, code);
 		return 1;
@@ -842,7 +848,9 @@ static int mt_input_mapping(struct hid_device *hdev, struct hid_input *hi,
 	if (!td->mtclass.export_all_inputs &&
 	    field->application != HID_DG_TOUCHSCREEN &&
 	    field->application != HID_DG_PEN &&
-	    field->application != HID_DG_TOUCHPAD)
+	    field->application != HID_DG_TOUCHPAD &&
+	    field->application != HID_GD_KEYBOARD &&
+	    field->application != HID_CP_CONSUMER_CONTROL)
 		return -1;
 
 	/*
@@ -1083,36 +1091,6 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		}
 	}
 
-	/* This allows the driver to correctly support devices
-	 * that emit events over several HID messages.
-	 */
-	hdev->quirks |= HID_QUIRK_NO_INPUT_SYNC;
-
-	/*
-	 * This allows the driver to handle different input sensors
-	 * that emits events through different reports on the same HID
-	 * device.
-	 */
-	hdev->quirks |= HID_QUIRK_MULTI_INPUT;
-	hdev->quirks |= HID_QUIRK_NO_EMPTY_INPUT;
-
-	/*
-	 * Handle special quirks for Windows 8 certified devices.
-	 */
-	if (id->group == HID_GROUP_MULTITOUCH_WIN_8)
-		/*
-		 * Some multitouch screens do not like to be polled for input
-		 * reports. Fortunately, the Win8 spec says that all touches
-		 * should be sent during each report, making the initialization
-		 * of input reports unnecessary.
-		 *
-		 * In addition some touchpads do not behave well if we read
-		 * all feature reports from them. Instead we prevent
-		 * initial report fetching and then selectively fetch each
-		 * report we are interested in.
-		 */
-		hdev->quirks |= HID_QUIRK_NO_INIT_REPORTS;
-
 	td = devm_kzalloc(&hdev->dev, sizeof(struct mt_device), GFP_KERNEL);
 	if (!td) {
 		dev_err(&hdev->dev, "cannot allocate multitouch data\n");
@@ -1136,6 +1114,39 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	if (id->vendor == HID_ANY_ID && id->product == HID_ANY_ID)
 		td->serial_maybe = true;
 
+	/*
+	 * Store the initial quirk state
+	 */
+	td->initial_quirks = hdev->quirks;
+
+	/* This allows the driver to correctly support devices
+	 * that emit events over several HID messages.
+	 */
+	hdev->quirks |= HID_QUIRK_NO_INPUT_SYNC;
+
+	/*
+	 * This allows the driver to handle different input sensors
+	 * that emits events through different reports on the same HID
+	 * device.
+	 */
+	hdev->quirks |= HID_QUIRK_MULTI_INPUT;
+	hdev->quirks |= HID_QUIRK_NO_EMPTY_INPUT;
+
+	/*
+	 * Some multitouch screens do not like to be polled for input
+	 * reports. Fortunately, the Win8 spec says that all touches
+	 * should be sent during each report, making the initialization
+	 * of input reports unnecessary. For Win7 devices, well, let's hope
+	 * they will still be happy (this is only be a problem if a touch
+	 * was already there while probing the device).
+	 *
+	 * In addition some touchpads do not behave well if we read
+	 * all feature reports from them. Instead we prevent
+	 * initial report fetching and then selectively fetch each
+	 * report we are interested in.
+	 */
+	hdev->quirks |= HID_QUIRK_NO_INIT_REPORTS;
+
 	ret = hid_parse(hdev);
 	if (ret != 0)
 		return ret;
@@ -1204,8 +1215,11 @@ static int mt_resume(struct hid_device *hdev)
 
 static void mt_remove(struct hid_device *hdev)
 {
+	struct mt_device *td = hid_get_drvdata(hdev);
+
 	sysfs_remove_group(&hdev->dev.kobj, &mt_attribute_group);
 	hid_hw_stop(hdev);
+	hdev->quirks = td->initial_quirks;
 }
 
 /*
diff --git a/drivers/hid/hid-sensor-hub.c b/drivers/hid/hid-sensor-hub.c
index 60875625cbdf..5c925228847c 100644
--- a/drivers/hid/hid-sensor-hub.c
+++ b/drivers/hid/hid-sensor-hub.c
@@ -795,6 +795,12 @@ static const struct hid_device_id sensor_hub_devices[] = {
 	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_MICROSOFT,
 			USB_DEVICE_ID_MS_TYPE_COVER_2),
 			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
+	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_MICROSOFT,
+			0x07bd), /* Microsoft Surface 3 */
+			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
+	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_MICROCHIP,
+			0x0f01), /* MM7150 */
+			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
 	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_STM_0,
 			USB_DEVICE_ID_STM_HID_SENSOR),
 			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c
index b0bb99a821bd..7687c0875395 100644
--- a/drivers/hid/hid-sony.c
+++ b/drivers/hid/hid-sony.c
@@ -36,6 +36,8 @@
 #include <linux/list.h>
 #include <linux/idr.h>
 #include <linux/input/mt.h>
+#include <linux/crc32.h>
+#include <asm/unaligned.h>
 
 #include "hid-ids.h"
 
@@ -374,7 +376,7 @@ static u8 dualshock4_usb_rdesc[] = {
 	0x65, 0x00,         /*      Unit,                           */
 	0x05, 0x09,         /*      Usage Page (Button),            */
 	0x19, 0x01,         /*      Usage Minimum (01h),            */
-	0x29, 0x0E,         /*      Usage Maximum (0Eh),            */
+	0x29, 0x0D,         /*      Usage Maximum (0Dh),            */
 	0x15, 0x00,         /*      Logical Minimum (0),            */
 	0x25, 0x01,         /*      Logical Maximum (1),            */
 	0x75, 0x01,         /*      Report Size (1),                */
@@ -403,14 +405,14 @@ static u8 dualshock4_usb_rdesc[] = {
 	0x19, 0x40,         /*      Usage Minimum (40h),            */
 	0x29, 0x42,         /*      Usage Maximum (42h),            */
 	0x16, 0x00, 0x80,   /*      Logical Minimum (-32768),       */
-	0x26, 0x00, 0x7F,   /*      Logical Maximum (32767),        */
+	0x26, 0xFF, 0x7F,   /*      Logical Maximum (32767),        */
 	0x75, 0x10,         /*      Report Size (16),               */
 	0x95, 0x03,         /*      Report Count (3),               */
 	0x81, 0x02,         /*      Input (Variable),               */
 	0x19, 0x43,         /*      Usage Minimum (43h),            */
 	0x29, 0x45,         /*      Usage Maximum (45h),            */
-	0x16, 0x00, 0xE0,   /*      Logical Minimum (-8192),        */
-	0x26, 0xFF, 0x1F,   /*      Logical Maximum (8191),         */
+	0x16, 0x00, 0x80,   /*      Logical Minimum (-32768),       */
+	0x26, 0xFF, 0x7F,   /*      Logical Maximum (32767),        */
 	0x95, 0x03,         /*      Report Count (3),               */
 	0x81, 0x02,         /*      Input (Variable),               */
 	0x06, 0x00, 0xFF,   /*      Usage Page (FF00h),             */
@@ -687,7 +689,7 @@ static u8 dualshock4_bt_rdesc[] = {
 	0x81, 0x42,         /*      Input (Variable, Null State),   */
 	0x05, 0x09,         /*      Usage Page (Button),            */
 	0x19, 0x01,         /*      Usage Minimum (01h),            */
-	0x29, 0x0E,         /*      Usage Maximum (0Eh),            */
+	0x29, 0x0D,         /*      Usage Maximum (0Dh),            */
 	0x15, 0x00,         /*      Logical Minimum (0),            */
 	0x25, 0x01,         /*      Logical Maximum (1),            */
 	0x75, 0x01,         /*      Report Size (1),                */
@@ -712,14 +714,14 @@ static u8 dualshock4_bt_rdesc[] = {
 	0x19, 0x40,         /*      Usage Minimum (40h),            */
 	0x29, 0x42,         /*      Usage Maximum (42h),            */
 	0x16, 0x00, 0x80,   /*      Logical Minimum (-32768),       */
-	0x26, 0x00, 0x7F,   /*      Logical Maximum (32767),        */
+	0x26, 0xFF, 0x7F,   /*      Logical Maximum (32767),        */
 	0x75, 0x10,         /*      Report Size (16),               */
 	0x95, 0x03,         /*      Report Count (3),               */
 	0x81, 0x02,         /*      Input (Variable),               */
 	0x19, 0x43,         /*      Usage Minimum (43h),            */
 	0x29, 0x45,         /*      Usage Maximum (45h),            */
-	0x16, 0x00, 0xE0,   /*      Logical Minimum (-8192),        */
-	0x26, 0xFF, 0x1F,   /*      Logical Maximum (8191),         */
+	0x16, 0x00, 0x80,   /*      Logical Minimum (-32768),       */
+	0x26, 0xFF, 0x7F,   /*      Logical Maximum (32767),        */
 	0x95, 0x03,         /*      Report Count (3),               */
 	0x81, 0x02,         /*      Input (Variable),               */
 	0x06, 0x00, 0xFF,   /*      Usage Page (FF00h),             */
@@ -975,6 +977,32 @@ static const unsigned int buzz_keymap[] = {
 	[20] = BTN_TRIGGER_HAPPY20,
 };
 
+static const unsigned int ds4_absmap[] = {
+	[0x30] = ABS_X,
+	[0x31] = ABS_Y,
+	[0x32] = ABS_RX, /* right stick X */
+	[0x33] = ABS_Z, /* L2 */
+	[0x34] = ABS_RZ, /* R2 */
+	[0x35] = ABS_RY, /* right stick Y */
+};
+
+static const unsigned int ds4_keymap[] = {
+	[0x1] = BTN_WEST, /* Square */
+	[0x2] = BTN_SOUTH, /* Cross */
+	[0x3] = BTN_EAST, /* Circle */
+	[0x4] = BTN_NORTH, /* Triangle */
+	[0x5] = BTN_TL, /* L1 */
+	[0x6] = BTN_TR, /* R1 */
+	[0x7] = BTN_TL2, /* L2 */
+	[0x8] = BTN_TR2, /* R2 */
+	[0x9] = BTN_SELECT, /* Share */
+	[0xa] = BTN_START, /* Options */
+	[0xb] = BTN_THUMBL, /* L3 */
+	[0xc] = BTN_THUMBR, /* R3 */
+	[0xd] = BTN_MODE, /* PS */
+};
+
+
 static enum power_supply_property sony_battery_props[] = {
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_CAPACITY,
@@ -1019,14 +1047,24 @@ struct motion_output_report_02 {
 	u8 rumble;
 };
 
-#define DS4_REPORT_0x02_SIZE 37
-#define DS4_REPORT_0x05_SIZE 32
-#define DS4_REPORT_0x11_SIZE 78
-#define DS4_REPORT_0x81_SIZE 7
+#define DS4_FEATURE_REPORT_0x02_SIZE 37
+#define DS4_FEATURE_REPORT_0x81_SIZE 7
+#define DS4_INPUT_REPORT_0x11_SIZE 78
+#define DS4_OUTPUT_REPORT_0x05_SIZE 32
+#define DS4_OUTPUT_REPORT_0x11_SIZE 78
 #define SIXAXIS_REPORT_0xF2_SIZE 17
 #define SIXAXIS_REPORT_0xF5_SIZE 8
 #define MOTION_REPORT_0x02_SIZE 49
 
+/* Offsets relative to USB input report (0x1). Bluetooth (0x11) requires an
+ * additional +2.
+ */
+#define DS4_INPUT_REPORT_BUTTON_OFFSET    5
+#define DS4_INPUT_REPORT_BATTERY_OFFSET  30
+#define DS4_INPUT_REPORT_TOUCHPAD_OFFSET 33
+
+#define DS4_TOUCHPAD_SUFFIX " Touchpad"
+
 static DEFINE_SPINLOCK(sony_dev_list_lock);
 static LIST_HEAD(sony_device_list);
 static DEFINE_IDA(sony_device_id_allocator);
@@ -1035,6 +1073,7 @@ struct sony_sc {
 	spinlock_t lock;
 	struct list_head list_node;
 	struct hid_device *hdev;
+	struct input_dev *touchpad;
 	struct led_classdev *leds[MAX_LEDS];
 	unsigned long quirks;
 	struct work_struct state_worker;
@@ -1130,6 +1169,37 @@ static int ps3remote_mapping(struct hid_device *hdev, struct hid_input *hi,
 	return 1;
 }
 
+static int ds4_mapping(struct hid_device *hdev, struct hid_input *hi,
+		       struct hid_field *field, struct hid_usage *usage,
+		       unsigned long **bit, int *max)
+{
+	if ((usage->hid & HID_USAGE_PAGE) == HID_UP_BUTTON) {
+		unsigned int key = usage->hid & HID_USAGE;
+
+		if (key >= ARRAY_SIZE(ds4_keymap))
+			return -1;
+
+		key = ds4_keymap[key];
+		hid_map_usage_clear(hi, usage, bit, max, EV_KEY, key);
+		return 1;
+	} else if ((usage->hid & HID_USAGE_PAGE) == HID_UP_GENDESK) {
+		unsigned int abs = usage->hid & HID_USAGE;
+
+		/* Let the HID parser deal with the HAT. */
+		if (usage->hid == HID_GD_HATSWITCH)
+			return 0;
+
+		if (abs >= ARRAY_SIZE(ds4_absmap))
+			return -1;
+
+		abs = ds4_absmap[abs];
+		hid_map_usage_clear(hi, usage, bit, max, EV_ABS, abs);
+		return 1;
+	}
+
+	return 0;
+}
+
 static u8 *sony_report_fixup(struct hid_device *hdev, u8 *rdesc,
 		unsigned int *rsize)
 {
@@ -1219,23 +1289,22 @@ static void sixaxis_parse_report(struct sony_sc *sc, u8 *rd, int size)
 
 static void dualshock4_parse_report(struct sony_sc *sc, u8 *rd, int size)
 {
-	struct hid_input *hidinput = list_entry(sc->hdev->inputs.next,
-						struct hid_input, list);
-	struct input_dev *input_dev = hidinput->input;
 	unsigned long flags;
-	int n, offset;
+	int n, m, offset, num_touch_data, max_touch_data;
 	u8 cable_state, battery_capacity, battery_charging;
 
-	/*
-	 * Battery and touchpad data starts at byte 30 in the USB report and
-	 * 32 in Bluetooth report.
-	 */
-	offset = (sc->quirks & DUALSHOCK4_CONTROLLER_USB) ? 30 : 32;
+	/* When using Bluetooth the header is 2 bytes longer, so skip these. */
+	int data_offset = (sc->quirks & DUALSHOCK4_CONTROLLER_USB) ? 0 : 2;
+
+	/* Second bit of third button byte is for the touchpad button. */
+	offset = data_offset + DS4_INPUT_REPORT_BUTTON_OFFSET;
+	input_report_key(sc->touchpad, BTN_LEFT, rd[offset+2] & 0x2);
 
 	/*
-	 * The lower 4 bits of byte 30 contain the battery level
+	 * The lower 4 bits of byte 30 (or 32 for BT) contain the battery level
 	 * and the 5th bit contains the USB cable state.
 	 */
+	offset = data_offset + DS4_INPUT_REPORT_BATTERY_OFFSET;
 	cable_state = (rd[offset] >> 4) & 0x01;
 	battery_capacity = rd[offset] & 0x0F;
 
@@ -1262,30 +1331,52 @@ static void dualshock4_parse_report(struct sony_sc *sc, u8 *rd, int size)
 	sc->battery_charging = battery_charging;
 	spin_unlock_irqrestore(&sc->lock, flags);
 
-	offset += 5;
-
 	/*
-	 * The Dualshock 4 multi-touch trackpad data starts at offset 35 on USB
-	 * and 37 on Bluetooth.
-	 * The first 7 bits of the first byte is a counter and bit 8 is a touch
-	 * indicator that is 0 when pressed and 1 when not pressed.
-	 * The next 3 bytes are two 12 bit touch coordinates, X and Y.
-	 * The data for the second touch is in the same format and immediatly
-	 * follows the data for the first.
+	 * The Dualshock 4 multi-touch trackpad data starts at offset 33 on USB
+	 * and 35 on Bluetooth.
+	 * The first byte indicates the number of touch data in the report.
+	 * Trackpad data starts 2 bytes later (e.g. 35 for USB).
 	 */
-	for (n = 0; n < 2; n++) {
-		u16 x, y;
+	offset = data_offset + DS4_INPUT_REPORT_TOUCHPAD_OFFSET;
+	max_touch_data = (sc->quirks & DUALSHOCK4_CONTROLLER_USB) ? 3 : 4;
+	if (rd[offset] > 0 && rd[offset] <= max_touch_data)
+		num_touch_data = rd[offset];
+	else
+		num_touch_data = 1;
+	offset += 1;
+
+	for (m = 0; m < num_touch_data; m++) {
+		/* Skip past timestamp */
+		offset += 1;
+
+		/*
+		 * The first 7 bits of the first byte is a counter and bit 8 is
+		 * a touch indicator that is 0 when pressed and 1 when not
+		 * pressed.
+		 * The next 3 bytes are two 12 bit touch coordinates, X and Y.
+		 * The data for the second touch is in the same format and
+		 * immediately follows the data for the first.
+		 */
+		for (n = 0; n < 2; n++) {
+			u16 x, y;
+			bool active;
 
-		x = rd[offset+1] | ((rd[offset+2] & 0xF) << 8);
-		y = ((rd[offset+2] & 0xF0) >> 4) | (rd[offset+3] << 4);
+			x = rd[offset+1] | ((rd[offset+2] & 0xF) << 8);
+			y = ((rd[offset+2] & 0xF0) >> 4) | (rd[offset+3] << 4);
 
-		input_mt_slot(input_dev, n);
-		input_mt_report_slot_state(input_dev, MT_TOOL_FINGER,
-					!(rd[offset] >> 7));
-		input_report_abs(input_dev, ABS_MT_POSITION_X, x);
-		input_report_abs(input_dev, ABS_MT_POSITION_Y, y);
+			active = !(rd[offset] >> 7);
+			input_mt_slot(sc->touchpad, n);
+			input_mt_report_slot_state(sc->touchpad, MT_TOOL_FINGER, active);
 
-		offset += 4;
+			if (active) {
+				input_report_abs(sc->touchpad, ABS_MT_POSITION_X, x);
+				input_report_abs(sc->touchpad, ABS_MT_POSITION_Y, y);
+			}
+
+			offset += 4;
+		}
+		input_mt_sync_frame(sc->touchpad);
+		input_sync(sc->touchpad);
 	}
 }
 
@@ -1324,6 +1415,21 @@ static int sony_raw_event(struct hid_device *hdev, struct hid_report *report,
 	} else if (((sc->quirks & DUALSHOCK4_CONTROLLER_USB) && rd[0] == 0x01 &&
 			size == 64) || ((sc->quirks & DUALSHOCK4_CONTROLLER_BT)
 			&& rd[0] == 0x11 && size == 78)) {
+		if (sc->quirks & DUALSHOCK4_CONTROLLER_BT) {
+			/* CRC check */
+			u8 bthdr = 0xA1;
+			u32 crc;
+			u32 report_crc;
+
+			crc = crc32_le(0xFFFFFFFF, &bthdr, 1);
+			crc = ~crc32_le(crc, rd, DS4_INPUT_REPORT_0x11_SIZE-4);
+			report_crc = get_unaligned_le32(&rd[DS4_INPUT_REPORT_0x11_SIZE-4]);
+			if (crc != report_crc) {
+				hid_dbg(sc->hdev, "DualShock 4 input report's CRC check failed, received crc 0x%0x != 0x%0x\n",
+					report_crc, crc);
+				return -EILSEQ;
+			}
+		}
 		dualshock4_parse_report(sc, rd, size);
 	}
 
@@ -1367,47 +1473,84 @@ static int sony_mapping(struct hid_device *hdev, struct hid_input *hi,
 	if (sc->quirks & PS3REMOTE)
 		return ps3remote_mapping(hdev, hi, field, usage, bit, max);
 
+
+	if (sc->quirks & DUALSHOCK4_CONTROLLER)
+		return ds4_mapping(hdev, hi, field, usage, bit, max);
+
 	/* Let hid-core decide for the others */
 	return 0;
 }
 
-static int sony_register_touchpad(struct hid_input *hi, int touch_count,
+static int sony_register_touchpad(struct sony_sc *sc, int touch_count,
 					int w, int h)
 {
-	struct input_dev *input_dev = hi->input;
+	size_t name_sz;
+	char *name;
 	int ret;
 
-	ret = input_mt_init_slots(input_dev, touch_count, 0);
+	sc->touchpad = input_allocate_device();
+	if (!sc->touchpad)
+		return -ENOMEM;
+
+	input_set_drvdata(sc->touchpad, sc);
+	sc->touchpad->dev.parent = &sc->hdev->dev;
+	sc->touchpad->phys = sc->hdev->phys;
+	sc->touchpad->uniq = sc->hdev->uniq;
+	sc->touchpad->id.bustype = sc->hdev->bus;
+	sc->touchpad->id.vendor = sc->hdev->vendor;
+	sc->touchpad->id.product = sc->hdev->product;
+	sc->touchpad->id.version = sc->hdev->version;
+
+	/* Append a suffix to the controller name as there are various
+	 * DS4 compatible non-Sony devices with different names.
+	 */
+	name_sz = strlen(sc->hdev->name) + sizeof(DS4_TOUCHPAD_SUFFIX);
+	name = kzalloc(name_sz, GFP_KERNEL);
+	if (!name) {
+		ret = -ENOMEM;
+		goto err;
+	}
+	snprintf(name, name_sz, "%s" DS4_TOUCHPAD_SUFFIX, sc->hdev->name);
+	sc->touchpad->name = name;
+
+	ret = input_mt_init_slots(sc->touchpad, touch_count, 0);
 	if (ret < 0)
-		return ret;
+		goto err;
+
+	/* We map the button underneath the touchpad to BTN_LEFT. */
+	__set_bit(EV_KEY, sc->touchpad->evbit);
+	__set_bit(BTN_LEFT, sc->touchpad->keybit);
+	__set_bit(INPUT_PROP_BUTTONPAD, sc->touchpad->propbit);
 
-	input_set_abs_params(input_dev, ABS_MT_POSITION_X, 0, w, 0, 0);
-	input_set_abs_params(input_dev, ABS_MT_POSITION_Y, 0, h, 0, 0);
+	input_set_abs_params(sc->touchpad, ABS_MT_POSITION_X, 0, w, 0, 0);
+	input_set_abs_params(sc->touchpad, ABS_MT_POSITION_Y, 0, h, 0, 0);
+
+	ret = input_register_device(sc->touchpad);
+	if (ret < 0)
+		goto err;
 
 	return 0;
+
+err:
+	kfree(sc->touchpad->name);
+	sc->touchpad->name = NULL;
+
+	input_free_device(sc->touchpad);
+	sc->touchpad = NULL;
+
+	return ret;
 }
 
-static int sony_input_configured(struct hid_device *hdev,
-					struct hid_input *hidinput)
+static void sony_unregister_touchpad(struct sony_sc *sc)
 {
-	struct sony_sc *sc = hid_get_drvdata(hdev);
-	int ret;
+	if (!sc->touchpad)
+		return;
 
-	/*
-	 * The Dualshock 4 touchpad supports 2 touches and has a
-	 * resolution of 1920x942 (44.86 dots/mm).
-	 */
-	if (sc->quirks & DUALSHOCK4_CONTROLLER) {
-		ret = sony_register_touchpad(hidinput, 2, 1920, 942);
-		if (ret) {
-			hid_err(sc->hdev,
-				"Unable to initialize multi-touch slots: %d\n",
-				ret);
-			return ret;
-		}
-	}
+	kfree(sc->touchpad->name);
+	sc->touchpad->name = NULL;
 
-	return 0;
+	input_unregister_device(sc->touchpad);
+	sc->touchpad = NULL;
 }
 
 /*
@@ -1483,11 +1626,11 @@ static int dualshock4_set_operational_bt(struct hid_device *hdev)
 	u8 *buf;
 	int ret;
 
-	buf = kmalloc(DS4_REPORT_0x02_SIZE, GFP_KERNEL);
+	buf = kmalloc(DS4_FEATURE_REPORT_0x02_SIZE, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
-	ret = hid_hw_raw_request(hdev, 0x02, buf, DS4_REPORT_0x02_SIZE,
+	ret = hid_hw_raw_request(hdev, 0x02, buf, DS4_FEATURE_REPORT_0x02_SIZE,
 				HID_FEATURE_REPORT, HID_REQ_GET_REPORT);
 
 	kfree(buf);
@@ -1892,14 +2035,14 @@ static void dualshock4_send_output_report(struct sony_sc *sc)
 	 * 0xD0 - 66hz
 	 */
 	if (sc->quirks & DUALSHOCK4_CONTROLLER_USB) {
-		memset(buf, 0, DS4_REPORT_0x05_SIZE);
+		memset(buf, 0, DS4_OUTPUT_REPORT_0x05_SIZE);
 		buf[0] = 0x05;
 		buf[1] = 0xFF;
 		offset = 4;
 	} else {
-		memset(buf, 0, DS4_REPORT_0x11_SIZE);
+		memset(buf, 0, DS4_OUTPUT_REPORT_0x11_SIZE);
 		buf[0] = 0x11;
-		buf[1] = 0x80;
+		buf[1] = 0xC0; /* HID + CRC */
 		buf[3] = 0x0F;
 		offset = 6;
 	}
@@ -1925,10 +2068,17 @@ static void dualshock4_send_output_report(struct sony_sc *sc)
 	buf[offset++] = sc->led_delay_off[3];
 
 	if (sc->quirks & DUALSHOCK4_CONTROLLER_USB)
-		hid_hw_output_report(hdev, buf, DS4_REPORT_0x05_SIZE);
-	else
-		hid_hw_raw_request(hdev, 0x11, buf, DS4_REPORT_0x11_SIZE,
-				HID_OUTPUT_REPORT, HID_REQ_SET_REPORT);
+		hid_hw_output_report(hdev, buf, DS4_OUTPUT_REPORT_0x05_SIZE);
+	else {
+		/* CRC generation */
+		u8 bthdr = 0xA2;
+		u32 crc;
+
+		crc = crc32_le(0xFFFFFFFF, &bthdr, 1);
+		crc = ~crc32_le(crc, buf, DS4_OUTPUT_REPORT_0x11_SIZE-4);
+		put_unaligned_le32(crc, &buf[74]);
+		hid_hw_output_report(hdev, buf, DS4_OUTPUT_REPORT_0x11_SIZE);
+	}
 }
 
 static void motion_send_output_report(struct sony_sc *sc)
@@ -1972,10 +2122,10 @@ static int sony_allocate_output_report(struct sony_sc *sc)
 			kmalloc(sizeof(union sixaxis_output_report_01),
 				GFP_KERNEL);
 	else if (sc->quirks & DUALSHOCK4_CONTROLLER_BT)
-		sc->output_report_dmabuf = kmalloc(DS4_REPORT_0x11_SIZE,
+		sc->output_report_dmabuf = kmalloc(DS4_OUTPUT_REPORT_0x11_SIZE,
 						GFP_KERNEL);
 	else if (sc->quirks & DUALSHOCK4_CONTROLLER_USB)
-		sc->output_report_dmabuf = kmalloc(DS4_REPORT_0x05_SIZE,
+		sc->output_report_dmabuf = kmalloc(DS4_OUTPUT_REPORT_0x05_SIZE,
 						GFP_KERNEL);
 	else if (sc->quirks & MOTION_CONTROLLER)
 		sc->output_report_dmabuf = kmalloc(MOTION_REPORT_0x02_SIZE,
@@ -2220,7 +2370,7 @@ static int sony_check_add(struct sony_sc *sc)
 			return 0;
 		}
 	} else if (sc->quirks & DUALSHOCK4_CONTROLLER_USB) {
-		buf = kmalloc(DS4_REPORT_0x81_SIZE, GFP_KERNEL);
+		buf = kmalloc(DS4_FEATURE_REPORT_0x81_SIZE, GFP_KERNEL);
 		if (!buf)
 			return -ENOMEM;
 
@@ -2230,10 +2380,10 @@ static int sony_check_add(struct sony_sc *sc)
 		 * offset 1.
 		 */
 		ret = hid_hw_raw_request(sc->hdev, 0x81, buf,
-				DS4_REPORT_0x81_SIZE, HID_FEATURE_REPORT,
+				DS4_FEATURE_REPORT_0x81_SIZE, HID_FEATURE_REPORT,
 				HID_REQ_GET_REPORT);
 
-		if (ret != DS4_REPORT_0x81_SIZE) {
+		if (ret != DS4_FEATURE_REPORT_0x81_SIZE) {
 			hid_err(sc->hdev, "failed to retrieve feature report 0x81 with the DualShock 4 MAC address\n");
 			ret = ret < 0 ? ret : -EINVAL;
 			goto out_free;
@@ -2329,45 +2479,12 @@ static inline void sony_cancel_work_sync(struct sony_sc *sc)
 		cancel_work_sync(&sc->state_worker);
 }
 
-static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
+static int sony_input_configured(struct hid_device *hdev,
+					struct hid_input *hidinput)
 {
-	int ret;
+	struct sony_sc *sc = hid_get_drvdata(hdev);
 	int append_dev_id;
-	unsigned long quirks = id->driver_data;
-	struct sony_sc *sc;
-	unsigned int connect_mask = HID_CONNECT_DEFAULT;
-
-	if (!strcmp(hdev->name, "FutureMax Dance Mat"))
-		quirks |= FUTUREMAX_DANCE_MAT;
-
-	sc = devm_kzalloc(&hdev->dev, sizeof(*sc), GFP_KERNEL);
-	if (sc == NULL) {
-		hid_err(hdev, "can't alloc sony descriptor\n");
-		return -ENOMEM;
-	}
-
-	spin_lock_init(&sc->lock);
-
-	sc->quirks = quirks;
-	hid_set_drvdata(hdev, sc);
-	sc->hdev = hdev;
-
-	ret = hid_parse(hdev);
-	if (ret) {
-		hid_err(hdev, "parse failed\n");
-		return ret;
-	}
-
-	if (sc->quirks & VAIO_RDESC_CONSTANT)
-		connect_mask |= HID_CONNECT_HIDDEV_FORCE;
-	else if (sc->quirks & SIXAXIS_CONTROLLER)
-		connect_mask |= HID_CONNECT_HIDDEV_FORCE;
-
-	ret = hid_hw_start(hdev, connect_mask);
-	if (ret) {
-		hid_err(hdev, "hw start failed\n");
-		return ret;
-	}
+	int ret;
 
 	ret = sony_set_device_id(sc);
 	if (ret < 0) {
@@ -2415,11 +2532,6 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		sony_init_output_report(sc, sixaxis_send_output_report);
 	} else if (sc->quirks & DUALSHOCK4_CONTROLLER) {
 		if (sc->quirks & DUALSHOCK4_CONTROLLER_BT) {
-			/*
-			 * The DualShock 4 wants output reports sent on the ctrl
-			 * endpoint when connected via Bluetooth.
-			 */
-			hdev->quirks |= HID_QUIRK_NO_OUTPUT_REPORTS_ON_INTR_EP;
 			ret = dualshock4_set_operational_bt(hdev);
 			if (ret < 0) {
 				hid_err(hdev, "failed to set the Dualshock 4 operational mode\n");
@@ -2427,6 +2539,18 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
 			}
 		}
 
+		/*
+		 * The Dualshock 4 touchpad supports 2 touches and has a
+		 * resolution of 1920x942 (44.86 dots/mm).
+		 */
+		ret = sony_register_touchpad(sc, 2, 1920, 942);
+		if (ret) {
+			hid_err(sc->hdev,
+			"Unable to initialize multi-touch slots: %d\n",
+			ret);
+			return ret;
+		}
+
 		sony_init_output_report(sc, dualshock4_send_output_report);
 	} else if (sc->quirks & MOTION_CONTROLLER) {
 		sony_init_output_report(sc, motion_send_output_report);
@@ -2482,17 +2606,84 @@ err_stop:
 	return ret;
 }
 
+static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id)
+{
+	int ret;
+	unsigned long quirks = id->driver_data;
+	struct sony_sc *sc;
+	unsigned int connect_mask = HID_CONNECT_DEFAULT;
+
+	if (!strcmp(hdev->name, "FutureMax Dance Mat"))
+		quirks |= FUTUREMAX_DANCE_MAT;
+
+	sc = devm_kzalloc(&hdev->dev, sizeof(*sc), GFP_KERNEL);
+	if (sc == NULL) {
+		hid_err(hdev, "can't alloc sony descriptor\n");
+		return -ENOMEM;
+	}
+
+	spin_lock_init(&sc->lock);
+
+	sc->quirks = quirks;
+	hid_set_drvdata(hdev, sc);
+	sc->hdev = hdev;
+
+	ret = hid_parse(hdev);
+	if (ret) {
+		hid_err(hdev, "parse failed\n");
+		return ret;
+	}
+
+	if (sc->quirks & VAIO_RDESC_CONSTANT)
+		connect_mask |= HID_CONNECT_HIDDEV_FORCE;
+	else if (sc->quirks & SIXAXIS_CONTROLLER)
+		connect_mask |= HID_CONNECT_HIDDEV_FORCE;
+
+	/* Patch the hw version on DS4 compatible devices, so applications can
+	 * distinguish between the default HID mappings and the mappings defined
+	 * by the Linux game controller spec. This is important for the SDL2
+	 * library, which has a game controller database, which uses device ids
+	 * in combination with version as a key.
+	 */
+	if (sc->quirks & DUALSHOCK4_CONTROLLER)
+		hdev->version |= 0x8000;
+
+	ret = hid_hw_start(hdev, connect_mask);
+	if (ret) {
+		hid_err(hdev, "hw start failed\n");
+		return ret;
+	}
+
+	/* sony_input_configured can fail, but this doesn't result
+	 * in hid_hw_start failures (intended). Check whether
+	 * the HID layer claimed the device else fail.
+	 * We don't know the actual reason for the failure, most
+	 * likely it is due to EEXIST in case of double connection
+	 * of USB and Bluetooth, but could have been due to ENOMEM
+	 * or other reasons as well.
+	 */
+	if (!(hdev->claimed & HID_CLAIMED_INPUT)) {
+		hid_err(hdev, "failed to claim input\n");
+		return -ENODEV;
+	}
+
+	return ret;
+}
+
 static void sony_remove(struct hid_device *hdev)
 {
 	struct sony_sc *sc = hid_get_drvdata(hdev);
 
+	hid_hw_close(hdev);
+
 	if (sc->quirks & SONY_LED_SUPPORT)
 		sony_leds_remove(sc);
 
-	if (sc->quirks & SONY_BATTERY_SUPPORT) {
-		hid_hw_close(hdev);
+	if (sc->quirks & SONY_BATTERY_SUPPORT)
 		sony_battery_remove(sc);
-	}
+
+	if (sc->touchpad)
+		sony_unregister_touchpad(sc);
 
 	sony_cancel_work_sync(sc);
 
@@ -2596,6 +2787,12 @@ static const struct hid_device_id sony_devices[] = {
 		.driver_data = DUALSHOCK4_CONTROLLER_USB },
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER),
 		.driver_data = DUALSHOCK4_CONTROLLER_BT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER_2),
+		.driver_data = DUALSHOCK4_CONTROLLER_USB },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER_2),
+		.driver_data = DUALSHOCK4_CONTROLLER_BT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER_DONGLE),
+		.driver_data = DUALSHOCK4_CONTROLLER_USB },
 	/* Nyko Core Controller for PS3 */
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SINO_LITE, USB_DEVICE_ID_SINO_LITE_CONTROLLER),
 		.driver_data = SIXAXIS_CONTROLLER_USB | SINO_LITE_CONTROLLER },
diff --git a/drivers/hid/hid-udraw-ps3.c b/drivers/hid/hid-udraw-ps3.c
new file mode 100644
index 000000000000..88ea390c10ad
--- /dev/null
+++ b/drivers/hid/hid-udraw-ps3.c
@@ -0,0 +1,474 @@
+/*
+ * HID driver for THQ PS3 uDraw tablet
+ *
+ * Copyright (C) 2016 Red Hat Inc. All Rights Reserved
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/device.h>
+#include <linux/hid.h>
+#include <linux/module.h>
+#include "hid-ids.h"
+
+MODULE_AUTHOR("Bastien Nocera <hadess@hadess.net>");
+MODULE_DESCRIPTION("PS3 uDraw tablet driver");
+MODULE_LICENSE("GPL");
+
+/*
+ * Protocol information from:
+ * http://brandonw.net/udraw/
+ * and the source code of:
+ * https://vvvv.org/contribution/udraw-hid
+ */
+
+/*
+ * The device is setup with multiple input devices:
+ * - the touch area which works as a touchpad
+ * - the tablet area which works as a touchpad/drawing tablet
+ * - a joypad with a d-pad, and 7 buttons
+ * - an accelerometer device
+ */
+
+enum {
+	TOUCH_NONE,
+	TOUCH_PEN,
+	TOUCH_FINGER,
+	TOUCH_TWOFINGER
+};
+
+enum {
+	AXIS_X,
+	AXIS_Y,
+	AXIS_Z
+};
+
+/*
+ * Accelerometer min/max values
+ * in order, X, Y and Z
+ */
+static struct {
+	int min;
+	int max;
+} accel_limits[] = {
+	[AXIS_X] = { 490, 534 },
+	[AXIS_Y] = { 490, 534 },
+	[AXIS_Z] = { 492, 536 }
+};
+
+#define DEVICE_NAME "THQ uDraw Game Tablet for PS3"
+/* resolution in pixels */
+#define RES_X 1920
+#define RES_Y 1080
+/* size in mm */
+#define WIDTH  160
+#define HEIGHT 90
+#define PRESSURE_OFFSET 113
+#define MAX_PRESSURE (255 - PRESSURE_OFFSET)
+
+struct udraw {
+	struct input_dev *joy_input_dev;
+	struct input_dev *touch_input_dev;
+	struct input_dev *pen_input_dev;
+	struct input_dev *accel_input_dev;
+	struct hid_device *hdev;
+
+	/*
+	 * The device's two-finger support is pretty unreliable, as
+	 * the device could report a single touch when the two fingers
+	 * are too close together, and the distance between fingers, even
+	 * though reported is not in the same unit as the touches.
+	 *
+	 * We'll make do without it, and try to report the first touch
+	 * as reliably as possible.
+	 */
+	int last_one_finger_x;
+	int last_one_finger_y;
+	int last_two_finger_x;
+	int last_two_finger_y;
+};
+
+static int clamp_accel(int axis, int offset)
+{
+	axis = clamp(axis,
+			accel_limits[offset].min,
+			accel_limits[offset].max);
+	axis = (axis - accel_limits[offset].min) /
+			((accel_limits[offset].max -
+			  accel_limits[offset].min) * 0xFF);
+	return axis;
+}
+
+static int udraw_raw_event(struct hid_device *hdev, struct hid_report *report,
+	 u8 *data, int len)
+{
+	struct udraw *udraw = hid_get_drvdata(hdev);
+	int touch;
+	int x, y, z;
+
+	if (len != 27)
+		return 0;
+
+	if (data[11] == 0x00)
+		touch = TOUCH_NONE;
+	else if (data[11] == 0x40)
+		touch = TOUCH_PEN;
+	else if (data[11] == 0x80)
+		touch = TOUCH_FINGER;
+	else
+		touch = TOUCH_TWOFINGER;
+
+	/* joypad */
+	input_report_key(udraw->joy_input_dev, BTN_WEST, data[0] & 1);
+	input_report_key(udraw->joy_input_dev, BTN_SOUTH, !!(data[0] & 2));
+	input_report_key(udraw->joy_input_dev, BTN_EAST, !!(data[0] & 4));
+	input_report_key(udraw->joy_input_dev, BTN_NORTH, !!(data[0] & 8));
+
+	input_report_key(udraw->joy_input_dev, BTN_SELECT, !!(data[1] & 1));
+	input_report_key(udraw->joy_input_dev, BTN_START, !!(data[1] & 2));
+	input_report_key(udraw->joy_input_dev, BTN_MODE, !!(data[1] & 16));
+
+	x = y = 0;
+	switch (data[2]) {
+	case 0x0:
+		y = -127;
+		break;
+	case 0x1:
+		y = -127;
+		x = 127;
+		break;
+	case 0x2:
+		x = 127;
+		break;
+	case 0x3:
+		y = 127;
+		x = 127;
+		break;
+	case 0x4:
+		y = 127;
+		break;
+	case 0x5:
+		y = 127;
+		x = -127;
+		break;
+	case 0x6:
+		x = -127;
+		break;
+	case 0x7:
+		y = -127;
+		x = -127;
+		break;
+	default:
+		break;
+	}
+
+	input_report_abs(udraw->joy_input_dev, ABS_X, x);
+	input_report_abs(udraw->joy_input_dev, ABS_Y, y);
+
+	input_sync(udraw->joy_input_dev);
+
+	/* For pen and touchpad */
+	x = y = 0;
+	if (touch != TOUCH_NONE) {
+		if (data[15] != 0x0F)
+			x = data[15] * 256 + data[17];
+		if (data[16] != 0x0F)
+			y = data[16] * 256 + data[18];
+	}
+
+	if (touch == TOUCH_FINGER) {
+		/* Save the last one-finger touch */
+		udraw->last_one_finger_x = x;
+		udraw->last_one_finger_y = y;
+		udraw->last_two_finger_x = -1;
+		udraw->last_two_finger_y = -1;
+	} else if (touch == TOUCH_TWOFINGER) {
+		/*
+		 * We have a problem because x/y is the one for the
+		 * second finger but we want the first finger given
+		 * to user-space otherwise it'll look as if it jumped.
+		 *
+		 * See the udraw struct definition for why this was
+		 * implemented this way.
+		 */
+		if (udraw->last_two_finger_x == -1) {
+			/* Save the position of the 2nd finger */
+			udraw->last_two_finger_x = x;
+			udraw->last_two_finger_y = y;
+
+			x = udraw->last_one_finger_x;
+			y = udraw->last_one_finger_y;
+		} else {
+			/*
+			 * Offset the 2-finger coords using the
+			 * saved data from the first finger
+			 */
+			x = x - (udraw->last_two_finger_x
+				- udraw->last_one_finger_x);
+			y = y - (udraw->last_two_finger_y
+				- udraw->last_one_finger_y);
+		}
+	}
+
+	/* touchpad */
+	if (touch == TOUCH_FINGER || touch == TOUCH_TWOFINGER) {
+		input_report_key(udraw->touch_input_dev, BTN_TOUCH, 1);
+		input_report_key(udraw->touch_input_dev, BTN_TOOL_FINGER,
+				touch == TOUCH_FINGER);
+		input_report_key(udraw->touch_input_dev, BTN_TOOL_DOUBLETAP,
+				touch == TOUCH_TWOFINGER);
+
+		input_report_abs(udraw->touch_input_dev, ABS_X, x);
+		input_report_abs(udraw->touch_input_dev, ABS_Y, y);
+	} else {
+		input_report_key(udraw->touch_input_dev, BTN_TOUCH, 0);
+		input_report_key(udraw->touch_input_dev, BTN_TOOL_FINGER, 0);
+		input_report_key(udraw->touch_input_dev, BTN_TOOL_DOUBLETAP, 0);
+	}
+	input_sync(udraw->touch_input_dev);
+
+	/* pen */
+	if (touch == TOUCH_PEN) {
+		int level;
+
+		level = clamp(data[13] - PRESSURE_OFFSET,
+				0, MAX_PRESSURE);
+
+		input_report_key(udraw->pen_input_dev, BTN_TOUCH, (level != 0));
+		input_report_key(udraw->pen_input_dev, BTN_TOOL_PEN, 1);
+		input_report_abs(udraw->pen_input_dev, ABS_PRESSURE, level);
+		input_report_abs(udraw->pen_input_dev, ABS_X, x);
+		input_report_abs(udraw->pen_input_dev, ABS_Y, y);
+	} else {
+		input_report_key(udraw->pen_input_dev, BTN_TOUCH, 0);
+		input_report_key(udraw->pen_input_dev, BTN_TOOL_PEN, 0);
+		input_report_abs(udraw->pen_input_dev, ABS_PRESSURE, 0);
+	}
+	input_sync(udraw->pen_input_dev);
+
+	/* accel */
+	x = (data[19] + (data[20] << 8));
+	x = clamp_accel(x, AXIS_X);
+	y = (data[21] + (data[22] << 8));
+	y = clamp_accel(y, AXIS_Y);
+	z = (data[23] + (data[24] << 8));
+	z = clamp_accel(z, AXIS_Z);
+	input_report_abs(udraw->accel_input_dev, ABS_X, x);
+	input_report_abs(udraw->accel_input_dev, ABS_Y, y);
+	input_report_abs(udraw->accel_input_dev, ABS_Z, z);
+	input_sync(udraw->accel_input_dev);
+
+	/* let hidraw and hiddev handle the report */
+	return 0;
+}
+
+static int udraw_open(struct input_dev *dev)
+{
+	struct udraw *udraw = input_get_drvdata(dev);
+
+	return hid_hw_open(udraw->hdev);
+}
+
+static void udraw_close(struct input_dev *dev)
+{
+	struct udraw *udraw = input_get_drvdata(dev);
+
+	hid_hw_close(udraw->hdev);
+}
+
+static struct input_dev *allocate_and_setup(struct hid_device *hdev,
+		const char *name)
+{
+	struct input_dev *input_dev;
+
+	input_dev = devm_input_allocate_device(&hdev->dev);
+	if (!input_dev)
+		return NULL;
+
+	input_dev->name = name;
+	input_dev->phys = hdev->phys;
+	input_dev->dev.parent = &hdev->dev;
+	input_dev->open = udraw_open;
+	input_dev->close = udraw_close;
+	input_dev->uniq = hdev->uniq;
+	input_dev->id.bustype = hdev->bus;
+	input_dev->id.vendor  = hdev->vendor;
+	input_dev->id.product = hdev->product;
+	input_dev->id.version = hdev->version;
+	input_set_drvdata(input_dev, hid_get_drvdata(hdev));
+
+	return input_dev;
+}
+
+static bool udraw_setup_touch(struct udraw *udraw,
+		struct hid_device *hdev)
+{
+	struct input_dev *input_dev;
+
+	input_dev = allocate_and_setup(hdev, DEVICE_NAME " Touchpad");
+	if (!input_dev)
+		return false;
+
+	input_dev->evbit[0] = BIT(EV_ABS) | BIT(EV_KEY);
+
+	input_set_abs_params(input_dev, ABS_X, 0, RES_X, 1, 0);
+	input_abs_set_res(input_dev, ABS_X, RES_X / WIDTH);
+	input_set_abs_params(input_dev, ABS_Y, 0, RES_Y, 1, 0);
+	input_abs_set_res(input_dev, ABS_Y, RES_Y / HEIGHT);
+
+	set_bit(BTN_TOUCH, input_dev->keybit);
+	set_bit(BTN_TOOL_FINGER, input_dev->keybit);
+	set_bit(BTN_TOOL_DOUBLETAP, input_dev->keybit);
+
+	set_bit(INPUT_PROP_POINTER, input_dev->propbit);
+
+	udraw->touch_input_dev = input_dev;
+
+	return true;
+}
+
+static bool udraw_setup_pen(struct udraw *udraw,
+		struct hid_device *hdev)
+{
+	struct input_dev *input_dev;
+
+	input_dev = allocate_and_setup(hdev, DEVICE_NAME " Pen");
+	if (!input_dev)
+		return false;
+
+	input_dev->evbit[0] = BIT(EV_ABS) | BIT(EV_KEY);
+
+	input_set_abs_params(input_dev, ABS_X, 0, RES_X, 1, 0);
+	input_abs_set_res(input_dev, ABS_X, RES_X / WIDTH);
+	input_set_abs_params(input_dev, ABS_Y, 0, RES_Y, 1, 0);
+	input_abs_set_res(input_dev, ABS_Y, RES_Y / HEIGHT);
+	input_set_abs_params(input_dev, ABS_PRESSURE,
+			0, MAX_PRESSURE, 0, 0);
+
+	set_bit(BTN_TOUCH, input_dev->keybit);
+	set_bit(BTN_TOOL_PEN, input_dev->keybit);
+
+	set_bit(INPUT_PROP_POINTER, input_dev->propbit);
+
+	udraw->pen_input_dev = input_dev;
+
+	return true;
+}
+
+static bool udraw_setup_accel(struct udraw *udraw,
+		struct hid_device *hdev)
+{
+	struct input_dev *input_dev;
+
+	input_dev = allocate_and_setup(hdev, DEVICE_NAME " Accelerometer");
+	if (!input_dev)
+		return false;
+
+	input_dev->evbit[0] = BIT(EV_ABS);
+
+	/* 1G accel is reported as ~256, so clamp to 2G */
+	input_set_abs_params(input_dev, ABS_X, -512, 512, 0, 0);
+	input_set_abs_params(input_dev, ABS_Y, -512, 512, 0, 0);
+	input_set_abs_params(input_dev, ABS_Z, -512, 512, 0, 0);
+
+	set_bit(INPUT_PROP_ACCELEROMETER, input_dev->propbit);
+
+	udraw->accel_input_dev = input_dev;
+
+	return true;
+}
+
+static bool udraw_setup_joypad(struct udraw *udraw,
+		struct hid_device *hdev)
+{
+	struct input_dev *input_dev;
+
+	input_dev = allocate_and_setup(hdev, DEVICE_NAME " Joypad");
+	if (!input_dev)
+		return false;
+
+	input_dev->evbit[0] = BIT(EV_KEY) | BIT(EV_ABS);
+
+	set_bit(BTN_SOUTH, input_dev->keybit);
+	set_bit(BTN_NORTH, input_dev->keybit);
+	set_bit(BTN_EAST, input_dev->keybit);
+	set_bit(BTN_WEST, input_dev->keybit);
+	set_bit(BTN_SELECT, input_dev->keybit);
+	set_bit(BTN_START, input_dev->keybit);
+	set_bit(BTN_MODE, input_dev->keybit);
+
+	input_set_abs_params(input_dev, ABS_X, -127, 127, 0, 0);
+	input_set_abs_params(input_dev, ABS_Y, -127, 127, 0, 0);
+
+	udraw->joy_input_dev = input_dev;
+
+	return true;
+}
+
+static int udraw_probe(struct hid_device *hdev, const struct hid_device_id *id)
+{
+	struct udraw *udraw;
+	int ret;
+
+	udraw = devm_kzalloc(&hdev->dev, sizeof(struct udraw), GFP_KERNEL);
+	if (!udraw)
+		return -ENOMEM;
+
+	udraw->hdev = hdev;
+	udraw->last_two_finger_x = -1;
+	udraw->last_two_finger_y = -1;
+
+	hid_set_drvdata(hdev, udraw);
+
+	ret = hid_parse(hdev);
+	if (ret) {
+		hid_err(hdev, "parse failed\n");
+		return ret;
+	}
+
+	if (!udraw_setup_joypad(udraw, hdev) ||
+	    !udraw_setup_touch(udraw, hdev) ||
+	    !udraw_setup_pen(udraw, hdev) ||
+	    !udraw_setup_accel(udraw, hdev)) {
+		hid_err(hdev, "could not allocate interfaces\n");
+		return -ENOMEM;
+	}
+
+	ret = input_register_device(udraw->joy_input_dev) ||
+		input_register_device(udraw->touch_input_dev) ||
+		input_register_device(udraw->pen_input_dev) ||
+		input_register_device(udraw->accel_input_dev);
+	if (ret) {
+		hid_err(hdev, "failed to register interfaces\n");
+		return ret;
+	}
+
+	ret = hid_hw_start(hdev, HID_CONNECT_HIDRAW | HID_CONNECT_DRIVER);
+	if (ret) {
+		hid_err(hdev, "hw start failed\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct hid_device_id udraw_devices[] = {
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THQ, USB_DEVICE_ID_THQ_PS3_UDRAW) },
+	{ }
+};
+MODULE_DEVICE_TABLE(hid, udraw_devices);
+
+static struct hid_driver udraw_driver = {
+	.name = "hid-udraw",
+	.id_table = udraw_devices,
+	.raw_event = udraw_raw_event,
+	.probe = udraw_probe,
+};
+module_hid_driver(udraw_driver);
diff --git a/drivers/hid/i2c-hid/i2c-hid.c b/drivers/hid/i2c-hid/i2c-hid.c
index b3ec4f2de875..78fb32a7b103 100644
--- a/drivers/hid/i2c-hid/i2c-hid.c
+++ b/drivers/hid/i2c-hid/i2c-hid.c
@@ -22,6 +22,7 @@
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/input.h>
+#include <linux/irq.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/pm.h>
@@ -37,10 +38,15 @@
 #include <linux/mutex.h>
 #include <linux/acpi.h>
 #include <linux/of.h>
-#include <linux/gpio/consumer.h>
+#include <linux/regulator/consumer.h>
 
 #include <linux/i2c/i2c-hid.h>
 
+#include "../hid-ids.h"
+
+/* quirks to control the device */
+#define I2C_HID_QUIRK_SET_PWR_WAKEUP_DEV	BIT(0)
+
 /* flags */
 #define I2C_HID_STARTED		0
 #define I2C_HID_RESET_PENDING	1
@@ -143,10 +149,9 @@ struct i2c_hid {
 	char			*argsbuf;	/* Command arguments buffer */
 
 	unsigned long		flags;		/* device flags */
+	unsigned long		quirks;		/* Various quirks */
 
 	wait_queue_head_t	wait;		/* For waiting the interrupt */
-	struct gpio_desc	*desc;
-	int			irq;
 
 	struct i2c_hid_platform_data pdata;
 
@@ -154,6 +159,39 @@ struct i2c_hid {
 	struct mutex		reset_lock;
 };
 
+static const struct i2c_hid_quirks {
+	__u16 idVendor;
+	__u16 idProduct;
+	__u32 quirks;
+} i2c_hid_quirks[] = {
+	{ USB_VENDOR_ID_WEIDA, USB_DEVICE_ID_WEIDA_8752,
+		I2C_HID_QUIRK_SET_PWR_WAKEUP_DEV },
+	{ USB_VENDOR_ID_WEIDA, USB_DEVICE_ID_WEIDA_8755,
+		I2C_HID_QUIRK_SET_PWR_WAKEUP_DEV },
+	{ 0, 0 }
+};
+
+/*
+ * i2c_hid_lookup_quirk: return any quirks associated with a I2C HID device
+ * @idVendor: the 16-bit vendor ID
+ * @idProduct: the 16-bit product ID
+ *
+ * Returns: a u32 quirks value.
+ */
+static u32 i2c_hid_lookup_quirk(const u16 idVendor, const u16 idProduct)
+{
+	u32 quirks = 0;
+	int n;
+
+	for (n = 0; i2c_hid_quirks[n].idVendor; n++)
+		if (i2c_hid_quirks[n].idVendor == idVendor &&
+		    (i2c_hid_quirks[n].idProduct == (__u16)HID_ANY_ID ||
+		     i2c_hid_quirks[n].idProduct == idProduct))
+			quirks = i2c_hid_quirks[n].quirks;
+
+	return quirks;
+}
+
 static int __i2c_hid_command(struct i2c_client *client,
 		const struct i2c_hid_cmd *command, u8 reportID,
 		u8 reportType, u8 *args, int args_len,
@@ -346,11 +384,27 @@ static int i2c_hid_set_power(struct i2c_client *client, int power_state)
 
 	i2c_hid_dbg(ihid, "%s\n", __func__);
 
+	/*
+	 * Some devices require to send a command to wakeup before power on.
+	 * The call will get a return value (EREMOTEIO) but device will be
+	 * triggered and activated. After that, it goes like a normal device.
+	 */
+	if (power_state == I2C_HID_PWR_ON &&
+	    ihid->quirks & I2C_HID_QUIRK_SET_PWR_WAKEUP_DEV) {
+		ret = i2c_hid_command(client, &hid_set_power_cmd, NULL, 0);
+
+		/* Device was already activated */
+		if (!ret)
+			goto set_pwr_exit;
+	}
+
 	ret = __i2c_hid_command(client, &hid_set_power_cmd, power_state,
 		0, NULL, 0, NULL, 0);
+
 	if (ret)
 		dev_err(&client->dev, "failed to change power setting.\n");
 
+set_pwr_exit:
 	return ret;
 }
 
@@ -716,9 +770,11 @@ static int i2c_hid_start(struct hid_device *hid)
 	i2c_hid_find_max_report(hid, HID_FEATURE_REPORT, &bufsize);
 
 	if (bufsize > ihid->bufsize) {
+		disable_irq(client->irq);
 		i2c_hid_free_buffers(ihid);
 
 		ret = i2c_hid_alloc_buffers(ihid, bufsize);
+		enable_irq(client->irq);
 
 		if (ret)
 			return ret;
@@ -806,18 +862,21 @@ static struct hid_ll_driver i2c_hid_ll_driver = {
 static int i2c_hid_init_irq(struct i2c_client *client)
 {
 	struct i2c_hid *ihid = i2c_get_clientdata(client);
+	unsigned long irqflags = 0;
 	int ret;
 
-	dev_dbg(&client->dev, "Requesting IRQ: %d\n", ihid->irq);
+	dev_dbg(&client->dev, "Requesting IRQ: %d\n", client->irq);
 
-	ret = request_threaded_irq(ihid->irq, NULL, i2c_hid_irq,
-			IRQF_TRIGGER_LOW | IRQF_ONESHOT,
-			client->name, ihid);
+	if (!irq_get_trigger_type(client->irq))
+		irqflags = IRQF_TRIGGER_LOW;
+
+	ret = request_threaded_irq(client->irq, NULL, i2c_hid_irq,
+				   irqflags | IRQF_ONESHOT, client->name, ihid);
 	if (ret < 0) {
 		dev_warn(&client->dev,
 			"Could not register for %s interrupt, irq = %d,"
 			" ret = %d\n",
-			client->name, ihid->irq, ret);
+			client->name, client->irq, ret);
 
 		return ret;
 	}
@@ -864,14 +923,6 @@ static int i2c_hid_fetch_hid_descriptor(struct i2c_hid *ihid)
 }
 
 #ifdef CONFIG_ACPI
-
-/* Default GPIO mapping */
-static const struct acpi_gpio_params i2c_hid_irq_gpio = { 0, 0, true };
-static const struct acpi_gpio_mapping i2c_hid_acpi_gpios[] = {
-	{ "gpios", &i2c_hid_irq_gpio, 1 },
-	{ },
-};
-
 static int i2c_hid_acpi_pdata(struct i2c_client *client,
 		struct i2c_hid_platform_data *pdata)
 {
@@ -882,7 +933,6 @@ static int i2c_hid_acpi_pdata(struct i2c_client *client,
 	union acpi_object *obj;
 	struct acpi_device *adev;
 	acpi_handle handle;
-	int ret;
 
 	handle = ACPI_HANDLE(&client->dev);
 	if (!handle || acpi_bus_get_device(handle, &adev))
@@ -898,9 +948,7 @@ static int i2c_hid_acpi_pdata(struct i2c_client *client,
 	pdata->hid_descriptor_address = obj->integer.value;
 	ACPI_FREE(obj);
 
-	/* GPIOs are optional */
-	ret = acpi_dev_add_driver_gpios(adev, i2c_hid_acpi_gpios);
-	return ret < 0 && ret != -ENXIO ? ret : 0;
+	return 0;
 }
 
 static const struct acpi_device_id i2c_hid_acpi_match[] = {
@@ -964,6 +1012,19 @@ static int i2c_hid_probe(struct i2c_client *client,
 
 	dbg_hid("HID probe called for i2c 0x%02x\n", client->addr);
 
+	if (!client->irq) {
+		dev_err(&client->dev,
+			"HID over i2c has not been provided an Int IRQ\n");
+		return -EINVAL;
+	}
+
+	if (client->irq < 0) {
+		if (client->irq != -EPROBE_DEFER)
+			dev_err(&client->dev,
+				"HID over i2c doesn't have a valid IRQ\n");
+		return client->irq;
+	}
+
 	ihid = kzalloc(sizeof(struct i2c_hid), GFP_KERNEL);
 	if (!ihid)
 		return -ENOMEM;
@@ -983,23 +1044,6 @@ static int i2c_hid_probe(struct i2c_client *client,
 		ihid->pdata = *platform_data;
 	}
 
-	if (client->irq > 0) {
-		ihid->irq = client->irq;
-	} else if (ACPI_COMPANION(&client->dev)) {
-		ihid->desc = gpiod_get(&client->dev, NULL, GPIOD_IN);
-		if (IS_ERR(ihid->desc)) {
-			dev_err(&client->dev, "Failed to get GPIO interrupt\n");
-			return PTR_ERR(ihid->desc);
-		}
-
-		ihid->irq = gpiod_to_irq(ihid->desc);
-		if (ihid->irq < 0) {
-			gpiod_put(ihid->desc);
-			dev_err(&client->dev, "Failed to convert GPIO to IRQ\n");
-			return ihid->irq;
-		}
-	}
-
 	i2c_set_clientdata(client, ihid);
 
 	ihid->client = client;
@@ -1050,6 +1094,8 @@ static int i2c_hid_probe(struct i2c_client *client,
 		 client->name, hid->vendor, hid->product);
 	strlcpy(hid->phys, dev_name(&client->dev), sizeof(hid->phys));
 
+	ihid->quirks = i2c_hid_lookup_quirk(hid->vendor, hid->product);
+
 	ret = hid_add_device(hid);
 	if (ret) {
 		if (ret != -ENODEV)
@@ -1064,16 +1110,13 @@ err_mem_free:
 	hid_destroy_device(hid);
 
 err_irq:
-	free_irq(ihid->irq, ihid);
+	free_irq(client->irq, ihid);
 
 err_pm:
 	pm_runtime_put_noidle(&client->dev);
 	pm_runtime_disable(&client->dev);
 
 err:
-	if (ihid->desc)
-		gpiod_put(ihid->desc);
-
 	i2c_hid_free_buffers(ihid);
 	kfree(ihid);
 	return ret;
@@ -1092,18 +1135,13 @@ static int i2c_hid_remove(struct i2c_client *client)
 	hid = ihid->hid;
 	hid_destroy_device(hid);
 
-	free_irq(ihid->irq, ihid);
+	free_irq(client->irq, ihid);
 
 	if (ihid->bufsize)
 		i2c_hid_free_buffers(ihid);
 
-	if (ihid->desc)
-		gpiod_put(ihid->desc);
-
 	kfree(ihid);
 
-	acpi_dev_remove_driver_gpios(ACPI_COMPANION(&client->dev));
-
 	return 0;
 }
 
@@ -1142,11 +1180,11 @@ static int i2c_hid_suspend(struct device *dev)
 		/* Save some power */
 		i2c_hid_set_power(client, I2C_HID_PWR_SLEEP);
 
-		disable_irq(ihid->irq);
+		disable_irq(client->irq);
 	}
 
 	if (device_may_wakeup(&client->dev)) {
-		wake_status = enable_irq_wake(ihid->irq);
+		wake_status = enable_irq_wake(client->irq);
 		if (!wake_status)
 			ihid->irq_wake_enabled = true;
 		else
@@ -1166,7 +1204,7 @@ static int i2c_hid_resume(struct device *dev)
 	int wake_status;
 
 	if (device_may_wakeup(&client->dev) && ihid->irq_wake_enabled) {
-		wake_status = disable_irq_wake(ihid->irq);
+		wake_status = disable_irq_wake(client->irq);
 		if (!wake_status)
 			ihid->irq_wake_enabled = false;
 		else
@@ -1179,7 +1217,7 @@ static int i2c_hid_resume(struct device *dev)
 	pm_runtime_set_active(dev);
 	pm_runtime_enable(dev);
 
-	enable_irq(ihid->irq);
+	enable_irq(client->irq);
 	ret = i2c_hid_hwreset(client);
 	if (ret)
 		return ret;
@@ -1197,19 +1235,17 @@ static int i2c_hid_resume(struct device *dev)
 static int i2c_hid_runtime_suspend(struct device *dev)
 {
 	struct i2c_client *client = to_i2c_client(dev);
-	struct i2c_hid *ihid = i2c_get_clientdata(client);
 
 	i2c_hid_set_power(client, I2C_HID_PWR_SLEEP);
-	disable_irq(ihid->irq);
+	disable_irq(client->irq);
 	return 0;
 }
 
 static int i2c_hid_runtime_resume(struct device *dev)
 {
 	struct i2c_client *client = to_i2c_client(dev);
-	struct i2c_hid *ihid = i2c_get_clientdata(client);
 
-	enable_irq(ihid->irq);
+	enable_irq(client->irq);
 	i2c_hid_set_power(client, I2C_HID_PWR_ON);
 	return 0;
 }
diff --git a/drivers/hid/intel-ish-hid/ipc/ipc.c b/drivers/hid/intel-ish-hid/ipc/ipc.c
index 0c9ac4d5d850..842d8416a7a6 100644
--- a/drivers/hid/intel-ish-hid/ipc/ipc.c
+++ b/drivers/hid/intel-ish-hid/ipc/ipc.c
@@ -19,7 +19,6 @@
 #include <linux/jiffies.h>
 #include "client.h"
 #include "hw-ish.h"
-#include "utils.h"
 #include "hbm.h"
 
 /* For FW reset flow */
@@ -310,6 +309,7 @@ static int write_ipc_from_queue(struct ishtp_device *dev)
 						((uint32_t)tv_utc.tv_usec);
 		ts_format.ts1_source = HOST_SYSTEM_TIME_USEC;
 		ts_format.ts2_source = HOST_UTC_TIME_USEC;
+		ts_format.reserved = 0;
 
 		time_update.primary_host_time = usec_system;
 		time_update.secondary_host_time = usec_utc;
@@ -427,6 +427,59 @@ static int ipc_send_mng_msg(struct ishtp_device *dev, uint32_t msg_code,
 		sizeof(uint32_t) + size);
 }
 
+#define WAIT_FOR_FW_RDY			0x1
+#define WAIT_FOR_INPUT_RDY		0x2
+
+/**
+ * timed_wait_for_timeout() - wait special event with timeout
+ * @dev: ISHTP device pointer
+ * @condition: indicate the condition for waiting
+ * @timeinc: time slice for every wait cycle, in ms
+ * @timeout: time in ms for timeout
+ *
+ * This function will check special event to be ready in a loop, the loop
+ * period is specificd in timeinc. Wait timeout will causes failure.
+ *
+ * Return: 0 for success else failure code
+ */
+static int timed_wait_for_timeout(struct ishtp_device *dev, int condition,
+				unsigned int timeinc, unsigned int timeout)
+{
+	bool complete = false;
+	int ret;
+
+	do {
+		if (condition == WAIT_FOR_FW_RDY) {
+			complete = ishtp_fw_is_ready(dev);
+		} else if (condition == WAIT_FOR_INPUT_RDY) {
+			complete = ish_is_input_ready(dev);
+		} else {
+			ret = -EINVAL;
+			goto out;
+		}
+
+		if (!complete) {
+			unsigned long left_time;
+
+			left_time = msleep_interruptible(timeinc);
+			timeout -= (timeinc - left_time);
+		}
+	} while (!complete && timeout > 0);
+
+	if (complete)
+		ret = 0;
+	else
+		ret = -EBUSY;
+
+out:
+	return ret;
+}
+
+#define TIME_SLICE_FOR_FW_RDY_MS		100
+#define TIME_SLICE_FOR_INPUT_RDY_MS		100
+#define TIMEOUT_FOR_FW_RDY_MS			2000
+#define TIMEOUT_FOR_INPUT_RDY_MS		2000
+
 /**
  * ish_fw_reset_handler() - FW reset handler
  * @dev: ishtp device pointer
@@ -456,8 +509,8 @@ static int ish_fw_reset_handler(struct ishtp_device *dev)
 	ishtp_reset_handler(dev);
 
 	if (!ish_is_input_ready(dev))
-		timed_wait_for_timeout(WAIT_FOR_SEND_SLICE,
-			ish_is_input_ready(dev), (2 * HZ));
+		timed_wait_for_timeout(dev, WAIT_FOR_INPUT_RDY,
+			TIME_SLICE_FOR_INPUT_RDY_MS, TIMEOUT_FOR_INPUT_RDY_MS);
 
 	/* ISH FW is dead */
 	if (!ish_is_input_ready(dev))
@@ -472,8 +525,8 @@ static int ish_fw_reset_handler(struct ishtp_device *dev)
 			 sizeof(uint32_t));
 
 	/* Wait for ISH FW'es ILUP and ISHTP_READY */
-	timed_wait_for_timeout(WAIT_FOR_SEND_SLICE, ishtp_fw_is_ready(dev),
-		(2 * HZ));
+	timed_wait_for_timeout(dev, WAIT_FOR_FW_RDY,
+			TIME_SLICE_FOR_FW_RDY_MS, TIMEOUT_FOR_FW_RDY_MS);
 	if (!ishtp_fw_is_ready(dev)) {
 		/* ISH FW is dead */
 		uint32_t	ish_status;
@@ -487,6 +540,8 @@ static int ish_fw_reset_handler(struct ishtp_device *dev)
 	return	0;
 }
 
+#define TIMEOUT_FOR_HW_RDY_MS			300
+
 /**
  * ish_fw_reset_work_fn() - FW reset worker function
  * @unused: not used
@@ -500,7 +555,7 @@ static void fw_reset_work_fn(struct work_struct *unused)
 	rv = ish_fw_reset_handler(ishtp_dev);
 	if (!rv) {
 		/* ISH is ILUP & ISHTP-ready. Restart ISHTP */
-		schedule_timeout(HZ / 3);
+		msleep_interruptible(TIMEOUT_FOR_HW_RDY_MS);
 		ishtp_dev->recvd_hw_ready = 1;
 		wake_up_interruptible(&ishtp_dev->wait_hw_ready);
 
diff --git a/drivers/hid/intel-ish-hid/ipc/utils.h b/drivers/hid/intel-ish-hid/ipc/utils.h
deleted file mode 100644
index 5a82123dc7b4..000000000000
--- a/drivers/hid/intel-ish-hid/ipc/utils.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Utility macros of ISH
- *
- * Copyright (c) 2014-2016, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- */
-#ifndef UTILS__H
-#define UTILS__H
-
-#define	WAIT_FOR_SEND_SLICE	(HZ / 10)
-#define	WAIT_FOR_CONNECT_SLICE	(HZ / 10)
-
-/*
- * Waits for specified event when a thread that triggers event can't signal
- * Also, waits *at_least* `timeinc` after condition is satisfied
- */
-#define	timed_wait_for(timeinc, condition)			\
-	do {							\
-		int completed = 0;				\
-		do {						\
-			unsigned long	j;			\
-			int	done = 0;			\
-								\
-			completed = (condition);		\
-			for (j = jiffies, done = 0; !done; ) {	\
-				schedule_timeout(timeinc);	\
-				if (time_is_before_eq_jiffies(j + timeinc)) \
-					done = 1;		\
-			}					\
-		} while (!(completed));				\
-	} while (0)
-
-
-/*
- * Waits for specified event when a thread that triggers event
- * can't signal with timeout (use whenever we may hang)
- */
-#define	timed_wait_for_timeout(timeinc, condition, timeout)	\
-	do {							\
-		int	t = timeout;				\
-		do {						\
-			unsigned long	j;			\
-			int	done = 0;			\
-								\
-			for (j = jiffies, done = 0; !done; ) {	\
-				schedule_timeout(timeinc);	\
-				if (time_is_before_eq_jiffies(j + timeinc)) \
-					done = 1;		\
-			} \
-			t -= timeinc;				\
-			if (t <= 0)				\
-				break;				\
-		} while (!(condition));				\
-	} while (0)
-
-#endif /* UTILS__H */
diff --git a/drivers/hid/intel-ish-hid/ishtp/bus.c b/drivers/hid/intel-ish-hid/ishtp/bus.c
index 256521509d20..f4cbc744e657 100644
--- a/drivers/hid/intel-ish-hid/ishtp/bus.c
+++ b/drivers/hid/intel-ish-hid/ishtp/bus.c
@@ -585,14 +585,7 @@ int ishtp_bus_new_client(struct ishtp_device *dev)
 	 */
 	i = dev->fw_client_presentation_num - 1;
 	device_uuid = dev->fw_clients[i].props.protocol_name;
-	dev_name = kasprintf(GFP_KERNEL,
-		"{%02X%02X%02X%02X-%02X%02X-%02X%02X-%02X%02X-%02X%02X%02X%02X%02X%02X}",
-		device_uuid.b[3], device_uuid.b[2], device_uuid.b[1],
-		device_uuid.b[0], device_uuid.b[5], device_uuid.b[4],
-		device_uuid.b[7], device_uuid.b[6], device_uuid.b[8],
-		device_uuid.b[9], device_uuid.b[10], device_uuid.b[11],
-		device_uuid.b[12], device_uuid.b[13], device_uuid.b[14],
-		device_uuid.b[15]);
+	dev_name = kasprintf(GFP_KERNEL, "{%pUL}", device_uuid.b);
 	if (!dev_name)
 		return	-ENOMEM;
 
diff --git a/drivers/hid/intel-ish-hid/ishtp/hbm.c b/drivers/hid/intel-ish-hid/ishtp/hbm.c
index 74bffee60774..59460b66e689 100644
--- a/drivers/hid/intel-ish-hid/ishtp/hbm.c
+++ b/drivers/hid/intel-ish-hid/ishtp/hbm.c
@@ -378,11 +378,10 @@ static void ishtp_hbm_cl_disconnect_res(struct ishtp_device *dev,
 	list_for_each_entry(cl, &dev->cl_list, link) {
 		if (!rs->status && ishtp_hbm_cl_addr_equal(cl, rs)) {
 			cl->state = ISHTP_CL_DISCONNECTED;
+			wake_up_interruptible(&cl->wait_ctrl_res);
 			break;
 		}
 	}
-	if (cl)
-		wake_up_interruptible(&cl->wait_ctrl_res);
 	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
 }
 
@@ -431,11 +430,10 @@ static void ishtp_hbm_cl_connect_res(struct ishtp_device *dev,
 				cl->state = ISHTP_CL_DISCONNECTED;
 				cl->status = -ENODEV;
 			}
+			wake_up_interruptible(&cl->wait_ctrl_res);
 			break;
 		}
 	}
-	if (cl)
-		wake_up_interruptible(&cl->wait_ctrl_res);
 	spin_unlock_irqrestore(&dev->cl_list_lock, flags);
 }
 
diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c
index ae83af649a60..333108ef18cf 100644
--- a/drivers/hid/usbhid/hid-core.c
+++ b/drivers/hid/usbhid/hid-core.c
@@ -1459,7 +1459,7 @@ static int hid_post_reset(struct usb_interface *intf)
 	rdesc = kmalloc(hid->dev_rsize, GFP_KERNEL);
 	if (!rdesc) {
 		dbg_hid("couldn't allocate rdesc memory (post_reset)\n");
-		return 1;
+		return -ENOMEM;
 	}
 	status = hid_get_class_descriptor(dev,
 				interface->desc.bInterfaceNumber,
@@ -1467,13 +1467,13 @@ static int hid_post_reset(struct usb_interface *intf)
 	if (status < 0) {
 		dbg_hid("reading report descriptor failed (post_reset)\n");
 		kfree(rdesc);
-		return 1;
+		return status;
 	}
 	status = memcmp(rdesc, hid->dev_rdesc, hid->dev_rsize);
 	kfree(rdesc);
 	if (status != 0) {
 		dbg_hid("report descriptor changed\n");
-		return 1;
+		return -EPERM;
 	}
 
 	/* No need to do another reset or clear a halted endpoint */
diff --git a/drivers/hid/usbhid/hid-quirks.c b/drivers/hid/usbhid/hid-quirks.c
index e6cfd323babc..b3e01c82af05 100644
--- a/drivers/hid/usbhid/hid-quirks.c
+++ b/drivers/hid/usbhid/hid-quirks.c
@@ -82,6 +82,8 @@ static const struct hid_blacklist {
 	{ USB_VENDOR_ID_CREATIVELABS, USB_DEVICE_ID_CREATIVE_SB_OMNI_SURROUND_51, HID_QUIRK_NOGET },
 	{ USB_VENDOR_ID_DMI, USB_DEVICE_ID_DMI_ENC, HID_QUIRK_NOGET },
 	{ USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_WIIU, HID_QUIRK_MULTI_INPUT },
+	{ USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_PS3, HID_QUIRK_MULTI_INPUT },
+	{ USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_GAMECUBE, HID_QUIRK_MULTI_INPUT },
 	{ USB_VENDOR_ID_ELAN, HID_ANY_ID, HID_QUIRK_ALWAYS_POLL },
 	{ USB_VENDOR_ID_ELO, USB_DEVICE_ID_ELO_TS2700, HID_QUIRK_NOGET },
 	{ USB_VENDOR_ID_FORMOSA, USB_DEVICE_ID_FORMOSA_IR_RECEIVER, HID_QUIRK_NO_INIT_REPORTS },
@@ -101,8 +103,9 @@ static const struct hid_blacklist {
 	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_2, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_3_JP, HID_QUIRK_NO_INIT_REPORTS },
+	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4, HID_QUIRK_NO_INIT_REPORTS },
+	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_2, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_PRO_4_JP, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_3, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_POWER_COVER, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_MSI, USB_DEVICE_ID_MSI_GT683R_LED_PANEL, HID_QUIRK_NO_INIT_REPORTS },
 	{ USB_VENDOR_ID_NEXIO, USB_DEVICE_ID_NEXIO_MULTITOUCH_PTI0750, HID_QUIRK_NO_INIT_REPORTS },
diff --git a/drivers/hid/wacom.h b/drivers/hid/wacom.h
index b4800ea891cb..d303e413306d 100644
--- a/drivers/hid/wacom.h
+++ b/drivers/hid/wacom.h
@@ -210,7 +210,7 @@ int wacom_setup_pad_input_capabilities(struct input_dev *input_dev,
 				       struct wacom_wac *wacom_wac);
 void wacom_wac_usage_mapping(struct hid_device *hdev,
 		struct hid_field *field, struct hid_usage *usage);
-int wacom_wac_event(struct hid_device *hdev, struct hid_field *field,
+void wacom_wac_event(struct hid_device *hdev, struct hid_field *field,
 		struct hid_usage *usage, __s32 value);
 void wacom_wac_report(struct hid_device *hdev, struct hid_report *report);
 void wacom_battery_work(struct work_struct *work);
diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c
index 5e7a5648e708..b9779bcbd140 100644
--- a/drivers/hid/wacom_sys.c
+++ b/drivers/hid/wacom_sys.c
@@ -122,6 +122,7 @@ static void wacom_feature_mapping(struct hid_device *hdev,
 	struct hid_data *hid_data = &wacom->wacom_wac.hid_data;
 	u8 *data;
 	int ret;
+	int n;
 
 	switch (usage->hid) {
 	case HID_DG_CONTACTMAX:
@@ -159,22 +160,48 @@ static void wacom_feature_mapping(struct hid_device *hdev,
 
 	case HID_UP_DIGITIZER:
 		if (field->report->id == 0x0B &&
-		    (field->application == WACOM_G9_DIGITIZER ||
-		     field->application == WACOM_G11_DIGITIZER)) {
+		    (field->application == WACOM_HID_G9_PEN ||
+		     field->application == WACOM_HID_G11_PEN)) {
 			wacom->wacom_wac.mode_report = field->report->id;
 			wacom->wacom_wac.mode_value = 0;
 		}
 		break;
 
-	case WACOM_G9_PAGE:
-	case WACOM_G11_PAGE:
+	case WACOM_HID_WD_DATAMODE:
+		wacom->wacom_wac.mode_report = field->report->id;
+		wacom->wacom_wac.mode_value = 2;
+		break;
+
+	case WACOM_HID_UP_G9:
+	case WACOM_HID_UP_G11:
 		if (field->report->id == 0x03 &&
-		    (field->application == WACOM_G9_TOUCHSCREEN ||
-		     field->application == WACOM_G11_TOUCHSCREEN)) {
+		    (field->application == WACOM_HID_G9_TOUCHSCREEN ||
+		     field->application == WACOM_HID_G11_TOUCHSCREEN)) {
 			wacom->wacom_wac.mode_report = field->report->id;
 			wacom->wacom_wac.mode_value = 0;
 		}
 		break;
+	case WACOM_HID_WD_OFFSETLEFT:
+	case WACOM_HID_WD_OFFSETTOP:
+	case WACOM_HID_WD_OFFSETRIGHT:
+	case WACOM_HID_WD_OFFSETBOTTOM:
+		/* read manually */
+		n = hid_report_len(field->report);
+		data = hid_alloc_report_buf(field->report, GFP_KERNEL);
+		if (!data)
+			break;
+		data[0] = field->report->id;
+		ret = wacom_get_report(hdev, HID_FEATURE_REPORT,
+					data, n, WAC_CMD_RETRIES);
+		if (ret == n) {
+			ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT,
+						   data, n, 0);
+		} else {
+			hid_warn(hdev, "%s: could not retrieve sensor offsets\n",
+				 __func__);
+		}
+		kfree(data);
+		break;
 	}
 }
 
@@ -240,6 +267,30 @@ static void wacom_usage_mapping(struct hid_device *hdev,
 			features->touch_max = 1;
 	}
 
+	/*
+	 * ISDv4 devices which predate HID's adoption of the
+	 * HID_DG_BARELSWITCH2 usage use 0x000D0000 in its
+	 * position instead. We can accurately detect if a
+	 * usage with that value should be HID_DG_BARRELSWITCH2
+	 * based on the surrounding usages, which have remained
+	 * constant across generations.
+	 */
+	if (features->type == HID_GENERIC &&
+	    usage->hid == 0x000D0000 &&
+	    field->application == HID_DG_PEN &&
+	    field->physical == HID_DG_STYLUS) {
+		int i = usage->usage_index;
+
+		if (i-4 >= 0 && i+1 < field->maxusage &&
+		    field->usage[i-4].hid == HID_DG_TIPSWITCH &&
+		    field->usage[i-3].hid == HID_DG_BARRELSWITCH &&
+		    field->usage[i-2].hid == HID_DG_ERASER &&
+		    field->usage[i-1].hid == HID_DG_INVERT &&
+		    field->usage[i+1].hid == HID_DG_INRANGE) {
+			usage->hid = HID_DG_BARRELSWITCH2;
+		}
+	}
+
 	switch (usage->hid) {
 	case HID_GD_X:
 		features->x_max = field->logical_maximum;
@@ -689,11 +740,6 @@ static int wacom_add_shared_data(struct hid_device *hdev)
 		return retval;
 	}
 
-	if (wacom_wac->features.device_type & WACOM_DEVICETYPE_TOUCH)
-		wacom_wac->shared->touch = hdev;
-	else if (wacom_wac->features.device_type & WACOM_DEVICETYPE_PEN)
-		wacom_wac->shared->pen = hdev;
-
 out:
 	mutex_unlock(&wacom_udev_list_lock);
 	return retval;
@@ -1916,6 +1962,19 @@ static void wacom_update_name(struct wacom *wacom, const char *suffix)
 				/* shift everything including the terminator */
 				memmove(gap, gap+1, strlen(gap));
 			}
+
+			/* strip off excessive prefixing */
+			if (strstr(name, "Wacom Co.,Ltd. Wacom ") == name) {
+				int n = strlen(name);
+				int x = strlen("Wacom Co.,Ltd. ");
+				memmove(name, name+x, n-x+1);
+			}
+			if (strstr(name, "Wacom Co., Ltd. Wacom ") == name) {
+				int n = strlen(name);
+				int x = strlen("Wacom Co., Ltd. ");
+				memmove(name, name+x, n-x+1);
+			}
+
 			/* get rid of trailing whitespace */
 			if (name[strlen(name)-1] == ' ')
 				name[strlen(name)-1] = '\0';
@@ -1977,6 +2036,10 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 	if (error)
 		goto fail;
 
+	error = wacom_add_shared_data(hdev);
+	if (error)
+		goto fail;
+
 	/*
 	 * Bamboo Pad has a generic hid handling for the Pen, and we switch it
 	 * into debug mode for the touch part.
@@ -2017,9 +2080,10 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 
 	wacom_update_name(wacom, wireless ? " (WL)" : "");
 
-	error = wacom_add_shared_data(hdev);
-	if (error)
-		goto fail;
+	if (wacom_wac->features.device_type & WACOM_DEVICETYPE_TOUCH)
+		wacom_wac->shared->touch = hdev;
+	else if (wacom_wac->features.device_type & WACOM_DEVICETYPE_PEN)
+		wacom_wac->shared->pen = hdev;
 
 	if (!(features->device_type & WACOM_DEVICETYPE_WL_MONITOR) &&
 	     (features->quirks & WACOM_QUIRK_BATTERY)) {
diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c
index 1cb79925730d..b1a9a3ca6d56 100644
--- a/drivers/hid/wacom_wac.c
+++ b/drivers/hid/wacom_wac.c
@@ -41,6 +41,8 @@ MODULE_PARM_DESC(touch_arbitration, " on (Y) off (N)");
 static void wacom_report_numbered_buttons(struct input_dev *input_dev,
 				int button_count, int mask);
 
+static int wacom_numbered_button_to_key(int n);
+
 /*
  * Percent of battery capacity for Graphire.
  * 8th value means AC online and show 100% capacity.
@@ -588,6 +590,11 @@ static int wacom_intuos_pad(struct wacom_wac *wacom)
 	return 1;
 }
 
+static int wacom_intuos_id_mangle(int tool_id)
+{
+	return (tool_id & ~0xFFF) << 4 | (tool_id & 0xFFF);
+}
+
 static int wacom_intuos_get_tool_type(int tool_id)
 {
 	int tool_type;
@@ -595,7 +602,7 @@ static int wacom_intuos_get_tool_type(int tool_id)
 	switch (tool_id) {
 	case 0x812: /* Inking pen */
 	case 0x801: /* Intuos3 Inking pen */
-	case 0x120802: /* Intuos4/5 Inking Pen */
+	case 0x12802: /* Intuos4/5 Inking Pen */
 	case 0x012:
 		tool_type = BTN_TOOL_PENCIL;
 		break;
@@ -610,11 +617,11 @@ static int wacom_intuos_get_tool_type(int tool_id)
 	case 0x804: /* Intuos4/5 13HD/24HD Marker Pen */
 	case 0x8e2: /* IntuosHT2 pen */
 	case 0x022:
-	case 0x100804: /* Intuos4/5 13HD/24HD Art Pen */
-	case 0x140802: /* Intuos4/5 13HD/24HD Classic Pen */
-	case 0x160802: /* Cintiq 13HD Pro Pen */
-	case 0x180802: /* DTH2242 Pen */
-	case 0x100802: /* Intuos4/5 13HD/24HD General Pen */
+	case 0x10804: /* Intuos4/5 13HD/24HD Art Pen */
+	case 0x14802: /* Intuos4/5 13HD/24HD Classic Pen */
+	case 0x16802: /* Cintiq 13HD Pro Pen */
+	case 0x18802: /* DTH2242 Pen */
+	case 0x10802: /* Intuos4/5 13HD/24HD General Pen */
 		tool_type = BTN_TOOL_PEN;
 		break;
 
@@ -638,6 +645,7 @@ static int wacom_intuos_get_tool_type(int tool_id)
 		break;
 
 	case 0x82a: /* Eraser */
+	case 0x84a:
 	case 0x85a:
 	case 0x91a:
 	case 0xd1a:
@@ -648,12 +656,12 @@ static int wacom_intuos_get_tool_type(int tool_id)
 	case 0x80c: /* Intuos4/5 13HD/24HD Marker Pen Eraser */
 	case 0x80a: /* Intuos4/5 13HD/24HD General Pen Eraser */
 	case 0x90a: /* Intuos4/5 13HD/24HD Airbrush Eraser */
-	case 0x14080a: /* Intuos4/5 13HD/24HD Classic Pen Eraser */
-	case 0x10090a: /* Intuos4/5 13HD/24HD Airbrush Eraser */
-	case 0x10080c: /* Intuos4/5 13HD/24HD Art Pen Eraser */
-	case 0x16080a: /* Cintiq 13HD Pro Pen Eraser */
-	case 0x18080a: /* DTH2242 Eraser */
-	case 0x10080a: /* Intuos4/5 13HD/24HD General Pen Eraser */
+	case 0x1480a: /* Intuos4/5 13HD/24HD Classic Pen Eraser */
+	case 0x1090a: /* Intuos4/5 13HD/24HD Airbrush Eraser */
+	case 0x1080c: /* Intuos4/5 13HD/24HD Art Pen Eraser */
+	case 0x1680a: /* Cintiq 13HD Pro Pen Eraser */
+	case 0x1880a: /* DTH2242 Eraser */
+	case 0x1080a: /* Intuos4/5 13HD/24HD General Pen Eraser */
 		tool_type = BTN_TOOL_RUBBER;
 		break;
 
@@ -662,7 +670,7 @@ static int wacom_intuos_get_tool_type(int tool_id)
 	case 0x112:
 	case 0x913: /* Intuos3 Airbrush */
 	case 0x902: /* Intuos4/5 13HD/24HD Airbrush */
-	case 0x100902: /* Intuos4/5 13HD/24HD Airbrush */
+	case 0x10902: /* Intuos4/5 13HD/24HD Airbrush */
 		tool_type = BTN_TOOL_AIRBRUSH;
 		break;
 
@@ -693,7 +701,7 @@ static int wacom_intuos_inout(struct wacom_wac *wacom)
 			(data[6] << 4) + (data[7] >> 4);
 
 		wacom->id[idx] = (data[2] << 4) | (data[3] >> 4) |
-			((data[7] & 0x0f) << 20) | ((data[8] & 0xf0) << 12);
+		     ((data[7] & 0x0f) << 16) | ((data[8] & 0xf0) << 8);
 
 		wacom->tool[idx] = wacom_intuos_get_tool_type(wacom->id[idx]);
 
@@ -923,7 +931,7 @@ static int wacom_intuos_general(struct wacom_wac *wacom)
 	 * don't report events for invalid data
 	 */
 	/* older I4 styli don't work with new Cintiqs */
-	if ((!((wacom->id[idx] >> 20) & 0x01) &&
+	if ((!((wacom->id[idx] >> 16) & 0x01) &&
 			(features->type == WACOM_21UX2)) ||
 	    /* Only large Intuos support Lense Cursor */
 	    (wacom->tool[idx] == BTN_TOOL_LENS &&
@@ -1059,7 +1067,8 @@ static int wacom_intuos_general(struct wacom_wac *wacom)
 		break;
 	}
 
-	input_report_abs(input, ABS_MISC, wacom->id[idx]); /* report tool id */
+	input_report_abs(input, ABS_MISC,
+			 wacom_intuos_id_mangle(wacom->id[idx])); /* report tool id */
 	input_report_key(input, wacom->tool[idx], 1);
 	input_event(input, EV_MSC, MSC_SERIAL, wacom->serial[idx]);
 	wacom->reporting_data = true;
@@ -1435,11 +1444,59 @@ static int wacom_tpc_irq(struct wacom_wac *wacom, size_t len)
 	return 0;
 }
 
+static int wacom_equivalent_usage(int usage)
+{
+	if ((usage & HID_USAGE_PAGE) == WACOM_HID_UP_WACOMDIGITIZER) {
+		int subpage = (usage & 0xFF00) << 8;
+		int subusage = (usage & 0xFF);
+
+		if (subpage == WACOM_HID_SP_PAD ||
+		    subpage == WACOM_HID_SP_BUTTON ||
+		    subpage == WACOM_HID_SP_DIGITIZER ||
+		    subpage == WACOM_HID_SP_DIGITIZERINFO ||
+		    usage == WACOM_HID_WD_SENSE ||
+		    usage == WACOM_HID_WD_SERIALHI ||
+		    usage == WACOM_HID_WD_TOOLTYPE ||
+		    usage == WACOM_HID_WD_DISTANCE ||
+		    usage == WACOM_HID_WD_TOUCHSTRIP ||
+		    usage == WACOM_HID_WD_TOUCHSTRIP2 ||
+		    usage == WACOM_HID_WD_TOUCHRING ||
+		    usage == WACOM_HID_WD_TOUCHRINGSTATUS) {
+			return usage;
+		}
+
+		if (subpage == HID_UP_UNDEFINED)
+			subpage = HID_UP_DIGITIZER;
+
+		return subpage | subusage;
+	}
+
+	return usage;
+}
+
 static void wacom_map_usage(struct input_dev *input, struct hid_usage *usage,
 		struct hid_field *field, __u8 type, __u16 code, int fuzz)
 {
+	struct wacom *wacom = input_get_drvdata(input);
+	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	struct wacom_features *features = &wacom_wac->features;
 	int fmin = field->logical_minimum;
 	int fmax = field->logical_maximum;
+	unsigned int equivalent_usage = wacom_equivalent_usage(usage->hid);
+	int resolution_code = code;
+
+	if (equivalent_usage == HID_DG_TWIST) {
+		resolution_code = ABS_RZ;
+	}
+
+	if (equivalent_usage == HID_GD_X) {
+		fmin += features->offset_left;
+		fmax -= features->offset_right;
+	}
+	if (equivalent_usage == HID_GD_Y) {
+		fmin += features->offset_top;
+		fmax -= features->offset_bottom;
+	}
 
 	usage->type = type;
 	usage->code = code;
@@ -1450,7 +1507,7 @@ static void wacom_map_usage(struct input_dev *input, struct hid_usage *usage,
 	case EV_ABS:
 		input_set_abs_params(input, code, fmin, fmax, fuzz, 0);
 		input_abs_set_res(input, code,
-				  hidinput_calc_abs_res(field, code));
+				  hidinput_calc_abs_res(field, resolution_code));
 		break;
 	case EV_KEY:
 		input_set_capability(input, EV_KEY, code);
@@ -1458,6 +1515,172 @@ static void wacom_map_usage(struct input_dev *input, struct hid_usage *usage,
 	case EV_MSC:
 		input_set_capability(input, EV_MSC, code);
 		break;
+	case EV_SW:
+		input_set_capability(input, EV_SW, code);
+		break;
+	}
+}
+
+static void wacom_wac_pad_usage_mapping(struct hid_device *hdev,
+		struct hid_field *field, struct hid_usage *usage)
+{
+	struct wacom *wacom = hid_get_drvdata(hdev);
+	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	struct wacom_features *features = &wacom_wac->features;
+	struct input_dev *input = wacom_wac->pad_input;
+	unsigned equivalent_usage = wacom_equivalent_usage(usage->hid);
+
+	switch (equivalent_usage) {
+	case WACOM_HID_WD_BATTERY_LEVEL:
+	case WACOM_HID_WD_BATTERY_CHARGING:
+		features->quirks |= WACOM_QUIRK_BATTERY;
+		break;
+	case WACOM_HID_WD_ACCELEROMETER_X:
+		__set_bit(INPUT_PROP_ACCELEROMETER, input->propbit);
+		wacom_map_usage(input, usage, field, EV_ABS, ABS_X, 0);
+		features->device_type |= WACOM_DEVICETYPE_PAD;
+		break;
+	case WACOM_HID_WD_ACCELEROMETER_Y:
+		__set_bit(INPUT_PROP_ACCELEROMETER, input->propbit);
+		wacom_map_usage(input, usage, field, EV_ABS, ABS_Y, 0);
+		features->device_type |= WACOM_DEVICETYPE_PAD;
+		break;
+	case WACOM_HID_WD_ACCELEROMETER_Z:
+		__set_bit(INPUT_PROP_ACCELEROMETER, input->propbit);
+		wacom_map_usage(input, usage, field, EV_ABS, ABS_Z, 0);
+		features->device_type |= WACOM_DEVICETYPE_PAD;
+		break;
+	case WACOM_HID_WD_BUTTONHOME:
+	case WACOM_HID_WD_BUTTONUP:
+	case WACOM_HID_WD_BUTTONDOWN:
+	case WACOM_HID_WD_BUTTONLEFT:
+	case WACOM_HID_WD_BUTTONRIGHT:
+	case WACOM_HID_WD_BUTTONCENTER:
+		wacom_map_usage(input, usage, field, EV_KEY,
+				wacom_numbered_button_to_key(features->numbered_buttons),
+				0);
+		features->numbered_buttons++;
+		features->device_type |= WACOM_DEVICETYPE_PAD;
+		break;
+	case WACOM_HID_WD_TOUCHONOFF:
+		wacom_map_usage(input, usage, field, EV_SW, SW_MUTE_DEVICE, 0);
+		features->device_type |= WACOM_DEVICETYPE_PAD;
+		break;
+	case WACOM_HID_WD_TOUCHSTRIP:
+		wacom_map_usage(input, usage, field, EV_ABS, ABS_RX, 0);
+		features->device_type |= WACOM_DEVICETYPE_PAD;
+		break;
+	case WACOM_HID_WD_TOUCHSTRIP2:
+		wacom_map_usage(input, usage, field, EV_ABS, ABS_RY, 0);
+		features->device_type |= WACOM_DEVICETYPE_PAD;
+		break;
+	case WACOM_HID_WD_TOUCHRING:
+		wacom_map_usage(input, usage, field, EV_ABS, ABS_WHEEL, 0);
+		features->device_type |= WACOM_DEVICETYPE_PAD;
+		break;
+	}
+
+	switch (equivalent_usage & 0xfffffff0) {
+	case WACOM_HID_WD_EXPRESSKEY00:
+		wacom_map_usage(input, usage, field, EV_KEY,
+				wacom_numbered_button_to_key(features->numbered_buttons),
+				0);
+		features->numbered_buttons++;
+		features->device_type |= WACOM_DEVICETYPE_PAD;
+		break;
+	}
+}
+
+static void wacom_wac_pad_battery_event(struct hid_device *hdev, struct hid_field *field,
+		struct hid_usage *usage, __s32 value)
+{
+	struct wacom *wacom = hid_get_drvdata(hdev);
+	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	unsigned equivalent_usage = wacom_equivalent_usage(usage->hid);
+
+	switch (equivalent_usage) {
+	case WACOM_HID_WD_BATTERY_LEVEL:
+		wacom_wac->hid_data.battery_capacity = value;
+		wacom_wac->hid_data.bat_connected = 1;
+		break;
+
+	case WACOM_HID_WD_BATTERY_CHARGING:
+		wacom_wac->hid_data.bat_charging = value;
+		wacom_wac->hid_data.ps_connected = value;
+		wacom_wac->hid_data.bat_connected = 1;
+		break;
+	}
+}
+
+static void wacom_wac_pad_event(struct hid_device *hdev, struct hid_field *field,
+		struct hid_usage *usage, __s32 value)
+{
+	struct wacom *wacom = hid_get_drvdata(hdev);
+	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	struct input_dev *input = wacom_wac->pad_input;
+	struct wacom_features *features = &wacom_wac->features;
+	unsigned equivalent_usage = wacom_equivalent_usage(usage->hid);
+
+	if (wacom_equivalent_usage(field->physical) == HID_DG_TABLETFUNCTIONKEY) {
+		wacom_wac->hid_data.inrange_state |= value;
+	}
+
+	switch (equivalent_usage) {
+	case WACOM_HID_WD_TOUCHRINGSTATUS:
+		break;
+
+	default:
+		features->input_event_flag = true;
+		input_event(input, usage->type, usage->code, value);
+		break;
+	}
+}
+
+static void wacom_wac_pad_pre_report(struct hid_device *hdev,
+		struct hid_report *report)
+{
+	struct wacom *wacom = hid_get_drvdata(hdev);
+	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+
+	wacom_wac->hid_data.inrange_state = 0;
+}
+
+static void wacom_wac_pad_battery_report(struct hid_device *hdev,
+		struct hid_report *report)
+{
+	struct wacom *wacom = hid_get_drvdata(hdev);
+	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	struct wacom_features *features = &wacom_wac->features;
+
+	if (features->quirks & WACOM_QUIRK_BATTERY) {
+		int capacity = wacom_wac->hid_data.battery_capacity;
+		bool charging = wacom_wac->hid_data.bat_charging;
+		bool connected = wacom_wac->hid_data.bat_connected;
+		bool powered = wacom_wac->hid_data.ps_connected;
+
+		wacom_notify_battery(wacom_wac, capacity, charging,
+				     connected, powered);
+	}
+}
+
+static void wacom_wac_pad_report(struct hid_device *hdev,
+		struct hid_report *report)
+{
+	struct wacom *wacom = hid_get_drvdata(hdev);
+	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	struct wacom_features *features = &wacom_wac->features;
+	struct input_dev *input = wacom_wac->pad_input;
+	bool active = wacom_wac->hid_data.inrange_state != 0;
+
+	/* report prox for expresskey events */
+	if (wacom_equivalent_usage(report->field[0]->physical) == HID_DG_TABLETFUNCTIONKEY) {
+		features->input_event_flag = true;
+		input_event(input, EV_ABS, ABS_MISC, active ? PAD_DEVICE_ID : 0);
+	}
+
+	if (features->input_event_flag) {
+		features->input_event_flag = false;
+		input_sync(input);
 	}
 }
 
@@ -1466,25 +1689,43 @@ static void wacom_wac_pen_usage_mapping(struct hid_device *hdev,
 {
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	struct wacom_features *features = &wacom_wac->features;
 	struct input_dev *input = wacom_wac->pen_input;
+	unsigned equivalent_usage = wacom_equivalent_usage(usage->hid);
 
-	switch (usage->hid) {
+	switch (equivalent_usage) {
 	case HID_GD_X:
 		wacom_map_usage(input, usage, field, EV_ABS, ABS_X, 4);
 		break;
 	case HID_GD_Y:
 		wacom_map_usage(input, usage, field, EV_ABS, ABS_Y, 4);
 		break;
+	case WACOM_HID_WD_DISTANCE:
+	case HID_GD_Z:
+		wacom_map_usage(input, usage, field, EV_ABS, ABS_DISTANCE, 0);
+		break;
 	case HID_DG_TIPPRESSURE:
 		wacom_map_usage(input, usage, field, EV_ABS, ABS_PRESSURE, 0);
 		break;
 	case HID_DG_INRANGE:
 		wacom_map_usage(input, usage, field, EV_KEY, BTN_TOOL_PEN, 0);
 		break;
+	case HID_DG_BATTERYSTRENGTH:
+		features->quirks |= WACOM_QUIRK_BATTERY;
+		break;
 	case HID_DG_INVERT:
 		wacom_map_usage(input, usage, field, EV_KEY,
 				BTN_TOOL_RUBBER, 0);
 		break;
+	case HID_DG_TILT_X:
+		wacom_map_usage(input, usage, field, EV_ABS, ABS_TILT_X, 0);
+		break;
+	case HID_DG_TILT_Y:
+		wacom_map_usage(input, usage, field, EV_ABS, ABS_TILT_Y, 0);
+		break;
+	case HID_DG_TWIST:
+		wacom_map_usage(input, usage, field, EV_ABS, ABS_Z, 0);
+		break;
 	case HID_DG_ERASER:
 	case HID_DG_TIPSWITCH:
 		wacom_map_usage(input, usage, field, EV_KEY, BTN_TOUCH, 0);
@@ -1498,39 +1739,131 @@ static void wacom_wac_pen_usage_mapping(struct hid_device *hdev,
 	case HID_DG_TOOLSERIALNUMBER:
 		wacom_map_usage(input, usage, field, EV_MSC, MSC_SERIAL, 0);
 		break;
+	case WACOM_HID_WD_SENSE:
+		features->quirks |= WACOM_QUIRK_SENSE;
+		wacom_map_usage(input, usage, field, EV_KEY, BTN_TOOL_PEN, 0);
+		break;
+	case WACOM_HID_WD_SERIALHI:
+		wacom_map_usage(input, usage, field, EV_ABS, ABS_MISC, 0);
+		set_bit(EV_KEY, input->evbit);
+		input_set_capability(input, EV_KEY, BTN_TOOL_PEN);
+		input_set_capability(input, EV_KEY, BTN_TOOL_RUBBER);
+		input_set_capability(input, EV_KEY, BTN_TOOL_BRUSH);
+		input_set_capability(input, EV_KEY, BTN_TOOL_PENCIL);
+		input_set_capability(input, EV_KEY, BTN_TOOL_AIRBRUSH);
+		input_set_capability(input, EV_KEY, BTN_TOOL_MOUSE);
+		input_set_capability(input, EV_KEY, BTN_TOOL_LENS);
+		break;
+	case WACOM_HID_WD_FINGERWHEEL:
+		wacom_map_usage(input, usage, field, EV_ABS, ABS_WHEEL, 0);
+		break;
 	}
 }
 
-static int wacom_wac_pen_event(struct hid_device *hdev, struct hid_field *field,
+static void wacom_wac_pen_event(struct hid_device *hdev, struct hid_field *field,
 		struct hid_usage *usage, __s32 value)
 {
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	struct wacom_features *features = &wacom_wac->features;
 	struct input_dev *input = wacom_wac->pen_input;
+	unsigned equivalent_usage = wacom_equivalent_usage(usage->hid);
 
-	/* checking which Tool / tip switch to send */
-	switch (usage->hid) {
+	switch (equivalent_usage) {
+	case HID_GD_Z:
+		/*
+		 * HID_GD_Z "should increase as the control's position is
+		 * moved from high to low", while ABS_DISTANCE instead
+		 * increases in value as the tool moves from low to high.
+		 */
+		value = field->logical_maximum - value;
+		break;
 	case HID_DG_INRANGE:
 		wacom_wac->hid_data.inrange_state = value;
-		return 0;
+		if (!(features->quirks & WACOM_QUIRK_SENSE))
+			wacom_wac->hid_data.sense_state = value;
+		return;
+	case HID_DG_BATTERYSTRENGTH:
+		wacom_wac->hid_data.battery_capacity = value;
+		wacom_wac->hid_data.bat_connected = 1;
+		break;
 	case HID_DG_INVERT:
 		wacom_wac->hid_data.invert_state = value;
-		return 0;
+		return;
 	case HID_DG_ERASER:
 	case HID_DG_TIPSWITCH:
 		wacom_wac->hid_data.tipswitch |= value;
-		return 0;
+		return;
+	case HID_DG_TOOLSERIALNUMBER:
+		wacom_wac->serial[0] = (wacom_wac->serial[0] & ~0xFFFFFFFFULL);
+		wacom_wac->serial[0] |= value;
+		return;
+	case WACOM_HID_WD_SENSE:
+		wacom_wac->hid_data.sense_state = value;
+		return;
+	case WACOM_HID_WD_SERIALHI:
+		wacom_wac->serial[0] = (wacom_wac->serial[0] & 0xFFFFFFFF);
+		wacom_wac->serial[0] |= ((__u64)value) << 32;
+		/*
+		 * Non-USI EMR devices may contain additional tool type
+		 * information here. See WACOM_HID_WD_TOOLTYPE case for
+		 * more details.
+		 */
+		if (value >> 20 == 1) {
+			wacom_wac->id[0] |= value & 0xFFFFF;
+		}
+		return;
+	case WACOM_HID_WD_TOOLTYPE:
+		/*
+		 * Some devices (MobileStudio Pro, and possibly later
+		 * devices as well) do not return the complete tool
+		 * type in their WACOM_HID_WD_TOOLTYPE usage. Use a
+		 * bitwise OR so the complete value can be built
+		 * up over time :(
+		 */
+		wacom_wac->id[0] |= value;
+		return;
+	case WACOM_HID_WD_OFFSETLEFT:
+		if (features->offset_left && value != features->offset_left)
+			hid_warn(hdev, "%s: overriding exising left offset "
+				 "%d -> %d\n", __func__, value,
+				 features->offset_left);
+		features->offset_left = value;
+		return;
+	case WACOM_HID_WD_OFFSETRIGHT:
+		if (features->offset_right && value != features->offset_right)
+			hid_warn(hdev, "%s: overriding exising right offset "
+				 "%d -> %d\n", __func__, value,
+				 features->offset_right);
+		features->offset_right = value;
+		return;
+	case WACOM_HID_WD_OFFSETTOP:
+		if (features->offset_top && value != features->offset_top)
+			hid_warn(hdev, "%s: overriding exising top offset "
+				 "%d -> %d\n", __func__, value,
+				 features->offset_top);
+		features->offset_top = value;
+		return;
+	case WACOM_HID_WD_OFFSETBOTTOM:
+		if (features->offset_bottom && value != features->offset_bottom)
+			hid_warn(hdev, "%s: overriding exising bottom offset "
+				 "%d -> %d\n", __func__, value,
+				 features->offset_bottom);
+		features->offset_bottom = value;
+		return;
 	}
 
 	/* send pen events only when touch is up or forced out
 	 * or touch arbitration is off
 	 */
 	if (!usage->type || delay_pen_events(wacom_wac))
-		return 0;
+		return;
 
-	input_event(input, usage->type, usage->code, value);
+	/* send pen events only when the pen is in/entering/leaving proximity */
+	if (!wacom_wac->hid_data.inrange_state && !wacom_wac->tool[0])
+		return;
 
-	return 0;
+	input_event(input, usage->type, usage->code, value);
 }
 
 static void wacom_wac_pen_pre_report(struct hid_device *hdev,
@@ -1546,24 +1879,53 @@ static void wacom_wac_pen_report(struct hid_device *hdev,
 	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
 	struct input_dev *input = wacom_wac->pen_input;
 	bool prox = wacom_wac->hid_data.inrange_state;
+	bool range = wacom_wac->hid_data.sense_state;
 
-	if (!wacom_wac->shared->stylus_in_proximity) /* first in prox */
+	if (!wacom_wac->tool[0] && prox) { /* first in prox */
 		/* Going into proximity select tool */
-		wacom_wac->tool[0] = wacom_wac->hid_data.invert_state ?
-						BTN_TOOL_RUBBER : BTN_TOOL_PEN;
+		if (wacom_wac->hid_data.invert_state)
+			wacom_wac->tool[0] = BTN_TOOL_RUBBER;
+		else if (wacom_wac->id[0])
+			wacom_wac->tool[0] = wacom_intuos_get_tool_type(wacom_wac->id[0]);
+		else
+			wacom_wac->tool[0] = BTN_TOOL_PEN;
+	}
 
 	/* keep pen state for touch events */
-	wacom_wac->shared->stylus_in_proximity = prox;
+	wacom_wac->shared->stylus_in_proximity = range;
 
-	if (!delay_pen_events(wacom_wac)) {
+	if (!delay_pen_events(wacom_wac) && wacom_wac->tool[0]) {
+		int id = wacom_wac->id[0];
+
+		/*
+		 * Non-USI EMR tools should have their IDs mangled to
+		 * match the legacy behavior of wacom_intuos_general
+		 */
+		if (wacom_wac->serial[0] >> 52 == 1)
+			id = wacom_intuos_id_mangle(id);
+
+		/*
+		 * To ensure compatibility with xf86-input-wacom, we should
+		 * report the BTN_TOOL_* event prior to the ABS_MISC or
+		 * MSC_SERIAL events.
+		 */
 		input_report_key(input, BTN_TOUCH,
 				wacom_wac->hid_data.tipswitch);
 		input_report_key(input, wacom_wac->tool[0], prox);
+		if (wacom_wac->serial[0]) {
+			input_event(input, EV_MSC, MSC_SERIAL, wacom_wac->serial[0]);
+			input_report_abs(input, ABS_MISC, id);
+		}
 
 		wacom_wac->hid_data.tipswitch = false;
 
 		input_sync(input);
 	}
+
+	if (!prox) {
+		wacom_wac->tool[0] = 0;
+		wacom_wac->id[0] = 0;
+	}
 }
 
 static void wacom_wac_finger_usage_mapping(struct hid_device *hdev,
@@ -1573,8 +1935,9 @@ static void wacom_wac_finger_usage_mapping(struct hid_device *hdev,
 	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
 	struct input_dev *input = wacom_wac->touch_input;
 	unsigned touch_max = wacom_wac->features.touch_max;
+	unsigned equivalent_usage = wacom_equivalent_usage(usage->hid);
 
-	switch (usage->hid) {
+	switch (equivalent_usage) {
 	case HID_GD_X:
 		if (touch_max == 1)
 			wacom_map_usage(input, usage, field, EV_ABS, ABS_X, 4);
@@ -1644,13 +2007,14 @@ static void wacom_wac_finger_slot(struct wacom_wac *wacom_wac,
 	}
 }
 
-static int wacom_wac_finger_event(struct hid_device *hdev,
+static void wacom_wac_finger_event(struct hid_device *hdev,
 		struct hid_field *field, struct hid_usage *usage, __s32 value)
 {
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	unsigned equivalent_usage = wacom_equivalent_usage(usage->hid);
 
-	switch (usage->hid) {
+	switch (equivalent_usage) {
 	case HID_GD_X:
 		wacom_wac->hid_data.x = value;
 		break;
@@ -1673,11 +2037,9 @@ static int wacom_wac_finger_event(struct hid_device *hdev,
 
 
 	if (usage->usage_index + 1 == field->report_count) {
-		if (usage->hid == wacom_wac->hid_data.last_slot_field)
+		if (equivalent_usage == wacom_wac->hid_data.last_slot_field)
 			wacom_wac_finger_slot(wacom_wac, wacom_wac->touch_input);
 	}
-
-	return 0;
 }
 
 static void wacom_wac_finger_pre_report(struct hid_device *hdev,
@@ -1762,28 +2124,30 @@ void wacom_wac_usage_mapping(struct hid_device *hdev,
 	/* currently, only direct devices have proper hid report descriptors */
 	features->device_type |= WACOM_DEVICETYPE_DIRECT;
 
-	if (WACOM_PEN_FIELD(field))
-		return wacom_wac_pen_usage_mapping(hdev, field, usage);
-
-	if (WACOM_FINGER_FIELD(field))
-		return wacom_wac_finger_usage_mapping(hdev, field, usage);
+	if (WACOM_PAD_FIELD(field))
+		wacom_wac_pad_usage_mapping(hdev, field, usage);
+	else if (WACOM_PEN_FIELD(field))
+		wacom_wac_pen_usage_mapping(hdev, field, usage);
+	else if (WACOM_FINGER_FIELD(field))
+		wacom_wac_finger_usage_mapping(hdev, field, usage);
 }
 
-int wacom_wac_event(struct hid_device *hdev, struct hid_field *field,
+void wacom_wac_event(struct hid_device *hdev, struct hid_field *field,
 		struct hid_usage *usage, __s32 value)
 {
 	struct wacom *wacom = hid_get_drvdata(hdev);
 
 	if (wacom->wacom_wac.features.type != HID_GENERIC)
-		return 0;
-
-	if (WACOM_PEN_FIELD(field))
-		return wacom_wac_pen_event(hdev, field, usage, value);
-
-	if (WACOM_FINGER_FIELD(field))
-		return wacom_wac_finger_event(hdev, field, usage, value);
+		return;
 
-	return 0;
+	if (WACOM_PAD_FIELD(field)) {
+		wacom_wac_pad_battery_event(hdev, field, usage, value);
+		if (wacom->wacom_wac.pad_input)
+			wacom_wac_pad_event(hdev, field, usage, value);
+	} else if (WACOM_PEN_FIELD(field) && wacom->wacom_wac.pen_input)
+		wacom_wac_pen_event(hdev, field, usage, value);
+	else if (WACOM_FINGER_FIELD(field) && wacom->wacom_wac.touch_input)
+		wacom_wac_finger_event(hdev, field, usage, value);
 }
 
 static void wacom_report_events(struct hid_device *hdev, struct hid_report *report)
@@ -1814,19 +2178,23 @@ void wacom_wac_report(struct hid_device *hdev, struct hid_report *report)
 	if (wacom_wac->features.type != HID_GENERIC)
 		return;
 
-	if (WACOM_PEN_FIELD(field))
+	if (WACOM_PAD_FIELD(field) && wacom->wacom_wac.pad_input)
+		wacom_wac_pad_pre_report(hdev, report);
+	else if (WACOM_PEN_FIELD(field) && wacom->wacom_wac.pen_input)
 		wacom_wac_pen_pre_report(hdev, report);
-
-	if (WACOM_FINGER_FIELD(field))
+	else if (WACOM_FINGER_FIELD(field) && wacom->wacom_wac.touch_input)
 		wacom_wac_finger_pre_report(hdev, report);
 
 	wacom_report_events(hdev, report);
 
-	if (WACOM_PEN_FIELD(field))
-		return wacom_wac_pen_report(hdev, report);
-
-	if (WACOM_FINGER_FIELD(field))
-		return wacom_wac_finger_report(hdev, report);
+	if (WACOM_PAD_FIELD(field)) {
+		wacom_wac_pad_battery_report(hdev, report);
+		if (wacom->wacom_wac.pad_input)
+			wacom_wac_pad_report(hdev, report);
+	} else if (WACOM_PEN_FIELD(field) && wacom->wacom_wac.pen_input)
+		wacom_wac_pen_report(hdev, report);
+	else if (WACOM_FINGER_FIELD(field) && wacom->wacom_wac.touch_input)
+		wacom_wac_finger_report(hdev, report);
 }
 
 static int wacom_bpt_touch(struct wacom_wac *wacom)
@@ -2399,6 +2767,8 @@ void wacom_setup_device_quirks(struct wacom *wacom)
 	struct wacom_features *features = &wacom->wacom_wac.features;
 
 	/* The pen and pad share the same interface on most devices */
+	if (features->numbered_buttons > 0)
+		features->device_type |= WACOM_DEVICETYPE_PAD;
 	if (features->type == GRAPHIRE_BT || features->type == WACOM_G4 ||
 	    features->type == DTUS ||
 	    (features->type >= INTUOS3S && features->type <= WACOM_MO)) {
@@ -2448,7 +2818,7 @@ void wacom_setup_device_quirks(struct wacom *wacom)
 	/*
 	 * Raw Wacom-mode pen and touch events both come from interface
 	 * 0, whose HID descriptor has an application usage of 0xFF0D
-	 * (i.e., WACOM_VENDORDEFINED_PEN). We route pen packets back
+	 * (i.e., WACOM_HID_WD_DIGITIZER). We route pen packets back
 	 * out through the HID_GENERIC device created for interface 1,
 	 * so rewrite this one to be of type WACOM_DEVICETYPE_TOUCH.
 	 */
@@ -2530,10 +2900,12 @@ int wacom_setup_pen_input_capabilities(struct input_dev *input_dev,
 	__set_bit(BTN_TOUCH, input_dev->keybit);
 	__set_bit(ABS_MISC, input_dev->absbit);
 
-	input_set_abs_params(input_dev, ABS_X, features->x_min,
-			     features->x_max, features->x_fuzz, 0);
-	input_set_abs_params(input_dev, ABS_Y, features->y_min,
-			     features->y_max, features->y_fuzz, 0);
+	input_set_abs_params(input_dev, ABS_X, 0 + features->offset_left,
+			     features->x_max - features->offset_right,
+			     features->x_fuzz, 0);
+	input_set_abs_params(input_dev, ABS_Y, 0 + features->offset_top,
+			     features->y_max - features->offset_bottom,
+			     features->y_fuzz, 0);
 	input_set_abs_params(input_dev, ABS_PRESSURE, 0,
 		features->pressure_max, features->pressure_fuzz, 0);
 
@@ -2769,17 +3141,29 @@ int wacom_setup_touch_input_capabilities(struct input_dev *input_dev,
 	return 0;
 }
 
+static int wacom_numbered_button_to_key(int n)
+{
+	if (n < 10)
+		return BTN_0 + n;
+	else if (n < 16)
+		return BTN_A + (n-10);
+	else if (n < 18)
+		return BTN_BASE + (n-16);
+	else
+		return 0;
+}
+
 static void wacom_setup_numbered_buttons(struct input_dev *input_dev,
 				int button_count)
 {
 	int i;
 
-	for (i = 0; i < button_count && i < 10; i++)
-		__set_bit(BTN_0 + i, input_dev->keybit);
-	for (i = 10; i < button_count && i < 16; i++)
-		__set_bit(BTN_A + (i-10), input_dev->keybit);
-	for (i = 16; i < button_count && i < 18; i++)
-		__set_bit(BTN_BASE + (i-16), input_dev->keybit);
+	for (i = 0; i < button_count; i++) {
+		int key = wacom_numbered_button_to_key(i);
+
+		if (key)
+			__set_bit(key, input_dev->keybit);
+	}
 }
 
 static void wacom_24hd_update_leds(struct wacom *wacom, int mask, int group)
@@ -2881,12 +3265,12 @@ static void wacom_report_numbered_buttons(struct input_dev *input_dev,
 	for (i = 0; i < wacom->led.count; i++)
 		wacom_update_led(wacom,  button_count, mask, i);
 
-	for (i = 0; i < button_count && i < 10; i++)
-		input_report_key(input_dev, BTN_0 + i, mask & (1 << i));
-	for (i = 10; i < button_count && i < 16; i++)
-		input_report_key(input_dev, BTN_A + (i-10), mask & (1 << i));
-	for (i = 16; i < button_count && i < 18; i++)
-		input_report_key(input_dev, BTN_BASE + (i-16), mask & (1 << i));
+	for (i = 0; i < button_count; i++) {
+		int key = wacom_numbered_button_to_key(i);
+
+		if (key)
+			input_report_key(input_dev, key, mask & (1 << i));
+	}
 }
 
 int wacom_setup_pad_input_capabilities(struct input_dev *input_dev,
@@ -2906,8 +3290,12 @@ int wacom_setup_pad_input_capabilities(struct input_dev *input_dev,
 	__set_bit(ABS_MISC, input_dev->absbit);
 
 	/* kept for making legacy xf86-input-wacom accepting the pad */
-	input_set_abs_params(input_dev, ABS_X, 0, 1, 0, 0);
-	input_set_abs_params(input_dev, ABS_Y, 0, 1, 0, 0);
+	if (!(input_dev->absinfo && (input_dev->absinfo[ABS_X].minimum ||
+	      input_dev->absinfo[ABS_X].maximum)))
+		input_set_abs_params(input_dev, ABS_X, 0, 1, 0, 0);
+	if (!(input_dev->absinfo && (input_dev->absinfo[ABS_Y].minimum ||
+	      input_dev->absinfo[ABS_Y].maximum)))
+		input_set_abs_params(input_dev, ABS_Y, 0, 1, 0, 0);
 
 	/* kept for making udev and libwacom accepting the pad */
 	__set_bit(BTN_STYLUS, input_dev->keybit);
@@ -3027,6 +3415,9 @@ int wacom_setup_pad_input_capabilities(struct input_dev *input_dev,
 		input_set_abs_params(input_dev, ABS_WHEEL, 0, 71, 0, 0);
 		break;
 
+	case HID_GENERIC:
+		break;
+
 	default:
 		/* no pad supported */
 		return -ENODEV;
@@ -3233,26 +3624,30 @@ static const struct wacom_features wacom_features_0x317 =
 	  INTUOSPL, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 9, .touch_max = 16,
 	  .check_for_hid_type = true, .hid_type = HID_TYPE_USBNONE };
 static const struct wacom_features wacom_features_0xF4 =
-	{ "Wacom Cintiq 24HD", 104080, 65200, 2047, 63,
+	{ "Wacom Cintiq 24HD", 104480, 65600, 2047, 63,
 	  WACOM_24HD, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 16,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET };
 static const struct wacom_features wacom_features_0xF8 =
-	{ "Wacom Cintiq 24HD touch", 104080, 65200, 2047, 63, /* Pen */
+	{ "Wacom Cintiq 24HD touch", 104480, 65600, 2047, 63, /* Pen */
 	  WACOM_24HD, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 16,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  .oVid = USB_VENDOR_ID_WACOM, .oPid = 0xf6 };
 static const struct wacom_features wacom_features_0xF6 =
 	{ "Wacom Cintiq 24HD touch", .type = WACOM_24HDT, /* Touch */
 	  .oVid = USB_VENDOR_ID_WACOM, .oPid = 0xf8, .touch_max = 10,
 	  .check_for_hid_type = true, .hid_type = HID_TYPE_USBNONE };
 static const struct wacom_features wacom_features_0x32A =
-	{ "Wacom Cintiq 27QHD", 119740, 67520, 2047, 63,
+	{ "Wacom Cintiq 27QHD", 120140, 67920, 2047, 63,
 	  WACOM_27QHD, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 0,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET };
 static const struct wacom_features wacom_features_0x32B =
-	{ "Wacom Cintiq 27QHD touch", 119740, 67520, 2047, 63,
+	{ "Wacom Cintiq 27QHD touch", 120140, 67920, 2047, 63,
 	  WACOM_27QHD, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 0,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  .oVid = USB_VENDOR_ID_WACOM, .oPid = 0x32C };
 static const struct wacom_features wacom_features_0x32C =
 	{ "Wacom Cintiq 27QHD touch", .type = WACOM_27QHDT,
@@ -3267,13 +3662,15 @@ static const struct wacom_features wacom_features_0xC6 =
 	{ "Wacom Cintiq 12WX", 53020, 33440, 1023, 63,
 	  WACOM_BEE, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 10 };
 static const struct wacom_features wacom_features_0x304 =
-	{ "Wacom Cintiq 13HD", 59152, 33448, 1023, 63,
+	{ "Wacom Cintiq 13HD", 59552, 33848, 1023, 63,
 	  WACOM_13HD, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 9,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET };
 static const struct wacom_features wacom_features_0x333 =
-	{ "Wacom Cintiq 13HD touch", 59152, 33448, 2047, 63,
+	{ "Wacom Cintiq 13HD touch", 59552, 33848, 2047, 63,
 	  WACOM_13HD, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 9,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  .oVid = USB_VENDOR_ID_WACOM, .oPid = 0x335 };
 static const struct wacom_features wacom_features_0x335 =
 	{ "Wacom Cintiq 13HD touch", .type = WACOM_24HDT, /* Touch */
@@ -3290,42 +3687,50 @@ static const struct wacom_features wacom_features_0xF0 =
 	{ "Wacom DTU1631", 34623, 19553, 511, 0,
 	  DTU, WACOM_INTUOS_RES, WACOM_INTUOS_RES };
 static const struct wacom_features wacom_features_0xFB =
-	{ "Wacom DTU1031", 21896, 13760, 511, 0,
+	{ "Wacom DTU1031", 22096, 13960, 511, 0,
 	  DTUS, WACOM_INTUOS_RES, WACOM_INTUOS_RES, 4,
+	  WACOM_DTU_OFFSET, WACOM_DTU_OFFSET,
 	  WACOM_DTU_OFFSET, WACOM_DTU_OFFSET };
 static const struct wacom_features wacom_features_0x32F =
-	{ "Wacom DTU1031X", 22472, 12728, 511, 0,
+	{ "Wacom DTU1031X", 22672, 12928, 511, 0,
 	  DTUSX, WACOM_INTUOS_RES, WACOM_INTUOS_RES, 0,
+	  WACOM_DTU_OFFSET, WACOM_DTU_OFFSET,
 	  WACOM_DTU_OFFSET, WACOM_DTU_OFFSET };
 static const struct wacom_features wacom_features_0x336 =
-	{ "Wacom DTU1141", 23472, 13203, 1023, 0,
+	{ "Wacom DTU1141", 23672, 13403, 1023, 0,
 	  DTUS, WACOM_INTUOS_RES, WACOM_INTUOS_RES, 4,
+	  WACOM_DTU_OFFSET, WACOM_DTU_OFFSET,
 	  WACOM_DTU_OFFSET, WACOM_DTU_OFFSET };
 static const struct wacom_features wacom_features_0x57 =
-	{ "Wacom DTK2241", 95640, 54060, 2047, 63,
+	{ "Wacom DTK2241", 95840, 54260, 2047, 63,
 	  DTK, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 6,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET };
 static const struct wacom_features wacom_features_0x59 = /* Pen */
-	{ "Wacom DTH2242", 95640, 54060, 2047, 63,
+	{ "Wacom DTH2242", 95840, 54260, 2047, 63,
 	  DTK, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 6,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  .oVid = USB_VENDOR_ID_WACOM, .oPid = 0x5D };
 static const struct wacom_features wacom_features_0x5D = /* Touch */
 	{ "Wacom DTH2242",       .type = WACOM_24HDT,
 	  .oVid = USB_VENDOR_ID_WACOM, .oPid = 0x59, .touch_max = 10,
 	  .check_for_hid_type = true, .hid_type = HID_TYPE_USBNONE };
 static const struct wacom_features wacom_features_0xCC =
-	{ "Wacom Cintiq 21UX2", 86800, 65200, 2047, 63,
+	{ "Wacom Cintiq 21UX2", 87200, 65600, 2047, 63,
 	  WACOM_21UX2, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 18,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET };
 static const struct wacom_features wacom_features_0xFA =
-	{ "Wacom Cintiq 22HD", 95440, 53860, 2047, 63,
+	{ "Wacom Cintiq 22HD", 95840, 54260, 2047, 63,
 	  WACOM_22HD, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 18,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET };
 static const struct wacom_features wacom_features_0x5B =
-	{ "Wacom Cintiq 22HDT", 95440, 53860, 2047, 63,
+	{ "Wacom Cintiq 22HDT", 95840, 54260, 2047, 63,
 	  WACOM_22HD, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 18,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  .oVid = USB_VENDOR_ID_WACOM, .oPid = 0x5e };
 static const struct wacom_features wacom_features_0x5E =
 	{ "Wacom Cintiq 22HDT", .type = WACOM_24HDT,
@@ -3469,18 +3874,20 @@ static const struct wacom_features wacom_features_0x6004 =
 	{ "ISD-V4", 12800, 8000, 255, 0,
 	  TABLETPC, WACOM_INTUOS_RES, WACOM_INTUOS_RES };
 static const struct wacom_features wacom_features_0x307 =
-	{ "Wacom ISDv5 307", 59152, 33448, 2047, 63,
+	{ "Wacom ISDv5 307", 59552, 33848, 2047, 63,
 	  CINTIQ_HYBRID, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 9,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  .oVid = USB_VENDOR_ID_WACOM, .oPid = 0x309 };
 static const struct wacom_features wacom_features_0x309 =
 	{ "Wacom ISDv5 309", .type = WACOM_24HDT, /* Touch */
 	  .oVid = USB_VENDOR_ID_WACOM, .oPid = 0x0307, .touch_max = 10,
 	  .check_for_hid_type = true, .hid_type = HID_TYPE_USBNONE };
 static const struct wacom_features wacom_features_0x30A =
-	{ "Wacom ISDv5 30A", 59152, 33448, 2047, 63,
+	{ "Wacom ISDv5 30A", 59552, 33848, 2047, 63,
 	  CINTIQ_HYBRID, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 9,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  .oVid = USB_VENDOR_ID_WACOM, .oPid = 0x30C };
 static const struct wacom_features wacom_features_0x30C =
 	{ "Wacom ISDv5 30C", .type = WACOM_24HDT, /* Touch */
@@ -3496,6 +3903,7 @@ static const struct wacom_features wacom_features_0x325 =
 	{ "Wacom ISDv5 325", 59552, 33848, 2047, 63,
 	  CINTIQ_COMPANION_2, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 11,
 	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
+	  WACOM_CINTIQ_OFFSET, WACOM_CINTIQ_OFFSET,
 	  .oVid = USB_VENDOR_ID_WACOM, .oPid = 0x326 };
 static const struct wacom_features wacom_features_0x326 = /* Touch */
 	{ "Wacom ISDv5 326", .type = HID_GENERIC, .oVid = USB_VENDOR_ID_WACOM,
@@ -3525,8 +3933,9 @@ static const struct wacom_features wacom_features_0x33E =
 	  INTUOSHT2, WACOM_INTUOS_RES, WACOM_INTUOS_RES, .touch_max = 16,
 	  .check_for_hid_type = true, .hid_type = HID_TYPE_USBNONE };
 static const struct wacom_features wacom_features_0x343 =
-	{ "Wacom DTK1651", 34616, 19559, 1023, 0,
+	{ "Wacom DTK1651", 34816, 19759, 1023, 0,
 	  DTUS, WACOM_INTUOS_RES, WACOM_INTUOS_RES, 4,
+	  WACOM_DTU_OFFSET, WACOM_DTU_OFFSET,
 	  WACOM_DTU_OFFSET, WACOM_DTU_OFFSET };
 
 static const struct wacom_features wacom_features_HID_ANY_ID =
diff --git a/drivers/hid/wacom_wac.h b/drivers/hid/wacom_wac.h
index 324c40b0c119..fb0e50acb10d 100644
--- a/drivers/hid/wacom_wac.h
+++ b/drivers/hid/wacom_wac.h
@@ -74,6 +74,7 @@
 
 /* device quirks */
 #define WACOM_QUIRK_BBTOUCH_LOWRES	0x0001
+#define WACOM_QUIRK_SENSE		0x0002
 #define WACOM_QUIRK_BATTERY		0x0008
 
 /* device types */
@@ -84,23 +85,66 @@
 #define WACOM_DEVICETYPE_WL_MONITOR     0x0008
 #define WACOM_DEVICETYPE_DIRECT         0x0010
 
-#define WACOM_VENDORDEFINED_PEN		0xff0d0001
-#define WACOM_G9_PAGE			0xff090000
-#define WACOM_G9_DIGITIZER		(WACOM_G9_PAGE | 0x02)
-#define WACOM_G9_TOUCHSCREEN		(WACOM_G9_PAGE | 0x11)
-#define WACOM_G11_PAGE			0xff110000
-#define WACOM_G11_DIGITIZER		(WACOM_G11_PAGE | 0x02)
-#define WACOM_G11_TOUCHSCREEN		(WACOM_G11_PAGE | 0x11)
+#define WACOM_HID_UP_WACOMDIGITIZER     0xff0d0000
+#define WACOM_HID_SP_PAD                0x00040000
+#define WACOM_HID_SP_BUTTON             0x00090000
+#define WACOM_HID_SP_DIGITIZER          0x000d0000
+#define WACOM_HID_SP_DIGITIZERINFO      0x00100000
+#define WACOM_HID_WD_DIGITIZER          (WACOM_HID_UP_WACOMDIGITIZER | 0x01)
+#define WACOM_HID_WD_SENSE              (WACOM_HID_UP_WACOMDIGITIZER | 0x36)
+#define WACOM_HID_WD_DIGITIZERFNKEYS    (WACOM_HID_UP_WACOMDIGITIZER | 0x39)
+#define WACOM_HID_WD_SERIALHI           (WACOM_HID_UP_WACOMDIGITIZER | 0x5c)
+#define WACOM_HID_WD_TOOLTYPE           (WACOM_HID_UP_WACOMDIGITIZER | 0x77)
+#define WACOM_HID_WD_DISTANCE           (WACOM_HID_UP_WACOMDIGITIZER | 0x0132)
+#define WACOM_HID_WD_TOUCHSTRIP         (WACOM_HID_UP_WACOMDIGITIZER | 0x0136)
+#define WACOM_HID_WD_TOUCHSTRIP2        (WACOM_HID_UP_WACOMDIGITIZER | 0x0137)
+#define WACOM_HID_WD_TOUCHRING          (WACOM_HID_UP_WACOMDIGITIZER | 0x0138)
+#define WACOM_HID_WD_TOUCHRINGSTATUS    (WACOM_HID_UP_WACOMDIGITIZER | 0x0139)
+#define WACOM_HID_WD_ACCELEROMETER_X    (WACOM_HID_UP_WACOMDIGITIZER | 0x0401)
+#define WACOM_HID_WD_ACCELEROMETER_Y    (WACOM_HID_UP_WACOMDIGITIZER | 0x0402)
+#define WACOM_HID_WD_ACCELEROMETER_Z    (WACOM_HID_UP_WACOMDIGITIZER | 0x0403)
+#define WACOM_HID_WD_BATTERY_CHARGING   (WACOM_HID_UP_WACOMDIGITIZER | 0x0404)
+#define WACOM_HID_WD_BATTERY_LEVEL      (WACOM_HID_UP_WACOMDIGITIZER | 0x043b)
+#define WACOM_HID_WD_EXPRESSKEY00       (WACOM_HID_UP_WACOMDIGITIZER | 0x0910)
+#define WACOM_HID_WD_EXPRESSKEYCAP00    (WACOM_HID_UP_WACOMDIGITIZER | 0x0950)
+#define WACOM_HID_WD_BUTTONHOME         (WACOM_HID_UP_WACOMDIGITIZER | 0x0990)
+#define WACOM_HID_WD_BUTTONUP           (WACOM_HID_UP_WACOMDIGITIZER | 0x0991)
+#define WACOM_HID_WD_BUTTONDOWN         (WACOM_HID_UP_WACOMDIGITIZER | 0x0992)
+#define WACOM_HID_WD_BUTTONLEFT         (WACOM_HID_UP_WACOMDIGITIZER | 0x0993)
+#define WACOM_HID_WD_BUTTONRIGHT        (WACOM_HID_UP_WACOMDIGITIZER | 0x0994)
+#define WACOM_HID_WD_BUTTONCENTER       (WACOM_HID_UP_WACOMDIGITIZER | 0x0995)
+#define WACOM_HID_WD_TOUCHONOFF         (WACOM_HID_UP_WACOMDIGITIZER | 0x0996)
+#define WACOM_HID_WD_FINGERWHEEL        (WACOM_HID_UP_WACOMDIGITIZER | 0x0d03)
+#define WACOM_HID_WD_OFFSETLEFT         (WACOM_HID_UP_WACOMDIGITIZER | 0x0d30)
+#define WACOM_HID_WD_OFFSETTOP          (WACOM_HID_UP_WACOMDIGITIZER | 0x0d31)
+#define WACOM_HID_WD_OFFSETRIGHT        (WACOM_HID_UP_WACOMDIGITIZER | 0x0d32)
+#define WACOM_HID_WD_OFFSETBOTTOM       (WACOM_HID_UP_WACOMDIGITIZER | 0x0d33)
+#define WACOM_HID_WD_DATAMODE           (WACOM_HID_UP_WACOMDIGITIZER | 0x1002)
+#define WACOM_HID_WD_DIGITIZERINFO      (WACOM_HID_UP_WACOMDIGITIZER | 0x1013)
+#define WACOM_HID_UP_G9                 0xff090000
+#define WACOM_HID_G9_PEN                (WACOM_HID_UP_G9 | 0x02)
+#define WACOM_HID_G9_TOUCHSCREEN        (WACOM_HID_UP_G9 | 0x11)
+#define WACOM_HID_UP_G11                0xff110000
+#define WACOM_HID_G11_PEN               (WACOM_HID_UP_G11 | 0x02)
+#define WACOM_HID_G11_TOUCHSCREEN       (WACOM_HID_UP_G11 | 0x11)
+
+#define WACOM_PAD_FIELD(f)	(((f)->physical == HID_DG_TABLETFUNCTIONKEY) || \
+				 ((f)->physical == WACOM_HID_WD_DIGITIZERFNKEYS) || \
+				 ((f)->physical == WACOM_HID_WD_DIGITIZERINFO))
 
 #define WACOM_PEN_FIELD(f)	(((f)->logical == HID_DG_STYLUS) || \
 				 ((f)->physical == HID_DG_STYLUS) || \
 				 ((f)->physical == HID_DG_PEN) || \
 				 ((f)->application == HID_DG_PEN) || \
 				 ((f)->application == HID_DG_DIGITIZER) || \
-				 ((f)->application == WACOM_VENDORDEFINED_PEN))
+				 ((f)->application == WACOM_HID_WD_DIGITIZER) || \
+				 ((f)->application == WACOM_HID_G9_PEN) || \
+				 ((f)->application == WACOM_HID_G11_PEN))
 #define WACOM_FINGER_FIELD(f)	(((f)->logical == HID_DG_FINGER) || \
 				 ((f)->physical == HID_DG_FINGER) || \
-				 ((f)->application == HID_DG_TOUCHSCREEN))
+				 ((f)->application == HID_DG_TOUCHSCREEN) || \
+				 ((f)->application == WACOM_HID_G9_TOUCHSCREEN) || \
+				 ((f)->application == WACOM_HID_G11_TOUCHSCREEN))
 
 enum {
 	PENPARTNER = 0,
@@ -167,8 +211,10 @@ struct wacom_features {
 	int x_resolution;
 	int y_resolution;
 	int numbered_buttons;
-	int x_min;
-	int y_min;
+	int offset_left;
+	int offset_right;
+	int offset_top;
+	int offset_bottom;
 	int device_type;
 	int x_phy;
 	int y_phy;
@@ -186,6 +232,7 @@ struct wacom_features {
 	int pktlen;
 	bool check_for_hid_type;
 	int hid_type;
+	bool input_event_flag;
 };
 
 struct wacom_shared {
@@ -202,6 +249,7 @@ struct wacom_shared {
 struct hid_data {
 	__s16 inputmode;	/* InputMode HID feature, -1 if non-existent */
 	__s16 inputmode_index;	/* InputMode HID feature index in the report */
+	bool sense_state;
 	bool inrange_state;
 	bool invert_state;
 	bool tipswitch;
@@ -217,6 +265,10 @@ struct hid_data {
 	int last_slot_field;
 	int num_expected;
 	int num_received;
+	int battery_capacity;
+	int bat_charging;
+	int bat_connected;
+	int ps_connected;
 };
 
 struct wacom_remote_data {
@@ -234,7 +286,7 @@ struct wacom_wac {
 	unsigned char data[WACOM_PKGLEN_MAX];
 	int tool[2];
 	int id[2];
-	__u32 serial[2];
+	__u64 serial[2];
 	bool reporting_data;
 	struct wacom_features features;
 	struct wacom_shared *shared;
diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c
index 16f91c8490fe..5fb4c6d9209b 100644
--- a/drivers/hv/channel.c
+++ b/drivers/hv/channel.c
@@ -39,7 +39,7 @@
  * vmbus_setevent- Trigger an event notification on the specified
  * channel.
  */
-static void vmbus_setevent(struct vmbus_channel *channel)
+void vmbus_setevent(struct vmbus_channel *channel)
 {
 	struct hv_monitor_page *monitorpage;
 
@@ -65,6 +65,7 @@ static void vmbus_setevent(struct vmbus_channel *channel)
 		vmbus_set_event(channel);
 	}
 }
+EXPORT_SYMBOL_GPL(vmbus_setevent);
 
 /*
  * vmbus_open - Open the specified channel.
@@ -635,8 +636,6 @@ int vmbus_sendpacket_ctl(struct vmbus_channel *channel, void *buffer,
 	u32 packetlen_aligned = ALIGN(packetlen, sizeof(u64));
 	struct kvec bufferlist[3];
 	u64 aligned_data = 0;
-	int ret;
-	bool signal = false;
 	bool lock = channel->acquire_ring_lock;
 	int num_vecs = ((bufferlen != 0) ? 3 : 1);
 
@@ -656,33 +655,9 @@ int vmbus_sendpacket_ctl(struct vmbus_channel *channel, void *buffer,
 	bufferlist[2].iov_base = &aligned_data;
 	bufferlist[2].iov_len = (packetlen_aligned - packetlen);
 
-	ret = hv_ringbuffer_write(&channel->outbound, bufferlist, num_vecs,
-				  &signal, lock, channel->signal_policy);
-
-	/*
-	 * Signalling the host is conditional on many factors:
-	 * 1. The ring state changed from being empty to non-empty.
-	 *    This is tracked by the variable "signal".
-	 * 2. The variable kick_q tracks if more data will be placed
-	 *    on the ring. We will not signal if more data is
-	 *    to be placed.
-	 *
-	 * Based on the channel signal state, we will decide
-	 * which signaling policy will be applied.
-	 *
-	 * If we cannot write to the ring-buffer; signal the host
-	 * even if we may not have written anything. This is a rare
-	 * enough condition that it should not matter.
-	 * NOTE: in this case, the hvsock channel is an exception, because
-	 * it looks the host side's hvsock implementation has a throttling
-	 * mechanism which can hurt the performance otherwise.
-	 */
-
-	if (((ret == 0) && kick_q && signal) ||
-	    (ret && !is_hvsock_channel(channel)))
-		vmbus_setevent(channel);
+	return hv_ringbuffer_write(channel, bufferlist, num_vecs,
+				   lock, kick_q);
 
-	return ret;
 }
 EXPORT_SYMBOL(vmbus_sendpacket_ctl);
 
@@ -723,7 +698,6 @@ int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel,
 				     u32 flags,
 				     bool kick_q)
 {
-	int ret;
 	int i;
 	struct vmbus_channel_packet_page_buffer desc;
 	u32 descsize;
@@ -731,7 +705,6 @@ int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel,
 	u32 packetlen_aligned;
 	struct kvec bufferlist[3];
 	u64 aligned_data = 0;
-	bool signal = false;
 	bool lock = channel->acquire_ring_lock;
 
 	if (pagecount > MAX_PAGE_BUFFER_COUNT)
@@ -769,29 +742,8 @@ int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel,
 	bufferlist[2].iov_base = &aligned_data;
 	bufferlist[2].iov_len = (packetlen_aligned - packetlen);
 
-	ret = hv_ringbuffer_write(&channel->outbound, bufferlist, 3,
-				  &signal, lock, channel->signal_policy);
-
-	/*
-	 * Signalling the host is conditional on many factors:
-	 * 1. The ring state changed from being empty to non-empty.
-	 *    This is tracked by the variable "signal".
-	 * 2. The variable kick_q tracks if more data will be placed
-	 *    on the ring. We will not signal if more data is
-	 *    to be placed.
-	 *
-	 * Based on the channel signal state, we will decide
-	 * which signaling policy will be applied.
-	 *
-	 * If we cannot write to the ring-buffer; signal the host
-	 * even if we may not have written anything. This is a rare
-	 * enough condition that it should not matter.
-	 */
-
-	if (((ret == 0) && kick_q && signal) || (ret))
-		vmbus_setevent(channel);
-
-	return ret;
+	return hv_ringbuffer_write(channel, bufferlist, 3,
+				   lock, kick_q);
 }
 EXPORT_SYMBOL_GPL(vmbus_sendpacket_pagebuffer_ctl);
 
@@ -822,12 +774,10 @@ int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel,
 			      u32 desc_size,
 			      void *buffer, u32 bufferlen, u64 requestid)
 {
-	int ret;
 	u32 packetlen;
 	u32 packetlen_aligned;
 	struct kvec bufferlist[3];
 	u64 aligned_data = 0;
-	bool signal = false;
 	bool lock = channel->acquire_ring_lock;
 
 	packetlen = desc_size + bufferlen;
@@ -848,13 +798,8 @@ int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel,
 	bufferlist[2].iov_base = &aligned_data;
 	bufferlist[2].iov_len = (packetlen_aligned - packetlen);
 
-	ret = hv_ringbuffer_write(&channel->outbound, bufferlist, 3,
-				  &signal, lock, channel->signal_policy);
-
-	if (ret == 0 && signal)
-		vmbus_setevent(channel);
-
-	return ret;
+	return hv_ringbuffer_write(channel, bufferlist, 3,
+				   lock, true);
 }
 EXPORT_SYMBOL_GPL(vmbus_sendpacket_mpb_desc);
 
@@ -866,14 +811,12 @@ int vmbus_sendpacket_multipagebuffer(struct vmbus_channel *channel,
 				struct hv_multipage_buffer *multi_pagebuffer,
 				void *buffer, u32 bufferlen, u64 requestid)
 {
-	int ret;
 	struct vmbus_channel_packet_multipage_buffer desc;
 	u32 descsize;
 	u32 packetlen;
 	u32 packetlen_aligned;
 	struct kvec bufferlist[3];
 	u64 aligned_data = 0;
-	bool signal = false;
 	bool lock = channel->acquire_ring_lock;
 	u32 pfncount = NUM_PAGES_SPANNED(multi_pagebuffer->offset,
 					 multi_pagebuffer->len);
@@ -913,13 +856,8 @@ int vmbus_sendpacket_multipagebuffer(struct vmbus_channel *channel,
 	bufferlist[2].iov_base = &aligned_data;
 	bufferlist[2].iov_len = (packetlen_aligned - packetlen);
 
-	ret = hv_ringbuffer_write(&channel->outbound, bufferlist, 3,
-				  &signal, lock, channel->signal_policy);
-
-	if (ret == 0 && signal)
-		vmbus_setevent(channel);
-
-	return ret;
+	return hv_ringbuffer_write(channel, bufferlist, 3,
+				   lock, true);
 }
 EXPORT_SYMBOL_GPL(vmbus_sendpacket_multipagebuffer);
 
@@ -941,16 +879,9 @@ __vmbus_recvpacket(struct vmbus_channel *channel, void *buffer,
 		   u32 bufferlen, u32 *buffer_actual_len, u64 *requestid,
 		   bool raw)
 {
-	int ret;
-	bool signal = false;
+	return hv_ringbuffer_read(channel, buffer, bufferlen,
+				  buffer_actual_len, requestid, raw);
 
-	ret = hv_ringbuffer_read(&channel->inbound, buffer, bufferlen,
-				 buffer_actual_len, requestid, &signal, raw);
-
-	if (signal)
-		vmbus_setevent(channel);
-
-	return ret;
 }
 
 int vmbus_recvpacket(struct vmbus_channel *channel, void *buffer,
diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c
index 96a85cd39580..26b419203f16 100644
--- a/drivers/hv/channel_mgmt.c
+++ b/drivers/hv/channel_mgmt.c
@@ -134,7 +134,7 @@ static const struct vmbus_device vmbus_devs[] = {
 	},
 
 	/* Unknown GUID */
-	{ .dev_type = HV_UNKOWN,
+	{ .dev_type = HV_UNKNOWN,
 	  .perf_device = false,
 	},
 };
@@ -163,9 +163,9 @@ static u16 hv_get_dev_type(const struct vmbus_channel *channel)
 	u16 i;
 
 	if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid))
-		return HV_UNKOWN;
+		return HV_UNKNOWN;
 
-	for (i = HV_IDE; i < HV_UNKOWN; i++) {
+	for (i = HV_IDE; i < HV_UNKNOWN; i++) {
 		if (!uuid_le_cmp(*guid, vmbus_devs[i].guid))
 			return i;
 	}
@@ -389,6 +389,7 @@ void vmbus_free_channels(void)
 {
 	struct vmbus_channel *channel, *tmp;
 
+	mutex_lock(&vmbus_connection.channel_mutex);
 	list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list,
 		listentry) {
 		/* hv_process_channel_removal() needs this */
@@ -396,6 +397,7 @@ void vmbus_free_channels(void)
 
 		vmbus_device_unregister(channel->device_obj);
 	}
+	mutex_unlock(&vmbus_connection.channel_mutex);
 }
 
 /*
@@ -447,8 +449,6 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel)
 	}
 
 	dev_type = hv_get_dev_type(newchannel);
-	if (dev_type == HV_NIC)
-		set_channel_signal_state(newchannel, HV_SIGNAL_POLICY_EXPLICIT);
 
 	init_vp_index(newchannel, dev_type);
 
diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c
index 78e6368a4423..6ce8b874e833 100644
--- a/drivers/hv/connection.c
+++ b/drivers/hv/connection.c
@@ -39,6 +39,7 @@ struct vmbus_connection vmbus_connection = {
 	.conn_state		= DISCONNECTED,
 	.next_gpadl_handle	= ATOMIC_INIT(0xE1E10),
 };
+EXPORT_SYMBOL_GPL(vmbus_connection);
 
 /*
  * Negotiated protocol version with the host.
diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c
index 60dbd6cb4640..446802ae8f1b 100644
--- a/drivers/hv/hv.c
+++ b/drivers/hv/hv.c
@@ -575,7 +575,7 @@ void hv_synic_clockevents_cleanup(void)
 	if (!(ms_hyperv.features & HV_X64_MSR_SYNTIMER_AVAILABLE))
 		return;
 
-	for_each_online_cpu(cpu)
+	for_each_present_cpu(cpu)
 		clockevents_unbind_device(hv_context.clk_evt[cpu], cpu);
 }
 
@@ -594,8 +594,10 @@ void hv_synic_cleanup(void *arg)
 		return;
 
 	/* Turn off clockevent device */
-	if (ms_hyperv.features & HV_X64_MSR_SYNTIMER_AVAILABLE)
+	if (ms_hyperv.features & HV_X64_MSR_SYNTIMER_AVAILABLE) {
+		clockevents_unbind_device(hv_context.clk_evt[cpu], cpu);
 		hv_ce_shutdown(hv_context.clk_evt[cpu]);
+	}
 
 	rdmsrl(HV_X64_MSR_SINT0 + VMBUS_MESSAGE_SINT, shared_sint.as_uint64);
 
diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c
index fdf8da929cbe..14c3dc4bd23c 100644
--- a/drivers/hv/hv_balloon.c
+++ b/drivers/hv/hv_balloon.c
@@ -564,6 +564,11 @@ struct hv_dynmem_device {
 	 * next version to try.
 	 */
 	__u32 next_version;
+
+	/*
+	 * The negotiated version agreed by host.
+	 */
+	__u32 version;
 };
 
 static struct hv_dynmem_device dm_device;
@@ -645,6 +650,7 @@ static void hv_bring_pgs_online(struct hv_hotadd_state *has,
 {
 	int i;
 
+	pr_debug("Online %lu pages starting at pfn 0x%lx\n", size, start_pfn);
 	for (i = 0; i < size; i++)
 		hv_page_online_one(has, pfn_to_page(start_pfn + i));
 }
@@ -685,7 +691,7 @@ static void hv_mem_hot_add(unsigned long start, unsigned long size,
 				(HA_CHUNK << PAGE_SHIFT));
 
 		if (ret) {
-			pr_info("hot_add memory failed error is %d\n", ret);
+			pr_warn("hot_add memory failed error is %d\n", ret);
 			if (ret == -EEXIST) {
 				/*
 				 * This error indicates that the error
@@ -814,6 +820,9 @@ static unsigned long handle_pg_range(unsigned long pg_start,
 	unsigned long old_covered_state;
 	unsigned long res = 0, flags;
 
+	pr_debug("Hot adding %lu pages starting at pfn 0x%lx.\n", pg_count,
+		pg_start);
+
 	spin_lock_irqsave(&dm_device.ha_lock, flags);
 	list_for_each_entry(has, &dm_device.ha_region_list, list) {
 		/*
@@ -1025,8 +1034,13 @@ static void process_info(struct hv_dynmem_device *dm, struct dm_info_msg *msg)
 
 	switch (info_hdr->type) {
 	case INFO_TYPE_MAX_PAGE_CNT:
-		pr_info("Received INFO_TYPE_MAX_PAGE_CNT\n");
-		pr_info("Data Size is %d\n", info_hdr->data_size);
+		if (info_hdr->data_size == sizeof(__u64)) {
+			__u64 *max_page_count = (__u64 *)&info_hdr[1];
+
+			pr_info("INFO_TYPE_MAX_PAGE_CNT = %llu\n",
+				*max_page_count);
+		}
+
 		break;
 	default:
 		pr_info("Received Unknown type: %d\n", info_hdr->type);
@@ -1196,8 +1210,6 @@ static unsigned int alloc_balloon_pages(struct hv_dynmem_device *dm,
 	return num_pages;
 }
 
-
-
 static void balloon_up(struct work_struct *dummy)
 {
 	unsigned int num_pages = dm_device.balloon_wrk.num_pages;
@@ -1224,6 +1236,10 @@ static void balloon_up(struct work_struct *dummy)
 
 	/* Refuse to balloon below the floor, keep the 2M granularity. */
 	if (avail_pages < num_pages || avail_pages - num_pages < floor) {
+		pr_warn("Balloon request will be partially fulfilled. %s\n",
+			avail_pages < num_pages ? "Not enough memory." :
+			"Balloon floor reached.");
+
 		num_pages = avail_pages > floor ? (avail_pages - floor) : 0;
 		num_pages -= num_pages % PAGES_IN_2M;
 	}
@@ -1245,6 +1261,9 @@ static void balloon_up(struct work_struct *dummy)
 		}
 
 		if (num_ballooned == 0 || num_ballooned == num_pages) {
+			pr_debug("Ballooned %u out of %u requested pages.\n",
+				num_pages, dm_device.balloon_wrk.num_pages);
+
 			bl_resp->more_pages = 0;
 			done = true;
 			dm_device.state = DM_INITIALIZED;
@@ -1292,12 +1311,16 @@ static void balloon_down(struct hv_dynmem_device *dm,
 	int range_count = req->range_count;
 	struct dm_unballoon_response resp;
 	int i;
+	unsigned int prev_pages_ballooned = dm->num_pages_ballooned;
 
 	for (i = 0; i < range_count; i++) {
 		free_balloon_pages(dm, &range_array[i]);
 		complete(&dm_device.config_event);
 	}
 
+	pr_debug("Freed %u ballooned pages.\n",
+		prev_pages_ballooned - dm->num_pages_ballooned);
+
 	if (req->more_pages == 1)
 		return;
 
@@ -1365,6 +1388,7 @@ static void version_resp(struct hv_dynmem_device *dm,
 	version_req.hdr.size = sizeof(struct dm_version_request);
 	version_req.hdr.trans_id = atomic_inc_return(&trans_id);
 	version_req.version.version = dm->next_version;
+	dm->version = version_req.version.version;
 
 	/*
 	 * Set the next version to try in case current version fails.
@@ -1501,7 +1525,11 @@ static int balloon_probe(struct hv_device *dev,
 	struct dm_version_request version_req;
 	struct dm_capabilities cap_msg;
 
+#ifdef CONFIG_MEMORY_HOTPLUG
 	do_hot_add = hot_add;
+#else
+	do_hot_add = false;
+#endif
 
 	/*
 	 * First allocate a send buffer.
@@ -1553,6 +1581,7 @@ static int balloon_probe(struct hv_device *dev,
 	version_req.hdr.trans_id = atomic_inc_return(&trans_id);
 	version_req.version.version = DYNMEM_PROTOCOL_VERSION_WIN10;
 	version_req.is_last_attempt = 0;
+	dm_device.version = version_req.version.version;
 
 	ret = vmbus_sendpacket(dev->channel, &version_req,
 				sizeof(struct dm_version_request),
@@ -1575,6 +1604,11 @@ static int balloon_probe(struct hv_device *dev,
 		ret = -ETIMEDOUT;
 		goto probe_error2;
 	}
+
+	pr_info("Using Dynamic Memory protocol version %u.%u\n",
+		DYNMEM_MAJOR_VERSION(dm_device.version),
+		DYNMEM_MINOR_VERSION(dm_device.version));
+
 	/*
 	 * Now submit our capabilities to the host.
 	 */
diff --git a/drivers/hv/hv_snapshot.c b/drivers/hv/hv_snapshot.c
index a6707133c297..eee238cc60bd 100644
--- a/drivers/hv/hv_snapshot.c
+++ b/drivers/hv/hv_snapshot.c
@@ -31,7 +31,10 @@
 #define VSS_MINOR  0
 #define VSS_VERSION    (VSS_MAJOR << 16 | VSS_MINOR)
 
-#define VSS_USERSPACE_TIMEOUT (msecs_to_jiffies(10 * 1000))
+/*
+ * Timeout values are based on expecations from host
+ */
+#define VSS_FREEZE_TIMEOUT (15 * 60)
 
 /*
  * Global state maintained for transaction that is being processed. For a class
@@ -120,7 +123,7 @@ static int vss_handle_handshake(struct hv_vss_msg *vss_msg)
 	default:
 		return -EINVAL;
 	}
-	pr_debug("VSS: userspace daemon ver. %d connected\n", dm_reg_value);
+	pr_info("VSS: userspace daemon ver. %d connected\n", dm_reg_value);
 	return 0;
 }
 
@@ -128,8 +131,10 @@ static int vss_on_msg(void *msg, int len)
 {
 	struct hv_vss_msg *vss_msg = (struct hv_vss_msg *)msg;
 
-	if (len != sizeof(*vss_msg))
+	if (len != sizeof(*vss_msg)) {
+		pr_debug("VSS: Message size does not match length\n");
 		return -EINVAL;
+	}
 
 	if (vss_msg->vss_hdr.operation == VSS_OP_REGISTER ||
 	    vss_msg->vss_hdr.operation == VSS_OP_REGISTER1) {
@@ -137,8 +142,11 @@ static int vss_on_msg(void *msg, int len)
 		 * Don't process registration messages if we're in the middle
 		 * of a transaction processing.
 		 */
-		if (vss_transaction.state > HVUTIL_READY)
+		if (vss_transaction.state > HVUTIL_READY) {
+			pr_debug("VSS: Got unexpected registration request\n");
 			return -EINVAL;
+		}
+
 		return vss_handle_handshake(vss_msg);
 	} else if (vss_transaction.state == HVUTIL_USERSPACE_REQ) {
 		vss_transaction.state = HVUTIL_USERSPACE_RECV;
@@ -155,7 +163,7 @@ static int vss_on_msg(void *msg, int len)
 		}
 	} else {
 		/* This is a spurious call! */
-		pr_warn("VSS: Transaction not active\n");
+		pr_debug("VSS: Transaction not active\n");
 		return -EINVAL;
 	}
 	return 0;
@@ -168,8 +176,10 @@ static void vss_send_op(void)
 	struct hv_vss_msg *vss_msg;
 
 	/* The transaction state is wrong. */
-	if (vss_transaction.state != HVUTIL_HOSTMSG_RECEIVED)
+	if (vss_transaction.state != HVUTIL_HOSTMSG_RECEIVED) {
+		pr_debug("VSS: Unexpected attempt to send to daemon\n");
 		return;
+	}
 
 	vss_msg = kzalloc(sizeof(*vss_msg), GFP_KERNEL);
 	if (!vss_msg)
@@ -179,7 +189,8 @@ static void vss_send_op(void)
 
 	vss_transaction.state = HVUTIL_USERSPACE_REQ;
 
-	schedule_delayed_work(&vss_timeout_work, VSS_USERSPACE_TIMEOUT);
+	schedule_delayed_work(&vss_timeout_work, op == VSS_OP_FREEZE ?
+			VSS_FREEZE_TIMEOUT * HZ : HV_UTIL_TIMEOUT * HZ);
 
 	rc = hvutil_transport_send(hvt, vss_msg, sizeof(*vss_msg), NULL);
 	if (rc) {
@@ -210,9 +221,13 @@ static void vss_handle_request(struct work_struct *dummy)
 	case VSS_OP_HOT_BACKUP:
 		if (vss_transaction.state < HVUTIL_READY) {
 			/* Userspace is not registered yet */
+			pr_debug("VSS: Not ready for request.\n");
 			vss_respond_to_host(HV_E_FAIL);
 			return;
 		}
+
+		pr_debug("VSS: Received request for op code: %d\n",
+			vss_transaction.msg->vss_hdr.operation);
 		vss_transaction.state = HVUTIL_HOSTMSG_RECEIVED;
 		vss_send_op();
 		return;
@@ -353,8 +368,10 @@ hv_vss_init(struct hv_util_service *srv)
 
 	hvt = hvutil_transport_init(vss_devname, CN_VSS_IDX, CN_VSS_VAL,
 				    vss_on_msg, vss_on_reset);
-	if (!hvt)
+	if (!hvt) {
+		pr_warn("VSS: Failed to initialize transport\n");
 		return -EFAULT;
+	}
 
 	return 0;
 }
diff --git a/drivers/hv/hv_util.c b/drivers/hv/hv_util.c
index bcd06306f3e8..e7707747f56d 100644
--- a/drivers/hv/hv_util.c
+++ b/drivers/hv/hv_util.c
@@ -389,16 +389,19 @@ static int util_probe(struct hv_device *dev,
 		ts_srv_version = TS_VERSION_1;
 		hb_srv_version = HB_VERSION_1;
 		break;
-	case(VERSION_WIN10):
+	case VERSION_WIN7:
+	case VERSION_WIN8:
+	case VERSION_WIN8_1:
 		util_fw_version = UTIL_FW_VERSION;
 		sd_srv_version = SD_VERSION;
-		ts_srv_version = TS_VERSION;
+		ts_srv_version = TS_VERSION_3;
 		hb_srv_version = HB_VERSION;
 		break;
+	case VERSION_WIN10:
 	default:
 		util_fw_version = UTIL_FW_VERSION;
 		sd_srv_version = SD_VERSION;
-		ts_srv_version = TS_VERSION_3;
+		ts_srv_version = TS_VERSION;
 		hb_srv_version = HB_VERSION;
 	}
 
diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h
index a5b4442433c8..0675b395ce5c 100644
--- a/drivers/hv/hyperv_vmbus.h
+++ b/drivers/hv/hyperv_vmbus.h
@@ -38,7 +38,7 @@
 /*
  * Timeout for guest-host handshake for services.
  */
-#define HV_UTIL_NEGO_TIMEOUT 60
+#define HV_UTIL_NEGO_TIMEOUT 55
 
 /*
  * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
@@ -527,14 +527,14 @@ int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info,
 
 void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info);
 
-int hv_ringbuffer_write(struct hv_ring_buffer_info *ring_info,
+int hv_ringbuffer_write(struct vmbus_channel *channel,
 		    struct kvec *kv_list,
-		    u32 kv_count, bool *signal, bool lock,
-		    enum hv_signal_policy policy);
+		    u32 kv_count, bool lock,
+		    bool kick_q);
 
-int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info,
+int hv_ringbuffer_read(struct vmbus_channel *channel,
 		       void *buffer, u32 buflen, u32 *buffer_actual_len,
-		       u64 *requestid, bool *signal, bool raw);
+		       u64 *requestid, bool raw);
 
 void hv_ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info,
 			    struct hv_ring_buffer_debug_info *debug_info);
diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c
index 08043da1a61c..cd49cb17eb7f 100644
--- a/drivers/hv/ring_buffer.c
+++ b/drivers/hv/ring_buffer.c
@@ -66,21 +66,25 @@ u32 hv_end_read(struct hv_ring_buffer_info *rbi)
  *	   once the ring buffer is empty, it will clear the
  *	   interrupt_mask and re-check to see if new data has
  *	   arrived.
+ *
+ * KYS: Oct. 30, 2016:
+ * It looks like Windows hosts have logic to deal with DOS attacks that
+ * can be triggered if it receives interrupts when it is not expecting
+ * the interrupt. The host expects interrupts only when the ring
+ * transitions from empty to non-empty (or full to non full on the guest
+ * to host ring).
+ * So, base the signaling decision solely on the ring state until the
+ * host logic is fixed.
  */
 
-static bool hv_need_to_signal(u32 old_write, struct hv_ring_buffer_info *rbi,
-			      enum hv_signal_policy policy)
+static void hv_signal_on_write(u32 old_write, struct vmbus_channel *channel,
+			       bool kick_q)
 {
+	struct hv_ring_buffer_info *rbi = &channel->outbound;
+
 	virt_mb();
 	if (READ_ONCE(rbi->ring_buffer->interrupt_mask))
-		return false;
-
-	/*
-	 * When the client wants to control signaling,
-	 * we only honour the host interrupt mask.
-	 */
-	if (policy == HV_SIGNAL_POLICY_EXPLICIT)
-		return true;
+		return;
 
 	/* check interrupt_mask before read_index */
 	virt_rmb();
@@ -89,9 +93,9 @@ static bool hv_need_to_signal(u32 old_write, struct hv_ring_buffer_info *rbi,
 	 * ring transitions from being empty to non-empty.
 	 */
 	if (old_write == READ_ONCE(rbi->ring_buffer->read_index))
-		return true;
+		vmbus_setevent(channel);
 
-	return false;
+	return;
 }
 
 /* Get the next write location for the specified ring buffer. */
@@ -280,9 +284,9 @@ void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info)
 }
 
 /* Write to the ring buffer. */
-int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info,
-		    struct kvec *kv_list, u32 kv_count, bool *signal, bool lock,
-		    enum hv_signal_policy policy)
+int hv_ringbuffer_write(struct vmbus_channel *channel,
+		    struct kvec *kv_list, u32 kv_count, bool lock,
+		    bool kick_q)
 {
 	int i = 0;
 	u32 bytes_avail_towrite;
@@ -292,6 +296,7 @@ int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info,
 	u32 old_write;
 	u64 prev_indices = 0;
 	unsigned long flags = 0;
+	struct hv_ring_buffer_info *outring_info = &channel->outbound;
 
 	for (i = 0; i < kv_count; i++)
 		totalbytes_towrite += kv_list[i].iov_len;
@@ -344,13 +349,13 @@ int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info,
 	if (lock)
 		spin_unlock_irqrestore(&outring_info->ring_lock, flags);
 
-	*signal = hv_need_to_signal(old_write, outring_info, policy);
+	hv_signal_on_write(old_write, channel, kick_q);
 	return 0;
 }
 
-int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info,
+int hv_ringbuffer_read(struct vmbus_channel *channel,
 		       void *buffer, u32 buflen, u32 *buffer_actual_len,
-		       u64 *requestid, bool *signal, bool raw)
+		       u64 *requestid, bool raw)
 {
 	u32 bytes_avail_toread;
 	u32 next_read_location = 0;
@@ -359,6 +364,7 @@ int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info,
 	u32 offset;
 	u32 packetlen;
 	int ret = 0;
+	struct hv_ring_buffer_info *inring_info = &channel->inbound;
 
 	if (buflen <= 0)
 		return -EINVAL;
@@ -416,7 +422,7 @@ int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info,
 	/* Update the read index */
 	hv_set_next_read_location(inring_info, next_read_location);
 
-	*signal = hv_need_to_signal_on_read(inring_info);
+	hv_signal_on_read(channel);
 
 	return ret;
 }
diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
index 0276d2ef06ee..230c62e7f567 100644
--- a/drivers/hv/vmbus_drv.c
+++ b/drivers/hv/vmbus_drv.c
@@ -45,6 +45,11 @@
 #include <linux/random.h>
 #include "hyperv_vmbus.h"
 
+struct vmbus_dynid {
+	struct list_head node;
+	struct hv_vmbus_device_id id;
+};
+
 static struct acpi_device  *hv_acpi_dev;
 
 static struct completion probe_event;
@@ -500,7 +505,7 @@ static ssize_t device_show(struct device *dev,
 static DEVICE_ATTR_RO(device);
 
 /* Set up per device attributes in /sys/bus/vmbus/devices/<bus device> */
-static struct attribute *vmbus_attrs[] = {
+static struct attribute *vmbus_dev_attrs[] = {
 	&dev_attr_id.attr,
 	&dev_attr_state.attr,
 	&dev_attr_monitor_id.attr,
@@ -528,7 +533,7 @@ static struct attribute *vmbus_attrs[] = {
 	&dev_attr_device.attr,
 	NULL,
 };
-ATTRIBUTE_GROUPS(vmbus);
+ATTRIBUTE_GROUPS(vmbus_dev);
 
 /*
  * vmbus_uevent - add uevent for our device
@@ -565,10 +570,29 @@ static inline bool is_null_guid(const uuid_le *guid)
  * Return a matching hv_vmbus_device_id pointer.
  * If there is no match, return NULL.
  */
-static const struct hv_vmbus_device_id *hv_vmbus_get_id(
-					const struct hv_vmbus_device_id *id,
+static const struct hv_vmbus_device_id *hv_vmbus_get_id(struct hv_driver *drv,
 					const uuid_le *guid)
 {
+	const struct hv_vmbus_device_id *id = NULL;
+	struct vmbus_dynid *dynid;
+
+	/* Look at the dynamic ids first, before the static ones */
+	spin_lock(&drv->dynids.lock);
+	list_for_each_entry(dynid, &drv->dynids.list, node) {
+		if (!uuid_le_cmp(dynid->id.guid, *guid)) {
+			id = &dynid->id;
+			break;
+		}
+	}
+	spin_unlock(&drv->dynids.lock);
+
+	if (id)
+		return id;
+
+	id = drv->id_table;
+	if (id == NULL)
+		return NULL; /* empty device table */
+
 	for (; !is_null_guid(&id->guid); id++)
 		if (!uuid_le_cmp(id->guid, *guid))
 			return id;
@@ -576,6 +600,134 @@ static const struct hv_vmbus_device_id *hv_vmbus_get_id(
 	return NULL;
 }
 
+/* vmbus_add_dynid - add a new device ID to this driver and re-probe devices */
+static int vmbus_add_dynid(struct hv_driver *drv, uuid_le *guid)
+{
+	struct vmbus_dynid *dynid;
+
+	dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
+	if (!dynid)
+		return -ENOMEM;
+
+	dynid->id.guid = *guid;
+
+	spin_lock(&drv->dynids.lock);
+	list_add_tail(&dynid->node, &drv->dynids.list);
+	spin_unlock(&drv->dynids.lock);
+
+	return driver_attach(&drv->driver);
+}
+
+static void vmbus_free_dynids(struct hv_driver *drv)
+{
+	struct vmbus_dynid *dynid, *n;
+
+	spin_lock(&drv->dynids.lock);
+	list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
+		list_del(&dynid->node);
+		kfree(dynid);
+	}
+	spin_unlock(&drv->dynids.lock);
+}
+
+/* Parse string of form: 1b4e28ba-2fa1-11d2-883f-b9a761bde3f */
+static int get_uuid_le(const char *str, uuid_le *uu)
+{
+	unsigned int b[16];
+	int i;
+
+	if (strlen(str) < 37)
+		return -1;
+
+	for (i = 0; i < 36; i++) {
+		switch (i) {
+		case 8: case 13: case 18: case 23:
+			if (str[i] != '-')
+				return -1;
+			break;
+		default:
+			if (!isxdigit(str[i]))
+				return -1;
+		}
+	}
+
+	/* unparse little endian output byte order */
+	if (sscanf(str,
+		   "%2x%2x%2x%2x-%2x%2x-%2x%2x-%2x%2x-%2x%2x%2x%2x%2x%2x",
+		   &b[3], &b[2], &b[1], &b[0],
+		   &b[5], &b[4], &b[7], &b[6], &b[8], &b[9],
+		   &b[10], &b[11], &b[12], &b[13], &b[14], &b[15]) != 16)
+		return -1;
+
+	for (i = 0; i < 16; i++)
+		uu->b[i] = b[i];
+	return 0;
+}
+
+/*
+ * store_new_id - sysfs frontend to vmbus_add_dynid()
+ *
+ * Allow GUIDs to be added to an existing driver via sysfs.
+ */
+static ssize_t new_id_store(struct device_driver *driver, const char *buf,
+			    size_t count)
+{
+	struct hv_driver *drv = drv_to_hv_drv(driver);
+	uuid_le guid = NULL_UUID_LE;
+	ssize_t retval;
+
+	if (get_uuid_le(buf, &guid) != 0)
+		return -EINVAL;
+
+	if (hv_vmbus_get_id(drv, &guid))
+		return -EEXIST;
+
+	retval = vmbus_add_dynid(drv, &guid);
+	if (retval)
+		return retval;
+	return count;
+}
+static DRIVER_ATTR_WO(new_id);
+
+/*
+ * store_remove_id - remove a PCI device ID from this driver
+ *
+ * Removes a dynamic pci device ID to this driver.
+ */
+static ssize_t remove_id_store(struct device_driver *driver, const char *buf,
+			       size_t count)
+{
+	struct hv_driver *drv = drv_to_hv_drv(driver);
+	struct vmbus_dynid *dynid, *n;
+	uuid_le guid = NULL_UUID_LE;
+	size_t retval = -ENODEV;
+
+	if (get_uuid_le(buf, &guid))
+		return -EINVAL;
+
+	spin_lock(&drv->dynids.lock);
+	list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
+		struct hv_vmbus_device_id *id = &dynid->id;
+
+		if (!uuid_le_cmp(id->guid, guid)) {
+			list_del(&dynid->node);
+			kfree(dynid);
+			retval = count;
+			break;
+		}
+	}
+	spin_unlock(&drv->dynids.lock);
+
+	return retval;
+}
+static DRIVER_ATTR_WO(remove_id);
+
+static struct attribute *vmbus_drv_attrs[] = {
+	&driver_attr_new_id.attr,
+	&driver_attr_remove_id.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(vmbus_drv);
 
 
 /*
@@ -590,7 +742,7 @@ static int vmbus_match(struct device *device, struct device_driver *driver)
 	if (is_hvsock_channel(hv_dev->channel))
 		return drv->hvsock;
 
-	if (hv_vmbus_get_id(drv->id_table, &hv_dev->dev_type))
+	if (hv_vmbus_get_id(drv, &hv_dev->dev_type))
 		return 1;
 
 	return 0;
@@ -607,7 +759,7 @@ static int vmbus_probe(struct device *child_device)
 	struct hv_device *dev = device_to_hv_device(child_device);
 	const struct hv_vmbus_device_id *dev_id;
 
-	dev_id = hv_vmbus_get_id(drv->id_table, &dev->dev_type);
+	dev_id = hv_vmbus_get_id(drv, &dev->dev_type);
 	if (drv->probe) {
 		ret = drv->probe(dev, dev_id);
 		if (ret != 0)
@@ -684,7 +836,8 @@ static struct bus_type  hv_bus = {
 	.remove =		vmbus_remove,
 	.probe =		vmbus_probe,
 	.uevent =		vmbus_uevent,
-	.dev_groups =		vmbus_groups,
+	.dev_groups =		vmbus_dev_groups,
+	.drv_groups =		vmbus_drv_groups,
 };
 
 struct onmessage_work_context {
@@ -905,6 +1058,9 @@ int __vmbus_driver_register(struct hv_driver *hv_driver, struct module *owner, c
 	hv_driver->driver.mod_name = mod_name;
 	hv_driver->driver.bus = &hv_bus;
 
+	spin_lock_init(&hv_driver->dynids.lock);
+	INIT_LIST_HEAD(&hv_driver->dynids.list);
+
 	ret = driver_register(&hv_driver->driver);
 
 	return ret;
@@ -923,8 +1079,10 @@ void vmbus_driver_unregister(struct hv_driver *hv_driver)
 {
 	pr_info("unregistering driver %s\n", hv_driver->name);
 
-	if (!vmbus_exists())
+	if (!vmbus_exists()) {
 		driver_unregister(&hv_driver->driver);
+		vmbus_free_dynids(hv_driver);
+	}
 }
 EXPORT_SYMBOL_GPL(vmbus_driver_unregister);
 
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index 45cef3d2c75c..190d270b20a2 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -907,6 +907,17 @@ config SENSORS_MCP3021
 	  This driver can also be built as a module.  If so, the module
 	  will be called mcp3021.
 
+config SENSORS_TC654
+	tristate "Microchip TC654/TC655 and compatibles"
+	depends on I2C
+	help
+	  If you say yes here you get support for TC654 and TC655.
+	  The TC654 and TC655 are PWM mode fan speed controllers with
+	  FanSense technology for use with brushless DC fans.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called tc654.
+
 config SENSORS_MENF21BMC_HWMON
 	tristate "MEN 14F021P00 BMC Hardware Monitoring"
 	depends on MFD_MENF21BMC
@@ -1068,8 +1079,8 @@ config SENSORS_LM90
 	  LM86, LM89 and LM99, Analog Devices ADM1032, ADT7461, and ADT7461A,
 	  Maxim MAX6646, MAX6647, MAX6648, MAX6649, MAX6657, MAX6658, MAX6659,
 	  MAX6680, MAX6681, MAX6692, MAX6695, MAX6696, ON Semiconductor NCT1008,
-	  Winbond/Nuvoton W83L771W/G/AWG/ASG, Philips SA56004, and GMT G781
-	  sensor chips.
+	  Winbond/Nuvoton W83L771W/G/AWG/ASG, Philips SA56004, GMT G781, and
+	  Texas Instruments TMP451 sensor chips.
 
 	  This driver can also be built as a module.  If so, the module
 	  will be called lm90.
@@ -1591,6 +1602,17 @@ config SENSORS_TMP103
 	  This driver can also be built as a module.  If so, the module
 	  will be called tmp103.
 
+config SENSORS_TMP108
+	tristate "Texas Instruments TMP108"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  If you say yes here you get support for Texas Instruments TMP108
+	  sensor chips.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called tmp108.
+
 config SENSORS_TMP401
 	tristate "Texas Instruments TMP401 and compatibles"
 	depends on I2C
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index aecf4ba17460..d2cb7e804a0f 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -122,6 +122,7 @@ obj-$(CONFIG_SENSORS_MAX6697)	+= max6697.o
 obj-$(CONFIG_SENSORS_MAX31790)	+= max31790.o
 obj-$(CONFIG_SENSORS_MC13783_ADC)+= mc13783-adc.o
 obj-$(CONFIG_SENSORS_MCP3021)	+= mcp3021.o
+obj-$(CONFIG_SENSORS_TC654)	+= tc654.o
 obj-$(CONFIG_SENSORS_MENF21BMC_HWMON) += menf21bmc_hwmon.o
 obj-$(CONFIG_SENSORS_NCT6683)	+= nct6683.o
 obj-$(CONFIG_SENSORS_NCT6775)	+= nct6775.o
@@ -152,6 +153,7 @@ obj-$(CONFIG_SENSORS_TC74)	+= tc74.o
 obj-$(CONFIG_SENSORS_THMC50)	+= thmc50.o
 obj-$(CONFIG_SENSORS_TMP102)	+= tmp102.o
 obj-$(CONFIG_SENSORS_TMP103)	+= tmp103.o
+obj-$(CONFIG_SENSORS_TMP108)	+= tmp108.o
 obj-$(CONFIG_SENSORS_TMP401)	+= tmp401.o
 obj-$(CONFIG_SENSORS_TMP421)	+= tmp421.o
 obj-$(CONFIG_SENSORS_TWL4030_MADC)+= twl4030-madc-hwmon.o
diff --git a/drivers/hwmon/adm1025.c b/drivers/hwmon/adm1025.c
index d6c767ace916..1abb4609b412 100644
--- a/drivers/hwmon/adm1025.c
+++ b/drivers/hwmon/adm1025.c
@@ -93,7 +93,7 @@ static const int in_scale[6] = { 2500, 2250, 3300, 5000, 12000, 3300 };
 
 #define IN_FROM_REG(reg, scale)	(((reg) * (scale) + 96) / 192)
 #define IN_TO_REG(val, scale)	((val) <= 0 ? 0 : \
-				 (val) * 192 >= (scale) * 255 ? 255 : \
+				 (val) >= (scale) * 255 / 192 ? 255 : \
 				 ((val) * 192 + (scale) / 2) / (scale))
 
 #define TEMP_FROM_REG(reg)	((reg) * 1000)
diff --git a/drivers/hwmon/adm1026.c b/drivers/hwmon/adm1026.c
index e67b9a50ac7c..b2a5d9e5c590 100644
--- a/drivers/hwmon/adm1026.c
+++ b/drivers/hwmon/adm1026.c
@@ -197,8 +197,9 @@ static int adm1026_scaling[] = { /* .001 Volts */
 	};
 #define NEG12_OFFSET  16000
 #define SCALE(val, from, to) (((val)*(to) + ((from)/2))/(from))
-#define INS_TO_REG(n, val)  (clamp_val(SCALE(val, adm1026_scaling[n], 192),\
-	0, 255))
+#define INS_TO_REG(n, val)	\
+		SCALE(clamp_val(val, 0, 255 * adm1026_scaling[n] / 192), \
+		      adm1026_scaling[n], 192)
 #define INS_FROM_REG(n, val) (SCALE(val, 192, adm1026_scaling[n]))
 
 /*
@@ -215,11 +216,11 @@ static int adm1026_scaling[] = { /* .001 Volts */
 #define DIV_TO_REG(val) ((val) >= 8 ? 3 : (val) >= 4 ? 2 : (val) >= 2 ? 1 : 0)
 
 /* Temperature is reported in 1 degC increments */
-#define TEMP_TO_REG(val) (clamp_val(((val) + ((val) < 0 ? -500 : 500)) \
-					/ 1000, -127, 127))
+#define TEMP_TO_REG(val) DIV_ROUND_CLOSEST(clamp_val(val, -128000, 127000), \
+					   1000)
 #define TEMP_FROM_REG(val) ((val) * 1000)
-#define OFFSET_TO_REG(val) (clamp_val(((val) + ((val) < 0 ? -500 : 500)) \
-					  / 1000, -127, 127))
+#define OFFSET_TO_REG(val) DIV_ROUND_CLOSEST(clamp_val(val, -128000, 127000), \
+					     1000)
 #define OFFSET_FROM_REG(val) ((val) * 1000)
 
 #define PWM_TO_REG(val) (clamp_val(val, 0, 255))
@@ -233,7 +234,8 @@ static int adm1026_scaling[] = { /* .001 Volts */
  *   indicates that the DAC could be used to drive the fans, but in our
  *   example board (Arima HDAMA) it isn't connected to the fans at all.
  */
-#define DAC_TO_REG(val) (clamp_val(((((val) * 255) + 500) / 2500), 0, 255))
+#define DAC_TO_REG(val) DIV_ROUND_CLOSEST(clamp_val(val, 0, 2500) * 255, \
+					  2500)
 #define DAC_FROM_REG(val) (((val) * 2500) / 255)
 
 /*
@@ -593,7 +595,10 @@ static ssize_t set_in16_min(struct device *dev, struct device_attribute *attr,
 		return err;
 
 	mutex_lock(&data->update_lock);
-	data->in_min[16] = INS_TO_REG(16, val + NEG12_OFFSET);
+	data->in_min[16] = INS_TO_REG(16,
+				      clamp_val(val, INT_MIN,
+						INT_MAX - NEG12_OFFSET) +
+				      NEG12_OFFSET);
 	adm1026_write_value(client, ADM1026_REG_IN_MIN[16], data->in_min[16]);
 	mutex_unlock(&data->update_lock);
 	return count;
@@ -618,7 +623,10 @@ static ssize_t set_in16_max(struct device *dev, struct device_attribute *attr,
 		return err;
 
 	mutex_lock(&data->update_lock);
-	data->in_max[16] = INS_TO_REG(16, val+NEG12_OFFSET);
+	data->in_max[16] = INS_TO_REG(16,
+				      clamp_val(val, INT_MIN,
+						INT_MAX - NEG12_OFFSET) +
+				      NEG12_OFFSET);
 	adm1026_write_value(client, ADM1026_REG_IN_MAX[16], data->in_max[16]);
 	mutex_unlock(&data->update_lock);
 	return count;
diff --git a/drivers/hwmon/adm9240.c b/drivers/hwmon/adm9240.c
index 2fe1828bd10b..72bf2489511e 100644
--- a/drivers/hwmon/adm9240.c
+++ b/drivers/hwmon/adm9240.c
@@ -98,13 +98,15 @@ static inline unsigned int IN_FROM_REG(u8 reg, int n)
 
 static inline u8 IN_TO_REG(unsigned long val, int n)
 {
-	return clamp_val(SCALE(val, 192, nom_mv[n]), 0, 255);
+	val = clamp_val(val, 0, nom_mv[n] * 255 / 192);
+	return SCALE(val, 192, nom_mv[n]);
 }
 
 /* temperature range: -40..125, 127 disables temperature alarm */
 static inline s8 TEMP_TO_REG(long val)
 {
-	return clamp_val(SCALE(val, 1, 1000), -40, 127);
+	val = clamp_val(val, -40000, 127000);
+	return SCALE(val, 1, 1000);
 }
 
 /* two fans, each with low fan speed limit */
@@ -122,7 +124,8 @@ static inline unsigned int FAN_FROM_REG(u8 reg, u8 div)
 /* analog out 0..1250mV */
 static inline u8 AOUT_TO_REG(unsigned long val)
 {
-	return clamp_val(SCALE(val, 255, 1250), 0, 255);
+	val = clamp_val(val, 0, 1250);
+	return SCALE(val, 255, 1250);
 }
 
 static inline unsigned int AOUT_FROM_REG(u8 reg)
diff --git a/drivers/hwmon/adt7411.c b/drivers/hwmon/adt7411.c
index 812fbc00f693..bdeaece9641d 100644
--- a/drivers/hwmon/adt7411.c
+++ b/drivers/hwmon/adt7411.c
@@ -55,7 +55,7 @@ struct adt7411_data {
 	struct mutex device_lock;	/* for "atomic" device accesses */
 	struct mutex update_lock;
 	unsigned long next_update;
-	int vref_cached;
+	long vref_cached;
 	struct i2c_client *client;
 	bool use_ext_temp;
 };
@@ -114,85 +114,6 @@ static int adt7411_modify_bit(struct i2c_client *client, u8 reg, u8 bit,
 	return ret;
 }
 
-static ssize_t adt7411_show_vdd(struct device *dev,
-				struct device_attribute *attr, char *buf)
-{
-	struct adt7411_data *data = dev_get_drvdata(dev);
-	struct i2c_client *client = data->client;
-	int ret = adt7411_read_10_bit(client, ADT7411_REG_INT_TEMP_VDD_LSB,
-			ADT7411_REG_VDD_MSB, 2);
-
-	return ret < 0 ? ret : sprintf(buf, "%u\n", ret * 7000 / 1024);
-}
-
-static ssize_t adt7411_show_temp(struct device *dev,
-			struct device_attribute *attr, char *buf)
-{
-	int nr = to_sensor_dev_attr(attr)->index;
-	struct adt7411_data *data = dev_get_drvdata(dev);
-	struct i2c_client *client = data->client;
-	int val;
-	struct {
-		u8 low;
-		u8 high;
-	} reg[2] = {
-		{ ADT7411_REG_INT_TEMP_VDD_LSB, ADT7411_REG_INT_TEMP_MSB },
-		{ ADT7411_REG_EXT_TEMP_AIN14_LSB,
-		  ADT7411_REG_EXT_TEMP_AIN1_MSB },
-	};
-
-	val = adt7411_read_10_bit(client, reg[nr].low, reg[nr].high, 0);
-	if (val < 0)
-		return val;
-
-	val = val & 0x200 ? val - 0x400 : val; /* 10 bit signed */
-
-	return sprintf(buf, "%d\n", val * 250);
-}
-
-static ssize_t adt7411_show_input(struct device *dev,
-				  struct device_attribute *attr, char *buf)
-{
-	int nr = to_sensor_dev_attr(attr)->index;
-	struct adt7411_data *data = dev_get_drvdata(dev);
-	struct i2c_client *client = data->client;
-	int val;
-	u8 lsb_reg, lsb_shift;
-
-	mutex_lock(&data->update_lock);
-	if (time_after_eq(jiffies, data->next_update)) {
-		val = i2c_smbus_read_byte_data(client, ADT7411_REG_CFG3);
-		if (val < 0)
-			goto exit_unlock;
-
-		if (val & ADT7411_CFG3_REF_VDD) {
-			val = adt7411_read_10_bit(client,
-					ADT7411_REG_INT_TEMP_VDD_LSB,
-					ADT7411_REG_VDD_MSB, 2);
-			if (val < 0)
-				goto exit_unlock;
-
-			data->vref_cached = val * 7000 / 1024;
-		} else {
-			data->vref_cached = 2250;
-		}
-
-		data->next_update = jiffies + HZ;
-	}
-
-	lsb_reg = ADT7411_REG_EXT_TEMP_AIN14_LSB + (nr >> 2);
-	lsb_shift = 2 * (nr & 0x03);
-	val = adt7411_read_10_bit(client, lsb_reg,
-			ADT7411_REG_EXT_TEMP_AIN1_MSB + nr, lsb_shift);
-	if (val < 0)
-		goto exit_unlock;
-
-	val = sprintf(buf, "%u\n", val * data->vref_cached / 1024);
- exit_unlock:
-	mutex_unlock(&data->update_lock);
-	return val;
-}
-
 static ssize_t adt7411_show_bit(struct device *dev,
 				struct device_attribute *attr, char *buf)
 {
@@ -228,65 +149,157 @@ static ssize_t adt7411_set_bit(struct device *dev,
 	return ret < 0 ? ret : count;
 }
 
-
 #define ADT7411_BIT_ATTR(__name, __reg, __bit) \
 	SENSOR_DEVICE_ATTR_2(__name, S_IRUGO | S_IWUSR, adt7411_show_bit, \
 	adt7411_set_bit, __bit, __reg)
 
-static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, adt7411_show_temp, NULL, 0);
-static SENSOR_DEVICE_ATTR(temp2_input, S_IRUGO, adt7411_show_temp, NULL, 1);
-static DEVICE_ATTR(in0_input, S_IRUGO, adt7411_show_vdd, NULL);
-static SENSOR_DEVICE_ATTR(in1_input, S_IRUGO, adt7411_show_input, NULL, 0);
-static SENSOR_DEVICE_ATTR(in2_input, S_IRUGO, adt7411_show_input, NULL, 1);
-static SENSOR_DEVICE_ATTR(in3_input, S_IRUGO, adt7411_show_input, NULL, 2);
-static SENSOR_DEVICE_ATTR(in4_input, S_IRUGO, adt7411_show_input, NULL, 3);
-static SENSOR_DEVICE_ATTR(in5_input, S_IRUGO, adt7411_show_input, NULL, 4);
-static SENSOR_DEVICE_ATTR(in6_input, S_IRUGO, adt7411_show_input, NULL, 5);
-static SENSOR_DEVICE_ATTR(in7_input, S_IRUGO, adt7411_show_input, NULL, 6);
-static SENSOR_DEVICE_ATTR(in8_input, S_IRUGO, adt7411_show_input, NULL, 7);
 static ADT7411_BIT_ATTR(no_average, ADT7411_REG_CFG2, ADT7411_CFG2_DISABLE_AVG);
 static ADT7411_BIT_ATTR(fast_sampling, ADT7411_REG_CFG3, ADT7411_CFG3_ADC_CLK_225);
 static ADT7411_BIT_ATTR(adc_ref_vdd, ADT7411_REG_CFG3, ADT7411_CFG3_REF_VDD);
 
 static struct attribute *adt7411_attrs[] = {
-	&sensor_dev_attr_temp1_input.dev_attr.attr,
-	&sensor_dev_attr_temp2_input.dev_attr.attr,
-	&dev_attr_in0_input.attr,
-	&sensor_dev_attr_in1_input.dev_attr.attr,
-	&sensor_dev_attr_in2_input.dev_attr.attr,
-	&sensor_dev_attr_in3_input.dev_attr.attr,
-	&sensor_dev_attr_in4_input.dev_attr.attr,
-	&sensor_dev_attr_in5_input.dev_attr.attr,
-	&sensor_dev_attr_in6_input.dev_attr.attr,
-	&sensor_dev_attr_in7_input.dev_attr.attr,
-	&sensor_dev_attr_in8_input.dev_attr.attr,
 	&sensor_dev_attr_no_average.dev_attr.attr,
 	&sensor_dev_attr_fast_sampling.dev_attr.attr,
 	&sensor_dev_attr_adc_ref_vdd.dev_attr.attr,
 	NULL
 };
+ATTRIBUTE_GROUPS(adt7411);
 
-static umode_t adt7411_attrs_visible(struct kobject *kobj,
-				     struct attribute *attr, int index)
+static int adt7411_read_in_vdd(struct device *dev, u32 attr, long *val)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
 	struct adt7411_data *data = dev_get_drvdata(dev);
-	bool visible = true;
+	struct i2c_client *client = data->client;
+	int ret;
 
-	if (attr == &sensor_dev_attr_temp2_input.dev_attr.attr)
-		visible = data->use_ext_temp;
-	else if (attr == &sensor_dev_attr_in1_input.dev_attr.attr ||
-		 attr == &sensor_dev_attr_in2_input.dev_attr.attr)
-		visible = !data->use_ext_temp;
+	switch (attr) {
+	case hwmon_in_input:
+		ret = adt7411_read_10_bit(client, ADT7411_REG_INT_TEMP_VDD_LSB,
+					  ADT7411_REG_VDD_MSB, 2);
+		if (ret < 0)
+			return ret;
+		*val = ret * 7000 / 1024;
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int adt7411_read_in_chan(struct device *dev, u32 attr, int channel,
+				long *val)
+{
+	struct adt7411_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
 
-	return visible ? attr->mode : 0;
+	int ret;
+	int lsb_reg, lsb_shift;
+	int nr = channel - 1;
+
+	mutex_lock(&data->update_lock);
+	if (time_after_eq(jiffies, data->next_update)) {
+		ret = i2c_smbus_read_byte_data(client, ADT7411_REG_CFG3);
+		if (ret < 0)
+			goto exit_unlock;
+
+		if (ret & ADT7411_CFG3_REF_VDD) {
+			ret = adt7411_read_in_vdd(dev, hwmon_in_input,
+						  &data->vref_cached);
+			if (ret < 0)
+				goto exit_unlock;
+		} else {
+			data->vref_cached = 2250;
+		}
+
+		data->next_update = jiffies + HZ;
+	}
+
+	switch (attr) {
+	case hwmon_in_input:
+		lsb_reg = ADT7411_REG_EXT_TEMP_AIN14_LSB + (nr >> 2);
+		lsb_shift = 2 * (nr & 0x03);
+		ret = adt7411_read_10_bit(client, lsb_reg,
+					  ADT7411_REG_EXT_TEMP_AIN1_MSB + nr,
+					  lsb_shift);
+		if (ret < 0)
+			goto exit_unlock;
+		*val = ret * data->vref_cached / 1024;
+		ret = 0;
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+		break;
+	}
+ exit_unlock:
+	mutex_unlock(&data->update_lock);
+	return ret;
 }
 
-static const struct attribute_group adt7411_group = {
-	.attrs = adt7411_attrs,
-	.is_visible = adt7411_attrs_visible,
-};
-__ATTRIBUTE_GROUPS(adt7411);
+static int adt7411_read_in(struct device *dev, u32 attr, int channel,
+			   long *val)
+{
+	if (channel == 0)
+		return adt7411_read_in_vdd(dev, attr, val);
+	else
+		return adt7411_read_in_chan(dev, attr, channel, val);
+}
+
+static int adt7411_read_temp(struct device *dev, u32 attr, int channel,
+			     long *val)
+{
+	struct adt7411_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
+	int ret, regl, regh;
+
+	switch (attr) {
+	case hwmon_temp_input:
+		regl = channel ? ADT7411_REG_EXT_TEMP_AIN14_LSB :
+				 ADT7411_REG_INT_TEMP_VDD_LSB;
+		regh = channel ? ADT7411_REG_EXT_TEMP_AIN1_MSB :
+				 ADT7411_REG_INT_TEMP_MSB;
+		ret = adt7411_read_10_bit(client, regl, regh, 0);
+		if (ret < 0)
+			return ret;
+		ret = ret & 0x200 ? ret - 0x400 : ret; /* 10 bit signed */
+		*val = ret * 250;
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int adt7411_read(struct device *dev, enum hwmon_sensor_types type,
+			u32 attr, int channel, long *val)
+{
+	switch (type) {
+	case hwmon_in:
+		return adt7411_read_in(dev, attr, channel, val);
+	case hwmon_temp:
+		return adt7411_read_temp(dev, attr, channel, val);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static umode_t adt7411_is_visible(const void *_data,
+				  enum hwmon_sensor_types type,
+				  u32 attr, int channel)
+{
+	const struct adt7411_data *data = _data;
+
+	switch (type) {
+	case hwmon_in:
+		if (channel > 0 && channel < 3)
+			return data->use_ext_temp ? 0 : S_IRUGO;
+		else
+			return S_IRUGO;
+	case hwmon_temp:
+		if (channel == 1)
+			return data->use_ext_temp ? S_IRUGO : 0;
+		else
+			return S_IRUGO;
+	default:
+		return 0;
+	}
+}
 
 static int adt7411_detect(struct i2c_client *client,
 			  struct i2c_board_info *info)
@@ -358,6 +371,51 @@ static int adt7411_init_device(struct adt7411_data *data)
 	return i2c_smbus_write_byte_data(data->client, ADT7411_REG_CFG1, val);
 }
 
+static const u32 adt7411_in_config[] = {
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	HWMON_I_INPUT,
+	0
+};
+
+static const struct hwmon_channel_info adt7411_in = {
+	.type = hwmon_in,
+	.config = adt7411_in_config,
+};
+
+static const u32 adt7411_temp_config[] = {
+	HWMON_T_INPUT,
+	HWMON_T_INPUT,
+	0
+};
+
+static const struct hwmon_channel_info adt7411_temp = {
+	.type = hwmon_temp,
+	.config = adt7411_temp_config,
+};
+
+static const struct hwmon_channel_info *adt7411_info[] = {
+	&adt7411_in,
+	&adt7411_temp,
+	NULL
+};
+
+static const struct hwmon_ops adt7411_hwmon_ops = {
+	.is_visible = adt7411_is_visible,
+	.read = adt7411_read,
+};
+
+static const struct hwmon_chip_info adt7411_chip_info = {
+	.ops = &adt7411_hwmon_ops,
+	.info = adt7411_info,
+};
+
 static int adt7411_probe(struct i2c_client *client,
 				   const struct i2c_device_id *id)
 {
@@ -382,9 +440,10 @@ static int adt7411_probe(struct i2c_client *client,
 	/* force update on first occasion */
 	data->next_update = jiffies;
 
-	hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name,
-							   data,
-							   adt7411_groups);
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name,
+							 data,
+							 &adt7411_chip_info,
+							 adt7411_groups);
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
diff --git a/drivers/hwmon/adt7462.c b/drivers/hwmon/adt7462.c
index 5929e126da63..19f2a6d48bac 100644
--- a/drivers/hwmon/adt7462.c
+++ b/drivers/hwmon/adt7462.c
@@ -810,8 +810,8 @@ static ssize_t set_temp_min(struct device *dev,
 	if (kstrtol(buf, 10, &temp) || !temp_enabled(data, attr->index))
 		return -EINVAL;
 
+	temp = clamp_val(temp, -64000, 191000);
 	temp = DIV_ROUND_CLOSEST(temp, 1000) + 64;
-	temp = clamp_val(temp, 0, 255);
 
 	mutex_lock(&data->lock);
 	data->temp_min[attr->index] = temp;
@@ -848,8 +848,8 @@ static ssize_t set_temp_max(struct device *dev,
 	if (kstrtol(buf, 10, &temp) || !temp_enabled(data, attr->index))
 		return -EINVAL;
 
+	temp = clamp_val(temp, -64000, 191000);
 	temp = DIV_ROUND_CLOSEST(temp, 1000) + 64;
-	temp = clamp_val(temp, 0, 255);
 
 	mutex_lock(&data->lock);
 	data->temp_max[attr->index] = temp;
@@ -912,9 +912,9 @@ static ssize_t set_volt_max(struct device *dev,
 	if (kstrtol(buf, 10, &temp) || !x)
 		return -EINVAL;
 
+	temp = clamp_val(temp, 0, 255 * x / 1000);
 	temp *= 1000; /* convert mV to uV */
 	temp = DIV_ROUND_CLOSEST(temp, x);
-	temp = clamp_val(temp, 0, 255);
 
 	mutex_lock(&data->lock);
 	data->volt_max[attr->index] = temp;
@@ -954,9 +954,9 @@ static ssize_t set_volt_min(struct device *dev,
 	if (kstrtol(buf, 10, &temp) || !x)
 		return -EINVAL;
 
+	temp = clamp_val(temp, 0, 255 * x / 1000);
 	temp *= 1000; /* convert mV to uV */
 	temp = DIV_ROUND_CLOSEST(temp, x);
-	temp = clamp_val(temp, 0, 255);
 
 	mutex_lock(&data->lock);
 	data->volt_min[attr->index] = temp;
@@ -1220,8 +1220,8 @@ static ssize_t set_pwm_hyst(struct device *dev,
 	if (kstrtol(buf, 10, &temp))
 		return -EINVAL;
 
+	temp = clamp_val(temp, 0, 15000);
 	temp = DIV_ROUND_CLOSEST(temp, 1000);
-	temp = clamp_val(temp, 0, 15);
 
 	/* package things up */
 	temp &= ADT7462_PWM_HYST_MASK;
@@ -1306,8 +1306,8 @@ static ssize_t set_pwm_tmin(struct device *dev,
 	if (kstrtol(buf, 10, &temp))
 		return -EINVAL;
 
+	temp = clamp_val(temp, -64000, 191000);
 	temp = DIV_ROUND_CLOSEST(temp, 1000) + 64;
-	temp = clamp_val(temp, 0, 255);
 
 	mutex_lock(&data->lock);
 	data->pwm_tmin[attr->index] = temp;
diff --git a/drivers/hwmon/adt7470.c b/drivers/hwmon/adt7470.c
index 6e60ca53406e..c9a1d9c25572 100644
--- a/drivers/hwmon/adt7470.c
+++ b/drivers/hwmon/adt7470.c
@@ -483,8 +483,8 @@ static ssize_t set_temp_min(struct device *dev,
 	if (kstrtol(buf, 10, &temp))
 		return -EINVAL;
 
+	temp = clamp_val(temp, -128000, 127000);
 	temp = DIV_ROUND_CLOSEST(temp, 1000);
-	temp = clamp_val(temp, -128, 127);
 
 	mutex_lock(&data->lock);
 	data->temp_min[attr->index] = temp;
@@ -517,8 +517,8 @@ static ssize_t set_temp_max(struct device *dev,
 	if (kstrtol(buf, 10, &temp))
 		return -EINVAL;
 
+	temp = clamp_val(temp, -128000, 127000);
 	temp = DIV_ROUND_CLOSEST(temp, 1000);
-	temp = clamp_val(temp, -128, 127);
 
 	mutex_lock(&data->lock);
 	data->temp_max[attr->index] = temp;
@@ -880,8 +880,8 @@ static ssize_t set_pwm_tmin(struct device *dev,
 	if (kstrtol(buf, 10, &temp))
 		return -EINVAL;
 
+	temp = clamp_val(temp, -128000, 127000);
 	temp = DIV_ROUND_CLOSEST(temp, 1000);
-	temp = clamp_val(temp, -128, 127);
 
 	mutex_lock(&data->lock);
 	data->pwm_tmin[attr->index] = temp;
diff --git a/drivers/hwmon/amc6821.c b/drivers/hwmon/amc6821.c
index 12e851a5af48..46b4e35fd555 100644
--- a/drivers/hwmon/amc6821.c
+++ b/drivers/hwmon/amc6821.c
@@ -188,8 +188,8 @@ static struct amc6821_data *amc6821_update_device(struct device *dev)
 			!data->valid) {
 
 		for (i = 0; i < TEMP_IDX_LEN; i++)
-			data->temp[i] = i2c_smbus_read_byte_data(client,
-				temp_reg[i]);
+			data->temp[i] = (int8_t)i2c_smbus_read_byte_data(
+				client, temp_reg[i]);
 
 		data->stat1 = i2c_smbus_read_byte_data(client,
 			AMC6821_REG_STAT1);
diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c
index 6a27eb2fed17..3ac4c03ba77b 100644
--- a/drivers/hwmon/coretemp.c
+++ b/drivers/hwmon/coretemp.c
@@ -51,6 +51,7 @@ static int force_tjmax;
 module_param_named(tjmax, force_tjmax, int, 0444);
 MODULE_PARM_DESC(tjmax, "TjMax value in degrees Celsius");
 
+#define PKG_SYSFS_ATTR_NO	1	/* Sysfs attribute for package temp */
 #define BASE_SYSFS_ATTR_NO	2	/* Sysfs Base attr no for coretemp */
 #define NUM_REAL_CORES		128	/* Number of Real cores per cpu */
 #define CORETEMP_NAME_LENGTH	19	/* String Length of attrs */
@@ -58,7 +59,6 @@ MODULE_PARM_DESC(tjmax, "TjMax value in degrees Celsius");
 #define TOTAL_ATTRS		(MAX_CORE_ATTRS + 1)
 #define MAX_CORE_DATA		(NUM_REAL_CORES + BASE_SYSFS_ATTR_NO)
 
-#define TO_PHYS_ID(cpu)		(cpu_data(cpu).phys_proc_id)
 #define TO_CORE_ID(cpu)		(cpu_data(cpu).cpu_core_id)
 #define TO_ATTR_NO(cpu)		(TO_CORE_ID(cpu) + BASE_SYSFS_ATTR_NO)
 
@@ -102,20 +102,17 @@ struct temp_data {
 
 /* Platform Data per Physical CPU */
 struct platform_data {
-	struct device *hwmon_dev;
-	u16 phys_proc_id;
-	struct temp_data *core_data[MAX_CORE_DATA];
+	struct device		*hwmon_dev;
+	u16			pkg_id;
+	struct cpumask		cpumask;
+	struct temp_data	*core_data[MAX_CORE_DATA];
 	struct device_attribute name_attr;
 };
 
-struct pdev_entry {
-	struct list_head list;
-	struct platform_device *pdev;
-	u16 phys_proc_id;
-};
-
-static LIST_HEAD(pdev_list);
-static DEFINE_MUTEX(pdev_list_mutex);
+/* Keep track of how many package pointers we allocated in init() */
+static int max_packages __read_mostly;
+/* Array of package pointers. Serialized by cpu hotplug lock */
+static struct platform_device **pkg_devices;
 
 static ssize_t show_label(struct device *dev,
 				struct device_attribute *devattr, char *buf)
@@ -125,7 +122,7 @@ static ssize_t show_label(struct device *dev,
 	struct temp_data *tdata = pdata->core_data[attr->index];
 
 	if (tdata->is_pkg_data)
-		return sprintf(buf, "Physical id %u\n", pdata->phys_proc_id);
+		return sprintf(buf, "Package id %u\n", pdata->pkg_id);
 
 	return sprintf(buf, "Core %u\n", tdata->cpu_core_id);
 }
@@ -138,7 +135,9 @@ static ssize_t show_crit_alarm(struct device *dev,
 	struct platform_data *pdata = dev_get_drvdata(dev);
 	struct temp_data *tdata = pdata->core_data[attr->index];
 
+	mutex_lock(&tdata->update_lock);
 	rdmsr_on_cpu(tdata->cpu, tdata->status_reg, &eax, &edx);
+	mutex_unlock(&tdata->update_lock);
 
 	return sprintf(buf, "%d\n", (eax >> 5) & 1);
 }
@@ -435,18 +434,10 @@ static int chk_ucode_version(unsigned int cpu)
 
 static struct platform_device *coretemp_get_pdev(unsigned int cpu)
 {
-	u16 phys_proc_id = TO_PHYS_ID(cpu);
-	struct pdev_entry *p;
-
-	mutex_lock(&pdev_list_mutex);
+	int pkgid = topology_logical_package_id(cpu);
 
-	list_for_each_entry(p, &pdev_list, list)
-		if (p->phys_proc_id == phys_proc_id) {
-			mutex_unlock(&pdev_list_mutex);
-			return p->pdev;
-		}
-
-	mutex_unlock(&pdev_list_mutex);
+	if (pkgid >= 0 && pkgid < max_packages)
+		return pkg_devices[pkgid];
 	return NULL;
 }
 
@@ -483,21 +474,11 @@ static int create_core_data(struct platform_device *pdev, unsigned int cpu,
 	 * The attr number is always core id + 2
 	 * The Pkgtemp will always show up as temp1_*, if available
 	 */
-	attr_no = pkg_flag ? 1 : TO_ATTR_NO(cpu);
+	attr_no = pkg_flag ? PKG_SYSFS_ATTR_NO : TO_ATTR_NO(cpu);
 
 	if (attr_no > MAX_CORE_DATA - 1)
 		return -ERANGE;
 
-	/*
-	 * Provide a single set of attributes for all HT siblings of a core
-	 * to avoid duplicate sensors (the processor ID and core ID of all
-	 * HT siblings of a core are the same).
-	 * Skip if a HT sibling of this core is already registered.
-	 * This is not an error.
-	 */
-	if (pdata->core_data[attr_no] != NULL)
-		return 0;
-
 	tdata = init_temp_data(cpu, pkg_flag);
 	if (!tdata)
 		return -ENOMEM;
@@ -539,21 +520,14 @@ exit_free:
 	return err;
 }
 
-static void coretemp_add_core(unsigned int cpu, int pkg_flag)
+static void
+coretemp_add_core(struct platform_device *pdev, unsigned int cpu, int pkg_flag)
 {
-	struct platform_device *pdev = coretemp_get_pdev(cpu);
-	int err;
-
-	if (!pdev)
-		return;
-
-	err = create_core_data(pdev, cpu, pkg_flag);
-	if (err)
+	if (create_core_data(pdev, cpu, pkg_flag))
 		dev_err(&pdev->dev, "Adding Core %u failed\n", cpu);
 }
 
-static void coretemp_remove_core(struct platform_data *pdata,
-				 int indx)
+static void coretemp_remove_core(struct platform_data *pdata, int indx)
 {
 	struct temp_data *tdata = pdata->core_data[indx];
 
@@ -574,7 +548,7 @@ static int coretemp_probe(struct platform_device *pdev)
 	if (!pdata)
 		return -ENOMEM;
 
-	pdata->phys_proc_id = pdev->id;
+	pdata->pkg_id = pdev->id;
 	platform_set_drvdata(pdev, pdata);
 
 	pdata->hwmon_dev = devm_hwmon_device_register_with_groups(dev, DRVNAME,
@@ -602,85 +576,33 @@ static struct platform_driver coretemp_driver = {
 	.remove = coretemp_remove,
 };
 
-static int coretemp_device_add(unsigned int cpu)
+static struct platform_device *coretemp_device_add(unsigned int cpu)
 {
-	int err;
+	int err, pkgid = topology_logical_package_id(cpu);
 	struct platform_device *pdev;
-	struct pdev_entry *pdev_entry;
 
-	mutex_lock(&pdev_list_mutex);
+	if (pkgid < 0)
+		return ERR_PTR(-ENOMEM);
 
-	pdev = platform_device_alloc(DRVNAME, TO_PHYS_ID(cpu));
-	if (!pdev) {
-		err = -ENOMEM;
-		pr_err("Device allocation failed\n");
-		goto exit;
-	}
-
-	pdev_entry = kzalloc(sizeof(struct pdev_entry), GFP_KERNEL);
-	if (!pdev_entry) {
-		err = -ENOMEM;
-		goto exit_device_put;
-	}
+	pdev = platform_device_alloc(DRVNAME, pkgid);
+	if (!pdev)
+		return ERR_PTR(-ENOMEM);
 
 	err = platform_device_add(pdev);
 	if (err) {
-		pr_err("Device addition failed (%d)\n", err);
-		goto exit_device_free;
-	}
-
-	pdev_entry->pdev = pdev;
-	pdev_entry->phys_proc_id = pdev->id;
-
-	list_add_tail(&pdev_entry->list, &pdev_list);
-	mutex_unlock(&pdev_list_mutex);
-
-	return 0;
-
-exit_device_free:
-	kfree(pdev_entry);
-exit_device_put:
-	platform_device_put(pdev);
-exit:
-	mutex_unlock(&pdev_list_mutex);
-	return err;
-}
-
-static void coretemp_device_remove(unsigned int cpu)
-{
-	struct pdev_entry *p, *n;
-	u16 phys_proc_id = TO_PHYS_ID(cpu);
-
-	mutex_lock(&pdev_list_mutex);
-	list_for_each_entry_safe(p, n, &pdev_list, list) {
-		if (p->phys_proc_id != phys_proc_id)
-			continue;
-		platform_device_unregister(p->pdev);
-		list_del(&p->list);
-		kfree(p);
+		platform_device_put(pdev);
+		return ERR_PTR(err);
 	}
-	mutex_unlock(&pdev_list_mutex);
-}
-
-static bool is_any_core_online(struct platform_data *pdata)
-{
-	int i;
 
-	/* Find online cores, except pkgtemp data */
-	for (i = MAX_CORE_DATA - 1; i >= 0; --i) {
-		if (pdata->core_data[i] &&
-			!pdata->core_data[i]->is_pkg_data) {
-			return true;
-		}
-	}
-	return false;
+	pkg_devices[pkgid] = pdev;
+	return pdev;
 }
 
-static void get_core_online(unsigned int cpu)
+static int coretemp_cpu_online(unsigned int cpu)
 {
-	struct cpuinfo_x86 *c = &cpu_data(cpu);
 	struct platform_device *pdev = coretemp_get_pdev(cpu);
-	int err;
+	struct cpuinfo_x86 *c = &cpu_data(cpu);
+	struct platform_data *pdata;
 
 	/*
 	 * CPUID.06H.EAX[0] indicates whether the CPU has thermal
@@ -688,12 +610,12 @@ static void get_core_online(unsigned int cpu)
 	 * without thermal sensors will be filtered out.
 	 */
 	if (!cpu_has(c, X86_FEATURE_DTHERM))
-		return;
+		return -ENODEV;
 
 	if (!pdev) {
 		/* Check the microcode version of the CPU */
 		if (chk_ucode_version(cpu))
-			return;
+			return -EINVAL;
 
 		/*
 		 * Alright, we have DTS support.
@@ -701,101 +623,100 @@ static void get_core_online(unsigned int cpu)
 		 * online. So, initialize per-pkg data structures and
 		 * then bring this core online.
 		 */
-		err = coretemp_device_add(cpu);
-		if (err)
-			return;
+		pdev = coretemp_device_add(cpu);
+		if (IS_ERR(pdev))
+			return PTR_ERR(pdev);
+
 		/*
 		 * Check whether pkgtemp support is available.
 		 * If so, add interfaces for pkgtemp.
 		 */
 		if (cpu_has(c, X86_FEATURE_PTS))
-			coretemp_add_core(cpu, 1);
+			coretemp_add_core(pdev, cpu, 1);
 	}
+
+	pdata = platform_get_drvdata(pdev);
 	/*
-	 * Physical CPU device already exists.
-	 * So, just add interfaces for this core.
+	 * Check whether a thread sibling is already online. If not add the
+	 * interface for this CPU core.
 	 */
-	coretemp_add_core(cpu, 0);
+	if (!cpumask_intersects(&pdata->cpumask, topology_sibling_cpumask(cpu)))
+		coretemp_add_core(pdev, cpu, 0);
+
+	cpumask_set_cpu(cpu, &pdata->cpumask);
+	return 0;
 }
 
-static void put_core_offline(unsigned int cpu)
+static int coretemp_cpu_offline(unsigned int cpu)
 {
-	int i, indx;
-	struct platform_data *pdata;
 	struct platform_device *pdev = coretemp_get_pdev(cpu);
+	struct platform_data *pd;
+	struct temp_data *tdata;
+	int indx, target;
 
 	/* If the physical CPU device does not exist, just return */
 	if (!pdev)
-		return;
-
-	pdata = platform_get_drvdata(pdev);
-
-	indx = TO_ATTR_NO(cpu);
+		return 0;
 
 	/* The core id is too big, just return */
+	indx = TO_ATTR_NO(cpu);
 	if (indx > MAX_CORE_DATA - 1)
-		return;
+		return 0;
 
-	if (pdata->core_data[indx] && pdata->core_data[indx]->cpu == cpu)
-		coretemp_remove_core(pdata, indx);
+	pd = platform_get_drvdata(pdev);
+	tdata = pd->core_data[indx];
+
+	cpumask_clear_cpu(cpu, &pd->cpumask);
 
 	/*
-	 * If a HT sibling of a core is taken offline, but another HT sibling
-	 * of the same core is still online, register the alternate sibling.
-	 * This ensures that exactly one set of attributes is provided as long
-	 * as at least one HT sibling of a core is online.
+	 * If this is the last thread sibling, remove the CPU core
+	 * interface, If there is still a sibling online, transfer the
+	 * target cpu of that core interface to it.
 	 */
-	for_each_sibling(i, cpu) {
-		if (i != cpu) {
-			get_core_online(i);
-			/*
-			 * Display temperature sensor data for one HT sibling
-			 * per core only, so abort the loop after one such
-			 * sibling has been found.
-			 */
-			break;
-		}
+	target = cpumask_any_and(&pd->cpumask, topology_sibling_cpumask(cpu));
+	if (target >= nr_cpu_ids) {
+		coretemp_remove_core(pd, indx);
+	} else if (tdata && tdata->cpu == cpu) {
+		mutex_lock(&tdata->update_lock);
+		tdata->cpu = target;
+		mutex_unlock(&tdata->update_lock);
 	}
+
 	/*
-	 * If all cores in this pkg are offline, remove the device.
-	 * coretemp_device_remove calls unregister_platform_device,
-	 * which in turn calls coretemp_remove. This removes the
-	 * pkgtemp entry and does other clean ups.
+	 * If all cores in this pkg are offline, remove the device. This
+	 * will invoke the platform driver remove function, which cleans up
+	 * the rest.
 	 */
-	if (!is_any_core_online(pdata))
-		coretemp_device_remove(cpu);
-}
+	if (cpumask_empty(&pd->cpumask)) {
+		pkg_devices[topology_logical_package_id(cpu)] = NULL;
+		platform_device_unregister(pdev);
+		return 0;
+	}
 
-static int coretemp_cpu_callback(struct notifier_block *nfb,
-				 unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long) hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		get_core_online(cpu);
-		break;
-	case CPU_DOWN_PREPARE:
-		put_core_offline(cpu);
-		break;
+	/*
+	 * Check whether this core is the target for the package
+	 * interface. We need to assign it to some other cpu.
+	 */
+	tdata = pd->core_data[PKG_SYSFS_ATTR_NO];
+	if (tdata && tdata->cpu == cpu) {
+		target = cpumask_first(&pd->cpumask);
+		mutex_lock(&tdata->update_lock);
+		tdata->cpu = target;
+		mutex_unlock(&tdata->update_lock);
 	}
-	return NOTIFY_OK;
+	return 0;
 }
-
-static struct notifier_block coretemp_cpu_notifier __refdata = {
-	.notifier_call = coretemp_cpu_callback,
-};
-
 static const struct x86_cpu_id __initconst coretemp_ids[] = {
 	{ X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_DTHERM },
 	{}
 };
 MODULE_DEVICE_TABLE(x86cpu, coretemp_ids);
 
+static enum cpuhp_state coretemp_hp_online;
+
 static int __init coretemp_init(void)
 {
-	int i, err;
+	int err;
 
 	/*
 	 * CPUID.06H.EAX[0] indicates whether the CPU has thermal
@@ -805,54 +726,38 @@ static int __init coretemp_init(void)
 	if (!x86_match_cpu(coretemp_ids))
 		return -ENODEV;
 
+	max_packages = topology_max_packages();
+	pkg_devices = kzalloc(max_packages * sizeof(struct platform_device *),
+			      GFP_KERNEL);
+	if (!pkg_devices)
+		return -ENOMEM;
+
 	err = platform_driver_register(&coretemp_driver);
 	if (err)
-		goto exit;
+		return err;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i)
-		get_core_online(i);
-
-#ifndef CONFIG_HOTPLUG_CPU
-	if (list_empty(&pdev_list)) {
-		cpu_notifier_register_done();
-		err = -ENODEV;
-		goto exit_driver_unreg;
-	}
-#endif
-
-	__register_hotcpu_notifier(&coretemp_cpu_notifier);
-	cpu_notifier_register_done();
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hwmon/coretemp:online",
+				coretemp_cpu_online, coretemp_cpu_offline);
+	if (err < 0)
+		goto outdrv;
+	coretemp_hp_online = err;
 	return 0;
 
-#ifndef CONFIG_HOTPLUG_CPU
-exit_driver_unreg:
+outdrv:
 	platform_driver_unregister(&coretemp_driver);
-#endif
-exit:
+	kfree(pkg_devices);
 	return err;
 }
+module_init(coretemp_init)
 
 static void __exit coretemp_exit(void)
 {
-	struct pdev_entry *p, *n;
-
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&coretemp_cpu_notifier);
-	mutex_lock(&pdev_list_mutex);
-	list_for_each_entry_safe(p, n, &pdev_list, list) {
-		platform_device_unregister(p->pdev);
-		list_del(&p->list);
-		kfree(p);
-	}
-	mutex_unlock(&pdev_list_mutex);
-	cpu_notifier_register_done();
+	cpuhp_remove_state(coretemp_hp_online);
 	platform_driver_unregister(&coretemp_driver);
+	kfree(pkg_devices);
 }
+module_exit(coretemp_exit)
 
 MODULE_AUTHOR("Rudolf Marek <r.marek@assembler.cz>");
 MODULE_DESCRIPTION("Intel Core temperature monitor");
 MODULE_LICENSE("GPL");
-
-module_init(coretemp_init)
-module_exit(coretemp_exit)
diff --git a/drivers/hwmon/ds620.c b/drivers/hwmon/ds620.c
index edf550fc4eef..0043a4c02b85 100644
--- a/drivers/hwmon/ds620.c
+++ b/drivers/hwmon/ds620.c
@@ -166,7 +166,7 @@ static ssize_t set_temp(struct device *dev, struct device_attribute *da,
 	if (res)
 		return res;
 
-	val = (val * 10 / 625) * 8;
+	val = (clamp_val(val, -128000, 128000) * 10 / 625) * 8;
 
 	mutex_lock(&data->update_lock);
 	data->temp[attr->index] = val;
diff --git a/drivers/hwmon/emc2103.c b/drivers/hwmon/emc2103.c
index 24e395c5907d..4b870ee9b0d3 100644
--- a/drivers/hwmon/emc2103.c
+++ b/drivers/hwmon/emc2103.c
@@ -251,7 +251,7 @@ static ssize_t set_temp_min(struct device *dev, struct device_attribute *da,
 	if (result < 0)
 		return result;
 
-	val = clamp_val(DIV_ROUND_CLOSEST(val, 1000), -63, 127);
+	val = DIV_ROUND_CLOSEST(clamp_val(val, -63000, 127000), 1000);
 
 	mutex_lock(&data->update_lock);
 	data->temp_min[nr] = val;
@@ -273,7 +273,7 @@ static ssize_t set_temp_max(struct device *dev, struct device_attribute *da,
 	if (result < 0)
 		return result;
 
-	val = clamp_val(DIV_ROUND_CLOSEST(val, 1000), -63, 127);
+	val = DIV_ROUND_CLOSEST(clamp_val(val, -63000, 127000), 1000);
 
 	mutex_lock(&data->update_lock);
 	data->temp_max[nr] = val;
diff --git a/drivers/hwmon/emc6w201.c b/drivers/hwmon/emc6w201.c
index f37fe2011640..4aee5adf9ef2 100644
--- a/drivers/hwmon/emc6w201.c
+++ b/drivers/hwmon/emc6w201.c
@@ -215,12 +215,13 @@ static ssize_t set_in(struct device *dev, struct device_attribute *devattr,
 	if (err < 0)
 		return err;
 
-	val = DIV_ROUND_CLOSEST(val * 0xC0, nominal_mv[nr]);
+	val = clamp_val(val, 0, 255 * nominal_mv[nr] / 192);
+	val = DIV_ROUND_CLOSEST(val * 192, nominal_mv[nr]);
 	reg = (sf == min) ? EMC6W201_REG_IN_LOW(nr)
 			  : EMC6W201_REG_IN_HIGH(nr);
 
 	mutex_lock(&data->update_lock);
-	data->in[sf][nr] = clamp_val(val, 0, 255);
+	data->in[sf][nr] = val;
 	err = emc6w201_write8(client, reg, data->in[sf][nr]);
 	mutex_unlock(&data->update_lock);
 
@@ -252,12 +253,13 @@ static ssize_t set_temp(struct device *dev, struct device_attribute *devattr,
 	if (err < 0)
 		return err;
 
+	val = clamp_val(val, -127000, 127000);
 	val = DIV_ROUND_CLOSEST(val, 1000);
 	reg = (sf == min) ? EMC6W201_REG_TEMP_LOW(nr)
 			  : EMC6W201_REG_TEMP_HIGH(nr);
 
 	mutex_lock(&data->update_lock);
-	data->temp[sf][nr] = clamp_val(val, -127, 127);
+	data->temp[sf][nr] = val;
 	err = emc6w201_write8(client, reg, data->temp[sf][nr]);
 	mutex_unlock(&data->update_lock);
 
diff --git a/drivers/hwmon/g762.c b/drivers/hwmon/g762.c
index b96a2a9e4df7..628be9c95ff9 100644
--- a/drivers/hwmon/g762.c
+++ b/drivers/hwmon/g762.c
@@ -193,14 +193,17 @@ static inline unsigned int rpm_from_cnt(u8 cnt, u32 clk_freq, u16 p,
  * Convert fan RPM value from sysfs into count value for fan controller
  * register (FAN_SET_CNT).
  */
-static inline unsigned char cnt_from_rpm(u32 rpm, u32 clk_freq, u16 p,
+static inline unsigned char cnt_from_rpm(unsigned long rpm, u32 clk_freq, u16 p,
 					 u8 clk_div, u8 gear_mult)
 {
-	if (!rpm)         /* to stop the fan, set cnt to 255 */
+	unsigned long f1 = clk_freq * 30 * gear_mult;
+	unsigned long f2 = p * clk_div;
+
+	if (!rpm)	/* to stop the fan, set cnt to 255 */
 		return 0xff;
 
-	return clamp_val(((clk_freq * 30 * gear_mult) / (rpm * p * clk_div)),
-			 0, 255);
+	rpm = clamp_val(rpm, f1 / (255 * f2), ULONG_MAX / f2);
+	return DIV_ROUND_CLOSEST(f1, rpm * f2);
 }
 
 /* helper to grab and cache data, at most one time per second */
diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c
index a74c075a30ec..3932f9276c07 100644
--- a/drivers/hwmon/hwmon.c
+++ b/drivers/hwmon/hwmon.c
@@ -38,12 +38,15 @@ struct hwmon_device {
 
 #define to_hwmon_device(d) container_of(d, struct hwmon_device, dev)
 
+#define MAX_SYSFS_ATTR_NAME_LENGTH	32
+
 struct hwmon_device_attribute {
 	struct device_attribute dev_attr;
 	const struct hwmon_ops *ops;
 	enum hwmon_sensor_types type;
 	u32 attr;
 	int index;
+	char name[MAX_SYSFS_ATTR_NAME_LENGTH];
 };
 
 #define to_hwmon_attr(d) \
@@ -178,6 +181,22 @@ static ssize_t hwmon_attr_show(struct device *dev,
 	return sprintf(buf, "%ld\n", val);
 }
 
+static ssize_t hwmon_attr_show_string(struct device *dev,
+				      struct device_attribute *devattr,
+				      char *buf)
+{
+	struct hwmon_device_attribute *hattr = to_hwmon_attr(devattr);
+	char *s;
+	int ret;
+
+	ret = hattr->ops->read_string(dev, hattr->type, hattr->attr,
+				      hattr->index, &s);
+	if (ret < 0)
+		return ret;
+
+	return sprintf(buf, "%s\n", s);
+}
+
 static ssize_t hwmon_attr_store(struct device *dev,
 				struct device_attribute *devattr,
 				const char *buf, size_t count)
@@ -205,6 +224,17 @@ static int hwmon_attr_base(enum hwmon_sensor_types type)
 	return 1;
 }
 
+static bool is_string_attr(enum hwmon_sensor_types type, u32 attr)
+{
+	return (type == hwmon_temp && attr == hwmon_temp_label) ||
+	       (type == hwmon_in && attr == hwmon_in_label) ||
+	       (type == hwmon_curr && attr == hwmon_curr_label) ||
+	       (type == hwmon_power && attr == hwmon_power_label) ||
+	       (type == hwmon_energy && attr == hwmon_energy_label) ||
+	       (type == hwmon_humidity && attr == hwmon_humidity_label) ||
+	       (type == hwmon_fan && attr == hwmon_fan_label);
+}
+
 static struct attribute *hwmon_genattr(struct device *dev,
 				       const void *drvdata,
 				       enum hwmon_sensor_types type,
@@ -218,6 +248,7 @@ static struct attribute *hwmon_genattr(struct device *dev,
 	struct attribute *a;
 	umode_t mode;
 	char *name;
+	bool is_string = is_string_attr(type, attr);
 
 	/* The attribute is invisible if there is no template string */
 	if (!template)
@@ -227,32 +258,31 @@ static struct attribute *hwmon_genattr(struct device *dev,
 	if (!mode)
 		return ERR_PTR(-ENOENT);
 
-	if ((mode & S_IRUGO) && !ops->read)
+	if ((mode & S_IRUGO) && ((is_string && !ops->read_string) ||
+				 (!is_string && !ops->read)))
 		return ERR_PTR(-EINVAL);
 	if ((mode & S_IWUGO) && !ops->write)
 		return ERR_PTR(-EINVAL);
 
+	hattr = devm_kzalloc(dev, sizeof(*hattr), GFP_KERNEL);
+	if (!hattr)
+		return ERR_PTR(-ENOMEM);
+
 	if (type == hwmon_chip) {
 		name = (char *)template;
 	} else {
-		name = devm_kzalloc(dev, strlen(template) + 16, GFP_KERNEL);
-		if (!name)
-			return ERR_PTR(-ENOMEM);
-		scnprintf(name, strlen(template) + 16, template,
+		scnprintf(hattr->name, sizeof(hattr->name), template,
 			  index + hwmon_attr_base(type));
+		name = hattr->name;
 	}
 
-	hattr = devm_kzalloc(dev, sizeof(*hattr), GFP_KERNEL);
-	if (!hattr)
-		return ERR_PTR(-ENOMEM);
-
 	hattr->type = type;
 	hattr->attr = attr;
 	hattr->index = index;
 	hattr->ops = ops;
 
 	dattr = &hattr->dev_attr;
-	dattr->show = hwmon_attr_show;
+	dattr->show = is_string ? hwmon_attr_show_string : hwmon_attr_show;
 	dattr->store = hwmon_attr_store;
 
 	a = &dattr->attr;
@@ -263,7 +293,11 @@ static struct attribute *hwmon_genattr(struct device *dev,
 	return a;
 }
 
-static const char * const hwmon_chip_attr_templates[] = {
+/*
+ * Chip attributes are not attribute templates but actual sysfs attributes.
+ * See hwmon_genattr() for special handling.
+ */
+static const char * const hwmon_chip_attrs[] = {
 	[hwmon_chip_temp_reset_history] = "temp_reset_history",
 	[hwmon_chip_in_reset_history] = "in_reset_history",
 	[hwmon_chip_curr_reset_history] = "curr_reset_history",
@@ -400,7 +434,7 @@ static const char * const hwmon_pwm_attr_templates[] = {
 };
 
 static const char * const *__templates[] = {
-	[hwmon_chip] = hwmon_chip_attr_templates,
+	[hwmon_chip] = hwmon_chip_attrs,
 	[hwmon_temp] = hwmon_temp_attr_templates,
 	[hwmon_in] = hwmon_in_attr_templates,
 	[hwmon_curr] = hwmon_curr_attr_templates,
@@ -412,7 +446,7 @@ static const char * const *__templates[] = {
 };
 
 static const int __templates_size[] = {
-	[hwmon_chip] = ARRAY_SIZE(hwmon_chip_attr_templates),
+	[hwmon_chip] = ARRAY_SIZE(hwmon_chip_attrs),
 	[hwmon_temp] = ARRAY_SIZE(hwmon_temp_attr_templates),
 	[hwmon_in] = ARRAY_SIZE(hwmon_in_attr_templates),
 	[hwmon_curr] = ARRAY_SIZE(hwmon_curr_attr_templates),
@@ -526,9 +560,9 @@ __hwmon_device_register(struct device *dev, const char *name, void *drvdata,
 
 	hdev = &hwdev->dev;
 
-	if (chip && chip->ops->is_visible) {
+	if (chip) {
 		struct attribute **attrs;
-		int ngroups = 2;
+		int ngroups = 2; /* terminating NULL plus &hwdev->groups */
 
 		if (groups)
 			for (i = 0; groups[i]; i++)
@@ -572,7 +606,7 @@ __hwmon_device_register(struct device *dev, const char *name, void *drvdata,
 	if (err)
 		goto free_hwmon;
 
-	if (chip && chip->ops->is_visible && chip->ops->read &&
+	if (chip && chip->ops->read &&
 	    chip->info[0]->type == hwmon_chip &&
 	    (chip->info[0]->config[0] & HWMON_C_REGISTER_TZ)) {
 		const struct hwmon_channel_info **info = chip->info;
@@ -626,8 +660,8 @@ EXPORT_SYMBOL_GPL(hwmon_device_register_with_groups);
  * @dev: the parent device
  * @name: hwmon name attribute
  * @drvdata: driver data to attach to created device
- * @info: Pointer to hwmon chip information
- * @groups - pointer to list of driver specific attribute groups
+ * @info: pointer to hwmon chip information
+ * @extra_groups: pointer to list of additional non-standard attribute groups
  *
  * hwmon_device_unregister() must be called when the device is no
  * longer needed.
@@ -638,12 +672,12 @@ struct device *
 hwmon_device_register_with_info(struct device *dev, const char *name,
 				void *drvdata,
 				const struct hwmon_chip_info *chip,
-				const struct attribute_group **groups)
+				const struct attribute_group **extra_groups)
 {
-	if (chip && (!chip->ops || !chip->info))
+	if (chip && (!chip->ops || !chip->ops->is_visible || !chip->info))
 		return ERR_PTR(-EINVAL);
 
-	return __hwmon_device_register(dev, name, drvdata, chip, groups);
+	return __hwmon_device_register(dev, name, drvdata, chip, extra_groups);
 }
 EXPORT_SYMBOL_GPL(hwmon_device_register_with_info);
 
@@ -658,6 +692,9 @@ EXPORT_SYMBOL_GPL(hwmon_device_register_with_info);
  */
 struct device *hwmon_device_register(struct device *dev)
 {
+	dev_warn(dev,
+		 "hwmon_device_register() is deprecated. Please convert the driver to use hwmon_device_register_with_info().\n");
+
 	return hwmon_device_register_with_groups(dev, NULL, NULL, NULL);
 }
 EXPORT_SYMBOL_GPL(hwmon_device_register);
diff --git a/drivers/hwmon/lm85.c b/drivers/hwmon/lm85.c
index 6ff773fcaefb..29c8136ce9c5 100644
--- a/drivers/hwmon/lm85.c
+++ b/drivers/hwmon/lm85.c
@@ -136,7 +136,8 @@ static const int lm85_scaling[] = {  /* .001 Volts */
 #define SCALE(val, from, to)	(((val) * (to) + ((from) / 2)) / (from))
 
 #define INS_TO_REG(n, val)	\
-		clamp_val(SCALE(val, lm85_scaling[n], 192), 0, 255)
+		SCALE(clamp_val(val, 0, 255 * lm85_scaling[n] / 192), \
+		      lm85_scaling[n], 192)
 
 #define INSEXT_FROM_REG(n, val, ext)	\
 		SCALE(((val) << 4) + (ext), 192 << 4, lm85_scaling[n])
diff --git a/drivers/hwmon/lm87.c b/drivers/hwmon/lm87.c
index a5e295826aea..13cca3606e06 100644
--- a/drivers/hwmon/lm87.c
+++ b/drivers/hwmon/lm87.c
@@ -121,7 +121,7 @@ static u8 LM87_REG_TEMP_LOW[3] = { 0x3A, 0x38, 0x2C };
 
 #define IN_FROM_REG(reg, scale)	(((reg) * (scale) + 96) / 192)
 #define IN_TO_REG(val, scale)	((val) <= 0 ? 0 : \
-				 (val) * 192 >= (scale) * 255 ? 255 : \
+				 (val) >= (scale) * 255 / 192 ? 255 : \
 				 ((val) * 192 + (scale) / 2) / (scale))
 
 #define TEMP_FROM_REG(reg)	((reg) * 1000)
@@ -154,7 +154,6 @@ static u8 LM87_REG_TEMP_LOW[3] = { 0x3A, 0x38, 0x2C };
  */
 
 struct lm87_data {
-	struct device *hwmon_dev;
 	struct mutex update_lock;
 	char valid; /* zero until following fields are valid */
 	unsigned long last_updated; /* In jiffies */
@@ -181,6 +180,8 @@ struct lm87_data {
 	u16 alarms;		/* register values, combined */
 	u8 vid;			/* register values, combined */
 	u8 vrm;
+
+	const struct attribute_group *attr_groups[6];
 };
 
 static inline int lm87_read_value(struct i2c_client *client, u8 reg)
@@ -195,7 +196,7 @@ static inline int lm87_write_value(struct i2c_client *client, u8 reg, u8 value)
 
 static struct lm87_data *lm87_update_device(struct device *dev)
 {
-	struct i2c_client *client = to_i2c_client(dev);
+	struct i2c_client *client = dev_get_drvdata(dev);
 	struct lm87_data *data = i2c_get_clientdata(client);
 
 	mutex_lock(&data->update_lock);
@@ -309,7 +310,7 @@ static ssize_t show_in_max(struct device *dev,
 static ssize_t set_in_min(struct device *dev, struct device_attribute *attr,
 			  const char *buf, size_t count)
 {
-	struct i2c_client *client = to_i2c_client(dev);
+	struct i2c_client *client = dev_get_drvdata(dev);
 	struct lm87_data *data = i2c_get_clientdata(client);
 	int nr = to_sensor_dev_attr(attr)->index;
 	long val;
@@ -330,7 +331,7 @@ static ssize_t set_in_min(struct device *dev, struct device_attribute *attr,
 static ssize_t set_in_max(struct device *dev,  struct device_attribute *attr,
 			  const char *buf, size_t count)
 {
-	struct i2c_client *client = to_i2c_client(dev);
+	struct i2c_client *client = dev_get_drvdata(dev);
 	struct lm87_data *data = i2c_get_clientdata(client);
 	int nr = to_sensor_dev_attr(attr)->index;
 	long val;
@@ -396,7 +397,7 @@ static ssize_t show_temp_high(struct device *dev,
 static ssize_t set_temp_low(struct device *dev, struct device_attribute *attr,
 			    const char *buf, size_t count)
 {
-	struct i2c_client *client = to_i2c_client(dev);
+	struct i2c_client *client = dev_get_drvdata(dev);
 	struct lm87_data *data = i2c_get_clientdata(client);
 	int nr = to_sensor_dev_attr(attr)->index;
 	long val;
@@ -416,7 +417,7 @@ static ssize_t set_temp_low(struct device *dev, struct device_attribute *attr,
 static ssize_t set_temp_high(struct device *dev, struct device_attribute *attr,
 			     const char *buf, size_t count)
 {
-	struct i2c_client *client = to_i2c_client(dev);
+	struct i2c_client *client = dev_get_drvdata(dev);
 	struct lm87_data *data = i2c_get_clientdata(client);
 	int nr = to_sensor_dev_attr(attr)->index;
 	long val;
@@ -495,7 +496,7 @@ static ssize_t show_fan_div(struct device *dev,
 static ssize_t set_fan_min(struct device *dev, struct device_attribute *attr,
 			   const char *buf, size_t count)
 {
-	struct i2c_client *client = to_i2c_client(dev);
+	struct i2c_client *client = dev_get_drvdata(dev);
 	struct lm87_data *data = i2c_get_clientdata(client);
 	int nr = to_sensor_dev_attr(attr)->index;
 	long val;
@@ -522,7 +523,7 @@ static ssize_t set_fan_min(struct device *dev, struct device_attribute *attr,
 static ssize_t set_fan_div(struct device *dev, struct device_attribute *attr,
 			   const char *buf, size_t count)
 {
-	struct i2c_client *client = to_i2c_client(dev);
+	struct i2c_client *client = dev_get_drvdata(dev);
 	struct lm87_data *data = i2c_get_clientdata(client);
 	int nr = to_sensor_dev_attr(attr)->index;
 	long val;
@@ -635,7 +636,7 @@ static ssize_t show_aout(struct device *dev, struct device_attribute *attr,
 static ssize_t set_aout(struct device *dev, struct device_attribute *attr,
 			const char *buf, size_t count)
 {
-	struct i2c_client *client = to_i2c_client(dev);
+	struct i2c_client *client = dev_get_drvdata(dev);
 	struct lm87_data *data = i2c_get_clientdata(client);
 	long val;
 	int err;
@@ -841,23 +842,18 @@ static int lm87_detect(struct i2c_client *client, struct i2c_board_info *info)
 	return 0;
 }
 
-static void lm87_remove_files(struct i2c_client *client)
+static void lm87_restore_config(void *arg)
 {
-	struct device *dev = &client->dev;
-
-	sysfs_remove_group(&dev->kobj, &lm87_group);
-	sysfs_remove_group(&dev->kobj, &lm87_group_in6);
-	sysfs_remove_group(&dev->kobj, &lm87_group_fan1);
-	sysfs_remove_group(&dev->kobj, &lm87_group_in7);
-	sysfs_remove_group(&dev->kobj, &lm87_group_fan2);
-	sysfs_remove_group(&dev->kobj, &lm87_group_temp3);
-	sysfs_remove_group(&dev->kobj, &lm87_group_in0_5);
-	sysfs_remove_group(&dev->kobj, &lm87_group_vid);
+	struct i2c_client *client = arg;
+	struct lm87_data *data = i2c_get_clientdata(client);
+
+	lm87_write_value(client, LM87_REG_CONFIG, data->config);
 }
 
-static void lm87_init_client(struct i2c_client *client)
+static int lm87_init_client(struct i2c_client *client)
 {
 	struct lm87_data *data = i2c_get_clientdata(client);
+	int rc;
 
 	if (dev_get_platdata(&client->dev)) {
 		data->channel = *(u8 *)dev_get_platdata(&client->dev);
@@ -868,6 +864,10 @@ static void lm87_init_client(struct i2c_client *client)
 	}
 	data->config = lm87_read_value(client, LM87_REG_CONFIG) & 0x6F;
 
+	rc = devm_add_action(&client->dev, lm87_restore_config, client);
+	if (rc)
+		return rc;
+
 	if (!(data->config & 0x01)) {
 		int i;
 
@@ -895,12 +895,15 @@ static void lm87_init_client(struct i2c_client *client)
 	if ((data->config & 0x09) != 0x01)
 		lm87_write_value(client, LM87_REG_CONFIG,
 				 (data->config & 0x77) | 0x01);
+	return 0;
 }
 
 static int lm87_probe(struct i2c_client *client, const struct i2c_device_id *id)
 {
 	struct lm87_data *data;
+	struct device *hwmon_dev;
 	int err;
+	unsigned int group_tail = 0;
 
 	data = devm_kzalloc(&client->dev, sizeof(struct lm87_data), GFP_KERNEL);
 	if (!data)
@@ -910,7 +913,9 @@ static int lm87_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	mutex_init(&data->update_lock);
 
 	/* Initialize the LM87 chip */
-	lm87_init_client(client);
+	err = lm87_init_client(client);
+	if (err)
+		return err;
 
 	data->in_scale[0] = 2500;
 	data->in_scale[1] = 2700;
@@ -921,72 +926,34 @@ static int lm87_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	data->in_scale[6] = 1875;
 	data->in_scale[7] = 1875;
 
-	/* Register sysfs hooks */
-	err = sysfs_create_group(&client->dev.kobj, &lm87_group);
-	if (err)
-		goto exit_stop;
-
-	if (data->channel & CHAN_NO_FAN(0)) {
-		err = sysfs_create_group(&client->dev.kobj, &lm87_group_in6);
-		if (err)
-			goto exit_remove;
-	} else {
-		err = sysfs_create_group(&client->dev.kobj, &lm87_group_fan1);
-		if (err)
-			goto exit_remove;
-	}
-
-	if (data->channel & CHAN_NO_FAN(1)) {
-		err = sysfs_create_group(&client->dev.kobj, &lm87_group_in7);
-		if (err)
-			goto exit_remove;
-	} else {
-		err = sysfs_create_group(&client->dev.kobj, &lm87_group_fan2);
-		if (err)
-			goto exit_remove;
-	}
-
-	if (data->channel & CHAN_TEMP3) {
-		err = sysfs_create_group(&client->dev.kobj, &lm87_group_temp3);
-		if (err)
-			goto exit_remove;
-	} else {
-		err = sysfs_create_group(&client->dev.kobj, &lm87_group_in0_5);
-		if (err)
-			goto exit_remove;
-	}
+	/*
+	 * Construct the list of attributes, the list depends on the
+	 * configuration of the chip
+	 */
+	data->attr_groups[group_tail++] = &lm87_group;
+	if (data->channel & CHAN_NO_FAN(0))
+		data->attr_groups[group_tail++] = &lm87_group_in6;
+	else
+		data->attr_groups[group_tail++] = &lm87_group_fan1;
+
+	if (data->channel & CHAN_NO_FAN(1))
+		data->attr_groups[group_tail++] = &lm87_group_in7;
+	else
+		data->attr_groups[group_tail++] = &lm87_group_fan2;
+
+	if (data->channel & CHAN_TEMP3)
+		data->attr_groups[group_tail++] = &lm87_group_temp3;
+	else
+		data->attr_groups[group_tail++] = &lm87_group_in0_5;
 
 	if (!(data->channel & CHAN_NO_VID)) {
 		data->vrm = vid_which_vrm();
-		err = sysfs_create_group(&client->dev.kobj, &lm87_group_vid);
-		if (err)
-			goto exit_remove;
-	}
-
-	data->hwmon_dev = hwmon_device_register(&client->dev);
-	if (IS_ERR(data->hwmon_dev)) {
-		err = PTR_ERR(data->hwmon_dev);
-		goto exit_remove;
+		data->attr_groups[group_tail++] = &lm87_group_vid;
 	}
 
-	return 0;
-
-exit_remove:
-	lm87_remove_files(client);
-exit_stop:
-	lm87_write_value(client, LM87_REG_CONFIG, data->config);
-	return err;
-}
-
-static int lm87_remove(struct i2c_client *client)
-{
-	struct lm87_data *data = i2c_get_clientdata(client);
-
-	hwmon_device_unregister(data->hwmon_dev);
-	lm87_remove_files(client);
-
-	lm87_write_value(client, LM87_REG_CONFIG, data->config);
-	return 0;
+	hwmon_dev = devm_hwmon_device_register_with_groups(
+	    &client->dev, client->name, client, data->attr_groups);
+	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
 /*
@@ -1006,7 +973,6 @@ static struct i2c_driver lm87_driver = {
 		.name	= "lm87",
 	},
 	.probe		= lm87_probe,
-	.remove		= lm87_remove,
 	.id_table	= lm87_id,
 	.detect		= lm87_detect,
 	.address_list	= normal_i2c,
diff --git a/drivers/hwmon/mcp3021.c b/drivers/hwmon/mcp3021.c
index 972444a14cca..1929734c3b1d 100644
--- a/drivers/hwmon/mcp3021.c
+++ b/drivers/hwmon/mcp3021.c
@@ -4,6 +4,7 @@
  * Copyright (C) 2008-2009, 2012 Freescale Semiconductor, Inc.
  * Author: Mingkai Hu <Mingkai.hu@freescale.com>
  * Reworked by Sven Schuchmann <schuchmann@schleissheimer.de>
+ * DT support added by Clemens Gruber <clemens.gruber@pqgruber.com>
  *
  * This driver export the value of analog input voltage to sysfs, the
  * voltage unit is mV. Through the sysfs interface, lm-sensors tool
@@ -22,11 +23,13 @@
 #include <linux/i2c.h>
 #include <linux/err.h>
 #include <linux/device.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
 
-/* Vdd info */
-#define MCP3021_VDD_MAX		5500
-#define MCP3021_VDD_MIN		2700
-#define MCP3021_VDD_REF		3300
+/* Vdd / reference voltage in millivolt */
+#define MCP3021_VDD_REF_MAX	5500
+#define MCP3021_VDD_REF_MIN	2700
+#define MCP3021_VDD_REF_DEFAULT	3300
 
 /* output format */
 #define MCP3021_SAR_SHIFT	2
@@ -47,7 +50,7 @@ enum chips {
  */
 struct mcp3021_data {
 	struct device *hwmon_dev;
-	u32 vdd;	/* device power supply */
+	u32 vdd;        /* supply and reference voltage in millivolt */
 	u16 sar_shift;
 	u16 sar_mask;
 	u8 output_res;
@@ -99,13 +102,14 @@ static ssize_t show_in_input(struct device *dev, struct device_attribute *attr,
 	return sprintf(buf, "%d\n", in_input);
 }
 
-static DEVICE_ATTR(in0_input, S_IRUGO, show_in_input, NULL);
+static DEVICE_ATTR(in0_input, 0444, show_in_input, NULL);
 
 static int mcp3021_probe(struct i2c_client *client,
 				const struct i2c_device_id *id)
 {
 	int err;
 	struct mcp3021_data *data = NULL;
+	struct device_node *np = client->dev.of_node;
 
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
 		return -ENODEV;
@@ -117,6 +121,21 @@ static int mcp3021_probe(struct i2c_client *client,
 
 	i2c_set_clientdata(client, data);
 
+	if (np) {
+		if (!of_property_read_u32(np, "reference-voltage-microvolt",
+					  &data->vdd))
+			data->vdd /= 1000;
+		else
+			data->vdd = MCP3021_VDD_REF_DEFAULT;
+	} else {
+		u32 *pdata = dev_get_platdata(&client->dev);
+
+		if (pdata)
+			data->vdd = *pdata;
+		else
+			data->vdd = MCP3021_VDD_REF_DEFAULT;
+	}
+
 	switch (id->driver_data) {
 	case mcp3021:
 		data->sar_shift = MCP3021_SAR_SHIFT;
@@ -131,13 +150,8 @@ static int mcp3021_probe(struct i2c_client *client,
 		break;
 	}
 
-	if (dev_get_platdata(&client->dev)) {
-		data->vdd = *(u32 *)dev_get_platdata(&client->dev);
-		if (data->vdd > MCP3021_VDD_MAX || data->vdd < MCP3021_VDD_MIN)
-			return -EINVAL;
-	} else {
-		data->vdd = MCP3021_VDD_REF;
-	}
+	if (data->vdd > MCP3021_VDD_REF_MAX || data->vdd < MCP3021_VDD_REF_MIN)
+		return -EINVAL;
 
 	err = sysfs_create_file(&client->dev.kobj, &dev_attr_in0_input.attr);
 	if (err)
@@ -173,9 +187,19 @@ static const struct i2c_device_id mcp3021_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, mcp3021_id);
 
+#ifdef CONFIG_OF
+static const struct of_device_id of_mcp3021_match[] = {
+	{ .compatible = "microchip,mcp3021", .data = (void *)mcp3021 },
+	{ .compatible = "microchip,mcp3221", .data = (void *)mcp3221 },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, of_mcp3021_match);
+#endif
+
 static struct i2c_driver mcp3021_driver = {
 	.driver = {
 		.name = "mcp3021",
+		.of_match_table = of_match_ptr(of_mcp3021_match),
 	},
 	.probe = mcp3021_probe,
 	.remove = mcp3021_remove,
diff --git a/drivers/hwmon/nct7802.c b/drivers/hwmon/nct7802.c
index 3ce33d244cc0..12b94b094c0d 100644
--- a/drivers/hwmon/nct7802.c
+++ b/drivers/hwmon/nct7802.c
@@ -259,13 +259,15 @@ static int nct7802_read_fan_min(struct nct7802_data *data, u8 reg_fan_low,
 		ret = 0;
 	else if (ret)
 		ret = DIV_ROUND_CLOSEST(1350000U, ret);
+	else
+		ret = 1350000U;
 abort:
 	mutex_unlock(&data->access_lock);
 	return ret;
 }
 
 static int nct7802_write_fan_min(struct nct7802_data *data, u8 reg_fan_low,
-				 u8 reg_fan_high, unsigned int limit)
+				 u8 reg_fan_high, unsigned long limit)
 {
 	int err;
 
@@ -326,8 +328,8 @@ static int nct7802_write_voltage(struct nct7802_data *data, int nr, int index,
 	int shift = 8 - REG_VOLTAGE_LIMIT_MSB_SHIFT[index - 1][nr];
 	int err;
 
+	voltage = clamp_val(voltage, 0, 0x3ff * nct7802_vmul[nr]);
 	voltage = DIV_ROUND_CLOSEST(voltage, nct7802_vmul[nr]);
-	voltage = clamp_val(voltage, 0, 0x3ff);
 
 	mutex_lock(&data->access_lock);
 	err = regmap_write(data->regmap,
@@ -402,7 +404,7 @@ static ssize_t store_temp(struct device *dev, struct device_attribute *attr,
 	if (err < 0)
 		return err;
 
-	val = clamp_val(DIV_ROUND_CLOSEST(val, 1000), -128, 127);
+	val = DIV_ROUND_CLOSEST(clamp_val(val, -128000, 127000), 1000);
 
 	err = regmap_write(data->regmap, nr, val & 0xff);
 	return err ? : count;
diff --git a/drivers/hwmon/pmbus/adm1275.c b/drivers/hwmon/pmbus/adm1275.c
index 3baa4f4a8c5e..4ab5293c7bf0 100644
--- a/drivers/hwmon/pmbus/adm1275.c
+++ b/drivers/hwmon/pmbus/adm1275.c
@@ -499,15 +499,27 @@ static int adm1275_probe(struct i2c_client *client,
 		pindex = 2;
 		tindex = 3;
 
-		info->func[0] |= PMBUS_HAVE_PIN | PMBUS_HAVE_STATUS_INPUT;
+		info->func[0] |= PMBUS_HAVE_PIN | PMBUS_HAVE_STATUS_INPUT |
+			PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT;
+
+		/* Enable VOUT if not enabled (it is disabled by default) */
+		if (!(config & ADM1278_VOUT_EN)) {
+			config |= ADM1278_VOUT_EN;
+			ret = i2c_smbus_write_byte_data(client,
+							ADM1275_PMON_CONFIG,
+							config);
+			if (ret < 0) {
+				dev_err(&client->dev,
+					"Failed to enable VOUT monitoring\n");
+				return -ENODEV;
+			}
+		}
+
 		if (config & ADM1278_TEMP1_EN)
 			info->func[0] |=
 				PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP;
 		if (config & ADM1278_VIN_EN)
 			info->func[0] |= PMBUS_HAVE_VIN;
-		if (config & ADM1278_VOUT_EN)
-			info->func[0] |=
-				PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT;
 		break;
 	case adm1293:
 	case adm1294:
diff --git a/drivers/hwmon/scpi-hwmon.c b/drivers/hwmon/scpi-hwmon.c
index 559a3dcd64d8..094f948f99ff 100644
--- a/drivers/hwmon/scpi-hwmon.c
+++ b/drivers/hwmon/scpi-hwmon.c
@@ -251,6 +251,7 @@ static const struct of_device_id scpi_of_match[] = {
 	{.compatible = "arm,scpi-sensors"},
 	{},
 };
+MODULE_DEVICE_TABLE(of, scpi_of_match);
 
 static struct platform_driver scpi_hwmon_platdrv = {
 	.driver = {
diff --git a/drivers/hwmon/smsc47m192.c b/drivers/hwmon/smsc47m192.c
index 6ac7cda72d4c..15650f247679 100644
--- a/drivers/hwmon/smsc47m192.c
+++ b/drivers/hwmon/smsc47m192.c
@@ -77,14 +77,15 @@ static inline unsigned int IN_FROM_REG(u8 reg, int n)
 
 static inline u8 IN_TO_REG(unsigned long val, int n)
 {
-	return clamp_val(SCALE(val, 192, nom_mv[n]), 0, 255);
+	val = clamp_val(val, 0, nom_mv[n] * 255 / 192);
+	return SCALE(val, 192, nom_mv[n]);
 }
 
 /*
  * TEMP: 0.001 degC units (-128C to +127C)
  * REG: 1C/bit, two's complement
  */
-static inline s8 TEMP_TO_REG(int val)
+static inline s8 TEMP_TO_REG(long val)
 {
 	return SCALE(clamp_val(val, -128000, 127000), 1, 1000);
 }
diff --git a/drivers/hwmon/tc654.c b/drivers/hwmon/tc654.c
new file mode 100644
index 000000000000..18136e1f95fd
--- /dev/null
+++ b/drivers/hwmon/tc654.c
@@ -0,0 +1,514 @@
+/*
+ * tc654.c - Linux kernel modules for fan speed controller
+ *
+ * Copyright (C) 2016 Allied Telesis Labs NZ
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/jiffies.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/util_macros.h>
+
+enum tc654_regs {
+	TC654_REG_RPM1 = 0x00,	/* RPM Output 1 */
+	TC654_REG_RPM2 = 0x01,	/* RPM Output 2 */
+	TC654_REG_FAN_FAULT1 = 0x02,	/* Fan Fault 1 Threshold */
+	TC654_REG_FAN_FAULT2 = 0x03,	/* Fan Fault 2 Threshold */
+	TC654_REG_CONFIG = 0x04,	/* Configuration */
+	TC654_REG_STATUS = 0x05,	/* Status */
+	TC654_REG_DUTY_CYCLE = 0x06,	/* Fan Speed Duty Cycle */
+	TC654_REG_MFR_ID = 0x07,	/* Manufacturer Identification */
+	TC654_REG_VER_ID = 0x08,	/* Version Identification */
+};
+
+/* Macros to easily index the registers */
+#define TC654_REG_RPM(idx)		(TC654_REG_RPM1 + (idx))
+#define TC654_REG_FAN_FAULT(idx)	(TC654_REG_FAN_FAULT1 + (idx))
+
+/* Config register bits */
+#define TC654_REG_CONFIG_RES		BIT(6)	/* Resolution Selection */
+#define TC654_REG_CONFIG_DUTYC		BIT(5)	/* Duty Cycle Control */
+#define TC654_REG_CONFIG_SDM		BIT(0)	/* Shutdown Mode */
+
+/* Status register bits */
+#define TC654_REG_STATUS_F2F		BIT(1)	/* Fan 2 Fault */
+#define TC654_REG_STATUS_F1F		BIT(0)	/* Fan 1 Fault */
+
+/* RPM resolution for RPM Output registers */
+#define TC654_HIGH_RPM_RESOLUTION	25	/* 25 RPM resolution */
+#define TC654_LOW_RPM_RESOLUTION	50	/* 50 RPM resolution */
+
+/* Convert to the fan fault RPM threshold from register value */
+#define TC654_FAN_FAULT_FROM_REG(val)	((val) * 50)	/* 50 RPM resolution */
+
+/* Convert to register value from the fan fault RPM threshold */
+#define TC654_FAN_FAULT_TO_REG(val)	(((val) / 50) & 0xff)
+
+/* Register data is read (and cached) at most once per second. */
+#define TC654_UPDATE_INTERVAL		HZ
+
+struct tc654_data {
+	struct i2c_client *client;
+
+	/* update mutex */
+	struct mutex update_lock;
+
+	/* tc654 register cache */
+	bool valid;
+	unsigned long last_updated;	/* in jiffies */
+
+	u8 rpm_output[2];	/* The fan RPM data for fans 1 and 2 is then
+				 * written to registers RPM1 and RPM2
+				 */
+	u8 fan_fault[2];	/* The Fan Fault Threshold Registers are used to
+				 * set the fan fault threshold levels for fan 1
+				 * and fan 2
+				 */
+	u8 config;	/* The Configuration Register is an 8-bit read/
+			 * writable multi-function control register
+			 *   7: Fan Fault Clear
+			 *      1 = Clear Fan Fault
+			 *      0 = Normal Operation (default)
+			 *   6: Resolution Selection for RPM Output Registers
+			 *      RPM Output Registers (RPM1 and RPM2) will be
+			 *      set for
+			 *      1 = 25 RPM (9-bit) resolution
+			 *      0 = 50 RPM (8-bit) resolution (default)
+			 *   5: Duty Cycle Control Method
+			 *      The V OUT duty cycle will be controlled via
+			 *      1 = the SMBus interface.
+			 *      0 = via the V IN analog input pin. (default)
+			 * 4,3: Fan 2 Pulses Per Rotation
+			 *      00 = 1
+			 *      01 = 2 (default)
+			 *      10 = 4
+			 *      11 = 8
+			 * 2,1: Fan 1 Pulses Per Rotation
+			 *      00 = 1
+			 *      01 = 2 (default)
+			 *      10 = 4
+			 *      11 = 8
+			 *   0: Shutdown Mode
+			 *      1 = Shutdown mode.
+			 *      0 = Normal operation. (default)
+			 */
+	u8 status;	/* The Status register provides all the information
+			 * about what is going on within the TC654/TC655
+			 * devices.
+			 * 7,6: Unimplemented, Read as '0'
+			 *   5: Over-Temperature Fault Condition
+			 *      1 = Over-Temperature condition has occurred
+			 *      0 = Normal operation. V IN is less than 2.6V
+			 *   4: RPM2 Counter Overflow
+			 *      1 = Fault condition
+			 *      0 = Normal operation
+			 *   3: RPM1 Counter Overflow
+			 *      1 = Fault condition
+			 *      0 = Normal operation
+			 *   2: V IN Input Status
+			 *      1 = V IN is open
+			 *      0 = Normal operation. voltage present at V IN
+			 *   1: Fan 2 Fault
+			 *      1 = Fault condition
+			 *      0 = Normal operation
+			 *   0: Fan 1 Fault
+			 *      1 = Fault condition
+			 *      0 = Normal operation
+			 */
+	u8 duty_cycle;	/* The DUTY_CYCLE register is a 4-bit read/
+			 * writable register used to control the duty
+			 * cycle of the V OUT output.
+			 */
+};
+
+/* helper to grab and cache data, at most one time per second */
+static struct tc654_data *tc654_update_client(struct device *dev)
+{
+	struct tc654_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
+	int ret = 0;
+
+	mutex_lock(&data->update_lock);
+	if (time_before(jiffies, data->last_updated + TC654_UPDATE_INTERVAL) &&
+	    likely(data->valid))
+		goto out;
+
+	ret = i2c_smbus_read_byte_data(client, TC654_REG_RPM(0));
+	if (ret < 0)
+		goto out;
+	data->rpm_output[0] = ret;
+
+	ret = i2c_smbus_read_byte_data(client, TC654_REG_RPM(1));
+	if (ret < 0)
+		goto out;
+	data->rpm_output[1] = ret;
+
+	ret = i2c_smbus_read_byte_data(client, TC654_REG_FAN_FAULT(0));
+	if (ret < 0)
+		goto out;
+	data->fan_fault[0] = ret;
+
+	ret = i2c_smbus_read_byte_data(client, TC654_REG_FAN_FAULT(1));
+	if (ret < 0)
+		goto out;
+	data->fan_fault[1] = ret;
+
+	ret = i2c_smbus_read_byte_data(client, TC654_REG_CONFIG);
+	if (ret < 0)
+		goto out;
+	data->config = ret;
+
+	ret = i2c_smbus_read_byte_data(client, TC654_REG_STATUS);
+	if (ret < 0)
+		goto out;
+	data->status = ret;
+
+	ret = i2c_smbus_read_byte_data(client, TC654_REG_DUTY_CYCLE);
+	if (ret < 0)
+		goto out;
+	data->duty_cycle = ret & 0x0f;
+
+	data->last_updated = jiffies;
+	data->valid = true;
+out:
+	mutex_unlock(&data->update_lock);
+
+	if (ret < 0)		/* upon error, encode it in return value */
+		data = ERR_PTR(ret);
+
+	return data;
+}
+
+/*
+ * sysfs attributes
+ */
+
+static ssize_t show_fan(struct device *dev, struct device_attribute *da,
+			char *buf)
+{
+	int nr = to_sensor_dev_attr(da)->index;
+	struct tc654_data *data = tc654_update_client(dev);
+	int val;
+
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
+	if (data->config & TC654_REG_CONFIG_RES)
+		val = data->rpm_output[nr] * TC654_HIGH_RPM_RESOLUTION;
+	else
+		val = data->rpm_output[nr] * TC654_LOW_RPM_RESOLUTION;
+
+	return sprintf(buf, "%d\n", val);
+}
+
+static ssize_t show_fan_min(struct device *dev, struct device_attribute *da,
+			    char *buf)
+{
+	int nr = to_sensor_dev_attr(da)->index;
+	struct tc654_data *data = tc654_update_client(dev);
+
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
+	return sprintf(buf, "%d\n",
+		       TC654_FAN_FAULT_FROM_REG(data->fan_fault[nr]));
+}
+
+static ssize_t set_fan_min(struct device *dev, struct device_attribute *da,
+			   const char *buf, size_t count)
+{
+	int nr = to_sensor_dev_attr(da)->index;
+	struct tc654_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
+	unsigned long val;
+	int ret;
+
+	if (kstrtoul(buf, 10, &val))
+		return -EINVAL;
+
+	val = clamp_val(val, 0, 12750);
+
+	mutex_lock(&data->update_lock);
+
+	data->fan_fault[nr] = TC654_FAN_FAULT_TO_REG(val);
+	ret = i2c_smbus_write_byte_data(client, TC654_REG_FAN_FAULT(nr),
+					data->fan_fault[nr]);
+
+	mutex_unlock(&data->update_lock);
+	return ret < 0 ? ret : count;
+}
+
+static ssize_t show_fan_alarm(struct device *dev, struct device_attribute *da,
+			      char *buf)
+{
+	int nr = to_sensor_dev_attr(da)->index;
+	struct tc654_data *data = tc654_update_client(dev);
+	int val;
+
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
+	if (nr == 0)
+		val = !!(data->status & TC654_REG_STATUS_F1F);
+	else
+		val = !!(data->status & TC654_REG_STATUS_F2F);
+
+	return sprintf(buf, "%d\n", val);
+}
+
+static const u8 TC654_FAN_PULSE_SHIFT[] = { 1, 3 };
+
+static ssize_t show_fan_pulses(struct device *dev, struct device_attribute *da,
+			       char *buf)
+{
+	int nr = to_sensor_dev_attr(da)->index;
+	struct tc654_data *data = tc654_update_client(dev);
+	u8 val;
+
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
+	val = BIT((data->config >> TC654_FAN_PULSE_SHIFT[nr]) & 0x03);
+	return sprintf(buf, "%d\n", val);
+}
+
+static ssize_t set_fan_pulses(struct device *dev, struct device_attribute *da,
+			      const char *buf, size_t count)
+{
+	int nr = to_sensor_dev_attr(da)->index;
+	struct tc654_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
+	u8 config;
+	unsigned long val;
+	int ret;
+
+	if (kstrtoul(buf, 10, &val))
+		return -EINVAL;
+
+	switch (val) {
+	case 1:
+		config = 0;
+		break;
+	case 2:
+		config = 1;
+		break;
+	case 4:
+		config = 2;
+		break;
+	case 8:
+		config = 3;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	mutex_lock(&data->update_lock);
+
+	data->config &= ~(0x03 << TC654_FAN_PULSE_SHIFT[nr]);
+	data->config |= (config << TC654_FAN_PULSE_SHIFT[nr]);
+	ret = i2c_smbus_write_byte_data(client, TC654_REG_CONFIG, data->config);
+
+	mutex_unlock(&data->update_lock);
+	return ret < 0 ? ret : count;
+}
+
+static ssize_t show_pwm_mode(struct device *dev,
+			     struct device_attribute *da, char *buf)
+{
+	struct tc654_data *data = tc654_update_client(dev);
+
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
+	return sprintf(buf, "%d\n", !!(data->config & TC654_REG_CONFIG_DUTYC));
+}
+
+static ssize_t set_pwm_mode(struct device *dev,
+			    struct device_attribute *da,
+			    const char *buf, size_t count)
+{
+	struct tc654_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
+	unsigned long val;
+	int ret;
+
+	if (kstrtoul(buf, 10, &val))
+		return -EINVAL;
+
+	if (val != 0 && val != 1)
+		return -EINVAL;
+
+	mutex_lock(&data->update_lock);
+
+	if (val)
+		data->config |= TC654_REG_CONFIG_DUTYC;
+	else
+		data->config &= ~TC654_REG_CONFIG_DUTYC;
+
+	ret = i2c_smbus_write_byte_data(client, TC654_REG_CONFIG, data->config);
+
+	mutex_unlock(&data->update_lock);
+	return ret < 0 ? ret : count;
+}
+
+static const int tc654_pwm_map[16] = { 77,  88, 102, 112, 124, 136, 148, 160,
+				      172, 184, 196, 207, 219, 231, 243, 255};
+
+static ssize_t show_pwm(struct device *dev, struct device_attribute *da,
+			char *buf)
+{
+	struct tc654_data *data = tc654_update_client(dev);
+	int pwm;
+
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
+	if (data->config & TC654_REG_CONFIG_SDM)
+		pwm = 0;
+	else
+		pwm = tc654_pwm_map[data->duty_cycle];
+
+	return sprintf(buf, "%d\n", pwm);
+}
+
+static ssize_t set_pwm(struct device *dev, struct device_attribute *da,
+		       const char *buf, size_t count)
+{
+	struct tc654_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
+	unsigned long val;
+	int ret;
+
+	if (kstrtoul(buf, 10, &val))
+		return -EINVAL;
+	if (val > 255)
+		return -EINVAL;
+
+	mutex_lock(&data->update_lock);
+
+	if (val == 0)
+		data->config |= TC654_REG_CONFIG_SDM;
+	else
+		data->config &= ~TC654_REG_CONFIG_SDM;
+
+	data->duty_cycle = find_closest(val, tc654_pwm_map,
+					ARRAY_SIZE(tc654_pwm_map));
+
+	ret = i2c_smbus_write_byte_data(client, TC654_REG_CONFIG, data->config);
+	if (ret < 0)
+		goto out;
+
+	ret = i2c_smbus_write_byte_data(client, TC654_REG_DUTY_CYCLE,
+					data->duty_cycle);
+
+out:
+	mutex_unlock(&data->update_lock);
+	return ret < 0 ? ret : count;
+}
+
+static SENSOR_DEVICE_ATTR(fan1_input, S_IRUGO, show_fan, NULL, 0);
+static SENSOR_DEVICE_ATTR(fan2_input, S_IRUGO, show_fan, NULL, 1);
+static SENSOR_DEVICE_ATTR(fan1_min, S_IWUSR | S_IRUGO, show_fan_min,
+			  set_fan_min, 0);
+static SENSOR_DEVICE_ATTR(fan2_min, S_IWUSR | S_IRUGO, show_fan_min,
+			  set_fan_min, 1);
+static SENSOR_DEVICE_ATTR(fan1_alarm, S_IRUGO, show_fan_alarm, NULL, 0);
+static SENSOR_DEVICE_ATTR(fan2_alarm, S_IRUGO, show_fan_alarm, NULL, 1);
+static SENSOR_DEVICE_ATTR(fan1_pulses, S_IWUSR | S_IRUGO, show_fan_pulses,
+			  set_fan_pulses, 0);
+static SENSOR_DEVICE_ATTR(fan2_pulses, S_IWUSR | S_IRUGO, show_fan_pulses,
+			  set_fan_pulses, 1);
+static SENSOR_DEVICE_ATTR(pwm1_mode, S_IWUSR | S_IRUGO,
+			  show_pwm_mode, set_pwm_mode, 0);
+static SENSOR_DEVICE_ATTR(pwm1, S_IWUSR | S_IRUGO, show_pwm,
+			  set_pwm, 0);
+
+/* Driver data */
+static struct attribute *tc654_attrs[] = {
+	&sensor_dev_attr_fan1_input.dev_attr.attr,
+	&sensor_dev_attr_fan2_input.dev_attr.attr,
+	&sensor_dev_attr_fan1_min.dev_attr.attr,
+	&sensor_dev_attr_fan2_min.dev_attr.attr,
+	&sensor_dev_attr_fan1_alarm.dev_attr.attr,
+	&sensor_dev_attr_fan2_alarm.dev_attr.attr,
+	&sensor_dev_attr_fan1_pulses.dev_attr.attr,
+	&sensor_dev_attr_fan2_pulses.dev_attr.attr,
+	&sensor_dev_attr_pwm1_mode.dev_attr.attr,
+	&sensor_dev_attr_pwm1.dev_attr.attr,
+	NULL
+};
+
+ATTRIBUTE_GROUPS(tc654);
+
+/*
+ * device probe and removal
+ */
+
+static int tc654_probe(struct i2c_client *client,
+		       const struct i2c_device_id *id)
+{
+	struct device *dev = &client->dev;
+	struct tc654_data *data;
+	struct device *hwmon_dev;
+	int ret;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_BYTE_DATA))
+		return -ENODEV;
+
+	data = devm_kzalloc(dev, sizeof(struct tc654_data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->client = client;
+	mutex_init(&data->update_lock);
+
+	ret = i2c_smbus_read_byte_data(client, TC654_REG_CONFIG);
+	if (ret < 0)
+		return ret;
+
+	data->config = ret;
+
+	hwmon_dev =
+	    devm_hwmon_device_register_with_groups(dev, client->name, data,
+						   tc654_groups);
+	return PTR_ERR_OR_ZERO(hwmon_dev);
+}
+
+static const struct i2c_device_id tc654_id[] = {
+	{"tc654", 0},
+	{"tc655", 0},
+	{}
+};
+
+MODULE_DEVICE_TABLE(i2c, tc654_id);
+
+static struct i2c_driver tc654_driver = {
+	.driver = {
+		   .name = "tc654",
+		   },
+	.probe = tc654_probe,
+	.id_table = tc654_id,
+};
+
+module_i2c_driver(tc654_driver);
+
+MODULE_AUTHOR("Allied Telesis Labs");
+MODULE_DESCRIPTION("Microchip TC654/TC655 driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/tmp108.c b/drivers/hwmon/tmp108.c
new file mode 100644
index 000000000000..91bb94639286
--- /dev/null
+++ b/drivers/hwmon/tmp108.c
@@ -0,0 +1,469 @@
+/* Texas Instruments TMP108 SMBus temperature sensor driver
+ *
+ * Copyright (C) 2016 John Muir <john@jmuir.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/jiffies.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#define	DRIVER_NAME "tmp108"
+
+#define	TMP108_REG_TEMP		0x00
+#define	TMP108_REG_CONF		0x01
+#define	TMP108_REG_TLOW		0x02
+#define	TMP108_REG_THIGH	0x03
+
+#define TMP108_TEMP_MIN_MC	-50000 /* Minimum millicelcius. */
+#define TMP108_TEMP_MAX_MC	127937 /* Maximum millicelcius. */
+
+/* Configuration register bits.
+ * Note: these bit definitions are byte swapped.
+ */
+#define TMP108_CONF_M0		0x0100 /* Sensor mode. */
+#define TMP108_CONF_M1		0x0200
+#define TMP108_CONF_TM		0x0400 /* Thermostat mode. */
+#define TMP108_CONF_FL		0x0800 /* Watchdog flag - TLOW */
+#define TMP108_CONF_FH		0x1000 /* Watchdog flag - THIGH */
+#define TMP108_CONF_CR0		0x2000 /* Conversion rate. */
+#define TMP108_CONF_CR1		0x4000
+#define TMP108_CONF_ID		0x8000
+#define TMP108_CONF_HYS0	0x0010 /* Hysteresis. */
+#define TMP108_CONF_HYS1	0x0020
+#define TMP108_CONF_POL		0x0080 /* Polarity of alert. */
+
+/* Defaults set by the hardware upon reset. */
+#define TMP108_CONF_DEFAULTS		(TMP108_CONF_CR0 | TMP108_CONF_TM |\
+					 TMP108_CONF_HYS0 | TMP108_CONF_M1)
+/* These bits are read-only. */
+#define TMP108_CONF_READ_ONLY		(TMP108_CONF_FL | TMP108_CONF_FH |\
+					 TMP108_CONF_ID)
+
+#define TMP108_CONF_MODE_MASK		(TMP108_CONF_M0|TMP108_CONF_M1)
+#define TMP108_MODE_SHUTDOWN		0x0000
+#define TMP108_MODE_ONE_SHOT		TMP108_CONF_M0
+#define TMP108_MODE_CONTINUOUS		TMP108_CONF_M1		/* Default */
+					/* When M1 is set, M0 is ignored. */
+
+#define TMP108_CONF_CONVRATE_MASK	(TMP108_CONF_CR0|TMP108_CONF_CR1)
+#define TMP108_CONVRATE_0P25HZ		0x0000
+#define TMP108_CONVRATE_1HZ		TMP108_CONF_CR0		/* Default */
+#define TMP108_CONVRATE_4HZ		TMP108_CONF_CR1
+#define TMP108_CONVRATE_16HZ		(TMP108_CONF_CR0|TMP108_CONF_CR1)
+
+#define TMP108_CONF_HYSTERESIS_MASK	(TMP108_CONF_HYS0|TMP108_CONF_HYS1)
+#define TMP108_HYSTERESIS_0C		0x0000
+#define TMP108_HYSTERESIS_1C		TMP108_CONF_HYS0	/* Default */
+#define TMP108_HYSTERESIS_2C		TMP108_CONF_HYS1
+#define TMP108_HYSTERESIS_4C		(TMP108_CONF_HYS0|TMP108_CONF_HYS1)
+
+#define TMP108_CONVERSION_TIME_MS	30	/* in milli-seconds */
+
+struct tmp108 {
+	struct regmap *regmap;
+	u16 orig_config;
+	unsigned long ready_time;
+};
+
+/* convert 12-bit TMP108 register value to milliCelsius */
+static inline int tmp108_temp_reg_to_mC(s16 val)
+{
+	return (val & ~0x0f) * 1000 / 256;
+}
+
+/* convert milliCelsius to left adjusted 12-bit TMP108 register value */
+static inline u16 tmp108_mC_to_temp_reg(int val)
+{
+	return (val * 256) / 1000;
+}
+
+static int tmp108_read(struct device *dev, enum hwmon_sensor_types type,
+		       u32 attr, int channel, long *temp)
+{
+	struct tmp108 *tmp108 = dev_get_drvdata(dev);
+	unsigned int regval;
+	int err, hyst;
+
+	if (type == hwmon_chip) {
+		if (attr == hwmon_chip_update_interval) {
+			err = regmap_read(tmp108->regmap, TMP108_REG_CONF,
+					  &regval);
+			if (err < 0)
+				return err;
+			switch (regval & TMP108_CONF_CONVRATE_MASK) {
+			case TMP108_CONVRATE_0P25HZ:
+			default:
+				*temp = 4000;
+				break;
+			case TMP108_CONVRATE_1HZ:
+				*temp = 1000;
+				break;
+			case TMP108_CONVRATE_4HZ:
+				*temp = 250;
+				break;
+			case TMP108_CONVRATE_16HZ:
+				*temp = 63;
+				break;
+			}
+			return 0;
+		}
+		return -EOPNOTSUPP;
+	}
+
+	switch (attr) {
+	case hwmon_temp_input:
+		/* Is it too early to return a conversion ? */
+		if (time_before(jiffies, tmp108->ready_time)) {
+			dev_dbg(dev, "%s: Conversion not ready yet..\n",
+				__func__);
+			return -EAGAIN;
+		}
+		err = regmap_read(tmp108->regmap, TMP108_REG_TEMP, &regval);
+		if (err < 0)
+			return err;
+		*temp = tmp108_temp_reg_to_mC(regval);
+		break;
+	case hwmon_temp_min:
+	case hwmon_temp_max:
+		err = regmap_read(tmp108->regmap, attr == hwmon_temp_min ?
+				  TMP108_REG_TLOW : TMP108_REG_THIGH, &regval);
+		if (err < 0)
+			return err;
+		*temp = tmp108_temp_reg_to_mC(regval);
+		break;
+	case hwmon_temp_min_alarm:
+	case hwmon_temp_max_alarm:
+		err = regmap_read(tmp108->regmap, TMP108_REG_CONF, &regval);
+		if (err < 0)
+			return err;
+		*temp = !!(regval & (attr == hwmon_temp_min_alarm ?
+				     TMP108_CONF_FL : TMP108_CONF_FH));
+		break;
+	case hwmon_temp_min_hyst:
+	case hwmon_temp_max_hyst:
+		err = regmap_read(tmp108->regmap, TMP108_REG_CONF, &regval);
+		if (err < 0)
+			return err;
+		switch (regval & TMP108_CONF_HYSTERESIS_MASK) {
+		case TMP108_HYSTERESIS_0C:
+		default:
+			hyst = 0;
+			break;
+		case TMP108_HYSTERESIS_1C:
+			hyst = 1000;
+			break;
+		case TMP108_HYSTERESIS_2C:
+			hyst = 2000;
+			break;
+		case TMP108_HYSTERESIS_4C:
+			hyst = 4000;
+			break;
+		}
+		err = regmap_read(tmp108->regmap, attr == hwmon_temp_min_hyst ?
+				  TMP108_REG_TLOW : TMP108_REG_THIGH, &regval);
+		if (err < 0)
+			return err;
+		*temp = tmp108_temp_reg_to_mC(regval);
+		if (attr == hwmon_temp_min_hyst)
+			*temp += hyst;
+		else
+			*temp -= hyst;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int tmp108_write(struct device *dev, enum hwmon_sensor_types type,
+			u32 attr, int channel, long temp)
+{
+	struct tmp108 *tmp108 = dev_get_drvdata(dev);
+	u32 regval, mask;
+	int err;
+
+	if (type == hwmon_chip) {
+		if (attr == hwmon_chip_update_interval) {
+			if (temp < 156)
+				mask = TMP108_CONVRATE_16HZ;
+			else if (temp < 625)
+				mask = TMP108_CONVRATE_4HZ;
+			else if (temp < 2500)
+				mask = TMP108_CONVRATE_1HZ;
+			else
+				mask = TMP108_CONVRATE_0P25HZ;
+			return regmap_update_bits(tmp108->regmap,
+						  TMP108_REG_CONF,
+						  TMP108_CONF_CONVRATE_MASK,
+						  mask);
+		}
+		return -EOPNOTSUPP;
+	}
+
+	switch (attr) {
+	case hwmon_temp_min:
+	case hwmon_temp_max:
+		temp = clamp_val(temp, TMP108_TEMP_MIN_MC, TMP108_TEMP_MAX_MC);
+		return regmap_write(tmp108->regmap,
+				    attr == hwmon_temp_min ?
+					TMP108_REG_TLOW : TMP108_REG_THIGH,
+				    tmp108_mC_to_temp_reg(temp));
+	case hwmon_temp_min_hyst:
+	case hwmon_temp_max_hyst:
+		temp = clamp_val(temp, TMP108_TEMP_MIN_MC, TMP108_TEMP_MAX_MC);
+		err = regmap_read(tmp108->regmap,
+				  attr == hwmon_temp_min_hyst ?
+					TMP108_REG_TLOW : TMP108_REG_THIGH,
+				  &regval);
+		if (err < 0)
+			return err;
+		if (attr == hwmon_temp_min_hyst)
+			temp -= tmp108_temp_reg_to_mC(regval);
+		else
+			temp = tmp108_temp_reg_to_mC(regval) - temp;
+		if (temp < 500)
+			mask = TMP108_HYSTERESIS_0C;
+		else if (temp < 1500)
+			mask = TMP108_HYSTERESIS_1C;
+		else if (temp < 3000)
+			mask = TMP108_HYSTERESIS_2C;
+		else
+			mask = TMP108_HYSTERESIS_4C;
+		return regmap_update_bits(tmp108->regmap, TMP108_REG_CONF,
+					  TMP108_CONF_HYSTERESIS_MASK, mask);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static umode_t tmp108_is_visible(const void *data, enum hwmon_sensor_types type,
+				 u32 attr, int channel)
+{
+	if (type == hwmon_chip && attr == hwmon_chip_update_interval)
+		return 0644;
+
+	if (type != hwmon_temp)
+		return 0;
+
+	switch (attr) {
+	case hwmon_temp_input:
+	case hwmon_temp_min_alarm:
+	case hwmon_temp_max_alarm:
+		return 0444;
+	case hwmon_temp_min:
+	case hwmon_temp_max:
+	case hwmon_temp_min_hyst:
+	case hwmon_temp_max_hyst:
+		return 0644;
+	default:
+		return 0;
+	}
+}
+
+static u32 tmp108_chip_config[] = {
+	HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL,
+	0
+};
+
+static const struct hwmon_channel_info tmp108_chip = {
+	.type = hwmon_chip,
+	.config = tmp108_chip_config,
+};
+
+static u32 tmp108_temp_config[] = {
+	HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MIN | HWMON_T_MIN_HYST
+		| HWMON_T_MAX_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM,
+	0
+};
+
+static const struct hwmon_channel_info tmp108_temp = {
+	.type = hwmon_temp,
+	.config = tmp108_temp_config,
+};
+
+static const struct hwmon_channel_info *tmp108_info[] = {
+	&tmp108_chip,
+	&tmp108_temp,
+	NULL
+};
+
+static const struct hwmon_ops tmp108_hwmon_ops = {
+	.is_visible = tmp108_is_visible,
+	.read = tmp108_read,
+	.write = tmp108_write,
+};
+
+static const struct hwmon_chip_info tmp108_chip_info = {
+	.ops = &tmp108_hwmon_ops,
+	.info = tmp108_info,
+};
+
+static void tmp108_restore_config(void *data)
+{
+	struct tmp108 *tmp108 = data;
+
+	regmap_write(tmp108->regmap, TMP108_REG_CONF, tmp108->orig_config);
+}
+
+static bool tmp108_is_writeable_reg(struct device *dev, unsigned int reg)
+{
+	return reg != TMP108_REG_TEMP;
+}
+
+static bool tmp108_is_volatile_reg(struct device *dev, unsigned int reg)
+{
+	/* Configuration register must be volatile to enable FL and FH. */
+	return reg == TMP108_REG_TEMP || reg == TMP108_REG_CONF;
+}
+
+static const struct regmap_config tmp108_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 16,
+	.max_register = TMP108_REG_THIGH,
+	.writeable_reg = tmp108_is_writeable_reg,
+	.volatile_reg = tmp108_is_volatile_reg,
+	.val_format_endian = REGMAP_ENDIAN_BIG,
+	.cache_type = REGCACHE_RBTREE,
+	.use_single_rw = true,
+};
+
+static int tmp108_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	struct device *dev = &client->dev;
+	struct device *hwmon_dev;
+	struct tmp108 *tmp108;
+	int err;
+	u32 config;
+
+	if (!i2c_check_functionality(client->adapter,
+				     I2C_FUNC_SMBUS_WORD_DATA)) {
+		dev_err(dev,
+			"adapter doesn't support SMBus word transactions\n");
+		return -ENODEV;
+	}
+
+	tmp108 = devm_kzalloc(dev, sizeof(*tmp108), GFP_KERNEL);
+	if (!tmp108)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, tmp108);
+
+	tmp108->regmap = devm_regmap_init_i2c(client, &tmp108_regmap_config);
+	if (IS_ERR(tmp108->regmap)) {
+		err = PTR_ERR(tmp108->regmap);
+		dev_err(dev, "regmap init failed: %d", err);
+		return err;
+	}
+
+	err = regmap_read(tmp108->regmap, TMP108_REG_CONF, &config);
+	if (err < 0) {
+		dev_err(dev, "error reading config register: %d", err);
+		return err;
+	}
+	tmp108->orig_config = config;
+
+	/* Only continuous mode is supported. */
+	config &= ~TMP108_CONF_MODE_MASK;
+	config |= TMP108_MODE_CONTINUOUS;
+
+	/* Only comparator mode is supported. */
+	config &= ~TMP108_CONF_TM;
+
+	err = regmap_write(tmp108->regmap, TMP108_REG_CONF, config);
+	if (err < 0) {
+		dev_err(dev, "error writing config register: %d", err);
+		return err;
+	}
+
+	tmp108->ready_time = jiffies;
+	if ((tmp108->orig_config & TMP108_CONF_MODE_MASK) ==
+	    TMP108_MODE_SHUTDOWN)
+		tmp108->ready_time +=
+			msecs_to_jiffies(TMP108_CONVERSION_TIME_MS);
+
+	err = devm_add_action_or_reset(dev, tmp108_restore_config, tmp108);
+	if (err) {
+		dev_err(dev, "add action or reset failed: %d", err);
+		return err;
+	}
+
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name,
+							 tmp108,
+							 &tmp108_chip_info,
+							 NULL);
+	return PTR_ERR_OR_ZERO(hwmon_dev);
+}
+
+static int __maybe_unused tmp108_suspend(struct device *dev)
+{
+	struct tmp108 *tmp108 = dev_get_drvdata(dev);
+
+	return regmap_update_bits(tmp108->regmap, TMP108_REG_CONF,
+				  TMP108_CONF_MODE_MASK, TMP108_MODE_SHUTDOWN);
+}
+
+static int __maybe_unused tmp108_resume(struct device *dev)
+{
+	struct tmp108 *tmp108 = dev_get_drvdata(dev);
+	int err;
+
+	err = regmap_update_bits(tmp108->regmap, TMP108_REG_CONF,
+				 TMP108_CONF_MODE_MASK, TMP108_MODE_CONTINUOUS);
+	tmp108->ready_time = jiffies +
+			     msecs_to_jiffies(TMP108_CONVERSION_TIME_MS);
+	return err;
+}
+
+static SIMPLE_DEV_PM_OPS(tmp108_dev_pm_ops, tmp108_suspend, tmp108_resume);
+
+static const struct i2c_device_id tmp108_i2c_ids[] = {
+	{ "tmp108", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, tmp108_i2c_ids);
+
+#ifdef CONFIG_OF
+static const struct of_device_id tmp108_of_ids[] = {
+	{ .compatible = "ti,tmp108", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, tmp108_of_ids);
+#endif
+
+static struct i2c_driver tmp108_driver = {
+	.driver = {
+		.name	= DRIVER_NAME,
+		.pm	= &tmp108_dev_pm_ops,
+		.of_match_table = of_match_ptr(tmp108_of_ids),
+	},
+	.probe		= tmp108_probe,
+	.id_table	= tmp108_i2c_ids,
+};
+
+module_i2c_driver(tmp108_driver);
+
+MODULE_AUTHOR("John Muir <john@jmuir.com>");
+MODULE_DESCRIPTION("Texas Instruments TMP108 temperature sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/via-cputemp.c b/drivers/hwmon/via-cputemp.c
index ac91c07e3f90..d1f209a5feac 100644
--- a/drivers/hwmon/via-cputemp.c
+++ b/drivers/hwmon/via-cputemp.c
@@ -220,7 +220,7 @@ struct pdev_entry {
 static LIST_HEAD(pdev_list);
 static DEFINE_MUTEX(pdev_list_mutex);
 
-static int via_cputemp_device_add(unsigned int cpu)
+static int via_cputemp_online(unsigned int cpu)
 {
 	int err;
 	struct platform_device *pdev;
@@ -261,7 +261,7 @@ exit:
 	return err;
 }
 
-static void via_cputemp_device_remove(unsigned int cpu)
+static int via_cputemp_down_prep(unsigned int cpu)
 {
 	struct pdev_entry *p;
 
@@ -272,33 +272,13 @@ static void via_cputemp_device_remove(unsigned int cpu)
 			list_del(&p->list);
 			mutex_unlock(&pdev_list_mutex);
 			kfree(p);
-			return;
+			return 0;
 		}
 	}
 	mutex_unlock(&pdev_list_mutex);
+	return 0;
 }
 
-static int via_cputemp_cpu_callback(struct notifier_block *nfb,
-				    unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long) hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		via_cputemp_device_add(cpu);
-		break;
-	case CPU_DOWN_PREPARE:
-		via_cputemp_device_remove(cpu);
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block via_cputemp_cpu_notifier __refdata = {
-	.notifier_call = via_cputemp_cpu_callback,
-};
-
 static const struct x86_cpu_id __initconst cputemp_ids[] = {
 	{ X86_VENDOR_CENTAUR, 6, 0xa, }, /* C7 A */
 	{ X86_VENDOR_CENTAUR, 6, 0xd, }, /* C7 D */
@@ -307,9 +287,11 @@ static const struct x86_cpu_id __initconst cputemp_ids[] = {
 };
 MODULE_DEVICE_TABLE(x86cpu, cputemp_ids);
 
+static enum cpuhp_state via_temp_online;
+
 static int __init via_cputemp_init(void)
 {
-	int i, err;
+	int err;
 
 	if (!x86_match_cpu(cputemp_ids))
 		return -ENODEV;
@@ -318,58 +300,33 @@ static int __init via_cputemp_init(void)
 	if (err)
 		goto exit;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i) {
-		struct cpuinfo_x86 *c = &cpu_data(i);
-
-		if (c->x86 != 6)
-			continue;
-
-		if (c->x86_model < 0x0a)
-			continue;
-
-		if (c->x86_model > 0x0f) {
-			pr_warn("Unknown CPU model 0x%x\n", c->x86_model);
-			continue;
-		}
-
-		via_cputemp_device_add(i);
-	}
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hwmon/via:online",
+				via_cputemp_online, via_cputemp_down_prep);
+	if (err < 0)
+		goto exit_driver_unreg;
+	via_temp_online = err;
 
 #ifndef CONFIG_HOTPLUG_CPU
 	if (list_empty(&pdev_list)) {
-		cpu_notifier_register_done();
 		err = -ENODEV;
-		goto exit_driver_unreg;
+		goto exit_hp_unreg;
 	}
 #endif
-
-	__register_hotcpu_notifier(&via_cputemp_cpu_notifier);
-	cpu_notifier_register_done();
 	return 0;
 
 #ifndef CONFIG_HOTPLUG_CPU
+exit_hp_unreg:
+	cpuhp_remove_state_nocalls(via_temp_online);
+#endif
 exit_driver_unreg:
 	platform_driver_unregister(&via_cputemp_driver);
-#endif
 exit:
 	return err;
 }
 
 static void __exit via_cputemp_exit(void)
 {
-	struct pdev_entry *p, *n;
-
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&via_cputemp_cpu_notifier);
-	mutex_lock(&pdev_list_mutex);
-	list_for_each_entry_safe(p, n, &pdev_list, list) {
-		platform_device_unregister(p->pdev);
-		list_del(&p->list);
-		kfree(p);
-	}
-	mutex_unlock(&pdev_list_mutex);
-	cpu_notifier_register_done();
+	cpuhp_remove_state(via_temp_online);
 	platform_driver_unregister(&via_cputemp_driver);
 }
 
diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c
index 2cd7c718198a..17741969026e 100644
--- a/drivers/hwtracing/coresight/coresight-etm-perf.c
+++ b/drivers/hwtracing/coresight/coresight-etm-perf.c
@@ -202,6 +202,21 @@ static void *etm_setup_aux(int event_cpu, void **pages,
 	if (!event_data)
 		return NULL;
 
+	/*
+	 * In theory nothing prevent tracers in a trace session from being
+	 * associated with different sinks, nor having a sink per tracer.  But
+	 * until we have HW with this kind of topology we need to assume tracers
+	 * in a trace session are using the same sink.  Therefore go through
+	 * the coresight bus and pick the first enabled sink.
+	 *
+	 * When operated from sysFS users are responsible to enable the sink
+	 * while from perf, the perf tools will do it based on the choice made
+	 * on the cmd line.  As such the "enable_sink" flag in sysFS is reset.
+	 */
+	sink = coresight_get_enabled_sink(true);
+	if (!sink)
+		goto err;
+
 	INIT_WORK(&event_data->work, free_event_data);
 
 	mask = &event_data->mask;
@@ -219,25 +234,11 @@ static void *etm_setup_aux(int event_cpu, void **pages,
 		 * list of devices from source to sink that can be
 		 * referenced later when the path is actually needed.
 		 */
-		event_data->path[cpu] = coresight_build_path(csdev);
+		event_data->path[cpu] = coresight_build_path(csdev, sink);
 		if (IS_ERR(event_data->path[cpu]))
 			goto err;
 	}
 
-	/*
-	 * In theory nothing prevent tracers in a trace session from being
-	 * associated with different sinks, nor having a sink per tracer.  But
-	 * until we have HW with this kind of topology and a way to convey
-	 * sink assignement from the perf cmd line we need to assume tracers
-	 * in a trace session are using the same sink.  Therefore pick the sink
-	 * found at the end of the first available path.
-	 */
-	cpu = cpumask_first(mask);
-	/* Grab the sink at the end of the path */
-	sink = coresight_get_sink(event_data->path[cpu]);
-	if (!sink)
-		goto err;
-
 	if (!sink_ops(sink)->alloc_buffer)
 		goto err;
 
diff --git a/drivers/hwtracing/coresight/coresight-etm.h b/drivers/hwtracing/coresight/coresight-etm.h
index 4a18ee499965..ad063d7444e1 100644
--- a/drivers/hwtracing/coresight/coresight-etm.h
+++ b/drivers/hwtracing/coresight/coresight-etm.h
@@ -89,11 +89,13 @@
 /* ETMCR - 0x00 */
 #define ETMCR_PWD_DWN		BIT(0)
 #define ETMCR_STALL_MODE	BIT(7)
+#define ETMCR_BRANCH_BROADCAST	BIT(8)
 #define ETMCR_ETM_PRG		BIT(10)
 #define ETMCR_ETM_EN		BIT(11)
 #define ETMCR_CYC_ACC		BIT(12)
 #define ETMCR_CTXID_SIZE	(BIT(14)|BIT(15))
 #define ETMCR_TIMESTAMP_EN	BIT(28)
+#define ETMCR_RETURN_STACK	BIT(29)
 /* ETMCCR - 0x04 */
 #define ETMCCR_FIFOFULL		BIT(23)
 /* ETMPDCR - 0x310 */
@@ -110,8 +112,11 @@
 #define ETM_MODE_STALL		BIT(2)
 #define ETM_MODE_TIMESTAMP	BIT(3)
 #define ETM_MODE_CTXID		BIT(4)
+#define ETM_MODE_BBROAD		BIT(5)
+#define ETM_MODE_RET_STACK	BIT(6)
 #define ETM_MODE_ALL		(ETM_MODE_EXCLUDE | ETM_MODE_CYCACC | \
 				 ETM_MODE_STALL | ETM_MODE_TIMESTAMP | \
+				 ETM_MODE_BBROAD | ETM_MODE_RET_STACK | \
 				 ETM_MODE_CTXID | ETM_MODE_EXCL_KERN | \
 				 ETM_MODE_EXCL_USER)
 
diff --git a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
index e9b071953f80..ca98ad13bb8c 100644
--- a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
@@ -146,7 +146,7 @@ static ssize_t mode_store(struct device *dev,
 			goto err_unlock;
 		}
 		config->ctrl |= ETMCR_STALL_MODE;
-	 } else
+	} else
 		config->ctrl &= ~ETMCR_STALL_MODE;
 
 	if (config->mode & ETM_MODE_TIMESTAMP) {
@@ -164,6 +164,16 @@ static ssize_t mode_store(struct device *dev,
 	else
 		config->ctrl &= ~ETMCR_CTXID_SIZE;
 
+	if (config->mode & ETM_MODE_BBROAD)
+		config->ctrl |= ETMCR_BRANCH_BROADCAST;
+	else
+		config->ctrl &= ~ETMCR_BRANCH_BROADCAST;
+
+	if (config->mode & ETM_MODE_RET_STACK)
+		config->ctrl |= ETMCR_RETURN_STACK;
+	else
+		config->ctrl &= ~ETMCR_RETURN_STACK;
+
 	if (config->mode & (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER))
 		etm_config_trace_mode(config);
 
diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h
index 196a14be4b3d..ef9d8e93e3b2 100644
--- a/drivers/hwtracing/coresight/coresight-priv.h
+++ b/drivers/hwtracing/coresight/coresight-priv.h
@@ -111,7 +111,9 @@ static inline void CS_UNLOCK(void __iomem *addr)
 void coresight_disable_path(struct list_head *path);
 int coresight_enable_path(struct list_head *path, u32 mode);
 struct coresight_device *coresight_get_sink(struct list_head *path);
-struct list_head *coresight_build_path(struct coresight_device *csdev);
+struct coresight_device *coresight_get_enabled_sink(bool reset);
+struct list_head *coresight_build_path(struct coresight_device *csdev,
+				       struct coresight_device *sink);
 void coresight_release_path(struct list_head *path);
 
 #ifdef CONFIG_CORESIGHT_SOURCE_ETM3X
diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c
index 49e0f1b925a5..e4c55c5f9988 100644
--- a/drivers/hwtracing/coresight/coresight-stm.c
+++ b/drivers/hwtracing/coresight/coresight-stm.c
@@ -406,7 +406,7 @@ static long stm_generic_set_options(struct stm_data *stm_data,
 	return 0;
 }
 
-static ssize_t stm_generic_packet(struct stm_data *stm_data,
+static ssize_t notrace stm_generic_packet(struct stm_data *stm_data,
 				  unsigned int master,
 				  unsigned int channel,
 				  unsigned int packet,
@@ -419,10 +419,10 @@ static ssize_t stm_generic_packet(struct stm_data *stm_data,
 						   struct stm_drvdata, stm);
 
 	if (!(drvdata && local_read(&drvdata->mode)))
-		return 0;
+		return -EACCES;
 
 	if (channel >= drvdata->numsp)
-		return 0;
+		return -EINVAL;
 
 	ch_addr = (unsigned long)stm_channel_addr(drvdata, channel);
 
@@ -920,6 +920,11 @@ static struct amba_id stm_ids[] = {
 		.mask   = 0x0003ffff,
 		.data	= "STM32",
 	},
+	{
+		.id	= 0x0003b963,
+		.mask	= 0x0003ffff,
+		.data	= "STM500",
+	},
 	{ 0, 0},
 };
 
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c
index d6941ea24d8d..1549436e2492 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etf.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c
@@ -70,7 +70,7 @@ static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
 	 * When operating in sysFS mode the content of the buffer needs to be
 	 * read before the TMC is disabled.
 	 */
-	if (local_read(&drvdata->mode) == CS_MODE_SYSFS)
+	if (drvdata->mode == CS_MODE_SYSFS)
 		tmc_etb_dump_hw(drvdata);
 	tmc_disable_hw(drvdata);
 
@@ -103,19 +103,14 @@ static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
 	CS_LOCK(drvdata->base);
 }
 
-static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev, u32 mode)
+static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
 {
 	int ret = 0;
 	bool used = false;
 	char *buf = NULL;
-	long val;
 	unsigned long flags;
 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
-	 /* This shouldn't be happening */
-	if (WARN_ON(mode != CS_MODE_SYSFS))
-		return -EINVAL;
-
 	/*
 	 * If we don't have a buffer release the lock and allocate memory.
 	 * Otherwise keep the lock and move along.
@@ -138,13 +133,12 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev, u32 mode)
 		goto out;
 	}
 
-	val = local_xchg(&drvdata->mode, mode);
 	/*
 	 * In sysFS mode we can have multiple writers per sink.  Since this
 	 * sink is already enabled no memory is needed and the HW need not be
 	 * touched.
 	 */
-	if (val == CS_MODE_SYSFS)
+	if (drvdata->mode == CS_MODE_SYSFS)
 		goto out;
 
 	/*
@@ -163,6 +157,7 @@ static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev, u32 mode)
 		drvdata->buf = buf;
 	}
 
+	drvdata->mode = CS_MODE_SYSFS;
 	tmc_etb_enable_hw(drvdata);
 out:
 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
@@ -177,34 +172,29 @@ out:
 	return ret;
 }
 
-static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, u32 mode)
+static int tmc_enable_etf_sink_perf(struct coresight_device *csdev)
 {
 	int ret = 0;
-	long val;
 	unsigned long flags;
 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
-	 /* This shouldn't be happening */
-	if (WARN_ON(mode != CS_MODE_PERF))
-		return -EINVAL;
-
 	spin_lock_irqsave(&drvdata->spinlock, flags);
 	if (drvdata->reading) {
 		ret = -EINVAL;
 		goto out;
 	}
 
-	val = local_xchg(&drvdata->mode, mode);
 	/*
 	 * In Perf mode there can be only one writer per sink.  There
 	 * is also no need to continue if the ETB/ETR is already operated
 	 * from sysFS.
 	 */
-	if (val != CS_MODE_DISABLED) {
+	if (drvdata->mode != CS_MODE_DISABLED) {
 		ret = -EINVAL;
 		goto out;
 	}
 
+	drvdata->mode = CS_MODE_PERF;
 	tmc_etb_enable_hw(drvdata);
 out:
 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
@@ -216,9 +206,9 @@ static int tmc_enable_etf_sink(struct coresight_device *csdev, u32 mode)
 {
 	switch (mode) {
 	case CS_MODE_SYSFS:
-		return tmc_enable_etf_sink_sysfs(csdev, mode);
+		return tmc_enable_etf_sink_sysfs(csdev);
 	case CS_MODE_PERF:
-		return tmc_enable_etf_sink_perf(csdev, mode);
+		return tmc_enable_etf_sink_perf(csdev);
 	}
 
 	/* We shouldn't be here */
@@ -227,7 +217,6 @@ static int tmc_enable_etf_sink(struct coresight_device *csdev, u32 mode)
 
 static void tmc_disable_etf_sink(struct coresight_device *csdev)
 {
-	long val;
 	unsigned long flags;
 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
@@ -237,10 +226,11 @@ static void tmc_disable_etf_sink(struct coresight_device *csdev)
 		return;
 	}
 
-	val = local_xchg(&drvdata->mode, CS_MODE_DISABLED);
 	/* Disable the TMC only if it needs to */
-	if (val != CS_MODE_DISABLED)
+	if (drvdata->mode != CS_MODE_DISABLED) {
 		tmc_etb_disable_hw(drvdata);
+		drvdata->mode = CS_MODE_DISABLED;
+	}
 
 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
 
@@ -260,7 +250,7 @@ static int tmc_enable_etf_link(struct coresight_device *csdev,
 	}
 
 	tmc_etf_enable_hw(drvdata);
-	local_set(&drvdata->mode, CS_MODE_SYSFS);
+	drvdata->mode = CS_MODE_SYSFS;
 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
 
 	dev_info(drvdata->dev, "TMC-ETF enabled\n");
@@ -280,7 +270,7 @@ static void tmc_disable_etf_link(struct coresight_device *csdev,
 	}
 
 	tmc_etf_disable_hw(drvdata);
-	local_set(&drvdata->mode, CS_MODE_DISABLED);
+	drvdata->mode = CS_MODE_DISABLED;
 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
 
 	dev_info(drvdata->dev, "TMC disabled\n");
@@ -383,7 +373,7 @@ static void tmc_update_etf_buffer(struct coresight_device *csdev,
 		return;
 
 	/* This shouldn't happen */
-	if (WARN_ON_ONCE(local_read(&drvdata->mode) != CS_MODE_PERF))
+	if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF))
 		return;
 
 	CS_UNLOCK(drvdata->base);
@@ -504,7 +494,6 @@ const struct coresight_ops tmc_etf_cs_ops = {
 
 int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
 {
-	long val;
 	enum tmc_mode mode;
 	int ret = 0;
 	unsigned long flags;
@@ -528,9 +517,8 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
 		goto out;
 	}
 
-	val = local_read(&drvdata->mode);
 	/* Don't interfere if operated from Perf */
-	if (val == CS_MODE_PERF) {
+	if (drvdata->mode == CS_MODE_PERF) {
 		ret = -EINVAL;
 		goto out;
 	}
@@ -542,7 +530,7 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
 	}
 
 	/* Disable the TMC if need be */
-	if (val == CS_MODE_SYSFS)
+	if (drvdata->mode == CS_MODE_SYSFS)
 		tmc_etb_disable_hw(drvdata);
 
 	drvdata->reading = true;
@@ -573,7 +561,7 @@ int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
 	}
 
 	/* Re-enable the TMC if need be */
-	if (local_read(&drvdata->mode) == CS_MODE_SYSFS) {
+	if (drvdata->mode == CS_MODE_SYSFS) {
 		/*
 		 * The trace run will continue with the same allocated trace
 		 * buffer. As such zero-out the buffer so that we don't end
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c
index 886ea83c68e0..5d312699b3b9 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etr.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c
@@ -86,26 +86,22 @@ static void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
 	 * When operating in sysFS mode the content of the buffer needs to be
 	 * read before the TMC is disabled.
 	 */
-	if (local_read(&drvdata->mode) == CS_MODE_SYSFS)
+	if (drvdata->mode == CS_MODE_SYSFS)
 		tmc_etr_dump_hw(drvdata);
 	tmc_disable_hw(drvdata);
 
 	CS_LOCK(drvdata->base);
 }
 
-static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev, u32 mode)
+static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
 {
 	int ret = 0;
 	bool used = false;
-	long val;
 	unsigned long flags;
 	void __iomem *vaddr = NULL;
 	dma_addr_t paddr;
 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
-	 /* This shouldn't be happening */
-	if (WARN_ON(mode != CS_MODE_SYSFS))
-		return -EINVAL;
 
 	/*
 	 * If we don't have a buffer release the lock and allocate memory.
@@ -134,13 +130,12 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev, u32 mode)
 		goto out;
 	}
 
-	val = local_xchg(&drvdata->mode, mode);
 	/*
 	 * In sysFS mode we can have multiple writers per sink.  Since this
 	 * sink is already enabled no memory is needed and the HW need not be
 	 * touched.
 	 */
-	if (val == CS_MODE_SYSFS)
+	if (drvdata->mode == CS_MODE_SYSFS)
 		goto out;
 
 	/*
@@ -155,8 +150,7 @@ static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev, u32 mode)
 		drvdata->buf = drvdata->vaddr;
 	}
 
-	memset(drvdata->vaddr, 0, drvdata->size);
-
+	drvdata->mode = CS_MODE_SYSFS;
 	tmc_etr_enable_hw(drvdata);
 out:
 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
@@ -171,34 +165,29 @@ out:
 	return ret;
 }
 
-static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, u32 mode)
+static int tmc_enable_etr_sink_perf(struct coresight_device *csdev)
 {
 	int ret = 0;
-	long val;
 	unsigned long flags;
 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
-	 /* This shouldn't be happening */
-	if (WARN_ON(mode != CS_MODE_PERF))
-		return -EINVAL;
-
 	spin_lock_irqsave(&drvdata->spinlock, flags);
 	if (drvdata->reading) {
 		ret = -EINVAL;
 		goto out;
 	}
 
-	val = local_xchg(&drvdata->mode, mode);
 	/*
 	 * In Perf mode there can be only one writer per sink.  There
 	 * is also no need to continue if the ETR is already operated
 	 * from sysFS.
 	 */
-	if (val != CS_MODE_DISABLED) {
+	if (drvdata->mode != CS_MODE_DISABLED) {
 		ret = -EINVAL;
 		goto out;
 	}
 
+	drvdata->mode = CS_MODE_PERF;
 	tmc_etr_enable_hw(drvdata);
 out:
 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
@@ -210,9 +199,9 @@ static int tmc_enable_etr_sink(struct coresight_device *csdev, u32 mode)
 {
 	switch (mode) {
 	case CS_MODE_SYSFS:
-		return tmc_enable_etr_sink_sysfs(csdev, mode);
+		return tmc_enable_etr_sink_sysfs(csdev);
 	case CS_MODE_PERF:
-		return tmc_enable_etr_sink_perf(csdev, mode);
+		return tmc_enable_etr_sink_perf(csdev);
 	}
 
 	/* We shouldn't be here */
@@ -221,7 +210,6 @@ static int tmc_enable_etr_sink(struct coresight_device *csdev, u32 mode)
 
 static void tmc_disable_etr_sink(struct coresight_device *csdev)
 {
-	long val;
 	unsigned long flags;
 	struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
@@ -231,10 +219,11 @@ static void tmc_disable_etr_sink(struct coresight_device *csdev)
 		return;
 	}
 
-	val = local_xchg(&drvdata->mode, CS_MODE_DISABLED);
 	/* Disable the TMC only if it needs to */
-	if (val != CS_MODE_DISABLED)
+	if (drvdata->mode != CS_MODE_DISABLED) {
 		tmc_etr_disable_hw(drvdata);
+		drvdata->mode = CS_MODE_DISABLED;
+	}
 
 	spin_unlock_irqrestore(&drvdata->spinlock, flags);
 
@@ -253,7 +242,6 @@ const struct coresight_ops tmc_etr_cs_ops = {
 int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
 {
 	int ret = 0;
-	long val;
 	unsigned long flags;
 
 	/* config types are set a boot time and never change */
@@ -266,9 +254,8 @@ int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
 		goto out;
 	}
 
-	val = local_read(&drvdata->mode);
 	/* Don't interfere if operated from Perf */
-	if (val == CS_MODE_PERF) {
+	if (drvdata->mode == CS_MODE_PERF) {
 		ret = -EINVAL;
 		goto out;
 	}
@@ -280,7 +267,7 @@ int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
 	}
 
 	/* Disable the TMC if need be */
-	if (val == CS_MODE_SYSFS)
+	if (drvdata->mode == CS_MODE_SYSFS)
 		tmc_etr_disable_hw(drvdata);
 
 	drvdata->reading = true;
@@ -303,7 +290,7 @@ int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
 	spin_lock_irqsave(&drvdata->spinlock, flags);
 
 	/* RE-enable the TMC if need be */
-	if (local_read(&drvdata->mode) == CS_MODE_SYSFS) {
+	if (drvdata->mode == CS_MODE_SYSFS) {
 		/*
 		 * The trace run will continue with the same allocated trace
 		 * buffer. The trace buffer is cleared in tmc_etr_enable_hw(),
diff --git a/drivers/hwtracing/coresight/coresight-tmc.h b/drivers/hwtracing/coresight/coresight-tmc.h
index 44b3ae346118..51c01851533e 100644
--- a/drivers/hwtracing/coresight/coresight-tmc.h
+++ b/drivers/hwtracing/coresight/coresight-tmc.h
@@ -117,7 +117,7 @@ struct tmc_drvdata {
 	void __iomem		*vaddr;
 	u32			size;
 	u32			len;
-	local_t			mode;
+	u32			mode;
 	enum tmc_config_type	config_type;
 	enum tmc_mem_intf_width	memwidth;
 	u32			trigger_cntr;
diff --git a/drivers/hwtracing/coresight/coresight.c b/drivers/hwtracing/coresight/coresight.c
index 7bf00a0beb6f..0c37356e417c 100644
--- a/drivers/hwtracing/coresight/coresight.c
+++ b/drivers/hwtracing/coresight/coresight.c
@@ -368,6 +368,52 @@ struct coresight_device *coresight_get_sink(struct list_head *path)
 	return csdev;
 }
 
+static int coresight_enabled_sink(struct device *dev, void *data)
+{
+	bool *reset = data;
+	struct coresight_device *csdev = to_coresight_device(dev);
+
+	if ((csdev->type == CORESIGHT_DEV_TYPE_SINK ||
+	     csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) &&
+	     csdev->activated) {
+		/*
+		 * Now that we have a handle on the sink for this session,
+		 * disable the sysFS "enable_sink" flag so that possible
+		 * concurrent perf session that wish to use another sink don't
+		 * trip on it.  Doing so has no ramification for the current
+		 * session.
+		 */
+		if (*reset)
+			csdev->activated = false;
+
+		return 1;
+	}
+
+	return 0;
+}
+
+/**
+ * coresight_get_enabled_sink - returns the first enabled sink found on the bus
+ * @deactivate:	Whether the 'enable_sink' flag should be reset
+ *
+ * When operated from perf the deactivate parameter should be set to 'true'.
+ * That way the "enabled_sink" flag of the sink that was selected can be reset,
+ * allowing for other concurrent perf sessions to choose a different sink.
+ *
+ * When operated from sysFS users have full control and as such the deactivate
+ * parameter should be set to 'false', hence mandating users to explicitly
+ * clear the flag.
+ */
+struct coresight_device *coresight_get_enabled_sink(bool deactivate)
+{
+	struct device *dev = NULL;
+
+	dev = bus_find_device(&coresight_bustype, NULL, &deactivate,
+			      coresight_enabled_sink);
+
+	return dev ? to_coresight_device(dev) : NULL;
+}
+
 /**
  * _coresight_build_path - recursively build a path from a @csdev to a sink.
  * @csdev:	The device to start from.
@@ -380,6 +426,7 @@ struct coresight_device *coresight_get_sink(struct list_head *path)
  * last one.
  */
 static int _coresight_build_path(struct coresight_device *csdev,
+				 struct coresight_device *sink,
 				 struct list_head *path)
 {
 	int i;
@@ -387,15 +434,15 @@ static int _coresight_build_path(struct coresight_device *csdev,
 	struct coresight_node *node;
 
 	/* An activated sink has been found.  Enqueue the element */
-	if ((csdev->type == CORESIGHT_DEV_TYPE_SINK ||
-	     csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) && csdev->activated)
+	if (csdev == sink)
 		goto out;
 
 	/* Not a sink - recursively explore each port found on this element */
 	for (i = 0; i < csdev->nr_outport; i++) {
 		struct coresight_device *child_dev = csdev->conns[i].child_dev;
 
-		if (child_dev && _coresight_build_path(child_dev, path) == 0) {
+		if (child_dev &&
+		    _coresight_build_path(child_dev, sink, path) == 0) {
 			found = true;
 			break;
 		}
@@ -422,18 +469,22 @@ out:
 	return 0;
 }
 
-struct list_head *coresight_build_path(struct coresight_device *csdev)
+struct list_head *coresight_build_path(struct coresight_device *source,
+				       struct coresight_device *sink)
 {
 	struct list_head *path;
 	int rc;
 
+	if (!sink)
+		return ERR_PTR(-EINVAL);
+
 	path = kzalloc(sizeof(struct list_head), GFP_KERNEL);
 	if (!path)
 		return ERR_PTR(-ENOMEM);
 
 	INIT_LIST_HEAD(path);
 
-	rc = _coresight_build_path(csdev, path);
+	rc = _coresight_build_path(source, sink, path);
 	if (rc) {
 		kfree(path);
 		return ERR_PTR(rc);
@@ -497,6 +548,7 @@ static int coresight_validate_source(struct coresight_device *csdev,
 int coresight_enable(struct coresight_device *csdev)
 {
 	int cpu, ret = 0;
+	struct coresight_device *sink;
 	struct list_head *path;
 
 	mutex_lock(&coresight_mutex);
@@ -508,7 +560,17 @@ int coresight_enable(struct coresight_device *csdev)
 	if (csdev->enable)
 		goto out;
 
-	path = coresight_build_path(csdev);
+	/*
+	 * Search for a valid sink for this session but don't reset the
+	 * "enable_sink" flag in sysFS.  Users get to do that explicitly.
+	 */
+	sink = coresight_get_enabled_sink(false);
+	if (!sink) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	path = coresight_build_path(csdev, sink);
 	if (IS_ERR(path)) {
 		pr_err("building path(s) failed\n");
 		ret = PTR_ERR(path);
diff --git a/drivers/hwtracing/intel_th/core.c b/drivers/hwtracing/intel_th/core.c
index 6f0a51a2c6ec..cdd9b3b26195 100644
--- a/drivers/hwtracing/intel_th/core.c
+++ b/drivers/hwtracing/intel_th/core.c
@@ -29,6 +29,9 @@
 #include "intel_th.h"
 #include "debug.h"
 
+static bool host_mode __read_mostly;
+module_param(host_mode, bool, 0444);
+
 static DEFINE_IDA(intel_th_ida);
 
 static int intel_th_match(struct device *dev, struct device_driver *driver)
@@ -380,7 +383,7 @@ static void intel_th_device_free(struct intel_th_device *thdev)
 /*
  * Intel(R) Trace Hub subdevices
  */
-static struct intel_th_subdevice {
+static const struct intel_th_subdevice {
 	const char		*name;
 	struct resource		res[3];
 	unsigned		nres;
@@ -527,14 +530,19 @@ static int intel_th_populate(struct intel_th *th, struct resource *devres,
 {
 	struct resource res[3];
 	unsigned int req = 0;
-	int i, err;
+	int src, dst, err;
 
 	/* create devices for each intel_th_subdevice */
-	for (i = 0; i < ARRAY_SIZE(intel_th_subdevices); i++) {
-		struct intel_th_subdevice *subdev = &intel_th_subdevices[i];
+	for (src = 0, dst = 0; src < ARRAY_SIZE(intel_th_subdevices); src++) {
+		const struct intel_th_subdevice *subdev =
+			&intel_th_subdevices[src];
 		struct intel_th_device *thdev;
 		int r;
 
+		/* only allow SOURCE and SWITCH devices in host mode */
+		if (host_mode && subdev->type == INTEL_TH_OUTPUT)
+			continue;
+
 		thdev = intel_th_device_alloc(th, subdev->type, subdev->name,
 					      subdev->id);
 		if (!thdev) {
@@ -577,10 +585,12 @@ static int intel_th_populate(struct intel_th *th, struct resource *devres,
 		}
 
 		if (subdev->type == INTEL_TH_OUTPUT) {
-			thdev->dev.devt = MKDEV(th->major, i);
+			thdev->dev.devt = MKDEV(th->major, dst);
 			thdev->output.type = subdev->otype;
 			thdev->output.port = -1;
 			thdev->output.scratchpad = subdev->scrpd;
+		} else if (subdev->type == INTEL_TH_SWITCH) {
+			thdev->host_mode = host_mode;
 		}
 
 		err = device_add(&thdev->dev);
@@ -597,14 +607,14 @@ static int intel_th_populate(struct intel_th *th, struct resource *devres,
 				req++;
 		}
 
-		th->thdev[i] = thdev;
+		th->thdev[dst++] = thdev;
 	}
 
 	return 0;
 
 kill_subdevs:
-	for (i-- ; i >= 0; i--)
-		intel_th_device_remove(th->thdev[i]);
+	for (; dst >= 0; dst--)
+		intel_th_device_remove(th->thdev[dst]);
 
 	return err;
 }
@@ -717,7 +727,7 @@ void intel_th_free(struct intel_th *th)
 
 	intel_th_request_hub_module_flush(th);
 	for (i = 0; i < TH_SUBDEVICE_MAX; i++)
-		if (th->thdev[i] != th->hub)
+		if (th->thdev[i] && th->thdev[i] != th->hub)
 			intel_th_device_remove(th->thdev[i]);
 
 	intel_th_device_remove(th->hub);
diff --git a/drivers/hwtracing/intel_th/gth.c b/drivers/hwtracing/intel_th/gth.c
index 33e09369a491..dd32d0bad687 100644
--- a/drivers/hwtracing/intel_th/gth.c
+++ b/drivers/hwtracing/intel_th/gth.c
@@ -564,6 +564,9 @@ static int intel_th_gth_assign(struct intel_th_device *thdev,
 	struct gth_device *gth = dev_get_drvdata(&thdev->dev);
 	int i, id;
 
+	if (thdev->host_mode)
+		return -EBUSY;
+
 	if (othdev->type != INTEL_TH_OUTPUT)
 		return -EINVAL;
 
@@ -600,6 +603,9 @@ static void intel_th_gth_unassign(struct intel_th_device *thdev,
 	struct gth_device *gth = dev_get_drvdata(&thdev->dev);
 	int port = othdev->output.port;
 
+	if (thdev->host_mode)
+		return;
+
 	spin_lock(&gth->gth_lock);
 	othdev->output.port = -1;
 	othdev->output.active = false;
@@ -654,9 +660,24 @@ static int intel_th_gth_probe(struct intel_th_device *thdev)
 	gth->base = base;
 	spin_lock_init(&gth->gth_lock);
 
+	/*
+	 * Host mode can be signalled via SW means or via SCRPD_DEBUGGER_IN_USE
+	 * bit. Either way, don't reset HW in this case, and don't export any
+	 * capture configuration attributes. Also, refuse to assign output
+	 * drivers to ports, see intel_th_gth_assign().
+	 */
+	if (thdev->host_mode)
+		goto done;
+
 	ret = intel_th_gth_reset(gth);
-	if (ret)
-		return ret;
+	if (ret) {
+		if (ret != -EBUSY)
+			return ret;
+
+		thdev->host_mode = true;
+
+		goto done;
+	}
 
 	for (i = 0; i < TH_CONFIGURABLE_MASTERS + 1; i++)
 		gth->master[i] = -1;
@@ -677,6 +698,7 @@ static int intel_th_gth_probe(struct intel_th_device *thdev)
 		return -ENOMEM;
 	}
 
+done:
 	dev_set_drvdata(dev, gth);
 
 	return 0;
diff --git a/drivers/hwtracing/intel_th/intel_th.h b/drivers/hwtracing/intel_th/intel_th.h
index 4c195786bf1f..3096e7054f6d 100644
--- a/drivers/hwtracing/intel_th/intel_th.h
+++ b/drivers/hwtracing/intel_th/intel_th.h
@@ -54,6 +54,7 @@ struct intel_th_output {
  * @num_resources:	number of resources in @resource array
  * @type:		INTEL_TH_{SOURCE,OUTPUT,SWITCH}
  * @id:			device instance or -1
+ * @host_mode:		Intel TH is controlled by an external debug host
  * @output:		output descriptor for INTEL_TH_OUTPUT devices
  * @name:		device name to match the driver
  */
@@ -64,6 +65,9 @@ struct intel_th_device {
 	unsigned int	type;
 	int		id;
 
+	/* INTEL_TH_SWITCH specific */
+	bool			host_mode;
+
 	/* INTEL_TH_OUTPUT specific */
 	struct intel_th_output	output;
 
diff --git a/drivers/hwtracing/intel_th/sth.c b/drivers/hwtracing/intel_th/sth.c
index e1aee61dd7b3..b03444624648 100644
--- a/drivers/hwtracing/intel_th/sth.c
+++ b/drivers/hwtracing/intel_th/sth.c
@@ -67,10 +67,13 @@ static void sth_iowrite(void __iomem *dest, const unsigned char *payload,
 	}
 }
 
-static ssize_t sth_stm_packet(struct stm_data *stm_data, unsigned int master,
-			      unsigned int channel, unsigned int packet,
-			      unsigned int flags, unsigned int size,
-			      const unsigned char *payload)
+static ssize_t notrace sth_stm_packet(struct stm_data *stm_data,
+				      unsigned int master,
+				      unsigned int channel,
+				      unsigned int packet,
+				      unsigned int flags,
+				      unsigned int size,
+				      const unsigned char *payload)
 {
 	struct sth_device *sth = container_of(stm_data, struct sth_device, stm);
 	struct intel_th_channel __iomem *out =
diff --git a/drivers/hwtracing/stm/Kconfig b/drivers/hwtracing/stm/Kconfig
index 847a39b35307..723e2d90083d 100644
--- a/drivers/hwtracing/stm/Kconfig
+++ b/drivers/hwtracing/stm/Kconfig
@@ -39,4 +39,15 @@ config STM_SOURCE_HEARTBEAT
 	  If you want to send heartbeat messages over STM devices,
 	  say Y.
 
+config STM_SOURCE_FTRACE
+	tristate "Copy the output from kernel Ftrace to STM engine"
+	depends on FUNCTION_TRACER
+	help
+	  This option can be used to copy the output from kernel Ftrace
+	  to STM engine. Enabling this option will introduce a slight
+	  timing effect.
+
+	  If you want to send kernel Ftrace messages over STM devices,
+	  say Y.
+
 endif
diff --git a/drivers/hwtracing/stm/Makefile b/drivers/hwtracing/stm/Makefile
index a9ce3d487e57..3abd84ce13d4 100644
--- a/drivers/hwtracing/stm/Makefile
+++ b/drivers/hwtracing/stm/Makefile
@@ -6,6 +6,8 @@ obj-$(CONFIG_STM_DUMMY)	+= dummy_stm.o
 
 obj-$(CONFIG_STM_SOURCE_CONSOLE)	+= stm_console.o
 obj-$(CONFIG_STM_SOURCE_HEARTBEAT)	+= stm_heartbeat.o
+obj-$(CONFIG_STM_SOURCE_FTRACE)		+= stm_ftrace.o
 
 stm_console-y		:= console.o
 stm_heartbeat-y		:= heartbeat.o
+stm_ftrace-y		:= ftrace.o
diff --git a/drivers/hwtracing/stm/core.c b/drivers/hwtracing/stm/core.c
index 51f81d64ca37..0e731143f6a4 100644
--- a/drivers/hwtracing/stm/core.c
+++ b/drivers/hwtracing/stm/core.c
@@ -361,7 +361,7 @@ static int stm_char_open(struct inode *inode, struct file *file)
 	struct stm_file *stmf;
 	struct device *dev;
 	unsigned int major = imajor(inode);
-	int err = -ENODEV;
+	int err = -ENOMEM;
 
 	dev = class_find_device(&stm_class, NULL, &major, major_match);
 	if (!dev)
@@ -369,8 +369,9 @@ static int stm_char_open(struct inode *inode, struct file *file)
 
 	stmf = kzalloc(sizeof(*stmf), GFP_KERNEL);
 	if (!stmf)
-		return -ENOMEM;
+		goto err_put_device;
 
+	err = -ENODEV;
 	stm_output_init(&stmf->output);
 	stmf->stm = to_stm_device(dev);
 
@@ -382,9 +383,10 @@ static int stm_char_open(struct inode *inode, struct file *file)
 	return nonseekable_open(inode, file);
 
 err_free:
+	kfree(stmf);
+err_put_device:
 	/* matches class_find_device() above */
 	put_device(dev);
-	kfree(stmf);
 
 	return err;
 }
@@ -425,7 +427,7 @@ static int stm_file_assign(struct stm_file *stmf, char *id, unsigned int width)
 	return ret;
 }
 
-static ssize_t stm_write(struct stm_data *data, unsigned int master,
+static ssize_t notrace stm_write(struct stm_data *data, unsigned int master,
 			  unsigned int channel, const char *buf, size_t count)
 {
 	unsigned int flags = STP_PACKET_TIMESTAMPED;
@@ -1121,8 +1123,9 @@ void stm_source_unregister_device(struct stm_source_data *data)
 }
 EXPORT_SYMBOL_GPL(stm_source_unregister_device);
 
-int stm_source_write(struct stm_source_data *data, unsigned int chan,
-		     const char *buf, size_t count)
+int notrace stm_source_write(struct stm_source_data *data,
+			     unsigned int chan,
+			     const char *buf, size_t count)
 {
 	struct stm_source_device *src = data->src;
 	struct stm_device *stm;
diff --git a/drivers/hwtracing/stm/dummy_stm.c b/drivers/hwtracing/stm/dummy_stm.c
index a86612d989f9..c5f94ca31c4d 100644
--- a/drivers/hwtracing/stm/dummy_stm.c
+++ b/drivers/hwtracing/stm/dummy_stm.c
@@ -21,7 +21,7 @@
 #include <linux/slab.h>
 #include <linux/stm.h>
 
-static ssize_t
+static ssize_t notrace
 dummy_stm_packet(struct stm_data *stm_data, unsigned int master,
 		 unsigned int channel, unsigned int packet, unsigned int flags,
 		 unsigned int size, const unsigned char *payload)
diff --git a/drivers/hwtracing/stm/ftrace.c b/drivers/hwtracing/stm/ftrace.c
new file mode 100644
index 000000000000..bd126a7c6da2
--- /dev/null
+++ b/drivers/hwtracing/stm/ftrace.c
@@ -0,0 +1,87 @@
+/*
+ * Simple kernel driver to link kernel Ftrace and an STM device
+ * Copyright (c) 2016, Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * STM Ftrace will be registered as a trace_export.
+ */
+
+#include <linux/module.h>
+#include <linux/stm.h>
+#include <linux/trace.h>
+
+#define STM_FTRACE_NR_CHANNELS 1
+#define STM_FTRACE_CHAN 0
+
+static int stm_ftrace_link(struct stm_source_data *data);
+static void stm_ftrace_unlink(struct stm_source_data *data);
+
+static struct stm_ftrace {
+	struct stm_source_data	data;
+	struct trace_export	ftrace;
+} stm_ftrace = {
+	.data	= {
+		.name		= "ftrace",
+		.nr_chans	= STM_FTRACE_NR_CHANNELS,
+		.link		= stm_ftrace_link,
+		.unlink		= stm_ftrace_unlink,
+	},
+};
+
+/**
+ * stm_ftrace_write() - write data to STM via 'stm_ftrace' source
+ * @buf:	buffer containing the data packet
+ * @len:	length of the data packet
+ */
+static void notrace
+stm_ftrace_write(const void *buf, unsigned int len)
+{
+	stm_source_write(&stm_ftrace.data, STM_FTRACE_CHAN, buf, len);
+}
+
+static int stm_ftrace_link(struct stm_source_data *data)
+{
+	struct stm_ftrace *sf = container_of(data, struct stm_ftrace, data);
+
+	sf->ftrace.write = stm_ftrace_write;
+
+	return register_ftrace_export(&sf->ftrace);
+}
+
+static void stm_ftrace_unlink(struct stm_source_data *data)
+{
+	struct stm_ftrace *sf = container_of(data, struct stm_ftrace, data);
+
+	unregister_ftrace_export(&sf->ftrace);
+}
+
+static int __init stm_ftrace_init(void)
+{
+	int ret;
+
+	ret = stm_source_register_device(NULL, &stm_ftrace.data);
+	if (ret)
+		pr_err("Failed to register stm_source - ftrace.\n");
+
+	return ret;
+}
+
+static void __exit stm_ftrace_exit(void)
+{
+	stm_source_unregister_device(&stm_ftrace.data);
+}
+
+module_init(stm_ftrace_init);
+module_exit(stm_ftrace_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("stm_ftrace driver");
+MODULE_AUTHOR("Chunyan Zhang <zhang.chunyan@linaro.org>");
diff --git a/drivers/i2c/Kconfig b/drivers/i2c/Kconfig
index 11edabf425ae..efc3354d60ae 100644
--- a/drivers/i2c/Kconfig
+++ b/drivers/i2c/Kconfig
@@ -7,6 +7,7 @@ menu "I2C support"
 config I2C
 	tristate "I2C support"
 	select RT_MUTEXES
+	select IRQ_DOMAIN
 	---help---
 	  I2C (pronounce: I-squared-C) is a slow serial bus protocol used in
 	  many micro controller applications and developed by Philips.  SMBus,
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index d252276feadf..0cdc8443deab 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -426,7 +426,7 @@ config I2C_BLACKFIN_TWI_CLK_KHZ
 
 config I2C_CADENCE
 	tristate "Cadence I2C Controller"
-	depends on ARCH_ZYNQ || ARM64
+	depends on ARCH_ZYNQ || ARM64 || XTENSA
 	help
 	  Say yes here to select Cadence I2C Host Controller. This controller is
 	  e.g. used by Xilinx Zynq.
@@ -597,6 +597,16 @@ config I2C_IMX
 	  This driver can also be built as a module.  If so, the module
 	  will be called i2c-imx.
 
+config I2C_IMX_LPI2C
+	tristate "IMX Low Power I2C interface"
+	depends on ARCH_MXC || COMPILE_TEST
+	help
+          Say Y here if you want to use the Low Power IIC bus controller
+          on the Freescale i.MX processors.
+
+          This driver can also be built as a module. If so, the module
+          will be called i2c-imx-lpi2c.
+
 config I2C_IOP3XX
 	tristate "Intel IOPx3xx and IXP4xx on-chip I2C interface"
 	depends on ARCH_IOP32X || ARCH_IOP33X || ARCH_IXP4XX || ARCH_IOP13XX
@@ -763,7 +773,7 @@ config I2C_PUV3
 
 config I2C_PXA
 	tristate "Intel PXA2XX I2C adapter"
-	depends on ARCH_PXA || ARCH_MMP || (X86_32 && PCI && OF)
+	depends on ARCH_PXA || ARCH_MMP || ARCH_MVEBU || (X86_32 && PCI && OF)
 	help
 	  If you have devices in the PXA I2C bus, say yes to this option.
 	  This driver can also be built as a module.  If so, the module
@@ -1150,6 +1160,17 @@ config I2C_ELEKTOR
 	  This support is also available as a module.  If so, the module
 	  will be called i2c-elektor.
 
+config I2C_MLXCPLD
+	tristate "Mellanox I2C driver"
+	depends on X86_64
+	help
+	  This exposes the Mellanox platform I2C busses to the linux I2C layer
+	  for X86 based systems.
+	  Controller is implemented as CPLD logic.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called as i2c-mlxcpld.
+
 config I2C_PCA_ISA
 	tristate "PCA9564/PCA9665 on an ISA bus"
 	depends on ISA
diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile
index 29764cc20a44..1c1bac87a9db 100644
--- a/drivers/i2c/busses/Makefile
+++ b/drivers/i2c/busses/Makefile
@@ -56,6 +56,7 @@ obj-$(CONFIG_I2C_HIX5HD2)	+= i2c-hix5hd2.o
 obj-$(CONFIG_I2C_IBM_IIC)	+= i2c-ibm_iic.o
 obj-$(CONFIG_I2C_IMG)		+= i2c-img-scb.o
 obj-$(CONFIG_I2C_IMX)		+= i2c-imx.o
+obj-$(CONFIG_I2C_IMX_LPI2C)	+= i2c-imx-lpi2c.o
 obj-$(CONFIG_I2C_IOP3XX)	+= i2c-iop3xx.o
 obj-$(CONFIG_I2C_JZ4780)	+= i2c-jz4780.o
 obj-$(CONFIG_I2C_KEMPLD)	+= i2c-kempld.o
@@ -116,6 +117,7 @@ obj-$(CONFIG_I2C_BCM_KONA)	+= i2c-bcm-kona.o
 obj-$(CONFIG_I2C_BRCMSTB)	+= i2c-brcmstb.o
 obj-$(CONFIG_I2C_CROS_EC_TUNNEL)	+= i2c-cros-ec-tunnel.o
 obj-$(CONFIG_I2C_ELEKTOR)	+= i2c-elektor.o
+obj-$(CONFIG_I2C_MLXCPLD)	+= i2c-mlxcpld.o
 obj-$(CONFIG_I2C_OPAL)		+= i2c-opal.o
 obj-$(CONFIG_I2C_PCA_ISA)	+= i2c-pca-isa.o
 obj-$(CONFIG_I2C_SIBYTE)	+= i2c-sibyte.o
diff --git a/drivers/i2c/busses/i2c-axxia.c b/drivers/i2c/busses/i2c-axxia.c
index 4351a9343058..13f07482ec68 100644
--- a/drivers/i2c/busses/i2c-axxia.c
+++ b/drivers/i2c/busses/i2c-axxia.c
@@ -489,7 +489,7 @@ static const struct i2c_algorithm axxia_i2c_algo = {
 	.functionality = axxia_i2c_func,
 };
 
-static struct i2c_adapter_quirks axxia_i2c_quirks = {
+static const struct i2c_adapter_quirks axxia_i2c_quirks = {
 	.max_read_len = 255,
 	.max_write_len = 255,
 };
diff --git a/drivers/i2c/busses/i2c-bcm-iproc.c b/drivers/i2c/busses/i2c-bcm-iproc.c
index 326b3db02c48..318df559adc5 100644
--- a/drivers/i2c/busses/i2c-bcm-iproc.c
+++ b/drivers/i2c/busses/i2c-bcm-iproc.c
@@ -395,7 +395,7 @@ static const struct i2c_algorithm bcm_iproc_algo = {
 	.functionality = bcm_iproc_i2c_functionality,
 };
 
-static struct i2c_adapter_quirks bcm_iproc_i2c_quirks = {
+static const struct i2c_adapter_quirks bcm_iproc_i2c_quirks = {
 	/* need to reserve one byte in the FIFO for the slave address */
 	.max_read_len = M_TX_RX_FIFO_SIZE - 1,
 };
diff --git a/drivers/i2c/busses/i2c-bcm2835.c b/drivers/i2c/busses/i2c-bcm2835.c
index d4f3239b5686..c3436f627028 100644
--- a/drivers/i2c/busses/i2c-bcm2835.c
+++ b/drivers/i2c/busses/i2c-bcm2835.c
@@ -50,20 +50,19 @@
 #define BCM2835_I2C_S_CLKT	BIT(9)
 #define BCM2835_I2C_S_LEN	BIT(10) /* Fake bit for SW error reporting */
 
-#define BCM2835_I2C_BITMSK_S	0x03FF
-
 #define BCM2835_I2C_CDIV_MIN	0x0002
 #define BCM2835_I2C_CDIV_MAX	0xFFFE
 
-#define BCM2835_I2C_TIMEOUT (msecs_to_jiffies(1000))
-
 struct bcm2835_i2c_dev {
 	struct device *dev;
 	void __iomem *regs;
 	struct clk *clk;
 	int irq;
+	u32 bus_clk_rate;
 	struct i2c_adapter adapter;
 	struct completion completion;
+	struct i2c_msg *curr_msg;
+	int num_msgs;
 	u32 msg_err;
 	u8 *msg_buf;
 	size_t msg_buf_remaining;
@@ -80,6 +79,30 @@ static inline u32 bcm2835_i2c_readl(struct bcm2835_i2c_dev *i2c_dev, u32 reg)
 	return readl(i2c_dev->regs + reg);
 }
 
+static int bcm2835_i2c_set_divider(struct bcm2835_i2c_dev *i2c_dev)
+{
+	u32 divider;
+
+	divider = DIV_ROUND_UP(clk_get_rate(i2c_dev->clk),
+			       i2c_dev->bus_clk_rate);
+	/*
+	 * Per the datasheet, the register is always interpreted as an even
+	 * number, by rounding down. In other words, the LSB is ignored. So,
+	 * if the LSB is set, increment the divider to avoid any issue.
+	 */
+	if (divider & 1)
+		divider++;
+	if ((divider < BCM2835_I2C_CDIV_MIN) ||
+	    (divider > BCM2835_I2C_CDIV_MAX)) {
+		dev_err_ratelimited(i2c_dev->dev, "Invalid clock-frequency\n");
+		return -EINVAL;
+	}
+
+	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_DIV, divider);
+
+	return 0;
+}
+
 static void bcm2835_fill_txfifo(struct bcm2835_i2c_dev *i2c_dev)
 {
 	u32 val;
@@ -110,106 +133,159 @@ static void bcm2835_drain_rxfifo(struct bcm2835_i2c_dev *i2c_dev)
 	}
 }
 
+/*
+ * Repeated Start Condition (Sr)
+ * The BCM2835 ARM Peripherals datasheet mentions a way to trigger a Sr when it
+ * talks about reading from a slave with 10 bit address. This is achieved by
+ * issuing a write, poll the I2CS.TA flag and wait for it to be set, and then
+ * issue a read.
+ * A comment in https://github.com/raspberrypi/linux/issues/254 shows how the
+ * firmware actually does it using polling and says that it's a workaround for
+ * a problem in the state machine.
+ * It turns out that it is possible to use the TXW interrupt to know when the
+ * transfer is active, provided the FIFO has not been prefilled.
+ */
+
+static void bcm2835_i2c_start_transfer(struct bcm2835_i2c_dev *i2c_dev)
+{
+	u32 c = BCM2835_I2C_C_ST | BCM2835_I2C_C_I2CEN;
+	struct i2c_msg *msg = i2c_dev->curr_msg;
+	bool last_msg = (i2c_dev->num_msgs == 1);
+
+	if (!i2c_dev->num_msgs)
+		return;
+
+	i2c_dev->num_msgs--;
+	i2c_dev->msg_buf = msg->buf;
+	i2c_dev->msg_buf_remaining = msg->len;
+
+	if (msg->flags & I2C_M_RD)
+		c |= BCM2835_I2C_C_READ | BCM2835_I2C_C_INTR;
+	else
+		c |= BCM2835_I2C_C_INTT;
+
+	if (last_msg)
+		c |= BCM2835_I2C_C_INTD;
+
+	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_A, msg->addr);
+	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_DLEN, msg->len);
+	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_C, c);
+}
+
+/*
+ * Note about I2C_C_CLEAR on error:
+ * The I2C_C_CLEAR on errors will take some time to resolve -- if you were in
+ * non-idle state and I2C_C_READ, it sets an abort_rx flag and runs through
+ * the state machine to send a NACK and a STOP. Since we're setting CLEAR
+ * without I2CEN, that NACK will be hanging around queued up for next time
+ * we start the engine.
+ */
+
 static irqreturn_t bcm2835_i2c_isr(int this_irq, void *data)
 {
 	struct bcm2835_i2c_dev *i2c_dev = data;
 	u32 val, err;
 
 	val = bcm2835_i2c_readl(i2c_dev, BCM2835_I2C_S);
-	val &= BCM2835_I2C_BITMSK_S;
-	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_S, val);
 
 	err = val & (BCM2835_I2C_S_CLKT | BCM2835_I2C_S_ERR);
 	if (err) {
 		i2c_dev->msg_err = err;
-		complete(&i2c_dev->completion);
-		return IRQ_HANDLED;
-	}
-
-	if (val & BCM2835_I2C_S_RXD) {
-		bcm2835_drain_rxfifo(i2c_dev);
-		if (!(val & BCM2835_I2C_S_DONE))
-			return IRQ_HANDLED;
+		goto complete;
 	}
 
 	if (val & BCM2835_I2C_S_DONE) {
-		if (i2c_dev->msg_buf_remaining)
+		if (i2c_dev->curr_msg->flags & I2C_M_RD) {
+			bcm2835_drain_rxfifo(i2c_dev);
+			val = bcm2835_i2c_readl(i2c_dev, BCM2835_I2C_S);
+		}
+
+		if ((val & BCM2835_I2C_S_RXD) || i2c_dev->msg_buf_remaining)
 			i2c_dev->msg_err = BCM2835_I2C_S_LEN;
 		else
 			i2c_dev->msg_err = 0;
-		complete(&i2c_dev->completion);
-		return IRQ_HANDLED;
+		goto complete;
 	}
 
-	if (val & BCM2835_I2C_S_TXD) {
+	if (val & BCM2835_I2C_S_TXW) {
+		if (!i2c_dev->msg_buf_remaining) {
+			i2c_dev->msg_err = val | BCM2835_I2C_S_LEN;
+			goto complete;
+		}
+
 		bcm2835_fill_txfifo(i2c_dev);
+
+		if (i2c_dev->num_msgs && !i2c_dev->msg_buf_remaining) {
+			i2c_dev->curr_msg++;
+			bcm2835_i2c_start_transfer(i2c_dev);
+		}
+
+		return IRQ_HANDLED;
+	}
+
+	if (val & BCM2835_I2C_S_RXR) {
+		if (!i2c_dev->msg_buf_remaining) {
+			i2c_dev->msg_err = val | BCM2835_I2C_S_LEN;
+			goto complete;
+		}
+
+		bcm2835_drain_rxfifo(i2c_dev);
 		return IRQ_HANDLED;
 	}
 
 	return IRQ_NONE;
+
+complete:
+	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_C, BCM2835_I2C_C_CLEAR);
+	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_S, BCM2835_I2C_S_CLKT |
+			   BCM2835_I2C_S_ERR | BCM2835_I2C_S_DONE);
+	complete(&i2c_dev->completion);
+
+	return IRQ_HANDLED;
 }
 
-static int bcm2835_i2c_xfer_msg(struct bcm2835_i2c_dev *i2c_dev,
-				struct i2c_msg *msg)
+static int bcm2835_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[],
+			    int num)
 {
-	u32 c;
+	struct bcm2835_i2c_dev *i2c_dev = i2c_get_adapdata(adap);
 	unsigned long time_left;
+	int i, ret;
 
-	i2c_dev->msg_buf = msg->buf;
-	i2c_dev->msg_buf_remaining = msg->len;
-	reinit_completion(&i2c_dev->completion);
+	for (i = 0; i < (num - 1); i++)
+		if (msgs[i].flags & I2C_M_RD) {
+			dev_warn_once(i2c_dev->dev,
+				      "only one read message supported, has to be last\n");
+			return -EOPNOTSUPP;
+		}
 
-	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_C, BCM2835_I2C_C_CLEAR);
+	ret = bcm2835_i2c_set_divider(i2c_dev);
+	if (ret)
+		return ret;
 
-	if (msg->flags & I2C_M_RD) {
-		c = BCM2835_I2C_C_READ | BCM2835_I2C_C_INTR;
-	} else {
-		c = BCM2835_I2C_C_INTT;
-		bcm2835_fill_txfifo(i2c_dev);
-	}
-	c |= BCM2835_I2C_C_ST | BCM2835_I2C_C_INTD | BCM2835_I2C_C_I2CEN;
+	i2c_dev->curr_msg = msgs;
+	i2c_dev->num_msgs = num;
+	reinit_completion(&i2c_dev->completion);
 
-	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_A, msg->addr);
-	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_DLEN, msg->len);
-	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_C, c);
+	bcm2835_i2c_start_transfer(i2c_dev);
 
 	time_left = wait_for_completion_timeout(&i2c_dev->completion,
-						BCM2835_I2C_TIMEOUT);
-	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_C, BCM2835_I2C_C_CLEAR);
+						adap->timeout);
 	if (!time_left) {
+		bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_C,
+				   BCM2835_I2C_C_CLEAR);
 		dev_err(i2c_dev->dev, "i2c transfer timed out\n");
 		return -ETIMEDOUT;
 	}
 
-	if (likely(!i2c_dev->msg_err))
-		return 0;
+	if (!i2c_dev->msg_err)
+		return num;
 
-	if ((i2c_dev->msg_err & BCM2835_I2C_S_ERR) &&
-	    (msg->flags & I2C_M_IGNORE_NAK))
-		return 0;
-
-	dev_err(i2c_dev->dev, "i2c transfer failed: %x\n", i2c_dev->msg_err);
+	dev_dbg(i2c_dev->dev, "i2c transfer failed: %x\n", i2c_dev->msg_err);
 
 	if (i2c_dev->msg_err & BCM2835_I2C_S_ERR)
 		return -EREMOTEIO;
-	else
-		return -EIO;
-}
-
-static int bcm2835_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[],
-			    int num)
-{
-	struct bcm2835_i2c_dev *i2c_dev = i2c_get_adapdata(adap);
-	int i;
-	int ret = 0;
-
-	for (i = 0; i < num; i++) {
-		ret = bcm2835_i2c_xfer_msg(i2c_dev, &msgs[i]);
-		if (ret)
-			break;
-	}
 
-	return ret ?: i;
+	return -EIO;
 }
 
 static u32 bcm2835_i2c_func(struct i2c_adapter *adap)
@@ -235,7 +311,6 @@ static int bcm2835_i2c_probe(struct platform_device *pdev)
 {
 	struct bcm2835_i2c_dev *i2c_dev;
 	struct resource *mem, *irq;
-	u32 bus_clk_rate, divider;
 	int ret;
 	struct i2c_adapter *adap;
 
@@ -259,27 +334,12 @@ static int bcm2835_i2c_probe(struct platform_device *pdev)
 	}
 
 	ret = of_property_read_u32(pdev->dev.of_node, "clock-frequency",
-				   &bus_clk_rate);
+				   &i2c_dev->bus_clk_rate);
 	if (ret < 0) {
 		dev_warn(&pdev->dev,
 			 "Could not read clock-frequency property\n");
-		bus_clk_rate = 100000;
-	}
-
-	divider = DIV_ROUND_UP(clk_get_rate(i2c_dev->clk), bus_clk_rate);
-	/*
-	 * Per the datasheet, the register is always interpreted as an even
-	 * number, by rounding down. In other words, the LSB is ignored. So,
-	 * if the LSB is set, increment the divider to avoid any issue.
-	 */
-	if (divider & 1)
-		divider++;
-	if ((divider < BCM2835_I2C_CDIV_MIN) ||
-	    (divider > BCM2835_I2C_CDIV_MAX)) {
-		dev_err(&pdev->dev, "Invalid clock-frequency\n");
-		return -ENODEV;
+		i2c_dev->bus_clk_rate = 100000;
 	}
-	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_DIV, divider);
 
 	irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
 	if (!irq) {
diff --git a/drivers/i2c/busses/i2c-designware-core.c b/drivers/i2c/busses/i2c-designware-core.c
index b403fa5ecf49..6d81c56184d3 100644
--- a/drivers/i2c/busses/i2c-designware-core.c
+++ b/drivers/i2c/busses/i2c-designware-core.c
@@ -536,6 +536,8 @@ i2c_dw_xfer_msg(struct dw_i2c_dev *dev)
 	intr_mask = DW_IC_INTR_DEFAULT_MASK;
 
 	for (; dev->msg_write_idx < dev->msgs_num; dev->msg_write_idx++) {
+		u32 flags = msgs[dev->msg_write_idx].flags;
+
 		/*
 		 * if target address has changed, we need to
 		 * reprogram the target address in the i2c
@@ -581,8 +583,15 @@ i2c_dw_xfer_msg(struct dw_i2c_dev *dev)
 			 * detected from the registers so we set it always
 			 * when writing/reading the last byte.
 			 */
+
+			/*
+			 * i2c-core.c always sets the buffer length of
+			 * I2C_FUNC_SMBUS_BLOCK_DATA to 1. The length will
+			 * be adjusted when receiving the first byte.
+			 * Thus we can't stop the transaction here.
+			 */
 			if (dev->msg_write_idx == dev->msgs_num - 1 &&
-			    buf_len == 1)
+			    buf_len == 1 && !(flags & I2C_M_RECV_LEN))
 				cmd |= BIT(9);
 
 			if (need_restart) {
@@ -607,7 +616,12 @@ i2c_dw_xfer_msg(struct dw_i2c_dev *dev)
 		dev->tx_buf = buf;
 		dev->tx_buf_len = buf_len;
 
-		if (buf_len > 0) {
+		/*
+		 * Because we don't know the buffer length in the
+		 * I2C_FUNC_SMBUS_BLOCK_DATA case, we can't stop
+		 * the transaction here.
+		 */
+		if (buf_len > 0 || flags & I2C_M_RECV_LEN) {
 			/* more bytes to be written */
 			dev->status |= STATUS_WRITE_IN_PROGRESS;
 			break;
@@ -628,6 +642,24 @@ i2c_dw_xfer_msg(struct dw_i2c_dev *dev)
 	dw_writel(dev, intr_mask,  DW_IC_INTR_MASK);
 }
 
+static u8
+i2c_dw_recv_len(struct dw_i2c_dev *dev, u8 len)
+{
+	struct i2c_msg *msgs = dev->msgs;
+	u32 flags = msgs[dev->msg_read_idx].flags;
+
+	/*
+	 * Adjust the buffer length and mask the flag
+	 * after receiving the first byte.
+	 */
+	len += (flags & I2C_CLIENT_PEC) ? 2 : 1;
+	dev->tx_buf_len = len - min_t(u8, len, dev->rx_outstanding);
+	msgs[dev->msg_read_idx].len = len;
+	msgs[dev->msg_read_idx].flags &= ~I2C_M_RECV_LEN;
+
+	return len;
+}
+
 static void
 i2c_dw_read(struct dw_i2c_dev *dev)
 {
@@ -652,7 +684,15 @@ i2c_dw_read(struct dw_i2c_dev *dev)
 		rx_valid = dw_readl(dev, DW_IC_RXFLR);
 
 		for (; len > 0 && rx_valid > 0; len--, rx_valid--) {
-			*buf++ = dw_readl(dev, DW_IC_DATA_CMD);
+			u32 flags = msgs[dev->msg_read_idx].flags;
+
+			*buf = dw_readl(dev, DW_IC_DATA_CMD);
+			/* Ensure length byte is a valid value */
+			if (flags & I2C_M_RECV_LEN &&
+				*buf <= I2C_SMBUS_BLOCK_MAX && *buf > 0) {
+				len = i2c_dw_recv_len(dev, *buf);
+			}
+			buf++;
 			dev->rx_outstanding--;
 		}
 
diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h
index 0d44d2ae7d4c..26250b425e2f 100644
--- a/drivers/i2c/busses/i2c-designware-core.h
+++ b/drivers/i2c/busses/i2c-designware-core.h
@@ -22,6 +22,14 @@
  *
  */
 
+#include <linux/i2c.h>
+
+#define DW_IC_DEFAULT_FUNCTIONALITY (I2C_FUNC_I2C |			\
+					I2C_FUNC_SMBUS_BYTE |		\
+					I2C_FUNC_SMBUS_BYTE_DATA |	\
+					I2C_FUNC_SMBUS_WORD_DATA |	\
+					I2C_FUNC_SMBUS_BLOCK_DATA |	\
+					I2C_FUNC_SMBUS_I2C_BLOCK)
 
 #define DW_IC_CON_MASTER		0x1
 #define DW_IC_CON_SPEED_STD		0x2
diff --git a/drivers/i2c/busses/i2c-designware-pcidrv.c b/drivers/i2c/busses/i2c-designware-pcidrv.c
index 96f8230cd2d3..d6423cfac588 100644
--- a/drivers/i2c/busses/i2c-designware-pcidrv.c
+++ b/drivers/i2c/busses/i2c-designware-pcidrv.c
@@ -71,12 +71,6 @@ struct dw_pci_controller {
 				DW_IC_CON_SLAVE_DISABLE |	\
 				DW_IC_CON_RESTART_EN)
 
-#define DW_DEFAULT_FUNCTIONALITY (I2C_FUNC_I2C |			\
-					I2C_FUNC_SMBUS_BYTE |		\
-					I2C_FUNC_SMBUS_BYTE_DATA |	\
-					I2C_FUNC_SMBUS_WORD_DATA |	\
-					I2C_FUNC_SMBUS_I2C_BLOCK)
-
 /* Merrifield HCNT/LCNT/SDA hold time */
 static struct dw_scl_sda_cfg mrfld_config = {
 	.ss_hcnt = 0x2f8,
@@ -147,6 +141,7 @@ static struct dw_pci_controller dw_pci_controllers[] = {
 		.bus_cfg   = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST,
 		.tx_fifo_depth = 32,
 		.rx_fifo_depth = 32,
+		.functionality = I2C_FUNC_10BIT_ADDR,
 		.clk_khz      = 25000,
 		.setup = mfld_setup,
 	},
@@ -155,6 +150,7 @@ static struct dw_pci_controller dw_pci_controllers[] = {
 		.bus_cfg = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST,
 		.tx_fifo_depth = 64,
 		.rx_fifo_depth = 64,
+		.functionality = I2C_FUNC_10BIT_ADDR,
 		.scl_sda_cfg = &mrfld_config,
 		.setup = mrfld_setup,
 	},
@@ -249,7 +245,7 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev,
 	}
 
 	dev->functionality = controller->functionality |
-				DW_DEFAULT_FUNCTIONALITY;
+				DW_IC_DEFAULT_FUNCTIONALITY;
 
 	dev->master_cfg = controller->bus_cfg;
 	if (controller->scl_sda_cfg) {
diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c
index 0b42a12171f3..08153ea4d848 100644
--- a/drivers/i2c/busses/i2c-designware-platdrv.c
+++ b/drivers/i2c/busses/i2c-designware-platdrv.c
@@ -176,9 +176,6 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 	dev->irq = irq;
 	platform_set_drvdata(pdev, dev);
 
-	/* fast mode by default because of legacy reasons */
-	dev->clk_freq = 400000;
-
 	if (pdata) {
 		dev->clk_freq = pdata->i2c_scl_freq;
 	} else {
@@ -193,8 +190,16 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 	}
 
 	acpi_speed = i2c_acpi_find_bus_speed(&pdev->dev);
-	if (acpi_speed)
-		dev->clk_freq = acpi_speed;
+	/*
+	 * Find bus speed from the "clock-frequency" device property, ACPI
+	 * or by using fast mode if neither is set.
+	 */
+	if (acpi_speed && dev->clk_freq)
+		dev->clk_freq = min(dev->clk_freq, acpi_speed);
+	else if (acpi_speed || dev->clk_freq)
+		dev->clk_freq = max(dev->clk_freq, acpi_speed);
+	else
+		dev->clk_freq = 400000;
 
 	if (has_acpi_companion(&pdev->dev))
 		dw_i2c_acpi_configure(pdev);
@@ -214,13 +219,7 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 	if (r)
 		return r;
 
-	dev->functionality =
-		I2C_FUNC_I2C |
-		I2C_FUNC_10BIT_ADDR |
-		I2C_FUNC_SMBUS_BYTE |
-		I2C_FUNC_SMBUS_BYTE_DATA |
-		I2C_FUNC_SMBUS_WORD_DATA |
-		I2C_FUNC_SMBUS_I2C_BLOCK;
+	dev->functionality = I2C_FUNC_10BIT_ADDR | DW_IC_DEFAULT_FUNCTIONALITY;
 
 	dev->master_cfg = DW_IC_CON_MASTER | DW_IC_CON_SLAVE_DISABLE |
 			  DW_IC_CON_RESTART_EN;
diff --git a/drivers/i2c/busses/i2c-dln2.c b/drivers/i2c/busses/i2c-dln2.c
index 8acda2aa1558..69075a32073e 100644
--- a/drivers/i2c/busses/i2c-dln2.c
+++ b/drivers/i2c/busses/i2c-dln2.c
@@ -182,7 +182,7 @@ static const struct i2c_algorithm dln2_i2c_usb_algorithm = {
 	.functionality = dln2_i2c_func,
 };
 
-static struct i2c_adapter_quirks dln2_i2c_quirks = {
+static const struct i2c_adapter_quirks dln2_i2c_quirks = {
 	.max_read_len = DLN2_I2C_MAX_XFER_SIZE,
 	.max_write_len = DLN2_I2C_MAX_XFER_SIZE,
 };
diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c
index eb3627f35d12..e242db43774b 100644
--- a/drivers/i2c/busses/i2c-i801.c
+++ b/drivers/i2c/busses/i2c-i801.c
@@ -118,7 +118,6 @@
 #define SMBSLVSTS(p)	(16 + (p)->smba)	/* ICH3 and later */
 #define SMBSLVCMD(p)	(17 + (p)->smba)	/* ICH3 and later */
 #define SMBNTFDADD(p)	(20 + (p)->smba)	/* ICH3 and later */
-#define SMBNTFDDAT(p)	(22 + (p)->smba)	/* ICH3 and later */
 
 /* PCI Address Constants */
 #define SMBBAR		4
@@ -137,27 +136,27 @@
 #define SBREG_SMBCTRL		0xc6000c
 
 /* Host status bits for SMBPCISTS */
-#define SMBPCISTS_INTS		0x08
+#define SMBPCISTS_INTS		BIT(3)
 
 /* Control bits for SMBPCICTL */
-#define SMBPCICTL_INTDIS	0x0400
+#define SMBPCICTL_INTDIS	BIT(10)
 
 /* Host configuration bits for SMBHSTCFG */
-#define SMBHSTCFG_HST_EN	1
-#define SMBHSTCFG_SMB_SMI_EN	2
-#define SMBHSTCFG_I2C_EN	4
-#define SMBHSTCFG_SPD_WD	0x10
+#define SMBHSTCFG_HST_EN	BIT(0)
+#define SMBHSTCFG_SMB_SMI_EN	BIT(1)
+#define SMBHSTCFG_I2C_EN	BIT(2)
+#define SMBHSTCFG_SPD_WD	BIT(4)
 
 /* TCO configuration bits for TCOCTL */
-#define TCOCTL_EN		0x0100
+#define TCOCTL_EN		BIT(8)
 
 /* Auxiliary status register bits, ICH4+ only */
-#define SMBAUXSTS_CRCE		1
-#define SMBAUXSTS_STCO		2
+#define SMBAUXSTS_CRCE		BIT(0)
+#define SMBAUXSTS_STCO		BIT(1)
 
 /* Auxiliary control register bits, ICH4+ only */
-#define SMBAUXCTL_CRC		1
-#define SMBAUXCTL_E32B		2
+#define SMBAUXCTL_CRC		BIT(0)
+#define SMBAUXCTL_E32B		BIT(1)
 
 /* Other settings */
 #define MAX_RETRIES		400
@@ -172,27 +171,27 @@
 #define I801_I2C_BLOCK_DATA	0x18	/* ICH5 and later */
 
 /* I801 Host Control register bits */
-#define SMBHSTCNT_INTREN	0x01
-#define SMBHSTCNT_KILL		0x02
-#define SMBHSTCNT_LAST_BYTE	0x20
-#define SMBHSTCNT_START		0x40
-#define SMBHSTCNT_PEC_EN	0x80	/* ICH3 and later */
+#define SMBHSTCNT_INTREN	BIT(0)
+#define SMBHSTCNT_KILL		BIT(1)
+#define SMBHSTCNT_LAST_BYTE	BIT(5)
+#define SMBHSTCNT_START		BIT(6)
+#define SMBHSTCNT_PEC_EN	BIT(7)	/* ICH3 and later */
 
 /* I801 Hosts Status register bits */
-#define SMBHSTSTS_BYTE_DONE	0x80
-#define SMBHSTSTS_INUSE_STS	0x40
-#define SMBHSTSTS_SMBALERT_STS	0x20
-#define SMBHSTSTS_FAILED	0x10
-#define SMBHSTSTS_BUS_ERR	0x08
-#define SMBHSTSTS_DEV_ERR	0x04
-#define SMBHSTSTS_INTR		0x02
-#define SMBHSTSTS_HOST_BUSY	0x01
+#define SMBHSTSTS_BYTE_DONE	BIT(7)
+#define SMBHSTSTS_INUSE_STS	BIT(6)
+#define SMBHSTSTS_SMBALERT_STS	BIT(5)
+#define SMBHSTSTS_FAILED	BIT(4)
+#define SMBHSTSTS_BUS_ERR	BIT(3)
+#define SMBHSTSTS_DEV_ERR	BIT(2)
+#define SMBHSTSTS_INTR		BIT(1)
+#define SMBHSTSTS_HOST_BUSY	BIT(0)
 
-/* Host Notify Status registers bits */
-#define SMBSLVSTS_HST_NTFY_STS	1
+/* Host Notify Status register bits */
+#define SMBSLVSTS_HST_NTFY_STS	BIT(0)
 
-/* Host Notify Command registers bits */
-#define SMBSLVCMD_HST_NTFY_INTREN	0x01
+/* Host Notify Command register bits */
+#define SMBSLVCMD_HST_NTFY_INTREN	BIT(0)
 
 #define STATUS_ERROR_FLAGS	(SMBHSTSTS_FAILED | SMBHSTSTS_BUS_ERR | \
 				 SMBHSTSTS_DEV_ERR)
@@ -243,6 +242,7 @@ struct i801_priv {
 	struct i2c_adapter adapter;
 	unsigned long smba;
 	unsigned char original_hstcfg;
+	unsigned char original_slvcmd;
 	struct pci_dev *pci_dev;
 	unsigned int features;
 
@@ -269,20 +269,17 @@ struct i801_priv {
 	 */
 	bool acpi_reserved;
 	struct mutex acpi_lock;
-	struct smbus_host_notify *host_notify;
 };
 
-#define SMBHSTNTFY_SIZE		8
-
-#define FEATURE_SMBUS_PEC	(1 << 0)
-#define FEATURE_BLOCK_BUFFER	(1 << 1)
-#define FEATURE_BLOCK_PROC	(1 << 2)
-#define FEATURE_I2C_BLOCK_READ	(1 << 3)
-#define FEATURE_IRQ		(1 << 4)
-#define FEATURE_HOST_NOTIFY	(1 << 5)
+#define FEATURE_SMBUS_PEC	BIT(0)
+#define FEATURE_BLOCK_BUFFER	BIT(1)
+#define FEATURE_BLOCK_PROC	BIT(2)
+#define FEATURE_I2C_BLOCK_READ	BIT(3)
+#define FEATURE_IRQ		BIT(4)
+#define FEATURE_HOST_NOTIFY	BIT(5)
 /* Not really a feature, but it's convenient to handle it as such */
-#define FEATURE_IDF		(1 << 15)
-#define FEATURE_TCO		(1 << 16)
+#define FEATURE_IDF		BIT(15)
+#define FEATURE_TCO		BIT(16)
 
 static const char *i801_feature_names[] = {
 	"SMBus PEC",
@@ -582,12 +579,15 @@ static void i801_isr_byte_done(struct i801_priv *priv)
 static irqreturn_t i801_host_notify_isr(struct i801_priv *priv)
 {
 	unsigned short addr;
-	unsigned int data;
 
 	addr = inb_p(SMBNTFDADD(priv)) >> 1;
-	data = inw_p(SMBNTFDDAT(priv));
 
-	i2c_handle_smbus_host_notify(priv->host_notify, addr, data);
+	/*
+	 * With the tested platforms, reading SMBNTFDDAT (22 + (p)->smba)
+	 * always returns 0. Our current implementation doesn't provide
+	 * data, so we just ignore it.
+	 */
+	i2c_handle_smbus_host_notify(&priv->adapter, addr);
 
 	/* clear Host Notify bit and return */
 	outb_p(SMBSLVSTS_HST_NTFY_STS, SMBSLVSTS(priv));
@@ -950,23 +950,29 @@ static u32 i801_func(struct i2c_adapter *adapter)
 		I2C_FUNC_SMBUS_HOST_NOTIFY : 0);
 }
 
-static int i801_enable_host_notify(struct i2c_adapter *adapter)
+static void i801_enable_host_notify(struct i2c_adapter *adapter)
 {
 	struct i801_priv *priv = i2c_get_adapdata(adapter);
 
 	if (!(priv->features & FEATURE_HOST_NOTIFY))
-		return -ENOTSUPP;
+		return;
 
-	if (!priv->host_notify)
-		priv->host_notify = i2c_setup_smbus_host_notify(adapter);
-	if (!priv->host_notify)
-		return -ENOMEM;
+	priv->original_slvcmd = inb_p(SMBSLVCMD(priv));
+
+	if (!(SMBSLVCMD_HST_NTFY_INTREN & priv->original_slvcmd))
+		outb_p(SMBSLVCMD_HST_NTFY_INTREN | priv->original_slvcmd,
+		       SMBSLVCMD(priv));
 
-	outb_p(SMBSLVCMD_HST_NTFY_INTREN, SMBSLVCMD(priv));
 	/* clear Host Notify bit to allow a new notification */
 	outb_p(SMBSLVSTS_HST_NTFY_STS, SMBSLVSTS(priv));
+}
 
-	return 0;
+static void i801_disable_host_notify(struct i801_priv *priv)
+{
+	if (!(priv->features & FEATURE_HOST_NOTIFY))
+		return;
+
+	outb_p(priv->original_slvcmd, SMBSLVCMD(priv));
 }
 
 static const struct i2c_algorithm smbus_algorithm = {
@@ -1633,14 +1639,7 @@ static int i801_probe(struct pci_dev *dev, const struct pci_device_id *id)
 		return err;
 	}
 
-	/*
-	 * Enable Host Notify for chips that supports it.
-	 * It is done after i2c_add_adapter() so that we are sure the work queue
-	 * is not used if i2c_add_adapter() fails.
-	 */
-	err = i801_enable_host_notify(&priv->adapter);
-	if (err && err != -ENOTSUPP)
-		dev_warn(&dev->dev, "Unable to enable SMBus Host Notify\n");
+	i801_enable_host_notify(&priv->adapter);
 
 	i801_probe_optional_slaves(priv);
 	/* We ignore errors - multiplexing is optional */
@@ -1663,6 +1662,7 @@ static void i801_remove(struct pci_dev *dev)
 	pm_runtime_forbid(&dev->dev);
 	pm_runtime_get_noresume(&dev->dev);
 
+	i801_disable_host_notify(priv);
 	i801_del_mux(priv);
 	i2c_del_adapter(&priv->adapter);
 	i801_acpi_remove(priv);
@@ -1690,11 +1690,8 @@ static int i801_resume(struct device *dev)
 {
 	struct pci_dev *pci_dev = to_pci_dev(dev);
 	struct i801_priv *priv = pci_get_drvdata(pci_dev);
-	int err;
 
-	err = i801_enable_host_notify(&priv->adapter);
-	if (err && err != -ENOTSUPP)
-		dev_warn(dev, "Unable to enable SMBus Host Notify\n");
+	i801_enable_host_notify(&priv->adapter);
 
 	return 0;
 }
diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c
new file mode 100644
index 000000000000..c62b7cd475f8
--- /dev/null
+++ b/drivers/i2c/busses/i2c-imx-lpi2c.c
@@ -0,0 +1,652 @@
+/*
+ * This is i.MX low power i2c controller driver.
+ *
+ * Copyright 2016 Freescale Semiconductor, Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+
+#define DRIVER_NAME "imx-lpi2c"
+
+#define LPI2C_PARAM	0x04	/* i2c RX/TX FIFO size */
+#define LPI2C_MCR	0x10	/* i2c contrl register */
+#define LPI2C_MSR	0x14	/* i2c status register */
+#define LPI2C_MIER	0x18	/* i2c interrupt enable */
+#define LPI2C_MCFGR0	0x20	/* i2c master configuration */
+#define LPI2C_MCFGR1	0x24	/* i2c master configuration */
+#define LPI2C_MCFGR2	0x28	/* i2c master configuration */
+#define LPI2C_MCFGR3	0x2C	/* i2c master configuration */
+#define LPI2C_MCCR0	0x48	/* i2c master clk configuration */
+#define LPI2C_MCCR1	0x50	/* i2c master clk configuration */
+#define LPI2C_MFCR	0x58	/* i2c master FIFO control */
+#define LPI2C_MFSR	0x5C	/* i2c master FIFO status */
+#define LPI2C_MTDR	0x60	/* i2c master TX data register */
+#define LPI2C_MRDR	0x70	/* i2c master RX data register */
+
+/* i2c command */
+#define TRAN_DATA	0X00
+#define RECV_DATA	0X01
+#define GEN_STOP	0X02
+#define RECV_DISCARD	0X03
+#define GEN_START	0X04
+#define START_NACK	0X05
+#define START_HIGH	0X06
+#define START_HIGH_NACK	0X07
+
+#define MCR_MEN		BIT(0)
+#define MCR_RST		BIT(1)
+#define MCR_DOZEN	BIT(2)
+#define MCR_DBGEN	BIT(3)
+#define MCR_RTF		BIT(8)
+#define MCR_RRF		BIT(9)
+#define MSR_TDF		BIT(0)
+#define MSR_RDF		BIT(1)
+#define MSR_SDF		BIT(9)
+#define MSR_NDF		BIT(10)
+#define MSR_ALF		BIT(11)
+#define MSR_MBF		BIT(24)
+#define MSR_BBF		BIT(25)
+#define MIER_TDIE	BIT(0)
+#define MIER_RDIE	BIT(1)
+#define MIER_SDIE	BIT(9)
+#define MIER_NDIE	BIT(10)
+#define MCFGR1_AUTOSTOP	BIT(8)
+#define MCFGR1_IGNACK	BIT(9)
+#define MRDR_RXEMPTY	BIT(14)
+
+#define I2C_CLK_RATIO	2
+#define CHUNK_DATA	256
+
+#define LPI2C_DEFAULT_RATE	100000
+#define STARDARD_MAX_BITRATE	400000
+#define FAST_MAX_BITRATE	1000000
+#define FAST_PLUS_MAX_BITRATE	3400000
+#define HIGHSPEED_MAX_BITRATE	5000000
+
+enum lpi2c_imx_mode {
+	STANDARD,	/* 100+Kbps */
+	FAST,		/* 400+Kbps */
+	FAST_PLUS,	/* 1.0+Mbps */
+	HS,		/* 3.4+Mbps */
+	ULTRA_FAST,	/* 5.0+Mbps */
+};
+
+enum lpi2c_imx_pincfg {
+	TWO_PIN_OD,
+	TWO_PIN_OO,
+	TWO_PIN_PP,
+	FOUR_PIN_PP,
+};
+
+struct lpi2c_imx_struct {
+	struct i2c_adapter	adapter;
+	struct clk		*clk;
+	void __iomem		*base;
+	__u8			*rx_buf;
+	__u8			*tx_buf;
+	struct completion	complete;
+	unsigned int		msglen;
+	unsigned int		delivered;
+	unsigned int		block_data;
+	unsigned int		bitrate;
+	unsigned int		txfifosize;
+	unsigned int		rxfifosize;
+	enum lpi2c_imx_mode	mode;
+};
+
+static void lpi2c_imx_intctrl(struct lpi2c_imx_struct *lpi2c_imx,
+			      unsigned int enable)
+{
+	writel(enable, lpi2c_imx->base + LPI2C_MIER);
+}
+
+static int lpi2c_imx_bus_busy(struct lpi2c_imx_struct *lpi2c_imx)
+{
+	unsigned long orig_jiffies = jiffies;
+	unsigned int temp;
+
+	while (1) {
+		temp = readl(lpi2c_imx->base + LPI2C_MSR);
+
+		/* check for arbitration lost, clear if set */
+		if (temp & MSR_ALF) {
+			writel(temp, lpi2c_imx->base + LPI2C_MSR);
+			return -EAGAIN;
+		}
+
+		if (temp & (MSR_BBF | MSR_MBF))
+			break;
+
+		if (time_after(jiffies, orig_jiffies + msecs_to_jiffies(500))) {
+			dev_dbg(&lpi2c_imx->adapter.dev, "bus not work\n");
+			return -ETIMEDOUT;
+		}
+		schedule();
+	}
+
+	return 0;
+}
+
+static void lpi2c_imx_set_mode(struct lpi2c_imx_struct *lpi2c_imx)
+{
+	unsigned int bitrate = lpi2c_imx->bitrate;
+	enum lpi2c_imx_mode mode;
+
+	if (bitrate < STARDARD_MAX_BITRATE)
+		mode = STANDARD;
+	else if (bitrate < FAST_MAX_BITRATE)
+		mode = FAST;
+	else if (bitrate < FAST_PLUS_MAX_BITRATE)
+		mode = FAST_PLUS;
+	else if (bitrate < HIGHSPEED_MAX_BITRATE)
+		mode = HS;
+	else
+		mode = ULTRA_FAST;
+
+	lpi2c_imx->mode = mode;
+}
+
+static int lpi2c_imx_start(struct lpi2c_imx_struct *lpi2c_imx,
+			   struct i2c_msg *msgs)
+{
+	unsigned int temp;
+	u8 read;
+
+	temp = readl(lpi2c_imx->base + LPI2C_MCR);
+	temp |= MCR_RRF | MCR_RTF;
+	writel(temp, lpi2c_imx->base + LPI2C_MCR);
+	writel(0x7f00, lpi2c_imx->base + LPI2C_MSR);
+
+	read = msgs->flags & I2C_M_RD;
+	temp = (msgs->addr << 1 | read) | (GEN_START << 8);
+	writel(temp, lpi2c_imx->base + LPI2C_MTDR);
+
+	return lpi2c_imx_bus_busy(lpi2c_imx);
+}
+
+static void lpi2c_imx_stop(struct lpi2c_imx_struct *lpi2c_imx)
+{
+	unsigned long orig_jiffies = jiffies;
+	unsigned int temp;
+
+	writel(GEN_STOP << 8, lpi2c_imx->base + LPI2C_MTDR);
+
+	do {
+		temp = readl(lpi2c_imx->base + LPI2C_MSR);
+		if (temp & MSR_SDF)
+			break;
+
+		if (time_after(jiffies, orig_jiffies + msecs_to_jiffies(500))) {
+			dev_dbg(&lpi2c_imx->adapter.dev, "stop timeout\n");
+			break;
+		}
+		schedule();
+
+	} while (1);
+}
+
+/* CLKLO = I2C_CLK_RATIO * CLKHI, SETHOLD = CLKHI, DATAVD = CLKHI/2 */
+static int lpi2c_imx_config(struct lpi2c_imx_struct *lpi2c_imx)
+{
+	u8 prescale, filt, sethold, clkhi, clklo, datavd;
+	unsigned int clk_rate, clk_cycle;
+	enum lpi2c_imx_pincfg pincfg;
+	unsigned int temp;
+
+	lpi2c_imx_set_mode(lpi2c_imx);
+
+	clk_rate = clk_get_rate(lpi2c_imx->clk);
+	if (lpi2c_imx->mode == HS || lpi2c_imx->mode == ULTRA_FAST)
+		filt = 0;
+	else
+		filt = 2;
+
+	for (prescale = 0; prescale <= 7; prescale++) {
+		clk_cycle = clk_rate / ((1 << prescale) * lpi2c_imx->bitrate)
+			    - 3 - (filt >> 1);
+		clkhi = (clk_cycle + I2C_CLK_RATIO) / (I2C_CLK_RATIO + 1);
+		clklo = clk_cycle - clkhi;
+		if (clklo < 64)
+			break;
+	}
+
+	if (prescale > 7)
+		return -EINVAL;
+
+	/* set MCFGR1: PINCFG, PRESCALE, IGNACK */
+	if (lpi2c_imx->mode == ULTRA_FAST)
+		pincfg = TWO_PIN_OO;
+	else
+		pincfg = TWO_PIN_OD;
+	temp = prescale | pincfg << 24;
+
+	if (lpi2c_imx->mode == ULTRA_FAST)
+		temp |= MCFGR1_IGNACK;
+
+	writel(temp, lpi2c_imx->base + LPI2C_MCFGR1);
+
+	/* set MCFGR2: FILTSDA, FILTSCL */
+	temp = (filt << 16) | (filt << 24);
+	writel(temp, lpi2c_imx->base + LPI2C_MCFGR2);
+
+	/* set MCCR: DATAVD, SETHOLD, CLKHI, CLKLO */
+	sethold = clkhi;
+	datavd = clkhi >> 1;
+	temp = datavd << 24 | sethold << 16 | clkhi << 8 | clklo;
+
+	if (lpi2c_imx->mode == HS)
+		writel(temp, lpi2c_imx->base + LPI2C_MCCR1);
+	else
+		writel(temp, lpi2c_imx->base + LPI2C_MCCR0);
+
+	return 0;
+}
+
+static int lpi2c_imx_master_enable(struct lpi2c_imx_struct *lpi2c_imx)
+{
+	unsigned int temp;
+	int ret;
+
+	ret = clk_enable(lpi2c_imx->clk);
+	if (ret)
+		return ret;
+
+	temp = MCR_RST;
+	writel(temp, lpi2c_imx->base + LPI2C_MCR);
+	writel(0, lpi2c_imx->base + LPI2C_MCR);
+
+	ret = lpi2c_imx_config(lpi2c_imx);
+	if (ret)
+		goto clk_disable;
+
+	temp = readl(lpi2c_imx->base + LPI2C_MCR);
+	temp |= MCR_MEN;
+	writel(temp, lpi2c_imx->base + LPI2C_MCR);
+
+	return 0;
+
+clk_disable:
+	clk_disable(lpi2c_imx->clk);
+
+	return ret;
+}
+
+static int lpi2c_imx_master_disable(struct lpi2c_imx_struct *lpi2c_imx)
+{
+	u32 temp;
+
+	temp = readl(lpi2c_imx->base + LPI2C_MCR);
+	temp &= ~MCR_MEN;
+	writel(temp, lpi2c_imx->base + LPI2C_MCR);
+
+	clk_disable(lpi2c_imx->clk);
+
+	return 0;
+}
+
+static int lpi2c_imx_msg_complete(struct lpi2c_imx_struct *lpi2c_imx)
+{
+	unsigned long timeout;
+
+	timeout = wait_for_completion_timeout(&lpi2c_imx->complete, HZ);
+
+	return timeout ? 0 : -ETIMEDOUT;
+}
+
+static int lpi2c_imx_txfifo_empty(struct lpi2c_imx_struct *lpi2c_imx)
+{
+	unsigned long orig_jiffies = jiffies;
+	u32 txcnt;
+
+	do {
+		txcnt = readl(lpi2c_imx->base + LPI2C_MFSR) & 0xff;
+
+		if (readl(lpi2c_imx->base + LPI2C_MSR) & MSR_NDF) {
+			dev_dbg(&lpi2c_imx->adapter.dev, "NDF detected\n");
+			return -EIO;
+		}
+
+		if (time_after(jiffies, orig_jiffies + msecs_to_jiffies(500))) {
+			dev_dbg(&lpi2c_imx->adapter.dev, "txfifo empty timeout\n");
+			return -ETIMEDOUT;
+		}
+		schedule();
+
+	} while (txcnt);
+
+	return 0;
+}
+
+static void lpi2c_imx_set_tx_watermark(struct lpi2c_imx_struct *lpi2c_imx)
+{
+	writel(lpi2c_imx->txfifosize >> 1, lpi2c_imx->base + LPI2C_MFCR);
+}
+
+static void lpi2c_imx_set_rx_watermark(struct lpi2c_imx_struct *lpi2c_imx)
+{
+	unsigned int temp, remaining;
+
+	remaining = lpi2c_imx->msglen - lpi2c_imx->delivered;
+
+	if (remaining > (lpi2c_imx->rxfifosize >> 1))
+		temp = lpi2c_imx->rxfifosize >> 1;
+	else
+		temp = 0;
+
+	writel(temp << 16, lpi2c_imx->base + LPI2C_MFCR);
+}
+
+static void lpi2c_imx_write_txfifo(struct lpi2c_imx_struct *lpi2c_imx)
+{
+	unsigned int data, txcnt;
+
+	txcnt = readl(lpi2c_imx->base + LPI2C_MFSR) & 0xff;
+
+	while (txcnt < lpi2c_imx->txfifosize) {
+		if (lpi2c_imx->delivered == lpi2c_imx->msglen)
+			break;
+
+		data = lpi2c_imx->tx_buf[lpi2c_imx->delivered++];
+		writel(data, lpi2c_imx->base + LPI2C_MTDR);
+		txcnt++;
+	}
+
+	if (lpi2c_imx->delivered < lpi2c_imx->msglen)
+		lpi2c_imx_intctrl(lpi2c_imx, MIER_TDIE | MIER_NDIE);
+	else
+		complete(&lpi2c_imx->complete);
+}
+
+static void lpi2c_imx_read_rxfifo(struct lpi2c_imx_struct *lpi2c_imx)
+{
+	unsigned int blocklen, remaining;
+	unsigned int temp, data;
+
+	do {
+		data = readl(lpi2c_imx->base + LPI2C_MRDR);
+		if (data & MRDR_RXEMPTY)
+			break;
+
+		lpi2c_imx->rx_buf[lpi2c_imx->delivered++] = data & 0xff;
+	} while (1);
+
+	/*
+	 * First byte is the length of remaining packet in the SMBus block
+	 * data read. Add it to msgs->len.
+	 */
+	if (lpi2c_imx->block_data) {
+		blocklen = lpi2c_imx->rx_buf[0];
+		lpi2c_imx->msglen += blocklen;
+	}
+
+	remaining = lpi2c_imx->msglen - lpi2c_imx->delivered;
+
+	if (!remaining) {
+		complete(&lpi2c_imx->complete);
+		return;
+	}
+
+	/* not finished, still waiting for rx data */
+	lpi2c_imx_set_rx_watermark(lpi2c_imx);
+
+	/* multiple receive commands */
+	if (lpi2c_imx->block_data) {
+		lpi2c_imx->block_data = 0;
+		temp = remaining;
+		temp |= (RECV_DATA << 8);
+		writel(temp, lpi2c_imx->base + LPI2C_MTDR);
+	} else if (!(lpi2c_imx->delivered & 0xff)) {
+		temp = (remaining > CHUNK_DATA ? CHUNK_DATA : remaining) - 1;
+		temp |= (RECV_DATA << 8);
+		writel(temp, lpi2c_imx->base + LPI2C_MTDR);
+	}
+
+	lpi2c_imx_intctrl(lpi2c_imx, MIER_RDIE);
+}
+
+static void lpi2c_imx_write(struct lpi2c_imx_struct *lpi2c_imx,
+			    struct i2c_msg *msgs)
+{
+	lpi2c_imx->tx_buf = msgs->buf;
+	lpi2c_imx_set_tx_watermark(lpi2c_imx);
+	lpi2c_imx_write_txfifo(lpi2c_imx);
+}
+
+static void lpi2c_imx_read(struct lpi2c_imx_struct *lpi2c_imx,
+			   struct i2c_msg *msgs)
+{
+	unsigned int temp;
+
+	lpi2c_imx->rx_buf = msgs->buf;
+	lpi2c_imx->block_data = msgs->flags & I2C_M_RECV_LEN;
+
+	lpi2c_imx_set_rx_watermark(lpi2c_imx);
+	temp = msgs->len > CHUNK_DATA ? CHUNK_DATA - 1 : msgs->len - 1;
+	temp |= (RECV_DATA << 8);
+	writel(temp, lpi2c_imx->base + LPI2C_MTDR);
+
+	lpi2c_imx_intctrl(lpi2c_imx, MIER_RDIE | MIER_NDIE);
+}
+
+static int lpi2c_imx_xfer(struct i2c_adapter *adapter,
+			  struct i2c_msg *msgs, int num)
+{
+	struct lpi2c_imx_struct *lpi2c_imx = i2c_get_adapdata(adapter);
+	unsigned int temp;
+	int i, result;
+
+	result = lpi2c_imx_master_enable(lpi2c_imx);
+	if (result)
+		return result;
+
+	for (i = 0; i < num; i++) {
+		result = lpi2c_imx_start(lpi2c_imx, &msgs[i]);
+		if (result)
+			goto disable;
+
+		/* quick smbus */
+		if (num == 1 && msgs[0].len == 0)
+			goto stop;
+
+		lpi2c_imx->delivered = 0;
+		lpi2c_imx->msglen = msgs[i].len;
+		init_completion(&lpi2c_imx->complete);
+
+		if (msgs[i].flags & I2C_M_RD)
+			lpi2c_imx_read(lpi2c_imx, &msgs[i]);
+		else
+			lpi2c_imx_write(lpi2c_imx, &msgs[i]);
+
+		result = lpi2c_imx_msg_complete(lpi2c_imx);
+		if (result)
+			goto stop;
+
+		if (!(msgs[i].flags & I2C_M_RD)) {
+			result = lpi2c_imx_txfifo_empty(lpi2c_imx);
+			if (result)
+				goto stop;
+		}
+	}
+
+stop:
+	lpi2c_imx_stop(lpi2c_imx);
+
+	temp = readl(lpi2c_imx->base + LPI2C_MSR);
+	if ((temp & MSR_NDF) && !result)
+		result = -EIO;
+
+disable:
+	lpi2c_imx_master_disable(lpi2c_imx);
+
+	dev_dbg(&lpi2c_imx->adapter.dev, "<%s> exit with: %s: %d\n", __func__,
+		(result < 0) ? "error" : "success msg",
+		(result < 0) ? result : num);
+
+	return (result < 0) ? result : num;
+}
+
+static irqreturn_t lpi2c_imx_isr(int irq, void *dev_id)
+{
+	struct lpi2c_imx_struct *lpi2c_imx = dev_id;
+	unsigned int temp;
+
+	lpi2c_imx_intctrl(lpi2c_imx, 0);
+	temp = readl(lpi2c_imx->base + LPI2C_MSR);
+
+	if (temp & MSR_RDF)
+		lpi2c_imx_read_rxfifo(lpi2c_imx);
+
+	if (temp & MSR_TDF)
+		lpi2c_imx_write_txfifo(lpi2c_imx);
+
+	if (temp & MSR_NDF)
+		complete(&lpi2c_imx->complete);
+
+	return IRQ_HANDLED;
+}
+
+static u32 lpi2c_imx_func(struct i2c_adapter *adapter)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL |
+		I2C_FUNC_SMBUS_READ_BLOCK_DATA;
+}
+
+static struct i2c_algorithm lpi2c_imx_algo = {
+	.master_xfer	= lpi2c_imx_xfer,
+	.functionality	= lpi2c_imx_func,
+};
+
+static const struct of_device_id lpi2c_imx_of_match[] = {
+	{ .compatible = "fsl,imx7ulp-lpi2c" },
+	{ .compatible = "fsl,imx8dv-lpi2c" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, lpi2c_imx_of_match);
+
+static int lpi2c_imx_probe(struct platform_device *pdev)
+{
+	struct lpi2c_imx_struct *lpi2c_imx;
+	struct resource *res;
+	unsigned int temp;
+	int irq, ret;
+
+	lpi2c_imx = devm_kzalloc(&pdev->dev, sizeof(*lpi2c_imx), GFP_KERNEL);
+	if (!lpi2c_imx)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	lpi2c_imx->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(lpi2c_imx->base))
+		return PTR_ERR(lpi2c_imx->base);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "can't get irq number\n");
+		return irq;
+	}
+
+	lpi2c_imx->adapter.owner	= THIS_MODULE;
+	lpi2c_imx->adapter.algo		= &lpi2c_imx_algo;
+	lpi2c_imx->adapter.dev.parent	= &pdev->dev;
+	lpi2c_imx->adapter.dev.of_node	= pdev->dev.of_node;
+	strlcpy(lpi2c_imx->adapter.name, pdev->name,
+		sizeof(lpi2c_imx->adapter.name));
+
+	lpi2c_imx->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(lpi2c_imx->clk)) {
+		dev_err(&pdev->dev, "can't get I2C peripheral clock\n");
+		return PTR_ERR(lpi2c_imx->clk);
+	}
+
+	ret = of_property_read_u32(pdev->dev.of_node,
+				   "clock-frequency", &lpi2c_imx->bitrate);
+	if (ret)
+		lpi2c_imx->bitrate = LPI2C_DEFAULT_RATE;
+
+	ret = devm_request_irq(&pdev->dev, irq, lpi2c_imx_isr, 0,
+			       pdev->name, lpi2c_imx);
+	if (ret) {
+		dev_err(&pdev->dev, "can't claim irq %d\n", irq);
+		return ret;
+	}
+
+	i2c_set_adapdata(&lpi2c_imx->adapter, lpi2c_imx);
+	platform_set_drvdata(pdev, lpi2c_imx);
+
+	ret = clk_prepare_enable(lpi2c_imx->clk);
+	if (ret) {
+		dev_err(&pdev->dev, "clk enable failed %d\n", ret);
+		return ret;
+	}
+
+	temp = readl(lpi2c_imx->base + LPI2C_PARAM);
+	lpi2c_imx->txfifosize = 1 << (temp & 0x0f);
+	lpi2c_imx->rxfifosize = 1 << ((temp >> 8) & 0x0f);
+
+	clk_disable(lpi2c_imx->clk);
+
+	ret = i2c_add_adapter(&lpi2c_imx->adapter);
+	if (ret)
+		goto clk_unprepare;
+
+	dev_info(&lpi2c_imx->adapter.dev, "LPI2C adapter registered\n");
+
+	return 0;
+
+clk_unprepare:
+	clk_unprepare(lpi2c_imx->clk);
+
+	return ret;
+}
+
+static int lpi2c_imx_remove(struct platform_device *pdev)
+{
+	struct lpi2c_imx_struct *lpi2c_imx = platform_get_drvdata(pdev);
+
+	i2c_del_adapter(&lpi2c_imx->adapter);
+
+	clk_unprepare(lpi2c_imx->clk);
+
+	return 0;
+}
+
+static struct platform_driver lpi2c_imx_driver = {
+	.probe = lpi2c_imx_probe,
+	.remove = lpi2c_imx_remove,
+	.driver = {
+		.name = DRIVER_NAME,
+		.of_match_table = lpi2c_imx_of_match,
+	},
+};
+
+module_platform_driver(lpi2c_imx_driver);
+
+MODULE_AUTHOR("Gao Pan <pandy.gao@nxp.com>");
+MODULE_DESCRIPTION("I2C adapter driver for LPI2C bus");
+MODULE_LICENSE("GPL");
diff --git a/drivers/i2c/busses/i2c-mlxcpld.c b/drivers/i2c/busses/i2c-mlxcpld.c
new file mode 100644
index 000000000000..d271e6a0954c
--- /dev/null
+++ b/drivers/i2c/busses/i2c-mlxcpld.c
@@ -0,0 +1,504 @@
+/*
+ * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2016 Michael Shych <michaels@mellanox.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the names of the copyright holders nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+/* General defines */
+#define MLXPLAT_CPLD_LPC_I2C_BASE_ADDR	0x2000
+#define MLXCPLD_I2C_DEVICE_NAME		"i2c_mlxcpld"
+#define MLXCPLD_I2C_VALID_FLAG		(I2C_M_RECV_LEN | I2C_M_RD)
+#define MLXCPLD_I2C_BUS_NUM		1
+#define MLXCPLD_I2C_DATA_REG_SZ		36
+#define MLXCPLD_I2C_MAX_ADDR_LEN	4
+#define MLXCPLD_I2C_RETR_NUM		2
+#define MLXCPLD_I2C_XFER_TO		500000 /* usec */
+#define MLXCPLD_I2C_POLL_TIME		2000   /* usec */
+
+/* LPC I2C registers */
+#define MLXCPLD_LPCI2C_LPF_REG		0x0
+#define MLXCPLD_LPCI2C_CTRL_REG		0x1
+#define MLXCPLD_LPCI2C_HALF_CYC_REG	0x4
+#define MLXCPLD_LPCI2C_I2C_HOLD_REG	0x5
+#define MLXCPLD_LPCI2C_CMD_REG		0x6
+#define MLXCPLD_LPCI2C_NUM_DAT_REG	0x7
+#define MLXCPLD_LPCI2C_NUM_ADDR_REG	0x8
+#define MLXCPLD_LPCI2C_STATUS_REG	0x9
+#define MLXCPLD_LPCI2C_DATA_REG		0xa
+
+/* LPC I2C masks and parametres */
+#define MLXCPLD_LPCI2C_RST_SEL_MASK	0x1
+#define MLXCPLD_LPCI2C_TRANS_END	0x1
+#define MLXCPLD_LPCI2C_STATUS_NACK	0x10
+#define MLXCPLD_LPCI2C_NO_IND		0
+#define MLXCPLD_LPCI2C_ACK_IND		1
+#define MLXCPLD_LPCI2C_NACK_IND		2
+
+struct  mlxcpld_i2c_curr_xfer {
+	u8 cmd;
+	u8 addr_width;
+	u8 data_len;
+	u8 msg_num;
+	struct i2c_msg *msg;
+};
+
+struct mlxcpld_i2c_priv {
+	struct i2c_adapter adap;
+	u32 base_addr;
+	struct mutex lock;
+	struct  mlxcpld_i2c_curr_xfer xfer;
+	struct device *dev;
+};
+
+static void mlxcpld_i2c_lpc_write_buf(u8 *data, u8 len, u32 addr)
+{
+	int i;
+
+	for (i = 0; i < len - len % 4; i += 4)
+		outl(*(u32 *)(data + i), addr + i);
+	for (; i < len; ++i)
+		outb(*(data + i), addr + i);
+}
+
+static void mlxcpld_i2c_lpc_read_buf(u8 *data, u8 len, u32 addr)
+{
+	int i;
+
+	for (i = 0; i < len - len % 4; i += 4)
+		*(u32 *)(data + i) = inl(addr + i);
+	for (; i < len; ++i)
+		*(data + i) = inb(addr + i);
+}
+
+static void mlxcpld_i2c_read_comm(struct mlxcpld_i2c_priv *priv, u8 offs,
+				  u8 *data, u8 datalen)
+{
+	u32 addr = priv->base_addr + offs;
+
+	switch (datalen) {
+	case 1:
+		*(data) = inb(addr);
+		break;
+	case 2:
+		*((u16 *)data) = inw(addr);
+		break;
+	case 3:
+		*((u16 *)data) = inw(addr);
+		*(data + 2) = inb(addr + 2);
+		break;
+	case 4:
+		*((u32 *)data) = inl(addr);
+		break;
+	default:
+		mlxcpld_i2c_lpc_read_buf(data, datalen, addr);
+		break;
+	}
+}
+
+static void mlxcpld_i2c_write_comm(struct mlxcpld_i2c_priv *priv, u8 offs,
+				   u8 *data, u8 datalen)
+{
+	u32 addr = priv->base_addr + offs;
+
+	switch (datalen) {
+	case 1:
+		outb(*(data), addr);
+		break;
+	case 2:
+		outw(*((u16 *)data), addr);
+		break;
+	case 3:
+		outw(*((u16 *)data), addr);
+		outb(*(data + 2), addr + 2);
+		break;
+	case 4:
+		outl(*((u32 *)data), addr);
+		break;
+	default:
+		mlxcpld_i2c_lpc_write_buf(data, datalen, addr);
+		break;
+	}
+}
+
+/*
+ * Check validity of received i2c messages parameters.
+ * Returns 0 if OK, other - in case of invalid parameters.
+ */
+static int mlxcpld_i2c_check_msg_params(struct mlxcpld_i2c_priv *priv,
+					struct i2c_msg *msgs, int num)
+{
+	int i;
+
+	if (!num) {
+		dev_err(priv->dev, "Incorrect 0 num of messages\n");
+		return -EINVAL;
+	}
+
+	if (unlikely(msgs[0].addr > 0x7f)) {
+		dev_err(priv->dev, "Invalid address 0x%03x\n",
+			msgs[0].addr);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < num; ++i) {
+		if (unlikely(!msgs[i].buf)) {
+			dev_err(priv->dev, "Invalid buf in msg[%d]\n",
+				i);
+			return -EINVAL;
+		}
+		if (unlikely(msgs[0].addr != msgs[i].addr)) {
+			dev_err(priv->dev, "Invalid addr in msg[%d]\n",
+				i);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * Check if transfer is completed and status of operation.
+ * Returns 0 - transfer completed (both ACK or NACK),
+ * negative - transfer isn't finished.
+ */
+static int mlxcpld_i2c_check_status(struct mlxcpld_i2c_priv *priv, int *status)
+{
+	u8 val;
+
+	mlxcpld_i2c_read_comm(priv, MLXCPLD_LPCI2C_STATUS_REG, &val, 1);
+
+	if (val & MLXCPLD_LPCI2C_TRANS_END) {
+		if (val & MLXCPLD_LPCI2C_STATUS_NACK)
+			/*
+			 * The slave is unable to accept the data. No such
+			 * slave, command not understood, or unable to accept
+			 * any more data.
+			 */
+			*status = MLXCPLD_LPCI2C_NACK_IND;
+		else
+			*status = MLXCPLD_LPCI2C_ACK_IND;
+		return 0;
+	}
+	*status = MLXCPLD_LPCI2C_NO_IND;
+
+	return -EIO;
+}
+
+static void mlxcpld_i2c_set_transf_data(struct mlxcpld_i2c_priv *priv,
+					struct i2c_msg *msgs, int num,
+					u8 comm_len)
+{
+	priv->xfer.msg = msgs;
+	priv->xfer.msg_num = num;
+
+	/*
+	 * All upper layers currently are never use transfer with more than
+	 * 2 messages. Actually, it's also not so relevant in Mellanox systems
+	 * because of HW limitation. Max size of transfer is not more than 32
+	 * bytes in the current x86 LPCI2C bridge.
+	 */
+	priv->xfer.cmd = msgs[num - 1].flags & I2C_M_RD;
+
+	if (priv->xfer.cmd == I2C_M_RD && comm_len != msgs[0].len) {
+		priv->xfer.addr_width = msgs[0].len;
+		priv->xfer.data_len = comm_len - priv->xfer.addr_width;
+	} else {
+		priv->xfer.addr_width = 0;
+		priv->xfer.data_len = comm_len;
+	}
+}
+
+/* Reset CPLD LPCI2C block */
+static void mlxcpld_i2c_reset(struct mlxcpld_i2c_priv *priv)
+{
+	u8 val;
+
+	mutex_lock(&priv->lock);
+
+	mlxcpld_i2c_read_comm(priv, MLXCPLD_LPCI2C_CTRL_REG, &val, 1);
+	val &= ~MLXCPLD_LPCI2C_RST_SEL_MASK;
+	mlxcpld_i2c_write_comm(priv, MLXCPLD_LPCI2C_CTRL_REG, &val, 1);
+
+	mutex_unlock(&priv->lock);
+}
+
+/* Make sure the CPLD is ready to start transmitting. */
+static int mlxcpld_i2c_check_busy(struct mlxcpld_i2c_priv *priv)
+{
+	u8 val;
+
+	mlxcpld_i2c_read_comm(priv, MLXCPLD_LPCI2C_STATUS_REG, &val, 1);
+
+	if (val & MLXCPLD_LPCI2C_TRANS_END)
+		return 0;
+
+	return -EIO;
+}
+
+static int mlxcpld_i2c_wait_for_free(struct mlxcpld_i2c_priv *priv)
+{
+	int timeout = 0;
+
+	do {
+		if (!mlxcpld_i2c_check_busy(priv))
+			break;
+		usleep_range(MLXCPLD_I2C_POLL_TIME / 2, MLXCPLD_I2C_POLL_TIME);
+		timeout += MLXCPLD_I2C_POLL_TIME;
+	} while (timeout <= MLXCPLD_I2C_XFER_TO);
+
+	if (timeout > MLXCPLD_I2C_XFER_TO)
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+/*
+ * Wait for master transfer to complete.
+ * It puts current process to sleep until we get interrupt or timeout expires.
+ * Returns the number of transferred or read bytes or error (<0).
+ */
+static int mlxcpld_i2c_wait_for_tc(struct mlxcpld_i2c_priv *priv)
+{
+	int status, i, timeout = 0;
+	u8 datalen;
+
+	do {
+		usleep_range(MLXCPLD_I2C_POLL_TIME / 2, MLXCPLD_I2C_POLL_TIME);
+		if (!mlxcpld_i2c_check_status(priv, &status))
+			break;
+		timeout += MLXCPLD_I2C_POLL_TIME;
+	} while (status == 0 && timeout < MLXCPLD_I2C_XFER_TO);
+
+	switch (status) {
+	case MLXCPLD_LPCI2C_NO_IND:
+		return -ETIMEDOUT;
+
+	case MLXCPLD_LPCI2C_ACK_IND:
+		if (priv->xfer.cmd != I2C_M_RD)
+			return (priv->xfer.addr_width + priv->xfer.data_len);
+
+		if (priv->xfer.msg_num == 1)
+			i = 0;
+		else
+			i = 1;
+
+		if (!priv->xfer.msg[i].buf)
+			return -EINVAL;
+
+		/*
+		 * Actual read data len will be always the same as
+		 * requested len. 0xff (line pull-up) will be returned
+		 * if slave has no data to return. Thus don't read
+		 * MLXCPLD_LPCI2C_NUM_DAT_REG reg from CPLD.
+		 */
+		datalen = priv->xfer.data_len;
+
+		mlxcpld_i2c_read_comm(priv, MLXCPLD_LPCI2C_DATA_REG,
+				      priv->xfer.msg[i].buf, datalen);
+
+		return datalen;
+
+	case MLXCPLD_LPCI2C_NACK_IND:
+		return -ENXIO;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static void mlxcpld_i2c_xfer_msg(struct mlxcpld_i2c_priv *priv)
+{
+	int i, len = 0;
+	u8 cmd;
+
+	mlxcpld_i2c_write_comm(priv, MLXCPLD_LPCI2C_NUM_DAT_REG,
+			       &priv->xfer.data_len, 1);
+	mlxcpld_i2c_write_comm(priv, MLXCPLD_LPCI2C_NUM_ADDR_REG,
+			       &priv->xfer.addr_width, 1);
+
+	for (i = 0; i < priv->xfer.msg_num; i++) {
+		if ((priv->xfer.msg[i].flags & I2C_M_RD) != I2C_M_RD) {
+			/* Don't write to CPLD buffer in read transaction */
+			mlxcpld_i2c_write_comm(priv, MLXCPLD_LPCI2C_DATA_REG +
+					       len, priv->xfer.msg[i].buf,
+					       priv->xfer.msg[i].len);
+			len += priv->xfer.msg[i].len;
+		}
+	}
+
+	/*
+	 * Set target slave address with command for master transfer.
+	 * It should be latest executed function before CPLD transaction.
+	 */
+	cmd = (priv->xfer.msg[0].addr << 1) | priv->xfer.cmd;
+	mlxcpld_i2c_write_comm(priv, MLXCPLD_LPCI2C_CMD_REG, &cmd, 1);
+}
+
+/*
+ * Generic lpc-i2c transfer.
+ * Returns the number of processed messages or error (<0).
+ */
+static int mlxcpld_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
+			    int num)
+{
+	struct mlxcpld_i2c_priv *priv = i2c_get_adapdata(adap);
+	u8 comm_len = 0;
+	int i, err;
+
+	err = mlxcpld_i2c_check_msg_params(priv, msgs, num);
+	if (err) {
+		dev_err(priv->dev, "Incorrect message\n");
+		return err;
+	}
+
+	for (i = 0; i < num; ++i)
+		comm_len += msgs[i].len;
+
+	/* Check bus state */
+	if (mlxcpld_i2c_wait_for_free(priv)) {
+		dev_err(priv->dev, "LPCI2C bridge is busy\n");
+
+		/*
+		 * Usually it means something serious has happened.
+		 * We can not have unfinished previous transfer
+		 * so it doesn't make any sense to try to stop it.
+		 * Probably we were not able to recover from the
+		 * previous error.
+		 * The only reasonable thing - is soft reset.
+		 */
+		mlxcpld_i2c_reset(priv);
+		if (mlxcpld_i2c_check_busy(priv)) {
+			dev_err(priv->dev, "LPCI2C bridge is busy after reset\n");
+			return -EIO;
+		}
+	}
+
+	mlxcpld_i2c_set_transf_data(priv, msgs, num, comm_len);
+
+	mutex_lock(&priv->lock);
+
+	/* Do real transfer. Can't fail */
+	mlxcpld_i2c_xfer_msg(priv);
+
+	/* Wait for transaction complete */
+	err = mlxcpld_i2c_wait_for_tc(priv);
+
+	mutex_unlock(&priv->lock);
+
+	return err < 0 ? err : num;
+}
+
+static u32 mlxcpld_i2c_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL | I2C_FUNC_SMBUS_BLOCK_DATA;
+}
+
+static const struct i2c_algorithm mlxcpld_i2c_algo = {
+	.master_xfer	= mlxcpld_i2c_xfer,
+	.functionality	= mlxcpld_i2c_func
+};
+
+static struct i2c_adapter_quirks mlxcpld_i2c_quirks = {
+	.flags = I2C_AQ_COMB_WRITE_THEN_READ,
+	.max_read_len = MLXCPLD_I2C_DATA_REG_SZ - MLXCPLD_I2C_MAX_ADDR_LEN,
+	.max_write_len = MLXCPLD_I2C_DATA_REG_SZ,
+	.max_comb_1st_msg_len = 4,
+};
+
+static struct i2c_adapter mlxcpld_i2c_adapter = {
+	.owner          = THIS_MODULE,
+	.name           = "i2c-mlxcpld",
+	.class          = I2C_CLASS_HWMON | I2C_CLASS_SPD,
+	.algo           = &mlxcpld_i2c_algo,
+	.quirks		= &mlxcpld_i2c_quirks,
+	.retries	= MLXCPLD_I2C_RETR_NUM,
+	.nr		= MLXCPLD_I2C_BUS_NUM,
+};
+
+static int mlxcpld_i2c_probe(struct platform_device *pdev)
+{
+	struct mlxcpld_i2c_priv *priv;
+	int err;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	mutex_init(&priv->lock);
+	platform_set_drvdata(pdev, priv);
+
+	priv->dev = &pdev->dev;
+
+	/* Register with i2c layer */
+	mlxcpld_i2c_adapter.timeout = usecs_to_jiffies(MLXCPLD_I2C_XFER_TO);
+	priv->adap = mlxcpld_i2c_adapter;
+	priv->adap.dev.parent = &pdev->dev;
+	priv->base_addr = MLXPLAT_CPLD_LPC_I2C_BASE_ADDR;
+	i2c_set_adapdata(&priv->adap, priv);
+
+	err = i2c_add_numbered_adapter(&priv->adap);
+	if (err)
+		mutex_destroy(&priv->lock);
+
+	return err;
+}
+
+static int mlxcpld_i2c_remove(struct platform_device *pdev)
+{
+	struct mlxcpld_i2c_priv *priv = platform_get_drvdata(pdev);
+
+	i2c_del_adapter(&priv->adap);
+	mutex_destroy(&priv->lock);
+
+	return 0;
+}
+
+static struct platform_driver mlxcpld_i2c_driver = {
+	.probe		= mlxcpld_i2c_probe,
+	.remove		= mlxcpld_i2c_remove,
+	.driver = {
+		.name = MLXCPLD_I2C_DEVICE_NAME,
+	},
+};
+
+module_platform_driver(mlxcpld_i2c_driver);
+
+MODULE_AUTHOR("Michael Shych <michaels@mellanox.com>");
+MODULE_DESCRIPTION("Mellanox I2C-CPLD controller driver");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_ALIAS("platform:i2c-mlxcpld");
diff --git a/drivers/i2c/busses/i2c-octeon-core.c b/drivers/i2c/busses/i2c-octeon-core.c
index 5e63b17f935d..3d10f1a802be 100644
--- a/drivers/i2c/busses/i2c-octeon-core.c
+++ b/drivers/i2c/busses/i2c-octeon-core.c
@@ -36,24 +36,6 @@ static bool octeon_i2c_test_iflg(struct octeon_i2c *i2c)
 	return (octeon_i2c_ctl_read(i2c) & TWSI_CTL_IFLG);
 }
 
-static bool octeon_i2c_test_ready(struct octeon_i2c *i2c, bool *first)
-{
-	if (octeon_i2c_test_iflg(i2c))
-		return true;
-
-	if (*first) {
-		*first = false;
-		return false;
-	}
-
-	/*
-	 * IRQ has signaled an event but IFLG hasn't changed.
-	 * Sleep and retry once.
-	 */
-	usleep_range(I2C_OCTEON_EVENT_WAIT, 2 * I2C_OCTEON_EVENT_WAIT);
-	return octeon_i2c_test_iflg(i2c);
-}
-
 /**
  * octeon_i2c_wait - wait for the IFLG to be set
  * @i2c: The struct octeon_i2c
@@ -63,7 +45,6 @@ static bool octeon_i2c_test_ready(struct octeon_i2c *i2c, bool *first)
 static int octeon_i2c_wait(struct octeon_i2c *i2c)
 {
 	long time_left;
-	bool first = true;
 
 	/*
 	 * Some chip revisions don't assert the irq in the interrupt
@@ -80,7 +61,7 @@ static int octeon_i2c_wait(struct octeon_i2c *i2c)
 	}
 
 	i2c->int_enable(i2c);
-	time_left = wait_event_timeout(i2c->queue, octeon_i2c_test_ready(i2c, &first),
+	time_left = wait_event_timeout(i2c->queue, octeon_i2c_test_iflg(i2c),
 				       i2c->adap.timeout);
 	i2c->int_disable(i2c);
 
@@ -102,25 +83,6 @@ static bool octeon_i2c_hlc_test_valid(struct octeon_i2c *i2c)
 	return (__raw_readq(i2c->twsi_base + SW_TWSI(i2c)) & SW_TWSI_V) == 0;
 }
 
-static bool octeon_i2c_hlc_test_ready(struct octeon_i2c *i2c, bool *first)
-{
-	/* check if valid bit is cleared */
-	if (octeon_i2c_hlc_test_valid(i2c))
-		return true;
-
-	if (*first) {
-		*first = false;
-		return false;
-	}
-
-	/*
-	 * IRQ has signaled an event but valid bit isn't cleared.
-	 * Sleep and retry once.
-	 */
-	usleep_range(I2C_OCTEON_EVENT_WAIT, 2 * I2C_OCTEON_EVENT_WAIT);
-	return octeon_i2c_hlc_test_valid(i2c);
-}
-
 static void octeon_i2c_hlc_int_clear(struct octeon_i2c *i2c)
 {
 	/* clear ST/TS events, listen for neither */
@@ -176,7 +138,6 @@ static void octeon_i2c_hlc_disable(struct octeon_i2c *i2c)
  */
 static int octeon_i2c_hlc_wait(struct octeon_i2c *i2c)
 {
-	bool first = true;
 	int time_left;
 
 	/*
@@ -195,7 +156,7 @@ static int octeon_i2c_hlc_wait(struct octeon_i2c *i2c)
 
 	i2c->hlc_int_enable(i2c);
 	time_left = wait_event_timeout(i2c->queue,
-				       octeon_i2c_hlc_test_ready(i2c, &first),
+				       octeon_i2c_hlc_test_valid(i2c),
 				       i2c->adap.timeout);
 	i2c->hlc_int_disable(i2c);
 	if (!time_left)
@@ -789,6 +750,9 @@ static void octeon_i2c_prepare_recovery(struct i2c_adapter *adap)
 	struct octeon_i2c *i2c = i2c_get_adapdata(adap);
 
 	octeon_i2c_hlc_disable(i2c);
+	octeon_i2c_reg_write(i2c, SW_TWSI_EOP_TWSI_RST, 0);
+	/* wait for software reset to settle */
+	udelay(5);
 
 	/*
 	 * Bring control register to a good state regardless
diff --git a/drivers/i2c/busses/i2c-pxa-pci.c b/drivers/i2c/busses/i2c-pxa-pci.c
index 417464e9ea2a..004deb96afe3 100644
--- a/drivers/i2c/busses/i2c-pxa-pci.c
+++ b/drivers/i2c/busses/i2c-pxa-pci.c
@@ -1,9 +1,13 @@
 /*
+ * CE4100 PCI-I2C glue code for PXA's driver
+ * Author: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
+ * License: GPL v2
+ *
  * The CE4100's I2C device is more or less the same one as found on PXA.
  * It does not support slave mode, the register slightly moved. This PCI
  * device provides three bars, every contains a single I2C controller.
  */
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
 #include <linux/i2c/pxa-i2c.h>
@@ -134,35 +138,17 @@ err_mem:
 	return ret;
 }
 
-static void ce4100_i2c_remove(struct pci_dev *dev)
-{
-	struct ce4100_devices *sds;
-	unsigned int i;
-
-	sds = pci_get_drvdata(dev);
-
-	for (i = 0; i < ARRAY_SIZE(sds->pdev); i++)
-		platform_device_unregister(sds->pdev[i]);
-
-	pci_disable_device(dev);
-	kfree(sds);
-}
-
 static const struct pci_device_id ce4100_i2c_devices[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2e68)},
 	{ },
 };
-MODULE_DEVICE_TABLE(pci, ce4100_i2c_devices);
 
 static struct pci_driver ce4100_i2c_driver = {
+	.driver = {
+		.suppress_bind_attrs = true,
+	},
 	.name           = "ce4100_i2c",
 	.id_table       = ce4100_i2c_devices,
 	.probe          = ce4100_i2c_probe,
-	.remove         = ce4100_i2c_remove,
 };
-
-module_pci_driver(ce4100_i2c_driver);
-
-MODULE_DESCRIPTION("CE4100 PCI-I2C glue code for PXA's driver");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Sebastian Andrzej Siewior <bigeasy@linutronix.de>");
+builtin_pci_driver(ce4100_i2c_driver);
diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c
index e28b825b0433..6cf333ecc8b8 100644
--- a/drivers/i2c/busses/i2c-pxa.c
+++ b/drivers/i2c/busses/i2c-pxa.c
@@ -48,6 +48,8 @@ struct pxa_reg_layout {
 	u32 isar;
 	u32 ilcr;
 	u32 iwcr;
+	u32 fm;
+	u32 hs;
 };
 
 enum pxa_i2c_types {
@@ -55,8 +57,12 @@ enum pxa_i2c_types {
 	REGS_PXA3XX,
 	REGS_CE4100,
 	REGS_PXA910,
+	REGS_A3700,
 };
 
+#define ICR_BUSMODE_FM	(1 << 16)	   /* shifted fast mode for armada-3700 */
+#define ICR_BUSMODE_HS	(1 << 17)	   /* shifted high speed mode for armada-3700 */
+
 /*
  * I2C registers definitions
  */
@@ -91,6 +97,15 @@ static struct pxa_reg_layout pxa_reg_layout[] = {
 		.ilcr = 0x28,
 		.iwcr = 0x30,
 	},
+	[REGS_A3700] = {
+		.ibmr =	0x00,
+		.idbr =	0x04,
+		.icr =	0x08,
+		.isr =	0x0c,
+		.isar =	0x10,
+		.fm = ICR_BUSMODE_FM,
+		.hs = ICR_BUSMODE_HS,
+	},
 };
 
 static const struct platform_device_id i2c_pxa_id_table[] = {
@@ -98,6 +113,7 @@ static const struct platform_device_id i2c_pxa_id_table[] = {
 	{ "pxa3xx-pwri2c",	REGS_PXA3XX },
 	{ "ce4100-i2c",		REGS_CE4100 },
 	{ "pxa910-i2c",		REGS_PXA910 },
+	{ "armada-3700-i2c",	REGS_A3700  },
 	{ },
 };
 MODULE_DEVICE_TABLE(platform, i2c_pxa_id_table);
@@ -193,6 +209,8 @@ struct pxa_i2c {
 	unsigned char		master_code;
 	unsigned long		rate;
 	bool			highmode_enter;
+	u32			fm_mask;
+	u32			hs_mask;
 };
 
 #define _IBMR(i2c)	((i2c)->reg_ibmr)
@@ -503,8 +521,8 @@ static void i2c_pxa_reset(struct pxa_i2c *i2c)
 		writel(i2c->slave_addr, _ISAR(i2c));
 
 	/* set control register values */
-	writel(I2C_ICR_INIT | (i2c->fast_mode ? ICR_FM : 0), _ICR(i2c));
-	writel(readl(_ICR(i2c)) | (i2c->high_mode ? ICR_HS : 0), _ICR(i2c));
+	writel(I2C_ICR_INIT | (i2c->fast_mode ? i2c->fm_mask : 0), _ICR(i2c));
+	writel(readl(_ICR(i2c)) | (i2c->high_mode ? i2c->hs_mask : 0), _ICR(i2c));
 
 #ifdef CONFIG_I2C_PXA_SLAVE
 	dev_info(&i2c->adap.dev, "Enabling slave mode\n");
@@ -1137,6 +1155,7 @@ static const struct of_device_id i2c_pxa_dt_ids[] = {
 	{ .compatible = "mrvl,pxa-i2c", .data = (void *)REGS_PXA2XX },
 	{ .compatible = "mrvl,pwri2c", .data = (void *)REGS_PXA3XX },
 	{ .compatible = "mrvl,mmp-twsi", .data = (void *)REGS_PXA910 },
+	{ .compatible = "marvell,armada-3700-i2c", .data = (void *)REGS_A3700 },
 	{}
 };
 MODULE_DEVICE_TABLE(of, i2c_pxa_dt_ids);
@@ -1234,6 +1253,9 @@ static int i2c_pxa_probe(struct platform_device *dev)
 	i2c->reg_idbr = i2c->reg_base + pxa_reg_layout[i2c_type].idbr;
 	i2c->reg_icr = i2c->reg_base + pxa_reg_layout[i2c_type].icr;
 	i2c->reg_isr = i2c->reg_base + pxa_reg_layout[i2c_type].isr;
+	i2c->fm_mask = pxa_reg_layout[i2c_type].fm ? : ICR_FM;
+	i2c->hs_mask = pxa_reg_layout[i2c_type].hs ? : ICR_HS;
+
 	if (i2c_type != REGS_CE4100)
 		i2c->reg_isar = i2c->reg_base + pxa_reg_layout[i2c_type].isar;
 
diff --git a/drivers/i2c/busses/i2c-qup.c b/drivers/i2c/busses/i2c-qup.c
index a8497cfdae6f..1902d8ac9753 100644
--- a/drivers/i2c/busses/i2c-qup.c
+++ b/drivers/i2c/busses/i2c-qup.c
@@ -14,6 +14,7 @@
  *
  */
 
+#include <linux/acpi.h>
 #include <linux/atomic.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
@@ -132,6 +133,10 @@
 /* Max timeout in ms for 32k bytes */
 #define TOUT_MAX			300
 
+/* Default values. Use these if FW query fails */
+#define DEFAULT_CLK_FREQ 100000
+#define DEFAULT_SRC_CLK 20000000
+
 struct qup_i2c_block {
 	int	count;
 	int	pos;
@@ -525,6 +530,33 @@ static int qup_i2c_get_data_len(struct qup_i2c_dev *qup)
 	return data_len;
 }
 
+static bool qup_i2c_check_msg_len(struct i2c_msg *msg)
+{
+	return ((msg->flags & I2C_M_RD) && (msg->flags & I2C_M_RECV_LEN));
+}
+
+static int qup_i2c_set_tags_smb(u16 addr, u8 *tags, struct qup_i2c_dev *qup,
+			struct i2c_msg *msg)
+{
+	int len = 0;
+
+	if (msg->len > 1) {
+		tags[len++] = QUP_TAG_V2_DATARD_STOP;
+		tags[len++] = qup_i2c_get_data_len(qup) - 1;
+	} else {
+		tags[len++] = QUP_TAG_V2_START;
+		tags[len++] = addr & 0xff;
+
+		if (msg->flags & I2C_M_TEN)
+			tags[len++] = addr >> 8;
+
+		tags[len++] = QUP_TAG_V2_DATARD;
+		/* Read 1 byte indicating the length of the SMBus message */
+		tags[len++] = 1;
+	}
+	return len;
+}
+
 static int qup_i2c_set_tags(u8 *tags, struct qup_i2c_dev *qup,
 			    struct i2c_msg *msg,  int is_dma)
 {
@@ -534,6 +566,10 @@ static int qup_i2c_set_tags(u8 *tags, struct qup_i2c_dev *qup,
 
 	int last = (qup->blk.pos == (qup->blk.count - 1)) && (qup->is_last);
 
+	/* Handle tags for SMBus block read */
+	if (qup_i2c_check_msg_len(msg))
+		return qup_i2c_set_tags_smb(addr, tags, qup, msg);
+
 	if (qup->blk.pos == 0) {
 		tags[len++] = QUP_TAG_V2_START;
 		tags[len++] = addr & 0xff;
@@ -1056,9 +1092,17 @@ static int qup_i2c_read_fifo_v2(struct qup_i2c_dev *qup,
 				struct i2c_msg *msg)
 {
 	u32 val;
-	int idx, pos = 0, ret = 0, total;
+	int idx, pos = 0, ret = 0, total, msg_offset = 0;
 
+	/*
+	 * If the message length is already read in
+	 * the first byte of the buffer, account for
+	 * that by setting the offset
+	 */
+	if (qup_i2c_check_msg_len(msg) && (msg->len > 1))
+		msg_offset = 1;
 	total = qup_i2c_get_data_len(qup);
+	total -= msg_offset;
 
 	/* 2 extra bytes for read tags */
 	while (pos < (total + 2)) {
@@ -1078,8 +1122,8 @@ static int qup_i2c_read_fifo_v2(struct qup_i2c_dev *qup,
 
 			if (pos >= (total + 2))
 				goto out;
-
-			msg->buf[qup->pos++] = val & 0xff;
+			msg->buf[qup->pos + msg_offset] = val & 0xff;
+			qup->pos++;
 		}
 	}
 
@@ -1119,6 +1163,20 @@ static int qup_i2c_read_one_v2(struct qup_i2c_dev *qup, struct i2c_msg *msg)
 			goto err;
 
 		qup->blk.pos++;
+
+		/* Handle SMBus block read length */
+		if (qup_i2c_check_msg_len(msg) && (msg->len == 1)) {
+			if (msg->buf[0] > I2C_SMBUS_BLOCK_MAX) {
+				ret = -EPROTO;
+				goto err;
+			}
+			msg->len += msg->buf[0];
+			qup->pos = 0;
+			qup_i2c_set_blk_data(qup, msg);
+			/* set tag length for block read */
+			qup->blk.tx_tag_len = 2;
+			qup_i2c_set_read_mode_v2(qup, msg->buf[0]);
+		}
 	} while (qup->blk.pos < qup->blk.count);
 
 err:
@@ -1204,6 +1262,11 @@ static int qup_i2c_xfer(struct i2c_adapter *adap,
 			goto out;
 		}
 
+		if (qup_i2c_check_msg_len(&msgs[idx])) {
+			ret = -EINVAL;
+			goto out;
+		}
+
 		if (msgs[idx].flags & I2C_M_RD)
 			ret = qup_i2c_read_one(qup, &msgs[idx]);
 		else
@@ -1358,14 +1421,13 @@ static void qup_i2c_disable_clocks(struct qup_i2c_dev *qup)
 static int qup_i2c_probe(struct platform_device *pdev)
 {
 	static const int blk_sizes[] = {4, 16, 32};
-	struct device_node *node = pdev->dev.of_node;
 	struct qup_i2c_dev *qup;
 	unsigned long one_bit_t;
 	struct resource *res;
 	u32 io_mode, hw_ver, size;
 	int ret, fs_div, hs_div;
-	int src_clk_freq;
-	u32 clk_freq = 100000;
+	u32 src_clk_freq = DEFAULT_SRC_CLK;
+	u32 clk_freq = DEFAULT_CLK_FREQ;
 	int blocks;
 
 	qup = devm_kzalloc(&pdev->dev, sizeof(*qup), GFP_KERNEL);
@@ -1376,7 +1438,11 @@ static int qup_i2c_probe(struct platform_device *pdev)
 	init_completion(&qup->xfer);
 	platform_set_drvdata(pdev, qup);
 
-	of_property_read_u32(node, "clock-frequency", &clk_freq);
+	ret = device_property_read_u32(qup->dev, "clock-frequency", &clk_freq);
+	if (ret) {
+		dev_notice(qup->dev, "using default clock-frequency %d",
+			DEFAULT_CLK_FREQ);
+	}
 
 	if (of_device_is_compatible(pdev->dev.of_node, "qcom,i2c-qup-v1.1.1")) {
 		qup->adap.algo = &qup_i2c_algo;
@@ -1452,20 +1518,30 @@ nodma:
 		return qup->irq;
 	}
 
-	qup->clk = devm_clk_get(qup->dev, "core");
-	if (IS_ERR(qup->clk)) {
-		dev_err(qup->dev, "Could not get core clock\n");
-		return PTR_ERR(qup->clk);
-	}
+	if (has_acpi_companion(qup->dev)) {
+		ret = device_property_read_u32(qup->dev,
+				"src-clock-hz", &src_clk_freq);
+		if (ret) {
+			dev_notice(qup->dev, "using default src-clock-hz %d",
+				DEFAULT_SRC_CLK);
+		}
+		ACPI_COMPANION_SET(&qup->adap.dev, ACPI_COMPANION(qup->dev));
+	} else {
+		qup->clk = devm_clk_get(qup->dev, "core");
+		if (IS_ERR(qup->clk)) {
+			dev_err(qup->dev, "Could not get core clock\n");
+			return PTR_ERR(qup->clk);
+		}
 
-	qup->pclk = devm_clk_get(qup->dev, "iface");
-	if (IS_ERR(qup->pclk)) {
-		dev_err(qup->dev, "Could not get iface clock\n");
-		return PTR_ERR(qup->pclk);
+		qup->pclk = devm_clk_get(qup->dev, "iface");
+		if (IS_ERR(qup->pclk)) {
+			dev_err(qup->dev, "Could not get iface clock\n");
+			return PTR_ERR(qup->pclk);
+		}
+		qup_i2c_enable_clocks(qup);
+		src_clk_freq = clk_get_rate(qup->clk);
 	}
 
-	qup_i2c_enable_clocks(qup);
-
 	/*
 	 * Bootloaders might leave a pending interrupt on certain QUP's,
 	 * so we reset the core before registering for interrupts.
@@ -1512,7 +1588,6 @@ nodma:
 	size = QUP_INPUT_FIFO_SIZE(io_mode);
 	qup->in_fifo_sz = qup->in_blk_sz * (2 << size);
 
-	src_clk_freq = clk_get_rate(qup->clk);
 	fs_div = ((src_clk_freq / clk_freq) / 2) - 3;
 	hs_div = 3;
 	qup->clk_ctl = (hs_div << 8) | (fs_div & 0xff);
@@ -1631,6 +1706,14 @@ static const struct of_device_id qup_i2c_dt_match[] = {
 };
 MODULE_DEVICE_TABLE(of, qup_i2c_dt_match);
 
+#if IS_ENABLED(CONFIG_ACPI)
+static const struct acpi_device_id qup_i2c_acpi_match[] = {
+	{ "QCOM8010"},
+	{ },
+};
+MODULE_DEVICE_TABLE(acpi, qup_i2c_acpi_match);
+#endif
+
 static struct platform_driver qup_i2c_driver = {
 	.probe  = qup_i2c_probe,
 	.remove = qup_i2c_remove,
@@ -1638,6 +1721,7 @@ static struct platform_driver qup_i2c_driver = {
 		.name = "i2c_qup",
 		.pm = &qup_i2c_qup_pm_ops,
 		.of_match_table = qup_i2c_dt_match,
+		.acpi_match_table = ACPI_PTR(qup_i2c_acpi_match),
 	},
 };
 
diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c
index 726615e54f2a..26f2ff22e97e 100644
--- a/drivers/i2c/busses/i2c-rcar.c
+++ b/drivers/i2c/busses/i2c-rcar.c
@@ -793,7 +793,6 @@ static const struct i2c_algorithm rcar_i2c_algo = {
 };
 
 static const struct of_device_id rcar_i2c_dt_ids[] = {
-	{ .compatible = "renesas,i2c-rcar", .data = (void *)I2C_RCAR_GEN1 },
 	{ .compatible = "renesas,i2c-r8a7778", .data = (void *)I2C_RCAR_GEN1 },
 	{ .compatible = "renesas,i2c-r8a7779", .data = (void *)I2C_RCAR_GEN1 },
 	{ .compatible = "renesas,i2c-r8a7790", .data = (void *)I2C_RCAR_GEN2 },
@@ -803,6 +802,10 @@ static const struct of_device_id rcar_i2c_dt_ids[] = {
 	{ .compatible = "renesas,i2c-r8a7794", .data = (void *)I2C_RCAR_GEN2 },
 	{ .compatible = "renesas,i2c-r8a7795", .data = (void *)I2C_RCAR_GEN3 },
 	{ .compatible = "renesas,i2c-r8a7796", .data = (void *)I2C_RCAR_GEN3 },
+	{ .compatible = "renesas,i2c-rcar", .data = (void *)I2C_RCAR_GEN1 },	/* Deprecated */
+	{ .compatible = "renesas,rcar-gen1-i2c", .data = (void *)I2C_RCAR_GEN1 },
+	{ .compatible = "renesas,rcar-gen2-i2c", .data = (void *)I2C_RCAR_GEN2 },
+	{ .compatible = "renesas,rcar-gen3-i2c", .data = (void *)I2C_RCAR_GEN3 },
 	{},
 };
 MODULE_DEVICE_TABLE(of, rcar_i2c_dt_ids);
diff --git a/drivers/i2c/busses/i2c-sh_mobile.c b/drivers/i2c/busses/i2c-sh_mobile.c
index 192f36f00e4d..3d9ebe6e5716 100644
--- a/drivers/i2c/busses/i2c-sh_mobile.c
+++ b/drivers/i2c/busses/i2c-sh_mobile.c
@@ -827,7 +827,6 @@ static const struct sh_mobile_dt_config r8a7740_dt_config = {
 };
 
 static const struct of_device_id sh_mobile_i2c_dt_ids[] = {
-	{ .compatible = "renesas,rmobile-iic", .data = &default_dt_config },
 	{ .compatible = "renesas,iic-r8a73a4", .data = &fast_clock_dt_config },
 	{ .compatible = "renesas,iic-r8a7740", .data = &r8a7740_dt_config },
 	{ .compatible = "renesas,iic-r8a7790", .data = &fast_clock_dt_config },
@@ -835,8 +834,11 @@ static const struct of_device_id sh_mobile_i2c_dt_ids[] = {
 	{ .compatible = "renesas,iic-r8a7792", .data = &fast_clock_dt_config },
 	{ .compatible = "renesas,iic-r8a7793", .data = &fast_clock_dt_config },
 	{ .compatible = "renesas,iic-r8a7794", .data = &fast_clock_dt_config },
+	{ .compatible = "renesas,rcar-gen2-iic", .data = &fast_clock_dt_config },
 	{ .compatible = "renesas,iic-r8a7795", .data = &fast_clock_dt_config },
+	{ .compatible = "renesas,rcar-gen3-iic", .data = &fast_clock_dt_config },
 	{ .compatible = "renesas,iic-sh73a0", .data = &fast_clock_dt_config },
+	{ .compatible = "renesas,rmobile-iic", .data = &default_dt_config },
 	{},
 };
 MODULE_DEVICE_TABLE(of, sh_mobile_i2c_dt_ids);
diff --git a/drivers/i2c/busses/i2c-uniphier-f.c b/drivers/i2c/busses/i2c-uniphier-f.c
index db9105e52c79..beee31892295 100644
--- a/drivers/i2c/busses/i2c-uniphier-f.c
+++ b/drivers/i2c/busses/i2c-uniphier-f.c
@@ -528,7 +528,7 @@ static int uniphier_fi2c_probe(struct platform_device *pdev)
 	if (!clk_rate) {
 		dev_err(dev, "input clock rate should not be zero\n");
 		ret = -EINVAL;
-		goto err;
+		goto disable_clk;
 	}
 
 	init_completion(&priv->comp);
@@ -547,11 +547,11 @@ static int uniphier_fi2c_probe(struct platform_device *pdev)
 			       pdev->name, priv);
 	if (ret) {
 		dev_err(dev, "failed to request irq %d\n", irq);
-		goto err;
+		goto disable_clk;
 	}
 
 	ret = i2c_add_adapter(&priv->adap);
-err:
+disable_clk:
 	if (ret)
 		clk_disable_unprepare(priv->clk);
 
diff --git a/drivers/i2c/busses/i2c-uniphier.c b/drivers/i2c/busses/i2c-uniphier.c
index 56e92af46ddc..777c0fe93653 100644
--- a/drivers/i2c/busses/i2c-uniphier.c
+++ b/drivers/i2c/busses/i2c-uniphier.c
@@ -373,7 +373,7 @@ static int uniphier_i2c_probe(struct platform_device *pdev)
 	if (!clk_rate) {
 		dev_err(dev, "input clock rate should not be zero\n");
 		ret = -EINVAL;
-		goto err;
+		goto disable_clk;
 	}
 
 	init_completion(&priv->comp);
@@ -392,11 +392,11 @@ static int uniphier_i2c_probe(struct platform_device *pdev)
 			       priv);
 	if (ret) {
 		dev_err(dev, "failed to request irq %d\n", irq);
-		goto err;
+		goto disable_clk;
 	}
 
 	ret = i2c_add_adapter(&priv->adap);
-err:
+disable_clk:
 	if (ret)
 		clk_disable_unprepare(priv->clk);
 
diff --git a/drivers/i2c/busses/i2c-viperboard.c b/drivers/i2c/busses/i2c-viperboard.c
index 543456a0a338..e4be86b3de9a 100644
--- a/drivers/i2c/busses/i2c-viperboard.c
+++ b/drivers/i2c/busses/i2c-viperboard.c
@@ -354,7 +354,7 @@ static const struct i2c_algorithm vprbrd_algorithm = {
 	.functionality	= vprbrd_i2c_func,
 };
 
-static struct i2c_adapter_quirks vprbrd_quirks = {
+static const struct i2c_adapter_quirks vprbrd_quirks = {
 	.max_read_len = 2048,
 	.max_write_len = 2048,
 };
diff --git a/drivers/i2c/busses/i2c-xlp9xx.c b/drivers/i2c/busses/i2c-xlp9xx.c
index e29ff37a43bd..84a8b2eccffb 100644
--- a/drivers/i2c/busses/i2c-xlp9xx.c
+++ b/drivers/i2c/busses/i2c-xlp9xx.c
@@ -393,6 +393,7 @@ static int xlp9xx_i2c_probe(struct platform_device *pdev)
 	init_completion(&priv->msg_complete);
 	priv->adapter.dev.parent = &pdev->dev;
 	priv->adapter.algo = &xlp9xx_i2c_algo;
+	ACPI_COMPANION_SET(&priv->adapter.dev, ACPI_COMPANION(&pdev->dev));
 	priv->adapter.dev.of_node = pdev->dev.of_node;
 	priv->dev = &pdev->dev;
 
diff --git a/drivers/i2c/i2c-core.c b/drivers/i2c/i2c-core.c
index b432b64e307a..3de95a29024c 100644
--- a/drivers/i2c/i2c-core.c
+++ b/drivers/i2c/i2c-core.c
@@ -65,6 +65,9 @@
 #define I2C_ADDR_OFFSET_TEN_BIT	0xa000
 #define I2C_ADDR_OFFSET_SLAVE	0x1000
 
+#define I2C_ADDR_7BITS_MAX	0x77
+#define I2C_ADDR_7BITS_COUNT	(I2C_ADDR_7BITS_MAX + 1)
+
 /* core_lock protects i2c_adapter_idr, and guarantees
    that device detection, deletion of detected devices, and attach_adapter
    calls are serialized */
@@ -77,9 +80,10 @@ static int i2c_detect(struct i2c_adapter *adapter, struct i2c_driver *driver);
 static struct static_key i2c_trace_msg = STATIC_KEY_INIT_FALSE;
 static bool is_registered;
 
-void i2c_transfer_trace_reg(void)
+int i2c_transfer_trace_reg(void)
 {
 	static_key_slow_inc(&i2c_trace_msg);
+	return 0;
 }
 
 void i2c_transfer_trace_unreg(void)
@@ -676,9 +680,12 @@ static inline int i2c_acpi_install_space_handler(struct i2c_adapter *adapter)
 
 /* ------------------------------------------------------------------------- */
 
-static const struct i2c_device_id *i2c_match_id(const struct i2c_device_id *id,
+const struct i2c_device_id *i2c_match_id(const struct i2c_device_id *id,
 						const struct i2c_client *client)
 {
+	if (!(id && client))
+		return NULL;
+
 	while (id->name[0]) {
 		if (strcmp(client->name, id->name) == 0)
 			return id;
@@ -686,17 +693,16 @@ static const struct i2c_device_id *i2c_match_id(const struct i2c_device_id *id,
 	}
 	return NULL;
 }
+EXPORT_SYMBOL_GPL(i2c_match_id);
 
 static int i2c_device_match(struct device *dev, struct device_driver *drv)
 {
 	struct i2c_client	*client = i2c_verify_client(dev);
 	struct i2c_driver	*driver;
 
-	if (!client)
-		return 0;
 
 	/* Attempt an OF style match */
-	if (of_driver_match_device(dev, drv))
+	if (i2c_of_match_device(drv->of_match_table, client))
 		return 1;
 
 	/* Then ACPI style match */
@@ -704,9 +710,10 @@ static int i2c_device_match(struct device *dev, struct device_driver *drv)
 		return 1;
 
 	driver = to_i2c_driver(drv);
-	/* match on an id table if there is one */
-	if (driver->id_table)
-		return i2c_match_id(driver->id_table, client) != NULL;
+
+	/* Finally an I2C match */
+	if (i2c_match_id(driver->id_table, client))
+		return 1;
 
 	return 0;
 }
@@ -893,6 +900,25 @@ static void i2c_init_recovery(struct i2c_adapter *adap)
 	adap->bus_recovery_info = NULL;
 }
 
+static int i2c_smbus_host_notify_to_irq(const struct i2c_client *client)
+{
+	struct i2c_adapter *adap = client->adapter;
+	unsigned int irq;
+
+	if (!adap->host_notify_domain)
+		return -ENXIO;
+
+	if (client->flags & I2C_CLIENT_TEN)
+		return -EINVAL;
+
+	irq = irq_find_mapping(adap->host_notify_domain, client->addr);
+	if (!irq)
+		irq = irq_create_mapping(adap->host_notify_domain,
+					 client->addr);
+
+	return irq > 0 ? irq : -ENXIO;
+}
+
 static int i2c_device_probe(struct device *dev)
 {
 	struct i2c_client	*client = i2c_verify_client(dev);
@@ -914,6 +940,14 @@ static int i2c_device_probe(struct device *dev)
 		}
 		if (irq == -EPROBE_DEFER)
 			return irq;
+		/*
+		 * ACPI and OF did not find any useful IRQ, try to see
+		 * if Host Notify can be used.
+		 */
+		if (irq < 0) {
+			dev_dbg(dev, "Using Host Notify IRQ\n");
+			irq = i2c_smbus_host_notify_to_irq(client);
+		}
 		if (irq < 0)
 			irq = 0;
 
@@ -921,7 +955,13 @@ static int i2c_device_probe(struct device *dev)
 	}
 
 	driver = to_i2c_driver(dev->driver);
-	if (!driver->probe || !driver->id_table)
+
+	/*
+	 * An I2C ID table is not mandatory, if and only if, a suitable Device
+	 * Tree match table entry is supplied for the probing device.
+	 */
+	if (!driver->id_table &&
+	    !i2c_of_match_device(dev->driver->of_match_table, client))
 		return -ENODEV;
 
 	if (client->flags & I2C_CLIENT_WAKE) {
@@ -956,7 +996,18 @@ static int i2c_device_probe(struct device *dev)
 	if (status == -EPROBE_DEFER)
 		goto err_clear_wakeup_irq;
 
-	status = driver->probe(client, i2c_match_id(driver->id_table, client));
+	/*
+	 * When there are no more users of probe(),
+	 * rename probe_new to probe.
+	 */
+	if (driver->probe_new)
+		status = driver->probe_new(client);
+	else if (driver->probe)
+		status = driver->probe(client,
+				       i2c_match_id(driver->id_table, client));
+	else
+		status = -EINVAL;
+
 	if (status)
 		goto err_detach_pm_domain;
 
@@ -1767,6 +1818,52 @@ struct i2c_adapter *of_get_i2c_adapter_by_node(struct device_node *node)
 	return adapter;
 }
 EXPORT_SYMBOL(of_get_i2c_adapter_by_node);
+
+static const struct of_device_id*
+i2c_of_match_device_sysfs(const struct of_device_id *matches,
+				  struct i2c_client *client)
+{
+	const char *name;
+
+	for (; matches->compatible[0]; matches++) {
+		/*
+		 * Adding devices through the i2c sysfs interface provides us
+		 * a string to match which may be compatible with the device
+		 * tree compatible strings, however with no actual of_node the
+		 * of_match_device() will not match
+		 */
+		if (sysfs_streq(client->name, matches->compatible))
+			return matches;
+
+		name = strchr(matches->compatible, ',');
+		if (!name)
+			name = matches->compatible;
+		else
+			name++;
+
+		if (sysfs_streq(client->name, name))
+			return matches;
+	}
+
+	return NULL;
+}
+
+const struct of_device_id
+*i2c_of_match_device(const struct of_device_id *matches,
+		     struct i2c_client *client)
+{
+	const struct of_device_id *match;
+
+	if (!(client && matches))
+		return NULL;
+
+	match = of_match_device(matches, &client->dev);
+	if (match)
+		return match;
+
+	return i2c_of_match_device_sysfs(matches, client);
+}
+EXPORT_SYMBOL_GPL(i2c_of_match_device);
 #else
 static void of_i2c_register_devices(struct i2c_adapter *adap) { }
 #endif /* CONFIG_OF */
@@ -1800,6 +1897,79 @@ static const struct i2c_lock_operations i2c_adapter_lock_ops = {
 	.unlock_bus =  i2c_adapter_unlock_bus,
 };
 
+static void i2c_host_notify_irq_teardown(struct i2c_adapter *adap)
+{
+	struct irq_domain *domain = adap->host_notify_domain;
+	irq_hw_number_t hwirq;
+
+	if (!domain)
+		return;
+
+	for (hwirq = 0 ; hwirq < I2C_ADDR_7BITS_COUNT ; hwirq++)
+		irq_dispose_mapping(irq_find_mapping(domain, hwirq));
+
+	irq_domain_remove(domain);
+	adap->host_notify_domain = NULL;
+}
+
+static int i2c_host_notify_irq_map(struct irq_domain *h,
+					  unsigned int virq,
+					  irq_hw_number_t hw_irq_num)
+{
+	irq_set_chip_and_handler(virq, &dummy_irq_chip, handle_simple_irq);
+
+	return 0;
+}
+
+static const struct irq_domain_ops i2c_host_notify_irq_ops = {
+	.map = i2c_host_notify_irq_map,
+};
+
+static int i2c_setup_host_notify_irq_domain(struct i2c_adapter *adap)
+{
+	struct irq_domain *domain;
+
+	if (!i2c_check_functionality(adap, I2C_FUNC_SMBUS_HOST_NOTIFY))
+		return 0;
+
+	domain = irq_domain_create_linear(adap->dev.fwnode,
+					  I2C_ADDR_7BITS_COUNT,
+					  &i2c_host_notify_irq_ops, adap);
+	if (!domain)
+		return -ENOMEM;
+
+	adap->host_notify_domain = domain;
+
+	return 0;
+}
+
+/**
+ * i2c_handle_smbus_host_notify - Forward a Host Notify event to the correct
+ * I2C client.
+ * @adap: the adapter
+ * @addr: the I2C address of the notifying device
+ * Context: can't sleep
+ *
+ * Helper function to be called from an I2C bus driver's interrupt
+ * handler. It will schedule the Host Notify IRQ.
+ */
+int i2c_handle_smbus_host_notify(struct i2c_adapter *adap, unsigned short addr)
+{
+	int irq;
+
+	if (!adap)
+		return -EINVAL;
+
+	irq = irq_find_mapping(adap->host_notify_domain, addr);
+	if (irq <= 0)
+		return -ENXIO;
+
+	generic_handle_irq(irq);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(i2c_handle_smbus_host_notify);
+
 static int i2c_register_adapter(struct i2c_adapter *adap)
 {
 	int res = -EINVAL;
@@ -1831,6 +2001,14 @@ static int i2c_register_adapter(struct i2c_adapter *adap)
 	if (adap->timeout == 0)
 		adap->timeout = HZ;
 
+	/* register soft irqs for Host Notify */
+	res = i2c_setup_host_notify_irq_domain(adap);
+	if (res) {
+		pr_err("adapter '%s': can't create Host Notify IRQs (%d)\n",
+		       adap->name, res);
+		goto out_list;
+	}
+
 	dev_set_name(&adap->dev, "i2c-%d", adap->nr);
 	adap->dev.bus = &i2c_bus_type;
 	adap->dev.type = &i2c_adapter_type;
@@ -2068,6 +2246,8 @@ void i2c_del_adapter(struct i2c_adapter *adap)
 
 	pm_runtime_disable(&adap->dev);
 
+	i2c_host_notify_irq_teardown(adap);
+
 	/* wait until all references to the device are gone
 	 *
 	 * FIXME: This is old code and should ideally be replaced by an
diff --git a/drivers/i2c/i2c-smbus.c b/drivers/i2c/i2c-smbus.c
index b0d2679c60d1..f9271c713d20 100644
--- a/drivers/i2c/i2c-smbus.c
+++ b/drivers/i2c/i2c-smbus.c
@@ -241,108 +241,6 @@ int i2c_handle_smbus_alert(struct i2c_client *ara)
 }
 EXPORT_SYMBOL_GPL(i2c_handle_smbus_alert);
 
-static void smbus_host_notify_work(struct work_struct *work)
-{
-	struct alert_data alert;
-	struct i2c_adapter *adapter;
-	unsigned long flags;
-	u16 payload;
-	u8 addr;
-	struct smbus_host_notify *data;
-
-	data = container_of(work, struct smbus_host_notify, work);
-
-	spin_lock_irqsave(&data->lock, flags);
-	payload = data->payload;
-	addr = data->addr;
-	adapter = data->adapter;
-
-	/* clear the pending bit and release the spinlock */
-	data->pending = false;
-	spin_unlock_irqrestore(&data->lock, flags);
-
-	if (!adapter || !addr)
-		return;
-
-	alert.type = I2C_PROTOCOL_SMBUS_HOST_NOTIFY;
-	alert.addr = addr;
-	alert.data = payload;
-
-	device_for_each_child(&adapter->dev, &alert, smbus_do_alert);
-}
-
-/**
- * i2c_setup_smbus_host_notify - Allocate a new smbus_host_notify for the given
- * I2C adapter.
- * @adapter: the adapter we want to associate a Host Notify function
- *
- * Returns a struct smbus_host_notify pointer on success, and NULL on failure.
- * The resulting smbus_host_notify must not be freed afterwards, it is a
- * managed resource already.
- */
-struct smbus_host_notify *i2c_setup_smbus_host_notify(struct i2c_adapter *adap)
-{
-	struct smbus_host_notify *host_notify;
-
-	host_notify = devm_kzalloc(&adap->dev, sizeof(struct smbus_host_notify),
-				   GFP_KERNEL);
-	if (!host_notify)
-		return NULL;
-
-	host_notify->adapter = adap;
-
-	spin_lock_init(&host_notify->lock);
-	INIT_WORK(&host_notify->work, smbus_host_notify_work);
-
-	return host_notify;
-}
-EXPORT_SYMBOL_GPL(i2c_setup_smbus_host_notify);
-
-/**
- * i2c_handle_smbus_host_notify - Forward a Host Notify event to the correct
- * I2C client.
- * @host_notify: the struct host_notify attached to the relevant adapter
- * @addr: the I2C address of the notifying device
- * @data: the payload of the notification
- * Context: can't sleep
- *
- * Helper function to be called from an I2C bus driver's interrupt
- * handler. It will schedule the Host Notify work, in turn calling the
- * corresponding I2C device driver's alert function.
- *
- * host_notify should be a valid pointer previously returned by
- * i2c_setup_smbus_host_notify().
- */
-int i2c_handle_smbus_host_notify(struct smbus_host_notify *host_notify,
-				 unsigned short addr, unsigned int data)
-{
-	unsigned long flags;
-	struct i2c_adapter *adapter;
-
-	if (!host_notify || !host_notify->adapter)
-		return -EINVAL;
-
-	adapter = host_notify->adapter;
-
-	spin_lock_irqsave(&host_notify->lock, flags);
-
-	if (host_notify->pending) {
-		spin_unlock_irqrestore(&host_notify->lock, flags);
-		dev_warn(&adapter->dev, "Host Notify already scheduled.\n");
-		return -EBUSY;
-	}
-
-	host_notify->payload = data;
-	host_notify->addr = addr;
-
-	/* Mark that there is a pending notification and release the lock */
-	host_notify->pending = true;
-	spin_unlock_irqrestore(&host_notify->lock, flags);
-
-	return schedule_work(&host_notify->work);
-}
-EXPORT_SYMBOL_GPL(i2c_handle_smbus_host_notify);
-
 module_i2c_driver(smbalert_driver);
 
 MODULE_AUTHOR("Jean Delvare <jdelvare@suse.de>");
diff --git a/drivers/i2c/muxes/Kconfig b/drivers/i2c/muxes/Kconfig
index 96de9ce5669b..10b3d17ae3ea 100644
--- a/drivers/i2c/muxes/Kconfig
+++ b/drivers/i2c/muxes/Kconfig
@@ -82,4 +82,15 @@ config I2C_DEMUX_PINCTRL
 	  demultiplexer that uses the pinctrl subsystem. This is useful if you
 	  want to change the I2C master at run-time depending on features.
 
+config I2C_MUX_MLXCPLD
+        tristate "Mellanox CPLD based I2C multiplexer"
+        help
+          If you say yes to this option, support will be included for a
+          CPLD based I2C multiplexer. This driver provides access to
+          I2C busses connected through a MUX, which is controlled
+          by a CPLD register.
+
+          This driver can also be built as a module.  If so, the module
+          will be called i2c-mux-mlxcpld.
+
 endmenu
diff --git a/drivers/i2c/muxes/Makefile b/drivers/i2c/muxes/Makefile
index 7c267c29b191..9948fa45037f 100644
--- a/drivers/i2c/muxes/Makefile
+++ b/drivers/i2c/muxes/Makefile
@@ -6,6 +6,7 @@ obj-$(CONFIG_I2C_ARB_GPIO_CHALLENGE)	+= i2c-arb-gpio-challenge.o
 obj-$(CONFIG_I2C_DEMUX_PINCTRL)		+= i2c-demux-pinctrl.o
 
 obj-$(CONFIG_I2C_MUX_GPIO)	+= i2c-mux-gpio.o
+obj-$(CONFIG_I2C_MUX_MLXCPLD)	+= i2c-mux-mlxcpld.o
 obj-$(CONFIG_I2C_MUX_PCA9541)	+= i2c-mux-pca9541.o
 obj-$(CONFIG_I2C_MUX_PCA954x)	+= i2c-mux-pca954x.o
 obj-$(CONFIG_I2C_MUX_PINCTRL)	+= i2c-mux-pinctrl.o
diff --git a/drivers/i2c/muxes/i2c-mux-gpio.c b/drivers/i2c/muxes/i2c-mux-gpio.c
index e5cf26eefa97..655684d621a4 100644
--- a/drivers/i2c/muxes/i2c-mux-gpio.c
+++ b/drivers/i2c/muxes/i2c-mux-gpio.c
@@ -21,6 +21,8 @@
 struct gpiomux {
 	struct i2c_mux_gpio_platform_data data;
 	unsigned gpio_base;
+	struct gpio_desc **gpios;
+	int *values;
 };
 
 static void i2c_mux_gpio_set(const struct gpiomux *mux, unsigned val)
@@ -28,8 +30,10 @@ static void i2c_mux_gpio_set(const struct gpiomux *mux, unsigned val)
 	int i;
 
 	for (i = 0; i < mux->data.n_gpios; i++)
-		gpio_set_value_cansleep(mux->gpio_base + mux->data.gpios[i],
-					val & (1 << i));
+		mux->values[i] = (val >> i) & 1;
+
+	gpiod_set_array_value_cansleep(mux->data.n_gpios,
+				       mux->gpios, mux->values);
 }
 
 static int i2c_mux_gpio_select(struct i2c_mux_core *muxc, u32 chan)
@@ -176,12 +180,16 @@ static int i2c_mux_gpio_probe(struct platform_device *pdev)
 	if (!parent)
 		return -EPROBE_DEFER;
 
-	muxc = i2c_mux_alloc(parent, &pdev->dev, mux->data.n_values, 0, 0,
+	muxc = i2c_mux_alloc(parent, &pdev->dev, mux->data.n_values,
+			     mux->data.n_gpios * sizeof(*mux->gpios) +
+			     mux->data.n_gpios * sizeof(*mux->values), 0,
 			     i2c_mux_gpio_select, NULL);
 	if (!muxc) {
 		ret = -ENOMEM;
 		goto alloc_failed;
 	}
+	mux->gpios = muxc->priv;
+	mux->values = (int *)(mux->gpios + mux->data.n_gpios);
 	muxc->priv = mux;
 
 	platform_set_drvdata(pdev, muxc);
@@ -219,10 +227,12 @@ static int i2c_mux_gpio_probe(struct platform_device *pdev)
 			goto err_request_gpio;
 		}
 
+		gpio_desc = gpio_to_desc(gpio_base + mux->data.gpios[i]);
+		mux->gpios[i] = gpio_desc;
+
 		if (!muxc->mux_locked)
 			continue;
 
-		gpio_desc = gpio_to_desc(gpio_base + mux->data.gpios[i]);
 		gpio_dev = &gpio_desc->gdev->dev;
 		muxc->mux_locked = i2c_root_adapter(gpio_dev) == root;
 	}
diff --git a/drivers/i2c/muxes/i2c-mux-mlxcpld.c b/drivers/i2c/muxes/i2c-mux-mlxcpld.c
new file mode 100644
index 000000000000..3ab654bbfab5
--- /dev/null
+++ b/drivers/i2c/muxes/i2c-mux-mlxcpld.c
@@ -0,0 +1,220 @@
+/*
+ * drivers/i2c/muxes/i2c-mux-mlxcpld.c
+ * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2016 Michael Shych <michaels@mellanox.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the names of the copyright holders nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/device.h>
+#include <linux/i2c.h>
+#include <linux/i2c-mux.h>
+#include <linux/io.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/version.h>
+#include <linux/i2c/mlxcpld.h>
+
+#define CPLD_MUX_MAX_NCHANS	8
+
+/* mlxcpld_mux - mux control structure:
+ * @last_chan - last register value
+ * @client - I2C device client
+ */
+struct mlxcpld_mux {
+	u8 last_chan;
+	struct i2c_client *client;
+};
+
+/* MUX logic description.
+ * Driver can support different mux control logic, according to CPLD
+ * implementation.
+ *
+ * Connectivity schema.
+ *
+ * i2c-mlxcpld                                 Digital               Analog
+ * driver
+ * *--------*                                 * -> mux1 (virt bus2) -> mux -> |
+ * | I2CLPC | i2c physical                    * -> mux2 (virt bus3) -> mux -> |
+ * | bridge | bus 1                 *---------*                               |
+ * | logic  |---------------------> * mux reg *                               |
+ * | in CPLD|                       *---------*                               |
+ * *--------*   i2c-mux-mlxpcld          ^    * -> muxn (virt busn) -> mux -> |
+ *     |        driver                   |                                    |
+ *     |        *---------------*        |                              Devices
+ *     |        * CPLD (i2c bus)* select |
+ *     |        * registers for *--------*
+ *     |        * mux selection * deselect
+ *     |        *---------------*
+ *     |                 |
+ * <-------->     <----------->
+ * i2c cntrl      Board cntrl reg
+ * reg space      space (mux select,
+ *                IO, LED, WD, info)
+ *
+ */
+
+static const struct i2c_device_id mlxcpld_mux_id[] = {
+	{ "mlxcpld_mux_module", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, mlxcpld_mux_id);
+
+/* Write to mux register. Don't use i2c_transfer() and i2c_smbus_xfer()
+ * for this as they will try to lock adapter a second time.
+ */
+static int mlxcpld_mux_reg_write(struct i2c_adapter *adap,
+				 struct i2c_client *client, u8 val)
+{
+	struct mlxcpld_mux_plat_data *pdata = dev_get_platdata(&client->dev);
+
+	if (adap->algo->master_xfer) {
+		struct i2c_msg msg;
+		u8 msgbuf[] = {pdata->sel_reg_addr, val};
+
+		msg.addr = client->addr;
+		msg.flags = 0;
+		msg.len = 2;
+		msg.buf = msgbuf;
+		return __i2c_transfer(adap, &msg, 1);
+	} else if (adap->algo->smbus_xfer) {
+		union i2c_smbus_data data;
+
+		data.byte = val;
+		return adap->algo->smbus_xfer(adap, client->addr,
+					      client->flags, I2C_SMBUS_WRITE,
+					      pdata->sel_reg_addr,
+					      I2C_SMBUS_BYTE_DATA, &data);
+	} else
+		return -ENODEV;
+}
+
+static int mlxcpld_mux_select_chan(struct i2c_mux_core *muxc, u32 chan)
+{
+	struct mlxcpld_mux *data = i2c_mux_priv(muxc);
+	struct i2c_client *client = data->client;
+	u8 regval = chan + 1;
+	int err = 0;
+
+	/* Only select the channel if its different from the last channel */
+	if (data->last_chan != regval) {
+		err = mlxcpld_mux_reg_write(muxc->parent, client, regval);
+		if (err)
+			data->last_chan = 0;
+		else
+			data->last_chan = regval;
+	}
+
+	return err;
+}
+
+static int mlxcpld_mux_deselect(struct i2c_mux_core *muxc, u32 chan)
+{
+	struct mlxcpld_mux *data = i2c_mux_priv(muxc);
+	struct i2c_client *client = data->client;
+
+	/* Deselect active channel */
+	data->last_chan = 0;
+
+	return mlxcpld_mux_reg_write(muxc->parent, client, data->last_chan);
+}
+
+/* Probe/reomove functions */
+static int mlxcpld_mux_probe(struct i2c_client *client,
+			     const struct i2c_device_id *id)
+{
+	struct i2c_adapter *adap = to_i2c_adapter(client->dev.parent);
+	struct mlxcpld_mux_plat_data *pdata = dev_get_platdata(&client->dev);
+	struct i2c_mux_core *muxc;
+	int num, force;
+	struct mlxcpld_mux *data;
+	int err;
+
+	if (!pdata)
+		return -EINVAL;
+
+	if (!i2c_check_functionality(adap, I2C_FUNC_SMBUS_WRITE_BYTE_DATA))
+		return -ENODEV;
+
+	muxc = i2c_mux_alloc(adap, &client->dev, CPLD_MUX_MAX_NCHANS,
+			     sizeof(*data), 0, mlxcpld_mux_select_chan,
+			     mlxcpld_mux_deselect);
+	if (!muxc)
+		return -ENOMEM;
+
+	data = i2c_mux_priv(muxc);
+	i2c_set_clientdata(client, muxc);
+	data->client = client;
+	data->last_chan = 0; /* force the first selection */
+
+	/* Create an adapter for each channel. */
+	for (num = 0; num < CPLD_MUX_MAX_NCHANS; num++) {
+		if (num >= pdata->num_adaps)
+			/* discard unconfigured channels */
+			break;
+
+		force = pdata->adap_ids[num];
+
+		err = i2c_mux_add_adapter(muxc, force, num, 0);
+		if (err)
+			goto virt_reg_failed;
+	}
+
+	return 0;
+
+virt_reg_failed:
+	i2c_mux_del_adapters(muxc);
+	return err;
+}
+
+static int mlxcpld_mux_remove(struct i2c_client *client)
+{
+	struct i2c_mux_core *muxc = i2c_get_clientdata(client);
+
+	i2c_mux_del_adapters(muxc);
+	return 0;
+}
+
+static struct i2c_driver mlxcpld_mux_driver = {
+	.driver		= {
+		.name	= "mlxcpld-mux",
+	},
+	.probe		= mlxcpld_mux_probe,
+	.remove		= mlxcpld_mux_remove,
+	.id_table	= mlxcpld_mux_id,
+};
+
+module_i2c_driver(mlxcpld_mux_driver);
+
+MODULE_AUTHOR("Michael Shych (michaels@mellanox.com)");
+MODULE_DESCRIPTION("Mellanox I2C-CPLD-MUX driver");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_ALIAS("platform:i2c-mux-mlxcpld");
diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c
index 8bc3d36d2837..9a348ee4dc14 100644
--- a/drivers/i2c/muxes/i2c-mux-pca954x.c
+++ b/drivers/i2c/muxes/i2c-mux-pca954x.c
@@ -35,6 +35,7 @@
  * warranty of any kind, whether express or implied.
  */
 
+#include <linux/acpi.h>
 #include <linux/device.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
@@ -120,6 +121,21 @@ static const struct i2c_device_id pca954x_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, pca954x_id);
 
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id pca954x_acpi_ids[] = {
+	{ .id = "PCA9540", .driver_data = pca_9540 },
+	{ .id = "PCA9542", .driver_data = pca_9540 },
+	{ .id = "PCA9543", .driver_data = pca_9543 },
+	{ .id = "PCA9544", .driver_data = pca_9544 },
+	{ .id = "PCA9545", .driver_data = pca_9545 },
+	{ .id = "PCA9546", .driver_data = pca_9545 },
+	{ .id = "PCA9547", .driver_data = pca_9547 },
+	{ .id = "PCA9548", .driver_data = pca_9548 },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, pca954x_acpi_ids);
+#endif
+
 #ifdef CONFIG_OF
 static const struct of_device_id pca954x_of_match[] = {
 	{ .compatible = "nxp,pca9540", .data = &chips[pca_9540] },
@@ -245,8 +261,17 @@ static int pca954x_probe(struct i2c_client *client,
 	match = of_match_device(of_match_ptr(pca954x_of_match), &client->dev);
 	if (match)
 		data->chip = of_device_get_match_data(&client->dev);
-	else
+	else if (id)
 		data->chip = &chips[id->driver_data];
+	else {
+		const struct acpi_device_id *acpi_id;
+
+		acpi_id = acpi_match_device(ACPI_PTR(pca954x_acpi_ids),
+						&client->dev);
+		if (!acpi_id)
+			return -ENODEV;
+		data->chip = &chips[acpi_id->driver_data];
+	}
 
 	data->last_chan = 0;		   /* force the first selection */
 
@@ -321,6 +346,7 @@ static struct i2c_driver pca954x_driver = {
 		.name	= "pca954x",
 		.pm	= &pca954x_pm,
 		.of_match_table = of_match_ptr(pca954x_of_match),
+		.acpi_match_table = ACPI_PTR(pca954x_acpi_ids),
 	},
 	.probe		= pca954x_probe,
 	.remove		= pca954x_remove,
diff --git a/drivers/ide/ide-atapi.c b/drivers/ide/ide-atapi.c
index 05352f490d60..f90ea221f7f2 100644
--- a/drivers/ide/ide-atapi.c
+++ b/drivers/ide/ide-atapi.c
@@ -211,7 +211,7 @@ void ide_prep_sense(ide_drive_t *drive, struct request *rq)
 	sense_rq->cmd[0] = GPCMD_REQUEST_SENSE;
 	sense_rq->cmd[4] = cmd_len;
 	sense_rq->cmd_type = REQ_TYPE_ATA_SENSE;
-	sense_rq->cmd_flags |= REQ_PREEMPT;
+	sense_rq->rq_flags |= RQF_PREEMPT;
 
 	if (drive->media == ide_tape)
 		sense_rq->cmd[13] = REQ_IDETAPE_PC1;
@@ -295,7 +295,7 @@ int ide_cd_expiry(ide_drive_t *drive)
 		wait = ATAPI_WAIT_PC;
 		break;
 	default:
-		if (!(rq->cmd_flags & REQ_QUIET))
+		if (!(rq->rq_flags & RQF_QUIET))
 			printk(KERN_INFO PFX "cmd 0x%x timed out\n",
 					 rq->cmd[0]);
 		wait = 0;
@@ -375,7 +375,7 @@ int ide_check_ireason(ide_drive_t *drive, struct request *rq, int len,
 	}
 
 	if (dev_is_idecd(drive) && rq->cmd_type == REQ_TYPE_ATA_PC)
-		rq->cmd_flags |= REQ_FAILED;
+		rq->rq_flags |= RQF_FAILED;
 
 	return 1;
 }
diff --git a/drivers/ide/ide-cd.c b/drivers/ide/ide-cd.c
index bf9a2ad296ed..9cbd217bc0c9 100644
--- a/drivers/ide/ide-cd.c
+++ b/drivers/ide/ide-cd.c
@@ -98,7 +98,7 @@ static int cdrom_log_sense(ide_drive_t *drive, struct request *rq)
 	struct request_sense *sense = &drive->sense_data;
 	int log = 0;
 
-	if (!sense || !rq || (rq->cmd_flags & REQ_QUIET))
+	if (!sense || !rq || (rq->rq_flags & RQF_QUIET))
 		return 0;
 
 	ide_debug_log(IDE_DBG_SENSE, "sense_key: 0x%x", sense->sense_key);
@@ -291,7 +291,7 @@ static int cdrom_decode_status(ide_drive_t *drive, u8 stat)
 		 * (probably while trying to recover from a former error).
 		 * Just give up.
 		 */
-		rq->cmd_flags |= REQ_FAILED;
+		rq->rq_flags |= RQF_FAILED;
 		return 2;
 	}
 
@@ -311,7 +311,7 @@ static int cdrom_decode_status(ide_drive_t *drive, u8 stat)
 			cdrom_saw_media_change(drive);
 
 			if (rq->cmd_type == REQ_TYPE_FS &&
-			    !(rq->cmd_flags & REQ_QUIET))
+			    !(rq->rq_flags & RQF_QUIET))
 				printk(KERN_ERR PFX "%s: tray open\n",
 					drive->name);
 		}
@@ -346,7 +346,7 @@ static int cdrom_decode_status(ide_drive_t *drive, u8 stat)
 		 * No point in retrying after an illegal request or data
 		 * protect error.
 		 */
-		if (!(rq->cmd_flags & REQ_QUIET))
+		if (!(rq->rq_flags & RQF_QUIET))
 			ide_dump_status(drive, "command error", stat);
 		do_end_request = 1;
 		break;
@@ -355,14 +355,14 @@ static int cdrom_decode_status(ide_drive_t *drive, u8 stat)
 		 * No point in re-trying a zillion times on a bad sector.
 		 * If we got here the error is not correctable.
 		 */
-		if (!(rq->cmd_flags & REQ_QUIET))
+		if (!(rq->rq_flags & RQF_QUIET))
 			ide_dump_status(drive, "media error "
 					"(bad sector)", stat);
 		do_end_request = 1;
 		break;
 	case BLANK_CHECK:
 		/* disk appears blank? */
-		if (!(rq->cmd_flags & REQ_QUIET))
+		if (!(rq->rq_flags & RQF_QUIET))
 			ide_dump_status(drive, "media error (blank)",
 					stat);
 		do_end_request = 1;
@@ -380,7 +380,7 @@ static int cdrom_decode_status(ide_drive_t *drive, u8 stat)
 	}
 
 	if (rq->cmd_type != REQ_TYPE_FS) {
-		rq->cmd_flags |= REQ_FAILED;
+		rq->rq_flags |= RQF_FAILED;
 		do_end_request = 1;
 	}
 
@@ -422,19 +422,19 @@ static void ide_cd_request_sense_fixup(ide_drive_t *drive, struct ide_cmd *cmd)
 int ide_cd_queue_pc(ide_drive_t *drive, const unsigned char *cmd,
 		    int write, void *buffer, unsigned *bufflen,
 		    struct request_sense *sense, int timeout,
-		    unsigned int cmd_flags)
+		    req_flags_t rq_flags)
 {
 	struct cdrom_info *info = drive->driver_data;
 	struct request_sense local_sense;
 	int retries = 10;
-	unsigned int flags = 0;
+	req_flags_t flags = 0;
 
 	if (!sense)
 		sense = &local_sense;
 
 	ide_debug_log(IDE_DBG_PC, "cmd[0]: 0x%x, write: 0x%x, timeout: %d, "
-				  "cmd_flags: 0x%x",
-				  cmd[0], write, timeout, cmd_flags);
+				  "rq_flags: 0x%x",
+				  cmd[0], write, timeout, rq_flags);
 
 	/* start of retry loop */
 	do {
@@ -446,7 +446,7 @@ int ide_cd_queue_pc(ide_drive_t *drive, const unsigned char *cmd,
 		memcpy(rq->cmd, cmd, BLK_MAX_CDB);
 		rq->cmd_type = REQ_TYPE_ATA_PC;
 		rq->sense = sense;
-		rq->cmd_flags |= cmd_flags;
+		rq->rq_flags |= rq_flags;
 		rq->timeout = timeout;
 		if (buffer) {
 			error = blk_rq_map_kern(drive->queue, rq, buffer,
@@ -462,14 +462,14 @@ int ide_cd_queue_pc(ide_drive_t *drive, const unsigned char *cmd,
 		if (buffer)
 			*bufflen = rq->resid_len;
 
-		flags = rq->cmd_flags;
+		flags = rq->rq_flags;
 		blk_put_request(rq);
 
 		/*
 		 * FIXME: we should probably abort/retry or something in case of
 		 * failure.
 		 */
-		if (flags & REQ_FAILED) {
+		if (flags & RQF_FAILED) {
 			/*
 			 * The request failed.  Retry if it was due to a unit
 			 * attention status (usually means media was changed).
@@ -494,10 +494,10 @@ int ide_cd_queue_pc(ide_drive_t *drive, const unsigned char *cmd,
 		}
 
 		/* end of retry loop */
-	} while ((flags & REQ_FAILED) && retries >= 0);
+	} while ((flags & RQF_FAILED) && retries >= 0);
 
 	/* return an error if the command failed */
-	return (flags & REQ_FAILED) ? -EIO : 0;
+	return (flags & RQF_FAILED) ? -EIO : 0;
 }
 
 /*
@@ -589,7 +589,7 @@ static ide_startstop_t cdrom_newpc_intr(ide_drive_t *drive)
 					"(%u bytes)\n", drive->name, __func__,
 					cmd->nleft);
 				if (!write)
-					rq->cmd_flags |= REQ_FAILED;
+					rq->rq_flags |= RQF_FAILED;
 				uptodate = 0;
 			}
 		} else if (rq->cmd_type != REQ_TYPE_BLOCK_PC) {
@@ -607,7 +607,7 @@ static ide_startstop_t cdrom_newpc_intr(ide_drive_t *drive)
 			}
 
 			if (!uptodate)
-				rq->cmd_flags |= REQ_FAILED;
+				rq->rq_flags |= RQF_FAILED;
 		}
 		goto out_end;
 	}
@@ -745,9 +745,9 @@ static void cdrom_do_block_pc(ide_drive_t *drive, struct request *rq)
 				  rq->cmd[0], rq->cmd_type);
 
 	if (rq->cmd_type == REQ_TYPE_BLOCK_PC)
-		rq->cmd_flags |= REQ_QUIET;
+		rq->rq_flags |= RQF_QUIET;
 	else
-		rq->cmd_flags &= ~REQ_FAILED;
+		rq->rq_flags &= ~RQF_FAILED;
 
 	drive->dma = 0;
 
@@ -867,7 +867,7 @@ int cdrom_check_status(ide_drive_t *drive, struct request_sense *sense)
 	 */
 	cmd[7] = cdi->sanyo_slot % 3;
 
-	return ide_cd_queue_pc(drive, cmd, 0, NULL, NULL, sense, 0, REQ_QUIET);
+	return ide_cd_queue_pc(drive, cmd, 0, NULL, NULL, sense, 0, RQF_QUIET);
 }
 
 static int cdrom_read_capacity(ide_drive_t *drive, unsigned long *capacity,
@@ -890,7 +890,7 @@ static int cdrom_read_capacity(ide_drive_t *drive, unsigned long *capacity,
 	cmd[0] = GPCMD_READ_CDVD_CAPACITY;
 
 	stat = ide_cd_queue_pc(drive, cmd, 0, &capbuf, &len, sense, 0,
-			       REQ_QUIET);
+			       RQF_QUIET);
 	if (stat)
 		return stat;
 
@@ -943,7 +943,7 @@ static int cdrom_read_tocentry(ide_drive_t *drive, int trackno, int msf_flag,
 	if (msf_flag)
 		cmd[1] = 2;
 
-	return ide_cd_queue_pc(drive, cmd, 0, buf, &buflen, sense, 0, REQ_QUIET);
+	return ide_cd_queue_pc(drive, cmd, 0, buf, &buflen, sense, 0, RQF_QUIET);
 }
 
 /* Try to read the entire TOC for the disk into our internal buffer. */
diff --git a/drivers/ide/ide-cd.h b/drivers/ide/ide-cd.h
index 1efc936f5b66..eea60c986c4f 100644
--- a/drivers/ide/ide-cd.h
+++ b/drivers/ide/ide-cd.h
@@ -101,7 +101,7 @@ void ide_cd_log_error(const char *, struct request *, struct request_sense *);
 
 /* ide-cd.c functions used by ide-cd_ioctl.c */
 int ide_cd_queue_pc(ide_drive_t *, const unsigned char *, int, void *,
-		    unsigned *, struct request_sense *, int, unsigned int);
+		    unsigned *, struct request_sense *, int, req_flags_t);
 int ide_cd_read_toc(ide_drive_t *, struct request_sense *);
 int ide_cdrom_get_capabilities(ide_drive_t *, u8 *);
 void ide_cdrom_update_speed(ide_drive_t *, u8 *);
diff --git a/drivers/ide/ide-cd_ioctl.c b/drivers/ide/ide-cd_ioctl.c
index 5887a7a09e37..f085e3a2e1d6 100644
--- a/drivers/ide/ide-cd_ioctl.c
+++ b/drivers/ide/ide-cd_ioctl.c
@@ -305,7 +305,7 @@ int ide_cdrom_reset(struct cdrom_device_info *cdi)
 
 	rq = blk_get_request(drive->queue, READ, __GFP_RECLAIM);
 	rq->cmd_type = REQ_TYPE_DRV_PRIV;
-	rq->cmd_flags = REQ_QUIET;
+	rq->rq_flags = RQF_QUIET;
 	ret = blk_execute_rq(drive->queue, cd->disk, rq, 0);
 	blk_put_request(rq);
 	/*
@@ -449,7 +449,7 @@ int ide_cdrom_packet(struct cdrom_device_info *cdi,
 			    struct packet_command *cgc)
 {
 	ide_drive_t *drive = cdi->handle;
-	unsigned int flags = 0;
+	req_flags_t flags = 0;
 	unsigned len = cgc->buflen;
 
 	if (cgc->timeout <= 0)
@@ -463,7 +463,7 @@ int ide_cdrom_packet(struct cdrom_device_info *cdi,
 		memset(cgc->sense, 0, sizeof(struct request_sense));
 
 	if (cgc->quiet)
-		flags |= REQ_QUIET;
+		flags |= RQF_QUIET;
 
 	cgc->stat = ide_cd_queue_pc(drive, cgc->cmd,
 				    cgc->data_direction == CGC_DATA_WRITE,
diff --git a/drivers/ide/ide-io.c b/drivers/ide/ide-io.c
index 669ea1e45795..6360bbd37efe 100644
--- a/drivers/ide/ide-io.c
+++ b/drivers/ide/ide-io.c
@@ -307,7 +307,7 @@ static ide_startstop_t start_request (ide_drive_t *drive, struct request *rq)
 {
 	ide_startstop_t startstop;
 
-	BUG_ON(!(rq->cmd_flags & REQ_STARTED));
+	BUG_ON(!(rq->rq_flags & RQF_STARTED));
 
 #ifdef DEBUG
 	printk("%s: start_request: current=0x%08lx\n",
@@ -316,7 +316,7 @@ static ide_startstop_t start_request (ide_drive_t *drive, struct request *rq)
 
 	/* bail early if we've exceeded max_failures */
 	if (drive->max_failures && (drive->failures > drive->max_failures)) {
-		rq->cmd_flags |= REQ_FAILED;
+		rq->rq_flags |= RQF_FAILED;
 		goto kill_rq;
 	}
 
@@ -539,7 +539,7 @@ repeat:
 		 */
 		if ((drive->dev_flags & IDE_DFLAG_BLOCKED) &&
 		    ata_pm_request(rq) == 0 &&
-		    (rq->cmd_flags & REQ_PREEMPT) == 0) {
+		    (rq->rq_flags & RQF_PREEMPT) == 0) {
 			/* there should be no pending command at this point */
 			ide_unlock_port(hwif);
 			goto plug_device;
diff --git a/drivers/ide/ide-pm.c b/drivers/ide/ide-pm.c
index e34af488693a..a015acdffb39 100644
--- a/drivers/ide/ide-pm.c
+++ b/drivers/ide/ide-pm.c
@@ -53,7 +53,7 @@ static int ide_pm_execute_rq(struct request *rq)
 
 	spin_lock_irq(q->queue_lock);
 	if (unlikely(blk_queue_dying(q))) {
-		rq->cmd_flags |= REQ_QUIET;
+		rq->rq_flags |= RQF_QUIET;
 		rq->errors = -ENXIO;
 		__blk_end_request_all(rq, rq->errors);
 		spin_unlock_irq(q->queue_lock);
@@ -90,7 +90,7 @@ int generic_ide_resume(struct device *dev)
 	memset(&rqpm, 0, sizeof(rqpm));
 	rq = blk_get_request(drive->queue, READ, __GFP_RECLAIM);
 	rq->cmd_type = REQ_TYPE_ATA_PM_RESUME;
-	rq->cmd_flags |= REQ_PREEMPT;
+	rq->rq_flags |= RQF_PREEMPT;
 	rq->special = &rqpm;
 	rqpm.pm_step = IDE_PM_START_RESUME;
 	rqpm.pm_state = PM_EVENT_ON;
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
index 4466a2f969d7..7d8ea3d5fda6 100644
--- a/drivers/idle/intel_idle.c
+++ b/drivers/idle/intel_idle.c
@@ -98,8 +98,6 @@ static int intel_idle(struct cpuidle_device *dev,
 			struct cpuidle_driver *drv, int index);
 static void intel_idle_freeze(struct cpuidle_device *dev,
 			      struct cpuidle_driver *drv, int index);
-static int intel_idle_cpu_init(int cpu);
-
 static struct cpuidle_state *cpuidle_state_table;
 
 /*
@@ -724,6 +722,50 @@ static struct cpuidle_state atom_cstates[] = {
 	{
 		.enter = NULL }
 };
+static struct cpuidle_state tangier_cstates[] = {
+	{
+		.name = "C1-TNG",
+		.desc = "MWAIT 0x00",
+		.flags = MWAIT2flg(0x00),
+		.exit_latency = 1,
+		.target_residency = 4,
+		.enter = &intel_idle,
+		.enter_freeze = intel_idle_freeze, },
+	{
+		.name = "C4-TNG",
+		.desc = "MWAIT 0x30",
+		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
+		.exit_latency = 100,
+		.target_residency = 400,
+		.enter = &intel_idle,
+		.enter_freeze = intel_idle_freeze, },
+	{
+		.name = "C6-TNG",
+		.desc = "MWAIT 0x52",
+		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
+		.exit_latency = 140,
+		.target_residency = 560,
+		.enter = &intel_idle,
+		.enter_freeze = intel_idle_freeze, },
+	{
+		.name = "C7-TNG",
+		.desc = "MWAIT 0x60",
+		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
+		.exit_latency = 1200,
+		.target_residency = 4000,
+		.enter = &intel_idle,
+		.enter_freeze = intel_idle_freeze, },
+	{
+		.name = "C9-TNG",
+		.desc = "MWAIT 0x64",
+		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
+		.exit_latency = 10000,
+		.target_residency = 20000,
+		.enter = &intel_idle,
+		.enter_freeze = intel_idle_freeze, },
+	{
+		.enter = NULL }
+};
 static struct cpuidle_state avn_cstates[] = {
 	{
 		.name = "C1-AVN",
@@ -907,51 +949,15 @@ static void intel_idle_freeze(struct cpuidle_device *dev,
 	mwait_idle_with_hints(eax, ecx);
 }
 
-static void __setup_broadcast_timer(void *arg)
+static void __setup_broadcast_timer(bool on)
 {
-	unsigned long on = (unsigned long)arg;
-
 	if (on)
 		tick_broadcast_enable();
 	else
 		tick_broadcast_disable();
 }
 
-static int cpu_hotplug_notify(struct notifier_block *n,
-			      unsigned long action, void *hcpu)
-{
-	int hotcpu = (unsigned long)hcpu;
-	struct cpuidle_device *dev;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-
-		if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE)
-			smp_call_function_single(hotcpu, __setup_broadcast_timer,
-						 (void *)true, 1);
-
-		/*
-		 * Some systems can hotplug a cpu at runtime after
-		 * the kernel has booted, we have to initialize the
-		 * driver in this case
-		 */
-		dev = per_cpu_ptr(intel_idle_cpuidle_devices, hotcpu);
-		if (dev->registered)
-			break;
-
-		if (intel_idle_cpu_init(hotcpu))
-			return NOTIFY_BAD;
-
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block cpu_hotplug_notifier = {
-	.notifier_call = cpu_hotplug_notify,
-};
-
-static void auto_demotion_disable(void *dummy)
+static void auto_demotion_disable(void)
 {
 	unsigned long long msr_bits;
 
@@ -959,7 +965,7 @@ static void auto_demotion_disable(void *dummy)
 	msr_bits &= ~(icpu->auto_demotion_disable_flags);
 	wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
 }
-static void c1e_promotion_disable(void *dummy)
+static void c1e_promotion_disable(void)
 {
 	unsigned long long msr_bits;
 
@@ -978,6 +984,10 @@ static const struct idle_cpu idle_cpu_atom = {
 	.state_table = atom_cstates,
 };
 
+static const struct idle_cpu idle_cpu_tangier = {
+	.state_table = tangier_cstates,
+};
+
 static const struct idle_cpu idle_cpu_lincroft = {
 	.state_table = atom_cstates,
 	.auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
@@ -1066,6 +1076,7 @@ static const struct x86_cpu_id intel_idle_ids[] __initconst = {
 	ICPU(INTEL_FAM6_SANDYBRIDGE_X,		idle_cpu_snb),
 	ICPU(INTEL_FAM6_ATOM_CEDARVIEW,		idle_cpu_atom),
 	ICPU(INTEL_FAM6_ATOM_SILVERMONT1,	idle_cpu_byt),
+	ICPU(INTEL_FAM6_ATOM_MERRIFIELD,	idle_cpu_tangier),
 	ICPU(INTEL_FAM6_ATOM_AIRMONT,		idle_cpu_cht),
 	ICPU(INTEL_FAM6_IVYBRIDGE,		idle_cpu_ivb),
 	ICPU(INTEL_FAM6_IVYBRIDGE_X,		idle_cpu_ivt),
@@ -1084,6 +1095,7 @@ static const struct x86_cpu_id intel_idle_ids[] __initconst = {
 	ICPU(INTEL_FAM6_KABYLAKE_DESKTOP,	idle_cpu_skl),
 	ICPU(INTEL_FAM6_SKYLAKE_X,		idle_cpu_skx),
 	ICPU(INTEL_FAM6_XEON_PHI_KNL,		idle_cpu_knl),
+	ICPU(INTEL_FAM6_XEON_PHI_KNM,		idle_cpu_knl),
 	ICPU(INTEL_FAM6_ATOM_GOLDMONT,		idle_cpu_bxt),
 	ICPU(INTEL_FAM6_ATOM_DENVERTON,		idle_cpu_dnv),
 	{}
@@ -1373,12 +1385,11 @@ static void __init intel_idle_cpuidle_driver_init(void)
  * allocate, initialize, register cpuidle_devices
  * @cpu: cpu/core to initialize
  */
-static int intel_idle_cpu_init(int cpu)
+static int intel_idle_cpu_init(unsigned int cpu)
 {
 	struct cpuidle_device *dev;
 
 	dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
-
 	dev->cpu = cpu;
 
 	if (cpuidle_register_device(dev)) {
@@ -1387,17 +1398,36 @@ static int intel_idle_cpu_init(int cpu)
 	}
 
 	if (icpu->auto_demotion_disable_flags)
-		smp_call_function_single(cpu, auto_demotion_disable, NULL, 1);
+		auto_demotion_disable();
 
 	if (icpu->disable_promotion_to_c1e)
-		smp_call_function_single(cpu, c1e_promotion_disable, NULL, 1);
+		c1e_promotion_disable();
+
+	return 0;
+}
+
+static int intel_idle_cpu_online(unsigned int cpu)
+{
+	struct cpuidle_device *dev;
+
+	if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE)
+		__setup_broadcast_timer(true);
+
+	/*
+	 * Some systems can hotplug a cpu at runtime after
+	 * the kernel has booted, we have to initialize the
+	 * driver in this case
+	 */
+	dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
+	if (!dev->registered)
+		return intel_idle_cpu_init(cpu);
 
 	return 0;
 }
 
 static int __init intel_idle_init(void)
 {
-	int retval, i;
+	int retval;
 
 	/* Do not load intel_idle at all for now if idle= is passed */
 	if (boot_option_idle_override != IDLE_NO_OVERRIDE)
@@ -1417,35 +1447,29 @@ static int __init intel_idle_init(void)
 		struct cpuidle_driver *drv = cpuidle_get_driver();
 		printk(KERN_DEBUG PREFIX "intel_idle yielding to %s",
 			drv ? drv->name : "none");
-		free_percpu(intel_idle_cpuidle_devices);
-		return retval;
+		goto init_driver_fail;
 	}
 
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(i) {
-		retval = intel_idle_cpu_init(i);
-		if (retval) {
-			intel_idle_cpuidle_devices_uninit();
-			cpu_notifier_register_done();
-			cpuidle_unregister_driver(&intel_idle_driver);
-			free_percpu(intel_idle_cpuidle_devices);
-			return retval;
-		}
-	}
-	__register_cpu_notifier(&cpu_hotplug_notifier);
-
 	if (boot_cpu_has(X86_FEATURE_ARAT))	/* Always Reliable APIC Timer */
 		lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE;
-	else
-		on_each_cpu(__setup_broadcast_timer, (void *)true, 1);
 
-	cpu_notifier_register_done();
+	retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
+				   intel_idle_cpu_online, NULL);
+	if (retval < 0)
+		goto hp_setup_fail;
 
 	pr_debug(PREFIX "lapic_timer_reliable_states 0x%x\n",
 		lapic_timer_reliable_states);
 
 	return 0;
+
+hp_setup_fail:
+	intel_idle_cpuidle_devices_uninit();
+	cpuidle_unregister_driver(&intel_idle_driver);
+init_driver_fail:
+	free_percpu(intel_idle_cpuidle_devices);
+	return retval;
+
 }
 device_initcall(intel_idle_init);
 
diff --git a/drivers/iio/Kconfig b/drivers/iio/Kconfig
index 6743b18194fb..a918270d6f54 100644
--- a/drivers/iio/Kconfig
+++ b/drivers/iio/Kconfig
@@ -73,6 +73,7 @@ source "drivers/iio/adc/Kconfig"
 source "drivers/iio/amplifiers/Kconfig"
 source "drivers/iio/chemical/Kconfig"
 source "drivers/iio/common/Kconfig"
+source "drivers/iio/counter/Kconfig"
 source "drivers/iio/dac/Kconfig"
 source "drivers/iio/dummy/Kconfig"
 source "drivers/iio/frequency/Kconfig"
@@ -87,6 +88,7 @@ if IIO_TRIGGER
    source "drivers/iio/trigger/Kconfig"
 endif #IIO_TRIGGER
 source "drivers/iio/potentiometer/Kconfig"
+source "drivers/iio/potentiostat/Kconfig"
 source "drivers/iio/pressure/Kconfig"
 source "drivers/iio/proximity/Kconfig"
 source "drivers/iio/temperature/Kconfig"
diff --git a/drivers/iio/Makefile b/drivers/iio/Makefile
index 87e4c4369e2f..33fa4026f92c 100644
--- a/drivers/iio/Makefile
+++ b/drivers/iio/Makefile
@@ -18,6 +18,7 @@ obj-y += amplifiers/
 obj-y += buffer/
 obj-y += chemical/
 obj-y += common/
+obj-y += counter/
 obj-y += dac/
 obj-y += dummy/
 obj-y += gyro/
@@ -29,6 +30,7 @@ obj-y += light/
 obj-y += magnetometer/
 obj-y += orientation/
 obj-y += potentiometer/
+obj-y += potentiostat/
 obj-y += pressure/
 obj-y += proximity/
 obj-y += temperature/
diff --git a/drivers/iio/accel/Kconfig b/drivers/iio/accel/Kconfig
index 2b791fe1e2bc..c68bdb649005 100644
--- a/drivers/iio/accel/Kconfig
+++ b/drivers/iio/accel/Kconfig
@@ -52,6 +52,26 @@ config BMC150_ACCEL_SPI
 	tristate
 	select REGMAP_SPI
 
+config DA280
+	tristate "MiraMEMS DA280 3-axis 14-bit digital accelerometer driver"
+	depends on I2C
+	help
+	  Say yes here to build support for the MiraMEMS DA280 3-axis 14-bit
+	  digital accelerometer.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called da280.
+
+config DA311
+	tristate "MiraMEMS DA311 3-axis 12-bit digital accelerometer driver"
+	depends on I2C
+	help
+	  Say yes here to build support for the MiraMEMS DA311 3-axis 12-bit
+	  digital accelerometer.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called da311.
+
 config DMARD06
 	tristate "Domintech DMARD06 Digital Accelerometer Driver"
 	depends on OF || COMPILE_TEST
@@ -73,6 +93,16 @@ config DMARD09
 	  Choosing M will build the driver as a module. If so, the module
 	  will be called dmard09.
 
+config DMARD10
+	tristate "Domintech DMARD10 3-axis Accelerometer Driver"
+	depends on I2C
+	help
+	  Say yes here to get support for the Domintech DMARD10 3-axis
+	  accelerometer.
+
+	  Choosing M will build the driver as a module. If so, the module
+	  will be called dmard10.
+
 config HID_SENSOR_ACCEL_3D
 	depends on HID_SENSOR_HUB
 	select IIO_BUFFER
@@ -97,7 +127,8 @@ config IIO_ST_ACCEL_3AXIS
 	help
 	  Say yes here to build support for STMicroelectronics accelerometers:
 	  LSM303DLH, LSM303DLHC, LIS3DH, LSM330D, LSM330DL, LSM330DLC,
-	  LIS331DLH, LSM303DL, LSM303DLM, LSM330, LIS2DH12, H3LIS331DL.
+	  LIS331DLH, LSM303DL, LSM303DLM, LSM330, LIS2DH12, H3LIS331DL,
+	  LNG2DM
 
 	  This driver can also be built as a module. If so, these modules
 	  will be created:
@@ -273,6 +304,18 @@ config MXC6255
 	  To compile this driver as a module, choose M here: the module will be
 	  called mxc6255.
 
+config SCA3000
+	select IIO_BUFFER
+	select IIO_KFIFO_BUF
+	depends on SPI
+	tristate "VTI SCA3000 series accelerometers"
+	help
+	  Say Y here to build support for the VTI SCA3000 series of SPI
+	  accelerometers. These devices use a hardware ring buffer.
+
+	  To compile this driver as a module, say M here: the module will be
+	  called sca3000.
+
 config STK8312
 	tristate "Sensortek STK8312 3-Axis Accelerometer Driver"
 	depends on I2C
diff --git a/drivers/iio/accel/Makefile b/drivers/iio/accel/Makefile
index f5d3ddee619e..69fe8edc57a2 100644
--- a/drivers/iio/accel/Makefile
+++ b/drivers/iio/accel/Makefile
@@ -8,8 +8,11 @@ obj-$(CONFIG_BMA220) += bma220_spi.o
 obj-$(CONFIG_BMC150_ACCEL) += bmc150-accel-core.o
 obj-$(CONFIG_BMC150_ACCEL_I2C) += bmc150-accel-i2c.o
 obj-$(CONFIG_BMC150_ACCEL_SPI) += bmc150-accel-spi.o
+obj-$(CONFIG_DA280)	+= da280.o
+obj-$(CONFIG_DA311)	+= da311.o
 obj-$(CONFIG_DMARD06)	+= dmard06.o
 obj-$(CONFIG_DMARD09)	+= dmard09.o
+obj-$(CONFIG_DMARD10)	+= dmard10.o
 obj-$(CONFIG_HID_SENSOR_ACCEL_3D) += hid-sensor-accel-3d.o
 obj-$(CONFIG_KXCJK1013) += kxcjk-1013.o
 obj-$(CONFIG_KXSD9)	+= kxsd9.o
@@ -32,6 +35,8 @@ obj-$(CONFIG_MMA9553)		+= mma9553.o
 obj-$(CONFIG_MXC4005)		+= mxc4005.o
 obj-$(CONFIG_MXC6255)		+= mxc6255.o
 
+obj-$(CONFIG_SCA3000)		+= sca3000.o
+
 obj-$(CONFIG_STK8312)		+= stk8312.o
 obj-$(CONFIG_STK8BA50)		+= stk8ba50.o
 
diff --git a/drivers/iio/accel/da280.c b/drivers/iio/accel/da280.c
new file mode 100644
index 000000000000..ed8343aeac9c
--- /dev/null
+++ b/drivers/iio/accel/da280.c
@@ -0,0 +1,183 @@
+/**
+ * IIO driver for the MiraMEMS DA280 3-axis accelerometer and
+ * IIO driver for the MiraMEMS DA226 2-axis accelerometer
+ *
+ * Copyright (c) 2016 Hans de Goede <hdegoede@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/byteorder/generic.h>
+
+#define DA280_REG_CHIP_ID		0x01
+#define DA280_REG_ACC_X_LSB		0x02
+#define DA280_REG_ACC_Y_LSB		0x04
+#define DA280_REG_ACC_Z_LSB		0x06
+#define DA280_REG_MODE_BW		0x11
+
+#define DA280_CHIP_ID			0x13
+#define DA280_MODE_ENABLE		0x1e
+#define DA280_MODE_DISABLE		0x9e
+
+enum { da226, da280 };
+
+/*
+ * a value of + or -4096 corresponds to + or - 1G
+ * scale = 9.81 / 4096 = 0.002395019
+ */
+
+static const int da280_nscale = 2395019;
+
+#define DA280_CHANNEL(reg, axis) {	\
+	.type = IIO_ACCEL,	\
+	.address = reg,	\
+	.modified = 1,	\
+	.channel2 = IIO_MOD_##axis,	\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+}
+
+static const struct iio_chan_spec da280_channels[] = {
+	DA280_CHANNEL(DA280_REG_ACC_X_LSB, X),
+	DA280_CHANNEL(DA280_REG_ACC_Y_LSB, Y),
+	DA280_CHANNEL(DA280_REG_ACC_Z_LSB, Z),
+};
+
+struct da280_data {
+	struct i2c_client *client;
+};
+
+static int da280_enable(struct i2c_client *client, bool enable)
+{
+	u8 data = enable ? DA280_MODE_ENABLE : DA280_MODE_DISABLE;
+
+	return i2c_smbus_write_byte_data(client, DA280_REG_MODE_BW, data);
+}
+
+static int da280_read_raw(struct iio_dev *indio_dev,
+				struct iio_chan_spec const *chan,
+				int *val, int *val2, long mask)
+{
+	struct da280_data *data = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = i2c_smbus_read_word_data(data->client, chan->address);
+		if (ret < 0)
+			return ret;
+		/*
+		 * Values are 14 bits, stored as 16 bits with the 2
+		 * least significant bits always 0.
+		 */
+		*val = (short)ret >> 2;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		*val = 0;
+		*val2 = da280_nscale;
+		return IIO_VAL_INT_PLUS_NANO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info da280_info = {
+	.driver_module	= THIS_MODULE,
+	.read_raw	= da280_read_raw,
+};
+
+static int da280_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	int ret;
+	struct iio_dev *indio_dev;
+	struct da280_data *data;
+
+	ret = i2c_smbus_read_byte_data(client, DA280_REG_CHIP_ID);
+	if (ret != DA280_CHIP_ID)
+		return (ret < 0) ? ret : -ENODEV;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	data->client = client;
+	i2c_set_clientdata(client, indio_dev);
+
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->info = &da280_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = da280_channels;
+	if (id->driver_data == da226) {
+		indio_dev->name = "da226";
+		indio_dev->num_channels = 2;
+	} else {
+		indio_dev->name = "da280";
+		indio_dev->num_channels = 3;
+	}
+
+	ret = da280_enable(client, true);
+	if (ret < 0)
+		return ret;
+
+	ret = iio_device_register(indio_dev);
+	if (ret < 0) {
+		dev_err(&client->dev, "device_register failed\n");
+		da280_enable(client, false);
+	}
+
+	return ret;
+}
+
+static int da280_remove(struct i2c_client *client)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(client);
+
+	iio_device_unregister(indio_dev);
+
+	return da280_enable(client, false);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int da280_suspend(struct device *dev)
+{
+	return da280_enable(to_i2c_client(dev), false);
+}
+
+static int da280_resume(struct device *dev)
+{
+	return da280_enable(to_i2c_client(dev), true);
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(da280_pm_ops, da280_suspend, da280_resume);
+
+static const struct i2c_device_id da280_i2c_id[] = {
+	{ "da226", da226 },
+	{ "da280", da280 },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, da280_i2c_id);
+
+static struct i2c_driver da280_driver = {
+	.driver = {
+		.name = "da280",
+		.pm = &da280_pm_ops,
+	},
+	.probe		= da280_probe,
+	.remove		= da280_remove,
+	.id_table	= da280_i2c_id,
+};
+
+module_i2c_driver(da280_driver);
+
+MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
+MODULE_DESCRIPTION("MiraMEMS DA280 3-Axis Accelerometer driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/accel/da311.c b/drivers/iio/accel/da311.c
new file mode 100644
index 000000000000..537cfa8b6edf
--- /dev/null
+++ b/drivers/iio/accel/da311.c
@@ -0,0 +1,305 @@
+/**
+ * IIO driver for the MiraMEMS DA311 3-axis accelerometer
+ *
+ * Copyright (c) 2016 Hans de Goede <hdegoede@redhat.com>
+ * Copyright (c) 2011-2013 MiraMEMS Sensing Technology Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/byteorder/generic.h>
+
+#define DA311_CHIP_ID			0x13
+
+/*
+ * Note register addressed go from 0 - 0x3f and then wrap.
+ * For some reason there are 2 banks with 0 - 0x3f addresses,
+ * rather then a single 0-0x7f bank.
+ */
+
+/* Bank 0 regs */
+#define DA311_REG_BANK			0x0000
+#define DA311_REG_LDO_REG		0x0006
+#define DA311_REG_CHIP_ID		0x000f
+#define DA311_REG_TEMP_CFG_REG		0x001f
+#define DA311_REG_CTRL_REG1		0x0020
+#define DA311_REG_CTRL_REG3		0x0022
+#define DA311_REG_CTRL_REG4		0x0023
+#define DA311_REG_CTRL_REG5		0x0024
+#define DA311_REG_CTRL_REG6		0x0025
+#define DA311_REG_STATUS_REG		0x0027
+#define DA311_REG_OUT_X_L		0x0028
+#define DA311_REG_OUT_X_H		0x0029
+#define DA311_REG_OUT_Y_L		0x002a
+#define DA311_REG_OUT_Y_H		0x002b
+#define DA311_REG_OUT_Z_L		0x002c
+#define DA311_REG_OUT_Z_H		0x002d
+#define DA311_REG_INT1_CFG		0x0030
+#define DA311_REG_INT1_SRC		0x0031
+#define DA311_REG_INT1_THS		0x0032
+#define DA311_REG_INT1_DURATION		0x0033
+#define DA311_REG_INT2_CFG		0x0034
+#define DA311_REG_INT2_SRC		0x0035
+#define DA311_REG_INT2_THS		0x0036
+#define DA311_REG_INT2_DURATION		0x0037
+#define DA311_REG_CLICK_CFG		0x0038
+#define DA311_REG_CLICK_SRC		0x0039
+#define DA311_REG_CLICK_THS		0x003a
+#define DA311_REG_TIME_LIMIT		0x003b
+#define DA311_REG_TIME_LATENCY		0x003c
+#define DA311_REG_TIME_WINDOW		0x003d
+
+/* Bank 1 regs */
+#define DA311_REG_SOFT_RESET		0x0105
+#define DA311_REG_OTP_XOFF_L		0x0110
+#define DA311_REG_OTP_XOFF_H		0x0111
+#define DA311_REG_OTP_YOFF_L		0x0112
+#define DA311_REG_OTP_YOFF_H		0x0113
+#define DA311_REG_OTP_ZOFF_L		0x0114
+#define DA311_REG_OTP_ZOFF_H		0x0115
+#define DA311_REG_OTP_XSO		0x0116
+#define DA311_REG_OTP_YSO		0x0117
+#define DA311_REG_OTP_ZSO		0x0118
+#define DA311_REG_OTP_TRIM_OSC		0x011b
+#define DA311_REG_LPF_ABSOLUTE		0x011c
+#define DA311_REG_TEMP_OFF1		0x0127
+#define DA311_REG_TEMP_OFF2		0x0128
+#define DA311_REG_TEMP_OFF3		0x0129
+#define DA311_REG_OTP_TRIM_THERM_H	0x011a
+
+/*
+ * a value of + or -1024 corresponds to + or - 1G
+ * scale = 9.81 / 1024 = 0.009580078
+ */
+
+static const int da311_nscale = 9580078;
+
+#define DA311_CHANNEL(reg, axis) {	\
+	.type = IIO_ACCEL,	\
+	.address = reg,	\
+	.modified = 1,	\
+	.channel2 = IIO_MOD_##axis,	\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+}
+
+static const struct iio_chan_spec da311_channels[] = {
+	/* | 0x80 comes from the android driver */
+	DA311_CHANNEL(DA311_REG_OUT_X_L | 0x80, X),
+	DA311_CHANNEL(DA311_REG_OUT_Y_L | 0x80, Y),
+	DA311_CHANNEL(DA311_REG_OUT_Z_L | 0x80, Z),
+};
+
+struct da311_data {
+	struct i2c_client *client;
+};
+
+static int da311_register_mask_write(struct i2c_client *client, u16 addr,
+				     u8 mask, u8 data)
+{
+	int ret;
+	u8 tmp_data = 0;
+
+	if (addr & 0xff00) {
+		/* Select bank 1 */
+		ret = i2c_smbus_write_byte_data(client, DA311_REG_BANK, 0x01);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (mask != 0xff) {
+		ret = i2c_smbus_read_byte_data(client, addr);
+		if (ret < 0)
+			return ret;
+		tmp_data = ret;
+	}
+
+	tmp_data &= ~mask;
+	tmp_data |= data & mask;
+	ret = i2c_smbus_write_byte_data(client, addr & 0xff, tmp_data);
+	if (ret < 0)
+		return ret;
+
+	if (addr & 0xff00) {
+		/* Back to bank 0 */
+		ret = i2c_smbus_write_byte_data(client, DA311_REG_BANK, 0x00);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+/* Init sequence taken from the android driver */
+static int da311_reset(struct i2c_client *client)
+{
+	const struct {
+		u16 addr;
+		u8 mask;
+		u8 data;
+	} init_data[] = {
+		{ DA311_REG_TEMP_CFG_REG,       0xff,   0x08 },
+		{ DA311_REG_CTRL_REG5,          0xff,   0x80 },
+		{ DA311_REG_CTRL_REG4,          0x30,   0x00 },
+		{ DA311_REG_CTRL_REG1,          0xff,   0x6f },
+		{ DA311_REG_TEMP_CFG_REG,       0xff,   0x88 },
+		{ DA311_REG_LDO_REG,            0xff,   0x02 },
+		{ DA311_REG_OTP_TRIM_OSC,       0xff,   0x27 },
+		{ DA311_REG_LPF_ABSOLUTE,       0xff,   0x30 },
+		{ DA311_REG_TEMP_OFF1,          0xff,   0x3f },
+		{ DA311_REG_TEMP_OFF2,          0xff,   0xff },
+		{ DA311_REG_TEMP_OFF3,          0xff,   0x0f },
+	};
+	int i, ret;
+
+	/* Reset */
+	ret = da311_register_mask_write(client, DA311_REG_SOFT_RESET,
+					0xff, 0xaa);
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < ARRAY_SIZE(init_data); i++) {
+		ret = da311_register_mask_write(client,
+						init_data[i].addr,
+						init_data[i].mask,
+						init_data[i].data);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int da311_enable(struct i2c_client *client, bool enable)
+{
+	u8 data = enable ? 0x00 : 0x20;
+
+	return da311_register_mask_write(client, DA311_REG_TEMP_CFG_REG,
+					 0x20, data);
+}
+
+static int da311_read_raw(struct iio_dev *indio_dev,
+				struct iio_chan_spec const *chan,
+				int *val, int *val2, long mask)
+{
+	struct da311_data *data = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = i2c_smbus_read_word_data(data->client, chan->address);
+		if (ret < 0)
+			return ret;
+		/*
+		 * Values are 12 bits, stored as 16 bits with the 4
+		 * least significant bits always 0.
+		 */
+		*val = (short)ret >> 4;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		*val = 0;
+		*val2 = da311_nscale;
+		return IIO_VAL_INT_PLUS_NANO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info da311_info = {
+	.driver_module	= THIS_MODULE,
+	.read_raw	= da311_read_raw,
+};
+
+static int da311_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	int ret;
+	struct iio_dev *indio_dev;
+	struct da311_data *data;
+
+	ret = i2c_smbus_read_byte_data(client, DA311_REG_CHIP_ID);
+	if (ret != DA311_CHIP_ID)
+		return (ret < 0) ? ret : -ENODEV;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	data->client = client;
+	i2c_set_clientdata(client, indio_dev);
+
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->info = &da311_info;
+	indio_dev->name = "da311";
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = da311_channels;
+	indio_dev->num_channels = ARRAY_SIZE(da311_channels);
+
+	ret = da311_reset(client);
+	if (ret < 0)
+		return ret;
+
+	ret = da311_enable(client, true);
+	if (ret < 0)
+		return ret;
+
+	ret = iio_device_register(indio_dev);
+	if (ret < 0) {
+		dev_err(&client->dev, "device_register failed\n");
+		da311_enable(client, false);
+	}
+
+	return ret;
+}
+
+static int da311_remove(struct i2c_client *client)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(client);
+
+	iio_device_unregister(indio_dev);
+
+	return da311_enable(client, false);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int da311_suspend(struct device *dev)
+{
+	return da311_enable(to_i2c_client(dev), false);
+}
+
+static int da311_resume(struct device *dev)
+{
+	return da311_enable(to_i2c_client(dev), true);
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(da311_pm_ops, da311_suspend, da311_resume);
+
+static const struct i2c_device_id da311_i2c_id[] = {
+	{"da311", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, da311_i2c_id);
+
+static struct i2c_driver da311_driver = {
+	.driver = {
+		.name = "da311",
+		.pm = &da311_pm_ops,
+	},
+	.probe		= da311_probe,
+	.remove		= da311_remove,
+	.id_table	= da311_i2c_id,
+};
+
+module_i2c_driver(da311_driver);
+
+MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
+MODULE_DESCRIPTION("MiraMEMS DA311 3-Axis Accelerometer driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/accel/dmard10.c b/drivers/iio/accel/dmard10.c
new file mode 100644
index 000000000000..b8736cc75656
--- /dev/null
+++ b/drivers/iio/accel/dmard10.c
@@ -0,0 +1,266 @@
+/**
+ * IIO driver for the 3-axis accelerometer Domintech ARD10.
+ *
+ * Copyright (c) 2016 Hans de Goede <hdegoede@redhat.com>
+ * Copyright (c) 2012 Domintech Technology Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/byteorder/generic.h>
+
+#define DMARD10_REG_ACTR			0x00
+#define DMARD10_REG_AFEM			0x0c
+#define DMARD10_REG_STADR			0x12
+#define DMARD10_REG_STAINT			0x1c
+#define DMARD10_REG_MISC2			0x1f
+#define DMARD10_REG_PD				0x21
+
+#define DMARD10_MODE_OFF			0x00
+#define DMARD10_MODE_STANDBY			0x02
+#define DMARD10_MODE_ACTIVE			0x06
+#define DMARD10_MODE_READ_OTP			0x12
+#define DMARD10_MODE_RESET_DATA_PATH		0x82
+
+/* AFEN set 1, ATM[2:0]=b'000 (normal), EN_Z/Y/X/T=1 */
+#define DMARD10_VALUE_AFEM_AFEN_NORMAL		0x8f
+/* ODR[3:0]=b'0111 (100Hz), CCK[3:0]=b'0100 (204.8kHZ) */
+#define DMARD10_VALUE_CKSEL_ODR_100_204		0x74
+/* INTC[6:5]=b'00 */
+#define DMARD10_VALUE_INTC			0x00
+/* TAP1/TAP2 Average 2 */
+#define DMARD10_VALUE_TAPNS_AVE_2		0x11
+
+#define DMARD10_VALUE_STADR			0x55
+#define DMARD10_VALUE_STAINT			0xaa
+#define DMARD10_VALUE_MISC2_OSCA_EN		0x08
+#define DMARD10_VALUE_PD_RST			0x52
+
+/* Offsets into the buffer read in dmard10_read_raw() */
+#define DMARD10_X_OFFSET			1
+#define DMARD10_Y_OFFSET			2
+#define DMARD10_Z_OFFSET			3
+
+/*
+ * a value of + or -128 corresponds to + or - 1G
+ * scale = 9.81 / 128 = 0.076640625
+ */
+
+static const int dmard10_nscale = 76640625;
+
+#define DMARD10_CHANNEL(reg, axis) {	\
+	.type = IIO_ACCEL,	\
+	.address = reg,	\
+	.modified = 1,	\
+	.channel2 = IIO_MOD_##axis,	\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+}
+
+static const struct iio_chan_spec dmard10_channels[] = {
+	DMARD10_CHANNEL(DMARD10_X_OFFSET, X),
+	DMARD10_CHANNEL(DMARD10_Y_OFFSET, Y),
+	DMARD10_CHANNEL(DMARD10_Z_OFFSET, Z),
+};
+
+struct dmard10_data {
+	struct i2c_client *client;
+};
+
+/* Init sequence taken from the android driver */
+static int dmard10_reset(struct i2c_client *client)
+{
+	unsigned char buffer[7];
+	int ret;
+
+	/* 1. Powerdown reset */
+	ret = i2c_smbus_write_byte_data(client, DMARD10_REG_PD,
+						DMARD10_VALUE_PD_RST);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * 2. ACTR => Standby mode => Download OTP to parameter reg =>
+	 *    Standby mode => Reset data path => Standby mode
+	 */
+	buffer[0] = DMARD10_REG_ACTR;
+	buffer[1] = DMARD10_MODE_STANDBY;
+	buffer[2] = DMARD10_MODE_READ_OTP;
+	buffer[3] = DMARD10_MODE_STANDBY;
+	buffer[4] = DMARD10_MODE_RESET_DATA_PATH;
+	buffer[5] = DMARD10_MODE_STANDBY;
+	ret = i2c_master_send(client, buffer, 6);
+	if (ret < 0)
+		return ret;
+
+	/* 3. OSCA_EN = 1, TSTO = b'000 (INT1 = normal, TEST0 = normal) */
+	ret = i2c_smbus_write_byte_data(client, DMARD10_REG_MISC2,
+						DMARD10_VALUE_MISC2_OSCA_EN);
+	if (ret < 0)
+		return ret;
+
+	/* 4. AFEN = 1 (AFE will powerdown after ADC) */
+	buffer[0] = DMARD10_REG_AFEM;
+	buffer[1] = DMARD10_VALUE_AFEM_AFEN_NORMAL;
+	buffer[2] = DMARD10_VALUE_CKSEL_ODR_100_204;
+	buffer[3] = DMARD10_VALUE_INTC;
+	buffer[4] = DMARD10_VALUE_TAPNS_AVE_2;
+	buffer[5] = 0x00; /* DLYC, no delay timing */
+	buffer[6] = 0x07; /* INTD=1 push-pull, INTA=1 active high, AUTOT=1 */
+	ret = i2c_master_send(client, buffer, 7);
+	if (ret < 0)
+		return ret;
+
+	/* 5. Activation mode */
+	ret = i2c_smbus_write_byte_data(client, DMARD10_REG_ACTR,
+						DMARD10_MODE_ACTIVE);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+/* Shutdown sequence taken from the android driver */
+static int dmard10_shutdown(struct i2c_client *client)
+{
+	unsigned char buffer[3];
+
+	buffer[0] = DMARD10_REG_ACTR;
+	buffer[1] = DMARD10_MODE_STANDBY;
+	buffer[2] = DMARD10_MODE_OFF;
+
+	return i2c_master_send(client, buffer, 3);
+}
+
+static int dmard10_read_raw(struct iio_dev *indio_dev,
+				struct iio_chan_spec const *chan,
+				int *val, int *val2, long mask)
+{
+	struct dmard10_data *data = iio_priv(indio_dev);
+	__le16 buf[4];
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		/*
+		 * Read 8 bytes starting at the REG_STADR register, trying to
+		 * read the individual X, Y, Z registers will always read 0.
+		 */
+		ret = i2c_smbus_read_i2c_block_data(data->client,
+						    DMARD10_REG_STADR,
+						    sizeof(buf), (u8 *)buf);
+		if (ret < 0)
+			return ret;
+		ret = le16_to_cpu(buf[chan->address]);
+		*val = sign_extend32(ret, 12);
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		*val = 0;
+		*val2 = dmard10_nscale;
+		return IIO_VAL_INT_PLUS_NANO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info dmard10_info = {
+	.driver_module	= THIS_MODULE,
+	.read_raw	= dmard10_read_raw,
+};
+
+static int dmard10_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	int ret;
+	struct iio_dev *indio_dev;
+	struct dmard10_data *data;
+
+	/* These 2 registers have special POR reset values used for id */
+	ret = i2c_smbus_read_byte_data(client, DMARD10_REG_STADR);
+	if (ret != DMARD10_VALUE_STADR)
+		return (ret < 0) ? ret : -ENODEV;
+
+	ret = i2c_smbus_read_byte_data(client, DMARD10_REG_STAINT);
+	if (ret != DMARD10_VALUE_STAINT)
+		return (ret < 0) ? ret : -ENODEV;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev) {
+		dev_err(&client->dev, "iio allocation failed!\n");
+		return -ENOMEM;
+	}
+
+	data = iio_priv(indio_dev);
+	data->client = client;
+	i2c_set_clientdata(client, indio_dev);
+
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->info = &dmard10_info;
+	indio_dev->name = "dmard10";
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = dmard10_channels;
+	indio_dev->num_channels = ARRAY_SIZE(dmard10_channels);
+
+	ret = dmard10_reset(client);
+	if (ret < 0)
+		return ret;
+
+	ret = iio_device_register(indio_dev);
+	if (ret < 0) {
+		dev_err(&client->dev, "device_register failed\n");
+		dmard10_shutdown(client);
+	}
+
+	return ret;
+}
+
+static int dmard10_remove(struct i2c_client *client)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(client);
+
+	iio_device_unregister(indio_dev);
+
+	return dmard10_shutdown(client);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int dmard10_suspend(struct device *dev)
+{
+	return dmard10_shutdown(to_i2c_client(dev));
+}
+
+static int dmard10_resume(struct device *dev)
+{
+	return dmard10_reset(to_i2c_client(dev));
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(dmard10_pm_ops, dmard10_suspend, dmard10_resume);
+
+static const struct i2c_device_id dmard10_i2c_id[] = {
+	{"dmard10", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, dmard10_i2c_id);
+
+static struct i2c_driver dmard10_driver = {
+	.driver = {
+		.name = "dmard10",
+		.pm = &dmard10_pm_ops,
+	},
+	.probe		= dmard10_probe,
+	.remove		= dmard10_remove,
+	.id_table	= dmard10_i2c_id,
+};
+
+module_i2c_driver(dmard10_driver);
+
+MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
+MODULE_DESCRIPTION("Domintech ARD10 3-Axis Accelerometer driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/accel/mma7660.c b/drivers/iio/accel/mma7660.c
index 03beadf14ad3..3a40774cca74 100644
--- a/drivers/iio/accel/mma7660.c
+++ b/drivers/iio/accel/mma7660.c
@@ -39,7 +39,7 @@
 
 #define MMA7660_SCALE_AVAIL	"0.467142857"
 
-const int mma7660_nscale = 467142857;
+static const int mma7660_nscale = 467142857;
 
 #define MMA7660_CHANNEL(reg, axis) {	\
 	.type = IIO_ACCEL,	\
diff --git a/drivers/iio/accel/mma8452.c b/drivers/iio/accel/mma8452.c
index d41e1b588e68..f418c588af6a 100644
--- a/drivers/iio/accel/mma8452.c
+++ b/drivers/iio/accel/mma8452.c
@@ -459,12 +459,14 @@ static int mma8452_read_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-		if (iio_buffer_enabled(indio_dev))
-			return -EBUSY;
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
 
 		mutex_lock(&data->lock);
 		ret = mma8452_read(data, buffer);
 		mutex_unlock(&data->lock);
+		iio_device_release_direct_mode(indio_dev);
 		if (ret < 0)
 			return ret;
 
@@ -664,37 +666,46 @@ static int mma8452_write_raw(struct iio_dev *indio_dev,
 	struct mma8452_data *data = iio_priv(indio_dev);
 	int i, ret;
 
-	if (iio_buffer_enabled(indio_dev))
-		return -EBUSY;
+	ret = iio_device_claim_direct_mode(indio_dev);
+	if (ret)
+		return ret;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_SAMP_FREQ:
 		i = mma8452_get_samp_freq_index(data, val, val2);
-		if (i < 0)
-			return i;
-
+		if (i < 0) {
+			ret = i;
+			break;
+		}
 		data->ctrl_reg1 &= ~MMA8452_CTRL_DR_MASK;
 		data->ctrl_reg1 |= i << MMA8452_CTRL_DR_SHIFT;
 
-		return mma8452_change_config(data, MMA8452_CTRL_REG1,
-					     data->ctrl_reg1);
+		ret = mma8452_change_config(data, MMA8452_CTRL_REG1,
+					    data->ctrl_reg1);
+		break;
 	case IIO_CHAN_INFO_SCALE:
 		i = mma8452_get_scale_index(data, val, val2);
-		if (i < 0)
-			return i;
+		if (i < 0) {
+			ret = i;
+			break;
+		}
 
 		data->data_cfg &= ~MMA8452_DATA_CFG_FS_MASK;
 		data->data_cfg |= i;
 
-		return mma8452_change_config(data, MMA8452_DATA_CFG,
-					     data->data_cfg);
+		ret = mma8452_change_config(data, MMA8452_DATA_CFG,
+					    data->data_cfg);
+		break;
 	case IIO_CHAN_INFO_CALIBBIAS:
-		if (val < -128 || val > 127)
-			return -EINVAL;
+		if (val < -128 || val > 127) {
+			ret = -EINVAL;
+			break;
+		}
 
-		return mma8452_change_config(data,
-					     MMA8452_OFF_X + chan->scan_index,
-					     val);
+		ret = mma8452_change_config(data,
+					    MMA8452_OFF_X + chan->scan_index,
+					    val);
+		break;
 
 	case IIO_CHAN_INFO_HIGH_PASS_FILTER_3DB_FREQUENCY:
 		if (val == 0 && val2 == 0) {
@@ -703,23 +714,30 @@ static int mma8452_write_raw(struct iio_dev *indio_dev,
 			data->data_cfg |= MMA8452_DATA_CFG_HPF_MASK;
 			ret = mma8452_set_hp_filter_frequency(data, val, val2);
 			if (ret < 0)
-				return ret;
+				break;
 		}
 
-		return mma8452_change_config(data, MMA8452_DATA_CFG,
+		ret = mma8452_change_config(data, MMA8452_DATA_CFG,
 					     data->data_cfg);
+		break;
 
 	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
 		ret = mma8452_get_odr_index(data);
 
 		for (i = 0; i < ARRAY_SIZE(mma8452_os_ratio); i++) {
-			if (mma8452_os_ratio[i][ret] == val)
-				return mma8452_set_power_mode(data, i);
+			if (mma8452_os_ratio[i][ret] == val) {
+				ret = mma8452_set_power_mode(data, i);
+				break;
+			}
 		}
-
+		break;
 	default:
-		return -EINVAL;
+		ret = -EINVAL;
+		break;
 	}
+
+	iio_device_release_direct_mode(indio_dev);
+	return ret;
 }
 
 static int mma8452_read_thresh(struct iio_dev *indio_dev,
@@ -1347,20 +1365,9 @@ static int mma8452_data_rdy_trigger_set_state(struct iio_trigger *trig,
 	return mma8452_change_config(data, MMA8452_CTRL_REG4, reg);
 }
 
-static int mma8452_validate_device(struct iio_trigger *trig,
-				   struct iio_dev *indio_dev)
-{
-	struct iio_dev *indio = iio_trigger_get_drvdata(trig);
-
-	if (indio != indio_dev)
-		return -EINVAL;
-
-	return 0;
-}
-
 static const struct iio_trigger_ops mma8452_trigger_ops = {
 	.set_trigger_state = mma8452_data_rdy_trigger_set_state,
-	.validate_device = mma8452_validate_device,
+	.validate_device = iio_trigger_validate_own_device,
 	.owner = THIS_MODULE,
 };
 
diff --git a/drivers/iio/accel/sca3000.c b/drivers/iio/accel/sca3000.c
new file mode 100644
index 000000000000..cb1d83fa19a0
--- /dev/null
+++ b/drivers/iio/accel/sca3000.c
@@ -0,0 +1,1576 @@
+/*
+ * sca3000_core.c -- support VTI sca3000 series accelerometers via SPI
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * Copyright (c) 2009 Jonathan Cameron <jic23@kernel.org>
+ *
+ * See industrialio/accels/sca3000.h for comments.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/fs.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/kernel.h>
+#include <linux/spi/spi.h>
+#include <linux/sysfs.h>
+#include <linux/module.h>
+#include <linux/uaccess.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/iio/events.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/kfifo_buf.h>
+
+#define SCA3000_WRITE_REG(a) (((a) << 2) | 0x02)
+#define SCA3000_READ_REG(a) ((a) << 2)
+
+#define SCA3000_REG_REVID_ADDR				0x00
+#define   SCA3000_REG_REVID_MAJOR_MASK			GENMASK(8, 4)
+#define   SCA3000_REG_REVID_MINOR_MASK			GENMASK(3, 0)
+
+#define SCA3000_REG_STATUS_ADDR				0x02
+#define   SCA3000_LOCKED				BIT(5)
+#define   SCA3000_EEPROM_CS_ERROR			BIT(1)
+#define   SCA3000_SPI_FRAME_ERROR			BIT(0)
+ 
+/* All reads done using register decrement so no need to directly access LSBs */
+#define SCA3000_REG_X_MSB_ADDR				0x05
+#define SCA3000_REG_Y_MSB_ADDR				0x07
+#define SCA3000_REG_Z_MSB_ADDR				0x09
+
+#define SCA3000_REG_RING_OUT_ADDR			0x0f
+
+/* Temp read untested - the e05 doesn't have the sensor */
+#define SCA3000_REG_TEMP_MSB_ADDR			0x13
+
+#define SCA3000_REG_MODE_ADDR				0x14
+#define SCA3000_MODE_PROT_MASK				0x28
+#define   SCA3000_REG_MODE_RING_BUF_ENABLE		BIT(7)
+#define   SCA3000_REG_MODE_RING_BUF_8BIT		BIT(6)
+
+/*
+ * Free fall detection triggers an interrupt if the acceleration
+ * is below a threshold for equivalent of 25cm drop
+ */
+#define   SCA3000_REG_MODE_FREE_FALL_DETECT		BIT(4)
+#define   SCA3000_REG_MODE_MEAS_MODE_NORMAL		0x00
+#define   SCA3000_REG_MODE_MEAS_MODE_OP_1		0x01
+#define   SCA3000_REG_MODE_MEAS_MODE_OP_2		0x02
+
+/*
+ * In motion detection mode the accelerations are band pass filtered
+ * (approx 1 - 25Hz) and then a programmable threshold used to trigger
+ * and interrupt.
+ */
+#define   SCA3000_REG_MODE_MEAS_MODE_MOT_DET		0x03
+#define   SCA3000_REG_MODE_MODE_MASK			0x03
+
+#define SCA3000_REG_BUF_COUNT_ADDR			0x15
+
+#define SCA3000_REG_INT_STATUS_ADDR			0x16
+#define   SCA3000_REG_INT_STATUS_THREE_QUARTERS		BIT(7)
+#define   SCA3000_REG_INT_STATUS_HALF			BIT(6)
+	
+#define SCA3000_INT_STATUS_FREE_FALL			BIT(3)
+#define SCA3000_INT_STATUS_Y_TRIGGER			BIT(2)
+#define SCA3000_INT_STATUS_X_TRIGGER			BIT(1)
+#define SCA3000_INT_STATUS_Z_TRIGGER			BIT(0)
+
+/* Used to allow access to multiplexed registers */
+#define SCA3000_REG_CTRL_SEL_ADDR			0x18
+/* Only available for SCA3000-D03 and SCA3000-D01 */
+#define   SCA3000_REG_CTRL_SEL_I2C_DISABLE		0x01
+#define   SCA3000_REG_CTRL_SEL_MD_CTRL			0x02
+#define   SCA3000_REG_CTRL_SEL_MD_Y_TH			0x03
+#define   SCA3000_REG_CTRL_SEL_MD_X_TH			0x04
+#define   SCA3000_REG_CTRL_SEL_MD_Z_TH			0x05
+/*
+ * BE VERY CAREFUL WITH THIS, IF 3 BITS ARE NOT SET the device
+ * will not function
+ */
+#define   SCA3000_REG_CTRL_SEL_OUT_CTRL			0x0B
+
+#define     SCA3000_REG_OUT_CTRL_PROT_MASK		0xE0
+#define     SCA3000_REG_OUT_CTRL_BUF_X_EN		0x10
+#define     SCA3000_REG_OUT_CTRL_BUF_Y_EN		0x08
+#define     SCA3000_REG_OUT_CTRL_BUF_Z_EN		0x04
+#define     SCA3000_REG_OUT_CTRL_BUF_DIV_MASK		0x03
+#define     SCA3000_REG_OUT_CTRL_BUF_DIV_4		0x02
+#define     SCA3000_REG_OUT_CTRL_BUF_DIV_2		0x01
+
+
+/*
+ * Control which motion detector interrupts are on.
+ * For now only OR combinations are supported.
+ */
+#define SCA3000_MD_CTRL_PROT_MASK			0xC0
+#define SCA3000_MD_CTRL_OR_Y				BIT(0)
+#define SCA3000_MD_CTRL_OR_X				BIT(1)
+#define SCA3000_MD_CTRL_OR_Z				BIT(2)
+/* Currently unsupported */
+#define SCA3000_MD_CTRL_AND_Y				BIT(3)
+#define SCA3000_MD_CTRL_AND_X				BIT(4)
+#define SAC3000_MD_CTRL_AND_Z				BIT(5)
+
+/*
+ * Some control registers of complex access methods requiring this register to
+ * be used to remove a lock.
+ */
+#define SCA3000_REG_UNLOCK_ADDR				0x1e
+
+#define SCA3000_REG_INT_MASK_ADDR			0x21
+#define   SCA3000_REG_INT_MASK_PROT_MASK		0x1C
+ 
+#define   SCA3000_REG_INT_MASK_RING_THREE_QUARTER	BIT(7)
+#define   SCA3000_REG_INT_MASK_RING_HALF		BIT(6)
+
+#define SCA3000_REG_INT_MASK_ALL_INTS			0x02
+#define SCA3000_REG_INT_MASK_ACTIVE_HIGH		0x01
+#define SCA3000_REG_INT_MASK_ACTIVE_LOW			0x00
+/* Values of multiplexed registers (write to ctrl_data after select) */
+#define SCA3000_REG_CTRL_DATA_ADDR			0x22
+
+/*
+ * Measurement modes available on some sca3000 series chips. Code assumes others
+ * may become available in the future.
+ *
+ * Bypass - Bypass the low-pass filter in the signal channel so as to increase
+ *          signal bandwidth.
+ *
+ * Narrow - Narrow low-pass filtering of the signal channel and half output
+ *          data rate by decimation.
+ *
+ * Wide - Widen low-pass filtering of signal channel to increase bandwidth
+ */
+#define SCA3000_OP_MODE_BYPASS				0x01
+#define SCA3000_OP_MODE_NARROW				0x02
+#define SCA3000_OP_MODE_WIDE				0x04
+#define SCA3000_MAX_TX 6
+#define SCA3000_MAX_RX 2
+
+/**
+ * struct sca3000_state - device instance state information
+ * @us:			the associated spi device
+ * @info:			chip variant information
+ * @last_timestamp:		the timestamp of the last event
+ * @mo_det_use_count:		reference counter for the motion detection unit
+ * @lock:			lock used to protect elements of sca3000_state
+ *				and the underlying device state.
+ * @tx:			dma-able transmit buffer
+ * @rx:			dma-able receive buffer
+ **/
+struct sca3000_state {
+	struct spi_device		*us;
+	const struct sca3000_chip_info	*info;
+	s64				last_timestamp;
+	int				mo_det_use_count;
+	struct mutex			lock;
+	/* Can these share a cacheline ? */
+	u8				rx[384] ____cacheline_aligned;
+	u8				tx[6] ____cacheline_aligned;
+};
+
+/**
+ * struct sca3000_chip_info - model dependent parameters
+ * @scale:			scale * 10^-6
+ * @temp_output:		some devices have temperature sensors.
+ * @measurement_mode_freq:	normal mode sampling frequency
+ * @measurement_mode_3db_freq:	3db cutoff frequency of the low pass filter for
+ * the normal measurement mode.
+ * @option_mode_1:		first optional mode. Not all models have one
+ * @option_mode_1_freq:		option mode 1 sampling frequency
+ * @option_mode_1_3db_freq:	3db cutoff frequency of the low pass filter for
+ * the first option mode.
+ * @option_mode_2:		second optional mode. Not all chips have one
+ * @option_mode_2_freq:		option mode 2 sampling frequency
+ * @option_mode_2_3db_freq:	3db cutoff frequency of the low pass filter for
+ * the second option mode.
+ * @mod_det_mult_xz:		Bit wise multipliers to calculate the threshold
+ * for motion detection in the x and z axis.
+ * @mod_det_mult_y:		Bit wise multipliers to calculate the threshold
+ * for motion detection in the y axis.
+ *
+ * This structure is used to hold information about the functionality of a given
+ * sca3000 variant.
+ **/
+struct sca3000_chip_info {
+	unsigned int		scale;
+	bool			temp_output;
+	int			measurement_mode_freq;
+	int			measurement_mode_3db_freq;
+	int			option_mode_1;
+	int			option_mode_1_freq;
+	int			option_mode_1_3db_freq;
+	int			option_mode_2;
+	int			option_mode_2_freq;
+	int			option_mode_2_3db_freq;
+	int			mot_det_mult_xz[6];
+	int			mot_det_mult_y[7];
+};
+
+enum sca3000_variant {
+	d01,
+	e02,
+	e04,
+	e05,
+};
+
+/*
+ * Note where option modes are not defined, the chip simply does not
+ * support any.
+ * Other chips in the sca3000 series use i2c and are not included here.
+ *
+ * Some of these devices are only listed in the family data sheet and
+ * do not actually appear to be available.
+ */
+static const struct sca3000_chip_info sca3000_spi_chip_info_tbl[] = {
+	[d01] = {
+		.scale = 7357,
+		.temp_output = true,
+		.measurement_mode_freq = 250,
+		.measurement_mode_3db_freq = 45,
+		.option_mode_1 = SCA3000_OP_MODE_BYPASS,
+		.option_mode_1_freq = 250,
+		.option_mode_1_3db_freq = 70,
+		.mot_det_mult_xz = {50, 100, 200, 350, 650, 1300},
+		.mot_det_mult_y = {50, 100, 150, 250, 450, 850, 1750},
+	},
+	[e02] = {
+		.scale = 9810,
+		.measurement_mode_freq = 125,
+		.measurement_mode_3db_freq = 40,
+		.option_mode_1 = SCA3000_OP_MODE_NARROW,
+		.option_mode_1_freq = 63,
+		.option_mode_1_3db_freq = 11,
+		.mot_det_mult_xz = {100, 150, 300, 550, 1050, 2050},
+		.mot_det_mult_y = {50, 100, 200, 350, 700, 1350, 2700},
+	},
+	[e04] = {
+		.scale = 19620,
+		.measurement_mode_freq = 100,
+		.measurement_mode_3db_freq = 38,
+		.option_mode_1 = SCA3000_OP_MODE_NARROW,
+		.option_mode_1_freq = 50,
+		.option_mode_1_3db_freq = 9,
+		.option_mode_2 = SCA3000_OP_MODE_WIDE,
+		.option_mode_2_freq = 400,
+		.option_mode_2_3db_freq = 70,
+		.mot_det_mult_xz = {200, 300, 600, 1100, 2100, 4100},
+		.mot_det_mult_y = {100, 200, 400, 7000, 1400, 2700, 54000},
+	},
+	[e05] = {
+		.scale = 61313,
+		.measurement_mode_freq = 200,
+		.measurement_mode_3db_freq = 60,
+		.option_mode_1 = SCA3000_OP_MODE_NARROW,
+		.option_mode_1_freq = 50,
+		.option_mode_1_3db_freq = 9,
+		.option_mode_2 = SCA3000_OP_MODE_WIDE,
+		.option_mode_2_freq = 400,
+		.option_mode_2_3db_freq = 75,
+		.mot_det_mult_xz = {600, 900, 1700, 3200, 6100, 11900},
+		.mot_det_mult_y = {300, 600, 1200, 2000, 4100, 7800, 15600},
+	},
+};
+
+static int sca3000_write_reg(struct sca3000_state *st, u8 address, u8 val)
+{
+	st->tx[0] = SCA3000_WRITE_REG(address);
+	st->tx[1] = val;
+	return spi_write(st->us, st->tx, 2);
+}
+
+static int sca3000_read_data_short(struct sca3000_state *st,
+				   u8 reg_address_high,
+				   int len)
+{
+	struct spi_transfer xfer[2] = {
+		{
+			.len = 1,
+			.tx_buf = st->tx,
+		}, {
+			.len = len,
+			.rx_buf = st->rx,
+		}
+	};
+	st->tx[0] = SCA3000_READ_REG(reg_address_high);
+
+	return spi_sync_transfer(st->us, xfer, ARRAY_SIZE(xfer));
+}
+
+/**
+ * sca3000_reg_lock_on() - test if the ctrl register lock is on
+ * @st: Driver specific device instance data.
+ *
+ * Lock must be held.
+ **/
+static int sca3000_reg_lock_on(struct sca3000_state *st)
+{
+	int ret;
+
+	ret = sca3000_read_data_short(st, SCA3000_REG_STATUS_ADDR, 1);
+	if (ret < 0)
+		return ret;
+
+	return !(st->rx[0] & SCA3000_LOCKED);
+}
+
+/**
+ * __sca3000_unlock_reg_lock() - unlock the control registers
+ * @st: Driver specific device instance data.
+ *
+ * Note the device does not appear to support doing this in a single transfer.
+ * This should only ever be used as part of ctrl reg read.
+ * Lock must be held before calling this
+ */
+static int __sca3000_unlock_reg_lock(struct sca3000_state *st)
+{
+	struct spi_transfer xfer[3] = {
+		{
+			.len = 2,
+			.cs_change = 1,
+			.tx_buf = st->tx,
+		}, {
+			.len = 2,
+			.cs_change = 1,
+			.tx_buf = st->tx + 2,
+		}, {
+			.len = 2,
+			.tx_buf = st->tx + 4,
+		},
+	};
+	st->tx[0] = SCA3000_WRITE_REG(SCA3000_REG_UNLOCK_ADDR);
+	st->tx[1] = 0x00;
+	st->tx[2] = SCA3000_WRITE_REG(SCA3000_REG_UNLOCK_ADDR);
+	st->tx[3] = 0x50;
+	st->tx[4] = SCA3000_WRITE_REG(SCA3000_REG_UNLOCK_ADDR);
+	st->tx[5] = 0xA0;
+
+	return spi_sync_transfer(st->us, xfer, ARRAY_SIZE(xfer));
+}
+
+/**
+ * sca3000_write_ctrl_reg() write to a lock protect ctrl register
+ * @st: Driver specific device instance data.
+ * @sel: selects which registers we wish to write to
+ * @val: the value to be written
+ *
+ * Certain control registers are protected against overwriting by the lock
+ * register and use a shared write address. This function allows writing of
+ * these registers.
+ * Lock must be held.
+ */
+static int sca3000_write_ctrl_reg(struct sca3000_state *st,
+				  u8 sel,
+				  uint8_t val)
+{
+	int ret;
+
+	ret = sca3000_reg_lock_on(st);
+	if (ret < 0)
+		goto error_ret;
+	if (ret) {
+		ret = __sca3000_unlock_reg_lock(st);
+		if (ret)
+			goto error_ret;
+	}
+
+	/* Set the control select register */
+	ret = sca3000_write_reg(st, SCA3000_REG_CTRL_SEL_ADDR, sel);
+	if (ret)
+		goto error_ret;
+
+	/* Write the actual value into the register */
+	ret = sca3000_write_reg(st, SCA3000_REG_CTRL_DATA_ADDR, val);
+
+error_ret:
+	return ret;
+}
+
+/**
+ * sca3000_read_ctrl_reg() read from lock protected control register.
+ * @st: Driver specific device instance data.
+ * @ctrl_reg: Which ctrl register do we want to read.
+ *
+ * Lock must be held.
+ */
+static int sca3000_read_ctrl_reg(struct sca3000_state *st,
+				 u8 ctrl_reg)
+{
+	int ret;
+
+	ret = sca3000_reg_lock_on(st);
+	if (ret < 0)
+		goto error_ret;
+	if (ret) {
+		ret = __sca3000_unlock_reg_lock(st);
+		if (ret)
+			goto error_ret;
+	}
+	/* Set the control select register */
+	ret = sca3000_write_reg(st, SCA3000_REG_CTRL_SEL_ADDR, ctrl_reg);
+	if (ret)
+		goto error_ret;
+	ret = sca3000_read_data_short(st, SCA3000_REG_CTRL_DATA_ADDR, 1);
+	if (ret)
+		goto error_ret;
+	return st->rx[0];
+error_ret:
+	return ret;
+}
+
+/**
+ * sca3000_show_rev() - sysfs interface to read the chip revision number
+ * @indio_dev: Device instance specific generic IIO data.
+ * Driver specific device instance data can be obtained via
+ * via iio_priv(indio_dev)
+ */
+static int sca3000_print_rev(struct iio_dev *indio_dev)
+{
+	int ret;
+	struct sca3000_state *st = iio_priv(indio_dev);
+
+	mutex_lock(&st->lock);
+	ret = sca3000_read_data_short(st, SCA3000_REG_REVID_ADDR, 1);
+	if (ret < 0)
+		goto error_ret;
+	dev_info(&indio_dev->dev,
+		 "sca3000 revision major=%lu, minor=%lu\n",
+		 st->rx[0] & SCA3000_REG_REVID_MAJOR_MASK,
+		 st->rx[0] & SCA3000_REG_REVID_MINOR_MASK);
+error_ret:
+	mutex_unlock(&st->lock);
+
+	return ret;
+}
+
+static ssize_t
+sca3000_show_available_3db_freqs(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int len;
+
+	len = sprintf(buf, "%d", st->info->measurement_mode_3db_freq);
+	if (st->info->option_mode_1)
+		len += sprintf(buf + len, " %d",
+			       st->info->option_mode_1_3db_freq);
+	if (st->info->option_mode_2)
+		len += sprintf(buf + len, " %d",
+			       st->info->option_mode_2_3db_freq);
+	len += sprintf(buf + len, "\n");
+
+	return len;
+}
+
+static IIO_DEVICE_ATTR(in_accel_filter_low_pass_3db_frequency_available,
+		       S_IRUGO, sca3000_show_available_3db_freqs,
+		       NULL, 0);
+
+static const struct iio_event_spec sca3000_event = {
+	.type = IIO_EV_TYPE_MAG,
+	.dir = IIO_EV_DIR_RISING,
+	.mask_separate = BIT(IIO_EV_INFO_VALUE) | BIT(IIO_EV_INFO_ENABLE),
+};
+
+/*
+ * Note the hack in the number of bits to pretend we have 2 more than
+ * we do in the fifo.
+ */
+#define SCA3000_CHAN(index, mod)				\
+	{							\
+		.type = IIO_ACCEL,				\
+		.modified = 1,					\
+		.channel2 = mod,				\
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) |\
+			BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY),\
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),\
+		.address = index,				\
+		.scan_index = index,				\
+		.scan_type = {					\
+			.sign = 's',				\
+			.realbits = 13,				\
+			.storagebits = 16,			\
+			.shift = 3,				\
+			.endianness = IIO_BE,			\
+		},						\
+		.event_spec = &sca3000_event,			\
+		.num_event_specs = 1,				\
+	}
+
+static const struct iio_event_spec sca3000_freefall_event_spec = {
+	.type = IIO_EV_TYPE_MAG,
+	.dir = IIO_EV_DIR_FALLING,
+	.mask_separate = BIT(IIO_EV_INFO_ENABLE) |
+		BIT(IIO_EV_INFO_PERIOD),
+};
+
+static const struct iio_chan_spec sca3000_channels[] = {
+	SCA3000_CHAN(0, IIO_MOD_X),
+	SCA3000_CHAN(1, IIO_MOD_Y),
+	SCA3000_CHAN(2, IIO_MOD_Z),
+	{
+		.type = IIO_ACCEL,
+		.modified = 1,
+		.channel2 = IIO_MOD_X_AND_Y_AND_Z,
+		.scan_index = -1, /* Fake channel */
+		.event_spec = &sca3000_freefall_event_spec,
+		.num_event_specs = 1,
+	},
+};
+
+static const struct iio_chan_spec sca3000_channels_with_temp[] = {
+	SCA3000_CHAN(0, IIO_MOD_X),
+	SCA3000_CHAN(1, IIO_MOD_Y),
+	SCA3000_CHAN(2, IIO_MOD_Z),
+	{
+		.type = IIO_TEMP,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) |
+			BIT(IIO_CHAN_INFO_OFFSET),
+		/* No buffer support */
+		.scan_index = -1,
+	},
+	{
+		.type = IIO_ACCEL,
+		.modified = 1,
+		.channel2 = IIO_MOD_X_AND_Y_AND_Z,
+		.scan_index = -1, /* Fake channel */
+		.event_spec = &sca3000_freefall_event_spec,
+		.num_event_specs = 1,
+	},
+};
+
+static u8 sca3000_addresses[3][3] = {
+	[0] = {SCA3000_REG_X_MSB_ADDR, SCA3000_REG_CTRL_SEL_MD_X_TH,
+	       SCA3000_MD_CTRL_OR_X},
+	[1] = {SCA3000_REG_Y_MSB_ADDR, SCA3000_REG_CTRL_SEL_MD_Y_TH,
+	       SCA3000_MD_CTRL_OR_Y},
+	[2] = {SCA3000_REG_Z_MSB_ADDR, SCA3000_REG_CTRL_SEL_MD_Z_TH,
+	       SCA3000_MD_CTRL_OR_Z},
+};
+
+/**
+ * __sca3000_get_base_freq() - obtain mode specific base frequency
+ * @st: Private driver specific device instance specific state.
+ * @info: chip type specific information.
+ * @base_freq: Base frequency for the current measurement mode.
+ *
+ * lock must be held
+ */
+static inline int __sca3000_get_base_freq(struct sca3000_state *st,
+					  const struct sca3000_chip_info *info,
+					  int *base_freq)
+{
+	int ret;
+
+	ret = sca3000_read_data_short(st, SCA3000_REG_MODE_ADDR, 1);
+	if (ret)
+		goto error_ret;
+	switch (SCA3000_REG_MODE_MODE_MASK & st->rx[0]) {
+	case SCA3000_REG_MODE_MEAS_MODE_NORMAL:
+		*base_freq = info->measurement_mode_freq;
+		break;
+	case SCA3000_REG_MODE_MEAS_MODE_OP_1:
+		*base_freq = info->option_mode_1_freq;
+		break;
+	case SCA3000_REG_MODE_MEAS_MODE_OP_2:
+		*base_freq = info->option_mode_2_freq;
+		break;
+	default:
+		ret = -EINVAL;
+	}
+error_ret:
+	return ret;
+}
+
+/**
+ * sca3000_read_raw_samp_freq() - read_raw handler for IIO_CHAN_INFO_SAMP_FREQ
+ * @st: Private driver specific device instance specific state.
+ * @val: The frequency read back.
+ *
+ * lock must be held
+ **/
+static int sca3000_read_raw_samp_freq(struct sca3000_state *st, int *val)
+{
+	int ret;
+
+	ret = __sca3000_get_base_freq(st, st->info, val);
+	if (ret)
+		return ret;
+
+	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL);
+	if (ret < 0)
+		return ret;
+
+	if (*val > 0) {
+		ret &= SCA3000_REG_OUT_CTRL_BUF_DIV_MASK;
+		switch (ret) {
+		case SCA3000_REG_OUT_CTRL_BUF_DIV_2:
+			*val /= 2;
+			break;
+		case SCA3000_REG_OUT_CTRL_BUF_DIV_4:
+			*val /= 4;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * sca3000_write_raw_samp_freq() - write_raw handler for IIO_CHAN_INFO_SAMP_FREQ
+ * @st: Private driver specific device instance specific state.
+ * @val: The frequency desired.
+ *
+ * lock must be held
+ */
+static int sca3000_write_raw_samp_freq(struct sca3000_state *st, int val)
+{
+	int ret, base_freq, ctrlval;
+
+	ret = __sca3000_get_base_freq(st, st->info, &base_freq);
+	if (ret)
+		return ret;
+
+	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL);
+	if (ret < 0)
+		return ret;
+
+	ctrlval = ret & ~SCA3000_REG_OUT_CTRL_BUF_DIV_MASK;
+
+	if (val == base_freq / 2)
+		ctrlval |= SCA3000_REG_OUT_CTRL_BUF_DIV_2;
+	if (val == base_freq / 4)
+		ctrlval |= SCA3000_REG_OUT_CTRL_BUF_DIV_4;
+	else if (val != base_freq)
+		return -EINVAL;
+
+	return sca3000_write_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL,
+				     ctrlval);
+}
+
+static int sca3000_read_3db_freq(struct sca3000_state *st, int *val)
+{
+	int ret;
+
+	ret = sca3000_read_data_short(st, SCA3000_REG_MODE_ADDR, 1);
+	if (ret)
+		return ret;
+
+	/* mask bottom 2 bits - only ones that are relevant */
+	st->rx[0] &= SCA3000_REG_MODE_MODE_MASK;
+	switch (st->rx[0]) {
+	case SCA3000_REG_MODE_MEAS_MODE_NORMAL:
+		*val = st->info->measurement_mode_3db_freq;
+		return IIO_VAL_INT;
+	case SCA3000_REG_MODE_MEAS_MODE_MOT_DET:
+		return -EBUSY;
+	case SCA3000_REG_MODE_MEAS_MODE_OP_1:
+		*val = st->info->option_mode_1_3db_freq;
+		return IIO_VAL_INT;
+	case SCA3000_REG_MODE_MEAS_MODE_OP_2:
+		*val = st->info->option_mode_2_3db_freq;
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int sca3000_write_3db_freq(struct sca3000_state *st, int val)
+{
+	int ret;
+	int mode;
+
+	if (val == st->info->measurement_mode_3db_freq)
+		mode = SCA3000_REG_MODE_MEAS_MODE_NORMAL;
+	else if (st->info->option_mode_1 &&
+		 (val == st->info->option_mode_1_3db_freq))
+		mode = SCA3000_REG_MODE_MEAS_MODE_OP_1;
+	else if (st->info->option_mode_2 &&
+		 (val == st->info->option_mode_2_3db_freq))
+		mode = SCA3000_REG_MODE_MEAS_MODE_OP_2;
+	else
+		return -EINVAL;
+	ret = sca3000_read_data_short(st, SCA3000_REG_MODE_ADDR, 1);
+	if (ret)
+		return ret;
+
+	st->rx[0] &= ~SCA3000_REG_MODE_MODE_MASK;
+	st->rx[0] |= (mode & SCA3000_REG_MODE_MODE_MASK);
+
+	return sca3000_write_reg(st, SCA3000_REG_MODE_ADDR, st->rx[0]);
+}
+
+static int sca3000_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val,
+			    int *val2,
+			    long mask)
+{
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int ret;
+	u8 address;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		mutex_lock(&st->lock);
+		if (chan->type == IIO_ACCEL) {
+			if (st->mo_det_use_count) {
+				mutex_unlock(&st->lock);
+				return -EBUSY;
+			}
+			address = sca3000_addresses[chan->address][0];
+			ret = sca3000_read_data_short(st, address, 2);
+			if (ret < 0) {
+				mutex_unlock(&st->lock);
+				return ret;
+			}
+			*val = (be16_to_cpup((__be16 *)st->rx) >> 3) & 0x1FFF;
+			*val = ((*val) << (sizeof(*val) * 8 - 13)) >>
+				(sizeof(*val) * 8 - 13);
+		} else {
+			/* get the temperature when available */
+			ret = sca3000_read_data_short(st,
+						      SCA3000_REG_TEMP_MSB_ADDR,
+						      2);
+			if (ret < 0) {
+				mutex_unlock(&st->lock);
+				return ret;
+			}
+			*val = ((st->rx[0] & 0x3F) << 3) |
+			       ((st->rx[1] & 0xE0) >> 5);
+		}
+		mutex_unlock(&st->lock);
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		*val = 0;
+		if (chan->type == IIO_ACCEL)
+			*val2 = st->info->scale;
+		else /* temperature */
+			*val2 = 555556;
+		return IIO_VAL_INT_PLUS_MICRO;
+	case IIO_CHAN_INFO_OFFSET:
+		*val = -214;
+		*val2 = 600000;
+		return IIO_VAL_INT_PLUS_MICRO;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		mutex_lock(&st->lock);
+		ret = sca3000_read_raw_samp_freq(st, val);
+		mutex_unlock(&st->lock);
+		return ret ? ret : IIO_VAL_INT;
+	case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY:
+		mutex_lock(&st->lock);
+		ret = sca3000_read_3db_freq(st, val);
+		mutex_unlock(&st->lock);
+		return ret;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int sca3000_write_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
+{
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (val2)
+			return -EINVAL;
+		mutex_lock(&st->lock);
+		ret = sca3000_write_raw_samp_freq(st, val);
+		mutex_unlock(&st->lock);
+		return ret;
+	case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY:
+		if (val2)
+			return -EINVAL;
+		mutex_lock(&st->lock);
+		ret = sca3000_write_3db_freq(st, val);
+		mutex_unlock(&st->lock);
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+/**
+ * sca3000_read_av_freq() - sysfs function to get available frequencies
+ * @dev: Device structure for this device.
+ * @attr: Description of the attribute.
+ * @buf: Incoming string
+ *
+ * The later modes are only relevant to the ring buffer - and depend on current
+ * mode. Note that data sheet gives rather wide tolerances for these so integer
+ * division will give good enough answer and not all chips have them specified
+ * at all.
+ **/
+static ssize_t sca3000_read_av_freq(struct device *dev,
+				    struct device_attribute *attr,
+				    char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int len = 0, ret, val;
+
+	mutex_lock(&st->lock);
+	ret = sca3000_read_data_short(st, SCA3000_REG_MODE_ADDR, 1);
+	val = st->rx[0];
+	mutex_unlock(&st->lock);
+	if (ret)
+		goto error_ret;
+
+	switch (val & SCA3000_REG_MODE_MODE_MASK) {
+	case SCA3000_REG_MODE_MEAS_MODE_NORMAL:
+		len += sprintf(buf + len, "%d %d %d\n",
+			       st->info->measurement_mode_freq,
+			       st->info->measurement_mode_freq / 2,
+			       st->info->measurement_mode_freq / 4);
+		break;
+	case SCA3000_REG_MODE_MEAS_MODE_OP_1:
+		len += sprintf(buf + len, "%d %d %d\n",
+			       st->info->option_mode_1_freq,
+			       st->info->option_mode_1_freq / 2,
+			       st->info->option_mode_1_freq / 4);
+		break;
+	case SCA3000_REG_MODE_MEAS_MODE_OP_2:
+		len += sprintf(buf + len, "%d %d %d\n",
+			       st->info->option_mode_2_freq,
+			       st->info->option_mode_2_freq / 2,
+			       st->info->option_mode_2_freq / 4);
+		break;
+	}
+	return len;
+error_ret:
+	return ret;
+}
+
+/*
+ * Should only really be registered if ring buffer support is compiled in.
+ * Does no harm however and doing it right would add a fair bit of complexity
+ */
+static IIO_DEV_ATTR_SAMP_FREQ_AVAIL(sca3000_read_av_freq);
+
+/**
+ * sca3000_read_event_value() - query of a threshold or period
+ **/
+static int sca3000_read_event_value(struct iio_dev *indio_dev,
+				    const struct iio_chan_spec *chan,
+				    enum iio_event_type type,
+				    enum iio_event_direction dir,
+				    enum iio_event_info info,
+				    int *val, int *val2)
+{
+	int ret, i;
+	struct sca3000_state *st = iio_priv(indio_dev);
+
+	switch (info) {
+	case IIO_EV_INFO_VALUE:
+		mutex_lock(&st->lock);
+		ret = sca3000_read_ctrl_reg(st,
+					    sca3000_addresses[chan->address][1]);
+		mutex_unlock(&st->lock);
+		if (ret < 0)
+			return ret;
+		*val = 0;
+		if (chan->channel2 == IIO_MOD_Y)
+			for_each_set_bit(i, (unsigned long *)&ret,
+					 ARRAY_SIZE(st->info->mot_det_mult_y))
+				*val += st->info->mot_det_mult_y[i];
+		else
+			for_each_set_bit(i, (unsigned long *)&ret,
+					 ARRAY_SIZE(st->info->mot_det_mult_xz))
+				*val += st->info->mot_det_mult_xz[i];
+
+		return IIO_VAL_INT;
+	case IIO_EV_INFO_PERIOD:
+		*val = 0;
+		*val2 = 226000;
+		return IIO_VAL_INT_PLUS_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+/**
+ * sca3000_write_value() - control of threshold and period
+ * @indio_dev: Device instance specific IIO information.
+ * @chan: Description of the channel for which the event is being
+ * configured.
+ * @type: The type of event being configured, here magnitude rising
+ * as everything else is read only.
+ * @dir: Direction of the event (here rising)
+ * @info: What information about the event are we configuring.
+ * Here the threshold only.
+ * @val: Integer part of the value being written..
+ * @val2: Non integer part of the value being written. Here always 0.
+ */
+static int sca3000_write_event_value(struct iio_dev *indio_dev,
+				     const struct iio_chan_spec *chan,
+				     enum iio_event_type type,
+				     enum iio_event_direction dir,
+				     enum iio_event_info info,
+				     int val, int val2)
+{
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int ret;
+	int i;
+	u8 nonlinear = 0;
+
+	if (chan->channel2 == IIO_MOD_Y) {
+		i = ARRAY_SIZE(st->info->mot_det_mult_y);
+		while (i > 0)
+			if (val >= st->info->mot_det_mult_y[--i]) {
+				nonlinear |= (1 << i);
+				val -= st->info->mot_det_mult_y[i];
+			}
+	} else {
+		i = ARRAY_SIZE(st->info->mot_det_mult_xz);
+		while (i > 0)
+			if (val >= st->info->mot_det_mult_xz[--i]) {
+				nonlinear |= (1 << i);
+				val -= st->info->mot_det_mult_xz[i];
+			}
+	}
+
+	mutex_lock(&st->lock);
+	ret = sca3000_write_ctrl_reg(st,
+				     sca3000_addresses[chan->address][1],
+				     nonlinear);
+	mutex_unlock(&st->lock);
+
+	return ret;
+}
+
+static struct attribute *sca3000_attributes[] = {
+	&iio_dev_attr_in_accel_filter_low_pass_3db_frequency_available.dev_attr.attr,
+	&iio_dev_attr_sampling_frequency_available.dev_attr.attr,
+	NULL,
+};
+
+static const struct attribute_group sca3000_attribute_group = {
+	.attrs = sca3000_attributes,
+};
+
+static int sca3000_read_data(struct sca3000_state *st,
+			     u8 reg_address_high,
+			     u8 *rx,
+			     int len)
+{
+	int ret;
+	struct spi_transfer xfer[2] = {
+		{
+			.len = 1,
+			.tx_buf = st->tx,
+		}, {
+			.len = len,
+			.rx_buf = rx,
+		}
+	};
+
+	st->tx[0] = SCA3000_READ_REG(reg_address_high);
+	ret = spi_sync_transfer(st->us, xfer, ARRAY_SIZE(xfer));
+	if (ret) {
+		dev_err(get_device(&st->us->dev), "problem reading register");
+		return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * sca3000_ring_int_process() - ring specific interrupt handling.
+ * @val: Value of the interrupt status register.
+ * @indio_dev: Device instance specific IIO device structure.
+ */
+static void sca3000_ring_int_process(u8 val, struct iio_dev *indio_dev)
+{
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int ret, i, num_available;
+
+	mutex_lock(&st->lock);
+
+	if (val & SCA3000_REG_INT_STATUS_HALF) {
+		ret = sca3000_read_data_short(st, SCA3000_REG_BUF_COUNT_ADDR,
+					      1);
+		if (ret)
+			goto error_ret;
+		num_available = st->rx[0];
+		/*
+		 * num_available is the total number of samples available
+		 * i.e. number of time points * number of channels.
+		 */
+		ret = sca3000_read_data(st, SCA3000_REG_RING_OUT_ADDR, st->rx,
+					num_available * 2);
+		if (ret)
+			goto error_ret;
+		for (i = 0; i < num_available / 3; i++) {
+			/*
+			 * Dirty hack to cover for 11 bit in fifo, 13 bit
+			 * direct reading.
+			 *
+			 * In theory the bottom two bits are undefined.
+			 * In reality they appear to always be 0.
+			 */
+			iio_push_to_buffers(indio_dev, st->rx + i * 3 * 2);
+		}
+	}
+error_ret:
+	mutex_unlock(&st->lock);
+}
+
+/**
+ * sca3000_event_handler() - handling ring and non ring events
+ * @irq: The irq being handled.
+ * @private: struct iio_device pointer for the device.
+ *
+ * Ring related interrupt handler. Depending on event, push to
+ * the ring buffer event chrdev or the event one.
+ *
+ * This function is complicated by the fact that the devices can signify ring
+ * and non ring events via the same interrupt line and they can only
+ * be distinguished via a read of the relevant status register.
+ */
+static irqreturn_t sca3000_event_handler(int irq, void *private)
+{
+	struct iio_dev *indio_dev = private;
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int ret, val;
+	s64 last_timestamp = iio_get_time_ns(indio_dev);
+
+	/*
+	 * Could lead if badly timed to an extra read of status reg,
+	 * but ensures no interrupt is missed.
+	 */
+	mutex_lock(&st->lock);
+	ret = sca3000_read_data_short(st, SCA3000_REG_INT_STATUS_ADDR, 1);
+	val = st->rx[0];
+	mutex_unlock(&st->lock);
+	if (ret)
+		goto done;
+
+	sca3000_ring_int_process(val, indio_dev);
+
+	if (val & SCA3000_INT_STATUS_FREE_FALL)
+		iio_push_event(indio_dev,
+			       IIO_MOD_EVENT_CODE(IIO_ACCEL,
+						  0,
+						  IIO_MOD_X_AND_Y_AND_Z,
+						  IIO_EV_TYPE_MAG,
+						  IIO_EV_DIR_FALLING),
+			       last_timestamp);
+
+	if (val & SCA3000_INT_STATUS_Y_TRIGGER)
+		iio_push_event(indio_dev,
+			       IIO_MOD_EVENT_CODE(IIO_ACCEL,
+						  0,
+						  IIO_MOD_Y,
+						  IIO_EV_TYPE_MAG,
+						  IIO_EV_DIR_RISING),
+			       last_timestamp);
+
+	if (val & SCA3000_INT_STATUS_X_TRIGGER)
+		iio_push_event(indio_dev,
+			       IIO_MOD_EVENT_CODE(IIO_ACCEL,
+						  0,
+						  IIO_MOD_X,
+						  IIO_EV_TYPE_MAG,
+						  IIO_EV_DIR_RISING),
+			       last_timestamp);
+
+	if (val & SCA3000_INT_STATUS_Z_TRIGGER)
+		iio_push_event(indio_dev,
+			       IIO_MOD_EVENT_CODE(IIO_ACCEL,
+						  0,
+						  IIO_MOD_Z,
+						  IIO_EV_TYPE_MAG,
+						  IIO_EV_DIR_RISING),
+			       last_timestamp);
+
+done:
+	return IRQ_HANDLED;
+}
+
+/**
+ * sca3000_read_event_config() what events are enabled
+ **/
+static int sca3000_read_event_config(struct iio_dev *indio_dev,
+				     const struct iio_chan_spec *chan,
+				     enum iio_event_type type,
+				     enum iio_event_direction dir)
+{
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int ret;
+	/* read current value of mode register */
+	mutex_lock(&st->lock);
+
+	ret = sca3000_read_data_short(st, SCA3000_REG_MODE_ADDR, 1);
+	if (ret)
+		goto error_ret;
+
+	switch (chan->channel2) {
+	case IIO_MOD_X_AND_Y_AND_Z:
+		ret = !!(st->rx[0] & SCA3000_REG_MODE_FREE_FALL_DETECT);
+		break;
+	case IIO_MOD_X:
+	case IIO_MOD_Y:
+	case IIO_MOD_Z:
+		/*
+		 * Motion detection mode cannot run at the same time as
+		 * acceleration data being read.
+		 */
+		if ((st->rx[0] & SCA3000_REG_MODE_MODE_MASK)
+		    != SCA3000_REG_MODE_MEAS_MODE_MOT_DET) {
+			ret = 0;
+		} else {
+			ret = sca3000_read_ctrl_reg(st,
+						SCA3000_REG_CTRL_SEL_MD_CTRL);
+			if (ret < 0)
+				goto error_ret;
+			/* only supporting logical or's for now */
+			ret = !!(ret & sca3000_addresses[chan->address][2]);
+		}
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+error_ret:
+	mutex_unlock(&st->lock);
+
+	return ret;
+}
+
+static int sca3000_freefall_set_state(struct iio_dev *indio_dev, int state)
+{
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int ret;
+
+	/* read current value of mode register */
+	ret = sca3000_read_data_short(st, SCA3000_REG_MODE_ADDR, 1);
+	if (ret)
+		return ret;
+
+	/* if off and should be on */
+	if (state && !(st->rx[0] & SCA3000_REG_MODE_FREE_FALL_DETECT))
+		return sca3000_write_reg(st, SCA3000_REG_MODE_ADDR,
+					 st->rx[0] | SCA3000_REG_MODE_FREE_FALL_DETECT);
+	/* if on and should be off */
+	else if (!state && (st->rx[0] & SCA3000_REG_MODE_FREE_FALL_DETECT))
+		return sca3000_write_reg(st, SCA3000_REG_MODE_ADDR,
+					 st->rx[0] & ~SCA3000_REG_MODE_FREE_FALL_DETECT);
+	else
+		return 0;
+}
+
+static int sca3000_motion_detect_set_state(struct iio_dev *indio_dev, int axis,
+					   int state)
+{
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int ret, ctrlval;
+
+	/*
+	 * First read the motion detector config to find out if
+	 * this axis is on
+	 */
+	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_MD_CTRL);
+	if (ret < 0)
+		return ret;
+	ctrlval = ret;
+	/* if off and should be on */
+	if (state && !(ctrlval & sca3000_addresses[axis][2])) {
+		ret = sca3000_write_ctrl_reg(st,
+					     SCA3000_REG_CTRL_SEL_MD_CTRL,
+					     ctrlval |
+					     sca3000_addresses[axis][2]);
+		if (ret)
+			return ret;
+		st->mo_det_use_count++;
+	} else if (!state && (ctrlval & sca3000_addresses[axis][2])) {
+		ret = sca3000_write_ctrl_reg(st,
+					     SCA3000_REG_CTRL_SEL_MD_CTRL,
+					     ctrlval &
+					     ~(sca3000_addresses[axis][2]));
+		if (ret)
+			return ret;
+		st->mo_det_use_count--;
+	}
+
+	/* read current value of mode register */
+	ret = sca3000_read_data_short(st, SCA3000_REG_MODE_ADDR, 1);
+	if (ret)
+		return ret;
+	/* if off and should be on */
+	if ((st->mo_det_use_count) &&
+	    ((st->rx[0] & SCA3000_REG_MODE_MODE_MASK)
+	     != SCA3000_REG_MODE_MEAS_MODE_MOT_DET))
+		return sca3000_write_reg(st, SCA3000_REG_MODE_ADDR,
+			(st->rx[0] & ~SCA3000_REG_MODE_MODE_MASK)
+			| SCA3000_REG_MODE_MEAS_MODE_MOT_DET);
+	/* if on and should be off */
+	else if (!(st->mo_det_use_count) &&
+		 ((st->rx[0] & SCA3000_REG_MODE_MODE_MASK)
+		  == SCA3000_REG_MODE_MEAS_MODE_MOT_DET))
+		return sca3000_write_reg(st, SCA3000_REG_MODE_ADDR,
+			st->rx[0] & SCA3000_REG_MODE_MODE_MASK);
+	else
+		return 0;
+}
+
+/**
+ * sca3000_write_event_config() - simple on off control for motion detector
+ * @indio_dev: IIO device instance specific structure. Data specific to this
+ * particular driver may be accessed via iio_priv(indio_dev).
+ * @chan: Description of the channel whose event we are configuring.
+ * @type: The type of event.
+ * @dir: The direction of the event.
+ * @state: Desired state of event being configured.
+ *
+ * This is a per axis control, but enabling any will result in the
+ * motion detector unit being enabled.
+ * N.B. enabling motion detector stops normal data acquisition.
+ * There is a complexity in knowing which mode to return to when
+ * this mode is disabled.  Currently normal mode is assumed.
+ **/
+static int sca3000_write_event_config(struct iio_dev *indio_dev,
+				      const struct iio_chan_spec *chan,
+				      enum iio_event_type type,
+				      enum iio_event_direction dir,
+				      int state)
+{
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&st->lock);
+	switch (chan->channel2) {
+	case IIO_MOD_X_AND_Y_AND_Z:
+		ret = sca3000_freefall_set_state(indio_dev, state);
+		break;
+
+	case IIO_MOD_X:
+	case IIO_MOD_Y:
+	case IIO_MOD_Z:
+		ret = sca3000_motion_detect_set_state(indio_dev,
+						      chan->address,
+						      state);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	mutex_unlock(&st->lock);
+
+	return ret;
+}
+
+static int sca3000_configure_ring(struct iio_dev *indio_dev)
+{
+	struct iio_buffer *buffer;
+
+	buffer = iio_kfifo_allocate();
+	if (!buffer)
+		return -ENOMEM;
+
+	iio_device_attach_buffer(indio_dev, buffer);
+	indio_dev->modes |= INDIO_BUFFER_SOFTWARE;
+
+	return 0;
+}
+
+static void sca3000_unconfigure_ring(struct iio_dev *indio_dev)
+{
+	iio_kfifo_free(indio_dev->buffer);
+}
+
+static inline
+int __sca3000_hw_ring_state_set(struct iio_dev *indio_dev, bool state)
+{
+	struct sca3000_state *st = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&st->lock);
+	ret = sca3000_read_data_short(st, SCA3000_REG_MODE_ADDR, 1);
+	if (ret)
+		goto error_ret;
+	if (state) {
+		dev_info(&indio_dev->dev, "supposedly enabling ring buffer\n");
+		ret = sca3000_write_reg(st,
+			SCA3000_REG_MODE_ADDR,
+			(st->rx[0] | SCA3000_REG_MODE_RING_BUF_ENABLE));
+	} else
+		ret = sca3000_write_reg(st,
+			SCA3000_REG_MODE_ADDR,
+			(st->rx[0] & ~SCA3000_REG_MODE_RING_BUF_ENABLE));
+error_ret:
+	mutex_unlock(&st->lock);
+
+	return ret;
+}
+
+/**
+ * sca3000_hw_ring_preenable() - hw ring buffer preenable function
+ * @indio_dev: structure representing the IIO device. Device instance
+ * specific state can be accessed via iio_priv(indio_dev).
+ *
+ * Very simple enable function as the chip will allows normal reads
+ * during ring buffer operation so as long as it is indeed running
+ * before we notify the core, the precise ordering does not matter.
+ */
+static int sca3000_hw_ring_preenable(struct iio_dev *indio_dev)
+{
+	int ret;
+	struct sca3000_state *st = iio_priv(indio_dev);
+
+	mutex_lock(&st->lock);
+
+	/* Enable the 50% full interrupt */
+	ret = sca3000_read_data_short(st, SCA3000_REG_INT_MASK_ADDR, 1);
+	if (ret)
+		goto error_unlock;
+	ret = sca3000_write_reg(st,
+				SCA3000_REG_INT_MASK_ADDR,
+				st->rx[0] | SCA3000_REG_INT_MASK_RING_HALF);
+	if (ret)
+		goto error_unlock;
+
+	mutex_unlock(&st->lock);
+
+	return __sca3000_hw_ring_state_set(indio_dev, 1);
+
+error_unlock:
+	mutex_unlock(&st->lock);
+
+	return ret;
+}
+
+static int sca3000_hw_ring_postdisable(struct iio_dev *indio_dev)
+{
+	int ret;
+	struct sca3000_state *st = iio_priv(indio_dev);
+
+	ret = __sca3000_hw_ring_state_set(indio_dev, 0);
+	if (ret)
+		return ret;
+
+	/* Disable the 50% full interrupt */
+	mutex_lock(&st->lock);
+
+	ret = sca3000_read_data_short(st, SCA3000_REG_INT_MASK_ADDR, 1);
+	if (ret)
+		goto unlock;
+	ret = sca3000_write_reg(st,
+				SCA3000_REG_INT_MASK_ADDR,
+				st->rx[0] & ~SCA3000_REG_INT_MASK_RING_HALF);
+unlock:
+	mutex_unlock(&st->lock);
+	return ret;
+}
+
+static const struct iio_buffer_setup_ops sca3000_ring_setup_ops = {
+	.preenable = &sca3000_hw_ring_preenable,
+	.postdisable = &sca3000_hw_ring_postdisable,
+};
+
+/**
+ * sca3000_clean_setup() - get the device into a predictable state
+ * @st: Device instance specific private data structure
+ *
+ * Devices use flash memory to store many of the register values
+ * and hence can come up in somewhat unpredictable states.
+ * Hence reset everything on driver load.
+ */
+static int sca3000_clean_setup(struct sca3000_state *st)
+{
+	int ret;
+
+	mutex_lock(&st->lock);
+	/* Ensure all interrupts have been acknowledged */
+	ret = sca3000_read_data_short(st, SCA3000_REG_INT_STATUS_ADDR, 1);
+	if (ret)
+		goto error_ret;
+
+	/* Turn off all motion detection channels */
+	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_MD_CTRL);
+	if (ret < 0)
+		goto error_ret;
+	ret = sca3000_write_ctrl_reg(st, SCA3000_REG_CTRL_SEL_MD_CTRL,
+				     ret & SCA3000_MD_CTRL_PROT_MASK);
+	if (ret)
+		goto error_ret;
+
+	/* Disable ring buffer */
+	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL);
+	if (ret < 0)
+		goto error_ret;
+	ret = sca3000_write_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL,
+				     (ret & SCA3000_REG_OUT_CTRL_PROT_MASK)
+				     | SCA3000_REG_OUT_CTRL_BUF_X_EN
+				     | SCA3000_REG_OUT_CTRL_BUF_Y_EN
+				     | SCA3000_REG_OUT_CTRL_BUF_Z_EN
+				     | SCA3000_REG_OUT_CTRL_BUF_DIV_4);
+	if (ret)
+		goto error_ret;
+	/* Enable interrupts, relevant to mode and set up as active low */
+	ret = sca3000_read_data_short(st, SCA3000_REG_INT_MASK_ADDR, 1);
+	if (ret)
+		goto error_ret;
+	ret = sca3000_write_reg(st,
+				SCA3000_REG_INT_MASK_ADDR,
+				(ret & SCA3000_REG_INT_MASK_PROT_MASK)
+				| SCA3000_REG_INT_MASK_ACTIVE_LOW);
+	if (ret)
+		goto error_ret;
+	/*
+	 * Select normal measurement mode, free fall off, ring off
+	 * Ring in 12 bit mode - it is fine to overwrite reserved bits 3,5
+	 * as that occurs in one of the example on the datasheet
+	 */
+	ret = sca3000_read_data_short(st, SCA3000_REG_MODE_ADDR, 1);
+	if (ret)
+		goto error_ret;
+	ret = sca3000_write_reg(st, SCA3000_REG_MODE_ADDR,
+				(st->rx[0] & SCA3000_MODE_PROT_MASK));
+
+error_ret:
+	mutex_unlock(&st->lock);
+	return ret;
+}
+
+static const struct iio_info sca3000_info = {
+	.attrs = &sca3000_attribute_group,
+	.read_raw = &sca3000_read_raw,
+	.write_raw = &sca3000_write_raw,
+	.read_event_value = &sca3000_read_event_value,
+	.write_event_value = &sca3000_write_event_value,
+	.read_event_config = &sca3000_read_event_config,
+	.write_event_config = &sca3000_write_event_config,
+	.driver_module = THIS_MODULE,
+};
+
+static int sca3000_probe(struct spi_device *spi)
+{
+	int ret;
+	struct sca3000_state *st;
+	struct iio_dev *indio_dev;
+
+	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+	spi_set_drvdata(spi, indio_dev);
+	st->us = spi;
+	mutex_init(&st->lock);
+	st->info = &sca3000_spi_chip_info_tbl[spi_get_device_id(spi)
+					      ->driver_data];
+
+	indio_dev->dev.parent = &spi->dev;
+	indio_dev->name = spi_get_device_id(spi)->name;
+	indio_dev->info = &sca3000_info;
+	if (st->info->temp_output) {
+		indio_dev->channels = sca3000_channels_with_temp;
+		indio_dev->num_channels =
+			ARRAY_SIZE(sca3000_channels_with_temp);
+	} else {
+		indio_dev->channels = sca3000_channels;
+		indio_dev->num_channels = ARRAY_SIZE(sca3000_channels);
+	}
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	sca3000_configure_ring(indio_dev);
+
+	if (spi->irq) {
+		ret = request_threaded_irq(spi->irq,
+					   NULL,
+					   &sca3000_event_handler,
+					   IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+					   "sca3000",
+					   indio_dev);
+		if (ret)
+			return ret;
+	}
+	indio_dev->setup_ops = &sca3000_ring_setup_ops;
+	ret = sca3000_clean_setup(st);
+	if (ret)
+		goto error_free_irq;
+
+	ret = sca3000_print_rev(indio_dev);
+	if (ret)
+		goto error_free_irq;
+
+	return iio_device_register(indio_dev);
+
+error_free_irq:
+	if (spi->irq)
+		free_irq(spi->irq, indio_dev);
+
+	return ret;
+}
+
+static int sca3000_stop_all_interrupts(struct sca3000_state *st)
+{
+	int ret;
+
+	mutex_lock(&st->lock);
+	ret = sca3000_read_data_short(st, SCA3000_REG_INT_MASK_ADDR, 1);
+	if (ret)
+		goto error_ret;
+	ret = sca3000_write_reg(st, SCA3000_REG_INT_MASK_ADDR,
+				(st->rx[0] &
+				 ~(SCA3000_REG_INT_MASK_RING_THREE_QUARTER |
+				   SCA3000_REG_INT_MASK_RING_HALF |
+				   SCA3000_REG_INT_MASK_ALL_INTS)));
+error_ret:
+	mutex_unlock(&st->lock);
+	return ret;
+}
+
+static int sca3000_remove(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev = spi_get_drvdata(spi);
+	struct sca3000_state *st = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+
+	/* Must ensure no interrupts can be generated after this! */
+	sca3000_stop_all_interrupts(st);
+	if (spi->irq)
+		free_irq(spi->irq, indio_dev);
+
+	sca3000_unconfigure_ring(indio_dev);
+
+	return 0;
+}
+
+static const struct spi_device_id sca3000_id[] = {
+	{"sca3000_d01", d01},
+	{"sca3000_e02", e02},
+	{"sca3000_e04", e04},
+	{"sca3000_e05", e05},
+	{}
+};
+MODULE_DEVICE_TABLE(spi, sca3000_id);
+
+static struct spi_driver sca3000_driver = {
+	.driver = {
+		.name = "sca3000",
+	},
+	.probe = sca3000_probe,
+	.remove = sca3000_remove,
+	.id_table = sca3000_id,
+};
+module_spi_driver(sca3000_driver);
+
+MODULE_AUTHOR("Jonathan Cameron <jic23@kernel.org>");
+MODULE_DESCRIPTION("VTI SCA3000 Series Accelerometers SPI driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/accel/st_accel.h b/drivers/iio/accel/st_accel.h
index f8dfdb690563..7c231687109a 100644
--- a/drivers/iio/accel/st_accel.h
+++ b/drivers/iio/accel/st_accel.h
@@ -30,6 +30,7 @@
 #define LSM303AGR_ACCEL_DEV_NAME	"lsm303agr_accel"
 #define LIS2DH12_ACCEL_DEV_NAME		"lis2dh12_accel"
 #define LIS3L02DQ_ACCEL_DEV_NAME	"lis3l02dq"
+#define LNG2DM_ACCEL_DEV_NAME		"lng2dm"
 
 /**
 * struct st_sensors_platform_data - default accel platform data
diff --git a/drivers/iio/accel/st_accel_core.c b/drivers/iio/accel/st_accel_core.c
index ce69048c88e9..f6b6d42385e1 100644
--- a/drivers/iio/accel/st_accel_core.c
+++ b/drivers/iio/accel/st_accel_core.c
@@ -43,194 +43,6 @@
 #define ST_ACCEL_FS_AVL_200G			200
 #define ST_ACCEL_FS_AVL_400G			400
 
-/* CUSTOM VALUES FOR SENSOR 1 */
-#define ST_ACCEL_1_WAI_EXP			0x33
-#define ST_ACCEL_1_ODR_ADDR			0x20
-#define ST_ACCEL_1_ODR_MASK			0xf0
-#define ST_ACCEL_1_ODR_AVL_1HZ_VAL		0x01
-#define ST_ACCEL_1_ODR_AVL_10HZ_VAL		0x02
-#define ST_ACCEL_1_ODR_AVL_25HZ_VAL		0x03
-#define ST_ACCEL_1_ODR_AVL_50HZ_VAL		0x04
-#define ST_ACCEL_1_ODR_AVL_100HZ_VAL		0x05
-#define ST_ACCEL_1_ODR_AVL_200HZ_VAL		0x06
-#define ST_ACCEL_1_ODR_AVL_400HZ_VAL		0x07
-#define ST_ACCEL_1_ODR_AVL_1600HZ_VAL		0x08
-#define ST_ACCEL_1_FS_ADDR			0x23
-#define ST_ACCEL_1_FS_MASK			0x30
-#define ST_ACCEL_1_FS_AVL_2_VAL			0x00
-#define ST_ACCEL_1_FS_AVL_4_VAL			0x01
-#define ST_ACCEL_1_FS_AVL_8_VAL			0x02
-#define ST_ACCEL_1_FS_AVL_16_VAL		0x03
-#define ST_ACCEL_1_FS_AVL_2_GAIN		IIO_G_TO_M_S_2(1000)
-#define ST_ACCEL_1_FS_AVL_4_GAIN		IIO_G_TO_M_S_2(2000)
-#define ST_ACCEL_1_FS_AVL_8_GAIN		IIO_G_TO_M_S_2(4000)
-#define ST_ACCEL_1_FS_AVL_16_GAIN		IIO_G_TO_M_S_2(12000)
-#define ST_ACCEL_1_BDU_ADDR			0x23
-#define ST_ACCEL_1_BDU_MASK			0x80
-#define ST_ACCEL_1_DRDY_IRQ_ADDR		0x22
-#define ST_ACCEL_1_DRDY_IRQ_INT1_MASK		0x10
-#define ST_ACCEL_1_DRDY_IRQ_INT2_MASK		0x08
-#define ST_ACCEL_1_IHL_IRQ_ADDR			0x25
-#define ST_ACCEL_1_IHL_IRQ_MASK			0x02
-#define ST_ACCEL_1_MULTIREAD_BIT		true
-
-/* CUSTOM VALUES FOR SENSOR 2 */
-#define ST_ACCEL_2_WAI_EXP			0x32
-#define ST_ACCEL_2_ODR_ADDR			0x20
-#define ST_ACCEL_2_ODR_MASK			0x18
-#define ST_ACCEL_2_ODR_AVL_50HZ_VAL		0x00
-#define ST_ACCEL_2_ODR_AVL_100HZ_VAL		0x01
-#define ST_ACCEL_2_ODR_AVL_400HZ_VAL		0x02
-#define ST_ACCEL_2_ODR_AVL_1000HZ_VAL		0x03
-#define ST_ACCEL_2_PW_ADDR			0x20
-#define ST_ACCEL_2_PW_MASK			0xe0
-#define ST_ACCEL_2_FS_ADDR			0x23
-#define ST_ACCEL_2_FS_MASK			0x30
-#define ST_ACCEL_2_FS_AVL_2_VAL			0X00
-#define ST_ACCEL_2_FS_AVL_4_VAL			0X01
-#define ST_ACCEL_2_FS_AVL_8_VAL			0x03
-#define ST_ACCEL_2_FS_AVL_2_GAIN		IIO_G_TO_M_S_2(1000)
-#define ST_ACCEL_2_FS_AVL_4_GAIN		IIO_G_TO_M_S_2(2000)
-#define ST_ACCEL_2_FS_AVL_8_GAIN		IIO_G_TO_M_S_2(3900)
-#define ST_ACCEL_2_BDU_ADDR			0x23
-#define ST_ACCEL_2_BDU_MASK			0x80
-#define ST_ACCEL_2_DRDY_IRQ_ADDR		0x22
-#define ST_ACCEL_2_DRDY_IRQ_INT1_MASK		0x02
-#define ST_ACCEL_2_DRDY_IRQ_INT2_MASK		0x10
-#define ST_ACCEL_2_IHL_IRQ_ADDR			0x22
-#define ST_ACCEL_2_IHL_IRQ_MASK			0x80
-#define ST_ACCEL_2_OD_IRQ_ADDR			0x22
-#define ST_ACCEL_2_OD_IRQ_MASK			0x40
-#define ST_ACCEL_2_MULTIREAD_BIT		true
-
-/* CUSTOM VALUES FOR SENSOR 3 */
-#define ST_ACCEL_3_WAI_EXP			0x40
-#define ST_ACCEL_3_ODR_ADDR			0x20
-#define ST_ACCEL_3_ODR_MASK			0xf0
-#define ST_ACCEL_3_ODR_AVL_3HZ_VAL		0x01
-#define ST_ACCEL_3_ODR_AVL_6HZ_VAL		0x02
-#define ST_ACCEL_3_ODR_AVL_12HZ_VAL		0x03
-#define ST_ACCEL_3_ODR_AVL_25HZ_VAL		0x04
-#define ST_ACCEL_3_ODR_AVL_50HZ_VAL		0x05
-#define ST_ACCEL_3_ODR_AVL_100HZ_VAL		0x06
-#define ST_ACCEL_3_ODR_AVL_200HZ_VAL		0x07
-#define ST_ACCEL_3_ODR_AVL_400HZ_VAL		0x08
-#define ST_ACCEL_3_ODR_AVL_800HZ_VAL		0x09
-#define ST_ACCEL_3_ODR_AVL_1600HZ_VAL		0x0a
-#define ST_ACCEL_3_FS_ADDR			0x24
-#define ST_ACCEL_3_FS_MASK			0x38
-#define ST_ACCEL_3_FS_AVL_2_VAL			0X00
-#define ST_ACCEL_3_FS_AVL_4_VAL			0X01
-#define ST_ACCEL_3_FS_AVL_6_VAL			0x02
-#define ST_ACCEL_3_FS_AVL_8_VAL			0x03
-#define ST_ACCEL_3_FS_AVL_16_VAL		0x04
-#define ST_ACCEL_3_FS_AVL_2_GAIN		IIO_G_TO_M_S_2(61)
-#define ST_ACCEL_3_FS_AVL_4_GAIN		IIO_G_TO_M_S_2(122)
-#define ST_ACCEL_3_FS_AVL_6_GAIN		IIO_G_TO_M_S_2(183)
-#define ST_ACCEL_3_FS_AVL_8_GAIN		IIO_G_TO_M_S_2(244)
-#define ST_ACCEL_3_FS_AVL_16_GAIN		IIO_G_TO_M_S_2(732)
-#define ST_ACCEL_3_BDU_ADDR			0x20
-#define ST_ACCEL_3_BDU_MASK			0x08
-#define ST_ACCEL_3_DRDY_IRQ_ADDR		0x23
-#define ST_ACCEL_3_DRDY_IRQ_INT1_MASK		0x80
-#define ST_ACCEL_3_DRDY_IRQ_INT2_MASK		0x00
-#define ST_ACCEL_3_IHL_IRQ_ADDR			0x23
-#define ST_ACCEL_3_IHL_IRQ_MASK			0x40
-#define ST_ACCEL_3_IG1_EN_ADDR			0x23
-#define ST_ACCEL_3_IG1_EN_MASK			0x08
-#define ST_ACCEL_3_MULTIREAD_BIT		false
-
-/* CUSTOM VALUES FOR SENSOR 4 */
-#define ST_ACCEL_4_WAI_EXP			0x3a
-#define ST_ACCEL_4_ODR_ADDR			0x20
-#define ST_ACCEL_4_ODR_MASK			0x30 /* DF1 and DF0 */
-#define ST_ACCEL_4_ODR_AVL_40HZ_VAL		0x00
-#define ST_ACCEL_4_ODR_AVL_160HZ_VAL		0x01
-#define ST_ACCEL_4_ODR_AVL_640HZ_VAL		0x02
-#define ST_ACCEL_4_ODR_AVL_2560HZ_VAL		0x03
-#define ST_ACCEL_4_PW_ADDR			0x20
-#define ST_ACCEL_4_PW_MASK			0xc0
-#define ST_ACCEL_4_FS_ADDR			0x21
-#define ST_ACCEL_4_FS_MASK			0x80
-#define ST_ACCEL_4_FS_AVL_2_VAL			0X00
-#define ST_ACCEL_4_FS_AVL_6_VAL			0X01
-#define ST_ACCEL_4_FS_AVL_2_GAIN		IIO_G_TO_M_S_2(1024)
-#define ST_ACCEL_4_FS_AVL_6_GAIN		IIO_G_TO_M_S_2(340)
-#define ST_ACCEL_4_BDU_ADDR			0x21
-#define ST_ACCEL_4_BDU_MASK			0x40
-#define ST_ACCEL_4_DRDY_IRQ_ADDR		0x21
-#define ST_ACCEL_4_DRDY_IRQ_INT1_MASK		0x04
-#define ST_ACCEL_4_MULTIREAD_BIT		true
-
-/* CUSTOM VALUES FOR SENSOR 5 */
-#define ST_ACCEL_5_WAI_EXP			0x3b
-#define ST_ACCEL_5_ODR_ADDR			0x20
-#define ST_ACCEL_5_ODR_MASK			0x80
-#define ST_ACCEL_5_ODR_AVL_100HZ_VAL		0x00
-#define ST_ACCEL_5_ODR_AVL_400HZ_VAL		0x01
-#define ST_ACCEL_5_PW_ADDR			0x20
-#define ST_ACCEL_5_PW_MASK			0x40
-#define ST_ACCEL_5_FS_ADDR			0x20
-#define ST_ACCEL_5_FS_MASK			0x20
-#define ST_ACCEL_5_FS_AVL_2_VAL			0X00
-#define ST_ACCEL_5_FS_AVL_8_VAL			0X01
-/* TODO: check these resulting gain settings, these are not in the datsheet */
-#define ST_ACCEL_5_FS_AVL_2_GAIN		IIO_G_TO_M_S_2(18000)
-#define ST_ACCEL_5_FS_AVL_8_GAIN		IIO_G_TO_M_S_2(72000)
-#define ST_ACCEL_5_DRDY_IRQ_ADDR		0x22
-#define ST_ACCEL_5_DRDY_IRQ_INT1_MASK		0x04
-#define ST_ACCEL_5_DRDY_IRQ_INT2_MASK		0x20
-#define ST_ACCEL_5_IHL_IRQ_ADDR			0x22
-#define ST_ACCEL_5_IHL_IRQ_MASK			0x80
-#define ST_ACCEL_5_OD_IRQ_ADDR			0x22
-#define ST_ACCEL_5_OD_IRQ_MASK			0x40
-#define ST_ACCEL_5_IG1_EN_ADDR			0x21
-#define ST_ACCEL_5_IG1_EN_MASK			0x08
-#define ST_ACCEL_5_MULTIREAD_BIT		false
-
-/* CUSTOM VALUES FOR SENSOR 6 */
-#define ST_ACCEL_6_WAI_EXP			0x32
-#define ST_ACCEL_6_ODR_ADDR			0x20
-#define ST_ACCEL_6_ODR_MASK			0x18
-#define ST_ACCEL_6_ODR_AVL_50HZ_VAL		0x00
-#define ST_ACCEL_6_ODR_AVL_100HZ_VAL		0x01
-#define ST_ACCEL_6_ODR_AVL_400HZ_VAL		0x02
-#define ST_ACCEL_6_ODR_AVL_1000HZ_VAL		0x03
-#define ST_ACCEL_6_PW_ADDR			0x20
-#define ST_ACCEL_6_PW_MASK			0x20
-#define ST_ACCEL_6_FS_ADDR			0x23
-#define ST_ACCEL_6_FS_MASK			0x30
-#define ST_ACCEL_6_FS_AVL_100_VAL		0x00
-#define ST_ACCEL_6_FS_AVL_200_VAL		0x01
-#define ST_ACCEL_6_FS_AVL_400_VAL		0x03
-#define ST_ACCEL_6_FS_AVL_100_GAIN		IIO_G_TO_M_S_2(49000)
-#define ST_ACCEL_6_FS_AVL_200_GAIN		IIO_G_TO_M_S_2(98000)
-#define ST_ACCEL_6_FS_AVL_400_GAIN		IIO_G_TO_M_S_2(195000)
-#define ST_ACCEL_6_BDU_ADDR			0x23
-#define ST_ACCEL_6_BDU_MASK			0x80
-#define ST_ACCEL_6_DRDY_IRQ_ADDR		0x22
-#define ST_ACCEL_6_DRDY_IRQ_INT1_MASK		0x02
-#define ST_ACCEL_6_DRDY_IRQ_INT2_MASK		0x10
-#define ST_ACCEL_6_IHL_IRQ_ADDR			0x22
-#define ST_ACCEL_6_IHL_IRQ_MASK			0x80
-#define ST_ACCEL_6_MULTIREAD_BIT		true
-
-/* CUSTOM VALUES FOR SENSOR 7 */
-#define ST_ACCEL_7_ODR_ADDR			0x20
-#define ST_ACCEL_7_ODR_MASK			0x30
-#define ST_ACCEL_7_ODR_AVL_280HZ_VAL		0x00
-#define ST_ACCEL_7_ODR_AVL_560HZ_VAL		0x01
-#define ST_ACCEL_7_ODR_AVL_1120HZ_VAL		0x02
-#define ST_ACCEL_7_ODR_AVL_4480HZ_VAL		0x03
-#define ST_ACCEL_7_PW_ADDR			0x20
-#define ST_ACCEL_7_PW_MASK			0xc0
-#define ST_ACCEL_7_FS_AVL_2_GAIN		IIO_G_TO_M_S_2(488)
-#define ST_ACCEL_7_BDU_ADDR			0x21
-#define ST_ACCEL_7_BDU_MASK			0x40
-#define ST_ACCEL_7_DRDY_IRQ_ADDR		0x21
-#define ST_ACCEL_7_DRDY_IRQ_INT1_MASK		0x04
-#define ST_ACCEL_7_MULTIREAD_BIT		false
-
 static const struct iio_chan_spec st_accel_8bit_channels[] = {
 	ST_SENSORS_LSM_CHANNELS(IIO_ACCEL,
 			BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
@@ -281,7 +93,7 @@ static const struct iio_chan_spec st_accel_16bit_channels[] = {
 
 static const struct st_sensor_settings st_accel_sensors_settings[] = {
 	{
-		.wai = ST_ACCEL_1_WAI_EXP,
+		.wai = 0x33,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = LIS3DH_ACCEL_DEV_NAME,
@@ -294,22 +106,22 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 		},
 		.ch = (struct iio_chan_spec *)st_accel_12bit_channels,
 		.odr = {
-			.addr = ST_ACCEL_1_ODR_ADDR,
-			.mask = ST_ACCEL_1_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0xf0,
 			.odr_avl = {
-				{ 1, ST_ACCEL_1_ODR_AVL_1HZ_VAL, },
-				{ 10, ST_ACCEL_1_ODR_AVL_10HZ_VAL, },
-				{ 25, ST_ACCEL_1_ODR_AVL_25HZ_VAL, },
-				{ 50, ST_ACCEL_1_ODR_AVL_50HZ_VAL, },
-				{ 100, ST_ACCEL_1_ODR_AVL_100HZ_VAL, },
-				{ 200, ST_ACCEL_1_ODR_AVL_200HZ_VAL, },
-				{ 400, ST_ACCEL_1_ODR_AVL_400HZ_VAL, },
-				{ 1600, ST_ACCEL_1_ODR_AVL_1600HZ_VAL, },
+				{ .hz = 1, .value = 0x01, },
+				{ .hz = 10, .value = 0x02, },
+				{ .hz = 25, .value = 0x03, },
+				{ .hz = 50, .value = 0x04, },
+				{ .hz = 100, .value = 0x05, },
+				{ .hz = 200, .value = 0x06, },
+				{ .hz = 400, .value = 0x07, },
+				{ .hz = 1600, .value = 0x08, },
 			},
 		},
 		.pw = {
-			.addr = ST_ACCEL_1_ODR_ADDR,
-			.mask = ST_ACCEL_1_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0xf0,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
 		.enable_axis = {
@@ -317,48 +129,48 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 			.mask = ST_SENSORS_DEFAULT_AXIS_MASK,
 		},
 		.fs = {
-			.addr = ST_ACCEL_1_FS_ADDR,
-			.mask = ST_ACCEL_1_FS_MASK,
+			.addr = 0x23,
+			.mask = 0x30,
 			.fs_avl = {
 				[0] = {
 					.num = ST_ACCEL_FS_AVL_2G,
-					.value = ST_ACCEL_1_FS_AVL_2_VAL,
-					.gain = ST_ACCEL_1_FS_AVL_2_GAIN,
+					.value = 0x00,
+					.gain = IIO_G_TO_M_S_2(1000),
 				},
 				[1] = {
 					.num = ST_ACCEL_FS_AVL_4G,
-					.value = ST_ACCEL_1_FS_AVL_4_VAL,
-					.gain = ST_ACCEL_1_FS_AVL_4_GAIN,
+					.value = 0x01,
+					.gain = IIO_G_TO_M_S_2(2000),
 				},
 				[2] = {
 					.num = ST_ACCEL_FS_AVL_8G,
-					.value = ST_ACCEL_1_FS_AVL_8_VAL,
-					.gain = ST_ACCEL_1_FS_AVL_8_GAIN,
+					.value = 0x02,
+					.gain = IIO_G_TO_M_S_2(4000),
 				},
 				[3] = {
 					.num = ST_ACCEL_FS_AVL_16G,
-					.value = ST_ACCEL_1_FS_AVL_16_VAL,
-					.gain = ST_ACCEL_1_FS_AVL_16_GAIN,
+					.value = 0x03,
+					.gain = IIO_G_TO_M_S_2(12000),
 				},
 			},
 		},
 		.bdu = {
-			.addr = ST_ACCEL_1_BDU_ADDR,
-			.mask = ST_ACCEL_1_BDU_MASK,
+			.addr = 0x23,
+			.mask = 0x80,
 		},
 		.drdy_irq = {
-			.addr = ST_ACCEL_1_DRDY_IRQ_ADDR,
-			.mask_int1 = ST_ACCEL_1_DRDY_IRQ_INT1_MASK,
-			.mask_int2 = ST_ACCEL_1_DRDY_IRQ_INT2_MASK,
-			.addr_ihl = ST_ACCEL_1_IHL_IRQ_ADDR,
-			.mask_ihl = ST_ACCEL_1_IHL_IRQ_MASK,
+			.addr = 0x22,
+			.mask_int1 = 0x10,
+			.mask_int2 = 0x08,
+			.addr_ihl = 0x25,
+			.mask_ihl = 0x02,
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 		},
-		.multi_read_bit = ST_ACCEL_1_MULTIREAD_BIT,
+		.multi_read_bit = true,
 		.bootime = 2,
 	},
 	{
-		.wai = ST_ACCEL_2_WAI_EXP,
+		.wai = 0x32,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = LIS331DLH_ACCEL_DEV_NAME,
@@ -368,18 +180,18 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 		},
 		.ch = (struct iio_chan_spec *)st_accel_12bit_channels,
 		.odr = {
-			.addr = ST_ACCEL_2_ODR_ADDR,
-			.mask = ST_ACCEL_2_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0x18,
 			.odr_avl = {
-				{ 50, ST_ACCEL_2_ODR_AVL_50HZ_VAL, },
-				{ 100, ST_ACCEL_2_ODR_AVL_100HZ_VAL, },
-				{ 400, ST_ACCEL_2_ODR_AVL_400HZ_VAL, },
-				{ 1000, ST_ACCEL_2_ODR_AVL_1000HZ_VAL, },
+				{ .hz = 50, .value = 0x00, },
+				{ .hz = 100, .value = 0x01, },
+				{ .hz = 400, .value = 0x02, },
+				{ .hz = 1000, .value = 0x03, },
 			},
 		},
 		.pw = {
-			.addr = ST_ACCEL_2_PW_ADDR,
-			.mask = ST_ACCEL_2_PW_MASK,
+			.addr = 0x20,
+			.mask = 0xe0,
 			.value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
@@ -388,69 +200,69 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 			.mask = ST_SENSORS_DEFAULT_AXIS_MASK,
 		},
 		.fs = {
-			.addr = ST_ACCEL_2_FS_ADDR,
-			.mask = ST_ACCEL_2_FS_MASK,
+			.addr = 0x23,
+			.mask = 0x30,
 			.fs_avl = {
 				[0] = {
 					.num = ST_ACCEL_FS_AVL_2G,
-					.value = ST_ACCEL_2_FS_AVL_2_VAL,
-					.gain = ST_ACCEL_2_FS_AVL_2_GAIN,
+					.value = 0x00,
+					.gain = IIO_G_TO_M_S_2(1000),
 				},
 				[1] = {
 					.num = ST_ACCEL_FS_AVL_4G,
-					.value = ST_ACCEL_2_FS_AVL_4_VAL,
-					.gain = ST_ACCEL_2_FS_AVL_4_GAIN,
+					.value = 0x01,
+					.gain = IIO_G_TO_M_S_2(2000),
 				},
 				[2] = {
 					.num = ST_ACCEL_FS_AVL_8G,
-					.value = ST_ACCEL_2_FS_AVL_8_VAL,
-					.gain = ST_ACCEL_2_FS_AVL_8_GAIN,
+					.value = 0x03,
+					.gain = IIO_G_TO_M_S_2(3900),
 				},
 			},
 		},
 		.bdu = {
-			.addr = ST_ACCEL_2_BDU_ADDR,
-			.mask = ST_ACCEL_2_BDU_MASK,
+			.addr = 0x23,
+			.mask = 0x80,
 		},
 		.drdy_irq = {
-			.addr = ST_ACCEL_2_DRDY_IRQ_ADDR,
-			.mask_int1 = ST_ACCEL_2_DRDY_IRQ_INT1_MASK,
-			.mask_int2 = ST_ACCEL_2_DRDY_IRQ_INT2_MASK,
-			.addr_ihl = ST_ACCEL_2_IHL_IRQ_ADDR,
-			.mask_ihl = ST_ACCEL_2_IHL_IRQ_MASK,
-			.addr_od = ST_ACCEL_2_OD_IRQ_ADDR,
-			.mask_od = ST_ACCEL_2_OD_IRQ_MASK,
+			.addr = 0x22,
+			.mask_int1 = 0x02,
+			.mask_int2 = 0x10,
+			.addr_ihl = 0x22,
+			.mask_ihl = 0x80,
+			.addr_od = 0x22,
+			.mask_od = 0x40,
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 		},
-		.multi_read_bit = ST_ACCEL_2_MULTIREAD_BIT,
+		.multi_read_bit = true,
 		.bootime = 2,
 	},
 	{
-		.wai = ST_ACCEL_3_WAI_EXP,
+		.wai = 0x40,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = LSM330_ACCEL_DEV_NAME,
 		},
 		.ch = (struct iio_chan_spec *)st_accel_16bit_channels,
 		.odr = {
-			.addr = ST_ACCEL_3_ODR_ADDR,
-			.mask = ST_ACCEL_3_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0xf0,
 			.odr_avl = {
-				{ 3, ST_ACCEL_3_ODR_AVL_3HZ_VAL },
-				{ 6, ST_ACCEL_3_ODR_AVL_6HZ_VAL, },
-				{ 12, ST_ACCEL_3_ODR_AVL_12HZ_VAL, },
-				{ 25, ST_ACCEL_3_ODR_AVL_25HZ_VAL, },
-				{ 50, ST_ACCEL_3_ODR_AVL_50HZ_VAL, },
-				{ 100, ST_ACCEL_3_ODR_AVL_100HZ_VAL, },
-				{ 200, ST_ACCEL_3_ODR_AVL_200HZ_VAL, },
-				{ 400, ST_ACCEL_3_ODR_AVL_400HZ_VAL, },
-				{ 800, ST_ACCEL_3_ODR_AVL_800HZ_VAL, },
-				{ 1600, ST_ACCEL_3_ODR_AVL_1600HZ_VAL, },
+				{ .hz = 3, .value = 0x01, },
+				{ .hz = 6, .value = 0x02, },
+				{ .hz = 12, .value = 0x03, },
+				{ .hz = 25, .value = 0x04, },
+				{ .hz = 50, .value = 0x05, },
+				{ .hz = 100, .value = 0x06, },
+				{ .hz = 200, .value = 0x07, },
+				{ .hz = 400, .value = 0x08, },
+				{ .hz = 800, .value = 0x09, },
+				{ .hz = 1600, .value = 0x0a, },
 			},
 		},
 		.pw = {
-			.addr = ST_ACCEL_3_ODR_ADDR,
-			.mask = ST_ACCEL_3_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0xf0,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
 		.enable_axis = {
@@ -458,75 +270,75 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 			.mask = ST_SENSORS_DEFAULT_AXIS_MASK,
 		},
 		.fs = {
-			.addr = ST_ACCEL_3_FS_ADDR,
-			.mask = ST_ACCEL_3_FS_MASK,
+			.addr = 0x24,
+			.mask = 0x38,
 			.fs_avl = {
 				[0] = {
 					.num = ST_ACCEL_FS_AVL_2G,
-					.value = ST_ACCEL_3_FS_AVL_2_VAL,
-					.gain = ST_ACCEL_3_FS_AVL_2_GAIN,
+					.value = 0x00,
+					.gain = IIO_G_TO_M_S_2(61),
 				},
 				[1] = {
 					.num = ST_ACCEL_FS_AVL_4G,
-					.value = ST_ACCEL_3_FS_AVL_4_VAL,
-					.gain = ST_ACCEL_3_FS_AVL_4_GAIN,
+					.value = 0x01,
+					.gain = IIO_G_TO_M_S_2(122),
 				},
 				[2] = {
 					.num = ST_ACCEL_FS_AVL_6G,
-					.value = ST_ACCEL_3_FS_AVL_6_VAL,
-					.gain = ST_ACCEL_3_FS_AVL_6_GAIN,
+					.value = 0x02,
+					.gain = IIO_G_TO_M_S_2(183),
 				},
 				[3] = {
 					.num = ST_ACCEL_FS_AVL_8G,
-					.value = ST_ACCEL_3_FS_AVL_8_VAL,
-					.gain = ST_ACCEL_3_FS_AVL_8_GAIN,
+					.value = 0x03,
+					.gain = IIO_G_TO_M_S_2(244),
 				},
 				[4] = {
 					.num = ST_ACCEL_FS_AVL_16G,
-					.value = ST_ACCEL_3_FS_AVL_16_VAL,
-					.gain = ST_ACCEL_3_FS_AVL_16_GAIN,
+					.value = 0x04,
+					.gain = IIO_G_TO_M_S_2(732),
 				},
 			},
 		},
 		.bdu = {
-			.addr = ST_ACCEL_3_BDU_ADDR,
-			.mask = ST_ACCEL_3_BDU_MASK,
+			.addr = 0x20,
+			.mask = 0x08,
 		},
 		.drdy_irq = {
-			.addr = ST_ACCEL_3_DRDY_IRQ_ADDR,
-			.mask_int1 = ST_ACCEL_3_DRDY_IRQ_INT1_MASK,
-			.mask_int2 = ST_ACCEL_3_DRDY_IRQ_INT2_MASK,
-			.addr_ihl = ST_ACCEL_3_IHL_IRQ_ADDR,
-			.mask_ihl = ST_ACCEL_3_IHL_IRQ_MASK,
+			.addr = 0x23,
+			.mask_int1 = 0x80,
+			.mask_int2 = 0x00,
+			.addr_ihl = 0x23,
+			.mask_ihl = 0x40,
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 			.ig1 = {
-				.en_addr = ST_ACCEL_3_IG1_EN_ADDR,
-				.en_mask = ST_ACCEL_3_IG1_EN_MASK,
+				.en_addr = 0x23,
+				.en_mask = 0x08,
 			},
 		},
-		.multi_read_bit = ST_ACCEL_3_MULTIREAD_BIT,
+		.multi_read_bit = false,
 		.bootime = 2,
 	},
 	{
-		.wai = ST_ACCEL_4_WAI_EXP,
+		.wai = 0x3a,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = LIS3LV02DL_ACCEL_DEV_NAME,
 		},
 		.ch = (struct iio_chan_spec *)st_accel_12bit_channels,
 		.odr = {
-			.addr = ST_ACCEL_4_ODR_ADDR,
-			.mask = ST_ACCEL_4_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0x30, /* DF1 and DF0 */
 			.odr_avl = {
-				{ 40, ST_ACCEL_4_ODR_AVL_40HZ_VAL },
-				{ 160, ST_ACCEL_4_ODR_AVL_160HZ_VAL, },
-				{ 640, ST_ACCEL_4_ODR_AVL_640HZ_VAL, },
-				{ 2560, ST_ACCEL_4_ODR_AVL_2560HZ_VAL, },
+				{ .hz = 40, .value = 0x00, },
+				{ .hz = 160, .value = 0x01, },
+				{ .hz = 640, .value = 0x02, },
+				{ .hz = 2560, .value = 0x03, },
 			},
 		},
 		.pw = {
-			.addr = ST_ACCEL_4_PW_ADDR,
-			.mask = ST_ACCEL_4_PW_MASK,
+			.addr = 0x20,
+			.mask = 0xc0,
 			.value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
@@ -535,51 +347,51 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 			.mask = ST_SENSORS_DEFAULT_AXIS_MASK,
 		},
 		.fs = {
-			.addr = ST_ACCEL_4_FS_ADDR,
-			.mask = ST_ACCEL_4_FS_MASK,
+			.addr = 0x21,
+			.mask = 0x80,
 			.fs_avl = {
 				[0] = {
 					.num = ST_ACCEL_FS_AVL_2G,
-					.value = ST_ACCEL_4_FS_AVL_2_VAL,
-					.gain = ST_ACCEL_4_FS_AVL_2_GAIN,
+					.value = 0x00,
+					.gain = IIO_G_TO_M_S_2(1024),
 				},
 				[1] = {
 					.num = ST_ACCEL_FS_AVL_6G,
-					.value = ST_ACCEL_4_FS_AVL_6_VAL,
-					.gain = ST_ACCEL_4_FS_AVL_6_GAIN,
+					.value = 0x01,
+					.gain = IIO_G_TO_M_S_2(340),
 				},
 			},
 		},
 		.bdu = {
-			.addr = ST_ACCEL_4_BDU_ADDR,
-			.mask = ST_ACCEL_4_BDU_MASK,
+			.addr = 0x21,
+			.mask = 0x40,
 		},
 		.drdy_irq = {
-			.addr = ST_ACCEL_4_DRDY_IRQ_ADDR,
-			.mask_int1 = ST_ACCEL_4_DRDY_IRQ_INT1_MASK,
+			.addr = 0x21,
+			.mask_int1 = 0x04,
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 		},
-		.multi_read_bit = ST_ACCEL_4_MULTIREAD_BIT,
+		.multi_read_bit = true,
 		.bootime = 2, /* guess */
 	},
 	{
-		.wai = ST_ACCEL_5_WAI_EXP,
+		.wai = 0x3b,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = LIS331DL_ACCEL_DEV_NAME,
 		},
 		.ch = (struct iio_chan_spec *)st_accel_8bit_channels,
 		.odr = {
-			.addr = ST_ACCEL_5_ODR_ADDR,
-			.mask = ST_ACCEL_5_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0x80,
 			.odr_avl = {
-				{ 100, ST_ACCEL_5_ODR_AVL_100HZ_VAL },
-				{ 400, ST_ACCEL_5_ODR_AVL_400HZ_VAL, },
+				{ .hz = 100, .value = 0x00, },
+				{ .hz = 400, .value = 0x01, },
 			},
 		},
 		.pw = {
-			.addr = ST_ACCEL_5_PW_ADDR,
-			.mask = ST_ACCEL_5_PW_MASK,
+			.addr = 0x20,
+			.mask = 0x40,
 			.value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
@@ -588,54 +400,58 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 			.mask = ST_SENSORS_DEFAULT_AXIS_MASK,
 		},
 		.fs = {
-			.addr = ST_ACCEL_5_FS_ADDR,
-			.mask = ST_ACCEL_5_FS_MASK,
+			.addr = 0x20,
+			.mask = 0x20,
+			/*
+			 * TODO: check these resulting gain settings, these are
+			 * not in the datsheet
+			 */
 			.fs_avl = {
 				[0] = {
 					.num = ST_ACCEL_FS_AVL_2G,
-					.value = ST_ACCEL_5_FS_AVL_2_VAL,
-					.gain = ST_ACCEL_5_FS_AVL_2_GAIN,
+					.value = 0x00,
+					.gain = IIO_G_TO_M_S_2(18000),
 				},
 				[1] = {
 					.num = ST_ACCEL_FS_AVL_8G,
-					.value = ST_ACCEL_5_FS_AVL_8_VAL,
-					.gain = ST_ACCEL_5_FS_AVL_8_GAIN,
+					.value = 0x01,
+					.gain = IIO_G_TO_M_S_2(72000),
 				},
 			},
 		},
 		.drdy_irq = {
-			.addr = ST_ACCEL_5_DRDY_IRQ_ADDR,
-			.mask_int1 = ST_ACCEL_5_DRDY_IRQ_INT1_MASK,
-			.mask_int2 = ST_ACCEL_5_DRDY_IRQ_INT2_MASK,
-			.addr_ihl = ST_ACCEL_5_IHL_IRQ_ADDR,
-			.mask_ihl = ST_ACCEL_5_IHL_IRQ_MASK,
-			.addr_od = ST_ACCEL_5_OD_IRQ_ADDR,
-			.mask_od = ST_ACCEL_5_OD_IRQ_MASK,
+			.addr = 0x22,
+			.mask_int1 = 0x04,
+			.mask_int2 = 0x20,
+			.addr_ihl = 0x22,
+			.mask_ihl = 0x80,
+			.addr_od = 0x22,
+			.mask_od = 0x40,
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 		},
-		.multi_read_bit = ST_ACCEL_5_MULTIREAD_BIT,
+		.multi_read_bit = false,
 		.bootime = 2, /* guess */
 	},
 	{
-		.wai = ST_ACCEL_6_WAI_EXP,
+		.wai = 0x32,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = H3LIS331DL_DRIVER_NAME,
 		},
 		.ch = (struct iio_chan_spec *)st_accel_12bit_channels,
 		.odr = {
-			.addr = ST_ACCEL_6_ODR_ADDR,
-			.mask = ST_ACCEL_6_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0x18,
 			.odr_avl = {
-				{ 50, ST_ACCEL_6_ODR_AVL_50HZ_VAL },
-				{ 100, ST_ACCEL_6_ODR_AVL_100HZ_VAL, },
-				{ 400, ST_ACCEL_6_ODR_AVL_400HZ_VAL, },
-				{ 1000, ST_ACCEL_6_ODR_AVL_1000HZ_VAL, },
+				{ .hz = 50, .value = 0x00, },
+				{ .hz = 100, .value = 0x01, },
+				{ .hz = 400, .value = 0x02, },
+				{ .hz = 1000, .value = 0x03, },
 			},
 		},
 		.pw = {
-			.addr = ST_ACCEL_6_PW_ADDR,
-			.mask = ST_ACCEL_6_PW_MASK,
+			.addr = 0x20,
+			.mask = 0x20,
 			.value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
@@ -644,38 +460,38 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 			.mask = ST_SENSORS_DEFAULT_AXIS_MASK,
 		},
 		.fs = {
-			.addr = ST_ACCEL_6_FS_ADDR,
-			.mask = ST_ACCEL_6_FS_MASK,
+			.addr = 0x23,
+			.mask = 0x30,
 			.fs_avl = {
 				[0] = {
 					.num = ST_ACCEL_FS_AVL_100G,
-					.value = ST_ACCEL_6_FS_AVL_100_VAL,
-					.gain = ST_ACCEL_6_FS_AVL_100_GAIN,
+					.value = 0x00,
+					.gain = IIO_G_TO_M_S_2(49000),
 				},
 				[1] = {
 					.num = ST_ACCEL_FS_AVL_200G,
-					.value = ST_ACCEL_6_FS_AVL_200_VAL,
-					.gain = ST_ACCEL_6_FS_AVL_200_GAIN,
+					.value = 0x01,
+					.gain = IIO_G_TO_M_S_2(98000),
 				},
 				[2] = {
 					.num = ST_ACCEL_FS_AVL_400G,
-					.value = ST_ACCEL_6_FS_AVL_400_VAL,
-					.gain = ST_ACCEL_6_FS_AVL_400_GAIN,
+					.value = 0x03,
+					.gain = IIO_G_TO_M_S_2(195000),
 				},
 			},
 		},
 		.bdu = {
-			.addr = ST_ACCEL_6_BDU_ADDR,
-			.mask = ST_ACCEL_6_BDU_MASK,
+			.addr = 0x23,
+			.mask = 0x80,
 		},
 		.drdy_irq = {
-			.addr = ST_ACCEL_6_DRDY_IRQ_ADDR,
-			.mask_int1 = ST_ACCEL_6_DRDY_IRQ_INT1_MASK,
-			.mask_int2 = ST_ACCEL_6_DRDY_IRQ_INT2_MASK,
-			.addr_ihl = ST_ACCEL_6_IHL_IRQ_ADDR,
-			.mask_ihl = ST_ACCEL_6_IHL_IRQ_MASK,
+			.addr = 0x22,
+			.mask_int1 = 0x02,
+			.mask_int2 = 0x10,
+			.addr_ihl = 0x22,
+			.mask_ihl = 0x80,
 		},
-		.multi_read_bit = ST_ACCEL_6_MULTIREAD_BIT,
+		.multi_read_bit = true,
 		.bootime = 2,
 	},
 	{
@@ -685,18 +501,18 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 		},
 		.ch = (struct iio_chan_spec *)st_accel_12bit_channels,
 		.odr = {
-			.addr = ST_ACCEL_7_ODR_ADDR,
-			.mask = ST_ACCEL_7_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0x30,
 			.odr_avl = {
-				{ 280, ST_ACCEL_7_ODR_AVL_280HZ_VAL, },
-				{ 560, ST_ACCEL_7_ODR_AVL_560HZ_VAL, },
-				{ 1120, ST_ACCEL_7_ODR_AVL_1120HZ_VAL, },
-				{ 4480, ST_ACCEL_7_ODR_AVL_4480HZ_VAL, },
+				{ .hz = 280, .value = 0x00, },
+				{ .hz = 560, .value = 0x01, },
+				{ .hz = 1120, .value = 0x02, },
+				{ .hz = 4480, .value = 0x03, },
 			},
 		},
 		.pw = {
-			.addr = ST_ACCEL_7_PW_ADDR,
-			.mask = ST_ACCEL_7_PW_MASK,
+			.addr = 0x20,
+			.mask = 0xc0,
 			.value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
@@ -708,7 +524,7 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 			.fs_avl = {
 				[0] = {
 					.num = ST_ACCEL_FS_AVL_2G,
-					.gain = ST_ACCEL_7_FS_AVL_2_GAIN,
+					.gain = IIO_G_TO_M_S_2(488),
 				},
 			},
 		},
@@ -719,11 +535,78 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 		.bdu = {
 		},
 		.drdy_irq = {
-			.addr = ST_ACCEL_7_DRDY_IRQ_ADDR,
-			.mask_int1 = ST_ACCEL_7_DRDY_IRQ_INT1_MASK,
+			.addr = 0x21,
+			.mask_int1 = 0x04,
+			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
+		},
+		.multi_read_bit = false,
+		.bootime = 2,
+	},
+	{
+		.wai = 0x33,
+		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
+		.sensors_supported = {
+			[0] = LNG2DM_ACCEL_DEV_NAME,
+		},
+		.ch = (struct iio_chan_spec *)st_accel_8bit_channels,
+		.odr = {
+			.addr = 0x20,
+			.mask = 0xf0,
+			.odr_avl = {
+				{ .hz = 1, .value = 0x01, },
+				{ .hz = 10, .value = 0x02, },
+				{ .hz = 25, .value = 0x03, },
+				{ .hz = 50, .value = 0x04, },
+				{ .hz = 100, .value = 0x05, },
+				{ .hz = 200, .value = 0x06, },
+				{ .hz = 400, .value = 0x07, },
+				{ .hz = 1600, .value = 0x08, },
+			},
+		},
+		.pw = {
+			.addr = 0x20,
+			.mask = 0xf0,
+			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
+		},
+		.enable_axis = {
+			.addr = ST_SENSORS_DEFAULT_AXIS_ADDR,
+			.mask = ST_SENSORS_DEFAULT_AXIS_MASK,
+		},
+		.fs = {
+			.addr = 0x23,
+			.mask = 0x30,
+			.fs_avl = {
+				[0] = {
+					.num = ST_ACCEL_FS_AVL_2G,
+					.value = 0x00,
+					.gain = IIO_G_TO_M_S_2(15600),
+				},
+				[1] = {
+					.num = ST_ACCEL_FS_AVL_4G,
+					.value = 0x01,
+					.gain = IIO_G_TO_M_S_2(31200),
+				},
+				[2] = {
+					.num = ST_ACCEL_FS_AVL_8G,
+					.value = 0x02,
+					.gain = IIO_G_TO_M_S_2(62500),
+				},
+				[3] = {
+					.num = ST_ACCEL_FS_AVL_16G,
+					.value = 0x03,
+					.gain = IIO_G_TO_M_S_2(187500),
+				},
+			},
+		},
+		.drdy_irq = {
+			.addr = 0x22,
+			.mask_int1 = 0x10,
+			.mask_int2 = 0x08,
+			.addr_ihl = 0x25,
+			.mask_ihl = 0x02,
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 		},
-		.multi_read_bit = ST_ACCEL_7_MULTIREAD_BIT,
+		.multi_read_bit = true,
 		.bootime = 2,
 	},
 };
diff --git a/drivers/iio/accel/st_accel_i2c.c b/drivers/iio/accel/st_accel_i2c.c
index e9d427a5df7c..c0f8867aa1ea 100644
--- a/drivers/iio/accel/st_accel_i2c.c
+++ b/drivers/iio/accel/st_accel_i2c.c
@@ -84,6 +84,10 @@ static const struct of_device_id st_accel_of_match[] = {
 		.compatible = "st,lis3l02dq",
 		.data = LIS3L02DQ_ACCEL_DEV_NAME,
 	},
+	{
+		.compatible = "st,lng2dm-accel",
+		.data = LNG2DM_ACCEL_DEV_NAME,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, st_accel_of_match);
@@ -135,6 +139,7 @@ static const struct i2c_device_id st_accel_id_table[] = {
 	{ LSM303AGR_ACCEL_DEV_NAME },
 	{ LIS2DH12_ACCEL_DEV_NAME },
 	{ LIS3L02DQ_ACCEL_DEV_NAME },
+	{ LNG2DM_ACCEL_DEV_NAME },
 	{},
 };
 MODULE_DEVICE_TABLE(i2c, st_accel_id_table);
diff --git a/drivers/iio/accel/st_accel_spi.c b/drivers/iio/accel/st_accel_spi.c
index efd43941d45d..c25ac50d4600 100644
--- a/drivers/iio/accel/st_accel_spi.c
+++ b/drivers/iio/accel/st_accel_spi.c
@@ -60,6 +60,7 @@ static const struct spi_device_id st_accel_id_table[] = {
 	{ LSM303AGR_ACCEL_DEV_NAME },
 	{ LIS2DH12_ACCEL_DEV_NAME },
 	{ LIS3L02DQ_ACCEL_DEV_NAME },
+	{ LNG2DM_ACCEL_DEV_NAME },
 	{},
 };
 MODULE_DEVICE_TABLE(spi, st_accel_id_table);
diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
index 99c051490eff..38bc319904c4 100644
--- a/drivers/iio/adc/Kconfig
+++ b/drivers/iio/adc/Kconfig
@@ -58,6 +58,18 @@ config AD7476
 	  To compile this driver as a module, choose M here: the
 	  module will be called ad7476.
 
+config AD7766
+	tristate "Analog Devices AD7766/AD7767 ADC driver"
+	depends on SPI_MASTER
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	help
+	  Say yes here to build support for Analog Devices AD7766, AD7766-1,
+	  AD7766-2, AD7767, AD7767-1, AD7767-2 SPI analog to digital converters.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called ad7766.
+
 config AD7791
 	tristate "Analog Devices AD7791 ADC driver"
 	depends on SPI
@@ -195,6 +207,16 @@ config DA9150_GPADC
 	  To compile this driver as a module, choose M here: the module will be
 	  called berlin2-adc.
 
+config ENVELOPE_DETECTOR
+	tristate "Envelope detector using a DAC and a comparator"
+	depends on OF
+	help
+	  Say yes here to build support for an envelope detector using a DAC
+	  and a comparator.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called envelope-detector.
+
 config EXYNOS_ADC
 	tristate "Exynos ADC driver support"
 	depends on ARCH_EXYNOS || ARCH_S3C24XX || ARCH_S3C64XX || (OF && COMPILE_TEST)
@@ -419,6 +441,28 @@ config ROCKCHIP_SARADC
 	  To compile this driver as a module, choose M here: the
 	  module will be called rockchip_saradc.
 
+config STM32_ADC_CORE
+	tristate "STMicroelectronics STM32 adc core"
+	depends on ARCH_STM32 || COMPILE_TEST
+	depends on OF
+	depends on REGULATOR
+	help
+	  Select this option to enable the core driver for STMicroelectronics
+	  STM32 analog-to-digital converter (ADC).
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called stm32-adc-core.
+
+config STM32_ADC
+	tristate "STMicroelectronics STM32 adc"
+	depends on STM32_ADC_CORE
+	help
+	  Say yes here to build support for STMicroelectronics stm32 Analog
+	  to Digital Converter (ADC).
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called stm32-adc.
+
 config STX104
 	tristate "Apex Embedded Systems STX104 driver"
 	depends on X86 && ISA_BUS_API
@@ -449,6 +493,8 @@ config TI_ADC081C
 config TI_ADC0832
 	tristate "Texas Instruments ADC0831/ADC0832/ADC0834/ADC0838"
 	depends on SPI
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
 	help
 	  If you say yes here you get support for Texas Instruments ADC0831,
 	  ADC0832, ADC0834, ADC0838 ADC chips.
diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile
index 7a40c04c311f..d36c4be8d1fc 100644
--- a/drivers/iio/adc/Makefile
+++ b/drivers/iio/adc/Makefile
@@ -9,6 +9,7 @@ obj-$(CONFIG_AD7291) += ad7291.o
 obj-$(CONFIG_AD7298) += ad7298.o
 obj-$(CONFIG_AD7923) += ad7923.o
 obj-$(CONFIG_AD7476) += ad7476.o
+obj-$(CONFIG_AD7766) += ad7766.o
 obj-$(CONFIG_AD7791) += ad7791.o
 obj-$(CONFIG_AD7793) += ad7793.o
 obj-$(CONFIG_AD7887) += ad7887.o
@@ -20,6 +21,7 @@ obj-$(CONFIG_BCM_IPROC_ADC) += bcm_iproc_adc.o
 obj-$(CONFIG_BERLIN2_ADC) += berlin2-adc.o
 obj-$(CONFIG_CC10001_ADC) += cc10001_adc.o
 obj-$(CONFIG_DA9150_GPADC) += da9150-gpadc.o
+obj-$(CONFIG_ENVELOPE_DETECTOR) += envelope-detector.o
 obj-$(CONFIG_EXYNOS_ADC) += exynos_adc.o
 obj-$(CONFIG_FSL_MX25_ADC) += fsl-imx25-gcq.o
 obj-$(CONFIG_HI8435) += hi8435.o
@@ -41,6 +43,8 @@ obj-$(CONFIG_QCOM_SPMI_IADC) += qcom-spmi-iadc.o
 obj-$(CONFIG_QCOM_SPMI_VADC) += qcom-spmi-vadc.o
 obj-$(CONFIG_ROCKCHIP_SARADC) += rockchip_saradc.o
 obj-$(CONFIG_STX104) += stx104.o
+obj-$(CONFIG_STM32_ADC_CORE) += stm32-adc-core.o
+obj-$(CONFIG_STM32_ADC) += stm32-adc.o
 obj-$(CONFIG_TI_ADC081C) += ti-adc081c.o
 obj-$(CONFIG_TI_ADC0832) += ti-adc0832.o
 obj-$(CONFIG_TI_ADC12138) += ti-adc12138.o
diff --git a/drivers/iio/adc/ad7766.c b/drivers/iio/adc/ad7766.c
new file mode 100644
index 000000000000..75cca42b6e70
--- /dev/null
+++ b/drivers/iio/adc/ad7766.c
@@ -0,0 +1,330 @@
+/*
+ * AD7766/AD7767 SPI ADC driver
+ *
+ * Copyright 2016 Analog Devices Inc.
+ *
+ * Licensed under the GPL-2 or later.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+#include <linux/spi/spi.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+
+struct ad7766_chip_info {
+	unsigned int decimation_factor;
+};
+
+enum {
+	AD7766_SUPPLY_AVDD = 0,
+	AD7766_SUPPLY_DVDD = 1,
+	AD7766_SUPPLY_VREF = 2,
+	AD7766_NUM_SUPPLIES = 3
+};
+
+struct ad7766 {
+	const struct ad7766_chip_info *chip_info;
+	struct spi_device *spi;
+	struct clk *mclk;
+	struct gpio_desc *pd_gpio;
+	struct regulator_bulk_data reg[AD7766_NUM_SUPPLIES];
+
+	struct iio_trigger *trig;
+
+	struct spi_transfer xfer;
+	struct spi_message msg;
+
+	/*
+	 * DMA (thus cache coherency maintenance) requires the
+	 * transfer buffers to live in their own cache lines.
+	 * Make the buffer large enough for one 24 bit sample and one 64 bit
+	 * aligned 64 bit timestamp.
+	 */
+	unsigned char data[ALIGN(3, sizeof(s64)) + sizeof(s64)]
+			____cacheline_aligned;
+};
+
+/*
+ * AD7766 and AD7767 variations are interface compatible, the main difference is
+ * analog performance. Both parts will use the same ID.
+ */
+enum ad7766_device_ids {
+	ID_AD7766,
+	ID_AD7766_1,
+	ID_AD7766_2,
+};
+
+static irqreturn_t ad7766_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct ad7766 *ad7766 = iio_priv(indio_dev);
+	int ret;
+
+	ret = spi_sync(ad7766->spi, &ad7766->msg);
+	if (ret < 0)
+		goto done;
+
+	iio_push_to_buffers_with_timestamp(indio_dev, ad7766->data,
+		pf->timestamp);
+done:
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int ad7766_preenable(struct iio_dev *indio_dev)
+{
+	struct ad7766 *ad7766 = iio_priv(indio_dev);
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(ad7766->reg), ad7766->reg);
+	if (ret < 0) {
+		dev_err(&ad7766->spi->dev, "Failed to enable supplies: %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(ad7766->mclk);
+	if (ret < 0) {
+		dev_err(&ad7766->spi->dev, "Failed to enable MCLK: %d\n", ret);
+		regulator_bulk_disable(ARRAY_SIZE(ad7766->reg), ad7766->reg);
+		return ret;
+	}
+
+	if (ad7766->pd_gpio)
+		gpiod_set_value(ad7766->pd_gpio, 0);
+
+	return 0;
+}
+
+static int ad7766_postdisable(struct iio_dev *indio_dev)
+{
+	struct ad7766 *ad7766 = iio_priv(indio_dev);
+
+	if (ad7766->pd_gpio)
+		gpiod_set_value(ad7766->pd_gpio, 1);
+
+	/*
+	 * The PD pin is synchronous to the clock, so give it some time to
+	 * notice the change before we disable the clock.
+	 */
+	msleep(20);
+
+	clk_disable_unprepare(ad7766->mclk);
+	regulator_bulk_disable(ARRAY_SIZE(ad7766->reg), ad7766->reg);
+
+	return 0;
+}
+
+static int ad7766_read_raw(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan, int *val, int *val2, long info)
+{
+	struct ad7766 *ad7766 = iio_priv(indio_dev);
+	struct regulator *vref = ad7766->reg[AD7766_SUPPLY_VREF].consumer;
+	int scale_uv;
+
+	switch (info) {
+	case IIO_CHAN_INFO_SCALE:
+		scale_uv = regulator_get_voltage(vref);
+		if (scale_uv < 0)
+			return scale_uv;
+		*val = scale_uv / 1000;
+		*val2 = chan->scan_type.realbits;
+		return IIO_VAL_FRACTIONAL_LOG2;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*val = clk_get_rate(ad7766->mclk) /
+			ad7766->chip_info->decimation_factor;
+		return IIO_VAL_INT;
+	}
+	return -EINVAL;
+}
+
+static const struct iio_chan_spec ad7766_channels[] = {
+	{
+		.type = IIO_VOLTAGE,
+		.indexed = 1,
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.scan_type = {
+			.sign = 's',
+			.realbits = 24,
+			.storagebits = 32,
+			.endianness = IIO_BE,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(1),
+};
+
+static const struct ad7766_chip_info ad7766_chip_info[] = {
+	[ID_AD7766] = {
+		.decimation_factor = 8,
+	},
+	[ID_AD7766_1] = {
+		.decimation_factor = 16,
+	},
+	[ID_AD7766_2] = {
+		.decimation_factor = 32,
+	},
+};
+
+static const struct iio_buffer_setup_ops ad7766_buffer_setup_ops = {
+	.preenable = &ad7766_preenable,
+	.postenable = &iio_triggered_buffer_postenable,
+	.predisable = &iio_triggered_buffer_predisable,
+	.postdisable = &ad7766_postdisable,
+};
+
+static const struct iio_info ad7766_info = {
+	.driver_module = THIS_MODULE,
+	.read_raw = &ad7766_read_raw,
+};
+
+static irqreturn_t ad7766_irq(int irq, void *private)
+{
+	iio_trigger_poll(private);
+	return IRQ_HANDLED;
+}
+
+static int ad7766_set_trigger_state(struct iio_trigger *trig, bool enable)
+{
+	struct ad7766 *ad7766 = iio_trigger_get_drvdata(trig);
+
+	if (enable)
+		enable_irq(ad7766->spi->irq);
+	else
+		disable_irq(ad7766->spi->irq);
+
+	return 0;
+}
+
+static const struct iio_trigger_ops ad7766_trigger_ops = {
+	.owner = THIS_MODULE,
+	.set_trigger_state = ad7766_set_trigger_state,
+	.validate_device = iio_trigger_validate_own_device,
+};
+
+static int ad7766_probe(struct spi_device *spi)
+{
+	const struct spi_device_id *id = spi_get_device_id(spi);
+	struct iio_dev *indio_dev;
+	struct ad7766 *ad7766;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*ad7766));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	ad7766 = iio_priv(indio_dev);
+	ad7766->chip_info = &ad7766_chip_info[id->driver_data];
+
+	ad7766->mclk = devm_clk_get(&spi->dev, "mclk");
+	if (IS_ERR(ad7766->mclk))
+		return PTR_ERR(ad7766->mclk);
+
+	ad7766->reg[AD7766_SUPPLY_AVDD].supply = "avdd";
+	ad7766->reg[AD7766_SUPPLY_DVDD].supply = "dvdd";
+	ad7766->reg[AD7766_SUPPLY_VREF].supply = "vref";
+
+	ret = devm_regulator_bulk_get(&spi->dev, ARRAY_SIZE(ad7766->reg),
+		ad7766->reg);
+	if (ret)
+		return ret;
+
+	ad7766->pd_gpio = devm_gpiod_get_optional(&spi->dev, "powerdown",
+		GPIOD_OUT_HIGH);
+	if (IS_ERR(ad7766->pd_gpio))
+		return PTR_ERR(ad7766->pd_gpio);
+
+	indio_dev->dev.parent = &spi->dev;
+	indio_dev->name = spi_get_device_id(spi)->name;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = ad7766_channels;
+	indio_dev->num_channels = ARRAY_SIZE(ad7766_channels);
+	indio_dev->info = &ad7766_info;
+
+	if (spi->irq > 0) {
+		ad7766->trig = devm_iio_trigger_alloc(&spi->dev, "%s-dev%d",
+			indio_dev->name, indio_dev->id);
+		if (!ad7766->trig)
+			return -ENOMEM;
+
+		ad7766->trig->ops = &ad7766_trigger_ops;
+		ad7766->trig->dev.parent = &spi->dev;
+		iio_trigger_set_drvdata(ad7766->trig, ad7766);
+
+		ret = devm_request_irq(&spi->dev, spi->irq, ad7766_irq,
+			IRQF_TRIGGER_FALLING, dev_name(&spi->dev),
+			ad7766->trig);
+		if (ret < 0)
+			return ret;
+
+		/*
+		 * The device generates interrupts as long as it is powered up.
+		 * Some platforms might not allow the option to power it down so
+		 * disable the interrupt to avoid extra load on the system
+		 */
+		disable_irq(spi->irq);
+
+		ret = devm_iio_trigger_register(&spi->dev, ad7766->trig);
+		if (ret)
+			return ret;
+	}
+
+	spi_set_drvdata(spi, indio_dev);
+
+	ad7766->spi = spi;
+
+	/* First byte always 0 */
+	ad7766->xfer.rx_buf = &ad7766->data[1];
+	ad7766->xfer.len = 3;
+
+	spi_message_init(&ad7766->msg);
+	spi_message_add_tail(&ad7766->xfer, &ad7766->msg);
+
+	ret = devm_iio_triggered_buffer_setup(&spi->dev, indio_dev,
+		&iio_pollfunc_store_time, &ad7766_trigger_handler,
+		&ad7766_buffer_setup_ops);
+	if (ret)
+		return ret;
+
+	ret = devm_iio_device_register(&spi->dev, indio_dev);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+static const struct spi_device_id ad7766_id[] = {
+	{"ad7766", ID_AD7766},
+	{"ad7766-1", ID_AD7766_1},
+	{"ad7766-2", ID_AD7766_2},
+	{"ad7767", ID_AD7766},
+	{"ad7767-1", ID_AD7766_1},
+	{"ad7767-2", ID_AD7766_2},
+	{}
+};
+MODULE_DEVICE_TABLE(spi, ad7766_id);
+
+static struct spi_driver ad7766_driver = {
+	.driver = {
+		.name	= "ad7766",
+	},
+	.probe		= ad7766_probe,
+	.id_table	= ad7766_id,
+};
+module_spi_driver(ad7766_driver);
+
+MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
+MODULE_DESCRIPTION("Analog Devices AD7766 and AD7767 ADCs driver support");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/at91_adc.c b/drivers/iio/adc/at91_adc.c
index bbdac07f4aaa..34b928cefeed 100644
--- a/drivers/iio/adc/at91_adc.c
+++ b/drivers/iio/adc/at91_adc.c
@@ -30,6 +30,7 @@
 #include <linux/iio/trigger.h>
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/triggered_buffer.h>
+#include <linux/pinctrl/consumer.h>
 
 /* Registers */
 #define AT91_ADC_CR		0x00		/* Control Register */
@@ -1347,6 +1348,32 @@ static int at91_adc_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_PM_SLEEP
+static int at91_adc_suspend(struct device *dev)
+{
+	struct iio_dev *idev = platform_get_drvdata(to_platform_device(dev));
+	struct at91_adc_state *st = iio_priv(idev);
+
+	pinctrl_pm_select_sleep_state(dev);
+	clk_disable_unprepare(st->clk);
+
+	return 0;
+}
+
+static int at91_adc_resume(struct device *dev)
+{
+	struct iio_dev *idev = platform_get_drvdata(to_platform_device(dev));
+	struct at91_adc_state *st = iio_priv(idev);
+
+	clk_prepare_enable(st->clk);
+	pinctrl_pm_select_default_state(dev);
+
+	return 0;
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(at91_adc_pm_ops, at91_adc_suspend, at91_adc_resume);
+
 static struct at91_adc_caps at91sam9260_caps = {
 	.calc_startup_ticks = calc_startup_ticks_9260,
 	.num_channels = 4,
@@ -1441,6 +1468,7 @@ static struct platform_driver at91_adc_driver = {
 	.driver = {
 		   .name = DRIVER_NAME,
 		   .of_match_table = of_match_ptr(at91_adc_dt_ids),
+		   .pm = &at91_adc_pm_ops,
 	},
 };
 
diff --git a/drivers/iio/adc/envelope-detector.c b/drivers/iio/adc/envelope-detector.c
new file mode 100644
index 000000000000..fef15c0d7c9c
--- /dev/null
+++ b/drivers/iio/adc/envelope-detector.c
@@ -0,0 +1,422 @@
+/*
+ * Driver for an envelope detector using a DAC and a comparator
+ *
+ * Copyright (C) 2016 Axentia Technologies AB
+ *
+ * Author: Peter Rosin <peda@axentia.se>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/*
+ * The DAC is used to find the peak level of an alternating voltage input
+ * signal by a binary search using the output of a comparator wired to
+ * an interrupt pin. Like so:
+ *                           _
+ *                          | \
+ *     input +------>-------|+ \
+ *                          |   \
+ *            .-------.     |    }---.
+ *            |       |     |   /    |
+ *            |    dac|-->--|- /     |
+ *            |       |     |_/      |
+ *            |       |              |
+ *            |       |              |
+ *            |    irq|------<-------'
+ *            |       |
+ *            '-------'
+ */
+
+#include <linux/completion.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/iio/consumer.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+
+struct envelope {
+	spinlock_t comp_lock; /* protects comp */
+	int comp;
+
+	struct mutex read_lock; /* protects everything else */
+
+	int comp_irq;
+	u32 comp_irq_trigger;
+	u32 comp_irq_trigger_inv;
+
+	struct iio_channel *dac;
+	struct delayed_work comp_timeout;
+
+	unsigned int comp_interval;
+	bool invert;
+	u32 dac_max;
+
+	int high;
+	int level;
+	int low;
+
+	struct completion done;
+};
+
+/*
+ * The envelope_detector_comp_latch function works together with the compare
+ * interrupt service routine below (envelope_detector_comp_isr) as a latch
+ * (one-bit memory) for if the interrupt has triggered since last calling
+ * this function.
+ * The ..._comp_isr function disables the interrupt so that the cpu does not
+ * need to service a possible interrupt flood from the comparator when no-one
+ * cares anyway, and this ..._comp_latch function reenables them again if
+ * needed.
+ */
+static int envelope_detector_comp_latch(struct envelope *env)
+{
+	int comp;
+
+	spin_lock_irq(&env->comp_lock);
+	comp = env->comp;
+	env->comp = 0;
+	spin_unlock_irq(&env->comp_lock);
+
+	if (!comp)
+		return 0;
+
+	/*
+	 * The irq was disabled, and is reenabled just now.
+	 * But there might have been a pending irq that
+	 * happened while the irq was disabled that fires
+	 * just as the irq is reenabled. That is not what
+	 * is desired.
+	 */
+	enable_irq(env->comp_irq);
+
+	/* So, synchronize this possibly pending irq... */
+	synchronize_irq(env->comp_irq);
+
+	/* ...and redo the whole dance. */
+	spin_lock_irq(&env->comp_lock);
+	comp = env->comp;
+	env->comp = 0;
+	spin_unlock_irq(&env->comp_lock);
+
+	if (comp)
+		enable_irq(env->comp_irq);
+
+	return 1;
+}
+
+static irqreturn_t envelope_detector_comp_isr(int irq, void *ctx)
+{
+	struct envelope *env = ctx;
+
+	spin_lock(&env->comp_lock);
+	env->comp = 1;
+	disable_irq_nosync(env->comp_irq);
+	spin_unlock(&env->comp_lock);
+
+	return IRQ_HANDLED;
+}
+
+static void envelope_detector_setup_compare(struct envelope *env)
+{
+	int ret;
+
+	/*
+	 * Do a binary search for the peak input level, and stop
+	 * when that level is "trapped" between two adjacent DAC
+	 * values.
+	 * When invert is active, use the midpoint floor so that
+	 * env->level ends up as env->low when the termination
+	 * criteria below is fulfilled, and use the midpoint
+	 * ceiling when invert is not active so that env->level
+	 * ends up as env->high in that case.
+	 */
+	env->level = (env->high + env->low + !env->invert) / 2;
+
+	if (env->high == env->low + 1) {
+		complete(&env->done);
+		return;
+	}
+
+	/* Set a "safe" DAC level (if there is such a thing)... */
+	ret = iio_write_channel_raw(env->dac, env->invert ? 0 : env->dac_max);
+	if (ret < 0)
+		goto err;
+
+	/* ...clear the comparison result... */
+	envelope_detector_comp_latch(env);
+
+	/* ...set the real DAC level... */
+	ret = iio_write_channel_raw(env->dac, env->level);
+	if (ret < 0)
+		goto err;
+
+	/* ...and wait for a bit to see if the latch catches anything. */
+	schedule_delayed_work(&env->comp_timeout,
+			      msecs_to_jiffies(env->comp_interval));
+	return;
+
+err:
+	env->level = ret;
+	complete(&env->done);
+}
+
+static void envelope_detector_timeout(struct work_struct *work)
+{
+	struct envelope *env = container_of(work, struct envelope,
+					    comp_timeout.work);
+
+	/* Adjust low/high depending on the latch content... */
+	if (!envelope_detector_comp_latch(env) ^ !env->invert)
+		env->low = env->level;
+	else
+		env->high = env->level;
+
+	/* ...and continue the search. */
+	envelope_detector_setup_compare(env);
+}
+
+static int envelope_detector_read_raw(struct iio_dev *indio_dev,
+				      struct iio_chan_spec const *chan,
+				      int *val, int *val2, long mask)
+{
+	struct envelope *env = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		/*
+		 * When invert is active, start with high=max+1 and low=0
+		 * since we will end up with the low value when the
+		 * termination criteria is fulfilled (rounding down). And
+		 * start with high=max and low=-1 when invert is not active
+		 * since we will end up with the high value in that case.
+		 * This ensures that the returned value in both cases are
+		 * in the same range as the DAC and is a value that has not
+		 * triggered the comparator.
+		 */
+		mutex_lock(&env->read_lock);
+		env->high = env->dac_max + env->invert;
+		env->low = -1 + env->invert;
+		envelope_detector_setup_compare(env);
+		wait_for_completion(&env->done);
+		if (env->level < 0) {
+			ret = env->level;
+			goto err_unlock;
+		}
+		*val = env->invert ? env->dac_max - env->level : env->level;
+		mutex_unlock(&env->read_lock);
+
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_SCALE:
+		return iio_read_channel_scale(env->dac, val, val2);
+	}
+
+	return -EINVAL;
+
+err_unlock:
+	mutex_unlock(&env->read_lock);
+	return ret;
+}
+
+static ssize_t envelope_show_invert(struct iio_dev *indio_dev,
+				    uintptr_t private,
+				    struct iio_chan_spec const *ch, char *buf)
+{
+	struct envelope *env = iio_priv(indio_dev);
+
+	return sprintf(buf, "%u\n", env->invert);
+}
+
+static ssize_t envelope_store_invert(struct iio_dev *indio_dev,
+				     uintptr_t private,
+				     struct iio_chan_spec const *ch,
+				     const char *buf, size_t len)
+{
+	struct envelope *env = iio_priv(indio_dev);
+	unsigned long invert;
+	int ret;
+	u32 trigger;
+
+	ret = kstrtoul(buf, 0, &invert);
+	if (ret < 0)
+		return ret;
+	if (invert > 1)
+		return -EINVAL;
+
+	trigger = invert ? env->comp_irq_trigger_inv : env->comp_irq_trigger;
+
+	mutex_lock(&env->read_lock);
+	if (invert != env->invert)
+		ret = irq_set_irq_type(env->comp_irq, trigger);
+	if (!ret) {
+		env->invert = invert;
+		ret = len;
+	}
+	mutex_unlock(&env->read_lock);
+
+	return ret;
+}
+
+static ssize_t envelope_show_comp_interval(struct iio_dev *indio_dev,
+					   uintptr_t private,
+					   struct iio_chan_spec const *ch,
+					   char *buf)
+{
+	struct envelope *env = iio_priv(indio_dev);
+
+	return sprintf(buf, "%u\n", env->comp_interval);
+}
+
+static ssize_t envelope_store_comp_interval(struct iio_dev *indio_dev,
+					    uintptr_t private,
+					    struct iio_chan_spec const *ch,
+					    const char *buf, size_t len)
+{
+	struct envelope *env = iio_priv(indio_dev);
+	unsigned long interval;
+	int ret;
+
+	ret = kstrtoul(buf, 0, &interval);
+	if (ret < 0)
+		return ret;
+	if (interval > 1000)
+		return -EINVAL;
+
+	mutex_lock(&env->read_lock);
+	env->comp_interval = interval;
+	mutex_unlock(&env->read_lock);
+
+	return len;
+}
+
+static const struct iio_chan_spec_ext_info envelope_detector_ext_info[] = {
+	{ .name = "invert",
+	  .read = envelope_show_invert,
+	  .write = envelope_store_invert, },
+	{ .name = "compare_interval",
+	  .read = envelope_show_comp_interval,
+	  .write = envelope_store_comp_interval, },
+	{ /* sentinel */ }
+};
+
+static const struct iio_chan_spec envelope_detector_iio_channel = {
+	.type = IIO_ALTVOLTAGE,
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW)
+			    | BIT(IIO_CHAN_INFO_SCALE),
+	.ext_info = envelope_detector_ext_info,
+	.indexed = 1,
+};
+
+static const struct iio_info envelope_detector_info = {
+	.read_raw = &envelope_detector_read_raw,
+	.driver_module = THIS_MODULE,
+};
+
+static int envelope_detector_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct iio_dev *indio_dev;
+	struct envelope *env;
+	enum iio_chan_type type;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*env));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, indio_dev);
+	env = iio_priv(indio_dev);
+	env->comp_interval = 50; /* some sensible default? */
+
+	spin_lock_init(&env->comp_lock);
+	mutex_init(&env->read_lock);
+	init_completion(&env->done);
+	INIT_DELAYED_WORK(&env->comp_timeout, envelope_detector_timeout);
+
+	indio_dev->name = dev_name(dev);
+	indio_dev->dev.parent = dev;
+	indio_dev->dev.of_node = dev->of_node;
+	indio_dev->info = &envelope_detector_info;
+	indio_dev->channels = &envelope_detector_iio_channel;
+	indio_dev->num_channels = 1;
+
+	env->dac = devm_iio_channel_get(dev, "dac");
+	if (IS_ERR(env->dac)) {
+		if (PTR_ERR(env->dac) != -EPROBE_DEFER)
+			dev_err(dev, "failed to get dac input channel\n");
+		return PTR_ERR(env->dac);
+	}
+
+	env->comp_irq = platform_get_irq_byname(pdev, "comp");
+	if (env->comp_irq < 0) {
+		if (env->comp_irq != -EPROBE_DEFER)
+			dev_err(dev, "failed to get compare interrupt\n");
+		return env->comp_irq;
+	}
+
+	ret = devm_request_irq(dev, env->comp_irq, envelope_detector_comp_isr,
+			       0, "envelope-detector", env);
+	if (ret) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "failed to request interrupt\n");
+		return ret;
+	}
+	env->comp_irq_trigger = irq_get_trigger_type(env->comp_irq);
+	if (env->comp_irq_trigger & IRQF_TRIGGER_RISING)
+		env->comp_irq_trigger_inv |= IRQF_TRIGGER_FALLING;
+	if (env->comp_irq_trigger & IRQF_TRIGGER_FALLING)
+		env->comp_irq_trigger_inv |= IRQF_TRIGGER_RISING;
+	if (env->comp_irq_trigger & IRQF_TRIGGER_HIGH)
+		env->comp_irq_trigger_inv |= IRQF_TRIGGER_LOW;
+	if (env->comp_irq_trigger & IRQF_TRIGGER_LOW)
+		env->comp_irq_trigger_inv |= IRQF_TRIGGER_HIGH;
+
+	ret = iio_get_channel_type(env->dac, &type);
+	if (ret < 0)
+		return ret;
+
+	if (type != IIO_VOLTAGE) {
+		dev_err(dev, "dac is of the wrong type\n");
+		return -EINVAL;
+	}
+
+	ret = iio_read_max_channel_raw(env->dac, &env->dac_max);
+	if (ret < 0) {
+		dev_err(dev, "dac does not indicate its raw maximum value\n");
+		return ret;
+	}
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct of_device_id envelope_detector_match[] = {
+	{ .compatible = "axentia,tse850-envelope-detector", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, envelope_detector_match);
+
+static struct platform_driver envelope_detector_driver = {
+	.probe = envelope_detector_probe,
+	.driver = {
+		.name = "iio-envelope-detector",
+		.of_match_table = envelope_detector_match,
+	},
+};
+module_platform_driver(envelope_detector_driver);
+
+MODULE_DESCRIPTION("Envelope detector using a DAC and a comparator");
+MODULE_AUTHOR("Peter Rosin <peda@axentia.se>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/max1027.c b/drivers/iio/adc/max1027.c
index 712fbd2b1f16..3b7c4f78f37a 100644
--- a/drivers/iio/adc/max1027.c
+++ b/drivers/iio/adc/max1027.c
@@ -238,7 +238,9 @@ static int max1027_read_single_value(struct iio_dev *indio_dev,
 
 	/* Configure conversion register with the requested chan */
 	st->reg = MAX1027_CONV_REG | MAX1027_CHAN(chan->channel) |
-		  MAX1027_NOSCAN | !!(chan->type == IIO_TEMP);
+		  MAX1027_NOSCAN;
+	if (chan->type == IIO_TEMP)
+		st->reg |= MAX1027_TEMP;
 	ret = spi_write(st->spi, &st->reg, 1);
 	if (ret < 0) {
 		dev_err(&indio_dev->dev,
@@ -360,17 +362,6 @@ static int max1027_set_trigger_state(struct iio_trigger *trig, bool state)
 	return 0;
 }
 
-static int max1027_validate_device(struct iio_trigger *trig,
-				   struct iio_dev *indio_dev)
-{
-	struct iio_dev *indio = iio_trigger_get_drvdata(trig);
-
-	if (indio != indio_dev)
-		return -EINVAL;
-
-	return 0;
-}
-
 static irqreturn_t max1027_trigger_handler(int irq, void *private)
 {
 	struct iio_poll_func *pf = (struct iio_poll_func *)private;
@@ -391,7 +382,7 @@ static irqreturn_t max1027_trigger_handler(int irq, void *private)
 
 static const struct iio_trigger_ops max1027_trigger_ops = {
 	.owner = THIS_MODULE,
-	.validate_device = &max1027_validate_device,
+	.validate_device = &iio_trigger_validate_own_device,
 	.set_trigger_state = &max1027_set_trigger_state,
 };
 
diff --git a/drivers/iio/adc/stm32-adc-core.c b/drivers/iio/adc/stm32-adc-core.c
new file mode 100644
index 000000000000..4214b0cd6b1b
--- /dev/null
+++ b/drivers/iio/adc/stm32-adc-core.c
@@ -0,0 +1,303 @@
+/*
+ * This file is part of STM32 ADC driver
+ *
+ * Copyright (C) 2016, STMicroelectronics - All Rights Reserved
+ * Author: Fabrice Gasnier <fabrice.gasnier@st.com>.
+ *
+ * Inspired from: fsl-imx25-tsadc
+ *
+ * License type: GPLv2
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/clk.h>
+#include <linux/interrupt.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdesc.h>
+#include <linux/irqdomain.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+
+#include "stm32-adc-core.h"
+
+/* STM32F4 - common registers for all ADC instances: 1, 2 & 3 */
+#define STM32F4_ADC_CSR			(STM32_ADCX_COMN_OFFSET + 0x00)
+#define STM32F4_ADC_CCR			(STM32_ADCX_COMN_OFFSET + 0x04)
+
+/* STM32F4_ADC_CSR - bit fields */
+#define STM32F4_EOC3			BIT(17)
+#define STM32F4_EOC2			BIT(9)
+#define STM32F4_EOC1			BIT(1)
+
+/* STM32F4_ADC_CCR - bit fields */
+#define STM32F4_ADC_ADCPRE_SHIFT	16
+#define STM32F4_ADC_ADCPRE_MASK		GENMASK(17, 16)
+
+/* STM32 F4 maximum analog clock rate (from datasheet) */
+#define STM32F4_ADC_MAX_CLK_RATE	36000000
+
+/**
+ * struct stm32_adc_priv - stm32 ADC core private data
+ * @irq:		irq for ADC block
+ * @domain:		irq domain reference
+ * @aclk:		clock reference for the analog circuitry
+ * @vref:		regulator reference
+ * @common:		common data for all ADC instances
+ */
+struct stm32_adc_priv {
+	int				irq;
+	struct irq_domain		*domain;
+	struct clk			*aclk;
+	struct regulator		*vref;
+	struct stm32_adc_common		common;
+};
+
+static struct stm32_adc_priv *to_stm32_adc_priv(struct stm32_adc_common *com)
+{
+	return container_of(com, struct stm32_adc_priv, common);
+}
+
+/* STM32F4 ADC internal common clock prescaler division ratios */
+static int stm32f4_pclk_div[] = {2, 4, 6, 8};
+
+/**
+ * stm32f4_adc_clk_sel() - Select stm32f4 ADC common clock prescaler
+ * @priv: stm32 ADC core private data
+ * Select clock prescaler used for analog conversions, before using ADC.
+ */
+static int stm32f4_adc_clk_sel(struct platform_device *pdev,
+			       struct stm32_adc_priv *priv)
+{
+	unsigned long rate;
+	u32 val;
+	int i;
+
+	rate = clk_get_rate(priv->aclk);
+	for (i = 0; i < ARRAY_SIZE(stm32f4_pclk_div); i++) {
+		if ((rate / stm32f4_pclk_div[i]) <= STM32F4_ADC_MAX_CLK_RATE)
+			break;
+	}
+	if (i >= ARRAY_SIZE(stm32f4_pclk_div))
+		return -EINVAL;
+
+	val = readl_relaxed(priv->common.base + STM32F4_ADC_CCR);
+	val &= ~STM32F4_ADC_ADCPRE_MASK;
+	val |= i << STM32F4_ADC_ADCPRE_SHIFT;
+	writel_relaxed(val, priv->common.base + STM32F4_ADC_CCR);
+
+	dev_dbg(&pdev->dev, "Using analog clock source at %ld kHz\n",
+		rate / (stm32f4_pclk_div[i] * 1000));
+
+	return 0;
+}
+
+/* ADC common interrupt for all instances */
+static void stm32_adc_irq_handler(struct irq_desc *desc)
+{
+	struct stm32_adc_priv *priv = irq_desc_get_handler_data(desc);
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	u32 status;
+
+	chained_irq_enter(chip, desc);
+	status = readl_relaxed(priv->common.base + STM32F4_ADC_CSR);
+
+	if (status & STM32F4_EOC1)
+		generic_handle_irq(irq_find_mapping(priv->domain, 0));
+
+	if (status & STM32F4_EOC2)
+		generic_handle_irq(irq_find_mapping(priv->domain, 1));
+
+	if (status & STM32F4_EOC3)
+		generic_handle_irq(irq_find_mapping(priv->domain, 2));
+
+	chained_irq_exit(chip, desc);
+};
+
+static int stm32_adc_domain_map(struct irq_domain *d, unsigned int irq,
+				irq_hw_number_t hwirq)
+{
+	irq_set_chip_data(irq, d->host_data);
+	irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_level_irq);
+
+	return 0;
+}
+
+static void stm32_adc_domain_unmap(struct irq_domain *d, unsigned int irq)
+{
+	irq_set_chip_and_handler(irq, NULL, NULL);
+	irq_set_chip_data(irq, NULL);
+}
+
+static const struct irq_domain_ops stm32_adc_domain_ops = {
+	.map = stm32_adc_domain_map,
+	.unmap  = stm32_adc_domain_unmap,
+	.xlate = irq_domain_xlate_onecell,
+};
+
+static int stm32_adc_irq_probe(struct platform_device *pdev,
+			       struct stm32_adc_priv *priv)
+{
+	struct device_node *np = pdev->dev.of_node;
+
+	priv->irq = platform_get_irq(pdev, 0);
+	if (priv->irq < 0) {
+		dev_err(&pdev->dev, "failed to get irq\n");
+		return priv->irq;
+	}
+
+	priv->domain = irq_domain_add_simple(np, STM32_ADC_MAX_ADCS, 0,
+					     &stm32_adc_domain_ops,
+					     priv);
+	if (!priv->domain) {
+		dev_err(&pdev->dev, "Failed to add irq domain\n");
+		return -ENOMEM;
+	}
+
+	irq_set_chained_handler(priv->irq, stm32_adc_irq_handler);
+	irq_set_handler_data(priv->irq, priv);
+
+	return 0;
+}
+
+static void stm32_adc_irq_remove(struct platform_device *pdev,
+				 struct stm32_adc_priv *priv)
+{
+	int hwirq;
+
+	for (hwirq = 0; hwirq < STM32_ADC_MAX_ADCS; hwirq++)
+		irq_dispose_mapping(irq_find_mapping(priv->domain, hwirq));
+	irq_domain_remove(priv->domain);
+	irq_set_chained_handler(priv->irq, NULL);
+}
+
+static int stm32_adc_probe(struct platform_device *pdev)
+{
+	struct stm32_adc_priv *priv;
+	struct device_node *np = pdev->dev.of_node;
+	struct resource *res;
+	int ret;
+
+	if (!pdev->dev.of_node)
+		return -ENODEV;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->common.base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(priv->common.base))
+		return PTR_ERR(priv->common.base);
+
+	priv->vref = devm_regulator_get(&pdev->dev, "vref");
+	if (IS_ERR(priv->vref)) {
+		ret = PTR_ERR(priv->vref);
+		dev_err(&pdev->dev, "vref get failed, %d\n", ret);
+		return ret;
+	}
+
+	ret = regulator_enable(priv->vref);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "vref enable failed\n");
+		return ret;
+	}
+
+	ret = regulator_get_voltage(priv->vref);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "vref get voltage failed, %d\n", ret);
+		goto err_regulator_disable;
+	}
+	priv->common.vref_mv = ret / 1000;
+	dev_dbg(&pdev->dev, "vref+=%dmV\n", priv->common.vref_mv);
+
+	priv->aclk = devm_clk_get(&pdev->dev, "adc");
+	if (IS_ERR(priv->aclk)) {
+		ret = PTR_ERR(priv->aclk);
+		dev_err(&pdev->dev, "Can't get 'adc' clock\n");
+		goto err_regulator_disable;
+	}
+
+	ret = clk_prepare_enable(priv->aclk);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "adc clk enable failed\n");
+		goto err_regulator_disable;
+	}
+
+	ret = stm32f4_adc_clk_sel(pdev, priv);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "adc clk selection failed\n");
+		goto err_clk_disable;
+	}
+
+	ret = stm32_adc_irq_probe(pdev, priv);
+	if (ret < 0)
+		goto err_clk_disable;
+
+	platform_set_drvdata(pdev, &priv->common);
+
+	ret = of_platform_populate(np, NULL, NULL, &pdev->dev);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to populate DT children\n");
+		goto err_irq_remove;
+	}
+
+	return 0;
+
+err_irq_remove:
+	stm32_adc_irq_remove(pdev, priv);
+
+err_clk_disable:
+	clk_disable_unprepare(priv->aclk);
+
+err_regulator_disable:
+	regulator_disable(priv->vref);
+
+	return ret;
+}
+
+static int stm32_adc_remove(struct platform_device *pdev)
+{
+	struct stm32_adc_common *common = platform_get_drvdata(pdev);
+	struct stm32_adc_priv *priv = to_stm32_adc_priv(common);
+
+	of_platform_depopulate(&pdev->dev);
+	stm32_adc_irq_remove(pdev, priv);
+	clk_disable_unprepare(priv->aclk);
+	regulator_disable(priv->vref);
+
+	return 0;
+}
+
+static const struct of_device_id stm32_adc_of_match[] = {
+	{ .compatible = "st,stm32f4-adc-core" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, stm32_adc_of_match);
+
+static struct platform_driver stm32_adc_driver = {
+	.probe = stm32_adc_probe,
+	.remove = stm32_adc_remove,
+	.driver = {
+		.name = "stm32-adc-core",
+		.of_match_table = stm32_adc_of_match,
+	},
+};
+module_platform_driver(stm32_adc_driver);
+
+MODULE_AUTHOR("Fabrice Gasnier <fabrice.gasnier@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics STM32 ADC core driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:stm32-adc-core");
diff --git a/drivers/iio/adc/stm32-adc-core.h b/drivers/iio/adc/stm32-adc-core.h
new file mode 100644
index 000000000000..081fa5f55015
--- /dev/null
+++ b/drivers/iio/adc/stm32-adc-core.h
@@ -0,0 +1,52 @@
+/*
+ * This file is part of STM32 ADC driver
+ *
+ * Copyright (C) 2016, STMicroelectronics - All Rights Reserved
+ * Author: Fabrice Gasnier <fabrice.gasnier@st.com>.
+ *
+ * License type: GPLv2
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __STM32_ADC_H
+#define __STM32_ADC_H
+
+/*
+ * STM32 - ADC global register map
+ * ________________________________________________________
+ * | Offset |                 Register                    |
+ * --------------------------------------------------------
+ * | 0x000  |                Master ADC1                  |
+ * --------------------------------------------------------
+ * | 0x100  |                Slave ADC2                   |
+ * --------------------------------------------------------
+ * | 0x200  |                Slave ADC3                   |
+ * --------------------------------------------------------
+ * | 0x300  |         Master & Slave common regs          |
+ * --------------------------------------------------------
+ */
+#define STM32_ADC_MAX_ADCS		3
+#define STM32_ADCX_COMN_OFFSET		0x300
+
+/**
+ * struct stm32_adc_common - stm32 ADC driver common data (for all instances)
+ * @base:		control registers base cpu addr
+ * @vref_mv:		vref voltage (mv)
+ */
+struct stm32_adc_common {
+	void __iomem			*base;
+	int				vref_mv;
+};
+
+#endif
diff --git a/drivers/iio/adc/stm32-adc.c b/drivers/iio/adc/stm32-adc.c
new file mode 100644
index 000000000000..5715e79f4935
--- /dev/null
+++ b/drivers/iio/adc/stm32-adc.c
@@ -0,0 +1,518 @@
+/*
+ * This file is part of STM32 ADC driver
+ *
+ * Copyright (C) 2016, STMicroelectronics - All Rights Reserved
+ * Author: Fabrice Gasnier <fabrice.gasnier@st.com>.
+ *
+ * License type: GPLv2
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/iio/iio.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+
+#include "stm32-adc-core.h"
+
+/* STM32F4 - Registers for each ADC instance */
+#define STM32F4_ADC_SR			0x00
+#define STM32F4_ADC_CR1			0x04
+#define STM32F4_ADC_CR2			0x08
+#define STM32F4_ADC_SMPR1		0x0C
+#define STM32F4_ADC_SMPR2		0x10
+#define STM32F4_ADC_HTR			0x24
+#define STM32F4_ADC_LTR			0x28
+#define STM32F4_ADC_SQR1		0x2C
+#define STM32F4_ADC_SQR2		0x30
+#define STM32F4_ADC_SQR3		0x34
+#define STM32F4_ADC_JSQR		0x38
+#define STM32F4_ADC_JDR1		0x3C
+#define STM32F4_ADC_JDR2		0x40
+#define STM32F4_ADC_JDR3		0x44
+#define STM32F4_ADC_JDR4		0x48
+#define STM32F4_ADC_DR			0x4C
+
+/* STM32F4_ADC_SR - bit fields */
+#define STM32F4_STRT			BIT(4)
+#define STM32F4_EOC			BIT(1)
+
+/* STM32F4_ADC_CR1 - bit fields */
+#define STM32F4_SCAN			BIT(8)
+#define STM32F4_EOCIE			BIT(5)
+
+/* STM32F4_ADC_CR2 - bit fields */
+#define STM32F4_SWSTART			BIT(30)
+#define STM32F4_EXTEN_MASK		GENMASK(29, 28)
+#define STM32F4_EOCS			BIT(10)
+#define STM32F4_ADON			BIT(0)
+
+/* STM32F4_ADC_SQR1 - bit fields */
+#define STM32F4_L_SHIFT			20
+#define STM32F4_L_MASK			GENMASK(23, 20)
+
+/* STM32F4_ADC_SQR3 - bit fields */
+#define STM32F4_SQ1_SHIFT		0
+#define STM32F4_SQ1_MASK		GENMASK(4, 0)
+
+#define STM32_ADC_TIMEOUT_US		100000
+#define STM32_ADC_TIMEOUT	(msecs_to_jiffies(STM32_ADC_TIMEOUT_US / 1000))
+
+/**
+ * struct stm32_adc - private data of each ADC IIO instance
+ * @common:		reference to ADC block common data
+ * @offset:		ADC instance register offset in ADC block
+ * @completion:		end of single conversion completion
+ * @buffer:		data buffer
+ * @clk:		clock for this adc instance
+ * @irq:		interrupt for this adc instance
+ * @lock:		spinlock
+ */
+struct stm32_adc {
+	struct stm32_adc_common	*common;
+	u32			offset;
+	struct completion	completion;
+	u16			*buffer;
+	struct clk		*clk;
+	int			irq;
+	spinlock_t		lock;		/* interrupt lock */
+};
+
+/**
+ * struct stm32_adc_chan_spec - specification of stm32 adc channel
+ * @type:	IIO channel type
+ * @channel:	channel number (single ended)
+ * @name:	channel name (single ended)
+ */
+struct stm32_adc_chan_spec {
+	enum iio_chan_type	type;
+	int			channel;
+	const char		*name;
+};
+
+/* Input definitions common for all STM32F4 instances */
+static const struct stm32_adc_chan_spec stm32f4_adc123_channels[] = {
+	{ IIO_VOLTAGE, 0, "in0" },
+	{ IIO_VOLTAGE, 1, "in1" },
+	{ IIO_VOLTAGE, 2, "in2" },
+	{ IIO_VOLTAGE, 3, "in3" },
+	{ IIO_VOLTAGE, 4, "in4" },
+	{ IIO_VOLTAGE, 5, "in5" },
+	{ IIO_VOLTAGE, 6, "in6" },
+	{ IIO_VOLTAGE, 7, "in7" },
+	{ IIO_VOLTAGE, 8, "in8" },
+	{ IIO_VOLTAGE, 9, "in9" },
+	{ IIO_VOLTAGE, 10, "in10" },
+	{ IIO_VOLTAGE, 11, "in11" },
+	{ IIO_VOLTAGE, 12, "in12" },
+	{ IIO_VOLTAGE, 13, "in13" },
+	{ IIO_VOLTAGE, 14, "in14" },
+	{ IIO_VOLTAGE, 15, "in15" },
+};
+
+/**
+ * STM32 ADC registers access routines
+ * @adc: stm32 adc instance
+ * @reg: reg offset in adc instance
+ *
+ * Note: All instances share same base, with 0x0, 0x100 or 0x200 offset resp.
+ * for adc1, adc2 and adc3.
+ */
+static u32 stm32_adc_readl(struct stm32_adc *adc, u32 reg)
+{
+	return readl_relaxed(adc->common->base + adc->offset + reg);
+}
+
+static u16 stm32_adc_readw(struct stm32_adc *adc, u32 reg)
+{
+	return readw_relaxed(adc->common->base + adc->offset + reg);
+}
+
+static void stm32_adc_writel(struct stm32_adc *adc, u32 reg, u32 val)
+{
+	writel_relaxed(val, adc->common->base + adc->offset + reg);
+}
+
+static void stm32_adc_set_bits(struct stm32_adc *adc, u32 reg, u32 bits)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adc->lock, flags);
+	stm32_adc_writel(adc, reg, stm32_adc_readl(adc, reg) | bits);
+	spin_unlock_irqrestore(&adc->lock, flags);
+}
+
+static void stm32_adc_clr_bits(struct stm32_adc *adc, u32 reg, u32 bits)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adc->lock, flags);
+	stm32_adc_writel(adc, reg, stm32_adc_readl(adc, reg) & ~bits);
+	spin_unlock_irqrestore(&adc->lock, flags);
+}
+
+/**
+ * stm32_adc_conv_irq_enable() - Enable end of conversion interrupt
+ * @adc: stm32 adc instance
+ */
+static void stm32_adc_conv_irq_enable(struct stm32_adc *adc)
+{
+	stm32_adc_set_bits(adc, STM32F4_ADC_CR1, STM32F4_EOCIE);
+};
+
+/**
+ * stm32_adc_conv_irq_disable() - Disable end of conversion interrupt
+ * @adc: stm32 adc instance
+ */
+static void stm32_adc_conv_irq_disable(struct stm32_adc *adc)
+{
+	stm32_adc_clr_bits(adc, STM32F4_ADC_CR1, STM32F4_EOCIE);
+}
+
+/**
+ * stm32_adc_start_conv() - Start conversions for regular channels.
+ * @adc: stm32 adc instance
+ */
+static void stm32_adc_start_conv(struct stm32_adc *adc)
+{
+	stm32_adc_set_bits(adc, STM32F4_ADC_CR1, STM32F4_SCAN);
+	stm32_adc_set_bits(adc, STM32F4_ADC_CR2, STM32F4_EOCS | STM32F4_ADON);
+
+	/* Wait for Power-up time (tSTAB from datasheet) */
+	usleep_range(2, 3);
+
+	/* Software start ? (e.g. trigger detection disabled ?) */
+	if (!(stm32_adc_readl(adc, STM32F4_ADC_CR2) & STM32F4_EXTEN_MASK))
+		stm32_adc_set_bits(adc, STM32F4_ADC_CR2, STM32F4_SWSTART);
+}
+
+static void stm32_adc_stop_conv(struct stm32_adc *adc)
+{
+	stm32_adc_clr_bits(adc, STM32F4_ADC_CR2, STM32F4_EXTEN_MASK);
+	stm32_adc_clr_bits(adc, STM32F4_ADC_SR, STM32F4_STRT);
+
+	stm32_adc_clr_bits(adc, STM32F4_ADC_CR1, STM32F4_SCAN);
+	stm32_adc_clr_bits(adc, STM32F4_ADC_CR2, STM32F4_ADON);
+}
+
+/**
+ * stm32_adc_single_conv() - Performs a single conversion
+ * @indio_dev: IIO device
+ * @chan: IIO channel
+ * @res: conversion result
+ *
+ * The function performs a single conversion on a given channel:
+ * - Program sequencer with one channel (e.g. in SQ1 with len = 1)
+ * - Use SW trigger
+ * - Start conversion, then wait for interrupt completion.
+ */
+static int stm32_adc_single_conv(struct iio_dev *indio_dev,
+				 const struct iio_chan_spec *chan,
+				 int *res)
+{
+	struct stm32_adc *adc = iio_priv(indio_dev);
+	long timeout;
+	u32 val;
+	u16 result;
+	int ret;
+
+	reinit_completion(&adc->completion);
+
+	adc->buffer = &result;
+
+	/* Program chan number in regular sequence */
+	val = stm32_adc_readl(adc, STM32F4_ADC_SQR3);
+	val &= ~STM32F4_SQ1_MASK;
+	val |= chan->channel << STM32F4_SQ1_SHIFT;
+	stm32_adc_writel(adc, STM32F4_ADC_SQR3, val);
+
+	/* Set regular sequence len (0 for 1 conversion) */
+	stm32_adc_clr_bits(adc, STM32F4_ADC_SQR1, STM32F4_L_MASK);
+
+	/* Trigger detection disabled (conversion can be launched in SW) */
+	stm32_adc_clr_bits(adc, STM32F4_ADC_CR2, STM32F4_EXTEN_MASK);
+
+	stm32_adc_conv_irq_enable(adc);
+
+	stm32_adc_start_conv(adc);
+
+	timeout = wait_for_completion_interruptible_timeout(
+					&adc->completion, STM32_ADC_TIMEOUT);
+	if (timeout == 0) {
+		ret = -ETIMEDOUT;
+	} else if (timeout < 0) {
+		ret = timeout;
+	} else {
+		*res = result;
+		ret = IIO_VAL_INT;
+	}
+
+	stm32_adc_stop_conv(adc);
+
+	stm32_adc_conv_irq_disable(adc);
+
+	return ret;
+}
+
+static int stm32_adc_read_raw(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      int *val, int *val2, long mask)
+{
+	struct stm32_adc *adc = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+		if (chan->type == IIO_VOLTAGE)
+			ret = stm32_adc_single_conv(indio_dev, chan, val);
+		else
+			ret = -EINVAL;
+		iio_device_release_direct_mode(indio_dev);
+		return ret;
+
+	case IIO_CHAN_INFO_SCALE:
+		*val = adc->common->vref_mv;
+		*val2 = chan->scan_type.realbits;
+		return IIO_VAL_FRACTIONAL_LOG2;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static irqreturn_t stm32_adc_isr(int irq, void *data)
+{
+	struct stm32_adc *adc = data;
+	u32 status = stm32_adc_readl(adc, STM32F4_ADC_SR);
+
+	if (status & STM32F4_EOC) {
+		*adc->buffer = stm32_adc_readw(adc, STM32F4_ADC_DR);
+		complete(&adc->completion);
+		return IRQ_HANDLED;
+	}
+
+	return IRQ_NONE;
+}
+
+static int stm32_adc_of_xlate(struct iio_dev *indio_dev,
+			      const struct of_phandle_args *iiospec)
+{
+	int i;
+
+	for (i = 0; i < indio_dev->num_channels; i++)
+		if (indio_dev->channels[i].channel == iiospec->args[0])
+			return i;
+
+	return -EINVAL;
+}
+
+/**
+ * stm32_adc_debugfs_reg_access - read or write register value
+ *
+ * To read a value from an ADC register:
+ *   echo [ADC reg offset] > direct_reg_access
+ *   cat direct_reg_access
+ *
+ * To write a value in a ADC register:
+ *   echo [ADC_reg_offset] [value] > direct_reg_access
+ */
+static int stm32_adc_debugfs_reg_access(struct iio_dev *indio_dev,
+					unsigned reg, unsigned writeval,
+					unsigned *readval)
+{
+	struct stm32_adc *adc = iio_priv(indio_dev);
+
+	if (!readval)
+		stm32_adc_writel(adc, reg, writeval);
+	else
+		*readval = stm32_adc_readl(adc, reg);
+
+	return 0;
+}
+
+static const struct iio_info stm32_adc_iio_info = {
+	.read_raw = stm32_adc_read_raw,
+	.debugfs_reg_access = stm32_adc_debugfs_reg_access,
+	.of_xlate = stm32_adc_of_xlate,
+	.driver_module = THIS_MODULE,
+};
+
+static void stm32_adc_chan_init_one(struct iio_dev *indio_dev,
+				    struct iio_chan_spec *chan,
+				    const struct stm32_adc_chan_spec *channel,
+				    int scan_index)
+{
+	chan->type = channel->type;
+	chan->channel = channel->channel;
+	chan->datasheet_name = channel->name;
+	chan->scan_index = scan_index;
+	chan->indexed = 1;
+	chan->info_mask_separate = BIT(IIO_CHAN_INFO_RAW);
+	chan->info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE);
+	chan->scan_type.sign = 'u';
+	chan->scan_type.realbits = 12;
+	chan->scan_type.storagebits = 16;
+}
+
+static int stm32_adc_chan_of_init(struct iio_dev *indio_dev)
+{
+	struct device_node *node = indio_dev->dev.of_node;
+	struct property *prop;
+	const __be32 *cur;
+	struct iio_chan_spec *channels;
+	int scan_index = 0, num_channels;
+	u32 val;
+
+	num_channels = of_property_count_u32_elems(node, "st,adc-channels");
+	if (num_channels < 0 ||
+	    num_channels >= ARRAY_SIZE(stm32f4_adc123_channels)) {
+		dev_err(&indio_dev->dev, "Bad st,adc-channels?\n");
+		return num_channels < 0 ? num_channels : -EINVAL;
+	}
+
+	channels = devm_kcalloc(&indio_dev->dev, num_channels,
+				sizeof(struct iio_chan_spec), GFP_KERNEL);
+	if (!channels)
+		return -ENOMEM;
+
+	of_property_for_each_u32(node, "st,adc-channels", prop, cur, val) {
+		if (val >= ARRAY_SIZE(stm32f4_adc123_channels)) {
+			dev_err(&indio_dev->dev, "Invalid channel %d\n", val);
+			return -EINVAL;
+		}
+		stm32_adc_chan_init_one(indio_dev, &channels[scan_index],
+					&stm32f4_adc123_channels[val],
+					scan_index);
+		scan_index++;
+	}
+
+	indio_dev->num_channels = scan_index;
+	indio_dev->channels = channels;
+
+	return 0;
+}
+
+static int stm32_adc_probe(struct platform_device *pdev)
+{
+	struct iio_dev *indio_dev;
+	struct stm32_adc *adc;
+	int ret;
+
+	if (!pdev->dev.of_node)
+		return -ENODEV;
+
+	indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*adc));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	adc = iio_priv(indio_dev);
+	adc->common = dev_get_drvdata(pdev->dev.parent);
+	spin_lock_init(&adc->lock);
+	init_completion(&adc->completion);
+
+	indio_dev->name = dev_name(&pdev->dev);
+	indio_dev->dev.parent = &pdev->dev;
+	indio_dev->dev.of_node = pdev->dev.of_node;
+	indio_dev->info = &stm32_adc_iio_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	platform_set_drvdata(pdev, adc);
+
+	ret = of_property_read_u32(pdev->dev.of_node, "reg", &adc->offset);
+	if (ret != 0) {
+		dev_err(&pdev->dev, "missing reg property\n");
+		return -EINVAL;
+	}
+
+	adc->irq = platform_get_irq(pdev, 0);
+	if (adc->irq < 0) {
+		dev_err(&pdev->dev, "failed to get irq\n");
+		return adc->irq;
+	}
+
+	ret = devm_request_irq(&pdev->dev, adc->irq, stm32_adc_isr,
+			       0, pdev->name, adc);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to request IRQ\n");
+		return ret;
+	}
+
+	adc->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(adc->clk)) {
+		dev_err(&pdev->dev, "Can't get clock\n");
+		return PTR_ERR(adc->clk);
+	}
+
+	ret = clk_prepare_enable(adc->clk);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "clk enable failed\n");
+		return ret;
+	}
+
+	ret = stm32_adc_chan_of_init(indio_dev);
+	if (ret < 0)
+		goto err_clk_disable;
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "iio dev register failed\n");
+		goto err_clk_disable;
+	}
+
+	return 0;
+
+err_clk_disable:
+	clk_disable_unprepare(adc->clk);
+
+	return ret;
+}
+
+static int stm32_adc_remove(struct platform_device *pdev)
+{
+	struct stm32_adc *adc = platform_get_drvdata(pdev);
+	struct iio_dev *indio_dev = iio_priv_to_dev(adc);
+
+	iio_device_unregister(indio_dev);
+	clk_disable_unprepare(adc->clk);
+
+	return 0;
+}
+
+static const struct of_device_id stm32_adc_of_match[] = {
+	{ .compatible = "st,stm32f4-adc" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, stm32_adc_of_match);
+
+static struct platform_driver stm32_adc_driver = {
+	.probe = stm32_adc_probe,
+	.remove = stm32_adc_remove,
+	.driver = {
+		.name = "stm32-adc",
+		.of_match_table = stm32_adc_of_match,
+	},
+};
+module_platform_driver(stm32_adc_driver);
+
+MODULE_AUTHOR("Fabrice Gasnier <fabrice.gasnier@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics STM32 ADC IIO driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:stm32-adc");
diff --git a/drivers/iio/adc/ti-adc0832.c b/drivers/iio/adc/ti-adc0832.c
index f4ba23effe9a..e952e94a14af 100644
--- a/drivers/iio/adc/ti-adc0832.c
+++ b/drivers/iio/adc/ti-adc0832.c
@@ -14,6 +14,10 @@
 #include <linux/spi/spi.h>
 #include <linux/iio/iio.h>
 #include <linux/regulator/consumer.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/trigger_consumer.h>
 
 enum {
 	adc0831,
@@ -38,10 +42,16 @@ struct adc0832 {
 		.indexed = 1,						\
 		.channel = chan,					\
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
-		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE)	\
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+		.scan_index = chan,					\
+		.scan_type = {						\
+			.sign = 'u',					\
+			.realbits = 8,					\
+			.storagebits = 8,				\
+		},							\
 	}
 
-#define ADC0832_VOLTAGE_CHANNEL_DIFF(chan1, chan2)			\
+#define ADC0832_VOLTAGE_CHANNEL_DIFF(chan1, chan2, si)			\
 	{								\
 		.type = IIO_VOLTAGE,					\
 		.indexed = 1,						\
@@ -49,18 +59,26 @@ struct adc0832 {
 		.channel2 = (chan2),					\
 		.differential = 1,					\
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
-		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE)	\
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+		.scan_index = si,					\
+		.scan_type = {						\
+			.sign = 'u',					\
+			.realbits = 8,					\
+			.storagebits = 8,				\
+		},							\
 	}
 
 static const struct iio_chan_spec adc0831_channels[] = {
-	ADC0832_VOLTAGE_CHANNEL_DIFF(0, 1),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(0, 1, 0),
+	IIO_CHAN_SOFT_TIMESTAMP(1),
 };
 
 static const struct iio_chan_spec adc0832_channels[] = {
 	ADC0832_VOLTAGE_CHANNEL(0),
 	ADC0832_VOLTAGE_CHANNEL(1),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(0, 1),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(1, 0),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(0, 1, 2),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(1, 0, 3),
+	IIO_CHAN_SOFT_TIMESTAMP(4),
 };
 
 static const struct iio_chan_spec adc0834_channels[] = {
@@ -68,10 +86,11 @@ static const struct iio_chan_spec adc0834_channels[] = {
 	ADC0832_VOLTAGE_CHANNEL(1),
 	ADC0832_VOLTAGE_CHANNEL(2),
 	ADC0832_VOLTAGE_CHANNEL(3),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(0, 1),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(1, 0),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(2, 3),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(3, 2),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(0, 1, 4),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(1, 0, 5),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(2, 3, 6),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(3, 2, 7),
+	IIO_CHAN_SOFT_TIMESTAMP(8),
 };
 
 static const struct iio_chan_spec adc0838_channels[] = {
@@ -83,14 +102,15 @@ static const struct iio_chan_spec adc0838_channels[] = {
 	ADC0832_VOLTAGE_CHANNEL(5),
 	ADC0832_VOLTAGE_CHANNEL(6),
 	ADC0832_VOLTAGE_CHANNEL(7),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(0, 1),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(1, 0),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(2, 3),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(3, 2),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(4, 5),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(5, 4),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(6, 7),
-	ADC0832_VOLTAGE_CHANNEL_DIFF(7, 6),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(0, 1, 8),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(1, 0, 9),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(2, 3, 10),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(3, 2, 11),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(4, 5, 12),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(5, 4, 13),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(6, 7, 14),
+	ADC0832_VOLTAGE_CHANNEL_DIFF(7, 6, 15),
+	IIO_CHAN_SOFT_TIMESTAMP(16),
 };
 
 static int adc0831_adc_conversion(struct adc0832 *adc)
@@ -178,6 +198,42 @@ static const struct iio_info adc0832_info = {
 	.driver_module = THIS_MODULE,
 };
 
+static irqreturn_t adc0832_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct adc0832 *adc = iio_priv(indio_dev);
+	u8 data[24] = { }; /* 16x 1 byte ADC data + 8 bytes timestamp */
+	int scan_index;
+	int i = 0;
+
+	mutex_lock(&adc->lock);
+
+	for_each_set_bit(scan_index, indio_dev->active_scan_mask,
+			 indio_dev->masklength) {
+		const struct iio_chan_spec *scan_chan =
+				&indio_dev->channels[scan_index];
+		int ret = adc0832_adc_conversion(adc, scan_chan->channel,
+						 scan_chan->differential);
+		if (ret < 0) {
+			dev_warn(&adc->spi->dev,
+				 "failed to get conversion data\n");
+			goto out;
+		}
+
+		data[i] = ret;
+		i++;
+	}
+	iio_push_to_buffers_with_timestamp(indio_dev, data,
+					   iio_get_time_ns(indio_dev));
+out:
+	mutex_unlock(&adc->lock);
+
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
 static int adc0832_probe(struct spi_device *spi)
 {
 	struct iio_dev *indio_dev;
@@ -233,9 +289,20 @@ static int adc0832_probe(struct spi_device *spi)
 
 	spi_set_drvdata(spi, indio_dev);
 
+	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+					 adc0832_trigger_handler, NULL);
+	if (ret)
+		goto err_reg_disable;
+
 	ret = iio_device_register(indio_dev);
 	if (ret)
-		regulator_disable(adc->reg);
+		goto err_buffer_cleanup;
+
+	return 0;
+err_buffer_cleanup:
+	iio_triggered_buffer_cleanup(indio_dev);
+err_reg_disable:
+	regulator_disable(adc->reg);
 
 	return ret;
 }
@@ -246,6 +313,7 @@ static int adc0832_remove(struct spi_device *spi)
 	struct adc0832 *adc = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
+	iio_triggered_buffer_cleanup(indio_dev);
 	regulator_disable(adc->reg);
 
 	return 0;
diff --git a/drivers/iio/adc/ti-adc161s626.c b/drivers/iio/adc/ti-adc161s626.c
index f94b69f9c288..4836a0d7aef5 100644
--- a/drivers/iio/adc/ti-adc161s626.c
+++ b/drivers/iio/adc/ti-adc161s626.c
@@ -27,6 +27,7 @@
 #include <linux/iio/buffer.h>
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/triggered_buffer.h>
+#include <linux/regulator/consumer.h>
 
 #define TI_ADC_DRV_NAME	"ti-adc161s626"
 
@@ -39,7 +40,9 @@ static const struct iio_chan_spec ti_adc141s626_channels[] = {
 	{
 		.type = IIO_VOLTAGE,
 		.channel = 0,
-		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE) |
+				      BIT(IIO_CHAN_INFO_OFFSET),
 		.scan_index = 0,
 		.scan_type = {
 			.sign = 's',
@@ -54,7 +57,9 @@ static const struct iio_chan_spec ti_adc161s626_channels[] = {
 	{
 		.type = IIO_VOLTAGE,
 		.channel = 0,
-		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE) |
+				      BIT(IIO_CHAN_INFO_OFFSET),
 		.scan_index = 0,
 		.scan_type = {
 			.sign = 's',
@@ -68,6 +73,8 @@ static const struct iio_chan_spec ti_adc161s626_channels[] = {
 struct ti_adc_data {
 	struct iio_dev *indio_dev;
 	struct spi_device *spi;
+	struct regulator *ref;
+
 	u8 read_size;
 	u8 shift;
 
@@ -135,18 +142,32 @@ static int ti_adc_read_raw(struct iio_dev *indio_dev,
 	struct ti_adc_data *data = iio_priv(indio_dev);
 	int ret;
 
-	if (mask != IIO_CHAN_INFO_RAW)
-		return -EINVAL;
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
 
-	ret = iio_device_claim_direct_mode(indio_dev);
-	if (ret)
-		return ret;
+		ret = ti_adc_read_measurement(data, chan, val);
+		iio_device_release_direct_mode(indio_dev);
 
-	ret = ti_adc_read_measurement(data, chan, val);
-	iio_device_release_direct_mode(indio_dev);
+		if (ret)
+			return ret;
 
-	if (!ret)
 		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		ret = regulator_get_voltage(data->ref);
+		if (ret < 0)
+			return ret;
+
+		*val = ret / 1000;
+		*val2 = chan->scan_type.realbits;
+
+		return IIO_VAL_FRACTIONAL_LOG2;
+	case IIO_CHAN_INFO_OFFSET:
+		*val = 1 << (chan->scan_type.realbits - 1);
+		return IIO_VAL_INT;
+	}
 
 	return 0;
 }
@@ -191,10 +212,17 @@ static int ti_adc_probe(struct spi_device *spi)
 		break;
 	}
 
+	data->ref = devm_regulator_get(&spi->dev, "vdda");
+	if (!IS_ERR(data->ref)) {
+		ret = regulator_enable(data->ref);
+		if (ret < 0)
+			return ret;
+	}
+
 	ret = iio_triggered_buffer_setup(indio_dev, NULL,
 					 ti_adc_trigger_handler, NULL);
 	if (ret)
-		return ret;
+		goto error_regulator_disable;
 
 	ret = iio_device_register(indio_dev);
 	if (ret)
@@ -205,15 +233,20 @@ static int ti_adc_probe(struct spi_device *spi)
 error_unreg_buffer:
 	iio_triggered_buffer_cleanup(indio_dev);
 
+error_regulator_disable:
+	regulator_disable(data->ref);
+
 	return ret;
 }
 
 static int ti_adc_remove(struct spi_device *spi)
 {
 	struct iio_dev *indio_dev = spi_get_drvdata(spi);
+	struct ti_adc_data *data = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
 	iio_triggered_buffer_cleanup(indio_dev);
+	regulator_disable(data->ref);
 
 	return 0;
 }
diff --git a/drivers/iio/adc/ti_am335x_adc.c b/drivers/iio/adc/ti_am335x_adc.c
index c3cfacca2541..ad9dec30bb30 100644
--- a/drivers/iio/adc/ti_am335x_adc.c
+++ b/drivers/iio/adc/ti_am335x_adc.c
@@ -30,10 +30,28 @@
 #include <linux/iio/buffer.h>
 #include <linux/iio/kfifo_buf.h>
 
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+
+#define DMA_BUFFER_SIZE		SZ_2K
+
+struct tiadc_dma {
+	struct dma_slave_config	conf;
+	struct dma_chan		*chan;
+	dma_addr_t		addr;
+	dma_cookie_t		cookie;
+	u8			*buf;
+	int			current_period;
+	int			period_size;
+	u8			fifo_thresh;
+};
+
 struct tiadc_device {
 	struct ti_tscadc_dev *mfd_tscadc;
+	struct tiadc_dma dma;
 	struct mutex fifo1_lock; /* to protect fifo access */
 	int channels;
+	int total_ch_enabled;
 	u8 channel_line[8];
 	u8 channel_step[8];
 	int buffer_en_ch_steps;
@@ -198,6 +216,67 @@ static irqreturn_t tiadc_worker_h(int irq, void *private)
 	return IRQ_HANDLED;
 }
 
+static void tiadc_dma_rx_complete(void *param)
+{
+	struct iio_dev *indio_dev = param;
+	struct tiadc_device *adc_dev = iio_priv(indio_dev);
+	struct tiadc_dma *dma = &adc_dev->dma;
+	u8 *data;
+	int i;
+
+	data = dma->buf + dma->current_period * dma->period_size;
+	dma->current_period = 1 - dma->current_period; /* swap the buffer ID */
+
+	for (i = 0; i < dma->period_size; i += indio_dev->scan_bytes) {
+		iio_push_to_buffers(indio_dev, data);
+		data += indio_dev->scan_bytes;
+	}
+}
+
+static int tiadc_start_dma(struct iio_dev *indio_dev)
+{
+	struct tiadc_device *adc_dev = iio_priv(indio_dev);
+	struct tiadc_dma *dma = &adc_dev->dma;
+	struct dma_async_tx_descriptor *desc;
+
+	dma->current_period = 0; /* We start to fill period 0 */
+	/*
+	 * Make the fifo thresh as the multiple of total number of
+	 * channels enabled, so make sure that cyclic DMA period
+	 * length is also a multiple of total number of channels
+	 * enabled. This ensures that no invalid data is reported
+	 * to the stack via iio_push_to_buffers().
+	 */
+	dma->fifo_thresh = rounddown(FIFO1_THRESHOLD + 1,
+				     adc_dev->total_ch_enabled) - 1;
+	/* Make sure that period length is multiple of fifo thresh level */
+	dma->period_size = rounddown(DMA_BUFFER_SIZE / 2,
+				    (dma->fifo_thresh + 1) * sizeof(u16));
+
+	dma->conf.src_maxburst = dma->fifo_thresh + 1;
+	dmaengine_slave_config(dma->chan, &dma->conf);
+
+	desc = dmaengine_prep_dma_cyclic(dma->chan, dma->addr,
+					 dma->period_size * 2,
+					 dma->period_size, DMA_DEV_TO_MEM,
+					 DMA_PREP_INTERRUPT);
+	if (!desc)
+		return -EBUSY;
+
+	desc->callback = tiadc_dma_rx_complete;
+	desc->callback_param = indio_dev;
+
+	dma->cookie = dmaengine_submit(desc);
+
+	dma_async_issue_pending(dma->chan);
+
+	tiadc_writel(adc_dev, REG_FIFO1THR, dma->fifo_thresh);
+	tiadc_writel(adc_dev, REG_DMA1REQ, dma->fifo_thresh);
+	tiadc_writel(adc_dev, REG_DMAENABLE_SET, DMA_FIFO1);
+
+	return 0;
+}
+
 static int tiadc_buffer_preenable(struct iio_dev *indio_dev)
 {
 	struct tiadc_device *adc_dev = iio_priv(indio_dev);
@@ -218,20 +297,30 @@ static int tiadc_buffer_preenable(struct iio_dev *indio_dev)
 static int tiadc_buffer_postenable(struct iio_dev *indio_dev)
 {
 	struct tiadc_device *adc_dev = iio_priv(indio_dev);
+	struct tiadc_dma *dma = &adc_dev->dma;
+	unsigned int irq_enable;
 	unsigned int enb = 0;
 	u8 bit;
 
 	tiadc_step_config(indio_dev);
-	for_each_set_bit(bit, indio_dev->active_scan_mask, adc_dev->channels)
+	for_each_set_bit(bit, indio_dev->active_scan_mask, adc_dev->channels) {
 		enb |= (get_adc_step_bit(adc_dev, bit) << 1);
+		adc_dev->total_ch_enabled++;
+	}
 	adc_dev->buffer_en_ch_steps = enb;
 
+	if (dma->chan)
+		tiadc_start_dma(indio_dev);
+
 	am335x_tsc_se_set_cache(adc_dev->mfd_tscadc, enb);
 
 	tiadc_writel(adc_dev,  REG_IRQSTATUS, IRQENB_FIFO1THRES
 				| IRQENB_FIFO1OVRRUN | IRQENB_FIFO1UNDRFLW);
-	tiadc_writel(adc_dev,  REG_IRQENABLE, IRQENB_FIFO1THRES
-				| IRQENB_FIFO1OVRRUN);
+
+	irq_enable = IRQENB_FIFO1OVRRUN;
+	if (!dma->chan)
+		irq_enable |= IRQENB_FIFO1THRES;
+	tiadc_writel(adc_dev,  REG_IRQENABLE, irq_enable);
 
 	return 0;
 }
@@ -239,12 +328,18 @@ static int tiadc_buffer_postenable(struct iio_dev *indio_dev)
 static int tiadc_buffer_predisable(struct iio_dev *indio_dev)
 {
 	struct tiadc_device *adc_dev = iio_priv(indio_dev);
+	struct tiadc_dma *dma = &adc_dev->dma;
 	int fifo1count, i, read;
 
 	tiadc_writel(adc_dev, REG_IRQCLR, (IRQENB_FIFO1THRES |
 				IRQENB_FIFO1OVRRUN | IRQENB_FIFO1UNDRFLW));
 	am335x_tsc_se_clr(adc_dev->mfd_tscadc, adc_dev->buffer_en_ch_steps);
 	adc_dev->buffer_en_ch_steps = 0;
+	adc_dev->total_ch_enabled = 0;
+	if (dma->chan) {
+		tiadc_writel(adc_dev, REG_DMAENABLE_CLEAR, 0x2);
+		dmaengine_terminate_async(dma->chan);
+	}
 
 	/* Flush FIFO of leftover data in the time it takes to disable adc */
 	fifo1count = tiadc_readl(adc_dev, REG_FIFO1CNT);
@@ -430,6 +525,41 @@ static const struct iio_info tiadc_info = {
 	.driver_module = THIS_MODULE,
 };
 
+static int tiadc_request_dma(struct platform_device *pdev,
+			     struct tiadc_device *adc_dev)
+{
+	struct tiadc_dma	*dma = &adc_dev->dma;
+	dma_cap_mask_t		mask;
+
+	/* Default slave configuration parameters */
+	dma->conf.direction = DMA_DEV_TO_MEM;
+	dma->conf.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
+	dma->conf.src_addr = adc_dev->mfd_tscadc->tscadc_phys_base + REG_FIFO1;
+
+	dma_cap_zero(mask);
+	dma_cap_set(DMA_CYCLIC, mask);
+
+	/* Get a channel for RX */
+	dma->chan = dma_request_chan(adc_dev->mfd_tscadc->dev, "fifo1");
+	if (IS_ERR(dma->chan)) {
+		int ret = PTR_ERR(dma->chan);
+
+		dma->chan = NULL;
+		return ret;
+	}
+
+	/* RX buffer */
+	dma->buf = dma_alloc_coherent(dma->chan->device->dev, DMA_BUFFER_SIZE,
+				      &dma->addr, GFP_KERNEL);
+	if (!dma->buf)
+		goto err;
+
+	return 0;
+err:
+	dma_release_channel(dma->chan);
+	return -ENOMEM;
+}
+
 static int tiadc_parse_dt(struct platform_device *pdev,
 			  struct tiadc_device *adc_dev)
 {
@@ -512,8 +642,14 @@ static int tiadc_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, indio_dev);
 
+	err = tiadc_request_dma(pdev, adc_dev);
+	if (err && err == -EPROBE_DEFER)
+		goto err_dma;
+
 	return 0;
 
+err_dma:
+	iio_device_unregister(indio_dev);
 err_buffer_unregister:
 	tiadc_iio_buffered_hardware_remove(indio_dev);
 err_free_channels:
@@ -525,8 +661,14 @@ static int tiadc_remove(struct platform_device *pdev)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct tiadc_device *adc_dev = iio_priv(indio_dev);
+	struct tiadc_dma *dma = &adc_dev->dma;
 	u32 step_en;
 
+	if (dma->chan) {
+		dma_free_coherent(dma->chan->device->dev, DMA_BUFFER_SIZE,
+				  dma->buf, dma->addr);
+		dma_release_channel(dma->chan);
+	}
 	iio_device_unregister(indio_dev);
 	tiadc_iio_buffered_hardware_remove(indio_dev);
 	tiadc_channels_remove(indio_dev);
diff --git a/drivers/iio/common/Kconfig b/drivers/iio/common/Kconfig
index 26a6026de614..e108996a9627 100644
--- a/drivers/iio/common/Kconfig
+++ b/drivers/iio/common/Kconfig
@@ -2,6 +2,7 @@
 # IIO common modules
 #
 
+source "drivers/iio/common/cros_ec_sensors/Kconfig"
 source "drivers/iio/common/hid-sensors/Kconfig"
 source "drivers/iio/common/ms_sensors/Kconfig"
 source "drivers/iio/common/ssp_sensors/Kconfig"
diff --git a/drivers/iio/common/Makefile b/drivers/iio/common/Makefile
index 585da6a1b188..6fa760e1bdd5 100644
--- a/drivers/iio/common/Makefile
+++ b/drivers/iio/common/Makefile
@@ -7,6 +7,7 @@
 #
 
 # When adding new entries keep the list in alphabetical order
+obj-y += cros_ec_sensors/
 obj-y += hid-sensors/
 obj-y += ms_sensors/
 obj-y += ssp_sensors/
diff --git a/drivers/iio/common/cros_ec_sensors/Kconfig b/drivers/iio/common/cros_ec_sensors/Kconfig
new file mode 100644
index 000000000000..135f6825903f
--- /dev/null
+++ b/drivers/iio/common/cros_ec_sensors/Kconfig
@@ -0,0 +1,22 @@
+#
+# Chrome OS Embedded Controller managed sensors library
+#
+config IIO_CROS_EC_SENSORS_CORE
+	tristate "ChromeOS EC Sensors Core"
+	depends on SYSFS && MFD_CROS_EC
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	help
+	  Base module for the ChromeOS EC Sensors module.
+	  Contains core functions used by other IIO CrosEC sensor
+	  drivers.
+	  Define common attributes and sysfs interrupt handler.
+
+config IIO_CROS_EC_SENSORS
+	tristate "ChromeOS EC Contiguous Sensors"
+	depends on IIO_CROS_EC_SENSORS_CORE
+	help
+	  Module to handle 3d contiguous sensors like
+	  Accelerometers, Gyroscope and Magnetometer that are
+	  presented by the ChromeOS EC Sensor hub.
+	  Creates an IIO device for each functions.
diff --git a/drivers/iio/common/cros_ec_sensors/Makefile b/drivers/iio/common/cros_ec_sensors/Makefile
new file mode 100644
index 000000000000..ec716ff2a775
--- /dev/null
+++ b/drivers/iio/common/cros_ec_sensors/Makefile
@@ -0,0 +1,6 @@
+#
+# Makefile for sensors seen through the ChromeOS EC sensor hub.
+#
+
+obj-$(CONFIG_IIO_CROS_EC_SENSORS_CORE) += cros_ec_sensors_core.o
+obj-$(CONFIG_IIO_CROS_EC_SENSORS) += cros_ec_sensors.o
diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c
new file mode 100644
index 000000000000..d6c372bb433b
--- /dev/null
+++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c
@@ -0,0 +1,322 @@
+/*
+ * cros_ec_sensors - Driver for Chrome OS Embedded Controller sensors.
+ *
+ * Copyright (C) 2016 Google, Inc
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * This driver uses the cros-ec interface to communicate with the Chrome OS
+ * EC about sensors data. Data access is presented through iio sysfs.
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/kfifo_buf.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/kernel.h>
+#include <linux/mfd/cros_ec.h>
+#include <linux/mfd/cros_ec_commands.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/sysfs.h>
+
+#include "cros_ec_sensors_core.h"
+
+#define CROS_EC_SENSORS_MAX_CHANNELS 4
+
+/* State data for ec_sensors iio driver. */
+struct cros_ec_sensors_state {
+	/* Shared by all sensors */
+	struct cros_ec_sensors_core_state core;
+
+	struct iio_chan_spec channels[CROS_EC_SENSORS_MAX_CHANNELS];
+};
+
+static int cros_ec_sensors_read(struct iio_dev *indio_dev,
+			  struct iio_chan_spec const *chan,
+			  int *val, int *val2, long mask)
+{
+	struct cros_ec_sensors_state *st = iio_priv(indio_dev);
+	s16 data = 0;
+	s64 val64;
+	int i;
+	int ret;
+	int idx = chan->scan_index;
+
+	mutex_lock(&st->core.cmd_lock);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = st->core.read_ec_sensors_data(indio_dev, 1 << idx, &data);
+		if (ret < 0)
+			break;
+
+		*val = data;
+		break;
+	case IIO_CHAN_INFO_CALIBBIAS:
+		st->core.param.cmd = MOTIONSENSE_CMD_SENSOR_OFFSET;
+		st->core.param.sensor_offset.flags = 0;
+
+		ret = cros_ec_motion_send_host_cmd(&st->core, 0);
+		if (ret < 0)
+			break;
+
+		/* Save values */
+		for (i = CROS_EC_SENSOR_X; i < CROS_EC_SENSOR_MAX_AXIS; i++)
+			st->core.calib[i] =
+				st->core.resp->sensor_offset.offset[i];
+
+		*val = st->core.calib[idx];
+		break;
+	case IIO_CHAN_INFO_SCALE:
+		st->core.param.cmd = MOTIONSENSE_CMD_SENSOR_RANGE;
+		st->core.param.sensor_range.data = EC_MOTION_SENSE_NO_VALUE;
+
+		ret = cros_ec_motion_send_host_cmd(&st->core, 0);
+		if (ret < 0)
+			break;
+
+		val64 = st->core.resp->sensor_range.ret;
+		switch (st->core.type) {
+		case MOTIONSENSE_TYPE_ACCEL:
+			/*
+			 * EC returns data in g, iio exepects m/s^2.
+			 * Do not use IIO_G_TO_M_S_2 to avoid precision loss.
+			 */
+			*val = div_s64(val64 * 980665, 10);
+			*val2 = 10000 << (CROS_EC_SENSOR_BITS - 1);
+			ret = IIO_VAL_FRACTIONAL;
+			break;
+		case MOTIONSENSE_TYPE_GYRO:
+			/*
+			 * EC returns data in dps, iio expects rad/s.
+			 * Do not use IIO_DEGREE_TO_RAD to avoid precision
+			 * loss. Round to the nearest integer.
+			 */
+			*val = div_s64(val64 * 314159 + 9000000ULL, 1000);
+			*val2 = 18000 << (CROS_EC_SENSOR_BITS - 1);
+			ret = IIO_VAL_FRACTIONAL;
+			break;
+		case MOTIONSENSE_TYPE_MAG:
+			/*
+			 * EC returns data in 16LSB / uT,
+			 * iio expects Gauss
+			 */
+			*val = val64;
+			*val2 = 100 << (CROS_EC_SENSOR_BITS - 1);
+			ret = IIO_VAL_FRACTIONAL;
+			break;
+		default:
+			ret = -EINVAL;
+		}
+		break;
+	default:
+		ret = cros_ec_sensors_core_read(&st->core, chan, val, val2,
+						mask);
+		break;
+	}
+	mutex_unlock(&st->core.cmd_lock);
+
+	return ret;
+}
+
+static int cros_ec_sensors_write(struct iio_dev *indio_dev,
+			       struct iio_chan_spec const *chan,
+			       int val, int val2, long mask)
+{
+	struct cros_ec_sensors_state *st = iio_priv(indio_dev);
+	int i;
+	int ret;
+	int idx = chan->scan_index;
+
+	mutex_lock(&st->core.cmd_lock);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_CALIBBIAS:
+		st->core.calib[idx] = val;
+
+		/* Send to EC for each axis, even if not complete */
+		st->core.param.cmd = MOTIONSENSE_CMD_SENSOR_OFFSET;
+		st->core.param.sensor_offset.flags =
+			MOTION_SENSE_SET_OFFSET;
+		for (i = CROS_EC_SENSOR_X; i < CROS_EC_SENSOR_MAX_AXIS; i++)
+			st->core.param.sensor_offset.offset[i] =
+				st->core.calib[i];
+		st->core.param.sensor_offset.temp =
+			EC_MOTION_SENSE_INVALID_CALIB_TEMP;
+
+		ret = cros_ec_motion_send_host_cmd(&st->core, 0);
+		break;
+	case IIO_CHAN_INFO_SCALE:
+		if (st->core.type == MOTIONSENSE_TYPE_MAG) {
+			ret = -EINVAL;
+			break;
+		}
+		st->core.param.cmd = MOTIONSENSE_CMD_SENSOR_RANGE;
+		st->core.param.sensor_range.data = val;
+
+		/* Always roundup, so caller gets at least what it asks for. */
+		st->core.param.sensor_range.roundup = 1;
+
+		ret = cros_ec_motion_send_host_cmd(&st->core, 0);
+		break;
+	default:
+		ret = cros_ec_sensors_core_write(
+				&st->core, chan, val, val2, mask);
+		break;
+	}
+
+	mutex_unlock(&st->core.cmd_lock);
+
+	return ret;
+}
+
+static const struct iio_info ec_sensors_info = {
+	.read_raw = &cros_ec_sensors_read,
+	.write_raw = &cros_ec_sensors_write,
+	.driver_module = THIS_MODULE,
+};
+
+static int cros_ec_sensors_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct cros_ec_dev *ec_dev = dev_get_drvdata(dev->parent);
+	struct cros_ec_device *ec_device;
+	struct iio_dev *indio_dev;
+	struct cros_ec_sensors_state *state;
+	struct iio_chan_spec *channel;
+	int ret, i;
+
+	if (!ec_dev || !ec_dev->ec_dev) {
+		dev_warn(&pdev->dev, "No CROS EC device found.\n");
+		return -EINVAL;
+	}
+	ec_device = ec_dev->ec_dev;
+
+	indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*state));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	ret = cros_ec_sensors_core_init(pdev, indio_dev, true);
+	if (ret)
+		return ret;
+
+	indio_dev->info = &ec_sensors_info;
+	state = iio_priv(indio_dev);
+	for (channel = state->channels, i = CROS_EC_SENSOR_X;
+	     i < CROS_EC_SENSOR_MAX_AXIS; i++, channel++) {
+		/* Common part */
+		channel->info_mask_separate =
+			BIT(IIO_CHAN_INFO_RAW) |
+			BIT(IIO_CHAN_INFO_CALIBBIAS);
+		channel->info_mask_shared_by_all =
+			BIT(IIO_CHAN_INFO_SCALE) |
+			BIT(IIO_CHAN_INFO_FREQUENCY) |
+			BIT(IIO_CHAN_INFO_SAMP_FREQ);
+		channel->scan_type.realbits = CROS_EC_SENSOR_BITS;
+		channel->scan_type.storagebits = CROS_EC_SENSOR_BITS;
+		channel->scan_index = i;
+		channel->ext_info = cros_ec_sensors_ext_info;
+		channel->modified = 1;
+		channel->channel2 = IIO_MOD_X + i;
+		channel->scan_type.sign = 's';
+
+		/* Sensor specific */
+		switch (state->core.type) {
+		case MOTIONSENSE_TYPE_ACCEL:
+			channel->type = IIO_ACCEL;
+			break;
+		case MOTIONSENSE_TYPE_GYRO:
+			channel->type = IIO_ANGL_VEL;
+			break;
+		case MOTIONSENSE_TYPE_MAG:
+			channel->type = IIO_MAGN;
+			break;
+		default:
+			dev_err(&pdev->dev, "Unknown motion sensor\n");
+			return -EINVAL;
+		}
+	}
+
+	/* Timestamp */
+	channel->type = IIO_TIMESTAMP;
+	channel->channel = -1;
+	channel->scan_index = CROS_EC_SENSOR_MAX_AXIS;
+	channel->scan_type.sign = 's';
+	channel->scan_type.realbits = 64;
+	channel->scan_type.storagebits = 64;
+
+	indio_dev->channels = state->channels;
+	indio_dev->num_channels = CROS_EC_SENSORS_MAX_CHANNELS;
+
+	/* There is only enough room for accel and gyro in the io space */
+	if ((state->core.ec->cmd_readmem != NULL) &&
+	    (state->core.type != MOTIONSENSE_TYPE_MAG))
+		state->core.read_ec_sensors_data = cros_ec_sensors_read_lpc;
+	else
+		state->core.read_ec_sensors_data = cros_ec_sensors_read_cmd;
+
+	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+					 cros_ec_sensors_capture, NULL);
+	if (ret)
+		return ret;
+
+	ret = iio_device_register(indio_dev);
+	if (ret)
+		goto error_uninit_buffer;
+
+	return 0;
+
+error_uninit_buffer:
+	iio_triggered_buffer_cleanup(indio_dev);
+
+	return ret;
+}
+
+static int cros_ec_sensors_remove(struct platform_device *pdev)
+{
+	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+
+	iio_device_unregister(indio_dev);
+	iio_triggered_buffer_cleanup(indio_dev);
+
+	return 0;
+}
+
+static const struct platform_device_id cros_ec_sensors_ids[] = {
+	{
+		.name = "cros-ec-accel",
+	},
+	{
+		.name = "cros-ec-gyro",
+	},
+	{
+		.name = "cros-ec-mag",
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(platform, cros_ec_sensors_ids);
+
+static struct platform_driver cros_ec_sensors_platform_driver = {
+	.driver = {
+		.name	= "cros-ec-sensors",
+	},
+	.probe		= cros_ec_sensors_probe,
+	.remove		= cros_ec_sensors_remove,
+	.id_table	= cros_ec_sensors_ids,
+};
+module_platform_driver(cros_ec_sensors_platform_driver);
+
+MODULE_DESCRIPTION("ChromeOS EC 3-axis sensors driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c
new file mode 100644
index 000000000000..416cae5ebbd0
--- /dev/null
+++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c
@@ -0,0 +1,450 @@
+/*
+ * cros_ec_sensors_core - Common function for Chrome OS EC sensor driver.
+ *
+ * Copyright (C) 2016 Google, Inc
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/kfifo_buf.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/kernel.h>
+#include <linux/mfd/cros_ec.h>
+#include <linux/mfd/cros_ec_commands.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/sysfs.h>
+#include <linux/platform_device.h>
+
+#include "cros_ec_sensors_core.h"
+
+static char *cros_ec_loc[] = {
+	[MOTIONSENSE_LOC_BASE] = "base",
+	[MOTIONSENSE_LOC_LID] = "lid",
+	[MOTIONSENSE_LOC_MAX] = "unknown",
+};
+
+int cros_ec_sensors_core_init(struct platform_device *pdev,
+			      struct iio_dev *indio_dev,
+			      bool physical_device)
+{
+	struct device *dev = &pdev->dev;
+	struct cros_ec_sensors_core_state *state = iio_priv(indio_dev);
+	struct cros_ec_dev *ec = dev_get_drvdata(pdev->dev.parent);
+	struct cros_ec_sensor_platform *sensor_platform = dev_get_platdata(dev);
+
+	platform_set_drvdata(pdev, indio_dev);
+
+	state->ec = ec->ec_dev;
+	state->msg = devm_kzalloc(&pdev->dev,
+				max((u16)sizeof(struct ec_params_motion_sense),
+				state->ec->max_response), GFP_KERNEL);
+	if (!state->msg)
+		return -ENOMEM;
+
+	state->resp = (struct ec_response_motion_sense *)state->msg->data;
+
+	mutex_init(&state->cmd_lock);
+
+	/* Set up the host command structure. */
+	state->msg->version = 2;
+	state->msg->command = EC_CMD_MOTION_SENSE_CMD + ec->cmd_offset;
+	state->msg->outsize = sizeof(struct ec_params_motion_sense);
+
+	indio_dev->dev.parent = &pdev->dev;
+	indio_dev->name = pdev->name;
+
+	if (physical_device) {
+		indio_dev->modes = INDIO_DIRECT_MODE;
+
+		state->param.cmd = MOTIONSENSE_CMD_INFO;
+		state->param.info.sensor_num = sensor_platform->sensor_num;
+		if (cros_ec_motion_send_host_cmd(state, 0)) {
+			dev_warn(dev, "Can not access sensor info\n");
+			return -EIO;
+		}
+		state->type = state->resp->info.type;
+		state->loc = state->resp->info.location;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cros_ec_sensors_core_init);
+
+int cros_ec_motion_send_host_cmd(struct cros_ec_sensors_core_state *state,
+				 u16 opt_length)
+{
+	int ret;
+
+	if (opt_length)
+		state->msg->insize = min(opt_length, state->ec->max_response);
+	else
+		state->msg->insize = state->ec->max_response;
+
+	memcpy(state->msg->data, &state->param, sizeof(state->param));
+
+	ret = cros_ec_cmd_xfer_status(state->ec, state->msg);
+	if (ret < 0)
+		return -EIO;
+
+	if (ret &&
+	    state->resp != (struct ec_response_motion_sense *)state->msg->data)
+		memcpy(state->resp, state->msg->data, ret);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cros_ec_motion_send_host_cmd);
+
+static ssize_t cros_ec_sensors_calibrate(struct iio_dev *indio_dev,
+		uintptr_t private, const struct iio_chan_spec *chan,
+		const char *buf, size_t len)
+{
+	struct cros_ec_sensors_core_state *st = iio_priv(indio_dev);
+	int ret, i;
+	bool calibrate;
+
+	ret = strtobool(buf, &calibrate);
+	if (ret < 0)
+		return ret;
+	if (!calibrate)
+		return -EINVAL;
+
+	mutex_lock(&st->cmd_lock);
+	st->param.cmd = MOTIONSENSE_CMD_PERFORM_CALIB;
+	ret = cros_ec_motion_send_host_cmd(st, 0);
+	if (ret != 0) {
+		dev_warn(&indio_dev->dev, "Unable to calibrate sensor\n");
+	} else {
+		/* Save values */
+		for (i = CROS_EC_SENSOR_X; i < CROS_EC_SENSOR_MAX_AXIS; i++)
+			st->calib[i] = st->resp->perform_calib.offset[i];
+	}
+	mutex_unlock(&st->cmd_lock);
+
+	return ret ? ret : len;
+}
+
+static ssize_t cros_ec_sensors_loc(struct iio_dev *indio_dev,
+		uintptr_t private, const struct iio_chan_spec *chan,
+		char *buf)
+{
+	struct cros_ec_sensors_core_state *st = iio_priv(indio_dev);
+
+	return snprintf(buf, PAGE_SIZE, "%s\n", cros_ec_loc[st->loc]);
+}
+
+const struct iio_chan_spec_ext_info cros_ec_sensors_ext_info[] = {
+	{
+		.name = "calibrate",
+		.shared = IIO_SHARED_BY_ALL,
+		.write = cros_ec_sensors_calibrate
+	},
+	{
+		.name = "location",
+		.shared = IIO_SHARED_BY_ALL,
+		.read = cros_ec_sensors_loc
+	},
+	{ },
+};
+EXPORT_SYMBOL_GPL(cros_ec_sensors_ext_info);
+
+/**
+ * cros_ec_sensors_idx_to_reg - convert index into offset in shared memory
+ * @st:		pointer to state information for device
+ * @idx:	sensor index (should be element of enum sensor_index)
+ *
+ * Return:	address to read at
+ */
+static unsigned int cros_ec_sensors_idx_to_reg(
+					struct cros_ec_sensors_core_state *st,
+					unsigned int idx)
+{
+	/*
+	 * When using LPC interface, only space for 2 Accel and one Gyro.
+	 * First halfword of MOTIONSENSE_TYPE_ACCEL is used by angle.
+	 */
+	if (st->type == MOTIONSENSE_TYPE_ACCEL)
+		return EC_MEMMAP_ACC_DATA + sizeof(u16) *
+			(1 + idx + st->param.info.sensor_num *
+			 CROS_EC_SENSOR_MAX_AXIS);
+
+	return EC_MEMMAP_GYRO_DATA + sizeof(u16) * idx;
+}
+
+static int cros_ec_sensors_cmd_read_u8(struct cros_ec_device *ec,
+				       unsigned int offset, u8 *dest)
+{
+	return ec->cmd_readmem(ec, offset, 1, dest);
+}
+
+static int cros_ec_sensors_cmd_read_u16(struct cros_ec_device *ec,
+					 unsigned int offset, u16 *dest)
+{
+	__le16 tmp;
+	int ret = ec->cmd_readmem(ec, offset, 2, &tmp);
+
+	if (ret >= 0)
+		*dest = le16_to_cpu(tmp);
+
+	return ret;
+}
+
+/**
+ * cros_ec_sensors_read_until_not_busy() - read until is not busy
+ *
+ * @st:	pointer to state information for device
+ *
+ * Read from EC status byte until it reads not busy.
+ * Return: 8-bit status if ok, -errno on failure.
+ */
+static int cros_ec_sensors_read_until_not_busy(
+					struct cros_ec_sensors_core_state *st)
+{
+	struct cros_ec_device *ec = st->ec;
+	u8 status;
+	int ret, attempts = 0;
+
+	ret = cros_ec_sensors_cmd_read_u8(ec, EC_MEMMAP_ACC_STATUS, &status);
+	if (ret < 0)
+		return ret;
+
+	while (status & EC_MEMMAP_ACC_STATUS_BUSY_BIT) {
+		/* Give up after enough attempts, return error. */
+		if (attempts++ >= 50)
+			return -EIO;
+
+		/* Small delay every so often. */
+		if (attempts % 5 == 0)
+			msleep(25);
+
+		ret = cros_ec_sensors_cmd_read_u8(ec, EC_MEMMAP_ACC_STATUS,
+						  &status);
+		if (ret < 0)
+			return ret;
+	}
+
+	return status;
+}
+
+/**
+ * read_ec_sensors_data_unsafe() - read acceleration data from EC shared memory
+ * @indio_dev:	pointer to IIO device
+ * @scan_mask:	bitmap of the sensor indices to scan
+ * @data:	location to store data
+ *
+ * This is the unsafe function for reading the EC data. It does not guarantee
+ * that the EC will not modify the data as it is being read in.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+static int cros_ec_sensors_read_data_unsafe(struct iio_dev *indio_dev,
+			 unsigned long scan_mask, s16 *data)
+{
+	struct cros_ec_sensors_core_state *st = iio_priv(indio_dev);
+	struct cros_ec_device *ec = st->ec;
+	unsigned int i;
+	int ret;
+
+	/* Read all sensors enabled in scan_mask. Each value is 2 bytes. */
+	for_each_set_bit(i, &scan_mask, indio_dev->masklength) {
+		ret = cros_ec_sensors_cmd_read_u16(ec,
+					     cros_ec_sensors_idx_to_reg(st, i),
+					     data);
+		if (ret < 0)
+			return ret;
+
+		data++;
+	}
+
+	return 0;
+}
+
+int cros_ec_sensors_read_lpc(struct iio_dev *indio_dev,
+			     unsigned long scan_mask, s16 *data)
+{
+	struct cros_ec_sensors_core_state *st = iio_priv(indio_dev);
+	struct cros_ec_device *ec = st->ec;
+	u8 samp_id = 0xff, status = 0;
+	int ret, attempts = 0;
+
+	/*
+	 * Continually read all data from EC until the status byte after
+	 * all reads reflects that the EC is not busy and the sample id
+	 * matches the sample id from before all reads. This guarantees
+	 * that data read in was not modified by the EC while reading.
+	 */
+	while ((status & (EC_MEMMAP_ACC_STATUS_BUSY_BIT |
+			  EC_MEMMAP_ACC_STATUS_SAMPLE_ID_MASK)) != samp_id) {
+		/* If we have tried to read too many times, return error. */
+		if (attempts++ >= 5)
+			return -EIO;
+
+		/* Read status byte until EC is not busy. */
+		ret = cros_ec_sensors_read_until_not_busy(st);
+		if (ret < 0)
+			return ret;
+
+		/*
+		 * Store the current sample id so that we can compare to the
+		 * sample id after reading the data.
+		 */
+		samp_id = ret & EC_MEMMAP_ACC_STATUS_SAMPLE_ID_MASK;
+
+		/* Read all EC data, format it, and store it into data. */
+		ret = cros_ec_sensors_read_data_unsafe(indio_dev, scan_mask,
+						       data);
+		if (ret < 0)
+			return ret;
+
+		/* Read status byte. */
+		ret = cros_ec_sensors_cmd_read_u8(ec, EC_MEMMAP_ACC_STATUS,
+						  &status);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cros_ec_sensors_read_lpc);
+
+int cros_ec_sensors_read_cmd(struct iio_dev *indio_dev,
+			     unsigned long scan_mask, s16 *data)
+{
+	struct cros_ec_sensors_core_state *st = iio_priv(indio_dev);
+	int ret;
+	unsigned int i;
+
+	/* Read all sensor data through a command. */
+	st->param.cmd = MOTIONSENSE_CMD_DATA;
+	ret = cros_ec_motion_send_host_cmd(st, sizeof(st->resp->data));
+	if (ret != 0) {
+		dev_warn(&indio_dev->dev, "Unable to read sensor data\n");
+		return ret;
+	}
+
+	for_each_set_bit(i, &scan_mask, indio_dev->masklength) {
+		*data = st->resp->data.data[i];
+		data++;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cros_ec_sensors_read_cmd);
+
+irqreturn_t cros_ec_sensors_capture(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct cros_ec_sensors_core_state *st = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&st->cmd_lock);
+
+	/* Clear capture data. */
+	memset(st->samples, 0, indio_dev->scan_bytes);
+
+	/* Read data based on which channels are enabled in scan mask. */
+	ret = st->read_ec_sensors_data(indio_dev,
+				       *(indio_dev->active_scan_mask),
+				       (s16 *)st->samples);
+	if (ret < 0)
+		goto done;
+
+	iio_push_to_buffers_with_timestamp(indio_dev, st->samples,
+					   iio_get_time_ns(indio_dev));
+
+done:
+	/*
+	 * Tell the core we are done with this trigger and ready for the
+	 * next one.
+	 */
+	iio_trigger_notify_done(indio_dev->trig);
+
+	mutex_unlock(&st->cmd_lock);
+
+	return IRQ_HANDLED;
+}
+EXPORT_SYMBOL_GPL(cros_ec_sensors_capture);
+
+int cros_ec_sensors_core_read(struct cros_ec_sensors_core_state *st,
+			  struct iio_chan_spec const *chan,
+			  int *val, int *val2, long mask)
+{
+	int ret = IIO_VAL_INT;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		st->param.cmd = MOTIONSENSE_CMD_EC_RATE;
+		st->param.ec_rate.data =
+			EC_MOTION_SENSE_NO_VALUE;
+
+		if (cros_ec_motion_send_host_cmd(st, 0))
+			ret = -EIO;
+		else
+			*val = st->resp->ec_rate.ret;
+		break;
+	case IIO_CHAN_INFO_FREQUENCY:
+		st->param.cmd = MOTIONSENSE_CMD_SENSOR_ODR;
+		st->param.sensor_odr.data =
+			EC_MOTION_SENSE_NO_VALUE;
+
+		if (cros_ec_motion_send_host_cmd(st, 0))
+			ret = -EIO;
+		else
+			*val = st->resp->sensor_odr.ret;
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(cros_ec_sensors_core_read);
+
+int cros_ec_sensors_core_write(struct cros_ec_sensors_core_state *st,
+			       struct iio_chan_spec const *chan,
+			       int val, int val2, long mask)
+{
+	int ret = 0;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_FREQUENCY:
+		st->param.cmd = MOTIONSENSE_CMD_SENSOR_ODR;
+		st->param.sensor_odr.data = val;
+
+		/* Always roundup, so caller gets at least what it asks for. */
+		st->param.sensor_odr.roundup = 1;
+
+		if (cros_ec_motion_send_host_cmd(st, 0))
+			ret = -EIO;
+		break;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		st->param.cmd = MOTIONSENSE_CMD_EC_RATE;
+		st->param.ec_rate.data = val;
+
+		if (cros_ec_motion_send_host_cmd(st, 0))
+			ret = -EIO;
+		else
+			st->curr_sampl_freq = val;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+EXPORT_SYMBOL_GPL(cros_ec_sensors_core_write);
+
+MODULE_DESCRIPTION("ChromeOS EC sensor hub core functions");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.h b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.h
new file mode 100644
index 000000000000..8bc2ca3c2e2e
--- /dev/null
+++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.h
@@ -0,0 +1,175 @@
+/*
+ * ChromeOS EC sensor hub
+ *
+ * Copyright (C) 2016 Google, Inc
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __CROS_EC_SENSORS_CORE_H
+#define __CROS_EC_SENSORS_CORE_H
+
+#include <linux/irqreturn.h>
+
+enum {
+	CROS_EC_SENSOR_X,
+	CROS_EC_SENSOR_Y,
+	CROS_EC_SENSOR_Z,
+	CROS_EC_SENSOR_MAX_AXIS,
+};
+
+/* EC returns sensor values using signed 16 bit registers */
+#define CROS_EC_SENSOR_BITS 16
+
+/*
+ * 4 16 bit channels are allowed.
+ * Good enough for current sensors, they use up to 3 16 bit vectors.
+ */
+#define CROS_EC_SAMPLE_SIZE  (sizeof(s64) * 2)
+
+/* Minimum sampling period to use when device is suspending */
+#define CROS_EC_MIN_SUSPEND_SAMPLING_FREQUENCY 1000  /* 1 second */
+
+/**
+ * struct cros_ec_sensors_core_state - state data for EC sensors IIO driver
+ * @ec:				cros EC device structure
+ * @cmd_lock:			lock used to prevent simultaneous access to the
+ *				commands.
+ * @msg:			cros EC command structure
+ * @param:			motion sensor parameters structure
+ * @resp:			motion sensor response structure
+ * @type:			type of motion sensor
+ * @loc:			location where the motion sensor is placed
+ * @calib:			calibration parameters. Note that trigger
+ *				captured data will always provide the calibrated
+ *				data
+ * @samples:			static array to hold data from a single capture.
+ *				For each channel we need 2 bytes, except for
+ *				the timestamp. The timestamp is always last and
+ *				is always 8-byte aligned.
+ * @read_ec_sensors_data:	function used for accessing sensors values
+ * @cuur_sampl_freq:		current sampling period
+ */
+struct cros_ec_sensors_core_state {
+	struct cros_ec_device *ec;
+	struct mutex cmd_lock;
+
+	struct cros_ec_command *msg;
+	struct ec_params_motion_sense param;
+	struct ec_response_motion_sense *resp;
+
+	enum motionsensor_type type;
+	enum motionsensor_location loc;
+
+	s16 calib[CROS_EC_SENSOR_MAX_AXIS];
+
+	u8 samples[CROS_EC_SAMPLE_SIZE];
+
+	int (*read_ec_sensors_data)(struct iio_dev *indio_dev,
+				    unsigned long scan_mask, s16 *data);
+
+	int curr_sampl_freq;
+};
+
+/**
+ * cros_ec_sensors_read_lpc() - retrieve data from EC shared memory
+ * @indio_dev:	pointer to IIO device
+ * @scan_mask:	bitmap of the sensor indices to scan
+ * @data:	location to store data
+ *
+ * This is the safe function for reading the EC data. It guarantees that the
+ * data sampled was not modified by the EC while being read.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+int cros_ec_sensors_read_lpc(struct iio_dev *indio_dev, unsigned long scan_mask,
+			     s16 *data);
+
+/**
+ * cros_ec_sensors_read_cmd() - retrieve data using the EC command protocol
+ * @indio_dev:	pointer to IIO device
+ * @scan_mask:	bitmap of the sensor indices to scan
+ * @data:	location to store data
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+int cros_ec_sensors_read_cmd(struct iio_dev *indio_dev, unsigned long scan_mask,
+			     s16 *data);
+
+/**
+ * cros_ec_sensors_core_init() - basic initialization of the core structure
+ * @pdev:		platform device created for the sensors
+ * @indio_dev:		iio device structure of the device
+ * @physical_device:	true if the device refers to a physical device
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+int cros_ec_sensors_core_init(struct platform_device *pdev,
+			      struct iio_dev *indio_dev, bool physical_device);
+
+/**
+ * cros_ec_sensors_capture() - the trigger handler function
+ * @irq:	the interrupt number.
+ * @p:		a pointer to the poll function.
+ *
+ * On a trigger event occurring, if the pollfunc is attached then this
+ * handler is called as a threaded interrupt (and hence may sleep). It
+ * is responsible for grabbing data from the device and pushing it into
+ * the associated buffer.
+ *
+ * Return: IRQ_HANDLED
+ */
+irqreturn_t cros_ec_sensors_capture(int irq, void *p);
+
+/**
+ * cros_ec_motion_send_host_cmd() - send motion sense host command
+ * @st:		pointer to state information for device
+ * @opt_length:	optional length to reduce the response size, useful on the data
+ *		path. Otherwise, the maximal allowed response size is used
+ *
+ * When called, the sub-command is assumed to be set in param->cmd.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+int cros_ec_motion_send_host_cmd(struct cros_ec_sensors_core_state *st,
+				 u16 opt_length);
+
+/**
+ * cros_ec_sensors_core_read() - function to request a value from the sensor
+ * @st:		pointer to state information for device
+ * @chan:	channel specification structure table
+ * @val:	will contain one element making up the returned value
+ * @val2:	will contain another element making up the returned value
+ * @mask:	specifies which values to be requested
+ *
+ * Return:	the type of value returned by the device
+ */
+int cros_ec_sensors_core_read(struct cros_ec_sensors_core_state *st,
+			      struct iio_chan_spec const *chan,
+			      int *val, int *val2, long mask);
+
+/**
+ * cros_ec_sensors_core_write() - function to write a value to the sensor
+ * @st:		pointer to state information for device
+ * @chan:	channel specification structure table
+ * @val:	first part of value to write
+ * @val2:	second part of value to write
+ * @mask:	specifies which values to write
+ *
+ * Return:	the type of value returned by the device
+ */
+int cros_ec_sensors_core_write(struct cros_ec_sensors_core_state *st,
+			       struct iio_chan_spec const *chan,
+			       int val, int val2, long mask);
+
+/* List of extended channel specification for all sensors */
+extern const struct iio_chan_spec_ext_info cros_ec_sensors_ext_info[];
+
+#endif  /* __CROS_EC_SENSORS_CORE_H */
diff --git a/drivers/iio/common/hid-sensors/hid-sensor-attributes.c b/drivers/iio/common/hid-sensors/hid-sensor-attributes.c
index b5beea53d6f6..7ef94a90ecf7 100644
--- a/drivers/iio/common/hid-sensors/hid-sensor-attributes.c
+++ b/drivers/iio/common/hid-sensors/hid-sensor-attributes.c
@@ -201,7 +201,7 @@ int hid_sensor_write_samp_freq_value(struct hid_sensor_common *st,
 	int ret;
 
 	if (val1 < 0 || val2 < 0)
-		ret = -EINVAL;
+		return -EINVAL;
 
 	value = val1 * pow_10(6) + val2;
 	if (value) {
@@ -250,6 +250,9 @@ int hid_sensor_write_raw_hyst_value(struct hid_sensor_common *st,
 	s32 value;
 	int ret;
 
+	if (val1 < 0 || val2 < 0)
+		return -EINVAL;
+
 	value = convert_to_vtf_format(st->sensitivity.size,
 				st->sensitivity.unit_expo,
 				val1, val2);
diff --git a/drivers/iio/counter/104-quad-8.c b/drivers/iio/counter/104-quad-8.c
new file mode 100644
index 000000000000..2d2ee353dde7
--- /dev/null
+++ b/drivers/iio/counter/104-quad-8.c
@@ -0,0 +1,593 @@
+/*
+ * IIO driver for the ACCES 104-QUAD-8
+ * Copyright (C) 2016 William Breathitt Gray
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * This driver supports the ACCES 104-QUAD-8 and ACCES 104-QUAD-4.
+ */
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/types.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/isa.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+
+#define QUAD8_EXTENT 32
+
+static unsigned int base[max_num_isa_dev(QUAD8_EXTENT)];
+static unsigned int num_quad8;
+module_param_array(base, uint, &num_quad8, 0);
+MODULE_PARM_DESC(base, "ACCES 104-QUAD-8 base addresses");
+
+#define QUAD8_NUM_COUNTERS 8
+
+/**
+ * struct quad8_iio - IIO device private data structure
+ * @preset:		array of preset values
+ * @count_mode:		array of count mode configurations
+ * @quadrature_mode:	array of quadrature mode configurations
+ * @quadrature_scale:	array of quadrature mode scale configurations
+ * @ab_enable:		array of A and B inputs enable configurations
+ * @preset_enable:	array of set_to_preset_on_index attribute configurations
+ * @synchronous_mode:	array of index function synchronous mode configurations
+ * @index_polarity:	array of index function polarity configurations
+ * @base:		base port address of the IIO device
+ */
+struct quad8_iio {
+	unsigned int preset[QUAD8_NUM_COUNTERS];
+	unsigned int count_mode[QUAD8_NUM_COUNTERS];
+	unsigned int quadrature_mode[QUAD8_NUM_COUNTERS];
+	unsigned int quadrature_scale[QUAD8_NUM_COUNTERS];
+	unsigned int ab_enable[QUAD8_NUM_COUNTERS];
+	unsigned int preset_enable[QUAD8_NUM_COUNTERS];
+	unsigned int synchronous_mode[QUAD8_NUM_COUNTERS];
+	unsigned int index_polarity[QUAD8_NUM_COUNTERS];
+	unsigned int base;
+};
+
+static int quad8_read_raw(struct iio_dev *indio_dev,
+	struct iio_chan_spec const *chan, int *val, int *val2, long mask)
+{
+	struct quad8_iio *const priv = iio_priv(indio_dev);
+	const int base_offset = priv->base + 2 * chan->channel;
+	unsigned int flags;
+	unsigned int borrow;
+	unsigned int carry;
+	int i;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		if (chan->type == IIO_INDEX) {
+			*val = !!(inb(priv->base + 0x16) & BIT(chan->channel));
+			return IIO_VAL_INT;
+		}
+
+		flags = inb(base_offset);
+		borrow = flags & BIT(0);
+		carry = !!(flags & BIT(1));
+
+		/* Borrow XOR Carry effectively doubles count range */
+		*val = (borrow ^ carry) << 24;
+
+		/* Reset Byte Pointer; transfer Counter to Output Latch */
+		outb(0x11, base_offset + 1);
+
+		for (i = 0; i < 3; i++)
+			*val |= (unsigned int)inb(base_offset) << (8 * i);
+
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_ENABLE:
+		*val = priv->ab_enable[chan->channel];
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		*val = 1;
+		*val2 = priv->quadrature_scale[chan->channel];
+		return IIO_VAL_FRACTIONAL_LOG2;
+	}
+
+	return -EINVAL;
+}
+
+static int quad8_write_raw(struct iio_dev *indio_dev,
+	struct iio_chan_spec const *chan, int val, int val2, long mask)
+{
+	struct quad8_iio *const priv = iio_priv(indio_dev);
+	const int base_offset = priv->base + 2 * chan->channel;
+	int i;
+	unsigned int ior_cfg;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		if (chan->type == IIO_INDEX)
+			return -EINVAL;
+
+		/* Only 24-bit values are supported */
+		if ((unsigned int)val > 0xFFFFFF)
+			return -EINVAL;
+
+		/* Reset Byte Pointer */
+		outb(0x01, base_offset + 1);
+
+		/* Counter can only be set via Preset Register */
+		for (i = 0; i < 3; i++)
+			outb(val >> (8 * i), base_offset);
+
+		/* Transfer Preset Register to Counter */
+		outb(0x08, base_offset + 1);
+
+		/* Reset Byte Pointer */
+		outb(0x01, base_offset + 1);
+
+		/* Set Preset Register back to original value */
+		val = priv->preset[chan->channel];
+		for (i = 0; i < 3; i++)
+			outb(val >> (8 * i), base_offset);
+
+		/* Reset Borrow, Carry, Compare, and Sign flags */
+		outb(0x02, base_offset + 1);
+		/* Reset Error flag */
+		outb(0x06, base_offset + 1);
+
+		return 0;
+	case IIO_CHAN_INFO_ENABLE:
+		/* only boolean values accepted */
+		if (val < 0 || val > 1)
+			return -EINVAL;
+
+		priv->ab_enable[chan->channel] = val;
+
+		ior_cfg = val | priv->preset_enable[chan->channel] << 1;
+
+		/* Load I/O control configuration */
+		outb(0x40 | ior_cfg, base_offset);
+
+		return 0;
+	case IIO_CHAN_INFO_SCALE:
+		/* Quadrature scaling only available in quadrature mode */
+		if (!priv->quadrature_mode[chan->channel] && (val2 || val != 1))
+			return -EINVAL;
+
+		/* Only three gain states (1, 0.5, 0.25) */
+		if (val == 1 && !val2)
+			priv->quadrature_scale[chan->channel] = 0;
+		else if (!val)
+			switch (val2) {
+			case 500000:
+				priv->quadrature_scale[chan->channel] = 1;
+				break;
+			case 250000:
+				priv->quadrature_scale[chan->channel] = 2;
+				break;
+			default:
+				return -EINVAL;
+			}
+		else
+			return -EINVAL;
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static const struct iio_info quad8_info = {
+	.driver_module = THIS_MODULE,
+	.read_raw = quad8_read_raw,
+	.write_raw = quad8_write_raw
+};
+
+static ssize_t quad8_read_preset(struct iio_dev *indio_dev, uintptr_t private,
+	const struct iio_chan_spec *chan, char *buf)
+{
+	const struct quad8_iio *const priv = iio_priv(indio_dev);
+
+	return snprintf(buf, PAGE_SIZE, "%u\n", priv->preset[chan->channel]);
+}
+
+static ssize_t quad8_write_preset(struct iio_dev *indio_dev, uintptr_t private,
+	const struct iio_chan_spec *chan, const char *buf, size_t len)
+{
+	struct quad8_iio *const priv = iio_priv(indio_dev);
+	const int base_offset = priv->base + 2 * chan->channel;
+	unsigned int preset;
+	int ret;
+	int i;
+
+	ret = kstrtouint(buf, 0, &preset);
+	if (ret)
+		return ret;
+
+	/* Only 24-bit values are supported */
+	if (preset > 0xFFFFFF)
+		return -EINVAL;
+
+	priv->preset[chan->channel] = preset;
+
+	/* Reset Byte Pointer */
+	outb(0x01, base_offset + 1);
+
+	/* Set Preset Register */
+	for (i = 0; i < 3; i++)
+		outb(preset >> (8 * i), base_offset);
+
+	return len;
+}
+
+static ssize_t quad8_read_set_to_preset_on_index(struct iio_dev *indio_dev,
+	uintptr_t private, const struct iio_chan_spec *chan, char *buf)
+{
+	const struct quad8_iio *const priv = iio_priv(indio_dev);
+
+	return snprintf(buf, PAGE_SIZE, "%u\n",
+		priv->preset_enable[chan->channel]);
+}
+
+static ssize_t quad8_write_set_to_preset_on_index(struct iio_dev *indio_dev,
+	uintptr_t private, const struct iio_chan_spec *chan, const char *buf,
+	size_t len)
+{
+	struct quad8_iio *const priv = iio_priv(indio_dev);
+	const int base_offset = priv->base + 2 * chan->channel;
+	bool preset_enable;
+	int ret;
+	unsigned int ior_cfg;
+
+	ret = kstrtobool(buf, &preset_enable);
+	if (ret)
+		return ret;
+
+	priv->preset_enable[chan->channel] = preset_enable;
+
+	ior_cfg = priv->ab_enable[chan->channel] |
+		(unsigned int)preset_enable << 1;
+
+	/* Load I/O control configuration to Input / Output Control Register */
+	outb(0x40 | ior_cfg, base_offset);
+
+	return len;
+}
+
+static const char *const quad8_noise_error_states[] = {
+	"No excessive noise is present at the count inputs",
+	"Excessive noise is present at the count inputs"
+};
+
+static int quad8_get_noise_error(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan)
+{
+	struct quad8_iio *const priv = iio_priv(indio_dev);
+	const int base_offset = priv->base + 2 * chan->channel + 1;
+
+	return !!(inb(base_offset) & BIT(4));
+}
+
+static const struct iio_enum quad8_noise_error_enum = {
+	.items = quad8_noise_error_states,
+	.num_items = ARRAY_SIZE(quad8_noise_error_states),
+	.get = quad8_get_noise_error
+};
+
+static const char *const quad8_count_direction_states[] = {
+	"down",
+	"up"
+};
+
+static int quad8_get_count_direction(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan)
+{
+	struct quad8_iio *const priv = iio_priv(indio_dev);
+	const int base_offset = priv->base + 2 * chan->channel + 1;
+
+	return !!(inb(base_offset) & BIT(5));
+}
+
+static const struct iio_enum quad8_count_direction_enum = {
+	.items = quad8_count_direction_states,
+	.num_items = ARRAY_SIZE(quad8_count_direction_states),
+	.get = quad8_get_count_direction
+};
+
+static const char *const quad8_count_modes[] = {
+	"normal",
+	"range limit",
+	"non-recycle",
+	"modulo-n"
+};
+
+static int quad8_set_count_mode(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan, unsigned int count_mode)
+{
+	struct quad8_iio *const priv = iio_priv(indio_dev);
+	unsigned int mode_cfg = count_mode << 1;
+	const int base_offset = priv->base + 2 * chan->channel + 1;
+
+	priv->count_mode[chan->channel] = count_mode;
+
+	/* Add quadrature mode configuration */
+	if (priv->quadrature_mode[chan->channel])
+		mode_cfg |= (priv->quadrature_scale[chan->channel] + 1) << 3;
+
+	/* Load mode configuration to Counter Mode Register */
+	outb(0x20 | mode_cfg, base_offset);
+
+	return 0;
+}
+
+static int quad8_get_count_mode(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan)
+{
+	const struct quad8_iio *const priv = iio_priv(indio_dev);
+
+	return priv->count_mode[chan->channel];
+}
+
+static const struct iio_enum quad8_count_mode_enum = {
+	.items = quad8_count_modes,
+	.num_items = ARRAY_SIZE(quad8_count_modes),
+	.set = quad8_set_count_mode,
+	.get = quad8_get_count_mode
+};
+
+static const char *const quad8_synchronous_modes[] = {
+	"non-synchronous",
+	"synchronous"
+};
+
+static int quad8_set_synchronous_mode(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan, unsigned int synchronous_mode)
+{
+	struct quad8_iio *const priv = iio_priv(indio_dev);
+	const unsigned int idr_cfg = synchronous_mode |
+		priv->index_polarity[chan->channel] << 1;
+	const int base_offset = priv->base + 2 * chan->channel + 1;
+
+	/* Index function must be non-synchronous in non-quadrature mode */
+	if (synchronous_mode && !priv->quadrature_mode[chan->channel])
+		return -EINVAL;
+
+	priv->synchronous_mode[chan->channel] = synchronous_mode;
+
+	/* Load Index Control configuration to Index Control Register */
+	outb(0x40 | idr_cfg, base_offset);
+
+	return 0;
+}
+
+static int quad8_get_synchronous_mode(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan)
+{
+	const struct quad8_iio *const priv = iio_priv(indio_dev);
+
+	return priv->synchronous_mode[chan->channel];
+}
+
+static const struct iio_enum quad8_synchronous_mode_enum = {
+	.items = quad8_synchronous_modes,
+	.num_items = ARRAY_SIZE(quad8_synchronous_modes),
+	.set = quad8_set_synchronous_mode,
+	.get = quad8_get_synchronous_mode
+};
+
+static const char *const quad8_quadrature_modes[] = {
+	"non-quadrature",
+	"quadrature"
+};
+
+static int quad8_set_quadrature_mode(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan, unsigned int quadrature_mode)
+{
+	struct quad8_iio *const priv = iio_priv(indio_dev);
+	unsigned int mode_cfg = priv->count_mode[chan->channel] << 1;
+	const int base_offset = priv->base + 2 * chan->channel + 1;
+
+	if (quadrature_mode)
+		mode_cfg |= (priv->quadrature_scale[chan->channel] + 1) << 3;
+	else {
+		/* Quadrature scaling only available in quadrature mode */
+		priv->quadrature_scale[chan->channel] = 0;
+
+		/* Synchronous function not supported in non-quadrature mode */
+		if (priv->synchronous_mode[chan->channel])
+			quad8_set_synchronous_mode(indio_dev, chan, 0);
+	}
+
+	priv->quadrature_mode[chan->channel] = quadrature_mode;
+
+	/* Load mode configuration to Counter Mode Register */
+	outb(0x20 | mode_cfg, base_offset);
+
+	return 0;
+}
+
+static int quad8_get_quadrature_mode(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan)
+{
+	const struct quad8_iio *const priv = iio_priv(indio_dev);
+
+	return priv->quadrature_mode[chan->channel];
+}
+
+static const struct iio_enum quad8_quadrature_mode_enum = {
+	.items = quad8_quadrature_modes,
+	.num_items = ARRAY_SIZE(quad8_quadrature_modes),
+	.set = quad8_set_quadrature_mode,
+	.get = quad8_get_quadrature_mode
+};
+
+static const char *const quad8_index_polarity_modes[] = {
+	"negative",
+	"positive"
+};
+
+static int quad8_set_index_polarity(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan, unsigned int index_polarity)
+{
+	struct quad8_iio *const priv = iio_priv(indio_dev);
+	const unsigned int idr_cfg = priv->synchronous_mode[chan->channel] |
+		index_polarity << 1;
+	const int base_offset = priv->base + 2 * chan->channel + 1;
+
+	priv->index_polarity[chan->channel] = index_polarity;
+
+	/* Load Index Control configuration to Index Control Register */
+	outb(0x40 | idr_cfg, base_offset);
+
+	return 0;
+}
+
+static int quad8_get_index_polarity(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan)
+{
+	const struct quad8_iio *const priv = iio_priv(indio_dev);
+
+	return priv->index_polarity[chan->channel];
+}
+
+static const struct iio_enum quad8_index_polarity_enum = {
+	.items = quad8_index_polarity_modes,
+	.num_items = ARRAY_SIZE(quad8_index_polarity_modes),
+	.set = quad8_set_index_polarity,
+	.get = quad8_get_index_polarity
+};
+
+static const struct iio_chan_spec_ext_info quad8_count_ext_info[] = {
+	{
+		.name = "preset",
+		.shared = IIO_SEPARATE,
+		.read = quad8_read_preset,
+		.write = quad8_write_preset
+	},
+	{
+		.name = "set_to_preset_on_index",
+		.shared = IIO_SEPARATE,
+		.read = quad8_read_set_to_preset_on_index,
+		.write = quad8_write_set_to_preset_on_index
+	},
+	IIO_ENUM("noise_error", IIO_SEPARATE, &quad8_noise_error_enum),
+	IIO_ENUM_AVAILABLE("noise_error", &quad8_noise_error_enum),
+	IIO_ENUM("count_direction", IIO_SEPARATE, &quad8_count_direction_enum),
+	IIO_ENUM_AVAILABLE("count_direction", &quad8_count_direction_enum),
+	IIO_ENUM("count_mode", IIO_SEPARATE, &quad8_count_mode_enum),
+	IIO_ENUM_AVAILABLE("count_mode", &quad8_count_mode_enum),
+	IIO_ENUM("quadrature_mode", IIO_SEPARATE, &quad8_quadrature_mode_enum),
+	IIO_ENUM_AVAILABLE("quadrature_mode", &quad8_quadrature_mode_enum),
+	{}
+};
+
+static const struct iio_chan_spec_ext_info quad8_index_ext_info[] = {
+	IIO_ENUM("synchronous_mode", IIO_SEPARATE,
+		&quad8_synchronous_mode_enum),
+	IIO_ENUM_AVAILABLE("synchronous_mode", &quad8_synchronous_mode_enum),
+	IIO_ENUM("index_polarity", IIO_SEPARATE, &quad8_index_polarity_enum),
+	IIO_ENUM_AVAILABLE("index_polarity", &quad8_index_polarity_enum),
+	{}
+};
+
+#define QUAD8_COUNT_CHAN(_chan) {					\
+	.type = IIO_COUNT,						\
+	.channel = (_chan),						\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |			\
+		BIT(IIO_CHAN_INFO_ENABLE) | BIT(IIO_CHAN_INFO_SCALE),	\
+	.ext_info = quad8_count_ext_info,				\
+	.indexed = 1							\
+}
+
+#define QUAD8_INDEX_CHAN(_chan) {			\
+	.type = IIO_INDEX,				\
+	.channel = (_chan),				\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
+	.ext_info = quad8_index_ext_info,		\
+	.indexed = 1					\
+}
+
+static const struct iio_chan_spec quad8_channels[] = {
+	QUAD8_COUNT_CHAN(0), QUAD8_INDEX_CHAN(0),
+	QUAD8_COUNT_CHAN(1), QUAD8_INDEX_CHAN(1),
+	QUAD8_COUNT_CHAN(2), QUAD8_INDEX_CHAN(2),
+	QUAD8_COUNT_CHAN(3), QUAD8_INDEX_CHAN(3),
+	QUAD8_COUNT_CHAN(4), QUAD8_INDEX_CHAN(4),
+	QUAD8_COUNT_CHAN(5), QUAD8_INDEX_CHAN(5),
+	QUAD8_COUNT_CHAN(6), QUAD8_INDEX_CHAN(6),
+	QUAD8_COUNT_CHAN(7), QUAD8_INDEX_CHAN(7)
+};
+
+static int quad8_probe(struct device *dev, unsigned int id)
+{
+	struct iio_dev *indio_dev;
+	struct quad8_iio *priv;
+	int i, j;
+	unsigned int base_offset;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*priv));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	if (!devm_request_region(dev, base[id], QUAD8_EXTENT,
+		dev_name(dev))) {
+		dev_err(dev, "Unable to lock port addresses (0x%X-0x%X)\n",
+			base[id], base[id] + QUAD8_EXTENT);
+		return -EBUSY;
+	}
+
+	indio_dev->info = &quad8_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->num_channels = ARRAY_SIZE(quad8_channels);
+	indio_dev->channels = quad8_channels;
+	indio_dev->name = dev_name(dev);
+
+	priv = iio_priv(indio_dev);
+	priv->base = base[id];
+
+	/* Reset all counters and disable interrupt function */
+	outb(0x01, base[id] + 0x11);
+	/* Set initial configuration for all counters */
+	for (i = 0; i < QUAD8_NUM_COUNTERS; i++) {
+		base_offset = base[id] + 2 * i;
+		/* Reset Byte Pointer */
+		outb(0x01, base_offset + 1);
+		/* Reset Preset Register */
+		for (j = 0; j < 3; j++)
+			outb(0x00, base_offset);
+		/* Reset Borrow, Carry, Compare, and Sign flags */
+		outb(0x04, base_offset + 1);
+		/* Reset Error flag */
+		outb(0x06, base_offset + 1);
+		/* Binary encoding; Normal count; non-quadrature mode */
+		outb(0x20, base_offset + 1);
+		/* Disable A and B inputs; preset on index; FLG1 as Carry */
+		outb(0x40, base_offset + 1);
+		/* Disable index function; negative index polarity */
+		outb(0x60, base_offset + 1);
+	}
+	/* Enable all counters */
+	outb(0x00, base[id] + 0x11);
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static struct isa_driver quad8_driver = {
+	.probe = quad8_probe,
+	.driver = {
+		.name = "104-quad-8"
+	}
+};
+
+module_isa_driver(quad8_driver, num_quad8);
+
+MODULE_AUTHOR("William Breathitt Gray <vilhelm.gray@gmail.com>");
+MODULE_DESCRIPTION("ACCES 104-QUAD-8 IIO driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/counter/Kconfig b/drivers/iio/counter/Kconfig
new file mode 100644
index 000000000000..44627f6e4861
--- /dev/null
+++ b/drivers/iio/counter/Kconfig
@@ -0,0 +1,24 @@
+#
+# Counter devices
+#
+# When adding new entries keep the list in alphabetical order
+
+menu "Counters"
+
+config 104_QUAD_8
+	tristate "ACCES 104-QUAD-8 driver"
+	depends on X86 && ISA_BUS_API
+	help
+	  Say yes here to build support for the ACCES 104-QUAD-8 quadrature
+	  encoder counter/interface device family (104-QUAD-8, 104-QUAD-4).
+
+	  Performing a write to a counter's IIO_CHAN_INFO_RAW sets the counter and
+	  also clears the counter's respective error flag. Although the counters
+	  have a 25-bit range, only the lower 24 bits may be set, either directly
+	  or via a counter's preset attribute. Interrupts are not supported by
+	  this driver.
+
+	  The base port addresses for the devices may be configured via the base
+	  array module parameter.
+
+endmenu
diff --git a/drivers/iio/counter/Makefile b/drivers/iio/counter/Makefile
new file mode 100644
index 000000000000..007e88411648
--- /dev/null
+++ b/drivers/iio/counter/Makefile
@@ -0,0 +1,7 @@
+#
+# Makefile for IIO counter devices
+#
+
+# When adding new entries keep the list in alphabetical order
+
+obj-$(CONFIG_104_QUAD_8)	+= 104-quad-8.o
diff --git a/drivers/iio/dac/Kconfig b/drivers/iio/dac/Kconfig
index 120b24478469..d3084028905b 100644
--- a/drivers/iio/dac/Kconfig
+++ b/drivers/iio/dac/Kconfig
@@ -200,6 +200,16 @@ config AD8801
 	  To compile this driver as a module choose M here: the module will be called
 	  ad8801.
 
+config DPOT_DAC
+	tristate "DAC emulation using a DPOT"
+	depends on OF
+	help
+	  Say yes here to build support for DAC emulation using a digital
+	  potentiometer.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called dpot-dac.
+
 config LPC18XX_DAC
 	tristate "NXP LPC18xx DAC driver"
 	depends on ARCH_LPC18XX || COMPILE_TEST
diff --git a/drivers/iio/dac/Makefile b/drivers/iio/dac/Makefile
index 27642bbf75f2..f01bf4a99867 100644
--- a/drivers/iio/dac/Makefile
+++ b/drivers/iio/dac/Makefile
@@ -22,6 +22,7 @@ obj-$(CONFIG_AD5686) += ad5686.o
 obj-$(CONFIG_AD7303) += ad7303.o
 obj-$(CONFIG_AD8801) += ad8801.o
 obj-$(CONFIG_CIO_DAC) += cio-dac.o
+obj-$(CONFIG_DPOT_DAC) += dpot-dac.o
 obj-$(CONFIG_LPC18XX_DAC) += lpc18xx_dac.o
 obj-$(CONFIG_M62332) += m62332.o
 obj-$(CONFIG_MAX517) += max517.o
diff --git a/drivers/iio/dac/ad5592r.c b/drivers/iio/dac/ad5592r.c
index 0b235a2c7359..6eed5b7729be 100644
--- a/drivers/iio/dac/ad5592r.c
+++ b/drivers/iio/dac/ad5592r.c
@@ -17,7 +17,7 @@
 #define AD5592R_GPIO_READBACK_EN	BIT(10)
 #define AD5592R_LDAC_READBACK_EN	BIT(6)
 
-static int ad5592r_spi_wnop_r16(struct ad5592r_state *st, u16 *buf)
+static int ad5592r_spi_wnop_r16(struct ad5592r_state *st, __be16 *buf)
 {
 	struct spi_device *spi = container_of(st->dev, struct spi_device, dev);
 	struct spi_transfer t = {
diff --git a/drivers/iio/dac/dpot-dac.c b/drivers/iio/dac/dpot-dac.c
new file mode 100644
index 000000000000..960a2b430480
--- /dev/null
+++ b/drivers/iio/dac/dpot-dac.c
@@ -0,0 +1,266 @@
+/*
+ * IIO DAC emulation driver using a digital potentiometer
+ *
+ * Copyright (C) 2016 Axentia Technologies AB
+ *
+ * Author: Peter Rosin <peda@axentia.se>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/*
+ * It is assumed that the dpot is used as a voltage divider between the
+ * current dpot wiper setting and the maximum resistance of the dpot. The
+ * divided voltage is provided by a vref regulator.
+ *
+ *                   .------.
+ *    .-----------.  |      |
+ *    | vref      |--'    .---.
+ *    | regulator |--.    |   |
+ *    '-----------'  |    | d |
+ *                   |    | p |
+ *                   |    | o |  wiper
+ *                   |    | t |<---------+
+ *                   |    |   |
+ *                   |    '---'       dac output voltage
+ *                   |      |
+ *                   '------+------------+
+ */
+
+#include <linux/err.h>
+#include <linux/iio/consumer.h>
+#include <linux/iio/iio.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+
+struct dpot_dac {
+	struct regulator *vref;
+	struct iio_channel *dpot;
+	u32 max_ohms;
+};
+
+static const struct iio_chan_spec dpot_dac_iio_channel = {
+	.type = IIO_VOLTAGE,
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW)
+			    | BIT(IIO_CHAN_INFO_SCALE),
+	.info_mask_separate_available = BIT(IIO_CHAN_INFO_RAW),
+	.output = 1,
+	.indexed = 1,
+};
+
+static int dpot_dac_read_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int *val, int *val2, long mask)
+{
+	struct dpot_dac *dac = iio_priv(indio_dev);
+	int ret;
+	unsigned long long tmp;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		return iio_read_channel_raw(dac->dpot, val);
+
+	case IIO_CHAN_INFO_SCALE:
+		ret = iio_read_channel_scale(dac->dpot, val, val2);
+		switch (ret) {
+		case IIO_VAL_FRACTIONAL_LOG2:
+			tmp = *val * 1000000000LL;
+			do_div(tmp, dac->max_ohms);
+			tmp *= regulator_get_voltage(dac->vref) / 1000;
+			do_div(tmp, 1000000000LL);
+			*val = tmp;
+			return ret;
+		case IIO_VAL_INT:
+			/*
+			 * Convert integer scale to fractional scale by
+			 * setting the denominator (val2) to one...
+			 */
+			*val2 = 1;
+			ret = IIO_VAL_FRACTIONAL;
+			/* ...and fall through. */
+		case IIO_VAL_FRACTIONAL:
+			*val *= regulator_get_voltage(dac->vref) / 1000;
+			*val2 *= dac->max_ohms;
+			break;
+		}
+
+		return ret;
+	}
+
+	return -EINVAL;
+}
+
+static int dpot_dac_read_avail(struct iio_dev *indio_dev,
+			       struct iio_chan_spec const *chan,
+			       const int **vals, int *type, int *length,
+			       long mask)
+{
+	struct dpot_dac *dac = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		*type = IIO_VAL_INT;
+		return iio_read_avail_channel_raw(dac->dpot, vals, length);
+	}
+
+	return -EINVAL;
+}
+
+static int dpot_dac_write_raw(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      int val, int val2, long mask)
+{
+	struct dpot_dac *dac = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		return iio_write_channel_raw(dac->dpot, val);
+	}
+
+	return -EINVAL;
+}
+
+static const struct iio_info dpot_dac_info = {
+	.read_raw = dpot_dac_read_raw,
+	.read_avail = dpot_dac_read_avail,
+	.write_raw = dpot_dac_write_raw,
+	.driver_module = THIS_MODULE,
+};
+
+static int dpot_dac_channel_max_ohms(struct iio_dev *indio_dev)
+{
+	struct device *dev = &indio_dev->dev;
+	struct dpot_dac *dac = iio_priv(indio_dev);
+	unsigned long long tmp;
+	int ret;
+	int val;
+	int val2;
+	int max;
+
+	ret = iio_read_max_channel_raw(dac->dpot, &max);
+	if (ret < 0) {
+		dev_err(dev, "dpot does not indicate its raw maximum value\n");
+		return ret;
+	}
+
+	switch (iio_read_channel_scale(dac->dpot, &val, &val2)) {
+	case IIO_VAL_INT:
+		return max * val;
+	case IIO_VAL_FRACTIONAL:
+		tmp = (unsigned long long)max * val;
+		do_div(tmp, val2);
+		return tmp;
+	case IIO_VAL_FRACTIONAL_LOG2:
+		tmp = val * 1000000000LL * max >> val2;
+		do_div(tmp, 1000000000LL);
+		return tmp;
+	default:
+		dev_err(dev, "dpot has a scale that is too weird\n");
+	}
+
+	return -EINVAL;
+}
+
+static int dpot_dac_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct iio_dev *indio_dev;
+	struct dpot_dac *dac;
+	enum iio_chan_type type;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*dac));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, indio_dev);
+	dac = iio_priv(indio_dev);
+
+	indio_dev->name = dev_name(dev);
+	indio_dev->dev.parent = dev;
+	indio_dev->info = &dpot_dac_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = &dpot_dac_iio_channel;
+	indio_dev->num_channels = 1;
+
+	dac->vref = devm_regulator_get(dev, "vref");
+	if (IS_ERR(dac->vref)) {
+		if (PTR_ERR(dac->vref) != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "failed to get vref regulator\n");
+		return PTR_ERR(dac->vref);
+	}
+
+	dac->dpot = devm_iio_channel_get(dev, "dpot");
+	if (IS_ERR(dac->dpot)) {
+		if (PTR_ERR(dac->dpot) != -EPROBE_DEFER)
+			dev_err(dev, "failed to get dpot input channel\n");
+		return PTR_ERR(dac->dpot);
+	}
+
+	ret = iio_get_channel_type(dac->dpot, &type);
+	if (ret < 0)
+		return ret;
+
+	if (type != IIO_RESISTANCE) {
+		dev_err(dev, "dpot is of the wrong type\n");
+		return -EINVAL;
+	}
+
+	ret = dpot_dac_channel_max_ohms(indio_dev);
+	if (ret < 0)
+		return ret;
+	dac->max_ohms = ret;
+
+	ret = regulator_enable(dac->vref);
+	if (ret) {
+		dev_err(dev, "failed to enable the vref regulator\n");
+		return ret;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(dev, "failed to register iio device\n");
+		goto disable_reg;
+	}
+
+	return 0;
+
+disable_reg:
+	regulator_disable(dac->vref);
+	return ret;
+}
+
+static int dpot_dac_remove(struct platform_device *pdev)
+{
+	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct dpot_dac *dac = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+	regulator_disable(dac->vref);
+
+	return 0;
+}
+
+static const struct of_device_id dpot_dac_match[] = {
+	{ .compatible = "dpot-dac" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, dpot_dac_match);
+
+static struct platform_driver dpot_dac_driver = {
+	.probe = dpot_dac_probe,
+	.remove = dpot_dac_remove,
+	.driver = {
+		.name = "iio-dpot-dac",
+		.of_match_table = dpot_dac_match,
+	},
+};
+module_platform_driver(dpot_dac_driver);
+
+MODULE_DESCRIPTION("DAC emulation driver using a digital potentiometer");
+MODULE_AUTHOR("Peter Rosin <peda@axentia.se>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/dac/mcp4725.c b/drivers/iio/dac/mcp4725.c
index cca935c06f2b..db109f0cdd8c 100644
--- a/drivers/iio/dac/mcp4725.c
+++ b/drivers/iio/dac/mcp4725.c
@@ -18,6 +18,8 @@
 #include <linux/i2c.h>
 #include <linux/err.h>
 #include <linux/delay.h>
+#include <linux/regulator/consumer.h>
+#include <linux/of.h>
 
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
@@ -26,12 +28,20 @@
 
 #define MCP4725_DRV_NAME "mcp4725"
 
+#define MCP472X_REF_VDD			0x00
+#define MCP472X_REF_VREF_UNBUFFERED	0x02
+#define MCP472X_REF_VREF_BUFFERED	0x03
+
 struct mcp4725_data {
 	struct i2c_client *client;
-	u16 vref_mv;
+	int id;
+	unsigned ref_mode;
+	bool vref_buffered;
 	u16 dac_value;
 	bool powerdown;
 	unsigned powerdown_mode;
+	struct regulator *vdd_reg;
+	struct regulator *vref_reg;
 };
 
 static int mcp4725_suspend(struct device *dev)
@@ -86,6 +96,7 @@ static ssize_t mcp4725_store_eeprom(struct device *dev,
 		return 0;
 
 	inoutbuf[0] = 0x60; /* write EEPROM */
+	inoutbuf[0] |= data->ref_mode << 3;
 	inoutbuf[1] = data->dac_value >> 4;
 	inoutbuf[2] = (data->dac_value & 0xf) << 4;
 
@@ -278,18 +289,49 @@ static int mcp4725_set_value(struct iio_dev *indio_dev, int val)
 		return 0;
 }
 
+static int mcp4726_set_cfg(struct iio_dev *indio_dev)
+{
+	struct mcp4725_data *data = iio_priv(indio_dev);
+	u8 outbuf[3];
+	int ret;
+
+	outbuf[0] = 0x40;
+	outbuf[0] |= data->ref_mode << 3;
+	if (data->powerdown)
+		outbuf[0] |= data->powerdown << 1;
+	outbuf[1] = data->dac_value >> 4;
+	outbuf[2] = (data->dac_value & 0xf) << 4;
+
+	ret = i2c_master_send(data->client, outbuf, 3);
+	if (ret < 0)
+		return ret;
+	else if (ret != 3)
+		return -EIO;
+	else
+		return 0;
+}
+
 static int mcp4725_read_raw(struct iio_dev *indio_dev,
 			   struct iio_chan_spec const *chan,
 			   int *val, int *val2, long mask)
 {
 	struct mcp4725_data *data = iio_priv(indio_dev);
+	int ret;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
 		*val = data->dac_value;
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_SCALE:
-		*val = data->vref_mv;
+		if (data->ref_mode == MCP472X_REF_VDD)
+			ret = regulator_get_voltage(data->vdd_reg);
+		else
+			ret = regulator_get_voltage(data->vref_reg);
+
+		if (ret < 0)
+			return ret;
+
+		*val = ret / 1000;
 		*val2 = 12;
 		return IIO_VAL_FRACTIONAL_LOG2;
 	}
@@ -323,27 +365,98 @@ static const struct iio_info mcp4725_info = {
 	.driver_module = THIS_MODULE,
 };
 
+#ifdef CONFIG_OF
+static int mcp4725_probe_dt(struct device *dev,
+			    struct mcp4725_platform_data *pdata)
+{
+	struct device_node *np = dev->of_node;
+
+	if (!np)
+		return -ENODEV;
+
+	/* check if is the vref-supply defined */
+	pdata->use_vref = of_property_read_bool(np, "vref-supply");
+	pdata->vref_buffered =
+		of_property_read_bool(np, "microchip,vref-buffered");
+
+	return 0;
+}
+#else
+static int mcp4725_probe_dt(struct device *dev,
+			    struct mcp4725_platform_data *platform_data)
+{
+	return -ENODEV;
+}
+#endif
+
 static int mcp4725_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
 	struct mcp4725_data *data;
 	struct iio_dev *indio_dev;
-	struct mcp4725_platform_data *platform_data = client->dev.platform_data;
-	u8 inbuf[3];
+	struct mcp4725_platform_data *pdata, pdata_dt;
+	u8 inbuf[4];
 	u8 pd;
+	u8 ref;
 	int err;
 
-	if (!platform_data || !platform_data->vref_mv) {
-		dev_err(&client->dev, "invalid platform data");
-		return -EINVAL;
-	}
-
 	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
 	if (indio_dev == NULL)
 		return -ENOMEM;
 	data = iio_priv(indio_dev);
 	i2c_set_clientdata(client, indio_dev);
 	data->client = client;
+	data->id = id->driver_data;
+	pdata = dev_get_platdata(&client->dev);
+
+	if (!pdata) {
+		err = mcp4725_probe_dt(&client->dev, &pdata_dt);
+		if (err) {
+			dev_err(&client->dev,
+				"invalid platform or devicetree data");
+			return err;
+		}
+		pdata = &pdata_dt;
+	}
+
+	if (data->id == MCP4725 && pdata->use_vref) {
+		dev_err(&client->dev,
+			"external reference is unavailable on MCP4725");
+		return -EINVAL;
+	}
+
+	if (!pdata->use_vref && pdata->vref_buffered) {
+		dev_err(&client->dev,
+			"buffering is unavailable on the internal reference");
+		return -EINVAL;
+	}
+
+	if (!pdata->use_vref)
+		data->ref_mode = MCP472X_REF_VDD;
+	else
+		data->ref_mode = pdata->vref_buffered ?
+			MCP472X_REF_VREF_BUFFERED :
+			MCP472X_REF_VREF_UNBUFFERED;
+
+	data->vdd_reg = devm_regulator_get(&client->dev, "vdd");
+	if (IS_ERR(data->vdd_reg))
+		return PTR_ERR(data->vdd_reg);
+
+	err = regulator_enable(data->vdd_reg);
+	if (err)
+		return err;
+
+	if (pdata->use_vref) {
+		data->vref_reg = devm_regulator_get(&client->dev, "vref");
+		if (IS_ERR(data->vref_reg)) {
+			err = PTR_ERR(data->vref_reg);
+			goto err_disable_vdd_reg;
+		}
+
+		err = regulator_enable(data->vref_reg);
+		if (err)
+			goto err_disable_vdd_reg;
+	}
 
 	indio_dev->dev.parent = &client->dev;
 	indio_dev->name = id->name;
@@ -352,25 +465,56 @@ static int mcp4725_probe(struct i2c_client *client,
 	indio_dev->num_channels = 1;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	data->vref_mv = platform_data->vref_mv;
+	/* read current DAC value and settings */
+	err = i2c_master_recv(client, inbuf, data->id == MCP4725 ? 3 : 4);
 
-	/* read current DAC value */
-	err = i2c_master_recv(client, inbuf, 3);
 	if (err < 0) {
 		dev_err(&client->dev, "failed to read DAC value");
-		return err;
+		goto err_disable_vref_reg;
 	}
 	pd = (inbuf[0] >> 1) & 0x3;
 	data->powerdown = pd > 0 ? true : false;
-	data->powerdown_mode = pd ? pd - 1 : 2; /* largest register to gnd */
+	data->powerdown_mode = pd ? pd - 1 : 2; /* largest resistor to gnd */
 	data->dac_value = (inbuf[1] << 4) | (inbuf[2] >> 4);
+	if (data->id == MCP4726)
+		ref = (inbuf[3] >> 3) & 0x3;
+
+	if (data->id == MCP4726 && ref != data->ref_mode) {
+		dev_info(&client->dev,
+			"voltage reference mode differs (conf: %u, eeprom: %u), setting %u",
+			data->ref_mode, ref, data->ref_mode);
+		err = mcp4726_set_cfg(indio_dev);
+		if (err < 0)
+			goto err_disable_vref_reg;
+	}
+ 
+	err = iio_device_register(indio_dev);
+	if (err)
+		goto err_disable_vref_reg;
+
+	return 0;
+
+err_disable_vref_reg:
+	if (data->vref_reg)
+		regulator_disable(data->vref_reg);
 
-	return iio_device_register(indio_dev);
+err_disable_vdd_reg:
+	regulator_disable(data->vdd_reg);
+
+	return err;
 }
 
 static int mcp4725_remove(struct i2c_client *client)
 {
-	iio_device_unregister(i2c_get_clientdata(client));
+	struct iio_dev *indio_dev = i2c_get_clientdata(client);
+	struct mcp4725_data *data = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+
+	if (data->vref_reg)
+		regulator_disable(data->vref_reg);
+	regulator_disable(data->vdd_reg);
+
 	return 0;
 }
 
diff --git a/drivers/iio/gyro/Kconfig b/drivers/iio/gyro/Kconfig
index 205a84420ae9..3126cf05e6b9 100644
--- a/drivers/iio/gyro/Kconfig
+++ b/drivers/iio/gyro/Kconfig
@@ -84,6 +84,24 @@ config HID_SENSOR_GYRO_3D
 	  Say yes here to build support for the HID SENSOR
 	  Gyroscope 3D.
 
+config MPU3050
+	tristate
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	select REGMAP
+
+config MPU3050_I2C
+	tristate "Invensense MPU3050 devices on I2C"
+	depends on !(INPUT_MPU3050=y || INPUT_MPU3050=m)
+	depends on I2C
+	select MPU3050
+	select REGMAP_I2C
+	select I2C_MUX
+	help
+	  This driver supports the Invensense MPU3050 gyroscope over I2C.
+	  This driver can be built as a module. The module will be called
+	  inv-mpu3050-i2c.
+
 config IIO_ST_GYRO_3AXIS
 	tristate "STMicroelectronics gyroscopes 3-Axis Driver"
 	depends on (I2C || SPI_MASTER) && SYSFS
diff --git a/drivers/iio/gyro/Makefile b/drivers/iio/gyro/Makefile
index f866a4be0667..f0e149a606b0 100644
--- a/drivers/iio/gyro/Makefile
+++ b/drivers/iio/gyro/Makefile
@@ -14,6 +14,11 @@ obj-$(CONFIG_BMG160_SPI) += bmg160_spi.o
 
 obj-$(CONFIG_HID_SENSOR_GYRO_3D) += hid-sensor-gyro-3d.o
 
+# Currently this is rolled into one module, split it if
+# we ever create a separate SPI interface for MPU-3050
+obj-$(CONFIG_MPU3050) += mpu3050.o
+mpu3050-objs := mpu3050-core.o mpu3050-i2c.o
+
 itg3200-y               := itg3200_core.o
 itg3200-$(CONFIG_IIO_BUFFER) += itg3200_buffer.o
 obj-$(CONFIG_ITG3200)   += itg3200.o
diff --git a/drivers/iio/gyro/mpu3050-core.c b/drivers/iio/gyro/mpu3050-core.c
new file mode 100644
index 000000000000..2be2a5d287e6
--- /dev/null
+++ b/drivers/iio/gyro/mpu3050-core.c
@@ -0,0 +1,1306 @@
+/*
+ * MPU3050 gyroscope driver
+ *
+ * Copyright (C) 2016 Linaro Ltd.
+ * Author: Linus Walleij <linus.walleij@linaro.org>
+ *
+ * Based on the input subsystem driver, Copyright (C) 2011 Wistron Co.Ltd
+ * Joseph Lai <joseph_lai@wistron.com> and trimmed down by
+ * Alan Cox <alan@linux.intel.com> in turn based on bma023.c.
+ * Device behaviour based on a misc driver posted by Nathan Royer in 2011.
+ *
+ * TODO: add support for setting up the low pass 3dB frequency.
+ */
+
+#include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+
+#include "mpu3050.h"
+
+#define MPU3050_CHIP_ID		0x69
+
+/*
+ * Register map: anything suffixed *_H is a big-endian high byte and always
+ * followed by the corresponding low byte (*_L) even though these are not
+ * explicitly included in the register definitions.
+ */
+#define MPU3050_CHIP_ID_REG	0x00
+#define MPU3050_PRODUCT_ID_REG	0x01
+#define MPU3050_XG_OFFS_TC	0x05
+#define MPU3050_YG_OFFS_TC	0x08
+#define MPU3050_ZG_OFFS_TC	0x0B
+#define MPU3050_X_OFFS_USR_H	0x0C
+#define MPU3050_Y_OFFS_USR_H	0x0E
+#define MPU3050_Z_OFFS_USR_H	0x10
+#define MPU3050_FIFO_EN		0x12
+#define MPU3050_AUX_VDDIO	0x13
+#define MPU3050_SLV_ADDR	0x14
+#define MPU3050_SMPLRT_DIV	0x15
+#define MPU3050_DLPF_FS_SYNC	0x16
+#define MPU3050_INT_CFG		0x17
+#define MPU3050_AUX_ADDR	0x18
+#define MPU3050_INT_STATUS	0x1A
+#define MPU3050_TEMP_H		0x1B
+#define MPU3050_XOUT_H		0x1D
+#define MPU3050_YOUT_H		0x1F
+#define MPU3050_ZOUT_H		0x21
+#define MPU3050_DMP_CFG1	0x35
+#define MPU3050_DMP_CFG2	0x36
+#define MPU3050_BANK_SEL	0x37
+#define MPU3050_MEM_START_ADDR	0x38
+#define MPU3050_MEM_R_W		0x39
+#define MPU3050_FIFO_COUNT_H	0x3A
+#define MPU3050_FIFO_R		0x3C
+#define MPU3050_USR_CTRL	0x3D
+#define MPU3050_PWR_MGM		0x3E
+
+/* MPU memory bank read options */
+#define MPU3050_MEM_PRFTCH	BIT(5)
+#define MPU3050_MEM_USER_BANK	BIT(4)
+/* Bits 8-11 select memory bank */
+#define MPU3050_MEM_RAM_BANK_0	0
+#define MPU3050_MEM_RAM_BANK_1	1
+#define MPU3050_MEM_RAM_BANK_2	2
+#define MPU3050_MEM_RAM_BANK_3	3
+#define MPU3050_MEM_OTP_BANK_0	4
+
+#define MPU3050_AXIS_REGS(axis) (MPU3050_XOUT_H + (axis * 2))
+
+/* Register bits */
+
+/* FIFO Enable */
+#define MPU3050_FIFO_EN_FOOTER		BIT(0)
+#define MPU3050_FIFO_EN_AUX_ZOUT	BIT(1)
+#define MPU3050_FIFO_EN_AUX_YOUT	BIT(2)
+#define MPU3050_FIFO_EN_AUX_XOUT	BIT(3)
+#define MPU3050_FIFO_EN_GYRO_ZOUT	BIT(4)
+#define MPU3050_FIFO_EN_GYRO_YOUT	BIT(5)
+#define MPU3050_FIFO_EN_GYRO_XOUT	BIT(6)
+#define MPU3050_FIFO_EN_TEMP_OUT	BIT(7)
+
+/*
+ * Digital Low Pass filter (DLPF)
+ * Full Scale (FS)
+ * and Synchronization
+ */
+#define MPU3050_EXT_SYNC_NONE		0x00
+#define MPU3050_EXT_SYNC_TEMP		0x20
+#define MPU3050_EXT_SYNC_GYROX		0x40
+#define MPU3050_EXT_SYNC_GYROY		0x60
+#define MPU3050_EXT_SYNC_GYROZ		0x80
+#define MPU3050_EXT_SYNC_ACCELX	0xA0
+#define MPU3050_EXT_SYNC_ACCELY	0xC0
+#define MPU3050_EXT_SYNC_ACCELZ	0xE0
+#define MPU3050_EXT_SYNC_MASK		0xE0
+#define MPU3050_EXT_SYNC_SHIFT		5
+
+#define MPU3050_FS_250DPS		0x00
+#define MPU3050_FS_500DPS		0x08
+#define MPU3050_FS_1000DPS		0x10
+#define MPU3050_FS_2000DPS		0x18
+#define MPU3050_FS_MASK			0x18
+#define MPU3050_FS_SHIFT		3
+
+#define MPU3050_DLPF_CFG_256HZ_NOLPF2	0x00
+#define MPU3050_DLPF_CFG_188HZ		0x01
+#define MPU3050_DLPF_CFG_98HZ		0x02
+#define MPU3050_DLPF_CFG_42HZ		0x03
+#define MPU3050_DLPF_CFG_20HZ		0x04
+#define MPU3050_DLPF_CFG_10HZ		0x05
+#define MPU3050_DLPF_CFG_5HZ		0x06
+#define MPU3050_DLPF_CFG_2100HZ_NOLPF	0x07
+#define MPU3050_DLPF_CFG_MASK		0x07
+#define MPU3050_DLPF_CFG_SHIFT		0
+
+/* Interrupt config */
+#define MPU3050_INT_RAW_RDY_EN		BIT(0)
+#define MPU3050_INT_DMP_DONE_EN		BIT(1)
+#define MPU3050_INT_MPU_RDY_EN		BIT(2)
+#define MPU3050_INT_ANYRD_2CLEAR	BIT(4)
+#define MPU3050_INT_LATCH_EN		BIT(5)
+#define MPU3050_INT_OPEN		BIT(6)
+#define MPU3050_INT_ACTL		BIT(7)
+/* Interrupt status */
+#define MPU3050_INT_STATUS_RAW_RDY	BIT(0)
+#define MPU3050_INT_STATUS_DMP_DONE	BIT(1)
+#define MPU3050_INT_STATUS_MPU_RDY	BIT(2)
+#define MPU3050_INT_STATUS_FIFO_OVFLW	BIT(7)
+/* USR_CTRL */
+#define MPU3050_USR_CTRL_FIFO_EN	BIT(6)
+#define MPU3050_USR_CTRL_AUX_IF_EN	BIT(5)
+#define MPU3050_USR_CTRL_AUX_IF_RST	BIT(3)
+#define MPU3050_USR_CTRL_FIFO_RST	BIT(1)
+#define MPU3050_USR_CTRL_GYRO_RST	BIT(0)
+/* PWR_MGM */
+#define MPU3050_PWR_MGM_PLL_X		0x01
+#define MPU3050_PWR_MGM_PLL_Y		0x02
+#define MPU3050_PWR_MGM_PLL_Z		0x03
+#define MPU3050_PWR_MGM_CLKSEL_MASK	0x07
+#define MPU3050_PWR_MGM_STBY_ZG		BIT(3)
+#define MPU3050_PWR_MGM_STBY_YG		BIT(4)
+#define MPU3050_PWR_MGM_STBY_XG		BIT(5)
+#define MPU3050_PWR_MGM_SLEEP		BIT(6)
+#define MPU3050_PWR_MGM_RESET		BIT(7)
+#define MPU3050_PWR_MGM_MASK		0xff
+
+/*
+ * Fullscale precision is (for finest precision) +/- 250 deg/s, so the full
+ * scale is actually 500 deg/s. All 16 bits are then used to cover this scale,
+ * in two's complement.
+ */
+static unsigned int mpu3050_fs_precision[] = {
+	IIO_DEGREE_TO_RAD(250),
+	IIO_DEGREE_TO_RAD(500),
+	IIO_DEGREE_TO_RAD(1000),
+	IIO_DEGREE_TO_RAD(2000)
+};
+
+/*
+ * Regulator names
+ */
+static const char mpu3050_reg_vdd[] = "vdd";
+static const char mpu3050_reg_vlogic[] = "vlogic";
+
+static unsigned int mpu3050_get_freq(struct mpu3050 *mpu3050)
+{
+	unsigned int freq;
+
+	if (mpu3050->lpf == MPU3050_DLPF_CFG_256HZ_NOLPF2)
+		freq = 8000;
+	else
+		freq = 1000;
+	freq /= (mpu3050->divisor + 1);
+
+	return freq;
+}
+
+static int mpu3050_start_sampling(struct mpu3050 *mpu3050)
+{
+	__be16 raw_val[3];
+	int ret;
+	int i;
+
+	/* Reset */
+	ret = regmap_update_bits(mpu3050->map, MPU3050_PWR_MGM,
+				 MPU3050_PWR_MGM_RESET, MPU3050_PWR_MGM_RESET);
+	if (ret)
+		return ret;
+
+	/* Turn on the Z-axis PLL */
+	ret = regmap_update_bits(mpu3050->map, MPU3050_PWR_MGM,
+				 MPU3050_PWR_MGM_CLKSEL_MASK,
+				 MPU3050_PWR_MGM_PLL_Z);
+	if (ret)
+		return ret;
+
+	/* Write calibration offset registers */
+	for (i = 0; i < 3; i++)
+		raw_val[i] = cpu_to_be16(mpu3050->calibration[i]);
+
+	ret = regmap_bulk_write(mpu3050->map, MPU3050_X_OFFS_USR_H, raw_val,
+				sizeof(raw_val));
+	if (ret)
+		return ret;
+
+	/* Set low pass filter (sample rate), sync and full scale */
+	ret = regmap_write(mpu3050->map, MPU3050_DLPF_FS_SYNC,
+			   MPU3050_EXT_SYNC_NONE << MPU3050_EXT_SYNC_SHIFT |
+			   mpu3050->fullscale << MPU3050_FS_SHIFT |
+			   mpu3050->lpf << MPU3050_DLPF_CFG_SHIFT);
+	if (ret)
+		return ret;
+
+	/* Set up sampling frequency */
+	ret = regmap_write(mpu3050->map, MPU3050_SMPLRT_DIV, mpu3050->divisor);
+	if (ret)
+		return ret;
+
+	/*
+	 * Max 50 ms start-up time after setting DLPF_FS_SYNC
+	 * according to the data sheet, then wait for the next sample
+	 * at this frequency T = 1000/f ms.
+	 */
+	msleep(50 + 1000 / mpu3050_get_freq(mpu3050));
+
+	return 0;
+}
+
+static int mpu3050_set_8khz_samplerate(struct mpu3050 *mpu3050)
+{
+	int ret;
+	u8 divisor;
+	enum mpu3050_lpf lpf;
+
+	lpf = mpu3050->lpf;
+	divisor = mpu3050->divisor;
+
+	mpu3050->lpf = LPF_256_HZ_NOLPF; /* 8 kHz base frequency */
+	mpu3050->divisor = 0; /* Divide by 1 */
+	ret = mpu3050_start_sampling(mpu3050);
+
+	mpu3050->lpf = lpf;
+	mpu3050->divisor = divisor;
+
+	return ret;
+}
+
+static int mpu3050_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2,
+			    long mask)
+{
+	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
+	int ret;
+	__be16 raw_val;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_OFFSET:
+		switch (chan->type) {
+		case IIO_TEMP:
+			/* The temperature scaling is (x+23000)/280 Celsius */
+			*val = 23000;
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_CALIBBIAS:
+		switch (chan->type) {
+		case IIO_ANGL_VEL:
+			*val = mpu3050->calibration[chan->scan_index-1];
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*val = mpu3050_get_freq(mpu3050);
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->type) {
+		case IIO_TEMP:
+			/* Millidegrees, see about temperature scaling above */
+			*val = 1000;
+			*val2 = 280;
+			return IIO_VAL_FRACTIONAL;
+		case IIO_ANGL_VEL:
+			/*
+			 * Convert to the corresponding full scale in
+			 * radians. All 16 bits are used with sign to
+			 * span the available scale: to account for the one
+			 * missing value if we multiply by 1/S16_MAX, instead
+			 * multiply with 2/U16_MAX.
+			 */
+			*val = mpu3050_fs_precision[mpu3050->fullscale] * 2;
+			*val2 = U16_MAX;
+			return IIO_VAL_FRACTIONAL;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_RAW:
+		/* Resume device */
+		pm_runtime_get_sync(mpu3050->dev);
+		mutex_lock(&mpu3050->lock);
+
+		ret = mpu3050_set_8khz_samplerate(mpu3050);
+		if (ret)
+			goto out_read_raw_unlock;
+
+		switch (chan->type) {
+		case IIO_TEMP:
+			ret = regmap_bulk_read(mpu3050->map, MPU3050_TEMP_H,
+					       &raw_val, sizeof(raw_val));
+			if (ret) {
+				dev_err(mpu3050->dev,
+					"error reading temperature\n");
+				goto out_read_raw_unlock;
+			}
+
+			*val = be16_to_cpu(raw_val);
+			ret = IIO_VAL_INT;
+
+			goto out_read_raw_unlock;
+		case IIO_ANGL_VEL:
+			ret = regmap_bulk_read(mpu3050->map,
+				       MPU3050_AXIS_REGS(chan->scan_index-1),
+				       &raw_val,
+				       sizeof(raw_val));
+			if (ret) {
+				dev_err(mpu3050->dev,
+					"error reading axis data\n");
+				goto out_read_raw_unlock;
+			}
+
+			*val = be16_to_cpu(raw_val);
+			ret = IIO_VAL_INT;
+
+			goto out_read_raw_unlock;
+		default:
+			ret = -EINVAL;
+			goto out_read_raw_unlock;
+		}
+	default:
+		break;
+	}
+
+	return -EINVAL;
+
+out_read_raw_unlock:
+	mutex_unlock(&mpu3050->lock);
+	pm_runtime_mark_last_busy(mpu3050->dev);
+	pm_runtime_put_autosuspend(mpu3050->dev);
+
+	return ret;
+}
+
+static int mpu3050_write_raw(struct iio_dev *indio_dev,
+			     const struct iio_chan_spec *chan,
+			     int val, int val2, long mask)
+{
+	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
+	/*
+	 * Couldn't figure out a way to precalculate these at compile time.
+	 */
+	unsigned int fs250 =
+		DIV_ROUND_CLOSEST(mpu3050_fs_precision[0] * 1000000 * 2,
+				  U16_MAX);
+	unsigned int fs500 =
+		DIV_ROUND_CLOSEST(mpu3050_fs_precision[1] * 1000000 * 2,
+				  U16_MAX);
+	unsigned int fs1000 =
+		DIV_ROUND_CLOSEST(mpu3050_fs_precision[2] * 1000000 * 2,
+				  U16_MAX);
+	unsigned int fs2000 =
+		DIV_ROUND_CLOSEST(mpu3050_fs_precision[3] * 1000000 * 2,
+				  U16_MAX);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_CALIBBIAS:
+		if (chan->type != IIO_ANGL_VEL)
+			return -EINVAL;
+		mpu3050->calibration[chan->scan_index-1] = val;
+		return 0;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		/*
+		 * The max samplerate is 8000 Hz, the minimum
+		 * 1000 / 256 ~= 4 Hz
+		 */
+		if (val < 4 || val > 8000)
+			return -EINVAL;
+
+		/*
+		 * Above 1000 Hz we must turn off the digital low pass filter
+		 * so we get a base frequency of 8kHz to the divider
+		 */
+		if (val > 1000) {
+			mpu3050->lpf = LPF_256_HZ_NOLPF;
+			mpu3050->divisor = DIV_ROUND_CLOSEST(8000, val) - 1;
+			return 0;
+		}
+
+		mpu3050->lpf = LPF_188_HZ;
+		mpu3050->divisor = DIV_ROUND_CLOSEST(1000, val) - 1;
+		return 0;
+	case IIO_CHAN_INFO_SCALE:
+		if (chan->type != IIO_ANGL_VEL)
+			return -EINVAL;
+		/*
+		 * We support +/-250, +/-500, +/-1000 and +/2000 deg/s
+		 * which means we need to round to the closest radians
+		 * which will be roughly +/-4.3, +/-8.7, +/-17.5, +/-35
+		 * rad/s. The scale is then for the 16 bits used to cover
+		 * it 2/(2^16) of that.
+		 */
+
+		/* Just too large, set the max range */
+		if (val != 0) {
+			mpu3050->fullscale = FS_2000_DPS;
+			return 0;
+		}
+
+		/*
+		 * Now we're dealing with fractions below zero in millirad/s
+		 * do some integer interpolation and match with the closest
+		 * fullscale in the table.
+		 */
+		if (val2 <= fs250 ||
+		    val2 < ((fs500 + fs250) / 2))
+			mpu3050->fullscale = FS_250_DPS;
+		else if (val2 <= fs500 ||
+			 val2 < ((fs1000 + fs500) / 2))
+			mpu3050->fullscale = FS_500_DPS;
+		else if (val2 <= fs1000 ||
+			 val2 < ((fs2000 + fs1000) / 2))
+			mpu3050->fullscale = FS_1000_DPS;
+		else
+			/* Catch-all */
+			mpu3050->fullscale = FS_2000_DPS;
+		return 0;
+	default:
+		break;
+	}
+
+	return -EINVAL;
+}
+
+static irqreturn_t mpu3050_trigger_handler(int irq, void *p)
+{
+	const struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
+	int ret;
+	/*
+	 * Temperature 1*16 bits
+	 * Three axes 3*16 bits
+	 * Timestamp 64 bits (4*16 bits)
+	 * Sum total 8*16 bits
+	 */
+	__be16 hw_values[8];
+	s64 timestamp;
+	unsigned int datums_from_fifo = 0;
+
+	/*
+	 * If we're using the hardware trigger, get the precise timestamp from
+	 * the top half of the threaded IRQ handler. Otherwise get the
+	 * timestamp here so it will be close in time to the actual values
+	 * read from the registers.
+	 */
+	if (iio_trigger_using_own(indio_dev))
+		timestamp = mpu3050->hw_timestamp;
+	else
+		timestamp = iio_get_time_ns(indio_dev);
+
+	mutex_lock(&mpu3050->lock);
+
+	/* Using the hardware IRQ trigger? Check the buffer then. */
+	if (mpu3050->hw_irq_trigger) {
+		__be16 raw_fifocnt;
+		u16 fifocnt;
+		/* X, Y, Z + temperature */
+		unsigned int bytes_per_datum = 8;
+		bool fifo_overflow = false;
+
+		ret = regmap_bulk_read(mpu3050->map,
+				       MPU3050_FIFO_COUNT_H,
+				       &raw_fifocnt,
+				       sizeof(raw_fifocnt));
+		if (ret)
+			goto out_trigger_unlock;
+		fifocnt = be16_to_cpu(raw_fifocnt);
+
+		if (fifocnt == 512) {
+			dev_info(mpu3050->dev,
+				 "FIFO overflow! Emptying and resetting FIFO\n");
+			fifo_overflow = true;
+			/* Reset and enable the FIFO */
+			ret = regmap_update_bits(mpu3050->map,
+						 MPU3050_USR_CTRL,
+						 MPU3050_USR_CTRL_FIFO_EN |
+						 MPU3050_USR_CTRL_FIFO_RST,
+						 MPU3050_USR_CTRL_FIFO_EN |
+						 MPU3050_USR_CTRL_FIFO_RST);
+			if (ret) {
+				dev_info(mpu3050->dev, "error resetting FIFO\n");
+				goto out_trigger_unlock;
+			}
+			mpu3050->pending_fifo_footer = false;
+		}
+
+		if (fifocnt)
+			dev_dbg(mpu3050->dev,
+				"%d bytes in the FIFO\n",
+				fifocnt);
+
+		while (!fifo_overflow && fifocnt > bytes_per_datum) {
+			unsigned int toread;
+			unsigned int offset;
+			__be16 fifo_values[5];
+
+			/*
+			 * If there is a FIFO footer in the pipe, first clear
+			 * that out. This follows the complex algorithm in the
+			 * datasheet that states that you may never leave the
+			 * FIFO empty after the first reading: you have to
+			 * always leave two footer bytes in it. The footer is
+			 * in practice just two zero bytes.
+			 */
+			if (mpu3050->pending_fifo_footer) {
+				toread = bytes_per_datum + 2;
+				offset = 0;
+			} else {
+				toread = bytes_per_datum;
+				offset = 1;
+				/* Put in some dummy value */
+				fifo_values[0] = 0xAAAA;
+			}
+
+			ret = regmap_bulk_read(mpu3050->map,
+					       MPU3050_FIFO_R,
+					       &fifo_values[offset],
+					       toread);
+
+			dev_dbg(mpu3050->dev,
+				"%04x %04x %04x %04x %04x\n",
+				fifo_values[0],
+				fifo_values[1],
+				fifo_values[2],
+				fifo_values[3],
+				fifo_values[4]);
+
+			/* Index past the footer (fifo_values[0]) and push */
+			iio_push_to_buffers_with_timestamp(indio_dev,
+							   &fifo_values[1],
+							   timestamp);
+
+			fifocnt -= toread;
+			datums_from_fifo++;
+			mpu3050->pending_fifo_footer = true;
+
+			/*
+			 * If we're emptying the FIFO, just make sure to
+			 * check if something new appeared.
+			 */
+			if (fifocnt < bytes_per_datum) {
+				ret = regmap_bulk_read(mpu3050->map,
+						       MPU3050_FIFO_COUNT_H,
+						       &raw_fifocnt,
+						       sizeof(raw_fifocnt));
+				if (ret)
+					goto out_trigger_unlock;
+				fifocnt = be16_to_cpu(raw_fifocnt);
+			}
+
+			if (fifocnt < bytes_per_datum)
+				dev_dbg(mpu3050->dev,
+					"%d bytes left in the FIFO\n",
+					fifocnt);
+
+			/*
+			 * At this point, the timestamp that triggered the
+			 * hardware interrupt is no longer valid for what
+			 * we are reading (the interrupt likely fired for
+			 * the value on the top of the FIFO), so set the
+			 * timestamp to zero and let userspace deal with it.
+			 */
+			timestamp = 0;
+		}
+	}
+
+	/*
+	 * If we picked some datums from the FIFO that's enough, else
+	 * fall through and just read from the current value registers.
+	 * This happens in two cases:
+	 *
+	 * - We are using some other trigger (external, like an HRTimer)
+	 *   than the sensor's own sample generator. In this case the
+	 *   sensor is just set to the max sampling frequency and we give
+	 *   the trigger a copy of the latest value every time we get here.
+	 *
+	 * - The hardware trigger is active but unused and we actually use
+	 *   another trigger which calls here with a frequency higher
+	 *   than what the device provides data. We will then just read
+	 *   duplicate values directly from the hardware registers.
+	 */
+	if (datums_from_fifo) {
+		dev_dbg(mpu3050->dev,
+			"read %d datums from the FIFO\n",
+			datums_from_fifo);
+		goto out_trigger_unlock;
+	}
+
+	ret = regmap_bulk_read(mpu3050->map, MPU3050_TEMP_H, &hw_values,
+			       sizeof(hw_values));
+	if (ret) {
+		dev_err(mpu3050->dev,
+			"error reading axis data\n");
+		goto out_trigger_unlock;
+	}
+
+	iio_push_to_buffers_with_timestamp(indio_dev, hw_values, timestamp);
+
+out_trigger_unlock:
+	mutex_unlock(&mpu3050->lock);
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int mpu3050_buffer_preenable(struct iio_dev *indio_dev)
+{
+	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
+
+	pm_runtime_get_sync(mpu3050->dev);
+
+	/* Unless we have OUR trigger active, run at full speed */
+	if (!mpu3050->hw_irq_trigger)
+		return mpu3050_set_8khz_samplerate(mpu3050);
+
+	return 0;
+}
+
+static int mpu3050_buffer_postdisable(struct iio_dev *indio_dev)
+{
+	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
+
+	pm_runtime_mark_last_busy(mpu3050->dev);
+	pm_runtime_put_autosuspend(mpu3050->dev);
+
+	return 0;
+}
+
+static const struct iio_buffer_setup_ops mpu3050_buffer_setup_ops = {
+	.preenable = mpu3050_buffer_preenable,
+	.postenable = iio_triggered_buffer_postenable,
+	.predisable = iio_triggered_buffer_predisable,
+	.postdisable = mpu3050_buffer_postdisable,
+};
+
+static const struct iio_mount_matrix *
+mpu3050_get_mount_matrix(const struct iio_dev *indio_dev,
+			 const struct iio_chan_spec *chan)
+{
+	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
+
+	return &mpu3050->orientation;
+}
+
+static const struct iio_chan_spec_ext_info mpu3050_ext_info[] = {
+	IIO_MOUNT_MATRIX(IIO_SHARED_BY_TYPE, mpu3050_get_mount_matrix),
+	{ },
+};
+
+#define MPU3050_AXIS_CHANNEL(axis, index)				\
+	{								\
+		.type = IIO_ANGL_VEL,					\
+		.modified = 1,						\
+		.channel2 = IIO_MOD_##axis,				\
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |		\
+			BIT(IIO_CHAN_INFO_CALIBBIAS),			\
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),\
+		.ext_info = mpu3050_ext_info,				\
+		.scan_index = index,					\
+		.scan_type = {						\
+			.sign = 's',					\
+			.realbits = 16,					\
+			.storagebits = 16,				\
+			.endianness = IIO_BE,				\
+		},							\
+	}
+
+static const struct iio_chan_spec mpu3050_channels[] = {
+	{
+		.type = IIO_TEMP,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE) |
+				      BIT(IIO_CHAN_INFO_OFFSET),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_BE,
+		},
+	},
+	MPU3050_AXIS_CHANNEL(X, 1),
+	MPU3050_AXIS_CHANNEL(Y, 2),
+	MPU3050_AXIS_CHANNEL(Z, 3),
+	IIO_CHAN_SOFT_TIMESTAMP(4),
+};
+
+/* Four channels apart from timestamp, scan mask = 0x0f */
+static const unsigned long mpu3050_scan_masks[] = { 0xf, 0 };
+
+/*
+ * These are just the hardcoded factors resulting from the more elaborate
+ * calculations done with fractions in the scale raw get/set functions.
+ */
+static IIO_CONST_ATTR(anglevel_scale_available,
+		      "0.000122070 "
+		      "0.000274658 "
+		      "0.000518798 "
+		      "0.001068115");
+
+static struct attribute *mpu3050_attributes[] = {
+	&iio_const_attr_anglevel_scale_available.dev_attr.attr,
+	NULL,
+};
+
+static const struct attribute_group mpu3050_attribute_group = {
+	.attrs = mpu3050_attributes,
+};
+
+static const struct iio_info mpu3050_info = {
+	.driver_module = THIS_MODULE,
+	.read_raw = mpu3050_read_raw,
+	.write_raw = mpu3050_write_raw,
+	.attrs = &mpu3050_attribute_group,
+};
+
+/**
+ * mpu3050_read_mem() - read MPU-3050 internal memory
+ * @mpu3050: device to read from
+ * @bank: target bank
+ * @addr: target address
+ * @len: number of bytes
+ * @buf: the buffer to store the read bytes in
+ */
+static int mpu3050_read_mem(struct mpu3050 *mpu3050,
+			    u8 bank,
+			    u8 addr,
+			    u8 len,
+			    u8 *buf)
+{
+	int ret;
+
+	ret = regmap_write(mpu3050->map,
+			   MPU3050_BANK_SEL,
+			   bank);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(mpu3050->map,
+			   MPU3050_MEM_START_ADDR,
+			   addr);
+	if (ret)
+		return ret;
+
+	return regmap_bulk_read(mpu3050->map,
+				MPU3050_MEM_R_W,
+				buf,
+				len);
+}
+
+static int mpu3050_hw_init(struct mpu3050 *mpu3050)
+{
+	int ret;
+	u8 otp[8];
+
+	/* Reset */
+	ret = regmap_update_bits(mpu3050->map,
+				 MPU3050_PWR_MGM,
+				 MPU3050_PWR_MGM_RESET,
+				 MPU3050_PWR_MGM_RESET);
+	if (ret)
+		return ret;
+
+	/* Turn on the PLL */
+	ret = regmap_update_bits(mpu3050->map,
+				 MPU3050_PWR_MGM,
+				 MPU3050_PWR_MGM_CLKSEL_MASK,
+				 MPU3050_PWR_MGM_PLL_Z);
+	if (ret)
+		return ret;
+
+	/* Disable IRQs */
+	ret = regmap_write(mpu3050->map,
+			   MPU3050_INT_CFG,
+			   0);
+	if (ret)
+		return ret;
+
+	/* Read out the 8 bytes of OTP (one-time-programmable) memory */
+	ret = mpu3050_read_mem(mpu3050,
+			       (MPU3050_MEM_PRFTCH |
+				MPU3050_MEM_USER_BANK |
+				MPU3050_MEM_OTP_BANK_0),
+			       0,
+			       sizeof(otp),
+			       otp);
+	if (ret)
+		return ret;
+
+	/* This is device-unique data so it goes into the entropy pool */
+	add_device_randomness(otp, sizeof(otp));
+
+	dev_info(mpu3050->dev,
+		 "die ID: %04X, wafer ID: %02X, A lot ID: %04X, "
+		 "W lot ID: %03X, WP ID: %01X, rev ID: %02X\n",
+		 /* Die ID, bits 0-12 */
+		 (otp[1] << 8 | otp[0]) & 0x1fff,
+		 /* Wafer ID, bits 13-17 */
+		 ((otp[2] << 8 | otp[1]) & 0x03e0) >> 5,
+		 /* A lot ID, bits 18-33 */
+		 ((otp[4] << 16 | otp[3] << 8 | otp[2]) & 0x3fffc) >> 2,
+		 /* W lot ID, bits 34-45 */
+		 ((otp[5] << 8 | otp[4]) & 0x3ffc) >> 2,
+		 /* WP ID, bits 47-49 */
+		 ((otp[6] << 8 | otp[5]) & 0x0380) >> 7,
+		 /* rev ID, bits 50-55 */
+		 otp[6] >> 2);
+
+	return 0;
+}
+
+static int mpu3050_power_up(struct mpu3050 *mpu3050)
+{
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(mpu3050->regs), mpu3050->regs);
+	if (ret) {
+		dev_err(mpu3050->dev, "cannot enable regulators\n");
+		return ret;
+	}
+	/*
+	 * 20-100 ms start-up time for register read/write according to
+	 * the datasheet, be on the safe side and wait 200 ms.
+	 */
+	msleep(200);
+
+	/* Take device out of sleep mode */
+	ret = regmap_update_bits(mpu3050->map, MPU3050_PWR_MGM,
+				 MPU3050_PWR_MGM_SLEEP, 0);
+	if (ret) {
+		dev_err(mpu3050->dev, "error setting power mode\n");
+		return ret;
+	}
+	msleep(10);
+
+	return 0;
+}
+
+static int mpu3050_power_down(struct mpu3050 *mpu3050)
+{
+	int ret;
+
+	/*
+	 * Put MPU-3050 into sleep mode before cutting regulators.
+	 * This is important, because we may not be the sole user
+	 * of the regulator so the power may stay on after this, and
+	 * then we would be wasting power unless we go to sleep mode
+	 * first.
+	 */
+	ret = regmap_update_bits(mpu3050->map, MPU3050_PWR_MGM,
+				 MPU3050_PWR_MGM_SLEEP, MPU3050_PWR_MGM_SLEEP);
+	if (ret)
+		dev_err(mpu3050->dev, "error putting to sleep\n");
+
+	ret = regulator_bulk_disable(ARRAY_SIZE(mpu3050->regs), mpu3050->regs);
+	if (ret)
+		dev_err(mpu3050->dev, "error disabling regulators\n");
+
+	return 0;
+}
+
+static irqreturn_t mpu3050_irq_handler(int irq, void *p)
+{
+	struct iio_trigger *trig = p;
+	struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig);
+	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
+
+	if (!mpu3050->hw_irq_trigger)
+		return IRQ_NONE;
+
+	/* Get the time stamp as close in time as possible */
+	mpu3050->hw_timestamp = iio_get_time_ns(indio_dev);
+
+	return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t mpu3050_irq_thread(int irq, void *p)
+{
+	struct iio_trigger *trig = p;
+	struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig);
+	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
+	unsigned int val;
+	int ret;
+
+	/* ACK IRQ and check if it was from us */
+	ret = regmap_read(mpu3050->map, MPU3050_INT_STATUS, &val);
+	if (ret) {
+		dev_err(mpu3050->dev, "error reading IRQ status\n");
+		return IRQ_HANDLED;
+	}
+	if (!(val & MPU3050_INT_STATUS_RAW_RDY))
+		return IRQ_NONE;
+
+	iio_trigger_poll_chained(p);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * mpu3050_drdy_trigger_set_state() - set data ready interrupt state
+ * @trig: trigger instance
+ * @enable: true if trigger should be enabled, false to disable
+ */
+static int mpu3050_drdy_trigger_set_state(struct iio_trigger *trig,
+					  bool enable)
+{
+	struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig);
+	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
+	unsigned int val;
+	int ret;
+
+	/* Disabling trigger: disable interrupt and return */
+	if (!enable) {
+		/* Disable all interrupts */
+		ret = regmap_write(mpu3050->map,
+				   MPU3050_INT_CFG,
+				   0);
+		if (ret)
+			dev_err(mpu3050->dev, "error disabling IRQ\n");
+
+		/* Clear IRQ flag */
+		ret = regmap_read(mpu3050->map, MPU3050_INT_STATUS, &val);
+		if (ret)
+			dev_err(mpu3050->dev, "error clearing IRQ status\n");
+
+		/* Disable all things in the FIFO and reset it */
+		ret = regmap_write(mpu3050->map, MPU3050_FIFO_EN, 0);
+		if (ret)
+			dev_err(mpu3050->dev, "error disabling FIFO\n");
+
+		ret = regmap_write(mpu3050->map, MPU3050_USR_CTRL,
+				   MPU3050_USR_CTRL_FIFO_RST);
+		if (ret)
+			dev_err(mpu3050->dev, "error resetting FIFO\n");
+
+		pm_runtime_mark_last_busy(mpu3050->dev);
+		pm_runtime_put_autosuspend(mpu3050->dev);
+		mpu3050->hw_irq_trigger = false;
+
+		return 0;
+	} else {
+		/* Else we're enabling the trigger from this point */
+		pm_runtime_get_sync(mpu3050->dev);
+		mpu3050->hw_irq_trigger = true;
+
+		/* Disable all things in the FIFO */
+		ret = regmap_write(mpu3050->map, MPU3050_FIFO_EN, 0);
+		if (ret)
+			return ret;
+
+		/* Reset and enable the FIFO */
+		ret = regmap_update_bits(mpu3050->map, MPU3050_USR_CTRL,
+					 MPU3050_USR_CTRL_FIFO_EN |
+					 MPU3050_USR_CTRL_FIFO_RST,
+					 MPU3050_USR_CTRL_FIFO_EN |
+					 MPU3050_USR_CTRL_FIFO_RST);
+		if (ret)
+			return ret;
+
+		mpu3050->pending_fifo_footer = false;
+
+		/* Turn on the FIFO for temp+X+Y+Z */
+		ret = regmap_write(mpu3050->map, MPU3050_FIFO_EN,
+				   MPU3050_FIFO_EN_TEMP_OUT |
+				   MPU3050_FIFO_EN_GYRO_XOUT |
+				   MPU3050_FIFO_EN_GYRO_YOUT |
+				   MPU3050_FIFO_EN_GYRO_ZOUT |
+				   MPU3050_FIFO_EN_FOOTER);
+		if (ret)
+			return ret;
+
+		/* Configure the sample engine */
+		ret = mpu3050_start_sampling(mpu3050);
+		if (ret)
+			return ret;
+
+		/* Clear IRQ flag */
+		ret = regmap_read(mpu3050->map, MPU3050_INT_STATUS, &val);
+		if (ret)
+			dev_err(mpu3050->dev, "error clearing IRQ status\n");
+
+		/* Give us interrupts whenever there is new data ready */
+		val = MPU3050_INT_RAW_RDY_EN;
+
+		if (mpu3050->irq_actl)
+			val |= MPU3050_INT_ACTL;
+		if (mpu3050->irq_latch)
+			val |= MPU3050_INT_LATCH_EN;
+		if (mpu3050->irq_opendrain)
+			val |= MPU3050_INT_OPEN;
+
+		ret = regmap_write(mpu3050->map, MPU3050_INT_CFG, val);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static const struct iio_trigger_ops mpu3050_trigger_ops = {
+	.owner = THIS_MODULE,
+	.set_trigger_state = mpu3050_drdy_trigger_set_state,
+};
+
+static int mpu3050_trigger_probe(struct iio_dev *indio_dev, int irq)
+{
+	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
+	unsigned long irq_trig;
+	int ret;
+
+	mpu3050->trig = devm_iio_trigger_alloc(&indio_dev->dev,
+					       "%s-dev%d",
+					       indio_dev->name,
+					       indio_dev->id);
+	if (!mpu3050->trig)
+		return -ENOMEM;
+
+	/* Check if IRQ is open drain */
+	if (of_property_read_bool(mpu3050->dev->of_node, "drive-open-drain"))
+		mpu3050->irq_opendrain = true;
+
+	irq_trig = irqd_get_trigger_type(irq_get_irq_data(irq));
+	/*
+	 * Configure the interrupt generator hardware to supply whatever
+	 * the interrupt is configured for, edges low/high level low/high,
+	 * we can provide it all.
+	 */
+	switch (irq_trig) {
+	case IRQF_TRIGGER_RISING:
+		dev_info(&indio_dev->dev,
+			 "pulse interrupts on the rising edge\n");
+		if (mpu3050->irq_opendrain) {
+			dev_info(&indio_dev->dev,
+				 "rising edge incompatible with open drain\n");
+			mpu3050->irq_opendrain = false;
+		}
+		break;
+	case IRQF_TRIGGER_FALLING:
+		mpu3050->irq_actl = true;
+		dev_info(&indio_dev->dev,
+			 "pulse interrupts on the falling edge\n");
+		break;
+	case IRQF_TRIGGER_HIGH:
+		mpu3050->irq_latch = true;
+		dev_info(&indio_dev->dev,
+			 "interrupts active high level\n");
+		if (mpu3050->irq_opendrain) {
+			dev_info(&indio_dev->dev,
+				 "active high incompatible with open drain\n");
+			mpu3050->irq_opendrain = false;
+		}
+		/*
+		 * With level IRQs, we mask the IRQ until it is processed,
+		 * but with edge IRQs (pulses) we can queue several interrupts
+		 * in the top half.
+		 */
+		irq_trig |= IRQF_ONESHOT;
+		break;
+	case IRQF_TRIGGER_LOW:
+		mpu3050->irq_latch = true;
+		mpu3050->irq_actl = true;
+		irq_trig |= IRQF_ONESHOT;
+		dev_info(&indio_dev->dev,
+			 "interrupts active low level\n");
+		break;
+	default:
+		/* This is the most preferred mode, if possible */
+		dev_err(&indio_dev->dev,
+			"unsupported IRQ trigger specified (%lx), enforce "
+			"rising edge\n", irq_trig);
+		irq_trig = IRQF_TRIGGER_RISING;
+		break;
+	}
+
+	/* An open drain line can be shared with several devices */
+	if (mpu3050->irq_opendrain)
+		irq_trig |= IRQF_SHARED;
+
+	ret = request_threaded_irq(irq,
+				   mpu3050_irq_handler,
+				   mpu3050_irq_thread,
+				   irq_trig,
+				   mpu3050->trig->name,
+				   mpu3050->trig);
+	if (ret) {
+		dev_err(mpu3050->dev,
+			"can't get IRQ %d, error %d\n", irq, ret);
+		return ret;
+	}
+
+	mpu3050->irq = irq;
+	mpu3050->trig->dev.parent = mpu3050->dev;
+	mpu3050->trig->ops = &mpu3050_trigger_ops;
+	iio_trigger_set_drvdata(mpu3050->trig, indio_dev);
+
+	ret = iio_trigger_register(mpu3050->trig);
+	if (ret)
+		return ret;
+
+	indio_dev->trig = iio_trigger_get(mpu3050->trig);
+
+	return 0;
+}
+
+int mpu3050_common_probe(struct device *dev,
+			 struct regmap *map,
+			 int irq,
+			 const char *name)
+{
+	struct iio_dev *indio_dev;
+	struct mpu3050 *mpu3050;
+	unsigned int val;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*mpu3050));
+	if (!indio_dev)
+		return -ENOMEM;
+	mpu3050 = iio_priv(indio_dev);
+
+	mpu3050->dev = dev;
+	mpu3050->map = map;
+	mutex_init(&mpu3050->lock);
+	/* Default fullscale: 2000 degrees per second */
+	mpu3050->fullscale = FS_2000_DPS;
+	/* 1 kHz, divide by 100, default frequency = 10 Hz */
+	mpu3050->lpf = MPU3050_DLPF_CFG_188HZ;
+	mpu3050->divisor = 99;
+
+	/* Read the mounting matrix, if present */
+	ret = of_iio_read_mount_matrix(dev, "mount-matrix",
+				       &mpu3050->orientation);
+	if (ret)
+		return ret;
+
+	/* Fetch and turn on regulators */
+	mpu3050->regs[0].supply = mpu3050_reg_vdd;
+	mpu3050->regs[1].supply = mpu3050_reg_vlogic;
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(mpu3050->regs),
+				      mpu3050->regs);
+	if (ret) {
+		dev_err(dev, "Cannot get regulators\n");
+		return ret;
+	}
+
+	ret = mpu3050_power_up(mpu3050);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(map, MPU3050_CHIP_ID_REG, &val);
+	if (ret) {
+		dev_err(dev, "could not read device ID\n");
+		ret = -ENODEV;
+
+		goto err_power_down;
+	}
+
+	if (val != MPU3050_CHIP_ID) {
+		dev_err(dev, "unsupported chip id %02x\n", (u8)val);
+		ret = -ENODEV;
+		goto err_power_down;
+	}
+
+	ret = regmap_read(map, MPU3050_PRODUCT_ID_REG, &val);
+	if (ret) {
+		dev_err(dev, "could not read device ID\n");
+		ret = -ENODEV;
+
+		goto err_power_down;
+	}
+	dev_info(dev, "found MPU-3050 part no: %d, version: %d\n",
+		 ((val >> 4) & 0xf), (val & 0xf));
+
+	ret = mpu3050_hw_init(mpu3050);
+	if (ret)
+		goto err_power_down;
+
+	indio_dev->dev.parent = dev;
+	indio_dev->channels = mpu3050_channels;
+	indio_dev->num_channels = ARRAY_SIZE(mpu3050_channels);
+	indio_dev->info = &mpu3050_info;
+	indio_dev->available_scan_masks = mpu3050_scan_masks;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->name = name;
+
+	ret = iio_triggered_buffer_setup(indio_dev, iio_pollfunc_store_time,
+					 mpu3050_trigger_handler,
+					 &mpu3050_buffer_setup_ops);
+	if (ret) {
+		dev_err(dev, "triggered buffer setup failed\n");
+		goto err_power_down;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(dev, "device register failed\n");
+		goto err_cleanup_buffer;
+	}
+
+	dev_set_drvdata(dev, indio_dev);
+
+	/* Check if we have an assigned IRQ to use as trigger */
+	if (irq) {
+		ret = mpu3050_trigger_probe(indio_dev, irq);
+		if (ret)
+			dev_err(dev, "failed to register trigger\n");
+	}
+
+	/* Enable runtime PM */
+	pm_runtime_get_noresume(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+	/*
+	 * Set autosuspend to two orders of magnitude larger than the
+	 * start-up time. 100ms start-up time means 10000ms autosuspend,
+	 * i.e. 10 seconds.
+	 */
+	pm_runtime_set_autosuspend_delay(dev, 10000);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_put(dev);
+
+	return 0;
+
+err_cleanup_buffer:
+	iio_triggered_buffer_cleanup(indio_dev);
+err_power_down:
+	mpu3050_power_down(mpu3050);
+
+	return ret;
+}
+EXPORT_SYMBOL(mpu3050_common_probe);
+
+int mpu3050_common_remove(struct device *dev)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
+
+	pm_runtime_get_sync(dev);
+	pm_runtime_put_noidle(dev);
+	pm_runtime_disable(dev);
+	iio_triggered_buffer_cleanup(indio_dev);
+	if (mpu3050->irq)
+		free_irq(mpu3050->irq, mpu3050);
+	iio_device_unregister(indio_dev);
+	mpu3050_power_down(mpu3050);
+
+	return 0;
+}
+EXPORT_SYMBOL(mpu3050_common_remove);
+
+#ifdef CONFIG_PM
+static int mpu3050_runtime_suspend(struct device *dev)
+{
+	return mpu3050_power_down(iio_priv(dev_get_drvdata(dev)));
+}
+
+static int mpu3050_runtime_resume(struct device *dev)
+{
+	return mpu3050_power_up(iio_priv(dev_get_drvdata(dev)));
+}
+#endif /* CONFIG_PM */
+
+const struct dev_pm_ops mpu3050_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(mpu3050_runtime_suspend,
+			   mpu3050_runtime_resume, NULL)
+};
+EXPORT_SYMBOL(mpu3050_dev_pm_ops);
+
+MODULE_AUTHOR("Linus Walleij");
+MODULE_DESCRIPTION("MPU3050 gyroscope driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/gyro/mpu3050-i2c.c b/drivers/iio/gyro/mpu3050-i2c.c
new file mode 100644
index 000000000000..06007200bf49
--- /dev/null
+++ b/drivers/iio/gyro/mpu3050-i2c.c
@@ -0,0 +1,124 @@
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/i2c-mux.h>
+#include <linux/iio/iio.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/pm_runtime.h>
+
+#include "mpu3050.h"
+
+static const struct regmap_config mpu3050_i2c_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+};
+
+static int mpu3050_i2c_bypass_select(struct i2c_mux_core *mux, u32 chan_id)
+{
+	struct mpu3050 *mpu3050 = i2c_mux_priv(mux);
+
+	/* Just power up the device, that is all that is needed */
+	pm_runtime_get_sync(mpu3050->dev);
+	return 0;
+}
+
+static int mpu3050_i2c_bypass_deselect(struct i2c_mux_core *mux, u32 chan_id)
+{
+	struct mpu3050 *mpu3050 = i2c_mux_priv(mux);
+
+	pm_runtime_mark_last_busy(mpu3050->dev);
+	pm_runtime_put_autosuspend(mpu3050->dev);
+	return 0;
+}
+
+static int mpu3050_i2c_probe(struct i2c_client *client,
+			     const struct i2c_device_id *id)
+{
+	struct regmap *regmap;
+	const char *name;
+	struct mpu3050 *mpu3050;
+	int ret;
+
+	if (!i2c_check_functionality(client->adapter,
+				     I2C_FUNC_SMBUS_I2C_BLOCK))
+		return -EOPNOTSUPP;
+
+	if (id)
+		name = id->name;
+	else
+		return -ENODEV;
+
+	regmap = devm_regmap_init_i2c(client, &mpu3050_i2c_regmap_config);
+	if (IS_ERR(regmap)) {
+		dev_err(&client->dev, "Failed to register i2c regmap %d\n",
+			(int)PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+	}
+
+	ret = mpu3050_common_probe(&client->dev, regmap, client->irq, name);
+	if (ret)
+		return ret;
+
+	/* The main driver is up, now register the I2C mux */
+	mpu3050 = iio_priv(dev_get_drvdata(&client->dev));
+	mpu3050->i2cmux = i2c_mux_alloc(client->adapter, &client->dev,
+					1, 0, I2C_MUX_LOCKED | I2C_MUX_GATE,
+					mpu3050_i2c_bypass_select,
+					mpu3050_i2c_bypass_deselect);
+	/* Just fail the mux, there is no point in killing the driver */
+	if (!mpu3050->i2cmux)
+		dev_err(&client->dev, "failed to allocate I2C mux\n");
+	else {
+		mpu3050->i2cmux->priv = mpu3050;
+		ret = i2c_mux_add_adapter(mpu3050->i2cmux, 0, 0, 0);
+		if (ret)
+			dev_err(&client->dev, "failed to add I2C mux\n");
+	}
+
+	return 0;
+}
+
+static int mpu3050_i2c_remove(struct i2c_client *client)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(&client->dev);
+	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
+
+	if (mpu3050->i2cmux)
+		i2c_mux_del_adapters(mpu3050->i2cmux);
+
+	return mpu3050_common_remove(&client->dev);
+}
+
+/*
+ * device id table is used to identify what device can be
+ * supported by this driver
+ */
+static const struct i2c_device_id mpu3050_i2c_id[] = {
+	{ "mpu3050" },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, mpu3050_i2c_id);
+
+static const struct of_device_id mpu3050_i2c_of_match[] = {
+	{ .compatible = "invensense,mpu3050", .data = "mpu3050" },
+	/* Deprecated vendor ID from the Input driver */
+	{ .compatible = "invn,mpu3050", .data = "mpu3050" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, mpu3050_i2c_of_match);
+
+static struct i2c_driver mpu3050_i2c_driver = {
+	.probe = mpu3050_i2c_probe,
+	.remove = mpu3050_i2c_remove,
+	.id_table = mpu3050_i2c_id,
+	.driver = {
+		.of_match_table = mpu3050_i2c_of_match,
+		.name = "mpu3050-i2c",
+		.pm = &mpu3050_dev_pm_ops,
+	},
+};
+module_i2c_driver(mpu3050_i2c_driver);
+
+MODULE_AUTHOR("Linus Walleij");
+MODULE_DESCRIPTION("Invensense MPU3050 gyroscope driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/gyro/mpu3050.h b/drivers/iio/gyro/mpu3050.h
new file mode 100644
index 000000000000..bef87a714dc5
--- /dev/null
+++ b/drivers/iio/gyro/mpu3050.h
@@ -0,0 +1,96 @@
+#include <linux/iio/iio.h>
+#include <linux/mutex.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/i2c.h>
+
+/**
+ * enum mpu3050_fullscale - indicates the full range of the sensor in deg/sec
+ */
+enum mpu3050_fullscale {
+	FS_250_DPS = 0,
+	FS_500_DPS,
+	FS_1000_DPS,
+	FS_2000_DPS,
+};
+
+/**
+ * enum mpu3050_lpf - indicates the low pass filter width
+ */
+enum mpu3050_lpf {
+	/* This implicity sets sample frequency to 8 kHz */
+	LPF_256_HZ_NOLPF = 0,
+	/* All others sets the sample frequency to 1 kHz */
+	LPF_188_HZ,
+	LPF_98_HZ,
+	LPF_42_HZ,
+	LPF_20_HZ,
+	LPF_10_HZ,
+	LPF_5_HZ,
+	LPF_2100_HZ_NOLPF,
+};
+
+enum mpu3050_axis {
+	AXIS_X = 0,
+	AXIS_Y,
+	AXIS_Z,
+	AXIS_MAX,
+};
+
+/**
+ * struct mpu3050 - instance state container for the device
+ * @dev: parent device for this instance
+ * @orientation: mounting matrix, flipped axis etc
+ * @map: regmap to reach the registers
+ * @lock: serialization lock to marshal all requests
+ * @irq: the IRQ used for this device
+ * @regs: the regulators to power this device
+ * @fullscale: the current fullscale setting for the device
+ * @lpf: digital low pass filter setting for the device
+ * @divisor: base frequency divider: divides 8 or 1 kHz
+ * @calibration: the three signed 16-bit calibration settings that
+ * get written into the offset registers for each axis to compensate
+ * for DC offsets
+ * @trig: trigger for the MPU-3050 interrupt, if present
+ * @hw_irq_trigger: hardware interrupt trigger is in use
+ * @irq_actl: interrupt is active low
+ * @irq_latch: latched IRQ, this means that it is a level IRQ
+ * @irq_opendrain: the interrupt line shall be configured open drain
+ * @pending_fifo_footer: tells us if there is a pending footer in the FIFO
+ * that we have to read out first when handling the FIFO
+ * @hw_timestamp: latest hardware timestamp from the trigger IRQ, when in
+ * use
+ * @i2cmux: an I2C mux reflecting the fact that this sensor is a hub with
+ * a pass-through I2C interface coming out of it: this device needs to be
+ * powered up in order to reach devices on the other side of this mux
+ */
+struct mpu3050 {
+	struct device *dev;
+	struct iio_mount_matrix orientation;
+	struct regmap *map;
+	struct mutex lock;
+	int irq;
+	struct regulator_bulk_data regs[2];
+	enum mpu3050_fullscale fullscale;
+	enum mpu3050_lpf lpf;
+	u8 divisor;
+	s16 calibration[3];
+	struct iio_trigger *trig;
+	bool hw_irq_trigger;
+	bool irq_actl;
+	bool irq_latch;
+	bool irq_opendrain;
+	bool pending_fifo_footer;
+	s64 hw_timestamp;
+	struct i2c_mux_core *i2cmux;
+};
+
+/* Probe called from different transports */
+int mpu3050_common_probe(struct device *dev,
+			 struct regmap *map,
+			 int irq,
+			 const char *name);
+int mpu3050_common_remove(struct device *dev);
+
+/* PM ops */
+extern const struct dev_pm_ops mpu3050_dev_pm_ops;
diff --git a/drivers/iio/gyro/st_gyro_core.c b/drivers/iio/gyro/st_gyro_core.c
index aea034d8fe0f..2a42b3d583e8 100644
--- a/drivers/iio/gyro/st_gyro_core.c
+++ b/drivers/iio/gyro/st_gyro_core.c
@@ -39,79 +39,6 @@
 #define ST_GYRO_FS_AVL_500DPS			500
 #define ST_GYRO_FS_AVL_2000DPS			2000
 
-/* CUSTOM VALUES FOR SENSOR 1 */
-#define ST_GYRO_1_WAI_EXP			0xd3
-#define ST_GYRO_1_ODR_ADDR			0x20
-#define ST_GYRO_1_ODR_MASK			0xc0
-#define ST_GYRO_1_ODR_AVL_100HZ_VAL		0x00
-#define ST_GYRO_1_ODR_AVL_200HZ_VAL		0x01
-#define ST_GYRO_1_ODR_AVL_400HZ_VAL		0x02
-#define ST_GYRO_1_ODR_AVL_800HZ_VAL		0x03
-#define ST_GYRO_1_PW_ADDR			0x20
-#define ST_GYRO_1_PW_MASK			0x08
-#define ST_GYRO_1_FS_ADDR			0x23
-#define ST_GYRO_1_FS_MASK			0x30
-#define ST_GYRO_1_FS_AVL_250_VAL		0x00
-#define ST_GYRO_1_FS_AVL_500_VAL		0x01
-#define ST_GYRO_1_FS_AVL_2000_VAL		0x02
-#define ST_GYRO_1_FS_AVL_250_GAIN		IIO_DEGREE_TO_RAD(8750)
-#define ST_GYRO_1_FS_AVL_500_GAIN		IIO_DEGREE_TO_RAD(17500)
-#define ST_GYRO_1_FS_AVL_2000_GAIN		IIO_DEGREE_TO_RAD(70000)
-#define ST_GYRO_1_BDU_ADDR			0x23
-#define ST_GYRO_1_BDU_MASK			0x80
-#define ST_GYRO_1_DRDY_IRQ_ADDR			0x22
-#define ST_GYRO_1_DRDY_IRQ_INT2_MASK		0x08
-#define ST_GYRO_1_MULTIREAD_BIT			true
-
-/* CUSTOM VALUES FOR SENSOR 2 */
-#define ST_GYRO_2_WAI_EXP			0xd4
-#define ST_GYRO_2_ODR_ADDR			0x20
-#define ST_GYRO_2_ODR_MASK			0xc0
-#define ST_GYRO_2_ODR_AVL_95HZ_VAL		0x00
-#define ST_GYRO_2_ODR_AVL_190HZ_VAL		0x01
-#define ST_GYRO_2_ODR_AVL_380HZ_VAL		0x02
-#define ST_GYRO_2_ODR_AVL_760HZ_VAL		0x03
-#define ST_GYRO_2_PW_ADDR			0x20
-#define ST_GYRO_2_PW_MASK			0x08
-#define ST_GYRO_2_FS_ADDR			0x23
-#define ST_GYRO_2_FS_MASK			0x30
-#define ST_GYRO_2_FS_AVL_250_VAL		0x00
-#define ST_GYRO_2_FS_AVL_500_VAL		0x01
-#define ST_GYRO_2_FS_AVL_2000_VAL		0x02
-#define ST_GYRO_2_FS_AVL_250_GAIN		IIO_DEGREE_TO_RAD(8750)
-#define ST_GYRO_2_FS_AVL_500_GAIN		IIO_DEGREE_TO_RAD(17500)
-#define ST_GYRO_2_FS_AVL_2000_GAIN		IIO_DEGREE_TO_RAD(70000)
-#define ST_GYRO_2_BDU_ADDR			0x23
-#define ST_GYRO_2_BDU_MASK			0x80
-#define ST_GYRO_2_DRDY_IRQ_ADDR			0x22
-#define ST_GYRO_2_DRDY_IRQ_INT2_MASK		0x08
-#define ST_GYRO_2_MULTIREAD_BIT			true
-
-/* CUSTOM VALUES FOR SENSOR 3 */
-#define ST_GYRO_3_WAI_EXP			0xd7
-#define ST_GYRO_3_ODR_ADDR			0x20
-#define ST_GYRO_3_ODR_MASK			0xc0
-#define ST_GYRO_3_ODR_AVL_95HZ_VAL		0x00
-#define ST_GYRO_3_ODR_AVL_190HZ_VAL		0x01
-#define ST_GYRO_3_ODR_AVL_380HZ_VAL		0x02
-#define ST_GYRO_3_ODR_AVL_760HZ_VAL		0x03
-#define ST_GYRO_3_PW_ADDR			0x20
-#define ST_GYRO_3_PW_MASK			0x08
-#define ST_GYRO_3_FS_ADDR			0x23
-#define ST_GYRO_3_FS_MASK			0x30
-#define ST_GYRO_3_FS_AVL_250_VAL		0x00
-#define ST_GYRO_3_FS_AVL_500_VAL		0x01
-#define ST_GYRO_3_FS_AVL_2000_VAL		0x02
-#define ST_GYRO_3_FS_AVL_250_GAIN		IIO_DEGREE_TO_RAD(8750)
-#define ST_GYRO_3_FS_AVL_500_GAIN		IIO_DEGREE_TO_RAD(17500)
-#define ST_GYRO_3_FS_AVL_2000_GAIN		IIO_DEGREE_TO_RAD(70000)
-#define ST_GYRO_3_BDU_ADDR			0x23
-#define ST_GYRO_3_BDU_MASK			0x80
-#define ST_GYRO_3_DRDY_IRQ_ADDR			0x22
-#define ST_GYRO_3_DRDY_IRQ_INT2_MASK		0x08
-#define ST_GYRO_3_MULTIREAD_BIT			true
-
-
 static const struct iio_chan_spec st_gyro_16bit_channels[] = {
 	ST_SENSORS_LSM_CHANNELS(IIO_ANGL_VEL,
 			BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
@@ -130,7 +57,7 @@ static const struct iio_chan_spec st_gyro_16bit_channels[] = {
 
 static const struct st_sensor_settings st_gyro_sensors_settings[] = {
 	{
-		.wai = ST_GYRO_1_WAI_EXP,
+		.wai = 0xd3,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = L3G4200D_GYRO_DEV_NAME,
@@ -138,18 +65,18 @@ static const struct st_sensor_settings st_gyro_sensors_settings[] = {
 		},
 		.ch = (struct iio_chan_spec *)st_gyro_16bit_channels,
 		.odr = {
-			.addr = ST_GYRO_1_ODR_ADDR,
-			.mask = ST_GYRO_1_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0xc0,
 			.odr_avl = {
-				{ 100, ST_GYRO_1_ODR_AVL_100HZ_VAL, },
-				{ 200, ST_GYRO_1_ODR_AVL_200HZ_VAL, },
-				{ 400, ST_GYRO_1_ODR_AVL_400HZ_VAL, },
-				{ 800, ST_GYRO_1_ODR_AVL_800HZ_VAL, },
+				{ .hz = 100, .value = 0x00, },
+				{ .hz = 200, .value = 0x01, },
+				{ .hz = 400, .value = 0x02, },
+				{ .hz = 800, .value = 0x03, },
 			},
 		},
 		.pw = {
-			.addr = ST_GYRO_1_PW_ADDR,
-			.mask = ST_GYRO_1_PW_MASK,
+			.addr = 0x20,
+			.mask = 0x08,
 			.value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
@@ -158,33 +85,33 @@ static const struct st_sensor_settings st_gyro_sensors_settings[] = {
 			.mask = ST_SENSORS_DEFAULT_AXIS_MASK,
 		},
 		.fs = {
-			.addr = ST_GYRO_1_FS_ADDR,
-			.mask = ST_GYRO_1_FS_MASK,
+			.addr = 0x23,
+			.mask = 0x30,
 			.fs_avl = {
 				[0] = {
 					.num = ST_GYRO_FS_AVL_250DPS,
-					.value = ST_GYRO_1_FS_AVL_250_VAL,
-					.gain = ST_GYRO_1_FS_AVL_250_GAIN,
+					.value = 0x00,
+					.gain = IIO_DEGREE_TO_RAD(8750),
 				},
 				[1] = {
 					.num = ST_GYRO_FS_AVL_500DPS,
-					.value = ST_GYRO_1_FS_AVL_500_VAL,
-					.gain = ST_GYRO_1_FS_AVL_500_GAIN,
+					.value = 0x01,
+					.gain = IIO_DEGREE_TO_RAD(17500),
 				},
 				[2] = {
 					.num = ST_GYRO_FS_AVL_2000DPS,
-					.value = ST_GYRO_1_FS_AVL_2000_VAL,
-					.gain = ST_GYRO_1_FS_AVL_2000_GAIN,
+					.value = 0x02,
+					.gain = IIO_DEGREE_TO_RAD(70000),
 				},
 			},
 		},
 		.bdu = {
-			.addr = ST_GYRO_1_BDU_ADDR,
-			.mask = ST_GYRO_1_BDU_MASK,
+			.addr = 0x23,
+			.mask = 0x80,
 		},
 		.drdy_irq = {
-			.addr = ST_GYRO_1_DRDY_IRQ_ADDR,
-			.mask_int2 = ST_GYRO_1_DRDY_IRQ_INT2_MASK,
+			.addr = 0x22,
+			.mask_int2 = 0x08,
 			/*
 			 * The sensor has IHL (active low) and open
 			 * drain settings, but only for INT1 and not
@@ -192,11 +119,11 @@ static const struct st_sensor_settings st_gyro_sensors_settings[] = {
 			 */
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 		},
-		.multi_read_bit = ST_GYRO_1_MULTIREAD_BIT,
+		.multi_read_bit = true,
 		.bootime = 2,
 	},
 	{
-		.wai = ST_GYRO_2_WAI_EXP,
+		.wai = 0xd4,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = L3GD20_GYRO_DEV_NAME,
@@ -208,18 +135,18 @@ static const struct st_sensor_settings st_gyro_sensors_settings[] = {
 		},
 		.ch = (struct iio_chan_spec *)st_gyro_16bit_channels,
 		.odr = {
-			.addr = ST_GYRO_2_ODR_ADDR,
-			.mask = ST_GYRO_2_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0xc0,
 			.odr_avl = {
-				{ 95, ST_GYRO_2_ODR_AVL_95HZ_VAL, },
-				{ 190, ST_GYRO_2_ODR_AVL_190HZ_VAL, },
-				{ 380, ST_GYRO_2_ODR_AVL_380HZ_VAL, },
-				{ 760, ST_GYRO_2_ODR_AVL_760HZ_VAL, },
+				{ .hz = 95, .value = 0x00, },
+				{ .hz = 190, .value = 0x01, },
+				{ .hz = 380, .value = 0x02, },
+				{ .hz = 760, .value = 0x03, },
 			},
 		},
 		.pw = {
-			.addr = ST_GYRO_2_PW_ADDR,
-			.mask = ST_GYRO_2_PW_MASK,
+			.addr = 0x20,
+			.mask = 0x08,
 			.value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
@@ -228,33 +155,33 @@ static const struct st_sensor_settings st_gyro_sensors_settings[] = {
 			.mask = ST_SENSORS_DEFAULT_AXIS_MASK,
 		},
 		.fs = {
-			.addr = ST_GYRO_2_FS_ADDR,
-			.mask = ST_GYRO_2_FS_MASK,
+			.addr = 0x23,
+			.mask = 0x30,
 			.fs_avl = {
 				[0] = {
 					.num = ST_GYRO_FS_AVL_250DPS,
-					.value = ST_GYRO_2_FS_AVL_250_VAL,
-					.gain = ST_GYRO_2_FS_AVL_250_GAIN,
+					.value = 0x00,
+					.gain = IIO_DEGREE_TO_RAD(8750),
 				},
 				[1] = {
 					.num = ST_GYRO_FS_AVL_500DPS,
-					.value = ST_GYRO_2_FS_AVL_500_VAL,
-					.gain = ST_GYRO_2_FS_AVL_500_GAIN,
+					.value = 0x01,
+					.gain = IIO_DEGREE_TO_RAD(17500),
 				},
 				[2] = {
 					.num = ST_GYRO_FS_AVL_2000DPS,
-					.value = ST_GYRO_2_FS_AVL_2000_VAL,
-					.gain = ST_GYRO_2_FS_AVL_2000_GAIN,
+					.value = 0x02,
+					.gain = IIO_DEGREE_TO_RAD(70000),
 				},
 			},
 		},
 		.bdu = {
-			.addr = ST_GYRO_2_BDU_ADDR,
-			.mask = ST_GYRO_2_BDU_MASK,
+			.addr = 0x23,
+			.mask = 0x80,
 		},
 		.drdy_irq = {
-			.addr = ST_GYRO_2_DRDY_IRQ_ADDR,
-			.mask_int2 = ST_GYRO_2_DRDY_IRQ_INT2_MASK,
+			.addr = 0x22,
+			.mask_int2 = 0x08,
 			/*
 			 * The sensor has IHL (active low) and open
 			 * drain settings, but only for INT1 and not
@@ -262,29 +189,29 @@ static const struct st_sensor_settings st_gyro_sensors_settings[] = {
 			 */
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 		},
-		.multi_read_bit = ST_GYRO_2_MULTIREAD_BIT,
+		.multi_read_bit = true,
 		.bootime = 2,
 	},
 	{
-		.wai = ST_GYRO_3_WAI_EXP,
+		.wai = 0xd7,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = L3GD20_GYRO_DEV_NAME,
 		},
 		.ch = (struct iio_chan_spec *)st_gyro_16bit_channels,
 		.odr = {
-			.addr = ST_GYRO_3_ODR_ADDR,
-			.mask = ST_GYRO_3_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0xc0,
 			.odr_avl = {
-				{ 95, ST_GYRO_3_ODR_AVL_95HZ_VAL, },
-				{ 190, ST_GYRO_3_ODR_AVL_190HZ_VAL, },
-				{ 380, ST_GYRO_3_ODR_AVL_380HZ_VAL, },
-				{ 760, ST_GYRO_3_ODR_AVL_760HZ_VAL, },
+				{ .hz = 95, .value = 0x00, },
+				{ .hz = 190, .value = 0x01, },
+				{ .hz = 380, .value = 0x02, },
+				{ .hz = 760, .value = 0x03, },
 			},
 		},
 		.pw = {
-			.addr = ST_GYRO_3_PW_ADDR,
-			.mask = ST_GYRO_3_PW_MASK,
+			.addr = 0x20,
+			.mask = 0x08,
 			.value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
@@ -293,33 +220,33 @@ static const struct st_sensor_settings st_gyro_sensors_settings[] = {
 			.mask = ST_SENSORS_DEFAULT_AXIS_MASK,
 		},
 		.fs = {
-			.addr = ST_GYRO_3_FS_ADDR,
-			.mask = ST_GYRO_3_FS_MASK,
+			.addr = 0x23,
+			.mask = 0x30,
 			.fs_avl = {
 				[0] = {
 					.num = ST_GYRO_FS_AVL_250DPS,
-					.value = ST_GYRO_3_FS_AVL_250_VAL,
-					.gain = ST_GYRO_3_FS_AVL_250_GAIN,
+					.value = 0x00,
+					.gain = IIO_DEGREE_TO_RAD(8750),
 				},
 				[1] = {
 					.num = ST_GYRO_FS_AVL_500DPS,
-					.value = ST_GYRO_3_FS_AVL_500_VAL,
-					.gain = ST_GYRO_3_FS_AVL_500_GAIN,
+					.value = 0x01,
+					.gain = IIO_DEGREE_TO_RAD(17500),
 				},
 				[2] = {
 					.num = ST_GYRO_FS_AVL_2000DPS,
-					.value = ST_GYRO_3_FS_AVL_2000_VAL,
-					.gain = ST_GYRO_3_FS_AVL_2000_GAIN,
+					.value = 0x02,
+					.gain = IIO_DEGREE_TO_RAD(70000),
 				},
 			},
 		},
 		.bdu = {
-			.addr = ST_GYRO_3_BDU_ADDR,
-			.mask = ST_GYRO_3_BDU_MASK,
+			.addr = 0x23,
+			.mask = 0x80,
 		},
 		.drdy_irq = {
-			.addr = ST_GYRO_3_DRDY_IRQ_ADDR,
-			.mask_int2 = ST_GYRO_3_DRDY_IRQ_INT2_MASK,
+			.addr = 0x22,
+			.mask_int2 = 0x08,
 			/*
 			 * The sensor has IHL (active low) and open
 			 * drain settings, but only for INT1 and not
@@ -327,7 +254,7 @@ static const struct st_sensor_settings st_gyro_sensors_settings[] = {
 			 */
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 		},
-		.multi_read_bit = ST_GYRO_3_MULTIREAD_BIT,
+		.multi_read_bit = true,
 		.bootime = 2,
 	},
 };
diff --git a/drivers/iio/humidity/Kconfig b/drivers/iio/humidity/Kconfig
index b17e2e2bd4f5..912477d54be2 100644
--- a/drivers/iio/humidity/Kconfig
+++ b/drivers/iio/humidity/Kconfig
@@ -27,6 +27,8 @@ config DHT11
 config HDC100X
 	tristate "TI HDC100x relative humidity and temperature sensor"
 	depends on I2C
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
 	help
 	  Say yes here to build support for the Texas Instruments
 	  HDC1000 and HDC1008 relative humidity and temperature sensors.
@@ -34,6 +36,28 @@ config HDC100X
 	  To compile this driver as a module, choose M here: the module
 	  will be called hdc100x.
 
+config HTS221
+	tristate "STMicroelectronics HTS221 sensor Driver"
+	depends on (I2C || SPI)
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	select HTS221_I2C if (I2C)
+	select HTS221_SPI if (SPI_MASTER)
+	help
+	  Say yes here to build support for STMicroelectronics HTS221
+	  temperature-humidity sensor
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called hts221.
+
+config HTS221_I2C
+	tristate
+	depends on HTS221
+
+config HTS221_SPI
+	tristate
+	depends on HTS221
+
 config HTU21
 	tristate "Measurement Specialties HTU21 humidity & temperature sensor"
 	depends on I2C
diff --git a/drivers/iio/humidity/Makefile b/drivers/iio/humidity/Makefile
index 4a73442fcd9c..a6850e47c100 100644
--- a/drivers/iio/humidity/Makefile
+++ b/drivers/iio/humidity/Makefile
@@ -5,6 +5,13 @@
 obj-$(CONFIG_AM2315) += am2315.o
 obj-$(CONFIG_DHT11) += dht11.o
 obj-$(CONFIG_HDC100X) += hdc100x.o
+
+hts221-y := hts221_core.o \
+	    hts221_buffer.o
+obj-$(CONFIG_HTS221) += hts221.o
+obj-$(CONFIG_HTS221_I2C) += hts221_i2c.o
+obj-$(CONFIG_HTS221_SPI) += hts221_spi.o
+
 obj-$(CONFIG_HTU21) += htu21.o
 obj-$(CONFIG_SI7005) += si7005.o
 obj-$(CONFIG_SI7020) += si7020.o
diff --git a/drivers/iio/humidity/hdc100x.c b/drivers/iio/humidity/hdc100x.c
index e0c9c70c2a4a..265c34da52d1 100644
--- a/drivers/iio/humidity/hdc100x.c
+++ b/drivers/iio/humidity/hdc100x.c
@@ -22,11 +22,15 @@
 
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
 
 #define HDC100X_REG_TEMP			0x00
 #define HDC100X_REG_HUMIDITY			0x01
 
 #define HDC100X_REG_CONFIG			0x02
+#define HDC100X_REG_CONFIG_ACQ_MODE		BIT(12)
 #define HDC100X_REG_CONFIG_HEATER_EN		BIT(13)
 
 struct hdc100x_data {
@@ -87,22 +91,40 @@ static const struct iio_chan_spec hdc100x_channels[] = {
 			BIT(IIO_CHAN_INFO_SCALE) |
 			BIT(IIO_CHAN_INFO_INT_TIME) |
 			BIT(IIO_CHAN_INFO_OFFSET),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_BE,
+		},
 	},
 	{
 		.type = IIO_HUMIDITYRELATIVE,
 		.address = HDC100X_REG_HUMIDITY,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
 			BIT(IIO_CHAN_INFO_SCALE) |
-			BIT(IIO_CHAN_INFO_INT_TIME)
+			BIT(IIO_CHAN_INFO_INT_TIME),
+		.scan_index = 1,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_BE,
+		},
 	},
 	{
 		.type = IIO_CURRENT,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
 		.extend_name = "heater",
 		.output = 1,
+		.scan_index = -1,
 	},
+	IIO_CHAN_SOFT_TIMESTAMP(2),
 };
 
+static const unsigned long hdc100x_scan_masks[] = {0x3, 0};
+
 static int hdc100x_update_config(struct hdc100x_data *data, int mask, int val)
 {
 	int tmp = (~mask & data->config) | val;
@@ -183,7 +205,14 @@ static int hdc100x_read_raw(struct iio_dev *indio_dev,
 			*val = hdc100x_get_heater_status(data);
 			ret = IIO_VAL_INT;
 		} else {
+			ret = iio_device_claim_direct_mode(indio_dev);
+			if (ret) {
+				mutex_unlock(&data->lock);
+				return ret;
+			}
+
 			ret = hdc100x_get_measurement(data, chan);
+			iio_device_release_direct_mode(indio_dev);
 			if (ret >= 0) {
 				*val = ret;
 				ret = IIO_VAL_INT;
@@ -246,6 +275,78 @@ static int hdc100x_write_raw(struct iio_dev *indio_dev,
 	}
 }
 
+static int hdc100x_buffer_postenable(struct iio_dev *indio_dev)
+{
+	struct hdc100x_data *data = iio_priv(indio_dev);
+	int ret;
+
+	/* Buffer is enabled. First set ACQ Mode, then attach poll func */
+	mutex_lock(&data->lock);
+	ret = hdc100x_update_config(data, HDC100X_REG_CONFIG_ACQ_MODE,
+				    HDC100X_REG_CONFIG_ACQ_MODE);
+	mutex_unlock(&data->lock);
+	if (ret)
+		return ret;
+
+	return iio_triggered_buffer_postenable(indio_dev);
+}
+
+static int hdc100x_buffer_predisable(struct iio_dev *indio_dev)
+{
+	struct hdc100x_data *data = iio_priv(indio_dev);
+	int ret;
+
+	/* First detach poll func, then reset ACQ mode. OK to disable buffer */
+	ret = iio_triggered_buffer_predisable(indio_dev);
+	if (ret)
+		return ret;
+
+	mutex_lock(&data->lock);
+	ret = hdc100x_update_config(data, HDC100X_REG_CONFIG_ACQ_MODE, 0);
+	mutex_unlock(&data->lock);
+
+	return ret;
+}
+
+static const struct iio_buffer_setup_ops hdc_buffer_setup_ops = {
+	.postenable  = hdc100x_buffer_postenable,
+	.predisable  = hdc100x_buffer_predisable,
+};
+
+static irqreturn_t hdc100x_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct hdc100x_data *data = iio_priv(indio_dev);
+	struct i2c_client *client = data->client;
+	int delay = data->adc_int_us[0] + data->adc_int_us[1];
+	int ret;
+	s16 buf[8];  /* 2x s16 + padding + 8 byte timestamp */
+
+	/* dual read starts at temp register */
+	mutex_lock(&data->lock);
+	ret = i2c_smbus_write_byte(client, HDC100X_REG_TEMP);
+	if (ret < 0) {
+		dev_err(&client->dev, "cannot start measurement\n");
+		goto err;
+	}
+	usleep_range(delay, delay + 1000);
+
+	ret = i2c_master_recv(client, (u8 *)buf, 4);
+	if (ret < 0) {
+		dev_err(&client->dev, "cannot read sensor data\n");
+		goto err;
+	}
+
+	iio_push_to_buffers_with_timestamp(indio_dev, buf,
+					   iio_get_time_ns(indio_dev));
+err:
+	mutex_unlock(&data->lock);
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
 static const struct iio_info hdc100x_info = {
 	.read_raw = hdc100x_read_raw,
 	.write_raw = hdc100x_write_raw,
@@ -258,6 +359,7 @@ static int hdc100x_probe(struct i2c_client *client,
 {
 	struct iio_dev *indio_dev;
 	struct hdc100x_data *data;
+	int ret;
 
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_WORD_DATA |
 				     I2C_FUNC_SMBUS_BYTE | I2C_FUNC_I2C))
@@ -279,12 +381,35 @@ static int hdc100x_probe(struct i2c_client *client,
 
 	indio_dev->channels = hdc100x_channels;
 	indio_dev->num_channels = ARRAY_SIZE(hdc100x_channels);
+	indio_dev->available_scan_masks = hdc100x_scan_masks;
 
 	/* be sure we are in a known state */
 	hdc100x_set_it_time(data, 0, hdc100x_int_time[0][0]);
 	hdc100x_set_it_time(data, 1, hdc100x_int_time[1][0]);
+	hdc100x_update_config(data, HDC100X_REG_CONFIG_ACQ_MODE, 0);
+
+	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+					 hdc100x_trigger_handler,
+					 &hdc_buffer_setup_ops);
+	if (ret < 0) {
+		dev_err(&client->dev, "iio triggered buffer setup failed\n");
+		return ret;
+	}
+	ret = iio_device_register(indio_dev);
+	if (ret < 0)
+		iio_triggered_buffer_cleanup(indio_dev);
+
+	return ret;
+}
+
+static int hdc100x_remove(struct i2c_client *client)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(client);
+
+	iio_device_unregister(indio_dev);
+	iio_triggered_buffer_cleanup(indio_dev);
 
-	return devm_iio_device_register(&client->dev, indio_dev);
+	return 0;
 }
 
 static const struct i2c_device_id hdc100x_id[] = {
@@ -298,6 +423,7 @@ static struct i2c_driver hdc100x_driver = {
 		.name	= "hdc100x",
 	},
 	.probe = hdc100x_probe,
+	.remove = hdc100x_remove,
 	.id_table = hdc100x_id,
 };
 module_i2c_driver(hdc100x_driver);
diff --git a/drivers/iio/humidity/hts221.h b/drivers/iio/humidity/hts221.h
new file mode 100644
index 000000000000..c7154665512e
--- /dev/null
+++ b/drivers/iio/humidity/hts221.h
@@ -0,0 +1,73 @@
+/*
+ * STMicroelectronics hts221 sensor driver
+ *
+ * Copyright 2016 STMicroelectronics Inc.
+ *
+ * Lorenzo Bianconi <lorenzo.bianconi@st.com>
+ *
+ * Licensed under the GPL-2.
+ */
+
+#ifndef HTS221_H
+#define HTS221_H
+
+#define HTS221_DEV_NAME		"hts221"
+
+#include <linux/iio/iio.h>
+
+#define HTS221_RX_MAX_LENGTH	8
+#define HTS221_TX_MAX_LENGTH	8
+
+#define HTS221_DATA_SIZE	2
+
+struct hts221_transfer_buffer {
+	u8 rx_buf[HTS221_RX_MAX_LENGTH];
+	u8 tx_buf[HTS221_TX_MAX_LENGTH] ____cacheline_aligned;
+};
+
+struct hts221_transfer_function {
+	int (*read)(struct device *dev, u8 addr, int len, u8 *data);
+	int (*write)(struct device *dev, u8 addr, int len, u8 *data);
+};
+
+#define HTS221_AVG_DEPTH	8
+struct hts221_avg_avl {
+	u16 avg;
+	u8 val;
+};
+
+enum hts221_sensor_type {
+	HTS221_SENSOR_H,
+	HTS221_SENSOR_T,
+	HTS221_SENSOR_MAX,
+};
+
+struct hts221_sensor {
+	u8 cur_avg_idx;
+	int slope, b_gen;
+};
+
+struct hts221_hw {
+	const char *name;
+	struct device *dev;
+
+	struct mutex lock;
+	struct iio_trigger *trig;
+	int irq;
+
+	struct hts221_sensor sensors[HTS221_SENSOR_MAX];
+
+	u8 odr;
+
+	const struct hts221_transfer_function *tf;
+	struct hts221_transfer_buffer tb;
+};
+
+int hts221_config_drdy(struct hts221_hw *hw, bool enable);
+int hts221_probe(struct iio_dev *iio_dev);
+int hts221_power_on(struct hts221_hw *hw);
+int hts221_power_off(struct hts221_hw *hw);
+int hts221_allocate_buffers(struct hts221_hw *hw);
+int hts221_allocate_trigger(struct hts221_hw *hw);
+
+#endif /* HTS221_H */
diff --git a/drivers/iio/humidity/hts221_buffer.c b/drivers/iio/humidity/hts221_buffer.c
new file mode 100644
index 000000000000..72ddcdac21a2
--- /dev/null
+++ b/drivers/iio/humidity/hts221_buffer.c
@@ -0,0 +1,168 @@
+/*
+ * STMicroelectronics hts221 sensor driver
+ *
+ * Copyright 2016 STMicroelectronics Inc.
+ *
+ * Lorenzo Bianconi <lorenzo.bianconi@st.com>
+ *
+ * Licensed under the GPL-2.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/irqreturn.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/events.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/buffer.h>
+
+#include "hts221.h"
+
+#define HTS221_REG_STATUS_ADDR		0x27
+#define HTS221_RH_DRDY_MASK		BIT(1)
+#define HTS221_TEMP_DRDY_MASK		BIT(0)
+
+static int hts221_trig_set_state(struct iio_trigger *trig, bool state)
+{
+	struct iio_dev *iio_dev = iio_trigger_get_drvdata(trig);
+	struct hts221_hw *hw = iio_priv(iio_dev);
+
+	return hts221_config_drdy(hw, state);
+}
+
+static const struct iio_trigger_ops hts221_trigger_ops = {
+	.owner = THIS_MODULE,
+	.set_trigger_state = hts221_trig_set_state,
+};
+
+static irqreturn_t hts221_trigger_handler_thread(int irq, void *private)
+{
+	struct hts221_hw *hw = (struct hts221_hw *)private;
+	u8 status;
+	int err;
+
+	err = hw->tf->read(hw->dev, HTS221_REG_STATUS_ADDR, sizeof(status),
+			   &status);
+	if (err < 0)
+		return IRQ_HANDLED;
+
+	/* 
+	 * H_DA bit (humidity data available) is routed to DRDY line.
+	 * Humidity sample is computed after temperature one.
+	 * Here we can assume data channels are both available if H_DA bit
+	 * is set in status register
+	 */
+	if (!(status & HTS221_RH_DRDY_MASK))
+		return IRQ_NONE;
+
+	iio_trigger_poll_chained(hw->trig);
+
+	return IRQ_HANDLED;
+}
+
+int hts221_allocate_trigger(struct hts221_hw *hw)
+{
+	struct iio_dev *iio_dev = iio_priv_to_dev(hw);
+	unsigned long irq_type;
+	int err;
+
+	irq_type = irqd_get_trigger_type(irq_get_irq_data(hw->irq));
+
+	switch (irq_type) {
+	case IRQF_TRIGGER_HIGH:
+	case IRQF_TRIGGER_RISING:
+		break;
+	default:
+		dev_info(hw->dev,
+			 "mode %lx unsupported, using IRQF_TRIGGER_RISING\n",
+			 irq_type);
+		irq_type = IRQF_TRIGGER_RISING;
+		break;
+	}
+
+	err = devm_request_threaded_irq(hw->dev, hw->irq, NULL,
+					hts221_trigger_handler_thread,
+					irq_type | IRQF_ONESHOT,
+					hw->name, hw);
+	if (err) {
+		dev_err(hw->dev, "failed to request trigger irq %d\n",
+			hw->irq);
+		return err;
+	}
+
+	hw->trig = devm_iio_trigger_alloc(hw->dev, "%s-trigger",
+					  iio_dev->name);
+	if (!hw->trig)
+		return -ENOMEM;
+
+	iio_trigger_set_drvdata(hw->trig, iio_dev);
+	hw->trig->ops = &hts221_trigger_ops;
+	hw->trig->dev.parent = hw->dev;
+	iio_dev->trig = iio_trigger_get(hw->trig);
+
+	return devm_iio_trigger_register(hw->dev, hw->trig);
+}
+
+static int hts221_buffer_preenable(struct iio_dev *iio_dev)
+{
+	return hts221_power_on(iio_priv(iio_dev));
+}
+
+static int hts221_buffer_postdisable(struct iio_dev *iio_dev)
+{
+	return hts221_power_off(iio_priv(iio_dev));
+}
+
+static const struct iio_buffer_setup_ops hts221_buffer_ops = {
+	.preenable = hts221_buffer_preenable,
+	.postenable = iio_triggered_buffer_postenable,
+	.predisable = iio_triggered_buffer_predisable,
+	.postdisable = hts221_buffer_postdisable,
+};
+
+static irqreturn_t hts221_buffer_handler_thread(int irq, void *p)
+{
+	u8 buffer[ALIGN(2 * HTS221_DATA_SIZE, sizeof(s64)) + sizeof(s64)];
+	struct iio_poll_func *pf = p;
+	struct iio_dev *iio_dev = pf->indio_dev;
+	struct hts221_hw *hw = iio_priv(iio_dev);
+	struct iio_chan_spec const *ch;
+	int err;
+
+	/* humidity data */
+	ch = &iio_dev->channels[HTS221_SENSOR_H];
+	err = hw->tf->read(hw->dev, ch->address, HTS221_DATA_SIZE,
+			   buffer);
+	if (err < 0)
+		goto out;
+
+	/* temperature data */
+	ch = &iio_dev->channels[HTS221_SENSOR_T];
+	err = hw->tf->read(hw->dev, ch->address, HTS221_DATA_SIZE,
+			   buffer + HTS221_DATA_SIZE);
+	if (err < 0)
+		goto out;
+
+	iio_push_to_buffers_with_timestamp(iio_dev, buffer,
+					   iio_get_time_ns(iio_dev));
+
+out:
+	iio_trigger_notify_done(hw->trig);
+
+	return IRQ_HANDLED;
+}
+
+int hts221_allocate_buffers(struct hts221_hw *hw)
+{
+	return devm_iio_triggered_buffer_setup(hw->dev, iio_priv_to_dev(hw),
+					NULL, hts221_buffer_handler_thread,
+					&hts221_buffer_ops);
+}
+
+MODULE_AUTHOR("Lorenzo Bianconi <lorenzo.bianconi@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics hts221 buffer driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/humidity/hts221_core.c b/drivers/iio/humidity/hts221_core.c
new file mode 100644
index 000000000000..3f3ef4a1a474
--- /dev/null
+++ b/drivers/iio/humidity/hts221_core.c
@@ -0,0 +1,687 @@
+/*
+ * STMicroelectronics hts221 sensor driver
+ *
+ * Copyright 2016 STMicroelectronics Inc.
+ *
+ * Lorenzo Bianconi <lorenzo.bianconi@st.com>
+ *
+ * Licensed under the GPL-2.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/iio/sysfs.h>
+#include <linux/delay.h>
+#include <asm/unaligned.h>
+
+#include "hts221.h"
+
+#define HTS221_REG_WHOAMI_ADDR		0x0f
+#define HTS221_REG_WHOAMI_VAL		0xbc
+
+#define HTS221_REG_CNTRL1_ADDR		0x20
+#define HTS221_REG_CNTRL2_ADDR		0x21
+#define HTS221_REG_CNTRL3_ADDR		0x22
+
+#define HTS221_REG_AVG_ADDR		0x10
+#define HTS221_REG_H_OUT_L		0x28
+#define HTS221_REG_T_OUT_L		0x2a
+
+#define HTS221_HUMIDITY_AVG_MASK	0x07
+#define HTS221_TEMP_AVG_MASK		0x38
+
+#define HTS221_ODR_MASK			0x87
+#define HTS221_BDU_MASK			BIT(2)
+
+#define HTS221_DRDY_MASK		BIT(2)
+
+#define HTS221_ENABLE_SENSOR		BIT(7)
+
+#define HTS221_HUMIDITY_AVG_4		0x00 /* 0.4 %RH */
+#define HTS221_HUMIDITY_AVG_8		0x01 /* 0.3 %RH */
+#define HTS221_HUMIDITY_AVG_16		0x02 /* 0.2 %RH */
+#define HTS221_HUMIDITY_AVG_32		0x03 /* 0.15 %RH */
+#define HTS221_HUMIDITY_AVG_64		0x04 /* 0.1 %RH */
+#define HTS221_HUMIDITY_AVG_128		0x05 /* 0.07 %RH */
+#define HTS221_HUMIDITY_AVG_256		0x06 /* 0.05 %RH */
+#define HTS221_HUMIDITY_AVG_512		0x07 /* 0.03 %RH */
+
+#define HTS221_TEMP_AVG_2		0x00 /* 0.08 degC */
+#define HTS221_TEMP_AVG_4		0x08 /* 0.05 degC */
+#define HTS221_TEMP_AVG_8		0x10 /* 0.04 degC */
+#define HTS221_TEMP_AVG_16		0x18 /* 0.03 degC */
+#define HTS221_TEMP_AVG_32		0x20 /* 0.02 degC */
+#define HTS221_TEMP_AVG_64		0x28 /* 0.015 degC */
+#define HTS221_TEMP_AVG_128		0x30 /* 0.01 degC */
+#define HTS221_TEMP_AVG_256		0x38 /* 0.007 degC */
+
+/* calibration registers */
+#define HTS221_REG_0RH_CAL_X_H		0x36
+#define HTS221_REG_1RH_CAL_X_H		0x3a
+#define HTS221_REG_0RH_CAL_Y_H		0x30
+#define HTS221_REG_1RH_CAL_Y_H		0x31
+#define HTS221_REG_0T_CAL_X_L		0x3c
+#define HTS221_REG_1T_CAL_X_L		0x3e
+#define HTS221_REG_0T_CAL_Y_H		0x32
+#define HTS221_REG_1T_CAL_Y_H		0x33
+#define HTS221_REG_T1_T0_CAL_Y_H	0x35
+
+struct hts221_odr {
+	u8 hz;
+	u8 val;
+};
+
+struct hts221_avg {
+	u8 addr;
+	u8 mask;
+	struct hts221_avg_avl avg_avl[HTS221_AVG_DEPTH];
+};
+
+static const struct hts221_odr hts221_odr_table[] = {
+	{  1, 0x01 },	/* 1Hz */
+	{  7, 0x02 },	/* 7Hz */
+	{ 13, 0x03 },	/* 12.5Hz */
+};
+
+static const struct hts221_avg hts221_avg_list[] = {
+	{
+		.addr = HTS221_REG_AVG_ADDR,
+		.mask = HTS221_HUMIDITY_AVG_MASK,
+		.avg_avl = {
+			{ 4, HTS221_HUMIDITY_AVG_4 },
+			{ 8, HTS221_HUMIDITY_AVG_8 },
+			{ 16, HTS221_HUMIDITY_AVG_16 },
+			{ 32, HTS221_HUMIDITY_AVG_32 },
+			{ 64, HTS221_HUMIDITY_AVG_64 },
+			{ 128, HTS221_HUMIDITY_AVG_128 },
+			{ 256, HTS221_HUMIDITY_AVG_256 },
+			{ 512, HTS221_HUMIDITY_AVG_512 },
+		},
+	},
+	{
+		.addr = HTS221_REG_AVG_ADDR,
+		.mask = HTS221_TEMP_AVG_MASK,
+		.avg_avl = {
+			{ 2, HTS221_TEMP_AVG_2 },
+			{ 4, HTS221_TEMP_AVG_4 },
+			{ 8, HTS221_TEMP_AVG_8 },
+			{ 16, HTS221_TEMP_AVG_16 },
+			{ 32, HTS221_TEMP_AVG_32 },
+			{ 64, HTS221_TEMP_AVG_64 },
+			{ 128, HTS221_TEMP_AVG_128 },
+			{ 256, HTS221_TEMP_AVG_256 },
+		},
+	},
+};
+
+static const struct iio_chan_spec hts221_channels[] = {
+	{
+		.type = IIO_HUMIDITYRELATIVE,
+		.address = HTS221_REG_H_OUT_L,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_OFFSET) |
+				      BIT(IIO_CHAN_INFO_SCALE) |
+				      BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_LE,
+		},
+	},
+	{
+		.type = IIO_TEMP,
+		.address = HTS221_REG_T_OUT_L,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_OFFSET) |
+				      BIT(IIO_CHAN_INFO_SCALE) |
+				      BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.scan_index = 1,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_LE,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(2),
+};
+
+static int hts221_write_with_mask(struct hts221_hw *hw, u8 addr, u8 mask,
+				  u8 val)
+{
+	u8 data;
+	int err;
+
+	mutex_lock(&hw->lock);
+
+	err = hw->tf->read(hw->dev, addr, sizeof(data), &data);
+	if (err < 0) {
+		dev_err(hw->dev, "failed to read %02x register\n", addr);
+		goto unlock;
+	}
+
+	data = (data & ~mask) | (val & mask);
+
+	err = hw->tf->write(hw->dev, addr, sizeof(data), &data);
+	if (err < 0)
+		dev_err(hw->dev, "failed to write %02x register\n", addr);
+
+unlock:
+	mutex_unlock(&hw->lock);
+
+	return err;
+}
+
+static int hts221_check_whoami(struct hts221_hw *hw)
+{
+	u8 data;
+	int err;
+
+	err = hw->tf->read(hw->dev, HTS221_REG_WHOAMI_ADDR, sizeof(data),
+			   &data);
+	if (err < 0) {
+		dev_err(hw->dev, "failed to read whoami register\n");
+		return err;
+	}
+
+	if (data != HTS221_REG_WHOAMI_VAL) {
+		dev_err(hw->dev, "wrong whoami {%02x vs %02x}\n",
+			data, HTS221_REG_WHOAMI_VAL);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+int hts221_config_drdy(struct hts221_hw *hw, bool enable)
+{
+	u8 val = enable ? BIT(2) : 0;
+	int err;
+
+	err = hts221_write_with_mask(hw, HTS221_REG_CNTRL3_ADDR,
+				     HTS221_DRDY_MASK, val);
+
+	return err < 0 ? err : 0;
+}
+
+static int hts221_update_odr(struct hts221_hw *hw, u8 odr)
+{
+	int i, err;
+	u8 val;
+
+	for (i = 0; i < ARRAY_SIZE(hts221_odr_table); i++)
+		if (hts221_odr_table[i].hz == odr)
+			break;
+
+	if (i == ARRAY_SIZE(hts221_odr_table))
+		return -EINVAL;
+
+	val = HTS221_ENABLE_SENSOR | HTS221_BDU_MASK | hts221_odr_table[i].val;
+	err = hts221_write_with_mask(hw, HTS221_REG_CNTRL1_ADDR,
+				     HTS221_ODR_MASK, val);
+	if (err < 0)
+		return err;
+
+	hw->odr = odr;
+
+	return 0;
+}
+
+static int hts221_update_avg(struct hts221_hw *hw,
+			     enum hts221_sensor_type type,
+			     u16 val)
+{
+	int i, err;
+	const struct hts221_avg *avg = &hts221_avg_list[type];
+
+	for (i = 0; i < HTS221_AVG_DEPTH; i++)
+		if (avg->avg_avl[i].avg == val)
+			break;
+
+	if (i == HTS221_AVG_DEPTH)
+		return -EINVAL;
+
+	err = hts221_write_with_mask(hw, avg->addr, avg->mask,
+				     avg->avg_avl[i].val);
+	if (err < 0)
+		return err;
+
+	hw->sensors[type].cur_avg_idx = i;
+
+	return 0;
+}
+
+static ssize_t hts221_sysfs_sampling_freq(struct device *dev,
+					  struct device_attribute *attr,
+					  char *buf)
+{
+	int i;
+	ssize_t len = 0;
+
+	for (i = 0; i < ARRAY_SIZE(hts221_odr_table); i++)
+		len += scnprintf(buf + len, PAGE_SIZE - len, "%d ",
+				 hts221_odr_table[i].hz);
+	buf[len - 1] = '\n';
+
+	return len;
+}
+
+static ssize_t
+hts221_sysfs_rh_oversampling_avail(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	const struct hts221_avg *avg = &hts221_avg_list[HTS221_SENSOR_H];
+	ssize_t len = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(avg->avg_avl); i++)
+		len += scnprintf(buf + len, PAGE_SIZE - len, "%d ",
+				 avg->avg_avl[i].avg);
+	buf[len - 1] = '\n';
+
+	return len;
+}
+
+static ssize_t
+hts221_sysfs_temp_oversampling_avail(struct device *dev,
+				     struct device_attribute *attr,
+				     char *buf)
+{
+	const struct hts221_avg *avg = &hts221_avg_list[HTS221_SENSOR_T];
+	ssize_t len = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(avg->avg_avl); i++)
+		len += scnprintf(buf + len, PAGE_SIZE - len, "%d ",
+				 avg->avg_avl[i].avg);
+	buf[len - 1] = '\n';
+
+	return len;
+}
+
+int hts221_power_on(struct hts221_hw *hw)
+{
+	return hts221_update_odr(hw, hw->odr);
+}
+
+int hts221_power_off(struct hts221_hw *hw)
+{
+	u8 data[] = {0x00, 0x00};
+
+	return hw->tf->write(hw->dev, HTS221_REG_CNTRL1_ADDR, sizeof(data),
+			     data);
+}
+
+static int hts221_parse_temp_caldata(struct hts221_hw *hw)
+{
+	int err, *slope, *b_gen;
+	s16 cal_x0, cal_x1, cal_y0, cal_y1;
+	u8 cal0, cal1;
+
+	err = hw->tf->read(hw->dev, HTS221_REG_0T_CAL_Y_H,
+			   sizeof(cal0), &cal0);
+	if (err < 0)
+		return err;
+
+	err = hw->tf->read(hw->dev, HTS221_REG_T1_T0_CAL_Y_H,
+			   sizeof(cal1), &cal1);
+	if (err < 0)
+		return err;
+	cal_y0 = (le16_to_cpu(cal1 & 0x3) << 8) | cal0;
+
+	err = hw->tf->read(hw->dev, HTS221_REG_1T_CAL_Y_H,
+			   sizeof(cal0), &cal0);
+	if (err < 0)
+		return err;
+	cal_y1 = (((cal1 & 0xc) >> 2) << 8) | cal0;
+
+	err = hw->tf->read(hw->dev, HTS221_REG_0T_CAL_X_L, sizeof(cal_x0),
+			   (u8 *)&cal_x0);
+	if (err < 0)
+		return err;
+	cal_x0 = le16_to_cpu(cal_x0);
+
+	err = hw->tf->read(hw->dev, HTS221_REG_1T_CAL_X_L, sizeof(cal_x1),
+			   (u8 *)&cal_x1);
+	if (err < 0)
+		return err;
+	cal_x1 = le16_to_cpu(cal_x1);
+
+	slope = &hw->sensors[HTS221_SENSOR_T].slope;
+	b_gen = &hw->sensors[HTS221_SENSOR_T].b_gen;
+
+	*slope = ((cal_y1 - cal_y0) * 8000) / (cal_x1 - cal_x0);
+	*b_gen = (((s32)cal_x1 * cal_y0 - (s32)cal_x0 * cal_y1) * 1000) /
+		 (cal_x1 - cal_x0);
+	*b_gen *= 8;
+
+	return 0;
+}
+
+static int hts221_parse_rh_caldata(struct hts221_hw *hw)
+{
+	int err, *slope, *b_gen;
+	s16 cal_x0, cal_x1, cal_y0, cal_y1;
+	u8 data;
+
+	err = hw->tf->read(hw->dev, HTS221_REG_0RH_CAL_Y_H, sizeof(data),
+			   &data);
+	if (err < 0)
+		return err;
+	cal_y0 = data;
+
+	err = hw->tf->read(hw->dev, HTS221_REG_1RH_CAL_Y_H, sizeof(data),
+			   &data);
+	if (err < 0)
+		return err;
+	cal_y1 = data;
+
+	err = hw->tf->read(hw->dev, HTS221_REG_0RH_CAL_X_H, sizeof(cal_x0),
+			   (u8 *)&cal_x0);
+	if (err < 0)
+		return err;
+	cal_x0 = le16_to_cpu(cal_x0);
+
+	err = hw->tf->read(hw->dev, HTS221_REG_1RH_CAL_X_H, sizeof(cal_x1),
+			   (u8 *)&cal_x1);
+	if (err < 0)
+		return err;
+	cal_x1 = le16_to_cpu(cal_x1);
+
+	slope = &hw->sensors[HTS221_SENSOR_H].slope;
+	b_gen = &hw->sensors[HTS221_SENSOR_H].b_gen;
+
+	*slope = ((cal_y1 - cal_y0) * 8000) / (cal_x1 - cal_x0);
+	*b_gen = (((s32)cal_x1 * cal_y0 - (s32)cal_x0 * cal_y1) * 1000) /
+		 (cal_x1 - cal_x0);
+	*b_gen *= 8;
+
+	return 0;
+}
+
+static int hts221_get_sensor_scale(struct hts221_hw *hw,
+				   enum iio_chan_type ch_type,
+				   int *val, int *val2)
+{
+	s64 tmp;
+	s32 rem, div, data;
+
+	switch (ch_type) {
+	case IIO_HUMIDITYRELATIVE:
+		data = hw->sensors[HTS221_SENSOR_H].slope;
+		div = (1 << 4) * 1000;
+		break;
+	case IIO_TEMP:
+		data = hw->sensors[HTS221_SENSOR_T].slope;
+		div = (1 << 6) * 1000;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	tmp = div_s64(data * 1000000000LL, div);
+	tmp = div_s64_rem(tmp, 1000000000LL, &rem);
+
+	*val = tmp;
+	*val2 = rem;
+
+	return IIO_VAL_INT_PLUS_NANO;
+}
+
+static int hts221_get_sensor_offset(struct hts221_hw *hw,
+				    enum iio_chan_type ch_type,
+				    int *val, int *val2)
+{
+	s64 tmp;
+	s32 rem, div, data;
+
+	switch (ch_type) {
+	case IIO_HUMIDITYRELATIVE:
+		data = hw->sensors[HTS221_SENSOR_H].b_gen;
+		div = hw->sensors[HTS221_SENSOR_H].slope;
+		break;
+	case IIO_TEMP:
+		data = hw->sensors[HTS221_SENSOR_T].b_gen;
+		div = hw->sensors[HTS221_SENSOR_T].slope;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	tmp = div_s64(data * 1000000000LL, div);
+	tmp = div_s64_rem(tmp, 1000000000LL, &rem);
+
+	*val = tmp;
+	*val2 = rem;
+
+	return IIO_VAL_INT_PLUS_NANO;
+}
+
+static int hts221_read_oneshot(struct hts221_hw *hw, u8 addr, int *val)
+{
+	u8 data[HTS221_DATA_SIZE];
+	int err;
+
+	err = hts221_power_on(hw);
+	if (err < 0)
+		return err;
+
+	msleep(50);
+
+	err = hw->tf->read(hw->dev, addr, sizeof(data), data);
+	if (err < 0)
+		return err;
+
+	hts221_power_off(hw);
+
+	*val = (s16)get_unaligned_le16(data);
+
+	return IIO_VAL_INT;
+}
+
+static int hts221_read_raw(struct iio_dev *iio_dev,
+			   struct iio_chan_spec const *ch,
+			   int *val, int *val2, long mask)
+{
+	struct hts221_hw *hw = iio_priv(iio_dev);
+	int ret;
+
+	ret = iio_device_claim_direct_mode(iio_dev);
+	if (ret)
+		return ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = hts221_read_oneshot(hw, ch->address, val);
+		break;
+	case IIO_CHAN_INFO_SCALE:
+		ret = hts221_get_sensor_scale(hw, ch->type, val, val2);
+		break;
+	case IIO_CHAN_INFO_OFFSET:
+		ret = hts221_get_sensor_offset(hw, ch->type, val, val2);
+		break;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*val = hw->odr;
+		ret = IIO_VAL_INT;
+		break;
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO: {
+		u8 idx;
+		const struct hts221_avg *avg;
+
+		switch (ch->type) {
+		case IIO_HUMIDITYRELATIVE:
+			avg = &hts221_avg_list[HTS221_SENSOR_H];
+			idx = hw->sensors[HTS221_SENSOR_H].cur_avg_idx;
+			*val = avg->avg_avl[idx].avg;
+			ret = IIO_VAL_INT;
+			break;
+		case IIO_TEMP:
+			avg = &hts221_avg_list[HTS221_SENSOR_T];
+			idx = hw->sensors[HTS221_SENSOR_T].cur_avg_idx;
+			*val = avg->avg_avl[idx].avg;
+			ret = IIO_VAL_INT;
+			break;
+		default:
+			ret = -EINVAL;
+			break;
+		}
+		break;
+	}
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	iio_device_release_direct_mode(iio_dev);
+
+	return ret;
+}
+
+static int hts221_write_raw(struct iio_dev *iio_dev,
+			    struct iio_chan_spec const *chan,
+			    int val, int val2, long mask)
+{
+	struct hts221_hw *hw = iio_priv(iio_dev);
+	int ret;
+
+	ret = iio_device_claim_direct_mode(iio_dev);
+	if (ret)
+		return ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		ret = hts221_update_odr(hw, val);
+		break;
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		switch (chan->type) {
+		case IIO_HUMIDITYRELATIVE:
+			ret = hts221_update_avg(hw, HTS221_SENSOR_H, val);
+			break;
+		case IIO_TEMP:
+			ret = hts221_update_avg(hw, HTS221_SENSOR_T, val);
+			break;
+		default:
+			ret = -EINVAL;
+			break;
+		}
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	iio_device_release_direct_mode(iio_dev);
+
+	return ret;
+}
+
+static int hts221_validate_trigger(struct iio_dev *iio_dev,
+				   struct iio_trigger *trig)
+{
+	struct hts221_hw *hw = iio_priv(iio_dev);
+
+	return hw->trig == trig ? 0 : -EINVAL;
+}
+
+static IIO_DEVICE_ATTR(in_humidity_oversampling_ratio_available, S_IRUGO,
+		       hts221_sysfs_rh_oversampling_avail, NULL, 0);
+static IIO_DEVICE_ATTR(in_temp_oversampling_ratio_available, S_IRUGO,
+		       hts221_sysfs_temp_oversampling_avail, NULL, 0);
+static IIO_DEV_ATTR_SAMP_FREQ_AVAIL(hts221_sysfs_sampling_freq);
+
+static struct attribute *hts221_attributes[] = {
+	&iio_dev_attr_sampling_frequency_available.dev_attr.attr,
+	&iio_dev_attr_in_humidity_oversampling_ratio_available.dev_attr.attr,
+	&iio_dev_attr_in_temp_oversampling_ratio_available.dev_attr.attr,
+	NULL,
+};
+
+static const struct attribute_group hts221_attribute_group = {
+	.attrs = hts221_attributes,
+};
+
+static const struct iio_info hts221_info = {
+	.driver_module = THIS_MODULE,
+	.attrs = &hts221_attribute_group,
+	.read_raw = hts221_read_raw,
+	.write_raw = hts221_write_raw,
+	.validate_trigger = hts221_validate_trigger,
+};
+
+static const unsigned long hts221_scan_masks[] = {0x3, 0x0};
+
+int hts221_probe(struct iio_dev *iio_dev)
+{
+	struct hts221_hw *hw = iio_priv(iio_dev);
+	int err;
+	u8 data;
+
+	mutex_init(&hw->lock);
+
+	err = hts221_check_whoami(hw);
+	if (err < 0)
+		return err;
+
+	hw->odr = hts221_odr_table[0].hz;
+
+	iio_dev->modes = INDIO_DIRECT_MODE;
+	iio_dev->dev.parent = hw->dev;
+	iio_dev->available_scan_masks = hts221_scan_masks;
+	iio_dev->channels = hts221_channels;
+	iio_dev->num_channels = ARRAY_SIZE(hts221_channels);
+	iio_dev->name = HTS221_DEV_NAME;
+	iio_dev->info = &hts221_info;
+
+	/* configure humidity sensor */
+	err = hts221_parse_rh_caldata(hw);
+	if (err < 0) {
+		dev_err(hw->dev, "failed to get rh calibration data\n");
+		return err;
+	}
+
+	data = hts221_avg_list[HTS221_SENSOR_H].avg_avl[3].avg;
+	err = hts221_update_avg(hw, HTS221_SENSOR_H, data);
+	if (err < 0) {
+		dev_err(hw->dev, "failed to set rh oversampling ratio\n");
+		return err;
+	}
+
+	/* configure temperature sensor */
+	err = hts221_parse_temp_caldata(hw);
+	if (err < 0) {
+		dev_err(hw->dev,
+			"failed to get temperature calibration data\n");
+		return err;
+	}
+
+	data = hts221_avg_list[HTS221_SENSOR_T].avg_avl[3].avg;
+	err = hts221_update_avg(hw, HTS221_SENSOR_T, data);
+	if (err < 0) {
+		dev_err(hw->dev,
+			"failed to set temperature oversampling ratio\n");
+		return err;
+	}
+
+	if (hw->irq > 0) {
+		err = hts221_allocate_buffers(hw);
+		if (err < 0)
+			return err;
+
+		err = hts221_allocate_trigger(hw);
+		if (err)
+			return err;
+	}
+
+	return devm_iio_device_register(hw->dev, iio_dev);
+}
+EXPORT_SYMBOL(hts221_probe);
+
+MODULE_AUTHOR("Lorenzo Bianconi <lorenzo.bianconi@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics hts221 sensor driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/humidity/hts221_i2c.c b/drivers/iio/humidity/hts221_i2c.c
new file mode 100644
index 000000000000..367ecd509f31
--- /dev/null
+++ b/drivers/iio/humidity/hts221_i2c.c
@@ -0,0 +1,110 @@
+/*
+ * STMicroelectronics hts221 i2c driver
+ *
+ * Copyright 2016 STMicroelectronics Inc.
+ *
+ * Lorenzo Bianconi <lorenzo.bianconi@st.com>
+ *
+ * Licensed under the GPL-2.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/slab.h>
+#include "hts221.h"
+
+#define I2C_AUTO_INCREMENT	0x80
+
+static int hts221_i2c_read(struct device *dev, u8 addr, int len, u8 *data)
+{
+	struct i2c_msg msg[2];
+	struct i2c_client *client = to_i2c_client(dev);
+
+	if (len > 1)
+		addr |= I2C_AUTO_INCREMENT;
+
+	msg[0].addr = client->addr;
+	msg[0].flags = client->flags;
+	msg[0].len = 1;
+	msg[0].buf = &addr;
+
+	msg[1].addr = client->addr;
+	msg[1].flags = client->flags | I2C_M_RD;
+	msg[1].len = len;
+	msg[1].buf = data;
+
+	return i2c_transfer(client->adapter, msg, 2);
+}
+
+static int hts221_i2c_write(struct device *dev, u8 addr, int len, u8 *data)
+{
+	u8 send[len + 1];
+	struct i2c_msg msg;
+	struct i2c_client *client = to_i2c_client(dev);
+
+	if (len > 1)
+		addr |= I2C_AUTO_INCREMENT;
+
+	send[0] = addr;
+	memcpy(&send[1], data, len * sizeof(u8));
+
+	msg.addr = client->addr;
+	msg.flags = client->flags;
+	msg.len = len + 1;
+	msg.buf = send;
+
+	return i2c_transfer(client->adapter, &msg, 1);
+}
+
+static const struct hts221_transfer_function hts221_transfer_fn = {
+	.read = hts221_i2c_read,
+	.write = hts221_i2c_write,
+};
+
+static int hts221_i2c_probe(struct i2c_client *client,
+			    const struct i2c_device_id *id)
+{
+	struct hts221_hw *hw;
+	struct iio_dev *iio_dev;
+
+	iio_dev = devm_iio_device_alloc(&client->dev, sizeof(*hw));
+	if (!iio_dev)
+		return -ENOMEM;
+
+	i2c_set_clientdata(client, iio_dev);
+
+	hw = iio_priv(iio_dev);
+	hw->name = client->name;
+	hw->dev = &client->dev;
+	hw->irq = client->irq;
+	hw->tf = &hts221_transfer_fn;
+
+	return hts221_probe(iio_dev);
+}
+
+static const struct of_device_id hts221_i2c_of_match[] = {
+	{ .compatible = "st,hts221", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, hts221_i2c_of_match);
+
+static const struct i2c_device_id hts221_i2c_id_table[] = {
+	{ HTS221_DEV_NAME },
+	{},
+};
+MODULE_DEVICE_TABLE(i2c, hts221_i2c_id_table);
+
+static struct i2c_driver hts221_driver = {
+	.driver = {
+		.name = "hts221_i2c",
+		.of_match_table = of_match_ptr(hts221_i2c_of_match),
+	},
+	.probe = hts221_i2c_probe,
+	.id_table = hts221_i2c_id_table,
+};
+module_i2c_driver(hts221_driver);
+
+MODULE_AUTHOR("Lorenzo Bianconi <lorenzo.bianconi@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics hts221 i2c driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/humidity/hts221_spi.c b/drivers/iio/humidity/hts221_spi.c
new file mode 100644
index 000000000000..70df5e7150c1
--- /dev/null
+++ b/drivers/iio/humidity/hts221_spi.c
@@ -0,0 +1,125 @@
+/*
+ * STMicroelectronics hts221 spi driver
+ *
+ * Copyright 2016 STMicroelectronics Inc.
+ *
+ * Lorenzo Bianconi <lorenzo.bianconi@st.com>
+ *
+ * Licensed under the GPL-2.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/spi/spi.h>
+#include <linux/slab.h>
+#include "hts221.h"
+
+#define SENSORS_SPI_READ	0x80
+#define SPI_AUTO_INCREMENT	0x40
+
+static int hts221_spi_read(struct device *dev, u8 addr, int len, u8 *data)
+{
+	int err;
+	struct spi_device *spi = to_spi_device(dev);
+	struct iio_dev *iio_dev = spi_get_drvdata(spi);
+	struct hts221_hw *hw = iio_priv(iio_dev);
+
+	struct spi_transfer xfers[] = {
+		{
+			.tx_buf = hw->tb.tx_buf,
+			.bits_per_word = 8,
+			.len = 1,
+		},
+		{
+			.rx_buf = hw->tb.rx_buf,
+			.bits_per_word = 8,
+			.len = len,
+		}
+	};
+
+	if (len > 1)
+		addr |= SPI_AUTO_INCREMENT;
+	hw->tb.tx_buf[0] = addr | SENSORS_SPI_READ;
+
+	err = spi_sync_transfer(spi, xfers,  ARRAY_SIZE(xfers));
+	if (err < 0)
+		return err;
+
+	memcpy(data, hw->tb.rx_buf, len * sizeof(u8));
+
+	return len;
+}
+
+static int hts221_spi_write(struct device *dev, u8 addr, int len, u8 *data)
+{
+	struct spi_device *spi = to_spi_device(dev);
+	struct iio_dev *iio_dev = spi_get_drvdata(spi);
+	struct hts221_hw *hw = iio_priv(iio_dev);
+
+	struct spi_transfer xfers = {
+		.tx_buf = hw->tb.tx_buf,
+		.bits_per_word = 8,
+		.len = len + 1,
+	};
+
+	if (len >= HTS221_TX_MAX_LENGTH)
+		return -ENOMEM;
+
+	if (len > 1)
+		addr |= SPI_AUTO_INCREMENT;
+	hw->tb.tx_buf[0] = addr;
+	memcpy(&hw->tb.tx_buf[1], data, len);
+
+	return spi_sync_transfer(spi, &xfers, 1);
+}
+
+static const struct hts221_transfer_function hts221_transfer_fn = {
+	.read = hts221_spi_read,
+	.write = hts221_spi_write,
+};
+
+static int hts221_spi_probe(struct spi_device *spi)
+{
+	struct hts221_hw *hw;
+	struct iio_dev *iio_dev;
+
+	iio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*hw));
+	if (!iio_dev)
+		return -ENOMEM;
+
+	spi_set_drvdata(spi, iio_dev);
+
+	hw = iio_priv(iio_dev);
+	hw->name = spi->modalias;
+	hw->dev = &spi->dev;
+	hw->irq = spi->irq;
+	hw->tf = &hts221_transfer_fn;
+
+	return hts221_probe(iio_dev);
+}
+
+static const struct of_device_id hts221_spi_of_match[] = {
+	{ .compatible = "st,hts221", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, hts221_spi_of_match);
+
+static const struct spi_device_id hts221_spi_id_table[] = {
+	{ HTS221_DEV_NAME },
+	{},
+};
+MODULE_DEVICE_TABLE(spi, hts221_spi_id_table);
+
+static struct spi_driver hts221_driver = {
+	.driver = {
+		.name = "hts221_spi",
+		.of_match_table = of_match_ptr(hts221_spi_of_match),
+	},
+	.probe = hts221_spi_probe,
+	.id_table = hts221_spi_id_table,
+};
+module_spi_driver(hts221_driver);
+
+MODULE_AUTHOR("Lorenzo Bianconi <lorenzo.bianconi@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics hts221 spi driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/humidity/si7020.c b/drivers/iio/humidity/si7020.c
index ffc2ccf6374e..345a7656c5ef 100644
--- a/drivers/iio/humidity/si7020.c
+++ b/drivers/iio/humidity/si7020.c
@@ -154,8 +154,17 @@ static const struct i2c_device_id si7020_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, si7020_id);
 
+static const struct of_device_id si7020_dt_ids[] = {
+	{ .compatible = "silabs,si7020" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, si7020_dt_ids);
+
 static struct i2c_driver si7020_driver = {
-	.driver.name	= "si7020",
+	.driver = {
+		.name = "si7020",
+		.of_match_table = of_match_ptr(si7020_dt_ids),
+	},
 	.probe		= si7020_probe,
 	.id_table	= si7020_id,
 };
diff --git a/drivers/iio/imu/bmi160/bmi160_core.c b/drivers/iio/imu/bmi160/bmi160_core.c
index e0251b8c1a52..5355507f8fa1 100644
--- a/drivers/iio/imu/bmi160/bmi160_core.c
+++ b/drivers/iio/imu/bmi160/bmi160_core.c
@@ -398,7 +398,8 @@ static irqreturn_t bmi160_trigger_handler(int irq, void *p)
 	struct iio_poll_func *pf = p;
 	struct iio_dev *indio_dev = pf->indio_dev;
 	struct bmi160_data *data = iio_priv(indio_dev);
-	s16 buf[16]; /* 3 sens x 3 axis x s16 + 3 x s16 pad + 4 x s16 tstamp */
+	__le16 buf[16];
+	/* 3 sens x 3 axis x __le16 + 3 x __le16 pad + 4 x __le16 tstamp */
 	int i, ret, j = 0, base = BMI160_REG_DATA_MAGN_XOUT_L;
 	__le16 sample;
 
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c
index 19580d1db597..2c3f8964a3ea 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c
@@ -126,7 +126,7 @@ static int inv_mpu_probe(struct i2c_client *client,
 
 	st = iio_priv(dev_get_drvdata(&client->dev));
 	st->muxc = i2c_mux_alloc(client->adapter, &client->dev,
-				 1, 0, I2C_MUX_LOCKED,
+				 1, 0, I2C_MUX_LOCKED | I2C_MUX_GATE,
 				 inv_mpu6050_select_bypass,
 				 inv_mpu6050_deselect_bypass);
 	if (!st->muxc) {
diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c
index 158aaf44dd95..b12830b09c7d 100644
--- a/drivers/iio/industrialio-buffer.c
+++ b/drivers/iio/industrialio-buffer.c
@@ -307,10 +307,9 @@ static int iio_scan_mask_set(struct iio_dev *indio_dev,
 	const unsigned long *mask;
 	unsigned long *trialmask;
 
-	trialmask = kmalloc(sizeof(*trialmask)*
-			    BITS_TO_LONGS(indio_dev->masklength),
-			    GFP_KERNEL);
-
+	trialmask = kmalloc_array(BITS_TO_LONGS(indio_dev->masklength),
+				  sizeof(*trialmask),
+				  GFP_KERNEL);
 	if (trialmask == NULL)
 		return -ENOMEM;
 	if (!indio_dev->masklength) {
diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c
index fc340ed3dca1..aaca42862389 100644
--- a/drivers/iio/industrialio-core.c
+++ b/drivers/iio/industrialio-core.c
@@ -81,6 +81,8 @@ static const char * const iio_chan_type_name_spec[] = {
 	[IIO_PH] = "ph",
 	[IIO_UVINDEX] = "uvindex",
 	[IIO_ELECTRICALCONDUCTIVITY] = "electricalconductivity",
+	[IIO_COUNT] = "count",
+	[IIO_INDEX] = "index",
 };
 
 static const char * const iio_modifier_names[] = {
@@ -575,66 +577,82 @@ int of_iio_read_mount_matrix(const struct device *dev,
 #endif
 EXPORT_SYMBOL(of_iio_read_mount_matrix);
 
-/**
- * iio_format_value() - Formats a IIO value into its string representation
- * @buf:	The buffer to which the formatted value gets written
- * @type:	One of the IIO_VAL_... constants. This decides how the val
- *		and val2 parameters are formatted.
- * @size:	Number of IIO value entries contained in vals
- * @vals:	Pointer to the values, exact meaning depends on the
- *		type parameter.
- *
- * Return: 0 by default, a negative number on failure or the
- *	   total number of characters written for a type that belongs
- *	   to the IIO_VAL_... constant.
- */
-ssize_t iio_format_value(char *buf, unsigned int type, int size, int *vals)
+static ssize_t __iio_format_value(char *buf, size_t len, unsigned int type,
+				  int size, const int *vals)
 {
 	unsigned long long tmp;
+	int tmp0, tmp1;
 	bool scale_db = false;
 
 	switch (type) {
 	case IIO_VAL_INT:
-		return sprintf(buf, "%d\n", vals[0]);
+		return snprintf(buf, len, "%d", vals[0]);
 	case IIO_VAL_INT_PLUS_MICRO_DB:
 		scale_db = true;
 	case IIO_VAL_INT_PLUS_MICRO:
 		if (vals[1] < 0)
-			return sprintf(buf, "-%d.%06u%s\n", abs(vals[0]),
-				       -vals[1], scale_db ? " dB" : "");
+			return snprintf(buf, len, "-%d.%06u%s", abs(vals[0]),
+					-vals[1], scale_db ? " dB" : "");
 		else
-			return sprintf(buf, "%d.%06u%s\n", vals[0], vals[1],
-				scale_db ? " dB" : "");
+			return snprintf(buf, len, "%d.%06u%s", vals[0], vals[1],
+					scale_db ? " dB" : "");
 	case IIO_VAL_INT_PLUS_NANO:
 		if (vals[1] < 0)
-			return sprintf(buf, "-%d.%09u\n", abs(vals[0]),
-				       -vals[1]);
+			return snprintf(buf, len, "-%d.%09u", abs(vals[0]),
+					-vals[1]);
 		else
-			return sprintf(buf, "%d.%09u\n", vals[0], vals[1]);
+			return snprintf(buf, len, "%d.%09u", vals[0], vals[1]);
 	case IIO_VAL_FRACTIONAL:
 		tmp = div_s64((s64)vals[0] * 1000000000LL, vals[1]);
-		vals[0] = (int)div_s64_rem(tmp, 1000000000, &vals[1]);
-		return sprintf(buf, "%d.%09u\n", vals[0], abs(vals[1]));
+		tmp1 = vals[1];
+		tmp0 = (int)div_s64_rem(tmp, 1000000000, &tmp1);
+		return snprintf(buf, len, "%d.%09u", tmp0, abs(tmp1));
 	case IIO_VAL_FRACTIONAL_LOG2:
 		tmp = (s64)vals[0] * 1000000000LL >> vals[1];
-		vals[1] = do_div(tmp, 1000000000LL);
-		vals[0] = tmp;
-		return sprintf(buf, "%d.%09u\n", vals[0], vals[1]);
+		tmp1 = do_div(tmp, 1000000000LL);
+		tmp0 = tmp;
+		return snprintf(buf, len, "%d.%09u", tmp0, tmp1);
 	case IIO_VAL_INT_MULTIPLE:
 	{
 		int i;
-		int len = 0;
+		int l = 0;
 
-		for (i = 0; i < size; ++i)
-			len += snprintf(&buf[len], PAGE_SIZE - len, "%d ",
-								vals[i]);
-		len += snprintf(&buf[len], PAGE_SIZE - len, "\n");
-		return len;
+		for (i = 0; i < size; ++i) {
+			l += snprintf(&buf[l], len - l, "%d ", vals[i]);
+			if (l >= len)
+				break;
+		}
+		return l;
 	}
 	default:
 		return 0;
 	}
 }
+
+/**
+ * iio_format_value() - Formats a IIO value into its string representation
+ * @buf:	The buffer to which the formatted value gets written
+ *		which is assumed to be big enough (i.e. PAGE_SIZE).
+ * @type:	One of the IIO_VAL_... constants. This decides how the val
+ *		and val2 parameters are formatted.
+ * @size:	Number of IIO value entries contained in vals
+ * @vals:	Pointer to the values, exact meaning depends on the
+ *		type parameter.
+ *
+ * Return: 0 by default, a negative number on failure or the
+ *	   total number of characters written for a type that belongs
+ *	   to the IIO_VAL_... constant.
+ */
+ssize_t iio_format_value(char *buf, unsigned int type, int size, int *vals)
+{
+	ssize_t len;
+
+	len = __iio_format_value(buf, PAGE_SIZE, type, size, vals);
+	if (len >= PAGE_SIZE - 1)
+		return -EFBIG;
+
+	return len + sprintf(buf + len, "\n");
+}
 EXPORT_SYMBOL_GPL(iio_format_value);
 
 static ssize_t iio_read_channel_info(struct device *dev,
@@ -662,6 +680,119 @@ static ssize_t iio_read_channel_info(struct device *dev,
 	return iio_format_value(buf, ret, val_len, vals);
 }
 
+static ssize_t iio_format_avail_list(char *buf, const int *vals,
+				     int type, int length)
+{
+	int i;
+	ssize_t len = 0;
+
+	switch (type) {
+	case IIO_VAL_INT:
+		for (i = 0; i < length; i++) {
+			len += __iio_format_value(buf + len, PAGE_SIZE - len,
+						  type, 1, &vals[i]);
+			if (len >= PAGE_SIZE)
+				return -EFBIG;
+			if (i < length - 1)
+				len += snprintf(buf + len, PAGE_SIZE - len,
+						" ");
+			else
+				len += snprintf(buf + len, PAGE_SIZE - len,
+						"\n");
+			if (len >= PAGE_SIZE)
+				return -EFBIG;
+		}
+		break;
+	default:
+		for (i = 0; i < length / 2; i++) {
+			len += __iio_format_value(buf + len, PAGE_SIZE - len,
+						  type, 2, &vals[i * 2]);
+			if (len >= PAGE_SIZE)
+				return -EFBIG;
+			if (i < length / 2 - 1)
+				len += snprintf(buf + len, PAGE_SIZE - len,
+						" ");
+			else
+				len += snprintf(buf + len, PAGE_SIZE - len,
+						"\n");
+			if (len >= PAGE_SIZE)
+				return -EFBIG;
+		}
+	}
+
+	return len;
+}
+
+static ssize_t iio_format_avail_range(char *buf, const int *vals, int type)
+{
+	int i;
+	ssize_t len;
+
+	len = snprintf(buf, PAGE_SIZE, "[");
+	switch (type) {
+	case IIO_VAL_INT:
+		for (i = 0; i < 3; i++) {
+			len += __iio_format_value(buf + len, PAGE_SIZE - len,
+						  type, 1, &vals[i]);
+			if (len >= PAGE_SIZE)
+				return -EFBIG;
+			if (i < 2)
+				len += snprintf(buf + len, PAGE_SIZE - len,
+						" ");
+			else
+				len += snprintf(buf + len, PAGE_SIZE - len,
+						"]\n");
+			if (len >= PAGE_SIZE)
+				return -EFBIG;
+		}
+		break;
+	default:
+		for (i = 0; i < 3; i++) {
+			len += __iio_format_value(buf + len, PAGE_SIZE - len,
+						  type, 2, &vals[i * 2]);
+			if (len >= PAGE_SIZE)
+				return -EFBIG;
+			if (i < 2)
+				len += snprintf(buf + len, PAGE_SIZE - len,
+						" ");
+			else
+				len += snprintf(buf + len, PAGE_SIZE - len,
+						"]\n");
+			if (len >= PAGE_SIZE)
+				return -EFBIG;
+		}
+	}
+
+	return len;
+}
+
+static ssize_t iio_read_channel_info_avail(struct device *dev,
+					   struct device_attribute *attr,
+					   char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
+	const int *vals;
+	int ret;
+	int length;
+	int type;
+
+	ret = indio_dev->info->read_avail(indio_dev, this_attr->c,
+					  &vals, &type, &length,
+					  this_attr->address);
+
+	if (ret < 0)
+		return ret;
+	switch (ret) {
+	case IIO_AVAIL_LIST:
+		return iio_format_avail_list(buf, vals, type, length);
+	case IIO_AVAIL_RANGE:
+		return iio_format_avail_range(buf, vals, type);
+	default:
+		return -EINVAL;
+	}
+}
+
 /**
  * iio_str_to_fixpoint() - Parse a fixed-point number from a string
  * @str: The string to parse
@@ -978,6 +1109,40 @@ static int iio_device_add_info_mask_type(struct iio_dev *indio_dev,
 	return attrcount;
 }
 
+static int iio_device_add_info_mask_type_avail(struct iio_dev *indio_dev,
+					       struct iio_chan_spec const *chan,
+					       enum iio_shared_by shared_by,
+					       const long *infomask)
+{
+	int i, ret, attrcount = 0;
+	char *avail_postfix;
+
+	for_each_set_bit(i, infomask, sizeof(infomask) * 8) {
+		avail_postfix = kasprintf(GFP_KERNEL,
+					  "%s_available",
+					  iio_chan_info_postfix[i]);
+		if (!avail_postfix)
+			return -ENOMEM;
+
+		ret = __iio_add_chan_devattr(avail_postfix,
+					     chan,
+					     &iio_read_channel_info_avail,
+					     NULL,
+					     i,
+					     shared_by,
+					     &indio_dev->dev,
+					     &indio_dev->channel_attr_list);
+		kfree(avail_postfix);
+		if ((ret == -EBUSY) && (shared_by != IIO_SEPARATE))
+			continue;
+		else if (ret < 0)
+			return ret;
+		attrcount++;
+	}
+
+	return attrcount;
+}
+
 static int iio_device_add_channel_sysfs(struct iio_dev *indio_dev,
 					struct iio_chan_spec const *chan)
 {
@@ -993,6 +1158,14 @@ static int iio_device_add_channel_sysfs(struct iio_dev *indio_dev,
 		return ret;
 	attrcount += ret;
 
+	ret = iio_device_add_info_mask_type_avail(indio_dev, chan,
+						  IIO_SEPARATE,
+						  &chan->
+						  info_mask_separate_available);
+	if (ret < 0)
+		return ret;
+	attrcount += ret;
+
 	ret = iio_device_add_info_mask_type(indio_dev, chan,
 					    IIO_SHARED_BY_TYPE,
 					    &chan->info_mask_shared_by_type);
@@ -1000,6 +1173,14 @@ static int iio_device_add_channel_sysfs(struct iio_dev *indio_dev,
 		return ret;
 	attrcount += ret;
 
+	ret = iio_device_add_info_mask_type_avail(indio_dev, chan,
+						  IIO_SHARED_BY_TYPE,
+						  &chan->
+						  info_mask_shared_by_type_available);
+	if (ret < 0)
+		return ret;
+	attrcount += ret;
+
 	ret = iio_device_add_info_mask_type(indio_dev, chan,
 					    IIO_SHARED_BY_DIR,
 					    &chan->info_mask_shared_by_dir);
@@ -1007,6 +1188,13 @@ static int iio_device_add_channel_sysfs(struct iio_dev *indio_dev,
 		return ret;
 	attrcount += ret;
 
+	ret = iio_device_add_info_mask_type_avail(indio_dev, chan,
+						  IIO_SHARED_BY_DIR,
+						  &chan->info_mask_shared_by_dir_available);
+	if (ret < 0)
+		return ret;
+	attrcount += ret;
+
 	ret = iio_device_add_info_mask_type(indio_dev, chan,
 					    IIO_SHARED_BY_ALL,
 					    &chan->info_mask_shared_by_all);
@@ -1014,6 +1202,13 @@ static int iio_device_add_channel_sysfs(struct iio_dev *indio_dev,
 		return ret;
 	attrcount += ret;
 
+	ret = iio_device_add_info_mask_type_avail(indio_dev, chan,
+						  IIO_SHARED_BY_ALL,
+						  &chan->info_mask_shared_by_all_available);
+	if (ret < 0)
+		return ret;
+	attrcount += ret;
+
 	if (chan->ext_info) {
 		unsigned int i = 0;
 		for (ext_info = chan->ext_info; ext_info->name; ext_info++) {
diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c
index e1e104845e38..978729f6d7c4 100644
--- a/drivers/iio/industrialio-trigger.c
+++ b/drivers/iio/industrialio-trigger.c
@@ -717,6 +717,27 @@ bool iio_trigger_using_own(struct iio_dev *indio_dev)
 }
 EXPORT_SYMBOL(iio_trigger_using_own);
 
+/**
+ * iio_trigger_validate_own_device - Check if a trigger and IIO device belong to
+ *  the same device
+ * @trig: The IIO trigger to check
+ * @indio_dev: the IIO device to check
+ *
+ * This function can be used as the validate_device callback for triggers that
+ * can only be attached to their own device.
+ *
+ * Return: 0 if both the trigger and the IIO device belong to the same
+ * device, -EINVAL otherwise.
+ */
+int iio_trigger_validate_own_device(struct iio_trigger *trig,
+	struct iio_dev *indio_dev)
+{
+	if (indio_dev->dev.parent != trig->dev.parent)
+		return -EINVAL;
+	return 0;
+}
+EXPORT_SYMBOL(iio_trigger_validate_own_device);
+
 void iio_device_register_trigger_consumer(struct iio_dev *indio_dev)
 {
 	indio_dev->groups[indio_dev->groupcounter++] =
diff --git a/drivers/iio/inkern.c b/drivers/iio/inkern.c
index c4757e6367e7..b0f4630a163f 100644
--- a/drivers/iio/inkern.c
+++ b/drivers/iio/inkern.c
@@ -658,6 +658,31 @@ err_unlock:
 }
 EXPORT_SYMBOL_GPL(iio_convert_raw_to_processed);
 
+static int iio_read_channel_attribute(struct iio_channel *chan,
+				      int *val, int *val2,
+				      enum iio_chan_info_enum attribute)
+{
+	int ret;
+
+	mutex_lock(&chan->indio_dev->info_exist_lock);
+	if (chan->indio_dev->info == NULL) {
+		ret = -ENODEV;
+		goto err_unlock;
+	}
+
+	ret = iio_channel_read(chan, val, val2, attribute);
+err_unlock:
+	mutex_unlock(&chan->indio_dev->info_exist_lock);
+
+	return ret;
+}
+
+int iio_read_channel_offset(struct iio_channel *chan, int *val, int *val2)
+{
+	return iio_read_channel_attribute(chan, val, val2, IIO_CHAN_INFO_OFFSET);
+}
+EXPORT_SYMBOL_GPL(iio_read_channel_offset);
+
 int iio_read_channel_processed(struct iio_channel *chan, int *val)
 {
 	int ret;
@@ -687,21 +712,113 @@ EXPORT_SYMBOL_GPL(iio_read_channel_processed);
 
 int iio_read_channel_scale(struct iio_channel *chan, int *val, int *val2)
 {
+	return iio_read_channel_attribute(chan, val, val2, IIO_CHAN_INFO_SCALE);
+}
+EXPORT_SYMBOL_GPL(iio_read_channel_scale);
+
+static int iio_channel_read_avail(struct iio_channel *chan,
+				  const int **vals, int *type, int *length,
+				  enum iio_chan_info_enum info)
+{
+	if (!iio_channel_has_available(chan->channel, info))
+		return -EINVAL;
+
+	return chan->indio_dev->info->read_avail(chan->indio_dev, chan->channel,
+						 vals, type, length, info);
+}
+
+int iio_read_avail_channel_raw(struct iio_channel *chan,
+			       const int **vals, int *length)
+{
 	int ret;
+	int type;
 
 	mutex_lock(&chan->indio_dev->info_exist_lock);
-	if (chan->indio_dev->info == NULL) {
+	if (!chan->indio_dev->info) {
 		ret = -ENODEV;
 		goto err_unlock;
 	}
 
-	ret = iio_channel_read(chan, val, val2, IIO_CHAN_INFO_SCALE);
+	ret = iio_channel_read_avail(chan,
+				     vals, &type, length, IIO_CHAN_INFO_RAW);
 err_unlock:
 	mutex_unlock(&chan->indio_dev->info_exist_lock);
 
+	if (ret >= 0 && type != IIO_VAL_INT) {
+		/* raw values are assumed to be IIO_VAL_INT */
+		ret = -EINVAL;
+		goto err_unlock;
+	}
+
 	return ret;
 }
-EXPORT_SYMBOL_GPL(iio_read_channel_scale);
+EXPORT_SYMBOL_GPL(iio_read_avail_channel_raw);
+
+static int iio_channel_read_max(struct iio_channel *chan,
+				int *val, int *val2, int *type,
+				enum iio_chan_info_enum info)
+{
+	int unused;
+	const int *vals;
+	int length;
+	int ret;
+
+	if (!val2)
+		val2 = &unused;
+
+	ret = iio_channel_read_avail(chan, &vals, type, &length, info);
+	switch (ret) {
+	case IIO_AVAIL_RANGE:
+		switch (*type) {
+		case IIO_VAL_INT:
+			*val = vals[2];
+			break;
+		default:
+			*val = vals[4];
+			*val2 = vals[5];
+		}
+		return 0;
+
+	case IIO_AVAIL_LIST:
+		if (length <= 0)
+			return -EINVAL;
+		switch (*type) {
+		case IIO_VAL_INT:
+			*val = vals[--length];
+			while (length) {
+				if (vals[--length] > *val)
+					*val = vals[length];
+			}
+			break;
+		default:
+			/* FIXME: learn about max for other iio values */
+			return -EINVAL;
+		}
+		return 0;
+
+	default:
+		return ret;
+	}
+}
+
+int iio_read_max_channel_raw(struct iio_channel *chan, int *val)
+{
+	int ret;
+	int type;
+
+	mutex_lock(&chan->indio_dev->info_exist_lock);
+	if (!chan->indio_dev->info) {
+		ret = -ENODEV;
+		goto err_unlock;
+	}
+
+	ret = iio_channel_read_max(chan, val, NULL, &type, IIO_CHAN_INFO_RAW);
+err_unlock:
+	mutex_unlock(&chan->indio_dev->info_exist_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(iio_read_max_channel_raw);
 
 int iio_get_channel_type(struct iio_channel *chan, enum iio_chan_type *type)
 {
diff --git a/drivers/iio/light/Kconfig b/drivers/iio/light/Kconfig
index ba2e64d7ee58..298ea5081a96 100644
--- a/drivers/iio/light/Kconfig
+++ b/drivers/iio/light/Kconfig
@@ -140,6 +140,18 @@ config GP2AP020A00F
 	  To compile this driver as a module, choose M here: the
 	  module will be called gp2ap020a00f.
 
+config SENSORS_ISL29018
+	tristate "Intersil 29018 light and proximity sensor"
+	depends on I2C
+	select REGMAP_I2C
+	default n
+	help
+	 If you say yes here you get support for ambient light sensing and
+	 proximity infrared sensing from Intersil ISL29018.
+	 This driver will provide the measurements of ambient light intensity
+	 in lux, proximity infrared sensing and normal infrared sensing.
+	 Data from sensor is accessible via sysfs.
+
 config ISL29125
 	tristate "Intersil ISL29125 digital color light sensor"
 	depends on I2C
@@ -326,6 +338,13 @@ config SENSORS_TSL2563
 	 This driver can also be built as a module.  If so, the module
 	 will be called tsl2563.
 
+config TSL2583
+	tristate "TAOS TSL2580, TSL2581 and TSL2583 light-to-digital converters"
+	depends on I2C
+	help
+	 Provides support for the TAOS tsl2580, tsl2581 and tsl2583 devices.
+	 Access ALS data via iio, sysfs.
+
 config TSL4531
 	tristate "TAOS TSL4531 ambient light sensors"
 	depends on I2C
diff --git a/drivers/iio/light/Makefile b/drivers/iio/light/Makefile
index c5768df87a17..4de520036e6e 100644
--- a/drivers/iio/light/Makefile
+++ b/drivers/iio/light/Makefile
@@ -17,6 +17,7 @@ obj-$(CONFIG_CM36651)		+= cm36651.o
 obj-$(CONFIG_GP2AP020A00F)	+= gp2ap020a00f.o
 obj-$(CONFIG_HID_SENSOR_ALS)	+= hid-sensor-als.o
 obj-$(CONFIG_HID_SENSOR_PROX)	+= hid-sensor-prox.o
+obj-$(CONFIG_SENSORS_ISL29018)	+= isl29018.o
 obj-$(CONFIG_ISL29125)		+= isl29125.o
 obj-$(CONFIG_JSA1212)		+= jsa1212.o
 obj-$(CONFIG_SENSORS_LM3533)	+= lm3533-als.o
@@ -30,6 +31,7 @@ obj-$(CONFIG_SI1145)		+= si1145.o
 obj-$(CONFIG_STK3310)          += stk3310.o
 obj-$(CONFIG_TCS3414)		+= tcs3414.o
 obj-$(CONFIG_TCS3472)		+= tcs3472.o
+obj-$(CONFIG_TSL2583)		+= tsl2583.o
 obj-$(CONFIG_TSL4531)		+= tsl4531.o
 obj-$(CONFIG_US5182D)		+= us5182d.o
 obj-$(CONFIG_VCNL4000)		+= vcnl4000.o
diff --git a/drivers/staging/iio/light/isl29018.c b/drivers/iio/light/isl29018.c
index a767a43c995c..917dd8b43e72 100644
--- a/drivers/staging/iio/light/isl29018.c
+++ b/drivers/iio/light/isl29018.c
@@ -62,16 +62,6 @@
 #define ISL29035_BOUT_SHIFT		0x07
 #define ISL29035_BOUT_MASK		(0x01 << ISL29035_BOUT_SHIFT)
 
-#define ISL29018_INT_TIME_AVAIL		"0.090000 0.005630 0.000351 0.000021"
-#define ISL29023_INT_TIME_AVAIL		"0.090000 0.005600 0.000352 0.000022"
-#define ISL29035_INT_TIME_AVAIL		"0.105000 0.006500 0.000410 0.000025"
-
-static const char * const int_time_avail[] = {
-	ISL29018_INT_TIME_AVAIL,
-	ISL29023_INT_TIME_AVAIL,
-	ISL29035_INT_TIME_AVAIL,
-};
-
 enum isl29018_int_time {
 	ISL29018_INT_TIME_16,
 	ISL29018_INT_TIME_12,
@@ -110,7 +100,8 @@ struct isl29018_chip {
 static int isl29018_set_integration_time(struct isl29018_chip *chip,
 					 unsigned int utime)
 {
-	int i, ret;
+	unsigned int i;
+	int ret;
 	unsigned int int_time, new_int_time;
 
 	for (i = 0; i < ARRAY_SIZE(isl29018_int_utimes[chip->type]); ++i) {
@@ -145,7 +136,8 @@ static int isl29018_set_integration_time(struct isl29018_chip *chip,
 
 static int isl29018_set_scale(struct isl29018_chip *chip, int scale, int uscale)
 {
-	int i, ret;
+	unsigned int i;
+	int ret;
 	struct isl29018_scale new_scale;
 
 	for (i = 0; i < ARRAY_SIZE(isl29018_scales[chip->int_time]); ++i) {
@@ -276,29 +268,35 @@ static int isl29018_read_proximity_ir(struct isl29018_chip *chip, int scheme,
 	return 0;
 }
 
-static ssize_t isl29018_show_scale_available(struct device *dev,
-				    struct device_attribute *attr, char *buf)
+static ssize_t in_illuminance_scale_available_show
+			(struct device *dev, struct device_attribute *attr,
+			 char *buf)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct isl29018_chip *chip = iio_priv(indio_dev);
-	int i, len = 0;
+	unsigned int i;
+	int len = 0;
 
+	mutex_lock(&chip->lock);
 	for (i = 0; i < ARRAY_SIZE(isl29018_scales[chip->int_time]); ++i)
 		len += sprintf(buf + len, "%d.%06d ",
 			       isl29018_scales[chip->int_time][i].scale,
 			       isl29018_scales[chip->int_time][i].uscale);
+	mutex_unlock(&chip->lock);
 
 	buf[len - 1] = '\n';
 
 	return len;
 }
 
-static ssize_t isl29018_show_int_time_available(struct device *dev,
-				       struct device_attribute *attr, char *buf)
+static ssize_t in_illuminance_integration_time_available_show
+			(struct device *dev, struct device_attribute *attr,
+			 char *buf)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct isl29018_chip *chip = iio_priv(indio_dev);
-	int i, len = 0;
+	unsigned int i;
+	int len = 0;
 
 	for (i = 0; i < ARRAY_SIZE(isl29018_int_utimes[chip->type]); ++i)
 		len += sprintf(buf + len, "0.%06d ",
@@ -309,9 +307,27 @@ static ssize_t isl29018_show_int_time_available(struct device *dev,
 	return len;
 }
 
-static ssize_t isl29018_show_prox_infrared_suppression(struct device *dev,
-					      struct device_attribute *attr,
-					      char *buf)
+/*
+ * From ISL29018 Data Sheet (FN6619.4, Oct 8, 2012) regarding the
+ * infrared suppression:
+ *
+ *   Proximity Sensing Scheme: Bit 7. This bit programs the function
+ * of the proximity detection. Logic 0 of this bit, Scheme 0, makes
+ * full n (4, 8, 12, 16) bits (unsigned) proximity detection. The range
+ * of Scheme 0 proximity count is from 0 to 2^n. Logic 1 of this bit,
+ * Scheme 1, makes n-1 (3, 7, 11, 15) bits (2's complementary)
+ * proximity_less_ambient detection. The range of Scheme 1
+ * proximity count is from -2^(n-1) to 2^(n-1) . The sign bit is extended
+ * for resolutions less than 16. While Scheme 0 has wider dynamic
+ * range, Scheme 1 proximity detection is less affected by the
+ * ambient IR noise variation.
+ *
+ * 0 Sensing IR from LED and ambient
+ * 1 Sensing IR from LED with ambient IR rejection
+ */
+static ssize_t proximity_on_chip_ambient_infrared_suppression_show
+			(struct device *dev, struct device_attribute *attr,
+			 char *buf)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct isl29018_chip *chip = iio_priv(indio_dev);
@@ -323,9 +339,9 @@ static ssize_t isl29018_show_prox_infrared_suppression(struct device *dev,
 	return sprintf(buf, "%d\n", chip->prox_scheme);
 }
 
-static ssize_t isl29018_store_prox_infrared_suppression(struct device *dev,
-					       struct device_attribute *attr,
-					       const char *buf, size_t count)
+static ssize_t proximity_on_chip_ambient_infrared_suppression_store
+			(struct device *dev, struct device_attribute *attr,
+			 const char *buf, size_t count)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct isl29018_chip *chip = iio_priv(indio_dev);
@@ -357,6 +373,10 @@ static int isl29018_write_raw(struct iio_dev *indio_dev,
 	int ret = -EINVAL;
 
 	mutex_lock(&chip->lock);
+	if (chip->suspended) {
+		ret = -EBUSY;
+		goto write_done;
+	}
 	switch (mask) {
 	case IIO_CHAN_INFO_CALIBSCALE:
 		if (chan->type == IIO_LIGHT) {
@@ -366,13 +386,8 @@ static int isl29018_write_raw(struct iio_dev *indio_dev,
 		}
 		break;
 	case IIO_CHAN_INFO_INT_TIME:
-		if (chan->type == IIO_LIGHT) {
-			if (val) {
-				mutex_unlock(&chip->lock);
-				return -EINVAL;
-			}
+		if (chan->type == IIO_LIGHT && !val)
 			ret = isl29018_set_integration_time(chip, val2);
-		}
 		break;
 	case IIO_CHAN_INFO_SCALE:
 		if (chan->type == IIO_LIGHT)
@@ -381,6 +396,8 @@ static int isl29018_write_raw(struct iio_dev *indio_dev,
 	default:
 		break;
 	}
+
+write_done:
 	mutex_unlock(&chip->lock);
 
 	return ret;
@@ -397,8 +414,8 @@ static int isl29018_read_raw(struct iio_dev *indio_dev,
 
 	mutex_lock(&chip->lock);
 	if (chip->suspended) {
-		mutex_unlock(&chip->lock);
-		return -EBUSY;
+		ret = -EBUSY;
+		goto read_done;
 	}
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
@@ -445,7 +462,10 @@ static int isl29018_read_raw(struct iio_dev *indio_dev,
 	default:
 		break;
 	}
+
+read_done:
 	mutex_unlock(&chip->lock);
+
 	return ret;
 }
 
@@ -482,14 +502,9 @@ static const struct iio_chan_spec isl29023_channels[] = {
 	ISL29018_IR_CHANNEL,
 };
 
-static IIO_DEVICE_ATTR(in_illuminance_integration_time_available, S_IRUGO,
-		       isl29018_show_int_time_available, NULL, 0);
-static IIO_DEVICE_ATTR(in_illuminance_scale_available, S_IRUGO,
-		      isl29018_show_scale_available, NULL, 0);
-static IIO_DEVICE_ATTR(proximity_on_chip_ambient_infrared_suppression,
-					S_IRUGO | S_IWUSR,
-					isl29018_show_prox_infrared_suppression,
-					isl29018_store_prox_infrared_suppression, 0);
+static IIO_DEVICE_ATTR_RO(in_illuminance_integration_time_available, 0);
+static IIO_DEVICE_ATTR_RO(in_illuminance_scale_available, 0);
+static IIO_DEVICE_ATTR_RW(proximity_on_chip_ambient_infrared_suppression, 0);
 
 #define ISL29018_DEV_ATTR(name) (&iio_dev_attr_##name.dev_attr.attr)
 
@@ -514,30 +529,6 @@ static const struct attribute_group isl29023_group = {
 	.attrs = isl29023_attributes,
 };
 
-static int isl29035_detect(struct isl29018_chip *chip)
-{
-	int status;
-	unsigned int id;
-	struct device *dev = regmap_get_device(chip->regmap);
-
-	status = regmap_read(chip->regmap, ISL29035_REG_DEVICE_ID, &id);
-	if (status < 0) {
-		dev_err(dev,
-			"Error reading ID register with error %d\n",
-			status);
-		return status;
-	}
-
-	id = (id & ISL29035_DEVICE_ID_MASK) >> ISL29035_DEVICE_ID_SHIFT;
-
-	if (id != ISL29035_DEVICE_ID)
-		return -ENODEV;
-
-	/* Clear brownout bit */
-	return regmap_update_bits(chip->regmap, ISL29035_REG_DEVICE_ID,
-				  ISL29035_BOUT_MASK, 0);
-}
-
 enum {
 	isl29018,
 	isl29023,
@@ -550,12 +541,31 @@ static int isl29018_chip_init(struct isl29018_chip *chip)
 	struct device *dev = regmap_get_device(chip->regmap);
 
 	if (chip->type == isl29035) {
-		status = isl29035_detect(chip);
+		unsigned int id;
+
+		status = regmap_read(chip->regmap, ISL29035_REG_DEVICE_ID, &id);
+		if (status < 0) {
+			dev_err(dev,
+				"Error reading ID register with error %d\n",
+				status);
+			return status;
+		}
+
+		id = (id & ISL29035_DEVICE_ID_MASK) >> ISL29035_DEVICE_ID_SHIFT;
+
+		if (id != ISL29035_DEVICE_ID)
+			return -ENODEV;
+
+		/* Clear brownout bit */
+		status = regmap_update_bits(chip->regmap,
+					    ISL29035_REG_DEVICE_ID,
+					    ISL29035_BOUT_MASK, 0);
 		if (status < 0)
 			return status;
 	}
 
-	/* Code added per Intersil Application Note 1534:
+	/*
+	 * Code added per Intersil Application Note 1534:
 	 *     When VDD sinks to approximately 1.8V or below, some of
 	 * the part's registers may change their state. When VDD
 	 * recovers to 2.25V (or greater), the part may thus be in an
@@ -582,7 +592,8 @@ static int isl29018_chip_init(struct isl29018_chip *chip)
 		return status;
 	}
 
-	/* See Intersil AN1534 comments above.
+	/*
+	 * See Intersil AN1534 comments above.
 	 * "Operating Mode" (COMMAND1) register is reprogrammed when
 	 * data is read from the device.
 	 */
@@ -605,12 +616,10 @@ static int isl29018_chip_init(struct isl29018_chip *chip)
 
 	status = isl29018_set_integration_time(chip,
 			isl29018_int_utimes[chip->type][chip->int_time]);
-	if (status < 0) {
+	if (status < 0)
 		dev_err(dev, "Init of isl29018 fails\n");
-		return status;
-	}
 
-	return 0;
+	return status;
 }
 
 static const struct iio_info isl29018_info = {
@@ -713,6 +722,7 @@ static int isl29018_probe(struct i2c_client *client,
 	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*chip));
 	if (!indio_dev)
 		return -ENOMEM;
+
 	chip = iio_priv(indio_dev);
 
 	i2c_set_clientdata(client, indio_dev);
@@ -752,6 +762,7 @@ static int isl29018_probe(struct i2c_client *client,
 	indio_dev->name = name;
 	indio_dev->dev.parent = &client->dev;
 	indio_dev->modes = INDIO_DIRECT_MODE;
+
 	return devm_iio_device_register(&client->dev, indio_dev);
 }
 
@@ -762,13 +773,15 @@ static int isl29018_suspend(struct device *dev)
 
 	mutex_lock(&chip->lock);
 
-	/* Since this driver uses only polling commands, we are by default in
+	/*
+	 * Since this driver uses only polling commands, we are by default in
 	 * auto shutdown (ie, power-down) mode.
 	 * So we do not have much to do here.
 	 */
 	chip->suspended = true;
 
 	mutex_unlock(&chip->lock);
+
 	return 0;
 }
 
@@ -784,6 +797,7 @@ static int isl29018_resume(struct device *dev)
 		chip->suspended = false;
 
 	mutex_unlock(&chip->lock);
+
 	return err;
 }
 
@@ -807,7 +821,6 @@ static const struct i2c_device_id isl29018_id[] = {
 	{"isl29035", isl29035},
 	{}
 };
-
 MODULE_DEVICE_TABLE(i2c, isl29018_id);
 
 static const struct of_device_id isl29018_of_match[] = {
diff --git a/drivers/iio/light/ltr501.c b/drivers/iio/light/ltr501.c
index 3afc53a3d0b6..b30e0c1c6cc4 100644
--- a/drivers/iio/light/ltr501.c
+++ b/drivers/iio/light/ltr501.c
@@ -631,14 +631,16 @@ static int ltr501_read_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_PROCESSED:
-		if (iio_buffer_enabled(indio_dev))
-			return -EBUSY;
-
 		switch (chan->type) {
 		case IIO_LIGHT:
+			ret = iio_device_claim_direct_mode(indio_dev);
+			if (ret)
+				return ret;
+
 			mutex_lock(&data->lock_als);
 			ret = ltr501_read_als(data, buf);
 			mutex_unlock(&data->lock_als);
+			iio_device_release_direct_mode(indio_dev);
 			if (ret < 0)
 				return ret;
 			*val = ltr501_calculate_lux(le16_to_cpu(buf[1]),
@@ -648,8 +650,9 @@ static int ltr501_read_raw(struct iio_dev *indio_dev,
 			return -EINVAL;
 		}
 	case IIO_CHAN_INFO_RAW:
-		if (iio_buffer_enabled(indio_dev))
-			return -EBUSY;
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
 
 		switch (chan->type) {
 		case IIO_INTENSITY:
@@ -657,21 +660,28 @@ static int ltr501_read_raw(struct iio_dev *indio_dev,
 			ret = ltr501_read_als(data, buf);
 			mutex_unlock(&data->lock_als);
 			if (ret < 0)
-				return ret;
+				break;
 			*val = le16_to_cpu(chan->address == LTR501_ALS_DATA1 ?
 					   buf[0] : buf[1]);
-			return IIO_VAL_INT;
+			ret = IIO_VAL_INT;
+			break;
 		case IIO_PROXIMITY:
 			mutex_lock(&data->lock_ps);
 			ret = ltr501_read_ps(data);
 			mutex_unlock(&data->lock_ps);
 			if (ret < 0)
-				return ret;
+				break;
 			*val = ret & LTR501_PS_DATA_MASK;
-			return IIO_VAL_INT;
+			ret = IIO_VAL_INT;
+			break;
 		default:
-			return -EINVAL;
+			ret = -EINVAL;
+			break;
 		}
+
+		iio_device_release_direct_mode(indio_dev);
+		return ret;
+
 	case IIO_CHAN_INFO_SCALE:
 		switch (chan->type) {
 		case IIO_INTENSITY:
@@ -729,8 +739,9 @@ static int ltr501_write_raw(struct iio_dev *indio_dev,
 	int i, ret, freq_val, freq_val2;
 	struct ltr501_chip_info *info = data->chip_info;
 
-	if (iio_buffer_enabled(indio_dev))
-		return -EBUSY;
+	ret = iio_device_claim_direct_mode(indio_dev);
+	if (ret)
+		return ret;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_SCALE:
@@ -739,85 +750,105 @@ static int ltr501_write_raw(struct iio_dev *indio_dev,
 			i = ltr501_get_gain_index(info->als_gain,
 						  info->als_gain_tbl_size,
 						  val, val2);
-			if (i < 0)
-				return -EINVAL;
+			if (i < 0) {
+				ret = -EINVAL;
+				break;
+			}
 
 			data->als_contr &= ~info->als_gain_mask;
 			data->als_contr |= i << info->als_gain_shift;
 
-			return regmap_write(data->regmap, LTR501_ALS_CONTR,
-					    data->als_contr);
+			ret = regmap_write(data->regmap, LTR501_ALS_CONTR,
+					   data->als_contr);
+			break;
 		case IIO_PROXIMITY:
 			i = ltr501_get_gain_index(info->ps_gain,
 						  info->ps_gain_tbl_size,
 						  val, val2);
-			if (i < 0)
-				return -EINVAL;
+			if (i < 0) {
+				ret = -EINVAL;
+				break;
+			}
 			data->ps_contr &= ~LTR501_CONTR_PS_GAIN_MASK;
 			data->ps_contr |= i << LTR501_CONTR_PS_GAIN_SHIFT;
 
-			return regmap_write(data->regmap, LTR501_PS_CONTR,
-					    data->ps_contr);
+			ret = regmap_write(data->regmap, LTR501_PS_CONTR,
+					   data->ps_contr);
+			break;
 		default:
-			return -EINVAL;
+			ret = -EINVAL;
+			break;
 		}
+		break;
+
 	case IIO_CHAN_INFO_INT_TIME:
 		switch (chan->type) {
 		case IIO_INTENSITY:
-			if (val != 0)
-				return -EINVAL;
+			if (val != 0) {
+				ret = -EINVAL;
+				break;
+			}
 			mutex_lock(&data->lock_als);
-			i = ltr501_set_it_time(data, val2);
+			ret = ltr501_set_it_time(data, val2);
 			mutex_unlock(&data->lock_als);
-			return i;
+			break;
 		default:
-			return -EINVAL;
+			ret = -EINVAL;
+			break;
 		}
+		break;
+
 	case IIO_CHAN_INFO_SAMP_FREQ:
 		switch (chan->type) {
 		case IIO_INTENSITY:
 			ret = ltr501_als_read_samp_freq(data, &freq_val,
 							&freq_val2);
 			if (ret < 0)
-				return ret;
+				break;
 
 			ret = ltr501_als_write_samp_freq(data, val, val2);
 			if (ret < 0)
-				return ret;
+				break;
 
 			/* update persistence count when changing frequency */
 			ret = ltr501_write_intr_prst(data, chan->type,
 						     0, data->als_period);
 
 			if (ret < 0)
-				return ltr501_als_write_samp_freq(data,
-								  freq_val,
-								  freq_val2);
-			return ret;
+				ret = ltr501_als_write_samp_freq(data, freq_val,
+								 freq_val2);
+			break;
 		case IIO_PROXIMITY:
 			ret = ltr501_ps_read_samp_freq(data, &freq_val,
 						       &freq_val2);
 			if (ret < 0)
-				return ret;
+				break;
 
 			ret = ltr501_ps_write_samp_freq(data, val, val2);
 			if (ret < 0)
-				return ret;
+				break;
 
 			/* update persistence count when changing frequency */
 			ret = ltr501_write_intr_prst(data, chan->type,
 						     0, data->ps_period);
 
 			if (ret < 0)
-				return ltr501_ps_write_samp_freq(data,
-								 freq_val,
-								 freq_val2);
-			return ret;
+				ret = ltr501_ps_write_samp_freq(data, freq_val,
+								freq_val2);
+			break;
 		default:
-			return -EINVAL;
+			ret = -EINVAL;
+			break;
 		}
+		break;
+
+	default:
+		ret = -EINVAL;
+		break;
 	}
-	return -EINVAL;
+
+	iio_device_release_direct_mode(indio_dev);
+	return ret;
 }
 
 static int ltr501_read_thresh(struct iio_dev *indio_dev,
diff --git a/drivers/iio/light/max44000.c b/drivers/iio/light/max44000.c
index 6511b20a2a29..a144ca3461fc 100644
--- a/drivers/iio/light/max44000.c
+++ b/drivers/iio/light/max44000.c
@@ -204,17 +204,18 @@ static int max44000_write_alspga(struct max44000_data *data, int val)
 static int max44000_read_alsval(struct max44000_data *data)
 {
 	u16 regval;
+	__be16 val;
 	int alstim, ret;
 
 	ret = regmap_bulk_read(data->regmap, MAX44000_REG_ALS_DATA_HI,
-			       &regval, sizeof(regval));
+			       &val, sizeof(val));
 	if (ret < 0)
 		return ret;
 	alstim = ret = max44000_read_alstim(data);
 	if (ret < 0)
 		return ret;
 
-	regval = be16_to_cpu(regval);
+	regval = be16_to_cpu(val);
 
 	/*
 	 * Overflow is explained on datasheet page 17.
diff --git a/drivers/iio/light/tsl2583.c b/drivers/iio/light/tsl2583.c
new file mode 100644
index 000000000000..a78b6025c465
--- /dev/null
+++ b/drivers/iio/light/tsl2583.c
@@ -0,0 +1,913 @@
+/*
+ * Device driver for monitoring ambient light intensity (lux)
+ * within the TAOS tsl258x family of devices (tsl2580, tsl2581, tsl2583).
+ *
+ * Copyright (c) 2011, TAOS Corporation.
+ * Copyright (c) 2016 Brian Masney <masneyb@onstation.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/i2c.h>
+#include <linux/errno.h>
+#include <linux/delay.h>
+#include <linux/string.h>
+#include <linux/mutex.h>
+#include <linux/unistd.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+
+/* Device Registers and Masks */
+#define TSL2583_CNTRL			0x00
+#define TSL2583_ALS_TIME		0X01
+#define TSL2583_INTERRUPT		0x02
+#define TSL2583_GAIN			0x07
+#define TSL2583_REVID			0x11
+#define TSL2583_CHIPID			0x12
+#define TSL2583_ALS_CHAN0LO		0x14
+#define TSL2583_ALS_CHAN0HI		0x15
+#define TSL2583_ALS_CHAN1LO		0x16
+#define TSL2583_ALS_CHAN1HI		0x17
+#define TSL2583_TMR_LO			0x18
+#define TSL2583_TMR_HI			0x19
+
+/* tsl2583 cmd reg masks */
+#define TSL2583_CMD_REG			0x80
+#define TSL2583_CMD_SPL_FN		0x60
+#define TSL2583_CMD_ALS_INT_CLR		0x01
+
+/* tsl2583 cntrl reg masks */
+#define TSL2583_CNTL_ADC_ENBL		0x02
+#define TSL2583_CNTL_PWR_OFF		0x00
+#define TSL2583_CNTL_PWR_ON		0x01
+
+/* tsl2583 status reg masks */
+#define TSL2583_STA_ADC_VALID		0x01
+#define TSL2583_STA_ADC_INTR		0x10
+
+/* Lux calculation constants */
+#define TSL2583_LUX_CALC_OVER_FLOW	65535
+
+#define TSL2583_INTERRUPT_DISABLED	0x00
+
+#define TSL2583_CHIP_ID			0x90
+#define TSL2583_CHIP_ID_MASK		0xf0
+
+/* Per-device data */
+struct tsl2583_als_info {
+	u16 als_ch0;
+	u16 als_ch1;
+	u16 lux;
+};
+
+struct tsl2583_lux {
+	unsigned int ratio;
+	unsigned int ch0;
+	unsigned int ch1;
+};
+
+static const struct tsl2583_lux tsl2583_default_lux[] = {
+	{  9830,  8520, 15729 },
+	{ 12452, 10807, 23344 },
+	{ 14746,  6383, 11705 },
+	{ 17695,  4063,  6554 },
+	{     0,     0,     0 }  /* Termination segment */
+};
+
+#define TSL2583_MAX_LUX_TABLE_ENTRIES 11
+
+struct tsl2583_settings {
+	int als_time;
+	int als_gain;
+	int als_gain_trim;
+	int als_cal_target;
+
+	/*
+	 * This structure is intentionally large to accommodate updates via
+	 * sysfs. Sized to 11 = max 10 segments + 1 termination segment.
+	 * Assumption is that one and only one type of glass used.
+	 */
+	struct tsl2583_lux als_device_lux[TSL2583_MAX_LUX_TABLE_ENTRIES];
+};
+
+struct tsl2583_chip {
+	struct mutex als_mutex;
+	struct i2c_client *client;
+	struct tsl2583_als_info als_cur_info;
+	struct tsl2583_settings als_settings;
+	int als_time_scale;
+	int als_saturation;
+	bool suspended;
+};
+
+struct gainadj {
+	s16 ch0;
+	s16 ch1;
+	s16 mean;
+};
+
+/* Index = (0 - 3) Used to validate the gain selection index */
+static const struct gainadj gainadj[] = {
+	{ 1, 1, 1 },
+	{ 8, 8, 8 },
+	{ 16, 16, 16 },
+	{ 107, 115, 111 }
+};
+
+/*
+ * Provides initial operational parameter defaults.
+ * These defaults may be changed through the device's sysfs files.
+ */
+static void tsl2583_defaults(struct tsl2583_chip *chip)
+{
+	/*
+	 * The integration time must be a multiple of 50ms and within the
+	 * range [50, 600] ms.
+	 */
+	chip->als_settings.als_time = 100;
+
+	/*
+	 * This is an index into the gainadj table. Assume clear glass as the
+	 * default.
+	 */
+	chip->als_settings.als_gain = 0;
+
+	/* Default gain trim to account for aperture effects */
+	chip->als_settings.als_gain_trim = 1000;
+
+	/* Known external ALS reading used for calibration */
+	chip->als_settings.als_cal_target = 130;
+
+	/* Default lux table. */
+	memcpy(chip->als_settings.als_device_lux, tsl2583_default_lux,
+	       sizeof(tsl2583_default_lux));
+}
+
+/*
+ * Reads and calculates current lux value.
+ * The raw ch0 and ch1 values of the ambient light sensed in the last
+ * integration cycle are read from the device.
+ * Time scale factor array values are adjusted based on the integration time.
+ * The raw values are multiplied by a scale factor, and device gain is obtained
+ * using gain index. Limit checks are done next, then the ratio of a multiple
+ * of ch1 value, to the ch0 value, is calculated. The array als_device_lux[]
+ * declared above is then scanned to find the first ratio value that is just
+ * above the ratio we just calculated. The ch0 and ch1 multiplier constants in
+ * the array are then used along with the time scale factor array values, to
+ * calculate the lux.
+ */
+static int tsl2583_get_lux(struct iio_dev *indio_dev)
+{
+	u16 ch0, ch1; /* separated ch0/ch1 data from device */
+	u32 lux; /* raw lux calculated from device data */
+	u64 lux64;
+	u32 ratio;
+	u8 buf[5];
+	struct tsl2583_lux *p;
+	struct tsl2583_chip *chip = iio_priv(indio_dev);
+	int i, ret;
+
+	ret = i2c_smbus_read_byte_data(chip->client, TSL2583_CMD_REG);
+	if (ret < 0) {
+		dev_err(&chip->client->dev, "%s: failed to read CMD_REG register\n",
+			__func__);
+		goto done;
+	}
+
+	/* is data new & valid */
+	if (!(ret & TSL2583_STA_ADC_INTR)) {
+		dev_err(&chip->client->dev, "%s: data not valid; returning last value\n",
+			__func__);
+		ret = chip->als_cur_info.lux; /* return LAST VALUE */
+		goto done;
+	}
+
+	for (i = 0; i < 4; i++) {
+		int reg = TSL2583_CMD_REG | (TSL2583_ALS_CHAN0LO + i);
+
+		ret = i2c_smbus_read_byte_data(chip->client, reg);
+		if (ret < 0) {
+			dev_err(&chip->client->dev, "%s: failed to read register %x\n",
+				__func__, reg);
+			goto done;
+		}
+		buf[i] = ret;
+	}
+
+	/*
+	 * Clear the pending interrupt status bit on the chip to allow the next
+	 * integration cycle to start. This has to be done even though this
+	 * driver currently does not support interrupts.
+	 */
+	ret = i2c_smbus_write_byte(chip->client,
+				   (TSL2583_CMD_REG | TSL2583_CMD_SPL_FN |
+				    TSL2583_CMD_ALS_INT_CLR));
+	if (ret < 0) {
+		dev_err(&chip->client->dev, "%s: failed to clear the interrupt bit\n",
+			__func__);
+		goto done; /* have no data, so return failure */
+	}
+
+	/* extract ALS/lux data */
+	ch0 = le16_to_cpup((const __le16 *)&buf[0]);
+	ch1 = le16_to_cpup((const __le16 *)&buf[2]);
+
+	chip->als_cur_info.als_ch0 = ch0;
+	chip->als_cur_info.als_ch1 = ch1;
+
+	if ((ch0 >= chip->als_saturation) || (ch1 >= chip->als_saturation))
+		goto return_max;
+
+	if (!ch0) {
+		/*
+		 * The sensor appears to be in total darkness so set the
+		 * calculated lux to 0 and return early to avoid a division by
+		 * zero below when calculating the ratio.
+		 */
+		ret = 0;
+		chip->als_cur_info.lux = 0;
+		goto done;
+	}
+
+	/* calculate ratio */
+	ratio = (ch1 << 15) / ch0;
+
+	/* convert to unscaled lux using the pointer to the table */
+	for (p = (struct tsl2583_lux *)chip->als_settings.als_device_lux;
+	     p->ratio != 0 && p->ratio < ratio; p++)
+		;
+
+	if (p->ratio == 0) {
+		lux = 0;
+	} else {
+		u32 ch0lux, ch1lux;
+
+		ch0lux = ((ch0 * p->ch0) +
+			  (gainadj[chip->als_settings.als_gain].ch0 >> 1))
+			 / gainadj[chip->als_settings.als_gain].ch0;
+		ch1lux = ((ch1 * p->ch1) +
+			  (gainadj[chip->als_settings.als_gain].ch1 >> 1))
+			 / gainadj[chip->als_settings.als_gain].ch1;
+
+		/* note: lux is 31 bit max at this point */
+		if (ch1lux > ch0lux) {
+			dev_dbg(&chip->client->dev, "%s: No Data - Returning 0\n",
+				__func__);
+			ret = 0;
+			chip->als_cur_info.lux = 0;
+			goto done;
+		}
+
+		lux = ch0lux - ch1lux;
+	}
+
+	/* adjust for active time scale */
+	if (chip->als_time_scale == 0)
+		lux = 0;
+	else
+		lux = (lux + (chip->als_time_scale >> 1)) /
+			chip->als_time_scale;
+
+	/*
+	 * Adjust for active gain scale.
+	 * The tsl2583_default_lux tables above have a factor of 8192 built in,
+	 * so we need to shift right.
+	 * User-specified gain provides a multiplier.
+	 * Apply user-specified gain before shifting right to retain precision.
+	 * Use 64 bits to avoid overflow on multiplication.
+	 * Then go back to 32 bits before division to avoid using div_u64().
+	 */
+	lux64 = lux;
+	lux64 = lux64 * chip->als_settings.als_gain_trim;
+	lux64 >>= 13;
+	lux = lux64;
+	lux = (lux + 500) / 1000;
+
+	if (lux > TSL2583_LUX_CALC_OVER_FLOW) { /* check for overflow */
+return_max:
+		lux = TSL2583_LUX_CALC_OVER_FLOW;
+	}
+
+	/* Update the structure with the latest VALID lux. */
+	chip->als_cur_info.lux = lux;
+	ret = lux;
+
+done:
+	return ret;
+}
+
+/*
+ * Obtain single reading and calculate the als_gain_trim (later used
+ * to derive actual lux).
+ * Return updated gain_trim value.
+ */
+static int tsl2583_als_calibrate(struct iio_dev *indio_dev)
+{
+	struct tsl2583_chip *chip = iio_priv(indio_dev);
+	unsigned int gain_trim_val;
+	int ret;
+	int lux_val;
+
+	ret = i2c_smbus_read_byte_data(chip->client,
+				       TSL2583_CMD_REG | TSL2583_CNTRL);
+	if (ret < 0) {
+		dev_err(&chip->client->dev,
+			"%s: failed to read from the CNTRL register\n",
+			__func__);
+		return ret;
+	}
+
+	if ((ret & (TSL2583_CNTL_ADC_ENBL | TSL2583_CNTL_PWR_ON))
+			!= (TSL2583_CNTL_ADC_ENBL | TSL2583_CNTL_PWR_ON)) {
+		dev_err(&chip->client->dev,
+			"%s: Device is not powered on and/or ADC is not enabled\n",
+			__func__);
+		return -EINVAL;
+	} else if ((ret & TSL2583_STA_ADC_VALID) != TSL2583_STA_ADC_VALID) {
+		dev_err(&chip->client->dev,
+			"%s: The two ADC channels have not completed an integration cycle\n",
+			__func__);
+		return -ENODATA;
+	}
+
+	lux_val = tsl2583_get_lux(indio_dev);
+	if (lux_val < 0) {
+		dev_err(&chip->client->dev, "%s: failed to get lux\n",
+			__func__);
+		return lux_val;
+	}
+
+	gain_trim_val = (unsigned int)(((chip->als_settings.als_cal_target)
+			* chip->als_settings.als_gain_trim) / lux_val);
+	if ((gain_trim_val < 250) || (gain_trim_val > 4000)) {
+		dev_err(&chip->client->dev,
+			"%s: trim_val of %d is not within the range [250, 4000]\n",
+			__func__, gain_trim_val);
+		return -ENODATA;
+	}
+
+	chip->als_settings.als_gain_trim = (int)gain_trim_val;
+
+	return 0;
+}
+
+static int tsl2583_set_als_time(struct tsl2583_chip *chip)
+{
+	int als_count, als_time, ret;
+	u8 val;
+
+	/* determine als integration register */
+	als_count = (chip->als_settings.als_time * 100 + 135) / 270;
+	if (!als_count)
+		als_count = 1; /* ensure at least one cycle */
+
+	/* convert back to time (encompasses overrides) */
+	als_time = (als_count * 27 + 5) / 10;
+
+	val = 256 - als_count;
+	ret = i2c_smbus_write_byte_data(chip->client,
+					TSL2583_CMD_REG | TSL2583_ALS_TIME,
+					val);
+	if (ret < 0) {
+		dev_err(&chip->client->dev, "%s: failed to set the als time to %d\n",
+			__func__, val);
+		return ret;
+	}
+
+	/* set chip struct re scaling and saturation */
+	chip->als_saturation = als_count * 922; /* 90% of full scale */
+	chip->als_time_scale = (als_time + 25) / 50;
+
+	return ret;
+}
+
+static int tsl2583_set_als_gain(struct tsl2583_chip *chip)
+{
+	int ret;
+
+	/* Set the gain based on als_settings struct */
+	ret = i2c_smbus_write_byte_data(chip->client,
+					TSL2583_CMD_REG | TSL2583_GAIN,
+					chip->als_settings.als_gain);
+	if (ret < 0)
+		dev_err(&chip->client->dev,
+			"%s: failed to set the gain to %d\n", __func__,
+			chip->als_settings.als_gain);
+
+	return ret;
+}
+
+static int tsl2583_set_power_state(struct tsl2583_chip *chip, u8 state)
+{
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(chip->client,
+					TSL2583_CMD_REG | TSL2583_CNTRL, state);
+	if (ret < 0)
+		dev_err(&chip->client->dev,
+			"%s: failed to set the power state to %d\n", __func__,
+			state);
+
+	return ret;
+}
+
+/*
+ * Turn the device on.
+ * Configuration must be set before calling this function.
+ */
+static int tsl2583_chip_init_and_power_on(struct iio_dev *indio_dev)
+{
+	struct tsl2583_chip *chip = iio_priv(indio_dev);
+	int ret;
+
+	/* Power on the device; ADC off. */
+	ret = tsl2583_set_power_state(chip, TSL2583_CNTL_PWR_ON);
+	if (ret < 0)
+		return ret;
+
+	ret = i2c_smbus_write_byte_data(chip->client,
+					TSL2583_CMD_REG | TSL2583_INTERRUPT,
+					TSL2583_INTERRUPT_DISABLED);
+	if (ret < 0) {
+		dev_err(&chip->client->dev,
+			"%s: failed to disable interrupts\n", __func__);
+		return ret;
+	}
+
+	ret = tsl2583_set_als_time(chip);
+	if (ret < 0)
+		return ret;
+
+	ret = tsl2583_set_als_gain(chip);
+	if (ret < 0)
+		return ret;
+
+	usleep_range(3000, 3500);
+
+	ret = tsl2583_set_power_state(chip, TSL2583_CNTL_PWR_ON |
+					    TSL2583_CNTL_ADC_ENBL);
+	if (ret < 0)
+		return ret;
+
+	chip->suspended = false;
+
+	return ret;
+}
+
+/* Sysfs Interface Functions */
+
+static ssize_t in_illuminance_input_target_show(struct device *dev,
+						struct device_attribute *attr,
+						char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct tsl2583_chip *chip = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&chip->als_mutex);
+	ret = sprintf(buf, "%d\n", chip->als_settings.als_cal_target);
+	mutex_unlock(&chip->als_mutex);
+
+	return ret;
+}
+
+static ssize_t in_illuminance_input_target_store(struct device *dev,
+						 struct device_attribute *attr,
+						 const char *buf, size_t len)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct tsl2583_chip *chip = iio_priv(indio_dev);
+	int value;
+
+	if (kstrtoint(buf, 0, &value) || !value)
+		return -EINVAL;
+
+	mutex_lock(&chip->als_mutex);
+	chip->als_settings.als_cal_target = value;
+	mutex_unlock(&chip->als_mutex);
+
+	return len;
+}
+
+static ssize_t in_illuminance_calibrate_store(struct device *dev,
+					      struct device_attribute *attr,
+					      const char *buf, size_t len)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct tsl2583_chip *chip = iio_priv(indio_dev);
+	int value, ret;
+
+	if (kstrtoint(buf, 0, &value) || value != 1)
+		return -EINVAL;
+
+	mutex_lock(&chip->als_mutex);
+
+	if (chip->suspended) {
+		ret = -EBUSY;
+		goto done;
+	}
+
+	ret = tsl2583_als_calibrate(indio_dev);
+	if (ret < 0)
+		goto done;
+
+	ret = len;
+done:
+	mutex_unlock(&chip->als_mutex);
+
+	return ret;
+}
+
+static ssize_t in_illuminance_lux_table_show(struct device *dev,
+					     struct device_attribute *attr,
+					     char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct tsl2583_chip *chip = iio_priv(indio_dev);
+	unsigned int i;
+	int offset = 0;
+
+	for (i = 0; i < ARRAY_SIZE(chip->als_settings.als_device_lux); i++) {
+		offset += sprintf(buf + offset, "%u,%u,%u,",
+				  chip->als_settings.als_device_lux[i].ratio,
+				  chip->als_settings.als_device_lux[i].ch0,
+				  chip->als_settings.als_device_lux[i].ch1);
+		if (chip->als_settings.als_device_lux[i].ratio == 0) {
+			/*
+			 * We just printed the first "0" entry.
+			 * Now get rid of the extra "," and break.
+			 */
+			offset--;
+			break;
+		}
+	}
+
+	offset += sprintf(buf + offset, "\n");
+
+	return offset;
+}
+
+static ssize_t in_illuminance_lux_table_store(struct device *dev,
+					      struct device_attribute *attr,
+					      const char *buf, size_t len)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct tsl2583_chip *chip = iio_priv(indio_dev);
+	const unsigned int max_ints = TSL2583_MAX_LUX_TABLE_ENTRIES * 3;
+	int value[TSL2583_MAX_LUX_TABLE_ENTRIES * 3 + 1];
+	int ret = -EINVAL;
+	unsigned int n;
+
+	mutex_lock(&chip->als_mutex);
+
+	get_options(buf, ARRAY_SIZE(value), value);
+
+	/*
+	 * We now have an array of ints starting at value[1], and
+	 * enumerated by value[0].
+	 * We expect each group of three ints is one table entry,
+	 * and the last table entry is all 0.
+	 */
+	n = value[0];
+	if ((n % 3) || n < 6 || n > max_ints) {
+		dev_err(dev,
+			"%s: The number of entries in the lux table must be a multiple of 3 and within the range [6, %d]\n",
+			__func__, max_ints);
+		goto done;
+	}
+	if ((value[n - 2] | value[n - 1] | value[n]) != 0) {
+		dev_err(dev, "%s: The last 3 entries in the lux table must be zeros.\n",
+			__func__);
+		goto done;
+	}
+
+	memcpy(chip->als_settings.als_device_lux, &value[1],
+	       value[0] * sizeof(value[1]));
+
+	ret = len;
+
+done:
+	mutex_unlock(&chip->als_mutex);
+
+	return ret;
+}
+
+static IIO_CONST_ATTR(in_illuminance_calibscale_available, "1 8 16 111");
+static IIO_CONST_ATTR(in_illuminance_integration_time_available,
+		      "0.000050 0.000100 0.000150 0.000200 0.000250 0.000300 0.000350 0.000400 0.000450 0.000500 0.000550 0.000600 0.000650");
+static IIO_DEVICE_ATTR_RW(in_illuminance_input_target, 0);
+static IIO_DEVICE_ATTR_WO(in_illuminance_calibrate, 0);
+static IIO_DEVICE_ATTR_RW(in_illuminance_lux_table, 0);
+
+static struct attribute *sysfs_attrs_ctrl[] = {
+	&iio_const_attr_in_illuminance_calibscale_available.dev_attr.attr,
+	&iio_const_attr_in_illuminance_integration_time_available.dev_attr.attr,
+	&iio_dev_attr_in_illuminance_input_target.dev_attr.attr,
+	&iio_dev_attr_in_illuminance_calibrate.dev_attr.attr,
+	&iio_dev_attr_in_illuminance_lux_table.dev_attr.attr,
+	NULL
+};
+
+static const struct attribute_group tsl2583_attribute_group = {
+	.attrs = sysfs_attrs_ctrl,
+};
+
+static const struct iio_chan_spec tsl2583_channels[] = {
+	{
+		.type = IIO_LIGHT,
+		.modified = 1,
+		.channel2 = IIO_MOD_LIGHT_IR,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+	},
+	{
+		.type = IIO_LIGHT,
+		.modified = 1,
+		.channel2 = IIO_MOD_LIGHT_BOTH,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+	},
+	{
+		.type = IIO_LIGHT,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED) |
+				      BIT(IIO_CHAN_INFO_CALIBBIAS) |
+				      BIT(IIO_CHAN_INFO_CALIBSCALE) |
+				      BIT(IIO_CHAN_INFO_INT_TIME),
+	},
+};
+
+static int tsl2583_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long mask)
+{
+	struct tsl2583_chip *chip = iio_priv(indio_dev);
+	int ret = -EINVAL;
+
+	mutex_lock(&chip->als_mutex);
+
+	if (chip->suspended) {
+		ret = -EBUSY;
+		goto read_done;
+	}
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		if (chan->type == IIO_LIGHT) {
+			ret = tsl2583_get_lux(indio_dev);
+			if (ret < 0)
+				goto read_done;
+
+			/*
+			 * From page 20 of the TSL2581, TSL2583 data
+			 * sheet (TAOS134 − MARCH 2011):
+			 *
+			 * One of the photodiodes (channel 0) is
+			 * sensitive to both visible and infrared light,
+			 * while the second photodiode (channel 1) is
+			 * sensitive primarily to infrared light.
+			 */
+			if (chan->channel2 == IIO_MOD_LIGHT_BOTH)
+				*val = chip->als_cur_info.als_ch0;
+			else
+				*val = chip->als_cur_info.als_ch1;
+
+			ret = IIO_VAL_INT;
+		}
+		break;
+	case IIO_CHAN_INFO_PROCESSED:
+		if (chan->type == IIO_LIGHT) {
+			ret = tsl2583_get_lux(indio_dev);
+			if (ret < 0)
+				goto read_done;
+
+			*val = ret;
+			ret = IIO_VAL_INT;
+		}
+		break;
+	case IIO_CHAN_INFO_CALIBBIAS:
+		if (chan->type == IIO_LIGHT) {
+			*val = chip->als_settings.als_gain_trim;
+			ret = IIO_VAL_INT;
+		}
+		break;
+	case IIO_CHAN_INFO_CALIBSCALE:
+		if (chan->type == IIO_LIGHT) {
+			*val = gainadj[chip->als_settings.als_gain].mean;
+			ret = IIO_VAL_INT;
+		}
+		break;
+	case IIO_CHAN_INFO_INT_TIME:
+		if (chan->type == IIO_LIGHT) {
+			*val = 0;
+			*val2 = chip->als_settings.als_time;
+			ret = IIO_VAL_INT_PLUS_MICRO;
+		}
+		break;
+	default:
+		break;
+	}
+
+read_done:
+	mutex_unlock(&chip->als_mutex);
+
+	return ret;
+}
+
+static int tsl2583_write_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
+{
+	struct tsl2583_chip *chip = iio_priv(indio_dev);
+	int ret = -EINVAL;
+
+	mutex_lock(&chip->als_mutex);
+
+	if (chip->suspended) {
+		ret = -EBUSY;
+		goto write_done;
+	}
+
+	switch (mask) {
+	case IIO_CHAN_INFO_CALIBBIAS:
+		if (chan->type == IIO_LIGHT) {
+			chip->als_settings.als_gain_trim = val;
+			ret = 0;
+		}
+		break;
+	case IIO_CHAN_INFO_CALIBSCALE:
+		if (chan->type == IIO_LIGHT) {
+			unsigned int i;
+
+			for (i = 0; i < ARRAY_SIZE(gainadj); i++) {
+				if (gainadj[i].mean == val) {
+					chip->als_settings.als_gain = i;
+					ret = tsl2583_set_als_gain(chip);
+					break;
+				}
+			}
+		}
+		break;
+	case IIO_CHAN_INFO_INT_TIME:
+		if (chan->type == IIO_LIGHT && !val && val2 >= 50 &&
+		    val2 <= 650 && !(val2 % 50)) {
+			chip->als_settings.als_time = val2;
+			ret = tsl2583_set_als_time(chip);
+		}
+		break;
+	default:
+		break;
+	}
+
+write_done:
+	mutex_unlock(&chip->als_mutex);
+
+	return ret;
+}
+
+static const struct iio_info tsl2583_info = {
+	.attrs = &tsl2583_attribute_group,
+	.driver_module = THIS_MODULE,
+	.read_raw = tsl2583_read_raw,
+	.write_raw = tsl2583_write_raw,
+};
+
+static int tsl2583_probe(struct i2c_client *clientp,
+			 const struct i2c_device_id *idp)
+{
+	int ret;
+	struct tsl2583_chip *chip;
+	struct iio_dev *indio_dev;
+
+	if (!i2c_check_functionality(clientp->adapter,
+				     I2C_FUNC_SMBUS_BYTE_DATA)) {
+		dev_err(&clientp->dev, "%s: i2c smbus byte data functionality is unsupported\n",
+			__func__);
+		return -EOPNOTSUPP;
+	}
+
+	indio_dev = devm_iio_device_alloc(&clientp->dev, sizeof(*chip));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	chip = iio_priv(indio_dev);
+	chip->client = clientp;
+	i2c_set_clientdata(clientp, indio_dev);
+
+	mutex_init(&chip->als_mutex);
+	chip->suspended = true;
+
+	ret = i2c_smbus_read_byte_data(clientp,
+				       TSL2583_CMD_REG | TSL2583_CHIPID);
+	if (ret < 0) {
+		dev_err(&clientp->dev,
+			"%s: failed to read the chip ID register\n", __func__);
+		return ret;
+	}
+
+	if ((ret & TSL2583_CHIP_ID_MASK) != TSL2583_CHIP_ID) {
+		dev_err(&clientp->dev, "%s: received an unknown chip ID %x\n",
+			__func__, ret);
+		return -EINVAL;
+	}
+
+	indio_dev->info = &tsl2583_info;
+	indio_dev->channels = tsl2583_channels;
+	indio_dev->num_channels = ARRAY_SIZE(tsl2583_channels);
+	indio_dev->dev.parent = &clientp->dev;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->name = chip->client->name;
+
+	ret = devm_iio_device_register(indio_dev->dev.parent, indio_dev);
+	if (ret) {
+		dev_err(&clientp->dev, "%s: iio registration failed\n",
+			__func__);
+		return ret;
+	}
+
+	/* Load up the V2 defaults (these are hard coded defaults for now) */
+	tsl2583_defaults(chip);
+
+	/* Make sure the chip is on */
+	ret = tsl2583_chip_init_and_power_on(indio_dev);
+	if (ret < 0)
+		return ret;
+
+	dev_info(&clientp->dev, "Light sensor found.\n");
+
+	return 0;
+}
+
+static int __maybe_unused tsl2583_suspend(struct device *dev)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev));
+	struct tsl2583_chip *chip = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&chip->als_mutex);
+
+	ret = tsl2583_set_power_state(chip, TSL2583_CNTL_PWR_OFF);
+	chip->suspended = true;
+
+	mutex_unlock(&chip->als_mutex);
+
+	return ret;
+}
+
+static int __maybe_unused tsl2583_resume(struct device *dev)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev));
+	struct tsl2583_chip *chip = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&chip->als_mutex);
+
+	ret = tsl2583_chip_init_and_power_on(indio_dev);
+
+	mutex_unlock(&chip->als_mutex);
+
+	return ret;
+}
+
+static SIMPLE_DEV_PM_OPS(tsl2583_pm_ops, tsl2583_suspend, tsl2583_resume);
+
+static struct i2c_device_id tsl2583_idtable[] = {
+	{ "tsl2580", 0 },
+	{ "tsl2581", 1 },
+	{ "tsl2583", 2 },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, tsl2583_idtable);
+
+static const struct of_device_id tsl2583_of_match[] = {
+	{ .compatible = "amstaos,tsl2580", },
+	{ .compatible = "amstaos,tsl2581", },
+	{ .compatible = "amstaos,tsl2583", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, tsl2583_of_match);
+
+/* Driver definition */
+static struct i2c_driver tsl2583_driver = {
+	.driver = {
+		.name = "tsl2583",
+		.pm = &tsl2583_pm_ops,
+		.of_match_table = tsl2583_of_match,
+	},
+	.id_table = tsl2583_idtable,
+	.probe = tsl2583_probe,
+};
+module_i2c_driver(tsl2583_driver);
+
+MODULE_AUTHOR("J. August Brenner <jbrenner@taosinc.com>");
+MODULE_AUTHOR("Brian Masney <masneyb@onstation.org>");
+MODULE_DESCRIPTION("TAOS tsl2583 ambient light sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/magnetometer/ak8974.c b/drivers/iio/magnetometer/ak8974.c
index 217353145676..ce09d771c1fb 100644
--- a/drivers/iio/magnetometer/ak8974.c
+++ b/drivers/iio/magnetometer/ak8974.c
@@ -287,7 +287,7 @@ static int ak8974_await_drdy(struct ak8974 *ak8974)
 	return 0;
 }
 
-static int ak8974_getresult(struct ak8974 *ak8974, s16 *result)
+static int ak8974_getresult(struct ak8974 *ak8974, __le16 *result)
 {
 	unsigned int src;
 	int ret;
@@ -395,7 +395,7 @@ static int ak8974_selftest(struct ak8974 *ak8974)
 static int ak8974_get_u16_val(struct ak8974 *ak8974, u8 reg, u16 *val)
 {
 	int ret;
-	u16 bulk;
+	__le16 bulk;
 
 	ret = regmap_bulk_read(ak8974->map, reg, &bulk, 2);
 	if (ret)
@@ -453,7 +453,7 @@ static int ak8974_read_raw(struct iio_dev *indio_dev,
 			   long mask)
 {
 	struct ak8974 *ak8974 = iio_priv(indio_dev);
-	s16 hw_values[3];
+	__le16 hw_values[3];
 	int ret = -EINVAL;
 
 	pm_runtime_get_sync(&ak8974->i2c->dev);
@@ -494,7 +494,7 @@ static void ak8974_fill_buffer(struct iio_dev *indio_dev)
 {
 	struct ak8974 *ak8974 = iio_priv(indio_dev);
 	int ret;
-	s16 hw_values[8]; /* Three axes + 64bit padding */
+	__le16 hw_values[8]; /* Three axes + 64bit padding */
 
 	pm_runtime_get_sync(&ak8974->i2c->dev);
 	mutex_lock(&ak8974->lock);
diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c
index af8606cc7812..825369fb1c57 100644
--- a/drivers/iio/magnetometer/ak8975.c
+++ b/drivers/iio/magnetometer/ak8975.c
@@ -690,6 +690,7 @@ static int ak8975_read_axis(struct iio_dev *indio_dev, int index, int *val)
 	struct ak8975_data *data = iio_priv(indio_dev);
 	const struct i2c_client *client = data->client;
 	const struct ak_def *def = data->def;
+	__le16 rval;
 	u16 buff;
 	int ret;
 
@@ -703,7 +704,7 @@ static int ak8975_read_axis(struct iio_dev *indio_dev, int index, int *val)
 
 	ret = i2c_smbus_read_i2c_block_data_or_emulated(
 			client, def->data_regs[index],
-			sizeof(buff), (u8*)&buff);
+			sizeof(rval), (u8*)&rval);
 	if (ret < 0)
 		goto exit;
 
@@ -713,7 +714,7 @@ static int ak8975_read_axis(struct iio_dev *indio_dev, int index, int *val)
 	pm_runtime_put_autosuspend(&data->client->dev);
 
 	/* Swap bytes and convert to valid range. */
-	buff = le16_to_cpu(buff);
+	buff = le16_to_cpu(rval);
 	*val = clamp_t(s16, buff, -def->range, def->range);
 	return IIO_VAL_INT;
 
@@ -813,6 +814,7 @@ static void ak8975_fill_buffer(struct iio_dev *indio_dev)
 	const struct ak_def *def = data->def;
 	int ret;
 	s16 buff[8]; /* 3 x 16 bits axis values + 1 aligned 64 bits timestamp */
+	__le16 fval[3];
 
 	mutex_lock(&data->lock);
 
@@ -826,17 +828,17 @@ static void ak8975_fill_buffer(struct iio_dev *indio_dev)
 	 */
 	ret = i2c_smbus_read_i2c_block_data_or_emulated(client,
 							def->data_regs[0],
-							3 * sizeof(buff[0]),
-							(u8 *)buff);
+							3 * sizeof(fval[0]),
+							(u8 *)fval);
 	if (ret < 0)
 		goto unlock;
 
 	mutex_unlock(&data->lock);
 
 	/* Clamp to valid range. */
-	buff[0] = clamp_t(s16, le16_to_cpu(buff[0]), -def->range, def->range);
-	buff[1] = clamp_t(s16, le16_to_cpu(buff[1]), -def->range, def->range);
-	buff[2] = clamp_t(s16, le16_to_cpu(buff[2]), -def->range, def->range);
+	buff[0] = clamp_t(s16, le16_to_cpu(fval[0]), -def->range, def->range);
+	buff[1] = clamp_t(s16, le16_to_cpu(fval[1]), -def->range, def->range);
+	buff[2] = clamp_t(s16, le16_to_cpu(fval[2]), -def->range, def->range);
 
 	iio_push_to_buffers_with_timestamp(indio_dev, buff,
 					   iio_get_time_ns(indio_dev));
diff --git a/drivers/iio/magnetometer/hid-sensor-magn-3d.c b/drivers/iio/magnetometer/hid-sensor-magn-3d.c
index d8a0c8da8db0..0e791b02ed4a 100644
--- a/drivers/iio/magnetometer/hid-sensor-magn-3d.c
+++ b/drivers/iio/magnetometer/hid-sensor-magn-3d.c
@@ -42,9 +42,17 @@ enum magn_3d_channel {
 	MAGN_3D_CHANNEL_MAX,
 };
 
+struct common_attributes {
+	int scale_pre_decml;
+	int scale_post_decml;
+	int scale_precision;
+	int value_offset;
+};
+
 struct magn_3d_state {
 	struct hid_sensor_hub_callbacks callbacks;
-	struct hid_sensor_common common_attributes;
+	struct hid_sensor_common magn_flux_attributes;
+	struct hid_sensor_common rot_attributes;
 	struct hid_sensor_hub_attribute_info magn[MAGN_3D_CHANNEL_MAX];
 
 	/* dynamically sized array to hold sensor values */
@@ -52,10 +60,8 @@ struct magn_3d_state {
 	/* array of pointers to sensor value */
 	u32 *magn_val_addr[MAGN_3D_CHANNEL_MAX];
 
-	int scale_pre_decml;
-	int scale_post_decml;
-	int scale_precision;
-	int value_offset;
+	struct common_attributes magn_flux_attr;
+	struct common_attributes rot_attr;
 };
 
 static const u32 magn_3d_addresses[MAGN_3D_CHANNEL_MAX] = {
@@ -162,41 +168,74 @@ static int magn_3d_read_raw(struct iio_dev *indio_dev,
 	*val2 = 0;
 	switch (mask) {
 	case 0:
-		hid_sensor_power_state(&magn_state->common_attributes, true);
+		hid_sensor_power_state(&magn_state->magn_flux_attributes, true);
 		report_id =
 			magn_state->magn[chan->address].report_id;
 		address = magn_3d_addresses[chan->address];
 		if (report_id >= 0)
 			*val = sensor_hub_input_attr_get_raw_value(
-				magn_state->common_attributes.hsdev,
+				magn_state->magn_flux_attributes.hsdev,
 				HID_USAGE_SENSOR_COMPASS_3D, address,
 				report_id,
 				SENSOR_HUB_SYNC);
 		else {
 			*val = 0;
-			hid_sensor_power_state(&magn_state->common_attributes,
-						false);
+			hid_sensor_power_state(
+				&magn_state->magn_flux_attributes,
+				false);
 			return -EINVAL;
 		}
-		hid_sensor_power_state(&magn_state->common_attributes, false);
+		hid_sensor_power_state(&magn_state->magn_flux_attributes,
+					false);
 		ret_type = IIO_VAL_INT;
 		break;
 	case IIO_CHAN_INFO_SCALE:
-		*val = magn_state->scale_pre_decml;
-		*val2 = magn_state->scale_post_decml;
-		ret_type = magn_state->scale_precision;
+		switch (chan->type) {
+		case IIO_MAGN:
+			*val = magn_state->magn_flux_attr.scale_pre_decml;
+			*val2 = magn_state->magn_flux_attr.scale_post_decml;
+			ret_type = magn_state->magn_flux_attr.scale_precision;
+			break;
+		case IIO_ROT:
+			*val = magn_state->rot_attr.scale_pre_decml;
+			*val2 = magn_state->rot_attr.scale_post_decml;
+			ret_type = magn_state->rot_attr.scale_precision;
+			break;
+		default:
+			ret_type = -EINVAL;
+		}
 		break;
 	case IIO_CHAN_INFO_OFFSET:
-		*val = magn_state->value_offset;
-		ret_type = IIO_VAL_INT;
+		switch (chan->type) {
+		case IIO_MAGN:
+			*val = magn_state->magn_flux_attr.value_offset;
+			ret_type = IIO_VAL_INT;
+			break;
+		case IIO_ROT:
+			*val = magn_state->rot_attr.value_offset;
+			ret_type = IIO_VAL_INT;
+			break;
+		default:
+			ret_type = -EINVAL;
+		}
 		break;
 	case IIO_CHAN_INFO_SAMP_FREQ:
 		ret_type = hid_sensor_read_samp_freq_value(
-			&magn_state->common_attributes, val, val2);
+			&magn_state->magn_flux_attributes, val, val2);
 		break;
 	case IIO_CHAN_INFO_HYSTERESIS:
-		ret_type = hid_sensor_read_raw_hyst_value(
-			&magn_state->common_attributes, val, val2);
+		switch (chan->type) {
+		case IIO_MAGN:
+			ret_type = hid_sensor_read_raw_hyst_value(
+				&magn_state->magn_flux_attributes, val, val2);
+			break;
+		case IIO_ROT:
+			ret_type = hid_sensor_read_raw_hyst_value(
+				&magn_state->rot_attributes, val, val2);
+			break;
+		default:
+			ret_type = -EINVAL;
+		}
 		break;
 	default:
 		ret_type = -EINVAL;
@@ -219,11 +258,21 @@ static int magn_3d_write_raw(struct iio_dev *indio_dev,
 	switch (mask) {
 	case IIO_CHAN_INFO_SAMP_FREQ:
 		ret = hid_sensor_write_samp_freq_value(
-				&magn_state->common_attributes, val, val2);
+				&magn_state->magn_flux_attributes, val, val2);
 		break;
 	case IIO_CHAN_INFO_HYSTERESIS:
-		ret = hid_sensor_write_raw_hyst_value(
-				&magn_state->common_attributes, val, val2);
+		switch (chan->type) {
+		case IIO_MAGN:
+			ret = hid_sensor_write_raw_hyst_value(
+				&magn_state->magn_flux_attributes, val, val2);
+			break;
+		case IIO_ROT:
+			ret = hid_sensor_write_raw_hyst_value(
+				&magn_state->rot_attributes, val, val2);
+			break;
+		default:
+			ret = -EINVAL;
+		}
 		break;
 	default:
 		ret = -EINVAL;
@@ -254,7 +303,7 @@ static int magn_3d_proc_event(struct hid_sensor_hub_device *hsdev,
 	struct magn_3d_state *magn_state = iio_priv(indio_dev);
 
 	dev_dbg(&indio_dev->dev, "magn_3d_proc_event\n");
-	if (atomic_read(&magn_state->common_attributes.data_ready))
+	if (atomic_read(&magn_state->magn_flux_attributes.data_ready))
 		hid_sensor_push_data(indio_dev, magn_state->iio_vals);
 
 	return 0;
@@ -389,21 +438,48 @@ static int magn_3d_parse_report(struct platform_device *pdev,
 	dev_dbg(&pdev->dev, "magn_3d Setup %d IIO channels\n",
 			*chan_count);
 
-	st->scale_precision = hid_sensor_format_scale(
+	st->magn_flux_attr.scale_precision = hid_sensor_format_scale(
 				HID_USAGE_SENSOR_COMPASS_3D,
 				&st->magn[CHANNEL_SCAN_INDEX_X],
-				&st->scale_pre_decml, &st->scale_post_decml);
+				&st->magn_flux_attr.scale_pre_decml,
+				&st->magn_flux_attr.scale_post_decml);
+	st->rot_attr.scale_precision
+		= hid_sensor_format_scale(
+			HID_USAGE_SENSOR_ORIENT_COMP_MAGN_NORTH,
+			&st->magn[CHANNEL_SCAN_INDEX_NORTH_MAGN_TILT_COMP],
+			&st->rot_attr.scale_pre_decml,
+			&st->rot_attr.scale_post_decml);
 
 	/* Set Sensitivity field ids, when there is no individual modifier */
-	if (st->common_attributes.sensitivity.index < 0) {
+	if (st->magn_flux_attributes.sensitivity.index < 0) {
 		sensor_hub_input_get_attribute_info(hsdev,
 			HID_FEATURE_REPORT, usage_id,
 			HID_USAGE_SENSOR_DATA_MOD_CHANGE_SENSITIVITY_ABS |
 			HID_USAGE_SENSOR_DATA_ORIENTATION,
-			&st->common_attributes.sensitivity);
+			&st->magn_flux_attributes.sensitivity);
+		dev_dbg(&pdev->dev, "Sensitivity index:report %d:%d\n",
+			st->magn_flux_attributes.sensitivity.index,
+			st->magn_flux_attributes.sensitivity.report_id);
+	}
+	if (st->magn_flux_attributes.sensitivity.index < 0) {
+		sensor_hub_input_get_attribute_info(hsdev,
+			HID_FEATURE_REPORT, usage_id,
+			HID_USAGE_SENSOR_DATA_MOD_CHANGE_SENSITIVITY_ABS |
+			HID_USAGE_SENSOR_ORIENT_MAGN_FLUX,
+			&st->magn_flux_attributes.sensitivity);
+		dev_dbg(&pdev->dev, "Sensitivity index:report %d:%d\n",
+			st->magn_flux_attributes.sensitivity.index,
+			st->magn_flux_attributes.sensitivity.report_id);
+	}
+	if (st->rot_attributes.sensitivity.index < 0) {
+		sensor_hub_input_get_attribute_info(hsdev,
+			HID_FEATURE_REPORT, usage_id,
+			HID_USAGE_SENSOR_DATA_MOD_CHANGE_SENSITIVITY_ABS |
+			HID_USAGE_SENSOR_ORIENT_COMP_MAGN_NORTH,
+			&st->rot_attributes.sensitivity);
 		dev_dbg(&pdev->dev, "Sensitivity index:report %d:%d\n",
-			st->common_attributes.sensitivity.index,
-			st->common_attributes.sensitivity.report_id);
+			st->rot_attributes.sensitivity.index,
+			st->rot_attributes.sensitivity.report_id);
 	}
 
 	return 0;
@@ -428,16 +504,17 @@ static int hid_magn_3d_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, indio_dev);
 
 	magn_state = iio_priv(indio_dev);
-	magn_state->common_attributes.hsdev = hsdev;
-	magn_state->common_attributes.pdev = pdev;
+	magn_state->magn_flux_attributes.hsdev = hsdev;
+	magn_state->magn_flux_attributes.pdev = pdev;
 
 	ret = hid_sensor_parse_common_attributes(hsdev,
 				HID_USAGE_SENSOR_COMPASS_3D,
-				&magn_state->common_attributes);
+				&magn_state->magn_flux_attributes);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to setup common attributes\n");
 		return ret;
 	}
+	magn_state->rot_attributes = magn_state->magn_flux_attributes;
 
 	ret = magn_3d_parse_report(pdev, hsdev,
 				&channels, &chan_count,
@@ -460,9 +537,9 @@ static int hid_magn_3d_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "failed to initialize trigger buffer\n");
 		return ret;
 	}
-	atomic_set(&magn_state->common_attributes.data_ready, 0);
+	atomic_set(&magn_state->magn_flux_attributes.data_ready, 0);
 	ret = hid_sensor_setup_trigger(indio_dev, name,
-					&magn_state->common_attributes);
+					&magn_state->magn_flux_attributes);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
 		goto error_unreg_buffer_funcs;
@@ -489,7 +566,7 @@ static int hid_magn_3d_probe(struct platform_device *pdev)
 error_iio_unreg:
 	iio_device_unregister(indio_dev);
 error_remove_trigger:
-	hid_sensor_remove_trigger(&magn_state->common_attributes);
+	hid_sensor_remove_trigger(&magn_state->magn_flux_attributes);
 error_unreg_buffer_funcs:
 	iio_triggered_buffer_cleanup(indio_dev);
 	return ret;
@@ -504,7 +581,7 @@ static int hid_magn_3d_remove(struct platform_device *pdev)
 
 	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_COMPASS_3D);
 	iio_device_unregister(indio_dev);
-	hid_sensor_remove_trigger(&magn_state->common_attributes);
+	hid_sensor_remove_trigger(&magn_state->magn_flux_attributes);
 	iio_triggered_buffer_cleanup(indio_dev);
 
 	return 0;
diff --git a/drivers/iio/magnetometer/st_magn_core.c b/drivers/iio/magnetometer/st_magn_core.c
index 3e1f06b2224c..8e1b0861fbe4 100644
--- a/drivers/iio/magnetometer/st_magn_core.c
+++ b/drivers/iio/magnetometer/st_magn_core.c
@@ -46,139 +46,12 @@
 #define ST_MAGN_FS_AVL_15000MG			15000
 #define ST_MAGN_FS_AVL_16000MG			16000
 
-/* CUSTOM VALUES FOR SENSOR 0 */
-#define ST_MAGN_0_ODR_ADDR			0x00
-#define ST_MAGN_0_ODR_MASK			0x1c
-#define ST_MAGN_0_ODR_AVL_1HZ_VAL		0x00
-#define ST_MAGN_0_ODR_AVL_2HZ_VAL		0x01
-#define ST_MAGN_0_ODR_AVL_3HZ_VAL		0x02
-#define ST_MAGN_0_ODR_AVL_8HZ_VAL		0x03
-#define ST_MAGN_0_ODR_AVL_15HZ_VAL		0x04
-#define ST_MAGN_0_ODR_AVL_30HZ_VAL		0x05
-#define ST_MAGN_0_ODR_AVL_75HZ_VAL		0x06
-#define ST_MAGN_0_ODR_AVL_220HZ_VAL		0x07
-#define ST_MAGN_0_PW_ADDR			0x02
-#define ST_MAGN_0_PW_MASK			0x03
-#define ST_MAGN_0_PW_ON				0x00
-#define ST_MAGN_0_PW_OFF			0x03
-#define ST_MAGN_0_FS_ADDR			0x01
-#define ST_MAGN_0_FS_MASK			0xe0
-#define ST_MAGN_0_FS_AVL_1300_VAL		0x01
-#define ST_MAGN_0_FS_AVL_1900_VAL		0x02
-#define ST_MAGN_0_FS_AVL_2500_VAL		0x03
-#define ST_MAGN_0_FS_AVL_4000_VAL		0x04
-#define ST_MAGN_0_FS_AVL_4700_VAL		0x05
-#define ST_MAGN_0_FS_AVL_5600_VAL		0x06
-#define ST_MAGN_0_FS_AVL_8100_VAL		0x07
-#define ST_MAGN_0_FS_AVL_1300_GAIN_XY		1100
-#define ST_MAGN_0_FS_AVL_1900_GAIN_XY		855
-#define ST_MAGN_0_FS_AVL_2500_GAIN_XY		670
-#define ST_MAGN_0_FS_AVL_4000_GAIN_XY		450
-#define ST_MAGN_0_FS_AVL_4700_GAIN_XY		400
-#define ST_MAGN_0_FS_AVL_5600_GAIN_XY		330
-#define ST_MAGN_0_FS_AVL_8100_GAIN_XY		230
-#define ST_MAGN_0_FS_AVL_1300_GAIN_Z		980
-#define ST_MAGN_0_FS_AVL_1900_GAIN_Z		760
-#define ST_MAGN_0_FS_AVL_2500_GAIN_Z		600
-#define ST_MAGN_0_FS_AVL_4000_GAIN_Z		400
-#define ST_MAGN_0_FS_AVL_4700_GAIN_Z		355
-#define ST_MAGN_0_FS_AVL_5600_GAIN_Z		295
-#define ST_MAGN_0_FS_AVL_8100_GAIN_Z		205
-#define ST_MAGN_0_MULTIREAD_BIT			false
-
-/* CUSTOM VALUES FOR SENSOR 1 */
-#define ST_MAGN_1_WAI_EXP			0x3c
-#define ST_MAGN_1_ODR_ADDR			0x00
-#define ST_MAGN_1_ODR_MASK			0x1c
-#define ST_MAGN_1_ODR_AVL_1HZ_VAL		0x00
-#define ST_MAGN_1_ODR_AVL_2HZ_VAL		0x01
-#define ST_MAGN_1_ODR_AVL_3HZ_VAL		0x02
-#define ST_MAGN_1_ODR_AVL_8HZ_VAL		0x03
-#define ST_MAGN_1_ODR_AVL_15HZ_VAL		0x04
-#define ST_MAGN_1_ODR_AVL_30HZ_VAL		0x05
-#define ST_MAGN_1_ODR_AVL_75HZ_VAL		0x06
-#define ST_MAGN_1_ODR_AVL_220HZ_VAL		0x07
-#define ST_MAGN_1_PW_ADDR			0x02
-#define ST_MAGN_1_PW_MASK			0x03
-#define ST_MAGN_1_PW_ON				0x00
-#define ST_MAGN_1_PW_OFF			0x03
-#define ST_MAGN_1_FS_ADDR			0x01
-#define ST_MAGN_1_FS_MASK			0xe0
-#define ST_MAGN_1_FS_AVL_1300_VAL		0x01
-#define ST_MAGN_1_FS_AVL_1900_VAL		0x02
-#define ST_MAGN_1_FS_AVL_2500_VAL		0x03
-#define ST_MAGN_1_FS_AVL_4000_VAL		0x04
-#define ST_MAGN_1_FS_AVL_4700_VAL		0x05
-#define ST_MAGN_1_FS_AVL_5600_VAL		0x06
-#define ST_MAGN_1_FS_AVL_8100_VAL		0x07
-#define ST_MAGN_1_FS_AVL_1300_GAIN_XY		909
-#define ST_MAGN_1_FS_AVL_1900_GAIN_XY		1169
-#define ST_MAGN_1_FS_AVL_2500_GAIN_XY		1492
-#define ST_MAGN_1_FS_AVL_4000_GAIN_XY		2222
-#define ST_MAGN_1_FS_AVL_4700_GAIN_XY		2500
-#define ST_MAGN_1_FS_AVL_5600_GAIN_XY		3030
-#define ST_MAGN_1_FS_AVL_8100_GAIN_XY		4347
-#define ST_MAGN_1_FS_AVL_1300_GAIN_Z		1020
-#define ST_MAGN_1_FS_AVL_1900_GAIN_Z		1315
-#define ST_MAGN_1_FS_AVL_2500_GAIN_Z		1666
-#define ST_MAGN_1_FS_AVL_4000_GAIN_Z		2500
-#define ST_MAGN_1_FS_AVL_4700_GAIN_Z		2816
-#define ST_MAGN_1_FS_AVL_5600_GAIN_Z		3389
-#define ST_MAGN_1_FS_AVL_8100_GAIN_Z		4878
-#define ST_MAGN_1_MULTIREAD_BIT			false
-
-/* CUSTOM VALUES FOR SENSOR 2 */
-#define ST_MAGN_2_WAI_EXP			0x3d
-#define ST_MAGN_2_ODR_ADDR			0x20
-#define ST_MAGN_2_ODR_MASK			0x1c
-#define ST_MAGN_2_ODR_AVL_1HZ_VAL		0x00
-#define ST_MAGN_2_ODR_AVL_2HZ_VAL		0x01
-#define ST_MAGN_2_ODR_AVL_3HZ_VAL		0x02
-#define ST_MAGN_2_ODR_AVL_5HZ_VAL		0x03
-#define ST_MAGN_2_ODR_AVL_10HZ_VAL		0x04
-#define ST_MAGN_2_ODR_AVL_20HZ_VAL		0x05
-#define ST_MAGN_2_ODR_AVL_40HZ_VAL		0x06
-#define ST_MAGN_2_ODR_AVL_80HZ_VAL		0x07
-#define ST_MAGN_2_PW_ADDR			0x22
-#define ST_MAGN_2_PW_MASK			0x03
-#define ST_MAGN_2_PW_ON				0x00
-#define ST_MAGN_2_PW_OFF			0x03
-#define ST_MAGN_2_FS_ADDR			0x21
-#define ST_MAGN_2_FS_MASK			0x60
-#define ST_MAGN_2_FS_AVL_4000_VAL		0x00
-#define ST_MAGN_2_FS_AVL_8000_VAL		0x01
-#define ST_MAGN_2_FS_AVL_12000_VAL		0x02
-#define ST_MAGN_2_FS_AVL_16000_VAL		0x03
-#define ST_MAGN_2_FS_AVL_4000_GAIN		146
-#define ST_MAGN_2_FS_AVL_8000_GAIN		292
-#define ST_MAGN_2_FS_AVL_12000_GAIN		438
-#define ST_MAGN_2_FS_AVL_16000_GAIN		584
-#define ST_MAGN_2_MULTIREAD_BIT			false
+/* Special L addresses for Sensor 2 */
 #define ST_MAGN_2_OUT_X_L_ADDR			0x28
 #define ST_MAGN_2_OUT_Y_L_ADDR			0x2a
 #define ST_MAGN_2_OUT_Z_L_ADDR			0x2c
 
-/* CUSTOM VALUES FOR SENSOR 3 */
-#define ST_MAGN_3_WAI_ADDR			0x4f
-#define ST_MAGN_3_WAI_EXP			0x40
-#define ST_MAGN_3_ODR_ADDR			0x60
-#define ST_MAGN_3_ODR_MASK			0x0c
-#define ST_MAGN_3_ODR_AVL_10HZ_VAL		0x00
-#define ST_MAGN_3_ODR_AVL_20HZ_VAL		0x01
-#define ST_MAGN_3_ODR_AVL_50HZ_VAL		0x02
-#define ST_MAGN_3_ODR_AVL_100HZ_VAL		0x03
-#define ST_MAGN_3_PW_ADDR			0x60
-#define ST_MAGN_3_PW_MASK			0x03
-#define ST_MAGN_3_PW_ON				0x00
-#define ST_MAGN_3_PW_OFF			0x03
-#define ST_MAGN_3_BDU_ADDR			0x62
-#define ST_MAGN_3_BDU_MASK			0x10
-#define ST_MAGN_3_DRDY_IRQ_ADDR			0x62
-#define ST_MAGN_3_DRDY_INT_MASK			0x01
-#define ST_MAGN_3_IHL_IRQ_ADDR			0x63
-#define ST_MAGN_3_IHL_IRQ_MASK			0x04
-#define ST_MAGN_3_FS_AVL_15000_GAIN		1500
-#define ST_MAGN_3_MULTIREAD_BIT			false
+/* Special L addresses for sensor 3 */
 #define ST_MAGN_3_OUT_X_L_ADDR			0x68
 #define ST_MAGN_3_OUT_Y_L_ADDR			0x6a
 #define ST_MAGN_3_OUT_Z_L_ADDR			0x6c
@@ -240,77 +113,78 @@ static const struct st_sensor_settings st_magn_sensors_settings[] = {
 		},
 		.ch = (struct iio_chan_spec *)st_magn_16bit_channels,
 		.odr = {
-			.addr = ST_MAGN_0_ODR_ADDR,
-			.mask = ST_MAGN_0_ODR_MASK,
+			.addr = 0x00,
+			.mask = 0x1c,
 			.odr_avl = {
-				{ 1, ST_MAGN_0_ODR_AVL_1HZ_VAL, },
-				{ 2, ST_MAGN_0_ODR_AVL_2HZ_VAL, },
-				{ 3, ST_MAGN_0_ODR_AVL_3HZ_VAL, },
-				{ 8, ST_MAGN_0_ODR_AVL_8HZ_VAL, },
-				{ 15, ST_MAGN_0_ODR_AVL_15HZ_VAL, },
-				{ 30, ST_MAGN_0_ODR_AVL_30HZ_VAL, },
-				{ 75, ST_MAGN_0_ODR_AVL_75HZ_VAL, },
+				{ .hz = 1, .value = 0x00 },
+				{ .hz = 2, .value = 0x01 },
+				{ .hz = 3, .value = 0x02 },
+				{ .hz = 8, .value = 0x03 },
+				{ .hz = 15, .value = 0x04 },
+				{ .hz = 30, .value = 0x05 },
+				{ .hz = 75, .value = 0x06 },
+				/* 220 Hz, 0x07 reportedly exist */
 			},
 		},
 		.pw = {
-			.addr = ST_MAGN_0_PW_ADDR,
-			.mask = ST_MAGN_0_PW_MASK,
-			.value_on = ST_MAGN_0_PW_ON,
-			.value_off = ST_MAGN_0_PW_OFF,
+			.addr = 0x02,
+			.mask = 0x03,
+			.value_on = 0x00,
+			.value_off = 0x03,
 		},
 		.fs = {
-			.addr = ST_MAGN_0_FS_ADDR,
-			.mask = ST_MAGN_0_FS_MASK,
+			.addr = 0x01,
+			.mask = 0xe0,
 			.fs_avl = {
 				[0] = {
 					.num = ST_MAGN_FS_AVL_1300MG,
-					.value = ST_MAGN_0_FS_AVL_1300_VAL,
-					.gain = ST_MAGN_0_FS_AVL_1300_GAIN_XY,
-					.gain2 = ST_MAGN_0_FS_AVL_1300_GAIN_Z,
+					.value = 0x01,
+					.gain = 1100,
+					.gain2 = 980,
 				},
 				[1] = {
 					.num = ST_MAGN_FS_AVL_1900MG,
-					.value = ST_MAGN_0_FS_AVL_1900_VAL,
-					.gain = ST_MAGN_0_FS_AVL_1900_GAIN_XY,
-					.gain2 = ST_MAGN_0_FS_AVL_1900_GAIN_Z,
+					.value = 0x02,
+					.gain = 855,
+					.gain2 = 760,
 				},
 				[2] = {
 					.num = ST_MAGN_FS_AVL_2500MG,
-					.value = ST_MAGN_0_FS_AVL_2500_VAL,
-					.gain = ST_MAGN_0_FS_AVL_2500_GAIN_XY,
-					.gain2 = ST_MAGN_0_FS_AVL_2500_GAIN_Z,
+					.value = 0x03,
+					.gain = 670,
+					.gain2 = 600,
 				},
 				[3] = {
 					.num = ST_MAGN_FS_AVL_4000MG,
-					.value = ST_MAGN_0_FS_AVL_4000_VAL,
-					.gain = ST_MAGN_0_FS_AVL_4000_GAIN_XY,
-					.gain2 = ST_MAGN_0_FS_AVL_4000_GAIN_Z,
+					.value = 0x04,
+					.gain = 450,
+					.gain2 = 400,
 				},
 				[4] = {
 					.num = ST_MAGN_FS_AVL_4700MG,
-					.value = ST_MAGN_0_FS_AVL_4700_VAL,
-					.gain = ST_MAGN_0_FS_AVL_4700_GAIN_XY,
-					.gain2 = ST_MAGN_0_FS_AVL_4700_GAIN_Z,
+					.value = 0x05,
+					.gain = 400,
+					.gain2 = 355,
 				},
 				[5] = {
 					.num = ST_MAGN_FS_AVL_5600MG,
-					.value = ST_MAGN_0_FS_AVL_5600_VAL,
-					.gain = ST_MAGN_0_FS_AVL_5600_GAIN_XY,
-					.gain2 = ST_MAGN_0_FS_AVL_5600_GAIN_Z,
+					.value = 0x06,
+					.gain = 330,
+					.gain2 = 295,
 				},
 				[6] = {
 					.num = ST_MAGN_FS_AVL_8100MG,
-					.value = ST_MAGN_0_FS_AVL_8100_VAL,
-					.gain = ST_MAGN_0_FS_AVL_8100_GAIN_XY,
-					.gain2 = ST_MAGN_0_FS_AVL_8100_GAIN_Z,
+					.value = 0x07,
+					.gain = 230,
+					.gain2 = 205,
 				},
 			},
 		},
-		.multi_read_bit = ST_MAGN_0_MULTIREAD_BIT,
+		.multi_read_bit = false,
 		.bootime = 2,
 	},
 	{
-		.wai = ST_MAGN_1_WAI_EXP,
+		.wai = 0x3c,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = LSM303DLHC_MAGN_DEV_NAME,
@@ -318,175 +192,175 @@ static const struct st_sensor_settings st_magn_sensors_settings[] = {
 		},
 		.ch = (struct iio_chan_spec *)st_magn_16bit_channels,
 		.odr = {
-			.addr = ST_MAGN_1_ODR_ADDR,
-			.mask = ST_MAGN_1_ODR_MASK,
+			.addr = 0x00,
+			.mask = 0x1c,
 			.odr_avl = {
-				{ 1, ST_MAGN_1_ODR_AVL_1HZ_VAL, },
-				{ 2, ST_MAGN_1_ODR_AVL_2HZ_VAL, },
-				{ 3, ST_MAGN_1_ODR_AVL_3HZ_VAL, },
-				{ 8, ST_MAGN_1_ODR_AVL_8HZ_VAL, },
-				{ 15, ST_MAGN_1_ODR_AVL_15HZ_VAL, },
-				{ 30, ST_MAGN_1_ODR_AVL_30HZ_VAL, },
-				{ 75, ST_MAGN_1_ODR_AVL_75HZ_VAL, },
-				{ 220, ST_MAGN_1_ODR_AVL_220HZ_VAL, },
+				{ .hz = 1, .value = 0x00 },
+				{ .hz = 2, .value = 0x01 },
+				{ .hz = 3, .value = 0x02 },
+				{ .hz = 8, .value = 0x03 },
+				{ .hz = 15, .value = 0x04 },
+				{ .hz = 30, .value = 0x05 },
+				{ .hz = 75, .value = 0x06 },
+				{ .hz = 220, .value = 0x07 },
 			},
 		},
 		.pw = {
-			.addr = ST_MAGN_1_PW_ADDR,
-			.mask = ST_MAGN_1_PW_MASK,
-			.value_on = ST_MAGN_1_PW_ON,
-			.value_off = ST_MAGN_1_PW_OFF,
+			.addr = 0x02,
+			.mask = 0x03,
+			.value_on = 0x00,
+			.value_off = 0x03,
 		},
 		.fs = {
-			.addr = ST_MAGN_1_FS_ADDR,
-			.mask = ST_MAGN_1_FS_MASK,
+			.addr = 0x01,
+			.mask = 0xe0,
 			.fs_avl = {
 				[0] = {
 					.num = ST_MAGN_FS_AVL_1300MG,
-					.value = ST_MAGN_1_FS_AVL_1300_VAL,
-					.gain = ST_MAGN_1_FS_AVL_1300_GAIN_XY,
-					.gain2 = ST_MAGN_1_FS_AVL_1300_GAIN_Z,
+					.value = 0x01,
+					.gain = 909,
+					.gain2 = 1020,
 				},
 				[1] = {
 					.num = ST_MAGN_FS_AVL_1900MG,
-					.value = ST_MAGN_1_FS_AVL_1900_VAL,
-					.gain = ST_MAGN_1_FS_AVL_1900_GAIN_XY,
-					.gain2 = ST_MAGN_1_FS_AVL_1900_GAIN_Z,
+					.value = 0x02,
+					.gain = 1169,
+					.gain2 = 1315,
 				},
 				[2] = {
 					.num = ST_MAGN_FS_AVL_2500MG,
-					.value = ST_MAGN_1_FS_AVL_2500_VAL,
-					.gain = ST_MAGN_1_FS_AVL_2500_GAIN_XY,
-					.gain2 = ST_MAGN_1_FS_AVL_2500_GAIN_Z,
+					.value = 0x03,
+					.gain = 1492,
+					.gain2 = 1666,
 				},
 				[3] = {
 					.num = ST_MAGN_FS_AVL_4000MG,
-					.value = ST_MAGN_1_FS_AVL_4000_VAL,
-					.gain = ST_MAGN_1_FS_AVL_4000_GAIN_XY,
-					.gain2 = ST_MAGN_1_FS_AVL_4000_GAIN_Z,
+					.value = 0x04,
+					.gain = 2222,
+					.gain2 = 2500,
 				},
 				[4] = {
 					.num = ST_MAGN_FS_AVL_4700MG,
-					.value = ST_MAGN_1_FS_AVL_4700_VAL,
-					.gain = ST_MAGN_1_FS_AVL_4700_GAIN_XY,
-					.gain2 = ST_MAGN_1_FS_AVL_4700_GAIN_Z,
+					.value = 0x05,
+					.gain = 2500,
+					.gain2 = 2816,
 				},
 				[5] = {
 					.num = ST_MAGN_FS_AVL_5600MG,
-					.value = ST_MAGN_1_FS_AVL_5600_VAL,
-					.gain = ST_MAGN_1_FS_AVL_5600_GAIN_XY,
-					.gain2 = ST_MAGN_1_FS_AVL_5600_GAIN_Z,
+					.value = 0x06,
+					.gain = 3030,
+					.gain2 = 3389,
 				},
 				[6] = {
 					.num = ST_MAGN_FS_AVL_8100MG,
-					.value = ST_MAGN_1_FS_AVL_8100_VAL,
-					.gain = ST_MAGN_1_FS_AVL_8100_GAIN_XY,
-					.gain2 = ST_MAGN_1_FS_AVL_8100_GAIN_Z,
+					.value = 0x07,
+					.gain = 4347,
+					.gain2 = 4878,
 				},
 			},
 		},
-		.multi_read_bit = ST_MAGN_1_MULTIREAD_BIT,
+		.multi_read_bit = false,
 		.bootime = 2,
 	},
 	{
-		.wai = ST_MAGN_2_WAI_EXP,
+		.wai = 0x3d,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = LIS3MDL_MAGN_DEV_NAME,
 		},
 		.ch = (struct iio_chan_spec *)st_magn_2_16bit_channels,
 		.odr = {
-			.addr = ST_MAGN_2_ODR_ADDR,
-			.mask = ST_MAGN_2_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0x1c,
 			.odr_avl = {
-				{ 1, ST_MAGN_2_ODR_AVL_1HZ_VAL, },
-				{ 2, ST_MAGN_2_ODR_AVL_2HZ_VAL, },
-				{ 3, ST_MAGN_2_ODR_AVL_3HZ_VAL, },
-				{ 5, ST_MAGN_2_ODR_AVL_5HZ_VAL, },
-				{ 10, ST_MAGN_2_ODR_AVL_10HZ_VAL, },
-				{ 20, ST_MAGN_2_ODR_AVL_20HZ_VAL, },
-				{ 40, ST_MAGN_2_ODR_AVL_40HZ_VAL, },
-				{ 80, ST_MAGN_2_ODR_AVL_80HZ_VAL, },
+				{ .hz = 1, .value = 0x00 },
+				{ .hz = 2, .value = 0x01 },
+				{ .hz = 3, .value = 0x02 },
+				{ .hz = 5, .value = 0x03 },
+				{ .hz = 10, .value = 0x04 },
+				{ .hz = 20, .value = 0x05 },
+				{ .hz = 40, .value = 0x06 },
+				{ .hz = 80, .value = 0x07 },
 			},
 		},
 		.pw = {
-			.addr = ST_MAGN_2_PW_ADDR,
-			.mask = ST_MAGN_2_PW_MASK,
-			.value_on = ST_MAGN_2_PW_ON,
-			.value_off = ST_MAGN_2_PW_OFF,
+			.addr = 0x22,
+			.mask = 0x03,
+			.value_on = 0x00,
+			.value_off = 0x03,
 		},
 		.fs = {
-			.addr = ST_MAGN_2_FS_ADDR,
-			.mask = ST_MAGN_2_FS_MASK,
+			.addr = 0x21,
+			.mask = 0x60,
 			.fs_avl = {
 				[0] = {
 					.num = ST_MAGN_FS_AVL_4000MG,
-					.value = ST_MAGN_2_FS_AVL_4000_VAL,
-					.gain = ST_MAGN_2_FS_AVL_4000_GAIN,
+					.value = 0x00,
+					.gain = 146,
 				},
 				[1] = {
 					.num = ST_MAGN_FS_AVL_8000MG,
-					.value = ST_MAGN_2_FS_AVL_8000_VAL,
-					.gain = ST_MAGN_2_FS_AVL_8000_GAIN,
+					.value = 0x01,
+					.gain = 292,
 				},
 				[2] = {
 					.num = ST_MAGN_FS_AVL_12000MG,
-					.value = ST_MAGN_2_FS_AVL_12000_VAL,
-					.gain = ST_MAGN_2_FS_AVL_12000_GAIN,
+					.value = 0x02,
+					.gain = 438,
 				},
 				[3] = {
 					.num = ST_MAGN_FS_AVL_16000MG,
-					.value = ST_MAGN_2_FS_AVL_16000_VAL,
-					.gain = ST_MAGN_2_FS_AVL_16000_GAIN,
+					.value = 0x03,
+					.gain = 584,
 				},
 			},
 		},
-		.multi_read_bit = ST_MAGN_2_MULTIREAD_BIT,
+		.multi_read_bit = false,
 		.bootime = 2,
 	},
 	{
-		.wai = ST_MAGN_3_WAI_EXP,
-		.wai_addr = ST_MAGN_3_WAI_ADDR,
+		.wai = 0x40,
+		.wai_addr = 0x4f,
 		.sensors_supported = {
 			[0] = LSM303AGR_MAGN_DEV_NAME,
 		},
 		.ch = (struct iio_chan_spec *)st_magn_3_16bit_channels,
 		.odr = {
-			.addr = ST_MAGN_3_ODR_ADDR,
-			.mask = ST_MAGN_3_ODR_MASK,
+			.addr = 0x60,
+			.mask = 0x0c,
 			.odr_avl = {
-				{ 10, ST_MAGN_3_ODR_AVL_10HZ_VAL, },
-				{ 20, ST_MAGN_3_ODR_AVL_20HZ_VAL, },
-				{ 50, ST_MAGN_3_ODR_AVL_50HZ_VAL, },
-				{ 100, ST_MAGN_3_ODR_AVL_100HZ_VAL, },
+				{ .hz = 10, .value = 0x00 },
+				{ .hz = 20, .value = 0x01 },
+				{ .hz = 50, .value = 0x02 },
+				{ .hz = 100, .value = 0x03 },
 			},
 		},
 		.pw = {
-			.addr = ST_MAGN_3_PW_ADDR,
-			.mask = ST_MAGN_3_PW_MASK,
-			.value_on = ST_MAGN_3_PW_ON,
-			.value_off = ST_MAGN_3_PW_OFF,
+			.addr = 0x60,
+			.mask = 0x03,
+			.value_on = 0x00,
+			.value_off = 0x03,
 		},
 		.fs = {
 			.fs_avl = {
 				[0] = {
 					.num = ST_MAGN_FS_AVL_15000MG,
-					.gain = ST_MAGN_3_FS_AVL_15000_GAIN,
+					.gain = 1500,
 				},
 			},
 		},
 		.bdu = {
-			.addr = ST_MAGN_3_BDU_ADDR,
-			.mask = ST_MAGN_3_BDU_MASK,
+			.addr = 0x62,
+			.mask = 0x10,
 		},
 		.drdy_irq = {
-			.addr = ST_MAGN_3_DRDY_IRQ_ADDR,
-			.mask_int1 = ST_MAGN_3_DRDY_INT_MASK,
-			.addr_ihl = ST_MAGN_3_IHL_IRQ_ADDR,
-			.mask_ihl = ST_MAGN_3_IHL_IRQ_MASK,
+			.addr = 0x62,
+			.mask_int1 = 0x01,
+			.addr_ihl = 0x63,
+			.mask_ihl = 0x04,
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 		},
-		.multi_read_bit = ST_MAGN_3_MULTIREAD_BIT,
+		.multi_read_bit = false,
 		.bootime = 2,
 	},
 };
diff --git a/drivers/iio/potentiometer/mcp4531.c b/drivers/iio/potentiometer/mcp4531.c
index 13b6ae2fcf7b..0d1bcf89ae17 100644
--- a/drivers/iio/potentiometer/mcp4531.c
+++ b/drivers/iio/potentiometer/mcp4531.c
@@ -38,7 +38,7 @@
 
 struct mcp4531_cfg {
 	int wipers;
-	int max_pos;
+	int avail[3];
 	int kohms;
 };
 
@@ -78,38 +78,38 @@ enum mcp4531_type {
 };
 
 static const struct mcp4531_cfg mcp4531_cfg[] = {
-	[MCP453x_502] = { .wipers = 1, .max_pos = 128, .kohms =   5, },
-	[MCP453x_103] = { .wipers = 1, .max_pos = 128, .kohms =  10, },
-	[MCP453x_503] = { .wipers = 1, .max_pos = 128, .kohms =  50, },
-	[MCP453x_104] = { .wipers = 1, .max_pos = 128, .kohms = 100, },
-	[MCP454x_502] = { .wipers = 1, .max_pos = 128, .kohms =   5, },
-	[MCP454x_103] = { .wipers = 1, .max_pos = 128, .kohms =  10, },
-	[MCP454x_503] = { .wipers = 1, .max_pos = 128, .kohms =  50, },
-	[MCP454x_104] = { .wipers = 1, .max_pos = 128, .kohms = 100, },
-	[MCP455x_502] = { .wipers = 1, .max_pos = 256, .kohms =   5, },
-	[MCP455x_103] = { .wipers = 1, .max_pos = 256, .kohms =  10, },
-	[MCP455x_503] = { .wipers = 1, .max_pos = 256, .kohms =  50, },
-	[MCP455x_104] = { .wipers = 1, .max_pos = 256, .kohms = 100, },
-	[MCP456x_502] = { .wipers = 1, .max_pos = 256, .kohms =   5, },
-	[MCP456x_103] = { .wipers = 1, .max_pos = 256, .kohms =  10, },
-	[MCP456x_503] = { .wipers = 1, .max_pos = 256, .kohms =  50, },
-	[MCP456x_104] = { .wipers = 1, .max_pos = 256, .kohms = 100, },
-	[MCP463x_502] = { .wipers = 2, .max_pos = 128, .kohms =   5, },
-	[MCP463x_103] = { .wipers = 2, .max_pos = 128, .kohms =  10, },
-	[MCP463x_503] = { .wipers = 2, .max_pos = 128, .kohms =  50, },
-	[MCP463x_104] = { .wipers = 2, .max_pos = 128, .kohms = 100, },
-	[MCP464x_502] = { .wipers = 2, .max_pos = 128, .kohms =   5, },
-	[MCP464x_103] = { .wipers = 2, .max_pos = 128, .kohms =  10, },
-	[MCP464x_503] = { .wipers = 2, .max_pos = 128, .kohms =  50, },
-	[MCP464x_104] = { .wipers = 2, .max_pos = 128, .kohms = 100, },
-	[MCP465x_502] = { .wipers = 2, .max_pos = 256, .kohms =   5, },
-	[MCP465x_103] = { .wipers = 2, .max_pos = 256, .kohms =  10, },
-	[MCP465x_503] = { .wipers = 2, .max_pos = 256, .kohms =  50, },
-	[MCP465x_104] = { .wipers = 2, .max_pos = 256, .kohms = 100, },
-	[MCP466x_502] = { .wipers = 2, .max_pos = 256, .kohms =   5, },
-	[MCP466x_103] = { .wipers = 2, .max_pos = 256, .kohms =  10, },
-	[MCP466x_503] = { .wipers = 2, .max_pos = 256, .kohms =  50, },
-	[MCP466x_104] = { .wipers = 2, .max_pos = 256, .kohms = 100, },
+	[MCP453x_502] = { .wipers = 1, .avail = { 0, 1, 128 }, .kohms =   5, },
+	[MCP453x_103] = { .wipers = 1, .avail = { 0, 1, 128 }, .kohms =  10, },
+	[MCP453x_503] = { .wipers = 1, .avail = { 0, 1, 128 }, .kohms =  50, },
+	[MCP453x_104] = { .wipers = 1, .avail = { 0, 1, 128 }, .kohms = 100, },
+	[MCP454x_502] = { .wipers = 1, .avail = { 0, 1, 128 }, .kohms =   5, },
+	[MCP454x_103] = { .wipers = 1, .avail = { 0, 1, 128 }, .kohms =  10, },
+	[MCP454x_503] = { .wipers = 1, .avail = { 0, 1, 128 }, .kohms =  50, },
+	[MCP454x_104] = { .wipers = 1, .avail = { 0, 1, 128 }, .kohms = 100, },
+	[MCP455x_502] = { .wipers = 1, .avail = { 0, 1, 256 }, .kohms =   5, },
+	[MCP455x_103] = { .wipers = 1, .avail = { 0, 1, 256 }, .kohms =  10, },
+	[MCP455x_503] = { .wipers = 1, .avail = { 0, 1, 256 }, .kohms =  50, },
+	[MCP455x_104] = { .wipers = 1, .avail = { 0, 1, 256 }, .kohms = 100, },
+	[MCP456x_502] = { .wipers = 1, .avail = { 0, 1, 256 }, .kohms =   5, },
+	[MCP456x_103] = { .wipers = 1, .avail = { 0, 1, 256 }, .kohms =  10, },
+	[MCP456x_503] = { .wipers = 1, .avail = { 0, 1, 256 }, .kohms =  50, },
+	[MCP456x_104] = { .wipers = 1, .avail = { 0, 1, 256 }, .kohms = 100, },
+	[MCP463x_502] = { .wipers = 2, .avail = { 0, 1, 128 }, .kohms =   5, },
+	[MCP463x_103] = { .wipers = 2, .avail = { 0, 1, 128 }, .kohms =  10, },
+	[MCP463x_503] = { .wipers = 2, .avail = { 0, 1, 128 }, .kohms =  50, },
+	[MCP463x_104] = { .wipers = 2, .avail = { 0, 1, 128 }, .kohms = 100, },
+	[MCP464x_502] = { .wipers = 2, .avail = { 0, 1, 128 }, .kohms =   5, },
+	[MCP464x_103] = { .wipers = 2, .avail = { 0, 1, 128 }, .kohms =  10, },
+	[MCP464x_503] = { .wipers = 2, .avail = { 0, 1, 128 }, .kohms =  50, },
+	[MCP464x_104] = { .wipers = 2, .avail = { 0, 1, 128 }, .kohms = 100, },
+	[MCP465x_502] = { .wipers = 2, .avail = { 0, 1, 256 }, .kohms =   5, },
+	[MCP465x_103] = { .wipers = 2, .avail = { 0, 1, 256 }, .kohms =  10, },
+	[MCP465x_503] = { .wipers = 2, .avail = { 0, 1, 256 }, .kohms =  50, },
+	[MCP465x_104] = { .wipers = 2, .avail = { 0, 1, 256 }, .kohms = 100, },
+	[MCP466x_502] = { .wipers = 2, .avail = { 0, 1, 256 }, .kohms =   5, },
+	[MCP466x_103] = { .wipers = 2, .avail = { 0, 1, 256 }, .kohms =  10, },
+	[MCP466x_503] = { .wipers = 2, .avail = { 0, 1, 256 }, .kohms =  50, },
+	[MCP466x_104] = { .wipers = 2, .avail = { 0, 1, 256 }, .kohms = 100, },
 };
 
 #define MCP4531_WRITE (0 << 2)
@@ -124,13 +124,14 @@ struct mcp4531_data {
 	const struct mcp4531_cfg *cfg;
 };
 
-#define MCP4531_CHANNEL(ch) {					\
-	.type = IIO_RESISTANCE,					\
-	.indexed = 1,						\
-	.output = 1,						\
-	.channel = (ch),					\
-	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
-	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+#define MCP4531_CHANNEL(ch) {						\
+	.type = IIO_RESISTANCE,						\
+	.indexed = 1,							\
+	.output = 1,							\
+	.channel = (ch),						\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),			\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),		\
+	.info_mask_shared_by_type_available = BIT(IIO_CHAN_INFO_RAW),	\
 }
 
 static const struct iio_chan_spec mcp4531_channels[] = {
@@ -156,13 +157,31 @@ static int mcp4531_read_raw(struct iio_dev *indio_dev,
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_SCALE:
 		*val = 1000 * data->cfg->kohms;
-		*val2 = data->cfg->max_pos;
+		*val2 = data->cfg->avail[2];
 		return IIO_VAL_FRACTIONAL;
 	}
 
 	return -EINVAL;
 }
 
+static int mcp4531_read_avail(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      const int **vals, int *type, int *length,
+			      long mask)
+{
+	struct mcp4531_data *data = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		*length = ARRAY_SIZE(data->cfg->avail);
+		*vals = data->cfg->avail;
+		*type = IIO_VAL_INT;
+		return IIO_AVAIL_RANGE;
+	}
+
+	return -EINVAL;
+}
+
 static int mcp4531_write_raw(struct iio_dev *indio_dev,
 			     struct iio_chan_spec const *chan,
 			     int val, int val2, long mask)
@@ -172,7 +191,7 @@ static int mcp4531_write_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-		if (val > data->cfg->max_pos || val < 0)
+		if (val > data->cfg->avail[2] || val < 0)
 			return -EINVAL;
 		break;
 	default:
@@ -186,6 +205,7 @@ static int mcp4531_write_raw(struct iio_dev *indio_dev,
 
 static const struct iio_info mcp4531_info = {
 	.read_raw = mcp4531_read_raw,
+	.read_avail = mcp4531_read_avail,
 	.write_raw = mcp4531_write_raw,
 	.driver_module = THIS_MODULE,
 };
diff --git a/drivers/iio/potentiostat/Kconfig b/drivers/iio/potentiostat/Kconfig
new file mode 100644
index 000000000000..1e3baf2cc97d
--- /dev/null
+++ b/drivers/iio/potentiostat/Kconfig
@@ -0,0 +1,22 @@
+#
+# Potentiostat drivers
+#
+# When adding new entries keep the list in alphabetical order
+
+menu "Digital potentiostats"
+
+config LMP91000
+	tristate "Texas Instruments LMP91000 potentiostat driver"
+	depends on I2C
+	select REGMAP_I2C
+	select IIO_BUFFER
+	select IIO_BUFFER_CB
+	select IIO_TRIGGERED_BUFFER
+	help
+	  Say yes here to build support for the Texas Instruments
+	  LMP91000 digital potentiostat chip.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called lmp91000
+
+endmenu
diff --git a/drivers/iio/potentiostat/Makefile b/drivers/iio/potentiostat/Makefile
new file mode 100644
index 000000000000..64d315ef4449
--- /dev/null
+++ b/drivers/iio/potentiostat/Makefile
@@ -0,0 +1,6 @@
+#
+# Makefile for industrial I/O potentiostat drivers
+#
+
+# When adding new entries keep the list in alphabetical order
+obj-$(CONFIG_LMP91000) += lmp91000.o
diff --git a/drivers/iio/potentiostat/lmp91000.c b/drivers/iio/potentiostat/lmp91000.c
new file mode 100644
index 000000000000..e22714365022
--- /dev/null
+++ b/drivers/iio/potentiostat/lmp91000.c
@@ -0,0 +1,446 @@
+/*
+ * lmp91000.c - Support for Texas Instruments digital potentiostats
+ *
+ * Copyright (C) 2016 Matt Ranostay <mranostay@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * TODO: bias voltage + polarity control, and multiple chip support
+ */
+
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/delay.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/consumer.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+
+#define LMP91000_REG_LOCK		0x01
+#define LMP91000_REG_TIACN		0x10
+#define LMP91000_REG_TIACN_GAIN_SHIFT	2
+
+#define LMP91000_REG_REFCN		0x11
+#define LMP91000_REG_REFCN_EXT_REF	0x20
+#define LMP91000_REG_REFCN_50_ZERO	0x80
+
+#define LMP91000_REG_MODECN		0x12
+#define LMP91000_REG_MODECN_3LEAD	0x03
+#define LMP91000_REG_MODECN_TEMP	0x07
+
+#define LMP91000_DRV_NAME	"lmp91000"
+
+static const int lmp91000_tia_gain[] = { 0, 2750, 3500, 7000, 14000, 35000,
+					 120000, 350000 };
+
+static const int lmp91000_rload[] = { 10, 33, 50, 100 };
+
+#define LMP91000_TEMP_BASE	-40
+
+static const u16 lmp91000_temp_lut[] = {
+	1875, 1867, 1860, 1852, 1844, 1836, 1828, 1821, 1813, 1805,
+	1797, 1789, 1782, 1774, 1766, 1758, 1750, 1742, 1734, 1727,
+	1719, 1711, 1703, 1695, 1687, 1679, 1671, 1663, 1656, 1648,
+	1640, 1632, 1624, 1616, 1608, 1600, 1592, 1584, 1576, 1568,
+	1560, 1552, 1544, 1536, 1528, 1520, 1512, 1504, 1496, 1488,
+	1480, 1472, 1464, 1456, 1448, 1440, 1432, 1424, 1415, 1407,
+	1399, 1391, 1383, 1375, 1367, 1359, 1351, 1342, 1334, 1326,
+	1318, 1310, 1302, 1293, 1285, 1277, 1269, 1261, 1253, 1244,
+	1236, 1228, 1220, 1212, 1203, 1195, 1187, 1179, 1170, 1162,
+	1154, 1146, 1137, 1129, 1121, 1112, 1104, 1096, 1087, 1079,
+	1071, 1063, 1054, 1046, 1038, 1029, 1021, 1012, 1004,  996,
+	 987,  979,  971,  962,  954,  945,  937,  929,  920,  912,
+	 903,  895,  886,  878,  870,  861 };
+
+static const struct regmap_config lmp91000_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+};
+
+struct lmp91000_data {
+	struct regmap *regmap;
+	struct device *dev;
+
+	struct iio_trigger *trig;
+	struct iio_cb_buffer *cb_buffer;
+	struct iio_channel *adc_chan;
+
+	struct completion completion;
+	u8 chan_select;
+
+	u32 buffer[4]; /* 64-bit data + 64-bit timestamp */
+};
+
+static const struct iio_chan_spec lmp91000_channels[] = {
+	{ /* chemical channel mV */
+		.type = IIO_VOLTAGE,
+		.channel = 0,
+		.address = LMP91000_REG_MODECN_3LEAD,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_OFFSET) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 32,
+			.storagebits = 32,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(1),
+	{ /* temperature channel mV */
+		.type = IIO_TEMP,
+		.channel = 1,
+		.address = LMP91000_REG_MODECN_TEMP,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED),
+		.scan_index = -1,
+	},
+};
+
+static int lmp91000_read(struct lmp91000_data *data, int channel, int *val)
+{
+	int state, ret;
+
+	ret = regmap_read(data->regmap, LMP91000_REG_MODECN, &state);
+	if (ret)
+		return -EINVAL;
+
+	ret = regmap_write(data->regmap, LMP91000_REG_MODECN, channel);
+	if (ret)
+		return -EINVAL;
+
+	/* delay till first temperature reading is complete */
+	if ((state != channel) && (channel == LMP91000_REG_MODECN_TEMP))
+		usleep_range(3000, 4000);
+
+	data->chan_select = channel != LMP91000_REG_MODECN_3LEAD;
+
+	iio_trigger_poll_chained(data->trig);
+
+	ret = wait_for_completion_timeout(&data->completion, HZ);
+	reinit_completion(&data->completion);
+
+	if (!ret)
+		return -ETIMEDOUT;
+
+	*val = data->buffer[data->chan_select];
+
+	return 0;
+}
+
+static irqreturn_t lmp91000_buffer_handler(int irq, void *private)
+{
+	struct iio_poll_func *pf = private;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct lmp91000_data *data = iio_priv(indio_dev);
+	int ret, val;
+
+	memset(data->buffer, 0, sizeof(data->buffer));
+
+	ret = lmp91000_read(data, LMP91000_REG_MODECN_3LEAD, &val);
+	if (!ret) {
+		data->buffer[0] = val;
+		iio_push_to_buffers_with_timestamp(indio_dev, data->buffer,
+						   iio_get_time_ns(indio_dev));
+	}
+
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int lmp91000_read_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int *val, int *val2, long mask)
+{
+	struct lmp91000_data *data = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+	case IIO_CHAN_INFO_PROCESSED: {
+		int ret = iio_channel_start_all_cb(data->cb_buffer);
+
+		if (ret)
+			return ret;
+
+		ret = lmp91000_read(data, chan->address, val);
+
+		iio_channel_stop_all_cb(data->cb_buffer);
+
+		if (ret)
+			return ret;
+
+		if (mask == IIO_CHAN_INFO_PROCESSED) {
+			int tmp, i;
+
+			ret = iio_convert_raw_to_processed(data->adc_chan,
+							   *val, &tmp, 1);
+			if (ret)
+				return ret;
+
+			for (i = 0; i < ARRAY_SIZE(lmp91000_temp_lut); i++)
+				if (lmp91000_temp_lut[i] < tmp)
+					break;
+
+			*val = (LMP91000_TEMP_BASE + i) * 1000;
+		}
+		return IIO_VAL_INT;
+	}
+	case IIO_CHAN_INFO_OFFSET:
+		return iio_read_channel_offset(data->adc_chan, val, val2);
+	case IIO_CHAN_INFO_SCALE:
+		return iio_read_channel_scale(data->adc_chan, val, val2);
+	}
+
+	return -EINVAL;
+}
+
+static const struct iio_info lmp91000_info = {
+	.driver_module = THIS_MODULE,
+	.read_raw = lmp91000_read_raw,
+};
+
+static int lmp91000_read_config(struct lmp91000_data *data)
+{
+	struct device *dev = data->dev;
+	struct device_node *np = dev->of_node;
+	unsigned int reg, val;
+	int i, ret;
+
+	ret = of_property_read_u32(np, "ti,tia-gain-ohm", &val);
+	if (ret) {
+		if (of_property_read_bool(np, "ti,external-tia-resistor"))
+			val = 0;
+		else {
+			dev_err(dev, "no ti,tia-gain-ohm defined");
+			return ret;
+		}
+	}
+
+	ret = -EINVAL;
+	for (i = 0; i < ARRAY_SIZE(lmp91000_tia_gain); i++) {
+		if (lmp91000_tia_gain[i] == val) {
+			reg = i << LMP91000_REG_TIACN_GAIN_SHIFT;
+			ret = 0;
+			break;
+		}
+	}
+
+	if (ret) {
+		dev_err(dev, "invalid ti,tia-gain-ohm %d\n", val);
+		return ret;
+	}
+
+	ret = of_property_read_u32(np, "ti,rload-ohm", &val);
+	if (ret) {
+		val = 100;
+		dev_info(dev, "no ti,rload-ohm defined, default to %d\n", val);
+	}
+
+	ret = -EINVAL;
+	for (i = 0; i < ARRAY_SIZE(lmp91000_rload); i++) {
+		if (lmp91000_rload[i] == val) {
+			reg |= i;
+			ret = 0;
+			break;
+		}
+	}
+
+	if (ret) {
+		dev_err(dev, "invalid ti,rload-ohm %d\n", val);
+		return ret;
+	}
+
+	regmap_write(data->regmap, LMP91000_REG_LOCK, 0);
+	regmap_write(data->regmap, LMP91000_REG_TIACN, reg);
+	regmap_write(data->regmap, LMP91000_REG_REFCN, LMP91000_REG_REFCN_EXT_REF
+					| LMP91000_REG_REFCN_50_ZERO);
+	regmap_write(data->regmap, LMP91000_REG_LOCK, 1);
+
+	return 0;
+}
+
+static int lmp91000_buffer_cb(const void *val, void *private)
+{
+	struct iio_dev *indio_dev = private;
+	struct lmp91000_data *data = iio_priv(indio_dev);
+
+	data->buffer[data->chan_select] = *((int *)val);
+	complete_all(&data->completion);
+
+	return 0;
+}
+
+static const struct iio_trigger_ops lmp91000_trigger_ops = {
+	.owner = THIS_MODULE,
+};
+
+
+static int lmp91000_buffer_preenable(struct iio_dev *indio_dev)
+{
+	struct lmp91000_data *data = iio_priv(indio_dev);
+
+	return iio_channel_start_all_cb(data->cb_buffer);
+}
+
+static int lmp91000_buffer_predisable(struct iio_dev *indio_dev)
+{
+	struct lmp91000_data *data = iio_priv(indio_dev);
+
+	iio_channel_stop_all_cb(data->cb_buffer);
+
+	return 0;
+}
+
+static const struct iio_buffer_setup_ops lmp91000_buffer_setup_ops = {
+	.preenable = lmp91000_buffer_preenable,
+	.postenable = iio_triggered_buffer_postenable,
+	.predisable = lmp91000_buffer_predisable,
+};
+
+static int lmp91000_probe(struct i2c_client *client,
+			  const struct i2c_device_id *id)
+{
+	struct device *dev = &client->dev;
+	struct lmp91000_data *data;
+	struct iio_dev *indio_dev;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	indio_dev->info = &lmp91000_info;
+	indio_dev->channels = lmp91000_channels;
+	indio_dev->num_channels = ARRAY_SIZE(lmp91000_channels);
+	indio_dev->name = LMP91000_DRV_NAME;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	i2c_set_clientdata(client, indio_dev);
+
+	data = iio_priv(indio_dev);
+	data->dev = dev;
+	data->regmap = devm_regmap_init_i2c(client, &lmp91000_regmap_config);
+	if (IS_ERR(data->regmap)) {
+		dev_err(dev, "regmap initialization failed.\n");
+		return PTR_ERR(data->regmap);
+	}
+
+	data->trig = devm_iio_trigger_alloc(data->dev, "%s-mux%d",
+					    indio_dev->name, indio_dev->id);
+	if (!data->trig) {
+		dev_err(dev, "cannot allocate iio trigger.\n");
+		return -ENOMEM;
+	}
+
+	data->trig->ops = &lmp91000_trigger_ops;
+	data->trig->dev.parent = dev;
+	init_completion(&data->completion);
+
+	ret = lmp91000_read_config(data);
+	if (ret)
+		return ret;
+
+	ret = iio_trigger_set_immutable(iio_channel_cb_get_iio_dev(data->cb_buffer),
+					data->trig);
+	if (ret) {
+		dev_err(dev, "cannot set immutable trigger.\n");
+		return ret;
+	}
+
+	ret = iio_trigger_register(data->trig);
+	if (ret) {
+		dev_err(dev, "cannot register iio trigger.\n");
+		return ret;
+	}
+
+	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+					 &lmp91000_buffer_handler,
+					 &lmp91000_buffer_setup_ops);
+	if (ret)
+		goto error_unreg_trigger;
+
+	data->cb_buffer = iio_channel_get_all_cb(dev, &lmp91000_buffer_cb,
+						 indio_dev);
+
+	if (IS_ERR(data->cb_buffer)) {
+		if (PTR_ERR(data->cb_buffer) == -ENODEV)
+			ret = -EPROBE_DEFER;
+		else
+			ret = PTR_ERR(data->cb_buffer);
+
+		goto error_unreg_buffer;
+	}
+
+	data->adc_chan = iio_channel_cb_get_channels(data->cb_buffer);
+
+	ret = iio_device_register(indio_dev);
+	if (ret)
+		goto error_unreg_cb_buffer;
+
+	return 0;
+
+error_unreg_cb_buffer:
+	iio_channel_release_all_cb(data->cb_buffer);
+
+error_unreg_buffer:
+	iio_triggered_buffer_cleanup(indio_dev);
+
+error_unreg_trigger:
+	iio_trigger_unregister(data->trig);
+
+	return ret;
+}
+
+static int lmp91000_remove(struct i2c_client *client)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(client);
+	struct lmp91000_data *data = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+
+	iio_channel_stop_all_cb(data->cb_buffer);
+	iio_channel_release_all_cb(data->cb_buffer);
+
+	iio_triggered_buffer_cleanup(indio_dev);
+	iio_trigger_unregister(data->trig);
+
+	return 0;
+}
+
+static const struct of_device_id lmp91000_of_match[] = {
+	{ .compatible = "ti,lmp91000", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, lmp91000_of_match);
+
+static const struct i2c_device_id lmp91000_id[] = {
+	{ "lmp91000", 0 },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, lmp91000_id);
+
+static struct i2c_driver lmp91000_driver = {
+	.driver = {
+		.name = LMP91000_DRV_NAME,
+		.of_match_table = of_match_ptr(lmp91000_of_match),
+	},
+	.probe = lmp91000_probe,
+	.remove = lmp91000_remove,
+	.id_table = lmp91000_id,
+};
+module_i2c_driver(lmp91000_driver);
+
+MODULE_AUTHOR("Matt Ranostay <mranostay@gmail.com>");
+MODULE_DESCRIPTION("LMP91000 digital potentiostat");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/pressure/Kconfig b/drivers/iio/pressure/Kconfig
index 15cd416365c1..bd8d96b96771 100644
--- a/drivers/iio/pressure/Kconfig
+++ b/drivers/iio/pressure/Kconfig
@@ -5,6 +5,16 @@
 
 menu "Pressure sensors"
 
+config ABP060MG
+	tristate "Honeywell ABP pressure sensor driver"
+	depends on I2C
+	help
+	  Say yes here to build support for the Honeywell ABP pressure
+	  sensors.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called abp060mg.
+
 config BMP280
 	tristate "Bosch Sensortec BMP180/BMP280 pressure sensor I2C driver"
 	depends on (I2C || SPI_MASTER)
diff --git a/drivers/iio/pressure/Makefile b/drivers/iio/pressure/Makefile
index fff77185a5cc..de3dbc81dc5a 100644
--- a/drivers/iio/pressure/Makefile
+++ b/drivers/iio/pressure/Makefile
@@ -3,6 +3,7 @@
 #
 
 # When adding new entries keep the list in alphabetical order
+obj-$(CONFIG_ABP060MG) += abp060mg.o
 obj-$(CONFIG_BMP280) += bmp280.o
 bmp280-objs := bmp280-core.o bmp280-regmap.o
 obj-$(CONFIG_BMP280_I2C) += bmp280-i2c.o
diff --git a/drivers/iio/pressure/abp060mg.c b/drivers/iio/pressure/abp060mg.c
new file mode 100644
index 000000000000..43bdd0b9155f
--- /dev/null
+++ b/drivers/iio/pressure/abp060mg.c
@@ -0,0 +1,276 @@
+/*
+ * Copyright (C) 2016 - Marcin Malagowski <mrc@bourne.st>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/iio/iio.h>
+
+#define ABP060MG_ERROR_MASK   0xC000
+#define ABP060MG_RESP_TIME_MS 40
+#define ABP060MG_MIN_COUNTS   1638  /* = 0x0666 (10% of u14) */
+#define ABP060MG_MAX_COUNTS   14745 /* = 0x3999 (90% of u14) */
+#define ABP060MG_NUM_COUNTS   (ABP060MG_MAX_COUNTS - ABP060MG_MIN_COUNTS)
+
+enum abp_variant {
+	/* gage [kPa] */
+	ABP006KG, ABP010KG, ABP016KG, ABP025KG, ABP040KG, ABP060KG, ABP100KG,
+	ABP160KG, ABP250KG, ABP400KG, ABP600KG, ABP001GG,
+	/* differential [kPa] */
+	ABP006KD, ABP010KD, ABP016KD, ABP025KD, ABP040KD, ABP060KD, ABP100KD,
+	ABP160KD, ABP250KD, ABP400KD,
+	/* gage [psi] */
+	ABP001PG, ABP005PG, ABP015PG, ABP030PG, ABP060PG, ABP100PG, ABP150PG,
+	/* differential [psi] */
+	ABP001PD, ABP005PD, ABP015PD, ABP030PD, ABP060PD,
+};
+
+struct abp_config {
+	int min;
+	int max;
+};
+
+static struct abp_config abp_config[] = {
+	/* mbar & kPa variants */
+	[ABP006KG] = { .min =       0, .max =     6000 },
+	[ABP010KG] = { .min =       0, .max =    10000 },
+	[ABP016KG] = { .min =       0, .max =    16000 },
+	[ABP025KG] = { .min =       0, .max =    25000 },
+	[ABP040KG] = { .min =       0, .max =    40000 },
+	[ABP060KG] = { .min =       0, .max =    60000 },
+	[ABP100KG] = { .min =       0, .max =   100000 },
+	[ABP160KG] = { .min =       0, .max =   160000 },
+	[ABP250KG] = { .min =       0, .max =   250000 },
+	[ABP400KG] = { .min =       0, .max =   400000 },
+	[ABP600KG] = { .min =       0, .max =   600000 },
+	[ABP001GG] = { .min =       0, .max =  1000000 },
+	[ABP006KD] = { .min =   -6000, .max =     6000 },
+	[ABP010KD] = { .min =  -10000, .max =    10000 },
+	[ABP016KD] = { .min =  -16000, .max =    16000 },
+	[ABP025KD] = { .min =  -25000, .max =    25000 },
+	[ABP040KD] = { .min =  -40000, .max =    40000 },
+	[ABP060KD] = { .min =  -60000, .max =    60000 },
+	[ABP100KD] = { .min = -100000, .max =   100000 },
+	[ABP160KD] = { .min = -160000, .max =   160000 },
+	[ABP250KD] = { .min = -250000, .max =   250000 },
+	[ABP400KD] = { .min = -400000, .max =   400000 },
+	/* psi variants (1 psi ~ 6895 Pa) */
+	[ABP001PG] = { .min =       0, .max =     6985 },
+	[ABP005PG] = { .min =       0, .max =    34474 },
+	[ABP015PG] = { .min =       0, .max =   103421 },
+	[ABP030PG] = { .min =       0, .max =   206843 },
+	[ABP060PG] = { .min =       0, .max =   413686 },
+	[ABP100PG] = { .min =       0, .max =   689476 },
+	[ABP150PG] = { .min =       0, .max =  1034214 },
+	[ABP001PD] = { .min =   -6895, .max =     6895 },
+	[ABP005PD] = { .min =  -34474, .max =    34474 },
+	[ABP015PD] = { .min = -103421, .max =   103421 },
+	[ABP030PD] = { .min = -206843, .max =   206843 },
+	[ABP060PD] = { .min = -413686, .max =   413686 },
+};
+
+struct abp_state {
+	struct i2c_client *client;
+	struct mutex lock;
+
+	/*
+	 * bus-dependent MEASURE_REQUEST length.
+	 * If no SMBUS_QUICK support, need to send dummy byte
+	 */
+	int mreq_len;
+
+	/* model-dependent values (calculated on probe) */
+	int scale;
+	int offset;
+};
+
+static const struct iio_chan_spec abp060mg_channels[] = {
+	{
+		.type = IIO_PRESSURE,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+			BIT(IIO_CHAN_INFO_OFFSET) | BIT(IIO_CHAN_INFO_SCALE),
+	},
+};
+
+static int abp060mg_get_measurement(struct abp_state *state, int *val)
+{
+	struct i2c_client *client = state->client;
+	__be16 buf[2];
+	u16 pressure;
+	int ret;
+
+	buf[0] = 0;
+	ret = i2c_master_send(client, (u8 *)&buf, state->mreq_len);
+	if (ret < 0)
+		return ret;
+
+	msleep_interruptible(ABP060MG_RESP_TIME_MS);
+
+	ret = i2c_master_recv(client, (u8 *)&buf, sizeof(buf));
+	if (ret < 0)
+		return ret;
+
+	pressure = be16_to_cpu(buf[0]);
+	if (pressure & ABP060MG_ERROR_MASK)
+		return -EIO;
+
+	if (pressure < ABP060MG_MIN_COUNTS || pressure > ABP060MG_MAX_COUNTS)
+		return -EIO;
+
+	*val = pressure;
+
+	return IIO_VAL_INT;
+}
+
+static int abp060mg_read_raw(struct iio_dev *indio_dev,
+			struct iio_chan_spec const *chan, int *val,
+			int *val2, long mask)
+{
+	struct abp_state *state = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&state->lock);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = abp060mg_get_measurement(state, val);
+		break;
+	case IIO_CHAN_INFO_OFFSET:
+		*val = state->offset;
+		ret = IIO_VAL_INT;
+		break;
+	case IIO_CHAN_INFO_SCALE:
+		*val = state->scale;
+		*val2 = ABP060MG_NUM_COUNTS * 1000; /* to kPa */
+		ret = IIO_VAL_FRACTIONAL;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	mutex_unlock(&state->lock);
+	return ret;
+}
+
+static const struct iio_info abp060mg_info = {
+	.driver_module = THIS_MODULE,
+	.read_raw = abp060mg_read_raw,
+};
+
+static void abp060mg_init_device(struct iio_dev *indio_dev, unsigned long id)
+{
+	struct abp_state *state = iio_priv(indio_dev);
+	struct abp_config *cfg = &abp_config[id];
+
+	state->scale = cfg->max - cfg->min;
+	state->offset = -ABP060MG_MIN_COUNTS;
+
+	if (cfg->min < 0) /* differential */
+		state->offset -= ABP060MG_NUM_COUNTS >> 1;
+}
+
+static int abp060mg_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	struct iio_dev *indio_dev;
+	struct abp_state *state;
+	unsigned long cfg_id = id->driver_data;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*state));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	state = iio_priv(indio_dev);
+	i2c_set_clientdata(client, state);
+	state->client = client;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_QUICK))
+		state->mreq_len = 1;
+
+	abp060mg_init_device(indio_dev, cfg_id);
+
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->name = dev_name(&client->dev);
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->info = &abp060mg_info;
+
+	indio_dev->channels = abp060mg_channels;
+	indio_dev->num_channels = ARRAY_SIZE(abp060mg_channels);
+
+	mutex_init(&state->lock);
+
+	return devm_iio_device_register(&client->dev, indio_dev);
+}
+
+static const struct i2c_device_id abp060mg_id_table[] = {
+	/* mbar & kPa variants (abp060m [60 mbar] == abp006k [6 kPa]) */
+	/*    gage: */
+	{ "abp060mg", ABP006KG }, { "abp006kg", ABP006KG },
+	{ "abp100mg", ABP010KG }, { "abp010kg", ABP010KG },
+	{ "abp160mg", ABP016KG }, { "abp016kg", ABP016KG },
+	{ "abp250mg", ABP025KG }, { "abp025kg", ABP025KG },
+	{ "abp400mg", ABP040KG }, { "abp040kg", ABP040KG },
+	{ "abp600mg", ABP060KG }, { "abp060kg", ABP060KG },
+	{ "abp001bg", ABP100KG }, { "abp100kg", ABP100KG },
+	{ "abp1_6bg", ABP160KG }, { "abp160kg", ABP160KG },
+	{ "abp2_5bg", ABP250KG }, { "abp250kg", ABP250KG },
+	{ "abp004bg", ABP400KG }, { "abp400kg", ABP400KG },
+	{ "abp006bg", ABP600KG }, { "abp600kg", ABP600KG },
+	{ "abp010bg", ABP001GG }, { "abp001gg", ABP001GG },
+	/*    differential: */
+	{ "abp060md", ABP006KD }, { "abp006kd", ABP006KD },
+	{ "abp100md", ABP010KD }, { "abp010kd", ABP010KD },
+	{ "abp160md", ABP016KD }, { "abp016kd", ABP016KD },
+	{ "abp250md", ABP025KD }, { "abp025kd", ABP025KD },
+	{ "abp400md", ABP040KD }, { "abp040kd", ABP040KD },
+	{ "abp600md", ABP060KD }, { "abp060kd", ABP060KD },
+	{ "abp001bd", ABP100KD }, { "abp100kd", ABP100KD },
+	{ "abp1_6bd", ABP160KD }, { "abp160kd", ABP160KD },
+	{ "abp2_5bd", ABP250KD }, { "abp250kd", ABP250KD },
+	{ "abp004bd", ABP400KD }, { "abp400kd", ABP400KD },
+	/* psi variants */
+	/*    gage: */
+	{ "abp001pg", ABP001PG },
+	{ "abp005pg", ABP005PG },
+	{ "abp015pg", ABP015PG },
+	{ "abp030pg", ABP030PG },
+	{ "abp060pg", ABP060PG },
+	{ "abp100pg", ABP100PG },
+	{ "abp150pg", ABP150PG },
+	/*    differential: */
+	{ "abp001pd", ABP001PD },
+	{ "abp005pd", ABP005PD },
+	{ "abp015pd", ABP015PD },
+	{ "abp030pd", ABP030PD },
+	{ "abp060pd", ABP060PD },
+	{ /* empty */ },
+};
+MODULE_DEVICE_TABLE(i2c, abp060mg_id_table);
+
+static struct i2c_driver abp060mg_driver = {
+	.driver = {
+		.name = "abp060mg",
+	},
+	.probe = abp060mg_probe,
+	.id_table = abp060mg_id_table,
+};
+module_i2c_driver(abp060mg_driver);
+
+MODULE_AUTHOR("Marcin Malagowski <mrc@bourne.st>");
+MODULE_DESCRIPTION("Honeywell ABP pressure sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/pressure/mpl3115.c b/drivers/iio/pressure/mpl3115.c
index 6392d7b62841..cc3f84139157 100644
--- a/drivers/iio/pressure/mpl3115.c
+++ b/drivers/iio/pressure/mpl3115.c
@@ -82,8 +82,9 @@ static int mpl3115_read_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-		if (iio_buffer_enabled(indio_dev))
-			return -EBUSY;
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
 
 		switch (chan->type) {
 		case IIO_PRESSURE: /* in 0.25 pascal / LSB */
@@ -91,32 +92,39 @@ static int mpl3115_read_raw(struct iio_dev *indio_dev,
 			ret = mpl3115_request(data);
 			if (ret < 0) {
 				mutex_unlock(&data->lock);
-				return ret;
+				break;
 			}
 			ret = i2c_smbus_read_i2c_block_data(data->client,
 				MPL3115_OUT_PRESS, 3, (u8 *) &tmp);
 			mutex_unlock(&data->lock);
 			if (ret < 0)
-				return ret;
+				break;
 			*val = be32_to_cpu(tmp) >> 12;
-			return IIO_VAL_INT;
+			ret = IIO_VAL_INT;
+			break;
 		case IIO_TEMP: /* in 0.0625 celsius / LSB */
 			mutex_lock(&data->lock);
 			ret = mpl3115_request(data);
 			if (ret < 0) {
 				mutex_unlock(&data->lock);
-				return ret;
+				break;
 			}
 			ret = i2c_smbus_read_i2c_block_data(data->client,
 				MPL3115_OUT_TEMP, 2, (u8 *) &tmp);
 			mutex_unlock(&data->lock);
 			if (ret < 0)
-				return ret;
+				break;
 			*val = sign_extend32(be32_to_cpu(tmp) >> 20, 11);
-			return IIO_VAL_INT;
+			ret = IIO_VAL_INT;
+			break;
 		default:
-			return -EINVAL;
+			ret = -EINVAL;
+			break;
 		}
+
+		iio_device_release_direct_mode(indio_dev);
+		return ret;
+
 	case IIO_CHAN_INFO_SCALE:
 		switch (chan->type) {
 		case IIO_PRESSURE:
diff --git a/drivers/iio/pressure/ms5611_core.c b/drivers/iio/pressure/ms5611_core.c
index a74ed1f0c880..6bd53e702667 100644
--- a/drivers/iio/pressure/ms5611_core.c
+++ b/drivers/iio/pressure/ms5611_core.c
@@ -392,17 +392,14 @@ static int ms5611_init(struct iio_dev *indio_dev)
 
 	/* Enable attached regulator if any. */
 	st->vdd = devm_regulator_get(indio_dev->dev.parent, "vdd");
-	if (!IS_ERR(st->vdd)) {
-		ret = regulator_enable(st->vdd);
-		if (ret) {
-			dev_err(indio_dev->dev.parent,
-				"failed to enable Vdd supply: %d\n", ret);
-			return ret;
-		}
-	} else {
-		ret = PTR_ERR(st->vdd);
-		if (ret != -ENODEV)
-			return ret;
+	if (IS_ERR(st->vdd))
+		return PTR_ERR(st->vdd);
+
+	ret = regulator_enable(st->vdd);
+	if (ret) {
+		dev_err(indio_dev->dev.parent,
+			"failed to enable Vdd supply: %d\n", ret);
+		return ret;
 	}
 
 	ret = ms5611_reset(indio_dev);
diff --git a/drivers/iio/pressure/st_pressure_core.c b/drivers/iio/pressure/st_pressure_core.c
index 55df9a75eb3a..e19e0787864c 100644
--- a/drivers/iio/pressure/st_pressure_core.c
+++ b/drivers/iio/pressure/st_pressure_core.c
@@ -112,115 +112,24 @@
 #define ST_PRESS_1_OUT_XL_ADDR			0x28
 #define ST_TEMP_1_OUT_L_ADDR			0x2b
 
-/*
- * CUSTOM VALUES FOR LPS331AP SENSOR
- * See LPS331AP datasheet:
- * http://www2.st.com/resource/en/datasheet/lps331ap.pdf
- */
-#define ST_PRESS_LPS331AP_WAI_EXP		0xbb
-#define ST_PRESS_LPS331AP_ODR_ADDR		0x20
-#define ST_PRESS_LPS331AP_ODR_MASK		0x70
-#define ST_PRESS_LPS331AP_ODR_AVL_1HZ_VAL	0x01
-#define ST_PRESS_LPS331AP_ODR_AVL_7HZ_VAL	0x05
-#define ST_PRESS_LPS331AP_ODR_AVL_13HZ_VAL	0x06
-#define ST_PRESS_LPS331AP_ODR_AVL_25HZ_VAL	0x07
-#define ST_PRESS_LPS331AP_PW_ADDR		0x20
-#define ST_PRESS_LPS331AP_PW_MASK		0x80
-#define ST_PRESS_LPS331AP_FS_ADDR		0x23
-#define ST_PRESS_LPS331AP_FS_MASK		0x30
-#define ST_PRESS_LPS331AP_BDU_ADDR		0x20
-#define ST_PRESS_LPS331AP_BDU_MASK		0x04
-#define ST_PRESS_LPS331AP_DRDY_IRQ_ADDR		0x22
-#define ST_PRESS_LPS331AP_DRDY_IRQ_INT1_MASK	0x04
-#define ST_PRESS_LPS331AP_DRDY_IRQ_INT2_MASK	0x20
-#define ST_PRESS_LPS331AP_IHL_IRQ_ADDR		0x22
-#define ST_PRESS_LPS331AP_IHL_IRQ_MASK		0x80
-#define ST_PRESS_LPS331AP_OD_IRQ_ADDR		0x22
-#define ST_PRESS_LPS331AP_OD_IRQ_MASK		0x40
-#define ST_PRESS_LPS331AP_MULTIREAD_BIT		true
-
-/*
- * CUSTOM VALUES FOR THE OBSOLETE LPS001WP SENSOR
- */
-
 /* LPS001WP pressure resolution */
 #define ST_PRESS_LPS001WP_LSB_PER_MBAR		16UL
 /* LPS001WP temperature resolution */
 #define ST_PRESS_LPS001WP_LSB_PER_CELSIUS	64UL
-
-#define ST_PRESS_LPS001WP_WAI_EXP		0xba
-#define ST_PRESS_LPS001WP_ODR_ADDR		0x20
-#define ST_PRESS_LPS001WP_ODR_MASK		0x30
-#define ST_PRESS_LPS001WP_ODR_AVL_1HZ_VAL	0x01
-#define ST_PRESS_LPS001WP_ODR_AVL_7HZ_VAL	0x02
-#define ST_PRESS_LPS001WP_ODR_AVL_13HZ_VAL	0x03
-#define ST_PRESS_LPS001WP_PW_ADDR		0x20
-#define ST_PRESS_LPS001WP_PW_MASK		0x40
+/* LPS001WP pressure gain */
 #define ST_PRESS_LPS001WP_FS_AVL_PRESS_GAIN \
 	(100000000UL / ST_PRESS_LPS001WP_LSB_PER_MBAR)
-#define ST_PRESS_LPS001WP_BDU_ADDR		0x20
-#define ST_PRESS_LPS001WP_BDU_MASK		0x04
-#define ST_PRESS_LPS001WP_MULTIREAD_BIT		true
+/* LPS001WP pressure and temp L addresses */
 #define ST_PRESS_LPS001WP_OUT_L_ADDR		0x28
 #define ST_TEMP_LPS001WP_OUT_L_ADDR		0x2a
 
-/*
- * CUSTOM VALUES FOR LPS25H SENSOR
- * See LPS25H datasheet:
- * http://www2.st.com/resource/en/datasheet/lps25h.pdf
- */
-#define ST_PRESS_LPS25H_WAI_EXP			0xbd
-#define ST_PRESS_LPS25H_ODR_ADDR		0x20
-#define ST_PRESS_LPS25H_ODR_MASK		0x70
-#define ST_PRESS_LPS25H_ODR_AVL_1HZ_VAL		0x01
-#define ST_PRESS_LPS25H_ODR_AVL_7HZ_VAL		0x02
-#define ST_PRESS_LPS25H_ODR_AVL_13HZ_VAL	0x03
-#define ST_PRESS_LPS25H_ODR_AVL_25HZ_VAL	0x04
-#define ST_PRESS_LPS25H_PW_ADDR			0x20
-#define ST_PRESS_LPS25H_PW_MASK			0x80
-#define ST_PRESS_LPS25H_BDU_ADDR		0x20
-#define ST_PRESS_LPS25H_BDU_MASK		0x04
-#define ST_PRESS_LPS25H_DRDY_IRQ_ADDR		0x23
-#define ST_PRESS_LPS25H_DRDY_IRQ_INT1_MASK	0x01
-#define ST_PRESS_LPS25H_DRDY_IRQ_INT2_MASK	0x10
-#define ST_PRESS_LPS25H_IHL_IRQ_ADDR		0x22
-#define ST_PRESS_LPS25H_IHL_IRQ_MASK		0x80
-#define ST_PRESS_LPS25H_OD_IRQ_ADDR		0x22
-#define ST_PRESS_LPS25H_OD_IRQ_MASK		0x40
-#define ST_PRESS_LPS25H_MULTIREAD_BIT		true
+/* LPS25H pressure and temp L addresses */
 #define ST_PRESS_LPS25H_OUT_XL_ADDR		0x28
 #define ST_TEMP_LPS25H_OUT_L_ADDR		0x2b
 
-/*
- * CUSTOM VALUES FOR LPS22HB SENSOR
- * See LPS22HB datasheet:
- * http://www2.st.com/resource/en/datasheet/lps22hb.pdf
- */
-
 /* LPS22HB temperature sensitivity */
 #define ST_PRESS_LPS22HB_LSB_PER_CELSIUS	100UL
 
-#define ST_PRESS_LPS22HB_WAI_EXP		0xb1
-#define ST_PRESS_LPS22HB_ODR_ADDR		0x10
-#define ST_PRESS_LPS22HB_ODR_MASK		0x70
-#define ST_PRESS_LPS22HB_ODR_AVL_1HZ_VAL	0x01
-#define ST_PRESS_LPS22HB_ODR_AVL_10HZ_VAL	0x02
-#define ST_PRESS_LPS22HB_ODR_AVL_25HZ_VAL	0x03
-#define ST_PRESS_LPS22HB_ODR_AVL_50HZ_VAL	0x04
-#define ST_PRESS_LPS22HB_ODR_AVL_75HZ_VAL	0x05
-#define ST_PRESS_LPS22HB_PW_ADDR		0x10
-#define ST_PRESS_LPS22HB_PW_MASK		0x70
-#define ST_PRESS_LPS22HB_BDU_ADDR		0x10
-#define ST_PRESS_LPS22HB_BDU_MASK		0x02
-#define ST_PRESS_LPS22HB_DRDY_IRQ_ADDR		0x12
-#define ST_PRESS_LPS22HB_DRDY_IRQ_INT1_MASK	0x04
-#define ST_PRESS_LPS22HB_DRDY_IRQ_INT2_MASK	0x08
-#define ST_PRESS_LPS22HB_IHL_IRQ_ADDR		0x12
-#define ST_PRESS_LPS22HB_IHL_IRQ_MASK		0x80
-#define ST_PRESS_LPS22HB_OD_IRQ_ADDR		0x12
-#define ST_PRESS_LPS22HB_OD_IRQ_MASK		0x40
-#define ST_PRESS_LPS22HB_MULTIREAD_BIT		true
-
 static const struct iio_chan_spec st_press_1_channels[] = {
 	{
 		.type = IIO_PRESSURE,
@@ -321,7 +230,12 @@ static const struct iio_chan_spec st_press_lps22hb_channels[] = {
 
 static const struct st_sensor_settings st_press_sensors_settings[] = {
 	{
-		.wai = ST_PRESS_LPS331AP_WAI_EXP,
+		/*
+		 * CUSTOM VALUES FOR LPS331AP SENSOR
+		 * See LPS331AP datasheet:
+		 * http://www2.st.com/resource/en/datasheet/lps331ap.pdf
+		 */
+		.wai = 0xbb,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = LPS331AP_PRESS_DEV_NAME,
@@ -329,24 +243,24 @@ static const struct st_sensor_settings st_press_sensors_settings[] = {
 		.ch = (struct iio_chan_spec *)st_press_1_channels,
 		.num_ch = ARRAY_SIZE(st_press_1_channels),
 		.odr = {
-			.addr = ST_PRESS_LPS331AP_ODR_ADDR,
-			.mask = ST_PRESS_LPS331AP_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0x70,
 			.odr_avl = {
-				{ 1, ST_PRESS_LPS331AP_ODR_AVL_1HZ_VAL, },
-				{ 7, ST_PRESS_LPS331AP_ODR_AVL_7HZ_VAL, },
-				{ 13, ST_PRESS_LPS331AP_ODR_AVL_13HZ_VAL, },
-				{ 25, ST_PRESS_LPS331AP_ODR_AVL_25HZ_VAL, },
+				{ .hz = 1, .value = 0x01 },
+				{ .hz = 7, .value = 0x05 },
+				{ .hz = 13, .value = 0x06 },
+				{ .hz = 25, .value = 0x07 },
 			},
 		},
 		.pw = {
-			.addr = ST_PRESS_LPS331AP_PW_ADDR,
-			.mask = ST_PRESS_LPS331AP_PW_MASK,
+			.addr = 0x20,
+			.mask = 0x80,
 			.value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
 		.fs = {
-			.addr = ST_PRESS_LPS331AP_FS_ADDR,
-			.mask = ST_PRESS_LPS331AP_FS_MASK,
+			.addr = 0x23,
+			.mask = 0x30,
 			.fs_avl = {
 				/*
 				 * Pressure and temperature sensitivity values
@@ -360,24 +274,27 @@ static const struct st_sensor_settings st_press_sensors_settings[] = {
 			},
 		},
 		.bdu = {
-			.addr = ST_PRESS_LPS331AP_BDU_ADDR,
-			.mask = ST_PRESS_LPS331AP_BDU_MASK,
+			.addr = 0x20,
+			.mask = 0x04,
 		},
 		.drdy_irq = {
-			.addr = ST_PRESS_LPS331AP_DRDY_IRQ_ADDR,
-			.mask_int1 = ST_PRESS_LPS331AP_DRDY_IRQ_INT1_MASK,
-			.mask_int2 = ST_PRESS_LPS331AP_DRDY_IRQ_INT2_MASK,
-			.addr_ihl = ST_PRESS_LPS331AP_IHL_IRQ_ADDR,
-			.mask_ihl = ST_PRESS_LPS331AP_IHL_IRQ_MASK,
-			.addr_od = ST_PRESS_LPS331AP_OD_IRQ_ADDR,
-			.mask_od = ST_PRESS_LPS331AP_OD_IRQ_MASK,
+			.addr = 0x22,
+			.mask_int1 = 0x04,
+			.mask_int2 = 0x20,
+			.addr_ihl = 0x22,
+			.mask_ihl = 0x80,
+			.addr_od = 0x22,
+			.mask_od = 0x40,
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 		},
-		.multi_read_bit = ST_PRESS_LPS331AP_MULTIREAD_BIT,
+		.multi_read_bit = true,
 		.bootime = 2,
 	},
 	{
-		.wai = ST_PRESS_LPS001WP_WAI_EXP,
+		/*
+		 * CUSTOM VALUES FOR LPS001WP SENSOR
+		 */
+		.wai = 0xba,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = LPS001WP_PRESS_DEV_NAME,
@@ -385,17 +302,17 @@ static const struct st_sensor_settings st_press_sensors_settings[] = {
 		.ch = (struct iio_chan_spec *)st_press_lps001wp_channels,
 		.num_ch = ARRAY_SIZE(st_press_lps001wp_channels),
 		.odr = {
-			.addr = ST_PRESS_LPS001WP_ODR_ADDR,
-			.mask = ST_PRESS_LPS001WP_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0x30,
 			.odr_avl = {
-				{ 1, ST_PRESS_LPS001WP_ODR_AVL_1HZ_VAL, },
-				{ 7, ST_PRESS_LPS001WP_ODR_AVL_7HZ_VAL, },
-				{ 13, ST_PRESS_LPS001WP_ODR_AVL_13HZ_VAL, },
+				{ .hz = 1, .value = 0x01 },
+				{ .hz = 7, .value = 0x02 },
+				{ .hz = 13, .value = 0x03 },
 			},
 		},
 		.pw = {
-			.addr = ST_PRESS_LPS001WP_PW_ADDR,
-			.mask = ST_PRESS_LPS001WP_PW_MASK,
+			.addr = 0x20,
+			.mask = 0x40,
 			.value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
@@ -413,17 +330,22 @@ static const struct st_sensor_settings st_press_sensors_settings[] = {
 			},
 		},
 		.bdu = {
-			.addr = ST_PRESS_LPS001WP_BDU_ADDR,
-			.mask = ST_PRESS_LPS001WP_BDU_MASK,
+			.addr = 0x20,
+			.mask = 0x04,
 		},
 		.drdy_irq = {
 			.addr = 0,
 		},
-		.multi_read_bit = ST_PRESS_LPS001WP_MULTIREAD_BIT,
+		.multi_read_bit = true,
 		.bootime = 2,
 	},
 	{
-		.wai = ST_PRESS_LPS25H_WAI_EXP,
+		/*
+		 * CUSTOM VALUES FOR LPS25H SENSOR
+		 * See LPS25H datasheet:
+		 * http://www2.st.com/resource/en/datasheet/lps25h.pdf
+		 */
+		.wai = 0xbd,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = LPS25H_PRESS_DEV_NAME,
@@ -431,18 +353,18 @@ static const struct st_sensor_settings st_press_sensors_settings[] = {
 		.ch = (struct iio_chan_spec *)st_press_1_channels,
 		.num_ch = ARRAY_SIZE(st_press_1_channels),
 		.odr = {
-			.addr = ST_PRESS_LPS25H_ODR_ADDR,
-			.mask = ST_PRESS_LPS25H_ODR_MASK,
+			.addr = 0x20,
+			.mask = 0x70,
 			.odr_avl = {
-				{ 1, ST_PRESS_LPS25H_ODR_AVL_1HZ_VAL, },
-				{ 7, ST_PRESS_LPS25H_ODR_AVL_7HZ_VAL, },
-				{ 13, ST_PRESS_LPS25H_ODR_AVL_13HZ_VAL, },
-				{ 25, ST_PRESS_LPS25H_ODR_AVL_25HZ_VAL, },
+				{ .hz = 1, .value = 0x01 },
+				{ .hz = 7, .value = 0x02 },
+				{ .hz = 13, .value = 0x03 },
+				{ .hz = 25, .value = 0x04 },
 			},
 		},
 		.pw = {
-			.addr = ST_PRESS_LPS25H_PW_ADDR,
-			.mask = ST_PRESS_LPS25H_PW_MASK,
+			.addr = 0x20,
+			.mask = 0x80,
 			.value_on = ST_SENSORS_DEFAULT_POWER_ON_VALUE,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
@@ -460,24 +382,29 @@ static const struct st_sensor_settings st_press_sensors_settings[] = {
 			},
 		},
 		.bdu = {
-			.addr = ST_PRESS_LPS25H_BDU_ADDR,
-			.mask = ST_PRESS_LPS25H_BDU_MASK,
+			.addr = 0x20,
+			.mask = 0x04,
 		},
 		.drdy_irq = {
-			.addr = ST_PRESS_LPS25H_DRDY_IRQ_ADDR,
-			.mask_int1 = ST_PRESS_LPS25H_DRDY_IRQ_INT1_MASK,
-			.mask_int2 = ST_PRESS_LPS25H_DRDY_IRQ_INT2_MASK,
-			.addr_ihl = ST_PRESS_LPS25H_IHL_IRQ_ADDR,
-			.mask_ihl = ST_PRESS_LPS25H_IHL_IRQ_MASK,
-			.addr_od = ST_PRESS_LPS25H_OD_IRQ_ADDR,
-			.mask_od = ST_PRESS_LPS25H_OD_IRQ_MASK,
+			.addr = 0x23,
+			.mask_int1 = 0x01,
+			.mask_int2 = 0x10,
+			.addr_ihl = 0x22,
+			.mask_ihl = 0x80,
+			.addr_od = 0x22,
+			.mask_od = 0x40,
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 		},
-		.multi_read_bit = ST_PRESS_LPS25H_MULTIREAD_BIT,
+		.multi_read_bit = true,
 		.bootime = 2,
 	},
 	{
-		.wai = ST_PRESS_LPS22HB_WAI_EXP,
+		/*
+		 * CUSTOM VALUES FOR LPS22HB SENSOR
+		 * See LPS22HB datasheet:
+		 * http://www2.st.com/resource/en/datasheet/lps22hb.pdf
+		 */
+		.wai = 0xb1,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
 			[0] = LPS22HB_PRESS_DEV_NAME,
@@ -485,19 +412,19 @@ static const struct st_sensor_settings st_press_sensors_settings[] = {
 		.ch = (struct iio_chan_spec *)st_press_lps22hb_channels,
 		.num_ch = ARRAY_SIZE(st_press_lps22hb_channels),
 		.odr = {
-			.addr = ST_PRESS_LPS22HB_ODR_ADDR,
-			.mask = ST_PRESS_LPS22HB_ODR_MASK,
+			.addr = 0x10,
+			.mask = 0x70,
 			.odr_avl = {
-				{ 1, ST_PRESS_LPS22HB_ODR_AVL_1HZ_VAL, },
-				{ 10, ST_PRESS_LPS22HB_ODR_AVL_10HZ_VAL, },
-				{ 25, ST_PRESS_LPS22HB_ODR_AVL_25HZ_VAL, },
-				{ 50, ST_PRESS_LPS22HB_ODR_AVL_50HZ_VAL, },
-				{ 75, ST_PRESS_LPS22HB_ODR_AVL_75HZ_VAL, },
+				{ .hz = 1, .value = 0x01 },
+				{ .hz = 10, .value = 0x02 },
+				{ .hz = 25, .value = 0x03 },
+				{ .hz = 50, .value = 0x04 },
+				{ .hz = 75, .value = 0x05 },
 			},
 		},
 		.pw = {
-			.addr = ST_PRESS_LPS22HB_PW_ADDR,
-			.mask = ST_PRESS_LPS22HB_PW_MASK,
+			.addr = 0x10,
+			.mask = 0x70,
 			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
 		},
 		.fs = {
@@ -514,20 +441,20 @@ static const struct st_sensor_settings st_press_sensors_settings[] = {
 			},
 		},
 		.bdu = {
-			.addr = ST_PRESS_LPS22HB_BDU_ADDR,
-			.mask = ST_PRESS_LPS22HB_BDU_MASK,
+			.addr = 0x10,
+			.mask = 0x02,
 		},
 		.drdy_irq = {
-			.addr = ST_PRESS_LPS22HB_DRDY_IRQ_ADDR,
-			.mask_int1 = ST_PRESS_LPS22HB_DRDY_IRQ_INT1_MASK,
-			.mask_int2 = ST_PRESS_LPS22HB_DRDY_IRQ_INT2_MASK,
-			.addr_ihl = ST_PRESS_LPS22HB_IHL_IRQ_ADDR,
-			.mask_ihl = ST_PRESS_LPS22HB_IHL_IRQ_MASK,
-			.addr_od = ST_PRESS_LPS22HB_OD_IRQ_ADDR,
-			.mask_od = ST_PRESS_LPS22HB_OD_IRQ_MASK,
+			.addr = 0x12,
+			.mask_int1 = 0x04,
+			.mask_int2 = 0x08,
+			.addr_ihl = 0x12,
+			.mask_ihl = 0x80,
+			.addr_od = 0x12,
+			.mask_od = 0x40,
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
 		},
-		.multi_read_bit = ST_PRESS_LPS22HB_MULTIREAD_BIT,
+		.multi_read_bit = true,
 	},
 };
 
diff --git a/drivers/iio/pressure/zpa2326.c b/drivers/iio/pressure/zpa2326.c
index 19d2eb46fda6..c720c3ac0b9b 100644
--- a/drivers/iio/pressure/zpa2326.c
+++ b/drivers/iio/pressure/zpa2326.c
@@ -147,12 +147,8 @@ struct zpa2326_private {
 #define zpa2326_warn(_idev, _format, _arg...) \
 	dev_warn(_idev->dev.parent, _format, ##_arg)
 
-#ifdef DEBUG
 #define zpa2326_dbg(_idev, _format, _arg...) \
 	dev_dbg(_idev->dev.parent, _format, ##_arg)
-#else
-#define zpa2326_dbg(_idev, _format, _arg...)
-#endif
 
 bool zpa2326_isreg_writeable(struct device *dev, unsigned int reg)
 {
diff --git a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c
index 3141c3c161bb..1fa9eefa0982 100644
--- a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c
+++ b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c
@@ -301,8 +301,6 @@ static int lidar_probe(struct i2c_client *client,
 	if (ret)
 		goto error_unreg_buffer;
 	pm_runtime_enable(&client->dev);
-
-	pm_runtime_mark_last_busy(&client->dev);
 	pm_runtime_idle(&client->dev);
 
 	return 0;
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index fb3fb89640e5..670917387eda 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -73,6 +73,7 @@ source "drivers/infiniband/hw/mlx4/Kconfig"
 source "drivers/infiniband/hw/mlx5/Kconfig"
 source "drivers/infiniband/hw/nes/Kconfig"
 source "drivers/infiniband/hw/ocrdma/Kconfig"
+source "drivers/infiniband/hw/vmw_pvrdma/Kconfig"
 source "drivers/infiniband/hw/usnic/Kconfig"
 source "drivers/infiniband/hw/hns/Kconfig"
 
diff --git a/drivers/infiniband/core/agent.c b/drivers/infiniband/core/agent.c
index 4fa524dfb6cf..11dacd97a667 100644
--- a/drivers/infiniband/core/agent.c
+++ b/drivers/infiniband/core/agent.c
@@ -156,7 +156,6 @@ int ib_agent_port_open(struct ib_device *device, int port_num)
 	/* Create new device info */
 	port_priv = kzalloc(sizeof *port_priv, GFP_KERNEL);
 	if (!port_priv) {
-		dev_err(&device->dev, "No memory for ib_agent_port_private\n");
 		ret = -ENOMEM;
 		goto error1;
 	}
diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c
index 1a2984c28b95..ae04826e82fc 100644
--- a/drivers/infiniband/core/cache.c
+++ b/drivers/infiniband/core/cache.c
@@ -770,12 +770,8 @@ static int _gid_table_setup_one(struct ib_device *ib_dev)
 	int err = 0;
 
 	table = kcalloc(ib_dev->phys_port_cnt, sizeof(*table), GFP_KERNEL);
-
-	if (!table) {
-		pr_warn("failed to allocate ib gid cache for %s\n",
-			ib_dev->name);
+	if (!table)
 		return -ENOMEM;
-	}
 
 	for (port = 0; port < ib_dev->phys_port_cnt; port++) {
 		u8 rdma_port = port + rdma_start_port(ib_dev);
@@ -1170,14 +1166,13 @@ int ib_cache_setup_one(struct ib_device *device)
 					  GFP_KERNEL);
 	if (!device->cache.pkey_cache ||
 	    !device->cache.lmc_cache) {
-		pr_warn("Couldn't allocate cache for %s\n", device->name);
-		return -ENOMEM;
+		err = -ENOMEM;
+		goto free;
 	}
 
 	err = gid_table_setup_one(device);
 	if (err)
-		/* Allocated memory will be cleaned in the release function */
-		return err;
+		goto free;
 
 	for (p = 0; p <= rdma_end_port(device) - rdma_start_port(device); ++p)
 		ib_cache_update(device, p + rdma_start_port(device));
@@ -1192,6 +1187,9 @@ int ib_cache_setup_one(struct ib_device *device)
 
 err:
 	gid_table_cleanup_one(device);
+free:
+	kfree(device->cache.pkey_cache);
+	kfree(device->cache.lmc_cache);
 	return err;
 }
 
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index 71c7c4c328ef..cf1edfa1cbac 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -57,6 +57,54 @@ MODULE_AUTHOR("Sean Hefty");
 MODULE_DESCRIPTION("InfiniBand CM");
 MODULE_LICENSE("Dual BSD/GPL");
 
+static const char * const ibcm_rej_reason_strs[] = {
+	[IB_CM_REJ_NO_QP]			= "no QP",
+	[IB_CM_REJ_NO_EEC]			= "no EEC",
+	[IB_CM_REJ_NO_RESOURCES]		= "no resources",
+	[IB_CM_REJ_TIMEOUT]			= "timeout",
+	[IB_CM_REJ_UNSUPPORTED]			= "unsupported",
+	[IB_CM_REJ_INVALID_COMM_ID]		= "invalid comm ID",
+	[IB_CM_REJ_INVALID_COMM_INSTANCE]	= "invalid comm instance",
+	[IB_CM_REJ_INVALID_SERVICE_ID]		= "invalid service ID",
+	[IB_CM_REJ_INVALID_TRANSPORT_TYPE]	= "invalid transport type",
+	[IB_CM_REJ_STALE_CONN]			= "stale conn",
+	[IB_CM_REJ_RDC_NOT_EXIST]		= "RDC not exist",
+	[IB_CM_REJ_INVALID_GID]			= "invalid GID",
+	[IB_CM_REJ_INVALID_LID]			= "invalid LID",
+	[IB_CM_REJ_INVALID_SL]			= "invalid SL",
+	[IB_CM_REJ_INVALID_TRAFFIC_CLASS]	= "invalid traffic class",
+	[IB_CM_REJ_INVALID_HOP_LIMIT]		= "invalid hop limit",
+	[IB_CM_REJ_INVALID_PACKET_RATE]		= "invalid packet rate",
+	[IB_CM_REJ_INVALID_ALT_GID]		= "invalid alt GID",
+	[IB_CM_REJ_INVALID_ALT_LID]		= "invalid alt LID",
+	[IB_CM_REJ_INVALID_ALT_SL]		= "invalid alt SL",
+	[IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS]	= "invalid alt traffic class",
+	[IB_CM_REJ_INVALID_ALT_HOP_LIMIT]	= "invalid alt hop limit",
+	[IB_CM_REJ_INVALID_ALT_PACKET_RATE]	= "invalid alt packet rate",
+	[IB_CM_REJ_PORT_CM_REDIRECT]		= "port CM redirect",
+	[IB_CM_REJ_PORT_REDIRECT]		= "port redirect",
+	[IB_CM_REJ_INVALID_MTU]			= "invalid MTU",
+	[IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES]	= "insufficient resp resources",
+	[IB_CM_REJ_CONSUMER_DEFINED]		= "consumer defined",
+	[IB_CM_REJ_INVALID_RNR_RETRY]		= "invalid RNR retry",
+	[IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID]	= "duplicate local comm ID",
+	[IB_CM_REJ_INVALID_CLASS_VERSION]	= "invalid class version",
+	[IB_CM_REJ_INVALID_FLOW_LABEL]		= "invalid flow label",
+	[IB_CM_REJ_INVALID_ALT_FLOW_LABEL]	= "invalid alt flow label",
+};
+
+const char *__attribute_const__ ibcm_reject_msg(int reason)
+{
+	size_t index = reason;
+
+	if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
+	    ibcm_rej_reason_strs[index])
+		return ibcm_rej_reason_strs[index];
+	else
+		return "unrecognized reason";
+}
+EXPORT_SYMBOL(ibcm_reject_msg);
+
 static void cm_add_one(struct ib_device *device);
 static void cm_remove_one(struct ib_device *device, void *client_data);
 
@@ -1582,6 +1630,7 @@ static struct cm_id_private * cm_match_req(struct cm_work *work,
 	struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
 	struct cm_timewait_info *timewait_info;
 	struct cm_req_msg *req_msg;
+	struct ib_cm_id *cm_id;
 
 	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
 
@@ -1603,10 +1652,18 @@ static struct cm_id_private * cm_match_req(struct cm_work *work,
 	timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
 	if (timewait_info) {
 		cm_cleanup_timewait(cm_id_priv->timewait_info);
+		cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
+					   timewait_info->work.remote_id);
+
 		spin_unlock_irq(&cm.lock);
 		cm_issue_rej(work->port, work->mad_recv_wc,
 			     IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
 			     NULL, 0);
+		if (cur_cm_id_priv) {
+			cm_id = &cur_cm_id_priv->id;
+			ib_send_cm_dreq(cm_id, NULL, 0);
+			cm_deref_id(cur_cm_id_priv);
+		}
 		return NULL;
 	}
 
@@ -1984,6 +2041,9 @@ static int cm_rep_handler(struct cm_work *work)
 	struct cm_id_private *cm_id_priv;
 	struct cm_rep_msg *rep_msg;
 	int ret;
+	struct cm_id_private *cur_cm_id_priv;
+	struct ib_cm_id *cm_id;
+	struct cm_timewait_info *timewait_info;
 
 	rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
 	cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, 0);
@@ -2018,16 +2078,26 @@ static int cm_rep_handler(struct cm_work *work)
 		goto error;
 	}
 	/* Check for a stale connection. */
-	if (cm_insert_remote_qpn(cm_id_priv->timewait_info)) {
+	timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
+	if (timewait_info) {
 		rb_erase(&cm_id_priv->timewait_info->remote_id_node,
 			 &cm.remote_id_table);
 		cm_id_priv->timewait_info->inserted_remote_id = 0;
+		cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
+					   timewait_info->work.remote_id);
+
 		spin_unlock(&cm.lock);
 		spin_unlock_irq(&cm_id_priv->lock);
 		cm_issue_rej(work->port, work->mad_recv_wc,
 			     IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
 			     NULL, 0);
 		ret = -EINVAL;
+		if (cur_cm_id_priv) {
+			cm_id = &cur_cm_id_priv->id;
+			ib_send_cm_dreq(cm_id, NULL, 0);
+			cm_deref_id(cur_cm_id_priv);
+		}
+
 		goto error;
 	}
 	spin_unlock(&cm.lock);
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 22fcf284dd8b..e7dcfac877ca 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -101,6 +101,49 @@ const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event)
 }
 EXPORT_SYMBOL(rdma_event_msg);
 
+const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id,
+						int reason)
+{
+	if (rdma_ib_or_roce(id->device, id->port_num))
+		return ibcm_reject_msg(reason);
+
+	if (rdma_protocol_iwarp(id->device, id->port_num))
+		return iwcm_reject_msg(reason);
+
+	WARN_ON_ONCE(1);
+	return "unrecognized transport";
+}
+EXPORT_SYMBOL(rdma_reject_msg);
+
+bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)
+{
+	if (rdma_ib_or_roce(id->device, id->port_num))
+		return reason == IB_CM_REJ_CONSUMER_DEFINED;
+
+	if (rdma_protocol_iwarp(id->device, id->port_num))
+		return reason == -ECONNREFUSED;
+
+	WARN_ON_ONCE(1);
+	return false;
+}
+EXPORT_SYMBOL(rdma_is_consumer_reject);
+
+const void *rdma_consumer_reject_data(struct rdma_cm_id *id,
+				      struct rdma_cm_event *ev, u8 *data_len)
+{
+	const void *p;
+
+	if (rdma_is_consumer_reject(id, ev->status)) {
+		*data_len = ev->param.conn.private_data_len;
+		p = ev->param.conn.private_data;
+	} else {
+		*data_len = 0;
+		p = NULL;
+	}
+	return p;
+}
+EXPORT_SYMBOL(rdma_consumer_reject_data);
+
 static void cma_add_one(struct ib_device *device);
 static void cma_remove_one(struct ib_device *device, void *client_data);
 
diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index 0c0bea091de8..d29372624f3a 100644
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -72,9 +72,6 @@ void ib_device_unregister_sysfs(struct ib_device *device);
 void ib_cache_setup(void);
 void ib_cache_cleanup(void);
 
-int ib_resolve_eth_dmac(struct ib_qp *qp,
-			struct ib_qp_attr *qp_attr, int *qp_attr_mask);
-
 typedef void (*roce_netdev_callback)(struct ib_device *device, u8 port,
 	      struct net_device *idev, void *cookie);
 
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 760ef603a468..571974cd3919 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -254,11 +254,8 @@ static int add_client_context(struct ib_device *device, struct ib_client *client
 	unsigned long flags;
 
 	context = kmalloc(sizeof *context, GFP_KERNEL);
-	if (!context) {
-		pr_warn("Couldn't allocate client context for %s/%s\n",
-			device->name, client->name);
+	if (!context)
 		return -ENOMEM;
-	}
 
 	context->client = client;
 	context->data   = NULL;
diff --git a/drivers/infiniband/core/fmr_pool.c b/drivers/infiniband/core/fmr_pool.c
index cdbb1f1a6d97..cdfad5f26212 100644
--- a/drivers/infiniband/core/fmr_pool.c
+++ b/drivers/infiniband/core/fmr_pool.c
@@ -247,7 +247,6 @@ struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd             *pd,
 			kmalloc(IB_FMR_HASH_SIZE * sizeof *pool->cache_bucket,
 				GFP_KERNEL);
 		if (!pool->cache_bucket) {
-			pr_warn(PFX "Failed to allocate cache in pool\n");
 			ret = -ENOMEM;
 			goto out_free_pool;
 		}
diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c
index 5495e22839a7..31661b5c1743 100644
--- a/drivers/infiniband/core/iwcm.c
+++ b/drivers/infiniband/core/iwcm.c
@@ -59,6 +59,27 @@ MODULE_AUTHOR("Tom Tucker");
 MODULE_DESCRIPTION("iWARP CM");
 MODULE_LICENSE("Dual BSD/GPL");
 
+static const char * const iwcm_rej_reason_strs[] = {
+	[ECONNRESET]			= "reset by remote host",
+	[ECONNREFUSED]			= "refused by remote application",
+	[ETIMEDOUT]			= "setup timeout",
+};
+
+const char *__attribute_const__ iwcm_reject_msg(int reason)
+{
+	size_t index;
+
+	/* iWARP uses negative errnos */
+	index = -reason;
+
+	if (index < ARRAY_SIZE(iwcm_rej_reason_strs) &&
+	    iwcm_rej_reason_strs[index])
+		return iwcm_rej_reason_strs[index];
+	else
+		return "unrecognized reason";
+}
+EXPORT_SYMBOL(iwcm_reject_msg);
+
 static struct ibnl_client_cbs iwcm_nl_cb_table[] = {
 	[RDMA_NL_IWPM_REG_PID] = {.dump = iwpm_register_pid_cb},
 	[RDMA_NL_IWPM_ADD_MAPPING] = {.dump = iwpm_add_mapping_cb},
diff --git a/drivers/infiniband/core/iwpm_msg.c b/drivers/infiniband/core/iwpm_msg.c
index 1c41b95cefec..a0e7c16d8bd8 100644
--- a/drivers/infiniband/core/iwpm_msg.c
+++ b/drivers/infiniband/core/iwpm_msg.c
@@ -604,7 +604,6 @@ int iwpm_remote_info_cb(struct sk_buff *skb, struct netlink_callback *cb)
 	}
 	rem_info = kzalloc(sizeof(struct iwpm_remote_info), GFP_ATOMIC);
 	if (!rem_info) {
-		pr_err("%s: Unable to allocate a remote info\n", __func__);
 		ret = -ENOMEM;
 		return ret;
 	}
diff --git a/drivers/infiniband/core/iwpm_util.c b/drivers/infiniband/core/iwpm_util.c
index ade71e7f0131..3ef51a96bbf1 100644
--- a/drivers/infiniband/core/iwpm_util.c
+++ b/drivers/infiniband/core/iwpm_util.c
@@ -62,7 +62,6 @@ int iwpm_init(u8 nl_client)
 					sizeof(struct hlist_head), GFP_KERNEL);
 		if (!iwpm_hash_bucket) {
 			ret = -ENOMEM;
-			pr_err("%s Unable to create mapinfo hash table\n", __func__);
 			goto init_exit;
 		}
 		iwpm_reminfo_bucket = kzalloc(IWPM_REMINFO_HASH_SIZE *
@@ -70,7 +69,6 @@ int iwpm_init(u8 nl_client)
 		if (!iwpm_reminfo_bucket) {
 			kfree(iwpm_hash_bucket);
 			ret = -ENOMEM;
-			pr_err("%s Unable to create reminfo hash table\n", __func__);
 			goto init_exit;
 		}
 	}
@@ -128,10 +126,9 @@ int iwpm_create_mapinfo(struct sockaddr_storage *local_sockaddr,
 	if (!iwpm_valid_client(nl_client))
 		return ret;
 	map_info = kzalloc(sizeof(struct iwpm_mapping_info), GFP_KERNEL);
-	if (!map_info) {
-		pr_err("%s: Unable to allocate a mapping info\n", __func__);
+	if (!map_info)
 		return -ENOMEM;
-	}
+
 	memcpy(&map_info->local_sockaddr, local_sockaddr,
 	       sizeof(struct sockaddr_storage));
 	memcpy(&map_info->mapped_sockaddr, mapped_sockaddr,
@@ -309,10 +306,9 @@ struct iwpm_nlmsg_request *iwpm_get_nlmsg_request(__u32 nlmsg_seq,
 	unsigned long flags;
 
 	nlmsg_request = kzalloc(sizeof(struct iwpm_nlmsg_request), gfp);
-	if (!nlmsg_request) {
-		pr_err("%s Unable to allocate a nlmsg_request\n", __func__);
+	if (!nlmsg_request)
 		return NULL;
-	}
+
 	spin_lock_irqsave(&iwpm_nlmsg_req_lock, flags);
 	list_add_tail(&nlmsg_request->inprocess_list, &iwpm_nlmsg_req_list);
 	spin_unlock_irqrestore(&iwpm_nlmsg_req_lock, flags);
diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index 40cbd6bdb73b..a009f7132c73 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -769,7 +769,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 	 * If we are at the start of the LID routed part, don't update the
 	 * hop_ptr or hop_cnt.  See section 14.2.2, Vol 1 IB spec.
 	 */
-	if (opa && smp->class_version == OPA_SMP_CLASS_VERSION) {
+	if (opa && smp->class_version == OPA_SM_CLASS_VERSION) {
 		u32 opa_drslid;
 
 		if ((opa_get_smp_direction(opa_smp)
@@ -816,7 +816,6 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 	local = kmalloc(sizeof *local, GFP_ATOMIC);
 	if (!local) {
 		ret = -ENOMEM;
-		dev_err(&device->dev, "No memory for ib_mad_local_private\n");
 		goto out;
 	}
 	local->mad_priv = NULL;
@@ -824,7 +823,6 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 	mad_priv = alloc_mad_private(mad_size, GFP_ATOMIC);
 	if (!mad_priv) {
 		ret = -ENOMEM;
-		dev_err(&device->dev, "No memory for local response MAD\n");
 		kfree(local);
 		goto out;
 	}
@@ -947,9 +945,6 @@ static int alloc_send_rmpp_list(struct ib_mad_send_wr_private *send_wr,
 	for (left = send_buf->data_len + pad; left > 0; left -= seg_size) {
 		seg = kmalloc(sizeof (*seg) + seg_size, gfp_mask);
 		if (!seg) {
-			dev_err(&send_buf->mad_agent->device->dev,
-				"alloc_send_rmpp_segs: RMPP mem alloc failed for len %zd, gfp %#x\n",
-				sizeof (*seg) + seg_size, gfp_mask);
 			free_send_rmpp_list(send_wr);
 			return -ENOMEM;
 		}
@@ -1362,12 +1357,7 @@ static int allocate_method_table(struct ib_mad_mgmt_method_table **method)
 {
 	/* Allocate management method table */
 	*method = kzalloc(sizeof **method, GFP_ATOMIC);
-	if (!*method) {
-		pr_err("No memory for ib_mad_mgmt_method_table\n");
-		return -ENOMEM;
-	}
-
-	return 0;
+	return (*method) ? 0 : (-ENOMEM);
 }
 
 /*
@@ -1458,8 +1448,6 @@ static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,
 		/* Allocate management class table for "new" class version */
 		*class = kzalloc(sizeof **class, GFP_ATOMIC);
 		if (!*class) {
-			dev_err(&agent_priv->agent.device->dev,
-				"No memory for ib_mad_mgmt_class_table\n");
 			ret = -ENOMEM;
 			goto error1;
 		}
@@ -1524,22 +1512,16 @@ static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req,
 	if (!*vendor_table) {
 		/* Allocate mgmt vendor class table for "new" class version */
 		vendor = kzalloc(sizeof *vendor, GFP_ATOMIC);
-		if (!vendor) {
-			dev_err(&agent_priv->agent.device->dev,
-				"No memory for ib_mad_mgmt_vendor_class_table\n");
+		if (!vendor)
 			goto error1;
-		}
 
 		*vendor_table = vendor;
 	}
 	if (!(*vendor_table)->vendor_class[vclass]) {
 		/* Allocate table for this management vendor class */
 		vendor_class = kzalloc(sizeof *vendor_class, GFP_ATOMIC);
-		if (!vendor_class) {
-			dev_err(&agent_priv->agent.device->dev,
-				"No memory for ib_mad_mgmt_vendor_class\n");
+		if (!vendor_class)
 			goto error2;
-		}
 
 		(*vendor_table)->vendor_class[vclass] = vendor_class;
 	}
@@ -1746,7 +1728,7 @@ find_mad_agent(struct ib_mad_port_private *port_priv,
 			if (!class)
 				goto out;
 			if (convert_mgmt_class(mad_hdr->mgmt_class) >=
-			    IB_MGMT_MAX_METHODS)
+			    ARRAY_SIZE(class->method_table))
 				goto out;
 			method = class->method_table[convert_mgmt_class(
 							mad_hdr->mgmt_class)];
@@ -2167,7 +2149,7 @@ handle_smi(struct ib_mad_port_private *port_priv,
 	struct ib_mad_hdr *mad_hdr = (struct ib_mad_hdr *)recv->mad;
 
 	if (opa && mad_hdr->base_version == OPA_MGMT_BASE_VERSION &&
-	    mad_hdr->class_version == OPA_SMI_CLASS_VERSION)
+	    mad_hdr->class_version == OPA_SM_CLASS_VERSION)
 		return handle_opa_smi(port_priv, qp_info, wc, port_num, recv,
 				      response);
 
@@ -2238,11 +2220,8 @@ static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)
 
 	mad_size = recv->mad_size;
 	response = alloc_mad_private(mad_size, GFP_KERNEL);
-	if (!response) {
-		dev_err(&port_priv->device->dev,
-			"%s: no memory for response buffer\n", __func__);
+	if (!response)
 		goto out;
-	}
 
 	if (rdma_cap_ib_switch(port_priv->device))
 		port_num = wc->port_num;
@@ -2869,8 +2848,6 @@ static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,
 			mad_priv = alloc_mad_private(port_mad_size(qp_info->port_priv),
 						     GFP_ATOMIC);
 			if (!mad_priv) {
-				dev_err(&qp_info->port_priv->device->dev,
-					"No memory for receive buffer\n");
 				ret = -ENOMEM;
 				break;
 			}
@@ -2961,11 +2938,8 @@ static int ib_mad_port_start(struct ib_mad_port_private *port_priv)
 	u16 pkey_index;
 
 	attr = kmalloc(sizeof *attr, GFP_KERNEL);
-	if (!attr) {
-		dev_err(&port_priv->device->dev,
-			"Couldn't kmalloc ib_qp_attr\n");
+	if (!attr)
 		return -ENOMEM;
-	}
 
 	ret = ib_find_pkey(port_priv->device, port_priv->port_num,
 			   IB_DEFAULT_PKEY_FULL, &pkey_index);
@@ -3135,10 +3109,8 @@ static int ib_mad_port_open(struct ib_device *device,
 
 	/* Create new device info */
 	port_priv = kzalloc(sizeof *port_priv, GFP_KERNEL);
-	if (!port_priv) {
-		dev_err(&device->dev, "No memory for ib_mad_port_private\n");
+	if (!port_priv)
 		return -ENOMEM;
-	}
 
 	port_priv->device = device;
 	port_priv->port_num = port_num;
diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c
index e51b739f6ea3..322cb67b07a9 100644
--- a/drivers/infiniband/core/multicast.c
+++ b/drivers/infiniband/core/multicast.c
@@ -518,8 +518,11 @@ static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
 		process_join_error(group, status);
 	else {
 		int mgids_changed, is_mgid0;
-		ib_find_pkey(group->port->dev->device, group->port->port_num,
-			     be16_to_cpu(rec->pkey), &pkey_index);
+
+		if (ib_find_pkey(group->port->dev->device,
+				 group->port->port_num, be16_to_cpu(rec->pkey),
+				 &pkey_index))
+			pkey_index = MCAST_INVALID_PKEY_INDEX;
 
 		spin_lock_irq(&group->port->lock);
 		if (group->state == MCAST_BUSY &&
diff --git a/drivers/infiniband/core/roce_gid_mgmt.c b/drivers/infiniband/core/roce_gid_mgmt.c
index 3a64a0881882..0621f4455732 100644
--- a/drivers/infiniband/core/roce_gid_mgmt.c
+++ b/drivers/infiniband/core/roce_gid_mgmt.c
@@ -304,10 +304,9 @@ static void enum_netdev_ipv4_ips(struct ib_device *ib_dev,
 	for_ifa(in_dev) {
 		struct sin_list *entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
 
-		if (!entry) {
-			pr_warn("roce_gid_mgmt: couldn't allocate entry for IPv4 update\n");
+		if (!entry)
 			continue;
-		}
+
 		entry->ip.sin_family = AF_INET;
 		entry->ip.sin_addr.s_addr = ifa->ifa_address;
 		list_add_tail(&entry->list, &sin_list);
@@ -348,10 +347,8 @@ static void enum_netdev_ipv6_ips(struct ib_device *ib_dev,
 	list_for_each_entry(ifp, &in6_dev->addr_list, if_list) {
 		struct sin6_list *entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
 
-		if (!entry) {
-			pr_warn("roce_gid_mgmt: couldn't allocate entry for IPv6 update\n");
+		if (!entry)
 			continue;
-		}
 
 		entry->sin6.sin6_family = AF_INET6;
 		entry->sin6.sin6_addr = ifp->addr;
@@ -447,10 +444,8 @@ static int netdev_upper_walk(struct net_device *upper, void *data)
 	struct upper_list *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
 	struct list_head *upper_list = data;
 
-	if (!entry) {
-		pr_info("roce_gid_mgmt: couldn't allocate entry to delete ndev\n");
+	if (!entry)
 		return 0;
-	}
 
 	list_add_tail(&entry->list, upper_list);
 	dev_hold(upper);
@@ -559,10 +554,8 @@ static int netdevice_queue_work(struct netdev_event_work_cmd *cmds,
 	struct netdev_event_work *ndev_work =
 		kmalloc(sizeof(*ndev_work), GFP_KERNEL);
 
-	if (!ndev_work) {
-		pr_warn("roce_gid_mgmt: can't allocate work for netdevice_event\n");
+	if (!ndev_work)
 		return NOTIFY_DONE;
-	}
 
 	memcpy(ndev_work->cmds, cmds, sizeof(ndev_work->cmds));
 	for (i = 0; i < ARRAY_SIZE(ndev_work->cmds) && ndev_work->cmds[i].cb; i++) {
@@ -696,10 +689,8 @@ static int addr_event(struct notifier_block *this, unsigned long event,
 	}
 
 	work = kmalloc(sizeof(*work), GFP_ATOMIC);
-	if (!work) {
-		pr_warn("roce_gid_mgmt: Couldn't allocate work for addr_event\n");
+	if (!work)
 		return NOTIFY_DONE;
-	}
 
 	INIT_WORK(&work->work, update_gid_event_work_handler);
 
diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c
index 7713ef089c3c..579f9a7f6283 100644
--- a/drivers/infiniband/core/ucm.c
+++ b/drivers/infiniband/core/ucm.c
@@ -1104,8 +1104,11 @@ static ssize_t ib_ucm_write(struct file *filp, const char __user *buf,
 	struct ib_ucm_cmd_hdr hdr;
 	ssize_t result;
 
-	if (WARN_ON_ONCE(!ib_safe_file_access(filp)))
+	if (!ib_safe_file_access(filp)) {
+		pr_err_once("ucm_write: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",
+			    task_tgid_vnr(current), current->comm);
 		return -EACCES;
+	}
 
 	if (len < sizeof(hdr))
 		return -EINVAL;
diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
index 9520154f1d7c..e12f8faf8c23 100644
--- a/drivers/infiniband/core/ucma.c
+++ b/drivers/infiniband/core/ucma.c
@@ -1584,8 +1584,11 @@ static ssize_t ucma_write(struct file *filp, const char __user *buf,
 	struct rdma_ucm_cmd_hdr hdr;
 	ssize_t ret;
 
-	if (WARN_ON_ONCE(!ib_safe_file_access(filp)))
+	if (!ib_safe_file_access(filp)) {
+		pr_err_once("ucma_write: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",
+			    task_tgid_vnr(current), current->comm);
 		return -EACCES;
+	}
 
 	if (len < sizeof(hdr))
 		return -EINVAL;
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index 84b4eff90395..1e62a5f0cb28 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -51,7 +51,7 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
 
 	if (umem->nmap > 0)
 		ib_dma_unmap_sg(dev, umem->sg_head.sgl,
-				umem->nmap,
+				umem->npages,
 				DMA_BIDIRECTIONAL);
 
 	for_each_sg(umem->sg_head.sgl, sg, umem->npages, i) {
diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
index 1f0fe3217f23..6b079a31dced 100644
--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -578,7 +578,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt,
 		 */
 		npages = get_user_pages_remote(owning_process, owning_mm,
 				user_virt, gup_num_pages,
-				flags, local_page_list, NULL);
+				flags, local_page_list, NULL, NULL);
 		up_read(&owning_mm->mmap_sem);
 
 		if (npages < 0)
diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
index df26a741cda6..455034ac994e 100644
--- a/drivers/infiniband/core/uverbs.h
+++ b/drivers/infiniband/core/uverbs.h
@@ -289,5 +289,6 @@ IB_UVERBS_DECLARE_EX_CMD(modify_wq);
 IB_UVERBS_DECLARE_EX_CMD(destroy_wq);
 IB_UVERBS_DECLARE_EX_CMD(create_rwq_ind_table);
 IB_UVERBS_DECLARE_EX_CMD(destroy_rwq_ind_table);
+IB_UVERBS_DECLARE_EX_CMD(modify_qp);
 
 #endif /* UVERBS_H */
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index cb3f515a2285..09b649159e6c 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -2328,94 +2328,88 @@ static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
 	}
 }
 
-ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
-			    struct ib_device *ib_dev,
-			    const char __user *buf, int in_len,
-			    int out_len)
+static int modify_qp(struct ib_uverbs_file *file,
+		     struct ib_uverbs_ex_modify_qp *cmd, struct ib_udata *udata)
 {
-	struct ib_uverbs_modify_qp cmd;
-	struct ib_udata            udata;
-	struct ib_qp              *qp;
-	struct ib_qp_attr         *attr;
-	int                        ret;
-
-	if (copy_from_user(&cmd, buf, sizeof cmd))
-		return -EFAULT;
-
-	INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd,
-		   out_len);
+	struct ib_qp_attr *attr;
+	struct ib_qp *qp;
+	int ret;
 
 	attr = kmalloc(sizeof *attr, GFP_KERNEL);
 	if (!attr)
 		return -ENOMEM;
 
-	qp = idr_read_qp(cmd.qp_handle, file->ucontext);
+	qp = idr_read_qp(cmd->base.qp_handle, file->ucontext);
 	if (!qp) {
 		ret = -EINVAL;
 		goto out;
 	}
 
-	attr->qp_state 		  = cmd.qp_state;
-	attr->cur_qp_state 	  = cmd.cur_qp_state;
-	attr->path_mtu 		  = cmd.path_mtu;
-	attr->path_mig_state 	  = cmd.path_mig_state;
-	attr->qkey 		  = cmd.qkey;
-	attr->rq_psn 		  = cmd.rq_psn;
-	attr->sq_psn 		  = cmd.sq_psn;
-	attr->dest_qp_num 	  = cmd.dest_qp_num;
-	attr->qp_access_flags 	  = cmd.qp_access_flags;
-	attr->pkey_index 	  = cmd.pkey_index;
-	attr->alt_pkey_index 	  = cmd.alt_pkey_index;
-	attr->en_sqd_async_notify = cmd.en_sqd_async_notify;
-	attr->max_rd_atomic 	  = cmd.max_rd_atomic;
-	attr->max_dest_rd_atomic  = cmd.max_dest_rd_atomic;
-	attr->min_rnr_timer 	  = cmd.min_rnr_timer;
-	attr->port_num 		  = cmd.port_num;
-	attr->timeout 		  = cmd.timeout;
-	attr->retry_cnt 	  = cmd.retry_cnt;
-	attr->rnr_retry 	  = cmd.rnr_retry;
-	attr->alt_port_num 	  = cmd.alt_port_num;
-	attr->alt_timeout 	  = cmd.alt_timeout;
-
-	memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16);
-	attr->ah_attr.grh.flow_label        = cmd.dest.flow_label;
-	attr->ah_attr.grh.sgid_index        = cmd.dest.sgid_index;
-	attr->ah_attr.grh.hop_limit         = cmd.dest.hop_limit;
-	attr->ah_attr.grh.traffic_class     = cmd.dest.traffic_class;
-	attr->ah_attr.dlid 	    	    = cmd.dest.dlid;
-	attr->ah_attr.sl   	    	    = cmd.dest.sl;
-	attr->ah_attr.src_path_bits 	    = cmd.dest.src_path_bits;
-	attr->ah_attr.static_rate   	    = cmd.dest.static_rate;
-	attr->ah_attr.ah_flags 	    	    = cmd.dest.is_global ? IB_AH_GRH : 0;
-	attr->ah_attr.port_num 	    	    = cmd.dest.port_num;
-
-	memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16);
-	attr->alt_ah_attr.grh.flow_label    = cmd.alt_dest.flow_label;
-	attr->alt_ah_attr.grh.sgid_index    = cmd.alt_dest.sgid_index;
-	attr->alt_ah_attr.grh.hop_limit     = cmd.alt_dest.hop_limit;
-	attr->alt_ah_attr.grh.traffic_class = cmd.alt_dest.traffic_class;
-	attr->alt_ah_attr.dlid 	    	    = cmd.alt_dest.dlid;
-	attr->alt_ah_attr.sl   	    	    = cmd.alt_dest.sl;
-	attr->alt_ah_attr.src_path_bits     = cmd.alt_dest.src_path_bits;
-	attr->alt_ah_attr.static_rate       = cmd.alt_dest.static_rate;
-	attr->alt_ah_attr.ah_flags 	    = cmd.alt_dest.is_global ? IB_AH_GRH : 0;
-	attr->alt_ah_attr.port_num 	    = cmd.alt_dest.port_num;
+	attr->qp_state		  = cmd->base.qp_state;
+	attr->cur_qp_state	  = cmd->base.cur_qp_state;
+	attr->path_mtu		  = cmd->base.path_mtu;
+	attr->path_mig_state	  = cmd->base.path_mig_state;
+	attr->qkey		  = cmd->base.qkey;
+	attr->rq_psn		  = cmd->base.rq_psn;
+	attr->sq_psn		  = cmd->base.sq_psn;
+	attr->dest_qp_num	  = cmd->base.dest_qp_num;
+	attr->qp_access_flags	  = cmd->base.qp_access_flags;
+	attr->pkey_index	  = cmd->base.pkey_index;
+	attr->alt_pkey_index	  = cmd->base.alt_pkey_index;
+	attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify;
+	attr->max_rd_atomic	  = cmd->base.max_rd_atomic;
+	attr->max_dest_rd_atomic  = cmd->base.max_dest_rd_atomic;
+	attr->min_rnr_timer	  = cmd->base.min_rnr_timer;
+	attr->port_num		  = cmd->base.port_num;
+	attr->timeout		  = cmd->base.timeout;
+	attr->retry_cnt		  = cmd->base.retry_cnt;
+	attr->rnr_retry		  = cmd->base.rnr_retry;
+	attr->alt_port_num	  = cmd->base.alt_port_num;
+	attr->alt_timeout	  = cmd->base.alt_timeout;
+	attr->rate_limit	  = cmd->rate_limit;
+
+	memcpy(attr->ah_attr.grh.dgid.raw, cmd->base.dest.dgid, 16);
+	attr->ah_attr.grh.flow_label	= cmd->base.dest.flow_label;
+	attr->ah_attr.grh.sgid_index	= cmd->base.dest.sgid_index;
+	attr->ah_attr.grh.hop_limit	= cmd->base.dest.hop_limit;
+	attr->ah_attr.grh.traffic_class	= cmd->base.dest.traffic_class;
+	attr->ah_attr.dlid		= cmd->base.dest.dlid;
+	attr->ah_attr.sl		= cmd->base.dest.sl;
+	attr->ah_attr.src_path_bits	= cmd->base.dest.src_path_bits;
+	attr->ah_attr.static_rate	= cmd->base.dest.static_rate;
+	attr->ah_attr.ah_flags		= cmd->base.dest.is_global ?
+					  IB_AH_GRH : 0;
+	attr->ah_attr.port_num		= cmd->base.dest.port_num;
+
+	memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd->base.alt_dest.dgid, 16);
+	attr->alt_ah_attr.grh.flow_label    = cmd->base.alt_dest.flow_label;
+	attr->alt_ah_attr.grh.sgid_index    = cmd->base.alt_dest.sgid_index;
+	attr->alt_ah_attr.grh.hop_limit     = cmd->base.alt_dest.hop_limit;
+	attr->alt_ah_attr.grh.traffic_class = cmd->base.alt_dest.traffic_class;
+	attr->alt_ah_attr.dlid		    = cmd->base.alt_dest.dlid;
+	attr->alt_ah_attr.sl		    = cmd->base.alt_dest.sl;
+	attr->alt_ah_attr.src_path_bits	    = cmd->base.alt_dest.src_path_bits;
+	attr->alt_ah_attr.static_rate	    = cmd->base.alt_dest.static_rate;
+	attr->alt_ah_attr.ah_flags	    = cmd->base.alt_dest.is_global ?
+					      IB_AH_GRH : 0;
+	attr->alt_ah_attr.port_num	    = cmd->base.alt_dest.port_num;
 
 	if (qp->real_qp == qp) {
-		ret = ib_resolve_eth_dmac(qp, attr, &cmd.attr_mask);
-		if (ret)
-			goto release_qp;
+		if (cmd->base.attr_mask & IB_QP_AV) {
+			ret = ib_resolve_eth_dmac(qp->device, &attr->ah_attr);
+			if (ret)
+				goto release_qp;
+		}
 		ret = qp->device->modify_qp(qp, attr,
-			modify_qp_mask(qp->qp_type, cmd.attr_mask), &udata);
+					    modify_qp_mask(qp->qp_type,
+							   cmd->base.attr_mask),
+					    udata);
 	} else {
-		ret = ib_modify_qp(qp, attr, modify_qp_mask(qp->qp_type, cmd.attr_mask));
+		ret = ib_modify_qp(qp, attr,
+				   modify_qp_mask(qp->qp_type,
+						  cmd->base.attr_mask));
 	}
 
-	if (ret)
-		goto release_qp;
-
-	ret = in_len;
-
 release_qp:
 	put_qp_read(qp);
 
@@ -2425,6 +2419,68 @@ out:
 	return ret;
 }
 
+ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
+			    struct ib_device *ib_dev,
+			    const char __user *buf, int in_len,
+			    int out_len)
+{
+	struct ib_uverbs_ex_modify_qp cmd = {};
+	struct ib_udata udata;
+	int ret;
+
+	if (copy_from_user(&cmd.base, buf, sizeof(cmd.base)))
+		return -EFAULT;
+
+	if (cmd.base.attr_mask &
+	    ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1))
+		return -EOPNOTSUPP;
+
+	INIT_UDATA(&udata, buf + sizeof(cmd.base), NULL,
+		   in_len - sizeof(cmd.base), out_len);
+
+	ret = modify_qp(file, &cmd, &udata);
+	if (ret)
+		return ret;
+
+	return in_len;
+}
+
+int ib_uverbs_ex_modify_qp(struct ib_uverbs_file *file,
+			   struct ib_device *ib_dev,
+			   struct ib_udata *ucore,
+			   struct ib_udata *uhw)
+{
+	struct ib_uverbs_ex_modify_qp cmd = {};
+	int ret;
+
+	/*
+	 * Last bit is reserved for extending the attr_mask by
+	 * using another field.
+	 */
+	BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31));
+
+	if (ucore->inlen < sizeof(cmd.base))
+		return -EINVAL;
+
+	ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
+	if (ret)
+		return ret;
+
+	if (cmd.base.attr_mask &
+	    ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))
+		return -EOPNOTSUPP;
+
+	if (ucore->inlen > sizeof(cmd)) {
+		if (ib_is_udata_cleared(ucore, sizeof(cmd),
+					ucore->inlen - sizeof(cmd)))
+			return -EOPNOTSUPP;
+	}
+
+	ret = modify_qp(file, &cmd, uhw);
+
+	return ret;
+}
+
 ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file,
 			     struct ib_device *ib_dev,
 			     const char __user *buf, int in_len,
@@ -2875,6 +2931,7 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
 	struct ib_ah			*ah;
 	struct ib_ah_attr		attr;
 	int ret;
+	struct ib_udata                   udata;
 
 	if (out_len < sizeof resp)
 		return -ENOSPC;
@@ -2882,6 +2939,10 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
+	INIT_UDATA(&udata, buf + sizeof(cmd),
+		   (unsigned long)cmd.response + sizeof(resp),
+		   in_len - sizeof(cmd), out_len - sizeof(resp));
+
 	uobj = kmalloc(sizeof *uobj, GFP_KERNEL);
 	if (!uobj)
 		return -ENOMEM;
@@ -2908,12 +2969,16 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
 	memset(&attr.dmac, 0, sizeof(attr.dmac));
 	memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16);
 
-	ah = ib_create_ah(pd, &attr);
+	ah = pd->device->create_ah(pd, &attr, &udata);
+
 	if (IS_ERR(ah)) {
 		ret = PTR_ERR(ah);
 		goto err_put;
 	}
 
+	ah->device  = pd->device;
+	ah->pd      = pd;
+	atomic_inc(&pd->usecnt);
 	ah->uobject  = uobj;
 	uobj->object = ah;
 
@@ -3124,8 +3189,10 @@ static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec *kern_spec,
 	kern_spec_val = (void *)kern_spec +
 		sizeof(struct ib_uverbs_flow_spec_hdr);
 	kern_spec_mask = kern_spec_val + kern_filter_sz;
+	if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL))
+		return -EINVAL;
 
-	switch (ib_spec->type) {
+	switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
 	case IB_FLOW_SPEC_ETH:
 		ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz);
 		actual_filter_sz = spec_filter_size(kern_spec_mask,
@@ -3175,6 +3242,21 @@ static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec *kern_spec,
 		memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz);
 		memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz);
 		break;
+	case IB_FLOW_SPEC_VXLAN_TUNNEL:
+		ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz);
+		actual_filter_sz = spec_filter_size(kern_spec_mask,
+						    kern_filter_sz,
+						    ib_filter_sz);
+		if (actual_filter_sz <= 0)
+			return -EINVAL;
+		ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel);
+		memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz);
+		memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz);
+
+		if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) ||
+		    (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24))
+			return -EINVAL;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -3745,7 +3827,6 @@ int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file,
 		err = PTR_ERR(flow_id);
 		goto err_free;
 	}
-	flow_id->qp = qp;
 	flow_id->uobject = uobj;
 	uobj->object = flow_id;
 
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 44b1104eb168..813593550c4b 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -137,6 +137,7 @@ static int (*uverbs_ex_cmd_table[])(struct ib_uverbs_file *file,
 	[IB_USER_VERBS_EX_CMD_DESTROY_WQ]       = ib_uverbs_ex_destroy_wq,
 	[IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL] = ib_uverbs_ex_create_rwq_ind_table,
 	[IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL] = ib_uverbs_ex_destroy_rwq_ind_table,
+	[IB_USER_VERBS_EX_CMD_MODIFY_QP]        = ib_uverbs_ex_modify_qp,
 };
 
 static void ib_uverbs_add_one(struct ib_device *device);
@@ -746,8 +747,11 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
 	int srcu_key;
 	ssize_t ret;
 
-	if (WARN_ON_ONCE(!ib_safe_file_access(filp)))
+	if (!ib_safe_file_access(filp)) {
+		pr_err_once("uverbs_write: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",
+			    task_tgid_vnr(current), current->comm);
 		return -EACCES;
+	}
 
 	if (count < sizeof hdr)
 		return -EINVAL;
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 83687646da68..71580cc28c9e 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -315,7 +315,7 @@ struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
 {
 	struct ib_ah *ah;
 
-	ah = pd->device->create_ah(pd, ah_attr);
+	ah = pd->device->create_ah(pd, ah_attr, NULL);
 
 	if (!IS_ERR(ah)) {
 		ah->device  = pd->device;
@@ -328,7 +328,7 @@ struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
 }
 EXPORT_SYMBOL(ib_create_ah);
 
-static int ib_get_header_version(const union rdma_network_hdr *hdr)
+int ib_get_rdma_header_version(const union rdma_network_hdr *hdr)
 {
 	const struct iphdr *ip4h = (struct iphdr *)&hdr->roce4grh;
 	struct iphdr ip4h_checked;
@@ -359,6 +359,7 @@ static int ib_get_header_version(const union rdma_network_hdr *hdr)
 		return 4;
 	return 6;
 }
+EXPORT_SYMBOL(ib_get_rdma_header_version);
 
 static enum rdma_network_type ib_get_net_type_by_grh(struct ib_device *device,
 						     u8 port_num,
@@ -369,7 +370,7 @@ static enum rdma_network_type ib_get_net_type_by_grh(struct ib_device *device,
 	if (rdma_protocol_ib(device, port_num))
 		return RDMA_NETWORK_IB;
 
-	grh_version = ib_get_header_version((union rdma_network_hdr *)grh);
+	grh_version = ib_get_rdma_header_version((union rdma_network_hdr *)grh);
 
 	if (grh_version == 4)
 		return RDMA_NETWORK_IPV4;
@@ -415,9 +416,9 @@ static int get_sgid_index_from_eth(struct ib_device *device, u8 port_num,
 				     &context, gid_index);
 }
 
-static int get_gids_from_rdma_hdr(union rdma_network_hdr *hdr,
-				  enum rdma_network_type net_type,
-				  union ib_gid *sgid, union ib_gid *dgid)
+int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
+			      enum rdma_network_type net_type,
+			      union ib_gid *sgid, union ib_gid *dgid)
 {
 	struct sockaddr_in  src_in;
 	struct sockaddr_in  dst_in;
@@ -447,6 +448,7 @@ static int get_gids_from_rdma_hdr(union rdma_network_hdr *hdr,
 		return -EINVAL;
 	}
 }
+EXPORT_SYMBOL(ib_get_gids_from_rdma_hdr);
 
 int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
 		       const struct ib_wc *wc, const struct ib_grh *grh,
@@ -469,8 +471,8 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
 			net_type = ib_get_net_type_by_grh(device, port_num, grh);
 		gid_type = ib_network_to_gid_type(net_type);
 	}
-	ret = get_gids_from_rdma_hdr((union rdma_network_hdr *)grh, net_type,
-				     &sgid, &dgid);
+	ret = ib_get_gids_from_rdma_hdr((union rdma_network_hdr *)grh, net_type,
+					&sgid, &dgid);
 	if (ret)
 		return ret;
 
@@ -1014,6 +1016,7 @@ static const struct {
 						 IB_QP_QKEY),
 				 [IB_QPT_GSI] = (IB_QP_CUR_STATE		|
 						 IB_QP_QKEY),
+				 [IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
 			 }
 		}
 	},
@@ -1047,6 +1050,7 @@ static const struct {
 						IB_QP_QKEY),
 				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
 						IB_QP_QKEY),
+				[IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
 			}
 		},
 		[IB_QPS_SQD]   = {
@@ -1196,66 +1200,66 @@ int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
 }
 EXPORT_SYMBOL(ib_modify_qp_is_ok);
 
-int ib_resolve_eth_dmac(struct ib_qp *qp,
-			struct ib_qp_attr *qp_attr, int *qp_attr_mask)
+int ib_resolve_eth_dmac(struct ib_device *device,
+			struct ib_ah_attr *ah_attr)
 {
 	int           ret = 0;
 
-	if (*qp_attr_mask & IB_QP_AV) {
-		if (qp_attr->ah_attr.port_num < rdma_start_port(qp->device) ||
-		    qp_attr->ah_attr.port_num > rdma_end_port(qp->device))
-			return -EINVAL;
-
-		if (!rdma_cap_eth_ah(qp->device, qp_attr->ah_attr.port_num))
-			return 0;
-
-		if (rdma_link_local_addr((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw)) {
-			rdma_get_ll_mac((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw,
-					qp_attr->ah_attr.dmac);
-		} else {
-			union ib_gid		sgid;
-			struct ib_gid_attr	sgid_attr;
-			int			ifindex;
-			int			hop_limit;
-
-			ret = ib_query_gid(qp->device,
-					   qp_attr->ah_attr.port_num,
-					   qp_attr->ah_attr.grh.sgid_index,
-					   &sgid, &sgid_attr);
-
-			if (ret || !sgid_attr.ndev) {
-				if (!ret)
-					ret = -ENXIO;
-				goto out;
-			}
+	if (ah_attr->port_num < rdma_start_port(device) ||
+	    ah_attr->port_num > rdma_end_port(device))
+		return -EINVAL;
 
-			ifindex = sgid_attr.ndev->ifindex;
+	if (!rdma_cap_eth_ah(device, ah_attr->port_num))
+		return 0;
 
-			ret = rdma_addr_find_l2_eth_by_grh(&sgid,
-							   &qp_attr->ah_attr.grh.dgid,
-							   qp_attr->ah_attr.dmac,
-							   NULL, &ifindex, &hop_limit);
+	if (rdma_link_local_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) {
+		rdma_get_ll_mac((struct in6_addr *)ah_attr->grh.dgid.raw,
+				ah_attr->dmac);
+	} else {
+		union ib_gid		sgid;
+		struct ib_gid_attr	sgid_attr;
+		int			ifindex;
+		int			hop_limit;
+
+		ret = ib_query_gid(device,
+				   ah_attr->port_num,
+				   ah_attr->grh.sgid_index,
+				   &sgid, &sgid_attr);
+
+		if (ret || !sgid_attr.ndev) {
+			if (!ret)
+				ret = -ENXIO;
+			goto out;
+		}
 
-			dev_put(sgid_attr.ndev);
+		ifindex = sgid_attr.ndev->ifindex;
 
-			qp_attr->ah_attr.grh.hop_limit = hop_limit;
-		}
+		ret = rdma_addr_find_l2_eth_by_grh(&sgid,
+						   &ah_attr->grh.dgid,
+						   ah_attr->dmac,
+						   NULL, &ifindex, &hop_limit);
+
+		dev_put(sgid_attr.ndev);
+
+		ah_attr->grh.hop_limit = hop_limit;
 	}
 out:
 	return ret;
 }
 EXPORT_SYMBOL(ib_resolve_eth_dmac);
 
-
 int ib_modify_qp(struct ib_qp *qp,
 		 struct ib_qp_attr *qp_attr,
 		 int qp_attr_mask)
 {
-	int ret;
 
-	ret = ib_resolve_eth_dmac(qp, qp_attr, &qp_attr_mask);
-	if (ret)
-		return ret;
+	if (qp_attr_mask & IB_QP_AV) {
+		int ret;
+
+		ret = ib_resolve_eth_dmac(qp->device, &qp_attr->ah_attr);
+		if (ret)
+			return ret;
+	}
 
 	return qp->device->modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
 }
@@ -1734,8 +1738,10 @@ struct ib_flow *ib_create_flow(struct ib_qp *qp,
 		return ERR_PTR(-ENOSYS);
 
 	flow_id = qp->device->create_flow(qp, flow_attr, domain);
-	if (!IS_ERR(flow_id))
+	if (!IS_ERR(flow_id)) {
 		atomic_inc(&qp->usecnt);
+		flow_id->qp = qp;
+	}
 	return flow_id;
 }
 EXPORT_SYMBOL(ib_create_flow);
diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile
index e7a5ed9f6f3f..ed553de2ca12 100644
--- a/drivers/infiniband/hw/Makefile
+++ b/drivers/infiniband/hw/Makefile
@@ -7,6 +7,7 @@ obj-$(CONFIG_MLX4_INFINIBAND)		+= mlx4/
 obj-$(CONFIG_MLX5_INFINIBAND)		+= mlx5/
 obj-$(CONFIG_INFINIBAND_NES)		+= nes/
 obj-$(CONFIG_INFINIBAND_OCRDMA)		+= ocrdma/
+obj-$(CONFIG_INFINIBAND_VMWARE_PVRDMA)	+= vmw_pvrdma/
 obj-$(CONFIG_INFINIBAND_USNIC)		+= usnic/
 obj-$(CONFIG_INFINIBAND_HFI1)		+= hfi1/
 obj-$(CONFIG_INFINIBAND_HNS)		+= hns/
diff --git a/drivers/infiniband/hw/cxgb3/cxio_dbg.c b/drivers/infiniband/hw/cxgb3/cxio_dbg.c
index 8bca6b4ec9af..445e89e5e7cf 100644
--- a/drivers/infiniband/hw/cxgb3/cxio_dbg.c
+++ b/drivers/infiniband/hw/cxgb3/cxio_dbg.c
@@ -45,10 +45,9 @@ void cxio_dump_tpt(struct cxio_rdev *rdev, u32 stag)
 	int size = 32;
 
 	m = kmalloc(sizeof(*m) + size, GFP_ATOMIC);
-	if (!m) {
-		PDBG("%s couldn't allocate memory.\n", __func__);
+	if (!m)
 		return;
-	}
+
 	m->mem_id = MEM_PMRX;
 	m->addr = (stag>>8) * 32 + rdev->rnic_info.tpt_base;
 	m->len = size;
@@ -82,10 +81,9 @@ void cxio_dump_pbl(struct cxio_rdev *rdev, u32 pbl_addr, uint len, u8 shift)
 	size = npages * sizeof(u64);
 
 	m = kmalloc(sizeof(*m) + size, GFP_ATOMIC);
-	if (!m) {
-		PDBG("%s couldn't allocate memory.\n", __func__);
+	if (!m)
 		return;
-	}
+
 	m->mem_id = MEM_PMRX;
 	m->addr = pbl_addr;
 	m->len = size;
@@ -144,10 +142,9 @@ void cxio_dump_rqt(struct cxio_rdev *rdev, u32 hwtid, int nents)
 	int rc;
 
 	m = kmalloc(sizeof(*m) + size, GFP_ATOMIC);
-	if (!m) {
-		PDBG("%s couldn't allocate memory.\n", __func__);
+	if (!m)
 		return;
-	}
+
 	m->mem_id = MEM_PMRX;
 	m->addr = ((hwtid)<<10) + rdev->rnic_info.rqt_base;
 	m->len = size;
@@ -177,10 +174,9 @@ void cxio_dump_tcb(struct cxio_rdev *rdev, u32 hwtid)
 	int rc;
 
 	m = kmalloc(sizeof(*m) + size, GFP_ATOMIC);
-	if (!m) {
-		PDBG("%s couldn't allocate memory.\n", __func__);
+	if (!m)
 		return;
-	}
+
 	m->mem_id = MEM_CM;
 	m->addr = hwtid * size;
 	m->len = size;
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index cba57bb53dba..9d5fe1853da4 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -62,7 +62,8 @@
 #include "common.h"
 
 static struct ib_ah *iwch_ah_create(struct ib_pd *pd,
-				    struct ib_ah_attr *ah_attr)
+				    struct ib_ah_attr *ah_attr,
+				    struct ib_udata *udata)
 {
 	return ERR_PTR(-ENOSYS);
 }
diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c
index 4e5baf4fe15e..516b0ae6dc3f 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -828,8 +828,10 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)
 	}
 	rdev->status_page = (struct t4_dev_status_page *)
 			    __get_free_page(GFP_KERNEL);
-	if (!rdev->status_page)
+	if (!rdev->status_page) {
+		err = -ENOMEM;
 		goto destroy_ocqp_pool;
+	}
 	rdev->status_page->qp_start = rdev->lldi.vr->qp.start;
 	rdev->status_page->qp_size = rdev->lldi.vr->qp.size;
 	rdev->status_page->cq_start = rdev->lldi.vr->cq.start;
@@ -841,8 +843,6 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)
 		if (rdev->wr_log) {
 			rdev->wr_log_size = 1 << c4iw_wr_log_size_order;
 			atomic_set(&rdev->wr_log_idx, 0);
-		} else {
-			pr_err(MOD "error allocating wr_log. Logging disabled\n");
 		}
 	}
 
@@ -1424,8 +1424,6 @@ static void recover_queues(struct uld_ctx *ctx)
 
 	qp_list.qps = kzalloc(count * sizeof *qp_list.qps, GFP_ATOMIC);
 	if (!qp_list.qps) {
-		printk(KERN_ERR MOD "%s: Fatal error - DB overflow recovery failed\n",
-		       pci_name(ctx->lldi.pdev));
 		spin_unlock_irq(&ctx->dev->lock);
 		return;
 	}
diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index 645e606a17c5..49b51b7e0fd7 100644
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -59,7 +59,9 @@ module_param(fastreg_support, int, 0644);
 MODULE_PARM_DESC(fastreg_support, "Advertise fastreg support (default=1)");
 
 static struct ib_ah *c4iw_ah_create(struct ib_pd *pd,
-				    struct ib_ah_attr *ah_attr)
+				    struct ib_ah_attr *ah_attr,
+				    struct ib_udata *udata)
+
 {
 	return ERR_PTR(-ENOSYS);
 }
diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c
index 67ea85a56945..7a3d906b3671 100644
--- a/drivers/infiniband/hw/hfi1/affinity.c
+++ b/drivers/infiniband/hw/hfi1/affinity.c
@@ -125,6 +125,7 @@ int node_affinity_init(void)
 				cpumask_weight(topology_sibling_cpumask(
 					cpumask_first(&node_affinity.proc.mask)
 					));
+	node_affinity.num_possible_nodes = num_possible_nodes();
 	node_affinity.num_online_nodes = num_online_nodes();
 	node_affinity.num_online_cpus = num_online_cpus();
 
@@ -135,7 +136,7 @@ int node_affinity_init(void)
 	 */
 	init_real_cpu_mask();
 
-	hfi1_per_node_cntr = kcalloc(num_possible_nodes(),
+	hfi1_per_node_cntr = kcalloc(node_affinity.num_possible_nodes,
 				     sizeof(*hfi1_per_node_cntr), GFP_KERNEL);
 	if (!hfi1_per_node_cntr)
 		return -ENOMEM;
diff --git a/drivers/infiniband/hw/hfi1/affinity.h b/drivers/infiniband/hw/hfi1/affinity.h
index 42e63316afd1..e78c7aa094e0 100644
--- a/drivers/infiniband/hw/hfi1/affinity.h
+++ b/drivers/infiniband/hw/hfi1/affinity.h
@@ -70,14 +70,6 @@ struct cpu_mask_set {
 	uint gen;
 };
 
-struct hfi1_affinity {
-	struct cpu_mask_set def_intr;
-	struct cpu_mask_set rcv_intr;
-	struct cpumask real_cpu_mask;
-	/* spin lock to protect affinity struct */
-	spinlock_t lock;
-};
-
 struct hfi1_msix_entry;
 
 /* Initialize non-HT cpu cores mask */
@@ -115,6 +107,7 @@ struct hfi1_affinity_node_list {
 	struct cpumask real_cpu_mask;
 	struct cpu_mask_set proc;
 	int num_core_siblings;
+	int num_possible_nodes;
 	int num_online_nodes;
 	int num_online_cpus;
 	struct mutex lock; /* protects affinity nodes */
diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index 24d0820873cf..ef72bc2a9e1d 100644
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -8477,7 +8477,10 @@ static int do_8051_command(
 	 */
 	if (type == HCMD_WRITE_LCB_CSR) {
 		in_data |= ((*out_data) & 0xffffffffffull) << 8;
-		reg = ((((*out_data) >> 40) & 0xff) <<
+		/* must preserve COMPLETED - it is tied to hardware */
+		reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_0);
+		reg &= DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK;
+		reg |= ((((*out_data) >> 40) & 0xff) <<
 				DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT)
 		      | ((((*out_data) >> 48) & 0xffff) <<
 				DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
@@ -9556,11 +9559,11 @@ int bringup_serdes(struct hfi1_pportdata *ppd)
 	if (HFI1_CAP_IS_KSET(EXTENDED_PSN))
 		add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK);
 
-	guid = ppd->guid;
+	guid = ppd->guids[HFI1_PORT_GUID_INDEX];
 	if (!guid) {
 		if (dd->base_guid)
 			guid = dd->base_guid + ppd->port - 1;
-		ppd->guid = guid;
+		ppd->guids[HFI1_PORT_GUID_INDEX] = guid;
 	}
 
 	/* Set linkinit_reason on power up per OPA spec */
diff --git a/drivers/infiniband/hw/hfi1/chip_registers.h b/drivers/infiniband/hw/hfi1/chip_registers.h
index 5b9993899789..5bfa839d1c48 100644
--- a/drivers/infiniband/hw/hfi1/chip_registers.h
+++ b/drivers/infiniband/hw/hfi1/chip_registers.h
@@ -415,6 +415,9 @@
 #define ASIC_CFG_SBUS_REQUEST_DATA_IN_SHIFT 32
 #define ASIC_CFG_SBUS_REQUEST_RECEIVER_ADDR_SHIFT 0
 #define ASIC_CFG_SCRATCH (ASIC + 0x000000000020)
+#define ASIC_CFG_SCRATCH_1 (ASIC_CFG_SCRATCH + 0x08)
+#define ASIC_CFG_SCRATCH_2 (ASIC_CFG_SCRATCH + 0x10)
+#define ASIC_CFG_SCRATCH_3 (ASIC_CFG_SCRATCH + 0x18)
 #define ASIC_CFG_THERM_POLL_EN (ASIC + 0x000000000050)
 #define ASIC_EEP_ADDR_CMD (ASIC + 0x000000000308)
 #define ASIC_EEP_ADDR_CMD_EP_ADDR_MASK 0xFFFFFFull
diff --git a/drivers/infiniband/hw/hfi1/debugfs.c b/drivers/infiniband/hw/hfi1/debugfs.c
index 632ba21759ab..8725f4c086cf 100644
--- a/drivers/infiniband/hw/hfi1/debugfs.c
+++ b/drivers/infiniband/hw/hfi1/debugfs.c
@@ -541,6 +541,114 @@ static ssize_t asic_flags_write(struct file *file, const char __user *buf,
 	return ret;
 }
 
+/* read the dc8051 memory */
+static ssize_t dc8051_memory_read(struct file *file, char __user *buf,
+				  size_t count, loff_t *ppos)
+{
+	struct hfi1_pportdata *ppd = private2ppd(file);
+	ssize_t rval;
+	void *tmp;
+	loff_t start, end;
+
+	/* the checks below expect the position to be positive */
+	if (*ppos < 0)
+		return -EINVAL;
+
+	tmp = kzalloc(DC8051_DATA_MEM_SIZE, GFP_KERNEL);
+	if (!tmp)
+		return -ENOMEM;
+
+	/*
+	 * Fill in the requested portion of the temporary buffer from the
+	 * 8051 memory.  The 8051 memory read is done in terms of 8 bytes.
+	 * Adjust start and end to fit.  Skip reading anything if out of
+	 * range.
+	 */
+	start = *ppos & ~0x7;	/* round down */
+	if (start < DC8051_DATA_MEM_SIZE) {
+		end = (*ppos + count + 7) & ~0x7; /* round up */
+		if (end > DC8051_DATA_MEM_SIZE)
+			end = DC8051_DATA_MEM_SIZE;
+		rval = read_8051_data(ppd->dd, start, end - start,
+				      (u64 *)(tmp + start));
+		if (rval)
+			goto done;
+	}
+
+	rval = simple_read_from_buffer(buf, count, ppos, tmp,
+				       DC8051_DATA_MEM_SIZE);
+done:
+	kfree(tmp);
+	return rval;
+}
+
+static ssize_t debugfs_lcb_read(struct file *file, char __user *buf,
+				size_t count, loff_t *ppos)
+{
+	struct hfi1_pportdata *ppd = private2ppd(file);
+	struct hfi1_devdata *dd = ppd->dd;
+	unsigned long total, csr_off;
+	u64 data;
+
+	if (*ppos < 0)
+		return -EINVAL;
+	/* only read 8 byte quantities */
+	if ((count % 8) != 0)
+		return -EINVAL;
+	/* offset must be 8-byte aligned */
+	if ((*ppos % 8) != 0)
+		return -EINVAL;
+	/* do nothing if out of range or zero count */
+	if (*ppos >= (LCB_END - LCB_START) || !count)
+		return 0;
+	/* reduce count if needed */
+	if (*ppos + count > LCB_END - LCB_START)
+		count = (LCB_END - LCB_START) - *ppos;
+
+	csr_off = LCB_START + *ppos;
+	for (total = 0; total < count; total += 8, csr_off += 8) {
+		if (read_lcb_csr(dd, csr_off, (u64 *)&data))
+			break; /* failed */
+		if (put_user(data, (unsigned long __user *)(buf + total)))
+			break;
+	}
+	*ppos += total;
+	return total;
+}
+
+static ssize_t debugfs_lcb_write(struct file *file, const char __user *buf,
+				 size_t count, loff_t *ppos)
+{
+	struct hfi1_pportdata *ppd = private2ppd(file);
+	struct hfi1_devdata *dd = ppd->dd;
+	unsigned long total, csr_off, data;
+
+	if (*ppos < 0)
+		return -EINVAL;
+	/* only write 8 byte quantities */
+	if ((count % 8) != 0)
+		return -EINVAL;
+	/* offset must be 8-byte aligned */
+	if ((*ppos % 8) != 0)
+		return -EINVAL;
+	/* do nothing if out of range or zero count */
+	if (*ppos >= (LCB_END - LCB_START) || !count)
+		return 0;
+	/* reduce count if needed */
+	if (*ppos + count > LCB_END - LCB_START)
+		count = (LCB_END - LCB_START) - *ppos;
+
+	csr_off = LCB_START + *ppos;
+	for (total = 0; total < count; total += 8, csr_off += 8) {
+		if (get_user(data, (unsigned long __user *)(buf + total)))
+			break;
+		if (write_lcb_csr(dd, csr_off, data))
+			break; /* failed */
+	}
+	*ppos += total;
+	return total;
+}
+
 /*
  * read the per-port QSFP data for ppd
  */
@@ -931,6 +1039,8 @@ static const struct counter_info port_cntr_ops[] = {
 	DEBUGFS_XOPS("qsfp2", qsfp2_debugfs_read, qsfp2_debugfs_write,
 		     qsfp2_debugfs_open, qsfp2_debugfs_release),
 	DEBUGFS_OPS("asic_flags", asic_flags_read, asic_flags_write),
+	DEBUGFS_OPS("dc8051_memory", dc8051_memory_read, NULL),
+	DEBUGFS_OPS("lcb", debugfs_lcb_read, debugfs_lcb_write),
 };
 
 static void *_sdma_cpu_list_seq_start(struct seq_file *s, loff_t *pos)
diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c
index c5efff29c147..4fbaee68012b 100644
--- a/drivers/infiniband/hw/hfi1/driver.c
+++ b/drivers/infiniband/hw/hfi1/driver.c
@@ -795,8 +795,7 @@ static inline void process_rcv_qp_work(struct hfi1_packet *packet)
 				hfi1_schedule_send(qp);
 			spin_unlock_irqrestore(&qp->s_lock, flags);
 		}
-		if (atomic_dec_and_test(&qp->refcount))
-			wake_up(&qp->wait);
+		rvt_put_qp(qp);
 	}
 }
 
diff --git a/drivers/infiniband/hw/hfi1/eprom.c b/drivers/infiniband/hw/hfi1/eprom.c
index e70c223801b4..26da124c88e2 100644
--- a/drivers/infiniband/hw/hfi1/eprom.c
+++ b/drivers/infiniband/hw/hfi1/eprom.c
@@ -207,6 +207,40 @@ done_asic:
 /* magic character sequence that trails an image */
 #define IMAGE_TRAIL_MAGIC "egamiAPO"
 
+/* EPROM file types */
+#define HFI1_EFT_PLATFORM_CONFIG 2
+
+/* segment size - 128 KiB */
+#define SEG_SIZE (128 * 1024)
+
+struct hfi1_eprom_footer {
+	u32 oprom_size;		/* size of the oprom, in bytes */
+	u16 num_table_entries;
+	u16 version;		/* version of this footer */
+	u32 magic;		/* must be last */
+};
+
+struct hfi1_eprom_table_entry {
+	u32 type;		/* file type */
+	u32 offset;		/* file offset from start of EPROM */
+	u32 size;		/* file size, in bytes */
+};
+
+/*
+ * Calculate the max number of table entries that will fit within a directory
+ * buffer of size 'dir_size'.
+ */
+#define MAX_TABLE_ENTRIES(dir_size) \
+	(((dir_size) - sizeof(struct hfi1_eprom_footer)) / \
+		sizeof(struct hfi1_eprom_table_entry))
+
+#define DIRECTORY_SIZE(n) (sizeof(struct hfi1_eprom_footer) + \
+	(sizeof(struct hfi1_eprom_table_entry) * (n)))
+
+#define MAGIC4(a, b, c, d) ((d) << 24 | (c) << 16 | (b) << 8 | (a))
+#define FOOTER_MAGIC MAGIC4('e', 'p', 'r', 'm')
+#define FOOTER_VERSION 1
+
 /*
  * Read all of partition 1.  The actual file is at the front.  Adjust
  * the returned size if a trailing image magic is found.
@@ -242,6 +276,167 @@ static int read_partition_platform_config(struct hfi1_devdata *dd, void **data,
 }
 
 /*
+ * The segment magic has been checked.  There is a footer and table of
+ * contents present.
+ *
+ * directory is a u32 aligned buffer of size EP_PAGE_SIZE.
+ */
+static int read_segment_platform_config(struct hfi1_devdata *dd,
+					void *directory, void **data, u32 *size)
+{
+	struct hfi1_eprom_footer *footer;
+	struct hfi1_eprom_table_entry *table;
+	struct hfi1_eprom_table_entry *entry;
+	void *buffer = NULL;
+	void *table_buffer = NULL;
+	int ret, i;
+	u32 directory_size;
+	u32 seg_base, seg_offset;
+	u32 bytes_available, ncopied, to_copy;
+
+	/* the footer is at the end of the directory */
+	footer = (struct hfi1_eprom_footer *)
+			(directory + EP_PAGE_SIZE - sizeof(*footer));
+
+	/* make sure the structure version is supported */
+	if (footer->version != FOOTER_VERSION)
+		return -EINVAL;
+
+	/* oprom size cannot be larger than a segment */
+	if (footer->oprom_size >= SEG_SIZE)
+		return -EINVAL;
+
+	/* the file table must fit in a segment with the oprom */
+	if (footer->num_table_entries >
+			MAX_TABLE_ENTRIES(SEG_SIZE - footer->oprom_size))
+		return -EINVAL;
+
+	/* find the file table start, which precedes the footer */
+	directory_size = DIRECTORY_SIZE(footer->num_table_entries);
+	if (directory_size <= EP_PAGE_SIZE) {
+		/* the file table fits into the directory buffer handed in */
+		table = (struct hfi1_eprom_table_entry *)
+				(directory + EP_PAGE_SIZE - directory_size);
+	} else {
+		/* need to allocate and read more */
+		table_buffer = kmalloc(directory_size, GFP_KERNEL);
+		if (!table_buffer)
+			return -ENOMEM;
+		ret = read_length(dd, SEG_SIZE - directory_size,
+				  directory_size, table_buffer);
+		if (ret)
+			goto done;
+		table = table_buffer;
+	}
+
+	/* look for the platform configuration file in the table */
+	for (entry = NULL, i = 0; i < footer->num_table_entries; i++) {
+		if (table[i].type == HFI1_EFT_PLATFORM_CONFIG) {
+			entry = &table[i];
+			break;
+		}
+	}
+	if (!entry) {
+		ret = -ENOENT;
+		goto done;
+	}
+
+	/*
+	 * Sanity check on the configuration file size - it should never
+	 * be larger than 4 KiB.
+	 */
+	if (entry->size > (4 * 1024)) {
+		dd_dev_err(dd, "Bad configuration file size 0x%x\n",
+			   entry->size);
+		ret = -EINVAL;
+		goto done;
+	}
+
+	/* check for bogus offset and size that wrap when added together */
+	if (entry->offset + entry->size < entry->offset) {
+		dd_dev_err(dd,
+			   "Bad configuration file start + size 0x%x+0x%x\n",
+			   entry->offset, entry->size);
+		ret = -EINVAL;
+		goto done;
+	}
+
+	/* allocate the buffer to return */
+	buffer = kmalloc(entry->size, GFP_KERNEL);
+	if (!buffer) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	/*
+	 * Extract the file by looping over segments until it is fully read.
+	 */
+	seg_offset = entry->offset % SEG_SIZE;
+	seg_base = entry->offset - seg_offset;
+	ncopied = 0;
+	while (ncopied < entry->size) {
+		/* calculate data bytes available in this segment */
+
+		/* start with the bytes from the current offset to the end */
+		bytes_available = SEG_SIZE - seg_offset;
+		/* subtract off footer and table from segment 0 */
+		if (seg_base == 0) {
+			/*
+			 * Sanity check: should not have a starting point
+			 * at or within the directory.
+			 */
+			if (bytes_available <= directory_size) {
+				dd_dev_err(dd,
+					   "Bad configuration file - offset 0x%x within footer+table\n",
+					   entry->offset);
+				ret = -EINVAL;
+				goto done;
+			}
+			bytes_available -= directory_size;
+		}
+
+		/* calculate bytes wanted */
+		to_copy = entry->size - ncopied;
+
+		/* max out at the available bytes in this segment */
+		if (to_copy > bytes_available)
+			to_copy = bytes_available;
+
+		/*
+		 * Read from the EPROM.
+		 *
+		 * The sanity check for entry->offset is done in read_length().
+		 * The EPROM offset is validated against what the hardware
+		 * addressing supports.  In addition, if the offset is larger
+		 * than the actual EPROM, it silently wraps.  It will work
+		 * fine, though the reader may not get what they expected
+		 * from the EPROM.
+		 */
+		ret = read_length(dd, seg_base + seg_offset, to_copy,
+				  buffer + ncopied);
+		if (ret)
+			goto done;
+
+		ncopied += to_copy;
+
+		/* set up for next segment */
+		seg_offset = footer->oprom_size;
+		seg_base += SEG_SIZE;
+	}
+
+	/* success */
+	ret = 0;
+	*data = buffer;
+	*size = entry->size;
+
+done:
+	kfree(table_buffer);
+	if (ret)
+		kfree(buffer);
+	return ret;
+}
+
+/*
  * Read the platform configuration file from the EPROM.
  *
  * On success, an allocated buffer containing the data and its size are
@@ -253,6 +448,7 @@ static int read_partition_platform_config(struct hfi1_devdata *dd, void **data,
  *   -EBUSY   - not able to acquire access to the EPROM
  *   -ENOENT  - no recognizable file written
  *   -ENOMEM  - buffer could not be allocated
+ *   -EINVAL  - invalid EPROM contentents found
  */
 int eprom_read_platform_config(struct hfi1_devdata *dd, void **data, u32 *size)
 {
@@ -266,21 +462,20 @@ int eprom_read_platform_config(struct hfi1_devdata *dd, void **data, u32 *size)
 	if (ret)
 		return -EBUSY;
 
-	/* read the last page of P0 for the EPROM format magic */
-	ret = read_length(dd, P1_START - EP_PAGE_SIZE, EP_PAGE_SIZE, directory);
+	/* read the last page of the segment for the EPROM format magic */
+	ret = read_length(dd, SEG_SIZE - EP_PAGE_SIZE, EP_PAGE_SIZE, directory);
 	if (ret)
 		goto done;
 
-	/* last dword of P0 contains a magic indicator */
-	if (directory[EP_PAGE_DWORDS - 1] == 0) {
+	/* last dword of the segment contains a magic value */
+	if (directory[EP_PAGE_DWORDS - 1] == FOOTER_MAGIC) {
+		/* segment format */
+		ret = read_segment_platform_config(dd, directory, data, size);
+	} else {
 		/* partition format */
 		ret = read_partition_platform_config(dd, data, size);
-		goto done;
 	}
 
-	/* nothing recognized */
-	ret = -ENOENT;
-
 done:
 	release_chip_resource(dd, CR_EPROM);
 	return ret;
diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c
index 13db8eb4f4ec..0dd50cdb039a 100644
--- a/drivers/infiniband/hw/hfi1/firmware.c
+++ b/drivers/infiniband/hw/hfi1/firmware.c
@@ -239,6 +239,16 @@ static const u8 all_fabric_serdes_broadcast = 0xe1;
 const u8 pcie_serdes_broadcast[2] = { 0xe2, 0xe3 };
 static const u8 all_pcie_serdes_broadcast = 0xe0;
 
+static const u32 platform_config_table_limits[PLATFORM_CONFIG_TABLE_MAX] = {
+	0,
+	SYSTEM_TABLE_MAX,
+	PORT_TABLE_MAX,
+	RX_PRESET_TABLE_MAX,
+	TX_PRESET_TABLE_MAX,
+	QSFP_ATTEN_TABLE_MAX,
+	VARIABLE_SETTINGS_TABLE_MAX
+};
+
 /* forwards */
 static void dispose_one_firmware(struct firmware_details *fdet);
 static int load_fabric_serdes_firmware(struct hfi1_devdata *dd,
@@ -263,11 +273,13 @@ static int __read_8051_data(struct hfi1_devdata *dd, u32 addr, u64 *result)
 	u64 reg;
 	int count;
 
-	/* start the read at the given address */
-	reg = ((addr & DC_DC8051_CFG_RAM_ACCESS_CTRL_ADDRESS_MASK)
-			<< DC_DC8051_CFG_RAM_ACCESS_CTRL_ADDRESS_SHIFT)
-		| DC_DC8051_CFG_RAM_ACCESS_CTRL_READ_ENA_SMASK;
+	/* step 1: set the address, clear enable */
+	reg = (addr & DC_DC8051_CFG_RAM_ACCESS_CTRL_ADDRESS_MASK)
+			<< DC_DC8051_CFG_RAM_ACCESS_CTRL_ADDRESS_SHIFT;
 	write_csr(dd, DC_DC8051_CFG_RAM_ACCESS_CTRL, reg);
+	/* step 2: enable */
+	write_csr(dd, DC_DC8051_CFG_RAM_ACCESS_CTRL,
+		  reg | DC_DC8051_CFG_RAM_ACCESS_CTRL_READ_ENA_SMASK);
 
 	/* wait until ACCESS_COMPLETED is set */
 	count = 0;
@@ -707,6 +719,9 @@ static int obtain_firmware(struct hfi1_devdata *dd)
 				       &dd->pcidev->dev);
 		if (err) {
 			platform_config = NULL;
+			dd_dev_err(dd,
+				   "%s: No default platform config file found\n",
+				   __func__);
 			goto done;
 		}
 		dd->platform_config.data = platform_config->data;
@@ -1761,8 +1776,17 @@ int parse_platform_config(struct hfi1_devdata *dd)
 	u32 record_idx = 0, table_type = 0, table_length_dwords = 0;
 	int ret = -EINVAL; /* assume failure */
 
+	/*
+	 * For integrated devices that did not fall back to the default file,
+	 * the SI tuning information for active channels is acquired from the
+	 * scratch register bitmap, thus there is no platform config to parse.
+	 * Skip parsing in these situations.
+	 */
+	if (is_integrated(dd) && !platform_config_load)
+		return 0;
+
 	if (!dd->platform_config.data) {
-		dd_dev_info(dd, "%s: Missing config file\n", __func__);
+		dd_dev_err(dd, "%s: Missing config file\n", __func__);
 		goto bail;
 	}
 	ptr = (u32 *)dd->platform_config.data;
@@ -1770,7 +1794,7 @@ int parse_platform_config(struct hfi1_devdata *dd)
 	magic_num = *ptr;
 	ptr++;
 	if (magic_num != PLATFORM_CONFIG_MAGIC_NUM) {
-		dd_dev_info(dd, "%s: Bad config file\n", __func__);
+		dd_dev_err(dd, "%s: Bad config file\n", __func__);
 		goto bail;
 	}
 
@@ -1797,9 +1821,9 @@ int parse_platform_config(struct hfi1_devdata *dd)
 		header1 = *ptr;
 		header2 = *(ptr + 1);
 		if (header1 != ~header2) {
-			dd_dev_info(dd, "%s: Failed validation at offset %ld\n",
-				    __func__, (ptr - (u32 *)
-					       dd->platform_config.data));
+			dd_dev_err(dd, "%s: Failed validation at offset %ld\n",
+				   __func__, (ptr - (u32 *)
+					      dd->platform_config.data));
 			goto bail;
 		}
 
@@ -1841,11 +1865,11 @@ int parse_platform_config(struct hfi1_devdata *dd)
 							table_length_dwords;
 				break;
 			default:
-				dd_dev_info(dd,
-					    "%s: Unknown data table %d, offset %ld\n",
-					    __func__, table_type,
-					    (ptr - (u32 *)
-					     dd->platform_config.data));
+				dd_dev_err(dd,
+					   "%s: Unknown data table %d, offset %ld\n",
+					   __func__, table_type,
+					   (ptr - (u32 *)
+					    dd->platform_config.data));
 				goto bail; /* We don't trust this file now */
 			}
 			pcfgcache->config_tables[table_type].table = ptr;
@@ -1865,11 +1889,11 @@ int parse_platform_config(struct hfi1_devdata *dd)
 			case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
 				break;
 			default:
-				dd_dev_info(dd,
-					    "%s: Unknown meta table %d, offset %ld\n",
-					    __func__, table_type,
-					    (ptr -
-					     (u32 *)dd->platform_config.data));
+				dd_dev_err(dd,
+					   "%s: Unknown meta table %d, offset %ld\n",
+					   __func__, table_type,
+					   (ptr -
+					    (u32 *)dd->platform_config.data));
 				goto bail; /* We don't trust this file now */
 			}
 			pcfgcache->config_tables[table_type].table_metadata =
@@ -1884,10 +1908,9 @@ int parse_platform_config(struct hfi1_devdata *dd)
 		/* Jump the table */
 		ptr += table_length_dwords;
 		if (crc != *ptr) {
-			dd_dev_info(dd, "%s: Failed CRC check at offset %ld\n",
-				    __func__, (ptr -
-					       (u32 *)
-					       dd->platform_config.data));
+			dd_dev_err(dd, "%s: Failed CRC check at offset %ld\n",
+				   __func__, (ptr -
+				   (u32 *)dd->platform_config.data));
 			goto bail;
 		}
 		/* Jump the CRC DWORD */
@@ -1901,6 +1924,84 @@ bail:
 	return ret;
 }
 
+static void get_integrated_platform_config_field(
+		struct hfi1_devdata *dd,
+		enum platform_config_table_type_encoding table_type,
+		int field_index, u32 *data)
+{
+	struct hfi1_pportdata *ppd = dd->pport;
+	u8 *cache = ppd->qsfp_info.cache;
+	u32 tx_preset = 0;
+
+	switch (table_type) {
+	case PLATFORM_CONFIG_SYSTEM_TABLE:
+		if (field_index == SYSTEM_TABLE_QSFP_POWER_CLASS_MAX)
+			*data = ppd->max_power_class;
+		else if (field_index == SYSTEM_TABLE_QSFP_ATTENUATION_DEFAULT_25G)
+			*data = ppd->default_atten;
+		break;
+	case PLATFORM_CONFIG_PORT_TABLE:
+		if (field_index == PORT_TABLE_PORT_TYPE)
+			*data = ppd->port_type;
+		else if (field_index == PORT_TABLE_LOCAL_ATTEN_25G)
+			*data = ppd->local_atten;
+		else if (field_index == PORT_TABLE_REMOTE_ATTEN_25G)
+			*data = ppd->remote_atten;
+		break;
+	case PLATFORM_CONFIG_RX_PRESET_TABLE:
+		if (field_index == RX_PRESET_TABLE_QSFP_RX_CDR_APPLY)
+			*data = (ppd->rx_preset & QSFP_RX_CDR_APPLY_SMASK) >>
+				QSFP_RX_CDR_APPLY_SHIFT;
+		else if (field_index == RX_PRESET_TABLE_QSFP_RX_EMP_APPLY)
+			*data = (ppd->rx_preset & QSFP_RX_EMP_APPLY_SMASK) >>
+				QSFP_RX_EMP_APPLY_SHIFT;
+		else if (field_index == RX_PRESET_TABLE_QSFP_RX_AMP_APPLY)
+			*data = (ppd->rx_preset & QSFP_RX_AMP_APPLY_SMASK) >>
+				QSFP_RX_AMP_APPLY_SHIFT;
+		else if (field_index == RX_PRESET_TABLE_QSFP_RX_CDR)
+			*data = (ppd->rx_preset & QSFP_RX_CDR_SMASK) >>
+				QSFP_RX_CDR_SHIFT;
+		else if (field_index == RX_PRESET_TABLE_QSFP_RX_EMP)
+			*data = (ppd->rx_preset & QSFP_RX_EMP_SMASK) >>
+				QSFP_RX_EMP_SHIFT;
+		else if (field_index == RX_PRESET_TABLE_QSFP_RX_AMP)
+			*data = (ppd->rx_preset & QSFP_RX_AMP_SMASK) >>
+				QSFP_RX_AMP_SHIFT;
+		break;
+	case PLATFORM_CONFIG_TX_PRESET_TABLE:
+		if (cache[QSFP_EQ_INFO_OFFS] & 0x4)
+			tx_preset = ppd->tx_preset_eq;
+		else
+			tx_preset = ppd->tx_preset_noeq;
+		if (field_index == TX_PRESET_TABLE_PRECUR)
+			*data = (tx_preset & TX_PRECUR_SMASK) >>
+				TX_PRECUR_SHIFT;
+		else if (field_index == TX_PRESET_TABLE_ATTN)
+			*data = (tx_preset & TX_ATTN_SMASK) >>
+				TX_ATTN_SHIFT;
+		else if (field_index == TX_PRESET_TABLE_POSTCUR)
+			*data = (tx_preset & TX_POSTCUR_SMASK) >>
+				TX_POSTCUR_SHIFT;
+		else if (field_index == TX_PRESET_TABLE_QSFP_TX_CDR_APPLY)
+			*data = (tx_preset & QSFP_TX_CDR_APPLY_SMASK) >>
+				QSFP_TX_CDR_APPLY_SHIFT;
+		else if (field_index == TX_PRESET_TABLE_QSFP_TX_EQ_APPLY)
+			*data = (tx_preset & QSFP_TX_EQ_APPLY_SMASK) >>
+				QSFP_TX_EQ_APPLY_SHIFT;
+		else if (field_index == TX_PRESET_TABLE_QSFP_TX_CDR)
+			*data = (tx_preset & QSFP_TX_CDR_SMASK) >>
+				QSFP_TX_CDR_SHIFT;
+		else if (field_index == TX_PRESET_TABLE_QSFP_TX_EQ)
+			*data = (tx_preset & QSFP_TX_EQ_SMASK) >>
+				QSFP_TX_EQ_SHIFT;
+		break;
+	case PLATFORM_CONFIG_QSFP_ATTEN_TABLE:
+	case PLATFORM_CONFIG_VARIABLE_SETTINGS_TABLE:
+	default:
+		break;
+	}
+}
+
 static int get_platform_fw_field_metadata(struct hfi1_devdata *dd, int table,
 					  int field, u32 *field_len_bits,
 					  u32 *field_start_bits)
@@ -1976,6 +2077,15 @@ int get_platform_config_field(struct hfi1_devdata *dd,
 	else
 		return -EINVAL;
 
+	if (is_integrated(dd) && !platform_config_load) {
+		/*
+		 * Use saved configuration from ppd for integrated platforms
+		 */
+		get_integrated_platform_config_field(dd, table_type,
+						     field_index, data);
+		return 0;
+	}
+
 	ret = get_platform_fw_field_metadata(dd, table_type, field_index,
 					     &field_len_bits,
 					     &field_start_bits);
diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h
index cc87fd4e534b..751a0fb29fa5 100644
--- a/drivers/infiniband/hw/hfi1/hfi.h
+++ b/drivers/infiniband/hw/hfi1/hfi.h
@@ -492,6 +492,9 @@ struct rvt_sge_state;
 #define HFI1_MIN_VLS_SUPPORTED 1
 #define HFI1_MAX_VLS_SUPPORTED 8
 
+#define HFI1_GUIDS_PER_PORT  5
+#define HFI1_PORT_GUID_INDEX 0
+
 static inline void incr_cntr64(u64 *cntr)
 {
 	if (*cntr < (u64)-1LL)
@@ -559,11 +562,20 @@ struct hfi1_pportdata {
 	struct kobject vl2mtu_kobj;
 
 	/* PHY support */
-	u32 port_type;
 	struct qsfp_data qsfp_info;
+	/* Values for SI tuning of SerDes */
+	u32 port_type;
+	u32 tx_preset_eq;
+	u32 tx_preset_noeq;
+	u32 rx_preset;
+	u8  local_atten;
+	u8  remote_atten;
+	u8  default_atten;
+	u8  max_power_class;
+
+	/* GUIDs for this interface, in host order, guids[0] is a port guid */
+	u64 guids[HFI1_GUIDS_PER_PORT];
 
-	/* GUID for this interface, in host order */
-	u64 guid;
 	/* GUID for peer interface, in host order */
 	u64 neighbor_guid;
 
@@ -826,32 +838,29 @@ struct hfi1_devdata {
 	u8 __iomem *kregend;
 	/* physical address of chip for io_remap, etc. */
 	resource_size_t physaddr;
-	/* receive context data */
-	struct hfi1_ctxtdata **rcd;
+	/* Per VL data. Enough for all VLs but not all elements are set/used. */
+	struct per_vl_data vld[PER_VL_SEND_CONTEXTS];
 	/* send context data */
 	struct send_context_info *send_contexts;
 	/* map hardware send contexts to software index */
 	u8 *hw_to_sw;
 	/* spinlock for allocating and releasing send context resources */
 	spinlock_t sc_lock;
-	/* Per VL data. Enough for all VLs but not all elements are set/used. */
-	struct per_vl_data vld[PER_VL_SEND_CONTEXTS];
 	/* lock for pio_map */
 	spinlock_t pio_map_lock;
+	/* Send Context initialization lock. */
+	spinlock_t sc_init_lock;
+	/* lock for sdma_map */
+	spinlock_t                          sde_map_lock;
 	/* array of kernel send contexts */
 	struct send_context **kernel_send_context;
 	/* array of vl maps */
 	struct pio_vl_map __rcu *pio_map;
-	/* seqlock for sc2vl */
-	seqlock_t sc2vl_lock;
-	u64 sc2vl[4];
-	/* Send Context initialization lock. */
-	spinlock_t sc_init_lock;
+	/* default flags to last descriptor */
+	u64 default_desc1;
 
 	/* fields common to all SDMA engines */
 
-	/* default flags to last descriptor */
-	u64 default_desc1;
 	volatile __le64                    *sdma_heads_dma; /* DMA'ed by chip */
 	dma_addr_t                          sdma_heads_phys;
 	void                               *sdma_pad_dma; /* DMA'ed by chip */
@@ -862,8 +871,6 @@ struct hfi1_devdata {
 	u32                                 chip_sdma_engines;
 	/* num used */
 	u32                                 num_sdma;
-	/* lock for sdma_map */
-	spinlock_t                          sde_map_lock;
 	/* array of engines sized by num_sdma */
 	struct sdma_engine                 *per_sdma;
 	/* array of vl maps */
@@ -872,14 +879,11 @@ struct hfi1_devdata {
 	wait_queue_head_t		  sdma_unfreeze_wq;
 	atomic_t			  sdma_unfreeze_count;
 
+	u32 lcb_access_count;		/* count of LCB users */
+
 	/* common data between shared ASIC HFIs in this OS */
 	struct hfi1_asic_data *asic_data;
 
-	/* hfi1_pportdata, points to array of (physical) port-specific
-	 * data structs, indexed by pidx (0..n-1)
-	 */
-	struct hfi1_pportdata *pport;
-
 	/* mem-mapped pointer to base of PIO buffers */
 	void __iomem *piobase;
 	/*
@@ -896,20 +900,13 @@ struct hfi1_devdata {
 	/* send context numbers and sizes for each type */
 	struct sc_config_sizes sc_sizes[SC_MAX];
 
-	u32 lcb_access_count;		/* count of LCB users */
-
 	char *boardname; /* human readable board info */
 
-	/* device (not port) flags, basically device capabilities */
-	u32 flags;
-
 	/* reset value */
 	u64 z_int_counter;
 	u64 z_rcv_limit;
 	u64 z_send_schedule;
-	/* percpu int_counter */
-	u64 __percpu *int_counter;
-	u64 __percpu *rcv_limit;
+
 	u64 __percpu *send_schedule;
 	/* number of receive contexts in use by the driver */
 	u32 num_rcv_contexts;
@@ -924,6 +921,7 @@ struct hfi1_devdata {
 	/* base receive interrupt timeout, in CSR units */
 	u32 rcv_intr_timeout_csr;
 
+	u32 freezelen; /* max length of freezemsg */
 	u64 __iomem *egrtidbase;
 	spinlock_t sendctrl_lock; /* protect changes to SendCtrl */
 	spinlock_t rcvctrl_lock; /* protect changes to RcvCtrl */
@@ -945,7 +943,6 @@ struct hfi1_devdata {
 	 * IB link status cheaply
 	 */
 	struct hfi1_status *status;
-	u32 freezelen; /* max length of freezemsg */
 
 	/* revision register shadow */
 	u64 revision;
@@ -973,6 +970,8 @@ struct hfi1_devdata {
 	u16 rcvegrbufsize_shift;
 	/* both sides of the PCIe link are gen3 capable */
 	u8 link_gen3_capable;
+	/* default link down value (poll/sleep) */
+	u8 link_default;
 	/* localbus width (1, 2,4,8,16,32) from config space  */
 	u32 lbus_width;
 	/* localbus speed in MHz */
@@ -1008,8 +1007,6 @@ struct hfi1_devdata {
 	u8 hfi1_id;
 	/* implementation code */
 	u8 icode;
-	/* default link down value (poll/sleep) */
-	u8 link_default;
 	/* vAU of this device */
 	u8 vau;
 	/* vCU of this device */
@@ -1020,27 +1017,17 @@ struct hfi1_devdata {
 	u16 vl15_init;
 
 	/* Misc small ints */
-	/* Number of physical ports available */
-	u8 num_pports;
-	/* Lowest context number which can be used by user processes */
-	u8 first_user_ctxt;
 	u8 n_krcv_queues;
 	u8 qos_shift;
-	u8 qpn_mask;
 
-	u16 rhf_offset; /* offset of RHF within receive header entry */
 	u16 irev;	/* implementation revision */
 	u16 dc8051_ver; /* 8051 firmware version */
 
+	spinlock_t hfi1_diag_trans_lock; /* protect diag observer ops */
 	struct platform_config platform_config;
 	struct platform_config_cache pcfg_cache;
 
 	struct diag_client *diag_client;
-	spinlock_t hfi1_diag_trans_lock; /* protect diag observer ops */
-
-	u8 psxmitwait_supported;
-	/* cycle length of PS* counters in HW (in picoseconds) */
-	u16 psxmitwait_check_rate;
 
 	/* MSI-X information */
 	struct hfi1_msix_entry *msix_entries;
@@ -1055,6 +1042,9 @@ struct hfi1_devdata {
 
 	struct rcv_array_data rcv_entries;
 
+	/* cycle length of PS* counters in HW (in picoseconds) */
+	u16 psxmitwait_check_rate;
+
 	/*
 	 * 64 bit synthetic counters
 	 */
@@ -1085,11 +1075,11 @@ struct hfi1_devdata {
 	struct err_info_rcvport err_info_rcvport;
 	struct err_info_constraint err_info_rcv_constraint;
 	struct err_info_constraint err_info_xmit_constraint;
-	u8 err_info_uncorrectable;
-	u8 err_info_fmconfig;
 
 	atomic_t drop_packet;
 	u8 do_drop;
+	u8 err_info_uncorrectable;
+	u8 err_info_fmconfig;
 
 	/*
 	 * Software counters for the status bits defined by the
@@ -1112,40 +1102,60 @@ struct hfi1_devdata {
 	u64 sw_cce_err_status_aggregate;
 	/* Software counter that aggregates all bypass packet rcv errors */
 	u64 sw_rcv_bypass_packet_errors;
-	/* receive interrupt functions */
-	rhf_rcv_function_ptr *rhf_rcv_function_map;
+	/* receive interrupt function */
 	rhf_rcv_function_ptr normal_rhf_rcv_functions[8];
 
+	/* Save the enabled LCB error bits */
+	u64 lcb_err_en;
+
 	/*
 	 * Capability to have different send engines simply by changing a
 	 * pointer value.
 	 */
-	send_routine process_pio_send;
+	send_routine process_pio_send ____cacheline_aligned_in_smp;
 	send_routine process_dma_send;
 	void (*pio_inline_send)(struct hfi1_devdata *dd, struct pio_buf *pbuf,
 				u64 pbc, const void *from, size_t count);
+	/* hfi1_pportdata, points to array of (physical) port-specific
+	 * data structs, indexed by pidx (0..n-1)
+	 */
+	struct hfi1_pportdata *pport;
+	/* receive context data */
+	struct hfi1_ctxtdata **rcd;
+	u64 __percpu *int_counter;
+	/* device (not port) flags, basically device capabilities */
+	u16 flags;
+	/* Number of physical ports available */
+	u8 num_pports;
+	/* Lowest context number which can be used by user processes */
+	u8 first_user_ctxt;
+	/* adding a new field here would make it part of this cacheline */
+
+	/* seqlock for sc2vl */
+	seqlock_t sc2vl_lock ____cacheline_aligned_in_smp;
+	u64 sc2vl[4];
+	/* receive interrupt functions */
+	rhf_rcv_function_ptr *rhf_rcv_function_map;
+	u64 __percpu *rcv_limit;
+	u16 rhf_offset; /* offset of RHF within receive header entry */
+	/* adding a new field here would make it part of this cacheline */
 
 	/* OUI comes from the HW. Used everywhere as 3 separate bytes. */
 	u8 oui1;
 	u8 oui2;
 	u8 oui3;
+	u8 dc_shutdown;
+
 	/* Timer and counter used to detect RcvBufOvflCnt changes */
 	struct timer_list rcverr_timer;
-	u32 rcv_ovfl_cnt;
 
 	wait_queue_head_t event_queue;
 
-	/* Save the enabled LCB error bits */
-	u64 lcb_err_en;
-	u8 dc_shutdown;
-
 	/* receive context tail dummy address */
 	__le64 *rcvhdrtail_dummy_kvaddr;
 	dma_addr_t rcvhdrtail_dummy_dma;
 
-	bool eprom_available;	/* true if EPROM is available for this device */
-	bool aspm_supported;	/* Does HW support ASPM */
-	bool aspm_enabled;	/* ASPM state: enabled/disabled */
+	u32 rcv_ovfl_cnt;
 	/* Serialize ASPM enable/disable between multiple verbs contexts */
 	spinlock_t aspm_lock;
 	/* Number of verbs contexts which have disabled ASPM */
@@ -1155,8 +1165,11 @@ struct hfi1_devdata {
 	/* Used to wait for outstanding user space clients before dev removal */
 	struct completion user_comp;
 
-	struct hfi1_affinity *affinity;
+	bool eprom_available;	/* true if EPROM is available for this device */
+	bool aspm_supported;	/* Does HW support ASPM */
+	bool aspm_enabled;	/* ASPM state: enabled/disabled */
 	struct rhashtable sdma_rht;
+
 	struct kobject kobj;
 };
 
@@ -1604,6 +1617,17 @@ static inline u16 hfi1_get_pkey(struct hfi1_ibport *ibp, unsigned index)
 }
 
 /*
+ * Return the indexed GUID from the port GUIDs table.
+ */
+static inline __be64 get_sguid(struct hfi1_ibport *ibp, unsigned int index)
+{
+	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+
+	WARN_ON(index >= HFI1_GUIDS_PER_PORT);
+	return cpu_to_be64(ppd->guids[index]);
+}
+
+/*
  * Called by readers of cc_state only, must call under rcu_read_lock().
  */
 static inline struct cc_state *get_cc_state(struct hfi1_pportdata *ppd)
@@ -1982,6 +2006,12 @@ static inline u32 qsfp_resource(struct hfi1_devdata *dd)
 	return i2c_target(dd->hfi1_id);
 }
 
+/* Is this device integrated or discrete? */
+static inline bool is_integrated(struct hfi1_devdata *dd)
+{
+	return dd->pcidev->device == PCI_DEVICE_ID_INTEL1;
+}
+
 int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp);
 
 #define DD_DEV_ENTRY(dd)       __string(dev, dev_name(&(dd)->pcidev->dev))
diff --git a/drivers/infiniband/hw/hfi1/iowait.h b/drivers/infiniband/hw/hfi1/iowait.h
index 2ec6ef38d389..d9740ddea6f1 100644
--- a/drivers/infiniband/hw/hfi1/iowait.h
+++ b/drivers/infiniband/hw/hfi1/iowait.h
@@ -64,6 +64,7 @@ struct sdma_engine;
 /**
  * struct iowait - linkage for delayed progress/waiting
  * @list: used to add/insert into QP/PQ wait lists
+ * @lock: uses to record the list head lock
  * @tx_head: overflow list of sdma_txreq's
  * @sleep: no space callback
  * @wakeup: space callback wakeup
@@ -91,6 +92,11 @@ struct sdma_engine;
  * so sleeping is not allowed.
  *
  * The wait_dma member along with the iow
+ *
+ * The lock field is used by waiters to record
+ * the seqlock_t that guards the list head.
+ * Waiters explicity know that, but the destroy
+ * code that unwaits QPs does not.
  */
 
 struct iowait {
@@ -103,6 +109,7 @@ struct iowait {
 		unsigned seq);
 	void (*wakeup)(struct iowait *wait, int reason);
 	void (*sdma_drained)(struct iowait *wait);
+	seqlock_t *lock;
 	struct work_struct iowork;
 	wait_queue_head_t wait_dma;
 	wait_queue_head_t wait_pio;
@@ -141,6 +148,7 @@ static inline void iowait_init(
 	void (*sdma_drained)(struct iowait *wait))
 {
 	wait->count = 0;
+	wait->lock = NULL;
 	INIT_LIST_HEAD(&wait->list);
 	INIT_LIST_HEAD(&wait->tx_head);
 	INIT_WORK(&wait->iowork, func);
diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c
index 9487c9bb8920..6e595afca24c 100644
--- a/drivers/infiniband/hw/hfi1/mad.c
+++ b/drivers/infiniband/hw/hfi1/mad.c
@@ -128,7 +128,7 @@ static void send_trap(struct hfi1_ibport *ibp, void *data, unsigned len)
 	smp = send_buf->mad;
 	smp->base_version = OPA_MGMT_BASE_VERSION;
 	smp->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
-	smp->class_version = OPA_SMI_CLASS_VERSION;
+	smp->class_version = OPA_SM_CLASS_VERSION;
 	smp->method = IB_MGMT_METHOD_TRAP;
 	ibp->rvp.tid++;
 	smp->tid = cpu_to_be64(ibp->rvp.tid);
@@ -336,20 +336,20 @@ static int __subn_get_opa_nodeinfo(struct opa_smp *smp, u32 am, u8 *data,
 	ni = (struct opa_node_info *)data;
 
 	/* GUID 0 is illegal */
-	if (am || pidx >= dd->num_pports || dd->pport[pidx].guid == 0) {
+	if (am || pidx >= dd->num_pports || ibdev->node_guid == 0 ||
+	    get_sguid(to_iport(ibdev, port), HFI1_PORT_GUID_INDEX) == 0) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
 
-	ni->port_guid = cpu_to_be64(dd->pport[pidx].guid);
+	ni->port_guid = get_sguid(to_iport(ibdev, port), HFI1_PORT_GUID_INDEX);
 	ni->base_version = OPA_MGMT_BASE_VERSION;
-	ni->class_version = OPA_SMI_CLASS_VERSION;
+	ni->class_version = OPA_SM_CLASS_VERSION;
 	ni->node_type = 1;     /* channel adapter */
 	ni->num_ports = ibdev->phys_port_cnt;
 	/* This is already in network order */
 	ni->system_image_guid = ib_hfi1_sys_image_guid;
-	/* Use first-port GUID as node */
-	ni->node_guid = cpu_to_be64(dd->pport->guid);
+	ni->node_guid = ibdev->node_guid;
 	ni->partition_cap = cpu_to_be16(hfi1_get_npkeys(dd));
 	ni->device_id = cpu_to_be16(dd->pcidev->device);
 	ni->revision = cpu_to_be32(dd->minrev);
@@ -373,19 +373,20 @@ static int subn_get_nodeinfo(struct ib_smp *smp, struct ib_device *ibdev,
 
 	/* GUID 0 is illegal */
 	if (smp->attr_mod || pidx >= dd->num_pports ||
-	    dd->pport[pidx].guid == 0)
+	    ibdev->node_guid == 0 ||
+	    get_sguid(to_iport(ibdev, port), HFI1_PORT_GUID_INDEX) == 0) {
 		smp->status |= IB_SMP_INVALID_FIELD;
-	else
-		nip->port_guid = cpu_to_be64(dd->pport[pidx].guid);
+		return reply((struct ib_mad_hdr *)smp);
+	}
 
+	nip->port_guid = get_sguid(to_iport(ibdev, port), HFI1_PORT_GUID_INDEX);
 	nip->base_version = OPA_MGMT_BASE_VERSION;
-	nip->class_version = OPA_SMI_CLASS_VERSION;
+	nip->class_version = OPA_SM_CLASS_VERSION;
 	nip->node_type = 1;     /* channel adapter */
 	nip->num_ports = ibdev->phys_port_cnt;
 	/* This is already in network order */
 	nip->sys_guid = ib_hfi1_sys_image_guid;
-	 /* Use first-port GUID as node */
-	nip->node_guid = cpu_to_be64(dd->pport->guid);
+	nip->node_guid = ibdev->node_guid;
 	nip->partition_cap = cpu_to_be16(hfi1_get_npkeys(dd));
 	nip->device_id = cpu_to_be16(dd->pcidev->device);
 	nip->revision = cpu_to_be32(dd->minrev);
@@ -2302,7 +2303,7 @@ static int pma_get_opa_classportinfo(struct opa_pma_mad *pmp,
 		pmp->mad_hdr.status |= IB_SMP_INVALID_FIELD;
 
 	p->base_version = OPA_MGMT_BASE_VERSION;
-	p->class_version = OPA_SMI_CLASS_VERSION;
+	p->class_version = OPA_SM_CLASS_VERSION;
 	/*
 	 * Expected response time is 4.096 usec. * 2^18 == 1.073741824 sec.
 	 */
@@ -4022,7 +4023,7 @@ static int process_subn_opa(struct ib_device *ibdev, int mad_flags,
 
 	am = be32_to_cpu(smp->attr_mod);
 	attr_id = smp->attr_id;
-	if (smp->class_version != OPA_SMI_CLASS_VERSION) {
+	if (smp->class_version != OPA_SM_CLASS_VERSION) {
 		smp->status |= IB_SMP_UNSUP_VERSION;
 		ret = reply((struct ib_mad_hdr *)smp);
 		return ret;
@@ -4232,7 +4233,7 @@ static int process_perf_opa(struct ib_device *ibdev, u8 port,
 
 	*out_mad = *in_mad;
 
-	if (pmp->mad_hdr.class_version != OPA_SMI_CLASS_VERSION) {
+	if (pmp->mad_hdr.class_version != OPA_SM_CLASS_VERSION) {
 		pmp->mad_hdr.status |= IB_SMP_UNSUP_VERSION;
 		return reply((struct ib_mad_hdr *)pmp);
 	}
diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.c b/drivers/infiniband/hw/hfi1/mmu_rb.c
index 7ad30898fc19..ccbf52c8ff6f 100644
--- a/drivers/infiniband/hw/hfi1/mmu_rb.c
+++ b/drivers/infiniband/hw/hfi1/mmu_rb.c
@@ -81,7 +81,7 @@ static void do_remove(struct mmu_rb_handler *handler,
 		      struct list_head *del_list);
 static void handle_remove(struct work_struct *work);
 
-static struct mmu_notifier_ops mn_opts = {
+static const struct mmu_notifier_ops mn_opts = {
 	.invalidate_page = mmu_notifier_page,
 	.invalidate_range_start = mmu_notifier_range_start,
 };
diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c
index d89b8745d4c1..615be68e40b3 100644
--- a/drivers/infiniband/hw/hfi1/pio.c
+++ b/drivers/infiniband/hw/hfi1/pio.c
@@ -758,6 +758,7 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type,
 	sc->hw_context = hw_context;
 	cr_group_addresses(sc, &dma);
 	sc->credits = sci->credits;
+	sc->size = sc->credits * PIO_BLOCK_SIZE;
 
 /* PIO Send Memory Address details */
 #define PIO_ADDR_CONTEXT_MASK 0xfful
@@ -1242,6 +1243,7 @@ int sc_enable(struct send_context *sc)
 	sc->free = 0;
 	sc->alloc_free = 0;
 	sc->fill = 0;
+	sc->fill_wrap = 0;
 	sc->sr_head = 0;
 	sc->sr_tail = 0;
 	sc->flags = 0;
@@ -1385,7 +1387,7 @@ struct pio_buf *sc_buffer_alloc(struct send_context *sc, u32 dw_len,
 	unsigned long flags;
 	unsigned long avail;
 	unsigned long blocks = dwords_to_blocks(dw_len);
-	unsigned long start_fill;
+	u32 fill_wrap;
 	int trycount = 0;
 	u32 head, next;
 
@@ -1410,9 +1412,7 @@ retry:
 			(sc->fill - sc->alloc_free);
 		if (blocks > avail) {
 			/* still no room, actively update */
-			spin_unlock_irqrestore(&sc->alloc_lock, flags);
 			sc_release_update(sc);
-			spin_lock_irqsave(&sc->alloc_lock, flags);
 			sc->alloc_free = ACCESS_ONCE(sc->free);
 			trycount++;
 			goto retry;
@@ -1428,8 +1428,11 @@ retry:
 	head = sc->sr_head;
 
 	/* "allocate" the buffer */
-	start_fill = sc->fill;
 	sc->fill += blocks;
+	fill_wrap = sc->fill_wrap;
+	sc->fill_wrap += blocks;
+	if (sc->fill_wrap >= sc->credits)
+		sc->fill_wrap = sc->fill_wrap - sc->credits;
 
 	/*
 	 * Fill the parts that the releaser looks at before moving the head.
@@ -1458,11 +1461,8 @@ retry:
 	spin_unlock_irqrestore(&sc->alloc_lock, flags);
 
 	/* finish filling in the buffer outside the lock */
-	pbuf->start = sc->base_addr + ((start_fill % sc->credits)
-							* PIO_BLOCK_SIZE);
-	pbuf->size = sc->credits * PIO_BLOCK_SIZE;
-	pbuf->end = sc->base_addr + pbuf->size;
-	pbuf->block_count = blocks;
+	pbuf->start = sc->base_addr + fill_wrap * PIO_BLOCK_SIZE;
+	pbuf->end = sc->base_addr + sc->size;
 	pbuf->qw_written = 0;
 	pbuf->carry_bytes = 0;
 	pbuf->carry.val64 = 0;
@@ -1573,6 +1573,7 @@ static void sc_piobufavail(struct send_context *sc)
 		qp = iowait_to_qp(wait);
 		priv = qp->priv;
 		list_del_init(&priv->s_iowait.list);
+		priv->s_iowait.lock = NULL;
 		/* refcount held until actual wake up */
 		qps[n++] = qp;
 	}
@@ -2028,29 +2029,17 @@ freesc15:
 int init_credit_return(struct hfi1_devdata *dd)
 {
 	int ret;
-	int num_numa;
 	int i;
 
-	num_numa = num_online_nodes();
-	/* enforce the expectation that the numas are compact */
-	for (i = 0; i < num_numa; i++) {
-		if (!node_online(i)) {
-			dd_dev_err(dd, "NUMA nodes are not compact\n");
-			ret = -EINVAL;
-			goto done;
-		}
-	}
-
 	dd->cr_base = kcalloc(
-		num_numa,
+		node_affinity.num_possible_nodes,
 		sizeof(struct credit_return_base),
 		GFP_KERNEL);
 	if (!dd->cr_base) {
-		dd_dev_err(dd, "Unable to allocate credit return base\n");
 		ret = -ENOMEM;
 		goto done;
 	}
-	for (i = 0; i < num_numa; i++) {
+	for_each_node_with_cpus(i) {
 		int bytes = TXE_NUM_CONTEXTS * sizeof(struct credit_return);
 
 		set_dev_node(&dd->pcidev->dev, i);
@@ -2077,14 +2066,11 @@ done:
 
 void free_credit_return(struct hfi1_devdata *dd)
 {
-	int num_numa;
 	int i;
 
 	if (!dd->cr_base)
 		return;
-
-	num_numa = num_online_nodes();
-	for (i = 0; i < num_numa; i++) {
+	for (i = 0; i < node_affinity.num_possible_nodes; i++) {
 		if (dd->cr_base[i].va) {
 			dma_free_coherent(&dd->pcidev->dev,
 					  TXE_NUM_CONTEXTS *
diff --git a/drivers/infiniband/hw/hfi1/pio.h b/drivers/infiniband/hw/hfi1/pio.h
index e709eaf743b5..867e5ffc3595 100644
--- a/drivers/infiniband/hw/hfi1/pio.h
+++ b/drivers/infiniband/hw/hfi1/pio.h
@@ -83,53 +83,55 @@ struct pio_buf {
 	void *arg;		/* argument for cb */
 	void __iomem *start;	/* buffer start address */
 	void __iomem *end;	/* context end address */
-	unsigned long size;	/* context size, in bytes */
 	unsigned long sent_at;	/* buffer is sent when <= free */
-	u32 block_count;	/* size of buffer, in blocks */
-	u32 qw_written;		/* QW written so far */
-	u32 carry_bytes;	/* number of valid bytes in carry */
 	union mix carry;	/* pending unwritten bytes */
+	u16 qw_written;		/* QW written so far */
+	u8 carry_bytes;	/* number of valid bytes in carry */
 };
 
 /* cache line aligned pio buffer array */
 union pio_shadow_ring {
 	struct pio_buf pbuf;
-	u64 unused[16];		/* cache line spacer */
 } ____cacheline_aligned;
 
 /* per-NUMA send context */
 struct send_context {
 	/* read-only after init */
 	struct hfi1_devdata *dd;		/* device */
-	void __iomem *base_addr;	/* start of PIO memory */
 	union pio_shadow_ring *sr;	/* shadow ring */
+	void __iomem *base_addr;	/* start of PIO memory */
+	u32 __percpu *buffers_allocated;/* count of buffers allocated */
+	u32 size;			/* context size, in bytes */
 
-	volatile __le64 *hw_free;	/* HW free counter */
-	struct work_struct halt_work;	/* halted context work queue entry */
-	unsigned long flags;		/* flags */
 	int node;			/* context home node */
-	int type;			/* context type */
-	u32 sw_index;			/* software index number */
-	u32 hw_context;			/* hardware context number */
-	u32 credits;			/* number of blocks in context */
 	u32 sr_size;			/* size of the shadow ring */
-	u32 group;			/* credit return group */
+	u16 flags;			/* flags */
+	u8  type;			/* context type */
+	u8  sw_index;			/* software index number */
+	u8  hw_context;			/* hardware context number */
+	u8  group;			/* credit return group */
+
 	/* allocator fields */
 	spinlock_t alloc_lock ____cacheline_aligned_in_smp;
+	u32 sr_head;			/* shadow ring head */
 	unsigned long fill;		/* official alloc count */
 	unsigned long alloc_free;	/* copy of free (less cache thrash) */
-	u32 sr_head;			/* shadow ring head */
+	u32 fill_wrap;			/* tracks fill within ring */
+	u32 credits;			/* number of blocks in context */
+	/* adding a new field here would make it part of this cacheline */
+
 	/* releaser fields */
 	spinlock_t release_lock ____cacheline_aligned_in_smp;
-	unsigned long free;		/* official free count */
 	u32 sr_tail;			/* shadow ring tail */
+	unsigned long free;		/* official free count */
+	volatile __le64 *hw_free;	/* HW free counter */
 	/* list for PIO waiters */
 	struct list_head piowait  ____cacheline_aligned_in_smp;
 	spinlock_t credit_ctrl_lock ____cacheline_aligned_in_smp;
-	u64 credit_ctrl;		/* cache for credit control */
 	u32 credit_intr_count;		/* count of credit intr users */
-	u32 __percpu *buffers_allocated;/* count of buffers allocated */
+	u64 credit_ctrl;		/* cache for credit control */
 	wait_queue_head_t halt_wait;    /* wait until kernel sees interrupt */
+	struct work_struct halt_work;	/* halted context work queue entry */
 };
 
 /* send context flags */
diff --git a/drivers/infiniband/hw/hfi1/pio_copy.c b/drivers/infiniband/hw/hfi1/pio_copy.c
index aa7773643107..03024cec78dd 100644
--- a/drivers/infiniband/hw/hfi1/pio_copy.c
+++ b/drivers/infiniband/hw/hfi1/pio_copy.c
@@ -129,8 +129,8 @@ void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc,
 				dest += sizeof(u64);
 			}
 
-			dest -= pbuf->size;
-			dend -= pbuf->size;
+			dest -= pbuf->sc->size;
+			dend -= pbuf->sc->size;
 		}
 
 		/* write 8-byte non-SOP, non-wrap chunk data */
@@ -361,8 +361,8 @@ void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc,
 				dest += sizeof(u64);
 			}
 
-			dest -= pbuf->size;
-			dend -= pbuf->size;
+			dest -= pbuf->sc->size;
+			dend -= pbuf->sc->size;
 		}
 
 		/* write 8-byte non-SOP, non-wrap chunk data */
@@ -458,8 +458,8 @@ static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)
 			dest += sizeof(u64);
 		}
 
-		dest -= pbuf->size;
-		dend -= pbuf->size;
+		dest -= pbuf->sc->size;
+		dend -= pbuf->sc->size;
 	}
 
 	/* write 8-byte non-SOP, non-wrap chunk data */
@@ -492,7 +492,7 @@ static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)
 		 */
 		/* adjust if we have wrapped */
 		if (dest >= pbuf->end)
-			dest -= pbuf->size;
+			dest -= pbuf->sc->size;
 		/* jump to the SOP range if within the first block */
 		else if (pbuf->qw_written < PIO_BLOCK_QWS)
 			dest += SOP_DISTANCE;
@@ -584,8 +584,8 @@ static void mid_copy_straight(struct pio_buf *pbuf,
 			dest += sizeof(u64);
 		}
 
-		dest -= pbuf->size;
-		dend -= pbuf->size;
+		dest -= pbuf->sc->size;
+		dend -= pbuf->sc->size;
 	}
 
 	/* write 8-byte non-SOP, non-wrap chunk data */
@@ -666,7 +666,7 @@ void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)
 			 */
 			/* adjust if we've wrapped */
 			if (dest >= pbuf->end)
-				dest -= pbuf->size;
+				dest -= pbuf->sc->size;
 			/* jump to SOP range if within the first block */
 			else if (pbuf->qw_written < PIO_BLOCK_QWS)
 				dest += SOP_DISTANCE;
@@ -719,7 +719,7 @@ void seg_pio_copy_end(struct pio_buf *pbuf)
 	 */
 	/* adjust if we have wrapped */
 	if (dest >= pbuf->end)
-		dest -= pbuf->size;
+		dest -= pbuf->sc->size;
 	/* jump to the SOP range if within the first block */
 	else if (pbuf->qw_written < PIO_BLOCK_QWS)
 		dest += SOP_DISTANCE;
diff --git a/drivers/infiniband/hw/hfi1/platform.c b/drivers/infiniband/hw/hfi1/platform.c
index 202433178864..838fe84e285a 100644
--- a/drivers/infiniband/hw/hfi1/platform.c
+++ b/drivers/infiniband/hw/hfi1/platform.c
@@ -49,6 +49,90 @@
 #include "efivar.h"
 #include "eprom.h"
 
+static int validate_scratch_checksum(struct hfi1_devdata *dd)
+{
+	u64 checksum = 0, temp_scratch = 0;
+	int i, j, version;
+
+	temp_scratch = read_csr(dd, ASIC_CFG_SCRATCH);
+	version = (temp_scratch & BITMAP_VERSION_SMASK) >> BITMAP_VERSION_SHIFT;
+
+	/* Prevent power on default of all zeroes from passing checksum */
+	if (!version)
+		return 0;
+
+	/*
+	 * ASIC scratch 0 only contains the checksum and bitmap version as
+	 * fields of interest, both of which are handled separately from the
+	 * loop below, so skip it
+	 */
+	checksum += version;
+	for (i = 1; i < ASIC_NUM_SCRATCH; i++) {
+		temp_scratch = read_csr(dd, ASIC_CFG_SCRATCH + (8 * i));
+		for (j = sizeof(u64); j != 0; j -= 2) {
+			checksum += (temp_scratch & 0xFFFF);
+			temp_scratch >>= 16;
+		}
+	}
+
+	while (checksum >> 16)
+		checksum = (checksum & CHECKSUM_MASK) + (checksum >> 16);
+
+	temp_scratch = read_csr(dd, ASIC_CFG_SCRATCH);
+	temp_scratch &= CHECKSUM_SMASK;
+	temp_scratch >>= CHECKSUM_SHIFT;
+
+	if (checksum + temp_scratch == 0xFFFF)
+		return 1;
+	return 0;
+}
+
+static void save_platform_config_fields(struct hfi1_devdata *dd)
+{
+	struct hfi1_pportdata *ppd = dd->pport;
+	u64 temp_scratch = 0, temp_dest = 0;
+
+	temp_scratch = read_csr(dd, ASIC_CFG_SCRATCH_1);
+
+	temp_dest = temp_scratch &
+		    (dd->hfi1_id ? PORT1_PORT_TYPE_SMASK :
+		     PORT0_PORT_TYPE_SMASK);
+	ppd->port_type = temp_dest >>
+			 (dd->hfi1_id ? PORT1_PORT_TYPE_SHIFT :
+			  PORT0_PORT_TYPE_SHIFT);
+
+	temp_dest = temp_scratch &
+		    (dd->hfi1_id ? PORT1_LOCAL_ATTEN_SMASK :
+		     PORT0_LOCAL_ATTEN_SMASK);
+	ppd->local_atten = temp_dest >>
+			   (dd->hfi1_id ? PORT1_LOCAL_ATTEN_SHIFT :
+			    PORT0_LOCAL_ATTEN_SHIFT);
+
+	temp_dest = temp_scratch &
+		    (dd->hfi1_id ? PORT1_REMOTE_ATTEN_SMASK :
+		     PORT0_REMOTE_ATTEN_SMASK);
+	ppd->remote_atten = temp_dest >>
+			    (dd->hfi1_id ? PORT1_REMOTE_ATTEN_SHIFT :
+			     PORT0_REMOTE_ATTEN_SHIFT);
+
+	temp_dest = temp_scratch &
+		    (dd->hfi1_id ? PORT1_DEFAULT_ATTEN_SMASK :
+		     PORT0_DEFAULT_ATTEN_SMASK);
+	ppd->default_atten = temp_dest >>
+			     (dd->hfi1_id ? PORT1_DEFAULT_ATTEN_SHIFT :
+			      PORT0_DEFAULT_ATTEN_SHIFT);
+
+	temp_scratch = read_csr(dd, dd->hfi1_id ? ASIC_CFG_SCRATCH_3 :
+				ASIC_CFG_SCRATCH_2);
+
+	ppd->tx_preset_eq = (temp_scratch & TX_EQ_SMASK) >> TX_EQ_SHIFT;
+	ppd->tx_preset_noeq = (temp_scratch & TX_NO_EQ_SMASK) >> TX_NO_EQ_SHIFT;
+	ppd->rx_preset = (temp_scratch & RX_SMASK) >> RX_SHIFT;
+
+	ppd->max_power_class = (temp_scratch & QSFP_MAX_POWER_SMASK) >>
+				QSFP_MAX_POWER_SHIFT;
+}
+
 void get_platform_config(struct hfi1_devdata *dd)
 {
 	int ret = 0;
@@ -56,38 +140,49 @@ void get_platform_config(struct hfi1_devdata *dd)
 	u8 *temp_platform_config = NULL;
 	u32 esize;
 
-	ret = eprom_read_platform_config(dd, (void **)&temp_platform_config,
-					 &esize);
-	if (!ret) {
-		/* success */
-		size = esize;
-		goto success;
+	if (is_integrated(dd)) {
+		if (validate_scratch_checksum(dd)) {
+			save_platform_config_fields(dd);
+			return;
+		}
+		dd_dev_err(dd, "%s: Config bitmap corrupted/uninitialized\n",
+			   __func__);
+		dd_dev_err(dd,
+			   "%s: Please update your BIOS to support active channels\n",
+			   __func__);
+	} else {
+		ret = eprom_read_platform_config(dd,
+						 (void **)&temp_platform_config,
+						 &esize);
+		if (!ret) {
+			/* success */
+			dd->platform_config.data = temp_platform_config;
+			dd->platform_config.size = esize;
+			return;
+		}
+		/* fail, try EFI variable */
+
+		ret = read_hfi1_efi_var(dd, "configuration", &size,
+					(void **)&temp_platform_config);
+		if (!ret) {
+			dd->platform_config.data = temp_platform_config;
+			dd->platform_config.size = size;
+			return;
+		}
 	}
-	/* fail, try EFI variable */
-
-	ret = read_hfi1_efi_var(dd, "configuration", &size,
-				(void **)&temp_platform_config);
-	if (!ret)
-		goto success;
-
-	dd_dev_info(dd,
-		    "%s: Failed to get platform config from UEFI, falling back to request firmware\n",
-		    __func__);
+	dd_dev_err(dd,
+		   "%s: Failed to get platform config, falling back to sub-optimal default file\n",
+		   __func__);
 	/* fall back to request firmware */
 	platform_config_load = 1;
-	return;
-
-success:
-	dd->platform_config.data = temp_platform_config;
-	dd->platform_config.size = size;
 }
 
 void free_platform_config(struct hfi1_devdata *dd)
 {
 	if (!platform_config_load) {
 		/*
-		 * was loaded from EFI, release memory
-		 * allocated by read_efi_var
+		 * was loaded from EFI or the EPROM, release memory
+		 * allocated by read_efi_var/eprom_read_platform_config
 		 */
 		kfree(dd->platform_config.data);
 	}
@@ -100,12 +195,16 @@ void free_platform_config(struct hfi1_devdata *dd)
 void get_port_type(struct hfi1_pportdata *ppd)
 {
 	int ret;
+	u32 temp;
 
 	ret = get_platform_config_field(ppd->dd, PLATFORM_CONFIG_PORT_TABLE, 0,
-					PORT_TABLE_PORT_TYPE, &ppd->port_type,
+					PORT_TABLE_PORT_TYPE, &temp,
 					4);
-	if (ret)
+	if (ret) {
 		ppd->port_type = PORT_TYPE_UNKNOWN;
+		return;
+	}
+	ppd->port_type = temp;
 }
 
 int set_qsfp_tx(struct hfi1_pportdata *ppd, int on)
@@ -538,6 +637,38 @@ static void apply_tx_lanes(struct hfi1_pportdata *ppd, u8 field_id,
 	}
 }
 
+/*
+ * Return a special SerDes setting for low power AOC cables.  The power class
+ * threshold and setting being used were all found by empirical testing.
+ *
+ * Summary of the logic:
+ *
+ * if (QSFP and QSFP_TYPE == AOC and QSFP_POWER_CLASS < 4)
+ *     return 0xe
+ * return 0; // leave at default
+ */
+static u8 aoc_low_power_setting(struct hfi1_pportdata *ppd)
+{
+	u8 *cache = ppd->qsfp_info.cache;
+	int power_class;
+
+	/* QSFP only */
+	if (ppd->port_type != PORT_TYPE_QSFP)
+		return 0; /* leave at default */
+
+	/* active optical cables only */
+	switch ((cache[QSFP_MOD_TECH_OFFS] & 0xF0) >> 4) {
+	case 0x0 ... 0x9: /* fallthrough */
+	case 0xC: /* fallthrough */
+	case 0xE:
+		/* active AOC */
+		power_class = get_qsfp_power_class(cache[QSFP_MOD_PWR_OFFS]);
+		if (power_class < QSFP_POWER_CLASS_4)
+			return 0xe;
+	}
+	return 0; /* leave at default */
+}
+
 static void apply_tunings(
 		struct hfi1_pportdata *ppd, u32 tx_preset_index,
 		u8 tuning_method, u32 total_atten, u8 limiting_active)
@@ -606,7 +737,17 @@ static void apply_tunings(
 		tx_preset_index, TX_PRESET_TABLE_POSTCUR, &tx_preset, 4);
 	postcur = tx_preset;
 
-	config_data = precur | (attn << 8) | (postcur << 16);
+	/*
+	 * NOTES:
+	 * o The aoc_low_power_setting is applied to all lanes even
+	 *   though only lane 0's value is examined by the firmware.
+	 * o A lingering low power setting after a cable swap does
+	 *   not occur.  On cable unplug the 8051 is reset and
+	 *   restarted on cable insert.  This resets all settings to
+	 *   their default, erasing any previous low power setting.
+	 */
+	config_data = precur | (attn << 8) | (postcur << 16) |
+			(aoc_low_power_setting(ppd) << 24);
 
 	apply_tx_lanes(ppd, TX_EQ_SETTINGS, config_data,
 		       "Applying TX settings");
diff --git a/drivers/infiniband/hw/hfi1/platform.h b/drivers/infiniband/hw/hfi1/platform.h
index e2c21613c326..eed0aa9124fa 100644
--- a/drivers/infiniband/hw/hfi1/platform.h
+++ b/drivers/infiniband/hw/hfi1/platform.h
@@ -168,16 +168,6 @@ struct platform_config_cache {
 	struct platform_config_data config_tables[PLATFORM_CONFIG_TABLE_MAX];
 };
 
-static const u32 platform_config_table_limits[PLATFORM_CONFIG_TABLE_MAX] = {
-	0,
-	SYSTEM_TABLE_MAX,
-	PORT_TABLE_MAX,
-	RX_PRESET_TABLE_MAX,
-	TX_PRESET_TABLE_MAX,
-	QSFP_ATTEN_TABLE_MAX,
-	VARIABLE_SETTINGS_TABLE_MAX
-};
-
 /* This section defines default values and encodings for the
  * fields defined for each table above
  */
@@ -295,6 +285,123 @@ enum link_tuning_encoding {
 	OPA_UNKNOWN_TUNING
 };
 
+/*
+ * Shifts and masks for the link SI tuning values stuffed into the ASIC scratch
+ * registers for integrated platforms
+ */
+#define PORT0_PORT_TYPE_SHIFT		0
+#define PORT0_LOCAL_ATTEN_SHIFT		4
+#define PORT0_REMOTE_ATTEN_SHIFT	10
+#define PORT0_DEFAULT_ATTEN_SHIFT	32
+
+#define PORT1_PORT_TYPE_SHIFT		16
+#define PORT1_LOCAL_ATTEN_SHIFT		20
+#define PORT1_REMOTE_ATTEN_SHIFT	26
+#define PORT1_DEFAULT_ATTEN_SHIFT	40
+
+#define PORT0_PORT_TYPE_MASK		0xFUL
+#define PORT0_LOCAL_ATTEN_MASK		0x3FUL
+#define PORT0_REMOTE_ATTEN_MASK		0x3FUL
+#define PORT0_DEFAULT_ATTEN_MASK	0xFFUL
+
+#define PORT1_PORT_TYPE_MASK		0xFUL
+#define PORT1_LOCAL_ATTEN_MASK		0x3FUL
+#define PORT1_REMOTE_ATTEN_MASK		0x3FUL
+#define PORT1_DEFAULT_ATTEN_MASK	0xFFUL
+
+#define PORT0_PORT_TYPE_SMASK		(PORT0_PORT_TYPE_MASK << \
+					 PORT0_PORT_TYPE_SHIFT)
+#define PORT0_LOCAL_ATTEN_SMASK		(PORT0_LOCAL_ATTEN_MASK << \
+					 PORT0_LOCAL_ATTEN_SHIFT)
+#define PORT0_REMOTE_ATTEN_SMASK	(PORT0_REMOTE_ATTEN_MASK << \
+					 PORT0_REMOTE_ATTEN_SHIFT)
+#define PORT0_DEFAULT_ATTEN_SMASK	(PORT0_DEFAULT_ATTEN_MASK << \
+					 PORT0_DEFAULT_ATTEN_SHIFT)
+
+#define PORT1_PORT_TYPE_SMASK		(PORT1_PORT_TYPE_MASK << \
+					 PORT1_PORT_TYPE_SHIFT)
+#define PORT1_LOCAL_ATTEN_SMASK		(PORT1_LOCAL_ATTEN_MASK << \
+					 PORT1_LOCAL_ATTEN_SHIFT)
+#define PORT1_REMOTE_ATTEN_SMASK	(PORT1_REMOTE_ATTEN_MASK << \
+					 PORT1_REMOTE_ATTEN_SHIFT)
+#define PORT1_DEFAULT_ATTEN_SMASK	(PORT1_DEFAULT_ATTEN_MASK << \
+					 PORT1_DEFAULT_ATTEN_SHIFT)
+
+#define QSFP_MAX_POWER_SHIFT		0
+#define TX_NO_EQ_SHIFT			4
+#define TX_EQ_SHIFT			25
+#define RX_SHIFT			46
+
+#define QSFP_MAX_POWER_MASK		0xFUL
+#define TX_NO_EQ_MASK			0x1FFFFFUL
+#define TX_EQ_MASK			0x1FFFFFUL
+#define RX_MASK				0xFFFFUL
+
+#define QSFP_MAX_POWER_SMASK		(QSFP_MAX_POWER_MASK << \
+					 QSFP_MAX_POWER_SHIFT)
+#define TX_NO_EQ_SMASK			(TX_NO_EQ_MASK << TX_NO_EQ_SHIFT)
+#define TX_EQ_SMASK			(TX_EQ_MASK << TX_EQ_SHIFT)
+#define RX_SMASK			(RX_MASK << RX_SHIFT)
+
+#define TX_PRECUR_SHIFT			0
+#define TX_ATTN_SHIFT			4
+#define QSFP_TX_CDR_APPLY_SHIFT		9
+#define QSFP_TX_EQ_APPLY_SHIFT		10
+#define QSFP_TX_CDR_SHIFT		11
+#define QSFP_TX_EQ_SHIFT		12
+#define TX_POSTCUR_SHIFT		16
+
+#define TX_PRECUR_MASK			0xFUL
+#define TX_ATTN_MASK			0x1FUL
+#define QSFP_TX_CDR_APPLY_MASK		0x1UL
+#define QSFP_TX_EQ_APPLY_MASK		0x1UL
+#define QSFP_TX_CDR_MASK		0x1UL
+#define QSFP_TX_EQ_MASK			0xFUL
+#define TX_POSTCUR_MASK			0x1FUL
+
+#define TX_PRECUR_SMASK			(TX_PRECUR_MASK << TX_PRECUR_SHIFT)
+#define TX_ATTN_SMASK			(TX_ATTN_MASK << TX_ATTN_SHIFT)
+#define QSFP_TX_CDR_APPLY_SMASK		(QSFP_TX_CDR_APPLY_MASK << \
+					 QSFP_TX_CDR_APPLY_SHIFT)
+#define QSFP_TX_EQ_APPLY_SMASK		(QSFP_TX_EQ_APPLY_MASK << \
+					 QSFP_TX_EQ_APPLY_SHIFT)
+#define QSFP_TX_CDR_SMASK		(QSFP_TX_CDR_MASK << QSFP_TX_CDR_SHIFT)
+#define QSFP_TX_EQ_SMASK		(QSFP_TX_EQ_MASK << QSFP_TX_EQ_SHIFT)
+#define TX_POSTCUR_SMASK		(TX_POSTCUR_MASK << TX_POSTCUR_SHIFT)
+
+#define QSFP_RX_CDR_APPLY_SHIFT		0
+#define QSFP_RX_EMP_APPLY_SHIFT		1
+#define QSFP_RX_AMP_APPLY_SHIFT		2
+#define QSFP_RX_CDR_SHIFT		3
+#define QSFP_RX_EMP_SHIFT		4
+#define QSFP_RX_AMP_SHIFT		8
+
+#define QSFP_RX_CDR_APPLY_MASK		0x1UL
+#define QSFP_RX_EMP_APPLY_MASK		0x1UL
+#define QSFP_RX_AMP_APPLY_MASK		0x1UL
+#define QSFP_RX_CDR_MASK		0x1UL
+#define QSFP_RX_EMP_MASK		0xFUL
+#define QSFP_RX_AMP_MASK		0x3UL
+
+#define QSFP_RX_CDR_APPLY_SMASK		(QSFP_RX_CDR_APPLY_MASK << \
+					 QSFP_RX_CDR_APPLY_SHIFT)
+#define QSFP_RX_EMP_APPLY_SMASK		(QSFP_RX_EMP_APPLY_MASK << \
+					 QSFP_RX_EMP_APPLY_SHIFT)
+#define QSFP_RX_AMP_APPLY_SMASK		(QSFP_RX_AMP_APPLY_MASK << \
+					 QSFP_RX_AMP_APPLY_SHIFT)
+#define QSFP_RX_CDR_SMASK		(QSFP_RX_CDR_MASK << QSFP_RX_CDR_SHIFT)
+#define QSFP_RX_EMP_SMASK		(QSFP_RX_EMP_MASK << QSFP_RX_EMP_SHIFT)
+#define QSFP_RX_AMP_SMASK		(QSFP_RX_AMP_MASK << QSFP_RX_AMP_SHIFT)
+
+#define BITMAP_VERSION			1
+#define BITMAP_VERSION_SHIFT		44
+#define BITMAP_VERSION_MASK		0xFUL
+#define BITMAP_VERSION_SMASK		(BITMAP_VERSION_MASK << \
+					 BITMAP_VERSION_SHIFT)
+#define CHECKSUM_SHIFT			48
+#define CHECKSUM_MASK			0xFFFFUL
+#define CHECKSUM_SMASK			(CHECKSUM_MASK << CHECKSUM_SHIFT)
+
 /* platform.c */
 void get_platform_config(struct hfi1_devdata *dd);
 void free_platform_config(struct hfi1_devdata *dd);
diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c
index 9fc75e7e8781..d752d6768a49 100644
--- a/drivers/infiniband/hw/hfi1/qp.c
+++ b/drivers/infiniband/hw/hfi1/qp.c
@@ -196,15 +196,18 @@ static void flush_tx_list(struct rvt_qp *qp)
 static void flush_iowait(struct rvt_qp *qp)
 {
 	struct hfi1_qp_priv *priv = qp->priv;
-	struct hfi1_ibdev *dev = to_idev(qp->ibqp.device);
 	unsigned long flags;
+	seqlock_t *lock = priv->s_iowait.lock;
 
-	write_seqlock_irqsave(&dev->iowait_lock, flags);
+	if (!lock)
+		return;
+	write_seqlock_irqsave(lock, flags);
 	if (!list_empty(&priv->s_iowait.list)) {
 		list_del_init(&priv->s_iowait.list);
+		priv->s_iowait.lock = NULL;
 		rvt_put_qp(qp);
 	}
-	write_sequnlock_irqrestore(&dev->iowait_lock, flags);
+	write_sequnlock_irqrestore(lock, flags);
 }
 
 static inline int opa_mtu_enum_to_int(int mtu)
@@ -543,6 +546,7 @@ static int iowait_sleep(
 			ibp->rvp.n_dmawait++;
 			qp->s_flags |= RVT_S_WAIT_DMA_DESC;
 			list_add_tail(&priv->s_iowait.list, &sde->dmawait);
+			priv->s_iowait.lock = &dev->iowait_lock;
 			trace_hfi1_qpsleep(qp, RVT_S_WAIT_DMA_DESC);
 			rvt_get_qp(qp);
 		}
@@ -964,6 +968,7 @@ void notify_error_qp(struct rvt_qp *qp)
 	if (!list_empty(&priv->s_iowait.list) && !(qp->s_flags & RVT_S_BUSY)) {
 		qp->s_flags &= ~RVT_S_ANY_WAIT_IO;
 		list_del_init(&priv->s_iowait.list);
+		priv->s_iowait.lock = NULL;
 		rvt_put_qp(qp);
 	}
 	write_sequnlock(&dev->iowait_lock);
diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c
index 83198a8a8797..809b26eb6d3c 100644
--- a/drivers/infiniband/hw/hfi1/rc.c
+++ b/drivers/infiniband/hw/hfi1/rc.c
@@ -276,7 +276,7 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,
 				rvt_get_mr(ps->s_txreq->mr);
 			qp->s_ack_rdma_sge.sge = e->rdma_sge;
 			qp->s_ack_rdma_sge.num_sge = 1;
-			qp->s_cur_sge = &qp->s_ack_rdma_sge;
+			ps->s_txreq->ss = &qp->s_ack_rdma_sge;
 			if (len > pmtu) {
 				len = pmtu;
 				qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST);
@@ -290,7 +290,7 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,
 			bth2 = mask_psn(qp->s_ack_rdma_psn++);
 		} else {
 			/* COMPARE_SWAP or FETCH_ADD */
-			qp->s_cur_sge = NULL;
+			ps->s_txreq->ss = NULL;
 			len = 0;
 			qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
 			ohdr->u.at.aeth = hfi1_compute_aeth(qp);
@@ -306,7 +306,7 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,
 		qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);
 		/* FALLTHROUGH */
 	case OP(RDMA_READ_RESPONSE_MIDDLE):
-		qp->s_cur_sge = &qp->s_ack_rdma_sge;
+		ps->s_txreq->ss = &qp->s_ack_rdma_sge;
 		ps->s_txreq->mr = qp->s_ack_rdma_sge.sge.mr;
 		if (ps->s_txreq->mr)
 			rvt_get_mr(ps->s_txreq->mr);
@@ -335,7 +335,7 @@ normal:
 		 */
 		qp->s_ack_state = OP(SEND_ONLY);
 		qp->s_flags &= ~RVT_S_ACK_PENDING;
-		qp->s_cur_sge = NULL;
+		ps->s_txreq->ss = NULL;
 		if (qp->s_nak_state)
 			ohdr->u.aeth =
 				cpu_to_be32((qp->r_msn & HFI1_MSN_MASK) |
@@ -351,7 +351,7 @@ normal:
 	qp->s_rdma_ack_cnt++;
 	qp->s_hdrwords = hwords;
 	ps->s_txreq->sde = priv->s_sde;
-	qp->s_cur_size = len;
+	ps->s_txreq->s_cur_size = len;
 	hfi1_make_ruc_header(qp, ohdr, bth0, bth2, middle, ps);
 	/* pbc */
 	ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2;
@@ -801,8 +801,8 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 	qp->s_len -= len;
 	qp->s_hdrwords = hwords;
 	ps->s_txreq->sde = priv->s_sde;
-	qp->s_cur_sge = ss;
-	qp->s_cur_size = len;
+	ps->s_txreq->ss = ss;
+	ps->s_txreq->s_cur_size = len;
 	hfi1_make_ruc_header(
 		qp,
 		ohdr,
@@ -1146,8 +1146,6 @@ void hfi1_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
 {
 	struct ib_other_headers *ohdr;
 	struct rvt_swqe *wqe;
-	struct ib_wc wc;
-	unsigned i;
 	u32 opcode;
 	u32 psn;
 
@@ -1195,22 +1193,8 @@ void hfi1_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
 		qp->s_last = s_last;
 		/* see post_send() */
 		barrier();
-		for (i = 0; i < wqe->wr.num_sge; i++) {
-			struct rvt_sge *sge = &wqe->sg_list[i];
-
-			rvt_put_mr(sge->mr);
-		}
-		/* Post a send completion queue entry if requested. */
-		if (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
-		    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
-			memset(&wc, 0, sizeof(wc));
-			wc.wr_id = wqe->wr.wr_id;
-			wc.status = IB_WC_SUCCESS;
-			wc.opcode = ib_hfi1_wc_opcode[wqe->wr.opcode];
-			wc.byte_len = wqe->length;
-			wc.qp = &qp->ibqp;
-			rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.send_cq), &wc, 0);
-		}
+		rvt_put_swqe(wqe);
+		rvt_qp_swqe_complete(qp, wqe, IB_WC_SUCCESS);
 	}
 	/*
 	 * If we were waiting for sends to complete before re-sending,
@@ -1240,9 +1224,6 @@ static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
 					 struct rvt_swqe *wqe,
 					 struct hfi1_ibport *ibp)
 {
-	struct ib_wc wc;
-	unsigned i;
-
 	lockdep_assert_held(&qp->s_lock);
 	/*
 	 * Don't decrement refcount and don't generate a
@@ -1253,28 +1234,14 @@ static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
 	    cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
 		u32 s_last;
 
-		for (i = 0; i < wqe->wr.num_sge; i++) {
-			struct rvt_sge *sge = &wqe->sg_list[i];
-
-			rvt_put_mr(sge->mr);
-		}
+		rvt_put_swqe(wqe);
 		s_last = qp->s_last;
 		if (++s_last >= qp->s_size)
 			s_last = 0;
 		qp->s_last = s_last;
 		/* see post_send() */
 		barrier();
-		/* Post a send completion queue entry if requested. */
-		if (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
-		    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
-			memset(&wc, 0, sizeof(wc));
-			wc.wr_id = wqe->wr.wr_id;
-			wc.status = IB_WC_SUCCESS;
-			wc.opcode = ib_hfi1_wc_opcode[wqe->wr.opcode];
-			wc.byte_len = wqe->length;
-			wc.qp = &qp->ibqp;
-			rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.send_cq), &wc, 0);
-		}
+		rvt_qp_swqe_complete(qp, wqe, IB_WC_SUCCESS);
 	} else {
 		struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
 
@@ -2295,7 +2262,7 @@ send_last:
 		hfi1_copy_sge(&qp->r_sge, data, tlen, 1, copy_last);
 		rvt_put_ss(&qp->r_sge);
 		qp->r_msn++;
-		if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
+		if (!__test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
 			break;
 		wc.wr_id = qp->r_wr_id;
 		wc.status = IB_WC_SUCCESS;
@@ -2410,8 +2377,7 @@ send_last:
 			 * Update the next expected PSN.  We add 1 later
 			 * below, so only add the remainder here.
 			 */
-			if (len > pmtu)
-				qp->r_psn += (len - 1) / pmtu;
+			qp->r_psn += rvt_div_mtu(qp, len - 1);
 		} else {
 			e->rdma_sge.mr = NULL;
 			e->rdma_sge.vaddr = NULL;
diff --git a/drivers/infiniband/hw/hfi1/ruc.c b/drivers/infiniband/hw/hfi1/ruc.c
index a1576aea4756..717ed4b159d3 100644
--- a/drivers/infiniband/hw/hfi1/ruc.c
+++ b/drivers/infiniband/hw/hfi1/ruc.c
@@ -239,16 +239,6 @@ bail:
 	return ret;
 }
 
-static __be64 get_sguid(struct hfi1_ibport *ibp, unsigned index)
-{
-	if (!index) {
-		struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
-
-		return cpu_to_be64(ppd->guid);
-	}
-	return ibp->guids[index - 1];
-}
-
 static int gid_ok(union ib_gid *gid, __be64 gid_prefix, __be64 id)
 {
 	return (gid->global.interface_id == id &&
@@ -699,9 +689,9 @@ u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr,
 	/* The SGID is 32-bit aligned. */
 	hdr->sgid.global.subnet_prefix = ibp->rvp.gid_prefix;
 	hdr->sgid.global.interface_id =
-		grh->sgid_index && grh->sgid_index < ARRAY_SIZE(ibp->guids) ?
-		ibp->guids[grh->sgid_index - 1] :
-			cpu_to_be64(ppd_from_ibp(ibp)->guid);
+		grh->sgid_index < HFI1_GUIDS_PER_PORT ?
+		get_sguid(ibp, grh->sgid_index) :
+		get_sguid(ibp, HFI1_PORT_GUID_INDEX);
 	hdr->dgid = grh->dgid;
 
 	/* GRH header size in 32-bit words. */
@@ -777,8 +767,8 @@ void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,
 	u32 bth1;
 
 	/* Construct the header. */
-	extra_bytes = -qp->s_cur_size & 3;
-	nwords = (qp->s_cur_size + extra_bytes) >> 2;
+	extra_bytes = -ps->s_txreq->s_cur_size & 3;
+	nwords = (ps->s_txreq->s_cur_size + extra_bytes) >> 2;
 	lrh0 = HFI1_LRH_BTH;
 	if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
 		qp->s_hdrwords += hfi1_make_grh(ibp,
@@ -952,7 +942,6 @@ void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
 			enum ib_wc_status status)
 {
 	u32 old_last, last;
-	unsigned i;
 
 	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_OR_FLUSH_SEND))
 		return;
@@ -964,32 +953,13 @@ void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
 	qp->s_last = last;
 	/* See post_send() */
 	barrier();
-	for (i = 0; i < wqe->wr.num_sge; i++) {
-		struct rvt_sge *sge = &wqe->sg_list[i];
-
-		rvt_put_mr(sge->mr);
-	}
+	rvt_put_swqe(wqe);
 	if (qp->ibqp.qp_type == IB_QPT_UD ||
 	    qp->ibqp.qp_type == IB_QPT_SMI ||
 	    qp->ibqp.qp_type == IB_QPT_GSI)
 		atomic_dec(&ibah_to_rvtah(wqe->ud_wr.ah)->refcount);
 
-	/* See ch. 11.2.4.1 and 10.7.3.1 */
-	if (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
-	    (wqe->wr.send_flags & IB_SEND_SIGNALED) ||
-	    status != IB_WC_SUCCESS) {
-		struct ib_wc wc;
-
-		memset(&wc, 0, sizeof(wc));
-		wc.wr_id = wqe->wr.wr_id;
-		wc.status = status;
-		wc.opcode = ib_hfi1_wc_opcode[wqe->wr.opcode];
-		wc.qp = &qp->ibqp;
-		if (status == IB_WC_SUCCESS)
-			wc.byte_len = wqe->length;
-		rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.send_cq), &wc,
-			     status != IB_WC_SUCCESS);
-	}
+	rvt_qp_swqe_complete(qp, wqe, status);
 
 	if (qp->s_acked == old_last)
 		qp->s_acked = last;
diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c
index 9cbe52d21077..1d81cac1fa6c 100644
--- a/drivers/infiniband/hw/hfi1/sdma.c
+++ b/drivers/infiniband/hw/hfi1/sdma.c
@@ -375,7 +375,7 @@ static inline void complete_tx(struct sdma_engine *sde,
 			   sde->head_sn, tx->sn);
 	sde->head_sn++;
 #endif
-	sdma_txclean(sde->dd, tx);
+	__sdma_txclean(sde->dd, tx);
 	if (complete)
 		(*complete)(tx, res);
 	if (wait && iowait_sdma_dec(wait))
@@ -1643,7 +1643,7 @@ static inline u8 ahg_mode(struct sdma_txreq *tx)
 }
 
 /**
- * sdma_txclean() - clean tx of mappings, descp *kmalloc's
+ * __sdma_txclean() - clean tx of mappings, descp *kmalloc's
  * @dd: hfi1_devdata for unmapping
  * @tx: tx request to clean
  *
@@ -1653,7 +1653,7 @@ static inline u8 ahg_mode(struct sdma_txreq *tx)
  * The code can be called multiple times without issue.
  *
  */
-void sdma_txclean(
+void __sdma_txclean(
 	struct hfi1_devdata *dd,
 	struct sdma_txreq *tx)
 {
@@ -3065,7 +3065,7 @@ static int _extend_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)
 		tx->descp[i] = tx->descs[i];
 	return 0;
 enomem:
-	sdma_txclean(dd, tx);
+	__sdma_txclean(dd, tx);
 	return -ENOMEM;
 }
 
@@ -3094,14 +3094,14 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
 
 	rval = _extend_sdma_tx_descs(dd, tx);
 	if (rval) {
-		sdma_txclean(dd, tx);
+		__sdma_txclean(dd, tx);
 		return rval;
 	}
 
 	/* If coalesce buffer is allocated, copy data into it */
 	if (tx->coalesce_buf) {
 		if (type == SDMA_MAP_NONE) {
-			sdma_txclean(dd, tx);
+			__sdma_txclean(dd, tx);
 			return -EINVAL;
 		}
 
@@ -3109,7 +3109,7 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
 			kvaddr = kmap(page);
 			kvaddr += offset;
 		} else if (WARN_ON(!kvaddr)) {
-			sdma_txclean(dd, tx);
+			__sdma_txclean(dd, tx);
 			return -EINVAL;
 		}
 
@@ -3139,7 +3139,7 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
 				      DMA_TO_DEVICE);
 
 		if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) {
-			sdma_txclean(dd, tx);
+			__sdma_txclean(dd, tx);
 			return -ENOSPC;
 		}
 
@@ -3181,7 +3181,7 @@ int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)
 	if ((unlikely(tx->num_desc == tx->desc_limit))) {
 		rval = _extend_sdma_tx_descs(dd, tx);
 		if (rval) {
-			sdma_txclean(dd, tx);
+			__sdma_txclean(dd, tx);
 			return rval;
 		}
 	}
diff --git a/drivers/infiniband/hw/hfi1/sdma.h b/drivers/infiniband/hw/hfi1/sdma.h
index 56257ea3598f..21f1e2834f37 100644
--- a/drivers/infiniband/hw/hfi1/sdma.h
+++ b/drivers/infiniband/hw/hfi1/sdma.h
@@ -667,7 +667,13 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
 			   int type, void *kvaddr, struct page *page,
 			   unsigned long offset, u16 len);
 int _pad_sdma_tx_descs(struct hfi1_devdata *, struct sdma_txreq *);
-void sdma_txclean(struct hfi1_devdata *, struct sdma_txreq *);
+void __sdma_txclean(struct hfi1_devdata *, struct sdma_txreq *);
+
+static inline void sdma_txclean(struct hfi1_devdata *dd, struct sdma_txreq *tx)
+{
+	if (tx->num_desc)
+		__sdma_txclean(dd, tx);
+}
 
 /* helpers used by public routines */
 static inline void _sdma_close_tx(struct hfi1_devdata *dd,
@@ -753,7 +759,7 @@ static inline int sdma_txadd_page(
 		       DMA_TO_DEVICE);
 
 	if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) {
-		sdma_txclean(dd, tx);
+		__sdma_txclean(dd, tx);
 		return -ENOSPC;
 	}
 
@@ -834,7 +840,7 @@ static inline int sdma_txadd_kvaddr(
 		       DMA_TO_DEVICE);
 
 	if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) {
-		sdma_txclean(dd, tx);
+		__sdma_txclean(dd, tx);
 		return -ENOSPC;
 	}
 
diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c
index 5e6d1bac4914..b141a78ae38b 100644
--- a/drivers/infiniband/hw/hfi1/uc.c
+++ b/drivers/infiniband/hw/hfi1/uc.c
@@ -258,8 +258,8 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 	qp->s_len -= len;
 	qp->s_hdrwords = hwords;
 	ps->s_txreq->sde = priv->s_sde;
-	qp->s_cur_sge = &qp->s_sge;
-	qp->s_cur_size = len;
+	ps->s_txreq->ss = &qp->s_sge;
+	ps->s_txreq->s_cur_size = len;
 	hfi1_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24),
 			     mask_psn(qp->s_psn++), middle, ps);
 	/* pbc */
diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c
index 97ae24b6314c..c071955c0272 100644
--- a/drivers/infiniband/hw/hfi1/ud.c
+++ b/drivers/infiniband/hw/hfi1/ud.c
@@ -354,8 +354,8 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 
 	/* header size in 32-bit words LRH+BTH+DETH = (8+12+8)/4. */
 	qp->s_hdrwords = 7;
-	qp->s_cur_size = wqe->length;
-	qp->s_cur_sge = &qp->s_sge;
+	ps->s_txreq->s_cur_size = wqe->length;
+	ps->s_txreq->ss = &qp->s_sge;
 	qp->s_srate = ah_attr->static_rate;
 	qp->srate_mbps = ib_rate_to_mbps(qp->s_srate);
 	qp->s_wqe = wqe;
diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c
index 77697d690f3e..7d22f8ee98ef 100644
--- a/drivers/infiniband/hw/hfi1/user_sdma.c
+++ b/drivers/infiniband/hw/hfi1/user_sdma.c
@@ -115,6 +115,7 @@ MODULE_PARM_DESC(sdma_comp_size, "Size of User SDMA completion ring. Default: 12
 #define KDETH_HCRC_LOWER_MASK     0xff
 
 #define AHG_KDETH_INTR_SHIFT 12
+#define AHG_KDETH_SH_SHIFT   13
 
 #define PBC2LRH(x) ((((x) & 0xfff) << 2) - 4)
 #define LRH2PBC(x) ((((x) >> 2) + 1) & 0xfff)
@@ -144,8 +145,9 @@ MODULE_PARM_DESC(sdma_comp_size, "Size of User SDMA completion ring. Default: 12
 #define KDETH_OM_LARGE     64
 #define KDETH_OM_MAX_SIZE  (1 << ((KDETH_OM_LARGE / KDETH_OM_SMALL) + 1))
 
-/* Last packet in the request */
-#define TXREQ_FLAGS_REQ_LAST_PKT BIT(0)
+/* Tx request flag bits */
+#define TXREQ_FLAGS_REQ_ACK   BIT(0)      /* Set the ACK bit in the header */
+#define TXREQ_FLAGS_REQ_DISABLE_SH BIT(1) /* Disable header suppression */
 
 /* SDMA request flag bits */
 #define SDMA_REQ_FOR_THREAD 1
@@ -943,8 +945,13 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
 		tx->busycount = 0;
 		INIT_LIST_HEAD(&tx->list);
 
+		/*
+		 * For the last packet set the ACK request
+		 * and disable header suppression.
+		 */
 		if (req->seqnum == req->info.npkts - 1)
-			tx->flags |= TXREQ_FLAGS_REQ_LAST_PKT;
+			tx->flags |= (TXREQ_FLAGS_REQ_ACK |
+				      TXREQ_FLAGS_REQ_DISABLE_SH);
 
 		/*
 		 * Calculate the payload size - this is min of the fragment
@@ -963,11 +970,22 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
 			}
 
 			datalen = compute_data_length(req, tx);
+
+			/*
+			 * Disable header suppression for the payload <= 8DWS.
+			 * If there is an uncorrectable error in the receive
+			 * data FIFO when the received payload size is less than
+			 * or equal to 8DWS then the RxDmaDataFifoRdUncErr is
+			 * not reported.There is set RHF.EccErr if the header
+			 * is not suppressed.
+			 */
 			if (!datalen) {
 				SDMA_DBG(req,
 					 "Request has data but pkt len is 0");
 				ret = -EFAULT;
 				goto free_tx;
+			} else if (datalen <= 32) {
+				tx->flags |= TXREQ_FLAGS_REQ_DISABLE_SH;
 			}
 		}
 
@@ -990,6 +1008,10 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
 						LRH2PBC(lrhlen);
 					tx->hdr.pbc[0] = cpu_to_le16(pbclen);
 				}
+				ret = check_header_template(req, &tx->hdr,
+							    lrhlen, datalen);
+				if (ret)
+					goto free_tx;
 				ret = sdma_txinit_ahg(&tx->txreq,
 						      SDMA_TXREQ_F_AHG_COPY,
 						      sizeof(tx->hdr) + datalen,
@@ -1351,7 +1373,7 @@ static int set_txreq_header(struct user_sdma_request *req,
 				req->seqnum));
 
 	/* Set ACK request on last packet */
-	if (unlikely(tx->flags & TXREQ_FLAGS_REQ_LAST_PKT))
+	if (unlikely(tx->flags & TXREQ_FLAGS_REQ_ACK))
 		hdr->bth[2] |= cpu_to_be32(1UL << 31);
 
 	/* Set the new offset */
@@ -1384,8 +1406,8 @@ static int set_txreq_header(struct user_sdma_request *req,
 		/* Set KDETH.TID based on value for this TID */
 		KDETH_SET(hdr->kdeth.ver_tid_offset, TID,
 			  EXP_TID_GET(tidval, IDX));
-		/* Clear KDETH.SH only on the last packet */
-		if (unlikely(tx->flags & TXREQ_FLAGS_REQ_LAST_PKT))
+		/* Clear KDETH.SH when DISABLE_SH flag is set */
+		if (unlikely(tx->flags & TXREQ_FLAGS_REQ_DISABLE_SH))
 			KDETH_SET(hdr->kdeth.ver_tid_offset, SH, 0);
 		/*
 		 * Set the KDETH.OFFSET and KDETH.OM based on size of
@@ -1429,7 +1451,7 @@ static int set_txreq_header_ahg(struct user_sdma_request *req,
 	/* BTH.PSN and BTH.A */
 	val32 = (be32_to_cpu(hdr->bth[2]) + req->seqnum) &
 		(HFI1_CAP_IS_KSET(EXTENDED_PSN) ? 0x7fffffff : 0xffffff);
-	if (unlikely(tx->flags & TXREQ_FLAGS_REQ_LAST_PKT))
+	if (unlikely(tx->flags & TXREQ_FLAGS_REQ_ACK))
 		val32 |= 1UL << 31;
 	AHG_HEADER_SET(req->ahg, diff, 6, 0, 16, cpu_to_be16(val32 >> 16));
 	AHG_HEADER_SET(req->ahg, diff, 6, 16, 16, cpu_to_be16(val32 & 0xffff));
@@ -1468,19 +1490,23 @@ static int set_txreq_header_ahg(struct user_sdma_request *req,
 		AHG_HEADER_SET(req->ahg, diff, 7, 0, 16,
 			       ((!!(req->omfactor - KDETH_OM_SMALL)) << 15 |
 				((req->tidoffset / req->omfactor) & 0x7fff)));
-		/* KDETH.TIDCtrl, KDETH.TID */
+		/* KDETH.TIDCtrl, KDETH.TID, KDETH.Intr, KDETH.SH */
 		val = cpu_to_le16(((EXP_TID_GET(tidval, CTRL) & 0x3) << 10) |
-					(EXP_TID_GET(tidval, IDX) & 0x3ff));
-		/* Clear KDETH.SH on last packet */
-		if (unlikely(tx->flags & TXREQ_FLAGS_REQ_LAST_PKT)) {
-			val |= cpu_to_le16(KDETH_GET(hdr->kdeth.ver_tid_offset,
-						     INTR) <<
-					   AHG_KDETH_INTR_SHIFT);
-			val &= cpu_to_le16(~(1U << 13));
-			AHG_HEADER_SET(req->ahg, diff, 7, 16, 14, val);
+				   (EXP_TID_GET(tidval, IDX) & 0x3ff));
+
+		if (unlikely(tx->flags & TXREQ_FLAGS_REQ_DISABLE_SH)) {
+			val |= cpu_to_le16((KDETH_GET(hdr->kdeth.ver_tid_offset,
+						      INTR) <<
+					    AHG_KDETH_INTR_SHIFT));
 		} else {
-			AHG_HEADER_SET(req->ahg, diff, 7, 16, 12, val);
+			val |= KDETH_GET(hdr->kdeth.ver_tid_offset, SH) ?
+			       cpu_to_le16(0x1 << AHG_KDETH_SH_SHIFT) :
+			       cpu_to_le16((KDETH_GET(hdr->kdeth.ver_tid_offset,
+						      INTR) <<
+					     AHG_KDETH_INTR_SHIFT));
 		}
+
+		AHG_HEADER_SET(req->ahg, diff, 7, 16, 14, val);
 	}
 
 	trace_hfi1_sdma_user_header_ahg(pq->dd, pq->ctxt, pq->subctxt,
diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index 4b7a16ceb362..95ed4d6da510 100644
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -297,22 +297,6 @@ static inline int wss_exceeds_threshold(void)
 }
 
 /*
- * Translate ib_wr_opcode into ib_wc_opcode.
- */
-const enum ib_wc_opcode ib_hfi1_wc_opcode[] = {
-	[IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE,
-	[IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE,
-	[IB_WR_SEND] = IB_WC_SEND,
-	[IB_WR_SEND_WITH_IMM] = IB_WC_SEND,
-	[IB_WR_RDMA_READ] = IB_WC_RDMA_READ,
-	[IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP,
-	[IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD,
-	[IB_WR_SEND_WITH_INV] = IB_WC_SEND,
-	[IB_WR_LOCAL_INV] = IB_WC_LOCAL_INV,
-	[IB_WR_REG_MR] = IB_WC_REG_MR
-};
-
-/*
  * Length of header by opcode, 0 --> not supported
  */
 const u8 hdr_len_by_opcode[256] = {
@@ -694,6 +678,7 @@ static void mem_timer(unsigned long data)
 		qp = iowait_to_qp(wait);
 		priv = qp->priv;
 		list_del_init(&priv->s_iowait.list);
+		priv->s_iowait.lock = NULL;
 		/* refcount held until actual wake up */
 		if (!list_empty(list))
 			mod_timer(&dev->mem_timer, jiffies + 1);
@@ -769,6 +754,7 @@ static int wait_kmem(struct hfi1_ibdev *dev,
 				mod_timer(&dev->mem_timer, jiffies + 1);
 			qp->s_flags |= RVT_S_WAIT_KMEM;
 			list_add_tail(&priv->s_iowait.list, &dev->memwait);
+			priv->s_iowait.lock = &dev->iowait_lock;
 			trace_hfi1_qpsleep(qp, RVT_S_WAIT_KMEM);
 			rvt_get_qp(qp);
 		}
@@ -788,10 +774,10 @@ static int wait_kmem(struct hfi1_ibdev *dev,
  */
 static noinline int build_verbs_ulp_payload(
 	struct sdma_engine *sde,
-	struct rvt_sge_state *ss,
 	u32 length,
 	struct verbs_txreq *tx)
 {
+	struct rvt_sge_state *ss = tx->ss;
 	struct rvt_sge *sg_list = ss->sg_list;
 	struct rvt_sge sge = ss->sge;
 	u8 num_sge = ss->num_sge;
@@ -835,7 +821,6 @@ bail_txadd:
 /* New API */
 static int build_verbs_tx_desc(
 	struct sdma_engine *sde,
-	struct rvt_sge_state *ss,
 	u32 length,
 	struct verbs_txreq *tx,
 	struct hfi1_ahg_info *ahg_info,
@@ -879,9 +864,9 @@ static int build_verbs_tx_desc(
 			goto bail_txadd;
 	}
 
-	/* add the ulp payload - if any.  ss can be NULL for acks */
-	if (ss)
-		ret = build_verbs_ulp_payload(sde, ss, length, tx);
+	/* add the ulp payload - if any. tx->ss can be NULL for acks */
+	if (tx->ss)
+		ret = build_verbs_ulp_payload(sde, length, tx);
 bail_txadd:
 	return ret;
 }
@@ -892,8 +877,7 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 	struct hfi1_qp_priv *priv = qp->priv;
 	struct hfi1_ahg_info *ahg_info = priv->s_ahg;
 	u32 hdrwords = qp->s_hdrwords;
-	struct rvt_sge_state *ss = qp->s_cur_sge;
-	u32 len = qp->s_cur_size;
+	u32 len = ps->s_txreq->s_cur_size;
 	u32 plen = hdrwords + ((len + 3) >> 2) + 2; /* includes pbc */
 	struct hfi1_ibdev *dev = ps->dev;
 	struct hfi1_pportdata *ppd = ps->ppd;
@@ -918,7 +902,7 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 					 plen);
 		}
 		tx->wqe = qp->s_wqe;
-		ret = build_verbs_tx_desc(tx->sde, ss, len, tx, ahg_info, pbc);
+		ret = build_verbs_tx_desc(tx->sde, len, tx, ahg_info, pbc);
 		if (unlikely(ret))
 			goto bail_build;
 	}
@@ -980,6 +964,7 @@ static int pio_wait(struct rvt_qp *qp,
 			qp->s_flags |= flag;
 			was_empty = list_empty(&sc->piowait);
 			list_add_tail(&priv->s_iowait.list, &sc->piowait);
+			priv->s_iowait.lock = &dev->iowait_lock;
 			trace_hfi1_qpsleep(qp, RVT_S_WAIT_PIO);
 			rvt_get_qp(qp);
 			/* counting: only call wantpiobuf_intr if first user */
@@ -1008,8 +993,8 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 {
 	struct hfi1_qp_priv *priv = qp->priv;
 	u32 hdrwords = qp->s_hdrwords;
-	struct rvt_sge_state *ss = qp->s_cur_sge;
-	u32 len = qp->s_cur_size;
+	struct rvt_sge_state *ss = ps->s_txreq->ss;
+	u32 len = ps->s_txreq->s_cur_size;
 	u32 dwords = (len + 3) >> 2;
 	u32 plen = hdrwords + dwords + 2; /* includes pbc */
 	struct hfi1_pportdata *ppd = ps->ppd;
@@ -1237,7 +1222,7 @@ static inline send_routine get_send_routine(struct rvt_qp *qp,
 		u8 op = get_opcode(h);
 
 		if (piothreshold &&
-		    qp->s_cur_size <= min(piothreshold, qp->pmtu) &&
+		    tx->s_cur_size <= min(piothreshold, qp->pmtu) &&
 		    (BIT(op & OPMASK) & pio_opmask[op >> 5]) &&
 		    iowait_sdma_pending(&priv->s_iowait) == 0 &&
 		    !sdma_txreq_built(&tx->txreq))
@@ -1483,15 +1468,11 @@ static int hfi1_get_guid_be(struct rvt_dev_info *rdi, struct rvt_ibport *rvp,
 			    int guid_index, __be64 *guid)
 {
 	struct hfi1_ibport *ibp = container_of(rvp, struct hfi1_ibport, rvp);
-	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
 
-	if (guid_index == 0)
-		*guid = cpu_to_be64(ppd->guid);
-	else if (guid_index < HFI1_GUIDS_PER_PORT)
-		*guid = ibp->guids[guid_index - 1];
-	else
+	if (guid_index >= HFI1_GUIDS_PER_PORT)
 		return -EINVAL;
 
+	*guid = get_sguid(ibp, guid_index);
 	return 0;
 }
 
@@ -1610,6 +1591,154 @@ static void hfi1_get_dev_fw_str(struct ib_device *ibdev, char *str,
 		 dc8051_ver_min(ver));
 }
 
+static const char * const driver_cntr_names[] = {
+	/* must be element 0*/
+	"DRIVER_KernIntr",
+	"DRIVER_ErrorIntr",
+	"DRIVER_Tx_Errs",
+	"DRIVER_Rcv_Errs",
+	"DRIVER_HW_Errs",
+	"DRIVER_NoPIOBufs",
+	"DRIVER_CtxtsOpen",
+	"DRIVER_RcvLen_Errs",
+	"DRIVER_EgrBufFull",
+	"DRIVER_EgrHdrFull"
+};
+
+static const char **dev_cntr_names;
+static const char **port_cntr_names;
+static int num_driver_cntrs = ARRAY_SIZE(driver_cntr_names);
+static int num_dev_cntrs;
+static int num_port_cntrs;
+static int cntr_names_initialized;
+
+/*
+ * Convert a list of names separated by '\n' into an array of NULL terminated
+ * strings. Optionally some entries can be reserved in the array to hold extra
+ * external strings.
+ */
+static int init_cntr_names(const char *names_in,
+			   const int names_len,
+			   int num_extra_names,
+			   int *num_cntrs,
+			   const char ***cntr_names)
+{
+	char *names_out, *p, **q;
+	int i, n;
+
+	n = 0;
+	for (i = 0; i < names_len; i++)
+		if (names_in[i] == '\n')
+			n++;
+
+	names_out = kmalloc((n + num_extra_names) * sizeof(char *) + names_len,
+			    GFP_KERNEL);
+	if (!names_out) {
+		*num_cntrs = 0;
+		*cntr_names = NULL;
+		return -ENOMEM;
+	}
+
+	p = names_out + (n + num_extra_names) * sizeof(char *);
+	memcpy(p, names_in, names_len);
+
+	q = (char **)names_out;
+	for (i = 0; i < n; i++) {
+		q[i] = p;
+		p = strchr(p, '\n');
+		*p++ = '\0';
+	}
+
+	*num_cntrs = n;
+	*cntr_names = (const char **)names_out;
+	return 0;
+}
+
+static struct rdma_hw_stats *alloc_hw_stats(struct ib_device *ibdev,
+					    u8 port_num)
+{
+	int i, err;
+
+	if (!cntr_names_initialized) {
+		struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
+
+		err = init_cntr_names(dd->cntrnames,
+				      dd->cntrnameslen,
+				      num_driver_cntrs,
+				      &num_dev_cntrs,
+				      &dev_cntr_names);
+		if (err)
+			return NULL;
+
+		for (i = 0; i < num_driver_cntrs; i++)
+			dev_cntr_names[num_dev_cntrs + i] =
+				driver_cntr_names[i];
+
+		err = init_cntr_names(dd->portcntrnames,
+				      dd->portcntrnameslen,
+				      0,
+				      &num_port_cntrs,
+				      &port_cntr_names);
+		if (err) {
+			kfree(dev_cntr_names);
+			dev_cntr_names = NULL;
+			return NULL;
+		}
+		cntr_names_initialized = 1;
+	}
+
+	if (!port_num)
+		return rdma_alloc_hw_stats_struct(
+				dev_cntr_names,
+				num_dev_cntrs + num_driver_cntrs,
+				RDMA_HW_STATS_DEFAULT_LIFESPAN);
+	else
+		return rdma_alloc_hw_stats_struct(
+				port_cntr_names,
+				num_port_cntrs,
+				RDMA_HW_STATS_DEFAULT_LIFESPAN);
+}
+
+static u64 hfi1_sps_ints(void)
+{
+	unsigned long flags;
+	struct hfi1_devdata *dd;
+	u64 sps_ints = 0;
+
+	spin_lock_irqsave(&hfi1_devs_lock, flags);
+	list_for_each_entry(dd, &hfi1_dev_list, list) {
+		sps_ints += get_all_cpu_total(dd->int_counter);
+	}
+	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
+	return sps_ints;
+}
+
+static int get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
+			u8 port, int index)
+{
+	u64 *values;
+	int count;
+
+	if (!port) {
+		u64 *stats = (u64 *)&hfi1_stats;
+		int i;
+
+		hfi1_read_cntrs(dd_from_ibdev(ibdev), NULL, &values);
+		values[num_dev_cntrs] = hfi1_sps_ints();
+		for (i = 1; i < num_driver_cntrs; i++)
+			values[num_dev_cntrs + i] = stats[i];
+		count = num_dev_cntrs + num_driver_cntrs;
+	} else {
+		struct hfi1_ibport *ibp = to_iport(ibdev, port);
+
+		hfi1_read_portcntrs(ppd_from_ibp(ibp), NULL, &values);
+		count = num_port_cntrs;
+	}
+
+	memcpy(stats->value, values, count * sizeof(u64));
+	return count;
+}
+
 /**
  * hfi1_register_ib_device - register our device with the infiniband core
  * @dd: the device data structure
@@ -1620,6 +1749,7 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
 	struct hfi1_ibdev *dev = &dd->verbs_dev;
 	struct ib_device *ibdev = &dev->rdi.ibdev;
 	struct hfi1_pportdata *ppd = dd->pport;
+	struct hfi1_ibport *ibp = &ppd->ibport_data;
 	unsigned i;
 	int ret;
 	size_t lcpysz = IB_DEVICE_NAME_MAX;
@@ -1632,6 +1762,7 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
 	setup_timer(&dev->mem_timer, mem_timer, (unsigned long)dev);
 
 	seqlock_init(&dev->iowait_lock);
+	seqlock_init(&dev->txwait_lock);
 	INIT_LIST_HEAD(&dev->txwait);
 	INIT_LIST_HEAD(&dev->memwait);
 
@@ -1639,20 +1770,24 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
 	if (ret)
 		goto err_verbs_txreq;
 
+	/* Use first-port GUID as node guid */
+	ibdev->node_guid = get_sguid(ibp, HFI1_PORT_GUID_INDEX);
+
 	/*
 	 * The system image GUID is supposed to be the same for all
 	 * HFIs in a single system but since there can be other
 	 * device types in the system, we can't be sure this is unique.
 	 */
 	if (!ib_hfi1_sys_image_guid)
-		ib_hfi1_sys_image_guid = cpu_to_be64(ppd->guid);
+		ib_hfi1_sys_image_guid = ibdev->node_guid;
 	lcpysz = strlcpy(ibdev->name, class_name(), lcpysz);
 	strlcpy(ibdev->name + lcpysz, "_%d", IB_DEVICE_NAME_MAX - lcpysz);
 	ibdev->owner = THIS_MODULE;
-	ibdev->node_guid = cpu_to_be64(ppd->guid);
 	ibdev->phys_port_cnt = dd->num_pports;
 	ibdev->dma_device = &dd->pcidev->dev;
 	ibdev->modify_device = modify_device;
+	ibdev->alloc_hw_stats = alloc_hw_stats;
+	ibdev->get_hw_stats = get_hw_stats;
 
 	/* keep process mad in the driver */
 	ibdev->process_mad = hfi1_process_mad;
@@ -1767,6 +1902,10 @@ void hfi1_unregister_ib_device(struct hfi1_devdata *dd)
 
 	del_timer_sync(&dev->mem_timer);
 	verbs_txreq_exit(dev);
+
+	kfree(dev_cntr_names);
+	kfree(port_cntr_names);
+	cntr_names_initialized = 0;
 }
 
 void hfi1_cnp_rcv(struct hfi1_packet *packet)
diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h
index 1c3815d89eb7..e6b893010e6d 100644
--- a/drivers/infiniband/hw/hfi1/verbs.h
+++ b/drivers/infiniband/hw/hfi1/verbs.h
@@ -73,7 +73,6 @@ struct hfi1_packet;
 #include "iowait.h"
 
 #define HFI1_MAX_RDMA_ATOMIC     16
-#define HFI1_GUIDS_PER_PORT	5
 
 /*
  * Increment this value if any changes that break userspace ABI
@@ -169,8 +168,6 @@ struct hfi1_ibport {
 	struct rvt_qp __rcu *qp[2];
 	struct rvt_ibport rvp;
 
-	__be64 guids[HFI1_GUIDS_PER_PORT	- 1];	/* writable GUIDs */
-
 	/* the first 16 entries are sl_to_vl for !OPA */
 	u8 sl_to_sc[32];
 	u8 sc_to_sl[32];
@@ -180,18 +177,19 @@ struct hfi1_ibdev {
 	struct rvt_dev_info rdi; /* Must be first */
 
 	/* QP numbers are shared by all IB ports */
-	/* protect wait lists */
-	seqlock_t iowait_lock;
+	/* protect txwait list */
+	seqlock_t txwait_lock ____cacheline_aligned_in_smp;
 	struct list_head txwait;        /* list for wait verbs_txreq */
 	struct list_head memwait;       /* list for wait kernel memory */
-	struct list_head txreq_free;
 	struct kmem_cache *verbs_txreq_cache;
-	struct timer_list mem_timer;
+	u64 n_txwait;
+	u64 n_kmem_wait;
 
+	/* protect iowait lists */
+	seqlock_t iowait_lock ____cacheline_aligned_in_smp;
 	u64 n_piowait;
 	u64 n_piodrain;
-	u64 n_txwait;
-	u64 n_kmem_wait;
+	struct timer_list mem_timer;
 
 #ifdef CONFIG_DEBUG_FS
 	/* per HFI debugfs */
diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.c b/drivers/infiniband/hw/hfi1/verbs_txreq.c
index 094ab829ec42..5d23172c470f 100644
--- a/drivers/infiniband/hw/hfi1/verbs_txreq.c
+++ b/drivers/infiniband/hw/hfi1/verbs_txreq.c
@@ -72,22 +72,22 @@ void hfi1_put_txreq(struct verbs_txreq *tx)
 	kmem_cache_free(dev->verbs_txreq_cache, tx);
 
 	do {
-		seq = read_seqbegin(&dev->iowait_lock);
+		seq = read_seqbegin(&dev->txwait_lock);
 		if (!list_empty(&dev->txwait)) {
 			struct iowait *wait;
 
-			write_seqlock_irqsave(&dev->iowait_lock, flags);
+			write_seqlock_irqsave(&dev->txwait_lock, flags);
 			wait = list_first_entry(&dev->txwait, struct iowait,
 						list);
 			qp = iowait_to_qp(wait);
 			priv = qp->priv;
 			list_del_init(&priv->s_iowait.list);
 			/* refcount held until actual wake up */
-			write_sequnlock_irqrestore(&dev->iowait_lock, flags);
+			write_sequnlock_irqrestore(&dev->txwait_lock, flags);
 			hfi1_qp_wakeup(qp, RVT_S_WAIT_TX);
 			break;
 		}
-	} while (read_seqretry(&dev->iowait_lock, seq));
+	} while (read_seqretry(&dev->txwait_lock, seq));
 }
 
 struct verbs_txreq *__get_txreq(struct hfi1_ibdev *dev,
@@ -96,7 +96,7 @@ struct verbs_txreq *__get_txreq(struct hfi1_ibdev *dev,
 {
 	struct verbs_txreq *tx = ERR_PTR(-EBUSY);
 
-	write_seqlock(&dev->iowait_lock);
+	write_seqlock(&dev->txwait_lock);
 	if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
 		struct hfi1_qp_priv *priv;
 
@@ -108,13 +108,14 @@ struct verbs_txreq *__get_txreq(struct hfi1_ibdev *dev,
 			dev->n_txwait++;
 			qp->s_flags |= RVT_S_WAIT_TX;
 			list_add_tail(&priv->s_iowait.list, &dev->txwait);
+			priv->s_iowait.lock = &dev->txwait_lock;
 			trace_hfi1_qpsleep(qp, RVT_S_WAIT_TX);
 			rvt_get_qp(qp);
 		}
 		qp->s_flags &= ~RVT_S_BUSY;
 	}
 out:
-	write_sequnlock(&dev->iowait_lock);
+	write_sequnlock(&dev->txwait_lock);
 	return tx;
 }
 
diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.h b/drivers/infiniband/hw/hfi1/verbs_txreq.h
index 5660897593ba..76216f2ef35a 100644
--- a/drivers/infiniband/hw/hfi1/verbs_txreq.h
+++ b/drivers/infiniband/hw/hfi1/verbs_txreq.h
@@ -65,6 +65,7 @@ struct verbs_txreq {
 	struct sdma_engine     *sde;
 	struct send_context     *psc;
 	u16                     hdr_dwords;
+	u16			s_cur_size;
 };
 
 struct hfi1_ibdev;
diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c
index 24f79ee39fdf..0ac294db3b29 100644
--- a/drivers/infiniband/hw/hns/hns_roce_ah.c
+++ b/drivers/infiniband/hw/hns/hns_roce_ah.c
@@ -39,7 +39,8 @@
 #define HNS_ROCE_VLAN_SL_BIT_MASK	7
 #define HNS_ROCE_VLAN_SL_SHIFT		13
 
-struct ib_ah *hns_roce_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *ah_attr)
+struct ib_ah *hns_roce_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *ah_attr,
+				 struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibpd->device);
 	struct device *dev = &hr_dev->pdev->dev;
diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c
index 863a17a2de40..605962f2828c 100644
--- a/drivers/infiniband/hw/hns/hns_roce_alloc.c
+++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c
@@ -61,9 +61,10 @@ int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj)
 	return ret;
 }
 
-void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj)
+void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj,
+			  int rr)
 {
-	hns_roce_bitmap_free_range(bitmap, obj, 1);
+	hns_roce_bitmap_free_range(bitmap, obj, 1, rr);
 }
 
 int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt,
@@ -106,7 +107,8 @@ int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt,
 }
 
 void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap,
-				unsigned long obj, int cnt)
+				unsigned long obj, int cnt,
+				int rr)
 {
 	int i;
 
@@ -116,7 +118,8 @@ void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap,
 	for (i = 0; i < cnt; i++)
 		clear_bit(obj + i, bitmap->table);
 
-	bitmap->last = min(bitmap->last, obj);
+	if (!rr)
+		bitmap->last = min(bitmap->last, obj);
 	bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
 		       & bitmap->mask;
 	spin_unlock(&bitmap->lock);
diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c
index 2a0b6c05da5f..8c1f7a6f84d2 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cmd.c
+++ b/drivers/infiniband/hw/hns/hns_roce_cmd.c
@@ -216,10 +216,10 @@ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,
 		goto out;
 
 	/*
-	* It is timeout when wait_for_completion_timeout return 0
-	* The return value is the time limit set in advance
-	* how many seconds showing
-	*/
+	 * It is timeout when wait_for_completion_timeout return 0
+	 * The return value is the time limit set in advance
+	 * how many seconds showing
+	 */
 	if (!wait_for_completion_timeout(&context->done,
 					 msecs_to_jiffies(timeout))) {
 		dev_err(dev, "[cmd]wait_for_completion_timeout timeout\n");
diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.h b/drivers/infiniband/hw/hns/hns_roce_cmd.h
index e3997d312c55..f5a9ee2fc53d 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cmd.h
+++ b/drivers/infiniband/hw/hns/hns_roce_cmd.h
@@ -34,6 +34,7 @@
 #define _HNS_ROCE_CMD_H
 
 #define HNS_ROCE_MAILBOX_SIZE		4096
+#define HNS_ROCE_CMD_TIMEOUT_MSECS	10000
 
 enum {
 	/* TPT commands */
@@ -57,17 +58,6 @@ enum {
 	HNS_ROCE_CMD_QUERY_QP		= 0x22,
 };
 
-enum {
-	HNS_ROCE_CMD_TIME_CLASS_A	= 10000,
-	HNS_ROCE_CMD_TIME_CLASS_B	= 10000,
-	HNS_ROCE_CMD_TIME_CLASS_C	= 10000,
-};
-
-struct hns_roce_cmd_mailbox {
-	void		       *buf;
-	dma_addr_t		dma;
-};
-
 int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param,
 		      unsigned long in_modifier, u8 op_modifier, u16 op,
 		      unsigned long timeout);
diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h
index 297016103aa7..4af403e1348c 100644
--- a/drivers/infiniband/hw/hns/hns_roce_common.h
+++ b/drivers/infiniband/hw/hns/hns_roce_common.h
@@ -57,6 +57,32 @@
 #define roce_set_bit(origin, shift, val) \
 	roce_set_field((origin), (1ul << (shift)), (shift), (val))
 
+/*
+ * roce_hw_index_cmp_lt - Compare two hardware index values in hisilicon
+ *                        SOC, check if a is less than b.
+ * @a: hardware index value
+ * @b: hardware index value
+ * @bits: the number of bits of a and b, range: 0~31.
+ *
+ * Hardware index increases continuously till max value, and then restart
+ * from zero, again and again. Because the bits of reg field is often
+ * limited, the reg field can only hold the low bits of the hardware index
+ * in hisilicon SOC.
+ * In some scenes we need to compare two values(a,b) getted from two reg
+ * fields in this driver, for example:
+ * If a equals 0xfffe, b equals 0x1 and bits equals 16, we think b has
+ * incresed from 0xffff to 0x1 and a is less than b.
+ * If a equals 0xfffe, b equals 0x0xf001 and bits equals 16, we think a
+ * is bigger than b.
+ *
+ * Return true on a less than b, otherwise false.
+ */
+#define roce_hw_index_mask(bits)	((1ul << (bits)) - 1)
+#define roce_hw_index_shift(bits)	(32 - (bits))
+#define roce_hw_index_cmp_lt(a, b, bits) \
+	((int)((((a) - (b)) & roce_hw_index_mask(bits)) << \
+		roce_hw_index_shift(bits)) < 0)
+
 #define ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S 3
 #define ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S 4
 
@@ -245,16 +271,26 @@
 #define ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M   \
 	(((1UL << 28) - 1) << ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S)
 
+#define ROCEE_SDB_PTR_CMP_BITS 28
+
 #define ROCEE_SDB_INV_CNT_SDB_INV_CNT_S 0
 #define ROCEE_SDB_INV_CNT_SDB_INV_CNT_M   \
 	(((1UL << 16) - 1) << ROCEE_SDB_INV_CNT_SDB_INV_CNT_S)
 
+#define ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S	0
+#define ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M	\
+	(((1UL << 16) - 1) << ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S)
+
+#define ROCEE_SDB_CNT_CMP_BITS 16
+
+#define ROCEE_TSP_BP_ST_QH_FIFO_ENTRY_S	20
+
+#define ROCEE_CNT_CLR_CE_CNT_CLR_CE_S 0
+
 /*************ROCEE_REG DEFINITION****************/
 #define ROCEE_VENDOR_ID_REG			0x0
 #define ROCEE_VENDOR_PART_ID_REG		0x4
 
-#define ROCEE_HW_VERSION_REG			0x8
-
 #define ROCEE_SYS_IMAGE_GUID_L_REG		0xC
 #define ROCEE_SYS_IMAGE_GUID_H_REG		0x10
 
@@ -318,7 +354,11 @@
 
 #define ROCEE_SDB_ISSUE_PTR_REG			0x758
 #define ROCEE_SDB_SEND_PTR_REG			0x75C
+#define ROCEE_CAEP_CQE_WCMD_EMPTY		0x850
+#define ROCEE_SCAEP_WR_CQE_CNT			0x8D0
 #define ROCEE_SDB_INV_CNT_REG			0x9A4
+#define ROCEE_SDB_RETRY_CNT_REG			0x9AC
+#define ROCEE_TSP_BP_ST_REG			0x9EC
 #define ROCEE_ECC_UCERR_ALM0_REG		0xB34
 #define ROCEE_ECC_CERR_ALM0_REG			0xB40
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c
index 097365932b09..589496c8fb9e 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_cq.c
@@ -35,7 +35,7 @@
 #include "hns_roce_device.h"
 #include "hns_roce_cmd.h"
 #include "hns_roce_hem.h"
-#include "hns_roce_user.h"
+#include <rdma/hns-abi.h>
 #include "hns_roce_common.h"
 
 static void hns_roce_ib_cq_comp(struct hns_roce_cq *hr_cq)
@@ -77,7 +77,7 @@ static int hns_roce_sw2hw_cq(struct hns_roce_dev *dev,
 			     unsigned long cq_num)
 {
 	return hns_roce_cmd_mbox(dev, mailbox->dma, 0, cq_num, 0,
-			    HNS_ROCE_CMD_SW2HW_CQ, HNS_ROCE_CMD_TIME_CLASS_A);
+			    HNS_ROCE_CMD_SW2HW_CQ, HNS_ROCE_CMD_TIMEOUT_MSECS);
 }
 
 static int hns_roce_cq_alloc(struct hns_roce_dev *hr_dev, int nent,
@@ -166,7 +166,7 @@ err_put:
 	hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn);
 
 err_out:
-	hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn);
+	hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR);
 	return ret;
 }
 
@@ -176,11 +176,10 @@ static int hns_roce_hw2sw_cq(struct hns_roce_dev *dev,
 {
 	return hns_roce_cmd_mbox(dev, 0, mailbox ? mailbox->dma : 0, cq_num,
 				 mailbox ? 0 : 1, HNS_ROCE_CMD_HW2SW_CQ,
-				 HNS_ROCE_CMD_TIME_CLASS_A);
+				 HNS_ROCE_CMD_TIMEOUT_MSECS);
 }
 
-static void hns_roce_free_cq(struct hns_roce_dev *hr_dev,
-			     struct hns_roce_cq *hr_cq)
+void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
 {
 	struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;
 	struct device *dev = &hr_dev->pdev->dev;
@@ -204,7 +203,7 @@ static void hns_roce_free_cq(struct hns_roce_dev *hr_dev,
 	spin_unlock_irq(&cq_table->lock);
 
 	hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn);
-	hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn);
+	hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR);
 }
 
 static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev,
@@ -349,6 +348,15 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
 		goto err_mtt;
 	}
 
+	/*
+	 * For the QP created by kernel space, tptr value should be initialized
+	 * to zero; For the QP created by user space, it will cause synchronous
+	 * problems if tptr is set to zero here, so we initialze it in user
+	 * space.
+	 */
+	if (!context)
+		*hr_cq->tptr_addr = 0;
+
 	/* Get created cq handler and carry out event */
 	hr_cq->comp = hns_roce_ib_cq_comp;
 	hr_cq->event = hns_roce_ib_cq_event;
@@ -383,19 +391,25 @@ int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
+	int ret = 0;
 
-	hns_roce_free_cq(hr_dev, hr_cq);
-	hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
+	if (hr_dev->hw->destroy_cq) {
+		ret = hr_dev->hw->destroy_cq(ib_cq);
+	} else {
+		hns_roce_free_cq(hr_dev, hr_cq);
+		hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
 
-	if (ib_cq->uobject)
-		ib_umem_release(hr_cq->umem);
-	else
-		/* Free the buff of stored cq */
-		hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, ib_cq->cqe);
+		if (ib_cq->uobject)
+			ib_umem_release(hr_cq->umem);
+		else
+			/* Free the buff of stored cq */
+			hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf,
+						ib_cq->cqe);
 
-	kfree(hr_cq);
+		kfree(hr_cq);
+	}
 
-	return 0;
+	return ret;
 }
 
 void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn)
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index 341731553a60..1a6cb5d7a0dd 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -37,6 +37,8 @@
 
 #define DRV_NAME "hns_roce"
 
+#define HNS_ROCE_HW_VER1	('h' << 24 | 'i' << 16 | '0' << 8 | '6')
+
 #define MAC_ADDR_OCTET_NUM			6
 #define HNS_ROCE_MAX_MSG_LEN			0x80000000
 
@@ -54,6 +56,12 @@
 #define HNS_ROCE_MAX_INNER_MTPT_NUM		0x7
 #define HNS_ROCE_MAX_MTPT_PBL_NUM		0x100000
 
+#define HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS	20
+#define HNS_ROCE_MAX_FREE_CQ_WAIT_CNT	\
+	(5000 / HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS)
+#define HNS_ROCE_CQE_WCMD_EMPTY_BIT		0x2
+#define HNS_ROCE_MIN_CQE_CNT			16
+
 #define HNS_ROCE_MAX_IRQ_NUM			34
 
 #define HNS_ROCE_COMP_VEC_NUM			32
@@ -70,6 +78,9 @@
 #define HNS_ROCE_MAX_GID_NUM			16
 #define HNS_ROCE_GID_SIZE			16
 
+#define BITMAP_NO_RR				0
+#define BITMAP_RR				1
+
 #define MR_TYPE_MR				0x00
 #define MR_TYPE_DMA				0x03
 
@@ -196,9 +207,9 @@ struct hns_roce_bitmap {
 /* Order = 0: bitmap is biggest, order = max bitmap is least (only a bit) */
 /* Every bit repesent to a partner free/used status in bitmap */
 /*
-* Initial, bits of other bitmap are all 0 except that a bit of max_order is 1
-* Bit = 1 represent to idle and available; bit = 0: not available
-*/
+ * Initial, bits of other bitmap are all 0 except that a bit of max_order is 1
+ * Bit = 1 represent to idle and available; bit = 0: not available
+ */
 struct hns_roce_buddy {
 	/* Members point to every order level bitmap */
 	unsigned long **bits;
@@ -296,7 +307,7 @@ struct hns_roce_cq {
 	u32				cq_depth;
 	u32				cons_index;
 	void __iomem			*cq_db_l;
-	void __iomem			*tptr_addr;
+	u16				*tptr_addr;
 	unsigned long			cqn;
 	u32				vector;
 	atomic_t			refcount;
@@ -360,29 +371,34 @@ struct hns_roce_cmdq {
 	struct mutex		hcr_mutex;
 	struct semaphore	poll_sem;
 	/*
-	* Event mode: cmd register mutex protection,
-	* ensure to not exceed max_cmds and user use limit region
-	*/
+	 * Event mode: cmd register mutex protection,
+	 * ensure to not exceed max_cmds and user use limit region
+	 */
 	struct semaphore	event_sem;
 	int			max_cmds;
 	spinlock_t		context_lock;
 	int			free_head;
 	struct hns_roce_cmd_context *context;
 	/*
-	* Result of get integer part
-	* which max_comds compute according a power of 2
-	*/
+	 * Result of get integer part
+	 * which max_comds compute according a power of 2
+	 */
 	u16			token_mask;
 	/*
-	* Process whether use event mode, init default non-zero
-	* After the event queue of cmd event ready,
-	* can switch into event mode
-	* close device, switch into poll mode(non event mode)
-	*/
+	 * Process whether use event mode, init default non-zero
+	 * After the event queue of cmd event ready,
+	 * can switch into event mode
+	 * close device, switch into poll mode(non event mode)
+	 */
 	u8			use_events;
 	u8			toggle;
 };
 
+struct hns_roce_cmd_mailbox {
+	void		       *buf;
+	dma_addr_t		dma;
+};
+
 struct hns_roce_dev;
 
 struct hns_roce_qp {
@@ -424,8 +440,6 @@ struct hns_roce_ib_iboe {
 	struct net_device      *netdevs[HNS_ROCE_MAX_PORTS];
 	struct notifier_block	nb;
 	struct notifier_block	nb_inet;
-	/* 16 GID is shared by 6 port in v1 engine. */
-	union ib_gid		gid_table[HNS_ROCE_MAX_GID_NUM];
 	u8			phy_port[HNS_ROCE_MAX_PORTS];
 };
 
@@ -519,6 +533,8 @@ struct hns_roce_hw {
 			 struct ib_recv_wr **bad_recv_wr);
 	int (*req_notify_cq)(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
 	int (*poll_cq)(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
+	int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr);
+	int (*destroy_cq)(struct ib_cq *ibcq);
 	void	*priv;
 };
 
@@ -553,6 +569,8 @@ struct hns_roce_dev {
 
 	int			cmd_mod;
 	int			loop_idc;
+	dma_addr_t		tptr_dma_addr; /*only for hw v1*/
+	u32			tptr_size; /*only for hw v1*/
 	struct hns_roce_hw	*hw;
 };
 
@@ -657,7 +675,8 @@ void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev);
 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev);
 
 int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj);
-void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj);
+void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj,
+			 int rr);
 int hns_roce_bitmap_init(struct hns_roce_bitmap *bitmap, u32 num, u32 mask,
 			 u32 reserved_bot, u32 resetrved_top);
 void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap);
@@ -665,9 +684,11 @@ void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev);
 int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt,
 				int align, unsigned long *obj);
 void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap,
-				unsigned long obj, int cnt);
+				unsigned long obj, int cnt,
+				int rr);
 
-struct ib_ah *hns_roce_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
+struct ib_ah *hns_roce_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
+				 struct ib_udata *udata);
 int hns_roce_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr);
 int hns_roce_destroy_ah(struct ib_ah *ah);
 
@@ -681,6 +702,10 @@ struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 				   u64 virt_addr, int access_flags,
 				   struct ib_udata *udata);
 int hns_roce_dereg_mr(struct ib_mr *ibmr);
+int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
+		       struct hns_roce_cmd_mailbox *mailbox,
+		       unsigned long mpt_index);
+unsigned long key_to_hw_index(u32 key);
 
 void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size,
 		       struct hns_roce_buf *buf);
@@ -717,6 +742,7 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
 				    struct ib_udata *udata);
 
 int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq);
+void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq);
 
 void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn);
 void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type);
diff --git a/drivers/infiniband/hw/hns/hns_roce_eq.c b/drivers/infiniband/hw/hns/hns_roce_eq.c
index 21e21b03cfb5..50f864935a0e 100644
--- a/drivers/infiniband/hw/hns/hns_roce_eq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_eq.c
@@ -371,9 +371,9 @@ static int hns_roce_aeq_ovf_int(struct hns_roce_dev *hr_dev,
 	int i = 0;
 
 	/**
-	* AEQ overflow ECC mult bit err CEQ overflow alarm
-	* must clear interrupt, mask irq, clear irq, cancel mask operation
-	*/
+	 * AEQ overflow ECC mult bit err CEQ overflow alarm
+	 * must clear interrupt, mask irq, clear irq, cancel mask operation
+	 */
 	aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG);
 
 	if (roce_get_bit(aeshift_val,
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c
index 250d8f280390..c5104e0b2916 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hem.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hem.c
@@ -80,9 +80,9 @@ struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, int npages,
 			--order;
 
 		/*
-		* Alloc memory one time. If failed, don't alloc small block
-		* memory, directly return fail.
-		*/
+		 * Alloc memory one time. If failed, don't alloc small block
+		 * memory, directly return fail.
+		 */
 		mem = &chunk->mem[chunk->npages];
 		buf = dma_alloc_coherent(&hr_dev->pdev->dev, PAGE_SIZE << order,
 				&sg_dma_address(mem), gfp_mask);
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
index 71232e5fabf6..b8111b0c8877 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
@@ -32,6 +32,7 @@
 
 #include <linux/platform_device.h>
 #include <linux/acpi.h>
+#include <linux/etherdevice.h>
 #include <rdma/ib_umem.h>
 #include "hns_roce_common.h"
 #include "hns_roce_device.h"
@@ -72,6 +73,8 @@ int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 	int nreq = 0;
 	u32 ind = 0;
 	int ret = 0;
+	u8 *smac;
+	int loopback;
 
 	if (unlikely(ibqp->qp_type != IB_QPT_GSI &&
 		ibqp->qp_type != IB_QPT_RC)) {
@@ -129,6 +132,14 @@ int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 				       UD_SEND_WQE_U32_8_DMAC_5_M,
 				       UD_SEND_WQE_U32_8_DMAC_5_S,
 				       ah->av.mac[5]);
+
+			smac = (u8 *)hr_dev->dev_addr[qp->port];
+			loopback = ether_addr_equal_unaligned(ah->av.mac,
+							      smac) ? 1 : 0;
+			roce_set_bit(ud_sq_wqe->u32_8,
+				     UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S,
+				     loopback);
+
 			roce_set_field(ud_sq_wqe->u32_8,
 				       UD_SEND_WQE_U32_8_OPERATION_TYPE_M,
 				       UD_SEND_WQE_U32_8_OPERATION_TYPE_S,
@@ -284,6 +295,8 @@ out:
 		roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SQ_HEAD_M,
 			       SQ_DOORBELL_U32_4_SQ_HEAD_S,
 			      (qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)));
+		roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SL_M,
+			       SQ_DOORBELL_U32_4_SL_S, qp->sl);
 		roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_PORT_M,
 			       SQ_DOORBELL_U32_4_PORT_S, qp->phy_port);
 		roce_set_field(sq_db.u32_8, SQ_DOORBELL_U32_8_QPN_M,
@@ -611,6 +624,213 @@ ext_sdb_buf_fail_out:
 	return ret;
 }
 
+static struct hns_roce_qp *hns_roce_v1_create_lp_qp(struct hns_roce_dev *hr_dev,
+						    struct ib_pd *pd)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct ib_qp_init_attr init_attr;
+	struct ib_qp *qp;
+
+	memset(&init_attr, 0, sizeof(struct ib_qp_init_attr));
+	init_attr.qp_type		= IB_QPT_RC;
+	init_attr.sq_sig_type		= IB_SIGNAL_ALL_WR;
+	init_attr.cap.max_recv_wr	= HNS_ROCE_MIN_WQE_NUM;
+	init_attr.cap.max_send_wr	= HNS_ROCE_MIN_WQE_NUM;
+
+	qp = hns_roce_create_qp(pd, &init_attr, NULL);
+	if (IS_ERR(qp)) {
+		dev_err(dev, "Create loop qp for mr free failed!");
+		return NULL;
+	}
+
+	return to_hr_qp(qp);
+}
+
+static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_caps *caps = &hr_dev->caps;
+	struct device *dev = &hr_dev->pdev->dev;
+	struct ib_cq_init_attr cq_init_attr;
+	struct hns_roce_free_mr *free_mr;
+	struct ib_qp_attr attr = { 0 };
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_qp *hr_qp;
+	struct ib_cq *cq;
+	struct ib_pd *pd;
+	u64 subnet_prefix;
+	int attr_mask = 0;
+	int i;
+	int ret;
+	u8 phy_port;
+	u8 sl;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	free_mr = &priv->free_mr;
+
+	/* Reserved cq for loop qp */
+	cq_init_attr.cqe		= HNS_ROCE_MIN_WQE_NUM * 2;
+	cq_init_attr.comp_vector	= 0;
+	cq = hns_roce_ib_create_cq(&hr_dev->ib_dev, &cq_init_attr, NULL, NULL);
+	if (IS_ERR(cq)) {
+		dev_err(dev, "Create cq for reseved loop qp failed!");
+		return -ENOMEM;
+	}
+	free_mr->mr_free_cq = to_hr_cq(cq);
+	free_mr->mr_free_cq->ib_cq.device		= &hr_dev->ib_dev;
+	free_mr->mr_free_cq->ib_cq.uobject		= NULL;
+	free_mr->mr_free_cq->ib_cq.comp_handler		= NULL;
+	free_mr->mr_free_cq->ib_cq.event_handler	= NULL;
+	free_mr->mr_free_cq->ib_cq.cq_context		= NULL;
+	atomic_set(&free_mr->mr_free_cq->ib_cq.usecnt, 0);
+
+	pd = hns_roce_alloc_pd(&hr_dev->ib_dev, NULL, NULL);
+	if (IS_ERR(pd)) {
+		dev_err(dev, "Create pd for reseved loop qp failed!");
+		ret = -ENOMEM;
+		goto alloc_pd_failed;
+	}
+	free_mr->mr_free_pd = to_hr_pd(pd);
+	free_mr->mr_free_pd->ibpd.device  = &hr_dev->ib_dev;
+	free_mr->mr_free_pd->ibpd.uobject = NULL;
+	atomic_set(&free_mr->mr_free_pd->ibpd.usecnt, 0);
+
+	attr.qp_access_flags	= IB_ACCESS_REMOTE_WRITE;
+	attr.pkey_index		= 0;
+	attr.min_rnr_timer	= 0;
+	/* Disable read ability */
+	attr.max_dest_rd_atomic = 0;
+	attr.max_rd_atomic	= 0;
+	/* Use arbitrary values as rq_psn and sq_psn */
+	attr.rq_psn		= 0x0808;
+	attr.sq_psn		= 0x0808;
+	attr.retry_cnt		= 7;
+	attr.rnr_retry		= 7;
+	attr.timeout		= 0x12;
+	attr.path_mtu		= IB_MTU_256;
+	attr.ah_attr.ah_flags		= 1;
+	attr.ah_attr.static_rate	= 3;
+	attr.ah_attr.grh.sgid_index	= 0;
+	attr.ah_attr.grh.hop_limit	= 1;
+	attr.ah_attr.grh.flow_label	= 0;
+	attr.ah_attr.grh.traffic_class	= 0;
+
+	subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
+	for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
+		free_mr->mr_free_qp[i] = hns_roce_v1_create_lp_qp(hr_dev, pd);
+		if (IS_ERR(free_mr->mr_free_qp[i])) {
+			dev_err(dev, "Create loop qp failed!\n");
+			goto create_lp_qp_failed;
+		}
+		hr_qp = free_mr->mr_free_qp[i];
+
+		sl = i / caps->num_ports;
+
+		if (caps->num_ports == HNS_ROCE_MAX_PORTS)
+			phy_port = (i >= HNS_ROCE_MAX_PORTS) ? (i - 2) :
+				(i % caps->num_ports);
+		else
+			phy_port = i % caps->num_ports;
+
+		hr_qp->port		= phy_port + 1;
+		hr_qp->phy_port		= phy_port;
+		hr_qp->ibqp.qp_type	= IB_QPT_RC;
+		hr_qp->ibqp.device	= &hr_dev->ib_dev;
+		hr_qp->ibqp.uobject	= NULL;
+		atomic_set(&hr_qp->ibqp.usecnt, 0);
+		hr_qp->ibqp.pd		= pd;
+		hr_qp->ibqp.recv_cq	= cq;
+		hr_qp->ibqp.send_cq	= cq;
+
+		attr.ah_attr.port_num	= phy_port + 1;
+		attr.ah_attr.sl		= sl;
+		attr.port_num		= phy_port + 1;
+
+		attr.dest_qp_num	= hr_qp->qpn;
+		memcpy(attr.ah_attr.dmac, hr_dev->dev_addr[phy_port],
+		       MAC_ADDR_OCTET_NUM);
+
+		memcpy(attr.ah_attr.grh.dgid.raw,
+			&subnet_prefix, sizeof(u64));
+		memcpy(&attr.ah_attr.grh.dgid.raw[8],
+		       hr_dev->dev_addr[phy_port], 3);
+		memcpy(&attr.ah_attr.grh.dgid.raw[13],
+		       hr_dev->dev_addr[phy_port] + 3, 3);
+		attr.ah_attr.grh.dgid.raw[11] = 0xff;
+		attr.ah_attr.grh.dgid.raw[12] = 0xfe;
+		attr.ah_attr.grh.dgid.raw[8] ^= 2;
+
+		attr_mask |= IB_QP_PORT;
+
+		ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
+					    IB_QPS_RESET, IB_QPS_INIT);
+		if (ret) {
+			dev_err(dev, "modify qp failed(%d)!\n", ret);
+			goto create_lp_qp_failed;
+		}
+
+		ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
+					    IB_QPS_INIT, IB_QPS_RTR);
+		if (ret) {
+			dev_err(dev, "modify qp failed(%d)!\n", ret);
+			goto create_lp_qp_failed;
+		}
+
+		ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
+					    IB_QPS_RTR, IB_QPS_RTS);
+		if (ret) {
+			dev_err(dev, "modify qp failed(%d)!\n", ret);
+			goto create_lp_qp_failed;
+		}
+	}
+
+	return 0;
+
+create_lp_qp_failed:
+	for (i -= 1; i >= 0; i--) {
+		hr_qp = free_mr->mr_free_qp[i];
+		if (hns_roce_v1_destroy_qp(&hr_qp->ibqp))
+			dev_err(dev, "Destroy qp %d for mr free failed!\n", i);
+	}
+
+	if (hns_roce_dealloc_pd(pd))
+		dev_err(dev, "Destroy pd for create_lp_qp failed!\n");
+
+alloc_pd_failed:
+	if (hns_roce_ib_destroy_cq(cq))
+		dev_err(dev, "Destroy cq for create_lp_qp failed!\n");
+
+	return -EINVAL;
+}
+
+static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_free_mr *free_mr;
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_qp *hr_qp;
+	int ret;
+	int i;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	free_mr = &priv->free_mr;
+
+	for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
+		hr_qp = free_mr->mr_free_qp[i];
+		ret = hns_roce_v1_destroy_qp(&hr_qp->ibqp);
+		if (ret)
+			dev_err(dev, "Destroy qp %d for mr free failed(%d)!\n",
+				i, ret);
+	}
+
+	ret = hns_roce_ib_destroy_cq(&free_mr->mr_free_cq->ib_cq);
+	if (ret)
+		dev_err(dev, "Destroy cq for mr_free failed(%d)!\n", ret);
+
+	ret = hns_roce_dealloc_pd(&free_mr->mr_free_pd->ibpd);
+	if (ret)
+		dev_err(dev, "Destroy pd for mr_free failed(%d)!\n", ret);
+}
+
 static int hns_roce_db_init(struct hns_roce_dev *hr_dev)
 {
 	struct device *dev = &hr_dev->pdev->dev;
@@ -648,6 +868,223 @@ static int hns_roce_db_init(struct hns_roce_dev *hr_dev)
 	return 0;
 }
 
+void hns_roce_v1_recreate_lp_qp_work_fn(struct work_struct *work)
+{
+	struct hns_roce_recreate_lp_qp_work *lp_qp_work;
+	struct hns_roce_dev *hr_dev;
+
+	lp_qp_work = container_of(work, struct hns_roce_recreate_lp_qp_work,
+				  work);
+	hr_dev = to_hr_dev(lp_qp_work->ib_dev);
+
+	hns_roce_v1_release_lp_qp(hr_dev);
+
+	if (hns_roce_v1_rsv_lp_qp(hr_dev))
+		dev_err(&hr_dev->pdev->dev, "create reserver qp failed\n");
+
+	if (lp_qp_work->comp_flag)
+		complete(lp_qp_work->comp);
+
+	kfree(lp_qp_work);
+}
+
+static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_recreate_lp_qp_work *lp_qp_work;
+	struct hns_roce_free_mr *free_mr;
+	struct hns_roce_v1_priv *priv;
+	struct completion comp;
+	unsigned long end =
+	  msecs_to_jiffies(HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS) + jiffies;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	free_mr = &priv->free_mr;
+
+	lp_qp_work = kzalloc(sizeof(struct hns_roce_recreate_lp_qp_work),
+			     GFP_KERNEL);
+
+	INIT_WORK(&(lp_qp_work->work), hns_roce_v1_recreate_lp_qp_work_fn);
+
+	lp_qp_work->ib_dev = &(hr_dev->ib_dev);
+	lp_qp_work->comp = &comp;
+	lp_qp_work->comp_flag = 1;
+
+	init_completion(lp_qp_work->comp);
+
+	queue_work(free_mr->free_mr_wq, &(lp_qp_work->work));
+
+	while (time_before_eq(jiffies, end)) {
+		if (try_wait_for_completion(&comp))
+			return 0;
+		msleep(HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE);
+	}
+
+	lp_qp_work->comp_flag = 0;
+	if (try_wait_for_completion(&comp))
+		return 0;
+
+	dev_warn(dev, "recreate lp qp failed 20s timeout and return failed!\n");
+	return -ETIMEDOUT;
+}
+
+static int hns_roce_v1_send_lp_wqe(struct hns_roce_qp *hr_qp)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct ib_send_wr send_wr, *bad_wr;
+	int ret;
+
+	memset(&send_wr, 0, sizeof(send_wr));
+	send_wr.next	= NULL;
+	send_wr.num_sge	= 0;
+	send_wr.send_flags = 0;
+	send_wr.sg_list	= NULL;
+	send_wr.wr_id	= (unsigned long long)&send_wr;
+	send_wr.opcode	= IB_WR_RDMA_WRITE;
+
+	ret = hns_roce_v1_post_send(&hr_qp->ibqp, &send_wr, &bad_wr);
+	if (ret) {
+		dev_err(dev, "Post write wqe for mr free failed(%d)!", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void hns_roce_v1_mr_free_work_fn(struct work_struct *work)
+{
+	struct hns_roce_mr_free_work *mr_work;
+	struct ib_wc wc[HNS_ROCE_V1_RESV_QP];
+	struct hns_roce_free_mr *free_mr;
+	struct hns_roce_cq *mr_free_cq;
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_dev *hr_dev;
+	struct hns_roce_mr *hr_mr;
+	struct hns_roce_qp *hr_qp;
+	struct device *dev;
+	unsigned long end =
+		msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies;
+	int i;
+	int ret;
+	int ne;
+
+	mr_work = container_of(work, struct hns_roce_mr_free_work, work);
+	hr_mr = (struct hns_roce_mr *)mr_work->mr;
+	hr_dev = to_hr_dev(mr_work->ib_dev);
+	dev = &hr_dev->pdev->dev;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	free_mr = &priv->free_mr;
+	mr_free_cq = free_mr->mr_free_cq;
+
+	for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
+		hr_qp = free_mr->mr_free_qp[i];
+		ret = hns_roce_v1_send_lp_wqe(hr_qp);
+		if (ret) {
+			dev_err(dev,
+			     "Send wqe (qp:0x%lx) for mr free failed(%d)!\n",
+			     hr_qp->qpn, ret);
+			goto free_work;
+		}
+	}
+
+	ne = HNS_ROCE_V1_RESV_QP;
+	do {
+		ret = hns_roce_v1_poll_cq(&mr_free_cq->ib_cq, ne, wc);
+		if (ret < 0) {
+			dev_err(dev,
+			   "(qp:0x%lx) starts, Poll cqe failed(%d) for mr 0x%x free! Remain %d cqe\n",
+			   hr_qp->qpn, ret, hr_mr->key, ne);
+			goto free_work;
+		}
+		ne -= ret;
+		msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE);
+	} while (ne && time_before_eq(jiffies, end));
+
+	if (ne != 0)
+		dev_err(dev,
+			"Poll cqe for mr 0x%x free timeout! Remain %d cqe\n",
+			hr_mr->key, ne);
+
+free_work:
+	if (mr_work->comp_flag)
+		complete(mr_work->comp);
+	kfree(mr_work);
+}
+
+int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_mr_free_work *mr_work;
+	struct hns_roce_free_mr *free_mr;
+	struct hns_roce_v1_priv *priv;
+	struct completion comp;
+	unsigned long end =
+		msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies;
+	unsigned long start = jiffies;
+	int npages;
+	int ret = 0;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	free_mr = &priv->free_mr;
+
+	if (mr->enabled) {
+		if (hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key)
+				       & (hr_dev->caps.num_mtpts - 1)))
+			dev_warn(dev, "HW2SW_MPT failed!\n");
+	}
+
+	mr_work = kzalloc(sizeof(*mr_work), GFP_KERNEL);
+	if (!mr_work) {
+		ret = -ENOMEM;
+		goto free_mr;
+	}
+
+	INIT_WORK(&(mr_work->work), hns_roce_v1_mr_free_work_fn);
+
+	mr_work->ib_dev = &(hr_dev->ib_dev);
+	mr_work->comp = &comp;
+	mr_work->comp_flag = 1;
+	mr_work->mr = (void *)mr;
+	init_completion(mr_work->comp);
+
+	queue_work(free_mr->free_mr_wq, &(mr_work->work));
+
+	while (time_before_eq(jiffies, end)) {
+		if (try_wait_for_completion(&comp))
+			goto free_mr;
+		msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE);
+	}
+
+	mr_work->comp_flag = 0;
+	if (try_wait_for_completion(&comp))
+		goto free_mr;
+
+	dev_warn(dev, "Free mr work 0x%x over 50s and failed!\n", mr->key);
+	ret = -ETIMEDOUT;
+
+free_mr:
+	dev_dbg(dev, "Free mr 0x%x use 0x%x us.\n",
+		mr->key, jiffies_to_usecs(jiffies) - jiffies_to_usecs(start));
+
+	if (mr->size != ~0ULL) {
+		npages = ib_umem_page_count(mr->umem);
+		dma_free_coherent(dev, npages * 8, mr->pbl_buf,
+				  mr->pbl_dma_addr);
+	}
+
+	hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
+			     key_to_hw_index(mr->key), 0);
+
+	if (mr->umem)
+		ib_umem_release(mr->umem);
+
+	kfree(mr);
+
+	return ret;
+}
+
 static void hns_roce_db_free(struct hns_roce_dev *hr_dev)
 {
 	struct device *dev = &hr_dev->pdev->dev;
@@ -849,6 +1286,85 @@ static void hns_roce_bt_free(struct hns_roce_dev *hr_dev)
 		priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map);
 }
 
+static int hns_roce_tptr_init(struct hns_roce_dev *hr_dev)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_buf_list *tptr_buf;
+	struct hns_roce_v1_priv *priv;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	tptr_buf = &priv->tptr_table.tptr_buf;
+
+	/*
+	 * This buffer will be used for CQ's tptr(tail pointer), also
+	 * named ci(customer index). Every CQ will use 2 bytes to save
+	 * cqe ci in hip06. Hardware will read this area to get new ci
+	 * when the queue is almost full.
+	 */
+	tptr_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE,
+					   &tptr_buf->map, GFP_KERNEL);
+	if (!tptr_buf->buf)
+		return -ENOMEM;
+
+	hr_dev->tptr_dma_addr = tptr_buf->map;
+	hr_dev->tptr_size = HNS_ROCE_V1_TPTR_BUF_SIZE;
+
+	return 0;
+}
+
+static void hns_roce_tptr_free(struct hns_roce_dev *hr_dev)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_buf_list *tptr_buf;
+	struct hns_roce_v1_priv *priv;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	tptr_buf = &priv->tptr_table.tptr_buf;
+
+	dma_free_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE,
+			  tptr_buf->buf, tptr_buf->map);
+}
+
+static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_free_mr *free_mr;
+	struct hns_roce_v1_priv *priv;
+	int ret = 0;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	free_mr = &priv->free_mr;
+
+	free_mr->free_mr_wq = create_singlethread_workqueue("hns_roce_free_mr");
+	if (!free_mr->free_mr_wq) {
+		dev_err(dev, "Create free mr workqueue failed!\n");
+		return -ENOMEM;
+	}
+
+	ret = hns_roce_v1_rsv_lp_qp(hr_dev);
+	if (ret) {
+		dev_err(dev, "Reserved loop qp failed(%d)!\n", ret);
+		flush_workqueue(free_mr->free_mr_wq);
+		destroy_workqueue(free_mr->free_mr_wq);
+	}
+
+	return ret;
+}
+
+static void hns_roce_free_mr_free(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_free_mr *free_mr;
+	struct hns_roce_v1_priv *priv;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	free_mr = &priv->free_mr;
+
+	flush_workqueue(free_mr->free_mr_wq);
+	destroy_workqueue(free_mr->free_mr_wq);
+
+	hns_roce_v1_release_lp_qp(hr_dev);
+}
+
 /**
  * hns_roce_v1_reset - reset RoCE
  * @hr_dev: RoCE device struct pointer
@@ -898,6 +1414,38 @@ int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset)
 	return ret;
 }
 
+static int hns_roce_des_qp_init(struct hns_roce_dev *hr_dev)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_des_qp *des_qp;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	des_qp = &priv->des_qp;
+
+	des_qp->requeue_flag = 1;
+	des_qp->qp_wq = create_singlethread_workqueue("hns_roce_destroy_qp");
+	if (!des_qp->qp_wq) {
+		dev_err(dev, "Create destroy qp workqueue failed!\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void hns_roce_des_qp_free(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_des_qp *des_qp;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	des_qp = &priv->des_qp;
+
+	des_qp->requeue_flag = 0;
+	flush_workqueue(des_qp->qp_wq);
+	destroy_workqueue(des_qp->qp_wq);
+}
+
 void hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
 {
 	int i = 0;
@@ -906,12 +1454,11 @@ void hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
 	hr_dev->vendor_id = le32_to_cpu(roce_read(hr_dev, ROCEE_VENDOR_ID_REG));
 	hr_dev->vendor_part_id = le32_to_cpu(roce_read(hr_dev,
 					     ROCEE_VENDOR_PART_ID_REG));
-	hr_dev->hw_rev = le32_to_cpu(roce_read(hr_dev, ROCEE_HW_VERSION_REG));
-
 	hr_dev->sys_image_guid = le32_to_cpu(roce_read(hr_dev,
 					     ROCEE_SYS_IMAGE_GUID_L_REG)) |
 				((u64)le32_to_cpu(roce_read(hr_dev,
 					    ROCEE_SYS_IMAGE_GUID_H_REG)) << 32);
+	hr_dev->hw_rev		= HNS_ROCE_HW_VER1;
 
 	caps->num_qps		= HNS_ROCE_V1_MAX_QP_NUM;
 	caps->max_wqes		= HNS_ROCE_V1_MAX_WQE_NUM;
@@ -1001,18 +1548,44 @@ int hns_roce_v1_init(struct hns_roce_dev *hr_dev)
 		goto error_failed_raq_init;
 	}
 
-	hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP);
-
 	ret = hns_roce_bt_init(hr_dev);
 	if (ret) {
 		dev_err(dev, "bt init failed!\n");
 		goto error_failed_bt_init;
 	}
 
+	ret = hns_roce_tptr_init(hr_dev);
+	if (ret) {
+		dev_err(dev, "tptr init failed!\n");
+		goto error_failed_tptr_init;
+	}
+
+	ret = hns_roce_des_qp_init(hr_dev);
+	if (ret) {
+		dev_err(dev, "des qp init failed!\n");
+		goto error_failed_des_qp_init;
+	}
+
+	ret = hns_roce_free_mr_init(hr_dev);
+	if (ret) {
+		dev_err(dev, "free mr init failed!\n");
+		goto error_failed_free_mr_init;
+	}
+
+	hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP);
+
 	return 0;
 
+error_failed_free_mr_init:
+	hns_roce_des_qp_free(hr_dev);
+
+error_failed_des_qp_init:
+	hns_roce_tptr_free(hr_dev);
+
+error_failed_tptr_init:
+	hns_roce_bt_free(hr_dev);
+
 error_failed_bt_init:
-	hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
 	hns_roce_raq_free(hr_dev);
 
 error_failed_raq_init:
@@ -1022,8 +1595,11 @@ error_failed_raq_init:
 
 void hns_roce_v1_exit(struct hns_roce_dev *hr_dev)
 {
-	hns_roce_bt_free(hr_dev);
 	hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
+	hns_roce_free_mr_free(hr_dev);
+	hns_roce_des_qp_free(hr_dev);
+	hns_roce_tptr_free(hr_dev);
+	hns_roce_bt_free(hr_dev);
 	hns_roce_raq_free(hr_dev);
 	hns_roce_db_free(hr_dev);
 }
@@ -1061,6 +1637,14 @@ void hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr)
 	u32 *p;
 	u32 val;
 
+	/*
+	 * When mac changed, loopback may fail
+	 * because of smac not equal to dmac.
+	 * We Need to release and create reserved qp again.
+	 */
+	if (hr_dev->hw->dereg_mr && hns_roce_v1_recreate_lp_qp(hr_dev))
+		dev_warn(&hr_dev->pdev->dev, "recreate lp qp timeout!\n");
+
 	p = (u32 *)(&addr[0]);
 	reg_smac_l = *p;
 	roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_SMAC_L_0_REG +
@@ -1293,9 +1877,9 @@ static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
 	}
 
 	/*
-	* Now backwards through the CQ, removing CQ entries
-	* that match our QP by overwriting them with next entries.
-	*/
+	 * Now backwards through the CQ, removing CQ entries
+	 * that match our QP by overwriting them with next entries.
+	 */
 	while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
 		cqe = get_cqe(hr_cq, prod_index & hr_cq->ib_cq.cqe);
 		if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
@@ -1317,9 +1901,9 @@ static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
 	if (nfreed) {
 		hr_cq->cons_index += nfreed;
 		/*
-		* Make sure update of buffer contents is done before
-		* updating consumer index.
-		*/
+		 * Make sure update of buffer contents is done before
+		 * updating consumer index.
+		 */
 		wmb();
 
 		hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index);
@@ -1339,14 +1923,21 @@ void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev,
 			   dma_addr_t dma_handle, int nent, u32 vector)
 {
 	struct hns_roce_cq_context *cq_context = NULL;
-	void __iomem *tptr_addr;
+	struct hns_roce_buf_list *tptr_buf;
+	struct hns_roce_v1_priv *priv;
+	dma_addr_t tptr_dma_addr;
+	int offset;
+
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	tptr_buf = &priv->tptr_table.tptr_buf;
 
 	cq_context = mb_buf;
 	memset(cq_context, 0, sizeof(*cq_context));
 
-	tptr_addr = 0;
-	hr_dev->priv_addr = tptr_addr;
-	hr_cq->tptr_addr = tptr_addr;
+	/* Get the tptr for this CQ. */
+	offset = hr_cq->cqn * HNS_ROCE_V1_TPTR_ENTRY_SIZE;
+	tptr_dma_addr = tptr_buf->map + offset;
+	hr_cq->tptr_addr = (u16 *)(tptr_buf->buf + offset);
 
 	/* Register cq_context members */
 	roce_set_field(cq_context->cqc_byte_4,
@@ -1390,10 +1981,10 @@ void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev,
 	roce_set_field(cq_context->cqc_byte_20,
 		       CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M,
 		       CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S,
-		       (u64)tptr_addr >> 44);
+		       tptr_dma_addr >> 44);
 	cq_context->cqc_byte_20 = cpu_to_le32(cq_context->cqc_byte_20);
 
-	cq_context->cqe_tptr_addr_l = (u32)((u64)tptr_addr >> 12);
+	cq_context->cqe_tptr_addr_l = (u32)(tptr_dma_addr >> 12);
 
 	roce_set_field(cq_context->cqc_byte_32,
 		       CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M,
@@ -1407,7 +1998,7 @@ void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev,
 	roce_set_bit(cq_context->cqc_byte_32,
 		     CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S,
 		     0);
-	/*The initial value of cq's ci is 0 */
+	/* The initial value of cq's ci is 0 */
 	roce_set_field(cq_context->cqc_byte_32,
 		       CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M,
 		       CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S, 0);
@@ -1424,9 +2015,9 @@ int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 	notification_flag = (flags & IB_CQ_SOLICITED_MASK) ==
 			    IB_CQ_SOLICITED ? CQ_DB_REQ_NOT : CQ_DB_REQ_NOT_SOL;
 	/*
-	* flags = 0; Notification Flag = 1, next
-	* flags = 1; Notification Flag = 0, solocited
-	*/
+	 * flags = 0; Notification Flag = 1, next
+	 * flags = 1; Notification Flag = 0, solocited
+	 */
 	doorbell[0] = hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1);
 	roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
 	roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
@@ -1581,10 +2172,10 @@ static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
 		wq = &(*cur_qp)->sq;
 		if ((*cur_qp)->sq_signal_bits) {
 			/*
-			* If sg_signal_bit is 1,
-			* firstly tail pointer updated to wqe
-			* which current cqe correspond to
-			*/
+			 * If sg_signal_bit is 1,
+			 * firstly tail pointer updated to wqe
+			 * which current cqe correspond to
+			 */
 			wqe_ctr = (u16)roce_get_field(cqe->cqe_byte_4,
 						      CQE_BYTE_4_WQE_INDEX_M,
 						      CQE_BYTE_4_WQE_INDEX_S);
@@ -1659,8 +2250,14 @@ int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 			break;
 	}
 
-	if (npolled)
+	if (npolled) {
+		*hr_cq->tptr_addr = hr_cq->cons_index &
+			((hr_cq->cq_depth << 1) - 1);
+
+		/* Memroy barrier */
+		wmb();
 		hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index);
+	}
 
 	spin_unlock_irqrestore(&hr_cq->lock, flags);
 
@@ -1799,12 +2396,12 @@ static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev,
 	if (op[cur_state][new_state] == HNS_ROCE_CMD_2RST_QP)
 		return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
 					 HNS_ROCE_CMD_2RST_QP,
-					 HNS_ROCE_CMD_TIME_CLASS_A);
+					 HNS_ROCE_CMD_TIMEOUT_MSECS);
 
 	if (op[cur_state][new_state] == HNS_ROCE_CMD_2ERR_QP)
 		return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
 					 HNS_ROCE_CMD_2ERR_QP,
-					 HNS_ROCE_CMD_TIME_CLASS_A);
+					 HNS_ROCE_CMD_TIMEOUT_MSECS);
 
 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
 	if (IS_ERR(mailbox))
@@ -1814,7 +2411,7 @@ static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev,
 
 	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
 				op[cur_state][new_state],
-				HNS_ROCE_CMD_TIME_CLASS_C);
+				HNS_ROCE_CMD_TIMEOUT_MSECS);
 
 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
 	return ret;
@@ -2000,11 +2597,11 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 	}
 
 	/*
-	*Reset to init
-	*	Mandatory param:
-	*	IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS
-	*	Optional param: NA
-	*/
+	 * Reset to init
+	 *	Mandatory param:
+	 *	IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS
+	 *	Optional param: NA
+	 */
 	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
 		roce_set_field(context->qpc_bytes_4,
 			       QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
@@ -2172,24 +2769,14 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 			     QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S,
 			     hr_qp->sq_signal_bits);
 
-		for (port = 0; port < hr_dev->caps.num_ports; port++) {
-			smac = (u8 *)hr_dev->dev_addr[port];
-			dev_dbg(dev, "smac: %2x: %2x: %2x: %2x: %2x: %2x\n",
-				smac[0], smac[1], smac[2], smac[3], smac[4],
-				smac[5]);
-			if ((dmac[0] == smac[0]) && (dmac[1] == smac[1]) &&
-			    (dmac[2] == smac[2]) && (dmac[3] == smac[3]) &&
-			    (dmac[4] == smac[4]) && (dmac[5] == smac[5])) {
-				roce_set_bit(context->qpc_bytes_32,
-				    QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S,
-				    1);
-				break;
-			}
-		}
-
-		if (hr_dev->loop_idc == 0x1)
+		port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) :
+			hr_qp->port;
+		smac = (u8 *)hr_dev->dev_addr[port];
+		/* when dmac equals smac or loop_idc is 1, it should loopback */
+		if (ether_addr_equal_unaligned(dmac, smac) ||
+		    hr_dev->loop_idc == 0x1)
 			roce_set_bit(context->qpc_bytes_32,
-				QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S, 1);
+			      QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S, 1);
 
 		roce_set_bit(context->qpc_bytes_32,
 			     QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S,
@@ -2509,7 +3096,7 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 	/* Every status migrate must change state */
 	roce_set_field(context->qpc_bytes_144,
 		       QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
-		       QP_CONTEXT_QPC_BYTES_144_QP_STATE_S, attr->qp_state);
+		       QP_CONTEXT_QPC_BYTES_144_QP_STATE_S, new_state);
 
 	/* SW pass context to HW */
 	ret = hns_roce_v1_qp_modify(hr_dev, &hr_qp->mtt,
@@ -2522,9 +3109,9 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 	}
 
 	/*
-	* Use rst2init to instead of init2init with drv,
-	* need to hw to flash RQ HEAD by DB again
-	*/
+	 * Use rst2init to instead of init2init with drv,
+	 * need to hw to flash RQ HEAD by DB again
+	 */
 	if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
 		/* Memory barrier */
 		wmb();
@@ -2619,7 +3206,7 @@ static int hns_roce_v1_query_qpc(struct hns_roce_dev *hr_dev,
 
 	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
 				HNS_ROCE_CMD_QUERY_QP,
-				HNS_ROCE_CMD_TIME_CLASS_A);
+				HNS_ROCE_CMD_TIMEOUT_MSECS);
 	if (!ret)
 		memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
 	else
@@ -2630,8 +3217,78 @@ static int hns_roce_v1_query_qpc(struct hns_roce_dev *hr_dev,
 	return ret;
 }
 
-int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
-			 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
+static int hns_roce_v1_q_sqp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+			     int qp_attr_mask,
+			     struct ib_qp_init_attr *qp_init_attr)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+	struct hns_roce_sqp_context context;
+	u32 addr;
+
+	mutex_lock(&hr_qp->mutex);
+
+	if (hr_qp->state == IB_QPS_RESET) {
+		qp_attr->qp_state = IB_QPS_RESET;
+		goto done;
+	}
+
+	addr = ROCEE_QP1C_CFG0_0_REG +
+		hr_qp->port * sizeof(struct hns_roce_sqp_context);
+	context.qp1c_bytes_4 = roce_read(hr_dev, addr);
+	context.sq_rq_bt_l = roce_read(hr_dev, addr + 1);
+	context.qp1c_bytes_12 = roce_read(hr_dev, addr + 2);
+	context.qp1c_bytes_16 = roce_read(hr_dev, addr + 3);
+	context.qp1c_bytes_20 = roce_read(hr_dev, addr + 4);
+	context.cur_rq_wqe_ba_l = roce_read(hr_dev, addr + 5);
+	context.qp1c_bytes_28 = roce_read(hr_dev, addr + 6);
+	context.qp1c_bytes_32 = roce_read(hr_dev, addr + 7);
+	context.cur_sq_wqe_ba_l = roce_read(hr_dev, addr + 8);
+	context.qp1c_bytes_40 = roce_read(hr_dev, addr + 9);
+
+	hr_qp->state = roce_get_field(context.qp1c_bytes_4,
+				      QP1C_BYTES_4_QP_STATE_M,
+				      QP1C_BYTES_4_QP_STATE_S);
+	qp_attr->qp_state	= hr_qp->state;
+	qp_attr->path_mtu	= IB_MTU_256;
+	qp_attr->path_mig_state	= IB_MIG_ARMED;
+	qp_attr->qkey		= QKEY_VAL;
+	qp_attr->rq_psn		= 0;
+	qp_attr->sq_psn		= 0;
+	qp_attr->dest_qp_num	= 1;
+	qp_attr->qp_access_flags = 6;
+
+	qp_attr->pkey_index = roce_get_field(context.qp1c_bytes_20,
+					     QP1C_BYTES_20_PKEY_IDX_M,
+					     QP1C_BYTES_20_PKEY_IDX_S);
+	qp_attr->port_num = hr_qp->port + 1;
+	qp_attr->sq_draining = 0;
+	qp_attr->max_rd_atomic = 0;
+	qp_attr->max_dest_rd_atomic = 0;
+	qp_attr->min_rnr_timer = 0;
+	qp_attr->timeout = 0;
+	qp_attr->retry_cnt = 0;
+	qp_attr->rnr_retry = 0;
+	qp_attr->alt_timeout = 0;
+
+done:
+	qp_attr->cur_qp_state = qp_attr->qp_state;
+	qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
+	qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
+	qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
+	qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
+	qp_attr->cap.max_inline_data = 0;
+	qp_init_attr->cap = qp_attr->cap;
+	qp_init_attr->create_flags = 0;
+
+	mutex_unlock(&hr_qp->mutex);
+
+	return 0;
+}
+
+static int hns_roce_v1_q_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+			    int qp_attr_mask,
+			    struct ib_qp_init_attr *qp_init_attr)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
@@ -2725,9 +3382,7 @@ int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 	qp_attr->pkey_index = roce_get_field(context->qpc_bytes_12,
 			      QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
 			      QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S);
-	qp_attr->port_num = (u8)roce_get_field(context->qpc_bytes_156,
-			     QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
-			     QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S) + 1;
+	qp_attr->port_num = hr_qp->port + 1;
 	qp_attr->sq_draining = 0;
 	qp_attr->max_rd_atomic = roce_get_field(context->qpc_bytes_156,
 				 QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
@@ -2767,134 +3422,397 @@ out:
 	return ret;
 }
 
-static void hns_roce_v1_destroy_qp_common(struct hns_roce_dev *hr_dev,
-					  struct hns_roce_qp *hr_qp,
-					  int is_user)
+int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+			 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
+{
+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+
+	return hr_qp->doorbell_qpn <= 1 ?
+		hns_roce_v1_q_sqp(ibqp, qp_attr, qp_attr_mask, qp_init_attr) :
+		hns_roce_v1_q_qp(ibqp, qp_attr, qp_attr_mask, qp_init_attr);
+}
+
+static int check_qp_db_process_status(struct hns_roce_dev *hr_dev,
+				      struct hns_roce_qp *hr_qp,
+				      u32 sdb_issue_ptr,
+				      u32 *sdb_inv_cnt,
+				      u32 *wait_stage)
 {
-	u32 sdbinvcnt;
-	unsigned long end = 0;
-	u32 sdbinvcnt_val;
-	u32 sdbsendptr_val;
-	u32 sdbisusepr_val;
-	struct hns_roce_cq *send_cq, *recv_cq;
 	struct device *dev = &hr_dev->pdev->dev;
+	u32 sdb_retry_cnt, old_retry;
+	u32 sdb_send_ptr, old_send;
+	u32 success_flags = 0;
+	u32 cur_cnt, old_cnt;
+	unsigned long end;
+	u32 send_ptr;
+	u32 inv_cnt;
+	u32 tsp_st;
+
+	if (*wait_stage > HNS_ROCE_V1_DB_STAGE2 ||
+	    *wait_stage < HNS_ROCE_V1_DB_STAGE1) {
+		dev_err(dev, "QP(0x%lx) db status wait stage(%d) error!\n",
+			hr_qp->qpn, *wait_stage);
+		return -EINVAL;
+	}
 
-	if (hr_qp->ibqp.qp_type == IB_QPT_RC) {
-		if (hr_qp->state != IB_QPS_RESET) {
-			/*
-			* Set qp to ERR,
-			* waiting for hw complete processing all dbs
-			*/
-			if (hns_roce_v1_qp_modify(hr_dev, NULL,
-					to_hns_roce_state(
-						(enum ib_qp_state)hr_qp->state),
-						HNS_ROCE_QP_STATE_ERR, NULL,
-						hr_qp))
-				dev_err(dev, "modify QP %06lx to ERR failed.\n",
-					hr_qp->qpn);
-
-			/* Record issued doorbell */
-			sdbisusepr_val = roce_read(hr_dev,
-					 ROCEE_SDB_ISSUE_PTR_REG);
-			/*
-			* Query db process status,
-			* until hw process completely
-			*/
-			end = msecs_to_jiffies(
-			      HNS_ROCE_QP_DESTROY_TIMEOUT_MSECS) + jiffies;
-			do {
-				sdbsendptr_val = roce_read(hr_dev,
+	/* Calculate the total timeout for the entire verification process */
+	end = msecs_to_jiffies(HNS_ROCE_V1_CHECK_DB_TIMEOUT_MSECS) + jiffies;
+
+	if (*wait_stage == HNS_ROCE_V1_DB_STAGE1) {
+		/* Query db process status, until hw process completely */
+		sdb_send_ptr = roce_read(hr_dev, ROCEE_SDB_SEND_PTR_REG);
+		while (roce_hw_index_cmp_lt(sdb_send_ptr, sdb_issue_ptr,
+					    ROCEE_SDB_PTR_CMP_BITS)) {
+			if (!time_before(jiffies, end)) {
+				dev_dbg(dev, "QP(0x%lx) db process stage1 timeout. issue 0x%x send 0x%x.\n",
+					hr_qp->qpn, sdb_issue_ptr,
+					sdb_send_ptr);
+				return 0;
+			}
+
+			msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS);
+			sdb_send_ptr = roce_read(hr_dev,
 						 ROCEE_SDB_SEND_PTR_REG);
-				if (!time_before(jiffies, end)) {
-					dev_err(dev, "destroy qp(0x%lx) timeout!!!",
-						hr_qp->qpn);
-					break;
-				}
-			} while ((short)(roce_get_field(sdbsendptr_val,
-					ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
-					ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) -
-				roce_get_field(sdbisusepr_val,
-					ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M,
-					ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S)
-				) < 0);
+		}
 
-			/* Get list pointer */
-			sdbinvcnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
+		if (roce_get_field(sdb_issue_ptr,
+				   ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M,
+				   ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S) ==
+		    roce_get_field(sdb_send_ptr,
+				   ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
+				   ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S)) {
+			old_send = roce_read(hr_dev, ROCEE_SDB_SEND_PTR_REG);
+			old_retry = roce_read(hr_dev, ROCEE_SDB_RETRY_CNT_REG);
 
-			/* Query db's list status, until hw reversal */
 			do {
-				sdbinvcnt_val = roce_read(hr_dev,
-						ROCEE_SDB_INV_CNT_REG);
+				tsp_st = roce_read(hr_dev, ROCEE_TSP_BP_ST_REG);
+				if (roce_get_bit(tsp_st,
+					ROCEE_TSP_BP_ST_QH_FIFO_ENTRY_S) == 1) {
+					*wait_stage = HNS_ROCE_V1_DB_WAIT_OK;
+					return 0;
+				}
+
 				if (!time_before(jiffies, end)) {
-					dev_err(dev, "destroy qp(0x%lx) timeout!!!",
-						hr_qp->qpn);
-					dev_err(dev, "SdbInvCnt = 0x%x\n",
-						sdbinvcnt_val);
-					break;
+					dev_dbg(dev, "QP(0x%lx) db process stage1 timeout when send ptr equals issue ptr.\n"
+						     "issue 0x%x send 0x%x.\n",
+						hr_qp->qpn, sdb_issue_ptr,
+						sdb_send_ptr);
+					return 0;
+				}
+
+				msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS);
+
+				sdb_send_ptr = roce_read(hr_dev,
+							ROCEE_SDB_SEND_PTR_REG);
+				sdb_retry_cnt =	roce_read(hr_dev,
+						       ROCEE_SDB_RETRY_CNT_REG);
+				cur_cnt = roce_get_field(sdb_send_ptr,
+					ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
+					ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) +
+					roce_get_field(sdb_retry_cnt,
+					ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M,
+					ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S);
+				if (!roce_get_bit(tsp_st,
+					ROCEE_CNT_CLR_CE_CNT_CLR_CE_S)) {
+					old_cnt = roce_get_field(old_send,
+					ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
+					ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) +
+					roce_get_field(old_retry,
+					ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M,
+					ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S);
+					if (cur_cnt - old_cnt > SDB_ST_CMP_VAL)
+						success_flags = 1;
+				} else {
+					old_cnt = roce_get_field(old_send,
+					ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
+					ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S);
+					if (cur_cnt - old_cnt > SDB_ST_CMP_VAL)
+						success_flags = 1;
+					else {
+					    send_ptr = roce_get_field(old_send,
+					    ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
+					    ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) +
+					    roce_get_field(sdb_retry_cnt,
+					    ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M,
+					    ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S);
+					    roce_set_field(old_send,
+					    ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
+					    ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S,
+						send_ptr);
+					}
 				}
-			} while ((short)(roce_get_field(sdbinvcnt_val,
-				  ROCEE_SDB_INV_CNT_SDB_INV_CNT_M,
-				  ROCEE_SDB_INV_CNT_SDB_INV_CNT_S) -
-				  (sdbinvcnt + SDB_INV_CNT_OFFSET)) < 0);
-
-			/* Modify qp to reset before destroying qp */
-			if (hns_roce_v1_qp_modify(hr_dev, NULL,
-					to_hns_roce_state(
-					(enum ib_qp_state)hr_qp->state),
-					HNS_ROCE_QP_STATE_RST, NULL, hr_qp))
-				dev_err(dev, "modify QP %06lx to RESET failed.\n",
-					hr_qp->qpn);
+			} while (!success_flags);
 		}
+
+		*wait_stage = HNS_ROCE_V1_DB_STAGE2;
+
+		/* Get list pointer */
+		*sdb_inv_cnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
+		dev_dbg(dev, "QP(0x%lx) db process stage2. inv cnt = 0x%x.\n",
+			hr_qp->qpn, *sdb_inv_cnt);
+	}
+
+	if (*wait_stage == HNS_ROCE_V1_DB_STAGE2) {
+		/* Query db's list status, until hw reversal */
+		inv_cnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
+		while (roce_hw_index_cmp_lt(inv_cnt,
+					    *sdb_inv_cnt + SDB_INV_CNT_OFFSET,
+					    ROCEE_SDB_CNT_CMP_BITS)) {
+			if (!time_before(jiffies, end)) {
+				dev_dbg(dev, "QP(0x%lx) db process stage2 timeout. inv cnt 0x%x.\n",
+					hr_qp->qpn, inv_cnt);
+				return 0;
+			}
+
+			msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS);
+			inv_cnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
+		}
+
+		*wait_stage = HNS_ROCE_V1_DB_WAIT_OK;
+	}
+
+	return 0;
+}
+
+static int check_qp_reset_state(struct hns_roce_dev *hr_dev,
+				struct hns_roce_qp *hr_qp,
+				struct hns_roce_qp_work *qp_work_entry,
+				int *is_timeout)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	u32 sdb_issue_ptr;
+	int ret;
+
+	if (hr_qp->state != IB_QPS_RESET) {
+		/* Set qp to ERR, waiting for hw complete processing all dbs */
+		ret = hns_roce_v1_modify_qp(&hr_qp->ibqp, NULL, 0, hr_qp->state,
+					    IB_QPS_ERR);
+		if (ret) {
+			dev_err(dev, "Modify QP(0x%lx) to ERR failed!\n",
+				hr_qp->qpn);
+			return ret;
+		}
+
+		/* Record issued doorbell */
+		sdb_issue_ptr = roce_read(hr_dev, ROCEE_SDB_ISSUE_PTR_REG);
+		qp_work_entry->sdb_issue_ptr = sdb_issue_ptr;
+		qp_work_entry->db_wait_stage = HNS_ROCE_V1_DB_STAGE1;
+
+		/* Query db process status, until hw process completely */
+		ret = check_qp_db_process_status(hr_dev, hr_qp, sdb_issue_ptr,
+						 &qp_work_entry->sdb_inv_cnt,
+						 &qp_work_entry->db_wait_stage);
+		if (ret) {
+			dev_err(dev, "Check QP(0x%lx) db process status failed!\n",
+				hr_qp->qpn);
+			return ret;
+		}
+
+		if (qp_work_entry->db_wait_stage != HNS_ROCE_V1_DB_WAIT_OK) {
+			qp_work_entry->sche_cnt = 0;
+			*is_timeout = 1;
+			return 0;
+		}
+
+		/* Modify qp to reset before destroying qp */
+		ret = hns_roce_v1_modify_qp(&hr_qp->ibqp, NULL, 0, hr_qp->state,
+					    IB_QPS_RESET);
+		if (ret) {
+			dev_err(dev, "Modify QP(0x%lx) to RST failed!\n",
+				hr_qp->qpn);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void hns_roce_v1_destroy_qp_work_fn(struct work_struct *work)
+{
+	struct hns_roce_qp_work *qp_work_entry;
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_dev *hr_dev;
+	struct hns_roce_qp *hr_qp;
+	struct device *dev;
+	int ret;
+
+	qp_work_entry = container_of(work, struct hns_roce_qp_work, work);
+	hr_dev = to_hr_dev(qp_work_entry->ib_dev);
+	dev = &hr_dev->pdev->dev;
+	priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+	hr_qp = qp_work_entry->qp;
+
+	dev_dbg(dev, "Schedule destroy QP(0x%lx) work.\n", hr_qp->qpn);
+
+	qp_work_entry->sche_cnt++;
+
+	/* Query db process status, until hw process completely */
+	ret = check_qp_db_process_status(hr_dev, hr_qp,
+					 qp_work_entry->sdb_issue_ptr,
+					 &qp_work_entry->sdb_inv_cnt,
+					 &qp_work_entry->db_wait_stage);
+	if (ret) {
+		dev_err(dev, "Check QP(0x%lx) db process status failed!\n",
+			hr_qp->qpn);
+		return;
+	}
+
+	if (qp_work_entry->db_wait_stage != HNS_ROCE_V1_DB_WAIT_OK &&
+	    priv->des_qp.requeue_flag) {
+		queue_work(priv->des_qp.qp_wq, work);
+		return;
+	}
+
+	/* Modify qp to reset before destroying qp */
+	ret = hns_roce_v1_modify_qp(&hr_qp->ibqp, NULL, 0, hr_qp->state,
+				    IB_QPS_RESET);
+	if (ret) {
+		dev_err(dev, "Modify QP(0x%lx) to RST failed!\n", hr_qp->qpn);
+		return;
+	}
+
+	hns_roce_qp_remove(hr_dev, hr_qp);
+	hns_roce_qp_free(hr_dev, hr_qp);
+
+	if (hr_qp->ibqp.qp_type == IB_QPT_RC) {
+		/* RC QP, release QPN */
+		hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
+		kfree(hr_qp);
+	} else
+		kfree(hr_to_hr_sqp(hr_qp));
+
+	kfree(qp_work_entry);
+
+	dev_dbg(dev, "Accomplished destroy QP(0x%lx) work.\n", hr_qp->qpn);
+}
+
+int hns_roce_v1_destroy_qp(struct ib_qp *ibqp)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_qp_work qp_work_entry;
+	struct hns_roce_qp_work *qp_work;
+	struct hns_roce_v1_priv *priv;
+	struct hns_roce_cq *send_cq, *recv_cq;
+	int is_user = !!ibqp->pd->uobject;
+	int is_timeout = 0;
+	int ret;
+
+	ret = check_qp_reset_state(hr_dev, hr_qp, &qp_work_entry, &is_timeout);
+	if (ret) {
+		dev_err(dev, "QP reset state check failed(%d)!\n", ret);
+		return ret;
 	}
 
 	send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
 	recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
 
 	hns_roce_lock_cqs(send_cq, recv_cq);
-
 	if (!is_user) {
 		__hns_roce_v1_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
 				       to_hr_srq(hr_qp->ibqp.srq) : NULL);
 		if (send_cq != recv_cq)
 			__hns_roce_v1_cq_clean(send_cq, hr_qp->qpn, NULL);
 	}
-
-	hns_roce_qp_remove(hr_dev, hr_qp);
-
 	hns_roce_unlock_cqs(send_cq, recv_cq);
 
-	hns_roce_qp_free(hr_dev, hr_qp);
+	if (!is_timeout) {
+		hns_roce_qp_remove(hr_dev, hr_qp);
+		hns_roce_qp_free(hr_dev, hr_qp);
 
-	/* Not special_QP, free their QPN */
-	if ((hr_qp->ibqp.qp_type == IB_QPT_RC) ||
-	    (hr_qp->ibqp.qp_type == IB_QPT_UC) ||
-	    (hr_qp->ibqp.qp_type == IB_QPT_UD))
-		hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
+		/* RC QP, release QPN */
+		if (hr_qp->ibqp.qp_type == IB_QPT_RC)
+			hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
+	}
 
 	hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
 
-	if (is_user) {
+	if (is_user)
 		ib_umem_release(hr_qp->umem);
-	} else {
+	else {
 		kfree(hr_qp->sq.wrid);
 		kfree(hr_qp->rq.wrid);
+
 		hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
 	}
+
+	if (!is_timeout) {
+		if (hr_qp->ibqp.qp_type == IB_QPT_RC)
+			kfree(hr_qp);
+		else
+			kfree(hr_to_hr_sqp(hr_qp));
+	} else {
+		qp_work = kzalloc(sizeof(*qp_work), GFP_KERNEL);
+		if (!qp_work)
+			return -ENOMEM;
+
+		INIT_WORK(&qp_work->work, hns_roce_v1_destroy_qp_work_fn);
+		qp_work->ib_dev	= &hr_dev->ib_dev;
+		qp_work->qp		= hr_qp;
+		qp_work->db_wait_stage	= qp_work_entry.db_wait_stage;
+		qp_work->sdb_issue_ptr	= qp_work_entry.sdb_issue_ptr;
+		qp_work->sdb_inv_cnt	= qp_work_entry.sdb_inv_cnt;
+		qp_work->sche_cnt	= qp_work_entry.sche_cnt;
+
+		priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
+		queue_work(priv->des_qp.qp_wq, &qp_work->work);
+		dev_dbg(dev, "Begin destroy QP(0x%lx) work.\n", hr_qp->qpn);
+	}
+
+	return 0;
 }
 
-int hns_roce_v1_destroy_qp(struct ib_qp *ibqp)
+int hns_roce_v1_destroy_cq(struct ib_cq *ibcq)
 {
-	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
-	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
+	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
+	struct device *dev = &hr_dev->pdev->dev;
+	u32 cqe_cnt_ori;
+	u32 cqe_cnt_cur;
+	u32 cq_buf_size;
+	int wait_time = 0;
+	int ret = 0;
 
-	hns_roce_v1_destroy_qp_common(hr_dev, hr_qp, !!ibqp->pd->uobject);
+	hns_roce_free_cq(hr_dev, hr_cq);
 
-	if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
-		kfree(hr_to_hr_sqp(hr_qp));
-	else
-		kfree(hr_qp);
+	/*
+	 * Before freeing cq buffer, we need to ensure that the outstanding CQE
+	 * have been written by checking the CQE counter.
+	 */
+	cqe_cnt_ori = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT);
+	while (1) {
+		if (roce_read(hr_dev, ROCEE_CAEP_CQE_WCMD_EMPTY) &
+		    HNS_ROCE_CQE_WCMD_EMPTY_BIT)
+			break;
 
-	return 0;
+		cqe_cnt_cur = roce_read(hr_dev, ROCEE_SCAEP_WR_CQE_CNT);
+		if ((cqe_cnt_cur - cqe_cnt_ori) >= HNS_ROCE_MIN_CQE_CNT)
+			break;
+
+		msleep(HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS);
+		if (wait_time > HNS_ROCE_MAX_FREE_CQ_WAIT_CNT) {
+			dev_warn(dev, "Destroy cq 0x%lx timeout!\n",
+				hr_cq->cqn);
+			ret = -ETIMEDOUT;
+			break;
+		}
+		wait_time++;
+	}
+
+	hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
+
+	if (ibcq->uobject)
+		ib_umem_release(hr_cq->umem);
+	else {
+		/* Free the buff of stored cq */
+		cq_buf_size = (ibcq->cqe + 1) * hr_dev->caps.cq_entry_sz;
+		hns_roce_buf_free(hr_dev, cq_buf_size, &hr_cq->hr_buf.hr_buf);
+	}
+
+	kfree(hr_cq);
+
+	return ret;
 }
 
 struct hns_roce_v1_priv hr_v1_priv;
@@ -2917,5 +3835,7 @@ struct hns_roce_hw hns_roce_hw_v1 = {
 	.post_recv = hns_roce_v1_post_recv,
 	.req_notify_cq = hns_roce_v1_req_notify_cq,
 	.poll_cq = hns_roce_v1_poll_cq,
+	.dereg_mr = hns_roce_v1_dereg_mr,
+	.destroy_cq = hns_roce_v1_destroy_cq,
 	.priv = &hr_v1_priv,
 };
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
index 539b0a3b92b0..b213b5e6fef1 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
@@ -58,6 +58,7 @@
 #define HNS_ROCE_V1_PHY_UAR_NUM				8
 
 #define HNS_ROCE_V1_GID_NUM				16
+#define HNS_ROCE_V1_RESV_QP				8
 
 #define HNS_ROCE_V1_NUM_COMP_EQE			0x8000
 #define HNS_ROCE_V1_NUM_ASYNC_EQE			0x400
@@ -102,8 +103,22 @@
 #define HNS_ROCE_V1_EXT_ODB_ALFUL	\
 	(HNS_ROCE_V1_EXT_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD)
 
+#define HNS_ROCE_V1_DB_WAIT_OK				0
+#define HNS_ROCE_V1_DB_STAGE1				1
+#define HNS_ROCE_V1_DB_STAGE2				2
+#define HNS_ROCE_V1_CHECK_DB_TIMEOUT_MSECS		10000
+#define HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS		20
+#define HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS		50000
+#define HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS	10000
+#define HNS_ROCE_V1_FREE_MR_WAIT_VALUE			5
+#define HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE		20
+
 #define HNS_ROCE_BT_RSV_BUF_SIZE			(1 << 17)
 
+#define HNS_ROCE_V1_TPTR_ENTRY_SIZE			2
+#define HNS_ROCE_V1_TPTR_BUF_SIZE	\
+	(HNS_ROCE_V1_TPTR_ENTRY_SIZE * HNS_ROCE_V1_MAX_CQ_NUM)
+
 #define HNS_ROCE_ODB_POLL_MODE				0
 
 #define HNS_ROCE_SDB_NORMAL_MODE			0
@@ -140,6 +155,7 @@
 #define SQ_PSN_SHIFT					8
 #define QKEY_VAL					0x80010000
 #define SDB_INV_CNT_OFFSET				8
+#define SDB_ST_CMP_VAL					8
 
 struct hns_roce_cq_context {
 	u32 cqc_byte_4;
@@ -436,6 +452,8 @@ struct hns_roce_ud_send_wqe {
 #define UD_SEND_WQE_U32_8_DMAC_5_M   \
 	(((1UL << 8) - 1) << UD_SEND_WQE_U32_8_DMAC_5_S)
 
+#define UD_SEND_WQE_U32_8_LOOPBACK_INDICATOR_S 22
+
 #define UD_SEND_WQE_U32_8_OPERATION_TYPE_S 16
 #define UD_SEND_WQE_U32_8_OPERATION_TYPE_M   \
 	(((1UL << 4) - 1) << UD_SEND_WQE_U32_8_OPERATION_TYPE_S)
@@ -480,13 +498,17 @@ struct hns_roce_sqp_context {
 	u32 qp1c_bytes_12;
 	u32 qp1c_bytes_16;
 	u32 qp1c_bytes_20;
-	u32 qp1c_bytes_28;
 	u32 cur_rq_wqe_ba_l;
+	u32 qp1c_bytes_28;
 	u32 qp1c_bytes_32;
 	u32 cur_sq_wqe_ba_l;
 	u32 qp1c_bytes_40;
 };
 
+#define QP1C_BYTES_4_QP_STATE_S 0
+#define QP1C_BYTES_4_QP_STATE_M   \
+	(((1UL << 3) - 1) << QP1C_BYTES_4_QP_STATE_S)
+
 #define QP1C_BYTES_4_SQ_WQE_SHIFT_S 8
 #define QP1C_BYTES_4_SQ_WQE_SHIFT_M   \
 	(((1UL << 4) - 1) << QP1C_BYTES_4_SQ_WQE_SHIFT_S)
@@ -952,6 +974,10 @@ struct hns_roce_sq_db {
 #define SQ_DOORBELL_U32_4_SQ_HEAD_M   \
 	(((1UL << 15) - 1) << SQ_DOORBELL_U32_4_SQ_HEAD_S)
 
+#define SQ_DOORBELL_U32_4_SL_S 16
+#define SQ_DOORBELL_U32_4_SL_M   \
+	(((1UL << 2) - 1) << SQ_DOORBELL_U32_4_SL_S)
+
 #define SQ_DOORBELL_U32_4_PORT_S 18
 #define SQ_DOORBELL_U32_4_PORT_M  (((1UL << 3) - 1) << SQ_DOORBELL_U32_4_PORT_S)
 
@@ -979,12 +1005,58 @@ struct hns_roce_bt_table {
 	struct hns_roce_buf_list cqc_buf;
 };
 
+struct hns_roce_tptr_table {
+	struct hns_roce_buf_list tptr_buf;
+};
+
+struct hns_roce_qp_work {
+	struct	work_struct work;
+	struct	ib_device *ib_dev;
+	struct	hns_roce_qp *qp;
+	u32	db_wait_stage;
+	u32	sdb_issue_ptr;
+	u32	sdb_inv_cnt;
+	u32	sche_cnt;
+};
+
+struct hns_roce_des_qp {
+	struct workqueue_struct	*qp_wq;
+	int	requeue_flag;
+};
+
+struct hns_roce_mr_free_work {
+	struct	work_struct work;
+	struct	ib_device *ib_dev;
+	struct	completion *comp;
+	int	comp_flag;
+	void	*mr;
+};
+
+struct hns_roce_recreate_lp_qp_work {
+	struct	work_struct work;
+	struct	ib_device *ib_dev;
+	struct	completion *comp;
+	int	comp_flag;
+};
+
+struct hns_roce_free_mr {
+	struct workqueue_struct *free_mr_wq;
+	struct hns_roce_qp *mr_free_qp[HNS_ROCE_V1_RESV_QP];
+	struct hns_roce_cq *mr_free_cq;
+	struct hns_roce_pd *mr_free_pd;
+};
+
 struct hns_roce_v1_priv {
 	struct hns_roce_db_table  db_table;
 	struct hns_roce_raq_table raq_table;
 	struct hns_roce_bt_table  bt_table;
+	struct hns_roce_tptr_table tptr_table;
+	struct hns_roce_des_qp des_qp;
+	struct hns_roce_free_mr free_mr;
 };
 
 int hns_dsaf_roce_reset(struct fwnode_handle *dsaf_fwnode, bool dereset);
+int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
+int hns_roce_v1_destroy_qp(struct ib_qp *ibqp);
 
 #endif
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 764e35a54457..4953d9cb83a7 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -35,52 +35,13 @@
 #include <rdma/ib_addr.h>
 #include <rdma/ib_smi.h>
 #include <rdma/ib_user_verbs.h>
+#include <rdma/ib_cache.h>
 #include "hns_roce_common.h"
 #include "hns_roce_device.h"
-#include "hns_roce_user.h"
+#include <rdma/hns-abi.h>
 #include "hns_roce_hem.h"
 
 /**
- * hns_roce_addrconf_ifid_eui48 - Get default gid.
- * @eui: eui.
- * @vlan_id:  gid
- * @dev:  net device
- * Description:
- *    MAC convert to GID
- *        gid[0..7] = fe80 0000 0000 0000
- *        gid[8] = mac[0] ^ 2
- *        gid[9] = mac[1]
- *        gid[10] = mac[2]
- *        gid[11] = ff        (VLAN ID high byte (4 MS bits))
- *        gid[12] = fe        (VLAN ID low byte)
- *        gid[13] = mac[3]
- *        gid[14] = mac[4]
- *        gid[15] = mac[5]
- */
-static void hns_roce_addrconf_ifid_eui48(u8 *eui, u16 vlan_id,
-					 struct net_device *dev)
-{
-	memcpy(eui, dev->dev_addr, 3);
-	memcpy(eui + 5, dev->dev_addr + 3, 3);
-	if (vlan_id < 0x1000) {
-		eui[3] = vlan_id >> 8;
-		eui[4] = vlan_id & 0xff;
-	} else {
-		eui[3] = 0xff;
-		eui[4] = 0xfe;
-	}
-	eui[0] ^= 2;
-}
-
-static void hns_roce_make_default_gid(struct net_device *dev, union ib_gid *gid)
-{
-	memset(gid, 0, sizeof(*gid));
-	gid->raw[0] = 0xFE;
-	gid->raw[1] = 0x80;
-	hns_roce_addrconf_ifid_eui48(&gid->raw[8], 0xffff, dev);
-}
-
-/**
  * hns_get_gid_index - Get gid index.
  * @hr_dev: pointer to structure hns_roce_dev.
  * @port:  port, value range: 0 ~ MAX
@@ -96,30 +57,6 @@ int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index)
 	return gid_index * hr_dev->caps.num_ports + port;
 }
 
-static int hns_roce_set_gid(struct hns_roce_dev *hr_dev, u8 port, int gid_index,
-		     union ib_gid *gid)
-{
-	struct device *dev = &hr_dev->pdev->dev;
-	u8 gid_idx = 0;
-
-	if (gid_index >= hr_dev->caps.gid_table_len[port]) {
-		dev_err(dev, "gid_index %d illegal, port %d gid range: 0~%d\n",
-			gid_index, port, hr_dev->caps.gid_table_len[port] - 1);
-		return -EINVAL;
-	}
-
-	gid_idx = hns_get_gid_index(hr_dev, port, gid_index);
-
-	if (!memcmp(gid, &hr_dev->iboe.gid_table[gid_idx], sizeof(*gid)))
-		return -EINVAL;
-
-	memcpy(&hr_dev->iboe.gid_table[gid_idx], gid, sizeof(*gid));
-
-	hr_dev->hw->set_gid(hr_dev, port, gid_index, gid);
-
-	return 0;
-}
-
 static void hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr)
 {
 	u8 phy_port;
@@ -135,27 +72,44 @@ static void hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr)
 	hr_dev->hw->set_mac(hr_dev, phy_port, addr);
 }
 
-static void hns_roce_set_mtu(struct hns_roce_dev *hr_dev, u8 port, int mtu)
+static int hns_roce_add_gid(struct ib_device *device, u8 port_num,
+			    unsigned int index, const union ib_gid *gid,
+			    const struct ib_gid_attr *attr, void **context)
 {
-	u8 phy_port = hr_dev->iboe.phy_port[port];
-	enum ib_mtu tmp;
+	struct hns_roce_dev *hr_dev = to_hr_dev(device);
+	u8 port = port_num - 1;
+	unsigned long flags;
+
+	if (port >= hr_dev->caps.num_ports)
+		return -EINVAL;
+
+	spin_lock_irqsave(&hr_dev->iboe.lock, flags);
 
-	tmp = iboe_get_mtu(mtu);
-	if (!tmp)
-		tmp = IB_MTU_256;
+	hr_dev->hw->set_gid(hr_dev, port, index, (union ib_gid *)gid);
 
-	hr_dev->hw->set_mtu(hr_dev, phy_port, tmp);
+	spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
+
+	return 0;
 }
 
-static void hns_roce_update_gids(struct hns_roce_dev *hr_dev, int port)
+static int hns_roce_del_gid(struct ib_device *device, u8 port_num,
+			    unsigned int index, void **context)
 {
-	struct ib_event event;
+	struct hns_roce_dev *hr_dev = to_hr_dev(device);
+	union ib_gid zgid = { {0} };
+	u8 port = port_num - 1;
+	unsigned long flags;
+
+	if (port >= hr_dev->caps.num_ports)
+		return -EINVAL;
+
+	spin_lock_irqsave(&hr_dev->iboe.lock, flags);
 
-	/* Refresh gid in ib_cache */
-	event.device = &hr_dev->ib_dev;
-	event.element.port_num = port + 1;
-	event.event = IB_EVENT_GID_CHANGE;
-	ib_dispatch_event(&event);
+	hr_dev->hw->set_gid(hr_dev, port, index, &zgid);
+
+	spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
+
+	return 0;
 }
 
 static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port,
@@ -163,9 +117,6 @@ static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port,
 {
 	struct device *dev = &hr_dev->pdev->dev;
 	struct net_device *netdev;
-	unsigned long flags;
-	union ib_gid gid;
-	int ret = 0;
 
 	netdev = hr_dev->iboe.netdevs[port];
 	if (!netdev) {
@@ -173,7 +124,7 @@ static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port,
 		return -ENODEV;
 	}
 
-	spin_lock_irqsave(&hr_dev->iboe.lock, flags);
+	spin_lock_bh(&hr_dev->iboe.lock);
 
 	switch (event) {
 	case NETDEV_UP:
@@ -181,23 +132,19 @@ static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port,
 	case NETDEV_REGISTER:
 	case NETDEV_CHANGEADDR:
 		hns_roce_set_mac(hr_dev, port, netdev->dev_addr);
-		hns_roce_make_default_gid(netdev, &gid);
-		ret = hns_roce_set_gid(hr_dev, port, 0, &gid);
-		if (!ret)
-			hns_roce_update_gids(hr_dev, port);
 		break;
 	case NETDEV_DOWN:
 		/*
-		* In v1 engine, only support all ports closed together.
-		*/
+		 * In v1 engine, only support all ports closed together.
+		 */
 		break;
 	default:
 		dev_dbg(dev, "NETDEV event = 0x%x!\n", (u32)(event));
 		break;
 	}
 
-	spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
-	return ret;
+	spin_unlock_bh(&hr_dev->iboe.lock);
+	return 0;
 }
 
 static int hns_roce_netdev_event(struct notifier_block *self,
@@ -224,118 +171,17 @@ static int hns_roce_netdev_event(struct notifier_block *self,
 	return NOTIFY_DONE;
 }
 
-static void hns_roce_addr_event(int event, struct net_device *event_netdev,
-				struct hns_roce_dev *hr_dev, union ib_gid *gid)
+static int hns_roce_setup_mtu_mac(struct hns_roce_dev *hr_dev)
 {
-	struct hns_roce_ib_iboe *iboe = NULL;
-	int gid_table_len = 0;
-	unsigned long flags;
-	union ib_gid zgid;
-	u8 gid_idx = 0;
-	u8 port = 0;
-	int i = 0;
-	int free;
-	struct net_device *real_dev = rdma_vlan_dev_real_dev(event_netdev) ?
-				      rdma_vlan_dev_real_dev(event_netdev) :
-				      event_netdev;
-
-	if (event != NETDEV_UP && event != NETDEV_DOWN)
-		return;
-
-	iboe = &hr_dev->iboe;
-	while (port < hr_dev->caps.num_ports) {
-		if (real_dev == iboe->netdevs[port])
-			break;
-		port++;
-	}
-
-	if (port >= hr_dev->caps.num_ports) {
-		dev_dbg(&hr_dev->pdev->dev, "can't find netdev\n");
-		return;
-	}
-
-	memset(zgid.raw, 0, sizeof(zgid.raw));
-	free = -1;
-	gid_table_len = hr_dev->caps.gid_table_len[port];
-
-	spin_lock_irqsave(&hr_dev->iboe.lock, flags);
-
-	for (i = 0; i < gid_table_len; i++) {
-		gid_idx = hns_get_gid_index(hr_dev, port, i);
-		if (!memcmp(gid->raw, iboe->gid_table[gid_idx].raw,
-			    sizeof(gid->raw)))
-			break;
-		if (free < 0 && !memcmp(zgid.raw,
-			iboe->gid_table[gid_idx].raw, sizeof(zgid.raw)))
-			free = i;
-	}
-
-	if (i >= gid_table_len) {
-		if (free < 0) {
-			spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
-			dev_dbg(&hr_dev->pdev->dev,
-				"gid_index overflow, port(%d)\n", port);
-			return;
-		}
-		if (!hns_roce_set_gid(hr_dev, port, free, gid))
-			hns_roce_update_gids(hr_dev, port);
-	} else if (event == NETDEV_DOWN) {
-		if (!hns_roce_set_gid(hr_dev, port, i, &zgid))
-			hns_roce_update_gids(hr_dev, port);
-	}
-
-	spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
-}
-
-static int hns_roce_inet_event(struct notifier_block *self, unsigned long event,
-			       void *ptr)
-{
-	struct in_ifaddr *ifa = ptr;
-	struct hns_roce_dev *hr_dev;
-	struct net_device *dev = ifa->ifa_dev->dev;
-	union ib_gid gid;
-
-	ipv6_addr_set_v4mapped(ifa->ifa_address, (struct in6_addr *)&gid);
-
-	hr_dev = container_of(self, struct hns_roce_dev, iboe.nb_inet);
-
-	hns_roce_addr_event(event, dev, hr_dev, &gid);
-
-	return NOTIFY_DONE;
-}
-
-static int hns_roce_setup_mtu_gids(struct hns_roce_dev *hr_dev)
-{
-	struct in_ifaddr *ifa_list = NULL;
-	union ib_gid gid = {{0} };
-	u32 ipaddr = 0;
-	int index = 0;
-	int ret = 0;
-	u8 i = 0;
+	u8 i;
 
 	for (i = 0; i < hr_dev->caps.num_ports; i++) {
-		hns_roce_set_mtu(hr_dev, i,
-				 ib_mtu_enum_to_int(hr_dev->caps.max_mtu));
+		hr_dev->hw->set_mtu(hr_dev, hr_dev->iboe.phy_port[i],
+				    hr_dev->caps.max_mtu);
 		hns_roce_set_mac(hr_dev, i, hr_dev->iboe.netdevs[i]->dev_addr);
-
-		if (hr_dev->iboe.netdevs[i]->ip_ptr) {
-			ifa_list = hr_dev->iboe.netdevs[i]->ip_ptr->ifa_list;
-			index = 1;
-			while (ifa_list) {
-				ipaddr = ifa_list->ifa_address;
-				ipv6_addr_set_v4mapped(ipaddr,
-						       (struct in6_addr *)&gid);
-				ret = hns_roce_set_gid(hr_dev, i, index, &gid);
-				if (ret)
-					break;
-				index++;
-				ifa_list = ifa_list->ifa_next;
-			}
-			hns_roce_update_gids(hr_dev, i);
-		}
 	}
 
-	return ret;
+	return 0;
 }
 
 static int hns_roce_query_device(struct ib_device *ib_dev,
@@ -444,31 +290,6 @@ static enum rdma_link_layer hns_roce_get_link_layer(struct ib_device *device,
 static int hns_roce_query_gid(struct ib_device *ib_dev, u8 port_num, int index,
 			      union ib_gid *gid)
 {
-	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
-	struct device *dev = &hr_dev->pdev->dev;
-	u8 gid_idx = 0;
-	u8 port;
-
-	if (port_num < 1 || port_num > hr_dev->caps.num_ports ||
-	    index >= hr_dev->caps.gid_table_len[port_num - 1]) {
-		dev_err(dev,
-			"port_num %d index %d illegal! correct range: port_num 1~%d index 0~%d!\n",
-			port_num, index, hr_dev->caps.num_ports,
-			hr_dev->caps.gid_table_len[port_num - 1] - 1);
-		return -EINVAL;
-	}
-
-	port = port_num - 1;
-	gid_idx = hns_get_gid_index(hr_dev, port, index);
-	if (gid_idx >= HNS_ROCE_MAX_GID_NUM) {
-		dev_err(dev, "port_num %d index %d illegal! total gid num %d!\n",
-			port_num, index, HNS_ROCE_MAX_GID_NUM);
-		return -EINVAL;
-	}
-
-	memcpy(gid->raw, hr_dev->iboe.gid_table[gid_idx].raw,
-	       HNS_ROCE_GID_SIZE);
-
 	return 0;
 }
 
@@ -549,6 +370,8 @@ static int hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext)
 static int hns_roce_mmap(struct ib_ucontext *context,
 			 struct vm_area_struct *vma)
 {
+	struct hns_roce_dev *hr_dev = to_hr_dev(context->device);
+
 	if (((vma->vm_end - vma->vm_start) % PAGE_SIZE) != 0)
 		return -EINVAL;
 
@@ -558,10 +381,15 @@ static int hns_roce_mmap(struct ib_ucontext *context,
 				       to_hr_ucontext(context)->uar.pfn,
 				       PAGE_SIZE, vma->vm_page_prot))
 			return -EAGAIN;
-
-	} else {
+	} else if (vma->vm_pgoff == 1 && hr_dev->hw_rev == HNS_ROCE_HW_VER1) {
+		/* vm_pgoff: 1 -- TPTR */
+		if (io_remap_pfn_range(vma, vma->vm_start,
+				       hr_dev->tptr_dma_addr >> PAGE_SHIFT,
+				       hr_dev->tptr_size,
+				       vma->vm_page_prot))
+			return -EAGAIN;
+	} else
 		return -EINVAL;
-	}
 
 	return 0;
 }
@@ -605,7 +433,7 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
 	spin_lock_init(&iboe->lock);
 
 	ib_dev = &hr_dev->ib_dev;
-	strlcpy(ib_dev->name, "hisi_%d", IB_DEVICE_NAME_MAX);
+	strlcpy(ib_dev->name, "hns_%d", IB_DEVICE_NAME_MAX);
 
 	ib_dev->owner			= THIS_MODULE;
 	ib_dev->node_type		= RDMA_NODE_IB_CA;
@@ -639,6 +467,8 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
 	ib_dev->get_link_layer		= hns_roce_get_link_layer;
 	ib_dev->get_netdev		= hns_roce_get_netdev;
 	ib_dev->query_gid		= hns_roce_query_gid;
+	ib_dev->add_gid			= hns_roce_add_gid;
+	ib_dev->del_gid			= hns_roce_del_gid;
 	ib_dev->query_pkey		= hns_roce_query_pkey;
 	ib_dev->alloc_ucontext		= hns_roce_alloc_ucontext;
 	ib_dev->dealloc_ucontext	= hns_roce_dealloc_ucontext;
@@ -681,32 +511,22 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
 		return ret;
 	}
 
-	ret = hns_roce_setup_mtu_gids(hr_dev);
+	ret = hns_roce_setup_mtu_mac(hr_dev);
 	if (ret) {
-		dev_err(dev, "roce_setup_mtu_gids failed!\n");
-		goto error_failed_setup_mtu_gids;
+		dev_err(dev, "setup_mtu_mac failed!\n");
+		goto error_failed_setup_mtu_mac;
 	}
 
 	iboe->nb.notifier_call = hns_roce_netdev_event;
 	ret = register_netdevice_notifier(&iboe->nb);
 	if (ret) {
 		dev_err(dev, "register_netdevice_notifier failed!\n");
-		goto error_failed_setup_mtu_gids;
-	}
-
-	iboe->nb_inet.notifier_call = hns_roce_inet_event;
-	ret = register_inetaddr_notifier(&iboe->nb_inet);
-	if (ret) {
-		dev_err(dev, "register inet addr notifier failed!\n");
-		goto error_failed_register_inetaddr_notifier;
+		goto error_failed_setup_mtu_mac;
 	}
 
 	return 0;
 
-error_failed_register_inetaddr_notifier:
-	unregister_netdevice_notifier(&iboe->nb);
-
-error_failed_setup_mtu_gids:
+error_failed_setup_mtu_mac:
 	ib_unregister_device(ib_dev);
 
 	return ret;
@@ -940,10 +760,10 @@ err_unmap_mtt:
 }
 
 /**
-* hns_roce_setup_hca - setup host channel adapter
-* @hr_dev: pointer to hns roce device
-* Return : int
-*/
+ * hns_roce_setup_hca - setup host channel adapter
+ * @hr_dev: pointer to hns roce device
+ * Return : int
+ */
 static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev)
 {
 	int ret;
@@ -1008,11 +828,11 @@ err_uar_table_free:
 }
 
 /**
-* hns_roce_probe - RoCE driver entrance
-* @pdev: pointer to platform device
-* Return : int
-*
-*/
+ * hns_roce_probe - RoCE driver entrance
+ * @pdev: pointer to platform device
+ * Return : int
+ *
+ */
 static int hns_roce_probe(struct platform_device *pdev)
 {
 	int ret;
@@ -1023,9 +843,6 @@ static int hns_roce_probe(struct platform_device *pdev)
 	if (!hr_dev)
 		return -ENOMEM;
 
-	memset((u8 *)hr_dev + sizeof(struct ib_device), 0,
-		sizeof(struct hns_roce_dev) - sizeof(struct ib_device));
-
 	hr_dev->pdev = pdev;
 	platform_set_drvdata(pdev, hr_dev);
 
@@ -1125,9 +942,9 @@ error_failed_get_cfg:
 }
 
 /**
-* hns_roce_remove - remove RoCE device
-* @pdev: pointer to platform device
-*/
+ * hns_roce_remove - remove RoCE device
+ * @pdev: pointer to platform device
+ */
 static int hns_roce_remove(struct platform_device *pdev)
 {
 	struct hns_roce_dev *hr_dev = platform_get_drvdata(pdev);
diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c
index fb87883ead34..4139abee3b54 100644
--- a/drivers/infiniband/hw/hns/hns_roce_mr.c
+++ b/drivers/infiniband/hw/hns/hns_roce_mr.c
@@ -42,7 +42,7 @@ static u32 hw_index_to_key(unsigned long ind)
 	return (u32)(ind >> 24) | (ind << 8);
 }
 
-static unsigned long key_to_hw_index(u32 key)
+unsigned long key_to_hw_index(u32 key)
 {
 	return (key << 24) | (key >> 8);
 }
@@ -53,16 +53,16 @@ static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev,
 {
 	return hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, mpt_index, 0,
 				 HNS_ROCE_CMD_SW2HW_MPT,
-				 HNS_ROCE_CMD_TIME_CLASS_B);
+				 HNS_ROCE_CMD_TIMEOUT_MSECS);
 }
 
-static int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
+int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
 			      struct hns_roce_cmd_mailbox *mailbox,
 			      unsigned long mpt_index)
 {
 	return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox->dma : 0,
 				 mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT,
-				 HNS_ROCE_CMD_TIME_CLASS_B);
+				 HNS_ROCE_CMD_TIMEOUT_MSECS);
 }
 
 static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order,
@@ -137,11 +137,13 @@ static int hns_roce_buddy_init(struct hns_roce_buddy *buddy, int max_order)
 
 	for (i = 0; i <= buddy->max_order; ++i) {
 		s = BITS_TO_LONGS(1 << (buddy->max_order - i));
-		buddy->bits[i] = kmalloc_array(s, sizeof(long), GFP_KERNEL);
-		if (!buddy->bits[i])
-			goto err_out_free;
-
-		bitmap_zero(buddy->bits[i], 1 << (buddy->max_order - i));
+		buddy->bits[i] = kcalloc(s, sizeof(long), GFP_KERNEL |
+					 __GFP_NOWARN);
+		if (!buddy->bits[i]) {
+			buddy->bits[i] = vzalloc(s * sizeof(long));
+			if (!buddy->bits[i])
+				goto err_out_free;
+		}
 	}
 
 	set_bit(0, buddy->bits[buddy->max_order]);
@@ -151,7 +153,7 @@ static int hns_roce_buddy_init(struct hns_roce_buddy *buddy, int max_order)
 
 err_out_free:
 	for (i = 0; i <= buddy->max_order; ++i)
-		kfree(buddy->bits[i]);
+		kvfree(buddy->bits[i]);
 
 err_out:
 	kfree(buddy->bits);
@@ -164,7 +166,7 @@ static void hns_roce_buddy_cleanup(struct hns_roce_buddy *buddy)
 	int i;
 
 	for (i = 0; i <= buddy->max_order; ++i)
-		kfree(buddy->bits[i]);
+		kvfree(buddy->bits[i]);
 
 	kfree(buddy->bits);
 	kfree(buddy->num_free);
@@ -287,7 +289,7 @@ static void hns_roce_mr_free(struct hns_roce_dev *hr_dev,
 	}
 
 	hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
-			     key_to_hw_index(mr->key));
+			     key_to_hw_index(mr->key), BITMAP_NO_RR);
 }
 
 static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
@@ -605,13 +607,20 @@ err_free:
 
 int hns_roce_dereg_mr(struct ib_mr *ibmr)
 {
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
 	struct hns_roce_mr *mr = to_hr_mr(ibmr);
+	int ret = 0;
 
-	hns_roce_mr_free(to_hr_dev(ibmr->device), mr);
-	if (mr->umem)
-		ib_umem_release(mr->umem);
+	if (hr_dev->hw->dereg_mr) {
+		ret = hr_dev->hw->dereg_mr(hr_dev, mr);
+	} else {
+		hns_roce_mr_free(hr_dev, mr);
 
-	kfree(mr);
+		if (mr->umem)
+			ib_umem_release(mr->umem);
 
-	return 0;
+		kfree(mr);
+	}
+
+	return ret;
 }
diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c
index 05db7d59812a..a64500fa1145 100644
--- a/drivers/infiniband/hw/hns/hns_roce_pd.c
+++ b/drivers/infiniband/hw/hns/hns_roce_pd.c
@@ -40,7 +40,7 @@ static int hns_roce_pd_alloc(struct hns_roce_dev *hr_dev, unsigned long *pdn)
 
 static void hns_roce_pd_free(struct hns_roce_dev *hr_dev, unsigned long pdn)
 {
-	hns_roce_bitmap_free(&hr_dev->pd_bitmap, pdn);
+	hns_roce_bitmap_free(&hr_dev->pd_bitmap, pdn, BITMAP_NO_RR);
 }
 
 int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev)
@@ -121,7 +121,8 @@ int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)
 
 void hns_roce_uar_free(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar)
 {
-	hns_roce_bitmap_free(&hr_dev->uar_table.bitmap, uar->index);
+	hns_roce_bitmap_free(&hr_dev->uar_table.bitmap, uar->index,
+			     BITMAP_NO_RR);
 }
 
 int hns_roce_init_uar_table(struct hns_roce_dev *hr_dev)
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index e86dd8d06777..f036f32f15d3 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -37,7 +37,7 @@
 #include "hns_roce_common.h"
 #include "hns_roce_device.h"
 #include "hns_roce_hem.h"
-#include "hns_roce_user.h"
+#include <rdma/hns-abi.h>
 
 #define SQP_NUM				(2 * HNS_ROCE_MAX_PORTS)
 
@@ -250,7 +250,7 @@ void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,
 	if (base_qpn < SQP_NUM)
 		return;
 
-	hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt);
+	hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, BITMAP_RR);
 }
 
 static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev,
diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h
index 8ec09e470f84..da2eb5a281fa 100644
--- a/drivers/infiniband/hw/i40iw/i40iw.h
+++ b/drivers/infiniband/hw/i40iw/i40iw.h
@@ -112,9 +112,12 @@
 #define I40IW_DRV_OPT_MCAST_LOGPORT_MAP    0x00000800
 
 #define IW_HMC_OBJ_TYPE_NUM ARRAY_SIZE(iw_hmc_obj_types)
-#define IW_CFG_FPM_QP_COUNT		32768
-#define I40IW_MAX_PAGES_PER_FMR		512
-#define I40IW_MIN_PAGES_PER_FMR		1
+#define IW_CFG_FPM_QP_COUNT               32768
+#define I40IW_MAX_PAGES_PER_FMR           512
+#define I40IW_MIN_PAGES_PER_FMR           1
+#define I40IW_CQP_COMPL_RQ_WQE_FLUSHED    2
+#define I40IW_CQP_COMPL_SQ_WQE_FLUSHED    3
+#define I40IW_CQP_COMPL_RQ_SQ_WQE_FLUSHED 4
 
 #define I40IW_MTU_TO_MSS		40
 #define I40IW_DEFAULT_MSS		1460
@@ -210,6 +213,12 @@ struct i40iw_msix_vector {
 	u32 ceq_id;
 };
 
+struct l2params_work {
+	struct work_struct work;
+	struct i40iw_device *iwdev;
+	struct i40iw_l2params l2params;
+};
+
 #define I40IW_MSIX_TABLE_SIZE   65
 
 struct virtchnl_work {
@@ -227,6 +236,7 @@ struct i40iw_device {
 	struct net_device *netdev;
 	wait_queue_head_t vchnl_waitq;
 	struct i40iw_sc_dev sc_dev;
+	struct i40iw_sc_vsi vsi;
 	struct i40iw_handler *hdl;
 	struct i40e_info *ldev;
 	struct i40e_client *client;
@@ -280,7 +290,6 @@ struct i40iw_device {
 	u32 sd_type;
 	struct workqueue_struct *param_wq;
 	atomic_t params_busy;
-	u32 mss;
 	enum init_completion_state init_state;
 	u16 mac_ip_table_idx;
 	atomic_t vchnl_msgs;
@@ -297,6 +306,14 @@ struct i40iw_device {
 	u32 mr_stagmask;
 	u32 mpa_version;
 	bool dcb;
+	bool closing;
+	bool reset;
+	u32 used_pds;
+	u32 used_cqs;
+	u32 used_mrs;
+	u32 used_qps;
+	wait_queue_head_t close_wq;
+	atomic64_t use_count;
 };
 
 struct i40iw_ib_device {
@@ -498,7 +515,7 @@ u32 i40iw_initialize_hw_resources(struct i40iw_device *iwdev);
 
 int i40iw_register_rdma_device(struct i40iw_device *iwdev);
 void i40iw_port_ibevent(struct i40iw_device *iwdev);
-int i40iw_cm_disconn(struct i40iw_qp *);
+void i40iw_cm_disconn(struct i40iw_qp *iwqp);
 void i40iw_cm_disconn_worker(void *);
 int mini_cm_recv_pkt(struct i40iw_cm_core *, struct i40iw_device *,
 		     struct sk_buff *);
@@ -508,20 +525,26 @@ enum i40iw_status_code i40iw_handle_cqp_op(struct i40iw_device *iwdev,
 enum i40iw_status_code i40iw_add_mac_addr(struct i40iw_device *iwdev,
 					  u8 *mac_addr, u8 *mac_index);
 int i40iw_modify_qp(struct ib_qp *, struct ib_qp_attr *, int, struct ib_udata *);
+void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq);
 
 void i40iw_rem_pdusecount(struct i40iw_pd *iwpd, struct i40iw_device *iwdev);
 void i40iw_add_pdusecount(struct i40iw_pd *iwpd);
+void i40iw_rem_devusecount(struct i40iw_device *iwdev);
+void i40iw_add_devusecount(struct i40iw_device *iwdev);
 void i40iw_hw_modify_qp(struct i40iw_device *iwdev, struct i40iw_qp *iwqp,
 			struct i40iw_modify_qp_info *info, bool wait);
 
+void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev,
+			     struct i40iw_sc_qp *qp,
+			     bool suspend);
 enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev,
 					  struct i40iw_cm_info *cminfo,
 					  enum i40iw_quad_entry_type etype,
 					  enum i40iw_quad_hash_manage_type mtype,
 					  void *cmnode,
 					  bool wait);
-void i40iw_receive_ilq(struct i40iw_sc_dev *dev, struct i40iw_puda_buf *rbuf);
-void i40iw_free_sqbuf(struct i40iw_sc_dev *dev, void *bufp);
+void i40iw_receive_ilq(struct i40iw_sc_vsi *vsi, struct i40iw_puda_buf *rbuf);
+void i40iw_free_sqbuf(struct i40iw_sc_vsi *vsi, void *bufp);
 void i40iw_free_qp_resources(struct i40iw_device *iwdev,
 			     struct i40iw_qp *iwqp,
 			     u32 qp_num);
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c
index 85637696f6e9..95a0586a4da8 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c
@@ -68,13 +68,13 @@ static void i40iw_disconnect_worker(struct work_struct *work);
 
 /**
  * i40iw_free_sqbuf - put back puda buffer if refcount = 0
- * @dev: FPK device
+ * @vsi: pointer to vsi structure
  * @buf: puda buffer to free
  */
-void i40iw_free_sqbuf(struct i40iw_sc_dev *dev, void *bufp)
+void i40iw_free_sqbuf(struct i40iw_sc_vsi *vsi, void *bufp)
 {
 	struct i40iw_puda_buf *buf = (struct i40iw_puda_buf *)bufp;
-	struct i40iw_puda_rsrc *ilq = dev->ilq;
+	struct i40iw_puda_rsrc *ilq = vsi->ilq;
 
 	if (!atomic_dec_return(&buf->refcount))
 		i40iw_puda_ret_bufpool(ilq, buf);
@@ -221,6 +221,7 @@ static void i40iw_get_addr_info(struct i40iw_cm_node *cm_node,
 	memcpy(cm_info->rem_addr, cm_node->rem_addr, sizeof(cm_info->rem_addr));
 	cm_info->loc_port = cm_node->loc_port;
 	cm_info->rem_port = cm_node->rem_port;
+	cm_info->user_pri = cm_node->user_pri;
 }
 
 /**
@@ -271,6 +272,7 @@ static int i40iw_send_cm_event(struct i40iw_cm_node *cm_node,
 		event.provider_data = (void *)cm_node;
 		event.private_data = (void *)cm_node->pdata_buf;
 		event.private_data_len = (u8)cm_node->pdata.size;
+		event.ird = cm_node->ird_size;
 		break;
 	case IW_CM_EVENT_CONNECT_REPLY:
 		i40iw_get_cmevent_info(cm_node, cm_id, &event);
@@ -335,13 +337,13 @@ static struct i40iw_cm_event *i40iw_create_event(struct i40iw_cm_node *cm_node,
  */
 static void i40iw_free_retrans_entry(struct i40iw_cm_node *cm_node)
 {
-	struct i40iw_sc_dev *dev = cm_node->dev;
+	struct i40iw_device *iwdev = cm_node->iwdev;
 	struct i40iw_timer_entry *send_entry;
 
 	send_entry = cm_node->send_entry;
 	if (send_entry) {
 		cm_node->send_entry = NULL;
-		i40iw_free_sqbuf(dev, (void *)send_entry->sqbuf);
+		i40iw_free_sqbuf(&iwdev->vsi, (void *)send_entry->sqbuf);
 		kfree(send_entry);
 		atomic_dec(&cm_node->ref_count);
 	}
@@ -360,15 +362,6 @@ static void i40iw_cleanup_retrans_entry(struct i40iw_cm_node *cm_node)
 	spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
 }
 
-static bool is_remote_ne020_or_chelsio(struct i40iw_cm_node *cm_node)
-{
-	if ((cm_node->rem_mac[0] == 0x0) &&
-	    (((cm_node->rem_mac[1] == 0x12) && (cm_node->rem_mac[2] == 0x55)) ||
-	     ((cm_node->rem_mac[1] == 0x07 && (cm_node->rem_mac[2] == 0x43)))))
-		return true;
-	return false;
-}
-
 /**
  * i40iw_form_cm_frame - get a free packet and build frame
  * @cm_node: connection's node ionfo to use in frame
@@ -384,7 +377,7 @@ static struct i40iw_puda_buf *i40iw_form_cm_frame(struct i40iw_cm_node *cm_node,
 						  u8 flags)
 {
 	struct i40iw_puda_buf *sqbuf;
-	struct i40iw_sc_dev *dev = cm_node->dev;
+	struct i40iw_sc_vsi *vsi = &cm_node->iwdev->vsi;
 	u8 *buf;
 
 	struct tcphdr *tcph;
@@ -396,8 +389,9 @@ static struct i40iw_puda_buf *i40iw_form_cm_frame(struct i40iw_cm_node *cm_node,
 	u32 opts_len = 0;
 	u32 pd_len = 0;
 	u32 hdr_len = 0;
+	u16 vtag;
 
-	sqbuf = i40iw_puda_get_bufpool(dev->ilq);
+	sqbuf = i40iw_puda_get_bufpool(vsi->ilq);
 	if (!sqbuf)
 		return NULL;
 	buf = sqbuf->mem.va;
@@ -408,11 +402,8 @@ static struct i40iw_puda_buf *i40iw_form_cm_frame(struct i40iw_cm_node *cm_node,
 	if (hdr)
 		hdr_len = hdr->size;
 
-	if (pdata) {
+	if (pdata)
 		pd_len = pdata->size;
-		if (!is_remote_ne020_or_chelsio(cm_node))
-			pd_len += MPA_ZERO_PAD_LEN;
-	}
 
 	if (cm_node->vlan_id < VLAN_TAG_PRESENT)
 		eth_hlen += 4;
@@ -445,7 +436,8 @@ static struct i40iw_puda_buf *i40iw_form_cm_frame(struct i40iw_cm_node *cm_node,
 		ether_addr_copy(ethh->h_source, cm_node->loc_mac);
 		if (cm_node->vlan_id < VLAN_TAG_PRESENT) {
 			((struct vlan_ethhdr *)ethh)->h_vlan_proto = htons(ETH_P_8021Q);
-			((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(cm_node->vlan_id);
+			vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) | cm_node->vlan_id;
+			((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);
 
 			((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto = htons(ETH_P_IP);
 		} else {
@@ -454,7 +446,7 @@ static struct i40iw_puda_buf *i40iw_form_cm_frame(struct i40iw_cm_node *cm_node,
 
 		iph->version = IPVERSION;
 		iph->ihl = 5;	/* 5 * 4Byte words, IP headr len */
-		iph->tos = 0;
+		iph->tos = cm_node->tos;
 		iph->tot_len = htons(packetsize);
 		iph->id = htons(++cm_node->tcp_cntxt.loc_id);
 
@@ -474,13 +466,15 @@ static struct i40iw_puda_buf *i40iw_form_cm_frame(struct i40iw_cm_node *cm_node,
 		ether_addr_copy(ethh->h_source, cm_node->loc_mac);
 		if (cm_node->vlan_id < VLAN_TAG_PRESENT) {
 			((struct vlan_ethhdr *)ethh)->h_vlan_proto = htons(ETH_P_8021Q);
-			((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(cm_node->vlan_id);
+			vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) | cm_node->vlan_id;
+			((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);
 			((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto = htons(ETH_P_IPV6);
 		} else {
 			ethh->h_proto = htons(ETH_P_IPV6);
 		}
 		ip6h->version = 6;
-		ip6h->flow_lbl[0] = 0;
+		ip6h->priority = cm_node->tos >> 4;
+		ip6h->flow_lbl[0] = cm_node->tos << 4;
 		ip6h->flow_lbl[1] = 0;
 		ip6h->flow_lbl[2] = 0;
 		ip6h->payload_len = htons(packetsize - sizeof(*ip6h));
@@ -1065,7 +1059,7 @@ int i40iw_schedule_cm_timer(struct i40iw_cm_node *cm_node,
 			    int send_retrans,
 			    int close_when_complete)
 {
-	struct i40iw_sc_dev *dev = cm_node->dev;
+	struct i40iw_sc_vsi *vsi = &cm_node->iwdev->vsi;
 	struct i40iw_cm_core *cm_core = cm_node->cm_core;
 	struct i40iw_timer_entry *new_send;
 	int ret = 0;
@@ -1074,7 +1068,7 @@ int i40iw_schedule_cm_timer(struct i40iw_cm_node *cm_node,
 
 	new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC);
 	if (!new_send) {
-		i40iw_free_sqbuf(cm_node->dev, (void *)sqbuf);
+		i40iw_free_sqbuf(vsi, (void *)sqbuf);
 		return -ENOMEM;
 	}
 	new_send->retrycount = I40IW_DEFAULT_RETRYS;
@@ -1089,7 +1083,7 @@ int i40iw_schedule_cm_timer(struct i40iw_cm_node *cm_node,
 		new_send->timetosend += (HZ / 10);
 		if (cm_node->close_entry) {
 			kfree(new_send);
-			i40iw_free_sqbuf(cm_node->dev, (void *)sqbuf);
+			i40iw_free_sqbuf(vsi, (void *)sqbuf);
 			i40iw_pr_err("already close entry\n");
 			return -EINVAL;
 		}
@@ -1104,7 +1098,7 @@ int i40iw_schedule_cm_timer(struct i40iw_cm_node *cm_node,
 		new_send->timetosend = jiffies + I40IW_RETRY_TIMEOUT;
 
 		atomic_inc(&sqbuf->refcount);
-		i40iw_puda_send_buf(dev->ilq, sqbuf);
+		i40iw_puda_send_buf(vsi->ilq, sqbuf);
 		if (!send_retrans) {
 			i40iw_cleanup_retrans_entry(cm_node);
 			if (close_when_complete)
@@ -1201,6 +1195,7 @@ static void i40iw_cm_timer_tick(unsigned long pass)
 	struct i40iw_cm_node *cm_node;
 	struct i40iw_timer_entry *send_entry, *close_entry;
 	struct list_head *list_core_temp;
+	struct i40iw_sc_vsi *vsi;
 	struct list_head *list_node;
 	struct i40iw_cm_core *cm_core = (struct i40iw_cm_core *)pass;
 	u32 settimer = 0;
@@ -1276,9 +1271,10 @@ static void i40iw_cm_timer_tick(unsigned long pass)
 		cm_node->cm_core->stats_pkt_retrans++;
 		spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
 
+		vsi = &cm_node->iwdev->vsi;
 		dev = cm_node->dev;
 		atomic_inc(&send_entry->sqbuf->refcount);
-		i40iw_puda_send_buf(dev->ilq, send_entry->sqbuf);
+		i40iw_puda_send_buf(vsi->ilq, send_entry->sqbuf);
 		spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
 		if (send_entry->send_retrans) {
 			send_entry->retranscount--;
@@ -1379,10 +1375,11 @@ int i40iw_send_syn(struct i40iw_cm_node *cm_node, u32 sendack)
 static void i40iw_send_ack(struct i40iw_cm_node *cm_node)
 {
 	struct i40iw_puda_buf *sqbuf;
+	struct i40iw_sc_vsi *vsi = &cm_node->iwdev->vsi;
 
 	sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK);
 	if (sqbuf)
-		i40iw_puda_send_buf(cm_node->dev->ilq, sqbuf);
+		i40iw_puda_send_buf(vsi->ilq, sqbuf);
 	else
 		i40iw_pr_err("no sqbuf\n");
 }
@@ -1564,9 +1561,15 @@ static enum i40iw_status_code i40iw_del_multiple_qhash(
 		memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
 		       sizeof(cm_info->loc_addr));
 		cm_info->vlan_id = child_listen_node->vlan_id;
-		ret = i40iw_manage_qhash(iwdev, cm_info,
-					 I40IW_QHASH_TYPE_TCP_SYN,
-					 I40IW_QHASH_MANAGE_TYPE_DELETE, NULL, false);
+		if (child_listen_node->qhash_set) {
+			ret = i40iw_manage_qhash(iwdev, cm_info,
+						 I40IW_QHASH_TYPE_TCP_SYN,
+						 I40IW_QHASH_MANAGE_TYPE_DELETE,
+						 NULL, false);
+			child_listen_node->qhash_set = false;
+		} else {
+			ret = I40IW_SUCCESS;
+		}
 		i40iw_debug(&iwdev->sc_dev,
 			    I40IW_DEBUG_CM,
 			    "freed pointer = %p\n",
@@ -1591,9 +1594,10 @@ static enum i40iw_status_code i40iw_del_multiple_qhash(
 static struct net_device *i40iw_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id, u8 *mac)
 {
 	struct net_device *ip_dev = NULL;
-#if IS_ENABLED(CONFIG_IPV6)
 	struct in6_addr laddr6;
 
+	if (!IS_ENABLED(CONFIG_IPV6))
+		return NULL;
 	i40iw_copy_ip_htonl(laddr6.in6_u.u6_addr32, addr);
 	if (vlan_id)
 		*vlan_id = I40IW_NO_VLAN;
@@ -1610,7 +1614,6 @@ static struct net_device *i40iw_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id, u8 *ma
 		}
 	}
 	rcu_read_unlock();
-#endif
 	return ip_dev;
 }
 
@@ -1646,7 +1649,7 @@ static enum i40iw_status_code i40iw_add_mqh_6(struct i40iw_device *iwdev,
 {
 	struct net_device *ip_dev;
 	struct inet6_dev *idev;
-	struct inet6_ifaddr *ifp;
+	struct inet6_ifaddr *ifp, *tmp;
 	enum i40iw_status_code ret = 0;
 	struct i40iw_cm_listener *child_listen_node;
 	unsigned long flags;
@@ -1661,7 +1664,7 @@ static enum i40iw_status_code i40iw_add_mqh_6(struct i40iw_device *iwdev,
 				i40iw_pr_err("idev == NULL\n");
 				break;
 			}
-			list_for_each_entry(ifp, &idev->addr_list, if_list) {
+			list_for_each_entry_safe(ifp, tmp, &idev->addr_list, if_list) {
 				i40iw_debug(&iwdev->sc_dev,
 					    I40IW_DEBUG_CM,
 					    "IP=%pI6, vlan_id=%d, MAC=%pM\n",
@@ -1675,7 +1678,6 @@ static enum i40iw_status_code i40iw_add_mqh_6(struct i40iw_device *iwdev,
 					    "Allocating child listener %p\n",
 					    child_listen_node);
 				if (!child_listen_node) {
-					i40iw_pr_err("listener memory allocation\n");
 					ret = I40IW_ERR_NO_MEMORY;
 					goto exit;
 				}
@@ -1695,6 +1697,7 @@ static enum i40iw_status_code i40iw_add_mqh_6(struct i40iw_device *iwdev,
 							 I40IW_QHASH_MANAGE_TYPE_ADD,
 							 NULL, true);
 				if (!ret) {
+					child_listen_node->qhash_set = true;
 					spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
 					list_add(&child_listen_node->child_listen_list,
 						 &cm_parent_listen_node->child_listen_list);
@@ -1751,7 +1754,6 @@ static enum i40iw_status_code i40iw_add_mqh_4(
 					    "Allocating child listener %p\n",
 					    child_listen_node);
 				if (!child_listen_node) {
-					i40iw_pr_err("listener memory allocation\n");
 					in_dev_put(idev);
 					ret = I40IW_ERR_NO_MEMORY;
 					goto exit;
@@ -1773,6 +1775,7 @@ static enum i40iw_status_code i40iw_add_mqh_4(
 							 NULL,
 							 true);
 				if (!ret) {
+					child_listen_node->qhash_set = true;
 					spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
 					list_add(&child_listen_node->child_listen_list,
 						 &cm_parent_listen_node->child_listen_list);
@@ -1880,6 +1883,7 @@ static int i40iw_dec_refcnt_listen(struct i40iw_cm_core *cm_core,
 			nfo.loc_port = listener->loc_port;
 			nfo.ipv4 = listener->ipv4;
 			nfo.vlan_id = listener->vlan_id;
+			nfo.user_pri = listener->user_pri;
 
 			if (!list_empty(&listener->child_listen_list)) {
 				i40iw_del_multiple_qhash(listener->iwdev, &nfo, listener);
@@ -2138,6 +2142,20 @@ static struct i40iw_cm_node *i40iw_make_cm_node(
 	/* set our node specific transport info */
 	cm_node->ipv4 = cm_info->ipv4;
 	cm_node->vlan_id = cm_info->vlan_id;
+	if ((cm_node->vlan_id == I40IW_NO_VLAN) && iwdev->dcb)
+		cm_node->vlan_id = 0;
+	cm_node->tos = cm_info->tos;
+	cm_node->user_pri = cm_info->user_pri;
+	if (listener) {
+		if (listener->tos != cm_info->tos)
+			i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_DCB,
+				    "application TOS[%d] and remote client TOS[%d] mismatch\n",
+				     listener->tos, cm_info->tos);
+		cm_node->tos = max(listener->tos, cm_info->tos);
+		cm_node->user_pri = rt_tos2priority(cm_node->tos);
+		i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_DCB, "listener: TOS:[%d] UP:[%d]\n",
+			    cm_node->tos, cm_node->user_pri);
+	}
 	memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr));
 	memcpy(cm_node->rem_addr, cm_info->rem_addr, sizeof(cm_node->rem_addr));
 	cm_node->loc_port = cm_info->loc_port;
@@ -2162,7 +2180,7 @@ static struct i40iw_cm_node *i40iw_make_cm_node(
 			I40IW_CM_DEFAULT_RCV_WND_SCALED >> I40IW_CM_DEFAULT_RCV_WND_SCALE;
 	ts = current_kernel_time();
 	cm_node->tcp_cntxt.loc_seq_num = ts.tv_nsec;
-	cm_node->tcp_cntxt.mss = iwdev->mss;
+	cm_node->tcp_cntxt.mss = iwdev->vsi.mss;
 
 	cm_node->iwdev = iwdev;
 	cm_node->dev = &iwdev->sc_dev;
@@ -2236,7 +2254,7 @@ static void i40iw_rem_ref_cm_node(struct i40iw_cm_node *cm_node)
 		i40iw_dec_refcnt_listen(cm_core, cm_node->listener, 0, true);
 	} else {
 		if (!i40iw_listen_port_in_use(cm_core, cm_node->loc_port) &&
-		    cm_node->apbvt_set && cm_node->iwdev) {
+		    cm_node->apbvt_set) {
 			i40iw_manage_apbvt(cm_node->iwdev,
 					   cm_node->loc_port,
 					   I40IW_MANAGE_APBVT_DEL);
@@ -2861,7 +2879,7 @@ static struct i40iw_cm_node *i40iw_create_cm_node(
 	/* create a CM connection node */
 	cm_node = i40iw_make_cm_node(cm_core, iwdev, cm_info, NULL);
 	if (!cm_node)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 	/* set our node side to client (active) side */
 	cm_node->tcp_cntxt.client = 1;
 	cm_node->tcp_cntxt.rcv_wscale = I40IW_CM_DEFAULT_RCV_WND_SCALE;
@@ -2874,7 +2892,8 @@ static struct i40iw_cm_node *i40iw_create_cm_node(
 						cm_node->vlan_id,
 						I40IW_CM_LISTENER_ACTIVE_STATE);
 		if (!loopback_remotelistener) {
-			i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED);
+			i40iw_rem_ref_cm_node(cm_node);
+			return ERR_PTR(-ECONNREFUSED);
 		} else {
 			loopback_cm_info = *cm_info;
 			loopback_cm_info.loc_port = cm_info->rem_port;
@@ -2887,7 +2906,7 @@ static struct i40iw_cm_node *i40iw_create_cm_node(
 								 loopback_remotelistener);
 			if (!loopback_remotenode) {
 				i40iw_rem_ref_cm_node(cm_node);
-				return NULL;
+				return ERR_PTR(-ENOMEM);
 			}
 			cm_core->stats_loopbacks++;
 			loopback_remotenode->loopbackpartner = cm_node;
@@ -3041,10 +3060,10 @@ static int i40iw_cm_close(struct i40iw_cm_node *cm_node)
 /**
  * i40iw_receive_ilq - recv an ETHERNET packet, and process it
  * through CM
- * @dev: FPK dev struct
+ * @vsi: pointer to the vsi structure
  * @rbuf: receive buffer
  */
-void i40iw_receive_ilq(struct i40iw_sc_dev *dev, struct i40iw_puda_buf *rbuf)
+void i40iw_receive_ilq(struct i40iw_sc_vsi *vsi, struct i40iw_puda_buf *rbuf)
 {
 	struct i40iw_cm_node *cm_node;
 	struct i40iw_cm_listener *listener;
@@ -3052,9 +3071,11 @@ void i40iw_receive_ilq(struct i40iw_sc_dev *dev, struct i40iw_puda_buf *rbuf)
 	struct ipv6hdr *ip6h;
 	struct tcphdr *tcph;
 	struct i40iw_cm_info cm_info;
+	struct i40iw_sc_dev *dev = vsi->dev;
 	struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
 	struct i40iw_cm_core *cm_core = &iwdev->cm_core;
 	struct vlan_ethhdr *ethh;
+	u16 vtag;
 
 	/* if vlan, then maclen = 18 else 14 */
 	iph = (struct iphdr *)rbuf->iph;
@@ -3068,7 +3089,9 @@ void i40iw_receive_ilq(struct i40iw_sc_dev *dev, struct i40iw_puda_buf *rbuf)
 	ethh = (struct vlan_ethhdr *)rbuf->mem.va;
 
 	if (ethh->h_vlan_proto == htons(ETH_P_8021Q)) {
-		cm_info.vlan_id = ntohs(ethh->h_vlan_TCI) & VLAN_VID_MASK;
+		vtag = ntohs(ethh->h_vlan_TCI);
+		cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
+		cm_info.vlan_id = vtag & VLAN_VID_MASK;
 		i40iw_debug(cm_core->dev,
 			    I40IW_DEBUG_CM,
 			    "%s vlan_id=%d\n",
@@ -3083,6 +3106,7 @@ void i40iw_receive_ilq(struct i40iw_sc_dev *dev, struct i40iw_puda_buf *rbuf)
 		cm_info.loc_addr[0] = ntohl(iph->daddr);
 		cm_info.rem_addr[0] = ntohl(iph->saddr);
 		cm_info.ipv4 = true;
+		cm_info.tos = iph->tos;
 	} else {
 		ip6h = (struct ipv6hdr *)rbuf->iph;
 		i40iw_copy_ip_ntohl(cm_info.loc_addr,
@@ -3090,6 +3114,7 @@ void i40iw_receive_ilq(struct i40iw_sc_dev *dev, struct i40iw_puda_buf *rbuf)
 		i40iw_copy_ip_ntohl(cm_info.rem_addr,
 				    ip6h->saddr.in6_u.u6_addr32);
 		cm_info.ipv4 = false;
+		cm_info.tos = (ip6h->priority << 4) | (ip6h->flow_lbl[0] >> 4);
 	}
 	cm_info.loc_port = ntohs(tcph->dest);
 	cm_info.rem_port = ntohs(tcph->source);
@@ -3309,6 +3334,8 @@ static void i40iw_cm_init_tsa_conn(struct i40iw_qp *iwqp,
 
 	ctx_info->tcp_info_valid = true;
 	ctx_info->iwarp_info_valid = true;
+	ctx_info->add_to_qoslist = true;
+	ctx_info->user_pri = cm_node->user_pri;
 
 	i40iw_init_tcp_ctx(cm_node, &tcp_info, iwqp);
 	if (cm_node->snd_mark_en) {
@@ -3320,33 +3347,47 @@ static void i40iw_cm_init_tsa_conn(struct i40iw_qp *iwqp,
 	cm_node->state = I40IW_CM_STATE_OFFLOADED;
 	tcp_info.tcp_state = I40IW_TCP_STATE_ESTABLISHED;
 	tcp_info.src_mac_addr_idx = iwdev->mac_ip_table_idx;
+	tcp_info.tos = cm_node->tos;
 
 	dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp, (u64 *)(iwqp->host_ctx.va), ctx_info);
 
 	/* once tcp_info is set, no need to do it again */
 	ctx_info->tcp_info_valid = false;
 	ctx_info->iwarp_info_valid = false;
+	ctx_info->add_to_qoslist = false;
 }
 
 /**
  * i40iw_cm_disconn - when a connection is being closed
  * @iwqp: associate qp for the connection
  */
-int i40iw_cm_disconn(struct i40iw_qp *iwqp)
+void i40iw_cm_disconn(struct i40iw_qp *iwqp)
 {
 	struct disconn_work *work;
 	struct i40iw_device *iwdev = iwqp->iwdev;
 	struct i40iw_cm_core *cm_core = &iwdev->cm_core;
+	unsigned long flags;
 
 	work = kzalloc(sizeof(*work), GFP_ATOMIC);
 	if (!work)
-		return -ENOMEM;	/* Timer will clean up */
-
+		return;	/* Timer will clean up */
+
+	spin_lock_irqsave(&iwdev->qptable_lock, flags);
+	if (!iwdev->qp_table[iwqp->ibqp.qp_num]) {
+		spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
+		i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
+			    "%s qp_id %d is already freed\n",
+			     __func__, iwqp->ibqp.qp_num);
+		kfree(work);
+		return;
+	}
 	i40iw_add_ref(&iwqp->ibqp);
+	spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
+
 	work->iwqp = iwqp;
 	INIT_WORK(&work->work, i40iw_disconnect_worker);
 	queue_work(cm_core->disconn_wq, &work->work);
-	return 0;
+	return;
 }
 
 /**
@@ -3432,7 +3473,7 @@ static void i40iw_cm_disconn_true(struct i40iw_qp *iwqp)
 		 *terminate-handler to issue cm_disconn which can re-free
 		 *a QP even after its refcnt=0.
 		 */
-		del_timer(&iwqp->terminate_timer);
+		i40iw_terminate_del_timer(qp);
 		if (!iwqp->flush_issued) {
 			iwqp->flush_issued = 1;
 			issue_flush = 1;
@@ -3462,7 +3503,7 @@ static void i40iw_cm_disconn_true(struct i40iw_qp *iwqp)
 		/* Flush the queues */
 		i40iw_flush_wqes(iwdev, iwqp);
 
-		if (qp->term_flags) {
+		if (qp->term_flags && iwqp->ibqp.event_handler) {
 			ibevent.device = iwqp->ibqp.device;
 			ibevent.event = (qp->eventtype == TERM_EVENT_QP_FATAL) ?
 					IB_EVENT_QP_FATAL : IB_EVENT_QP_ACCESS_ERR;
@@ -3571,7 +3612,7 @@ int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 	iwqp->cm_node = (void *)cm_node;
 	cm_node->iwqp = iwqp;
 
-	buf_len = conn_param->private_data_len + I40IW_MAX_IETF_SIZE + MPA_ZERO_PAD_LEN;
+	buf_len = conn_param->private_data_len + I40IW_MAX_IETF_SIZE;
 
 	status = i40iw_allocate_dma_mem(dev->hw, &iwqp->ietf_mem, buf_len, 1);
 
@@ -3605,18 +3646,10 @@ int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 		iwqp->lsmm_mr = ibmr;
 		if (iwqp->page)
 			iwqp->sc_qp.qp_uk.sq_base = kmap(iwqp->page);
-		if (is_remote_ne020_or_chelsio(cm_node))
-			dev->iw_priv_qp_ops->qp_send_lsmm(
-							&iwqp->sc_qp,
+		dev->iw_priv_qp_ops->qp_send_lsmm(&iwqp->sc_qp,
 							iwqp->ietf_mem.va,
 							(accept.size + conn_param->private_data_len),
 							ibmr->lkey);
-		else
-			dev->iw_priv_qp_ops->qp_send_lsmm(
-							&iwqp->sc_qp,
-							iwqp->ietf_mem.va,
-							(accept.size + conn_param->private_data_len + MPA_ZERO_PAD_LEN),
-							ibmr->lkey);
 
 	} else {
 		if (iwqp->page)
@@ -3714,6 +3747,7 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 	struct sockaddr_in6 *raddr6;
 	bool qhash_set = false;
 	int apbvt_set = 0;
+	int err = 0;
 	enum i40iw_status_code status;
 
 	ibqp = i40iw_get_qp(cm_id->device, conn_param->qpn);
@@ -3759,6 +3793,10 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 		i40iw_netdev_vlan_ipv6(cm_info.loc_addr, &cm_info.vlan_id, NULL);
 	}
 	cm_info.cm_id = cm_id;
+	cm_info.tos = cm_id->tos;
+	cm_info.user_pri = rt_tos2priority(cm_id->tos);
+	i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_DCB, "%s TOS:[%d] UP:[%d]\n",
+		    __func__, cm_id->tos, cm_info.user_pri);
 	if ((cm_info.ipv4 && (laddr->sin_addr.s_addr != raddr->sin_addr.s_addr)) ||
 	    (!cm_info.ipv4 && memcmp(laddr6->sin6_addr.in6_u.u6_addr32,
 				     raddr6->sin6_addr.in6_u.u6_addr32,
@@ -3790,8 +3828,11 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 				       conn_param->private_data_len,
 				       (void *)conn_param->private_data,
 				       &cm_info);
-	if (!cm_node)
-		goto err;
+
+	if (IS_ERR(cm_node)) {
+		err = PTR_ERR(cm_node);
+		goto err_out;
+	}
 
 	i40iw_record_ird_ord(cm_node, (u16)conn_param->ird, (u16)conn_param->ord);
 	if (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO &&
@@ -3805,10 +3846,12 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 	iwqp->cm_id = cm_id;
 	i40iw_add_ref(&iwqp->ibqp);
 
-	if (cm_node->state == I40IW_CM_STATE_SYN_SENT) {
-		if (i40iw_send_syn(cm_node, 0)) {
+	if (cm_node->state != I40IW_CM_STATE_OFFLOADED) {
+		cm_node->state = I40IW_CM_STATE_SYN_SENT;
+		err = i40iw_send_syn(cm_node, 0);
+		if (err) {
 			i40iw_rem_ref_cm_node(cm_node);
-			goto err;
+			goto err_out;
 		}
 	}
 
@@ -3820,24 +3863,25 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 		    cm_node->cm_id);
 	return 0;
 
-err:
-	if (cm_node) {
-		if (cm_node->ipv4)
-			i40iw_debug(cm_node->dev,
-				    I40IW_DEBUG_CM,
-				    "Api - connect() FAILED: dest addr=%pI4",
-				    cm_node->rem_addr);
-		else
-			i40iw_debug(cm_node->dev, I40IW_DEBUG_CM,
-				    "Api - connect() FAILED: dest addr=%pI6",
-				    cm_node->rem_addr);
-	}
-	i40iw_manage_qhash(iwdev,
-			   &cm_info,
-			   I40IW_QHASH_TYPE_TCP_ESTABLISHED,
-			   I40IW_QHASH_MANAGE_TYPE_DELETE,
-			   NULL,
-			   false);
+err_out:
+	if (cm_info.ipv4)
+		i40iw_debug(&iwdev->sc_dev,
+			    I40IW_DEBUG_CM,
+			    "Api - connect() FAILED: dest addr=%pI4",
+			    cm_info.rem_addr);
+	else
+		i40iw_debug(&iwdev->sc_dev,
+			    I40IW_DEBUG_CM,
+			    "Api - connect() FAILED: dest addr=%pI6",
+			    cm_info.rem_addr);
+
+	if (qhash_set)
+		i40iw_manage_qhash(iwdev,
+				   &cm_info,
+				   I40IW_QHASH_TYPE_TCP_ESTABLISHED,
+				   I40IW_QHASH_MANAGE_TYPE_DELETE,
+				   NULL,
+				   false);
 
 	if (apbvt_set && !i40iw_listen_port_in_use(&iwdev->cm_core,
 						   cm_info.loc_port))
@@ -3846,7 +3890,7 @@ err:
 				   I40IW_MANAGE_APBVT_DEL);
 	cm_id->rem_ref(cm_id);
 	iwdev->cm_core.stats_connect_errs++;
-	return -ENOMEM;
+	return err;
 }
 
 /**
@@ -3904,6 +3948,10 @@ int i40iw_create_listen(struct iw_cm_id *cm_id, int backlog)
 
 	cm_id->provider_data = cm_listen_node;
 
+	cm_listen_node->tos = cm_id->tos;
+	cm_listen_node->user_pri = rt_tos2priority(cm_id->tos);
+	cm_info.user_pri = cm_listen_node->user_pri;
+
 	if (!cm_listen_node->reused_node) {
 		if (wildcard) {
 			if (cm_info.ipv4)
@@ -4124,3 +4172,158 @@ static void i40iw_cm_post_event(struct i40iw_cm_event *event)
 
 	queue_work(event->cm_node->cm_core->event_wq, &event->event_work);
 }
+
+/**
+ * i40iw_qhash_ctrl - enable/disable qhash for list
+ * @iwdev: device pointer
+ * @parent_listen_node: parent listen node
+ * @nfo: cm info node
+ * @ipaddr: Pointer to IPv4 or IPv6 address
+ * @ipv4: flag indicating IPv4 when true
+ * @ifup: flag indicating interface up when true
+ *
+ * Enables or disables the qhash for the node in the child
+ * listen list that matches ipaddr. If no matching IP was found
+ * it will allocate and add a new child listen node to the
+ * parent listen node. The listen_list_lock is assumed to be
+ * held when called.
+ */
+static void i40iw_qhash_ctrl(struct i40iw_device *iwdev,
+			     struct i40iw_cm_listener *parent_listen_node,
+			     struct i40iw_cm_info *nfo,
+			     u32 *ipaddr, bool ipv4, bool ifup)
+{
+	struct list_head *child_listen_list = &parent_listen_node->child_listen_list;
+	struct i40iw_cm_listener *child_listen_node;
+	struct list_head *pos, *tpos;
+	enum i40iw_status_code ret;
+	bool node_allocated = false;
+	enum i40iw_quad_hash_manage_type op =
+		ifup ? I40IW_QHASH_MANAGE_TYPE_ADD : I40IW_QHASH_MANAGE_TYPE_DELETE;
+
+	list_for_each_safe(pos, tpos, child_listen_list) {
+		child_listen_node =
+			list_entry(pos,
+				   struct i40iw_cm_listener,
+				   child_listen_list);
+		if (!memcmp(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16))
+			goto set_qhash;
+	}
+
+	/* if not found then add a child listener if interface is going up */
+	if (!ifup)
+		return;
+	child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_ATOMIC);
+	if (!child_listen_node)
+		return;
+	node_allocated = true;
+	memcpy(child_listen_node, parent_listen_node, sizeof(*child_listen_node));
+
+	memcpy(child_listen_node->loc_addr, ipaddr,  ipv4 ? 4 : 16);
+
+set_qhash:
+	memcpy(nfo->loc_addr,
+	       child_listen_node->loc_addr,
+	       sizeof(nfo->loc_addr));
+	nfo->vlan_id = child_listen_node->vlan_id;
+	ret = i40iw_manage_qhash(iwdev, nfo,
+				 I40IW_QHASH_TYPE_TCP_SYN,
+				 op,
+				 NULL, false);
+	if (!ret) {
+		child_listen_node->qhash_set = ifup;
+		if (node_allocated)
+			list_add(&child_listen_node->child_listen_list,
+				 &parent_listen_node->child_listen_list);
+	} else if (node_allocated) {
+		kfree(child_listen_node);
+	}
+}
+
+/**
+ * i40iw_cm_disconnect_all - disconnect all connected qp's
+ * @iwdev: device pointer
+ */
+void i40iw_cm_disconnect_all(struct i40iw_device *iwdev)
+{
+	struct i40iw_cm_core *cm_core = &iwdev->cm_core;
+	struct list_head *list_core_temp;
+	struct list_head *list_node;
+	struct i40iw_cm_node *cm_node;
+	unsigned long flags;
+	struct list_head connected_list;
+	struct ib_qp_attr attr;
+
+	INIT_LIST_HEAD(&connected_list);
+	spin_lock_irqsave(&cm_core->ht_lock, flags);
+	list_for_each_safe(list_node, list_core_temp, &cm_core->connected_nodes) {
+		cm_node = container_of(list_node, struct i40iw_cm_node, list);
+		atomic_inc(&cm_node->ref_count);
+		list_add(&cm_node->connected_entry, &connected_list);
+	}
+	spin_unlock_irqrestore(&cm_core->ht_lock, flags);
+
+	list_for_each_safe(list_node, list_core_temp, &connected_list) {
+		cm_node = container_of(list_node, struct i40iw_cm_node, connected_entry);
+		attr.qp_state = IB_QPS_ERR;
+		i40iw_modify_qp(&cm_node->iwqp->ibqp, &attr, IB_QP_STATE, NULL);
+		i40iw_rem_ref_cm_node(cm_node);
+	}
+}
+
+/**
+ * i40iw_ifdown_notify - process an ifdown on an interface
+ * @iwdev: device pointer
+ * @ipaddr: Pointer to IPv4 or IPv6 address
+ * @ipv4: flag indicating IPv4 when true
+ * @ifup: flag indicating interface up when true
+ */
+void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev,
+		     u32 *ipaddr, bool ipv4, bool ifup)
+{
+	struct i40iw_cm_core *cm_core = &iwdev->cm_core;
+	unsigned long flags;
+	struct i40iw_cm_listener *listen_node;
+	static const u32 ip_zero[4] = { 0, 0, 0, 0 };
+	struct i40iw_cm_info nfo;
+	u16 vlan_id = rdma_vlan_dev_vlan_id(netdev);
+	enum i40iw_status_code ret;
+	enum i40iw_quad_hash_manage_type op =
+		ifup ? I40IW_QHASH_MANAGE_TYPE_ADD : I40IW_QHASH_MANAGE_TYPE_DELETE;
+
+	/* Disable or enable qhash for listeners */
+	spin_lock_irqsave(&cm_core->listen_list_lock, flags);
+	list_for_each_entry(listen_node, &cm_core->listen_nodes, list) {
+		if (vlan_id == listen_node->vlan_id &&
+		    (!memcmp(listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16) ||
+		    !memcmp(listen_node->loc_addr, ip_zero, ipv4 ? 4 : 16))) {
+			memcpy(nfo.loc_addr, listen_node->loc_addr,
+			       sizeof(nfo.loc_addr));
+			nfo.loc_port = listen_node->loc_port;
+			nfo.ipv4 = listen_node->ipv4;
+			nfo.vlan_id = listen_node->vlan_id;
+			nfo.user_pri = listen_node->user_pri;
+			if (!list_empty(&listen_node->child_listen_list)) {
+				i40iw_qhash_ctrl(iwdev,
+						 listen_node,
+						 &nfo,
+						 ipaddr, ipv4, ifup);
+			} else if (memcmp(listen_node->loc_addr, ip_zero,
+					  ipv4 ? 4 : 16)) {
+				ret = i40iw_manage_qhash(iwdev,
+							 &nfo,
+							 I40IW_QHASH_TYPE_TCP_SYN,
+							 op,
+							 NULL,
+							 false);
+				if (!ret)
+					listen_node->qhash_set = ifup;
+			}
+		}
+	}
+	spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
+
+	/* disconnect any connected qp's on ifdown */
+	if (!ifup)
+		i40iw_cm_disconnect_all(iwdev);
+}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.h b/drivers/infiniband/hw/i40iw/i40iw_cm.h
index e9046d9f9645..2e52e38ffcf3 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_cm.h
@@ -56,8 +56,6 @@
 
 #define I40IW_MAX_IETF_SIZE      32
 
-#define MPA_ZERO_PAD_LEN	4
-
 /* IETF RTR MSG Fields               */
 #define IETF_PEER_TO_PEER       0x8000
 #define IETF_FLPDU_ZERO_LEN     0x4000
@@ -299,6 +297,7 @@ struct i40iw_cm_listener {
 	enum i40iw_cm_listener_state listener_state;
 	u32 reused_node;
 	u8 user_pri;
+	u8 tos;
 	u16 vlan_id;
 	bool qhash_set;
 	bool ipv4;
@@ -341,9 +340,11 @@ struct i40iw_cm_node {
 	int accept_pend;
 	struct list_head timer_entry;
 	struct list_head reset_entry;
+	struct list_head connected_entry;
 	atomic_t passive_state;
 	bool qhash_set;
 	u8 user_pri;
+	u8 tos;
 	bool ipv4;
 	bool snd_mark_en;
 	u16 lsmm_size;
@@ -368,7 +369,8 @@ struct i40iw_cm_info {
 	u32 rem_addr[4];
 	u16 vlan_id;
 	int backlog;
-	u16 user_pri;
+	u8 user_pri;
+	u8 tos;
 	bool ipv4;
 };
 
@@ -445,4 +447,7 @@ int i40iw_arp_table(struct i40iw_device *iwdev,
 		    u8 *mac_addr,
 		    u32 action);
 
+void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev,
+		     u32 *ipaddr, bool ipv4, bool ifup);
+void i40iw_cm_disconnect_all(struct i40iw_device *iwdev);
 #endif /* I40IW_CM_H */
diff --git a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
index 2c4b4d072d6a..392f78384a60 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
@@ -103,6 +103,7 @@ static enum i40iw_status_code i40iw_cqp_poll_registers(
 		if (newtail != tail) {
 			/* SUCCESS */
 			I40IW_RING_MOVE_TAIL(cqp->sq_ring);
+			cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
 			return 0;
 		}
 		udelay(I40IW_SLEEP_COUNT);
@@ -223,6 +224,136 @@ static enum i40iw_status_code i40iw_sc_parse_fpm_query_buf(
 }
 
 /**
+ * i40iw_fill_qos_list - Change all unknown qs handles to available ones
+ * @qs_list: list of qs_handles to be fixed with valid qs_handles
+ */
+static void i40iw_fill_qos_list(u16 *qs_list)
+{
+	u16 qshandle = qs_list[0];
+	int i;
+
+	for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
+		if (qs_list[i] == QS_HANDLE_UNKNOWN)
+			qs_list[i] = qshandle;
+		else
+			qshandle = qs_list[i];
+	}
+}
+
+/**
+ * i40iw_qp_from_entry - Given entry, get to the qp structure
+ * @entry: Points to list of qp structure
+ */
+static struct i40iw_sc_qp *i40iw_qp_from_entry(struct list_head *entry)
+{
+	if (!entry)
+		return NULL;
+
+	return (struct i40iw_sc_qp *)((char *)entry - offsetof(struct i40iw_sc_qp, list));
+}
+
+/**
+ * i40iw_get_qp - get the next qp from the list given current qp
+ * @head: Listhead of qp's
+ * @qp: current qp
+ */
+static struct i40iw_sc_qp *i40iw_get_qp(struct list_head *head, struct i40iw_sc_qp *qp)
+{
+	struct list_head *entry = NULL;
+	struct list_head *lastentry;
+
+	if (list_empty(head))
+		return NULL;
+
+	if (!qp) {
+		entry = head->next;
+	} else {
+		lastentry = &qp->list;
+		entry = (lastentry != head) ? lastentry->next : NULL;
+	}
+
+	return i40iw_qp_from_entry(entry);
+}
+
+/**
+ * i40iw_change_l2params - given the new l2 parameters, change all qp
+ * @vsi: pointer to the vsi structure
+ * @l2params: New paramaters from l2
+ */
+void i40iw_change_l2params(struct i40iw_sc_vsi *vsi, struct i40iw_l2params *l2params)
+{
+	struct i40iw_sc_dev *dev = vsi->dev;
+	struct i40iw_sc_qp *qp = NULL;
+	bool qs_handle_change = false;
+	bool mss_change = false;
+	unsigned long flags;
+	u16 qs_handle;
+	int i;
+
+	if (vsi->mss != l2params->mss) {
+		mss_change = true;
+		vsi->mss = l2params->mss;
+	}
+
+	i40iw_fill_qos_list(l2params->qs_handle_list);
+	for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
+		qs_handle = l2params->qs_handle_list[i];
+		if (vsi->qos[i].qs_handle != qs_handle)
+			qs_handle_change = true;
+		else if (!mss_change)
+			continue;       /* no MSS nor qs handle change */
+		spin_lock_irqsave(&vsi->qos[i].lock, flags);
+		qp = i40iw_get_qp(&vsi->qos[i].qplist, qp);
+		while (qp) {
+			if (mss_change)
+				i40iw_qp_mss_modify(dev, qp);
+			if (qs_handle_change) {
+				qp->qs_handle = qs_handle;
+				/* issue cqp suspend command */
+				i40iw_qp_suspend_resume(dev, qp, true);
+			}
+			qp = i40iw_get_qp(&vsi->qos[i].qplist, qp);
+		}
+		spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
+		vsi->qos[i].qs_handle = qs_handle;
+	}
+}
+
+/**
+ * i40iw_qp_rem_qos - remove qp from qos lists during destroy qp
+ * @qp: qp to be removed from qos
+ */
+static void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp)
+{
+	struct i40iw_sc_vsi *vsi = qp->vsi;
+	unsigned long flags;
+
+	if (!qp->on_qoslist)
+		return;
+	spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
+	list_del(&qp->list);
+	spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
+}
+
+/**
+ * i40iw_qp_add_qos - called during setctx fot qp to be added to qos
+ * @qp: qp to be added to qos
+ */
+void i40iw_qp_add_qos(struct i40iw_sc_qp *qp)
+{
+	struct i40iw_sc_vsi *vsi = qp->vsi;
+	unsigned long flags;
+
+	if (qp->on_qoslist)
+		return;
+	spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
+	qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
+	list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);
+	qp->on_qoslist = true;
+	spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
+}
+
+/**
  * i40iw_sc_pd_init - initialize sc pd struct
  * @dev: sc device struct
  * @pd: sc pd ptr
@@ -292,6 +423,9 @@ static enum i40iw_status_code i40iw_sc_cqp_init(struct i40iw_sc_cqp *cqp,
 	info->dev->cqp = cqp;
 
 	I40IW_RING_INIT(cqp->sq_ring, cqp->sq_size);
+	cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] = 0;
+	cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS] = 0;
+
 	i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
 		    "%s: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04X]\n",
 		    __func__, cqp->sq_size, cqp->hw_sq_size,
@@ -302,12 +436,10 @@ static enum i40iw_status_code i40iw_sc_cqp_init(struct i40iw_sc_cqp *cqp,
 /**
  * i40iw_sc_cqp_create - create cqp during bringup
  * @cqp: struct for cqp hw
- * @disable_pfpdus: if pfpdu to be disabled
  * @maj_err: If error, major err number
  * @min_err: If error, minor err number
  */
 static enum i40iw_status_code i40iw_sc_cqp_create(struct i40iw_sc_cqp *cqp,
-						  bool disable_pfpdus,
 						  u16 *maj_err,
 						  u16 *min_err)
 {
@@ -326,9 +458,6 @@ static enum i40iw_status_code i40iw_sc_cqp_create(struct i40iw_sc_cqp *cqp,
 	temp = LS_64(cqp->hw_sq_size, I40IW_CQPHC_SQSIZE) |
 	       LS_64(cqp->struct_ver, I40IW_CQPHC_SVER);
 
-	if (disable_pfpdus)
-		temp |= LS_64(1, I40IW_CQPHC_DISABLE_PFPDUS);
-
 	set_64bit_val(cqp->host_ctx, 0, temp);
 	set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);
 	temp = LS_64(cqp->enabled_vf_count, I40IW_CQPHC_ENABLED_VFS) |
@@ -424,6 +553,7 @@ u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch)
 		return NULL;
 	}
 	I40IW_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, wqe_idx, ret_code);
+	cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS]++;
 	if (ret_code)
 		return NULL;
 	if (!wqe_idx)
@@ -559,6 +689,8 @@ static enum i40iw_status_code i40iw_sc_ccq_get_cqe_info(
 		      I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring));
 	wmb(); /* write shadow area before tail */
 	I40IW_RING_MOVE_TAIL(cqp->sq_ring);
+	ccq->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
+
 	return ret_code;
 }
 
@@ -1051,6 +1183,7 @@ static enum i40iw_status_code i40iw_sc_manage_qhash_table_entry(
 	u64 qw1 = 0;
 	u64 qw2 = 0;
 	u64 temp;
+	struct i40iw_sc_vsi *vsi = info->vsi;
 
 	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 	if (!wqe)
@@ -1082,7 +1215,7 @@ static enum i40iw_status_code i40iw_sc_manage_qhash_table_entry(
 			      LS_64(info->dest_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |
 			      LS_64(info->dest_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
 	}
-	qw2 = LS_64(cqp->dev->qs_handle, I40IW_CQPSQ_QHASH_QS_HANDLE);
+	qw2 = LS_64(vsi->qos[info->user_pri].qs_handle, I40IW_CQPSQ_QHASH_QS_HANDLE);
 	if (info->vlan_valid)
 		qw2 |= LS_64(info->vlan_id, I40IW_CQPSQ_QHASH_VLANID);
 	set_64bit_val(wqe, 16, qw2);
@@ -2103,6 +2236,7 @@ static enum i40iw_status_code i40iw_sc_qp_init(struct i40iw_sc_qp *qp,
 	u32 offset;
 
 	qp->dev = info->pd->dev;
+	qp->vsi = info->vsi;
 	qp->sq_pa = info->sq_pa;
 	qp->rq_pa = info->rq_pa;
 	qp->hw_host_ctx_pa = info->host_ctx_pa;
@@ -2151,7 +2285,7 @@ static enum i40iw_status_code i40iw_sc_qp_init(struct i40iw_sc_qp *qp,
 	qp->rq_tph_en = info->rq_tph_en;
 	qp->rcv_tph_en = info->rcv_tph_en;
 	qp->xmit_tph_en = info->xmit_tph_en;
-	qp->qs_handle = qp->pd->dev->qs_handle;
+	qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;
 	qp->exception_lan_queue = qp->pd->dev->exception_lan_queue;
 
 	return 0;
@@ -2296,6 +2430,7 @@ static enum i40iw_status_code i40iw_sc_qp_destroy(
 	struct i40iw_sc_cqp *cqp;
 	u64 header;
 
+	i40iw_qp_rem_qos(qp);
 	cqp = qp->pd->dev->cqp;
 	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 	if (!wqe)
@@ -2443,10 +2578,20 @@ static enum i40iw_status_code i40iw_sc_qp_setctx(
 {
 	struct i40iwarp_offload_info *iw;
 	struct i40iw_tcp_offload_info *tcp;
+	struct i40iw_sc_vsi *vsi;
+	struct i40iw_sc_dev *dev;
 	u64 qw0, qw3, qw7 = 0;
 
 	iw = info->iwarp_info;
 	tcp = info->tcp_info;
+	vsi = qp->vsi;
+	dev = qp->dev;
+	if (info->add_to_qoslist) {
+		qp->user_pri = info->user_pri;
+		i40iw_qp_add_qos(qp);
+		i40iw_debug(qp->dev, I40IW_DEBUG_DCB, "%s qp[%d] UP[%d] qset[%d]\n",
+			    __func__, qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle);
+	}
 	qw0 = LS_64(qp->qp_uk.rq_wqe_size, I40IWQPC_RQWQESIZE) |
 	      LS_64(info->err_rq_idx_valid, I40IWQPC_ERR_RQ_IDX_VALID) |
 	      LS_64(qp->rcv_tph_en, I40IWQPC_RCVTPHEN) |
@@ -2487,16 +2632,14 @@ static enum i40iw_status_code i40iw_sc_qp_setctx(
 		       LS_64(iw->rdmap_ver, I40IWQPC_RDMAP_VER);
 
 		qw7 |= LS_64(iw->pd_id, I40IWQPC_PDIDX);
-		set_64bit_val(qp_ctx, 144, qp->q2_pa);
+		set_64bit_val(qp_ctx,
+			      144,
+			      LS_64(qp->q2_pa, I40IWQPC_Q2ADDR) |
+			      LS_64(vsi->fcn_id, I40IWQPC_STAT_INDEX));
 		set_64bit_val(qp_ctx,
 			      152,
 			      LS_64(iw->last_byte_sent, I40IWQPC_LASTBYTESENT));
 
-		/*
-		* Hard-code IRD_SIZE to hw-limit, 128, in qpctx, i.e matching an
-		*advertisable IRD of 64
-		*/
-		iw->ird_size = I40IW_QPCTX_ENCD_MAXIRD;
 		set_64bit_val(qp_ctx,
 			      160,
 			      LS_64(iw->ord_size, I40IWQPC_ORDSIZE) |
@@ -2507,6 +2650,9 @@ static enum i40iw_status_code i40iw_sc_qp_setctx(
 			      LS_64(iw->bind_en, I40IWQPC_BINDEN) |
 			      LS_64(iw->fast_reg_en, I40IWQPC_FASTREGEN) |
 			      LS_64(iw->priv_mode_en, I40IWQPC_PRIVEN) |
+			      LS_64((((vsi->stats_fcn_id_alloc) &&
+				      (dev->is_pf) && (vsi->fcn_id >= I40IW_FIRST_NON_PF_STAT)) ? 1 : 0),
+				    I40IWQPC_USESTATSINSTANCE) |
 			      LS_64(1, I40IWQPC_IWARPMODE) |
 			      LS_64(iw->rcv_mark_en, I40IWQPC_RCVMARKERS) |
 			      LS_64(iw->align_hdrs, I40IWQPC_ALIGNHDRS) |
@@ -2623,7 +2769,9 @@ static enum i40iw_status_code i40iw_sc_alloc_stag(
 	u64 *wqe;
 	struct i40iw_sc_cqp *cqp;
 	u64 header;
+	enum i40iw_page_size page_size;
 
+	page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
 	cqp = dev->cqp;
 	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 	if (!wqe)
@@ -2643,7 +2791,7 @@ static enum i40iw_status_code i40iw_sc_alloc_stag(
 		 LS_64(1, I40IW_CQPSQ_STAG_MR) |
 		 LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
 		 LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |
-		 LS_64(info->page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
+		 LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
 		 LS_64(info->remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
 		 LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |
 		 LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |
@@ -2679,7 +2827,9 @@ static enum i40iw_status_code i40iw_sc_mr_reg_non_shared(
 	u32 pble_obj_cnt;
 	bool remote_access;
 	u8 addr_type;
+	enum i40iw_page_size page_size;
 
+	page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
 	if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |
 				   I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
 		remote_access = true;
@@ -2722,7 +2872,7 @@ static enum i40iw_status_code i40iw_sc_mr_reg_non_shared(
 	header = LS_64(I40IW_CQP_OP_REG_MR, I40IW_CQPSQ_OPCODE) |
 		 LS_64(1, I40IW_CQPSQ_STAG_MR) |
 		 LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |
-		 LS_64(info->page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
+		 LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
 		 LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
 		 LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
 		 LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |
@@ -2937,7 +3087,9 @@ enum i40iw_status_code i40iw_sc_mr_fast_register(
 	u64 temp, header;
 	u64 *wqe;
 	u32 wqe_idx;
+	enum i40iw_page_size page_size;
 
+	page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
 	wqe = i40iw_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
 					 0, info->wr_id);
 	if (!wqe)
@@ -2964,7 +3116,7 @@ enum i40iw_status_code i40iw_sc_mr_fast_register(
 		 LS_64(info->stag_idx, I40IWQPSQ_STAGINDEX) |
 		 LS_64(I40IWQP_OP_FAST_REGISTER, I40IWQPSQ_OPCODE) |
 		 LS_64(info->chunk_size, I40IWQPSQ_LPBLSIZE) |
-		 LS_64(info->page_size, I40IWQPSQ_HPAGESIZE) |
+		 LS_64(page_size, I40IWQPSQ_HPAGESIZE) |
 		 LS_64(info->access_rights, I40IWQPSQ_STAGRIGHTS) |
 		 LS_64(info->addr_type, I40IWQPSQ_VABASEDTO) |
 		 LS_64(info->read_fence, I40IWQPSQ_READFENCE) |
@@ -3959,7 +4111,7 @@ enum i40iw_status_code i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev,
 					     struct cqp_commands_info *pcmdinfo)
 {
 	enum i40iw_status_code status = 0;
-	unsigned long	flags;
+	unsigned long flags;
 
 	spin_lock_irqsave(&dev->cqp_lock, flags);
 	if (list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp))
@@ -3978,7 +4130,7 @@ enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev)
 {
 	enum i40iw_status_code status = 0;
 	struct cqp_commands_info *pcmdinfo;
-	unsigned long	flags;
+	unsigned long flags;
 
 	spin_lock_irqsave(&dev->cqp_lock, flags);
 	while (!list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp)) {
@@ -4055,7 +4207,6 @@ static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp *qp,
 	u16 ddp_seg_len;
 	int copy_len = 0;
 	u8 is_tagged = 0;
-	enum i40iw_flush_opcode flush_code = FLUSH_INVALID;
 	u32 opcode;
 	struct i40iw_terminate_hdr *termhdr;
 
@@ -4228,9 +4379,6 @@ static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp *qp,
 	if (copy_len)
 		memcpy(termhdr + 1, pkt, copy_len);
 
-	if (flush_code && !info->in_rdrsp_wr)
-		qp->sq_flush = (info->sq) ? true : false;
-
 	return sizeof(struct i40iw_terminate_hdr) + copy_len;
 }
 
@@ -4321,286 +4469,370 @@ void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *in
 }
 
 /**
- * i40iw_hw_stat_init - Initiliaze HW stats table
- * @devstat: pestat struct
+ * i40iw_sc_vsi_init - Initialize virtual device
+ * @vsi: pointer to the vsi structure
+ * @info: parameters to initialize vsi
+ **/
+void i40iw_sc_vsi_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_init_info *info)
+{
+	int i;
+
+	vsi->dev = info->dev;
+	vsi->back_vsi = info->back_vsi;
+	vsi->mss = info->params->mss;
+	i40iw_fill_qos_list(info->params->qs_handle_list);
+
+	for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
+		vsi->qos[i].qs_handle =
+			info->params->qs_handle_list[i];
+			i40iw_debug(vsi->dev, I40IW_DEBUG_DCB, "qset[%d]: %d\n", i, vsi->qos[i].qs_handle);
+		spin_lock_init(&vsi->qos[i].lock);
+		INIT_LIST_HEAD(&vsi->qos[i].qplist);
+	}
+}
+
+/**
+ * i40iw_hw_stats_init - Initiliaze HW stats table
+ * @stats: pestat struct
  * @fcn_idx: PCI fn id
- * @hw: PF i40iw_hw structure.
  * @is_pf: Is it a PF?
  *
- * Populate the HW stat table with register offset addr for each
- * stat. And start the perioidic stats timer.
+ * Populate the HW stats table with register offset addr for each
+ * stats. And start the perioidic stats timer.
  */
-static void i40iw_hw_stat_init(struct i40iw_dev_pestat *devstat,
-			       u8 fcn_idx,
-			       struct i40iw_hw *hw, bool is_pf)
+void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 fcn_idx, bool is_pf)
 {
-	u32 stat_reg_offset;
-	u32 stat_index;
-	struct i40iw_dev_hw_stat_offsets *stat_table =
-		&devstat->hw_stat_offsets;
-	struct i40iw_dev_hw_stats *last_rd_stats = &devstat->last_read_hw_stats;
-
-	devstat->hw = hw;
+	u32 stats_reg_offset;
+	u32 stats_index;
+	struct i40iw_dev_hw_stats_offsets *stats_table =
+		&stats->hw_stats_offsets;
+	struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
 
 	if (is_pf) {
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
 				I40E_GLPES_PFIP4RXDISCARD(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
 				I40E_GLPES_PFIP4RXTRUNC(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
 				I40E_GLPES_PFIP4TXNOROUTE(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
 				I40E_GLPES_PFIP6RXDISCARD(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
 				I40E_GLPES_PFIP6RXTRUNC(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
 				I40E_GLPES_PFIP6TXNOROUTE(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
 				I40E_GLPES_PFTCPRTXSEG(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
 				I40E_GLPES_PFTCPRXOPTERR(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
 				I40E_GLPES_PFTCPRXPROTOERR(fcn_idx);
 
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
 				I40E_GLPES_PFIP4RXOCTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
 				I40E_GLPES_PFIP4RXPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
 				I40E_GLPES_PFIP4RXFRAGSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
 				I40E_GLPES_PFIP4RXMCPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
 				I40E_GLPES_PFIP4TXOCTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
 				I40E_GLPES_PFIP4TXPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
 				I40E_GLPES_PFIP4TXFRAGSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
 				I40E_GLPES_PFIP4TXMCPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
 				I40E_GLPES_PFIP6RXOCTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
 				I40E_GLPES_PFIP6RXPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
 				I40E_GLPES_PFIP6RXFRAGSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
 				I40E_GLPES_PFIP6RXMCPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
 				I40E_GLPES_PFIP6TXOCTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
 				I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
 				I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
 				I40E_GLPES_PFIP6TXFRAGSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
 				I40E_GLPES_PFTCPRXSEGSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
 				I40E_GLPES_PFTCPTXSEGLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
 				I40E_GLPES_PFRDMARXRDSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
 				I40E_GLPES_PFRDMARXSNDSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
 				I40E_GLPES_PFRDMARXWRSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
 				I40E_GLPES_PFRDMATXRDSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
 				I40E_GLPES_PFRDMATXSNDSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
 				I40E_GLPES_PFRDMATXWRSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
 				I40E_GLPES_PFRDMAVBNDLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
 				I40E_GLPES_PFRDMAVINVLO(fcn_idx);
 	} else {
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
 				I40E_GLPES_VFIP4RXDISCARD(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
 				I40E_GLPES_VFIP4RXTRUNC(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
 				I40E_GLPES_VFIP4TXNOROUTE(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
 				I40E_GLPES_VFIP6RXDISCARD(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
 				I40E_GLPES_VFIP6RXTRUNC(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
 				I40E_GLPES_VFIP6TXNOROUTE(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
 				I40E_GLPES_VFTCPRTXSEG(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
 				I40E_GLPES_VFTCPRXOPTERR(fcn_idx);
-		stat_table->stat_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
+		stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
 				I40E_GLPES_VFTCPRXPROTOERR(fcn_idx);
 
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
 				I40E_GLPES_VFIP4RXOCTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
 				I40E_GLPES_VFIP4RXPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
 				I40E_GLPES_VFIP4RXFRAGSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
 				I40E_GLPES_VFIP4RXMCPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
 				I40E_GLPES_VFIP4TXOCTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
 				I40E_GLPES_VFIP4TXPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
 				I40E_GLPES_VFIP4TXFRAGSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
 				I40E_GLPES_VFIP4TXMCPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
 				I40E_GLPES_VFIP6RXOCTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
 				I40E_GLPES_VFIP6RXPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
 				I40E_GLPES_VFIP6RXFRAGSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
 				I40E_GLPES_VFIP6RXMCPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
 				I40E_GLPES_VFIP6TXOCTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
 				I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
 				I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
 				I40E_GLPES_VFIP6TXFRAGSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
 				I40E_GLPES_VFTCPRXSEGSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
 				I40E_GLPES_VFTCPTXSEGLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
 				I40E_GLPES_VFRDMARXRDSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
 				I40E_GLPES_VFRDMARXSNDSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
 				I40E_GLPES_VFRDMARXWRSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
 				I40E_GLPES_VFRDMATXRDSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
 				I40E_GLPES_VFRDMATXSNDSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
 				I40E_GLPES_VFRDMATXWRSLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
 				I40E_GLPES_VFRDMAVBNDLO(fcn_idx);
-		stat_table->stat_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
+		stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
 				I40E_GLPES_VFRDMAVINVLO(fcn_idx);
 	}
 
-	for (stat_index = 0; stat_index < I40IW_HW_STAT_INDEX_MAX_64;
-	     stat_index++) {
-		stat_reg_offset = stat_table->stat_offset_64[stat_index];
-		last_rd_stats->stat_value_64[stat_index] =
-			readq(devstat->hw->hw_addr + stat_reg_offset);
+	for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
+	     stats_index++) {
+		stats_reg_offset = stats_table->stats_offset_64[stats_index];
+		last_rd_stats->stats_value_64[stats_index] =
+			readq(stats->hw->hw_addr + stats_reg_offset);
 	}
 
-	for (stat_index = 0; stat_index < I40IW_HW_STAT_INDEX_MAX_32;
-	     stat_index++) {
-		stat_reg_offset = stat_table->stat_offset_32[stat_index];
-		last_rd_stats->stat_value_32[stat_index] =
-			i40iw_rd32(devstat->hw, stat_reg_offset);
+	for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
+	     stats_index++) {
+		stats_reg_offset = stats_table->stats_offset_32[stats_index];
+		last_rd_stats->stats_value_32[stats_index] =
+			i40iw_rd32(stats->hw, stats_reg_offset);
 	}
 }
 
 /**
- * i40iw_hw_stat_read_32 - Read 32-bit HW stat counters and accommodates for roll-overs.
- * @devstat: pestat struct
- * @index: index in HW stat table which contains offset reg-addr
- * @value: hw stat value
+ * i40iw_hw_stats_read_32 - Read 32-bit HW stats counters and accommodates for roll-overs.
+ * @stat: pestat struct
+ * @index: index in HW stats table which contains offset reg-addr
+ * @value: hw stats value
  */
-static void i40iw_hw_stat_read_32(struct i40iw_dev_pestat *devstat,
-				  enum i40iw_hw_stat_index_32b index,
-				  u64 *value)
+void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats,
+			    enum i40iw_hw_stats_index_32b index,
+			    u64 *value)
 {
-	struct i40iw_dev_hw_stat_offsets *stat_table =
-		&devstat->hw_stat_offsets;
-	struct i40iw_dev_hw_stats *last_rd_stats = &devstat->last_read_hw_stats;
-	struct i40iw_dev_hw_stats *hw_stats = &devstat->hw_stats;
-	u64 new_stat_value = 0;
-	u32 stat_reg_offset = stat_table->stat_offset_32[index];
-
-	new_stat_value = i40iw_rd32(devstat->hw, stat_reg_offset);
+	struct i40iw_dev_hw_stats_offsets *stats_table =
+		&stats->hw_stats_offsets;
+	struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
+	struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
+	u64 new_stats_value = 0;
+	u32 stats_reg_offset = stats_table->stats_offset_32[index];
+
+	new_stats_value = i40iw_rd32(stats->hw, stats_reg_offset);
 	/*roll-over case */
-	if (new_stat_value < last_rd_stats->stat_value_32[index])
-		hw_stats->stat_value_32[index] += new_stat_value;
+	if (new_stats_value < last_rd_stats->stats_value_32[index])
+		hw_stats->stats_value_32[index] += new_stats_value;
 	else
-		hw_stats->stat_value_32[index] +=
-			new_stat_value - last_rd_stats->stat_value_32[index];
-	last_rd_stats->stat_value_32[index] = new_stat_value;
-	*value = hw_stats->stat_value_32[index];
+		hw_stats->stats_value_32[index] +=
+			new_stats_value - last_rd_stats->stats_value_32[index];
+	last_rd_stats->stats_value_32[index] = new_stats_value;
+	*value = hw_stats->stats_value_32[index];
 }
 
 /**
- * i40iw_hw_stat_read_64 - Read HW stat counters (greater than 32-bit) and accommodates for roll-overs.
- * @devstat: pestat struct
- * @index: index in HW stat table which contains offset reg-addr
- * @value: hw stat value
+ * i40iw_hw_stats_read_64 - Read HW stats counters (greater than 32-bit) and accommodates for roll-overs.
+ * @stats: pestat struct
+ * @index: index in HW stats table which contains offset reg-addr
+ * @value: hw stats value
  */
-static void i40iw_hw_stat_read_64(struct i40iw_dev_pestat *devstat,
-				  enum i40iw_hw_stat_index_64b index,
-				  u64 *value)
+void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats,
+			    enum i40iw_hw_stats_index_64b index,
+			    u64 *value)
 {
-	struct i40iw_dev_hw_stat_offsets *stat_table =
-		&devstat->hw_stat_offsets;
-	struct i40iw_dev_hw_stats *last_rd_stats = &devstat->last_read_hw_stats;
-	struct i40iw_dev_hw_stats *hw_stats = &devstat->hw_stats;
-	u64 new_stat_value = 0;
-	u32 stat_reg_offset = stat_table->stat_offset_64[index];
-
-	new_stat_value = readq(devstat->hw->hw_addr + stat_reg_offset);
+	struct i40iw_dev_hw_stats_offsets *stats_table =
+		&stats->hw_stats_offsets;
+	struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
+	struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
+	u64 new_stats_value = 0;
+	u32 stats_reg_offset = stats_table->stats_offset_64[index];
+
+	new_stats_value = readq(stats->hw->hw_addr + stats_reg_offset);
 	/*roll-over case */
-	if (new_stat_value < last_rd_stats->stat_value_64[index])
-		hw_stats->stat_value_64[index] += new_stat_value;
+	if (new_stats_value < last_rd_stats->stats_value_64[index])
+		hw_stats->stats_value_64[index] += new_stats_value;
 	else
-		hw_stats->stat_value_64[index] +=
-			new_stat_value - last_rd_stats->stat_value_64[index];
-	last_rd_stats->stat_value_64[index] = new_stat_value;
-	*value = hw_stats->stat_value_64[index];
+		hw_stats->stats_value_64[index] +=
+			new_stats_value - last_rd_stats->stats_value_64[index];
+	last_rd_stats->stats_value_64[index] = new_stats_value;
+	*value = hw_stats->stats_value_64[index];
 }
 
 /**
- * i40iw_hw_stat_read_all - read all HW stat counters
- * @devstat: pestat struct
- * @stat_values: hw stats structure
+ * i40iw_hw_stats_read_all - read all HW stat counters
+ * @stats: pestat struct
+ * @stats_values: hw stats structure
  *
  * Read all the HW stat counters and populates hw_stats structure
- * of passed-in dev's pestat as well as copy created in stat_values.
+ * of passed-in vsi's pestat as well as copy created in stat_values.
  */
-static void i40iw_hw_stat_read_all(struct i40iw_dev_pestat *devstat,
-				   struct i40iw_dev_hw_stats *stat_values)
+void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats,
+			     struct i40iw_dev_hw_stats *stats_values)
 {
-	u32 stat_index;
-
-	for (stat_index = 0; stat_index < I40IW_HW_STAT_INDEX_MAX_32;
-	     stat_index++)
-		i40iw_hw_stat_read_32(devstat, stat_index,
-				      &stat_values->stat_value_32[stat_index]);
-	for (stat_index = 0; stat_index < I40IW_HW_STAT_INDEX_MAX_64;
-	     stat_index++)
-		i40iw_hw_stat_read_64(devstat, stat_index,
-				      &stat_values->stat_value_64[stat_index]);
+	u32 stats_index;
+	unsigned long flags;
+
+	spin_lock_irqsave(&stats->lock, flags);
+
+	for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
+	     stats_index++)
+		i40iw_hw_stats_read_32(stats, stats_index,
+				       &stats_values->stats_value_32[stats_index]);
+	for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
+	     stats_index++)
+		i40iw_hw_stats_read_64(stats, stats_index,
+				       &stats_values->stats_value_64[stats_index]);
+	spin_unlock_irqrestore(&stats->lock, flags);
 }
 
 /**
- * i40iw_hw_stat_refresh_all - Update all HW stat structs
- * @devstat: pestat struct
- * @stat_values: hw stats structure
+ * i40iw_hw_stats_refresh_all - Update all HW stats structs
+ * @stats: pestat struct
  *
- * Read all the HW stat counters to refresh values in hw_stats structure
+ * Read all the HW stats counters to refresh values in hw_stats structure
  * of passed-in dev's pestat
  */
-static void i40iw_hw_stat_refresh_all(struct i40iw_dev_pestat *devstat)
+void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats)
+{
+	u64 stats_value;
+	u32 stats_index;
+	unsigned long flags;
+
+	spin_lock_irqsave(&stats->lock, flags);
+
+	for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
+	     stats_index++)
+		i40iw_hw_stats_read_32(stats, stats_index, &stats_value);
+	for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
+	     stats_index++)
+		i40iw_hw_stats_read_64(stats, stats_index, &stats_value);
+	spin_unlock_irqrestore(&stats->lock, flags);
+}
+
+/**
+ * i40iw_get_fcn_id - Return the function id
+ * @dev: pointer to the device
+ */
+static u8 i40iw_get_fcn_id(struct i40iw_sc_dev *dev)
+{
+	u8 fcn_id = I40IW_INVALID_FCN_ID;
+	u8 i;
+
+	for (i = I40IW_FIRST_NON_PF_STAT; i < I40IW_MAX_STATS_COUNT; i++)
+		if (!dev->fcn_id_array[i]) {
+			fcn_id = i;
+			dev->fcn_id_array[i] = true;
+			break;
+		}
+	return fcn_id;
+}
+
+/**
+ * i40iw_vsi_stats_init - Initialize the vsi statistics
+ * @vsi: pointer to the vsi structure
+ * @info: The info structure used for initialization
+ */
+enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_stats_info *info)
 {
-	u64 stat_value;
-	u32 stat_index;
-
-	for (stat_index = 0; stat_index < I40IW_HW_STAT_INDEX_MAX_32;
-	     stat_index++)
-		i40iw_hw_stat_read_32(devstat, stat_index, &stat_value);
-	for (stat_index = 0; stat_index < I40IW_HW_STAT_INDEX_MAX_64;
-	     stat_index++)
-		i40iw_hw_stat_read_64(devstat, stat_index, &stat_value);
+	u8 fcn_id = info->fcn_id;
+
+	if (info->alloc_fcn_id)
+		fcn_id = i40iw_get_fcn_id(vsi->dev);
+
+	if (fcn_id == I40IW_INVALID_FCN_ID)
+		return I40IW_ERR_NOT_READY;
+
+	vsi->pestat = info->pestat;
+	vsi->pestat->hw = vsi->dev->hw;
+
+	if (info->stats_initialize) {
+		i40iw_hw_stats_init(vsi->pestat, fcn_id, true);
+		spin_lock_init(&vsi->pestat->lock);
+		i40iw_hw_stats_start_timer(vsi);
+	}
+	vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
+	vsi->fcn_id = fcn_id;
+	return I40IW_SUCCESS;
+}
+
+/**
+ * i40iw_vsi_stats_free - Free the vsi stats
+ * @vsi: pointer to the vsi structure
+ */
+void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi)
+{
+	u8 fcn_id = vsi->fcn_id;
+
+	if ((vsi->stats_fcn_id_alloc) && (fcn_id != I40IW_INVALID_FCN_ID))
+		vsi->dev->fcn_id_array[fcn_id] = false;
+	i40iw_hw_stats_stop_timer(vsi);
 }
 
 static struct i40iw_cqp_ops iw_cqp_ops = {
@@ -4711,24 +4943,6 @@ static struct i40iw_hmc_ops iw_hmc_ops = {
 	NULL
 };
 
-static const struct i40iw_device_pestat_ops iw_device_pestat_ops = {
-	i40iw_hw_stat_init,
-	i40iw_hw_stat_read_32,
-	i40iw_hw_stat_read_64,
-	i40iw_hw_stat_read_all,
-	i40iw_hw_stat_refresh_all
-};
-
-/**
- * i40iw_device_init_pestat - Initialize the pestat structure
- * @dev: pestat struct
- */
-enum i40iw_status_code i40iw_device_init_pestat(struct i40iw_dev_pestat *devstat)
-{
-	devstat->ops = iw_device_pestat_ops;
-	return 0;
-}
-
 /**
  * i40iw_device_init - Initialize IWARP device
  * @dev: IWARP device pointer
@@ -4750,14 +4964,7 @@ enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev,
 
 	dev->debug_mask = info->debug_mask;
 
-	ret_code = i40iw_device_init_pestat(&dev->dev_pestat);
-	if (ret_code) {
-		i40iw_debug(dev, I40IW_DEBUG_DEV,
-			    "%s: i40iw_device_init_pestat failed\n", __func__);
-		return ret_code;
-	}
 	dev->hmc_fn_id = info->hmc_fn_id;
-	dev->qs_handle = info->qs_handle;
 	dev->exception_lan_queue = info->exception_lan_queue;
 	dev->is_pf = info->is_pf;
 
@@ -4770,15 +4977,10 @@ enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev,
 	dev->hw = info->hw;
 	dev->hw->hw_addr = info->bar0;
 
-	val = i40iw_rd32(dev->hw, I40E_GLPCI_DREVID);
-	dev->hw_rev = (u8)RS_32(val, I40E_GLPCI_DREVID_DEFAULT_REVID);
-
 	if (dev->is_pf) {
-		dev->dev_pestat.ops.iw_hw_stat_init(&dev->dev_pestat,
-			dev->hmc_fn_id, dev->hw, true);
-		spin_lock_init(&dev->dev_pestat.stats_lock);
-		/*start the periodic stats_timer */
-		i40iw_hw_stats_start_timer(dev);
+		val = i40iw_rd32(dev->hw, I40E_GLPCI_DREVID);
+		dev->hw_rev = (u8)RS_32(val, I40E_GLPCI_DREVID_DEFAULT_REVID);
+
 		val = i40iw_rd32(dev->hw, I40E_GLPCI_LBARCTRL);
 		db_size = (u8)RS_32(val, I40E_GLPCI_LBARCTRL_PE_DB_SIZE);
 		if ((db_size != I40IW_PE_DB_SIZE_4M) &&
diff --git a/drivers/infiniband/hw/i40iw/i40iw_d.h b/drivers/infiniband/hw/i40iw/i40iw_d.h
index 2fac1db0e0a0..a39ac12b6a7e 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_d.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_d.h
@@ -35,6 +35,8 @@
 #ifndef I40IW_D_H
 #define I40IW_D_H
 
+#define I40IW_FIRST_USER_QP_ID  2
+
 #define I40IW_DB_ADDR_OFFSET    (4 * 1024 * 1024 - 64 * 1024)
 #define I40IW_VF_DB_ADDR_OFFSET (64 * 1024)
 
@@ -67,6 +69,9 @@
 #define I40IW_STAG_TYPE_NONSHARED 1
 
 #define I40IW_MAX_USER_PRIORITY 8
+#define I40IW_MAX_STATS_COUNT 16
+#define I40IW_FIRST_NON_PF_STAT	4
+
 
 #define LS_64_1(val, bits)      ((u64)(uintptr_t)val << bits)
 #define RS_64_1(val, bits)      ((u64)(uintptr_t)val >> bits)
@@ -74,6 +79,8 @@
 #define RS_32_1(val, bits)      (u32)(val >> bits)
 #define I40E_HI_DWORD(x)        ((u32)((((x) >> 16) >> 16) & 0xFFFFFFFF))
 
+#define QS_HANDLE_UNKNOWN       0xffff
+
 #define LS_64(val, field) (((u64)val << field ## _SHIFT) & (field ## _MASK))
 
 #define RS_64(val, field) ((u64)(val & field ## _MASK) >> field ## _SHIFT)
@@ -1199,8 +1206,11 @@
 #define I40IWQPC_RXCQNUM_SHIFT 32
 #define I40IWQPC_RXCQNUM_MASK (0x1ffffULL << I40IWQPC_RXCQNUM_SHIFT)
 
-#define I40IWQPC_Q2ADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPC_Q2ADDR_MASK I40IW_CQPHC_QPCTX_MASK
+#define I40IWQPC_STAT_INDEX_SHIFT 0
+#define I40IWQPC_STAT_INDEX_MASK (0x1fULL << I40IWQPC_STAT_INDEX_SHIFT)
+
+#define I40IWQPC_Q2ADDR_SHIFT 0
+#define I40IWQPC_Q2ADDR_MASK (0xffffffffffffff00ULL << I40IWQPC_Q2ADDR_SHIFT)
 
 #define I40IWQPC_LASTBYTESENT_SHIFT 0
 #define I40IWQPC_LASTBYTESENT_MASK (0xffUL << I40IWQPC_LASTBYTESENT_SHIFT)
@@ -1232,11 +1242,8 @@
 #define I40IWQPC_PRIVEN_SHIFT 25
 #define I40IWQPC_PRIVEN_MASK (1UL << I40IWQPC_PRIVEN_SHIFT)
 
-#define I40IWQPC_LSMMPRESENT_SHIFT 26
-#define I40IWQPC_LSMMPRESENT_MASK (1UL << I40IWQPC_LSMMPRESENT_SHIFT)
-
-#define I40IWQPC_ADJUSTFORLSMM_SHIFT 27
-#define I40IWQPC_ADJUSTFORLSMM_MASK (1UL << I40IWQPC_ADJUSTFORLSMM_SHIFT)
+#define I40IWQPC_USESTATSINSTANCE_SHIFT 26
+#define I40IWQPC_USESTATSINSTANCE_MASK (1UL << I40IWQPC_USESTATSINSTANCE_SHIFT)
 
 #define I40IWQPC_IWARPMODE_SHIFT 28
 #define I40IWQPC_IWARPMODE_MASK (1UL << I40IWQPC_IWARPMODE_SHIFT)
@@ -1713,6 +1720,8 @@ enum i40iw_alignment {
 #define OP_MANAGE_VF_PBLE_BP                    28
 #define OP_QUERY_FPM_VALUES                     29
 #define OP_COMMIT_FPM_VALUES                    30
-#define OP_SIZE_CQP_STAT_ARRAY                  31
+#define OP_REQUESTED_COMMANDS                   31
+#define OP_COMPLETED_COMMANDS                   32
+#define OP_SIZE_CQP_STAT_ARRAY                  33
 
 #endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_hw.c b/drivers/infiniband/hw/i40iw/i40iw_hw.c
index 0c92a40b3e86..476867a3f584 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_hw.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_hw.c
@@ -62,7 +62,7 @@ u32 i40iw_initialize_hw_resources(struct i40iw_device *iwdev)
 	max_mr = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt;
 	arp_table_size = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_ARP].cnt;
 	iwdev->max_cqe = 0xFFFFF;
-	num_pds = max_qp * 4;
+	num_pds = I40IW_MAX_PDS;
 	resources_size = sizeof(struct i40iw_arp_entry) * arp_table_size;
 	resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 	resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
@@ -308,7 +308,9 @@ void i40iw_process_aeq(struct i40iw_device *iwdev)
 			iwqp = iwdev->qp_table[info->qp_cq_id];
 			if (!iwqp) {
 				spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
-				i40iw_pr_err("qp_id %d is already freed\n", info->qp_cq_id);
+				i40iw_debug(dev, I40IW_DEBUG_AEQ,
+					    "%s qp_id %d is already freed\n",
+					    __func__, info->qp_cq_id);
 				continue;
 			}
 			i40iw_add_ref(&iwqp->ibqp);
@@ -359,6 +361,9 @@ void i40iw_process_aeq(struct i40iw_device *iwdev)
 				continue;
 			i40iw_cm_disconn(iwqp);
 			break;
+		case I40IW_AE_QP_SUSPEND_COMPLETE:
+			i40iw_qp_suspend_resume(dev, &iwqp->sc_qp, false);
+			break;
 		case I40IW_AE_TERMINATE_SENT:
 			i40iw_terminate_send_fin(qp);
 			break;
@@ -404,19 +409,18 @@ void i40iw_process_aeq(struct i40iw_device *iwdev)
 		case I40IW_AE_LCE_CQ_CATASTROPHIC:
 		case I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG:
 		case I40IW_AE_UDA_XMIT_IPADDR_MISMATCH:
-		case I40IW_AE_QP_SUSPEND_COMPLETE:
 			ctx_info->err_rq_idx_valid = false;
 		default:
-				if (!info->sq && ctx_info->err_rq_idx_valid) {
-					ctx_info->err_rq_idx = info->wqe_idx;
-					ctx_info->tcp_info_valid = false;
-					ctx_info->iwarp_info_valid = false;
-					ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
-									     iwqp->host_ctx.va,
-									     ctx_info);
-				}
-				i40iw_terminate_connection(qp, info);
-				break;
+			if (!info->sq && ctx_info->err_rq_idx_valid) {
+				ctx_info->err_rq_idx = info->wqe_idx;
+				ctx_info->tcp_info_valid = false;
+				ctx_info->iwarp_info_valid = false;
+				ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
+								     iwqp->host_ctx.va,
+								     ctx_info);
+			}
+			i40iw_terminate_connection(qp, info);
+			break;
 		}
 		if (info->qp)
 			i40iw_rem_ref(&iwqp->ibqp);
@@ -538,6 +542,7 @@ enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev,
 {
 	struct i40iw_qhash_table_info *info;
 	struct i40iw_sc_dev *dev = &iwdev->sc_dev;
+	struct i40iw_sc_vsi *vsi = &iwdev->vsi;
 	enum i40iw_status_code status;
 	struct i40iw_cqp *iwcqp = &iwdev->cqp;
 	struct i40iw_cqp_request *cqp_request;
@@ -550,6 +555,7 @@ enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev,
 	info = &cqp_info->in.u.manage_qhash_table_entry.info;
 	memset(info, 0, sizeof(*info));
 
+	info->vsi = &iwdev->vsi;
 	info->manage = mtype;
 	info->entry_type = etype;
 	if (cminfo->vlan_id != 0xFFFF) {
@@ -560,8 +566,9 @@ enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev,
 	}
 
 	info->ipv4_valid = cminfo->ipv4;
+	info->user_pri = cminfo->user_pri;
 	ether_addr_copy(info->mac_addr, iwdev->netdev->dev_addr);
-	info->qp_num = cpu_to_le32(dev->ilq->qp_id);
+	info->qp_num = cpu_to_le32(vsi->ilq->qp_id);
 	info->dest_port = cpu_to_le16(cminfo->loc_port);
 	info->dest_ip[0] = cpu_to_le32(cminfo->loc_addr[0]);
 	info->dest_ip[1] = cpu_to_le32(cminfo->loc_addr[1]);
@@ -617,6 +624,7 @@ enum i40iw_status_code i40iw_hw_flush_wqes(struct i40iw_device *iwdev,
 	struct i40iw_qp_flush_info *hw_info;
 	struct i40iw_cqp_request *cqp_request;
 	struct cqp_commands_info *cqp_info;
+	struct i40iw_qp *iwqp = (struct i40iw_qp *)qp->back_qp;
 
 	cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait);
 	if (!cqp_request)
@@ -631,9 +639,30 @@ enum i40iw_status_code i40iw_hw_flush_wqes(struct i40iw_device *iwdev,
 	cqp_info->in.u.qp_flush_wqes.qp = qp;
 	cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request;
 	status = i40iw_handle_cqp_op(iwdev, cqp_request);
-	if (status)
+	if (status) {
 		i40iw_pr_err("CQP-OP Flush WQE's fail");
-	return status;
+		complete(&iwqp->sq_drained);
+		complete(&iwqp->rq_drained);
+		return status;
+	}
+	if (!cqp_request->compl_info.maj_err_code) {
+		switch (cqp_request->compl_info.min_err_code) {
+		case I40IW_CQP_COMPL_RQ_WQE_FLUSHED:
+			complete(&iwqp->sq_drained);
+			break;
+		case I40IW_CQP_COMPL_SQ_WQE_FLUSHED:
+			complete(&iwqp->rq_drained);
+			break;
+		case I40IW_CQP_COMPL_RQ_SQ_WQE_FLUSHED:
+			break;
+		default:
+			complete(&iwqp->sq_drained);
+			complete(&iwqp->rq_drained);
+			break;
+		}
+	}
+
+	return 0;
 }
 
 /**
diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c
index ac2f3cd9478c..2728af3103ce 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_main.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_main.c
@@ -237,14 +237,11 @@ static irqreturn_t i40iw_irq_handler(int irq, void *data)
  */
 static void i40iw_destroy_cqp(struct i40iw_device *iwdev, bool free_hwcqp)
 {
-	enum i40iw_status_code status = 0;
 	struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 	struct i40iw_cqp *cqp = &iwdev->cqp;
 
-	if (free_hwcqp && dev->cqp_ops->cqp_destroy)
-		status = dev->cqp_ops->cqp_destroy(dev->cqp);
-	if (status)
-		i40iw_pr_err("destroy cqp failed");
+	if (free_hwcqp)
+		dev->cqp_ops->cqp_destroy(dev->cqp);
 
 	i40iw_free_dma_mem(dev->hw, &cqp->sq);
 	kfree(cqp->scratch_array);
@@ -270,6 +267,7 @@ static void i40iw_disable_irq(struct i40iw_sc_dev *dev,
 		i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_vec->idx - 1), 0);
 	else
 		i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_vec->idx - 1), 0);
+	irq_set_affinity_hint(msix_vec->irq, NULL);
 	free_irq(msix_vec->irq, dev_id);
 }
 
@@ -603,7 +601,7 @@ static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev)
 		i40iw_pr_err("cqp init status %d\n", status);
 		goto exit;
 	}
-	status = dev->cqp_ops->cqp_create(dev->cqp, true, &maj_err, &min_err);
+	status = dev->cqp_ops->cqp_create(dev->cqp, &maj_err, &min_err);
 	if (status) {
 		i40iw_pr_err("cqp create status %d maj_err %d min_err %d\n",
 			     status, maj_err, min_err);
@@ -688,6 +686,7 @@ static enum i40iw_status_code i40iw_configure_ceq_vector(struct i40iw_device *iw
 							 struct i40iw_msix_vector *msix_vec)
 {
 	enum i40iw_status_code status;
+	cpumask_t mask;
 
 	if (iwdev->msix_shared && !ceq_id) {
 		tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);
@@ -697,12 +696,15 @@ static enum i40iw_status_code i40iw_configure_ceq_vector(struct i40iw_device *iw
 		status = request_irq(msix_vec->irq, i40iw_ceq_handler, 0, "CEQ", iwceq);
 	}
 
+	cpumask_clear(&mask);
+	cpumask_set_cpu(msix_vec->cpu_affinity, &mask);
+	irq_set_affinity_hint(msix_vec->irq, &mask);
+
 	if (status) {
 		i40iw_pr_err("ceq irq config fail\n");
 		return I40IW_ERR_CONFIG;
 	}
 	msix_vec->ceq_id = ceq_id;
-	msix_vec->cpu_affinity = 0;
 
 	return 0;
 }
@@ -930,6 +932,7 @@ static enum i40iw_status_code i40iw_initialize_ilq(struct i40iw_device *iwdev)
 	struct i40iw_puda_rsrc_info info;
 	enum i40iw_status_code status;
 
+	memset(&info, 0, sizeof(info));
 	info.type = I40IW_PUDA_RSRC_TYPE_ILQ;
 	info.cq_id = 1;
 	info.qp_id = 0;
@@ -939,10 +942,9 @@ static enum i40iw_status_code i40iw_initialize_ilq(struct i40iw_device *iwdev)
 	info.rq_size = 8192;
 	info.buf_size = 1024;
 	info.tx_buf_cnt = 16384;
-	info.mss = iwdev->mss;
 	info.receive = i40iw_receive_ilq;
 	info.xmit_complete = i40iw_free_sqbuf;
-	status = i40iw_puda_create_rsrc(&iwdev->sc_dev, &info);
+	status = i40iw_puda_create_rsrc(&iwdev->vsi, &info);
 	if (status)
 		i40iw_pr_err("ilq create fail\n");
 	return status;
@@ -959,6 +961,7 @@ static enum i40iw_status_code i40iw_initialize_ieq(struct i40iw_device *iwdev)
 	struct i40iw_puda_rsrc_info info;
 	enum i40iw_status_code status;
 
+	memset(&info, 0, sizeof(info));
 	info.type = I40IW_PUDA_RSRC_TYPE_IEQ;
 	info.cq_id = 2;
 	info.qp_id = iwdev->sc_dev.exception_lan_queue;
@@ -967,9 +970,8 @@ static enum i40iw_status_code i40iw_initialize_ieq(struct i40iw_device *iwdev)
 	info.sq_size = 8192;
 	info.rq_size = 8192;
 	info.buf_size = 2048;
-	info.mss = iwdev->mss;
 	info.tx_buf_cnt = 16384;
-	status = i40iw_puda_create_rsrc(&iwdev->sc_dev, &info);
+	status = i40iw_puda_create_rsrc(&iwdev->vsi, &info);
 	if (status)
 		i40iw_pr_err("ieq create fail\n");
 	return status;
@@ -1159,7 +1161,7 @@ static void i40iw_add_ipv6_addr(struct i40iw_device *iwdev)
 {
 	struct net_device *ip_dev;
 	struct inet6_dev *idev;
-	struct inet6_ifaddr *ifp;
+	struct inet6_ifaddr *ifp, *tmp;
 	u32 local_ipaddr6[4];
 
 	rcu_read_lock();
@@ -1172,7 +1174,7 @@ static void i40iw_add_ipv6_addr(struct i40iw_device *iwdev)
 				i40iw_pr_err("ipv6 inet device not found\n");
 				break;
 			}
-			list_for_each_entry(ifp, &idev->addr_list, if_list) {
+			list_for_each_entry_safe(ifp, tmp, &idev->addr_list, if_list) {
 				i40iw_pr_info("IP=%pI6, vlan_id=%d, MAC=%pM\n", &ifp->addr,
 					      rdma_vlan_dev_vlan_id(ip_dev), ip_dev->dev_addr);
 				i40iw_copy_ip_ntohl(local_ipaddr6,
@@ -1294,17 +1296,23 @@ static enum i40iw_status_code i40iw_initialize_dev(struct i40iw_device *iwdev,
 	enum i40iw_status_code status;
 	struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 	struct i40iw_device_init_info info;
+	struct i40iw_vsi_init_info vsi_info;
 	struct i40iw_dma_mem mem;
+	struct i40iw_l2params l2params;
 	u32 size;
+	struct i40iw_vsi_stats_info stats_info;
+	u16 last_qset = I40IW_NO_QSET;
+	u16 qset;
+	u32 i;
 
+	memset(&l2params, 0, sizeof(l2params));
 	memset(&info, 0, sizeof(info));
 	size = sizeof(struct i40iw_hmc_pble_rsrc) + sizeof(struct i40iw_hmc_info) +
 				(sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX);
 	iwdev->hmc_info_mem = kzalloc(size, GFP_KERNEL);
-	if (!iwdev->hmc_info_mem) {
-		i40iw_pr_err("memory alloc fail\n");
+	if (!iwdev->hmc_info_mem)
 		return I40IW_ERR_NO_MEMORY;
-	}
+
 	iwdev->pble_rsrc = (struct i40iw_hmc_pble_rsrc *)iwdev->hmc_info_mem;
 	dev->hmc_info = &iwdev->hw.hmc;
 	dev->hmc_info->hmc_obj = (struct i40iw_hmc_obj_info *)(iwdev->pble_rsrc + 1);
@@ -1325,7 +1333,17 @@ static enum i40iw_status_code i40iw_initialize_dev(struct i40iw_device *iwdev,
 	info.bar0 = ldev->hw_addr;
 	info.hw = &iwdev->hw;
 	info.debug_mask = debug;
-	info.qs_handle = ldev->params.qos.prio_qos[0].qs_handle;
+	l2params.mss =
+		(ldev->params.mtu) ? ldev->params.mtu - I40IW_MTU_TO_MSS : I40IW_DEFAULT_MSS;
+	for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++) {
+		qset = ldev->params.qos.prio_qos[i].qs_handle;
+		l2params.qs_handle_list[i] = qset;
+		if (last_qset == I40IW_NO_QSET)
+			last_qset = qset;
+		else if ((qset != last_qset) && (qset != I40IW_NO_QSET))
+			iwdev->dcb = true;
+	}
+	i40iw_pr_info("DCB is set/clear = %d\n", iwdev->dcb);
 	info.exception_lan_queue = 1;
 	info.vchnl_send = i40iw_virtchnl_send;
 	status = i40iw_device_init(&iwdev->sc_dev, &info);
@@ -1334,6 +1352,20 @@ exit:
 		kfree(iwdev->hmc_info_mem);
 		iwdev->hmc_info_mem = NULL;
 	}
+	memset(&vsi_info, 0, sizeof(vsi_info));
+	vsi_info.dev = &iwdev->sc_dev;
+	vsi_info.back_vsi = (void *)iwdev;
+	vsi_info.params = &l2params;
+	i40iw_sc_vsi_init(&iwdev->vsi, &vsi_info);
+
+	if (dev->is_pf) {
+		memset(&stats_info, 0, sizeof(stats_info));
+		stats_info.fcn_id = ldev->fid;
+		stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL);
+		stats_info.stats_initialize = true;
+		if (stats_info.pestat)
+			i40iw_vsi_stats_init(&iwdev->vsi, &stats_info);
+	}
 	return status;
 }
 
@@ -1384,6 +1416,7 @@ static enum i40iw_status_code i40iw_save_msix_info(struct i40iw_device *iwdev,
 	for (i = 0, ceq_idx = 0; i < iwdev->msix_count; i++, iw_qvinfo++) {
 		iwdev->iw_msixtbl[i].idx = ldev->msix_entries[i].entry;
 		iwdev->iw_msixtbl[i].irq = ldev->msix_entries[i].vector;
+		iwdev->iw_msixtbl[i].cpu_affinity = ceq_idx;
 		if (i == 0) {
 			iw_qvinfo->aeq_idx = 0;
 			if (iwdev->msix_shared)
@@ -1404,18 +1437,19 @@ static enum i40iw_status_code i40iw_save_msix_info(struct i40iw_device *iwdev,
  * i40iw_deinit_device - clean up the device resources
  * @iwdev: iwarp device
  * @reset: true if called before reset
- * @del_hdl: true if delete hdl entry
  *
  * Destroy the ib device interface, remove the mac ip entry and ipv4/ipv6 addresses,
  * destroy the device queues and free the pble and the hmc objects
  */
-static void i40iw_deinit_device(struct i40iw_device *iwdev, bool reset, bool del_hdl)
+static void i40iw_deinit_device(struct i40iw_device *iwdev, bool reset)
 {
 	struct i40e_info *ldev = iwdev->ldev;
 
 	struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 
 	i40iw_pr_info("state = %d\n", iwdev->init_state);
+	if (iwdev->param_wq)
+		destroy_workqueue(iwdev->param_wq);
 
 	switch (iwdev->init_state) {
 	case RDMA_DEV_REGISTERED:
@@ -1441,10 +1475,10 @@ static void i40iw_deinit_device(struct i40iw_device *iwdev, bool reset, bool del
 		i40iw_destroy_aeq(iwdev, reset);
 		/* fallthrough */
 	case IEQ_CREATED:
-		i40iw_puda_dele_resources(dev, I40IW_PUDA_RSRC_TYPE_IEQ, reset);
+		i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_IEQ, reset);
 		/* fallthrough */
 	case ILQ_CREATED:
-		i40iw_puda_dele_resources(dev, I40IW_PUDA_RSRC_TYPE_ILQ, reset);
+		i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_ILQ, reset);
 		/* fallthrough */
 	case CCQ_CREATED:
 		i40iw_destroy_ccq(iwdev, reset);
@@ -1456,13 +1490,14 @@ static void i40iw_deinit_device(struct i40iw_device *iwdev, bool reset, bool del
 		i40iw_del_hmc_objects(dev, dev->hmc_info, true, reset);
 		/* fallthrough */
 	case CQP_CREATED:
-		i40iw_destroy_cqp(iwdev, !reset);
+		i40iw_destroy_cqp(iwdev, true);
 		/* fallthrough */
 	case INITIAL_STATE:
 		i40iw_cleanup_cm_core(&iwdev->cm_core);
-		if (dev->is_pf)
-			i40iw_hw_stats_del_timer(dev);
-
+		if (iwdev->vsi.pestat) {
+			i40iw_vsi_stats_free(&iwdev->vsi);
+			kfree(iwdev->vsi.pestat);
+		}
 		i40iw_del_init_mem(iwdev);
 		break;
 	case INVALID_STATE:
@@ -1472,8 +1507,7 @@ static void i40iw_deinit_device(struct i40iw_device *iwdev, bool reset, bool del
 		break;
 	}
 
-	if (del_hdl)
-		i40iw_del_handler(i40iw_find_i40e_handler(ldev));
+	i40iw_del_handler(i40iw_find_i40e_handler(ldev));
 	kfree(iwdev->hdl);
 }
 
@@ -1508,7 +1542,6 @@ static enum i40iw_status_code i40iw_setup_init_state(struct i40iw_handler *hdl,
 	iwdev->max_enabled_vfs = iwdev->max_rdma_vfs;
 	iwdev->netdev = ldev->netdev;
 	hdl->client = client;
-	iwdev->mss = (!ldev->params.mtu) ? I40IW_DEFAULT_MSS : ldev->params.mtu - I40IW_MTU_TO_MSS;
 	if (!ldev->ftype)
 		iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_DB_ADDR_OFFSET;
 	else
@@ -1528,6 +1561,7 @@ static enum i40iw_status_code i40iw_setup_init_state(struct i40iw_handler *hdl,
 
 	init_waitqueue_head(&iwdev->vchnl_waitq);
 	init_waitqueue_head(&dev->vf_reqs);
+	init_waitqueue_head(&iwdev->close_wq);
 
 	status = i40iw_initialize_dev(iwdev, ldev);
 exit:
@@ -1540,6 +1574,20 @@ exit:
 }
 
 /**
+ * i40iw_get_used_rsrc - determine resources used internally
+ * @iwdev: iwarp device
+ *
+ * Called after internal allocations
+ */
+static void i40iw_get_used_rsrc(struct i40iw_device *iwdev)
+{
+	iwdev->used_pds = find_next_zero_bit(iwdev->allocated_pds, iwdev->max_pd, 0);
+	iwdev->used_qps = find_next_zero_bit(iwdev->allocated_qps, iwdev->max_qp, 0);
+	iwdev->used_cqs = find_next_zero_bit(iwdev->allocated_cqs, iwdev->max_cq, 0);
+	iwdev->used_mrs = find_next_zero_bit(iwdev->allocated_mrs, iwdev->max_mr, 0);
+}
+
+/**
  * i40iw_open - client interface operation open for iwarp/uda device
  * @ldev: lan device information
  * @client: iwarp client information, provided during registration
@@ -1611,6 +1659,7 @@ static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)
 		status = i40iw_initialize_hw_resources(iwdev);
 		if (status)
 			break;
+		i40iw_get_used_rsrc(iwdev);
 		dev->ccq_ops->ccq_arm(dev->ccq);
 		status = i40iw_hmc_init_pble(&iwdev->sc_dev, iwdev->pble_rsrc);
 		if (status)
@@ -1630,35 +1679,73 @@ static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)
 		iwdev->init_state = RDMA_DEV_REGISTERED;
 		iwdev->iw_status = 1;
 		i40iw_port_ibevent(iwdev);
+		iwdev->param_wq = alloc_ordered_workqueue("l2params", WQ_MEM_RECLAIM);
+		if(iwdev->param_wq == NULL)
+			break;
 		i40iw_pr_info("i40iw_open completed\n");
 		return 0;
 	} while (0);
 
 	i40iw_pr_err("status = %d last completion = %d\n", status, iwdev->init_state);
-	i40iw_deinit_device(iwdev, false, false);
+	i40iw_deinit_device(iwdev, false);
 	return -ERESTART;
 }
 
 /**
- * i40iw_l2param_change : handle qs handles for qos and mss change
+ * i40iw_l2params_worker - worker for l2 params change
+ * @work: work pointer for l2 params
+ */
+static void i40iw_l2params_worker(struct work_struct *work)
+{
+	struct l2params_work *dwork =
+	    container_of(work, struct l2params_work, work);
+	struct i40iw_device *iwdev = dwork->iwdev;
+
+	i40iw_change_l2params(&iwdev->vsi, &dwork->l2params);
+	atomic_dec(&iwdev->params_busy);
+	kfree(work);
+}
+
+/**
+ * i40iw_l2param_change - handle qs handles for qos and mss change
  * @ldev: lan device information
  * @client: client for paramater change
  * @params: new parameters from L2
  */
-static void i40iw_l2param_change(struct i40e_info *ldev,
-				 struct i40e_client *client,
+static void i40iw_l2param_change(struct i40e_info *ldev, struct i40e_client *client,
 				 struct i40e_params *params)
 {
 	struct i40iw_handler *hdl;
+	struct i40iw_l2params *l2params;
+	struct l2params_work *work;
 	struct i40iw_device *iwdev;
+	int i;
 
 	hdl = i40iw_find_i40e_handler(ldev);
 	if (!hdl)
 		return;
 
 	iwdev = &hdl->device;
-	if (params->mtu)
-		iwdev->mss = params->mtu - I40IW_MTU_TO_MSS;
+
+	if (atomic_read(&iwdev->params_busy))
+		return;
+
+
+	work = kzalloc(sizeof(*work), GFP_ATOMIC);
+	if (!work)
+		return;
+
+	atomic_inc(&iwdev->params_busy);
+
+	work->iwdev = iwdev;
+	l2params = &work->l2params;
+	for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++)
+		l2params->qs_handle_list[i] = params->qos.prio_qos[i].qs_handle;
+
+	l2params->mss = (params->mtu) ? params->mtu - I40IW_MTU_TO_MSS : iwdev->vsi.mss;
+
+	INIT_WORK(&work->work, i40iw_l2params_worker);
+	queue_work(iwdev->param_wq, &work->work);
 }
 
 /**
@@ -1679,8 +1766,11 @@ static void i40iw_close(struct i40e_info *ldev, struct i40e_client *client, bool
 		return;
 
 	iwdev = &hdl->device;
+	iwdev->closing = true;
+
+	i40iw_cm_disconnect_all(iwdev);
 	destroy_workqueue(iwdev->virtchnl_wq);
-	i40iw_deinit_device(iwdev, reset, true);
+	i40iw_deinit_device(iwdev, reset);
 }
 
 /**
@@ -1701,21 +1791,23 @@ static void i40iw_vf_reset(struct i40e_info *ldev, struct i40e_client *client, u
 	struct i40iw_vfdev *tmp_vfdev;
 	unsigned int i;
 	unsigned long flags;
+	struct i40iw_device *iwdev;
 
 	hdl = i40iw_find_i40e_handler(ldev);
 	if (!hdl)
 		return;
 
 	dev = &hdl->device.sc_dev;
+	iwdev = (struct i40iw_device *)dev->back_dev;
 
 	for (i = 0; i < I40IW_MAX_PE_ENABLED_VF_COUNT; i++) {
 		if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id != vf_id))
 			continue;
 		/* free all resources allocated on behalf of vf */
 		tmp_vfdev = dev->vf_dev[i];
-		spin_lock_irqsave(&dev->dev_pestat.stats_lock, flags);
+		spin_lock_irqsave(&iwdev->vsi.pestat->lock, flags);
 		dev->vf_dev[i] = NULL;
-		spin_unlock_irqrestore(&dev->dev_pestat.stats_lock, flags);
+		spin_unlock_irqrestore(&iwdev->vsi.pestat->lock, flags);
 		i40iw_del_hmc_objects(dev, &tmp_vfdev->hmc_info, false, false);
 		/* remove vf hmc function */
 		memset(&hmc_fcn_info, 0, sizeof(hmc_fcn_info));
diff --git a/drivers/infiniband/hw/i40iw/i40iw_osdep.h b/drivers/infiniband/hw/i40iw/i40iw_osdep.h
index 80f422bf3967..aa66c1c63dfa 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_osdep.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_osdep.h
@@ -198,6 +198,8 @@ enum i40iw_status_code i40iw_cqp_manage_vf_pble_bp(struct i40iw_sc_dev *dev,
 void i40iw_cqp_spawn_worker(struct i40iw_sc_dev *dev,
 			    struct i40iw_virtchnl_work_info *work_info, u32 iw_vf_idx);
 void *i40iw_remove_head(struct list_head *list);
+void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp, bool suspend);
+void i40iw_qp_mss_modify(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);
 
 void i40iw_term_modify_qp(struct i40iw_sc_qp *qp, u8 next_state, u8 term, u8 term_len);
 void i40iw_terminate_done(struct i40iw_sc_qp *qp, int timeout_occurred);
@@ -207,9 +209,9 @@ void i40iw_terminate_del_timer(struct i40iw_sc_qp *qp);
 enum i40iw_status_code i40iw_hw_manage_vf_pble_bp(struct i40iw_device *iwdev,
 						  struct i40iw_manage_vf_pble_info *info,
 						  bool wait);
-struct i40iw_dev_pestat;
-void i40iw_hw_stats_start_timer(struct i40iw_sc_dev *);
-void i40iw_hw_stats_del_timer(struct i40iw_sc_dev *);
+struct i40iw_sc_vsi;
+void i40iw_hw_stats_start_timer(struct i40iw_sc_vsi *vsi);
+void i40iw_hw_stats_stop_timer(struct i40iw_sc_vsi *vsi);
 #define i40iw_mmiowb() mmiowb()
 void i40iw_wr32(struct i40iw_hw *hw, u32 reg, u32 value);
 u32  i40iw_rd32(struct i40iw_hw *hw, u32 reg);
diff --git a/drivers/infiniband/hw/i40iw/i40iw_p.h b/drivers/infiniband/hw/i40iw/i40iw_p.h
index a0b8ca10d67e..28a92fee0822 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_p.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_p.h
@@ -47,8 +47,6 @@ void i40iw_debug_buf(struct i40iw_sc_dev *dev, enum i40iw_debug_flag mask,
 enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev,
 					 struct i40iw_device_init_info *info);
 
-enum i40iw_status_code i40iw_device_init_pestat(struct i40iw_dev_pestat *);
-
 void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp);
 
 u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch);
@@ -64,7 +62,24 @@ enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev,
 enum i40iw_status_code i40iw_pf_init_vfhmc(struct i40iw_sc_dev *dev, u8 vf_hmc_fn_id,
 					   u32 *vf_cnt_array);
 
-/* cqp misc functions */
+/* stats functions */
+void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats);
+void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats, struct i40iw_dev_hw_stats *stats_values);
+void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats,
+			    enum i40iw_hw_stats_index_32b index,
+			    u64 *value);
+void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats,
+			    enum i40iw_hw_stats_index_64b index,
+			    u64 *value);
+void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 index, bool is_pf);
+
+/* vsi misc functions */
+enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_stats_info *info);
+void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi);
+void i40iw_sc_vsi_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_init_info *info);
+
+void i40iw_change_l2params(struct i40iw_sc_vsi *vsi, struct i40iw_l2params *l2params);
+void i40iw_qp_add_qos(struct i40iw_sc_qp *qp);
 
 void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp);
 
diff --git a/drivers/infiniband/hw/i40iw/i40iw_pble.c b/drivers/infiniband/hw/i40iw/i40iw_pble.c
index 85993dc44f6e..c87ba1617087 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_pble.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_pble.c
@@ -353,10 +353,6 @@ static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev,
 	pages = (idx->rel_pd_idx) ? (I40IW_HMC_PD_CNT_IN_SD -
 			idx->rel_pd_idx) : I40IW_HMC_PD_CNT_IN_SD;
 	pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT);
-	if (!pages) {
-		ret_code = I40IW_ERR_NO_PBLCHUNKS_AVAILABLE;
-		goto error;
-	}
 	info.chunk = chunk;
 	info.hmc_info = hmc_info;
 	info.pages = pages;
diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.c b/drivers/infiniband/hw/i40iw/i40iw_puda.c
index c62d354f7810..449ba8c81ce7 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_puda.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_puda.c
@@ -42,12 +42,13 @@
 #include "i40iw_p.h"
 #include "i40iw_puda.h"
 
-static void i40iw_ieq_receive(struct i40iw_sc_dev *dev,
+static void i40iw_ieq_receive(struct i40iw_sc_vsi *vsi,
 			      struct i40iw_puda_buf *buf);
-static void i40iw_ieq_tx_compl(struct i40iw_sc_dev *dev, void *sqwrid);
+static void i40iw_ieq_tx_compl(struct i40iw_sc_vsi *vsi, void *sqwrid);
 static void i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx);
 static enum i40iw_status_code i40iw_puda_replenish_rq(struct i40iw_puda_rsrc
 						      *rsrc, bool initial);
+static void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp);
 /**
  * i40iw_puda_get_listbuf - get buffer from puda list
  * @list: list to use for buffers (ILQ or IEQ)
@@ -292,7 +293,7 @@ enum i40iw_status_code i40iw_puda_poll_completion(struct i40iw_sc_dev *dev,
 	unsigned long	flags;
 
 	if ((cq_type == I40IW_CQ_TYPE_ILQ) || (cq_type == I40IW_CQ_TYPE_IEQ)) {
-		rsrc = (cq_type == I40IW_CQ_TYPE_ILQ) ? dev->ilq : dev->ieq;
+		rsrc = (cq_type == I40IW_CQ_TYPE_ILQ) ? cq->vsi->ilq : cq->vsi->ieq;
 	} else {
 		i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s qp_type error\n", __func__);
 		return I40IW_ERR_BAD_PTR;
@@ -335,7 +336,7 @@ enum i40iw_status_code i40iw_puda_poll_completion(struct i40iw_sc_dev *dev,
 		rsrc->stats_pkt_rcvd++;
 		rsrc->compl_rxwqe_idx = info.wqe_idx;
 		i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s RQ completion\n", __func__);
-		rsrc->receive(rsrc->dev, buf);
+		rsrc->receive(rsrc->vsi, buf);
 		if (cq_type == I40IW_CQ_TYPE_ILQ)
 			i40iw_ilq_putback_rcvbuf(&rsrc->qp, info.wqe_idx);
 		else
@@ -345,12 +346,12 @@ enum i40iw_status_code i40iw_puda_poll_completion(struct i40iw_sc_dev *dev,
 		i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s SQ completion\n", __func__);
 		sqwrid = (void *)(uintptr_t)qp->sq_wrtrk_array[info.wqe_idx].wrid;
 		I40IW_RING_SET_TAIL(qp->sq_ring, info.wqe_idx);
-		rsrc->xmit_complete(rsrc->dev, sqwrid);
+		rsrc->xmit_complete(rsrc->vsi, sqwrid);
 		spin_lock_irqsave(&rsrc->bufpool_lock, flags);
 		rsrc->tx_wqe_avail_cnt++;
 		spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
-		if (!list_empty(&dev->ilq->txpend))
-			i40iw_puda_send_buf(dev->ilq, NULL);
+		if (!list_empty(&rsrc->vsi->ilq->txpend))
+			i40iw_puda_send_buf(rsrc->vsi->ilq, NULL);
 	}
 
 done:
@@ -513,10 +514,8 @@ static void i40iw_puda_qp_setctx(struct i40iw_puda_rsrc *rsrc)
  * i40iw_puda_qp_wqe - setup wqe for qp create
  * @rsrc: resource for qp
  */
-static enum i40iw_status_code i40iw_puda_qp_wqe(struct i40iw_puda_rsrc *rsrc)
+static enum i40iw_status_code i40iw_puda_qp_wqe(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp)
 {
-	struct i40iw_sc_qp *qp = &rsrc->qp;
-	struct i40iw_sc_dev *dev = rsrc->dev;
 	struct i40iw_sc_cqp *cqp;
 	u64 *wqe;
 	u64 header;
@@ -582,6 +581,7 @@ static enum i40iw_status_code i40iw_puda_qp_create(struct i40iw_puda_rsrc *rsrc)
 	qp->back_qp = (void *)rsrc;
 	qp->sq_pa = mem->pa;
 	qp->rq_pa = qp->sq_pa + sq_size;
+	qp->vsi = rsrc->vsi;
 	ukqp->sq_base = mem->va;
 	ukqp->rq_base = &ukqp->sq_base[rsrc->sq_size];
 	ukqp->shadow_area = ukqp->rq_base[rsrc->rq_size].elem;
@@ -608,15 +608,63 @@ static enum i40iw_status_code i40iw_puda_qp_create(struct i40iw_puda_rsrc *rsrc)
 		ukqp->wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) +
 						    I40E_VFPE_WQEALLOC1);
 
-	qp->qs_handle = qp->dev->qs_handle;
+	qp->user_pri = 0;
+	i40iw_qp_add_qos(qp);
 	i40iw_puda_qp_setctx(rsrc);
-	ret = i40iw_puda_qp_wqe(rsrc);
+	if (rsrc->ceq_valid)
+		ret = i40iw_cqp_qp_create_cmd(rsrc->dev, qp);
+	else
+		ret = i40iw_puda_qp_wqe(rsrc->dev, qp);
 	if (ret)
 		i40iw_free_dma_mem(rsrc->dev->hw, &rsrc->qpmem);
 	return ret;
 }
 
 /**
+ * i40iw_puda_cq_wqe - setup wqe for cq create
+ * @rsrc: resource for cq
+ */
+static enum i40iw_status_code i40iw_puda_cq_wqe(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq)
+{
+	u64 *wqe;
+	struct i40iw_sc_cqp *cqp;
+	u64 header;
+	struct i40iw_ccq_cqe_info compl_info;
+	enum i40iw_status_code status = 0;
+
+	cqp = dev->cqp;
+	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, 0);
+	if (!wqe)
+		return I40IW_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
+	set_64bit_val(wqe, 8, RS_64_1(cq, 1));
+	set_64bit_val(wqe, 16,
+		      LS_64(cq->shadow_read_threshold,
+			    I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
+	set_64bit_val(wqe, 32, cq->cq_pa);
+
+	set_64bit_val(wqe, 40, cq->shadow_area_pa);
+
+	header = cq->cq_uk.cq_id |
+	    LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
+	    LS_64(1, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
+	    LS_64(1, I40IW_CQPSQ_CQ_ENCEQEMASK) |
+	    LS_64(1, I40IW_CQPSQ_CQ_CEQIDVALID) |
+	    LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
+	set_64bit_val(wqe, 24, header);
+
+	i40iw_debug_buf(dev, I40IW_DEBUG_PUDA, "PUDA CQE",
+			wqe, I40IW_CQP_WQE_SIZE * 8);
+
+	i40iw_sc_cqp_post_sq(dev->cqp);
+	status = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
+						 I40IW_CQP_OP_CREATE_CQ,
+						 &compl_info);
+	return status;
+}
+
+/**
  * i40iw_puda_cq_create - create cq for resource
  * @rsrc: resource for which cq to create
  */
@@ -624,18 +672,13 @@ static enum i40iw_status_code i40iw_puda_cq_create(struct i40iw_puda_rsrc *rsrc)
 {
 	struct i40iw_sc_dev *dev = rsrc->dev;
 	struct i40iw_sc_cq *cq = &rsrc->cq;
-	u64 *wqe;
-	struct i40iw_sc_cqp *cqp;
-	u64 header;
 	enum i40iw_status_code ret = 0;
 	u32 tsize, cqsize;
-	u32 shadow_read_threshold = 128;
 	struct i40iw_dma_mem *mem;
-	struct i40iw_ccq_cqe_info compl_info;
 	struct i40iw_cq_init_info info;
 	struct i40iw_cq_uk_init_info *init_info = &info.cq_uk_init_info;
 
-	cq->back_cq = (void *)rsrc;
+	cq->vsi = rsrc->vsi;
 	cqsize = rsrc->cq_size * (sizeof(struct i40iw_cqe));
 	tsize = cqsize + sizeof(struct i40iw_cq_shadow_area);
 	ret = i40iw_allocate_dma_mem(dev->hw, &rsrc->cqmem, tsize,
@@ -656,43 +699,84 @@ static enum i40iw_status_code i40iw_puda_cq_create(struct i40iw_puda_rsrc *rsrc)
 	init_info->shadow_area = (u64 *)((u8 *)mem->va + cqsize);
 	init_info->cq_size = rsrc->cq_size;
 	init_info->cq_id = rsrc->cq_id;
+	info.ceqe_mask = true;
+	info.ceq_id_valid = true;
 	ret = dev->iw_priv_cq_ops->cq_init(cq, &info);
 	if (ret)
 		goto error;
-	cqp = dev->cqp;
-	wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, 0);
-	if (!wqe) {
-		ret = I40IW_ERR_RING_FULL;
-		goto error;
-	}
+	if (rsrc->ceq_valid)
+		ret = i40iw_cqp_cq_create_cmd(dev, cq);
+	else
+		ret = i40iw_puda_cq_wqe(dev, cq);
+error:
+	if (ret)
+		i40iw_free_dma_mem(dev->hw, &rsrc->cqmem);
+	return ret;
+}
 
-	set_64bit_val(wqe, 0, rsrc->cq_size);
-	set_64bit_val(wqe, 8, RS_64_1(cq, 1));
-	set_64bit_val(wqe, 16, LS_64(shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
-	set_64bit_val(wqe, 32, cq->cq_pa);
+/**
+ * i40iw_puda_free_qp - free qp for resource
+ * @rsrc: resource for which qp to free
+ */
+static void i40iw_puda_free_qp(struct i40iw_puda_rsrc *rsrc)
+{
+	enum i40iw_status_code ret;
+	struct i40iw_ccq_cqe_info compl_info;
+	struct i40iw_sc_dev *dev = rsrc->dev;
 
-	set_64bit_val(wqe, 40, cq->shadow_area_pa);
+	if (rsrc->ceq_valid) {
+		i40iw_cqp_qp_destroy_cmd(dev, &rsrc->qp);
+		return;
+	}
 
-	header = rsrc->cq_id |
-	    LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
-	    LS_64(1, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
-	    LS_64(1, I40IW_CQPSQ_CQ_ENCEQEMASK) |
-	    LS_64(1, I40IW_CQPSQ_CQ_CEQIDVALID) |
-	    LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-	set_64bit_val(wqe, 24, header);
+	ret = dev->iw_priv_qp_ops->qp_destroy(&rsrc->qp,
+			0, false, true, true);
+	if (ret)
+		i40iw_debug(dev, I40IW_DEBUG_PUDA,
+			    "%s error puda qp destroy wqe\n",
+			    __func__);
 
-	i40iw_debug_buf(dev, I40IW_DEBUG_PUDA, "PUDA CQE",
-			wqe, I40IW_CQP_WQE_SIZE * 8);
+	if (!ret) {
+		ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
+				I40IW_CQP_OP_DESTROY_QP,
+				&compl_info);
+		if (ret)
+			i40iw_debug(dev, I40IW_DEBUG_PUDA,
+				    "%s error puda qp destroy failed\n",
+				    __func__);
+	}
+}
 
-	i40iw_sc_cqp_post_sq(dev->cqp);
-	ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
-						 I40IW_CQP_OP_CREATE_CQ,
-						 &compl_info);
+/**
+ * i40iw_puda_free_cq - free cq for resource
+ * @rsrc: resource for which cq to free
+ */
+static void i40iw_puda_free_cq(struct i40iw_puda_rsrc *rsrc)
+{
+	enum i40iw_status_code ret;
+	struct i40iw_ccq_cqe_info compl_info;
+	struct i40iw_sc_dev *dev = rsrc->dev;
+
+	if (rsrc->ceq_valid) {
+		i40iw_cqp_cq_destroy_cmd(dev, &rsrc->cq);
+		return;
+	}
+	ret = dev->iw_priv_cq_ops->cq_destroy(&rsrc->cq, 0, true);
 
-error:
 	if (ret)
-		i40iw_free_dma_mem(dev->hw, &rsrc->cqmem);
-	return ret;
+		i40iw_debug(dev, I40IW_DEBUG_PUDA,
+			    "%s error ieq cq destroy\n",
+			    __func__);
+
+	if (!ret) {
+		ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
+				I40IW_CQP_OP_DESTROY_CQ,
+				&compl_info);
+		if (ret)
+			i40iw_debug(dev, I40IW_DEBUG_PUDA,
+				    "%s error ieq qp destroy done\n",
+				    __func__);
+	}
 }
 
 /**
@@ -701,25 +785,24 @@ error:
  * @type: type of resource to dele
  * @reset: true if reset chip
  */
-void i40iw_puda_dele_resources(struct i40iw_sc_dev *dev,
+void i40iw_puda_dele_resources(struct i40iw_sc_vsi *vsi,
 			       enum puda_resource_type type,
 			       bool reset)
 {
-	struct i40iw_ccq_cqe_info compl_info;
+	struct i40iw_sc_dev *dev = vsi->dev;
 	struct i40iw_puda_rsrc *rsrc;
 	struct i40iw_puda_buf *buf = NULL;
 	struct i40iw_puda_buf *nextbuf = NULL;
 	struct i40iw_virt_mem *vmem;
-	enum i40iw_status_code ret;
 
 	switch (type) {
 	case I40IW_PUDA_RSRC_TYPE_ILQ:
-		rsrc = dev->ilq;
-		vmem = &dev->ilq_mem;
+		rsrc = vsi->ilq;
+		vmem = &vsi->ilq_mem;
 		break;
 	case I40IW_PUDA_RSRC_TYPE_IEQ:
-		rsrc = dev->ieq;
-		vmem = &dev->ieq_mem;
+		rsrc = vsi->ieq;
+		vmem = &vsi->ieq_mem;
 		break;
 	default:
 		i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s: error resource type = 0x%x\n",
@@ -731,45 +814,14 @@ void i40iw_puda_dele_resources(struct i40iw_sc_dev *dev,
 	case PUDA_HASH_CRC_COMPLETE:
 		i40iw_free_hash_desc(rsrc->hash_desc);
 	case PUDA_QP_CREATED:
-		do {
-			if (reset)
-				break;
-			ret = dev->iw_priv_qp_ops->qp_destroy(&rsrc->qp,
-							      0, false, true, true);
-			if (ret)
-				i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA,
-					    "%s error ieq qp destroy\n",
-					    __func__);
-
-			ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
-								 I40IW_CQP_OP_DESTROY_QP,
-								 &compl_info);
-			if (ret)
-				i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA,
-					    "%s error ieq qp destroy done\n",
-					    __func__);
-		} while (0);
+		if (!reset)
+			i40iw_puda_free_qp(rsrc);
 
 		i40iw_free_dma_mem(dev->hw, &rsrc->qpmem);
 		/* fallthrough */
 	case PUDA_CQ_CREATED:
-		do {
-			if (reset)
-				break;
-			ret = dev->iw_priv_cq_ops->cq_destroy(&rsrc->cq, 0, true);
-			if (ret)
-				i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA,
-					    "%s error ieq cq destroy\n",
-					    __func__);
-
-			ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
-								 I40IW_CQP_OP_DESTROY_CQ,
-								 &compl_info);
-			if (ret)
-				i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA,
-					    "%s error ieq qp destroy done\n",
-					    __func__);
-		} while (0);
+		if (!reset)
+			i40iw_puda_free_cq(rsrc);
 
 		i40iw_free_dma_mem(dev->hw, &rsrc->cqmem);
 		break;
@@ -825,9 +877,10 @@ static enum i40iw_status_code i40iw_puda_allocbufs(struct i40iw_puda_rsrc *rsrc,
  * @dev: iwarp device
  * @info: resource information
  */
-enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_dev *dev,
+enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_vsi *vsi,
 					      struct i40iw_puda_rsrc_info *info)
 {
+	struct i40iw_sc_dev *dev = vsi->dev;
 	enum i40iw_status_code ret = 0;
 	struct i40iw_puda_rsrc *rsrc;
 	u32 pudasize;
@@ -840,10 +893,10 @@ enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_dev *dev,
 	rqwridsize = info->rq_size * 8;
 	switch (info->type) {
 	case I40IW_PUDA_RSRC_TYPE_ILQ:
-		vmem = &dev->ilq_mem;
+		vmem = &vsi->ilq_mem;
 		break;
 	case I40IW_PUDA_RSRC_TYPE_IEQ:
-		vmem = &dev->ieq_mem;
+		vmem = &vsi->ieq_mem;
 		break;
 	default:
 		return I40IW_NOT_SUPPORTED;
@@ -856,22 +909,22 @@ enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_dev *dev,
 	rsrc = (struct i40iw_puda_rsrc *)vmem->va;
 	spin_lock_init(&rsrc->bufpool_lock);
 	if (info->type == I40IW_PUDA_RSRC_TYPE_ILQ) {
-		dev->ilq = (struct i40iw_puda_rsrc *)vmem->va;
-		dev->ilq_count = info->count;
+		vsi->ilq = (struct i40iw_puda_rsrc *)vmem->va;
+		vsi->ilq_count = info->count;
 		rsrc->receive = info->receive;
 		rsrc->xmit_complete = info->xmit_complete;
 	} else {
-		vmem = &dev->ieq_mem;
-		dev->ieq_count = info->count;
-		dev->ieq = (struct i40iw_puda_rsrc *)vmem->va;
+		vmem = &vsi->ieq_mem;
+		vsi->ieq_count = info->count;
+		vsi->ieq = (struct i40iw_puda_rsrc *)vmem->va;
 		rsrc->receive = i40iw_ieq_receive;
 		rsrc->xmit_complete = i40iw_ieq_tx_compl;
 	}
 
+	rsrc->ceq_valid = info->ceq_valid;
 	rsrc->type = info->type;
 	rsrc->sq_wrtrk_array = (struct i40iw_sq_uk_wr_trk_info *)((u8 *)vmem->va + pudasize);
 	rsrc->rq_wrid_array = (u64 *)((u8 *)vmem->va + pudasize + sqwridsize);
-	rsrc->mss = info->mss;
 	/* Initialize all ieq lists */
 	INIT_LIST_HEAD(&rsrc->bufpool);
 	INIT_LIST_HEAD(&rsrc->txpend);
@@ -885,6 +938,7 @@ enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_dev *dev,
 	rsrc->cq_size = info->rq_size + info->sq_size;
 	rsrc->buf_size = info->buf_size;
 	rsrc->dev = dev;
+	rsrc->vsi = vsi;
 
 	ret = i40iw_puda_cq_create(rsrc);
 	if (!ret) {
@@ -919,7 +973,7 @@ enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_dev *dev,
 	dev->ccq_ops->ccq_arm(&rsrc->cq);
 	return ret;
  error:
-	i40iw_puda_dele_resources(dev, info->type, false);
+	i40iw_puda_dele_resources(vsi, info->type, false);
 
 	return ret;
 }
@@ -1131,7 +1185,7 @@ static enum i40iw_status_code i40iw_ieq_handle_partial(struct i40iw_puda_rsrc *i
 	list_add(&buf->list, &pbufl);
 
 	status = i40iw_ieq_create_pbufl(pfpdu, rxlist, &pbufl, buf, fpdu_len);
-	if (!status)
+	if (status)
 		goto error;
 
 	txbuf = i40iw_puda_get_bufpool(ieq);
@@ -1332,7 +1386,7 @@ static void i40iw_ieq_handle_exception(struct i40iw_puda_rsrc *ieq,
 	}
 	if (pfpdu->mode && (fps != pfpdu->fps)) {
 		/* clean up qp as it is new partial sequence */
-		i40iw_ieq_cleanup_qp(ieq->dev, qp);
+		i40iw_ieq_cleanup_qp(ieq, qp);
 		i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ,
 			    "%s: restarting new partial\n", __func__);
 		pfpdu->mode = false;
@@ -1344,7 +1398,7 @@ static void i40iw_ieq_handle_exception(struct i40iw_puda_rsrc *ieq,
 		pfpdu->rcv_nxt = fps;
 		pfpdu->fps = fps;
 		pfpdu->mode = true;
-		pfpdu->max_fpdu_data = ieq->mss;
+		pfpdu->max_fpdu_data = ieq->vsi->mss;
 		pfpdu->pmode_count++;
 		INIT_LIST_HEAD(rxlist);
 		i40iw_ieq_check_first_buf(buf, fps);
@@ -1379,14 +1433,14 @@ static void i40iw_ieq_handle_exception(struct i40iw_puda_rsrc *ieq,
  * @dev: iwarp device
  * @buf: exception buffer received
  */
-static void i40iw_ieq_receive(struct i40iw_sc_dev *dev,
+static void i40iw_ieq_receive(struct i40iw_sc_vsi *vsi,
 			      struct i40iw_puda_buf *buf)
 {
-	struct i40iw_puda_rsrc *ieq = dev->ieq;
+	struct i40iw_puda_rsrc *ieq = vsi->ieq;
 	struct i40iw_sc_qp *qp = NULL;
 	u32 wqe_idx = ieq->compl_rxwqe_idx;
 
-	qp = i40iw_ieq_get_qp(dev, buf);
+	qp = i40iw_ieq_get_qp(vsi->dev, buf);
 	if (!qp) {
 		ieq->stats_bad_qp_id++;
 		i40iw_puda_ret_bufpool(ieq, buf);
@@ -1404,12 +1458,12 @@ static void i40iw_ieq_receive(struct i40iw_sc_dev *dev,
 
 /**
  * i40iw_ieq_tx_compl - put back after sending completed exception buffer
- * @dev: iwarp device
+ * @vsi: pointer to the vsi structure
  * @sqwrid: pointer to puda buffer
  */
-static void i40iw_ieq_tx_compl(struct i40iw_sc_dev *dev, void *sqwrid)
+static void i40iw_ieq_tx_compl(struct i40iw_sc_vsi *vsi, void *sqwrid)
 {
-	struct i40iw_puda_rsrc *ieq = dev->ieq;
+	struct i40iw_puda_rsrc *ieq = vsi->ieq;
 	struct i40iw_puda_buf *buf = (struct i40iw_puda_buf *)sqwrid;
 
 	i40iw_puda_ret_bufpool(ieq, buf);
@@ -1421,15 +1475,14 @@ static void i40iw_ieq_tx_compl(struct i40iw_sc_dev *dev, void *sqwrid)
 
 /**
  * i40iw_ieq_cleanup_qp - qp is being destroyed
- * @dev: iwarp device
+ * @ieq: ieq resource
  * @qp: all pending fpdu buffers
  */
-void i40iw_ieq_cleanup_qp(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp)
+static void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp)
 {
 	struct i40iw_puda_buf *buf;
 	struct i40iw_pfpdu *pfpdu = &qp->pfpdu;
 	struct list_head *rxlist = &pfpdu->rxlist;
-	struct i40iw_puda_rsrc *ieq = dev->ieq;
 
 	if (!pfpdu->mode)
 		return;
diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.h b/drivers/infiniband/hw/i40iw/i40iw_puda.h
index 52bf7826ce4e..dba05ce7d392 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_puda.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_puda.h
@@ -100,6 +100,7 @@ struct i40iw_puda_rsrc_info {
 	enum puda_resource_type type;	/* ILQ or IEQ */
 	u32 count;
 	u16 pd_id;
+	bool ceq_valid;
 	u32 cq_id;
 	u32 qp_id;
 	u32 sq_size;
@@ -107,8 +108,8 @@ struct i40iw_puda_rsrc_info {
 	u16 buf_size;
 	u16 mss;
 	u32 tx_buf_cnt;		/* total bufs allocated will be rq_size + tx_buf_cnt */
-	void (*receive)(struct i40iw_sc_dev *, struct i40iw_puda_buf *);
-	void (*xmit_complete)(struct i40iw_sc_dev *, void *);
+	void (*receive)(struct i40iw_sc_vsi *, struct i40iw_puda_buf *);
+	void (*xmit_complete)(struct i40iw_sc_vsi *, void *);
 };
 
 struct i40iw_puda_rsrc {
@@ -116,6 +117,7 @@ struct i40iw_puda_rsrc {
 	struct i40iw_sc_qp qp;
 	struct i40iw_sc_pd sc_pd;
 	struct i40iw_sc_dev *dev;
+	struct i40iw_sc_vsi *vsi;
 	struct i40iw_dma_mem cqmem;
 	struct i40iw_dma_mem qpmem;
 	struct i40iw_virt_mem ilq_mem;
@@ -123,6 +125,7 @@ struct i40iw_puda_rsrc {
 	enum puda_resource_type type;
 	u16 buf_size;		/*buffer must be max datalen + tcpip hdr + mac */
 	u16 mss;
+	bool ceq_valid;
 	u32 cq_id;
 	u32 qp_id;
 	u32 sq_size;
@@ -142,8 +145,8 @@ struct i40iw_puda_rsrc {
 	u32 avail_buf_count;		/* snapshot of currently available buffers */
 	spinlock_t bufpool_lock;
 	struct i40iw_puda_buf *alloclist;
-	void (*receive)(struct i40iw_sc_dev *, struct i40iw_puda_buf *);
-	void (*xmit_complete)(struct i40iw_sc_dev *, void *);
+	void (*receive)(struct i40iw_sc_vsi *, struct i40iw_puda_buf *);
+	void (*xmit_complete)(struct i40iw_sc_vsi *, void *);
 	/* puda stats */
 	u64 stats_buf_alloc_fail;
 	u64 stats_pkt_rcvd;
@@ -160,14 +163,13 @@ void i40iw_puda_send_buf(struct i40iw_puda_rsrc *rsrc,
 			 struct i40iw_puda_buf *buf);
 enum i40iw_status_code i40iw_puda_send(struct i40iw_sc_qp *qp,
 				       struct i40iw_puda_send_info *info);
-enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_dev *dev,
+enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_vsi *vsi,
 					      struct i40iw_puda_rsrc_info *info);
-void i40iw_puda_dele_resources(struct i40iw_sc_dev *dev,
+void i40iw_puda_dele_resources(struct i40iw_sc_vsi *vsi,
 			       enum puda_resource_type type,
 			       bool reset);
 enum i40iw_status_code i40iw_puda_poll_completion(struct i40iw_sc_dev *dev,
 						  struct i40iw_sc_cq *cq, u32 *compl_err);
-void i40iw_ieq_cleanup_qp(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);
 
 struct i40iw_sc_qp *i40iw_ieq_get_qp(struct i40iw_sc_dev *dev,
 				     struct i40iw_puda_buf *buf);
@@ -180,4 +182,8 @@ void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);
 void i40iw_free_hash_desc(struct shash_desc *desc);
 void i40iw_ieq_update_tcpip_info(struct i40iw_puda_buf *buf, u16 length,
 				 u32 seqnum);
+enum i40iw_status_code i40iw_cqp_qp_create_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);
+enum i40iw_status_code i40iw_cqp_cq_create_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq);
+void i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);
+void i40iw_cqp_cq_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq);
 #endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_type.h b/drivers/infiniband/hw/i40iw/i40iw_type.h
index 2b1a04e9ca3c..f3f8e9cc3c05 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_type.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_type.h
@@ -61,7 +61,7 @@ struct i40iw_cq_shadow_area {
 
 struct i40iw_sc_dev;
 struct i40iw_hmc_info;
-struct i40iw_dev_pestat;
+struct i40iw_vsi_pestat;
 
 struct i40iw_cqp_ops;
 struct i40iw_ccq_ops;
@@ -74,6 +74,11 @@ struct i40iw_priv_qp_ops;
 struct i40iw_priv_cq_ops;
 struct i40iw_hmc_ops;
 
+enum i40iw_page_size {
+	I40IW_PAGE_SIZE_4K,
+	I40IW_PAGE_SIZE_2M
+};
+
 enum i40iw_resource_indicator_type {
 	I40IW_RSRC_INDICATOR_TYPE_ADAPTER = 0,
 	I40IW_RSRC_INDICATOR_TYPE_CQ,
@@ -186,7 +191,7 @@ enum i40iw_debug_flag {
 	I40IW_DEBUG_ALL		= 0xFFFFFFFF
 };
 
-enum i40iw_hw_stat_index_32b {
+enum i40iw_hw_stats_index_32b {
 	I40IW_HW_STAT_INDEX_IP4RXDISCARD = 0,
 	I40IW_HW_STAT_INDEX_IP4RXTRUNC,
 	I40IW_HW_STAT_INDEX_IP4TXNOROUTE,
@@ -199,7 +204,7 @@ enum i40iw_hw_stat_index_32b {
 	I40IW_HW_STAT_INDEX_MAX_32
 };
 
-enum i40iw_hw_stat_index_64b {
+enum i40iw_hw_stats_index_64b {
 	I40IW_HW_STAT_INDEX_IP4RXOCTS = 0,
 	I40IW_HW_STAT_INDEX_IP4RXPKTS,
 	I40IW_HW_STAT_INDEX_IP4RXFRAGS,
@@ -229,32 +234,23 @@ enum i40iw_hw_stat_index_64b {
 	I40IW_HW_STAT_INDEX_MAX_64
 };
 
-struct i40iw_dev_hw_stat_offsets {
-	u32 stat_offset_32[I40IW_HW_STAT_INDEX_MAX_32];
-	u32 stat_offset_64[I40IW_HW_STAT_INDEX_MAX_64];
+struct i40iw_dev_hw_stats_offsets {
+	u32 stats_offset_32[I40IW_HW_STAT_INDEX_MAX_32];
+	u32 stats_offset_64[I40IW_HW_STAT_INDEX_MAX_64];
 };
 
 struct i40iw_dev_hw_stats {
-	u64 stat_value_32[I40IW_HW_STAT_INDEX_MAX_32];
-	u64 stat_value_64[I40IW_HW_STAT_INDEX_MAX_64];
-};
-
-struct i40iw_device_pestat_ops {
-	void (*iw_hw_stat_init)(struct i40iw_dev_pestat *, u8, struct i40iw_hw *, bool);
-	void (*iw_hw_stat_read_32)(struct i40iw_dev_pestat *, enum i40iw_hw_stat_index_32b, u64 *);
-	void (*iw_hw_stat_read_64)(struct i40iw_dev_pestat *, enum i40iw_hw_stat_index_64b, u64 *);
-	void (*iw_hw_stat_read_all)(struct i40iw_dev_pestat *, struct i40iw_dev_hw_stats *);
-	void (*iw_hw_stat_refresh_all)(struct i40iw_dev_pestat *);
+	u64 stats_value_32[I40IW_HW_STAT_INDEX_MAX_32];
+	u64 stats_value_64[I40IW_HW_STAT_INDEX_MAX_64];
 };
 
-struct i40iw_dev_pestat {
+struct i40iw_vsi_pestat {
 	struct i40iw_hw *hw;
-	struct i40iw_device_pestat_ops ops;
 	struct i40iw_dev_hw_stats hw_stats;
 	struct i40iw_dev_hw_stats last_read_hw_stats;
-	struct i40iw_dev_hw_stat_offsets hw_stat_offsets;
+	struct i40iw_dev_hw_stats_offsets hw_stats_offsets;
 	struct timer_list stats_timer;
-	spinlock_t stats_lock; /* rdma stats lock */
+	spinlock_t lock; /* rdma stats lock */
 };
 
 struct i40iw_hw {
@@ -350,6 +346,7 @@ struct i40iw_sc_cq {
 	u64 cq_pa;
 	u64 shadow_area_pa;
 	struct i40iw_sc_dev *dev;
+	struct i40iw_sc_vsi *vsi;
 	void *pbl_list;
 	void *back_cq;
 	u32 ceq_id;
@@ -373,6 +370,7 @@ struct i40iw_sc_qp {
 	u64 shadow_area_pa;
 	u64 q2_pa;
 	struct i40iw_sc_dev *dev;
+	struct i40iw_sc_vsi *vsi;
 	struct i40iw_sc_pd *pd;
 	u64 *hw_host_ctx;
 	void *llp_stream_handle;
@@ -397,6 +395,9 @@ struct i40iw_sc_qp {
 	bool virtual_map;
 	bool flush_sq;
 	bool flush_rq;
+	u8 user_pri;
+	struct list_head list;
+	bool on_qoslist;
 	bool sq_flush;
 	enum i40iw_flush_opcode flush_code;
 	enum i40iw_term_eventtypes eventtype;
@@ -424,10 +425,16 @@ struct i40iw_vchnl_vf_msg_buffer {
 	char parm_buffer[I40IW_VCHNL_MAX_VF_MSG_SIZE - 1];
 };
 
+struct i40iw_qos {
+	struct list_head qplist;
+	spinlock_t lock;	/* qos list */
+	u16 qs_handle;
+};
+
 struct i40iw_vfdev {
 	struct i40iw_sc_dev *pf_dev;
 	u8 *hmc_info_mem;
-	struct i40iw_dev_pestat dev_pestat;
+	struct i40iw_vsi_pestat pestat;
 	struct i40iw_hmc_pble_info *pble_info;
 	struct i40iw_hmc_info hmc_info;
 	struct i40iw_vchnl_vf_msg_buffer vf_msg_buffer;
@@ -441,11 +448,28 @@ struct i40iw_vfdev {
 	bool stats_initialized;
 };
 
+#define I40IW_INVALID_FCN_ID 0xff
+struct i40iw_sc_vsi {
+	struct i40iw_sc_dev *dev;
+	void *back_vsi; /* Owned by OS */
+	u32 ilq_count;
+	struct i40iw_virt_mem ilq_mem;
+	struct i40iw_puda_rsrc *ilq;
+	u32 ieq_count;
+	struct i40iw_virt_mem ieq_mem;
+	struct i40iw_puda_rsrc *ieq;
+	u16 mss;
+	u8 fcn_id;
+	bool stats_fcn_id_alloc;
+	struct i40iw_qos qos[I40IW_MAX_USER_PRIORITY];
+	struct i40iw_vsi_pestat *pestat;
+};
+
 struct i40iw_sc_dev {
 	struct list_head cqp_cmd_head;	/* head of the CQP command list */
 	spinlock_t cqp_lock; /* cqp list sync */
 	struct i40iw_dev_uk dev_uk;
-	struct i40iw_dev_pestat dev_pestat;
+	bool fcn_id_array[I40IW_MAX_STATS_COUNT];
 	struct i40iw_dma_mem vf_fpm_query_buf[I40IW_MAX_PE_ENABLED_VF_COUNT];
 	u64 fpm_query_buf_pa;
 	u64 fpm_commit_buf_pa;
@@ -472,17 +496,9 @@ struct i40iw_sc_dev {
 	struct i40iw_cqp_misc_ops *cqp_misc_ops;
 	struct i40iw_hmc_ops *hmc_ops;
 	struct i40iw_vchnl_if vchnl_if;
-	u32 ilq_count;
-	struct i40iw_virt_mem ilq_mem;
-	struct i40iw_puda_rsrc *ilq;
-	u32 ieq_count;
-	struct i40iw_virt_mem ieq_mem;
-	struct i40iw_puda_rsrc *ieq;
-
 	const struct i40iw_vf_cqp_ops *iw_vf_cqp_ops;
 
 	struct i40iw_hmc_fpm_misc hmc_fpm_misc;
-	u16 qs_handle;
 	u32 debug_mask;
 	u16 exception_lan_queue;
 	u8 hmc_fn_id;
@@ -556,6 +572,19 @@ struct i40iw_l2params {
 	u16 mss;
 };
 
+struct i40iw_vsi_init_info {
+	struct i40iw_sc_dev *dev;
+	void  *back_vsi;
+	struct i40iw_l2params *params;
+};
+
+struct i40iw_vsi_stats_info {
+	struct i40iw_vsi_pestat *pestat;
+	u8 fcn_id;
+	bool alloc_fcn_id;
+	bool stats_initialize;
+};
+
 struct i40iw_device_init_info {
 	u64 fpm_query_buf_pa;
 	u64 fpm_commit_buf_pa;
@@ -564,7 +593,6 @@ struct i40iw_device_init_info {
 	struct i40iw_hw *hw;
 	void __iomem *bar0;
 	enum i40iw_status_code (*vchnl_send)(struct i40iw_sc_dev *, u32, u8 *, u16);
-	u16 qs_handle;
 	u16 exception_lan_queue;
 	u8 hmc_fn_id;
 	bool is_pf;
@@ -722,6 +750,8 @@ struct i40iw_qp_host_ctx_info {
 	bool iwarp_info_valid;
 	bool err_rq_idx_valid;
 	u16 err_rq_idx;
+	bool add_to_qoslist;
+	u8 user_pri;
 };
 
 struct i40iw_aeqe_info {
@@ -814,6 +844,7 @@ struct i40iw_register_shared_stag {
 struct i40iw_qp_init_info {
 	struct i40iw_qp_uk_init_info qp_uk_init_info;
 	struct i40iw_sc_pd *pd;
+	struct i40iw_sc_vsi *vsi;
 	u64 *host_ctx;
 	u8 *q2;
 	u64 sq_pa;
@@ -880,13 +911,14 @@ enum i40iw_quad_hash_manage_type {
 };
 
 struct i40iw_qhash_table_info {
+	struct i40iw_sc_vsi *vsi;
 	enum i40iw_quad_hash_manage_type manage;
 	enum i40iw_quad_entry_type entry_type;
 	bool vlan_valid;
 	bool ipv4_valid;
 	u8 mac_addr[6];
 	u16 vlan_id;
-	u16 qs_handle;
+	u8 user_pri;
 	u32 qp_num;
 	u32 dest_ip[4];
 	u32 src_ip[4];
@@ -976,7 +1008,7 @@ struct i40iw_cqp_query_fpm_values {
 struct i40iw_cqp_ops {
 	enum i40iw_status_code (*cqp_init)(struct i40iw_sc_cqp *,
 					   struct i40iw_cqp_init_info *);
-	enum i40iw_status_code (*cqp_create)(struct i40iw_sc_cqp *, bool, u16 *, u16 *);
+	enum i40iw_status_code (*cqp_create)(struct i40iw_sc_cqp *, u16 *, u16 *);
 	void (*cqp_post_sq)(struct i40iw_sc_cqp *);
 	u64 *(*cqp_get_next_send_wqe)(struct i40iw_sc_cqp *, u64 scratch);
 	enum i40iw_status_code (*cqp_destroy)(struct i40iw_sc_cqp *);
diff --git a/drivers/infiniband/hw/i40iw/i40iw_uk.c b/drivers/infiniband/hw/i40iw/i40iw_uk.c
index 4d28c3cb03cc..4376cd628774 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_uk.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_uk.c
@@ -175,12 +175,10 @@ u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp,
 		if (!*wqe_idx)
 			qp->swqe_polarity = !qp->swqe_polarity;
 	}
-
-	for (i = 0; i < wqe_size / I40IW_QP_WQE_MIN_SIZE; i++) {
-		I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code);
-		if (ret_code)
-			return NULL;
-	}
+	I40IW_RING_MOVE_HEAD_BY_COUNT(qp->sq_ring,
+				      wqe_size / I40IW_QP_WQE_MIN_SIZE, ret_code);
+	if (ret_code)
+		return NULL;
 
 	wqe = qp->sq_base[*wqe_idx].elem;
 
@@ -430,7 +428,7 @@ static enum i40iw_status_code i40iw_inline_rdma_write(struct i40iw_qp_uk *qp,
 	struct i40iw_inline_rdma_write *op_info;
 	u64 *push;
 	u64 header = 0;
-	u32 i, wqe_idx;
+	u32 wqe_idx;
 	enum i40iw_status_code ret_code;
 	bool read_fence = false;
 	u8 wqe_size;
@@ -465,14 +463,12 @@ static enum i40iw_status_code i40iw_inline_rdma_write(struct i40iw_qp_uk *qp,
 	src = (u8 *)(op_info->data);
 
 	if (op_info->len <= 16) {
-		for (i = 0; i < op_info->len; i++, src++, dest++)
-			*dest = *src;
+		memcpy(dest, src, op_info->len);
 	} else {
-		for (i = 0; i < 16; i++, src++, dest++)
-			*dest = *src;
+		memcpy(dest, src, 16);
+		src += 16;
 		dest = (u8 *)wqe + 32;
-		for (; i < op_info->len; i++, src++, dest++)
-			*dest = *src;
+		memcpy(dest, src, op_info->len - 16);
 	}
 
 	wmb(); /* make sure WQE is populated before valid bit is set */
@@ -507,7 +503,7 @@ static enum i40iw_status_code i40iw_inline_send(struct i40iw_qp_uk *qp,
 	u8 *dest, *src;
 	struct i40iw_post_inline_send *op_info;
 	u64 header;
-	u32 wqe_idx, i;
+	u32 wqe_idx;
 	enum i40iw_status_code ret_code;
 	bool read_fence = false;
 	u8 wqe_size;
@@ -540,14 +536,12 @@ static enum i40iw_status_code i40iw_inline_send(struct i40iw_qp_uk *qp,
 	src = (u8 *)(op_info->data);
 
 	if (op_info->len <= 16) {
-		for (i = 0; i < op_info->len; i++, src++, dest++)
-			*dest = *src;
+		memcpy(dest, src, op_info->len);
 	} else {
-		for (i = 0; i < 16; i++, src++, dest++)
-			*dest = *src;
+		memcpy(dest, src, 16);
+		src += 16;
 		dest = (u8 *)wqe + 32;
-		for (; i < op_info->len; i++, src++, dest++)
-			*dest = *src;
+		memcpy(dest, src, op_info->len - 16);
 	}
 
 	wmb(); /* make sure WQE is populated before valid bit is set */
@@ -1190,12 +1184,8 @@ enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size,
 
 	if (data_size <= 16)
 		*wqe_size = I40IW_QP_WQE_MIN_SIZE;
-	else if (data_size <= 48)
-		*wqe_size = 64;
-	else if (data_size <= 80)
-		*wqe_size = 96;
 	else
-		*wqe_size = 128;
+		*wqe_size = 64;
 
 	return 0;
 }
diff --git a/drivers/infiniband/hw/i40iw/i40iw_user.h b/drivers/infiniband/hw/i40iw/i40iw_user.h
index 276bcefffd7e..80d9f464f65e 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_user.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_user.h
@@ -72,12 +72,12 @@ enum i40iw_device_capabilities_const {
 	I40IW_MAX_SQ_PAYLOAD_SIZE =		2145386496,
 	I40IW_MAX_INLINE_DATA_SIZE =		48,
 	I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE =	48,
-	I40IW_MAX_IRD_SIZE =			32,
-	I40IW_QPCTX_ENCD_MAXIRD =		3,
+	I40IW_MAX_IRD_SIZE =			63,
+	I40IW_MAX_ORD_SIZE =			127,
 	I40IW_MAX_WQ_ENTRIES =			2048,
-	I40IW_MAX_ORD_SIZE =			32,
 	I40IW_Q2_BUFFER_SIZE =			(248 + 100),
-	I40IW_QP_CTX_SIZE =			248
+	I40IW_QP_CTX_SIZE =			248,
+	I40IW_MAX_PDS = 			32768
 };
 
 #define i40iw_handle void *
@@ -96,12 +96,6 @@ enum i40iw_device_capabilities_const {
 #define i40iw_physical_fragment u64
 #define i40iw_address_list u64 *
 
-#define I40IW_CREATE_STAG(index, key)       (((index) << 8) + (key))
-
-#define I40IW_STAG_KEY_FROM_STAG(stag)      ((stag) && 0x000000FF)
-
-#define I40IW_STAG_INDEX_FROM_STAG(stag)    (((stag) && 0xFFFFFF00) >> 8)
-
 #define	I40IW_MAX_MR_SIZE	0x10000000000L
 
 struct i40iw_qp_uk;
diff --git a/drivers/infiniband/hw/i40iw/i40iw_utils.c b/drivers/infiniband/hw/i40iw/i40iw_utils.c
index 6fd043b1d714..0f5d43d1f5fc 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_utils.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_utils.c
@@ -153,6 +153,7 @@ int i40iw_inetaddr_event(struct notifier_block *notifier,
 	struct i40iw_device *iwdev;
 	struct i40iw_handler *hdl;
 	u32 local_ipaddr;
+	u32 action = I40IW_ARP_ADD;
 
 	hdl = i40iw_find_netdev(event_netdev);
 	if (!hdl)
@@ -164,44 +165,25 @@ int i40iw_inetaddr_event(struct notifier_block *notifier,
 	if (netdev != event_netdev)
 		return NOTIFY_DONE;
 
+	if (upper_dev)
+		local_ipaddr = ntohl(
+			((struct in_device *)upper_dev->ip_ptr)->ifa_list->ifa_address);
+	else
+		local_ipaddr = ntohl(ifa->ifa_address);
 	switch (event) {
 	case NETDEV_DOWN:
-		if (upper_dev)
-			local_ipaddr = ntohl(
-				((struct in_device *)upper_dev->ip_ptr)->ifa_list->ifa_address);
-		else
-			local_ipaddr = ntohl(ifa->ifa_address);
-		i40iw_manage_arp_cache(iwdev,
-				       netdev->dev_addr,
-				       &local_ipaddr,
-				       true,
-				       I40IW_ARP_DELETE);
-		return NOTIFY_OK;
+		action = I40IW_ARP_DELETE;
+		/* Fall through */
 	case NETDEV_UP:
-		if (upper_dev)
-			local_ipaddr = ntohl(
-				((struct in_device *)upper_dev->ip_ptr)->ifa_list->ifa_address);
-		else
-			local_ipaddr = ntohl(ifa->ifa_address);
-		i40iw_manage_arp_cache(iwdev,
-				       netdev->dev_addr,
-				       &local_ipaddr,
-				       true,
-				       I40IW_ARP_ADD);
-		break;
+		/* Fall through */
 	case NETDEV_CHANGEADDR:
-		/* Add the address to the IP table */
-		if (upper_dev)
-			local_ipaddr = ntohl(
-				((struct in_device *)upper_dev->ip_ptr)->ifa_list->ifa_address);
-		else
-			local_ipaddr = ntohl(ifa->ifa_address);
-
 		i40iw_manage_arp_cache(iwdev,
 				       netdev->dev_addr,
 				       &local_ipaddr,
 				       true,
-				       I40IW_ARP_ADD);
+				       action);
+		i40iw_if_notify(iwdev, netdev, &local_ipaddr, true,
+				(action == I40IW_ARP_ADD) ? true : false);
 		break;
 	default:
 		break;
@@ -225,6 +207,7 @@ int i40iw_inet6addr_event(struct notifier_block *notifier,
 	struct i40iw_device *iwdev;
 	struct i40iw_handler *hdl;
 	u32 local_ipaddr6[4];
+	u32 action = I40IW_ARP_ADD;
 
 	hdl = i40iw_find_netdev(event_netdev);
 	if (!hdl)
@@ -235,24 +218,21 @@ int i40iw_inet6addr_event(struct notifier_block *notifier,
 	if (netdev != event_netdev)
 		return NOTIFY_DONE;
 
+	i40iw_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
 	switch (event) {
 	case NETDEV_DOWN:
-		i40iw_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
-		i40iw_manage_arp_cache(iwdev,
-				       netdev->dev_addr,
-				       local_ipaddr6,
-				       false,
-				       I40IW_ARP_DELETE);
-		return NOTIFY_OK;
+		action = I40IW_ARP_DELETE;
+		/* Fall through */
 	case NETDEV_UP:
 		/* Fall through */
 	case NETDEV_CHANGEADDR:
-		i40iw_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
 		i40iw_manage_arp_cache(iwdev,
 				       netdev->dev_addr,
 				       local_ipaddr6,
 				       false,
-				       I40IW_ARP_ADD);
+				       action);
+		i40iw_if_notify(iwdev, netdev, local_ipaddr6, false,
+				(action == I40IW_ARP_ADD) ? true : false);
 		break;
 	default:
 		break;
@@ -392,6 +372,7 @@ static void i40iw_free_qp(struct i40iw_cqp_request *cqp_request, u32 num)
 
 	i40iw_rem_pdusecount(iwqp->iwpd, iwdev);
 	i40iw_free_qp_resources(iwdev, iwqp, qp_num);
+	i40iw_rem_devusecount(iwdev);
 }
 
 /**
@@ -415,7 +396,10 @@ static int i40iw_wait_event(struct i40iw_device *iwdev,
 		i40iw_pr_err("error cqp command 0x%x timed out ret = %d\n",
 			     info->cqp_cmd, timeout_ret);
 		err_code = -ETIME;
-		i40iw_request_reset(iwdev);
+		if (!iwdev->reset) {
+			iwdev->reset = true;
+			i40iw_request_reset(iwdev);
+		}
 		goto done;
 	}
 	cqp_error = cqp_request->compl_info.error;
@@ -445,6 +429,11 @@ enum i40iw_status_code i40iw_handle_cqp_op(struct i40iw_device *iwdev,
 	struct cqp_commands_info *info = &cqp_request->info;
 	int err_code = 0;
 
+	if (iwdev->reset) {
+		i40iw_free_cqp_request(&iwdev->cqp, cqp_request);
+		return I40IW_ERR_CQP_COMPL_ERROR;
+	}
+
 	status = i40iw_process_cqp_cmd(dev, info);
 	if (status) {
 		i40iw_pr_err("error cqp command 0x%x failed\n", info->cqp_cmd);
@@ -459,6 +448,26 @@ enum i40iw_status_code i40iw_handle_cqp_op(struct i40iw_device *iwdev,
 }
 
 /**
+ * i40iw_add_devusecount - add dev refcount
+ * @iwdev: dev for refcount
+ */
+void i40iw_add_devusecount(struct i40iw_device *iwdev)
+{
+	atomic64_inc(&iwdev->use_count);
+}
+
+/**
+ * i40iw_rem_devusecount - decrement refcount for dev
+ * @iwdev: device
+ */
+void i40iw_rem_devusecount(struct i40iw_device *iwdev)
+{
+	if (!atomic64_dec_and_test(&iwdev->use_count))
+		return;
+	wake_up(&iwdev->close_wq);
+}
+
+/**
  * i40iw_add_pdusecount - add pd refcount
  * @iwpd: pd for refcount
  */
@@ -712,6 +721,51 @@ enum i40iw_status_code i40iw_cqp_sds_cmd(struct i40iw_sc_dev *dev,
 }
 
 /**
+ * i40iw_qp_suspend_resume - cqp command for suspend/resume
+ * @dev: hardware control device structure
+ * @qp: hardware control qp
+ * @suspend: flag if suspend or resume
+ */
+void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp, bool suspend)
+{
+	struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
+	struct i40iw_cqp_request *cqp_request;
+	struct i40iw_sc_cqp *cqp = dev->cqp;
+	struct cqp_commands_info *cqp_info;
+	enum i40iw_status_code status;
+
+	cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false);
+	if (!cqp_request)
+		return;
+
+	cqp_info = &cqp_request->info;
+	cqp_info->cqp_cmd = (suspend) ? OP_SUSPEND : OP_RESUME;
+	cqp_info->in.u.suspend_resume.cqp = cqp;
+	cqp_info->in.u.suspend_resume.qp = qp;
+	cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
+	status = i40iw_handle_cqp_op(iwdev, cqp_request);
+	if (status)
+		i40iw_pr_err("CQP-OP QP Suspend/Resume fail");
+}
+
+/**
+ * i40iw_qp_mss_modify - modify mss for qp
+ * @dev: hardware control device structure
+ * @qp: hardware control qp
+ */
+void i40iw_qp_mss_modify(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp)
+{
+	struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
+	struct i40iw_qp *iwqp = (struct i40iw_qp *)qp->back_qp;
+	struct i40iw_modify_qp_info info;
+
+	memset(&info, 0, sizeof(info));
+	info.mss_change = true;
+	info.new_mss = qp->vsi->mss;
+	i40iw_hw_modify_qp(iwdev, iwqp, &info, false);
+}
+
+/**
  * i40iw_term_modify_qp - modify qp for term message
  * @qp: hardware control qp
  * @next_state: qp's next state
@@ -769,6 +823,7 @@ static void i40iw_terminate_timeout(unsigned long context)
 	struct i40iw_sc_qp *qp = (struct i40iw_sc_qp *)&iwqp->sc_qp;
 
 	i40iw_terminate_done(qp, 1);
+	i40iw_rem_ref(&iwqp->ibqp);
 }
 
 /**
@@ -780,6 +835,7 @@ void i40iw_terminate_start_timer(struct i40iw_sc_qp *qp)
 	struct i40iw_qp *iwqp;
 
 	iwqp = (struct i40iw_qp *)qp->back_qp;
+	i40iw_add_ref(&iwqp->ibqp);
 	init_timer(&iwqp->terminate_timer);
 	iwqp->terminate_timer.function = i40iw_terminate_timeout;
 	iwqp->terminate_timer.expires = jiffies + HZ;
@@ -796,7 +852,8 @@ void i40iw_terminate_del_timer(struct i40iw_sc_qp *qp)
 	struct i40iw_qp *iwqp;
 
 	iwqp = (struct i40iw_qp *)qp->back_qp;
-	del_timer(&iwqp->terminate_timer);
+	if (del_timer(&iwqp->terminate_timer))
+		i40iw_rem_ref(&iwqp->ibqp);
 }
 
 /**
@@ -1011,6 +1068,116 @@ enum i40iw_status_code i40iw_vf_wait_vchnl_resp(struct i40iw_sc_dev *dev)
 }
 
 /**
+ * i40iw_cqp_cq_create_cmd - create a cq for the cqp
+ * @dev: device pointer
+ * @cq: pointer to created cq
+ */
+enum i40iw_status_code i40iw_cqp_cq_create_cmd(struct i40iw_sc_dev *dev,
+					       struct i40iw_sc_cq *cq)
+{
+	struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
+	struct i40iw_cqp *iwcqp = &iwdev->cqp;
+	struct i40iw_cqp_request *cqp_request;
+	struct cqp_commands_info *cqp_info;
+	enum i40iw_status_code status;
+
+	cqp_request = i40iw_get_cqp_request(iwcqp, true);
+	if (!cqp_request)
+		return I40IW_ERR_NO_MEMORY;
+
+	cqp_info = &cqp_request->info;
+	cqp_info->cqp_cmd = OP_CQ_CREATE;
+	cqp_info->post_sq = 1;
+	cqp_info->in.u.cq_create.cq = cq;
+	cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
+	status = i40iw_handle_cqp_op(iwdev, cqp_request);
+	if (status)
+		i40iw_pr_err("CQP-OP Create QP fail");
+
+	return status;
+}
+
+/**
+ * i40iw_cqp_qp_create_cmd - create a qp for the cqp
+ * @dev: device pointer
+ * @qp: pointer to created qp
+ */
+enum i40iw_status_code i40iw_cqp_qp_create_cmd(struct i40iw_sc_dev *dev,
+					       struct i40iw_sc_qp *qp)
+{
+	struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
+	struct i40iw_cqp *iwcqp = &iwdev->cqp;
+	struct i40iw_cqp_request *cqp_request;
+	struct cqp_commands_info *cqp_info;
+	struct i40iw_create_qp_info *qp_info;
+	enum i40iw_status_code status;
+
+	cqp_request = i40iw_get_cqp_request(iwcqp, true);
+	if (!cqp_request)
+		return I40IW_ERR_NO_MEMORY;
+
+	cqp_info = &cqp_request->info;
+	qp_info = &cqp_request->info.in.u.qp_create.info;
+
+	memset(qp_info, 0, sizeof(*qp_info));
+
+	qp_info->cq_num_valid = true;
+	qp_info->next_iwarp_state = I40IW_QP_STATE_RTS;
+
+	cqp_info->cqp_cmd = OP_QP_CREATE;
+	cqp_info->post_sq = 1;
+	cqp_info->in.u.qp_create.qp = qp;
+	cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
+	status = i40iw_handle_cqp_op(iwdev, cqp_request);
+	if (status)
+		i40iw_pr_err("CQP-OP QP create fail");
+	return status;
+}
+
+/**
+ * i40iw_cqp_cq_destroy_cmd - destroy the cqp cq
+ * @dev: device pointer
+ * @cq: pointer to cq
+ */
+void i40iw_cqp_cq_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq)
+{
+	struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
+
+	i40iw_cq_wq_destroy(iwdev, cq);
+}
+
+/**
+ * i40iw_cqp_qp_destroy_cmd - destroy the cqp
+ * @dev: device pointer
+ * @qp: pointer to qp
+ */
+void i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp)
+{
+	struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
+	struct i40iw_cqp *iwcqp = &iwdev->cqp;
+	struct i40iw_cqp_request *cqp_request;
+	struct cqp_commands_info *cqp_info;
+	enum i40iw_status_code status;
+
+	cqp_request = i40iw_get_cqp_request(iwcqp, true);
+	if (!cqp_request)
+		return;
+
+	cqp_info = &cqp_request->info;
+	memset(cqp_info, 0, sizeof(*cqp_info));
+
+	cqp_info->cqp_cmd = OP_QP_DESTROY;
+	cqp_info->post_sq = 1;
+	cqp_info->in.u.qp_destroy.qp = qp;
+	cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
+	cqp_info->in.u.qp_destroy.remove_hash_idx = true;
+	status = i40iw_handle_cqp_op(iwdev, cqp_request);
+	if (status)
+		i40iw_pr_err("CQP QP_DESTROY fail");
+}
+
+
+/**
  * i40iw_ieq_mpa_crc_ae - generate AE for crc error
  * @dev: hardware control device structure
  * @qp: hardware control qp
@@ -1208,7 +1375,7 @@ enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_in
 
 	buf->totallen = pkt_len + buf->maclen;
 
-	if (info->payload_len < buf->totallen - 4) {
+	if (info->payload_len < buf->totallen) {
 		i40iw_pr_err("payload_len = 0x%x totallen expected0x%x\n",
 			     info->payload_len, buf->totallen);
 		return I40IW_ERR_INVALID_SIZE;
@@ -1224,27 +1391,29 @@ enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_in
 
 /**
  * i40iw_hw_stats_timeout - Stats timer-handler which updates all HW stats
- * @dev: hardware control device structure
+ * @vsi: pointer to the vsi structure
  */
-static void i40iw_hw_stats_timeout(unsigned long dev)
+static void i40iw_hw_stats_timeout(unsigned long vsi)
 {
-	struct i40iw_sc_dev *pf_dev = (struct i40iw_sc_dev *)dev;
-	struct i40iw_dev_pestat *pf_devstat = &pf_dev->dev_pestat;
-	struct i40iw_dev_pestat *vf_devstat = NULL;
+	struct i40iw_sc_vsi *sc_vsi =  (struct i40iw_sc_vsi *)vsi;
+	struct i40iw_sc_dev *pf_dev = sc_vsi->dev;
+	struct i40iw_vsi_pestat *pf_devstat = sc_vsi->pestat;
+	struct i40iw_vsi_pestat *vf_devstat = NULL;
 	u16 iw_vf_idx;
 	unsigned long flags;
 
 	/*PF*/
-	pf_devstat->ops.iw_hw_stat_read_all(pf_devstat, &pf_devstat->hw_stats);
+	i40iw_hw_stats_read_all(pf_devstat, &pf_devstat->hw_stats);
+
 	for (iw_vf_idx = 0; iw_vf_idx < I40IW_MAX_PE_ENABLED_VF_COUNT; iw_vf_idx++) {
-		spin_lock_irqsave(&pf_devstat->stats_lock, flags);
+		spin_lock_irqsave(&pf_devstat->lock, flags);
 		if (pf_dev->vf_dev[iw_vf_idx]) {
 			if (pf_dev->vf_dev[iw_vf_idx]->stats_initialized) {
-				vf_devstat = &pf_dev->vf_dev[iw_vf_idx]->dev_pestat;
-				vf_devstat->ops.iw_hw_stat_read_all(vf_devstat, &vf_devstat->hw_stats);
+				vf_devstat = &pf_dev->vf_dev[iw_vf_idx]->pestat;
+				i40iw_hw_stats_read_all(vf_devstat, &vf_devstat->hw_stats);
 			}
 		}
-		spin_unlock_irqrestore(&pf_devstat->stats_lock, flags);
+		spin_unlock_irqrestore(&pf_devstat->lock, flags);
 	}
 
 	mod_timer(&pf_devstat->stats_timer,
@@ -1253,26 +1422,26 @@ static void i40iw_hw_stats_timeout(unsigned long dev)
 
 /**
  * i40iw_hw_stats_start_timer - Start periodic stats timer
- * @dev: hardware control device structure
+ * @vsi: pointer to the vsi structure
  */
-void i40iw_hw_stats_start_timer(struct i40iw_sc_dev *dev)
+void i40iw_hw_stats_start_timer(struct i40iw_sc_vsi *vsi)
 {
-	struct i40iw_dev_pestat *devstat = &dev->dev_pestat;
+	struct i40iw_vsi_pestat *devstat = vsi->pestat;
 
 	init_timer(&devstat->stats_timer);
 	devstat->stats_timer.function = i40iw_hw_stats_timeout;
-	devstat->stats_timer.data = (unsigned long)dev;
+	devstat->stats_timer.data = (unsigned long)vsi;
 	mod_timer(&devstat->stats_timer,
 		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
 }
 
 /**
- * i40iw_hw_stats_del_timer - Delete periodic stats timer
- * @dev: hardware control device structure
+ * i40iw_hw_stats_stop_timer - Delete periodic stats timer
+ * @vsi: pointer to the vsi structure
  */
-void i40iw_hw_stats_del_timer(struct i40iw_sc_dev *dev)
+void i40iw_hw_stats_stop_timer(struct i40iw_sc_vsi *vsi)
 {
-	struct i40iw_dev_pestat *devstat = &dev->dev_pestat;
+	struct i40iw_vsi_pestat *devstat = vsi->pestat;
 
 	del_timer_sync(&devstat->stats_timer);
 }
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
index 6329c971c22f..7368a50bbdaa 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
@@ -37,6 +37,7 @@
 #include <linux/random.h>
 #include <linux/highmem.h>
 #include <linux/time.h>
+#include <linux/hugetlb.h>
 #include <asm/byteorder.h>
 #include <net/ip.h>
 #include <rdma/ib_verbs.h>
@@ -67,13 +68,13 @@ static int i40iw_query_device(struct ib_device *ibdev,
 	props->vendor_part_id = iwdev->ldev->pcidev->device;
 	props->hw_ver = (u32)iwdev->sc_dev.hw_rev;
 	props->max_mr_size = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;
-	props->max_qp = iwdev->max_qp;
+	props->max_qp = iwdev->max_qp - iwdev->used_qps;
 	props->max_qp_wr = (I40IW_MAX_WQ_ENTRIES >> 2) - 1;
 	props->max_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
-	props->max_cq = iwdev->max_cq;
+	props->max_cq = iwdev->max_cq - iwdev->used_cqs;
 	props->max_cqe = iwdev->max_cqe;
-	props->max_mr = iwdev->max_mr;
-	props->max_pd = iwdev->max_pd;
+	props->max_mr = iwdev->max_mr - iwdev->used_mrs;
+	props->max_pd = iwdev->max_pd - iwdev->used_pds;
 	props->max_sge_rd = I40IW_MAX_SGE_RD;
 	props->max_qp_rd_atom = I40IW_MAX_IRD_SIZE;
 	props->max_qp_init_rd_atom = props->max_qp_rd_atom;
@@ -254,7 +255,6 @@ static void i40iw_alloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp
 {
 	struct i40iw_cqp_request *cqp_request;
 	struct cqp_commands_info *cqp_info;
-	struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 	enum i40iw_status_code status;
 
 	if (qp->push_idx != I40IW_INVALID_PUSH_PAGE_INDEX)
@@ -270,7 +270,7 @@ static void i40iw_alloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp
 	cqp_info->cqp_cmd = OP_MANAGE_PUSH_PAGE;
 	cqp_info->post_sq = 1;
 
-	cqp_info->in.u.manage_push_page.info.qs_handle = dev->qs_handle;
+	cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
 	cqp_info->in.u.manage_push_page.info.free_page = 0;
 	cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp;
 	cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
@@ -292,7 +292,6 @@ static void i40iw_dealloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_
 {
 	struct i40iw_cqp_request *cqp_request;
 	struct cqp_commands_info *cqp_info;
-	struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 	enum i40iw_status_code status;
 
 	if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX)
@@ -307,7 +306,7 @@ static void i40iw_dealloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_
 	cqp_info->post_sq = 1;
 
 	cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
-	cqp_info->in.u.manage_push_page.info.qs_handle = dev->qs_handle;
+	cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
 	cqp_info->in.u.manage_push_page.info.free_page = 1;
 	cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp;
 	cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
@@ -337,6 +336,9 @@ static struct ib_pd *i40iw_alloc_pd(struct ib_device *ibdev,
 	u32 pd_id = 0;
 	int err;
 
+	if (iwdev->closing)
+		return ERR_PTR(-ENODEV);
+
 	err = i40iw_alloc_resource(iwdev, iwdev->allocated_pds,
 				   iwdev->max_pd, &pd_id, &iwdev->next_pd);
 	if (err) {
@@ -602,6 +604,9 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd,
 	struct i40iwarp_offload_info *iwarp_info;
 	unsigned long flags;
 
+	if (iwdev->closing)
+		return ERR_PTR(-ENODEV);
+
 	if (init_attr->create_flags)
 		return ERR_PTR(-EINVAL);
 	if (init_attr->cap.max_inline_data > I40IW_MAX_INLINE_DATA_SIZE)
@@ -610,11 +615,15 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd,
 	if (init_attr->cap.max_send_sge > I40IW_MAX_WQ_FRAGMENT_COUNT)
 		init_attr->cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
 
+	if (init_attr->cap.max_recv_sge > I40IW_MAX_WQ_FRAGMENT_COUNT)
+		init_attr->cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
+
 	memset(&init_info, 0, sizeof(init_info));
 
 	sq_size = init_attr->cap.max_send_wr;
 	rq_size = init_attr->cap.max_recv_wr;
 
+	init_info.vsi = &iwdev->vsi;
 	init_info.qp_uk_init_info.sq_size = sq_size;
 	init_info.qp_uk_init_info.rq_size = rq_size;
 	init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge;
@@ -774,6 +783,7 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd,
 	iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;
 	iwdev->qp_table[qp_num] = iwqp;
 	i40iw_add_pdusecount(iwqp->iwpd);
+	i40iw_add_devusecount(iwdev);
 	if (ibpd->uobject && udata) {
 		memset(&uresp, 0, sizeof(uresp));
 		uresp.actual_sq_size = sq_size;
@@ -815,8 +825,9 @@ static int i40iw_query_qp(struct ib_qp *ibqp,
 	attr->qp_access_flags = 0;
 	attr->cap.max_send_wr = qp->qp_uk.sq_size;
 	attr->cap.max_recv_wr = qp->qp_uk.rq_size;
-	attr->cap.max_recv_sge = 1;
 	attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE;
+	attr->cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
+	attr->cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
 	init_attr->event_handler = iwqp->ibqp.event_handler;
 	init_attr->qp_context = iwqp->ibqp.qp_context;
 	init_attr->send_cq = iwqp->ibqp.send_cq;
@@ -884,6 +895,11 @@ int i40iw_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	spin_lock_irqsave(&iwqp->lock, flags);
 
 	if (attr_mask & IB_QP_STATE) {
+		if (iwdev->closing && attr->qp_state != IB_QPS_ERR) {
+			err = -EINVAL;
+			goto exit;
+		}
+
 		switch (attr->qp_state) {
 		case IB_QPS_INIT:
 		case IB_QPS_RTR:
@@ -944,7 +960,7 @@ int i40iw_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 				goto exit;
 			}
 			if (iwqp->sc_qp.term_flags)
-				del_timer(&iwqp->terminate_timer);
+				i40iw_terminate_del_timer(&iwqp->sc_qp);
 			info.next_iwarp_state = I40IW_QP_STATE_ERROR;
 			if ((iwqp->hw_tcp_state > I40IW_TCP_STATE_CLOSED) &&
 			    iwdev->iw_status &&
@@ -1037,11 +1053,11 @@ static void cq_free_resources(struct i40iw_device *iwdev, struct i40iw_cq *iwcq)
 }
 
 /**
- * cq_wq_destroy - send cq destroy cqp
+ * i40iw_cq_wq_destroy - send cq destroy cqp
  * @iwdev: iwarp device
  * @cq: hardware control cq
  */
-static void cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq)
+void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq)
 {
 	enum i40iw_status_code status;
 	struct i40iw_cqp_request *cqp_request;
@@ -1080,9 +1096,10 @@ static int i40iw_destroy_cq(struct ib_cq *ib_cq)
 	iwcq = to_iwcq(ib_cq);
 	iwdev = to_iwdev(ib_cq->device);
 	cq = &iwcq->sc_cq;
-	cq_wq_destroy(iwdev, cq);
+	i40iw_cq_wq_destroy(iwdev, cq);
 	cq_free_resources(iwdev, iwcq);
 	kfree(iwcq);
+	i40iw_rem_devusecount(iwdev);
 	return 0;
 }
 
@@ -1113,6 +1130,9 @@ static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev,
 	int err_code;
 	int entries = attr->cqe;
 
+	if (iwdev->closing)
+		return ERR_PTR(-ENODEV);
+
 	if (entries > iwdev->max_cqe)
 		return ERR_PTR(-EINVAL);
 
@@ -1137,7 +1157,8 @@ static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev,
 	ukinfo->cq_id = cq_num;
 	iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size;
 	info.ceqe_mask = 0;
-	info.ceq_id = 0;
+	if (attr->comp_vector < iwdev->ceqs_count)
+		info.ceq_id = attr->comp_vector;
 	info.ceq_id_valid = true;
 	info.ceqe_mask = 1;
 	info.type = I40IW_CQ_TYPE_IWARP;
@@ -1229,10 +1250,11 @@ static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev,
 		}
 	}
 
+	i40iw_add_devusecount(iwdev);
 	return (struct ib_cq *)iwcq;
 
 cq_destroy:
-	cq_wq_destroy(iwdev, cq);
+	i40iw_cq_wq_destroy(iwdev, cq);
 cq_free_resources:
 	cq_free_resources(iwdev, iwcq);
 error:
@@ -1266,6 +1288,7 @@ static void i40iw_free_stag(struct i40iw_device *iwdev, u32 stag)
 
 	stag_idx = (stag & iwdev->mr_stagmask) >> I40IW_CQPSQ_STAG_IDX_SHIFT;
 	i40iw_free_resource(iwdev, iwdev->allocated_mrs, stag_idx);
+	i40iw_rem_devusecount(iwdev);
 }
 
 /**
@@ -1296,19 +1319,18 @@ static u32 i40iw_create_stag(struct i40iw_device *iwdev)
 		stag = stag_index << I40IW_CQPSQ_STAG_IDX_SHIFT;
 		stag |= driver_key;
 		stag += (u32)consumer_key;
+		i40iw_add_devusecount(iwdev);
 	}
 	return stag;
 }
 
 /**
  * i40iw_next_pbl_addr - Get next pbl address
- * @palloc: Poiner to allocated pbles
  * @pbl: pointer to a pble
  * @pinfo: info pointer
  * @idx: index
  */
-static inline u64 *i40iw_next_pbl_addr(struct i40iw_pble_alloc *palloc,
-				       u64 *pbl,
+static inline u64 *i40iw_next_pbl_addr(u64 *pbl,
 				       struct i40iw_pble_info **pinfo,
 				       u32 *idx)
 {
@@ -1336,9 +1358,11 @@ static void i40iw_copy_user_pgaddrs(struct i40iw_mr *iwmr,
 	struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
 	struct i40iw_pble_info *pinfo;
 	struct scatterlist *sg;
+	u64 pg_addr = 0;
 	u32 idx = 0;
 
 	pinfo = (level == I40IW_LEVEL_1) ? NULL : palloc->level2.leaf;
+
 	pg_shift = ffs(region->page_size) - 1;
 	for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
 		chunk_pages = sg_dma_len(sg) >> pg_shift;
@@ -1346,17 +1370,96 @@ static void i40iw_copy_user_pgaddrs(struct i40iw_mr *iwmr,
 		    !iwpbl->qp_mr.sq_page)
 			iwpbl->qp_mr.sq_page = sg_page(sg);
 		for (i = 0; i < chunk_pages; i++) {
-			*pbl = cpu_to_le64(sg_dma_address(sg) + region->page_size * i);
-			pbl = i40iw_next_pbl_addr(palloc, pbl, &pinfo, &idx);
+			pg_addr = sg_dma_address(sg) + region->page_size * i;
+
+			if ((entry + i) == 0)
+				*pbl = cpu_to_le64(pg_addr & iwmr->page_msk);
+			else if (!(pg_addr & ~iwmr->page_msk))
+				*pbl = cpu_to_le64(pg_addr);
+			else
+				continue;
+			pbl = i40iw_next_pbl_addr(pbl, &pinfo, &idx);
+		}
+	}
+}
+
+/**
+ * i40iw_set_hugetlb_params - set MR pg size and mask to huge pg values.
+ * @addr: virtual address
+ * @iwmr: mr pointer for this memory registration
+ */
+static void i40iw_set_hugetlb_values(u64 addr, struct i40iw_mr *iwmr)
+{
+	struct vm_area_struct *vma;
+	struct hstate *h;
+
+	vma = find_vma(current->mm, addr);
+	if (vma && is_vm_hugetlb_page(vma)) {
+		h = hstate_vma(vma);
+		if (huge_page_size(h) == 0x200000) {
+			iwmr->page_size = huge_page_size(h);
+			iwmr->page_msk = huge_page_mask(h);
 		}
 	}
 }
 
 /**
+ * i40iw_check_mem_contiguous - check if pbls stored in arr are contiguous
+ * @arr: lvl1 pbl array
+ * @npages: page count
+ * pg_size: page size
+ *
+ */
+static bool i40iw_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size)
+{
+	u32 pg_idx;
+
+	for (pg_idx = 0; pg_idx < npages; pg_idx++) {
+		if ((*arr + (pg_size * pg_idx)) != arr[pg_idx])
+			return false;
+	}
+	return true;
+}
+
+/**
+ * i40iw_check_mr_contiguous - check if MR is physically contiguous
+ * @palloc: pbl allocation struct
+ * pg_size: page size
+ */
+static bool i40iw_check_mr_contiguous(struct i40iw_pble_alloc *palloc, u32 pg_size)
+{
+	struct i40iw_pble_level2 *lvl2 = &palloc->level2;
+	struct i40iw_pble_info *leaf = lvl2->leaf;
+	u64 *arr = NULL;
+	u64 *start_addr = NULL;
+	int i;
+	bool ret;
+
+	if (palloc->level == I40IW_LEVEL_1) {
+		arr = (u64 *)palloc->level1.addr;
+		ret = i40iw_check_mem_contiguous(arr, palloc->total_cnt, pg_size);
+		return ret;
+	}
+
+	start_addr = (u64 *)leaf->addr;
+
+	for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
+		arr = (u64 *)leaf->addr;
+		if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr)
+			return false;
+		ret = i40iw_check_mem_contiguous(arr, leaf->cnt, pg_size);
+		if (!ret)
+			return false;
+	}
+
+	return true;
+}
+
+/**
  * i40iw_setup_pbles - copy user pg address to pble's
  * @iwdev: iwarp device
  * @iwmr: mr pointer for this memory registration
- * @use_pbles: flag if to use pble's or memory (level 0)
+ * @use_pbles: flag if to use pble's
  */
 static int i40iw_setup_pbles(struct i40iw_device *iwdev,
 			     struct i40iw_mr *iwmr,
@@ -1369,9 +1472,6 @@ static int i40iw_setup_pbles(struct i40iw_device *iwdev,
 	enum i40iw_status_code status;
 	enum i40iw_pble_level level = I40IW_LEVEL_1;
 
-	if (!use_pbles && (iwmr->page_cnt > MAX_SAVE_PAGE_ADDRS))
-		return -ENOMEM;
-
 	if (use_pbles) {
 		mutex_lock(&iwdev->pbl_mutex);
 		status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);
@@ -1388,6 +1488,10 @@ static int i40iw_setup_pbles(struct i40iw_device *iwdev,
 	}
 
 	i40iw_copy_user_pgaddrs(iwmr, pbl, level);
+
+	if (use_pbles)
+		iwmr->pgaddrmem[0] = *pbl;
+
 	return 0;
 }
 
@@ -1409,14 +1513,18 @@ static int i40iw_handle_q_mem(struct i40iw_device *iwdev,
 	struct i40iw_cq_mr *cqmr = &iwpbl->cq_mr;
 	struct i40iw_hmc_pble *hmc_p;
 	u64 *arr = iwmr->pgaddrmem;
+	u32 pg_size;
 	int err;
 	int total;
+	bool ret = true;
 
 	total = req->sq_pages + req->rq_pages + req->cq_pages;
+	pg_size = iwmr->page_size;
 
 	err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);
 	if (err)
 		return err;
+
 	if (use_pbles && (palloc->level != I40IW_LEVEL_1)) {
 		i40iw_free_pble(iwdev->pble_rsrc, palloc);
 		iwpbl->pbl_allocated = false;
@@ -1425,26 +1533,44 @@ static int i40iw_handle_q_mem(struct i40iw_device *iwdev,
 
 	if (use_pbles)
 		arr = (u64 *)palloc->level1.addr;
-	if (req->reg_type == IW_MEMREG_TYPE_QP) {
+
+	if (iwmr->type == IW_MEMREG_TYPE_QP) {
 		hmc_p = &qpmr->sq_pbl;
 		qpmr->shadow = (dma_addr_t)arr[total];
+
 		if (use_pbles) {
+			ret = i40iw_check_mem_contiguous(arr, req->sq_pages, pg_size);
+			if (ret)
+				ret = i40iw_check_mem_contiguous(&arr[req->sq_pages], req->rq_pages, pg_size);
+		}
+
+		if (!ret) {
 			hmc_p->idx = palloc->level1.idx;
 			hmc_p = &qpmr->rq_pbl;
 			hmc_p->idx = palloc->level1.idx + req->sq_pages;
 		} else {
 			hmc_p->addr = arr[0];
 			hmc_p = &qpmr->rq_pbl;
-			hmc_p->addr = arr[1];
+			hmc_p->addr = arr[req->sq_pages];
 		}
 	} else {		/* CQ */
 		hmc_p = &cqmr->cq_pbl;
 		cqmr->shadow = (dma_addr_t)arr[total];
+
 		if (use_pbles)
+			ret = i40iw_check_mem_contiguous(arr, req->cq_pages, pg_size);
+
+		if (!ret)
 			hmc_p->idx = palloc->level1.idx;
 		else
 			hmc_p->addr = arr[0];
 	}
+
+	if (use_pbles && ret) {
+		i40iw_free_pble(iwdev->pble_rsrc, palloc);
+		iwpbl->pbl_allocated = false;
+	}
+
 	return err;
 }
 
@@ -1642,8 +1768,9 @@ static int i40iw_hwreg_mr(struct i40iw_device *iwdev,
 	stag_info->access_rights = access;
 	stag_info->pd_id = iwpd->sc_pd.pd_id;
 	stag_info->addr_type = I40IW_ADDR_TYPE_VA_BASED;
+	stag_info->page_size = iwmr->page_size;
 
-	if (iwmr->page_cnt > 1) {
+	if (iwpbl->pbl_allocated) {
 		if (palloc->level == I40IW_LEVEL_1) {
 			stag_info->first_pm_pbl_index = palloc->level1.idx;
 			stag_info->chunk_size = 1;
@@ -1699,6 +1826,11 @@ static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd,
 	bool use_pbles = false;
 	unsigned long flags;
 	int err = -ENOSYS;
+	int ret;
+	int pg_shift;
+
+	if (iwdev->closing)
+		return ERR_PTR(-ENODEV);
 
 	if (length > I40IW_MAX_MR_SIZE)
 		return ERR_PTR(-EINVAL);
@@ -1723,9 +1855,17 @@ static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd,
 	iwmr->ibmr.pd = pd;
 	iwmr->ibmr.device = pd->device;
 	ucontext = to_ucontext(pd->uobject->context);
-	region_length = region->length + (start & 0xfff);
-	pbl_depth = region_length >> 12;
-	pbl_depth += (region_length & (4096 - 1)) ? 1 : 0;
+
+	iwmr->page_size = region->page_size;
+	iwmr->page_msk = PAGE_MASK;
+
+	if (region->hugetlb && (req.reg_type == IW_MEMREG_TYPE_MEM))
+		i40iw_set_hugetlb_values(start, iwmr);
+
+	region_length = region->length + (start & (iwmr->page_size - 1));
+	pg_shift = ffs(iwmr->page_size) - 1;
+	pbl_depth = region_length >> pg_shift;
+	pbl_depth += (region_length & (iwmr->page_size - 1)) ? 1 : 0;
 	iwmr->length = region->length;
 
 	iwpbl->user_base = virt;
@@ -1755,13 +1895,21 @@ static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd,
 		spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
 		break;
 	case IW_MEMREG_TYPE_MEM:
+		use_pbles = (iwmr->page_cnt != 1);
 		access = I40IW_ACCESS_FLAGS_LOCALREAD;
 
-		use_pbles = (iwmr->page_cnt != 1);
 		err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);
 		if (err)
 			goto error;
 
+		if (use_pbles) {
+			ret = i40iw_check_mr_contiguous(palloc, iwmr->page_size);
+			if (ret) {
+				i40iw_free_pble(iwdev->pble_rsrc, palloc);
+				iwpbl->pbl_allocated = false;
+			}
+		}
+
 		access |= i40iw_get_user_access(acc);
 		stag = i40iw_create_stag(iwdev);
 		if (!stag) {
@@ -1778,6 +1926,7 @@ static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd,
 			i40iw_free_stag(iwdev, stag);
 			goto error;
 		}
+
 		break;
 	default:
 		goto error;
@@ -1789,7 +1938,7 @@ static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd,
 	return &iwmr->ibmr;
 
 error:
-	if (palloc->level != I40IW_LEVEL_0)
+	if (palloc->level != I40IW_LEVEL_0 && iwpbl->pbl_allocated)
 		i40iw_free_pble(iwdev->pble_rsrc, palloc);
 	ib_umem_release(region);
 	kfree(iwmr);
@@ -2142,7 +2291,6 @@ static int i40iw_post_send(struct ib_qp *ibqp,
 		case IB_WR_REG_MR:
 		{
 			struct i40iw_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr);
-			int page_shift = ilog2(reg_wr(ib_wr)->mr->page_size);
 			int flags = reg_wr(ib_wr)->access;
 			struct i40iw_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc;
 			struct i40iw_sc_dev *dev = &iwqp->iwdev->sc_dev;
@@ -2153,6 +2301,7 @@ static int i40iw_post_send(struct ib_qp *ibqp,
 			info.access_rights |= i40iw_get_user_access(flags);
 			info.stag_key = reg_wr(ib_wr)->key & 0xff;
 			info.stag_idx = reg_wr(ib_wr)->key >> 8;
+			info.page_size = reg_wr(ib_wr)->mr->page_size;
 			info.wr_id = ib_wr->wr_id;
 
 			info.addr_type = I40IW_ADDR_TYPE_VA_BASED;
@@ -2166,9 +2315,6 @@ static int i40iw_post_send(struct ib_qp *ibqp,
 			if (iwmr->npages > I40IW_MIN_PAGES_PER_FMR)
 				info.chunk_size = 1;
 
-			if (page_shift == 21)
-				info.page_size = 1; /* 2M page */
-
 			ret = dev->iw_priv_qp_ops->iw_mr_fast_register(&iwqp->sc_qp, &info, true);
 			if (ret)
 				err = -ENOMEM;
@@ -2487,21 +2633,17 @@ static int i40iw_get_hw_stats(struct ib_device *ibdev,
 {
 	struct i40iw_device *iwdev = to_iwdev(ibdev);
 	struct i40iw_sc_dev *dev = &iwdev->sc_dev;
-	struct i40iw_dev_pestat *devstat = &dev->dev_pestat;
+	struct i40iw_vsi_pestat *devstat = iwdev->vsi.pestat;
 	struct i40iw_dev_hw_stats *hw_stats = &devstat->hw_stats;
-	unsigned long flags;
 
 	if (dev->is_pf) {
-		spin_lock_irqsave(&devstat->stats_lock, flags);
-		devstat->ops.iw_hw_stat_read_all(devstat,
-			&devstat->hw_stats);
-		spin_unlock_irqrestore(&devstat->stats_lock, flags);
+		i40iw_hw_stats_read_all(devstat, &devstat->hw_stats);
 	} else {
 		if (i40iw_vchnl_vf_get_pe_stats(dev, &devstat->hw_stats))
 			return -ENOSYS;
 	}
 
-	memcpy(&stats->value[0], &hw_stats, sizeof(*hw_stats));
+	memcpy(&stats->value[0], hw_stats, sizeof(*hw_stats));
 
 	return stats->num_counters;
 }
@@ -2562,7 +2704,9 @@ static int i40iw_query_pkey(struct ib_device *ibdev,
  * @ah_attr: address handle attributes
  */
 static struct ib_ah *i40iw_create_ah(struct ib_pd *ibpd,
-				     struct ib_ah_attr *attr)
+				     struct ib_ah_attr *attr,
+				     struct ib_udata *udata)
+
 {
 	return ERR_PTR(-ENOSYS);
 }
@@ -2621,7 +2765,7 @@ static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev
 	    (1ull << IB_USER_VERBS_CMD_POST_RECV) |
 	    (1ull << IB_USER_VERBS_CMD_POST_SEND);
 	iwibdev->ibdev.phys_port_cnt = 1;
-	iwibdev->ibdev.num_comp_vectors = 1;
+	iwibdev->ibdev.num_comp_vectors = iwdev->ceqs_count;
 	iwibdev->ibdev.dma_device = &pcidev->dev;
 	iwibdev->ibdev.dev.parent = &pcidev->dev;
 	iwibdev->ibdev.query_port = i40iw_query_port;
@@ -2654,7 +2798,6 @@ static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev
 	iwibdev->ibdev.iwcm = kzalloc(sizeof(*iwibdev->ibdev.iwcm), GFP_KERNEL);
 	if (!iwibdev->ibdev.iwcm) {
 		ib_dealloc_device(&iwibdev->ibdev);
-		i40iw_pr_err("iwcm == NULL\n");
 		return NULL;
 	}
 
@@ -2719,6 +2862,9 @@ void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev)
 	i40iw_unregister_rdma_device(iwibdev);
 	kfree(iwibdev->ibdev.iwcm);
 	iwibdev->ibdev.iwcm = NULL;
+	wait_event_timeout(iwibdev->iwdev->close_wq,
+			   !atomic64_read(&iwibdev->iwdev->use_count),
+			   I40IW_EVENT_TIMEOUT);
 	ib_dealloc_device(&iwibdev->ibdev);
 }
 
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.h b/drivers/infiniband/hw/i40iw/i40iw_verbs.h
index 0069be8a5a38..6549c939500f 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.h
@@ -92,6 +92,8 @@ struct i40iw_mr {
 	struct ib_umem *region;
 	u16 type;
 	u32 page_cnt;
+	u32 page_size;
+	u64 page_msk;
 	u32 npages;
 	u32 stag;
 	u64 length;
diff --git a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c b/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c
index 3041003c94d2..f4d13683a403 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c
@@ -403,6 +403,19 @@ del_out:
 }
 
 /**
+ * i40iw_vf_init_pestat - Initialize stats for VF
+ * @devL pointer to the VF Device
+ * @stats: Statistics structure pointer
+ * @index: Stats index
+ */
+static void i40iw_vf_init_pestat(struct i40iw_sc_dev *dev, struct i40iw_vsi_pestat *stats, u16 index)
+{
+	stats->hw = dev->hw;
+	i40iw_hw_stats_init(stats, (u8)index, false);
+	spin_lock_init(&stats->lock);
+}
+
+/**
  * i40iw_vchnl_recv_pf - Receive PF virtual channel messages
  * @dev: IWARP device pointer
  * @vf_id: Virtual function ID associated with the message
@@ -421,9 +434,8 @@ enum i40iw_status_code i40iw_vchnl_recv_pf(struct i40iw_sc_dev *dev,
 	u16 first_avail_iw_vf = I40IW_MAX_PE_ENABLED_VF_COUNT;
 	struct i40iw_virt_mem vf_dev_mem;
 	struct i40iw_virtchnl_work_info work_info;
-	struct i40iw_dev_pestat *devstat;
+	struct i40iw_vsi_pestat *stats;
 	enum i40iw_status_code ret_code;
-	unsigned long flags;
 
 	if (!dev || !msg || !len)
 		return I40IW_ERR_PARAM;
@@ -496,14 +508,7 @@ enum i40iw_status_code i40iw_vchnl_recv_pf(struct i40iw_sc_dev *dev,
 				i40iw_debug(dev, I40IW_DEBUG_VIRT,
 					    "VF%u error CQP HMC Function operation.\n",
 					    vf_id);
-			ret_code = i40iw_device_init_pestat(&vf_dev->dev_pestat);
-			if (ret_code)
-				i40iw_debug(dev, I40IW_DEBUG_VIRT,
-					    "VF%u - i40iw_device_init_pestat failed\n",
-					    vf_id);
-			vf_dev->dev_pestat.ops.iw_hw_stat_init(&vf_dev->dev_pestat,
-							      (u8)vf_dev->pmf_index,
-							      dev->hw, false);
+			i40iw_vf_init_pestat(dev, &vf_dev->pestat, vf_dev->pmf_index);
 			vf_dev->stats_initialized = true;
 		} else {
 			if (vf_dev) {
@@ -534,12 +539,10 @@ enum i40iw_status_code i40iw_vchnl_recv_pf(struct i40iw_sc_dev *dev,
 	case I40IW_VCHNL_OP_GET_STATS:
 		if (!vf_dev)
 			return I40IW_ERR_BAD_PTR;
-		devstat = &vf_dev->dev_pestat;
-		spin_lock_irqsave(&dev->dev_pestat.stats_lock, flags);
-		devstat->ops.iw_hw_stat_read_all(devstat, &devstat->hw_stats);
-		spin_unlock_irqrestore(&dev->dev_pestat.stats_lock, flags);
+		stats = &vf_dev->pestat;
+		i40iw_hw_stats_read_all(stats, &stats->hw_stats);
 		vf_dev->msg_count--;
-		vchnl_pf_send_get_pe_stats_resp(dev, vf_id, vchnl_msg, &devstat->hw_stats);
+		vchnl_pf_send_get_pe_stats_resp(dev, vf_id, vchnl_msg, &stats->hw_stats);
 		break;
 	default:
 		i40iw_debug(dev, I40IW_DEBUG_VIRT,
diff --git a/drivers/infiniband/hw/mlx4/ah.c b/drivers/infiniband/hw/mlx4/ah.c
index b9bf0759f10a..077c33d2dc75 100644
--- a/drivers/infiniband/hw/mlx4/ah.c
+++ b/drivers/infiniband/hw/mlx4/ah.c
@@ -114,7 +114,9 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr
 		       !(1 << ah->av.eth.stat_rate & dev->caps.stat_rate_support))
 			--ah->av.eth.stat_rate;
 	}
-
+	ah->av.eth.sl_tclass_flowlabel |=
+			cpu_to_be32((ah_attr->grh.traffic_class << 20) |
+				    ah_attr->grh.flow_label);
 	/*
 	 * HW requires multicast LID so we just choose one.
 	 */
@@ -122,12 +124,14 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr
 		ah->av.ib.dlid = cpu_to_be16(0xc000);
 
 	memcpy(ah->av.eth.dgid, ah_attr->grh.dgid.raw, 16);
-	ah->av.eth.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 29);
+	ah->av.eth.sl_tclass_flowlabel |= cpu_to_be32(ah_attr->sl << 29);
 
 	return &ah->ibah;
 }
 
-struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
+struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
+				struct ib_udata *udata)
+
 {
 	struct mlx4_ib_ah *ah;
 	struct ib_ah *ret;
diff --git a/drivers/infiniband/hw/mlx4/alias_GUID.c b/drivers/infiniband/hw/mlx4/alias_GUID.c
index 5e9939045852..06020c54db20 100644
--- a/drivers/infiniband/hw/mlx4/alias_GUID.c
+++ b/drivers/infiniband/hw/mlx4/alias_GUID.c
@@ -755,10 +755,8 @@ static void alias_guid_work(struct work_struct *work)
 	struct mlx4_ib_dev *dev = container_of(ib_sriov, struct mlx4_ib_dev, sriov);
 
 	rec = kzalloc(sizeof *rec, GFP_KERNEL);
-	if (!rec) {
-		pr_err("alias_guid_work: No Memory\n");
+	if (!rec)
 		return;
-	}
 
 	pr_debug("starting [port: %d]...\n", sriov_alias_port->port + 1);
 	ret = get_next_record_to_update(dev, sriov_alias_port->port, rec);
diff --git a/drivers/infiniband/hw/mlx4/cm.c b/drivers/infiniband/hw/mlx4/cm.c
index 39a488889fc7..d64845335e87 100644
--- a/drivers/infiniband/hw/mlx4/cm.c
+++ b/drivers/infiniband/hw/mlx4/cm.c
@@ -247,10 +247,8 @@ id_map_alloc(struct ib_device *ibdev, int slave_id, u32 sl_cm_id)
 	struct mlx4_ib_sriov *sriov = &to_mdev(ibdev)->sriov;
 
 	ent = kmalloc(sizeof (struct id_map_entry), GFP_KERNEL);
-	if (!ent) {
-		mlx4_ib_warn(ibdev, "Couldn't allocate id cache entry - out of memory\n");
+	if (!ent)
 		return ERR_PTR(-ENOMEM);
-	}
 
 	ent->sl_cm_id = sl_cm_id;
 	ent->slave_id = slave_id;
diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c
index 1672907ff219..db564ccc0f92 100644
--- a/drivers/infiniband/hw/mlx4/mad.c
+++ b/drivers/infiniband/hw/mlx4/mad.c
@@ -39,6 +39,8 @@
 #include <linux/mlx4/cmd.h>
 #include <linux/gfp.h>
 #include <rdma/ib_pma.h>
+#include <linux/ip.h>
+#include <net/ipv6.h>
 
 #include <linux/mlx4/driver.h>
 #include "mlx4_ib.h"
@@ -480,6 +482,23 @@ static int find_slave_port_pkey_ix(struct mlx4_ib_dev *dev, int slave,
 	return -EINVAL;
 }
 
+static int get_gids_from_l3_hdr(struct ib_grh *grh, union ib_gid *sgid,
+				union ib_gid *dgid)
+{
+	int version = ib_get_rdma_header_version((const union rdma_network_hdr *)grh);
+	enum rdma_network_type net_type;
+
+	if (version == 4)
+		net_type = RDMA_NETWORK_IPV4;
+	else if (version == 6)
+		net_type = RDMA_NETWORK_IPV6;
+	else
+		return -EINVAL;
+
+	return ib_get_gids_from_rdma_hdr((union rdma_network_hdr *)grh, net_type,
+					 sgid, dgid);
+}
+
 int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,
 			  enum ib_qp_type dest_qpt, struct ib_wc *wc,
 			  struct ib_grh *grh, struct ib_mad *mad)
@@ -538,7 +557,10 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,
 	memset(&attr, 0, sizeof attr);
 	attr.port_num = port;
 	if (is_eth) {
-		memcpy(&attr.grh.dgid.raw[0], &grh->dgid.raw[0], 16);
+		union ib_gid sgid;
+
+		if (get_gids_from_l3_hdr(grh, &sgid, &attr.grh.dgid))
+			return -EINVAL;
 		attr.ah_flags = IB_AH_GRH;
 	}
 	ah = ib_create_ah(tun_ctx->pd, &attr);
@@ -651,6 +673,11 @@ static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port,
 		is_eth = 1;
 
 	if (is_eth) {
+		union ib_gid dgid;
+		union ib_gid sgid;
+
+		if (get_gids_from_l3_hdr(grh, &sgid, &dgid))
+			return -EINVAL;
 		if (!(wc->wc_flags & IB_WC_GRH)) {
 			mlx4_ib_warn(ibdev, "RoCE grh not present.\n");
 			return -EINVAL;
@@ -659,10 +686,10 @@ static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port,
 			mlx4_ib_warn(ibdev, "RoCE mgmt class is not CM\n");
 			return -EINVAL;
 		}
-		err = mlx4_get_slave_from_roce_gid(dev->dev, port, grh->dgid.raw, &slave);
+		err = mlx4_get_slave_from_roce_gid(dev->dev, port, dgid.raw, &slave);
 		if (err && mlx4_is_mf_bonded(dev->dev)) {
 			other_port = (port == 1) ? 2 : 1;
-			err = mlx4_get_slave_from_roce_gid(dev->dev, other_port, grh->dgid.raw, &slave);
+			err = mlx4_get_slave_from_roce_gid(dev->dev, other_port, dgid.raw, &slave);
 			if (!err) {
 				port = other_port;
 				pr_debug("resolved slave %d from gid %pI6 wire port %d other %d\n",
@@ -702,10 +729,18 @@ static int mlx4_ib_demux_mad(struct ib_device *ibdev, u8 port,
 
 	/* If a grh is present, we demux according to it */
 	if (wc->wc_flags & IB_WC_GRH) {
-		slave = mlx4_ib_find_real_gid(ibdev, port, grh->dgid.global.interface_id);
-		if (slave < 0) {
-			mlx4_ib_warn(ibdev, "failed matching grh\n");
-			return -ENOENT;
+		if (grh->dgid.global.interface_id ==
+			cpu_to_be64(IB_SA_WELL_KNOWN_GUID) &&
+		    grh->dgid.global.subnet_prefix == cpu_to_be64(
+			atomic64_read(&dev->sriov.demux[port - 1].subnet_prefix))) {
+			slave = 0;
+		} else {
+			slave = mlx4_ib_find_real_gid(ibdev, port,
+						      grh->dgid.global.interface_id);
+			if (slave < 0) {
+				mlx4_ib_warn(ibdev, "failed matching grh\n");
+				return -ENOENT;
+			}
 		}
 	}
 	/* Class-specific handling */
@@ -1102,10 +1137,8 @@ static void handle_slaves_guid_change(struct mlx4_ib_dev *dev, u8 port_num,
 
 	in_mad  = kmalloc(sizeof *in_mad, GFP_KERNEL);
 	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
-	if (!in_mad || !out_mad) {
-		mlx4_ib_warn(&dev->ib_dev, "failed to allocate memory for guid info mads\n");
+	if (!in_mad || !out_mad)
 		goto out;
-	}
 
 	guid_tbl_blk_num  *= 4;
 
@@ -1916,11 +1949,8 @@ static int alloc_pv_object(struct mlx4_ib_dev *dev, int slave, int port,
 
 	*ret_ctx = NULL;
 	ctx = kzalloc(sizeof (struct mlx4_ib_demux_pv_ctx), GFP_KERNEL);
-	if (!ctx) {
-		pr_err("failed allocating pv resource context "
-		       "for port %d, slave %d\n", port, slave);
+	if (!ctx)
 		return -ENOMEM;
-	}
 
 	ctx->ib_dev = &dev->ib_dev;
 	ctx->port = port;
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index b597e8227591..c8413fc120e6 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -430,7 +430,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
 	struct mlx4_ib_dev *dev = to_mdev(ibdev);
 	struct ib_smp *in_mad  = NULL;
 	struct ib_smp *out_mad = NULL;
-	int err = -ENOMEM;
+	int err;
 	int have_ib_ports;
 	struct mlx4_uverbs_ex_query_device cmd;
 	struct mlx4_uverbs_ex_query_device_resp resp = {.comp_mask = 0};
@@ -455,6 +455,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
 		sizeof(resp.response_length);
 	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
+	err = -ENOMEM;
 	if (!in_mad || !out_mad)
 		goto out;
 
@@ -547,6 +548,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
 	props->max_map_per_fmr = dev->dev->caps.max_fmr_maps;
 	props->hca_core_clock = dev->dev->caps.hca_core_clock * 1000UL;
 	props->timestamp_mask = 0xFFFFFFFFFFFFULL;
+	props->max_ah = INT_MAX;
 
 	if (!mlx4_is_slave(dev->dev))
 		err = mlx4_get_internal_clock_params(dev->dev, &clock_params);
@@ -697,9 +699,11 @@ static int eth_link_query_port(struct ib_device *ibdev, u8 port,
 	if (err)
 		goto out;
 
-	props->active_width	=  (((u8 *)mailbox->buf)[5] == 0x40) ?
-						IB_WIDTH_4X : IB_WIDTH_1X;
-	props->active_speed	= IB_SPEED_QDR;
+	props->active_width	=  (((u8 *)mailbox->buf)[5] == 0x40) ||
+				   (((u8 *)mailbox->buf)[5] == 0x20 /*56Gb*/) ?
+					   IB_WIDTH_4X : IB_WIDTH_1X;
+	props->active_speed	=  (((u8 *)mailbox->buf)[5] == 0x20 /*56Gb*/) ?
+					   IB_SPEED_FDR : IB_SPEED_QDR;
 	props->port_cap_flags	= IB_PORT_CM_SUP | IB_PORT_IP_BASED_GIDS;
 	props->gid_tbl_len	= mdev->dev->caps.gid_table_len[port];
 	props->max_msg_sz	= mdev->dev->caps.max_msg_sz;
@@ -2814,20 +2818,22 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
 			kmalloc(BITS_TO_LONGS(ibdev->steer_qpn_count) *
 				sizeof(long),
 				GFP_KERNEL);
-		if (!ibdev->ib_uc_qpns_bitmap) {
-			dev_err(&dev->persist->pdev->dev,
-				"bit map alloc failed\n");
+		if (!ibdev->ib_uc_qpns_bitmap)
 			goto err_steer_qp_release;
-		}
 
-		bitmap_zero(ibdev->ib_uc_qpns_bitmap, ibdev->steer_qpn_count);
-
-		err = mlx4_FLOW_STEERING_IB_UC_QP_RANGE(
-				dev, ibdev->steer_qpn_base,
-				ibdev->steer_qpn_base +
-				ibdev->steer_qpn_count - 1);
-		if (err)
-			goto err_steer_free_bitmap;
+		if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_DMFS_IPOIB) {
+			bitmap_zero(ibdev->ib_uc_qpns_bitmap,
+				    ibdev->steer_qpn_count);
+			err = mlx4_FLOW_STEERING_IB_UC_QP_RANGE(
+					dev, ibdev->steer_qpn_base,
+					ibdev->steer_qpn_base +
+					ibdev->steer_qpn_count - 1);
+			if (err)
+				goto err_steer_free_bitmap;
+		} else {
+			bitmap_fill(ibdev->ib_uc_qpns_bitmap,
+				    ibdev->steer_qpn_count);
+		}
 	}
 
 	for (j = 1; j <= ibdev->dev->caps.num_ports; j++)
@@ -3055,15 +3061,12 @@ static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init)
 	first_port = find_first_bit(actv_ports.ports, dev->caps.num_ports);
 
 	dm = kcalloc(ports, sizeof(*dm), GFP_ATOMIC);
-	if (!dm) {
-		pr_err("failed to allocate memory for tunneling qp update\n");
+	if (!dm)
 		return;
-	}
 
 	for (i = 0; i < ports; i++) {
 		dm[i] = kmalloc(sizeof (struct mlx4_ib_demux_work), GFP_ATOMIC);
 		if (!dm[i]) {
-			pr_err("failed to allocate memory for tunneling qp update work struct\n");
 			while (--i >= 0)
 				kfree(dm[i]);
 			goto out;
@@ -3223,8 +3226,6 @@ void mlx4_sched_ib_sl2vl_update_work(struct mlx4_ib_dev *ibdev,
 		ew->port = port;
 		ew->ib_dev = ibdev;
 		queue_work(wq, &ew->work);
-	} else {
-		pr_err("failed to allocate memory for sl2vl update work\n");
 	}
 }
 
@@ -3284,10 +3285,8 @@ static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,
 
 	case MLX4_DEV_EVENT_PORT_MGMT_CHANGE:
 		ew = kmalloc(sizeof *ew, GFP_ATOMIC);
-		if (!ew) {
-			pr_err("failed to allocate memory for events work\n");
+		if (!ew)
 			break;
-		}
 
 		INIT_WORK(&ew->work, handle_port_mgmt_change_event);
 		memcpy(&ew->ib_eqe, eqe, sizeof *eqe);
diff --git a/drivers/infiniband/hw/mlx4/mcg.c b/drivers/infiniband/hw/mlx4/mcg.c
index a21d37f02f35..e010fe459e67 100644
--- a/drivers/infiniband/hw/mlx4/mcg.c
+++ b/drivers/infiniband/hw/mlx4/mcg.c
@@ -1142,7 +1142,6 @@ void mlx4_ib_mcg_port_cleanup(struct mlx4_ib_demux_ctx *ctx, int destroy_wq)
 	work = kmalloc(sizeof *work, GFP_KERNEL);
 	if (!work) {
 		ctx->flushing = 0;
-		mcg_warn("failed allocating work for cleanup\n");
 		return;
 	}
 
@@ -1202,10 +1201,8 @@ static int push_deleteing_req(struct mcast_group *group, int slave)
 		return 0;
 
 	req = kzalloc(sizeof *req, GFP_KERNEL);
-	if (!req) {
-		mcg_warn_group(group, "failed allocation - may leave stall groups\n");
+	if (!req)
 		return -ENOMEM;
-	}
 
 	if (!list_empty(&group->func[slave].pending)) {
 		pend_req = list_entry(group->func[slave].pending.prev, struct mcast_req, group_list);
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 35141f451e5c..7f3d976d81ed 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -742,7 +742,8 @@ int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
 void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);
 void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);
 
-struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
+struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
+				struct ib_udata *udata);
 int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr);
 int mlx4_ib_destroy_ah(struct ib_ah *ah);
 
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 570bc866b1d6..c068add8838b 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -644,7 +644,7 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 	int qpn;
 	int err;
 	struct ib_qp_cap backup_cap;
-	struct mlx4_ib_sqp *sqp;
+	struct mlx4_ib_sqp *sqp = NULL;
 	struct mlx4_ib_qp *qp;
 	enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type;
 	struct mlx4_ib_cq *mcq;
@@ -933,7 +933,9 @@ err_db:
 		mlx4_db_free(dev->dev, &qp->db);
 
 err:
-	if (!*caller_qp)
+	if (sqp)
+		kfree(sqp);
+	else if (!*caller_qp)
 		kfree(qp);
 	return err;
 }
@@ -1280,7 +1282,8 @@ static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
 	if (is_qp0(dev, mqp))
 		mlx4_CLOSE_PORT(dev->dev, mqp->port);
 
-	if (dev->qp1_proxy[mqp->port - 1] == mqp) {
+	if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI &&
+	    dev->qp1_proxy[mqp->port - 1] == mqp) {
 		mutex_lock(&dev->qp1_proxy_lock[mqp->port - 1]);
 		dev->qp1_proxy[mqp->port - 1] = NULL;
 		mutex_unlock(&dev->qp1_proxy_lock[mqp->port - 1]);
@@ -1764,14 +1767,14 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 		u8 port_num = mlx4_is_bonded(to_mdev(ibqp->device)->dev) ? 1 :
 			attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
 		union ib_gid gid;
-		struct ib_gid_attr gid_attr;
+		struct ib_gid_attr gid_attr = {.gid_type = IB_GID_TYPE_IB};
 		u16 vlan = 0xffff;
 		u8 smac[ETH_ALEN];
 		int status = 0;
 		int is_eth = rdma_cap_eth_ah(&dev->ib_dev, port_num) &&
 			attr->ah_attr.ah_flags & IB_AH_GRH;
 
-		if (is_eth) {
+		if (is_eth && attr->ah_attr.ah_flags & IB_AH_GRH) {
 			int index = attr->ah_attr.grh.sgid_index;
 
 			status = ib_get_cached_gid(ibqp->device, port_num,
diff --git a/drivers/infiniband/hw/mlx5/ah.c b/drivers/infiniband/hw/mlx5/ah.c
index 745efa4cfc71..d090e96f6f01 100644
--- a/drivers/infiniband/hw/mlx5/ah.c
+++ b/drivers/infiniband/hw/mlx5/ah.c
@@ -64,7 +64,9 @@ static struct ib_ah *create_ib_ah(struct mlx5_ib_dev *dev,
 	return &ah->ibah;
 }
 
-struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
+struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
+				struct ib_udata *udata)
+
 {
 	struct mlx5_ib_ah *ah;
 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
@@ -75,6 +77,27 @@ struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
 	if (ll == IB_LINK_LAYER_ETHERNET && !(ah_attr->ah_flags & IB_AH_GRH))
 		return ERR_PTR(-EINVAL);
 
+	if (ll == IB_LINK_LAYER_ETHERNET && udata) {
+		int err;
+		struct mlx5_ib_create_ah_resp resp = {};
+		u32 min_resp_len = offsetof(typeof(resp), dmac) +
+				   sizeof(resp.dmac);
+
+		if (udata->outlen < min_resp_len)
+			return ERR_PTR(-EINVAL);
+
+		resp.response_length = min_resp_len;
+
+		err = ib_resolve_eth_dmac(pd->device, ah_attr);
+		if (err)
+			return ERR_PTR(err);
+
+		memcpy(resp.dmac, ah_attr->dmac, ETH_ALEN);
+		err = ib_copy_to_udata(udata, &resp, resp.response_length);
+		if (err)
+			return ERR_PTR(err);
+	}
+
 	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
 	if (!ah)
 		return ERR_PTR(-ENOMEM);
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index fcd04b881ec1..b3ef47c3ab73 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -731,7 +731,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 			  int entries, u32 **cqb,
 			  int *cqe_size, int *index, int *inlen)
 {
-	struct mlx5_ib_create_cq ucmd;
+	struct mlx5_ib_create_cq ucmd = {};
 	size_t ucmdlen;
 	int page_shift;
 	__be64 *pas;
@@ -770,7 +770,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 	if (err)
 		goto err_umem;
 
-	mlx5_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, &npages, &page_shift,
+	mlx5_ib_cont_pages(cq->buf.umem, ucmd.buf_addr, 0, &npages, &page_shift,
 			   &ncont, NULL);
 	mlx5_ib_dbg(dev, "addr 0x%llx, size %u, npages %d, page_shift %d, ncont %d\n",
 		    ucmd.buf_addr, entries * ucmd.cqe_size, npages, page_shift, ncont);
@@ -792,8 +792,36 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 
 	*index = to_mucontext(context)->uuari.uars[0].index;
 
+	if (ucmd.cqe_comp_en == 1) {
+		if (unlikely((*cqe_size != 64) ||
+			     !MLX5_CAP_GEN(dev->mdev, cqe_compression))) {
+			err = -EOPNOTSUPP;
+			mlx5_ib_warn(dev, "CQE compression is not supported for size %d!\n",
+				     *cqe_size);
+			goto err_cqb;
+		}
+
+		if (unlikely(!ucmd.cqe_comp_res_format ||
+			     !(ucmd.cqe_comp_res_format <
+			       MLX5_IB_CQE_RES_RESERVED) ||
+			     (ucmd.cqe_comp_res_format &
+			      (ucmd.cqe_comp_res_format - 1)))) {
+			err = -EOPNOTSUPP;
+			mlx5_ib_warn(dev, "CQE compression res format %d is not supported!\n",
+				     ucmd.cqe_comp_res_format);
+			goto err_cqb;
+		}
+
+		MLX5_SET(cqc, cqc, cqe_comp_en, 1);
+		MLX5_SET(cqc, cqc, mini_cqe_res_format,
+			 ilog2(ucmd.cqe_comp_res_format));
+	}
+
 	return 0;
 
+err_cqb:
+	kfree(cqb);
+
 err_db:
 	mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);
 
@@ -1124,7 +1152,7 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
 		return err;
 	}
 
-	mlx5_ib_cont_pages(umem, ucmd.buf_addr, &npages, page_shift,
+	mlx5_ib_cont_pages(umem, ucmd.buf_addr, 0, &npages, page_shift,
 			   npas, NULL);
 
 	cq->resize_umem = umem;
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 2be65ddf56ba..d566f6738833 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -127,7 +127,7 @@ static int mlx5_netdev_event(struct notifier_block *this,
 
 		if ((upper == ndev || (!upper && ndev == ibdev->roce.netdev))
 		    && ibdev->ib_active) {
-			struct ib_event ibev = {0};
+			struct ib_event ibev = { };
 
 			ibev.device = &ibdev->ib_dev;
 			ibev.event = (event == NETDEV_UP) ?
@@ -496,6 +496,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
 	struct mlx5_core_dev *mdev = dev->mdev;
 	int err = -ENOMEM;
+	int max_sq_desc;
 	int max_rq_sg;
 	int max_sq_sg;
 	u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
@@ -618,9 +619,10 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	props->max_qp_wr	   = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
 	max_rq_sg =  MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
 		     sizeof(struct mlx5_wqe_data_seg);
-	max_sq_sg = (MLX5_CAP_GEN(mdev, max_wqe_sz_sq) -
-		     sizeof(struct mlx5_wqe_ctrl_seg)) /
-		     sizeof(struct mlx5_wqe_data_seg);
+	max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
+	max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
+		     sizeof(struct mlx5_wqe_raddr_seg)) /
+		sizeof(struct mlx5_wqe_data_seg);
 	props->max_sge = min(max_rq_sg, max_sq_sg);
 	props->max_sge_rd	   = MLX5_MAX_SGE_RD;
 	props->max_cq		   = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
@@ -643,6 +645,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
 					   props->max_mcast_grp;
 	props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
+	props->max_ah = INT_MAX;
 	props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
 	props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
 
@@ -669,6 +672,40 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 			1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
 	}
 
+	if (field_avail(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes,
+			uhw->outlen)) {
+		resp.mlx5_ib_support_multi_pkt_send_wqes =
+			MLX5_CAP_ETH(mdev, multi_pkt_send_wqe);
+		resp.response_length +=
+			sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);
+	}
+
+	if (field_avail(typeof(resp), reserved, uhw->outlen))
+		resp.response_length += sizeof(resp.reserved);
+
+	if (field_avail(typeof(resp), cqe_comp_caps, uhw->outlen)) {
+		resp.cqe_comp_caps.max_num =
+			MLX5_CAP_GEN(dev->mdev, cqe_compression) ?
+			MLX5_CAP_GEN(dev->mdev, cqe_compression_max_num) : 0;
+		resp.cqe_comp_caps.supported_format =
+			MLX5_IB_CQE_RES_FORMAT_HASH |
+			MLX5_IB_CQE_RES_FORMAT_CSUM;
+		resp.response_length += sizeof(resp.cqe_comp_caps);
+	}
+
+	if (field_avail(typeof(resp), packet_pacing_caps, uhw->outlen)) {
+		if (MLX5_CAP_QOS(mdev, packet_pacing) &&
+		    MLX5_CAP_GEN(mdev, qos)) {
+			resp.packet_pacing_caps.qp_rate_limit_max =
+				MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
+			resp.packet_pacing_caps.qp_rate_limit_min =
+				MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
+			resp.packet_pacing_caps.supported_qpts |=
+				1 << IB_QPT_RAW_PACKET;
+		}
+		resp.response_length += sizeof(resp.packet_pacing_caps);
+	}
+
 	if (uhw->outlen) {
 		err = ib_copy_to_udata(uhw, &resp, resp.response_length);
 
@@ -1093,7 +1130,8 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 		resp.response_length += sizeof(resp.cqe_version);
 
 	if (field_avail(typeof(resp), cmds_supp_uhw, udata->outlen)) {
-		resp.cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE;
+		resp.cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
+				      MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
 		resp.response_length += sizeof(resp.cmds_supp_uhw);
 	}
 
@@ -1502,6 +1540,22 @@ static void set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
 	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
 }
 
+static void set_flow_label(void *misc_c, void *misc_v, u8 mask, u8 val,
+			   bool inner)
+{
+	if (inner) {
+		MLX5_SET(fte_match_set_misc,
+			 misc_c, inner_ipv6_flow_label, mask);
+		MLX5_SET(fte_match_set_misc,
+			 misc_v, inner_ipv6_flow_label, val);
+	} else {
+		MLX5_SET(fte_match_set_misc,
+			 misc_c, outer_ipv6_flow_label, mask);
+		MLX5_SET(fte_match_set_misc,
+			 misc_v, outer_ipv6_flow_label, val);
+	}
+}
+
 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
 {
 	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
@@ -1515,6 +1569,7 @@ static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
 #define LAST_IPV4_FIELD tos
 #define LAST_IPV6_FIELD traffic_class
 #define LAST_TCP_UDP_FIELD src_port
+#define LAST_TUNNEL_FIELD tunnel_id
 
 /* Field is the last supported field */
 #define FIELDS_NOT_SUPPORTED(filter, field)\
@@ -1527,155 +1582,164 @@ static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
 static int parse_flow_attr(u32 *match_c, u32 *match_v,
 			   const union ib_flow_spec *ib_spec)
 {
-	void *outer_headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
-					     outer_headers);
-	void *outer_headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
-					     outer_headers);
 	void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
 					   misc_parameters);
 	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
 					   misc_parameters);
+	void *headers_c;
+	void *headers_v;
+
+	if (ib_spec->type & IB_FLOW_SPEC_INNER) {
+		headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
+					 inner_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
+					 inner_headers);
+	} else {
+		headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
+					 outer_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
+					 outer_headers);
+	}
 
-	switch (ib_spec->type) {
+	switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
 	case IB_FLOW_SPEC_ETH:
 		if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
 			return -ENOTSUPP;
 
-		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
+		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 					     dmac_47_16),
 				ib_spec->eth.mask.dst_mac);
-		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
+		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 					     dmac_47_16),
 				ib_spec->eth.val.dst_mac);
 
-		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
+		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 					     smac_47_16),
 				ib_spec->eth.mask.src_mac);
-		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
+		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 					     smac_47_16),
 				ib_spec->eth.val.src_mac);
 
 		if (ib_spec->eth.mask.vlan_tag) {
-			MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
+			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 				 vlan_tag, 1);
-			MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
+			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 				 vlan_tag, 1);
 
-			MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
+			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 				 first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
-			MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
+			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 				 first_vid, ntohs(ib_spec->eth.val.vlan_tag));
 
-			MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
+			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 				 first_cfi,
 				 ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
-			MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
+			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 				 first_cfi,
 				 ntohs(ib_spec->eth.val.vlan_tag) >> 12);
 
-			MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
+			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 				 first_prio,
 				 ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
-			MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
+			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 				 first_prio,
 				 ntohs(ib_spec->eth.val.vlan_tag) >> 13);
 		}
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 			 ethertype, ntohs(ib_spec->eth.mask.ether_type));
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 			 ethertype, ntohs(ib_spec->eth.val.ether_type));
 		break;
 	case IB_FLOW_SPEC_IPV4:
 		if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
 			return -ENOTSUPP;
 
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 			 ethertype, 0xffff);
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 			 ethertype, ETH_P_IP);
 
-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 				    src_ipv4_src_ipv6.ipv4_layout.ipv4),
 		       &ib_spec->ipv4.mask.src_ip,
 		       sizeof(ib_spec->ipv4.mask.src_ip));
-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 				    src_ipv4_src_ipv6.ipv4_layout.ipv4),
 		       &ib_spec->ipv4.val.src_ip,
 		       sizeof(ib_spec->ipv4.val.src_ip));
-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 				    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 		       &ib_spec->ipv4.mask.dst_ip,
 		       sizeof(ib_spec->ipv4.mask.dst_ip));
-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 				    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 		       &ib_spec->ipv4.val.dst_ip,
 		       sizeof(ib_spec->ipv4.val.dst_ip));
 
-		set_tos(outer_headers_c, outer_headers_v,
+		set_tos(headers_c, headers_v,
 			ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
 
-		set_proto(outer_headers_c, outer_headers_v,
+		set_proto(headers_c, headers_v,
 			  ib_spec->ipv4.mask.proto, ib_spec->ipv4.val.proto);
 		break;
 	case IB_FLOW_SPEC_IPV6:
 		if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
 			return -ENOTSUPP;
 
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 			 ethertype, 0xffff);
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 			 ethertype, ETH_P_IPV6);
 
-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 				    src_ipv4_src_ipv6.ipv6_layout.ipv6),
 		       &ib_spec->ipv6.mask.src_ip,
 		       sizeof(ib_spec->ipv6.mask.src_ip));
-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 				    src_ipv4_src_ipv6.ipv6_layout.ipv6),
 		       &ib_spec->ipv6.val.src_ip,
 		       sizeof(ib_spec->ipv6.val.src_ip));
-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 		       &ib_spec->ipv6.mask.dst_ip,
 		       sizeof(ib_spec->ipv6.mask.dst_ip));
-		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
+		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 		       &ib_spec->ipv6.val.dst_ip,
 		       sizeof(ib_spec->ipv6.val.dst_ip));
 
-		set_tos(outer_headers_c, outer_headers_v,
+		set_tos(headers_c, headers_v,
 			ib_spec->ipv6.mask.traffic_class,
 			ib_spec->ipv6.val.traffic_class);
 
-		set_proto(outer_headers_c, outer_headers_v,
+		set_proto(headers_c, headers_v,
 			  ib_spec->ipv6.mask.next_hdr,
 			  ib_spec->ipv6.val.next_hdr);
 
-		MLX5_SET(fte_match_set_misc, misc_params_c,
-			 outer_ipv6_flow_label,
-			 ntohl(ib_spec->ipv6.mask.flow_label));
-		MLX5_SET(fte_match_set_misc, misc_params_v,
-			 outer_ipv6_flow_label,
-			 ntohl(ib_spec->ipv6.val.flow_label));
+		set_flow_label(misc_params_c, misc_params_v,
+			       ntohl(ib_spec->ipv6.mask.flow_label),
+			       ntohl(ib_spec->ipv6.val.flow_label),
+			       ib_spec->type & IB_FLOW_SPEC_INNER);
+
 		break;
 	case IB_FLOW_SPEC_TCP:
 		if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
 					 LAST_TCP_UDP_FIELD))
 			return -ENOTSUPP;
 
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
 			 0xff);
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v, ip_protocol,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 			 IPPROTO_TCP);
 
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
 			 ntohs(ib_spec->tcp_udp.mask.src_port));
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v, tcp_sport,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
 			 ntohs(ib_spec->tcp_udp.val.src_port));
 
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
 			 ntohs(ib_spec->tcp_udp.mask.dst_port));
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v, tcp_dport,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
 			 ntohs(ib_spec->tcp_udp.val.dst_port));
 		break;
 	case IB_FLOW_SPEC_UDP:
@@ -1683,21 +1747,31 @@ static int parse_flow_attr(u32 *match_c, u32 *match_v,
 					 LAST_TCP_UDP_FIELD))
 			return -ENOTSUPP;
 
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
 			 0xff);
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v, ip_protocol,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 			 IPPROTO_UDP);
 
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, udp_sport,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
 			 ntohs(ib_spec->tcp_udp.mask.src_port));
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v, udp_sport,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
 			 ntohs(ib_spec->tcp_udp.val.src_port));
 
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, udp_dport,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
 			 ntohs(ib_spec->tcp_udp.mask.dst_port));
-		MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v, udp_dport,
+		MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
 			 ntohs(ib_spec->tcp_udp.val.dst_port));
 		break;
+	case IB_FLOW_SPEC_VXLAN_TUNNEL:
+		if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
+					 LAST_TUNNEL_FIELD))
+			return -ENOTSUPP;
+
+		MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
+			 ntohl(ib_spec->tunnel.mask.tunnel_id));
+		MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
+			 ntohl(ib_spec->tunnel.val.tunnel_id));
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -2721,6 +2795,8 @@ static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
 			       struct ib_port_immutable *immutable)
 {
 	struct ib_port_attr attr;
+	struct mlx5_ib_dev *dev = to_mdev(ibdev);
+	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
 	int err;
 
 	err = mlx5_ib_query_port(ibdev, port_num, &attr);
@@ -2730,7 +2806,8 @@ static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
 	immutable->pkey_tbl_len = attr.pkey_tbl_len;
 	immutable->gid_tbl_len = attr.gid_tbl_len;
 	immutable->core_cap_flags = get_core_cap_flags(ibdev);
-	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+	if ((ll == IB_LINK_LAYER_INFINIBAND) || MLX5_CAP_GEN(dev->mdev, roce))
+		immutable->max_mad_size = IB_MGMT_MAD_SIZE;
 
 	return 0;
 }
@@ -2744,7 +2821,7 @@ static void get_dev_fw_str(struct ib_device *ibdev, char *str,
 		       fw_rev_min(dev->mdev), fw_rev_sub(dev->mdev));
 }
 
-static int mlx5_roce_lag_init(struct mlx5_ib_dev *dev)
+static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
 {
 	struct mlx5_core_dev *mdev = dev->mdev;
 	struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
@@ -2773,7 +2850,7 @@ err_destroy_vport_lag:
 	return err;
 }
 
-static void mlx5_roce_lag_cleanup(struct mlx5_ib_dev *dev)
+static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
 {
 	struct mlx5_core_dev *mdev = dev->mdev;
 
@@ -2785,7 +2862,21 @@ static void mlx5_roce_lag_cleanup(struct mlx5_ib_dev *dev)
 	}
 }
 
-static void mlx5_remove_roce_notifier(struct mlx5_ib_dev *dev)
+static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev)
+{
+	int err;
+
+	dev->roce.nb.notifier_call = mlx5_netdev_event;
+	err = register_netdevice_notifier(&dev->roce.nb);
+	if (err) {
+		dev->roce.nb.notifier_call = NULL;
+		return err;
+	}
+
+	return 0;
+}
+
+static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev)
 {
 	if (dev->roce.nb.notifier_call) {
 		unregister_netdevice_notifier(&dev->roce.nb);
@@ -2793,39 +2884,40 @@ static void mlx5_remove_roce_notifier(struct mlx5_ib_dev *dev)
 	}
 }
 
-static int mlx5_enable_roce(struct mlx5_ib_dev *dev)
+static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
 {
 	int err;
 
-	dev->roce.nb.notifier_call = mlx5_netdev_event;
-	err = register_netdevice_notifier(&dev->roce.nb);
-	if (err) {
-		dev->roce.nb.notifier_call = NULL;
+	err = mlx5_add_netdev_notifier(dev);
+	if (err)
 		return err;
-	}
 
-	err = mlx5_nic_vport_enable_roce(dev->mdev);
-	if (err)
-		goto err_unregister_netdevice_notifier;
+	if (MLX5_CAP_GEN(dev->mdev, roce)) {
+		err = mlx5_nic_vport_enable_roce(dev->mdev);
+		if (err)
+			goto err_unregister_netdevice_notifier;
+	}
 
-	err = mlx5_roce_lag_init(dev);
+	err = mlx5_eth_lag_init(dev);
 	if (err)
 		goto err_disable_roce;
 
 	return 0;
 
 err_disable_roce:
-	mlx5_nic_vport_disable_roce(dev->mdev);
+	if (MLX5_CAP_GEN(dev->mdev, roce))
+		mlx5_nic_vport_disable_roce(dev->mdev);
 
 err_unregister_netdevice_notifier:
-	mlx5_remove_roce_notifier(dev);
+	mlx5_remove_netdev_notifier(dev);
 	return err;
 }
 
-static void mlx5_disable_roce(struct mlx5_ib_dev *dev)
+static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
 {
-	mlx5_roce_lag_cleanup(dev);
-	mlx5_nic_vport_disable_roce(dev->mdev);
+	mlx5_eth_lag_cleanup(dev);
+	if (MLX5_CAP_GEN(dev->mdev, roce))
+		mlx5_nic_vport_disable_roce(dev->mdev);
 }
 
 static void mlx5_ib_dealloc_q_counters(struct mlx5_ib_dev *dev)
@@ -2947,9 +3039,6 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
 
-	if ((ll == IB_LINK_LAYER_ETHERNET) && !MLX5_CAP_GEN(mdev, roce))
-		return NULL;
-
 	printk_once(KERN_INFO "%s", mlx5_version);
 
 	dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
@@ -2995,6 +3084,8 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 		(1ull << IB_USER_VERBS_CMD_QUERY_PORT)		|
 		(1ull << IB_USER_VERBS_CMD_ALLOC_PD)		|
 		(1ull << IB_USER_VERBS_CMD_DEALLOC_PD)		|
+		(1ull << IB_USER_VERBS_CMD_CREATE_AH)		|
+		(1ull << IB_USER_VERBS_CMD_DESTROY_AH)		|
 		(1ull << IB_USER_VERBS_CMD_REG_MR)		|
 		(1ull << IB_USER_VERBS_CMD_REREG_MR)		|
 		(1ull << IB_USER_VERBS_CMD_DEREG_MR)		|
@@ -3017,7 +3108,8 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 	dev->ib_dev.uverbs_ex_cmd_mask =
 		(1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE)	|
 		(1ull << IB_USER_VERBS_EX_CMD_CREATE_CQ)	|
-		(1ull << IB_USER_VERBS_EX_CMD_CREATE_QP);
+		(1ull << IB_USER_VERBS_EX_CMD_CREATE_QP)	|
+		(1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP);
 
 	dev->ib_dev.query_device	= mlx5_ib_query_device;
 	dev->ib_dev.query_port		= mlx5_ib_query_port;
@@ -3128,14 +3220,14 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 	spin_lock_init(&dev->reset_flow_resource_lock);
 
 	if (ll == IB_LINK_LAYER_ETHERNET) {
-		err = mlx5_enable_roce(dev);
+		err = mlx5_enable_eth(dev);
 		if (err)
 			goto err_free_port;
 	}
 
 	err = create_dev_resources(&dev->devr);
 	if (err)
-		goto err_disable_roce;
+		goto err_disable_eth;
 
 	err = mlx5_ib_odp_init_one(dev);
 	if (err)
@@ -3179,10 +3271,10 @@ err_odp:
 err_rsrc:
 	destroy_dev_resources(&dev->devr);
 
-err_disable_roce:
+err_disable_eth:
 	if (ll == IB_LINK_LAYER_ETHERNET) {
-		mlx5_disable_roce(dev);
-		mlx5_remove_roce_notifier(dev);
+		mlx5_disable_eth(dev);
+		mlx5_remove_netdev_notifier(dev);
 	}
 
 err_free_port:
@@ -3199,14 +3291,14 @@ static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
 	struct mlx5_ib_dev *dev = context;
 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev, 1);
 
-	mlx5_remove_roce_notifier(dev);
+	mlx5_remove_netdev_notifier(dev);
 	ib_unregister_device(&dev->ib_dev);
 	mlx5_ib_dealloc_q_counters(dev);
 	destroy_umrc_res(dev);
 	mlx5_ib_odp_remove_one(dev);
 	destroy_dev_resources(&dev->devr);
 	if (ll == IB_LINK_LAYER_ETHERNET)
-		mlx5_disable_roce(dev);
+		mlx5_disable_eth(dev);
 	kfree(dev->port);
 	ib_dealloc_device(&dev->ib_dev);
 }
diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c
index 996b54e366b0..6851357c16f4 100644
--- a/drivers/infiniband/hw/mlx5/mem.c
+++ b/drivers/infiniband/hw/mlx5/mem.c
@@ -37,12 +37,15 @@
 
 /* @umem: umem object to scan
  * @addr: ib virtual address requested by the user
+ * @max_page_shift: high limit for page_shift - 0 means no limit
  * @count: number of PAGE_SIZE pages covered by umem
  * @shift: page shift for the compound pages found in the region
  * @ncont: number of compund pages
  * @order: log2 of the number of compound pages
  */
-void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
+void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
+			unsigned long max_page_shift,
+			int *count, int *shift,
 			int *ncont, int *order)
 {
 	unsigned long tmp;
@@ -72,6 +75,8 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
 	addr = addr >> page_shift;
 	tmp = (unsigned long)addr;
 	m = find_first_bit(&tmp, BITS_PER_LONG);
+	if (max_page_shift)
+		m = min_t(unsigned long, max_page_shift - page_shift, m);
 	skip = 1 << m;
 	mask = skip - 1;
 	i = 0;
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 854748b61212..6c6057eb60ea 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -63,6 +63,8 @@ pr_warn("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__,	\
 #define MLX5_IB_DEFAULT_UIDX 0xffffff
 #define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
 
+#define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size)
+
 enum {
 	MLX5_IB_MMAP_CMD_SHIFT	= 8,
 	MLX5_IB_MMAP_CMD_MASK	= 0xff,
@@ -387,6 +389,7 @@ struct mlx5_ib_qp {
 	struct list_head	qps_list;
 	struct list_head	cq_recv_list;
 	struct list_head	cq_send_list;
+	u32			rate_limit;
 };
 
 struct mlx5_ib_cq_buf {
@@ -418,7 +421,7 @@ struct mlx5_umr_wr {
 	struct ib_pd		       *pd;
 	unsigned int			page_shift;
 	unsigned int			npages;
-	u32				length;
+	u64				length;
 	int				access_flags;
 	u32				mkey;
 };
@@ -739,7 +742,8 @@ void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);
 int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, int ignore_bkey,
 		 u8 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh,
 		 const void *in_mad, void *response_mad);
-struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
+struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
+				struct ib_udata *udata);
 int mlx5_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr);
 int mlx5_ib_destroy_ah(struct ib_ah *ah);
 struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
@@ -825,7 +829,9 @@ int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
 		       struct ib_port_attr *props);
 int mlx5_ib_init_fmr(struct mlx5_ib_dev *dev);
 void mlx5_ib_cleanup_fmr(struct mlx5_ib_dev *dev);
-void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
+void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
+			unsigned long max_page_shift,
+			int *count, int *shift,
 			int *ncont, int *order);
 void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
 			    int page_shift, size_t offset, size_t num_pages,
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 4e9012463c37..8f608debe141 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -628,7 +628,8 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
 		ent->order = i + 2;
 		ent->dev = dev;
 
-		if (dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE)
+		if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) &&
+		    (mlx5_core_is_pf(dev->mdev)))
 			limit = dev->mdev->profile->mr_cache[i].limit;
 		else
 			limit = 0;
@@ -646,6 +647,33 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
 	return 0;
 }
 
+static void wait_for_async_commands(struct mlx5_ib_dev *dev)
+{
+	struct mlx5_mr_cache *cache = &dev->cache;
+	struct mlx5_cache_ent *ent;
+	int total = 0;
+	int i;
+	int j;
+
+	for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
+		ent = &cache->ent[i];
+		for (j = 0 ; j < 1000; j++) {
+			if (!ent->pending)
+				break;
+			msleep(50);
+		}
+	}
+	for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
+		ent = &cache->ent[i];
+		total += ent->pending;
+	}
+
+	if (total)
+		mlx5_ib_warn(dev, "aborted while there are %d pending mr requests\n", total);
+	else
+		mlx5_ib_warn(dev, "done with all pending requests\n");
+}
+
 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
 {
 	int i;
@@ -659,6 +687,7 @@ int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
 		clean_keys(dev, i);
 
 	destroy_workqueue(dev->cache.wq);
+	wait_for_async_commands(dev);
 	del_timer_sync(&dev->delay_timer);
 
 	return 0;
@@ -816,29 +845,34 @@ static void prep_umr_unreg_wqe(struct mlx5_ib_dev *dev,
 	umrwr->mkey = key;
 }
 
-static struct ib_umem *mr_umem_get(struct ib_pd *pd, u64 start, u64 length,
-				   int access_flags, int *npages,
-				   int *page_shift, int *ncont, int *order)
+static int mr_umem_get(struct ib_pd *pd, u64 start, u64 length,
+		       int access_flags, struct ib_umem **umem,
+		       int *npages, int *page_shift, int *ncont,
+		       int *order)
 {
 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
-	struct ib_umem *umem = ib_umem_get(pd->uobject->context, start, length,
-					   access_flags, 0);
-	if (IS_ERR(umem)) {
+	int err;
+
+	*umem = ib_umem_get(pd->uobject->context, start, length,
+			    access_flags, 0);
+	err = PTR_ERR_OR_ZERO(*umem);
+	if (err < 0) {
 		mlx5_ib_err(dev, "umem get failed (%ld)\n", PTR_ERR(umem));
-		return (void *)umem;
+		return err;
 	}
 
-	mlx5_ib_cont_pages(umem, start, npages, page_shift, ncont, order);
+	mlx5_ib_cont_pages(*umem, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages,
+			   page_shift, ncont, order);
 	if (!*npages) {
 		mlx5_ib_warn(dev, "avoid zero region\n");
-		ib_umem_release(umem);
-		return ERR_PTR(-EINVAL);
+		ib_umem_release(*umem);
+		return -EINVAL;
 	}
 
 	mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
 		    *npages, *ncont, *order, *page_shift);
 
-	return umem;
+	return 0;
 }
 
 static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)
@@ -1164,11 +1198,11 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 
 	mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
 		    start, virt_addr, length, access_flags);
-	umem = mr_umem_get(pd, start, length, access_flags, &npages,
+	err = mr_umem_get(pd, start, length, access_flags, &umem, &npages,
 			   &page_shift, &ncont, &order);
 
-	if (IS_ERR(umem))
-		return (void *)umem;
+        if (err < 0)
+		return ERR_PTR(err);
 
 	if (use_umr(order)) {
 		mr = reg_umr(pd, umem, virt_addr, length, ncont, page_shift,
@@ -1345,10 +1379,9 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
 		 */
 		flags |= IB_MR_REREG_TRANS;
 		ib_umem_release(mr->umem);
-		mr->umem = mr_umem_get(pd, addr, len, access_flags, &npages,
-				       &page_shift, &ncont, &order);
-		if (IS_ERR(mr->umem)) {
-			err = PTR_ERR(mr->umem);
+		err = mr_umem_get(pd, addr, len, access_flags, &mr->umem,
+				  &npages, &page_shift, &ncont, &order);
+		if (err < 0) {
 			mr->umem = NULL;
 			return err;
 		}
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index d1e921816bfe..a1b3125f0a6e 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -77,12 +77,14 @@ struct mlx5_wqe_eth_pad {
 
 enum raw_qp_set_mask_map {
 	MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID		= 1UL << 0,
+	MLX5_RAW_QP_RATE_LIMIT			= 1UL << 1,
 };
 
 struct mlx5_modify_raw_qp_param {
 	u16 operation;
 
 	u32 set_mask; /* raw_qp_set_mask_map */
+	u32 rate_limit;
 	u8 rq_q_ctr_id;
 };
 
@@ -351,6 +353,29 @@ static int calc_send_wqe(struct ib_qp_init_attr *attr)
 		return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
 }
 
+static int get_send_sge(struct ib_qp_init_attr *attr, int wqe_size)
+{
+	int max_sge;
+
+	if (attr->qp_type == IB_QPT_RC)
+		max_sge = (min_t(int, wqe_size, 512) -
+			   sizeof(struct mlx5_wqe_ctrl_seg) -
+			   sizeof(struct mlx5_wqe_raddr_seg)) /
+			sizeof(struct mlx5_wqe_data_seg);
+	else if (attr->qp_type == IB_QPT_XRC_INI)
+		max_sge = (min_t(int, wqe_size, 512) -
+			   sizeof(struct mlx5_wqe_ctrl_seg) -
+			   sizeof(struct mlx5_wqe_xrc_seg) -
+			   sizeof(struct mlx5_wqe_raddr_seg)) /
+			sizeof(struct mlx5_wqe_data_seg);
+	else
+		max_sge = (wqe_size - sq_overhead(attr)) /
+			sizeof(struct mlx5_wqe_data_seg);
+
+	return min_t(int, max_sge, wqe_size - sq_overhead(attr) /
+		     sizeof(struct mlx5_wqe_data_seg));
+}
+
 static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
 			struct mlx5_ib_qp *qp)
 {
@@ -381,13 +406,18 @@ static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
 	wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
 	qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
 	if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
-		mlx5_ib_dbg(dev, "wqe count(%d) exceeds limits(%d)\n",
+		mlx5_ib_dbg(dev, "send queue size (%d * %d / %d -> %d) exceeds limits(%d)\n",
+			    attr->cap.max_send_wr, wqe_size, MLX5_SEND_WQE_BB,
 			    qp->sq.wqe_cnt,
 			    1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
 		return -ENOMEM;
 	}
 	qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
-	qp->sq.max_gs = attr->cap.max_send_sge;
+	qp->sq.max_gs = get_send_sge(attr, wqe_size);
+	if (qp->sq.max_gs < attr->cap.max_send_sge)
+		return -ENOMEM;
+
+	attr->cap.max_send_sge = qp->sq.max_gs;
 	qp->sq.max_post = wq_size / wqe_size;
 	attr->cap.max_send_wr = qp->sq.max_post;
 
@@ -647,7 +677,7 @@ static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev,
 		return PTR_ERR(*umem);
 	}
 
-	mlx5_ib_cont_pages(*umem, addr, npages, page_shift, ncont, NULL);
+	mlx5_ib_cont_pages(*umem, addr, 0, npages, page_shift, ncont, NULL);
 
 	err = mlx5_ib_get_buf_offset(addr, *page_shift, offset);
 	if (err) {
@@ -700,7 +730,7 @@ static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		return err;
 	}
 
-	mlx5_ib_cont_pages(rwq->umem, ucmd->buf_addr, &npages, &page_shift,
+	mlx5_ib_cont_pages(rwq->umem, ucmd->buf_addr, 0, &npages, &page_shift,
 			   &ncont, NULL);
 	err = mlx5_ib_get_buf_offset(ucmd->buf_addr, page_shift,
 				     &rwq->rq_page_offset);
@@ -2442,8 +2472,14 @@ out:
 }
 
 static int modify_raw_packet_qp_sq(struct mlx5_core_dev *dev,
-				   struct mlx5_ib_sq *sq, int new_state)
+				   struct mlx5_ib_sq *sq,
+				   int new_state,
+				   const struct mlx5_modify_raw_qp_param *raw_qp_param)
 {
+	struct mlx5_ib_qp *ibqp = sq->base.container_mibqp;
+	u32 old_rate = ibqp->rate_limit;
+	u32 new_rate = old_rate;
+	u16 rl_index = 0;
 	void *in;
 	void *sqc;
 	int inlen;
@@ -2459,10 +2495,44 @@ static int modify_raw_packet_qp_sq(struct mlx5_core_dev *dev,
 	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
 	MLX5_SET(sqc, sqc, state, new_state);
 
+	if (raw_qp_param->set_mask & MLX5_RAW_QP_RATE_LIMIT) {
+		if (new_state != MLX5_SQC_STATE_RDY)
+			pr_warn("%s: Rate limit can only be changed when SQ is moving to RDY\n",
+				__func__);
+		else
+			new_rate = raw_qp_param->rate_limit;
+	}
+
+	if (old_rate != new_rate) {
+		if (new_rate) {
+			err = mlx5_rl_add_rate(dev, new_rate, &rl_index);
+			if (err) {
+				pr_err("Failed configuring rate %u: %d\n",
+				       new_rate, err);
+				goto out;
+			}
+		}
+
+		MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
+		MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);
+	}
+
 	err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in, inlen);
-	if (err)
+	if (err) {
+		/* Remove new rate from table if failed */
+		if (new_rate &&
+		    old_rate != new_rate)
+			mlx5_rl_remove_rate(dev, new_rate);
 		goto out;
+	}
+
+	/* Only remove the old rate after new rate was set */
+	if ((old_rate &&
+	    (old_rate != new_rate)) ||
+	    (new_state != MLX5_SQC_STATE_RDY))
+		mlx5_rl_remove_rate(dev, old_rate);
 
+	ibqp->rate_limit = new_rate;
 	sq->state = new_state;
 
 out:
@@ -2477,6 +2547,8 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
 	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
 	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
+	int modify_rq = !!qp->rq.wqe_cnt;
+	int modify_sq = !!qp->sq.wqe_cnt;
 	int rq_state;
 	int sq_state;
 	int err;
@@ -2494,10 +2566,18 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 		rq_state = MLX5_RQC_STATE_RST;
 		sq_state = MLX5_SQC_STATE_RST;
 		break;
-	case MLX5_CMD_OP_INIT2INIT_QP:
-	case MLX5_CMD_OP_INIT2RTR_QP:
 	case MLX5_CMD_OP_RTR2RTS_QP:
 	case MLX5_CMD_OP_RTS2RTS_QP:
+		if (raw_qp_param->set_mask ==
+		    MLX5_RAW_QP_RATE_LIMIT) {
+			modify_rq = 0;
+			sq_state = sq->state;
+		} else {
+			return raw_qp_param->set_mask ? -EINVAL : 0;
+		}
+		break;
+	case MLX5_CMD_OP_INIT2INIT_QP:
+	case MLX5_CMD_OP_INIT2RTR_QP:
 		if (raw_qp_param->set_mask)
 			return -EINVAL;
 		else
@@ -2507,13 +2587,13 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 		return -EINVAL;
 	}
 
-	if (qp->rq.wqe_cnt) {
-		err = modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param);
+	if (modify_rq) {
+		err =  modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param);
 		if (err)
 			return err;
 	}
 
-	if (qp->sq.wqe_cnt) {
+	if (modify_sq) {
 		if (tx_affinity) {
 			err = modify_raw_packet_tx_affinity(dev->mdev, sq,
 							    tx_affinity);
@@ -2521,7 +2601,7 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 				return err;
 		}
 
-		return modify_raw_packet_qp_sq(dev->mdev, sq, sq_state);
+		return modify_raw_packet_qp_sq(dev->mdev, sq, sq_state, raw_qp_param);
 	}
 
 	return 0;
@@ -2577,7 +2657,6 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	struct mlx5_ib_port *mibport = NULL;
 	enum mlx5_qp_state mlx5_cur, mlx5_new;
 	enum mlx5_qp_optpar optpar;
-	int sqd_event;
 	int mlx5_st;
 	int err;
 	u16 op;
@@ -2724,12 +2803,6 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
 		context->db_rec_addr = cpu_to_be64(qp->db.dma);
 
-	if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD	&&
-	    attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
-		sqd_event = 1;
-	else
-		sqd_event = 0;
-
 	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
 		u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
 			       qp->port) - 1;
@@ -2776,6 +2849,12 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 			raw_qp_param.rq_q_ctr_id = mibport->q_cnt_id;
 			raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
 		}
+
+		if (attr_mask & IB_QP_RATE_LIMIT) {
+			raw_qp_param.rate_limit = attr->rate_limit;
+			raw_qp_param.set_mask |= MLX5_RAW_QP_RATE_LIMIT;
+		}
+
 		err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity);
 	} else {
 		err = mlx5_core_qp_modify(dev->mdev, op, optpar, context,
@@ -3067,10 +3146,10 @@ static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
 {
 	memset(umr, 0, sizeof(*umr));
 	umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
-	umr->flags = 1 << 7;
+	umr->flags = MLX5_UMR_INLINE;
 }
 
-static __be64 get_umr_reg_mr_mask(void)
+static __be64 get_umr_reg_mr_mask(int atomic)
 {
 	u64 result;
 
@@ -3083,9 +3162,11 @@ static __be64 get_umr_reg_mr_mask(void)
 		 MLX5_MKEY_MASK_KEY		|
 		 MLX5_MKEY_MASK_RR		|
 		 MLX5_MKEY_MASK_RW		|
-		 MLX5_MKEY_MASK_A		|
 		 MLX5_MKEY_MASK_FREE;
 
+	if (atomic)
+		result |= MLX5_MKEY_MASK_A;
+
 	return cpu_to_be64(result);
 }
 
@@ -3146,7 +3227,7 @@ static __be64 get_umr_update_pd_mask(void)
 }
 
 static void set_reg_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
-				struct ib_send_wr *wr)
+				struct ib_send_wr *wr, int atomic)
 {
 	struct mlx5_umr_wr *umrwr = umr_wr(wr);
 
@@ -3171,7 +3252,7 @@ static void set_reg_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
 		if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD)
 			umr->mkey_mask |= get_umr_update_pd_mask();
 		if (!umr->mkey_mask)
-			umr->mkey_mask = get_umr_reg_mr_mask();
+			umr->mkey_mask = get_umr_reg_mr_mask(atomic);
 	} else {
 		umr->mkey_mask = get_umr_unreg_mr_mask();
 	}
@@ -4024,7 +4105,7 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 			}
 			qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
 			ctrl->imm = cpu_to_be32(umr_wr(wr)->mkey);
-			set_reg_umr_segment(seg, wr);
+			set_reg_umr_segment(seg, wr, !!(MLX5_CAP_GEN(mdev, atomic)));
 			seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
 			size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
 			if (unlikely((seg == qend)))
diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c
index 3857dbd9c956..6f4397ee1ed6 100644
--- a/drivers/infiniband/hw/mlx5/srq.c
+++ b/drivers/infiniband/hw/mlx5/srq.c
@@ -118,7 +118,7 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
 		return err;
 	}
 
-	mlx5_ib_cont_pages(srq->umem, ucmd.buf_addr, &npages,
+	mlx5_ib_cont_pages(srq->umem, ucmd.buf_addr, 0, &npages,
 			   &page_shift, &ncont, NULL);
 	err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift,
 				     &offset);
@@ -203,8 +203,6 @@ static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq,
 
 	srq->wrid = kmalloc(srq->msrq.max * sizeof(u64), GFP_KERNEL);
 	if (!srq->wrid) {
-		mlx5_ib_dbg(dev, "kmalloc failed %lu\n",
-			    (unsigned long)(srq->msrq.max * sizeof(u64)));
 		err = -ENOMEM;
 		goto err_in;
 	}
@@ -282,6 +280,7 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
 	mlx5_ib_dbg(dev, "desc_size 0x%x, req wr 0x%x, srq size 0x%x, max_gs 0x%x, max_avail_gather 0x%x\n",
 		    desc_size, init_attr->attr.max_wr, srq->msrq.max, srq->msrq.max_gs,
 		    srq->msrq.max_avail_gather);
+	in.type = init_attr->srq_type;
 
 	if (pd->uobject)
 		err = create_srq_user(pd, srq, &in, udata, buf_size);
@@ -294,7 +293,6 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
 		goto err_srq;
 	}
 
-	in.type = init_attr->srq_type;
 	in.log_size = ilog2(srq->msrq.max);
 	in.wqe_shift = srq->msrq.wqe_shift - 4;
 	if (srq->wq_sig)
diff --git a/drivers/infiniband/hw/mthca/mthca_av.c b/drivers/infiniband/hw/mthca/mthca_av.c
index bcac294042f5..c9f0f364f484 100644
--- a/drivers/infiniband/hw/mthca/mthca_av.c
+++ b/drivers/infiniband/hw/mthca/mthca_av.c
@@ -186,8 +186,8 @@ int mthca_create_ah(struct mthca_dev *dev,
 
 on_hca_fail:
 	if (ah->type == MTHCA_AH_PCI_POOL) {
-		ah->av = pci_pool_alloc(dev->av_table.pool,
-					GFP_ATOMIC, &ah->avdma);
+		ah->av = pci_pool_zalloc(dev->av_table.pool,
+					 GFP_ATOMIC, &ah->avdma);
 		if (!ah->av)
 			return -ENOMEM;
 
@@ -196,8 +196,6 @@ on_hca_fail:
 
 	ah->key = pd->ntmr.ibmr.lkey;
 
-	memset(av, 0, MTHCA_AV_SIZE);
-
 	av->port_pd = cpu_to_be32(pd->pd_num | (ah_attr->port_num << 24));
 	av->g_slid  = ah_attr->src_path_bits;
 	av->dlid    = cpu_to_be16(ah_attr->dlid);
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 358930a41e36..d31708742ba5 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -410,7 +410,9 @@ static int mthca_dealloc_pd(struct ib_pd *pd)
 }
 
 static struct ib_ah *mthca_ah_create(struct ib_pd *pd,
-				     struct ib_ah_attr *ah_attr)
+				     struct ib_ah_attr *ah_attr,
+				     struct ib_udata *udata)
+
 {
 	int err;
 	struct mthca_ah *ah;
diff --git a/drivers/infiniband/hw/mthca/mthca_reset.c b/drivers/infiniband/hw/mthca/mthca_reset.c
index 6727af27c017..2a6979e4ae1c 100644
--- a/drivers/infiniband/hw/mthca/mthca_reset.c
+++ b/drivers/infiniband/hw/mthca/mthca_reset.c
@@ -96,8 +96,6 @@ int mthca_reset(struct mthca_dev *mdev)
 	hca_header = kmalloc(256, GFP_KERNEL);
 	if (!hca_header) {
 		err = -ENOMEM;
-		mthca_err(mdev, "Couldn't allocate memory to save HCA "
-			  "PCI header, aborting.\n");
 		goto put_dev;
 	}
 
@@ -119,8 +117,6 @@ int mthca_reset(struct mthca_dev *mdev)
 		bridge_header = kmalloc(256, GFP_KERNEL);
 		if (!bridge_header) {
 			err = -ENOMEM;
-			mthca_err(mdev, "Couldn't allocate memory to save HCA "
-				  "bridge PCI header, aborting.\n");
 			goto free_hca;
 		}
 
diff --git a/drivers/infiniband/hw/nes/nes.c b/drivers/infiniband/hw/nes/nes.c
index 2baa45a8e401..5b9601014f0c 100644
--- a/drivers/infiniband/hw/nes/nes.c
+++ b/drivers/infiniband/hw/nes/nes.c
@@ -515,7 +515,6 @@ static int nes_probe(struct pci_dev *pcidev, const struct pci_device_id *ent)
 	/* Allocate hardware structure */
 	nesdev = kzalloc(sizeof(struct nes_device), GFP_KERNEL);
 	if (!nesdev) {
-		printk(KERN_ERR PFX "%s: Unable to alloc hardware struct\n", pci_name(pcidev));
 		ret = -ENOMEM;
 		goto bail2;
 	}
diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c
index 57db9b332f44..8e703479e7ce 100644
--- a/drivers/infiniband/hw/nes/nes_cm.c
+++ b/drivers/infiniband/hw/nes/nes_cm.c
@@ -2282,10 +2282,8 @@ static struct nes_cm_listener *mini_cm_listen(struct nes_cm_core *cm_core,
 	if (!listener) {
 		/* create a CM listen node (1/2 node to compare incoming traffic to) */
 		listener = kzalloc(sizeof(*listener), GFP_ATOMIC);
-		if (!listener) {
-			nes_debug(NES_DBG_CM, "Not creating listener memory allocation failed\n");
+		if (!listener)
 			return NULL;
-		}
 
 		listener->loc_addr = cm_info->loc_addr;
 		listener->loc_port = cm_info->loc_port;
diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c
index a1c6481d8038..19acd13c6cb1 100644
--- a/drivers/infiniband/hw/nes/nes_hw.c
+++ b/drivers/infiniband/hw/nes/nes_hw.c
@@ -351,9 +351,8 @@ struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 
 	/* allocate a new adapter struct */
 	nesadapter = kzalloc(adapter_size, GFP_KERNEL);
-	if (nesadapter == NULL) {
+	if (!nesadapter)
 		return NULL;
-	}
 
 	nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 			nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
@@ -1007,8 +1006,7 @@ int nes_init_cqp(struct nes_device *nesdev)
 	/* Allocate a twice the number of CQP requests as the SQ size */
 	nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 			2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
-	if (nesdev->nes_cqp_requests == NULL) {
-		nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
+	if (!nesdev->nes_cqp_requests) {
 		pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 				nesdev->cqp.sq_pbase);
 		return -ENOMEM;
diff --git a/drivers/infiniband/hw/nes/nes_mgt.c b/drivers/infiniband/hw/nes/nes_mgt.c
index 416645259b0f..33624f17c347 100644
--- a/drivers/infiniband/hw/nes/nes_mgt.c
+++ b/drivers/infiniband/hw/nes/nes_mgt.c
@@ -320,8 +320,7 @@ static int get_fpdu_info(struct nes_device *nesdev, struct nes_qp *nesqp,
 
 	/* Found one */
 	fpdu_info = kzalloc(sizeof(*fpdu_info), GFP_ATOMIC);
-	if (fpdu_info == NULL) {
-		nes_debug(NES_DBG_PAU, "Failed to alloc a fpdu_info.\n");
+	if (!fpdu_info) {
 		rc = -ENOMEM;
 		goto out;
 	}
@@ -729,8 +728,7 @@ static int nes_change_quad_hash(struct nes_device *nesdev,
 	}
 
 	qh_chg = kmalloc(sizeof *qh_chg, GFP_ATOMIC);
-	if (qh_chg == NULL) {
-		nes_debug(NES_DBG_PAU, "Failed to get a cqp_request.\n");
+	if (!qh_chg) {
 		ret = -ENOMEM;
 		goto chg_qh_err;
 	}
@@ -880,10 +878,8 @@ int nes_init_mgt_qp(struct nes_device *nesdev, struct net_device *netdev, struct
 
 	/* Allocate space the all mgt QPs once */
 	mgtvnic = kzalloc(NES_MGT_QP_COUNT * sizeof(struct nes_vnic_mgt), GFP_KERNEL);
-	if (mgtvnic == NULL) {
-		nes_debug(NES_DBG_INIT, "Unable to allocate memory for mgt structure\n");
+	if (!mgtvnic)
 		return -ENOMEM;
-	}
 
 	/* Allocate fragment, RQ, and CQ; Reuse CEQ based on the PCI function */
 	/* We are not sending from this NIC so sq is not allocated */
diff --git a/drivers/infiniband/hw/nes/nes_nic.c b/drivers/infiniband/hw/nes/nes_nic.c
index 7f8597d6738b..5921ea3d50ae 100644
--- a/drivers/infiniband/hw/nes/nes_nic.c
+++ b/drivers/infiniband/hw/nes/nes_nic.c
@@ -662,10 +662,14 @@ tso_sq_no_longer_full:
 				nesnic->sq_head &= nesnic->sq_size-1;
 			}
 		} else {
-			nesvnic->linearized_skbs++;
 			hoffset = skb_transport_header(skb) - skb->data;
 			nhoffset = skb_network_header(skb) - skb->data;
-			skb_linearize(skb);
+			if (skb_linearize(skb)) {
+				nesvnic->tx_sw_dropped++;
+				kfree_skb(skb);
+				return NETDEV_TX_OK;
+			}
+			nesvnic->linearized_skbs++;
 			skb_set_transport_header(skb, hoffset);
 			skb_set_network_header(skb, nhoffset);
 			if (!nes_nic_send(skb, netdev))
@@ -1461,7 +1465,8 @@ static int nes_netdev_set_pauseparam(struct net_device *netdev,
 /**
  * nes_netdev_get_settings
  */
-static int nes_netdev_get_settings(struct net_device *netdev, struct ethtool_cmd *et_cmd)
+static int nes_netdev_get_link_ksettings(struct net_device *netdev,
+					 struct ethtool_link_ksettings *cmd)
 {
 	struct nes_vnic *nesvnic = netdev_priv(netdev);
 	struct nes_device *nesdev = nesvnic->nesdev;
@@ -1470,54 +1475,59 @@ static int nes_netdev_get_settings(struct net_device *netdev, struct ethtool_cmd
 	u8 phy_type = nesadapter->phy_type[mac_index];
 	u8 phy_index = nesadapter->phy_index[mac_index];
 	u16 phy_data;
+	u32 supported, advertising;
 
-	et_cmd->duplex = DUPLEX_FULL;
-	et_cmd->port   = PORT_MII;
-	et_cmd->maxtxpkt = 511;
-	et_cmd->maxrxpkt = 511;
+	cmd->base.duplex = DUPLEX_FULL;
+	cmd->base.port   = PORT_MII;
 
 	if (nesadapter->OneG_Mode) {
-		ethtool_cmd_speed_set(et_cmd, SPEED_1000);
+		cmd->base.speed = SPEED_1000;
 		if (phy_type == NES_PHY_TYPE_PUMA_1G) {
-			et_cmd->supported   = SUPPORTED_1000baseT_Full;
-			et_cmd->advertising = ADVERTISED_1000baseT_Full;
-			et_cmd->autoneg     = AUTONEG_DISABLE;
-			et_cmd->transceiver = XCVR_INTERNAL;
-			et_cmd->phy_address = mac_index;
+			supported   = SUPPORTED_1000baseT_Full;
+			advertising = ADVERTISED_1000baseT_Full;
+			cmd->base.autoneg     = AUTONEG_DISABLE;
+			cmd->base.phy_address = mac_index;
 		} else {
 			unsigned long flags;
-			et_cmd->supported   = SUPPORTED_1000baseT_Full
-					    | SUPPORTED_Autoneg;
-			et_cmd->advertising = ADVERTISED_1000baseT_Full
-					    | ADVERTISED_Autoneg;
+
+			supported = SUPPORTED_1000baseT_Full
+				| SUPPORTED_Autoneg;
+			advertising = ADVERTISED_1000baseT_Full
+				| ADVERTISED_Autoneg;
 			spin_lock_irqsave(&nesadapter->phy_lock, flags);
 			nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
 			spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 			if (phy_data & 0x1000)
-				et_cmd->autoneg = AUTONEG_ENABLE;
+				cmd->base.autoneg = AUTONEG_ENABLE;
 			else
-				et_cmd->autoneg = AUTONEG_DISABLE;
-			et_cmd->transceiver = XCVR_EXTERNAL;
-			et_cmd->phy_address = phy_index;
+				cmd->base.autoneg = AUTONEG_DISABLE;
+			cmd->base.phy_address = phy_index;
 		}
+		ethtool_convert_legacy_u32_to_link_mode(
+			cmd->link_modes.supported, supported);
+		ethtool_convert_legacy_u32_to_link_mode(
+			cmd->link_modes.advertising, advertising);
 		return 0;
 	}
 	if ((phy_type == NES_PHY_TYPE_ARGUS) ||
 	    (phy_type == NES_PHY_TYPE_SFP_D) ||
 	    (phy_type == NES_PHY_TYPE_KR)) {
-		et_cmd->transceiver = XCVR_EXTERNAL;
-		et_cmd->port        = PORT_FIBRE;
-		et_cmd->supported   = SUPPORTED_FIBRE;
-		et_cmd->advertising = ADVERTISED_FIBRE;
-		et_cmd->phy_address = phy_index;
+		cmd->base.port        = PORT_FIBRE;
+		supported   = SUPPORTED_FIBRE;
+		advertising = ADVERTISED_FIBRE;
+		cmd->base.phy_address = phy_index;
 	} else {
-		et_cmd->transceiver = XCVR_INTERNAL;
-		et_cmd->supported   = SUPPORTED_10000baseT_Full;
-		et_cmd->advertising = ADVERTISED_10000baseT_Full;
-		et_cmd->phy_address = mac_index;
+		supported   = SUPPORTED_10000baseT_Full;
+		advertising = ADVERTISED_10000baseT_Full;
+		cmd->base.phy_address = mac_index;
 	}
-	ethtool_cmd_speed_set(et_cmd, SPEED_10000);
-	et_cmd->autoneg = AUTONEG_DISABLE;
+	cmd->base.speed = SPEED_10000;
+	cmd->base.autoneg = AUTONEG_DISABLE;
+	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
+						supported);
+	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
+						advertising);
+
 	return 0;
 }
 
@@ -1525,7 +1535,9 @@ static int nes_netdev_get_settings(struct net_device *netdev, struct ethtool_cmd
 /**
  * nes_netdev_set_settings
  */
-static int nes_netdev_set_settings(struct net_device *netdev, struct ethtool_cmd *et_cmd)
+static int
+nes_netdev_set_link_ksettings(struct net_device *netdev,
+			      const struct ethtool_link_ksettings *cmd)
 {
 	struct nes_vnic *nesvnic = netdev_priv(netdev);
 	struct nes_device *nesdev = nesvnic->nesdev;
@@ -1539,7 +1551,7 @@ static int nes_netdev_set_settings(struct net_device *netdev, struct ethtool_cmd
 
 		spin_lock_irqsave(&nesadapter->phy_lock, flags);
 		nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
-		if (et_cmd->autoneg) {
+		if (cmd->base.autoneg) {
 			/* Turn on Full duplex, Autoneg, and restart autonegotiation */
 			phy_data |= 0x1300;
 		} else {
@@ -1556,8 +1568,6 @@ static int nes_netdev_set_settings(struct net_device *netdev, struct ethtool_cmd
 
 static const struct ethtool_ops nes_ethtool_ops = {
 	.get_link = ethtool_op_get_link,
-	.get_settings = nes_netdev_get_settings,
-	.set_settings = nes_netdev_set_settings,
 	.get_strings = nes_netdev_get_strings,
 	.get_sset_count = nes_netdev_get_sset_count,
 	.get_ethtool_stats = nes_netdev_get_ethtool_stats,
@@ -1566,6 +1576,8 @@ static const struct ethtool_ops nes_ethtool_ops = {
 	.set_coalesce = nes_netdev_set_coalesce,
 	.get_pauseparam = nes_netdev_get_pauseparam,
 	.set_pauseparam = nes_netdev_set_pauseparam,
+	.get_link_ksettings = nes_netdev_get_link_ksettings,
+	.set_link_ksettings = nes_netdev_set_link_ksettings,
 };
 
 static void nes_vlan_mode(struct net_device *netdev, struct nes_device *nesdev, netdev_features_t features)
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index bd69125731c1..aff9fb14768b 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -771,7 +771,8 @@ static int nes_dealloc_pd(struct ib_pd *ibpd)
 /**
  * nes_create_ah
  */
-static struct ib_ah *nes_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
+static struct ib_ah *nes_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
+				   struct ib_udata *udata)
 {
 	return ERR_PTR(-ENOSYS);
 }
@@ -1075,7 +1076,6 @@ static struct ib_qp *nes_create_qp(struct ib_pd *ibpd,
 			mem = kzalloc(sizeof(*nesqp)+NES_SW_CONTEXT_ALIGN-1, GFP_KERNEL);
 			if (!mem) {
 				nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
-				nes_debug(NES_DBG_QP, "Unable to allocate QP\n");
 				return ERR_PTR(-ENOMEM);
 			}
 			u64nesqp = (unsigned long)mem;
@@ -1475,7 +1475,6 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev,
 	nescq = kzalloc(sizeof(struct nes_cq), GFP_KERNEL);
 	if (!nescq) {
 		nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
-		nes_debug(NES_DBG_CQ, "Unable to allocate nes_cq struct\n");
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -2408,7 +2407,6 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 			}
 			nespbl = kzalloc(sizeof(*nespbl), GFP_KERNEL);
 			if (!nespbl) {
-				nes_debug(NES_DBG_MR, "Unable to allocate PBL\n");
 				ib_umem_release(region);
 				return ERR_PTR(-ENOMEM);
 			}
@@ -2416,7 +2414,6 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 			if (!nesmr) {
 				ib_umem_release(region);
 				kfree(nespbl);
-				nes_debug(NES_DBG_MR, "Unable to allocate nesmr\n");
 				return ERR_PTR(-ENOMEM);
 			}
 			nesmr->region = region;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
index 797362a297b2..14d33b0f3950 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
@@ -154,7 +154,8 @@ static inline int set_av_attr(struct ocrdma_dev *dev, struct ocrdma_ah *ah,
 	return status;
 }
 
-struct ib_ah *ocrdma_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
+struct ib_ah *ocrdma_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
+			       struct ib_udata *udata)
 {
 	u32 *ahid_addr;
 	int status;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.h b/drivers/infiniband/hw/ocrdma/ocrdma_ah.h
index 3856dd4c7e3d..0704a24b17c8 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.h
@@ -50,7 +50,9 @@ enum {
 	OCRDMA_AH_L3_TYPE_MASK		= 0x03,
 	OCRDMA_AH_L3_TYPE_SHIFT		= 0x1D /* 29 bits */
 };
-struct ib_ah *ocrdma_create_ah(struct ib_pd *, struct ib_ah_attr *);
+
+struct ib_ah *ocrdma_create_ah(struct ib_pd *, struct ib_ah_attr *,
+			       struct ib_udata *);
 int ocrdma_destroy_ah(struct ib_ah *);
 int ocrdma_query_ah(struct ib_ah *, struct ib_ah_attr *);
 int ocrdma_modify_ah(struct ib_ah *, struct ib_ah_attr *);
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
index 67fc0b6857e1..9a305201545e 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
@@ -1596,10 +1596,9 @@ void ocrdma_alloc_pd_pool(struct ocrdma_dev *dev)
 
 	dev->pd_mgr = kzalloc(sizeof(struct ocrdma_pd_resource_mgr),
 			      GFP_KERNEL);
-	if (!dev->pd_mgr) {
-		pr_err("%s(%d)Memory allocation failure.\n", __func__, dev->id);
+	if (!dev->pd_mgr)
 		return;
-	}
+
 	status = ocrdma_mbx_alloc_pd_range(dev);
 	if (status) {
 		pr_err("%s(%d) Unable to initialize PD pool, using default.\n",
@@ -1642,7 +1641,7 @@ static int ocrdma_build_q_conf(u32 *num_entries, int entry_size,
 static int ocrdma_mbx_create_ah_tbl(struct ocrdma_dev *dev)
 {
 	int i;
-	int status = 0;
+	int status = -ENOMEM;
 	int max_ah;
 	struct ocrdma_create_ah_tbl *cmd;
 	struct ocrdma_create_ah_tbl_rsp *rsp;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_stats.c b/drivers/infiniband/hw/ocrdma/ocrdma_stats.c
index 8bef09a8c49f..f8e4b0a6486f 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_stats.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_stats.c
@@ -84,10 +84,8 @@ bool ocrdma_alloc_stats_resources(struct ocrdma_dev *dev)
 
 	/* Alloc debugfs mem */
 	mem->debugfs_mem = kzalloc(OCRDMA_MAX_DBGFS_MEM, GFP_KERNEL);
-	if (!mem->debugfs_mem) {
-		pr_err("%s: stats debugfs mem allocation failed\n", __func__);
+	if (!mem->debugfs_mem)
 		return false;
-	}
 
 	return true;
 }
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index a61514296767..302fb05e6e6f 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -511,8 +511,10 @@ int qedr_dealloc_pd(struct ib_pd *ibpd)
 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 	struct qedr_pd *pd = get_qedr_pd(ibpd);
 
-	if (!pd)
+	if (!pd) {
 		pr_err("Invalid PD received in dealloc_pd\n");
+		return -EINVAL;
+	}
 
 	DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 	dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
@@ -1477,6 +1479,7 @@ struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
 	struct qedr_ucontext *ctx = NULL;
 	struct qedr_create_qp_ureq ureq;
 	struct qedr_qp *qp;
+	struct ib_qp *ibqp;
 	int rc = 0;
 
 	DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
@@ -1486,13 +1489,13 @@ struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
 	if (rc)
 		return ERR_PTR(rc);
 
+	if (attrs->srq)
+		return ERR_PTR(-EINVAL);
+
 	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
 	if (!qp)
 		return ERR_PTR(-ENOMEM);
 
-	if (attrs->srq)
-		return ERR_PTR(-EINVAL);
-
 	DP_DEBUG(dev, QEDR_MSG_QP,
 		 "create qp: sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
 		 get_qedr_cq(attrs->send_cq),
@@ -1508,7 +1511,10 @@ struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
 			       "create qp: unexpected udata when creating GSI QP\n");
 			goto err0;
 		}
-		return qedr_create_gsi_qp(dev, attrs, qp);
+		ibqp = qedr_create_gsi_qp(dev, attrs, qp);
+		if (IS_ERR(ibqp))
+			kfree(qp);
+		return ibqp;
 	}
 
 	memset(&in_params, 0, sizeof(in_params));
@@ -2094,7 +2100,8 @@ int qedr_destroy_qp(struct ib_qp *ibqp)
 	return rc;
 }
 
-struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
+struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
+			     struct ib_udata *udata)
 {
 	struct qedr_ah *ah;
 
@@ -2413,8 +2420,7 @@ static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
 		 */
 		pbl = list_first_entry(&info->inuse_pbl_list,
 				       struct qedr_pbl, list_entry);
-		list_del(&pbl->list_entry);
-		list_add_tail(&pbl->list_entry, &info->free_pbl_list);
+		list_move_tail(&pbl->list_entry, &info->free_pbl_list);
 		info->completed_handled++;
 	}
 }
@@ -2981,11 +2987,6 @@ int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 		return -EINVAL;
 	}
 
-	if (!wr) {
-		DP_ERR(dev, "Got an empty post send.\n");
-		return -EINVAL;
-	}
-
 	while (wr) {
 		rc = __qedr_post_send(ibqp, wr, bad_wr);
 		if (rc)
diff --git a/drivers/infiniband/hw/qedr/verbs.h b/drivers/infiniband/hw/qedr/verbs.h
index a9b5e67bb81e..070677ca4d19 100644
--- a/drivers/infiniband/hw/qedr/verbs.h
+++ b/drivers/infiniband/hw/qedr/verbs.h
@@ -70,7 +70,8 @@ int qedr_query_qp(struct ib_qp *, struct ib_qp_attr *qp_attr,
 		  int qp_attr_mask, struct ib_qp_init_attr *);
 int qedr_destroy_qp(struct ib_qp *ibqp);
 
-struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr);
+struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
+			     struct ib_udata *udata);
 int qedr_destroy_ah(struct ib_ah *ibah);
 
 int qedr_dereg_mr(struct ib_mr *);
diff --git a/drivers/infiniband/hw/qib/qib_diag.c b/drivers/infiniband/hw/qib/qib_diag.c
index 8c34b23e5bf6..775018b32b0d 100644
--- a/drivers/infiniband/hw/qib/qib_diag.c
+++ b/drivers/infiniband/hw/qib/qib_diag.c
@@ -609,8 +609,6 @@ static ssize_t qib_diagpkt_write(struct file *fp,
 
 	tmpbuf = vmalloc(plen);
 	if (!tmpbuf) {
-		qib_devinfo(dd->pcidev,
-			"Unable to allocate tmp buffer, failing\n");
 		ret = -ENOMEM;
 		goto bail;
 	}
@@ -702,10 +700,8 @@ int qib_register_observer(struct qib_devdata *dd,
 	if (!dd || !op)
 		return -EINVAL;
 	olp = vmalloc(sizeof(*olp));
-	if (!olp) {
-		pr_err("vmalloc for observer failed\n");
+	if (!olp)
 		return -ENOMEM;
-	}
 
 	spin_lock_irqsave(&dd->qib_diag_trans_lock, flags);
 	olp->op = op;
diff --git a/drivers/infiniband/hw/qib/qib_driver.c b/drivers/infiniband/hw/qib/qib_driver.c
index 728e0a030d2e..2b5982f743ef 100644
--- a/drivers/infiniband/hw/qib/qib_driver.c
+++ b/drivers/infiniband/hw/qib/qib_driver.c
@@ -420,8 +420,7 @@ static u32 qib_rcv_hdrerr(struct qib_ctxtdata *rcd, struct qib_pportdata *ppd,
 						if (list_empty(&qp->rspwait)) {
 							qp->r_flags |=
 								RVT_R_RSP_NAK;
-							atomic_inc(
-								&qp->refcount);
+							rvt_get_qp(qp);
 							list_add_tail(
 							 &qp->rspwait,
 							 &rcd->qp_wait_list);
diff --git a/drivers/infiniband/hw/qib/qib_eeprom.c b/drivers/infiniband/hw/qib/qib_eeprom.c
index 311ee6c3dd5e..33a2e74c8495 100644
--- a/drivers/infiniband/hw/qib/qib_eeprom.c
+++ b/drivers/infiniband/hw/qib/qib_eeprom.c
@@ -182,12 +182,8 @@ void qib_get_eeprom_info(struct qib_devdata *dd)
 	 * */
 	len = sizeof(struct qib_flash);
 	buf = vmalloc(len);
-	if (!buf) {
-		qib_dev_err(dd,
-			"Couldn't allocate memory to read %u bytes from eeprom for GUID\n",
-			len);
+	if (!buf)
 		goto bail;
-	}
 
 	/*
 	 * Use "public" eeprom read function, which does locking and
diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c
index 382466a90da7..2d1eacf1dfed 100644
--- a/drivers/infiniband/hw/qib/qib_file_ops.c
+++ b/drivers/infiniband/hw/qib/qib_file_ops.c
@@ -2066,8 +2066,11 @@ static ssize_t qib_write(struct file *fp, const char __user *data,
 	ssize_t ret = 0;
 	void *dest;
 
-	if (WARN_ON_ONCE(!ib_safe_file_access(fp)))
+	if (!ib_safe_file_access(fp)) {
+		pr_err_once("qib_write: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",
+			    task_tgid_vnr(current), current->comm);
 		return -EACCES;
+	}
 
 	if (count < sizeof(cmd.type)) {
 		ret = -EINVAL;
diff --git a/drivers/infiniband/hw/qib/qib_iba6120.c b/drivers/infiniband/hw/qib/qib_iba6120.c
index a3733f25280f..92399d3ffd15 100644
--- a/drivers/infiniband/hw/qib/qib_iba6120.c
+++ b/drivers/infiniband/hw/qib/qib_iba6120.c
@@ -1759,9 +1759,7 @@ static void pe_boardname(struct qib_devdata *dd)
 	}
 	namelen = strlen(n) + 1;
 	dd->boardname = kmalloc(namelen, GFP_KERNEL);
-	if (!dd->boardname)
-		qib_dev_err(dd, "Failed allocation for board name: %s\n", n);
-	else
+	if (dd->boardname)
 		snprintf(dd->boardname, namelen, "%s", n);
 
 	if (dd->majrev != 4 || !dd->minrev || dd->minrev > 2)
@@ -2533,8 +2531,6 @@ static void init_6120_cntrnames(struct qib_devdata *dd)
 		dd->cspec->cntrnamelen = 1 + s - cntr6120names;
 	dd->cspec->cntrs = kmalloc(dd->cspec->ncntrs
 		* sizeof(u64), GFP_KERNEL);
-	if (!dd->cspec->cntrs)
-		qib_dev_err(dd, "Failed allocation for counters\n");
 
 	for (i = 0, s = (char *)portcntr6120names; s; i++)
 		s = strchr(s + 1, '\n');
@@ -2542,8 +2538,6 @@ static void init_6120_cntrnames(struct qib_devdata *dd)
 	dd->cspec->portcntrnamelen = sizeof(portcntr6120names) - 1;
 	dd->cspec->portcntrs = kmalloc(dd->cspec->nportcntrs
 		* sizeof(u64), GFP_KERNEL);
-	if (!dd->cspec->portcntrs)
-		qib_dev_err(dd, "Failed allocation for portcounters\n");
 }
 
 static u32 qib_read_6120cntrs(struct qib_devdata *dd, loff_t pos, char **namep,
diff --git a/drivers/infiniband/hw/qib/qib_iba7220.c b/drivers/infiniband/hw/qib/qib_iba7220.c
index 00b2af211157..e55e31a69195 100644
--- a/drivers/infiniband/hw/qib/qib_iba7220.c
+++ b/drivers/infiniband/hw/qib/qib_iba7220.c
@@ -2070,9 +2070,7 @@ static void qib_7220_boardname(struct qib_devdata *dd)
 
 	namelen = strlen(n) + 1;
 	dd->boardname = kmalloc(namelen, GFP_KERNEL);
-	if (!dd->boardname)
-		qib_dev_err(dd, "Failed allocation for board name: %s\n", n);
-	else
+	if (dd->boardname)
 		snprintf(dd->boardname, namelen, "%s", n);
 
 	if (dd->majrev != 5 || !dd->minrev || dd->minrev > 2)
@@ -3179,8 +3177,6 @@ static void init_7220_cntrnames(struct qib_devdata *dd)
 		dd->cspec->cntrnamelen = 1 + s - cntr7220names;
 	dd->cspec->cntrs = kmalloc(dd->cspec->ncntrs
 		* sizeof(u64), GFP_KERNEL);
-	if (!dd->cspec->cntrs)
-		qib_dev_err(dd, "Failed allocation for counters\n");
 
 	for (i = 0, s = (char *)portcntr7220names; s; i++)
 		s = strchr(s + 1, '\n');
@@ -3188,8 +3184,6 @@ static void init_7220_cntrnames(struct qib_devdata *dd)
 	dd->cspec->portcntrnamelen = sizeof(portcntr7220names) - 1;
 	dd->cspec->portcntrs = kmalloc(dd->cspec->nportcntrs
 		* sizeof(u64), GFP_KERNEL);
-	if (!dd->cspec->portcntrs)
-		qib_dev_err(dd, "Failed allocation for portcounters\n");
 }
 
 static u32 qib_read_7220cntrs(struct qib_devdata *dd, loff_t pos, char **namep,
diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c
index ded27172320e..c4a3616062f1 100644
--- a/drivers/infiniband/hw/qib/qib_iba7322.c
+++ b/drivers/infiniband/hw/qib/qib_iba7322.c
@@ -3627,9 +3627,7 @@ static unsigned qib_7322_boardname(struct qib_devdata *dd)
 
 	namelen = strlen(n) + 1;
 	dd->boardname = kmalloc(namelen, GFP_KERNEL);
-	if (!dd->boardname)
-		qib_dev_err(dd, "Failed allocation for board name: %s\n", n);
-	else
+	if (dd->boardname)
 		snprintf(dd->boardname, namelen, "%s", n);
 
 	snprintf(dd->boardversion, sizeof(dd->boardversion),
@@ -3656,7 +3654,7 @@ static unsigned qib_7322_boardname(struct qib_devdata *dd)
 static int qib_do_7322_reset(struct qib_devdata *dd)
 {
 	u64 val;
-	u64 *msix_vecsave;
+	u64 *msix_vecsave = NULL;
 	int i, msix_entries, ret = 1;
 	u16 cmdval;
 	u8 int_line, clinesz;
@@ -3677,10 +3675,7 @@ static int qib_do_7322_reset(struct qib_devdata *dd)
 		/* can be up to 512 bytes, too big for stack */
 		msix_vecsave = kmalloc(2 * dd->cspec->num_msix_entries *
 			sizeof(u64), GFP_KERNEL);
-		if (!msix_vecsave)
-			qib_dev_err(dd, "No mem to save MSIx data\n");
-	} else
-		msix_vecsave = NULL;
+	}
 
 	/*
 	 * Core PCI (as of 2.6.18) doesn't save or rewrite the full vector
@@ -5043,8 +5038,6 @@ static void init_7322_cntrnames(struct qib_devdata *dd)
 		dd->cspec->cntrnamelen = 1 + s - cntr7322names;
 	dd->cspec->cntrs = kmalloc(dd->cspec->ncntrs
 		* sizeof(u64), GFP_KERNEL);
-	if (!dd->cspec->cntrs)
-		qib_dev_err(dd, "Failed allocation for counters\n");
 
 	for (i = 0, s = (char *)portcntr7322names; s; i++)
 		s = strchr(s + 1, '\n');
@@ -5053,9 +5046,6 @@ static void init_7322_cntrnames(struct qib_devdata *dd)
 	for (i = 0; i < dd->num_pports; ++i) {
 		dd->pport[i].cpspec->portcntrs = kmalloc(dd->cspec->nportcntrs
 			* sizeof(u64), GFP_KERNEL);
-		if (!dd->pport[i].cpspec->portcntrs)
-			qib_dev_err(dd,
-				"Failed allocation for portcounters\n");
 	}
 }
 
@@ -6461,7 +6451,6 @@ static int qib_init_7322_variables(struct qib_devdata *dd)
 		sizeof(*dd->cspec->sendibchk), GFP_KERNEL);
 	if (!dd->cspec->sendchkenable || !dd->cspec->sendgrhchk ||
 		!dd->cspec->sendibchk) {
-		qib_dev_err(dd, "Failed allocation for hdrchk bitmaps\n");
 		ret = -ENOMEM;
 		goto bail;
 	}
@@ -7338,10 +7327,9 @@ struct qib_devdata *qib_init_iba7322_funcs(struct pci_dev *pdev,
 	tabsize = actual_cnt;
 	dd->cspec->msix_entries = kzalloc(tabsize *
 			sizeof(struct qib_msix_entry), GFP_KERNEL);
-	if (!dd->cspec->msix_entries) {
-		qib_dev_err(dd, "No memory for MSIx table\n");
+	if (!dd->cspec->msix_entries)
 		tabsize = 0;
-	}
+
 	for (i = 0; i < tabsize; i++)
 		dd->cspec->msix_entries[i].msix.entry = i;
 
diff --git a/drivers/infiniband/hw/qib/qib_init.c b/drivers/infiniband/hw/qib/qib_init.c
index 1730aa839a47..b50240b1d5a4 100644
--- a/drivers/infiniband/hw/qib/qib_init.c
+++ b/drivers/infiniband/hw/qib/qib_init.c
@@ -133,11 +133,8 @@ int qib_create_ctxts(struct qib_devdata *dd)
 	 * cleanup iterates across all possible ctxts.
 	 */
 	dd->rcd = kcalloc(dd->ctxtcnt, sizeof(*dd->rcd), GFP_KERNEL);
-	if (!dd->rcd) {
-		qib_dev_err(dd,
-			"Unable to allocate ctxtdata array, failing\n");
+	if (!dd->rcd)
 		return -ENOMEM;
-	}
 
 	/* create (one or more) kctxt */
 	for (i = 0; i < dd->first_user_ctxt; ++i) {
@@ -265,39 +262,23 @@ int qib_init_pportdata(struct qib_pportdata *ppd, struct qib_devdata *dd,
 	size = IB_CC_TABLE_CAP_DEFAULT * sizeof(struct ib_cc_table_entry)
 		* IB_CCT_ENTRIES;
 	ppd->ccti_entries = kzalloc(size, GFP_KERNEL);
-	if (!ppd->ccti_entries) {
-		qib_dev_err(dd,
-		  "failed to allocate congestion control table for port %d!\n",
-		  port);
+	if (!ppd->ccti_entries)
 		goto bail;
-	}
 
 	size = IB_CC_CCS_ENTRIES * sizeof(struct ib_cc_congestion_entry);
 	ppd->congestion_entries = kzalloc(size, GFP_KERNEL);
-	if (!ppd->congestion_entries) {
-		qib_dev_err(dd,
-		 "failed to allocate congestion setting list for port %d!\n",
-		 port);
+	if (!ppd->congestion_entries)
 		goto bail_1;
-	}
 
 	size = sizeof(struct cc_table_shadow);
 	ppd->ccti_entries_shadow = kzalloc(size, GFP_KERNEL);
-	if (!ppd->ccti_entries_shadow) {
-		qib_dev_err(dd,
-		 "failed to allocate shadow ccti list for port %d!\n",
-		 port);
+	if (!ppd->ccti_entries_shadow)
 		goto bail_2;
-	}
 
 	size = sizeof(struct ib_cc_congestion_setting_attr);
 	ppd->congestion_entries_shadow = kzalloc(size, GFP_KERNEL);
-	if (!ppd->congestion_entries_shadow) {
-		qib_dev_err(dd,
-		 "failed to allocate shadow congestion setting list for port %d!\n",
-		 port);
+	if (!ppd->congestion_entries_shadow)
 		goto bail_3;
-	}
 
 	return 0;
 
@@ -391,18 +372,12 @@ static void init_shadow_tids(struct qib_devdata *dd)
 	dma_addr_t *addrs;
 
 	pages = vzalloc(dd->cfgctxts * dd->rcvtidcnt * sizeof(struct page *));
-	if (!pages) {
-		qib_dev_err(dd,
-			"failed to allocate shadow page * array, no expected sends!\n");
+	if (!pages)
 		goto bail;
-	}
 
 	addrs = vzalloc(dd->cfgctxts * dd->rcvtidcnt * sizeof(dma_addr_t));
-	if (!addrs) {
-		qib_dev_err(dd,
-			"failed to allocate shadow dma handle array, no expected sends!\n");
+	if (!addrs)
 		goto bail_free;
-	}
 
 	dd->pageshadow = pages;
 	dd->physshadow = addrs;
@@ -1026,11 +1001,8 @@ static void qib_verify_pioperf(struct qib_devdata *dd)
 	cnt = 1024;
 
 	addr = vmalloc(cnt);
-	if (!addr) {
-		qib_devinfo(dd->pcidev,
-			 "Couldn't get memory for checking PIO perf, skipping\n");
+	if (!addr)
 		goto done;
-	}
 
 	preempt_disable();  /* we want reasonably accurate elapsed time */
 	msecs = 1 + jiffies_to_msecs(jiffies);
@@ -1172,9 +1144,6 @@ struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra)
 				      sizeof(long), GFP_KERNEL);
 		if (qib_cpulist)
 			qib_cpulist_count = count;
-		else
-			qib_early_err(&pdev->dev,
-				"Could not alloc cpulist info, cpu affinity might be wrong\n");
 	}
 #ifdef CONFIG_DEBUG_FS
 	qib_dbg_ibdev_init(&dd->verbs_dev);
diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c
index 2097512e75aa..031433cb7206 100644
--- a/drivers/infiniband/hw/qib/qib_rc.c
+++ b/drivers/infiniband/hw/qib/qib_rc.c
@@ -941,8 +941,6 @@ void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
 {
 	struct ib_other_headers *ohdr;
 	struct rvt_swqe *wqe;
-	struct ib_wc wc;
-	unsigned i;
 	u32 opcode;
 	u32 psn;
 
@@ -988,22 +986,8 @@ void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
 		qp->s_last = s_last;
 		/* see post_send() */
 		barrier();
-		for (i = 0; i < wqe->wr.num_sge; i++) {
-			struct rvt_sge *sge = &wqe->sg_list[i];
-
-			rvt_put_mr(sge->mr);
-		}
-		/* Post a send completion queue entry if requested. */
-		if (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
-		    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
-			memset(&wc, 0, sizeof(wc));
-			wc.wr_id = wqe->wr.wr_id;
-			wc.status = IB_WC_SUCCESS;
-			wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode];
-			wc.byte_len = wqe->length;
-			wc.qp = &qp->ibqp;
-			rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.send_cq), &wc, 0);
-		}
+		rvt_put_swqe(wqe);
+		rvt_qp_swqe_complete(qp, wqe, IB_WC_SUCCESS);
 	}
 	/*
 	 * If we were waiting for sends to complete before resending,
@@ -1032,9 +1016,6 @@ static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
 					 struct rvt_swqe *wqe,
 					 struct qib_ibport *ibp)
 {
-	struct ib_wc wc;
-	unsigned i;
-
 	/*
 	 * Don't decrement refcount and don't generate a
 	 * completion if the SWQE is being resent until the send
@@ -1044,28 +1025,14 @@ static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
 	    qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
 		u32 s_last;
 
-		for (i = 0; i < wqe->wr.num_sge; i++) {
-			struct rvt_sge *sge = &wqe->sg_list[i];
-
-			rvt_put_mr(sge->mr);
-		}
+		rvt_put_swqe(wqe);
 		s_last = qp->s_last;
 		if (++s_last >= qp->s_size)
 			s_last = 0;
 		qp->s_last = s_last;
 		/* see post_send() */
 		barrier();
-		/* Post a send completion queue entry if requested. */
-		if (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
-		    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
-			memset(&wc, 0, sizeof(wc));
-			wc.wr_id = wqe->wr.wr_id;
-			wc.status = IB_WC_SUCCESS;
-			wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode];
-			wc.byte_len = wqe->length;
-			wc.qp = &qp->ibqp;
-			rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.send_cq), &wc, 0);
-		}
+		rvt_qp_swqe_complete(qp, wqe, IB_WC_SUCCESS);
 	} else
 		this_cpu_inc(*ibp->rvp.rc_delayed_comp);
 
@@ -2112,8 +2079,7 @@ send_last:
 			 * Update the next expected PSN.  We add 1 later
 			 * below, so only add the remainder here.
 			 */
-			if (len > pmtu)
-				qp->r_psn += (len - 1) / pmtu;
+			qp->r_psn += rvt_div_mtu(qp, len - 1);
 		} else {
 			e->rdma_sge.mr = NULL;
 			e->rdma_sge.vaddr = NULL;
diff --git a/drivers/infiniband/hw/qib/qib_ruc.c b/drivers/infiniband/hw/qib/qib_ruc.c
index de1bde5950f5..e54a2feeeb10 100644
--- a/drivers/infiniband/hw/qib/qib_ruc.c
+++ b/drivers/infiniband/hw/qib/qib_ruc.c
@@ -793,7 +793,6 @@ void qib_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
 		       enum ib_wc_status status)
 {
 	u32 old_last, last;
-	unsigned i;
 
 	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_OR_FLUSH_SEND))
 		return;
@@ -805,32 +804,13 @@ void qib_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
 	qp->s_last = last;
 	/* See post_send() */
 	barrier();
-	for (i = 0; i < wqe->wr.num_sge; i++) {
-		struct rvt_sge *sge = &wqe->sg_list[i];
-
-		rvt_put_mr(sge->mr);
-	}
+	rvt_put_swqe(wqe);
 	if (qp->ibqp.qp_type == IB_QPT_UD ||
 	    qp->ibqp.qp_type == IB_QPT_SMI ||
 	    qp->ibqp.qp_type == IB_QPT_GSI)
 		atomic_dec(&ibah_to_rvtah(wqe->ud_wr.ah)->refcount);
 
-	/* See ch. 11.2.4.1 and 10.7.3.1 */
-	if (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
-	    (wqe->wr.send_flags & IB_SEND_SIGNALED) ||
-	    status != IB_WC_SUCCESS) {
-		struct ib_wc wc;
-
-		memset(&wc, 0, sizeof(wc));
-		wc.wr_id = wqe->wr.wr_id;
-		wc.status = status;
-		wc.opcode = ib_qib_wc_opcode[wqe->wr.opcode];
-		wc.qp = &qp->ibqp;
-		if (status == IB_WC_SUCCESS)
-			wc.byte_len = wqe->length;
-		rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.send_cq), &wc,
-			     status != IB_WC_SUCCESS);
-	}
+	rvt_qp_swqe_complete(qp, wqe, status);
 
 	if (qp->s_acked == old_last)
 		qp->s_acked = last;
diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c
index 954f15064514..4b54c0ddd08a 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -114,19 +114,6 @@ module_param_named(disable_sma, ib_qib_disable_sma, uint, S_IWUSR | S_IRUGO);
 MODULE_PARM_DESC(disable_sma, "Disable the SMA");
 
 /*
- * Translate ib_wr_opcode into ib_wc_opcode.
- */
-const enum ib_wc_opcode ib_qib_wc_opcode[] = {
-	[IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE,
-	[IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE,
-	[IB_WR_SEND] = IB_WC_SEND,
-	[IB_WR_SEND_WITH_IMM] = IB_WC_SEND,
-	[IB_WR_RDMA_READ] = IB_WC_RDMA_READ,
-	[IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP,
-	[IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD
-};
-
-/*
  * System image GUID.
  */
 __be64 ib_qib_sys_image_guid;
@@ -464,7 +451,7 @@ static void mem_timer(unsigned long data)
 		priv = list_entry(list->next, struct qib_qp_priv, iowait);
 		qp = priv->owner;
 		list_del_init(&priv->iowait);
-		atomic_inc(&qp->refcount);
+		rvt_get_qp(qp);
 		if (!list_empty(list))
 			mod_timer(&dev->mem_timer, jiffies + 1);
 	}
@@ -477,8 +464,7 @@ static void mem_timer(unsigned long data)
 			qib_schedule_send(qp);
 		}
 		spin_unlock_irqrestore(&qp->s_lock, flags);
-		if (atomic_dec_and_test(&qp->refcount))
-			wake_up(&qp->wait);
+		rvt_put_qp(qp);
 	}
 }
 
@@ -762,7 +748,7 @@ void qib_put_txreq(struct qib_verbs_txreq *tx)
 				  iowait);
 		qp = priv->owner;
 		list_del_init(&priv->iowait);
-		atomic_inc(&qp->refcount);
+		rvt_get_qp(qp);
 		spin_unlock_irqrestore(&dev->rdi.pending_lock, flags);
 
 		spin_lock_irqsave(&qp->s_lock, flags);
@@ -772,8 +758,7 @@ void qib_put_txreq(struct qib_verbs_txreq *tx)
 		}
 		spin_unlock_irqrestore(&qp->s_lock, flags);
 
-		if (atomic_dec_and_test(&qp->refcount))
-			wake_up(&qp->wait);
+		rvt_put_qp(qp);
 	} else
 		spin_unlock_irqrestore(&dev->rdi.pending_lock, flags);
 }
@@ -808,7 +793,7 @@ void qib_verbs_sdma_desc_avail(struct qib_pportdata *ppd, unsigned avail)
 			break;
 		avail -= qpp->s_tx->txreq.sg_count;
 		list_del_init(&qpp->iowait);
-		atomic_inc(&qp->refcount);
+		rvt_get_qp(qp);
 		qps[n++] = qp;
 	}
 
@@ -822,8 +807,7 @@ void qib_verbs_sdma_desc_avail(struct qib_pportdata *ppd, unsigned avail)
 			qib_schedule_send(qp);
 		}
 		spin_unlock(&qp->s_lock);
-		if (atomic_dec_and_test(&qp->refcount))
-			wake_up(&qp->wait);
+		rvt_put_qp(qp);
 	}
 }
 
@@ -1288,7 +1272,7 @@ void qib_ib_piobufavail(struct qib_devdata *dd)
 		priv = list_entry(list->next, struct qib_qp_priv, iowait);
 		qp = priv->owner;
 		list_del_init(&priv->iowait);
-		atomic_inc(&qp->refcount);
+		rvt_get_qp(qp);
 		qps[n++] = qp;
 	}
 	dd->f_wantpiobuf_intr(dd, 0);
@@ -1306,8 +1290,7 @@ full:
 		spin_unlock_irqrestore(&qp->s_lock, flags);
 
 		/* Notify qib_destroy_qp() if it is waiting. */
-		if (atomic_dec_and_test(&qp->refcount))
-			wake_up(&qp->wait);
+		rvt_put_qp(qp);
 	}
 }
 
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c b/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c
index 5b0248adf4ce..092d4e11a633 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_qp_grp.c
@@ -117,10 +117,10 @@ static int enable_qp_grp(struct usnic_ib_qp_grp *qp_grp)
 	vnic_idx = usnic_vnic_get_index(qp_grp->vf->vnic);
 
 	res_chunk = get_qp_res_chunk(qp_grp);
-	if (IS_ERR_OR_NULL(res_chunk)) {
+	if (IS_ERR(res_chunk)) {
 		usnic_err("Unable to get qp res with err %ld\n",
 				PTR_ERR(res_chunk));
-		return res_chunk ? PTR_ERR(res_chunk) : -ENOMEM;
+		return PTR_ERR(res_chunk);
 	}
 
 	for (i = 0; i < res_chunk->cnt; i++) {
@@ -158,10 +158,10 @@ static int disable_qp_grp(struct usnic_ib_qp_grp *qp_grp)
 	vnic_idx = usnic_vnic_get_index(qp_grp->vf->vnic);
 
 	res_chunk = get_qp_res_chunk(qp_grp);
-	if (IS_ERR_OR_NULL(res_chunk)) {
+	if (IS_ERR(res_chunk)) {
 		usnic_err("Unable to get qp res with err %ld\n",
 			PTR_ERR(res_chunk));
-		return res_chunk ? PTR_ERR(res_chunk) : -ENOMEM;
+		return PTR_ERR(res_chunk);
 	}
 
 	for (i = 0; i < res_chunk->cnt; i++) {
@@ -186,11 +186,11 @@ static int init_filter_action(struct usnic_ib_qp_grp *qp_grp,
 	struct usnic_vnic_res_chunk *res_chunk;
 
 	res_chunk = usnic_ib_qp_grp_get_chunk(qp_grp, USNIC_VNIC_RES_TYPE_RQ);
-	if (IS_ERR_OR_NULL(res_chunk)) {
+	if (IS_ERR(res_chunk)) {
 		usnic_err("Unable to get %s with err %ld\n",
 			usnic_vnic_res_type_to_str(USNIC_VNIC_RES_TYPE_RQ),
 			PTR_ERR(res_chunk));
-		return res_chunk ? PTR_ERR(res_chunk) : -ENOMEM;
+		return PTR_ERR(res_chunk);
 	}
 
 	uaction->vnic_idx = usnic_vnic_get_index(qp_grp->vf->vnic);
@@ -228,8 +228,6 @@ create_roce_custom_flow(struct usnic_ib_qp_grp *qp_grp,
 
 	flow = usnic_fwd_alloc_flow(qp_grp->ufdev, &filter, &uaction);
 	if (IS_ERR_OR_NULL(flow)) {
-		usnic_err("Unable to alloc flow failed with err %ld\n",
-				PTR_ERR(flow));
 		err = flow ? PTR_ERR(flow) : -EFAULT;
 		goto out_unreserve_port;
 	}
@@ -303,8 +301,6 @@ create_udp_flow(struct usnic_ib_qp_grp *qp_grp,
 
 	flow = usnic_fwd_alloc_flow(qp_grp->ufdev, &filter, &uaction);
 	if (IS_ERR_OR_NULL(flow)) {
-		usnic_err("Unable to alloc flow failed with err %ld\n",
-				PTR_ERR(flow));
 		err = flow ? PTR_ERR(flow) : -EFAULT;
 		goto out_put_sock;
 	}
@@ -694,18 +690,14 @@ usnic_ib_qp_grp_create(struct usnic_fwd_dev *ufdev, struct usnic_ib_vf *vf,
 	}
 
 	qp_grp = kzalloc(sizeof(*qp_grp), GFP_ATOMIC);
-	if (!qp_grp) {
-		usnic_err("Unable to alloc qp_grp - Out of memory\n");
+	if (!qp_grp)
 		return NULL;
-	}
 
 	qp_grp->res_chunk_list = alloc_res_chunk_list(vf->vnic, res_spec,
 							qp_grp);
 	if (IS_ERR_OR_NULL(qp_grp->res_chunk_list)) {
 		err = qp_grp->res_chunk_list ?
 				PTR_ERR(qp_grp->res_chunk_list) : -ENOMEM;
-		usnic_err("Unable to alloc res for %d with err %d\n",
-				qp_grp->grp_id, err);
 		goto out_free_qp_grp;
 	}
 
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index a5bfbba6bbac..74819a7951e2 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -87,12 +87,12 @@ static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp,
 	resp.bar_len = bar->len;
 
 	chunk = usnic_ib_qp_grp_get_chunk(qp_grp, USNIC_VNIC_RES_TYPE_RQ);
-	if (IS_ERR_OR_NULL(chunk)) {
+	if (IS_ERR(chunk)) {
 		usnic_err("Failed to get chunk %s for qp_grp %d with err %ld\n",
 			usnic_vnic_res_type_to_str(USNIC_VNIC_RES_TYPE_RQ),
 			qp_grp->grp_id,
 			PTR_ERR(chunk));
-		return chunk ? PTR_ERR(chunk) : -ENOMEM;
+		return PTR_ERR(chunk);
 	}
 
 	WARN_ON(chunk->type != USNIC_VNIC_RES_TYPE_RQ);
@@ -101,12 +101,12 @@ static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp,
 		resp.rq_idx[i] = chunk->res[i]->vnic_idx;
 
 	chunk = usnic_ib_qp_grp_get_chunk(qp_grp, USNIC_VNIC_RES_TYPE_WQ);
-	if (IS_ERR_OR_NULL(chunk)) {
+	if (IS_ERR(chunk)) {
 		usnic_err("Failed to get chunk %s for qp_grp %d with err %ld\n",
 			usnic_vnic_res_type_to_str(USNIC_VNIC_RES_TYPE_WQ),
 			qp_grp->grp_id,
 			PTR_ERR(chunk));
-		return chunk ? PTR_ERR(chunk) : -ENOMEM;
+		return PTR_ERR(chunk);
 	}
 
 	WARN_ON(chunk->type != USNIC_VNIC_RES_TYPE_WQ);
@@ -115,12 +115,12 @@ static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp,
 		resp.wq_idx[i] = chunk->res[i]->vnic_idx;
 
 	chunk = usnic_ib_qp_grp_get_chunk(qp_grp, USNIC_VNIC_RES_TYPE_CQ);
-	if (IS_ERR_OR_NULL(chunk)) {
+	if (IS_ERR(chunk)) {
 		usnic_err("Failed to get chunk %s for qp_grp %d with err %ld\n",
 			usnic_vnic_res_type_to_str(USNIC_VNIC_RES_TYPE_CQ),
 			qp_grp->grp_id,
 			PTR_ERR(chunk));
-		return chunk ? PTR_ERR(chunk) : -ENOMEM;
+		return PTR_ERR(chunk);
 	}
 
 	WARN_ON(chunk->type != USNIC_VNIC_RES_TYPE_CQ);
@@ -738,7 +738,9 @@ int usnic_ib_mmap(struct ib_ucontext *context,
 
 /* In ib callbacks section -  Start of stub funcs */
 struct ib_ah *usnic_ib_create_ah(struct ib_pd *pd,
-					struct ib_ah_attr *ah_attr)
+				 struct ib_ah_attr *ah_attr,
+				 struct ib_udata *udata)
+
 {
 	usnic_dbg("\n");
 	return ERR_PTR(-EPERM);
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
index 0d9d2e6a14d5..0ed8e072329e 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
@@ -75,7 +75,9 @@ int usnic_ib_dealloc_ucontext(struct ib_ucontext *ibcontext);
 int usnic_ib_mmap(struct ib_ucontext *context,
 			struct vm_area_struct *vma);
 struct ib_ah *usnic_ib_create_ah(struct ib_pd *pd,
-					struct ib_ah_attr *ah_attr);
+				 struct ib_ah_attr *ah_attr,
+				 struct ib_udata *udata);
+
 int usnic_ib_destroy_ah(struct ib_ah *ah);
 int usnic_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 			struct ib_send_wr **bad_wr);
diff --git a/drivers/infiniband/hw/usnic/usnic_vnic.c b/drivers/infiniband/hw/usnic/usnic_vnic.c
index 887510718690..e7b0030254da 100644
--- a/drivers/infiniband/hw/usnic/usnic_vnic.c
+++ b/drivers/infiniband/hw/usnic/usnic_vnic.c
@@ -241,17 +241,12 @@ usnic_vnic_get_resources(struct usnic_vnic *vnic, enum usnic_vnic_res_type type,
 		return ERR_PTR(-EINVAL);
 
 	ret = kzalloc(sizeof(*ret), GFP_ATOMIC);
-	if (!ret) {
-		usnic_err("Failed to allocate chunk for %s - Out of memory\n",
-				usnic_vnic_pci_name(vnic));
+	if (!ret)
 		return ERR_PTR(-ENOMEM);
-	}
 
 	if (cnt > 0) {
 		ret->res = kcalloc(cnt, sizeof(*(ret->res)), GFP_ATOMIC);
 		if (!ret->res) {
-			usnic_err("Failed to allocate resources for %s. Out of memory\n",
-					usnic_vnic_pci_name(vnic));
 			kfree(ret);
 			return ERR_PTR(-ENOMEM);
 		}
@@ -311,8 +306,10 @@ static int usnic_vnic_alloc_res_chunk(struct usnic_vnic *vnic,
 	struct usnic_vnic_res *res;
 
 	cnt = vnic_dev_get_res_count(vnic->vdev, _to_vnic_res_type(type));
-	if (cnt < 1)
+	if (cnt < 1) {
+		usnic_err("Wrong res count with cnt %d\n", cnt);
 		return -EINVAL;
+	}
 
 	chunk->cnt = chunk->free_cnt = cnt;
 	chunk->res = kzalloc(sizeof(*(chunk->res))*cnt, GFP_KERNEL);
@@ -384,12 +381,8 @@ static int usnic_vnic_discover_resources(struct pci_dev *pdev,
 			res_type < USNIC_VNIC_RES_TYPE_MAX; res_type++) {
 		err = usnic_vnic_alloc_res_chunk(vnic, res_type,
 						&vnic->chunks[res_type]);
-		if (err) {
-			usnic_err("Failed to alloc res %s with err %d\n",
-					usnic_vnic_res_type_to_str(res_type),
-					err);
+		if (err)
 			goto out_clean_chunks;
-		}
 	}
 
 	return 0;
@@ -454,11 +447,8 @@ struct usnic_vnic *usnic_vnic_alloc(struct pci_dev *pdev)
 	}
 
 	vnic = kzalloc(sizeof(*vnic), GFP_KERNEL);
-	if (!vnic) {
-		usnic_err("Failed to alloc vnic for %s - out of memory\n",
-				pci_name(pdev));
+	if (!vnic)
 		return ERR_PTR(-ENOMEM);
-	}
 
 	spin_lock_init(&vnic->res_lock);
 
diff --git a/drivers/infiniband/hw/vmw_pvrdma/Kconfig b/drivers/infiniband/hw/vmw_pvrdma/Kconfig
new file mode 100644
index 000000000000..5a9790ac0ede
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/Kconfig
@@ -0,0 +1,7 @@
+config INFINIBAND_VMWARE_PVRDMA
+	tristate "VMware Paravirtualized RDMA Driver"
+	depends on NETDEVICES && ETHERNET && PCI && INET && VMXNET3
+	---help---
+	  This driver provides low-level support for VMware Paravirtual
+	  RDMA adapter. It interacts with the VMXNet3 driver to provide
+	  Ethernet capabilities.
diff --git a/drivers/infiniband/hw/vmw_pvrdma/Makefile b/drivers/infiniband/hw/vmw_pvrdma/Makefile
new file mode 100644
index 000000000000..0194ed19f542
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/Makefile
@@ -0,0 +1,3 @@
+obj-$(CONFIG_INFINIBAND_VMWARE_PVRDMA) += vmw_pvrdma.o
+
+vmw_pvrdma-y := pvrdma_cmd.o pvrdma_cq.o pvrdma_doorbell.o pvrdma_main.o pvrdma_misc.o pvrdma_mr.o pvrdma_qp.o pvrdma_verbs.o
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
new file mode 100644
index 000000000000..71e1d55d69d6
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
@@ -0,0 +1,474 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __PVRDMA_H__
+#define __PVRDMA_H__
+
+#include <linux/compiler.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/semaphore.h>
+#include <linux/workqueue.h>
+#include <rdma/ib_umem.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/vmw_pvrdma-abi.h>
+
+#include "pvrdma_ring.h"
+#include "pvrdma_dev_api.h"
+#include "pvrdma_verbs.h"
+
+/* NOT the same as BIT_MASK(). */
+#define PVRDMA_MASK(n) ((n << 1) - 1)
+
+/*
+ * VMware PVRDMA PCI device id.
+ */
+#define PCI_DEVICE_ID_VMWARE_PVRDMA	0x0820
+
+struct pvrdma_dev;
+
+struct pvrdma_page_dir {
+	dma_addr_t dir_dma;
+	u64 *dir;
+	int ntables;
+	u64 **tables;
+	u64 npages;
+	void **pages;
+};
+
+struct pvrdma_cq {
+	struct ib_cq ibcq;
+	int offset;
+	spinlock_t cq_lock; /* Poll lock. */
+	struct pvrdma_uar_map *uar;
+	struct ib_umem *umem;
+	struct pvrdma_ring_state *ring_state;
+	struct pvrdma_page_dir pdir;
+	u32 cq_handle;
+	bool is_kernel;
+	atomic_t refcnt;
+	wait_queue_head_t wait;
+};
+
+struct pvrdma_id_table {
+	u32 last;
+	u32 top;
+	u32 max;
+	u32 mask;
+	spinlock_t lock; /* Table lock. */
+	unsigned long *table;
+};
+
+struct pvrdma_uar_map {
+	unsigned long pfn;
+	void __iomem *map;
+	int index;
+};
+
+struct pvrdma_uar_table {
+	struct pvrdma_id_table tbl;
+	int size;
+};
+
+struct pvrdma_ucontext {
+	struct ib_ucontext ibucontext;
+	struct pvrdma_dev *dev;
+	struct pvrdma_uar_map uar;
+	u64 ctx_handle;
+};
+
+struct pvrdma_pd {
+	struct ib_pd ibpd;
+	u32 pdn;
+	u32 pd_handle;
+	int privileged;
+};
+
+struct pvrdma_mr {
+	u32 mr_handle;
+	u64 iova;
+	u64 size;
+};
+
+struct pvrdma_user_mr {
+	struct ib_mr ibmr;
+	struct ib_umem *umem;
+	struct pvrdma_mr mmr;
+	struct pvrdma_page_dir pdir;
+	u64 *pages;
+	u32 npages;
+	u32 max_pages;
+	u32 page_shift;
+};
+
+struct pvrdma_wq {
+	struct pvrdma_ring *ring;
+	spinlock_t lock; /* Work queue lock. */
+	int wqe_cnt;
+	int wqe_size;
+	int max_sg;
+	int offset;
+};
+
+struct pvrdma_ah {
+	struct ib_ah ibah;
+	struct pvrdma_av av;
+};
+
+struct pvrdma_qp {
+	struct ib_qp ibqp;
+	u32 qp_handle;
+	u32 qkey;
+	struct pvrdma_wq sq;
+	struct pvrdma_wq rq;
+	struct ib_umem *rumem;
+	struct ib_umem *sumem;
+	struct pvrdma_page_dir pdir;
+	int npages;
+	int npages_send;
+	int npages_recv;
+	u32 flags;
+	u8 port;
+	u8 state;
+	bool is_kernel;
+	struct mutex mutex; /* QP state mutex. */
+	atomic_t refcnt;
+	wait_queue_head_t wait;
+};
+
+struct pvrdma_dev {
+	/* PCI device-related information. */
+	struct ib_device ib_dev;
+	struct pci_dev *pdev;
+	void __iomem *regs;
+	struct pvrdma_device_shared_region *dsr; /* Shared region pointer */
+	dma_addr_t dsrbase; /* Shared region base address */
+	void *cmd_slot;
+	void *resp_slot;
+	unsigned long flags;
+	struct list_head device_link;
+
+	/* Locking and interrupt information. */
+	spinlock_t cmd_lock; /* Command lock. */
+	struct semaphore cmd_sema;
+	struct completion cmd_done;
+	struct {
+		enum pvrdma_intr_type type; /* Intr type */
+		struct msix_entry msix_entry[PVRDMA_MAX_INTERRUPTS];
+		irq_handler_t handler[PVRDMA_MAX_INTERRUPTS];
+		u8 enabled[PVRDMA_MAX_INTERRUPTS];
+		u8 size;
+	} intr;
+
+	/* RDMA-related device information. */
+	union ib_gid *sgid_tbl;
+	struct pvrdma_ring_state *async_ring_state;
+	struct pvrdma_page_dir async_pdir;
+	struct pvrdma_ring_state *cq_ring_state;
+	struct pvrdma_page_dir cq_pdir;
+	struct pvrdma_cq **cq_tbl;
+	spinlock_t cq_tbl_lock;
+	struct pvrdma_qp **qp_tbl;
+	spinlock_t qp_tbl_lock;
+	struct pvrdma_uar_table uar_table;
+	struct pvrdma_uar_map driver_uar;
+	__be64 sys_image_guid;
+	spinlock_t desc_lock; /* Device modification lock. */
+	u32 port_cap_mask;
+	struct mutex port_mutex; /* Port modification mutex. */
+	bool ib_active;
+	atomic_t num_qps;
+	atomic_t num_cqs;
+	atomic_t num_pds;
+	atomic_t num_ahs;
+
+	/* Network device information. */
+	struct net_device *netdev;
+	struct notifier_block nb_netdev;
+};
+
+struct pvrdma_netdevice_work {
+	struct work_struct work;
+	struct net_device *event_netdev;
+	unsigned long event;
+};
+
+static inline struct pvrdma_dev *to_vdev(struct ib_device *ibdev)
+{
+	return container_of(ibdev, struct pvrdma_dev, ib_dev);
+}
+
+static inline struct
+pvrdma_ucontext *to_vucontext(struct ib_ucontext *ibucontext)
+{
+	return container_of(ibucontext, struct pvrdma_ucontext, ibucontext);
+}
+
+static inline struct pvrdma_pd *to_vpd(struct ib_pd *ibpd)
+{
+	return container_of(ibpd, struct pvrdma_pd, ibpd);
+}
+
+static inline struct pvrdma_cq *to_vcq(struct ib_cq *ibcq)
+{
+	return container_of(ibcq, struct pvrdma_cq, ibcq);
+}
+
+static inline struct pvrdma_user_mr *to_vmr(struct ib_mr *ibmr)
+{
+	return container_of(ibmr, struct pvrdma_user_mr, ibmr);
+}
+
+static inline struct pvrdma_qp *to_vqp(struct ib_qp *ibqp)
+{
+	return container_of(ibqp, struct pvrdma_qp, ibqp);
+}
+
+static inline struct pvrdma_ah *to_vah(struct ib_ah *ibah)
+{
+	return container_of(ibah, struct pvrdma_ah, ibah);
+}
+
+static inline void pvrdma_write_reg(struct pvrdma_dev *dev, u32 reg, u32 val)
+{
+	writel(cpu_to_le32(val), dev->regs + reg);
+}
+
+static inline u32 pvrdma_read_reg(struct pvrdma_dev *dev, u32 reg)
+{
+	return le32_to_cpu(readl(dev->regs + reg));
+}
+
+static inline void pvrdma_write_uar_cq(struct pvrdma_dev *dev, u32 val)
+{
+	writel(cpu_to_le32(val), dev->driver_uar.map + PVRDMA_UAR_CQ_OFFSET);
+}
+
+static inline void pvrdma_write_uar_qp(struct pvrdma_dev *dev, u32 val)
+{
+	writel(cpu_to_le32(val), dev->driver_uar.map + PVRDMA_UAR_QP_OFFSET);
+}
+
+static inline void *pvrdma_page_dir_get_ptr(struct pvrdma_page_dir *pdir,
+					    u64 offset)
+{
+	return pdir->pages[offset / PAGE_SIZE] + (offset % PAGE_SIZE);
+}
+
+static inline enum pvrdma_mtu ib_mtu_to_pvrdma(enum ib_mtu mtu)
+{
+	return (enum pvrdma_mtu)mtu;
+}
+
+static inline enum ib_mtu pvrdma_mtu_to_ib(enum pvrdma_mtu mtu)
+{
+	return (enum ib_mtu)mtu;
+}
+
+static inline enum pvrdma_port_state ib_port_state_to_pvrdma(
+					enum ib_port_state state)
+{
+	return (enum pvrdma_port_state)state;
+}
+
+static inline enum ib_port_state pvrdma_port_state_to_ib(
+					enum pvrdma_port_state state)
+{
+	return (enum ib_port_state)state;
+}
+
+static inline int ib_port_cap_flags_to_pvrdma(int flags)
+{
+	return flags & PVRDMA_MASK(PVRDMA_PORT_CAP_FLAGS_MAX);
+}
+
+static inline int pvrdma_port_cap_flags_to_ib(int flags)
+{
+	return flags;
+}
+
+static inline enum pvrdma_port_width ib_port_width_to_pvrdma(
+					enum ib_port_width width)
+{
+	return (enum pvrdma_port_width)width;
+}
+
+static inline enum ib_port_width pvrdma_port_width_to_ib(
+					enum pvrdma_port_width width)
+{
+	return (enum ib_port_width)width;
+}
+
+static inline enum pvrdma_port_speed ib_port_speed_to_pvrdma(
+					enum ib_port_speed speed)
+{
+	return (enum pvrdma_port_speed)speed;
+}
+
+static inline enum ib_port_speed pvrdma_port_speed_to_ib(
+					enum pvrdma_port_speed speed)
+{
+	return (enum ib_port_speed)speed;
+}
+
+static inline int pvrdma_qp_attr_mask_to_ib(int attr_mask)
+{
+	return attr_mask;
+}
+
+static inline int ib_qp_attr_mask_to_pvrdma(int attr_mask)
+{
+	return attr_mask & PVRDMA_MASK(PVRDMA_QP_ATTR_MASK_MAX);
+}
+
+static inline enum pvrdma_mig_state ib_mig_state_to_pvrdma(
+					enum ib_mig_state state)
+{
+	return (enum pvrdma_mig_state)state;
+}
+
+static inline enum ib_mig_state pvrdma_mig_state_to_ib(
+					enum pvrdma_mig_state state)
+{
+	return (enum ib_mig_state)state;
+}
+
+static inline int ib_access_flags_to_pvrdma(int flags)
+{
+	return flags;
+}
+
+static inline int pvrdma_access_flags_to_ib(int flags)
+{
+	return flags & PVRDMA_MASK(PVRDMA_ACCESS_FLAGS_MAX);
+}
+
+static inline enum pvrdma_qp_type ib_qp_type_to_pvrdma(enum ib_qp_type type)
+{
+	return (enum pvrdma_qp_type)type;
+}
+
+static inline enum ib_qp_type pvrdma_qp_type_to_ib(enum pvrdma_qp_type type)
+{
+	return (enum ib_qp_type)type;
+}
+
+static inline enum pvrdma_qp_state ib_qp_state_to_pvrdma(enum ib_qp_state state)
+{
+	return (enum pvrdma_qp_state)state;
+}
+
+static inline enum ib_qp_state pvrdma_qp_state_to_ib(enum pvrdma_qp_state state)
+{
+	return (enum ib_qp_state)state;
+}
+
+static inline enum pvrdma_wr_opcode ib_wr_opcode_to_pvrdma(enum ib_wr_opcode op)
+{
+	return (enum pvrdma_wr_opcode)op;
+}
+
+static inline enum ib_wc_status pvrdma_wc_status_to_ib(
+					enum pvrdma_wc_status status)
+{
+	return (enum ib_wc_status)status;
+}
+
+static inline int pvrdma_wc_opcode_to_ib(int opcode)
+{
+	return opcode;
+}
+
+static inline int pvrdma_wc_flags_to_ib(int flags)
+{
+	return flags;
+}
+
+static inline int ib_send_flags_to_pvrdma(int flags)
+{
+	return flags & PVRDMA_MASK(PVRDMA_SEND_FLAGS_MAX);
+}
+
+void pvrdma_qp_cap_to_ib(struct ib_qp_cap *dst,
+			 const struct pvrdma_qp_cap *src);
+void ib_qp_cap_to_pvrdma(struct pvrdma_qp_cap *dst,
+			 const struct ib_qp_cap *src);
+void pvrdma_gid_to_ib(union ib_gid *dst, const union pvrdma_gid *src);
+void ib_gid_to_pvrdma(union pvrdma_gid *dst, const union ib_gid *src);
+void pvrdma_global_route_to_ib(struct ib_global_route *dst,
+			       const struct pvrdma_global_route *src);
+void ib_global_route_to_pvrdma(struct pvrdma_global_route *dst,
+			       const struct ib_global_route *src);
+void pvrdma_ah_attr_to_ib(struct ib_ah_attr *dst,
+			  const struct pvrdma_ah_attr *src);
+void ib_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
+			  const struct ib_ah_attr *src);
+
+int pvrdma_uar_table_init(struct pvrdma_dev *dev);
+void pvrdma_uar_table_cleanup(struct pvrdma_dev *dev);
+
+int pvrdma_uar_alloc(struct pvrdma_dev *dev, struct pvrdma_uar_map *uar);
+void pvrdma_uar_free(struct pvrdma_dev *dev, struct pvrdma_uar_map *uar);
+
+void _pvrdma_flush_cqe(struct pvrdma_qp *qp, struct pvrdma_cq *cq);
+
+int pvrdma_page_dir_init(struct pvrdma_dev *dev, struct pvrdma_page_dir *pdir,
+			 u64 npages, bool alloc_pages);
+void pvrdma_page_dir_cleanup(struct pvrdma_dev *dev,
+			     struct pvrdma_page_dir *pdir);
+int pvrdma_page_dir_insert_dma(struct pvrdma_page_dir *pdir, u64 idx,
+			       dma_addr_t daddr);
+int pvrdma_page_dir_insert_umem(struct pvrdma_page_dir *pdir,
+				struct ib_umem *umem, u64 offset);
+dma_addr_t pvrdma_page_dir_get_dma(struct pvrdma_page_dir *pdir, u64 idx);
+int pvrdma_page_dir_insert_page_list(struct pvrdma_page_dir *pdir,
+				     u64 *page_list, int num_pages);
+
+int pvrdma_cmd_post(struct pvrdma_dev *dev, union pvrdma_cmd_req *req,
+		    union pvrdma_cmd_resp *rsp, unsigned resp_code);
+
+#endif /* __PVRDMA_H__ */
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cmd.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cmd.c
new file mode 100644
index 000000000000..4a78c537d8a1
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cmd.c
@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/list.h>
+
+#include "pvrdma.h"
+
+#define PVRDMA_CMD_TIMEOUT	10000 /* ms */
+
+static inline int pvrdma_cmd_recv(struct pvrdma_dev *dev,
+				  union pvrdma_cmd_resp *resp,
+				  unsigned resp_code)
+{
+	int err;
+
+	dev_dbg(&dev->pdev->dev, "receive response from device\n");
+
+	err = wait_for_completion_interruptible_timeout(&dev->cmd_done,
+			msecs_to_jiffies(PVRDMA_CMD_TIMEOUT));
+	if (err == 0 || err == -ERESTARTSYS) {
+		dev_warn(&dev->pdev->dev,
+			 "completion timeout or interrupted\n");
+		return -ETIMEDOUT;
+	}
+
+	spin_lock(&dev->cmd_lock);
+	memcpy(resp, dev->resp_slot, sizeof(*resp));
+	spin_unlock(&dev->cmd_lock);
+
+	if (resp->hdr.ack != resp_code) {
+		dev_warn(&dev->pdev->dev,
+			 "unknown response %#x expected %#x\n",
+			 resp->hdr.ack, resp_code);
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+int
+pvrdma_cmd_post(struct pvrdma_dev *dev, union pvrdma_cmd_req *req,
+		union pvrdma_cmd_resp *resp, unsigned resp_code)
+{
+	int err;
+
+	dev_dbg(&dev->pdev->dev, "post request to device\n");
+
+	/* Serializiation */
+	down(&dev->cmd_sema);
+
+	BUILD_BUG_ON(sizeof(union pvrdma_cmd_req) !=
+		     sizeof(struct pvrdma_cmd_modify_qp));
+
+	spin_lock(&dev->cmd_lock);
+	memcpy(dev->cmd_slot, req, sizeof(*req));
+	spin_unlock(&dev->cmd_lock);
+
+	init_completion(&dev->cmd_done);
+	pvrdma_write_reg(dev, PVRDMA_REG_REQUEST, 0);
+
+	/* Make sure the request is written before reading status. */
+	mb();
+
+	err = pvrdma_read_reg(dev, PVRDMA_REG_ERR);
+	if (err == 0) {
+		if (resp != NULL)
+			err = pvrdma_cmd_recv(dev, resp, resp_code);
+	} else {
+		dev_warn(&dev->pdev->dev,
+			 "failed to write request error reg: %d\n", err);
+		err = -EFAULT;
+	}
+
+	up(&dev->cmd_sema);
+
+	return err;
+}
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
new file mode 100644
index 000000000000..e429ca5b16aa
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
@@ -0,0 +1,425 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <asm/page.h>
+#include <linux/io.h>
+#include <linux/wait.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_smi.h>
+#include <rdma/ib_user_verbs.h>
+
+#include "pvrdma.h"
+
+/**
+ * pvrdma_req_notify_cq - request notification for a completion queue
+ * @ibcq: the completion queue
+ * @notify_flags: notification flags
+ *
+ * @return: 0 for success.
+ */
+int pvrdma_req_notify_cq(struct ib_cq *ibcq,
+			 enum ib_cq_notify_flags notify_flags)
+{
+	struct pvrdma_dev *dev = to_vdev(ibcq->device);
+	struct pvrdma_cq *cq = to_vcq(ibcq);
+	u32 val = cq->cq_handle;
+
+	val |= (notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
+		PVRDMA_UAR_CQ_ARM_SOL : PVRDMA_UAR_CQ_ARM;
+
+	pvrdma_write_uar_cq(dev, val);
+
+	return 0;
+}
+
+/**
+ * pvrdma_create_cq - create completion queue
+ * @ibdev: the device
+ * @attr: completion queue attributes
+ * @context: user context
+ * @udata: user data
+ *
+ * @return: ib_cq completion queue pointer on success,
+ *          otherwise returns negative errno.
+ */
+struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
+			       const struct ib_cq_init_attr *attr,
+			       struct ib_ucontext *context,
+			       struct ib_udata *udata)
+{
+	int entries = attr->cqe;
+	struct pvrdma_dev *dev = to_vdev(ibdev);
+	struct pvrdma_cq *cq;
+	int ret;
+	int npages;
+	unsigned long flags;
+	union pvrdma_cmd_req req;
+	union pvrdma_cmd_resp rsp;
+	struct pvrdma_cmd_create_cq *cmd = &req.create_cq;
+	struct pvrdma_cmd_create_cq_resp *resp = &rsp.create_cq_resp;
+	struct pvrdma_create_cq ucmd;
+
+	BUILD_BUG_ON(sizeof(struct pvrdma_cqe) != 64);
+
+	entries = roundup_pow_of_two(entries);
+	if (entries < 1 || entries > dev->dsr->caps.max_cqe)
+		return ERR_PTR(-EINVAL);
+
+	if (!atomic_add_unless(&dev->num_cqs, 1, dev->dsr->caps.max_cq))
+		return ERR_PTR(-ENOMEM);
+
+	cq = kzalloc(sizeof(*cq), GFP_KERNEL);
+	if (!cq) {
+		atomic_dec(&dev->num_cqs);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	cq->ibcq.cqe = entries;
+
+	if (context) {
+		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
+			ret = -EFAULT;
+			goto err_cq;
+		}
+
+		cq->umem = ib_umem_get(context, ucmd.buf_addr, ucmd.buf_size,
+				       IB_ACCESS_LOCAL_WRITE, 1);
+		if (IS_ERR(cq->umem)) {
+			ret = PTR_ERR(cq->umem);
+			goto err_cq;
+		}
+
+		npages = ib_umem_page_count(cq->umem);
+	} else {
+		cq->is_kernel = true;
+
+		/* One extra page for shared ring state */
+		npages = 1 + (entries * sizeof(struct pvrdma_cqe) +
+			      PAGE_SIZE - 1) / PAGE_SIZE;
+
+		/* Skip header page. */
+		cq->offset = PAGE_SIZE;
+	}
+
+	if (npages < 0 || npages > PVRDMA_PAGE_DIR_MAX_PAGES) {
+		dev_warn(&dev->pdev->dev,
+			 "overflow pages in completion queue\n");
+		ret = -EINVAL;
+		goto err_umem;
+	}
+
+	ret = pvrdma_page_dir_init(dev, &cq->pdir, npages, cq->is_kernel);
+	if (ret) {
+		dev_warn(&dev->pdev->dev,
+			 "could not allocate page directory\n");
+		goto err_umem;
+	}
+
+	/* Ring state is always the first page. Set in library for user cq. */
+	if (cq->is_kernel)
+		cq->ring_state = cq->pdir.pages[0];
+	else
+		pvrdma_page_dir_insert_umem(&cq->pdir, cq->umem, 0);
+
+	atomic_set(&cq->refcnt, 1);
+	init_waitqueue_head(&cq->wait);
+	spin_lock_init(&cq->cq_lock);
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_CREATE_CQ;
+	cmd->nchunks = npages;
+	cmd->ctx_handle = (context) ?
+		(u64)to_vucontext(context)->ctx_handle : 0;
+	cmd->cqe = entries;
+	cmd->pdir_dma = cq->pdir.dir_dma;
+	ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_CREATE_CQ_RESP);
+	if (ret < 0) {
+		dev_warn(&dev->pdev->dev,
+			 "could not create completion queue, error: %d\n", ret);
+		goto err_page_dir;
+	}
+
+	cq->ibcq.cqe = resp->cqe;
+	cq->cq_handle = resp->cq_handle;
+	spin_lock_irqsave(&dev->cq_tbl_lock, flags);
+	dev->cq_tbl[cq->cq_handle % dev->dsr->caps.max_cq] = cq;
+	spin_unlock_irqrestore(&dev->cq_tbl_lock, flags);
+
+	if (context) {
+		cq->uar = &(to_vucontext(context)->uar);
+
+		/* Copy udata back. */
+		if (ib_copy_to_udata(udata, &cq->cq_handle, sizeof(__u32))) {
+			dev_warn(&dev->pdev->dev,
+				 "failed to copy back udata\n");
+			pvrdma_destroy_cq(&cq->ibcq);
+			return ERR_PTR(-EINVAL);
+		}
+	}
+
+	return &cq->ibcq;
+
+err_page_dir:
+	pvrdma_page_dir_cleanup(dev, &cq->pdir);
+err_umem:
+	if (context)
+		ib_umem_release(cq->umem);
+err_cq:
+	atomic_dec(&dev->num_cqs);
+	kfree(cq);
+
+	return ERR_PTR(ret);
+}
+
+static void pvrdma_free_cq(struct pvrdma_dev *dev, struct pvrdma_cq *cq)
+{
+	atomic_dec(&cq->refcnt);
+	wait_event(cq->wait, !atomic_read(&cq->refcnt));
+
+	if (!cq->is_kernel)
+		ib_umem_release(cq->umem);
+
+	pvrdma_page_dir_cleanup(dev, &cq->pdir);
+	kfree(cq);
+}
+
+/**
+ * pvrdma_destroy_cq - destroy completion queue
+ * @cq: the completion queue to destroy.
+ *
+ * @return: 0 for success.
+ */
+int pvrdma_destroy_cq(struct ib_cq *cq)
+{
+	struct pvrdma_cq *vcq = to_vcq(cq);
+	union pvrdma_cmd_req req;
+	struct pvrdma_cmd_destroy_cq *cmd = &req.destroy_cq;
+	struct pvrdma_dev *dev = to_vdev(cq->device);
+	unsigned long flags;
+	int ret;
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_DESTROY_CQ;
+	cmd->cq_handle = vcq->cq_handle;
+
+	ret = pvrdma_cmd_post(dev, &req, NULL, 0);
+	if (ret < 0)
+		dev_warn(&dev->pdev->dev,
+			 "could not destroy completion queue, error: %d\n",
+			 ret);
+
+	/* free cq's resources */
+	spin_lock_irqsave(&dev->cq_tbl_lock, flags);
+	dev->cq_tbl[vcq->cq_handle] = NULL;
+	spin_unlock_irqrestore(&dev->cq_tbl_lock, flags);
+
+	pvrdma_free_cq(dev, vcq);
+	atomic_dec(&dev->num_cqs);
+
+	return ret;
+}
+
+/**
+ * pvrdma_modify_cq - modify the CQ moderation parameters
+ * @ibcq: the CQ to modify
+ * @cq_count: number of CQEs that will trigger an event
+ * @cq_period: max period of time in usec before triggering an event
+ *
+ * @return: -EOPNOTSUPP as CQ resize is not supported.
+ */
+int pvrdma_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline struct pvrdma_cqe *get_cqe(struct pvrdma_cq *cq, int i)
+{
+	return (struct pvrdma_cqe *)pvrdma_page_dir_get_ptr(
+					&cq->pdir,
+					cq->offset +
+					sizeof(struct pvrdma_cqe) * i);
+}
+
+void _pvrdma_flush_cqe(struct pvrdma_qp *qp, struct pvrdma_cq *cq)
+{
+	int head;
+	int has_data;
+
+	if (!cq->is_kernel)
+		return;
+
+	/* Lock held */
+	has_data = pvrdma_idx_ring_has_data(&cq->ring_state->rx,
+					    cq->ibcq.cqe, &head);
+	if (unlikely(has_data > 0)) {
+		int items;
+		int curr;
+		int tail = pvrdma_idx(&cq->ring_state->rx.prod_tail,
+				      cq->ibcq.cqe);
+		struct pvrdma_cqe *cqe;
+		struct pvrdma_cqe *curr_cqe;
+
+		items = (tail > head) ? (tail - head) :
+			(cq->ibcq.cqe - head + tail);
+		curr = --tail;
+		while (items-- > 0) {
+			if (curr < 0)
+				curr = cq->ibcq.cqe - 1;
+			if (tail < 0)
+				tail = cq->ibcq.cqe - 1;
+			curr_cqe = get_cqe(cq, curr);
+			if ((curr_cqe->qp & 0xFFFF) != qp->qp_handle) {
+				if (curr != tail) {
+					cqe = get_cqe(cq, tail);
+					*cqe = *curr_cqe;
+				}
+				tail--;
+			} else {
+				pvrdma_idx_ring_inc(
+					&cq->ring_state->rx.cons_head,
+					cq->ibcq.cqe);
+			}
+			curr--;
+		}
+	}
+}
+
+static int pvrdma_poll_one(struct pvrdma_cq *cq, struct pvrdma_qp **cur_qp,
+			   struct ib_wc *wc)
+{
+	struct pvrdma_dev *dev = to_vdev(cq->ibcq.device);
+	int has_data;
+	unsigned int head;
+	bool tried = false;
+	struct pvrdma_cqe *cqe;
+
+retry:
+	has_data = pvrdma_idx_ring_has_data(&cq->ring_state->rx,
+					    cq->ibcq.cqe, &head);
+	if (has_data == 0) {
+		if (tried)
+			return -EAGAIN;
+
+		pvrdma_write_uar_cq(dev, cq->cq_handle | PVRDMA_UAR_CQ_POLL);
+
+		tried = true;
+		goto retry;
+	} else if (has_data == PVRDMA_INVALID_IDX) {
+		dev_err(&dev->pdev->dev, "CQ ring state invalid\n");
+		return -EAGAIN;
+	}
+
+	cqe = get_cqe(cq, head);
+
+	/* Ensure cqe is valid. */
+	rmb();
+	if (dev->qp_tbl[cqe->qp & 0xffff])
+		*cur_qp = (struct pvrdma_qp *)dev->qp_tbl[cqe->qp & 0xffff];
+	else
+		return -EAGAIN;
+
+	wc->opcode = pvrdma_wc_opcode_to_ib(cqe->opcode);
+	wc->status = pvrdma_wc_status_to_ib(cqe->status);
+	wc->wr_id = cqe->wr_id;
+	wc->qp = &(*cur_qp)->ibqp;
+	wc->byte_len = cqe->byte_len;
+	wc->ex.imm_data = cqe->imm_data;
+	wc->src_qp = cqe->src_qp;
+	wc->wc_flags = pvrdma_wc_flags_to_ib(cqe->wc_flags);
+	wc->pkey_index = cqe->pkey_index;
+	wc->slid = cqe->slid;
+	wc->sl = cqe->sl;
+	wc->dlid_path_bits = cqe->dlid_path_bits;
+	wc->port_num = cqe->port_num;
+	wc->vendor_err = 0;
+
+	/* Update shared ring state */
+	pvrdma_idx_ring_inc(&cq->ring_state->rx.cons_head, cq->ibcq.cqe);
+
+	return 0;
+}
+
+/**
+ * pvrdma_poll_cq - poll for work completion queue entries
+ * @ibcq: completion queue
+ * @num_entries: the maximum number of entries
+ * @entry: pointer to work completion array
+ *
+ * @return: number of polled completion entries
+ */
+int pvrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
+{
+	struct pvrdma_cq *cq = to_vcq(ibcq);
+	struct pvrdma_qp *cur_qp = NULL;
+	unsigned long flags;
+	int npolled;
+
+	if (num_entries < 1 || wc == NULL)
+		return 0;
+
+	spin_lock_irqsave(&cq->cq_lock, flags);
+	for (npolled = 0; npolled < num_entries; ++npolled) {
+		if (pvrdma_poll_one(cq, &cur_qp, wc + npolled))
+			break;
+	}
+
+	spin_unlock_irqrestore(&cq->cq_lock, flags);
+
+	/* Ensure we do not return errors from poll_cq */
+	return npolled;
+}
+
+/**
+ * pvrdma_resize_cq - resize CQ
+ * @ibcq: the completion queue
+ * @entries: CQ entries
+ * @udata: user data
+ *
+ * @return: -EOPNOTSUPP as CQ resize is not supported.
+ */
+int pvrdma_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
+{
+	return -EOPNOTSUPP;
+}
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
new file mode 100644
index 000000000000..c06768635d65
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
@@ -0,0 +1,586 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __PVRDMA_DEV_API_H__
+#define __PVRDMA_DEV_API_H__
+
+#include <linux/types.h>
+
+#include "pvrdma_verbs.h"
+
+#define PVRDMA_VERSION			17
+#define PVRDMA_BOARD_ID			1
+#define PVRDMA_REV_ID			1
+
+/*
+ * Masks and accessors for page directory, which is a two-level lookup:
+ * page directory -> page table -> page. Only one directory for now, but we
+ * could expand that easily. 9 bits for tables, 9 bits for pages, gives one
+ * gigabyte for memory regions and so forth.
+ */
+
+#define PVRDMA_PDIR_SHIFT		18
+#define PVRDMA_PTABLE_SHIFT		9
+#define PVRDMA_PAGE_DIR_DIR(x)		(((x) >> PVRDMA_PDIR_SHIFT) & 0x1)
+#define PVRDMA_PAGE_DIR_TABLE(x)	(((x) >> PVRDMA_PTABLE_SHIFT) & 0x1ff)
+#define PVRDMA_PAGE_DIR_PAGE(x)		((x) & 0x1ff)
+#define PVRDMA_PAGE_DIR_MAX_PAGES	(1 * 512 * 512)
+#define PVRDMA_MAX_FAST_REG_PAGES	128
+
+/*
+ * Max MSI-X vectors.
+ */
+
+#define PVRDMA_MAX_INTERRUPTS	3
+
+/* Register offsets within PCI resource on BAR1. */
+#define PVRDMA_REG_VERSION	0x00	/* R: Version of device. */
+#define PVRDMA_REG_DSRLOW	0x04	/* W: Device shared region low PA. */
+#define PVRDMA_REG_DSRHIGH	0x08	/* W: Device shared region high PA. */
+#define PVRDMA_REG_CTL		0x0c	/* W: PVRDMA_DEVICE_CTL */
+#define PVRDMA_REG_REQUEST	0x10	/* W: Indicate device request. */
+#define PVRDMA_REG_ERR		0x14	/* R: Device error. */
+#define PVRDMA_REG_ICR		0x18	/* R: Interrupt cause. */
+#define PVRDMA_REG_IMR		0x1c	/* R/W: Interrupt mask. */
+#define PVRDMA_REG_MACL		0x20	/* R/W: MAC address low. */
+#define PVRDMA_REG_MACH		0x24	/* R/W: MAC address high. */
+
+/* Object flags. */
+#define PVRDMA_CQ_FLAG_ARMED_SOL	BIT(0)	/* Armed for solicited-only. */
+#define PVRDMA_CQ_FLAG_ARMED		BIT(1)	/* Armed. */
+#define PVRDMA_MR_FLAG_DMA		BIT(0)	/* DMA region. */
+#define PVRDMA_MR_FLAG_FRMR		BIT(1)	/* Fast reg memory region. */
+
+/*
+ * Atomic operation capability (masked versions are extended atomic
+ * operations.
+ */
+
+#define PVRDMA_ATOMIC_OP_COMP_SWAP	BIT(0)	/* Compare and swap. */
+#define PVRDMA_ATOMIC_OP_FETCH_ADD	BIT(1)	/* Fetch and add. */
+#define PVRDMA_ATOMIC_OP_MASK_COMP_SWAP	BIT(2)	/* Masked compare and swap. */
+#define PVRDMA_ATOMIC_OP_MASK_FETCH_ADD	BIT(3)	/* Masked fetch and add. */
+
+/*
+ * Base Memory Management Extension flags to support Fast Reg Memory Regions
+ * and Fast Reg Work Requests. Each flag represents a verb operation and we
+ * must support all of them to qualify for the BMME device cap.
+ */
+
+#define PVRDMA_BMME_FLAG_LOCAL_INV	BIT(0)	/* Local Invalidate. */
+#define PVRDMA_BMME_FLAG_REMOTE_INV	BIT(1)	/* Remote Invalidate. */
+#define PVRDMA_BMME_FLAG_FAST_REG_WR	BIT(2)	/* Fast Reg Work Request. */
+
+/*
+ * GID types. The interpretation of the gid_types bit field in the device
+ * capabilities will depend on the device mode. For now, the device only
+ * supports RoCE as mode, so only the different GID types for RoCE are
+ * defined.
+ */
+
+#define PVRDMA_GID_TYPE_FLAG_ROCE_V1	BIT(0)
+#define PVRDMA_GID_TYPE_FLAG_ROCE_V2	BIT(1)
+
+enum pvrdma_pci_resource {
+	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
+	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */
+	PVRDMA_PCI_RESOURCE_UAR,	/* BAR2: UAR pages, MMIO, 64-bit. */
+	PVRDMA_PCI_RESOURCE_LAST,	/* Last. */
+};
+
+enum pvrdma_device_ctl {
+	PVRDMA_DEVICE_CTL_ACTIVATE,	/* Activate device. */
+	PVRDMA_DEVICE_CTL_QUIESCE,	/* Quiesce device. */
+	PVRDMA_DEVICE_CTL_RESET,	/* Reset device. */
+};
+
+enum pvrdma_intr_vector {
+	PVRDMA_INTR_VECTOR_RESPONSE,	/* Command response. */
+	PVRDMA_INTR_VECTOR_ASYNC,	/* Async events. */
+	PVRDMA_INTR_VECTOR_CQ,		/* CQ notification. */
+	/* Additional CQ notification vectors. */
+};
+
+enum pvrdma_intr_cause {
+	PVRDMA_INTR_CAUSE_RESPONSE	= (1 << PVRDMA_INTR_VECTOR_RESPONSE),
+	PVRDMA_INTR_CAUSE_ASYNC		= (1 << PVRDMA_INTR_VECTOR_ASYNC),
+	PVRDMA_INTR_CAUSE_CQ		= (1 << PVRDMA_INTR_VECTOR_CQ),
+};
+
+enum pvrdma_intr_type {
+	PVRDMA_INTR_TYPE_INTX,		/* Legacy. */
+	PVRDMA_INTR_TYPE_MSI,		/* MSI. */
+	PVRDMA_INTR_TYPE_MSIX,		/* MSI-X. */
+};
+
+enum pvrdma_gos_bits {
+	PVRDMA_GOS_BITS_UNK,		/* Unknown. */
+	PVRDMA_GOS_BITS_32,		/* 32-bit. */
+	PVRDMA_GOS_BITS_64,		/* 64-bit. */
+};
+
+enum pvrdma_gos_type {
+	PVRDMA_GOS_TYPE_UNK,		/* Unknown. */
+	PVRDMA_GOS_TYPE_LINUX,		/* Linux. */
+};
+
+enum pvrdma_device_mode {
+	PVRDMA_DEVICE_MODE_ROCE,	/* RoCE. */
+	PVRDMA_DEVICE_MODE_IWARP,	/* iWarp. */
+	PVRDMA_DEVICE_MODE_IB,		/* InfiniBand. */
+};
+
+struct pvrdma_gos_info {
+	u32 gos_bits:2;			/* W: PVRDMA_GOS_BITS_ */
+	u32 gos_type:4;			/* W: PVRDMA_GOS_TYPE_ */
+	u32 gos_ver:16;			/* W: Guest OS version. */
+	u32 gos_misc:10;		/* W: Other. */
+	u32 pad;			/* Pad to 8-byte alignment. */
+};
+
+struct pvrdma_device_caps {
+	u64 fw_ver;				/* R: Query device. */
+	__be64 node_guid;
+	__be64 sys_image_guid;
+	u64 max_mr_size;
+	u64 page_size_cap;
+	u64 atomic_arg_sizes;			/* EX verbs. */
+	u32 ex_comp_mask;			/* EX verbs. */
+	u32 device_cap_flags2;			/* EX verbs. */
+	u32 max_fa_bit_boundary;		/* EX verbs. */
+	u32 log_max_atomic_inline_arg;		/* EX verbs. */
+	u32 vendor_id;
+	u32 vendor_part_id;
+	u32 hw_ver;
+	u32 max_qp;
+	u32 max_qp_wr;
+	u32 device_cap_flags;
+	u32 max_sge;
+	u32 max_sge_rd;
+	u32 max_cq;
+	u32 max_cqe;
+	u32 max_mr;
+	u32 max_pd;
+	u32 max_qp_rd_atom;
+	u32 max_ee_rd_atom;
+	u32 max_res_rd_atom;
+	u32 max_qp_init_rd_atom;
+	u32 max_ee_init_rd_atom;
+	u32 max_ee;
+	u32 max_rdd;
+	u32 max_mw;
+	u32 max_raw_ipv6_qp;
+	u32 max_raw_ethy_qp;
+	u32 max_mcast_grp;
+	u32 max_mcast_qp_attach;
+	u32 max_total_mcast_qp_attach;
+	u32 max_ah;
+	u32 max_fmr;
+	u32 max_map_per_fmr;
+	u32 max_srq;
+	u32 max_srq_wr;
+	u32 max_srq_sge;
+	u32 max_uar;
+	u32 gid_tbl_len;
+	u16 max_pkeys;
+	u8  local_ca_ack_delay;
+	u8  phys_port_cnt;
+	u8  mode;				/* PVRDMA_DEVICE_MODE_ */
+	u8  atomic_ops;				/* PVRDMA_ATOMIC_OP_* bits */
+	u8  bmme_flags;				/* FRWR Mem Mgmt Extensions */
+	u8  gid_types;				/* PVRDMA_GID_TYPE_FLAG_ */
+	u8  reserved[4];
+};
+
+struct pvrdma_ring_page_info {
+	u32 num_pages;				/* Num pages incl. header. */
+	u32 reserved;				/* Reserved. */
+	u64 pdir_dma;				/* Page directory PA. */
+};
+
+#pragma pack(push, 1)
+
+struct pvrdma_device_shared_region {
+	u32 driver_version;			/* W: Driver version. */
+	u32 pad;				/* Pad to 8-byte align. */
+	struct pvrdma_gos_info gos_info;	/* W: Guest OS information. */
+	u64 cmd_slot_dma;			/* W: Command slot address. */
+	u64 resp_slot_dma;			/* W: Response slot address. */
+	struct pvrdma_ring_page_info async_ring_pages;
+						/* W: Async ring page info. */
+	struct pvrdma_ring_page_info cq_ring_pages;
+						/* W: CQ ring page info. */
+	u32 uar_pfn;				/* W: UAR pageframe. */
+	u32 pad2;				/* Pad to 8-byte align. */
+	struct pvrdma_device_caps caps;		/* R: Device capabilities. */
+};
+
+#pragma pack(pop)
+
+/* Event types. Currently a 1:1 mapping with enum ib_event. */
+enum pvrdma_eqe_type {
+	PVRDMA_EVENT_CQ_ERR,
+	PVRDMA_EVENT_QP_FATAL,
+	PVRDMA_EVENT_QP_REQ_ERR,
+	PVRDMA_EVENT_QP_ACCESS_ERR,
+	PVRDMA_EVENT_COMM_EST,
+	PVRDMA_EVENT_SQ_DRAINED,
+	PVRDMA_EVENT_PATH_MIG,
+	PVRDMA_EVENT_PATH_MIG_ERR,
+	PVRDMA_EVENT_DEVICE_FATAL,
+	PVRDMA_EVENT_PORT_ACTIVE,
+	PVRDMA_EVENT_PORT_ERR,
+	PVRDMA_EVENT_LID_CHANGE,
+	PVRDMA_EVENT_PKEY_CHANGE,
+	PVRDMA_EVENT_SM_CHANGE,
+	PVRDMA_EVENT_SRQ_ERR,
+	PVRDMA_EVENT_SRQ_LIMIT_REACHED,
+	PVRDMA_EVENT_QP_LAST_WQE_REACHED,
+	PVRDMA_EVENT_CLIENT_REREGISTER,
+	PVRDMA_EVENT_GID_CHANGE,
+};
+
+/* Event queue element. */
+struct pvrdma_eqe {
+	u32 type;	/* Event type. */
+	u32 info;	/* Handle, other. */
+};
+
+/* CQ notification queue element. */
+struct pvrdma_cqne {
+	u32 info;	/* Handle */
+};
+
+enum {
+	PVRDMA_CMD_FIRST,
+	PVRDMA_CMD_QUERY_PORT = PVRDMA_CMD_FIRST,
+	PVRDMA_CMD_QUERY_PKEY,
+	PVRDMA_CMD_CREATE_PD,
+	PVRDMA_CMD_DESTROY_PD,
+	PVRDMA_CMD_CREATE_MR,
+	PVRDMA_CMD_DESTROY_MR,
+	PVRDMA_CMD_CREATE_CQ,
+	PVRDMA_CMD_RESIZE_CQ,
+	PVRDMA_CMD_DESTROY_CQ,
+	PVRDMA_CMD_CREATE_QP,
+	PVRDMA_CMD_MODIFY_QP,
+	PVRDMA_CMD_QUERY_QP,
+	PVRDMA_CMD_DESTROY_QP,
+	PVRDMA_CMD_CREATE_UC,
+	PVRDMA_CMD_DESTROY_UC,
+	PVRDMA_CMD_CREATE_BIND,
+	PVRDMA_CMD_DESTROY_BIND,
+	PVRDMA_CMD_MAX,
+};
+
+enum {
+	PVRDMA_CMD_FIRST_RESP = (1 << 31),
+	PVRDMA_CMD_QUERY_PORT_RESP = PVRDMA_CMD_FIRST_RESP,
+	PVRDMA_CMD_QUERY_PKEY_RESP,
+	PVRDMA_CMD_CREATE_PD_RESP,
+	PVRDMA_CMD_DESTROY_PD_RESP_NOOP,
+	PVRDMA_CMD_CREATE_MR_RESP,
+	PVRDMA_CMD_DESTROY_MR_RESP_NOOP,
+	PVRDMA_CMD_CREATE_CQ_RESP,
+	PVRDMA_CMD_RESIZE_CQ_RESP,
+	PVRDMA_CMD_DESTROY_CQ_RESP_NOOP,
+	PVRDMA_CMD_CREATE_QP_RESP,
+	PVRDMA_CMD_MODIFY_QP_RESP,
+	PVRDMA_CMD_QUERY_QP_RESP,
+	PVRDMA_CMD_DESTROY_QP_RESP,
+	PVRDMA_CMD_CREATE_UC_RESP,
+	PVRDMA_CMD_DESTROY_UC_RESP_NOOP,
+	PVRDMA_CMD_CREATE_BIND_RESP_NOOP,
+	PVRDMA_CMD_DESTROY_BIND_RESP_NOOP,
+	PVRDMA_CMD_MAX_RESP,
+};
+
+struct pvrdma_cmd_hdr {
+	u64 response;		/* Key for response lookup. */
+	u32 cmd;		/* PVRDMA_CMD_ */
+	u32 reserved;		/* Reserved. */
+};
+
+struct pvrdma_cmd_resp_hdr {
+	u64 response;		/* From cmd hdr. */
+	u32 ack;		/* PVRDMA_CMD_XXX_RESP */
+	u8 err;			/* Error. */
+	u8 reserved[3];		/* Reserved. */
+};
+
+struct pvrdma_cmd_query_port {
+	struct pvrdma_cmd_hdr hdr;
+	u8 port_num;
+	u8 reserved[7];
+};
+
+struct pvrdma_cmd_query_port_resp {
+	struct pvrdma_cmd_resp_hdr hdr;
+	struct pvrdma_port_attr attrs;
+};
+
+struct pvrdma_cmd_query_pkey {
+	struct pvrdma_cmd_hdr hdr;
+	u8 port_num;
+	u8 index;
+	u8 reserved[6];
+};
+
+struct pvrdma_cmd_query_pkey_resp {
+	struct pvrdma_cmd_resp_hdr hdr;
+	u16 pkey;
+	u8 reserved[6];
+};
+
+struct pvrdma_cmd_create_uc {
+	struct pvrdma_cmd_hdr hdr;
+	u32 pfn; /* UAR page frame number */
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_create_uc_resp {
+	struct pvrdma_cmd_resp_hdr hdr;
+	u32 ctx_handle;
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_destroy_uc {
+	struct pvrdma_cmd_hdr hdr;
+	u32 ctx_handle;
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_create_pd {
+	struct pvrdma_cmd_hdr hdr;
+	u32 ctx_handle;
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_create_pd_resp {
+	struct pvrdma_cmd_resp_hdr hdr;
+	u32 pd_handle;
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_destroy_pd {
+	struct pvrdma_cmd_hdr hdr;
+	u32 pd_handle;
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_create_mr {
+	struct pvrdma_cmd_hdr hdr;
+	u64 start;
+	u64 length;
+	u64 pdir_dma;
+	u32 pd_handle;
+	u32 access_flags;
+	u32 flags;
+	u32 nchunks;
+};
+
+struct pvrdma_cmd_create_mr_resp {
+	struct pvrdma_cmd_resp_hdr hdr;
+	u32 mr_handle;
+	u32 lkey;
+	u32 rkey;
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_destroy_mr {
+	struct pvrdma_cmd_hdr hdr;
+	u32 mr_handle;
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_create_cq {
+	struct pvrdma_cmd_hdr hdr;
+	u64 pdir_dma;
+	u32 ctx_handle;
+	u32 cqe;
+	u32 nchunks;
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_create_cq_resp {
+	struct pvrdma_cmd_resp_hdr hdr;
+	u32 cq_handle;
+	u32 cqe;
+};
+
+struct pvrdma_cmd_resize_cq {
+	struct pvrdma_cmd_hdr hdr;
+	u32 cq_handle;
+	u32 cqe;
+};
+
+struct pvrdma_cmd_resize_cq_resp {
+	struct pvrdma_cmd_resp_hdr hdr;
+	u32 cqe;
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_destroy_cq {
+	struct pvrdma_cmd_hdr hdr;
+	u32 cq_handle;
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_create_qp {
+	struct pvrdma_cmd_hdr hdr;
+	u64 pdir_dma;
+	u32 pd_handle;
+	u32 send_cq_handle;
+	u32 recv_cq_handle;
+	u32 srq_handle;
+	u32 max_send_wr;
+	u32 max_recv_wr;
+	u32 max_send_sge;
+	u32 max_recv_sge;
+	u32 max_inline_data;
+	u32 lkey;
+	u32 access_flags;
+	u16 total_chunks;
+	u16 send_chunks;
+	u16 max_atomic_arg;
+	u8 sq_sig_all;
+	u8 qp_type;
+	u8 is_srq;
+	u8 reserved[3];
+};
+
+struct pvrdma_cmd_create_qp_resp {
+	struct pvrdma_cmd_resp_hdr hdr;
+	u32 qpn;
+	u32 max_send_wr;
+	u32 max_recv_wr;
+	u32 max_send_sge;
+	u32 max_recv_sge;
+	u32 max_inline_data;
+};
+
+struct pvrdma_cmd_modify_qp {
+	struct pvrdma_cmd_hdr hdr;
+	u32 qp_handle;
+	u32 attr_mask;
+	struct pvrdma_qp_attr attrs;
+};
+
+struct pvrdma_cmd_query_qp {
+	struct pvrdma_cmd_hdr hdr;
+	u32 qp_handle;
+	u32 attr_mask;
+};
+
+struct pvrdma_cmd_query_qp_resp {
+	struct pvrdma_cmd_resp_hdr hdr;
+	struct pvrdma_qp_attr attrs;
+};
+
+struct pvrdma_cmd_destroy_qp {
+	struct pvrdma_cmd_hdr hdr;
+	u32 qp_handle;
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_destroy_qp_resp {
+	struct pvrdma_cmd_resp_hdr hdr;
+	u32 events_reported;
+	u8 reserved[4];
+};
+
+struct pvrdma_cmd_create_bind {
+	struct pvrdma_cmd_hdr hdr;
+	u32 mtu;
+	u32 vlan;
+	u32 index;
+	u8 new_gid[16];
+	u8 gid_type;
+	u8 reserved[3];
+};
+
+struct pvrdma_cmd_destroy_bind {
+	struct pvrdma_cmd_hdr hdr;
+	u32 index;
+	u8 dest_gid[16];
+	u8 reserved[4];
+};
+
+union pvrdma_cmd_req {
+	struct pvrdma_cmd_hdr hdr;
+	struct pvrdma_cmd_query_port query_port;
+	struct pvrdma_cmd_query_pkey query_pkey;
+	struct pvrdma_cmd_create_uc create_uc;
+	struct pvrdma_cmd_destroy_uc destroy_uc;
+	struct pvrdma_cmd_create_pd create_pd;
+	struct pvrdma_cmd_destroy_pd destroy_pd;
+	struct pvrdma_cmd_create_mr create_mr;
+	struct pvrdma_cmd_destroy_mr destroy_mr;
+	struct pvrdma_cmd_create_cq create_cq;
+	struct pvrdma_cmd_resize_cq resize_cq;
+	struct pvrdma_cmd_destroy_cq destroy_cq;
+	struct pvrdma_cmd_create_qp create_qp;
+	struct pvrdma_cmd_modify_qp modify_qp;
+	struct pvrdma_cmd_query_qp query_qp;
+	struct pvrdma_cmd_destroy_qp destroy_qp;
+	struct pvrdma_cmd_create_bind create_bind;
+	struct pvrdma_cmd_destroy_bind destroy_bind;
+};
+
+union pvrdma_cmd_resp {
+	struct pvrdma_cmd_resp_hdr hdr;
+	struct pvrdma_cmd_query_port_resp query_port_resp;
+	struct pvrdma_cmd_query_pkey_resp query_pkey_resp;
+	struct pvrdma_cmd_create_uc_resp create_uc_resp;
+	struct pvrdma_cmd_create_pd_resp create_pd_resp;
+	struct pvrdma_cmd_create_mr_resp create_mr_resp;
+	struct pvrdma_cmd_create_cq_resp create_cq_resp;
+	struct pvrdma_cmd_resize_cq_resp resize_cq_resp;
+	struct pvrdma_cmd_create_qp_resp create_qp_resp;
+	struct pvrdma_cmd_query_qp_resp query_qp_resp;
+	struct pvrdma_cmd_destroy_qp_resp destroy_qp_resp;
+};
+
+#endif /* __PVRDMA_DEV_API_H__ */
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_doorbell.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_doorbell.c
new file mode 100644
index 000000000000..bf51357ea3aa
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_doorbell.c
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/bitmap.h>
+#include <linux/errno.h>
+#include <linux/slab.h>
+
+#include "pvrdma.h"
+
+int pvrdma_uar_table_init(struct pvrdma_dev *dev)
+{
+	u32 num = dev->dsr->caps.max_uar;
+	u32 mask = num - 1;
+	struct pvrdma_id_table *tbl = &dev->uar_table.tbl;
+
+	if (!is_power_of_2(num))
+		return -EINVAL;
+
+	tbl->last = 0;
+	tbl->top = 0;
+	tbl->max = num;
+	tbl->mask = mask;
+	spin_lock_init(&tbl->lock);
+	tbl->table = kcalloc(BITS_TO_LONGS(num), sizeof(long), GFP_KERNEL);
+	if (!tbl->table)
+		return -ENOMEM;
+
+	/* 0th UAR is taken by the device. */
+	set_bit(0, tbl->table);
+
+	return 0;
+}
+
+void pvrdma_uar_table_cleanup(struct pvrdma_dev *dev)
+{
+	struct pvrdma_id_table *tbl = &dev->uar_table.tbl;
+
+	kfree(tbl->table);
+}
+
+int pvrdma_uar_alloc(struct pvrdma_dev *dev, struct pvrdma_uar_map *uar)
+{
+	struct pvrdma_id_table *tbl;
+	unsigned long flags;
+	u32 obj;
+
+	tbl = &dev->uar_table.tbl;
+
+	spin_lock_irqsave(&tbl->lock, flags);
+	obj = find_next_zero_bit(tbl->table, tbl->max, tbl->last);
+	if (obj >= tbl->max) {
+		tbl->top = (tbl->top + tbl->max) & tbl->mask;
+		obj = find_first_zero_bit(tbl->table, tbl->max);
+	}
+
+	if (obj >= tbl->max) {
+		spin_unlock_irqrestore(&tbl->lock, flags);
+		return -ENOMEM;
+	}
+
+	set_bit(obj, tbl->table);
+	obj |= tbl->top;
+
+	spin_unlock_irqrestore(&tbl->lock, flags);
+
+	uar->index = obj;
+	uar->pfn = (pci_resource_start(dev->pdev, PVRDMA_PCI_RESOURCE_UAR) >>
+		    PAGE_SHIFT) + uar->index;
+
+	return 0;
+}
+
+void pvrdma_uar_free(struct pvrdma_dev *dev, struct pvrdma_uar_map *uar)
+{
+	struct pvrdma_id_table *tbl = &dev->uar_table.tbl;
+	unsigned long flags;
+	u32 obj;
+
+	obj = uar->index & (tbl->max - 1);
+	spin_lock_irqsave(&tbl->lock, flags);
+	clear_bit(obj, tbl->table);
+	tbl->last = min(tbl->last, obj);
+	tbl->top = (tbl->top + tbl->max) & tbl->mask;
+	spin_unlock_irqrestore(&tbl->lock, flags);
+}
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
new file mode 100644
index 000000000000..231a1ce1f4be
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
@@ -0,0 +1,1211 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/errno.h>
+#include <linux/inetdevice.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_smi.h>
+#include <rdma/ib_user_verbs.h>
+#include <net/addrconf.h>
+
+#include "pvrdma.h"
+
+#define DRV_NAME	"vmw_pvrdma"
+#define DRV_VERSION	"1.0.0.0-k"
+
+static DEFINE_MUTEX(pvrdma_device_list_lock);
+static LIST_HEAD(pvrdma_device_list);
+static struct workqueue_struct *event_wq;
+
+static int pvrdma_add_gid(struct ib_device *ibdev,
+			  u8 port_num,
+			  unsigned int index,
+			  const union ib_gid *gid,
+			  const struct ib_gid_attr *attr,
+			  void **context);
+static int pvrdma_del_gid(struct ib_device *ibdev,
+			  u8 port_num,
+			  unsigned int index,
+			  void **context);
+
+
+static ssize_t show_hca(struct device *device, struct device_attribute *attr,
+			char *buf)
+{
+	return sprintf(buf, "VMW_PVRDMA-%s\n", DRV_VERSION);
+}
+
+static ssize_t show_rev(struct device *device, struct device_attribute *attr,
+			char *buf)
+{
+	return sprintf(buf, "%d\n", PVRDMA_REV_ID);
+}
+
+static ssize_t show_board(struct device *device, struct device_attribute *attr,
+			  char *buf)
+{
+	return sprintf(buf, "%d\n", PVRDMA_BOARD_ID);
+}
+
+static DEVICE_ATTR(hw_rev,   S_IRUGO, show_rev,	   NULL);
+static DEVICE_ATTR(hca_type, S_IRUGO, show_hca,	   NULL);
+static DEVICE_ATTR(board_id, S_IRUGO, show_board,  NULL);
+
+static struct device_attribute *pvrdma_class_attributes[] = {
+	&dev_attr_hw_rev,
+	&dev_attr_hca_type,
+	&dev_attr_board_id
+};
+
+static void pvrdma_get_fw_ver_str(struct ib_device *device, char *str,
+				  size_t str_len)
+{
+	struct pvrdma_dev *dev =
+		container_of(device, struct pvrdma_dev, ib_dev);
+	snprintf(str, str_len, "%d.%d.%d\n",
+		 (int) (dev->dsr->caps.fw_ver >> 32),
+		 (int) (dev->dsr->caps.fw_ver >> 16) & 0xffff,
+		 (int) dev->dsr->caps.fw_ver & 0xffff);
+}
+
+static int pvrdma_init_device(struct pvrdma_dev *dev)
+{
+	/*  Initialize some device related stuff */
+	spin_lock_init(&dev->cmd_lock);
+	sema_init(&dev->cmd_sema, 1);
+	atomic_set(&dev->num_qps, 0);
+	atomic_set(&dev->num_cqs, 0);
+	atomic_set(&dev->num_pds, 0);
+	atomic_set(&dev->num_ahs, 0);
+
+	return 0;
+}
+
+static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num,
+				 struct ib_port_immutable *immutable)
+{
+	struct ib_port_attr attr;
+	int err;
+
+	err = pvrdma_query_port(ibdev, port_num, &attr);
+	if (err)
+		return err;
+
+	immutable->pkey_tbl_len = attr.pkey_tbl_len;
+	immutable->gid_tbl_len = attr.gid_tbl_len;
+	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
+	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+	return 0;
+}
+
+static struct net_device *pvrdma_get_netdev(struct ib_device *ibdev,
+					    u8 port_num)
+{
+	struct net_device *netdev;
+	struct pvrdma_dev *dev = to_vdev(ibdev);
+
+	if (port_num != 1)
+		return NULL;
+
+	rcu_read_lock();
+	netdev = dev->netdev;
+	if (netdev)
+		dev_hold(netdev);
+	rcu_read_unlock();
+
+	return netdev;
+}
+
+static int pvrdma_register_device(struct pvrdma_dev *dev)
+{
+	int ret = -1;
+	int i = 0;
+
+	strlcpy(dev->ib_dev.name, "vmw_pvrdma%d", IB_DEVICE_NAME_MAX);
+	dev->ib_dev.node_guid = dev->dsr->caps.node_guid;
+	dev->sys_image_guid = dev->dsr->caps.sys_image_guid;
+	dev->flags = 0;
+	dev->ib_dev.owner = THIS_MODULE;
+	dev->ib_dev.num_comp_vectors = 1;
+	dev->ib_dev.dma_device = &dev->pdev->dev;
+	dev->ib_dev.uverbs_abi_ver = PVRDMA_UVERBS_ABI_VERSION;
+	dev->ib_dev.uverbs_cmd_mask =
+		(1ull << IB_USER_VERBS_CMD_GET_CONTEXT)		|
+		(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)	|
+		(1ull << IB_USER_VERBS_CMD_QUERY_PORT)		|
+		(1ull << IB_USER_VERBS_CMD_ALLOC_PD)		|
+		(1ull << IB_USER_VERBS_CMD_DEALLOC_PD)		|
+		(1ull << IB_USER_VERBS_CMD_REG_MR)		|
+		(1ull << IB_USER_VERBS_CMD_DEREG_MR)		|
+		(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)	|
+		(1ull << IB_USER_VERBS_CMD_CREATE_CQ)		|
+		(1ull << IB_USER_VERBS_CMD_POLL_CQ)		|
+		(1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ)	|
+		(1ull << IB_USER_VERBS_CMD_DESTROY_CQ)		|
+		(1ull << IB_USER_VERBS_CMD_CREATE_QP)		|
+		(1ull << IB_USER_VERBS_CMD_MODIFY_QP)		|
+		(1ull << IB_USER_VERBS_CMD_QUERY_QP)		|
+		(1ull << IB_USER_VERBS_CMD_DESTROY_QP)		|
+		(1ull << IB_USER_VERBS_CMD_POST_SEND)		|
+		(1ull << IB_USER_VERBS_CMD_POST_RECV)		|
+		(1ull << IB_USER_VERBS_CMD_CREATE_AH)		|
+		(1ull << IB_USER_VERBS_CMD_DESTROY_AH);
+
+	dev->ib_dev.node_type = RDMA_NODE_IB_CA;
+	dev->ib_dev.phys_port_cnt = dev->dsr->caps.phys_port_cnt;
+
+	dev->ib_dev.query_device = pvrdma_query_device;
+	dev->ib_dev.query_port = pvrdma_query_port;
+	dev->ib_dev.query_gid = pvrdma_query_gid;
+	dev->ib_dev.query_pkey = pvrdma_query_pkey;
+	dev->ib_dev.modify_port	= pvrdma_modify_port;
+	dev->ib_dev.alloc_ucontext = pvrdma_alloc_ucontext;
+	dev->ib_dev.dealloc_ucontext = pvrdma_dealloc_ucontext;
+	dev->ib_dev.mmap = pvrdma_mmap;
+	dev->ib_dev.alloc_pd = pvrdma_alloc_pd;
+	dev->ib_dev.dealloc_pd = pvrdma_dealloc_pd;
+	dev->ib_dev.create_ah = pvrdma_create_ah;
+	dev->ib_dev.destroy_ah = pvrdma_destroy_ah;
+	dev->ib_dev.create_qp = pvrdma_create_qp;
+	dev->ib_dev.modify_qp = pvrdma_modify_qp;
+	dev->ib_dev.query_qp = pvrdma_query_qp;
+	dev->ib_dev.destroy_qp = pvrdma_destroy_qp;
+	dev->ib_dev.post_send = pvrdma_post_send;
+	dev->ib_dev.post_recv = pvrdma_post_recv;
+	dev->ib_dev.create_cq = pvrdma_create_cq;
+	dev->ib_dev.modify_cq = pvrdma_modify_cq;
+	dev->ib_dev.resize_cq = pvrdma_resize_cq;
+	dev->ib_dev.destroy_cq = pvrdma_destroy_cq;
+	dev->ib_dev.poll_cq = pvrdma_poll_cq;
+	dev->ib_dev.req_notify_cq = pvrdma_req_notify_cq;
+	dev->ib_dev.get_dma_mr = pvrdma_get_dma_mr;
+	dev->ib_dev.reg_user_mr	= pvrdma_reg_user_mr;
+	dev->ib_dev.dereg_mr = pvrdma_dereg_mr;
+	dev->ib_dev.alloc_mr = pvrdma_alloc_mr;
+	dev->ib_dev.map_mr_sg = pvrdma_map_mr_sg;
+	dev->ib_dev.add_gid = pvrdma_add_gid;
+	dev->ib_dev.del_gid = pvrdma_del_gid;
+	dev->ib_dev.get_netdev = pvrdma_get_netdev;
+	dev->ib_dev.get_port_immutable = pvrdma_port_immutable;
+	dev->ib_dev.get_link_layer = pvrdma_port_link_layer;
+	dev->ib_dev.get_dev_fw_str = pvrdma_get_fw_ver_str;
+
+	mutex_init(&dev->port_mutex);
+	spin_lock_init(&dev->desc_lock);
+
+	dev->cq_tbl = kcalloc(dev->dsr->caps.max_cq, sizeof(void *),
+			      GFP_KERNEL);
+	if (!dev->cq_tbl)
+		return ret;
+	spin_lock_init(&dev->cq_tbl_lock);
+
+	dev->qp_tbl = kcalloc(dev->dsr->caps.max_qp, sizeof(void *),
+			      GFP_KERNEL);
+	if (!dev->qp_tbl)
+		goto err_cq_free;
+	spin_lock_init(&dev->qp_tbl_lock);
+
+	ret = ib_register_device(&dev->ib_dev, NULL);
+	if (ret)
+		goto err_qp_free;
+
+	for (i = 0; i < ARRAY_SIZE(pvrdma_class_attributes); ++i) {
+		ret = device_create_file(&dev->ib_dev.dev,
+					 pvrdma_class_attributes[i]);
+		if (ret)
+			goto err_class;
+	}
+
+	dev->ib_active = true;
+
+	return 0;
+
+err_class:
+	ib_unregister_device(&dev->ib_dev);
+err_qp_free:
+	kfree(dev->qp_tbl);
+err_cq_free:
+	kfree(dev->cq_tbl);
+
+	return ret;
+}
+
+static irqreturn_t pvrdma_intr0_handler(int irq, void *dev_id)
+{
+	u32 icr = PVRDMA_INTR_CAUSE_RESPONSE;
+	struct pvrdma_dev *dev = dev_id;
+
+	dev_dbg(&dev->pdev->dev, "interrupt 0 (response) handler\n");
+
+	if (dev->intr.type != PVRDMA_INTR_TYPE_MSIX) {
+		/* Legacy intr */
+		icr = pvrdma_read_reg(dev, PVRDMA_REG_ICR);
+		if (icr == 0)
+			return IRQ_NONE;
+	}
+
+	if (icr == PVRDMA_INTR_CAUSE_RESPONSE)
+		complete(&dev->cmd_done);
+
+	return IRQ_HANDLED;
+}
+
+static void pvrdma_qp_event(struct pvrdma_dev *dev, u32 qpn, int type)
+{
+	struct pvrdma_qp *qp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->qp_tbl_lock, flags);
+	qp = dev->qp_tbl[qpn % dev->dsr->caps.max_qp];
+	if (qp)
+		atomic_inc(&qp->refcnt);
+	spin_unlock_irqrestore(&dev->qp_tbl_lock, flags);
+
+	if (qp && qp->ibqp.event_handler) {
+		struct ib_qp *ibqp = &qp->ibqp;
+		struct ib_event e;
+
+		e.device = ibqp->device;
+		e.element.qp = ibqp;
+		e.event = type; /* 1:1 mapping for now. */
+		ibqp->event_handler(&e, ibqp->qp_context);
+	}
+	if (qp) {
+		atomic_dec(&qp->refcnt);
+		if (atomic_read(&qp->refcnt) == 0)
+			wake_up(&qp->wait);
+	}
+}
+
+static void pvrdma_cq_event(struct pvrdma_dev *dev, u32 cqn, int type)
+{
+	struct pvrdma_cq *cq;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->cq_tbl_lock, flags);
+	cq = dev->cq_tbl[cqn % dev->dsr->caps.max_cq];
+	if (cq)
+		atomic_inc(&cq->refcnt);
+	spin_unlock_irqrestore(&dev->cq_tbl_lock, flags);
+
+	if (cq && cq->ibcq.event_handler) {
+		struct ib_cq *ibcq = &cq->ibcq;
+		struct ib_event e;
+
+		e.device = ibcq->device;
+		e.element.cq = ibcq;
+		e.event = type; /* 1:1 mapping for now. */
+		ibcq->event_handler(&e, ibcq->cq_context);
+	}
+	if (cq) {
+		atomic_dec(&cq->refcnt);
+		if (atomic_read(&cq->refcnt) == 0)
+			wake_up(&cq->wait);
+	}
+}
+
+static void pvrdma_dispatch_event(struct pvrdma_dev *dev, int port,
+				  enum ib_event_type event)
+{
+	struct ib_event ib_event;
+
+	memset(&ib_event, 0, sizeof(ib_event));
+	ib_event.device = &dev->ib_dev;
+	ib_event.element.port_num = port;
+	ib_event.event = event;
+	ib_dispatch_event(&ib_event);
+}
+
+static void pvrdma_dev_event(struct pvrdma_dev *dev, u8 port, int type)
+{
+	if (port < 1 || port > dev->dsr->caps.phys_port_cnt) {
+		dev_warn(&dev->pdev->dev, "event on port %d\n", port);
+		return;
+	}
+
+	pvrdma_dispatch_event(dev, port, type);
+}
+
+static inline struct pvrdma_eqe *get_eqe(struct pvrdma_dev *dev, unsigned int i)
+{
+	return (struct pvrdma_eqe *)pvrdma_page_dir_get_ptr(
+					&dev->async_pdir,
+					PAGE_SIZE +
+					sizeof(struct pvrdma_eqe) * i);
+}
+
+static irqreturn_t pvrdma_intr1_handler(int irq, void *dev_id)
+{
+	struct pvrdma_dev *dev = dev_id;
+	struct pvrdma_ring *ring = &dev->async_ring_state->rx;
+	int ring_slots = (dev->dsr->async_ring_pages.num_pages - 1) *
+			 PAGE_SIZE / sizeof(struct pvrdma_eqe);
+	unsigned int head;
+
+	dev_dbg(&dev->pdev->dev, "interrupt 1 (async event) handler\n");
+
+	/*
+	 * Don't process events until the IB device is registered. Otherwise
+	 * we'll try to ib_dispatch_event() on an invalid device.
+	 */
+	if (!dev->ib_active)
+		return IRQ_HANDLED;
+
+	while (pvrdma_idx_ring_has_data(ring, ring_slots, &head) > 0) {
+		struct pvrdma_eqe *eqe;
+
+		eqe = get_eqe(dev, head);
+
+		switch (eqe->type) {
+		case PVRDMA_EVENT_QP_FATAL:
+		case PVRDMA_EVENT_QP_REQ_ERR:
+		case PVRDMA_EVENT_QP_ACCESS_ERR:
+		case PVRDMA_EVENT_COMM_EST:
+		case PVRDMA_EVENT_SQ_DRAINED:
+		case PVRDMA_EVENT_PATH_MIG:
+		case PVRDMA_EVENT_PATH_MIG_ERR:
+		case PVRDMA_EVENT_QP_LAST_WQE_REACHED:
+			pvrdma_qp_event(dev, eqe->info, eqe->type);
+			break;
+
+		case PVRDMA_EVENT_CQ_ERR:
+			pvrdma_cq_event(dev, eqe->info, eqe->type);
+			break;
+
+		case PVRDMA_EVENT_SRQ_ERR:
+		case PVRDMA_EVENT_SRQ_LIMIT_REACHED:
+			break;
+
+		case PVRDMA_EVENT_PORT_ACTIVE:
+		case PVRDMA_EVENT_PORT_ERR:
+		case PVRDMA_EVENT_LID_CHANGE:
+		case PVRDMA_EVENT_PKEY_CHANGE:
+		case PVRDMA_EVENT_SM_CHANGE:
+		case PVRDMA_EVENT_CLIENT_REREGISTER:
+		case PVRDMA_EVENT_GID_CHANGE:
+			pvrdma_dev_event(dev, eqe->info, eqe->type);
+			break;
+
+		case PVRDMA_EVENT_DEVICE_FATAL:
+			pvrdma_dev_event(dev, 1, eqe->type);
+			break;
+
+		default:
+			break;
+		}
+
+		pvrdma_idx_ring_inc(&ring->cons_head, ring_slots);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static inline struct pvrdma_cqne *get_cqne(struct pvrdma_dev *dev,
+					   unsigned int i)
+{
+	return (struct pvrdma_cqne *)pvrdma_page_dir_get_ptr(
+					&dev->cq_pdir,
+					PAGE_SIZE +
+					sizeof(struct pvrdma_cqne) * i);
+}
+
+static irqreturn_t pvrdma_intrx_handler(int irq, void *dev_id)
+{
+	struct pvrdma_dev *dev = dev_id;
+	struct pvrdma_ring *ring = &dev->cq_ring_state->rx;
+	int ring_slots = (dev->dsr->cq_ring_pages.num_pages - 1) * PAGE_SIZE /
+			 sizeof(struct pvrdma_cqne);
+	unsigned int head;
+	unsigned long flags;
+
+	dev_dbg(&dev->pdev->dev, "interrupt x (completion) handler\n");
+
+	while (pvrdma_idx_ring_has_data(ring, ring_slots, &head) > 0) {
+		struct pvrdma_cqne *cqne;
+		struct pvrdma_cq *cq;
+
+		cqne = get_cqne(dev, head);
+		spin_lock_irqsave(&dev->cq_tbl_lock, flags);
+		cq = dev->cq_tbl[cqne->info % dev->dsr->caps.max_cq];
+		if (cq)
+			atomic_inc(&cq->refcnt);
+		spin_unlock_irqrestore(&dev->cq_tbl_lock, flags);
+
+		if (cq && cq->ibcq.comp_handler)
+			cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+		if (cq) {
+			atomic_dec(&cq->refcnt);
+			if (atomic_read(&cq->refcnt))
+				wake_up(&cq->wait);
+		}
+		pvrdma_idx_ring_inc(&ring->cons_head, ring_slots);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void pvrdma_disable_msi_all(struct pvrdma_dev *dev)
+{
+	if (dev->intr.type == PVRDMA_INTR_TYPE_MSIX)
+		pci_disable_msix(dev->pdev);
+	else if (dev->intr.type == PVRDMA_INTR_TYPE_MSI)
+		pci_disable_msi(dev->pdev);
+}
+
+static void pvrdma_free_irq(struct pvrdma_dev *dev)
+{
+	int i;
+
+	dev_dbg(&dev->pdev->dev, "freeing interrupts\n");
+
+	if (dev->intr.type == PVRDMA_INTR_TYPE_MSIX) {
+		for (i = 0; i < dev->intr.size; i++) {
+			if (dev->intr.enabled[i]) {
+				free_irq(dev->intr.msix_entry[i].vector, dev);
+				dev->intr.enabled[i] = 0;
+			}
+		}
+	} else if (dev->intr.type == PVRDMA_INTR_TYPE_INTX ||
+		   dev->intr.type == PVRDMA_INTR_TYPE_MSI) {
+		free_irq(dev->pdev->irq, dev);
+	}
+}
+
+static void pvrdma_enable_intrs(struct pvrdma_dev *dev)
+{
+	dev_dbg(&dev->pdev->dev, "enable interrupts\n");
+	pvrdma_write_reg(dev, PVRDMA_REG_IMR, 0);
+}
+
+static void pvrdma_disable_intrs(struct pvrdma_dev *dev)
+{
+	dev_dbg(&dev->pdev->dev, "disable interrupts\n");
+	pvrdma_write_reg(dev, PVRDMA_REG_IMR, ~0);
+}
+
+static int pvrdma_enable_msix(struct pci_dev *pdev, struct pvrdma_dev *dev)
+{
+	int i;
+	int ret;
+
+	for (i = 0; i < PVRDMA_MAX_INTERRUPTS; i++) {
+		dev->intr.msix_entry[i].entry = i;
+		dev->intr.msix_entry[i].vector = i;
+
+		switch (i) {
+		case 0:
+			/* CMD ring handler */
+			dev->intr.handler[i] = pvrdma_intr0_handler;
+			break;
+		case 1:
+			/* Async event ring handler */
+			dev->intr.handler[i] = pvrdma_intr1_handler;
+			break;
+		default:
+			/* Completion queue handler */
+			dev->intr.handler[i] = pvrdma_intrx_handler;
+			break;
+		}
+	}
+
+	ret = pci_enable_msix(pdev, dev->intr.msix_entry,
+			      PVRDMA_MAX_INTERRUPTS);
+	if (!ret) {
+		dev->intr.type = PVRDMA_INTR_TYPE_MSIX;
+		dev->intr.size = PVRDMA_MAX_INTERRUPTS;
+	} else if (ret > 0) {
+		ret = pci_enable_msix(pdev, dev->intr.msix_entry, ret);
+		if (!ret) {
+			dev->intr.type = PVRDMA_INTR_TYPE_MSIX;
+			dev->intr.size = ret;
+		} else {
+			dev->intr.size = 0;
+		}
+	}
+
+	dev_dbg(&pdev->dev, "using interrupt type %d, size %d\n",
+		dev->intr.type, dev->intr.size);
+
+	return ret;
+}
+
+static int pvrdma_alloc_intrs(struct pvrdma_dev *dev)
+{
+	int ret = 0;
+	int i;
+
+	if (pci_find_capability(dev->pdev, PCI_CAP_ID_MSIX) &&
+	    pvrdma_enable_msix(dev->pdev, dev)) {
+		/* Try MSI */
+		ret = pci_enable_msi(dev->pdev);
+		if (!ret) {
+			dev->intr.type = PVRDMA_INTR_TYPE_MSI;
+		} else {
+			/* Legacy INTR */
+			dev->intr.type = PVRDMA_INTR_TYPE_INTX;
+		}
+	}
+
+	/* Request First IRQ */
+	switch (dev->intr.type) {
+	case PVRDMA_INTR_TYPE_INTX:
+	case PVRDMA_INTR_TYPE_MSI:
+		ret = request_irq(dev->pdev->irq, pvrdma_intr0_handler,
+				  IRQF_SHARED, DRV_NAME, dev);
+		if (ret) {
+			dev_err(&dev->pdev->dev,
+				"failed to request interrupt\n");
+			goto disable_msi;
+		}
+		break;
+	case PVRDMA_INTR_TYPE_MSIX:
+		ret = request_irq(dev->intr.msix_entry[0].vector,
+				  pvrdma_intr0_handler, 0, DRV_NAME, dev);
+		if (ret) {
+			dev_err(&dev->pdev->dev,
+				"failed to request interrupt 0\n");
+			goto disable_msi;
+		}
+		dev->intr.enabled[0] = 1;
+		break;
+	default:
+		/* Not reached */
+		break;
+	}
+
+	/* For MSIX: request intr for each vector */
+	if (dev->intr.size > 1) {
+		ret = request_irq(dev->intr.msix_entry[1].vector,
+				  pvrdma_intr1_handler, 0, DRV_NAME, dev);
+		if (ret) {
+			dev_err(&dev->pdev->dev,
+				"failed to request interrupt 1\n");
+			goto free_irq;
+		}
+		dev->intr.enabled[1] = 1;
+
+		for (i = 2; i < dev->intr.size; i++) {
+			ret = request_irq(dev->intr.msix_entry[i].vector,
+					  pvrdma_intrx_handler, 0,
+					  DRV_NAME, dev);
+			if (ret) {
+				dev_err(&dev->pdev->dev,
+					"failed to request interrupt %d\n", i);
+				goto free_irq;
+			}
+			dev->intr.enabled[i] = 1;
+		}
+	}
+
+	return 0;
+
+free_irq:
+	pvrdma_free_irq(dev);
+disable_msi:
+	pvrdma_disable_msi_all(dev);
+	return ret;
+}
+
+static void pvrdma_free_slots(struct pvrdma_dev *dev)
+{
+	struct pci_dev *pdev = dev->pdev;
+
+	if (dev->resp_slot)
+		dma_free_coherent(&pdev->dev, PAGE_SIZE, dev->resp_slot,
+				  dev->dsr->resp_slot_dma);
+	if (dev->cmd_slot)
+		dma_free_coherent(&pdev->dev, PAGE_SIZE, dev->cmd_slot,
+				  dev->dsr->cmd_slot_dma);
+}
+
+static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
+				   const union ib_gid *gid,
+				   int index)
+{
+	int ret;
+	union pvrdma_cmd_req req;
+	struct pvrdma_cmd_create_bind *cmd_bind = &req.create_bind;
+
+	if (!dev->sgid_tbl) {
+		dev_warn(&dev->pdev->dev, "sgid table not initialized\n");
+		return -EINVAL;
+	}
+
+	memset(cmd_bind, 0, sizeof(*cmd_bind));
+	cmd_bind->hdr.cmd = PVRDMA_CMD_CREATE_BIND;
+	memcpy(cmd_bind->new_gid, gid->raw, 16);
+	cmd_bind->mtu = ib_mtu_enum_to_int(IB_MTU_1024);
+	cmd_bind->vlan = 0xfff;
+	cmd_bind->index = index;
+	cmd_bind->gid_type = PVRDMA_GID_TYPE_FLAG_ROCE_V1;
+
+	ret = pvrdma_cmd_post(dev, &req, NULL, 0);
+	if (ret < 0) {
+		dev_warn(&dev->pdev->dev,
+			 "could not create binding, error: %d\n", ret);
+		return -EFAULT;
+	}
+	memcpy(&dev->sgid_tbl[index], gid, sizeof(*gid));
+	return 0;
+}
+
+static int pvrdma_add_gid(struct ib_device *ibdev,
+			  u8 port_num,
+			  unsigned int index,
+			  const union ib_gid *gid,
+			  const struct ib_gid_attr *attr,
+			  void **context)
+{
+	struct pvrdma_dev *dev = to_vdev(ibdev);
+
+	return pvrdma_add_gid_at_index(dev, gid, index);
+}
+
+static int pvrdma_del_gid_at_index(struct pvrdma_dev *dev, int index)
+{
+	int ret;
+	union pvrdma_cmd_req req;
+	struct pvrdma_cmd_destroy_bind *cmd_dest = &req.destroy_bind;
+
+	/* Update sgid table. */
+	if (!dev->sgid_tbl) {
+		dev_warn(&dev->pdev->dev, "sgid table not initialized\n");
+		return -EINVAL;
+	}
+
+	memset(cmd_dest, 0, sizeof(*cmd_dest));
+	cmd_dest->hdr.cmd = PVRDMA_CMD_DESTROY_BIND;
+	memcpy(cmd_dest->dest_gid, &dev->sgid_tbl[index], 16);
+	cmd_dest->index = index;
+
+	ret = pvrdma_cmd_post(dev, &req, NULL, 0);
+	if (ret < 0) {
+		dev_warn(&dev->pdev->dev,
+			 "could not destroy binding, error: %d\n", ret);
+		return ret;
+	}
+	memset(&dev->sgid_tbl[index], 0, 16);
+	return 0;
+}
+
+static int pvrdma_del_gid(struct ib_device *ibdev,
+			  u8 port_num,
+			  unsigned int index,
+			  void **context)
+{
+	struct pvrdma_dev *dev = to_vdev(ibdev);
+
+	dev_dbg(&dev->pdev->dev, "removing gid at index %u from %s",
+		index, dev->netdev->name);
+
+	return pvrdma_del_gid_at_index(dev, index);
+}
+
+static void pvrdma_netdevice_event_handle(struct pvrdma_dev *dev,
+					  unsigned long event)
+{
+	switch (event) {
+	case NETDEV_REBOOT:
+	case NETDEV_DOWN:
+		pvrdma_dispatch_event(dev, 1, IB_EVENT_PORT_ERR);
+		break;
+	case NETDEV_UP:
+		pvrdma_dispatch_event(dev, 1, IB_EVENT_PORT_ACTIVE);
+		break;
+	default:
+		dev_dbg(&dev->pdev->dev, "ignore netdevice event %ld on %s\n",
+			event, dev->ib_dev.name);
+		break;
+	}
+}
+
+static void pvrdma_netdevice_event_work(struct work_struct *work)
+{
+	struct pvrdma_netdevice_work *netdev_work;
+	struct pvrdma_dev *dev;
+
+	netdev_work = container_of(work, struct pvrdma_netdevice_work, work);
+
+	mutex_lock(&pvrdma_device_list_lock);
+	list_for_each_entry(dev, &pvrdma_device_list, device_link) {
+		if (dev->netdev == netdev_work->event_netdev) {
+			pvrdma_netdevice_event_handle(dev, netdev_work->event);
+			break;
+		}
+	}
+	mutex_unlock(&pvrdma_device_list_lock);
+
+	kfree(netdev_work);
+}
+
+static int pvrdma_netdevice_event(struct notifier_block *this,
+				  unsigned long event, void *ptr)
+{
+	struct net_device *event_netdev = netdev_notifier_info_to_dev(ptr);
+	struct pvrdma_netdevice_work *netdev_work;
+
+	netdev_work = kmalloc(sizeof(*netdev_work), GFP_ATOMIC);
+	if (!netdev_work)
+		return NOTIFY_BAD;
+
+	INIT_WORK(&netdev_work->work, pvrdma_netdevice_event_work);
+	netdev_work->event_netdev = event_netdev;
+	netdev_work->event = event;
+	queue_work(event_wq, &netdev_work->work);
+
+	return NOTIFY_DONE;
+}
+
+static int pvrdma_pci_probe(struct pci_dev *pdev,
+			    const struct pci_device_id *id)
+{
+	struct pci_dev *pdev_net;
+	struct pvrdma_dev *dev;
+	int ret;
+	unsigned long start;
+	unsigned long len;
+	unsigned int version;
+	dma_addr_t slot_dma = 0;
+
+	dev_dbg(&pdev->dev, "initializing driver %s\n", pci_name(pdev));
+
+	/* Allocate zero-out device */
+	dev = (struct pvrdma_dev *)ib_alloc_device(sizeof(*dev));
+	if (!dev) {
+		dev_err(&pdev->dev, "failed to allocate IB device\n");
+		return -ENOMEM;
+	}
+
+	mutex_lock(&pvrdma_device_list_lock);
+	list_add(&dev->device_link, &pvrdma_device_list);
+	mutex_unlock(&pvrdma_device_list_lock);
+
+	ret = pvrdma_init_device(dev);
+	if (ret)
+		goto err_free_device;
+
+	dev->pdev = pdev;
+	pci_set_drvdata(pdev, dev);
+
+	ret = pci_enable_device(pdev);
+	if (ret) {
+		dev_err(&pdev->dev, "cannot enable PCI device\n");
+		goto err_free_device;
+	}
+
+	dev_dbg(&pdev->dev, "PCI resource flags BAR0 %#lx\n",
+		pci_resource_flags(pdev, 0));
+	dev_dbg(&pdev->dev, "PCI resource len %#llx\n",
+		(unsigned long long)pci_resource_len(pdev, 0));
+	dev_dbg(&pdev->dev, "PCI resource start %#llx\n",
+		(unsigned long long)pci_resource_start(pdev, 0));
+	dev_dbg(&pdev->dev, "PCI resource flags BAR1 %#lx\n",
+		pci_resource_flags(pdev, 1));
+	dev_dbg(&pdev->dev, "PCI resource len %#llx\n",
+		(unsigned long long)pci_resource_len(pdev, 1));
+	dev_dbg(&pdev->dev, "PCI resource start %#llx\n",
+		(unsigned long long)pci_resource_start(pdev, 1));
+
+	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
+	    !(pci_resource_flags(pdev, 1) & IORESOURCE_MEM)) {
+		dev_err(&pdev->dev, "PCI BAR region not MMIO\n");
+		ret = -ENOMEM;
+		goto err_free_device;
+	}
+
+	ret = pci_request_regions(pdev, DRV_NAME);
+	if (ret) {
+		dev_err(&pdev->dev, "cannot request PCI resources\n");
+		goto err_disable_pdev;
+	}
+
+	/* Enable 64-Bit DMA */
+	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
+		ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
+		if (ret != 0) {
+			dev_err(&pdev->dev,
+				"pci_set_consistent_dma_mask failed\n");
+			goto err_free_resource;
+		}
+	} else {
+		ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+		if (ret != 0) {
+			dev_err(&pdev->dev,
+				"pci_set_dma_mask failed\n");
+			goto err_free_resource;
+		}
+	}
+
+	pci_set_master(pdev);
+
+	/* Map register space */
+	start = pci_resource_start(dev->pdev, PVRDMA_PCI_RESOURCE_REG);
+	len = pci_resource_len(dev->pdev, PVRDMA_PCI_RESOURCE_REG);
+	dev->regs = ioremap(start, len);
+	if (!dev->regs) {
+		dev_err(&pdev->dev, "register mapping failed\n");
+		ret = -ENOMEM;
+		goto err_free_resource;
+	}
+
+	/* Setup per-device UAR. */
+	dev->driver_uar.index = 0;
+	dev->driver_uar.pfn =
+		pci_resource_start(dev->pdev, PVRDMA_PCI_RESOURCE_UAR) >>
+		PAGE_SHIFT;
+	dev->driver_uar.map =
+		ioremap(dev->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
+	if (!dev->driver_uar.map) {
+		dev_err(&pdev->dev, "failed to remap UAR pages\n");
+		ret = -ENOMEM;
+		goto err_unmap_regs;
+	}
+
+	version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
+	dev_info(&pdev->dev, "device version %d, driver version %d\n",
+		 version, PVRDMA_VERSION);
+	if (version < PVRDMA_VERSION) {
+		dev_err(&pdev->dev, "incompatible device version\n");
+		goto err_uar_unmap;
+	}
+
+	dev->dsr = dma_alloc_coherent(&pdev->dev, sizeof(*dev->dsr),
+				      &dev->dsrbase, GFP_KERNEL);
+	if (!dev->dsr) {
+		dev_err(&pdev->dev, "failed to allocate shared region\n");
+		ret = -ENOMEM;
+		goto err_uar_unmap;
+	}
+
+	/* Setup the shared region */
+	memset(dev->dsr, 0, sizeof(*dev->dsr));
+	dev->dsr->driver_version = PVRDMA_VERSION;
+	dev->dsr->gos_info.gos_bits = sizeof(void *) == 4 ?
+		PVRDMA_GOS_BITS_32 :
+		PVRDMA_GOS_BITS_64;
+	dev->dsr->gos_info.gos_type = PVRDMA_GOS_TYPE_LINUX;
+	dev->dsr->gos_info.gos_ver = 1;
+	dev->dsr->uar_pfn = dev->driver_uar.pfn;
+
+	/* Command slot. */
+	dev->cmd_slot = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
+					   &slot_dma, GFP_KERNEL);
+	if (!dev->cmd_slot) {
+		ret = -ENOMEM;
+		goto err_free_dsr;
+	}
+
+	dev->dsr->cmd_slot_dma = (u64)slot_dma;
+
+	/* Response slot. */
+	dev->resp_slot = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
+					    &slot_dma, GFP_KERNEL);
+	if (!dev->resp_slot) {
+		ret = -ENOMEM;
+		goto err_free_slots;
+	}
+
+	dev->dsr->resp_slot_dma = (u64)slot_dma;
+
+	/* Async event ring */
+	dev->dsr->async_ring_pages.num_pages = 4;
+	ret = pvrdma_page_dir_init(dev, &dev->async_pdir,
+				   dev->dsr->async_ring_pages.num_pages, true);
+	if (ret)
+		goto err_free_slots;
+	dev->async_ring_state = dev->async_pdir.pages[0];
+	dev->dsr->async_ring_pages.pdir_dma = dev->async_pdir.dir_dma;
+
+	/* CQ notification ring */
+	dev->dsr->cq_ring_pages.num_pages = 4;
+	ret = pvrdma_page_dir_init(dev, &dev->cq_pdir,
+				   dev->dsr->cq_ring_pages.num_pages, true);
+	if (ret)
+		goto err_free_async_ring;
+	dev->cq_ring_state = dev->cq_pdir.pages[0];
+	dev->dsr->cq_ring_pages.pdir_dma = dev->cq_pdir.dir_dma;
+
+	/*
+	 * Write the PA of the shared region to the device. The writes must be
+	 * ordered such that the high bits are written last. When the writes
+	 * complete, the device will have filled out the capabilities.
+	 */
+
+	pvrdma_write_reg(dev, PVRDMA_REG_DSRLOW, (u32)dev->dsrbase);
+	pvrdma_write_reg(dev, PVRDMA_REG_DSRHIGH,
+			 (u32)((u64)(dev->dsrbase) >> 32));
+
+	/* Make sure the write is complete before reading status. */
+	mb();
+
+	/* Currently, the driver only supports RoCE mode. */
+	if (dev->dsr->caps.mode != PVRDMA_DEVICE_MODE_ROCE) {
+		dev_err(&pdev->dev, "unsupported transport %d\n",
+			dev->dsr->caps.mode);
+		ret = -EFAULT;
+		goto err_free_cq_ring;
+	}
+
+	/* Currently, the driver only supports RoCE V1. */
+	if (!(dev->dsr->caps.gid_types & PVRDMA_GID_TYPE_FLAG_ROCE_V1)) {
+		dev_err(&pdev->dev, "driver needs RoCE v1 support\n");
+		ret = -EFAULT;
+		goto err_free_cq_ring;
+	}
+
+	/* Paired vmxnet3 will have same bus, slot. But func will be 0 */
+	pdev_net = pci_get_slot(pdev->bus, PCI_DEVFN(PCI_SLOT(pdev->devfn), 0));
+	if (!pdev_net) {
+		dev_err(&pdev->dev, "failed to find paired net device\n");
+		ret = -ENODEV;
+		goto err_free_cq_ring;
+	}
+
+	if (pdev_net->vendor != PCI_VENDOR_ID_VMWARE ||
+	    pdev_net->device != PCI_DEVICE_ID_VMWARE_VMXNET3) {
+		dev_err(&pdev->dev, "failed to find paired vmxnet3 device\n");
+		pci_dev_put(pdev_net);
+		ret = -ENODEV;
+		goto err_free_cq_ring;
+	}
+
+	dev->netdev = pci_get_drvdata(pdev_net);
+	pci_dev_put(pdev_net);
+	if (!dev->netdev) {
+		dev_err(&pdev->dev, "failed to get vmxnet3 device\n");
+		ret = -ENODEV;
+		goto err_free_cq_ring;
+	}
+
+	dev_info(&pdev->dev, "paired device to %s\n", dev->netdev->name);
+
+	/* Interrupt setup */
+	ret = pvrdma_alloc_intrs(dev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to allocate interrupts\n");
+		ret = -ENOMEM;
+		goto err_netdevice;
+	}
+
+	/* Allocate UAR table. */
+	ret = pvrdma_uar_table_init(dev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to allocate UAR table\n");
+		ret = -ENOMEM;
+		goto err_free_intrs;
+	}
+
+	/* Allocate GID table */
+	dev->sgid_tbl = kcalloc(dev->dsr->caps.gid_tbl_len,
+				sizeof(union ib_gid), GFP_KERNEL);
+	if (!dev->sgid_tbl) {
+		ret = -ENOMEM;
+		goto err_free_uar_table;
+	}
+	dev_dbg(&pdev->dev, "gid table len %d\n", dev->dsr->caps.gid_tbl_len);
+
+	pvrdma_enable_intrs(dev);
+
+	/* Activate pvrdma device */
+	pvrdma_write_reg(dev, PVRDMA_REG_CTL, PVRDMA_DEVICE_CTL_ACTIVATE);
+
+	/* Make sure the write is complete before reading status. */
+	mb();
+
+	/* Check if device was successfully activated */
+	ret = pvrdma_read_reg(dev, PVRDMA_REG_ERR);
+	if (ret != 0) {
+		dev_err(&pdev->dev, "failed to activate device\n");
+		ret = -EFAULT;
+		goto err_disable_intr;
+	}
+
+	/* Register IB device */
+	ret = pvrdma_register_device(dev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to register IB device\n");
+		goto err_disable_intr;
+	}
+
+	dev->nb_netdev.notifier_call = pvrdma_netdevice_event;
+	ret = register_netdevice_notifier(&dev->nb_netdev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to register netdevice events\n");
+		goto err_unreg_ibdev;
+	}
+
+	dev_info(&pdev->dev, "attached to device\n");
+	return 0;
+
+err_unreg_ibdev:
+	ib_unregister_device(&dev->ib_dev);
+err_disable_intr:
+	pvrdma_disable_intrs(dev);
+	kfree(dev->sgid_tbl);
+err_free_uar_table:
+	pvrdma_uar_table_cleanup(dev);
+err_free_intrs:
+	pvrdma_free_irq(dev);
+	pvrdma_disable_msi_all(dev);
+err_netdevice:
+	unregister_netdevice_notifier(&dev->nb_netdev);
+err_free_cq_ring:
+	pvrdma_page_dir_cleanup(dev, &dev->cq_pdir);
+err_free_async_ring:
+	pvrdma_page_dir_cleanup(dev, &dev->async_pdir);
+err_free_slots:
+	pvrdma_free_slots(dev);
+err_free_dsr:
+	dma_free_coherent(&pdev->dev, sizeof(*dev->dsr), dev->dsr,
+			  dev->dsrbase);
+err_uar_unmap:
+	iounmap(dev->driver_uar.map);
+err_unmap_regs:
+	iounmap(dev->regs);
+err_free_resource:
+	pci_release_regions(pdev);
+err_disable_pdev:
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+err_free_device:
+	mutex_lock(&pvrdma_device_list_lock);
+	list_del(&dev->device_link);
+	mutex_unlock(&pvrdma_device_list_lock);
+	ib_dealloc_device(&dev->ib_dev);
+	return ret;
+}
+
+static void pvrdma_pci_remove(struct pci_dev *pdev)
+{
+	struct pvrdma_dev *dev = pci_get_drvdata(pdev);
+
+	if (!dev)
+		return;
+
+	dev_info(&pdev->dev, "detaching from device\n");
+
+	unregister_netdevice_notifier(&dev->nb_netdev);
+	dev->nb_netdev.notifier_call = NULL;
+
+	flush_workqueue(event_wq);
+
+	/* Unregister ib device */
+	ib_unregister_device(&dev->ib_dev);
+
+	mutex_lock(&pvrdma_device_list_lock);
+	list_del(&dev->device_link);
+	mutex_unlock(&pvrdma_device_list_lock);
+
+	pvrdma_disable_intrs(dev);
+	pvrdma_free_irq(dev);
+	pvrdma_disable_msi_all(dev);
+
+	/* Deactivate pvrdma device */
+	pvrdma_write_reg(dev, PVRDMA_REG_CTL, PVRDMA_DEVICE_CTL_RESET);
+	pvrdma_page_dir_cleanup(dev, &dev->cq_pdir);
+	pvrdma_page_dir_cleanup(dev, &dev->async_pdir);
+	pvrdma_free_slots(dev);
+
+	iounmap(dev->regs);
+	kfree(dev->sgid_tbl);
+	kfree(dev->cq_tbl);
+	kfree(dev->qp_tbl);
+	pvrdma_uar_table_cleanup(dev);
+	iounmap(dev->driver_uar.map);
+
+	ib_dealloc_device(&dev->ib_dev);
+
+	/* Free pci resources */
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+}
+
+static struct pci_device_id pvrdma_pci_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_VMWARE, PCI_DEVICE_ID_VMWARE_PVRDMA), },
+	{ 0 },
+};
+
+MODULE_DEVICE_TABLE(pci, pvrdma_pci_table);
+
+static struct pci_driver pvrdma_driver = {
+	.name		= DRV_NAME,
+	.id_table	= pvrdma_pci_table,
+	.probe		= pvrdma_pci_probe,
+	.remove		= pvrdma_pci_remove,
+};
+
+static int __init pvrdma_init(void)
+{
+	int err;
+
+	event_wq = alloc_ordered_workqueue("pvrdma_event_wq", WQ_MEM_RECLAIM);
+	if (!event_wq)
+		return -ENOMEM;
+
+	err = pci_register_driver(&pvrdma_driver);
+	if (err)
+		destroy_workqueue(event_wq);
+
+	return err;
+}
+
+static void __exit pvrdma_cleanup(void)
+{
+	pci_unregister_driver(&pvrdma_driver);
+
+	destroy_workqueue(event_wq);
+}
+
+module_init(pvrdma_init);
+module_exit(pvrdma_cleanup);
+
+MODULE_AUTHOR("VMware, Inc");
+MODULE_DESCRIPTION("VMware Paravirtual RDMA driver");
+MODULE_VERSION(DRV_VERSION);
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
new file mode 100644
index 000000000000..948b5ccd2a70
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
@@ -0,0 +1,304 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/bitmap.h>
+
+#include "pvrdma.h"
+
+int pvrdma_page_dir_init(struct pvrdma_dev *dev, struct pvrdma_page_dir *pdir,
+			 u64 npages, bool alloc_pages)
+{
+	u64 i;
+
+	if (npages > PVRDMA_PAGE_DIR_MAX_PAGES)
+		return -EINVAL;
+
+	memset(pdir, 0, sizeof(*pdir));
+
+	pdir->dir = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
+				       &pdir->dir_dma, GFP_KERNEL);
+	if (!pdir->dir)
+		goto err;
+
+	pdir->ntables = PVRDMA_PAGE_DIR_TABLE(npages - 1) + 1;
+	pdir->tables = kcalloc(pdir->ntables, sizeof(*pdir->tables),
+			       GFP_KERNEL);
+	if (!pdir->tables)
+		goto err;
+
+	for (i = 0; i < pdir->ntables; i++) {
+		pdir->tables[i] = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
+						(dma_addr_t *)&pdir->dir[i],
+						GFP_KERNEL);
+		if (!pdir->tables[i])
+			goto err;
+	}
+
+	pdir->npages = npages;
+
+	if (alloc_pages) {
+		pdir->pages = kcalloc(npages, sizeof(*pdir->pages),
+				      GFP_KERNEL);
+		if (!pdir->pages)
+			goto err;
+
+		for (i = 0; i < pdir->npages; i++) {
+			dma_addr_t page_dma;
+
+			pdir->pages[i] = dma_alloc_coherent(&dev->pdev->dev,
+							    PAGE_SIZE,
+							    &page_dma,
+							    GFP_KERNEL);
+			if (!pdir->pages[i])
+				goto err;
+
+			pvrdma_page_dir_insert_dma(pdir, i, page_dma);
+		}
+	}
+
+	return 0;
+
+err:
+	pvrdma_page_dir_cleanup(dev, pdir);
+
+	return -ENOMEM;
+}
+
+static u64 *pvrdma_page_dir_table(struct pvrdma_page_dir *pdir, u64 idx)
+{
+	return pdir->tables[PVRDMA_PAGE_DIR_TABLE(idx)];
+}
+
+dma_addr_t pvrdma_page_dir_get_dma(struct pvrdma_page_dir *pdir, u64 idx)
+{
+	return pvrdma_page_dir_table(pdir, idx)[PVRDMA_PAGE_DIR_PAGE(idx)];
+}
+
+static void pvrdma_page_dir_cleanup_pages(struct pvrdma_dev *dev,
+					  struct pvrdma_page_dir *pdir)
+{
+	if (pdir->pages) {
+		u64 i;
+
+		for (i = 0; i < pdir->npages && pdir->pages[i]; i++) {
+			dma_addr_t page_dma = pvrdma_page_dir_get_dma(pdir, i);
+
+			dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
+					  pdir->pages[i], page_dma);
+		}
+
+		kfree(pdir->pages);
+	}
+}
+
+static void pvrdma_page_dir_cleanup_tables(struct pvrdma_dev *dev,
+					   struct pvrdma_page_dir *pdir)
+{
+	if (pdir->tables) {
+		int i;
+
+		pvrdma_page_dir_cleanup_pages(dev, pdir);
+
+		for (i = 0; i < pdir->ntables; i++) {
+			u64 *table = pdir->tables[i];
+
+			if (table)
+				dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
+						  table, pdir->dir[i]);
+		}
+
+		kfree(pdir->tables);
+	}
+}
+
+void pvrdma_page_dir_cleanup(struct pvrdma_dev *dev,
+			     struct pvrdma_page_dir *pdir)
+{
+	if (pdir->dir) {
+		pvrdma_page_dir_cleanup_tables(dev, pdir);
+		dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
+				  pdir->dir, pdir->dir_dma);
+	}
+}
+
+int pvrdma_page_dir_insert_dma(struct pvrdma_page_dir *pdir, u64 idx,
+			       dma_addr_t daddr)
+{
+	u64 *table;
+
+	if (idx >= pdir->npages)
+		return -EINVAL;
+
+	table = pvrdma_page_dir_table(pdir, idx);
+	table[PVRDMA_PAGE_DIR_PAGE(idx)] = daddr;
+
+	return 0;
+}
+
+int pvrdma_page_dir_insert_umem(struct pvrdma_page_dir *pdir,
+				struct ib_umem *umem, u64 offset)
+{
+	u64 i = offset;
+	int j, entry;
+	int ret = 0, len = 0;
+	struct scatterlist *sg;
+
+	if (offset >= pdir->npages)
+		return -EINVAL;
+
+	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
+		len = sg_dma_len(sg) >> PAGE_SHIFT;
+		for (j = 0; j < len; j++) {
+			dma_addr_t addr = sg_dma_address(sg) +
+					  umem->page_size * j;
+
+			ret = pvrdma_page_dir_insert_dma(pdir, i, addr);
+			if (ret)
+				goto exit;
+
+			i++;
+		}
+	}
+
+exit:
+	return ret;
+}
+
+int pvrdma_page_dir_insert_page_list(struct pvrdma_page_dir *pdir,
+				     u64 *page_list,
+				     int num_pages)
+{
+	int i;
+	int ret;
+
+	if (num_pages > pdir->npages)
+		return -EINVAL;
+
+	for (i = 0; i < num_pages; i++) {
+		ret = pvrdma_page_dir_insert_dma(pdir, i, page_list[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+void pvrdma_qp_cap_to_ib(struct ib_qp_cap *dst, const struct pvrdma_qp_cap *src)
+{
+	dst->max_send_wr = src->max_send_wr;
+	dst->max_recv_wr = src->max_recv_wr;
+	dst->max_send_sge = src->max_send_sge;
+	dst->max_recv_sge = src->max_recv_sge;
+	dst->max_inline_data = src->max_inline_data;
+}
+
+void ib_qp_cap_to_pvrdma(struct pvrdma_qp_cap *dst, const struct ib_qp_cap *src)
+{
+	dst->max_send_wr = src->max_send_wr;
+	dst->max_recv_wr = src->max_recv_wr;
+	dst->max_send_sge = src->max_send_sge;
+	dst->max_recv_sge = src->max_recv_sge;
+	dst->max_inline_data = src->max_inline_data;
+}
+
+void pvrdma_gid_to_ib(union ib_gid *dst, const union pvrdma_gid *src)
+{
+	BUILD_BUG_ON(sizeof(union pvrdma_gid) != sizeof(union ib_gid));
+	memcpy(dst, src, sizeof(*src));
+}
+
+void ib_gid_to_pvrdma(union pvrdma_gid *dst, const union ib_gid *src)
+{
+	BUILD_BUG_ON(sizeof(union pvrdma_gid) != sizeof(union ib_gid));
+	memcpy(dst, src, sizeof(*src));
+}
+
+void pvrdma_global_route_to_ib(struct ib_global_route *dst,
+			       const struct pvrdma_global_route *src)
+{
+	pvrdma_gid_to_ib(&dst->dgid, &src->dgid);
+	dst->flow_label = src->flow_label;
+	dst->sgid_index = src->sgid_index;
+	dst->hop_limit = src->hop_limit;
+	dst->traffic_class = src->traffic_class;
+}
+
+void ib_global_route_to_pvrdma(struct pvrdma_global_route *dst,
+			       const struct ib_global_route *src)
+{
+	ib_gid_to_pvrdma(&dst->dgid, &src->dgid);
+	dst->flow_label = src->flow_label;
+	dst->sgid_index = src->sgid_index;
+	dst->hop_limit = src->hop_limit;
+	dst->traffic_class = src->traffic_class;
+}
+
+void pvrdma_ah_attr_to_ib(struct ib_ah_attr *dst,
+			  const struct pvrdma_ah_attr *src)
+{
+	pvrdma_global_route_to_ib(&dst->grh, &src->grh);
+	dst->dlid = src->dlid;
+	dst->sl = src->sl;
+	dst->src_path_bits = src->src_path_bits;
+	dst->static_rate = src->static_rate;
+	dst->ah_flags = src->ah_flags;
+	dst->port_num = src->port_num;
+	memcpy(&dst->dmac, &src->dmac, sizeof(dst->dmac));
+}
+
+void ib_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
+			  const struct ib_ah_attr *src)
+{
+	ib_global_route_to_pvrdma(&dst->grh, &src->grh);
+	dst->dlid = src->dlid;
+	dst->sl = src->sl;
+	dst->src_path_bits = src->src_path_bits;
+	dst->static_rate = src->static_rate;
+	dst->ah_flags = src->ah_flags;
+	dst->port_num = src->port_num;
+	memcpy(&dst->dmac, &src->dmac, sizeof(dst->dmac));
+}
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
new file mode 100644
index 000000000000..8519f3212e52
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
@@ -0,0 +1,334 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/list.h>
+#include <linux/slab.h>
+
+#include "pvrdma.h"
+
+/**
+ * pvrdma_get_dma_mr - get a DMA memory region
+ * @pd: protection domain
+ * @acc: access flags
+ *
+ * @return: ib_mr pointer on success, otherwise returns an errno.
+ */
+struct ib_mr *pvrdma_get_dma_mr(struct ib_pd *pd, int acc)
+{
+	struct pvrdma_dev *dev = to_vdev(pd->device);
+	struct pvrdma_user_mr *mr;
+	union pvrdma_cmd_req req;
+	union pvrdma_cmd_resp rsp;
+	struct pvrdma_cmd_create_mr *cmd = &req.create_mr;
+	struct pvrdma_cmd_create_mr_resp *resp = &rsp.create_mr_resp;
+	int ret;
+
+	/* Support only LOCAL_WRITE flag for DMA MRs */
+	if (acc & ~IB_ACCESS_LOCAL_WRITE) {
+		dev_warn(&dev->pdev->dev,
+			 "unsupported dma mr access flags %#x\n", acc);
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr)
+		return ERR_PTR(-ENOMEM);
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_CREATE_MR;
+	cmd->pd_handle = to_vpd(pd)->pd_handle;
+	cmd->access_flags = acc;
+	cmd->flags = PVRDMA_MR_FLAG_DMA;
+
+	ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_CREATE_MR_RESP);
+	if (ret < 0) {
+		dev_warn(&dev->pdev->dev,
+			 "could not get DMA mem region, error: %d\n", ret);
+		kfree(mr);
+		return ERR_PTR(ret);
+	}
+
+	mr->mmr.mr_handle = resp->mr_handle;
+	mr->ibmr.lkey = resp->lkey;
+	mr->ibmr.rkey = resp->rkey;
+
+	return &mr->ibmr;
+}
+
+/**
+ * pvrdma_reg_user_mr - register a userspace memory region
+ * @pd: protection domain
+ * @start: starting address
+ * @length: length of region
+ * @virt_addr: I/O virtual address
+ * @access_flags: access flags for memory region
+ * @udata: user data
+ *
+ * @return: ib_mr pointer on success, otherwise returns an errno.
+ */
+struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
+				 u64 virt_addr, int access_flags,
+				 struct ib_udata *udata)
+{
+	struct pvrdma_dev *dev = to_vdev(pd->device);
+	struct pvrdma_user_mr *mr = NULL;
+	struct ib_umem *umem;
+	union pvrdma_cmd_req req;
+	union pvrdma_cmd_resp rsp;
+	struct pvrdma_cmd_create_mr *cmd = &req.create_mr;
+	struct pvrdma_cmd_create_mr_resp *resp = &rsp.create_mr_resp;
+	int nchunks;
+	int ret;
+	int entry;
+	struct scatterlist *sg;
+
+	if (length == 0 || length > dev->dsr->caps.max_mr_size) {
+		dev_warn(&dev->pdev->dev, "invalid mem region length\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	umem = ib_umem_get(pd->uobject->context, start,
+			   length, access_flags, 0);
+	if (IS_ERR(umem)) {
+		dev_warn(&dev->pdev->dev,
+			 "could not get umem for mem region\n");
+		return ERR_CAST(umem);
+	}
+
+	nchunks = 0;
+	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry)
+		nchunks += sg_dma_len(sg) >> PAGE_SHIFT;
+
+	if (nchunks < 0 || nchunks > PVRDMA_PAGE_DIR_MAX_PAGES) {
+		dev_warn(&dev->pdev->dev, "overflow %d pages in mem region\n",
+			 nchunks);
+		ret = -EINVAL;
+		goto err_umem;
+	}
+
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr) {
+		ret = -ENOMEM;
+		goto err_umem;
+	}
+
+	mr->mmr.iova = virt_addr;
+	mr->mmr.size = length;
+	mr->umem = umem;
+
+	ret = pvrdma_page_dir_init(dev, &mr->pdir, nchunks, false);
+	if (ret) {
+		dev_warn(&dev->pdev->dev,
+			 "could not allocate page directory\n");
+		goto err_umem;
+	}
+
+	ret = pvrdma_page_dir_insert_umem(&mr->pdir, mr->umem, 0);
+	if (ret)
+		goto err_pdir;
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_CREATE_MR;
+	cmd->start = start;
+	cmd->length = length;
+	cmd->pd_handle = to_vpd(pd)->pd_handle;
+	cmd->access_flags = access_flags;
+	cmd->nchunks = nchunks;
+	cmd->pdir_dma = mr->pdir.dir_dma;
+
+	ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_CREATE_MR_RESP);
+	if (ret < 0) {
+		dev_warn(&dev->pdev->dev,
+			 "could not register mem region, error: %d\n", ret);
+		goto err_pdir;
+	}
+
+	mr->mmr.mr_handle = resp->mr_handle;
+	mr->ibmr.lkey = resp->lkey;
+	mr->ibmr.rkey = resp->rkey;
+
+	return &mr->ibmr;
+
+err_pdir:
+	pvrdma_page_dir_cleanup(dev, &mr->pdir);
+err_umem:
+	ib_umem_release(umem);
+	kfree(mr);
+
+	return ERR_PTR(ret);
+}
+
+/**
+ * pvrdma_alloc_mr - allocate a memory region
+ * @pd: protection domain
+ * @mr_type: type of memory region
+ * @max_num_sg: maximum number of pages
+ *
+ * @return: ib_mr pointer on success, otherwise returns an errno.
+ */
+struct ib_mr *pvrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+			      u32 max_num_sg)
+{
+	struct pvrdma_dev *dev = to_vdev(pd->device);
+	struct pvrdma_user_mr *mr;
+	union pvrdma_cmd_req req;
+	union pvrdma_cmd_resp rsp;
+	struct pvrdma_cmd_create_mr *cmd = &req.create_mr;
+	struct pvrdma_cmd_create_mr_resp *resp = &rsp.create_mr_resp;
+	int size = max_num_sg * sizeof(u64);
+	int ret;
+
+	if (mr_type != IB_MR_TYPE_MEM_REG ||
+	    max_num_sg > PVRDMA_MAX_FAST_REG_PAGES)
+		return ERR_PTR(-EINVAL);
+
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr)
+		return ERR_PTR(-ENOMEM);
+
+	mr->pages = kzalloc(size, GFP_KERNEL);
+	if (!mr->pages) {
+		ret = -ENOMEM;
+		goto freemr;
+	}
+
+	ret = pvrdma_page_dir_init(dev, &mr->pdir, max_num_sg, false);
+	if (ret) {
+		dev_warn(&dev->pdev->dev,
+			 "failed to allocate page dir for mr\n");
+		ret = -ENOMEM;
+		goto freepages;
+	}
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_CREATE_MR;
+	cmd->pd_handle = to_vpd(pd)->pd_handle;
+	cmd->access_flags = 0;
+	cmd->flags = PVRDMA_MR_FLAG_FRMR;
+	cmd->nchunks = max_num_sg;
+
+	ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_CREATE_MR_RESP);
+	if (ret < 0) {
+		dev_warn(&dev->pdev->dev,
+			 "could not create FR mem region, error: %d\n", ret);
+		goto freepdir;
+	}
+
+	mr->max_pages = max_num_sg;
+	mr->mmr.mr_handle = resp->mr_handle;
+	mr->ibmr.lkey = resp->lkey;
+	mr->ibmr.rkey = resp->rkey;
+	mr->page_shift = PAGE_SHIFT;
+	mr->umem = NULL;
+
+	return &mr->ibmr;
+
+freepdir:
+	pvrdma_page_dir_cleanup(dev, &mr->pdir);
+freepages:
+	kfree(mr->pages);
+freemr:
+	kfree(mr);
+	return ERR_PTR(ret);
+}
+
+/**
+ * pvrdma_dereg_mr - deregister a memory region
+ * @ibmr: memory region
+ *
+ * @return: 0 on success.
+ */
+int pvrdma_dereg_mr(struct ib_mr *ibmr)
+{
+	struct pvrdma_user_mr *mr = to_vmr(ibmr);
+	struct pvrdma_dev *dev = to_vdev(ibmr->device);
+	union pvrdma_cmd_req req;
+	struct pvrdma_cmd_destroy_mr *cmd = &req.destroy_mr;
+	int ret;
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_DESTROY_MR;
+	cmd->mr_handle = mr->mmr.mr_handle;
+	ret = pvrdma_cmd_post(dev, &req, NULL, 0);
+	if (ret < 0)
+		dev_warn(&dev->pdev->dev,
+			 "could not deregister mem region, error: %d\n", ret);
+
+	pvrdma_page_dir_cleanup(dev, &mr->pdir);
+	if (mr->umem)
+		ib_umem_release(mr->umem);
+
+	kfree(mr->pages);
+	kfree(mr);
+
+	return 0;
+}
+
+static int pvrdma_set_page(struct ib_mr *ibmr, u64 addr)
+{
+	struct pvrdma_user_mr *mr = to_vmr(ibmr);
+
+	if (mr->npages == mr->max_pages)
+		return -ENOMEM;
+
+	mr->pages[mr->npages++] = addr;
+	return 0;
+}
+
+int pvrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
+		     unsigned int *sg_offset)
+{
+	struct pvrdma_user_mr *mr = to_vmr(ibmr);
+	struct pvrdma_dev *dev = to_vdev(ibmr->device);
+	int ret;
+
+	mr->npages = 0;
+
+	ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, pvrdma_set_page);
+	if (ret < 0)
+		dev_warn(&dev->pdev->dev, "could not map sg to pages\n");
+
+	return ret;
+}
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
new file mode 100644
index 000000000000..c8c01e558125
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
@@ -0,0 +1,972 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <asm/page.h>
+#include <linux/io.h>
+#include <linux/wait.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_smi.h>
+#include <rdma/ib_user_verbs.h>
+
+#include "pvrdma.h"
+
+static inline void get_cqs(struct pvrdma_qp *qp, struct pvrdma_cq **send_cq,
+			   struct pvrdma_cq **recv_cq)
+{
+	*send_cq = to_vcq(qp->ibqp.send_cq);
+	*recv_cq = to_vcq(qp->ibqp.recv_cq);
+}
+
+static void pvrdma_lock_cqs(struct pvrdma_cq *scq, struct pvrdma_cq *rcq,
+			    unsigned long *scq_flags,
+			    unsigned long *rcq_flags)
+	__acquires(scq->cq_lock) __acquires(rcq->cq_lock)
+{
+	if (scq == rcq) {
+		spin_lock_irqsave(&scq->cq_lock, *scq_flags);
+		__acquire(rcq->cq_lock);
+	} else if (scq->cq_handle < rcq->cq_handle) {
+		spin_lock_irqsave(&scq->cq_lock, *scq_flags);
+		spin_lock_irqsave_nested(&rcq->cq_lock, *rcq_flags,
+					 SINGLE_DEPTH_NESTING);
+	} else {
+		spin_lock_irqsave(&rcq->cq_lock, *rcq_flags);
+		spin_lock_irqsave_nested(&scq->cq_lock, *scq_flags,
+					 SINGLE_DEPTH_NESTING);
+	}
+}
+
+static void pvrdma_unlock_cqs(struct pvrdma_cq *scq, struct pvrdma_cq *rcq,
+			      unsigned long *scq_flags,
+			      unsigned long *rcq_flags)
+	__releases(scq->cq_lock) __releases(rcq->cq_lock)
+{
+	if (scq == rcq) {
+		__release(rcq->cq_lock);
+		spin_unlock_irqrestore(&scq->cq_lock, *scq_flags);
+	} else if (scq->cq_handle < rcq->cq_handle) {
+		spin_unlock_irqrestore(&rcq->cq_lock, *rcq_flags);
+		spin_unlock_irqrestore(&scq->cq_lock, *scq_flags);
+	} else {
+		spin_unlock_irqrestore(&scq->cq_lock, *scq_flags);
+		spin_unlock_irqrestore(&rcq->cq_lock, *rcq_flags);
+	}
+}
+
+static void pvrdma_reset_qp(struct pvrdma_qp *qp)
+{
+	struct pvrdma_cq *scq, *rcq;
+	unsigned long scq_flags, rcq_flags;
+
+	/* Clean up cqes */
+	get_cqs(qp, &scq, &rcq);
+	pvrdma_lock_cqs(scq, rcq, &scq_flags, &rcq_flags);
+
+	_pvrdma_flush_cqe(qp, scq);
+	if (scq != rcq)
+		_pvrdma_flush_cqe(qp, rcq);
+
+	pvrdma_unlock_cqs(scq, rcq, &scq_flags, &rcq_flags);
+
+	/*
+	 * Reset queuepair. The checks are because usermode queuepairs won't
+	 * have kernel ringstates.
+	 */
+	if (qp->rq.ring) {
+		atomic_set(&qp->rq.ring->cons_head, 0);
+		atomic_set(&qp->rq.ring->prod_tail, 0);
+	}
+	if (qp->sq.ring) {
+		atomic_set(&qp->sq.ring->cons_head, 0);
+		atomic_set(&qp->sq.ring->prod_tail, 0);
+	}
+}
+
+static int pvrdma_set_rq_size(struct pvrdma_dev *dev,
+			      struct ib_qp_cap *req_cap,
+			      struct pvrdma_qp *qp)
+{
+	if (req_cap->max_recv_wr > dev->dsr->caps.max_qp_wr ||
+	    req_cap->max_recv_sge > dev->dsr->caps.max_sge) {
+		dev_warn(&dev->pdev->dev, "recv queue size invalid\n");
+		return -EINVAL;
+	}
+
+	qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, req_cap->max_recv_wr));
+	qp->rq.max_sg = roundup_pow_of_two(max(1U, req_cap->max_recv_sge));
+
+	/* Write back */
+	req_cap->max_recv_wr = qp->rq.wqe_cnt;
+	req_cap->max_recv_sge = qp->rq.max_sg;
+
+	qp->rq.wqe_size = roundup_pow_of_two(sizeof(struct pvrdma_rq_wqe_hdr) +
+					     sizeof(struct pvrdma_sge) *
+					     qp->rq.max_sg);
+	qp->npages_recv = (qp->rq.wqe_cnt * qp->rq.wqe_size + PAGE_SIZE - 1) /
+			  PAGE_SIZE;
+
+	return 0;
+}
+
+static int pvrdma_set_sq_size(struct pvrdma_dev *dev, struct ib_qp_cap *req_cap,
+			      enum ib_qp_type type, struct pvrdma_qp *qp)
+{
+	if (req_cap->max_send_wr > dev->dsr->caps.max_qp_wr ||
+	    req_cap->max_send_sge > dev->dsr->caps.max_sge) {
+		dev_warn(&dev->pdev->dev, "send queue size invalid\n");
+		return -EINVAL;
+	}
+
+	qp->sq.wqe_cnt = roundup_pow_of_two(max(1U, req_cap->max_send_wr));
+	qp->sq.max_sg = roundup_pow_of_two(max(1U, req_cap->max_send_sge));
+
+	/* Write back */
+	req_cap->max_send_wr = qp->sq.wqe_cnt;
+	req_cap->max_send_sge = qp->sq.max_sg;
+
+	qp->sq.wqe_size = roundup_pow_of_two(sizeof(struct pvrdma_sq_wqe_hdr) +
+					     sizeof(struct pvrdma_sge) *
+					     qp->sq.max_sg);
+	/* Note: one extra page for the header. */
+	qp->npages_send = 1 + (qp->sq.wqe_cnt * qp->sq.wqe_size +
+			       PAGE_SIZE - 1) / PAGE_SIZE;
+
+	return 0;
+}
+
+/**
+ * pvrdma_create_qp - create queue pair
+ * @pd: protection domain
+ * @init_attr: queue pair attributes
+ * @udata: user data
+ *
+ * @return: the ib_qp pointer on success, otherwise returns an errno.
+ */
+struct ib_qp *pvrdma_create_qp(struct ib_pd *pd,
+			       struct ib_qp_init_attr *init_attr,
+			       struct ib_udata *udata)
+{
+	struct pvrdma_qp *qp = NULL;
+	struct pvrdma_dev *dev = to_vdev(pd->device);
+	union pvrdma_cmd_req req;
+	union pvrdma_cmd_resp rsp;
+	struct pvrdma_cmd_create_qp *cmd = &req.create_qp;
+	struct pvrdma_cmd_create_qp_resp *resp = &rsp.create_qp_resp;
+	struct pvrdma_create_qp ucmd;
+	unsigned long flags;
+	int ret;
+
+	if (init_attr->create_flags) {
+		dev_warn(&dev->pdev->dev,
+			 "invalid create queuepair flags %#x\n",
+			 init_attr->create_flags);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (init_attr->qp_type != IB_QPT_RC &&
+	    init_attr->qp_type != IB_QPT_UD &&
+	    init_attr->qp_type != IB_QPT_GSI) {
+		dev_warn(&dev->pdev->dev, "queuepair type %d not supported\n",
+			 init_attr->qp_type);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (!atomic_add_unless(&dev->num_qps, 1, dev->dsr->caps.max_qp))
+		return ERR_PTR(-ENOMEM);
+
+	switch (init_attr->qp_type) {
+	case IB_QPT_GSI:
+		if (init_attr->port_num == 0 ||
+		    init_attr->port_num > pd->device->phys_port_cnt ||
+		    udata) {
+			dev_warn(&dev->pdev->dev, "invalid queuepair attrs\n");
+			ret = -EINVAL;
+			goto err_qp;
+		}
+		/* fall through */
+	case IB_QPT_RC:
+	case IB_QPT_UD:
+		qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+		if (!qp) {
+			ret = -ENOMEM;
+			goto err_qp;
+		}
+
+		spin_lock_init(&qp->sq.lock);
+		spin_lock_init(&qp->rq.lock);
+		mutex_init(&qp->mutex);
+		atomic_set(&qp->refcnt, 1);
+		init_waitqueue_head(&qp->wait);
+
+		qp->state = IB_QPS_RESET;
+
+		if (pd->uobject && udata) {
+			dev_dbg(&dev->pdev->dev,
+				"create queuepair from user space\n");
+
+			if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
+				ret = -EFAULT;
+				goto err_qp;
+			}
+
+			/* set qp->sq.wqe_cnt, shift, buf_size.. */
+			qp->rumem = ib_umem_get(pd->uobject->context,
+						ucmd.rbuf_addr,
+						ucmd.rbuf_size, 0, 0);
+			if (IS_ERR(qp->rumem)) {
+				ret = PTR_ERR(qp->rumem);
+				goto err_qp;
+			}
+
+			qp->sumem = ib_umem_get(pd->uobject->context,
+						ucmd.sbuf_addr,
+						ucmd.sbuf_size, 0, 0);
+			if (IS_ERR(qp->sumem)) {
+				ib_umem_release(qp->rumem);
+				ret = PTR_ERR(qp->sumem);
+				goto err_qp;
+			}
+
+			qp->npages_send = ib_umem_page_count(qp->sumem);
+			qp->npages_recv = ib_umem_page_count(qp->rumem);
+			qp->npages = qp->npages_send + qp->npages_recv;
+		} else {
+			qp->is_kernel = true;
+
+			ret = pvrdma_set_sq_size(to_vdev(pd->device),
+						 &init_attr->cap,
+						 init_attr->qp_type, qp);
+			if (ret)
+				goto err_qp;
+
+			ret = pvrdma_set_rq_size(to_vdev(pd->device),
+						 &init_attr->cap, qp);
+			if (ret)
+				goto err_qp;
+
+			qp->npages = qp->npages_send + qp->npages_recv;
+
+			/* Skip header page. */
+			qp->sq.offset = PAGE_SIZE;
+
+			/* Recv queue pages are after send pages. */
+			qp->rq.offset = qp->npages_send * PAGE_SIZE;
+		}
+
+		if (qp->npages < 0 || qp->npages > PVRDMA_PAGE_DIR_MAX_PAGES) {
+			dev_warn(&dev->pdev->dev,
+				 "overflow pages in queuepair\n");
+			ret = -EINVAL;
+			goto err_umem;
+		}
+
+		ret = pvrdma_page_dir_init(dev, &qp->pdir, qp->npages,
+					   qp->is_kernel);
+		if (ret) {
+			dev_warn(&dev->pdev->dev,
+				 "could not allocate page directory\n");
+			goto err_umem;
+		}
+
+		if (!qp->is_kernel) {
+			pvrdma_page_dir_insert_umem(&qp->pdir, qp->sumem, 0);
+			pvrdma_page_dir_insert_umem(&qp->pdir, qp->rumem,
+						    qp->npages_send);
+		} else {
+			/* Ring state is always the first page. */
+			qp->sq.ring = qp->pdir.pages[0];
+			qp->rq.ring = &qp->sq.ring[1];
+		}
+		break;
+	default:
+		ret = -EINVAL;
+		goto err_qp;
+	}
+
+	/* Not supported */
+	init_attr->cap.max_inline_data = 0;
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_CREATE_QP;
+	cmd->pd_handle = to_vpd(pd)->pd_handle;
+	cmd->send_cq_handle = to_vcq(init_attr->send_cq)->cq_handle;
+	cmd->recv_cq_handle = to_vcq(init_attr->recv_cq)->cq_handle;
+	cmd->max_send_wr = init_attr->cap.max_send_wr;
+	cmd->max_recv_wr = init_attr->cap.max_recv_wr;
+	cmd->max_send_sge = init_attr->cap.max_send_sge;
+	cmd->max_recv_sge = init_attr->cap.max_recv_sge;
+	cmd->max_inline_data = init_attr->cap.max_inline_data;
+	cmd->sq_sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;
+	cmd->qp_type = ib_qp_type_to_pvrdma(init_attr->qp_type);
+	cmd->access_flags = IB_ACCESS_LOCAL_WRITE;
+	cmd->total_chunks = qp->npages;
+	cmd->send_chunks = qp->npages_send - 1;
+	cmd->pdir_dma = qp->pdir.dir_dma;
+
+	dev_dbg(&dev->pdev->dev, "create queuepair with %d, %d, %d, %d\n",
+		cmd->max_send_wr, cmd->max_recv_wr, cmd->max_send_sge,
+		cmd->max_recv_sge);
+
+	ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_CREATE_QP_RESP);
+	if (ret < 0) {
+		dev_warn(&dev->pdev->dev,
+			 "could not create queuepair, error: %d\n", ret);
+		goto err_pdir;
+	}
+
+	/* max_send_wr/_recv_wr/_send_sge/_recv_sge/_inline_data */
+	qp->qp_handle = resp->qpn;
+	qp->port = init_attr->port_num;
+	qp->ibqp.qp_num = resp->qpn;
+	spin_lock_irqsave(&dev->qp_tbl_lock, flags);
+	dev->qp_tbl[qp->qp_handle % dev->dsr->caps.max_qp] = qp;
+	spin_unlock_irqrestore(&dev->qp_tbl_lock, flags);
+
+	return &qp->ibqp;
+
+err_pdir:
+	pvrdma_page_dir_cleanup(dev, &qp->pdir);
+err_umem:
+	if (pd->uobject && udata) {
+		if (qp->rumem)
+			ib_umem_release(qp->rumem);
+		if (qp->sumem)
+			ib_umem_release(qp->sumem);
+	}
+err_qp:
+	kfree(qp);
+	atomic_dec(&dev->num_qps);
+
+	return ERR_PTR(ret);
+}
+
+static void pvrdma_free_qp(struct pvrdma_qp *qp)
+{
+	struct pvrdma_dev *dev = to_vdev(qp->ibqp.device);
+	struct pvrdma_cq *scq;
+	struct pvrdma_cq *rcq;
+	unsigned long flags, scq_flags, rcq_flags;
+
+	/* In case cq is polling */
+	get_cqs(qp, &scq, &rcq);
+	pvrdma_lock_cqs(scq, rcq, &scq_flags, &rcq_flags);
+
+	_pvrdma_flush_cqe(qp, scq);
+	if (scq != rcq)
+		_pvrdma_flush_cqe(qp, rcq);
+
+	spin_lock_irqsave(&dev->qp_tbl_lock, flags);
+	dev->qp_tbl[qp->qp_handle] = NULL;
+	spin_unlock_irqrestore(&dev->qp_tbl_lock, flags);
+
+	pvrdma_unlock_cqs(scq, rcq, &scq_flags, &rcq_flags);
+
+	atomic_dec(&qp->refcnt);
+	wait_event(qp->wait, !atomic_read(&qp->refcnt));
+
+	pvrdma_page_dir_cleanup(dev, &qp->pdir);
+
+	kfree(qp);
+
+	atomic_dec(&dev->num_qps);
+}
+
+/**
+ * pvrdma_destroy_qp - destroy a queue pair
+ * @qp: the queue pair to destroy
+ *
+ * @return: 0 on success.
+ */
+int pvrdma_destroy_qp(struct ib_qp *qp)
+{
+	struct pvrdma_qp *vqp = to_vqp(qp);
+	union pvrdma_cmd_req req;
+	struct pvrdma_cmd_destroy_qp *cmd = &req.destroy_qp;
+	int ret;
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_DESTROY_QP;
+	cmd->qp_handle = vqp->qp_handle;
+
+	ret = pvrdma_cmd_post(to_vdev(qp->device), &req, NULL, 0);
+	if (ret < 0)
+		dev_warn(&to_vdev(qp->device)->pdev->dev,
+			 "destroy queuepair failed, error: %d\n", ret);
+
+	pvrdma_free_qp(vqp);
+
+	return 0;
+}
+
+/**
+ * pvrdma_modify_qp - modify queue pair attributes
+ * @ibqp: the queue pair
+ * @attr: the new queue pair's attributes
+ * @attr_mask: attributes mask
+ * @udata: user data
+ *
+ * @returns 0 on success, otherwise returns an errno.
+ */
+int pvrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+		     int attr_mask, struct ib_udata *udata)
+{
+	struct pvrdma_dev *dev = to_vdev(ibqp->device);
+	struct pvrdma_qp *qp = to_vqp(ibqp);
+	union pvrdma_cmd_req req;
+	union pvrdma_cmd_resp rsp;
+	struct pvrdma_cmd_modify_qp *cmd = &req.modify_qp;
+	int cur_state, next_state;
+	int ret;
+
+	/* Sanity checking. Should need lock here */
+	mutex_lock(&qp->mutex);
+	cur_state = (attr_mask & IB_QP_CUR_STATE) ? attr->cur_qp_state :
+		qp->state;
+	next_state = (attr_mask & IB_QP_STATE) ? attr->qp_state : cur_state;
+
+	if (!ib_modify_qp_is_ok(cur_state, next_state, ibqp->qp_type,
+				attr_mask, IB_LINK_LAYER_ETHERNET)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (attr_mask & IB_QP_PORT) {
+		if (attr->port_num == 0 ||
+		    attr->port_num > ibqp->device->phys_port_cnt) {
+			ret = -EINVAL;
+			goto out;
+		}
+	}
+
+	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
+		if (attr->min_rnr_timer > 31) {
+			ret = -EINVAL;
+			goto out;
+		}
+	}
+
+	if (attr_mask & IB_QP_PKEY_INDEX) {
+		if (attr->pkey_index >= dev->dsr->caps.max_pkeys) {
+			ret = -EINVAL;
+			goto out;
+		}
+	}
+
+	if (attr_mask & IB_QP_QKEY)
+		qp->qkey = attr->qkey;
+
+	if (cur_state == next_state && cur_state == IB_QPS_RESET) {
+		ret = 0;
+		goto out;
+	}
+
+	qp->state = next_state;
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_MODIFY_QP;
+	cmd->qp_handle = qp->qp_handle;
+	cmd->attr_mask = ib_qp_attr_mask_to_pvrdma(attr_mask);
+	cmd->attrs.qp_state = ib_qp_state_to_pvrdma(attr->qp_state);
+	cmd->attrs.cur_qp_state =
+		ib_qp_state_to_pvrdma(attr->cur_qp_state);
+	cmd->attrs.path_mtu = ib_mtu_to_pvrdma(attr->path_mtu);
+	cmd->attrs.path_mig_state =
+		ib_mig_state_to_pvrdma(attr->path_mig_state);
+	cmd->attrs.qkey = attr->qkey;
+	cmd->attrs.rq_psn = attr->rq_psn;
+	cmd->attrs.sq_psn = attr->sq_psn;
+	cmd->attrs.dest_qp_num = attr->dest_qp_num;
+	cmd->attrs.qp_access_flags =
+		ib_access_flags_to_pvrdma(attr->qp_access_flags);
+	cmd->attrs.pkey_index = attr->pkey_index;
+	cmd->attrs.alt_pkey_index = attr->alt_pkey_index;
+	cmd->attrs.en_sqd_async_notify = attr->en_sqd_async_notify;
+	cmd->attrs.sq_draining = attr->sq_draining;
+	cmd->attrs.max_rd_atomic = attr->max_rd_atomic;
+	cmd->attrs.max_dest_rd_atomic = attr->max_dest_rd_atomic;
+	cmd->attrs.min_rnr_timer = attr->min_rnr_timer;
+	cmd->attrs.port_num = attr->port_num;
+	cmd->attrs.timeout = attr->timeout;
+	cmd->attrs.retry_cnt = attr->retry_cnt;
+	cmd->attrs.rnr_retry = attr->rnr_retry;
+	cmd->attrs.alt_port_num = attr->alt_port_num;
+	cmd->attrs.alt_timeout = attr->alt_timeout;
+	ib_qp_cap_to_pvrdma(&cmd->attrs.cap, &attr->cap);
+	ib_ah_attr_to_pvrdma(&cmd->attrs.ah_attr, &attr->ah_attr);
+	ib_ah_attr_to_pvrdma(&cmd->attrs.alt_ah_attr, &attr->alt_ah_attr);
+
+	ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_MODIFY_QP_RESP);
+	if (ret < 0) {
+		dev_warn(&dev->pdev->dev,
+			 "could not modify queuepair, error: %d\n", ret);
+	} else if (rsp.hdr.err > 0) {
+		dev_warn(&dev->pdev->dev,
+			 "cannot modify queuepair, error: %d\n", rsp.hdr.err);
+		ret = -EINVAL;
+	}
+
+	if (ret == 0 && next_state == IB_QPS_RESET)
+		pvrdma_reset_qp(qp);
+
+out:
+	mutex_unlock(&qp->mutex);
+
+	return ret;
+}
+
+static inline void *get_sq_wqe(struct pvrdma_qp *qp, int n)
+{
+	return pvrdma_page_dir_get_ptr(&qp->pdir,
+				       qp->sq.offset + n * qp->sq.wqe_size);
+}
+
+static inline void *get_rq_wqe(struct pvrdma_qp *qp, int n)
+{
+	return pvrdma_page_dir_get_ptr(&qp->pdir,
+				       qp->rq.offset + n * qp->rq.wqe_size);
+}
+
+static int set_reg_seg(struct pvrdma_sq_wqe_hdr *wqe_hdr, struct ib_reg_wr *wr)
+{
+	struct pvrdma_user_mr *mr = to_vmr(wr->mr);
+
+	wqe_hdr->wr.fast_reg.iova_start = mr->ibmr.iova;
+	wqe_hdr->wr.fast_reg.pl_pdir_dma = mr->pdir.dir_dma;
+	wqe_hdr->wr.fast_reg.page_shift = mr->page_shift;
+	wqe_hdr->wr.fast_reg.page_list_len = mr->npages;
+	wqe_hdr->wr.fast_reg.length = mr->ibmr.length;
+	wqe_hdr->wr.fast_reg.access_flags = wr->access;
+	wqe_hdr->wr.fast_reg.rkey = wr->key;
+
+	return pvrdma_page_dir_insert_page_list(&mr->pdir, mr->pages,
+						mr->npages);
+}
+
+/**
+ * pvrdma_post_send - post send work request entries on a QP
+ * @ibqp: the QP
+ * @wr: work request list to post
+ * @bad_wr: the first bad WR returned
+ *
+ * @return: 0 on success, otherwise errno returned.
+ */
+int pvrdma_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
+		     struct ib_send_wr **bad_wr)
+{
+	struct pvrdma_qp *qp = to_vqp(ibqp);
+	struct pvrdma_dev *dev = to_vdev(ibqp->device);
+	unsigned long flags;
+	struct pvrdma_sq_wqe_hdr *wqe_hdr;
+	struct pvrdma_sge *sge;
+	int i, index;
+	int nreq;
+	int ret;
+
+	/*
+	 * In states lower than RTS, we can fail immediately. In other states,
+	 * just post and let the device figure it out.
+	 */
+	if (qp->state < IB_QPS_RTS) {
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&qp->sq.lock, flags);
+
+	index = pvrdma_idx(&qp->sq.ring->prod_tail, qp->sq.wqe_cnt);
+	for (nreq = 0; wr; nreq++, wr = wr->next) {
+		unsigned int tail;
+
+		if (unlikely(!pvrdma_idx_ring_has_space(
+				qp->sq.ring, qp->sq.wqe_cnt, &tail))) {
+			dev_warn_ratelimited(&dev->pdev->dev,
+					     "send queue is full\n");
+			*bad_wr = wr;
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		if (unlikely(wr->num_sge > qp->sq.max_sg || wr->num_sge < 0)) {
+			dev_warn_ratelimited(&dev->pdev->dev,
+					     "send SGE overflow\n");
+			*bad_wr = wr;
+			ret = -EINVAL;
+			goto out;
+		}
+
+		if (unlikely(wr->opcode < 0)) {
+			dev_warn_ratelimited(&dev->pdev->dev,
+					     "invalid send opcode\n");
+			*bad_wr = wr;
+			ret = -EINVAL;
+			goto out;
+		}
+
+		/*
+		 * Only support UD, RC.
+		 * Need to check opcode table for thorough checking.
+		 * opcode		_UD	_UC	_RC
+		 * _SEND		x	x	x
+		 * _SEND_WITH_IMM	x	x	x
+		 * _RDMA_WRITE			x	x
+		 * _RDMA_WRITE_WITH_IMM		x	x
+		 * _LOCAL_INV			x	x
+		 * _SEND_WITH_INV		x	x
+		 * _RDMA_READ				x
+		 * _ATOMIC_CMP_AND_SWP			x
+		 * _ATOMIC_FETCH_AND_ADD		x
+		 * _MASK_ATOMIC_CMP_AND_SWP		x
+		 * _MASK_ATOMIC_FETCH_AND_ADD		x
+		 * _REG_MR				x
+		 *
+		 */
+		if (qp->ibqp.qp_type != IB_QPT_UD &&
+		    qp->ibqp.qp_type != IB_QPT_RC &&
+			wr->opcode != IB_WR_SEND) {
+			dev_warn_ratelimited(&dev->pdev->dev,
+					     "unsupported queuepair type\n");
+			*bad_wr = wr;
+			ret = -EINVAL;
+			goto out;
+		} else if (qp->ibqp.qp_type == IB_QPT_UD ||
+			   qp->ibqp.qp_type == IB_QPT_GSI) {
+			if (wr->opcode != IB_WR_SEND &&
+			    wr->opcode != IB_WR_SEND_WITH_IMM) {
+				dev_warn_ratelimited(&dev->pdev->dev,
+						     "invalid send opcode\n");
+				*bad_wr = wr;
+				ret = -EINVAL;
+				goto out;
+			}
+		}
+
+		wqe_hdr = (struct pvrdma_sq_wqe_hdr *)get_sq_wqe(qp, index);
+		memset(wqe_hdr, 0, sizeof(*wqe_hdr));
+		wqe_hdr->wr_id = wr->wr_id;
+		wqe_hdr->num_sge = wr->num_sge;
+		wqe_hdr->opcode = ib_wr_opcode_to_pvrdma(wr->opcode);
+		wqe_hdr->send_flags = ib_send_flags_to_pvrdma(wr->send_flags);
+		if (wr->opcode == IB_WR_SEND_WITH_IMM ||
+		    wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM)
+			wqe_hdr->ex.imm_data = wr->ex.imm_data;
+
+		switch (qp->ibqp.qp_type) {
+		case IB_QPT_GSI:
+		case IB_QPT_UD:
+			if (unlikely(!ud_wr(wr)->ah)) {
+				dev_warn_ratelimited(&dev->pdev->dev,
+						     "invalid address handle\n");
+				*bad_wr = wr;
+				ret = -EINVAL;
+				goto out;
+			}
+
+			/*
+			 * Use qkey from qp context if high order bit set,
+			 * otherwise from work request.
+			 */
+			wqe_hdr->wr.ud.remote_qpn = ud_wr(wr)->remote_qpn;
+			wqe_hdr->wr.ud.remote_qkey =
+				ud_wr(wr)->remote_qkey & 0x80000000 ?
+				qp->qkey : ud_wr(wr)->remote_qkey;
+			wqe_hdr->wr.ud.av = to_vah(ud_wr(wr)->ah)->av;
+
+			break;
+		case IB_QPT_RC:
+			switch (wr->opcode) {
+			case IB_WR_RDMA_READ:
+			case IB_WR_RDMA_WRITE:
+			case IB_WR_RDMA_WRITE_WITH_IMM:
+				wqe_hdr->wr.rdma.remote_addr =
+					rdma_wr(wr)->remote_addr;
+				wqe_hdr->wr.rdma.rkey = rdma_wr(wr)->rkey;
+				break;
+			case IB_WR_LOCAL_INV:
+			case IB_WR_SEND_WITH_INV:
+				wqe_hdr->ex.invalidate_rkey =
+					wr->ex.invalidate_rkey;
+				break;
+			case IB_WR_ATOMIC_CMP_AND_SWP:
+			case IB_WR_ATOMIC_FETCH_AND_ADD:
+				wqe_hdr->wr.atomic.remote_addr =
+					atomic_wr(wr)->remote_addr;
+				wqe_hdr->wr.atomic.rkey = atomic_wr(wr)->rkey;
+				wqe_hdr->wr.atomic.compare_add =
+					atomic_wr(wr)->compare_add;
+				if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP)
+					wqe_hdr->wr.atomic.swap =
+						atomic_wr(wr)->swap;
+				break;
+			case IB_WR_REG_MR:
+				ret = set_reg_seg(wqe_hdr, reg_wr(wr));
+				if (ret < 0) {
+					dev_warn_ratelimited(&dev->pdev->dev,
+							     "Failed to set fast register work request\n");
+					*bad_wr = wr;
+					goto out;
+				}
+				break;
+			default:
+				break;
+			}
+
+			break;
+		default:
+			dev_warn_ratelimited(&dev->pdev->dev,
+					     "invalid queuepair type\n");
+			ret = -EINVAL;
+			*bad_wr = wr;
+			goto out;
+		}
+
+		sge = (struct pvrdma_sge *)(wqe_hdr + 1);
+		for (i = 0; i < wr->num_sge; i++) {
+			/* Need to check wqe_size 0 or max size */
+			sge->addr = wr->sg_list[i].addr;
+			sge->length = wr->sg_list[i].length;
+			sge->lkey = wr->sg_list[i].lkey;
+			sge++;
+		}
+
+		/* Make sure wqe is written before index update */
+		smp_wmb();
+
+		index++;
+		if (unlikely(index >= qp->sq.wqe_cnt))
+			index = 0;
+		/* Update shared sq ring */
+		pvrdma_idx_ring_inc(&qp->sq.ring->prod_tail,
+				    qp->sq.wqe_cnt);
+	}
+
+	ret = 0;
+
+out:
+	spin_unlock_irqrestore(&qp->sq.lock, flags);
+
+	if (!ret)
+		pvrdma_write_uar_qp(dev, PVRDMA_UAR_QP_SEND | qp->qp_handle);
+
+	return ret;
+}
+
+/**
+ * pvrdma_post_receive - post receive work request entries on a QP
+ * @ibqp: the QP
+ * @wr: the work request list to post
+ * @bad_wr: the first bad WR returned
+ *
+ * @return: 0 on success, otherwise errno returned.
+ */
+int pvrdma_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
+		     struct ib_recv_wr **bad_wr)
+{
+	struct pvrdma_dev *dev = to_vdev(ibqp->device);
+	unsigned long flags;
+	struct pvrdma_qp *qp = to_vqp(ibqp);
+	struct pvrdma_rq_wqe_hdr *wqe_hdr;
+	struct pvrdma_sge *sge;
+	int index, nreq;
+	int ret = 0;
+	int i;
+
+	/*
+	 * In the RESET state, we can fail immediately. For other states,
+	 * just post and let the device figure it out.
+	 */
+	if (qp->state == IB_QPS_RESET) {
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&qp->rq.lock, flags);
+
+	index = pvrdma_idx(&qp->rq.ring->prod_tail, qp->rq.wqe_cnt);
+	for (nreq = 0; wr; nreq++, wr = wr->next) {
+		unsigned int tail;
+
+		if (unlikely(wr->num_sge > qp->rq.max_sg ||
+			     wr->num_sge < 0)) {
+			ret = -EINVAL;
+			*bad_wr = wr;
+			dev_warn_ratelimited(&dev->pdev->dev,
+					     "recv SGE overflow\n");
+			goto out;
+		}
+
+		if (unlikely(!pvrdma_idx_ring_has_space(
+				qp->rq.ring, qp->rq.wqe_cnt, &tail))) {
+			ret = -ENOMEM;
+			*bad_wr = wr;
+			dev_warn_ratelimited(&dev->pdev->dev,
+					     "recv queue full\n");
+			goto out;
+		}
+
+		wqe_hdr = (struct pvrdma_rq_wqe_hdr *)get_rq_wqe(qp, index);
+		wqe_hdr->wr_id = wr->wr_id;
+		wqe_hdr->num_sge = wr->num_sge;
+		wqe_hdr->total_len = 0;
+
+		sge = (struct pvrdma_sge *)(wqe_hdr + 1);
+		for (i = 0; i < wr->num_sge; i++) {
+			sge->addr = wr->sg_list[i].addr;
+			sge->length = wr->sg_list[i].length;
+			sge->lkey = wr->sg_list[i].lkey;
+			sge++;
+		}
+
+		/* Make sure wqe is written before index update */
+		smp_wmb();
+
+		index++;
+		if (unlikely(index >= qp->rq.wqe_cnt))
+			index = 0;
+		/* Update shared rq ring */
+		pvrdma_idx_ring_inc(&qp->rq.ring->prod_tail,
+				    qp->rq.wqe_cnt);
+	}
+
+	spin_unlock_irqrestore(&qp->rq.lock, flags);
+
+	pvrdma_write_uar_qp(dev, PVRDMA_UAR_QP_RECV | qp->qp_handle);
+
+	return ret;
+
+out:
+	spin_unlock_irqrestore(&qp->rq.lock, flags);
+
+	return ret;
+}
+
+/**
+ * pvrdma_query_qp - query a queue pair's attributes
+ * @ibqp: the queue pair to query
+ * @attr: the queue pair's attributes
+ * @attr_mask: attributes mask
+ * @init_attr: initial queue pair attributes
+ *
+ * @returns 0 on success, otherwise returns an errno.
+ */
+int pvrdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+		    int attr_mask, struct ib_qp_init_attr *init_attr)
+{
+	struct pvrdma_dev *dev = to_vdev(ibqp->device);
+	struct pvrdma_qp *qp = to_vqp(ibqp);
+	union pvrdma_cmd_req req;
+	union pvrdma_cmd_resp rsp;
+	struct pvrdma_cmd_query_qp *cmd = &req.query_qp;
+	struct pvrdma_cmd_query_qp_resp *resp = &rsp.query_qp_resp;
+	int ret = 0;
+
+	mutex_lock(&qp->mutex);
+
+	if (qp->state == IB_QPS_RESET) {
+		attr->qp_state = IB_QPS_RESET;
+		goto out;
+	}
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_QUERY_QP;
+	cmd->qp_handle = qp->qp_handle;
+	cmd->attr_mask = ib_qp_attr_mask_to_pvrdma(attr_mask);
+
+	ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_QUERY_QP_RESP);
+	if (ret < 0) {
+		dev_warn(&dev->pdev->dev,
+			 "could not query queuepair, error: %d\n", ret);
+		goto out;
+	}
+
+	attr->qp_state = pvrdma_qp_state_to_ib(resp->attrs.qp_state);
+	attr->cur_qp_state =
+		pvrdma_qp_state_to_ib(resp->attrs.cur_qp_state);
+	attr->path_mtu = pvrdma_mtu_to_ib(resp->attrs.path_mtu);
+	attr->path_mig_state =
+		pvrdma_mig_state_to_ib(resp->attrs.path_mig_state);
+	attr->qkey = resp->attrs.qkey;
+	attr->rq_psn = resp->attrs.rq_psn;
+	attr->sq_psn = resp->attrs.sq_psn;
+	attr->dest_qp_num = resp->attrs.dest_qp_num;
+	attr->qp_access_flags =
+		pvrdma_access_flags_to_ib(resp->attrs.qp_access_flags);
+	attr->pkey_index = resp->attrs.pkey_index;
+	attr->alt_pkey_index = resp->attrs.alt_pkey_index;
+	attr->en_sqd_async_notify = resp->attrs.en_sqd_async_notify;
+	attr->sq_draining = resp->attrs.sq_draining;
+	attr->max_rd_atomic = resp->attrs.max_rd_atomic;
+	attr->max_dest_rd_atomic = resp->attrs.max_dest_rd_atomic;
+	attr->min_rnr_timer = resp->attrs.min_rnr_timer;
+	attr->port_num = resp->attrs.port_num;
+	attr->timeout = resp->attrs.timeout;
+	attr->retry_cnt = resp->attrs.retry_cnt;
+	attr->rnr_retry = resp->attrs.rnr_retry;
+	attr->alt_port_num = resp->attrs.alt_port_num;
+	attr->alt_timeout = resp->attrs.alt_timeout;
+	pvrdma_qp_cap_to_ib(&attr->cap, &resp->attrs.cap);
+	pvrdma_ah_attr_to_ib(&attr->ah_attr, &resp->attrs.ah_attr);
+	pvrdma_ah_attr_to_ib(&attr->alt_ah_attr, &resp->attrs.alt_ah_attr);
+
+	qp->state = attr->qp_state;
+
+	ret = 0;
+
+out:
+	attr->cur_qp_state = attr->qp_state;
+
+	init_attr->event_handler = qp->ibqp.event_handler;
+	init_attr->qp_context = qp->ibqp.qp_context;
+	init_attr->send_cq = qp->ibqp.send_cq;
+	init_attr->recv_cq = qp->ibqp.recv_cq;
+	init_attr->srq = qp->ibqp.srq;
+	init_attr->xrcd = NULL;
+	init_attr->cap = attr->cap;
+	init_attr->sq_sig_type = 0;
+	init_attr->qp_type = qp->ibqp.qp_type;
+	init_attr->create_flags = 0;
+	init_attr->port_num = qp->port;
+
+	mutex_unlock(&qp->mutex);
+	return ret;
+}
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h
new file mode 100644
index 000000000000..ed9022a91a1d
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __PVRDMA_RING_H__
+#define __PVRDMA_RING_H__
+
+#include <linux/types.h>
+
+#define PVRDMA_INVALID_IDX	-1	/* Invalid index. */
+
+struct pvrdma_ring {
+	atomic_t prod_tail;	/* Producer tail. */
+	atomic_t cons_head;	/* Consumer head. */
+};
+
+struct pvrdma_ring_state {
+	struct pvrdma_ring tx;	/* Tx ring. */
+	struct pvrdma_ring rx;	/* Rx ring. */
+};
+
+static inline int pvrdma_idx_valid(__u32 idx, __u32 max_elems)
+{
+	/* Generates fewer instructions than a less-than. */
+	return (idx & ~((max_elems << 1) - 1)) == 0;
+}
+
+static inline __s32 pvrdma_idx(atomic_t *var, __u32 max_elems)
+{
+	const unsigned int idx = atomic_read(var);
+
+	if (pvrdma_idx_valid(idx, max_elems))
+		return idx & (max_elems - 1);
+	return PVRDMA_INVALID_IDX;
+}
+
+static inline void pvrdma_idx_ring_inc(atomic_t *var, __u32 max_elems)
+{
+	__u32 idx = atomic_read(var) + 1;	/* Increment. */
+
+	idx &= (max_elems << 1) - 1;		/* Modulo size, flip gen. */
+	atomic_set(var, idx);
+}
+
+static inline __s32 pvrdma_idx_ring_has_space(const struct pvrdma_ring *r,
+					      __u32 max_elems, __u32 *out_tail)
+{
+	const __u32 tail = atomic_read(&r->prod_tail);
+	const __u32 head = atomic_read(&r->cons_head);
+
+	if (pvrdma_idx_valid(tail, max_elems) &&
+	    pvrdma_idx_valid(head, max_elems)) {
+		*out_tail = tail & (max_elems - 1);
+		return tail != (head ^ max_elems);
+	}
+	return PVRDMA_INVALID_IDX;
+}
+
+static inline __s32 pvrdma_idx_ring_has_data(const struct pvrdma_ring *r,
+					     __u32 max_elems, __u32 *out_head)
+{
+	const __u32 tail = atomic_read(&r->prod_tail);
+	const __u32 head = atomic_read(&r->cons_head);
+
+	if (pvrdma_idx_valid(tail, max_elems) &&
+	    pvrdma_idx_valid(head, max_elems)) {
+		*out_head = head & (max_elems - 1);
+		return tail != head;
+	}
+	return PVRDMA_INVALID_IDX;
+}
+
+static inline bool pvrdma_idx_ring_is_valid_idx(const struct pvrdma_ring *r,
+						__u32 max_elems, __u32 *idx)
+{
+	const __u32 tail = atomic_read(&r->prod_tail);
+	const __u32 head = atomic_read(&r->cons_head);
+
+	if (pvrdma_idx_valid(tail, max_elems) &&
+	    pvrdma_idx_valid(head, max_elems) &&
+	    pvrdma_idx_valid(*idx, max_elems)) {
+		if (tail > head && (*idx < tail && *idx >= head))
+			return true;
+		else if (head > tail && (*idx >= head || *idx < tail))
+			return true;
+	}
+	return false;
+}
+
+#endif /* __PVRDMA_RING_H__ */
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
new file mode 100644
index 000000000000..54891370d18a
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
@@ -0,0 +1,579 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <asm/page.h>
+#include <linux/inet.h>
+#include <linux/io.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_smi.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/vmw_pvrdma-abi.h>
+
+#include "pvrdma.h"
+
+/**
+ * pvrdma_query_device - query device
+ * @ibdev: the device to query
+ * @props: the device properties
+ * @uhw: user data
+ *
+ * @return: 0 on success, otherwise negative errno
+ */
+int pvrdma_query_device(struct ib_device *ibdev,
+			struct ib_device_attr *props,
+			struct ib_udata *uhw)
+{
+	struct pvrdma_dev *dev = to_vdev(ibdev);
+
+	if (uhw->inlen || uhw->outlen)
+		return -EINVAL;
+
+	memset(props, 0, sizeof(*props));
+
+	props->fw_ver = dev->dsr->caps.fw_ver;
+	props->sys_image_guid = dev->dsr->caps.sys_image_guid;
+	props->max_mr_size = dev->dsr->caps.max_mr_size;
+	props->page_size_cap = dev->dsr->caps.page_size_cap;
+	props->vendor_id = dev->dsr->caps.vendor_id;
+	props->vendor_part_id = dev->pdev->device;
+	props->hw_ver = dev->dsr->caps.hw_ver;
+	props->max_qp = dev->dsr->caps.max_qp;
+	props->max_qp_wr = dev->dsr->caps.max_qp_wr;
+	props->device_cap_flags = dev->dsr->caps.device_cap_flags;
+	props->max_sge = dev->dsr->caps.max_sge;
+	props->max_cq = dev->dsr->caps.max_cq;
+	props->max_cqe = dev->dsr->caps.max_cqe;
+	props->max_mr = dev->dsr->caps.max_mr;
+	props->max_pd = dev->dsr->caps.max_pd;
+	props->max_qp_rd_atom = dev->dsr->caps.max_qp_rd_atom;
+	props->max_qp_init_rd_atom = dev->dsr->caps.max_qp_init_rd_atom;
+	props->atomic_cap =
+		dev->dsr->caps.atomic_ops &
+		(PVRDMA_ATOMIC_OP_COMP_SWAP | PVRDMA_ATOMIC_OP_FETCH_ADD) ?
+		IB_ATOMIC_HCA : IB_ATOMIC_NONE;
+	props->masked_atomic_cap = props->atomic_cap;
+	props->max_ah = dev->dsr->caps.max_ah;
+	props->max_pkeys = dev->dsr->caps.max_pkeys;
+	props->local_ca_ack_delay = dev->dsr->caps.local_ca_ack_delay;
+	if ((dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_LOCAL_INV) &&
+	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_REMOTE_INV) &&
+	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_FAST_REG_WR)) {
+		props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
+	}
+
+	return 0;
+}
+
+/**
+ * pvrdma_query_port - query device port attributes
+ * @ibdev: the device to query
+ * @port: the port number
+ * @props: the device properties
+ *
+ * @return: 0 on success, otherwise negative errno
+ */
+int pvrdma_query_port(struct ib_device *ibdev, u8 port,
+		      struct ib_port_attr *props)
+{
+	struct pvrdma_dev *dev = to_vdev(ibdev);
+	union pvrdma_cmd_req req;
+	union pvrdma_cmd_resp rsp;
+	struct pvrdma_cmd_query_port *cmd = &req.query_port;
+	struct pvrdma_cmd_query_port_resp *resp = &rsp.query_port_resp;
+	int err;
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_QUERY_PORT;
+	cmd->port_num = port;
+
+	err = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_QUERY_PORT_RESP);
+	if (err < 0) {
+		dev_warn(&dev->pdev->dev,
+			 "could not query port, error: %d\n", err);
+		return err;
+	}
+
+	memset(props, 0, sizeof(*props));
+
+	props->state = pvrdma_port_state_to_ib(resp->attrs.state);
+	props->max_mtu = pvrdma_mtu_to_ib(resp->attrs.max_mtu);
+	props->active_mtu = pvrdma_mtu_to_ib(resp->attrs.active_mtu);
+	props->gid_tbl_len = resp->attrs.gid_tbl_len;
+	props->port_cap_flags =
+		pvrdma_port_cap_flags_to_ib(resp->attrs.port_cap_flags);
+	props->max_msg_sz = resp->attrs.max_msg_sz;
+	props->bad_pkey_cntr = resp->attrs.bad_pkey_cntr;
+	props->qkey_viol_cntr = resp->attrs.qkey_viol_cntr;
+	props->pkey_tbl_len = resp->attrs.pkey_tbl_len;
+	props->lid = resp->attrs.lid;
+	props->sm_lid = resp->attrs.sm_lid;
+	props->lmc = resp->attrs.lmc;
+	props->max_vl_num = resp->attrs.max_vl_num;
+	props->sm_sl = resp->attrs.sm_sl;
+	props->subnet_timeout = resp->attrs.subnet_timeout;
+	props->init_type_reply = resp->attrs.init_type_reply;
+	props->active_width = pvrdma_port_width_to_ib(resp->attrs.active_width);
+	props->active_speed = pvrdma_port_speed_to_ib(resp->attrs.active_speed);
+	props->phys_state = resp->attrs.phys_state;
+
+	return 0;
+}
+
+/**
+ * pvrdma_query_gid - query device gid
+ * @ibdev: the device to query
+ * @port: the port number
+ * @index: the index
+ * @gid: the device gid value
+ *
+ * @return: 0 on success, otherwise negative errno
+ */
+int pvrdma_query_gid(struct ib_device *ibdev, u8 port, int index,
+		     union ib_gid *gid)
+{
+	struct pvrdma_dev *dev = to_vdev(ibdev);
+
+	if (index >= dev->dsr->caps.gid_tbl_len)
+		return -EINVAL;
+
+	memcpy(gid, &dev->sgid_tbl[index], sizeof(union ib_gid));
+
+	return 0;
+}
+
+/**
+ * pvrdma_query_pkey - query device port's P_Key table
+ * @ibdev: the device to query
+ * @port: the port number
+ * @index: the index
+ * @pkey: the device P_Key value
+ *
+ * @return: 0 on success, otherwise negative errno
+ */
+int pvrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
+		      u16 *pkey)
+{
+	int err = 0;
+	union pvrdma_cmd_req req;
+	union pvrdma_cmd_resp rsp;
+	struct pvrdma_cmd_query_pkey *cmd = &req.query_pkey;
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_QUERY_PKEY;
+	cmd->port_num = port;
+	cmd->index = index;
+
+	err = pvrdma_cmd_post(to_vdev(ibdev), &req, &rsp,
+			      PVRDMA_CMD_QUERY_PKEY_RESP);
+	if (err < 0) {
+		dev_warn(&to_vdev(ibdev)->pdev->dev,
+			 "could not query pkey, error: %d\n", err);
+		return err;
+	}
+
+	*pkey = rsp.query_pkey_resp.pkey;
+
+	return 0;
+}
+
+enum rdma_link_layer pvrdma_port_link_layer(struct ib_device *ibdev,
+					    u8 port)
+{
+	return IB_LINK_LAYER_ETHERNET;
+}
+
+int pvrdma_modify_device(struct ib_device *ibdev, int mask,
+			 struct ib_device_modify *props)
+{
+	unsigned long flags;
+
+	if (mask & ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID |
+		     IB_DEVICE_MODIFY_NODE_DESC)) {
+		dev_warn(&to_vdev(ibdev)->pdev->dev,
+			 "unsupported device modify mask %#x\n", mask);
+		return -EOPNOTSUPP;
+	}
+
+	if (mask & IB_DEVICE_MODIFY_NODE_DESC) {
+		spin_lock_irqsave(&to_vdev(ibdev)->desc_lock, flags);
+		memcpy(ibdev->node_desc, props->node_desc, 64);
+		spin_unlock_irqrestore(&to_vdev(ibdev)->desc_lock, flags);
+	}
+
+	if (mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID) {
+		mutex_lock(&to_vdev(ibdev)->port_mutex);
+		to_vdev(ibdev)->sys_image_guid =
+			cpu_to_be64(props->sys_image_guid);
+		mutex_unlock(&to_vdev(ibdev)->port_mutex);
+	}
+
+	return 0;
+}
+
+/**
+ * pvrdma_modify_port - modify device port attributes
+ * @ibdev: the device to modify
+ * @port: the port number
+ * @mask: attributes to modify
+ * @props: the device properties
+ *
+ * @return: 0 on success, otherwise negative errno
+ */
+int pvrdma_modify_port(struct ib_device *ibdev, u8 port, int mask,
+		       struct ib_port_modify *props)
+{
+	struct ib_port_attr attr;
+	struct pvrdma_dev *vdev = to_vdev(ibdev);
+	int ret;
+
+	if (mask & ~IB_PORT_SHUTDOWN) {
+		dev_warn(&vdev->pdev->dev,
+			 "unsupported port modify mask %#x\n", mask);
+		return -EOPNOTSUPP;
+	}
+
+	mutex_lock(&vdev->port_mutex);
+	ret = pvrdma_query_port(ibdev, port, &attr);
+	if (ret)
+		goto out;
+
+	vdev->port_cap_mask |= props->set_port_cap_mask;
+	vdev->port_cap_mask &= ~props->clr_port_cap_mask;
+
+	if (mask & IB_PORT_SHUTDOWN)
+		vdev->ib_active = false;
+
+out:
+	mutex_unlock(&vdev->port_mutex);
+	return ret;
+}
+
+/**
+ * pvrdma_alloc_ucontext - allocate ucontext
+ * @ibdev: the IB device
+ * @udata: user data
+ *
+ * @return: the ib_ucontext pointer on success, otherwise errno.
+ */
+struct ib_ucontext *pvrdma_alloc_ucontext(struct ib_device *ibdev,
+					  struct ib_udata *udata)
+{
+	struct pvrdma_dev *vdev = to_vdev(ibdev);
+	struct pvrdma_ucontext *context;
+	union pvrdma_cmd_req req;
+	union pvrdma_cmd_resp rsp;
+	struct pvrdma_cmd_create_uc *cmd = &req.create_uc;
+	struct pvrdma_cmd_create_uc_resp *resp = &rsp.create_uc_resp;
+	struct pvrdma_alloc_ucontext_resp uresp;
+	int ret;
+	void *ptr;
+
+	if (!vdev->ib_active)
+		return ERR_PTR(-EAGAIN);
+
+	context = kmalloc(sizeof(*context), GFP_KERNEL);
+	if (!context)
+		return ERR_PTR(-ENOMEM);
+
+	context->dev = vdev;
+	ret = pvrdma_uar_alloc(vdev, &context->uar);
+	if (ret) {
+		kfree(context);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	/* get ctx_handle from host */
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->pfn = context->uar.pfn;
+	cmd->hdr.cmd = PVRDMA_CMD_CREATE_UC;
+	ret = pvrdma_cmd_post(vdev, &req, &rsp, PVRDMA_CMD_CREATE_UC_RESP);
+	if (ret < 0) {
+		dev_warn(&vdev->pdev->dev,
+			 "could not create ucontext, error: %d\n", ret);
+		ptr = ERR_PTR(ret);
+		goto err;
+	}
+
+	context->ctx_handle = resp->ctx_handle;
+
+	/* copy back to user */
+	uresp.qp_tab_size = vdev->dsr->caps.max_qp;
+	ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+	if (ret) {
+		pvrdma_uar_free(vdev, &context->uar);
+		context->ibucontext.device = ibdev;
+		pvrdma_dealloc_ucontext(&context->ibucontext);
+		return ERR_PTR(-EFAULT);
+	}
+
+	return &context->ibucontext;
+
+err:
+	pvrdma_uar_free(vdev, &context->uar);
+	kfree(context);
+	return ptr;
+}
+
+/**
+ * pvrdma_dealloc_ucontext - deallocate ucontext
+ * @ibcontext: the ucontext
+ *
+ * @return: 0 on success, otherwise errno.
+ */
+int pvrdma_dealloc_ucontext(struct ib_ucontext *ibcontext)
+{
+	struct pvrdma_ucontext *context = to_vucontext(ibcontext);
+	union pvrdma_cmd_req req;
+	struct pvrdma_cmd_destroy_uc *cmd = &req.destroy_uc;
+	int ret;
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_DESTROY_UC;
+	cmd->ctx_handle = context->ctx_handle;
+
+	ret = pvrdma_cmd_post(context->dev, &req, NULL, 0);
+	if (ret < 0)
+		dev_warn(&context->dev->pdev->dev,
+			 "destroy ucontext failed, error: %d\n", ret);
+
+	/* Free the UAR even if the device command failed */
+	pvrdma_uar_free(to_vdev(ibcontext->device), &context->uar);
+	kfree(context);
+
+	return ret;
+}
+
+/**
+ * pvrdma_mmap - create mmap region
+ * @ibcontext: the user context
+ * @vma: the VMA
+ *
+ * @return: 0 on success, otherwise errno.
+ */
+int pvrdma_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
+{
+	struct pvrdma_ucontext *context = to_vucontext(ibcontext);
+	unsigned long start = vma->vm_start;
+	unsigned long size = vma->vm_end - vma->vm_start;
+	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
+
+	dev_dbg(&context->dev->pdev->dev, "create mmap region\n");
+
+	if ((size != PAGE_SIZE) || (offset & ~PAGE_MASK)) {
+		dev_warn(&context->dev->pdev->dev,
+			 "invalid params for mmap region\n");
+		return -EINVAL;
+	}
+
+	/* Map UAR to kernel space, VM_LOCKED? */
+	vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	if (io_remap_pfn_range(vma, start, context->uar.pfn, size,
+			       vma->vm_page_prot))
+		return -EAGAIN;
+
+	return 0;
+}
+
+/**
+ * pvrdma_alloc_pd - allocate protection domain
+ * @ibdev: the IB device
+ * @context: user context
+ * @udata: user data
+ *
+ * @return: the ib_pd protection domain pointer on success, otherwise errno.
+ */
+struct ib_pd *pvrdma_alloc_pd(struct ib_device *ibdev,
+			      struct ib_ucontext *context,
+			      struct ib_udata *udata)
+{
+	struct pvrdma_pd *pd;
+	struct pvrdma_dev *dev = to_vdev(ibdev);
+	union pvrdma_cmd_req req;
+	union pvrdma_cmd_resp rsp;
+	struct pvrdma_cmd_create_pd *cmd = &req.create_pd;
+	struct pvrdma_cmd_create_pd_resp *resp = &rsp.create_pd_resp;
+	int ret;
+	void *ptr;
+
+	/* Check allowed max pds */
+	if (!atomic_add_unless(&dev->num_pds, 1, dev->dsr->caps.max_pd))
+		return ERR_PTR(-ENOMEM);
+
+	pd = kmalloc(sizeof(*pd), GFP_KERNEL);
+	if (!pd) {
+		ptr = ERR_PTR(-ENOMEM);
+		goto err;
+	}
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_CREATE_PD;
+	cmd->ctx_handle = (context) ? to_vucontext(context)->ctx_handle : 0;
+	ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_CREATE_PD_RESP);
+	if (ret < 0) {
+		dev_warn(&dev->pdev->dev,
+			 "failed to allocate protection domain, error: %d\n",
+			 ret);
+		ptr = ERR_PTR(ret);
+		goto freepd;
+	}
+
+	pd->privileged = !context;
+	pd->pd_handle = resp->pd_handle;
+	pd->pdn = resp->pd_handle;
+
+	if (context) {
+		if (ib_copy_to_udata(udata, &pd->pdn, sizeof(__u32))) {
+			dev_warn(&dev->pdev->dev,
+				 "failed to copy back protection domain\n");
+			pvrdma_dealloc_pd(&pd->ibpd);
+			return ERR_PTR(-EFAULT);
+		}
+	}
+
+	/* u32 pd handle */
+	return &pd->ibpd;
+
+freepd:
+	kfree(pd);
+err:
+	atomic_dec(&dev->num_pds);
+	return ptr;
+}
+
+/**
+ * pvrdma_dealloc_pd - deallocate protection domain
+ * @pd: the protection domain to be released
+ *
+ * @return: 0 on success, otherwise errno.
+ */
+int pvrdma_dealloc_pd(struct ib_pd *pd)
+{
+	struct pvrdma_dev *dev = to_vdev(pd->device);
+	union pvrdma_cmd_req req;
+	struct pvrdma_cmd_destroy_pd *cmd = &req.destroy_pd;
+	int ret;
+
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->hdr.cmd = PVRDMA_CMD_DESTROY_PD;
+	cmd->pd_handle = to_vpd(pd)->pd_handle;
+
+	ret = pvrdma_cmd_post(dev, &req, NULL, 0);
+	if (ret)
+		dev_warn(&dev->pdev->dev,
+			 "could not dealloc protection domain, error: %d\n",
+			 ret);
+
+	kfree(to_vpd(pd));
+	atomic_dec(&dev->num_pds);
+
+	return 0;
+}
+
+/**
+ * pvrdma_create_ah - create an address handle
+ * @pd: the protection domain
+ * @ah_attr: the attributes of the AH
+ * @udata: user data blob
+ *
+ * @return: the ib_ah pointer on success, otherwise errno.
+ */
+struct ib_ah *pvrdma_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
+			       struct ib_udata *udata)
+{
+	struct pvrdma_dev *dev = to_vdev(pd->device);
+	struct pvrdma_ah *ah;
+	enum rdma_link_layer ll;
+
+	if (!(ah_attr->ah_flags & IB_AH_GRH))
+		return ERR_PTR(-EINVAL);
+
+	ll = rdma_port_get_link_layer(pd->device, ah_attr->port_num);
+
+	if (ll != IB_LINK_LAYER_ETHERNET ||
+	    rdma_is_multicast_addr((struct in6_addr *)ah_attr->grh.dgid.raw))
+		return ERR_PTR(-EINVAL);
+
+	if (!atomic_add_unless(&dev->num_ahs, 1, dev->dsr->caps.max_ah))
+		return ERR_PTR(-ENOMEM);
+
+	ah = kzalloc(sizeof(*ah), GFP_KERNEL);
+	if (!ah) {
+		atomic_dec(&dev->num_ahs);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	ah->av.port_pd = to_vpd(pd)->pd_handle | (ah_attr->port_num << 24);
+	ah->av.src_path_bits = ah_attr->src_path_bits;
+	ah->av.src_path_bits |= 0x80;
+	ah->av.gid_index = ah_attr->grh.sgid_index;
+	ah->av.hop_limit = ah_attr->grh.hop_limit;
+	ah->av.sl_tclass_flowlabel = (ah_attr->grh.traffic_class << 20) |
+				      ah_attr->grh.flow_label;
+	memcpy(ah->av.dgid, ah_attr->grh.dgid.raw, 16);
+	memcpy(ah->av.dmac, ah_attr->dmac, 6);
+
+	ah->ibah.device = pd->device;
+	ah->ibah.pd = pd;
+	ah->ibah.uobject = NULL;
+
+	return &ah->ibah;
+}
+
+/**
+ * pvrdma_destroy_ah - destroy an address handle
+ * @ah: the address handle to destroyed
+ *
+ * @return: 0 on success.
+ */
+int pvrdma_destroy_ah(struct ib_ah *ah)
+{
+	struct pvrdma_dev *dev = to_vdev(ah->device);
+
+	kfree(to_vah(ah));
+	atomic_dec(&dev->num_ahs);
+
+	return 0;
+}
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
new file mode 100644
index 000000000000..bfbe96b56255
--- /dev/null
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
@@ -0,0 +1,436 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __PVRDMA_VERBS_H__
+#define __PVRDMA_VERBS_H__
+
+#include <linux/types.h>
+
+union pvrdma_gid {
+	u8	raw[16];
+	struct {
+		__be64	subnet_prefix;
+		__be64	interface_id;
+	} global;
+};
+
+enum pvrdma_link_layer {
+	PVRDMA_LINK_LAYER_UNSPECIFIED,
+	PVRDMA_LINK_LAYER_INFINIBAND,
+	PVRDMA_LINK_LAYER_ETHERNET,
+};
+
+enum pvrdma_mtu {
+	PVRDMA_MTU_256  = 1,
+	PVRDMA_MTU_512  = 2,
+	PVRDMA_MTU_1024 = 3,
+	PVRDMA_MTU_2048 = 4,
+	PVRDMA_MTU_4096 = 5,
+};
+
+static inline int pvrdma_mtu_enum_to_int(enum pvrdma_mtu mtu)
+{
+	switch (mtu) {
+	case PVRDMA_MTU_256:	return  256;
+	case PVRDMA_MTU_512:	return  512;
+	case PVRDMA_MTU_1024:	return 1024;
+	case PVRDMA_MTU_2048:	return 2048;
+	case PVRDMA_MTU_4096:	return 4096;
+	default:		return   -1;
+	}
+}
+
+static inline enum pvrdma_mtu pvrdma_mtu_int_to_enum(int mtu)
+{
+	switch (mtu) {
+	case 256:	return PVRDMA_MTU_256;
+	case 512:	return PVRDMA_MTU_512;
+	case 1024:	return PVRDMA_MTU_1024;
+	case 2048:	return PVRDMA_MTU_2048;
+	case 4096:
+	default:	return PVRDMA_MTU_4096;
+	}
+}
+
+enum pvrdma_port_state {
+	PVRDMA_PORT_NOP			= 0,
+	PVRDMA_PORT_DOWN		= 1,
+	PVRDMA_PORT_INIT		= 2,
+	PVRDMA_PORT_ARMED		= 3,
+	PVRDMA_PORT_ACTIVE		= 4,
+	PVRDMA_PORT_ACTIVE_DEFER	= 5,
+};
+
+enum pvrdma_port_cap_flags {
+	PVRDMA_PORT_SM				= 1 <<  1,
+	PVRDMA_PORT_NOTICE_SUP			= 1 <<  2,
+	PVRDMA_PORT_TRAP_SUP			= 1 <<  3,
+	PVRDMA_PORT_OPT_IPD_SUP			= 1 <<  4,
+	PVRDMA_PORT_AUTO_MIGR_SUP		= 1 <<  5,
+	PVRDMA_PORT_SL_MAP_SUP			= 1 <<  6,
+	PVRDMA_PORT_MKEY_NVRAM			= 1 <<  7,
+	PVRDMA_PORT_PKEY_NVRAM			= 1 <<  8,
+	PVRDMA_PORT_LED_INFO_SUP		= 1 <<  9,
+	PVRDMA_PORT_SM_DISABLED			= 1 << 10,
+	PVRDMA_PORT_SYS_IMAGE_GUID_SUP		= 1 << 11,
+	PVRDMA_PORT_PKEY_SW_EXT_PORT_TRAP_SUP	= 1 << 12,
+	PVRDMA_PORT_EXTENDED_SPEEDS_SUP		= 1 << 14,
+	PVRDMA_PORT_CM_SUP			= 1 << 16,
+	PVRDMA_PORT_SNMP_TUNNEL_SUP		= 1 << 17,
+	PVRDMA_PORT_REINIT_SUP			= 1 << 18,
+	PVRDMA_PORT_DEVICE_MGMT_SUP		= 1 << 19,
+	PVRDMA_PORT_VENDOR_CLASS_SUP		= 1 << 20,
+	PVRDMA_PORT_DR_NOTICE_SUP		= 1 << 21,
+	PVRDMA_PORT_CAP_MASK_NOTICE_SUP		= 1 << 22,
+	PVRDMA_PORT_BOOT_MGMT_SUP		= 1 << 23,
+	PVRDMA_PORT_LINK_LATENCY_SUP		= 1 << 24,
+	PVRDMA_PORT_CLIENT_REG_SUP		= 1 << 25,
+	PVRDMA_PORT_IP_BASED_GIDS		= 1 << 26,
+	PVRDMA_PORT_CAP_FLAGS_MAX		= PVRDMA_PORT_IP_BASED_GIDS,
+};
+
+enum pvrdma_port_width {
+	PVRDMA_WIDTH_1X		= 1,
+	PVRDMA_WIDTH_4X		= 2,
+	PVRDMA_WIDTH_8X		= 4,
+	PVRDMA_WIDTH_12X	= 8,
+};
+
+static inline int pvrdma_width_enum_to_int(enum pvrdma_port_width width)
+{
+	switch (width) {
+	case PVRDMA_WIDTH_1X:	return  1;
+	case PVRDMA_WIDTH_4X:	return  4;
+	case PVRDMA_WIDTH_8X:	return  8;
+	case PVRDMA_WIDTH_12X:	return 12;
+	default:		return -1;
+	}
+}
+
+enum pvrdma_port_speed {
+	PVRDMA_SPEED_SDR	= 1,
+	PVRDMA_SPEED_DDR	= 2,
+	PVRDMA_SPEED_QDR	= 4,
+	PVRDMA_SPEED_FDR10	= 8,
+	PVRDMA_SPEED_FDR	= 16,
+	PVRDMA_SPEED_EDR	= 32,
+};
+
+struct pvrdma_port_attr {
+	enum pvrdma_port_state	state;
+	enum pvrdma_mtu		max_mtu;
+	enum pvrdma_mtu		active_mtu;
+	u32			gid_tbl_len;
+	u32			port_cap_flags;
+	u32			max_msg_sz;
+	u32			bad_pkey_cntr;
+	u32			qkey_viol_cntr;
+	u16			pkey_tbl_len;
+	u16			lid;
+	u16			sm_lid;
+	u8			lmc;
+	u8			max_vl_num;
+	u8			sm_sl;
+	u8			subnet_timeout;
+	u8			init_type_reply;
+	u8			active_width;
+	u8			active_speed;
+	u8			phys_state;
+	u8			reserved[2];
+};
+
+struct pvrdma_global_route {
+	union pvrdma_gid	dgid;
+	u32			flow_label;
+	u8			sgid_index;
+	u8			hop_limit;
+	u8			traffic_class;
+	u8			reserved;
+};
+
+struct pvrdma_grh {
+	__be32			version_tclass_flow;
+	__be16			paylen;
+	u8			next_hdr;
+	u8			hop_limit;
+	union pvrdma_gid	sgid;
+	union pvrdma_gid	dgid;
+};
+
+enum pvrdma_ah_flags {
+	PVRDMA_AH_GRH = 1,
+};
+
+enum pvrdma_rate {
+	PVRDMA_RATE_PORT_CURRENT	= 0,
+	PVRDMA_RATE_2_5_GBPS		= 2,
+	PVRDMA_RATE_5_GBPS		= 5,
+	PVRDMA_RATE_10_GBPS		= 3,
+	PVRDMA_RATE_20_GBPS		= 6,
+	PVRDMA_RATE_30_GBPS		= 4,
+	PVRDMA_RATE_40_GBPS		= 7,
+	PVRDMA_RATE_60_GBPS		= 8,
+	PVRDMA_RATE_80_GBPS		= 9,
+	PVRDMA_RATE_120_GBPS		= 10,
+	PVRDMA_RATE_14_GBPS		= 11,
+	PVRDMA_RATE_56_GBPS		= 12,
+	PVRDMA_RATE_112_GBPS		= 13,
+	PVRDMA_RATE_168_GBPS		= 14,
+	PVRDMA_RATE_25_GBPS		= 15,
+	PVRDMA_RATE_100_GBPS		= 16,
+	PVRDMA_RATE_200_GBPS		= 17,
+	PVRDMA_RATE_300_GBPS		= 18,
+};
+
+struct pvrdma_ah_attr {
+	struct pvrdma_global_route	grh;
+	u16				dlid;
+	u16				vlan_id;
+	u8				sl;
+	u8				src_path_bits;
+	u8				static_rate;
+	u8				ah_flags;
+	u8				port_num;
+	u8				dmac[6];
+	u8				reserved;
+};
+
+enum pvrdma_cq_notify_flags {
+	PVRDMA_CQ_SOLICITED		= 1 << 0,
+	PVRDMA_CQ_NEXT_COMP		= 1 << 1,
+	PVRDMA_CQ_SOLICITED_MASK	= PVRDMA_CQ_SOLICITED |
+					  PVRDMA_CQ_NEXT_COMP,
+	PVRDMA_CQ_REPORT_MISSED_EVENTS	= 1 << 2,
+};
+
+struct pvrdma_qp_cap {
+	u32	max_send_wr;
+	u32	max_recv_wr;
+	u32	max_send_sge;
+	u32	max_recv_sge;
+	u32	max_inline_data;
+	u32	reserved;
+};
+
+enum pvrdma_sig_type {
+	PVRDMA_SIGNAL_ALL_WR,
+	PVRDMA_SIGNAL_REQ_WR,
+};
+
+enum pvrdma_qp_type {
+	PVRDMA_QPT_SMI,
+	PVRDMA_QPT_GSI,
+	PVRDMA_QPT_RC,
+	PVRDMA_QPT_UC,
+	PVRDMA_QPT_UD,
+	PVRDMA_QPT_RAW_IPV6,
+	PVRDMA_QPT_RAW_ETHERTYPE,
+	PVRDMA_QPT_RAW_PACKET = 8,
+	PVRDMA_QPT_XRC_INI = 9,
+	PVRDMA_QPT_XRC_TGT,
+	PVRDMA_QPT_MAX,
+};
+
+enum pvrdma_qp_create_flags {
+	PVRDMA_QP_CREATE_IPOPVRDMA_UD_LSO		= 1 << 0,
+	PVRDMA_QP_CREATE_BLOCK_MULTICAST_LOOPBACK	= 1 << 1,
+};
+
+enum pvrdma_qp_attr_mask {
+	PVRDMA_QP_STATE			= 1 << 0,
+	PVRDMA_QP_CUR_STATE		= 1 << 1,
+	PVRDMA_QP_EN_SQD_ASYNC_NOTIFY	= 1 << 2,
+	PVRDMA_QP_ACCESS_FLAGS		= 1 << 3,
+	PVRDMA_QP_PKEY_INDEX		= 1 << 4,
+	PVRDMA_QP_PORT			= 1 << 5,
+	PVRDMA_QP_QKEY			= 1 << 6,
+	PVRDMA_QP_AV			= 1 << 7,
+	PVRDMA_QP_PATH_MTU		= 1 << 8,
+	PVRDMA_QP_TIMEOUT		= 1 << 9,
+	PVRDMA_QP_RETRY_CNT		= 1 << 10,
+	PVRDMA_QP_RNR_RETRY		= 1 << 11,
+	PVRDMA_QP_RQ_PSN		= 1 << 12,
+	PVRDMA_QP_MAX_QP_RD_ATOMIC	= 1 << 13,
+	PVRDMA_QP_ALT_PATH		= 1 << 14,
+	PVRDMA_QP_MIN_RNR_TIMER		= 1 << 15,
+	PVRDMA_QP_SQ_PSN		= 1 << 16,
+	PVRDMA_QP_MAX_DEST_RD_ATOMIC	= 1 << 17,
+	PVRDMA_QP_PATH_MIG_STATE	= 1 << 18,
+	PVRDMA_QP_CAP			= 1 << 19,
+	PVRDMA_QP_DEST_QPN		= 1 << 20,
+	PVRDMA_QP_ATTR_MASK_MAX		= PVRDMA_QP_DEST_QPN,
+};
+
+enum pvrdma_qp_state {
+	PVRDMA_QPS_RESET,
+	PVRDMA_QPS_INIT,
+	PVRDMA_QPS_RTR,
+	PVRDMA_QPS_RTS,
+	PVRDMA_QPS_SQD,
+	PVRDMA_QPS_SQE,
+	PVRDMA_QPS_ERR,
+};
+
+enum pvrdma_mig_state {
+	PVRDMA_MIG_MIGRATED,
+	PVRDMA_MIG_REARM,
+	PVRDMA_MIG_ARMED,
+};
+
+enum pvrdma_mw_type {
+	PVRDMA_MW_TYPE_1 = 1,
+	PVRDMA_MW_TYPE_2 = 2,
+};
+
+struct pvrdma_qp_attr {
+	enum pvrdma_qp_state	qp_state;
+	enum pvrdma_qp_state	cur_qp_state;
+	enum pvrdma_mtu		path_mtu;
+	enum pvrdma_mig_state	path_mig_state;
+	u32			qkey;
+	u32			rq_psn;
+	u32			sq_psn;
+	u32			dest_qp_num;
+	u32			qp_access_flags;
+	u16			pkey_index;
+	u16			alt_pkey_index;
+	u8			en_sqd_async_notify;
+	u8			sq_draining;
+	u8			max_rd_atomic;
+	u8			max_dest_rd_atomic;
+	u8			min_rnr_timer;
+	u8			port_num;
+	u8			timeout;
+	u8			retry_cnt;
+	u8			rnr_retry;
+	u8			alt_port_num;
+	u8			alt_timeout;
+	u8			reserved[5];
+	struct pvrdma_qp_cap	cap;
+	struct pvrdma_ah_attr	ah_attr;
+	struct pvrdma_ah_attr	alt_ah_attr;
+};
+
+enum pvrdma_send_flags {
+	PVRDMA_SEND_FENCE	= 1 << 0,
+	PVRDMA_SEND_SIGNALED	= 1 << 1,
+	PVRDMA_SEND_SOLICITED	= 1 << 2,
+	PVRDMA_SEND_INLINE	= 1 << 3,
+	PVRDMA_SEND_IP_CSUM	= 1 << 4,
+	PVRDMA_SEND_FLAGS_MAX	= PVRDMA_SEND_IP_CSUM,
+};
+
+enum pvrdma_access_flags {
+	PVRDMA_ACCESS_LOCAL_WRITE	= 1 << 0,
+	PVRDMA_ACCESS_REMOTE_WRITE	= 1 << 1,
+	PVRDMA_ACCESS_REMOTE_READ	= 1 << 2,
+	PVRDMA_ACCESS_REMOTE_ATOMIC	= 1 << 3,
+	PVRDMA_ACCESS_MW_BIND		= 1 << 4,
+	PVRDMA_ZERO_BASED		= 1 << 5,
+	PVRDMA_ACCESS_ON_DEMAND		= 1 << 6,
+	PVRDMA_ACCESS_FLAGS_MAX		= PVRDMA_ACCESS_ON_DEMAND,
+};
+
+int pvrdma_query_device(struct ib_device *ibdev,
+			struct ib_device_attr *props,
+			struct ib_udata *udata);
+int pvrdma_query_port(struct ib_device *ibdev, u8 port,
+		      struct ib_port_attr *props);
+int pvrdma_query_gid(struct ib_device *ibdev, u8 port,
+		     int index, union ib_gid *gid);
+int pvrdma_query_pkey(struct ib_device *ibdev, u8 port,
+		      u16 index, u16 *pkey);
+enum rdma_link_layer pvrdma_port_link_layer(struct ib_device *ibdev,
+					    u8 port);
+int pvrdma_modify_device(struct ib_device *ibdev, int mask,
+			 struct ib_device_modify *props);
+int pvrdma_modify_port(struct ib_device *ibdev, u8 port,
+		       int mask, struct ib_port_modify *props);
+int pvrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+struct ib_ucontext *pvrdma_alloc_ucontext(struct ib_device *ibdev,
+					  struct ib_udata *udata);
+int pvrdma_dealloc_ucontext(struct ib_ucontext *context);
+struct ib_pd *pvrdma_alloc_pd(struct ib_device *ibdev,
+			      struct ib_ucontext *context,
+			      struct ib_udata *udata);
+int pvrdma_dealloc_pd(struct ib_pd *ibpd);
+struct ib_mr *pvrdma_get_dma_mr(struct ib_pd *pd, int acc);
+struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
+				 u64 virt_addr, int access_flags,
+				 struct ib_udata *udata);
+int pvrdma_dereg_mr(struct ib_mr *mr);
+struct ib_mr *pvrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+			      u32 max_num_sg);
+int pvrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+		     int sg_nents, unsigned int *sg_offset);
+int pvrdma_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
+int pvrdma_resize_cq(struct ib_cq *ibcq, int entries,
+		     struct ib_udata *udata);
+struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
+			       const struct ib_cq_init_attr *attr,
+			       struct ib_ucontext *context,
+			       struct ib_udata *udata);
+int pvrdma_resize_cq(struct ib_cq *ibcq, int entries,
+		     struct ib_udata *udata);
+int pvrdma_destroy_cq(struct ib_cq *cq);
+int pvrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
+int pvrdma_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
+struct ib_ah *pvrdma_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
+			       struct ib_udata *udata);
+int pvrdma_destroy_ah(struct ib_ah *ah);
+struct ib_qp *pvrdma_create_qp(struct ib_pd *pd,
+			       struct ib_qp_init_attr *init_attr,
+			       struct ib_udata *udata);
+int pvrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+		     int attr_mask, struct ib_udata *udata);
+int pvrdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+		    int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
+int pvrdma_destroy_qp(struct ib_qp *qp);
+int pvrdma_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
+		     struct ib_send_wr **bad_wr);
+int pvrdma_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
+		     struct ib_recv_wr **bad_wr);
+
+#endif /* __PVRDMA_VERBS_H__ */
diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c
index 6d9904a4a0ab..7aa7a4e312f1 100644
--- a/drivers/infiniband/sw/rdmavt/cq.c
+++ b/drivers/infiniband/sw/rdmavt/cq.c
@@ -119,18 +119,17 @@ void rvt_cq_enter(struct rvt_cq *cq, struct ib_wc *entry, bool solicited)
 	if (cq->notify == IB_CQ_NEXT_COMP ||
 	    (cq->notify == IB_CQ_SOLICITED &&
 	     (solicited || entry->status != IB_WC_SUCCESS))) {
-		struct kthread_worker *worker;
 		/*
 		 * This will cause send_complete() to be called in
 		 * another thread.
 		 */
-		smp_read_barrier_depends(); /* see rvt_cq_exit */
-		worker = cq->rdi->worker;
-		if (likely(worker)) {
+		spin_lock(&cq->rdi->n_cqs_lock);
+		if (likely(cq->rdi->worker)) {
 			cq->notify = RVT_CQ_NONE;
 			cq->triggered++;
-			kthread_queue_work(worker, &cq->comptask);
+			kthread_queue_work(cq->rdi->worker, &cq->comptask);
 		}
+		spin_unlock(&cq->rdi->n_cqs_lock);
 	}
 
 	spin_unlock_irqrestore(&cq->lock, flags);
@@ -240,15 +239,15 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
 		}
 	}
 
-	spin_lock(&rdi->n_cqs_lock);
+	spin_lock_irq(&rdi->n_cqs_lock);
 	if (rdi->n_cqs_allocated == rdi->dparms.props.max_cq) {
-		spin_unlock(&rdi->n_cqs_lock);
+		spin_unlock_irq(&rdi->n_cqs_lock);
 		ret = ERR_PTR(-ENOMEM);
 		goto bail_ip;
 	}
 
 	rdi->n_cqs_allocated++;
-	spin_unlock(&rdi->n_cqs_lock);
+	spin_unlock_irq(&rdi->n_cqs_lock);
 
 	if (cq->ip) {
 		spin_lock_irq(&rdi->pending_lock);
@@ -296,9 +295,9 @@ int rvt_destroy_cq(struct ib_cq *ibcq)
 	struct rvt_dev_info *rdi = cq->rdi;
 
 	kthread_flush_work(&cq->comptask);
-	spin_lock(&rdi->n_cqs_lock);
+	spin_lock_irq(&rdi->n_cqs_lock);
 	rdi->n_cqs_allocated--;
-	spin_unlock(&rdi->n_cqs_lock);
+	spin_unlock_irq(&rdi->n_cqs_lock);
 	if (cq->ip)
 		kref_put(&cq->ip->ref, rvt_release_mmap_info);
 	else
@@ -504,33 +503,23 @@ int rvt_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
  */
 int rvt_driver_cq_init(struct rvt_dev_info *rdi)
 {
-	int ret = 0;
 	int cpu;
-	struct task_struct *task;
+	struct kthread_worker *worker;
 
 	if (rdi->worker)
 		return 0;
+
 	spin_lock_init(&rdi->n_cqs_lock);
-	rdi->worker = kzalloc(sizeof(*rdi->worker), GFP_KERNEL);
-	if (!rdi->worker)
-		return -ENOMEM;
-	kthread_init_worker(rdi->worker);
-	task = kthread_create_on_node(
-		kthread_worker_fn,
-		rdi->worker,
-		rdi->dparms.node,
-		"%s", rdi->dparms.cq_name);
-	if (IS_ERR(task)) {
-		kfree(rdi->worker);
-		rdi->worker = NULL;
-		return PTR_ERR(task);
-	}
 
-	set_user_nice(task, MIN_NICE);
 	cpu = cpumask_first(cpumask_of_node(rdi->dparms.node));
-	kthread_bind(task, cpu);
-	wake_up_process(task);
-	return ret;
+	worker = kthread_create_worker_on_cpu(cpu, 0,
+					      "%s", rdi->dparms.cq_name);
+	if (IS_ERR(worker))
+		return PTR_ERR(worker);
+
+	set_user_nice(worker->task, MIN_NICE);
+	rdi->worker = worker;
+	return 0;
 }
 
 /**
@@ -541,13 +530,15 @@ void rvt_cq_exit(struct rvt_dev_info *rdi)
 {
 	struct kthread_worker *worker;
 
+	/* block future queuing from send_complete() */
+	spin_lock_irq(&rdi->n_cqs_lock);
 	worker = rdi->worker;
-	if (!worker)
+	if (!worker) {
+		spin_unlock_irq(&rdi->n_cqs_lock);
 		return;
-	/* blocks future queuing from send_complete() */
+	}
 	rdi->worker = NULL;
-	smp_wmb(); /* See rdi_cq_enter */
-	kthread_flush_worker(worker);
-	kthread_stop(worker->task);
-	kfree(worker);
+	spin_unlock_irq(&rdi->n_cqs_lock);
+
+	kthread_destroy_worker(worker);
 }
diff --git a/drivers/infiniband/sw/rdmavt/mcast.c b/drivers/infiniband/sw/rdmavt/mcast.c
index 983d319ac976..05c8c2afb0e3 100644
--- a/drivers/infiniband/sw/rdmavt/mcast.c
+++ b/drivers/infiniband/sw/rdmavt/mcast.c
@@ -81,7 +81,7 @@ static struct rvt_mcast_qp *rvt_mcast_qp_alloc(struct rvt_qp *qp)
 		goto bail;
 
 	mqp->qp = qp;
-	atomic_inc(&qp->refcount);
+	rvt_get_qp(qp);
 
 bail:
 	return mqp;
@@ -92,8 +92,7 @@ static void rvt_mcast_qp_free(struct rvt_mcast_qp *mqp)
 	struct rvt_qp *qp = mqp->qp;
 
 	/* Notify hfi1_destroy_qp() if it is waiting. */
-	if (atomic_dec_and_test(&qp->refcount))
-		wake_up(&qp->wait);
+	rvt_put_qp(qp);
 
 	kfree(mqp);
 }
diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c
index 46b64970058e..52fd15276ee6 100644
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -51,6 +51,7 @@
 #include <rdma/rdma_vt.h>
 #include "vt.h"
 #include "mr.h"
+#include "trace.h"
 
 /**
  * rvt_driver_mr_init - Init MR resources per driver
@@ -84,6 +85,7 @@ int rvt_driver_mr_init(struct rvt_dev_info *rdi)
 		lkey_table_size = rdi->dparms.lkey_table_size;
 	}
 	rdi->lkey_table.max = 1 << lkey_table_size;
+	rdi->lkey_table.shift = 32 - lkey_table_size;
 	lk_tab_size = rdi->lkey_table.max * sizeof(*rdi->lkey_table.table);
 	rdi->lkey_table.table = (struct rvt_mregion __rcu **)
 			       vmalloc_node(lk_tab_size, rdi->dparms.node);
@@ -402,6 +404,7 @@ struct ib_mr *rvt_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 		}
 		mr->mr.map[m]->segs[n].vaddr = vaddr;
 		mr->mr.map[m]->segs[n].length = umem->page_size;
+		trace_rvt_mr_user_seg(&mr->mr, m, n, vaddr, umem->page_size);
 		n++;
 		if (n == RVT_SEGSZ) {
 			m++;
@@ -506,6 +509,7 @@ static int rvt_set_page(struct ib_mr *ibmr, u64 addr)
 	n = mapped_segs % RVT_SEGSZ;
 	mr->mr.map[m]->segs[n].vaddr = (void *)addr;
 	mr->mr.map[m]->segs[n].length = ps;
+	trace_rvt_mr_page_seg(&mr->mr, m, n, (void *)addr, ps);
 	mr->mr.length += ps;
 
 	return 0;
@@ -692,6 +696,7 @@ int rvt_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
 	for (i = 0; i < list_len; i++) {
 		fmr->mr.map[m]->segs[n].vaddr = (void *)page_list[i];
 		fmr->mr.map[m]->segs[n].length = ps;
+		trace_rvt_mr_fmr_seg(&fmr->mr, m, n, (void *)page_list[i], ps);
 		if (++n == RVT_SEGSZ) {
 			m++;
 			n = 0;
@@ -774,7 +779,6 @@ int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
 	struct rvt_mregion *mr;
 	unsigned n, m;
 	size_t off;
-	struct rvt_dev_info *dev = ib_to_rvt(pd->ibpd.device);
 
 	/*
 	 * We use LKEY == zero for kernel virtual addresses
@@ -782,12 +786,14 @@ int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
 	 */
 	rcu_read_lock();
 	if (sge->lkey == 0) {
+		struct rvt_dev_info *dev = ib_to_rvt(pd->ibpd.device);
+
 		if (pd->user)
 			goto bail;
 		mr = rcu_dereference(dev->dma_mr);
 		if (!mr)
 			goto bail;
-		atomic_inc(&mr->refcount);
+		rvt_get_mr(mr);
 		rcu_read_unlock();
 
 		isge->mr = mr;
@@ -798,8 +804,7 @@ int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
 		isge->n = 0;
 		goto ok;
 	}
-	mr = rcu_dereference(
-		rkt->table[(sge->lkey >> (32 - dev->dparms.lkey_table_size))]);
+	mr = rcu_dereference(rkt->table[sge->lkey >> rkt->shift]);
 	if (unlikely(!mr || atomic_read(&mr->lkey_invalid) ||
 		     mr->lkey != sge->lkey || mr->pd != &pd->ibpd))
 		goto bail;
@@ -809,7 +814,7 @@ int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
 		     off + sge->length > mr->length ||
 		     (mr->access_flags & acc) != acc))
 		goto bail;
-	atomic_inc(&mr->refcount);
+	rvt_get_mr(mr);
 	rcu_read_unlock();
 
 	off += mr->offset;
@@ -887,7 +892,7 @@ int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
 		mr = rcu_dereference(rdi->dma_mr);
 		if (!mr)
 			goto bail;
-		atomic_inc(&mr->refcount);
+		rvt_get_mr(mr);
 		rcu_read_unlock();
 
 		sge->mr = mr;
@@ -899,8 +904,7 @@ int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
 		goto ok;
 	}
 
-	mr = rcu_dereference(
-		rkt->table[(rkey >> (32 - dev->dparms.lkey_table_size))]);
+	mr = rcu_dereference(rkt->table[rkey >> rkt->shift]);
 	if (unlikely(!mr || atomic_read(&mr->lkey_invalid) ||
 		     mr->lkey != rkey || qp->ibqp.pd != mr->pd))
 		goto bail;
@@ -909,7 +913,7 @@ int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
 	if (unlikely(vaddr < mr->iova || off + len > mr->length ||
 		     (mr->access_flags & acc) == 0))
 		goto bail;
-	atomic_inc(&mr->refcount);
+	rvt_get_mr(mr);
 	rcu_read_unlock();
 
 	off += mr->offset;
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 6500c3b5a89c..2a13ac660f2b 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -76,6 +76,23 @@ const int ib_rvt_state_ops[IB_QPS_ERR + 1] = {
 };
 EXPORT_SYMBOL(ib_rvt_state_ops);
 
+/*
+ * Translate ib_wr_opcode into ib_wc_opcode.
+ */
+const enum ib_wc_opcode ib_rvt_wc_opcode[] = {
+	[IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE,
+	[IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE,
+	[IB_WR_SEND] = IB_WC_SEND,
+	[IB_WR_SEND_WITH_IMM] = IB_WC_SEND,
+	[IB_WR_RDMA_READ] = IB_WC_RDMA_READ,
+	[IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP,
+	[IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD,
+	[IB_WR_SEND_WITH_INV] = IB_WC_SEND,
+	[IB_WR_LOCAL_INV] = IB_WC_LOCAL_INV,
+	[IB_WR_REG_MR] = IB_WC_REG_MR
+};
+EXPORT_SYMBOL(ib_rvt_wc_opcode);
+
 static void get_map_page(struct rvt_qpn_table *qpt,
 			 struct rvt_qpn_map *map,
 			 gfp_t gfp)
@@ -884,7 +901,8 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 	return ret;
 
 bail_ip:
-	kref_put(&qp->ip->ref, rvt_release_mmap_info);
+	if (qp->ip)
+		kref_put(&qp->ip->ref, rvt_release_mmap_info);
 
 bail_qpn:
 	free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num);
diff --git a/drivers/infiniband/sw/rdmavt/trace.h b/drivers/infiniband/sw/rdmavt/trace.h
index 6c0457db5499..e2d23acb6a7d 100644
--- a/drivers/infiniband/sw/rdmavt/trace.h
+++ b/drivers/infiniband/sw/rdmavt/trace.h
@@ -45,143 +45,10 @@
  *
  */
 
-#undef TRACE_SYSTEM_VAR
-#define TRACE_SYSTEM_VAR rdmavt
-
-#if !defined(__RDMAVT_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
-#define __RDMAVT_TRACE_H
-
-#include <linux/tracepoint.h>
-#include <linux/trace_seq.h>
-
-#include <rdma/ib_verbs.h>
-#include <rdma/rdma_vt.h>
-
 #define RDI_DEV_ENTRY(rdi)   __string(dev, rdi->driver_f.get_card_name(rdi))
 #define RDI_DEV_ASSIGN(rdi)  __assign_str(dev, rdi->driver_f.get_card_name(rdi))
 
-#undef TRACE_SYSTEM
-#define TRACE_SYSTEM rdmavt
-
-TRACE_EVENT(rvt_dbg,
-	TP_PROTO(struct rvt_dev_info *rdi,
-		 const char *msg),
-	TP_ARGS(rdi, msg),
-	TP_STRUCT__entry(
-		RDI_DEV_ENTRY(rdi)
-		__string(msg, msg)
-	),
-	TP_fast_assign(
-		RDI_DEV_ASSIGN(rdi);
-		__assign_str(msg, msg);
-	),
-	TP_printk("[%s]: %s", __get_str(dev), __get_str(msg))
-);
-
-#undef TRACE_SYSTEM
-#define TRACE_SYSTEM rvt_qphash
-DECLARE_EVENT_CLASS(rvt_qphash_template,
-	TP_PROTO(struct rvt_qp *qp, u32 bucket),
-	TP_ARGS(qp, bucket),
-	TP_STRUCT__entry(
-		RDI_DEV_ENTRY(ib_to_rvt(qp->ibqp.device))
-		__field(u32, qpn)
-		__field(u32, bucket)
-	),
-	TP_fast_assign(
-		RDI_DEV_ASSIGN(ib_to_rvt(qp->ibqp.device))
-		__entry->qpn = qp->ibqp.qp_num;
-		__entry->bucket = bucket;
-	),
-	TP_printk(
-		"[%s] qpn 0x%x bucket %u",
-		__get_str(dev),
-		__entry->qpn,
-		__entry->bucket
-	)
-);
-
-DEFINE_EVENT(rvt_qphash_template, rvt_qpinsert,
-	TP_PROTO(struct rvt_qp *qp, u32 bucket),
-	TP_ARGS(qp, bucket));
-
-DEFINE_EVENT(rvt_qphash_template, rvt_qpremove,
-	TP_PROTO(struct rvt_qp *qp, u32 bucket),
-	TP_ARGS(qp, bucket));
-
-#undef TRACE_SYSTEM
-#define TRACE_SYSTEM rvt_tx
-
-#define wr_opcode_name(opcode) { IB_WR_##opcode, #opcode  }
-#define show_wr_opcode(opcode)                             \
-__print_symbolic(opcode,                                   \
-	wr_opcode_name(RDMA_WRITE),                        \
-	wr_opcode_name(RDMA_WRITE_WITH_IMM),               \
-	wr_opcode_name(SEND),                              \
-	wr_opcode_name(SEND_WITH_IMM),                     \
-	wr_opcode_name(RDMA_READ),                         \
-	wr_opcode_name(ATOMIC_CMP_AND_SWP),                \
-	wr_opcode_name(ATOMIC_FETCH_AND_ADD),              \
-	wr_opcode_name(LSO),                               \
-	wr_opcode_name(SEND_WITH_INV),                     \
-	wr_opcode_name(RDMA_READ_WITH_INV),                \
-	wr_opcode_name(LOCAL_INV),                         \
-	wr_opcode_name(MASKED_ATOMIC_CMP_AND_SWP),         \
-	wr_opcode_name(MASKED_ATOMIC_FETCH_AND_ADD))
-
-#define POS_PRN \
-"[%s] wr_id %llx qpn %x psn 0x%x lpsn 0x%x length %u opcode 0x%.2x,%s size %u avail %u head %u last %u"
-
-TRACE_EVENT(
-	rvt_post_one_wr,
-	TP_PROTO(struct rvt_qp *qp, struct rvt_swqe *wqe),
-	TP_ARGS(qp, wqe),
-	TP_STRUCT__entry(
-		RDI_DEV_ENTRY(ib_to_rvt(qp->ibqp.device))
-		__field(u64, wr_id)
-		__field(u32, qpn)
-		__field(u32, psn)
-		__field(u32, lpsn)
-		__field(u32, length)
-		__field(u32, opcode)
-		__field(u32, size)
-		__field(u32, avail)
-		__field(u32, head)
-		__field(u32, last)
-	),
-	TP_fast_assign(
-		RDI_DEV_ASSIGN(ib_to_rvt(qp->ibqp.device))
-		__entry->wr_id = wqe->wr.wr_id;
-		__entry->qpn = qp->ibqp.qp_num;
-		__entry->psn = wqe->psn;
-		__entry->lpsn = wqe->lpsn;
-		__entry->length = wqe->length;
-		__entry->opcode = wqe->wr.opcode;
-		__entry->size = qp->s_size;
-		__entry->avail = qp->s_avail;
-		__entry->head = qp->s_head;
-		__entry->last = qp->s_last;
-	),
-	TP_printk(
-		POS_PRN,
-		__get_str(dev),
-		__entry->wr_id,
-		__entry->qpn,
-		__entry->psn,
-		__entry->lpsn,
-		__entry->length,
-		__entry->opcode, show_wr_opcode(__entry->opcode),
-		__entry->size,
-		__entry->avail,
-		__entry->head,
-		__entry->last
-	)
-);
-
-#endif /* __RDMAVT_TRACE_H */
-
-#undef TRACE_INCLUDE_PATH
-#undef TRACE_INCLUDE_FILE
-#define TRACE_INCLUDE_PATH .
-#define TRACE_INCLUDE_FILE trace
-#include <trace/define_trace.h>
+#include "trace_rvt.h"
+#include "trace_qp.h"
+#include "trace_tx.h"
+#include "trace_mr.h"
diff --git a/drivers/infiniband/sw/rdmavt/trace_mr.h b/drivers/infiniband/sw/rdmavt/trace_mr.h
new file mode 100644
index 000000000000..3318a6c36373
--- /dev/null
+++ b/drivers/infiniband/sw/rdmavt/trace_mr.h
@@ -0,0 +1,112 @@
+/*
+ * Copyright(c) 2016 Intel Corporation.
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * BSD LICENSE
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  - Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  - Neither the name of Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#if !defined(__RVT_TRACE_MR_H) || defined(TRACE_HEADER_MULTI_READ)
+#define __RVT_TRACE_MR_H
+
+#include <linux/tracepoint.h>
+#include <linux/trace_seq.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/rdma_vt.h>
+#include <rdma/rdmavt_mr.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM rvt_mr
+DECLARE_EVENT_CLASS(
+	rvt_mr_template,
+	TP_PROTO(struct rvt_mregion *mr, u16 m, u16 n, void *v, size_t len),
+	TP_ARGS(mr, m, n, v, len),
+	TP_STRUCT__entry(
+		RDI_DEV_ENTRY(ib_to_rvt(mr->pd->device))
+		__field(void *, vaddr)
+		__field(struct page *, page)
+		__field(size_t, len)
+		__field(u32, lkey)
+		__field(u16, m)
+		__field(u16, n)
+	),
+	TP_fast_assign(
+		RDI_DEV_ASSIGN(ib_to_rvt(mr->pd->device));
+		__entry->vaddr = v;
+		__entry->page = virt_to_page(v);
+		__entry->m = m;
+		__entry->n = n;
+		__entry->len = len;
+	),
+	TP_printk(
+		"[%s] vaddr %p page %p m %u n %u len %ld",
+		__get_str(dev),
+		__entry->vaddr,
+		__entry->page,
+		__entry->m,
+		__entry->n,
+		__entry->len
+	)
+);
+
+DEFINE_EVENT(
+	rvt_mr_template, rvt_mr_page_seg,
+	TP_PROTO(struct rvt_mregion *mr, u16 m, u16 n, void *v, size_t len),
+	TP_ARGS(mr, m, n, v, len));
+
+DEFINE_EVENT(
+	rvt_mr_template, rvt_mr_fmr_seg,
+	TP_PROTO(struct rvt_mregion *mr, u16 m, u16 n, void *v, size_t len),
+	TP_ARGS(mr, m, n, v, len));
+
+DEFINE_EVENT(
+	rvt_mr_template, rvt_mr_user_seg,
+	TP_PROTO(struct rvt_mregion *mr, u16 m, u16 n, void *v, size_t len),
+	TP_ARGS(mr, m, n, v, len));
+
+#endif /* __RVT_TRACE_MR_H */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE trace_mr
+#include <trace/define_trace.h>
diff --git a/drivers/infiniband/sw/rdmavt/trace_qp.h b/drivers/infiniband/sw/rdmavt/trace_qp.h
new file mode 100644
index 000000000000..4c77a3119bda
--- /dev/null
+++ b/drivers/infiniband/sw/rdmavt/trace_qp.h
@@ -0,0 +1,96 @@
+/*
+ * Copyright(c) 2016 Intel Corporation.
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * BSD LICENSE
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  - Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  - Neither the name of Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#if !defined(__RVT_TRACE_QP_H) || defined(TRACE_HEADER_MULTI_READ)
+#define __RVT_TRACE_QP_H
+
+#include <linux/tracepoint.h>
+#include <linux/trace_seq.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/rdma_vt.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM rvt_qp
+
+DECLARE_EVENT_CLASS(rvt_qphash_template,
+	TP_PROTO(struct rvt_qp *qp, u32 bucket),
+	TP_ARGS(qp, bucket),
+	TP_STRUCT__entry(
+		RDI_DEV_ENTRY(ib_to_rvt(qp->ibqp.device))
+		__field(u32, qpn)
+		__field(u32, bucket)
+	),
+	TP_fast_assign(
+		RDI_DEV_ASSIGN(ib_to_rvt(qp->ibqp.device))
+		__entry->qpn = qp->ibqp.qp_num;
+		__entry->bucket = bucket;
+	),
+	TP_printk(
+		"[%s] qpn 0x%x bucket %u",
+		__get_str(dev),
+		__entry->qpn,
+		__entry->bucket
+	)
+);
+
+DEFINE_EVENT(rvt_qphash_template, rvt_qpinsert,
+	TP_PROTO(struct rvt_qp *qp, u32 bucket),
+	TP_ARGS(qp, bucket));
+
+DEFINE_EVENT(rvt_qphash_template, rvt_qpremove,
+	TP_PROTO(struct rvt_qp *qp, u32 bucket),
+	TP_ARGS(qp, bucket));
+
+
+#endif /* __RVT_TRACE_QP_H */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE trace_qp
+#include <trace/define_trace.h>
+
diff --git a/drivers/infiniband/sw/rdmavt/trace_rvt.h b/drivers/infiniband/sw/rdmavt/trace_rvt.h
new file mode 100644
index 000000000000..746f33461d9a
--- /dev/null
+++ b/drivers/infiniband/sw/rdmavt/trace_rvt.h
@@ -0,0 +1,81 @@
+/*
+ * Copyright(c) 2016 Intel Corporation.
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * BSD LICENSE
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  - Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  - Neither the name of Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#if !defined(__RVT_TRACE_RVT_H) || defined(TRACE_HEADER_MULTI_READ)
+#define __RVT_TRACE_RVT_H
+
+#include <linux/tracepoint.h>
+#include <linux/trace_seq.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/rdma_vt.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM rvt
+
+TRACE_EVENT(rvt_dbg,
+	TP_PROTO(struct rvt_dev_info *rdi,
+		 const char *msg),
+	TP_ARGS(rdi, msg),
+	TP_STRUCT__entry(
+		RDI_DEV_ENTRY(rdi)
+		__string(msg, msg)
+	),
+	TP_fast_assign(
+		RDI_DEV_ASSIGN(rdi);
+		__assign_str(msg, msg);
+	),
+	TP_printk("[%s]: %s", __get_str(dev), __get_str(msg))
+);
+
+#endif /* __RVT_TRACE_MISC_H */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE trace_rvt
+#include <trace/define_trace.h>
+
diff --git a/drivers/infiniband/sw/rdmavt/trace_tx.h b/drivers/infiniband/sw/rdmavt/trace_tx.h
new file mode 100644
index 000000000000..0e03173662d8
--- /dev/null
+++ b/drivers/infiniband/sw/rdmavt/trace_tx.h
@@ -0,0 +1,132 @@
+/*
+ * Copyright(c) 2016 Intel Corporation.
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * BSD LICENSE
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  - Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  - Neither the name of Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#if !defined(__RVT_TRACE_TX_H) || defined(TRACE_HEADER_MULTI_READ)
+#define __RVT_TRACE_TX_H
+
+#include <linux/tracepoint.h>
+#include <linux/trace_seq.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/rdma_vt.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM rvt_tx
+
+#define wr_opcode_name(opcode) { IB_WR_##opcode, #opcode  }
+#define show_wr_opcode(opcode)                             \
+__print_symbolic(opcode,                                   \
+	wr_opcode_name(RDMA_WRITE),                        \
+	wr_opcode_name(RDMA_WRITE_WITH_IMM),               \
+	wr_opcode_name(SEND),                              \
+	wr_opcode_name(SEND_WITH_IMM),                     \
+	wr_opcode_name(RDMA_READ),                         \
+	wr_opcode_name(ATOMIC_CMP_AND_SWP),                \
+	wr_opcode_name(ATOMIC_FETCH_AND_ADD),              \
+	wr_opcode_name(LSO),                               \
+	wr_opcode_name(SEND_WITH_INV),                     \
+	wr_opcode_name(RDMA_READ_WITH_INV),                \
+	wr_opcode_name(LOCAL_INV),                         \
+	wr_opcode_name(MASKED_ATOMIC_CMP_AND_SWP),         \
+	wr_opcode_name(MASKED_ATOMIC_FETCH_AND_ADD))
+
+#define POS_PRN \
+"[%s] wr_id %llx qpn %x psn 0x%x lpsn 0x%x length %u opcode 0x%.2x,%s size %u avail %u head %u last %u"
+
+TRACE_EVENT(
+	rvt_post_one_wr,
+	TP_PROTO(struct rvt_qp *qp, struct rvt_swqe *wqe),
+	TP_ARGS(qp, wqe),
+	TP_STRUCT__entry(
+		RDI_DEV_ENTRY(ib_to_rvt(qp->ibqp.device))
+		__field(u64, wr_id)
+		__field(u32, qpn)
+		__field(u32, psn)
+		__field(u32, lpsn)
+		__field(u32, length)
+		__field(u32, opcode)
+		__field(u32, size)
+		__field(u32, avail)
+		__field(u32, head)
+		__field(u32, last)
+	),
+	TP_fast_assign(
+		RDI_DEV_ASSIGN(ib_to_rvt(qp->ibqp.device))
+		__entry->wr_id = wqe->wr.wr_id;
+		__entry->qpn = qp->ibqp.qp_num;
+		__entry->psn = wqe->psn;
+		__entry->lpsn = wqe->lpsn;
+		__entry->length = wqe->length;
+		__entry->opcode = wqe->wr.opcode;
+		__entry->size = qp->s_size;
+		__entry->avail = qp->s_avail;
+		__entry->head = qp->s_head;
+		__entry->last = qp->s_last;
+	),
+	TP_printk(
+		POS_PRN,
+		__get_str(dev),
+		__entry->wr_id,
+		__entry->qpn,
+		__entry->psn,
+		__entry->lpsn,
+		__entry->length,
+		__entry->opcode, show_wr_opcode(__entry->opcode),
+		__entry->size,
+		__entry->avail,
+		__entry->head,
+		__entry->last
+	)
+);
+
+#endif /* __RVT_TRACE_TX_H */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE trace_tx
+#include <trace/define_trace.h>
+
diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c
index 6c5e29db88e3..cd27cbde7652 100644
--- a/drivers/infiniband/sw/rxe/rxe_comp.c
+++ b/drivers/infiniband/sw/rxe/rxe_comp.c
@@ -420,11 +420,12 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
 	    (wqe->wr.send_flags & IB_SEND_SIGNALED) ||
 	    (qp->req.state == QP_STATE_ERROR)) {
 		make_send_cqe(qp, wqe, &cqe);
+		advance_consumer(qp->sq.queue);
 		rxe_cq_post(qp->scq, &cqe, 0);
+	} else {
+		advance_consumer(qp->sq.queue);
 	}
 
-	advance_consumer(qp->sq.queue);
-
 	/*
 	 * we completed something so let req run again
 	 * if it is trying to fence
@@ -510,6 +511,8 @@ int rxe_completer(void *arg)
 	struct rxe_pkt_info *pkt = NULL;
 	enum comp_state state;
 
+	rxe_add_ref(qp);
+
 	if (!qp->valid) {
 		while ((skb = skb_dequeue(&qp->resp_pkts))) {
 			rxe_drop_ref(qp);
@@ -739,11 +742,13 @@ exit:
 	/* we come here if we are done with processing and want the task to
 	 * exit from the loop calling us
 	 */
+	rxe_drop_ref(qp);
 	return -EAGAIN;
 
 done:
 	/* we come here if we have processed a packet we want the task to call
 	 * us again to see if there is anything else to do
 	 */
+	rxe_drop_ref(qp);
 	return 0;
 }
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index 73849a5a91b3..efe4c6a35442 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -266,8 +266,6 @@ static inline int rxe_xmit_packet(struct rxe_dev *rxe, struct rxe_qp *qp,
 		return err;
 	}
 
-	atomic_inc(&qp->skb_out);
-
 	if ((qp_type(qp) != IB_QPT_RC) &&
 	    (pkt->mask & RXE_END_MASK)) {
 		pkt->wqe->state = wqe_state_done;
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index 1869152f1d23..d0faca294006 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -355,6 +355,9 @@ int rxe_mem_copy(struct rxe_mem *mem, u64 iova, void *addr, int length,
 	size_t			offset;
 	u32			crc = crcp ? (*crcp) : 0;
 
+	if (length == 0)
+		return 0;
+
 	if (mem->type == RXE_MEM_TYPE_DMA) {
 		u8 *src, *dest;
 
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index ffff5a54cb34..16967cdb45df 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -46,7 +46,7 @@
 #include "rxe_loc.h"
 
 static LIST_HEAD(rxe_dev_list);
-static spinlock_t dev_list_lock; /* spinlock for device list */
+static DEFINE_SPINLOCK(dev_list_lock); /* spinlock for device list */
 
 struct rxe_dev *net_to_rxe(struct net_device *ndev)
 {
@@ -455,6 +455,8 @@ static int send(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
 		return -EAGAIN;
 	}
 
+	if (pkt->qp)
+		atomic_inc(&pkt->qp->skb_out);
 	kfree_skb(skb);
 
 	return 0;
@@ -659,8 +661,6 @@ struct notifier_block rxe_net_notifier = {
 
 int rxe_net_ipv4_init(void)
 {
-	spin_lock_init(&dev_list_lock);
-
 	recv_sockets.sk4 = rxe_setup_udp_tunnel(&init_net,
 				htons(ROCE_V2_UDP_DPORT), false);
 	if (IS_ERR(recv_sockets.sk4)) {
@@ -676,8 +676,6 @@ int rxe_net_ipv6_init(void)
 {
 #if IS_ENABLED(CONFIG_IPV6)
 
-	spin_lock_init(&dev_list_lock);
-
 	recv_sockets.sk6 = rxe_setup_udp_tunnel(&init_net,
 						htons(ROCE_V2_UDP_DPORT), true);
 	if (IS_ERR(recv_sockets.sk6)) {
diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
index f459c43a77c8..13ed2cc6eaa2 100644
--- a/drivers/infiniband/sw/rxe/rxe_param.h
+++ b/drivers/infiniband/sw/rxe/rxe_param.h
@@ -82,7 +82,7 @@ enum rxe_device_param {
 	RXE_MAX_SGE			= 32,
 	RXE_MAX_SGE_RD			= 32,
 	RXE_MAX_CQ			= 16384,
-	RXE_MAX_LOG_CQE			= 13,
+	RXE_MAX_LOG_CQE			= 15,
 	RXE_MAX_MR			= 2 * 1024,
 	RXE_MAX_PD			= 0x7ffc,
 	RXE_MAX_QP_RD_ATOM		= 128,
diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c
index 6bac0717c540..d723947a8542 100644
--- a/drivers/infiniband/sw/rxe/rxe_pool.c
+++ b/drivers/infiniband/sw/rxe/rxe_pool.c
@@ -180,7 +180,6 @@ static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
 	size = BITS_TO_LONGS(max - min + 1) * sizeof(long);
 	pool->table = kmalloc(size, GFP_KERNEL);
 	if (!pool->table) {
-		pr_warn("no memory for bit table\n");
 		err = -ENOMEM;
 		goto out;
 	}
diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c
index 46f062842a9a..252b4d637d45 100644
--- a/drivers/infiniband/sw/rxe/rxe_recv.c
+++ b/drivers/infiniband/sw/rxe/rxe_recv.c
@@ -391,16 +391,15 @@ int rxe_rcv(struct sk_buff *skb)
 			     payload_size(pkt));
 	calc_icrc = cpu_to_be32(~calc_icrc);
 	if (unlikely(calc_icrc != pack_icrc)) {
-		char saddr[sizeof(struct in6_addr)];
-
 		if (skb->protocol == htons(ETH_P_IPV6))
-			sprintf(saddr, "%pI6", &ipv6_hdr(skb)->saddr);
+			pr_warn_ratelimited("bad ICRC from %pI6c\n",
+					    &ipv6_hdr(skb)->saddr);
 		else if (skb->protocol == htons(ETH_P_IP))
-			sprintf(saddr, "%pI4", &ip_hdr(skb)->saddr);
+			pr_warn_ratelimited("bad ICRC from %pI4\n",
+					    &ip_hdr(skb)->saddr);
 		else
-			sprintf(saddr, "unknown");
+			pr_warn_ratelimited("bad ICRC from unknown\n");
 
-		pr_warn_ratelimited("bad ICRC from %s\n", saddr);
 		goto drop;
 	}
 
diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
index 22bd9630dcd9..73d4a97603a1 100644
--- a/drivers/infiniband/sw/rxe/rxe_req.c
+++ b/drivers/infiniband/sw/rxe/rxe_req.c
@@ -548,23 +548,23 @@ static void update_wqe_psn(struct rxe_qp *qp,
 static void save_state(struct rxe_send_wqe *wqe,
 		       struct rxe_qp *qp,
 		       struct rxe_send_wqe *rollback_wqe,
-		       struct rxe_qp *rollback_qp)
+		       u32 *rollback_psn)
 {
 	rollback_wqe->state     = wqe->state;
 	rollback_wqe->first_psn = wqe->first_psn;
 	rollback_wqe->last_psn  = wqe->last_psn;
-	rollback_qp->req.psn    = qp->req.psn;
+	*rollback_psn		= qp->req.psn;
 }
 
 static void rollback_state(struct rxe_send_wqe *wqe,
 			   struct rxe_qp *qp,
 			   struct rxe_send_wqe *rollback_wqe,
-			   struct rxe_qp *rollback_qp)
+			   u32 rollback_psn)
 {
 	wqe->state     = rollback_wqe->state;
 	wqe->first_psn = rollback_wqe->first_psn;
 	wqe->last_psn  = rollback_wqe->last_psn;
-	qp->req.psn    = rollback_qp->req.psn;
+	qp->req.psn    = rollback_psn;
 }
 
 static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
@@ -593,8 +593,10 @@ int rxe_requester(void *arg)
 	int mtu;
 	int opcode;
 	int ret;
-	struct rxe_qp rollback_qp;
 	struct rxe_send_wqe rollback_wqe;
+	u32 rollback_psn;
+
+	rxe_add_ref(qp);
 
 next_wqe:
 	if (unlikely(!qp->valid || qp->req.state == QP_STATE_ERROR))
@@ -697,6 +699,7 @@ next_wqe:
 			wqe->state = wqe_state_done;
 			wqe->status = IB_WC_SUCCESS;
 			__rxe_do_task(&qp->comp.task);
+			rxe_drop_ref(qp);
 			return 0;
 		}
 		payload = mtu;
@@ -719,7 +722,7 @@ next_wqe:
 	 * rxe_xmit_packet().
 	 * Otherwise, completer might initiate an unjustified retry flow.
 	 */
-	save_state(wqe, qp, &rollback_wqe, &rollback_qp);
+	save_state(wqe, qp, &rollback_wqe, &rollback_psn);
 	update_wqe_state(qp, wqe, &pkt);
 	update_wqe_psn(qp, wqe, &pkt, payload);
 	ret = rxe_xmit_packet(to_rdev(qp->ibqp.device), qp, &pkt, skb);
@@ -727,7 +730,7 @@ next_wqe:
 		qp->need_req_skb = 1;
 		kfree_skb(skb);
 
-		rollback_state(wqe, qp, &rollback_wqe, &rollback_qp);
+		rollback_state(wqe, qp, &rollback_wqe, rollback_psn);
 
 		if (ret == -EAGAIN) {
 			rxe_run_task(&qp->req.task, 1);
@@ -756,8 +759,7 @@ err:
 	 */
 	wqe->wr.send_flags |= IB_SEND_SIGNALED;
 	__rxe_do_task(&qp->comp.task);
-	return -EAGAIN;
-
 exit:
+	rxe_drop_ref(qp);
 	return -EAGAIN;
 }
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index dd3d88adc003..7a36ec9dbc0c 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -444,6 +444,13 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
 		return RESPST_EXECUTE;
 	}
 
+	/* A zero-byte op is not required to set an addr or rkey. */
+	if ((pkt->mask & (RXE_READ_MASK | RXE_WRITE_OR_SEND)) &&
+	    (pkt->mask & RXE_RETH_MASK) &&
+	    reth_len(pkt) == 0) {
+		return RESPST_EXECUTE;
+	}
+
 	va	= qp->resp.va;
 	rkey	= qp->resp.rkey;
 	resid	= qp->resp.resid;
@@ -680,9 +687,14 @@ static enum resp_states read_reply(struct rxe_qp *qp,
 		res->read.va_org	= qp->resp.va;
 
 		res->first_psn		= req_pkt->psn;
-		res->last_psn		= req_pkt->psn +
-					  (reth_len(req_pkt) + mtu - 1) /
-					  mtu - 1;
+
+		if (reth_len(req_pkt)) {
+			res->last_psn	= (req_pkt->psn +
+					   (reth_len(req_pkt) + mtu - 1) /
+					   mtu - 1) & BTH_PSN_MASK;
+		} else {
+			res->last_psn	= res->first_psn;
+		}
 		res->cur_psn		= req_pkt->psn;
 
 		res->read.resid		= qp->resp.resid;
@@ -742,7 +754,8 @@ static enum resp_states read_reply(struct rxe_qp *qp,
 	} else {
 		qp->resp.res = NULL;
 		qp->resp.opcode = -1;
-		qp->resp.psn = res->cur_psn;
+		if (psn_compare(res->cur_psn, qp->resp.psn) >= 0)
+			qp->resp.psn = res->cur_psn;
 		state = RESPST_CLEANUP;
 	}
 
@@ -1132,6 +1145,7 @@ static enum resp_states duplicate_request(struct rxe_qp *qp,
 					     pkt, skb_copy);
 			if (rc) {
 				pr_err("Failed resending result. This flow is not handled - skb ignored\n");
+				rxe_drop_ref(qp);
 				kfree_skb(skb_copy);
 				rc = RESPST_CLEANUP;
 				goto out;
@@ -1198,6 +1212,8 @@ int rxe_responder(void *arg)
 	struct rxe_pkt_info *pkt = NULL;
 	int ret = 0;
 
+	rxe_add_ref(qp);
+
 	qp->resp.aeth_syndrome = AETH_ACK_UNLIMITED;
 
 	if (!qp->valid) {
@@ -1386,5 +1402,6 @@ int rxe_responder(void *arg)
 exit:
 	ret = -EAGAIN;
 done:
+	rxe_drop_ref(qp);
 	return ret;
 }
diff --git a/drivers/infiniband/sw/rxe/rxe_srq.c b/drivers/infiniband/sw/rxe/rxe_srq.c
index 2a6e3cd2d4e8..efc832a2d7c6 100644
--- a/drivers/infiniband/sw/rxe/rxe_srq.c
+++ b/drivers/infiniband/sw/rxe/rxe_srq.c
@@ -169,7 +169,7 @@ int rxe_srq_from_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
 			}
 		}
 
-		err = rxe_queue_resize(q, (unsigned int *)&attr->max_wr,
+		err = rxe_queue_resize(q, &attr->max_wr,
 				       rcv_wqe_size(srq->rq.max_sge),
 				       srq->rq.queue->ip ?
 						srq->rq.queue->ip->context :
diff --git a/drivers/infiniband/sw/rxe/rxe_task.c b/drivers/infiniband/sw/rxe/rxe_task.c
index 1e19bf828a6e..d2a14a1bdc7f 100644
--- a/drivers/infiniband/sw/rxe/rxe_task.c
+++ b/drivers/infiniband/sw/rxe/rxe_task.c
@@ -121,6 +121,7 @@ int rxe_init_task(void *obj, struct rxe_task *task,
 	task->arg	= arg;
 	task->func	= func;
 	snprintf(task->name, sizeof(task->name), "%s", name);
+	task->destroyed	= false;
 
 	tasklet_init(&task->tasklet, rxe_do_task, (unsigned long)task);
 
@@ -132,11 +133,29 @@ int rxe_init_task(void *obj, struct rxe_task *task,
 
 void rxe_cleanup_task(struct rxe_task *task)
 {
+	unsigned long flags;
+	bool idle;
+
+	/*
+	 * Mark the task, then wait for it to finish. It might be
+	 * running in a non-tasklet (direct call) context.
+	 */
+	task->destroyed = true;
+
+	do {
+		spin_lock_irqsave(&task->state_lock, flags);
+		idle = (task->state == TASK_STATE_START);
+		spin_unlock_irqrestore(&task->state_lock, flags);
+	} while (!idle);
+
 	tasklet_kill(&task->tasklet);
 }
 
 void rxe_run_task(struct rxe_task *task, int sched)
 {
+	if (task->destroyed)
+		return;
+
 	if (sched)
 		tasklet_schedule(&task->tasklet);
 	else
diff --git a/drivers/infiniband/sw/rxe/rxe_task.h b/drivers/infiniband/sw/rxe/rxe_task.h
index d14aa6daed05..08ff42d451c6 100644
--- a/drivers/infiniband/sw/rxe/rxe_task.h
+++ b/drivers/infiniband/sw/rxe/rxe_task.h
@@ -54,6 +54,7 @@ struct rxe_task {
 	int			(*func)(void *arg);
 	int			ret;
 	char			name[16];
+	bool			destroyed;
 };
 
 /*
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 19841c863daf..beb7021ff18a 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -316,7 +316,9 @@ static int rxe_init_av(struct rxe_dev *rxe, struct ib_ah_attr *attr,
 	return err;
 }
 
-static struct ib_ah *rxe_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
+static struct ib_ah *rxe_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
+				   struct ib_udata *udata)
+
 {
 	int err;
 	struct rxe_dev *rxe = to_rdev(ibpd->device);
@@ -564,7 +566,7 @@ static struct ib_qp *rxe_create_qp(struct ib_pd *ibpd,
 	if (udata) {
 		if (udata->inlen) {
 			err = -EINVAL;
-			goto err1;
+			goto err2;
 		}
 		qp->is_user = 1;
 	}
@@ -573,12 +575,13 @@ static struct ib_qp *rxe_create_qp(struct ib_pd *ibpd,
 
 	err = rxe_qp_from_init(rxe, qp, pd, init, udata, ibpd);
 	if (err)
-		goto err2;
+		goto err3;
 
 	return &qp->ibqp;
 
-err2:
+err3:
 	rxe_drop_index(qp);
+err2:
 	rxe_drop_ref(qp);
 err1:
 	return ERR_PTR(err);
@@ -1007,11 +1010,19 @@ static int rxe_peek_cq(struct ib_cq *ibcq, int wc_cnt)
 static int rxe_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 {
 	struct rxe_cq *cq = to_rcq(ibcq);
+	unsigned long irq_flags;
+	int ret = 0;
 
+	spin_lock_irqsave(&cq->cq_lock, irq_flags);
 	if (cq->notify != IB_CQ_NEXT_COMP)
 		cq->notify = flags & IB_CQ_SOLICITED_MASK;
 
-	return 0;
+	if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !queue_empty(cq->queue))
+		ret = 1;
+
+	spin_unlock_irqrestore(&cq->cq_lock, irq_flags);
+
+	return ret;
 }
 
 static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
index 339a1eecdfe3..096c4f6fbd65 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
@@ -357,11 +357,8 @@ static int ipoib_cm_nonsrq_init_rx(struct net_device *dev, struct ib_cm_id *cm_i
 	int i;
 
 	rx->rx_ring = vzalloc(ipoib_recvq_size * sizeof *rx->rx_ring);
-	if (!rx->rx_ring) {
-		printk(KERN_WARNING "%s: failed to allocate CM non-SRQ ring (%d entries)\n",
-		       priv->ca->name, ipoib_recvq_size);
+	if (!rx->rx_ring)
 		return -ENOMEM;
-	}
 
 	t = kmalloc(sizeof *t, GFP_KERNEL);
 	if (!t) {
@@ -1054,8 +1051,6 @@ static struct ib_qp *ipoib_cm_create_tx_qp(struct net_device *dev, struct ipoib_
 
 	tx_qp = ib_create_qp(priv->pd, &attr);
 	if (PTR_ERR(tx_qp) == -EINVAL) {
-		ipoib_warn(priv, "can't use GFP_NOIO for QPs on device %s, using GFP_KERNEL\n",
-			   priv->ca->name);
 		attr.create_flags &= ~IB_QP_CREATE_USE_GFP_NOIO;
 		tx_qp = ib_create_qp(priv->pd, &attr);
 	}
@@ -1134,7 +1129,6 @@ static int ipoib_cm_tx_init(struct ipoib_cm_tx *p, u32 qpn,
 	p->tx_ring = __vmalloc(ipoib_sendq_size * sizeof *p->tx_ring,
 			       GFP_NOIO, PAGE_KERNEL);
 	if (!p->tx_ring) {
-		ipoib_warn(priv, "failed to allocate tx ring\n");
 		ret = -ENOMEM;
 		goto err_tx;
 	}
@@ -1550,8 +1544,6 @@ static void ipoib_cm_create_srq(struct net_device *dev, int max_sge)
 
 	priv->cm.srq_ring = vzalloc(ipoib_recvq_size * sizeof *priv->cm.srq_ring);
 	if (!priv->cm.srq_ring) {
-		printk(KERN_WARNING "%s: failed to allocate CM SRQ ring (%d entries)\n",
-		       priv->ca->name, ipoib_recvq_size);
 		ib_destroy_srq(priv->cm.srq);
 		priv->cm.srq = NULL;
 		return;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
index 830fecb6934c..5038f9d2d753 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
@@ -416,11 +416,8 @@ static void ipoib_ib_handle_tx_wc(struct net_device *dev, struct ib_wc *wc)
 			   "(status=%d, wrid=%d vend_err %x)\n",
 			   wc->status, wr_id, wc->vendor_err);
 		qp_work = kzalloc(sizeof(*qp_work), GFP_ATOMIC);
-		if (!qp_work) {
-			ipoib_warn(priv, "%s Failed alloc ipoib_qp_state_validate for qp: 0x%x\n",
-				   __func__, priv->qp->qp_num);
+		if (!qp_work)
 			return;
-		}
 
 		INIT_WORK(&qp_work->work, ipoib_qp_state_validate_work);
 		qp_work->priv = priv;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index c50794fb92db..3ce0765a05ab 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -1619,11 +1619,8 @@ int ipoib_dev_init(struct net_device *dev, struct ib_device *ca, int port)
 	/* Allocate RX/TX "rings" to hold queued skbs */
 	priv->rx_ring =	kzalloc(ipoib_recvq_size * sizeof *priv->rx_ring,
 				GFP_KERNEL);
-	if (!priv->rx_ring) {
-		printk(KERN_WARNING "%s: failed to allocate RX ring (%d entries)\n",
-		       ca->name, ipoib_recvq_size);
+	if (!priv->rx_ring)
 		goto out;
-	}
 
 	priv->tx_ring = vzalloc(ipoib_sendq_size * sizeof *priv->tx_ring);
 	if (!priv->tx_ring) {
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index 1909dd252c94..fddff403d5d2 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -575,8 +575,11 @@ void ipoib_mcast_join_task(struct work_struct *work)
 	if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags))
 		return;
 
-	if (ib_query_port(priv->ca, priv->port, &port_attr) ||
-	    port_attr.state != IB_PORT_ACTIVE) {
+	if (ib_query_port(priv->ca, priv->port, &port_attr)) {
+		ipoib_dbg(priv, "ib_query_port() failed\n");
+		return;
+	}
+	if (port_attr.state != IB_PORT_ACTIVE) {
 		ipoib_dbg(priv, "port state is not ACTIVE (state = %d) suspending join task\n",
 			  port_attr.state);
 		return;
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index a4b791dfaa1d..8ae7a3beddb7 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -890,11 +890,14 @@ static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *eve
 	case RDMA_CM_EVENT_ESTABLISHED:
 		iser_connected_handler(cma_id, event->param.conn.private_data);
 		break;
+	case RDMA_CM_EVENT_REJECTED:
+		iser_info("Connection rejected: %s\n",
+			 rdma_reject_msg(cma_id, event->status));
+		/* FALLTHROUGH */
 	case RDMA_CM_EVENT_ADDR_ERROR:
 	case RDMA_CM_EVENT_ROUTE_ERROR:
 	case RDMA_CM_EVENT_CONNECT_ERROR:
 	case RDMA_CM_EVENT_UNREACHABLE:
-	case RDMA_CM_EVENT_REJECTED:
 		iser_connect_error(cma_id);
 		break;
 	case RDMA_CM_EVENT_DISCONNECTED:
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 6dd43f63238e..314e95516068 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -184,7 +184,7 @@ isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
 	isert_conn->rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS *
 				sizeof(struct iser_rx_desc), GFP_KERNEL);
 	if (!isert_conn->rx_descs)
-		goto fail;
+		return -ENOMEM;
 
 	rx_desc = isert_conn->rx_descs;
 
@@ -213,9 +213,7 @@ dma_map_fail:
 	}
 	kfree(isert_conn->rx_descs);
 	isert_conn->rx_descs = NULL;
-fail:
 	isert_err("conn %p failed to allocate rx descriptors\n", isert_conn);
-
 	return -ENOMEM;
 }
 
@@ -269,10 +267,8 @@ isert_alloc_comps(struct isert_device *device)
 
 	device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp),
 				GFP_KERNEL);
-	if (!device->comps) {
-		isert_err("Unable to allocate completion contexts\n");
+	if (!device->comps)
 		return -ENOMEM;
-	}
 
 	max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe);
 
@@ -432,10 +428,8 @@ isert_alloc_login_buf(struct isert_conn *isert_conn,
 
 	isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf),
 			GFP_KERNEL);
-	if (!isert_conn->login_req_buf) {
-		isert_err("Unable to allocate isert_conn->login_buf\n");
+	if (!isert_conn->login_req_buf)
 		return -ENOMEM;
-	}
 
 	isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
 				isert_conn->login_req_buf,
@@ -795,6 +789,8 @@ isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 		 */
 		return 1;
 	case RDMA_CM_EVENT_REJECTED:       /* FALLTHRU */
+		isert_info("Connection rejected: %s\n",
+			   rdma_reject_msg(cma_id, event->status));
 	case RDMA_CM_EVENT_UNREACHABLE:    /* FALLTHRU */
 	case RDMA_CM_EVENT_CONNECT_ERROR:
 		ret = isert_connect_error(cma_id);
@@ -1276,11 +1272,8 @@ isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd
 
 	if (payload_length) {
 		text_in = kzalloc(payload_length, GFP_KERNEL);
-		if (!text_in) {
-			isert_err("Unable to allocate text_in of payload_length: %u\n",
-				  payload_length);
+		if (!text_in)
 			return -ENOMEM;
-		}
 	}
 	cmd->text_in_ptr = text_in;
 
@@ -1851,6 +1844,8 @@ isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
 		isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
 				(void *)cmd->sense_buffer, pdu_len,
 				DMA_TO_DEVICE);
+		if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
+			return -ENOMEM;
 
 		isert_cmd->pdu_buf_len = pdu_len;
 		tx_dsg->addr	= isert_cmd->pdu_buf_dma;
@@ -1978,6 +1973,8 @@ isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
 	isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
 			(void *)cmd->buf_ptr, ISCSI_HDR_LEN,
 			DMA_TO_DEVICE);
+	if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
+		return -ENOMEM;
 	isert_cmd->pdu_buf_len = ISCSI_HDR_LEN;
 	tx_dsg->addr	= isert_cmd->pdu_buf_dma;
 	tx_dsg->length	= ISCSI_HDR_LEN;
@@ -2018,6 +2015,8 @@ isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
 
 		isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
 				txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE);
+		if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
+			return -ENOMEM;
 
 		isert_cmd->pdu_buf_len = txt_rsp_len;
 		tx_dsg->addr	= isert_cmd->pdu_buf_dma;
@@ -2307,10 +2306,9 @@ isert_setup_np(struct iscsi_np *np,
 	int ret;
 
 	isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
-	if (!isert_np) {
-		isert_err("Unable to allocate struct isert_np\n");
+	if (!isert_np)
 		return -ENOMEM;
-	}
+
 	sema_init(&isert_np->sem, 0);
 	mutex_init(&isert_np->mutex);
 	INIT_LIST_HEAD(&isert_np->accepted);
@@ -2651,7 +2649,6 @@ static int __init isert_init(void)
 					WQ_UNBOUND | WQ_HIGHPRI, 0);
 	if (!isert_comp_wq) {
 		isert_err("Unable to allocate isert_comp_wq\n");
-		ret = -ENOMEM;
 		return -ENOMEM;
 	}
 
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index d980fb458ad4..8ddc07123193 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -64,6 +64,11 @@ MODULE_LICENSE("Dual BSD/GPL");
 MODULE_VERSION(DRV_VERSION);
 MODULE_INFO(release_date, DRV_RELDATE);
 
+#if !defined(CONFIG_DYNAMIC_DEBUG)
+#define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt)
+#define DYNAMIC_DEBUG_BRANCH(descriptor) false
+#endif
+
 static unsigned int srp_sg_tablesize;
 static unsigned int cmd_sg_entries;
 static unsigned int indirect_sg_entries;
@@ -384,6 +389,9 @@ static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
 				 max_page_list_len);
 		if (IS_ERR(mr)) {
 			ret = PTR_ERR(mr);
+			if (ret == -ENOMEM)
+				pr_info("%s: ib_alloc_mr() failed. Try to reduce max_cmd_per_lun, max_sect or ch_count\n",
+					dev_name(&device->dev));
 			goto destroy_pool;
 		}
 		d->mr = mr;
@@ -1266,8 +1274,12 @@ static int srp_map_finish_fmr(struct srp_map_state *state,
 	struct ib_pool_fmr *fmr;
 	u64 io_addr = 0;
 
-	if (state->fmr.next >= state->fmr.end)
+	if (state->fmr.next >= state->fmr.end) {
+		shost_printk(KERN_ERR, ch->target->scsi_host,
+			     PFX "Out of MRs (mr_per_cmd = %d)\n",
+			     ch->target->mr_per_cmd);
 		return -ENOMEM;
+	}
 
 	WARN_ON_ONCE(!dev->use_fmr);
 
@@ -1323,8 +1335,12 @@ static int srp_map_finish_fr(struct srp_map_state *state,
 	u32 rkey;
 	int n, err;
 
-	if (state->fr.next >= state->fr.end)
+	if (state->fr.next >= state->fr.end) {
+		shost_printk(KERN_ERR, ch->target->scsi_host,
+			     PFX "Out of MRs (mr_per_cmd = %d)\n",
+			     ch->target->mr_per_cmd);
 		return -ENOMEM;
+	}
 
 	WARN_ON_ONCE(!dev->use_fast_reg);
 
@@ -1556,7 +1572,6 @@ static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
 	return 0;
 }
 
-#if defined(DYNAMIC_DATA_DEBUG)
 static void srp_check_mapping(struct srp_map_state *state,
 			      struct srp_rdma_ch *ch, struct srp_request *req,
 			      struct scatterlist *scat, int count)
@@ -1580,7 +1595,6 @@ static void srp_check_mapping(struct srp_map_state *state,
 		       scsi_bufflen(req->scmnd), desc_len, mr_len,
 		       state->ndesc, state->nmdesc);
 }
-#endif
 
 /**
  * srp_map_data() - map SCSI data buffer onto an SRP request
@@ -1669,14 +1683,12 @@ static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
 	if (ret < 0)
 		goto unmap;
 
-#if defined(DYNAMIC_DEBUG)
 	{
 		DEFINE_DYNAMIC_DEBUG_METADATA(ddm,
 			"Memory mapping consistency check");
-		if (unlikely(ddm.flags & _DPRINTK_FLAGS_PRINT))
+		if (DYNAMIC_DEBUG_BRANCH(ddm))
 			srp_check_mapping(&state, ch, req, scat, count);
 	}
-#endif
 
 	/* We've mapped the request, now pull as much of the indirect
 	 * descriptor table as we can into the command buffer. If this
@@ -3287,7 +3299,9 @@ static ssize_t srp_create_target(struct device *dev,
 	 */
 	scsi_host_get(target->scsi_host);
 
-	mutex_lock(&host->add_target_mutex);
+	ret = mutex_lock_interruptible(&host->add_target_mutex);
+	if (ret < 0)
+		goto put;
 
 	ret = srp_parse_options(buf, target);
 	if (ret)
@@ -3443,6 +3457,7 @@ connected:
 out:
 	mutex_unlock(&host->add_target_mutex);
 
+put:
 	scsi_host_put(target->scsi_host);
 	if (ret < 0)
 		scsi_host_put(target->scsi_host);
@@ -3526,6 +3541,7 @@ free_host:
 static void srp_add_one(struct ib_device *device)
 {
 	struct srp_device *srp_dev;
+	struct ib_device_attr *attr = &device->attrs;
 	struct srp_host *host;
 	int mr_page_shift, p;
 	u64 max_pages_per_mr;
@@ -3540,25 +3556,25 @@ static void srp_add_one(struct ib_device *device)
 	 * minimum of 4096 bytes. We're unlikely to build large sglists
 	 * out of smaller entries.
 	 */
-	mr_page_shift		= max(12, ffs(device->attrs.page_size_cap) - 1);
+	mr_page_shift		= max(12, ffs(attr->page_size_cap) - 1);
 	srp_dev->mr_page_size	= 1 << mr_page_shift;
 	srp_dev->mr_page_mask	= ~((u64) srp_dev->mr_page_size - 1);
-	max_pages_per_mr	= device->attrs.max_mr_size;
+	max_pages_per_mr	= attr->max_mr_size;
 	do_div(max_pages_per_mr, srp_dev->mr_page_size);
 	pr_debug("%s: %llu / %u = %llu <> %u\n", __func__,
-		 device->attrs.max_mr_size, srp_dev->mr_page_size,
+		 attr->max_mr_size, srp_dev->mr_page_size,
 		 max_pages_per_mr, SRP_MAX_PAGES_PER_MR);
 	srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
 					  max_pages_per_mr);
 
 	srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
 			    device->map_phys_fmr && device->unmap_fmr);
-	srp_dev->has_fr = (device->attrs.device_cap_flags &
+	srp_dev->has_fr = (attr->device_cap_flags &
 			   IB_DEVICE_MEM_MGT_EXTENSIONS);
 	if (!never_register && !srp_dev->has_fmr && !srp_dev->has_fr) {
 		dev_warn(&device->dev, "neither FMR nor FR is supported\n");
 	} else if (!never_register &&
-		   device->attrs.max_mr_size >= 2 * srp_dev->mr_page_size) {
+		   attr->max_mr_size >= 2 * srp_dev->mr_page_size) {
 		srp_dev->use_fast_reg = (srp_dev->has_fr &&
 					 (!srp_dev->has_fmr || prefer_fr));
 		srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
@@ -3571,13 +3587,13 @@ static void srp_add_one(struct ib_device *device)
 	if (srp_dev->use_fast_reg) {
 		srp_dev->max_pages_per_mr =
 			min_t(u32, srp_dev->max_pages_per_mr,
-			      device->attrs.max_fast_reg_page_list_len);
+			      attr->max_fast_reg_page_list_len);
 	}
 	srp_dev->mr_max_size	= srp_dev->mr_page_size *
 				   srp_dev->max_pages_per_mr;
 	pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
-		 device->name, mr_page_shift, device->attrs.max_mr_size,
-		 device->attrs.max_fast_reg_page_list_len,
+		 device->name, mr_page_shift, attr->max_mr_size,
+		 attr->max_fast_reg_page_list_len,
 		 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
 
 	INIT_LIST_HEAD(&srp_dev->dev_list);
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 0b1f69ed2e92..d21ba9d857c3 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -1840,7 +1840,6 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
 	struct srpt_rdma_ch *ch, *tmp_ch;
 	u32 it_iu_len;
 	int i, ret = 0;
-	unsigned char *p;
 
 	WARN_ON_ONCE(irqs_disabled());
 
@@ -1994,21 +1993,18 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
 			be64_to_cpu(*(__be64 *)(ch->i_port_id + 8)));
 
 	pr_debug("registering session %s\n", ch->sess_name);
-	p = &ch->sess_name[0];
 
-try_again:
 	ch->sess = target_alloc_session(&sport->port_tpg_1, 0, 0,
-					TARGET_PROT_NORMAL, p, ch, NULL);
+					TARGET_PROT_NORMAL, ch->sess_name, ch,
+					NULL);
+	/* Retry without leading "0x" */
+	if (IS_ERR(ch->sess))
+		ch->sess = target_alloc_session(&sport->port_tpg_1, 0, 0,
+						TARGET_PROT_NORMAL,
+						ch->sess_name + 2, ch, NULL);
 	if (IS_ERR(ch->sess)) {
-		pr_info("Rejected login because no ACL has been"
-			" configured yet for initiator %s.\n", p);
-		/*
-		 * XXX: Hack to retry of ch->i_port_id without leading '0x'
-		 */
-		if (p == &ch->sess_name[0]) {
-			p += 2;
-			goto try_again;
-		}
+		pr_info("Rejected login because no ACL has been configured yet for initiator %s.\n",
+			ch->sess_name);
 		rej->reason = cpu_to_be32((PTR_ERR(ch->sess) == -ENOMEM) ?
 				SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES :
 				SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED);
diff --git a/drivers/input/misc/arizona-haptics.c b/drivers/input/misc/arizona-haptics.c
index 982936334537..07ec465f1095 100644
--- a/drivers/input/misc/arizona-haptics.c
+++ b/drivers/input/misc/arizona-haptics.c
@@ -37,6 +37,8 @@ static void arizona_haptics_work(struct work_struct *work)
 						       struct arizona_haptics,
 						       work);
 	struct arizona *arizona = haptics->arizona;
+	struct snd_soc_component *component =
+		snd_soc_dapm_to_component(arizona->dapm);
 	int ret;
 
 	if (!haptics->arizona->dapm) {
@@ -66,7 +68,7 @@ static void arizona_haptics_work(struct work_struct *work)
 			return;
 		}
 
-		ret = snd_soc_dapm_enable_pin(arizona->dapm, "HAPTICS");
+		ret = snd_soc_component_enable_pin(component, "HAPTICS");
 		if (ret != 0) {
 			dev_err(arizona->dev, "Failed to start HAPTICS: %d\n",
 				ret);
@@ -81,7 +83,7 @@ static void arizona_haptics_work(struct work_struct *work)
 		}
 	} else {
 		/* This disable sequence will be a noop if already enabled */
-		ret = snd_soc_dapm_disable_pin(arizona->dapm, "HAPTICS");
+		ret = snd_soc_component_disable_pin(component, "HAPTICS");
 		if (ret != 0) {
 			dev_err(arizona->dev, "Failed to disable HAPTICS: %d\n",
 				ret);
@@ -140,11 +142,14 @@ static int arizona_haptics_play(struct input_dev *input, void *data,
 static void arizona_haptics_close(struct input_dev *input)
 {
 	struct arizona_haptics *haptics = input_get_drvdata(input);
+	struct snd_soc_component *component;
 
 	cancel_work_sync(&haptics->work);
 
-	if (haptics->arizona->dapm)
-		snd_soc_dapm_disable_pin(haptics->arizona->dapm, "HAPTICS");
+	if (haptics->arizona->dapm) {
+		component = snd_soc_dapm_to_component(haptics->arizona->dapm);
+		snd_soc_component_disable_pin(component, "HAPTICS");
+	}
 }
 
 static int arizona_haptics_probe(struct platform_device *pdev)
diff --git a/drivers/input/misc/xen-kbdfront.c b/drivers/input/misc/xen-kbdfront.c
index 227fbd2dbb71..3900875dec10 100644
--- a/drivers/input/misc/xen-kbdfront.c
+++ b/drivers/input/misc/xen-kbdfront.c
@@ -108,7 +108,8 @@ static irqreturn_t input_handler(int rq, void *dev_id)
 static int xenkbd_probe(struct xenbus_device *dev,
 				  const struct xenbus_device_id *id)
 {
-	int ret, i, abs;
+	int ret, i;
+	unsigned int abs;
 	struct xenkbd_info *info;
 	struct input_dev *kbd, *ptr;
 
@@ -127,8 +128,7 @@ static int xenkbd_probe(struct xenbus_device *dev,
 	if (!info->page)
 		goto error_nomem;
 
-	if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-abs-pointer", "%d", &abs) < 0)
-		abs = 0;
+	abs = xenbus_read_unsigned(dev->otherend, "feature-abs-pointer", 0);
 	if (abs) {
 		ret = xenbus_write(XBT_NIL, dev->nodename,
 				   "request-abs-pointer", "1");
@@ -322,11 +322,8 @@ static void xenkbd_backend_changed(struct xenbus_device *dev,
 
 	case XenbusStateInitWait:
 InitWait:
-		ret = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-				   "feature-abs-pointer", "%d", &val);
-		if (ret < 0)
-			val = 0;
-		if (val) {
+		if (xenbus_read_unsigned(info->xbdev->otherend,
+					 "feature-abs-pointer", 0)) {
 			ret = xenbus_write(XBT_NIL, info->xbdev->nodename,
 					   "request-abs-pointer", "1");
 			if (ret)
diff --git a/drivers/input/touchscreen/wm97xx-core.c b/drivers/input/touchscreen/wm97xx-core.c
index 90d6be3c26cc..83cf11312fd9 100644
--- a/drivers/input/touchscreen/wm97xx-core.c
+++ b/drivers/input/touchscreen/wm97xx-core.c
@@ -682,7 +682,7 @@ static int wm97xx_probe(struct device *dev)
 	}
 	platform_set_drvdata(wm->battery_dev, wm);
 	wm->battery_dev->dev.parent = dev;
-	wm->battery_dev->dev.platform_data = pdata;
+	wm->battery_dev->dev.platform_data = pdata->batt_pdata;
 	ret = platform_device_add(wm->battery_dev);
 	if (ret < 0)
 		goto batt_reg_err;
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 754595ee11b6..019e02707cd5 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -373,6 +373,8 @@ static struct iommu_group *acpihid_device_group(struct device *dev)
 
 	if (!entry->group)
 		entry->group = generic_device_group(dev);
+	else
+		iommu_group_ref_get(entry->group);
 
 	return entry->group;
 }
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 157e93421fb8..971154cbbb03 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -28,6 +28,7 @@
 #include <linux/amd-iommu.h>
 #include <linux/export.h>
 #include <linux/iommu.h>
+#include <linux/kmemleak.h>
 #include <asm/pci-direct.h>
 #include <asm/iommu.h>
 #include <asm/gart.h>
@@ -2090,6 +2091,7 @@ static struct syscore_ops amd_iommu_syscore_ops = {
 
 static void __init free_on_init_error(void)
 {
+	kmemleak_free(irq_lookup_table);
 	free_pages((unsigned long)irq_lookup_table,
 		   get_order(rlookup_table_size));
 
@@ -2321,6 +2323,8 @@ static int __init early_amd_iommu_init(void)
 		irq_lookup_table = (void *)__get_free_pages(
 				GFP_KERNEL | __GFP_ZERO,
 				get_order(rlookup_table_size));
+		kmemleak_alloc(irq_lookup_table, rlookup_table_size,
+			       1, GFP_KERNEL);
 		if (!irq_lookup_table)
 			goto out;
 	}
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index 594849a3a9be..f8ed8c95b685 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -805,8 +805,10 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
 		goto out_free_domain;
 
 	group = iommu_group_get(&pdev->dev);
-	if (!group)
+	if (!group) {
+		ret = -EINVAL;
 		goto out_free_domain;
+	}
 
 	ret = iommu_attach_group(dev_state->domain, group);
 	if (ret != 0)
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index e6f9b2d745ca..4d6ec444a9d6 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -20,6 +20,8 @@
  * This driver is powered by bad coffee and bombay mix.
  */
 
+#include <linux/acpi.h>
+#include <linux/acpi_iort.h>
 #include <linux/delay.h>
 #include <linux/dma-iommu.h>
 #include <linux/err.h>
@@ -1358,7 +1360,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 	} while (size -= granule);
 }
 
-static struct iommu_gather_ops arm_smmu_gather_ops = {
+static const struct iommu_gather_ops arm_smmu_gather_ops = {
 	.tlb_flush_all	= arm_smmu_tlb_inv_context,
 	.tlb_add_flush	= arm_smmu_tlb_inv_range_nosync,
 	.tlb_sync	= arm_smmu_tlb_sync,
@@ -1723,13 +1725,14 @@ static struct platform_driver arm_smmu_driver;
 
 static int arm_smmu_match_node(struct device *dev, void *data)
 {
-	return dev->of_node == data;
+	return dev->fwnode == data;
 }
 
-static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np)
+static
+struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
 {
 	struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
-						np, arm_smmu_match_node);
+						fwnode, arm_smmu_match_node);
 	put_device(dev);
 	return dev ? dev_get_drvdata(dev) : NULL;
 }
@@ -1765,7 +1768,7 @@ static int arm_smmu_add_device(struct device *dev)
 		master = fwspec->iommu_priv;
 		smmu = master->smmu;
 	} else {
-		smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode));
+		smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
 		if (!smmu)
 			return -ENODEV;
 		master = kzalloc(sizeof(*master), GFP_KERNEL);
@@ -2380,10 +2383,10 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
 	return 0;
 }
 
-static int arm_smmu_device_probe(struct arm_smmu_device *smmu)
+static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
 {
 	u32 reg;
-	bool coherent;
+	bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
 
 	/* IDR0 */
 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
@@ -2435,13 +2438,9 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu)
 		smmu->features |= ARM_SMMU_FEAT_HYP;
 
 	/*
-	 * The dma-coherent property is used in preference to the ID
+	 * The coherency feature as set by FW is used in preference to the ID
 	 * register, but warn on mismatch.
 	 */
-	coherent = of_dma_is_coherent(smmu->dev->of_node);
-	if (coherent)
-		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
-
 	if (!!(reg & IDR0_COHACC) != coherent)
 		dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n",
 			 coherent ? "true" : "false");
@@ -2562,21 +2561,61 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu)
 	return 0;
 }
 
-static int arm_smmu_device_dt_probe(struct platform_device *pdev)
+#ifdef CONFIG_ACPI
+static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
+				      struct arm_smmu_device *smmu)
+{
+	struct acpi_iort_smmu_v3 *iort_smmu;
+	struct device *dev = smmu->dev;
+	struct acpi_iort_node *node;
+
+	node = *(struct acpi_iort_node **)dev_get_platdata(dev);
+
+	/* Retrieve SMMUv3 specific data */
+	iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
+
+	if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
+		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
+
+	return 0;
+}
+#else
+static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
+					     struct arm_smmu_device *smmu)
+{
+	return -ENODEV;
+}
+#endif
+
+static int arm_smmu_device_dt_probe(struct platform_device *pdev,
+				    struct arm_smmu_device *smmu)
 {
-	int irq, ret;
-	struct resource *res;
-	struct arm_smmu_device *smmu;
 	struct device *dev = &pdev->dev;
-	bool bypass = true;
 	u32 cells;
+	int ret = -EINVAL;
 
 	if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
 		dev_err(dev, "missing #iommu-cells property\n");
 	else if (cells != 1)
 		dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
 	else
-		bypass = false;
+		ret = 0;
+
+	parse_driver_options(smmu);
+
+	if (of_dma_is_coherent(dev->of_node))
+		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
+
+	return ret;
+}
+
+static int arm_smmu_device_probe(struct platform_device *pdev)
+{
+	int irq, ret;
+	struct resource *res;
+	struct arm_smmu_device *smmu;
+	struct device *dev = &pdev->dev;
+	bool bypass;
 
 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
 	if (!smmu) {
@@ -2613,10 +2652,19 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 	if (irq > 0)
 		smmu->gerr_irq = irq;
 
-	parse_driver_options(smmu);
+	if (dev->of_node) {
+		ret = arm_smmu_device_dt_probe(pdev, smmu);
+	} else {
+		ret = arm_smmu_device_acpi_probe(pdev, smmu);
+		if (ret == -ENODEV)
+			return ret;
+	}
+
+	/* Set bypass mode according to firmware probing result */
+	bypass = !!ret;
 
 	/* Probe the h/w */
-	ret = arm_smmu_device_probe(smmu);
+	ret = arm_smmu_device_hw_probe(smmu);
 	if (ret)
 		return ret;
 
@@ -2634,7 +2682,8 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 		return ret;
 
 	/* And we're up. Go go go! */
-	of_iommu_set_ops(dev->of_node, &arm_smmu_ops);
+	iommu_register_instance(dev->fwnode, &arm_smmu_ops);
+
 #ifdef CONFIG_PCI
 	if (pci_bus_type.iommu_ops != &arm_smmu_ops) {
 		pci_request_acs();
@@ -2677,7 +2726,7 @@ static struct platform_driver arm_smmu_driver = {
 		.name		= "arm-smmu-v3",
 		.of_match_table	= of_match_ptr(arm_smmu_of_match),
 	},
-	.probe	= arm_smmu_device_dt_probe,
+	.probe	= arm_smmu_device_probe,
 	.remove	= arm_smmu_device_remove,
 };
 
@@ -2715,6 +2764,17 @@ static int __init arm_smmu_of_init(struct device_node *np)
 }
 IOMMU_OF_DECLARE(arm_smmuv3, "arm,smmu-v3", arm_smmu_of_init);
 
+#ifdef CONFIG_ACPI
+static int __init acpi_smmu_v3_init(struct acpi_table_header *table)
+{
+	if (iort_node_match(ACPI_IORT_NODE_SMMU_V3))
+		return arm_smmu_init();
+
+	return 0;
+}
+IORT_ACPI_DECLARE(arm_smmu_v3, ACPI_SIG_IORT, acpi_smmu_v3_init);
+#endif
+
 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 8f7281444551..a60cded8a6ed 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -28,6 +28,8 @@
 
 #define pr_fmt(fmt) "arm-smmu: " fmt
 
+#include <linux/acpi.h>
+#include <linux/acpi_iort.h>
 #include <linux/atomic.h>
 #include <linux/delay.h>
 #include <linux/dma-iommu.h>
@@ -247,6 +249,7 @@ enum arm_smmu_s2cr_privcfg {
 #define ARM_MMU500_ACTLR_CPRE		(1 << 1)
 
 #define ARM_MMU500_ACR_CACHE_LOCK	(1 << 26)
+#define ARM_MMU500_ACR_SMTNMB_TLBEN	(1 << 8)
 
 #define CB_PAR_F			(1 << 0)
 
@@ -642,7 +645,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 	}
 }
 
-static struct iommu_gather_ops arm_smmu_gather_ops = {
+static const struct iommu_gather_ops arm_smmu_gather_ops = {
 	.tlb_flush_all	= arm_smmu_tlb_inv_context,
 	.tlb_add_flush	= arm_smmu_tlb_inv_range_nosync,
 	.tlb_sync	= arm_smmu_tlb_sync,
@@ -1379,13 +1382,14 @@ static bool arm_smmu_capable(enum iommu_cap cap)
 
 static int arm_smmu_match_node(struct device *dev, void *data)
 {
-	return dev->of_node == data;
+	return dev->fwnode == data;
 }
 
-static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np)
+static
+struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
 {
 	struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
-						np, arm_smmu_match_node);
+						fwnode, arm_smmu_match_node);
 	put_device(dev);
 	return dev ? dev_get_drvdata(dev) : NULL;
 }
@@ -1403,7 +1407,7 @@ static int arm_smmu_add_device(struct device *dev)
 		if (ret)
 			goto out_free;
 	} else if (fwspec && fwspec->ops == &arm_smmu_ops) {
-		smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode));
+		smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
 	} else {
 		return -ENODEV;
 	}
@@ -1478,7 +1482,7 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev)
 	}
 
 	if (group)
-		return group;
+		return iommu_group_ref_get(group);
 
 	if (dev_is_pci(dev))
 		group = pci_device_group(dev);
@@ -1581,16 +1585,22 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
 	for (i = 0; i < smmu->num_mapping_groups; ++i)
 		arm_smmu_write_sme(smmu, i);
 
-	/*
-	 * Before clearing ARM_MMU500_ACTLR_CPRE, need to
-	 * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
-	 * bit is only present in MMU-500r2 onwards.
-	 */
-	reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
-	major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
-	if ((smmu->model == ARM_MMU500) && (major >= 2)) {
+	if (smmu->model == ARM_MMU500) {
+		/*
+		 * Before clearing ARM_MMU500_ACTLR_CPRE, need to
+		 * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
+		 * bit is only present in MMU-500r2 onwards.
+		 */
+		reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
+		major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
 		reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
-		reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
+		if (major >= 2)
+			reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
+		/*
+		 * Allow unmatched Stream IDs to allocate bypass
+		 * TLB entries for reduced latency.
+		 */
+		reg |= ARM_MMU500_ACR_SMTNMB_TLBEN;
 		writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
 	}
 
@@ -1667,7 +1677,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
 	unsigned long size;
 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 	u32 id;
-	bool cttw_dt, cttw_reg;
+	bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
 	int i;
 
 	dev_notice(smmu->dev, "probing hardware configuration...\n");
@@ -1712,20 +1722,17 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
 
 	/*
 	 * In order for DMA API calls to work properly, we must defer to what
-	 * the DT says about coherency, regardless of what the hardware claims.
+	 * the FW says about coherency, regardless of what the hardware claims.
 	 * Fortunately, this also opens up a workaround for systems where the
 	 * ID register value has ended up configured incorrectly.
 	 */
-	cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
 	cttw_reg = !!(id & ID0_CTTW);
-	if (cttw_dt)
-		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
-	if (cttw_dt || cttw_reg)
+	if (cttw_fw || cttw_reg)
 		dev_notice(smmu->dev, "\t%scoherent table walk\n",
-			   cttw_dt ? "" : "non-");
-	if (cttw_dt != cttw_reg)
+			   cttw_fw ? "" : "non-");
+	if (cttw_fw != cttw_reg)
 		dev_notice(smmu->dev,
-			   "\t(IDR0.CTTW overridden by dma-coherent property)\n");
+			   "\t(IDR0.CTTW overridden by FW configuration)\n");
 
 	/* Max. number of entries we have for stream matching/indexing */
 	size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
@@ -1906,15 +1913,83 @@ static const struct of_device_id arm_smmu_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
 
-static int arm_smmu_device_dt_probe(struct platform_device *pdev)
+#ifdef CONFIG_ACPI
+static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
+{
+	int ret = 0;
+
+	switch (model) {
+	case ACPI_IORT_SMMU_V1:
+	case ACPI_IORT_SMMU_CORELINK_MMU400:
+		smmu->version = ARM_SMMU_V1;
+		smmu->model = GENERIC_SMMU;
+		break;
+	case ACPI_IORT_SMMU_V2:
+		smmu->version = ARM_SMMU_V2;
+		smmu->model = GENERIC_SMMU;
+		break;
+	case ACPI_IORT_SMMU_CORELINK_MMU500:
+		smmu->version = ARM_SMMU_V2;
+		smmu->model = ARM_MMU500;
+		break;
+	default:
+		ret = -ENODEV;
+	}
+
+	return ret;
+}
+
+static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
+				      struct arm_smmu_device *smmu)
+{
+	struct device *dev = smmu->dev;
+	struct acpi_iort_node *node =
+		*(struct acpi_iort_node **)dev_get_platdata(dev);
+	struct acpi_iort_smmu *iort_smmu;
+	int ret;
+
+	/* Retrieve SMMU1/2 specific data */
+	iort_smmu = (struct acpi_iort_smmu *)node->node_data;
+
+	ret = acpi_smmu_get_data(iort_smmu->model, smmu);
+	if (ret < 0)
+		return ret;
+
+	/* Ignore the configuration access interrupt */
+	smmu->num_global_irqs = 1;
+
+	if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
+		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
+
+	return 0;
+}
+#else
+static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
+					     struct arm_smmu_device *smmu)
+{
+	return -ENODEV;
+}
+#endif
+
+static int arm_smmu_device_dt_probe(struct platform_device *pdev,
+				    struct arm_smmu_device *smmu)
 {
 	const struct arm_smmu_match_data *data;
-	struct resource *res;
-	struct arm_smmu_device *smmu;
 	struct device *dev = &pdev->dev;
-	int num_irqs, i, err;
 	bool legacy_binding;
 
+	if (of_property_read_u32(dev->of_node, "#global-interrupts",
+				 &smmu->num_global_irqs)) {
+		dev_err(dev, "missing #global-interrupts property\n");
+		return -ENODEV;
+	}
+
+	data = of_device_get_match_data(dev);
+	smmu->version = data->version;
+	smmu->model = data->model;
+
+	parse_driver_options(smmu);
+
 	legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
 	if (legacy_binding && !using_generic_binding) {
 		if (!using_legacy_binding)
@@ -1927,6 +2002,19 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
+	if (of_dma_is_coherent(dev->of_node))
+		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
+
+	return 0;
+}
+
+static int arm_smmu_device_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	struct arm_smmu_device *smmu;
+	struct device *dev = &pdev->dev;
+	int num_irqs, i, err;
+
 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
 	if (!smmu) {
 		dev_err(dev, "failed to allocate arm_smmu_device\n");
@@ -1934,9 +2022,13 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 	}
 	smmu->dev = dev;
 
-	data = of_device_get_match_data(dev);
-	smmu->version = data->version;
-	smmu->model = data->model;
+	if (dev->of_node)
+		err = arm_smmu_device_dt_probe(pdev, smmu);
+	else
+		err = arm_smmu_device_acpi_probe(pdev, smmu);
+
+	if (err)
+		return err;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	smmu->base = devm_ioremap_resource(dev, res);
@@ -1944,12 +2036,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 		return PTR_ERR(smmu->base);
 	smmu->size = resource_size(res);
 
-	if (of_property_read_u32(dev->of_node, "#global-interrupts",
-				 &smmu->num_global_irqs)) {
-		dev_err(dev, "missing #global-interrupts property\n");
-		return -ENODEV;
-	}
-
 	num_irqs = 0;
 	while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
 		num_irqs++;
@@ -1984,8 +2070,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
-	parse_driver_options(smmu);
-
 	if (smmu->version == ARM_SMMU_V2 &&
 	    smmu->num_context_banks != smmu->num_context_irqs) {
 		dev_err(dev,
@@ -2007,7 +2091,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 		}
 	}
 
-	of_iommu_set_ops(dev->of_node, &arm_smmu_ops);
+	iommu_register_instance(dev->fwnode, &arm_smmu_ops);
 	platform_set_drvdata(pdev, smmu);
 	arm_smmu_device_reset(smmu);
 
@@ -2047,7 +2131,7 @@ static struct platform_driver arm_smmu_driver = {
 		.name		= "arm-smmu",
 		.of_match_table	= of_match_ptr(arm_smmu_of_match),
 	},
-	.probe	= arm_smmu_device_dt_probe,
+	.probe	= arm_smmu_device_probe,
 	.remove	= arm_smmu_device_remove,
 };
 
@@ -2090,6 +2174,17 @@ IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
 
+#ifdef CONFIG_ACPI
+static int __init arm_smmu_acpi_init(struct acpi_table_header *table)
+{
+	if (iort_node_match(ACPI_IORT_NODE_SMMU))
+		return arm_smmu_init();
+
+	return 0;
+}
+IORT_ACPI_DECLARE(arm_smmu, ACPI_SIG_IORT, arm_smmu_acpi_init);
+#endif
+
 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index c5ab8667e6f2..2db0d641cf45 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -432,13 +432,12 @@ int iommu_dma_mmap(struct page **pages, size_t size, struct vm_area_struct *vma)
 	return ret;
 }
 
-dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
-		unsigned long offset, size_t size, int prot)
+static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
+		size_t size, int prot)
 {
 	dma_addr_t dma_addr;
 	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
 	struct iova_domain *iovad = cookie_iovad(domain);
-	phys_addr_t phys = page_to_phys(page) + offset;
 	size_t iova_off = iova_offset(iovad, phys);
 	size_t len = iova_align(iovad, size + iova_off);
 	struct iova *iova = __alloc_iova(domain, len, dma_get_mask(dev));
@@ -454,6 +453,12 @@ dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
 	return dma_addr + iova_off;
 }
 
+dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
+		unsigned long offset, size_t size, int prot)
+{
+	return __iommu_dma_map(dev, page_to_phys(page) + offset, size, prot);
+}
+
 void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size,
 		enum dma_data_direction dir, unsigned long attrs)
 {
@@ -624,6 +629,19 @@ void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
 	__iommu_dma_unmap(iommu_get_domain_for_dev(dev), sg_dma_address(sg));
 }
 
+dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
+		size_t size, enum dma_data_direction dir, unsigned long attrs)
+{
+	return __iommu_dma_map(dev, phys, size,
+			dma_direction_to_prot(dir, false) | IOMMU_MMIO);
+}
+
+void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,
+		size_t size, enum dma_data_direction dir, unsigned long attrs)
+{
+	__iommu_dma_unmap(iommu_get_domain_for_dev(dev), handle);
+}
+
 int iommu_dma_supported(struct device *dev, u64 mask)
 {
 	/*
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 30808e91b775..57ba0d3091ea 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -70,6 +70,36 @@ static short PG_ENT_SHIFT = -1;
 #define SYSMMU_PG_ENT_SHIFT 0
 #define SYSMMU_V5_PG_ENT_SHIFT 4
 
+static const sysmmu_pte_t *LV1_PROT;
+static const sysmmu_pte_t SYSMMU_LV1_PROT[] = {
+	((0 << 15) | (0 << 10)), /* no access */
+	((1 << 15) | (1 << 10)), /* IOMMU_READ only */
+	((0 << 15) | (1 << 10)), /* IOMMU_WRITE not supported, use read/write */
+	((0 << 15) | (1 << 10)), /* IOMMU_READ | IOMMU_WRITE */
+};
+static const sysmmu_pte_t SYSMMU_V5_LV1_PROT[] = {
+	(0 << 4), /* no access */
+	(1 << 4), /* IOMMU_READ only */
+	(2 << 4), /* IOMMU_WRITE only */
+	(3 << 4), /* IOMMU_READ | IOMMU_WRITE */
+};
+
+static const sysmmu_pte_t *LV2_PROT;
+static const sysmmu_pte_t SYSMMU_LV2_PROT[] = {
+	((0 << 9) | (0 << 4)), /* no access */
+	((1 << 9) | (1 << 4)), /* IOMMU_READ only */
+	((0 << 9) | (1 << 4)), /* IOMMU_WRITE not supported, use read/write */
+	((0 << 9) | (1 << 4)), /* IOMMU_READ | IOMMU_WRITE */
+};
+static const sysmmu_pte_t SYSMMU_V5_LV2_PROT[] = {
+	(0 << 2), /* no access */
+	(1 << 2), /* IOMMU_READ only */
+	(2 << 2), /* IOMMU_WRITE only */
+	(3 << 2), /* IOMMU_READ | IOMMU_WRITE */
+};
+
+#define SYSMMU_SUPPORTED_PROT_BITS (IOMMU_READ | IOMMU_WRITE)
+
 #define sect_to_phys(ent) (((phys_addr_t) ent) << PG_ENT_SHIFT)
 #define section_phys(sent) (sect_to_phys(*(sent)) & SECT_MASK)
 #define section_offs(iova) (iova & (SECT_SIZE - 1))
@@ -97,16 +127,17 @@ static u32 lv2ent_offset(sysmmu_iova_t iova)
 #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE)
 #define lv2table_base(sent) (sect_to_phys(*(sent) & 0xFFFFFFC0))
 
-#define mk_lv1ent_sect(pa) ((pa >> PG_ENT_SHIFT) | 2)
+#define mk_lv1ent_sect(pa, prot) ((pa >> PG_ENT_SHIFT) | LV1_PROT[prot] | 2)
 #define mk_lv1ent_page(pa) ((pa >> PG_ENT_SHIFT) | 1)
-#define mk_lv2ent_lpage(pa) ((pa >> PG_ENT_SHIFT) | 1)
-#define mk_lv2ent_spage(pa) ((pa >> PG_ENT_SHIFT) | 2)
+#define mk_lv2ent_lpage(pa, prot) ((pa >> PG_ENT_SHIFT) | LV2_PROT[prot] | 1)
+#define mk_lv2ent_spage(pa, prot) ((pa >> PG_ENT_SHIFT) | LV2_PROT[prot] | 2)
 
 #define CTRL_ENABLE	0x5
 #define CTRL_BLOCK	0x7
 #define CTRL_DISABLE	0x0
 
 #define CFG_LRU		0x1
+#define CFG_EAP		(1 << 2)
 #define CFG_QOS(n)	((n & 0xF) << 7)
 #define CFG_ACGEN	(1 << 24) /* System MMU 3.3 only */
 #define CFG_SYSSEL	(1 << 22) /* System MMU 3.2 only */
@@ -206,6 +237,7 @@ static const struct sysmmu_fault_info sysmmu_v5_faults[] = {
 struct exynos_iommu_owner {
 	struct list_head controllers;	/* list of sysmmu_drvdata.owner_node */
 	struct iommu_domain *domain;	/* domain this device is attached */
+	struct mutex rpm_lock;		/* for runtime pm of all sysmmus */
 };
 
 /*
@@ -237,8 +269,8 @@ struct sysmmu_drvdata {
 	struct clk *aclk;		/* SYSMMU's aclk clock */
 	struct clk *pclk;		/* SYSMMU's pclk clock */
 	struct clk *clk_master;		/* master's device clock */
-	int activations;		/* number of calls to sysmmu_enable */
 	spinlock_t lock;		/* lock for modyfying state */
+	bool active;			/* current status */
 	struct exynos_iommu_domain *domain; /* domain we belong to */
 	struct list_head domain_node;	/* node for domain clients list */
 	struct list_head owner_node;	/* node for owner controllers list */
@@ -251,25 +283,6 @@ static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom)
 	return container_of(dom, struct exynos_iommu_domain, domain);
 }
 
-static bool set_sysmmu_active(struct sysmmu_drvdata *data)
-{
-	/* return true if the System MMU was not active previously
-	   and it needs to be initialized */
-	return ++data->activations == 1;
-}
-
-static bool set_sysmmu_inactive(struct sysmmu_drvdata *data)
-{
-	/* return true if the System MMU is needed to be disabled */
-	BUG_ON(data->activations < 1);
-	return --data->activations == 0;
-}
-
-static bool is_sysmmu_active(struct sysmmu_drvdata *data)
-{
-	return data->activations > 0;
-}
-
 static void sysmmu_unblock(struct sysmmu_drvdata *data)
 {
 	writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL);
@@ -388,7 +401,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
 	unsigned short reg_status, reg_clear;
 	int ret = -ENOSYS;
 
-	WARN_ON(!is_sysmmu_active(data));
+	WARN_ON(!data->active);
 
 	if (MMU_MAJ_VER(data->version) < 5) {
 		reg_status = REG_INT_STATUS;
@@ -434,40 +447,19 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data)
+static void __sysmmu_disable(struct sysmmu_drvdata *data)
 {
+	unsigned long flags;
+
 	clk_enable(data->clk_master);
 
+	spin_lock_irqsave(&data->lock, flags);
 	writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL);
 	writel(0, data->sfrbase + REG_MMU_CFG);
-
-	__sysmmu_disable_clocks(data);
-}
-
-static bool __sysmmu_disable(struct sysmmu_drvdata *data)
-{
-	bool disabled;
-	unsigned long flags;
-
-	spin_lock_irqsave(&data->lock, flags);
-
-	disabled = set_sysmmu_inactive(data);
-
-	if (disabled) {
-		data->pgtable = 0;
-		data->domain = NULL;
-
-		__sysmmu_disable_nocount(data);
-
-		dev_dbg(data->sysmmu, "Disabled\n");
-	} else  {
-		dev_dbg(data->sysmmu, "%d times left to disable\n",
-					data->activations);
-	}
-
+	data->active = false;
 	spin_unlock_irqrestore(&data->lock, flags);
 
-	return disabled;
+	__sysmmu_disable_clocks(data);
 }
 
 static void __sysmmu_init_config(struct sysmmu_drvdata *data)
@@ -481,20 +473,24 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data)
 	else
 		cfg = CFG_QOS(15) | CFG_FLPDCACHE | CFG_ACGEN;
 
+	cfg |= CFG_EAP; /* enable access protection bits check */
+
 	writel(cfg, data->sfrbase + REG_MMU_CFG);
 }
 
-static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
+static void __sysmmu_enable(struct sysmmu_drvdata *data)
 {
+	unsigned long flags;
+
 	__sysmmu_enable_clocks(data);
 
+	spin_lock_irqsave(&data->lock, flags);
 	writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
-
 	__sysmmu_init_config(data);
-
 	__sysmmu_set_ptbase(data, data->pgtable);
-
 	writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL);
+	data->active = true;
+	spin_unlock_irqrestore(&data->lock, flags);
 
 	/*
 	 * SYSMMU driver keeps master's clock enabled only for the short
@@ -505,48 +501,18 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
 	clk_disable(data->clk_master);
 }
 
-static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable,
-			   struct exynos_iommu_domain *domain)
-{
-	int ret = 0;
-	unsigned long flags;
-
-	spin_lock_irqsave(&data->lock, flags);
-	if (set_sysmmu_active(data)) {
-		data->pgtable = pgtable;
-		data->domain = domain;
-
-		__sysmmu_enable_nocount(data);
-
-		dev_dbg(data->sysmmu, "Enabled\n");
-	} else {
-		ret = (pgtable == data->pgtable) ? 1 : -EBUSY;
-
-		dev_dbg(data->sysmmu, "already enabled\n");
-	}
-
-	if (WARN_ON(ret < 0))
-		set_sysmmu_inactive(data); /* decrement count */
-
-	spin_unlock_irqrestore(&data->lock, flags);
-
-	return ret;
-}
-
 static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
 					    sysmmu_iova_t iova)
 {
 	unsigned long flags;
 
-
 	spin_lock_irqsave(&data->lock, flags);
-	if (is_sysmmu_active(data) && data->version >= MAKE_MMU_VER(3, 3)) {
+	if (data->active && data->version >= MAKE_MMU_VER(3, 3)) {
 		clk_enable(data->clk_master);
 		__sysmmu_tlb_invalidate_entry(data, iova, 1);
 		clk_disable(data->clk_master);
 	}
 	spin_unlock_irqrestore(&data->lock, flags);
-
 }
 
 static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
@@ -555,7 +521,7 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
 	unsigned long flags;
 
 	spin_lock_irqsave(&data->lock, flags);
-	if (is_sysmmu_active(data)) {
+	if (data->active) {
 		unsigned int num_inv = 1;
 
 		clk_enable(data->clk_master);
@@ -578,9 +544,6 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
 			sysmmu_unblock(data);
 		}
 		clk_disable(data->clk_master);
-	} else {
-		dev_dbg(data->master,
-			"disabled. Skipping TLB invalidation @ %#x\n", iova);
 	}
 	spin_unlock_irqrestore(&data->lock, flags);
 }
@@ -652,10 +615,15 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
 
 	__sysmmu_get_version(data);
 	if (PG_ENT_SHIFT < 0) {
-		if (MMU_MAJ_VER(data->version) < 5)
+		if (MMU_MAJ_VER(data->version) < 5) {
 			PG_ENT_SHIFT = SYSMMU_PG_ENT_SHIFT;
-		else
+			LV1_PROT = SYSMMU_LV1_PROT;
+			LV2_PROT = SYSMMU_LV2_PROT;
+		} else {
 			PG_ENT_SHIFT = SYSMMU_V5_PG_ENT_SHIFT;
+			LV1_PROT = SYSMMU_V5_LV1_PROT;
+			LV2_PROT = SYSMMU_V5_LV2_PROT;
+		}
 	}
 
 	pm_runtime_enable(dev);
@@ -665,34 +633,46 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int exynos_sysmmu_suspend(struct device *dev)
+static int __maybe_unused exynos_sysmmu_suspend(struct device *dev)
 {
 	struct sysmmu_drvdata *data = dev_get_drvdata(dev);
+	struct device *master = data->master;
+
+	if (master) {
+		struct exynos_iommu_owner *owner = master->archdata.iommu;
 
-	dev_dbg(dev, "suspend\n");
-	if (is_sysmmu_active(data)) {
-		__sysmmu_disable_nocount(data);
-		pm_runtime_put(dev);
+		mutex_lock(&owner->rpm_lock);
+		if (data->domain) {
+			dev_dbg(data->sysmmu, "saving state\n");
+			__sysmmu_disable(data);
+		}
+		mutex_unlock(&owner->rpm_lock);
 	}
 	return 0;
 }
 
-static int exynos_sysmmu_resume(struct device *dev)
+static int __maybe_unused exynos_sysmmu_resume(struct device *dev)
 {
 	struct sysmmu_drvdata *data = dev_get_drvdata(dev);
+	struct device *master = data->master;
+
+	if (master) {
+		struct exynos_iommu_owner *owner = master->archdata.iommu;
 
-	dev_dbg(dev, "resume\n");
-	if (is_sysmmu_active(data)) {
-		pm_runtime_get_sync(dev);
-		__sysmmu_enable_nocount(data);
+		mutex_lock(&owner->rpm_lock);
+		if (data->domain) {
+			dev_dbg(data->sysmmu, "restoring state\n");
+			__sysmmu_enable(data);
+		}
+		mutex_unlock(&owner->rpm_lock);
 	}
 	return 0;
 }
-#endif
 
 static const struct dev_pm_ops sysmmu_pm_ops = {
-	SET_LATE_SYSTEM_SLEEP_PM_OPS(exynos_sysmmu_suspend, exynos_sysmmu_resume)
+	SET_RUNTIME_PM_OPS(exynos_sysmmu_suspend, exynos_sysmmu_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 };
 
 static const struct of_device_id sysmmu_of_match[] __initconst = {
@@ -796,9 +776,12 @@ static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain)
 	spin_lock_irqsave(&domain->lock, flags);
 
 	list_for_each_entry_safe(data, next, &domain->clients, domain_node) {
-		if (__sysmmu_disable(data))
-			data->master = NULL;
+		spin_lock(&data->lock);
+		__sysmmu_disable(data);
+		data->pgtable = 0;
+		data->domain = NULL;
 		list_del_init(&data->domain_node);
+		spin_unlock(&data->lock);
 	}
 
 	spin_unlock_irqrestore(&domain->lock, flags);
@@ -832,31 +815,34 @@ static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain,
 	phys_addr_t pagetable = virt_to_phys(domain->pgtable);
 	struct sysmmu_drvdata *data, *next;
 	unsigned long flags;
-	bool found = false;
 
 	if (!has_sysmmu(dev) || owner->domain != iommu_domain)
 		return;
 
+	mutex_lock(&owner->rpm_lock);
+
+	list_for_each_entry(data, &owner->controllers, owner_node) {
+		pm_runtime_get_noresume(data->sysmmu);
+		if (pm_runtime_active(data->sysmmu))
+			__sysmmu_disable(data);
+		pm_runtime_put(data->sysmmu);
+	}
+
 	spin_lock_irqsave(&domain->lock, flags);
 	list_for_each_entry_safe(data, next, &domain->clients, domain_node) {
-		if (data->master == dev) {
-			if (__sysmmu_disable(data)) {
-				data->master = NULL;
-				list_del_init(&data->domain_node);
-			}
-			pm_runtime_put(data->sysmmu);
-			found = true;
-		}
+		spin_lock(&data->lock);
+		data->pgtable = 0;
+		data->domain = NULL;
+		list_del_init(&data->domain_node);
+		spin_unlock(&data->lock);
 	}
+	owner->domain = NULL;
 	spin_unlock_irqrestore(&domain->lock, flags);
 
-	owner->domain = NULL;
+	mutex_unlock(&owner->rpm_lock);
 
-	if (found)
-		dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n",
-					__func__, &pagetable);
-	else
-		dev_err(dev, "%s: No IOMMU is attached\n", __func__);
+	dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", __func__,
+		&pagetable);
 }
 
 static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain,
@@ -867,7 +853,6 @@ static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain,
 	struct sysmmu_drvdata *data;
 	phys_addr_t pagetable = virt_to_phys(domain->pgtable);
 	unsigned long flags;
-	int ret = -ENODEV;
 
 	if (!has_sysmmu(dev))
 		return -ENODEV;
@@ -875,29 +860,32 @@ static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain,
 	if (owner->domain)
 		exynos_iommu_detach_device(owner->domain, dev);
 
+	mutex_lock(&owner->rpm_lock);
+
+	spin_lock_irqsave(&domain->lock, flags);
 	list_for_each_entry(data, &owner->controllers, owner_node) {
-		pm_runtime_get_sync(data->sysmmu);
-		ret = __sysmmu_enable(data, pagetable, domain);
-		if (ret >= 0) {
-			data->master = dev;
-
-			spin_lock_irqsave(&domain->lock, flags);
-			list_add_tail(&data->domain_node, &domain->clients);
-			spin_unlock_irqrestore(&domain->lock, flags);
-		}
+		spin_lock(&data->lock);
+		data->pgtable = pagetable;
+		data->domain = domain;
+		list_add_tail(&data->domain_node, &domain->clients);
+		spin_unlock(&data->lock);
 	}
+	owner->domain = iommu_domain;
+	spin_unlock_irqrestore(&domain->lock, flags);
 
-	if (ret < 0) {
-		dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n",
-					__func__, &pagetable);
-		return ret;
+	list_for_each_entry(data, &owner->controllers, owner_node) {
+		pm_runtime_get_noresume(data->sysmmu);
+		if (pm_runtime_active(data->sysmmu))
+			__sysmmu_enable(data);
+		pm_runtime_put(data->sysmmu);
 	}
 
-	owner->domain = iommu_domain;
-	dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa %s\n",
-		__func__, &pagetable, (ret == 0) ? "" : ", again");
+	mutex_unlock(&owner->rpm_lock);
 
-	return ret;
+	dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa\n", __func__,
+		&pagetable);
+
+	return 0;
 }
 
 static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain,
@@ -954,7 +942,7 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain,
 
 static int lv1set_section(struct exynos_iommu_domain *domain,
 			  sysmmu_pte_t *sent, sysmmu_iova_t iova,
-			  phys_addr_t paddr, short *pgcnt)
+			  phys_addr_t paddr, int prot, short *pgcnt)
 {
 	if (lv1ent_section(sent)) {
 		WARN(1, "Trying mapping on 1MiB@%#08x that is mapped",
@@ -973,7 +961,7 @@ static int lv1set_section(struct exynos_iommu_domain *domain,
 		*pgcnt = 0;
 	}
 
-	update_pte(sent, mk_lv1ent_sect(paddr));
+	update_pte(sent, mk_lv1ent_sect(paddr, prot));
 
 	spin_lock(&domain->lock);
 	if (lv1ent_page_zero(sent)) {
@@ -991,13 +979,13 @@ static int lv1set_section(struct exynos_iommu_domain *domain,
 }
 
 static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
-								short *pgcnt)
+		       int prot, short *pgcnt)
 {
 	if (size == SPAGE_SIZE) {
 		if (WARN_ON(!lv2ent_fault(pent)))
 			return -EADDRINUSE;
 
-		update_pte(pent, mk_lv2ent_spage(paddr));
+		update_pte(pent, mk_lv2ent_spage(paddr, prot));
 		*pgcnt -= 1;
 	} else { /* size == LPAGE_SIZE */
 		int i;
@@ -1013,7 +1001,7 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
 				return -EADDRINUSE;
 			}
 
-			*pent = mk_lv2ent_lpage(paddr);
+			*pent = mk_lv2ent_lpage(paddr, prot);
 		}
 		dma_sync_single_for_device(dma_dev, pent_base,
 					   sizeof(*pent) * SPAGES_PER_LPAGE,
@@ -1061,13 +1049,14 @@ static int exynos_iommu_map(struct iommu_domain *iommu_domain,
 	int ret = -ENOMEM;
 
 	BUG_ON(domain->pgtable == NULL);
+	prot &= SYSMMU_SUPPORTED_PROT_BITS;
 
 	spin_lock_irqsave(&domain->pgtablelock, flags);
 
 	entry = section_entry(domain->pgtable, iova);
 
 	if (size == SECT_SIZE) {
-		ret = lv1set_section(domain, entry, iova, paddr,
+		ret = lv1set_section(domain, entry, iova, paddr, prot,
 				     &domain->lv2entcnt[lv1ent_offset(iova)]);
 	} else {
 		sysmmu_pte_t *pent;
@@ -1078,7 +1067,7 @@ static int exynos_iommu_map(struct iommu_domain *iommu_domain,
 		if (IS_ERR(pent))
 			ret = PTR_ERR(pent);
 		else
-			ret = lv2set_page(pent, paddr, size,
+			ret = lv2set_page(pent, paddr, size, prot,
 				       &domain->lv2entcnt[lv1ent_offset(iova)]);
 	}
 
@@ -1268,10 +1257,20 @@ static int exynos_iommu_of_xlate(struct device *dev,
 			return -ENOMEM;
 
 		INIT_LIST_HEAD(&owner->controllers);
+		mutex_init(&owner->rpm_lock);
 		dev->archdata.iommu = owner;
 	}
 
 	list_add_tail(&data->owner_node, &owner->controllers);
+	data->master = dev;
+
+	/*
+	 * SYSMMU will be runtime activated via device link (dependency) to its
+	 * master device, so there are no direct calls to pm_runtime_get/put
+	 * in this driver.
+	 */
+	device_link_add(dev, data->sysmmu, DL_FLAG_PM_RUNTIME);
+
 	return 0;
 }
 
diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c
index f50e51c1a9c8..0769276c0537 100644
--- a/drivers/iommu/io-pgtable-arm-v7s.c
+++ b/drivers/iommu/io-pgtable-arm-v7s.c
@@ -793,8 +793,7 @@ static int __init arm_v7s_do_selftests(void)
 	 * Distinct mappings of different granule sizes.
 	 */
 	iova = 0;
-	i = find_first_bit(&cfg.pgsize_bitmap, BITS_PER_LONG);
-	while (i != BITS_PER_LONG) {
+	for_each_set_bit(i, &cfg.pgsize_bitmap, BITS_PER_LONG) {
 		size = 1UL << i;
 		if (ops->map(ops, iova, iova, size, IOMMU_READ |
 						    IOMMU_WRITE |
@@ -811,8 +810,6 @@ static int __init arm_v7s_do_selftests(void)
 			return __FAIL(ops);
 
 		iova += SZ_16M;
-		i++;
-		i = find_next_bit(&cfg.pgsize_bitmap, BITS_PER_LONG, i);
 		loopnr++;
 	}
 
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index f5c90e1366ce..a40ce3406fef 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -916,7 +916,7 @@ static void dummy_tlb_sync(void *cookie)
 	WARN_ON(cookie != cfg_cookie);
 }
 
-static struct iommu_gather_ops dummy_tlb_ops __initdata = {
+static const struct iommu_gather_ops dummy_tlb_ops __initconst = {
 	.tlb_flush_all	= dummy_tlb_flush_all,
 	.tlb_add_flush	= dummy_tlb_add_flush,
 	.tlb_sync	= dummy_tlb_sync,
@@ -980,8 +980,7 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
 		 * Distinct mappings of different granule sizes.
 		 */
 		iova = 0;
-		j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG);
-		while (j != BITS_PER_LONG) {
+		for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
 			size = 1UL << j;
 
 			if (ops->map(ops, iova, iova, size, IOMMU_READ |
@@ -999,8 +998,6 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
 				return __FAIL(ops, i);
 
 			iova += SZ_1G;
-			j++;
-			j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j);
 		}
 
 		/* Partial unmap */
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 9a2f1960873b..dbe7f653bb7c 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -552,6 +552,19 @@ struct iommu_group *iommu_group_get(struct device *dev)
 EXPORT_SYMBOL_GPL(iommu_group_get);
 
 /**
+ * iommu_group_ref_get - Increment reference on a group
+ * @group: the group to use, must not be NULL
+ *
+ * This function is called by iommu drivers to take additional references on an
+ * existing group.  Returns the given group for convenience.
+ */
+struct iommu_group *iommu_group_ref_get(struct iommu_group *group)
+{
+	kobject_get(group->devices_kobj);
+	return group;
+}
+
+/**
  * iommu_group_put - Decrement group reference
  * @group: the group to use
  *
@@ -1615,6 +1628,46 @@ out:
 	return ret;
 }
 
+struct iommu_instance {
+	struct list_head list;
+	struct fwnode_handle *fwnode;
+	const struct iommu_ops *ops;
+};
+static LIST_HEAD(iommu_instance_list);
+static DEFINE_SPINLOCK(iommu_instance_lock);
+
+void iommu_register_instance(struct fwnode_handle *fwnode,
+			     const struct iommu_ops *ops)
+{
+	struct iommu_instance *iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
+
+	if (WARN_ON(!iommu))
+		return;
+
+	of_node_get(to_of_node(fwnode));
+	INIT_LIST_HEAD(&iommu->list);
+	iommu->fwnode = fwnode;
+	iommu->ops = ops;
+	spin_lock(&iommu_instance_lock);
+	list_add_tail(&iommu->list, &iommu_instance_list);
+	spin_unlock(&iommu_instance_lock);
+}
+
+const struct iommu_ops *iommu_get_instance(struct fwnode_handle *fwnode)
+{
+	struct iommu_instance *instance;
+	const struct iommu_ops *ops = NULL;
+
+	spin_lock(&iommu_instance_lock);
+	list_for_each_entry(instance, &iommu_instance_list, list)
+		if (instance->fwnode == fwnode) {
+			ops = instance->ops;
+			break;
+		}
+	spin_unlock(&iommu_instance_lock);
+	return ops;
+}
+
 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
 		      const struct iommu_ops *ops)
 {
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index e23001bfcfee..080beca0197d 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -56,7 +56,7 @@ EXPORT_SYMBOL_GPL(init_iova_domain);
 static struct rb_node *
 __get_cached_rbnode(struct iova_domain *iovad, unsigned long *limit_pfn)
 {
-	if ((*limit_pfn != iovad->dma_32bit_pfn) ||
+	if ((*limit_pfn > iovad->dma_32bit_pfn) ||
 		(iovad->cached32_node == NULL))
 		return rb_last(&iovad->rbroot);
 	else {
diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
index b12c12d74c33..1479c76ece9e 100644
--- a/drivers/iommu/mtk_iommu.c
+++ b/drivers/iommu/mtk_iommu.c
@@ -195,14 +195,14 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
 static void mtk_iommu_config(struct mtk_iommu_data *data,
 			     struct device *dev, bool enable)
 {
-	struct mtk_iommu_client_priv *head, *cur, *next;
 	struct mtk_smi_larb_iommu    *larb_mmu;
 	unsigned int                 larbid, portid;
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	int i;
 
-	head = dev->archdata.iommu;
-	list_for_each_entry_safe(cur, next, &head->client, client) {
-		larbid = MTK_M4U_TO_LARB(cur->mtk_m4u_id);
-		portid = MTK_M4U_TO_PORT(cur->mtk_m4u_id);
+	for (i = 0; i < fwspec->num_ids; ++i) {
+		larbid = MTK_M4U_TO_LARB(fwspec->ids[i]);
+		portid = MTK_M4U_TO_PORT(fwspec->ids[i]);
 		larb_mmu = &data->smi_imu.larb_imu[larbid];
 
 		dev_dbg(dev, "%s iommu port: %d\n",
@@ -282,14 +282,12 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain,
 				   struct device *dev)
 {
 	struct mtk_iommu_domain *dom = to_mtk_domain(domain);
-	struct mtk_iommu_client_priv *priv = dev->archdata.iommu;
-	struct mtk_iommu_data *data;
+	struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
 	int ret;
 
-	if (!priv)
+	if (!data)
 		return -ENODEV;
 
-	data = dev_get_drvdata(priv->m4udev);
 	if (!data->m4u_dom) {
 		data->m4u_dom = dom;
 		ret = mtk_iommu_domain_finalise(data);
@@ -310,13 +308,11 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain,
 static void mtk_iommu_detach_device(struct iommu_domain *domain,
 				    struct device *dev)
 {
-	struct mtk_iommu_client_priv *priv = dev->archdata.iommu;
-	struct mtk_iommu_data *data;
+	struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
 
-	if (!priv)
+	if (!data)
 		return;
 
-	data = dev_get_drvdata(priv->m4udev);
 	mtk_iommu_config(data, dev, false);
 }
 
@@ -366,8 +362,8 @@ static int mtk_iommu_add_device(struct device *dev)
 {
 	struct iommu_group *group;
 
-	if (!dev->archdata.iommu) /* Not a iommu client device */
-		return -ENODEV;
+	if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops)
+		return -ENODEV; /* Not a iommu client device */
 
 	group = iommu_group_get_for_dev(dev);
 	if (IS_ERR(group))
@@ -379,44 +375,33 @@ static int mtk_iommu_add_device(struct device *dev)
 
 static void mtk_iommu_remove_device(struct device *dev)
 {
-	struct mtk_iommu_client_priv *head, *cur, *next;
-
-	head = dev->archdata.iommu;
-	if (!head)
+	if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops)
 		return;
 
-	list_for_each_entry_safe(cur, next, &head->client, client) {
-		list_del(&cur->client);
-		kfree(cur);
-	}
-	kfree(head);
-	dev->archdata.iommu = NULL;
-
 	iommu_group_remove_device(dev);
+	iommu_fwspec_free(dev);
 }
 
 static struct iommu_group *mtk_iommu_device_group(struct device *dev)
 {
-	struct mtk_iommu_data *data;
-	struct mtk_iommu_client_priv *priv;
+	struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
 
-	priv = dev->archdata.iommu;
-	if (!priv)
+	if (!data)
 		return ERR_PTR(-ENODEV);
 
 	/* All the client devices are in the same m4u iommu-group */
-	data = dev_get_drvdata(priv->m4udev);
 	if (!data->m4u_group) {
 		data->m4u_group = iommu_group_alloc();
 		if (IS_ERR(data->m4u_group))
 			dev_err(dev, "Failed to allocate M4U IOMMU group\n");
+	} else {
+		iommu_group_ref_get(data->m4u_group);
 	}
 	return data->m4u_group;
 }
 
 static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
 {
-	struct mtk_iommu_client_priv *head, *priv, *next;
 	struct platform_device *m4updev;
 
 	if (args->args_count != 1) {
@@ -425,38 +410,16 @@ static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
 		return -EINVAL;
 	}
 
-	if (!dev->archdata.iommu) {
+	if (!dev->iommu_fwspec->iommu_priv) {
 		/* Get the m4u device */
 		m4updev = of_find_device_by_node(args->np);
 		if (WARN_ON(!m4updev))
 			return -EINVAL;
 
-		head = kzalloc(sizeof(*head), GFP_KERNEL);
-		if (!head)
-			return -ENOMEM;
-
-		dev->archdata.iommu = head;
-		INIT_LIST_HEAD(&head->client);
-		head->m4udev = &m4updev->dev;
-	} else {
-		head = dev->archdata.iommu;
+		dev->iommu_fwspec->iommu_priv = platform_get_drvdata(m4updev);
 	}
 
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		goto err_free_mem;
-
-	priv->mtk_m4u_id = args->args[0];
-	list_add_tail(&priv->client, &head->client);
-
-	return 0;
-
-err_free_mem:
-	list_for_each_entry_safe(priv, next, &head->client, client)
-		kfree(priv);
-	kfree(head);
-	dev->archdata.iommu = NULL;
-	return -ENOMEM;
+	return iommu_fwspec_add_ids(dev, args->args, 1);
 }
 
 static struct iommu_ops mtk_iommu_ops = {
@@ -583,17 +546,19 @@ static int mtk_iommu_probe(struct platform_device *pdev)
 			continue;
 
 		plarbdev = of_find_device_by_node(larbnode);
-		of_node_put(larbnode);
 		if (!plarbdev) {
 			plarbdev = of_platform_device_create(
 						larbnode, NULL,
 						platform_bus_type.dev_root);
-			if (!plarbdev)
+			if (!plarbdev) {
+				of_node_put(larbnode);
 				return -EPROBE_DEFER;
+			}
 		}
 		data->smi_imu.larb_imu[i].dev = &plarbdev->dev;
 
-		component_match_add(dev, &match, compare_of, larbnode);
+		component_match_add_release(dev, &match, release_of,
+					    compare_of, larbnode);
 	}
 
 	platform_set_drvdata(pdev, data);
diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h
index 3dab13b4a211..50177f738e4e 100644
--- a/drivers/iommu/mtk_iommu.h
+++ b/drivers/iommu/mtk_iommu.h
@@ -34,12 +34,6 @@ struct mtk_iommu_suspend_reg {
 	u32				int_main_control;
 };
 
-struct mtk_iommu_client_priv {
-	struct list_head		client;
-	unsigned int			mtk_m4u_id;
-	struct device			*m4udev;
-};
-
 struct mtk_iommu_domain;
 
 struct mtk_iommu_data {
@@ -60,6 +54,11 @@ static inline int compare_of(struct device *dev, void *data)
 	return dev->of_node == data;
 }
 
+static inline void release_of(struct device *dev, void *data)
+{
+	of_node_put(data);
+}
+
 static inline int mtk_iommu_bind(struct device *dev)
 {
 	struct mtk_iommu_data *data = dev_get_drvdata(dev);
diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
index b8aeb0768483..19e010083408 100644
--- a/drivers/iommu/mtk_iommu_v1.c
+++ b/drivers/iommu/mtk_iommu_v1.c
@@ -204,14 +204,14 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
 static void mtk_iommu_config(struct mtk_iommu_data *data,
 			     struct device *dev, bool enable)
 {
-	struct mtk_iommu_client_priv *head, *cur, *next;
 	struct mtk_smi_larb_iommu    *larb_mmu;
 	unsigned int                 larbid, portid;
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	int i;
 
-	head = dev->archdata.iommu;
-	list_for_each_entry_safe(cur, next, &head->client, client) {
-		larbid = mt2701_m4u_to_larb(cur->mtk_m4u_id);
-		portid = mt2701_m4u_to_port(cur->mtk_m4u_id);
+	for (i = 0; i < fwspec->num_ids; ++i) {
+		larbid = mt2701_m4u_to_larb(fwspec->ids[i]);
+		portid = mt2701_m4u_to_port(fwspec->ids[i]);
 		larb_mmu = &data->smi_imu.larb_imu[larbid];
 
 		dev_dbg(dev, "%s iommu port: %d\n",
@@ -271,14 +271,12 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain,
 				   struct device *dev)
 {
 	struct mtk_iommu_domain *dom = to_mtk_domain(domain);
-	struct mtk_iommu_client_priv *priv = dev->archdata.iommu;
-	struct mtk_iommu_data *data;
+	struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
 	int ret;
 
-	if (!priv)
+	if (!data)
 		return -ENODEV;
 
-	data = dev_get_drvdata(priv->m4udev);
 	if (!data->m4u_dom) {
 		data->m4u_dom = dom;
 		ret = mtk_iommu_domain_finalise(data);
@@ -295,13 +293,11 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain,
 static void mtk_iommu_detach_device(struct iommu_domain *domain,
 				    struct device *dev)
 {
-	struct mtk_iommu_client_priv *priv = dev->archdata.iommu;
-	struct mtk_iommu_data *data;
+	struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
 
-	if (!priv)
+	if (!data)
 		return;
 
-	data = dev_get_drvdata(priv->m4udev);
 	mtk_iommu_config(data, dev, false);
 }
 
@@ -366,6 +362,8 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
 	return pa;
 }
 
+static struct iommu_ops mtk_iommu_ops;
+
 /*
  * MTK generation one iommu HW only support one iommu domain, and all the client
  * sharing the same iova address space.
@@ -373,7 +371,7 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
 static int mtk_iommu_create_mapping(struct device *dev,
 				    struct of_phandle_args *args)
 {
-	struct mtk_iommu_client_priv *head, *priv, *next;
+	struct mtk_iommu_data *data;
 	struct platform_device *m4updev;
 	struct dma_iommu_mapping *mtk_mapping;
 	struct device *m4udev;
@@ -385,41 +383,37 @@ static int mtk_iommu_create_mapping(struct device *dev,
 		return -EINVAL;
 	}
 
-	if (!dev->archdata.iommu) {
+	if (!dev->iommu_fwspec) {
+		ret = iommu_fwspec_init(dev, &args->np->fwnode, &mtk_iommu_ops);
+		if (ret)
+			return ret;
+	} else if (dev->iommu_fwspec->ops != &mtk_iommu_ops) {
+		return -EINVAL;
+	}
+
+	if (!dev->iommu_fwspec->iommu_priv) {
 		/* Get the m4u device */
 		m4updev = of_find_device_by_node(args->np);
 		if (WARN_ON(!m4updev))
 			return -EINVAL;
 
-		head = kzalloc(sizeof(*head), GFP_KERNEL);
-		if (!head)
-			return -ENOMEM;
-
-		dev->archdata.iommu = head;
-		INIT_LIST_HEAD(&head->client);
-		head->m4udev = &m4updev->dev;
-	} else {
-		head = dev->archdata.iommu;
+		dev->iommu_fwspec->iommu_priv = platform_get_drvdata(m4updev);
 	}
 
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv) {
-		ret = -ENOMEM;
-		goto err_free_mem;
-	}
-	priv->mtk_m4u_id = args->args[0];
-	list_add_tail(&priv->client, &head->client);
+	ret = iommu_fwspec_add_ids(dev, args->args, 1);
+	if (ret)
+		return ret;
 
-	m4udev = head->m4udev;
+	data = dev->iommu_fwspec->iommu_priv;
+	m4udev = data->dev;
 	mtk_mapping = m4udev->archdata.iommu;
 	if (!mtk_mapping) {
 		/* MTK iommu support 4GB iova address space. */
 		mtk_mapping = arm_iommu_create_mapping(&platform_bus_type,
 						0, 1ULL << 32);
-		if (IS_ERR(mtk_mapping)) {
-			ret = PTR_ERR(mtk_mapping);
-			goto err_free_mem;
-		}
+		if (IS_ERR(mtk_mapping))
+			return PTR_ERR(mtk_mapping);
+
 		m4udev->archdata.iommu = mtk_mapping;
 	}
 
@@ -432,11 +426,6 @@ static int mtk_iommu_create_mapping(struct device *dev,
 err_release_mapping:
 	arm_iommu_release_mapping(mtk_mapping);
 	m4udev->archdata.iommu = NULL;
-err_free_mem:
-	list_for_each_entry_safe(priv, next, &head->client, client)
-		kfree(priv);
-	kfree(head);
-	dev->archdata.iommu = NULL;
 	return ret;
 }
 
@@ -458,8 +447,8 @@ static int mtk_iommu_add_device(struct device *dev)
 		of_node_put(iommu_spec.np);
 	}
 
-	if (!dev->archdata.iommu) /* Not a iommu client device */
-		return -ENODEV;
+	if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops)
+		return -ENODEV; /* Not a iommu client device */
 
 	group = iommu_group_get_for_dev(dev);
 	if (IS_ERR(group))
@@ -471,37 +460,27 @@ static int mtk_iommu_add_device(struct device *dev)
 
 static void mtk_iommu_remove_device(struct device *dev)
 {
-	struct mtk_iommu_client_priv *head, *cur, *next;
-
-	head = dev->archdata.iommu;
-	if (!head)
+	if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops)
 		return;
 
-	list_for_each_entry_safe(cur, next, &head->client, client) {
-		list_del(&cur->client);
-		kfree(cur);
-	}
-	kfree(head);
-	dev->archdata.iommu = NULL;
-
 	iommu_group_remove_device(dev);
+	iommu_fwspec_free(dev);
 }
 
 static struct iommu_group *mtk_iommu_device_group(struct device *dev)
 {
-	struct mtk_iommu_data *data;
-	struct mtk_iommu_client_priv *priv;
+	struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
 
-	priv = dev->archdata.iommu;
-	if (!priv)
+	if (!data)
 		return ERR_PTR(-ENODEV);
 
 	/* All the client devices are in the same m4u iommu-group */
-	data = dev_get_drvdata(priv->m4udev);
 	if (!data->m4u_group) {
 		data->m4u_group = iommu_group_alloc();
 		if (IS_ERR(data->m4u_group))
 			dev_err(dev, "Failed to allocate M4U IOMMU group\n");
+	} else {
+		iommu_group_ref_get(data->m4u_group);
 	}
 	return data->m4u_group;
 }
@@ -624,17 +603,19 @@ static int mtk_iommu_probe(struct platform_device *pdev)
 			continue;
 
 		plarbdev = of_find_device_by_node(larb_spec.np);
-		of_node_put(larb_spec.np);
 		if (!plarbdev) {
 			plarbdev = of_platform_device_create(
 						larb_spec.np, NULL,
 						platform_bus_type.dev_root);
-			if (!plarbdev)
+			if (!plarbdev) {
+				of_node_put(larb_spec.np);
 				return -EPROBE_DEFER;
+			}
 		}
 
 		data->smi_imu.larb_imu[larb_nr].dev = &plarbdev->dev;
-		component_match_add(dev, &match, compare_of, larb_spec.np);
+		component_match_add_release(dev, &match, release_of,
+					    compare_of, larb_spec.np);
 		larb_nr++;
 	}
 
diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
index 5b82862f571f..0f57ddc4ecc2 100644
--- a/drivers/iommu/of_iommu.c
+++ b/drivers/iommu/of_iommu.c
@@ -96,45 +96,6 @@ int of_get_dma_window(struct device_node *dn, const char *prefix, int index,
 }
 EXPORT_SYMBOL_GPL(of_get_dma_window);
 
-struct of_iommu_node {
-	struct list_head list;
-	struct device_node *np;
-	const struct iommu_ops *ops;
-};
-static LIST_HEAD(of_iommu_list);
-static DEFINE_SPINLOCK(of_iommu_lock);
-
-void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops)
-{
-	struct of_iommu_node *iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
-
-	if (WARN_ON(!iommu))
-		return;
-
-	of_node_get(np);
-	INIT_LIST_HEAD(&iommu->list);
-	iommu->np = np;
-	iommu->ops = ops;
-	spin_lock(&of_iommu_lock);
-	list_add_tail(&iommu->list, &of_iommu_list);
-	spin_unlock(&of_iommu_lock);
-}
-
-const struct iommu_ops *of_iommu_get_ops(struct device_node *np)
-{
-	struct of_iommu_node *node;
-	const struct iommu_ops *ops = NULL;
-
-	spin_lock(&of_iommu_lock);
-	list_for_each_entry(node, &of_iommu_list, list)
-		if (node->np == np) {
-			ops = node->ops;
-			break;
-		}
-	spin_unlock(&of_iommu_lock);
-	return ops;
-}
-
 static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data)
 {
 	struct of_phandle_args *iommu_spec = data;
diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index 3b44b1d82f3b..179e636a4d91 100644
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -8,7 +8,6 @@
 #include <linux/pci.h>
 #include <linux/iommu.h>
 #include <linux/iommu-helper.h>
-#include <linux/pci.h>
 #include <linux/sizes.h>
 #include <asm/pci_dma.h>
 
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 19d642eae096..26e1d7fafb1e 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -120,11 +120,10 @@ static void gic_redist_wait_for_rwp(void)
 }
 
 #ifdef CONFIG_ARM64
-static DEFINE_STATIC_KEY_FALSE(is_cavium_thunderx);
 
 static u64 __maybe_unused gic_read_iar(void)
 {
-	if (static_branch_unlikely(&is_cavium_thunderx))
+	if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154))
 		return gic_read_iar_cavium_thunderx();
 	else
 		return gic_read_iar_common();
@@ -905,14 +904,6 @@ static const struct irq_domain_ops partition_domain_ops = {
 	.select = gic_irq_domain_select,
 };
 
-static void gicv3_enable_quirks(void)
-{
-#ifdef CONFIG_ARM64
-	if (cpus_have_cap(ARM64_WORKAROUND_CAVIUM_23154))
-		static_branch_enable(&is_cavium_thunderx);
-#endif
-}
-
 static int __init gic_init_bases(void __iomem *dist_base,
 				 struct redist_region *rdist_regs,
 				 u32 nr_redist_regions,
@@ -935,8 +926,6 @@ static int __init gic_init_bases(void __iomem *dist_base,
 	gic_data.nr_redist_regions = nr_redist_regions;
 	gic_data.redist_stride = redist_stride;
 
-	gicv3_enable_quirks();
-
 	/*
 	 * Find out how many interrupts are supported.
 	 * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI)
diff --git a/drivers/isdn/hisax/q931.c b/drivers/isdn/hisax/q931.c
index ba4beb25d872..298c8dba0321 100644
--- a/drivers/isdn/hisax/q931.c
+++ b/drivers/isdn/hisax/q931.c
@@ -855,7 +855,7 @@ struct DTag { /* Display tags */
 	{ 0x8c, "Reason" },
 	{ 0x8d, "Calling party name" },
 	{ 0x8e, "Called party name" },
-	{ 0x8f, "Orignal called name" },
+	{ 0x8f, "Original called name" },
 	{ 0x90, "Redirecting name" },
 	{ 0x91, "Connected name" },
 	{ 0x92, "Originating restrictions" },
diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig
index 7a628c6516f6..c621cbbb5768 100644
--- a/drivers/leds/Kconfig
+++ b/drivers/leds/Kconfig
@@ -645,7 +645,7 @@ config LEDS_VERSATILE
 
 config LEDS_PM8058
 	tristate "LED Support for the Qualcomm PM8058 PMIC"
-	depends on MFD_PM8921_CORE
+	depends on MFD_PM8XXX
 	depends on LEDS_CLASS
 	help
 	  Choose this option if you want to use the LED drivers in
@@ -659,6 +659,25 @@ config LEDS_MLXCPLD
 	  This option enabled support for the LEDs on the Mellanox
 	  boards. Say Y to enabled these.
 
+config LEDS_USER
+	tristate "Userspace LED support"
+	depends on LEDS_CLASS
+	help
+	  This option enables support for userspace LEDs. Say 'y' to enable this
+	  support in kernel. To compile this driver as a module, choose 'm' here:
+	  the module will be called uleds.
+
+config LEDS_NIC78BX
+	tristate "LED support for NI PXI NIC78bx devices"
+	depends on LEDS_CLASS
+	depends on X86 && ACPI
+	help
+	  This option enables support for the User1 and User2 LEDs on NI
+	  PXI NIC78bx devices.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called leds-nic78bx.
+
 comment "LED Triggers"
 source "drivers/leds/trigger/Kconfig"
 
diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile
index 3965070190f5..6b8273736478 100644
--- a/drivers/leds/Makefile
+++ b/drivers/leds/Makefile
@@ -71,9 +71,13 @@ obj-$(CONFIG_LEDS_IS31FL319X)		+= leds-is31fl319x.o
 obj-$(CONFIG_LEDS_IS31FL32XX)		+= leds-is31fl32xx.o
 obj-$(CONFIG_LEDS_PM8058)		+= leds-pm8058.o
 obj-$(CONFIG_LEDS_MLXCPLD)		+= leds-mlxcpld.o
+obj-$(CONFIG_LEDS_NIC78BX)		+= leds-nic78bx.o
 
 # LED SPI Drivers
 obj-$(CONFIG_LEDS_DAC124S085)		+= leds-dac124s085.o
 
+# LED Userspace Drivers
+obj-$(CONFIG_LEDS_USER)			+= uleds.o
+
 # LED Triggers
 obj-$(CONFIG_LEDS_TRIGGERS)		+= trigger/
diff --git a/drivers/leds/led-class.c b/drivers/leds/led-class.c
index aa84e5b37593..326ee6e925a2 100644
--- a/drivers/leds/led-class.c
+++ b/drivers/leds/led-class.c
@@ -20,6 +20,7 @@
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/timer.h>
+#include <uapi/linux/uleds.h>
 #include "leds.h"
 
 static struct class *leds_class;
@@ -187,7 +188,7 @@ static int led_classdev_next_name(const char *init_name, char *name,
  */
 int led_classdev_register(struct device *parent, struct led_classdev *led_cdev)
 {
-	char name[64];
+	char name[LED_MAX_NAME_SIZE];
 	int ret;
 
 	ret = led_classdev_next_name(led_cdev->name, name, sizeof(name));
@@ -203,6 +204,7 @@ int led_classdev_register(struct device *parent, struct led_classdev *led_cdev)
 		dev_warn(parent, "Led %s renamed to %s due to name collision",
 				led_cdev->name, dev_name(led_cdev->dev));
 
+	led_cdev->work_flags = 0;
 #ifdef CONFIG_LEDS_TRIGGERS
 	init_rwsem(&led_cdev->trigger_lock);
 #endif
diff --git a/drivers/leds/led-core.c b/drivers/leds/led-core.c
index 3bce44893021..ef1360445413 100644
--- a/drivers/leds/led-core.c
+++ b/drivers/leds/led-core.c
@@ -53,30 +53,30 @@ static void led_timer_function(unsigned long data)
 
 	if (!led_cdev->blink_delay_on || !led_cdev->blink_delay_off) {
 		led_set_brightness_nosleep(led_cdev, LED_OFF);
-		led_cdev->flags &= ~LED_BLINK_SW;
+		clear_bit(LED_BLINK_SW, &led_cdev->work_flags);
 		return;
 	}
 
-	if (led_cdev->flags & LED_BLINK_ONESHOT_STOP) {
-		led_cdev->flags &=  ~(LED_BLINK_ONESHOT_STOP | LED_BLINK_SW);
+	if (test_and_clear_bit(LED_BLINK_ONESHOT_STOP,
+			       &led_cdev->work_flags)) {
+		clear_bit(LED_BLINK_SW, &led_cdev->work_flags);
 		return;
 	}
 
 	brightness = led_get_brightness(led_cdev);
 	if (!brightness) {
 		/* Time to switch the LED on. */
-		brightness = led_cdev->blink_brightness;
+		if (test_and_clear_bit(LED_BLINK_BRIGHTNESS_CHANGE,
+					&led_cdev->work_flags))
+			brightness = led_cdev->new_blink_brightness;
+		else
+			brightness = led_cdev->blink_brightness;
 		delay = led_cdev->blink_delay_on;
 	} else {
 		/* Store the current brightness value to be able
 		 * to restore it when the delay_off period is over.
-		 * Do it only if there is no pending blink brightness
-		 * change, to avoid overwriting the new value.
 		 */
-		if (!(led_cdev->flags & LED_BLINK_BRIGHTNESS_CHANGE))
-			led_cdev->blink_brightness = brightness;
-		else
-			led_cdev->flags &= ~LED_BLINK_BRIGHTNESS_CHANGE;
+		led_cdev->blink_brightness = brightness;
 		brightness = LED_OFF;
 		delay = led_cdev->blink_delay_off;
 	}
@@ -87,13 +87,15 @@ static void led_timer_function(unsigned long data)
 	 * the final blink state so that the led is toggled each delay_on +
 	 * delay_off milliseconds in worst case.
 	 */
-	if (led_cdev->flags & LED_BLINK_ONESHOT) {
-		if (led_cdev->flags & LED_BLINK_INVERT) {
+	if (test_bit(LED_BLINK_ONESHOT, &led_cdev->work_flags)) {
+		if (test_bit(LED_BLINK_INVERT, &led_cdev->work_flags)) {
 			if (brightness)
-				led_cdev->flags |= LED_BLINK_ONESHOT_STOP;
+				set_bit(LED_BLINK_ONESHOT_STOP,
+					&led_cdev->work_flags);
 		} else {
 			if (!brightness)
-				led_cdev->flags |= LED_BLINK_ONESHOT_STOP;
+				set_bit(LED_BLINK_ONESHOT_STOP,
+					&led_cdev->work_flags);
 		}
 	}
 
@@ -106,10 +108,9 @@ static void set_brightness_delayed(struct work_struct *ws)
 		container_of(ws, struct led_classdev, set_brightness_work);
 	int ret = 0;
 
-	if (led_cdev->flags & LED_BLINK_DISABLE) {
+	if (test_and_clear_bit(LED_BLINK_DISABLE, &led_cdev->work_flags)) {
 		led_cdev->delayed_set_value = LED_OFF;
 		led_stop_software_blink(led_cdev);
-		led_cdev->flags &= ~LED_BLINK_DISABLE;
 	}
 
 	ret = __led_set_brightness(led_cdev, led_cdev->delayed_set_value);
@@ -152,7 +153,7 @@ static void led_set_software_blink(struct led_classdev *led_cdev,
 		return;
 	}
 
-	led_cdev->flags |= LED_BLINK_SW;
+	set_bit(LED_BLINK_SW, &led_cdev->work_flags);
 	mod_timer(&led_cdev->blink_timer, jiffies + 1);
 }
 
@@ -161,7 +162,7 @@ static void led_blink_setup(struct led_classdev *led_cdev,
 		     unsigned long *delay_on,
 		     unsigned long *delay_off)
 {
-	if (!(led_cdev->flags & LED_BLINK_ONESHOT) &&
+	if (!test_bit(LED_BLINK_ONESHOT, &led_cdev->work_flags) &&
 	    led_cdev->blink_set &&
 	    !led_cdev->blink_set(led_cdev, delay_on, delay_off))
 		return;
@@ -188,8 +189,8 @@ void led_blink_set(struct led_classdev *led_cdev,
 {
 	del_timer_sync(&led_cdev->blink_timer);
 
-	led_cdev->flags &= ~LED_BLINK_ONESHOT;
-	led_cdev->flags &= ~LED_BLINK_ONESHOT_STOP;
+	clear_bit(LED_BLINK_ONESHOT, &led_cdev->work_flags);
+	clear_bit(LED_BLINK_ONESHOT_STOP, &led_cdev->work_flags);
 
 	led_blink_setup(led_cdev, delay_on, delay_off);
 }
@@ -200,17 +201,17 @@ void led_blink_set_oneshot(struct led_classdev *led_cdev,
 			   unsigned long *delay_off,
 			   int invert)
 {
-	if ((led_cdev->flags & LED_BLINK_ONESHOT) &&
+	if (test_bit(LED_BLINK_ONESHOT, &led_cdev->work_flags) &&
 	     timer_pending(&led_cdev->blink_timer))
 		return;
 
-	led_cdev->flags |= LED_BLINK_ONESHOT;
-	led_cdev->flags &= ~LED_BLINK_ONESHOT_STOP;
+	set_bit(LED_BLINK_ONESHOT, &led_cdev->work_flags);
+	clear_bit(LED_BLINK_ONESHOT_STOP, &led_cdev->work_flags);
 
 	if (invert)
-		led_cdev->flags |= LED_BLINK_INVERT;
+		set_bit(LED_BLINK_INVERT, &led_cdev->work_flags);
 	else
-		led_cdev->flags &= ~LED_BLINK_INVERT;
+		clear_bit(LED_BLINK_INVERT, &led_cdev->work_flags);
 
 	led_blink_setup(led_cdev, delay_on, delay_off);
 }
@@ -221,7 +222,7 @@ void led_stop_software_blink(struct led_classdev *led_cdev)
 	del_timer_sync(&led_cdev->blink_timer);
 	led_cdev->blink_delay_on = 0;
 	led_cdev->blink_delay_off = 0;
-	led_cdev->flags &= ~LED_BLINK_SW;
+	clear_bit(LED_BLINK_SW, &led_cdev->work_flags);
 }
 EXPORT_SYMBOL_GPL(led_stop_software_blink);
 
@@ -232,18 +233,19 @@ void led_set_brightness(struct led_classdev *led_cdev,
 	 * If software blink is active, delay brightness setting
 	 * until the next timer tick.
 	 */
-	if (led_cdev->flags & LED_BLINK_SW) {
+	if (test_bit(LED_BLINK_SW, &led_cdev->work_flags)) {
 		/*
 		 * If we need to disable soft blinking delegate this to the
 		 * work queue task to avoid problems in case we are called
 		 * from hard irq context.
 		 */
 		if (brightness == LED_OFF) {
-			led_cdev->flags |= LED_BLINK_DISABLE;
+			set_bit(LED_BLINK_DISABLE, &led_cdev->work_flags);
 			schedule_work(&led_cdev->set_brightness_work);
 		} else {
-			led_cdev->flags |= LED_BLINK_BRIGHTNESS_CHANGE;
-			led_cdev->blink_brightness = brightness;
+			set_bit(LED_BLINK_BRIGHTNESS_CHANGE,
+				&led_cdev->work_flags);
+			led_cdev->new_blink_brightness = brightness;
 		}
 		return;
 	}
diff --git a/drivers/leds/leds-cobalt-raq.c b/drivers/leds/leds-cobalt-raq.c
index b316df4a8c1e..8d066facdc73 100644
--- a/drivers/leds/leds-cobalt-raq.c
+++ b/drivers/leds/leds-cobalt-raq.c
@@ -115,8 +115,4 @@ static struct platform_driver cobalt_raq_led_driver = {
 	},
 };
 
-static int __init cobalt_raq_led_init(void)
-{
-	return platform_driver_register(&cobalt_raq_led_driver);
-}
-device_initcall(cobalt_raq_led_init);
+builtin_platform_driver(cobalt_raq_led_driver);
diff --git a/drivers/leds/leds-lp3952.c b/drivers/leds/leds-lp3952.c
index a73c8ff08530..4847e89883a7 100644
--- a/drivers/leds/leds-lp3952.c
+++ b/drivers/leds/leds-lp3952.c
@@ -274,6 +274,7 @@ static const struct i2c_device_id lp3952_id[] = {
 	{LP3952_NAME, 0},
 	{}
 };
+MODULE_DEVICE_TABLE(i2c, lp3952_id);
 
 #ifdef CONFIG_ACPI
 static const struct acpi_device_id lp3952_acpi_match[] = {
diff --git a/drivers/leds/leds-mc13783.c b/drivers/leds/leds-mc13783.c
index a2e4c1792e17..2421cf104991 100644
--- a/drivers/leds/leds-mc13783.c
+++ b/drivers/leds/leds-mc13783.c
@@ -84,8 +84,9 @@ static int mc13xxx_led_set(struct led_classdev *led_cdev,
 	case MC13892_LED_MD:
 	case MC13892_LED_AD:
 	case MC13892_LED_KP:
-		reg = (led->id - MC13892_LED_MD) / 2;
-		shift = 3 + (led->id - MC13892_LED_MD) * 12;
+		off = led->id - MC13892_LED_MD;
+		reg = off / 2;
+		shift = 3 + (off - reg * 2) * 12;
 		break;
 	case MC13892_LED_R:
 	case MC13892_LED_G:
diff --git a/drivers/leds/leds-mlxcpld.c b/drivers/leds/leds-mlxcpld.c
index 197ab9b29a9c..281482e1d50f 100644
--- a/drivers/leds/leds-mlxcpld.c
+++ b/drivers/leds/leds-mlxcpld.c
@@ -400,6 +400,9 @@ static int __init mlxcpld_led_init(void)
 	struct platform_device *pdev;
 	int err;
 
+	if (!dmi_match(DMI_CHASSIS_VENDOR, "Mellanox Technologies Ltd."))
+		return -ENODEV;
+
 	pdev = platform_device_register_simple(KBUILD_MODNAME, -1, NULL, 0);
 	if (IS_ERR(pdev)) {
 		pr_err("Device allocation failed\n");
@@ -426,5 +429,5 @@ module_exit(mlxcpld_led_exit);
 
 MODULE_AUTHOR("Vadim Pasternak <vadimp@mellanox.com>");
 MODULE_DESCRIPTION("Mellanox board LED driver");
-MODULE_LICENSE("GPL v2");
+MODULE_LICENSE("Dual BSD/GPL");
 MODULE_ALIAS("platform:leds_mlxcpld");
diff --git a/drivers/leds/leds-netxbig.c b/drivers/leds/leds-netxbig.c
index 4b88b93244be..f48b1aed9b4e 100644
--- a/drivers/leds/leds-netxbig.c
+++ b/drivers/leds/leds-netxbig.c
@@ -534,6 +534,7 @@ static const struct of_device_id of_netxbig_leds_match[] = {
 	{ .compatible = "lacie,netxbig-leds", },
 	{},
 };
+MODULE_DEVICE_TABLE(of, of_netxbig_leds_match);
 #else
 static inline int
 netxbig_leds_get_of_pdata(struct device *dev,
diff --git a/drivers/leds/leds-nic78bx.c b/drivers/leds/leds-nic78bx.c
new file mode 100644
index 000000000000..8d69e2b74a27
--- /dev/null
+++ b/drivers/leds/leds-nic78bx.c
@@ -0,0 +1,209 @@
+/*
+ * Copyright (C) 2016 National Instruments Corp.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/acpi.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+
+#define NIC78BX_USER1_LED_MASK		0x3
+#define NIC78BX_USER1_GREEN_LED		BIT(0)
+#define NIC78BX_USER1_YELLOW_LED	BIT(1)
+
+#define NIC78BX_USER2_LED_MASK		0xC
+#define NIC78BX_USER2_GREEN_LED		BIT(2)
+#define NIC78BX_USER2_YELLOW_LED	BIT(3)
+
+#define NIC78BX_LOCK_REG_OFFSET		1
+#define NIC78BX_LOCK_VALUE		0xA5
+#define NIC78BX_UNLOCK_VALUE		0x5A
+
+#define NIC78BX_USER_LED_IO_SIZE	2
+
+struct nic78bx_led_data {
+	u16 io_base;
+	spinlock_t lock;
+	struct platform_device *pdev;
+};
+
+struct nic78bx_led {
+	u8 bit;
+	u8 mask;
+	struct nic78bx_led_data *data;
+	struct led_classdev cdev;
+};
+
+static inline struct nic78bx_led *to_nic78bx_led(struct led_classdev *cdev)
+{
+	return container_of(cdev, struct nic78bx_led, cdev);
+}
+
+static void nic78bx_brightness_set(struct led_classdev *cdev,
+				  enum led_brightness brightness)
+{
+	struct nic78bx_led *nled = to_nic78bx_led(cdev);
+	unsigned long flags;
+	u8 value;
+
+	spin_lock_irqsave(&nled->data->lock, flags);
+	value = inb(nled->data->io_base);
+
+	if (brightness) {
+		value &= ~nled->mask;
+		value |= nled->bit;
+	} else {
+		value &= ~nled->bit;
+	}
+
+	outb(value, nled->data->io_base);
+	spin_unlock_irqrestore(&nled->data->lock, flags);
+}
+
+static enum led_brightness nic78bx_brightness_get(struct led_classdev *cdev)
+{
+	struct nic78bx_led *nled = to_nic78bx_led(cdev);
+	unsigned long flags;
+	u8 value;
+
+	spin_lock_irqsave(&nled->data->lock, flags);
+	value = inb(nled->data->io_base);
+	spin_unlock_irqrestore(&nled->data->lock, flags);
+
+	return (value & nled->bit) ? 1 : LED_OFF;
+}
+
+static struct nic78bx_led nic78bx_leds[] = {
+	{
+		.bit = NIC78BX_USER1_GREEN_LED,
+		.mask = NIC78BX_USER1_LED_MASK,
+		.cdev = {
+			.name = "nilrt:green:user1",
+			.max_brightness = 1,
+			.brightness_set = nic78bx_brightness_set,
+			.brightness_get = nic78bx_brightness_get,
+		}
+	},
+	{
+		.bit = NIC78BX_USER1_YELLOW_LED,
+		.mask = NIC78BX_USER1_LED_MASK,
+		.cdev = {
+			.name = "nilrt:yellow:user1",
+			.max_brightness = 1,
+			.brightness_set = nic78bx_brightness_set,
+			.brightness_get = nic78bx_brightness_get,
+		}
+	},
+	{
+		.bit = NIC78BX_USER2_GREEN_LED,
+		.mask = NIC78BX_USER2_LED_MASK,
+		.cdev = {
+			.name = "nilrt:green:user2",
+			.max_brightness = 1,
+			.brightness_set = nic78bx_brightness_set,
+			.brightness_get = nic78bx_brightness_get,
+		}
+	},
+	{
+		.bit = NIC78BX_USER2_YELLOW_LED,
+		.mask = NIC78BX_USER2_LED_MASK,
+		.cdev = {
+			.name = "nilrt:yellow:user2",
+			.max_brightness = 1,
+			.brightness_set = nic78bx_brightness_set,
+			.brightness_get = nic78bx_brightness_get,
+		}
+	}
+};
+
+static int nic78bx_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct nic78bx_led_data *led_data;
+	struct resource *io_rc;
+	int ret, i;
+
+	led_data = devm_kzalloc(dev, sizeof(*led_data), GFP_KERNEL);
+	if (!led_data)
+		return -ENOMEM;
+
+	led_data->pdev = pdev;
+	platform_set_drvdata(pdev, led_data);
+
+	io_rc = platform_get_resource(pdev, IORESOURCE_IO, 0);
+	if (!io_rc) {
+		dev_err(dev, "missing IO resources\n");
+		return -EINVAL;
+	}
+
+	if (resource_size(io_rc) < NIC78BX_USER_LED_IO_SIZE) {
+		dev_err(dev, "IO region too small\n");
+		return -EINVAL;
+	}
+
+	if (!devm_request_region(dev, io_rc->start, resource_size(io_rc),
+				 KBUILD_MODNAME)) {
+		dev_err(dev, "failed to get IO region\n");
+		return -EBUSY;
+	}
+
+	led_data->io_base = io_rc->start;
+	spin_lock_init(&led_data->lock);
+
+	for (i = 0; i < ARRAY_SIZE(nic78bx_leds); i++) {
+		nic78bx_leds[i].data = led_data;
+
+		ret = devm_led_classdev_register(dev, &nic78bx_leds[i].cdev);
+		if (ret)
+			return ret;
+	}
+
+	/* Unlock LED register */
+	outb(NIC78BX_UNLOCK_VALUE,
+	     led_data->io_base + NIC78BX_LOCK_REG_OFFSET);
+
+	return ret;
+}
+
+static int nic78bx_remove(struct platform_device *pdev)
+{
+	struct nic78bx_led_data *led_data = platform_get_drvdata(pdev);
+
+	/* Lock LED register */
+	outb(NIC78BX_LOCK_VALUE,
+	     led_data->io_base + NIC78BX_LOCK_REG_OFFSET);
+
+	return 0;
+}
+
+static const struct acpi_device_id led_device_ids[] = {
+	{"NIC78B3", 0},
+	{"", 0},
+};
+MODULE_DEVICE_TABLE(acpi, led_device_ids);
+
+static struct platform_driver led_driver = {
+	.probe = nic78bx_probe,
+	.remove = nic78bx_remove,
+	.driver = {
+		.name = KBUILD_MODNAME,
+		.acpi_match_table = ACPI_PTR(led_device_ids),
+	},
+};
+
+module_platform_driver(led_driver);
+
+MODULE_DESCRIPTION("National Instruments PXI User LEDs driver");
+MODULE_AUTHOR("Hui Chun Ong <hui.chun.ong@ni.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/leds/leds-pca9532.c b/drivers/leds/leds-pca9532.c
index 09a7cffbc46f..06e63106ae1e 100644
--- a/drivers/leds/leds-pca9532.c
+++ b/drivers/leds/leds-pca9532.c
@@ -369,7 +369,7 @@ static int pca9532_configure(struct i2c_client *client,
 			led->state = pled->state;
 			led->name = pled->name;
 			led->ldev.name = led->name;
-			led->ldev.default_trigger = led->default_trigger;
+			led->ldev.default_trigger = pled->default_trigger;
 			led->ldev.brightness = LED_OFF;
 			led->ldev.brightness_set_blocking =
 						pca9532_set_brightness;
diff --git a/drivers/leds/leds-pca955x.c b/drivers/leds/leds-pca955x.c
index 840401ae9a4e..78a7ce816a47 100644
--- a/drivers/leds/leds-pca955x.c
+++ b/drivers/leds/leds-pca955x.c
@@ -40,6 +40,7 @@
  *  bits the chip supports.
  */
 
+#include <linux/acpi.h>
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/string.h>
@@ -100,6 +101,15 @@ static const struct i2c_device_id pca955x_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, pca955x_id);
 
+static const struct acpi_device_id pca955x_acpi_ids[] = {
+	{ "PCA9550",  pca9550 },
+	{ "PCA9551",  pca9551 },
+	{ "PCA9552",  pca9552 },
+	{ "PCA9553",  pca9553 },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, pca955x_acpi_ids);
+
 struct pca955x {
 	struct mutex lock;
 	struct pca955x_led *leds;
@@ -250,7 +260,16 @@ static int pca955x_probe(struct i2c_client *client,
 	struct led_platform_data *pdata;
 	int i, err;
 
-	chip = &pca955x_chipdefs[id->driver_data];
+	if (id) {
+		chip = &pca955x_chipdefs[id->driver_data];
+	} else {
+		const struct acpi_device_id *acpi_id;
+
+		acpi_id = acpi_match_device(pca955x_acpi_ids, &client->dev);
+		if (!acpi_id)
+			return -ENODEV;
+		chip = &pca955x_chipdefs[acpi_id->driver_data];
+	}
 	adapter = to_i2c_adapter(client->dev.parent);
 	pdata = dev_get_platdata(&client->dev);
 
@@ -264,7 +283,7 @@ static int pca955x_probe(struct i2c_client *client,
 
 	dev_info(&client->dev, "leds-pca955x: Using %s %d-bit LED driver at "
 			"slave address 0x%02x\n",
-			id->name, chip->bits, client->addr);
+			client->name, chip->bits, client->addr);
 
 	if (!i2c_check_functionality(adapter, I2C_FUNC_I2C))
 		return -EIO;
@@ -358,6 +377,7 @@ static int pca955x_remove(struct i2c_client *client)
 static struct i2c_driver pca955x_driver = {
 	.driver = {
 		.name	= "leds-pca955x",
+		.acpi_match_table = ACPI_PTR(pca955x_acpi_ids),
 	},
 	.probe	= pca955x_probe,
 	.remove	= pca955x_remove,
diff --git a/drivers/leds/leds-pca963x.c b/drivers/leds/leds-pca963x.c
index 407eba11e187..ded1e4dac36a 100644
--- a/drivers/leds/leds-pca963x.c
+++ b/drivers/leds/leds-pca963x.c
@@ -25,6 +25,7 @@
  * or by adding the 'nxp,hw-blink' property to the DTS.
  */
 
+#include <linux/acpi.h>
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/string.h>
@@ -59,6 +60,7 @@ struct pca963x_chipdef {
 	u8			grpfreq;
 	u8			ledout_base;
 	int			n_leds;
+	unsigned int		scaling;
 };
 
 static struct pca963x_chipdef pca963x_chipdefs[] = {
@@ -95,6 +97,15 @@ static const struct i2c_device_id pca963x_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, pca963x_id);
 
+static const struct acpi_device_id pca963x_acpi_ids[] = {
+	{ "PCA9632", pca9633 },
+	{ "PCA9633", pca9633 },
+	{ "PCA9634", pca9634 },
+	{ "PCA9635", pca9635 },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, pca963x_acpi_ids);
+
 struct pca963x_led;
 
 struct pca963x {
@@ -102,6 +113,7 @@ struct pca963x {
 	struct mutex mutex;
 	struct i2c_client *client;
 	struct pca963x_led *leds;
+	unsigned long leds_on;
 };
 
 struct pca963x_led {
@@ -123,7 +135,6 @@ static int pca963x_brightness(struct pca963x_led *pca963x,
 	u8 mask = 0x3 << shift;
 	int ret;
 
-	mutex_lock(&pca963x->chip->mutex);
 	ledout = i2c_smbus_read_byte_data(pca963x->chip->client, ledout_addr);
 	switch (brightness) {
 	case LED_FULL:
@@ -140,14 +151,13 @@ static int pca963x_brightness(struct pca963x_led *pca963x,
 			PCA963X_PWM_BASE + pca963x->led_num,
 			brightness);
 		if (ret < 0)
-			goto unlock;
+			return ret;
 		ret = i2c_smbus_write_byte_data(pca963x->chip->client,
 			ledout_addr,
 			(ledout & ~mask) | (PCA963X_LED_PWM << shift));
 		break;
 	}
-unlock:
-	mutex_unlock(&pca963x->chip->mutex);
+
 	return ret;
 }
 
@@ -179,14 +189,49 @@ static void pca963x_blink(struct pca963x_led *pca963x)
 	mutex_unlock(&pca963x->chip->mutex);
 }
 
+static int pca963x_power_state(struct pca963x_led *pca963x)
+{
+	unsigned long *leds_on = &pca963x->chip->leds_on;
+	unsigned long cached_leds = pca963x->chip->leds_on;
+
+	if (pca963x->led_cdev.brightness)
+		set_bit(pca963x->led_num, leds_on);
+	else
+		clear_bit(pca963x->led_num, leds_on);
+
+	if (!(*leds_on) != !cached_leds)
+		return i2c_smbus_write_byte_data(pca963x->chip->client,
+			PCA963X_MODE1, *leds_on ? 0 : BIT(4));
+
+	return 0;
+}
+
 static int pca963x_led_set(struct led_classdev *led_cdev,
 	enum led_brightness value)
 {
 	struct pca963x_led *pca963x;
+	int ret;
 
 	pca963x = container_of(led_cdev, struct pca963x_led, led_cdev);
 
-	return pca963x_brightness(pca963x, value);
+	mutex_lock(&pca963x->chip->mutex);
+
+	ret = pca963x_brightness(pca963x, value);
+	if (ret < 0)
+		goto unlock;
+	ret = pca963x_power_state(pca963x);
+
+unlock:
+	mutex_unlock(&pca963x->chip->mutex);
+	return ret;
+}
+
+static unsigned int pca963x_period_scale(struct pca963x_led *pca963x,
+	unsigned int val)
+{
+	unsigned int scaling = pca963x->chip->chipdef->scaling;
+
+	return scaling ? DIV_ROUND_CLOSEST(val * scaling, 1000) : val;
 }
 
 static int pca963x_blink_set(struct led_classdev *led_cdev,
@@ -207,14 +252,14 @@ static int pca963x_blink_set(struct led_classdev *led_cdev,
 		time_off = 500;
 	}
 
-	period = time_on + time_off;
+	period = pca963x_period_scale(pca963x, time_on + time_off);
 
 	/* If period not supported by hardware, default to someting sane. */
 	if ((period < PCA963X_BLINK_PERIOD_MIN) ||
 	    (period > PCA963X_BLINK_PERIOD_MAX)) {
 		time_on = 500;
 		time_off = 500;
-		period = time_on + time_off;
+		period = pca963x_period_scale(pca963x, 1000);
 	}
 
 	/*
@@ -222,7 +267,7 @@ static int pca963x_blink_set(struct led_classdev *led_cdev,
 	 *	(time_on / period) = (GDC / 256) ->
 	 *		GDC = ((time_on * 256) / period)
 	 */
-	gdc = (time_on * 256) / period;
+	gdc = (pca963x_period_scale(pca963x, time_on) * 256) / period;
 
 	/*
 	 * From manual: period = ((GFRQ + 1) / 24) in seconds.
@@ -294,6 +339,9 @@ pca963x_dt_init(struct i2c_client *client, struct pca963x_chipdef *chip)
 	else
 		pdata->blink_type = PCA963X_SW_BLINK;
 
+	if (of_property_read_u32(np, "nxp,period-scale", &chip->scaling))
+		chip->scaling = 1000;
+
 	return pdata;
 }
 
@@ -322,7 +370,16 @@ static int pca963x_probe(struct i2c_client *client,
 	struct pca963x_chipdef *chip;
 	int i, err;
 
-	chip = &pca963x_chipdefs[id->driver_data];
+	if (id) {
+		chip = &pca963x_chipdefs[id->driver_data];
+	} else {
+		const struct acpi_device_id *acpi_id;
+
+		acpi_id = acpi_match_device(pca963x_acpi_ids, &client->dev);
+		if (!acpi_id)
+			return -ENODEV;
+		chip = &pca963x_chipdefs[acpi_id->driver_data];
+	}
 	pdata = dev_get_platdata(&client->dev);
 
 	if (!pdata) {
@@ -391,8 +448,8 @@ static int pca963x_probe(struct i2c_client *client,
 			goto exit;
 	}
 
-	/* Disable LED all-call address and set normal mode */
-	i2c_smbus_write_byte_data(client, PCA963X_MODE1, 0x00);
+	/* Disable LED all-call address, and power down initially */
+	i2c_smbus_write_byte_data(client, PCA963X_MODE1, BIT(4));
 
 	if (pdata) {
 		/* Configure output: open-drain or totem pole (push-pull) */
@@ -426,6 +483,7 @@ static struct i2c_driver pca963x_driver = {
 	.driver = {
 		.name	= "leds-pca963x",
 		.of_match_table = of_match_ptr(of_pca963x_match),
+		.acpi_match_table = ACPI_PTR(pca963x_acpi_ids),
 	},
 	.probe	= pca963x_probe,
 	.remove	= pca963x_remove,
diff --git a/drivers/leds/trigger/ledtrig-cpu.c b/drivers/leds/trigger/ledtrig-cpu.c
index 22f0634dd3fa..9719caf7437c 100644
--- a/drivers/leds/trigger/ledtrig-cpu.c
+++ b/drivers/leds/trigger/ledtrig-cpu.c
@@ -42,7 +42,7 @@ static DEFINE_PER_CPU(struct led_trigger_cpu, cpu_trig);
  * @evt: CPU event to be emitted
  *
  * Emit a CPU event on a CPU core, which will trigger a
- * binded LED to turn on or turn off.
+ * bound LED to turn on or turn off.
  */
 void ledtrig_cpu(enum cpu_led_event ledevt)
 {
diff --git a/drivers/leds/uleds.c b/drivers/leds/uleds.c
new file mode 100644
index 000000000000..5e9e8a1fdefb
--- /dev/null
+++ b/drivers/leds/uleds.c
@@ -0,0 +1,235 @@
+/*
+ * Userspace driver for the LED subsystem
+ *
+ * Copyright (C) 2016 David Lechner <david@lechnology.com>
+ *
+ * Based on uinput.c: Aristeu Sergio Rozanski Filho <aris@cathedrallabs.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/leds.h>
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/poll.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+
+#include <uapi/linux/uleds.h>
+
+#define ULEDS_NAME	"uleds"
+
+enum uleds_state {
+	ULEDS_STATE_UNKNOWN,
+	ULEDS_STATE_REGISTERED,
+};
+
+struct uleds_device {
+	struct uleds_user_dev	user_dev;
+	struct led_classdev	led_cdev;
+	struct mutex		mutex;
+	enum uleds_state	state;
+	wait_queue_head_t	waitq;
+	int			brightness;
+	bool			new_data;
+};
+
+static struct miscdevice uleds_misc;
+
+static void uleds_brightness_set(struct led_classdev *led_cdev,
+				 enum led_brightness brightness)
+{
+	struct uleds_device *udev = container_of(led_cdev, struct uleds_device,
+						 led_cdev);
+
+	if (udev->brightness != brightness) {
+		udev->brightness = brightness;
+		udev->new_data = true;
+		wake_up_interruptible(&udev->waitq);
+	}
+}
+
+static int uleds_open(struct inode *inode, struct file *file)
+{
+	struct uleds_device *udev;
+
+	udev = kzalloc(sizeof(*udev), GFP_KERNEL);
+	if (!udev)
+		return -ENOMEM;
+
+	udev->led_cdev.name = udev->user_dev.name;
+	udev->led_cdev.brightness_set = uleds_brightness_set;
+
+	mutex_init(&udev->mutex);
+	init_waitqueue_head(&udev->waitq);
+	udev->state = ULEDS_STATE_UNKNOWN;
+
+	file->private_data = udev;
+	nonseekable_open(inode, file);
+
+	return 0;
+}
+
+static ssize_t uleds_write(struct file *file, const char __user *buffer,
+			   size_t count, loff_t *ppos)
+{
+	struct uleds_device *udev = file->private_data;
+	const char *name;
+	int ret;
+
+	if (count == 0)
+		return 0;
+
+	ret = mutex_lock_interruptible(&udev->mutex);
+	if (ret)
+		return ret;
+
+	if (udev->state == ULEDS_STATE_REGISTERED) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	if (count != sizeof(struct uleds_user_dev)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (copy_from_user(&udev->user_dev, buffer,
+			   sizeof(struct uleds_user_dev))) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	name = udev->user_dev.name;
+	if (!name[0] || !strcmp(name, ".") || !strcmp(name, "..") ||
+	    strchr(name, '/')) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (udev->user_dev.max_brightness <= 0) {
+		ret = -EINVAL;
+		goto out;
+	}
+	udev->led_cdev.max_brightness = udev->user_dev.max_brightness;
+
+	ret = devm_led_classdev_register(uleds_misc.this_device,
+					 &udev->led_cdev);
+	if (ret < 0)
+		goto out;
+
+	udev->new_data = true;
+	udev->state = ULEDS_STATE_REGISTERED;
+	ret = count;
+
+out:
+	mutex_unlock(&udev->mutex);
+
+	return ret;
+}
+
+static ssize_t uleds_read(struct file *file, char __user *buffer, size_t count,
+			  loff_t *ppos)
+{
+	struct uleds_device *udev = file->private_data;
+	ssize_t retval;
+
+	if (count < sizeof(udev->brightness))
+		return 0;
+
+	do {
+		retval = mutex_lock_interruptible(&udev->mutex);
+		if (retval)
+			return retval;
+
+		if (udev->state != ULEDS_STATE_REGISTERED) {
+			retval = -ENODEV;
+		} else if (!udev->new_data && (file->f_flags & O_NONBLOCK)) {
+			retval = -EAGAIN;
+		} else if (udev->new_data) {
+			retval = copy_to_user(buffer, &udev->brightness,
+					      sizeof(udev->brightness));
+			udev->new_data = false;
+			retval = sizeof(udev->brightness);
+		}
+
+		mutex_unlock(&udev->mutex);
+
+		if (retval)
+			break;
+
+		if (!(file->f_flags & O_NONBLOCK))
+			retval = wait_event_interruptible(udev->waitq,
+					udev->new_data ||
+					udev->state != ULEDS_STATE_REGISTERED);
+	} while (retval == 0);
+
+	return retval;
+}
+
+static unsigned int uleds_poll(struct file *file, poll_table *wait)
+{
+	struct uleds_device *udev = file->private_data;
+
+	poll_wait(file, &udev->waitq, wait);
+
+	if (udev->new_data)
+		return POLLIN | POLLRDNORM;
+
+	return 0;
+}
+
+static int uleds_release(struct inode *inode, struct file *file)
+{
+	struct uleds_device *udev = file->private_data;
+
+	if (udev->state == ULEDS_STATE_REGISTERED) {
+		udev->state = ULEDS_STATE_UNKNOWN;
+		devm_led_classdev_unregister(uleds_misc.this_device,
+					     &udev->led_cdev);
+	}
+	kfree(udev);
+
+	return 0;
+}
+
+static const struct file_operations uleds_fops = {
+	.owner		= THIS_MODULE,
+	.open		= uleds_open,
+	.release	= uleds_release,
+	.read		= uleds_read,
+	.write		= uleds_write,
+	.poll		= uleds_poll,
+	.llseek		= no_llseek,
+};
+
+static struct miscdevice uleds_misc = {
+	.fops		= &uleds_fops,
+	.minor		= MISC_DYNAMIC_MINOR,
+	.name		= ULEDS_NAME,
+};
+
+static int __init uleds_init(void)
+{
+	return misc_register(&uleds_misc);
+}
+module_init(uleds_init);
+
+static void __exit uleds_exit(void)
+{
+	misc_deregister(&uleds_misc);
+}
+module_exit(uleds_exit);
+
+MODULE_AUTHOR("David Lechner <david@lechnology.com>");
+MODULE_DESCRIPTION("Userspace driver for the LED subsystem");
+MODULE_LICENSE("GPL");
diff --git a/drivers/lightnvm/Makefile b/drivers/lightnvm/Makefile
index 1f6b6521016a..a7a0a22cf1a5 100644
--- a/drivers/lightnvm/Makefile
+++ b/drivers/lightnvm/Makefile
@@ -2,6 +2,6 @@
 # Makefile for Open-Channel SSDs.
 #
 
-obj-$(CONFIG_NVM)		:= core.o sysblk.o sysfs.o
+obj-$(CONFIG_NVM)		:= core.o sysblk.o
 obj-$(CONFIG_NVM_GENNVM) 	+= gennvm.o
 obj-$(CONFIG_NVM_RRPC)		+= rrpc.o
diff --git a/drivers/lightnvm/core.c b/drivers/lightnvm/core.c
index 1cac0f8bc0dc..02240a0b39c9 100644
--- a/drivers/lightnvm/core.c
+++ b/drivers/lightnvm/core.c
@@ -22,13 +22,11 @@
 #include <linux/types.h>
 #include <linux/sem.h>
 #include <linux/bitmap.h>
-#include <linux/module.h>
+#include <linux/moduleparam.h>
 #include <linux/miscdevice.h>
 #include <linux/lightnvm.h>
 #include <linux/sched/sysctl.h>
 
-#include "lightnvm.h"
-
 static LIST_HEAD(nvm_tgt_types);
 static DECLARE_RWSEM(nvm_tgtt_lock);
 static LIST_HEAD(nvm_mgrs);
@@ -88,8 +86,7 @@ void *nvm_dev_dma_alloc(struct nvm_dev *dev, gfp_t mem_flags,
 }
 EXPORT_SYMBOL(nvm_dev_dma_alloc);
 
-void nvm_dev_dma_free(struct nvm_dev *dev, void *addr,
-							dma_addr_t dma_handler)
+void nvm_dev_dma_free(struct nvm_dev *dev, void *addr, dma_addr_t dma_handler)
 {
 	dev->ops->dev_dma_free(dev->dma_pool, addr, dma_handler);
 }
@@ -178,38 +175,133 @@ static struct nvm_dev *nvm_find_nvm_dev(const char *name)
 	return NULL;
 }
 
-struct nvm_block *nvm_get_blk(struct nvm_dev *dev, struct nvm_lun *lun,
-							unsigned long flags)
+static void nvm_tgt_generic_to_addr_mode(struct nvm_tgt_dev *tgt_dev,
+					 struct nvm_rq *rqd)
 {
-	return dev->mt->get_blk(dev, lun, flags);
+	struct nvm_dev *dev = tgt_dev->parent;
+	int i;
+
+	if (rqd->nr_ppas > 1) {
+		for (i = 0; i < rqd->nr_ppas; i++) {
+			rqd->ppa_list[i] = dev->mt->trans_ppa(tgt_dev,
+					rqd->ppa_list[i], TRANS_TGT_TO_DEV);
+			rqd->ppa_list[i] = generic_to_dev_addr(dev,
+							rqd->ppa_list[i]);
+		}
+	} else {
+		rqd->ppa_addr = dev->mt->trans_ppa(tgt_dev, rqd->ppa_addr,
+						TRANS_TGT_TO_DEV);
+		rqd->ppa_addr = generic_to_dev_addr(dev, rqd->ppa_addr);
+	}
+}
+
+int nvm_set_bb_tbl(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas,
+								int type)
+{
+	struct nvm_rq rqd;
+	int ret;
+
+	if (nr_ppas > dev->ops->max_phys_sect) {
+		pr_err("nvm: unable to update all sysblocks atomically\n");
+		return -EINVAL;
+	}
+
+	memset(&rqd, 0, sizeof(struct nvm_rq));
+
+	nvm_set_rqd_ppalist(dev, &rqd, ppas, nr_ppas, 1);
+	nvm_generic_to_addr_mode(dev, &rqd);
+
+	ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type);
+	nvm_free_rqd_ppalist(dev, &rqd);
+	if (ret) {
+		pr_err("nvm: sysblk failed bb mark\n");
+		return -EINVAL;
+	}
+
+	return 0;
 }
-EXPORT_SYMBOL(nvm_get_blk);
+EXPORT_SYMBOL(nvm_set_bb_tbl);
 
-/* Assumes that all valid pages have already been moved on release to bm */
-void nvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk)
+int nvm_set_tgt_bb_tbl(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *ppas,
+		       int nr_ppas, int type)
 {
-	return dev->mt->put_blk(dev, blk);
+	struct nvm_dev *dev = tgt_dev->parent;
+	struct nvm_rq rqd;
+	int ret;
+
+	if (nr_ppas > dev->ops->max_phys_sect) {
+		pr_err("nvm: unable to update all blocks atomically\n");
+		return -EINVAL;
+	}
+
+	memset(&rqd, 0, sizeof(struct nvm_rq));
+
+	nvm_set_rqd_ppalist(dev, &rqd, ppas, nr_ppas, 1);
+	nvm_tgt_generic_to_addr_mode(tgt_dev, &rqd);
+
+	ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type);
+	nvm_free_rqd_ppalist(dev, &rqd);
+	if (ret) {
+		pr_err("nvm: sysblk failed bb mark\n");
+		return -EINVAL;
+	}
+
+	return 0;
 }
-EXPORT_SYMBOL(nvm_put_blk);
+EXPORT_SYMBOL(nvm_set_tgt_bb_tbl);
 
-void nvm_mark_blk(struct nvm_dev *dev, struct ppa_addr ppa, int type)
+int nvm_max_phys_sects(struct nvm_tgt_dev *tgt_dev)
 {
-	return dev->mt->mark_blk(dev, ppa, type);
+	struct nvm_dev *dev = tgt_dev->parent;
+
+	return dev->ops->max_phys_sect;
 }
-EXPORT_SYMBOL(nvm_mark_blk);
+EXPORT_SYMBOL(nvm_max_phys_sects);
 
-int nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
+int nvm_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
 {
-	return dev->mt->submit_io(dev, rqd);
+	struct nvm_dev *dev = tgt_dev->parent;
+
+	return dev->mt->submit_io(tgt_dev, rqd);
 }
 EXPORT_SYMBOL(nvm_submit_io);
 
-int nvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk)
+int nvm_erase_blk(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p, int flags)
 {
-	return dev->mt->erase_blk(dev, blk, 0);
+	struct nvm_dev *dev = tgt_dev->parent;
+
+	return dev->mt->erase_blk(tgt_dev, p, flags);
 }
 EXPORT_SYMBOL(nvm_erase_blk);
 
+int nvm_get_l2p_tbl(struct nvm_tgt_dev *tgt_dev, u64 slba, u32 nlb,
+		    nvm_l2p_update_fn *update_l2p, void *priv)
+{
+	struct nvm_dev *dev = tgt_dev->parent;
+
+	if (!dev->ops->get_l2p_tbl)
+		return 0;
+
+	return dev->ops->get_l2p_tbl(dev, slba, nlb, update_l2p, priv);
+}
+EXPORT_SYMBOL(nvm_get_l2p_tbl);
+
+int nvm_get_area(struct nvm_tgt_dev *tgt_dev, sector_t *lba, sector_t len)
+{
+	struct nvm_dev *dev = tgt_dev->parent;
+
+	return dev->mt->get_area(dev, lba, len);
+}
+EXPORT_SYMBOL(nvm_get_area);
+
+void nvm_put_area(struct nvm_tgt_dev *tgt_dev, sector_t lba)
+{
+	struct nvm_dev *dev = tgt_dev->parent;
+
+	dev->mt->put_area(dev, lba);
+}
+EXPORT_SYMBOL(nvm_put_area);
+
 void nvm_addr_to_generic_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
 {
 	int i;
@@ -241,10 +333,11 @@ EXPORT_SYMBOL(nvm_generic_to_addr_mode);
 int nvm_set_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd,
 			const struct ppa_addr *ppas, int nr_ppas, int vblk)
 {
+	struct nvm_geo *geo = &dev->geo;
 	int i, plane_cnt, pl_idx;
 	struct ppa_addr ppa;
 
-	if ((!vblk || dev->plane_mode == NVM_PLANE_SINGLE) && nr_ppas == 1) {
+	if ((!vblk || geo->plane_mode == NVM_PLANE_SINGLE) && nr_ppas == 1) {
 		rqd->nr_ppas = nr_ppas;
 		rqd->ppa_addr = ppas[0];
 
@@ -262,7 +355,7 @@ int nvm_set_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd,
 		for (i = 0; i < nr_ppas; i++)
 			rqd->ppa_list[i] = ppas[i];
 	} else {
-		plane_cnt = dev->plane_mode;
+		plane_cnt = geo->plane_mode;
 		rqd->nr_ppas *= plane_cnt;
 
 		for (i = 0; i < nr_ppas; i++) {
@@ -287,7 +380,8 @@ void nvm_free_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd)
 }
 EXPORT_SYMBOL(nvm_free_rqd_ppalist);
 
-int nvm_erase_ppa(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas)
+int nvm_erase_ppa(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas,
+								int flags)
 {
 	struct nvm_rq rqd;
 	int ret;
@@ -303,6 +397,8 @@ int nvm_erase_ppa(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas)
 
 	nvm_generic_to_addr_mode(dev, &rqd);
 
+	rqd.flags = flags;
+
 	ret = dev->ops->erase_block(dev, &rqd);
 
 	nvm_free_rqd_ppalist(dev, &rqd);
@@ -341,7 +437,7 @@ static int __nvm_submit_ppa(struct nvm_dev *dev, struct nvm_rq *rqd, int opcode,
 
 	nvm_generic_to_addr_mode(dev, rqd);
 
-	rqd->dev = dev;
+	rqd->dev = NULL;
 	rqd->opcode = opcode;
 	rqd->flags = flags;
 	rqd->bio = bio;
@@ -437,17 +533,18 @@ EXPORT_SYMBOL(nvm_submit_ppa);
  */
 int nvm_bb_tbl_fold(struct nvm_dev *dev, u8 *blks, int nr_blks)
 {
+	struct nvm_geo *geo = &dev->geo;
 	int blk, offset, pl, blktype;
 
-	if (nr_blks != dev->blks_per_lun * dev->plane_mode)
+	if (nr_blks != geo->blks_per_lun * geo->plane_mode)
 		return -EINVAL;
 
-	for (blk = 0; blk < dev->blks_per_lun; blk++) {
-		offset = blk * dev->plane_mode;
+	for (blk = 0; blk < geo->blks_per_lun; blk++) {
+		offset = blk * geo->plane_mode;
 		blktype = blks[offset];
 
 		/* Bad blocks on any planes take precedence over other types */
-		for (pl = 0; pl < dev->plane_mode; pl++) {
+		for (pl = 0; pl < geo->plane_mode; pl++) {
 			if (blks[offset + pl] &
 					(NVM_BLK_T_BAD|NVM_BLK_T_GRWN_BAD)) {
 				blktype = blks[offset + pl];
@@ -458,7 +555,7 @@ int nvm_bb_tbl_fold(struct nvm_dev *dev, u8 *blks, int nr_blks)
 		blks[blk] = blktype;
 	}
 
-	return dev->blks_per_lun;
+	return geo->blks_per_lun;
 }
 EXPORT_SYMBOL(nvm_bb_tbl_fold);
 
@@ -470,11 +567,22 @@ int nvm_get_bb_tbl(struct nvm_dev *dev, struct ppa_addr ppa, u8 *blks)
 }
 EXPORT_SYMBOL(nvm_get_bb_tbl);
 
+int nvm_get_tgt_bb_tbl(struct nvm_tgt_dev *tgt_dev, struct ppa_addr ppa,
+		       u8 *blks)
+{
+	struct nvm_dev *dev = tgt_dev->parent;
+
+	ppa = dev->mt->trans_ppa(tgt_dev, ppa, TRANS_TGT_TO_DEV);
+	return nvm_get_bb_tbl(dev, ppa, blks);
+}
+EXPORT_SYMBOL(nvm_get_tgt_bb_tbl);
+
 static int nvm_init_slc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
 {
+	struct nvm_geo *geo = &dev->geo;
 	int i;
 
-	dev->lps_per_blk = dev->pgs_per_blk;
+	dev->lps_per_blk = geo->pgs_per_blk;
 	dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
 	if (!dev->lptbl)
 		return -ENOMEM;
@@ -520,29 +628,32 @@ static int nvm_core_init(struct nvm_dev *dev)
 {
 	struct nvm_id *id = &dev->identity;
 	struct nvm_id_group *grp = &id->groups[0];
+	struct nvm_geo *geo = &dev->geo;
 	int ret;
 
-	/* device values */
-	dev->nr_chnls = grp->num_ch;
-	dev->luns_per_chnl = grp->num_lun;
-	dev->pgs_per_blk = grp->num_pg;
-	dev->blks_per_lun = grp->num_blk;
-	dev->nr_planes = grp->num_pln;
-	dev->fpg_size = grp->fpg_sz;
-	dev->pfpg_size = grp->fpg_sz * grp->num_pln;
-	dev->sec_size = grp->csecs;
-	dev->oob_size = grp->sos;
-	dev->sec_per_pg = grp->fpg_sz / grp->csecs;
-	dev->mccap = grp->mccap;
-	memcpy(&dev->ppaf, &id->ppaf, sizeof(struct nvm_addr_format));
-
-	dev->plane_mode = NVM_PLANE_SINGLE;
-	dev->max_rq_size = dev->ops->max_phys_sect * dev->sec_size;
+	/* Whole device values */
+	geo->nr_chnls = grp->num_ch;
+	geo->luns_per_chnl = grp->num_lun;
+
+	/* Generic device values */
+	geo->pgs_per_blk = grp->num_pg;
+	geo->blks_per_lun = grp->num_blk;
+	geo->nr_planes = grp->num_pln;
+	geo->fpg_size = grp->fpg_sz;
+	geo->pfpg_size = grp->fpg_sz * grp->num_pln;
+	geo->sec_size = grp->csecs;
+	geo->oob_size = grp->sos;
+	geo->sec_per_pg = grp->fpg_sz / grp->csecs;
+	geo->mccap = grp->mccap;
+	memcpy(&geo->ppaf, &id->ppaf, sizeof(struct nvm_addr_format));
+
+	geo->plane_mode = NVM_PLANE_SINGLE;
+	geo->max_rq_size = dev->ops->max_phys_sect * geo->sec_size;
 
 	if (grp->mpos & 0x020202)
-		dev->plane_mode = NVM_PLANE_DOUBLE;
+		geo->plane_mode = NVM_PLANE_DOUBLE;
 	if (grp->mpos & 0x040404)
-		dev->plane_mode = NVM_PLANE_QUAD;
+		geo->plane_mode = NVM_PLANE_QUAD;
 
 	if (grp->mtype != 0) {
 		pr_err("nvm: memory type not supported\n");
@@ -550,13 +661,13 @@ static int nvm_core_init(struct nvm_dev *dev)
 	}
 
 	/* calculated values */
-	dev->sec_per_pl = dev->sec_per_pg * dev->nr_planes;
-	dev->sec_per_blk = dev->sec_per_pl * dev->pgs_per_blk;
-	dev->sec_per_lun = dev->sec_per_blk * dev->blks_per_lun;
-	dev->nr_luns = dev->luns_per_chnl * dev->nr_chnls;
+	geo->sec_per_pl = geo->sec_per_pg * geo->nr_planes;
+	geo->sec_per_blk = geo->sec_per_pl * geo->pgs_per_blk;
+	geo->sec_per_lun = geo->sec_per_blk * geo->blks_per_lun;
+	geo->nr_luns = geo->luns_per_chnl * geo->nr_chnls;
 
-	dev->total_secs = dev->nr_luns * dev->sec_per_lun;
-	dev->lun_map = kcalloc(BITS_TO_LONGS(dev->nr_luns),
+	dev->total_secs = geo->nr_luns * geo->sec_per_lun;
+	dev->lun_map = kcalloc(BITS_TO_LONGS(geo->nr_luns),
 					sizeof(unsigned long), GFP_KERNEL);
 	if (!dev->lun_map)
 		return -ENOMEM;
@@ -583,7 +694,7 @@ static int nvm_core_init(struct nvm_dev *dev)
 	mutex_init(&dev->mlock);
 	spin_lock_init(&dev->lock);
 
-	blk_queue_logical_block_size(dev->q, dev->sec_size);
+	blk_queue_logical_block_size(dev->q, geo->sec_size);
 
 	return 0;
 err_fmtype:
@@ -617,6 +728,7 @@ void nvm_free(struct nvm_dev *dev)
 
 static int nvm_init(struct nvm_dev *dev)
 {
+	struct nvm_geo *geo = &dev->geo;
 	int ret = -EINVAL;
 
 	if (!dev->q || !dev->ops)
@@ -648,20 +760,15 @@ static int nvm_init(struct nvm_dev *dev)
 	}
 
 	pr_info("nvm: registered %s [%u/%u/%u/%u/%u/%u]\n",
-			dev->name, dev->sec_per_pg, dev->nr_planes,
-			dev->pgs_per_blk, dev->blks_per_lun, dev->nr_luns,
-			dev->nr_chnls);
+			dev->name, geo->sec_per_pg, geo->nr_planes,
+			geo->pgs_per_blk, geo->blks_per_lun,
+			geo->nr_luns, geo->nr_chnls);
 	return 0;
 err:
 	pr_err("nvm: failed to initialize nvm\n");
 	return ret;
 }
 
-static void nvm_exit(struct nvm_dev *dev)
-{
-	nvm_sysfs_unregister_dev(dev);
-}
-
 struct nvm_dev *nvm_alloc_dev(int node)
 {
 	return kzalloc_node(sizeof(struct nvm_dev), GFP_KERNEL, node);
@@ -691,10 +798,6 @@ int nvm_register(struct nvm_dev *dev)
 		}
 	}
 
-	ret = nvm_sysfs_register_dev(dev);
-	if (ret)
-		goto err_ppalist;
-
 	if (dev->identity.cap & NVM_ID_DCAP_BBLKMGMT) {
 		ret = nvm_get_sysblock(dev, &dev->sb);
 		if (!ret)
@@ -711,8 +814,6 @@ int nvm_register(struct nvm_dev *dev)
 	up_write(&nvm_lock);
 
 	return 0;
-err_ppalist:
-	dev->ops->destroy_dma_pool(dev->dma_pool);
 err_init:
 	kfree(dev->lun_map);
 	return ret;
@@ -725,7 +826,7 @@ void nvm_unregister(struct nvm_dev *dev)
 	list_del(&dev->devices);
 	up_write(&nvm_lock);
 
-	nvm_exit(dev);
+	nvm_free(dev);
 }
 EXPORT_SYMBOL(nvm_unregister);
 
@@ -754,149 +855,15 @@ static int __nvm_configure_create(struct nvm_ioctl_create *create)
 	}
 	s = &create->conf.s;
 
-	if (s->lun_begin > s->lun_end || s->lun_end > dev->nr_luns) {
+	if (s->lun_begin > s->lun_end || s->lun_end > dev->geo.nr_luns) {
 		pr_err("nvm: lun out of bound (%u:%u > %u)\n",
-			s->lun_begin, s->lun_end, dev->nr_luns);
+			s->lun_begin, s->lun_end, dev->geo.nr_luns);
 		return -EINVAL;
 	}
 
 	return dev->mt->create_tgt(dev, create);
 }
 
-#ifdef CONFIG_NVM_DEBUG
-static int nvm_configure_show(const char *val)
-{
-	struct nvm_dev *dev;
-	char opcode, devname[DISK_NAME_LEN];
-	int ret;
-
-	ret = sscanf(val, "%c %32s", &opcode, devname);
-	if (ret != 2) {
-		pr_err("nvm: invalid command. Use \"opcode devicename\".\n");
-		return -EINVAL;
-	}
-
-	down_write(&nvm_lock);
-	dev = nvm_find_nvm_dev(devname);
-	up_write(&nvm_lock);
-	if (!dev) {
-		pr_err("nvm: device not found\n");
-		return -EINVAL;
-	}
-
-	if (!dev->mt)
-		return 0;
-
-	dev->mt->lun_info_print(dev);
-
-	return 0;
-}
-
-static int nvm_configure_remove(const char *val)
-{
-	struct nvm_ioctl_remove remove;
-	struct nvm_dev *dev;
-	char opcode;
-	int ret = 0;
-
-	ret = sscanf(val, "%c %256s", &opcode, remove.tgtname);
-	if (ret != 2) {
-		pr_err("nvm: invalid command. Use \"d targetname\".\n");
-		return -EINVAL;
-	}
-
-	remove.flags = 0;
-
-	list_for_each_entry(dev, &nvm_devices, devices) {
-		ret = dev->mt->remove_tgt(dev, &remove);
-		if (!ret)
-			break;
-	}
-
-	return ret;
-}
-
-static int nvm_configure_create(const char *val)
-{
-	struct nvm_ioctl_create create;
-	char opcode;
-	int lun_begin, lun_end, ret;
-
-	ret = sscanf(val, "%c %256s %256s %48s %u:%u", &opcode, create.dev,
-						create.tgtname, create.tgttype,
-						&lun_begin, &lun_end);
-	if (ret != 6) {
-		pr_err("nvm: invalid command. Use \"opcode device name tgttype lun_begin:lun_end\".\n");
-		return -EINVAL;
-	}
-
-	create.flags = 0;
-	create.conf.type = NVM_CONFIG_TYPE_SIMPLE;
-	create.conf.s.lun_begin = lun_begin;
-	create.conf.s.lun_end = lun_end;
-
-	return __nvm_configure_create(&create);
-}
-
-
-/* Exposes administrative interface through /sys/module/lnvm/configure_by_str */
-static int nvm_configure_by_str_event(const char *val,
-					const struct kernel_param *kp)
-{
-	char opcode;
-	int ret;
-
-	ret = sscanf(val, "%c", &opcode);
-	if (ret != 1) {
-		pr_err("nvm: string must have the format of \"cmd ...\"\n");
-		return -EINVAL;
-	}
-
-	switch (opcode) {
-	case 'a':
-		return nvm_configure_create(val);
-	case 'd':
-		return nvm_configure_remove(val);
-	case 's':
-		return nvm_configure_show(val);
-	default:
-		pr_err("nvm: invalid command\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int nvm_configure_get(char *buf, const struct kernel_param *kp)
-{
-	int sz;
-	struct nvm_dev *dev;
-
-	sz = sprintf(buf, "available devices:\n");
-	down_write(&nvm_lock);
-	list_for_each_entry(dev, &nvm_devices, devices) {
-		if (sz > 4095 - DISK_NAME_LEN - 2)
-			break;
-		sz += sprintf(buf + sz, " %32s\n", dev->name);
-	}
-	up_write(&nvm_lock);
-
-	return sz;
-}
-
-static const struct kernel_param_ops nvm_configure_by_str_event_param_ops = {
-	.set	= nvm_configure_by_str_event,
-	.get	= nvm_configure_get,
-};
-
-#undef MODULE_PARAM_PREFIX
-#define MODULE_PARAM_PREFIX	"lnvm."
-
-module_param_cb(configure_debug, &nvm_configure_by_str_event_param_ops, NULL,
-									0644);
-
-#endif /* CONFIG_NVM_DEBUG */
-
 static long nvm_ioctl_info(struct file *file, void __user *arg)
 {
 	struct nvm_ioctl_info *info;
@@ -1162,10 +1129,4 @@ static struct miscdevice _nvm_misc = {
 	.nodename	= "lightnvm/control",
 	.fops		= &_ctl_fops,
 };
-module_misc_device(_nvm_misc);
-
-MODULE_ALIAS_MISCDEV(MISC_DYNAMIC_MINOR);
-
-MODULE_AUTHOR("Matias Bjorling <m@bjorling.me>");
-MODULE_LICENSE("GPL v2");
-MODULE_VERSION("0.1");
+builtin_misc_device(_nvm_misc);
diff --git a/drivers/lightnvm/gennvm.c b/drivers/lightnvm/gennvm.c
index b74174c6d021..ca7880082d80 100644
--- a/drivers/lightnvm/gennvm.c
+++ b/drivers/lightnvm/gennvm.c
@@ -35,6 +35,165 @@ static const struct block_device_operations gen_fops = {
 	.owner		= THIS_MODULE,
 };
 
+static int gen_reserve_luns(struct nvm_dev *dev, struct nvm_target *t,
+			    int lun_begin, int lun_end)
+{
+	int i;
+
+	for (i = lun_begin; i <= lun_end; i++) {
+		if (test_and_set_bit(i, dev->lun_map)) {
+			pr_err("nvm: lun %d already allocated\n", i);
+			goto err;
+		}
+	}
+
+	return 0;
+
+err:
+	while (--i > lun_begin)
+		clear_bit(i, dev->lun_map);
+
+	return -EBUSY;
+}
+
+static void gen_release_luns_err(struct nvm_dev *dev, int lun_begin,
+				 int lun_end)
+{
+	int i;
+
+	for (i = lun_begin; i <= lun_end; i++)
+		WARN_ON(!test_and_clear_bit(i, dev->lun_map));
+}
+
+static void gen_remove_tgt_dev(struct nvm_tgt_dev *tgt_dev)
+{
+	struct nvm_dev *dev = tgt_dev->parent;
+	struct gen_dev_map *dev_map = tgt_dev->map;
+	int i, j;
+
+	for (i = 0; i < dev_map->nr_chnls; i++) {
+		struct gen_ch_map *ch_map = &dev_map->chnls[i];
+		int *lun_offs = ch_map->lun_offs;
+		int ch = i + ch_map->ch_off;
+
+		for (j = 0; j < ch_map->nr_luns; j++) {
+			int lun = j + lun_offs[j];
+			int lunid = (ch * dev->geo.luns_per_chnl) + lun;
+
+			WARN_ON(!test_and_clear_bit(lunid, dev->lun_map));
+		}
+
+		kfree(ch_map->lun_offs);
+	}
+
+	kfree(dev_map->chnls);
+	kfree(dev_map);
+	kfree(tgt_dev->luns);
+	kfree(tgt_dev);
+}
+
+static struct nvm_tgt_dev *gen_create_tgt_dev(struct nvm_dev *dev,
+					      int lun_begin, int lun_end)
+{
+	struct nvm_tgt_dev *tgt_dev = NULL;
+	struct gen_dev_map *dev_rmap = dev->rmap;
+	struct gen_dev_map *dev_map;
+	struct ppa_addr *luns;
+	int nr_luns = lun_end - lun_begin + 1;
+	int luns_left = nr_luns;
+	int nr_chnls = nr_luns / dev->geo.luns_per_chnl;
+	int nr_chnls_mod = nr_luns % dev->geo.luns_per_chnl;
+	int bch = lun_begin / dev->geo.luns_per_chnl;
+	int blun = lun_begin % dev->geo.luns_per_chnl;
+	int lunid = 0;
+	int lun_balanced = 1;
+	int prev_nr_luns;
+	int i, j;
+
+	nr_chnls = nr_luns / dev->geo.luns_per_chnl;
+	nr_chnls = (nr_chnls_mod == 0) ? nr_chnls : nr_chnls + 1;
+
+	dev_map = kmalloc(sizeof(struct gen_dev_map), GFP_KERNEL);
+	if (!dev_map)
+		goto err_dev;
+
+	dev_map->chnls = kcalloc(nr_chnls, sizeof(struct gen_ch_map),
+								GFP_KERNEL);
+	if (!dev_map->chnls)
+		goto err_chnls;
+
+	luns = kcalloc(nr_luns, sizeof(struct ppa_addr), GFP_KERNEL);
+	if (!luns)
+		goto err_luns;
+
+	prev_nr_luns = (luns_left > dev->geo.luns_per_chnl) ?
+					dev->geo.luns_per_chnl : luns_left;
+	for (i = 0; i < nr_chnls; i++) {
+		struct gen_ch_map *ch_rmap = &dev_rmap->chnls[i + bch];
+		int *lun_roffs = ch_rmap->lun_offs;
+		struct gen_ch_map *ch_map = &dev_map->chnls[i];
+		int *lun_offs;
+		int luns_in_chnl = (luns_left > dev->geo.luns_per_chnl) ?
+					dev->geo.luns_per_chnl : luns_left;
+
+		if (lun_balanced && prev_nr_luns != luns_in_chnl)
+			lun_balanced = 0;
+
+		ch_map->ch_off = ch_rmap->ch_off = bch;
+		ch_map->nr_luns = luns_in_chnl;
+
+		lun_offs = kcalloc(luns_in_chnl, sizeof(int), GFP_KERNEL);
+		if (!lun_offs)
+			goto err_ch;
+
+		for (j = 0; j < luns_in_chnl; j++) {
+			luns[lunid].ppa = 0;
+			luns[lunid].g.ch = i;
+			luns[lunid++].g.lun = j;
+
+			lun_offs[j] = blun;
+			lun_roffs[j + blun] = blun;
+		}
+
+		ch_map->lun_offs = lun_offs;
+
+		/* when starting a new channel, lun offset is reset */
+		blun = 0;
+		luns_left -= luns_in_chnl;
+	}
+
+	dev_map->nr_chnls = nr_chnls;
+
+	tgt_dev = kmalloc(sizeof(struct nvm_tgt_dev), GFP_KERNEL);
+	if (!tgt_dev)
+		goto err_ch;
+
+	memcpy(&tgt_dev->geo, &dev->geo, sizeof(struct nvm_geo));
+	/* Target device only owns a portion of the physical device */
+	tgt_dev->geo.nr_chnls = nr_chnls;
+	tgt_dev->geo.nr_luns = nr_luns;
+	tgt_dev->geo.luns_per_chnl = (lun_balanced) ? prev_nr_luns : -1;
+	tgt_dev->total_secs = nr_luns * tgt_dev->geo.sec_per_lun;
+	tgt_dev->q = dev->q;
+	tgt_dev->map = dev_map;
+	tgt_dev->luns = luns;
+	memcpy(&tgt_dev->identity, &dev->identity, sizeof(struct nvm_id));
+
+	tgt_dev->parent = dev;
+
+	return tgt_dev;
+err_ch:
+	while (--i > 0)
+		kfree(dev_map->chnls[i].lun_offs);
+	kfree(luns);
+err_luns:
+	kfree(dev_map->chnls);
+err_chnls:
+	kfree(dev_map);
+err_dev:
+	return tgt_dev;
+}
+
 static int gen_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create)
 {
 	struct gen_dev *gn = dev->mp;
@@ -43,6 +202,7 @@ static int gen_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create)
 	struct gendisk *tdisk;
 	struct nvm_tgt_type *tt;
 	struct nvm_target *t;
+	struct nvm_tgt_dev *tgt_dev;
 	void *targetdata;
 
 	tt = nvm_find_target_type(create->tgttype, 1);
@@ -64,9 +224,18 @@ static int gen_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create)
 	if (!t)
 		return -ENOMEM;
 
+	if (gen_reserve_luns(dev, t, s->lun_begin, s->lun_end))
+		goto err_t;
+
+	tgt_dev = gen_create_tgt_dev(dev, s->lun_begin, s->lun_end);
+	if (!tgt_dev) {
+		pr_err("nvm: could not create target device\n");
+		goto err_reserve;
+	}
+
 	tqueue = blk_alloc_queue_node(GFP_KERNEL, dev->q->node);
 	if (!tqueue)
-		goto err_t;
+		goto err_dev;
 	blk_queue_make_request(tqueue, tt->make_rq);
 
 	tdisk = alloc_disk(0);
@@ -80,7 +249,7 @@ static int gen_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create)
 	tdisk->fops = &gen_fops;
 	tdisk->queue = tqueue;
 
-	targetdata = tt->init(dev, tdisk, s->lun_begin, s->lun_end);
+	targetdata = tt->init(tgt_dev, tdisk);
 	if (IS_ERR(targetdata))
 		goto err_init;
 
@@ -94,7 +263,7 @@ static int gen_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create)
 
 	t->type = tt;
 	t->disk = tdisk;
-	t->dev = dev;
+	t->dev = tgt_dev;
 
 	mutex_lock(&gn->lock);
 	list_add_tail(&t->list, &gn->targets);
@@ -105,6 +274,10 @@ err_init:
 	put_disk(tdisk);
 err_queue:
 	blk_cleanup_queue(tqueue);
+err_dev:
+	kfree(tgt_dev);
+err_reserve:
+	gen_release_luns_err(dev, s->lun_begin, s->lun_end);
 err_t:
 	kfree(t);
 	return -ENOMEM;
@@ -122,6 +295,7 @@ static void __gen_remove_target(struct nvm_target *t)
 	if (tt->exit)
 		tt->exit(tdisk->private_data);
 
+	gen_remove_tgt_dev(t->dev);
 	put_disk(tdisk);
 
 	list_del(&t->list);
@@ -160,10 +334,11 @@ static int gen_remove_tgt(struct nvm_dev *dev, struct nvm_ioctl_remove *remove)
 
 static int gen_get_area(struct nvm_dev *dev, sector_t *lba, sector_t len)
 {
+	struct nvm_geo *geo = &dev->geo;
 	struct gen_dev *gn = dev->mp;
 	struct gen_area *area, *prev, *next;
 	sector_t begin = 0;
-	sector_t max_sectors = (dev->sec_size * dev->total_secs) >> 9;
+	sector_t max_sectors = (geo->sec_size * dev->total_secs) >> 9;
 
 	if (len > max_sectors)
 		return -EINVAL;
@@ -220,240 +395,74 @@ static void gen_put_area(struct nvm_dev *dev, sector_t begin)
 	spin_unlock(&dev->lock);
 }
 
-static void gen_blocks_free(struct nvm_dev *dev)
-{
-	struct gen_dev *gn = dev->mp;
-	struct gen_lun *lun;
-	int i;
-
-	gen_for_each_lun(gn, lun, i) {
-		if (!lun->vlun.blocks)
-			break;
-		vfree(lun->vlun.blocks);
-	}
-}
-
-static void gen_luns_free(struct nvm_dev *dev)
-{
-	struct gen_dev *gn = dev->mp;
-
-	kfree(gn->luns);
-}
-
-static int gen_luns_init(struct nvm_dev *dev, struct gen_dev *gn)
-{
-	struct gen_lun *lun;
-	int i;
-
-	gn->luns = kcalloc(dev->nr_luns, sizeof(struct gen_lun), GFP_KERNEL);
-	if (!gn->luns)
-		return -ENOMEM;
-
-	gen_for_each_lun(gn, lun, i) {
-		spin_lock_init(&lun->vlun.lock);
-		INIT_LIST_HEAD(&lun->free_list);
-		INIT_LIST_HEAD(&lun->used_list);
-		INIT_LIST_HEAD(&lun->bb_list);
-
-		lun->reserved_blocks = 2; /* for GC only */
-		lun->vlun.id = i;
-		lun->vlun.lun_id = i % dev->luns_per_chnl;
-		lun->vlun.chnl_id = i / dev->luns_per_chnl;
-		lun->vlun.nr_free_blocks = dev->blks_per_lun;
-	}
-	return 0;
-}
-
-static int gen_block_bb(struct gen_dev *gn, struct ppa_addr ppa,
-							u8 *blks, int nr_blks)
-{
-	struct nvm_dev *dev = gn->dev;
-	struct gen_lun *lun;
-	struct nvm_block *blk;
-	int i;
-
-	nr_blks = nvm_bb_tbl_fold(dev, blks, nr_blks);
-	if (nr_blks < 0)
-		return nr_blks;
-
-	lun = &gn->luns[(dev->luns_per_chnl * ppa.g.ch) + ppa.g.lun];
-
-	for (i = 0; i < nr_blks; i++) {
-		if (blks[i] == 0)
-			continue;
-
-		blk = &lun->vlun.blocks[i];
-		list_move_tail(&blk->list, &lun->bb_list);
-		lun->vlun.nr_free_blocks--;
-	}
-
-	return 0;
-}
-
-static int gen_block_map(u64 slba, u32 nlb, __le64 *entries, void *private)
-{
-	struct nvm_dev *dev = private;
-	struct gen_dev *gn = dev->mp;
-	u64 elba = slba + nlb;
-	struct gen_lun *lun;
-	struct nvm_block *blk;
-	u64 i;
-	int lun_id;
-
-	if (unlikely(elba > dev->total_secs)) {
-		pr_err("gen: L2P data from device is out of bounds!\n");
-		return -EINVAL;
-	}
-
-	for (i = 0; i < nlb; i++) {
-		u64 pba = le64_to_cpu(entries[i]);
-
-		if (unlikely(pba >= dev->total_secs && pba != U64_MAX)) {
-			pr_err("gen: L2P data entry is out of bounds!\n");
-			return -EINVAL;
-		}
-
-		/* Address zero is a special one. The first page on a disk is
-		 * protected. It often holds internal device boot
-		 * information.
-		 */
-		if (!pba)
-			continue;
-
-		/* resolve block from physical address */
-		lun_id = div_u64(pba, dev->sec_per_lun);
-		lun = &gn->luns[lun_id];
-
-		/* Calculate block offset into lun */
-		pba = pba - (dev->sec_per_lun * lun_id);
-		blk = &lun->vlun.blocks[div_u64(pba, dev->sec_per_blk)];
-
-		if (!blk->state) {
-			/* at this point, we don't know anything about the
-			 * block. It's up to the FTL on top to re-etablish the
-			 * block state. The block is assumed to be open.
-			 */
-			list_move_tail(&blk->list, &lun->used_list);
-			blk->state = NVM_BLK_ST_TGT;
-			lun->vlun.nr_free_blocks--;
-		}
-	}
-
-	return 0;
-}
-
-static int gen_blocks_init(struct nvm_dev *dev, struct gen_dev *gn)
-{
-	struct gen_lun *lun;
-	struct nvm_block *block;
-	sector_t lun_iter, blk_iter, cur_block_id = 0;
-	int ret, nr_blks;
-	u8 *blks;
-
-	nr_blks = dev->blks_per_lun * dev->plane_mode;
-	blks = kmalloc(nr_blks, GFP_KERNEL);
-	if (!blks)
-		return -ENOMEM;
-
-	gen_for_each_lun(gn, lun, lun_iter) {
-		lun->vlun.blocks = vzalloc(sizeof(struct nvm_block) *
-							dev->blks_per_lun);
-		if (!lun->vlun.blocks) {
-			kfree(blks);
-			return -ENOMEM;
-		}
-
-		for (blk_iter = 0; blk_iter < dev->blks_per_lun; blk_iter++) {
-			block = &lun->vlun.blocks[blk_iter];
-
-			INIT_LIST_HEAD(&block->list);
-
-			block->lun = &lun->vlun;
-			block->id = cur_block_id++;
-
-			/* First block is reserved for device */
-			if (unlikely(lun_iter == 0 && blk_iter == 0)) {
-				lun->vlun.nr_free_blocks--;
-				continue;
-			}
-
-			list_add_tail(&block->list, &lun->free_list);
-		}
-
-		if (dev->ops->get_bb_tbl) {
-			struct ppa_addr ppa;
-
-			ppa.ppa = 0;
-			ppa.g.ch = lun->vlun.chnl_id;
-			ppa.g.lun = lun->vlun.lun_id;
-
-			ret = nvm_get_bb_tbl(dev, ppa, blks);
-			if (ret)
-				pr_err("gen: could not get BB table\n");
-
-			ret = gen_block_bb(gn, ppa, blks, nr_blks);
-			if (ret)
-				pr_err("gen: BB table map failed\n");
-		}
-	}
-
-	if ((dev->identity.dom & NVM_RSP_L2P) && dev->ops->get_l2p_tbl) {
-		ret = dev->ops->get_l2p_tbl(dev, 0, dev->total_secs,
-							gen_block_map, dev);
-		if (ret) {
-			pr_err("gen: could not read L2P table.\n");
-			pr_warn("gen: default block initialization");
-		}
-	}
-
-	kfree(blks);
-	return 0;
-}
-
 static void gen_free(struct nvm_dev *dev)
 {
-	gen_blocks_free(dev);
-	gen_luns_free(dev);
 	kfree(dev->mp);
+	kfree(dev->rmap);
 	dev->mp = NULL;
 }
 
 static int gen_register(struct nvm_dev *dev)
 {
 	struct gen_dev *gn;
-	int ret;
+	struct gen_dev_map *dev_rmap;
+	int i, j;
 
 	if (!try_module_get(THIS_MODULE))
 		return -ENODEV;
 
 	gn = kzalloc(sizeof(struct gen_dev), GFP_KERNEL);
 	if (!gn)
-		return -ENOMEM;
+		goto err_gn;
+
+	dev_rmap = kmalloc(sizeof(struct gen_dev_map), GFP_KERNEL);
+	if (!dev_rmap)
+		goto err_rmap;
+
+	dev_rmap->chnls = kcalloc(dev->geo.nr_chnls, sizeof(struct gen_ch_map),
+								GFP_KERNEL);
+	if (!dev_rmap->chnls)
+		goto err_chnls;
+
+	for (i = 0; i < dev->geo.nr_chnls; i++) {
+		struct gen_ch_map *ch_rmap;
+		int *lun_roffs;
+		int luns_in_chnl = dev->geo.luns_per_chnl;
+
+		ch_rmap = &dev_rmap->chnls[i];
+
+		ch_rmap->ch_off = -1;
+		ch_rmap->nr_luns = luns_in_chnl;
+
+		lun_roffs = kcalloc(luns_in_chnl, sizeof(int), GFP_KERNEL);
+		if (!lun_roffs)
+			goto err_ch;
+
+		for (j = 0; j < luns_in_chnl; j++)
+			lun_roffs[j] = -1;
+
+		ch_rmap->lun_offs = lun_roffs;
+	}
 
 	gn->dev = dev;
-	gn->nr_luns = dev->nr_luns;
+	gn->nr_luns = dev->geo.nr_luns;
 	INIT_LIST_HEAD(&gn->area_list);
 	mutex_init(&gn->lock);
 	INIT_LIST_HEAD(&gn->targets);
 	dev->mp = gn;
-
-	ret = gen_luns_init(dev, gn);
-	if (ret) {
-		pr_err("gen: could not initialize luns\n");
-		goto err;
-	}
-
-	ret = gen_blocks_init(dev, gn);
-	if (ret) {
-		pr_err("gen: could not initialize blocks\n");
-		goto err;
-	}
+	dev->rmap = dev_rmap;
 
 	return 1;
-err:
+err_ch:
+	while (--i >= 0)
+		kfree(dev_rmap->chnls[i].lun_offs);
+err_chnls:
+	kfree(dev_rmap);
+err_rmap:
 	gen_free(dev);
+err_gn:
 	module_put(THIS_MODULE);
-	return ret;
+	return -ENOMEM;
 }
 
 static void gen_unregister(struct nvm_dev *dev)
@@ -463,7 +472,7 @@ static void gen_unregister(struct nvm_dev *dev)
 
 	mutex_lock(&gn->lock);
 	list_for_each_entry_safe(t, tmp, &gn->targets, list) {
-		if (t->dev != dev)
+		if (t->dev->parent != dev)
 			continue;
 		__gen_remove_target(t);
 	}
@@ -473,168 +482,142 @@ static void gen_unregister(struct nvm_dev *dev)
 	module_put(THIS_MODULE);
 }
 
-static struct nvm_block *gen_get_blk(struct nvm_dev *dev,
-				struct nvm_lun *vlun, unsigned long flags)
+static int gen_map_to_dev(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p)
 {
-	struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun);
-	struct nvm_block *blk = NULL;
-	int is_gc = flags & NVM_IOTYPE_GC;
-
-	spin_lock(&vlun->lock);
-	if (list_empty(&lun->free_list)) {
-		pr_err_ratelimited("gen: lun %u have no free pages available",
-								lun->vlun.id);
-		goto out;
+	struct gen_dev_map *dev_map = tgt_dev->map;
+	struct gen_ch_map *ch_map = &dev_map->chnls[p->g.ch];
+	int lun_off = ch_map->lun_offs[p->g.lun];
+	struct nvm_dev *dev = tgt_dev->parent;
+	struct gen_dev_map *dev_rmap = dev->rmap;
+	struct gen_ch_map *ch_rmap;
+	int lun_roff;
+
+	p->g.ch += ch_map->ch_off;
+	p->g.lun += lun_off;
+
+	ch_rmap = &dev_rmap->chnls[p->g.ch];
+	lun_roff = ch_rmap->lun_offs[p->g.lun];
+
+	if (unlikely(ch_rmap->ch_off < 0 || lun_roff < 0)) {
+		pr_err("nvm: corrupted device partition table\n");
+		return -EINVAL;
 	}
 
-	if (!is_gc && lun->vlun.nr_free_blocks < lun->reserved_blocks)
-		goto out;
-
-	blk = list_first_entry(&lun->free_list, struct nvm_block, list);
-
-	list_move_tail(&blk->list, &lun->used_list);
-	blk->state = NVM_BLK_ST_TGT;
-	lun->vlun.nr_free_blocks--;
-out:
-	spin_unlock(&vlun->lock);
-	return blk;
-}
-
-static void gen_put_blk(struct nvm_dev *dev, struct nvm_block *blk)
-{
-	struct nvm_lun *vlun = blk->lun;
-	struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun);
-
-	spin_lock(&vlun->lock);
-	if (blk->state & NVM_BLK_ST_TGT) {
-		list_move_tail(&blk->list, &lun->free_list);
-		lun->vlun.nr_free_blocks++;
-		blk->state = NVM_BLK_ST_FREE;
-	} else if (blk->state & NVM_BLK_ST_BAD) {
-		list_move_tail(&blk->list, &lun->bb_list);
-		blk->state = NVM_BLK_ST_BAD;
-	} else {
-		WARN_ON_ONCE(1);
-		pr_err("gen: erroneous block type (%lu -> %u)\n",
-							blk->id, blk->state);
-		list_move_tail(&blk->list, &lun->bb_list);
-	}
-	spin_unlock(&vlun->lock);
+	return 0;
 }
 
-static void gen_mark_blk(struct nvm_dev *dev, struct ppa_addr ppa, int type)
+static int gen_map_to_tgt(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p)
 {
-	struct gen_dev *gn = dev->mp;
-	struct gen_lun *lun;
-	struct nvm_block *blk;
-
-	pr_debug("gen: ppa  (ch: %u lun: %u blk: %u pg: %u) -> %u\n",
-			ppa.g.ch, ppa.g.lun, ppa.g.blk, ppa.g.pg, type);
-
-	if (unlikely(ppa.g.ch > dev->nr_chnls ||
-					ppa.g.lun > dev->luns_per_chnl ||
-					ppa.g.blk > dev->blks_per_lun)) {
-		WARN_ON_ONCE(1);
-		pr_err("gen: ppa broken (ch: %u > %u lun: %u > %u blk: %u > %u",
-				ppa.g.ch, dev->nr_chnls,
-				ppa.g.lun, dev->luns_per_chnl,
-				ppa.g.blk, dev->blks_per_lun);
-		return;
-	}
+	struct nvm_dev *dev = tgt_dev->parent;
+	struct gen_dev_map *dev_rmap = dev->rmap;
+	struct gen_ch_map *ch_rmap = &dev_rmap->chnls[p->g.ch];
+	int lun_roff = ch_rmap->lun_offs[p->g.lun];
 
-	lun = &gn->luns[(dev->luns_per_chnl * ppa.g.ch) + ppa.g.lun];
-	blk = &lun->vlun.blocks[ppa.g.blk];
+	p->g.ch -= ch_rmap->ch_off;
+	p->g.lun -= lun_roff;
 
-	/* will be moved to bb list on put_blk from target */
-	blk->state = type;
+	return 0;
 }
 
-/*
- * mark block bad in gen. It is expected that the target recovers separately
- */
-static void gen_mark_blk_bad(struct nvm_dev *dev, struct nvm_rq *rqd)
+static int gen_trans_rq(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd,
+			int flag)
 {
-	int bit = -1;
-	int max_secs = dev->ops->max_phys_sect;
-	void *comp_bits = &rqd->ppa_status;
+	gen_trans_fn *f;
+	int i;
+	int ret = 0;
 
-	nvm_addr_to_generic_mode(dev, rqd);
+	f = (flag == TRANS_TGT_TO_DEV) ? gen_map_to_dev : gen_map_to_tgt;
 
-	/* look up blocks and mark them as bad */
-	if (rqd->nr_ppas == 1) {
-		gen_mark_blk(dev, rqd->ppa_addr, NVM_BLK_ST_BAD);
-		return;
+	if (rqd->nr_ppas == 1)
+		return f(tgt_dev, &rqd->ppa_addr);
+
+	for (i = 0; i < rqd->nr_ppas; i++) {
+		ret = f(tgt_dev, &rqd->ppa_list[i]);
+		if (ret)
+			goto out;
 	}
 
-	while ((bit = find_next_bit(comp_bits, max_secs, bit + 1)) < max_secs)
-		gen_mark_blk(dev, rqd->ppa_list[bit], NVM_BLK_ST_BAD);
+out:
+	return ret;
 }
 
 static void gen_end_io(struct nvm_rq *rqd)
 {
+	struct nvm_tgt_dev *tgt_dev = rqd->dev;
 	struct nvm_tgt_instance *ins = rqd->ins;
 
-	if (rqd->error == NVM_RSP_ERR_FAILWRITE)
-		gen_mark_blk_bad(rqd->dev, rqd);
+	/* Convert address space */
+	if (tgt_dev)
+		gen_trans_rq(tgt_dev, rqd, TRANS_DEV_TO_TGT);
 
 	ins->tt->end_io(rqd);
 }
 
-static int gen_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
+static int gen_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
 {
+	struct nvm_dev *dev = tgt_dev->parent;
+
 	if (!dev->ops->submit_io)
 		return -ENODEV;
 
 	/* Convert address space */
+	gen_trans_rq(tgt_dev, rqd, TRANS_TGT_TO_DEV);
 	nvm_generic_to_addr_mode(dev, rqd);
 
-	rqd->dev = dev;
+	rqd->dev = tgt_dev;
 	rqd->end_io = gen_end_io;
 	return dev->ops->submit_io(dev, rqd);
 }
 
-static int gen_erase_blk(struct nvm_dev *dev, struct nvm_block *blk,
-							unsigned long flags)
+static int gen_erase_blk(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p,
+			 int flags)
 {
-	struct ppa_addr addr = block_to_ppa(dev, blk);
+	/* Convert address space */
+	gen_map_to_dev(tgt_dev, p);
 
-	return nvm_erase_ppa(dev, &addr, 1);
+	return nvm_erase_ppa(tgt_dev->parent, p, 1, flags);
 }
 
-static int gen_reserve_lun(struct nvm_dev *dev, int lunid)
+static struct ppa_addr gen_trans_ppa(struct nvm_tgt_dev *tgt_dev,
+				     struct ppa_addr p, int direction)
 {
-	return test_and_set_bit(lunid, dev->lun_map);
-}
+	gen_trans_fn *f;
+	struct ppa_addr ppa = p;
 
-static void gen_release_lun(struct nvm_dev *dev, int lunid)
-{
-	WARN_ON(!test_and_clear_bit(lunid, dev->lun_map));
+	f = (direction == TRANS_TGT_TO_DEV) ? gen_map_to_dev : gen_map_to_tgt;
+	f(tgt_dev, &ppa);
+
+	return ppa;
 }
 
-static struct nvm_lun *gen_get_lun(struct nvm_dev *dev, int lunid)
+static void gen_part_to_tgt(struct nvm_dev *dev, sector_t *entries,
+			       int len)
 {
-	struct gen_dev *gn = dev->mp;
-
-	if (unlikely(lunid >= dev->nr_luns))
-		return NULL;
+	struct nvm_geo *geo = &dev->geo;
+	struct gen_dev_map *dev_rmap = dev->rmap;
+	u64 i;
 
-	return &gn->luns[lunid].vlun;
-}
+	for (i = 0; i < len; i++) {
+		struct gen_ch_map *ch_rmap;
+		int *lun_roffs;
+		struct ppa_addr gaddr;
+		u64 pba = le64_to_cpu(entries[i]);
+		int off;
+		u64 diff;
 
-static void gen_lun_info_print(struct nvm_dev *dev)
-{
-	struct gen_dev *gn = dev->mp;
-	struct gen_lun *lun;
-	unsigned int i;
+		if (!pba)
+			continue;
 
+		gaddr = linear_to_generic_addr(geo, pba);
+		ch_rmap = &dev_rmap->chnls[gaddr.g.ch];
+		lun_roffs = ch_rmap->lun_offs;
 
-	gen_for_each_lun(gn, lun, i) {
-		spin_lock(&lun->vlun.lock);
+		off = gaddr.g.ch * geo->luns_per_chnl + gaddr.g.lun;
 
-		pr_info("%s: lun%8u\t%u\n", dev->name, i,
-						lun->vlun.nr_free_blocks);
+		diff = ((ch_rmap->ch_off * geo->luns_per_chnl) +
+				(lun_roffs[gaddr.g.lun])) * geo->sec_per_lun;
 
-		spin_unlock(&lun->vlun.lock);
+		entries[i] -= cpu_to_le64(diff);
 	}
 }
 
@@ -648,22 +631,14 @@ static struct nvmm_type gen = {
 	.create_tgt		= gen_create_tgt,
 	.remove_tgt		= gen_remove_tgt,
 
-	.get_blk		= gen_get_blk,
-	.put_blk		= gen_put_blk,
-
 	.submit_io		= gen_submit_io,
 	.erase_blk		= gen_erase_blk,
 
-	.mark_blk		= gen_mark_blk,
-
-	.get_lun		= gen_get_lun,
-	.reserve_lun		= gen_reserve_lun,
-	.release_lun		= gen_release_lun,
-	.lun_info_print		= gen_lun_info_print,
-
 	.get_area		= gen_get_area,
 	.put_area		= gen_put_area,
 
+	.trans_ppa		= gen_trans_ppa,
+	.part_to_tgt		= gen_part_to_tgt,
 };
 
 static int __init gen_module_init(void)
diff --git a/drivers/lightnvm/gennvm.h b/drivers/lightnvm/gennvm.h
index 8ecfa817d21d..6a4b3f368848 100644
--- a/drivers/lightnvm/gennvm.h
+++ b/drivers/lightnvm/gennvm.h
@@ -20,37 +20,41 @@
 
 #include <linux/lightnvm.h>
 
-struct gen_lun {
-	struct nvm_lun vlun;
-
-	int reserved_blocks;
-	/* lun block lists */
-	struct list_head used_list;	/* In-use blocks */
-	struct list_head free_list;	/* Not used blocks i.e. released
-					 * and ready for use
-					 */
-	struct list_head bb_list;	/* Bad blocks. Mutually exclusive with
-					 * free_list and used_list
-					 */
-};
-
 struct gen_dev {
 	struct nvm_dev *dev;
 
 	int nr_luns;
-	struct gen_lun *luns;
 	struct list_head area_list;
 
 	struct mutex lock;
 	struct list_head targets;
 };
 
+/* Map between virtual and physical channel and lun */
+struct gen_ch_map {
+	int ch_off;
+	int nr_luns;
+	int *lun_offs;
+};
+
+struct gen_dev_map {
+	struct gen_ch_map *chnls;
+	int nr_chnls;
+};
+
 struct gen_area {
 	struct list_head list;
 	sector_t begin;
 	sector_t end;	/* end is excluded */
 };
 
+static inline void *ch_map_to_lun_offs(struct gen_ch_map *ch_map)
+{
+	return ch_map + 1;
+}
+
+typedef int (gen_trans_fn)(struct nvm_tgt_dev *, struct ppa_addr *);
+
 #define gen_for_each_lun(bm, lun, i) \
 		for ((i) = 0, lun = &(bm)->luns[0]; \
 			(i) < (bm)->nr_luns; (i)++, lun = &(bm)->luns[(i)])
diff --git a/drivers/lightnvm/lightnvm.h b/drivers/lightnvm/lightnvm.h
deleted file mode 100644
index 305c181509a6..000000000000
--- a/drivers/lightnvm/lightnvm.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (C) 2016 CNEX Labs. All rights reserved.
- * Initial release: Matias Bjorling <matias@cnexlabs.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
- * USA.
- *
- */
-
-#ifndef LIGHTNVM_H
-#define LIGHTNVM_H
-
-#include <linux/lightnvm.h>
-
-/* core -> sysfs.c */
-int __must_check nvm_sysfs_register_dev(struct nvm_dev *);
-void nvm_sysfs_unregister_dev(struct nvm_dev *);
-int nvm_sysfs_register(void);
-void nvm_sysfs_unregister(void);
-
-/* sysfs > core */
-void nvm_free(struct nvm_dev *);
-
-#endif
diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c
index 37fcaadbf80c..9fb7de395915 100644
--- a/drivers/lightnvm/rrpc.c
+++ b/drivers/lightnvm/rrpc.c
@@ -28,6 +28,7 @@ static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
 
 static void rrpc_page_invalidate(struct rrpc *rrpc, struct rrpc_addr *a)
 {
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	struct rrpc_block *rblk = a->rblk;
 	unsigned int pg_offset;
 
@@ -38,13 +39,13 @@ static void rrpc_page_invalidate(struct rrpc *rrpc, struct rrpc_addr *a)
 
 	spin_lock(&rblk->lock);
 
-	div_u64_rem(a->addr, rrpc->dev->sec_per_blk, &pg_offset);
+	div_u64_rem(a->addr, dev->geo.sec_per_blk, &pg_offset);
 	WARN_ON(test_and_set_bit(pg_offset, rblk->invalid_pages));
 	rblk->nr_invalid_pages++;
 
 	spin_unlock(&rblk->lock);
 
-	rrpc->rev_trans_map[a->addr - rrpc->poffset].addr = ADDR_EMPTY;
+	rrpc->rev_trans_map[a->addr].addr = ADDR_EMPTY;
 }
 
 static void rrpc_invalidate_range(struct rrpc *rrpc, sector_t slba,
@@ -116,62 +117,35 @@ static void rrpc_discard(struct rrpc *rrpc, struct bio *bio)
 
 static int block_is_full(struct rrpc *rrpc, struct rrpc_block *rblk)
 {
-	return (rblk->next_page == rrpc->dev->sec_per_blk);
+	struct nvm_tgt_dev *dev = rrpc->dev;
+
+	return (rblk->next_page == dev->geo.sec_per_blk);
 }
 
 /* Calculate relative addr for the given block, considering instantiated LUNs */
 static u64 block_to_rel_addr(struct rrpc *rrpc, struct rrpc_block *rblk)
 {
-	struct nvm_block *blk = rblk->parent;
-	int lun_blk = blk->id % (rrpc->dev->blks_per_lun * rrpc->nr_luns);
-
-	return lun_blk * rrpc->dev->sec_per_blk;
-}
-
-/* Calculate global addr for the given block */
-static u64 block_to_addr(struct rrpc *rrpc, struct rrpc_block *rblk)
-{
-	struct nvm_block *blk = rblk->parent;
-
-	return blk->id * rrpc->dev->sec_per_blk;
-}
-
-static struct ppa_addr linear_to_generic_addr(struct nvm_dev *dev,
-							struct ppa_addr r)
-{
-	struct ppa_addr l;
-	int secs, pgs, blks, luns;
-	sector_t ppa = r.ppa;
-
-	l.ppa = 0;
-
-	div_u64_rem(ppa, dev->sec_per_pg, &secs);
-	l.g.sec = secs;
-
-	sector_div(ppa, dev->sec_per_pg);
-	div_u64_rem(ppa, dev->pgs_per_blk, &pgs);
-	l.g.pg = pgs;
-
-	sector_div(ppa, dev->pgs_per_blk);
-	div_u64_rem(ppa, dev->blks_per_lun, &blks);
-	l.g.blk = blks;
-
-	sector_div(ppa, dev->blks_per_lun);
-	div_u64_rem(ppa, dev->luns_per_chnl, &luns);
-	l.g.lun = luns;
-
-	sector_div(ppa, dev->luns_per_chnl);
-	l.g.ch = ppa;
+	struct nvm_tgt_dev *dev = rrpc->dev;
+	struct rrpc_lun *rlun = rblk->rlun;
 
-	return l;
+	return rlun->id * dev->geo.sec_per_blk;
 }
 
-static struct ppa_addr rrpc_ppa_to_gaddr(struct nvm_dev *dev, u64 addr)
+static struct ppa_addr rrpc_ppa_to_gaddr(struct nvm_tgt_dev *dev,
+					 struct rrpc_addr *gp)
 {
+	struct rrpc_block *rblk = gp->rblk;
+	struct rrpc_lun *rlun = rblk->rlun;
+	u64 addr = gp->addr;
 	struct ppa_addr paddr;
 
 	paddr.ppa = addr;
-	return linear_to_generic_addr(dev, paddr);
+	paddr = rrpc_linear_to_generic_addr(&dev->geo, paddr);
+	paddr.g.ch = rlun->bppa.g.ch;
+	paddr.g.lun = rlun->bppa.g.lun;
+	paddr.g.blk = rblk->id;
+
+	return paddr;
 }
 
 /* requires lun->lock taken */
@@ -188,21 +162,47 @@ static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *new_rblk,
 	*cur_rblk = new_rblk;
 }
 
+static struct rrpc_block *__rrpc_get_blk(struct rrpc *rrpc,
+							struct rrpc_lun *rlun)
+{
+	struct rrpc_block *rblk = NULL;
+
+	if (list_empty(&rlun->free_list))
+		goto out;
+
+	rblk = list_first_entry(&rlun->free_list, struct rrpc_block, list);
+
+	list_move_tail(&rblk->list, &rlun->used_list);
+	rblk->state = NVM_BLK_ST_TGT;
+	rlun->nr_free_blocks--;
+
+out:
+	return rblk;
+}
+
 static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun,
 							unsigned long flags)
 {
-	struct nvm_block *blk;
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	struct rrpc_block *rblk;
+	int is_gc = flags & NVM_IOTYPE_GC;
+
+	spin_lock(&rlun->lock);
+	if (!is_gc && rlun->nr_free_blocks < rlun->reserved_blocks) {
+		pr_err("nvm: rrpc: cannot give block to non GC request\n");
+		spin_unlock(&rlun->lock);
+		return NULL;
+	}
 
-	blk = nvm_get_blk(rrpc->dev, rlun->parent, flags);
-	if (!blk) {
-		pr_err("nvm: rrpc: cannot get new block from media manager\n");
+	rblk = __rrpc_get_blk(rrpc, rlun);
+	if (!rblk) {
+		pr_err("nvm: rrpc: cannot get new block\n");
+		spin_unlock(&rlun->lock);
 		return NULL;
 	}
+	spin_unlock(&rlun->lock);
 
-	rblk = rrpc_get_rblk(rlun, blk->id);
-	blk->priv = rblk;
-	bitmap_zero(rblk->invalid_pages, rrpc->dev->sec_per_blk);
+	bitmap_zero(rblk->invalid_pages, dev->geo.sec_per_blk);
 	rblk->next_page = 0;
 	rblk->nr_invalid_pages = 0;
 	atomic_set(&rblk->data_cmnt_size, 0);
@@ -212,7 +212,24 @@ static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun,
 
 static void rrpc_put_blk(struct rrpc *rrpc, struct rrpc_block *rblk)
 {
-	nvm_put_blk(rrpc->dev, rblk->parent);
+	struct rrpc_lun *rlun = rblk->rlun;
+
+	spin_lock(&rlun->lock);
+	if (rblk->state & NVM_BLK_ST_TGT) {
+		list_move_tail(&rblk->list, &rlun->free_list);
+		rlun->nr_free_blocks++;
+		rblk->state = NVM_BLK_ST_FREE;
+	} else if (rblk->state & NVM_BLK_ST_BAD) {
+		list_move_tail(&rblk->list, &rlun->bb_list);
+		rblk->state = NVM_BLK_ST_BAD;
+	} else {
+		WARN_ON_ONCE(1);
+		pr_err("rrpc: erroneous type (ch:%d,lun:%d,blk%d-> %u)\n",
+					rlun->bppa.g.ch, rlun->bppa.g.lun,
+					rblk->id, rblk->state);
+		list_move_tail(&rblk->list, &rlun->bb_list);
+	}
+	spin_unlock(&rlun->lock);
 }
 
 static void rrpc_put_blks(struct rrpc *rrpc)
@@ -280,13 +297,14 @@ static void rrpc_end_sync_bio(struct bio *bio)
  */
 static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk)
 {
-	struct request_queue *q = rrpc->dev->q;
+	struct nvm_tgt_dev *dev = rrpc->dev;
+	struct request_queue *q = dev->q;
 	struct rrpc_rev_addr *rev;
 	struct nvm_rq *rqd;
 	struct bio *bio;
 	struct page *page;
 	int slot;
-	int nr_sec_per_blk = rrpc->dev->sec_per_blk;
+	int nr_sec_per_blk = dev->geo.sec_per_blk;
 	u64 phys_addr;
 	DECLARE_COMPLETION_ONSTACK(wait);
 
@@ -309,12 +327,12 @@ static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk)
 					    nr_sec_per_blk)) < nr_sec_per_blk) {
 
 		/* Lock laddr */
-		phys_addr = rblk->parent->id * nr_sec_per_blk + slot;
+		phys_addr = rrpc_blk_to_ppa(rrpc, rblk) + slot;
 
 try:
 		spin_lock(&rrpc->rev_lock);
 		/* Get logical address from physical to logical table */
-		rev = &rrpc->rev_trans_map[phys_addr - rrpc->poffset];
+		rev = &rrpc->rev_trans_map[phys_addr];
 		/* already updated by previous regular write */
 		if (rev->addr == ADDR_EMPTY) {
 			spin_unlock(&rrpc->rev_lock);
@@ -396,15 +414,23 @@ static void rrpc_block_gc(struct work_struct *work)
 	struct rrpc *rrpc = gcb->rrpc;
 	struct rrpc_block *rblk = gcb->rblk;
 	struct rrpc_lun *rlun = rblk->rlun;
-	struct nvm_dev *dev = rrpc->dev;
+	struct nvm_tgt_dev *dev = rrpc->dev;
+	struct ppa_addr ppa;
 
 	mempool_free(gcb, rrpc->gcb_pool);
-	pr_debug("nvm: block '%lu' being reclaimed\n", rblk->parent->id);
+	pr_debug("nvm: block 'ch:%d,lun:%d,blk:%d' being reclaimed\n",
+			rlun->bppa.g.ch, rlun->bppa.g.lun,
+			rblk->id);
 
 	if (rrpc_move_valid_pages(rrpc, rblk))
 		goto put_back;
 
-	if (nvm_erase_blk(dev, rblk->parent))
+	ppa.ppa = 0;
+	ppa.g.ch = rlun->bppa.g.ch;
+	ppa.g.lun = rlun->bppa.g.lun;
+	ppa.g.blk = rblk->id;
+
+	if (nvm_erase_blk(dev, &ppa, 0))
 		goto put_back;
 
 	rrpc_put_blk(rrpc, rblk);
@@ -420,7 +446,7 @@ put_back:
 /* the block with highest number of invalid pages, will be in the beginning
  * of the list
  */
-static struct rrpc_block *rblock_max_invalid(struct rrpc_block *ra,
+static struct rrpc_block *rblk_max_invalid(struct rrpc_block *ra,
 							struct rrpc_block *rb)
 {
 	if (ra->nr_invalid_pages == rb->nr_invalid_pages)
@@ -435,13 +461,13 @@ static struct rrpc_block *rblock_max_invalid(struct rrpc_block *ra,
 static struct rrpc_block *block_prio_find_max(struct rrpc_lun *rlun)
 {
 	struct list_head *prio_list = &rlun->prio_list;
-	struct rrpc_block *rblock, *max;
+	struct rrpc_block *rblk, *max;
 
 	BUG_ON(list_empty(prio_list));
 
 	max = list_first_entry(prio_list, struct rrpc_block, prio);
-	list_for_each_entry(rblock, prio_list, prio)
-		max = rblock_max_invalid(max, rblock);
+	list_for_each_entry(rblk, prio_list, prio)
+		max = rblk_max_invalid(max, rblk);
 
 	return max;
 }
@@ -450,36 +476,37 @@ static void rrpc_lun_gc(struct work_struct *work)
 {
 	struct rrpc_lun *rlun = container_of(work, struct rrpc_lun, ws_gc);
 	struct rrpc *rrpc = rlun->rrpc;
-	struct nvm_lun *lun = rlun->parent;
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	struct rrpc_block_gc *gcb;
 	unsigned int nr_blocks_need;
 
-	nr_blocks_need = rrpc->dev->blks_per_lun / GC_LIMIT_INVERSE;
+	nr_blocks_need = dev->geo.blks_per_lun / GC_LIMIT_INVERSE;
 
 	if (nr_blocks_need < rrpc->nr_luns)
 		nr_blocks_need = rrpc->nr_luns;
 
 	spin_lock(&rlun->lock);
-	while (nr_blocks_need > lun->nr_free_blocks &&
+	while (nr_blocks_need > rlun->nr_free_blocks &&
 					!list_empty(&rlun->prio_list)) {
-		struct rrpc_block *rblock = block_prio_find_max(rlun);
-		struct nvm_block *block = rblock->parent;
+		struct rrpc_block *rblk = block_prio_find_max(rlun);
 
-		if (!rblock->nr_invalid_pages)
+		if (!rblk->nr_invalid_pages)
 			break;
 
 		gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC);
 		if (!gcb)
 			break;
 
-		list_del_init(&rblock->prio);
+		list_del_init(&rblk->prio);
 
-		BUG_ON(!block_is_full(rrpc, rblock));
+		WARN_ON(!block_is_full(rrpc, rblk));
 
-		pr_debug("rrpc: selected block '%lu' for GC\n", block->id);
+		pr_debug("rrpc: selected block 'ch:%d,lun:%d,blk:%d' for GC\n",
+					rlun->bppa.g.ch, rlun->bppa.g.lun,
+					rblk->id);
 
 		gcb->rrpc = rrpc;
-		gcb->rblk = rblock;
+		gcb->rblk = rblk;
 		INIT_WORK(&gcb->ws_gc, rrpc_block_gc);
 
 		queue_work(rrpc->kgc_wq, &gcb->ws_gc);
@@ -504,8 +531,9 @@ static void rrpc_gc_queue(struct work_struct *work)
 	spin_unlock(&rlun->lock);
 
 	mempool_free(gcb, rrpc->gcb_pool);
-	pr_debug("nvm: block '%lu' is full, allow GC (sched)\n",
-							rblk->parent->id);
+	pr_debug("nvm: block 'ch:%d,lun:%d,blk:%d' full, allow GC (sched)\n",
+					rlun->bppa.g.ch, rlun->bppa.g.lun,
+					rblk->id);
 }
 
 static const struct block_device_operations rrpc_fops = {
@@ -529,8 +557,7 @@ static struct rrpc_lun *rrpc_get_lun_rr(struct rrpc *rrpc, int is_gc)
 	 * estimate.
 	 */
 	rrpc_for_each_lun(rrpc, rlun, i) {
-		if (rlun->parent->nr_free_blocks >
-					max_free->parent->nr_free_blocks)
+		if (rlun->nr_free_blocks > max_free->nr_free_blocks)
 			max_free = rlun;
 	}
 
@@ -553,7 +580,7 @@ static struct rrpc_addr *rrpc_update_map(struct rrpc *rrpc, sector_t laddr,
 	gp->addr = paddr;
 	gp->rblk = rblk;
 
-	rev = &rrpc->rev_trans_map[gp->addr - rrpc->poffset];
+	rev = &rrpc->rev_trans_map[gp->addr];
 	rev->addr = laddr;
 	spin_unlock(&rrpc->rev_lock);
 
@@ -568,7 +595,7 @@ static u64 rrpc_alloc_addr(struct rrpc *rrpc, struct rrpc_block *rblk)
 	if (block_is_full(rrpc, rblk))
 		goto out;
 
-	addr = block_to_addr(rrpc, rblk) + rblk->next_page;
+	addr = rblk->next_page;
 
 	rblk->next_page++;
 out:
@@ -582,20 +609,22 @@ out:
  * Returns rrpc_addr with the physical address and block. Returns NULL if no
  * blocks in the next rlun are available.
  */
-static struct rrpc_addr *rrpc_map_page(struct rrpc *rrpc, sector_t laddr,
+static struct ppa_addr rrpc_map_page(struct rrpc *rrpc, sector_t laddr,
 								int is_gc)
 {
+	struct nvm_tgt_dev *tgt_dev = rrpc->dev;
 	struct rrpc_lun *rlun;
 	struct rrpc_block *rblk, **cur_rblk;
-	struct nvm_lun *lun;
+	struct rrpc_addr *p;
+	struct ppa_addr ppa;
 	u64 paddr;
 	int gc_force = 0;
 
+	ppa.ppa = ADDR_EMPTY;
 	rlun = rrpc_get_lun_rr(rrpc, is_gc);
-	lun = rlun->parent;
 
-	if (!is_gc && lun->nr_free_blocks < rrpc->nr_luns * 4)
-		return NULL;
+	if (!is_gc && rlun->nr_free_blocks < rrpc->nr_luns * 4)
+		return ppa;
 
 	/*
 	 * page allocation steps:
@@ -652,10 +681,15 @@ new_blk:
 	}
 
 	pr_err("rrpc: failed to allocate new block\n");
-	return NULL;
+	return ppa;
 done:
 	spin_unlock(&rlun->lock);
-	return rrpc_update_map(rrpc, laddr, rblk, paddr);
+	p = rrpc_update_map(rrpc, laddr, rblk, paddr);
+	if (!p)
+		return ppa;
+
+	/* return global address */
+	return rrpc_ppa_to_gaddr(tgt_dev, p);
 }
 
 static void rrpc_run_gc(struct rrpc *rrpc, struct rrpc_block *rblk)
@@ -675,21 +709,70 @@ static void rrpc_run_gc(struct rrpc *rrpc, struct rrpc_block *rblk)
 	queue_work(rrpc->kgc_wq, &gcb->ws_gc);
 }
 
+static struct rrpc_lun *rrpc_ppa_to_lun(struct rrpc *rrpc, struct ppa_addr p)
+{
+	struct rrpc_lun *rlun = NULL;
+	int i;
+
+	for (i = 0; i < rrpc->nr_luns; i++) {
+		if (rrpc->luns[i].bppa.g.ch == p.g.ch &&
+				rrpc->luns[i].bppa.g.lun == p.g.lun) {
+			rlun = &rrpc->luns[i];
+			break;
+		}
+	}
+
+	return rlun;
+}
+
+static void __rrpc_mark_bad_block(struct rrpc *rrpc, struct ppa_addr ppa)
+{
+	struct nvm_tgt_dev *dev = rrpc->dev;
+	struct rrpc_lun *rlun;
+	struct rrpc_block *rblk;
+
+	rlun = rrpc_ppa_to_lun(rrpc, ppa);
+	rblk = &rlun->blocks[ppa.g.blk];
+	rblk->state = NVM_BLK_ST_BAD;
+
+	nvm_set_tgt_bb_tbl(dev, &ppa, 1, NVM_BLK_T_GRWN_BAD);
+}
+
+static void rrpc_mark_bad_block(struct rrpc *rrpc, struct nvm_rq *rqd)
+{
+	void *comp_bits = &rqd->ppa_status;
+	struct ppa_addr ppa, prev_ppa;
+	int nr_ppas = rqd->nr_ppas;
+	int bit;
+
+	if (rqd->nr_ppas == 1)
+		__rrpc_mark_bad_block(rrpc, rqd->ppa_addr);
+
+	ppa_set_empty(&prev_ppa);
+	bit = -1;
+	while ((bit = find_next_bit(comp_bits, nr_ppas, bit + 1)) < nr_ppas) {
+		ppa = rqd->ppa_list[bit];
+		if (ppa_cmp_blk(ppa, prev_ppa))
+			continue;
+
+		__rrpc_mark_bad_block(rrpc, ppa);
+	}
+}
+
 static void rrpc_end_io_write(struct rrpc *rrpc, struct rrpc_rq *rrqd,
 						sector_t laddr, uint8_t npages)
 {
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	struct rrpc_addr *p;
 	struct rrpc_block *rblk;
-	struct nvm_lun *lun;
 	int cmnt_size, i;
 
 	for (i = 0; i < npages; i++) {
 		p = &rrpc->trans_map[laddr + i];
 		rblk = p->rblk;
-		lun = rblk->parent->lun;
 
 		cmnt_size = atomic_inc_return(&rblk->data_cmnt_size);
-		if (unlikely(cmnt_size == rrpc->dev->sec_per_blk))
+		if (unlikely(cmnt_size == dev->geo.sec_per_blk))
 			rrpc_run_gc(rrpc, rblk);
 	}
 }
@@ -697,12 +780,17 @@ static void rrpc_end_io_write(struct rrpc *rrpc, struct rrpc_rq *rrqd,
 static void rrpc_end_io(struct nvm_rq *rqd)
 {
 	struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance);
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
 	uint8_t npages = rqd->nr_ppas;
 	sector_t laddr = rrpc_get_laddr(rqd->bio) - npages;
 
-	if (bio_data_dir(rqd->bio) == WRITE)
+	if (bio_data_dir(rqd->bio) == WRITE) {
+		if (rqd->error == NVM_RSP_ERR_FAILWRITE)
+			rrpc_mark_bad_block(rrpc, rqd);
+
 		rrpc_end_io_write(rrpc, rrqd, laddr, npages);
+	}
 
 	bio_put(rqd->bio);
 
@@ -712,7 +800,7 @@ static void rrpc_end_io(struct nvm_rq *rqd)
 	rrpc_unlock_rq(rrpc, rqd);
 
 	if (npages > 1)
-		nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
+		nvm_dev_dma_free(dev->parent, rqd->ppa_list, rqd->dma_ppa_list);
 
 	mempool_free(rqd, rrpc->rq_pool);
 }
@@ -720,6 +808,7 @@ static void rrpc_end_io(struct nvm_rq *rqd)
 static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
 			struct nvm_rq *rqd, unsigned long flags, int npages)
 {
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd);
 	struct rrpc_addr *gp;
 	sector_t laddr = rrpc_get_laddr(bio);
@@ -727,7 +816,7 @@ static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
 	int i;
 
 	if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) {
-		nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
+		nvm_dev_dma_free(dev->parent, rqd->ppa_list, rqd->dma_ppa_list);
 		return NVM_IO_REQUEUE;
 	}
 
@@ -737,12 +826,11 @@ static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
 		gp = &rrpc->trans_map[laddr + i];
 
 		if (gp->rblk) {
-			rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
-								gp->addr);
+			rqd->ppa_list[i] = rrpc_ppa_to_gaddr(dev, gp);
 		} else {
 			BUG_ON(is_gc);
 			rrpc_unlock_laddr(rrpc, r);
-			nvm_dev_dma_free(rrpc->dev, rqd->ppa_list,
+			nvm_dev_dma_free(dev->parent, rqd->ppa_list,
 							rqd->dma_ppa_list);
 			return NVM_IO_DONE;
 		}
@@ -756,7 +844,6 @@ static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
 static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
 							unsigned long flags)
 {
-	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
 	int is_gc = flags & NVM_IOTYPE_GC;
 	sector_t laddr = rrpc_get_laddr(bio);
 	struct rrpc_addr *gp;
@@ -768,7 +855,7 @@ static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
 	gp = &rrpc->trans_map[laddr];
 
 	if (gp->rblk) {
-		rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr);
+		rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp);
 	} else {
 		BUG_ON(is_gc);
 		rrpc_unlock_rq(rrpc, rqd);
@@ -776,7 +863,6 @@ static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
 	}
 
 	rqd->opcode = NVM_OP_HBREAD;
-	rrqd->addr = gp;
 
 	return NVM_IO_OK;
 }
@@ -784,31 +870,31 @@ static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
 static int rrpc_write_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
 			struct nvm_rq *rqd, unsigned long flags, int npages)
 {
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd);
-	struct rrpc_addr *p;
+	struct ppa_addr p;
 	sector_t laddr = rrpc_get_laddr(bio);
 	int is_gc = flags & NVM_IOTYPE_GC;
 	int i;
 
 	if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) {
-		nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
+		nvm_dev_dma_free(dev->parent, rqd->ppa_list, rqd->dma_ppa_list);
 		return NVM_IO_REQUEUE;
 	}
 
 	for (i = 0; i < npages; i++) {
 		/* We assume that mapping occurs at 4KB granularity */
 		p = rrpc_map_page(rrpc, laddr + i, is_gc);
-		if (!p) {
+		if (p.ppa == ADDR_EMPTY) {
 			BUG_ON(is_gc);
 			rrpc_unlock_laddr(rrpc, r);
-			nvm_dev_dma_free(rrpc->dev, rqd->ppa_list,
+			nvm_dev_dma_free(dev->parent, rqd->ppa_list,
 							rqd->dma_ppa_list);
 			rrpc_gc_kick(rrpc);
 			return NVM_IO_REQUEUE;
 		}
 
-		rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
-								p->addr);
+		rqd->ppa_list[i] = p;
 	}
 
 	rqd->opcode = NVM_OP_HBWRITE;
@@ -819,8 +905,7 @@ static int rrpc_write_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
 static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio,
 				struct nvm_rq *rqd, unsigned long flags)
 {
-	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
-	struct rrpc_addr *p;
+	struct ppa_addr p;
 	int is_gc = flags & NVM_IOTYPE_GC;
 	sector_t laddr = rrpc_get_laddr(bio);
 
@@ -828,16 +913,15 @@ static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio,
 		return NVM_IO_REQUEUE;
 
 	p = rrpc_map_page(rrpc, laddr, is_gc);
-	if (!p) {
+	if (p.ppa == ADDR_EMPTY) {
 		BUG_ON(is_gc);
 		rrpc_unlock_rq(rrpc, rqd);
 		rrpc_gc_kick(rrpc);
 		return NVM_IO_REQUEUE;
 	}
 
-	rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, p->addr);
+	rqd->ppa_addr = p;
 	rqd->opcode = NVM_OP_HBWRITE;
-	rrqd->addr = p;
 
 	return NVM_IO_OK;
 }
@@ -845,8 +929,10 @@ static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio,
 static int rrpc_setup_rq(struct rrpc *rrpc, struct bio *bio,
 			struct nvm_rq *rqd, unsigned long flags, uint8_t npages)
 {
+	struct nvm_tgt_dev *dev = rrpc->dev;
+
 	if (npages > 1) {
-		rqd->ppa_list = nvm_dev_dma_alloc(rrpc->dev, GFP_KERNEL,
+		rqd->ppa_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL,
 							&rqd->dma_ppa_list);
 		if (!rqd->ppa_list) {
 			pr_err("rrpc: not able to allocate ppa list\n");
@@ -869,14 +955,15 @@ static int rrpc_setup_rq(struct rrpc *rrpc, struct bio *bio,
 static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
 				struct nvm_rq *rqd, unsigned long flags)
 {
-	int err;
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	struct rrpc_rq *rrq = nvm_rq_to_pdu(rqd);
 	uint8_t nr_pages = rrpc_get_pages(bio);
 	int bio_size = bio_sectors(bio) << 9;
+	int err;
 
-	if (bio_size < rrpc->dev->sec_size)
+	if (bio_size < dev->geo.sec_size)
 		return NVM_IO_ERR;
-	else if (bio_size > rrpc->dev->max_rq_size)
+	else if (bio_size > dev->geo.max_rq_size)
 		return NVM_IO_ERR;
 
 	err = rrpc_setup_rq(rrpc, bio, rqd, flags, nr_pages);
@@ -889,15 +976,15 @@ static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
 	rqd->nr_ppas = nr_pages;
 	rrq->flags = flags;
 
-	err = nvm_submit_io(rrpc->dev, rqd);
+	err = nvm_submit_io(dev, rqd);
 	if (err) {
 		pr_err("rrpc: I/O submission failed: %d\n", err);
 		bio_put(bio);
 		if (!(flags & NVM_IOTYPE_GC)) {
 			rrpc_unlock_rq(rrpc, rqd);
 			if (rqd->nr_ppas > 1)
-				nvm_dev_dma_free(rrpc->dev,
-			rqd->ppa_list, rqd->dma_ppa_list);
+				nvm_dev_dma_free(dev->parent, rqd->ppa_list,
+							rqd->dma_ppa_list);
 		}
 		return NVM_IO_ERR;
 	}
@@ -911,6 +998,8 @@ static blk_qc_t rrpc_make_rq(struct request_queue *q, struct bio *bio)
 	struct nvm_rq *rqd;
 	int err;
 
+	blk_queue_split(q, &bio, q->bio_split);
+
 	if (bio_op(bio) == REQ_OP_DISCARD) {
 		rrpc_discard(rrpc, bio);
 		return BLK_QC_T_NONE;
@@ -997,25 +1086,24 @@ static void rrpc_map_free(struct rrpc *rrpc)
 static int rrpc_l2p_update(u64 slba, u32 nlb, __le64 *entries, void *private)
 {
 	struct rrpc *rrpc = (struct rrpc *)private;
-	struct nvm_dev *dev = rrpc->dev;
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	struct rrpc_addr *addr = rrpc->trans_map + slba;
 	struct rrpc_rev_addr *raddr = rrpc->rev_trans_map;
-	u64 elba = slba + nlb;
+	struct rrpc_lun *rlun;
+	struct rrpc_block *rblk;
 	u64 i;
 
-	if (unlikely(elba > dev->total_secs)) {
-		pr_err("nvm: L2P data from device is out of bounds!\n");
-		return -EINVAL;
-	}
-
 	for (i = 0; i < nlb; i++) {
+		struct ppa_addr gaddr;
 		u64 pba = le64_to_cpu(entries[i]);
 		unsigned int mod;
+
 		/* LNVM treats address-spaces as silos, LBA and PBA are
 		 * equally large and zero-indexed.
 		 */
 		if (unlikely(pba >= dev->total_secs && pba != U64_MAX)) {
 			pr_err("nvm: L2P data entry is out of bounds!\n");
+			pr_err("nvm: Maybe loaded an old target L2P\n");
 			return -EINVAL;
 		}
 
@@ -1028,7 +1116,27 @@ static int rrpc_l2p_update(u64 slba, u32 nlb, __le64 *entries, void *private)
 
 		div_u64_rem(pba, rrpc->nr_sects, &mod);
 
+		gaddr = rrpc_recov_addr(dev, pba);
+		rlun = rrpc_ppa_to_lun(rrpc, gaddr);
+		if (!rlun) {
+			pr_err("rrpc: l2p corruption on lba %llu\n",
+							slba + i);
+			return -EINVAL;
+		}
+
+		rblk = &rlun->blocks[gaddr.g.blk];
+		if (!rblk->state) {
+			/* at this point, we don't know anything about the
+			 * block. It's up to the FTL on top to re-etablish the
+			 * block state. The block is assumed to be open.
+			 */
+			list_move_tail(&rblk->list, &rlun->used_list);
+			rblk->state = NVM_BLK_ST_TGT;
+			rlun->nr_free_blocks--;
+		}
+
 		addr[i].addr = pba;
+		addr[i].rblk = rblk;
 		raddr[mod].addr = slba + i;
 	}
 
@@ -1037,7 +1145,7 @@ static int rrpc_l2p_update(u64 slba, u32 nlb, __le64 *entries, void *private)
 
 static int rrpc_map_init(struct rrpc *rrpc)
 {
-	struct nvm_dev *dev = rrpc->dev;
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	sector_t i;
 	int ret;
 
@@ -1058,12 +1166,9 @@ static int rrpc_map_init(struct rrpc *rrpc)
 		r->addr = ADDR_EMPTY;
 	}
 
-	if (!dev->ops->get_l2p_tbl)
-		return 0;
-
 	/* Bring up the mapping table from device */
-	ret = dev->ops->get_l2p_tbl(dev, rrpc->soffset, rrpc->nr_sects,
-					rrpc_l2p_update, rrpc);
+	ret = nvm_get_l2p_tbl(dev, rrpc->soffset, rrpc->nr_sects,
+							rrpc_l2p_update, rrpc);
 	if (ret) {
 		pr_err("nvm: rrpc: could not read L2P table.\n");
 		return -EINVAL;
@@ -1102,7 +1207,7 @@ static int rrpc_core_init(struct rrpc *rrpc)
 	if (!rrpc->page_pool)
 		return -ENOMEM;
 
-	rrpc->gcb_pool = mempool_create_slab_pool(rrpc->dev->nr_luns,
+	rrpc->gcb_pool = mempool_create_slab_pool(rrpc->dev->geo.nr_luns,
 								rrpc_gcb_cache);
 	if (!rrpc->gcb_pool)
 		return -ENOMEM;
@@ -1126,8 +1231,6 @@ static void rrpc_core_free(struct rrpc *rrpc)
 
 static void rrpc_luns_free(struct rrpc *rrpc)
 {
-	struct nvm_dev *dev = rrpc->dev;
-	struct nvm_lun *lun;
 	struct rrpc_lun *rlun;
 	int i;
 
@@ -1136,23 +1239,74 @@ static void rrpc_luns_free(struct rrpc *rrpc)
 
 	for (i = 0; i < rrpc->nr_luns; i++) {
 		rlun = &rrpc->luns[i];
-		lun = rlun->parent;
-		if (!lun)
-			break;
-		dev->mt->release_lun(dev, lun->id);
 		vfree(rlun->blocks);
 	}
 
 	kfree(rrpc->luns);
 }
 
-static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end)
+static int rrpc_bb_discovery(struct nvm_tgt_dev *dev, struct rrpc_lun *rlun)
 {
-	struct nvm_dev *dev = rrpc->dev;
+	struct nvm_geo *geo = &dev->geo;
+	struct rrpc_block *rblk;
+	struct ppa_addr ppa;
+	u8 *blks;
+	int nr_blks;
+	int i;
+	int ret;
+
+	if (!dev->parent->ops->get_bb_tbl)
+		return 0;
+
+	nr_blks = geo->blks_per_lun * geo->plane_mode;
+	blks = kmalloc(nr_blks, GFP_KERNEL);
+	if (!blks)
+		return -ENOMEM;
+
+	ppa.ppa = 0;
+	ppa.g.ch = rlun->bppa.g.ch;
+	ppa.g.lun = rlun->bppa.g.lun;
+
+	ret = nvm_get_tgt_bb_tbl(dev, ppa, blks);
+	if (ret) {
+		pr_err("rrpc: could not get BB table\n");
+		goto out;
+	}
+
+	nr_blks = nvm_bb_tbl_fold(dev->parent, blks, nr_blks);
+	if (nr_blks < 0)
+		return nr_blks;
+
+	for (i = 0; i < nr_blks; i++) {
+		if (blks[i] == NVM_BLK_T_FREE)
+			continue;
+
+		rblk = &rlun->blocks[i];
+		list_move_tail(&rblk->list, &rlun->bb_list);
+		rblk->state = NVM_BLK_ST_BAD;
+		rlun->nr_free_blocks--;
+	}
+
+out:
+	kfree(blks);
+	return ret;
+}
+
+static void rrpc_set_lun_ppa(struct rrpc_lun *rlun, struct ppa_addr ppa)
+{
+	rlun->bppa.ppa = 0;
+	rlun->bppa.g.ch = ppa.g.ch;
+	rlun->bppa.g.lun = ppa.g.lun;
+}
+
+static int rrpc_luns_init(struct rrpc *rrpc, struct ppa_addr *luns)
+{
+	struct nvm_tgt_dev *dev = rrpc->dev;
+	struct nvm_geo *geo = &dev->geo;
 	struct rrpc_lun *rlun;
 	int i, j, ret = -EINVAL;
 
-	if (dev->sec_per_blk > MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) {
+	if (geo->sec_per_blk > MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) {
 		pr_err("rrpc: number of pages per block too high.");
 		return -EINVAL;
 	}
@@ -1166,43 +1320,46 @@ static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end)
 
 	/* 1:1 mapping */
 	for (i = 0; i < rrpc->nr_luns; i++) {
-		int lunid = lun_begin + i;
-		struct nvm_lun *lun;
-
-		if (dev->mt->reserve_lun(dev, lunid)) {
-			pr_err("rrpc: lun %u is already allocated\n", lunid);
-			goto err;
-		}
-
-		lun = dev->mt->get_lun(dev, lunid);
-		if (!lun)
-			goto err;
-
 		rlun = &rrpc->luns[i];
-		rlun->parent = lun;
+		rlun->id = i;
+		rrpc_set_lun_ppa(rlun, luns[i]);
 		rlun->blocks = vzalloc(sizeof(struct rrpc_block) *
-						rrpc->dev->blks_per_lun);
+							geo->blks_per_lun);
 		if (!rlun->blocks) {
 			ret = -ENOMEM;
 			goto err;
 		}
 
-		for (j = 0; j < rrpc->dev->blks_per_lun; j++) {
+		INIT_LIST_HEAD(&rlun->free_list);
+		INIT_LIST_HEAD(&rlun->used_list);
+		INIT_LIST_HEAD(&rlun->bb_list);
+
+		for (j = 0; j < geo->blks_per_lun; j++) {
 			struct rrpc_block *rblk = &rlun->blocks[j];
-			struct nvm_block *blk = &lun->blocks[j];
 
-			rblk->parent = blk;
+			rblk->id = j;
 			rblk->rlun = rlun;
+			rblk->state = NVM_BLK_T_FREE;
 			INIT_LIST_HEAD(&rblk->prio);
+			INIT_LIST_HEAD(&rblk->list);
 			spin_lock_init(&rblk->lock);
+
+			list_add_tail(&rblk->list, &rlun->free_list);
 		}
 
 		rlun->rrpc = rrpc;
+		rlun->nr_free_blocks = geo->blks_per_lun;
+		rlun->reserved_blocks = 2; /* for GC only */
+
 		INIT_LIST_HEAD(&rlun->prio_list);
 		INIT_LIST_HEAD(&rlun->wblk_list);
 
 		INIT_WORK(&rlun->ws_gc, rrpc_lun_gc);
 		spin_lock_init(&rlun->lock);
+
+		if (rrpc_bb_discovery(dev, rlun))
+			goto err;
+
 	}
 
 	return 0;
@@ -1213,27 +1370,25 @@ err:
 /* returns 0 on success and stores the beginning address in *begin */
 static int rrpc_area_init(struct rrpc *rrpc, sector_t *begin)
 {
-	struct nvm_dev *dev = rrpc->dev;
-	struct nvmm_type *mt = dev->mt;
-	sector_t size = rrpc->nr_sects * dev->sec_size;
+	struct nvm_tgt_dev *dev = rrpc->dev;
+	sector_t size = rrpc->nr_sects * dev->geo.sec_size;
 	int ret;
 
 	size >>= 9;
 
-	ret = mt->get_area(dev, begin, size);
+	ret = nvm_get_area(dev, begin, size);
 	if (!ret)
-		*begin >>= (ilog2(dev->sec_size) - 9);
+		*begin >>= (ilog2(dev->geo.sec_size) - 9);
 
 	return ret;
 }
 
 static void rrpc_area_free(struct rrpc *rrpc)
 {
-	struct nvm_dev *dev = rrpc->dev;
-	struct nvmm_type *mt = dev->mt;
-	sector_t begin = rrpc->soffset << (ilog2(dev->sec_size) - 9);
+	struct nvm_tgt_dev *dev = rrpc->dev;
+	sector_t begin = rrpc->soffset << (ilog2(dev->geo.sec_size) - 9);
 
-	mt->put_area(dev, begin);
+	nvm_put_area(dev, begin);
 }
 
 static void rrpc_free(struct rrpc *rrpc)
@@ -1262,11 +1417,11 @@ static void rrpc_exit(void *private)
 static sector_t rrpc_capacity(void *private)
 {
 	struct rrpc *rrpc = private;
-	struct nvm_dev *dev = rrpc->dev;
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	sector_t reserved, provisioned;
 
 	/* cur, gc, and two emergency blocks for each lun */
-	reserved = rrpc->nr_luns * dev->sec_per_blk * 4;
+	reserved = rrpc->nr_luns * dev->geo.sec_per_blk * 4;
 	provisioned = rrpc->nr_sects - reserved;
 
 	if (reserved > rrpc->nr_sects) {
@@ -1285,13 +1440,13 @@ static sector_t rrpc_capacity(void *private)
  */
 static void rrpc_block_map_update(struct rrpc *rrpc, struct rrpc_block *rblk)
 {
-	struct nvm_dev *dev = rrpc->dev;
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	int offset;
 	struct rrpc_addr *laddr;
 	u64 bpaddr, paddr, pladdr;
 
 	bpaddr = block_to_rel_addr(rrpc, rblk);
-	for (offset = 0; offset < dev->sec_per_blk; offset++) {
+	for (offset = 0; offset < dev->geo.sec_per_blk; offset++) {
 		paddr = bpaddr + offset;
 
 		pladdr = rrpc->rev_trans_map[paddr].addr;
@@ -1311,6 +1466,7 @@ static void rrpc_block_map_update(struct rrpc *rrpc, struct rrpc_block *rblk)
 
 static int rrpc_blocks_init(struct rrpc *rrpc)
 {
+	struct nvm_tgt_dev *dev = rrpc->dev;
 	struct rrpc_lun *rlun;
 	struct rrpc_block *rblk;
 	int lun_iter, blk_iter;
@@ -1318,7 +1474,7 @@ static int rrpc_blocks_init(struct rrpc *rrpc)
 	for (lun_iter = 0; lun_iter < rrpc->nr_luns; lun_iter++) {
 		rlun = &rrpc->luns[lun_iter];
 
-		for (blk_iter = 0; blk_iter < rrpc->dev->blks_per_lun;
+		for (blk_iter = 0; blk_iter < dev->geo.blks_per_lun;
 								blk_iter++) {
 			rblk = &rlun->blocks[blk_iter];
 			rrpc_block_map_update(rrpc, rblk);
@@ -1357,11 +1513,11 @@ err:
 
 static struct nvm_tgt_type tt_rrpc;
 
-static void *rrpc_init(struct nvm_dev *dev, struct gendisk *tdisk,
-						int lun_begin, int lun_end)
+static void *rrpc_init(struct nvm_tgt_dev *dev, struct gendisk *tdisk)
 {
 	struct request_queue *bqueue = dev->q;
 	struct request_queue *tqueue = tdisk->queue;
+	struct nvm_geo *geo = &dev->geo;
 	struct rrpc *rrpc;
 	sector_t soffset;
 	int ret;
@@ -1384,9 +1540,8 @@ static void *rrpc_init(struct nvm_dev *dev, struct gendisk *tdisk,
 	spin_lock_init(&rrpc->bio_lock);
 	INIT_WORK(&rrpc->ws_requeue, rrpc_requeue);
 
-	rrpc->nr_luns = lun_end - lun_begin + 1;
-	rrpc->total_blocks = (unsigned long)dev->blks_per_lun * rrpc->nr_luns;
-	rrpc->nr_sects = (unsigned long long)dev->sec_per_lun * rrpc->nr_luns;
+	rrpc->nr_luns = geo->nr_luns;
+	rrpc->nr_sects = (unsigned long long)geo->sec_per_lun * rrpc->nr_luns;
 
 	/* simple round-robin strategy */
 	atomic_set(&rrpc->next_lun, -1);
@@ -1398,15 +1553,12 @@ static void *rrpc_init(struct nvm_dev *dev, struct gendisk *tdisk,
 	}
 	rrpc->soffset = soffset;
 
-	ret = rrpc_luns_init(rrpc, lun_begin, lun_end);
+	ret = rrpc_luns_init(rrpc, dev->luns);
 	if (ret) {
 		pr_err("nvm: rrpc: could not initialize luns\n");
 		goto err;
 	}
 
-	rrpc->poffset = dev->sec_per_lun * lun_begin;
-	rrpc->lun_offset = lun_begin;
-
 	ret = rrpc_core_init(rrpc);
 	if (ret) {
 		pr_err("nvm: rrpc: could not initialize core\n");
diff --git a/drivers/lightnvm/rrpc.h b/drivers/lightnvm/rrpc.h
index 5e87d52cb983..94e4d73116b2 100644
--- a/drivers/lightnvm/rrpc.h
+++ b/drivers/lightnvm/rrpc.h
@@ -48,14 +48,15 @@ struct rrpc_inflight_rq {
 
 struct rrpc_rq {
 	struct rrpc_inflight_rq inflight_rq;
-	struct rrpc_addr *addr;
 	unsigned long flags;
 };
 
 struct rrpc_block {
-	struct nvm_block *parent;
+	int id;				/* id inside of LUN */
 	struct rrpc_lun *rlun;
-	struct list_head prio;
+
+	struct list_head prio;		/* LUN CG list */
+	struct list_head list;		/* LUN free, used, bb list */
 
 #define MAX_INVALID_PAGES_STORAGE 8
 	/* Bitmap for invalid page intries */
@@ -65,21 +66,38 @@ struct rrpc_block {
 	/* number of pages that are invalid, wrt host page size */
 	unsigned int nr_invalid_pages;
 
+	int state;
+
 	spinlock_t lock;
 	atomic_t data_cmnt_size; /* data pages committed to stable storage */
 };
 
 struct rrpc_lun {
 	struct rrpc *rrpc;
-	struct nvm_lun *parent;
+
+	int id;
+	struct ppa_addr bppa;
+
 	struct rrpc_block *cur, *gc_cur;
 	struct rrpc_block *blocks;	/* Reference to block allocation */
 
 	struct list_head prio_list;	/* Blocks that may be GC'ed */
 	struct list_head wblk_list;	/* Queued blocks to be written to */
 
+	/* lun block lists */
+	struct list_head used_list;	/* In-use blocks */
+	struct list_head free_list;	/* Not used blocks i.e. released
+					 * and ready for use
+					 */
+	struct list_head bb_list;	/* Bad blocks. Mutually exclusive with
+					 * free_list and used_list
+					 */
+	unsigned int nr_free_blocks;	/* Number of unused blocks */
+
 	struct work_struct ws_gc;
 
+	int reserved_blocks;
+
 	spinlock_t lock;
 };
 
@@ -87,19 +105,16 @@ struct rrpc {
 	/* instance must be kept in top to resolve rrpc in unprep */
 	struct nvm_tgt_instance instance;
 
-	struct nvm_dev *dev;
+	struct nvm_tgt_dev *dev;
 	struct gendisk *disk;
 
 	sector_t soffset; /* logical sector offset */
-	u64 poffset; /* physical page offset */
-	int lun_offset;
 
 	int nr_luns;
 	struct rrpc_lun *luns;
 
 	/* calculated values */
 	unsigned long long nr_sects;
-	unsigned long total_blocks;
 
 	/* Write strategy variables. Move these into each for structure for each
 	 * strategy
@@ -150,13 +165,37 @@ struct rrpc_rev_addr {
 	u64 addr;
 };
 
-static inline struct rrpc_block *rrpc_get_rblk(struct rrpc_lun *rlun,
-								int blk_id)
+static inline struct ppa_addr rrpc_linear_to_generic_addr(struct nvm_geo *geo,
+							  struct ppa_addr r)
+{
+	struct ppa_addr l;
+	int secs, pgs;
+	sector_t ppa = r.ppa;
+
+	l.ppa = 0;
+
+	div_u64_rem(ppa, geo->sec_per_pg, &secs);
+	l.g.sec = secs;
+
+	sector_div(ppa, geo->sec_per_pg);
+	div_u64_rem(ppa, geo->pgs_per_blk, &pgs);
+	l.g.pg = pgs;
+
+	return l;
+}
+
+static inline struct ppa_addr rrpc_recov_addr(struct nvm_tgt_dev *dev, u64 pba)
+{
+	return linear_to_generic_addr(&dev->geo, pba);
+}
+
+static inline u64 rrpc_blk_to_ppa(struct rrpc *rrpc, struct rrpc_block *rblk)
 {
-	struct rrpc *rrpc = rlun->rrpc;
-	int lun_blk = blk_id % rrpc->dev->blks_per_lun;
+	struct nvm_tgt_dev *dev = rrpc->dev;
+	struct nvm_geo *geo = &dev->geo;
+	struct rrpc_lun *rlun = rblk->rlun;
 
-	return &rlun->blocks[lun_blk];
+	return (rlun->id * geo->sec_per_lun) + (rblk->id * geo->sec_per_blk);
 }
 
 static inline sector_t rrpc_get_laddr(struct bio *bio)
diff --git a/drivers/lightnvm/sysblk.c b/drivers/lightnvm/sysblk.c
index a75bd28aaca3..12002bf4efc2 100644
--- a/drivers/lightnvm/sysblk.c
+++ b/drivers/lightnvm/sysblk.c
@@ -62,7 +62,8 @@ static void nvm_cpu_to_sysblk(struct nvm_system_block *sb,
 
 static int nvm_setup_sysblks(struct nvm_dev *dev, struct ppa_addr *sysblk_ppas)
 {
-	int nr_rows = min_t(int, MAX_SYSBLKS, dev->nr_chnls);
+	struct nvm_geo *geo = &dev->geo;
+	int nr_rows = min_t(int, MAX_SYSBLKS, geo->nr_chnls);
 	int i;
 
 	for (i = 0; i < nr_rows; i++)
@@ -71,7 +72,7 @@ static int nvm_setup_sysblks(struct nvm_dev *dev, struct ppa_addr *sysblk_ppas)
 	/* if possible, place sysblk at first channel, middle channel and last
 	 * channel of the device. If not, create only one or two sys blocks
 	 */
-	switch (dev->nr_chnls) {
+	switch (geo->nr_chnls) {
 	case 2:
 		sysblk_ppas[1].g.ch = 1;
 		/* fall-through */
@@ -80,8 +81,8 @@ static int nvm_setup_sysblks(struct nvm_dev *dev, struct ppa_addr *sysblk_ppas)
 		break;
 	default:
 		sysblk_ppas[0].g.ch = 0;
-		sysblk_ppas[1].g.ch = dev->nr_chnls / 2;
-		sysblk_ppas[2].g.ch = dev->nr_chnls - 1;
+		sysblk_ppas[1].g.ch = geo->nr_chnls / 2;
+		sysblk_ppas[2].g.ch = geo->nr_chnls - 1;
 		break;
 	}
 
@@ -162,11 +163,12 @@ static int sysblk_get_host_blks(struct nvm_dev *dev, struct ppa_addr ppa,
 static int nvm_get_all_sysblks(struct nvm_dev *dev, struct sysblk_scan *s,
 				struct ppa_addr *ppas, int get_free)
 {
+	struct nvm_geo *geo = &dev->geo;
 	int i, nr_blks, ret = 0;
 	u8 *blks;
 
 	s->nr_ppas = 0;
-	nr_blks = dev->blks_per_lun * dev->plane_mode;
+	nr_blks = geo->blks_per_lun * geo->plane_mode;
 
 	blks = kmalloc(nr_blks, GFP_KERNEL);
 	if (!blks)
@@ -210,13 +212,14 @@ err_get:
 static int nvm_scan_block(struct nvm_dev *dev, struct ppa_addr *ppa,
 						struct nvm_system_block *sblk)
 {
+	struct nvm_geo *geo = &dev->geo;
 	struct nvm_system_block *cur;
 	int pg, ret, found = 0;
 
 	/* the full buffer for a flash page is allocated. Only the first of it
 	 * contains the system block information
 	 */
-	cur = kmalloc(dev->pfpg_size, GFP_KERNEL);
+	cur = kmalloc(geo->pfpg_size, GFP_KERNEL);
 	if (!cur)
 		return -ENOMEM;
 
@@ -225,7 +228,7 @@ static int nvm_scan_block(struct nvm_dev *dev, struct ppa_addr *ppa,
 		ppa->g.pg = ppa_to_slc(dev, pg);
 
 		ret = nvm_submit_ppa(dev, ppa, 1, NVM_OP_PREAD, NVM_IO_SLC_MODE,
-							cur, dev->pfpg_size);
+							cur, geo->pfpg_size);
 		if (ret) {
 			if (ret == NVM_RSP_ERR_EMPTYPAGE) {
 				pr_debug("nvm: sysblk scan empty ppa (%u %u %u %u)\n",
@@ -267,34 +270,16 @@ static int nvm_scan_block(struct nvm_dev *dev, struct ppa_addr *ppa,
 	return found;
 }
 
-static int nvm_set_bb_tbl(struct nvm_dev *dev, struct sysblk_scan *s, int type)
+static int nvm_sysblk_set_bb_tbl(struct nvm_dev *dev, struct sysblk_scan *s,
+								int type)
 {
-	struct nvm_rq rqd;
-	int ret;
-
-	if (s->nr_ppas > dev->ops->max_phys_sect) {
-		pr_err("nvm: unable to update all sysblocks atomically\n");
-		return -EINVAL;
-	}
-
-	memset(&rqd, 0, sizeof(struct nvm_rq));
-
-	nvm_set_rqd_ppalist(dev, &rqd, s->ppas, s->nr_ppas, 1);
-	nvm_generic_to_addr_mode(dev, &rqd);
-
-	ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type);
-	nvm_free_rqd_ppalist(dev, &rqd);
-	if (ret) {
-		pr_err("nvm: sysblk failed bb mark\n");
-		return -EINVAL;
-	}
-
-	return 0;
+	return nvm_set_bb_tbl(dev, s->ppas, s->nr_ppas, type);
 }
 
 static int nvm_write_and_verify(struct nvm_dev *dev, struct nvm_sb_info *info,
 							struct sysblk_scan *s)
 {
+	struct nvm_geo *geo = &dev->geo;
 	struct nvm_system_block nvmsb;
 	void *buf;
 	int i, sect, ret = 0;
@@ -302,12 +287,12 @@ static int nvm_write_and_verify(struct nvm_dev *dev, struct nvm_sb_info *info,
 
 	nvm_cpu_to_sysblk(&nvmsb, info);
 
-	buf = kzalloc(dev->pfpg_size, GFP_KERNEL);
+	buf = kzalloc(geo->pfpg_size, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 	memcpy(buf, &nvmsb, sizeof(struct nvm_system_block));
 
-	ppas = kcalloc(dev->sec_per_pg, sizeof(struct ppa_addr), GFP_KERNEL);
+	ppas = kcalloc(geo->sec_per_pg, sizeof(struct ppa_addr), GFP_KERNEL);
 	if (!ppas) {
 		ret = -ENOMEM;
 		goto err;
@@ -324,15 +309,15 @@ static int nvm_write_and_verify(struct nvm_dev *dev, struct nvm_sb_info *info,
 							ppas[0].g.pg);
 
 		/* Expand to all sectors within a flash page */
-		if (dev->sec_per_pg > 1) {
-			for (sect = 1; sect < dev->sec_per_pg; sect++) {
+		if (geo->sec_per_pg > 1) {
+			for (sect = 1; sect < geo->sec_per_pg; sect++) {
 				ppas[sect].ppa = ppas[0].ppa;
 				ppas[sect].g.sec = sect;
 			}
 		}
 
-		ret = nvm_submit_ppa(dev, ppas, dev->sec_per_pg, NVM_OP_PWRITE,
-					NVM_IO_SLC_MODE, buf, dev->pfpg_size);
+		ret = nvm_submit_ppa(dev, ppas, geo->sec_per_pg, NVM_OP_PWRITE,
+					NVM_IO_SLC_MODE, buf, geo->pfpg_size);
 		if (ret) {
 			pr_err("nvm: sysblk failed program (%u %u %u)\n",
 							ppas[0].g.ch,
@@ -341,8 +326,8 @@ static int nvm_write_and_verify(struct nvm_dev *dev, struct nvm_sb_info *info,
 			break;
 		}
 
-		ret = nvm_submit_ppa(dev, ppas, dev->sec_per_pg, NVM_OP_PREAD,
-					NVM_IO_SLC_MODE, buf, dev->pfpg_size);
+		ret = nvm_submit_ppa(dev, ppas, geo->sec_per_pg, NVM_OP_PREAD,
+					NVM_IO_SLC_MODE, buf, geo->pfpg_size);
 		if (ret) {
 			pr_err("nvm: sysblk failed read (%u %u %u)\n",
 							ppas[0].g.ch,
@@ -379,7 +364,7 @@ static int nvm_prepare_new_sysblks(struct nvm_dev *dev, struct sysblk_scan *s)
 		ppa = &s->ppas[scan_ppa_idx(i, nxt_blk)];
 		ppa->g.pg = ppa_to_slc(dev, 0);
 
-		ret = nvm_erase_ppa(dev, ppa, 1);
+		ret = nvm_erase_ppa(dev, ppa, 1, 0);
 		if (ret)
 			return ret;
 
@@ -546,6 +531,7 @@ err_sysblk:
 
 int nvm_init_sysblock(struct nvm_dev *dev, struct nvm_sb_info *info)
 {
+	struct nvm_geo *geo = &dev->geo;
 	struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
 	struct sysblk_scan s;
 	int ret;
@@ -560,7 +546,7 @@ int nvm_init_sysblock(struct nvm_dev *dev, struct nvm_sb_info *info)
 	if (!dev->ops->get_bb_tbl || !dev->ops->set_bb_tbl)
 		return -EINVAL;
 
-	if (!(dev->mccap & NVM_ID_CAP_SLC) || !dev->lps_per_blk) {
+	if (!(geo->mccap & NVM_ID_CAP_SLC) || !dev->lps_per_blk) {
 		pr_err("nvm: memory does not support SLC access\n");
 		return -EINVAL;
 	}
@@ -573,7 +559,7 @@ int nvm_init_sysblock(struct nvm_dev *dev, struct nvm_sb_info *info)
 	if (ret)
 		goto err_mark;
 
-	ret = nvm_set_bb_tbl(dev, &s, NVM_BLK_T_HOST);
+	ret = nvm_sysblk_set_bb_tbl(dev, &s, NVM_BLK_T_HOST);
 	if (ret)
 		goto err_mark;
 
@@ -590,11 +576,11 @@ static int factory_nblks(int nblks)
 	return (nblks + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
 }
 
-static unsigned int factory_blk_offset(struct nvm_dev *dev, struct ppa_addr ppa)
+static unsigned int factory_blk_offset(struct nvm_geo *geo, struct ppa_addr ppa)
 {
-	int nblks = factory_nblks(dev->blks_per_lun);
+	int nblks = factory_nblks(geo->blks_per_lun);
 
-	return ((ppa.g.ch * dev->luns_per_chnl * nblks) + (ppa.g.lun * nblks)) /
+	return ((ppa.g.ch * geo->luns_per_chnl * nblks) + (ppa.g.lun * nblks)) /
 								BITS_PER_LONG;
 }
 
@@ -608,7 +594,7 @@ static int nvm_factory_blks(struct nvm_dev *dev, struct ppa_addr ppa,
 	if (nr_blks < 0)
 		return nr_blks;
 
-	lunoff = factory_blk_offset(dev, ppa);
+	lunoff = factory_blk_offset(&dev->geo, ppa);
 
 	/* non-set bits correspond to the block must be erased */
 	for (i = 0; i < nr_blks; i++) {
@@ -637,19 +623,19 @@ static int nvm_factory_blks(struct nvm_dev *dev, struct ppa_addr ppa,
 static int nvm_fact_get_blks(struct nvm_dev *dev, struct ppa_addr *erase_list,
 					int max_ppas, unsigned long *blk_bitmap)
 {
+	struct nvm_geo *geo = &dev->geo;
 	struct ppa_addr ppa;
 	int ch, lun, blkid, idx, done = 0, ppa_cnt = 0;
 	unsigned long *offset;
 
 	while (!done) {
 		done = 1;
-		nvm_for_each_lun_ppa(dev, ppa, ch, lun) {
-			idx = factory_blk_offset(dev, ppa);
+		nvm_for_each_lun_ppa(geo, ppa, ch, lun) {
+			idx = factory_blk_offset(geo, ppa);
 			offset = &blk_bitmap[idx];
 
-			blkid = find_first_zero_bit(offset,
-						dev->blks_per_lun);
-			if (blkid >= dev->blks_per_lun)
+			blkid = find_first_zero_bit(offset, geo->blks_per_lun);
+			if (blkid >= geo->blks_per_lun)
 				continue;
 			set_bit(blkid, offset);
 
@@ -674,16 +660,17 @@ static int nvm_fact_get_blks(struct nvm_dev *dev, struct ppa_addr *erase_list,
 static int nvm_fact_select_blks(struct nvm_dev *dev, unsigned long *blk_bitmap,
 								int flags)
 {
+	struct nvm_geo *geo = &dev->geo;
 	struct ppa_addr ppa;
 	int ch, lun, nr_blks, ret = 0;
 	u8 *blks;
 
-	nr_blks = dev->blks_per_lun * dev->plane_mode;
+	nr_blks = geo->blks_per_lun * geo->plane_mode;
 	blks = kmalloc(nr_blks, GFP_KERNEL);
 	if (!blks)
 		return -ENOMEM;
 
-	nvm_for_each_lun_ppa(dev, ppa, ch, lun) {
+	nvm_for_each_lun_ppa(geo, ppa, ch, lun) {
 		ret = nvm_get_bb_tbl(dev, ppa, blks);
 		if (ret)
 			pr_err("nvm: failed bb tbl for ch%u lun%u\n",
@@ -701,14 +688,15 @@ static int nvm_fact_select_blks(struct nvm_dev *dev, unsigned long *blk_bitmap,
 
 int nvm_dev_factory(struct nvm_dev *dev, int flags)
 {
+	struct nvm_geo *geo = &dev->geo;
 	struct ppa_addr *ppas;
 	int ppa_cnt, ret = -ENOMEM;
-	int max_ppas = dev->ops->max_phys_sect / dev->nr_planes;
+	int max_ppas = dev->ops->max_phys_sect / geo->nr_planes;
 	struct ppa_addr sysblk_ppas[MAX_SYSBLKS];
 	struct sysblk_scan s;
 	unsigned long *blk_bitmap;
 
-	blk_bitmap = kzalloc(factory_nblks(dev->blks_per_lun) * dev->nr_luns,
+	blk_bitmap = kzalloc(factory_nblks(geo->blks_per_lun) * geo->nr_luns,
 								GFP_KERNEL);
 	if (!blk_bitmap)
 		return ret;
@@ -725,7 +713,7 @@ int nvm_dev_factory(struct nvm_dev *dev, int flags)
 	/* continue to erase until list of blks until empty */
 	while ((ppa_cnt =
 			nvm_fact_get_blks(dev, ppas, max_ppas, blk_bitmap)) > 0)
-		nvm_erase_ppa(dev, ppas, ppa_cnt);
+		nvm_erase_ppa(dev, ppas, ppa_cnt, 0);
 
 	/* mark host reserved blocks free */
 	if (flags & NVM_FACTORY_RESET_HOST_BLKS) {
@@ -733,7 +721,7 @@ int nvm_dev_factory(struct nvm_dev *dev, int flags)
 		mutex_lock(&dev->mlock);
 		ret = nvm_get_all_sysblks(dev, &s, sysblk_ppas, 0);
 		if (!ret)
-			ret = nvm_set_bb_tbl(dev, &s, NVM_BLK_T_FREE);
+			ret = nvm_sysblk_set_bb_tbl(dev, &s, NVM_BLK_T_FREE);
 		mutex_unlock(&dev->mlock);
 	}
 err_ppas:
diff --git a/drivers/lightnvm/sysfs.c b/drivers/lightnvm/sysfs.c
deleted file mode 100644
index 0338c27ab95a..000000000000
--- a/drivers/lightnvm/sysfs.c
+++ /dev/null
@@ -1,198 +0,0 @@
-#include <linux/kernel.h>
-#include <linux/lightnvm.h>
-#include <linux/miscdevice.h>
-#include <linux/kobject.h>
-#include <linux/blk-mq.h>
-
-#include "lightnvm.h"
-
-static ssize_t nvm_dev_attr_show(struct device *dev,
-				 struct device_attribute *dattr, char *page)
-{
-	struct nvm_dev *ndev = container_of(dev, struct nvm_dev, dev);
-	struct nvm_id *id = &ndev->identity;
-	struct nvm_id_group *grp = &id->groups[0];
-	struct attribute *attr = &dattr->attr;
-
-	if (strcmp(attr->name, "version") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", id->ver_id);
-	} else if (strcmp(attr->name, "vendor_opcode") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", id->vmnt);
-	} else if (strcmp(attr->name, "capabilities") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", id->cap);
-	} else if (strcmp(attr->name, "device_mode") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", id->dom);
-	} else if (strcmp(attr->name, "media_manager") == 0) {
-		if (!ndev->mt)
-			return scnprintf(page, PAGE_SIZE, "%s\n", "none");
-		return scnprintf(page, PAGE_SIZE, "%s\n", ndev->mt->name);
-	} else if (strcmp(attr->name, "ppa_format") == 0) {
-		return scnprintf(page, PAGE_SIZE,
-			"0x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n",
-			id->ppaf.ch_offset, id->ppaf.ch_len,
-			id->ppaf.lun_offset, id->ppaf.lun_len,
-			id->ppaf.pln_offset, id->ppaf.pln_len,
-			id->ppaf.blk_offset, id->ppaf.blk_len,
-			id->ppaf.pg_offset, id->ppaf.pg_len,
-			id->ppaf.sect_offset, id->ppaf.sect_len);
-	} else if (strcmp(attr->name, "media_type") == 0) {	/* u8 */
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->mtype);
-	} else if (strcmp(attr->name, "flash_media_type") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->fmtype);
-	} else if (strcmp(attr->name, "num_channels") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_ch);
-	} else if (strcmp(attr->name, "num_luns") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_lun);
-	} else if (strcmp(attr->name, "num_planes") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_pln);
-	} else if (strcmp(attr->name, "num_blocks") == 0) {	/* u16 */
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_blk);
-	} else if (strcmp(attr->name, "num_pages") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_pg);
-	} else if (strcmp(attr->name, "page_size") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->fpg_sz);
-	} else if (strcmp(attr->name, "hw_sector_size") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->csecs);
-	} else if (strcmp(attr->name, "oob_sector_size") == 0) {/* u32 */
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->sos);
-	} else if (strcmp(attr->name, "read_typ") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->trdt);
-	} else if (strcmp(attr->name, "read_max") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->trdm);
-	} else if (strcmp(attr->name, "prog_typ") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->tprt);
-	} else if (strcmp(attr->name, "prog_max") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->tprm);
-	} else if (strcmp(attr->name, "erase_typ") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->tbet);
-	} else if (strcmp(attr->name, "erase_max") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->tbem);
-	} else if (strcmp(attr->name, "multiplane_modes") == 0) {
-		return scnprintf(page, PAGE_SIZE, "0x%08x\n", grp->mpos);
-	} else if (strcmp(attr->name, "media_capabilities") == 0) {
-		return scnprintf(page, PAGE_SIZE, "0x%08x\n", grp->mccap);
-	} else if (strcmp(attr->name, "max_phys_secs") == 0) {
-		return scnprintf(page, PAGE_SIZE, "%u\n",
-				ndev->ops->max_phys_sect);
-	} else {
-		return scnprintf(page,
-				 PAGE_SIZE,
-				 "Unhandled attr(%s) in `nvm_dev_attr_show`\n",
-				 attr->name);
-	}
-}
-
-#define NVM_DEV_ATTR_RO(_name)						\
-	DEVICE_ATTR(_name, S_IRUGO, nvm_dev_attr_show, NULL)
-
-static NVM_DEV_ATTR_RO(version);
-static NVM_DEV_ATTR_RO(vendor_opcode);
-static NVM_DEV_ATTR_RO(capabilities);
-static NVM_DEV_ATTR_RO(device_mode);
-static NVM_DEV_ATTR_RO(ppa_format);
-static NVM_DEV_ATTR_RO(media_manager);
-
-static NVM_DEV_ATTR_RO(media_type);
-static NVM_DEV_ATTR_RO(flash_media_type);
-static NVM_DEV_ATTR_RO(num_channels);
-static NVM_DEV_ATTR_RO(num_luns);
-static NVM_DEV_ATTR_RO(num_planes);
-static NVM_DEV_ATTR_RO(num_blocks);
-static NVM_DEV_ATTR_RO(num_pages);
-static NVM_DEV_ATTR_RO(page_size);
-static NVM_DEV_ATTR_RO(hw_sector_size);
-static NVM_DEV_ATTR_RO(oob_sector_size);
-static NVM_DEV_ATTR_RO(read_typ);
-static NVM_DEV_ATTR_RO(read_max);
-static NVM_DEV_ATTR_RO(prog_typ);
-static NVM_DEV_ATTR_RO(prog_max);
-static NVM_DEV_ATTR_RO(erase_typ);
-static NVM_DEV_ATTR_RO(erase_max);
-static NVM_DEV_ATTR_RO(multiplane_modes);
-static NVM_DEV_ATTR_RO(media_capabilities);
-static NVM_DEV_ATTR_RO(max_phys_secs);
-
-#define NVM_DEV_ATTR(_name) (dev_attr_##_name##)
-
-static struct attribute *nvm_dev_attrs[] = {
-	&dev_attr_version.attr,
-	&dev_attr_vendor_opcode.attr,
-	&dev_attr_capabilities.attr,
-	&dev_attr_device_mode.attr,
-	&dev_attr_media_manager.attr,
-
-	&dev_attr_ppa_format.attr,
-	&dev_attr_media_type.attr,
-	&dev_attr_flash_media_type.attr,
-	&dev_attr_num_channels.attr,
-	&dev_attr_num_luns.attr,
-	&dev_attr_num_planes.attr,
-	&dev_attr_num_blocks.attr,
-	&dev_attr_num_pages.attr,
-	&dev_attr_page_size.attr,
-	&dev_attr_hw_sector_size.attr,
-	&dev_attr_oob_sector_size.attr,
-	&dev_attr_read_typ.attr,
-	&dev_attr_read_max.attr,
-	&dev_attr_prog_typ.attr,
-	&dev_attr_prog_max.attr,
-	&dev_attr_erase_typ.attr,
-	&dev_attr_erase_max.attr,
-	&dev_attr_multiplane_modes.attr,
-	&dev_attr_media_capabilities.attr,
-	&dev_attr_max_phys_secs.attr,
-	NULL,
-};
-
-static struct attribute_group nvm_dev_attr_group = {
-	.name = "lightnvm",
-	.attrs = nvm_dev_attrs,
-};
-
-static const struct attribute_group *nvm_dev_attr_groups[] = {
-	&nvm_dev_attr_group,
-	NULL,
-};
-
-static void nvm_dev_release(struct device *device)
-{
-	struct nvm_dev *dev = container_of(device, struct nvm_dev, dev);
-	struct request_queue *q = dev->q;
-
-	pr_debug("nvm/sysfs: `nvm_dev_release`\n");
-
-	blk_mq_unregister_dev(device, q);
-
-	nvm_free(dev);
-}
-
-static struct device_type nvm_type = {
-	.name		= "lightnvm",
-	.groups		= nvm_dev_attr_groups,
-	.release	= nvm_dev_release,
-};
-
-int nvm_sysfs_register_dev(struct nvm_dev *dev)
-{
-	int ret;
-
-	if (!dev->parent_dev)
-		return 0;
-
-	dev->dev.parent = dev->parent_dev;
-	dev_set_name(&dev->dev, "%s", dev->name);
-	dev->dev.type = &nvm_type;
-	device_initialize(&dev->dev);
-	ret = device_add(&dev->dev);
-
-	if (!ret)
-		blk_mq_register_dev(&dev->dev, dev->q);
-
-	return ret;
-}
-
-void nvm_sysfs_unregister_dev(struct nvm_dev *dev)
-{
-	if (dev && dev->parent_dev)
-		kobject_put(&dev->dev.kobj);
-}
diff --git a/drivers/macintosh/Kconfig b/drivers/macintosh/Kconfig
index d28690f6e262..5d80810934df 100644
--- a/drivers/macintosh/Kconfig
+++ b/drivers/macintosh/Kconfig
@@ -102,7 +102,6 @@ config ADB_PMU_LED_DISK
 	bool "Use front LED as DISK LED by default"
 	depends on ADB_PMU_LED
 	depends on LEDS_CLASS
-	depends on IDE_GD_ATA
 	select LEDS_TRIGGERS
 	select LEDS_TRIGGER_DISK
 	help
diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig
index 11eebfe8a4cb..ceff415f201c 100644
--- a/drivers/mailbox/Kconfig
+++ b/drivers/mailbox/Kconfig
@@ -124,6 +124,15 @@ config MAILBOX_TEST
 	  Test client to help with testing new Controller driver
 	  implementations.
 
+config TEGRA_HSP_MBOX
+	bool "Tegra HSP (Hardware Synchronization Primitives) Driver"
+	depends on ARCH_TEGRA_186_SOC
+	help
+	  The Tegra HSP driver is used for the interprocessor communication
+	  between different remote processors and host processors on Tegra186
+	  and later SoCs. Say Y here if you want to have this support.
+	  If unsure say N.
+
 config XGENE_SLIMPRO_MBOX
 	tristate "APM SoC X-Gene SLIMpro Mailbox Controller"
 	depends on ARCH_XGENE
diff --git a/drivers/mailbox/Makefile b/drivers/mailbox/Makefile
index ace6fed8fea9..7dde4f609ae8 100644
--- a/drivers/mailbox/Makefile
+++ b/drivers/mailbox/Makefile
@@ -29,3 +29,5 @@ obj-$(CONFIG_XGENE_SLIMPRO_MBOX) += mailbox-xgene-slimpro.o
 obj-$(CONFIG_HI6220_MBOX)	+= hi6220-mailbox.o
 
 obj-$(CONFIG_BCM_PDC_MBOX)	+= bcm-pdc-mailbox.o
+
+obj-$(CONFIG_TEGRA_HSP_MBOX)	+= tegra-hsp.o
diff --git a/drivers/mailbox/tegra-hsp.c b/drivers/mailbox/tegra-hsp.c
new file mode 100644
index 000000000000..0cde356c11ab
--- /dev/null
+++ b/drivers/mailbox/tegra-hsp.c
@@ -0,0 +1,479 @@
+/*
+ * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/mailbox_controller.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include <dt-bindings/mailbox/tegra186-hsp.h>
+
+#define HSP_INT_DIMENSIONING	0x380
+#define HSP_nSM_SHIFT		0
+#define HSP_nSS_SHIFT		4
+#define HSP_nAS_SHIFT		8
+#define HSP_nDB_SHIFT		12
+#define HSP_nSI_SHIFT		16
+#define HSP_nINT_MASK		0xf
+
+#define HSP_DB_TRIGGER	0x0
+#define HSP_DB_ENABLE	0x4
+#define HSP_DB_RAW	0x8
+#define HSP_DB_PENDING	0xc
+
+#define HSP_DB_CCPLEX		1
+#define HSP_DB_BPMP		3
+#define HSP_DB_MAX		7
+
+struct tegra_hsp_channel;
+struct tegra_hsp;
+
+struct tegra_hsp_channel {
+	struct tegra_hsp *hsp;
+	struct mbox_chan *chan;
+	void __iomem *regs;
+};
+
+struct tegra_hsp_doorbell {
+	struct tegra_hsp_channel channel;
+	struct list_head list;
+	const char *name;
+	unsigned int master;
+	unsigned int index;
+};
+
+struct tegra_hsp_db_map {
+	const char *name;
+	unsigned int master;
+	unsigned int index;
+};
+
+struct tegra_hsp_soc {
+	const struct tegra_hsp_db_map *map;
+};
+
+struct tegra_hsp {
+	const struct tegra_hsp_soc *soc;
+	struct mbox_controller mbox;
+	void __iomem *regs;
+	unsigned int irq;
+	unsigned int num_sm;
+	unsigned int num_as;
+	unsigned int num_ss;
+	unsigned int num_db;
+	unsigned int num_si;
+	spinlock_t lock;
+
+	struct list_head doorbells;
+};
+
+static inline struct tegra_hsp *
+to_tegra_hsp(struct mbox_controller *mbox)
+{
+	return container_of(mbox, struct tegra_hsp, mbox);
+}
+
+static inline u32 tegra_hsp_readl(struct tegra_hsp *hsp, unsigned int offset)
+{
+	return readl(hsp->regs + offset);
+}
+
+static inline void tegra_hsp_writel(struct tegra_hsp *hsp, u32 value,
+				    unsigned int offset)
+{
+	writel(value, hsp->regs + offset);
+}
+
+static inline u32 tegra_hsp_channel_readl(struct tegra_hsp_channel *channel,
+					  unsigned int offset)
+{
+	return readl(channel->regs + offset);
+}
+
+static inline void tegra_hsp_channel_writel(struct tegra_hsp_channel *channel,
+					    u32 value, unsigned int offset)
+{
+	writel(value, channel->regs + offset);
+}
+
+static bool tegra_hsp_doorbell_can_ring(struct tegra_hsp_doorbell *db)
+{
+	u32 value;
+
+	value = tegra_hsp_channel_readl(&db->channel, HSP_DB_ENABLE);
+
+	return (value & BIT(TEGRA_HSP_DB_MASTER_CCPLEX)) != 0;
+}
+
+static struct tegra_hsp_doorbell *
+__tegra_hsp_doorbell_get(struct tegra_hsp *hsp, unsigned int master)
+{
+	struct tegra_hsp_doorbell *entry;
+
+	list_for_each_entry(entry, &hsp->doorbells, list)
+		if (entry->master == master)
+			return entry;
+
+	return NULL;
+}
+
+static struct tegra_hsp_doorbell *
+tegra_hsp_doorbell_get(struct tegra_hsp *hsp, unsigned int master)
+{
+	struct tegra_hsp_doorbell *db;
+	unsigned long flags;
+
+	spin_lock_irqsave(&hsp->lock, flags);
+	db = __tegra_hsp_doorbell_get(hsp, master);
+	spin_unlock_irqrestore(&hsp->lock, flags);
+
+	return db;
+}
+
+static irqreturn_t tegra_hsp_doorbell_irq(int irq, void *data)
+{
+	struct tegra_hsp *hsp = data;
+	struct tegra_hsp_doorbell *db;
+	unsigned long master, value;
+
+	db = tegra_hsp_doorbell_get(hsp, TEGRA_HSP_DB_MASTER_CCPLEX);
+	if (!db)
+		return IRQ_NONE;
+
+	value = tegra_hsp_channel_readl(&db->channel, HSP_DB_PENDING);
+	tegra_hsp_channel_writel(&db->channel, value, HSP_DB_PENDING);
+
+	spin_lock(&hsp->lock);
+
+	for_each_set_bit(master, &value, hsp->mbox.num_chans) {
+		struct tegra_hsp_doorbell *db;
+
+		db = __tegra_hsp_doorbell_get(hsp, master);
+		/*
+		 * Depending on the bootloader chain, the CCPLEX doorbell will
+		 * have some doorbells enabled, which means that requesting an
+		 * interrupt will immediately fire.
+		 *
+		 * In that case, db->channel.chan will still be NULL here and
+		 * cause a crash if not properly guarded.
+		 *
+		 * It remains to be seen if ignoring the doorbell in that case
+		 * is the correct solution.
+		 */
+		if (db && db->channel.chan)
+			mbox_chan_received_data(db->channel.chan, NULL);
+	}
+
+	spin_unlock(&hsp->lock);
+
+	return IRQ_HANDLED;
+}
+
+static struct tegra_hsp_channel *
+tegra_hsp_doorbell_create(struct tegra_hsp *hsp, const char *name,
+			  unsigned int master, unsigned int index)
+{
+	struct tegra_hsp_doorbell *db;
+	unsigned int offset;
+	unsigned long flags;
+
+	db = kzalloc(sizeof(*db), GFP_KERNEL);
+	if (!db)
+		return ERR_PTR(-ENOMEM);
+
+	offset = (1 + (hsp->num_sm / 2) + hsp->num_ss + hsp->num_as) << 16;
+	offset += index * 0x100;
+
+	db->channel.regs = hsp->regs + offset;
+	db->channel.hsp = hsp;
+
+	db->name = kstrdup_const(name, GFP_KERNEL);
+	db->master = master;
+	db->index = index;
+
+	spin_lock_irqsave(&hsp->lock, flags);
+	list_add_tail(&db->list, &hsp->doorbells);
+	spin_unlock_irqrestore(&hsp->lock, flags);
+
+	return &db->channel;
+}
+
+static void __tegra_hsp_doorbell_destroy(struct tegra_hsp_doorbell *db)
+{
+	list_del(&db->list);
+	kfree_const(db->name);
+	kfree(db);
+}
+
+static int tegra_hsp_doorbell_send_data(struct mbox_chan *chan, void *data)
+{
+	struct tegra_hsp_doorbell *db = chan->con_priv;
+
+	tegra_hsp_channel_writel(&db->channel, 1, HSP_DB_TRIGGER);
+
+	return 0;
+}
+
+static int tegra_hsp_doorbell_startup(struct mbox_chan *chan)
+{
+	struct tegra_hsp_doorbell *db = chan->con_priv;
+	struct tegra_hsp *hsp = db->channel.hsp;
+	struct tegra_hsp_doorbell *ccplex;
+	unsigned long flags;
+	u32 value;
+
+	if (db->master >= hsp->mbox.num_chans) {
+		dev_err(hsp->mbox.dev,
+			"invalid master ID %u for HSP channel\n",
+			db->master);
+		return -EINVAL;
+	}
+
+	ccplex = tegra_hsp_doorbell_get(hsp, TEGRA_HSP_DB_MASTER_CCPLEX);
+	if (!ccplex)
+		return -ENODEV;
+
+	if (!tegra_hsp_doorbell_can_ring(db))
+		return -ENODEV;
+
+	spin_lock_irqsave(&hsp->lock, flags);
+
+	value = tegra_hsp_channel_readl(&ccplex->channel, HSP_DB_ENABLE);
+	value |= BIT(db->master);
+	tegra_hsp_channel_writel(&ccplex->channel, value, HSP_DB_ENABLE);
+
+	spin_unlock_irqrestore(&hsp->lock, flags);
+
+	return 0;
+}
+
+static void tegra_hsp_doorbell_shutdown(struct mbox_chan *chan)
+{
+	struct tegra_hsp_doorbell *db = chan->con_priv;
+	struct tegra_hsp *hsp = db->channel.hsp;
+	struct tegra_hsp_doorbell *ccplex;
+	unsigned long flags;
+	u32 value;
+
+	ccplex = tegra_hsp_doorbell_get(hsp, TEGRA_HSP_DB_MASTER_CCPLEX);
+	if (!ccplex)
+		return;
+
+	spin_lock_irqsave(&hsp->lock, flags);
+
+	value = tegra_hsp_channel_readl(&ccplex->channel, HSP_DB_ENABLE);
+	value &= ~BIT(db->master);
+	tegra_hsp_channel_writel(&ccplex->channel, value, HSP_DB_ENABLE);
+
+	spin_unlock_irqrestore(&hsp->lock, flags);
+}
+
+static const struct mbox_chan_ops tegra_hsp_doorbell_ops = {
+	.send_data = tegra_hsp_doorbell_send_data,
+	.startup = tegra_hsp_doorbell_startup,
+	.shutdown = tegra_hsp_doorbell_shutdown,
+};
+
+static struct mbox_chan *of_tegra_hsp_xlate(struct mbox_controller *mbox,
+					    const struct of_phandle_args *args)
+{
+	struct tegra_hsp_channel *channel = ERR_PTR(-ENODEV);
+	struct tegra_hsp *hsp = to_tegra_hsp(mbox);
+	unsigned int type = args->args[0];
+	unsigned int master = args->args[1];
+	struct tegra_hsp_doorbell *db;
+	struct mbox_chan *chan;
+	unsigned long flags;
+	unsigned int i;
+
+	switch (type) {
+	case TEGRA_HSP_MBOX_TYPE_DB:
+		db = tegra_hsp_doorbell_get(hsp, master);
+		if (db)
+			channel = &db->channel;
+
+		break;
+
+	default:
+		break;
+	}
+
+	if (IS_ERR(channel))
+		return ERR_CAST(channel);
+
+	spin_lock_irqsave(&hsp->lock, flags);
+
+	for (i = 0; i < hsp->mbox.num_chans; i++) {
+		chan = &hsp->mbox.chans[i];
+		if (!chan->con_priv) {
+			chan->con_priv = channel;
+			channel->chan = chan;
+			break;
+		}
+
+		chan = NULL;
+	}
+
+	spin_unlock_irqrestore(&hsp->lock, flags);
+
+	return chan ?: ERR_PTR(-EBUSY);
+}
+
+static void tegra_hsp_remove_doorbells(struct tegra_hsp *hsp)
+{
+	struct tegra_hsp_doorbell *db, *tmp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&hsp->lock, flags);
+
+	list_for_each_entry_safe(db, tmp, &hsp->doorbells, list)
+		__tegra_hsp_doorbell_destroy(db);
+
+	spin_unlock_irqrestore(&hsp->lock, flags);
+}
+
+static int tegra_hsp_add_doorbells(struct tegra_hsp *hsp)
+{
+	const struct tegra_hsp_db_map *map = hsp->soc->map;
+	struct tegra_hsp_channel *channel;
+
+	while (map->name) {
+		channel = tegra_hsp_doorbell_create(hsp, map->name,
+						    map->master, map->index);
+		if (IS_ERR(channel)) {
+			tegra_hsp_remove_doorbells(hsp);
+			return PTR_ERR(channel);
+		}
+
+		map++;
+	}
+
+	return 0;
+}
+
+static int tegra_hsp_probe(struct platform_device *pdev)
+{
+	struct tegra_hsp *hsp;
+	struct resource *res;
+	u32 value;
+	int err;
+
+	hsp = devm_kzalloc(&pdev->dev, sizeof(*hsp), GFP_KERNEL);
+	if (!hsp)
+		return -ENOMEM;
+
+	hsp->soc = of_device_get_match_data(&pdev->dev);
+	INIT_LIST_HEAD(&hsp->doorbells);
+	spin_lock_init(&hsp->lock);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	hsp->regs = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(hsp->regs))
+		return PTR_ERR(hsp->regs);
+
+	value = tegra_hsp_readl(hsp, HSP_INT_DIMENSIONING);
+	hsp->num_sm = (value >> HSP_nSM_SHIFT) & HSP_nINT_MASK;
+	hsp->num_ss = (value >> HSP_nSS_SHIFT) & HSP_nINT_MASK;
+	hsp->num_as = (value >> HSP_nAS_SHIFT) & HSP_nINT_MASK;
+	hsp->num_db = (value >> HSP_nDB_SHIFT) & HSP_nINT_MASK;
+	hsp->num_si = (value >> HSP_nSI_SHIFT) & HSP_nINT_MASK;
+
+	err = platform_get_irq_byname(pdev, "doorbell");
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to get doorbell IRQ: %d\n", err);
+		return err;
+	}
+
+	hsp->irq = err;
+
+	hsp->mbox.of_xlate = of_tegra_hsp_xlate;
+	hsp->mbox.num_chans = 32;
+	hsp->mbox.dev = &pdev->dev;
+	hsp->mbox.txdone_irq = false;
+	hsp->mbox.txdone_poll = false;
+	hsp->mbox.ops = &tegra_hsp_doorbell_ops;
+
+	hsp->mbox.chans = devm_kcalloc(&pdev->dev, hsp->mbox.num_chans,
+					sizeof(*hsp->mbox.chans),
+					GFP_KERNEL);
+	if (!hsp->mbox.chans)
+		return -ENOMEM;
+
+	err = tegra_hsp_add_doorbells(hsp);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to add doorbells: %d\n", err);
+		return err;
+	}
+
+	platform_set_drvdata(pdev, hsp);
+
+	err = mbox_controller_register(&hsp->mbox);
+	if (err) {
+		dev_err(&pdev->dev, "failed to register mailbox: %d\n", err);
+		tegra_hsp_remove_doorbells(hsp);
+		return err;
+	}
+
+	err = devm_request_irq(&pdev->dev, hsp->irq, tegra_hsp_doorbell_irq,
+			       IRQF_NO_SUSPEND, dev_name(&pdev->dev), hsp);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n",
+			hsp->irq, err);
+		return err;
+	}
+
+	return 0;
+}
+
+static int tegra_hsp_remove(struct platform_device *pdev)
+{
+	struct tegra_hsp *hsp = platform_get_drvdata(pdev);
+
+	mbox_controller_unregister(&hsp->mbox);
+	tegra_hsp_remove_doorbells(hsp);
+
+	return 0;
+}
+
+static const struct tegra_hsp_db_map tegra186_hsp_db_map[] = {
+	{ "ccplex", TEGRA_HSP_DB_MASTER_CCPLEX, HSP_DB_CCPLEX, },
+	{ "bpmp",   TEGRA_HSP_DB_MASTER_BPMP,   HSP_DB_BPMP,   },
+	{ /* sentinel */ }
+};
+
+static const struct tegra_hsp_soc tegra186_hsp_soc = {
+	.map = tegra186_hsp_db_map,
+};
+
+static const struct of_device_id tegra_hsp_match[] = {
+	{ .compatible = "nvidia,tegra186-hsp", .data = &tegra186_hsp_soc },
+	{ }
+};
+
+static struct platform_driver tegra_hsp_driver = {
+	.driver = {
+		.name = "tegra-hsp",
+		.of_match_table = tegra_hsp_match,
+	},
+	.probe = tegra_hsp_probe,
+	.remove = tegra_hsp_remove,
+};
+
+static int __init tegra_hsp_init(void)
+{
+	return platform_driver_register(&tegra_hsp_driver);
+}
+core_initcall(tegra_hsp_init);
diff --git a/drivers/mcb/mcb-parse.c b/drivers/mcb/mcb-parse.c
index 4ca2739b4fad..ee7fb6ec96bd 100644
--- a/drivers/mcb/mcb-parse.c
+++ b/drivers/mcb/mcb-parse.c
@@ -149,7 +149,7 @@ static int chameleon_get_bar(char __iomem **base, phys_addr_t mapbase,
 		reg = readl(*base);
 
 		bar_count = BAR_CNT(reg);
-		if (bar_count <= 0 && bar_count > CHAMELEON_BAR_MAX)
+		if (bar_count <= 0 || bar_count > CHAMELEON_BAR_MAX)
 			return -ENODEV;
 
 		c = kcalloc(bar_count, sizeof(struct chameleon_bar),
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index 02a5345a44a6..b7767da50c26 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -240,9 +240,17 @@ config DM_BUFIO
 	 as a cache, holding recently-read blocks in memory and performing
 	 delayed writes.
 
+config DM_DEBUG_BLOCK_MANAGER_LOCKING
+       bool "Block manager locking"
+       depends on DM_BUFIO
+       ---help---
+	 Block manager locking can catch various metadata corruption issues.
+
+	 If unsure, say N.
+
 config DM_DEBUG_BLOCK_STACK_TRACING
        bool "Keep stack trace of persistent data block lock holders"
-       depends on STACKTRACE_SUPPORT && DM_BUFIO
+       depends on STACKTRACE_SUPPORT && DM_DEBUG_BLOCK_MANAGER_LOCKING
        select STACKTRACE
        ---help---
 	 Enable this for messages that may help debug problems with the
diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
index 81d3db40cd7b..6fdd8e252760 100644
--- a/drivers/md/bcache/btree.c
+++ b/drivers/md/bcache/btree.c
@@ -297,7 +297,7 @@ static void bch_btree_node_read(struct btree *b)
 	bio->bi_iter.bi_size = KEY_SIZE(&b->key) << 9;
 	bio->bi_end_io	= btree_node_read_endio;
 	bio->bi_private	= &cl;
-	bio_set_op_attrs(bio, REQ_OP_READ, REQ_META|READ_SYNC);
+	bio->bi_opf = REQ_OP_READ | REQ_META;
 
 	bch_bio_map(bio, b->keys.set[0].data);
 
@@ -393,7 +393,7 @@ static void do_btree_node_write(struct btree *b)
 	b->bio->bi_end_io	= btree_node_write_endio;
 	b->bio->bi_private	= cl;
 	b->bio->bi_iter.bi_size	= roundup(set_bytes(i), block_bytes(b->c));
-	bio_set_op_attrs(b->bio, REQ_OP_WRITE, REQ_META|WRITE_SYNC|REQ_FUA);
+	b->bio->bi_opf		= REQ_OP_WRITE | REQ_META | REQ_FUA;
 	bch_bio_map(b->bio, i);
 
 	/*
diff --git a/drivers/md/bcache/debug.c b/drivers/md/bcache/debug.c
index 333a1e5f6ae6..06f55056aaae 100644
--- a/drivers/md/bcache/debug.c
+++ b/drivers/md/bcache/debug.c
@@ -52,7 +52,7 @@ void bch_btree_verify(struct btree *b)
 	bio->bi_bdev		= PTR_CACHE(b->c, &b->key, 0)->bdev;
 	bio->bi_iter.bi_sector	= PTR_OFFSET(&b->key, 0);
 	bio->bi_iter.bi_size	= KEY_SIZE(&v->key) << 9;
-	bio_set_op_attrs(bio, REQ_OP_READ, REQ_META|READ_SYNC);
+	bio->bi_opf		= REQ_OP_READ | REQ_META;
 	bch_bio_map(bio, sorted);
 
 	submit_bio_wait(bio);
@@ -107,22 +107,26 @@ void bch_data_verify(struct cached_dev *dc, struct bio *bio)
 {
 	char name[BDEVNAME_SIZE];
 	struct bio *check;
-	struct bio_vec bv;
-	struct bvec_iter iter;
+	struct bio_vec bv, cbv;
+	struct bvec_iter iter, citer = { 0 };
 
 	check = bio_clone(bio, GFP_NOIO);
 	if (!check)
 		return;
-	bio_set_op_attrs(check, REQ_OP_READ, READ_SYNC);
+	check->bi_opf = REQ_OP_READ;
 
 	if (bio_alloc_pages(check, GFP_NOIO))
 		goto out_put;
 
 	submit_bio_wait(check);
 
+	citer.bi_size = UINT_MAX;
 	bio_for_each_segment(bv, bio, iter) {
 		void *p1 = kmap_atomic(bv.bv_page);
-		void *p2 = page_address(check->bi_io_vec[iter.bi_idx].bv_page);
+		void *p2;
+
+		cbv = bio_iter_iovec(check, citer);
+		p2 = page_address(cbv.bv_page);
 
 		cache_set_err_on(memcmp(p1 + bv.bv_offset,
 					p2 + bv.bv_offset,
@@ -133,6 +137,7 @@ void bch_data_verify(struct cached_dev *dc, struct bio *bio)
 				 (uint64_t) bio->bi_iter.bi_sector);
 
 		kunmap_atomic(p1);
+		bio_advance_iter(check, &citer, bv.bv_len);
 	}
 
 	bio_free_pages(check);
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index e97b0acf7b8d..db45a88c0ce9 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -24,9 +24,7 @@ struct bio *bch_bbio_alloc(struct cache_set *c)
 	struct bbio *b = mempool_alloc(c->bio_meta, GFP_NOIO);
 	struct bio *bio = &b->bio;
 
-	bio_init(bio);
-	bio->bi_max_vecs	 = bucket_pages(c);
-	bio->bi_io_vec		 = bio->bi_inline_vecs;
+	bio_init(bio, bio->bi_inline_vecs, bucket_pages(c));
 
 	return bio;
 }
diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c
index 6925023e12d4..1198e53d5670 100644
--- a/drivers/md/bcache/journal.c
+++ b/drivers/md/bcache/journal.c
@@ -448,13 +448,11 @@ static void do_journal_discard(struct cache *ca)
 
 		atomic_set(&ja->discard_in_flight, DISCARD_IN_FLIGHT);
 
-		bio_init(bio);
+		bio_init(bio, bio->bi_inline_vecs, 1);
 		bio_set_op_attrs(bio, REQ_OP_DISCARD, 0);
 		bio->bi_iter.bi_sector	= bucket_to_sector(ca->set,
 						ca->sb.d[ja->discard_idx]);
 		bio->bi_bdev		= ca->bdev;
-		bio->bi_max_vecs	= 1;
-		bio->bi_io_vec		= bio->bi_inline_vecs;
 		bio->bi_iter.bi_size	= bucket_bytes(ca);
 		bio->bi_end_io		= journal_discard_endio;
 
diff --git a/drivers/md/bcache/movinggc.c b/drivers/md/bcache/movinggc.c
index 5c4bddecfaf0..13b8a907006d 100644
--- a/drivers/md/bcache/movinggc.c
+++ b/drivers/md/bcache/movinggc.c
@@ -77,15 +77,13 @@ static void moving_init(struct moving_io *io)
 {
 	struct bio *bio = &io->bio.bio;
 
-	bio_init(bio);
+	bio_init(bio, bio->bi_inline_vecs,
+		 DIV_ROUND_UP(KEY_SIZE(&io->w->key), PAGE_SECTORS));
 	bio_get(bio);
 	bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
 
 	bio->bi_iter.bi_size	= KEY_SIZE(&io->w->key) << 9;
-	bio->bi_max_vecs	= DIV_ROUND_UP(KEY_SIZE(&io->w->key),
-					       PAGE_SECTORS);
 	bio->bi_private		= &io->cl;
-	bio->bi_io_vec		= bio->bi_inline_vecs;
 	bch_bio_map(bio, NULL);
 }
 
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index 40ffe5e424b3..f49c5417527d 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -404,8 +404,8 @@ static bool check_should_bypass(struct cached_dev *dc, struct bio *bio)
 
 	if (!congested &&
 	    mode == CACHE_MODE_WRITEBACK &&
-	    op_is_write(bio_op(bio)) &&
-	    (bio->bi_opf & REQ_SYNC))
+	    op_is_write(bio->bi_opf) &&
+	    op_is_sync(bio->bi_opf))
 		goto rescale;
 
 	spin_lock(&dc->io_lock);
@@ -623,7 +623,7 @@ static void do_bio_hook(struct search *s, struct bio *orig_bio)
 {
 	struct bio *bio = &s->bio.bio;
 
-	bio_init(bio);
+	bio_init(bio, NULL, 0);
 	__bio_clone_fast(bio, orig_bio);
 	bio->bi_end_io		= request_endio;
 	bio->bi_private		= &s->cl;
@@ -923,7 +923,7 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)
 			flush->bi_bdev	= bio->bi_bdev;
 			flush->bi_end_io = request_endio;
 			flush->bi_private = cl;
-			bio_set_op_attrs(flush, REQ_OP_WRITE, WRITE_FLUSH);
+			flush->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 
 			closure_bio_submit(flush, cl);
 		}
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index 849ad441cd76..2fb5bfeb43e2 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -381,7 +381,7 @@ static char *uuid_read(struct cache_set *c, struct jset *j, struct closure *cl)
 		return "bad uuid pointer";
 
 	bkey_copy(&c->uuid_bucket, k);
-	uuid_io(c, REQ_OP_READ, READ_SYNC, k, cl);
+	uuid_io(c, REQ_OP_READ, 0, k, cl);
 
 	if (j->version < BCACHE_JSET_VERSION_UUIDv1) {
 		struct uuid_entry_v0	*u0 = (void *) c->uuids;
@@ -600,7 +600,7 @@ static void prio_read(struct cache *ca, uint64_t bucket)
 			ca->prio_last_buckets[bucket_nr] = bucket;
 			bucket_nr++;
 
-			prio_io(ca, bucket, REQ_OP_READ, READ_SYNC);
+			prio_io(ca, bucket, REQ_OP_READ, 0);
 
 			if (p->csum != bch_crc64(&p->magic, bucket_bytes(ca) - 8))
 				pr_warn("bad csum reading priorities");
@@ -1152,9 +1152,7 @@ static void register_bdev(struct cache_sb *sb, struct page *sb_page,
 	dc->bdev = bdev;
 	dc->bdev->bd_holder = dc;
 
-	bio_init(&dc->sb_bio);
-	dc->sb_bio.bi_max_vecs	= 1;
-	dc->sb_bio.bi_io_vec	= dc->sb_bio.bi_inline_vecs;
+	bio_init(&dc->sb_bio, dc->sb_bio.bi_inline_vecs, 1);
 	dc->sb_bio.bi_io_vec[0].bv_page = sb_page;
 	get_page(sb_page);
 
@@ -1814,9 +1812,7 @@ static int cache_alloc(struct cache *ca)
 	__module_get(THIS_MODULE);
 	kobject_init(&ca->kobj, &bch_cache_ktype);
 
-	bio_init(&ca->journal.bio);
-	ca->journal.bio.bi_max_vecs = 8;
-	ca->journal.bio.bi_io_vec = ca->journal.bio.bi_inline_vecs;
+	bio_init(&ca->journal.bio, ca->journal.bio.bi_inline_vecs, 8);
 
 	free = roundup_pow_of_two(ca->sb.nbuckets) >> 10;
 
@@ -1852,9 +1848,7 @@ static int register_cache(struct cache_sb *sb, struct page *sb_page,
 	ca->bdev = bdev;
 	ca->bdev->bd_holder = ca;
 
-	bio_init(&ca->sb_bio);
-	ca->sb_bio.bi_max_vecs	= 1;
-	ca->sb_bio.bi_io_vec	= ca->sb_bio.bi_inline_vecs;
+	bio_init(&ca->sb_bio, ca->sb_bio.bi_inline_vecs, 1);
 	ca->sb_bio.bi_io_vec[0].bv_page = sb_page;
 	get_page(sb_page);
 
diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c
index e51644e503a5..69e1ae59cab8 100644
--- a/drivers/md/bcache/writeback.c
+++ b/drivers/md/bcache/writeback.c
@@ -106,14 +106,13 @@ static void dirty_init(struct keybuf_key *w)
 	struct dirty_io *io = w->private;
 	struct bio *bio = &io->bio;
 
-	bio_init(bio);
+	bio_init(bio, bio->bi_inline_vecs,
+		 DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS));
 	if (!io->dc->writeback_percent)
 		bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
 
 	bio->bi_iter.bi_size	= KEY_SIZE(&w->key) << 9;
-	bio->bi_max_vecs	= DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS);
 	bio->bi_private		= w;
-	bio->bi_io_vec		= bio->bi_inline_vecs;
 	bch_bio_map(bio, NULL);
 }
 
diff --git a/drivers/md/bcache/writeback.h b/drivers/md/bcache/writeback.h
index 301eaf565167..629bd1a502fd 100644
--- a/drivers/md/bcache/writeback.h
+++ b/drivers/md/bcache/writeback.h
@@ -57,8 +57,7 @@ static inline bool should_writeback(struct cached_dev *dc, struct bio *bio,
 	if (would_skip)
 		return false;
 
-	return bio->bi_opf & REQ_SYNC ||
-		in_use <= CUTOFF_WRITEBACK;
+	return op_is_sync(bio->bi_opf) || in_use <= CUTOFF_WRITEBACK;
 }
 
 static inline void bch_writeback_queue(struct cached_dev *dc)
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
index 2d826927a3bf..9fb2ccac958a 100644
--- a/drivers/md/bitmap.c
+++ b/drivers/md/bitmap.c
@@ -27,6 +27,7 @@
 #include <linux/mount.h>
 #include <linux/buffer_head.h>
 #include <linux/seq_file.h>
+#include <trace/events/block.h>
 #include "md.h"
 #include "bitmap.h"
 
@@ -208,11 +209,13 @@ static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mdde
 
 static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
 {
-	struct md_rdev *rdev = NULL;
+	struct md_rdev *rdev;
 	struct block_device *bdev;
 	struct mddev *mddev = bitmap->mddev;
 	struct bitmap_storage *store = &bitmap->storage;
 
+restart:
+	rdev = NULL;
 	while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
 		int size = PAGE_SIZE;
 		loff_t offset = mddev->bitmap_info.offset;
@@ -268,8 +271,8 @@ static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
 			       page);
 	}
 
-	if (wait)
-		md_super_wait(mddev);
+	if (wait && md_super_wait(mddev) < 0)
+		goto restart;
 	return 0;
 
  bad_alignment:
@@ -405,10 +408,10 @@ static int read_page(struct file *file, unsigned long index,
 		ret = -EIO;
 out:
 	if (ret)
-		printk(KERN_ALERT "md: bitmap read error: (%dB @ %llu): %d\n",
-			(int)PAGE_SIZE,
-			(unsigned long long)index << PAGE_SHIFT,
-			ret);
+		pr_err("md: bitmap read error: (%dB @ %llu): %d\n",
+		       (int)PAGE_SIZE,
+		       (unsigned long long)index << PAGE_SHIFT,
+		       ret);
 	return ret;
 }
 
@@ -416,6 +419,28 @@ out:
  * bitmap file superblock operations
  */
 
+/*
+ * bitmap_wait_writes() should be called before writing any bitmap
+ * blocks, to ensure previous writes, particularly from
+ * bitmap_daemon_work(), have completed.
+ */
+static void bitmap_wait_writes(struct bitmap *bitmap)
+{
+	if (bitmap->storage.file)
+		wait_event(bitmap->write_wait,
+			   atomic_read(&bitmap->pending_writes)==0);
+	else
+		/* Note that we ignore the return value.  The writes
+		 * might have failed, but that would just mean that
+		 * some bits which should be cleared haven't been,
+		 * which is safe.  The relevant bitmap blocks will
+		 * probably get written again, but there is no great
+		 * loss if they aren't.
+		 */
+		md_super_wait(bitmap->mddev);
+}
+
+
 /* update the event counter and sync the superblock to disk */
 void bitmap_update_sb(struct bitmap *bitmap)
 {
@@ -455,24 +480,24 @@ void bitmap_print_sb(struct bitmap *bitmap)
 	if (!bitmap || !bitmap->storage.sb_page)
 		return;
 	sb = kmap_atomic(bitmap->storage.sb_page);
-	printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
-	printk(KERN_DEBUG "         magic: %08x\n", le32_to_cpu(sb->magic));
-	printk(KERN_DEBUG "       version: %d\n", le32_to_cpu(sb->version));
-	printk(KERN_DEBUG "          uuid: %08x.%08x.%08x.%08x\n",
-					*(__u32 *)(sb->uuid+0),
-					*(__u32 *)(sb->uuid+4),
-					*(__u32 *)(sb->uuid+8),
-					*(__u32 *)(sb->uuid+12));
-	printk(KERN_DEBUG "        events: %llu\n",
-			(unsigned long long) le64_to_cpu(sb->events));
-	printk(KERN_DEBUG "events cleared: %llu\n",
-			(unsigned long long) le64_to_cpu(sb->events_cleared));
-	printk(KERN_DEBUG "         state: %08x\n", le32_to_cpu(sb->state));
-	printk(KERN_DEBUG "     chunksize: %d B\n", le32_to_cpu(sb->chunksize));
-	printk(KERN_DEBUG "  daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
-	printk(KERN_DEBUG "     sync size: %llu KB\n",
-			(unsigned long long)le64_to_cpu(sb->sync_size)/2);
-	printk(KERN_DEBUG "max write behind: %d\n", le32_to_cpu(sb->write_behind));
+	pr_debug("%s: bitmap file superblock:\n", bmname(bitmap));
+	pr_debug("         magic: %08x\n", le32_to_cpu(sb->magic));
+	pr_debug("       version: %d\n", le32_to_cpu(sb->version));
+	pr_debug("          uuid: %08x.%08x.%08x.%08x\n",
+		 *(__u32 *)(sb->uuid+0),
+		 *(__u32 *)(sb->uuid+4),
+		 *(__u32 *)(sb->uuid+8),
+		 *(__u32 *)(sb->uuid+12));
+	pr_debug("        events: %llu\n",
+		 (unsigned long long) le64_to_cpu(sb->events));
+	pr_debug("events cleared: %llu\n",
+		 (unsigned long long) le64_to_cpu(sb->events_cleared));
+	pr_debug("         state: %08x\n", le32_to_cpu(sb->state));
+	pr_debug("     chunksize: %d B\n", le32_to_cpu(sb->chunksize));
+	pr_debug("  daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
+	pr_debug("     sync size: %llu KB\n",
+		 (unsigned long long)le64_to_cpu(sb->sync_size)/2);
+	pr_debug("max write behind: %d\n", le32_to_cpu(sb->write_behind));
 	kunmap_atomic(sb);
 }
 
@@ -506,14 +531,14 @@ static int bitmap_new_disk_sb(struct bitmap *bitmap)
 	BUG_ON(!chunksize);
 	if (!is_power_of_2(chunksize)) {
 		kunmap_atomic(sb);
-		printk(KERN_ERR "bitmap chunksize not a power of 2\n");
+		pr_warn("bitmap chunksize not a power of 2\n");
 		return -EINVAL;
 	}
 	sb->chunksize = cpu_to_le32(chunksize);
 
 	daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
 	if (!daemon_sleep || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
-		printk(KERN_INFO "Choosing daemon_sleep default (5 sec)\n");
+		pr_debug("Choosing daemon_sleep default (5 sec)\n");
 		daemon_sleep = 5 * HZ;
 	}
 	sb->daemon_sleep = cpu_to_le32(daemon_sleep);
@@ -584,7 +609,7 @@ re_read:
 		/* to 4k blocks */
 		bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks, 4096);
 		offset = bitmap->mddev->bitmap_info.offset + (bitmap->cluster_slot * (bm_blocks << 3));
-		pr_info("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
+		pr_debug("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
 			bitmap->cluster_slot, offset);
 	}
 
@@ -634,7 +659,7 @@ re_read:
 	else if (write_behind > COUNTER_MAX)
 		reason = "write-behind limit out of range (0 - 16383)";
 	if (reason) {
-		printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
+		pr_warn("%s: invalid bitmap file superblock: %s\n",
 			bmname(bitmap), reason);
 		goto out;
 	}
@@ -648,18 +673,15 @@ re_read:
 		 * bitmap's UUID and event counter to the mddev's
 		 */
 		if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
-			printk(KERN_INFO
-			       "%s: bitmap superblock UUID mismatch\n",
-			       bmname(bitmap));
+			pr_warn("%s: bitmap superblock UUID mismatch\n",
+				bmname(bitmap));
 			goto out;
 		}
 		events = le64_to_cpu(sb->events);
 		if (!nodes && (events < bitmap->mddev->events)) {
-			printk(KERN_INFO
-			       "%s: bitmap file is out of date (%llu < %llu) "
-			       "-- forcing full recovery\n",
-			       bmname(bitmap), events,
-			       (unsigned long long) bitmap->mddev->events);
+			pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n",
+				bmname(bitmap), events,
+				(unsigned long long) bitmap->mddev->events);
 			set_bit(BITMAP_STALE, &bitmap->flags);
 		}
 	}
@@ -679,8 +701,8 @@ out:
 	if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
 		err = md_setup_cluster(bitmap->mddev, nodes);
 		if (err) {
-			pr_err("%s: Could not setup cluster service (%d)\n",
-					bmname(bitmap), err);
+			pr_warn("%s: Could not setup cluster service (%d)\n",
+				bmname(bitmap), err);
 			goto out_no_sb;
 		}
 		bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
@@ -847,15 +869,13 @@ static void bitmap_file_kick(struct bitmap *bitmap)
 				ptr = file_path(bitmap->storage.file,
 					     path, PAGE_SIZE);
 
-			printk(KERN_ALERT
-			      "%s: kicking failed bitmap file %s from array!\n",
-			      bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
+			pr_warn("%s: kicking failed bitmap file %s from array!\n",
+				bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
 
 			kfree(path);
 		} else
-			printk(KERN_ALERT
-			       "%s: disabling internal bitmap due to errors\n",
-			       bmname(bitmap));
+			pr_warn("%s: disabling internal bitmap due to errors\n",
+				bmname(bitmap));
 	}
 }
 
@@ -983,6 +1003,7 @@ void bitmap_unplug(struct bitmap *bitmap)
 {
 	unsigned long i;
 	int dirty, need_write;
+	int writing = 0;
 
 	if (!bitmap || !bitmap->storage.filemap ||
 	    test_bit(BITMAP_STALE, &bitmap->flags))
@@ -997,15 +1018,19 @@ void bitmap_unplug(struct bitmap *bitmap)
 		need_write = test_and_clear_page_attr(bitmap, i,
 						      BITMAP_PAGE_NEEDWRITE);
 		if (dirty || need_write) {
+			if (!writing) {
+				bitmap_wait_writes(bitmap);
+				if (bitmap->mddev->queue)
+					blk_add_trace_msg(bitmap->mddev->queue,
+							  "md bitmap_unplug");
+			}
 			clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
 			write_page(bitmap, bitmap->storage.filemap[i], 0);
+			writing = 1;
 		}
 	}
-	if (bitmap->storage.file)
-		wait_event(bitmap->write_wait,
-			   atomic_read(&bitmap->pending_writes)==0);
-	else
-		md_super_wait(bitmap->mddev);
+	if (writing)
+		bitmap_wait_writes(bitmap);
 
 	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
 		bitmap_file_kick(bitmap);
@@ -1056,14 +1081,13 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
 
 	outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
 	if (outofdate)
-		printk(KERN_INFO "%s: bitmap file is out of date, doing full "
-			"recovery\n", bmname(bitmap));
+		pr_warn("%s: bitmap file is out of date, doing full recovery\n", bmname(bitmap));
 
 	if (file && i_size_read(file->f_mapping->host) < store->bytes) {
-		printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
-		       bmname(bitmap),
-		       (unsigned long) i_size_read(file->f_mapping->host),
-		       store->bytes);
+		pr_warn("%s: bitmap file too short %lu < %lu\n",
+			bmname(bitmap),
+			(unsigned long) i_size_read(file->f_mapping->host),
+			store->bytes);
 		goto err;
 	}
 
@@ -1137,16 +1161,15 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
 		offset = 0;
 	}
 
-	printk(KERN_INFO "%s: bitmap initialized from disk: "
-	       "read %lu pages, set %lu of %lu bits\n",
-	       bmname(bitmap), store->file_pages,
-	       bit_cnt, chunks);
+	pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n",
+		 bmname(bitmap), store->file_pages,
+		 bit_cnt, chunks);
 
 	return 0;
 
  err:
-	printk(KERN_INFO "%s: bitmap initialisation failed: %d\n",
-	       bmname(bitmap), ret);
+	pr_warn("%s: bitmap initialisation failed: %d\n",
+		bmname(bitmap), ret);
 	return ret;
 }
 
@@ -1225,6 +1248,10 @@ void bitmap_daemon_work(struct mddev *mddev)
 	}
 	bitmap->allclean = 1;
 
+	if (bitmap->mddev->queue)
+		blk_add_trace_msg(bitmap->mddev->queue,
+				  "md bitmap_daemon_work");
+
 	/* Any file-page which is PENDING now needs to be written.
 	 * So set NEEDWRITE now, then after we make any last-minute changes
 	 * we will write it.
@@ -1289,6 +1316,7 @@ void bitmap_daemon_work(struct mddev *mddev)
 	}
 	spin_unlock_irq(&counts->lock);
 
+	bitmap_wait_writes(bitmap);
 	/* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
 	 * DIRTY pages need to be written by bitmap_unplug so it can wait
 	 * for them.
@@ -1595,7 +1623,7 @@ void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force)
 		   atomic_read(&bitmap->mddev->recovery_active) == 0);
 
 	bitmap->mddev->curr_resync_completed = sector;
-	set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags);
+	set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags);
 	sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
 	s = 0;
 	while (s < sector && s < bitmap->mddev->resync_max_sectors) {
@@ -1825,8 +1853,8 @@ struct bitmap *bitmap_create(struct mddev *mddev, int slot)
 	if (err)
 		goto error;
 
-	printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
-	       bitmap->counts.pages, bmname(bitmap));
+	pr_debug("created bitmap (%lu pages) for device %s\n",
+		 bitmap->counts.pages, bmname(bitmap));
 
 	err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
 	if (err)
@@ -2029,8 +2057,10 @@ int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
 					   !bitmap->mddev->bitmap_info.external,
 					   mddev_is_clustered(bitmap->mddev)
 					   ? bitmap->cluster_slot : 0);
-	if (ret)
+	if (ret) {
+		bitmap_file_unmap(&store);
 		goto err;
+	}
 
 	pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
 
@@ -2089,7 +2119,7 @@ int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
 				bitmap->mddev->bitmap_info.chunksize = 1 << (old_counts.chunkshift +
 									     BITMAP_BLOCK_SHIFT);
 				blocks = old_counts.chunks << old_counts.chunkshift;
-				pr_err("Could not pre-allocate in-memory bitmap for cluster raid\n");
+				pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n");
 				break;
 			} else
 				bitmap->counts.bp[page].count += 1;
@@ -2266,7 +2296,7 @@ location_store(struct mddev *mddev, const char *buf, size_t len)
 		/* Ensure new bitmap info is stored in
 		 * metadata promptly.
 		 */
-		set_bit(MD_CHANGE_DEVS, &mddev->flags);
+		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 		md_wakeup_thread(mddev->thread);
 	}
 	rv = 0;
diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index 125aedc3875f..84d2f0e4c754 100644
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -611,9 +611,7 @@ static void use_inline_bio(struct dm_buffer *b, int rw, sector_t block,
 	char *ptr;
 	int len;
 
-	bio_init(&b->bio);
-	b->bio.bi_io_vec = b->bio_vec;
-	b->bio.bi_max_vecs = DM_BUFIO_INLINE_VECS;
+	bio_init(&b->bio, b->bio_vec, DM_BUFIO_INLINE_VECS);
 	b->bio.bi_iter.bi_sector = block << b->c->sectors_per_block_bits;
 	b->bio.bi_bdev = b->c->bdev;
 	b->bio.bi_end_io = inline_endio;
@@ -822,12 +820,14 @@ enum new_flag {
 static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf)
 {
 	struct dm_buffer *b;
+	bool tried_noio_alloc = false;
 
 	/*
 	 * dm-bufio is resistant to allocation failures (it just keeps
 	 * one buffer reserved in cases all the allocations fail).
 	 * So set flags to not try too hard:
-	 *	GFP_NOIO: don't recurse into the I/O layer
+	 *	GFP_NOWAIT: don't wait; if we need to sleep we'll release our
+	 *		    mutex and wait ourselves.
 	 *	__GFP_NORETRY: don't retry and rather return failure
 	 *	__GFP_NOMEMALLOC: don't use emergency reserves
 	 *	__GFP_NOWARN: don't print a warning in case of failure
@@ -837,7 +837,7 @@ static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client
 	 */
 	while (1) {
 		if (dm_bufio_cache_size_latch != 1) {
-			b = alloc_buffer(c, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
+			b = alloc_buffer(c, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
 			if (b)
 				return b;
 		}
@@ -845,6 +845,15 @@ static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client
 		if (nf == NF_PREFETCH)
 			return NULL;
 
+		if (dm_bufio_cache_size_latch != 1 && !tried_noio_alloc) {
+			dm_bufio_unlock(c);
+			b = alloc_buffer(c, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
+			dm_bufio_lock(c);
+			if (b)
+				return b;
+			tried_noio_alloc = true;
+		}
+
 		if (!list_empty(&c->reserved_buffers)) {
 			b = list_entry(c->reserved_buffers.next,
 				       struct dm_buffer, lru_list);
@@ -1316,7 +1325,7 @@ int dm_bufio_issue_flush(struct dm_bufio_client *c)
 {
 	struct dm_io_request io_req = {
 		.bi_op = REQ_OP_WRITE,
-		.bi_op_flags = WRITE_FLUSH,
+		.bi_op_flags = REQ_PREFLUSH,
 		.mem.type = DM_IO_KMEM,
 		.mem.ptr.addr = NULL,
 		.client = c->dm_io,
@@ -1587,18 +1596,9 @@ dm_bufio_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
 static unsigned long
 dm_bufio_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
 {
-	struct dm_bufio_client *c;
-	unsigned long count;
-
-	c = container_of(shrink, struct dm_bufio_client, shrinker);
-	if (sc->gfp_mask & __GFP_FS)
-		dm_bufio_lock(c);
-	else if (!dm_bufio_trylock(c))
-		return 0;
+	struct dm_bufio_client *c = container_of(shrink, struct dm_bufio_client, shrinker);
 
-	count = c->n_buffers[LIST_CLEAN] + c->n_buffers[LIST_DIRTY];
-	dm_bufio_unlock(c);
-	return count;
+	return ACCESS_ONCE(c->n_buffers[LIST_CLEAN]) + ACCESS_ONCE(c->n_buffers[LIST_DIRTY]);
 }
 
 /*
diff --git a/drivers/md/dm-cache-block-types.h b/drivers/md/dm-cache-block-types.h
index bed4ad4e1b7c..389c9e8ac785 100644
--- a/drivers/md/dm-cache-block-types.h
+++ b/drivers/md/dm-cache-block-types.h
@@ -17,9 +17,9 @@
  * discard bitset.
  */
 
-typedef dm_block_t __bitwise__ dm_oblock_t;
-typedef uint32_t __bitwise__ dm_cblock_t;
-typedef dm_block_t __bitwise__ dm_dblock_t;
+typedef dm_block_t __bitwise dm_oblock_t;
+typedef uint32_t __bitwise dm_cblock_t;
+typedef dm_block_t __bitwise dm_dblock_t;
 
 static inline dm_oblock_t to_oblock(dm_block_t b)
 {
diff --git a/drivers/md/dm-cache-metadata.c b/drivers/md/dm-cache-metadata.c
index 695577812cf6..624fe4319b24 100644
--- a/drivers/md/dm-cache-metadata.c
+++ b/drivers/md/dm-cache-metadata.c
@@ -383,7 +383,6 @@ static int __format_metadata(struct dm_cache_metadata *cmd)
 		goto bad;
 
 	dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
-
 	r = dm_bitset_empty(&cmd->discard_info, &cmd->discard_root);
 	if (r < 0)
 		goto bad;
@@ -789,7 +788,7 @@ static struct dm_cache_metadata *lookup_or_open(struct block_device *bdev,
 static bool same_params(struct dm_cache_metadata *cmd, sector_t data_block_size)
 {
 	if (cmd->data_block_size != data_block_size) {
-		DMERR("data_block_size (%llu) different from that in metadata (%llu)\n",
+		DMERR("data_block_size (%llu) different from that in metadata (%llu)",
 		      (unsigned long long) data_block_size,
 		      (unsigned long long) cmd->data_block_size);
 		return false;
diff --git a/drivers/md/dm-cache-policy-smq.c b/drivers/md/dm-cache-policy-smq.c
index c33f4a6e1d7d..f19c6930a67c 100644
--- a/drivers/md/dm-cache-policy-smq.c
+++ b/drivers/md/dm-cache-policy-smq.c
@@ -1361,7 +1361,7 @@ static void smq_clear_dirty(struct dm_cache_policy *p, dm_oblock_t oblock)
 
 static unsigned random_level(dm_cblock_t cblock)
 {
-	return hash_32_generic(from_cblock(cblock), 9) & (NR_CACHE_LEVELS - 1);
+	return hash_32(from_cblock(cblock), 9) & (NR_CACHE_LEVELS - 1);
 }
 
 static int smq_load_mapping(struct dm_cache_policy *p,
diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index 59b2c50562e4..e04c61e0839e 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -989,7 +989,8 @@ static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mod
 	enum cache_metadata_mode old_mode = get_cache_mode(cache);
 
 	if (dm_cache_metadata_needs_check(cache->cmd, &needs_check)) {
-		DMERR("unable to read needs_check flag, setting failure mode");
+		DMERR("%s: unable to read needs_check flag, setting failure mode.",
+		      cache_device_name(cache));
 		new_mode = CM_FAIL;
 	}
 
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index a2768835d394..7c6c57216bf2 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -12,6 +12,7 @@
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
+#include <linux/key.h>
 #include <linux/bio.h>
 #include <linux/blkdev.h>
 #include <linux/mempool.h>
@@ -23,12 +24,14 @@
 #include <linux/atomic.h>
 #include <linux/scatterlist.h>
 #include <linux/rbtree.h>
+#include <linux/ctype.h>
 #include <asm/page.h>
 #include <asm/unaligned.h>
 #include <crypto/hash.h>
 #include <crypto/md5.h>
 #include <crypto/algapi.h>
 #include <crypto/skcipher.h>
+#include <keys/user-type.h>
 
 #include <linux/device-mapper.h>
 
@@ -140,8 +143,9 @@ struct crypt_config {
 
 	char *cipher;
 	char *cipher_string;
+	char *key_string;
 
-	struct crypt_iv_operations *iv_gen_ops;
+	const struct crypt_iv_operations *iv_gen_ops;
 	union {
 		struct iv_essiv_private essiv;
 		struct iv_benbi_private benbi;
@@ -758,15 +762,15 @@ static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv,
 	return r;
 }
 
-static struct crypt_iv_operations crypt_iv_plain_ops = {
+static const struct crypt_iv_operations crypt_iv_plain_ops = {
 	.generator = crypt_iv_plain_gen
 };
 
-static struct crypt_iv_operations crypt_iv_plain64_ops = {
+static const struct crypt_iv_operations crypt_iv_plain64_ops = {
 	.generator = crypt_iv_plain64_gen
 };
 
-static struct crypt_iv_operations crypt_iv_essiv_ops = {
+static const struct crypt_iv_operations crypt_iv_essiv_ops = {
 	.ctr       = crypt_iv_essiv_ctr,
 	.dtr       = crypt_iv_essiv_dtr,
 	.init      = crypt_iv_essiv_init,
@@ -774,17 +778,17 @@ static struct crypt_iv_operations crypt_iv_essiv_ops = {
 	.generator = crypt_iv_essiv_gen
 };
 
-static struct crypt_iv_operations crypt_iv_benbi_ops = {
+static const struct crypt_iv_operations crypt_iv_benbi_ops = {
 	.ctr	   = crypt_iv_benbi_ctr,
 	.dtr	   = crypt_iv_benbi_dtr,
 	.generator = crypt_iv_benbi_gen
 };
 
-static struct crypt_iv_operations crypt_iv_null_ops = {
+static const struct crypt_iv_operations crypt_iv_null_ops = {
 	.generator = crypt_iv_null_gen
 };
 
-static struct crypt_iv_operations crypt_iv_lmk_ops = {
+static const struct crypt_iv_operations crypt_iv_lmk_ops = {
 	.ctr	   = crypt_iv_lmk_ctr,
 	.dtr	   = crypt_iv_lmk_dtr,
 	.init	   = crypt_iv_lmk_init,
@@ -793,7 +797,7 @@ static struct crypt_iv_operations crypt_iv_lmk_ops = {
 	.post	   = crypt_iv_lmk_post
 };
 
-static struct crypt_iv_operations crypt_iv_tcw_ops = {
+static const struct crypt_iv_operations crypt_iv_tcw_ops = {
 	.ctr	   = crypt_iv_tcw_ctr,
 	.dtr	   = crypt_iv_tcw_dtr,
 	.init	   = crypt_iv_tcw_init,
@@ -994,7 +998,6 @@ static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size)
 	gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM;
 	unsigned i, len, remaining_size;
 	struct page *page;
-	struct bio_vec *bvec;
 
 retry:
 	if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
@@ -1019,12 +1022,7 @@ retry:
 
 		len = (remaining_size > PAGE_SIZE) ? PAGE_SIZE : remaining_size;
 
-		bvec = &clone->bi_io_vec[clone->bi_vcnt++];
-		bvec->bv_page = page;
-		bvec->bv_len = len;
-		bvec->bv_offset = 0;
-
-		clone->bi_iter.bi_size += len;
+		bio_add_page(clone, page, len, 0);
 
 		remaining_size -= len;
 	}
@@ -1135,7 +1133,7 @@ static void clone_init(struct dm_crypt_io *io, struct bio *clone)
 	clone->bi_private = io;
 	clone->bi_end_io  = crypt_endio;
 	clone->bi_bdev    = cc->dev->bdev;
-	bio_set_op_attrs(clone, bio_op(io->base_bio), bio_flags(io->base_bio));
+	clone->bi_opf	  = io->base_bio->bi_opf;
 }
 
 static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
@@ -1471,7 +1469,7 @@ static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
 	return 0;
 }
 
-static int crypt_setkey_allcpus(struct crypt_config *cc)
+static int crypt_setkey(struct crypt_config *cc)
 {
 	unsigned subkey_size;
 	int err = 0, i, r;
@@ -1490,25 +1488,157 @@ static int crypt_setkey_allcpus(struct crypt_config *cc)
 	return err;
 }
 
+#ifdef CONFIG_KEYS
+
+static bool contains_whitespace(const char *str)
+{
+	while (*str)
+		if (isspace(*str++))
+			return true;
+	return false;
+}
+
+static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
+{
+	char *new_key_string, *key_desc;
+	int ret;
+	struct key *key;
+	const struct user_key_payload *ukp;
+
+	/*
+	 * Reject key_string with whitespace. dm core currently lacks code for
+	 * proper whitespace escaping in arguments on DM_TABLE_STATUS path.
+	 */
+	if (contains_whitespace(key_string)) {
+		DMERR("whitespace chars not allowed in key string");
+		return -EINVAL;
+	}
+
+	/* look for next ':' separating key_type from key_description */
+	key_desc = strpbrk(key_string, ":");
+	if (!key_desc || key_desc == key_string || !strlen(key_desc + 1))
+		return -EINVAL;
+
+	if (strncmp(key_string, "logon:", key_desc - key_string + 1) &&
+	    strncmp(key_string, "user:", key_desc - key_string + 1))
+		return -EINVAL;
+
+	new_key_string = kstrdup(key_string, GFP_KERNEL);
+	if (!new_key_string)
+		return -ENOMEM;
+
+	key = request_key(key_string[0] == 'l' ? &key_type_logon : &key_type_user,
+			  key_desc + 1, NULL);
+	if (IS_ERR(key)) {
+		kzfree(new_key_string);
+		return PTR_ERR(key);
+	}
+
+	rcu_read_lock();
+
+	ukp = user_key_payload(key);
+	if (!ukp) {
+		rcu_read_unlock();
+		key_put(key);
+		kzfree(new_key_string);
+		return -EKEYREVOKED;
+	}
+
+	if (cc->key_size != ukp->datalen) {
+		rcu_read_unlock();
+		key_put(key);
+		kzfree(new_key_string);
+		return -EINVAL;
+	}
+
+	memcpy(cc->key, ukp->data, cc->key_size);
+
+	rcu_read_unlock();
+	key_put(key);
+
+	/* clear the flag since following operations may invalidate previously valid key */
+	clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
+
+	ret = crypt_setkey(cc);
+
+	/* wipe the kernel key payload copy in each case */
+	memset(cc->key, 0, cc->key_size * sizeof(u8));
+
+	if (!ret) {
+		set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
+		kzfree(cc->key_string);
+		cc->key_string = new_key_string;
+	} else
+		kzfree(new_key_string);
+
+	return ret;
+}
+
+static int get_key_size(char **key_string)
+{
+	char *colon, dummy;
+	int ret;
+
+	if (*key_string[0] != ':')
+		return strlen(*key_string) >> 1;
+
+	/* look for next ':' in key string */
+	colon = strpbrk(*key_string + 1, ":");
+	if (!colon)
+		return -EINVAL;
+
+	if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':')
+		return -EINVAL;
+
+	*key_string = colon;
+
+	/* remaining key string should be :<logon|user>:<key_desc> */
+
+	return ret;
+}
+
+#else
+
+static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
+{
+	return -EINVAL;
+}
+
+static int get_key_size(char **key_string)
+{
+	return (*key_string[0] == ':') ? -EINVAL : strlen(*key_string) >> 1;
+}
+
+#endif
+
 static int crypt_set_key(struct crypt_config *cc, char *key)
 {
 	int r = -EINVAL;
 	int key_string_len = strlen(key);
 
-	/* The key size may not be changed. */
-	if (cc->key_size != (key_string_len >> 1))
-		goto out;
-
 	/* Hyphen (which gives a key_size of zero) means there is no key. */
 	if (!cc->key_size && strcmp(key, "-"))
 		goto out;
 
-	if (cc->key_size && crypt_decode_key(cc->key, key, cc->key_size) < 0)
+	/* ':' means the key is in kernel keyring, short-circuit normal key processing */
+	if (key[0] == ':') {
+		r = crypt_set_keyring_key(cc, key + 1);
 		goto out;
+	}
 
-	set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
+	/* clear the flag since following operations may invalidate previously valid key */
+	clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
 
-	r = crypt_setkey_allcpus(cc);
+	/* wipe references to any kernel keyring key */
+	kzfree(cc->key_string);
+	cc->key_string = NULL;
+
+	if (cc->key_size && crypt_decode_key(cc->key, key, cc->key_size) < 0)
+		goto out;
+
+	r = crypt_setkey(cc);
+	if (!r)
+		set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
 
 out:
 	/* Hex key string not needed after here, so wipe it. */
@@ -1521,8 +1651,10 @@ static int crypt_wipe_key(struct crypt_config *cc)
 {
 	clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
 	memset(&cc->key, 0, cc->key_size * sizeof(u8));
+	kzfree(cc->key_string);
+	cc->key_string = NULL;
 
-	return crypt_setkey_allcpus(cc);
+	return crypt_setkey(cc);
 }
 
 static void crypt_dtr(struct dm_target *ti)
@@ -1558,6 +1690,7 @@ static void crypt_dtr(struct dm_target *ti)
 
 	kzfree(cc->cipher);
 	kzfree(cc->cipher_string);
+	kzfree(cc->key_string);
 
 	/* Must zero key material before freeing */
 	kzfree(cc);
@@ -1726,12 +1859,13 @@ bad_mem:
 
 /*
  * Construct an encryption mapping:
- * <cipher> <key> <iv_offset> <dev_path> <start>
+ * <cipher> [<key>|:<key_size>:<user|logon>:<key_description>] <iv_offset> <dev_path> <start>
  */
 static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 {
 	struct crypt_config *cc;
-	unsigned int key_size, opt_params;
+	int key_size;
+	unsigned int opt_params;
 	unsigned long long tmpll;
 	int ret;
 	size_t iv_size_padding;
@@ -1748,7 +1882,11 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		return -EINVAL;
 	}
 
-	key_size = strlen(argv[1]) >> 1;
+	key_size = get_key_size(&argv[1]);
+	if (key_size < 0) {
+		ti->error = "Cannot parse key size";
+		return -EINVAL;
+	}
 
 	cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL);
 	if (!cc) {
@@ -1955,10 +2093,13 @@ static void crypt_status(struct dm_target *ti, status_type_t type,
 	case STATUSTYPE_TABLE:
 		DMEMIT("%s ", cc->cipher_string);
 
-		if (cc->key_size > 0)
-			for (i = 0; i < cc->key_size; i++)
-				DMEMIT("%02x", cc->key[i]);
-		else
+		if (cc->key_size > 0) {
+			if (cc->key_string)
+				DMEMIT(":%u:%s", cc->key_size, cc->key_string);
+			else
+				for (i = 0; i < cc->key_size; i++)
+					DMEMIT("%02x", cc->key[i]);
+		} else
 			DMEMIT("-");
 
 		DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
@@ -2014,7 +2155,7 @@ static void crypt_resume(struct dm_target *ti)
 static int crypt_message(struct dm_target *ti, unsigned argc, char **argv)
 {
 	struct crypt_config *cc = ti->private;
-	int ret = -EINVAL;
+	int key_size, ret = -EINVAL;
 
 	if (argc < 2)
 		goto error;
@@ -2025,6 +2166,13 @@ static int crypt_message(struct dm_target *ti, unsigned argc, char **argv)
 			return -EINVAL;
 		}
 		if (argc == 3 && !strcasecmp(argv[1], "set")) {
+			/* The key size may not be changed. */
+			key_size = get_key_size(&argv[2]);
+			if (key_size < 0 || cc->key_size != key_size) {
+				memset(argv[2], '0', strlen(argv[2]));
+				return -EINVAL;
+			}
+
 			ret = crypt_set_key(cc, argv[2]);
 			if (ret)
 				return ret;
@@ -2068,7 +2216,7 @@ static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
 
 static struct target_type crypt_target = {
 	.name   = "crypt",
-	.version = {1, 14, 1},
+	.version = {1, 15, 0},
 	.module = THIS_MODULE,
 	.ctr    = crypt_ctr,
 	.dtr    = crypt_dtr,
diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c
index 6a2e8dd44a1b..13305a182611 100644
--- a/drivers/md/dm-flakey.c
+++ b/drivers/md/dm-flakey.c
@@ -36,7 +36,8 @@ struct flakey_c {
 };
 
 enum feature_flag_bits {
-	DROP_WRITES
+	DROP_WRITES,
+	ERROR_WRITES
 };
 
 struct per_bio_data {
@@ -76,6 +77,25 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc,
 			if (test_and_set_bit(DROP_WRITES, &fc->flags)) {
 				ti->error = "Feature drop_writes duplicated";
 				return -EINVAL;
+			} else if (test_bit(ERROR_WRITES, &fc->flags)) {
+				ti->error = "Feature drop_writes conflicts with feature error_writes";
+				return -EINVAL;
+			}
+
+			continue;
+		}
+
+		/*
+		 * error_writes
+		 */
+		if (!strcasecmp(arg_name, "error_writes")) {
+			if (test_and_set_bit(ERROR_WRITES, &fc->flags)) {
+				ti->error = "Feature error_writes duplicated";
+				return -EINVAL;
+
+			} else if (test_bit(DROP_WRITES, &fc->flags)) {
+				ti->error = "Feature error_writes conflicts with feature drop_writes";
+				return -EINVAL;
 			}
 
 			continue;
@@ -135,6 +155,10 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc,
 	if (test_bit(DROP_WRITES, &fc->flags) && (fc->corrupt_bio_rw == WRITE)) {
 		ti->error = "drop_writes is incompatible with corrupt_bio_byte with the WRITE flag set";
 		return -EINVAL;
+
+	} else if (test_bit(ERROR_WRITES, &fc->flags) && (fc->corrupt_bio_rw == WRITE)) {
+		ti->error = "error_writes is incompatible with corrupt_bio_byte with the WRITE flag set";
+		return -EINVAL;
 	}
 
 	return 0;
@@ -200,11 +224,13 @@ static int flakey_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 
 	if (!(fc->up_interval + fc->down_interval)) {
 		ti->error = "Total (up + down) interval is zero";
+		r = -EINVAL;
 		goto bad;
 	}
 
 	if (fc->up_interval + fc->down_interval < fc->up_interval) {
 		ti->error = "Interval overflow";
+		r = -EINVAL;
 		goto bad;
 	}
 
@@ -289,22 +315,27 @@ static int flakey_map(struct dm_target *ti, struct bio *bio)
 		pb->bio_submitted = true;
 
 		/*
-		 * Error reads if neither corrupt_bio_byte or drop_writes are set.
+		 * Error reads if neither corrupt_bio_byte or drop_writes or error_writes are set.
 		 * Otherwise, flakey_end_io() will decide if the reads should be modified.
 		 */
 		if (bio_data_dir(bio) == READ) {
-			if (!fc->corrupt_bio_byte && !test_bit(DROP_WRITES, &fc->flags))
+			if (!fc->corrupt_bio_byte && !test_bit(DROP_WRITES, &fc->flags) &&
+			    !test_bit(ERROR_WRITES, &fc->flags))
 				return -EIO;
 			goto map_bio;
 		}
 
 		/*
-		 * Drop writes?
+		 * Drop or error writes?
 		 */
 		if (test_bit(DROP_WRITES, &fc->flags)) {
 			bio_endio(bio);
 			return DM_MAPIO_SUBMITTED;
 		}
+		else if (test_bit(ERROR_WRITES, &fc->flags)) {
+			bio_io_error(bio);
+			return DM_MAPIO_SUBMITTED;
+		}
 
 		/*
 		 * Corrupt matching writes.
@@ -340,10 +371,11 @@ static int flakey_end_io(struct dm_target *ti, struct bio *bio, int error)
 			 */
 			corrupt_bio_data(bio, fc);
 
-		} else if (!test_bit(DROP_WRITES, &fc->flags)) {
+		} else if (!test_bit(DROP_WRITES, &fc->flags) &&
+			   !test_bit(ERROR_WRITES, &fc->flags)) {
 			/*
 			 * Error read during the down_interval if drop_writes
-			 * wasn't configured.
+			 * and error_writes were not configured.
 			 */
 			return -EIO;
 		}
@@ -357,7 +389,7 @@ static void flakey_status(struct dm_target *ti, status_type_t type,
 {
 	unsigned sz = 0;
 	struct flakey_c *fc = ti->private;
-	unsigned drop_writes;
+	unsigned drop_writes, error_writes;
 
 	switch (type) {
 	case STATUSTYPE_INFO:
@@ -370,10 +402,13 @@ static void flakey_status(struct dm_target *ti, status_type_t type,
 		       fc->down_interval);
 
 		drop_writes = test_bit(DROP_WRITES, &fc->flags);
-		DMEMIT("%u ", drop_writes + (fc->corrupt_bio_byte > 0) * 5);
+		error_writes = test_bit(ERROR_WRITES, &fc->flags);
+		DMEMIT("%u ", drop_writes + error_writes + (fc->corrupt_bio_byte > 0) * 5);
 
 		if (drop_writes)
 			DMEMIT("drop_writes ");
+		else if (error_writes)
+			DMEMIT("error_writes ");
 
 		if (fc->corrupt_bio_byte)
 			DMEMIT("corrupt_bio_byte %u %c %u %u ",
@@ -410,7 +445,7 @@ static int flakey_iterate_devices(struct dm_target *ti, iterate_devices_callout_
 
 static struct target_type flakey_target = {
 	.name   = "flakey",
-	.version = {1, 3, 1},
+	.version = {1, 4, 0},
 	.module = THIS_MODULE,
 	.ctr    = flakey_ctr,
 	.dtr    = flakey_dtr,
diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
index 0bf1a12e35fe..03940bf36f6c 100644
--- a/drivers/md/dm-io.c
+++ b/drivers/md/dm-io.c
@@ -162,7 +162,10 @@ struct dpages {
 			 struct page **p, unsigned long *len, unsigned *offset);
 	void (*next_page)(struct dpages *dp);
 
-	unsigned context_u;
+	union {
+		unsigned context_u;
+		struct bvec_iter context_bi;
+	};
 	void *context_ptr;
 
 	void *vma_invalidate_address;
@@ -204,25 +207,36 @@ static void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned offse
 static void bio_get_page(struct dpages *dp, struct page **p,
 			 unsigned long *len, unsigned *offset)
 {
-	struct bio_vec *bvec = dp->context_ptr;
-	*p = bvec->bv_page;
-	*len = bvec->bv_len - dp->context_u;
-	*offset = bvec->bv_offset + dp->context_u;
+	struct bio_vec bvec = bvec_iter_bvec((struct bio_vec *)dp->context_ptr,
+					     dp->context_bi);
+
+	*p = bvec.bv_page;
+	*len = bvec.bv_len;
+	*offset = bvec.bv_offset;
+
+	/* avoid figuring it out again in bio_next_page() */
+	dp->context_bi.bi_sector = (sector_t)bvec.bv_len;
 }
 
 static void bio_next_page(struct dpages *dp)
 {
-	struct bio_vec *bvec = dp->context_ptr;
-	dp->context_ptr = bvec + 1;
-	dp->context_u = 0;
+	unsigned int len = (unsigned int)dp->context_bi.bi_sector;
+
+	bvec_iter_advance((struct bio_vec *)dp->context_ptr,
+			  &dp->context_bi, len);
 }
 
 static void bio_dp_init(struct dpages *dp, struct bio *bio)
 {
 	dp->get_page = bio_get_page;
 	dp->next_page = bio_next_page;
-	dp->context_ptr = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
-	dp->context_u = bio->bi_iter.bi_bvec_done;
+
+	/*
+	 * We just use bvec iterator to retrieve pages, so it is ok to
+	 * access the bvec table directly here
+	 */
+	dp->context_ptr = bio->bi_io_vec;
+	dp->context_bi = bio->bi_iter;
 }
 
 /*
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index 966eb4b61aed..c72a77048b73 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -1697,7 +1697,7 @@ static int copy_params(struct dm_ioctl __user *user, struct dm_ioctl *param_kern
 {
 	struct dm_ioctl *dmi;
 	int secure_data;
-	const size_t minimum_data_size = sizeof(*param_kernel) - sizeof(param_kernel->data);
+	const size_t minimum_data_size = offsetof(struct dm_ioctl, data);
 
 	if (copy_from_user(param_kernel, user, minimum_data_size))
 		return -EFAULT;
diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c
index 07fc1ad42ec5..33e71ea6cc14 100644
--- a/drivers/md/dm-log.c
+++ b/drivers/md/dm-log.c
@@ -308,7 +308,7 @@ static int flush_header(struct log_c *lc)
 	};
 
 	lc->io_req.bi_op = REQ_OP_WRITE;
-	lc->io_req.bi_op_flags = WRITE_FLUSH;
+	lc->io_req.bi_op_flags = REQ_PREFLUSH;
 
 	return dm_io(&lc->io_req, 1, &null_location, NULL);
 }
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index e477af8596e2..6400cffb986d 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -372,16 +372,13 @@ static int __pg_init_all_paths(struct multipath *m)
 	return atomic_read(&m->pg_init_in_progress);
 }
 
-static int pg_init_all_paths(struct multipath *m)
+static void pg_init_all_paths(struct multipath *m)
 {
-	int r;
 	unsigned long flags;
 
 	spin_lock_irqsave(&m->lock, flags);
-	r = __pg_init_all_paths(m);
+	__pg_init_all_paths(m);
 	spin_unlock_irqrestore(&m->lock, flags);
-
-	return r;
 }
 
 static void __switch_pg(struct multipath *m, struct priority_group *pg)
@@ -583,16 +580,17 @@ static int __multipath_map(struct dm_target *ti, struct request *clone,
 		 * .request_fn stacked on blk-mq path(s) and
 		 * blk-mq stacked on blk-mq path(s).
 		 */
-		*__clone = blk_mq_alloc_request(bdev_get_queue(bdev),
-						rq_data_dir(rq), BLK_MQ_REQ_NOWAIT);
-		if (IS_ERR(*__clone)) {
-			/* ENOMEM, requeue */
+		clone = blk_mq_alloc_request(bdev_get_queue(bdev),
+					     rq_data_dir(rq), BLK_MQ_REQ_NOWAIT);
+		if (IS_ERR(clone)) {
+			/* EBUSY, ENODEV or EWOULDBLOCK: requeue */
 			clear_request_fn_mpio(m, map_context);
 			return r;
 		}
-		(*__clone)->bio = (*__clone)->biotail = NULL;
-		(*__clone)->rq_disk = bdev->bd_disk;
-		(*__clone)->cmd_flags |= REQ_FAILFAST_TRANSPORT;
+		clone->bio = clone->biotail = NULL;
+		clone->rq_disk = bdev->bd_disk;
+		clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
+		*__clone = clone;
 	}
 
 	if (pgpath->pg->ps.type->start_io)
@@ -852,18 +850,22 @@ retain:
 		attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
 		if (attached_handler_name) {
 			/*
+			 * Clear any hw_handler_params associated with a
+			 * handler that isn't already attached.
+			 */
+			if (m->hw_handler_name && strcmp(attached_handler_name, m->hw_handler_name)) {
+				kfree(m->hw_handler_params);
+				m->hw_handler_params = NULL;
+			}
+
+			/*
 			 * Reset hw_handler_name to match the attached handler
-			 * and clear any hw_handler_params associated with the
-			 * ignored handler.
 			 *
 			 * NB. This modifies the table line to show the actual
 			 * handler instead of the original table passed in.
 			 */
 			kfree(m->hw_handler_name);
 			m->hw_handler_name = attached_handler_name;
-
-			kfree(m->hw_handler_params);
-			m->hw_handler_params = NULL;
 		}
 	}
 
@@ -1002,6 +1004,8 @@ static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
 	}
 
 	m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
+	if (!m->hw_handler_name)
+		return -EINVAL;
 
 	if (hw_argc > 1) {
 		char *p;
@@ -1362,7 +1366,7 @@ static int switch_pg_num(struct multipath *m, const char *pgstr)
 	char dummy;
 
 	if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
-	    (pgnum > m->nr_priority_groups)) {
+	    !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
 		DMWARN("invalid PG number supplied to switch_pg_num");
 		return -EINVAL;
 	}
@@ -1394,7 +1398,7 @@ static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed)
 	char dummy;
 
 	if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
-	    (pgnum > m->nr_priority_groups)) {
+	    !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
 		DMWARN("invalid PG number supplied to bypass_pg");
 		return -EINVAL;
 	}
diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c
index 6d53810963f7..b8f978e551d7 100644
--- a/drivers/md/dm-raid.c
+++ b/drivers/md/dm-raid.c
@@ -160,7 +160,6 @@ struct raid_dev {
 				 CTR_FLAG_DAEMON_SLEEP | \
 				 CTR_FLAG_MIN_RECOVERY_RATE | \
 				 CTR_FLAG_MAX_RECOVERY_RATE | \
-				 CTR_FLAG_MAX_WRITE_BEHIND | \
 				 CTR_FLAG_STRIPE_CACHE | \
 				 CTR_FLAG_REGION_SIZE | \
 				 CTR_FLAG_DELTA_DISKS | \
@@ -171,7 +170,6 @@ struct raid_dev {
 				 CTR_FLAG_DAEMON_SLEEP | \
 				 CTR_FLAG_MIN_RECOVERY_RATE | \
 				 CTR_FLAG_MAX_RECOVERY_RATE | \
-				 CTR_FLAG_MAX_WRITE_BEHIND | \
 				 CTR_FLAG_STRIPE_CACHE | \
 				 CTR_FLAG_REGION_SIZE | \
 				 CTR_FLAG_DELTA_DISKS | \
@@ -2011,7 +2009,7 @@ static int super_load(struct md_rdev *rdev, struct md_rdev *refdev)
 		sb->compat_features = cpu_to_le32(FEATURE_FLAG_SUPPORTS_V190);
 
 		/* Force writing of superblocks to disk */
-		set_bit(MD_CHANGE_DEVS, &rdev->mddev->flags);
+		set_bit(MD_SB_CHANGE_DEVS, &rdev->mddev->sb_flags);
 
 		/* Any superblock is better than none, choose that if given */
 		return refdev ? 0 : 1;
@@ -2050,16 +2048,17 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev)
 
 	mddev->reshape_position = MaxSector;
 
+	mddev->raid_disks = le32_to_cpu(sb->num_devices);
+	mddev->level = le32_to_cpu(sb->level);
+	mddev->layout = le32_to_cpu(sb->layout);
+	mddev->chunk_sectors = le32_to_cpu(sb->stripe_sectors);
+
 	/*
 	 * Reshaping is supported, e.g. reshape_position is valid
 	 * in superblock and superblock content is authoritative.
 	 */
 	if (le32_to_cpu(sb->compat_features) & FEATURE_FLAG_SUPPORTS_V190) {
 		/* Superblock is authoritative wrt given raid set layout! */
-		mddev->raid_disks = le32_to_cpu(sb->num_devices);
-		mddev->level = le32_to_cpu(sb->level);
-		mddev->layout = le32_to_cpu(sb->layout);
-		mddev->chunk_sectors = le32_to_cpu(sb->stripe_sectors);
 		mddev->new_level = le32_to_cpu(sb->new_level);
 		mddev->new_layout = le32_to_cpu(sb->new_layout);
 		mddev->new_chunk_sectors = le32_to_cpu(sb->new_stripe_sectors);
@@ -2087,38 +2086,44 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev)
 		/*
 		 * No takeover/reshaping, because we don't have the extended v1.9.0 metadata
 		 */
-		if (le32_to_cpu(sb->level) != mddev->new_level) {
-			DMERR("Reshaping/takeover raid sets not yet supported. (raid level/stripes/size change)");
-			return -EINVAL;
-		}
-		if (le32_to_cpu(sb->layout) != mddev->new_layout) {
-			DMERR("Reshaping raid sets not yet supported. (raid layout change)");
-			DMERR("	 0x%X vs 0x%X", le32_to_cpu(sb->layout), mddev->layout);
-			DMERR("	 Old layout: %s w/ %d copies",
-			      raid10_md_layout_to_format(le32_to_cpu(sb->layout)),
-			      raid10_md_layout_to_copies(le32_to_cpu(sb->layout)));
-			DMERR("	 New layout: %s w/ %d copies",
-			      raid10_md_layout_to_format(mddev->layout),
-			      raid10_md_layout_to_copies(mddev->layout));
-			return -EINVAL;
-		}
-		if (le32_to_cpu(sb->stripe_sectors) != mddev->new_chunk_sectors) {
-			DMERR("Reshaping raid sets not yet supported. (stripe sectors change)");
-			return -EINVAL;
-		}
+		struct raid_type *rt_cur = get_raid_type_by_ll(mddev->level, mddev->layout);
+		struct raid_type *rt_new = get_raid_type_by_ll(mddev->new_level, mddev->new_layout);
 
-		/* We can only change the number of devices in raid1 with old (i.e. pre 1.0.7) metadata */
-		if (!rt_is_raid1(rs->raid_type) &&
-		    (le32_to_cpu(sb->num_devices) != mddev->raid_disks)) {
-			DMERR("Reshaping raid sets not yet supported. (device count change from %u to %u)",
-			      sb->num_devices, mddev->raid_disks);
+		if (rs_takeover_requested(rs)) {
+			if (rt_cur && rt_new)
+				DMERR("Takeover raid sets from %s to %s not yet supported by metadata. (raid level change)",
+				      rt_cur->name, rt_new->name);
+			else
+				DMERR("Takeover raid sets not yet supported by metadata. (raid level change)");
+			return -EINVAL;
+		} else if (rs_reshape_requested(rs)) {
+			DMERR("Reshaping raid sets not yet supported by metadata. (raid layout change keeping level)");
+			if (mddev->layout != mddev->new_layout) {
+				if (rt_cur && rt_new)
+					DMERR("	 current layout %s vs new layout %s",
+					      rt_cur->name, rt_new->name);
+				else
+					DMERR("	 current layout 0x%X vs new layout 0x%X",
+					      le32_to_cpu(sb->layout), mddev->new_layout);
+			}
+			if (mddev->chunk_sectors != mddev->new_chunk_sectors)
+				DMERR("	 current stripe sectors %u vs new stripe sectors %u",
+				      mddev->chunk_sectors, mddev->new_chunk_sectors);
+			if (rs->delta_disks)
+				DMERR("	 current %u disks vs new %u disks",
+				      mddev->raid_disks, mddev->raid_disks + rs->delta_disks);
+			if (rs_is_raid10(rs)) {
+				DMERR("	 Old layout: %s w/ %u copies",
+				      raid10_md_layout_to_format(mddev->layout),
+				      raid10_md_layout_to_copies(mddev->layout));
+				DMERR("	 New layout: %s w/ %u copies",
+				      raid10_md_layout_to_format(mddev->new_layout),
+				      raid10_md_layout_to_copies(mddev->new_layout));
+			}
 			return -EINVAL;
 		}
 
 		DMINFO("Discovered old metadata format; upgrading to extended metadata format");
-
-		/* Table line is checked vs. authoritative superblock */
-		rs_set_new(rs);
 	}
 
 	if (!test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags))
@@ -2211,7 +2216,7 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev)
 				continue;
 
 			if (role != r->raid_disk) {
-				if (__is_raid10_near(mddev->layout)) {
+				if (rs_is_raid10(rs) && __is_raid10_near(mddev->layout)) {
 					if (mddev->raid_disks % __raid10_near_copies(mddev->layout) ||
 					    rs->raid_disks % rs->raid10_copies) {
 						rs->ti->error =
@@ -2994,6 +2999,9 @@ static int raid_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		}
 	}
 
+	/* Disable/enable discard support on raid set. */
+	configure_discard_support(rs);
+
 	mddev_unlock(&rs->md);
 	return 0;
 
@@ -3497,7 +3505,7 @@ static void rs_update_sbs(struct raid_set *rs)
 	struct mddev *mddev = &rs->md;
 	int ro = mddev->ro;
 
-	set_bit(MD_CHANGE_DEVS, &mddev->flags);
+	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 	mddev->ro = 0;
 	md_update_sb(mddev, 1);
 	mddev->ro = ro;
@@ -3580,12 +3588,6 @@ static int raid_preresume(struct dm_target *ti)
 	if (test_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags))
 		rs_update_sbs(rs);
 
-	/*
-	 * Disable/enable discard support on raid set after any
-	 * conversion, because devices can have been added
-	 */
-	configure_discard_support(rs);
-
 	/* Load the bitmap from disk unless raid0 */
 	r = __load_dirty_region_bitmap(rs);
 	if (r)
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index 9a8b71067c6e..2ddc2d20e62d 100644
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -260,7 +260,7 @@ static int mirror_flush(struct dm_target *ti)
 	struct mirror *m;
 	struct dm_io_request io_req = {
 		.bi_op = REQ_OP_WRITE,
-		.bi_op_flags = WRITE_FLUSH,
+		.bi_op_flags = REQ_PREFLUSH,
 		.mem.type = DM_IO_KMEM,
 		.mem.ptr.addr = NULL,
 		.client = ms->io_client,
@@ -656,7 +656,7 @@ static void do_write(struct mirror_set *ms, struct bio *bio)
 	struct mirror *m;
 	struct dm_io_request io_req = {
 		.bi_op = REQ_OP_WRITE,
-		.bi_op_flags = bio->bi_opf & WRITE_FLUSH_FUA,
+		.bi_op_flags = bio->bi_opf & (REQ_FUA | REQ_PREFLUSH),
 		.mem.type = DM_IO_BIO,
 		.mem.ptr.bio = bio,
 		.notify.fn = write_callback,
diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c
index 1d0d2adc050a..9d7275fb541a 100644
--- a/drivers/md/dm-rq.c
+++ b/drivers/md/dm-rq.c
@@ -23,11 +23,7 @@ static unsigned dm_mq_queue_depth = DM_MQ_QUEUE_DEPTH;
 #define RESERVED_REQUEST_BASED_IOS	256
 static unsigned reserved_rq_based_ios = RESERVED_REQUEST_BASED_IOS;
 
-#ifdef CONFIG_DM_MQ_DEFAULT
-static bool use_blk_mq = true;
-#else
-static bool use_blk_mq = false;
-#endif
+static bool use_blk_mq = IS_ENABLED(CONFIG_DM_MQ_DEFAULT);
 
 bool dm_use_blk_mq_default(void)
 {
@@ -75,12 +71,6 @@ static void dm_old_start_queue(struct request_queue *q)
 
 static void dm_mq_start_queue(struct request_queue *q)
 {
-	unsigned long flags;
-
-	spin_lock_irqsave(q->queue_lock, flags);
-	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
-	spin_unlock_irqrestore(q->queue_lock, flags);
-
 	blk_mq_start_stopped_hw_queues(q, true);
 	blk_mq_kick_requeue_list(q);
 }
@@ -105,20 +95,10 @@ static void dm_old_stop_queue(struct request_queue *q)
 
 static void dm_mq_stop_queue(struct request_queue *q)
 {
-	unsigned long flags;
-
-	spin_lock_irqsave(q->queue_lock, flags);
-	if (blk_queue_stopped(q)) {
-		spin_unlock_irqrestore(q->queue_lock, flags);
+	if (blk_mq_queue_stopped(q))
 		return;
-	}
 
-	queue_flag_set(QUEUE_FLAG_STOPPED, q);
-	spin_unlock_irqrestore(q->queue_lock, flags);
-
-	/* Avoid that requeuing could restart the queue. */
-	blk_mq_cancel_requeue_work(q);
-	blk_mq_stop_hw_queues(q);
+	blk_mq_quiesce_queue(q);
 }
 
 void dm_stop_queue(struct request_queue *q)
@@ -226,6 +206,9 @@ static void rq_end_stats(struct mapped_device *md, struct request *orig)
  */
 static void rq_completed(struct mapped_device *md, int rw, bool run_queue)
 {
+	struct request_queue *q = md->queue;
+	unsigned long flags;
+
 	atomic_dec(&md->pending[rw]);
 
 	/* nudge anyone waiting on suspend queue */
@@ -238,8 +221,11 @@ static void rq_completed(struct mapped_device *md, int rw, bool run_queue)
 	 * back into ->request_fn() could deadlock attempting to grab the
 	 * queue lock again.
 	 */
-	if (!md->queue->mq_ops && run_queue)
-		blk_run_queue_async(md->queue);
+	if (!q->mq_ops && run_queue) {
+		spin_lock_irqsave(q->queue_lock, flags);
+		blk_run_queue_async(q);
+		spin_unlock_irqrestore(q->queue_lock, flags);
+	}
 
 	/*
 	 * dm_put() must be at the end of this function. See the comment above
@@ -313,7 +299,7 @@ static void dm_unprep_request(struct request *rq)
 
 	if (!rq->q->mq_ops) {
 		rq->special = NULL;
-		rq->cmd_flags &= ~REQ_DONTPREP;
+		rq->rq_flags &= ~RQF_DONTPREP;
 	}
 
 	if (clone)
@@ -338,12 +324,7 @@ static void dm_old_requeue_request(struct request *rq)
 
 static void __dm_mq_kick_requeue_list(struct request_queue *q, unsigned long msecs)
 {
-	unsigned long flags;
-
-	spin_lock_irqsave(q->queue_lock, flags);
-	if (!blk_queue_stopped(q))
-		blk_mq_delay_kick_requeue_list(q, msecs);
-	spin_unlock_irqrestore(q->queue_lock, flags);
+	blk_mq_delay_kick_requeue_list(q, msecs);
 }
 
 void dm_mq_kick_requeue_list(struct mapped_device *md)
@@ -354,7 +335,7 @@ EXPORT_SYMBOL(dm_mq_kick_requeue_list);
 
 static void dm_mq_delay_requeue_request(struct request *rq, unsigned long msecs)
 {
-	blk_mq_requeue_request(rq);
+	blk_mq_requeue_request(rq, false);
 	__dm_mq_kick_requeue_list(rq->q, msecs);
 }
 
@@ -431,7 +412,7 @@ static void dm_softirq_done(struct request *rq)
 		return;
 	}
 
-	if (rq->cmd_flags & REQ_FAILED)
+	if (rq->rq_flags & RQF_FAILED)
 		mapped = false;
 
 	dm_done(clone, tio->error, mapped);
@@ -460,7 +441,7 @@ static void dm_complete_request(struct request *rq, int error)
  */
 static void dm_kill_unmapped_request(struct request *rq, int error)
 {
-	rq->cmd_flags |= REQ_FAILED;
+	rq->rq_flags |= RQF_FAILED;
 	dm_complete_request(rq, error);
 }
 
@@ -476,7 +457,7 @@ static void end_clone_request(struct request *clone, int error)
 		 * For just cleaning up the information of the queue in which
 		 * the clone was dispatched.
 		 * The clone is *NOT* freed actually here because it is alloced
-		 * from dm own mempool (REQ_ALLOCED isn't set).
+		 * from dm own mempool (RQF_ALLOCED isn't set).
 		 */
 		__blk_put_request(clone->q, clone);
 	}
@@ -497,7 +478,7 @@ static void dm_dispatch_clone_request(struct request *clone, struct request *rq)
 	int r;
 
 	if (blk_queue_io_stat(clone->q))
-		clone->cmd_flags |= REQ_IO_STAT;
+		clone->rq_flags |= RQF_IO_STAT;
 
 	clone->start_time = jiffies;
 	r = blk_insert_cloned_request(clone->q, clone);
@@ -633,7 +614,7 @@ static int dm_old_prep_fn(struct request_queue *q, struct request *rq)
 		return BLKPREP_DEFER;
 
 	rq->special = tio;
-	rq->cmd_flags |= REQ_DONTPREP;
+	rq->rq_flags |= RQF_DONTPREP;
 
 	return BLKPREP_OK;
 }
@@ -819,7 +800,7 @@ static void dm_old_request_fn(struct request_queue *q)
 			pos = blk_rq_pos(rq);
 
 		if ((dm_old_request_peeked_before_merge_deadline(md) &&
-		     md_in_flight(md) && rq->bio && rq->bio->bi_vcnt == 1 &&
+		     md_in_flight(md) && rq->bio && !bio_multiple_segments(rq->bio) &&
 		     md->last_rq_pos == pos && md->last_rq_rw == rq_data_dir(rq)) ||
 		    (ti->type->busy && ti->type->busy(ti))) {
 			blk_delay_queue(q, 10);
@@ -904,17 +885,6 @@ static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
 		dm_put_live_table(md, srcu_idx);
 	}
 
-	/*
-	 * On suspend dm_stop_queue() handles stopping the blk-mq
-	 * request_queue BUT: even though the hw_queues are marked
-	 * BLK_MQ_S_STOPPED at that point there is still a race that
-	 * is allowing block/blk-mq.c to call ->queue_rq against a
-	 * hctx that it really shouldn't.  The following check guards
-	 * against this rarity (albeit _not_ race-free).
-	 */
-	if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
-		return BLK_MQ_RQ_QUEUE_BUSY;
-
 	if (ti->type->busy && ti->type->busy(ti))
 		return BLK_MQ_RQ_QUEUE_BUSY;
 
diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c
index b8cf956b577b..b93476c3ba3f 100644
--- a/drivers/md/dm-snap-persistent.c
+++ b/drivers/md/dm-snap-persistent.c
@@ -741,7 +741,7 @@ static void persistent_commit_exception(struct dm_exception_store *store,
 	/*
 	 * Commit exceptions to disk.
 	 */
-	if (ps->valid && area_io(ps, REQ_OP_WRITE, WRITE_FLUSH_FUA))
+	if (ps->valid && area_io(ps, REQ_OP_WRITE, REQ_PREFLUSH | REQ_FUA))
 		ps->valid = 0;
 
 	/*
@@ -818,7 +818,7 @@ static int persistent_commit_merge(struct dm_exception_store *store,
 	for (i = 0; i < nr_merged; i++)
 		clear_exception(ps, ps->current_committed - 1 - i);
 
-	r = area_io(ps, REQ_OP_WRITE, WRITE_FLUSH_FUA);
+	r = area_io(ps, REQ_OP_WRITE, REQ_PREFLUSH | REQ_FUA);
 	if (r < 0)
 		return r;
 
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index c4b53b332607..0a427de23ed2 100644
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -871,7 +871,7 @@ static int dm_table_determine_type(struct dm_table *t)
 {
 	unsigned i;
 	unsigned bio_based = 0, request_based = 0, hybrid = 0;
-	bool verify_blk_mq = false;
+	unsigned sq_count = 0, mq_count = 0;
 	struct dm_target *tgt;
 	struct dm_dev_internal *dd;
 	struct list_head *devices = dm_table_get_devices(t);
@@ -924,12 +924,6 @@ static int dm_table_determine_type(struct dm_table *t)
 
 	BUG_ON(!request_based); /* No targets in this table */
 
-	if (list_empty(devices) && __table_type_request_based(live_md_type)) {
-		/* inherit live MD type */
-		t->type = live_md_type;
-		return 0;
-	}
-
 	/*
 	 * The only way to establish DM_TYPE_MQ_REQUEST_BASED is by
 	 * having a compatible target use dm_table_set_type.
@@ -948,6 +942,19 @@ verify_rq_based:
 		return -EINVAL;
 	}
 
+	if (list_empty(devices)) {
+		int srcu_idx;
+		struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx);
+
+		/* inherit live table's type and all_blk_mq */
+		if (live_table) {
+			t->type = live_table->type;
+			t->all_blk_mq = live_table->all_blk_mq;
+		}
+		dm_put_live_table(t->md, srcu_idx);
+		return 0;
+	}
+
 	/* Non-request-stackable devices can't be used for request-based dm */
 	list_for_each_entry(dd, devices, list) {
 		struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev);
@@ -959,19 +966,19 @@ verify_rq_based:
 		}
 
 		if (q->mq_ops)
-			verify_blk_mq = true;
+			mq_count++;
+		else
+			sq_count++;
 	}
+	if (sq_count && mq_count) {
+		DMERR("table load rejected: not all devices are blk-mq request-stackable");
+		return -EINVAL;
+	}
+	t->all_blk_mq = mq_count > 0;
 
-	if (verify_blk_mq) {
-		/* verify _all_ devices in the table are blk-mq devices */
-		list_for_each_entry(dd, devices, list)
-			if (!bdev_get_queue(dd->dm_dev->bdev)->mq_ops) {
-				DMERR("table load rejected: not all devices"
-				      " are blk-mq request-stackable");
-				return -EINVAL;
-			}
-
-		t->all_blk_mq = true;
+	if (t->type == DM_TYPE_MQ_REQUEST_BASED && !t->all_blk_mq) {
+		DMERR("table load rejected: all devices are not blk-mq request-stackable");
+		return -EINVAL;
 	}
 
 	return 0;
diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c
index 0aba34a7b3b3..7335d8a3fc47 100644
--- a/drivers/md/dm-verity-target.c
+++ b/drivers/md/dm-verity-target.c
@@ -868,7 +868,7 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
 
 	r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);
 	if (r) {
-		ti->error = "Data device lookup failed";
+		ti->error = "Hash device lookup failed";
 		goto bad;
 	}
 
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index ef7bf1dd6900..3086da5664f3 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -1525,9 +1525,9 @@ static struct mapped_device *alloc_dev(int minor)
 	if (!md->bdev)
 		goto bad;
 
-	bio_init(&md->flush_bio);
+	bio_init(&md->flush_bio, NULL, 0);
 	md->flush_bio.bi_bdev = md->bdev;
-	bio_set_op_attrs(&md->flush_bio, REQ_OP_WRITE, WRITE_FLUSH);
+	md->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 
 	dm_stats_init(&md->stats);
 
@@ -1886,9 +1886,7 @@ static void __dm_destroy(struct mapped_device *md, bool wait)
 	set_bit(DMF_FREEING, &md->flags);
 	spin_unlock(&_minor_lock);
 
-	spin_lock_irq(q->queue_lock);
-	queue_flag_set(QUEUE_FLAG_DYING, q);
-	spin_unlock_irq(q->queue_lock);
+	blk_set_queue_dying(q);
 
 	if (dm_request_based(md) && md->kworker_task)
 		kthread_flush_worker(&md->kworker);
diff --git a/drivers/md/linear.c b/drivers/md/linear.c
index 86f5d435901d..5975c9915684 100644
--- a/drivers/md/linear.c
+++ b/drivers/md/linear.c
@@ -21,6 +21,7 @@
 #include <linux/seq_file.h>
 #include <linux/module.h>
 #include <linux/slab.h>
+#include <trace/events/block.h>
 #include "md.h"
 #include "linear.h"
 
@@ -101,8 +102,8 @@ static struct linear_conf *linear_conf(struct mddev *mddev, int raid_disks)
 		sector_t sectors;
 
 		if (j < 0 || j >= raid_disks || disk->rdev) {
-			printk(KERN_ERR "md/linear:%s: disk numbering problem. Aborting!\n",
-			       mdname(mddev));
+			pr_warn("md/linear:%s: disk numbering problem. Aborting!\n",
+				mdname(mddev));
 			goto out;
 		}
 
@@ -123,8 +124,8 @@ static struct linear_conf *linear_conf(struct mddev *mddev, int raid_disks)
 			discard_supported = true;
 	}
 	if (cnt != raid_disks) {
-		printk(KERN_ERR "md/linear:%s: not enough drives present. Aborting!\n",
-		       mdname(mddev));
+		pr_warn("md/linear:%s: not enough drives present. Aborting!\n",
+			mdname(mddev));
 		goto out;
 	}
 
@@ -227,22 +228,22 @@ static void linear_make_request(struct mddev *mddev, struct bio *bio)
 	}
 
 	do {
-		tmp_dev = which_dev(mddev, bio->bi_iter.bi_sector);
+		sector_t bio_sector = bio->bi_iter.bi_sector;
+		tmp_dev = which_dev(mddev, bio_sector);
 		start_sector = tmp_dev->end_sector - tmp_dev->rdev->sectors;
 		end_sector = tmp_dev->end_sector;
 		data_offset = tmp_dev->rdev->data_offset;
 		bio->bi_bdev = tmp_dev->rdev->bdev;
 
-		if (unlikely(bio->bi_iter.bi_sector >= end_sector ||
-			     bio->bi_iter.bi_sector < start_sector))
+		if (unlikely(bio_sector >= end_sector ||
+			     bio_sector < start_sector))
 			goto out_of_bounds;
 
 		if (unlikely(bio_end_sector(bio) > end_sector)) {
 			/* This bio crosses a device boundary, so we have to
 			 * split it.
 			 */
-			split = bio_split(bio, end_sector -
-					  bio->bi_iter.bi_sector,
+			split = bio_split(bio, end_sector - bio_sector,
 					  GFP_NOIO, fs_bio_set);
 			bio_chain(split, bio);
 		} else {
@@ -256,15 +257,18 @@ static void linear_make_request(struct mddev *mddev, struct bio *bio)
 			 !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) {
 			/* Just ignore it */
 			bio_endio(split);
-		} else
+		} else {
+			if (mddev->gendisk)
+				trace_block_bio_remap(bdev_get_queue(split->bi_bdev),
+						      split, disk_devt(mddev->gendisk),
+						      bio_sector);
 			generic_make_request(split);
+		}
 	} while (split != bio);
 	return;
 
 out_of_bounds:
-	printk(KERN_ERR
-	       "md/linear:%s: make_request: Sector %llu out of bounds on "
-	       "dev %s: %llu sectors, offset %llu\n",
+	pr_err("md/linear:%s: make_request: Sector %llu out of bounds on dev %s: %llu sectors, offset %llu\n",
 	       mdname(mddev),
 	       (unsigned long long)bio->bi_iter.bi_sector,
 	       bdevname(tmp_dev->rdev->bdev, b),
@@ -275,7 +279,6 @@ out_of_bounds:
 
 static void linear_status (struct seq_file *seq, struct mddev *mddev)
 {
-
 	seq_printf(seq, " %dk rounding", mddev->chunk_sectors / 2);
 }
 
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 2089d46b0eb8..82821ee0d57f 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -30,6 +30,18 @@
    You should have received a copy of the GNU General Public License
    (for example /usr/src/linux/COPYING); if not, write to the Free
    Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+   Errors, Warnings, etc.
+   Please use:
+     pr_crit() for error conditions that risk data loss
+     pr_err() for error conditions that are unexpected, like an IO error
+         or internal inconsistency
+     pr_warn() for error conditions that could have been predicated, like
+         adding a device to an array when it has incompatible metadata
+     pr_info() for every interesting, very rare events, like an array starting
+         or stopping, or resync starting or stopping
+     pr_debug() for everything else.
+
 */
 
 #include <linux/kthread.h>
@@ -52,6 +64,7 @@
 #include <linux/raid/md_p.h>
 #include <linux/raid/md_u.h>
 #include <linux/slab.h>
+#include <trace/events/block.h>
 #include "md.h"
 #include "bitmap.h"
 #include "md-cluster.h"
@@ -394,7 +407,7 @@ static void submit_flushes(struct work_struct *ws)
 			bi->bi_end_io = md_end_flush;
 			bi->bi_private = rdev;
 			bi->bi_bdev = rdev->bdev;
-			bio_set_op_attrs(bi, REQ_OP_WRITE, WRITE_FLUSH);
+			bi->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 			atomic_inc(&mddev->flush_pending);
 			submit_bio(bi);
 			rcu_read_lock();
@@ -684,11 +697,8 @@ static inline sector_t calc_dev_sboffset(struct md_rdev *rdev)
 static int alloc_disk_sb(struct md_rdev *rdev)
 {
 	rdev->sb_page = alloc_page(GFP_KERNEL);
-	if (!rdev->sb_page) {
-		printk(KERN_ALERT "md: out of memory.\n");
+	if (!rdev->sb_page)
 		return -ENOMEM;
-	}
-
 	return 0;
 }
 
@@ -715,9 +725,15 @@ static void super_written(struct bio *bio)
 	struct mddev *mddev = rdev->mddev;
 
 	if (bio->bi_error) {
-		printk("md: super_written gets error=%d\n", bio->bi_error);
+		pr_err("md: super_written gets error=%d\n", bio->bi_error);
 		md_error(mddev, rdev);
-	}
+		if (!test_bit(Faulty, &rdev->flags)
+		    && (bio->bi_opf & MD_FAILFAST)) {
+			set_bit(MD_SB_NEED_REWRITE, &mddev->sb_flags);
+			set_bit(LastDev, &rdev->flags);
+		}
+	} else
+		clear_bit(LastDev, &rdev->flags);
 
 	if (atomic_dec_and_test(&mddev->pending_writes))
 		wake_up(&mddev->sb_wait);
@@ -734,7 +750,13 @@ void md_super_write(struct mddev *mddev, struct md_rdev *rdev,
 	 * if zero is reached.
 	 * If an error occurred, call md_error
 	 */
-	struct bio *bio = bio_alloc_mddev(GFP_NOIO, 1, mddev);
+	struct bio *bio;
+	int ff = 0;
+
+	if (test_bit(Faulty, &rdev->flags))
+		return;
+
+	bio = bio_alloc_mddev(GFP_NOIO, 1, mddev);
 
 	atomic_inc(&rdev->nr_pending);
 
@@ -743,16 +765,24 @@ void md_super_write(struct mddev *mddev, struct md_rdev *rdev,
 	bio_add_page(bio, page, size, 0);
 	bio->bi_private = rdev;
 	bio->bi_end_io = super_written;
-	bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_FLUSH_FUA);
+
+	if (test_bit(MD_FAILFAST_SUPPORTED, &mddev->flags) &&
+	    test_bit(FailFast, &rdev->flags) &&
+	    !test_bit(LastDev, &rdev->flags))
+		ff = MD_FAILFAST;
+	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_FUA | ff;
 
 	atomic_inc(&mddev->pending_writes);
 	submit_bio(bio);
 }
 
-void md_super_wait(struct mddev *mddev)
+int md_super_wait(struct mddev *mddev)
 {
 	/* wait for all superblock writes that were scheduled to complete */
 	wait_event(mddev->sb_wait, atomic_read(&mddev->pending_writes)==0);
+	if (test_and_clear_bit(MD_SB_NEED_REWRITE, &mddev->sb_flags))
+		return -EAGAIN;
+	return 0;
 }
 
 int sync_page_io(struct md_rdev *rdev, sector_t sector, int size,
@@ -795,8 +825,8 @@ static int read_disk_sb(struct md_rdev *rdev, int size)
 	return 0;
 
 fail:
-	printk(KERN_WARNING "md: disabled device %s, could not read superblock.\n",
-		bdevname(rdev->bdev,b));
+	pr_err("md: disabled device %s, could not read superblock.\n",
+	       bdevname(rdev->bdev,b));
 	return -EINVAL;
 }
 
@@ -818,7 +848,6 @@ static int sb_equal(mdp_super_t *sb1, mdp_super_t *sb2)
 
 	if (!tmp1 || !tmp2) {
 		ret = 0;
-		printk(KERN_INFO "md.c sb_equal(): failed to allocate memory!\n");
 		goto abort;
 	}
 
@@ -932,7 +961,7 @@ int md_check_no_bitmap(struct mddev *mddev)
 {
 	if (!mddev->bitmap_info.file && !mddev->bitmap_info.offset)
 		return 0;
-	printk(KERN_ERR "%s: bitmaps are not supported for %s\n",
+	pr_warn("%s: bitmaps are not supported for %s\n",
 		mdname(mddev), mddev->pers->name);
 	return 1;
 }
@@ -956,7 +985,8 @@ static int super_90_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor
 	rdev->sb_start = calc_dev_sboffset(rdev);
 
 	ret = read_disk_sb(rdev, MD_SB_BYTES);
-	if (ret) return ret;
+	if (ret)
+		return ret;
 
 	ret = -EINVAL;
 
@@ -964,17 +994,15 @@ static int super_90_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor
 	sb = page_address(rdev->sb_page);
 
 	if (sb->md_magic != MD_SB_MAGIC) {
-		printk(KERN_ERR "md: invalid raid superblock magic on %s\n",
-		       b);
+		pr_warn("md: invalid raid superblock magic on %s\n", b);
 		goto abort;
 	}
 
 	if (sb->major_version != 0 ||
 	    sb->minor_version < 90 ||
 	    sb->minor_version > 91) {
-		printk(KERN_WARNING "Bad version number %d.%d on %s\n",
-			sb->major_version, sb->minor_version,
-			b);
+		pr_warn("Bad version number %d.%d on %s\n",
+			sb->major_version, sb->minor_version, b);
 		goto abort;
 	}
 
@@ -982,8 +1010,7 @@ static int super_90_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor
 		goto abort;
 
 	if (md_csum_fold(calc_sb_csum(sb)) != md_csum_fold(sb->sb_csum)) {
-		printk(KERN_WARNING "md: invalid superblock checksum on %s\n",
-			b);
+		pr_warn("md: invalid superblock checksum on %s\n", b);
 		goto abort;
 	}
 
@@ -1004,14 +1031,13 @@ static int super_90_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor
 		__u64 ev1, ev2;
 		mdp_super_t *refsb = page_address(refdev->sb_page);
 		if (!uuid_equal(refsb, sb)) {
-			printk(KERN_WARNING "md: %s has different UUID to %s\n",
+			pr_warn("md: %s has different UUID to %s\n",
 				b, bdevname(refdev->bdev,b2));
 			goto abort;
 		}
 		if (!sb_equal(refsb, sb)) {
-			printk(KERN_WARNING "md: %s has same UUID"
-			       " but different superblock to %s\n",
-			       b, bdevname(refdev->bdev, b2));
+			pr_warn("md: %s has same UUID but different superblock to %s\n",
+				b, bdevname(refdev->bdev, b2));
 			goto abort;
 		}
 		ev1 = md_event(sb);
@@ -1158,6 +1184,8 @@ static int super_90_validate(struct mddev *mddev, struct md_rdev *rdev)
 		}
 		if (desc->state & (1<<MD_DISK_WRITEMOSTLY))
 			set_bit(WriteMostly, &rdev->flags);
+		if (desc->state & (1<<MD_DISK_FAILFAST))
+			set_bit(FailFast, &rdev->flags);
 	} else /* MULTIPATH are always insync */
 		set_bit(In_sync, &rdev->flags);
 	return 0;
@@ -1283,6 +1311,8 @@ static void super_90_sync(struct mddev *mddev, struct md_rdev *rdev)
 		}
 		if (test_bit(WriteMostly, &rdev2->flags))
 			d->state |= (1<<MD_DISK_WRITEMOSTLY);
+		if (test_bit(FailFast, &rdev2->flags))
+			d->state |= (1<<MD_DISK_FAILFAST);
 	}
 	/* now set the "removed" and "faulty" bits on any missing devices */
 	for (i=0 ; i < mddev->raid_disks ; i++) {
@@ -1324,9 +1354,10 @@ super_90_rdev_size_change(struct md_rdev *rdev, sector_t num_sectors)
 	if (IS_ENABLED(CONFIG_LBDAF) && (u64)num_sectors >= (2ULL << 32) &&
 	    rdev->mddev->level >= 1)
 		num_sectors = (sector_t)(2ULL << 32) - 2;
-	md_super_write(rdev->mddev, rdev, rdev->sb_start, rdev->sb_size,
+	do {
+		md_super_write(rdev->mddev, rdev, rdev->sb_start, rdev->sb_size,
 		       rdev->sb_page);
-	md_super_wait(rdev->mddev);
+	} while (md_super_wait(rdev->mddev) < 0);
 	return num_sectors;
 }
 
@@ -1413,13 +1444,13 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_
 		return -EINVAL;
 
 	if (calc_sb_1_csum(sb) != sb->sb_csum) {
-		printk("md: invalid superblock checksum on %s\n",
+		pr_warn("md: invalid superblock checksum on %s\n",
 			bdevname(rdev->bdev,b));
 		return -EINVAL;
 	}
 	if (le64_to_cpu(sb->data_size) < 10) {
-		printk("md: data_size too small on %s\n",
-		       bdevname(rdev->bdev,b));
+		pr_warn("md: data_size too small on %s\n",
+			bdevname(rdev->bdev,b));
 		return -EINVAL;
 	}
 	if (sb->pad0 ||
@@ -1503,8 +1534,7 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_
 		    sb->level != refsb->level ||
 		    sb->layout != refsb->layout ||
 		    sb->chunksize != refsb->chunksize) {
-			printk(KERN_WARNING "md: %s has strangely different"
-				" superblock to %s\n",
+			pr_warn("md: %s has strangely different superblock to %s\n",
 				bdevname(rdev->bdev,b),
 				bdevname(refdev->bdev,b2));
 			return -EINVAL;
@@ -1646,8 +1676,7 @@ static int super_1_validate(struct mddev *mddev, struct md_rdev *rdev)
 		case MD_DISK_ROLE_JOURNAL: /* journal device */
 			if (!(le32_to_cpu(sb->feature_map) & MD_FEATURE_JOURNAL)) {
 				/* journal device without journal feature */
-				printk(KERN_WARNING
-				  "md: journal device provided without journal feature, ignoring the device\n");
+				pr_warn("md: journal device provided without journal feature, ignoring the device\n");
 				return -EINVAL;
 			}
 			set_bit(Journal, &rdev->flags);
@@ -1669,6 +1698,8 @@ static int super_1_validate(struct mddev *mddev, struct md_rdev *rdev)
 		}
 		if (sb->devflags & WriteMostly1)
 			set_bit(WriteMostly, &rdev->flags);
+		if (sb->devflags & FailFast1)
+			set_bit(FailFast, &rdev->flags);
 		if (le32_to_cpu(sb->feature_map) & MD_FEATURE_REPLACEMENT)
 			set_bit(Replacement, &rdev->flags);
 	} else /* MULTIPATH are always insync */
@@ -1707,6 +1738,10 @@ static void super_1_sync(struct mddev *mddev, struct md_rdev *rdev)
 	sb->chunksize = cpu_to_le32(mddev->chunk_sectors);
 	sb->level = cpu_to_le32(mddev->level);
 	sb->layout = cpu_to_le32(mddev->layout);
+	if (test_bit(FailFast, &rdev->flags))
+		sb->devflags |= FailFast1;
+	else
+		sb->devflags &= ~FailFast1;
 
 	if (test_bit(WriteMostly, &rdev->flags))
 		sb->devflags |= WriteMostly1;
@@ -1863,9 +1898,10 @@ super_1_rdev_size_change(struct md_rdev *rdev, sector_t num_sectors)
 	sb->data_size = cpu_to_le64(num_sectors);
 	sb->super_offset = rdev->sb_start;
 	sb->sb_csum = calc_sb_1_csum(sb);
-	md_super_write(rdev->mddev, rdev, rdev->sb_start, rdev->sb_size,
-		       rdev->sb_page);
-	md_super_wait(rdev->mddev);
+	do {
+		md_super_write(rdev->mddev, rdev, rdev->sb_start, rdev->sb_size,
+			       rdev->sb_page);
+	} while (md_super_wait(rdev->mddev) < 0);
 	return num_sectors;
 
 }
@@ -2004,9 +2040,9 @@ int md_integrity_register(struct mddev *mddev)
 	blk_integrity_register(mddev->gendisk,
 			       bdev_get_integrity(reference->bdev));
 
-	printk(KERN_NOTICE "md: data integrity enabled on %s\n", mdname(mddev));
+	pr_debug("md: data integrity enabled on %s\n", mdname(mddev));
 	if (bioset_integrity_create(mddev->bio_set, BIO_POOL_SIZE)) {
-		printk(KERN_ERR "md: failed to create integrity pool for %s\n",
+		pr_err("md: failed to create integrity pool for %s\n",
 		       mdname(mddev));
 		return -EINVAL;
 	}
@@ -2034,8 +2070,8 @@ int md_integrity_add_rdev(struct md_rdev *rdev, struct mddev *mddev)
 		return 0;
 
 	if (blk_integrity_compare(mddev->gendisk, rdev->bdev->bd_disk) != 0) {
-		printk(KERN_NOTICE "%s: incompatible integrity profile for %s\n",
-				mdname(mddev), bdevname(rdev->bdev, name));
+		pr_err("%s: incompatible integrity profile for %s\n",
+		       mdname(mddev), bdevname(rdev->bdev, name));
 		return -ENXIO;
 	}
 
@@ -2089,15 +2125,15 @@ static int bind_rdev_to_array(struct md_rdev *rdev, struct mddev *mddev)
 	rcu_read_unlock();
 	if (!test_bit(Journal, &rdev->flags) &&
 	    mddev->max_disks && rdev->desc_nr >= mddev->max_disks) {
-		printk(KERN_WARNING "md: %s: array is limited to %d devices\n",
-		       mdname(mddev), mddev->max_disks);
+		pr_warn("md: %s: array is limited to %d devices\n",
+			mdname(mddev), mddev->max_disks);
 		return -EBUSY;
 	}
 	bdevname(rdev->bdev,b);
 	strreplace(b, '/', '!');
 
 	rdev->mddev = mddev;
-	printk(KERN_INFO "md: bind<%s>\n", b);
+	pr_debug("md: bind<%s>\n", b);
 
 	if ((err = kobject_add(&rdev->kobj, &mddev->kobj, "dev-%s", b)))
 		goto fail;
@@ -2116,8 +2152,8 @@ static int bind_rdev_to_array(struct md_rdev *rdev, struct mddev *mddev)
 	return 0;
 
  fail:
-	printk(KERN_WARNING "md: failed to register dev-%s for %s\n",
-	       b, mdname(mddev));
+	pr_warn("md: failed to register dev-%s for %s\n",
+		b, mdname(mddev));
 	return err;
 }
 
@@ -2134,7 +2170,7 @@ static void unbind_rdev_from_array(struct md_rdev *rdev)
 
 	bd_unlink_disk_holder(rdev->bdev, rdev->mddev->gendisk);
 	list_del_rcu(&rdev->same_set);
-	printk(KERN_INFO "md: unbind<%s>\n", bdevname(rdev->bdev,b));
+	pr_debug("md: unbind<%s>\n", bdevname(rdev->bdev,b));
 	rdev->mddev = NULL;
 	sysfs_remove_link(&rdev->kobj, "block");
 	sysfs_put(rdev->sysfs_state);
@@ -2164,8 +2200,7 @@ static int lock_rdev(struct md_rdev *rdev, dev_t dev, int shared)
 	bdev = blkdev_get_by_dev(dev, FMODE_READ|FMODE_WRITE|FMODE_EXCL,
 				 shared ? (struct md_rdev *)lock_rdev : rdev);
 	if (IS_ERR(bdev)) {
-		printk(KERN_ERR "md: could not open %s.\n",
-			__bdevname(dev, b));
+		pr_warn("md: could not open %s.\n", __bdevname(dev, b));
 		return PTR_ERR(bdev);
 	}
 	rdev->bdev = bdev;
@@ -2185,8 +2220,7 @@ static void export_rdev(struct md_rdev *rdev)
 {
 	char b[BDEVNAME_SIZE];
 
-	printk(KERN_INFO "md: export_rdev(%s)\n",
-		bdevname(rdev->bdev,b));
+	pr_debug("md: export_rdev(%s)\n", bdevname(rdev->bdev,b));
 	md_rdev_clear(rdev);
 #ifndef MODULE
 	if (test_bit(AutoDetected, &rdev->flags))
@@ -2288,24 +2322,24 @@ void md_update_sb(struct mddev *mddev, int force_change)
 
 	if (mddev->ro) {
 		if (force_change)
-			set_bit(MD_CHANGE_DEVS, &mddev->flags);
+			set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 		return;
 	}
 
 repeat:
 	if (mddev_is_clustered(mddev)) {
-		if (test_and_clear_bit(MD_CHANGE_DEVS, &mddev->flags))
+		if (test_and_clear_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags))
 			force_change = 1;
-		if (test_and_clear_bit(MD_CHANGE_CLEAN, &mddev->flags))
+		if (test_and_clear_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags))
 			nospares = 1;
 		ret = md_cluster_ops->metadata_update_start(mddev);
 		/* Has someone else has updated the sb */
 		if (!does_sb_need_changing(mddev)) {
 			if (ret == 0)
 				md_cluster_ops->metadata_update_cancel(mddev);
-			bit_clear_unless(&mddev->flags, BIT(MD_CHANGE_PENDING),
-							 BIT(MD_CHANGE_DEVS) |
-							 BIT(MD_CHANGE_CLEAN));
+			bit_clear_unless(&mddev->sb_flags, BIT(MD_SB_CHANGE_PENDING),
+							 BIT(MD_SB_CHANGE_DEVS) |
+							 BIT(MD_SB_CHANGE_CLEAN));
 			return;
 		}
 	}
@@ -2321,10 +2355,10 @@ repeat:
 
 	}
 	if (!mddev->persistent) {
-		clear_bit(MD_CHANGE_CLEAN, &mddev->flags);
-		clear_bit(MD_CHANGE_DEVS, &mddev->flags);
+		clear_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
+		clear_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 		if (!mddev->external) {
-			clear_bit(MD_CHANGE_PENDING, &mddev->flags);
+			clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
 			rdev_for_each(rdev, mddev) {
 				if (rdev->badblocks.changed) {
 					rdev->badblocks.changed = 0;
@@ -2344,9 +2378,9 @@ repeat:
 
 	mddev->utime = ktime_get_real_seconds();
 
-	if (test_and_clear_bit(MD_CHANGE_DEVS, &mddev->flags))
+	if (test_and_clear_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags))
 		force_change = 1;
-	if (test_and_clear_bit(MD_CHANGE_CLEAN, &mddev->flags))
+	if (test_and_clear_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags))
 		/* just a clean<-> dirty transition, possibly leave spares alone,
 		 * though if events isn't the right even/odd, we will have to do
 		 * spares after all
@@ -2402,6 +2436,9 @@ repeat:
 	pr_debug("md: updating %s RAID superblock on device (in sync %d)\n",
 		 mdname(mddev), mddev->in_sync);
 
+	if (mddev->queue)
+		blk_add_trace_msg(mddev->queue, "md md_update_sb");
+rewrite:
 	bitmap_update_sb(mddev->bitmap);
 	rdev_for_each(rdev, mddev) {
 		char b[BDEVNAME_SIZE];
@@ -2433,15 +2470,16 @@ repeat:
 			/* only need to write one superblock... */
 			break;
 	}
-	md_super_wait(mddev);
-	/* if there was a failure, MD_CHANGE_DEVS was set, and we re-write super */
+	if (md_super_wait(mddev) < 0)
+		goto rewrite;
+	/* if there was a failure, MD_SB_CHANGE_DEVS was set, and we re-write super */
 
 	if (mddev_is_clustered(mddev) && ret == 0)
 		md_cluster_ops->metadata_update_finish(mddev);
 
 	if (mddev->in_sync != sync_req ||
-	    !bit_clear_unless(&mddev->flags, BIT(MD_CHANGE_PENDING),
-			       BIT(MD_CHANGE_DEVS) | BIT(MD_CHANGE_CLEAN)))
+	    !bit_clear_unless(&mddev->sb_flags, BIT(MD_SB_CHANGE_PENDING),
+			       BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_CLEAN)))
 		/* have to write it out again */
 		goto repeat;
 	wake_up(&mddev->sb_wait);
@@ -2485,7 +2523,7 @@ static int add_bound_rdev(struct md_rdev *rdev)
 	}
 	sysfs_notify_dirent_safe(rdev->sysfs_state);
 
-	set_bit(MD_CHANGE_DEVS, &mddev->flags);
+	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 	if (mddev->degraded)
 		set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
 	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
@@ -2523,51 +2561,41 @@ struct rdev_sysfs_entry {
 static ssize_t
 state_show(struct md_rdev *rdev, char *page)
 {
-	char *sep = "";
+	char *sep = ",";
 	size_t len = 0;
 	unsigned long flags = ACCESS_ONCE(rdev->flags);
 
 	if (test_bit(Faulty, &flags) ||
-	    rdev->badblocks.unacked_exist) {
-		len+= sprintf(page+len, "%sfaulty",sep);
-		sep = ",";
-	}
-	if (test_bit(In_sync, &flags)) {
-		len += sprintf(page+len, "%sin_sync",sep);
-		sep = ",";
-	}
-	if (test_bit(Journal, &flags)) {
-		len += sprintf(page+len, "%sjournal",sep);
-		sep = ",";
-	}
-	if (test_bit(WriteMostly, &flags)) {
-		len += sprintf(page+len, "%swrite_mostly",sep);
-		sep = ",";
-	}
+	    (!test_bit(ExternalBbl, &flags) &&
+	    rdev->badblocks.unacked_exist))
+		len += sprintf(page+len, "faulty%s", sep);
+	if (test_bit(In_sync, &flags))
+		len += sprintf(page+len, "in_sync%s", sep);
+	if (test_bit(Journal, &flags))
+		len += sprintf(page+len, "journal%s", sep);
+	if (test_bit(WriteMostly, &flags))
+		len += sprintf(page+len, "write_mostly%s", sep);
 	if (test_bit(Blocked, &flags) ||
 	    (rdev->badblocks.unacked_exist
-	     && !test_bit(Faulty, &flags))) {
-		len += sprintf(page+len, "%sblocked", sep);
-		sep = ",";
-	}
+	     && !test_bit(Faulty, &flags)))
+		len += sprintf(page+len, "blocked%s", sep);
 	if (!test_bit(Faulty, &flags) &&
 	    !test_bit(Journal, &flags) &&
-	    !test_bit(In_sync, &flags)) {
-		len += sprintf(page+len, "%sspare", sep);
-		sep = ",";
-	}
-	if (test_bit(WriteErrorSeen, &flags)) {
-		len += sprintf(page+len, "%swrite_error", sep);
-		sep = ",";
-	}
-	if (test_bit(WantReplacement, &flags)) {
-		len += sprintf(page+len, "%swant_replacement", sep);
-		sep = ",";
-	}
-	if (test_bit(Replacement, &flags)) {
-		len += sprintf(page+len, "%sreplacement", sep);
-		sep = ",";
-	}
+	    !test_bit(In_sync, &flags))
+		len += sprintf(page+len, "spare%s", sep);
+	if (test_bit(WriteErrorSeen, &flags))
+		len += sprintf(page+len, "write_error%s", sep);
+	if (test_bit(WantReplacement, &flags))
+		len += sprintf(page+len, "want_replacement%s", sep);
+	if (test_bit(Replacement, &flags))
+		len += sprintf(page+len, "replacement%s", sep);
+	if (test_bit(ExternalBbl, &flags))
+		len += sprintf(page+len, "external_bbl%s", sep);
+	if (test_bit(FailFast, &flags))
+		len += sprintf(page+len, "failfast%s", sep);
+
+	if (len)
+		len -= strlen(sep);
 
 	return len+sprintf(page+len, "\n");
 }
@@ -2587,6 +2615,7 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
 	 *            so that it gets rebuilt based on bitmap
 	 *  write_error - sets WriteErrorSeen
 	 *  -write_error - clears WriteErrorSeen
+	 *  {,-}failfast - set/clear FailFast
 	 */
 	int err = -EINVAL;
 	if (cmd_match(buf, "faulty") && rdev->mddev->pers) {
@@ -2610,8 +2639,10 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
 
 			if (err == 0) {
 				md_kick_rdev_from_array(rdev);
-				if (mddev->pers)
-					md_update_sb(mddev, 1);
+				if (mddev->pers) {
+					set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
+					md_wakeup_thread(mddev->thread);
+				}
 				md_new_event(mddev);
 			}
 		}
@@ -2626,6 +2657,7 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
 		err = 0;
 	} else if (cmd_match(buf, "-blocked")) {
 		if (!test_bit(Faulty, &rdev->flags) &&
+		    !test_bit(ExternalBbl, &rdev->flags) &&
 		    rdev->badblocks.unacked_exist) {
 			/* metadata handler doesn't understand badblocks,
 			 * so we need to fail the device
@@ -2642,6 +2674,12 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
 	} else if (cmd_match(buf, "insync") && rdev->raid_disk == -1) {
 		set_bit(In_sync, &rdev->flags);
 		err = 0;
+	} else if (cmd_match(buf, "failfast")) {
+		set_bit(FailFast, &rdev->flags);
+		err = 0;
+	} else if (cmd_match(buf, "-failfast")) {
+		clear_bit(FailFast, &rdev->flags);
+		err = 0;
 	} else if (cmd_match(buf, "-insync") && rdev->raid_disk >= 0 &&
 		   !test_bit(Journal, &rdev->flags)) {
 		if (rdev->mddev->pers == NULL) {
@@ -2708,6 +2746,13 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
 			}
 		} else
 			err = -EBUSY;
+	} else if (cmd_match(buf, "external_bbl") && (rdev->mddev->external)) {
+		set_bit(ExternalBbl, &rdev->flags);
+		rdev->badblocks.shift = 0;
+		err = 0;
+	} else if (cmd_match(buf, "-external_bbl") && (rdev->mddev->external)) {
+		clear_bit(ExternalBbl, &rdev->flags);
+		err = 0;
 	}
 	if (!err)
 		sysfs_notify_dirent_safe(rdev->sysfs_state);
@@ -3211,10 +3256,8 @@ static struct md_rdev *md_import_device(dev_t newdev, int super_format, int supe
 	sector_t size;
 
 	rdev = kzalloc(sizeof(*rdev), GFP_KERNEL);
-	if (!rdev) {
-		printk(KERN_ERR "md: could not alloc mem for new device!\n");
+	if (!rdev)
 		return ERR_PTR(-ENOMEM);
-	}
 
 	err = md_rdev_init(rdev);
 	if (err)
@@ -3231,8 +3274,7 @@ static struct md_rdev *md_import_device(dev_t newdev, int super_format, int supe
 
 	size = i_size_read(rdev->bdev->bd_inode) >> BLOCK_SIZE_BITS;
 	if (!size) {
-		printk(KERN_WARNING
-			"md: %s has zero or unknown size, marking faulty!\n",
+		pr_warn("md: %s has zero or unknown size, marking faulty!\n",
 			bdevname(rdev->bdev,b));
 		err = -EINVAL;
 		goto abort_free;
@@ -3242,16 +3284,13 @@ static struct md_rdev *md_import_device(dev_t newdev, int super_format, int supe
 		err = super_types[super_format].
 			load_super(rdev, NULL, super_minor);
 		if (err == -EINVAL) {
-			printk(KERN_WARNING
-				"md: %s does not have a valid v%d.%d "
-			       "superblock, not importing!\n",
+			pr_warn("md: %s does not have a valid v%d.%d superblock, not importing!\n",
 				bdevname(rdev->bdev,b),
-			       super_format, super_minor);
+				super_format, super_minor);
 			goto abort_free;
 		}
 		if (err < 0) {
-			printk(KERN_WARNING
-				"md: could not read %s's sb, not importing!\n",
+			pr_warn("md: could not read %s's sb, not importing!\n",
 				bdevname(rdev->bdev,b));
 			goto abort_free;
 		}
@@ -3287,9 +3326,7 @@ static void analyze_sbs(struct mddev *mddev)
 		case 0:
 			break;
 		default:
-			printk( KERN_ERR \
-				"md: fatal superblock inconsistency in %s"
-				" -- removing from array\n",
+			pr_warn("md: fatal superblock inconsistency in %s -- removing from array\n",
 				bdevname(rdev->bdev,b));
 			md_kick_rdev_from_array(rdev);
 		}
@@ -3302,18 +3339,16 @@ static void analyze_sbs(struct mddev *mddev)
 		if (mddev->max_disks &&
 		    (rdev->desc_nr >= mddev->max_disks ||
 		     i > mddev->max_disks)) {
-			printk(KERN_WARNING
-			       "md: %s: %s: only %d devices permitted\n",
-			       mdname(mddev), bdevname(rdev->bdev, b),
-			       mddev->max_disks);
+			pr_warn("md: %s: %s: only %d devices permitted\n",
+				mdname(mddev), bdevname(rdev->bdev, b),
+				mddev->max_disks);
 			md_kick_rdev_from_array(rdev);
 			continue;
 		}
 		if (rdev != freshest) {
 			if (super_types[mddev->major_version].
 			    validate_super(mddev, rdev)) {
-				printk(KERN_WARNING "md: kicking non-fresh %s"
-					" from array!\n",
+				pr_warn("md: kicking non-fresh %s from array!\n",
 					bdevname(rdev->bdev,b));
 				md_kick_rdev_from_array(rdev);
 				continue;
@@ -3384,7 +3419,7 @@ safe_delay_store(struct mddev *mddev, const char *cbuf, size_t len)
 	unsigned long msec;
 
 	if (mddev_is_clustered(mddev)) {
-		pr_info("md: Safemode is disabled for clustered mode\n");
+		pr_warn("md: Safemode is disabled for clustered mode\n");
 		return -EINVAL;
 	}
 
@@ -3472,8 +3507,8 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
 
 	rv = -EINVAL;
 	if (!mddev->pers->quiesce) {
-		printk(KERN_WARNING "md: %s: %s does not support online personality change\n",
-		       mdname(mddev), mddev->pers->name);
+		pr_warn("md: %s: %s does not support online personality change\n",
+			mdname(mddev), mddev->pers->name);
 		goto out_unlock;
 	}
 
@@ -3491,7 +3526,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
 	pers = find_pers(level, clevel);
 	if (!pers || !try_module_get(pers->owner)) {
 		spin_unlock(&pers_lock);
-		printk(KERN_WARNING "md: personality %s not loaded\n", clevel);
+		pr_warn("md: personality %s not loaded\n", clevel);
 		rv = -EINVAL;
 		goto out_unlock;
 	}
@@ -3505,8 +3540,8 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
 	}
 	if (!pers->takeover) {
 		module_put(pers->owner);
-		printk(KERN_WARNING "md: %s: %s does not support personality takeover\n",
-		       mdname(mddev), clevel);
+		pr_warn("md: %s: %s does not support personality takeover\n",
+			mdname(mddev), clevel);
 		rv = -EINVAL;
 		goto out_unlock;
 	}
@@ -3526,8 +3561,8 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
 		mddev->delta_disks = 0;
 		mddev->reshape_backwards = 0;
 		module_put(pers->owner);
-		printk(KERN_WARNING "md: %s: %s would not accept array\n",
-		       mdname(mddev), clevel);
+		pr_warn("md: %s: %s would not accept array\n",
+			mdname(mddev), clevel);
 		rv = PTR_ERR(priv);
 		goto out_unlock;
 	}
@@ -3570,9 +3605,8 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
 	    pers->sync_request != NULL) {
 		/* need to add the md_redundancy_group */
 		if (sysfs_create_group(&mddev->kobj, &md_redundancy_group))
-			printk(KERN_WARNING
-			       "md: cannot register extra attributes for %s\n",
-			       mdname(mddev));
+			pr_warn("md: cannot register extra attributes for %s\n",
+				mdname(mddev));
 		mddev->sysfs_action = sysfs_get_dirent(mddev->kobj.sd, "sync_action");
 	}
 	if (oldpers->sync_request != NULL &&
@@ -3603,9 +3637,8 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
 			clear_bit(In_sync, &rdev->flags);
 		else {
 			if (sysfs_link_rdev(mddev, rdev))
-				printk(KERN_WARNING "md: cannot register rd%d"
-				       " for %s after level change\n",
-				       rdev->raid_disk, mdname(mddev));
+				pr_warn("md: cannot register rd%d for %s after level change\n",
+					rdev->raid_disk, mdname(mddev));
 		}
 	}
 
@@ -3618,7 +3651,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
 	}
 	blk_set_stacking_limits(&mddev->queue->limits);
 	pers->run(mddev);
-	set_bit(MD_CHANGE_DEVS, &mddev->flags);
+	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 	mddev_resume(mddev);
 	if (!mddev->thread)
 		md_update_sb(mddev, 1);
@@ -3813,7 +3846,7 @@ resync_start_store(struct mddev *mddev, const char *buf, size_t len)
 	if (!err) {
 		mddev->recovery_cp = n;
 		if (mddev->pers)
-			set_bit(MD_CHANGE_CLEAN, &mddev->flags);
+			set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
 	}
 	mddev_unlock(mddev);
 	return err ?: len;
@@ -3887,7 +3920,7 @@ array_state_show(struct mddev *mddev, char *page)
 			st = read_auto;
 			break;
 		case 0:
-			if (test_bit(MD_CHANGE_PENDING, &mddev->flags))
+			if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))
 				st = write_pending;
 			else if (mddev->in_sync)
 				st = clean;
@@ -3925,7 +3958,8 @@ array_state_store(struct mddev *mddev, const char *buf, size_t len)
 		spin_lock(&mddev->lock);
 		if (st == active) {
 			restart_array(mddev);
-			clear_bit(MD_CHANGE_PENDING, &mddev->flags);
+			clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
+			md_wakeup_thread(mddev->thread);
 			wake_up(&mddev->sb_wait);
 			err = 0;
 		} else /* st == clean */ {
@@ -3935,7 +3969,7 @@ array_state_store(struct mddev *mddev, const char *buf, size_t len)
 					mddev->in_sync = 1;
 					if (mddev->safemode == 1)
 						mddev->safemode = 0;
-					set_bit(MD_CHANGE_CLEAN, &mddev->flags);
+					set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
 				}
 				err = 0;
 			} else
@@ -4001,7 +4035,7 @@ array_state_store(struct mddev *mddev, const char *buf, size_t len)
 					mddev->in_sync = 1;
 					if (mddev->safemode == 1)
 						mddev->safemode = 0;
-					set_bit(MD_CHANGE_CLEAN, &mddev->flags);
+					set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
 				}
 				err = 0;
 			} else
@@ -4015,7 +4049,7 @@ array_state_store(struct mddev *mddev, const char *buf, size_t len)
 			err = restart_array(mddev);
 			if (err)
 				break;
-			clear_bit(MD_CHANGE_PENDING, &mddev->flags);
+			clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
 			wake_up(&mddev->sb_wait);
 			err = 0;
 		} else {
@@ -5071,13 +5105,13 @@ static int md_alloc(dev_t dev, char *name)
 		/* This isn't possible, but as kobject_init_and_add is marked
 		 * __must_check, we must do something with the result
 		 */
-		printk(KERN_WARNING "md: cannot register %s/md - name in use\n",
-		       disk->disk_name);
+		pr_debug("md: cannot register %s/md - name in use\n",
+			 disk->disk_name);
 		error = 0;
 	}
 	if (mddev->kobj.sd &&
 	    sysfs_create_group(&mddev->kobj, &md_bitmap_group))
-		printk(KERN_DEBUG "pointless warning\n");
+		pr_debug("pointless warning\n");
 	mutex_unlock(&mddev->open_mutex);
  abort:
 	mutex_unlock(&disks_mutex);
@@ -5179,15 +5213,15 @@ int md_run(struct mddev *mddev)
 			if (mddev->dev_sectors &&
 			    rdev->data_offset + mddev->dev_sectors
 			    > rdev->sb_start) {
-				printk("md: %s: data overlaps metadata\n",
-				       mdname(mddev));
+				pr_warn("md: %s: data overlaps metadata\n",
+					mdname(mddev));
 				return -EINVAL;
 			}
 		} else {
 			if (rdev->sb_start + rdev->sb_size/512
 			    > rdev->data_offset) {
-				printk("md: %s: metadata overlaps data\n",
-				       mdname(mddev));
+				pr_warn("md: %s: metadata overlaps data\n",
+					mdname(mddev));
 				return -EINVAL;
 			}
 		}
@@ -5202,11 +5236,11 @@ int md_run(struct mddev *mddev)
 	if (!pers || !try_module_get(pers->owner)) {
 		spin_unlock(&pers_lock);
 		if (mddev->level != LEVEL_NONE)
-			printk(KERN_WARNING "md: personality for level %d is not loaded!\n",
-			       mddev->level);
+			pr_warn("md: personality for level %d is not loaded!\n",
+				mddev->level);
 		else
-			printk(KERN_WARNING "md: personality for level %s is not loaded!\n",
-			       mddev->clevel);
+			pr_warn("md: personality for level %s is not loaded!\n",
+				mddev->clevel);
 		return -EINVAL;
 	}
 	spin_unlock(&pers_lock);
@@ -5236,21 +5270,16 @@ int md_run(struct mddev *mddev)
 				if (rdev < rdev2 &&
 				    rdev->bdev->bd_contains ==
 				    rdev2->bdev->bd_contains) {
-					printk(KERN_WARNING
-					       "%s: WARNING: %s appears to be"
-					       " on the same physical disk as"
-					       " %s.\n",
-					       mdname(mddev),
-					       bdevname(rdev->bdev,b),
-					       bdevname(rdev2->bdev,b2));
+					pr_warn("%s: WARNING: %s appears to be on the same physical disk as %s.\n",
+						mdname(mddev),
+						bdevname(rdev->bdev,b),
+						bdevname(rdev2->bdev,b2));
 					warned = 1;
 				}
 			}
 
 		if (warned)
-			printk(KERN_WARNING
-			       "True protection against single-disk"
-			       " failure might be compromised.\n");
+			pr_warn("True protection against single-disk failure might be compromised.\n");
 	}
 
 	mddev->recovery = 0;
@@ -5264,14 +5293,14 @@ int md_run(struct mddev *mddev)
 
 	err = pers->run(mddev);
 	if (err)
-		printk(KERN_ERR "md: pers->run() failed ...\n");
+		pr_warn("md: pers->run() failed ...\n");
 	else if (pers->size(mddev, 0, 0) < mddev->array_sectors) {
-		WARN_ONCE(!mddev->external_size, "%s: default size too small,"
-			  " but 'external_size' not in effect?\n", __func__);
-		printk(KERN_ERR
-		       "md: invalid array_size %llu > default size %llu\n",
-		       (unsigned long long)mddev->array_sectors / 2,
-		       (unsigned long long)pers->size(mddev, 0, 0) / 2);
+		WARN_ONCE(!mddev->external_size,
+			  "%s: default size too small, but 'external_size' not in effect?\n",
+			  __func__);
+		pr_warn("md: invalid array_size %llu > default size %llu\n",
+			(unsigned long long)mddev->array_sectors / 2,
+			(unsigned long long)pers->size(mddev, 0, 0) / 2);
 		err = -EINVAL;
 	}
 	if (err == 0 && pers->sync_request &&
@@ -5281,8 +5310,8 @@ int md_run(struct mddev *mddev)
 		bitmap = bitmap_create(mddev, -1);
 		if (IS_ERR(bitmap)) {
 			err = PTR_ERR(bitmap);
-			printk(KERN_ERR "%s: failed to create bitmap (%d)\n",
-			       mdname(mddev), err);
+			pr_warn("%s: failed to create bitmap (%d)\n",
+				mdname(mddev), err);
 		} else
 			mddev->bitmap = bitmap;
 
@@ -5318,9 +5347,8 @@ int md_run(struct mddev *mddev)
 	if (pers->sync_request) {
 		if (mddev->kobj.sd &&
 		    sysfs_create_group(&mddev->kobj, &md_redundancy_group))
-			printk(KERN_WARNING
-			       "md: cannot register extra attributes for %s\n",
-			       mdname(mddev));
+			pr_warn("md: cannot register extra attributes for %s\n",
+				mdname(mddev));
 		mddev->sysfs_action = sysfs_get_dirent_safe(mddev->kobj.sd, "sync_action");
 	} else if (mddev->ro == 2) /* auto-readonly not meaningful */
 		mddev->ro = 0;
@@ -5350,7 +5378,7 @@ int md_run(struct mddev *mddev)
 		set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
 	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
 
-	if (mddev->flags & MD_UPDATE_SB_FLAGS)
+	if (mddev->sb_flags)
 		md_update_sb(mddev, 0);
 
 	md_new_event(mddev);
@@ -5421,8 +5449,7 @@ static int restart_array(struct mddev *mddev)
 	mddev->safemode = 0;
 	mddev->ro = 0;
 	set_disk_ro(disk, 0);
-	printk(KERN_INFO "md: %s switched to read-write mode.\n",
-		mdname(mddev));
+	pr_debug("md: %s switched to read-write mode.\n", mdname(mddev));
 	/* Kick recovery or resync if necessary */
 	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
 	md_wakeup_thread(mddev->thread);
@@ -5446,6 +5473,7 @@ static void md_clean(struct mddev *mddev)
 	mddev->level = LEVEL_NONE;
 	mddev->clevel[0] = 0;
 	mddev->flags = 0;
+	mddev->sb_flags = 0;
 	mddev->ro = 0;
 	mddev->metadata_type[0] = 0;
 	mddev->chunk_sectors = 0;
@@ -5490,12 +5518,15 @@ static void __md_stop_writes(struct mddev *mddev)
 
 	del_timer_sync(&mddev->safemode_timer);
 
+	if (mddev->pers && mddev->pers->quiesce) {
+		mddev->pers->quiesce(mddev, 1);
+		mddev->pers->quiesce(mddev, 0);
+	}
 	bitmap_flush(mddev);
-	md_super_wait(mddev);
 
 	if (mddev->ro == 0 &&
 	    ((!mddev->in_sync && !mddev_is_clustered(mddev)) ||
-	     (mddev->flags & MD_UPDATE_SB_FLAGS))) {
+	     mddev->sb_flags)) {
 		/* mark array as shutdown cleanly */
 		if (!mddev_is_clustered(mddev))
 			mddev->in_sync = 1;
@@ -5516,8 +5547,8 @@ static void mddev_detach(struct mddev *mddev)
 	struct bitmap *bitmap = mddev->bitmap;
 	/* wait for behind writes to complete */
 	if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
-		printk(KERN_INFO "md:%s: behind writes in progress - waiting to stop.\n",
-		       mdname(mddev));
+		pr_debug("md:%s: behind writes in progress - waiting to stop.\n",
+			 mdname(mddev));
 		/* need to kick something here to make sure I/O goes? */
 		wait_event(bitmap->behind_wait,
 			   atomic_read(&bitmap->behind_writes) == 0);
@@ -5578,20 +5609,20 @@ static int md_set_readonly(struct mddev *mddev, struct block_device *bdev)
 		 * which will now never happen */
 		wake_up_process(mddev->sync_thread->tsk);
 
-	if (mddev->external && test_bit(MD_CHANGE_PENDING, &mddev->flags))
+	if (mddev->external && test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))
 		return -EBUSY;
 	mddev_unlock(mddev);
 	wait_event(resync_wait, !test_bit(MD_RECOVERY_RUNNING,
 					  &mddev->recovery));
 	wait_event(mddev->sb_wait,
-		   !test_bit(MD_CHANGE_PENDING, &mddev->flags));
+		   !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
 	mddev_lock_nointr(mddev);
 
 	mutex_lock(&mddev->open_mutex);
 	if ((mddev->pers && atomic_read(&mddev->openers) > !!bdev) ||
 	    mddev->sync_thread ||
 	    test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
-		printk("md: %s still in use.\n",mdname(mddev));
+		pr_warn("md: %s still in use.\n",mdname(mddev));
 		if (did_freeze) {
 			clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
 			set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
@@ -5653,7 +5684,7 @@ static int do_md_stop(struct mddev *mddev, int mode,
 	    mddev->sysfs_active ||
 	    mddev->sync_thread ||
 	    test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
-		printk("md: %s still in use.\n",mdname(mddev));
+		pr_warn("md: %s still in use.\n",mdname(mddev));
 		mutex_unlock(&mddev->open_mutex);
 		if (did_freeze) {
 			clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
@@ -5690,7 +5721,7 @@ static int do_md_stop(struct mddev *mddev, int mode,
 	 * Free resources if final stop
 	 */
 	if (mode == 0) {
-		printk(KERN_INFO "md: %s stopped.\n", mdname(mddev));
+		pr_info("md: %s stopped.\n", mdname(mddev));
 
 		bitmap_destroy(mddev);
 		if (mddev->bitmap_info.file) {
@@ -5722,17 +5753,17 @@ static void autorun_array(struct mddev *mddev)
 	if (list_empty(&mddev->disks))
 		return;
 
-	printk(KERN_INFO "md: running: ");
+	pr_info("md: running: ");
 
 	rdev_for_each(rdev, mddev) {
 		char b[BDEVNAME_SIZE];
-		printk("<%s>", bdevname(rdev->bdev,b));
+		pr_cont("<%s>", bdevname(rdev->bdev,b));
 	}
-	printk("\n");
+	pr_cont("\n");
 
 	err = do_md_run(mddev);
 	if (err) {
-		printk(KERN_WARNING "md: do_md_run() returned %d\n", err);
+		pr_warn("md: do_md_run() returned %d\n", err);
 		do_md_stop(mddev, 0, NULL);
 	}
 }
@@ -5755,7 +5786,7 @@ static void autorun_devices(int part)
 	struct mddev *mddev;
 	char b[BDEVNAME_SIZE];
 
-	printk(KERN_INFO "md: autorun ...\n");
+	pr_info("md: autorun ...\n");
 	while (!list_empty(&pending_raid_disks)) {
 		int unit;
 		dev_t dev;
@@ -5763,13 +5794,12 @@ static void autorun_devices(int part)
 		rdev0 = list_entry(pending_raid_disks.next,
 					 struct md_rdev, same_set);
 
-		printk(KERN_INFO "md: considering %s ...\n",
-			bdevname(rdev0->bdev,b));
+		pr_debug("md: considering %s ...\n", bdevname(rdev0->bdev,b));
 		INIT_LIST_HEAD(&candidates);
 		rdev_for_each_list(rdev, tmp, &pending_raid_disks)
 			if (super_90_load(rdev, rdev0, 0) >= 0) {
-				printk(KERN_INFO "md:  adding %s ...\n",
-					bdevname(rdev->bdev,b));
+				pr_debug("md:  adding %s ...\n",
+					 bdevname(rdev->bdev,b));
 				list_move(&rdev->same_set, &candidates);
 			}
 		/*
@@ -5786,8 +5816,8 @@ static void autorun_devices(int part)
 			unit = MINOR(dev);
 		}
 		if (rdev0->preferred_minor != unit) {
-			printk(KERN_INFO "md: unit number in %s is bad: %d\n",
-			       bdevname(rdev0->bdev, b), rdev0->preferred_minor);
+			pr_warn("md: unit number in %s is bad: %d\n",
+				bdevname(rdev0->bdev, b), rdev0->preferred_minor);
 			break;
 		}
 
@@ -5796,21 +5826,17 @@ static void autorun_devices(int part)
 		if (!mddev || !mddev->gendisk) {
 			if (mddev)
 				mddev_put(mddev);
-			printk(KERN_ERR
-				"md: cannot allocate memory for md drive.\n");
 			break;
 		}
 		if (mddev_lock(mddev))
-			printk(KERN_WARNING "md: %s locked, cannot run\n",
-			       mdname(mddev));
+			pr_warn("md: %s locked, cannot run\n", mdname(mddev));
 		else if (mddev->raid_disks || mddev->major_version
 			 || !list_empty(&mddev->disks)) {
-			printk(KERN_WARNING
-				"md: %s already running, cannot run %s\n",
+			pr_warn("md: %s already running, cannot run %s\n",
 				mdname(mddev), bdevname(rdev0->bdev,b));
 			mddev_unlock(mddev);
 		} else {
-			printk(KERN_INFO "md: created %s\n", mdname(mddev));
+			pr_debug("md: created %s\n", mdname(mddev));
 			mddev->persistent = 1;
 			rdev_for_each_list(rdev, tmp, &candidates) {
 				list_del_init(&rdev->same_set);
@@ -5829,7 +5855,7 @@ static void autorun_devices(int part)
 		}
 		mddev_put(mddev);
 	}
-	printk(KERN_INFO "md: ... autorun DONE.\n");
+	pr_info("md: ... autorun DONE.\n");
 }
 #endif /* !MODULE */
 
@@ -5964,6 +5990,8 @@ static int get_disk_info(struct mddev *mddev, void __user * arg)
 			info.state |= (1<<MD_DISK_JOURNAL);
 		if (test_bit(WriteMostly, &rdev->flags))
 			info.state |= (1<<MD_DISK_WRITEMOSTLY);
+		if (test_bit(FailFast, &rdev->flags))
+			info.state |= (1<<MD_DISK_FAILFAST);
 	} else {
 		info.major = info.minor = 0;
 		info.raid_disk = -1;
@@ -5985,8 +6013,8 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info)
 
 	if (mddev_is_clustered(mddev) &&
 		!(info->state & ((1 << MD_DISK_CLUSTER_ADD) | (1 << MD_DISK_CANDIDATE)))) {
-		pr_err("%s: Cannot add to clustered mddev.\n",
-			       mdname(mddev));
+		pr_warn("%s: Cannot add to clustered mddev.\n",
+			mdname(mddev));
 		return -EINVAL;
 	}
 
@@ -5998,8 +6026,7 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info)
 		/* expecting a device which has a superblock */
 		rdev = md_import_device(dev, mddev->major_version, mddev->minor_version);
 		if (IS_ERR(rdev)) {
-			printk(KERN_WARNING
-				"md: md_import_device returned %ld\n",
+			pr_warn("md: md_import_device returned %ld\n",
 				PTR_ERR(rdev));
 			return PTR_ERR(rdev);
 		}
@@ -6010,8 +6037,7 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info)
 			err = super_types[mddev->major_version]
 				.load_super(rdev, rdev0, mddev->minor_version);
 			if (err < 0) {
-				printk(KERN_WARNING
-					"md: %s has different UUID to %s\n",
+				pr_warn("md: %s has different UUID to %s\n",
 					bdevname(rdev->bdev,b),
 					bdevname(rdev0->bdev,b2));
 				export_rdev(rdev);
@@ -6032,9 +6058,8 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info)
 	if (mddev->pers) {
 		int err;
 		if (!mddev->pers->hot_add_disk) {
-			printk(KERN_WARNING
-				"%s: personality does not support diskops!\n",
-			       mdname(mddev));
+			pr_warn("%s: personality does not support diskops!\n",
+				mdname(mddev));
 			return -EINVAL;
 		}
 		if (mddev->persistent)
@@ -6043,8 +6068,7 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info)
 		else
 			rdev = md_import_device(dev, -1, -1);
 		if (IS_ERR(rdev)) {
-			printk(KERN_WARNING
-				"md: md_import_device returned %ld\n",
+			pr_warn("md: md_import_device returned %ld\n",
 				PTR_ERR(rdev));
 			return PTR_ERR(rdev);
 		}
@@ -6075,6 +6099,10 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info)
 			set_bit(WriteMostly, &rdev->flags);
 		else
 			clear_bit(WriteMostly, &rdev->flags);
+		if (info->state & (1<<MD_DISK_FAILFAST))
+			set_bit(FailFast, &rdev->flags);
+		else
+			clear_bit(FailFast, &rdev->flags);
 
 		if (info->state & (1<<MD_DISK_JOURNAL)) {
 			struct md_rdev *rdev2;
@@ -6140,8 +6168,7 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info)
 	 * for major_version==0 superblocks
 	 */
 	if (mddev->major_version != 0) {
-		printk(KERN_WARNING "%s: ADD_NEW_DISK not supported\n",
-		       mdname(mddev));
+		pr_warn("%s: ADD_NEW_DISK not supported\n", mdname(mddev));
 		return -EINVAL;
 	}
 
@@ -6149,8 +6176,7 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info)
 		int err;
 		rdev = md_import_device(dev, -1, 0);
 		if (IS_ERR(rdev)) {
-			printk(KERN_WARNING
-				"md: error, md_import_device() returned %ld\n",
+			pr_warn("md: error, md_import_device() returned %ld\n",
 				PTR_ERR(rdev));
 			return PTR_ERR(rdev);
 		}
@@ -6166,9 +6192,11 @@ static int add_new_disk(struct mddev *mddev, mdu_disk_info_t *info)
 
 		if (info->state & (1<<MD_DISK_WRITEMOSTLY))
 			set_bit(WriteMostly, &rdev->flags);
+		if (info->state & (1<<MD_DISK_FAILFAST))
+			set_bit(FailFast, &rdev->flags);
 
 		if (!mddev->persistent) {
-			printk(KERN_INFO "md: nonpersistent superblock ...\n");
+			pr_debug("md: nonpersistent superblock ...\n");
 			rdev->sb_start = i_size_read(rdev->bdev->bd_inode) / 512;
 		} else
 			rdev->sb_start = calc_dev_sboffset(rdev);
@@ -6207,13 +6235,17 @@ kick_rdev:
 		md_cluster_ops->remove_disk(mddev, rdev);
 
 	md_kick_rdev_from_array(rdev);
-	md_update_sb(mddev, 1);
+	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
+	if (mddev->thread)
+		md_wakeup_thread(mddev->thread);
+	else
+		md_update_sb(mddev, 1);
 	md_new_event(mddev);
 
 	return 0;
 busy:
-	printk(KERN_WARNING "md: cannot remove active disk %s from %s ...\n",
-		bdevname(rdev->bdev,b), mdname(mddev));
+	pr_debug("md: cannot remove active disk %s from %s ...\n",
+		 bdevname(rdev->bdev,b), mdname(mddev));
 	return -EBUSY;
 }
 
@@ -6227,22 +6259,19 @@ static int hot_add_disk(struct mddev *mddev, dev_t dev)
 		return -ENODEV;
 
 	if (mddev->major_version != 0) {
-		printk(KERN_WARNING "%s: HOT_ADD may only be used with"
-			" version-0 superblocks.\n",
+		pr_warn("%s: HOT_ADD may only be used with version-0 superblocks.\n",
 			mdname(mddev));
 		return -EINVAL;
 	}
 	if (!mddev->pers->hot_add_disk) {
-		printk(KERN_WARNING
-			"%s: personality does not support diskops!\n",
+		pr_warn("%s: personality does not support diskops!\n",
 			mdname(mddev));
 		return -EINVAL;
 	}
 
 	rdev = md_import_device(dev, -1, 0);
 	if (IS_ERR(rdev)) {
-		printk(KERN_WARNING
-			"md: error, md_import_device() returned %ld\n",
+		pr_warn("md: error, md_import_device() returned %ld\n",
 			PTR_ERR(rdev));
 		return -EINVAL;
 	}
@@ -6255,8 +6284,7 @@ static int hot_add_disk(struct mddev *mddev, dev_t dev)
 	rdev->sectors = rdev->sb_start;
 
 	if (test_bit(Faulty, &rdev->flags)) {
-		printk(KERN_WARNING
-			"md: can not hot-add faulty %s disk to %s!\n",
+		pr_warn("md: can not hot-add faulty %s disk to %s!\n",
 			bdevname(rdev->bdev,b), mdname(mddev));
 		err = -EINVAL;
 		goto abort_export;
@@ -6276,7 +6304,9 @@ static int hot_add_disk(struct mddev *mddev, dev_t dev)
 
 	rdev->raid_disk = -1;
 
-	md_update_sb(mddev, 1);
+	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
+	if (!mddev->thread)
+		md_update_sb(mddev, 1);
 	/*
 	 * Kick recovery, maybe this spare has to be added to the
 	 * array immediately.
@@ -6312,23 +6342,23 @@ static int set_bitmap_file(struct mddev *mddev, int fd)
 		f = fget(fd);
 
 		if (f == NULL) {
-			printk(KERN_ERR "%s: error: failed to get bitmap file\n",
-			       mdname(mddev));
+			pr_warn("%s: error: failed to get bitmap file\n",
+				mdname(mddev));
 			return -EBADF;
 		}
 
 		inode = f->f_mapping->host;
 		if (!S_ISREG(inode->i_mode)) {
-			printk(KERN_ERR "%s: error: bitmap file must be a regular file\n",
-			       mdname(mddev));
+			pr_warn("%s: error: bitmap file must be a regular file\n",
+				mdname(mddev));
 			err = -EBADF;
 		} else if (!(f->f_mode & FMODE_WRITE)) {
-			printk(KERN_ERR "%s: error: bitmap file must open for write\n",
-			       mdname(mddev));
+			pr_warn("%s: error: bitmap file must open for write\n",
+				mdname(mddev));
 			err = -EBADF;
 		} else if (atomic_read(&inode->i_writecount) != 1) {
-			printk(KERN_ERR "%s: error: bitmap file is already in use\n",
-			       mdname(mddev));
+			pr_warn("%s: error: bitmap file is already in use\n",
+				mdname(mddev));
 			err = -EBUSY;
 		}
 		if (err) {
@@ -6393,8 +6423,7 @@ static int set_array_info(struct mddev *mddev, mdu_array_info_t *info)
 		    info->major_version >= ARRAY_SIZE(super_types) ||
 		    super_types[info->major_version].name == NULL) {
 			/* maybe try to auto-load a module? */
-			printk(KERN_INFO
-				"md: superblock version %d not known\n",
+			pr_warn("md: superblock version %d not known\n",
 				info->major_version);
 			return -EINVAL;
 		}
@@ -6432,9 +6461,11 @@ static int set_array_info(struct mddev *mddev, mdu_array_info_t *info)
 
 	mddev->max_disks     = MD_SB_DISKS;
 
-	if (mddev->persistent)
+	if (mddev->persistent) {
 		mddev->flags         = 0;
-	set_bit(MD_CHANGE_DEVS, &mddev->flags);
+		mddev->sb_flags         = 0;
+	}
+	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 
 	mddev->bitmap_info.default_offset = MD_SB_BYTES >> 9;
 	mddev->bitmap_info.default_space = 64*2 - (MD_SB_BYTES >> 9);
@@ -6660,8 +6691,7 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
 			if (mddev->bitmap_info.nodes) {
 				/* hold PW on all the bitmap lock */
 				if (md_cluster_ops->lock_all_bitmaps(mddev) <= 0) {
-					printk("md: can't change bitmap to none since the"
-					       " array is in use by more than one node\n");
+					pr_warn("md: can't change bitmap to none since the array is in use by more than one node\n");
 					rv = -EPERM;
 					md_cluster_ops->unlock_all_bitmaps(mddev);
 					goto err;
@@ -6829,7 +6859,7 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode,
 		/* need to ensure recovery thread has run */
 		wait_event_interruptible_timeout(mddev->sb_wait,
 						 !test_bit(MD_RECOVERY_NEEDED,
-							   &mddev->flags),
+							   &mddev->recovery),
 						 msecs_to_jiffies(5000));
 	if (cmd == STOP_ARRAY || cmd == STOP_ARRAY_RO) {
 		/* Need to flush page cache, and ensure no-one else opens
@@ -6847,9 +6877,8 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode,
 	}
 	err = mddev_lock(mddev);
 	if (err) {
-		printk(KERN_INFO
-			"md: ioctl lock interrupted, reason %d, cmd %d\n",
-			err, cmd);
+		pr_debug("md: ioctl lock interrupted, reason %d, cmd %d\n",
+			 err, cmd);
 		goto out;
 	}
 
@@ -6864,30 +6893,24 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode,
 		if (mddev->pers) {
 			err = update_array_info(mddev, &info);
 			if (err) {
-				printk(KERN_WARNING "md: couldn't update"
-				       " array info. %d\n", err);
+				pr_warn("md: couldn't update array info. %d\n", err);
 				goto unlock;
 			}
 			goto unlock;
 		}
 		if (!list_empty(&mddev->disks)) {
-			printk(KERN_WARNING
-			       "md: array %s already has disks!\n",
-			       mdname(mddev));
+			pr_warn("md: array %s already has disks!\n", mdname(mddev));
 			err = -EBUSY;
 			goto unlock;
 		}
 		if (mddev->raid_disks) {
-			printk(KERN_WARNING
-			       "md: array %s already initialised!\n",
-			       mdname(mddev));
+			pr_warn("md: array %s already initialised!\n", mdname(mddev));
 			err = -EBUSY;
 			goto unlock;
 		}
 		err = set_array_info(mddev, &info);
 		if (err) {
-			printk(KERN_WARNING "md: couldn't set"
-			       " array info. %d\n", err);
+			pr_warn("md: couldn't set array info. %d\n", err);
 			goto unlock;
 		}
 		goto unlock;
@@ -6987,11 +7010,11 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode,
 			/* If a device failed while we were read-only, we
 			 * need to make sure the metadata is updated now.
 			 */
-			if (test_bit(MD_CHANGE_DEVS, &mddev->flags)) {
+			if (test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags)) {
 				mddev_unlock(mddev);
 				wait_event(mddev->sb_wait,
-					   !test_bit(MD_CHANGE_DEVS, &mddev->flags) &&
-					   !test_bit(MD_CHANGE_PENDING, &mddev->flags));
+					   !test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags) &&
+					   !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
 				mddev_lock_nointr(mddev);
 			}
 		} else {
@@ -7092,7 +7115,8 @@ static int md_open(struct block_device *bdev, fmode_t mode)
 
 	if (test_bit(MD_CLOSING, &mddev->flags)) {
 		mutex_unlock(&mddev->open_mutex);
-		return -ENODEV;
+		err = -ENODEV;
+		goto out;
 	}
 
 	err = 0;
@@ -7101,6 +7125,8 @@ static int md_open(struct block_device *bdev, fmode_t mode)
 
 	check_disk_change(bdev);
  out:
+	if (err)
+		mddev_put(mddev);
 	return err;
 }
 
@@ -7171,10 +7197,12 @@ static int md_thread(void *arg)
 		wait_event_interruptible_timeout
 			(thread->wqueue,
 			 test_bit(THREAD_WAKEUP, &thread->flags)
-			 || kthread_should_stop(),
+			 || kthread_should_stop() || kthread_should_park(),
 			 thread->timeout);
 
 		clear_bit(THREAD_WAKEUP, &thread->flags);
+		if (kthread_should_park())
+			kthread_parkme();
 		if (!kthread_should_stop())
 			thread->run(thread);
 	}
@@ -7588,8 +7616,8 @@ static const struct file_operations md_seq_fops = {
 
 int register_md_personality(struct md_personality *p)
 {
-	printk(KERN_INFO "md: %s personality registered for level %d\n",
-						p->name, p->level);
+	pr_debug("md: %s personality registered for level %d\n",
+		 p->name, p->level);
 	spin_lock(&pers_lock);
 	list_add_tail(&p->list, &pers_list);
 	spin_unlock(&pers_lock);
@@ -7599,7 +7627,7 @@ EXPORT_SYMBOL(register_md_personality);
 
 int unregister_md_personality(struct md_personality *p)
 {
-	printk(KERN_INFO "md: %s personality unregistered\n", p->name);
+	pr_debug("md: %s personality unregistered\n", p->name);
 	spin_lock(&pers_lock);
 	list_del_init(&p->list);
 	spin_unlock(&pers_lock);
@@ -7639,7 +7667,7 @@ int md_setup_cluster(struct mddev *mddev, int nodes)
 	spin_lock(&pers_lock);
 	/* ensure module won't be unloaded */
 	if (!md_cluster_ops || !try_module_get(md_cluster_mod)) {
-		pr_err("can't find md-cluster module or get it's reference.\n");
+		pr_warn("can't find md-cluster module or get it's reference.\n");
 		spin_unlock(&pers_lock);
 		return -ENOENT;
 	}
@@ -7741,8 +7769,8 @@ void md_write_start(struct mddev *mddev, struct bio *bi)
 		spin_lock(&mddev->lock);
 		if (mddev->in_sync) {
 			mddev->in_sync = 0;
-			set_bit(MD_CHANGE_CLEAN, &mddev->flags);
-			set_bit(MD_CHANGE_PENDING, &mddev->flags);
+			set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
+			set_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
 			md_wakeup_thread(mddev->thread);
 			did_change = 1;
 		}
@@ -7751,7 +7779,7 @@ void md_write_start(struct mddev *mddev, struct bio *bi)
 	if (did_change)
 		sysfs_notify_dirent_safe(mddev->sysfs_state);
 	wait_event(mddev->sb_wait,
-		   !test_bit(MD_CHANGE_PENDING, &mddev->flags));
+		   !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
 }
 EXPORT_SYMBOL(md_write_start);
 
@@ -7772,7 +7800,7 @@ EXPORT_SYMBOL(md_write_end);
  * attempting a GFP_KERNEL allocation while holding the mddev lock.
  * Must be called with mddev_lock held.
  *
- * In the ->external case MD_CHANGE_PENDING can not be cleared until mddev->lock
+ * In the ->external case MD_SB_CHANGE_PENDING can not be cleared until mddev->lock
  * is dropped, so return -EAGAIN after notifying userspace.
  */
 int md_allow_write(struct mddev *mddev)
@@ -7787,8 +7815,8 @@ int md_allow_write(struct mddev *mddev)
 	spin_lock(&mddev->lock);
 	if (mddev->in_sync) {
 		mddev->in_sync = 0;
-		set_bit(MD_CHANGE_CLEAN, &mddev->flags);
-		set_bit(MD_CHANGE_PENDING, &mddev->flags);
+		set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
+		set_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
 		if (mddev->safemode_delay &&
 		    mddev->safemode == 0)
 			mddev->safemode = 1;
@@ -7798,7 +7826,7 @@ int md_allow_write(struct mddev *mddev)
 	} else
 		spin_unlock(&mddev->lock);
 
-	if (test_bit(MD_CHANGE_PENDING, &mddev->flags))
+	if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))
 		return -EAGAIN;
 	else
 		return 0;
@@ -7914,11 +7942,9 @@ void md_do_sync(struct md_thread *thread)
 				    mddev2->curr_resync >= mddev->curr_resync) {
 					if (mddev2_minor != mddev2->md_minor) {
 						mddev2_minor = mddev2->md_minor;
-						printk(KERN_INFO "md: delaying %s of %s"
-						       " until %s has finished (they"
-						       " share one or more physical units)\n",
-						       desc, mdname(mddev),
-						       mdname(mddev2));
+						pr_info("md: delaying %s of %s until %s has finished (they share one or more physical units)\n",
+							desc, mdname(mddev),
+							mdname(mddev2));
 					}
 					mddev_put(mddev2);
 					if (signal_pending(current))
@@ -7975,12 +8001,10 @@ void md_do_sync(struct md_thread *thread)
 		}
 	}
 
-	printk(KERN_INFO "md: %s of RAID array %s\n", desc, mdname(mddev));
-	printk(KERN_INFO "md: minimum _guaranteed_  speed:"
-		" %d KB/sec/disk.\n", speed_min(mddev));
-	printk(KERN_INFO "md: using maximum available idle IO bandwidth "
-	       "(but not more than %d KB/sec) for %s.\n",
-	       speed_max(mddev), desc);
+	pr_info("md: %s of RAID array %s\n", desc, mdname(mddev));
+	pr_debug("md: minimum _guaranteed_  speed: %d KB/sec/disk.\n", speed_min(mddev));
+	pr_debug("md: using maximum available idle IO bandwidth (but not more than %d KB/sec) for %s.\n",
+		 speed_max(mddev), desc);
 
 	is_mddev_idle(mddev, 1); /* this initializes IO event counters */
 
@@ -7997,16 +8021,15 @@ void md_do_sync(struct md_thread *thread)
 	 * Tune reconstruction:
 	 */
 	window = 32*(PAGE_SIZE/512);
-	printk(KERN_INFO "md: using %dk window, over a total of %lluk.\n",
-		window/2, (unsigned long long)max_sectors/2);
+	pr_debug("md: using %dk window, over a total of %lluk.\n",
+		 window/2, (unsigned long long)max_sectors/2);
 
 	atomic_set(&mddev->recovery_active, 0);
 	last_check = 0;
 
 	if (j>2) {
-		printk(KERN_INFO
-		       "md: resuming %s of %s from checkpoint.\n",
-		       desc, mdname(mddev));
+		pr_debug("md: resuming %s of %s from checkpoint.\n",
+			 desc, mdname(mddev));
 		mddev->curr_resync = j;
 	} else
 		mddev->curr_resync = 3; /* no longer delayed */
@@ -8038,7 +8061,7 @@ void md_do_sync(struct md_thread *thread)
 			    j > mddev->recovery_cp)
 				mddev->recovery_cp = j;
 			update_time = jiffies;
-			set_bit(MD_CHANGE_CLEAN, &mddev->flags);
+			set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
 			sysfs_notify(&mddev->kobj, NULL, "sync_completed");
 		}
 
@@ -8133,9 +8156,9 @@ void md_do_sync(struct md_thread *thread)
 			}
 		}
 	}
-	printk(KERN_INFO "md: %s: %s %s.\n",mdname(mddev), desc,
-	       test_bit(MD_RECOVERY_INTR, &mddev->recovery)
-	       ? "interrupted" : "done");
+	pr_info("md: %s: %s %s.\n",mdname(mddev), desc,
+		test_bit(MD_RECOVERY_INTR, &mddev->recovery)
+		? "interrupted" : "done");
 	/*
 	 * this also signals 'finished resyncing' to md_stop
 	 */
@@ -8155,9 +8178,8 @@ void md_do_sync(struct md_thread *thread)
 		if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
 			if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
 				if (mddev->curr_resync >= mddev->recovery_cp) {
-					printk(KERN_INFO
-					       "md: checkpointing %s of %s.\n",
-					       desc, mdname(mddev));
+					pr_debug("md: checkpointing %s of %s.\n",
+						 desc, mdname(mddev));
 					if (test_bit(MD_RECOVERY_ERROR,
 						&mddev->recovery))
 						mddev->recovery_cp =
@@ -8187,8 +8209,8 @@ void md_do_sync(struct md_thread *thread)
 	/* set CHANGE_PENDING here since maybe another update is needed,
 	 * so other nodes are informed. It should be harmless for normal
 	 * raid */
-	set_mask_bits(&mddev->flags, 0,
-		      BIT(MD_CHANGE_PENDING) | BIT(MD_CHANGE_DEVS));
+	set_mask_bits(&mddev->sb_flags, 0,
+		      BIT(MD_SB_CHANGE_PENDING) | BIT(MD_SB_CHANGE_DEVS));
 
 	spin_lock(&mddev->lock);
 	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
@@ -8288,12 +8310,12 @@ static int remove_and_add_spares(struct mddev *mddev,
 			if (!test_bit(Journal, &rdev->flags))
 				spares++;
 			md_new_event(mddev);
-			set_bit(MD_CHANGE_DEVS, &mddev->flags);
+			set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 		}
 	}
 no_add:
 	if (removed)
-		set_bit(MD_CHANGE_DEVS, &mddev->flags);
+		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 	return spares;
 }
 
@@ -8305,8 +8327,8 @@ static void md_start_sync(struct work_struct *ws)
 						mddev,
 						"resync");
 	if (!mddev->sync_thread) {
-		printk(KERN_ERR "%s: could not start resync thread...\n",
-		       mdname(mddev));
+		pr_warn("%s: could not start resync thread...\n",
+			mdname(mddev));
 		/* leave the spares where they are, it shouldn't hurt */
 		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
 		clear_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
@@ -8356,8 +8378,8 @@ void md_check_recovery(struct mddev *mddev)
 
 	if (signal_pending(current)) {
 		if (mddev->pers->sync_request && !mddev->external) {
-			printk(KERN_INFO "md: %s in immediate safe mode\n",
-			       mdname(mddev));
+			pr_debug("md: %s in immediate safe mode\n",
+				 mdname(mddev));
 			mddev->safemode = 2;
 		}
 		flush_signals(current);
@@ -8366,7 +8388,7 @@ void md_check_recovery(struct mddev *mddev)
 	if (mddev->ro && !test_bit(MD_RECOVERY_NEEDED, &mddev->recovery))
 		return;
 	if ( ! (
-		(mddev->flags & MD_UPDATE_SB_FLAGS & ~ (1<<MD_CHANGE_PENDING)) ||
+		(mddev->sb_flags & ~ (1<<MD_SB_CHANGE_PENDING)) ||
 		test_bit(MD_RECOVERY_NEEDED, &mddev->recovery) ||
 		test_bit(MD_RECOVERY_DONE, &mddev->recovery) ||
 		test_bit(MD_RELOAD_SB, &mddev->flags) ||
@@ -8404,7 +8426,7 @@ void md_check_recovery(struct mddev *mddev)
 			md_reap_sync_thread(mddev);
 			clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery);
 			clear_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
-			clear_bit(MD_CHANGE_PENDING, &mddev->flags);
+			clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
 			goto unlock;
 		}
 
@@ -8432,7 +8454,7 @@ void md_check_recovery(struct mddev *mddev)
 			    mddev->recovery_cp == MaxSector) {
 				mddev->in_sync = 1;
 				did_change = 1;
-				set_bit(MD_CHANGE_CLEAN, &mddev->flags);
+				set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags);
 			}
 			if (mddev->safemode == 1)
 				mddev->safemode = 0;
@@ -8441,7 +8463,7 @@ void md_check_recovery(struct mddev *mddev)
 				sysfs_notify_dirent_safe(mddev->sysfs_state);
 		}
 
-		if (mddev->flags & MD_UPDATE_SB_FLAGS)
+		if (mddev->sb_flags)
 			md_update_sb(mddev, 0);
 
 		if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) &&
@@ -8537,7 +8559,7 @@ void md_reap_sync_thread(struct mddev *mddev)
 		if (mddev->pers->spare_active(mddev)) {
 			sysfs_notify(&mddev->kobj, NULL,
 				     "degraded");
-			set_bit(MD_CHANGE_DEVS, &mddev->flags);
+			set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 		}
 	}
 	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
@@ -8552,7 +8574,7 @@ void md_reap_sync_thread(struct mddev *mddev)
 			rdev->saved_raid_disk = -1;
 
 	md_update_sb(mddev, 1);
-	/* MD_CHANGE_PENDING should be cleared by md_update_sb, so we can
+	/* MD_SB_CHANGE_PENDING should be cleared by md_update_sb, so we can
 	 * call resync_finish here if MD_CLUSTER_RESYNC_LOCKED is set by
 	 * clustered raid */
 	if (test_and_clear_bit(MD_CLUSTER_RESYNC_LOCKED, &mddev->flags))
@@ -8614,9 +8636,12 @@ int rdev_set_badblocks(struct md_rdev *rdev, sector_t s, int sectors,
 	rv = badblocks_set(&rdev->badblocks, s, sectors, 0);
 	if (rv == 0) {
 		/* Make sure they get written out promptly */
+		if (test_bit(ExternalBbl, &rdev->flags))
+			sysfs_notify(&rdev->kobj, NULL,
+				     "unacknowledged_bad_blocks");
 		sysfs_notify_dirent_safe(rdev->sysfs_state);
-		set_mask_bits(&mddev->flags, 0,
-			      BIT(MD_CHANGE_CLEAN) | BIT(MD_CHANGE_PENDING));
+		set_mask_bits(&mddev->sb_flags, 0,
+			      BIT(MD_SB_CHANGE_CLEAN) | BIT(MD_SB_CHANGE_PENDING));
 		md_wakeup_thread(rdev->mddev->thread);
 		return 1;
 	} else
@@ -8627,12 +8652,15 @@ EXPORT_SYMBOL_GPL(rdev_set_badblocks);
 int rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, int sectors,
 			 int is_new)
 {
+	int rv;
 	if (is_new)
 		s += rdev->new_data_offset;
 	else
 		s += rdev->data_offset;
-	return badblocks_clear(&rdev->badblocks,
-				  s, sectors);
+	rv = badblocks_clear(&rdev->badblocks, s, sectors);
+	if ((rv == 0) && test_bit(ExternalBbl, &rdev->flags))
+		sysfs_notify(&rdev->kobj, NULL, "bad_blocks");
+	return rv;
 }
 EXPORT_SYMBOL_GPL(rdev_clear_badblocks);
 
@@ -8749,7 +8777,7 @@ static void check_sb_changes(struct mddev *mddev, struct md_rdev *rdev)
 				rdev2->saved_raid_disk = role;
 				ret = remove_and_add_spares(mddev, rdev2);
 				pr_info("Activated spare: %s\n",
-						bdevname(rdev2->bdev,b));
+					bdevname(rdev2->bdev,b));
 				/* wakeup mddev->thread here, so array could
 				 * perform resync with the new activated disk */
 				set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
@@ -8785,15 +8813,18 @@ static int read_rdev(struct mddev *mddev, struct md_rdev *rdev)
 	 * variable in case we err in the future
 	 */
 	rdev->sb_page = NULL;
-	alloc_disk_sb(rdev);
-	ClearPageUptodate(rdev->sb_page);
-	rdev->sb_loaded = 0;
-	err = super_types[mddev->major_version].load_super(rdev, NULL, mddev->minor_version);
-
+	err = alloc_disk_sb(rdev);
+	if (err == 0) {
+		ClearPageUptodate(rdev->sb_page);
+		rdev->sb_loaded = 0;
+		err = super_types[mddev->major_version].
+			load_super(rdev, NULL, mddev->minor_version);
+	}
 	if (err < 0) {
 		pr_warn("%s: %d Could not reload rdev(%d) err: %d. Restoring old values\n",
 				__func__, __LINE__, rdev->desc_nr, err);
-		put_page(rdev->sb_page);
+		if (rdev->sb_page)
+			put_page(rdev->sb_page);
 		rdev->sb_page = swapout;
 		rdev->sb_loaded = 1;
 		return err;
@@ -8871,9 +8902,6 @@ void md_autodetect_dev(dev_t dev)
 		mutex_lock(&detected_devices_mutex);
 		list_add_tail(&node_detected_dev->list, &all_detected_devices);
 		mutex_unlock(&detected_devices_mutex);
-	} else {
-		printk(KERN_CRIT "md: md_autodetect_dev: kzalloc failed"
-			", skipping dev(%d,%d)\n", MAJOR(dev), MINOR(dev));
 	}
 }
 
@@ -8887,7 +8915,7 @@ static void autostart_arrays(int part)
 	i_scanned = 0;
 	i_passed = 0;
 
-	printk(KERN_INFO "md: Autodetecting RAID arrays.\n");
+	pr_info("md: Autodetecting RAID arrays.\n");
 
 	mutex_lock(&detected_devices_mutex);
 	while (!list_empty(&all_detected_devices) && i_scanned < INT_MAX) {
@@ -8912,8 +8940,7 @@ static void autostart_arrays(int part)
 	}
 	mutex_unlock(&detected_devices_mutex);
 
-	printk(KERN_INFO "md: Scanned %d and added %d devices.\n",
-						i_scanned, i_passed);
+	pr_debug("md: Scanned %d and added %d devices.\n", i_scanned, i_passed);
 
 	autorun_devices(part);
 }
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 2b2041773e79..e38936d05df1 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -30,6 +30,16 @@
 #define MaxSector (~(sector_t)0)
 
 /*
+ * These flags should really be called "NO_RETRY" rather than
+ * "FAILFAST" because they don't make any promise about time lapse,
+ * only about the number of retries, which will be zero.
+ * REQ_FAILFAST_DRIVER is not included because
+ * Commit: 4a27446f3e39 ("[SCSI] modify scsi to handle new fail fast flags.")
+ * seems to suggest that the errors it avoids retrying should usually
+ * be retried.
+ */
+#define	MD_FAILFAST	(REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT)
+/*
  * MD's 'extended' device
  */
 struct md_rdev {
@@ -168,6 +178,19 @@ enum flag_bits {
 				 * so it is safe to remove without
 				 * another synchronize_rcu() call.
 				 */
+	ExternalBbl,            /* External metadata provides bad
+				 * block management for a disk
+				 */
+	FailFast,		/* Minimal retries should be attempted on
+				 * this device, so use REQ_FAILFAST_DEV.
+				 * Also don't try to repair failed reads.
+				 * It is expects that no bad block log
+				 * is present.
+				 */
+	LastDev,		/* Seems to be the last working dev as
+				 * it didn't fail, so don't use FailFast
+				 * any more for metadata
+				 */
 };
 
 static inline int is_badblock(struct md_rdev *rdev, sector_t s, int sectors,
@@ -189,6 +212,31 @@ extern int rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, int sectors,
 				int is_new);
 struct md_cluster_info;
 
+enum mddev_flags {
+	MD_ARRAY_FIRST_USE,	/* First use of array, needs initialization */
+	MD_CLOSING,		/* If set, we are closing the array, do not open
+				 * it then */
+	MD_JOURNAL_CLEAN,	/* A raid with journal is already clean */
+	MD_HAS_JOURNAL,		/* The raid array has journal feature set */
+	MD_RELOAD_SB,		/* Reload the superblock because another node
+				 * updated it.
+				 */
+	MD_CLUSTER_RESYNC_LOCKED, /* cluster raid only, which means node
+				   * already took resync lock, need to
+				   * release the lock */
+	MD_FAILFAST_SUPPORTED,	/* Using MD_FAILFAST on metadata writes is
+				 * supported as calls to md_error() will
+				 * never cause the array to become failed.
+				 */
+};
+
+enum mddev_sb_flags {
+	MD_SB_CHANGE_DEVS,		/* Some device status has changed */
+	MD_SB_CHANGE_CLEAN,	/* transition to or from 'clean' */
+	MD_SB_CHANGE_PENDING,	/* switch from 'clean' to 'active' in progress */
+	MD_SB_NEED_REWRITE,	/* metadata write needs to be repeated */
+};
+
 struct mddev {
 	void				*private;
 	struct md_personality		*pers;
@@ -196,21 +244,7 @@ struct mddev {
 	int				md_minor;
 	struct list_head		disks;
 	unsigned long			flags;
-#define MD_CHANGE_DEVS	0	/* Some device status has changed */
-#define MD_CHANGE_CLEAN 1	/* transition to or from 'clean' */
-#define MD_CHANGE_PENDING 2	/* switch from 'clean' to 'active' in progress */
-#define MD_UPDATE_SB_FLAGS (1 | 2 | 4)	/* If these are set, md_update_sb needed */
-#define MD_ARRAY_FIRST_USE 3    /* First use of array, needs initialization */
-#define MD_CLOSING	4	/* If set, we are closing the array, do not open
-				 * it then */
-#define MD_JOURNAL_CLEAN 5	/* A raid with journal is already clean */
-#define MD_HAS_JOURNAL	6	/* The raid array has journal feature set */
-#define MD_RELOAD_SB	7	/* Reload the superblock because another node
-				 * updated it.
-				 */
-#define MD_CLUSTER_RESYNC_LOCKED 8 /* cluster raid only, which means node
-				    * already took resync lock, need to
-				    * release the lock */
+	unsigned long			sb_flags;
 
 	int				suspended;
 	atomic_t			active_io;
@@ -304,31 +338,6 @@ struct mddev {
 	int				parallel_resync;
 
 	int				ok_start_degraded;
-	/* recovery/resync flags
-	 * NEEDED:   we might need to start a resync/recover
-	 * RUNNING:  a thread is running, or about to be started
-	 * SYNC:     actually doing a resync, not a recovery
-	 * RECOVER:  doing recovery, or need to try it.
-	 * INTR:     resync needs to be aborted for some reason
-	 * DONE:     thread is done and is waiting to be reaped
-	 * REQUEST:  user-space has requested a sync (used with SYNC)
-	 * CHECK:    user-space request for check-only, no repair
-	 * RESHAPE:  A reshape is happening
-	 * ERROR:    sync-action interrupted because io-error
-	 *
-	 * If neither SYNC or RESHAPE are set, then it is a recovery.
-	 */
-#define	MD_RECOVERY_RUNNING	0
-#define	MD_RECOVERY_SYNC	1
-#define	MD_RECOVERY_RECOVER	2
-#define	MD_RECOVERY_INTR	3
-#define	MD_RECOVERY_DONE	4
-#define	MD_RECOVERY_NEEDED	5
-#define	MD_RECOVERY_REQUESTED	6
-#define	MD_RECOVERY_CHECK	7
-#define MD_RECOVERY_RESHAPE	8
-#define	MD_RECOVERY_FROZEN	9
-#define	MD_RECOVERY_ERROR	10
 
 	unsigned long			recovery;
 	/* If a RAID personality determines that recovery (of a particular
@@ -442,6 +451,23 @@ struct mddev {
 	unsigned int			good_device_nr;	/* good device num within cluster raid */
 };
 
+enum recovery_flags {
+	/*
+	 * If neither SYNC or RESHAPE are set, then it is a recovery.
+	 */
+	MD_RECOVERY_RUNNING,	/* a thread is running, or about to be started */
+	MD_RECOVERY_SYNC,	/* actually doing a resync, not a recovery */
+	MD_RECOVERY_RECOVER,	/* doing recovery, or need to try it. */
+	MD_RECOVERY_INTR,	/* resync needs to be aborted for some reason */
+	MD_RECOVERY_DONE,	/* thread is done and is waiting to be reaped */
+	MD_RECOVERY_NEEDED,	/* we might need to start a resync/recover */
+	MD_RECOVERY_REQUESTED,	/* user-space has requested a sync (used with SYNC) */
+	MD_RECOVERY_CHECK,	/* user-space request for check-only, no repair */
+	MD_RECOVERY_RESHAPE,	/* A reshape is happening */
+	MD_RECOVERY_FROZEN,	/* User request to abort, and not restart, any action */
+	MD_RECOVERY_ERROR,	/* sync-action interrupted because io-error */
+};
+
 static inline int __must_check mddev_lock(struct mddev *mddev)
 {
 	return mutex_lock_interruptible(&mddev->reconfig_mutex);
@@ -623,7 +649,7 @@ extern int mddev_congested(struct mddev *mddev, int bits);
 extern void md_flush_request(struct mddev *mddev, struct bio *bio);
 extern void md_super_write(struct mddev *mddev, struct md_rdev *rdev,
 			   sector_t sector, int size, struct page *page);
-extern void md_super_wait(struct mddev *mddev);
+extern int md_super_wait(struct mddev *mddev);
 extern int sync_page_io(struct md_rdev *rdev, sector_t sector, int size,
 			struct page *page, int op, int op_flags,
 			bool metadata_op);
diff --git a/drivers/md/multipath.c b/drivers/md/multipath.c
index 673efbd6fc47..aa8c4e5c1ee2 100644
--- a/drivers/md/multipath.c
+++ b/drivers/md/multipath.c
@@ -52,7 +52,7 @@ static int multipath_map (struct mpconf *conf)
 	}
 	rcu_read_unlock();
 
-	printk(KERN_ERR "multipath_map(): no more operational IO paths?\n");
+	pr_crit_ratelimited("multipath_map(): no more operational IO paths?\n");
 	return (-1);
 }
 
@@ -97,9 +97,9 @@ static void multipath_end_request(struct bio *bio)
 		 */
 		char b[BDEVNAME_SIZE];
 		md_error (mp_bh->mddev, rdev);
-		printk(KERN_ERR "multipath: %s: rescheduling sector %llu\n",
-		       bdevname(rdev->bdev,b),
-		       (unsigned long long)bio->bi_iter.bi_sector);
+		pr_info("multipath: %s: rescheduling sector %llu\n",
+			bdevname(rdev->bdev,b),
+			(unsigned long long)bio->bi_iter.bi_sector);
 		multipath_reschedule_retry(mp_bh);
 	} else
 		multipath_end_bh_io(mp_bh, bio->bi_error);
@@ -130,7 +130,7 @@ static void multipath_make_request(struct mddev *mddev, struct bio * bio)
 	}
 	multipath = conf->multipaths + mp_bh->path;
 
-	bio_init(&mp_bh->bio);
+	bio_init(&mp_bh->bio, NULL, 0);
 	__bio_clone_fast(&mp_bh->bio, bio);
 
 	mp_bh->bio.bi_iter.bi_sector += multipath->rdev->data_offset;
@@ -194,8 +194,7 @@ static void multipath_error (struct mddev *mddev, struct md_rdev *rdev)
 		 * first check if this is a queued request for a device
 		 * which has just failed.
 		 */
-		printk(KERN_ALERT
-		       "multipath: only one IO path left and IO error.\n");
+		pr_warn("multipath: only one IO path left and IO error.\n");
 		/* leave it active... it's all we have */
 		return;
 	}
@@ -209,11 +208,9 @@ static void multipath_error (struct mddev *mddev, struct md_rdev *rdev)
 		spin_unlock_irqrestore(&conf->device_lock, flags);
 	}
 	set_bit(Faulty, &rdev->flags);
-	set_bit(MD_CHANGE_DEVS, &mddev->flags);
-	printk(KERN_ALERT "multipath: IO failure on %s,"
-	       " disabling IO path.\n"
-	       "multipath: Operation continuing"
-	       " on %d IO paths.\n",
+	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
+	pr_err("multipath: IO failure on %s, disabling IO path.\n"
+	       "multipath: Operation continuing on %d IO paths.\n",
 	       bdevname(rdev->bdev, b),
 	       conf->raid_disks - mddev->degraded);
 }
@@ -223,21 +220,21 @@ static void print_multipath_conf (struct mpconf *conf)
 	int i;
 	struct multipath_info *tmp;
 
-	printk("MULTIPATH conf printout:\n");
+	pr_debug("MULTIPATH conf printout:\n");
 	if (!conf) {
-		printk("(conf==NULL)\n");
+		pr_debug("(conf==NULL)\n");
 		return;
 	}
-	printk(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
-			 conf->raid_disks);
+	pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
+		 conf->raid_disks);
 
 	for (i = 0; i < conf->raid_disks; i++) {
 		char b[BDEVNAME_SIZE];
 		tmp = conf->multipaths + i;
 		if (tmp->rdev)
-			printk(" disk%d, o:%d, dev:%s\n",
-				i,!test_bit(Faulty, &tmp->rdev->flags),
-			       bdevname(tmp->rdev->bdev,b));
+			pr_debug(" disk%d, o:%d, dev:%s\n",
+				 i,!test_bit(Faulty, &tmp->rdev->flags),
+				 bdevname(tmp->rdev->bdev,b));
 	}
 }
 
@@ -292,8 +289,7 @@ static int multipath_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
 	if (rdev == p->rdev) {
 		if (test_bit(In_sync, &rdev->flags) ||
 		    atomic_read(&rdev->nr_pending)) {
-			printk(KERN_ERR "hot-remove-disk, slot %d is identified"
-			       " but is still operational!\n", number);
+			pr_warn("hot-remove-disk, slot %d is identified but is still operational!\n", number);
 			err = -EBUSY;
 			goto abort;
 		}
@@ -346,16 +342,14 @@ static void multipathd(struct md_thread *thread)
 		bio->bi_iter.bi_sector = mp_bh->master_bio->bi_iter.bi_sector;
 
 		if ((mp_bh->path = multipath_map (conf))<0) {
-			printk(KERN_ALERT "multipath: %s: unrecoverable IO read"
-				" error for block %llu\n",
-				bdevname(bio->bi_bdev,b),
-				(unsigned long long)bio->bi_iter.bi_sector);
+			pr_err("multipath: %s: unrecoverable IO read error for block %llu\n",
+			       bdevname(bio->bi_bdev,b),
+			       (unsigned long long)bio->bi_iter.bi_sector);
 			multipath_end_bh_io(mp_bh, -EIO);
 		} else {
-			printk(KERN_ERR "multipath: %s: redirecting sector %llu"
-				" to another IO path\n",
-				bdevname(bio->bi_bdev,b),
-				(unsigned long long)bio->bi_iter.bi_sector);
+			pr_err("multipath: %s: redirecting sector %llu to another IO path\n",
+			       bdevname(bio->bi_bdev,b),
+			       (unsigned long long)bio->bi_iter.bi_sector);
 			*bio = *(mp_bh->master_bio);
 			bio->bi_iter.bi_sector +=
 				conf->multipaths[mp_bh->path].rdev->data_offset;
@@ -389,8 +383,8 @@ static int multipath_run (struct mddev *mddev)
 		return -EINVAL;
 
 	if (mddev->level != LEVEL_MULTIPATH) {
-		printk("multipath: %s: raid level not set to multipath IO (%d)\n",
-		       mdname(mddev), mddev->level);
+		pr_warn("multipath: %s: raid level not set to multipath IO (%d)\n",
+			mdname(mddev), mddev->level);
 		goto out;
 	}
 	/*
@@ -401,21 +395,13 @@ static int multipath_run (struct mddev *mddev)
 
 	conf = kzalloc(sizeof(struct mpconf), GFP_KERNEL);
 	mddev->private = conf;
-	if (!conf) {
-		printk(KERN_ERR
-			"multipath: couldn't allocate memory for %s\n",
-			mdname(mddev));
+	if (!conf)
 		goto out;
-	}
 
 	conf->multipaths = kzalloc(sizeof(struct multipath_info)*mddev->raid_disks,
 				   GFP_KERNEL);
-	if (!conf->multipaths) {
-		printk(KERN_ERR
-			"multipath: couldn't allocate memory for %s\n",
-			mdname(mddev));
+	if (!conf->multipaths)
 		goto out_free_conf;
-	}
 
 	working_disks = 0;
 	rdev_for_each(rdev, mddev) {
@@ -439,7 +425,7 @@ static int multipath_run (struct mddev *mddev)
 	INIT_LIST_HEAD(&conf->retry_list);
 
 	if (!working_disks) {
-		printk(KERN_ERR "multipath: no operational IO paths for %s\n",
+		pr_warn("multipath: no operational IO paths for %s\n",
 			mdname(mddev));
 		goto out_free_conf;
 	}
@@ -447,27 +433,17 @@ static int multipath_run (struct mddev *mddev)
 
 	conf->pool = mempool_create_kmalloc_pool(NR_RESERVED_BUFS,
 						 sizeof(struct multipath_bh));
-	if (conf->pool == NULL) {
-		printk(KERN_ERR
-			"multipath: couldn't allocate memory for %s\n",
-			mdname(mddev));
+	if (conf->pool == NULL)
 		goto out_free_conf;
-	}
 
-	{
-		mddev->thread = md_register_thread(multipathd, mddev,
-						   "multipath");
-		if (!mddev->thread) {
-			printk(KERN_ERR "multipath: couldn't allocate thread"
-				" for %s\n", mdname(mddev));
-			goto out_free_conf;
-		}
-	}
+	mddev->thread = md_register_thread(multipathd, mddev,
+					   "multipath");
+	if (!mddev->thread)
+		goto out_free_conf;
 
-	printk(KERN_INFO
-		"multipath: array %s active with %d out of %d IO paths\n",
+	pr_info("multipath: array %s active with %d out of %d IO paths\n",
 		mdname(mddev), conf->raid_disks - mddev->degraded,
-	       mddev->raid_disks);
+		mddev->raid_disks);
 	/*
 	 * Ok, everything is just fine now
 	 */
diff --git a/drivers/md/persistent-data/dm-array.c b/drivers/md/persistent-data/dm-array.c
index e83047cbb2da..7938cd21fa4c 100644
--- a/drivers/md/persistent-data/dm-array.c
+++ b/drivers/md/persistent-data/dm-array.c
@@ -700,13 +700,11 @@ static int populate_ablock_with_values(struct dm_array_info *info, struct array_
 {
 	int r;
 	unsigned i;
-	uint32_t nr_entries;
 	struct dm_btree_value_type *vt = &info->value_type;
 
 	BUG_ON(le32_to_cpu(ab->nr_entries));
 	BUG_ON(new_nr > le32_to_cpu(ab->max_entries));
 
-	nr_entries = le32_to_cpu(ab->nr_entries);
 	for (i = 0; i < new_nr; i++) {
 		r = fn(base + i, element_at(info, ab, i), context);
 		if (r)
diff --git a/drivers/md/persistent-data/dm-block-manager.c b/drivers/md/persistent-data/dm-block-manager.c
index 1e33dd51c21f..a6dde7cab458 100644
--- a/drivers/md/persistent-data/dm-block-manager.c
+++ b/drivers/md/persistent-data/dm-block-manager.c
@@ -18,6 +18,8 @@
 
 /*----------------------------------------------------------------*/
 
+#ifdef CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING
+
 /*
  * This is a read/write semaphore with a couple of differences.
  *
@@ -302,6 +304,18 @@ static void report_recursive_bug(dm_block_t b, int r)
 		      (unsigned long long) b);
 }
 
+#else  /* !CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING */
+
+#define bl_init(x) do { } while (0)
+#define bl_down_read(x) 0
+#define bl_down_read_nonblock(x) 0
+#define bl_up_read(x) do { } while (0)
+#define bl_down_write(x) 0
+#define bl_up_write(x) do { } while (0)
+#define report_recursive_bug(x, y) do { } while (0)
+
+#endif /* CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING */
+
 /*----------------------------------------------------------------*/
 
 /*
@@ -330,8 +344,11 @@ EXPORT_SYMBOL_GPL(dm_block_data);
 
 struct buffer_aux {
 	struct dm_block_validator *validator;
-	struct block_lock lock;
 	int write_locked;
+
+#ifdef CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING
+	struct block_lock lock;
+#endif
 };
 
 static void dm_block_manager_alloc_callback(struct dm_buffer *buf)
diff --git a/drivers/md/persistent-data/dm-space-map-common.c b/drivers/md/persistent-data/dm-space-map-common.c
index 306d2e4502c4..4c28608a0c94 100644
--- a/drivers/md/persistent-data/dm-space-map-common.c
+++ b/drivers/md/persistent-data/dm-space-map-common.c
@@ -464,7 +464,8 @@ static int sm_ll_mutate(struct ll_disk *ll, dm_block_t b,
 		ll->nr_allocated--;
 		le32_add_cpu(&ie_disk.nr_free, 1);
 		ie_disk.none_free_before = cpu_to_le32(min(le32_to_cpu(ie_disk.none_free_before), bit));
-	}
+	} else
+		*ev = SM_NONE;
 
 	return ll->save_ie(ll, index, &ie_disk);
 }
@@ -547,7 +548,6 @@ static int metadata_ll_init_index(struct ll_disk *ll)
 	if (r < 0)
 		return r;
 
-	memcpy(dm_block_data(b), &ll->mi_le, sizeof(ll->mi_le));
 	ll->bitmap_root = dm_block_location(b);
 
 	dm_tm_unlock(ll->tm, b);
diff --git a/drivers/md/persistent-data/dm-space-map-metadata.c b/drivers/md/persistent-data/dm-space-map-metadata.c
index 7e44005595c1..20557e2c60c6 100644
--- a/drivers/md/persistent-data/dm-space-map-metadata.c
+++ b/drivers/md/persistent-data/dm-space-map-metadata.c
@@ -775,17 +775,15 @@ int dm_sm_metadata_create(struct dm_space_map *sm,
 	memcpy(&smm->sm, &bootstrap_ops, sizeof(smm->sm));
 
 	r = sm_ll_new_metadata(&smm->ll, tm);
+	if (!r) {
+		if (nr_blocks > DM_SM_METADATA_MAX_BLOCKS)
+			nr_blocks = DM_SM_METADATA_MAX_BLOCKS;
+		r = sm_ll_extend(&smm->ll, nr_blocks);
+	}
+	memcpy(&smm->sm, &ops, sizeof(smm->sm));
 	if (r)
 		return r;
 
-	if (nr_blocks > DM_SM_METADATA_MAX_BLOCKS)
-		nr_blocks = DM_SM_METADATA_MAX_BLOCKS;
-	r = sm_ll_extend(&smm->ll, nr_blocks);
-	if (r)
-		return r;
-
-	memcpy(&smm->sm, &ops, sizeof(smm->sm));
-
 	/*
 	 * Now we need to update the newly created data structures with the
 	 * allocated blocks that they were built from.
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
index 258986a2699d..a162fedeb51a 100644
--- a/drivers/md/raid0.c
+++ b/drivers/md/raid0.c
@@ -21,6 +21,7 @@
 #include <linux/seq_file.h>
 #include <linux/module.h>
 #include <linux/slab.h>
+#include <trace/events/block.h>
 #include "md.h"
 #include "raid0.h"
 #include "raid5.h"
@@ -51,20 +52,21 @@ static void dump_zones(struct mddev *mddev)
 	char b[BDEVNAME_SIZE];
 	struct r0conf *conf = mddev->private;
 	int raid_disks = conf->strip_zone[0].nb_dev;
-	printk(KERN_INFO "md: RAID0 configuration for %s - %d zone%s\n",
-	       mdname(mddev),
-	       conf->nr_strip_zones, conf->nr_strip_zones==1?"":"s");
+	pr_debug("md: RAID0 configuration for %s - %d zone%s\n",
+		 mdname(mddev),
+		 conf->nr_strip_zones, conf->nr_strip_zones==1?"":"s");
 	for (j = 0; j < conf->nr_strip_zones; j++) {
-		printk(KERN_INFO "md: zone%d=[", j);
+		char line[200];
+		int len = 0;
+
 		for (k = 0; k < conf->strip_zone[j].nb_dev; k++)
-			printk(KERN_CONT "%s%s", k?"/":"",
-			bdevname(conf->devlist[j*raid_disks
-						+ k]->bdev, b));
-		printk(KERN_CONT "]\n");
+			len += snprintf(line+len, 200-len, "%s%s", k?"/":"",
+					bdevname(conf->devlist[j*raid_disks
+							       + k]->bdev, b));
+		pr_debug("md: zone%d=[%s]\n", j, line);
 
 		zone_size  = conf->strip_zone[j].zone_end - zone_start;
-		printk(KERN_INFO "      zone-offset=%10lluKB, "
-				"device-offset=%10lluKB, size=%10lluKB\n",
+		pr_debug("      zone-offset=%10lluKB, device-offset=%10lluKB, size=%10lluKB\n",
 			(unsigned long long)zone_start>>1,
 			(unsigned long long)conf->strip_zone[j].dev_start>>1,
 			(unsigned long long)zone_size>>1);
@@ -142,9 +144,9 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf)
 	 * chunk size is a multiple of that sector size
 	 */
 	if ((mddev->chunk_sectors << 9) % blksize) {
-		printk(KERN_ERR "md/raid0:%s: chunk_size of %d not multiple of block size %d\n",
-		       mdname(mddev),
-		       mddev->chunk_sectors << 9, blksize);
+		pr_warn("md/raid0:%s: chunk_size of %d not multiple of block size %d\n",
+			mdname(mddev),
+			mddev->chunk_sectors << 9, blksize);
 		err = -EINVAL;
 		goto abort;
 	}
@@ -186,19 +188,18 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf)
 		}
 
 		if (j < 0) {
-			printk(KERN_ERR
-			       "md/raid0:%s: remove inactive devices before converting to RAID0\n",
-			       mdname(mddev));
+			pr_warn("md/raid0:%s: remove inactive devices before converting to RAID0\n",
+				mdname(mddev));
 			goto abort;
 		}
 		if (j >= mddev->raid_disks) {
-			printk(KERN_ERR "md/raid0:%s: bad disk number %d - "
-			       "aborting!\n", mdname(mddev), j);
+			pr_warn("md/raid0:%s: bad disk number %d - aborting!\n",
+				mdname(mddev), j);
 			goto abort;
 		}
 		if (dev[j]) {
-			printk(KERN_ERR "md/raid0:%s: multiple devices for %d - "
-			       "aborting!\n", mdname(mddev), j);
+			pr_warn("md/raid0:%s: multiple devices for %d - aborting!\n",
+				mdname(mddev), j);
 			goto abort;
 		}
 		dev[j] = rdev1;
@@ -208,8 +209,8 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf)
 		cnt++;
 	}
 	if (cnt != mddev->raid_disks) {
-		printk(KERN_ERR "md/raid0:%s: too few disks (%d of %d) - "
-		       "aborting!\n", mdname(mddev), cnt, mddev->raid_disks);
+		pr_warn("md/raid0:%s: too few disks (%d of %d) - aborting!\n",
+			mdname(mddev), cnt, mddev->raid_disks);
 		goto abort;
 	}
 	zone->nb_dev = cnt;
@@ -357,8 +358,7 @@ static int raid0_run(struct mddev *mddev)
 	int ret;
 
 	if (mddev->chunk_sectors == 0) {
-		printk(KERN_ERR "md/raid0:%s: chunk size must be set.\n",
-		       mdname(mddev));
+		pr_warn("md/raid0:%s: chunk size must be set.\n", mdname(mddev));
 		return -EINVAL;
 	}
 	if (md_check_no_bitmap(mddev))
@@ -399,9 +399,9 @@ static int raid0_run(struct mddev *mddev)
 	/* calculate array device size */
 	md_set_array_sectors(mddev, raid0_size(mddev, 0, 0));
 
-	printk(KERN_INFO "md/raid0:%s: md_size is %llu sectors.\n",
-	       mdname(mddev),
-	       (unsigned long long)mddev->array_sectors);
+	pr_debug("md/raid0:%s: md_size is %llu sectors.\n",
+		 mdname(mddev),
+		 (unsigned long long)mddev->array_sectors);
 
 	if (mddev->queue) {
 		/* calculate the max read-ahead size.
@@ -464,7 +464,8 @@ static void raid0_make_request(struct mddev *mddev, struct bio *bio)
 	}
 
 	do {
-		sector_t sector = bio->bi_iter.bi_sector;
+		sector_t bio_sector = bio->bi_iter.bi_sector;
+		sector_t sector = bio_sector;
 		unsigned chunk_sects = mddev->chunk_sectors;
 
 		unsigned sectors = chunk_sects -
@@ -473,7 +474,7 @@ static void raid0_make_request(struct mddev *mddev, struct bio *bio)
 			 : sector_div(sector, chunk_sects));
 
 		/* Restore due to sector_div */
-		sector = bio->bi_iter.bi_sector;
+		sector = bio_sector;
 
 		if (sectors < bio_sectors(bio)) {
 			split = bio_split(bio, sectors, GFP_NOIO, fs_bio_set);
@@ -492,8 +493,13 @@ static void raid0_make_request(struct mddev *mddev, struct bio *bio)
 			 !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) {
 			/* Just ignore it */
 			bio_endio(split);
-		} else
+		} else {
+			if (mddev->gendisk)
+				trace_block_bio_remap(bdev_get_queue(split->bi_bdev),
+						      split, disk_devt(mddev->gendisk),
+						      bio_sector);
 			generic_make_request(split);
+		}
 	} while (split != bio);
 }
 
@@ -509,17 +515,17 @@ static void *raid0_takeover_raid45(struct mddev *mddev)
 	struct r0conf *priv_conf;
 
 	if (mddev->degraded != 1) {
-		printk(KERN_ERR "md/raid0:%s: raid5 must be degraded! Degraded disks: %d\n",
-		       mdname(mddev),
-		       mddev->degraded);
+		pr_warn("md/raid0:%s: raid5 must be degraded! Degraded disks: %d\n",
+			mdname(mddev),
+			mddev->degraded);
 		return ERR_PTR(-EINVAL);
 	}
 
 	rdev_for_each(rdev, mddev) {
 		/* check slot number for a disk */
 		if (rdev->raid_disk == mddev->raid_disks-1) {
-			printk(KERN_ERR "md/raid0:%s: raid5 must have missing parity disk!\n",
-			       mdname(mddev));
+			pr_warn("md/raid0:%s: raid5 must have missing parity disk!\n",
+				mdname(mddev));
 			return ERR_PTR(-EINVAL);
 		}
 		rdev->sectors = mddev->dev_sectors;
@@ -533,8 +539,11 @@ static void *raid0_takeover_raid45(struct mddev *mddev)
 	mddev->delta_disks = -1;
 	/* make sure it will be not marked as dirty */
 	mddev->recovery_cp = MaxSector;
+	clear_bit(MD_HAS_JOURNAL, &mddev->flags);
+	clear_bit(MD_JOURNAL_CLEAN, &mddev->flags);
 
 	create_strip_zones(mddev, &priv_conf);
+
 	return priv_conf;
 }
 
@@ -549,19 +558,19 @@ static void *raid0_takeover_raid10(struct mddev *mddev)
 	 *  - all mirrors must be already degraded
 	 */
 	if (mddev->layout != ((1 << 8) + 2)) {
-		printk(KERN_ERR "md/raid0:%s:: Raid0 cannot takeover layout: 0x%x\n",
-		       mdname(mddev),
-		       mddev->layout);
+		pr_warn("md/raid0:%s:: Raid0 cannot takeover layout: 0x%x\n",
+			mdname(mddev),
+			mddev->layout);
 		return ERR_PTR(-EINVAL);
 	}
 	if (mddev->raid_disks & 1) {
-		printk(KERN_ERR "md/raid0:%s: Raid0 cannot takeover Raid10 with odd disk number.\n",
-		       mdname(mddev));
+		pr_warn("md/raid0:%s: Raid0 cannot takeover Raid10 with odd disk number.\n",
+			mdname(mddev));
 		return ERR_PTR(-EINVAL);
 	}
 	if (mddev->degraded != (mddev->raid_disks>>1)) {
-		printk(KERN_ERR "md/raid0:%s: All mirrors must be already degraded!\n",
-		       mdname(mddev));
+		pr_warn("md/raid0:%s: All mirrors must be already degraded!\n",
+			mdname(mddev));
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -574,6 +583,7 @@ static void *raid0_takeover_raid10(struct mddev *mddev)
 	mddev->degraded = 0;
 	/* make sure it will be not marked as dirty */
 	mddev->recovery_cp = MaxSector;
+	clear_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
 
 	create_strip_zones(mddev, &priv_conf);
 	return priv_conf;
@@ -588,7 +598,7 @@ static void *raid0_takeover_raid1(struct mddev *mddev)
 	 *  - (N - 1) mirror drives must be already faulty
 	 */
 	if ((mddev->raid_disks - 1) != mddev->degraded) {
-		printk(KERN_ERR "md/raid0:%s: (N - 1) mirrors drives must be already faulty!\n",
+		pr_err("md/raid0:%s: (N - 1) mirrors drives must be already faulty!\n",
 		       mdname(mddev));
 		return ERR_PTR(-EINVAL);
 	}
@@ -616,6 +626,7 @@ static void *raid0_takeover_raid1(struct mddev *mddev)
 	mddev->raid_disks = 1;
 	/* make sure it will be not marked as dirty */
 	mddev->recovery_cp = MaxSector;
+	clear_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
 
 	create_strip_zones(mddev, &priv_conf);
 	return priv_conf;
@@ -631,8 +642,8 @@ static void *raid0_takeover(struct mddev *mddev)
 	 */
 
 	if (mddev->bitmap) {
-		printk(KERN_ERR "md/raid0: %s: cannot takeover array with bitmap\n",
-		       mdname(mddev));
+		pr_warn("md/raid0: %s: cannot takeover array with bitmap\n",
+			mdname(mddev));
 		return ERR_PTR(-EBUSY);
 	}
 	if (mddev->level == 4)
@@ -642,8 +653,8 @@ static void *raid0_takeover(struct mddev *mddev)
 		if (mddev->layout == ALGORITHM_PARITY_N)
 			return raid0_takeover_raid45(mddev);
 
-		printk(KERN_ERR "md/raid0:%s: Raid can only takeover Raid5 with layout: %d\n",
-		       mdname(mddev), ALGORITHM_PARITY_N);
+		pr_warn("md/raid0:%s: Raid can only takeover Raid5 with layout: %d\n",
+			mdname(mddev), ALGORITHM_PARITY_N);
 	}
 
 	if (mddev->level == 10)
@@ -652,7 +663,7 @@ static void *raid0_takeover(struct mddev *mddev)
 	if (mddev->level == 1)
 		return raid0_takeover_raid1(mddev);
 
-	printk(KERN_ERR "Takeover from raid%i to raid0 not supported\n",
+	pr_warn("Takeover from raid%i to raid0 not supported\n",
 		mddev->level);
 
 	return ERR_PTR(-EINVAL);
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 29e2df5cd77b..a1f3fbed9100 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -37,6 +37,7 @@
 #include <linux/module.h>
 #include <linux/seq_file.h>
 #include <linux/ratelimit.h>
+#include <trace/events/block.h>
 #include "md.h"
 #include "raid1.h"
 #include "bitmap.h"
@@ -70,6 +71,9 @@ static void allow_barrier(struct r1conf *conf, sector_t start_next_window,
 			  sector_t bi_sector);
 static void lower_barrier(struct r1conf *conf);
 
+#define raid1_log(md, fmt, args...)				\
+	do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid1 " fmt, ##args); } while (0)
+
 static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data)
 {
 	struct pool_info *pi = data;
@@ -325,6 +329,11 @@ static void raid1_end_read_request(struct bio *bio)
 
 	if (uptodate)
 		set_bit(R1BIO_Uptodate, &r1_bio->state);
+	else if (test_bit(FailFast, &rdev->flags) &&
+		 test_bit(R1BIO_FailFast, &r1_bio->state))
+		/* This was a fail-fast read so we definitely
+		 * want to retry */
+		;
 	else {
 		/* If all other devices have failed, we want to return
 		 * the error upwards rather than fail the last device.
@@ -347,13 +356,10 @@ static void raid1_end_read_request(struct bio *bio)
 		 * oops, read error:
 		 */
 		char b[BDEVNAME_SIZE];
-		printk_ratelimited(
-			KERN_ERR "md/raid1:%s: %s: "
-			"rescheduling sector %llu\n",
-			mdname(conf->mddev),
-			bdevname(rdev->bdev,
-				 b),
-			(unsigned long long)r1_bio->sector);
+		pr_err_ratelimited("md/raid1:%s: %s: rescheduling sector %llu\n",
+				   mdname(conf->mddev),
+				   bdevname(rdev->bdev, b),
+				   (unsigned long long)r1_bio->sector);
 		set_bit(R1BIO_ReadError, &r1_bio->state);
 		reschedule_retry(r1_bio);
 		/* don't drop the reference on read_disk yet */
@@ -416,7 +422,24 @@ static void raid1_end_write_request(struct bio *bio)
 			set_bit(MD_RECOVERY_NEEDED, &
 				conf->mddev->recovery);
 
-		set_bit(R1BIO_WriteError, &r1_bio->state);
+		if (test_bit(FailFast, &rdev->flags) &&
+		    (bio->bi_opf & MD_FAILFAST) &&
+		    /* We never try FailFast to WriteMostly devices */
+		    !test_bit(WriteMostly, &rdev->flags)) {
+			md_error(r1_bio->mddev, rdev);
+			if (!test_bit(Faulty, &rdev->flags))
+				/* This is the only remaining device,
+				 * We need to retry the write without
+				 * FailFast
+				 */
+				set_bit(R1BIO_WriteError, &r1_bio->state);
+			else {
+				/* Finished with this branch */
+				r1_bio->bios[mirror] = NULL;
+				to_put = bio;
+			}
+		} else
+			set_bit(R1BIO_WriteError, &r1_bio->state);
 	} else {
 		/*
 		 * Set R1BIO_Uptodate in our master bio, so that we
@@ -534,6 +557,7 @@ static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sect
 	best_good_sectors = 0;
 	has_nonrot_disk = 0;
 	choose_next_idle = 0;
+	clear_bit(R1BIO_FailFast, &r1_bio->state);
 
 	if ((conf->mddev->recovery_cp < this_sector + sectors) ||
 	    (mddev_is_clustered(conf->mddev) &&
@@ -607,6 +631,10 @@ static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sect
 		} else
 			best_good_sectors = sectors;
 
+		if (best_disk >= 0)
+			/* At least two disks to choose from so failfast is OK */
+			set_bit(R1BIO_FailFast, &r1_bio->state);
+
 		nonrot = blk_queue_nonrot(bdev_get_queue(rdev->bdev));
 		has_nonrot_disk |= nonrot;
 		pending = atomic_read(&rdev->nr_pending);
@@ -645,11 +673,6 @@ static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sect
 			}
 			break;
 		}
-		/* If device is idle, use it */
-		if (pending == 0) {
-			best_disk = disk;
-			break;
-		}
 
 		if (choose_next_idle)
 			continue;
@@ -672,7 +695,7 @@ static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sect
 	 * mixed ratation/non-rotational disks depending on workload.
 	 */
 	if (best_disk == -1) {
-		if (has_nonrot_disk)
+		if (has_nonrot_disk || min_pending == 0)
 			best_disk = best_pending_disk;
 		else
 			best_disk = best_dist_disk;
@@ -745,9 +768,14 @@ static void flush_pending_writes(struct r1conf *conf)
 
 		while (bio) { /* submit pending writes */
 			struct bio *next = bio->bi_next;
+			struct md_rdev *rdev = (void*)bio->bi_bdev;
 			bio->bi_next = NULL;
-			if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
-			    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
+			bio->bi_bdev = rdev->bdev;
+			if (test_bit(Faulty, &rdev->flags)) {
+				bio->bi_error = -EIO;
+				bio_endio(bio);
+			} else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
+					    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
 				/* Just ignore it */
 				bio_endio(bio);
 			else
@@ -832,7 +860,7 @@ static bool need_to_wait_for_sync(struct r1conf *conf, struct bio *bio)
 	else if (conf->barrier && bio_data_dir(bio) == WRITE) {
 		if ((conf->mddev->curr_resync_completed
 		     >= bio_end_sector(bio)) ||
-		    (conf->next_resync + NEXT_NORMALIO_DISTANCE
+		    (conf->start_next_window + NEXT_NORMALIO_DISTANCE
 		     <= bio->bi_iter.bi_sector))
 			wait = false;
 		else
@@ -858,6 +886,7 @@ static sector_t wait_barrier(struct r1conf *conf, struct bio *bio)
 		 * that queue to allow conf->start_next_window
 		 * to increase.
 		 */
+		raid1_log(conf->mddev, "wait barrier");
 		wait_event_lock_irq(conf->wait_barrier,
 				    !conf->array_frozen &&
 				    (!conf->barrier ||
@@ -937,6 +966,7 @@ static void freeze_array(struct r1conf *conf, int extra)
 	 */
 	spin_lock_irq(&conf->resync_lock);
 	conf->array_frozen = 1;
+	raid1_log(conf->mddev, "wait freeze");
 	wait_event_lock_irq_cmd(conf->wait_barrier,
 				conf->nr_pending == conf->nr_queued+extra,
 				conf->resync_lock,
@@ -1019,9 +1049,14 @@ static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule)
 
 	while (bio) { /* submit pending writes */
 		struct bio *next = bio->bi_next;
+		struct md_rdev *rdev = (void*)bio->bi_bdev;
 		bio->bi_next = NULL;
-		if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
-		    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
+		bio->bi_bdev = rdev->bdev;
+		if (test_bit(Faulty, &rdev->flags)) {
+			bio->bi_error = -EIO;
+			bio_endio(bio);
+		} else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
+				    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
 			/* Just ignore it */
 			bio_endio(bio);
 		else
@@ -1136,6 +1171,7 @@ read_again:
 			 * take care not to over-take any writes
 			 * that are 'behind'
 			 */
+			raid1_log(mddev, "wait behind writes");
 			wait_event(bitmap->behind_wait,
 				   atomic_read(&bitmap->behind_writes) == 0);
 		}
@@ -1153,8 +1189,16 @@ read_again:
 		read_bio->bi_bdev = mirror->rdev->bdev;
 		read_bio->bi_end_io = raid1_end_read_request;
 		bio_set_op_attrs(read_bio, op, do_sync);
+		if (test_bit(FailFast, &mirror->rdev->flags) &&
+		    test_bit(R1BIO_FailFast, &r1_bio->state))
+			read_bio->bi_opf |= MD_FAILFAST;
 		read_bio->bi_private = r1_bio;
 
+		if (mddev->gendisk)
+			trace_block_bio_remap(bdev_get_queue(read_bio->bi_bdev),
+					      read_bio, disk_devt(mddev->gendisk),
+					      r1_bio->sector);
+
 		if (max_sectors < r1_bio->sectors) {
 			/* could not read all from this device, so we will
 			 * need another r1_bio.
@@ -1195,6 +1239,7 @@ read_again:
 	 */
 	if (conf->pending_count >= max_queued_requests) {
 		md_wakeup_thread(mddev->thread);
+		raid1_log(mddev, "wait queued");
 		wait_event(conf->wait_barrier,
 			   conf->pending_count < max_queued_requests);
 	}
@@ -1286,6 +1331,7 @@ read_again:
 				rdev_dec_pending(conf->mirrors[j].rdev, mddev);
 		r1_bio->state = 0;
 		allow_barrier(conf, start_next_window, bio->bi_iter.bi_sector);
+		raid1_log(mddev, "wait rdev %d blocked", blocked_rdev->raid_disk);
 		md_wait_for_blocked_rdev(blocked_rdev, mddev);
 		start_next_window = wait_barrier(conf, bio);
 		/*
@@ -1363,10 +1409,21 @@ read_again:
 		mbio->bi_bdev = conf->mirrors[i].rdev->bdev;
 		mbio->bi_end_io	= raid1_end_write_request;
 		bio_set_op_attrs(mbio, op, do_flush_fua | do_sync);
+		if (test_bit(FailFast, &conf->mirrors[i].rdev->flags) &&
+		    !test_bit(WriteMostly, &conf->mirrors[i].rdev->flags) &&
+		    conf->raid_disks - mddev->degraded > 1)
+			mbio->bi_opf |= MD_FAILFAST;
 		mbio->bi_private = r1_bio;
 
 		atomic_inc(&r1_bio->remaining);
 
+		if (mddev->gendisk)
+			trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
+					      mbio, disk_devt(mddev->gendisk),
+					      r1_bio->sector);
+		/* flush_pending_writes() needs access to the rdev so...*/
+		mbio->bi_bdev = (void*)conf->mirrors[i].rdev;
+
 		cb = blk_check_plugged(raid1_unplug, mddev, sizeof(*plug));
 		if (cb)
 			plug = container_of(cb, struct raid1_plug_cb, cb);
@@ -1436,6 +1493,7 @@ static void raid1_error(struct mddev *mddev, struct md_rdev *rdev)
 	 * next level up know.
 	 * else mark the drive as failed
 	 */
+	spin_lock_irqsave(&conf->device_lock, flags);
 	if (test_bit(In_sync, &rdev->flags)
 	    && (conf->raid_disks - mddev->degraded) == 1) {
 		/*
@@ -1445,10 +1503,10 @@ static void raid1_error(struct mddev *mddev, struct md_rdev *rdev)
 		 * it is very likely to fail.
 		 */
 		conf->recovery_disabled = mddev->recovery_disabled;
+		spin_unlock_irqrestore(&conf->device_lock, flags);
 		return;
 	}
 	set_bit(Blocked, &rdev->flags);
-	spin_lock_irqsave(&conf->device_lock, flags);
 	if (test_and_clear_bit(In_sync, &rdev->flags)) {
 		mddev->degraded++;
 		set_bit(Faulty, &rdev->flags);
@@ -1459,36 +1517,35 @@ static void raid1_error(struct mddev *mddev, struct md_rdev *rdev)
 	 * if recovery is running, make sure it aborts.
 	 */
 	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
-	set_mask_bits(&mddev->flags, 0,
-		      BIT(MD_CHANGE_DEVS) | BIT(MD_CHANGE_PENDING));
-	printk(KERN_ALERT
-	       "md/raid1:%s: Disk failure on %s, disabling device.\n"
-	       "md/raid1:%s: Operation continuing on %d devices.\n",
-	       mdname(mddev), bdevname(rdev->bdev, b),
-	       mdname(mddev), conf->raid_disks - mddev->degraded);
+	set_mask_bits(&mddev->sb_flags, 0,
+		      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
+	pr_crit("md/raid1:%s: Disk failure on %s, disabling device.\n"
+		"md/raid1:%s: Operation continuing on %d devices.\n",
+		mdname(mddev), bdevname(rdev->bdev, b),
+		mdname(mddev), conf->raid_disks - mddev->degraded);
 }
 
 static void print_conf(struct r1conf *conf)
 {
 	int i;
 
-	printk(KERN_DEBUG "RAID1 conf printout:\n");
+	pr_debug("RAID1 conf printout:\n");
 	if (!conf) {
-		printk(KERN_DEBUG "(!conf)\n");
+		pr_debug("(!conf)\n");
 		return;
 	}
-	printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
-		conf->raid_disks);
+	pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
+		 conf->raid_disks);
 
 	rcu_read_lock();
 	for (i = 0; i < conf->raid_disks; i++) {
 		char b[BDEVNAME_SIZE];
 		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
 		if (rdev)
-			printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
-			       i, !test_bit(In_sync, &rdev->flags),
-			       !test_bit(Faulty, &rdev->flags),
-			       bdevname(rdev->bdev,b));
+			pr_debug(" disk %d, wo:%d, o:%d, dev:%s\n",
+				 i, !test_bit(In_sync, &rdev->flags),
+				 !test_bit(Faulty, &rdev->flags),
+				 bdevname(rdev->bdev,b));
 	}
 	rcu_read_unlock();
 }
@@ -1788,12 +1845,24 @@ static int fix_sync_read_error(struct r1bio *r1_bio)
 	sector_t sect = r1_bio->sector;
 	int sectors = r1_bio->sectors;
 	int idx = 0;
+	struct md_rdev *rdev;
+
+	rdev = conf->mirrors[r1_bio->read_disk].rdev;
+	if (test_bit(FailFast, &rdev->flags)) {
+		/* Don't try recovering from here - just fail it
+		 * ... unless it is the last working device of course */
+		md_error(mddev, rdev);
+		if (test_bit(Faulty, &rdev->flags))
+			/* Don't try to read from here, but make sure
+			 * put_buf does it's thing
+			 */
+			bio->bi_end_io = end_sync_write;
+	}
 
 	while(sectors) {
 		int s = sectors;
 		int d = r1_bio->read_disk;
 		int success = 0;
-		struct md_rdev *rdev;
 		int start;
 
 		if (s > (PAGE_SIZE>>9))
@@ -1825,11 +1894,10 @@ static int fix_sync_read_error(struct r1bio *r1_bio)
 			 * work just disable and interrupt the recovery.
 			 * Don't fail devices as that won't really help.
 			 */
-			printk(KERN_ALERT "md/raid1:%s: %s: unrecoverable I/O read error"
-			       " for block %llu\n",
-			       mdname(mddev),
-			       bdevname(bio->bi_bdev, b),
-			       (unsigned long long)r1_bio->sector);
+			pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n",
+					    mdname(mddev),
+					    bdevname(bio->bi_bdev, b),
+					    (unsigned long long)r1_bio->sector);
 			for (d = 0; d < conf->raid_disks * 2; d++) {
 				rdev = conf->mirrors[d].rdev;
 				if (!rdev || test_bit(Faulty, &rdev->flags))
@@ -2013,6 +2081,9 @@ static void sync_request_write(struct mddev *mddev, struct r1bio *r1_bio)
 			continue;
 
 		bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
+		if (test_bit(FailFast, &conf->mirrors[i].rdev->flags))
+			wbio->bi_opf |= MD_FAILFAST;
+
 		wbio->bi_end_io = end_sync_write;
 		atomic_inc(&r1_bio->remaining);
 		md_sync_acct(conf->mirrors[i].rdev->bdev, bio_sectors(wbio));
@@ -2122,13 +2193,11 @@ static void fix_read_error(struct r1conf *conf, int read_disk,
 				if (r1_sync_page_io(rdev, sect, s,
 						    conf->tmppage, READ)) {
 					atomic_add(s, &rdev->corrected_errors);
-					printk(KERN_INFO
-					       "md/raid1:%s: read error corrected "
-					       "(%d sectors at %llu on %s)\n",
-					       mdname(mddev), s,
-					       (unsigned long long)(sect +
-								    rdev->data_offset),
-					       bdevname(rdev->bdev, b));
+					pr_info("md/raid1:%s: read error corrected (%d sectors at %llu on %s)\n",
+						mdname(mddev), s,
+						(unsigned long long)(sect +
+								     rdev->data_offset),
+						bdevname(rdev->bdev, b));
 				}
 				rdev_dec_pending(rdev, mddev);
 			} else
@@ -2287,6 +2356,8 @@ static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)
 	struct bio *bio;
 	char b[BDEVNAME_SIZE];
 	struct md_rdev *rdev;
+	dev_t bio_dev;
+	sector_t bio_sector;
 
 	clear_bit(R1BIO_ReadError, &r1_bio->state);
 	/* we got a read error. Maybe the drive is bad.  Maybe just
@@ -2300,10 +2371,14 @@ static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)
 
 	bio = r1_bio->bios[r1_bio->read_disk];
 	bdevname(bio->bi_bdev, b);
+	bio_dev = bio->bi_bdev->bd_dev;
+	bio_sector = conf->mirrors[r1_bio->read_disk].rdev->data_offset + r1_bio->sector;
 	bio_put(bio);
 	r1_bio->bios[r1_bio->read_disk] = NULL;
 
-	if (mddev->ro == 0) {
+	rdev = conf->mirrors[r1_bio->read_disk].rdev;
+	if (mddev->ro == 0
+	    && !test_bit(FailFast, &rdev->flags)) {
 		freeze_array(conf, 1);
 		fix_read_error(conf, r1_bio->read_disk,
 			       r1_bio->sector, r1_bio->sectors);
@@ -2312,14 +2387,13 @@ static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)
 		r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED;
 	}
 
-	rdev_dec_pending(conf->mirrors[r1_bio->read_disk].rdev, conf->mddev);
+	rdev_dec_pending(rdev, conf->mddev);
 
 read_more:
 	disk = read_balance(conf, r1_bio, &max_sectors);
 	if (disk == -1) {
-		printk(KERN_ALERT "md/raid1:%s: %s: unrecoverable I/O"
-		       " read error for block %llu\n",
-		       mdname(mddev), b, (unsigned long long)r1_bio->sector);
+		pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n",
+				    mdname(mddev), b, (unsigned long long)r1_bio->sector);
 		raid_end_bio_io(r1_bio);
 	} else {
 		const unsigned long do_sync
@@ -2330,16 +2404,17 @@ read_more:
 			 max_sectors);
 		r1_bio->bios[r1_bio->read_disk] = bio;
 		rdev = conf->mirrors[disk].rdev;
-		printk_ratelimited(KERN_ERR
-				   "md/raid1:%s: redirecting sector %llu"
-				   " to other mirror: %s\n",
-				   mdname(mddev),
-				   (unsigned long long)r1_bio->sector,
-				   bdevname(rdev->bdev, b));
+		pr_info_ratelimited("md/raid1:%s: redirecting sector %llu to other mirror: %s\n",
+				    mdname(mddev),
+				    (unsigned long long)r1_bio->sector,
+				    bdevname(rdev->bdev, b));
 		bio->bi_iter.bi_sector = r1_bio->sector + rdev->data_offset;
 		bio->bi_bdev = rdev->bdev;
 		bio->bi_end_io = raid1_end_read_request;
 		bio_set_op_attrs(bio, REQ_OP_READ, do_sync);
+		if (test_bit(FailFast, &rdev->flags) &&
+		    test_bit(R1BIO_FailFast, &r1_bio->state))
+			bio->bi_opf |= MD_FAILFAST;
 		bio->bi_private = r1_bio;
 		if (max_sectors < r1_bio->sectors) {
 			/* Drat - have to split this up more */
@@ -2353,6 +2428,8 @@ read_more:
 			else
 				mbio->bi_phys_segments++;
 			spin_unlock_irq(&conf->device_lock);
+			trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
+					      bio, bio_dev, bio_sector);
 			generic_make_request(bio);
 			bio = NULL;
 
@@ -2367,8 +2444,11 @@ read_more:
 				sectors_handled;
 
 			goto read_more;
-		} else
+		} else {
+			trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
+					      bio, bio_dev, bio_sector);
 			generic_make_request(bio);
+		}
 	}
 }
 
@@ -2384,10 +2464,10 @@ static void raid1d(struct md_thread *thread)
 	md_check_recovery(mddev);
 
 	if (!list_empty_careful(&conf->bio_end_io_list) &&
-	    !test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
+	    !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
 		LIST_HEAD(tmp);
 		spin_lock_irqsave(&conf->device_lock, flags);
-		if (!test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
+		if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
 			while (!list_empty(&conf->bio_end_io_list)) {
 				list_move(conf->bio_end_io_list.prev, &tmp);
 				conf->nr_queued--;
@@ -2441,7 +2521,7 @@ static void raid1d(struct md_thread *thread)
 			generic_make_request(r1_bio->bios[r1_bio->read_disk]);
 
 		cond_resched();
-		if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
+		if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
 			md_check_recovery(mddev);
 	}
 	blk_finish_plug(&plug);
@@ -2623,6 +2703,8 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
 			bio->bi_iter.bi_sector = sector_nr + rdev->data_offset;
 			bio->bi_bdev = rdev->bdev;
 			bio->bi_private = r1_bio;
+			if (test_bit(FailFast, &rdev->flags))
+				bio->bi_opf |= MD_FAILFAST;
 		}
 	}
 	rcu_read_unlock();
@@ -2642,7 +2724,7 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
 							min_bad, 0
 					) && ok;
 			}
-		set_bit(MD_CHANGE_DEVS, &mddev->flags);
+		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 		*skipped = 1;
 		put_buf(r1_bio);
 
@@ -2753,6 +2835,8 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
 			if (bio->bi_end_io == end_sync_read) {
 				read_targets--;
 				md_sync_acct(bio->bi_bdev, nr_sectors);
+				if (read_targets == 1)
+					bio->bi_opf &= ~MD_FAILFAST;
 				generic_make_request(bio);
 			}
 		}
@@ -2760,6 +2844,8 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
 		atomic_set(&r1_bio->remaining, 1);
 		bio = r1_bio->bios[r1_bio->read_disk];
 		md_sync_acct(bio->bi_bdev, nr_sectors);
+		if (read_targets == 1)
+			bio->bi_opf &= ~MD_FAILFAST;
 		generic_make_request(bio);
 
 	}
@@ -2875,12 +2961,8 @@ static struct r1conf *setup_conf(struct mddev *mddev)
 
 	err = -ENOMEM;
 	conf->thread = md_register_thread(raid1d, mddev, "raid1");
-	if (!conf->thread) {
-		printk(KERN_ERR
-		       "md/raid1:%s: couldn't allocate thread\n",
-		       mdname(mddev));
+	if (!conf->thread)
 		goto abort;
-	}
 
 	return conf;
 
@@ -2905,13 +2987,13 @@ static int raid1_run(struct mddev *mddev)
 	bool discard_supported = false;
 
 	if (mddev->level != 1) {
-		printk(KERN_ERR "md/raid1:%s: raid level not set to mirroring (%d)\n",
-		       mdname(mddev), mddev->level);
+		pr_warn("md/raid1:%s: raid level not set to mirroring (%d)\n",
+			mdname(mddev), mddev->level);
 		return -EIO;
 	}
 	if (mddev->reshape_position != MaxSector) {
-		printk(KERN_ERR "md/raid1:%s: reshape_position set but not supported\n",
-		       mdname(mddev));
+		pr_warn("md/raid1:%s: reshape_position set but not supported\n",
+			mdname(mddev));
 		return -EIO;
 	}
 	/*
@@ -2950,11 +3032,9 @@ static int raid1_run(struct mddev *mddev)
 		mddev->recovery_cp = MaxSector;
 
 	if (mddev->recovery_cp != MaxSector)
-		printk(KERN_NOTICE "md/raid1:%s: not clean"
-		       " -- starting background reconstruction\n",
-		       mdname(mddev));
-	printk(KERN_INFO
-		"md/raid1:%s: active with %d out of %d mirrors\n",
+		pr_info("md/raid1:%s: not clean -- starting background reconstruction\n",
+			mdname(mddev));
+	pr_info("md/raid1:%s: active with %d out of %d mirrors\n",
 		mdname(mddev), mddev->raid_disks - mddev->degraded,
 		mddev->raid_disks);
 
@@ -2964,6 +3044,7 @@ static int raid1_run(struct mddev *mddev)
 	mddev->thread = conf->thread;
 	conf->thread = NULL;
 	mddev->private = conf;
+	set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
 
 	md_set_array_sectors(mddev, raid1_size(mddev, 0, 0));
 
@@ -3107,9 +3188,8 @@ static int raid1_reshape(struct mddev *mddev)
 			rdev->raid_disk = d2;
 			sysfs_unlink_rdev(mddev, rdev);
 			if (sysfs_link_rdev(mddev, rdev))
-				printk(KERN_WARNING
-				       "md/raid1:%s: cannot register rd%d\n",
-				       mdname(mddev), rdev->raid_disk);
+				pr_warn("md/raid1:%s: cannot register rd%d\n",
+					mdname(mddev), rdev->raid_disk);
 		}
 		if (rdev)
 			newmirrors[d2++].rdev = rdev;
@@ -3163,9 +3243,12 @@ static void *raid1_takeover(struct mddev *mddev)
 		mddev->new_layout = 0;
 		mddev->new_chunk_sectors = 0;
 		conf = setup_conf(mddev);
-		if (!IS_ERR(conf))
+		if (!IS_ERR(conf)) {
 			/* Array must appear to be quiesced */
 			conf->array_frozen = 1;
+			clear_bit(MD_HAS_JOURNAL, &mddev->flags);
+			clear_bit(MD_JOURNAL_CLEAN, &mddev->flags);
+		}
 		return conf;
 	}
 	return ERR_PTR(-EINVAL);
diff --git a/drivers/md/raid1.h b/drivers/md/raid1.h
index 61c39b390cd8..c52ef424a24b 100644
--- a/drivers/md/raid1.h
+++ b/drivers/md/raid1.h
@@ -161,14 +161,15 @@ struct r1bio {
 };
 
 /* bits for r1bio.state */
-#define	R1BIO_Uptodate	0
-#define	R1BIO_IsSync	1
-#define	R1BIO_Degraded	2
-#define	R1BIO_BehindIO	3
+enum r1bio_state {
+	R1BIO_Uptodate,
+	R1BIO_IsSync,
+	R1BIO_Degraded,
+	R1BIO_BehindIO,
 /* Set ReadError on bios that experience a readerror so that
  * raid1d knows what to do with them.
  */
-#define R1BIO_ReadError 4
+	R1BIO_ReadError,
 /* For write-behind requests, we call bi_end_io when
  * the last non-write-behind device completes, providing
  * any write was successful.  Otherwise we call when
@@ -176,10 +177,12 @@ struct r1bio {
  * with failure when last write completes (and all failed).
  * Record that bi_end_io was called with this flag...
  */
-#define	R1BIO_Returned 6
+	R1BIO_Returned,
 /* If a write for this request means we can clear some
  * known-bad-block records, we set this flag
  */
-#define	R1BIO_MadeGood 7
-#define	R1BIO_WriteError 8
+	R1BIO_MadeGood,
+	R1BIO_WriteError,
+	R1BIO_FailFast,
+};
 #endif
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 39fddda2fef2..ab5e86209322 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -25,6 +25,7 @@
 #include <linux/seq_file.h>
 #include <linux/ratelimit.h>
 #include <linux/kthread.h>
+#include <trace/events/block.h>
 #include "md.h"
 #include "raid10.h"
 #include "raid0.h"
@@ -99,12 +100,16 @@ static int max_queued_requests = 1024;
 static void allow_barrier(struct r10conf *conf);
 static void lower_barrier(struct r10conf *conf);
 static int _enough(struct r10conf *conf, int previous, int ignore);
+static int enough(struct r10conf *conf, int ignore);
 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
 				int *skipped);
 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
 static void end_reshape_write(struct bio *bio);
 static void end_reshape(struct r10conf *conf);
 
+#define raid10_log(md, fmt, args...)				\
+	do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid10 " fmt, ##args); } while (0)
+
 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
 {
 	struct r10conf *conf = data;
@@ -404,8 +409,7 @@ static void raid10_end_read_request(struct bio *bio)
 		 * oops, read error - keep the refcount on the rdev
 		 */
 		char b[BDEVNAME_SIZE];
-		printk_ratelimited(KERN_ERR
-				   "md/raid10:%s: %s: rescheduling sector %llu\n",
+		pr_err_ratelimited("md/raid10:%s: %s: rescheduling sector %llu\n",
 				   mdname(conf->mddev),
 				   bdevname(rdev->bdev, b),
 				   (unsigned long long)r10_bio->sector);
@@ -447,6 +451,7 @@ static void raid10_end_write_request(struct bio *bio)
 	struct r10conf *conf = r10_bio->mddev->private;
 	int slot, repl;
 	struct md_rdev *rdev = NULL;
+	struct bio *to_put = NULL;
 	bool discard_error;
 
 	discard_error = bio->bi_error && bio_op(bio) == REQ_OP_DISCARD;
@@ -474,8 +479,24 @@ static void raid10_end_write_request(struct bio *bio)
 			if (!test_and_set_bit(WantReplacement, &rdev->flags))
 				set_bit(MD_RECOVERY_NEEDED,
 					&rdev->mddev->recovery);
-			set_bit(R10BIO_WriteError, &r10_bio->state);
+
 			dec_rdev = 0;
+			if (test_bit(FailFast, &rdev->flags) &&
+			    (bio->bi_opf & MD_FAILFAST)) {
+				md_error(rdev->mddev, rdev);
+				if (!test_bit(Faulty, &rdev->flags))
+					/* This is the only remaining device,
+					 * We need to retry the write without
+					 * FailFast
+					 */
+					set_bit(R10BIO_WriteError, &r10_bio->state);
+				else {
+					r10_bio->devs[slot].bio = NULL;
+					to_put = bio;
+					dec_rdev = 1;
+				}
+			} else
+				set_bit(R10BIO_WriteError, &r10_bio->state);
 		}
 	} else {
 		/*
@@ -525,6 +546,8 @@ static void raid10_end_write_request(struct bio *bio)
 	one_write_done(r10_bio);
 	if (dec_rdev)
 		rdev_dec_pending(rdev, conf->mddev);
+	if (to_put)
+		bio_put(to_put);
 }
 
 /*
@@ -716,6 +739,7 @@ static struct md_rdev *read_balance(struct r10conf *conf,
 	best_dist = MaxSector;
 	best_good_sectors = 0;
 	do_balance = 1;
+	clear_bit(R10BIO_FailFast, &r10_bio->state);
 	/*
 	 * Check if we can balance. We can balance on the whole
 	 * device if no resync is going on (recovery is ok), or below
@@ -780,15 +804,18 @@ static struct md_rdev *read_balance(struct r10conf *conf,
 		if (!do_balance)
 			break;
 
+		if (best_slot >= 0)
+			/* At least 2 disks to choose from so failfast is OK */
+			set_bit(R10BIO_FailFast, &r10_bio->state);
 		/* This optimisation is debatable, and completely destroys
 		 * sequential read speed for 'far copies' arrays.  So only
 		 * keep it for 'near' arrays, and review those later.
 		 */
 		if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending))
-			break;
+			new_distance = 0;
 
 		/* for far > 1 always use the lowest address */
-		if (geo->far_copies > 1)
+		else if (geo->far_copies > 1)
 			new_distance = r10_bio->devs[slot].addr;
 		else
 			new_distance = abs(r10_bio->devs[slot].addr -
@@ -859,9 +886,14 @@ static void flush_pending_writes(struct r10conf *conf)
 
 		while (bio) { /* submit pending writes */
 			struct bio *next = bio->bi_next;
+			struct md_rdev *rdev = (void*)bio->bi_bdev;
 			bio->bi_next = NULL;
-			if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
-			    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
+			bio->bi_bdev = rdev->bdev;
+			if (test_bit(Faulty, &rdev->flags)) {
+				bio->bi_error = -EIO;
+				bio_endio(bio);
+			} else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
+					    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
 				/* Just ignore it */
 				bio_endio(bio);
 			else
@@ -937,6 +969,7 @@ static void wait_barrier(struct r10conf *conf)
 		 * that queue to get the nr_pending
 		 * count down.
 		 */
+		raid10_log(conf->mddev, "wait barrier");
 		wait_event_lock_irq(conf->wait_barrier,
 				    !conf->barrier ||
 				    (atomic_read(&conf->nr_pending) &&
@@ -1037,9 +1070,14 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
 
 	while (bio) { /* submit pending writes */
 		struct bio *next = bio->bi_next;
+		struct md_rdev *rdev = (void*)bio->bi_bdev;
 		bio->bi_next = NULL;
-		if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
-		    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
+		bio->bi_bdev = rdev->bdev;
+		if (test_bit(Faulty, &rdev->flags)) {
+			bio->bi_error = -EIO;
+			bio_endio(bio);
+		} else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
+				    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
 			/* Just ignore it */
 			bio_endio(bio);
 		else
@@ -1083,6 +1121,7 @@ static void __make_request(struct mddev *mddev, struct bio *bio)
 		/* IO spans the reshape position.  Need to wait for
 		 * reshape to pass
 		 */
+		raid10_log(conf->mddev, "wait reshape");
 		allow_barrier(conf);
 		wait_event(conf->wait_barrier,
 			   conf->reshape_progress <= bio->bi_iter.bi_sector ||
@@ -1099,11 +1138,12 @@ static void __make_request(struct mddev *mddev, struct bio *bio)
 		bio->bi_iter.bi_sector < conf->reshape_progress))) {
 		/* Need to update reshape_position in metadata */
 		mddev->reshape_position = conf->reshape_progress;
-		set_mask_bits(&mddev->flags, 0,
-			      BIT(MD_CHANGE_DEVS) | BIT(MD_CHANGE_PENDING));
+		set_mask_bits(&mddev->sb_flags, 0,
+			      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
 		md_wakeup_thread(mddev->thread);
+		raid10_log(conf->mddev, "wait reshape metadata");
 		wait_event(mddev->sb_wait,
-			   !test_bit(MD_CHANGE_PENDING, &mddev->flags));
+			   !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
 
 		conf->reshape_safe = mddev->reshape_position;
 	}
@@ -1154,8 +1194,15 @@ read_again:
 		read_bio->bi_bdev = rdev->bdev;
 		read_bio->bi_end_io = raid10_end_read_request;
 		bio_set_op_attrs(read_bio, op, do_sync);
+		if (test_bit(FailFast, &rdev->flags) &&
+		    test_bit(R10BIO_FailFast, &r10_bio->state))
+			read_bio->bi_opf |= MD_FAILFAST;
 		read_bio->bi_private = r10_bio;
 
+		if (mddev->gendisk)
+			trace_block_bio_remap(bdev_get_queue(read_bio->bi_bdev),
+					      read_bio, disk_devt(mddev->gendisk),
+					      r10_bio->sector);
 		if (max_sectors < r10_bio->sectors) {
 			/* Could not read all from this device, so we will
 			 * need another r10_bio.
@@ -1195,6 +1242,7 @@ read_again:
 	 */
 	if (conf->pending_count >= max_queued_requests) {
 		md_wakeup_thread(mddev->thread);
+		raid10_log(mddev, "wait queued");
 		wait_event(conf->wait_barrier,
 			   conf->pending_count < max_queued_requests);
 	}
@@ -1322,6 +1370,7 @@ retry_write:
 			}
 		}
 		allow_barrier(conf);
+		raid10_log(conf->mddev, "wait rdev %d blocked", blocked_rdev->raid_disk);
 		md_wait_for_blocked_rdev(blocked_rdev, mddev);
 		wait_barrier(conf);
 		goto retry_write;
@@ -1361,8 +1410,18 @@ retry_write:
 			mbio->bi_bdev = rdev->bdev;
 			mbio->bi_end_io	= raid10_end_write_request;
 			bio_set_op_attrs(mbio, op, do_sync | do_fua);
+			if (test_bit(FailFast, &conf->mirrors[d].rdev->flags) &&
+			    enough(conf, d))
+				mbio->bi_opf |= MD_FAILFAST;
 			mbio->bi_private = r10_bio;
 
+			if (conf->mddev->gendisk)
+				trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
+						      mbio, disk_devt(conf->mddev->gendisk),
+						      r10_bio->sector);
+			/* flush_pending_writes() needs access to the rdev so...*/
+			mbio->bi_bdev = (void*)rdev;
+
 			atomic_inc(&r10_bio->remaining);
 
 			cb = blk_check_plugged(raid10_unplug, mddev,
@@ -1405,6 +1464,13 @@ retry_write:
 			bio_set_op_attrs(mbio, op, do_sync | do_fua);
 			mbio->bi_private = r10_bio;
 
+			if (conf->mddev->gendisk)
+				trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
+						      mbio, disk_devt(conf->mddev->gendisk),
+						      r10_bio->sector);
+			/* flush_pending_writes() needs access to the rdev so...*/
+			mbio->bi_bdev = (void*)rdev;
+
 			atomic_inc(&r10_bio->remaining);
 			spin_lock_irqsave(&conf->device_lock, flags);
 			bio_list_add(&conf->pending_bio_list, mbio);
@@ -1586,14 +1652,13 @@ static void raid10_error(struct mddev *mddev, struct md_rdev *rdev)
 	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
 	set_bit(Blocked, &rdev->flags);
 	set_bit(Faulty, &rdev->flags);
-	set_mask_bits(&mddev->flags, 0,
-		      BIT(MD_CHANGE_DEVS) | BIT(MD_CHANGE_PENDING));
+	set_mask_bits(&mddev->sb_flags, 0,
+		      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
 	spin_unlock_irqrestore(&conf->device_lock, flags);
-	printk(KERN_ALERT
-	       "md/raid10:%s: Disk failure on %s, disabling device.\n"
-	       "md/raid10:%s: Operation continuing on %d devices.\n",
-	       mdname(mddev), bdevname(rdev->bdev, b),
-	       mdname(mddev), conf->geo.raid_disks - mddev->degraded);
+	pr_crit("md/raid10:%s: Disk failure on %s, disabling device.\n"
+		"md/raid10:%s: Operation continuing on %d devices.\n",
+		mdname(mddev), bdevname(rdev->bdev, b),
+		mdname(mddev), conf->geo.raid_disks - mddev->degraded);
 }
 
 static void print_conf(struct r10conf *conf)
@@ -1601,13 +1666,13 @@ static void print_conf(struct r10conf *conf)
 	int i;
 	struct md_rdev *rdev;
 
-	printk(KERN_DEBUG "RAID10 conf printout:\n");
+	pr_debug("RAID10 conf printout:\n");
 	if (!conf) {
-		printk(KERN_DEBUG "(!conf)\n");
+		pr_debug("(!conf)\n");
 		return;
 	}
-	printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
-		conf->geo.raid_disks);
+	pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
+		 conf->geo.raid_disks);
 
 	/* This is only called with ->reconfix_mutex held, so
 	 * rcu protection of rdev is not needed */
@@ -1615,10 +1680,10 @@ static void print_conf(struct r10conf *conf)
 		char b[BDEVNAME_SIZE];
 		rdev = conf->mirrors[i].rdev;
 		if (rdev)
-			printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
-				i, !test_bit(In_sync, &rdev->flags),
-			        !test_bit(Faulty, &rdev->flags),
-				bdevname(rdev->bdev,b));
+			pr_debug(" disk %d, wo:%d, o:%d, dev:%s\n",
+				 i, !test_bit(In_sync, &rdev->flags),
+				 !test_bit(Faulty, &rdev->flags),
+				 bdevname(rdev->bdev,b));
 	}
 }
 
@@ -1953,6 +2018,7 @@ static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
 	/* now find blocks with errors */
 	for (i=0 ; i < conf->copies ; i++) {
 		int  j, d;
+		struct md_rdev *rdev;
 
 		tbio = r10_bio->devs[i].bio;
 
@@ -1960,6 +2026,8 @@ static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
 			continue;
 		if (i == first)
 			continue;
+		d = r10_bio->devs[i].devnum;
+		rdev = conf->mirrors[d].rdev;
 		if (!r10_bio->devs[i].bio->bi_error) {
 			/* We know that the bi_io_vec layout is the same for
 			 * both 'first' and 'i', so we just compare them.
@@ -1982,6 +2050,10 @@ static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
 			if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
 				/* Don't fix anything. */
 				continue;
+		} else if (test_bit(FailFast, &rdev->flags)) {
+			/* Just give up on this device */
+			md_error(rdev->mddev, rdev);
+			continue;
 		}
 		/* Ok, we need to write this bio, either to correct an
 		 * inconsistency or to correct an unreadable block.
@@ -1999,11 +2071,12 @@ static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
 
 		bio_copy_data(tbio, fbio);
 
-		d = r10_bio->devs[i].devnum;
 		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
 		atomic_inc(&r10_bio->remaining);
 		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio));
 
+		if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
+			tbio->bi_opf |= MD_FAILFAST;
 		tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;
 		tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
 		generic_make_request(tbio);
@@ -2109,10 +2182,8 @@ static void fix_recovery_read_error(struct r10bio *r10_bio)
 				ok = rdev_set_badblocks(rdev2, addr, s, 0);
 				if (!ok) {
 					/* just abort the recovery */
-					printk(KERN_NOTICE
-					       "md/raid10:%s: recovery aborted"
-					       " due to read error\n",
-					       mdname(mddev));
+					pr_notice("md/raid10:%s: recovery aborted due to read error\n",
+						  mdname(mddev));
 
 					conf->mirrors[dw].recovery_disabled
 						= mddev->recovery_disabled;
@@ -2259,14 +2330,11 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10
 		char b[BDEVNAME_SIZE];
 		bdevname(rdev->bdev, b);
 
-		printk(KERN_NOTICE
-		       "md/raid10:%s: %s: Raid device exceeded "
-		       "read_error threshold [cur %d:max %d]\n",
-		       mdname(mddev), b,
-		       atomic_read(&rdev->read_errors), max_read_errors);
-		printk(KERN_NOTICE
-		       "md/raid10:%s: %s: Failing raid device\n",
-		       mdname(mddev), b);
+		pr_notice("md/raid10:%s: %s: Raid device exceeded read_error threshold [cur %d:max %d]\n",
+			  mdname(mddev), b,
+			  atomic_read(&rdev->read_errors), max_read_errors);
+		pr_notice("md/raid10:%s: %s: Failing raid device\n",
+			  mdname(mddev), b);
 		md_error(mddev, rdev);
 		r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
 		return;
@@ -2356,20 +2424,16 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10
 					     s, conf->tmppage, WRITE)
 			    == 0) {
 				/* Well, this device is dead */
-				printk(KERN_NOTICE
-				       "md/raid10:%s: read correction "
-				       "write failed"
-				       " (%d sectors at %llu on %s)\n",
-				       mdname(mddev), s,
-				       (unsigned long long)(
-					       sect +
-					       choose_data_offset(r10_bio,
-								  rdev)),
-				       bdevname(rdev->bdev, b));
-				printk(KERN_NOTICE "md/raid10:%s: %s: failing "
-				       "drive\n",
-				       mdname(mddev),
-				       bdevname(rdev->bdev, b));
+				pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %s)\n",
+					  mdname(mddev), s,
+					  (unsigned long long)(
+						  sect +
+						  choose_data_offset(r10_bio,
+								     rdev)),
+					  bdevname(rdev->bdev, b));
+				pr_notice("md/raid10:%s: %s: failing drive\n",
+					  mdname(mddev),
+					  bdevname(rdev->bdev, b));
 			}
 			rdev_dec_pending(rdev, mddev);
 			rcu_read_lock();
@@ -2397,24 +2461,18 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10
 						 READ)) {
 			case 0:
 				/* Well, this device is dead */
-				printk(KERN_NOTICE
-				       "md/raid10:%s: unable to read back "
-				       "corrected sectors"
-				       " (%d sectors at %llu on %s)\n",
+				pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %s)\n",
 				       mdname(mddev), s,
 				       (unsigned long long)(
 					       sect +
 					       choose_data_offset(r10_bio, rdev)),
 				       bdevname(rdev->bdev, b));
-				printk(KERN_NOTICE "md/raid10:%s: %s: failing "
-				       "drive\n",
+				pr_notice("md/raid10:%s: %s: failing drive\n",
 				       mdname(mddev),
 				       bdevname(rdev->bdev, b));
 				break;
 			case 1:
-				printk(KERN_INFO
-				       "md/raid10:%s: read error corrected"
-				       " (%d sectors at %llu on %s)\n",
+				pr_info("md/raid10:%s: read error corrected (%d sectors at %llu on %s)\n",
 				       mdname(mddev), s,
 				       (unsigned long long)(
 					       sect +
@@ -2503,6 +2561,8 @@ static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
 	char b[BDEVNAME_SIZE];
 	unsigned long do_sync;
 	int max_sectors;
+	dev_t bio_dev;
+	sector_t bio_last_sector;
 
 	/* we got a read error. Maybe the drive is bad.  Maybe just
 	 * the block and we can fix it.
@@ -2514,38 +2574,38 @@ static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
 	 */
 	bio = r10_bio->devs[slot].bio;
 	bdevname(bio->bi_bdev, b);
+	bio_dev = bio->bi_bdev->bd_dev;
+	bio_last_sector = r10_bio->devs[slot].addr + rdev->data_offset + r10_bio->sectors;
 	bio_put(bio);
 	r10_bio->devs[slot].bio = NULL;
 
-	if (mddev->ro == 0) {
+	if (mddev->ro)
+		r10_bio->devs[slot].bio = IO_BLOCKED;
+	else if (!test_bit(FailFast, &rdev->flags)) {
 		freeze_array(conf, 1);
 		fix_read_error(conf, mddev, r10_bio);
 		unfreeze_array(conf);
 	} else
-		r10_bio->devs[slot].bio = IO_BLOCKED;
+		md_error(mddev, rdev);
 
 	rdev_dec_pending(rdev, mddev);
 
 read_more:
 	rdev = read_balance(conf, r10_bio, &max_sectors);
 	if (rdev == NULL) {
-		printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O"
-		       " read error for block %llu\n",
-		       mdname(mddev), b,
-		       (unsigned long long)r10_bio->sector);
+		pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n",
+				    mdname(mddev), b,
+				    (unsigned long long)r10_bio->sector);
 		raid_end_bio_io(r10_bio);
 		return;
 	}
 
 	do_sync = (r10_bio->master_bio->bi_opf & REQ_SYNC);
 	slot = r10_bio->read_slot;
-	printk_ratelimited(
-		KERN_ERR
-		"md/raid10:%s: %s: redirecting "
-		"sector %llu to another mirror\n",
-		mdname(mddev),
-		bdevname(rdev->bdev, b),
-		(unsigned long long)r10_bio->sector);
+	pr_err_ratelimited("md/raid10:%s: %s: redirecting sector %llu to another mirror\n",
+			   mdname(mddev),
+			   bdevname(rdev->bdev, b),
+			   (unsigned long long)r10_bio->sector);
 	bio = bio_clone_mddev(r10_bio->master_bio,
 			      GFP_NOIO, mddev);
 	bio_trim(bio, r10_bio->sector - bio->bi_iter.bi_sector, max_sectors);
@@ -2555,8 +2615,15 @@ read_more:
 		+ choose_data_offset(r10_bio, rdev);
 	bio->bi_bdev = rdev->bdev;
 	bio_set_op_attrs(bio, REQ_OP_READ, do_sync);
+	if (test_bit(FailFast, &rdev->flags) &&
+	    test_bit(R10BIO_FailFast, &r10_bio->state))
+		bio->bi_opf |= MD_FAILFAST;
 	bio->bi_private = r10_bio;
 	bio->bi_end_io = raid10_end_read_request;
+	trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
+			      bio, bio_dev,
+			      bio_last_sector - r10_bio->sectors);
+
 	if (max_sectors < r10_bio->sectors) {
 		/* Drat - have to split this up more */
 		struct bio *mbio = r10_bio->master_bio;
@@ -2694,10 +2761,10 @@ static void raid10d(struct md_thread *thread)
 	md_check_recovery(mddev);
 
 	if (!list_empty_careful(&conf->bio_end_io_list) &&
-	    !test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
+	    !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
 		LIST_HEAD(tmp);
 		spin_lock_irqsave(&conf->device_lock, flags);
-		if (!test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
+		if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
 			while (!list_empty(&conf->bio_end_io_list)) {
 				list_move(conf->bio_end_io_list.prev, &tmp);
 				conf->nr_queued--;
@@ -2755,7 +2822,7 @@ static void raid10d(struct md_thread *thread)
 		}
 
 		cond_resched();
-		if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
+		if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
 			md_check_recovery(mddev);
 	}
 	blk_finish_plug(&plug);
@@ -3072,6 +3139,8 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 				bio->bi_private = r10_bio;
 				bio->bi_end_io = end_sync_read;
 				bio_set_op_attrs(bio, REQ_OP_READ, 0);
+				if (test_bit(FailFast, &rdev->flags))
+					bio->bi_opf |= MD_FAILFAST;
 				from_addr = r10_bio->devs[j].addr;
 				bio->bi_iter.bi_sector = from_addr +
 					rdev->data_offset;
@@ -3160,8 +3229,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 				if (!any_working)  {
 					if (!test_and_set_bit(MD_RECOVERY_INTR,
 							      &mddev->recovery))
-						printk(KERN_INFO "md/raid10:%s: insufficient "
-						       "working devices for recovery.\n",
+						pr_warn("md/raid10:%s: insufficient working devices for recovery.\n",
 						       mdname(mddev));
 					mirror->recovery_disabled
 						= mddev->recovery_disabled;
@@ -3178,6 +3246,23 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 			rdev_dec_pending(mrdev, mddev);
 			if (mreplace)
 				rdev_dec_pending(mreplace, mddev);
+			if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) {
+				/* Only want this if there is elsewhere to
+				 * read from. 'j' is currently the first
+				 * readable copy.
+				 */
+				int targets = 1;
+				for (; j < conf->copies; j++) {
+					int d = r10_bio->devs[j].devnum;
+					if (conf->mirrors[d].rdev &&
+					    test_bit(In_sync,
+						      &conf->mirrors[d].rdev->flags))
+						targets++;
+				}
+				if (targets == 1)
+					r10_bio->devs[0].bio->bi_opf
+						&= ~MD_FAILFAST;
+			}
 		}
 		if (biolist == NULL) {
 			while (r10_bio) {
@@ -3256,6 +3341,8 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 			bio->bi_private = r10_bio;
 			bio->bi_end_io = end_sync_read;
 			bio_set_op_attrs(bio, REQ_OP_READ, 0);
+			if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
+				bio->bi_opf |= MD_FAILFAST;
 			bio->bi_iter.bi_sector = sector + rdev->data_offset;
 			bio->bi_bdev = rdev->bdev;
 			count++;
@@ -3279,6 +3366,8 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 			bio->bi_private = r10_bio;
 			bio->bi_end_io = end_sync_write;
 			bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
+			if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
+				bio->bi_opf |= MD_FAILFAST;
 			bio->bi_iter.bi_sector = sector + rdev->data_offset;
 			bio->bi_bdev = rdev->bdev;
 			count++;
@@ -3489,15 +3578,14 @@ static struct r10conf *setup_conf(struct mddev *mddev)
 	copies = setup_geo(&geo, mddev, geo_new);
 
 	if (copies == -2) {
-		printk(KERN_ERR "md/raid10:%s: chunk size must be "
-		       "at least PAGE_SIZE(%ld) and be a power of 2.\n",
-		       mdname(mddev), PAGE_SIZE);
+		pr_warn("md/raid10:%s: chunk size must be at least PAGE_SIZE(%ld) and be a power of 2.\n",
+			mdname(mddev), PAGE_SIZE);
 		goto out;
 	}
 
 	if (copies < 2 || copies > mddev->raid_disks) {
-		printk(KERN_ERR "md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
-		       mdname(mddev), mddev->new_layout);
+		pr_warn("md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
+			mdname(mddev), mddev->new_layout);
 		goto out;
 	}
 
@@ -3557,9 +3645,6 @@ static struct r10conf *setup_conf(struct mddev *mddev)
 	return conf;
 
  out:
-	if (err == -ENOMEM)
-		printk(KERN_ERR "md/raid10:%s: couldn't allocate memory.\n",
-		       mdname(mddev));
 	if (conf) {
 		mempool_destroy(conf->r10bio_pool);
 		kfree(conf->mirrors);
@@ -3656,7 +3741,7 @@ static int raid10_run(struct mddev *mddev)
 	}
 	/* need to check that every block has at least one working mirror */
 	if (!enough(conf, -1)) {
-		printk(KERN_ERR "md/raid10:%s: not enough operational mirrors.\n",
+		pr_err("md/raid10:%s: not enough operational mirrors.\n",
 		       mdname(mddev));
 		goto out_free_conf;
 	}
@@ -3698,11 +3783,9 @@ static int raid10_run(struct mddev *mddev)
 	}
 
 	if (mddev->recovery_cp != MaxSector)
-		printk(KERN_NOTICE "md/raid10:%s: not clean"
-		       " -- starting background reconstruction\n",
-		       mdname(mddev));
-	printk(KERN_INFO
-		"md/raid10:%s: active with %d out of %d devices\n",
+		pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n",
+			  mdname(mddev));
+	pr_info("md/raid10:%s: active with %d out of %d devices\n",
 		mdname(mddev), conf->geo.raid_disks - mddev->degraded,
 		conf->geo.raid_disks);
 	/*
@@ -3712,6 +3795,7 @@ static int raid10_run(struct mddev *mddev)
 	size = raid10_size(mddev, 0, 0);
 	md_set_array_sectors(mddev, size);
 	mddev->resync_max_sectors = size;
+	set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
 
 	if (mddev->queue) {
 		int stripe = conf->geo.raid_disks *
@@ -3739,7 +3823,7 @@ static int raid10_run(struct mddev *mddev)
 
 		if (max(before_length, after_length) > min_offset_diff) {
 			/* This cannot work */
-			printk("md/raid10: offset difference not enough to continue reshape\n");
+			pr_warn("md/raid10: offset difference not enough to continue reshape\n");
 			goto out_free_conf;
 		}
 		conf->offset_diff = min_offset_diff;
@@ -3846,8 +3930,8 @@ static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs)
 	struct r10conf *conf;
 
 	if (mddev->degraded > 0) {
-		printk(KERN_ERR "md/raid10:%s: Error: degraded raid0!\n",
-		       mdname(mddev));
+		pr_warn("md/raid10:%s: Error: degraded raid0!\n",
+			mdname(mddev));
 		return ERR_PTR(-EINVAL);
 	}
 	sector_div(size, devs);
@@ -3887,9 +3971,8 @@ static void *raid10_takeover(struct mddev *mddev)
 		/* for raid0 takeover only one zone is supported */
 		raid0_conf = mddev->private;
 		if (raid0_conf->nr_strip_zones > 1) {
-			printk(KERN_ERR "md/raid10:%s: cannot takeover raid 0"
-			       " with more than one zone.\n",
-			       mdname(mddev));
+			pr_warn("md/raid10:%s: cannot takeover raid 0 with more than one zone.\n",
+				mdname(mddev));
 			return ERR_PTR(-EINVAL);
 		}
 		return raid10_takeover_raid0(mddev,
@@ -4078,8 +4161,8 @@ static int raid10_start_reshape(struct mddev *mddev)
 		sector_t size = raid10_size(mddev, 0, 0);
 		if (size < mddev->array_sectors) {
 			spin_unlock_irq(&conf->device_lock);
-			printk(KERN_ERR "md/raid10:%s: array size must be reduce before number of disks\n",
-			       mdname(mddev));
+			pr_warn("md/raid10:%s: array size must be reduce before number of disks\n",
+				mdname(mddev));
 			return -EINVAL;
 		}
 		mddev->resync_max_sectors = size;
@@ -4126,7 +4209,7 @@ static int raid10_start_reshape(struct mddev *mddev)
 	spin_unlock_irq(&conf->device_lock);
 	mddev->raid_disks = conf->geo.raid_disks;
 	mddev->reshape_position = conf->reshape_progress;
-	set_bit(MD_CHANGE_DEVS, &mddev->flags);
+	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 
 	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
 	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
@@ -4321,9 +4404,9 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
 		else
 			mddev->curr_resync_completed = conf->reshape_progress;
 		conf->reshape_checkpoint = jiffies;
-		set_bit(MD_CHANGE_DEVS, &mddev->flags);
+		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 		md_wakeup_thread(mddev->thread);
-		wait_event(mddev->sb_wait, mddev->flags == 0 ||
+		wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||
 			   test_bit(MD_RECOVERY_INTR, &mddev->recovery));
 		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
 			allow_barrier(conf);
diff --git a/drivers/md/raid10.h b/drivers/md/raid10.h
index 18ec1f7a98bf..3162615e57bd 100644
--- a/drivers/md/raid10.h
+++ b/drivers/md/raid10.h
@@ -156,5 +156,7 @@ enum r10bio_state {
  * flag is set
  */
 	R10BIO_Previous,
+/* failfast devices did receive failfast requests. */
+	R10BIO_FailFast,
 };
 #endif
diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
index a227a9f3ee65..d7bfb6fc8aef 100644
--- a/drivers/md/raid5-cache.c
+++ b/drivers/md/raid5-cache.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (C) 2015 Shaohua Li <shli@fb.com>
+ * Copyright (C) 2016 Song Liu <songliubraving@fb.com>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -18,8 +19,10 @@
 #include <linux/raid/md_p.h>
 #include <linux/crc32c.h>
 #include <linux/random.h>
+#include <linux/kthread.h>
 #include "md.h"
 #include "raid5.h"
+#include "bitmap.h"
 
 /*
  * metadata/data stored in disk with 4k size unit (a block) regardless
@@ -28,18 +31,70 @@
 #define BLOCK_SECTORS (8)
 
 /*
- * reclaim runs every 1/4 disk size or 10G reclaimable space. This can prevent
- * recovery scans a very long log
+ * log->max_free_space is min(1/4 disk size, 10G reclaimable space).
+ *
+ * In write through mode, the reclaim runs every log->max_free_space.
+ * This can prevent the recovery scans for too long
  */
 #define RECLAIM_MAX_FREE_SPACE (10 * 1024 * 1024 * 2) /* sector */
 #define RECLAIM_MAX_FREE_SPACE_SHIFT (2)
 
+/* wake up reclaim thread periodically */
+#define R5C_RECLAIM_WAKEUP_INTERVAL (30 * HZ)
+/* start flush with these full stripes */
+#define R5C_FULL_STRIPE_FLUSH_BATCH 256
+/* reclaim stripes in groups */
+#define R5C_RECLAIM_STRIPE_GROUP (NR_STRIPE_HASH_LOCKS * 2)
+
 /*
  * We only need 2 bios per I/O unit to make progress, but ensure we
  * have a few more available to not get too tight.
  */
 #define R5L_POOL_SIZE	4
 
+/*
+ * r5c journal modes of the array: write-back or write-through.
+ * write-through mode has identical behavior as existing log only
+ * implementation.
+ */
+enum r5c_journal_mode {
+	R5C_JOURNAL_MODE_WRITE_THROUGH = 0,
+	R5C_JOURNAL_MODE_WRITE_BACK = 1,
+};
+
+static char *r5c_journal_mode_str[] = {"write-through",
+				       "write-back"};
+/*
+ * raid5 cache state machine
+ *
+ * With the RAID cache, each stripe works in two phases:
+ *	- caching phase
+ *	- writing-out phase
+ *
+ * These two phases are controlled by bit STRIPE_R5C_CACHING:
+ *   if STRIPE_R5C_CACHING == 0, the stripe is in writing-out phase
+ *   if STRIPE_R5C_CACHING == 1, the stripe is in caching phase
+ *
+ * When there is no journal, or the journal is in write-through mode,
+ * the stripe is always in writing-out phase.
+ *
+ * For write-back journal, the stripe is sent to caching phase on write
+ * (r5c_try_caching_write). r5c_make_stripe_write_out() kicks off
+ * the write-out phase by clearing STRIPE_R5C_CACHING.
+ *
+ * Stripes in caching phase do not write the raid disks. Instead, all
+ * writes are committed from the log device. Therefore, a stripe in
+ * caching phase handles writes as:
+ *	- write to log device
+ *	- return IO
+ *
+ * Stripes in writing-out phase handle writes as:
+ *	- calculate parity
+ *	- write pending data and parity to journal
+ *	- write data and parity to raid disks
+ *	- return IO for pending writes
+ */
+
 struct r5l_log {
 	struct md_rdev *rdev;
 
@@ -58,7 +113,6 @@ struct r5l_log {
 	u64 seq;			/* log head sequence */
 
 	sector_t next_checkpoint;
-	u64 next_cp_seq;
 
 	struct mutex io_mutex;
 	struct r5l_io_unit *current_io;	/* current io_unit accepting new data */
@@ -96,6 +150,18 @@ struct r5l_log {
 	spinlock_t no_space_stripes_lock;
 
 	bool need_cache_flush;
+
+	/* for r5c_cache */
+	enum r5c_journal_mode r5c_journal_mode;
+
+	/* all stripes in r5cache, in the order of seq at sh->log_start */
+	struct list_head stripe_in_journal_list;
+
+	spinlock_t stripe_in_journal_lock;
+	atomic_t stripe_in_journal_count;
+
+	/* to submit async io_units, to fulfill ordering of flush */
+	struct work_struct deferred_io_work;
 };
 
 /*
@@ -122,6 +188,18 @@ struct r5l_io_unit {
 
 	int state;
 	bool need_split_bio;
+	struct bio *split_bio;
+
+	unsigned int has_flush:1;      /* include flush request */
+	unsigned int has_fua:1;        /* include fua request */
+	unsigned int has_null_flush:1; /* include empty flush request */
+	/*
+	 * io isn't sent yet, flush/fua request can only be submitted till it's
+	 * the first IO in running_ios list
+	 */
+	unsigned int io_deferred:1;
+
+	struct bio_list flush_barriers;   /* size == 0 flush bios */
 };
 
 /* r5l_io_unit state */
@@ -133,6 +211,12 @@ enum r5l_io_unit_state {
 	IO_UNIT_STRIPE_END = 3,	/* stripes data finished writing to raid */
 };
 
+bool r5c_is_writeback(struct r5l_log *log)
+{
+	return (log != NULL &&
+		log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK);
+}
+
 static sector_t r5l_ring_add(struct r5l_log *log, sector_t start, sector_t inc)
 {
 	start += inc;
@@ -168,12 +252,235 @@ static void __r5l_set_io_unit_state(struct r5l_io_unit *io,
 	io->state = state;
 }
 
+static void
+r5c_return_dev_pending_writes(struct r5conf *conf, struct r5dev *dev,
+			      struct bio_list *return_bi)
+{
+	struct bio *wbi, *wbi2;
+
+	wbi = dev->written;
+	dev->written = NULL;
+	while (wbi && wbi->bi_iter.bi_sector <
+	       dev->sector + STRIPE_SECTORS) {
+		wbi2 = r5_next_bio(wbi, dev->sector);
+		if (!raid5_dec_bi_active_stripes(wbi)) {
+			md_write_end(conf->mddev);
+			bio_list_add(return_bi, wbi);
+		}
+		wbi = wbi2;
+	}
+}
+
+void r5c_handle_cached_data_endio(struct r5conf *conf,
+	  struct stripe_head *sh, int disks, struct bio_list *return_bi)
+{
+	int i;
+
+	for (i = sh->disks; i--; ) {
+		if (sh->dev[i].written) {
+			set_bit(R5_UPTODATE, &sh->dev[i].flags);
+			r5c_return_dev_pending_writes(conf, &sh->dev[i],
+						      return_bi);
+			bitmap_endwrite(conf->mddev->bitmap, sh->sector,
+					STRIPE_SECTORS,
+					!test_bit(STRIPE_DEGRADED, &sh->state),
+					0);
+		}
+	}
+}
+
+/* Check whether we should flush some stripes to free up stripe cache */
+void r5c_check_stripe_cache_usage(struct r5conf *conf)
+{
+	int total_cached;
+
+	if (!r5c_is_writeback(conf->log))
+		return;
+
+	total_cached = atomic_read(&conf->r5c_cached_partial_stripes) +
+		atomic_read(&conf->r5c_cached_full_stripes);
+
+	/*
+	 * The following condition is true for either of the following:
+	 *   - stripe cache pressure high:
+	 *          total_cached > 3/4 min_nr_stripes ||
+	 *          empty_inactive_list_nr > 0
+	 *   - stripe cache pressure moderate:
+	 *          total_cached > 1/2 min_nr_stripes
+	 */
+	if (total_cached > conf->min_nr_stripes * 1 / 2 ||
+	    atomic_read(&conf->empty_inactive_list_nr) > 0)
+		r5l_wake_reclaim(conf->log, 0);
+}
+
+/*
+ * flush cache when there are R5C_FULL_STRIPE_FLUSH_BATCH or more full
+ * stripes in the cache
+ */
+void r5c_check_cached_full_stripe(struct r5conf *conf)
+{
+	if (!r5c_is_writeback(conf->log))
+		return;
+
+	/*
+	 * wake up reclaim for R5C_FULL_STRIPE_FLUSH_BATCH cached stripes
+	 * or a full stripe (chunk size / 4k stripes).
+	 */
+	if (atomic_read(&conf->r5c_cached_full_stripes) >=
+	    min(R5C_FULL_STRIPE_FLUSH_BATCH,
+		conf->chunk_sectors >> STRIPE_SHIFT))
+		r5l_wake_reclaim(conf->log, 0);
+}
+
+/*
+ * Total log space (in sectors) needed to flush all data in cache
+ *
+ * Currently, writing-out phase automatically includes all pending writes
+ * to the same sector. So the reclaim of each stripe takes up to
+ * (conf->raid_disks + 1) pages of log space.
+ *
+ * To totally avoid deadlock due to log space, the code reserves
+ * (conf->raid_disks + 1) pages for each stripe in cache, which is not
+ * necessary in most cases.
+ *
+ * To improve this, we will need writing-out phase to be able to NOT include
+ * pending writes, which will reduce the requirement to
+ * (conf->max_degraded + 1) pages per stripe in cache.
+ */
+static sector_t r5c_log_required_to_flush_cache(struct r5conf *conf)
+{
+	struct r5l_log *log = conf->log;
+
+	if (!r5c_is_writeback(log))
+		return 0;
+
+	return BLOCK_SECTORS * (conf->raid_disks + 1) *
+		atomic_read(&log->stripe_in_journal_count);
+}
+
+/*
+ * evaluate log space usage and update R5C_LOG_TIGHT and R5C_LOG_CRITICAL
+ *
+ * R5C_LOG_TIGHT is set when free space on the log device is less than 3x of
+ * reclaim_required_space. R5C_LOG_CRITICAL is set when free space on the log
+ * device is less than 2x of reclaim_required_space.
+ */
+static inline void r5c_update_log_state(struct r5l_log *log)
+{
+	struct r5conf *conf = log->rdev->mddev->private;
+	sector_t free_space;
+	sector_t reclaim_space;
+	bool wake_reclaim = false;
+
+	if (!r5c_is_writeback(log))
+		return;
+
+	free_space = r5l_ring_distance(log, log->log_start,
+				       log->last_checkpoint);
+	reclaim_space = r5c_log_required_to_flush_cache(conf);
+	if (free_space < 2 * reclaim_space)
+		set_bit(R5C_LOG_CRITICAL, &conf->cache_state);
+	else {
+		if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state))
+			wake_reclaim = true;
+		clear_bit(R5C_LOG_CRITICAL, &conf->cache_state);
+	}
+	if (free_space < 3 * reclaim_space)
+		set_bit(R5C_LOG_TIGHT, &conf->cache_state);
+	else
+		clear_bit(R5C_LOG_TIGHT, &conf->cache_state);
+
+	if (wake_reclaim)
+		r5l_wake_reclaim(log, 0);
+}
+
+/*
+ * Put the stripe into writing-out phase by clearing STRIPE_R5C_CACHING.
+ * This function should only be called in write-back mode.
+ */
+void r5c_make_stripe_write_out(struct stripe_head *sh)
+{
+	struct r5conf *conf = sh->raid_conf;
+	struct r5l_log *log = conf->log;
+
+	BUG_ON(!r5c_is_writeback(log));
+
+	WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
+	clear_bit(STRIPE_R5C_CACHING, &sh->state);
+
+	if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
+		atomic_inc(&conf->preread_active_stripes);
+
+	if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) {
+		BUG_ON(atomic_read(&conf->r5c_cached_partial_stripes) == 0);
+		atomic_dec(&conf->r5c_cached_partial_stripes);
+	}
+
+	if (test_and_clear_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
+		BUG_ON(atomic_read(&conf->r5c_cached_full_stripes) == 0);
+		atomic_dec(&conf->r5c_cached_full_stripes);
+	}
+}
+
+static void r5c_handle_data_cached(struct stripe_head *sh)
+{
+	int i;
+
+	for (i = sh->disks; i--; )
+		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
+			set_bit(R5_InJournal, &sh->dev[i].flags);
+			clear_bit(R5_LOCKED, &sh->dev[i].flags);
+		}
+	clear_bit(STRIPE_LOG_TRAPPED, &sh->state);
+}
+
+/*
+ * this journal write must contain full parity,
+ * it may also contain some data pages
+ */
+static void r5c_handle_parity_cached(struct stripe_head *sh)
+{
+	int i;
+
+	for (i = sh->disks; i--; )
+		if (test_bit(R5_InJournal, &sh->dev[i].flags))
+			set_bit(R5_Wantwrite, &sh->dev[i].flags);
+}
+
+/*
+ * Setting proper flags after writing (or flushing) data and/or parity to the
+ * log device. This is called from r5l_log_endio() or r5l_log_flush_endio().
+ */
+static void r5c_finish_cache_stripe(struct stripe_head *sh)
+{
+	struct r5l_log *log = sh->raid_conf->log;
+
+	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
+		BUG_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
+		/*
+		 * Set R5_InJournal for parity dev[pd_idx]. This means
+		 * all data AND parity in the journal. For RAID 6, it is
+		 * NOT necessary to set the flag for dev[qd_idx], as the
+		 * two parities are written out together.
+		 */
+		set_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
+	} else if (test_bit(STRIPE_R5C_CACHING, &sh->state)) {
+		r5c_handle_data_cached(sh);
+	} else {
+		r5c_handle_parity_cached(sh);
+		set_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
+	}
+}
+
 static void r5l_io_run_stripes(struct r5l_io_unit *io)
 {
 	struct stripe_head *sh, *next;
 
 	list_for_each_entry_safe(sh, next, &io->stripe_list, log_list) {
 		list_del_init(&sh->log_list);
+
+		r5c_finish_cache_stripe(sh);
+
 		set_bit(STRIPE_HANDLE, &sh->state);
 		raid5_release_stripe(sh);
 	}
@@ -209,9 +516,11 @@ static void r5l_move_to_end_ios(struct r5l_log *log)
 	}
 }
 
+static void __r5l_stripe_write_finished(struct r5l_io_unit *io);
 static void r5l_log_endio(struct bio *bio)
 {
 	struct r5l_io_unit *io = bio->bi_private;
+	struct r5l_io_unit *io_deferred;
 	struct r5l_log *log = io->log;
 	unsigned long flags;
 
@@ -227,18 +536,89 @@ static void r5l_log_endio(struct bio *bio)
 		r5l_move_to_end_ios(log);
 	else
 		r5l_log_run_stripes(log);
+	if (!list_empty(&log->running_ios)) {
+		/*
+		 * FLUSH/FUA io_unit is deferred because of ordering, now we
+		 * can dispatch it
+		 */
+		io_deferred = list_first_entry(&log->running_ios,
+					       struct r5l_io_unit, log_sibling);
+		if (io_deferred->io_deferred)
+			schedule_work(&log->deferred_io_work);
+	}
+
 	spin_unlock_irqrestore(&log->io_list_lock, flags);
 
 	if (log->need_cache_flush)
 		md_wakeup_thread(log->rdev->mddev->thread);
+
+	if (io->has_null_flush) {
+		struct bio *bi;
+
+		WARN_ON(bio_list_empty(&io->flush_barriers));
+		while ((bi = bio_list_pop(&io->flush_barriers)) != NULL) {
+			bio_endio(bi);
+			atomic_dec(&io->pending_stripe);
+		}
+		if (atomic_read(&io->pending_stripe) == 0)
+			__r5l_stripe_write_finished(io);
+	}
+}
+
+static void r5l_do_submit_io(struct r5l_log *log, struct r5l_io_unit *io)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&log->io_list_lock, flags);
+	__r5l_set_io_unit_state(io, IO_UNIT_IO_START);
+	spin_unlock_irqrestore(&log->io_list_lock, flags);
+
+	if (io->has_flush)
+		io->current_bio->bi_opf |= REQ_PREFLUSH;
+	if (io->has_fua)
+		io->current_bio->bi_opf |= REQ_FUA;
+	submit_bio(io->current_bio);
+
+	if (!io->split_bio)
+		return;
+
+	if (io->has_flush)
+		io->split_bio->bi_opf |= REQ_PREFLUSH;
+	if (io->has_fua)
+		io->split_bio->bi_opf |= REQ_FUA;
+	submit_bio(io->split_bio);
+}
+
+/* deferred io_unit will be dispatched here */
+static void r5l_submit_io_async(struct work_struct *work)
+{
+	struct r5l_log *log = container_of(work, struct r5l_log,
+					   deferred_io_work);
+	struct r5l_io_unit *io = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&log->io_list_lock, flags);
+	if (!list_empty(&log->running_ios)) {
+		io = list_first_entry(&log->running_ios, struct r5l_io_unit,
+				      log_sibling);
+		if (!io->io_deferred)
+			io = NULL;
+		else
+			io->io_deferred = 0;
+	}
+	spin_unlock_irqrestore(&log->io_list_lock, flags);
+	if (io)
+		r5l_do_submit_io(log, io);
 }
 
 static void r5l_submit_current_io(struct r5l_log *log)
 {
 	struct r5l_io_unit *io = log->current_io;
+	struct bio *bio;
 	struct r5l_meta_block *block;
 	unsigned long flags;
 	u32 crc;
+	bool do_submit = true;
 
 	if (!io)
 		return;
@@ -247,13 +627,20 @@ static void r5l_submit_current_io(struct r5l_log *log)
 	block->meta_size = cpu_to_le32(io->meta_offset);
 	crc = crc32c_le(log->uuid_checksum, block, PAGE_SIZE);
 	block->checksum = cpu_to_le32(crc);
+	bio = io->current_bio;
 
 	log->current_io = NULL;
 	spin_lock_irqsave(&log->io_list_lock, flags);
-	__r5l_set_io_unit_state(io, IO_UNIT_IO_START);
+	if (io->has_flush || io->has_fua) {
+		if (io != list_first_entry(&log->running_ios,
+					   struct r5l_io_unit, log_sibling)) {
+			io->io_deferred = 1;
+			do_submit = false;
+		}
+	}
 	spin_unlock_irqrestore(&log->io_list_lock, flags);
-
-	submit_bio(io->current_bio);
+	if (do_submit)
+		r5l_do_submit_io(log, io);
 }
 
 static struct bio *r5l_bio_alloc(struct r5l_log *log)
@@ -271,6 +658,7 @@ static void r5_reserve_log_entry(struct r5l_log *log, struct r5l_io_unit *io)
 {
 	log->log_start = r5l_ring_add(log, log->log_start, BLOCK_SECTORS);
 
+	r5c_update_log_state(log);
 	/*
 	 * If we filled up the log device start from the beginning again,
 	 * which will require a new bio.
@@ -297,6 +685,7 @@ static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
 	io->log = log;
 	INIT_LIST_HEAD(&io->log_sibling);
 	INIT_LIST_HEAD(&io->stripe_list);
+	bio_list_init(&io->flush_barriers);
 	io->state = IO_UNIT_RUNNING;
 
 	io->meta_page = mempool_alloc(log->meta_pool, GFP_NOIO);
@@ -367,12 +756,11 @@ static void r5l_append_payload_page(struct r5l_log *log, struct page *page)
 	struct r5l_io_unit *io = log->current_io;
 
 	if (io->need_split_bio) {
-		struct bio *prev = io->current_bio;
-
+		BUG_ON(io->split_bio);
+		io->split_bio = io->current_bio;
 		io->current_bio = r5l_bio_alloc(log);
-		bio_chain(io->current_bio, prev);
-
-		submit_bio(prev);
+		bio_chain(io->current_bio, io->split_bio);
+		io->need_split_bio = false;
 	}
 
 	if (!bio_add_page(io->current_bio, page, PAGE_SIZE, 0))
@@ -401,50 +789,85 @@ static int r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
 
 	io = log->current_io;
 
+	if (test_and_clear_bit(STRIPE_R5C_PREFLUSH, &sh->state))
+		io->has_flush = 1;
+
 	for (i = 0; i < sh->disks; i++) {
-		if (!test_bit(R5_Wantwrite, &sh->dev[i].flags))
+		if (!test_bit(R5_Wantwrite, &sh->dev[i].flags) ||
+		    test_bit(R5_InJournal, &sh->dev[i].flags))
 			continue;
 		if (i == sh->pd_idx || i == sh->qd_idx)
 			continue;
+		if (test_bit(R5_WantFUA, &sh->dev[i].flags) &&
+		    log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK) {
+			io->has_fua = 1;
+			/*
+			 * we need to flush journal to make sure recovery can
+			 * reach the data with fua flag
+			 */
+			io->has_flush = 1;
+		}
 		r5l_append_payload_meta(log, R5LOG_PAYLOAD_DATA,
 					raid5_compute_blocknr(sh, i, 0),
 					sh->dev[i].log_checksum, 0, false);
 		r5l_append_payload_page(log, sh->dev[i].page);
 	}
 
-	if (sh->qd_idx >= 0) {
+	if (parity_pages == 2) {
 		r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
 					sh->sector, sh->dev[sh->pd_idx].log_checksum,
 					sh->dev[sh->qd_idx].log_checksum, true);
 		r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
 		r5l_append_payload_page(log, sh->dev[sh->qd_idx].page);
-	} else {
+	} else if (parity_pages == 1) {
 		r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
 					sh->sector, sh->dev[sh->pd_idx].log_checksum,
 					0, false);
 		r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
-	}
+	} else  /* Just writing data, not parity, in caching phase */
+		BUG_ON(parity_pages != 0);
 
 	list_add_tail(&sh->log_list, &io->stripe_list);
 	atomic_inc(&io->pending_stripe);
 	sh->log_io = io;
 
+	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
+		return 0;
+
+	if (sh->log_start == MaxSector) {
+		BUG_ON(!list_empty(&sh->r5c));
+		sh->log_start = io->log_start;
+		spin_lock_irq(&log->stripe_in_journal_lock);
+		list_add_tail(&sh->r5c,
+			      &log->stripe_in_journal_list);
+		spin_unlock_irq(&log->stripe_in_journal_lock);
+		atomic_inc(&log->stripe_in_journal_count);
+	}
 	return 0;
 }
 
-static void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
+/* add stripe to no_space_stripes, and then wake up reclaim */
+static inline void r5l_add_no_space_stripe(struct r5l_log *log,
+					   struct stripe_head *sh)
+{
+	spin_lock(&log->no_space_stripes_lock);
+	list_add_tail(&sh->log_list, &log->no_space_stripes);
+	spin_unlock(&log->no_space_stripes_lock);
+}
+
 /*
  * running in raid5d, where reclaim could wait for raid5d too (when it flushes
  * data from log to raid disks), so we shouldn't wait for reclaim here
  */
 int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
 {
+	struct r5conf *conf = sh->raid_conf;
 	int write_disks = 0;
 	int data_pages, parity_pages;
-	int meta_size;
 	int reserve;
 	int i;
 	int ret = 0;
+	bool wake_reclaim = false;
 
 	if (!log)
 		return -EAGAIN;
@@ -456,11 +879,15 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
 		return -EAGAIN;
 	}
 
+	WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
+
 	for (i = 0; i < sh->disks; i++) {
 		void *addr;
 
-		if (!test_bit(R5_Wantwrite, &sh->dev[i].flags))
+		if (!test_bit(R5_Wantwrite, &sh->dev[i].flags) ||
+		    test_bit(R5_InJournal, &sh->dev[i].flags))
 			continue;
+
 		write_disks++;
 		/* checksum is already calculated in last run */
 		if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
@@ -473,15 +900,6 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
 	parity_pages = 1 + !!(sh->qd_idx >= 0);
 	data_pages = write_disks - parity_pages;
 
-	meta_size =
-		((sizeof(struct r5l_payload_data_parity) + sizeof(__le32))
-		 * data_pages) +
-		sizeof(struct r5l_payload_data_parity) +
-		sizeof(__le32) * parity_pages;
-	/* Doesn't work with very big raid array */
-	if (meta_size + sizeof(struct r5l_meta_block) > PAGE_SIZE)
-		return -EINVAL;
-
 	set_bit(STRIPE_LOG_TRAPPED, &sh->state);
 	/*
 	 * The stripe must enter state machine again to finish the write, so
@@ -493,22 +911,49 @@ int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
 	mutex_lock(&log->io_mutex);
 	/* meta + data */
 	reserve = (1 + write_disks) << (PAGE_SHIFT - 9);
-	if (!r5l_has_free_space(log, reserve)) {
-		spin_lock(&log->no_space_stripes_lock);
-		list_add_tail(&sh->log_list, &log->no_space_stripes);
-		spin_unlock(&log->no_space_stripes_lock);
 
-		r5l_wake_reclaim(log, reserve);
-	} else {
-		ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
-		if (ret) {
-			spin_lock_irq(&log->io_list_lock);
-			list_add_tail(&sh->log_list, &log->no_mem_stripes);
-			spin_unlock_irq(&log->io_list_lock);
+	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
+		if (!r5l_has_free_space(log, reserve)) {
+			r5l_add_no_space_stripe(log, sh);
+			wake_reclaim = true;
+		} else {
+			ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
+			if (ret) {
+				spin_lock_irq(&log->io_list_lock);
+				list_add_tail(&sh->log_list,
+					      &log->no_mem_stripes);
+				spin_unlock_irq(&log->io_list_lock);
+			}
+		}
+	} else {  /* R5C_JOURNAL_MODE_WRITE_BACK */
+		/*
+		 * log space critical, do not process stripes that are
+		 * not in cache yet (sh->log_start == MaxSector).
+		 */
+		if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
+		    sh->log_start == MaxSector) {
+			r5l_add_no_space_stripe(log, sh);
+			wake_reclaim = true;
+			reserve = 0;
+		} else if (!r5l_has_free_space(log, reserve)) {
+			if (sh->log_start == log->last_checkpoint)
+				BUG();
+			else
+				r5l_add_no_space_stripe(log, sh);
+		} else {
+			ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
+			if (ret) {
+				spin_lock_irq(&log->io_list_lock);
+				list_add_tail(&sh->log_list,
+					      &log->no_mem_stripes);
+				spin_unlock_irq(&log->io_list_lock);
+			}
 		}
 	}
 
 	mutex_unlock(&log->io_mutex);
+	if (wake_reclaim)
+		r5l_wake_reclaim(log, reserve);
 	return 0;
 }
 
@@ -525,17 +970,34 @@ int r5l_handle_flush_request(struct r5l_log *log, struct bio *bio)
 {
 	if (!log)
 		return -ENODEV;
-	/*
-	 * we flush log disk cache first, then write stripe data to raid disks.
-	 * So if bio is finished, the log disk cache is flushed already. The
-	 * recovery guarantees we can recovery the bio from log disk, so we
-	 * don't need to flush again
-	 */
-	if (bio->bi_iter.bi_size == 0) {
-		bio_endio(bio);
-		return 0;
+
+	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
+		/*
+		 * in write through (journal only)
+		 * we flush log disk cache first, then write stripe data to
+		 * raid disks. So if bio is finished, the log disk cache is
+		 * flushed already. The recovery guarantees we can recovery
+		 * the bio from log disk, so we don't need to flush again
+		 */
+		if (bio->bi_iter.bi_size == 0) {
+			bio_endio(bio);
+			return 0;
+		}
+		bio->bi_opf &= ~REQ_PREFLUSH;
+	} else {
+		/* write back (with cache) */
+		if (bio->bi_iter.bi_size == 0) {
+			mutex_lock(&log->io_mutex);
+			r5l_get_meta(log, 0);
+			bio_list_add(&log->current_io->flush_barriers, bio);
+			log->current_io->has_flush = 1;
+			log->current_io->has_null_flush = 1;
+			atomic_inc(&log->current_io->pending_stripe);
+			r5l_submit_current_io(log);
+			mutex_unlock(&log->io_mutex);
+			return 0;
+		}
 	}
-	bio->bi_opf &= ~REQ_PREFLUSH;
 	return -EAGAIN;
 }
 
@@ -555,10 +1017,40 @@ static void r5l_run_no_space_stripes(struct r5l_log *log)
 	spin_unlock(&log->no_space_stripes_lock);
 }
 
+/*
+ * calculate new last_checkpoint
+ * for write through mode, returns log->next_checkpoint
+ * for write back, returns log_start of first sh in stripe_in_journal_list
+ */
+static sector_t r5c_calculate_new_cp(struct r5conf *conf)
+{
+	struct stripe_head *sh;
+	struct r5l_log *log = conf->log;
+	sector_t new_cp;
+	unsigned long flags;
+
+	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
+		return log->next_checkpoint;
+
+	spin_lock_irqsave(&log->stripe_in_journal_lock, flags);
+	if (list_empty(&conf->log->stripe_in_journal_list)) {
+		/* all stripes flushed */
+		spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
+		return log->next_checkpoint;
+	}
+	sh = list_first_entry(&conf->log->stripe_in_journal_list,
+			      struct stripe_head, r5c);
+	new_cp = sh->log_start;
+	spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
+	return new_cp;
+}
+
 static sector_t r5l_reclaimable_space(struct r5l_log *log)
 {
+	struct r5conf *conf = log->rdev->mddev->private;
+
 	return r5l_ring_distance(log, log->last_checkpoint,
-				 log->next_checkpoint);
+				 r5c_calculate_new_cp(conf));
 }
 
 static void r5l_run_no_mem_stripe(struct r5l_log *log)
@@ -589,7 +1081,6 @@ static bool r5l_complete_finished_ios(struct r5l_log *log)
 			break;
 
 		log->next_checkpoint = io->log_start;
-		log->next_cp_seq = io->seq;
 
 		list_del(&io->log_sibling);
 		mempool_free(io, log->io_pool);
@@ -604,6 +1095,7 @@ static bool r5l_complete_finished_ios(struct r5l_log *log)
 static void __r5l_stripe_write_finished(struct r5l_io_unit *io)
 {
 	struct r5l_log *log = io->log;
+	struct r5conf *conf = log->rdev->mddev->private;
 	unsigned long flags;
 
 	spin_lock_irqsave(&log->io_list_lock, flags);
@@ -614,7 +1106,8 @@ static void __r5l_stripe_write_finished(struct r5l_io_unit *io)
 		return;
 	}
 
-	if (r5l_reclaimable_space(log) > log->max_free_space)
+	if (r5l_reclaimable_space(log) > log->max_free_space ||
+	    test_bit(R5C_LOG_TIGHT, &conf->cache_state))
 		r5l_wake_reclaim(log, 0);
 
 	spin_unlock_irqrestore(&log->io_list_lock, flags);
@@ -685,7 +1178,7 @@ void r5l_flush_stripe_to_raid(struct r5l_log *log)
 	bio_reset(&log->flush_bio);
 	log->flush_bio.bi_bdev = log->rdev->bdev;
 	log->flush_bio.bi_end_io = r5l_log_flush_endio;
-	bio_set_op_attrs(&log->flush_bio, REQ_OP_WRITE, WRITE_FLUSH);
+	log->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 	submit_bio(&log->flush_bio);
 }
 
@@ -713,8 +1206,8 @@ static void r5l_write_super_and_discard_space(struct r5l_log *log,
 	 * there is a deadlock. We workaround this issue with a trylock.
 	 * FIXME: we could miss discard if we can't take reconfig mutex
 	 */
-	set_mask_bits(&mddev->flags, 0,
-		BIT(MD_CHANGE_DEVS) | BIT(MD_CHANGE_PENDING));
+	set_mask_bits(&mddev->sb_flags, 0,
+		BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
 	if (!mddev_trylock(mddev))
 		return;
 	md_update_sb(mddev, 1);
@@ -735,15 +1228,148 @@ static void r5l_write_super_and_discard_space(struct r5l_log *log,
 	}
 }
 
+/*
+ * r5c_flush_stripe moves stripe from cached list to handle_list. When called,
+ * the stripe must be on r5c_cached_full_stripes or r5c_cached_partial_stripes.
+ *
+ * must hold conf->device_lock
+ */
+static void r5c_flush_stripe(struct r5conf *conf, struct stripe_head *sh)
+{
+	BUG_ON(list_empty(&sh->lru));
+	BUG_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
+	BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
+
+	/*
+	 * The stripe is not ON_RELEASE_LIST, so it is safe to call
+	 * raid5_release_stripe() while holding conf->device_lock
+	 */
+	BUG_ON(test_bit(STRIPE_ON_RELEASE_LIST, &sh->state));
+	assert_spin_locked(&conf->device_lock);
+
+	list_del_init(&sh->lru);
+	atomic_inc(&sh->count);
+
+	set_bit(STRIPE_HANDLE, &sh->state);
+	atomic_inc(&conf->active_stripes);
+	r5c_make_stripe_write_out(sh);
+
+	raid5_release_stripe(sh);
+}
+
+/*
+ * if num == 0, flush all full stripes
+ * if num > 0, flush all full stripes. If less than num full stripes are
+ *             flushed, flush some partial stripes until totally num stripes are
+ *             flushed or there is no more cached stripes.
+ */
+void r5c_flush_cache(struct r5conf *conf, int num)
+{
+	int count;
+	struct stripe_head *sh, *next;
+
+	assert_spin_locked(&conf->device_lock);
+	if (!conf->log)
+		return;
+
+	count = 0;
+	list_for_each_entry_safe(sh, next, &conf->r5c_full_stripe_list, lru) {
+		r5c_flush_stripe(conf, sh);
+		count++;
+	}
+
+	if (count >= num)
+		return;
+	list_for_each_entry_safe(sh, next,
+				 &conf->r5c_partial_stripe_list, lru) {
+		r5c_flush_stripe(conf, sh);
+		if (++count >= num)
+			break;
+	}
+}
+
+static void r5c_do_reclaim(struct r5conf *conf)
+{
+	struct r5l_log *log = conf->log;
+	struct stripe_head *sh;
+	int count = 0;
+	unsigned long flags;
+	int total_cached;
+	int stripes_to_flush;
+
+	if (!r5c_is_writeback(log))
+		return;
+
+	total_cached = atomic_read(&conf->r5c_cached_partial_stripes) +
+		atomic_read(&conf->r5c_cached_full_stripes);
+
+	if (total_cached > conf->min_nr_stripes * 3 / 4 ||
+	    atomic_read(&conf->empty_inactive_list_nr) > 0)
+		/*
+		 * if stripe cache pressure high, flush all full stripes and
+		 * some partial stripes
+		 */
+		stripes_to_flush = R5C_RECLAIM_STRIPE_GROUP;
+	else if (total_cached > conf->min_nr_stripes * 1 / 2 ||
+		 atomic_read(&conf->r5c_cached_full_stripes) >
+		 R5C_FULL_STRIPE_FLUSH_BATCH)
+		/*
+		 * if stripe cache pressure moderate, or if there is many full
+		 * stripes,flush all full stripes
+		 */
+		stripes_to_flush = 0;
+	else
+		/* no need to flush */
+		stripes_to_flush = -1;
+
+	if (stripes_to_flush >= 0) {
+		spin_lock_irqsave(&conf->device_lock, flags);
+		r5c_flush_cache(conf, stripes_to_flush);
+		spin_unlock_irqrestore(&conf->device_lock, flags);
+	}
+
+	/* if log space is tight, flush stripes on stripe_in_journal_list */
+	if (test_bit(R5C_LOG_TIGHT, &conf->cache_state)) {
+		spin_lock_irqsave(&log->stripe_in_journal_lock, flags);
+		spin_lock(&conf->device_lock);
+		list_for_each_entry(sh, &log->stripe_in_journal_list, r5c) {
+			/*
+			 * stripes on stripe_in_journal_list could be in any
+			 * state of the stripe_cache state machine. In this
+			 * case, we only want to flush stripe on
+			 * r5c_cached_full/partial_stripes. The following
+			 * condition makes sure the stripe is on one of the
+			 * two lists.
+			 */
+			if (!list_empty(&sh->lru) &&
+			    !test_bit(STRIPE_HANDLE, &sh->state) &&
+			    atomic_read(&sh->count) == 0) {
+				r5c_flush_stripe(conf, sh);
+			}
+			if (count++ >= R5C_RECLAIM_STRIPE_GROUP)
+				break;
+		}
+		spin_unlock(&conf->device_lock);
+		spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
+	}
+
+	if (!test_bit(R5C_LOG_CRITICAL, &conf->cache_state))
+		r5l_run_no_space_stripes(log);
+
+	md_wakeup_thread(conf->mddev->thread);
+}
 
 static void r5l_do_reclaim(struct r5l_log *log)
 {
+	struct r5conf *conf = log->rdev->mddev->private;
 	sector_t reclaim_target = xchg(&log->reclaim_target, 0);
 	sector_t reclaimable;
 	sector_t next_checkpoint;
-	u64 next_cp_seq;
+	bool write_super;
 
 	spin_lock_irq(&log->io_list_lock);
+	write_super = r5l_reclaimable_space(log) > log->max_free_space ||
+		reclaim_target != 0 || !list_empty(&log->no_space_stripes);
 	/*
 	 * move proper io_unit to reclaim list. We should not change the order.
 	 * reclaimable/unreclaimable io_unit can be mixed in the list, we
@@ -764,12 +1390,12 @@ static void r5l_do_reclaim(struct r5l_log *log)
 				    log->io_list_lock);
 	}
 
-	next_checkpoint = log->next_checkpoint;
-	next_cp_seq = log->next_cp_seq;
+	next_checkpoint = r5c_calculate_new_cp(conf);
 	spin_unlock_irq(&log->io_list_lock);
 
 	BUG_ON(reclaimable < 0);
-	if (reclaimable == 0)
+
+	if (reclaimable == 0 || !write_super)
 		return;
 
 	/*
@@ -781,7 +1407,7 @@ static void r5l_do_reclaim(struct r5l_log *log)
 
 	mutex_lock(&log->io_mutex);
 	log->last_checkpoint = next_checkpoint;
-	log->last_cp_seq = next_cp_seq;
+	r5c_update_log_state(log);
 	mutex_unlock(&log->io_mutex);
 
 	r5l_run_no_space_stripes(log);
@@ -795,14 +1421,17 @@ static void r5l_reclaim_thread(struct md_thread *thread)
 
 	if (!log)
 		return;
+	r5c_do_reclaim(conf);
 	r5l_do_reclaim(log);
 }
 
-static void r5l_wake_reclaim(struct r5l_log *log, sector_t space)
+void r5l_wake_reclaim(struct r5l_log *log, sector_t space)
 {
 	unsigned long target;
 	unsigned long new = (unsigned long)space; /* overflow in theory */
 
+	if (!log)
+		return;
 	do {
 		target = log->reclaim_target;
 		if (new < target)
@@ -816,22 +1445,14 @@ void r5l_quiesce(struct r5l_log *log, int state)
 	struct mddev *mddev;
 	if (!log || state == 2)
 		return;
-	if (state == 0) {
-		/*
-		 * This is a special case for hotadd. In suspend, the array has
-		 * no journal. In resume, journal is initialized as well as the
-		 * reclaim thread.
-		 */
-		if (log->reclaim_thread)
-			return;
-		log->reclaim_thread = md_register_thread(r5l_reclaim_thread,
-					log->rdev->mddev, "reclaim");
-	} else if (state == 1) {
+	if (state == 0)
+		kthread_unpark(log->reclaim_thread->tsk);
+	else if (state == 1) {
 		/* make sure r5l_write_super_and_discard_space exits */
 		mddev = log->rdev->mddev;
 		wake_up(&mddev->sb_wait);
-		r5l_wake_reclaim(log, -1L);
-		md_unregister_thread(&log->reclaim_thread);
+		kthread_park(log->reclaim_thread->tsk);
+		r5l_wake_reclaim(log, MaxSector);
 		r5l_do_reclaim(log);
 	}
 }
@@ -857,10 +1478,13 @@ struct r5l_recovery_ctx {
 	sector_t meta_total_blocks;	/* total size of current meta and data */
 	sector_t pos;			/* recovery position */
 	u64 seq;			/* recovery position seq */
+	int data_parity_stripes;	/* number of data_parity stripes */
+	int data_only_stripes;		/* number of data_only stripes */
+	struct list_head cached_list;
 };
 
-static int r5l_read_meta_block(struct r5l_log *log,
-			       struct r5l_recovery_ctx *ctx)
+static int r5l_recovery_read_meta_block(struct r5l_log *log,
+					struct r5l_recovery_ctx *ctx)
 {
 	struct page *page = ctx->meta_page;
 	struct r5l_meta_block *mb;
@@ -892,170 +1516,618 @@ static int r5l_read_meta_block(struct r5l_log *log,
 	return 0;
 }
 
-static int r5l_recovery_flush_one_stripe(struct r5l_log *log,
-					 struct r5l_recovery_ctx *ctx,
-					 sector_t stripe_sect,
-					 int *offset, sector_t *log_offset)
+static void
+r5l_recovery_create_empty_meta_block(struct r5l_log *log,
+				     struct page *page,
+				     sector_t pos, u64 seq)
 {
-	struct r5conf *conf = log->rdev->mddev->private;
-	struct stripe_head *sh;
-	struct r5l_payload_data_parity *payload;
-	int disk_index;
+	struct r5l_meta_block *mb;
 
-	sh = raid5_get_active_stripe(conf, stripe_sect, 0, 0, 0);
-	while (1) {
-		payload = page_address(ctx->meta_page) + *offset;
+	mb = page_address(page);
+	clear_page(mb);
+	mb->magic = cpu_to_le32(R5LOG_MAGIC);
+	mb->version = R5LOG_VERSION;
+	mb->meta_size = cpu_to_le32(sizeof(struct r5l_meta_block));
+	mb->seq = cpu_to_le64(seq);
+	mb->position = cpu_to_le64(pos);
+}
 
-		if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
-			raid5_compute_sector(conf,
-					     le64_to_cpu(payload->location), 0,
-					     &disk_index, sh);
+static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos,
+					  u64 seq)
+{
+	struct page *page;
+	struct r5l_meta_block *mb;
 
-			sync_page_io(log->rdev, *log_offset, PAGE_SIZE,
-				     sh->dev[disk_index].page, REQ_OP_READ, 0,
-				     false);
-			sh->dev[disk_index].log_checksum =
-				le32_to_cpu(payload->checksum[0]);
-			set_bit(R5_Wantwrite, &sh->dev[disk_index].flags);
-			ctx->meta_total_blocks += BLOCK_SECTORS;
-		} else {
-			disk_index = sh->pd_idx;
-			sync_page_io(log->rdev, *log_offset, PAGE_SIZE,
-				     sh->dev[disk_index].page, REQ_OP_READ, 0,
-				     false);
-			sh->dev[disk_index].log_checksum =
-				le32_to_cpu(payload->checksum[0]);
-			set_bit(R5_Wantwrite, &sh->dev[disk_index].flags);
-
-			if (sh->qd_idx >= 0) {
-				disk_index = sh->qd_idx;
-				sync_page_io(log->rdev,
-					     r5l_ring_add(log, *log_offset, BLOCK_SECTORS),
-					     PAGE_SIZE, sh->dev[disk_index].page,
-					     REQ_OP_READ, 0, false);
-				sh->dev[disk_index].log_checksum =
-					le32_to_cpu(payload->checksum[1]);
-				set_bit(R5_Wantwrite,
-					&sh->dev[disk_index].flags);
-			}
-			ctx->meta_total_blocks += BLOCK_SECTORS * conf->max_degraded;
-		}
+	page = alloc_page(GFP_KERNEL);
+	if (!page)
+		return -ENOMEM;
+	r5l_recovery_create_empty_meta_block(log, page, pos, seq);
+	mb = page_address(page);
+	mb->checksum = cpu_to_le32(crc32c_le(log->uuid_checksum,
+					     mb, PAGE_SIZE));
+	if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, REQ_OP_WRITE,
+			  REQ_FUA, false)) {
+		__free_page(page);
+		return -EIO;
+	}
+	__free_page(page);
+	return 0;
+}
 
-		*log_offset = r5l_ring_add(log, *log_offset,
-					   le32_to_cpu(payload->size));
-		*offset += sizeof(struct r5l_payload_data_parity) +
-			sizeof(__le32) *
-			(le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
-		if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY)
-			break;
+/*
+ * r5l_recovery_load_data and r5l_recovery_load_parity uses flag R5_Wantwrite
+ * to mark valid (potentially not flushed) data in the journal.
+ *
+ * We already verified checksum in r5l_recovery_verify_data_checksum_for_mb,
+ * so there should not be any mismatch here.
+ */
+static void r5l_recovery_load_data(struct r5l_log *log,
+				   struct stripe_head *sh,
+				   struct r5l_recovery_ctx *ctx,
+				   struct r5l_payload_data_parity *payload,
+				   sector_t log_offset)
+{
+	struct mddev *mddev = log->rdev->mddev;
+	struct r5conf *conf = mddev->private;
+	int dd_idx;
+
+	raid5_compute_sector(conf,
+			     le64_to_cpu(payload->location), 0,
+			     &dd_idx, sh);
+	sync_page_io(log->rdev, log_offset, PAGE_SIZE,
+		     sh->dev[dd_idx].page, REQ_OP_READ, 0, false);
+	sh->dev[dd_idx].log_checksum =
+		le32_to_cpu(payload->checksum[0]);
+	ctx->meta_total_blocks += BLOCK_SECTORS;
+
+	set_bit(R5_Wantwrite, &sh->dev[dd_idx].flags);
+	set_bit(STRIPE_R5C_CACHING, &sh->state);
+}
+
+static void r5l_recovery_load_parity(struct r5l_log *log,
+				     struct stripe_head *sh,
+				     struct r5l_recovery_ctx *ctx,
+				     struct r5l_payload_data_parity *payload,
+				     sector_t log_offset)
+{
+	struct mddev *mddev = log->rdev->mddev;
+	struct r5conf *conf = mddev->private;
+
+	ctx->meta_total_blocks += BLOCK_SECTORS * conf->max_degraded;
+	sync_page_io(log->rdev, log_offset, PAGE_SIZE,
+		     sh->dev[sh->pd_idx].page, REQ_OP_READ, 0, false);
+	sh->dev[sh->pd_idx].log_checksum =
+		le32_to_cpu(payload->checksum[0]);
+	set_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags);
+
+	if (sh->qd_idx >= 0) {
+		sync_page_io(log->rdev,
+			     r5l_ring_add(log, log_offset, BLOCK_SECTORS),
+			     PAGE_SIZE, sh->dev[sh->qd_idx].page,
+			     REQ_OP_READ, 0, false);
+		sh->dev[sh->qd_idx].log_checksum =
+			le32_to_cpu(payload->checksum[1]);
+		set_bit(R5_Wantwrite, &sh->dev[sh->qd_idx].flags);
 	}
+	clear_bit(STRIPE_R5C_CACHING, &sh->state);
+}
 
-	for (disk_index = 0; disk_index < sh->disks; disk_index++) {
-		void *addr;
-		u32 checksum;
+static void r5l_recovery_reset_stripe(struct stripe_head *sh)
+{
+	int i;
 
+	sh->state = 0;
+	sh->log_start = MaxSector;
+	for (i = sh->disks; i--; )
+		sh->dev[i].flags = 0;
+}
+
+static void
+r5l_recovery_replay_one_stripe(struct r5conf *conf,
+			       struct stripe_head *sh,
+			       struct r5l_recovery_ctx *ctx)
+{
+	struct md_rdev *rdev, *rrdev;
+	int disk_index;
+	int data_count = 0;
+
+	for (disk_index = 0; disk_index < sh->disks; disk_index++) {
 		if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
 			continue;
-		addr = kmap_atomic(sh->dev[disk_index].page);
-		checksum = crc32c_le(log->uuid_checksum, addr, PAGE_SIZE);
-		kunmap_atomic(addr);
-		if (checksum != sh->dev[disk_index].log_checksum)
-			goto error;
+		if (disk_index == sh->qd_idx || disk_index == sh->pd_idx)
+			continue;
+		data_count++;
 	}
 
-	for (disk_index = 0; disk_index < sh->disks; disk_index++) {
-		struct md_rdev *rdev, *rrdev;
+	/*
+	 * stripes that only have parity must have been flushed
+	 * before the crash that we are now recovering from, so
+	 * there is nothing more to recovery.
+	 */
+	if (data_count == 0)
+		goto out;
 
-		if (!test_and_clear_bit(R5_Wantwrite,
-					&sh->dev[disk_index].flags))
+	for (disk_index = 0; disk_index < sh->disks; disk_index++) {
+		if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
 			continue;
 
 		/* in case device is broken */
+		rcu_read_lock();
 		rdev = rcu_dereference(conf->disks[disk_index].rdev);
-		if (rdev)
-			sync_page_io(rdev, stripe_sect, PAGE_SIZE,
+		if (rdev) {
+			atomic_inc(&rdev->nr_pending);
+			rcu_read_unlock();
+			sync_page_io(rdev, sh->sector, PAGE_SIZE,
 				     sh->dev[disk_index].page, REQ_OP_WRITE, 0,
 				     false);
+			rdev_dec_pending(rdev, rdev->mddev);
+			rcu_read_lock();
+		}
 		rrdev = rcu_dereference(conf->disks[disk_index].replacement);
-		if (rrdev)
-			sync_page_io(rrdev, stripe_sect, PAGE_SIZE,
+		if (rrdev) {
+			atomic_inc(&rrdev->nr_pending);
+			rcu_read_unlock();
+			sync_page_io(rrdev, sh->sector, PAGE_SIZE,
 				     sh->dev[disk_index].page, REQ_OP_WRITE, 0,
 				     false);
+			rdev_dec_pending(rrdev, rrdev->mddev);
+			rcu_read_lock();
+		}
+		rcu_read_unlock();
 	}
-	raid5_release_stripe(sh);
+	ctx->data_parity_stripes++;
+out:
+	r5l_recovery_reset_stripe(sh);
+}
+
+static struct stripe_head *
+r5c_recovery_alloc_stripe(struct r5conf *conf,
+			  sector_t stripe_sect,
+			  sector_t log_start)
+{
+	struct stripe_head *sh;
+
+	sh = raid5_get_active_stripe(conf, stripe_sect, 0, 1, 0);
+	if (!sh)
+		return NULL;  /* no more stripe available */
+
+	r5l_recovery_reset_stripe(sh);
+	sh->log_start = log_start;
+
+	return sh;
+}
+
+static struct stripe_head *
+r5c_recovery_lookup_stripe(struct list_head *list, sector_t sect)
+{
+	struct stripe_head *sh;
+
+	list_for_each_entry(sh, list, lru)
+		if (sh->sector == sect)
+			return sh;
+	return NULL;
+}
+
+static void
+r5c_recovery_drop_stripes(struct list_head *cached_stripe_list,
+			  struct r5l_recovery_ctx *ctx)
+{
+	struct stripe_head *sh, *next;
+
+	list_for_each_entry_safe(sh, next, cached_stripe_list, lru) {
+		r5l_recovery_reset_stripe(sh);
+		list_del_init(&sh->lru);
+		raid5_release_stripe(sh);
+	}
+}
+
+static void
+r5c_recovery_replay_stripes(struct list_head *cached_stripe_list,
+			    struct r5l_recovery_ctx *ctx)
+{
+	struct stripe_head *sh, *next;
+
+	list_for_each_entry_safe(sh, next, cached_stripe_list, lru)
+		if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
+			r5l_recovery_replay_one_stripe(sh->raid_conf, sh, ctx);
+			list_del_init(&sh->lru);
+			raid5_release_stripe(sh);
+		}
+}
+
+/* if matches return 0; otherwise return -EINVAL */
+static int
+r5l_recovery_verify_data_checksum(struct r5l_log *log, struct page *page,
+				  sector_t log_offset, __le32 log_checksum)
+{
+	void *addr;
+	u32 checksum;
+
+	sync_page_io(log->rdev, log_offset, PAGE_SIZE,
+		     page, REQ_OP_READ, 0, false);
+	addr = kmap_atomic(page);
+	checksum = crc32c_le(log->uuid_checksum, addr, PAGE_SIZE);
+	kunmap_atomic(addr);
+	return (le32_to_cpu(log_checksum) == checksum) ? 0 : -EINVAL;
+}
+
+/*
+ * before loading data to stripe cache, we need verify checksum for all data,
+ * if there is mismatch for any data page, we drop all data in the mata block
+ */
+static int
+r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log,
+					 struct r5l_recovery_ctx *ctx)
+{
+	struct mddev *mddev = log->rdev->mddev;
+	struct r5conf *conf = mddev->private;
+	struct r5l_meta_block *mb = page_address(ctx->meta_page);
+	sector_t mb_offset = sizeof(struct r5l_meta_block);
+	sector_t log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
+	struct page *page;
+	struct r5l_payload_data_parity *payload;
+
+	page = alloc_page(GFP_KERNEL);
+	if (!page)
+		return -ENOMEM;
+
+	while (mb_offset < le32_to_cpu(mb->meta_size)) {
+		payload = (void *)mb + mb_offset;
+
+		if (payload->header.type == R5LOG_PAYLOAD_DATA) {
+			if (r5l_recovery_verify_data_checksum(
+				    log, page, log_offset,
+				    payload->checksum[0]) < 0)
+				goto mismatch;
+		} else if (payload->header.type == R5LOG_PAYLOAD_PARITY) {
+			if (r5l_recovery_verify_data_checksum(
+				    log, page, log_offset,
+				    payload->checksum[0]) < 0)
+				goto mismatch;
+			if (conf->max_degraded == 2 && /* q for RAID 6 */
+			    r5l_recovery_verify_data_checksum(
+				    log, page,
+				    r5l_ring_add(log, log_offset,
+						 BLOCK_SECTORS),
+				    payload->checksum[1]) < 0)
+				goto mismatch;
+		} else /* not R5LOG_PAYLOAD_DATA or R5LOG_PAYLOAD_PARITY */
+			goto mismatch;
+
+		log_offset = r5l_ring_add(log, log_offset,
+					  le32_to_cpu(payload->size));
+
+		mb_offset += sizeof(struct r5l_payload_data_parity) +
+			sizeof(__le32) *
+			(le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
+	}
+
+	put_page(page);
 	return 0;
 
-error:
-	for (disk_index = 0; disk_index < sh->disks; disk_index++)
-		sh->dev[disk_index].flags = 0;
-	raid5_release_stripe(sh);
+mismatch:
+	put_page(page);
 	return -EINVAL;
 }
 
-static int r5l_recovery_flush_one_meta(struct r5l_log *log,
-				       struct r5l_recovery_ctx *ctx)
+/*
+ * Analyze all data/parity pages in one meta block
+ * Returns:
+ * 0 for success
+ * -EINVAL for unknown playload type
+ * -EAGAIN for checksum mismatch of data page
+ * -ENOMEM for run out of memory (alloc_page failed or run out of stripes)
+ */
+static int
+r5c_recovery_analyze_meta_block(struct r5l_log *log,
+				struct r5l_recovery_ctx *ctx,
+				struct list_head *cached_stripe_list)
 {
-	struct r5conf *conf = log->rdev->mddev->private;
-	struct r5l_payload_data_parity *payload;
+	struct mddev *mddev = log->rdev->mddev;
+	struct r5conf *conf = mddev->private;
 	struct r5l_meta_block *mb;
-	int offset;
+	struct r5l_payload_data_parity *payload;
+	int mb_offset;
 	sector_t log_offset;
-	sector_t stripe_sector;
+	sector_t stripe_sect;
+	struct stripe_head *sh;
+	int ret;
+
+	/*
+	 * for mismatch in data blocks, we will drop all data in this mb, but
+	 * we will still read next mb for other data with FLUSH flag, as
+	 * io_unit could finish out of order.
+	 */
+	ret = r5l_recovery_verify_data_checksum_for_mb(log, ctx);
+	if (ret == -EINVAL)
+		return -EAGAIN;
+	else if (ret)
+		return ret;   /* -ENOMEM duo to alloc_page() failed */
 
 	mb = page_address(ctx->meta_page);
-	offset = sizeof(struct r5l_meta_block);
+	mb_offset = sizeof(struct r5l_meta_block);
 	log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
 
-	while (offset < le32_to_cpu(mb->meta_size)) {
+	while (mb_offset < le32_to_cpu(mb->meta_size)) {
 		int dd;
 
-		payload = (void *)mb + offset;
-		stripe_sector = raid5_compute_sector(conf,
-						     le64_to_cpu(payload->location), 0, &dd, NULL);
-		if (r5l_recovery_flush_one_stripe(log, ctx, stripe_sector,
-						  &offset, &log_offset))
+		payload = (void *)mb + mb_offset;
+		stripe_sect = (payload->header.type == R5LOG_PAYLOAD_DATA) ?
+			raid5_compute_sector(
+				conf, le64_to_cpu(payload->location), 0, &dd,
+				NULL)
+			: le64_to_cpu(payload->location);
+
+		sh = r5c_recovery_lookup_stripe(cached_stripe_list,
+						stripe_sect);
+
+		if (!sh) {
+			sh = r5c_recovery_alloc_stripe(conf, stripe_sect, ctx->pos);
+			/*
+			 * cannot get stripe from raid5_get_active_stripe
+			 * try replay some stripes
+			 */
+			if (!sh) {
+				r5c_recovery_replay_stripes(
+					cached_stripe_list, ctx);
+				sh = r5c_recovery_alloc_stripe(
+					conf, stripe_sect, ctx->pos);
+			}
+			if (!sh) {
+				pr_debug("md/raid:%s: Increasing stripe cache size to %d to recovery data on journal.\n",
+					mdname(mddev),
+					conf->min_nr_stripes * 2);
+				raid5_set_cache_size(mddev,
+						     conf->min_nr_stripes * 2);
+				sh = r5c_recovery_alloc_stripe(
+					conf, stripe_sect, ctx->pos);
+			}
+			if (!sh) {
+				pr_err("md/raid:%s: Cannot get enough stripes due to memory pressure. Recovery failed.\n",
+				       mdname(mddev));
+				return -ENOMEM;
+			}
+			list_add_tail(&sh->lru, cached_stripe_list);
+		}
+
+		if (payload->header.type == R5LOG_PAYLOAD_DATA) {
+			if (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
+			    test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags)) {
+				r5l_recovery_replay_one_stripe(conf, sh, ctx);
+				sh->log_start = ctx->pos;
+				list_move_tail(&sh->lru, cached_stripe_list);
+			}
+			r5l_recovery_load_data(log, sh, ctx, payload,
+					       log_offset);
+		} else if (payload->header.type == R5LOG_PAYLOAD_PARITY)
+			r5l_recovery_load_parity(log, sh, ctx, payload,
+						 log_offset);
+		else
 			return -EINVAL;
+
+		log_offset = r5l_ring_add(log, log_offset,
+					  le32_to_cpu(payload->size));
+
+		mb_offset += sizeof(struct r5l_payload_data_parity) +
+			sizeof(__le32) *
+			(le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
 	}
+
 	return 0;
 }
 
-/* copy data/parity from log to raid disks */
-static void r5l_recovery_flush_log(struct r5l_log *log,
-				   struct r5l_recovery_ctx *ctx)
+/*
+ * Load the stripe into cache. The stripe will be written out later by
+ * the stripe cache state machine.
+ */
+static void r5c_recovery_load_one_stripe(struct r5l_log *log,
+					 struct stripe_head *sh)
 {
+	struct r5dev *dev;
+	int i;
+
+	for (i = sh->disks; i--; ) {
+		dev = sh->dev + i;
+		if (test_and_clear_bit(R5_Wantwrite, &dev->flags)) {
+			set_bit(R5_InJournal, &dev->flags);
+			set_bit(R5_UPTODATE, &dev->flags);
+		}
+	}
+	list_add_tail(&sh->r5c, &log->stripe_in_journal_list);
+	atomic_inc(&log->stripe_in_journal_count);
+}
+
+/*
+ * Scan through the log for all to-be-flushed data
+ *
+ * For stripes with data and parity, namely Data-Parity stripe
+ * (STRIPE_R5C_CACHING == 0), we simply replay all the writes.
+ *
+ * For stripes with only data, namely Data-Only stripe
+ * (STRIPE_R5C_CACHING == 1), we load them to stripe cache state machine.
+ *
+ * For a stripe, if we see data after parity, we should discard all previous
+ * data and parity for this stripe, as these data are already flushed to
+ * the array.
+ *
+ * At the end of the scan, we return the new journal_tail, which points to
+ * first data-only stripe on the journal device, or next invalid meta block.
+ */
+static int r5c_recovery_flush_log(struct r5l_log *log,
+				  struct r5l_recovery_ctx *ctx)
+{
+	struct stripe_head *sh;
+	int ret = 0;
+
+	/* scan through the log */
 	while (1) {
-		if (r5l_read_meta_block(log, ctx))
-			return;
-		if (r5l_recovery_flush_one_meta(log, ctx))
-			return;
+		if (r5l_recovery_read_meta_block(log, ctx))
+			break;
+
+		ret = r5c_recovery_analyze_meta_block(log, ctx,
+						      &ctx->cached_list);
+		/*
+		 * -EAGAIN means mismatch in data block, in this case, we still
+		 * try scan the next metablock
+		 */
+		if (ret && ret != -EAGAIN)
+			break;   /* ret == -EINVAL or -ENOMEM */
 		ctx->seq++;
 		ctx->pos = r5l_ring_add(log, ctx->pos, ctx->meta_total_blocks);
 	}
+
+	if (ret == -ENOMEM) {
+		r5c_recovery_drop_stripes(&ctx->cached_list, ctx);
+		return ret;
+	}
+
+	/* replay data-parity stripes */
+	r5c_recovery_replay_stripes(&ctx->cached_list, ctx);
+
+	/* load data-only stripes to stripe cache */
+	list_for_each_entry(sh, &ctx->cached_list, lru) {
+		WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
+		r5c_recovery_load_one_stripe(log, sh);
+		ctx->data_only_stripes++;
+	}
+
+	return 0;
 }
 
-static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos,
-					  u64 seq)
+/*
+ * we did a recovery. Now ctx.pos points to an invalid meta block. New
+ * log will start here. but we can't let superblock point to last valid
+ * meta block. The log might looks like:
+ * | meta 1| meta 2| meta 3|
+ * meta 1 is valid, meta 2 is invalid. meta 3 could be valid. If
+ * superblock points to meta 1, we write a new valid meta 2n.  if crash
+ * happens again, new recovery will start from meta 1. Since meta 2n is
+ * valid now, recovery will think meta 3 is valid, which is wrong.
+ * The solution is we create a new meta in meta2 with its seq == meta
+ * 1's seq + 10000 and let superblock points to meta2. The same recovery
+ * will not think meta 3 is a valid meta, because its seq doesn't match
+ */
+
+/*
+ * Before recovery, the log looks like the following
+ *
+ *   ---------------------------------------------
+ *   |           valid log        | invalid log  |
+ *   ---------------------------------------------
+ *   ^
+ *   |- log->last_checkpoint
+ *   |- log->last_cp_seq
+ *
+ * Now we scan through the log until we see invalid entry
+ *
+ *   ---------------------------------------------
+ *   |           valid log        | invalid log  |
+ *   ---------------------------------------------
+ *   ^                            ^
+ *   |- log->last_checkpoint      |- ctx->pos
+ *   |- log->last_cp_seq          |- ctx->seq
+ *
+ * From this point, we need to increase seq number by 10 to avoid
+ * confusing next recovery.
+ *
+ *   ---------------------------------------------
+ *   |           valid log        | invalid log  |
+ *   ---------------------------------------------
+ *   ^                              ^
+ *   |- log->last_checkpoint        |- ctx->pos+1
+ *   |- log->last_cp_seq            |- ctx->seq+10001
+ *
+ * However, it is not safe to start the state machine yet, because data only
+ * parities are not yet secured in RAID. To save these data only parities, we
+ * rewrite them from seq+11.
+ *
+ *   -----------------------------------------------------------------
+ *   |           valid log        | data only stripes | invalid log  |
+ *   -----------------------------------------------------------------
+ *   ^                                                ^
+ *   |- log->last_checkpoint                          |- ctx->pos+n
+ *   |- log->last_cp_seq                              |- ctx->seq+10000+n
+ *
+ * If failure happens again during this process, the recovery can safe start
+ * again from log->last_checkpoint.
+ *
+ * Once data only stripes are rewritten to journal, we move log_tail
+ *
+ *   -----------------------------------------------------------------
+ *   |     old log        |    data only stripes    | invalid log  |
+ *   -----------------------------------------------------------------
+ *                        ^                         ^
+ *                        |- log->last_checkpoint   |- ctx->pos+n
+ *                        |- log->last_cp_seq       |- ctx->seq+10000+n
+ *
+ * Then we can safely start the state machine. If failure happens from this
+ * point on, the recovery will start from new log->last_checkpoint.
+ */
+static int
+r5c_recovery_rewrite_data_only_stripes(struct r5l_log *log,
+				       struct r5l_recovery_ctx *ctx)
 {
+	struct stripe_head *sh, *next;
+	struct mddev *mddev = log->rdev->mddev;
 	struct page *page;
-	struct r5l_meta_block *mb;
-	u32 crc;
 
-	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-	if (!page)
+	page = alloc_page(GFP_KERNEL);
+	if (!page) {
+		pr_err("md/raid:%s: cannot allocate memory to rewrite data only stripes\n",
+		       mdname(mddev));
 		return -ENOMEM;
-	mb = page_address(page);
-	mb->magic = cpu_to_le32(R5LOG_MAGIC);
-	mb->version = R5LOG_VERSION;
-	mb->meta_size = cpu_to_le32(sizeof(struct r5l_meta_block));
-	mb->seq = cpu_to_le64(seq);
-	mb->position = cpu_to_le64(pos);
-	crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE);
-	mb->checksum = cpu_to_le32(crc);
+	}
 
-	if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, REQ_OP_WRITE,
-			  WRITE_FUA, false)) {
-		__free_page(page);
-		return -EIO;
+	list_for_each_entry_safe(sh, next, &ctx->cached_list, lru) {
+		struct r5l_meta_block *mb;
+		int i;
+		int offset;
+		sector_t write_pos;
+
+		WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
+		r5l_recovery_create_empty_meta_block(log, page,
+						     ctx->pos, ctx->seq);
+		mb = page_address(page);
+		offset = le32_to_cpu(mb->meta_size);
+		write_pos = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
+
+		for (i = sh->disks; i--; ) {
+			struct r5dev *dev = &sh->dev[i];
+			struct r5l_payload_data_parity *payload;
+			void *addr;
+
+			if (test_bit(R5_InJournal, &dev->flags)) {
+				payload = (void *)mb + offset;
+				payload->header.type = cpu_to_le16(
+					R5LOG_PAYLOAD_DATA);
+				payload->size = BLOCK_SECTORS;
+				payload->location = cpu_to_le64(
+					raid5_compute_blocknr(sh, i, 0));
+				addr = kmap_atomic(dev->page);
+				payload->checksum[0] = cpu_to_le32(
+					crc32c_le(log->uuid_checksum, addr,
+						  PAGE_SIZE));
+				kunmap_atomic(addr);
+				sync_page_io(log->rdev, write_pos, PAGE_SIZE,
+					     dev->page, REQ_OP_WRITE, 0, false);
+				write_pos = r5l_ring_add(log, write_pos,
+							 BLOCK_SECTORS);
+				offset += sizeof(__le32) +
+					sizeof(struct r5l_payload_data_parity);
+
+			}
+		}
+		mb->meta_size = cpu_to_le32(offset);
+		mb->checksum = cpu_to_le32(crc32c_le(log->uuid_checksum,
+						     mb, PAGE_SIZE));
+		sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page,
+			     REQ_OP_WRITE, REQ_FUA, false);
+		sh->log_start = ctx->pos;
+		ctx->pos = write_pos;
+		ctx->seq += 1;
+
+		list_del_init(&sh->lru);
+		raid5_release_stripe(sh);
 	}
 	__free_page(page);
 	return 0;
@@ -1063,45 +2135,60 @@ static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos,
 
 static int r5l_recovery_log(struct r5l_log *log)
 {
+	struct mddev *mddev = log->rdev->mddev;
 	struct r5l_recovery_ctx ctx;
+	int ret;
+	sector_t pos;
+	struct stripe_head *sh;
 
 	ctx.pos = log->last_checkpoint;
 	ctx.seq = log->last_cp_seq;
 	ctx.meta_page = alloc_page(GFP_KERNEL);
+	ctx.data_only_stripes = 0;
+	ctx.data_parity_stripes = 0;
+	INIT_LIST_HEAD(&ctx.cached_list);
+
 	if (!ctx.meta_page)
 		return -ENOMEM;
 
-	r5l_recovery_flush_log(log, &ctx);
+	ret = r5c_recovery_flush_log(log, &ctx);
 	__free_page(ctx.meta_page);
 
-	/*
-	 * we did a recovery. Now ctx.pos points to an invalid meta block. New
-	 * log will start here. but we can't let superblock point to last valid
-	 * meta block. The log might looks like:
-	 * | meta 1| meta 2| meta 3|
-	 * meta 1 is valid, meta 2 is invalid. meta 3 could be valid. If
-	 * superblock points to meta 1, we write a new valid meta 2n.  if crash
-	 * happens again, new recovery will start from meta 1. Since meta 2n is
-	 * valid now, recovery will think meta 3 is valid, which is wrong.
-	 * The solution is we create a new meta in meta2 with its seq == meta
-	 * 1's seq + 10 and let superblock points to meta2. The same recovery will
-	 * not think meta 3 is a valid meta, because its seq doesn't match
-	 */
-	if (ctx.seq > log->last_cp_seq) {
-		int ret;
-
-		ret = r5l_log_write_empty_meta_block(log, ctx.pos, ctx.seq + 10);
-		if (ret)
-			return ret;
-		log->seq = ctx.seq + 11;
-		log->log_start = r5l_ring_add(log, ctx.pos, BLOCK_SECTORS);
-		r5l_write_super(log, ctx.pos);
-		log->last_checkpoint = ctx.pos;
+	if (ret)
+		return ret;
+
+	pos = ctx.pos;
+	ctx.seq += 10000;
+
+	if (ctx.data_only_stripes == 0) {
 		log->next_checkpoint = ctx.pos;
+		r5l_log_write_empty_meta_block(log, ctx.pos, ctx.seq++);
+		ctx.pos = r5l_ring_add(log, ctx.pos, BLOCK_SECTORS);
 	} else {
-		log->log_start = ctx.pos;
-		log->seq = ctx.seq;
+		sh = list_last_entry(&ctx.cached_list, struct stripe_head, lru);
+		log->next_checkpoint = sh->log_start;
+	}
+
+	if ((ctx.data_only_stripes == 0) && (ctx.data_parity_stripes == 0))
+		pr_debug("md/raid:%s: starting from clean shutdown\n",
+			 mdname(mddev));
+	else {
+		pr_debug("md/raid:%s: recoverying %d data-only stripes and %d data-parity stripes\n",
+			 mdname(mddev), ctx.data_only_stripes,
+			 ctx.data_parity_stripes);
+
+		if (ctx.data_only_stripes > 0)
+			if (r5c_recovery_rewrite_data_only_stripes(log, &ctx)) {
+				pr_err("md/raid:%s: failed to rewrite stripes to journal\n",
+				       mdname(mddev));
+				return -EIO;
+			}
 	}
+
+	log->log_start = ctx.pos;
+	log->seq = ctx.seq;
+	log->last_checkpoint = pos;
+	r5l_write_super(log, pos);
 	return 0;
 }
 
@@ -1110,7 +2197,293 @@ static void r5l_write_super(struct r5l_log *log, sector_t cp)
 	struct mddev *mddev = log->rdev->mddev;
 
 	log->rdev->journal_tail = cp;
-	set_bit(MD_CHANGE_DEVS, &mddev->flags);
+	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
+}
+
+static ssize_t r5c_journal_mode_show(struct mddev *mddev, char *page)
+{
+	struct r5conf *conf = mddev->private;
+	int ret;
+
+	if (!conf->log)
+		return 0;
+
+	switch (conf->log->r5c_journal_mode) {
+	case R5C_JOURNAL_MODE_WRITE_THROUGH:
+		ret = snprintf(
+			page, PAGE_SIZE, "[%s] %s\n",
+			r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_THROUGH],
+			r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_BACK]);
+		break;
+	case R5C_JOURNAL_MODE_WRITE_BACK:
+		ret = snprintf(
+			page, PAGE_SIZE, "%s [%s]\n",
+			r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_THROUGH],
+			r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_BACK]);
+		break;
+	default:
+		ret = 0;
+	}
+	return ret;
+}
+
+static ssize_t r5c_journal_mode_store(struct mddev *mddev,
+				      const char *page, size_t length)
+{
+	struct r5conf *conf = mddev->private;
+	struct r5l_log *log = conf->log;
+	int val = -1, i;
+	int len = length;
+
+	if (!log)
+		return -ENODEV;
+
+	if (len && page[len - 1] == '\n')
+		len -= 1;
+	for (i = 0; i < ARRAY_SIZE(r5c_journal_mode_str); i++)
+		if (strlen(r5c_journal_mode_str[i]) == len &&
+		    strncmp(page, r5c_journal_mode_str[i], len) == 0) {
+			val = i;
+			break;
+		}
+	if (val < R5C_JOURNAL_MODE_WRITE_THROUGH ||
+	    val > R5C_JOURNAL_MODE_WRITE_BACK)
+		return -EINVAL;
+
+	mddev_suspend(mddev);
+	conf->log->r5c_journal_mode = val;
+	mddev_resume(mddev);
+
+	pr_debug("md/raid:%s: setting r5c cache mode to %d: %s\n",
+		 mdname(mddev), val, r5c_journal_mode_str[val]);
+	return length;
+}
+
+struct md_sysfs_entry
+r5c_journal_mode = __ATTR(journal_mode, 0644,
+			  r5c_journal_mode_show, r5c_journal_mode_store);
+
+/*
+ * Try handle write operation in caching phase. This function should only
+ * be called in write-back mode.
+ *
+ * If all outstanding writes can be handled in caching phase, returns 0
+ * If writes requires write-out phase, call r5c_make_stripe_write_out()
+ * and returns -EAGAIN
+ */
+int r5c_try_caching_write(struct r5conf *conf,
+			  struct stripe_head *sh,
+			  struct stripe_head_state *s,
+			  int disks)
+{
+	struct r5l_log *log = conf->log;
+	int i;
+	struct r5dev *dev;
+	int to_cache = 0;
+
+	BUG_ON(!r5c_is_writeback(log));
+
+	if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
+		/*
+		 * There are two different scenarios here:
+		 *  1. The stripe has some data cached, and it is sent to
+		 *     write-out phase for reclaim
+		 *  2. The stripe is clean, and this is the first write
+		 *
+		 * For 1, return -EAGAIN, so we continue with
+		 * handle_stripe_dirtying().
+		 *
+		 * For 2, set STRIPE_R5C_CACHING and continue with caching
+		 * write.
+		 */
+
+		/* case 1: anything injournal or anything in written */
+		if (s->injournal > 0 || s->written > 0)
+			return -EAGAIN;
+		/* case 2 */
+		set_bit(STRIPE_R5C_CACHING, &sh->state);
+	}
+
+	for (i = disks; i--; ) {
+		dev = &sh->dev[i];
+		/* if non-overwrite, use writing-out phase */
+		if (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags) &&
+		    !test_bit(R5_InJournal, &dev->flags)) {
+			r5c_make_stripe_write_out(sh);
+			return -EAGAIN;
+		}
+	}
+
+	for (i = disks; i--; ) {
+		dev = &sh->dev[i];
+		if (dev->towrite) {
+			set_bit(R5_Wantwrite, &dev->flags);
+			set_bit(R5_Wantdrain, &dev->flags);
+			set_bit(R5_LOCKED, &dev->flags);
+			to_cache++;
+		}
+	}
+
+	if (to_cache) {
+		set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
+		/*
+		 * set STRIPE_LOG_TRAPPED, which triggers r5c_cache_data()
+		 * in ops_run_io(). STRIPE_LOG_TRAPPED will be cleared in
+		 * r5c_handle_data_cached()
+		 */
+		set_bit(STRIPE_LOG_TRAPPED, &sh->state);
+	}
+
+	return 0;
+}
+
+/*
+ * free extra pages (orig_page) we allocated for prexor
+ */
+void r5c_release_extra_page(struct stripe_head *sh)
+{
+	struct r5conf *conf = sh->raid_conf;
+	int i;
+	bool using_disk_info_extra_page;
+
+	using_disk_info_extra_page =
+		sh->dev[0].orig_page == conf->disks[0].extra_page;
+
+	for (i = sh->disks; i--; )
+		if (sh->dev[i].page != sh->dev[i].orig_page) {
+			struct page *p = sh->dev[i].orig_page;
+
+			sh->dev[i].orig_page = sh->dev[i].page;
+			if (!using_disk_info_extra_page)
+				put_page(p);
+		}
+
+	if (using_disk_info_extra_page) {
+		clear_bit(R5C_EXTRA_PAGE_IN_USE, &conf->cache_state);
+		md_wakeup_thread(conf->mddev->thread);
+	}
+}
+
+void r5c_use_extra_page(struct stripe_head *sh)
+{
+	struct r5conf *conf = sh->raid_conf;
+	int i;
+	struct r5dev *dev;
+
+	for (i = sh->disks; i--; ) {
+		dev = &sh->dev[i];
+		if (dev->orig_page != dev->page)
+			put_page(dev->orig_page);
+		dev->orig_page = conf->disks[i].extra_page;
+	}
+}
+
+/*
+ * clean up the stripe (clear R5_InJournal for dev[pd_idx] etc.) after the
+ * stripe is committed to RAID disks.
+ */
+void r5c_finish_stripe_write_out(struct r5conf *conf,
+				 struct stripe_head *sh,
+				 struct stripe_head_state *s)
+{
+	int i;
+	int do_wakeup = 0;
+
+	if (!conf->log ||
+	    !test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags))
+		return;
+
+	WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
+	clear_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
+
+	if (conf->log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
+		return;
+
+	for (i = sh->disks; i--; ) {
+		clear_bit(R5_InJournal, &sh->dev[i].flags);
+		if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
+			do_wakeup = 1;
+	}
+
+	/*
+	 * analyse_stripe() runs before r5c_finish_stripe_write_out(),
+	 * We updated R5_InJournal, so we also update s->injournal.
+	 */
+	s->injournal = 0;
+
+	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
+		if (atomic_dec_and_test(&conf->pending_full_writes))
+			md_wakeup_thread(conf->mddev->thread);
+
+	if (do_wakeup)
+		wake_up(&conf->wait_for_overlap);
+
+	if (conf->log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
+		return;
+
+	spin_lock_irq(&conf->log->stripe_in_journal_lock);
+	list_del_init(&sh->r5c);
+	spin_unlock_irq(&conf->log->stripe_in_journal_lock);
+	sh->log_start = MaxSector;
+	atomic_dec(&conf->log->stripe_in_journal_count);
+	r5c_update_log_state(conf->log);
+}
+
+int
+r5c_cache_data(struct r5l_log *log, struct stripe_head *sh,
+	       struct stripe_head_state *s)
+{
+	struct r5conf *conf = sh->raid_conf;
+	int pages = 0;
+	int reserve;
+	int i;
+	int ret = 0;
+
+	BUG_ON(!log);
+
+	for (i = 0; i < sh->disks; i++) {
+		void *addr;
+
+		if (!test_bit(R5_Wantwrite, &sh->dev[i].flags))
+			continue;
+		addr = kmap_atomic(sh->dev[i].page);
+		sh->dev[i].log_checksum = crc32c_le(log->uuid_checksum,
+						    addr, PAGE_SIZE);
+		kunmap_atomic(addr);
+		pages++;
+	}
+	WARN_ON(pages == 0);
+
+	/*
+	 * The stripe must enter state machine again to call endio, so
+	 * don't delay.
+	 */
+	clear_bit(STRIPE_DELAYED, &sh->state);
+	atomic_inc(&sh->count);
+
+	mutex_lock(&log->io_mutex);
+	/* meta + data */
+	reserve = (1 + pages) << (PAGE_SHIFT - 9);
+
+	if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
+	    sh->log_start == MaxSector)
+		r5l_add_no_space_stripe(log, sh);
+	else if (!r5l_has_free_space(log, reserve)) {
+		if (sh->log_start == log->last_checkpoint)
+			BUG();
+		else
+			r5l_add_no_space_stripe(log, sh);
+	} else {
+		ret = r5l_log_stripe(log, sh, pages, 0);
+		if (ret) {
+			spin_lock_irq(&log->io_list_lock);
+			list_add_tail(&sh->log_list, &log->no_mem_stripes);
+			spin_unlock_irq(&log->io_list_lock);
+		}
+	}
+
+	mutex_unlock(&log->io_mutex);
+	return 0;
 }
 
 static int r5l_load_log(struct r5l_log *log)
@@ -1121,7 +2494,7 @@ static int r5l_load_log(struct r5l_log *log)
 	sector_t cp = log->rdev->journal_tail;
 	u32 stored_crc, expected_crc;
 	bool create_super = false;
-	int ret;
+	int ret = 0;
 
 	/* Make sure it's valid */
 	if (cp >= rdev->sectors || round_down(cp, BLOCK_SECTORS) != cp)
@@ -1171,11 +2544,18 @@ create:
 	if (log->max_free_space > RECLAIM_MAX_FREE_SPACE)
 		log->max_free_space = RECLAIM_MAX_FREE_SPACE;
 	log->last_checkpoint = cp;
-	log->next_checkpoint = cp;
 
 	__free_page(page);
 
-	return r5l_recovery_log(log);
+	if (create_super) {
+		log->log_start = r5l_ring_add(log, cp, BLOCK_SECTORS);
+		log->seq = log->last_cp_seq + 1;
+		log->next_checkpoint = cp;
+	} else
+		ret = r5l_recovery_log(log);
+
+	r5c_update_log_state(log);
+	return ret;
 ioerr:
 	__free_page(page);
 	return ret;
@@ -1188,6 +2568,22 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
 
 	if (PAGE_SIZE != 4096)
 		return -EINVAL;
+
+	/*
+	 * The PAGE_SIZE must be big enough to hold 1 r5l_meta_block and
+	 * raid_disks r5l_payload_data_parity.
+	 *
+	 * Write journal and cache does not work for very big array
+	 * (raid_disks > 203)
+	 */
+	if (sizeof(struct r5l_meta_block) +
+	    ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32)) *
+	     conf->raid_disks) > PAGE_SIZE) {
+		pr_err("md/raid:%s: write journal/cache doesn't work for array with %d disks\n",
+		       mdname(conf->mddev), conf->raid_disks);
+		return -EINVAL;
+	}
+
 	log = kzalloc(sizeof(*log), GFP_KERNEL);
 	if (!log)
 		return -ENOMEM;
@@ -1205,7 +2601,7 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
 	INIT_LIST_HEAD(&log->io_end_ios);
 	INIT_LIST_HEAD(&log->flushing_ios);
 	INIT_LIST_HEAD(&log->finished_ios);
-	bio_init(&log->flush_bio);
+	bio_init(&log->flush_bio, NULL, 0);
 
 	log->io_kc = KMEM_CACHE(r5l_io_unit, 0);
 	if (!log->io_kc)
@@ -1227,6 +2623,8 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
 						 log->rdev->mddev, "reclaim");
 	if (!log->reclaim_thread)
 		goto reclaim_thread;
+	log->reclaim_thread->timeout = R5C_RECLAIM_WAKEUP_INTERVAL;
+
 	init_waitqueue_head(&log->iounit_wait);
 
 	INIT_LIST_HEAD(&log->no_mem_stripes);
@@ -1234,6 +2632,13 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
 	INIT_LIST_HEAD(&log->no_space_stripes);
 	spin_lock_init(&log->no_space_stripes_lock);
 
+	INIT_WORK(&log->deferred_io_work, r5l_submit_io_async);
+
+	log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
+	INIT_LIST_HEAD(&log->stripe_in_journal_list);
+	spin_lock_init(&log->stripe_in_journal_lock);
+	atomic_set(&log->stripe_in_journal_count, 0);
+
 	if (r5l_load_log(log))
 		goto error;
 
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 92ac251e91e6..06d7279bdd04 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -70,19 +70,6 @@ module_param(devices_handle_discard_safely, bool, 0644);
 MODULE_PARM_DESC(devices_handle_discard_safely,
 		 "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
 static struct workqueue_struct *raid5_wq;
-/*
- * Stripe cache
- */
-
-#define NR_STRIPES		256
-#define STRIPE_SIZE		PAGE_SIZE
-#define STRIPE_SHIFT		(PAGE_SHIFT - 9)
-#define STRIPE_SECTORS		(STRIPE_SIZE>>9)
-#define	IO_THRESHOLD		1
-#define BYPASS_THRESHOLD	1
-#define NR_HASH			(PAGE_SIZE / sizeof(struct hlist_head))
-#define HASH_MASK		(NR_HASH - 1)
-#define MAX_STRIPE_BATCH	8
 
 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
 {
@@ -126,64 +113,6 @@ static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)
 	local_irq_enable();
 }
 
-/* bio's attached to a stripe+device for I/O are linked together in bi_sector
- * order without overlap.  There may be several bio's per stripe+device, and
- * a bio could span several devices.
- * When walking this list for a particular stripe+device, we must never proceed
- * beyond a bio that extends past this device, as the next bio might no longer
- * be valid.
- * This function is used to determine the 'next' bio in the list, given the sector
- * of the current stripe+device
- */
-static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
-{
-	int sectors = bio_sectors(bio);
-	if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS)
-		return bio->bi_next;
-	else
-		return NULL;
-}
-
-/*
- * We maintain a biased count of active stripes in the bottom 16 bits of
- * bi_phys_segments, and a count of processed stripes in the upper 16 bits
- */
-static inline int raid5_bi_processed_stripes(struct bio *bio)
-{
-	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
-	return (atomic_read(segments) >> 16) & 0xffff;
-}
-
-static inline int raid5_dec_bi_active_stripes(struct bio *bio)
-{
-	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
-	return atomic_sub_return(1, segments) & 0xffff;
-}
-
-static inline void raid5_inc_bi_active_stripes(struct bio *bio)
-{
-	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
-	atomic_inc(segments);
-}
-
-static inline void raid5_set_bi_processed_stripes(struct bio *bio,
-	unsigned int cnt)
-{
-	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
-	int old, new;
-
-	do {
-		old = atomic_read(segments);
-		new = (old & 0xffff) | (cnt << 16);
-	} while (atomic_cmpxchg(segments, old, new) != old);
-}
-
-static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
-{
-	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
-	atomic_set(segments, cnt);
-}
-
 /* Find first data disk in a raid6 stripe */
 static inline int raid6_d0(struct stripe_head *sh)
 {
@@ -289,8 +218,27 @@ static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
 			      struct list_head *temp_inactive_list)
 {
+	int i;
+	int injournal = 0;	/* number of date pages with R5_InJournal */
+
 	BUG_ON(!list_empty(&sh->lru));
 	BUG_ON(atomic_read(&conf->active_stripes)==0);
+
+	if (r5c_is_writeback(conf->log))
+		for (i = sh->disks; i--; )
+			if (test_bit(R5_InJournal, &sh->dev[i].flags))
+				injournal++;
+	/*
+	 * When quiesce in r5c write back, set STRIPE_HANDLE for stripes with
+	 * data in journal, so they are not released to cached lists
+	 */
+	if (conf->quiesce && r5c_is_writeback(conf->log) &&
+	    !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0) {
+		if (test_bit(STRIPE_R5C_CACHING, &sh->state))
+			r5c_make_stripe_write_out(sh);
+		set_bit(STRIPE_HANDLE, &sh->state);
+	}
+
 	if (test_bit(STRIPE_HANDLE, &sh->state)) {
 		if (test_bit(STRIPE_DELAYED, &sh->state) &&
 		    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
@@ -316,8 +264,30 @@ static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
 			    < IO_THRESHOLD)
 				md_wakeup_thread(conf->mddev->thread);
 		atomic_dec(&conf->active_stripes);
-		if (!test_bit(STRIPE_EXPANDING, &sh->state))
-			list_add_tail(&sh->lru, temp_inactive_list);
+		if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
+			if (!r5c_is_writeback(conf->log))
+				list_add_tail(&sh->lru, temp_inactive_list);
+			else {
+				WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags));
+				if (injournal == 0)
+					list_add_tail(&sh->lru, temp_inactive_list);
+				else if (injournal == conf->raid_disks - conf->max_degraded) {
+					/* full stripe */
+					if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state))
+						atomic_inc(&conf->r5c_cached_full_stripes);
+					if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))
+						atomic_dec(&conf->r5c_cached_partial_stripes);
+					list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);
+					r5c_check_cached_full_stripe(conf);
+				} else {
+					/* partial stripe */
+					if (!test_and_set_bit(STRIPE_R5C_PARTIAL_STRIPE,
+							      &sh->state))
+						atomic_inc(&conf->r5c_cached_partial_stripes);
+					list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);
+				}
+			}
+		}
 	}
 }
 
@@ -541,7 +511,7 @@ retry:
 
 		if (dev->toread || dev->read || dev->towrite || dev->written ||
 		    test_bit(R5_LOCKED, &dev->flags)) {
-			printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
+			pr_err("sector=%llx i=%d %p %p %p %p %d\n",
 			       (unsigned long long)sh->sector, i, dev->toread,
 			       dev->read, dev->towrite, dev->written,
 			       test_bit(R5_LOCKED, &dev->flags));
@@ -680,9 +650,12 @@ raid5_get_active_stripe(struct r5conf *conf, sector_t sector,
 			}
 			if (noblock && sh == NULL)
 				break;
+
+			r5c_check_stripe_cache_usage(conf);
 			if (!sh) {
 				set_bit(R5_INACTIVE_BLOCKED,
 					&conf->cache_state);
+				r5l_wake_reclaim(conf->log, 0);
 				wait_event_lock_irq(
 					conf->wait_for_stripe,
 					!list_empty(conf->inactive_list + hash) &&
@@ -901,8 +874,19 @@ static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
 
 	might_sleep();
 
-	if (r5l_write_stripe(conf->log, sh) == 0)
-		return;
+	if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
+		/* writing out phase */
+		if (s->waiting_extra_page)
+			return;
+		if (r5l_write_stripe(conf->log, sh) == 0)
+			return;
+	} else {  /* caching phase */
+		if (test_bit(STRIPE_LOG_TRAPPED, &sh->state)) {
+			r5c_cache_data(conf->log, sh, s);
+			return;
+		}
+	}
+
 	for (i = disks; i--; ) {
 		int op, op_flags = 0;
 		int replace_only = 0;
@@ -913,7 +897,7 @@ static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
 		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
 			op = REQ_OP_WRITE;
 			if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
-				op_flags = WRITE_FUA;
+				op_flags = REQ_FUA;
 			if (test_bit(R5_Discard, &sh->dev[i].flags))
 				op = REQ_OP_DISCARD;
 		} else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
@@ -977,7 +961,7 @@ again:
 			if (bad < 0) {
 				set_bit(BlockedBadBlocks, &rdev->flags);
 				if (!conf->mddev->external &&
-				    conf->mddev->flags) {
+				    conf->mddev->sb_flags) {
 					/* It is very unlikely, but we might
 					 * still need to write out the
 					 * bad block log - better give it
@@ -1115,7 +1099,7 @@ again:
 static struct dma_async_tx_descriptor *
 async_copy_data(int frombio, struct bio *bio, struct page **page,
 	sector_t sector, struct dma_async_tx_descriptor *tx,
-	struct stripe_head *sh)
+	struct stripe_head *sh, int no_skipcopy)
 {
 	struct bio_vec bvl;
 	struct bvec_iter iter;
@@ -1155,7 +1139,8 @@ async_copy_data(int frombio, struct bio *bio, struct page **page,
 			if (frombio) {
 				if (sh->raid_conf->skip_copy &&
 				    b_offset == 0 && page_offset == 0 &&
-				    clen == STRIPE_SIZE)
+				    clen == STRIPE_SIZE &&
+				    !no_skipcopy)
 					*page = bio_page;
 				else
 					tx = async_memcpy(*page, bio_page, page_offset,
@@ -1237,7 +1222,7 @@ static void ops_run_biofill(struct stripe_head *sh)
 			while (rbi && rbi->bi_iter.bi_sector <
 				dev->sector + STRIPE_SECTORS) {
 				tx = async_copy_data(0, rbi, &dev->page,
-					dev->sector, tx, sh);
+						     dev->sector, tx, sh, 0);
 				rbi = r5_next_bio(rbi, dev->sector);
 			}
 		}
@@ -1364,10 +1349,15 @@ static int set_syndrome_sources(struct page **srcs,
 		if (i == sh->qd_idx || i == sh->pd_idx ||
 		    (srctype == SYNDROME_SRC_ALL) ||
 		    (srctype == SYNDROME_SRC_WANT_DRAIN &&
-		     test_bit(R5_Wantdrain, &dev->flags)) ||
+		     (test_bit(R5_Wantdrain, &dev->flags) ||
+		      test_bit(R5_InJournal, &dev->flags))) ||
 		    (srctype == SYNDROME_SRC_WRITTEN &&
-		     dev->written))
-			srcs[slot] = sh->dev[i].page;
+		     dev->written)) {
+			if (test_bit(R5_InJournal, &dev->flags))
+				srcs[slot] = sh->dev[i].orig_page;
+			else
+				srcs[slot] = sh->dev[i].page;
+		}
 		i = raid6_next_disk(i, disks);
 	} while (i != d0_idx);
 
@@ -1546,6 +1536,13 @@ static void ops_complete_prexor(void *stripe_head_ref)
 
 	pr_debug("%s: stripe %llu\n", __func__,
 		(unsigned long long)sh->sector);
+
+	if (r5c_is_writeback(sh->raid_conf->log))
+		/*
+		 * raid5-cache write back uses orig_page during prexor.
+		 * After prexor, it is time to free orig_page
+		 */
+		r5c_release_extra_page(sh);
 }
 
 static struct dma_async_tx_descriptor *
@@ -1567,7 +1564,9 @@ ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
 	for (i = disks; i--; ) {
 		struct r5dev *dev = &sh->dev[i];
 		/* Only process blocks that are known to be uptodate */
-		if (test_bit(R5_Wantdrain, &dev->flags))
+		if (test_bit(R5_InJournal, &dev->flags))
+			xor_srcs[count++] = dev->orig_page;
+		else if (test_bit(R5_Wantdrain, &dev->flags))
 			xor_srcs[count++] = dev->page;
 	}
 
@@ -1601,6 +1600,7 @@ ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
 static struct dma_async_tx_descriptor *
 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
 {
+	struct r5conf *conf = sh->raid_conf;
 	int disks = sh->disks;
 	int i;
 	struct stripe_head *head_sh = sh;
@@ -1618,6 +1618,11 @@ ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
 
 again:
 			dev = &sh->dev[i];
+			/*
+			 * clear R5_InJournal, so when rewriting a page in
+			 * journal, it is not skipped by r5l_log_stripe()
+			 */
+			clear_bit(R5_InJournal, &dev->flags);
 			spin_lock_irq(&sh->stripe_lock);
 			chosen = dev->towrite;
 			dev->towrite = NULL;
@@ -1637,8 +1642,10 @@ again:
 					set_bit(R5_Discard, &dev->flags);
 				else {
 					tx = async_copy_data(1, wbi, &dev->page,
-						dev->sector, tx, sh);
-					if (dev->page != dev->orig_page) {
+							     dev->sector, tx, sh,
+							     r5c_is_writeback(conf->log));
+					if (dev->page != dev->orig_page &&
+					    !r5c_is_writeback(conf->log)) {
 						set_bit(R5_SkipCopy, &dev->flags);
 						clear_bit(R5_UPTODATE, &dev->flags);
 						clear_bit(R5_OVERWRITE, &dev->flags);
@@ -1746,7 +1753,8 @@ again:
 		xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
 		for (i = disks; i--; ) {
 			struct r5dev *dev = &sh->dev[i];
-			if (head_sh->dev[i].written)
+			if (head_sh->dev[i].written ||
+			    test_bit(R5_InJournal, &head_sh->dev[i].flags))
 				xor_srcs[count++] = dev->page;
 		}
 	} else {
@@ -2000,17 +2008,15 @@ static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp,
 		spin_lock_init(&sh->batch_lock);
 		INIT_LIST_HEAD(&sh->batch_list);
 		INIT_LIST_HEAD(&sh->lru);
+		INIT_LIST_HEAD(&sh->r5c);
+		INIT_LIST_HEAD(&sh->log_list);
 		atomic_set(&sh->count, 1);
+		sh->log_start = MaxSector;
 		for (i = 0; i < disks; i++) {
 			struct r5dev *dev = &sh->dev[i];
 
-			bio_init(&dev->req);
-			dev->req.bi_io_vec = &dev->vec;
-			dev->req.bi_max_vecs = 1;
-
-			bio_init(&dev->rreq);
-			dev->rreq.bi_io_vec = &dev->rvec;
-			dev->rreq.bi_max_vecs = 1;
+			bio_init(&dev->req, &dev->vec, 1);
+			bio_init(&dev->rreq, &dev->rvec, 1);
 		}
 	}
 	return sh;
@@ -2245,10 +2251,24 @@ static int resize_stripes(struct r5conf *conf, int newsize)
 	 */
 	ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
 	if (ndisks) {
-		for (i=0; i<conf->raid_disks; i++)
+		for (i = 0; i < conf->pool_size; i++)
 			ndisks[i] = conf->disks[i];
-		kfree(conf->disks);
-		conf->disks = ndisks;
+
+		for (i = conf->pool_size; i < newsize; i++) {
+			ndisks[i].extra_page = alloc_page(GFP_NOIO);
+			if (!ndisks[i].extra_page)
+				err = -ENOMEM;
+		}
+
+		if (err) {
+			for (i = conf->pool_size; i < newsize; i++)
+				if (ndisks[i].extra_page)
+					put_page(ndisks[i].extra_page);
+			kfree(ndisks);
+		} else {
+			kfree(conf->disks);
+			conf->disks = ndisks;
+		}
 	} else
 		err = -ENOMEM;
 
@@ -2347,10 +2367,8 @@ static void raid5_end_read_request(struct bio * bi)
 			 * replacement device.  We just fail those on
 			 * any error
 			 */
-			printk_ratelimited(
-				KERN_INFO
-				"md/raid:%s: read error corrected"
-				" (%lu sectors at %llu on %s)\n",
+			pr_info_ratelimited(
+				"md/raid:%s: read error corrected (%lu sectors at %llu on %s)\n",
 				mdname(conf->mddev), STRIPE_SECTORS,
 				(unsigned long long)s,
 				bdevname(rdev->bdev, b));
@@ -2370,36 +2388,29 @@ static void raid5_end_read_request(struct bio * bi)
 		clear_bit(R5_UPTODATE, &sh->dev[i].flags);
 		atomic_inc(&rdev->read_errors);
 		if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
-			printk_ratelimited(
-				KERN_WARNING
-				"md/raid:%s: read error on replacement device "
-				"(sector %llu on %s).\n",
+			pr_warn_ratelimited(
+				"md/raid:%s: read error on replacement device (sector %llu on %s).\n",
 				mdname(conf->mddev),
 				(unsigned long long)s,
 				bdn);
 		else if (conf->mddev->degraded >= conf->max_degraded) {
 			set_bad = 1;
-			printk_ratelimited(
-				KERN_WARNING
-				"md/raid:%s: read error not correctable "
-				"(sector %llu on %s).\n",
+			pr_warn_ratelimited(
+				"md/raid:%s: read error not correctable (sector %llu on %s).\n",
 				mdname(conf->mddev),
 				(unsigned long long)s,
 				bdn);
 		} else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
 			/* Oh, no!!! */
 			set_bad = 1;
-			printk_ratelimited(
-				KERN_WARNING
-				"md/raid:%s: read error NOT corrected!! "
-				"(sector %llu on %s).\n",
+			pr_warn_ratelimited(
+				"md/raid:%s: read error NOT corrected!! (sector %llu on %s).\n",
 				mdname(conf->mddev),
 				(unsigned long long)s,
 				bdn);
 		} else if (atomic_read(&rdev->read_errors)
 			 > conf->max_nr_stripes)
-			printk(KERN_WARNING
-			       "md/raid:%s: Too many read errors, failing device %s.\n",
+			pr_warn("md/raid:%s: Too many read errors, failing device %s.\n",
 			       mdname(conf->mddev), bdn);
 		else
 			retry = 1;
@@ -2531,15 +2542,14 @@ static void raid5_error(struct mddev *mddev, struct md_rdev *rdev)
 
 	set_bit(Blocked, &rdev->flags);
 	set_bit(Faulty, &rdev->flags);
-	set_mask_bits(&mddev->flags, 0,
-		      BIT(MD_CHANGE_DEVS) | BIT(MD_CHANGE_PENDING));
-	printk(KERN_ALERT
-	       "md/raid:%s: Disk failure on %s, disabling device.\n"
-	       "md/raid:%s: Operation continuing on %d devices.\n",
-	       mdname(mddev),
-	       bdevname(rdev->bdev, b),
-	       mdname(mddev),
-	       conf->raid_disks - mddev->degraded);
+	set_mask_bits(&mddev->sb_flags, 0,
+		      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
+	pr_crit("md/raid:%s: Disk failure on %s, disabling device.\n"
+		"md/raid:%s: Operation continuing on %d devices.\n",
+		mdname(mddev),
+		bdevname(rdev->bdev, b),
+		mdname(mddev),
+		conf->raid_disks - mddev->degraded);
 }
 
 /*
@@ -2861,8 +2871,8 @@ sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous)
 				     previous, &dummy1, &sh2);
 	if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
 		|| sh2.qd_idx != sh->qd_idx) {
-		printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
-		       mdname(conf->mddev));
+		pr_warn("md/raid:%s: compute_blocknr: map not correct\n",
+			mdname(conf->mddev));
 		return 0;
 	}
 	return r_sector;
@@ -2877,6 +2887,13 @@ schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
 	int level = conf->level;
 
 	if (rcw) {
+		/*
+		 * In some cases, handle_stripe_dirtying initially decided to
+		 * run rmw and allocates extra page for prexor. However, rcw is
+		 * cheaper later on. We need to free the extra page now,
+		 * because we won't be able to do that in ops_complete_prexor().
+		 */
+		r5c_release_extra_page(sh);
 
 		for (i = disks; i--; ) {
 			struct r5dev *dev = &sh->dev[i];
@@ -2887,6 +2904,9 @@ schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
 				if (!expand)
 					clear_bit(R5_UPTODATE, &dev->flags);
 				s->locked++;
+			} else if (test_bit(R5_InJournal, &dev->flags)) {
+				set_bit(R5_LOCKED, &dev->flags);
+				s->locked++;
 			}
 		}
 		/* if we are not expanding this is a proper write request, and
@@ -2926,6 +2946,9 @@ schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
 				set_bit(R5_LOCKED, &dev->flags);
 				clear_bit(R5_UPTODATE, &dev->flags);
 				s->locked++;
+			} else if (test_bit(R5_InJournal, &dev->flags)) {
+				set_bit(R5_LOCKED, &dev->flags);
+				s->locked++;
 			}
 		}
 		if (!s->locked)
@@ -3569,10 +3592,10 @@ unhash:
 		break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS);
 }
 
-static void handle_stripe_dirtying(struct r5conf *conf,
-				   struct stripe_head *sh,
-				   struct stripe_head_state *s,
-				   int disks)
+static int handle_stripe_dirtying(struct r5conf *conf,
+				  struct stripe_head *sh,
+				  struct stripe_head_state *s,
+				  int disks)
 {
 	int rmw = 0, rcw = 0, i;
 	sector_t recovery_cp = conf->mddev->recovery_cp;
@@ -3597,9 +3620,12 @@ static void handle_stripe_dirtying(struct r5conf *conf,
 	} else for (i = disks; i--; ) {
 		/* would I have to read this buffer for read_modify_write */
 		struct r5dev *dev = &sh->dev[i];
-		if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) &&
+		if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx ||
+		     test_bit(R5_InJournal, &dev->flags)) &&
 		    !test_bit(R5_LOCKED, &dev->flags) &&
-		    !(test_bit(R5_UPTODATE, &dev->flags) ||
+		    !((test_bit(R5_UPTODATE, &dev->flags) &&
+		       (!test_bit(R5_InJournal, &dev->flags) ||
+			dev->page != dev->orig_page)) ||
 		      test_bit(R5_Wantcompute, &dev->flags))) {
 			if (test_bit(R5_Insync, &dev->flags))
 				rmw++;
@@ -3611,13 +3637,15 @@ static void handle_stripe_dirtying(struct r5conf *conf,
 		    i != sh->pd_idx && i != sh->qd_idx &&
 		    !test_bit(R5_LOCKED, &dev->flags) &&
 		    !(test_bit(R5_UPTODATE, &dev->flags) ||
-		    test_bit(R5_Wantcompute, &dev->flags))) {
+		      test_bit(R5_InJournal, &dev->flags) ||
+		      test_bit(R5_Wantcompute, &dev->flags))) {
 			if (test_bit(R5_Insync, &dev->flags))
 				rcw++;
 			else
 				rcw += 2*disks;
 		}
 	}
+
 	pr_debug("for sector %llu, rmw=%d rcw=%d\n",
 		(unsigned long long)sh->sector, rmw, rcw);
 	set_bit(STRIPE_HANDLE, &sh->state);
@@ -3629,10 +3657,44 @@ static void handle_stripe_dirtying(struct r5conf *conf,
 					  (unsigned long long)sh->sector, rmw);
 		for (i = disks; i--; ) {
 			struct r5dev *dev = &sh->dev[i];
-			if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) &&
+			if (test_bit(R5_InJournal, &dev->flags) &&
+			    dev->page == dev->orig_page &&
+			    !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) {
+				/* alloc page for prexor */
+				struct page *p = alloc_page(GFP_NOIO);
+
+				if (p) {
+					dev->orig_page = p;
+					continue;
+				}
+
+				/*
+				 * alloc_page() failed, try use
+				 * disk_info->extra_page
+				 */
+				if (!test_and_set_bit(R5C_EXTRA_PAGE_IN_USE,
+						      &conf->cache_state)) {
+					r5c_use_extra_page(sh);
+					break;
+				}
+
+				/* extra_page in use, add to delayed_list */
+				set_bit(STRIPE_DELAYED, &sh->state);
+				s->waiting_extra_page = 1;
+				return -EAGAIN;
+			}
+		}
+
+		for (i = disks; i--; ) {
+			struct r5dev *dev = &sh->dev[i];
+			if ((dev->towrite ||
+			     i == sh->pd_idx || i == sh->qd_idx ||
+			     test_bit(R5_InJournal, &dev->flags)) &&
 			    !test_bit(R5_LOCKED, &dev->flags) &&
-			    !(test_bit(R5_UPTODATE, &dev->flags) ||
-			    test_bit(R5_Wantcompute, &dev->flags)) &&
+			    !((test_bit(R5_UPTODATE, &dev->flags) &&
+			       (!test_bit(R5_InJournal, &dev->flags) ||
+				dev->page != dev->orig_page)) ||
+			      test_bit(R5_Wantcompute, &dev->flags)) &&
 			    test_bit(R5_Insync, &dev->flags)) {
 				if (test_bit(STRIPE_PREREAD_ACTIVE,
 					     &sh->state)) {
@@ -3658,6 +3720,7 @@ static void handle_stripe_dirtying(struct r5conf *conf,
 			    i != sh->pd_idx && i != sh->qd_idx &&
 			    !test_bit(R5_LOCKED, &dev->flags) &&
 			    !(test_bit(R5_UPTODATE, &dev->flags) ||
+			      test_bit(R5_InJournal, &dev->flags) ||
 			      test_bit(R5_Wantcompute, &dev->flags))) {
 				rcw++;
 				if (test_bit(R5_Insync, &dev->flags) &&
@@ -3697,8 +3760,9 @@ static void handle_stripe_dirtying(struct r5conf *conf,
 	 */
 	if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
 	    (s->locked == 0 && (rcw == 0 || rmw == 0) &&
-	    !test_bit(STRIPE_BIT_DELAY, &sh->state)))
+	     !test_bit(STRIPE_BIT_DELAY, &sh->state)))
 		schedule_reconstruction(sh, s, rcw == 0, 0);
+	return 0;
 }
 
 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
@@ -3782,7 +3846,7 @@ static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
 	case check_state_compute_run:
 		break;
 	default:
-		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
+		pr_err("%s: unknown check_state: %d sector: %llu\n",
 		       __func__, sh->check_state,
 		       (unsigned long long) sh->sector);
 		BUG();
@@ -3946,9 +4010,9 @@ static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
 	case check_state_compute_run:
 		break;
 	default:
-		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
-		       __func__, sh->check_state,
-		       (unsigned long long) sh->sector);
+		pr_warn("%s: unknown check_state: %d sector: %llu\n",
+			__func__, sh->check_state,
+			(unsigned long long) sh->sector);
 		BUG();
 	}
 }
@@ -4188,6 +4252,11 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
 			if (rdev && !test_bit(Faulty, &rdev->flags))
 				do_recovery = 1;
 		}
+
+		if (test_bit(R5_InJournal, &dev->flags))
+			s->injournal++;
+		if (test_bit(R5_InJournal, &dev->flags) && dev->written)
+			s->just_cached++;
 	}
 	if (test_bit(STRIPE_SYNCING, &sh->state)) {
 		/* If there is a failed device being replaced,
@@ -4416,7 +4485,8 @@ static void handle_stripe(struct stripe_head *sh)
 			struct r5dev *dev = &sh->dev[i];
 			if (test_bit(R5_LOCKED, &dev->flags) &&
 				(i == sh->pd_idx || i == sh->qd_idx ||
-				 dev->written)) {
+				 dev->written || test_bit(R5_InJournal,
+							  &dev->flags))) {
 				pr_debug("Writing block %d\n", i);
 				set_bit(R5_Wantwrite, &dev->flags);
 				if (prexor)
@@ -4456,6 +4526,10 @@ static void handle_stripe(struct stripe_head *sh)
 				 test_bit(R5_Discard, &qdev->flags))))))
 		handle_stripe_clean_event(conf, sh, disks, &s.return_bi);
 
+	if (s.just_cached)
+		r5c_handle_cached_data_endio(conf, sh, disks, &s.return_bi);
+	r5l_stripe_write_finished(sh);
+
 	/* Now we might consider reading some blocks, either to check/generate
 	 * parity, or to satisfy requests
 	 * or to load a block that is being partially written.
@@ -4467,14 +4541,51 @@ static void handle_stripe(struct stripe_head *sh)
 	    || s.expanding)
 		handle_stripe_fill(sh, &s, disks);
 
-	/* Now to consider new write requests and what else, if anything
-	 * should be read.  We do not handle new writes when:
+	/*
+	 * When the stripe finishes full journal write cycle (write to journal
+	 * and raid disk), this is the clean up procedure so it is ready for
+	 * next operation.
+	 */
+	r5c_finish_stripe_write_out(conf, sh, &s);
+
+	/*
+	 * Now to consider new write requests, cache write back and what else,
+	 * if anything should be read.  We do not handle new writes when:
 	 * 1/ A 'write' operation (copy+xor) is already in flight.
 	 * 2/ A 'check' operation is in flight, as it may clobber the parity
 	 *    block.
+	 * 3/ A r5c cache log write is in flight.
 	 */
-	if (s.to_write && !sh->reconstruct_state && !sh->check_state)
-		handle_stripe_dirtying(conf, sh, &s, disks);
+
+	if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) {
+		if (!r5c_is_writeback(conf->log)) {
+			if (s.to_write)
+				handle_stripe_dirtying(conf, sh, &s, disks);
+		} else { /* write back cache */
+			int ret = 0;
+
+			/* First, try handle writes in caching phase */
+			if (s.to_write)
+				ret = r5c_try_caching_write(conf, sh, &s,
+							    disks);
+			/*
+			 * If caching phase failed: ret == -EAGAIN
+			 *    OR
+			 * stripe under reclaim: !caching && injournal
+			 *
+			 * fall back to handle_stripe_dirtying()
+			 */
+			if (ret == -EAGAIN ||
+			    /* stripe under reclaim: !caching && injournal */
+			    (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
+			     s.injournal > 0)) {
+				ret = handle_stripe_dirtying(conf, sh, &s,
+							     disks);
+				if (ret == -EAGAIN)
+					goto finish;
+			}
+		}
+	}
 
 	/* maybe we need to check and possibly fix the parity for this stripe
 	 * Any reads will already have been scheduled, so we just see if enough
@@ -4645,9 +4756,7 @@ finish:
 	}
 
 	if (!bio_list_empty(&s.return_bi)) {
-		if (test_bit(MD_CHANGE_PENDING, &conf->mddev->flags) &&
-				(s.failed <= conf->max_degraded ||
-					conf->mddev->external == 0)) {
+		if (test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags)) {
 			spin_lock_irq(&conf->device_lock);
 			bio_list_merge(&conf->return_bi, &s.return_bi);
 			spin_unlock_irq(&conf->device_lock);
@@ -4703,6 +4812,10 @@ static int raid5_congested(struct mddev *mddev, int bits)
 
 	if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))
 		return 1;
+
+	/* Also checks whether there is pressure on r5cache log space */
+	if (test_bit(R5C_LOG_TIGHT, &conf->cache_state))
+		return 1;
 	if (conf->quiesce)
 		return 1;
 	if (atomic_read(&conf->empty_inactive_list_nr))
@@ -5172,6 +5285,7 @@ static void raid5_make_request(struct mddev *mddev, struct bio * bi)
 	int remaining;
 	DEFINE_WAIT(w);
 	bool do_prepare;
+	bool do_flush = false;
 
 	if (unlikely(bi->bi_opf & REQ_PREFLUSH)) {
 		int ret = r5l_handle_flush_request(conf->log, bi);
@@ -5183,6 +5297,11 @@ static void raid5_make_request(struct mddev *mddev, struct bio * bi)
 			return;
 		}
 		/* ret == -EAGAIN, fallback */
+		/*
+		 * if r5l_handle_flush_request() didn't clear REQ_PREFLUSH,
+		 * we need to flush journal device
+		 */
+		do_flush = bi->bi_opf & REQ_PREFLUSH;
 	}
 
 	md_write_start(mddev, bi);
@@ -5193,6 +5312,7 @@ static void raid5_make_request(struct mddev *mddev, struct bio * bi)
 	 * data on failed drives.
 	 */
 	if (rw == READ && mddev->degraded == 0 &&
+	    !r5c_is_writeback(conf->log) &&
 	    mddev->reshape_position == MaxSector) {
 		bi = chunk_aligned_read(mddev, bi);
 		if (!bi)
@@ -5321,6 +5441,12 @@ static void raid5_make_request(struct mddev *mddev, struct bio * bi)
 				do_prepare = true;
 				goto retry;
 			}
+			if (do_flush) {
+				set_bit(STRIPE_R5C_PREFLUSH, &sh->state);
+				/* we only need flush for one stripe */
+				do_flush = false;
+			}
+
 			set_bit(STRIPE_HANDLE, &sh->state);
 			clear_bit(STRIPE_DELAYED, &sh->state);
 			if ((!sh->batch_head || sh == sh->batch_head) &&
@@ -5486,9 +5612,9 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk
 		mddev->reshape_position = conf->reshape_progress;
 		mddev->curr_resync_completed = sector_nr;
 		conf->reshape_checkpoint = jiffies;
-		set_bit(MD_CHANGE_DEVS, &mddev->flags);
+		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 		md_wakeup_thread(mddev->thread);
-		wait_event(mddev->sb_wait, mddev->flags == 0 ||
+		wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||
 			   test_bit(MD_RECOVERY_INTR, &mddev->recovery));
 		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
 			return 0;
@@ -5584,10 +5710,10 @@ finish:
 		mddev->reshape_position = conf->reshape_progress;
 		mddev->curr_resync_completed = sector_nr;
 		conf->reshape_checkpoint = jiffies;
-		set_bit(MD_CHANGE_DEVS, &mddev->flags);
+		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 		md_wakeup_thread(mddev->thread);
 		wait_event(mddev->sb_wait,
-			   !test_bit(MD_CHANGE_DEVS, &mddev->flags)
+			   !test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags)
 			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
 		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
 			goto ret;
@@ -5862,10 +5988,10 @@ static void raid5d(struct md_thread *thread)
 	md_check_recovery(mddev);
 
 	if (!bio_list_empty(&conf->return_bi) &&
-	    !test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
+	    !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
 		struct bio_list tmp = BIO_EMPTY_LIST;
 		spin_lock_irq(&conf->device_lock);
-		if (!test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
+		if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
 			bio_list_merge(&tmp, &conf->return_bi);
 			bio_list_init(&conf->return_bi);
 		}
@@ -5912,7 +6038,7 @@ static void raid5d(struct md_thread *thread)
 			break;
 		handled += batch_size;
 
-		if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) {
+		if (mddev->sb_flags & ~(1 << MD_SB_CHANGE_PENDING)) {
 			spin_unlock_irq(&conf->device_lock);
 			md_check_recovery(mddev);
 			spin_lock_irq(&conf->device_lock);
@@ -6242,6 +6368,7 @@ static struct attribute *raid5_attrs[] =  {
 	&raid5_group_thread_cnt.attr,
 	&raid5_skip_copy.attr,
 	&raid5_rmw_level.attr,
+	&r5c_journal_mode.attr,
 	NULL,
 };
 static struct attribute_group raid5_attrs_group = {
@@ -6368,6 +6495,8 @@ static void raid5_free_percpu(struct r5conf *conf)
 
 static void free_conf(struct r5conf *conf)
 {
+	int i;
+
 	if (conf->log)
 		r5l_exit_log(conf->log);
 	if (conf->shrinker.nr_deferred)
@@ -6376,6 +6505,9 @@ static void free_conf(struct r5conf *conf)
 	free_thread_groups(conf);
 	shrink_stripes(conf);
 	raid5_free_percpu(conf);
+	for (i = 0; i < conf->pool_size; i++)
+		if (conf->disks[i].extra_page)
+			put_page(conf->disks[i].extra_page);
 	kfree(conf->disks);
 	kfree(conf->stripe_hashtbl);
 	kfree(conf);
@@ -6387,8 +6519,8 @@ static int raid456_cpu_up_prepare(unsigned int cpu, struct hlist_node *node)
 	struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
 
 	if (alloc_scratch_buffer(conf, percpu)) {
-		pr_err("%s: failed memory allocation for cpu%u\n",
-		       __func__, cpu);
+		pr_warn("%s: failed memory allocation for cpu%u\n",
+			__func__, cpu);
 		return -ENOMEM;
 	}
 	return 0;
@@ -6458,29 +6590,29 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 	if (mddev->new_level != 5
 	    && mddev->new_level != 4
 	    && mddev->new_level != 6) {
-		printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
-		       mdname(mddev), mddev->new_level);
+		pr_warn("md/raid:%s: raid level not set to 4/5/6 (%d)\n",
+			mdname(mddev), mddev->new_level);
 		return ERR_PTR(-EIO);
 	}
 	if ((mddev->new_level == 5
 	     && !algorithm_valid_raid5(mddev->new_layout)) ||
 	    (mddev->new_level == 6
 	     && !algorithm_valid_raid6(mddev->new_layout))) {
-		printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
-		       mdname(mddev), mddev->new_layout);
+		pr_warn("md/raid:%s: layout %d not supported\n",
+			mdname(mddev), mddev->new_layout);
 		return ERR_PTR(-EIO);
 	}
 	if (mddev->new_level == 6 && mddev->raid_disks < 4) {
-		printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
-		       mdname(mddev), mddev->raid_disks);
+		pr_warn("md/raid:%s: not enough configured devices (%d, minimum 4)\n",
+			mdname(mddev), mddev->raid_disks);
 		return ERR_PTR(-EINVAL);
 	}
 
 	if (!mddev->new_chunk_sectors ||
 	    (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
 	    !is_power_of_2(mddev->new_chunk_sectors)) {
-		printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
-		       mdname(mddev), mddev->new_chunk_sectors << 9);
+		pr_warn("md/raid:%s: invalid chunk size %d\n",
+			mdname(mddev), mddev->new_chunk_sectors << 9);
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -6522,9 +6654,16 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 
 	conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
 			      GFP_KERNEL);
+
 	if (!conf->disks)
 		goto abort;
 
+	for (i = 0; i < max_disks; i++) {
+		conf->disks[i].extra_page = alloc_page(GFP_KERNEL);
+		if (!conf->disks[i].extra_page)
+			goto abort;
+	}
+
 	conf->mddev = mddev;
 
 	if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
@@ -6545,6 +6684,11 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
 		INIT_LIST_HEAD(conf->temp_inactive_list + i);
 
+	atomic_set(&conf->r5c_cached_full_stripes, 0);
+	INIT_LIST_HEAD(&conf->r5c_full_stripe_list);
+	atomic_set(&conf->r5c_cached_partial_stripes, 0);
+	INIT_LIST_HEAD(&conf->r5c_partial_stripe_list);
+
 	conf->level = mddev->new_level;
 	conf->chunk_sectors = mddev->new_chunk_sectors;
 	if (raid5_alloc_percpu(conf) != 0)
@@ -6571,9 +6715,8 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 
 		if (test_bit(In_sync, &rdev->flags)) {
 			char b[BDEVNAME_SIZE];
-			printk(KERN_INFO "md/raid:%s: device %s operational as raid"
-			       " disk %d\n",
-			       mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
+			pr_info("md/raid:%s: device %s operational as raid disk %d\n",
+				mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
 		} else if (rdev->saved_raid_disk != raid_disk)
 			/* Cannot rely on bitmap to complete recovery */
 			conf->fullsync = 1;
@@ -6607,21 +6750,18 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 			((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4);
 		conf->min_nr_stripes = max(NR_STRIPES, stripes);
 		if (conf->min_nr_stripes != NR_STRIPES)
-			printk(KERN_INFO
-				"md/raid:%s: force stripe size %d for reshape\n",
+			pr_info("md/raid:%s: force stripe size %d for reshape\n",
 				mdname(mddev), conf->min_nr_stripes);
 	}
 	memory = conf->min_nr_stripes * (sizeof(struct stripe_head) +
 		 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
 	atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
 	if (grow_stripes(conf, conf->min_nr_stripes)) {
-		printk(KERN_ERR
-		       "md/raid:%s: couldn't allocate %dkB for buffers\n",
-		       mdname(mddev), memory);
+		pr_warn("md/raid:%s: couldn't allocate %dkB for buffers\n",
+			mdname(mddev), memory);
 		goto abort;
 	} else
-		printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
-		       mdname(mddev), memory);
+		pr_debug("md/raid:%s: allocated %dkB\n", mdname(mddev), memory);
 	/*
 	 * Losing a stripe head costs more than the time to refill it,
 	 * it reduces the queue depth and so can hurt throughput.
@@ -6633,18 +6773,16 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 	conf->shrinker.batch = 128;
 	conf->shrinker.flags = 0;
 	if (register_shrinker(&conf->shrinker)) {
-		printk(KERN_ERR
-		       "md/raid:%s: couldn't register shrinker.\n",
-		       mdname(mddev));
+		pr_warn("md/raid:%s: couldn't register shrinker.\n",
+			mdname(mddev));
 		goto abort;
 	}
 
 	sprintf(pers_name, "raid%d", mddev->new_level);
 	conf->thread = md_register_thread(raid5d, mddev, pers_name);
 	if (!conf->thread) {
-		printk(KERN_ERR
-		       "md/raid:%s: couldn't allocate thread.\n",
-		       mdname(mddev));
+		pr_warn("md/raid:%s: couldn't allocate thread.\n",
+			mdname(mddev));
 		goto abort;
 	}
 
@@ -6697,9 +6835,8 @@ static int raid5_run(struct mddev *mddev)
 	int first = 1;
 
 	if (mddev->recovery_cp != MaxSector)
-		printk(KERN_NOTICE "md/raid:%s: not clean"
-		       " -- starting background reconstruction\n",
-		       mdname(mddev));
+		pr_notice("md/raid:%s: not clean -- starting background reconstruction\n",
+			  mdname(mddev));
 
 	rdev_for_each(rdev, mddev) {
 		long long diff;
@@ -6742,15 +6879,14 @@ static int raid5_run(struct mddev *mddev)
 		int new_data_disks;
 
 		if (journal_dev) {
-			printk(KERN_ERR "md/raid:%s: don't support reshape with journal - aborting.\n",
-			       mdname(mddev));
+			pr_warn("md/raid:%s: don't support reshape with journal - aborting.\n",
+				mdname(mddev));
 			return -EINVAL;
 		}
 
 		if (mddev->new_level != mddev->level) {
-			printk(KERN_ERR "md/raid:%s: unsupported reshape "
-			       "required - aborting.\n",
-			       mdname(mddev));
+			pr_warn("md/raid:%s: unsupported reshape required - aborting.\n",
+				mdname(mddev));
 			return -EINVAL;
 		}
 		old_disks = mddev->raid_disks - mddev->delta_disks;
@@ -6765,8 +6901,8 @@ static int raid5_run(struct mddev *mddev)
 		chunk_sectors = max(mddev->chunk_sectors, mddev->new_chunk_sectors);
 		new_data_disks = mddev->raid_disks - max_degraded;
 		if (sector_div(here_new, chunk_sectors * new_data_disks)) {
-			printk(KERN_ERR "md/raid:%s: reshape_position not "
-			       "on a stripe boundary\n", mdname(mddev));
+			pr_warn("md/raid:%s: reshape_position not on a stripe boundary\n",
+				mdname(mddev));
 			return -EINVAL;
 		}
 		reshape_offset = here_new * chunk_sectors;
@@ -6787,10 +6923,8 @@ static int raid5_run(struct mddev *mddev)
 			    abs(min_offset_diff) >= mddev->new_chunk_sectors)
 				/* not really in-place - so OK */;
 			else if (mddev->ro == 0) {
-				printk(KERN_ERR "md/raid:%s: in-place reshape "
-				       "must be started in read-only mode "
-				       "- aborting\n",
-				       mdname(mddev));
+				pr_warn("md/raid:%s: in-place reshape must be started in read-only mode - aborting\n",
+					mdname(mddev));
 				return -EINVAL;
 			}
 		} else if (mddev->reshape_backwards
@@ -6799,13 +6933,11 @@ static int raid5_run(struct mddev *mddev)
 		    : (here_new * chunk_sectors >=
 		       here_old * chunk_sectors + (-min_offset_diff))) {
 			/* Reading from the same stripe as writing to - bad */
-			printk(KERN_ERR "md/raid:%s: reshape_position too early for "
-			       "auto-recovery - aborting.\n",
-			       mdname(mddev));
+			pr_warn("md/raid:%s: reshape_position too early for auto-recovery - aborting.\n",
+				mdname(mddev));
 			return -EINVAL;
 		}
-		printk(KERN_INFO "md/raid:%s: reshape will continue\n",
-		       mdname(mddev));
+		pr_debug("md/raid:%s: reshape will continue\n", mdname(mddev));
 		/* OK, we should be able to continue; */
 	} else {
 		BUG_ON(mddev->level != mddev->new_level);
@@ -6824,8 +6956,8 @@ static int raid5_run(struct mddev *mddev)
 
 	if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
 		if (!journal_dev) {
-			pr_err("md/raid:%s: journal disk is missing, force array readonly\n",
-			       mdname(mddev));
+			pr_warn("md/raid:%s: journal disk is missing, force array readonly\n",
+				mdname(mddev));
 			mddev->ro = 1;
 			set_disk_ro(mddev->gendisk, 1);
 		} else if (mddev->recovery_cp == MaxSector)
@@ -6852,8 +6984,7 @@ static int raid5_run(struct mddev *mddev)
 		if (conf->disks[i].replacement &&
 		    conf->reshape_progress != MaxSector) {
 			/* replacements and reshape simply do not mix. */
-			printk(KERN_ERR "md: cannot handle concurrent "
-			       "replacement and reshape.\n");
+			pr_warn("md: cannot handle concurrent replacement and reshape.\n");
 			goto abort;
 		}
 		if (test_bit(In_sync, &rdev->flags)) {
@@ -6895,8 +7026,7 @@ static int raid5_run(struct mddev *mddev)
 	mddev->degraded = calc_degraded(conf);
 
 	if (has_failed(conf)) {
-		printk(KERN_ERR "md/raid:%s: not enough operational devices"
-			" (%d/%d failed)\n",
+		pr_crit("md/raid:%s: not enough operational devices (%d/%d failed)\n",
 			mdname(mddev), mddev->degraded, conf->raid_disks);
 		goto abort;
 	}
@@ -6908,29 +7038,19 @@ static int raid5_run(struct mddev *mddev)
 	if (mddev->degraded > dirty_parity_disks &&
 	    mddev->recovery_cp != MaxSector) {
 		if (mddev->ok_start_degraded)
-			printk(KERN_WARNING
-			       "md/raid:%s: starting dirty degraded array"
-			       " - data corruption possible.\n",
-			       mdname(mddev));
+			pr_crit("md/raid:%s: starting dirty degraded array - data corruption possible.\n",
+				mdname(mddev));
 		else {
-			printk(KERN_ERR
-			       "md/raid:%s: cannot start dirty degraded array.\n",
-			       mdname(mddev));
+			pr_crit("md/raid:%s: cannot start dirty degraded array.\n",
+				mdname(mddev));
 			goto abort;
 		}
 	}
 
-	if (mddev->degraded == 0)
-		printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
-		       " devices, algorithm %d\n", mdname(mddev), conf->level,
-		       mddev->raid_disks-mddev->degraded, mddev->raid_disks,
-		       mddev->new_layout);
-	else
-		printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
-		       " out of %d devices, algorithm %d\n",
-		       mdname(mddev), conf->level,
-		       mddev->raid_disks - mddev->degraded,
-		       mddev->raid_disks, mddev->new_layout);
+	pr_info("md/raid:%s: raid level %d active with %d out of %d devices, algorithm %d\n",
+		mdname(mddev), conf->level,
+		mddev->raid_disks-mddev->degraded, mddev->raid_disks,
+		mddev->new_layout);
 
 	print_raid5_conf(conf);
 
@@ -6950,9 +7070,8 @@ static int raid5_run(struct mddev *mddev)
 		mddev->to_remove = NULL;
 	else if (mddev->kobj.sd &&
 	    sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
-		printk(KERN_WARNING
-		       "raid5: failed to create sysfs attributes for %s\n",
-		       mdname(mddev));
+		pr_warn("raid5: failed to create sysfs attributes for %s\n",
+			mdname(mddev));
 	md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
 
 	if (mddev->queue) {
@@ -6984,6 +7103,15 @@ static int raid5_run(struct mddev *mddev)
 			stripe = (stripe | (stripe-1)) + 1;
 		mddev->queue->limits.discard_alignment = stripe;
 		mddev->queue->limits.discard_granularity = stripe;
+
+		/*
+		 * We use 16-bit counter of active stripes in bi_phys_segments
+		 * (minus one for over-loaded initialization)
+		 */
+		blk_queue_max_hw_sectors(mddev->queue, 0xfffe * STRIPE_SECTORS);
+		blk_queue_max_discard_sectors(mddev->queue,
+					      0xfffe * STRIPE_SECTORS);
+
 		/*
 		 * unaligned part of discard request will be ignored, so can't
 		 * guarantee discard_zeroes_data
@@ -7040,9 +7168,10 @@ static int raid5_run(struct mddev *mddev)
 	if (journal_dev) {
 		char b[BDEVNAME_SIZE];
 
-		printk(KERN_INFO"md/raid:%s: using device %s as journal\n",
-		       mdname(mddev), bdevname(journal_dev->bdev, b));
-		r5l_init_log(conf, journal_dev);
+		pr_debug("md/raid:%s: using device %s as journal\n",
+			 mdname(mddev), bdevname(journal_dev->bdev, b));
+		if (r5l_init_log(conf, journal_dev))
+			goto abort;
 	}
 
 	return 0;
@@ -7051,7 +7180,7 @@ abort:
 	print_raid5_conf(conf);
 	free_conf(conf);
 	mddev->private = NULL;
-	printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
+	pr_warn("md/raid:%s: failed to run raid set.\n", mdname(mddev));
 	return -EIO;
 }
 
@@ -7085,12 +7214,12 @@ static void print_raid5_conf (struct r5conf *conf)
 	int i;
 	struct disk_info *tmp;
 
-	printk(KERN_DEBUG "RAID conf printout:\n");
+	pr_debug("RAID conf printout:\n");
 	if (!conf) {
-		printk("(conf==NULL)\n");
+		pr_debug("(conf==NULL)\n");
 		return;
 	}
-	printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
+	pr_debug(" --- level:%d rd:%d wd:%d\n", conf->level,
 	       conf->raid_disks,
 	       conf->raid_disks - conf->mddev->degraded);
 
@@ -7098,7 +7227,7 @@ static void print_raid5_conf (struct r5conf *conf)
 		char b[BDEVNAME_SIZE];
 		tmp = conf->disks + i;
 		if (tmp->rdev)
-			printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
+			pr_debug(" disk %d, o:%d, dev:%s\n",
 			       i, !test_bit(Faulty, &tmp->rdev->flags),
 			       bdevname(tmp->rdev->bdev, b));
 	}
@@ -7246,8 +7375,8 @@ static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
 		 * write requests running. We should be safe
 		 */
 		r5l_init_log(conf, rdev);
-		printk(KERN_INFO"md/raid:%s: using device %s as journal\n",
-		       mdname(mddev), bdevname(rdev->bdev, b));
+		pr_debug("md/raid:%s: using device %s as journal\n",
+			 mdname(mddev), bdevname(rdev->bdev, b));
 		return 0;
 	}
 	if (mddev->recovery_disabled == conf->recovery_disabled)
@@ -7351,10 +7480,10 @@ static int check_stripe_cache(struct mddev *mddev)
 	    > conf->min_nr_stripes ||
 	    ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
 	    > conf->min_nr_stripes) {
-		printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes.  Needed %lu\n",
-		       mdname(mddev),
-		       ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
-			/ STRIPE_SIZE)*4);
+		pr_warn("md/raid:%s: reshape: not enough stripes.  Needed %lu\n",
+			mdname(mddev),
+			((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
+			 / STRIPE_SIZE)*4);
 		return 0;
 	}
 	return 1;
@@ -7435,8 +7564,8 @@ static int raid5_start_reshape(struct mddev *mddev)
 	 */
 	if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
 	    < mddev->array_sectors) {
-		printk(KERN_ERR "md/raid:%s: array size must be reduced "
-		       "before number of disks\n", mdname(mddev));
+		pr_warn("md/raid:%s: array size must be reduced before number of disks\n",
+			mdname(mddev));
 		return -EINVAL;
 	}
 
@@ -7506,7 +7635,7 @@ static int raid5_start_reshape(struct mddev *mddev)
 	}
 	mddev->raid_disks = conf->raid_disks;
 	mddev->reshape_position = conf->reshape_progress;
-	set_bit(MD_CHANGE_DEVS, &mddev->flags);
+	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 
 	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
 	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
@@ -7624,6 +7753,7 @@ static void raid5_quiesce(struct mddev *mddev, int state)
 		/* '2' tells resync/reshape to pause so that all
 		 * active stripes can drain
 		 */
+		r5c_flush_cache(conf, INT_MAX);
 		conf->quiesce = 2;
 		wait_event_cmd(conf->wait_for_quiescent,
 				    atomic_read(&conf->active_stripes) == 0 &&
@@ -7654,8 +7784,8 @@ static void *raid45_takeover_raid0(struct mddev *mddev, int level)
 
 	/* for raid0 takeover only one zone is supported */
 	if (raid0_conf->nr_strip_zones > 1) {
-		printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
-		       mdname(mddev));
+		pr_warn("md/raid:%s: cannot takeover raid0 with more than one zone.\n",
+			mdname(mddev));
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -7676,6 +7806,7 @@ static void *raid45_takeover_raid0(struct mddev *mddev, int level)
 static void *raid5_takeover_raid1(struct mddev *mddev)
 {
 	int chunksect;
+	void *ret;
 
 	if (mddev->raid_disks != 2 ||
 	    mddev->degraded > 1)
@@ -7697,7 +7828,10 @@ static void *raid5_takeover_raid1(struct mddev *mddev)
 	mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
 	mddev->new_chunk_sectors = chunksect;
 
-	return setup_conf(mddev);
+	ret = setup_conf(mddev);
+	if (!IS_ERR_VALUE(ret))
+		clear_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
+	return ret;
 }
 
 static void *raid5_takeover_raid6(struct mddev *mddev)
@@ -7767,7 +7901,7 @@ static int raid5_check_reshape(struct mddev *mddev)
 			conf->chunk_sectors = new_chunk ;
 			mddev->chunk_sectors = new_chunk;
 		}
-		set_bit(MD_CHANGE_DEVS, &mddev->flags);
+		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 		md_wakeup_thread(mddev->thread);
 	}
 	return check_reshape(mddev);
diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h
index 57ec49f0839e..ed8e1362ab36 100644
--- a/drivers/md/raid5.h
+++ b/drivers/md/raid5.h
@@ -226,6 +226,8 @@ struct stripe_head {
 
 	struct r5l_io_unit	*log_io;
 	struct list_head	log_list;
+	sector_t		log_start; /* first meta block on the journal */
+	struct list_head	r5c; /* for r5c_cache->stripe_in_journal */
 	/**
 	 * struct stripe_operations
 	 * @target - STRIPE_OP_COMPUTE_BLK target
@@ -264,6 +266,7 @@ struct stripe_head_state {
 	int syncing, expanding, expanded, replacing;
 	int locked, uptodate, to_read, to_write, failed, written;
 	int to_fill, compute, req_compute, non_overwrite;
+	int injournal, just_cached;
 	int failed_num[2];
 	int p_failed, q_failed;
 	int dec_preread_active;
@@ -273,6 +276,7 @@ struct stripe_head_state {
 	struct md_rdev *blocked_rdev;
 	int handle_bad_blocks;
 	int log_failed;
+	int waiting_extra_page;
 };
 
 /* Flags for struct r5dev.flags */
@@ -313,6 +317,11 @@ enum r5dev_flags {
 			 */
 	R5_Discard,	/* Discard the stripe */
 	R5_SkipCopy,	/* Don't copy data from bio to stripe cache */
+	R5_InJournal,	/* data being written is in the journal device.
+			 * if R5_InJournal is set for parity pd_idx, all the
+			 * data and parity being written are in the journal
+			 * device
+			 */
 };
 
 /*
@@ -345,7 +354,30 @@ enum {
 	STRIPE_BITMAP_PENDING,	/* Being added to bitmap, don't add
 				 * to batch yet.
 				 */
-	STRIPE_LOG_TRAPPED, /* trapped into log */
+	STRIPE_LOG_TRAPPED,	/* trapped into log (see raid5-cache.c)
+				 * this bit is used in two scenarios:
+				 *
+				 * 1. write-out phase
+				 *  set in first entry of r5l_write_stripe
+				 *  clear in second entry of r5l_write_stripe
+				 *  used to bypass logic in handle_stripe
+				 *
+				 * 2. caching phase
+				 *  set in r5c_try_caching_write()
+				 *  clear when journal write is done
+				 *  used to initiate r5c_cache_data()
+				 *  also used to bypass logic in handle_stripe
+				 */
+	STRIPE_R5C_CACHING,	/* the stripe is in caching phase
+				 * see more detail in the raid5-cache.c
+				 */
+	STRIPE_R5C_PARTIAL_STRIPE,	/* in r5c cache (to-be/being handled or
+					 * in conf->r5c_partial_stripe_list)
+					 */
+	STRIPE_R5C_FULL_STRIPE,	/* in r5c cache (to-be/being handled or
+				 * in conf->r5c_full_stripe_list)
+				 */
+	STRIPE_R5C_PREFLUSH,	/* need to flush journal device */
 };
 
 #define STRIPE_EXPAND_SYNC_FLAGS \
@@ -408,8 +440,86 @@ enum {
 
 struct disk_info {
 	struct md_rdev	*rdev, *replacement;
+	struct page	*extra_page; /* extra page to use in prexor */
 };
 
+/*
+ * Stripe cache
+ */
+
+#define NR_STRIPES		256
+#define STRIPE_SIZE		PAGE_SIZE
+#define STRIPE_SHIFT		(PAGE_SHIFT - 9)
+#define STRIPE_SECTORS		(STRIPE_SIZE>>9)
+#define	IO_THRESHOLD		1
+#define BYPASS_THRESHOLD	1
+#define NR_HASH			(PAGE_SIZE / sizeof(struct hlist_head))
+#define HASH_MASK		(NR_HASH - 1)
+#define MAX_STRIPE_BATCH	8
+
+/* bio's attached to a stripe+device for I/O are linked together in bi_sector
+ * order without overlap.  There may be several bio's per stripe+device, and
+ * a bio could span several devices.
+ * When walking this list for a particular stripe+device, we must never proceed
+ * beyond a bio that extends past this device, as the next bio might no longer
+ * be valid.
+ * This function is used to determine the 'next' bio in the list, given the
+ * sector of the current stripe+device
+ */
+static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
+{
+	int sectors = bio_sectors(bio);
+
+	if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS)
+		return bio->bi_next;
+	else
+		return NULL;
+}
+
+/*
+ * We maintain a biased count of active stripes in the bottom 16 bits of
+ * bi_phys_segments, and a count of processed stripes in the upper 16 bits
+ */
+static inline int raid5_bi_processed_stripes(struct bio *bio)
+{
+	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
+
+	return (atomic_read(segments) >> 16) & 0xffff;
+}
+
+static inline int raid5_dec_bi_active_stripes(struct bio *bio)
+{
+	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
+
+	return atomic_sub_return(1, segments) & 0xffff;
+}
+
+static inline void raid5_inc_bi_active_stripes(struct bio *bio)
+{
+	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
+
+	atomic_inc(segments);
+}
+
+static inline void raid5_set_bi_processed_stripes(struct bio *bio,
+	unsigned int cnt)
+{
+	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
+	int old, new;
+
+	do {
+		old = atomic_read(segments);
+		new = (old & 0xffff) | (cnt << 16);
+	} while (atomic_cmpxchg(segments, old, new) != old);
+}
+
+static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
+{
+	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
+
+	atomic_set(segments, cnt);
+}
+
 /* NOTE NR_STRIPE_HASH_LOCKS must remain below 64.
  * This is because we sometimes take all the spinlocks
  * and creating that much locking depth can cause
@@ -432,6 +542,30 @@ struct r5worker_group {
 	int stripes_cnt;
 };
 
+enum r5_cache_state {
+	R5_INACTIVE_BLOCKED,	/* release of inactive stripes blocked,
+				 * waiting for 25% to be free
+				 */
+	R5_ALLOC_MORE,		/* It might help to allocate another
+				 * stripe.
+				 */
+	R5_DID_ALLOC,		/* A stripe was allocated, don't allocate
+				 * more until at least one has been
+				 * released.  This avoids flooding
+				 * the cache.
+				 */
+	R5C_LOG_TIGHT,		/* log device space tight, need to
+				 * prioritize stripes at last_checkpoint
+				 */
+	R5C_LOG_CRITICAL,	/* log device is running out of space,
+				 * only process stripes that are already
+				 * occupying the log
+				 */
+	R5C_EXTRA_PAGE_IN_USE,	/* a stripe is using disk_info.extra_page
+				 * for prexor
+				 */
+};
+
 struct r5conf {
 	struct hlist_head	*stripe_hashtbl;
 	/* only protect corresponding hash list and inactive_list */
@@ -519,23 +653,18 @@ struct r5conf {
 	 */
 	atomic_t		active_stripes;
 	struct list_head	inactive_list[NR_STRIPE_HASH_LOCKS];
+
+	atomic_t		r5c_cached_full_stripes;
+	struct list_head	r5c_full_stripe_list;
+	atomic_t		r5c_cached_partial_stripes;
+	struct list_head	r5c_partial_stripe_list;
+
 	atomic_t		empty_inactive_list_nr;
 	struct llist_head	released_stripes;
 	wait_queue_head_t	wait_for_quiescent;
 	wait_queue_head_t	wait_for_stripe;
 	wait_queue_head_t	wait_for_overlap;
 	unsigned long		cache_state;
-#define R5_INACTIVE_BLOCKED	1	/* release of inactive stripes blocked,
-					 * waiting for 25% to be free
-					 */
-#define R5_ALLOC_MORE		2	/* It might help to allocate another
-					 * stripe.
-					 */
-#define R5_DID_ALLOC		4	/* A stripe was allocated, don't allocate
-					 * more until at least one has been
-					 * released.  This avoids flooding
-					 * the cache.
-					 */
 	struct shrinker		shrinker;
 	int			pool_size; /* number of disks in stripeheads in pool */
 	spinlock_t		device_lock;
@@ -633,4 +762,23 @@ extern void r5l_stripe_write_finished(struct stripe_head *sh);
 extern int r5l_handle_flush_request(struct r5l_log *log, struct bio *bio);
 extern void r5l_quiesce(struct r5l_log *log, int state);
 extern bool r5l_log_disk_error(struct r5conf *conf);
+extern bool r5c_is_writeback(struct r5l_log *log);
+extern int
+r5c_try_caching_write(struct r5conf *conf, struct stripe_head *sh,
+		      struct stripe_head_state *s, int disks);
+extern void
+r5c_finish_stripe_write_out(struct r5conf *conf, struct stripe_head *sh,
+			    struct stripe_head_state *s);
+extern void r5c_release_extra_page(struct stripe_head *sh);
+extern void r5c_use_extra_page(struct stripe_head *sh);
+extern void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
+extern void r5c_handle_cached_data_endio(struct r5conf *conf,
+	struct stripe_head *sh, int disks, struct bio_list *return_bi);
+extern int r5c_cache_data(struct r5l_log *log, struct stripe_head *sh,
+			  struct stripe_head_state *s);
+extern void r5c_make_stripe_write_out(struct stripe_head *sh);
+extern void r5c_flush_cache(struct r5conf *conf, int num);
+extern void r5c_check_stripe_cache_usage(struct r5conf *conf);
+extern void r5c_check_cached_full_stripe(struct r5conf *conf);
+extern struct md_sysfs_entry r5c_journal_mode;
 #endif
diff --git a/drivers/media/usb/dvb-usb-v2/af9015.c b/drivers/media/usb/dvb-usb-v2/af9015.c
index 941ceff9b268..29011dfabb11 100644
--- a/drivers/media/usb/dvb-usb-v2/af9015.c
+++ b/drivers/media/usb/dvb-usb-v2/af9015.c
@@ -1455,7 +1455,7 @@ static const struct usb_device_id af9015_id_table[] = {
 	{ DVB_USB_DEVICE(USB_VID_KWORLD_2, USB_PID_CONCEPTRONIC_CTVDIGRCU,
 		&af9015_props, "Conceptronic USB2.0 DVB-T CTVDIGRCU V3.0", NULL) },
 	{ DVB_USB_DEVICE(USB_VID_KWORLD_2, USB_PID_KWORLD_MC810,
-		&af9015_props, "KWorld Digial MC-810", NULL) },
+		&af9015_props, "KWorld Digital MC-810", NULL) },
 	{ DVB_USB_DEVICE(USB_VID_KYE, USB_PID_GENIUS_TVGO_DVB_T03,
 		&af9015_props, "Genius TVGo DVB-T03", NULL) },
 	{ DVB_USB_DEVICE(USB_VID_KWORLD_2, USB_PID_KWORLD_399U_2,
diff --git a/drivers/media/usb/usbtv/usbtv-core.c b/drivers/media/usb/usbtv/usbtv-core.c
index dc76fd41e00f..ceb953be0770 100644
--- a/drivers/media/usb/usbtv/usbtv-core.c
+++ b/drivers/media/usb/usbtv/usbtv-core.c
@@ -71,6 +71,7 @@ static int usbtv_probe(struct usb_interface *intf,
 	int size;
 	struct device *dev = &intf->dev;
 	struct usbtv *usbtv;
+	struct usb_host_endpoint *ep;
 
 	/* Checks that the device is what we think it is. */
 	if (intf->num_altsetting != 2)
@@ -78,10 +79,12 @@ static int usbtv_probe(struct usb_interface *intf,
 	if (intf->altsetting[1].desc.bNumEndpoints != 4)
 		return -ENODEV;
 
+	ep = &intf->altsetting[1].endpoint[0];
+
 	/* Packet size is split into 11 bits of base size and count of
 	 * extra multiplies of it.*/
-	size = usb_endpoint_maxp(&intf->altsetting[1].endpoint[0].desc);
-	size = (size & 0x07ff) * (((size & 0x1800) >> 11) + 1);
+	size = usb_endpoint_maxp(&ep->desc);
+	size = (size & 0x07ff) * usb_endpoint_maxp_mult(&ep->desc);
 
 	/* Device structure */
 	usbtv = kzalloc(sizeof(struct usbtv), GFP_KERNEL);
diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c
index b5589d5f5da4..f3c1c852e401 100644
--- a/drivers/media/usb/uvc/uvc_video.c
+++ b/drivers/media/usb/uvc/uvc_video.c
@@ -1467,6 +1467,7 @@ static unsigned int uvc_endpoint_max_bpi(struct usb_device *dev,
 					 struct usb_host_endpoint *ep)
 {
 	u16 psize;
+	u16 mult;
 
 	switch (dev->speed) {
 	case USB_SPEED_SUPER:
@@ -1474,7 +1475,8 @@ static unsigned int uvc_endpoint_max_bpi(struct usb_device *dev,
 		return le16_to_cpu(ep->ss_ep_comp.wBytesPerInterval);
 	case USB_SPEED_HIGH:
 		psize = usb_endpoint_maxp(&ep->desc);
-		return (psize & 0x07ff) * (1 + ((psize >> 11) & 3));
+		mult = usb_endpoint_maxp_mult(&ep->desc);
+		return (psize & 0x07ff) * mult;
 	case USB_SPEED_WIRELESS:
 		psize = usb_endpoint_maxp(&ep->desc);
 		return psize;
@@ -1551,7 +1553,7 @@ static int uvc_init_video_bulk(struct uvc_streaming *stream,
 	u16 psize;
 	u32 size;
 
-	psize = usb_endpoint_maxp(&ep->desc) & 0x7ff;
+	psize = usb_endpoint_maxp(&ep->desc);
 	size = stream->ctrl.dwMaxPayloadTransferSize;
 	stream->bulk.max_payload_size = size;
 
diff --git a/drivers/media/v4l2-core/videobuf-dma-sg.c b/drivers/media/v4l2-core/videobuf-dma-sg.c
index 1db0af6c7f94..ba63ca57ed7e 100644
--- a/drivers/media/v4l2-core/videobuf-dma-sg.c
+++ b/drivers/media/v4l2-core/videobuf-dma-sg.c
@@ -439,13 +439,12 @@ static int videobuf_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	struct page *page;
 
 	dprintk(3, "fault: fault @ %08lx [vma %08lx-%08lx]\n",
-		(unsigned long)vmf->virtual_address,
-		vma->vm_start, vma->vm_end);
+		vmf->address, vma->vm_start, vma->vm_end);
 
 	page = alloc_page(GFP_USER | __GFP_DMA32);
 	if (!page)
 		return VM_FAULT_OOM;
-	clear_user_highpage(page, (unsigned long)vmf->virtual_address);
+	clear_user_highpage(page, vmf->address);
 	vmf->page = page;
 
 	return 0;
diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig
index 4b4c0c3c3d2f..ec80e35c8dfe 100644
--- a/drivers/memory/Kconfig
+++ b/drivers/memory/Kconfig
@@ -134,6 +134,14 @@ config MTK_SMI
 	  mainly help enable/disable iommu and control the power domain and
 	  clocks for each local arbiter.
 
+config DA8XX_DDRCTL
+	bool "Texas Instruments da8xx DDR2/mDDR driver"
+	depends on ARCH_DAVINCI_DA8XX
+	help
+	  This driver is for the DDR2/mDDR Memory Controller present on
+	  Texas Instruments da8xx SoCs. It's used to tweak various memory
+	  controller configuration options.
+
 source "drivers/memory/samsung/Kconfig"
 source "drivers/memory/tegra/Kconfig"
 
diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile
index b20ae38b5bfb..e88097fbc085 100644
--- a/drivers/memory/Makefile
+++ b/drivers/memory/Makefile
@@ -17,6 +17,7 @@ obj-$(CONFIG_MVEBU_DEVBUS)	+= mvebu-devbus.o
 obj-$(CONFIG_TEGRA20_MC)	+= tegra20-mc.o
 obj-$(CONFIG_JZ4780_NEMC)	+= jz4780-nemc.o
 obj-$(CONFIG_MTK_SMI)		+= mtk-smi.o
+obj-$(CONFIG_DA8XX_DDRCTL)	+= da8xx-ddrctl.o
 
 obj-$(CONFIG_SAMSUNG_MC)	+= samsung/
 obj-$(CONFIG_TEGRA_MC)		+= tegra/
diff --git a/drivers/memory/atmel-ebi.c b/drivers/memory/atmel-ebi.c
index b5ed3bd082b5..047d6fcdcec2 100644
--- a/drivers/memory/atmel-ebi.c
+++ b/drivers/memory/atmel-ebi.c
@@ -657,7 +657,7 @@ static int at91_ebi_dev_disable(struct at91_ebi *ebi, struct device_node *np)
 		return -ENOMEM;
 
 	newprop->value = devm_kstrdup(dev, "disabled", GFP_KERNEL);
-	if (!newprop->name)
+	if (!newprop->value)
 		return -ENOMEM;
 
 	newprop->length = sizeof("disabled");
diff --git a/drivers/memory/atmel-sdramc.c b/drivers/memory/atmel-sdramc.c
index 12080b05e3e6..b418b39af180 100644
--- a/drivers/memory/atmel-sdramc.c
+++ b/drivers/memory/atmel-sdramc.c
@@ -85,8 +85,4 @@ static struct platform_driver atmel_ramc_driver = {
 	},
 };
 
-static int __init atmel_ramc_init(void)
-{
-	return platform_driver_register(&atmel_ramc_driver);
-}
-device_initcall(atmel_ramc_init);
+builtin_platform_driver(atmel_ramc_driver);
diff --git a/drivers/memory/da8xx-ddrctl.c b/drivers/memory/da8xx-ddrctl.c
new file mode 100644
index 000000000000..030afbe29d0c
--- /dev/null
+++ b/drivers/memory/da8xx-ddrctl.c
@@ -0,0 +1,173 @@
+/*
+ * TI da8xx DDR2/mDDR controller driver
+ *
+ * Copyright (C) 2016 BayLibre SAS
+ *
+ * Author:
+ *   Bartosz Golaszewski <bgolaszewski@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+
+/*
+ * REVISIT: Linux doesn't have a good framework for the kind of performance
+ * knobs this driver controls. We can't use device tree properties as it deals
+ * with hardware configuration rather than description. We also don't want to
+ * commit to maintaining some random sysfs attributes.
+ *
+ * For now we just hardcode the register values for the boards that need
+ * some changes (as is the case for the LCD controller on da850-lcdk - the
+ * first board we support here). When linux gets an appropriate framework,
+ * we'll easily convert the driver to it.
+ */
+
+struct da8xx_ddrctl_config_knob {
+	const char *name;
+	u32 reg;
+	u32 mask;
+	u32 shift;
+};
+
+static const struct da8xx_ddrctl_config_knob da8xx_ddrctl_knobs[] = {
+	{
+		.name = "da850-pbbpr",
+		.reg = 0x20,
+		.mask = 0xffffff00,
+		.shift = 0,
+	},
+};
+
+struct da8xx_ddrctl_setting {
+	const char *name;
+	u32 val;
+};
+
+struct da8xx_ddrctl_board_settings {
+	const char *board;
+	const struct da8xx_ddrctl_setting *settings;
+};
+
+static const struct da8xx_ddrctl_setting da850_lcdk_ddrctl_settings[] = {
+	{
+		.name = "da850-pbbpr",
+		.val = 0x20,
+	},
+	{ }
+};
+
+static const struct da8xx_ddrctl_board_settings da8xx_ddrctl_board_confs[] = {
+	{
+		.board = "ti,da850-lcdk",
+		.settings = da850_lcdk_ddrctl_settings,
+	},
+};
+
+static const struct da8xx_ddrctl_config_knob *
+da8xx_ddrctl_match_knob(const struct da8xx_ddrctl_setting *setting)
+{
+	const struct da8xx_ddrctl_config_knob *knob;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(da8xx_ddrctl_knobs); i++) {
+		knob = &da8xx_ddrctl_knobs[i];
+
+		if (strcmp(knob->name, setting->name) == 0)
+			return knob;
+	}
+
+	return NULL;
+}
+
+static const struct da8xx_ddrctl_setting *da8xx_ddrctl_get_board_settings(void)
+{
+	const struct da8xx_ddrctl_board_settings *board_settings;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(da8xx_ddrctl_board_confs); i++) {
+		board_settings = &da8xx_ddrctl_board_confs[i];
+
+		if (of_machine_is_compatible(board_settings->board))
+			return board_settings->settings;
+	}
+
+	return NULL;
+}
+
+static int da8xx_ddrctl_probe(struct platform_device *pdev)
+{
+	const struct da8xx_ddrctl_config_knob *knob;
+	const struct da8xx_ddrctl_setting *setting;
+	struct device_node *node;
+	struct resource *res;
+	void __iomem *ddrctl;
+	struct device *dev;
+	u32 reg;
+
+	dev = &pdev->dev;
+	node = dev->of_node;
+
+	setting = da8xx_ddrctl_get_board_settings();
+	if (!setting) {
+		dev_err(dev, "no settings defined for this board\n");
+		return -EINVAL;
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	ddrctl = devm_ioremap_resource(dev, res);
+	if (IS_ERR(ddrctl)) {
+		dev_err(dev, "unable to map memory controller registers\n");
+		return PTR_ERR(ddrctl);
+	}
+
+	for (; setting->name; setting++) {
+		knob = da8xx_ddrctl_match_knob(setting);
+		if (!knob) {
+			dev_warn(dev,
+				 "no such config option: %s\n", setting->name);
+			continue;
+		}
+
+		if (knob->reg + sizeof(u32) > resource_size(res)) {
+			dev_warn(dev,
+				 "register offset of '%s' exceeds mapped memory size\n",
+				 knob->name);
+			continue;
+		}
+
+		reg = readl(ddrctl + knob->reg);
+		reg &= knob->mask;
+		reg |= setting->val << knob->shift;
+
+		dev_dbg(dev, "writing 0x%08x to %s\n", reg, setting->name);
+
+		writel(reg, ddrctl + knob->reg);
+	}
+
+	return 0;
+}
+
+static const struct of_device_id da8xx_ddrctl_of_match[] = {
+	{ .compatible = "ti,da850-ddr-controller", },
+	{ },
+};
+
+static struct platform_driver da8xx_ddrctl_driver = {
+	.probe = da8xx_ddrctl_probe,
+	.driver = {
+		.name = "da850-ddr-controller",
+		.of_match_table = da8xx_ddrctl_of_match,
+	},
+};
+module_platform_driver(da8xx_ddrctl_driver);
+
+MODULE_AUTHOR("Bartosz Golaszewski <bgolaszewski@baylibre.com>");
+MODULE_DESCRIPTION("TI da8xx DDR2/mDDR controller driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/memstick/core/ms_block.c b/drivers/memstick/core/ms_block.c
index aacf584f2a42..f3512404bc52 100644
--- a/drivers/memstick/core/ms_block.c
+++ b/drivers/memstick/core/ms_block.c
@@ -2006,7 +2006,7 @@ static int msb_prepare_req(struct request_queue *q, struct request *req)
 		blk_dump_rq_flags(req, "MS unsupported request");
 		return BLKPREP_KILL;
 	}
-	req->cmd_flags |= REQ_DONTPREP;
+	req->rq_flags |= RQF_DONTPREP;
 	return BLKPREP_OK;
 }
 
diff --git a/drivers/memstick/core/mspro_block.c b/drivers/memstick/core/mspro_block.c
index c1472275fe57..fa0746d182ff 100644
--- a/drivers/memstick/core/mspro_block.c
+++ b/drivers/memstick/core/mspro_block.c
@@ -834,7 +834,7 @@ static int mspro_block_prepare_req(struct request_queue *q, struct request *req)
 		return BLKPREP_KILL;
 	}
 
-	req->cmd_flags |= REQ_DONTPREP;
+	req->rq_flags |= RQF_DONTPREP;
 
 	return BLKPREP_OK;
 }
diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c
index 89c7ed16b4df..1e73064b0fb2 100644
--- a/drivers/message/fusion/mptbase.c
+++ b/drivers/message/fusion/mptbase.c
@@ -2585,10 +2585,7 @@ mpt_do_ioc_recovery(MPT_ADAPTER *ioc, u32 reason, int sleepFlag)
 				(void) GetLanConfigPages(ioc);
 				a = (u8*)&ioc->lan_cnfg_page1.HardwareAddressLow;
 				dprintk(ioc, printk(MYIOC_s_DEBUG_FMT
-					"LanAddr = %02X:%02X:%02X"
-					":%02X:%02X:%02X\n",
-					ioc->name, a[5], a[4],
-					a[3], a[2], a[1], a[0]));
+					"LanAddr = %pMR\n", ioc->name, a));
 			}
 			break;
 
@@ -2868,21 +2865,21 @@ MptDisplayIocCapabilities(MPT_ADAPTER *ioc)
 
 	printk(KERN_INFO "%s: ", ioc->name);
 	if (ioc->prod_name)
-		printk("%s: ", ioc->prod_name);
-	printk("Capabilities={");
+		pr_cont("%s: ", ioc->prod_name);
+	pr_cont("Capabilities={");
 
 	if (ioc->pfacts[0].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_INITIATOR) {
-		printk("Initiator");
+		pr_cont("Initiator");
 		i++;
 	}
 
 	if (ioc->pfacts[0].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_TARGET) {
-		printk("%sTarget", i ? "," : "");
+		pr_cont("%sTarget", i ? "," : "");
 		i++;
 	}
 
 	if (ioc->pfacts[0].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_LAN) {
-		printk("%sLAN", i ? "," : "");
+		pr_cont("%sLAN", i ? "," : "");
 		i++;
 	}
 
@@ -2891,12 +2888,12 @@ MptDisplayIocCapabilities(MPT_ADAPTER *ioc)
 	 *  This would probably evoke more questions than it's worth
 	 */
 	if (ioc->pfacts[0].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_TARGET) {
-		printk("%sLogBusAddr", i ? "," : "");
+		pr_cont("%sLogBusAddr", i ? "," : "");
 		i++;
 	}
 #endif
 
-	printk("}\n");
+	pr_cont("}\n");
 }
 
 /*=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=*/
@@ -6783,8 +6780,7 @@ static int mpt_iocinfo_proc_show(struct seq_file *m, void *v)
 		if (ioc->bus_type == FC) {
 			if (ioc->pfacts[p].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_LAN) {
 				u8 *a = (u8*)&ioc->lan_cnfg_page1.HardwareAddressLow;
-				seq_printf(m, "    LanAddr = %02X:%02X:%02X:%02X:%02X:%02X\n",
-						a[5], a[4], a[3], a[2], a[1], a[0]);
+				seq_printf(m, "    LanAddr = %pMR\n", a);
 			}
 			seq_printf(m, "    WWN = %08X%08X:%08X%08X\n",
 					ioc->fc_port_page0[p].WWNN.High,
@@ -6861,8 +6857,7 @@ mpt_print_ioc_summary(MPT_ADAPTER *ioc, char *buffer, int *size, int len, int sh
 
 	if (showlan && (ioc->pfacts[0].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_LAN)) {
 		u8 *a = (u8*)&ioc->lan_cnfg_page1.HardwareAddressLow;
-		y += sprintf(buffer+len+y, ", LanAddr=%02X:%02X:%02X:%02X:%02X:%02X",
-			a[5], a[4], a[3], a[2], a[1], a[0]);
+		y += sprintf(buffer+len+y, ", LanAddr=%pMR", a);
 	}
 
 	y += sprintf(buffer+len+y, ", IRQ=%d", ioc->pci_irq);
@@ -6896,8 +6891,7 @@ static void seq_mpt_print_ioc_summary(MPT_ADAPTER *ioc, struct seq_file *m, int
 
 	if (showlan && (ioc->pfacts[0].ProtocolFlags & MPI_PORTFACTS_PROTOCOL_LAN)) {
 		u8 *a = (u8*)&ioc->lan_cnfg_page1.HardwareAddressLow;
-		seq_printf(m, ", LanAddr=%02X:%02X:%02X:%02X:%02X:%02X",
-			a[5], a[4], a[3], a[2], a[1], a[0]);
+		seq_printf(m, ", LanAddr=%pMR", a);
 	}
 
 	seq_printf(m, ", IRQ=%d", ioc->pci_irq);
diff --git a/drivers/message/fusion/mptscsih.c b/drivers/message/fusion/mptscsih.c
index 6c9fc11efb87..08a807d6a44f 100644
--- a/drivers/message/fusion/mptscsih.c
+++ b/drivers/message/fusion/mptscsih.c
@@ -1366,15 +1366,10 @@ mptscsih_qcmd(struct scsi_cmnd *SCpnt)
 	/* Default to untagged. Once a target structure has been allocated,
 	 * use the Inquiry data to determine if device supports tagged.
 	 */
-	if ((vdevice->vtarget->tflags & MPT_TARGET_FLAGS_Q_YES)
-	    && (SCpnt->device->tagged_supported)) {
+	if ((vdevice->vtarget->tflags & MPT_TARGET_FLAGS_Q_YES) &&
+	    SCpnt->device->tagged_supported)
 		scsictl = scsidir | MPI_SCSIIO_CONTROL_SIMPLEQ;
-		if (SCpnt->request && SCpnt->request->ioprio) {
-			if (((SCpnt->request->ioprio & 0x7) == 1) ||
-				!(SCpnt->request->ioprio & 0x7))
-				scsictl |= MPI_SCSIIO_CONTROL_HEADOFQ;
-		}
-	} else
+	else
 		scsictl = scsidir | MPI_SCSIIO_CONTROL_UNTAGGED;
 
 
diff --git a/drivers/mfd/88pm860x-core.c b/drivers/mfd/88pm860x-core.c
index 25e1aafae60c..227b99018657 100644
--- a/drivers/mfd/88pm860x-core.c
+++ b/drivers/mfd/88pm860x-core.c
@@ -1132,8 +1132,7 @@ static int pm860x_dt_init(struct device_node *np,
 	return 0;
 }
 
-static int pm860x_probe(struct i2c_client *client,
-				  const struct i2c_device_id *id)
+static int pm860x_probe(struct i2c_client *client)
 {
 	struct pm860x_platform_data *pdata = dev_get_platdata(&client->dev);
 	struct device_node *node = client->dev.of_node;
@@ -1259,7 +1258,7 @@ static struct i2c_driver pm860x_driver = {
 		.pm     = &pm860x_pm_ops,
 		.of_match_table	= pm860x_dt_ids,
 	},
-	.probe		= pm860x_probe,
+	.probe_new	= pm860x_probe,
 	.remove		= pm860x_remove,
 	.id_table	= pm860x_id_table,
 };
diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig
index c6df6442ba2b..1ed0584f494e 100644
--- a/drivers/mfd/Kconfig
+++ b/drivers/mfd/Kconfig
@@ -756,24 +756,20 @@ config UCB1400_CORE
 	  module will be called ucb1400_core.
 
 config MFD_PM8XXX
-	tristate
-
-config MFD_PM8921_CORE
-	tristate "Qualcomm PM8921 PMIC chip"
+	tristate "Qualcomm PM8xxx PMIC chips driver"
 	depends on (ARM || HEXAGON)
 	select IRQ_DOMAIN
 	select MFD_CORE
-	select MFD_PM8XXX
 	select REGMAP
 	help
 	  If you say yes to this option, support will be included for the
-	  built-in PM8921 PMIC chip.
+	  built-in PM8xxx PMIC chips.
 
-	  This is required if your board has a PM8921 and uses its features,
+	  This is required if your board has a PM8xxx and uses its features,
 	  such as: MPPs, GPIOs, regulators, interrupts, and PWM.
 
-	  Say M here if you want to include support for PM8921 chip as a module.
-	  This will build a module called "pm8921-core".
+	  Say M here if you want to include support for PM8xxx chips as a
+	  module. This will build a module called "pm8xxx-core".
 
 config MFD_QCOM_RPM
 	tristate "Qualcomm Resource Power Manager (RPM)"
diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile
index 9834e669d985..7bb5a50127cb 100644
--- a/drivers/mfd/Makefile
+++ b/drivers/mfd/Makefile
@@ -172,7 +172,7 @@ obj-$(CONFIG_MFD_SI476X_CORE)	+= si476x-core.o
 
 obj-$(CONFIG_MFD_CS5535)	+= cs5535-mfd.o
 obj-$(CONFIG_MFD_OMAP_USB_HOST)	+= omap-usb-host.o omap-usb-tll.o
-obj-$(CONFIG_MFD_PM8921_CORE) 	+= pm8921-core.o ssbi.o
+obj-$(CONFIG_MFD_PM8XXX) 	+= qcom-pm8xxx.o ssbi.o
 obj-$(CONFIG_MFD_QCOM_RPM)	+= qcom_rpm.o
 obj-$(CONFIG_MFD_SPMI_PMIC)	+= qcom-spmi-pmic.o
 obj-$(CONFIG_TPS65911_COMPARATOR)	+= tps65911-comparator.o
diff --git a/drivers/mfd/max77620.c b/drivers/mfd/max77620.c
index 258757e216c4..b1700b5fa640 100644
--- a/drivers/mfd/max77620.c
+++ b/drivers/mfd/max77620.c
@@ -461,7 +461,7 @@ static int max77620_probe(struct i2c_client *client,
 	chip->rmap = devm_regmap_init_i2c(client, rmap_config);
 	if (IS_ERR(chip->rmap)) {
 		ret = PTR_ERR(chip->rmap);
-		dev_err(chip->dev, "Failed to intialise regmap: %d\n", ret);
+		dev_err(chip->dev, "Failed to initialise regmap: %d\n", ret);
 		return ret;
 	}
 
diff --git a/drivers/mfd/pm8921-core.c b/drivers/mfd/qcom-pm8xxx.c
index 0e3a2ea25942..7f9620ec61e8 100644
--- a/drivers/mfd/pm8921-core.c
+++ b/drivers/mfd/qcom-pm8xxx.c
@@ -53,7 +53,7 @@
 #define REG_HWREV		0x002  /* PMIC4 revision */
 #define REG_HWREV_2		0x0E8  /* PMIC4 revision 2 */
 
-#define PM8921_NR_IRQS		256
+#define PM8XXX_NR_IRQS		256
 
 struct pm_irq_chip {
 	struct regmap		*regmap;
@@ -308,22 +308,22 @@ static const struct regmap_config ssbi_regmap_config = {
 	.reg_write = ssbi_reg_write
 };
 
-static const struct of_device_id pm8921_id_table[] = {
+static const struct of_device_id pm8xxx_id_table[] = {
 	{ .compatible = "qcom,pm8018", },
 	{ .compatible = "qcom,pm8058", },
 	{ .compatible = "qcom,pm8921", },
 	{ }
 };
-MODULE_DEVICE_TABLE(of, pm8921_id_table);
+MODULE_DEVICE_TABLE(of, pm8xxx_id_table);
 
-static int pm8921_probe(struct platform_device *pdev)
+static int pm8xxx_probe(struct platform_device *pdev)
 {
 	struct regmap *regmap;
 	int irq, rc;
 	unsigned int val;
 	u32 rev;
 	struct pm_irq_chip *chip;
-	unsigned int nirqs = PM8921_NR_IRQS;
+	unsigned int nirqs = PM8XXX_NR_IRQS;
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
@@ -384,46 +384,46 @@ static int pm8921_probe(struct platform_device *pdev)
 	return rc;
 }
 
-static int pm8921_remove_child(struct device *dev, void *unused)
+static int pm8xxx_remove_child(struct device *dev, void *unused)
 {
 	platform_device_unregister(to_platform_device(dev));
 	return 0;
 }
 
-static int pm8921_remove(struct platform_device *pdev)
+static int pm8xxx_remove(struct platform_device *pdev)
 {
 	int irq = platform_get_irq(pdev, 0);
 	struct pm_irq_chip *chip = platform_get_drvdata(pdev);
 
-	device_for_each_child(&pdev->dev, NULL, pm8921_remove_child);
+	device_for_each_child(&pdev->dev, NULL, pm8xxx_remove_child);
 	irq_set_chained_handler_and_data(irq, NULL, NULL);
 	irq_domain_remove(chip->irqdomain);
 
 	return 0;
 }
 
-static struct platform_driver pm8921_driver = {
-	.probe		= pm8921_probe,
-	.remove		= pm8921_remove,
+static struct platform_driver pm8xxx_driver = {
+	.probe		= pm8xxx_probe,
+	.remove		= pm8xxx_remove,
 	.driver		= {
-		.name	= "pm8921-core",
-		.of_match_table = pm8921_id_table,
+		.name	= "pm8xxx-core",
+		.of_match_table = pm8xxx_id_table,
 	},
 };
 
-static int __init pm8921_init(void)
+static int __init pm8xxx_init(void)
 {
-	return platform_driver_register(&pm8921_driver);
+	return platform_driver_register(&pm8xxx_driver);
 }
-subsys_initcall(pm8921_init);
+subsys_initcall(pm8xxx_init);
 
-static void __exit pm8921_exit(void)
+static void __exit pm8xxx_exit(void)
 {
-	platform_driver_unregister(&pm8921_driver);
+	platform_driver_unregister(&pm8xxx_driver);
 }
-module_exit(pm8921_exit);
+module_exit(pm8xxx_exit);
 
 MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("PMIC 8921 core driver");
+MODULE_DESCRIPTION("PMIC 8xxx core driver");
 MODULE_VERSION("1.0");
-MODULE_ALIAS("platform:pm8921-core");
+MODULE_ALIAS("platform:pm8xxx-core");
diff --git a/drivers/mfd/ti_am335x_tscadc.c b/drivers/mfd/ti_am335x_tscadc.c
index c8f027b4ea4c..0f3fab47fe48 100644
--- a/drivers/mfd/ti_am335x_tscadc.c
+++ b/drivers/mfd/ti_am335x_tscadc.c
@@ -183,6 +183,7 @@ static	int ti_tscadc_probe(struct platform_device *pdev)
 		tscadc->irq = err;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	tscadc->tscadc_phys_base = res->start;
 	tscadc->tscadc_base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(tscadc->tscadc_base))
 		return PTR_ERR(tscadc->tscadc_base);
diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
index 2e5233b60971..1b35e33d2434 100644
--- a/drivers/misc/cxl/api.c
+++ b/drivers/misc/cxl/api.c
@@ -9,18 +9,119 @@
 
 #include <linux/pci.h>
 #include <linux/slab.h>
-#include <linux/anon_inodes.h>
 #include <linux/file.h>
 #include <misc/cxl.h>
-#include <linux/fs.h>
 #include <asm/pnv-pci.h>
 #include <linux/msi.h>
+#include <linux/module.h>
+#include <linux/mount.h>
 
 #include "cxl.h"
 
+/*
+ * Since we want to track memory mappings to be able to force-unmap
+ * when the AFU is no longer reachable, we need an inode. For devices
+ * opened through the cxl user API, this is not a problem, but a
+ * userland process can also get a cxl fd through the cxl_get_fd()
+ * API, which is used by the cxlflash driver.
+ *
+ * Therefore we implement our own simple pseudo-filesystem and inode
+ * allocator. We don't use the anonymous inode, as we need the
+ * meta-data associated with it (address_space) and it is shared by
+ * other drivers/processes, so it could lead to cxl unmapping VMAs
+ * from random processes.
+ */
+
+#define CXL_PSEUDO_FS_MAGIC	0x1697697f
+
+static int cxl_fs_cnt;
+static struct vfsmount *cxl_vfs_mount;
+
+static const struct dentry_operations cxl_fs_dops = {
+	.d_dname	= simple_dname,
+};
+
+static struct dentry *cxl_fs_mount(struct file_system_type *fs_type, int flags,
+				const char *dev_name, void *data)
+{
+	return mount_pseudo(fs_type, "cxl:", NULL, &cxl_fs_dops,
+			CXL_PSEUDO_FS_MAGIC);
+}
+
+static struct file_system_type cxl_fs_type = {
+	.name		= "cxl",
+	.owner		= THIS_MODULE,
+	.mount		= cxl_fs_mount,
+	.kill_sb	= kill_anon_super,
+};
+
+
+void cxl_release_mapping(struct cxl_context *ctx)
+{
+	if (ctx->kernelapi && ctx->mapping)
+		simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);
+}
+
+static struct file *cxl_getfile(const char *name,
+				const struct file_operations *fops,
+				void *priv, int flags)
+{
+	struct qstr this;
+	struct path path;
+	struct file *file;
+	struct inode *inode = NULL;
+	int rc;
+
+	/* strongly inspired by anon_inode_getfile() */
+
+	if (fops->owner && !try_module_get(fops->owner))
+		return ERR_PTR(-ENOENT);
+
+	rc = simple_pin_fs(&cxl_fs_type, &cxl_vfs_mount, &cxl_fs_cnt);
+	if (rc < 0) {
+		pr_err("Cannot mount cxl pseudo filesystem: %d\n", rc);
+		file = ERR_PTR(rc);
+		goto err_module;
+	}
+
+	inode = alloc_anon_inode(cxl_vfs_mount->mnt_sb);
+	if (IS_ERR(inode)) {
+		file = ERR_CAST(inode);
+		goto err_fs;
+	}
+
+	file = ERR_PTR(-ENOMEM);
+	this.name = name;
+	this.len = strlen(name);
+	this.hash = 0;
+	path.dentry = d_alloc_pseudo(cxl_vfs_mount->mnt_sb, &this);
+	if (!path.dentry)
+		goto err_inode;
+
+	path.mnt = mntget(cxl_vfs_mount);
+	d_instantiate(path.dentry, inode);
+
+	file = alloc_file(&path, OPEN_FMODE(flags), fops);
+	if (IS_ERR(file))
+		goto err_dput;
+	file->f_flags = flags & (O_ACCMODE | O_NONBLOCK);
+	file->private_data = priv;
+
+	return file;
+
+err_dput:
+	path_put(&path);
+err_inode:
+	iput(inode);
+err_fs:
+	simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);
+err_module:
+	module_put(fops->owner);
+	return file;
+}
+
 struct cxl_context *cxl_dev_context_init(struct pci_dev *dev)
 {
-	struct address_space *mapping;
 	struct cxl_afu *afu;
 	struct cxl_context  *ctx;
 	int rc;
@@ -30,38 +131,20 @@ struct cxl_context *cxl_dev_context_init(struct pci_dev *dev)
 		return ERR_CAST(afu);
 
 	ctx = cxl_context_alloc();
-	if (IS_ERR(ctx)) {
-		rc = PTR_ERR(ctx);
-		goto err_dev;
-	}
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
 
 	ctx->kernelapi = true;
 
-	/*
-	 * Make our own address space since we won't have one from the
-	 * filesystem like the user api has, and even if we do associate a file
-	 * with this context we don't want to use the global anonymous inode's
-	 * address space as that can invalidate unrelated users:
-	 */
-	mapping = kmalloc(sizeof(struct address_space), GFP_KERNEL);
-	if (!mapping) {
-		rc = -ENOMEM;
-		goto err_ctx;
-	}
-	address_space_init_once(mapping);
-
 	/* Make it a slave context.  We can promote it later? */
-	rc = cxl_context_init(ctx, afu, false, mapping);
+	rc = cxl_context_init(ctx, afu, false);
 	if (rc)
-		goto err_mapping;
+		goto err_ctx;
 
 	return ctx;
 
-err_mapping:
-	kfree(mapping);
 err_ctx:
 	kfree(ctx);
-err_dev:
 	return ERR_PTR(rc);
 }
 EXPORT_SYMBOL_GPL(cxl_dev_context_init);
@@ -340,6 +423,11 @@ struct file *cxl_get_fd(struct cxl_context *ctx, struct file_operations *fops,
 {
 	struct file *file;
 	int rc, flags, fdtmp;
+	char *name = NULL;
+
+	/* only allow one per context */
+	if (ctx->mapping)
+		return ERR_PTR(-EEXIST);
 
 	flags = O_RDWR | O_CLOEXEC;
 
@@ -363,12 +451,13 @@ struct file *cxl_get_fd(struct cxl_context *ctx, struct file_operations *fops,
 	} else /* use default ops */
 		fops = (struct file_operations *)&afu_fops;
 
-	file = anon_inode_getfile("cxl", fops, ctx, flags);
+	name = kasprintf(GFP_KERNEL, "cxl:%d", ctx->pe);
+	file = cxl_getfile(name, fops, ctx, flags);
+	kfree(name);
 	if (IS_ERR(file))
 		goto err_fd;
 
-	file->f_mapping = ctx->mapping;
-
+	cxl_context_set_mapping(ctx, file->f_mapping);
 	*fd = fdtmp;
 	return file;
 
@@ -541,7 +630,7 @@ int _cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
 
 		if (remaining > 0) {
 			new_ctx = cxl_dev_context_init(pdev);
-			if (!new_ctx) {
+			if (IS_ERR(new_ctx)) {
 				pr_warn("%s: Failed to allocate enough contexts for MSIs\n", pci_name(pdev));
 				return -ENOSPC;
 			}
diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
index 5e506c19108a..3907387b6d15 100644
--- a/drivers/misc/cxl/context.c
+++ b/drivers/misc/cxl/context.c
@@ -34,8 +34,7 @@ struct cxl_context *cxl_context_alloc(void)
 /*
  * Initialises a CXL context.
  */
-int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master,
-		     struct address_space *mapping)
+int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)
 {
 	int i;
 
@@ -44,7 +43,7 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master,
 	ctx->master = master;
 	ctx->pid = ctx->glpid = NULL; /* Set in start work ioctl */
 	mutex_init(&ctx->mapping_lock);
-	ctx->mapping = mapping;
+	ctx->mapping = NULL;
 
 	/*
 	 * Allocate the segment table before we put it in the IDR so that we
@@ -114,16 +113,23 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master,
 	return 0;
 }
 
+void cxl_context_set_mapping(struct cxl_context *ctx,
+			struct address_space *mapping)
+{
+	mutex_lock(&ctx->mapping_lock);
+	ctx->mapping = mapping;
+	mutex_unlock(&ctx->mapping_lock);
+}
+
 static int cxl_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct cxl_context *ctx = vma->vm_file->private_data;
-	unsigned long address = (unsigned long)vmf->virtual_address;
 	u64 area, offset;
 
 	offset = vmf->pgoff << PAGE_SHIFT;
 
 	pr_devel("%s: pe: %i address: 0x%lx offset: 0x%llx\n",
-			__func__, ctx->pe, address, offset);
+			__func__, ctx->pe, vmf->address, offset);
 
 	if (ctx->afu->current_mode == CXL_MODE_DEDICATED) {
 		area = ctx->afu->psn_phys;
@@ -155,7 +161,7 @@ static int cxl_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 		return VM_FAULT_SIGBUS;
 	}
 
-	vm_insert_pfn(vma, address, (area + offset) >> PAGE_SHIFT);
+	vm_insert_pfn(vma, vmf->address, (area + offset) >> PAGE_SHIFT);
 
 	mutex_unlock(&ctx->status_mutex);
 
@@ -300,8 +306,6 @@ static void reclaim_ctx(struct rcu_head *rcu)
 	if (ctx->ff_page)
 		__free_page(ctx->ff_page);
 	ctx->sstp = NULL;
-	if (ctx->kernelapi)
-		kfree(ctx->mapping);
 
 	kfree(ctx->irq_bitmap);
 
@@ -313,6 +317,8 @@ static void reclaim_ctx(struct rcu_head *rcu)
 
 void cxl_context_free(struct cxl_context *ctx)
 {
+	if (ctx->kernelapi && ctx->mapping)
+		cxl_release_mapping(ctx);
 	mutex_lock(&ctx->afu->contexts_lock);
 	idr_remove(&ctx->afu->contexts_idr, ctx->pe);
 	mutex_unlock(&ctx->afu->contexts_lock);
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index a144073593fa..b24d76723fb0 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -817,8 +817,9 @@ void cxl_dump_debug_buffer(void *addr, size_t size);
 void init_cxl_native(void);
 
 struct cxl_context *cxl_context_alloc(void);
-int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master,
-		     struct address_space *mapping);
+int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master);
+void cxl_context_set_mapping(struct cxl_context *ctx,
+			struct address_space *mapping);
 void cxl_context_free(struct cxl_context *ctx);
 int cxl_context_iomap(struct cxl_context *ctx, struct vm_area_struct *vma);
 unsigned int cxl_map_irq(struct cxl *adapter, irq_hw_number_t hwirq,
@@ -877,6 +878,7 @@ void cxl_native_err_irq_dump_regs(struct cxl *adapter);
 void cxl_stop_trace(struct cxl *cxl);
 int cxl_pci_vphb_add(struct cxl_afu *afu);
 void cxl_pci_vphb_remove(struct cxl_afu *afu);
+void cxl_release_mapping(struct cxl_context *ctx);
 
 extern struct pci_driver cxl_pci_driver;
 extern struct platform_driver cxl_of_driver;
diff --git a/drivers/misc/cxl/debugfs.c b/drivers/misc/cxl/debugfs.c
index ec7b8a017439..9c06ac8fa5ac 100644
--- a/drivers/misc/cxl/debugfs.c
+++ b/drivers/misc/cxl/debugfs.c
@@ -43,12 +43,14 @@ static int debugfs_io_u64_set(void *data, u64 val)
 	out_be64((u64 __iomem *)data, val);
 	return 0;
 }
-DEFINE_SIMPLE_ATTRIBUTE(fops_io_x64, debugfs_io_u64_get, debugfs_io_u64_set, "0x%016llx\n");
+DEFINE_DEBUGFS_ATTRIBUTE(fops_io_x64, debugfs_io_u64_get, debugfs_io_u64_set,
+			 "0x%016llx\n");
 
 static struct dentry *debugfs_create_io_x64(const char *name, umode_t mode,
 					    struct dentry *parent, u64 __iomem *value)
 {
-	return debugfs_create_file(name, mode, parent, (void __force *)value, &fops_io_x64);
+	return debugfs_create_file_unsafe(name, mode, parent,
+					  (void __force *)value, &fops_io_x64);
 }
 
 void cxl_debugfs_add_adapter_psl_regs(struct cxl *adapter, struct dentry *dir)
diff --git a/drivers/misc/cxl/file.c b/drivers/misc/cxl/file.c
index 77080cc5fa0a..859959f19f10 100644
--- a/drivers/misc/cxl/file.c
+++ b/drivers/misc/cxl/file.c
@@ -86,9 +86,12 @@ static int __afu_open(struct inode *inode, struct file *file, bool master)
 		goto err_put_afu;
 	}
 
-	if ((rc = cxl_context_init(ctx, afu, master, inode->i_mapping)))
+	rc = cxl_context_init(ctx, afu, master);
+	if (rc)
 		goto err_put_afu;
 
+	cxl_context_set_mapping(ctx, inode->i_mapping);
+
 	pr_devel("afu_open pe: %i\n", ctx->pe);
 	file->private_data = ctx;
 	cxl_ctx_get();
diff --git a/drivers/misc/cxl/guest.c b/drivers/misc/cxl/guest.c
index 3e102cd6ed91..e04bc4ddfd74 100644
--- a/drivers/misc/cxl/guest.c
+++ b/drivers/misc/cxl/guest.c
@@ -887,7 +887,7 @@ static void afu_handle_errstate(struct work_struct *work)
 	    afu_guest->previous_state == H_STATE_PERM_UNAVAILABLE)
 		return;
 
-	if (afu_guest->handle_err == true)
+	if (afu_guest->handle_err)
 		schedule_delayed_work(&afu_guest->work_err,
 				      msecs_to_jiffies(3000));
 }
diff --git a/drivers/misc/cxl/irq.c b/drivers/misc/cxl/irq.c
index dec60f58a767..1a402bbed687 100644
--- a/drivers/misc/cxl/irq.c
+++ b/drivers/misc/cxl/irq.c
@@ -104,7 +104,7 @@ irqreturn_t cxl_irq(int irq, struct cxl_context *ctx, struct cxl_irq_info *irq_i
 		} else {
 			spin_lock(&ctx->lock);
 			ctx->afu_err = irq_info->afu_err;
-			ctx->pending_afu_err = 1;
+			ctx->pending_afu_err = true;
 			spin_unlock(&ctx->lock);
 
 			wake_up_all(&ctx->wq);
diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c
index a217a74ccc98..09505f432eda 100644
--- a/drivers/misc/cxl/native.c
+++ b/drivers/misc/cxl/native.c
@@ -10,7 +10,6 @@
 #include <linux/spinlock.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
-#include <linux/sched.h>
 #include <linux/mutex.h>
 #include <linux/mm.h>
 #include <linux/uaccess.h>
@@ -54,7 +53,7 @@ static int afu_control(struct cxl_afu *afu, u64 command, u64 clear,
 				     AFU_Cntl | command);
 		cpu_relax();
 		AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
-	};
+	}
 
 	if (AFU_Cntl & CXL_AFU_Cntl_An_RA) {
 		/*
@@ -167,7 +166,7 @@ int cxl_psl_purge(struct cxl_afu *afu)
 			cpu_relax();
 		}
 		PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
-	};
+	}
 	end = local_clock();
 	pr_devel("PSL purged in %lld ns\n", end - start);
 
@@ -931,9 +930,18 @@ static irqreturn_t native_irq_multiplexed(int irq, void *data)
 	struct cxl_afu *afu = data;
 	struct cxl_context *ctx;
 	struct cxl_irq_info irq_info;
-	int ph = cxl_p2n_read(afu, CXL_PSL_PEHandle_An) & 0xffff;
-	int ret;
-
+	u64 phreg = cxl_p2n_read(afu, CXL_PSL_PEHandle_An);
+	int ph, ret;
+
+	/* check if eeh kicked in while the interrupt was in flight */
+	if (unlikely(phreg == ~0ULL)) {
+		dev_warn(&afu->dev,
+			 "Ignoring slice interrupt(%d) due to fenced card",
+			 irq);
+		return IRQ_HANDLED;
+	}
+	/* Mask the pe-handle from register value */
+	ph = phreg & 0xffff;
 	if ((ret = native_get_irq_info(afu, &irq_info))) {
 		WARN(1, "Unable to get CXL IRQ Info: %i\n", ret);
 		return fail_psl_irq(afu, &irq_info);
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index e96be9ca4e60..80a87ab25b83 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -1921,7 +1921,7 @@ static pci_ers_result_t cxl_pci_slot_reset(struct pci_dev *pdev)
 				goto err;
 
 			ctx = cxl_dev_context_init(afu_dev);
-			if (!ctx)
+			if (IS_ERR(ctx))
 				goto err;
 
 			afu_dev->dev.archdata.cxl_ctx = ctx;
diff --git a/drivers/misc/cxl/phb.c b/drivers/misc/cxl/phb.c
index 0935d44c1770..6ec69ada19f4 100644
--- a/drivers/misc/cxl/phb.c
+++ b/drivers/misc/cxl/phb.c
@@ -20,7 +20,7 @@ bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu
 	 * in the virtual phb, we'll need a default context to attach them to.
 	 */
 	ctx = cxl_dev_context_init(dev);
-	if (!ctx)
+	if (IS_ERR(ctx))
 		return false;
 	dev->dev.archdata.cxl_ctx = ctx;
 
diff --git a/drivers/misc/genwqe/card_base.h b/drivers/misc/genwqe/card_base.h
index cb851c14ca4b..5813b5f25006 100644
--- a/drivers/misc/genwqe/card_base.h
+++ b/drivers/misc/genwqe/card_base.h
@@ -41,7 +41,6 @@
 #include "genwqe_driver.h"
 
 #define GENWQE_MSI_IRQS			4  /* Just one supported, no MSIx */
-#define GENWQE_FLAG_MSI_ENABLED		(1 << 0)
 
 #define GENWQE_MAX_VFS			15 /* maximum 15 VFs are possible */
 #define GENWQE_MAX_FUNCS		16 /* 1 PF and 15 VFs */
diff --git a/drivers/misc/genwqe/card_utils.c b/drivers/misc/genwqe/card_utils.c
index fc2794b513fa..147b83011b58 100644
--- a/drivers/misc/genwqe/card_utils.c
+++ b/drivers/misc/genwqe/card_utils.c
@@ -740,13 +740,10 @@ int genwqe_read_softreset(struct genwqe_dev *cd)
 int genwqe_set_interrupt_capability(struct genwqe_dev *cd, int count)
 {
 	int rc;
-	struct pci_dev *pci_dev = cd->pci_dev;
 
-	rc = pci_enable_msi_range(pci_dev, 1, count);
+	rc = pci_alloc_irq_vectors(cd->pci_dev, 1, count, PCI_IRQ_MSI);
 	if (rc < 0)
 		return rc;
-
-	cd->flags |= GENWQE_FLAG_MSI_ENABLED;
 	return 0;
 }
 
@@ -756,12 +753,7 @@ int genwqe_set_interrupt_capability(struct genwqe_dev *cd, int count)
  */
 void genwqe_reset_interrupt_capability(struct genwqe_dev *cd)
 {
-	struct pci_dev *pci_dev = cd->pci_dev;
-
-	if (cd->flags & GENWQE_FLAG_MSI_ENABLED) {
-		pci_disable_msi(pci_dev);
-		cd->flags &= ~GENWQE_FLAG_MSI_ENABLED;
-	}
+	pci_free_irq_vectors(cd->pci_dev);
 }
 
 /**
diff --git a/drivers/misc/ibmasm/module.c b/drivers/misc/ibmasm/module.c
index 6b3bf9ab051d..c5a456b0a564 100644
--- a/drivers/misc/ibmasm/module.c
+++ b/drivers/misc/ibmasm/module.c
@@ -170,7 +170,7 @@ static void ibmasm_remove_one(struct pci_dev *pdev)
 	ibmasm_unregister_uart(sp);
 	dbg("Sending OS down message\n");
 	if (ibmasm_send_os_state(sp, SYSTEM_STATE_OS_DOWN))
-		err("failed to get repsonse to 'Send OS State' command\n");
+		err("failed to get response to 'Send OS State' command\n");
 	dbg("Disabling heartbeats\n");
 	ibmasm_heartbeat_exit(sp);
 	dbg("Disabling interrupts\n");
diff --git a/drivers/misc/lkdtm_bugs.c b/drivers/misc/lkdtm_bugs.c
index f336206d4b1f..91edd0b55e5c 100644
--- a/drivers/misc/lkdtm_bugs.c
+++ b/drivers/misc/lkdtm_bugs.c
@@ -85,7 +85,8 @@ noinline void lkdtm_CORRUPT_STACK(void)
 	/* Use default char array length that triggers stack protection. */
 	char data[8];
 
-	memset((void *)data, 0, 64);
+	memset((void *)data, 'a', 64);
+	pr_info("Corrupted stack with '%16s'...\n", data);
 }
 
 void lkdtm_UNALIGNED_LOAD_STORE_WRITE(void)
diff --git a/drivers/misc/lkdtm_perms.c b/drivers/misc/lkdtm_perms.c
index 45f1c0f96612..c7635a79341f 100644
--- a/drivers/misc/lkdtm_perms.c
+++ b/drivers/misc/lkdtm_perms.c
@@ -60,15 +60,18 @@ static noinline void execute_location(void *dst, bool write)
 
 static void execute_user_location(void *dst)
 {
+	int copied;
+
 	/* Intentionally crossing kernel/user memory boundary. */
 	void (*func)(void) = dst;
 
 	pr_info("attempting ok execution at %p\n", do_nothing);
 	do_nothing();
 
-	if (copy_to_user((void __user *)dst, do_nothing, EXEC_SIZE))
+	copied = access_process_vm(current, (unsigned long)dst, do_nothing,
+				   EXEC_SIZE, FOLL_WRITE);
+	if (copied < EXEC_SIZE)
 		return;
-	flush_icache_range((unsigned long)dst, (unsigned long)dst + EXEC_SIZE);
 	pr_info("attempting bad execution at %p\n", func);
 	func();
 }
diff --git a/drivers/misc/mei/amthif.c b/drivers/misc/mei/amthif.c
index 7ae89b4a21d5..466afb2611c6 100644
--- a/drivers/misc/mei/amthif.c
+++ b/drivers/misc/mei/amthif.c
@@ -144,7 +144,7 @@ int mei_amthif_run_next_cmd(struct mei_device *dev)
 	dev->iamthif_state = MEI_IAMTHIF_WRITING;
 	cl->fp = cb->fp;
 
-	ret = mei_cl_write(cl, cb, false);
+	ret = mei_cl_write(cl, cb);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/misc/mei/bus-fixup.c b/drivers/misc/mei/bus-fixup.c
index 75b9d4ac8b1e..18e05ca7584f 100644
--- a/drivers/misc/mei/bus-fixup.c
+++ b/drivers/misc/mei/bus-fixup.c
@@ -38,6 +38,9 @@ static const uuid_le mei_nfc_info_guid = MEI_UUID_NFC_INFO;
 #define MEI_UUID_WD UUID_LE(0x05B79A6F, 0x4628, 0x4D7F, \
 			    0x89, 0x9D, 0xA9, 0x15, 0x14, 0xCB, 0x32, 0xAB)
 
+#define MEI_UUID_MKHIF_FIX UUID_LE(0x55213584, 0x9a29, 0x4916, \
+			0xba, 0xdf, 0xf, 0xb7, 0xed, 0x68, 0x2a, 0xeb)
+
 #define MEI_UUID_ANY NULL_UUID_LE
 
 /**
@@ -69,6 +72,97 @@ static void blacklist(struct mei_cl_device *cldev)
 	cldev->do_match = 0;
 }
 
+#define OSTYPE_LINUX    2
+struct mei_os_ver {
+	__le16 build;
+	__le16 reserved1;
+	u8  os_type;
+	u8  major;
+	u8  minor;
+	u8  reserved2;
+} __packed;
+
+#define MKHI_FEATURE_PTT 0x10
+
+struct mkhi_rule_id {
+	__le16 rule_type;
+	u8 feature_id;
+	u8 reserved;
+} __packed;
+
+struct mkhi_fwcaps {
+	struct mkhi_rule_id id;
+	u8 len;
+	u8 data[0];
+} __packed;
+
+#define MKHI_FWCAPS_GROUP_ID 0x3
+#define MKHI_FWCAPS_SET_OS_VER_APP_RULE_CMD 6
+struct mkhi_msg_hdr {
+	u8  group_id;
+	u8  command;
+	u8  reserved;
+	u8  result;
+} __packed;
+
+struct mkhi_msg {
+	struct mkhi_msg_hdr hdr;
+	u8 data[0];
+} __packed;
+
+static int mei_osver(struct mei_cl_device *cldev)
+{
+	int ret;
+	const size_t size = sizeof(struct mkhi_msg_hdr) +
+			    sizeof(struct mkhi_fwcaps) +
+			    sizeof(struct mei_os_ver);
+	size_t length = 8;
+	char buf[size];
+	struct mkhi_msg *req;
+	struct mkhi_fwcaps *fwcaps;
+	struct mei_os_ver *os_ver;
+	unsigned int mode = MEI_CL_IO_TX_BLOCKING | MEI_CL_IO_TX_INTERNAL;
+
+	memset(buf, 0, size);
+
+	req = (struct mkhi_msg *)buf;
+	req->hdr.group_id = MKHI_FWCAPS_GROUP_ID;
+	req->hdr.command = MKHI_FWCAPS_SET_OS_VER_APP_RULE_CMD;
+
+	fwcaps = (struct mkhi_fwcaps *)req->data;
+
+	fwcaps->id.rule_type = 0x0;
+	fwcaps->id.feature_id = MKHI_FEATURE_PTT;
+	fwcaps->len = sizeof(*os_ver);
+	os_ver = (struct mei_os_ver *)fwcaps->data;
+	os_ver->os_type = OSTYPE_LINUX;
+
+	ret = __mei_cl_send(cldev->cl, buf, size, mode);
+	if (ret < 0)
+		return ret;
+
+	ret = __mei_cl_recv(cldev->cl, buf, length, 0);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static void mei_mkhi_fix(struct mei_cl_device *cldev)
+{
+	int ret;
+
+	ret = mei_cldev_enable(cldev);
+	if (ret)
+		return;
+
+	ret = mei_osver(cldev);
+	if (ret)
+		dev_err(&cldev->dev, "OS version command failed %d\n", ret);
+
+	mei_cldev_disable(cldev);
+}
+
 /**
  * mei_wd - wd client on the bus, change protocol version
  *   as the API has changed.
@@ -162,7 +256,8 @@ static int mei_nfc_if_version(struct mei_cl *cl,
 
 	WARN_ON(mutex_is_locked(&bus->device_lock));
 
-	ret = __mei_cl_send(cl, (u8 *)&cmd, sizeof(struct mei_nfc_cmd), 1);
+	ret = __mei_cl_send(cl, (u8 *)&cmd, sizeof(struct mei_nfc_cmd),
+			    MEI_CL_IO_TX_BLOCKING);
 	if (ret < 0) {
 		dev_err(bus->dev, "Could not send IF version cmd\n");
 		return ret;
@@ -177,7 +272,7 @@ static int mei_nfc_if_version(struct mei_cl *cl,
 		return -ENOMEM;
 
 	ret = 0;
-	bytes_recv = __mei_cl_recv(cl, (u8 *)reply, if_version_length);
+	bytes_recv = __mei_cl_recv(cl, (u8 *)reply, if_version_length, 0);
 	if (bytes_recv < if_version_length) {
 		dev_err(bus->dev, "Could not read IF version\n");
 		ret = -EIO;
@@ -309,6 +404,7 @@ static struct mei_fixup {
 	MEI_FIXUP(MEI_UUID_NFC_INFO, blacklist),
 	MEI_FIXUP(MEI_UUID_NFC_HCI, mei_nfc),
 	MEI_FIXUP(MEI_UUID_WD, mei_wd),
+	MEI_FIXUP(MEI_UUID_MKHIF_FIX, mei_mkhi_fix),
 };
 
 /**
diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c
index 8cac7ef9ad0d..0037153c80a6 100644
--- a/drivers/misc/mei/bus.c
+++ b/drivers/misc/mei/bus.c
@@ -36,12 +36,12 @@
  * @cl: host client
  * @buf: buffer to send
  * @length: buffer length
- * @blocking: wait for write completion
+ * @mode: sending mode
  *
  * Return: written size bytes or < 0 on error
  */
 ssize_t __mei_cl_send(struct mei_cl *cl, u8 *buf, size_t length,
-			bool blocking)
+		      unsigned int mode)
 {
 	struct mei_device *bus;
 	struct mei_cl_cb *cb;
@@ -80,9 +80,11 @@ ssize_t __mei_cl_send(struct mei_cl *cl, u8 *buf, size_t length,
 		goto out;
 	}
 
+	cb->internal = !!(mode & MEI_CL_IO_TX_INTERNAL);
+	cb->blocking = !!(mode & MEI_CL_IO_TX_BLOCKING);
 	memcpy(cb->buf.data, buf, length);
 
-	rets = mei_cl_write(cl, cb, blocking);
+	rets = mei_cl_write(cl, cb);
 
 out:
 	mutex_unlock(&bus->device_lock);
@@ -96,15 +98,18 @@ out:
  * @cl: host client
  * @buf: buffer to receive
  * @length: buffer length
+ * @mode: io mode
  *
  * Return: read size in bytes of < 0 on error
  */
-ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length)
+ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length,
+		      unsigned int mode)
 {
 	struct mei_device *bus;
 	struct mei_cl_cb *cb;
 	size_t r_length;
 	ssize_t rets;
+	bool nonblock = !!(mode & MEI_CL_IO_RX_NONBLOCK);
 
 	if (WARN_ON(!cl || !cl->dev))
 		return -ENODEV;
@@ -125,6 +130,11 @@ ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length)
 	if (rets && rets != -EBUSY)
 		goto out;
 
+	if (nonblock) {
+		rets = -EAGAIN;
+		goto out;
+	}
+
 	/* wait on event only if there is no other waiter */
 	/* synchronized under device mutex */
 	if (!waitqueue_active(&cl->rx_wait)) {
@@ -185,14 +195,30 @@ ssize_t mei_cldev_send(struct mei_cl_device *cldev, u8 *buf, size_t length)
 {
 	struct mei_cl *cl = cldev->cl;
 
-	if (cl == NULL)
-		return -ENODEV;
-
-	return __mei_cl_send(cl, buf, length, 1);
+	return __mei_cl_send(cl, buf, length, MEI_CL_IO_TX_BLOCKING);
 }
 EXPORT_SYMBOL_GPL(mei_cldev_send);
 
 /**
+ * mei_cldev_recv_nonblock - non block client receive (read)
+ *
+ * @cldev: me client device
+ * @buf: buffer to receive
+ * @length: buffer length
+ *
+ * Return: read size in bytes of < 0 on error
+ *         -EAGAIN if function will block.
+ */
+ssize_t mei_cldev_recv_nonblock(struct mei_cl_device *cldev, u8 *buf,
+				size_t length)
+{
+	struct mei_cl *cl = cldev->cl;
+
+	return __mei_cl_recv(cl, buf, length, MEI_CL_IO_RX_NONBLOCK);
+}
+EXPORT_SYMBOL_GPL(mei_cldev_recv_nonblock);
+
+/**
  * mei_cldev_recv - client receive (read)
  *
  * @cldev: me client device
@@ -205,39 +231,45 @@ ssize_t mei_cldev_recv(struct mei_cl_device *cldev, u8 *buf, size_t length)
 {
 	struct mei_cl *cl = cldev->cl;
 
-	if (cl == NULL)
-		return -ENODEV;
-
-	return __mei_cl_recv(cl, buf, length);
+	return __mei_cl_recv(cl, buf, length, 0);
 }
 EXPORT_SYMBOL_GPL(mei_cldev_recv);
 
 /**
- * mei_cl_bus_event_work  - dispatch rx event for a bus device
- *    and schedule new work
+ * mei_cl_bus_rx_work - dispatch rx event for a bus device
  *
  * @work: work
  */
-static void mei_cl_bus_event_work(struct work_struct *work)
+static void mei_cl_bus_rx_work(struct work_struct *work)
 {
 	struct mei_cl_device *cldev;
 	struct mei_device *bus;
 
-	cldev = container_of(work, struct mei_cl_device, event_work);
+	cldev = container_of(work, struct mei_cl_device, rx_work);
 
 	bus = cldev->bus;
 
-	if (cldev->event_cb)
-		cldev->event_cb(cldev, cldev->events, cldev->event_context);
+	if (cldev->rx_cb)
+		cldev->rx_cb(cldev);
+
+	mutex_lock(&bus->device_lock);
+	mei_cl_read_start(cldev->cl, mei_cl_mtu(cldev->cl), NULL);
+	mutex_unlock(&bus->device_lock);
+}
+
+/**
+ * mei_cl_bus_notif_work - dispatch FW notif event for a bus device
+ *
+ * @work: work
+ */
+static void mei_cl_bus_notif_work(struct work_struct *work)
+{
+	struct mei_cl_device *cldev;
 
-	cldev->events = 0;
+	cldev = container_of(work, struct mei_cl_device, notif_work);
 
-	/* Prepare for the next read */
-	if (cldev->events_mask & BIT(MEI_CL_EVENT_RX)) {
-		mutex_lock(&bus->device_lock);
-		mei_cl_read_start(cldev->cl, mei_cl_mtu(cldev->cl), NULL);
-		mutex_unlock(&bus->device_lock);
-	}
+	if (cldev->notif_cb)
+		cldev->notif_cb(cldev);
 }
 
 /**
@@ -252,18 +284,13 @@ bool mei_cl_bus_notify_event(struct mei_cl *cl)
 {
 	struct mei_cl_device *cldev = cl->cldev;
 
-	if (!cldev || !cldev->event_cb)
-		return false;
-
-	if (!(cldev->events_mask & BIT(MEI_CL_EVENT_NOTIF)))
+	if (!cldev || !cldev->notif_cb)
 		return false;
 
 	if (!cl->notify_ev)
 		return false;
 
-	set_bit(MEI_CL_EVENT_NOTIF, &cldev->events);
-
-	schedule_work(&cldev->event_work);
+	schedule_work(&cldev->notif_work);
 
 	cl->notify_ev = false;
 
@@ -271,7 +298,7 @@ bool mei_cl_bus_notify_event(struct mei_cl *cl)
 }
 
 /**
- * mei_cl_bus_rx_event  - schedule rx event
+ * mei_cl_bus_rx_event - schedule rx event
  *
  * @cl: host client
  *
@@ -282,66 +309,81 @@ bool mei_cl_bus_rx_event(struct mei_cl *cl)
 {
 	struct mei_cl_device *cldev = cl->cldev;
 
-	if (!cldev || !cldev->event_cb)
-		return false;
-
-	if (!(cldev->events_mask & BIT(MEI_CL_EVENT_RX)))
+	if (!cldev || !cldev->rx_cb)
 		return false;
 
-	set_bit(MEI_CL_EVENT_RX, &cldev->events);
-
-	schedule_work(&cldev->event_work);
+	schedule_work(&cldev->rx_work);
 
 	return true;
 }
 
 /**
- * mei_cldev_register_event_cb - register event callback
+ * mei_cldev_register_rx_cb - register Rx event callback
  *
  * @cldev: me client devices
- * @event_cb: callback function
- * @events_mask: requested events bitmask
- * @context: driver context data
+ * @rx_cb: callback function
  *
  * Return: 0 on success
  *         -EALREADY if an callback is already registered
  *         <0 on other errors
  */
-int mei_cldev_register_event_cb(struct mei_cl_device *cldev,
-				unsigned long events_mask,
-				mei_cldev_event_cb_t event_cb, void *context)
+int mei_cldev_register_rx_cb(struct mei_cl_device *cldev, mei_cldev_cb_t rx_cb)
 {
 	struct mei_device *bus = cldev->bus;
 	int ret;
 
-	if (cldev->event_cb)
+	if (!rx_cb)
+		return -EINVAL;
+	if (cldev->rx_cb)
 		return -EALREADY;
 
-	cldev->events = 0;
-	cldev->events_mask = events_mask;
-	cldev->event_cb = event_cb;
-	cldev->event_context = context;
-	INIT_WORK(&cldev->event_work, mei_cl_bus_event_work);
+	cldev->rx_cb = rx_cb;
+	INIT_WORK(&cldev->rx_work, mei_cl_bus_rx_work);
 
-	if (cldev->events_mask & BIT(MEI_CL_EVENT_RX)) {
-		mutex_lock(&bus->device_lock);
-		ret = mei_cl_read_start(cldev->cl, mei_cl_mtu(cldev->cl), NULL);
-		mutex_unlock(&bus->device_lock);
-		if (ret && ret != -EBUSY)
-			return ret;
-	}
+	mutex_lock(&bus->device_lock);
+	ret = mei_cl_read_start(cldev->cl, mei_cl_mtu(cldev->cl), NULL);
+	mutex_unlock(&bus->device_lock);
+	if (ret && ret != -EBUSY)
+		return ret;
 
-	if (cldev->events_mask & BIT(MEI_CL_EVENT_NOTIF)) {
-		mutex_lock(&bus->device_lock);
-		ret = mei_cl_notify_request(cldev->cl, NULL, event_cb ? 1 : 0);
-		mutex_unlock(&bus->device_lock);
-		if (ret)
-			return ret;
-	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mei_cldev_register_rx_cb);
+
+/**
+ * mei_cldev_register_notif_cb - register FW notification event callback
+ *
+ * @cldev: me client devices
+ * @notif_cb: callback function
+ *
+ * Return: 0 on success
+ *         -EALREADY if an callback is already registered
+ *         <0 on other errors
+ */
+int mei_cldev_register_notif_cb(struct mei_cl_device *cldev,
+				mei_cldev_cb_t notif_cb)
+{
+	struct mei_device *bus = cldev->bus;
+	int ret;
+
+	if (!notif_cb)
+		return -EINVAL;
+
+	if (cldev->notif_cb)
+		return -EALREADY;
+
+	cldev->notif_cb = notif_cb;
+	INIT_WORK(&cldev->notif_work, mei_cl_bus_notif_work);
+
+	mutex_lock(&bus->device_lock);
+	ret = mei_cl_notify_request(cldev->cl, NULL, 1);
+	mutex_unlock(&bus->device_lock);
+	if (ret)
+		return ret;
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(mei_cldev_register_event_cb);
+EXPORT_SYMBOL_GPL(mei_cldev_register_notif_cb);
 
 /**
  * mei_cldev_get_drvdata - driver data getter
@@ -403,7 +445,7 @@ EXPORT_SYMBOL_GPL(mei_cldev_ver);
  */
 bool mei_cldev_enabled(struct mei_cl_device *cldev)
 {
-	return cldev->cl && mei_cl_is_connected(cldev->cl);
+	return mei_cl_is_connected(cldev->cl);
 }
 EXPORT_SYMBOL_GPL(mei_cldev_enabled);
 
@@ -423,14 +465,13 @@ int mei_cldev_enable(struct mei_cl_device *cldev)
 
 	cl = cldev->cl;
 
-	if (!cl) {
+	if (cl->state == MEI_FILE_UNINITIALIZED) {
 		mutex_lock(&bus->device_lock);
-		cl = mei_cl_alloc_linked(bus);
+		ret = mei_cl_link(cl);
 		mutex_unlock(&bus->device_lock);
-		if (IS_ERR(cl))
-			return PTR_ERR(cl);
+		if (ret)
+			return ret;
 		/* update pointers */
-		cldev->cl = cl;
 		cl->cldev = cldev;
 	}
 
@@ -471,19 +512,17 @@ int mei_cldev_disable(struct mei_cl_device *cldev)
 	struct mei_cl *cl;
 	int err;
 
-	if (!cldev || !cldev->cl)
+	if (!cldev)
 		return -ENODEV;
 
 	cl = cldev->cl;
 
 	bus = cldev->bus;
 
-	cldev->event_cb = NULL;
-
 	mutex_lock(&bus->device_lock);
 
 	if (!mei_cl_is_connected(cl)) {
-		dev_err(bus->dev, "Already disconnected");
+		dev_dbg(bus->dev, "Already disconnected");
 		err = 0;
 		goto out;
 	}
@@ -497,9 +536,6 @@ out:
 	mei_cl_flush_queues(cl, NULL);
 	mei_cl_unlink(cl);
 
-	kfree(cl);
-	cldev->cl = NULL;
-
 	mutex_unlock(&bus->device_lock);
 	return err;
 }
@@ -629,9 +665,13 @@ static int mei_cl_device_remove(struct device *dev)
 	if (!cldev || !dev->driver)
 		return 0;
 
-	if (cldev->event_cb) {
-		cldev->event_cb = NULL;
-		cancel_work_sync(&cldev->event_work);
+	if (cldev->rx_cb) {
+		cancel_work_sync(&cldev->rx_work);
+		cldev->rx_cb = NULL;
+	}
+	if (cldev->notif_cb) {
+		cancel_work_sync(&cldev->notif_work);
+		cldev->notif_cb = NULL;
 	}
 
 	cldrv = to_mei_cl_driver(dev->driver);
@@ -754,6 +794,7 @@ static void mei_cl_bus_dev_release(struct device *dev)
 
 	mei_me_cl_put(cldev->me_cl);
 	mei_dev_bus_put(cldev->bus);
+	kfree(cldev->cl);
 	kfree(cldev);
 }
 
@@ -786,17 +827,25 @@ static struct mei_cl_device *mei_cl_bus_dev_alloc(struct mei_device *bus,
 						  struct mei_me_client *me_cl)
 {
 	struct mei_cl_device *cldev;
+	struct mei_cl *cl;
 
 	cldev = kzalloc(sizeof(struct mei_cl_device), GFP_KERNEL);
 	if (!cldev)
 		return NULL;
 
+	cl = mei_cl_allocate(bus);
+	if (!cl) {
+		kfree(cldev);
+		return NULL;
+	}
+
 	device_initialize(&cldev->dev);
 	cldev->dev.parent = bus->dev;
 	cldev->dev.bus    = &mei_cl_bus_type;
 	cldev->dev.type   = &mei_cl_device_type;
 	cldev->bus        = mei_dev_bus_get(bus);
 	cldev->me_cl      = mei_me_cl_get(me_cl);
+	cldev->cl         = cl;
 	mei_cl_bus_set_name(cldev);
 	cldev->is_added   = 0;
 	INIT_LIST_HEAD(&cldev->bus_list);
diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c
index 6fe02350578d..391936c1aa04 100644
--- a/drivers/misc/mei/client.c
+++ b/drivers/misc/mei/client.c
@@ -425,7 +425,7 @@ static inline void mei_io_list_free(struct mei_cl_cb *list, struct mei_cl *cl)
  *
  * @cl: host client
  * @length: size of the buffer
- * @type: operation type
+ * @fop_type: operation type
  * @fp: associated file pointer (might be NULL)
  *
  * Return: cb on success and NULL on failure
@@ -459,7 +459,7 @@ struct mei_cl_cb *mei_cl_alloc_cb(struct mei_cl *cl, size_t length,
  *
  * @cl: host client
  * @length: size of the buffer
- * @type: operation type
+ * @fop_type: operation type
  * @fp: associated file pointer (might be NULL)
  *
  * Return: cb on success and NULL on failure
@@ -571,7 +571,7 @@ void mei_cl_init(struct mei_cl *cl, struct mei_device *dev)
 	INIT_LIST_HEAD(&cl->rd_pending);
 	INIT_LIST_HEAD(&cl->link);
 	cl->writing_state = MEI_IDLE;
-	cl->state = MEI_FILE_INITIALIZING;
+	cl->state = MEI_FILE_UNINITIALIZED;
 	cl->dev = dev;
 }
 
@@ -672,7 +672,12 @@ int mei_cl_unlink(struct mei_cl *cl)
 
 	list_del_init(&cl->link);
 
-	cl->state = MEI_FILE_INITIALIZING;
+	cl->state = MEI_FILE_UNINITIALIZED;
+	cl->writing_state = MEI_IDLE;
+
+	WARN_ON(!list_empty(&cl->rd_completed) ||
+		!list_empty(&cl->rd_pending) ||
+		!list_empty(&cl->link));
 
 	return 0;
 }
@@ -686,7 +691,7 @@ void mei_host_client_init(struct mei_device *dev)
 
 	pm_runtime_mark_last_busy(dev->dev);
 	dev_dbg(dev->dev, "rpm: autosuspend\n");
-	pm_runtime_autosuspend(dev->dev);
+	pm_request_autosuspend(dev->dev);
 }
 
 /**
@@ -756,7 +761,7 @@ void mei_cl_set_disconnected(struct mei_cl *cl)
 	struct mei_device *dev = cl->dev;
 
 	if (cl->state == MEI_FILE_DISCONNECTED ||
-	    cl->state == MEI_FILE_INITIALIZING)
+	    cl->state <= MEI_FILE_INITIALIZING)
 		return;
 
 	cl->state = MEI_FILE_DISCONNECTED;
@@ -1598,18 +1603,17 @@ int mei_cl_irq_write(struct mei_cl *cl, struct mei_cl_cb *cb,
  *
  * @cl: host client
  * @cb: write callback with filled data
- * @blocking: block until completed
  *
  * Return: number of bytes sent on success, <0 on failure.
  */
-int mei_cl_write(struct mei_cl *cl, struct mei_cl_cb *cb, bool blocking)
+int mei_cl_write(struct mei_cl *cl, struct mei_cl_cb *cb)
 {
 	struct mei_device *dev;
 	struct mei_msg_data *buf;
 	struct mei_msg_hdr mei_hdr;
 	int size;
 	int rets;
-
+	bool blocking;
 
 	if (WARN_ON(!cl || !cl->dev))
 		return -ENODEV;
@@ -1621,6 +1625,7 @@ int mei_cl_write(struct mei_cl *cl, struct mei_cl_cb *cb, bool blocking)
 
 	buf = &cb->buf;
 	size = buf->size;
+	blocking = cb->blocking;
 
 	cl_dbg(dev, cl, "size=%d\n", size);
 
diff --git a/drivers/misc/mei/client.h b/drivers/misc/mei/client.h
index d2bfabecd882..f2545af9be7b 100644
--- a/drivers/misc/mei/client.h
+++ b/drivers/misc/mei/client.h
@@ -219,7 +219,7 @@ int mei_cl_irq_connect(struct mei_cl *cl, struct mei_cl_cb *cb,
 int mei_cl_read_start(struct mei_cl *cl, size_t length, const struct file *fp);
 int mei_cl_irq_read_msg(struct mei_cl *cl, struct mei_msg_hdr *hdr,
 			struct mei_cl_cb *cmpl_list);
-int mei_cl_write(struct mei_cl *cl, struct mei_cl_cb *cb, bool blocking);
+int mei_cl_write(struct mei_cl *cl, struct mei_cl_cb *cb);
 int mei_cl_irq_write(struct mei_cl *cl, struct mei_cl_cb *cb,
 		     struct mei_cl_cb *cmpl_list);
 
diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h
index 7ad15d678878..c8307e8b4c16 100644
--- a/drivers/misc/mei/hw-me-regs.h
+++ b/drivers/misc/mei/hw-me-regs.h
@@ -122,6 +122,8 @@
 #define MEI_DEV_ID_SPT_H      0xA13A  /* Sunrise Point H */
 #define MEI_DEV_ID_SPT_H_2    0xA13B  /* Sunrise Point H 2 */
 
+#define MEI_DEV_ID_LBG        0xA1BA  /* Lewisburg (SPT) */
+
 #define MEI_DEV_ID_BXT_M      0x1A9A  /* Broxton M */
 #define MEI_DEV_ID_APL_I      0x5A9A  /* Apollo Lake I */
 
diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c
index 56c2101e80ad..a05375a3338a 100644
--- a/drivers/misc/mei/hw-me.c
+++ b/drivers/misc/mei/hw-me.c
@@ -246,6 +246,36 @@ static inline enum mei_pg_state mei_me_pg_state(struct mei_device *dev)
 	return hw->pg_state;
 }
 
+static inline u32 me_intr_src(u32 hcsr)
+{
+	return hcsr & H_CSR_IS_MASK;
+}
+
+/**
+ * me_intr_disable - disables mei device interrupts
+ *      using supplied hcsr register value.
+ *
+ * @dev: the device structure
+ * @hcsr: supplied hcsr register value
+ */
+static inline void me_intr_disable(struct mei_device *dev, u32 hcsr)
+{
+	hcsr &= ~H_CSR_IE_MASK;
+	mei_hcsr_set(dev, hcsr);
+}
+
+/**
+ * mei_me_intr_clear - clear and stop interrupts
+ *
+ * @dev: the device structure
+ * @hcsr: supplied hcsr register value
+ */
+static inline void me_intr_clear(struct mei_device *dev, u32 hcsr)
+{
+	if (me_intr_src(hcsr))
+		mei_hcsr_write(dev, hcsr);
+}
+
 /**
  * mei_me_intr_clear - clear and stop interrupts
  *
@@ -255,8 +285,7 @@ static void mei_me_intr_clear(struct mei_device *dev)
 {
 	u32 hcsr = mei_hcsr_read(dev);
 
-	if (hcsr & H_CSR_IS_MASK)
-		mei_hcsr_write(dev, hcsr);
+	me_intr_clear(dev, hcsr);
 }
 /**
  * mei_me_intr_enable - enables mei device interrupts
@@ -280,8 +309,19 @@ static void mei_me_intr_disable(struct mei_device *dev)
 {
 	u32 hcsr = mei_hcsr_read(dev);
 
-	hcsr  &= ~H_CSR_IE_MASK;
-	mei_hcsr_set(dev, hcsr);
+	me_intr_disable(dev, hcsr);
+}
+
+/**
+ * mei_me_synchronize_irq - wait for pending IRQ handlers
+ *
+ * @dev: the device structure
+ */
+static void mei_me_synchronize_irq(struct mei_device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
+
+	synchronize_irq(pdev->irq);
 }
 
 /**
@@ -450,7 +490,7 @@ static size_t mei_me_hbuf_max_len(const struct mei_device *dev)
 
 
 /**
- * mei_me_write_message - writes a message to mei device.
+ * mei_me_hbuf_write - writes a message to host hw buffer.
  *
  * @dev: the device structure
  * @header: mei HECI header of message
@@ -458,9 +498,9 @@ static size_t mei_me_hbuf_max_len(const struct mei_device *dev)
  *
  * Return: -EIO if write has failed
  */
-static int mei_me_write_message(struct mei_device *dev,
-			struct mei_msg_hdr *header,
-			unsigned char *buf)
+static int mei_me_hbuf_write(struct mei_device *dev,
+			     struct mei_msg_hdr *header,
+			     const unsigned char *buf)
 {
 	unsigned long rem;
 	unsigned long length = header->length;
@@ -956,13 +996,14 @@ static void mei_me_pg_legacy_intr(struct mei_device *dev)
  * mei_me_d0i3_intr - perform d0i3 processing in interrupt thread handler
  *
  * @dev: the device structure
+ * @intr_source: interrupt source
  */
-static void mei_me_d0i3_intr(struct mei_device *dev)
+static void mei_me_d0i3_intr(struct mei_device *dev, u32 intr_source)
 {
 	struct mei_me_hw *hw = to_me_hw(dev);
 
 	if (dev->pg_event == MEI_PG_EVENT_INTR_WAIT &&
-	    (hw->intr_source & H_D0I3C_IS)) {
+	    (intr_source & H_D0I3C_IS)) {
 		dev->pg_event = MEI_PG_EVENT_INTR_RECEIVED;
 		if (hw->pg_state == MEI_PG_ON) {
 			hw->pg_state = MEI_PG_OFF;
@@ -981,7 +1022,7 @@ static void mei_me_d0i3_intr(struct mei_device *dev)
 		wake_up(&dev->wait_pg);
 	}
 
-	if (hw->pg_state == MEI_PG_ON && (hw->intr_source & H_IS)) {
+	if (hw->pg_state == MEI_PG_ON && (intr_source & H_IS)) {
 		/*
 		 * HW sent some data and we are in D0i3, so
 		 * we got here because of HW initiated exit from D0i3.
@@ -996,13 +1037,14 @@ static void mei_me_d0i3_intr(struct mei_device *dev)
  * mei_me_pg_intr - perform pg processing in interrupt thread handler
  *
  * @dev: the device structure
+ * @intr_source: interrupt source
  */
-static void mei_me_pg_intr(struct mei_device *dev)
+static void mei_me_pg_intr(struct mei_device *dev, u32 intr_source)
 {
 	struct mei_me_hw *hw = to_me_hw(dev);
 
 	if (hw->d0i3_supported)
-		mei_me_d0i3_intr(dev);
+		mei_me_d0i3_intr(dev, intr_source);
 	else
 		mei_me_pg_legacy_intr(dev);
 }
@@ -1121,19 +1163,16 @@ static int mei_me_hw_reset(struct mei_device *dev, bool intr_enable)
 irqreturn_t mei_me_irq_quick_handler(int irq, void *dev_id)
 {
 	struct mei_device *dev = (struct mei_device *)dev_id;
-	struct mei_me_hw *hw = to_me_hw(dev);
 	u32 hcsr;
 
 	hcsr = mei_hcsr_read(dev);
-	if (!(hcsr & H_CSR_IS_MASK))
+	if (!me_intr_src(hcsr))
 		return IRQ_NONE;
 
-	hw->intr_source = hcsr & H_CSR_IS_MASK;
-	dev_dbg(dev->dev, "interrupt source 0x%08X.\n", hw->intr_source);
-
-	/* clear H_IS and H_D0I3C_IS bits in H_CSR to clear the interrupts */
-	mei_hcsr_write(dev, hcsr);
+	dev_dbg(dev->dev, "interrupt source 0x%08X\n", me_intr_src(hcsr));
 
+	/* disable interrupts on device */
+	me_intr_disable(dev, hcsr);
 	return IRQ_WAKE_THREAD;
 }
 
@@ -1152,11 +1191,16 @@ irqreturn_t mei_me_irq_thread_handler(int irq, void *dev_id)
 	struct mei_device *dev = (struct mei_device *) dev_id;
 	struct mei_cl_cb complete_list;
 	s32 slots;
+	u32 hcsr;
 	int rets = 0;
 
 	dev_dbg(dev->dev, "function called after ISR to handle the interrupt processing.\n");
 	/* initialize our complete list */
 	mutex_lock(&dev->device_lock);
+
+	hcsr = mei_hcsr_read(dev);
+	me_intr_clear(dev, hcsr);
+
 	mei_io_list_init(&complete_list);
 
 	/* check if ME wants a reset */
@@ -1166,7 +1210,7 @@ irqreturn_t mei_me_irq_thread_handler(int irq, void *dev_id)
 		goto end;
 	}
 
-	mei_me_pg_intr(dev);
+	mei_me_pg_intr(dev, me_intr_src(hcsr));
 
 	/*  check if we need to start the dev */
 	if (!mei_host_is_ready(dev)) {
@@ -1216,6 +1260,7 @@ irqreturn_t mei_me_irq_thread_handler(int irq, void *dev_id)
 
 end:
 	dev_dbg(dev->dev, "interrupt thread end ret = %d\n", rets);
+	mei_me_intr_enable(dev);
 	mutex_unlock(&dev->device_lock);
 	return IRQ_HANDLED;
 }
@@ -1238,12 +1283,13 @@ static const struct mei_hw_ops mei_me_hw_ops = {
 	.intr_clear = mei_me_intr_clear,
 	.intr_enable = mei_me_intr_enable,
 	.intr_disable = mei_me_intr_disable,
+	.synchronize_irq = mei_me_synchronize_irq,
 
 	.hbuf_free_slots = mei_me_hbuf_empty_slots,
 	.hbuf_is_ready = mei_me_hbuf_is_empty,
 	.hbuf_max_len = mei_me_hbuf_max_len,
 
-	.write = mei_me_write_message,
+	.write = mei_me_hbuf_write,
 
 	.rdbuf_full_slots = mei_me_count_full_read_slots,
 	.read_hdr = mei_me_mecbrw_read,
diff --git a/drivers/misc/mei/hw-me.h b/drivers/misc/mei/hw-me.h
index 2ee14dc1b2ea..cf64847a35b9 100644
--- a/drivers/misc/mei/hw-me.h
+++ b/drivers/misc/mei/hw-me.h
@@ -51,14 +51,12 @@ struct mei_cfg {
  *
  * @cfg: per device generation config and ops
  * @mem_addr: io memory address
- * @intr_source: interrupt source
  * @pg_state: power gating state
  * @d0i3_supported: di03 support
  */
 struct mei_me_hw {
 	const struct mei_cfg *cfg;
 	void __iomem *mem_addr;
-	u32 intr_source;
 	enum mei_pg_state pg_state;
 	bool d0i3_supported;
 };
diff --git a/drivers/misc/mei/hw-txe.c b/drivers/misc/mei/hw-txe.c
index 60415a2bfcbd..e9f8c0aeec13 100644
--- a/drivers/misc/mei/hw-txe.c
+++ b/drivers/misc/mei/hw-txe.c
@@ -19,7 +19,7 @@
 #include <linux/ktime.h>
 #include <linux/delay.h>
 #include <linux/kthread.h>
-#include <linux/irqreturn.h>
+#include <linux/interrupt.h>
 #include <linux/pm_runtime.h>
 
 #include <linux/mei.h>
@@ -441,6 +441,18 @@ static void mei_txe_intr_enable(struct mei_device *dev)
 }
 
 /**
+ * mei_txe_synchronize_irq - wait for pending IRQ handlers
+ *
+ * @dev: the device structure
+ */
+static void mei_txe_synchronize_irq(struct mei_device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
+
+	synchronize_irq(pdev->irq);
+}
+
+/**
  * mei_txe_pending_interrupts - check if there are pending interrupts
  *	only Aliveness, Input ready, and output doorbell are of relevance
  *
@@ -691,7 +703,8 @@ static void mei_txe_hw_config(struct mei_device *dev)
  */
 
 static int mei_txe_write(struct mei_device *dev,
-		struct mei_msg_hdr *header, unsigned char *buf)
+			 struct mei_msg_hdr *header,
+			 const unsigned char *buf)
 {
 	struct mei_txe_hw *hw = to_txe_hw(dev);
 	unsigned long rem;
@@ -1167,6 +1180,7 @@ static const struct mei_hw_ops mei_txe_hw_ops = {
 	.intr_clear = mei_txe_intr_clear,
 	.intr_enable = mei_txe_intr_enable,
 	.intr_disable = mei_txe_intr_disable,
+	.synchronize_irq = mei_txe_synchronize_irq,
 
 	.hbuf_free_slots = mei_txe_hbuf_empty_slots,
 	.hbuf_is_ready = mei_txe_is_input_ready,
diff --git a/drivers/misc/mei/init.c b/drivers/misc/mei/init.c
index 9a9c2484d107..41e5760a6886 100644
--- a/drivers/misc/mei/init.c
+++ b/drivers/misc/mei/init.c
@@ -122,6 +122,10 @@ int mei_reset(struct mei_device *dev)
 			 mei_dev_state_str(state), fw_sts_str);
 	}
 
+	mei_clear_interrupts(dev);
+
+	mei_synchronize_irq(dev);
+
 	/* we're already in reset, cancel the init timer
 	 * if the reset was called due the hbm protocol error
 	 * we need to call it before hw start
@@ -273,8 +277,6 @@ int mei_restart(struct mei_device *dev)
 
 	mutex_lock(&dev->device_lock);
 
-	mei_clear_interrupts(dev);
-
 	dev->dev_state = MEI_DEV_POWER_UP;
 	dev->reset_count = 0;
 
diff --git a/drivers/misc/mei/interrupt.c b/drivers/misc/mei/interrupt.c
index 5a4893ce9c24..b584749bcc4a 100644
--- a/drivers/misc/mei/interrupt.c
+++ b/drivers/misc/mei/interrupt.c
@@ -118,7 +118,6 @@ int mei_cl_irq_read_msg(struct mei_cl *cl,
 
 	if (!mei_cl_is_connected(cl)) {
 		cl_dbg(dev, cl, "not connected\n");
-		list_move_tail(&cb->list, &complete_list->list);
 		cb->status = -ENODEV;
 		goto discard;
 	}
@@ -128,8 +127,6 @@ int mei_cl_irq_read_msg(struct mei_cl *cl,
 	if (buf_sz < cb->buf_idx) {
 		cl_err(dev, cl, "message is too big len %d idx %zu\n",
 		       mei_hdr->length, cb->buf_idx);
-
-		list_move_tail(&cb->list, &complete_list->list);
 		cb->status = -EMSGSIZE;
 		goto discard;
 	}
@@ -137,8 +134,6 @@ int mei_cl_irq_read_msg(struct mei_cl *cl,
 	if (cb->buf.size < buf_sz) {
 		cl_dbg(dev, cl, "message overflow. size %zu len %d idx %zu\n",
 			cb->buf.size, mei_hdr->length, cb->buf_idx);
-
-		list_move_tail(&cb->list, &complete_list->list);
 		cb->status = -EMSGSIZE;
 		goto discard;
 	}
@@ -158,6 +153,8 @@ int mei_cl_irq_read_msg(struct mei_cl *cl,
 	return 0;
 
 discard:
+	if (cb)
+		list_move_tail(&cb->list, &complete_list->list);
 	mei_irq_discard_msg(dev, mei_hdr);
 	return 0;
 }
diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c
index fa50635512e8..e1bf54481fd6 100644
--- a/drivers/misc/mei/main.c
+++ b/drivers/misc/mei/main.c
@@ -322,7 +322,7 @@ static ssize_t mei_write(struct file *file, const char __user *ubuf,
 		goto out;
 	}
 
-	rets = mei_cl_write(cl, cb, false);
+	rets = mei_cl_write(cl, cb);
 out:
 	mutex_unlock(&dev->device_lock);
 	return rets;
@@ -653,7 +653,7 @@ static int mei_fasync(int fd, struct file *file, int band)
 }
 
 /**
- * fw_status_show - mei device attribute show method
+ * fw_status_show - mei device fw_status attribute show method
  *
  * @device: device pointer
  * @attr: attribute pointer
@@ -684,8 +684,49 @@ static ssize_t fw_status_show(struct device *device,
 }
 static DEVICE_ATTR_RO(fw_status);
 
+/**
+ * hbm_ver_show - display HBM protocol version negotiated with FW
+ *
+ * @device: device pointer
+ * @attr: attribute pointer
+ * @buf:  char out buffer
+ *
+ * Return: number of the bytes printed into buf or error
+ */
+static ssize_t hbm_ver_show(struct device *device,
+			    struct device_attribute *attr, char *buf)
+{
+	struct mei_device *dev = dev_get_drvdata(device);
+	struct hbm_version ver;
+
+	mutex_lock(&dev->device_lock);
+	ver = dev->version;
+	mutex_unlock(&dev->device_lock);
+
+	return sprintf(buf, "%u.%u\n", ver.major_version, ver.minor_version);
+}
+static DEVICE_ATTR_RO(hbm_ver);
+
+/**
+ * hbm_ver_drv_show - display HBM protocol version advertised by driver
+ *
+ * @device: device pointer
+ * @attr: attribute pointer
+ * @buf:  char out buffer
+ *
+ * Return: number of the bytes printed into buf or error
+ */
+static ssize_t hbm_ver_drv_show(struct device *device,
+				struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%u.%u\n", HBM_MAJOR_VERSION, HBM_MINOR_VERSION);
+}
+static DEVICE_ATTR_RO(hbm_ver_drv);
+
 static struct attribute *mei_attrs[] = {
 	&dev_attr_fw_status.attr,
+	&dev_attr_hbm_ver.attr,
+	&dev_attr_hbm_ver_drv.attr,
 	NULL
 };
 ATTRIBUTE_GROUPS(mei);
diff --git a/drivers/misc/mei/mei_dev.h b/drivers/misc/mei/mei_dev.h
index 1169fd9e7d02..699693cd8c59 100644
--- a/drivers/misc/mei/mei_dev.h
+++ b/drivers/misc/mei/mei_dev.h
@@ -55,7 +55,8 @@ extern const uuid_le mei_amthif_guid;
 
 /* File state */
 enum file_state {
-	MEI_FILE_INITIALIZING = 0,
+	MEI_FILE_UNINITIALIZED = 0,
+	MEI_FILE_INITIALIZING,
 	MEI_FILE_CONNECTING,
 	MEI_FILE_CONNECTED,
 	MEI_FILE_DISCONNECTING,
@@ -109,6 +110,21 @@ enum mei_cb_file_ops {
 	MEI_FOP_NOTIFY_STOP,
 };
 
+/**
+ * enum mei_cl_io_mode - io mode between driver and fw
+ *
+ * @MEI_CL_IO_TX_BLOCKING: send is blocking
+ * @MEI_CL_IO_TX_INTERNAL: internal communication between driver and FW
+ *
+ * @MEI_CL_IO_RX_NONBLOCK: recv is non-blocking
+ */
+enum mei_cl_io_mode {
+	MEI_CL_IO_TX_BLOCKING = BIT(0),
+	MEI_CL_IO_TX_INTERNAL = BIT(1),
+
+	MEI_CL_IO_RX_NONBLOCK = BIT(2),
+};
+
 /*
  * Intel MEI message data struct
  */
@@ -169,6 +185,7 @@ struct mei_cl;
  * @fp: pointer to file structure
  * @status: io status of the cb
  * @internal: communication between driver and FW flag
+ * @blocking: transmission blocking mode
  * @completed: the transfer or reception has completed
  */
 struct mei_cl_cb {
@@ -180,6 +197,7 @@ struct mei_cl_cb {
 	const struct file *fp;
 	int status;
 	u32 internal:1;
+	u32 blocking:1;
 	u32 completed:1;
 };
 
@@ -253,6 +271,7 @@ struct mei_cl {
  * @intr_clear       : clear pending interrupts
  * @intr_enable      : enable interrupts
  * @intr_disable     : disable interrupts
+ * @synchronize_irq  : synchronize irqs
  *
  * @hbuf_free_slots  : query for write buffer empty slots
  * @hbuf_is_ready    : query if write buffer is empty
@@ -274,7 +293,6 @@ struct mei_hw_ops {
 	int (*hw_start)(struct mei_device *dev);
 	void (*hw_config)(struct mei_device *dev);
 
-
 	int (*fw_status)(struct mei_device *dev, struct mei_fw_status *fw_sts);
 	enum mei_pg_state (*pg_state)(struct mei_device *dev);
 	bool (*pg_in_transition)(struct mei_device *dev);
@@ -283,14 +301,14 @@ struct mei_hw_ops {
 	void (*intr_clear)(struct mei_device *dev);
 	void (*intr_enable)(struct mei_device *dev);
 	void (*intr_disable)(struct mei_device *dev);
+	void (*synchronize_irq)(struct mei_device *dev);
 
 	int (*hbuf_free_slots)(struct mei_device *dev);
 	bool (*hbuf_is_ready)(struct mei_device *dev);
 	size_t (*hbuf_max_len)(const struct mei_device *dev);
-
 	int (*write)(struct mei_device *dev,
 		     struct mei_msg_hdr *hdr,
-		     unsigned char *buf);
+		     const unsigned char *buf);
 
 	int (*rdbuf_full_slots)(struct mei_device *dev);
 
@@ -304,8 +322,9 @@ void mei_cl_bus_rescan(struct mei_device *bus);
 void mei_cl_bus_rescan_work(struct work_struct *work);
 void mei_cl_bus_dev_fixup(struct mei_cl_device *dev);
 ssize_t __mei_cl_send(struct mei_cl *cl, u8 *buf, size_t length,
-			bool blocking);
-ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length);
+		      unsigned int mode);
+ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length,
+		      unsigned int mode);
 bool mei_cl_bus_rx_event(struct mei_cl *cl);
 bool mei_cl_bus_notify_event(struct mei_cl *cl);
 void mei_cl_bus_remove_devices(struct mei_device *bus);
@@ -627,6 +646,11 @@ static inline void mei_disable_interrupts(struct mei_device *dev)
 	dev->ops->intr_disable(dev);
 }
 
+static inline void mei_synchronize_irq(struct mei_device *dev)
+{
+	dev->ops->synchronize_irq(dev);
+}
+
 static inline bool mei_host_is_ready(struct mei_device *dev)
 {
 	return dev->ops->host_is_ready(dev);
@@ -652,7 +676,7 @@ static inline size_t mei_hbuf_max_len(const struct mei_device *dev)
 }
 
 static inline int mei_write_message(struct mei_device *dev,
-			struct mei_msg_hdr *hdr, void *buf)
+				    struct mei_msg_hdr *hdr, const void *buf)
 {
 	return dev->ops->write(dev, hdr, buf);
 }
diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c
index f3ffd883b232..f9c6ec4b98ab 100644
--- a/drivers/misc/mei/pci-me.c
+++ b/drivers/misc/mei/pci-me.c
@@ -87,6 +87,7 @@ static const struct pci_device_id mei_me_pci_tbl[] = {
 	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_2, mei_me_pch8_cfg)},
 	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H, mei_me_pch8_sps_cfg)},
 	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H_2, mei_me_pch8_sps_cfg)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_LBG, mei_me_pch8_cfg)},
 
 	{MEI_PCI_DEVICE(MEI_DEV_ID_BXT_M, mei_me_pch8_cfg)},
 	{MEI_PCI_DEVICE(MEI_DEV_ID_APL_I, mei_me_pch8_cfg)},
diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c
index 33741ad4a74a..af2e077da4b8 100644
--- a/drivers/misc/sgi-gru/grumain.c
+++ b/drivers/misc/sgi-gru/grumain.c
@@ -932,7 +932,7 @@ int gru_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	unsigned long paddr, vaddr;
 	unsigned long expires;
 
-	vaddr = (unsigned long)vmf->virtual_address;
+	vaddr = vmf->address;
 	gru_dbg(grudev, "vma %p, vaddr 0x%lx (0x%lx)\n",
 		vma, vaddr, GSEG_BASE(vaddr));
 	STAT(nopfn);
diff --git a/drivers/misc/sram.c b/drivers/misc/sram.c
index f84b53d6ce50..b33ab8ce47ab 100644
--- a/drivers/misc/sram.c
+++ b/drivers/misc/sram.c
@@ -19,12 +19,17 @@
  */
 
 #include <linux/clk.h>
+#include <linux/delay.h>
 #include <linux/genalloc.h>
 #include <linux/io.h>
 #include <linux/list_sort.h>
 #include <linux/of_address.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
+#include <linux/regmap.h>
 #include <linux/slab.h>
+#include <linux/mfd/syscon.h>
+#include <soc/at91/atmel-secumod.h>
 
 #define SRAM_GRANULARITY	32
 
@@ -334,12 +339,33 @@ static int sram_reserve_regions(struct sram_dev *sram, struct resource *res)
 	return ret;
 }
 
+static int atmel_securam_wait(void)
+{
+	struct regmap *regmap;
+	u32 val;
+
+	regmap = syscon_regmap_lookup_by_compatible("atmel,sama5d2-secumod");
+	if (IS_ERR(regmap))
+		return -ENODEV;
+
+	return regmap_read_poll_timeout(regmap, AT91_SECUMOD_RAMRDY, val,
+					val & AT91_SECUMOD_RAMRDY_READY,
+					10000, 500000);
+}
+
+static const struct of_device_id sram_dt_ids[] = {
+	{ .compatible = "mmio-sram" },
+	{ .compatible = "atmel,sama5d2-securam", .data = atmel_securam_wait },
+	{}
+};
+
 static int sram_probe(struct platform_device *pdev)
 {
 	struct sram_dev *sram;
 	struct resource *res;
 	size_t size;
 	int ret;
+	int (*init_func)(void);
 
 	sram = devm_kzalloc(&pdev->dev, sizeof(*sram), GFP_KERNEL);
 	if (!sram)
@@ -384,6 +410,13 @@ static int sram_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, sram);
 
+	init_func = of_device_get_match_data(&pdev->dev);
+	if (init_func) {
+		ret = init_func();
+		if (ret)
+			return ret;
+	}
+
 	dev_dbg(sram->dev, "SRAM pool: %zu KiB @ 0x%p\n",
 		gen_pool_size(sram->pool) / 1024, sram->virt_base);
 
@@ -405,17 +438,10 @@ static int sram_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_OF
-static const struct of_device_id sram_dt_ids[] = {
-	{ .compatible = "mmio-sram" },
-	{}
-};
-#endif
-
 static struct platform_driver sram_driver = {
 	.driver = {
 		.name = "sram",
-		.of_match_table = of_match_ptr(sram_dt_ids),
+		.of_match_table = sram_dt_ids,
 	},
 	.probe = sram_probe,
 	.remove = sram_remove,
diff --git a/drivers/mmc/Kconfig b/drivers/mmc/Kconfig
index f2eeb38efa65..7e803fc454d1 100644
--- a/drivers/mmc/Kconfig
+++ b/drivers/mmc/Kconfig
@@ -23,8 +23,6 @@ if MMC
 
 source "drivers/mmc/core/Kconfig"
 
-source "drivers/mmc/card/Kconfig"
-
 source "drivers/mmc/host/Kconfig"
 
 endif # MMC
diff --git a/drivers/mmc/Makefile b/drivers/mmc/Makefile
index 400756ec7c49..416b6d1c9ec6 100644
--- a/drivers/mmc/Makefile
+++ b/drivers/mmc/Makefile
@@ -5,5 +5,4 @@
 subdir-ccflags-$(CONFIG_MMC_DEBUG) := -DDEBUG
 
 obj-$(CONFIG_MMC)		+= core/
-obj-$(CONFIG_MMC)		+= card/
 obj-$(subst m,y,$(CONFIG_MMC))	+= host/
diff --git a/drivers/mmc/card/Kconfig b/drivers/mmc/card/Kconfig
deleted file mode 100644
index 5562308699bc..000000000000
--- a/drivers/mmc/card/Kconfig
+++ /dev/null
@@ -1,70 +0,0 @@
-#
-# MMC/SD card drivers
-#
-
-comment "MMC/SD/SDIO Card Drivers"
-
-config MMC_BLOCK
-	tristate "MMC block device driver"
-	depends on BLOCK
-	default y
-	help
-	  Say Y here to enable the MMC block device driver support.
-	  This provides a block device driver, which you can use to
-	  mount the filesystem. Almost everyone wishing MMC support
-	  should say Y or M here.
-
-config MMC_BLOCK_MINORS
-	int "Number of minors per block device"
-	depends on MMC_BLOCK
-	range 4 256
-	default 8
-	help
-	  Number of minors per block device. One is needed for every
-	  partition on the disk (plus one for the whole disk).
-
-	  Number of total MMC minors available is 256, so your number
-	  of supported block devices will be limited to 256 divided
-	  by this number.
-
-	  Default is 8 to be backwards compatible with previous
-	  hardwired device numbering.
-
-	  If unsure, say 8 here.
-
-config MMC_BLOCK_BOUNCE
-	bool "Use bounce buffer for simple hosts"
-	depends on MMC_BLOCK
-	default y
-	help
-	  SD/MMC is a high latency protocol where it is crucial to
-	  send large requests in order to get high performance. Many
-	  controllers, however, are restricted to continuous memory
-	  (i.e. they can't do scatter-gather), something the kernel
-	  rarely can provide.
-
-	  Say Y here to help these restricted hosts by bouncing
-	  requests back and forth from a large buffer. You will get
-	  a big performance gain at the cost of up to 64 KiB of
-	  physical memory.
-
-	  If unsure, say Y here.
-
-config SDIO_UART
-	tristate "SDIO UART/GPS class support"
-	depends on TTY
-	help
-	  SDIO function driver for SDIO cards that implements the UART
-	  class, as well as the GPS class which appears like a UART.
-
-config MMC_TEST
-	tristate "MMC host test driver"
-	help
-	  Development driver that performs a series of reads and writes
-	  to a memory card in order to expose certain well known bugs
-	  in host controllers. The tests are executed by writing to the
-	  "test" file in debugfs under each card. Note that whatever is
-	  on your card will be overwritten by these tests.
-
-	  This driver is only of interest to those developing or
-	  testing a host driver. Most people should say N here.
diff --git a/drivers/mmc/card/Makefile b/drivers/mmc/card/Makefile
deleted file mode 100644
index c73b406a06cd..000000000000
--- a/drivers/mmc/card/Makefile
+++ /dev/null
@@ -1,10 +0,0 @@
-#
-# Makefile for MMC/SD card drivers
-#
-
-obj-$(CONFIG_MMC_BLOCK)		+= mmc_block.o
-mmc_block-objs			:= block.o queue.o
-obj-$(CONFIG_MMC_TEST)		+= mmc_test.o
-
-obj-$(CONFIG_SDIO_UART)		+= sdio_uart.o
-
diff --git a/drivers/mmc/core/Kconfig b/drivers/mmc/core/Kconfig
index 250f223aaa80..cdfa8520a4b1 100644
--- a/drivers/mmc/core/Kconfig
+++ b/drivers/mmc/core/Kconfig
@@ -22,3 +22,69 @@ config PWRSEQ_SIMPLE
 
 	  This driver can also be built as a module. If so, the module
 	  will be called pwrseq_simple.
+
+config MMC_BLOCK
+	tristate "MMC block device driver"
+	depends on BLOCK
+	default y
+	help
+	  Say Y here to enable the MMC block device driver support.
+	  This provides a block device driver, which you can use to
+	  mount the filesystem. Almost everyone wishing MMC support
+	  should say Y or M here.
+
+config MMC_BLOCK_MINORS
+	int "Number of minors per block device"
+	depends on MMC_BLOCK
+	range 4 256
+	default 8
+	help
+	  Number of minors per block device. One is needed for every
+	  partition on the disk (plus one for the whole disk).
+
+	  Number of total MMC minors available is 256, so your number
+	  of supported block devices will be limited to 256 divided
+	  by this number.
+
+	  Default is 8 to be backwards compatible with previous
+	  hardwired device numbering.
+
+	  If unsure, say 8 here.
+
+config MMC_BLOCK_BOUNCE
+	bool "Use bounce buffer for simple hosts"
+	depends on MMC_BLOCK
+	default y
+	help
+	  SD/MMC is a high latency protocol where it is crucial to
+	  send large requests in order to get high performance. Many
+	  controllers, however, are restricted to continuous memory
+	  (i.e. they can't do scatter-gather), something the kernel
+	  rarely can provide.
+
+	  Say Y here to help these restricted hosts by bouncing
+	  requests back and forth from a large buffer. You will get
+	  a big performance gain at the cost of up to 64 KiB of
+	  physical memory.
+
+	  If unsure, say Y here.
+
+config SDIO_UART
+	tristate "SDIO UART/GPS class support"
+	depends on TTY
+	help
+	  SDIO function driver for SDIO cards that implements the UART
+	  class, as well as the GPS class which appears like a UART.
+
+config MMC_TEST
+	tristate "MMC host test driver"
+	help
+	  Development driver that performs a series of reads and writes
+	  to a memory card in order to expose certain well known bugs
+	  in host controllers. The tests are executed by writing to the
+	  "test" file in debugfs under each card. Note that whatever is
+	  on your card will be overwritten by these tests.
+
+	  This driver is only of interest to those developing or
+	  testing a host driver. Most people should say N here.
+
diff --git a/drivers/mmc/core/Makefile b/drivers/mmc/core/Makefile
index f007151dfdc6..b2a257dc644f 100644
--- a/drivers/mmc/core/Makefile
+++ b/drivers/mmc/core/Makefile
@@ -12,3 +12,7 @@ mmc_core-$(CONFIG_OF)		+= pwrseq.o
 obj-$(CONFIG_PWRSEQ_SIMPLE)	+= pwrseq_simple.o
 obj-$(CONFIG_PWRSEQ_EMMC)	+= pwrseq_emmc.o
 mmc_core-$(CONFIG_DEBUG_FS)	+= debugfs.o
+obj-$(CONFIG_MMC_BLOCK)		+= mmc_block.o
+mmc_block-objs			:= block.o queue.o
+obj-$(CONFIG_MMC_TEST)		+= mmc_test.o
+obj-$(CONFIG_SDIO_UART)		+= sdio_uart.o
diff --git a/drivers/mmc/card/block.c b/drivers/mmc/core/block.c
index 709a872ed484..bab3f07b1117 100644
--- a/drivers/mmc/card/block.c
+++ b/drivers/mmc/core/block.c
@@ -66,9 +66,6 @@ MODULE_ALIAS("mmc:block");
 
 #define mmc_req_rel_wr(req)	((req->cmd_flags & REQ_FUA) && \
 				  (rq_data_dir(req) == WRITE))
-#define PACKED_CMD_VER	0x01
-#define PACKED_CMD_WR	0x02
-
 static DEFINE_MUTEX(block_mutex);
 
 /*
@@ -102,7 +99,6 @@ struct mmc_blk_data {
 	unsigned int	flags;
 #define MMC_BLK_CMD23	(1 << 0)	/* Can do SET_BLOCK_COUNT for multiblock */
 #define MMC_BLK_REL_WR	(1 << 1)	/* MMC Reliable write support */
-#define MMC_BLK_PACKED_CMD	(1 << 2)	/* MMC packed command support */
 
 	unsigned int	usage;
 	unsigned int	read_only;
@@ -126,12 +122,6 @@ struct mmc_blk_data {
 
 static DEFINE_MUTEX(open_lock);
 
-enum {
-	MMC_PACKED_NR_IDX = -1,
-	MMC_PACKED_NR_ZERO,
-	MMC_PACKED_NR_SINGLE,
-};
-
 module_param(perdev_minors, int, 0444);
 MODULE_PARM_DESC(perdev_minors, "Minors numbers to allocate per device");
 
@@ -139,17 +129,6 @@ static inline int mmc_blk_part_switch(struct mmc_card *card,
 				      struct mmc_blk_data *md);
 static int get_card_status(struct mmc_card *card, u32 *status, int retries);
 
-static inline void mmc_blk_clear_packed(struct mmc_queue_req *mqrq)
-{
-	struct mmc_packed *packed = mqrq->packed;
-
-	mqrq->cmd_type = MMC_PACKED_NONE;
-	packed->nr_entries = MMC_PACKED_NR_ZERO;
-	packed->idx_failure = MMC_PACKED_NR_IDX;
-	packed->retries = 0;
-	packed->blocks = 0;
-}
-
 static struct mmc_blk_data *mmc_blk_get(struct gendisk *disk)
 {
 	struct mmc_blk_data *md;
@@ -854,7 +833,7 @@ static int get_card_status(struct mmc_card *card, u32 *status, int retries)
 }
 
 static int card_busy_detect(struct mmc_card *card, unsigned int timeout_ms,
-		bool hw_busy_detect, struct request *req, int *gen_err)
+		bool hw_busy_detect, struct request *req, bool *gen_err)
 {
 	unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
 	int err = 0;
@@ -871,7 +850,7 @@ static int card_busy_detect(struct mmc_card *card, unsigned int timeout_ms,
 		if (status & R1_ERROR) {
 			pr_err("%s: %s: error sending status cmd, status %#x\n",
 				req->rq_disk->disk_name, __func__, status);
-			*gen_err = 1;
+			*gen_err = true;
 		}
 
 		/* We may rely on the host hw to handle busy detection.*/
@@ -902,7 +881,7 @@ static int card_busy_detect(struct mmc_card *card, unsigned int timeout_ms,
 }
 
 static int send_stop(struct mmc_card *card, unsigned int timeout_ms,
-		struct request *req, int *gen_err, u32 *stop_status)
+		struct request *req, bool *gen_err, u32 *stop_status)
 {
 	struct mmc_host *host = card->host;
 	struct mmc_command cmd = {0};
@@ -940,7 +919,7 @@ static int send_stop(struct mmc_card *card, unsigned int timeout_ms,
 		(*stop_status & R1_ERROR)) {
 		pr_err("%s: %s: general error sending stop command, resp %#x\n",
 			req->rq_disk->disk_name, __func__, *stop_status);
-		*gen_err = 1;
+		*gen_err = true;
 	}
 
 	return card_busy_detect(card, timeout_ms, use_r1b_resp, req, gen_err);
@@ -1014,7 +993,7 @@ static int mmc_blk_cmd_error(struct request *req, const char *name, int error,
  * Otherwise we don't understand what happened, so abort.
  */
 static int mmc_blk_cmd_recovery(struct mmc_card *card, struct request *req,
-	struct mmc_blk_request *brq, int *ecc_err, int *gen_err)
+	struct mmc_blk_request *brq, bool *ecc_err, bool *gen_err)
 {
 	bool prev_cmd_status_valid = true;
 	u32 status, stop_status = 0;
@@ -1053,7 +1032,7 @@ static int mmc_blk_cmd_recovery(struct mmc_card *card, struct request *req,
 	if ((status & R1_CARD_ECC_FAILED) ||
 	    (brq->stop.resp[0] & R1_CARD_ECC_FAILED) ||
 	    (brq->cmd.resp[0] & R1_CARD_ECC_FAILED))
-		*ecc_err = 1;
+		*ecc_err = true;
 
 	/* Flag General errors */
 	if (!mmc_host_is_spi(card->host) && rq_data_dir(req) != READ)
@@ -1062,7 +1041,7 @@ static int mmc_blk_cmd_recovery(struct mmc_card *card, struct request *req,
 			pr_err("%s: %s: general error sending stop or status command, stop cmd response %#x, card status %#x\n",
 			       req->rq_disk->disk_name, __func__,
 			       brq->stop.resp[0], status);
-			*gen_err = 1;
+			*gen_err = true;
 		}
 
 	/*
@@ -1085,7 +1064,7 @@ static int mmc_blk_cmd_recovery(struct mmc_card *card, struct request *req,
 		}
 
 		if (stop_status & R1_CARD_ECC_FAILED)
-			*ecc_err = 1;
+			*ecc_err = true;
 	}
 
 	/* Check for set block count errors */
@@ -1154,7 +1133,7 @@ static inline void mmc_blk_reset_success(struct mmc_blk_data *md, int type)
 
 int mmc_access_rpmb(struct mmc_queue *mq)
 {
-	struct mmc_blk_data *md = mq->data;
+	struct mmc_blk_data *md = mq->blkdata;
 	/*
 	 * If this is a RPMB partition access, return ture
 	 */
@@ -1166,7 +1145,7 @@ int mmc_access_rpmb(struct mmc_queue *mq)
 
 static int mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)
 {
-	struct mmc_blk_data *md = mq->data;
+	struct mmc_blk_data *md = mq->blkdata;
 	struct mmc_card *card = md->queue.card;
 	unsigned int from, nr, arg;
 	int err = 0, type = MMC_BLK_DISCARD;
@@ -1210,7 +1189,7 @@ out:
 static int mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
 				       struct request *req)
 {
-	struct mmc_blk_data *md = mq->data;
+	struct mmc_blk_data *md = mq->blkdata;
 	struct mmc_card *card = md->queue.card;
 	unsigned int from, nr, arg;
 	int err = 0, type = MMC_BLK_SECDISCARD;
@@ -1276,7 +1255,7 @@ out:
 
 static int mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)
 {
-	struct mmc_blk_data *md = mq->data;
+	struct mmc_blk_data *md = mq->blkdata;
 	struct mmc_card *card = md->queue.card;
 	int ret = 0;
 
@@ -1320,15 +1299,16 @@ static inline void mmc_apply_rel_rw(struct mmc_blk_request *brq,
 	 R1_CC_ERROR |		/* Card controller error */		\
 	 R1_ERROR)		/* General/unknown error */
 
-static int mmc_blk_err_check(struct mmc_card *card,
-			     struct mmc_async_req *areq)
+static enum mmc_blk_status mmc_blk_err_check(struct mmc_card *card,
+					     struct mmc_async_req *areq)
 {
 	struct mmc_queue_req *mq_mrq = container_of(areq, struct mmc_queue_req,
 						    mmc_active);
 	struct mmc_blk_request *brq = &mq_mrq->brq;
 	struct request *req = mq_mrq->req;
 	int need_retune = card->host->need_retune;
-	int ecc_err = 0, gen_err = 0;
+	bool ecc_err = false;
+	bool gen_err = false;
 
 	/*
 	 * sbc.error indicates a problem with the set block count
@@ -1378,7 +1358,7 @@ static int mmc_blk_err_check(struct mmc_card *card,
 			pr_err("%s: %s: general error sending stop command, stop cmd response %#x\n",
 			       req->rq_disk->disk_name, __func__,
 			       brq->stop.resp[0]);
-			gen_err = 1;
+			gen_err = true;
 		}
 
 		err = card_busy_detect(card, MMC_BLK_TIMEOUT_MS, false, req,
@@ -1419,67 +1399,12 @@ static int mmc_blk_err_check(struct mmc_card *card,
 	if (!brq->data.bytes_xfered)
 		return MMC_BLK_RETRY;
 
-	if (mmc_packed_cmd(mq_mrq->cmd_type)) {
-		if (unlikely(brq->data.blocks << 9 != brq->data.bytes_xfered))
-			return MMC_BLK_PARTIAL;
-		else
-			return MMC_BLK_SUCCESS;
-	}
-
 	if (blk_rq_bytes(req) != brq->data.bytes_xfered)
 		return MMC_BLK_PARTIAL;
 
 	return MMC_BLK_SUCCESS;
 }
 
-static int mmc_blk_packed_err_check(struct mmc_card *card,
-				    struct mmc_async_req *areq)
-{
-	struct mmc_queue_req *mq_rq = container_of(areq, struct mmc_queue_req,
-			mmc_active);
-	struct request *req = mq_rq->req;
-	struct mmc_packed *packed = mq_rq->packed;
-	int err, check, status;
-	u8 *ext_csd;
-
-	packed->retries--;
-	check = mmc_blk_err_check(card, areq);
-	err = get_card_status(card, &status, 0);
-	if (err) {
-		pr_err("%s: error %d sending status command\n",
-		       req->rq_disk->disk_name, err);
-		return MMC_BLK_ABORT;
-	}
-
-	if (status & R1_EXCEPTION_EVENT) {
-		err = mmc_get_ext_csd(card, &ext_csd);
-		if (err) {
-			pr_err("%s: error %d sending ext_csd\n",
-			       req->rq_disk->disk_name, err);
-			return MMC_BLK_ABORT;
-		}
-
-		if ((ext_csd[EXT_CSD_EXP_EVENTS_STATUS] &
-		     EXT_CSD_PACKED_FAILURE) &&
-		    (ext_csd[EXT_CSD_PACKED_CMD_STATUS] &
-		     EXT_CSD_PACKED_GENERIC_ERROR)) {
-			if (ext_csd[EXT_CSD_PACKED_CMD_STATUS] &
-			    EXT_CSD_PACKED_INDEXED_ERROR) {
-				packed->idx_failure =
-				  ext_csd[EXT_CSD_PACKED_FAILURE_INDEX] - 1;
-				check = MMC_BLK_PARTIAL;
-			}
-			pr_err("%s: packed cmd failed, nr %u, sectors %u, "
-			       "failure index: %d\n",
-			       req->rq_disk->disk_name, packed->nr_entries,
-			       packed->blocks, packed->idx_failure);
-		}
-		kfree(ext_csd);
-	}
-
-	return check;
-}
-
 static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
 			       struct mmc_card *card,
 			       int disable_multi,
@@ -1488,7 +1413,7 @@ static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
 	u32 readcmd, writecmd;
 	struct mmc_blk_request *brq = &mqrq->brq;
 	struct request *req = mqrq->req;
-	struct mmc_blk_data *md = mq->data;
+	struct mmc_blk_data *md = mq->blkdata;
 	bool do_data_tag;
 
 	/*
@@ -1640,224 +1565,6 @@ static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
 	mmc_queue_bounce_pre(mqrq);
 }
 
-static inline u8 mmc_calc_packed_hdr_segs(struct request_queue *q,
-					  struct mmc_card *card)
-{
-	unsigned int hdr_sz = mmc_large_sector(card) ? 4096 : 512;
-	unsigned int max_seg_sz = queue_max_segment_size(q);
-	unsigned int len, nr_segs = 0;
-
-	do {
-		len = min(hdr_sz, max_seg_sz);
-		hdr_sz -= len;
-		nr_segs++;
-	} while (hdr_sz);
-
-	return nr_segs;
-}
-
-static u8 mmc_blk_prep_packed_list(struct mmc_queue *mq, struct request *req)
-{
-	struct request_queue *q = mq->queue;
-	struct mmc_card *card = mq->card;
-	struct request *cur = req, *next = NULL;
-	struct mmc_blk_data *md = mq->data;
-	struct mmc_queue_req *mqrq = mq->mqrq_cur;
-	bool en_rel_wr = card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN;
-	unsigned int req_sectors = 0, phys_segments = 0;
-	unsigned int max_blk_count, max_phys_segs;
-	bool put_back = true;
-	u8 max_packed_rw = 0;
-	u8 reqs = 0;
-
-	/*
-	 * We don't need to check packed for any further
-	 * operation of packed stuff as we set MMC_PACKED_NONE
-	 * and return zero for reqs if geting null packed. Also
-	 * we clean the flag of MMC_BLK_PACKED_CMD to avoid doing
-	 * it again when removing blk req.
-	 */
-	if (!mqrq->packed) {
-		md->flags &= (~MMC_BLK_PACKED_CMD);
-		goto no_packed;
-	}
-
-	if (!(md->flags & MMC_BLK_PACKED_CMD))
-		goto no_packed;
-
-	if ((rq_data_dir(cur) == WRITE) &&
-	    mmc_host_packed_wr(card->host))
-		max_packed_rw = card->ext_csd.max_packed_writes;
-
-	if (max_packed_rw == 0)
-		goto no_packed;
-
-	if (mmc_req_rel_wr(cur) &&
-	    (md->flags & MMC_BLK_REL_WR) && !en_rel_wr)
-		goto no_packed;
-
-	if (mmc_large_sector(card) &&
-	    !IS_ALIGNED(blk_rq_sectors(cur), 8))
-		goto no_packed;
-
-	mmc_blk_clear_packed(mqrq);
-
-	max_blk_count = min(card->host->max_blk_count,
-			    card->host->max_req_size >> 9);
-	if (unlikely(max_blk_count > 0xffff))
-		max_blk_count = 0xffff;
-
-	max_phys_segs = queue_max_segments(q);
-	req_sectors += blk_rq_sectors(cur);
-	phys_segments += cur->nr_phys_segments;
-
-	if (rq_data_dir(cur) == WRITE) {
-		req_sectors += mmc_large_sector(card) ? 8 : 1;
-		phys_segments += mmc_calc_packed_hdr_segs(q, card);
-	}
-
-	do {
-		if (reqs >= max_packed_rw - 1) {
-			put_back = false;
-			break;
-		}
-
-		spin_lock_irq(q->queue_lock);
-		next = blk_fetch_request(q);
-		spin_unlock_irq(q->queue_lock);
-		if (!next) {
-			put_back = false;
-			break;
-		}
-
-		if (mmc_large_sector(card) &&
-		    !IS_ALIGNED(blk_rq_sectors(next), 8))
-			break;
-
-		if (req_op(next) == REQ_OP_DISCARD ||
-		    req_op(next) == REQ_OP_SECURE_ERASE ||
-		    req_op(next) == REQ_OP_FLUSH)
-			break;
-
-		if (rq_data_dir(cur) != rq_data_dir(next))
-			break;
-
-		if (mmc_req_rel_wr(next) &&
-		    (md->flags & MMC_BLK_REL_WR) && !en_rel_wr)
-			break;
-
-		req_sectors += blk_rq_sectors(next);
-		if (req_sectors > max_blk_count)
-			break;
-
-		phys_segments +=  next->nr_phys_segments;
-		if (phys_segments > max_phys_segs)
-			break;
-
-		list_add_tail(&next->queuelist, &mqrq->packed->list);
-		cur = next;
-		reqs++;
-	} while (1);
-
-	if (put_back) {
-		spin_lock_irq(q->queue_lock);
-		blk_requeue_request(q, next);
-		spin_unlock_irq(q->queue_lock);
-	}
-
-	if (reqs > 0) {
-		list_add(&req->queuelist, &mqrq->packed->list);
-		mqrq->packed->nr_entries = ++reqs;
-		mqrq->packed->retries = reqs;
-		return reqs;
-	}
-
-no_packed:
-	mqrq->cmd_type = MMC_PACKED_NONE;
-	return 0;
-}
-
-static void mmc_blk_packed_hdr_wrq_prep(struct mmc_queue_req *mqrq,
-					struct mmc_card *card,
-					struct mmc_queue *mq)
-{
-	struct mmc_blk_request *brq = &mqrq->brq;
-	struct request *req = mqrq->req;
-	struct request *prq;
-	struct mmc_blk_data *md = mq->data;
-	struct mmc_packed *packed = mqrq->packed;
-	bool do_rel_wr, do_data_tag;
-	__le32 *packed_cmd_hdr;
-	u8 hdr_blocks;
-	u8 i = 1;
-
-	mqrq->cmd_type = MMC_PACKED_WRITE;
-	packed->blocks = 0;
-	packed->idx_failure = MMC_PACKED_NR_IDX;
-
-	packed_cmd_hdr = packed->cmd_hdr;
-	memset(packed_cmd_hdr, 0, sizeof(packed->cmd_hdr));
-	packed_cmd_hdr[0] = cpu_to_le32((packed->nr_entries << 16) |
-		(PACKED_CMD_WR << 8) | PACKED_CMD_VER);
-	hdr_blocks = mmc_large_sector(card) ? 8 : 1;
-
-	/*
-	 * Argument for each entry of packed group
-	 */
-	list_for_each_entry(prq, &packed->list, queuelist) {
-		do_rel_wr = mmc_req_rel_wr(prq) && (md->flags & MMC_BLK_REL_WR);
-		do_data_tag = (card->ext_csd.data_tag_unit_size) &&
-			(prq->cmd_flags & REQ_META) &&
-			(rq_data_dir(prq) == WRITE) &&
-			blk_rq_bytes(prq) >= card->ext_csd.data_tag_unit_size;
-		/* Argument of CMD23 */
-		packed_cmd_hdr[(i * 2)] = cpu_to_le32(
-			(do_rel_wr ? MMC_CMD23_ARG_REL_WR : 0) |
-			(do_data_tag ? MMC_CMD23_ARG_TAG_REQ : 0) |
-			blk_rq_sectors(prq));
-		/* Argument of CMD18 or CMD25 */
-		packed_cmd_hdr[((i * 2)) + 1] = cpu_to_le32(
-			mmc_card_blockaddr(card) ?
-			blk_rq_pos(prq) : blk_rq_pos(prq) << 9);
-		packed->blocks += blk_rq_sectors(prq);
-		i++;
-	}
-
-	memset(brq, 0, sizeof(struct mmc_blk_request));
-	brq->mrq.cmd = &brq->cmd;
-	brq->mrq.data = &brq->data;
-	brq->mrq.sbc = &brq->sbc;
-	brq->mrq.stop = &brq->stop;
-
-	brq->sbc.opcode = MMC_SET_BLOCK_COUNT;
-	brq->sbc.arg = MMC_CMD23_ARG_PACKED | (packed->blocks + hdr_blocks);
-	brq->sbc.flags = MMC_RSP_R1 | MMC_CMD_AC;
-
-	brq->cmd.opcode = MMC_WRITE_MULTIPLE_BLOCK;
-	brq->cmd.arg = blk_rq_pos(req);
-	if (!mmc_card_blockaddr(card))
-		brq->cmd.arg <<= 9;
-	brq->cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC;
-
-	brq->data.blksz = 512;
-	brq->data.blocks = packed->blocks + hdr_blocks;
-	brq->data.flags = MMC_DATA_WRITE;
-
-	brq->stop.opcode = MMC_STOP_TRANSMISSION;
-	brq->stop.arg = 0;
-	brq->stop.flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC;
-
-	mmc_set_data_timeout(&brq->data, card);
-
-	brq->data.sg = mqrq->sg;
-	brq->data.sg_len = mmc_queue_map_sg(mq, mqrq);
-
-	mqrq->mmc_active.mrq = &brq->mrq;
-	mqrq->mmc_active.err_check = mmc_blk_packed_err_check;
-
-	mmc_queue_bounce_pre(mqrq);
-}
-
 static int mmc_blk_cmd_err(struct mmc_blk_data *md, struct mmc_card *card,
 			   struct mmc_blk_request *brq, struct request *req,
 			   int ret)
@@ -1881,97 +1588,25 @@ static int mmc_blk_cmd_err(struct mmc_blk_data *md, struct mmc_card *card,
 			ret = blk_end_request(req, 0, blocks << 9);
 		}
 	} else {
-		if (!mmc_packed_cmd(mq_rq->cmd_type))
-			ret = blk_end_request(req, 0, brq->data.bytes_xfered);
+		ret = blk_end_request(req, 0, brq->data.bytes_xfered);
 	}
 	return ret;
 }
 
-static int mmc_blk_end_packed_req(struct mmc_queue_req *mq_rq)
-{
-	struct request *prq;
-	struct mmc_packed *packed = mq_rq->packed;
-	int idx = packed->idx_failure, i = 0;
-	int ret = 0;
-
-	while (!list_empty(&packed->list)) {
-		prq = list_entry_rq(packed->list.next);
-		if (idx == i) {
-			/* retry from error index */
-			packed->nr_entries -= idx;
-			mq_rq->req = prq;
-			ret = 1;
-
-			if (packed->nr_entries == MMC_PACKED_NR_SINGLE) {
-				list_del_init(&prq->queuelist);
-				mmc_blk_clear_packed(mq_rq);
-			}
-			return ret;
-		}
-		list_del_init(&prq->queuelist);
-		blk_end_request(prq, 0, blk_rq_bytes(prq));
-		i++;
-	}
-
-	mmc_blk_clear_packed(mq_rq);
-	return ret;
-}
-
-static void mmc_blk_abort_packed_req(struct mmc_queue_req *mq_rq)
-{
-	struct request *prq;
-	struct mmc_packed *packed = mq_rq->packed;
-
-	while (!list_empty(&packed->list)) {
-		prq = list_entry_rq(packed->list.next);
-		list_del_init(&prq->queuelist);
-		blk_end_request(prq, -EIO, blk_rq_bytes(prq));
-	}
-
-	mmc_blk_clear_packed(mq_rq);
-}
-
-static void mmc_blk_revert_packed_req(struct mmc_queue *mq,
-				      struct mmc_queue_req *mq_rq)
-{
-	struct request *prq;
-	struct request_queue *q = mq->queue;
-	struct mmc_packed *packed = mq_rq->packed;
-
-	while (!list_empty(&packed->list)) {
-		prq = list_entry_rq(packed->list.prev);
-		if (prq->queuelist.prev != &packed->list) {
-			list_del_init(&prq->queuelist);
-			spin_lock_irq(q->queue_lock);
-			blk_requeue_request(mq->queue, prq);
-			spin_unlock_irq(q->queue_lock);
-		} else {
-			list_del_init(&prq->queuelist);
-		}
-	}
-
-	mmc_blk_clear_packed(mq_rq);
-}
-
 static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
 {
-	struct mmc_blk_data *md = mq->data;
+	struct mmc_blk_data *md = mq->blkdata;
 	struct mmc_card *card = md->queue.card;
-	struct mmc_blk_request *brq = &mq->mqrq_cur->brq;
+	struct mmc_blk_request *brq;
 	int ret = 1, disable_multi = 0, retry = 0, type, retune_retry_done = 0;
 	enum mmc_blk_status status;
 	struct mmc_queue_req *mq_rq;
-	struct request *req = rqc;
+	struct request *req;
 	struct mmc_async_req *areq;
-	const u8 packed_nr = 2;
-	u8 reqs = 0;
 
 	if (!rqc && !mq->mqrq_prev->req)
 		return 0;
 
-	if (rqc)
-		reqs = mmc_blk_prep_packed_list(mq, rqc);
-
 	do {
 		if (rqc) {
 			/*
@@ -1981,20 +1616,18 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
 			if (mmc_large_sector(card) &&
 				!IS_ALIGNED(blk_rq_sectors(rqc), 8)) {
 				pr_err("%s: Transfer size is not 4KB sector size aligned\n",
-					req->rq_disk->disk_name);
+					rqc->rq_disk->disk_name);
 				mq_rq = mq->mqrq_cur;
+				req = rqc;
+				rqc = NULL;
 				goto cmd_abort;
 			}
 
-			if (reqs >= packed_nr)
-				mmc_blk_packed_hdr_wrq_prep(mq->mqrq_cur,
-							    card, mq);
-			else
-				mmc_blk_rw_rq_prep(mq->mqrq_cur, card, 0, mq);
+			mmc_blk_rw_rq_prep(mq->mqrq_cur, card, 0, mq);
 			areq = &mq->mqrq_cur->mmc_active;
 		} else
 			areq = NULL;
-		areq = mmc_start_req(card->host, areq, (int *) &status);
+		areq = mmc_start_req(card->host, areq, &status);
 		if (!areq) {
 			if (status == MMC_BLK_NEW_REQUEST)
 				mq->flags |= MMC_QUEUE_NEW_REQUEST;
@@ -2015,13 +1648,8 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
 			 */
 			mmc_blk_reset_success(md, type);
 
-			if (mmc_packed_cmd(mq_rq->cmd_type)) {
-				ret = mmc_blk_end_packed_req(mq_rq);
-				break;
-			} else {
-				ret = blk_end_request(req, 0,
-						brq->data.bytes_xfered);
-			}
+			ret = blk_end_request(req, 0,
+					brq->data.bytes_xfered);
 
 			/*
 			 * If the blk_end_request function returns non-zero even
@@ -2058,8 +1686,7 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
 			err = mmc_blk_reset(md, card->host, type);
 			if (!err)
 				break;
-			if (err == -ENODEV ||
-				mmc_packed_cmd(mq_rq->cmd_type))
+			if (err == -ENODEV)
 				goto cmd_abort;
 			/* Fall through */
 		}
@@ -2090,23 +1717,14 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
 		}
 
 		if (ret) {
-			if (mmc_packed_cmd(mq_rq->cmd_type)) {
-				if (!mq_rq->packed->retries)
-					goto cmd_abort;
-				mmc_blk_packed_hdr_wrq_prep(mq_rq, card, mq);
-				mmc_start_req(card->host,
-					      &mq_rq->mmc_active, NULL);
-			} else {
-
-				/*
-				 * In case of a incomplete request
-				 * prepare it again and resend.
-				 */
-				mmc_blk_rw_rq_prep(mq_rq, card,
-						disable_multi, mq);
-				mmc_start_req(card->host,
-						&mq_rq->mmc_active, NULL);
-			}
+			/*
+			 * In case of a incomplete request
+			 * prepare it again and resend.
+			 */
+			mmc_blk_rw_rq_prep(mq_rq, card,
+					disable_multi, mq);
+			mmc_start_req(card->host,
+					&mq_rq->mmc_active, NULL);
 			mq_rq->brq.retune_retry_done = retune_retry_done;
 		}
 	} while (ret);
@@ -2114,28 +1732,18 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
 	return 1;
 
  cmd_abort:
-	if (mmc_packed_cmd(mq_rq->cmd_type)) {
-		mmc_blk_abort_packed_req(mq_rq);
-	} else {
-		if (mmc_card_removed(card))
-			req->cmd_flags |= REQ_QUIET;
-		while (ret)
-			ret = blk_end_request(req, -EIO,
-					blk_rq_cur_bytes(req));
-	}
+	if (mmc_card_removed(card))
+		req->rq_flags |= RQF_QUIET;
+	while (ret)
+		ret = blk_end_request(req, -EIO,
+				blk_rq_cur_bytes(req));
 
  start_new_req:
 	if (rqc) {
 		if (mmc_card_removed(card)) {
-			rqc->cmd_flags |= REQ_QUIET;
+			rqc->rq_flags |= RQF_QUIET;
 			blk_end_request_all(rqc, -EIO);
 		} else {
-			/*
-			 * If current request is packed, it needs to put back.
-			 */
-			if (mmc_packed_cmd(mq->mqrq_cur->cmd_type))
-				mmc_blk_revert_packed_req(mq, mq->mqrq_cur);
-
 			mmc_blk_rw_rq_prep(mq->mqrq_cur, card, 0, mq);
 			mmc_start_req(card->host,
 				      &mq->mqrq_cur->mmc_active, NULL);
@@ -2148,10 +1756,8 @@ static int mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *rqc)
 int mmc_blk_issue_rq(struct mmc_queue *mq, struct request *req)
 {
 	int ret;
-	struct mmc_blk_data *md = mq->data;
+	struct mmc_blk_data *md = mq->blkdata;
 	struct mmc_card *card = md->queue.card;
-	struct mmc_host *host = card->host;
-	unsigned long flags;
 	bool req_is_special = mmc_req_is_special(req);
 
 	if (req && !mq->mqrq_prev->req)
@@ -2184,11 +1790,6 @@ int mmc_blk_issue_rq(struct mmc_queue *mq, struct request *req)
 			mmc_blk_issue_rw_rq(mq, NULL);
 		ret = mmc_blk_issue_flush(mq, req);
 	} else {
-		if (!req && host->areq) {
-			spin_lock_irqsave(&host->context_info.lock, flags);
-			host->context_info.is_waiting_last_req = true;
-			spin_unlock_irqrestore(&host->context_info.lock, flags);
-		}
 		ret = mmc_blk_issue_rw_rq(mq, req);
 	}
 
@@ -2266,7 +1867,7 @@ again:
 	if (ret)
 		goto err_putdisk;
 
-	md->queue.data = md;
+	md->queue.blkdata = md;
 
 	md->disk->major	= MMC_BLOCK_MAJOR;
 	md->disk->first_minor = devidx * perdev_minors;
@@ -2318,14 +1919,6 @@ again:
 		blk_queue_write_cache(md->queue.queue, true, true);
 	}
 
-	if (mmc_card_mmc(card) &&
-	    (area_type == MMC_BLK_DATA_AREA_MAIN) &&
-	    (md->flags & MMC_BLK_CMD23) &&
-	    card->ext_csd.packed_event_en) {
-		if (!mmc_packed_init(&md->queue, card))
-			md->flags |= MMC_BLK_PACKED_CMD;
-	}
-
 	return md;
 
  err_putdisk:
@@ -2429,8 +2022,6 @@ static void mmc_blk_remove_req(struct mmc_blk_data *md)
 		 */
 		card = md->queue.card;
 		mmc_cleanup_queue(&md->queue);
-		if (md->flags & MMC_BLK_PACKED_CMD)
-			mmc_packed_clean(&md->queue);
 		if (md->disk->flags & GENHD_FL_UP) {
 			device_remove_file(disk_to_dev(md->disk), &md->force_ro);
 			if ((md->area_type & MMC_BLK_DATA_AREA_BOOT) &&
diff --git a/drivers/mmc/card/block.h b/drivers/mmc/core/block.h
index cdabb2ee74be..cdabb2ee74be 100644
--- a/drivers/mmc/card/block.h
+++ b/drivers/mmc/core/block.h
diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c
index 2553d903a82b..543eadd230e5 100644
--- a/drivers/mmc/core/core.c
+++ b/drivers/mmc/core/core.c
@@ -306,16 +306,16 @@ static int mmc_start_request(struct mmc_host *host, struct mmc_request *mrq)
 		mrq->sbc->mrq = mrq;
 	}
 	if (mrq->data) {
-		BUG_ON(mrq->data->blksz > host->max_blk_size);
-		BUG_ON(mrq->data->blocks > host->max_blk_count);
-		BUG_ON(mrq->data->blocks * mrq->data->blksz >
-			host->max_req_size);
-
+		if (mrq->data->blksz > host->max_blk_size ||
+		    mrq->data->blocks > host->max_blk_count ||
+		    mrq->data->blocks * mrq->data->blksz > host->max_req_size)
+			return -EINVAL;
 #ifdef CONFIG_MMC_DEBUG
 		sz = 0;
 		for_each_sg(mrq->data->sg, sg, mrq->data->sg_len, i)
 			sz += sg->length;
-		BUG_ON(sz != mrq->data->blocks * mrq->data->blksz);
+		if (sz != mrq->data->blocks * mrq->data->blksz)
+			return -EINVAL;
 #endif
 
 		mrq->cmd->data = mrq->data;
@@ -349,8 +349,6 @@ void mmc_start_bkops(struct mmc_card *card, bool from_exception)
 	int timeout;
 	bool use_busy_signal;
 
-	BUG_ON(!card);
-
 	if (!card->ext_csd.man_bkops_en || mmc_card_doing_bkops(card))
 		return;
 
@@ -380,7 +378,7 @@ void mmc_start_bkops(struct mmc_card *card, bool from_exception)
 	mmc_retune_hold(card->host);
 
 	err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
-			EXT_CSD_BKOPS_START, 1, timeout,
+			EXT_CSD_BKOPS_START, 1, timeout, 0,
 			use_busy_signal, true, false);
 	if (err) {
 		pr_warn("%s: Error %d starting bkops\n",
@@ -497,32 +495,28 @@ static int __mmc_start_req(struct mmc_host *host, struct mmc_request *mrq)
  *
  * Returns enum mmc_blk_status after checking errors.
  */
-static int mmc_wait_for_data_req_done(struct mmc_host *host,
+static enum mmc_blk_status mmc_wait_for_data_req_done(struct mmc_host *host,
 				      struct mmc_request *mrq,
 				      struct mmc_async_req *next_req)
 {
 	struct mmc_command *cmd;
 	struct mmc_context_info *context_info = &host->context_info;
-	int err;
-	unsigned long flags;
+	enum mmc_blk_status status;
 
 	while (1) {
 		wait_event_interruptible(context_info->wait,
 				(context_info->is_done_rcv ||
 				 context_info->is_new_req));
-		spin_lock_irqsave(&context_info->lock, flags);
 		context_info->is_waiting_last_req = false;
-		spin_unlock_irqrestore(&context_info->lock, flags);
 		if (context_info->is_done_rcv) {
 			context_info->is_done_rcv = false;
-			context_info->is_new_req = false;
 			cmd = mrq->cmd;
 
 			if (!cmd->error || !cmd->retries ||
 			    mmc_card_removed(host->card)) {
-				err = host->areq->err_check(host->card,
-							    host->areq);
-				break; /* return err */
+				status = host->areq->err_check(host->card,
+							       host->areq);
+				break; /* return status */
 			} else {
 				mmc_retune_recheck(host);
 				pr_info("%s: req failed (CMD%u): %d, retrying...\n",
@@ -534,13 +528,12 @@ static int mmc_wait_for_data_req_done(struct mmc_host *host,
 				continue; /* wait for done/new event again */
 			}
 		} else if (context_info->is_new_req) {
-			context_info->is_new_req = false;
 			if (!next_req)
 				return MMC_BLK_NEW_REQUEST;
 		}
 	}
 	mmc_retune_release(host);
-	return err;
+	return status;
 }
 
 void mmc_wait_for_req_done(struct mmc_host *host, struct mmc_request *mrq)
@@ -611,18 +604,15 @@ EXPORT_SYMBOL(mmc_is_req_done);
  *	mmc_pre_req - Prepare for a new request
  *	@host: MMC host to prepare command
  *	@mrq: MMC request to prepare for
- *	@is_first_req: true if there is no previous started request
- *                     that may run in parellel to this call, otherwise false
  *
  *	mmc_pre_req() is called in prior to mmc_start_req() to let
  *	host prepare for the new request. Preparation of a request may be
  *	performed while another request is running on the host.
  */
-static void mmc_pre_req(struct mmc_host *host, struct mmc_request *mrq,
-		 bool is_first_req)
+static void mmc_pre_req(struct mmc_host *host, struct mmc_request *mrq)
 {
 	if (host->ops->pre_req)
-		host->ops->pre_req(host, mrq, is_first_req);
+		host->ops->pre_req(host, mrq);
 }
 
 /**
@@ -658,21 +648,22 @@ static void mmc_post_req(struct mmc_host *host, struct mmc_request *mrq,
  *	is returned without waiting. NULL is not an error condition.
  */
 struct mmc_async_req *mmc_start_req(struct mmc_host *host,
-				    struct mmc_async_req *areq, int *error)
+				    struct mmc_async_req *areq,
+				    enum mmc_blk_status *ret_stat)
 {
-	int err = 0;
+	enum mmc_blk_status status = MMC_BLK_SUCCESS;
 	int start_err = 0;
 	struct mmc_async_req *data = host->areq;
 
 	/* Prepare a new request */
 	if (areq)
-		mmc_pre_req(host, areq->mrq, !host->areq);
+		mmc_pre_req(host, areq->mrq);
 
 	if (host->areq) {
-		err = mmc_wait_for_data_req_done(host, host->areq->mrq,	areq);
-		if (err == MMC_BLK_NEW_REQUEST) {
-			if (error)
-				*error = err;
+		status = mmc_wait_for_data_req_done(host, host->areq->mrq, areq);
+		if (status == MMC_BLK_NEW_REQUEST) {
+			if (ret_stat)
+				*ret_stat = status;
 			/*
 			 * The previous request was not completed,
 			 * nothing to return
@@ -695,27 +686,27 @@ struct mmc_async_req *mmc_start_req(struct mmc_host *host,
 
 			/* prepare the request again */
 			if (areq)
-				mmc_pre_req(host, areq->mrq, !host->areq);
+				mmc_pre_req(host, areq->mrq);
 		}
 	}
 
-	if (!err && areq)
+	if (status == MMC_BLK_SUCCESS && areq)
 		start_err = __mmc_start_data_req(host, areq->mrq);
 
 	if (host->areq)
 		mmc_post_req(host, host->areq->mrq, 0);
 
 	 /* Cancel a prepared request if it was not started. */
-	if ((err || start_err) && areq)
+	if ((status != MMC_BLK_SUCCESS || start_err) && areq)
 		mmc_post_req(host, areq->mrq, -EINVAL);
 
-	if (err)
+	if (status != MMC_BLK_SUCCESS)
 		host->areq = NULL;
 	else
 		host->areq = areq;
 
-	if (error)
-		*error = err;
+	if (ret_stat)
+		*ret_stat = status;
 	return data;
 }
 EXPORT_SYMBOL(mmc_start_req);
@@ -754,8 +745,6 @@ int mmc_interrupt_hpi(struct mmc_card *card)
 	u32 status;
 	unsigned long prg_wait;
 
-	BUG_ON(!card);
-
 	if (!card->ext_csd.hpi_en) {
 		pr_info("%s: HPI enable bit unset\n", mmc_hostname(card->host));
 		return 1;
@@ -850,7 +839,6 @@ int mmc_stop_bkops(struct mmc_card *card)
 {
 	int err = 0;
 
-	BUG_ON(!card);
 	err = mmc_interrupt_hpi(card);
 
 	/*
@@ -1666,8 +1654,6 @@ int mmc_set_signal_voltage(struct mmc_host *host, int signal_voltage, u32 ocr)
 	int err = 0;
 	u32 clock;
 
-	BUG_ON(!host);
-
 	/*
 	 * Send CMD11 only if the request is to switch the card to
 	 * 1.8V signalling.
@@ -1884,9 +1870,7 @@ void mmc_power_cycle(struct mmc_host *host, u32 ocr)
  */
 static void __mmc_release_bus(struct mmc_host *host)
 {
-	BUG_ON(!host);
-	BUG_ON(host->bus_refs);
-	BUG_ON(!host->bus_dead);
+	WARN_ON(!host->bus_dead);
 
 	host->bus_ops = NULL;
 }
@@ -1926,15 +1910,12 @@ void mmc_attach_bus(struct mmc_host *host, const struct mmc_bus_ops *ops)
 {
 	unsigned long flags;
 
-	BUG_ON(!host);
-	BUG_ON(!ops);
-
 	WARN_ON(!host->claimed);
 
 	spin_lock_irqsave(&host->lock, flags);
 
-	BUG_ON(host->bus_ops);
-	BUG_ON(host->bus_refs);
+	WARN_ON(host->bus_ops);
+	WARN_ON(host->bus_refs);
 
 	host->bus_ops = ops;
 	host->bus_refs = 1;
@@ -1950,8 +1931,6 @@ void mmc_detach_bus(struct mmc_host *host)
 {
 	unsigned long flags;
 
-	BUG_ON(!host);
-
 	WARN_ON(!host->claimed);
 	WARN_ON(!host->bus_ops);
 
@@ -2824,12 +2803,11 @@ void mmc_start_host(struct mmc_host *host)
 	host->rescan_disable = 0;
 	host->ios.power_mode = MMC_POWER_UNDEFINED;
 
-	mmc_claim_host(host);
-	if (host->caps2 & MMC_CAP2_NO_PRESCAN_POWERUP)
-		mmc_power_off(host);
-	else
+	if (!(host->caps2 & MMC_CAP2_NO_PRESCAN_POWERUP)) {
+		mmc_claim_host(host);
 		mmc_power_up(host, host->ocr_avail);
-	mmc_release_host(host);
+		mmc_release_host(host);
+	}
 
 	mmc_gpiod_request_cd_irq(host);
 	_mmc_detect_change(host, 0, false);
@@ -2865,8 +2843,6 @@ void mmc_stop_host(struct mmc_host *host)
 	}
 	mmc_bus_put(host);
 
-	BUG_ON(host->card);
-
 	mmc_claim_host(host);
 	mmc_power_off(host);
 	mmc_release_host(host);
@@ -3019,7 +2995,6 @@ void mmc_unregister_pm_notifier(struct mmc_host *host)
  */
 void mmc_init_context_info(struct mmc_host *host)
 {
-	spin_lock_init(&host->context_info.lock);
 	host->context_info.is_new_req = false;
 	host->context_info.is_done_rcv = false;
 	host->context_info.is_waiting_last_req = false;
diff --git a/drivers/mmc/core/debugfs.c b/drivers/mmc/core/debugfs.c
index c8451ce557ae..30623b8b86a4 100644
--- a/drivers/mmc/core/debugfs.c
+++ b/drivers/mmc/core/debugfs.c
@@ -321,7 +321,11 @@ static int mmc_ext_csd_open(struct inode *inode, struct file *filp)
 	for (i = 0; i < 512; i++)
 		n += sprintf(buf + n, "%02x", ext_csd[i]);
 	n += sprintf(buf + n, "\n");
-	BUG_ON(n != EXT_CSD_STR_LEN);
+
+	if (n != EXT_CSD_STR_LEN) {
+		err = -EINVAL;
+		goto out_free;
+	}
 
 	filp->private_data = buf;
 	kfree(ext_csd);
diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c
index df19777068a6..b61b52f9da3d 100644
--- a/drivers/mmc/core/mmc.c
+++ b/drivers/mmc/core/mmc.c
@@ -618,6 +618,24 @@ static int mmc_decode_ext_csd(struct mmc_card *card, u8 *ext_csd)
 			(ext_csd[EXT_CSD_SUPPORTED_MODE] & 0x1) &&
 			!(ext_csd[EXT_CSD_FW_CONFIG] & 0x1);
 	}
+
+	/* eMMC v5.1 or later */
+	if (card->ext_csd.rev >= 8) {
+		card->ext_csd.cmdq_support = ext_csd[EXT_CSD_CMDQ_SUPPORT] &
+					     EXT_CSD_CMDQ_SUPPORTED;
+		card->ext_csd.cmdq_depth = (ext_csd[EXT_CSD_CMDQ_DEPTH] &
+					    EXT_CSD_CMDQ_DEPTH_MASK) + 1;
+		/* Exclude inefficiently small queue depths */
+		if (card->ext_csd.cmdq_depth <= 2) {
+			card->ext_csd.cmdq_support = false;
+			card->ext_csd.cmdq_depth = 0;
+		}
+		if (card->ext_csd.cmdq_support) {
+			pr_debug("%s: Command Queue supported depth %u\n",
+				 mmc_hostname(card->host),
+				 card->ext_csd.cmdq_depth);
+		}
+	}
 out:
 	return err;
 }
@@ -1003,19 +1021,6 @@ static int mmc_select_bus_width(struct mmc_card *card)
 	return err;
 }
 
-/* Caller must hold re-tuning */
-static int mmc_switch_status(struct mmc_card *card)
-{
-	u32 status;
-	int err;
-
-	err = mmc_send_status(card, &status);
-	if (err)
-		return err;
-
-	return mmc_switch_status_error(card->host, status);
-}
-
 /*
  * Switch to the high-speed mode
  */
@@ -1025,13 +1030,8 @@ static int mmc_select_hs(struct mmc_card *card)
 
 	err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
 			   EXT_CSD_HS_TIMING, EXT_CSD_TIMING_HS,
-			   card->ext_csd.generic_cmd6_time,
-			   true, false, true);
-	if (!err) {
-		mmc_set_timing(card->host, MMC_TIMING_MMC_HS);
-		err = mmc_switch_status(card);
-	}
-
+			   card->ext_csd.generic_cmd6_time, MMC_TIMING_MMC_HS,
+			   true, true, true);
 	if (err)
 		pr_warn("%s: switch to high-speed failed, err:%d\n",
 			mmc_hostname(card->host), err);
@@ -1058,10 +1058,12 @@ static int mmc_select_hs_ddr(struct mmc_card *card)
 	ext_csd_bits = (bus_width == MMC_BUS_WIDTH_8) ?
 		EXT_CSD_DDR_BUS_WIDTH_8 : EXT_CSD_DDR_BUS_WIDTH_4;
 
-	err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
-			EXT_CSD_BUS_WIDTH,
-			ext_csd_bits,
-			card->ext_csd.generic_cmd6_time);
+	err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
+			   EXT_CSD_BUS_WIDTH,
+			   ext_csd_bits,
+			   card->ext_csd.generic_cmd6_time,
+			   MMC_TIMING_MMC_DDR52,
+			   true, true, true);
 	if (err) {
 		pr_err("%s: switch to bus width %d ddr failed\n",
 			mmc_hostname(host), 1 << bus_width);
@@ -1104,9 +1106,6 @@ static int mmc_select_hs_ddr(struct mmc_card *card)
 	if (err)
 		err = __mmc_set_signal_voltage(host, MMC_SIGNAL_VOLTAGE_330);
 
-	if (!err)
-		mmc_set_timing(host, MMC_TIMING_MMC_DDR52);
-
 	return err;
 }
 
@@ -1128,7 +1127,7 @@ static int mmc_select_hs400(struct mmc_card *card)
 	val = EXT_CSD_TIMING_HS;
 	err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
 			   EXT_CSD_HS_TIMING, val,
-			   card->ext_csd.generic_cmd6_time,
+			   card->ext_csd.generic_cmd6_time, 0,
 			   true, false, true);
 	if (err) {
 		pr_err("%s: switch to high-speed from hs200 failed, err:%d\n",
@@ -1163,7 +1162,7 @@ static int mmc_select_hs400(struct mmc_card *card)
 	      card->drive_strength << EXT_CSD_DRV_STR_SHIFT;
 	err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
 			   EXT_CSD_HS_TIMING, val,
-			   card->ext_csd.generic_cmd6_time,
+			   card->ext_csd.generic_cmd6_time, 0,
 			   true, false, true);
 	if (err) {
 		pr_err("%s: switch to hs400 failed, err:%d\n",
@@ -1206,7 +1205,7 @@ int mmc_hs400_to_hs200(struct mmc_card *card)
 	/* Switch HS400 to HS DDR */
 	val = EXT_CSD_TIMING_HS;
 	err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_HS_TIMING,
-			   val, card->ext_csd.generic_cmd6_time,
+			   val, card->ext_csd.generic_cmd6_time, 0,
 			   true, false, true);
 	if (err)
 		goto out_err;
@@ -1220,7 +1219,7 @@ int mmc_hs400_to_hs200(struct mmc_card *card)
 	/* Switch HS DDR to HS */
 	err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BUS_WIDTH,
 			   EXT_CSD_BUS_WIDTH_8, card->ext_csd.generic_cmd6_time,
-			   true, false, true);
+			   0, true, false, true);
 	if (err)
 		goto out_err;
 
@@ -1234,14 +1233,19 @@ int mmc_hs400_to_hs200(struct mmc_card *card)
 	val = EXT_CSD_TIMING_HS200 |
 	      card->drive_strength << EXT_CSD_DRV_STR_SHIFT;
 	err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_HS_TIMING,
-			   val, card->ext_csd.generic_cmd6_time,
+			   val, card->ext_csd.generic_cmd6_time, 0,
 			   true, false, true);
 	if (err)
 		goto out_err;
 
 	mmc_set_timing(host, MMC_TIMING_MMC_HS200);
 
-	err = mmc_switch_status(card);
+	/*
+	 * For HS200, CRC errors are not a reliable way to know the switch
+	 * failed. If there really is a problem, we would expect tuning will
+	 * fail and the result ends up the same.
+	 */
+	err = __mmc_switch_status(card, false);
 	if (err)
 		goto out_err;
 
@@ -1281,16 +1285,23 @@ static int mmc_select_hs400es(struct mmc_card *card)
 		goto out_err;
 
 	/* Switch card to HS mode */
-	err = mmc_select_hs(card);
-	if (err)
+	err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
+			   EXT_CSD_HS_TIMING, EXT_CSD_TIMING_HS,
+			   card->ext_csd.generic_cmd6_time, 0,
+			   true, false, true);
+	if (err) {
+		pr_err("%s: switch to hs for hs400es failed, err:%d\n",
+			mmc_hostname(host), err);
 		goto out_err;
+	}
 
-	mmc_set_clock(host, card->ext_csd.hs_max_dtr);
-
+	mmc_set_timing(host, MMC_TIMING_MMC_HS);
 	err = mmc_switch_status(card);
 	if (err)
 		goto out_err;
 
+	mmc_set_clock(host, card->ext_csd.hs_max_dtr);
+
 	/* Switch card to DDR with strobe bit */
 	val = EXT_CSD_DDR_BUS_WIDTH_8 | EXT_CSD_BUS_WIDTH_STROBE;
 	err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
@@ -1308,7 +1319,7 @@ static int mmc_select_hs400es(struct mmc_card *card)
 	      card->drive_strength << EXT_CSD_DRV_STR_SHIFT;
 	err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
 			   EXT_CSD_HS_TIMING, val,
-			   card->ext_csd.generic_cmd6_time,
+			   card->ext_csd.generic_cmd6_time, 0,
 			   true, false, true);
 	if (err) {
 		pr_err("%s: switch to hs400es failed, err:%d\n",
@@ -1390,14 +1401,20 @@ static int mmc_select_hs200(struct mmc_card *card)
 		      card->drive_strength << EXT_CSD_DRV_STR_SHIFT;
 		err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
 				   EXT_CSD_HS_TIMING, val,
-				   card->ext_csd.generic_cmd6_time,
+				   card->ext_csd.generic_cmd6_time, 0,
 				   true, false, true);
 		if (err)
 			goto err;
 		old_timing = host->ios.timing;
 		mmc_set_timing(host, MMC_TIMING_MMC_HS200);
 
-		err = mmc_switch_status(card);
+		/*
+		 * For HS200, CRC errors are not a reliable way to know the
+		 * switch failed. If there really is a problem, we would expect
+		 * tuning will fail and the result ends up the same.
+		 */
+		err = __mmc_switch_status(card, false);
+
 		/*
 		 * mmc_select_timing() assumes timing has not changed if
 		 * it is a switch error.
@@ -1480,7 +1497,6 @@ static int mmc_init_card(struct mmc_host *host, u32 ocr,
 	u32 cid[4];
 	u32 rocr;
 
-	BUG_ON(!host);
 	WARN_ON(!host->claimed);
 
 	/* Set correct bus mode for MMC before attempting init */
@@ -1854,7 +1870,7 @@ static int mmc_poweroff_notify(struct mmc_card *card, unsigned int notify_type)
 
 	err = __mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
 			EXT_CSD_POWER_OFF_NOTIFICATION,
-			notify_type, timeout, true, false, false);
+			notify_type, timeout, 0, true, false, false);
 	if (err)
 		pr_err("%s: Power Off Notification timed out, %u\n",
 		       mmc_hostname(card->host), timeout);
@@ -1870,9 +1886,6 @@ static int mmc_poweroff_notify(struct mmc_card *card, unsigned int notify_type)
  */
 static void mmc_remove(struct mmc_host *host)
 {
-	BUG_ON(!host);
-	BUG_ON(!host->card);
-
 	mmc_remove_card(host->card);
 	host->card = NULL;
 }
@@ -1892,9 +1905,6 @@ static void mmc_detect(struct mmc_host *host)
 {
 	int err;
 
-	BUG_ON(!host);
-	BUG_ON(!host->card);
-
 	mmc_get_card(host->card);
 
 	/*
@@ -1920,9 +1930,6 @@ static int _mmc_suspend(struct mmc_host *host, bool is_suspend)
 	unsigned int notify_type = is_suspend ? EXT_CSD_POWER_OFF_SHORT :
 					EXT_CSD_POWER_OFF_LONG;
 
-	BUG_ON(!host);
-	BUG_ON(!host->card);
-
 	mmc_claim_host(host);
 
 	if (mmc_card_suspended(host->card))
@@ -1979,9 +1986,6 @@ static int _mmc_resume(struct mmc_host *host)
 {
 	int err = 0;
 
-	BUG_ON(!host);
-	BUG_ON(!host->card);
-
 	mmc_claim_host(host);
 
 	if (!mmc_card_suspended(host->card))
@@ -2114,7 +2118,6 @@ int mmc_attach_mmc(struct mmc_host *host)
 	int err;
 	u32 ocr, rocr;
 
-	BUG_ON(!host);
 	WARN_ON(!host->claimed);
 
 	/* Set correct bus mode for MMC before attempting attach */
diff --git a/drivers/mmc/core/mmc_ops.c b/drivers/mmc/core/mmc_ops.c
index ad6e9798e949..b11c3455b040 100644
--- a/drivers/mmc/core/mmc_ops.c
+++ b/drivers/mmc/core/mmc_ops.c
@@ -54,21 +54,15 @@ static const u8 tuning_blk_pattern_8bit[] = {
 	0xff, 0x77, 0x77, 0xff, 0x77, 0xbb, 0xdd, 0xee,
 };
 
-static inline int __mmc_send_status(struct mmc_card *card, u32 *status,
-				    bool ignore_crc)
+int mmc_send_status(struct mmc_card *card, u32 *status)
 {
 	int err;
 	struct mmc_command cmd = {0};
 
-	BUG_ON(!card);
-	BUG_ON(!card->host);
-
 	cmd.opcode = MMC_SEND_STATUS;
 	if (!mmc_host_is_spi(card->host))
 		cmd.arg = card->rca << 16;
 	cmd.flags = MMC_RSP_SPI_R2 | MMC_RSP_R1 | MMC_CMD_AC;
-	if (ignore_crc)
-		cmd.flags &= ~MMC_RSP_CRC;
 
 	err = mmc_wait_for_cmd(card->host, &cmd, MMC_CMD_RETRIES);
 	if (err)
@@ -83,17 +77,10 @@ static inline int __mmc_send_status(struct mmc_card *card, u32 *status,
 	return 0;
 }
 
-int mmc_send_status(struct mmc_card *card, u32 *status)
-{
-	return __mmc_send_status(card, status, false);
-}
-
 static int _mmc_select_card(struct mmc_host *host, struct mmc_card *card)
 {
 	struct mmc_command cmd = {0};
 
-	BUG_ON(!host);
-
 	cmd.opcode = MMC_SELECT_CARD;
 
 	if (card) {
@@ -109,7 +96,6 @@ static int _mmc_select_card(struct mmc_host *host, struct mmc_card *card)
 
 int mmc_select_card(struct mmc_card *card)
 {
-	BUG_ON(!card);
 
 	return _mmc_select_card(card->host, card);
 }
@@ -181,8 +167,6 @@ int mmc_send_op_cond(struct mmc_host *host, u32 ocr, u32 *rocr)
 	struct mmc_command cmd = {0};
 	int i, err = 0;
 
-	BUG_ON(!host);
-
 	cmd.opcode = MMC_SEND_OP_COND;
 	cmd.arg = mmc_host_is_spi(host) ? 0 : ocr;
 	cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R3 | MMC_CMD_BCR;
@@ -221,9 +205,6 @@ int mmc_all_send_cid(struct mmc_host *host, u32 *cid)
 	int err;
 	struct mmc_command cmd = {0};
 
-	BUG_ON(!host);
-	BUG_ON(!cid);
-
 	cmd.opcode = MMC_ALL_SEND_CID;
 	cmd.arg = 0;
 	cmd.flags = MMC_RSP_R2 | MMC_CMD_BCR;
@@ -241,9 +222,6 @@ int mmc_set_relative_addr(struct mmc_card *card)
 {
 	struct mmc_command cmd = {0};
 
-	BUG_ON(!card);
-	BUG_ON(!card->host);
-
 	cmd.opcode = MMC_SET_RELATIVE_ADDR;
 	cmd.arg = card->rca << 16;
 	cmd.flags = MMC_RSP_R1 | MMC_CMD_AC;
@@ -257,9 +235,6 @@ mmc_send_cxd_native(struct mmc_host *host, u32 arg, u32 *cxd, int opcode)
 	int err;
 	struct mmc_command cmd = {0};
 
-	BUG_ON(!host);
-	BUG_ON(!cxd);
-
 	cmd.opcode = opcode;
 	cmd.arg = arg;
 	cmd.flags = MMC_RSP_R2 | MMC_CMD_AC;
@@ -440,7 +415,7 @@ int mmc_spi_set_crc(struct mmc_host *host, int use_crc)
 	return err;
 }
 
-int mmc_switch_status_error(struct mmc_host *host, u32 status)
+static int mmc_switch_status_error(struct mmc_host *host, u32 status)
 {
 	if (mmc_host_is_spi(host)) {
 		if (status & R1_SPI_ILLEGAL_COMMAND)
@@ -455,6 +430,88 @@ int mmc_switch_status_error(struct mmc_host *host, u32 status)
 	return 0;
 }
 
+/* Caller must hold re-tuning */
+int __mmc_switch_status(struct mmc_card *card, bool crc_err_fatal)
+{
+	u32 status;
+	int err;
+
+	err = mmc_send_status(card, &status);
+	if (!crc_err_fatal && err == -EILSEQ)
+		return 0;
+	if (err)
+		return err;
+
+	return mmc_switch_status_error(card->host, status);
+}
+
+int mmc_switch_status(struct mmc_card *card)
+{
+	return __mmc_switch_status(card, true);
+}
+
+static int mmc_poll_for_busy(struct mmc_card *card, unsigned int timeout_ms,
+			bool send_status, bool retry_crc_err)
+{
+	struct mmc_host *host = card->host;
+	int err;
+	unsigned long timeout;
+	u32 status = 0;
+	bool expired = false;
+	bool busy = false;
+
+	/* We have an unspecified cmd timeout, use the fallback value. */
+	if (!timeout_ms)
+		timeout_ms = MMC_OPS_TIMEOUT_MS;
+
+	/*
+	 * In cases when not allowed to poll by using CMD13 or because we aren't
+	 * capable of polling by using ->card_busy(), then rely on waiting the
+	 * stated timeout to be sufficient.
+	 */
+	if (!send_status && !host->ops->card_busy) {
+		mmc_delay(timeout_ms);
+		return 0;
+	}
+
+	timeout = jiffies + msecs_to_jiffies(timeout_ms) + 1;
+	do {
+		/*
+		 * Due to the possibility of being preempted while polling,
+		 * check the expiration time first.
+		 */
+		expired = time_after(jiffies, timeout);
+
+		if (host->ops->card_busy) {
+			busy = host->ops->card_busy(host);
+		} else {
+			err = mmc_send_status(card, &status);
+			if (retry_crc_err && err == -EILSEQ) {
+				busy = true;
+			} else if (err) {
+				return err;
+			} else {
+				err = mmc_switch_status_error(host, status);
+				if (err)
+					return err;
+				busy = R1_CURRENT_STATE(status) == R1_STATE_PRG;
+			}
+		}
+
+		/* Timeout if the device still remains busy. */
+		if (expired && busy) {
+			pr_err("%s: Card stuck being busy! %s\n",
+				mmc_hostname(host), __func__);
+			return -ETIMEDOUT;
+		}
+	} while (busy);
+
+	if (host->ops->card_busy && send_status)
+		return mmc_switch_status(card);
+
+	return 0;
+}
+
 /**
  *	__mmc_switch - modify EXT_CSD register
  *	@card: the MMC card associated with the data transfer
@@ -463,24 +520,22 @@ int mmc_switch_status_error(struct mmc_host *host, u32 status)
  *	@value: value to program into EXT_CSD register
  *	@timeout_ms: timeout (ms) for operation performed by register write,
  *                   timeout of zero implies maximum possible timeout
+ *	@timing: new timing to change to
  *	@use_busy_signal: use the busy signal as response type
  *	@send_status: send status cmd to poll for busy
- *	@ignore_crc: ignore CRC errors when sending status cmd to poll for busy
+ *	@retry_crc_err: retry when CRC errors when polling with CMD13 for busy
  *
  *	Modifies the EXT_CSD register for selected card.
  */
 int __mmc_switch(struct mmc_card *card, u8 set, u8 index, u8 value,
-		unsigned int timeout_ms, bool use_busy_signal, bool send_status,
-		bool ignore_crc)
+		unsigned int timeout_ms, unsigned char timing,
+		bool use_busy_signal, bool send_status,	bool retry_crc_err)
 {
 	struct mmc_host *host = card->host;
 	int err;
 	struct mmc_command cmd = {0};
-	unsigned long timeout;
-	u32 status = 0;
 	bool use_r1b_resp = use_busy_signal;
-	bool expired = false;
-	bool busy = false;
+	unsigned char old_timing = host->ios.timing;
 
 	mmc_retune_hold(host);
 
@@ -522,62 +577,24 @@ int __mmc_switch(struct mmc_card *card, u8 set, u8 index, u8 value,
 	if (!use_busy_signal)
 		goto out;
 
-	/*
-	 * CRC errors shall only be ignored in cases were CMD13 is used to poll
-	 * to detect busy completion.
-	 */
-	if ((host->caps & MMC_CAP_WAIT_WHILE_BUSY) && use_r1b_resp)
-		ignore_crc = false;
-
-	/* We have an unspecified cmd timeout, use the fallback value. */
-	if (!timeout_ms)
-		timeout_ms = MMC_OPS_TIMEOUT_MS;
-
-	/* Must check status to be sure of no errors. */
-	timeout = jiffies + msecs_to_jiffies(timeout_ms) + 1;
-	do {
-		/*
-		 * Due to the possibility of being preempted after
-		 * sending the status command, check the expiration
-		 * time first.
-		 */
-		expired = time_after(jiffies, timeout);
-		if (send_status) {
-			err = __mmc_send_status(card, &status, ignore_crc);
-			if (err)
-				goto out;
-		}
-		if ((host->caps & MMC_CAP_WAIT_WHILE_BUSY) && use_r1b_resp)
-			break;
-		if (host->ops->card_busy) {
-			if (!host->ops->card_busy(host))
-				break;
-			busy = true;
-		}
-		if (mmc_host_is_spi(host))
-			break;
+	/* Switch to new timing before poll and check switch status. */
+	if (timing)
+		mmc_set_timing(host, timing);
 
-		/*
-		 * We are not allowed to issue a status command and the host
-		 * does'nt support MMC_CAP_WAIT_WHILE_BUSY, then we can only
-		 * rely on waiting for the stated timeout to be sufficient.
-		 */
-		if (!send_status && !host->ops->card_busy) {
-			mmc_delay(timeout_ms);
-			goto out;
-		}
+	/*If SPI or used HW busy detection above, then we don't need to poll. */
+	if (((host->caps & MMC_CAP_WAIT_WHILE_BUSY) && use_r1b_resp) ||
+		mmc_host_is_spi(host)) {
+		if (send_status)
+			err = mmc_switch_status(card);
+		goto out_tim;
+	}
 
-		/* Timeout if the device never leaves the program state. */
-		if (expired &&
-		    (R1_CURRENT_STATE(status) == R1_STATE_PRG || busy)) {
-			pr_err("%s: Card stuck in programming state! %s\n",
-				mmc_hostname(host), __func__);
-			err = -ETIMEDOUT;
-			goto out;
-		}
-	} while (R1_CURRENT_STATE(status) == R1_STATE_PRG || busy);
+	/* Let's try to poll to find out when the command is completed. */
+	err = mmc_poll_for_busy(card, timeout_ms, send_status, retry_crc_err);
 
-	err = mmc_switch_status_error(host, status);
+out_tim:
+	if (err && timing)
+		mmc_set_timing(host, old_timing);
 out:
 	mmc_retune_release(host);
 
@@ -587,8 +604,8 @@ out:
 int mmc_switch(struct mmc_card *card, u8 set, u8 index, u8 value,
 		unsigned int timeout_ms)
 {
-	return __mmc_switch(card, set, index, value, timeout_ms, true, true,
-				false);
+	return __mmc_switch(card, set, index, value, timeout_ms, 0,
+			true, true, false);
 }
 EXPORT_SYMBOL_GPL(mmc_switch);
 
@@ -661,6 +678,31 @@ out:
 }
 EXPORT_SYMBOL_GPL(mmc_send_tuning);
 
+int mmc_abort_tuning(struct mmc_host *host, u32 opcode)
+{
+	struct mmc_command cmd = {0};
+
+	/*
+	 * eMMC specification specifies that CMD12 can be used to stop a tuning
+	 * command, but SD specification does not, so do nothing unless it is
+	 * eMMC.
+	 */
+	if (opcode != MMC_SEND_TUNING_BLOCK_HS200)
+		return 0;
+
+	cmd.opcode = MMC_STOP_TRANSMISSION;
+	cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC;
+
+	/*
+	 * For drivers that override R1 to R1b, set an arbitrary timeout based
+	 * on the tuning timeout i.e. 150ms.
+	 */
+	cmd.busy_timeout = 150;
+
+	return mmc_wait_for_cmd(host, &cmd, 0);
+}
+EXPORT_SYMBOL_GPL(mmc_abort_tuning);
+
 static int
 mmc_send_bus_test(struct mmc_card *card, struct mmc_host *host, u8 opcode,
 		  u8 len)
diff --git a/drivers/mmc/core/mmc_ops.h b/drivers/mmc/core/mmc_ops.h
index f1b8e81aaa28..abd525ed74be 100644
--- a/drivers/mmc/core/mmc_ops.h
+++ b/drivers/mmc/core/mmc_ops.h
@@ -27,10 +27,11 @@ int mmc_spi_set_crc(struct mmc_host *host, int use_crc);
 int mmc_bus_test(struct mmc_card *card, u8 bus_width);
 int mmc_send_hpi_cmd(struct mmc_card *card, u32 *status);
 int mmc_can_ext_csd(struct mmc_card *card);
-int mmc_switch_status_error(struct mmc_host *host, u32 status);
+int mmc_switch_status(struct mmc_card *card);
+int __mmc_switch_status(struct mmc_card *card, bool crc_err_fatal);
 int __mmc_switch(struct mmc_card *card, u8 set, u8 index, u8 value,
-		unsigned int timeout_ms, bool use_busy_signal, bool send_status,
-		bool ignore_crc);
+		unsigned int timeout_ms, unsigned char timing,
+		bool use_busy_signal, bool send_status,	bool retry_crc_err);
 
 #endif
 
diff --git a/drivers/mmc/card/mmc_test.c b/drivers/mmc/core/mmc_test.c
index 3678220964fe..3ab6e52d106c 100644
--- a/drivers/mmc/card/mmc_test.c
+++ b/drivers/mmc/core/mmc_test.c
@@ -1,6 +1,4 @@
 /*
- *  linux/drivers/mmc/card/mmc_test.c
- *
  *  Copyright 2007-2008 Pierre Ossman
  *
  * This program is free software; you can redistribute it and/or modify
@@ -214,7 +212,8 @@ static void mmc_test_prepare_mrq(struct mmc_test_card *test,
 	struct mmc_request *mrq, struct scatterlist *sg, unsigned sg_len,
 	unsigned dev_addr, unsigned blocks, unsigned blksz, int write)
 {
-	BUG_ON(!mrq || !mrq->cmd || !mrq->data || !mrq->stop);
+	if (WARN_ON(!mrq || !mrq->cmd || !mrq->data || !mrq->stop))
+		return;
 
 	if (blocks > 1) {
 		mrq->cmd->opcode = write ?
@@ -694,7 +693,8 @@ static int mmc_test_cleanup(struct mmc_test_card *test)
 static void mmc_test_prepare_broken_mrq(struct mmc_test_card *test,
 	struct mmc_request *mrq, int write)
 {
-	BUG_ON(!mrq || !mrq->cmd || !mrq->data);
+	if (WARN_ON(!mrq || !mrq->cmd || !mrq->data))
+		return;
 
 	if (mrq->data->blocks > 1) {
 		mrq->cmd->opcode = write ?
@@ -714,7 +714,8 @@ static int mmc_test_check_result(struct mmc_test_card *test,
 {
 	int ret;
 
-	BUG_ON(!mrq || !mrq->cmd || !mrq->data);
+	if (WARN_ON(!mrq || !mrq->cmd || !mrq->data))
+		return -EINVAL;
 
 	ret = 0;
 
@@ -736,15 +737,28 @@ static int mmc_test_check_result(struct mmc_test_card *test,
 	return ret;
 }
 
-static int mmc_test_check_result_async(struct mmc_card *card,
+static enum mmc_blk_status mmc_test_check_result_async(struct mmc_card *card,
 				       struct mmc_async_req *areq)
 {
 	struct mmc_test_async_req *test_async =
 		container_of(areq, struct mmc_test_async_req, areq);
+	int ret;
 
 	mmc_test_wait_busy(test_async->test);
 
-	return mmc_test_check_result(test_async->test, areq->mrq);
+	/*
+	 * FIXME: this would earlier just casts a regular error code,
+	 * either of the kernel type -ERRORCODE or the local test framework
+	 * RESULT_* errorcode, into an enum mmc_blk_status and return as
+	 * result check. Instead, convert it to some reasonable type by just
+	 * returning either MMC_BLK_SUCCESS or MMC_BLK_CMD_ERR.
+	 * If possible, a reasonable error code should be returned.
+	 */
+	ret = mmc_test_check_result(test_async->test, areq->mrq);
+	if (ret)
+		return MMC_BLK_CMD_ERR;
+
+	return MMC_BLK_SUCCESS;
 }
 
 /*
@@ -755,7 +769,8 @@ static int mmc_test_check_broken_result(struct mmc_test_card *test,
 {
 	int ret;
 
-	BUG_ON(!mrq || !mrq->cmd || !mrq->data);
+	if (WARN_ON(!mrq || !mrq->cmd || !mrq->data))
+		return -EINVAL;
 
 	ret = 0;
 
@@ -817,8 +832,9 @@ static int mmc_test_nonblock_transfer(struct mmc_test_card *test,
 	struct mmc_async_req *done_areq;
 	struct mmc_async_req *cur_areq = &test_areq[0].areq;
 	struct mmc_async_req *other_areq = &test_areq[1].areq;
+	enum mmc_blk_status status;
 	int i;
-	int ret;
+	int ret = RESULT_OK;
 
 	test_areq[0].test = test;
 	test_areq[1].test = test;
@@ -834,10 +850,12 @@ static int mmc_test_nonblock_transfer(struct mmc_test_card *test,
 	for (i = 0; i < count; i++) {
 		mmc_test_prepare_mrq(test, cur_areq->mrq, sg, sg_len, dev_addr,
 				     blocks, blksz, write);
-		done_areq = mmc_start_req(test->card->host, cur_areq, &ret);
+		done_areq = mmc_start_req(test->card->host, cur_areq, &status);
 
-		if (ret || (!done_areq && i > 0))
+		if (status != MMC_BLK_SUCCESS || (!done_areq && i > 0)) {
+			ret = RESULT_FAIL;
 			goto err;
+		}
 
 		if (done_areq) {
 			if (done_areq->mrq == &mrq2)
@@ -851,7 +869,9 @@ static int mmc_test_nonblock_transfer(struct mmc_test_card *test,
 		dev_addr += blocks;
 	}
 
-	done_areq = mmc_start_req(test->card->host, NULL, &ret);
+	done_areq = mmc_start_req(test->card->host, NULL, &status);
+	if (status != MMC_BLK_SUCCESS)
+		ret = RESULT_FAIL;
 
 	return ret;
 err:
@@ -2351,6 +2371,7 @@ static int mmc_test_ongoing_transfer(struct mmc_test_card *test,
 	struct mmc_request *mrq;
 	unsigned long timeout;
 	bool expired = false;
+	enum mmc_blk_status blkstat = MMC_BLK_SUCCESS;
 	int ret = 0, cmd_ret;
 	u32 status = 0;
 	int count = 0;
@@ -2378,9 +2399,11 @@ static int mmc_test_ongoing_transfer(struct mmc_test_card *test,
 
 	/* Start ongoing data request */
 	if (use_areq) {
-		mmc_start_req(host, &test_areq.areq, &ret);
-		if (ret)
+		mmc_start_req(host, &test_areq.areq, &blkstat);
+		if (blkstat != MMC_BLK_SUCCESS) {
+			ret = RESULT_FAIL;
 			goto out_free;
+		}
 	} else {
 		mmc_wait_for_req(host, mrq);
 	}
@@ -2413,10 +2436,13 @@ static int mmc_test_ongoing_transfer(struct mmc_test_card *test,
 	} while (repeat_cmd && R1_CURRENT_STATE(status) != R1_STATE_TRAN);
 
 	/* Wait for data request to complete */
-	if (use_areq)
-		mmc_start_req(host, NULL, &ret);
-	else
+	if (use_areq) {
+		mmc_start_req(host, NULL, &blkstat);
+		if (blkstat != MMC_BLK_SUCCESS)
+			ret = RESULT_FAIL;
+	} else {
 		mmc_wait_for_req_done(test->card->host, mrq);
+	}
 
 	/*
 	 * For cap_cmd_during_tfr request, upper layer must send stop if
diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/core/queue.c
index 8037f73a109a..a6496d8027bc 100644
--- a/drivers/mmc/card/queue.c
+++ b/drivers/mmc/core/queue.c
@@ -1,6 +1,4 @@
 /*
- *  linux/drivers/mmc/card/queue.c
- *
  *  Copyright (C) 2003 Russell King, All Rights Reserved.
  *  Copyright 2006-2007 Pierre Ossman
  *
@@ -44,7 +42,7 @@ static int mmc_prep_request(struct request_queue *q, struct request *req)
 	if (mq && (mmc_card_removed(mq->card) || mmc_access_rpmb(mq)))
 		return BLKPREP_KILL;
 
-	req->cmd_flags |= REQ_DONTPREP;
+	req->rq_flags |= RQF_DONTPREP;
 
 	return BLKPREP_OK;
 }
@@ -53,6 +51,7 @@ static int mmc_queue_thread(void *d)
 {
 	struct mmc_queue *mq = d;
 	struct request_queue *q = mq->queue;
+	struct mmc_context_info *cntx = &mq->card->host->context_info;
 
 	current->flags |= PF_MEMALLOC;
 
@@ -63,6 +62,19 @@ static int mmc_queue_thread(void *d)
 		spin_lock_irq(q->queue_lock);
 		set_current_state(TASK_INTERRUPTIBLE);
 		req = blk_fetch_request(q);
+		mq->asleep = false;
+		cntx->is_waiting_last_req = false;
+		cntx->is_new_req = false;
+		if (!req) {
+			/*
+			 * Dispatch queue is empty so set flags for
+			 * mmc_request_fn() to wake us up.
+			 */
+			if (mq->mqrq_prev->req)
+				cntx->is_waiting_last_req = true;
+			else
+				mq->asleep = true;
+		}
 		mq->mqrq_cur->req = req;
 		spin_unlock_irq(q->queue_lock);
 
@@ -115,31 +127,24 @@ static void mmc_request_fn(struct request_queue *q)
 {
 	struct mmc_queue *mq = q->queuedata;
 	struct request *req;
-	unsigned long flags;
 	struct mmc_context_info *cntx;
 
 	if (!mq) {
 		while ((req = blk_fetch_request(q)) != NULL) {
-			req->cmd_flags |= REQ_QUIET;
+			req->rq_flags |= RQF_QUIET;
 			__blk_end_request_all(req, -EIO);
 		}
 		return;
 	}
 
 	cntx = &mq->card->host->context_info;
-	if (!mq->mqrq_cur->req && mq->mqrq_prev->req) {
-		/*
-		 * New MMC request arrived when MMC thread may be
-		 * blocked on the previous request to be complete
-		 * with no current request fetched
-		 */
-		spin_lock_irqsave(&cntx->lock, flags);
-		if (cntx->is_waiting_last_req) {
-			cntx->is_new_req = true;
-			wake_up_interruptible(&cntx->wait);
-		}
-		spin_unlock_irqrestore(&cntx->lock, flags);
-	} else if (!mq->mqrq_cur->req && !mq->mqrq_prev->req)
+
+	if (cntx->is_waiting_last_req) {
+		cntx->is_new_req = true;
+		wake_up_interruptible(&cntx->wait);
+	}
+
+	if (mq->asleep)
 		wake_up_process(mq->thread);
 }
 
@@ -179,6 +184,82 @@ static void mmc_queue_setup_discard(struct request_queue *q,
 		queue_flag_set_unlocked(QUEUE_FLAG_SECERASE, q);
 }
 
+#ifdef CONFIG_MMC_BLOCK_BOUNCE
+static bool mmc_queue_alloc_bounce_bufs(struct mmc_queue *mq,
+					unsigned int bouncesz)
+{
+	int i;
+
+	for (i = 0; i < mq->qdepth; i++) {
+		mq->mqrq[i].bounce_buf = kmalloc(bouncesz, GFP_KERNEL);
+		if (!mq->mqrq[i].bounce_buf)
+			goto out_err;
+	}
+
+	return true;
+
+out_err:
+	while (--i >= 0) {
+		kfree(mq->mqrq[i].bounce_buf);
+		mq->mqrq[i].bounce_buf = NULL;
+	}
+	pr_warn("%s: unable to allocate bounce buffers\n",
+		mmc_card_name(mq->card));
+	return false;
+}
+
+static int mmc_queue_alloc_bounce_sgs(struct mmc_queue *mq,
+				      unsigned int bouncesz)
+{
+	int i, ret;
+
+	for (i = 0; i < mq->qdepth; i++) {
+		mq->mqrq[i].sg = mmc_alloc_sg(1, &ret);
+		if (ret)
+			return ret;
+
+		mq->mqrq[i].bounce_sg = mmc_alloc_sg(bouncesz / 512, &ret);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+#endif
+
+static int mmc_queue_alloc_sgs(struct mmc_queue *mq, int max_segs)
+{
+	int i, ret;
+
+	for (i = 0; i < mq->qdepth; i++) {
+		mq->mqrq[i].sg = mmc_alloc_sg(max_segs, &ret);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void mmc_queue_req_free_bufs(struct mmc_queue_req *mqrq)
+{
+	kfree(mqrq->bounce_sg);
+	mqrq->bounce_sg = NULL;
+
+	kfree(mqrq->sg);
+	mqrq->sg = NULL;
+
+	kfree(mqrq->bounce_buf);
+	mqrq->bounce_buf = NULL;
+}
+
+static void mmc_queue_reqs_free_bufs(struct mmc_queue *mq)
+{
+	int i;
+
+	for (i = 0; i < mq->qdepth; i++)
+		mmc_queue_req_free_bufs(&mq->mqrq[i]);
+}
+
 /**
  * mmc_init_queue - initialise a queue structure.
  * @mq: mmc queue
@@ -193,9 +274,8 @@ int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
 {
 	struct mmc_host *host = card->host;
 	u64 limit = BLK_BOUNCE_HIGH;
-	int ret;
-	struct mmc_queue_req *mqrq_cur = &mq->mqrq[0];
-	struct mmc_queue_req *mqrq_prev = &mq->mqrq[1];
+	bool bounce = false;
+	int ret = -ENOMEM;
 
 	if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
 		limit = (u64)dma_max_pfn(mmc_dev(host)) << PAGE_SHIFT;
@@ -205,8 +285,13 @@ int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
 	if (!mq->queue)
 		return -ENOMEM;
 
-	mq->mqrq_cur = mqrq_cur;
-	mq->mqrq_prev = mqrq_prev;
+	mq->qdepth = 2;
+	mq->mqrq = kcalloc(mq->qdepth, sizeof(struct mmc_queue_req),
+			   GFP_KERNEL);
+	if (!mq->mqrq)
+		goto blk_cleanup;
+	mq->mqrq_cur = &mq->mqrq[0];
+	mq->mqrq_prev = &mq->mqrq[1];
 	mq->queue->queuedata = mq;
 
 	blk_queue_prep_rq(mq->queue, mmc_prep_request);
@@ -228,63 +313,29 @@ int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
 		if (bouncesz > (host->max_blk_count * 512))
 			bouncesz = host->max_blk_count * 512;
 
-		if (bouncesz > 512) {
-			mqrq_cur->bounce_buf = kmalloc(bouncesz, GFP_KERNEL);
-			if (!mqrq_cur->bounce_buf) {
-				pr_warn("%s: unable to allocate bounce cur buffer\n",
-					mmc_card_name(card));
-			} else {
-				mqrq_prev->bounce_buf =
-						kmalloc(bouncesz, GFP_KERNEL);
-				if (!mqrq_prev->bounce_buf) {
-					pr_warn("%s: unable to allocate bounce prev buffer\n",
-						mmc_card_name(card));
-					kfree(mqrq_cur->bounce_buf);
-					mqrq_cur->bounce_buf = NULL;
-				}
-			}
-		}
-
-		if (mqrq_cur->bounce_buf && mqrq_prev->bounce_buf) {
+		if (bouncesz > 512 &&
+		    mmc_queue_alloc_bounce_bufs(mq, bouncesz)) {
 			blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_ANY);
 			blk_queue_max_hw_sectors(mq->queue, bouncesz / 512);
 			blk_queue_max_segments(mq->queue, bouncesz / 512);
 			blk_queue_max_segment_size(mq->queue, bouncesz);
 
-			mqrq_cur->sg = mmc_alloc_sg(1, &ret);
-			if (ret)
-				goto cleanup_queue;
-
-			mqrq_cur->bounce_sg =
-				mmc_alloc_sg(bouncesz / 512, &ret);
-			if (ret)
-				goto cleanup_queue;
-
-			mqrq_prev->sg = mmc_alloc_sg(1, &ret);
-			if (ret)
-				goto cleanup_queue;
-
-			mqrq_prev->bounce_sg =
-				mmc_alloc_sg(bouncesz / 512, &ret);
+			ret = mmc_queue_alloc_bounce_sgs(mq, bouncesz);
 			if (ret)
 				goto cleanup_queue;
+			bounce = true;
 		}
 	}
 #endif
 
-	if (!mqrq_cur->bounce_buf && !mqrq_prev->bounce_buf) {
+	if (!bounce) {
 		blk_queue_bounce_limit(mq->queue, limit);
 		blk_queue_max_hw_sectors(mq->queue,
 			min(host->max_blk_count, host->max_req_size / 512));
 		blk_queue_max_segments(mq->queue, host->max_segs);
 		blk_queue_max_segment_size(mq->queue, host->max_seg_size);
 
-		mqrq_cur->sg = mmc_alloc_sg(host->max_segs, &ret);
-		if (ret)
-			goto cleanup_queue;
-
-
-		mqrq_prev->sg = mmc_alloc_sg(host->max_segs, &ret);
+		ret = mmc_queue_alloc_sgs(mq, host->max_segs);
 		if (ret)
 			goto cleanup_queue;
 	}
@@ -296,27 +347,16 @@ int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
 
 	if (IS_ERR(mq->thread)) {
 		ret = PTR_ERR(mq->thread);
-		goto free_bounce_sg;
+		goto cleanup_queue;
 	}
 
 	return 0;
- free_bounce_sg:
-	kfree(mqrq_cur->bounce_sg);
-	mqrq_cur->bounce_sg = NULL;
-	kfree(mqrq_prev->bounce_sg);
-	mqrq_prev->bounce_sg = NULL;
 
  cleanup_queue:
-	kfree(mqrq_cur->sg);
-	mqrq_cur->sg = NULL;
-	kfree(mqrq_cur->bounce_buf);
-	mqrq_cur->bounce_buf = NULL;
-
-	kfree(mqrq_prev->sg);
-	mqrq_prev->sg = NULL;
-	kfree(mqrq_prev->bounce_buf);
-	mqrq_prev->bounce_buf = NULL;
-
+	mmc_queue_reqs_free_bufs(mq);
+	kfree(mq->mqrq);
+	mq->mqrq = NULL;
+blk_cleanup:
 	blk_cleanup_queue(mq->queue);
 	return ret;
 }
@@ -325,8 +365,6 @@ void mmc_cleanup_queue(struct mmc_queue *mq)
 {
 	struct request_queue *q = mq->queue;
 	unsigned long flags;
-	struct mmc_queue_req *mqrq_cur = mq->mqrq_cur;
-	struct mmc_queue_req *mqrq_prev = mq->mqrq_prev;
 
 	/* Make sure the queue isn't suspended, as that will deadlock */
 	mmc_queue_resume(mq);
@@ -340,71 +378,14 @@ void mmc_cleanup_queue(struct mmc_queue *mq)
 	blk_start_queue(q);
 	spin_unlock_irqrestore(q->queue_lock, flags);
 
-	kfree(mqrq_cur->bounce_sg);
-	mqrq_cur->bounce_sg = NULL;
-
-	kfree(mqrq_cur->sg);
-	mqrq_cur->sg = NULL;
-
-	kfree(mqrq_cur->bounce_buf);
-	mqrq_cur->bounce_buf = NULL;
-
-	kfree(mqrq_prev->bounce_sg);
-	mqrq_prev->bounce_sg = NULL;
-
-	kfree(mqrq_prev->sg);
-	mqrq_prev->sg = NULL;
-
-	kfree(mqrq_prev->bounce_buf);
-	mqrq_prev->bounce_buf = NULL;
+	mmc_queue_reqs_free_bufs(mq);
+	kfree(mq->mqrq);
+	mq->mqrq = NULL;
 
 	mq->card = NULL;
 }
 EXPORT_SYMBOL(mmc_cleanup_queue);
 
-int mmc_packed_init(struct mmc_queue *mq, struct mmc_card *card)
-{
-	struct mmc_queue_req *mqrq_cur = &mq->mqrq[0];
-	struct mmc_queue_req *mqrq_prev = &mq->mqrq[1];
-	int ret = 0;
-
-
-	mqrq_cur->packed = kzalloc(sizeof(struct mmc_packed), GFP_KERNEL);
-	if (!mqrq_cur->packed) {
-		pr_warn("%s: unable to allocate packed cmd for mqrq_cur\n",
-			mmc_card_name(card));
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	mqrq_prev->packed = kzalloc(sizeof(struct mmc_packed), GFP_KERNEL);
-	if (!mqrq_prev->packed) {
-		pr_warn("%s: unable to allocate packed cmd for mqrq_prev\n",
-			mmc_card_name(card));
-		kfree(mqrq_cur->packed);
-		mqrq_cur->packed = NULL;
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	INIT_LIST_HEAD(&mqrq_cur->packed->list);
-	INIT_LIST_HEAD(&mqrq_prev->packed->list);
-
-out:
-	return ret;
-}
-
-void mmc_packed_clean(struct mmc_queue *mq)
-{
-	struct mmc_queue_req *mqrq_cur = &mq->mqrq[0];
-	struct mmc_queue_req *mqrq_prev = &mq->mqrq[1];
-
-	kfree(mqrq_cur->packed);
-	mqrq_cur->packed = NULL;
-	kfree(mqrq_prev->packed);
-	mqrq_prev->packed = NULL;
-}
-
 /**
  * mmc_queue_suspend - suspend a MMC request queue
  * @mq: MMC queue to suspend
@@ -449,41 +430,6 @@ void mmc_queue_resume(struct mmc_queue *mq)
 	}
 }
 
-static unsigned int mmc_queue_packed_map_sg(struct mmc_queue *mq,
-					    struct mmc_packed *packed,
-					    struct scatterlist *sg,
-					    enum mmc_packed_type cmd_type)
-{
-	struct scatterlist *__sg = sg;
-	unsigned int sg_len = 0;
-	struct request *req;
-
-	if (mmc_packed_wr(cmd_type)) {
-		unsigned int hdr_sz = mmc_large_sector(mq->card) ? 4096 : 512;
-		unsigned int max_seg_sz = queue_max_segment_size(mq->queue);
-		unsigned int len, remain, offset = 0;
-		u8 *buf = (u8 *)packed->cmd_hdr;
-
-		remain = hdr_sz;
-		do {
-			len = min(remain, max_seg_sz);
-			sg_set_buf(__sg, buf + offset, len);
-			offset += len;
-			remain -= len;
-			sg_unmark_end(__sg++);
-			sg_len++;
-		} while (remain);
-	}
-
-	list_for_each_entry(req, &packed->list, queuelist) {
-		sg_len += blk_rq_map_sg(mq->queue, req, __sg);
-		__sg = sg + (sg_len - 1);
-		sg_unmark_end(__sg++);
-	}
-	sg_mark_end(sg + (sg_len - 1));
-	return sg_len;
-}
-
 /*
  * Prepare the sg list(s) to be handed of to the host driver
  */
@@ -492,26 +438,12 @@ unsigned int mmc_queue_map_sg(struct mmc_queue *mq, struct mmc_queue_req *mqrq)
 	unsigned int sg_len;
 	size_t buflen;
 	struct scatterlist *sg;
-	enum mmc_packed_type cmd_type;
 	int i;
 
-	cmd_type = mqrq->cmd_type;
-
-	if (!mqrq->bounce_buf) {
-		if (mmc_packed_cmd(cmd_type))
-			return mmc_queue_packed_map_sg(mq, mqrq->packed,
-						       mqrq->sg, cmd_type);
-		else
-			return blk_rq_map_sg(mq->queue, mqrq->req, mqrq->sg);
-	}
-
-	BUG_ON(!mqrq->bounce_sg);
+	if (!mqrq->bounce_buf)
+		return blk_rq_map_sg(mq->queue, mqrq->req, mqrq->sg);
 
-	if (mmc_packed_cmd(cmd_type))
-		sg_len = mmc_queue_packed_map_sg(mq, mqrq->packed,
-						 mqrq->bounce_sg, cmd_type);
-	else
-		sg_len = blk_rq_map_sg(mq->queue, mqrq->req, mqrq->bounce_sg);
+	sg_len = blk_rq_map_sg(mq->queue, mqrq->req, mqrq->bounce_sg);
 
 	mqrq->bounce_sg_len = sg_len;
 
diff --git a/drivers/mmc/card/queue.h b/drivers/mmc/core/queue.h
index 342f1e3f301e..dac8c3d010dd 100644
--- a/drivers/mmc/card/queue.h
+++ b/drivers/mmc/core/queue.h
@@ -11,6 +11,7 @@ static inline bool mmc_req_is_special(struct request *req)
 
 struct request;
 struct task_struct;
+struct mmc_blk_data;
 
 struct mmc_blk_request {
 	struct mmc_request	mrq;
@@ -21,23 +22,6 @@ struct mmc_blk_request {
 	int			retune_retry_done;
 };
 
-enum mmc_packed_type {
-	MMC_PACKED_NONE = 0,
-	MMC_PACKED_WRITE,
-};
-
-#define mmc_packed_cmd(type)	((type) != MMC_PACKED_NONE)
-#define mmc_packed_wr(type)	((type) == MMC_PACKED_WRITE)
-
-struct mmc_packed {
-	struct list_head	list;
-	__le32			cmd_hdr[1024];
-	unsigned int		blocks;
-	u8			nr_entries;
-	u8			retries;
-	s16			idx_failure;
-};
-
 struct mmc_queue_req {
 	struct request		*req;
 	struct mmc_blk_request	brq;
@@ -46,8 +30,6 @@ struct mmc_queue_req {
 	struct scatterlist	*bounce_sg;
 	unsigned int		bounce_sg_len;
 	struct mmc_async_req	mmc_active;
-	enum mmc_packed_type	cmd_type;
-	struct mmc_packed	*packed;
 };
 
 struct mmc_queue {
@@ -57,11 +39,13 @@ struct mmc_queue {
 	unsigned int		flags;
 #define MMC_QUEUE_SUSPENDED	(1 << 0)
 #define MMC_QUEUE_NEW_REQUEST	(1 << 1)
-	void			*data;
+	bool			asleep;
+	struct mmc_blk_data	*blkdata;
 	struct request_queue	*queue;
-	struct mmc_queue_req	mqrq[2];
+	struct mmc_queue_req	*mqrq;
 	struct mmc_queue_req	*mqrq_cur;
 	struct mmc_queue_req	*mqrq_prev;
+	int			qdepth;
 };
 
 extern int mmc_init_queue(struct mmc_queue *, struct mmc_card *, spinlock_t *,
@@ -75,9 +59,6 @@ extern unsigned int mmc_queue_map_sg(struct mmc_queue *,
 extern void mmc_queue_bounce_pre(struct mmc_queue_req *);
 extern void mmc_queue_bounce_post(struct mmc_queue_req *);
 
-extern int mmc_packed_init(struct mmc_queue *, struct mmc_card *);
-extern void mmc_packed_clean(struct mmc_queue *);
-
 extern int mmc_access_rpmb(struct mmc_queue *);
 
 #endif
diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c
index 73c762a28dfe..deb90c2ff6b4 100644
--- a/drivers/mmc/core/sd.c
+++ b/drivers/mmc/core/sd.c
@@ -927,7 +927,6 @@ static int mmc_sd_init_card(struct mmc_host *host, u32 ocr,
 	u32 cid[4];
 	u32 rocr = 0;
 
-	BUG_ON(!host);
 	WARN_ON(!host->claimed);
 
 	err = mmc_sd_get_cid(host, ocr, cid, &rocr);
@@ -1043,9 +1042,6 @@ free_card:
  */
 static void mmc_sd_remove(struct mmc_host *host)
 {
-	BUG_ON(!host);
-	BUG_ON(!host->card);
-
 	mmc_remove_card(host->card);
 	host->card = NULL;
 }
@@ -1065,9 +1061,6 @@ static void mmc_sd_detect(struct mmc_host *host)
 {
 	int err;
 
-	BUG_ON(!host);
-	BUG_ON(!host->card);
-
 	mmc_get_card(host->card);
 
 	/*
@@ -1091,9 +1084,6 @@ static int _mmc_sd_suspend(struct mmc_host *host)
 {
 	int err = 0;
 
-	BUG_ON(!host);
-	BUG_ON(!host->card);
-
 	mmc_claim_host(host);
 
 	if (mmc_card_suspended(host->card))
@@ -1136,9 +1126,6 @@ static int _mmc_sd_resume(struct mmc_host *host)
 {
 	int err = 0;
 
-	BUG_ON(!host);
-	BUG_ON(!host->card);
-
 	mmc_claim_host(host);
 
 	if (!mmc_card_suspended(host->card))
@@ -1221,7 +1208,6 @@ int mmc_attach_sd(struct mmc_host *host)
 	int err;
 	u32 ocr, rocr;
 
-	BUG_ON(!host);
 	WARN_ON(!host->claimed);
 
 	err = mmc_send_app_op_cond(host, 0, &ocr);
diff --git a/drivers/mmc/core/sd_ops.c b/drivers/mmc/core/sd_ops.c
index 16b774c18e75..de125a41aa7a 100644
--- a/drivers/mmc/core/sd_ops.c
+++ b/drivers/mmc/core/sd_ops.c
@@ -27,8 +27,8 @@ int mmc_app_cmd(struct mmc_host *host, struct mmc_card *card)
 	int err;
 	struct mmc_command cmd = {0};
 
-	BUG_ON(!host);
-	BUG_ON(card && (card->host != host));
+	if (WARN_ON(card && card->host != host))
+		return -EINVAL;
 
 	cmd.opcode = MMC_APP_CMD;
 
@@ -72,8 +72,8 @@ int mmc_wait_for_app_cmd(struct mmc_host *host, struct mmc_card *card,
 
 	int i, err;
 
-	BUG_ON(!cmd);
-	BUG_ON(retries < 0);
+	if (retries < 0)
+		retries = MMC_CMD_RETRIES;
 
 	err = -EIO;
 
@@ -122,9 +122,6 @@ int mmc_app_set_bus_width(struct mmc_card *card, int width)
 {
 	struct mmc_command cmd = {0};
 
-	BUG_ON(!card);
-	BUG_ON(!card->host);
-
 	cmd.opcode = SD_APP_SET_BUS_WIDTH;
 	cmd.flags = MMC_RSP_R1 | MMC_CMD_AC;
 
@@ -147,8 +144,6 @@ int mmc_send_app_op_cond(struct mmc_host *host, u32 ocr, u32 *rocr)
 	struct mmc_command cmd = {0};
 	int i, err = 0;
 
-	BUG_ON(!host);
-
 	cmd.opcode = SD_APP_OP_COND;
 	if (mmc_host_is_spi(host))
 		cmd.arg = ocr & (1 << 30); /* SPI only defines one bit */
@@ -224,9 +219,6 @@ int mmc_send_relative_addr(struct mmc_host *host, unsigned int *rca)
 	int err;
 	struct mmc_command cmd = {0};
 
-	BUG_ON(!host);
-	BUG_ON(!rca);
-
 	cmd.opcode = SD_SEND_RELATIVE_ADDR;
 	cmd.arg = 0;
 	cmd.flags = MMC_RSP_R6 | MMC_CMD_BCR;
@@ -249,10 +241,6 @@ int mmc_app_send_scr(struct mmc_card *card, u32 *scr)
 	struct scatterlist sg;
 	void *data_buf;
 
-	BUG_ON(!card);
-	BUG_ON(!card->host);
-	BUG_ON(!scr);
-
 	/* NOTE: caller guarantees scr is heap-allocated */
 
 	err = mmc_app_cmd(card->host, card);
@@ -307,9 +295,6 @@ int mmc_sd_switch(struct mmc_card *card, int mode, int group,
 	struct mmc_data data = {0};
 	struct scatterlist sg;
 
-	BUG_ON(!card);
-	BUG_ON(!card->host);
-
 	/* NOTE: caller guarantees resp is heap-allocated */
 
 	mode = !!mode;
@@ -352,10 +337,6 @@ int mmc_app_sd_status(struct mmc_card *card, void *ssr)
 	struct mmc_data data = {0};
 	struct scatterlist sg;
 
-	BUG_ON(!card);
-	BUG_ON(!card->host);
-	BUG_ON(!ssr);
-
 	/* NOTE: caller guarantees ssr is heap-allocated */
 
 	err = mmc_app_cmd(card->host, card);
diff --git a/drivers/mmc/core/sdio.c b/drivers/mmc/core/sdio.c
index bd44ba8116d1..ecbc52981ba5 100644
--- a/drivers/mmc/core/sdio.c
+++ b/drivers/mmc/core/sdio.c
@@ -63,7 +63,8 @@ static int sdio_init_func(struct mmc_card *card, unsigned int fn)
 	int ret;
 	struct sdio_func *func;
 
-	BUG_ON(fn > SDIO_MAX_FUNCS);
+	if (WARN_ON(fn > SDIO_MAX_FUNCS))
+		return -EINVAL;
 
 	func = sdio_alloc_func(card);
 	if (IS_ERR(func))
@@ -555,7 +556,6 @@ static int mmc_sdio_init_card(struct mmc_host *host, u32 ocr,
 	u32 rocr = 0;
 	u32 ocr_card = ocr;
 
-	BUG_ON(!host);
 	WARN_ON(!host->claimed);
 
 	/* to query card if 1.8V signalling is supported */
@@ -791,9 +791,6 @@ static void mmc_sdio_remove(struct mmc_host *host)
 {
 	int i;
 
-	BUG_ON(!host);
-	BUG_ON(!host->card);
-
 	for (i = 0;i < host->card->sdio_funcs;i++) {
 		if (host->card->sdio_func[i]) {
 			sdio_remove_func(host->card->sdio_func[i]);
@@ -820,9 +817,6 @@ static void mmc_sdio_detect(struct mmc_host *host)
 {
 	int err;
 
-	BUG_ON(!host);
-	BUG_ON(!host->card);
-
 	/* Make sure card is powered before detecting it */
 	if (host->caps & MMC_CAP_POWER_OFF_CARD) {
 		err = pm_runtime_get_sync(&host->card->dev);
@@ -916,9 +910,6 @@ static int mmc_sdio_resume(struct mmc_host *host)
 {
 	int err = 0;
 
-	BUG_ON(!host);
-	BUG_ON(!host->card);
-
 	/* Basic card reinitialization. */
 	mmc_claim_host(host);
 
@@ -970,9 +961,6 @@ static int mmc_sdio_power_restore(struct mmc_host *host)
 {
 	int ret;
 
-	BUG_ON(!host);
-	BUG_ON(!host->card);
-
 	mmc_claim_host(host);
 
 	/*
@@ -1063,7 +1051,6 @@ int mmc_attach_sdio(struct mmc_host *host)
 	u32 ocr, rocr;
 	struct mmc_card *card;
 
-	BUG_ON(!host);
 	WARN_ON(!host->claimed);
 
 	err = mmc_send_io_op_cond(host, 0, &ocr);
diff --git a/drivers/mmc/core/sdio_cis.c b/drivers/mmc/core/sdio_cis.c
index dcb3dee59fa5..f8c372839d24 100644
--- a/drivers/mmc/core/sdio_cis.c
+++ b/drivers/mmc/core/sdio_cis.c
@@ -262,7 +262,8 @@ static int sdio_read_cis(struct mmc_card *card, struct sdio_func *func)
 	else
 		prev = &card->tuples;
 
-	BUG_ON(*prev);
+	if (*prev)
+		return -EINVAL;
 
 	do {
 		unsigned char tpl_code, tpl_link;
diff --git a/drivers/mmc/core/sdio_irq.c b/drivers/mmc/core/sdio_irq.c
index 91bbbfb29f3f..f1faf9acc007 100644
--- a/drivers/mmc/core/sdio_irq.c
+++ b/drivers/mmc/core/sdio_irq.c
@@ -214,7 +214,9 @@ static int sdio_card_irq_put(struct mmc_card *card)
 	struct mmc_host *host = card->host;
 
 	WARN_ON(!host->claimed);
-	BUG_ON(host->sdio_irqs < 1);
+
+	if (host->sdio_irqs < 1)
+		return -EINVAL;
 
 	if (!--host->sdio_irqs) {
 		if (!(host->caps2 & MMC_CAP2_SDIO_IRQ_NOTHREAD)) {
@@ -261,8 +263,8 @@ int sdio_claim_irq(struct sdio_func *func, sdio_irq_handler_t *handler)
 	int ret;
 	unsigned char reg;
 
-	BUG_ON(!func);
-	BUG_ON(!func->card);
+	if (!func)
+		return -EINVAL;
 
 	pr_debug("SDIO: Enabling IRQ for %s...\n", sdio_func_id(func));
 
@@ -304,8 +306,8 @@ int sdio_release_irq(struct sdio_func *func)
 	int ret;
 	unsigned char reg;
 
-	BUG_ON(!func);
-	BUG_ON(!func->card);
+	if (!func)
+		return -EINVAL;
 
 	pr_debug("SDIO: Disabling IRQ for %s...\n", sdio_func_id(func));
 
diff --git a/drivers/mmc/card/sdio_uart.c b/drivers/mmc/core/sdio_uart.c
index 5af6fb9a9ce2..d3c91f412b69 100644
--- a/drivers/mmc/card/sdio_uart.c
+++ b/drivers/mmc/core/sdio_uart.c
@@ -1,5 +1,5 @@
 /*
- * linux/drivers/mmc/card/sdio_uart.c - SDIO UART/GPS driver
+ * SDIO UART/GPS driver
  *
  * Based on drivers/serial/8250.c and drivers/serial/serial_core.c
  * by Russell King.
@@ -135,8 +135,6 @@ static void sdio_uart_port_remove(struct sdio_uart_port *port)
 {
 	struct sdio_func *func;
 
-	BUG_ON(sdio_uart_table[port->index] != port);
-
 	spin_lock(&sdio_uart_table_lock);
 	sdio_uart_table[port->index] = NULL;
 	spin_unlock(&sdio_uart_table_lock);
diff --git a/drivers/mmc/core/slot-gpio.c b/drivers/mmc/core/slot-gpio.c
index 27117ba47073..babe591aea96 100644
--- a/drivers/mmc/core/slot-gpio.c
+++ b/drivers/mmc/core/slot-gpio.c
@@ -258,6 +258,14 @@ int mmc_gpiod_request_cd(struct mmc_host *host, const char *con_id,
 }
 EXPORT_SYMBOL(mmc_gpiod_request_cd);
 
+bool mmc_can_gpio_cd(struct mmc_host *host)
+{
+	struct mmc_gpio *ctx = host->slot.handler_priv;
+
+	return ctx->cd_gpio ? true : false;
+}
+EXPORT_SYMBOL(mmc_can_gpio_cd);
+
 /**
  * mmc_gpiod_request_ro - request a gpio descriptor for write protection
  * @host: mmc host
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index 5274f503a39a..2eb97014dc3f 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -135,7 +135,6 @@ config MMC_SDHCI_OF_AT91
 	tristate "SDHCI OF support for the Atmel SDMMC controller"
 	depends on MMC_SDHCI_PLTFM
 	depends on OF
-	select MMC_SDHCI_IO_ACCESSORS
 	help
 	  This selects the Atmel SDMMC driver
 
@@ -144,6 +143,7 @@ config MMC_SDHCI_OF_ESDHC
 	depends on MMC_SDHCI_PLTFM
 	depends on PPC || ARCH_MXC || ARCH_LAYERSCAPE
 	select MMC_SDHCI_IO_ACCESSORS
+	select FSL_GUTS
 	help
 	  This selects the Freescale eSDHC controller support.
 
@@ -165,6 +165,17 @@ config MMC_SDHCI_OF_HLWD
 
 	  If unsure, say N.
 
+config MMC_SDHCI_CADENCE
+	tristate "SDHCI support for the Cadence SD/SDIO/eMMC controller"
+	depends on MMC_SDHCI_PLTFM
+	depends on OF
+	help
+	  This selects the Cadence SD/SDIO/eMMC driver.
+
+	  If you have a controller with this interface, say Y or M here.
+
+	  If unsure, say N.
+
 config MMC_SDHCI_CNS3XXX
 	tristate "SDHCI support on the Cavium Networks CNS3xxx SoC"
 	depends on ARCH_CNS3XXX
@@ -322,6 +333,16 @@ config MMC_SDHCI_IPROC
 
 	  If unsure, say N.
 
+config MMC_MESON_GX
+	tristate "Amlogic S905/GX* SD/MMC Host Controller support"
+	depends on ARCH_MESON && MMC
+	help
+	  This selects support for the Amlogic SD/MMC Host Controller
+	  found on the S905/GX* family of SoCs.  This controller is
+	  MMC 5.1 compliant and supports SD, eMMC and SDIO interfaces.
+
+	  If you have a controller with this interface, say Y here.
+
 config MMC_MOXART
 	tristate "MOXART SD/MMC Host Controller support"
 	depends on ARCH_MOXART && MMC
diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile
index e2bdaaf43184..ccc9c4cba154 100644
--- a/drivers/mmc/host/Makefile
+++ b/drivers/mmc/host/Makefile
@@ -53,6 +53,7 @@ obj-$(CONFIG_MMC_JZ4740)	+= jz4740_mmc.o
 obj-$(CONFIG_MMC_VUB300)	+= vub300.o
 obj-$(CONFIG_MMC_USHC)		+= ushc.o
 obj-$(CONFIG_MMC_WMT)		+= wmt-sdmmc.o
+obj-$(CONFIG_MMC_MESON_GX)	+= meson-gx-mmc.o
 obj-$(CONFIG_MMC_MOXART)	+= moxart-mmc.o
 obj-$(CONFIG_MMC_SUNXI)		+= sunxi-mmc.o
 obj-$(CONFIG_MMC_USDHI6ROL0)	+= usdhi6rol0.o
@@ -62,6 +63,7 @@ obj-$(CONFIG_MMC_REALTEK_PCI)	+= rtsx_pci_sdmmc.o
 obj-$(CONFIG_MMC_REALTEK_USB)	+= rtsx_usb_sdmmc.o
 
 obj-$(CONFIG_MMC_SDHCI_PLTFM)		+= sdhci-pltfm.o
+obj-$(CONFIG_MMC_SDHCI_CADENCE)		+= sdhci-cadence.o
 obj-$(CONFIG_MMC_SDHCI_CNS3XXX)		+= sdhci-cns3xxx.o
 obj-$(CONFIG_MMC_SDHCI_ESDHC_IMX)	+= sdhci-esdhc-imx.o
 obj-$(CONFIG_MMC_SDHCI_DOVE)		+= sdhci-dove.o
diff --git a/drivers/mmc/host/davinci_mmc.c b/drivers/mmc/host/davinci_mmc.c
index 8fa478c3b0db..36b5af8eadb8 100644
--- a/drivers/mmc/host/davinci_mmc.c
+++ b/drivers/mmc/host/davinci_mmc.c
@@ -35,6 +35,7 @@
 #include <linux/mmc/mmc.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/mmc/slot-gpio.h>
 
 #include <linux/platform_data/mmc-davinci.h>
 
@@ -1029,9 +1030,10 @@ static int mmc_davinci_get_cd(struct mmc_host *mmc)
 	struct platform_device *pdev = to_platform_device(mmc->parent);
 	struct davinci_mmc_config *config = pdev->dev.platform_data;
 
-	if (!config || !config->get_cd)
-		return -ENOSYS;
-	return config->get_cd(pdev->id);
+	if (config && config->get_cd)
+		return config->get_cd(pdev->id);
+
+	return mmc_gpio_get_cd(mmc);
 }
 
 static int mmc_davinci_get_ro(struct mmc_host *mmc)
@@ -1039,9 +1041,10 @@ static int mmc_davinci_get_ro(struct mmc_host *mmc)
 	struct platform_device *pdev = to_platform_device(mmc->parent);
 	struct davinci_mmc_config *config = pdev->dev.platform_data;
 
-	if (!config || !config->get_ro)
-		return -ENOSYS;
-	return config->get_ro(pdev->id);
+	if (config && config->get_ro)
+		return config->get_ro(pdev->id);
+
+	return mmc_gpio_get_ro(mmc);
 }
 
 static void mmc_davinci_enable_sdio_irq(struct mmc_host *mmc, int enable)
@@ -1159,49 +1162,53 @@ static const struct of_device_id davinci_mmc_dt_ids[] = {
 };
 MODULE_DEVICE_TABLE(of, davinci_mmc_dt_ids);
 
-static struct davinci_mmc_config
-	*mmc_parse_pdata(struct platform_device *pdev)
+static int mmc_davinci_parse_pdata(struct mmc_host *mmc)
 {
-	struct device_node *np;
+	struct platform_device *pdev = to_platform_device(mmc->parent);
 	struct davinci_mmc_config *pdata = pdev->dev.platform_data;
-	const struct of_device_id *match =
-		of_match_device(davinci_mmc_dt_ids, &pdev->dev);
-	u32 data;
-
-	np = pdev->dev.of_node;
-	if (!np)
-		return pdata;
-
-	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
-	if (!pdata) {
-		dev_err(&pdev->dev, "Failed to allocate memory for struct davinci_mmc_config\n");
-		goto nodata;
-	}
+	struct mmc_davinci_host *host;
+	int ret;
 
-	if (match)
-		pdev->id_entry = match->data;
+	if (!pdata)
+		return -EINVAL;
 
-	if (of_property_read_u32(np, "max-frequency", &pdata->max_freq))
-		dev_info(&pdev->dev, "'max-frequency' property not specified, defaulting to 25MHz\n");
+	host = mmc_priv(mmc);
+	if (!host)
+		return -EINVAL;
 
-	of_property_read_u32(np, "bus-width", &data);
-	switch (data) {
-	case 1:
-	case 4:
-	case 8:
-		pdata->wires = data;
-		break;
-	default:
-		pdata->wires = 1;
-		dev_info(&pdev->dev, "Unsupported buswidth, defaulting to 1 bit\n");
-	}
-nodata:
-	return pdata;
+	if (pdata && pdata->nr_sg)
+		host->nr_sg = pdata->nr_sg - 1;
+
+	if (pdata && (pdata->wires == 4 || pdata->wires == 0))
+		mmc->caps |= MMC_CAP_4_BIT_DATA;
+
+	if (pdata && (pdata->wires == 8))
+		mmc->caps |= (MMC_CAP_4_BIT_DATA | MMC_CAP_8_BIT_DATA);
+
+	mmc->f_min = 312500;
+	mmc->f_max = 25000000;
+	if (pdata && pdata->max_freq)
+		mmc->f_max = pdata->max_freq;
+	if (pdata && pdata->caps)
+		mmc->caps |= pdata->caps;
+
+	/* Register a cd gpio, if there is not one, enable polling */
+	ret = mmc_gpiod_request_cd(mmc, "cd", 0, false, 0, NULL);
+	if (ret == -EPROBE_DEFER)
+		return ret;
+	else if (ret)
+		mmc->caps |= MMC_CAP_NEEDS_POLL;
+
+	ret = mmc_gpiod_request_ro(mmc, "wp", 0, false, 0, NULL);
+	if (ret == -EPROBE_DEFER)
+		return ret;
+
+	return 0;
 }
 
 static int __init davinci_mmcsd_probe(struct platform_device *pdev)
 {
-	struct davinci_mmc_config *pdata = NULL;
+	const struct of_device_id *match;
 	struct mmc_davinci_host *host = NULL;
 	struct mmc_host *mmc = NULL;
 	struct resource *r, *mem = NULL;
@@ -1209,12 +1216,6 @@ static int __init davinci_mmcsd_probe(struct platform_device *pdev)
 	size_t mem_size;
 	const struct platform_device_id *id_entry;
 
-	pdata = mmc_parse_pdata(pdev);
-	if (pdata == NULL) {
-		dev_err(&pdev->dev, "Couldn't get platform data\n");
-		return -ENOENT;
-	}
-
 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!r)
 		return -ENODEV;
@@ -1253,14 +1254,28 @@ static int __init davinci_mmcsd_probe(struct platform_device *pdev)
 
 	host->mmc_input_clk = clk_get_rate(host->clk);
 
-	init_mmcsd_host(host);
-
-	if (pdata->nr_sg)
-		host->nr_sg = pdata->nr_sg - 1;
+	match = of_match_device(davinci_mmc_dt_ids, &pdev->dev);
+	if (match) {
+		pdev->id_entry = match->data;
+		ret = mmc_of_parse(mmc);
+		if (ret) {
+			dev_err(&pdev->dev,
+				"could not parse of data: %d\n", ret);
+			goto parse_fail;
+		}
+	} else {
+		ret = mmc_davinci_parse_pdata(mmc);
+		if (ret) {
+			dev_err(&pdev->dev,
+				"could not parse platform data: %d\n", ret);
+			goto parse_fail;
+	}	}
 
 	if (host->nr_sg > MAX_NR_SG || !host->nr_sg)
 		host->nr_sg = MAX_NR_SG;
 
+	init_mmcsd_host(host);
+
 	host->use_dma = use_dma;
 	host->mmc_irq = irq;
 	host->sdio_irq = platform_get_irq(pdev, 1);
@@ -1273,27 +1288,13 @@ static int __init davinci_mmcsd_probe(struct platform_device *pdev)
 			host->use_dma = 0;
 	}
 
-	/* REVISIT:  someday, support IRQ-driven card detection.  */
-	mmc->caps |= MMC_CAP_NEEDS_POLL;
 	mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY;
 
-	if (pdata && (pdata->wires == 4 || pdata->wires == 0))
-		mmc->caps |= MMC_CAP_4_BIT_DATA;
-
-	if (pdata && (pdata->wires == 8))
-		mmc->caps |= (MMC_CAP_4_BIT_DATA | MMC_CAP_8_BIT_DATA);
-
 	id_entry = platform_get_device_id(pdev);
 	if (id_entry)
 		host->version = id_entry->driver_data;
 
 	mmc->ops = &mmc_davinci_ops;
-	mmc->f_min = 312500;
-	mmc->f_max = 25000000;
-	if (pdata && pdata->max_freq)
-		mmc->f_max = pdata->max_freq;
-	if (pdata && pdata->caps)
-		mmc->caps |= pdata->caps;
 	mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34;
 
 	/* With no iommu coalescing pages, each phys_seg is a hw_seg.
@@ -1354,6 +1355,7 @@ mmc_add_host_fail:
 	mmc_davinci_cpufreq_deregister(host);
 cpu_freq_fail:
 	davinci_release_dma_channels(host);
+parse_fail:
 dma_probe_defer:
 	clk_disable_unprepare(host->clk);
 clk_prepare_enable_fail:
diff --git a/drivers/mmc/host/dw_mmc-exynos.c b/drivers/mmc/host/dw_mmc-exynos.c
index 7ab3d749b5ae..e1335289316c 100644
--- a/drivers/mmc/host/dw_mmc-exynos.c
+++ b/drivers/mmc/host/dw_mmc-exynos.c
@@ -17,6 +17,7 @@
 #include <linux/mmc/mmc.h>
 #include <linux/of.h>
 #include <linux/of_gpio.h>
+#include <linux/pm_runtime.h>
 #include <linux/slab.h>
 
 #include "dw_mmc.h"
@@ -161,20 +162,13 @@ static void dw_mci_exynos_set_clksel_timing(struct dw_mci *host, u32 timing)
 		set_bit(DW_MMC_CARD_NO_USE_HOLD, &host->cur_slot->flags);
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int dw_mci_exynos_suspend(struct device *dev)
-{
-	struct dw_mci *host = dev_get_drvdata(dev);
-
-	return dw_mci_suspend(host);
-}
-
-static int dw_mci_exynos_resume(struct device *dev)
+#ifdef CONFIG_PM
+static int dw_mci_exynos_runtime_resume(struct device *dev)
 {
 	struct dw_mci *host = dev_get_drvdata(dev);
 
 	dw_mci_exynos_config_smu(host);
-	return dw_mci_resume(host);
+	return dw_mci_runtime_resume(dev);
 }
 
 /**
@@ -211,10 +205,8 @@ static int dw_mci_exynos_resume_noirq(struct device *dev)
 	return 0;
 }
 #else
-#define dw_mci_exynos_suspend		NULL
-#define dw_mci_exynos_resume		NULL
 #define dw_mci_exynos_resume_noirq	NULL
-#endif /* CONFIG_PM_SLEEP */
+#endif /* CONFIG_PM */
 
 static void dw_mci_exynos_config_hs400(struct dw_mci *host, u32 timing)
 {
@@ -524,14 +516,42 @@ static int dw_mci_exynos_probe(struct platform_device *pdev)
 {
 	const struct dw_mci_drv_data *drv_data;
 	const struct of_device_id *match;
+	int ret;
 
 	match = of_match_node(dw_mci_exynos_match, pdev->dev.of_node);
 	drv_data = match->data;
-	return dw_mci_pltfm_register(pdev, drv_data);
+
+	pm_runtime_get_noresume(&pdev->dev);
+	pm_runtime_set_active(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
+
+	ret = dw_mci_pltfm_register(pdev, drv_data);
+	if (ret) {
+		pm_runtime_disable(&pdev->dev);
+		pm_runtime_set_suspended(&pdev->dev);
+		pm_runtime_put_noidle(&pdev->dev);
+
+		return ret;
+	}
+
+	return 0;
+}
+
+static int dw_mci_exynos_remove(struct platform_device *pdev)
+{
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_set_suspended(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
+
+	return dw_mci_pltfm_remove(pdev);
 }
 
 static const struct dev_pm_ops dw_mci_exynos_pmops = {
-	SET_SYSTEM_SLEEP_PM_OPS(dw_mci_exynos_suspend, dw_mci_exynos_resume)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(dw_mci_runtime_suspend,
+			   dw_mci_exynos_runtime_resume,
+			   NULL)
 	.resume_noirq = dw_mci_exynos_resume_noirq,
 	.thaw_noirq = dw_mci_exynos_resume_noirq,
 	.restore_noirq = dw_mci_exynos_resume_noirq,
@@ -539,7 +559,7 @@ static const struct dev_pm_ops dw_mci_exynos_pmops = {
 
 static struct platform_driver dw_mci_exynos_pltfm_driver = {
 	.probe		= dw_mci_exynos_probe,
-	.remove		= dw_mci_pltfm_remove,
+	.remove		= dw_mci_exynos_remove,
 	.driver		= {
 		.name		= "dwmmc_exynos",
 		.of_match_table	= dw_mci_exynos_match,
diff --git a/drivers/mmc/host/dw_mmc-k3.c b/drivers/mmc/host/dw_mmc-k3.c
index 624789496dce..9821e6bd5d5e 100644
--- a/drivers/mmc/host/dw_mmc-k3.c
+++ b/drivers/mmc/host/dw_mmc-k3.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
 
@@ -162,35 +163,13 @@ static int dw_mci_k3_probe(struct platform_device *pdev)
 	return dw_mci_pltfm_register(pdev, drv_data);
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int dw_mci_k3_suspend(struct device *dev)
-{
-	struct dw_mci *host = dev_get_drvdata(dev);
-	int ret;
-
-	ret = dw_mci_suspend(host);
-	if (!ret)
-		clk_disable_unprepare(host->ciu_clk);
-
-	return ret;
-}
-
-static int dw_mci_k3_resume(struct device *dev)
-{
-	struct dw_mci *host = dev_get_drvdata(dev);
-	int ret;
-
-	ret = clk_prepare_enable(host->ciu_clk);
-	if (ret) {
-		dev_err(host->dev, "failed to enable ciu clock\n");
-		return ret;
-	}
-
-	return dw_mci_resume(host);
-}
-#endif /* CONFIG_PM_SLEEP */
-
-static SIMPLE_DEV_PM_OPS(dw_mci_k3_pmops, dw_mci_k3_suspend, dw_mci_k3_resume);
+static const struct dev_pm_ops dw_mci_k3_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(dw_mci_runtime_suspend,
+			   dw_mci_runtime_resume,
+			   NULL)
+};
 
 static struct platform_driver dw_mci_k3_pltfm_driver = {
 	.probe		= dw_mci_k3_probe,
@@ -198,7 +177,7 @@ static struct platform_driver dw_mci_k3_pltfm_driver = {
 	.driver		= {
 		.name		= "dwmmc_k3",
 		.of_match_table	= dw_mci_k3_match,
-		.pm		= &dw_mci_k3_pmops,
+		.pm		= &dw_mci_k3_dev_pm_ops,
 	},
 };
 
diff --git a/drivers/mmc/host/dw_mmc-pci.c b/drivers/mmc/host/dw_mmc-pci.c
index 4c69fbd29811..ab82796b01e2 100644
--- a/drivers/mmc/host/dw_mmc-pci.c
+++ b/drivers/mmc/host/dw_mmc-pci.c
@@ -14,6 +14,7 @@
 #include <linux/io.h>
 #include <linux/irq.h>
 #include <linux/pci.h>
+#include <linux/pm_runtime.h>
 #include <linux/slab.h>
 #include <linux/mmc/host.h>
 #include <linux/mmc/mmc.h>
@@ -79,25 +80,13 @@ static void dw_mci_pci_remove(struct pci_dev *pdev)
 	dw_mci_remove(host);
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int dw_mci_pci_suspend(struct device *dev)
-{
-	struct pci_dev *pdev = to_pci_dev(dev);
-	struct dw_mci *host = pci_get_drvdata(pdev);
-
-	return dw_mci_suspend(host);
-}
-
-static int dw_mci_pci_resume(struct device *dev)
-{
-	struct pci_dev *pdev = to_pci_dev(dev);
-	struct dw_mci *host = pci_get_drvdata(pdev);
-
-	return dw_mci_resume(host);
-}
-#endif /* CONFIG_PM_SLEEP */
-
-static SIMPLE_DEV_PM_OPS(dw_mci_pci_pmops, dw_mci_pci_suspend, dw_mci_pci_resume);
+static const struct dev_pm_ops dw_mci_pci_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(dw_mci_runtime_suspend,
+			   dw_mci_runtime_resume,
+			   NULL)
+};
 
 static const struct pci_device_id dw_mci_pci_id[] = {
 	{ PCI_DEVICE(SYNOPSYS_DW_MCI_VENDOR_ID, SYNOPSYS_DW_MCI_DEVICE_ID) },
@@ -111,7 +100,7 @@ static struct pci_driver dw_mci_pci_driver = {
 	.probe		= dw_mci_pci_probe,
 	.remove		= dw_mci_pci_remove,
 	.driver		=	{
-		.pm =   &dw_mci_pci_pmops
+		.pm =   &dw_mci_pci_dev_pm_ops,
 	},
 };
 
diff --git a/drivers/mmc/host/dw_mmc-pltfm.c b/drivers/mmc/host/dw_mmc-pltfm.c
index dbbc4303bdd0..1236d49ba36e 100644
--- a/drivers/mmc/host/dw_mmc-pltfm.c
+++ b/drivers/mmc/host/dw_mmc-pltfm.c
@@ -16,6 +16,7 @@
 #include <linux/io.h>
 #include <linux/irq.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/slab.h>
 #include <linux/mmc/host.h>
 #include <linux/mmc/mmc.h>
@@ -58,26 +59,13 @@ int dw_mci_pltfm_register(struct platform_device *pdev,
 }
 EXPORT_SYMBOL_GPL(dw_mci_pltfm_register);
 
-#ifdef CONFIG_PM_SLEEP
-/*
- * TODO: we should probably disable the clock to the card in the suspend path.
- */
-static int dw_mci_pltfm_suspend(struct device *dev)
-{
-	struct dw_mci *host = dev_get_drvdata(dev);
-
-	return dw_mci_suspend(host);
-}
-
-static int dw_mci_pltfm_resume(struct device *dev)
-{
-	struct dw_mci *host = dev_get_drvdata(dev);
-
-	return dw_mci_resume(host);
-}
-#endif /* CONFIG_PM_SLEEP */
-
-SIMPLE_DEV_PM_OPS(dw_mci_pltfm_pmops, dw_mci_pltfm_suspend, dw_mci_pltfm_resume);
+const struct dev_pm_ops dw_mci_pltfm_pmops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(dw_mci_runtime_suspend,
+			   dw_mci_runtime_resume,
+			   NULL)
+};
 EXPORT_SYMBOL_GPL(dw_mci_pltfm_pmops);
 
 static const struct of_device_id dw_mci_pltfm_match[] = {
diff --git a/drivers/mmc/host/dw_mmc-rockchip.c b/drivers/mmc/host/dw_mmc-rockchip.c
index 25eae359a5ea..9a46e4694227 100644
--- a/drivers/mmc/host/dw_mmc-rockchip.c
+++ b/drivers/mmc/host/dw_mmc-rockchip.c
@@ -13,6 +13,8 @@
 #include <linux/mmc/host.h>
 #include <linux/mmc/dw_mmc.h>
 #include <linux/of_address.h>
+#include <linux/mmc/slot-gpio.h>
+#include <linux/pm_runtime.h>
 #include <linux/slab.h>
 
 #include "dw_mmc.h"
@@ -325,6 +327,7 @@ static int dw_mci_rockchip_probe(struct platform_device *pdev)
 {
 	const struct dw_mci_drv_data *drv_data;
 	const struct of_device_id *match;
+	int ret;
 
 	if (!pdev->dev.of_node)
 		return -ENODEV;
@@ -332,16 +335,49 @@ static int dw_mci_rockchip_probe(struct platform_device *pdev)
 	match = of_match_node(dw_mci_rockchip_match, pdev->dev.of_node);
 	drv_data = match->data;
 
-	return dw_mci_pltfm_register(pdev, drv_data);
+	pm_runtime_get_noresume(&pdev->dev);
+	pm_runtime_set_active(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
+	pm_runtime_set_autosuspend_delay(&pdev->dev, 50);
+	pm_runtime_use_autosuspend(&pdev->dev);
+
+	ret = dw_mci_pltfm_register(pdev, drv_data);
+	if (ret) {
+		pm_runtime_disable(&pdev->dev);
+		pm_runtime_set_suspended(&pdev->dev);
+		pm_runtime_put_noidle(&pdev->dev);
+		return ret;
+	}
+
+	pm_runtime_put_autosuspend(&pdev->dev);
+
+	return 0;
 }
 
+static int dw_mci_rockchip_remove(struct platform_device *pdev)
+{
+	pm_runtime_get_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
+
+	return dw_mci_pltfm_remove(pdev);
+}
+
+static const struct dev_pm_ops dw_mci_rockchip_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(dw_mci_runtime_suspend,
+			   dw_mci_runtime_resume,
+			   NULL)
+};
+
 static struct platform_driver dw_mci_rockchip_pltfm_driver = {
 	.probe		= dw_mci_rockchip_probe,
-	.remove		= dw_mci_pltfm_remove,
+	.remove		= dw_mci_rockchip_remove,
 	.driver		= {
 		.name		= "dwmmc_rockchip",
 		.of_match_table	= dw_mci_rockchip_match,
-		.pm		= &dw_mci_pltfm_pmops,
+		.pm		= &dw_mci_rockchip_dev_pm_ops,
 	},
 };
 
diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
index df478ae72e23..b44306b886cb 100644
--- a/drivers/mmc/host/dw_mmc.c
+++ b/drivers/mmc/host/dw_mmc.c
@@ -54,7 +54,7 @@
 #define DW_MCI_DMA_THRESHOLD	16
 
 #define DW_MCI_FREQ_MAX	200000000	/* unit: HZ */
-#define DW_MCI_FREQ_MIN	400000		/* unit: HZ */
+#define DW_MCI_FREQ_MIN	100000		/* unit: HZ */
 
 #define IDMAC_INT_CLR		(SDMMC_IDMAC_INT_AI | SDMMC_IDMAC_INT_NI | \
 				 SDMMC_IDMAC_INT_CES | SDMMC_IDMAC_INT_DU | \
@@ -165,12 +165,14 @@ static const struct file_operations dw_mci_req_fops = {
 
 static int dw_mci_regs_show(struct seq_file *s, void *v)
 {
-	seq_printf(s, "STATUS:\t0x%08x\n", SDMMC_STATUS);
-	seq_printf(s, "RINTSTS:\t0x%08x\n", SDMMC_RINTSTS);
-	seq_printf(s, "CMD:\t0x%08x\n", SDMMC_CMD);
-	seq_printf(s, "CTRL:\t0x%08x\n", SDMMC_CTRL);
-	seq_printf(s, "INTMASK:\t0x%08x\n", SDMMC_INTMASK);
-	seq_printf(s, "CLKENA:\t0x%08x\n", SDMMC_CLKENA);
+	struct dw_mci *host = s->private;
+
+	seq_printf(s, "STATUS:\t0x%08x\n", mci_readl(host, STATUS));
+	seq_printf(s, "RINTSTS:\t0x%08x\n", mci_readl(host, RINTSTS));
+	seq_printf(s, "CMD:\t0x%08x\n", mci_readl(host, CMD));
+	seq_printf(s, "CTRL:\t0x%08x\n", mci_readl(host, CTRL));
+	seq_printf(s, "INTMASK:\t0x%08x\n", mci_readl(host, INTMASK));
+	seq_printf(s, "CLKENA:\t0x%08x\n", mci_readl(host, CLKENA));
 
 	return 0;
 }
@@ -234,7 +236,6 @@ static void mci_send_cmd(struct dw_mci_slot *slot, u32 cmd, u32 arg);
 
 static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd)
 {
-	struct mmc_data	*data;
 	struct dw_mci_slot *slot = mmc_priv(mmc);
 	struct dw_mci *host = slot->host;
 	u32 cmdr;
@@ -289,10 +290,9 @@ static u32 dw_mci_prepare_command(struct mmc_host *mmc, struct mmc_command *cmd)
 	if (cmd->flags & MMC_RSP_CRC)
 		cmdr |= SDMMC_CMD_RESP_CRC;
 
-	data = cmd->data;
-	if (data) {
+	if (cmd->data) {
 		cmdr |= SDMMC_CMD_DAT_EXP;
-		if (data->flags & MMC_DATA_WRITE)
+		if (cmd->data->flags & MMC_DATA_WRITE)
 			cmdr |= SDMMC_CMD_DAT_WR;
 	}
 
@@ -335,6 +335,9 @@ static u32 dw_mci_prep_stop_abort(struct dw_mci *host, struct mmc_command *cmd)
 	cmdr = stop->opcode | SDMMC_CMD_STOP |
 		SDMMC_CMD_RESP_CRC | SDMMC_CMD_RESP_EXP;
 
+	if (!test_bit(DW_MMC_CARD_NO_USE_HOLD, &host->cur_slot->flags))
+		cmdr |= SDMMC_CMD_USE_HOLD_REG;
+
 	return cmdr;
 }
 
@@ -380,7 +383,7 @@ static void dw_mci_start_command(struct dw_mci *host,
 
 static inline void send_stop_abort(struct dw_mci *host, struct mmc_data *data)
 {
-	struct mmc_command *stop = data->stop ? data->stop : &host->stop_abort;
+	struct mmc_command *stop = &host->stop_abort;
 
 	dw_mci_start_command(host, stop, host->stop_cmdr);
 }
@@ -409,12 +412,13 @@ static void dw_mci_dma_cleanup(struct dw_mci *host)
 {
 	struct mmc_data *data = host->data;
 
-	if (data)
-		if (!data->host_cookie)
-			dma_unmap_sg(host->dev,
-				     data->sg,
-				     data->sg_len,
-				     dw_mci_get_dma_dir(data));
+	if (data && data->host_cookie == COOKIE_MAPPED) {
+		dma_unmap_sg(host->dev,
+			     data->sg,
+			     data->sg_len,
+			     dw_mci_get_dma_dir(data));
+		data->host_cookie = COOKIE_UNMAPPED;
+	}
 }
 
 static void dw_mci_idmac_reset(struct dw_mci *host)
@@ -612,7 +616,7 @@ static inline int dw_mci_prepare_desc64(struct dw_mci *host,
 	return 0;
 err_own_bit:
 	/* restore the descriptor chain as it's polluted */
-	dev_dbg(host->dev, "desciptor is still owned by IDMAC.\n");
+	dev_dbg(host->dev, "descriptor is still owned by IDMAC.\n");
 	memset(host->sg_cpu, 0, DESC_RING_BUF_SZ);
 	dw_mci_idmac_init(host);
 	return -EINVAL;
@@ -688,7 +692,7 @@ static inline int dw_mci_prepare_desc32(struct dw_mci *host,
 	return 0;
 err_own_bit:
 	/* restore the descriptor chain as it's polluted */
-	dev_dbg(host->dev, "desciptor is still owned by IDMAC.\n");
+	dev_dbg(host->dev, "descriptor is still owned by IDMAC.\n");
 	memset(host->sg_cpu, 0, DESC_RING_BUF_SZ);
 	dw_mci_idmac_init(host);
 	return -EINVAL;
@@ -845,13 +849,13 @@ static const struct dw_mci_dma_ops dw_mci_edmac_ops = {
 
 static int dw_mci_pre_dma_transfer(struct dw_mci *host,
 				   struct mmc_data *data,
-				   bool next)
+				   int cookie)
 {
 	struct scatterlist *sg;
 	unsigned int i, sg_len;
 
-	if (!next && data->host_cookie)
-		return data->host_cookie;
+	if (data->host_cookie == COOKIE_PRE_MAPPED)
+		return data->sg_len;
 
 	/*
 	 * We don't do DMA on "complex" transfers, i.e. with
@@ -876,15 +880,13 @@ static int dw_mci_pre_dma_transfer(struct dw_mci *host,
 	if (sg_len == 0)
 		return -EINVAL;
 
-	if (next)
-		data->host_cookie = sg_len;
+	data->host_cookie = cookie;
 
 	return sg_len;
 }
 
 static void dw_mci_pre_req(struct mmc_host *mmc,
-			   struct mmc_request *mrq,
-			   bool is_first_req)
+			   struct mmc_request *mrq)
 {
 	struct dw_mci_slot *slot = mmc_priv(mmc);
 	struct mmc_data *data = mrq->data;
@@ -892,13 +894,12 @@ static void dw_mci_pre_req(struct mmc_host *mmc,
 	if (!slot->host->use_dma || !data)
 		return;
 
-	if (data->host_cookie) {
-		data->host_cookie = 0;
-		return;
-	}
+	/* This data might be unmapped at this time */
+	data->host_cookie = COOKIE_UNMAPPED;
 
-	if (dw_mci_pre_dma_transfer(slot->host, mrq->data, 1) < 0)
-		data->host_cookie = 0;
+	if (dw_mci_pre_dma_transfer(slot->host, mrq->data,
+				COOKIE_PRE_MAPPED) < 0)
+		data->host_cookie = COOKIE_UNMAPPED;
 }
 
 static void dw_mci_post_req(struct mmc_host *mmc,
@@ -911,12 +912,12 @@ static void dw_mci_post_req(struct mmc_host *mmc,
 	if (!slot->host->use_dma || !data)
 		return;
 
-	if (data->host_cookie)
+	if (data->host_cookie != COOKIE_UNMAPPED)
 		dma_unmap_sg(slot->host->dev,
 			     data->sg,
 			     data->sg_len,
 			     dw_mci_get_dma_dir(data));
-	data->host_cookie = 0;
+	data->host_cookie = COOKIE_UNMAPPED;
 }
 
 static void dw_mci_adjust_fifoth(struct dw_mci *host, struct mmc_data *data)
@@ -1022,7 +1023,7 @@ static int dw_mci_submit_data_dma(struct dw_mci *host, struct mmc_data *data)
 	if (!host->use_dma)
 		return -ENODEV;
 
-	sg_len = dw_mci_pre_dma_transfer(host, data, 0);
+	sg_len = dw_mci_pre_dma_transfer(host, data, COOKIE_MAPPED);
 	if (sg_len < 0) {
 		host->dma_ops->stop(host);
 		return sg_len;
@@ -1175,13 +1176,24 @@ static void dw_mci_setup_bus(struct dw_mci_slot *slot, bool force_clkinit)
 
 		div = (host->bus_hz != clock) ? DIV_ROUND_UP(div, 2) : 0;
 
-		if (clock != slot->__clk_old || force_clkinit)
+		if ((clock != slot->__clk_old &&
+			!test_bit(DW_MMC_CARD_NEEDS_POLL, &slot->flags)) ||
+			force_clkinit) {
 			dev_info(&slot->mmc->class_dev,
 				 "Bus speed (slot %d) = %dHz (slot req %dHz, actual %dHZ div = %d)\n",
 				 slot->id, host->bus_hz, clock,
 				 div ? ((host->bus_hz / div) >> 1) :
 				 host->bus_hz, div);
 
+			/*
+			 * If card is polling, display the message only
+			 * one time at boot time.
+			 */
+			if (slot->mmc->caps & MMC_CAP_NEEDS_POLL &&
+					slot->mmc->f_min == clock)
+				set_bit(DW_MMC_CARD_NEEDS_POLL, &slot->flags);
+		}
+
 		/* disable clock */
 		mci_writel(host, CLKENA, 0);
 		mci_writel(host, CLKSRC, 0);
@@ -1273,10 +1285,7 @@ static void __dw_mci_start_request(struct dw_mci *host,
 		spin_unlock_irqrestore(&host->irq_lock, irqflags);
 	}
 
-	if (mrq->stop)
-		host->stop_cmdr = dw_mci_prepare_command(slot->mmc, mrq->stop);
-	else
-		host->stop_cmdr = dw_mci_prep_stop_abort(host, cmd);
+	host->stop_cmdr = dw_mci_prep_stop_abort(host, cmd);
 }
 
 static void dw_mci_start_request(struct dw_mci *host,
@@ -1527,22 +1536,34 @@ static int dw_mci_get_cd(struct mmc_host *mmc)
 	int gpio_cd = mmc_gpio_get_cd(mmc);
 
 	/* Use platform get_cd function, else try onboard card detect */
-	if ((mmc->caps & MMC_CAP_NEEDS_POLL) || !mmc_card_is_removable(mmc))
+	if (((mmc->caps & MMC_CAP_NEEDS_POLL)
+				|| !mmc_card_is_removable(mmc))) {
 		present = 1;
-	else if (gpio_cd >= 0)
+
+		if (!test_bit(DW_MMC_CARD_PRESENT, &slot->flags)) {
+			if (mmc->caps & MMC_CAP_NEEDS_POLL) {
+				dev_info(&mmc->class_dev,
+					"card is polling.\n");
+			} else {
+				dev_info(&mmc->class_dev,
+					"card is non-removable.\n");
+			}
+			set_bit(DW_MMC_CARD_PRESENT, &slot->flags);
+		}
+
+		return present;
+	} else if (gpio_cd >= 0)
 		present = gpio_cd;
 	else
 		present = (mci_readl(slot->host, CDETECT) & (1 << slot->id))
 			== 0 ? 1 : 0;
 
 	spin_lock_bh(&host->lock);
-	if (present) {
-		set_bit(DW_MMC_CARD_PRESENT, &slot->flags);
+	if (present && !test_and_set_bit(DW_MMC_CARD_PRESENT, &slot->flags))
 		dev_dbg(&mmc->class_dev, "card is present\n");
-	} else {
-		clear_bit(DW_MMC_CARD_PRESENT, &slot->flags);
+	else if (!present &&
+			!test_and_clear_bit(DW_MMC_CARD_PRESENT, &slot->flags))
 		dev_dbg(&mmc->class_dev, "card is not present\n");
-	}
 	spin_unlock_bh(&host->lock);
 
 	return present;
@@ -1889,8 +1910,7 @@ static void dw_mci_tasklet_func(unsigned long priv)
 			if (test_and_clear_bit(EVENT_DATA_ERROR,
 					       &host->pending_events)) {
 				dw_mci_stop_dma(host);
-				if (data->stop ||
-				    !(host->data_status & (SDMMC_INT_DRTO |
+				if (!(host->data_status & (SDMMC_INT_DRTO |
 							   SDMMC_INT_EBE)))
 					send_stop_abort(host, data);
 				state = STATE_DATA_ERROR;
@@ -1926,8 +1946,7 @@ static void dw_mci_tasklet_func(unsigned long priv)
 			if (test_and_clear_bit(EVENT_DATA_ERROR,
 					       &host->pending_events)) {
 				dw_mci_stop_dma(host);
-				if (data->stop ||
-				    !(host->data_status & (SDMMC_INT_DRTO |
+				if (!(host->data_status & (SDMMC_INT_DRTO |
 							   SDMMC_INT_EBE)))
 					send_stop_abort(host, data);
 				state = STATE_DATA_ERROR;
@@ -2003,7 +2022,7 @@ static void dw_mci_tasklet_func(unsigned long priv)
 			host->cmd = NULL;
 			host->data = NULL;
 
-			if (mrq->stop)
+			if (!mrq->sbc && mrq->stop)
 				dw_mci_command_complete(host, mrq->stop);
 			else
 				host->cmd_status = 0;
@@ -2615,6 +2634,8 @@ static int dw_mci_init_slot(struct dw_mci *host, unsigned int id)
 		mmc->f_min = DW_MCI_FREQ_MIN;
 		mmc->f_max = DW_MCI_FREQ_MAX;
 	} else {
+		dev_info(host->dev,
+			"'clock-freq-min-max' property was deprecated.\n");
 		mmc->f_min = freq[0];
 		mmc->f_max = freq[1];
 	}
@@ -3267,26 +3288,46 @@ EXPORT_SYMBOL(dw_mci_remove);
 
 
 
-#ifdef CONFIG_PM_SLEEP
-/*
- * TODO: we should probably disable the clock to the card in the suspend path.
- */
-int dw_mci_suspend(struct dw_mci *host)
+#ifdef CONFIG_PM
+int dw_mci_runtime_suspend(struct device *dev)
 {
+	struct dw_mci *host = dev_get_drvdata(dev);
+
 	if (host->use_dma && host->dma_ops->exit)
 		host->dma_ops->exit(host);
 
+	clk_disable_unprepare(host->ciu_clk);
+
+	if (host->cur_slot &&
+	    (mmc_can_gpio_cd(host->cur_slot->mmc) ||
+	     !mmc_card_is_removable(host->cur_slot->mmc)))
+		clk_disable_unprepare(host->biu_clk);
+
 	return 0;
 }
-EXPORT_SYMBOL(dw_mci_suspend);
+EXPORT_SYMBOL(dw_mci_runtime_suspend);
 
-int dw_mci_resume(struct dw_mci *host)
+int dw_mci_runtime_resume(struct device *dev)
 {
-	int i, ret;
+	int i, ret = 0;
+	struct dw_mci *host = dev_get_drvdata(dev);
+
+	if (host->cur_slot &&
+	    (mmc_can_gpio_cd(host->cur_slot->mmc) ||
+	     !mmc_card_is_removable(host->cur_slot->mmc))) {
+		ret = clk_prepare_enable(host->biu_clk);
+		if (ret)
+			return ret;
+	}
+
+	ret = clk_prepare_enable(host->ciu_clk);
+	if (ret)
+		goto err;
 
 	if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS)) {
+		clk_disable_unprepare(host->ciu_clk);
 		ret = -ENODEV;
-		return ret;
+		goto err;
 	}
 
 	if (host->use_dma && host->dma_ops->init)
@@ -3296,8 +3337,8 @@ int dw_mci_resume(struct dw_mci *host)
 	 * Restore the initial value at FIFOTH register
 	 * And Invalidate the prev_blksz with zero
 	 */
-	mci_writel(host, FIFOTH, host->fifoth_val);
-	host->prev_blksz = 0;
+	 mci_writel(host, FIFOTH, host->fifoth_val);
+	 host->prev_blksz = 0;
 
 	/* Put in max timeout */
 	mci_writel(host, TMOUT, 0xFFFFFFFF);
@@ -3323,9 +3364,17 @@ int dw_mci_resume(struct dw_mci *host)
 	dw_mci_enable_cd(host);
 
 	return 0;
+
+err:
+	if (host->cur_slot &&
+	    (mmc_can_gpio_cd(host->cur_slot->mmc) ||
+	     !mmc_card_is_removable(host->cur_slot->mmc)))
+		clk_disable_unprepare(host->biu_clk);
+
+	return ret;
 }
-EXPORT_SYMBOL(dw_mci_resume);
-#endif /* CONFIG_PM_SLEEP */
+EXPORT_SYMBOL(dw_mci_runtime_resume);
+#endif /* CONFIG_PM */
 
 static int __init dw_mci_init(void)
 {
diff --git a/drivers/mmc/host/dw_mmc.h b/drivers/mmc/host/dw_mmc.h
index e8cd2dec3263..c59465829387 100644
--- a/drivers/mmc/host/dw_mmc.h
+++ b/drivers/mmc/host/dw_mmc.h
@@ -234,9 +234,9 @@
 
 extern int dw_mci_probe(struct dw_mci *host);
 extern void dw_mci_remove(struct dw_mci *host);
-#ifdef CONFIG_PM_SLEEP
-extern int dw_mci_suspend(struct dw_mci *host);
-extern int dw_mci_resume(struct dw_mci *host);
+#ifdef CONFIG_PM
+extern int dw_mci_runtime_suspend(struct device *device);
+extern int dw_mci_runtime_resume(struct device *device);
 #endif
 
 /**
@@ -272,6 +272,7 @@ struct dw_mci_slot {
 #define DW_MMC_CARD_NEED_INIT	1
 #define DW_MMC_CARD_NO_LOW_PWR	2
 #define DW_MMC_CARD_NO_USE_HOLD 3
+#define DW_MMC_CARD_NEEDS_POLL	4
 	int			id;
 	int			sdio_id;
 };
diff --git a/drivers/mmc/host/jz4740_mmc.c b/drivers/mmc/host/jz4740_mmc.c
index 684087db170b..819ad32964fc 100644
--- a/drivers/mmc/host/jz4740_mmc.c
+++ b/drivers/mmc/host/jz4740_mmc.c
@@ -320,8 +320,7 @@ dma_unmap:
 }
 
 static void jz4740_mmc_pre_request(struct mmc_host *mmc,
-				   struct mmc_request *mrq,
-				   bool is_first_req)
+				   struct mmc_request *mrq)
 {
 	struct jz4740_mmc_host *host = mmc_priv(mmc);
 	struct mmc_data *data = mrq->data;
diff --git a/drivers/mmc/host/meson-gx-mmc.c b/drivers/mmc/host/meson-gx-mmc.c
new file mode 100644
index 000000000000..b352760c041e
--- /dev/null
+++ b/drivers/mmc/host/meson-gx-mmc.c
@@ -0,0 +1,851 @@
+/*
+ * Amlogic SD/eMMC driver for the GX/S905 family SoCs
+ *
+ * Copyright (c) 2016 BayLibre, SAS.
+ * Author: Kevin Hilman <khilman@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/device.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/ioport.h>
+#include <linux/spinlock.h>
+#include <linux/dma-mapping.h>
+#include <linux/mmc/host.h>
+#include <linux/mmc/mmc.h>
+#include <linux/mmc/sdio.h>
+#include <linux/mmc/slot-gpio.h>
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/regulator/consumer.h>
+
+#define DRIVER_NAME "meson-gx-mmc"
+
+#define SD_EMMC_CLOCK 0x0
+#define   CLK_DIV_SHIFT 0
+#define   CLK_DIV_WIDTH 6
+#define   CLK_DIV_MASK 0x3f
+#define   CLK_DIV_MAX 63
+#define   CLK_SRC_SHIFT 6
+#define   CLK_SRC_WIDTH 2
+#define   CLK_SRC_MASK 0x3
+#define   CLK_SRC_XTAL 0   /* external crystal */
+#define   CLK_SRC_XTAL_RATE 24000000
+#define   CLK_SRC_PLL 1    /* FCLK_DIV2 */
+#define   CLK_SRC_PLL_RATE 1000000000
+#define   CLK_PHASE_SHIFT 8
+#define   CLK_PHASE_MASK 0x3
+#define   CLK_PHASE_0 0
+#define   CLK_PHASE_90 1
+#define   CLK_PHASE_180 2
+#define   CLK_PHASE_270 3
+#define   CLK_ALWAYS_ON BIT(24)
+
+#define SD_EMMC_DElAY 0x4
+#define SD_EMMC_ADJUST 0x8
+#define SD_EMMC_CALOUT 0x10
+#define SD_EMMC_START 0x40
+#define   START_DESC_INIT BIT(0)
+#define   START_DESC_BUSY BIT(1)
+#define   START_DESC_ADDR_SHIFT 2
+#define   START_DESC_ADDR_MASK (~0x3)
+
+#define SD_EMMC_CFG 0x44
+#define   CFG_BUS_WIDTH_SHIFT 0
+#define   CFG_BUS_WIDTH_MASK 0x3
+#define   CFG_BUS_WIDTH_1 0x0
+#define   CFG_BUS_WIDTH_4 0x1
+#define   CFG_BUS_WIDTH_8 0x2
+#define   CFG_DDR BIT(2)
+#define   CFG_BLK_LEN_SHIFT 4
+#define   CFG_BLK_LEN_MASK 0xf
+#define   CFG_RESP_TIMEOUT_SHIFT 8
+#define   CFG_RESP_TIMEOUT_MASK 0xf
+#define   CFG_RC_CC_SHIFT 12
+#define   CFG_RC_CC_MASK 0xf
+#define   CFG_STOP_CLOCK BIT(22)
+#define   CFG_CLK_ALWAYS_ON BIT(18)
+#define   CFG_AUTO_CLK BIT(23)
+
+#define SD_EMMC_STATUS 0x48
+#define   STATUS_BUSY BIT(31)
+
+#define SD_EMMC_IRQ_EN 0x4c
+#define   IRQ_EN_MASK 0x3fff
+#define   IRQ_RXD_ERR_SHIFT 0
+#define   IRQ_RXD_ERR_MASK 0xff
+#define   IRQ_TXD_ERR BIT(8)
+#define   IRQ_DESC_ERR BIT(9)
+#define   IRQ_RESP_ERR BIT(10)
+#define   IRQ_RESP_TIMEOUT BIT(11)
+#define   IRQ_DESC_TIMEOUT BIT(12)
+#define   IRQ_END_OF_CHAIN BIT(13)
+#define   IRQ_RESP_STATUS BIT(14)
+#define   IRQ_SDIO BIT(15)
+
+#define SD_EMMC_CMD_CFG 0x50
+#define SD_EMMC_CMD_ARG 0x54
+#define SD_EMMC_CMD_DAT 0x58
+#define SD_EMMC_CMD_RSP 0x5c
+#define SD_EMMC_CMD_RSP1 0x60
+#define SD_EMMC_CMD_RSP2 0x64
+#define SD_EMMC_CMD_RSP3 0x68
+
+#define SD_EMMC_RXD 0x94
+#define SD_EMMC_TXD 0x94
+#define SD_EMMC_LAST_REG SD_EMMC_TXD
+
+#define SD_EMMC_CFG_BLK_SIZE 512 /* internal buffer max: 512 bytes */
+#define SD_EMMC_CFG_RESP_TIMEOUT 256 /* in clock cycles */
+#define SD_EMMC_CFG_CMD_GAP 16 /* in clock cycles */
+#define MUX_CLK_NUM_PARENTS 2
+
+struct meson_host {
+	struct	device		*dev;
+	struct	mmc_host	*mmc;
+	struct	mmc_request	*mrq;
+	struct	mmc_command	*cmd;
+
+	spinlock_t lock;
+	void __iomem *regs;
+	int irq;
+	u32 ocr_mask;
+	struct clk *core_clk;
+	struct clk_mux mux;
+	struct clk *mux_clk;
+	struct clk *mux_parent[MUX_CLK_NUM_PARENTS];
+	unsigned long mux_parent_rate[MUX_CLK_NUM_PARENTS];
+
+	struct clk_divider cfg_div;
+	struct clk *cfg_div_clk;
+
+	unsigned int bounce_buf_size;
+	void *bounce_buf;
+	dma_addr_t bounce_dma_addr;
+
+	bool vqmmc_enabled;
+};
+
+struct sd_emmc_desc {
+	u32 cmd_cfg;
+	u32 cmd_arg;
+	u32 cmd_data;
+	u32 cmd_resp;
+};
+#define CMD_CFG_LENGTH_SHIFT 0
+#define CMD_CFG_LENGTH_MASK 0x1ff
+#define CMD_CFG_BLOCK_MODE BIT(9)
+#define CMD_CFG_R1B BIT(10)
+#define CMD_CFG_END_OF_CHAIN BIT(11)
+#define CMD_CFG_TIMEOUT_SHIFT 12
+#define CMD_CFG_TIMEOUT_MASK 0xf
+#define CMD_CFG_NO_RESP BIT(16)
+#define CMD_CFG_NO_CMD BIT(17)
+#define CMD_CFG_DATA_IO BIT(18)
+#define CMD_CFG_DATA_WR BIT(19)
+#define CMD_CFG_RESP_NOCRC BIT(20)
+#define CMD_CFG_RESP_128 BIT(21)
+#define CMD_CFG_RESP_NUM BIT(22)
+#define CMD_CFG_DATA_NUM BIT(23)
+#define CMD_CFG_CMD_INDEX_SHIFT 24
+#define CMD_CFG_CMD_INDEX_MASK 0x3f
+#define CMD_CFG_ERROR BIT(30)
+#define CMD_CFG_OWNER BIT(31)
+
+#define CMD_DATA_MASK (~0x3)
+#define CMD_DATA_BIG_ENDIAN BIT(1)
+#define CMD_DATA_SRAM BIT(0)
+#define CMD_RESP_MASK (~0x1)
+#define CMD_RESP_SRAM BIT(0)
+
+static int meson_mmc_clk_set(struct meson_host *host, unsigned long clk_rate)
+{
+	struct mmc_host *mmc = host->mmc;
+	int ret = 0;
+	u32 cfg;
+
+	if (clk_rate) {
+		if (WARN_ON(clk_rate > mmc->f_max))
+			clk_rate = mmc->f_max;
+		else if (WARN_ON(clk_rate < mmc->f_min))
+			clk_rate = mmc->f_min;
+	}
+
+	if (clk_rate == mmc->actual_clock)
+		return 0;
+
+	/* stop clock */
+	cfg = readl(host->regs + SD_EMMC_CFG);
+	if (!(cfg & CFG_STOP_CLOCK)) {
+		cfg |= CFG_STOP_CLOCK;
+		writel(cfg, host->regs + SD_EMMC_CFG);
+	}
+
+	dev_dbg(host->dev, "change clock rate %u -> %lu\n",
+		mmc->actual_clock, clk_rate);
+
+	if (clk_rate == 0) {
+		mmc->actual_clock = 0;
+		return 0;
+	}
+
+	ret = clk_set_rate(host->cfg_div_clk, clk_rate);
+	if (ret)
+		dev_warn(host->dev, "Unable to set cfg_div_clk to %lu. ret=%d\n",
+			 clk_rate, ret);
+	else if (clk_rate && clk_rate != clk_get_rate(host->cfg_div_clk))
+		dev_warn(host->dev, "divider requested rate %lu != actual rate %lu: ret=%d\n",
+			 clk_rate, clk_get_rate(host->cfg_div_clk), ret);
+	else
+		mmc->actual_clock = clk_rate;
+
+	/* (re)start clock, if non-zero */
+	if (!ret && clk_rate) {
+		cfg = readl(host->regs + SD_EMMC_CFG);
+		cfg &= ~CFG_STOP_CLOCK;
+		writel(cfg, host->regs + SD_EMMC_CFG);
+	}
+
+	return ret;
+}
+
+/*
+ * The SD/eMMC IP block has an internal mux and divider used for
+ * generating the MMC clock.  Use the clock framework to create and
+ * manage these clocks.
+ */
+static int meson_mmc_clk_init(struct meson_host *host)
+{
+	struct clk_init_data init;
+	char clk_name[32];
+	int i, ret = 0;
+	const char *mux_parent_names[MUX_CLK_NUM_PARENTS];
+	unsigned int mux_parent_count = 0;
+	const char *clk_div_parents[1];
+	unsigned int f_min = UINT_MAX;
+	u32 clk_reg, cfg;
+
+	/* get the mux parents */
+	for (i = 0; i < MUX_CLK_NUM_PARENTS; i++) {
+		char name[16];
+
+		snprintf(name, sizeof(name), "clkin%d", i);
+		host->mux_parent[i] = devm_clk_get(host->dev, name);
+		if (IS_ERR(host->mux_parent[i])) {
+			ret = PTR_ERR(host->mux_parent[i]);
+			if (PTR_ERR(host->mux_parent[i]) != -EPROBE_DEFER)
+				dev_err(host->dev, "Missing clock %s\n", name);
+			host->mux_parent[i] = NULL;
+			return ret;
+		}
+
+		host->mux_parent_rate[i] = clk_get_rate(host->mux_parent[i]);
+		mux_parent_names[i] = __clk_get_name(host->mux_parent[i]);
+		mux_parent_count++;
+		if (host->mux_parent_rate[i] < f_min)
+			f_min = host->mux_parent_rate[i];
+	}
+
+	/* cacluate f_min based on input clocks, and max divider value */
+	if (f_min != UINT_MAX)
+		f_min = DIV_ROUND_UP(CLK_SRC_XTAL_RATE, CLK_DIV_MAX);
+	else
+		f_min = 4000000;  /* default min: 400 MHz */
+	host->mmc->f_min = f_min;
+
+	/* create the mux */
+	snprintf(clk_name, sizeof(clk_name), "%s#mux", dev_name(host->dev));
+	init.name = clk_name;
+	init.ops = &clk_mux_ops;
+	init.flags = 0;
+	init.parent_names = mux_parent_names;
+	init.num_parents = mux_parent_count;
+
+	host->mux.reg = host->regs + SD_EMMC_CLOCK;
+	host->mux.shift = CLK_SRC_SHIFT;
+	host->mux.mask = CLK_SRC_MASK;
+	host->mux.flags = 0;
+	host->mux.table = NULL;
+	host->mux.hw.init = &init;
+
+	host->mux_clk = devm_clk_register(host->dev, &host->mux.hw);
+	if (WARN_ON(IS_ERR(host->mux_clk)))
+		return PTR_ERR(host->mux_clk);
+
+	/* create the divider */
+	snprintf(clk_name, sizeof(clk_name), "%s#div", dev_name(host->dev));
+	init.name = devm_kstrdup(host->dev, clk_name, GFP_KERNEL);
+	init.ops = &clk_divider_ops;
+	init.flags = CLK_SET_RATE_PARENT;
+	clk_div_parents[0] = __clk_get_name(host->mux_clk);
+	init.parent_names = clk_div_parents;
+	init.num_parents = ARRAY_SIZE(clk_div_parents);
+
+	host->cfg_div.reg = host->regs + SD_EMMC_CLOCK;
+	host->cfg_div.shift = CLK_DIV_SHIFT;
+	host->cfg_div.width = CLK_DIV_WIDTH;
+	host->cfg_div.hw.init = &init;
+	host->cfg_div.flags = CLK_DIVIDER_ONE_BASED |
+		CLK_DIVIDER_ROUND_CLOSEST | CLK_DIVIDER_ALLOW_ZERO;
+
+	host->cfg_div_clk = devm_clk_register(host->dev, &host->cfg_div.hw);
+	if (WARN_ON(PTR_ERR_OR_ZERO(host->cfg_div_clk)))
+		return PTR_ERR(host->cfg_div_clk);
+
+	/* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
+	clk_reg = 0;
+	clk_reg |= CLK_PHASE_180 << CLK_PHASE_SHIFT;
+	clk_reg |= CLK_SRC_XTAL << CLK_SRC_SHIFT;
+	clk_reg |= CLK_DIV_MAX << CLK_DIV_SHIFT;
+	clk_reg &= ~CLK_ALWAYS_ON;
+	writel(clk_reg, host->regs + SD_EMMC_CLOCK);
+
+	/* Ensure clock starts in "auto" mode, not "always on" */
+	cfg = readl(host->regs + SD_EMMC_CFG);
+	cfg &= ~CFG_CLK_ALWAYS_ON;
+	cfg |= CFG_AUTO_CLK;
+	writel(cfg, host->regs + SD_EMMC_CFG);
+
+	ret = clk_prepare_enable(host->cfg_div_clk);
+	if (!ret)
+		ret = meson_mmc_clk_set(host, f_min);
+
+	if (!ret)
+		clk_disable_unprepare(host->cfg_div_clk);
+
+	return ret;
+}
+
+static void meson_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
+{
+	struct meson_host *host = mmc_priv(mmc);
+	u32 bus_width;
+	u32 val, orig;
+
+	/*
+	 * GPIO regulator, only controls switching between 1v8 and
+	 * 3v3, doesn't support MMC_POWER_OFF, MMC_POWER_ON.
+	 */
+	switch (ios->power_mode) {
+	case MMC_POWER_OFF:
+		if (!IS_ERR(mmc->supply.vmmc))
+			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
+
+		if (!IS_ERR(mmc->supply.vqmmc) && host->vqmmc_enabled) {
+			regulator_disable(mmc->supply.vqmmc);
+			host->vqmmc_enabled = false;
+		}
+
+		break;
+
+	case MMC_POWER_UP:
+		if (!IS_ERR(mmc->supply.vmmc))
+			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, ios->vdd);
+		break;
+
+	case MMC_POWER_ON:
+		if (!IS_ERR(mmc->supply.vqmmc) && !host->vqmmc_enabled) {
+			int ret = regulator_enable(mmc->supply.vqmmc);
+
+			if (ret < 0)
+				dev_err(mmc_dev(mmc),
+					"failed to enable vqmmc regulator\n");
+			else
+				host->vqmmc_enabled = true;
+		}
+
+		break;
+	}
+
+
+	meson_mmc_clk_set(host, ios->clock);
+
+	/* Bus width */
+	val = readl(host->regs + SD_EMMC_CFG);
+	switch (ios->bus_width) {
+	case MMC_BUS_WIDTH_1:
+		bus_width = CFG_BUS_WIDTH_1;
+		break;
+	case MMC_BUS_WIDTH_4:
+		bus_width = CFG_BUS_WIDTH_4;
+		break;
+	case MMC_BUS_WIDTH_8:
+		bus_width = CFG_BUS_WIDTH_8;
+		break;
+	default:
+		dev_err(host->dev, "Invalid ios->bus_width: %u.  Setting to 4.\n",
+			ios->bus_width);
+		bus_width = CFG_BUS_WIDTH_4;
+		return;
+	}
+
+	val = readl(host->regs + SD_EMMC_CFG);
+	orig = val;
+
+	val &= ~(CFG_BUS_WIDTH_MASK << CFG_BUS_WIDTH_SHIFT);
+	val |= bus_width << CFG_BUS_WIDTH_SHIFT;
+
+	val &= ~(CFG_BLK_LEN_MASK << CFG_BLK_LEN_SHIFT);
+	val |= ilog2(SD_EMMC_CFG_BLK_SIZE) << CFG_BLK_LEN_SHIFT;
+
+	val &= ~(CFG_RESP_TIMEOUT_MASK << CFG_RESP_TIMEOUT_SHIFT);
+	val |= ilog2(SD_EMMC_CFG_RESP_TIMEOUT) << CFG_RESP_TIMEOUT_SHIFT;
+
+	val &= ~(CFG_RC_CC_MASK << CFG_RC_CC_SHIFT);
+	val |= ilog2(SD_EMMC_CFG_CMD_GAP) << CFG_RC_CC_SHIFT;
+
+	writel(val, host->regs + SD_EMMC_CFG);
+
+	if (val != orig)
+		dev_dbg(host->dev, "%s: SD_EMMC_CFG: 0x%08x -> 0x%08x\n",
+			__func__, orig, val);
+}
+
+static int meson_mmc_request_done(struct mmc_host *mmc, struct mmc_request *mrq)
+{
+	struct meson_host *host = mmc_priv(mmc);
+
+	WARN_ON(host->mrq != mrq);
+
+	host->mrq = NULL;
+	host->cmd = NULL;
+	mmc_request_done(host->mmc, mrq);
+
+	return 0;
+}
+
+static void meson_mmc_start_cmd(struct mmc_host *mmc, struct mmc_command *cmd)
+{
+	struct meson_host *host = mmc_priv(mmc);
+	struct sd_emmc_desc *desc, desc_tmp;
+	u32 cfg;
+	u8 blk_len, cmd_cfg_timeout;
+	unsigned int xfer_bytes = 0;
+
+	/* Setup descriptors */
+	dma_rmb();
+	desc = &desc_tmp;
+	memset(desc, 0, sizeof(struct sd_emmc_desc));
+
+	desc->cmd_cfg |= (cmd->opcode & CMD_CFG_CMD_INDEX_MASK)	<<
+		CMD_CFG_CMD_INDEX_SHIFT;
+	desc->cmd_cfg |= CMD_CFG_OWNER;  /* owned by CPU */
+	desc->cmd_arg = cmd->arg;
+
+	/* Response */
+	if (cmd->flags & MMC_RSP_PRESENT) {
+		desc->cmd_cfg &= ~CMD_CFG_NO_RESP;
+		if (cmd->flags & MMC_RSP_136)
+			desc->cmd_cfg |= CMD_CFG_RESP_128;
+		desc->cmd_cfg |= CMD_CFG_RESP_NUM;
+		desc->cmd_resp = 0;
+
+		if (!(cmd->flags & MMC_RSP_CRC))
+			desc->cmd_cfg |= CMD_CFG_RESP_NOCRC;
+
+		if (cmd->flags & MMC_RSP_BUSY)
+			desc->cmd_cfg |= CMD_CFG_R1B;
+	} else {
+		desc->cmd_cfg |= CMD_CFG_NO_RESP;
+	}
+
+	/* data? */
+	if (cmd->data) {
+		desc->cmd_cfg |= CMD_CFG_DATA_IO;
+		if (cmd->data->blocks > 1) {
+			desc->cmd_cfg |= CMD_CFG_BLOCK_MODE;
+			desc->cmd_cfg |=
+				(cmd->data->blocks & CMD_CFG_LENGTH_MASK) <<
+				CMD_CFG_LENGTH_SHIFT;
+
+			/* check if block-size matches, if not update */
+			cfg = readl(host->regs + SD_EMMC_CFG);
+			blk_len = cfg & (CFG_BLK_LEN_MASK << CFG_BLK_LEN_SHIFT);
+			blk_len >>= CFG_BLK_LEN_SHIFT;
+			if (blk_len != ilog2(cmd->data->blksz)) {
+				dev_warn(host->dev, "%s: update blk_len %d -> %d\n",
+					__func__, blk_len,
+					 ilog2(cmd->data->blksz));
+				blk_len = ilog2(cmd->data->blksz);
+				cfg &= ~(CFG_BLK_LEN_MASK << CFG_BLK_LEN_SHIFT);
+				cfg |= blk_len << CFG_BLK_LEN_SHIFT;
+				writel(cfg, host->regs + SD_EMMC_CFG);
+			}
+		} else {
+			desc->cmd_cfg &= ~CMD_CFG_BLOCK_MODE;
+			desc->cmd_cfg |=
+				(cmd->data->blksz & CMD_CFG_LENGTH_MASK) <<
+				CMD_CFG_LENGTH_SHIFT;
+		}
+
+		cmd->data->bytes_xfered = 0;
+		xfer_bytes = cmd->data->blksz * cmd->data->blocks;
+		if (cmd->data->flags & MMC_DATA_WRITE) {
+			desc->cmd_cfg |= CMD_CFG_DATA_WR;
+			WARN_ON(xfer_bytes > host->bounce_buf_size);
+			sg_copy_to_buffer(cmd->data->sg, cmd->data->sg_len,
+					  host->bounce_buf, xfer_bytes);
+			cmd->data->bytes_xfered = xfer_bytes;
+			dma_wmb();
+		} else {
+			desc->cmd_cfg &= ~CMD_CFG_DATA_WR;
+		}
+
+		if (xfer_bytes > 0) {
+			desc->cmd_cfg &= ~CMD_CFG_DATA_NUM;
+			desc->cmd_data = host->bounce_dma_addr & CMD_DATA_MASK;
+		} else {
+			/* write data to data_addr */
+			desc->cmd_cfg |= CMD_CFG_DATA_NUM;
+			desc->cmd_data = 0;
+		}
+
+		cmd_cfg_timeout = 12;
+	} else {
+		desc->cmd_cfg &= ~CMD_CFG_DATA_IO;
+		cmd_cfg_timeout = 10;
+	}
+	desc->cmd_cfg |= (cmd_cfg_timeout & CMD_CFG_TIMEOUT_MASK) <<
+		CMD_CFG_TIMEOUT_SHIFT;
+
+	host->cmd = cmd;
+
+	/* Last descriptor */
+	desc->cmd_cfg |= CMD_CFG_END_OF_CHAIN;
+	writel(desc->cmd_cfg, host->regs + SD_EMMC_CMD_CFG);
+	writel(desc->cmd_data, host->regs + SD_EMMC_CMD_DAT);
+	writel(desc->cmd_resp, host->regs + SD_EMMC_CMD_RSP);
+	wmb(); /* ensure descriptor is written before kicked */
+	writel(desc->cmd_arg, host->regs + SD_EMMC_CMD_ARG);
+}
+
+static void meson_mmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
+{
+	struct meson_host *host = mmc_priv(mmc);
+
+	WARN_ON(host->mrq != NULL);
+
+	/* Stop execution */
+	writel(0, host->regs + SD_EMMC_START);
+
+	/* clear, ack, enable all interrupts */
+	writel(0, host->regs + SD_EMMC_IRQ_EN);
+	writel(IRQ_EN_MASK, host->regs + SD_EMMC_STATUS);
+	writel(IRQ_EN_MASK, host->regs + SD_EMMC_IRQ_EN);
+
+	host->mrq = mrq;
+
+	if (mrq->sbc)
+		meson_mmc_start_cmd(mmc, mrq->sbc);
+	else
+		meson_mmc_start_cmd(mmc, mrq->cmd);
+}
+
+static int meson_mmc_read_resp(struct mmc_host *mmc, struct mmc_command *cmd)
+{
+	struct meson_host *host = mmc_priv(mmc);
+
+	if (cmd->flags & MMC_RSP_136) {
+		cmd->resp[0] = readl(host->regs + SD_EMMC_CMD_RSP3);
+		cmd->resp[1] = readl(host->regs + SD_EMMC_CMD_RSP2);
+		cmd->resp[2] = readl(host->regs + SD_EMMC_CMD_RSP1);
+		cmd->resp[3] = readl(host->regs + SD_EMMC_CMD_RSP);
+	} else if (cmd->flags & MMC_RSP_PRESENT) {
+		cmd->resp[0] = readl(host->regs + SD_EMMC_CMD_RSP);
+	}
+
+	return 0;
+}
+
+static irqreturn_t meson_mmc_irq(int irq, void *dev_id)
+{
+	struct meson_host *host = dev_id;
+	struct mmc_request *mrq;
+	struct mmc_command *cmd = host->cmd;
+	u32 irq_en, status, raw_status;
+	irqreturn_t ret = IRQ_HANDLED;
+
+	if (WARN_ON(!host))
+		return IRQ_NONE;
+
+	mrq = host->mrq;
+
+	if (WARN_ON(!mrq))
+		return IRQ_NONE;
+
+	if (WARN_ON(!cmd))
+		return IRQ_NONE;
+
+	spin_lock(&host->lock);
+	irq_en = readl(host->regs + SD_EMMC_IRQ_EN);
+	raw_status = readl(host->regs + SD_EMMC_STATUS);
+	status = raw_status & irq_en;
+
+	if (!status) {
+		dev_warn(host->dev, "Spurious IRQ! status=0x%08x, irq_en=0x%08x\n",
+			 raw_status, irq_en);
+		ret = IRQ_NONE;
+		goto out;
+	}
+
+	cmd->error = 0;
+	if (status & IRQ_RXD_ERR_MASK) {
+		dev_dbg(host->dev, "Unhandled IRQ: RXD error\n");
+		cmd->error = -EILSEQ;
+	}
+	if (status & IRQ_TXD_ERR) {
+		dev_dbg(host->dev, "Unhandled IRQ: TXD error\n");
+		cmd->error = -EILSEQ;
+	}
+	if (status & IRQ_DESC_ERR)
+		dev_dbg(host->dev, "Unhandled IRQ: Descriptor error\n");
+	if (status & IRQ_RESP_ERR) {
+		dev_dbg(host->dev, "Unhandled IRQ: Response error\n");
+		cmd->error = -EILSEQ;
+	}
+	if (status & IRQ_RESP_TIMEOUT) {
+		dev_dbg(host->dev, "Unhandled IRQ: Response timeout\n");
+		cmd->error = -ETIMEDOUT;
+	}
+	if (status & IRQ_DESC_TIMEOUT) {
+		dev_dbg(host->dev, "Unhandled IRQ: Descriptor timeout\n");
+		cmd->error = -ETIMEDOUT;
+	}
+	if (status & IRQ_SDIO)
+		dev_dbg(host->dev, "Unhandled IRQ: SDIO.\n");
+
+	if (status & (IRQ_END_OF_CHAIN | IRQ_RESP_STATUS))
+		ret = IRQ_WAKE_THREAD;
+	else  {
+		dev_warn(host->dev, "Unknown IRQ! status=0x%04x: MMC CMD%u arg=0x%08x flags=0x%08x stop=%d\n",
+			 status, cmd->opcode, cmd->arg,
+			 cmd->flags, mrq->stop ? 1 : 0);
+		if (cmd->data) {
+			struct mmc_data *data = cmd->data;
+
+			dev_warn(host->dev, "\tblksz %u blocks %u flags 0x%08x (%s%s)",
+				 data->blksz, data->blocks, data->flags,
+				 data->flags & MMC_DATA_WRITE ? "write" : "",
+				 data->flags & MMC_DATA_READ ? "read" : "");
+		}
+	}
+
+out:
+	/* ack all (enabled) interrupts */
+	writel(status, host->regs + SD_EMMC_STATUS);
+
+	if (ret == IRQ_HANDLED) {
+		meson_mmc_read_resp(host->mmc, cmd);
+		meson_mmc_request_done(host->mmc, cmd->mrq);
+	}
+
+	spin_unlock(&host->lock);
+	return ret;
+}
+
+static irqreturn_t meson_mmc_irq_thread(int irq, void *dev_id)
+{
+	struct meson_host *host = dev_id;
+	struct mmc_request *mrq = host->mrq;
+	struct mmc_command *cmd = host->cmd;
+	struct mmc_data *data;
+	unsigned int xfer_bytes;
+	int ret = IRQ_HANDLED;
+
+	if (WARN_ON(!mrq))
+		ret = IRQ_NONE;
+
+	if (WARN_ON(!cmd))
+		ret = IRQ_NONE;
+
+	data = cmd->data;
+	if (data) {
+		xfer_bytes = data->blksz * data->blocks;
+		if (data->flags & MMC_DATA_READ) {
+			WARN_ON(xfer_bytes > host->bounce_buf_size);
+			sg_copy_from_buffer(data->sg, data->sg_len,
+					    host->bounce_buf, xfer_bytes);
+			data->bytes_xfered = xfer_bytes;
+		}
+	}
+
+	meson_mmc_read_resp(host->mmc, cmd);
+	if (!data || !data->stop || mrq->sbc)
+		meson_mmc_request_done(host->mmc, mrq);
+	else
+		meson_mmc_start_cmd(host->mmc, data->stop);
+
+	return ret;
+}
+
+/*
+ * NOTE: we only need this until the GPIO/pinctrl driver can handle
+ * interrupts.  For now, the MMC core will use this for polling.
+ */
+static int meson_mmc_get_cd(struct mmc_host *mmc)
+{
+	int status = mmc_gpio_get_cd(mmc);
+
+	if (status == -ENOSYS)
+		return 1; /* assume present */
+
+	return status;
+}
+
+static const struct mmc_host_ops meson_mmc_ops = {
+	.request	= meson_mmc_request,
+	.set_ios	= meson_mmc_set_ios,
+	.get_cd         = meson_mmc_get_cd,
+};
+
+static int meson_mmc_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	struct meson_host *host;
+	struct mmc_host *mmc;
+	int ret;
+
+	mmc = mmc_alloc_host(sizeof(struct meson_host), &pdev->dev);
+	if (!mmc)
+		return -ENOMEM;
+	host = mmc_priv(mmc);
+	host->mmc = mmc;
+	host->dev = &pdev->dev;
+	dev_set_drvdata(&pdev->dev, host);
+
+	spin_lock_init(&host->lock);
+
+	/* Get regulators and the supported OCR mask */
+	host->vqmmc_enabled = false;
+	ret = mmc_regulator_get_supply(mmc);
+	if (ret == -EPROBE_DEFER)
+		goto free_host;
+
+	ret = mmc_of_parse(mmc);
+	if (ret) {
+		dev_warn(&pdev->dev, "error parsing DT: %d\n", ret);
+		goto free_host;
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	host->regs = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(host->regs)) {
+		ret = PTR_ERR(host->regs);
+		goto free_host;
+	}
+
+	host->irq = platform_get_irq(pdev, 0);
+	if (host->irq == 0) {
+		dev_err(&pdev->dev, "failed to get interrupt resource.\n");
+		ret = -EINVAL;
+		goto free_host;
+	}
+
+	host->core_clk = devm_clk_get(&pdev->dev, "core");
+	if (IS_ERR(host->core_clk)) {
+		ret = PTR_ERR(host->core_clk);
+		goto free_host;
+	}
+
+	ret = clk_prepare_enable(host->core_clk);
+	if (ret)
+		goto free_host;
+
+	ret = meson_mmc_clk_init(host);
+	if (ret)
+		goto free_host;
+
+	/* Stop execution */
+	writel(0, host->regs + SD_EMMC_START);
+
+	/* clear, ack, enable all interrupts */
+	writel(0, host->regs + SD_EMMC_IRQ_EN);
+	writel(IRQ_EN_MASK, host->regs + SD_EMMC_STATUS);
+
+	ret = devm_request_threaded_irq(&pdev->dev, host->irq,
+					meson_mmc_irq, meson_mmc_irq_thread,
+					IRQF_SHARED, DRIVER_NAME, host);
+	if (ret)
+		goto free_host;
+
+	/* data bounce buffer */
+	host->bounce_buf_size = SZ_512K;
+	host->bounce_buf =
+		dma_alloc_coherent(host->dev, host->bounce_buf_size,
+				   &host->bounce_dma_addr, GFP_KERNEL);
+	if (host->bounce_buf == NULL) {
+		dev_err(host->dev, "Unable to map allocate DMA bounce buffer.\n");
+		ret = -ENOMEM;
+		goto free_host;
+	}
+
+	mmc->ops = &meson_mmc_ops;
+	mmc_add_host(mmc);
+
+	return 0;
+
+free_host:
+	clk_disable_unprepare(host->cfg_div_clk);
+	clk_disable_unprepare(host->core_clk);
+	mmc_free_host(mmc);
+	return ret;
+}
+
+static int meson_mmc_remove(struct platform_device *pdev)
+{
+	struct meson_host *host = dev_get_drvdata(&pdev->dev);
+
+	if (WARN_ON(!host))
+		return 0;
+
+	if (host->bounce_buf)
+		dma_free_coherent(host->dev, host->bounce_buf_size,
+				  host->bounce_buf, host->bounce_dma_addr);
+
+	clk_disable_unprepare(host->cfg_div_clk);
+	clk_disable_unprepare(host->core_clk);
+
+	mmc_free_host(host->mmc);
+	return 0;
+}
+
+static const struct of_device_id meson_mmc_of_match[] = {
+	{ .compatible = "amlogic,meson-gx-mmc", },
+	{ .compatible = "amlogic,meson-gxbb-mmc", },
+	{ .compatible = "amlogic,meson-gxl-mmc", },
+	{ .compatible = "amlogic,meson-gxm-mmc", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, meson_mmc_of_match);
+
+static struct platform_driver meson_mmc_driver = {
+	.probe		= meson_mmc_probe,
+	.remove		= meson_mmc_remove,
+	.driver		= {
+		.name = DRIVER_NAME,
+		.of_match_table = of_match_ptr(meson_mmc_of_match),
+	},
+};
+
+module_platform_driver(meson_mmc_driver);
+
+MODULE_DESCRIPTION("Amlogic S905*/GX* SD/eMMC driver");
+MODULE_AUTHOR("Kevin Hilman <khilman@baylibre.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mmc/host/mmci.c b/drivers/mmc/host/mmci.c
index df990bb8c873..01a804792f30 100644
--- a/drivers/mmc/host/mmci.c
+++ b/drivers/mmc/host/mmci.c
@@ -71,7 +71,12 @@ static unsigned int fmax = 515633;
  * @f_max: maximum clk frequency supported by the controller.
  * @signal_direction: input/out direction of bus signals can be indicated
  * @pwrreg_clkgate: MMCIPOWER register must be used to gate the clock
- * @busy_detect: true if busy detection on dat0 is supported
+ * @busy_detect: true if the variant supports busy detection on DAT0.
+ * @busy_dpsm_flag: bitmask enabling busy detection in the DPSM
+ * @busy_detect_flag: bitmask identifying the bit in the MMCISTATUS register
+ *		      indicating that the card is busy
+ * @busy_detect_mask: bitmask identifying the bit in the MMCIMASK0 to mask for
+ *		      getting busy end detection interrupts
  * @pwrreg_nopower: bits in MMCIPOWER don't controls ext. power supply
  * @explicit_mclk_control: enable explicit mclk control in driver.
  * @qcom_fifo: enables qcom specific fifo pio read logic.
@@ -98,6 +103,9 @@ struct variant_data {
 	bool			signal_direction;
 	bool			pwrreg_clkgate;
 	bool			busy_detect;
+	u32			busy_dpsm_flag;
+	u32			busy_detect_flag;
+	u32			busy_detect_mask;
 	bool			pwrreg_nopower;
 	bool			explicit_mclk_control;
 	bool			qcom_fifo;
@@ -137,7 +145,7 @@ static struct variant_data variant_u300 = {
 	.clkreg_enable		= MCI_ST_U300_HWFCEN,
 	.clkreg_8bit_bus_enable = MCI_ST_8BIT_BUS,
 	.datalength_bits	= 16,
-	.datactrl_mask_sdio	= MCI_ST_DPSM_SDIOEN,
+	.datactrl_mask_sdio	= MCI_DPSM_ST_SDIOEN,
 	.st_sdio			= true,
 	.pwrreg_powerup		= MCI_PWR_ON,
 	.f_max			= 100000000,
@@ -152,7 +160,7 @@ static struct variant_data variant_nomadik = {
 	.clkreg			= MCI_CLK_ENABLE,
 	.clkreg_8bit_bus_enable = MCI_ST_8BIT_BUS,
 	.datalength_bits	= 24,
-	.datactrl_mask_sdio	= MCI_ST_DPSM_SDIOEN,
+	.datactrl_mask_sdio	= MCI_DPSM_ST_SDIOEN,
 	.st_sdio		= true,
 	.st_clkdiv		= true,
 	.pwrreg_powerup		= MCI_PWR_ON,
@@ -170,7 +178,7 @@ static struct variant_data variant_ux500 = {
 	.clkreg_8bit_bus_enable = MCI_ST_8BIT_BUS,
 	.clkreg_neg_edge_enable	= MCI_ST_UX500_NEG_EDGE,
 	.datalength_bits	= 24,
-	.datactrl_mask_sdio	= MCI_ST_DPSM_SDIOEN,
+	.datactrl_mask_sdio	= MCI_DPSM_ST_SDIOEN,
 	.st_sdio		= true,
 	.st_clkdiv		= true,
 	.pwrreg_powerup		= MCI_PWR_ON,
@@ -178,6 +186,9 @@ static struct variant_data variant_ux500 = {
 	.signal_direction	= true,
 	.pwrreg_clkgate		= true,
 	.busy_detect		= true,
+	.busy_dpsm_flag		= MCI_DPSM_ST_BUSYMODE,
+	.busy_detect_flag	= MCI_ST_CARDBUSY,
+	.busy_detect_mask	= MCI_ST_BUSYENDMASK,
 	.pwrreg_nopower		= true,
 };
 
@@ -188,9 +199,9 @@ static struct variant_data variant_ux500v2 = {
 	.clkreg_enable		= MCI_ST_UX500_HWFCEN,
 	.clkreg_8bit_bus_enable = MCI_ST_8BIT_BUS,
 	.clkreg_neg_edge_enable	= MCI_ST_UX500_NEG_EDGE,
-	.datactrl_mask_ddrmode	= MCI_ST_DPSM_DDRMODE,
+	.datactrl_mask_ddrmode	= MCI_DPSM_ST_DDRMODE,
 	.datalength_bits	= 24,
-	.datactrl_mask_sdio	= MCI_ST_DPSM_SDIOEN,
+	.datactrl_mask_sdio	= MCI_DPSM_ST_SDIOEN,
 	.st_sdio		= true,
 	.st_clkdiv		= true,
 	.blksz_datactrl16	= true,
@@ -199,6 +210,9 @@ static struct variant_data variant_ux500v2 = {
 	.signal_direction	= true,
 	.pwrreg_clkgate		= true,
 	.busy_detect		= true,
+	.busy_dpsm_flag		= MCI_DPSM_ST_BUSYMODE,
+	.busy_detect_flag	= MCI_ST_CARDBUSY,
+	.busy_detect_mask	= MCI_ST_BUSYENDMASK,
 	.pwrreg_nopower		= true,
 };
 
@@ -210,7 +224,7 @@ static struct variant_data variant_qcom = {
 				  MCI_QCOM_CLK_SELECT_IN_FBCLK,
 	.clkreg_8bit_bus_enable = MCI_QCOM_CLK_WIDEBUS_8,
 	.datactrl_mask_ddrmode	= MCI_QCOM_CLK_SELECT_IN_DDR_MODE,
-	.data_cmd_enable	= MCI_QCOM_CSPM_DATCMD,
+	.data_cmd_enable	= MCI_CPSM_QCOM_DATCMD,
 	.blksz_datactrl4	= true,
 	.datalength_bits	= 24,
 	.pwrreg_powerup		= MCI_PWR_UP,
@@ -220,6 +234,7 @@ static struct variant_data variant_qcom = {
 	.qcom_dml		= true,
 };
 
+/* Busy detection for the ST Micro variant */
 static int mmci_card_busy(struct mmc_host *mmc)
 {
 	struct mmci_host *host = mmc_priv(mmc);
@@ -227,7 +242,7 @@ static int mmci_card_busy(struct mmc_host *mmc)
 	int busy = 0;
 
 	spin_lock_irqsave(&host->lock, flags);
-	if (readl(host->base + MMCISTATUS) & MCI_ST_CARDBUSY)
+	if (readl(host->base + MMCISTATUS) & host->variant->busy_detect_flag)
 		busy = 1;
 	spin_unlock_irqrestore(&host->lock, flags);
 
@@ -294,8 +309,8 @@ static void mmci_write_pwrreg(struct mmci_host *host, u32 pwr)
  */
 static void mmci_write_datactrlreg(struct mmci_host *host, u32 datactrl)
 {
-	/* Keep ST Micro busy mode if enabled */
-	datactrl |= host->datactrl_reg & MCI_ST_DPSM_BUSYMODE;
+	/* Keep busy mode in DPSM if enabled */
+	datactrl |= host->datactrl_reg & host->variant->busy_dpsm_flag;
 
 	if (host->datactrl_reg != datactrl) {
 		host->datactrl_reg = datactrl;
@@ -684,8 +699,7 @@ static void mmci_get_next_data(struct mmci_host *host, struct mmc_data *data)
 	next->dma_chan = NULL;
 }
 
-static void mmci_pre_request(struct mmc_host *mmc, struct mmc_request *mrq,
-			     bool is_first_req)
+static void mmci_pre_request(struct mmc_host *mmc, struct mmc_request *mrq)
 {
 	struct mmci_host *host = mmc_priv(mmc);
 	struct mmc_data *data = mrq->data;
@@ -973,37 +987,66 @@ mmci_cmd_irq(struct mmci_host *host, struct mmc_command *cmd,
 	     unsigned int status)
 {
 	void __iomem *base = host->base;
-	bool sbc, busy_resp;
+	bool sbc;
 
 	if (!cmd)
 		return;
 
 	sbc = (cmd == host->mrq->sbc);
-	busy_resp = host->variant->busy_detect && (cmd->flags & MMC_RSP_BUSY);
 
-	if (!((status|host->busy_status) & (MCI_CMDCRCFAIL|MCI_CMDTIMEOUT|
-		MCI_CMDSENT|MCI_CMDRESPEND)))
+	/*
+	 * We need to be one of these interrupts to be considered worth
+	 * handling. Note that we tag on any latent IRQs postponed
+	 * due to waiting for busy status.
+	 */
+	if (!((status|host->busy_status) &
+	      (MCI_CMDCRCFAIL|MCI_CMDTIMEOUT|MCI_CMDSENT|MCI_CMDRESPEND)))
 		return;
 
-	/* Check if we need to wait for busy completion. */
-	if (host->busy_status && (status & MCI_ST_CARDBUSY))
-		return;
+	/*
+	 * ST Micro variant: handle busy detection.
+	 */
+	if (host->variant->busy_detect) {
+		bool busy_resp = !!(cmd->flags & MMC_RSP_BUSY);
 
-	/* Enable busy completion if needed and supported. */
-	if (!host->busy_status && busy_resp &&
-		!(status & (MCI_CMDCRCFAIL|MCI_CMDTIMEOUT)) &&
-		(readl(base + MMCISTATUS) & MCI_ST_CARDBUSY)) {
-		writel(readl(base + MMCIMASK0) | MCI_ST_BUSYEND,
-			base + MMCIMASK0);
-		host->busy_status = status & (MCI_CMDSENT|MCI_CMDRESPEND);
-		return;
-	}
+		/* We are busy with a command, return */
+		if (host->busy_status &&
+		    (status & host->variant->busy_detect_flag))
+			return;
+
+		/*
+		 * We were not busy, but we now got a busy response on
+		 * something that was not an error, and we double-check
+		 * that the special busy status bit is still set before
+		 * proceeding.
+		 */
+		if (!host->busy_status && busy_resp &&
+		    !(status & (MCI_CMDCRCFAIL|MCI_CMDTIMEOUT)) &&
+		    (readl(base + MMCISTATUS) & host->variant->busy_detect_flag)) {
+			/* Unmask the busy IRQ */
+			writel(readl(base + MMCIMASK0) |
+			       host->variant->busy_detect_mask,
+			       base + MMCIMASK0);
+			/*
+			 * Now cache the last response status code (until
+			 * the busy bit goes low), and return.
+			 */
+			host->busy_status =
+				status & (MCI_CMDSENT|MCI_CMDRESPEND);
+			return;
+		}
 
-	/* At busy completion, mask the IRQ and complete the request. */
-	if (host->busy_status) {
-		writel(readl(base + MMCIMASK0) & ~MCI_ST_BUSYEND,
-			base + MMCIMASK0);
-		host->busy_status = 0;
+		/*
+		 * At this point we are not busy with a command, we have
+		 * not received a new busy request, mask the busy IRQ and
+		 * fall through to process the IRQ.
+		 */
+		if (host->busy_status) {
+			writel(readl(base + MMCIMASK0) &
+			       ~host->variant->busy_detect_mask,
+			       base + MMCIMASK0);
+			host->busy_status = 0;
+		}
 	}
 
 	host->cmd = NULL;
@@ -1257,9 +1300,11 @@ static irqreturn_t mmci_irq(int irq, void *dev_id)
 			mmci_data_irq(host, host->data, status);
 		}
 
-		/* Don't poll for busy completion in irq context. */
-		if (host->busy_status)
-			status &= ~MCI_ST_CARDBUSY;
+		/*
+		 * Don't poll for busy completion in irq context.
+		 */
+		if (host->variant->busy_detect && host->busy_status)
+			status &= ~host->variant->busy_detect_flag;
 
 		ret = 1;
 	} while (status);
@@ -1612,9 +1657,18 @@ static int mmci_probe(struct amba_device *dev,
 	/* We support these capabilities. */
 	mmc->caps |= MMC_CAP_CMD23;
 
+	/*
+	 * Enable busy detection.
+	 */
 	if (variant->busy_detect) {
 		mmci_ops.card_busy = mmci_card_busy;
-		mmci_write_datactrlreg(host, MCI_ST_DPSM_BUSYMODE);
+		/*
+		 * Not all variants have a flag to enable busy detection
+		 * in the DPSM, but if they do, set it here.
+		 */
+		if (variant->busy_dpsm_flag)
+			mmci_write_datactrlreg(host,
+					       host->variant->busy_dpsm_flag);
 		mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY;
 		mmc->max_busy_timeout = 0;
 	}
diff --git a/drivers/mmc/host/mmci.h b/drivers/mmc/host/mmci.h
index a1f5e4f49e2a..56322c6afba4 100644
--- a/drivers/mmc/host/mmci.h
+++ b/drivers/mmc/host/mmci.h
@@ -51,25 +51,27 @@
 #define MCI_QCOM_CLK_SELECT_IN_DDR_MODE	(BIT(14) | BIT(15))
 
 #define MMCIARGUMENT		0x008
-#define MMCICOMMAND		0x00c
-#define MCI_CPSM_RESPONSE	(1 << 6)
-#define MCI_CPSM_LONGRSP	(1 << 7)
-#define MCI_CPSM_INTERRUPT	(1 << 8)
-#define MCI_CPSM_PENDING	(1 << 9)
-#define MCI_CPSM_ENABLE		(1 << 10)
-/* Argument flag extenstions in the ST Micro versions */
-#define MCI_ST_SDIO_SUSP	(1 << 11)
-#define MCI_ST_ENCMD_COMPL	(1 << 12)
-#define MCI_ST_NIEN		(1 << 13)
-#define MCI_ST_CE_ATACMD	(1 << 14)
 
-/* Modified on Qualcomm Integrations */
-#define MCI_QCOM_CSPM_DATCMD		BIT(12)
-#define MCI_QCOM_CSPM_MCIABORT		BIT(13)
-#define MCI_QCOM_CSPM_CCSENABLE		BIT(14)
-#define MCI_QCOM_CSPM_CCSDISABLE	BIT(15)
-#define MCI_QCOM_CSPM_AUTO_CMD19	BIT(16)
-#define MCI_QCOM_CSPM_AUTO_CMD21	BIT(21)
+/* The command register controls the Command Path State Machine (CPSM) */
+#define MMCICOMMAND		0x00c
+#define MCI_CPSM_RESPONSE	BIT(6)
+#define MCI_CPSM_LONGRSP	BIT(7)
+#define MCI_CPSM_INTERRUPT	BIT(8)
+#define MCI_CPSM_PENDING	BIT(9)
+#define MCI_CPSM_ENABLE		BIT(10)
+/* Command register flag extenstions in the ST Micro versions */
+#define MCI_CPSM_ST_SDIO_SUSP		BIT(11)
+#define MCI_CPSM_ST_ENCMD_COMPL		BIT(12)
+#define MCI_CPSM_ST_NIEN		BIT(13)
+#define MCI_CPSM_ST_CE_ATACMD		BIT(14)
+/* Command register flag extensions in the Qualcomm versions */
+#define MCI_CPSM_QCOM_PROGENA		BIT(11)
+#define MCI_CPSM_QCOM_DATCMD		BIT(12)
+#define MCI_CPSM_QCOM_MCIABORT		BIT(13)
+#define MCI_CPSM_QCOM_CCSENABLE		BIT(14)
+#define MCI_CPSM_QCOM_CCSDISABLE	BIT(15)
+#define MCI_CPSM_QCOM_AUTO_CMD19	BIT(16)
+#define MCI_CPSM_QCOM_AUTO_CMD21	BIT(21)
 
 #define MMCIRESPCMD		0x010
 #define MMCIRESPONSE0		0x014
@@ -78,22 +80,27 @@
 #define MMCIRESPONSE3		0x020
 #define MMCIDATATIMER		0x024
 #define MMCIDATALENGTH		0x028
+
+/* The data control register controls the Data Path State Machine (DPSM) */
 #define MMCIDATACTRL		0x02c
-#define MCI_DPSM_ENABLE		(1 << 0)
-#define MCI_DPSM_DIRECTION	(1 << 1)
-#define MCI_DPSM_MODE		(1 << 2)
-#define MCI_DPSM_DMAENABLE	(1 << 3)
-#define MCI_DPSM_BLOCKSIZE	(1 << 4)
+#define MCI_DPSM_ENABLE		BIT(0)
+#define MCI_DPSM_DIRECTION	BIT(1)
+#define MCI_DPSM_MODE		BIT(2)
+#define MCI_DPSM_DMAENABLE	BIT(3)
+#define MCI_DPSM_BLOCKSIZE	BIT(4)
 /* Control register extensions in the ST Micro U300 and Ux500 versions */
-#define MCI_ST_DPSM_RWSTART	(1 << 8)
-#define MCI_ST_DPSM_RWSTOP	(1 << 9)
-#define MCI_ST_DPSM_RWMOD	(1 << 10)
-#define MCI_ST_DPSM_SDIOEN	(1 << 11)
+#define MCI_DPSM_ST_RWSTART	BIT(8)
+#define MCI_DPSM_ST_RWSTOP	BIT(9)
+#define MCI_DPSM_ST_RWMOD	BIT(10)
+#define MCI_DPSM_ST_SDIOEN	BIT(11)
 /* Control register extensions in the ST Micro Ux500 versions */
-#define MCI_ST_DPSM_DMAREQCTL	(1 << 12)
-#define MCI_ST_DPSM_DBOOTMODEEN	(1 << 13)
-#define MCI_ST_DPSM_BUSYMODE	(1 << 14)
-#define MCI_ST_DPSM_DDRMODE	(1 << 15)
+#define MCI_DPSM_ST_DMAREQCTL	BIT(12)
+#define MCI_DPSM_ST_DBOOTMODEEN	BIT(13)
+#define MCI_DPSM_ST_BUSYMODE	BIT(14)
+#define MCI_DPSM_ST_DDRMODE	BIT(15)
+/* Control register extensions in the Qualcomm versions */
+#define MCI_DPSM_QCOM_DATA_PEND	BIT(17)
+#define MCI_DPSM_QCOM_RX_DATA_PEND BIT(20)
 
 #define MMCIDATACNT		0x030
 #define MMCISTATUS		0x034
@@ -167,7 +174,7 @@
 /* Extended status bits for the ST Micro variants */
 #define MCI_ST_SDIOITMASK	(1 << 22)
 #define MCI_ST_CEATAENDMASK	(1 << 23)
-#define MCI_ST_BUSYEND		(1 << 24)
+#define MCI_ST_BUSYENDMASK	(1 << 24)
 
 #define MMCIMASK1		0x040
 #define MMCIFIFOCNT		0x048
diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c
index 84e9afcb5c09..10ef2ae1d2f6 100644
--- a/drivers/mmc/host/mtk-sd.c
+++ b/drivers/mmc/host/mtk-sd.c
@@ -927,8 +927,7 @@ static void msdc_ops_request(struct mmc_host *mmc, struct mmc_request *mrq)
 		msdc_start_command(host, mrq, mrq->cmd);
 }
 
-static void msdc_pre_req(struct mmc_host *mmc, struct mmc_request *mrq,
-		bool is_first_req)
+static void msdc_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
 {
 	struct msdc_host *host = mmc_priv(mmc);
 	struct mmc_data *data = mrq->data;
@@ -1713,6 +1712,7 @@ static const struct of_device_id msdc_of_ids[] = {
 	{   .compatible = "mediatek,mt8135-mmc", },
 	{}
 };
+MODULE_DEVICE_TABLE(of, msdc_of_ids);
 
 static struct platform_driver mt_msdc_driver = {
 	.probe = msdc_drv_probe,
diff --git a/drivers/mmc/host/omap_hsmmc.c b/drivers/mmc/host/omap_hsmmc.c
index 5f2f24a7360d..ad11c4cc12ed 100644
--- a/drivers/mmc/host/omap_hsmmc.c
+++ b/drivers/mmc/host/omap_hsmmc.c
@@ -1565,8 +1565,7 @@ static void omap_hsmmc_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
 	}
 }
 
-static void omap_hsmmc_pre_req(struct mmc_host *mmc, struct mmc_request *mrq,
-			       bool is_first_req)
+static void omap_hsmmc_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
 {
 	struct omap_hsmmc_host *host = mmc_priv(mmc);
 
diff --git a/drivers/mmc/host/rtsx_pci_sdmmc.c b/drivers/mmc/host/rtsx_pci_sdmmc.c
index 3ccaa1415f33..ecb99a8d2fa2 100644
--- a/drivers/mmc/host/rtsx_pci_sdmmc.c
+++ b/drivers/mmc/host/rtsx_pci_sdmmc.c
@@ -190,8 +190,7 @@ static int sd_pre_dma_transfer(struct realtek_pci_sdmmc *host,
 	return using_cookie;
 }
 
-static void sdmmc_pre_req(struct mmc_host *mmc, struct mmc_request *mrq,
-		bool is_first_req)
+static void sdmmc_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
 {
 	struct realtek_pci_sdmmc *host = mmc_priv(mmc);
 	struct mmc_data *data = mrq->data;
diff --git a/drivers/mmc/host/rtsx_usb_sdmmc.c b/drivers/mmc/host/rtsx_usb_sdmmc.c
index 6e9c0f8fddb1..dc1abd14acbc 100644
--- a/drivers/mmc/host/rtsx_usb_sdmmc.c
+++ b/drivers/mmc/host/rtsx_usb_sdmmc.c
@@ -1374,6 +1374,8 @@ static int rtsx_usb_sdmmc_drv_probe(struct platform_device *pdev)
 
 	mutex_init(&host->host_mutex);
 	rtsx_usb_init_host(host);
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_runtime_set_autosuspend_delay(&pdev->dev, 50);
 	pm_runtime_enable(&pdev->dev);
 
 #ifdef RTSX_USB_USE_LEDS_CLASS
@@ -1428,6 +1430,7 @@ static int rtsx_usb_sdmmc_drv_remove(struct platform_device *pdev)
 
 	mmc_free_host(mmc);
 	pm_runtime_disable(&pdev->dev);
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	platform_set_drvdata(pdev, NULL);
 
 	dev_dbg(&(pdev->dev),
diff --git a/drivers/mmc/host/s3cmci.c b/drivers/mmc/host/s3cmci.c
index c531deef3258..932a4b1fed33 100644
--- a/drivers/mmc/host/s3cmci.c
+++ b/drivers/mmc/host/s3cmci.c
@@ -28,7 +28,6 @@
 #include <mach/dma.h>
 #include <mach/gpio-samsung.h>
 
-#include <linux/platform_data/dma-s3c24xx.h>
 #include <linux/platform_data/mmc-s3cmci.h>
 
 #include "s3cmci.h"
@@ -1682,19 +1681,13 @@ static int s3cmci_probe(struct platform_device *pdev)
 		gpio_direction_input(host->pdata->gpio_wprotect);
 	}
 
-	/* depending on the dma state, get a dma channel to use. */
+	/* Depending on the dma state, get a DMA channel to use. */
 
 	if (s3cmci_host_usedma(host)) {
-		dma_cap_mask_t mask;
-
-		dma_cap_zero(mask);
-		dma_cap_set(DMA_SLAVE, mask);
-
-		host->dma = dma_request_slave_channel_compat(mask,
-			s3c24xx_dma_filter, (void *)DMACH_SDI, &pdev->dev, "rx-tx");
-		if (!host->dma) {
+		host->dma = dma_request_chan(&pdev->dev, "rx-tx");
+		ret = PTR_ERR_OR_ZERO(host->dma);
+		if (ret) {
 			dev_err(&pdev->dev, "cannot get DMA channel.\n");
-			ret = -EBUSY;
 			goto probe_free_gpio_wp;
 		}
 	}
diff --git a/drivers/mmc/host/sdhci-acpi.c b/drivers/mmc/host/sdhci-acpi.c
index 81d4dc034793..160f695cc09c 100644
--- a/drivers/mmc/host/sdhci-acpi.c
+++ b/drivers/mmc/host/sdhci-acpi.c
@@ -328,6 +328,7 @@ static const struct sdhci_acpi_uid_slot sdhci_acpi_uids[] = {
 	{ "80865ACC", NULL, &sdhci_acpi_slot_int_emmc },
 	{ "80865AD0", NULL, &sdhci_acpi_slot_int_sdio },
 	{ "80860F14" , "1" , &sdhci_acpi_slot_int_emmc },
+	{ "80860F14" , "2" , &sdhci_acpi_slot_int_sdio },
 	{ "80860F14" , "3" , &sdhci_acpi_slot_int_sd   },
 	{ "80860F16" , NULL, &sdhci_acpi_slot_int_sd   },
 	{ "INT33BB"  , "2" , &sdhci_acpi_slot_int_sdio },
diff --git a/drivers/mmc/host/sdhci-cadence.c b/drivers/mmc/host/sdhci-cadence.c
new file mode 100644
index 000000000000..1501cfdac473
--- /dev/null
+++ b/drivers/mmc/host/sdhci-cadence.c
@@ -0,0 +1,283 @@
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/bitops.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/mmc/host.h>
+
+#include "sdhci-pltfm.h"
+
+/* HRS - Host Register Set (specific to Cadence) */
+#define SDHCI_CDNS_HRS04		0x10		/* PHY access port */
+#define   SDHCI_CDNS_HRS04_ACK			BIT(26)
+#define   SDHCI_CDNS_HRS04_RD			BIT(25)
+#define   SDHCI_CDNS_HRS04_WR			BIT(24)
+#define   SDHCI_CDNS_HRS04_RDATA_SHIFT		12
+#define   SDHCI_CDNS_HRS04_WDATA_SHIFT		8
+#define   SDHCI_CDNS_HRS04_ADDR_SHIFT		0
+
+#define SDHCI_CDNS_HRS06		0x18		/* eMMC control */
+#define   SDHCI_CDNS_HRS06_TUNE_UP		BIT(15)
+#define   SDHCI_CDNS_HRS06_TUNE_SHIFT		8
+#define   SDHCI_CDNS_HRS06_TUNE_MASK		0x3f
+#define   SDHCI_CDNS_HRS06_MODE_MASK		0x7
+#define   SDHCI_CDNS_HRS06_MODE_SD		0x0
+#define   SDHCI_CDNS_HRS06_MODE_MMC_SDR		0x2
+#define   SDHCI_CDNS_HRS06_MODE_MMC_DDR		0x3
+#define   SDHCI_CDNS_HRS06_MODE_MMC_HS200	0x4
+#define   SDHCI_CDNS_HRS06_MODE_MMC_HS400	0x5
+
+/* SRS - Slot Register Set (SDHCI-compatible) */
+#define SDHCI_CDNS_SRS_BASE		0x200
+
+/* PHY */
+#define SDHCI_CDNS_PHY_DLY_SD_HS	0x00
+#define SDHCI_CDNS_PHY_DLY_SD_DEFAULT	0x01
+#define SDHCI_CDNS_PHY_DLY_UHS_SDR12	0x02
+#define SDHCI_CDNS_PHY_DLY_UHS_SDR25	0x03
+#define SDHCI_CDNS_PHY_DLY_UHS_SDR50	0x04
+#define SDHCI_CDNS_PHY_DLY_UHS_DDR50	0x05
+#define SDHCI_CDNS_PHY_DLY_EMMC_LEGACY	0x06
+#define SDHCI_CDNS_PHY_DLY_EMMC_SDR	0x07
+#define SDHCI_CDNS_PHY_DLY_EMMC_DDR	0x08
+
+/*
+ * The tuned val register is 6 bit-wide, but not the whole of the range is
+ * available.  The range 0-42 seems to be available (then 43 wraps around to 0)
+ * but I am not quite sure if it is official.  Use only 0 to 39 for safety.
+ */
+#define SDHCI_CDNS_MAX_TUNING_LOOP	40
+
+struct sdhci_cdns_priv {
+	void __iomem *hrs_addr;
+};
+
+static void sdhci_cdns_write_phy_reg(struct sdhci_cdns_priv *priv,
+				     u8 addr, u8 data)
+{
+	void __iomem *reg = priv->hrs_addr + SDHCI_CDNS_HRS04;
+	u32 tmp;
+
+	tmp = (data << SDHCI_CDNS_HRS04_WDATA_SHIFT) |
+	      (addr << SDHCI_CDNS_HRS04_ADDR_SHIFT);
+	writel(tmp, reg);
+
+	tmp |= SDHCI_CDNS_HRS04_WR;
+	writel(tmp, reg);
+
+	tmp &= ~SDHCI_CDNS_HRS04_WR;
+	writel(tmp, reg);
+}
+
+static void sdhci_cdns_phy_init(struct sdhci_cdns_priv *priv)
+{
+	sdhci_cdns_write_phy_reg(priv, SDHCI_CDNS_PHY_DLY_SD_HS, 4);
+	sdhci_cdns_write_phy_reg(priv, SDHCI_CDNS_PHY_DLY_SD_DEFAULT, 4);
+	sdhci_cdns_write_phy_reg(priv, SDHCI_CDNS_PHY_DLY_EMMC_LEGACY, 9);
+	sdhci_cdns_write_phy_reg(priv, SDHCI_CDNS_PHY_DLY_EMMC_SDR, 2);
+	sdhci_cdns_write_phy_reg(priv, SDHCI_CDNS_PHY_DLY_EMMC_DDR, 3);
+}
+
+static inline void *sdhci_cdns_priv(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+
+	return sdhci_pltfm_priv(pltfm_host);
+}
+
+static unsigned int sdhci_cdns_get_timeout_clock(struct sdhci_host *host)
+{
+	/*
+	 * Cadence's spec says the Timeout Clock Frequency is the same as the
+	 * Base Clock Frequency.  Divide it by 1000 to return a value in kHz.
+	 */
+	return host->max_clk / 1000;
+}
+
+static void sdhci_cdns_set_uhs_signaling(struct sdhci_host *host,
+					 unsigned int timing)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	u32 mode, tmp;
+
+	switch (timing) {
+	case MMC_TIMING_MMC_HS:
+		mode = SDHCI_CDNS_HRS06_MODE_MMC_SDR;
+		break;
+	case MMC_TIMING_MMC_DDR52:
+		mode = SDHCI_CDNS_HRS06_MODE_MMC_DDR;
+		break;
+	case MMC_TIMING_MMC_HS200:
+		mode = SDHCI_CDNS_HRS06_MODE_MMC_HS200;
+		break;
+	case MMC_TIMING_MMC_HS400:
+		mode = SDHCI_CDNS_HRS06_MODE_MMC_HS400;
+		break;
+	default:
+		mode = SDHCI_CDNS_HRS06_MODE_SD;
+		break;
+	}
+
+	/* The speed mode for eMMC is selected by HRS06 register */
+	tmp = readl(priv->hrs_addr + SDHCI_CDNS_HRS06);
+	tmp &= ~SDHCI_CDNS_HRS06_MODE_MASK;
+	tmp |= mode;
+	writel(tmp, priv->hrs_addr + SDHCI_CDNS_HRS06);
+
+	/* For SD, fall back to the default handler */
+	if (mode == SDHCI_CDNS_HRS06_MODE_SD)
+		sdhci_set_uhs_signaling(host, timing);
+}
+
+static const struct sdhci_ops sdhci_cdns_ops = {
+	.set_clock = sdhci_set_clock,
+	.get_timeout_clock = sdhci_cdns_get_timeout_clock,
+	.set_bus_width = sdhci_set_bus_width,
+	.reset = sdhci_reset,
+	.set_uhs_signaling = sdhci_cdns_set_uhs_signaling,
+};
+
+static const struct sdhci_pltfm_data sdhci_cdns_pltfm_data = {
+	.ops = &sdhci_cdns_ops,
+};
+
+static int sdhci_cdns_set_tune_val(struct sdhci_host *host, unsigned int val)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	void __iomem *reg = priv->hrs_addr + SDHCI_CDNS_HRS06;
+	u32 tmp;
+
+	if (WARN_ON(val > SDHCI_CDNS_HRS06_TUNE_MASK))
+		return -EINVAL;
+
+	tmp = readl(reg);
+	tmp &= ~(SDHCI_CDNS_HRS06_TUNE_MASK << SDHCI_CDNS_HRS06_TUNE_SHIFT);
+	tmp |= val << SDHCI_CDNS_HRS06_TUNE_SHIFT;
+	tmp |= SDHCI_CDNS_HRS06_TUNE_UP;
+	writel(tmp, reg);
+
+	return readl_poll_timeout(reg, tmp, !(tmp & SDHCI_CDNS_HRS06_TUNE_UP),
+				  0, 1);
+}
+
+static int sdhci_cdns_execute_tuning(struct mmc_host *mmc, u32 opcode)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	int cur_streak = 0;
+	int max_streak = 0;
+	int end_of_streak = 0;
+	int i;
+
+	/*
+	 * This handler only implements the eMMC tuning that is specific to
+	 * this controller.  Fall back to the standard method for SD timing.
+	 */
+	if (host->timing != MMC_TIMING_MMC_HS200)
+		return sdhci_execute_tuning(mmc, opcode);
+
+	if (WARN_ON(opcode != MMC_SEND_TUNING_BLOCK_HS200))
+		return -EINVAL;
+
+	for (i = 0; i < SDHCI_CDNS_MAX_TUNING_LOOP; i++) {
+		if (sdhci_cdns_set_tune_val(host, i) ||
+		    mmc_send_tuning(host->mmc, opcode, NULL)) { /* bad */
+			cur_streak = 0;
+		} else { /* good */
+			cur_streak++;
+			if (cur_streak > max_streak) {
+				max_streak = cur_streak;
+				end_of_streak = i;
+			}
+		}
+	}
+
+	if (!max_streak) {
+		dev_err(mmc_dev(host->mmc), "no tuning point found\n");
+		return -EIO;
+	}
+
+	return sdhci_cdns_set_tune_val(host, end_of_streak - max_streak / 2);
+}
+
+static int sdhci_cdns_probe(struct platform_device *pdev)
+{
+	struct sdhci_host *host;
+	struct sdhci_pltfm_host *pltfm_host;
+	struct sdhci_cdns_priv *priv;
+	struct clk *clk;
+	int ret;
+
+	clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	ret = clk_prepare_enable(clk);
+	if (ret)
+		return ret;
+
+	host = sdhci_pltfm_init(pdev, &sdhci_cdns_pltfm_data, sizeof(*priv));
+	if (IS_ERR(host)) {
+		ret = PTR_ERR(host);
+		goto disable_clk;
+	}
+
+	pltfm_host = sdhci_priv(host);
+	pltfm_host->clk = clk;
+
+	priv = sdhci_cdns_priv(host);
+	priv->hrs_addr = host->ioaddr;
+	host->ioaddr += SDHCI_CDNS_SRS_BASE;
+	host->mmc_host_ops.execute_tuning = sdhci_cdns_execute_tuning;
+
+	ret = mmc_of_parse(host->mmc);
+	if (ret)
+		goto free;
+
+	sdhci_cdns_phy_init(priv);
+
+	ret = sdhci_add_host(host);
+	if (ret)
+		goto free;
+
+	return 0;
+free:
+	sdhci_pltfm_free(pdev);
+disable_clk:
+	clk_disable_unprepare(clk);
+
+	return ret;
+}
+
+static const struct of_device_id sdhci_cdns_match[] = {
+	{ .compatible = "cdns,sd4hc" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, sdhci_cdns_match);
+
+static struct platform_driver sdhci_cdns_driver = {
+	.driver = {
+		.name = "sdhci-cdns",
+		.pm = &sdhci_pltfm_pmops,
+		.of_match_table = sdhci_cdns_match,
+	},
+	.probe = sdhci_cdns_probe,
+	.remove = sdhci_pltfm_unregister,
+};
+module_platform_driver(sdhci_cdns_driver);
+
+MODULE_AUTHOR("Masahiro Yamada <yamada.masahiro@socionext.com>");
+MODULE_DESCRIPTION("Cadence SD/SDIO/eMMC Host Controller Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/mmc/host/sdhci-iproc.c b/drivers/mmc/host/sdhci-iproc.c
index 726246665850..d7046d67415a 100644
--- a/drivers/mmc/host/sdhci-iproc.c
+++ b/drivers/mmc/host/sdhci-iproc.c
@@ -143,6 +143,14 @@ static void sdhci_iproc_writeb(struct sdhci_host *host, u8 val, int reg)
 }
 
 static const struct sdhci_ops sdhci_iproc_ops = {
+	.set_clock = sdhci_set_clock,
+	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
+	.set_bus_width = sdhci_set_bus_width,
+	.reset = sdhci_reset,
+	.set_uhs_signaling = sdhci_set_uhs_signaling,
+};
+
+static const struct sdhci_ops sdhci_iproc_32only_ops = {
 	.read_l = sdhci_iproc_readl,
 	.read_w = sdhci_iproc_readw,
 	.read_b = sdhci_iproc_readb,
@@ -156,6 +164,28 @@ static const struct sdhci_ops sdhci_iproc_ops = {
 	.set_uhs_signaling = sdhci_set_uhs_signaling,
 };
 
+static const struct sdhci_pltfm_data sdhci_iproc_cygnus_pltfm_data = {
+	.quirks = SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK,
+	.quirks2 = SDHCI_QUIRK2_ACMD23_BROKEN,
+	.ops = &sdhci_iproc_32only_ops,
+};
+
+static const struct sdhci_iproc_data iproc_cygnus_data = {
+	.pdata = &sdhci_iproc_cygnus_pltfm_data,
+	.caps = ((0x1 << SDHCI_MAX_BLOCK_SHIFT)
+			& SDHCI_MAX_BLOCK_MASK) |
+		SDHCI_CAN_VDD_330 |
+		SDHCI_CAN_VDD_180 |
+		SDHCI_CAN_DO_SUSPEND |
+		SDHCI_CAN_DO_HISPD |
+		SDHCI_CAN_DO_ADMA2 |
+		SDHCI_CAN_DO_SDMA,
+	.caps1 = SDHCI_DRIVER_TYPE_C |
+		 SDHCI_DRIVER_TYPE_D |
+		 SDHCI_SUPPORT_DDR50,
+	.mmc_caps = MMC_CAP_1_8V_DDR,
+};
+
 static const struct sdhci_pltfm_data sdhci_iproc_pltfm_data = {
 	.quirks = SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK,
 	.quirks2 = SDHCI_QUIRK2_ACMD23_BROKEN,
@@ -182,7 +212,7 @@ static const struct sdhci_pltfm_data sdhci_bcm2835_pltfm_data = {
 	.quirks = SDHCI_QUIRK_BROKEN_CARD_DETECTION |
 		  SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
 		  SDHCI_QUIRK_MISSING_CAPS,
-	.ops = &sdhci_iproc_ops,
+	.ops = &sdhci_iproc_32only_ops,
 };
 
 static const struct sdhci_iproc_data bcm2835_data = {
@@ -194,7 +224,8 @@ static const struct sdhci_iproc_data bcm2835_data = {
 
 static const struct of_device_id sdhci_iproc_of_match[] = {
 	{ .compatible = "brcm,bcm2835-sdhci", .data = &bcm2835_data },
-	{ .compatible = "brcm,sdhci-iproc-cygnus", .data = &iproc_data },
+	{ .compatible = "brcm,sdhci-iproc-cygnus", .data = &iproc_cygnus_data},
+	{ .compatible = "brcm,sdhci-iproc", .data = &iproc_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, sdhci_iproc_of_match);
diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c
index 90ed2e12d345..32879b845b75 100644
--- a/drivers/mmc/host/sdhci-msm.c
+++ b/drivers/mmc/host/sdhci-msm.c
@@ -18,7 +18,9 @@
 #include <linux/of_device.h>
 #include <linux/delay.h>
 #include <linux/mmc/mmc.h>
+#include <linux/pm_runtime.h>
 #include <linux/slab.h>
+#include <linux/iopoll.h>
 
 #include "sdhci-pltfm.h"
 
@@ -31,6 +33,7 @@
 #define HC_MODE_EN		0x1
 #define CORE_POWER		0x0
 #define CORE_SW_RST		BIT(7)
+#define FF_CLK_SW_RST_DIS	BIT(13)
 
 #define CORE_PWRCTL_STATUS	0xdc
 #define CORE_PWRCTL_MASK	0xe0
@@ -49,6 +52,7 @@
 #define INT_MASK		0xf
 #define MAX_PHASES		16
 #define CORE_DLL_LOCK		BIT(7)
+#define CORE_DDR_DLL_LOCK	BIT(11)
 #define CORE_DLL_EN		BIT(16)
 #define CORE_CDR_EN		BIT(17)
 #define CORE_CK_OUT_EN		BIT(18)
@@ -56,18 +60,67 @@
 #define CORE_DLL_PDN		BIT(29)
 #define CORE_DLL_RST		BIT(30)
 #define CORE_DLL_CONFIG		0x100
+#define CORE_CMD_DAT_TRACK_SEL	BIT(0)
 #define CORE_DLL_STATUS		0x108
 
+#define CORE_DLL_CONFIG_2	0x1b4
+#define CORE_DDR_CAL_EN		BIT(0)
+#define CORE_FLL_CYCLE_CNT	BIT(18)
+#define CORE_DLL_CLOCK_DISABLE	BIT(21)
+
 #define CORE_VENDOR_SPEC	0x10c
 #define CORE_CLK_PWRSAVE	BIT(1)
+#define CORE_HC_MCLK_SEL_DFLT	(2 << 8)
+#define CORE_HC_MCLK_SEL_HS400	(3 << 8)
+#define CORE_HC_MCLK_SEL_MASK	(3 << 8)
+#define CORE_HC_SELECT_IN_EN	BIT(18)
+#define CORE_HC_SELECT_IN_HS400	(6 << 19)
+#define CORE_HC_SELECT_IN_MASK	(7 << 19)
+
+#define CORE_CSR_CDC_CTLR_CFG0		0x130
+#define CORE_SW_TRIG_FULL_CALIB		BIT(16)
+#define CORE_HW_AUTOCAL_ENA		BIT(17)
+
+#define CORE_CSR_CDC_CTLR_CFG1		0x134
+#define CORE_CSR_CDC_CAL_TIMER_CFG0	0x138
+#define CORE_TIMER_ENA			BIT(16)
+
+#define CORE_CSR_CDC_CAL_TIMER_CFG1	0x13C
+#define CORE_CSR_CDC_REFCOUNT_CFG	0x140
+#define CORE_CSR_CDC_COARSE_CAL_CFG	0x144
+#define CORE_CDC_OFFSET_CFG		0x14C
+#define CORE_CSR_CDC_DELAY_CFG		0x150
+#define CORE_CDC_SLAVE_DDA_CFG		0x160
+#define CORE_CSR_CDC_STATUS0		0x164
+#define CORE_CALIBRATION_DONE		BIT(0)
+
+#define CORE_CDC_ERROR_CODE_MASK	0x7000000
+
+#define CORE_CSR_CDC_GEN_CFG		0x178
+#define CORE_CDC_SWITCH_BYPASS_OFF	BIT(0)
+#define CORE_CDC_SWITCH_RC_EN		BIT(1)
+
+#define CORE_DDR_200_CFG		0x184
+#define CORE_CDC_T4_DLY_SEL		BIT(0)
+#define CORE_START_CDC_TRAFFIC		BIT(6)
+#define CORE_VENDOR_SPEC3	0x1b0
+#define CORE_PWRSAVE_DLL	BIT(3)
+
+#define CORE_DDR_CONFIG		0x1b8
+#define DDR_CONFIG_POR_VAL	0x80040853
 
 #define CORE_VENDOR_SPEC_CAPABILITIES0	0x11c
 
+#define INVALID_TUNING_PHASE	-1
+#define SDHCI_MSM_MIN_CLOCK	400000
+#define CORE_FREQ_100MHZ	(100 * 1000 * 1000)
+
 #define CDR_SELEXT_SHIFT	20
 #define CDR_SELEXT_MASK		(0xf << CDR_SELEXT_SHIFT)
 #define CMUX_SHIFT_PHASE_SHIFT	24
 #define CMUX_SHIFT_PHASE_MASK	(7 << CMUX_SHIFT_PHASE_SHIFT)
 
+#define MSM_MMC_AUTOSUSPEND_DELAY_MS	50
 struct sdhci_msm_host {
 	struct platform_device *pdev;
 	void __iomem *core_mem;	/* MSM SDCC mapped address */
@@ -75,7 +128,14 @@ struct sdhci_msm_host {
 	struct clk *clk;	/* main SD/MMC bus clock */
 	struct clk *pclk;	/* SDHC peripheral bus clock */
 	struct clk *bus_clk;	/* SDHC bus voter clock */
+	struct clk *xo_clk;	/* TCXO clk needed for FLL feature of cm_dll*/
+	unsigned long clk_rate;
 	struct mmc_host *mmc;
+	bool use_14lpp_dll_reset;
+	bool tuning_done;
+	bool calibration_done;
+	u8 saved_tuning_phase;
+	bool use_cdclp533;
 };
 
 /* Platform specific tuning */
@@ -115,6 +175,9 @@ static int msm_config_cm_dll_phase(struct sdhci_host *host, u8 phase)
 	u32 config;
 	struct mmc_host *mmc = host->mmc;
 
+	if (phase > 0xf)
+		return -EINVAL;
+
 	spin_lock_irqsave(&host->lock, flags);
 
 	config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
@@ -136,9 +199,9 @@ static int msm_config_cm_dll_phase(struct sdhci_host *host, u8 phase)
 	config |= grey_coded_phase_table[phase] << CDR_SELEXT_SHIFT;
 	writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
 
-	/* Set CK_OUT_EN bit of DLL_CONFIG register to 1. */
-	writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG)
-			| CORE_CK_OUT_EN), host->ioaddr + CORE_DLL_CONFIG);
+	config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
+	config |= CORE_CK_OUT_EN;
+	writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
 
 	/* Wait until CK_OUT_EN bit of DLL_CONFIG register becomes '1' */
 	rc = msm_dll_poll_ck_out_en(host, 1);
@@ -163,8 +226,8 @@ out:
  * Find out the greatest range of consecuitive selected
  * DLL clock output phases that can be used as sampling
  * setting for SD3.0 UHS-I card read operation (in SDR104
- * timing mode) or for eMMC4.5 card read operation (in HS200
- * timing mode).
+ * timing mode) or for eMMC4.5 card read operation (in
+ * HS400/HS200 timing mode).
  * Select the 3/4 of the range and configure the DLL with the
  * selected DLL clock output phase.
  */
@@ -303,8 +366,11 @@ static inline void msm_cm_dll_set_freq(struct sdhci_host *host)
 static int msm_init_cm_dll(struct sdhci_host *host)
 {
 	struct mmc_host *mmc = host->mmc;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
 	int wait_cnt = 50;
 	unsigned long flags;
+	u32 config;
 
 	spin_lock_irqsave(&host->lock, flags);
 
@@ -313,33 +379,73 @@ static int msm_init_cm_dll(struct sdhci_host *host)
 	 * tuning is in progress. Keeping PWRSAVE ON may
 	 * turn off the clock.
 	 */
-	writel_relaxed((readl_relaxed(host->ioaddr + CORE_VENDOR_SPEC)
-			& ~CORE_CLK_PWRSAVE), host->ioaddr + CORE_VENDOR_SPEC);
+	config = readl_relaxed(host->ioaddr + CORE_VENDOR_SPEC);
+	config &= ~CORE_CLK_PWRSAVE;
+	writel_relaxed(config, host->ioaddr + CORE_VENDOR_SPEC);
+
+	if (msm_host->use_14lpp_dll_reset) {
+		config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
+		config &= ~CORE_CK_OUT_EN;
+		writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
+
+		config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG_2);
+		config |= CORE_DLL_CLOCK_DISABLE;
+		writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG_2);
+	}
 
-	/* Write 1 to DLL_RST bit of DLL_CONFIG register */
-	writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG)
-			| CORE_DLL_RST), host->ioaddr + CORE_DLL_CONFIG);
+	config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
+	config |= CORE_DLL_RST;
+	writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
 
-	/* Write 1 to DLL_PDN bit of DLL_CONFIG register */
-	writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG)
-			| CORE_DLL_PDN), host->ioaddr + CORE_DLL_CONFIG);
+	config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
+	config |= CORE_DLL_PDN;
+	writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
 	msm_cm_dll_set_freq(host);
 
-	/* Write 0 to DLL_RST bit of DLL_CONFIG register */
-	writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG)
-			& ~CORE_DLL_RST), host->ioaddr + CORE_DLL_CONFIG);
+	if (msm_host->use_14lpp_dll_reset &&
+	    !IS_ERR_OR_NULL(msm_host->xo_clk)) {
+		u32 mclk_freq = 0;
+
+		config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG_2);
+		config &= CORE_FLL_CYCLE_CNT;
+		if (config)
+			mclk_freq = DIV_ROUND_CLOSEST_ULL((host->clock * 8),
+					clk_get_rate(msm_host->xo_clk));
+		else
+			mclk_freq = DIV_ROUND_CLOSEST_ULL((host->clock * 4),
+					clk_get_rate(msm_host->xo_clk));
+
+		config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG_2);
+		config &= ~(0xFF << 10);
+		config |= mclk_freq << 10;
 
-	/* Write 0 to DLL_PDN bit of DLL_CONFIG register */
-	writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG)
-			& ~CORE_DLL_PDN), host->ioaddr + CORE_DLL_CONFIG);
+		writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG_2);
+		/* wait for 5us before enabling DLL clock */
+		udelay(5);
+	}
+
+	config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
+	config &= ~CORE_DLL_RST;
+	writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
+
+	config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
+	config &= ~CORE_DLL_PDN;
+	writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
+
+	if (msm_host->use_14lpp_dll_reset) {
+		msm_cm_dll_set_freq(host);
+		config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG_2);
+		config &= ~CORE_DLL_CLOCK_DISABLE;
+		writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG_2);
+	}
 
-	/* Set DLL_EN bit to 1. */
-	writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG)
-			| CORE_DLL_EN), host->ioaddr + CORE_DLL_CONFIG);
+	config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
+	config |= CORE_DLL_EN;
+	writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
 
-	/* Set CK_OUT_EN bit to 1. */
-	writel_relaxed((readl_relaxed(host->ioaddr + CORE_DLL_CONFIG)
-			| CORE_CK_OUT_EN), host->ioaddr + CORE_DLL_CONFIG);
+	config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
+	config |= CORE_CK_OUT_EN;
+	writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
 
 	/* Wait until DLL_LOCK bit of DLL_STATUS register becomes '1' */
 	while (!(readl_relaxed(host->ioaddr + CORE_DLL_STATUS) &
@@ -358,6 +464,200 @@ static int msm_init_cm_dll(struct sdhci_host *host)
 	return 0;
 }
 
+static int sdhci_msm_cdclp533_calibration(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
+	u32 config, calib_done;
+	int ret;
+
+	pr_debug("%s: %s: Enter\n", mmc_hostname(host->mmc), __func__);
+
+	/*
+	 * Retuning in HS400 (DDR mode) will fail, just reset the
+	 * tuning block and restore the saved tuning phase.
+	 */
+	ret = msm_init_cm_dll(host);
+	if (ret)
+		goto out;
+
+	/* Set the selected phase in delay line hw block */
+	ret = msm_config_cm_dll_phase(host, msm_host->saved_tuning_phase);
+	if (ret)
+		goto out;
+
+	config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
+	config |= CORE_CMD_DAT_TRACK_SEL;
+	writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
+
+	config = readl_relaxed(host->ioaddr + CORE_DDR_200_CFG);
+	config &= ~CORE_CDC_T4_DLY_SEL;
+	writel_relaxed(config, host->ioaddr + CORE_DDR_200_CFG);
+
+	config = readl_relaxed(host->ioaddr + CORE_CSR_CDC_GEN_CFG);
+	config &= ~CORE_CDC_SWITCH_BYPASS_OFF;
+	writel_relaxed(config, host->ioaddr + CORE_CSR_CDC_GEN_CFG);
+
+	config = readl_relaxed(host->ioaddr + CORE_CSR_CDC_GEN_CFG);
+	config |= CORE_CDC_SWITCH_RC_EN;
+	writel_relaxed(config, host->ioaddr + CORE_CSR_CDC_GEN_CFG);
+
+	config = readl_relaxed(host->ioaddr + CORE_DDR_200_CFG);
+	config &= ~CORE_START_CDC_TRAFFIC;
+	writel_relaxed(config, host->ioaddr + CORE_DDR_200_CFG);
+
+	/*
+	 * Perform CDC Register Initialization Sequence
+	 *
+	 * CORE_CSR_CDC_CTLR_CFG0	0x11800EC
+	 * CORE_CSR_CDC_CTLR_CFG1	0x3011111
+	 * CORE_CSR_CDC_CAL_TIMER_CFG0	0x1201000
+	 * CORE_CSR_CDC_CAL_TIMER_CFG1	0x4
+	 * CORE_CSR_CDC_REFCOUNT_CFG	0xCB732020
+	 * CORE_CSR_CDC_COARSE_CAL_CFG	0xB19
+	 * CORE_CSR_CDC_DELAY_CFG	0x3AC
+	 * CORE_CDC_OFFSET_CFG		0x0
+	 * CORE_CDC_SLAVE_DDA_CFG	0x16334
+	 */
+
+	writel_relaxed(0x11800EC, host->ioaddr + CORE_CSR_CDC_CTLR_CFG0);
+	writel_relaxed(0x3011111, host->ioaddr + CORE_CSR_CDC_CTLR_CFG1);
+	writel_relaxed(0x1201000, host->ioaddr + CORE_CSR_CDC_CAL_TIMER_CFG0);
+	writel_relaxed(0x4, host->ioaddr + CORE_CSR_CDC_CAL_TIMER_CFG1);
+	writel_relaxed(0xCB732020, host->ioaddr + CORE_CSR_CDC_REFCOUNT_CFG);
+	writel_relaxed(0xB19, host->ioaddr + CORE_CSR_CDC_COARSE_CAL_CFG);
+	writel_relaxed(0x3AC, host->ioaddr + CORE_CSR_CDC_DELAY_CFG);
+	writel_relaxed(0x0, host->ioaddr + CORE_CDC_OFFSET_CFG);
+	writel_relaxed(0x16334, host->ioaddr + CORE_CDC_SLAVE_DDA_CFG);
+
+	/* CDC HW Calibration */
+
+	config = readl_relaxed(host->ioaddr + CORE_CSR_CDC_CTLR_CFG0);
+	config |= CORE_SW_TRIG_FULL_CALIB;
+	writel_relaxed(config, host->ioaddr + CORE_CSR_CDC_CTLR_CFG0);
+
+	config = readl_relaxed(host->ioaddr + CORE_CSR_CDC_CTLR_CFG0);
+	config &= ~CORE_SW_TRIG_FULL_CALIB;
+	writel_relaxed(config, host->ioaddr + CORE_CSR_CDC_CTLR_CFG0);
+
+	config = readl_relaxed(host->ioaddr + CORE_CSR_CDC_CTLR_CFG0);
+	config |= CORE_HW_AUTOCAL_ENA;
+	writel_relaxed(config, host->ioaddr + CORE_CSR_CDC_CTLR_CFG0);
+
+	config = readl_relaxed(host->ioaddr + CORE_CSR_CDC_CAL_TIMER_CFG0);
+	config |= CORE_TIMER_ENA;
+	writel_relaxed(config, host->ioaddr + CORE_CSR_CDC_CAL_TIMER_CFG0);
+
+	ret = readl_relaxed_poll_timeout(host->ioaddr + CORE_CSR_CDC_STATUS0,
+					 calib_done,
+					 (calib_done & CORE_CALIBRATION_DONE),
+					 1, 50);
+
+	if (ret == -ETIMEDOUT) {
+		pr_err("%s: %s: CDC calibration was not completed\n",
+		       mmc_hostname(host->mmc), __func__);
+		goto out;
+	}
+
+	ret = readl_relaxed(host->ioaddr + CORE_CSR_CDC_STATUS0)
+			& CORE_CDC_ERROR_CODE_MASK;
+	if (ret) {
+		pr_err("%s: %s: CDC error code %d\n",
+		       mmc_hostname(host->mmc), __func__, ret);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	config = readl_relaxed(host->ioaddr + CORE_DDR_200_CFG);
+	config |= CORE_START_CDC_TRAFFIC;
+	writel_relaxed(config, host->ioaddr + CORE_DDR_200_CFG);
+out:
+	pr_debug("%s: %s: Exit, ret %d\n", mmc_hostname(host->mmc),
+		 __func__, ret);
+	return ret;
+}
+
+static int sdhci_msm_cm_dll_sdc4_calibration(struct sdhci_host *host)
+{
+	u32 dll_status, config;
+	int ret;
+
+	pr_debug("%s: %s: Enter\n", mmc_hostname(host->mmc), __func__);
+
+	/*
+	 * Currently the CORE_DDR_CONFIG register defaults to desired
+	 * configuration on reset. Currently reprogramming the power on
+	 * reset (POR) value in case it might have been modified by
+	 * bootloaders. In the future, if this changes, then the desired
+	 * values will need to be programmed appropriately.
+	 */
+	writel_relaxed(DDR_CONFIG_POR_VAL, host->ioaddr + CORE_DDR_CONFIG);
+
+	config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG_2);
+	config |= CORE_DDR_CAL_EN;
+	writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG_2);
+
+	ret = readl_relaxed_poll_timeout(host->ioaddr + CORE_DLL_STATUS,
+					 dll_status,
+					 (dll_status & CORE_DDR_DLL_LOCK),
+					 10, 1000);
+
+	if (ret == -ETIMEDOUT) {
+		pr_err("%s: %s: CM_DLL_SDC4 calibration was not completed\n",
+		       mmc_hostname(host->mmc), __func__);
+		goto out;
+	}
+
+	config = readl_relaxed(host->ioaddr + CORE_VENDOR_SPEC3);
+	config |= CORE_PWRSAVE_DLL;
+	writel_relaxed(config, host->ioaddr + CORE_VENDOR_SPEC3);
+
+	/*
+	 * Drain writebuffer to ensure above DLL calibration
+	 * and PWRSAVE DLL is enabled.
+	 */
+	wmb();
+out:
+	pr_debug("%s: %s: Exit, ret %d\n", mmc_hostname(host->mmc),
+		 __func__, ret);
+	return ret;
+}
+
+static int sdhci_msm_hs400_dll_calibration(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+	u32 config;
+
+	pr_debug("%s: %s: Enter\n", mmc_hostname(host->mmc), __func__);
+
+	/*
+	 * Retuning in HS400 (DDR mode) will fail, just reset the
+	 * tuning block and restore the saved tuning phase.
+	 */
+	ret = msm_init_cm_dll(host);
+	if (ret)
+		goto out;
+
+	/* Set the selected phase in delay line hw block */
+	ret = msm_config_cm_dll_phase(host, msm_host->saved_tuning_phase);
+	if (ret)
+		goto out;
+
+	config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
+	config |= CORE_CMD_DAT_TRACK_SEL;
+	writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
+	if (msm_host->use_cdclp533)
+		ret = sdhci_msm_cdclp533_calibration(host);
+	else
+		ret = sdhci_msm_cm_dll_sdc4_calibration(host);
+out:
+	pr_debug("%s: %s: Exit, ret %d\n", mmc_hostname(host->mmc),
+		 __func__, ret);
+	return ret;
+}
+
 static int sdhci_msm_execute_tuning(struct sdhci_host *host, u32 opcode)
 {
 	int tuning_seq_cnt = 3;
@@ -365,14 +665,17 @@ static int sdhci_msm_execute_tuning(struct sdhci_host *host, u32 opcode)
 	int rc;
 	struct mmc_host *mmc = host->mmc;
 	struct mmc_ios ios = host->mmc->ios;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
 
 	/*
 	 * Tuning is required for SDR104, HS200 and HS400 cards and
 	 * if clock frequency is greater than 100MHz in these modes.
 	 */
-	if (host->clock <= 100 * 1000 * 1000 ||
-	    !((ios.timing == MMC_TIMING_MMC_HS200) ||
-	      (ios.timing == MMC_TIMING_UHS_SDR104)))
+	if (host->clock <= CORE_FREQ_100MHZ ||
+	    !(ios.timing == MMC_TIMING_MMC_HS400 ||
+	    ios.timing == MMC_TIMING_MMC_HS200 ||
+	    ios.timing == MMC_TIMING_UHS_SDR104))
 		return 0;
 
 retry:
@@ -388,6 +691,7 @@ retry:
 		if (rc)
 			return rc;
 
+		msm_host->saved_tuning_phase = phase;
 		rc = mmc_send_tuning(mmc, opcode, NULL);
 		if (!rc) {
 			/* Tuning is successful at this tuning point */
@@ -423,6 +727,8 @@ retry:
 		rc = -EIO;
 	}
 
+	if (!rc)
+		msm_host->tuning_done = true;
 	return rc;
 }
 
@@ -430,7 +736,10 @@ static void sdhci_msm_set_uhs_signaling(struct sdhci_host *host,
 					unsigned int uhs)
 {
 	struct mmc_host *mmc = host->mmc;
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
 	u16 ctrl_2;
+	u32 config;
 
 	ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
 	/* Select Bus Speed Mode for host */
@@ -445,6 +754,7 @@ static void sdhci_msm_set_uhs_signaling(struct sdhci_host *host,
 	case MMC_TIMING_UHS_SDR50:
 		ctrl_2 |= SDHCI_CTRL_UHS_SDR50;
 		break;
+	case MMC_TIMING_MMC_HS400:
 	case MMC_TIMING_MMC_HS200:
 	case MMC_TIMING_UHS_SDR104:
 		ctrl_2 |= SDHCI_CTRL_UHS_SDR104;
@@ -461,15 +771,42 @@ static void sdhci_msm_set_uhs_signaling(struct sdhci_host *host,
 	 * provide feedback clock, the mode selection can be any value less
 	 * than 3'b011 in bits [2:0] of HOST CONTROL2 register.
 	 */
-	if (host->clock <= 100000000 &&
-	    (uhs == MMC_TIMING_MMC_HS400 ||
-	     uhs == MMC_TIMING_MMC_HS200 ||
-	     uhs == MMC_TIMING_UHS_SDR104))
-		ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
+	if (host->clock <= CORE_FREQ_100MHZ) {
+		if (uhs == MMC_TIMING_MMC_HS400 ||
+		    uhs == MMC_TIMING_MMC_HS200 ||
+		    uhs == MMC_TIMING_UHS_SDR104)
+			ctrl_2 &= ~SDHCI_CTRL_UHS_MASK;
+		/*
+		 * DLL is not required for clock <= 100MHz
+		 * Thus, make sure DLL it is disabled when not required
+		 */
+		config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
+		config |= CORE_DLL_RST;
+		writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
+
+		config = readl_relaxed(host->ioaddr + CORE_DLL_CONFIG);
+		config |= CORE_DLL_PDN;
+		writel_relaxed(config, host->ioaddr + CORE_DLL_CONFIG);
+
+		/*
+		 * The DLL needs to be restored and CDCLP533 recalibrated
+		 * when the clock frequency is set back to 400MHz.
+		 */
+		msm_host->calibration_done = false;
+	}
 
 	dev_dbg(mmc_dev(mmc), "%s: clock=%u uhs=%u ctrl_2=0x%x\n",
 		mmc_hostname(host->mmc), host->clock, uhs, ctrl_2);
 	sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
+
+	spin_unlock_irq(&host->lock);
+	/* CDCLP533 HW calibration is only required for HS400 mode*/
+	if (host->clock > CORE_FREQ_100MHZ &&
+	    msm_host->tuning_done && !msm_host->calibration_done &&
+	    mmc->ios.timing == MMC_TIMING_MMC_HS400)
+		if (!sdhci_msm_hs400_dll_calibration(host))
+			msm_host->calibration_done = true;
+	spin_lock_irq(&host->lock);
 }
 
 static void sdhci_msm_voltage_switch(struct sdhci_host *host)
@@ -505,6 +842,183 @@ static irqreturn_t sdhci_msm_pwr_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static unsigned int sdhci_msm_get_max_clock(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
+
+	return clk_round_rate(msm_host->clk, ULONG_MAX);
+}
+
+static unsigned int sdhci_msm_get_min_clock(struct sdhci_host *host)
+{
+	return SDHCI_MSM_MIN_CLOCK;
+}
+
+/**
+ * __sdhci_msm_set_clock - sdhci_msm clock control.
+ *
+ * Description:
+ * MSM controller does not use internal divider and
+ * instead directly control the GCC clock as per
+ * HW recommendation.
+ **/
+void __sdhci_msm_set_clock(struct sdhci_host *host, unsigned int clock)
+{
+	u16 clk;
+	/*
+	 * Keep actual_clock as zero -
+	 * - since there is no divider used so no need of having actual_clock.
+	 * - MSM controller uses SDCLK for data timeout calculation. If
+	 *   actual_clock is zero, host->clock is taken for calculation.
+	 */
+	host->mmc->actual_clock = 0;
+
+	sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
+
+	if (clock == 0)
+		return;
+
+	/*
+	 * MSM controller do not use clock divider.
+	 * Thus read SDHCI_CLOCK_CONTROL and only enable
+	 * clock with no divider value programmed.
+	 */
+	clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
+	sdhci_enable_clk(host, clk);
+}
+
+/* sdhci_msm_set_clock - Called with (host->lock) spinlock held. */
+static void sdhci_msm_set_clock(struct sdhci_host *host, unsigned int clock)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
+	struct mmc_ios curr_ios = host->mmc->ios;
+	u32 config, dll_lock;
+	int rc;
+
+	if (!clock) {
+		msm_host->clk_rate = clock;
+		goto out;
+	}
+
+	spin_unlock_irq(&host->lock);
+	/*
+	 * The SDHC requires internal clock frequency to be double the
+	 * actual clock that will be set for DDR mode. The controller
+	 * uses the faster clock(100/400MHz) for some of its parts and
+	 * send the actual required clock (50/200MHz) to the card.
+	 */
+	if (curr_ios.timing == MMC_TIMING_UHS_DDR50 ||
+	    curr_ios.timing == MMC_TIMING_MMC_DDR52 ||
+	    curr_ios.timing == MMC_TIMING_MMC_HS400)
+		clock *= 2;
+	/*
+	 * In general all timing modes are controlled via UHS mode select in
+	 * Host Control2 register. eMMC specific HS200/HS400 doesn't have
+	 * their respective modes defined here, hence we use these values.
+	 *
+	 * HS200 - SDR104 (Since they both are equivalent in functionality)
+	 * HS400 - This involves multiple configurations
+	 *		Initially SDR104 - when tuning is required as HS200
+	 *		Then when switching to DDR @ 400MHz (HS400) we use
+	 *		the vendor specific HC_SELECT_IN to control the mode.
+	 *
+	 * In addition to controlling the modes we also need to select the
+	 * correct input clock for DLL depending on the mode.
+	 *
+	 * HS400 - divided clock (free running MCLK/2)
+	 * All other modes - default (free running MCLK)
+	 */
+	if (curr_ios.timing == MMC_TIMING_MMC_HS400) {
+		/* Select the divided clock (free running MCLK/2) */
+		config = readl_relaxed(host->ioaddr + CORE_VENDOR_SPEC);
+		config &= ~CORE_HC_MCLK_SEL_MASK;
+		config |= CORE_HC_MCLK_SEL_HS400;
+
+		writel_relaxed(config, host->ioaddr + CORE_VENDOR_SPEC);
+		/*
+		 * Select HS400 mode using the HC_SELECT_IN from VENDOR SPEC
+		 * register
+		 */
+		if (msm_host->tuning_done && !msm_host->calibration_done) {
+			/*
+			 * Write 0x6 to HC_SELECT_IN and 1 to HC_SELECT_IN_EN
+			 * field in VENDOR_SPEC_FUNC
+			 */
+			config = readl_relaxed(host->ioaddr + CORE_VENDOR_SPEC);
+			config |= CORE_HC_SELECT_IN_HS400;
+			config |= CORE_HC_SELECT_IN_EN;
+			writel_relaxed(config, host->ioaddr + CORE_VENDOR_SPEC);
+		}
+		if (!msm_host->clk_rate && !msm_host->use_cdclp533) {
+			/*
+			 * Poll on DLL_LOCK or DDR_DLL_LOCK bits in
+			 * CORE_DLL_STATUS to be set.  This should get set
+			 * within 15 us at 200 MHz.
+			 */
+			rc = readl_relaxed_poll_timeout(host->ioaddr +
+							CORE_DLL_STATUS,
+							dll_lock,
+							(dll_lock &
+							(CORE_DLL_LOCK |
+							CORE_DDR_DLL_LOCK)), 10,
+							1000);
+			if (rc == -ETIMEDOUT)
+				pr_err("%s: Unable to get DLL_LOCK/DDR_DLL_LOCK, dll_status: 0x%08x\n",
+				       mmc_hostname(host->mmc), dll_lock);
+		}
+	} else {
+		if (!msm_host->use_cdclp533) {
+			config = readl_relaxed(host->ioaddr +
+					CORE_VENDOR_SPEC3);
+			config &= ~CORE_PWRSAVE_DLL;
+			writel_relaxed(config, host->ioaddr +
+					CORE_VENDOR_SPEC3);
+		}
+
+		config = readl_relaxed(host->ioaddr + CORE_VENDOR_SPEC);
+		config &= ~CORE_HC_MCLK_SEL_MASK;
+		config |= CORE_HC_MCLK_SEL_DFLT;
+		writel_relaxed(config, host->ioaddr + CORE_VENDOR_SPEC);
+
+		/*
+		 * Disable HC_SELECT_IN to be able to use the UHS mode select
+		 * configuration from Host Control2 register for all other
+		 * modes.
+		 * Write 0 to HC_SELECT_IN and HC_SELECT_IN_EN field
+		 * in VENDOR_SPEC_FUNC
+		 */
+		config = readl_relaxed(host->ioaddr + CORE_VENDOR_SPEC);
+		config &= ~CORE_HC_SELECT_IN_EN;
+		config &= ~CORE_HC_SELECT_IN_MASK;
+		writel_relaxed(config, host->ioaddr + CORE_VENDOR_SPEC);
+	}
+
+	/*
+	 * Make sure above writes impacting free running MCLK are completed
+	 * before changing the clk_rate at GCC.
+	 */
+	wmb();
+
+	rc = clk_set_rate(msm_host->clk, clock);
+	if (rc) {
+		pr_err("%s: Failed to set clock at rate %u at timing %d\n",
+		       mmc_hostname(host->mmc), clock,
+		       curr_ios.timing);
+		goto out_lock;
+	}
+	msm_host->clk_rate = clock;
+	pr_debug("%s: Setting clock at rate %lu at timing %d\n",
+		 mmc_hostname(host->mmc), clk_get_rate(msm_host->clk),
+		 curr_ios.timing);
+
+out_lock:
+	spin_lock_irq(&host->lock);
+out:
+	__sdhci_msm_set_clock(host, clock);
+}
+
 static const struct of_device_id sdhci_msm_dt_match[] = {
 	{ .compatible = "qcom,sdhci-msm-v4" },
 	{},
@@ -515,7 +1029,9 @@ MODULE_DEVICE_TABLE(of, sdhci_msm_dt_match);
 static const struct sdhci_ops sdhci_msm_ops = {
 	.platform_execute_tuning = sdhci_msm_execute_tuning,
 	.reset = sdhci_reset,
-	.set_clock = sdhci_set_clock,
+	.set_clock = sdhci_msm_set_clock,
+	.get_min_clock = sdhci_msm_get_min_clock,
+	.get_max_clock = sdhci_msm_get_max_clock,
 	.set_bus_width = sdhci_set_bus_width,
 	.set_uhs_signaling = sdhci_msm_set_uhs_signaling,
 	.voltage_switch = sdhci_msm_voltage_switch,
@@ -524,7 +1040,9 @@ static const struct sdhci_ops sdhci_msm_ops = {
 static const struct sdhci_pltfm_data sdhci_msm_pdata = {
 	.quirks = SDHCI_QUIRK_BROKEN_CARD_DETECTION |
 		  SDHCI_QUIRK_NO_CARD_NO_RESET |
-		  SDHCI_QUIRK_SINGLE_POWER_WRITE,
+		  SDHCI_QUIRK_SINGLE_POWER_WRITE |
+		  SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
 	.ops = &sdhci_msm_ops,
 };
 
@@ -536,7 +1054,7 @@ static int sdhci_msm_probe(struct platform_device *pdev)
 	struct resource *core_memres;
 	int ret;
 	u16 host_version, core_minor;
-	u32 core_version, caps;
+	u32 core_version, config;
 	u8 core_major;
 
 	host = sdhci_pltfm_init(pdev, &sdhci_msm_pdata, sizeof(*msm_host));
@@ -554,6 +1072,8 @@ static int sdhci_msm_probe(struct platform_device *pdev)
 
 	sdhci_get_of_property(pdev);
 
+	msm_host->saved_tuning_phase = INVALID_TUNING_PHASE;
+
 	/* Setup SDCC bus voter clock. */
 	msm_host->bus_clk = devm_clk_get(&pdev->dev, "bus");
 	if (!IS_ERR(msm_host->bus_clk)) {
@@ -586,6 +1106,16 @@ static int sdhci_msm_probe(struct platform_device *pdev)
 		goto pclk_disable;
 	}
 
+	/*
+	 * xo clock is needed for FLL feature of cm_dll.
+	 * In case if xo clock is not mentioned in DT, warn and proceed.
+	 */
+	msm_host->xo_clk = devm_clk_get(&pdev->dev, "xo");
+	if (IS_ERR(msm_host->xo_clk)) {
+		ret = PTR_ERR(msm_host->xo_clk);
+		dev_warn(&pdev->dev, "TCXO clk not present (%d)\n", ret);
+	}
+
 	/* Vote for maximum clock rate for maximum performance */
 	ret = clk_set_rate(msm_host->clk, INT_MAX);
 	if (ret)
@@ -604,9 +1134,9 @@ static int sdhci_msm_probe(struct platform_device *pdev)
 		goto clk_disable;
 	}
 
-	/* Reset the core and Enable SDHC mode */
-	writel_relaxed(readl_relaxed(msm_host->core_mem + CORE_POWER) |
-		       CORE_SW_RST, msm_host->core_mem + CORE_POWER);
+	config = readl_relaxed(msm_host->core_mem + CORE_POWER);
+	config |= CORE_SW_RST;
+	writel_relaxed(config, msm_host->core_mem + CORE_POWER);
 
 	/* SW reset can take upto 10HCLK + 15MCLK cycles. (min 40us) */
 	usleep_range(1000, 5000);
@@ -619,6 +1149,10 @@ static int sdhci_msm_probe(struct platform_device *pdev)
 	/* Set HC_MODE_EN bit in HC_MODE register */
 	writel_relaxed(HC_MODE_EN, (msm_host->core_mem + CORE_HC_MODE));
 
+	config = readl_relaxed(msm_host->core_mem + CORE_HC_MODE);
+	config |= FF_CLK_SW_RST_DIS;
+	writel_relaxed(config, msm_host->core_mem + CORE_HC_MODE);
+
 	host_version = readw_relaxed((host->ioaddr + SDHCI_HOST_VERSION));
 	dev_dbg(&pdev->dev, "Host Version: 0x%x Vendor Version 0x%x\n",
 		host_version, ((host_version & SDHCI_VENDOR_VER_MASK) >>
@@ -631,14 +1165,24 @@ static int sdhci_msm_probe(struct platform_device *pdev)
 	dev_dbg(&pdev->dev, "MCI Version: 0x%08x, major: 0x%04x, minor: 0x%02x\n",
 		core_version, core_major, core_minor);
 
+	if (core_major == 1 && core_minor >= 0x42)
+		msm_host->use_14lpp_dll_reset = true;
+
+	/*
+	 * SDCC 5 controller with major version 1, minor version 0x34 and later
+	 * with HS 400 mode support will use CM DLL instead of CDC LP 533 DLL.
+	 */
+	if (core_major == 1 && core_minor < 0x34)
+		msm_host->use_cdclp533 = true;
+
 	/*
 	 * Support for some capabilities is not advertised by newer
 	 * controller versions and must be explicitly enabled.
 	 */
 	if (core_major >= 1 && core_minor != 0x11 && core_minor != 0x12) {
-		caps = readl_relaxed(host->ioaddr + SDHCI_CAPABILITIES);
-		caps |= SDHCI_CAN_VDD_300 | SDHCI_CAN_DO_8BIT;
-		writel_relaxed(caps, host->ioaddr +
+		config = readl_relaxed(host->ioaddr + SDHCI_CAPABILITIES);
+		config |= SDHCI_CAN_VDD_300 | SDHCI_CAN_DO_8BIT;
+		writel_relaxed(config, host->ioaddr +
 			       CORE_VENDOR_SPEC_CAPABILITIES0);
 	}
 
@@ -659,12 +1203,26 @@ static int sdhci_msm_probe(struct platform_device *pdev)
 		goto clk_disable;
 	}
 
+	pm_runtime_get_noresume(&pdev->dev);
+	pm_runtime_set_active(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
+	pm_runtime_set_autosuspend_delay(&pdev->dev,
+					 MSM_MMC_AUTOSUSPEND_DELAY_MS);
+	pm_runtime_use_autosuspend(&pdev->dev);
+
 	ret = sdhci_add_host(host);
 	if (ret)
-		goto clk_disable;
+		goto pm_runtime_disable;
+
+	pm_runtime_mark_last_busy(&pdev->dev);
+	pm_runtime_put_autosuspend(&pdev->dev);
 
 	return 0;
 
+pm_runtime_disable:
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_set_suspended(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
 clk_disable:
 	clk_disable_unprepare(msm_host->clk);
 pclk_disable:
@@ -686,6 +1244,11 @@ static int sdhci_msm_remove(struct platform_device *pdev)
 		    0xffffffff);
 
 	sdhci_remove_host(host, dead);
+
+	pm_runtime_get_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
+
 	clk_disable_unprepare(msm_host->clk);
 	clk_disable_unprepare(msm_host->pclk);
 	if (!IS_ERR(msm_host->bus_clk))
@@ -694,12 +1257,57 @@ static int sdhci_msm_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_PM
+static int sdhci_msm_runtime_suspend(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
+
+	clk_disable_unprepare(msm_host->clk);
+	clk_disable_unprepare(msm_host->pclk);
+
+	return 0;
+}
+
+static int sdhci_msm_runtime_resume(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_msm_host *msm_host = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	ret = clk_prepare_enable(msm_host->clk);
+	if (ret) {
+		dev_err(dev, "clk_enable failed for core_clk: %d\n", ret);
+		return ret;
+	}
+	ret = clk_prepare_enable(msm_host->pclk);
+	if (ret) {
+		dev_err(dev, "clk_enable failed for iface_clk: %d\n", ret);
+		clk_disable_unprepare(msm_host->clk);
+		return ret;
+	}
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops sdhci_msm_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(sdhci_msm_runtime_suspend,
+			   sdhci_msm_runtime_resume,
+			   NULL)
+};
+
 static struct platform_driver sdhci_msm_driver = {
 	.probe = sdhci_msm_probe,
 	.remove = sdhci_msm_remove,
 	.driver = {
 		   .name = "sdhci_msm",
 		   .of_match_table = sdhci_msm_dt_match,
+		   .pm = &sdhci_msm_pm_ops,
 	},
 };
 
diff --git a/drivers/mmc/host/sdhci-of-at91.c b/drivers/mmc/host/sdhci-of-at91.c
index a9b7fc06c434..2f9ad213377a 100644
--- a/drivers/mmc/host/sdhci-of-at91.c
+++ b/drivers/mmc/host/sdhci-of-at91.c
@@ -100,6 +100,7 @@ static const struct of_device_id sdhci_at91_dt_match[] = {
 	{ .compatible = "atmel,sama5d2-sdhci", .data = &soc_data_sama5d2 },
 	{}
 };
+MODULE_DEVICE_TABLE(of, sdhci_at91_dt_match);
 
 #ifdef CONFIG_PM
 static int sdhci_at91_runtime_suspend(struct device *dev)
diff --git a/drivers/mmc/host/sdhci-of-esdhc.c b/drivers/mmc/host/sdhci-of-esdhc.c
index 1bb11e4a9fe5..9a6eb4492172 100644
--- a/drivers/mmc/host/sdhci-of-esdhc.c
+++ b/drivers/mmc/host/sdhci-of-esdhc.c
@@ -18,6 +18,7 @@
 #include <linux/of.h>
 #include <linux/delay.h>
 #include <linux/module.h>
+#include <linux/sys_soc.h>
 #include <linux/mmc/host.h>
 #include "sdhci-pltfm.h"
 #include "sdhci-esdhc.h"
@@ -28,6 +29,7 @@
 struct sdhci_esdhc {
 	u8 vendor_ver;
 	u8 spec_ver;
+	bool quirk_incorrect_hostver;
 };
 
 /**
@@ -87,6 +89,8 @@ static u32 esdhc_readl_fixup(struct sdhci_host *host,
 static u16 esdhc_readw_fixup(struct sdhci_host *host,
 				     int spec_reg, u32 value)
 {
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_esdhc *esdhc = sdhci_pltfm_priv(pltfm_host);
 	u16 ret;
 	int shift = (spec_reg & 0x2) * 8;
 
@@ -94,6 +98,12 @@ static u16 esdhc_readw_fixup(struct sdhci_host *host,
 		ret = value & 0xffff;
 	else
 		ret = (value >> shift) & 0xffff;
+	/* Workaround for T4240-R1.0-R2.0 eSDHC which has incorrect
+	 * vendor version and spec version information.
+	 */
+	if ((spec_reg == SDHCI_HOST_VERSION) &&
+	    (esdhc->quirk_incorrect_hostver))
+		ret = (VENDOR_V_23 << SDHCI_VENDOR_VER_SHIFT) | SDHCI_SPEC_200;
 	return ret;
 }
 
@@ -572,6 +582,12 @@ static const struct sdhci_pltfm_data sdhci_esdhc_le_pdata = {
 	.ops = &sdhci_esdhc_le_ops,
 };
 
+static struct soc_device_attribute soc_incorrect_hostver[] = {
+	{ .family = "QorIQ T4240", .revision = "1.0", },
+	{ .family = "QorIQ T4240", .revision = "2.0", },
+	{ },
+};
+
 static void esdhc_init(struct platform_device *pdev, struct sdhci_host *host)
 {
 	struct sdhci_pltfm_host *pltfm_host;
@@ -585,6 +601,10 @@ static void esdhc_init(struct platform_device *pdev, struct sdhci_host *host)
 	esdhc->vendor_ver = (host_ver & SDHCI_VENDOR_VER_MASK) >>
 			     SDHCI_VENDOR_VER_SHIFT;
 	esdhc->spec_ver = host_ver & SDHCI_SPEC_VER_MASK;
+	if (soc_device_match(soc_incorrect_hostver))
+		esdhc->quirk_incorrect_hostver = true;
+	else
+		esdhc->quirk_incorrect_hostver = false;
 }
 
 static int sdhci_esdhc_probe(struct platform_device *pdev)
diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c
index 1d9e00a00e9f..1a72d32af07f 100644
--- a/drivers/mmc/host/sdhci-pci-core.c
+++ b/drivers/mmc/host/sdhci-pci-core.c
@@ -27,6 +27,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/mmc/slot-gpio.h>
 #include <linux/mmc/sdhci-pci-data.h>
+#include <linux/acpi.h>
 
 #include "sdhci.h"
 #include "sdhci-pci.h"
@@ -375,6 +376,44 @@ static int byt_emmc_probe_slot(struct sdhci_pci_slot *slot)
 	return 0;
 }
 
+#ifdef CONFIG_ACPI
+static int ni_set_max_freq(struct sdhci_pci_slot *slot)
+{
+	acpi_status status;
+	unsigned long long max_freq;
+
+	status = acpi_evaluate_integer(ACPI_HANDLE(&slot->chip->pdev->dev),
+				       "MXFQ", NULL, &max_freq);
+	if (ACPI_FAILURE(status)) {
+		dev_err(&slot->chip->pdev->dev,
+			"MXFQ not found in acpi table\n");
+		return -EINVAL;
+	}
+
+	slot->host->mmc->f_max = max_freq * 1000000;
+
+	return 0;
+}
+#else
+static inline int ni_set_max_freq(struct sdhci_pci_slot *slot)
+{
+	return 0;
+}
+#endif
+
+static int ni_byt_sdio_probe_slot(struct sdhci_pci_slot *slot)
+{
+	int err;
+
+	err = ni_set_max_freq(slot);
+	if (err)
+		return err;
+
+	slot->host->mmc->caps |= MMC_CAP_POWER_OFF_CARD | MMC_CAP_NONREMOVABLE |
+				 MMC_CAP_WAIT_WHILE_BUSY;
+	return 0;
+}
+
 static int byt_sdio_probe_slot(struct sdhci_pci_slot *slot)
 {
 	slot->host->mmc->caps |= MMC_CAP_POWER_OFF_CARD | MMC_CAP_NONREMOVABLE |
@@ -390,7 +429,8 @@ static int byt_sd_probe_slot(struct sdhci_pci_slot *slot)
 	slot->cd_override_level = true;
 	if (slot->chip->pdev->device == PCI_DEVICE_ID_INTEL_BXT_SD ||
 	    slot->chip->pdev->device == PCI_DEVICE_ID_INTEL_BXTM_SD ||
-	    slot->chip->pdev->device == PCI_DEVICE_ID_INTEL_APL_SD) {
+	    slot->chip->pdev->device == PCI_DEVICE_ID_INTEL_APL_SD ||
+	    slot->chip->pdev->device == PCI_DEVICE_ID_INTEL_GLK_SD) {
 		slot->host->mmc_host_ops.get_cd = bxt_get_cd;
 		slot->host->mmc->caps |= MMC_CAP_AGGRESSIVE_PM;
 	}
@@ -447,6 +487,15 @@ static const struct sdhci_pci_fixes sdhci_intel_byt_emmc = {
 	.ops		= &sdhci_intel_byt_ops,
 };
 
+static const struct sdhci_pci_fixes sdhci_ni_byt_sdio = {
+	.quirks		= SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC,
+	.quirks2	= SDHCI_QUIRK2_HOST_OFF_CARD_ON |
+			  SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+	.allow_runtime_pm = true,
+	.probe_slot	= ni_byt_sdio_probe_slot,
+	.ops		= &sdhci_intel_byt_ops,
+};
+
 static const struct sdhci_pci_fixes sdhci_intel_byt_sdio = {
 	.quirks		= SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC,
 	.quirks2	= SDHCI_QUIRK2_HOST_OFF_CARD_ON |
@@ -1079,6 +1128,14 @@ static const struct pci_device_id pci_ids[] = {
 	{
 		.vendor		= PCI_VENDOR_ID_INTEL,
 		.device		= PCI_DEVICE_ID_INTEL_BYT_SDIO,
+		.subvendor	= PCI_VENDOR_ID_NI,
+		.subdevice	= 0x7884,
+		.driver_data	= (kernel_ulong_t)&sdhci_ni_byt_sdio,
+	},
+
+	{
+		.vendor		= PCI_VENDOR_ID_INTEL,
+		.device		= PCI_DEVICE_ID_INTEL_BYT_SDIO,
 		.subvendor	= PCI_ANY_ID,
 		.subdevice	= PCI_ANY_ID,
 		.driver_data	= (kernel_ulong_t)&sdhci_intel_byt_sdio,
@@ -1277,6 +1334,30 @@ static const struct pci_device_id pci_ids[] = {
 	},
 
 	{
+		.vendor		= PCI_VENDOR_ID_INTEL,
+		.device		= PCI_DEVICE_ID_INTEL_GLK_EMMC,
+		.subvendor	= PCI_ANY_ID,
+		.subdevice	= PCI_ANY_ID,
+		.driver_data	= (kernel_ulong_t)&sdhci_intel_byt_emmc,
+	},
+
+	{
+		.vendor		= PCI_VENDOR_ID_INTEL,
+		.device		= PCI_DEVICE_ID_INTEL_GLK_SDIO,
+		.subvendor	= PCI_ANY_ID,
+		.subdevice	= PCI_ANY_ID,
+		.driver_data	= (kernel_ulong_t)&sdhci_intel_byt_sdio,
+	},
+
+	{
+		.vendor		= PCI_VENDOR_ID_INTEL,
+		.device		= PCI_DEVICE_ID_INTEL_GLK_SD,
+		.subvendor	= PCI_ANY_ID,
+		.subdevice	= PCI_ANY_ID,
+		.driver_data	= (kernel_ulong_t)&sdhci_intel_byt_sd,
+	},
+
+	{
 		.vendor		= PCI_VENDOR_ID_O2,
 		.device		= PCI_DEVICE_ID_O2_8120,
 		.subvendor	= PCI_ANY_ID,
@@ -1735,11 +1816,16 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot(
 	host->mmc->slotno = slotno;
 	host->mmc->caps2 |= MMC_CAP2_NO_PRESCAN_POWERUP;
 
-	if (slot->cd_idx >= 0 &&
-	    mmc_gpiod_request_cd(host->mmc, slot->cd_con_id, slot->cd_idx,
-				 slot->cd_override_level, 0, NULL)) {
-		dev_warn(&pdev->dev, "failed to setup card detect gpio\n");
-		slot->cd_idx = -1;
+	if (slot->cd_idx >= 0) {
+		ret = mmc_gpiod_request_cd(host->mmc, slot->cd_con_id, slot->cd_idx,
+					   slot->cd_override_level, 0, NULL);
+		if (ret == -EPROBE_DEFER)
+			goto remove;
+
+		if (ret) {
+			dev_warn(&pdev->dev, "failed to setup card detect gpio\n");
+			slot->cd_idx = -1;
+		}
 	}
 
 	ret = sdhci_add_host(host);
diff --git a/drivers/mmc/host/sdhci-pci.h b/drivers/mmc/host/sdhci-pci.h
index 6bccf56bc5ff..4abdaed72bd4 100644
--- a/drivers/mmc/host/sdhci-pci.h
+++ b/drivers/mmc/host/sdhci-pci.h
@@ -34,6 +34,9 @@
 #define PCI_DEVICE_ID_INTEL_APL_SD	0x5aca
 #define PCI_DEVICE_ID_INTEL_APL_EMMC	0x5acc
 #define PCI_DEVICE_ID_INTEL_APL_SDIO	0x5ad0
+#define PCI_DEVICE_ID_INTEL_GLK_SD	0x31ca
+#define PCI_DEVICE_ID_INTEL_GLK_EMMC	0x31cc
+#define PCI_DEVICE_ID_INTEL_GLK_SDIO	0x31d0
 
 /*
  * PCI registers
diff --git a/drivers/mmc/host/sdhci-pltfm.h b/drivers/mmc/host/sdhci-pltfm.h
index 3280f2077959..957839d0fe37 100644
--- a/drivers/mmc/host/sdhci-pltfm.h
+++ b/drivers/mmc/host/sdhci-pltfm.h
@@ -106,7 +106,7 @@ extern unsigned int sdhci_pltfm_clk_get_max_clock(struct sdhci_host *host);
 
 static inline void *sdhci_pltfm_priv(struct sdhci_pltfm_host *host)
 {
-	return (void *)host->private;
+	return host->private;
 }
 
 extern const struct dev_pm_ops sdhci_pltfm_pmops;
diff --git a/drivers/mmc/host/sdhci-s3c.c b/drivers/mmc/host/sdhci-s3c.c
index 784c5a848fb4..de219ca7ea7c 100644
--- a/drivers/mmc/host/sdhci-s3c.c
+++ b/drivers/mmc/host/sdhci-s3c.c
@@ -121,7 +121,9 @@ static unsigned int sdhci_s3c_consider_clock(struct sdhci_s3c *ourhost,
 	 * speed possible with selected clock source and skip the division.
 	 */
 	if (ourhost->no_divider) {
+		spin_unlock_irq(&ourhost->host->lock);
 		rate = clk_round_rate(clksrc, wanted);
+		spin_lock_irq(&ourhost->host->lock);
 		return wanted - rate;
 	}
 
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index 42ef3ebb1d8c..111991e5b9a0 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -22,6 +22,7 @@
 #include <linux/scatterlist.h>
 #include <linux/regulator/consumer.h>
 #include <linux/pm_runtime.h>
+#include <linux/of.h>
 
 #include <linux/leds.h>
 
@@ -1343,20 +1344,10 @@ clock_set:
 }
 EXPORT_SYMBOL_GPL(sdhci_calc_clk);
 
-void sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
+void sdhci_enable_clk(struct sdhci_host *host, u16 clk)
 {
-	u16 clk;
 	unsigned long timeout;
 
-	host->mmc->actual_clock = 0;
-
-	sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
-
-	if (clock == 0)
-		return;
-
-	clk = sdhci_calc_clk(host, clock, &host->mmc->actual_clock);
-
 	clk |= SDHCI_CLOCK_INT_EN;
 	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
 
@@ -1377,6 +1368,22 @@ void sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
 	clk |= SDHCI_CLOCK_CARD_EN;
 	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
 }
+EXPORT_SYMBOL_GPL(sdhci_enable_clk);
+
+void sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
+{
+	u16 clk;
+
+	host->mmc->actual_clock = 0;
+
+	sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL);
+
+	if (clock == 0)
+		return;
+
+	clk = sdhci_calc_clk(host, clock, &host->mmc->actual_clock);
+	sdhci_enable_clk(host, clk);
+}
 EXPORT_SYMBOL_GPL(sdhci_set_clock);
 
 static void sdhci_set_power_reg(struct sdhci_host *host, unsigned char mode,
@@ -1623,7 +1630,14 @@ static void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 
 	if ((ios->timing == MMC_TIMING_SD_HS ||
-	     ios->timing == MMC_TIMING_MMC_HS)
+	     ios->timing == MMC_TIMING_MMC_HS ||
+	     ios->timing == MMC_TIMING_MMC_HS400 ||
+	     ios->timing == MMC_TIMING_MMC_HS200 ||
+	     ios->timing == MMC_TIMING_MMC_DDR52 ||
+	     ios->timing == MMC_TIMING_UHS_SDR50 ||
+	     ios->timing == MMC_TIMING_UHS_SDR104 ||
+	     ios->timing == MMC_TIMING_UHS_DDR50 ||
+	     ios->timing == MMC_TIMING_UHS_SDR25)
 	    && !(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT))
 		ctrl |= SDHCI_CTRL_HISPD;
 	else
@@ -1632,16 +1646,6 @@ static void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 	if (host->version >= SDHCI_SPEC_300) {
 		u16 clk, ctrl_2;
 
-		/* In case of UHS-I modes, set High Speed Enable */
-		if ((ios->timing == MMC_TIMING_MMC_HS400) ||
-		    (ios->timing == MMC_TIMING_MMC_HS200) ||
-		    (ios->timing == MMC_TIMING_MMC_DDR52) ||
-		    (ios->timing == MMC_TIMING_UHS_SDR50) ||
-		    (ios->timing == MMC_TIMING_UHS_SDR104) ||
-		    (ios->timing == MMC_TIMING_UHS_DDR50) ||
-		    (ios->timing == MMC_TIMING_UHS_SDR25))
-			ctrl |= SDHCI_CTRL_HISPD;
-
 		if (!host->preset_enabled) {
 			sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 			/*
@@ -1948,11 +1952,157 @@ static int sdhci_prepare_hs400_tuning(struct mmc_host *mmc, struct mmc_ios *ios)
 	return 0;
 }
 
-static int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
+static void sdhci_start_tuning(struct sdhci_host *host)
 {
-	struct sdhci_host *host = mmc_priv(mmc);
 	u16 ctrl;
-	int tuning_loop_counter = MAX_TUNING_LOOP;
+
+	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+	ctrl |= SDHCI_CTRL_EXEC_TUNING;
+	if (host->quirks2 & SDHCI_QUIRK2_TUNING_WORK_AROUND)
+		ctrl |= SDHCI_CTRL_TUNED_CLK;
+	sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
+
+	/*
+	 * As per the Host Controller spec v3.00, tuning command
+	 * generates Buffer Read Ready interrupt, so enable that.
+	 *
+	 * Note: The spec clearly says that when tuning sequence
+	 * is being performed, the controller does not generate
+	 * interrupts other than Buffer Read Ready interrupt. But
+	 * to make sure we don't hit a controller bug, we _only_
+	 * enable Buffer Read Ready interrupt here.
+	 */
+	sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_INT_ENABLE);
+	sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_SIGNAL_ENABLE);
+}
+
+static void sdhci_end_tuning(struct sdhci_host *host)
+{
+	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
+	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
+}
+
+static void sdhci_reset_tuning(struct sdhci_host *host)
+{
+	u16 ctrl;
+
+	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+	ctrl &= ~SDHCI_CTRL_TUNED_CLK;
+	ctrl &= ~SDHCI_CTRL_EXEC_TUNING;
+	sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
+}
+
+static void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode,
+			       unsigned long flags)
+{
+	sdhci_reset_tuning(host);
+
+	sdhci_do_reset(host, SDHCI_RESET_CMD);
+	sdhci_do_reset(host, SDHCI_RESET_DATA);
+
+	sdhci_end_tuning(host);
+
+	spin_unlock_irqrestore(&host->lock, flags);
+	mmc_abort_tuning(host->mmc, opcode);
+	spin_lock_irqsave(&host->lock, flags);
+}
+
+/*
+ * We use sdhci_send_tuning() because mmc_send_tuning() is not a good fit. SDHCI
+ * tuning command does not have a data payload (or rather the hardware does it
+ * automatically) so mmc_send_tuning() will return -EIO. Also the tuning command
+ * interrupt setup is different to other commands and there is no timeout
+ * interrupt so special handling is needed.
+ */
+static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode,
+			      unsigned long flags)
+{
+	struct mmc_host *mmc = host->mmc;
+	struct mmc_command cmd = {0};
+	struct mmc_request mrq = {NULL};
+
+	cmd.opcode = opcode;
+	cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
+	cmd.mrq = &mrq;
+
+	mrq.cmd = &cmd;
+	/*
+	 * In response to CMD19, the card sends 64 bytes of tuning
+	 * block to the Host Controller. So we set the block size
+	 * to 64 here.
+	 */
+	if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
+	    mmc->ios.bus_width == MMC_BUS_WIDTH_8)
+		sdhci_writew(host, SDHCI_MAKE_BLKSZ(7, 128), SDHCI_BLOCK_SIZE);
+	else
+		sdhci_writew(host, SDHCI_MAKE_BLKSZ(7, 64), SDHCI_BLOCK_SIZE);
+
+	/*
+	 * The tuning block is sent by the card to the host controller.
+	 * So we set the TRNS_READ bit in the Transfer Mode register.
+	 * This also takes care of setting DMA Enable and Multi Block
+	 * Select in the same register to 0.
+	 */
+	sdhci_writew(host, SDHCI_TRNS_READ, SDHCI_TRANSFER_MODE);
+
+	sdhci_send_command(host, &cmd);
+
+	host->cmd = NULL;
+
+	sdhci_del_timer(host, &mrq);
+
+	host->tuning_done = 0;
+
+	spin_unlock_irqrestore(&host->lock, flags);
+
+	/* Wait for Buffer Read Ready interrupt */
+	wait_event_timeout(host->buf_ready_int, (host->tuning_done == 1),
+			   msecs_to_jiffies(50));
+
+	spin_lock_irqsave(&host->lock, flags);
+}
+
+static void __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode,
+				   unsigned long flags)
+{
+	int i;
+
+	/*
+	 * Issue opcode repeatedly till Execute Tuning is set to 0 or the number
+	 * of loops reaches 40 times.
+	 */
+	for (i = 0; i < MAX_TUNING_LOOP; i++) {
+		u16 ctrl;
+
+		sdhci_send_tuning(host, opcode, flags);
+
+		if (!host->tuning_done) {
+			pr_info("%s: Tuning timeout, falling back to fixed sampling clock\n",
+				mmc_hostname(host->mmc));
+			sdhci_abort_tuning(host, opcode, flags);
+			return;
+		}
+
+		ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+		if (!(ctrl & SDHCI_CTRL_EXEC_TUNING)) {
+			if (ctrl & SDHCI_CTRL_TUNED_CLK)
+				return; /* Success! */
+			break;
+		}
+
+		/* eMMC spec does not require a delay between tuning cycles */
+		if (opcode == MMC_SEND_TUNING_BLOCK)
+			mdelay(1);
+	}
+
+	pr_info("%s: Tuning failed, falling back to fixed sampling clock\n",
+		mmc_hostname(host->mmc));
+	sdhci_reset_tuning(host);
+}
+
+int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
 	int err = 0;
 	unsigned long flags;
 	unsigned int tuning_count = 0;
@@ -2003,144 +2153,22 @@ static int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
 
 	if (host->ops->platform_execute_tuning) {
 		spin_unlock_irqrestore(&host->lock, flags);
-		err = host->ops->platform_execute_tuning(host, opcode);
-		return err;
+		return host->ops->platform_execute_tuning(host, opcode);
 	}
 
-	ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
-	ctrl |= SDHCI_CTRL_EXEC_TUNING;
-	if (host->quirks2 & SDHCI_QUIRK2_TUNING_WORK_AROUND)
-		ctrl |= SDHCI_CTRL_TUNED_CLK;
-	sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
-
-	/*
-	 * As per the Host Controller spec v3.00, tuning command
-	 * generates Buffer Read Ready interrupt, so enable that.
-	 *
-	 * Note: The spec clearly says that when tuning sequence
-	 * is being performed, the controller does not generate
-	 * interrupts other than Buffer Read Ready interrupt. But
-	 * to make sure we don't hit a controller bug, we _only_
-	 * enable Buffer Read Ready interrupt here.
-	 */
-	sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_INT_ENABLE);
-	sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_SIGNAL_ENABLE);
-
-	/*
-	 * Issue CMD19 repeatedly till Execute Tuning is set to 0 or the number
-	 * of loops reaches 40 times.
-	 */
-	do {
-		struct mmc_command cmd = {0};
-		struct mmc_request mrq = {NULL};
-
-		cmd.opcode = opcode;
-		cmd.arg = 0;
-		cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
-		cmd.retries = 0;
-		cmd.data = NULL;
-		cmd.mrq = &mrq;
-		cmd.error = 0;
-
-		if (tuning_loop_counter-- == 0)
-			break;
-
-		mrq.cmd = &cmd;
-
-		/*
-		 * In response to CMD19, the card sends 64 bytes of tuning
-		 * block to the Host Controller. So we set the block size
-		 * to 64 here.
-		 */
-		if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200) {
-			if (mmc->ios.bus_width == MMC_BUS_WIDTH_8)
-				sdhci_writew(host, SDHCI_MAKE_BLKSZ(7, 128),
-					     SDHCI_BLOCK_SIZE);
-			else if (mmc->ios.bus_width == MMC_BUS_WIDTH_4)
-				sdhci_writew(host, SDHCI_MAKE_BLKSZ(7, 64),
-					     SDHCI_BLOCK_SIZE);
-		} else {
-			sdhci_writew(host, SDHCI_MAKE_BLKSZ(7, 64),
-				     SDHCI_BLOCK_SIZE);
-		}
-
-		/*
-		 * The tuning block is sent by the card to the host controller.
-		 * So we set the TRNS_READ bit in the Transfer Mode register.
-		 * This also takes care of setting DMA Enable and Multi Block
-		 * Select in the same register to 0.
-		 */
-		sdhci_writew(host, SDHCI_TRNS_READ, SDHCI_TRANSFER_MODE);
-
-		sdhci_send_command(host, &cmd);
-
-		host->cmd = NULL;
-		sdhci_del_timer(host, &mrq);
-
-		spin_unlock_irqrestore(&host->lock, flags);
-		/* Wait for Buffer Read Ready interrupt */
-		wait_event_timeout(host->buf_ready_int,
-					(host->tuning_done == 1),
-					msecs_to_jiffies(50));
-		spin_lock_irqsave(&host->lock, flags);
-
-		if (!host->tuning_done) {
-			pr_info(DRIVER_NAME ": Timeout waiting for Buffer Read Ready interrupt during tuning procedure, falling back to fixed sampling clock\n");
-
-			sdhci_do_reset(host, SDHCI_RESET_CMD);
-			sdhci_do_reset(host, SDHCI_RESET_DATA);
-
-			ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
-			ctrl &= ~SDHCI_CTRL_TUNED_CLK;
-			ctrl &= ~SDHCI_CTRL_EXEC_TUNING;
-			sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
-
-			err = -EIO;
-			goto out;
-		}
+	host->mmc->retune_period = tuning_count;
 
-		host->tuning_done = 0;
+	sdhci_start_tuning(host);
 
-		ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+	__sdhci_execute_tuning(host, opcode, flags);
 
-		/* eMMC spec does not require a delay between tuning cycles */
-		if (opcode == MMC_SEND_TUNING_BLOCK)
-			mdelay(1);
-	} while (ctrl & SDHCI_CTRL_EXEC_TUNING);
-
-	/*
-	 * The Host Driver has exhausted the maximum number of loops allowed,
-	 * so use fixed sampling frequency.
-	 */
-	if (tuning_loop_counter < 0) {
-		ctrl &= ~SDHCI_CTRL_TUNED_CLK;
-		sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2);
-	}
-	if (!(ctrl & SDHCI_CTRL_TUNED_CLK)) {
-		pr_info(DRIVER_NAME ": Tuning procedure failed, falling back to fixed sampling clock\n");
-		err = -EIO;
-	}
-
-out:
-	if (tuning_count) {
-		/*
-		 * In case tuning fails, host controllers which support
-		 * re-tuning can try tuning again at a later time, when the
-		 * re-tuning timer expires.  So for these controllers, we
-		 * return 0. Since there might be other controllers who do not
-		 * have this capability, we return error for them.
-		 */
-		err = 0;
-	}
-
-	host->mmc->retune_period = err ? 0 : tuning_count;
-
-	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
-	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
+	sdhci_end_tuning(host);
 out_unlock:
 	spin_unlock_irqrestore(&host->lock, flags);
+
 	return err;
 }
+EXPORT_SYMBOL_GPL(sdhci_execute_tuning);
 
 static int sdhci_select_drive_strength(struct mmc_card *card,
 				       unsigned int max_dtr, int host_drv,
@@ -2198,8 +2226,7 @@ static void sdhci_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
 	data->host_cookie = COOKIE_UNMAPPED;
 }
 
-static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq,
-			       bool is_first_req)
+static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
 {
 	struct sdhci_host *host = mmc_priv(mmc);
 
@@ -3010,6 +3037,8 @@ static int sdhci_set_dma_mask(struct sdhci_host *host)
 void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1)
 {
 	u16 v;
+	u64 dt_caps_mask = 0;
+	u64 dt_caps = 0;
 
 	if (host->read_caps)
 		return;
@@ -3024,18 +3053,35 @@ void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1)
 
 	sdhci_do_reset(host, SDHCI_RESET_ALL);
 
+	of_property_read_u64(mmc_dev(host->mmc)->of_node,
+			     "sdhci-caps-mask", &dt_caps_mask);
+	of_property_read_u64(mmc_dev(host->mmc)->of_node,
+			     "sdhci-caps", &dt_caps);
+
 	v = ver ? *ver : sdhci_readw(host, SDHCI_HOST_VERSION);
 	host->version = (v & SDHCI_SPEC_VER_MASK) >> SDHCI_SPEC_VER_SHIFT;
 
 	if (host->quirks & SDHCI_QUIRK_MISSING_CAPS)
 		return;
 
-	host->caps = caps ? *caps : sdhci_readl(host, SDHCI_CAPABILITIES);
+	if (caps) {
+		host->caps = *caps;
+	} else {
+		host->caps = sdhci_readl(host, SDHCI_CAPABILITIES);
+		host->caps &= ~lower_32_bits(dt_caps_mask);
+		host->caps |= lower_32_bits(dt_caps);
+	}
 
 	if (host->version < SDHCI_SPEC_300)
 		return;
 
-	host->caps1 = caps1 ? *caps1 : sdhci_readl(host, SDHCI_CAPABILITIES_1);
+	if (caps1) {
+		host->caps1 = *caps1;
+	} else {
+		host->caps1 = sdhci_readl(host, SDHCI_CAPABILITIES_1);
+		host->caps1 &= ~upper_32_bits(dt_caps_mask);
+		host->caps1 |= upper_32_bits(dt_caps);
+	}
 }
 EXPORT_SYMBOL_GPL(__sdhci_read_caps);
 
diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h
index 2570455b219a..0b66f210ae82 100644
--- a/drivers/mmc/host/sdhci.h
+++ b/drivers/mmc/host/sdhci.h
@@ -656,7 +656,7 @@ extern void sdhci_free_host(struct sdhci_host *host);
 
 static inline void *sdhci_priv(struct sdhci_host *host)
 {
-	return (void *)host->private;
+	return host->private;
 }
 
 extern void sdhci_card_detect(struct sdhci_host *host);
@@ -682,6 +682,7 @@ static inline bool sdhci_sdio_irq_enabled(struct sdhci_host *host)
 u16 sdhci_calc_clk(struct sdhci_host *host, unsigned int clock,
 		   unsigned int *actual_clock);
 void sdhci_set_clock(struct sdhci_host *host, unsigned int clock);
+void sdhci_enable_clk(struct sdhci_host *host, u16 clk);
 void sdhci_set_power(struct sdhci_host *host, unsigned char mode,
 		     unsigned short vdd);
 void sdhci_set_power_noreg(struct sdhci_host *host, unsigned char mode,
@@ -689,6 +690,7 @@ void sdhci_set_power_noreg(struct sdhci_host *host, unsigned char mode,
 void sdhci_set_bus_width(struct sdhci_host *host, int width);
 void sdhci_reset(struct sdhci_host *host, u8 mask);
 void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing);
+int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode);
 
 #ifdef CONFIG_PM
 extern int sdhci_suspend_host(struct sdhci_host *host);
diff --git a/drivers/mmc/host/sh_mobile_sdhi.c b/drivers/mmc/host/sh_mobile_sdhi.c
index 49edff7fee49..d46c2d00c182 100644
--- a/drivers/mmc/host/sh_mobile_sdhi.c
+++ b/drivers/mmc/host/sh_mobile_sdhi.c
@@ -47,31 +47,69 @@
 
 #define host_to_priv(host) container_of((host)->pdata, struct sh_mobile_sdhi, mmc_data)
 
+struct sh_mobile_sdhi_scc {
+	unsigned long clk_rate;	/* clock rate for SDR104 */
+	u32 tap;		/* sampling clock position for SDR104 */
+};
+
 struct sh_mobile_sdhi_of_data {
 	unsigned long tmio_flags;
+	u32	      tmio_ocr_mask;
 	unsigned long capabilities;
 	unsigned long capabilities2;
 	enum dma_slave_buswidth dma_buswidth;
 	dma_addr_t dma_rx_offset;
 	unsigned bus_shift;
+	int scc_offset;
+	struct sh_mobile_sdhi_scc *taps;
+	int taps_num;
 };
 
 static const struct sh_mobile_sdhi_of_data of_default_cfg = {
 	.tmio_flags = TMIO_MMC_HAS_IDLE_WAIT,
 };
 
+static const struct sh_mobile_sdhi_of_data of_rz_compatible = {
+	.tmio_flags	= TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_32BIT_DATA_PORT,
+	.tmio_ocr_mask	= MMC_VDD_32_33,
+	.capabilities	= MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ,
+};
+
 static const struct sh_mobile_sdhi_of_data of_rcar_gen1_compatible = {
 	.tmio_flags	= TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_WRPROTECT_DISABLE |
 			  TMIO_MMC_CLK_ACTUAL,
 	.capabilities	= MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ,
 };
 
+/* Definitions for sampling clocks */
+static struct sh_mobile_sdhi_scc rcar_gen2_scc_taps[] = {
+	{
+		.clk_rate = 156000000,
+		.tap = 0x00000703,
+	},
+	{
+		.clk_rate = 0,
+		.tap = 0x00000300,
+	},
+};
+
 static const struct sh_mobile_sdhi_of_data of_rcar_gen2_compatible = {
 	.tmio_flags	= TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_WRPROTECT_DISABLE |
 			  TMIO_MMC_CLK_ACTUAL | TMIO_MMC_MIN_RCAR2,
 	.capabilities	= MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ,
 	.dma_buswidth	= DMA_SLAVE_BUSWIDTH_4_BYTES,
 	.dma_rx_offset	= 0x2000,
+	.scc_offset	= 0x0300,
+	.taps		= rcar_gen2_scc_taps,
+	.taps_num	= ARRAY_SIZE(rcar_gen2_scc_taps),
+};
+
+/* Definitions for sampling clocks */
+static struct sh_mobile_sdhi_scc rcar_gen3_scc_taps[] = {
+	{
+		.clk_rate = 0,
+		.tap = 0x00000300,
+	},
 };
 
 static const struct sh_mobile_sdhi_of_data of_rcar_gen3_compatible = {
@@ -79,6 +117,9 @@ static const struct sh_mobile_sdhi_of_data of_rcar_gen3_compatible = {
 			  TMIO_MMC_CLK_ACTUAL | TMIO_MMC_MIN_RCAR2,
 	.capabilities	= MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ,
 	.bus_shift	= 2,
+	.scc_offset	= 0x1000,
+	.taps		= rcar_gen3_scc_taps,
+	.taps_num	= ARRAY_SIZE(rcar_gen3_scc_taps),
 };
 
 static const struct of_device_id sh_mobile_sdhi_of_match[] = {
@@ -86,6 +127,7 @@ static const struct of_device_id sh_mobile_sdhi_of_match[] = {
 	{ .compatible = "renesas,sdhi-sh73a0", .data = &of_default_cfg, },
 	{ .compatible = "renesas,sdhi-r8a73a4", .data = &of_default_cfg, },
 	{ .compatible = "renesas,sdhi-r8a7740", .data = &of_default_cfg, },
+	{ .compatible = "renesas,sdhi-r7s72100", .data = &of_rz_compatible, },
 	{ .compatible = "renesas,sdhi-r8a7778", .data = &of_rcar_gen1_compatible, },
 	{ .compatible = "renesas,sdhi-r8a7779", .data = &of_rcar_gen1_compatible, },
 	{ .compatible = "renesas,sdhi-r8a7790", .data = &of_rcar_gen2_compatible, },
@@ -105,6 +147,7 @@ struct sh_mobile_sdhi {
 	struct tmio_mmc_dma dma_priv;
 	struct pinctrl *pinctrl;
 	struct pinctrl_state *pins_default, *pins_uhs;
+	void __iomem *scc_ctl;
 };
 
 static void sh_mobile_sdhi_sdbuf_width(struct tmio_mmc_host *host, int width)
@@ -255,6 +298,201 @@ static int sh_mobile_sdhi_start_signal_voltage_switch(struct mmc_host *mmc,
 	return pinctrl_select_state(priv->pinctrl, pin_state);
 }
 
+/* SCC registers */
+#define SH_MOBILE_SDHI_SCC_DTCNTL	0x000
+#define SH_MOBILE_SDHI_SCC_TAPSET	0x002
+#define SH_MOBILE_SDHI_SCC_DT2FF	0x004
+#define SH_MOBILE_SDHI_SCC_CKSEL	0x006
+#define SH_MOBILE_SDHI_SCC_RVSCNTL	0x008
+#define SH_MOBILE_SDHI_SCC_RVSREQ	0x00A
+
+/* Definitions for values the SH_MOBILE_SDHI_SCC_DTCNTL register */
+#define SH_MOBILE_SDHI_SCC_DTCNTL_TAPEN		BIT(0)
+#define SH_MOBILE_SDHI_SCC_DTCNTL_TAPNUM_SHIFT	16
+#define SH_MOBILE_SDHI_SCC_DTCNTL_TAPNUM_MASK	0xff
+
+/* Definitions for values the SH_MOBILE_SDHI_SCC_CKSEL register */
+#define SH_MOBILE_SDHI_SCC_CKSEL_DTSEL		BIT(0)
+/* Definitions for values the SH_MOBILE_SDHI_SCC_RVSCNTL register */
+#define SH_MOBILE_SDHI_SCC_RVSCNTL_RVSEN	BIT(0)
+/* Definitions for values the SH_MOBILE_SDHI_SCC_RVSREQ register */
+#define SH_MOBILE_SDHI_SCC_RVSREQ_RVSERR	BIT(2)
+
+static inline u32 sd_scc_read32(struct tmio_mmc_host *host,
+				struct sh_mobile_sdhi *priv, int addr)
+{
+	return readl(priv->scc_ctl + (addr << host->bus_shift));
+}
+
+static inline void sd_scc_write32(struct tmio_mmc_host *host,
+				  struct sh_mobile_sdhi *priv,
+				  int addr, u32 val)
+{
+	writel(val, priv->scc_ctl + (addr << host->bus_shift));
+}
+
+static unsigned int sh_mobile_sdhi_init_tuning(struct tmio_mmc_host *host)
+{
+	struct sh_mobile_sdhi *priv;
+
+	if (!(host->mmc->caps & MMC_CAP_UHS_SDR104))
+		return 0;
+
+	priv = host_to_priv(host);
+
+	/* set sampling clock selection range */
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_DTCNTL,
+		       0x8 << SH_MOBILE_SDHI_SCC_DTCNTL_TAPNUM_SHIFT);
+
+	/* Initialize SCC */
+	sd_ctrl_write32_as_16_and_16(host, CTL_STATUS, 0x0);
+
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_DTCNTL,
+		       SH_MOBILE_SDHI_SCC_DTCNTL_TAPEN |
+		       sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_DTCNTL));
+
+	sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, ~CLK_CTL_SCLKEN &
+			sd_ctrl_read16(host, CTL_SD_CARD_CLK_CTL));
+
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_CKSEL,
+		       SH_MOBILE_SDHI_SCC_CKSEL_DTSEL |
+		       sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_CKSEL));
+
+	sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, CLK_CTL_SCLKEN |
+			sd_ctrl_read16(host, CTL_SD_CARD_CLK_CTL));
+
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL,
+		       ~SH_MOBILE_SDHI_SCC_RVSCNTL_RVSEN &
+		       sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL));
+
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_DT2FF, host->scc_tappos);
+
+	/* Read TAPNUM */
+	return (sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_DTCNTL) >>
+		SH_MOBILE_SDHI_SCC_DTCNTL_TAPNUM_SHIFT) &
+		SH_MOBILE_SDHI_SCC_DTCNTL_TAPNUM_MASK;
+}
+
+static void sh_mobile_sdhi_prepare_tuning(struct tmio_mmc_host *host,
+					 unsigned long tap)
+{
+	struct sh_mobile_sdhi *priv = host_to_priv(host);
+
+	/* Set sampling clock position */
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TAPSET, tap);
+}
+
+#define SH_MOBILE_SDHI_MAX_TAP 3
+
+static int sh_mobile_sdhi_select_tuning(struct tmio_mmc_host *host)
+{
+	struct sh_mobile_sdhi *priv = host_to_priv(host);
+	unsigned long tap_cnt;  /* counter of tuning success */
+	unsigned long tap_set;  /* tap position */
+	unsigned long tap_start;/* start position of tuning success */
+	unsigned long tap_end;  /* end position of tuning success */
+	unsigned long ntap;     /* temporary counter of tuning success */
+	unsigned long i;
+
+	/* Clear SCC_RVSREQ */
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_RVSREQ, 0);
+
+	/*
+	 * Find the longest consecutive run of successful probes.  If that
+	 * is more than SH_MOBILE_SDHI_MAX_TAP probes long then use the
+	 * center index as the tap.
+	 */
+	tap_cnt = 0;
+	ntap = 0;
+	tap_start = 0;
+	tap_end = 0;
+	for (i = 0; i < host->tap_num * 2; i++) {
+		if (test_bit(i, host->taps))
+			ntap++;
+		else {
+			if (ntap > tap_cnt) {
+				tap_start = i - ntap;
+				tap_end = i - 1;
+				tap_cnt = ntap;
+			}
+			ntap = 0;
+		}
+	}
+
+	if (ntap > tap_cnt) {
+		tap_start = i - ntap;
+		tap_end = i - 1;
+		tap_cnt = ntap;
+	}
+
+	if (tap_cnt >= SH_MOBILE_SDHI_MAX_TAP)
+		tap_set = (tap_start + tap_end) / 2 % host->tap_num;
+	else
+		return -EIO;
+
+	/* Set SCC */
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TAPSET, tap_set);
+
+	/* Enable auto re-tuning */
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL,
+		       SH_MOBILE_SDHI_SCC_RVSCNTL_RVSEN |
+		       sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL));
+
+	return 0;
+}
+
+
+static bool sh_mobile_sdhi_check_scc_error(struct tmio_mmc_host *host)
+{
+	struct sh_mobile_sdhi *priv;
+
+	if (!(host->mmc->caps & MMC_CAP_UHS_SDR104))
+		return 0;
+
+	priv = host_to_priv(host);
+
+	/* Check SCC error */
+	if (sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL) &
+	    SH_MOBILE_SDHI_SCC_RVSCNTL_RVSEN &&
+	    sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_RVSREQ) &
+	    SH_MOBILE_SDHI_SCC_RVSREQ_RVSERR) {
+		/* Clear SCC error */
+		sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_RVSREQ, 0);
+		return true;
+	}
+
+	return false;
+}
+
+static void sh_mobile_sdhi_hw_reset(struct tmio_mmc_host *host)
+{
+	struct sh_mobile_sdhi *priv;
+
+	if (!(host->mmc->caps & MMC_CAP_UHS_SDR104))
+		return;
+
+	priv = host_to_priv(host);
+
+	/* Reset SCC */
+	sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, ~CLK_CTL_SCLKEN &
+			sd_ctrl_read16(host, CTL_SD_CARD_CLK_CTL));
+
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_CKSEL,
+		       ~SH_MOBILE_SDHI_SCC_CKSEL_DTSEL &
+		       sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_CKSEL));
+
+	sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, CLK_CTL_SCLKEN |
+			sd_ctrl_read16(host, CTL_SD_CARD_CLK_CTL));
+
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL,
+		       ~SH_MOBILE_SDHI_SCC_RVSCNTL_RVSEN &
+		       sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL));
+
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL,
+		       ~SH_MOBILE_SDHI_SCC_RVSCNTL_RVSEN &
+		       sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL));
+}
+
 static int sh_mobile_sdhi_wait_idle(struct tmio_mmc_host *host)
 {
 	int timeout = 1000;
@@ -325,7 +563,7 @@ static int sh_mobile_sdhi_probe(struct platform_device *pdev)
 	struct tmio_mmc_data *mmd = pdev->dev.platform_data;
 	struct tmio_mmc_host *host;
 	struct resource *res;
-	int irq, ret, i = 0;
+	int irq, ret, i;
 	struct tmio_mmc_dma *dma_priv;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -364,6 +602,7 @@ static int sh_mobile_sdhi_probe(struct platform_device *pdev)
 		const struct sh_mobile_sdhi_of_data *of_data = of_id->data;
 
 		mmc_data->flags |= of_data->tmio_flags;
+		mmc_data->ocr_mask = of_data->tmio_ocr_mask;
 		mmc_data->capabilities |= of_data->capabilities;
 		mmc_data->capabilities2 |= of_data->capabilities2;
 		mmc_data->dma_rx_offset = of_data->dma_rx_offset;
@@ -384,6 +623,11 @@ static int sh_mobile_sdhi_probe(struct platform_device *pdev)
 		host->card_busy	= sh_mobile_sdhi_card_busy;
 		host->start_signal_voltage_switch =
 			sh_mobile_sdhi_start_signal_voltage_switch;
+		host->init_tuning	= sh_mobile_sdhi_init_tuning;
+		host->prepare_tuning	= sh_mobile_sdhi_prepare_tuning;
+		host->select_tuning	= sh_mobile_sdhi_select_tuning;
+		host->check_scc_error	= sh_mobile_sdhi_check_scc_error;
+		host->hw_reset		= sh_mobile_sdhi_hw_reset;
 	}
 
 	/* Orginally registers were 16 bit apart, could be 32 or 64 nowadays */
@@ -424,6 +668,34 @@ static int sh_mobile_sdhi_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto efree;
 
+	if (host->mmc->caps & MMC_CAP_UHS_SDR104) {
+		host->mmc->caps |= MMC_CAP_HW_RESET;
+
+		if (of_id && of_id->data) {
+			const struct sh_mobile_sdhi_of_data *of_data;
+			const struct sh_mobile_sdhi_scc *taps;
+			bool hit = false;
+
+			of_data = of_id->data;
+			taps = of_data->taps;
+
+			for (i = 0; i < of_data->taps_num; i++) {
+				if (taps[i].clk_rate == 0 ||
+				    taps[i].clk_rate == host->mmc->f_max) {
+					host->scc_tappos = taps->tap;
+					hit = true;
+					break;
+				}
+			}
+
+			if (!hit)
+				dev_warn(&host->pdev->dev, "Unknown clock rate for SDR104\n");
+
+			priv->scc_ctl = host->ctl + of_data->scc_offset;
+		}
+	}
+
+	i = 0;
 	while (1) {
 		irq = platform_get_irq(pdev, i);
 		if (irq < 0)
diff --git a/drivers/mmc/host/sunxi-mmc.c b/drivers/mmc/host/sunxi-mmc.c
index c0a5c676d0e8..b1d1303389a7 100644
--- a/drivers/mmc/host/sunxi-mmc.c
+++ b/drivers/mmc/host/sunxi-mmc.c
@@ -822,10 +822,13 @@ static void sunxi_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 		break;
 
 	case MMC_POWER_UP:
-		host->ferror = mmc_regulator_set_ocr(mmc, mmc->supply.vmmc,
-						     ios->vdd);
-		if (host->ferror)
-			return;
+		if (!IS_ERR(mmc->supply.vmmc)) {
+			host->ferror = mmc_regulator_set_ocr(mmc,
+							     mmc->supply.vmmc,
+							     ios->vdd);
+			if (host->ferror)
+				return;
+		}
 
 		if (!IS_ERR(mmc->supply.vqmmc)) {
 			host->ferror = regulator_enable(mmc->supply.vqmmc);
@@ -847,7 +850,9 @@ static void sunxi_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 	case MMC_POWER_OFF:
 		dev_dbg(mmc_dev(mmc), "power off!\n");
 		sunxi_mmc_reset_host(host);
-		mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
+		if (!IS_ERR(mmc->supply.vmmc))
+			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0);
+
 		if (!IS_ERR(mmc->supply.vqmmc) && host->vqmmc_enabled)
 			regulator_disable(mmc->supply.vqmmc);
 		host->vqmmc_enabled = false;
diff --git a/drivers/mmc/host/tmio_mmc.h b/drivers/mmc/host/tmio_mmc.h
index 8e126afd988c..9e20bcf3aa8d 100644
--- a/drivers/mmc/host/tmio_mmc.h
+++ b/drivers/mmc/host/tmio_mmc.h
@@ -153,9 +153,12 @@ struct tmio_mmc_host {
 	struct mutex		ios_lock;	/* protect set_ios() context */
 	bool			native_hotplug;
 	bool			sdio_irq_enabled;
+	u32			scc_tappos;
 
-	int (*write16_hook)(struct tmio_mmc_host *host, int addr);
+	/* Mandatory callback */
 	int (*clk_enable)(struct tmio_mmc_host *host);
+
+	/* Optional callbacks */
 	unsigned int (*clk_update)(struct tmio_mmc_host *host,
 				   unsigned int new_clock);
 	void (*clk_disable)(struct tmio_mmc_host *host);
@@ -164,6 +167,21 @@ struct tmio_mmc_host {
 	int (*card_busy)(struct mmc_host *mmc);
 	int (*start_signal_voltage_switch)(struct mmc_host *mmc,
 					   struct mmc_ios *ios);
+	int (*write16_hook)(struct tmio_mmc_host *host, int addr);
+	void (*hw_reset)(struct tmio_mmc_host *host);
+	void (*prepare_tuning)(struct tmio_mmc_host *host, unsigned long tap);
+	bool (*check_scc_error)(struct tmio_mmc_host *host);
+
+	/*
+	 * Mandatory callback for tuning to occur which is optional for SDR50
+	 * and mandatory for SDR104.
+	 */
+	unsigned int (*init_tuning)(struct tmio_mmc_host *host);
+	int (*select_tuning)(struct tmio_mmc_host *host);
+
+	/* Tuning values: 1 for success, 0 for failure */
+	DECLARE_BITMAP(taps, BITS_PER_BYTE * sizeof(long));
+	unsigned int tap_num;
 };
 
 struct tmio_mmc_host *tmio_mmc_host_alloc(struct platform_device *pdev);
@@ -245,6 +263,12 @@ static inline u32 sd_ctrl_read16_and_16_as_32(struct tmio_mmc_host *host, int ad
 	       readw(host->ctl + ((addr + 2) << host->bus_shift)) << 16;
 }
 
+static inline void sd_ctrl_read32_rep(struct tmio_mmc_host *host, int addr,
+		u32 *buf, int count)
+{
+	readsl(host->ctl + (addr << host->bus_shift), buf, count);
+}
+
 static inline void sd_ctrl_write16(struct tmio_mmc_host *host, int addr, u16 val)
 {
 	/* If there is a hook and it returns non-zero then there
@@ -267,4 +291,10 @@ static inline void sd_ctrl_write32_as_16_and_16(struct tmio_mmc_host *host, int
 	writew(val >> 16, host->ctl + ((addr + 2) << host->bus_shift));
 }
 
+static inline void sd_ctrl_write32_rep(struct tmio_mmc_host *host, int addr,
+		const u32 *buf, int count)
+{
+	writesl(host->ctl + (addr << host->bus_shift), buf, count);
+}
+
 #endif
diff --git a/drivers/mmc/host/tmio_mmc_pio.c b/drivers/mmc/host/tmio_mmc_pio.c
index 700567603107..2064fa1a5bf1 100644
--- a/drivers/mmc/host/tmio_mmc_pio.c
+++ b/drivers/mmc/host/tmio_mmc_pio.c
@@ -22,7 +22,6 @@
  * TODO:
  *   Investigate using a workqueue for PIO transfers
  *   Eliminate FIXMEs
- *   SDIO support
  *   Better Power management
  *   Handle MMC errors better
  *   double buffer support
@@ -36,6 +35,7 @@
 #include <linux/io.h>
 #include <linux/irq.h>
 #include <linux/mfd/tmio.h>
+#include <linux/mmc/card.h>
 #include <linux/mmc/host.h>
 #include <linux/mmc/mmc.h>
 #include <linux/mmc/slot-gpio.h>
@@ -298,6 +298,9 @@ static void tmio_mmc_finish_request(struct tmio_mmc_host *host)
 	if (mrq->cmd->error || (mrq->data && mrq->data->error))
 		tmio_mmc_abort_dma(host);
 
+	if (host->check_scc_error)
+		host->check_scc_error(host);
+
 	mmc_request_done(host->mmc, mrq);
 }
 
@@ -393,6 +396,36 @@ static void tmio_mmc_transfer_data(struct tmio_mmc_host *host,
 	/*
 	 * Transfer the data
 	 */
+	if (host->pdata->flags & TMIO_MMC_32BIT_DATA_PORT) {
+		u8 data[4] = { };
+
+		if (is_read)
+			sd_ctrl_read32_rep(host, CTL_SD_DATA_PORT, (u32 *)buf,
+					   count >> 2);
+		else
+			sd_ctrl_write32_rep(host, CTL_SD_DATA_PORT, (u32 *)buf,
+					    count >> 2);
+
+		/* if count was multiple of 4 */
+		if (!(count & 0x3))
+			return;
+
+		buf8 = (u8 *)(buf + (count >> 2));
+		count %= 4;
+
+		if (is_read) {
+			sd_ctrl_read32_rep(host, CTL_SD_DATA_PORT,
+					   (u32 *)data, 1);
+			memcpy(buf8, data, count);
+		} else {
+			memcpy(data, buf8, count);
+			sd_ctrl_write32_rep(host, CTL_SD_DATA_PORT,
+					    (u32 *)data, 1);
+		}
+
+		return;
+	}
+
 	if (is_read)
 		sd_ctrl_read16_rep(host, CTL_SD_DATA_PORT, buf, count >> 1);
 	else
@@ -522,7 +555,7 @@ void tmio_mmc_do_data_irq(struct tmio_mmc_host *host)
 	schedule_work(&host->done);
 }
 
-static void tmio_mmc_data_irq(struct tmio_mmc_host *host)
+static void tmio_mmc_data_irq(struct tmio_mmc_host *host, unsigned int stat)
 {
 	struct mmc_data *data;
 	spin_lock(&host->lock);
@@ -531,6 +564,9 @@ static void tmio_mmc_data_irq(struct tmio_mmc_host *host)
 	if (!data)
 		goto out;
 
+	if (stat & TMIO_STAT_CRCFAIL || stat & TMIO_STAT_STOPBIT_ERR ||
+	    stat & TMIO_STAT_TXUNDERRUN)
+		data->error = -EILSEQ;
 	if (host->chan_tx && (data->flags & MMC_DATA_WRITE) && !host->force_pio) {
 		u32 status = sd_ctrl_read16_and_16_as_32(host, CTL_STATUS);
 		bool done = false;
@@ -579,8 +615,6 @@ static void tmio_mmc_cmd_irq(struct tmio_mmc_host *host,
 		goto out;
 	}
 
-	host->cmd = NULL;
-
 	/* This controller is sicker than the PXA one. Not only do we need to
 	 * drop the top 8 bits of the first response word, we also need to
 	 * modify the order of the response for short response command types.
@@ -600,14 +634,16 @@ static void tmio_mmc_cmd_irq(struct tmio_mmc_host *host,
 
 	if (stat & TMIO_STAT_CMDTIMEOUT)
 		cmd->error = -ETIMEDOUT;
-	else if (stat & TMIO_STAT_CRCFAIL && cmd->flags & MMC_RSP_CRC)
+	else if ((stat & TMIO_STAT_CRCFAIL && cmd->flags & MMC_RSP_CRC) ||
+		 stat & TMIO_STAT_STOPBIT_ERR ||
+		 stat & TMIO_STAT_CMD_IDX_ERR)
 		cmd->error = -EILSEQ;
 
 	/* If there is data to handle we enable data IRQs here, and
 	 * we will ultimatley finish the request in the data_end handler.
 	 * If theres no data or we encountered an error, finish now.
 	 */
-	if (host->data && !cmd->error) {
+	if (host->data && (!cmd->error || cmd->error == -EILSEQ)) {
 		if (host->data->flags & MMC_DATA_READ) {
 			if (host->force_pio || !host->chan_rx)
 				tmio_mmc_enable_mmc_irqs(host, TMIO_MASK_READOP);
@@ -668,7 +704,7 @@ static bool __tmio_mmc_sdcard_irq(struct tmio_mmc_host *host,
 	/* Data transfer completion */
 	if (ireg & TMIO_STAT_DATAEND) {
 		tmio_mmc_ack_mmc_irqs(host, TMIO_STAT_DATAEND);
-		tmio_mmc_data_irq(host);
+		tmio_mmc_data_irq(host, status);
 		return true;
 	}
 
@@ -687,7 +723,7 @@ static void tmio_mmc_sdio_irq(int irq, void *devid)
 		return;
 
 	status = sd_ctrl_read16(host, CTL_SDIO_STATUS);
-	ireg = status & TMIO_SDIO_MASK_ALL & ~host->sdcard_irq_mask;
+	ireg = status & TMIO_SDIO_MASK_ALL & ~host->sdio_irq_mask;
 
 	sdio_status = status & ~TMIO_SDIO_MASK_ALL;
 	if (pdata->flags & TMIO_MMC_SDIO_STATUS_QUIRK)
@@ -756,6 +792,63 @@ static int tmio_mmc_start_data(struct tmio_mmc_host *host,
 	return 0;
 }
 
+static void tmio_mmc_hw_reset(struct mmc_host *mmc)
+{
+	struct tmio_mmc_host *host = mmc_priv(mmc);
+
+	if (host->hw_reset)
+		host->hw_reset(host);
+}
+
+static int tmio_mmc_execute_tuning(struct mmc_host *mmc, u32 opcode)
+{
+	struct tmio_mmc_host *host = mmc_priv(mmc);
+	int i, ret = 0;
+
+	if (!host->tap_num) {
+		if (!host->init_tuning || !host->select_tuning)
+			/* Tuning is not supported */
+			goto out;
+
+		host->tap_num = host->init_tuning(host);
+		if (!host->tap_num)
+			/* Tuning is not supported */
+			goto out;
+	}
+
+	if (host->tap_num * 2 >= sizeof(host->taps) * BITS_PER_BYTE) {
+		dev_warn_once(&host->pdev->dev,
+		      "Too many taps, skipping tuning. Please consider updating size of taps field of tmio_mmc_host\n");
+		goto out;
+	}
+
+	bitmap_zero(host->taps, host->tap_num * 2);
+
+	/* Issue CMD19 twice for each tap */
+	for (i = 0; i < 2 * host->tap_num; i++) {
+		if (host->prepare_tuning)
+			host->prepare_tuning(host, i % host->tap_num);
+
+		ret = mmc_send_tuning(mmc, opcode, NULL);
+		if (ret && ret != -EILSEQ)
+			goto out;
+		if (ret == 0)
+			set_bit(i, host->taps);
+
+		mdelay(1);
+	}
+
+	ret = host->select_tuning(host);
+
+out:
+	if (ret < 0) {
+		dev_warn(&host->pdev->dev, "Tuning procedure failed\n");
+		tmio_mmc_hw_reset(mmc);
+	}
+
+	return ret;
+}
+
 /* Process requests from the MMC layer */
 static void tmio_mmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
 {
@@ -972,6 +1065,8 @@ static struct mmc_host_ops tmio_mmc_ops = {
 	.get_cd		= mmc_gpio_get_cd,
 	.enable_sdio_irq = tmio_mmc_enable_sdio_irq,
 	.multi_io_quirk	= tmio_multi_io_quirk,
+	.hw_reset	= tmio_mmc_hw_reset,
+	.execute_tuning = tmio_mmc_execute_tuning,
 };
 
 static int tmio_mmc_init_ocr(struct tmio_mmc_host *host)
@@ -1218,6 +1313,11 @@ int tmio_mmc_host_runtime_suspend(struct device *dev)
 }
 EXPORT_SYMBOL(tmio_mmc_host_runtime_suspend);
 
+static bool tmio_mmc_can_retune(struct tmio_mmc_host *host)
+{
+	return host->tap_num && mmc_can_retune(host->mmc);
+}
+
 int tmio_mmc_host_runtime_resume(struct device *dev)
 {
 	struct mmc_host *mmc = dev_get_drvdata(dev);
@@ -1231,6 +1331,9 @@ int tmio_mmc_host_runtime_resume(struct device *dev)
 
 	tmio_mmc_enable_dma(host, true);
 
+	if (tmio_mmc_can_retune(host) && host->select_tuning(host))
+		dev_warn(&host->pdev->dev, "Tuning selection failed\n");
+
 	return 0;
 }
 EXPORT_SYMBOL(tmio_mmc_host_runtime_resume);
diff --git a/drivers/mmc/host/wbsd.c b/drivers/mmc/host/wbsd.c
index c3fd16d997ca..80a3b11f3217 100644
--- a/drivers/mmc/host/wbsd.c
+++ b/drivers/mmc/host/wbsd.c
@@ -1395,23 +1395,25 @@ static void wbsd_request_dma(struct wbsd_host *host, int dma)
 	 */
 	host->dma_addr = dma_map_single(mmc_dev(host->mmc), host->dma_buffer,
 		WBSD_DMA_SIZE, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(mmc_dev(host->mmc), host->dma_addr))
+		goto kfree;
 
 	/*
 	 * ISA DMA must be aligned on a 64k basis.
 	 */
 	if ((host->dma_addr & 0xffff) != 0)
-		goto kfree;
+		goto unmap;
 	/*
 	 * ISA cannot access memory above 16 MB.
 	 */
 	else if (host->dma_addr >= 0x1000000)
-		goto kfree;
+		goto unmap;
 
 	host->dma = dma;
 
 	return;
 
-kfree:
+unmap:
 	/*
 	 * If we've gotten here then there is some kind of alignment bug
 	 */
@@ -1421,6 +1423,7 @@ kfree:
 		WBSD_DMA_SIZE, DMA_BIDIRECTIONAL);
 	host->dma_addr = 0;
 
+kfree:
 	kfree(host->dma_buffer);
 	host->dma_buffer = NULL;
 
@@ -1434,7 +1437,7 @@ err:
 
 static void wbsd_release_dma(struct wbsd_host *host)
 {
-	if (host->dma_addr) {
+	if (!dma_mapping_error(mmc_dev(host->mmc), host->dma_addr)) {
 		dma_unmap_single(mmc_dev(host->mmc), host->dma_addr,
 			WBSD_DMA_SIZE, DMA_BIDIRECTIONAL);
 	}
diff --git a/drivers/net/can/Makefile b/drivers/net/can/Makefile
index 26ba4b794a0b..7a85495dbb0c 100644
--- a/drivers/net/can/Makefile
+++ b/drivers/net/can/Makefile
@@ -31,5 +31,4 @@ obj-$(CONFIG_CAN_TI_HECC)	+= ti_hecc.o
 obj-$(CONFIG_CAN_XILINXCAN)	+= xilinx_can.o
 obj-$(CONFIG_PCH_CAN)		+= pch_can.o
 
-subdir-ccflags-y += -D__CHECK_ENDIAN__
 subdir-ccflags-$(CONFIG_CAN_DEBUG_DEVICES) += -DDEBUG
diff --git a/drivers/net/ethernet/alacritech/slicoss.c b/drivers/net/ethernet/alacritech/slicoss.c
index b9fbd0107008..b21d8aa8d653 100644
--- a/drivers/net/ethernet/alacritech/slicoss.c
+++ b/drivers/net/ethernet/alacritech/slicoss.c
@@ -1567,7 +1567,6 @@ static const struct net_device_ops slic_netdev_ops = {
 	.ndo_set_mac_address	= eth_mac_addr,
 	.ndo_get_stats64	= slic_get_stats,
 	.ndo_set_rx_mode	= slic_set_rx_mode,
-	.ndo_change_mtu		= eth_change_mtu,
 	.ndo_validate_addr	= eth_validate_addr,
 };
 
diff --git a/drivers/net/ethernet/altera/Makefile b/drivers/net/ethernet/altera/Makefile
index 3eff2fd3997e..d4a187e45369 100644
--- a/drivers/net/ethernet/altera/Makefile
+++ b/drivers/net/ethernet/altera/Makefile
@@ -5,4 +5,3 @@
 obj-$(CONFIG_ALTERA_TSE) += altera_tse.o
 altera_tse-objs := altera_tse_main.o altera_tse_ethtool.o \
 altera_msgdma.o altera_sgdma.o altera_utils.o
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/ethernet/atheros/alx/Makefile b/drivers/net/ethernet/atheros/alx/Makefile
index 5901fa407d52..ed4a605874a3 100644
--- a/drivers/net/ethernet/atheros/alx/Makefile
+++ b/drivers/net/ethernet/atheros/alx/Makefile
@@ -1,3 +1,2 @@
 obj-$(CONFIG_ALX) += alx.o
 alx-objs := main.o ethtool.o hw.o
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
index 3f77d0863543..6fad22adbbb9 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
@@ -585,7 +585,7 @@ int bnx2x_vf_mcast(struct bnx2x *bp, struct bnx2x_virtf *vf,
 		mcast.mcast_list_len = mc_num;
 		rc = bnx2x_config_mcast(bp, &mcast, BNX2X_MCAST_CMD_SET);
 		if (rc)
-			BNX2X_ERR("Faled to set multicasts\n");
+			BNX2X_ERR("Failed to set multicasts\n");
 	} else {
 		/* clear existing mcasts */
 		rc = bnx2x_config_mcast(bp, &mcast, BNX2X_MCAST_CMD_DEL);
diff --git a/drivers/net/ethernet/freescale/Makefile b/drivers/net/ethernet/freescale/Makefile
index 4a13115155c9..c46df5c82af5 100644
--- a/drivers/net/ethernet/freescale/Makefile
+++ b/drivers/net/ethernet/freescale/Makefile
@@ -4,8 +4,6 @@
 
 obj-$(CONFIG_FEC) += fec.o
 fec-objs :=fec_main.o fec_ptp.o
-CFLAGS_fec_main.o := -D__CHECK_ENDIAN__
-CFLAGS_fec_ptp.o := -D__CHECK_ENDIAN__
 
 obj-$(CONFIG_FEC_MPC52xx) += fec_mpc52xx.o
 ifeq ($(CONFIG_FEC_MPC52xx_MDIO),y)
diff --git a/drivers/net/ethernet/intel/igb/igb.h b/drivers/net/ethernet/intel/igb/igb.h
index d11093dce1b9..acbc3abe2ddd 100644
--- a/drivers/net/ethernet/intel/igb/igb.h
+++ b/drivers/net/ethernet/intel/igb/igb.h
@@ -210,7 +210,12 @@ struct igb_tx_buffer {
 struct igb_rx_buffer {
 	dma_addr_t dma;
 	struct page *page;
-	unsigned int page_offset;
+#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
+	__u32 page_offset;
+#else
+	__u16 page_offset;
+#endif
+	__u16 pagecnt_bias;
 };
 
 struct igb_tx_queue_stats {
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index cae24a8ccf47..a761001308dc 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -3947,11 +3947,23 @@ static void igb_clean_rx_ring(struct igb_ring *rx_ring)
 		if (!buffer_info->page)
 			continue;
 
-		dma_unmap_page(rx_ring->dev,
-			       buffer_info->dma,
-			       PAGE_SIZE,
-			       DMA_FROM_DEVICE);
-		__free_page(buffer_info->page);
+		/* Invalidate cache lines that may have been written to by
+		 * device so that we avoid corrupting memory.
+		 */
+		dma_sync_single_range_for_cpu(rx_ring->dev,
+					      buffer_info->dma,
+					      buffer_info->page_offset,
+					      IGB_RX_BUFSZ,
+					      DMA_FROM_DEVICE);
+
+		/* free resources associated with mapping */
+		dma_unmap_page_attrs(rx_ring->dev,
+				     buffer_info->dma,
+				     PAGE_SIZE,
+				     DMA_FROM_DEVICE,
+				     DMA_ATTR_SKIP_CPU_SYNC);
+		__page_frag_drain(buffer_info->page, 0,
+				  buffer_info->pagecnt_bias);
 
 		buffer_info->page = NULL;
 	}
@@ -6812,12 +6824,6 @@ static void igb_reuse_rx_page(struct igb_ring *rx_ring,
 
 	/* transfer page from old buffer to new buffer */
 	*new_buff = *old_buff;
-
-	/* sync the buffer for use by the device */
-	dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma,
-					 old_buff->page_offset,
-					 IGB_RX_BUFSZ,
-					 DMA_FROM_DEVICE);
 }
 
 static inline bool igb_page_is_reserved(struct page *page)
@@ -6829,13 +6835,15 @@ static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
 				  struct page *page,
 				  unsigned int truesize)
 {
+	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias--;
+
 	/* avoid re-using remote pages */
 	if (unlikely(igb_page_is_reserved(page)))
 		return false;
 
 #if (PAGE_SIZE < 8192)
 	/* if we are only owner of page we can reuse it */
-	if (unlikely(page_count(page) != 1))
+	if (unlikely(page_ref_count(page) != pagecnt_bias))
 		return false;
 
 	/* flip page offset to other buffer */
@@ -6848,10 +6856,14 @@ static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
 		return false;
 #endif
 
-	/* Even if we own the page, we are not allowed to use atomic_set()
-	 * This would break get_page_unless_zero() users.
+	/* If we have drained the page fragment pool we need to update
+	 * the pagecnt_bias and page count so that we fully restock the
+	 * number of references the driver holds.
 	 */
-	page_ref_inc(page);
+	if (unlikely(pagecnt_bias == 1)) {
+		page_ref_add(page, USHRT_MAX);
+		rx_buffer->pagecnt_bias = USHRT_MAX;
+	}
 
 	return true;
 }
@@ -6903,7 +6915,6 @@ static bool igb_add_rx_frag(struct igb_ring *rx_ring,
 			return true;
 
 		/* this page cannot be reused so discard it */
-		__free_page(page);
 		return false;
 	}
 
@@ -6938,6 +6949,13 @@ static struct sk_buff *igb_fetch_rx_buffer(struct igb_ring *rx_ring,
 	page = rx_buffer->page;
 	prefetchw(page);
 
+	/* we are reusing so sync this buffer for CPU use */
+	dma_sync_single_range_for_cpu(rx_ring->dev,
+				      rx_buffer->dma,
+				      rx_buffer->page_offset,
+				      size,
+				      DMA_FROM_DEVICE);
+
 	if (likely(!skb)) {
 		void *page_addr = page_address(page) +
 				  rx_buffer->page_offset;
@@ -6962,21 +6980,18 @@ static struct sk_buff *igb_fetch_rx_buffer(struct igb_ring *rx_ring,
 		prefetchw(skb->data);
 	}
 
-	/* we are reusing so sync this buffer for CPU use */
-	dma_sync_single_range_for_cpu(rx_ring->dev,
-				      rx_buffer->dma,
-				      rx_buffer->page_offset,
-				      size,
-				      DMA_FROM_DEVICE);
-
 	/* pull page into skb */
 	if (igb_add_rx_frag(rx_ring, rx_buffer, size, rx_desc, skb)) {
 		/* hand second half of page back to the ring */
 		igb_reuse_rx_page(rx_ring, rx_buffer);
 	} else {
-		/* we are not reusing the buffer so unmap it */
-		dma_unmap_page(rx_ring->dev, rx_buffer->dma,
-			       PAGE_SIZE, DMA_FROM_DEVICE);
+		/* We are not reusing the buffer so unmap it and free
+		 * any references we are holding to it
+		 */
+		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
+				     PAGE_SIZE, DMA_FROM_DEVICE,
+				     DMA_ATTR_SKIP_CPU_SYNC);
+		__page_frag_drain(page, 0, rx_buffer->pagecnt_bias);
 	}
 
 	/* clear contents of rx_buffer */
@@ -7234,7 +7249,8 @@ static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
 	}
 
 	/* map page for use */
-	dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
+	dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE,
+				 DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
 
 	/* if mapping failed free memory back to system since
 	 * there isn't much point in holding memory we can't use
@@ -7249,6 +7265,7 @@ static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
 	bi->dma = dma;
 	bi->page = page;
 	bi->page_offset = 0;
+	bi->pagecnt_bias = 1;
 
 	return true;
 }
@@ -7275,6 +7292,12 @@ void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
 		if (!igb_alloc_mapped_page(rx_ring, bi))
 			break;
 
+		/* sync the buffer for use by the device */
+		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
+						 bi->page_offset,
+						 IGB_RX_BUFSZ,
+						 DMA_FROM_DEVICE);
+
 		/* Refresh the desc even if buffer_addrs didn't change
 		 * because each write-back erases this info.
 		 */
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index 75d07fa9d0b1..b2ca8a635b2e 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -4020,49 +4020,51 @@ int mlx4_restart_one(struct pci_dev *pdev)
 	return err;
 }
 
+#define MLX_SP(id) { PCI_VDEVICE(MELLANOX, id), MLX4_PCI_DEV_FORCE_SENSE_PORT }
+#define MLX_VF(id) { PCI_VDEVICE(MELLANOX, id), MLX4_PCI_DEV_IS_VF }
+#define MLX_GN(id) { PCI_VDEVICE(MELLANOX, id), 0 }
+
 static const struct pci_device_id mlx4_pci_table[] = {
-	/* MT25408 "Hermon" SDR */
-	{ PCI_VDEVICE(MELLANOX, 0x6340), MLX4_PCI_DEV_FORCE_SENSE_PORT },
-	/* MT25408 "Hermon" DDR */
-	{ PCI_VDEVICE(MELLANOX, 0x634a), MLX4_PCI_DEV_FORCE_SENSE_PORT },
-	/* MT25408 "Hermon" QDR */
-	{ PCI_VDEVICE(MELLANOX, 0x6354), MLX4_PCI_DEV_FORCE_SENSE_PORT },
-	/* MT25408 "Hermon" DDR PCIe gen2 */
-	{ PCI_VDEVICE(MELLANOX, 0x6732), MLX4_PCI_DEV_FORCE_SENSE_PORT },
-	/* MT25408 "Hermon" QDR PCIe gen2 */
-	{ PCI_VDEVICE(MELLANOX, 0x673c), MLX4_PCI_DEV_FORCE_SENSE_PORT },
-	/* MT25408 "Hermon" EN 10GigE */
-	{ PCI_VDEVICE(MELLANOX, 0x6368), MLX4_PCI_DEV_FORCE_SENSE_PORT },
-	/* MT25408 "Hermon" EN 10GigE PCIe gen2 */
-	{ PCI_VDEVICE(MELLANOX, 0x6750), MLX4_PCI_DEV_FORCE_SENSE_PORT },
-	/* MT25458 ConnectX EN 10GBASE-T 10GigE */
-	{ PCI_VDEVICE(MELLANOX, 0x6372), MLX4_PCI_DEV_FORCE_SENSE_PORT },
-	/* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */
-	{ PCI_VDEVICE(MELLANOX, 0x675a), MLX4_PCI_DEV_FORCE_SENSE_PORT },
-	/* MT26468 ConnectX EN 10GigE PCIe gen2*/
-	{ PCI_VDEVICE(MELLANOX, 0x6764), MLX4_PCI_DEV_FORCE_SENSE_PORT },
-	/* MT26438 ConnectX EN 40GigE PCIe gen2 5GT/s */
-	{ PCI_VDEVICE(MELLANOX, 0x6746), MLX4_PCI_DEV_FORCE_SENSE_PORT },
-	/* MT26478 ConnectX2 40GigE PCIe gen2 */
-	{ PCI_VDEVICE(MELLANOX, 0x676e), MLX4_PCI_DEV_FORCE_SENSE_PORT },
-	/* MT25400 Family [ConnectX-2 Virtual Function] */
-	{ PCI_VDEVICE(MELLANOX, 0x1002), MLX4_PCI_DEV_IS_VF },
+	/* MT25408 "Hermon" */
+	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_SDR),	/* SDR */
+	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_DDR),	/* DDR */
+	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_QDR),	/* QDR */
+	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_DDR_GEN2), /* DDR Gen2 */
+	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_QDR_GEN2),	/* QDR Gen2 */
+	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_EN),	/* EN 10GigE */
+	MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_EN_GEN2),  /* EN 10GigE Gen2 */
+	/* MT25458 ConnectX EN 10GBASE-T */
+	MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX_EN),
+	MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_T_GEN2),	/* Gen2 */
+	/* MT26468 ConnectX EN 10GigE PCIe Gen2*/
+	MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_GEN2),
+	/* MT26438 ConnectX EN 40GigE PCIe Gen2 5GT/s */
+	MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_5_GEN2),
+	/* MT26478 ConnectX2 40GigE PCIe Gen2 */
+	MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX2),
+	/* MT25400 Family [ConnectX-2] */
+	MLX_VF(0x1002),					/* Virtual Function */
 	/* MT27500 Family [ConnectX-3] */
-	{ PCI_VDEVICE(MELLANOX, 0x1003), 0 },
-	/* MT27500 Family [ConnectX-3 Virtual Function] */
-	{ PCI_VDEVICE(MELLANOX, 0x1004), MLX4_PCI_DEV_IS_VF },
-	{ PCI_VDEVICE(MELLANOX, 0x1005), 0 }, /* MT27510 Family */
-	{ PCI_VDEVICE(MELLANOX, 0x1006), 0 }, /* MT27511 Family */
-	{ PCI_VDEVICE(MELLANOX, 0x1007), 0 }, /* MT27520 Family */
-	{ PCI_VDEVICE(MELLANOX, 0x1008), 0 }, /* MT27521 Family */
-	{ PCI_VDEVICE(MELLANOX, 0x1009), 0 }, /* MT27530 Family */
-	{ PCI_VDEVICE(MELLANOX, 0x100a), 0 }, /* MT27531 Family */
-	{ PCI_VDEVICE(MELLANOX, 0x100b), 0 }, /* MT27540 Family */
-	{ PCI_VDEVICE(MELLANOX, 0x100c), 0 }, /* MT27541 Family */
-	{ PCI_VDEVICE(MELLANOX, 0x100d), 0 }, /* MT27550 Family */
-	{ PCI_VDEVICE(MELLANOX, 0x100e), 0 }, /* MT27551 Family */
-	{ PCI_VDEVICE(MELLANOX, 0x100f), 0 }, /* MT27560 Family */
-	{ PCI_VDEVICE(MELLANOX, 0x1010), 0 }, /* MT27561 Family */
+	MLX_GN(PCI_DEVICE_ID_MELLANOX_CONNECTX3),
+	MLX_VF(0x1004),					/* Virtual Function */
+	MLX_GN(0x1005),					/* MT27510 Family */
+	MLX_GN(0x1006),					/* MT27511 Family */
+	MLX_GN(PCI_DEVICE_ID_MELLANOX_CONNECTX3_PRO),	/* MT27520 Family */
+	MLX_GN(0x1008),					/* MT27521 Family */
+	MLX_GN(0x1009),					/* MT27530 Family */
+	MLX_GN(0x100a),					/* MT27531 Family */
+	MLX_GN(0x100b),					/* MT27540 Family */
+	MLX_GN(0x100c),					/* MT27541 Family */
+	MLX_GN(0x100d),					/* MT27550 Family */
+	MLX_GN(0x100e),					/* MT27551 Family */
+	MLX_GN(0x100f),					/* MT27560 Family */
+	MLX_GN(0x1010),					/* MT27561 Family */
+
+	/*
+	 * See the mellanox_check_broken_intx_masking() quirk when
+	 * adding devices
+	 */
+
 	{ 0, }
 };
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.c b/drivers/net/ethernet/qlogic/qed/qed_int.c
index bb74e1c10ffe..c68dbf7092b1 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_int.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_int.c
@@ -1377,7 +1377,7 @@ static const char *attn_master_to_str(u8 master)
 	case 9: return "DBU";
 	case 10: return "DMAE";
 	default:
-		return "Unkown";
+		return "Unknown";
 	}
 }
 
@@ -1555,7 +1555,7 @@ static int qed_dorq_attn_cb(struct qed_hwfn *p_hwfn)
 				     DORQ_REG_DB_DROP_DETAILS);
 
 		DP_INFO(p_hwfn->cdev,
-			"DORQ db_drop: adress 0x%08x Opaque FID 0x%04x Size [bytes] 0x%08x Reason: 0x%08x\n",
+			"DORQ db_drop: address 0x%08x Opaque FID 0x%04x Size [bytes] 0x%08x Reason: 0x%08x\n",
 			qed_rd(p_hwfn, p_hwfn->p_dpc_ptt,
 			       DORQ_REG_DB_DROP_DETAILS_ADDRESS),
 			(u16)(details & QED_DORQ_ATTENTION_OPAQUE_MASK),
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
index d0a58282f2a8..a39ef2e7a9a6 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
@@ -369,7 +369,7 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
 		p_ramrod->personality = PERSONALITY_RDMA_AND_ETH;
 		break;
 	default:
-		DP_NOTICE(p_hwfn, "Unkown personality %d\n",
+		DP_NOTICE(p_hwfn, "Unknown personality %d\n",
 			  p_hwfn->hw_info.personality);
 		p_ramrod->personality = PERSONALITY_ETH;
 	}
diff --git a/drivers/net/ethernet/qlogic/qede/qede_roce.c b/drivers/net/ethernet/qlogic/qede/qede_roce.c
index 9867f960b063..49272716a7c4 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_roce.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_roce.c
@@ -191,8 +191,8 @@ int qede_roce_register_driver(struct qedr_driver *drv)
 	}
 	mutex_unlock(&qedr_dev_list_lock);
 
-	DP_INFO(edev, "qedr: discovered and registered %d RoCE funcs\n",
-		qedr_counter);
+	pr_notice("qedr: discovered and registered %d RoCE funcs\n",
+		  qedr_counter);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/smsc/smsc911x.c b/drivers/net/ethernet/smsc/smsc911x.c
index c48fc0c4abd9..fa5ca0992be6 100644
--- a/drivers/net/ethernet/smsc/smsc911x.c
+++ b/drivers/net/ethernet/smsc/smsc911x.c
@@ -2585,6 +2585,9 @@ static int smsc911x_suspend(struct device *dev)
 		PMT_CTRL_PM_MODE_D1_ | PMT_CTRL_WOL_EN_ |
 		PMT_CTRL_ED_EN_ | PMT_CTRL_PME_EN_);
 
+	pm_runtime_disable(dev);
+	pm_runtime_set_suspended(dev);
+
 	return 0;
 }
 
@@ -2594,6 +2597,9 @@ static int smsc911x_resume(struct device *dev)
 	struct smsc911x_data *pdata = netdev_priv(ndev);
 	unsigned int to = 100;
 
+	pm_runtime_enable(dev);
+	pm_runtime_resume(dev);
+
 	/* Note 3.11 from the datasheet:
 	 * 	"When the LAN9220 is in a power saving state, a write of any
 	 * 	 data to the BYTE_TEST register will wake-up the device."
diff --git a/drivers/net/ethernet/sun/sunhme.h b/drivers/net/ethernet/sun/sunhme.h
index f4307654e4ae..4a8d5b18dfd5 100644
--- a/drivers/net/ethernet/sun/sunhme.h
+++ b/drivers/net/ethernet/sun/sunhme.h
@@ -302,7 +302,7 @@
  * Always write the address first before setting the ownership
  * bits to avoid races with the hardware scanning the ring.
  */
-typedef u32 __bitwise__ hme32;
+typedef u32 __bitwise hme32;
 
 struct happy_meal_rxd {
 	hme32 rx_flags;
diff --git a/drivers/net/vmxnet3/vmxnet3_int.h b/drivers/net/vmxnet3/vmxnet3_int.h
index 7dc37a090549..59e077be8829 100644
--- a/drivers/net/vmxnet3/vmxnet3_int.h
+++ b/drivers/net/vmxnet3/vmxnet3_int.h
@@ -119,9 +119,8 @@ enum {
 };
 
 /*
- * PCI vendor and device IDs.
+ * Maximum devices supported.
  */
-#define PCI_DEVICE_ID_VMWARE_VMXNET3    0x07B0
 #define MAX_ETHERNET_CARDS		10
 #define MAX_PCI_PASSTHRU_DEVICE		6
 
diff --git a/drivers/net/wireless/ath/Makefile b/drivers/net/wireless/ath/Makefile
index 89f8d5979402..4cdebc7659dd 100644
--- a/drivers/net/wireless/ath/Makefile
+++ b/drivers/net/wireless/ath/Makefile
@@ -19,6 +19,4 @@ ath-objs :=	main.o \
 ath-$(CONFIG_ATH_DEBUG) += debug.o
 ath-$(CONFIG_ATH_TRACEPOINTS) += trace.o
 
-ccflags-y += -D__CHECK_ENDIAN__
-
 CFLAGS_trace.o := -I$(src)
diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c
index 0457e315d336..b541a1c74488 100644
--- a/drivers/net/wireless/ath/ath10k/pci.c
+++ b/drivers/net/wireless/ath/ath10k/pci.c
@@ -2091,7 +2091,7 @@ int ath10k_pci_init_config(struct ath10k *ar)
 
 	ret = ath10k_pci_diag_read32(ar, ealloc_targ_addr, &ealloc_value);
 	if (ret != 0) {
-		ath10k_err(ar, "Faile to get early alloc val: %d\n", ret);
+		ath10k_err(ar, "Failed to get early alloc val: %d\n", ret);
 		return ret;
 	}
 
diff --git a/drivers/net/wireless/ath/wil6210/Makefile b/drivers/net/wireless/ath/wil6210/Makefile
index 11b544b26c74..89bf2f9eca1d 100644
--- a/drivers/net/wireless/ath/wil6210/Makefile
+++ b/drivers/net/wireless/ath/wil6210/Makefile
@@ -22,5 +22,3 @@ wil6210-y += p2p.o
 
 # for tracing framework to find trace.h
 CFLAGS_trace.o := -I$(src)
-
-subdir-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/ath/wil6210/txrx.c b/drivers/net/wireless/ath/wil6210/txrx.c
index 4ac9ba04afed..c1b4bb03e997 100644
--- a/drivers/net/wireless/ath/wil6210/txrx.c
+++ b/drivers/net/wireless/ath/wil6210/txrx.c
@@ -338,7 +338,7 @@ static void wil_rx_add_radiotap_header(struct wil6210_priv *wil,
 
 	if (skb_headroom(skb) < rtap_len &&
 	    pskb_expand_head(skb, rtap_len, 0, GFP_ATOMIC)) {
-		wil_err(wil, "Unable to expand headrom to %d\n", rtap_len);
+		wil_err(wil, "Unable to expand headroom to %d\n", rtap_len);
 		return;
 	}
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile
index d1568bed1ad1..0383ba559edc 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile
@@ -19,8 +19,6 @@ ccflags-y += \
 	-Idrivers/net/wireless/broadcom/brcm80211/brcmfmac	\
 	-Idrivers/net/wireless/broadcom/brcm80211/include
 
-ccflags-y += -D__CHECK_ENDIAN__
-
 obj-$(CONFIG_BRCMFMAC) += brcmfmac.o
 brcmfmac-objs += \
 		cfg80211.o \
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/Makefile
index 960e6b86bbcb..ed83f33aceb7 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/Makefile
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/Makefile
@@ -16,7 +16,6 @@
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 ccflags-y := \
-	-D__CHECK_ENDIAN__ \
 	-Idrivers/net/wireless/broadcom/brcm80211/brcmsmac \
 	-Idrivers/net/wireless/broadcom/brcm80211/brcmsmac/phy \
 	-Idrivers/net/wireless/broadcom/brcm80211/include
diff --git a/drivers/net/wireless/intel/iwlegacy/Makefile b/drivers/net/wireless/intel/iwlegacy/Makefile
index c985a01a0731..c826a6b985bb 100644
--- a/drivers/net/wireless/intel/iwlegacy/Makefile
+++ b/drivers/net/wireless/intel/iwlegacy/Makefile
@@ -13,5 +13,3 @@ iwl4965-$(CONFIG_IWLEGACY_DEBUGFS) += 4965-debug.o
 obj-$(CONFIG_IWL3945)	+= iwl3945.o
 iwl3945-objs		:= 3945-mac.o 3945.o 3945-rs.o
 iwl3945-$(CONFIG_IWLEGACY_DEBUGFS) += 3945-debug.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/intel/iwlwifi/Makefile b/drivers/net/wireless/intel/iwlwifi/Makefile
index 6e7ed908de0c..92e611841200 100644
--- a/drivers/net/wireless/intel/iwlwifi/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/Makefile
@@ -15,7 +15,7 @@ iwlwifi-objs += $(iwlwifi-m)
 
 iwlwifi-$(CONFIG_IWLWIFI_DEVICE_TRACING) += iwl-devtrace.o
 
-ccflags-y += -D__CHECK_ENDIAN__ -I$(src)
+ccflags-y += -I$(src)
 
 obj-$(CONFIG_IWLDVM)	+= dvm/
 obj-$(CONFIG_IWLMVM)	+= mvm/
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/Makefile b/drivers/net/wireless/intel/iwlwifi/dvm/Makefile
index 4d19685f31c3..b256a354953a 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/Makefile
@@ -10,4 +10,4 @@ iwldvm-objs		+= rxon.o devices.o
 iwldvm-$(CONFIG_IWLWIFI_LEDS) += led.o
 iwldvm-$(CONFIG_IWLWIFI_DEBUGFS) += debugfs.o
 
-ccflags-y += -D__CHECK_ENDIAN__ -I$(src)/../
+ccflags-y += -I$(src)/../
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/calib.c b/drivers/net/wireless/intel/iwlwifi/dvm/calib.c
index e9cef9de9ed8..c96f9b1d948a 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/calib.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/calib.c
@@ -900,8 +900,7 @@ static void iwlagn_gain_computation(struct iwl_priv *priv,
 
 		/* bound gain by 2 bits value max, 3rd bit is sign */
 		data->delta_gain_code[i] =
-			min(abs(delta_g),
-			(s32) CHAIN_NOISE_MAX_DELTA_GAIN_CODE);
+			min(abs(delta_g), CHAIN_NOISE_MAX_DELTA_GAIN_CODE);
 
 		if (delta_g < 0)
 			/*
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-fw-file.h b/drivers/net/wireless/intel/iwlwifi/iwl-fw-file.h
index 1ad0ec180d5d..84813b550ef1 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-fw-file.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-fw-file.h
@@ -228,7 +228,7 @@ enum iwl_ucode_tlv_flag {
 	IWL_UCODE_TLV_FLAGS_BCAST_FILTERING	= BIT(29),
 };
 
-typedef unsigned int __bitwise__ iwl_ucode_tlv_api_t;
+typedef unsigned int __bitwise iwl_ucode_tlv_api_t;
 
 /**
  * enum iwl_ucode_tlv_api - ucode api
@@ -258,7 +258,7 @@ enum iwl_ucode_tlv_api {
 #endif
 };
 
-typedef unsigned int __bitwise__ iwl_ucode_tlv_capa_t;
+typedef unsigned int __bitwise iwl_ucode_tlv_capa_t;
 
 /**
  * enum iwl_ucode_tlv_capa - ucode capabilities
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile
index 2e06dfc1c477..83ac807e547d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile
@@ -9,4 +9,4 @@ iwlmvm-$(CONFIG_IWLWIFI_LEDS) += led.o
 iwlmvm-y += tof.o fw-dbg.o
 iwlmvm-$(CONFIG_PM) += d3.o
 
-ccflags-y += -D__CHECK_ENDIAN__ -I$(src)/../
+ccflags-y += -I$(src)/../
diff --git a/drivers/net/wireless/intersil/orinoco/Makefile b/drivers/net/wireless/intersil/orinoco/Makefile
index bfdefb85abcd..b7ecef820f76 100644
--- a/drivers/net/wireless/intersil/orinoco/Makefile
+++ b/drivers/net/wireless/intersil/orinoco/Makefile
@@ -12,6 +12,3 @@ obj-$(CONFIG_TMD_HERMES)	+= orinoco_tmd.o
 obj-$(CONFIG_NORTEL_HERMES)	+= orinoco_nortel.o
 obj-$(CONFIG_PCMCIA_SPECTRUM)	+= spectrum_cs.o
 obj-$(CONFIG_ORINOCO_USB)	+= orinoco_usb.o
-
-# Orinoco should be endian clean.
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/mediatek/mt7601u/Makefile b/drivers/net/wireless/mediatek/mt7601u/Makefile
index ea9ed8a5db4d..08fc802ead4b 100644
--- a/drivers/net/wireless/mediatek/mt7601u/Makefile
+++ b/drivers/net/wireless/mediatek/mt7601u/Makefile
@@ -1,5 +1,3 @@
-ccflags-y += -D__CHECK_ENDIAN__
-
 obj-$(CONFIG_MT7601U)	+= mt7601u.o
 
 mt7601u-objs	= \
diff --git a/drivers/net/wireless/realtek/rtlwifi/Makefile b/drivers/net/wireless/realtek/rtlwifi/Makefile
index ad6d3c52ec57..84c2e826fa1d 100644
--- a/drivers/net/wireless/realtek/rtlwifi/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/Makefile
@@ -30,5 +30,3 @@ obj-$(CONFIG_RTLBTCOEXIST)	+= btcoexist/
 obj-$(CONFIG_RTL8723_COMMON)	+= rtl8723com/
 obj-$(CONFIG_RTL8821AE)		+= rtl8821ae/
 obj-$(CONFIG_RTL8192EE)		+= rtl8192ee/
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/Makefile b/drivers/net/wireless/realtek/rtlwifi/btcoexist/Makefile
index 47ceecfcb7dc..d1454d4f08a5 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/Makefile
@@ -3,5 +3,3 @@ btcoexist-objs :=	halbtc8723b2ant.o	\
 			rtl_btc.o
 
 obj-$(CONFIG_RTLBTCOEXIST) += btcoexist.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/Makefile b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/Makefile
index 676e7de27f27..dae4f0f19cd3 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/Makefile
@@ -11,5 +11,3 @@ rtl8188ee-objs :=		\
 		trx.o
 
 obj-$(CONFIG_RTL8188EE) += rtl8188ee.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192c/Makefile b/drivers/net/wireless/realtek/rtlwifi/rtl8192c/Makefile
index aee42d7ae8a2..0546b7556259 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192c/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192c/Makefile
@@ -5,5 +5,3 @@ rtl8192c-common-objs :=		\
 		phy_common.o
 
 obj-$(CONFIG_RTL8192C_COMMON) += rtl8192c-common.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/Makefile b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/Makefile
index c0cb0cfe7d37..577c7adbc322 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/Makefile
@@ -9,5 +9,3 @@ rtl8192ce-objs :=		\
 		trx.o
 
 obj-$(CONFIG_RTL8192CE) += rtl8192ce.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/Makefile b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/Makefile
index ad2de6b839ef..97437dadc287 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/Makefile
@@ -10,5 +10,3 @@ rtl8192cu-objs :=		\
 		trx.o
 
 obj-$(CONFIG_RTL8192CU) += rtl8192cu.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/Makefile b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/Makefile
index e3213c8264b6..d0703f20d30c 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/Makefile
@@ -10,5 +10,3 @@ rtl8192de-objs :=		\
 		trx.o
 
 obj-$(CONFIG_RTL8192DE) += rtl8192de.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/Makefile b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/Makefile
index 0315eeda9b60..f254b9f64326 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/Makefile
@@ -12,5 +12,3 @@ rtl8192ee-objs :=		\
 
 
 obj-$(CONFIG_RTL8192EE) += rtl8192ee.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/Makefile b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/Makefile
index b7eb13819cbc..dfa9dbbe2cdf 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/Makefile
@@ -11,5 +11,3 @@ rtl8192se-objs :=		\
 
 obj-$(CONFIG_RTL8192SE) += rtl8192se.o
 
-ccflags-y += -D__CHECK_ENDIAN__
-
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/Makefile b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/Makefile
index 6220672a96f4..e7607d2cb2ef 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/Makefile
@@ -14,5 +14,3 @@ rtl8723ae-objs :=		\
 
 
 obj-$(CONFIG_RTL8723AE) += rtl8723ae.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/fw.c
index 1186755e55b8..e5505387260b 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/fw.c
@@ -134,7 +134,7 @@ static void _rtl8723e_fill_h2c_command(struct ieee80211_hw *hw, u8 element_id,
 			wait_h2c_limmit--;
 			if (wait_h2c_limmit == 0) {
 				RT_TRACE(rtlpriv, COMP_CMD, DBG_LOUD,
-					 "Wating too long for FW read clear HMEBox(%d)!\n",
+					 "Waiting too long for FW read clear HMEBox(%d)!\n",
 					 boxnum);
 				break;
 			}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/Makefile b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/Makefile
index a77c34102792..a841cbd55d8e 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/Makefile
@@ -12,5 +12,3 @@ rtl8723be-objs :=		\
 
 
 obj-$(CONFIG_RTL8723BE) += rtl8723be.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723com/Makefile b/drivers/net/wireless/realtek/rtlwifi/rtl8723com/Makefile
index 345a68adcf38..73da75526e2a 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723com/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723com/Makefile
@@ -5,5 +5,3 @@ rtl8723-common-objs :=		\
 		phy_common.o
 
 obj-$(CONFIG_RTL8723_COMMON) += rtl8723-common.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/Makefile b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/Makefile
index f7a26f71197e..8ca406b95f02 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/Makefile
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/Makefile
@@ -12,5 +12,3 @@ rtl8821ae-objs :=		\
 
 
 obj-$(CONFIG_RTL8821AE) += rtl8821ae.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/ti/wl1251/Makefile b/drivers/net/wireless/ti/wl1251/Makefile
index a5c6328b5f72..58b4f935a3f6 100644
--- a/drivers/net/wireless/ti/wl1251/Makefile
+++ b/drivers/net/wireless/ti/wl1251/Makefile
@@ -6,5 +6,3 @@ wl1251_sdio-objs	+= sdio.o
 obj-$(CONFIG_WL1251)		+= wl1251.o
 obj-$(CONFIG_WL1251_SPI)	+= wl1251_spi.o
 obj-$(CONFIG_WL1251_SDIO)	+= wl1251_sdio.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/wireless/ti/wlcore/Makefile b/drivers/net/wireless/ti/wlcore/Makefile
index 0a69c1373643..e286713b3c18 100644
--- a/drivers/net/wireless/ti/wlcore/Makefile
+++ b/drivers/net/wireless/ti/wlcore/Makefile
@@ -8,5 +8,3 @@ wlcore-$(CONFIG_NL80211_TESTMODE)	+= testmode.o
 obj-$(CONFIG_WLCORE)			+= wlcore.o
 obj-$(CONFIG_WLCORE_SPI)		+= wlcore_spi.o
 obj-$(CONFIG_WLCORE_SDIO)		+= wlcore_sdio.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c
index 55a4488633e4..3124eaec9427 100644
--- a/drivers/net/xen-netback/xenbus.c
+++ b/drivers/net/xen-netback/xenbus.c
@@ -785,12 +785,9 @@ static void xen_mcast_ctrl_changed(struct xenbus_watch *watch,
 	struct xenvif *vif = container_of(watch, struct xenvif,
 					  mcast_ctrl_watch);
 	struct xenbus_device *dev = xenvif_to_xenbus_device(vif);
-	int val;
 
-	if (xenbus_scanf(XBT_NIL, dev->otherend,
-			 "request-multicast-control", "%d", &val) < 0)
-		val = 0;
-	vif->multicast_control = !!val;
+	vif->multicast_control = !!xenbus_read_unsigned(dev->otherend,
+					"request-multicast-control", 0);
 }
 
 static int xen_register_mcast_ctrl_watch(struct xenbus_device *dev,
@@ -934,14 +931,11 @@ static void connect(struct backend_info *be)
 	/* Check whether the frontend requested multiple queues
 	 * and read the number requested.
 	 */
-	err = xenbus_scanf(XBT_NIL, dev->otherend,
-			   "multi-queue-num-queues",
-			   "%u", &requested_num_queues);
-	if (err < 0) {
-		requested_num_queues = 1; /* Fall back to single queue */
-	} else if (requested_num_queues > xenvif_max_queues) {
+	requested_num_queues = xenbus_read_unsigned(dev->otherend,
+					"multi-queue-num-queues", 1);
+	if (requested_num_queues > xenvif_max_queues) {
 		/* buggy or malicious guest */
-		xenbus_dev_fatal(dev, err,
+		xenbus_dev_fatal(dev, -EINVAL,
 				 "guest requested %u queues, exceeding the maximum of %u.",
 				 requested_num_queues, xenvif_max_queues);
 		return;
@@ -1134,7 +1128,7 @@ static int read_xenbus_vif_flags(struct backend_info *be)
 	struct xenvif *vif = be->vif;
 	struct xenbus_device *dev = be->dev;
 	unsigned int rx_copy;
-	int err, val;
+	int err;
 
 	err = xenbus_scanf(XBT_NIL, dev->otherend, "request-rx-copy", "%u",
 			   &rx_copy);
@@ -1150,10 +1144,7 @@ static int read_xenbus_vif_flags(struct backend_info *be)
 	if (!rx_copy)
 		return -EOPNOTSUPP;
 
-	if (xenbus_scanf(XBT_NIL, dev->otherend,
-			 "feature-rx-notify", "%d", &val) < 0)
-		val = 0;
-	if (!val) {
+	if (!xenbus_read_unsigned(dev->otherend, "feature-rx-notify", 0)) {
 		/* - Reduce drain timeout to poll more frequently for
 		 *   Rx requests.
 		 * - Disable Rx stall detection.
@@ -1162,34 +1153,21 @@ static int read_xenbus_vif_flags(struct backend_info *be)
 		be->vif->stall_timeout = 0;
 	}
 
-	if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-sg",
-			 "%d", &val) < 0)
-		val = 0;
-	vif->can_sg = !!val;
+	vif->can_sg = !!xenbus_read_unsigned(dev->otherend, "feature-sg", 0);
 
 	vif->gso_mask = 0;
 
-	if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-gso-tcpv4",
-			 "%d", &val) < 0)
-		val = 0;
-	if (val)
+	if (xenbus_read_unsigned(dev->otherend, "feature-gso-tcpv4", 0))
 		vif->gso_mask |= GSO_BIT(TCPV4);
 
-	if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-gso-tcpv6",
-			 "%d", &val) < 0)
-		val = 0;
-	if (val)
+	if (xenbus_read_unsigned(dev->otherend, "feature-gso-tcpv6", 0))
 		vif->gso_mask |= GSO_BIT(TCPV6);
 
-	if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-no-csum-offload",
-			 "%d", &val) < 0)
-		val = 0;
-	vif->ip_csum = !val;
+	vif->ip_csum = !xenbus_read_unsigned(dev->otherend,
+					     "feature-no-csum-offload", 0);
 
-	if (xenbus_scanf(XBT_NIL, dev->otherend, "feature-ipv6-csum-offload",
-			 "%d", &val) < 0)
-		val = 0;
-	vif->ipv6_csum = !!val;
+	vif->ipv6_csum = !!xenbus_read_unsigned(dev->otherend,
+						"feature-ipv6-csum-offload", 0);
 
 	return 0;
 }
diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
index e085c8c31cfe..a479cd99911d 100644
--- a/drivers/net/xen-netfront.c
+++ b/drivers/net/xen-netfront.c
@@ -1169,43 +1169,23 @@ static netdev_features_t xennet_fix_features(struct net_device *dev,
 	netdev_features_t features)
 {
 	struct netfront_info *np = netdev_priv(dev);
-	int val;
 
-	if (features & NETIF_F_SG) {
-		if (xenbus_scanf(XBT_NIL, np->xbdev->otherend, "feature-sg",
-				 "%d", &val) < 0)
-			val = 0;
+	if (features & NETIF_F_SG &&
+	    !xenbus_read_unsigned(np->xbdev->otherend, "feature-sg", 0))
+		features &= ~NETIF_F_SG;
 
-		if (!val)
-			features &= ~NETIF_F_SG;
-	}
-
-	if (features & NETIF_F_IPV6_CSUM) {
-		if (xenbus_scanf(XBT_NIL, np->xbdev->otherend,
-				 "feature-ipv6-csum-offload", "%d", &val) < 0)
-			val = 0;
-
-		if (!val)
-			features &= ~NETIF_F_IPV6_CSUM;
-	}
-
-	if (features & NETIF_F_TSO) {
-		if (xenbus_scanf(XBT_NIL, np->xbdev->otherend,
-				 "feature-gso-tcpv4", "%d", &val) < 0)
-			val = 0;
+	if (features & NETIF_F_IPV6_CSUM &&
+	    !xenbus_read_unsigned(np->xbdev->otherend,
+				  "feature-ipv6-csum-offload", 0))
+		features &= ~NETIF_F_IPV6_CSUM;
 
-		if (!val)
-			features &= ~NETIF_F_TSO;
-	}
+	if (features & NETIF_F_TSO &&
+	    !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv4", 0))
+		features &= ~NETIF_F_TSO;
 
-	if (features & NETIF_F_TSO6) {
-		if (xenbus_scanf(XBT_NIL, np->xbdev->otherend,
-				 "feature-gso-tcpv6", "%d", &val) < 0)
-			val = 0;
-
-		if (!val)
-			features &= ~NETIF_F_TSO6;
-	}
+	if (features & NETIF_F_TSO6 &&
+	    !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv6", 0))
+		features &= ~NETIF_F_TSO6;
 
 	return features;
 }
@@ -1823,18 +1803,13 @@ static int talk_to_netback(struct xenbus_device *dev,
 	info->netdev->irq = 0;
 
 	/* Check if backend supports multiple queues */
-	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-			   "multi-queue-max-queues", "%u", &max_queues);
-	if (err < 0)
-		max_queues = 1;
+	max_queues = xenbus_read_unsigned(info->xbdev->otherend,
+					  "multi-queue-max-queues", 1);
 	num_queues = min(max_queues, xennet_max_queues);
 
 	/* Check feature-split-event-channels */
-	err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-			   "feature-split-event-channels", "%u",
-			   &feature_split_evtchn);
-	if (err < 0)
-		feature_split_evtchn = 0;
+	feature_split_evtchn = xenbus_read_unsigned(info->xbdev->otherend,
+					"feature-split-event-channels", 0);
 
 	/* Read mac addr. */
 	err = xen_net_read_mac(dev, info->netdev->dev_addr);
@@ -1968,16 +1943,10 @@ static int xennet_connect(struct net_device *dev)
 	struct netfront_info *np = netdev_priv(dev);
 	unsigned int num_queues = 0;
 	int err;
-	unsigned int feature_rx_copy;
 	unsigned int j = 0;
 	struct netfront_queue *queue = NULL;
 
-	err = xenbus_scanf(XBT_NIL, np->xbdev->otherend,
-			   "feature-rx-copy", "%u", &feature_rx_copy);
-	if (err != 1)
-		feature_rx_copy = 0;
-
-	if (!feature_rx_copy) {
+	if (!xenbus_read_unsigned(np->xbdev->otherend, "feature-rx-copy", 0)) {
 		dev_info(&dev->dev,
 			 "backend does not support copying receive path\n");
 		return -ENODEV;
diff --git a/drivers/nfc/mei_phy.c b/drivers/nfc/mei_phy.c
index 6f9563a96488..8a04c5e02999 100644
--- a/drivers/nfc/mei_phy.c
+++ b/drivers/nfc/mei_phy.c
@@ -297,35 +297,34 @@ static int mei_nfc_recv(struct nfc_mei_phy *phy, u8 *buf, size_t length)
 }
 
 
-static void nfc_mei_event_cb(struct mei_cl_device *cldev, u32 events,
-			     void *context)
+static void nfc_mei_rx_cb(struct mei_cl_device *cldev)
 {
-	struct nfc_mei_phy *phy = context;
+	struct nfc_mei_phy *phy = mei_cldev_get_drvdata(cldev);
+	struct sk_buff *skb;
+	int reply_size;
 
-	if (phy->hard_fault != 0)
+	if (!phy)
 		return;
 
-	if (events & BIT(MEI_CL_EVENT_RX)) {
-		struct sk_buff *skb;
-		int reply_size;
+	if (phy->hard_fault != 0)
+		return;
 
-		skb = alloc_skb(MEI_NFC_MAX_READ, GFP_KERNEL);
-		if (!skb)
-			return;
+	skb = alloc_skb(MEI_NFC_MAX_READ, GFP_KERNEL);
+	if (!skb)
+		return;
 
-		reply_size = mei_nfc_recv(phy, skb->data, MEI_NFC_MAX_READ);
-		if (reply_size < MEI_NFC_HEADER_SIZE) {
-			kfree_skb(skb);
-			return;
-		}
+	reply_size = mei_nfc_recv(phy, skb->data, MEI_NFC_MAX_READ);
+	if (reply_size < MEI_NFC_HEADER_SIZE) {
+		kfree_skb(skb);
+		return;
+	}
 
-		skb_put(skb, reply_size);
-		skb_pull(skb, MEI_NFC_HEADER_SIZE);
+	skb_put(skb, reply_size);
+	skb_pull(skb, MEI_NFC_HEADER_SIZE);
 
-		MEI_DUMP_SKB_IN("mei frame read", skb);
+	MEI_DUMP_SKB_IN("mei frame read", skb);
 
-		nfc_hci_recv_frame(phy->hdev, skb);
-	}
+	nfc_hci_recv_frame(phy->hdev, skb);
 }
 
 static int nfc_mei_phy_enable(void *phy_id)
@@ -356,8 +355,7 @@ static int nfc_mei_phy_enable(void *phy_id)
 		goto err;
 	}
 
-	r = mei_cldev_register_event_cb(phy->cldev, BIT(MEI_CL_EVENT_RX),
-				     nfc_mei_event_cb, phy);
+	r = mei_cldev_register_rx_cb(phy->cldev, nfc_mei_rx_cb);
 	if (r) {
 		pr_err("Event cb registration failed %d\n", r);
 		goto err;
diff --git a/drivers/nfc/microread/mei.c b/drivers/nfc/microread/mei.c
index 3092501f26c4..eb5eddf1794e 100644
--- a/drivers/nfc/microread/mei.c
+++ b/drivers/nfc/microread/mei.c
@@ -82,28 +82,7 @@ static struct mei_cl_driver microread_driver = {
 	.remove = microread_mei_remove,
 };
 
-static int microread_mei_init(void)
-{
-	int r;
-
-	pr_debug(DRIVER_DESC ": %s\n", __func__);
-
-	r = mei_cldev_driver_register(&microread_driver);
-	if (r) {
-		pr_err(MICROREAD_DRIVER_NAME ": driver registration failed\n");
-		return r;
-	}
-
-	return 0;
-}
-
-static void microread_mei_exit(void)
-{
-	mei_cldev_driver_unregister(&microread_driver);
-}
-
-module_init(microread_mei_init);
-module_exit(microread_mei_exit);
+module_mei_cl_driver(microread_driver);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION(DRIVER_DESC);
diff --git a/drivers/nfc/pn544/mei.c b/drivers/nfc/pn544/mei.c
index 46d0eb24eef9..ad57a8ec00d6 100644
--- a/drivers/nfc/pn544/mei.c
+++ b/drivers/nfc/pn544/mei.c
@@ -82,28 +82,7 @@ static struct mei_cl_driver pn544_driver = {
 	.remove = pn544_mei_remove,
 };
 
-static int pn544_mei_init(void)
-{
-	int r;
-
-	pr_debug(DRIVER_DESC ": %s\n", __func__);
-
-	r = mei_cldev_driver_register(&pn544_driver);
-	if (r) {
-		pr_err(PN544_DRIVER_NAME ": driver registration failed\n");
-		return r;
-	}
-
-	return 0;
-}
-
-static void pn544_mei_exit(void)
-{
-	mei_cldev_driver_unregister(&pn544_driver);
-}
-
-module_init(pn544_mei_init);
-module_exit(pn544_mei_exit);
+module_mei_cl_driver(pn544_driver);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION(DRIVER_DESC);
diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
index f7d37a62f874..90745a616df7 100644
--- a/drivers/nvme/host/Kconfig
+++ b/drivers/nvme/host/Kconfig
@@ -43,3 +43,20 @@ config NVME_RDMA
 	  from https://github.com/linux-nvme/nvme-cli.
 
 	  If unsure, say N.
+
+config NVME_FC
+	tristate "NVM Express over Fabrics FC host driver"
+	depends on BLOCK
+	depends on HAS_DMA
+	select NVME_CORE
+	select NVME_FABRICS
+	select SG_POOL
+	help
+	  This provides support for the NVMe over Fabrics protocol using
+	  the FC transport.  This allows you to use remote block devices
+	  exported using the NVMe protocol set.
+
+	  To configure a NVMe over Fabrics controller use the nvme-cli tool
+	  from https://github.com/linux-nvme/nvme-cli.
+
+	  If unsure, say N.
diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile
index 47abcec23514..f1a7d945fbb6 100644
--- a/drivers/nvme/host/Makefile
+++ b/drivers/nvme/host/Makefile
@@ -2,6 +2,7 @@ obj-$(CONFIG_NVME_CORE)			+= nvme-core.o
 obj-$(CONFIG_BLK_DEV_NVME)		+= nvme.o
 obj-$(CONFIG_NVME_FABRICS)		+= nvme-fabrics.o
 obj-$(CONFIG_NVME_RDMA)			+= nvme-rdma.o
+obj-$(CONFIG_NVME_FC)			+= nvme-fc.o
 
 nvme-core-y				:= core.o
 nvme-core-$(CONFIG_BLK_DEV_NVME_SCSI)	+= scsi.o
@@ -12,3 +13,5 @@ nvme-y					+= pci.o
 nvme-fabrics-y				+= fabrics.o
 
 nvme-rdma-y				+= rdma.o
+
+nvme-fc-y				+= fc.o
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 79e679d12f3b..b40cfb076f02 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -201,13 +201,7 @@ fail:
 
 void nvme_requeue_req(struct request *req)
 {
-	unsigned long flags;
-
-	blk_mq_requeue_request(req);
-	spin_lock_irqsave(req->q->queue_lock, flags);
-	if (!blk_queue_stopped(req->q))
-		blk_mq_kick_requeue_list(req->q);
-	spin_unlock_irqrestore(req->q->queue_lock, flags);
+	blk_mq_requeue_request(req, !blk_mq_queue_stopped(req->q));
 }
 EXPORT_SYMBOL_GPL(nvme_requeue_req);
 
@@ -227,8 +221,7 @@ struct request *nvme_alloc_request(struct request_queue *q,
 
 	req->cmd_type = REQ_TYPE_DRV_PRIV;
 	req->cmd_flags |= REQ_FAILFAST_DRIVER;
-	req->cmd = (unsigned char *)cmd;
-	req->cmd_len = sizeof(struct nvme_command);
+	nvme_req(req)->cmd = cmd;
 
 	return req;
 }
@@ -246,8 +239,6 @@ static inline int nvme_setup_discard(struct nvme_ns *ns, struct request *req,
 		struct nvme_command *cmnd)
 {
 	struct nvme_dsm_range *range;
-	struct page *page;
-	int offset;
 	unsigned int nr_bytes = blk_rq_bytes(req);
 
 	range = kmalloc(sizeof(*range), GFP_ATOMIC);
@@ -264,19 +255,12 @@ static inline int nvme_setup_discard(struct nvme_ns *ns, struct request *req,
 	cmnd->dsm.nr = 0;
 	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 
-	req->completion_data = range;
-	page = virt_to_page(range);
-	offset = offset_in_page(range);
-	blk_add_request_payload(req, page, offset, sizeof(*range));
-
-	/*
-	 * we set __data_len back to the size of the area to be discarded
-	 * on disk. This allows us to report completion on the full amount
-	 * of blocks described by the request.
-	 */
-	req->__data_len = nr_bytes;
+	req->special_vec.bv_page = virt_to_page(range);
+	req->special_vec.bv_offset = offset_in_page(range);
+	req->special_vec.bv_len = sizeof(*range);
+	req->rq_flags |= RQF_SPECIAL_PAYLOAD;
 
-	return 0;
+	return BLK_MQ_RQ_QUEUE_OK;
 }
 
 static inline void nvme_setup_rw(struct nvme_ns *ns, struct request *req,
@@ -295,7 +279,6 @@ static inline void nvme_setup_rw(struct nvme_ns *ns, struct request *req,
 
 	memset(cmnd, 0, sizeof(*cmnd));
 	cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
-	cmnd->rw.command_id = req->tag;
 	cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
 	cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
 	cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
@@ -324,10 +307,10 @@ static inline void nvme_setup_rw(struct nvme_ns *ns, struct request *req,
 int nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
 		struct nvme_command *cmd)
 {
-	int ret = 0;
+	int ret = BLK_MQ_RQ_QUEUE_OK;
 
 	if (req->cmd_type == REQ_TYPE_DRV_PRIV)
-		memcpy(cmd, req->cmd, sizeof(*cmd));
+		memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
 	else if (req_op(req) == REQ_OP_FLUSH)
 		nvme_setup_flush(ns, cmd);
 	else if (req_op(req) == REQ_OP_DISCARD)
@@ -335,6 +318,8 @@ int nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
 	else
 		nvme_setup_rw(ns, req, cmd);
 
+	cmd->common.command_id = req->tag;
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
@@ -344,7 +329,7 @@ EXPORT_SYMBOL_GPL(nvme_setup_cmd);
  * if the result is positive, it's an NVM Express status code
  */
 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
-		struct nvme_completion *cqe, void *buffer, unsigned bufflen,
+		union nvme_result *result, void *buffer, unsigned bufflen,
 		unsigned timeout, int qid, int at_head, int flags)
 {
 	struct request *req;
@@ -355,7 +340,6 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 		return PTR_ERR(req);
 
 	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
-	req->special = cqe;
 
 	if (buffer && bufflen) {
 		ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
@@ -364,6 +348,8 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 	}
 
 	blk_execute_rq(req->q, NULL, req, at_head);
+	if (result)
+		*result = nvme_req(req)->result;
 	ret = req->errors;
  out:
 	blk_mq_free_request(req);
@@ -385,7 +371,6 @@ int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
 		u32 *result, unsigned timeout)
 {
 	bool write = nvme_is_write(cmd);
-	struct nvme_completion cqe;
 	struct nvme_ns *ns = q->queuedata;
 	struct gendisk *disk = ns ? ns->disk : NULL;
 	struct request *req;
@@ -398,7 +383,6 @@ int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
 		return PTR_ERR(req);
 
 	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
-	req->special = &cqe;
 
 	if (ubuffer && bufflen) {
 		ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
@@ -453,7 +437,7 @@ int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
 	blk_execute_rq(req->q, disk, req, 0);
 	ret = req->errors;
 	if (result)
-		*result = le32_to_cpu(cqe.result);
+		*result = le32_to_cpu(nvme_req(req)->result.u32);
 	if (meta && !ret && !write) {
 		if (copy_to_user(meta_buffer, meta, meta_len))
 			ret = -EFAULT;
@@ -602,7 +586,7 @@ int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
 		      void *buffer, size_t buflen, u32 *result)
 {
 	struct nvme_command c;
-	struct nvme_completion cqe;
+	union nvme_result res;
 	int ret;
 
 	memset(&c, 0, sizeof(c));
@@ -610,10 +594,10 @@ int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
 	c.features.nsid = cpu_to_le32(nsid);
 	c.features.fid = cpu_to_le32(fid);
 
-	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, buffer, buflen, 0,
+	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res, buffer, buflen, 0,
 			NVME_QID_ANY, 0, 0);
 	if (ret >= 0 && result)
-		*result = le32_to_cpu(cqe.result);
+		*result = le32_to_cpu(res.u32);
 	return ret;
 }
 
@@ -621,7 +605,7 @@ int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
 		      void *buffer, size_t buflen, u32 *result)
 {
 	struct nvme_command c;
-	struct nvme_completion cqe;
+	union nvme_result res;
 	int ret;
 
 	memset(&c, 0, sizeof(c));
@@ -629,10 +613,10 @@ int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
 	c.features.fid = cpu_to_le32(fid);
 	c.features.dword11 = cpu_to_le32(dword11);
 
-	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe,
+	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
 			buffer, buflen, 0, NVME_QID_ANY, 0, 0);
 	if (ret >= 0 && result)
-		*result = le32_to_cpu(cqe.result);
+		*result = le32_to_cpu(res.u32);
 	return ret;
 }
 
@@ -1683,27 +1667,24 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 	if (nvme_revalidate_ns(ns, &id))
 		goto out_free_queue;
 
-	if (nvme_nvm_ns_supported(ns, id)) {
-		if (nvme_nvm_register(ns, disk_name, node,
-							&nvme_ns_attr_group)) {
-			dev_warn(ctrl->dev, "%s: LightNVM init failure\n",
-								__func__);
-			goto out_free_id;
-		}
-	} else {
-		disk = alloc_disk_node(0, node);
-		if (!disk)
-			goto out_free_id;
+	if (nvme_nvm_ns_supported(ns, id) &&
+				nvme_nvm_register(ns, disk_name, node)) {
+		dev_warn(ctrl->dev, "%s: LightNVM init failure\n", __func__);
+		goto out_free_id;
+	}
 
-		disk->fops = &nvme_fops;
-		disk->private_data = ns;
-		disk->queue = ns->queue;
-		disk->flags = GENHD_FL_EXT_DEVT;
-		memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
-		ns->disk = disk;
+	disk = alloc_disk_node(0, node);
+	if (!disk)
+		goto out_free_id;
 
-		__nvme_revalidate_disk(disk, id);
-	}
+	disk->fops = &nvme_fops;
+	disk->private_data = ns;
+	disk->queue = ns->queue;
+	disk->flags = GENHD_FL_EXT_DEVT;
+	memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
+	ns->disk = disk;
+
+	__nvme_revalidate_disk(disk, id);
 
 	mutex_lock(&ctrl->namespaces_mutex);
 	list_add_tail(&ns->list, &ctrl->namespaces);
@@ -1713,14 +1694,14 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 
 	kfree(id);
 
-	if (ns->ndev)
-		return;
-
 	device_add_disk(ctrl->device, ns->disk);
 	if (sysfs_create_group(&disk_to_dev(ns->disk)->kobj,
 					&nvme_ns_attr_group))
 		pr_warn("%s: failed to create sysfs group for identification\n",
 			ns->disk->disk_name);
+	if (ns->ndev && nvme_nvm_register_sysfs(ns))
+		pr_warn("%s: failed to register lightnvm sysfs group for identification\n",
+			ns->disk->disk_name);
 	return;
  out_free_id:
 	kfree(id);
@@ -1742,6 +1723,8 @@ static void nvme_ns_remove(struct nvme_ns *ns)
 			blk_integrity_unregister(ns->disk);
 		sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
 					&nvme_ns_attr_group);
+		if (ns->ndev)
+			nvme_nvm_unregister_sysfs(ns);
 		del_gendisk(ns->disk);
 		blk_mq_abort_requeue_list(ns->queue);
 		blk_cleanup_queue(ns->queue);
@@ -1905,18 +1888,25 @@ static void nvme_async_event_work(struct work_struct *work)
 	spin_unlock_irq(&ctrl->lock);
 }
 
-void nvme_complete_async_event(struct nvme_ctrl *ctrl,
-		struct nvme_completion *cqe)
+void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
+		union nvme_result *res)
 {
-	u16 status = le16_to_cpu(cqe->status) >> 1;
-	u32 result = le32_to_cpu(cqe->result);
+	u32 result = le32_to_cpu(res->u32);
+	bool done = true;
 
-	if (status == NVME_SC_SUCCESS || status == NVME_SC_ABORT_REQ) {
+	switch (le16_to_cpu(status) >> 1) {
+	case NVME_SC_SUCCESS:
+		done = false;
+		/*FALLTHRU*/
+	case NVME_SC_ABORT_REQ:
 		++ctrl->event_limit;
 		schedule_work(&ctrl->async_event_work);
+		break;
+	default:
+		break;
 	}
 
-	if (status != NVME_SC_SUCCESS)
+	if (done)
 		return;
 
 	switch (result & 0xff07) {
@@ -2078,14 +2068,8 @@ void nvme_stop_queues(struct nvme_ctrl *ctrl)
 	struct nvme_ns *ns;
 
 	mutex_lock(&ctrl->namespaces_mutex);
-	list_for_each_entry(ns, &ctrl->namespaces, list) {
-		spin_lock_irq(ns->queue->queue_lock);
-		queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
-		spin_unlock_irq(ns->queue->queue_lock);
-
-		blk_mq_cancel_requeue_work(ns->queue);
-		blk_mq_stop_hw_queues(ns->queue);
-	}
+	list_for_each_entry(ns, &ctrl->namespaces, list)
+		blk_mq_quiesce_queue(ns->queue);
 	mutex_unlock(&ctrl->namespaces_mutex);
 }
 EXPORT_SYMBOL_GPL(nvme_stop_queues);
@@ -2096,7 +2080,6 @@ void nvme_start_queues(struct nvme_ctrl *ctrl)
 
 	mutex_lock(&ctrl->namespaces_mutex);
 	list_for_each_entry(ns, &ctrl->namespaces, list) {
-		queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
 		blk_mq_start_stopped_hw_queues(ns->queue, true);
 		blk_mq_kick_requeue_list(ns->queue);
 	}
diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c
index 5a3f008d3480..916d13608059 100644
--- a/drivers/nvme/host/fabrics.c
+++ b/drivers/nvme/host/fabrics.c
@@ -161,7 +161,7 @@ EXPORT_SYMBOL_GPL(nvmf_get_subsysnqn);
 int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
 {
 	struct nvme_command cmd;
-	struct nvme_completion cqe;
+	union nvme_result res;
 	int ret;
 
 	memset(&cmd, 0, sizeof(cmd));
@@ -169,11 +169,11 @@ int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
 	cmd.prop_get.fctype = nvme_fabrics_type_property_get;
 	cmd.prop_get.offset = cpu_to_le32(off);
 
-	ret = __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, &cqe, NULL, 0, 0,
+	ret = __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, &res, NULL, 0, 0,
 			NVME_QID_ANY, 0, 0);
 
 	if (ret >= 0)
-		*val = le64_to_cpu(cqe.result64);
+		*val = le64_to_cpu(res.u64);
 	if (unlikely(ret != 0))
 		dev_err(ctrl->device,
 			"Property Get error: %d, offset %#x\n",
@@ -207,7 +207,7 @@ EXPORT_SYMBOL_GPL(nvmf_reg_read32);
 int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
 {
 	struct nvme_command cmd;
-	struct nvme_completion cqe;
+	union nvme_result res;
 	int ret;
 
 	memset(&cmd, 0, sizeof(cmd));
@@ -216,11 +216,11 @@ int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
 	cmd.prop_get.attrib = 1;
 	cmd.prop_get.offset = cpu_to_le32(off);
 
-	ret = __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, &cqe, NULL, 0, 0,
+	ret = __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, &res, NULL, 0, 0,
 			NVME_QID_ANY, 0, 0);
 
 	if (ret >= 0)
-		*val = le64_to_cpu(cqe.result64);
+		*val = le64_to_cpu(res.u64);
 	if (unlikely(ret != 0))
 		dev_err(ctrl->device,
 			"Property Get error: %d, offset %#x\n",
@@ -368,7 +368,7 @@ static void nvmf_log_connect_error(struct nvme_ctrl *ctrl,
 int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
 {
 	struct nvme_command cmd;
-	struct nvme_completion cqe;
+	union nvme_result res;
 	struct nvmf_connect_data *data;
 	int ret;
 
@@ -400,16 +400,16 @@ int nvmf_connect_admin_queue(struct nvme_ctrl *ctrl)
 	strncpy(data->subsysnqn, ctrl->opts->subsysnqn, NVMF_NQN_SIZE);
 	strncpy(data->hostnqn, ctrl->opts->host->nqn, NVMF_NQN_SIZE);
 
-	ret = __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, &cqe,
+	ret = __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, &res,
 			data, sizeof(*data), 0, NVME_QID_ANY, 1,
 			BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
 	if (ret) {
-		nvmf_log_connect_error(ctrl, ret, le32_to_cpu(cqe.result),
+		nvmf_log_connect_error(ctrl, ret, le32_to_cpu(res.u32),
 				       &cmd, data);
 		goto out_free_data;
 	}
 
-	ctrl->cntlid = le16_to_cpu(cqe.result16);
+	ctrl->cntlid = le16_to_cpu(res.u16);
 
 out_free_data:
 	kfree(data);
@@ -441,7 +441,7 @@ int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid)
 {
 	struct nvme_command cmd;
 	struct nvmf_connect_data *data;
-	struct nvme_completion cqe;
+	union nvme_result res;
 	int ret;
 
 	memset(&cmd, 0, sizeof(cmd));
@@ -459,11 +459,11 @@ int nvmf_connect_io_queue(struct nvme_ctrl *ctrl, u16 qid)
 	strncpy(data->subsysnqn, ctrl->opts->subsysnqn, NVMF_NQN_SIZE);
 	strncpy(data->hostnqn, ctrl->opts->host->nqn, NVMF_NQN_SIZE);
 
-	ret = __nvme_submit_sync_cmd(ctrl->connect_q, &cmd, &cqe,
+	ret = __nvme_submit_sync_cmd(ctrl->connect_q, &cmd, &res,
 			data, sizeof(*data), 0, qid, 1,
 			BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
 	if (ret) {
-		nvmf_log_connect_error(ctrl, ret, le32_to_cpu(cqe.result),
+		nvmf_log_connect_error(ctrl, ret, le32_to_cpu(res.u32),
 				       &cmd, data);
 	}
 	kfree(data);
@@ -576,7 +576,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
 			nqnlen = strlen(opts->subsysnqn);
 			if (nqnlen >= NVMF_NQN_SIZE) {
 				pr_err("%s needs to be < %d bytes\n",
-				opts->subsysnqn, NVMF_NQN_SIZE);
+					opts->subsysnqn, NVMF_NQN_SIZE);
 				ret = -EINVAL;
 				goto out;
 			}
@@ -666,10 +666,12 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
 			if (nqnlen >= NVMF_NQN_SIZE) {
 				pr_err("%s needs to be < %d bytes\n",
 					p, NVMF_NQN_SIZE);
+				kfree(p);
 				ret = -EINVAL;
 				goto out;
 			}
 			opts->host = nvmf_host_add(p);
+			kfree(p);
 			if (!opts->host) {
 				ret = -ENOMEM;
 				goto out;
@@ -825,8 +827,7 @@ nvmf_create_ctrl(struct device *dev, const char *buf, size_t count)
 out_unlock:
 	mutex_unlock(&nvmf_transports_mutex);
 out_free_opts:
-	nvmf_host_put(opts->host);
-	kfree(opts);
+	nvmf_free_options(opts);
 	return ERR_PTR(ret);
 }
 
diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
new file mode 100644
index 000000000000..771e2e761872
--- /dev/null
+++ b/drivers/nvme/host/fc.c
@@ -0,0 +1,2586 @@
+/*
+ * Copyright (c) 2016 Avago Technologies.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful.
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
+ * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
+ * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO
+ * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
+ * See the GNU General Public License for more details, a copy of which
+ * can be found in the file COPYING included with this package
+ *
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/module.h>
+#include <linux/parser.h>
+#include <uapi/scsi/fc/fc_fs.h>
+#include <uapi/scsi/fc/fc_els.h>
+
+#include "nvme.h"
+#include "fabrics.h"
+#include <linux/nvme-fc-driver.h>
+#include <linux/nvme-fc.h>
+
+
+/* *************************** Data Structures/Defines ****************** */
+
+
+/*
+ * We handle AEN commands ourselves and don't even let the
+ * block layer know about them.
+ */
+#define NVME_FC_NR_AEN_COMMANDS	1
+#define NVME_FC_AQ_BLKMQ_DEPTH	\
+	(NVMF_AQ_DEPTH - NVME_FC_NR_AEN_COMMANDS)
+#define AEN_CMDID_BASE		(NVME_FC_AQ_BLKMQ_DEPTH + 1)
+
+enum nvme_fc_queue_flags {
+	NVME_FC_Q_CONNECTED = (1 << 0),
+};
+
+#define NVMEFC_QUEUE_DELAY	3		/* ms units */
+
+struct nvme_fc_queue {
+	struct nvme_fc_ctrl	*ctrl;
+	struct device		*dev;
+	struct blk_mq_hw_ctx	*hctx;
+	void			*lldd_handle;
+	int			queue_size;
+	size_t			cmnd_capsule_len;
+	u32			qnum;
+	u32			rqcnt;
+	u32			seqno;
+
+	u64			connection_id;
+	atomic_t		csn;
+
+	unsigned long		flags;
+} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
+
+struct nvmefc_ls_req_op {
+	struct nvmefc_ls_req	ls_req;
+
+	struct nvme_fc_ctrl	*ctrl;
+	struct nvme_fc_queue	*queue;
+	struct request		*rq;
+
+	int			ls_error;
+	struct completion	ls_done;
+	struct list_head	lsreq_list;	/* ctrl->ls_req_list */
+	bool			req_queued;
+};
+
+enum nvme_fcpop_state {
+	FCPOP_STATE_UNINIT	= 0,
+	FCPOP_STATE_IDLE	= 1,
+	FCPOP_STATE_ACTIVE	= 2,
+	FCPOP_STATE_ABORTED	= 3,
+};
+
+struct nvme_fc_fcp_op {
+	struct nvme_request	nreq;		/*
+						 * nvme/host/core.c
+						 * requires this to be
+						 * the 1st element in the
+						 * private structure
+						 * associated with the
+						 * request.
+						 */
+	struct nvmefc_fcp_req	fcp_req;
+
+	struct nvme_fc_ctrl	*ctrl;
+	struct nvme_fc_queue	*queue;
+	struct request		*rq;
+
+	atomic_t		state;
+	u32			rqno;
+	u32			nents;
+
+	struct nvme_fc_cmd_iu	cmd_iu;
+	struct nvme_fc_ersp_iu	rsp_iu;
+};
+
+struct nvme_fc_lport {
+	struct nvme_fc_local_port	localport;
+
+	struct ida			endp_cnt;
+	struct list_head		port_list;	/* nvme_fc_port_list */
+	struct list_head		endp_list;
+	struct device			*dev;	/* physical device for dma */
+	struct nvme_fc_port_template	*ops;
+	struct kref			ref;
+} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
+
+struct nvme_fc_rport {
+	struct nvme_fc_remote_port	remoteport;
+
+	struct list_head		endp_list; /* for lport->endp_list */
+	struct list_head		ctrl_list;
+	spinlock_t			lock;
+	struct kref			ref;
+} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
+
+enum nvme_fcctrl_state {
+	FCCTRL_INIT		= 0,
+	FCCTRL_ACTIVE		= 1,
+};
+
+struct nvme_fc_ctrl {
+	spinlock_t		lock;
+	struct nvme_fc_queue	*queues;
+	u32			queue_count;
+
+	struct device		*dev;
+	struct nvme_fc_lport	*lport;
+	struct nvme_fc_rport	*rport;
+	u32			cnum;
+
+	u64			association_id;
+
+	u64			cap;
+
+	struct list_head	ctrl_list;	/* rport->ctrl_list */
+	struct list_head	ls_req_list;
+
+	struct blk_mq_tag_set	admin_tag_set;
+	struct blk_mq_tag_set	tag_set;
+
+	struct work_struct	delete_work;
+	struct kref		ref;
+	int			state;
+
+	struct nvme_fc_fcp_op	aen_ops[NVME_FC_NR_AEN_COMMANDS];
+
+	struct nvme_ctrl	ctrl;
+};
+
+static inline struct nvme_fc_ctrl *
+to_fc_ctrl(struct nvme_ctrl *ctrl)
+{
+	return container_of(ctrl, struct nvme_fc_ctrl, ctrl);
+}
+
+static inline struct nvme_fc_lport *
+localport_to_lport(struct nvme_fc_local_port *portptr)
+{
+	return container_of(portptr, struct nvme_fc_lport, localport);
+}
+
+static inline struct nvme_fc_rport *
+remoteport_to_rport(struct nvme_fc_remote_port *portptr)
+{
+	return container_of(portptr, struct nvme_fc_rport, remoteport);
+}
+
+static inline struct nvmefc_ls_req_op *
+ls_req_to_lsop(struct nvmefc_ls_req *lsreq)
+{
+	return container_of(lsreq, struct nvmefc_ls_req_op, ls_req);
+}
+
+static inline struct nvme_fc_fcp_op *
+fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq)
+{
+	return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req);
+}
+
+
+
+/* *************************** Globals **************************** */
+
+
+static DEFINE_SPINLOCK(nvme_fc_lock);
+
+static LIST_HEAD(nvme_fc_lport_list);
+static DEFINE_IDA(nvme_fc_local_port_cnt);
+static DEFINE_IDA(nvme_fc_ctrl_cnt);
+
+static struct workqueue_struct *nvme_fc_wq;
+
+
+
+/* *********************** FC-NVME Port Management ************************ */
+
+static int __nvme_fc_del_ctrl(struct nvme_fc_ctrl *);
+static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
+			struct nvme_fc_queue *, unsigned int);
+
+
+/**
+ * nvme_fc_register_localport - transport entry point called by an
+ *                              LLDD to register the existence of a NVME
+ *                              host FC port.
+ * @pinfo:     pointer to information about the port to be registered
+ * @template:  LLDD entrypoints and operational parameters for the port
+ * @dev:       physical hardware device node port corresponds to. Will be
+ *             used for DMA mappings
+ * @lport_p:   pointer to a local port pointer. Upon success, the routine
+ *             will allocate a nvme_fc_local_port structure and place its
+ *             address in the local port pointer. Upon failure, local port
+ *             pointer will be set to 0.
+ *
+ * Returns:
+ * a completion status. Must be 0 upon success; a negative errno
+ * (ex: -ENXIO) upon failure.
+ */
+int
+nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
+			struct nvme_fc_port_template *template,
+			struct device *dev,
+			struct nvme_fc_local_port **portptr)
+{
+	struct nvme_fc_lport *newrec;
+	unsigned long flags;
+	int ret, idx;
+
+	if (!template->localport_delete || !template->remoteport_delete ||
+	    !template->ls_req || !template->fcp_io ||
+	    !template->ls_abort || !template->fcp_abort ||
+	    !template->max_hw_queues || !template->max_sgl_segments ||
+	    !template->max_dif_sgl_segments || !template->dma_boundary) {
+		ret = -EINVAL;
+		goto out_reghost_failed;
+	}
+
+	newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz),
+			 GFP_KERNEL);
+	if (!newrec) {
+		ret = -ENOMEM;
+		goto out_reghost_failed;
+	}
+
+	idx = ida_simple_get(&nvme_fc_local_port_cnt, 0, 0, GFP_KERNEL);
+	if (idx < 0) {
+		ret = -ENOSPC;
+		goto out_fail_kfree;
+	}
+
+	if (!get_device(dev) && dev) {
+		ret = -ENODEV;
+		goto out_ida_put;
+	}
+
+	INIT_LIST_HEAD(&newrec->port_list);
+	INIT_LIST_HEAD(&newrec->endp_list);
+	kref_init(&newrec->ref);
+	newrec->ops = template;
+	newrec->dev = dev;
+	ida_init(&newrec->endp_cnt);
+	newrec->localport.private = &newrec[1];
+	newrec->localport.node_name = pinfo->node_name;
+	newrec->localport.port_name = pinfo->port_name;
+	newrec->localport.port_role = pinfo->port_role;
+	newrec->localport.port_id = pinfo->port_id;
+	newrec->localport.port_state = FC_OBJSTATE_ONLINE;
+	newrec->localport.port_num = idx;
+
+	spin_lock_irqsave(&nvme_fc_lock, flags);
+	list_add_tail(&newrec->port_list, &nvme_fc_lport_list);
+	spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+	if (dev)
+		dma_set_seg_boundary(dev, template->dma_boundary);
+
+	*portptr = &newrec->localport;
+	return 0;
+
+out_ida_put:
+	ida_simple_remove(&nvme_fc_local_port_cnt, idx);
+out_fail_kfree:
+	kfree(newrec);
+out_reghost_failed:
+	*portptr = NULL;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(nvme_fc_register_localport);
+
+static void
+nvme_fc_free_lport(struct kref *ref)
+{
+	struct nvme_fc_lport *lport =
+		container_of(ref, struct nvme_fc_lport, ref);
+	unsigned long flags;
+
+	WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
+	WARN_ON(!list_empty(&lport->endp_list));
+
+	/* remove from transport list */
+	spin_lock_irqsave(&nvme_fc_lock, flags);
+	list_del(&lport->port_list);
+	spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+	/* let the LLDD know we've finished tearing it down */
+	lport->ops->localport_delete(&lport->localport);
+
+	ida_simple_remove(&nvme_fc_local_port_cnt, lport->localport.port_num);
+	ida_destroy(&lport->endp_cnt);
+
+	put_device(lport->dev);
+
+	kfree(lport);
+}
+
+static void
+nvme_fc_lport_put(struct nvme_fc_lport *lport)
+{
+	kref_put(&lport->ref, nvme_fc_free_lport);
+}
+
+static int
+nvme_fc_lport_get(struct nvme_fc_lport *lport)
+{
+	return kref_get_unless_zero(&lport->ref);
+}
+
+/**
+ * nvme_fc_unregister_localport - transport entry point called by an
+ *                              LLDD to deregister/remove a previously
+ *                              registered a NVME host FC port.
+ * @localport: pointer to the (registered) local port that is to be
+ *             deregistered.
+ *
+ * Returns:
+ * a completion status. Must be 0 upon success; a negative errno
+ * (ex: -ENXIO) upon failure.
+ */
+int
+nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr)
+{
+	struct nvme_fc_lport *lport = localport_to_lport(portptr);
+	unsigned long flags;
+
+	if (!portptr)
+		return -EINVAL;
+
+	spin_lock_irqsave(&nvme_fc_lock, flags);
+
+	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
+		spin_unlock_irqrestore(&nvme_fc_lock, flags);
+		return -EINVAL;
+	}
+	portptr->port_state = FC_OBJSTATE_DELETED;
+
+	spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+	nvme_fc_lport_put(lport);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport);
+
+/**
+ * nvme_fc_register_remoteport - transport entry point called by an
+ *                              LLDD to register the existence of a NVME
+ *                              subsystem FC port on its fabric.
+ * @localport: pointer to the (registered) local port that the remote
+ *             subsystem port is connected to.
+ * @pinfo:     pointer to information about the port to be registered
+ * @rport_p:   pointer to a remote port pointer. Upon success, the routine
+ *             will allocate a nvme_fc_remote_port structure and place its
+ *             address in the remote port pointer. Upon failure, remote port
+ *             pointer will be set to 0.
+ *
+ * Returns:
+ * a completion status. Must be 0 upon success; a negative errno
+ * (ex: -ENXIO) upon failure.
+ */
+int
+nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
+				struct nvme_fc_port_info *pinfo,
+				struct nvme_fc_remote_port **portptr)
+{
+	struct nvme_fc_lport *lport = localport_to_lport(localport);
+	struct nvme_fc_rport *newrec;
+	unsigned long flags;
+	int ret, idx;
+
+	newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz),
+			 GFP_KERNEL);
+	if (!newrec) {
+		ret = -ENOMEM;
+		goto out_reghost_failed;
+	}
+
+	if (!nvme_fc_lport_get(lport)) {
+		ret = -ESHUTDOWN;
+		goto out_kfree_rport;
+	}
+
+	idx = ida_simple_get(&lport->endp_cnt, 0, 0, GFP_KERNEL);
+	if (idx < 0) {
+		ret = -ENOSPC;
+		goto out_lport_put;
+	}
+
+	INIT_LIST_HEAD(&newrec->endp_list);
+	INIT_LIST_HEAD(&newrec->ctrl_list);
+	kref_init(&newrec->ref);
+	spin_lock_init(&newrec->lock);
+	newrec->remoteport.localport = &lport->localport;
+	newrec->remoteport.private = &newrec[1];
+	newrec->remoteport.port_role = pinfo->port_role;
+	newrec->remoteport.node_name = pinfo->node_name;
+	newrec->remoteport.port_name = pinfo->port_name;
+	newrec->remoteport.port_id = pinfo->port_id;
+	newrec->remoteport.port_state = FC_OBJSTATE_ONLINE;
+	newrec->remoteport.port_num = idx;
+
+	spin_lock_irqsave(&nvme_fc_lock, flags);
+	list_add_tail(&newrec->endp_list, &lport->endp_list);
+	spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+	*portptr = &newrec->remoteport;
+	return 0;
+
+out_lport_put:
+	nvme_fc_lport_put(lport);
+out_kfree_rport:
+	kfree(newrec);
+out_reghost_failed:
+	*portptr = NULL;
+	return ret;
+
+}
+EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);
+
+static void
+nvme_fc_free_rport(struct kref *ref)
+{
+	struct nvme_fc_rport *rport =
+		container_of(ref, struct nvme_fc_rport, ref);
+	struct nvme_fc_lport *lport =
+			localport_to_lport(rport->remoteport.localport);
+	unsigned long flags;
+
+	WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED);
+	WARN_ON(!list_empty(&rport->ctrl_list));
+
+	/* remove from lport list */
+	spin_lock_irqsave(&nvme_fc_lock, flags);
+	list_del(&rport->endp_list);
+	spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+	/* let the LLDD know we've finished tearing it down */
+	lport->ops->remoteport_delete(&rport->remoteport);
+
+	ida_simple_remove(&lport->endp_cnt, rport->remoteport.port_num);
+
+	kfree(rport);
+
+	nvme_fc_lport_put(lport);
+}
+
+static void
+nvme_fc_rport_put(struct nvme_fc_rport *rport)
+{
+	kref_put(&rport->ref, nvme_fc_free_rport);
+}
+
+static int
+nvme_fc_rport_get(struct nvme_fc_rport *rport)
+{
+	return kref_get_unless_zero(&rport->ref);
+}
+
+/**
+ * nvme_fc_unregister_remoteport - transport entry point called by an
+ *                              LLDD to deregister/remove a previously
+ *                              registered a NVME subsystem FC port.
+ * @remoteport: pointer to the (registered) remote port that is to be
+ *              deregistered.
+ *
+ * Returns:
+ * a completion status. Must be 0 upon success; a negative errno
+ * (ex: -ENXIO) upon failure.
+ */
+int
+nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
+{
+	struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
+	struct nvme_fc_ctrl *ctrl;
+	unsigned long flags;
+
+	if (!portptr)
+		return -EINVAL;
+
+	spin_lock_irqsave(&rport->lock, flags);
+
+	if (portptr->port_state != FC_OBJSTATE_ONLINE) {
+		spin_unlock_irqrestore(&rport->lock, flags);
+		return -EINVAL;
+	}
+	portptr->port_state = FC_OBJSTATE_DELETED;
+
+	/* tear down all associations to the remote port */
+	list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list)
+		__nvme_fc_del_ctrl(ctrl);
+
+	spin_unlock_irqrestore(&rport->lock, flags);
+
+	nvme_fc_rport_put(rport);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport);
+
+
+/* *********************** FC-NVME DMA Handling **************************** */
+
+/*
+ * The fcloop device passes in a NULL device pointer. Real LLD's will
+ * pass in a valid device pointer. If NULL is passed to the dma mapping
+ * routines, depending on the platform, it may or may not succeed, and
+ * may crash.
+ *
+ * As such:
+ * Wrapper all the dma routines and check the dev pointer.
+ *
+ * If simple mappings (return just a dma address, we'll noop them,
+ * returning a dma address of 0.
+ *
+ * On more complex mappings (dma_map_sg), a pseudo routine fills
+ * in the scatter list, setting all dma addresses to 0.
+ */
+
+static inline dma_addr_t
+fc_dma_map_single(struct device *dev, void *ptr, size_t size,
+		enum dma_data_direction dir)
+{
+	return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
+}
+
+static inline int
+fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
+{
+	return dev ? dma_mapping_error(dev, dma_addr) : 0;
+}
+
+static inline void
+fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
+	enum dma_data_direction dir)
+{
+	if (dev)
+		dma_unmap_single(dev, addr, size, dir);
+}
+
+static inline void
+fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
+		enum dma_data_direction dir)
+{
+	if (dev)
+		dma_sync_single_for_cpu(dev, addr, size, dir);
+}
+
+static inline void
+fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
+		enum dma_data_direction dir)
+{
+	if (dev)
+		dma_sync_single_for_device(dev, addr, size, dir);
+}
+
+/* pseudo dma_map_sg call */
+static int
+fc_map_sg(struct scatterlist *sg, int nents)
+{
+	struct scatterlist *s;
+	int i;
+
+	WARN_ON(nents == 0 || sg[0].length == 0);
+
+	for_each_sg(sg, s, nents, i) {
+		s->dma_address = 0L;
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+		s->dma_length = s->length;
+#endif
+	}
+	return nents;
+}
+
+static inline int
+fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
+		enum dma_data_direction dir)
+{
+	return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
+}
+
+static inline void
+fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
+		enum dma_data_direction dir)
+{
+	if (dev)
+		dma_unmap_sg(dev, sg, nents, dir);
+}
+
+
+/* *********************** FC-NVME LS Handling **************************** */
+
+static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
+static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
+
+
+static void
+__nvme_fc_finish_ls_req(struct nvme_fc_ctrl *ctrl,
+		struct nvmefc_ls_req_op *lsop)
+{
+	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ctrl->lock, flags);
+
+	if (!lsop->req_queued) {
+		spin_unlock_irqrestore(&ctrl->lock, flags);
+		return;
+	}
+
+	list_del(&lsop->lsreq_list);
+
+	lsop->req_queued = false;
+
+	spin_unlock_irqrestore(&ctrl->lock, flags);
+
+	fc_dma_unmap_single(ctrl->dev, lsreq->rqstdma,
+				  (lsreq->rqstlen + lsreq->rsplen),
+				  DMA_BIDIRECTIONAL);
+
+	nvme_fc_ctrl_put(ctrl);
+}
+
+static int
+__nvme_fc_send_ls_req(struct nvme_fc_ctrl *ctrl,
+		struct nvmefc_ls_req_op *lsop,
+		void (*done)(struct nvmefc_ls_req *req, int status))
+{
+	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
+	unsigned long flags;
+	int ret;
+
+	if (!nvme_fc_ctrl_get(ctrl))
+		return -ESHUTDOWN;
+
+	lsreq->done = done;
+	lsop->ctrl = ctrl;
+	lsop->req_queued = false;
+	INIT_LIST_HEAD(&lsop->lsreq_list);
+	init_completion(&lsop->ls_done);
+
+	lsreq->rqstdma = fc_dma_map_single(ctrl->dev, lsreq->rqstaddr,
+				  lsreq->rqstlen + lsreq->rsplen,
+				  DMA_BIDIRECTIONAL);
+	if (fc_dma_mapping_error(ctrl->dev, lsreq->rqstdma)) {
+		nvme_fc_ctrl_put(ctrl);
+		dev_err(ctrl->dev,
+			"els request command failed EFAULT.\n");
+		return -EFAULT;
+	}
+	lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
+
+	spin_lock_irqsave(&ctrl->lock, flags);
+
+	list_add_tail(&lsop->lsreq_list, &ctrl->ls_req_list);
+
+	lsop->req_queued = true;
+
+	spin_unlock_irqrestore(&ctrl->lock, flags);
+
+	ret = ctrl->lport->ops->ls_req(&ctrl->lport->localport,
+					&ctrl->rport->remoteport, lsreq);
+	if (ret)
+		lsop->ls_error = ret;
+
+	return ret;
+}
+
+static void
+nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
+{
+	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
+
+	lsop->ls_error = status;
+	complete(&lsop->ls_done);
+}
+
+static int
+nvme_fc_send_ls_req(struct nvme_fc_ctrl *ctrl, struct nvmefc_ls_req_op *lsop)
+{
+	struct nvmefc_ls_req *lsreq = &lsop->ls_req;
+	struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
+	int ret;
+
+	ret = __nvme_fc_send_ls_req(ctrl, lsop, nvme_fc_send_ls_req_done);
+
+	if (!ret)
+		/*
+		 * No timeout/not interruptible as we need the struct
+		 * to exist until the lldd calls us back. Thus mandate
+		 * wait until driver calls back. lldd responsible for
+		 * the timeout action
+		 */
+		wait_for_completion(&lsop->ls_done);
+
+	__nvme_fc_finish_ls_req(ctrl, lsop);
+
+	if (ret) {
+		dev_err(ctrl->dev,
+			"ls request command failed (%d).\n", ret);
+		return ret;
+	}
+
+	/* ACC or RJT payload ? */
+	if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
+		return -ENXIO;
+
+	return 0;
+}
+
+static void
+nvme_fc_send_ls_req_async(struct nvme_fc_ctrl *ctrl,
+		struct nvmefc_ls_req_op *lsop,
+		void (*done)(struct nvmefc_ls_req *req, int status))
+{
+	int ret;
+
+	ret = __nvme_fc_send_ls_req(ctrl, lsop, done);
+
+	/* don't wait for completion */
+
+	if (ret)
+		done(&lsop->ls_req, ret);
+}
+
+/* Validation Error indexes into the string table below */
+enum {
+	VERR_NO_ERROR		= 0,
+	VERR_LSACC		= 1,
+	VERR_LSDESC_RQST	= 2,
+	VERR_LSDESC_RQST_LEN	= 3,
+	VERR_ASSOC_ID		= 4,
+	VERR_ASSOC_ID_LEN	= 5,
+	VERR_CONN_ID		= 6,
+	VERR_CONN_ID_LEN	= 7,
+	VERR_CR_ASSOC		= 8,
+	VERR_CR_ASSOC_ACC_LEN	= 9,
+	VERR_CR_CONN		= 10,
+	VERR_CR_CONN_ACC_LEN	= 11,
+	VERR_DISCONN		= 12,
+	VERR_DISCONN_ACC_LEN	= 13,
+};
+
+static char *validation_errors[] = {
+	"OK",
+	"Not LS_ACC",
+	"Not LSDESC_RQST",
+	"Bad LSDESC_RQST Length",
+	"Not Association ID",
+	"Bad Association ID Length",
+	"Not Connection ID",
+	"Bad Connection ID Length",
+	"Not CR_ASSOC Rqst",
+	"Bad CR_ASSOC ACC Length",
+	"Not CR_CONN Rqst",
+	"Bad CR_CONN ACC Length",
+	"Not Disconnect Rqst",
+	"Bad Disconnect ACC Length",
+};
+
+static int
+nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
+	struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio)
+{
+	struct nvmefc_ls_req_op *lsop;
+	struct nvmefc_ls_req *lsreq;
+	struct fcnvme_ls_cr_assoc_rqst *assoc_rqst;
+	struct fcnvme_ls_cr_assoc_acc *assoc_acc;
+	int ret, fcret = 0;
+
+	lsop = kzalloc((sizeof(*lsop) +
+			 ctrl->lport->ops->lsrqst_priv_sz +
+			 sizeof(*assoc_rqst) + sizeof(*assoc_acc)), GFP_KERNEL);
+	if (!lsop) {
+		ret = -ENOMEM;
+		goto out_no_memory;
+	}
+	lsreq = &lsop->ls_req;
+
+	lsreq->private = (void *)&lsop[1];
+	assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)
+			(lsreq->private + ctrl->lport->ops->lsrqst_priv_sz);
+	assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1];
+
+	assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION;
+	assoc_rqst->desc_list_len =
+			cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
+
+	assoc_rqst->assoc_cmd.desc_tag =
+			cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD);
+	assoc_rqst->assoc_cmd.desc_len =
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
+
+	assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
+	assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize);
+	/* Linux supports only Dynamic controllers */
+	assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
+	memcpy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id,
+		min_t(size_t, FCNVME_ASSOC_HOSTID_LEN, sizeof(uuid_be)));
+	strncpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn,
+		min(FCNVME_ASSOC_HOSTNQN_LEN, NVMF_NQN_SIZE));
+	strncpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn,
+		min(FCNVME_ASSOC_SUBNQN_LEN, NVMF_NQN_SIZE));
+
+	lsop->queue = queue;
+	lsreq->rqstaddr = assoc_rqst;
+	lsreq->rqstlen = sizeof(*assoc_rqst);
+	lsreq->rspaddr = assoc_acc;
+	lsreq->rsplen = sizeof(*assoc_acc);
+	lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC;
+
+	ret = nvme_fc_send_ls_req(ctrl, lsop);
+	if (ret)
+		goto out_free_buffer;
+
+	/* process connect LS completion */
+
+	/* validate the ACC response */
+	if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
+		fcret = VERR_LSACC;
+	if (assoc_acc->hdr.desc_list_len !=
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_ls_cr_assoc_acc)))
+		fcret = VERR_CR_ASSOC_ACC_LEN;
+	if (assoc_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
+		fcret = VERR_LSDESC_RQST;
+	else if (assoc_acc->hdr.rqst.desc_len !=
+			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
+		fcret = VERR_LSDESC_RQST_LEN;
+	else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION)
+		fcret = VERR_CR_ASSOC;
+	else if (assoc_acc->associd.desc_tag !=
+			cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
+		fcret = VERR_ASSOC_ID;
+	else if (assoc_acc->associd.desc_len !=
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_assoc_id)))
+		fcret = VERR_ASSOC_ID_LEN;
+	else if (assoc_acc->connectid.desc_tag !=
+			cpu_to_be32(FCNVME_LSDESC_CONN_ID))
+		fcret = VERR_CONN_ID;
+	else if (assoc_acc->connectid.desc_len !=
+			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
+		fcret = VERR_CONN_ID_LEN;
+
+	if (fcret) {
+		ret = -EBADF;
+		dev_err(ctrl->dev,
+			"q %d connect failed: %s\n",
+			queue->qnum, validation_errors[fcret]);
+	} else {
+		ctrl->association_id =
+			be64_to_cpu(assoc_acc->associd.association_id);
+		queue->connection_id =
+			be64_to_cpu(assoc_acc->connectid.connection_id);
+		set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
+	}
+
+out_free_buffer:
+	kfree(lsop);
+out_no_memory:
+	if (ret)
+		dev_err(ctrl->dev,
+			"queue %d connect admin queue failed (%d).\n",
+			queue->qnum, ret);
+	return ret;
+}
+
+static int
+nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
+			u16 qsize, u16 ersp_ratio)
+{
+	struct nvmefc_ls_req_op *lsop;
+	struct nvmefc_ls_req *lsreq;
+	struct fcnvme_ls_cr_conn_rqst *conn_rqst;
+	struct fcnvme_ls_cr_conn_acc *conn_acc;
+	int ret, fcret = 0;
+
+	lsop = kzalloc((sizeof(*lsop) +
+			 ctrl->lport->ops->lsrqst_priv_sz +
+			 sizeof(*conn_rqst) + sizeof(*conn_acc)), GFP_KERNEL);
+	if (!lsop) {
+		ret = -ENOMEM;
+		goto out_no_memory;
+	}
+	lsreq = &lsop->ls_req;
+
+	lsreq->private = (void *)&lsop[1];
+	conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)
+			(lsreq->private + ctrl->lport->ops->lsrqst_priv_sz);
+	conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1];
+
+	conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION;
+	conn_rqst->desc_list_len = cpu_to_be32(
+				sizeof(struct fcnvme_lsdesc_assoc_id) +
+				sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
+
+	conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
+	conn_rqst->associd.desc_len =
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_assoc_id));
+	conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
+	conn_rqst->connect_cmd.desc_tag =
+			cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD);
+	conn_rqst->connect_cmd.desc_len =
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
+	conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
+	conn_rqst->connect_cmd.qid  = cpu_to_be16(queue->qnum);
+	conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize);
+
+	lsop->queue = queue;
+	lsreq->rqstaddr = conn_rqst;
+	lsreq->rqstlen = sizeof(*conn_rqst);
+	lsreq->rspaddr = conn_acc;
+	lsreq->rsplen = sizeof(*conn_acc);
+	lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC;
+
+	ret = nvme_fc_send_ls_req(ctrl, lsop);
+	if (ret)
+		goto out_free_buffer;
+
+	/* process connect LS completion */
+
+	/* validate the ACC response */
+	if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
+		fcret = VERR_LSACC;
+	if (conn_acc->hdr.desc_list_len !=
+			fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)))
+		fcret = VERR_CR_CONN_ACC_LEN;
+	if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
+		fcret = VERR_LSDESC_RQST;
+	else if (conn_acc->hdr.rqst.desc_len !=
+			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
+		fcret = VERR_LSDESC_RQST_LEN;
+	else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION)
+		fcret = VERR_CR_CONN;
+	else if (conn_acc->connectid.desc_tag !=
+			cpu_to_be32(FCNVME_LSDESC_CONN_ID))
+		fcret = VERR_CONN_ID;
+	else if (conn_acc->connectid.desc_len !=
+			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
+		fcret = VERR_CONN_ID_LEN;
+
+	if (fcret) {
+		ret = -EBADF;
+		dev_err(ctrl->dev,
+			"q %d connect failed: %s\n",
+			queue->qnum, validation_errors[fcret]);
+	} else {
+		queue->connection_id =
+			be64_to_cpu(conn_acc->connectid.connection_id);
+		set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
+	}
+
+out_free_buffer:
+	kfree(lsop);
+out_no_memory:
+	if (ret)
+		dev_err(ctrl->dev,
+			"queue %d connect command failed (%d).\n",
+			queue->qnum, ret);
+	return ret;
+}
+
+static void
+nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
+{
+	struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
+	struct nvme_fc_ctrl *ctrl = lsop->ctrl;
+
+	__nvme_fc_finish_ls_req(ctrl, lsop);
+
+	if (status)
+		dev_err(ctrl->dev,
+			"disconnect assoc ls request command failed (%d).\n",
+			status);
+
+	/* fc-nvme iniator doesn't care about success or failure of cmd */
+
+	kfree(lsop);
+}
+
+/*
+ * This routine sends a FC-NVME LS to disconnect (aka terminate)
+ * the FC-NVME Association.  Terminating the association also
+ * terminates the FC-NVME connections (per queue, both admin and io
+ * queues) that are part of the association. E.g. things are torn
+ * down, and the related FC-NVME Association ID and Connection IDs
+ * become invalid.
+ *
+ * The behavior of the fc-nvme initiator is such that it's
+ * understanding of the association and connections will implicitly
+ * be torn down. The action is implicit as it may be due to a loss of
+ * connectivity with the fc-nvme target, so you may never get a
+ * response even if you tried.  As such, the action of this routine
+ * is to asynchronously send the LS, ignore any results of the LS, and
+ * continue on with terminating the association. If the fc-nvme target
+ * is present and receives the LS, it too can tear down.
+ */
+static void
+nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
+{
+	struct fcnvme_ls_disconnect_rqst *discon_rqst;
+	struct fcnvme_ls_disconnect_acc *discon_acc;
+	struct nvmefc_ls_req_op *lsop;
+	struct nvmefc_ls_req *lsreq;
+
+	lsop = kzalloc((sizeof(*lsop) +
+			 ctrl->lport->ops->lsrqst_priv_sz +
+			 sizeof(*discon_rqst) + sizeof(*discon_acc)),
+			GFP_KERNEL);
+	if (!lsop)
+		/* couldn't sent it... too bad */
+		return;
+
+	lsreq = &lsop->ls_req;
+
+	lsreq->private = (void *)&lsop[1];
+	discon_rqst = (struct fcnvme_ls_disconnect_rqst *)
+			(lsreq->private + ctrl->lport->ops->lsrqst_priv_sz);
+	discon_acc = (struct fcnvme_ls_disconnect_acc *)&discon_rqst[1];
+
+	discon_rqst->w0.ls_cmd = FCNVME_LS_DISCONNECT;
+	discon_rqst->desc_list_len = cpu_to_be32(
+				sizeof(struct fcnvme_lsdesc_assoc_id) +
+				sizeof(struct fcnvme_lsdesc_disconn_cmd));
+
+	discon_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
+	discon_rqst->associd.desc_len =
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_assoc_id));
+
+	discon_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
+
+	discon_rqst->discon_cmd.desc_tag = cpu_to_be32(
+						FCNVME_LSDESC_DISCONN_CMD);
+	discon_rqst->discon_cmd.desc_len =
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_disconn_cmd));
+	discon_rqst->discon_cmd.scope = FCNVME_DISCONN_ASSOCIATION;
+	discon_rqst->discon_cmd.id = cpu_to_be64(ctrl->association_id);
+
+	lsreq->rqstaddr = discon_rqst;
+	lsreq->rqstlen = sizeof(*discon_rqst);
+	lsreq->rspaddr = discon_acc;
+	lsreq->rsplen = sizeof(*discon_acc);
+	lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC;
+
+	nvme_fc_send_ls_req_async(ctrl, lsop, nvme_fc_disconnect_assoc_done);
+
+	/* only meaningful part to terminating the association */
+	ctrl->association_id = 0;
+}
+
+
+/* *********************** NVME Ctrl Routines **************************** */
+
+
+static int
+nvme_fc_reinit_request(void *data, struct request *rq)
+{
+	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
+	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
+
+	memset(cmdiu, 0, sizeof(*cmdiu));
+	cmdiu->scsi_id = NVME_CMD_SCSI_ID;
+	cmdiu->fc_id = NVME_CMD_FC_ID;
+	cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
+	memset(&op->rsp_iu, 0, sizeof(op->rsp_iu));
+
+	return 0;
+}
+
+static void
+__nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl,
+		struct nvme_fc_fcp_op *op)
+{
+	fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.rspdma,
+				sizeof(op->rsp_iu), DMA_FROM_DEVICE);
+	fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma,
+				sizeof(op->cmd_iu), DMA_TO_DEVICE);
+
+	atomic_set(&op->state, FCPOP_STATE_UNINIT);
+}
+
+static void
+nvme_fc_exit_request(void *data, struct request *rq,
+				unsigned int hctx_idx, unsigned int rq_idx)
+{
+	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
+
+	return __nvme_fc_exit_request(data, op);
+}
+
+static void
+nvme_fc_exit_aen_ops(struct nvme_fc_ctrl *ctrl)
+{
+	struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
+	int i;
+
+	for (i = 0; i < NVME_FC_NR_AEN_COMMANDS; i++, aen_op++) {
+		if (atomic_read(&aen_op->state) == FCPOP_STATE_UNINIT)
+			continue;
+		__nvme_fc_exit_request(ctrl, aen_op);
+		nvme_fc_ctrl_put(ctrl);
+	}
+}
+
+void
+nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
+{
+	struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
+	struct request *rq = op->rq;
+	struct nvmefc_fcp_req *freq = &op->fcp_req;
+	struct nvme_fc_ctrl *ctrl = op->ctrl;
+	struct nvme_fc_queue *queue = op->queue;
+	struct nvme_completion *cqe = &op->rsp_iu.cqe;
+	u16 status;
+
+	/*
+	 * WARNING:
+	 * The current linux implementation of a nvme controller
+	 * allocates a single tag set for all io queues and sizes
+	 * the io queues to fully hold all possible tags. Thus, the
+	 * implementation does not reference or care about the sqhd
+	 * value as it never needs to use the sqhd/sqtail pointers
+	 * for submission pacing.
+	 *
+	 * This affects the FC-NVME implementation in two ways:
+	 * 1) As the value doesn't matter, we don't need to waste
+	 *    cycles extracting it from ERSPs and stamping it in the
+	 *    cases where the transport fabricates CQEs on successful
+	 *    completions.
+	 * 2) The FC-NVME implementation requires that delivery of
+	 *    ERSP completions are to go back to the nvme layer in order
+	 *    relative to the rsn, such that the sqhd value will always
+	 *    be "in order" for the nvme layer. As the nvme layer in
+	 *    linux doesn't care about sqhd, there's no need to return
+	 *    them in order.
+	 *
+	 * Additionally:
+	 * As the core nvme layer in linux currently does not look at
+	 * every field in the cqe - in cases where the FC transport must
+	 * fabricate a CQE, the following fields will not be set as they
+	 * are not referenced:
+	 *      cqe.sqid,  cqe.sqhd,  cqe.command_id
+	 */
+
+	fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma,
+				sizeof(op->rsp_iu), DMA_FROM_DEVICE);
+
+	if (atomic_read(&op->state) == FCPOP_STATE_ABORTED)
+		status = NVME_SC_ABORT_REQ | NVME_SC_DNR;
+	else
+		status = freq->status;
+
+	/*
+	 * For the linux implementation, if we have an unsuccesful
+	 * status, they blk-mq layer can typically be called with the
+	 * non-zero status and the content of the cqe isn't important.
+	 */
+	if (status)
+		goto done;
+
+	/*
+	 * command completed successfully relative to the wire
+	 * protocol. However, validate anything received and
+	 * extract the status and result from the cqe (create it
+	 * where necessary).
+	 */
+
+	switch (freq->rcv_rsplen) {
+
+	case 0:
+	case NVME_FC_SIZEOF_ZEROS_RSP:
+		/*
+		 * No response payload or 12 bytes of payload (which
+		 * should all be zeros) are considered successful and
+		 * no payload in the CQE by the transport.
+		 */
+		if (freq->transferred_length !=
+			be32_to_cpu(op->cmd_iu.data_len)) {
+			status = -EIO;
+			goto done;
+		}
+		op->nreq.result.u64 = 0;
+		break;
+
+	case sizeof(struct nvme_fc_ersp_iu):
+		/*
+		 * The ERSP IU contains a full completion with CQE.
+		 * Validate ERSP IU and look at cqe.
+		 */
+		if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) !=
+					(freq->rcv_rsplen / 4) ||
+			     be32_to_cpu(op->rsp_iu.xfrd_len) !=
+					freq->transferred_length ||
+			     op->rqno != le16_to_cpu(cqe->command_id))) {
+			status = -EIO;
+			goto done;
+		}
+		op->nreq.result = cqe->result;
+		status = le16_to_cpu(cqe->status) >> 1;
+		break;
+
+	default:
+		status = -EIO;
+		goto done;
+	}
+
+done:
+	if (!queue->qnum && op->rqno >= AEN_CMDID_BASE) {
+		nvme_complete_async_event(&queue->ctrl->ctrl, status,
+					&op->nreq.result);
+		nvme_fc_ctrl_put(ctrl);
+		return;
+	}
+
+	blk_mq_complete_request(rq, status);
+}
+
+static int
+__nvme_fc_init_request(struct nvme_fc_ctrl *ctrl,
+		struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op,
+		struct request *rq, u32 rqno)
+{
+	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
+	int ret = 0;
+
+	memset(op, 0, sizeof(*op));
+	op->fcp_req.cmdaddr = &op->cmd_iu;
+	op->fcp_req.cmdlen = sizeof(op->cmd_iu);
+	op->fcp_req.rspaddr = &op->rsp_iu;
+	op->fcp_req.rsplen = sizeof(op->rsp_iu);
+	op->fcp_req.done = nvme_fc_fcpio_done;
+	op->fcp_req.first_sgl = (struct scatterlist *)&op[1];
+	op->fcp_req.private = &op->fcp_req.first_sgl[SG_CHUNK_SIZE];
+	op->ctrl = ctrl;
+	op->queue = queue;
+	op->rq = rq;
+	op->rqno = rqno;
+
+	cmdiu->scsi_id = NVME_CMD_SCSI_ID;
+	cmdiu->fc_id = NVME_CMD_FC_ID;
+	cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
+
+	op->fcp_req.cmddma = fc_dma_map_single(ctrl->lport->dev,
+				&op->cmd_iu, sizeof(op->cmd_iu), DMA_TO_DEVICE);
+	if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.cmddma)) {
+		dev_err(ctrl->dev,
+			"FCP Op failed - cmdiu dma mapping failed.\n");
+		ret = EFAULT;
+		goto out_on_error;
+	}
+
+	op->fcp_req.rspdma = fc_dma_map_single(ctrl->lport->dev,
+				&op->rsp_iu, sizeof(op->rsp_iu),
+				DMA_FROM_DEVICE);
+	if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.rspdma)) {
+		dev_err(ctrl->dev,
+			"FCP Op failed - rspiu dma mapping failed.\n");
+		ret = EFAULT;
+	}
+
+	atomic_set(&op->state, FCPOP_STATE_IDLE);
+out_on_error:
+	return ret;
+}
+
+static int
+nvme_fc_init_request(void *data, struct request *rq,
+				unsigned int hctx_idx, unsigned int rq_idx,
+				unsigned int numa_node)
+{
+	struct nvme_fc_ctrl *ctrl = data;
+	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
+	struct nvme_fc_queue *queue = &ctrl->queues[hctx_idx+1];
+
+	return __nvme_fc_init_request(ctrl, queue, op, rq, queue->rqcnt++);
+}
+
+static int
+nvme_fc_init_admin_request(void *data, struct request *rq,
+				unsigned int hctx_idx, unsigned int rq_idx,
+				unsigned int numa_node)
+{
+	struct nvme_fc_ctrl *ctrl = data;
+	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
+	struct nvme_fc_queue *queue = &ctrl->queues[0];
+
+	return __nvme_fc_init_request(ctrl, queue, op, rq, queue->rqcnt++);
+}
+
+static int
+nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl)
+{
+	struct nvme_fc_fcp_op *aen_op;
+	struct nvme_fc_cmd_iu *cmdiu;
+	struct nvme_command *sqe;
+	int i, ret;
+
+	aen_op = ctrl->aen_ops;
+	for (i = 0; i < NVME_FC_NR_AEN_COMMANDS; i++, aen_op++) {
+		cmdiu = &aen_op->cmd_iu;
+		sqe = &cmdiu->sqe;
+		ret = __nvme_fc_init_request(ctrl, &ctrl->queues[0],
+				aen_op, (struct request *)NULL,
+				(AEN_CMDID_BASE + i));
+		if (ret)
+			return ret;
+
+		memset(sqe, 0, sizeof(*sqe));
+		sqe->common.opcode = nvme_admin_async_event;
+		sqe->common.command_id = AEN_CMDID_BASE + i;
+	}
+	return 0;
+}
+
+
+static inline void
+__nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, struct nvme_fc_ctrl *ctrl,
+		unsigned int qidx)
+{
+	struct nvme_fc_queue *queue = &ctrl->queues[qidx];
+
+	hctx->driver_data = queue;
+	queue->hctx = hctx;
+}
+
+static int
+nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
+		unsigned int hctx_idx)
+{
+	struct nvme_fc_ctrl *ctrl = data;
+
+	__nvme_fc_init_hctx(hctx, ctrl, hctx_idx + 1);
+
+	return 0;
+}
+
+static int
+nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
+		unsigned int hctx_idx)
+{
+	struct nvme_fc_ctrl *ctrl = data;
+
+	__nvme_fc_init_hctx(hctx, ctrl, hctx_idx);
+
+	return 0;
+}
+
+static void
+nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx, size_t queue_size)
+{
+	struct nvme_fc_queue *queue;
+
+	queue = &ctrl->queues[idx];
+	memset(queue, 0, sizeof(*queue));
+	queue->ctrl = ctrl;
+	queue->qnum = idx;
+	atomic_set(&queue->csn, 1);
+	queue->dev = ctrl->dev;
+
+	if (idx > 0)
+		queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
+	else
+		queue->cmnd_capsule_len = sizeof(struct nvme_command);
+
+	queue->queue_size = queue_size;
+
+	/*
+	 * Considered whether we should allocate buffers for all SQEs
+	 * and CQEs and dma map them - mapping their respective entries
+	 * into the request structures (kernel vm addr and dma address)
+	 * thus the driver could use the buffers/mappings directly.
+	 * It only makes sense if the LLDD would use them for its
+	 * messaging api. It's very unlikely most adapter api's would use
+	 * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
+	 * structures were used instead.
+	 */
+}
+
+/*
+ * This routine terminates a queue at the transport level.
+ * The transport has already ensured that all outstanding ios on
+ * the queue have been terminated.
+ * The transport will send a Disconnect LS request to terminate
+ * the queue's connection. Termination of the admin queue will also
+ * terminate the association at the target.
+ */
+static void
+nvme_fc_free_queue(struct nvme_fc_queue *queue)
+{
+	if (!test_and_clear_bit(NVME_FC_Q_CONNECTED, &queue->flags))
+		return;
+
+	/*
+	 * Current implementation never disconnects a single queue.
+	 * It always terminates a whole association. So there is never
+	 * a disconnect(queue) LS sent to the target.
+	 */
+
+	queue->connection_id = 0;
+	clear_bit(NVME_FC_Q_CONNECTED, &queue->flags);
+}
+
+static void
+__nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl,
+	struct nvme_fc_queue *queue, unsigned int qidx)
+{
+	if (ctrl->lport->ops->delete_queue)
+		ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx,
+				queue->lldd_handle);
+	queue->lldd_handle = NULL;
+}
+
+static void
+nvme_fc_destroy_admin_queue(struct nvme_fc_ctrl *ctrl)
+{
+	__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
+	blk_cleanup_queue(ctrl->ctrl.admin_q);
+	blk_mq_free_tag_set(&ctrl->admin_tag_set);
+	nvme_fc_free_queue(&ctrl->queues[0]);
+}
+
+static void
+nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl)
+{
+	int i;
+
+	for (i = 1; i < ctrl->queue_count; i++)
+		nvme_fc_free_queue(&ctrl->queues[i]);
+}
+
+static int
+__nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl,
+	struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize)
+{
+	int ret = 0;
+
+	queue->lldd_handle = NULL;
+	if (ctrl->lport->ops->create_queue)
+		ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport,
+				qidx, qsize, &queue->lldd_handle);
+
+	return ret;
+}
+
+static void
+nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)
+{
+	struct nvme_fc_queue *queue = &ctrl->queues[ctrl->queue_count - 1];
+	int i;
+
+	for (i = ctrl->queue_count - 1; i >= 1; i--, queue--)
+		__nvme_fc_delete_hw_queue(ctrl, queue, i);
+}
+
+static int
+nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
+{
+	struct nvme_fc_queue *queue = &ctrl->queues[1];
+	int i, j, ret;
+
+	for (i = 1; i < ctrl->queue_count; i++, queue++) {
+		ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize);
+		if (ret) {
+			for (j = i-1; j >= 0; j--)
+				__nvme_fc_delete_hw_queue(ctrl,
+						&ctrl->queues[j], j);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
+nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
+{
+	int i, ret = 0;
+
+	for (i = 1; i < ctrl->queue_count; i++) {
+		ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize,
+					(qsize / 5));
+		if (ret)
+			break;
+		ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
+		if (ret)
+			break;
+	}
+
+	return ret;
+}
+
+static void
+nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl)
+{
+	int i;
+
+	for (i = 1; i < ctrl->queue_count; i++)
+		nvme_fc_init_queue(ctrl, i, ctrl->ctrl.sqsize);
+}
+
+static void
+nvme_fc_ctrl_free(struct kref *ref)
+{
+	struct nvme_fc_ctrl *ctrl =
+		container_of(ref, struct nvme_fc_ctrl, ref);
+	unsigned long flags;
+
+	if (ctrl->state != FCCTRL_INIT) {
+		/* remove from rport list */
+		spin_lock_irqsave(&ctrl->rport->lock, flags);
+		list_del(&ctrl->ctrl_list);
+		spin_unlock_irqrestore(&ctrl->rport->lock, flags);
+	}
+
+	put_device(ctrl->dev);
+	nvme_fc_rport_put(ctrl->rport);
+
+	kfree(ctrl->queues);
+	ida_simple_remove(&nvme_fc_ctrl_cnt, ctrl->cnum);
+	nvmf_free_options(ctrl->ctrl.opts);
+	kfree(ctrl);
+}
+
+static void
+nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl)
+{
+	kref_put(&ctrl->ref, nvme_fc_ctrl_free);
+}
+
+static int
+nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl)
+{
+	return kref_get_unless_zero(&ctrl->ref);
+}
+
+/*
+ * All accesses from nvme core layer done - can now free the
+ * controller. Called after last nvme_put_ctrl() call
+ */
+static void
+nvme_fc_free_nvme_ctrl(struct nvme_ctrl *nctrl)
+{
+	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
+
+	WARN_ON(nctrl != &ctrl->ctrl);
+
+	/*
+	 * Tear down the association, which will generate link
+	 * traffic to terminate connections
+	 */
+
+	if (ctrl->state != FCCTRL_INIT) {
+		/* send a Disconnect(association) LS to fc-nvme target */
+		nvme_fc_xmt_disconnect_assoc(ctrl);
+
+		if (ctrl->ctrl.tagset) {
+			blk_cleanup_queue(ctrl->ctrl.connect_q);
+			blk_mq_free_tag_set(&ctrl->tag_set);
+			nvme_fc_delete_hw_io_queues(ctrl);
+			nvme_fc_free_io_queues(ctrl);
+		}
+
+		nvme_fc_exit_aen_ops(ctrl);
+
+		nvme_fc_destroy_admin_queue(ctrl);
+	}
+
+	nvme_fc_ctrl_put(ctrl);
+}
+
+
+static int
+__nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
+{
+	int state;
+
+	state = atomic_xchg(&op->state, FCPOP_STATE_ABORTED);
+	if (state != FCPOP_STATE_ACTIVE) {
+		atomic_set(&op->state, state);
+		return -ECANCELED; /* fail */
+	}
+
+	ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
+					&ctrl->rport->remoteport,
+					op->queue->lldd_handle,
+					&op->fcp_req);
+
+	return 0;
+}
+
+enum blk_eh_timer_return
+nvme_fc_timeout(struct request *rq, bool reserved)
+{
+	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
+	struct nvme_fc_ctrl *ctrl = op->ctrl;
+	int ret;
+
+	if (reserved)
+		return BLK_EH_RESET_TIMER;
+
+	ret = __nvme_fc_abort_op(ctrl, op);
+	if (ret)
+		/* io wasn't active to abort consider it done */
+		return BLK_EH_HANDLED;
+
+	/*
+	 * TODO: force a controller reset
+	 *   when that happens, queues will be torn down and outstanding
+	 *   ios will be terminated, and the above abort, on a single io
+	 *   will no longer be needed.
+	 */
+
+	return BLK_EH_HANDLED;
+}
+
+static int
+nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
+		struct nvme_fc_fcp_op *op)
+{
+	struct nvmefc_fcp_req *freq = &op->fcp_req;
+	u32 map_len = nvme_map_len(rq);
+	enum dma_data_direction dir;
+	int ret;
+
+	freq->sg_cnt = 0;
+
+	if (!map_len)
+		return 0;
+
+	freq->sg_table.sgl = freq->first_sgl;
+	ret = sg_alloc_table_chained(&freq->sg_table, rq->nr_phys_segments,
+			freq->sg_table.sgl);
+	if (ret)
+		return -ENOMEM;
+
+	op->nents = blk_rq_map_sg(rq->q, rq, freq->sg_table.sgl);
+	WARN_ON(op->nents > rq->nr_phys_segments);
+	dir = (rq_data_dir(rq) == WRITE) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	freq->sg_cnt = fc_dma_map_sg(ctrl->lport->dev, freq->sg_table.sgl,
+				op->nents, dir);
+	if (unlikely(freq->sg_cnt <= 0)) {
+		sg_free_table_chained(&freq->sg_table, true);
+		freq->sg_cnt = 0;
+		return -EFAULT;
+	}
+
+	/*
+	 * TODO: blk_integrity_rq(rq)  for DIF
+	 */
+	return 0;
+}
+
+static void
+nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
+		struct nvme_fc_fcp_op *op)
+{
+	struct nvmefc_fcp_req *freq = &op->fcp_req;
+
+	if (!freq->sg_cnt)
+		return;
+
+	fc_dma_unmap_sg(ctrl->lport->dev, freq->sg_table.sgl, op->nents,
+				((rq_data_dir(rq) == WRITE) ?
+					DMA_TO_DEVICE : DMA_FROM_DEVICE));
+
+	nvme_cleanup_cmd(rq);
+
+	sg_free_table_chained(&freq->sg_table, true);
+
+	freq->sg_cnt = 0;
+}
+
+/*
+ * In FC, the queue is a logical thing. At transport connect, the target
+ * creates its "queue" and returns a handle that is to be given to the
+ * target whenever it posts something to the corresponding SQ.  When an
+ * SQE is sent on a SQ, FC effectively considers the SQE, or rather the
+ * command contained within the SQE, an io, and assigns a FC exchange
+ * to it. The SQE and the associated SQ handle are sent in the initial
+ * CMD IU sents on the exchange. All transfers relative to the io occur
+ * as part of the exchange.  The CQE is the last thing for the io,
+ * which is transferred (explicitly or implicitly) with the RSP IU
+ * sent on the exchange. After the CQE is received, the FC exchange is
+ * terminaed and the Exchange may be used on a different io.
+ *
+ * The transport to LLDD api has the transport making a request for a
+ * new fcp io request to the LLDD. The LLDD then allocates a FC exchange
+ * resource and transfers the command. The LLDD will then process all
+ * steps to complete the io. Upon completion, the transport done routine
+ * is called.
+ *
+ * So - while the operation is outstanding to the LLDD, there is a link
+ * level FC exchange resource that is also outstanding. This must be
+ * considered in all cleanup operations.
+ */
+static int
+nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
+	struct nvme_fc_fcp_op *op, u32 data_len,
+	enum nvmefc_fcp_datadir	io_dir)
+{
+	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
+	struct nvme_command *sqe = &cmdiu->sqe;
+	u32 csn;
+	int ret;
+
+	if (!nvme_fc_ctrl_get(ctrl))
+		return BLK_MQ_RQ_QUEUE_ERROR;
+
+	/* format the FC-NVME CMD IU and fcp_req */
+	cmdiu->connection_id = cpu_to_be64(queue->connection_id);
+	csn = atomic_inc_return(&queue->csn);
+	cmdiu->csn = cpu_to_be32(csn);
+	cmdiu->data_len = cpu_to_be32(data_len);
+	switch (io_dir) {
+	case NVMEFC_FCP_WRITE:
+		cmdiu->flags = FCNVME_CMD_FLAGS_WRITE;
+		break;
+	case NVMEFC_FCP_READ:
+		cmdiu->flags = FCNVME_CMD_FLAGS_READ;
+		break;
+	case NVMEFC_FCP_NODATA:
+		cmdiu->flags = 0;
+		break;
+	}
+	op->fcp_req.payload_length = data_len;
+	op->fcp_req.io_dir = io_dir;
+	op->fcp_req.transferred_length = 0;
+	op->fcp_req.rcv_rsplen = 0;
+	op->fcp_req.status = 0;
+	op->fcp_req.sqid = cpu_to_le16(queue->qnum);
+
+	/*
+	 * validate per fabric rules, set fields mandated by fabric spec
+	 * as well as those by FC-NVME spec.
+	 */
+	WARN_ON_ONCE(sqe->common.metadata);
+	WARN_ON_ONCE(sqe->common.dptr.prp1);
+	WARN_ON_ONCE(sqe->common.dptr.prp2);
+	sqe->common.flags |= NVME_CMD_SGL_METABUF;
+
+	/*
+	 * format SQE DPTR field per FC-NVME rules
+	 *    type=data block descr; subtype=offset;
+	 *    offset is currently 0.
+	 */
+	sqe->rw.dptr.sgl.type = NVME_SGL_FMT_OFFSET;
+	sqe->rw.dptr.sgl.length = cpu_to_le32(data_len);
+	sqe->rw.dptr.sgl.addr = 0;
+
+	/* odd that we set the command_id - should come from nvme-fabrics */
+	WARN_ON_ONCE(sqe->common.command_id != cpu_to_le16(op->rqno));
+
+	if (op->rq) {				/* skipped on aens */
+		ret = nvme_fc_map_data(ctrl, op->rq, op);
+		if (ret < 0) {
+			dev_err(queue->ctrl->ctrl.device,
+			     "Failed to map data (%d)\n", ret);
+			nvme_cleanup_cmd(op->rq);
+			nvme_fc_ctrl_put(ctrl);
+			return (ret == -ENOMEM || ret == -EAGAIN) ?
+				BLK_MQ_RQ_QUEUE_BUSY : BLK_MQ_RQ_QUEUE_ERROR;
+		}
+	}
+
+	fc_dma_sync_single_for_device(ctrl->lport->dev, op->fcp_req.cmddma,
+				  sizeof(op->cmd_iu), DMA_TO_DEVICE);
+
+	atomic_set(&op->state, FCPOP_STATE_ACTIVE);
+
+	if (op->rq)
+		blk_mq_start_request(op->rq);
+
+	ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport,
+					&ctrl->rport->remoteport,
+					queue->lldd_handle, &op->fcp_req);
+
+	if (ret) {
+		dev_err(ctrl->dev,
+			"Send nvme command failed - lldd returned %d.\n", ret);
+
+		if (op->rq) {			/* normal request */
+			nvme_fc_unmap_data(ctrl, op->rq, op);
+			nvme_cleanup_cmd(op->rq);
+		}
+		/* else - aen. no cleanup needed */
+
+		nvme_fc_ctrl_put(ctrl);
+
+		if (ret != -EBUSY)
+			return BLK_MQ_RQ_QUEUE_ERROR;
+
+		if (op->rq) {
+			blk_mq_stop_hw_queues(op->rq->q);
+			blk_mq_delay_queue(queue->hctx, NVMEFC_QUEUE_DELAY);
+		}
+		return BLK_MQ_RQ_QUEUE_BUSY;
+	}
+
+	return BLK_MQ_RQ_QUEUE_OK;
+}
+
+static int
+nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx,
+			const struct blk_mq_queue_data *bd)
+{
+	struct nvme_ns *ns = hctx->queue->queuedata;
+	struct nvme_fc_queue *queue = hctx->driver_data;
+	struct nvme_fc_ctrl *ctrl = queue->ctrl;
+	struct request *rq = bd->rq;
+	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
+	struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
+	struct nvme_command *sqe = &cmdiu->sqe;
+	enum nvmefc_fcp_datadir	io_dir;
+	u32 data_len;
+	int ret;
+
+	ret = nvme_setup_cmd(ns, rq, sqe);
+	if (ret)
+		return ret;
+
+	data_len = nvme_map_len(rq);
+	if (data_len)
+		io_dir = ((rq_data_dir(rq) == WRITE) ?
+					NVMEFC_FCP_WRITE : NVMEFC_FCP_READ);
+	else
+		io_dir = NVMEFC_FCP_NODATA;
+
+	return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir);
+}
+
+static struct blk_mq_tags *
+nvme_fc_tagset(struct nvme_fc_queue *queue)
+{
+	if (queue->qnum == 0)
+		return queue->ctrl->admin_tag_set.tags[queue->qnum];
+
+	return queue->ctrl->tag_set.tags[queue->qnum - 1];
+}
+
+static int
+nvme_fc_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
+
+{
+	struct nvme_fc_queue *queue = hctx->driver_data;
+	struct nvme_fc_ctrl *ctrl = queue->ctrl;
+	struct request *req;
+	struct nvme_fc_fcp_op *op;
+
+	req = blk_mq_tag_to_rq(nvme_fc_tagset(queue), tag);
+	if (!req) {
+		dev_err(queue->ctrl->ctrl.device,
+			 "tag 0x%x on QNum %#x not found\n",
+			tag, queue->qnum);
+		return 0;
+	}
+
+	op = blk_mq_rq_to_pdu(req);
+
+	if ((atomic_read(&op->state) == FCPOP_STATE_ACTIVE) &&
+		 (ctrl->lport->ops->poll_queue))
+		ctrl->lport->ops->poll_queue(&ctrl->lport->localport,
+						 queue->lldd_handle);
+
+	return ((atomic_read(&op->state) != FCPOP_STATE_ACTIVE));
+}
+
+static void
+nvme_fc_submit_async_event(struct nvme_ctrl *arg, int aer_idx)
+{
+	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(arg);
+	struct nvme_fc_fcp_op *aen_op;
+	int ret;
+
+	if (aer_idx > NVME_FC_NR_AEN_COMMANDS)
+		return;
+
+	aen_op = &ctrl->aen_ops[aer_idx];
+
+	ret = nvme_fc_start_fcp_op(ctrl, aen_op->queue, aen_op, 0,
+					NVMEFC_FCP_NODATA);
+	if (ret)
+		dev_err(ctrl->ctrl.device,
+			"failed async event work [%d]\n", aer_idx);
+}
+
+static void
+nvme_fc_complete_rq(struct request *rq)
+{
+	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
+	struct nvme_fc_ctrl *ctrl = op->ctrl;
+	int error = 0, state;
+
+	state = atomic_xchg(&op->state, FCPOP_STATE_IDLE);
+
+	nvme_cleanup_cmd(rq);
+
+	nvme_fc_unmap_data(ctrl, rq, op);
+
+	if (unlikely(rq->errors)) {
+		if (nvme_req_needs_retry(rq, rq->errors)) {
+			nvme_requeue_req(rq);
+			return;
+		}
+
+		if (rq->cmd_type == REQ_TYPE_DRV_PRIV)
+			error = rq->errors;
+		else
+			error = nvme_error_status(rq->errors);
+	}
+
+	nvme_fc_ctrl_put(ctrl);
+
+	blk_mq_end_request(rq, error);
+}
+
+static struct blk_mq_ops nvme_fc_mq_ops = {
+	.queue_rq	= nvme_fc_queue_rq,
+	.complete	= nvme_fc_complete_rq,
+	.init_request	= nvme_fc_init_request,
+	.exit_request	= nvme_fc_exit_request,
+	.reinit_request	= nvme_fc_reinit_request,
+	.init_hctx	= nvme_fc_init_hctx,
+	.poll		= nvme_fc_poll,
+	.timeout	= nvme_fc_timeout,
+};
+
+static struct blk_mq_ops nvme_fc_admin_mq_ops = {
+	.queue_rq	= nvme_fc_queue_rq,
+	.complete	= nvme_fc_complete_rq,
+	.init_request	= nvme_fc_init_admin_request,
+	.exit_request	= nvme_fc_exit_request,
+	.reinit_request	= nvme_fc_reinit_request,
+	.init_hctx	= nvme_fc_init_admin_hctx,
+	.timeout	= nvme_fc_timeout,
+};
+
+static int
+nvme_fc_configure_admin_queue(struct nvme_fc_ctrl *ctrl)
+{
+	u32 segs;
+	int error;
+
+	nvme_fc_init_queue(ctrl, 0, NVME_FC_AQ_BLKMQ_DEPTH);
+
+	error = nvme_fc_connect_admin_queue(ctrl, &ctrl->queues[0],
+				NVME_FC_AQ_BLKMQ_DEPTH,
+				(NVME_FC_AQ_BLKMQ_DEPTH / 4));
+	if (error)
+		return error;
+
+	memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
+	ctrl->admin_tag_set.ops = &nvme_fc_admin_mq_ops;
+	ctrl->admin_tag_set.queue_depth = NVME_FC_AQ_BLKMQ_DEPTH;
+	ctrl->admin_tag_set.reserved_tags = 2; /* fabric connect + Keep-Alive */
+	ctrl->admin_tag_set.numa_node = NUMA_NO_NODE;
+	ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_fc_fcp_op) +
+					(SG_CHUNK_SIZE *
+						sizeof(struct scatterlist)) +
+					ctrl->lport->ops->fcprqst_priv_sz;
+	ctrl->admin_tag_set.driver_data = ctrl;
+	ctrl->admin_tag_set.nr_hw_queues = 1;
+	ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT;
+
+	error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
+	if (error)
+		goto out_free_queue;
+
+	ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
+	if (IS_ERR(ctrl->ctrl.admin_q)) {
+		error = PTR_ERR(ctrl->ctrl.admin_q);
+		goto out_free_tagset;
+	}
+
+	error = __nvme_fc_create_hw_queue(ctrl, &ctrl->queues[0], 0,
+				NVME_FC_AQ_BLKMQ_DEPTH);
+	if (error)
+		goto out_cleanup_queue;
+
+	error = nvmf_connect_admin_queue(&ctrl->ctrl);
+	if (error)
+		goto out_delete_hw_queue;
+
+	error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->cap);
+	if (error) {
+		dev_err(ctrl->ctrl.device,
+			"prop_get NVME_REG_CAP failed\n");
+		goto out_delete_hw_queue;
+	}
+
+	ctrl->ctrl.sqsize =
+		min_t(int, NVME_CAP_MQES(ctrl->cap) + 1, ctrl->ctrl.sqsize);
+
+	error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->cap);
+	if (error)
+		goto out_delete_hw_queue;
+
+	segs = min_t(u32, NVME_FC_MAX_SEGMENTS,
+			ctrl->lport->ops->max_sgl_segments);
+	ctrl->ctrl.max_hw_sectors = (segs - 1) << (PAGE_SHIFT - 9);
+
+	error = nvme_init_identify(&ctrl->ctrl);
+	if (error)
+		goto out_delete_hw_queue;
+
+	nvme_start_keep_alive(&ctrl->ctrl);
+
+	return 0;
+
+out_delete_hw_queue:
+	__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
+out_cleanup_queue:
+	blk_cleanup_queue(ctrl->ctrl.admin_q);
+out_free_tagset:
+	blk_mq_free_tag_set(&ctrl->admin_tag_set);
+out_free_queue:
+	nvme_fc_free_queue(&ctrl->queues[0]);
+	return error;
+}
+
+/*
+ * This routine is used by the transport when it needs to find active
+ * io on a queue that is to be terminated. The transport uses
+ * blk_mq_tagset_busy_itr() to find the busy requests, which then invoke
+ * this routine to kill them on a 1 by 1 basis.
+ *
+ * As FC allocates FC exchange for each io, the transport must contact
+ * the LLDD to terminate the exchange, thus releasing the FC exchange.
+ * After terminating the exchange the LLDD will call the transport's
+ * normal io done path for the request, but it will have an aborted
+ * status. The done path will return the io request back to the block
+ * layer with an error status.
+ */
+static void
+nvme_fc_terminate_exchange(struct request *req, void *data, bool reserved)
+{
+	struct nvme_ctrl *nctrl = data;
+	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
+	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req);
+int status;
+
+	if (!blk_mq_request_started(req))
+		return;
+
+	/* this performs an ABTS-LS on the FC exchange for the io */
+	status = __nvme_fc_abort_op(ctrl, op);
+	/*
+	 * if __nvme_fc_abort_op failed: io wasn't active to abort
+	 * consider it done. Assume completion path already completing
+	 * in parallel
+	 */
+	if (status)
+		/* io wasn't active to abort consider it done */
+		/* assume completion path already completing in parallel */
+		return;
+}
+
+
+/*
+ * This routine stops operation of the controller. Admin and IO queues
+ * are stopped, outstanding ios on them terminated, and the nvme ctrl
+ * is shutdown.
+ */
+static void
+nvme_fc_shutdown_ctrl(struct nvme_fc_ctrl *ctrl)
+{
+	/*
+	 * If io queues are present, stop them and terminate all outstanding
+	 * ios on them. As FC allocates FC exchange for each io, the
+	 * transport must contact the LLDD to terminate the exchange,
+	 * thus releasing the FC exchange. We use blk_mq_tagset_busy_itr()
+	 * to tell us what io's are busy and invoke a transport routine
+	 * to kill them with the LLDD.  After terminating the exchange
+	 * the LLDD will call the transport's normal io done path, but it
+	 * will have an aborted status. The done path will return the
+	 * io requests back to the block layer as part of normal completions
+	 * (but with error status).
+	 */
+	if (ctrl->queue_count > 1) {
+		nvme_stop_queues(&ctrl->ctrl);
+		blk_mq_tagset_busy_iter(&ctrl->tag_set,
+				nvme_fc_terminate_exchange, &ctrl->ctrl);
+	}
+
+	if (ctrl->ctrl.state == NVME_CTRL_LIVE)
+		nvme_shutdown_ctrl(&ctrl->ctrl);
+
+	/*
+	 * now clean up the admin queue. Same thing as above.
+	 * use blk_mq_tagset_busy_itr() and the transport routine to
+	 * terminate the exchanges.
+	 */
+	blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);
+	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
+				nvme_fc_terminate_exchange, &ctrl->ctrl);
+}
+
+/*
+ * Called to teardown an association.
+ * May be called with association fully in place or partially in place.
+ */
+static void
+__nvme_fc_remove_ctrl(struct nvme_fc_ctrl *ctrl)
+{
+	nvme_stop_keep_alive(&ctrl->ctrl);
+
+	/* stop and terminate ios on admin and io queues */
+	nvme_fc_shutdown_ctrl(ctrl);
+
+	/*
+	 * tear down the controller
+	 * This will result in the last reference on the nvme ctrl to
+	 * expire, calling the transport nvme_fc_free_nvme_ctrl() callback.
+	 * From there, the transport will tear down it's logical queues and
+	 * association.
+	 */
+	nvme_uninit_ctrl(&ctrl->ctrl);
+
+	nvme_put_ctrl(&ctrl->ctrl);
+}
+
+static void
+nvme_fc_del_ctrl_work(struct work_struct *work)
+{
+	struct nvme_fc_ctrl *ctrl =
+			container_of(work, struct nvme_fc_ctrl, delete_work);
+
+	__nvme_fc_remove_ctrl(ctrl);
+}
+
+static int
+__nvme_fc_del_ctrl(struct nvme_fc_ctrl *ctrl)
+{
+	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING))
+		return -EBUSY;
+
+	if (!queue_work(nvme_fc_wq, &ctrl->delete_work))
+		return -EBUSY;
+
+	return 0;
+}
+
+/*
+ * Request from nvme core layer to delete the controller
+ */
+static int
+nvme_fc_del_nvme_ctrl(struct nvme_ctrl *nctrl)
+{
+	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
+	struct nvme_fc_rport *rport = ctrl->rport;
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&rport->lock, flags);
+	ret = __nvme_fc_del_ctrl(ctrl);
+	spin_unlock_irqrestore(&rport->lock, flags);
+	if (ret)
+		return ret;
+
+	flush_work(&ctrl->delete_work);
+
+	return 0;
+}
+
+static int
+nvme_fc_reset_nvme_ctrl(struct nvme_ctrl *nctrl)
+{
+	return -EIO;
+}
+
+static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
+	.name			= "fc",
+	.module			= THIS_MODULE,
+	.is_fabrics		= true,
+	.reg_read32		= nvmf_reg_read32,
+	.reg_read64		= nvmf_reg_read64,
+	.reg_write32		= nvmf_reg_write32,
+	.reset_ctrl		= nvme_fc_reset_nvme_ctrl,
+	.free_ctrl		= nvme_fc_free_nvme_ctrl,
+	.submit_async_event	= nvme_fc_submit_async_event,
+	.delete_ctrl		= nvme_fc_del_nvme_ctrl,
+	.get_subsysnqn		= nvmf_get_subsysnqn,
+	.get_address		= nvmf_get_address,
+};
+
+static int
+nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
+{
+	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
+	int ret;
+
+	ret = nvme_set_queue_count(&ctrl->ctrl, &opts->nr_io_queues);
+	if (ret) {
+		dev_info(ctrl->ctrl.device,
+			"set_queue_count failed: %d\n", ret);
+		return ret;
+	}
+
+	ctrl->queue_count = opts->nr_io_queues + 1;
+	if (!opts->nr_io_queues)
+		return 0;
+
+	dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n",
+			opts->nr_io_queues);
+
+	nvme_fc_init_io_queues(ctrl);
+
+	memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
+	ctrl->tag_set.ops = &nvme_fc_mq_ops;
+	ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
+	ctrl->tag_set.reserved_tags = 1; /* fabric connect */
+	ctrl->tag_set.numa_node = NUMA_NO_NODE;
+	ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
+	ctrl->tag_set.cmd_size = sizeof(struct nvme_fc_fcp_op) +
+					(SG_CHUNK_SIZE *
+						sizeof(struct scatterlist)) +
+					ctrl->lport->ops->fcprqst_priv_sz;
+	ctrl->tag_set.driver_data = ctrl;
+	ctrl->tag_set.nr_hw_queues = ctrl->queue_count - 1;
+	ctrl->tag_set.timeout = NVME_IO_TIMEOUT;
+
+	ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
+	if (ret)
+		return ret;
+
+	ctrl->ctrl.tagset = &ctrl->tag_set;
+
+	ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);
+	if (IS_ERR(ctrl->ctrl.connect_q)) {
+		ret = PTR_ERR(ctrl->ctrl.connect_q);
+		goto out_free_tag_set;
+	}
+
+	ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.opts->queue_size);
+	if (ret)
+		goto out_cleanup_blk_queue;
+
+	ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.opts->queue_size);
+	if (ret)
+		goto out_delete_hw_queues;
+
+	return 0;
+
+out_delete_hw_queues:
+	nvme_fc_delete_hw_io_queues(ctrl);
+out_cleanup_blk_queue:
+	nvme_stop_keep_alive(&ctrl->ctrl);
+	blk_cleanup_queue(ctrl->ctrl.connect_q);
+out_free_tag_set:
+	blk_mq_free_tag_set(&ctrl->tag_set);
+	nvme_fc_free_io_queues(ctrl);
+
+	/* force put free routine to ignore io queues */
+	ctrl->ctrl.tagset = NULL;
+
+	return ret;
+}
+
+
+static struct nvme_ctrl *
+__nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
+	struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
+{
+	struct nvme_fc_ctrl *ctrl;
+	unsigned long flags;
+	int ret, idx;
+	bool changed;
+
+	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
+	if (!ctrl) {
+		ret = -ENOMEM;
+		goto out_fail;
+	}
+
+	idx = ida_simple_get(&nvme_fc_ctrl_cnt, 0, 0, GFP_KERNEL);
+	if (idx < 0) {
+		ret = -ENOSPC;
+		goto out_free_ctrl;
+	}
+
+	ctrl->ctrl.opts = opts;
+	INIT_LIST_HEAD(&ctrl->ctrl_list);
+	INIT_LIST_HEAD(&ctrl->ls_req_list);
+	ctrl->lport = lport;
+	ctrl->rport = rport;
+	ctrl->dev = lport->dev;
+	ctrl->state = FCCTRL_INIT;
+	ctrl->cnum = idx;
+
+	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0);
+	if (ret)
+		goto out_free_ida;
+
+	get_device(ctrl->dev);
+	kref_init(&ctrl->ref);
+
+	INIT_WORK(&ctrl->delete_work, nvme_fc_del_ctrl_work);
+	spin_lock_init(&ctrl->lock);
+
+	/* io queue count */
+	ctrl->queue_count = min_t(unsigned int,
+				opts->nr_io_queues,
+				lport->ops->max_hw_queues);
+	opts->nr_io_queues = ctrl->queue_count;	/* so opts has valid value */
+	ctrl->queue_count++;	/* +1 for admin queue */
+
+	ctrl->ctrl.sqsize = opts->queue_size - 1;
+	ctrl->ctrl.kato = opts->kato;
+
+	ret = -ENOMEM;
+	ctrl->queues = kcalloc(ctrl->queue_count, sizeof(struct nvme_fc_queue),
+				GFP_KERNEL);
+	if (!ctrl->queues)
+		goto out_uninit_ctrl;
+
+	ret = nvme_fc_configure_admin_queue(ctrl);
+	if (ret)
+		goto out_uninit_ctrl;
+
+	/* sanity checks */
+
+	/* FC-NVME supports 64-byte SQE only */
+	if (ctrl->ctrl.ioccsz != 4) {
+		dev_err(ctrl->ctrl.device, "ioccsz %d is not supported!\n",
+				ctrl->ctrl.ioccsz);
+		goto out_remove_admin_queue;
+	}
+	/* FC-NVME supports 16-byte CQE only */
+	if (ctrl->ctrl.iorcsz != 1) {
+		dev_err(ctrl->ctrl.device, "iorcsz %d is not supported!\n",
+				ctrl->ctrl.iorcsz);
+		goto out_remove_admin_queue;
+	}
+	/* FC-NVME does not have other data in the capsule */
+	if (ctrl->ctrl.icdoff) {
+		dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n",
+				ctrl->ctrl.icdoff);
+		goto out_remove_admin_queue;
+	}
+
+	/* FC-NVME supports normal SGL Data Block Descriptors */
+
+	if (opts->queue_size > ctrl->ctrl.maxcmd) {
+		/* warn if maxcmd is lower than queue_size */
+		dev_warn(ctrl->ctrl.device,
+			"queue_size %zu > ctrl maxcmd %u, reducing "
+			"to queue_size\n",
+			opts->queue_size, ctrl->ctrl.maxcmd);
+		opts->queue_size = ctrl->ctrl.maxcmd;
+	}
+
+	ret = nvme_fc_init_aen_ops(ctrl);
+	if (ret)
+		goto out_exit_aen_ops;
+
+	if (ctrl->queue_count > 1) {
+		ret = nvme_fc_create_io_queues(ctrl);
+		if (ret)
+			goto out_exit_aen_ops;
+	}
+
+	spin_lock_irqsave(&ctrl->lock, flags);
+	ctrl->state = FCCTRL_ACTIVE;
+	spin_unlock_irqrestore(&ctrl->lock, flags);
+
+	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
+	WARN_ON_ONCE(!changed);
+
+	dev_info(ctrl->ctrl.device,
+		"NVME-FC{%d}: new ctrl: NQN \"%s\" (%p)\n",
+		ctrl->cnum, ctrl->ctrl.opts->subsysnqn, &ctrl);
+
+	kref_get(&ctrl->ctrl.kref);
+
+	spin_lock_irqsave(&rport->lock, flags);
+	list_add_tail(&ctrl->ctrl_list, &rport->ctrl_list);
+	spin_unlock_irqrestore(&rport->lock, flags);
+
+	if (opts->nr_io_queues) {
+		nvme_queue_scan(&ctrl->ctrl);
+		nvme_queue_async_events(&ctrl->ctrl);
+	}
+
+	return &ctrl->ctrl;
+
+out_exit_aen_ops:
+	nvme_fc_exit_aen_ops(ctrl);
+out_remove_admin_queue:
+	/* send a Disconnect(association) LS to fc-nvme target */
+	nvme_fc_xmt_disconnect_assoc(ctrl);
+	nvme_stop_keep_alive(&ctrl->ctrl);
+	nvme_fc_destroy_admin_queue(ctrl);
+out_uninit_ctrl:
+	nvme_uninit_ctrl(&ctrl->ctrl);
+	nvme_put_ctrl(&ctrl->ctrl);
+	if (ret > 0)
+		ret = -EIO;
+	/* exit via here will follow ctlr ref point callbacks to free */
+	return ERR_PTR(ret);
+
+out_free_ida:
+	ida_simple_remove(&nvme_fc_ctrl_cnt, ctrl->cnum);
+out_free_ctrl:
+	kfree(ctrl);
+out_fail:
+	nvme_fc_rport_put(rport);
+	/* exit via here doesn't follow ctlr ref points */
+	return ERR_PTR(ret);
+}
+
+enum {
+	FCT_TRADDR_ERR		= 0,
+	FCT_TRADDR_WWNN		= 1 << 0,
+	FCT_TRADDR_WWPN		= 1 << 1,
+};
+
+struct nvmet_fc_traddr {
+	u64	nn;
+	u64	pn;
+};
+
+static const match_table_t traddr_opt_tokens = {
+	{ FCT_TRADDR_WWNN,	"nn-%s"		},
+	{ FCT_TRADDR_WWPN,	"pn-%s"		},
+	{ FCT_TRADDR_ERR,	NULL		}
+};
+
+static int
+nvme_fc_parse_address(struct nvmet_fc_traddr *traddr, char *buf)
+{
+	substring_t args[MAX_OPT_ARGS];
+	char *options, *o, *p;
+	int token, ret = 0;
+	u64 token64;
+
+	options = o = kstrdup(buf, GFP_KERNEL);
+	if (!options)
+		return -ENOMEM;
+
+	while ((p = strsep(&o, ":\n")) != NULL) {
+		if (!*p)
+			continue;
+
+		token = match_token(p, traddr_opt_tokens, args);
+		switch (token) {
+		case FCT_TRADDR_WWNN:
+			if (match_u64(args, &token64)) {
+				ret = -EINVAL;
+				goto out;
+			}
+			traddr->nn = token64;
+			break;
+		case FCT_TRADDR_WWPN:
+			if (match_u64(args, &token64)) {
+				ret = -EINVAL;
+				goto out;
+			}
+			traddr->pn = token64;
+			break;
+		default:
+			pr_warn("unknown traddr token or missing value '%s'\n",
+					p);
+			ret = -EINVAL;
+			goto out;
+		}
+	}
+
+out:
+	kfree(options);
+	return ret;
+}
+
+static struct nvme_ctrl *
+nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
+{
+	struct nvme_fc_lport *lport;
+	struct nvme_fc_rport *rport;
+	struct nvmet_fc_traddr laddr = { 0L, 0L };
+	struct nvmet_fc_traddr raddr = { 0L, 0L };
+	unsigned long flags;
+	int ret;
+
+	ret = nvme_fc_parse_address(&raddr, opts->traddr);
+	if (ret || !raddr.nn || !raddr.pn)
+		return ERR_PTR(-EINVAL);
+
+	ret = nvme_fc_parse_address(&laddr, opts->host_traddr);
+	if (ret || !laddr.nn || !laddr.pn)
+		return ERR_PTR(-EINVAL);
+
+	/* find the host and remote ports to connect together */
+	spin_lock_irqsave(&nvme_fc_lock, flags);
+	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
+		if (lport->localport.node_name != laddr.nn ||
+		    lport->localport.port_name != laddr.pn)
+			continue;
+
+		list_for_each_entry(rport, &lport->endp_list, endp_list) {
+			if (rport->remoteport.node_name != raddr.nn ||
+			    rport->remoteport.port_name != raddr.pn)
+				continue;
+
+			/* if fail to get reference fall through. Will error */
+			if (!nvme_fc_rport_get(rport))
+				break;
+
+			spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+			return __nvme_fc_create_ctrl(dev, opts, lport, rport);
+		}
+	}
+	spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+	return ERR_PTR(-ENOENT);
+}
+
+
+static struct nvmf_transport_ops nvme_fc_transport = {
+	.name		= "fc",
+	.required_opts	= NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR,
+	.allowed_opts	= NVMF_OPT_RECONNECT_DELAY,
+	.create_ctrl	= nvme_fc_create_ctrl,
+};
+
+static int __init nvme_fc_init_module(void)
+{
+	nvme_fc_wq = create_workqueue("nvme_fc_wq");
+	if (!nvme_fc_wq)
+		return -ENOMEM;
+
+	nvmf_register_transport(&nvme_fc_transport);
+	return 0;
+}
+
+static void __exit nvme_fc_exit_module(void)
+{
+	/* sanity check - all lports should be removed */
+	if (!list_empty(&nvme_fc_lport_list))
+		pr_warn("%s: localport list not empty\n", __func__);
+
+	nvmf_unregister_transport(&nvme_fc_transport);
+
+	destroy_workqueue(nvme_fc_wq);
+
+	ida_destroy(&nvme_fc_local_port_cnt);
+	ida_destroy(&nvme_fc_ctrl_cnt);
+}
+
+module_init(nvme_fc_init_module);
+module_exit(nvme_fc_exit_module);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c
index 5daf2f4be0cd..588d4a34c083 100644
--- a/drivers/nvme/host/lightnvm.c
+++ b/drivers/nvme/host/lightnvm.c
@@ -146,14 +146,6 @@ struct nvme_nvm_command {
 	};
 };
 
-struct nvme_nvm_completion {
-	__le64	result;		/* Used by LightNVM to return ppa completions */
-	__le16	sq_head;	/* how much of this queue may be reclaimed */
-	__le16	sq_id;		/* submission queue that generated this entry */
-	__u16	command_id;	/* of the command which completed */
-	__le16	status;		/* did the command fail, and if so, why? */
-};
-
 #define NVME_NVM_LP_MLC_PAIRS 886
 struct nvme_nvm_lp_mlc {
 	__le16			num_pairs;
@@ -360,6 +352,7 @@ static int nvme_nvm_get_l2p_tbl(struct nvm_dev *nvmdev, u64 slba, u32 nlb,
 
 	while (nlb) {
 		u32 cmd_nlb = min(nlb_pr_rq, nlb);
+		u64 elba = slba + cmd_nlb;
 
 		c.l2p.slba = cpu_to_le64(cmd_slba);
 		c.l2p.nlb = cpu_to_le32(cmd_nlb);
@@ -373,6 +366,14 @@ static int nvme_nvm_get_l2p_tbl(struct nvm_dev *nvmdev, u64 slba, u32 nlb,
 			goto out;
 		}
 
+		if (unlikely(elba > nvmdev->total_secs)) {
+			pr_err("nvm: L2P data from device is out of bounds!\n");
+			return -EINVAL;
+		}
+
+		/* Transform physical address to target address space */
+		nvmdev->mt->part_to_tgt(nvmdev, entries, cmd_nlb);
+
 		if (update_l2p(cmd_slba, cmd_nlb, entries, priv)) {
 			ret = -EINTR;
 			goto out;
@@ -391,11 +392,12 @@ static int nvme_nvm_get_bb_tbl(struct nvm_dev *nvmdev, struct ppa_addr ppa,
 								u8 *blks)
 {
 	struct request_queue *q = nvmdev->q;
+	struct nvm_geo *geo = &nvmdev->geo;
 	struct nvme_ns *ns = q->queuedata;
 	struct nvme_ctrl *ctrl = ns->ctrl;
 	struct nvme_nvm_command c = {};
 	struct nvme_nvm_bb_tbl *bb_tbl;
-	int nr_blks = nvmdev->blks_per_lun * nvmdev->plane_mode;
+	int nr_blks = geo->blks_per_lun * geo->plane_mode;
 	int tblsz = sizeof(struct nvme_nvm_bb_tbl) + nr_blks;
 	int ret = 0;
 
@@ -436,7 +438,7 @@ static int nvme_nvm_get_bb_tbl(struct nvm_dev *nvmdev, struct ppa_addr ppa,
 		goto out;
 	}
 
-	memcpy(blks, bb_tbl->blk, nvmdev->blks_per_lun * nvmdev->plane_mode);
+	memcpy(blks, bb_tbl->blk, geo->blks_per_lun * geo->plane_mode);
 out:
 	kfree(bb_tbl);
 	return ret;
@@ -481,14 +483,11 @@ static inline void nvme_nvm_rqtocmd(struct request *rq, struct nvm_rq *rqd,
 static void nvme_nvm_end_io(struct request *rq, int error)
 {
 	struct nvm_rq *rqd = rq->end_io_data;
-	struct nvme_nvm_completion *cqe = rq->special;
-
-	if (cqe)
-		rqd->ppa_status = le64_to_cpu(cqe->result);
 
+	rqd->ppa_status = nvme_req(rq)->result.u64;
 	nvm_end_io(rqd, error);
 
-	kfree(rq->cmd);
+	kfree(nvme_req(rq)->cmd);
 	blk_mq_free_request(rq);
 }
 
@@ -500,20 +499,18 @@ static int nvme_nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
 	struct bio *bio = rqd->bio;
 	struct nvme_nvm_command *cmd;
 
-	rq = blk_mq_alloc_request(q, bio_data_dir(bio), 0);
-	if (IS_ERR(rq))
+	cmd = kzalloc(sizeof(struct nvme_nvm_command), GFP_KERNEL);
+	if (!cmd)
 		return -ENOMEM;
 
-	cmd = kzalloc(sizeof(struct nvme_nvm_command) +
-				sizeof(struct nvme_nvm_completion), GFP_KERNEL);
-	if (!cmd) {
-		blk_mq_free_request(rq);
+	rq = nvme_alloc_request(q, (struct nvme_command *)cmd, 0, NVME_QID_ANY);
+	if (IS_ERR(rq)) {
+		kfree(cmd);
 		return -ENOMEM;
 	}
+	rq->cmd_flags &= ~REQ_FAILFAST_DRIVER;
 
-	rq->cmd_type = REQ_TYPE_DRV_PRIV;
 	rq->ioprio = bio_prio(bio);
-
 	if (bio_has_data(bio))
 		rq->nr_phys_segments = bio_phys_segments(q, bio);
 
@@ -522,10 +519,6 @@ static int nvme_nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
 
 	nvme_nvm_rqtocmd(rq, rqd, ns, cmd);
 
-	rq->cmd = (unsigned char *)cmd;
-	rq->cmd_len = sizeof(struct nvme_nvm_command);
-	rq->special = cmd + 1;
-
 	rq->end_io_data = rqd;
 
 	blk_execute_rq_nowait(q, NULL, rq, 0, nvme_nvm_end_io);
@@ -543,6 +536,7 @@ static int nvme_nvm_erase_block(struct nvm_dev *dev, struct nvm_rq *rqd)
 	c.erase.nsid = cpu_to_le32(ns->ns_id);
 	c.erase.spba = cpu_to_le64(rqd->ppa_addr.ppa);
 	c.erase.length = cpu_to_le16(rqd->nr_ppas - 1);
+	c.erase.control = cpu_to_le16(rqd->flags);
 
 	return nvme_submit_sync_cmd(q, (struct nvme_command *)&c, NULL, 0);
 }
@@ -592,12 +586,10 @@ static struct nvm_dev_ops nvme_nvm_dev_ops = {
 	.max_phys_sect		= 64,
 };
 
-int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node,
-		      const struct attribute_group *attrs)
+int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node)
 {
 	struct request_queue *q = ns->queue;
 	struct nvm_dev *dev;
-	int ret;
 
 	dev = nvm_alloc_dev(node);
 	if (!dev)
@@ -606,18 +598,10 @@ int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node,
 	dev->q = q;
 	memcpy(dev->name, disk_name, DISK_NAME_LEN);
 	dev->ops = &nvme_nvm_dev_ops;
-	dev->parent_dev = ns->ctrl->device;
 	dev->private_data = ns;
 	ns->ndev = dev;
 
-	ret = nvm_register(dev);
-
-	ns->lba_shift = ilog2(dev->sec_size);
-
-	if (sysfs_create_group(&dev->dev.kobj, attrs))
-		pr_warn("%s: failed to create sysfs group for identification\n",
-			disk_name);
-	return ret;
+	return nvm_register(dev);
 }
 
 void nvme_nvm_unregister(struct nvme_ns *ns)
@@ -625,6 +609,167 @@ void nvme_nvm_unregister(struct nvme_ns *ns)
 	nvm_unregister(ns->ndev);
 }
 
+static ssize_t nvm_dev_attr_show(struct device *dev,
+				 struct device_attribute *dattr, char *page)
+{
+	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
+	struct nvm_dev *ndev = ns->ndev;
+	struct nvm_id *id;
+	struct nvm_id_group *grp;
+	struct attribute *attr;
+
+	if (!ndev)
+		return 0;
+
+	id = &ndev->identity;
+	grp = &id->groups[0];
+	attr = &dattr->attr;
+
+	if (strcmp(attr->name, "version") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", id->ver_id);
+	} else if (strcmp(attr->name, "vendor_opcode") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", id->vmnt);
+	} else if (strcmp(attr->name, "capabilities") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", id->cap);
+	} else if (strcmp(attr->name, "device_mode") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", id->dom);
+	} else if (strcmp(attr->name, "media_manager") == 0) {
+		if (!ndev->mt)
+			return scnprintf(page, PAGE_SIZE, "%s\n", "none");
+		return scnprintf(page, PAGE_SIZE, "%s\n", ndev->mt->name);
+	} else if (strcmp(attr->name, "ppa_format") == 0) {
+		return scnprintf(page, PAGE_SIZE,
+			"0x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n",
+			id->ppaf.ch_offset, id->ppaf.ch_len,
+			id->ppaf.lun_offset, id->ppaf.lun_len,
+			id->ppaf.pln_offset, id->ppaf.pln_len,
+			id->ppaf.blk_offset, id->ppaf.blk_len,
+			id->ppaf.pg_offset, id->ppaf.pg_len,
+			id->ppaf.sect_offset, id->ppaf.sect_len);
+	} else if (strcmp(attr->name, "media_type") == 0) {	/* u8 */
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->mtype);
+	} else if (strcmp(attr->name, "flash_media_type") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->fmtype);
+	} else if (strcmp(attr->name, "num_channels") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_ch);
+	} else if (strcmp(attr->name, "num_luns") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_lun);
+	} else if (strcmp(attr->name, "num_planes") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_pln);
+	} else if (strcmp(attr->name, "num_blocks") == 0) {	/* u16 */
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_blk);
+	} else if (strcmp(attr->name, "num_pages") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_pg);
+	} else if (strcmp(attr->name, "page_size") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->fpg_sz);
+	} else if (strcmp(attr->name, "hw_sector_size") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->csecs);
+	} else if (strcmp(attr->name, "oob_sector_size") == 0) {/* u32 */
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->sos);
+	} else if (strcmp(attr->name, "read_typ") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->trdt);
+	} else if (strcmp(attr->name, "read_max") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->trdm);
+	} else if (strcmp(attr->name, "prog_typ") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->tprt);
+	} else if (strcmp(attr->name, "prog_max") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->tprm);
+	} else if (strcmp(attr->name, "erase_typ") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->tbet);
+	} else if (strcmp(attr->name, "erase_max") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->tbem);
+	} else if (strcmp(attr->name, "multiplane_modes") == 0) {
+		return scnprintf(page, PAGE_SIZE, "0x%08x\n", grp->mpos);
+	} else if (strcmp(attr->name, "media_capabilities") == 0) {
+		return scnprintf(page, PAGE_SIZE, "0x%08x\n", grp->mccap);
+	} else if (strcmp(attr->name, "max_phys_secs") == 0) {
+		return scnprintf(page, PAGE_SIZE, "%u\n",
+				ndev->ops->max_phys_sect);
+	} else {
+		return scnprintf(page,
+				 PAGE_SIZE,
+				 "Unhandled attr(%s) in `nvm_dev_attr_show`\n",
+				 attr->name);
+	}
+}
+
+#define NVM_DEV_ATTR_RO(_name)						\
+	DEVICE_ATTR(_name, S_IRUGO, nvm_dev_attr_show, NULL)
+
+static NVM_DEV_ATTR_RO(version);
+static NVM_DEV_ATTR_RO(vendor_opcode);
+static NVM_DEV_ATTR_RO(capabilities);
+static NVM_DEV_ATTR_RO(device_mode);
+static NVM_DEV_ATTR_RO(ppa_format);
+static NVM_DEV_ATTR_RO(media_manager);
+
+static NVM_DEV_ATTR_RO(media_type);
+static NVM_DEV_ATTR_RO(flash_media_type);
+static NVM_DEV_ATTR_RO(num_channels);
+static NVM_DEV_ATTR_RO(num_luns);
+static NVM_DEV_ATTR_RO(num_planes);
+static NVM_DEV_ATTR_RO(num_blocks);
+static NVM_DEV_ATTR_RO(num_pages);
+static NVM_DEV_ATTR_RO(page_size);
+static NVM_DEV_ATTR_RO(hw_sector_size);
+static NVM_DEV_ATTR_RO(oob_sector_size);
+static NVM_DEV_ATTR_RO(read_typ);
+static NVM_DEV_ATTR_RO(read_max);
+static NVM_DEV_ATTR_RO(prog_typ);
+static NVM_DEV_ATTR_RO(prog_max);
+static NVM_DEV_ATTR_RO(erase_typ);
+static NVM_DEV_ATTR_RO(erase_max);
+static NVM_DEV_ATTR_RO(multiplane_modes);
+static NVM_DEV_ATTR_RO(media_capabilities);
+static NVM_DEV_ATTR_RO(max_phys_secs);
+
+static struct attribute *nvm_dev_attrs[] = {
+	&dev_attr_version.attr,
+	&dev_attr_vendor_opcode.attr,
+	&dev_attr_capabilities.attr,
+	&dev_attr_device_mode.attr,
+	&dev_attr_media_manager.attr,
+
+	&dev_attr_ppa_format.attr,
+	&dev_attr_media_type.attr,
+	&dev_attr_flash_media_type.attr,
+	&dev_attr_num_channels.attr,
+	&dev_attr_num_luns.attr,
+	&dev_attr_num_planes.attr,
+	&dev_attr_num_blocks.attr,
+	&dev_attr_num_pages.attr,
+	&dev_attr_page_size.attr,
+	&dev_attr_hw_sector_size.attr,
+	&dev_attr_oob_sector_size.attr,
+	&dev_attr_read_typ.attr,
+	&dev_attr_read_max.attr,
+	&dev_attr_prog_typ.attr,
+	&dev_attr_prog_max.attr,
+	&dev_attr_erase_typ.attr,
+	&dev_attr_erase_max.attr,
+	&dev_attr_multiplane_modes.attr,
+	&dev_attr_media_capabilities.attr,
+	&dev_attr_max_phys_secs.attr,
+	NULL,
+};
+
+static const struct attribute_group nvm_dev_attr_group = {
+	.name		= "lightnvm",
+	.attrs		= nvm_dev_attrs,
+};
+
+int nvme_nvm_register_sysfs(struct nvme_ns *ns)
+{
+	return sysfs_create_group(&disk_to_dev(ns->disk)->kobj,
+					&nvm_dev_attr_group);
+}
+
+void nvme_nvm_unregister_sysfs(struct nvme_ns *ns)
+{
+	sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
+					&nvm_dev_attr_group);
+}
+
 /* move to shared place when used in multiple places. */
 #define PCI_VENDOR_ID_CNEX 0x1d1d
 #define PCI_DEVICE_ID_CNEX_WL 0x2807
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index d47f5a5d18c7..bd5321441d12 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -79,6 +79,20 @@ enum nvme_quirks {
 	NVME_QUIRK_DELAY_BEFORE_CHK_RDY		= (1 << 3),
 };
 
+/*
+ * Common request structure for NVMe passthrough.  All drivers must have
+ * this structure as the first member of their request-private data.
+ */
+struct nvme_request {
+	struct nvme_command	*cmd;
+	union nvme_result	result;
+};
+
+static inline struct nvme_request *nvme_req(struct request *req)
+{
+	return blk_mq_rq_to_pdu(req);
+}
+
 /* The below value is the specific amount of delay needed before checking
  * readiness in case of the PCI_DEVICE(0x1c58, 0x0003), which needs the
  * NVME_QUIRK_DELAY_BEFORE_CHK_RDY quirk enabled. The value (in ms) was
@@ -222,8 +236,10 @@ static inline unsigned nvme_map_len(struct request *rq)
 
 static inline void nvme_cleanup_cmd(struct request *req)
 {
-	if (req_op(req) == REQ_OP_DISCARD)
-		kfree(req->completion_data);
+	if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
+		kfree(page_address(req->special_vec.bv_page) +
+		      req->special_vec.bv_offset);
+	}
 }
 
 static inline int nvme_error_status(u16 status)
@@ -261,8 +277,8 @@ void nvme_queue_scan(struct nvme_ctrl *ctrl);
 void nvme_remove_namespaces(struct nvme_ctrl *ctrl);
 
 #define NVME_NR_AERS	1
-void nvme_complete_async_event(struct nvme_ctrl *ctrl,
-		struct nvme_completion *cqe);
+void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
+		union nvme_result *res);
 void nvme_queue_async_events(struct nvme_ctrl *ctrl);
 
 void nvme_stop_queues(struct nvme_ctrl *ctrl);
@@ -278,7 +294,7 @@ int nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 		void *buf, unsigned bufflen);
 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
-		struct nvme_completion *cqe, void *buffer, unsigned bufflen,
+		union nvme_result *result, void *buffer, unsigned bufflen,
 		unsigned timeout, int qid, int at_head, int flags);
 int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
 		void __user *ubuffer, unsigned bufflen, u32 *result,
@@ -307,36 +323,33 @@ int nvme_sg_get_version_num(int __user *ip);
 
 #ifdef CONFIG_NVM
 int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id);
-int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node,
-		      const struct attribute_group *attrs);
+int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node);
 void nvme_nvm_unregister(struct nvme_ns *ns);
-
-static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev)
-{
-	if (dev->type->devnode)
-		return dev_to_disk(dev)->private_data;
-
-	return (container_of(dev, struct nvm_dev, dev))->private_data;
-}
+int nvme_nvm_register_sysfs(struct nvme_ns *ns);
+void nvme_nvm_unregister_sysfs(struct nvme_ns *ns);
 #else
 static inline int nvme_nvm_register(struct nvme_ns *ns, char *disk_name,
-				    int node,
-				    const struct attribute_group *attrs)
+				    int node)
 {
 	return 0;
 }
 
 static inline void nvme_nvm_unregister(struct nvme_ns *ns) {};
-
+static inline int nvme_nvm_register_sysfs(struct nvme_ns *ns)
+{
+	return 0;
+}
+static inline void nvme_nvm_unregister_sysfs(struct nvme_ns *ns) {};
 static inline int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id)
 {
 	return 0;
 }
+#endif /* CONFIG_NVM */
+
 static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev)
 {
 	return dev_to_disk(dev)->private_data;
 }
-#endif /* CONFIG_NVM */
 
 int __init nvme_core_init(void);
 void nvme_core_exit(void);
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 5e52034ab010..2fd7dc2e8fc4 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -141,6 +141,7 @@ struct nvme_queue {
  * allocated to store the PRP list.
  */
 struct nvme_iod {
+	struct nvme_request req;
 	struct nvme_queue *nvmeq;
 	int aborted;
 	int npages;		/* In the PRP list. 0 means small pool in use */
@@ -302,14 +303,14 @@ static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
 static __le64 **iod_list(struct request *req)
 {
 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
-	return (__le64 **)(iod->sg + req->nr_phys_segments);
+	return (__le64 **)(iod->sg + blk_rq_nr_phys_segments(req));
 }
 
 static int nvme_init_iod(struct request *rq, unsigned size,
 		struct nvme_dev *dev)
 {
 	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq);
-	int nseg = rq->nr_phys_segments;
+	int nseg = blk_rq_nr_phys_segments(rq);
 
 	if (nseg > NVME_INT_PAGES || size > NVME_INT_BYTES(dev)) {
 		iod->sg = kmalloc(nvme_iod_alloc_size(dev, size, nseg), GFP_ATOMIC);
@@ -324,11 +325,11 @@ static int nvme_init_iod(struct request *rq, unsigned size,
 	iod->nents = 0;
 	iod->length = size;
 
-	if (!(rq->cmd_flags & REQ_DONTPREP)) {
+	if (!(rq->rq_flags & RQF_DONTPREP)) {
 		rq->retries = 0;
-		rq->cmd_flags |= REQ_DONTPREP;
+		rq->rq_flags |= RQF_DONTPREP;
 	}
-	return 0;
+	return BLK_MQ_RQ_QUEUE_OK;
 }
 
 static void nvme_free_iod(struct nvme_dev *dev, struct request *req)
@@ -339,8 +340,6 @@ static void nvme_free_iod(struct nvme_dev *dev, struct request *req)
 	__le64 **list = iod_list(req);
 	dma_addr_t prp_dma = iod->first_dma;
 
-	nvme_cleanup_cmd(req);
-
 	if (iod->npages == 0)
 		dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
 	for (i = 0; i < iod->npages; i++) {
@@ -510,7 +509,7 @@ static int nvme_map_data(struct nvme_dev *dev, struct request *req,
 			DMA_TO_DEVICE : DMA_FROM_DEVICE;
 	int ret = BLK_MQ_RQ_QUEUE_ERROR;
 
-	sg_init_table(iod->sg, req->nr_phys_segments);
+	sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));
 	iod->nents = blk_rq_map_sg(q, req, iod->sg);
 	if (!iod->nents)
 		goto out;
@@ -566,6 +565,7 @@ static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
 		}
 	}
 
+	nvme_cleanup_cmd(req);
 	nvme_free_iod(dev, req);
 }
 
@@ -596,22 +596,21 @@ static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 		}
 	}
 
-	map_len = nvme_map_len(req);
-	ret = nvme_init_iod(req, map_len, dev);
-	if (ret)
+	ret = nvme_setup_cmd(ns, req, &cmnd);
+	if (ret != BLK_MQ_RQ_QUEUE_OK)
 		return ret;
 
-	ret = nvme_setup_cmd(ns, req, &cmnd);
-	if (ret)
-		goto out;
+	map_len = nvme_map_len(req);
+	ret = nvme_init_iod(req, map_len, dev);
+	if (ret != BLK_MQ_RQ_QUEUE_OK)
+		goto out_free_cmd;
 
-	if (req->nr_phys_segments)
+	if (blk_rq_nr_phys_segments(req))
 		ret = nvme_map_data(dev, req, map_len, &cmnd);
 
-	if (ret)
-		goto out;
+	if (ret != BLK_MQ_RQ_QUEUE_OK)
+		goto out_cleanup_iod;
 
-	cmnd.common.command_id = req->tag;
 	blk_mq_start_request(req);
 
 	spin_lock_irq(&nvmeq->q_lock);
@@ -621,14 +620,16 @@ static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 		else
 			ret = BLK_MQ_RQ_QUEUE_ERROR;
 		spin_unlock_irq(&nvmeq->q_lock);
-		goto out;
+		goto out_cleanup_iod;
 	}
 	__nvme_submit_cmd(nvmeq, &cmnd);
 	nvme_process_cq(nvmeq);
 	spin_unlock_irq(&nvmeq->q_lock);
 	return BLK_MQ_RQ_QUEUE_OK;
-out:
+out_cleanup_iod:
 	nvme_free_iod(dev, req);
+out_free_cmd:
+	nvme_cleanup_cmd(req);
 	return ret;
 }
 
@@ -703,13 +704,13 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
 		 */
 		if (unlikely(nvmeq->qid == 0 &&
 				cqe.command_id >= NVME_AQ_BLKMQ_DEPTH)) {
-			nvme_complete_async_event(&nvmeq->dev->ctrl, &cqe);
+			nvme_complete_async_event(&nvmeq->dev->ctrl,
+					cqe.status, &cqe.result);
 			continue;
 		}
 
 		req = blk_mq_tag_to_rq(*nvmeq->tags, cqe.command_id);
-		if (req->cmd_type == REQ_TYPE_DRV_PRIV && req->special)
-			memcpy(req->special, &cqe, sizeof(cqe));
+		nvme_req(req)->result = cqe.result;
 		blk_mq_complete_request(req, le16_to_cpu(cqe.status) >> 1);
 
 	}
@@ -1281,6 +1282,24 @@ static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
 	return true;
 }
 
+static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
+{
+	/* Read a config register to help see what died. */
+	u16 pci_status;
+	int result;
+
+	result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS,
+				      &pci_status);
+	if (result == PCIBIOS_SUCCESSFUL)
+		dev_warn(dev->dev,
+			 "controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n",
+			 csts, pci_status);
+	else
+		dev_warn(dev->dev,
+			 "controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n",
+			 csts, result);
+}
+
 static void nvme_watchdog_timer(unsigned long data)
 {
 	struct nvme_dev *dev = (struct nvme_dev *)data;
@@ -1289,9 +1308,7 @@ static void nvme_watchdog_timer(unsigned long data)
 	/* Skip controllers under certain specific conditions. */
 	if (nvme_should_reset(dev, csts)) {
 		if (!nvme_reset(dev))
-			dev_warn(dev->dev,
-				"Failed status: 0x%x, reset controller.\n",
-				csts);
+			nvme_warn_reset(dev, csts);
 		return;
 	}
 
@@ -2085,9 +2102,6 @@ static const struct pci_error_handlers nvme_err_handler = {
 	.reset_notify	= nvme_reset_notify,
 };
 
-/* Move to pci_ids.h later */
-#define PCI_CLASS_STORAGE_EXPRESS	0x010802
-
 static const struct pci_device_id nvme_id_table[] = {
 	{ PCI_VDEVICE(INTEL, 0x0953),
 		.driver_data = NVME_QUIRK_STRIPE_SIZE |
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 3d25add36d91..f587af345889 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -28,7 +28,6 @@
 
 #include <rdma/ib_verbs.h>
 #include <rdma/rdma_cm.h>
-#include <rdma/ib_cm.h>
 #include <linux/nvme-rdma.h>
 
 #include "nvme.h"
@@ -43,6 +42,28 @@
 
 #define NVME_RDMA_MAX_INLINE_SEGMENTS	1
 
+static const char *const nvme_rdma_cm_status_strs[] = {
+	[NVME_RDMA_CM_INVALID_LEN]	= "invalid length",
+	[NVME_RDMA_CM_INVALID_RECFMT]	= "invalid record format",
+	[NVME_RDMA_CM_INVALID_QID]	= "invalid queue ID",
+	[NVME_RDMA_CM_INVALID_HSQSIZE]	= "invalid host SQ size",
+	[NVME_RDMA_CM_INVALID_HRQSIZE]	= "invalid host RQ size",
+	[NVME_RDMA_CM_NO_RSC]		= "resource not found",
+	[NVME_RDMA_CM_INVALID_IRD]	= "invalid IRD",
+	[NVME_RDMA_CM_INVALID_ORD]	= "Invalid ORD",
+};
+
+static const char *nvme_rdma_cm_msg(enum nvme_rdma_cm_status status)
+{
+	size_t index = status;
+
+	if (index < ARRAY_SIZE(nvme_rdma_cm_status_strs) &&
+	    nvme_rdma_cm_status_strs[index])
+		return nvme_rdma_cm_status_strs[index];
+	else
+		return "unrecognized reason";
+};
+
 /*
  * We handle AEN commands ourselves and don't even let the
  * block layer know about them.
@@ -66,6 +87,7 @@ struct nvme_rdma_qe {
 
 struct nvme_rdma_queue;
 struct nvme_rdma_request {
+	struct nvme_request	req;
 	struct ib_mr		*mr;
 	struct nvme_rdma_qe	sqe;
 	struct ib_sge		sge[1 + NVME_RDMA_MAX_INLINE_SEGMENTS];
@@ -241,7 +263,9 @@ out_free_ring:
 
 static void nvme_rdma_qp_event(struct ib_event *event, void *context)
 {
-	pr_debug("QP event %d\n", event->event);
+	pr_debug("QP event %s (%d)\n",
+		 ib_event_msg(event->event), event->event);
+
 }
 
 static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue *queue)
@@ -963,8 +987,7 @@ static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
 	struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
 	struct nvme_rdma_device *dev = queue->device;
 	struct ib_device *ibdev = dev->dev;
-	int nents, count;
-	int ret;
+	int count, ret;
 
 	req->num_sge = 1;
 	req->inline_data = false;
@@ -976,16 +999,14 @@ static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
 		return nvme_rdma_set_sg_null(c);
 
 	req->sg_table.sgl = req->first_sgl;
-	ret = sg_alloc_table_chained(&req->sg_table, rq->nr_phys_segments,
-				req->sg_table.sgl);
+	ret = sg_alloc_table_chained(&req->sg_table,
+			blk_rq_nr_phys_segments(rq), req->sg_table.sgl);
 	if (ret)
 		return -ENOMEM;
 
-	nents = blk_rq_map_sg(rq->q, rq, req->sg_table.sgl);
-	BUG_ON(nents > rq->nr_phys_segments);
-	req->nents = nents;
+	req->nents = blk_rq_map_sg(rq->q, rq, req->sg_table.sgl);
 
-	count = ib_dma_map_sg(ibdev, req->sg_table.sgl, nents,
+	count = ib_dma_map_sg(ibdev, req->sg_table.sgl, req->nents,
 		    rq_data_dir(rq) == WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
 	if (unlikely(count <= 0)) {
 		sg_free_table_chained(&req->sg_table, true);
@@ -1130,13 +1151,10 @@ static void nvme_rdma_submit_async_event(struct nvme_ctrl *arg, int aer_idx)
 static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
 		struct nvme_completion *cqe, struct ib_wc *wc, int tag)
 {
-	u16 status = le16_to_cpu(cqe->status);
 	struct request *rq;
 	struct nvme_rdma_request *req;
 	int ret = 0;
 
-	status >>= 1;
-
 	rq = blk_mq_tag_to_rq(nvme_rdma_tagset(queue), cqe->command_id);
 	if (!rq) {
 		dev_err(queue->ctrl->ctrl.device,
@@ -1147,9 +1165,6 @@ static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
 	}
 	req = blk_mq_rq_to_pdu(rq);
 
-	if (rq->cmd_type == REQ_TYPE_DRV_PRIV && rq->special)
-		memcpy(rq->special, cqe, sizeof(*cqe));
-
 	if (rq->tag == tag)
 		ret = 1;
 
@@ -1157,8 +1172,8 @@ static int nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
 	    wc->ex.invalidate_rkey == req->mr->rkey)
 		req->mr->need_inval = false;
 
-	blk_mq_complete_request(rq, status);
-
+	req->req.result = cqe->result;
+	blk_mq_complete_request(rq, le16_to_cpu(cqe->status) >> 1);
 	return ret;
 }
 
@@ -1186,7 +1201,8 @@ static int __nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc, int tag)
 	 */
 	if (unlikely(nvme_rdma_queue_idx(queue) == 0 &&
 			cqe->command_id >= NVME_RDMA_AQ_BLKMQ_DEPTH))
-		nvme_complete_async_event(&queue->ctrl->ctrl, cqe);
+		nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
+				&cqe->result);
 	else
 		ret = nvme_rdma_process_nvme_rsp(queue, cqe, wc, tag);
 	ib_dma_sync_single_for_device(ibdev, qe->dma, len, DMA_FROM_DEVICE);
@@ -1220,16 +1236,24 @@ out_destroy_queue_ib:
 static int nvme_rdma_conn_rejected(struct nvme_rdma_queue *queue,
 		struct rdma_cm_event *ev)
 {
-	if (ev->param.conn.private_data_len) {
-		struct nvme_rdma_cm_rej *rej =
-			(struct nvme_rdma_cm_rej *)ev->param.conn.private_data;
+	struct rdma_cm_id *cm_id = queue->cm_id;
+	int status = ev->status;
+	const char *rej_msg;
+	const struct nvme_rdma_cm_rej *rej_data;
+	u8 rej_data_len;
+
+	rej_msg = rdma_reject_msg(cm_id, status);
+	rej_data = rdma_consumer_reject_data(cm_id, ev, &rej_data_len);
+
+	if (rej_data && rej_data_len >= sizeof(u16)) {
+		u16 sts = le16_to_cpu(rej_data->sts);
 
 		dev_err(queue->ctrl->ctrl.device,
-			"Connect rejected, status %d.", le16_to_cpu(rej->sts));
-		/* XXX: Think of something clever to do here... */
+		      "Connect rejected: status %d (%s) nvme status %d (%s).\n",
+		      status, rej_msg, sts, nvme_rdma_cm_msg(sts));
 	} else {
 		dev_err(queue->ctrl->ctrl.device,
-			"Connect rejected, no private data.\n");
+			"Connect rejected: status %d (%s).\n", status, rej_msg);
 	}
 
 	return -ECONNRESET;
@@ -1433,10 +1457,9 @@ static int nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
 			sizeof(struct nvme_command), DMA_TO_DEVICE);
 
 	ret = nvme_setup_cmd(ns, rq, c);
-	if (ret)
+	if (ret != BLK_MQ_RQ_QUEUE_OK)
 		return ret;
 
-	c->common.command_id = rq->tag;
 	blk_mq_start_request(rq);
 
 	map_len = nvme_map_len(rq);
@@ -1944,6 +1967,14 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
 		opts->queue_size = ctrl->ctrl.maxcmd;
 	}
 
+	if (opts->queue_size > ctrl->ctrl.sqsize + 1) {
+		/* warn if sqsize is lower than queue_size */
+		dev_warn(ctrl->ctrl.device,
+			"queue_size %zu > ctrl sqsize %u, clamping down\n",
+			opts->queue_size, ctrl->ctrl.sqsize + 1);
+		opts->queue_size = ctrl->ctrl.sqsize + 1;
+	}
+
 	if (opts->nr_io_queues) {
 		ret = nvme_rdma_create_io_queues(ctrl);
 		if (ret)
diff --git a/drivers/nvme/host/scsi.c b/drivers/nvme/host/scsi.c
index 3eaa4d27801e..b71e95044b43 100644
--- a/drivers/nvme/host/scsi.c
+++ b/drivers/nvme/host/scsi.c
@@ -1280,10 +1280,6 @@ static inline void nvme_trans_modesel_get_bd_len(u8 *parm_list, u8 cdb10,
 static void nvme_trans_modesel_save_bd(struct nvme_ns *ns, u8 *parm_list,
 					u16 idx, u16 bd_len, u8 llbaa)
 {
-	u16 bd_num;
-
-	bd_num = bd_len / ((llbaa == 0) ?
-			SHORT_DESC_BLOCK : LONG_DESC_BLOCK);
 	/* Store block descriptor info if a FORMAT UNIT comes later */
 	/* TODO Saving 1st BD info; what to do if multiple BD received? */
 	if (llbaa == 0) {
@@ -1528,7 +1524,7 @@ static int nvme_trans_fmt_send_cmd(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	int nvme_sc;
 	struct nvme_id_ns *id_ns;
 	u8 i;
-	u8 flbas, nlbaf;
+	u8 nlbaf;
 	u8 selected_lbaf = 0xFF;
 	u32 cdw10 = 0;
 	struct nvme_command c;
@@ -1539,7 +1535,6 @@ static int nvme_trans_fmt_send_cmd(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 	if (res)
 		return res;
 
-	flbas = (id_ns->flbas) & 0x0F;
 	nlbaf = id_ns->nlbaf;
 
 	for (i = 0; i < nlbaf; i++) {
@@ -2168,12 +2163,10 @@ static int nvme_trans_synchronize_cache(struct nvme_ns *ns,
 static int nvme_trans_start_stop(struct nvme_ns *ns, struct sg_io_hdr *hdr,
 							u8 *cmd)
 {
-	u8 immed, pcmod, no_flush, start;
+	u8 immed, no_flush;
 
 	immed = cmd[1] & 0x01;
-	pcmod = cmd[3] & 0x0f;
 	no_flush = cmd[4] & 0x04;
-	start = cmd[4] & 0x01;
 
 	if (immed != 0) {
 		return nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION,
diff --git a/drivers/nvme/target/Kconfig b/drivers/nvme/target/Kconfig
index 3a5b9d0576cb..03e4ab65fe77 100644
--- a/drivers/nvme/target/Kconfig
+++ b/drivers/nvme/target/Kconfig
@@ -34,3 +34,27 @@ config NVME_TARGET_RDMA
 	  devices over RDMA.
 
 	  If unsure, say N.
+
+config NVME_TARGET_FC
+	tristate "NVMe over Fabrics FC target driver"
+	depends on NVME_TARGET
+	depends on HAS_DMA
+	help
+	  This enables the NVMe FC target support, which allows exporting NVMe
+	  devices over FC.
+
+	  If unsure, say N.
+
+config NVME_TARGET_FCLOOP
+	tristate "NVMe over Fabrics FC Transport Loopback Test driver"
+	depends on NVME_TARGET
+	select NVME_CORE
+	select NVME_FABRICS
+	select SG_POOL
+	depends on NVME_FC
+	depends on NVME_TARGET_FC
+	help
+	  This enables the NVMe FC loopback test support, which can be useful
+	  to test NVMe-FC transport interfaces.
+
+	  If unsure, say N.
diff --git a/drivers/nvme/target/Makefile b/drivers/nvme/target/Makefile
index b7a06232c9da..fecc14f535b2 100644
--- a/drivers/nvme/target/Makefile
+++ b/drivers/nvme/target/Makefile
@@ -2,8 +2,12 @@
 obj-$(CONFIG_NVME_TARGET)		+= nvmet.o
 obj-$(CONFIG_NVME_TARGET_LOOP)		+= nvme-loop.o
 obj-$(CONFIG_NVME_TARGET_RDMA)		+= nvmet-rdma.o
+obj-$(CONFIG_NVME_TARGET_FC)		+= nvmet-fc.o
+obj-$(CONFIG_NVME_TARGET_FCLOOP)	+= nvme-fcloop.o
 
 nvmet-y		+= core.o configfs.o admin-cmd.o io-cmd.o fabrics-cmd.o \
 			discovery.o
 nvme-loop-y	+= loop.o
 nvmet-rdma-y	+= rdma.o
+nvmet-fc-y	+= fc.o
+nvme-fcloop-y	+= fcloop.o
diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
index 6fe4c48a21e4..ec1ad2aa0a4c 100644
--- a/drivers/nvme/target/admin-cmd.c
+++ b/drivers/nvme/target/admin-cmd.c
@@ -237,7 +237,8 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 	id->maxcmd = cpu_to_le16(NVMET_MAX_CMD);
 
 	id->nn = cpu_to_le32(ctrl->subsys->max_nsid);
-	id->oncs = cpu_to_le16(NVME_CTRL_ONCS_DSM);
+	id->oncs = cpu_to_le16(NVME_CTRL_ONCS_DSM |
+			NVME_CTRL_ONCS_WRITE_ZEROES);
 
 	/* XXX: don't report vwc if the underlying device is write through */
 	id->vwc = NVME_CTRL_VWC_PRESENT;
diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c
index af5e2dc4a3d5..6f5074153dcd 100644
--- a/drivers/nvme/target/configfs.c
+++ b/drivers/nvme/target/configfs.c
@@ -37,6 +37,8 @@ static ssize_t nvmet_addr_adrfam_show(struct config_item *item,
 		return sprintf(page, "ipv6\n");
 	case NVMF_ADDR_FAMILY_IB:
 		return sprintf(page, "ib\n");
+	case NVMF_ADDR_FAMILY_FC:
+		return sprintf(page, "fc\n");
 	default:
 		return sprintf(page, "\n");
 	}
@@ -59,6 +61,8 @@ static ssize_t nvmet_addr_adrfam_store(struct config_item *item,
 		port->disc_addr.adrfam = NVMF_ADDR_FAMILY_IP6;
 	} else if (sysfs_streq(page, "ib")) {
 		port->disc_addr.adrfam = NVMF_ADDR_FAMILY_IB;
+	} else if (sysfs_streq(page, "fc")) {
+		port->disc_addr.adrfam = NVMF_ADDR_FAMILY_FC;
 	} else {
 		pr_err("Invalid value '%s' for adrfam\n", page);
 		return -EINVAL;
@@ -209,6 +213,8 @@ static ssize_t nvmet_addr_trtype_show(struct config_item *item,
 		return sprintf(page, "rdma\n");
 	case NVMF_TRTYPE_LOOP:
 		return sprintf(page, "loop\n");
+	case NVMF_TRTYPE_FC:
+		return sprintf(page, "fc\n");
 	default:
 		return sprintf(page, "\n");
 	}
@@ -229,6 +235,12 @@ static void nvmet_port_init_tsas_loop(struct nvmet_port *port)
 	memset(&port->disc_addr.tsas, 0, NVMF_TSAS_SIZE);
 }
 
+static void nvmet_port_init_tsas_fc(struct nvmet_port *port)
+{
+	port->disc_addr.trtype = NVMF_TRTYPE_FC;
+	memset(&port->disc_addr.tsas, 0, NVMF_TSAS_SIZE);
+}
+
 static ssize_t nvmet_addr_trtype_store(struct config_item *item,
 		const char *page, size_t count)
 {
@@ -244,6 +256,8 @@ static ssize_t nvmet_addr_trtype_store(struct config_item *item,
 		nvmet_port_init_tsas_rdma(port);
 	} else if (sysfs_streq(page, "loop")) {
 		nvmet_port_init_tsas_loop(port);
+	} else if (sysfs_streq(page, "fc")) {
+		nvmet_port_init_tsas_fc(port);
 	} else {
 		pr_err("Invalid value '%s' for trtype\n", page);
 		return -EINVAL;
@@ -271,7 +285,7 @@ static ssize_t nvmet_ns_device_path_store(struct config_item *item,
 
 	mutex_lock(&subsys->lock);
 	ret = -EBUSY;
-	if (nvmet_ns_enabled(ns))
+	if (ns->enabled)
 		goto out_unlock;
 
 	kfree(ns->device_path);
@@ -307,7 +321,7 @@ static ssize_t nvmet_ns_device_nguid_store(struct config_item *item,
 	int ret = 0;
 
 	mutex_lock(&subsys->lock);
-	if (nvmet_ns_enabled(ns)) {
+	if (ns->enabled) {
 		ret = -EBUSY;
 		goto out_unlock;
 	}
@@ -339,7 +353,7 @@ CONFIGFS_ATTR(nvmet_ns_, device_nguid);
 
 static ssize_t nvmet_ns_enable_show(struct config_item *item, char *page)
 {
-	return sprintf(page, "%d\n", nvmet_ns_enabled(to_nvmet_ns(item)));
+	return sprintf(page, "%d\n", to_nvmet_ns(item)->enabled);
 }
 
 static ssize_t nvmet_ns_enable_store(struct config_item *item,
@@ -466,7 +480,7 @@ out_free_link:
 	return ret;
 }
 
-static int nvmet_port_subsys_drop_link(struct config_item *parent,
+static void nvmet_port_subsys_drop_link(struct config_item *parent,
 		struct config_item *target)
 {
 	struct nvmet_port *port = to_nvmet_port(parent->ci_parent);
@@ -479,7 +493,7 @@ static int nvmet_port_subsys_drop_link(struct config_item *parent,
 			goto found;
 	}
 	up_write(&nvmet_config_sem);
-	return -EINVAL;
+	return;
 
 found:
 	list_del(&p->entry);
@@ -488,7 +502,6 @@ found:
 		nvmet_disable_port(port);
 	up_write(&nvmet_config_sem);
 	kfree(p);
-	return 0;
 }
 
 static struct configfs_item_operations nvmet_port_subsys_item_ops = {
@@ -542,7 +555,7 @@ out_free_link:
 	return ret;
 }
 
-static int nvmet_allowed_hosts_drop_link(struct config_item *parent,
+static void nvmet_allowed_hosts_drop_link(struct config_item *parent,
 		struct config_item *target)
 {
 	struct nvmet_subsys *subsys = to_subsys(parent->ci_parent);
@@ -555,14 +568,13 @@ static int nvmet_allowed_hosts_drop_link(struct config_item *parent,
 			goto found;
 	}
 	up_write(&nvmet_config_sem);
-	return -EINVAL;
+	return;
 
 found:
 	list_del(&p->entry);
 	nvmet_genctr++;
 	up_write(&nvmet_config_sem);
 	kfree(p);
-	return 0;
 }
 
 static struct configfs_item_operations nvmet_allowed_hosts_item_ops = {
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index a21437a33adb..b1d66ed655c9 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -264,7 +264,7 @@ int nvmet_ns_enable(struct nvmet_ns *ns)
 	int ret = 0;
 
 	mutex_lock(&subsys->lock);
-	if (!list_empty(&ns->dev_link))
+	if (ns->enabled)
 		goto out_unlock;
 
 	ns->bdev = blkdev_get_by_path(ns->device_path, FMODE_READ | FMODE_WRITE,
@@ -309,6 +309,7 @@ int nvmet_ns_enable(struct nvmet_ns *ns)
 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
 		nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE, 0, 0);
 
+	ns->enabled = true;
 	ret = 0;
 out_unlock:
 	mutex_unlock(&subsys->lock);
@@ -325,11 +326,11 @@ void nvmet_ns_disable(struct nvmet_ns *ns)
 	struct nvmet_ctrl *ctrl;
 
 	mutex_lock(&subsys->lock);
-	if (list_empty(&ns->dev_link)) {
-		mutex_unlock(&subsys->lock);
-		return;
-	}
-	list_del_init(&ns->dev_link);
+	if (!ns->enabled)
+		goto out_unlock;
+
+	ns->enabled = false;
+	list_del_rcu(&ns->dev_link);
 	mutex_unlock(&subsys->lock);
 
 	/*
@@ -351,6 +352,7 @@ void nvmet_ns_disable(struct nvmet_ns *ns)
 
 	if (ns->bdev)
 		blkdev_put(ns->bdev, FMODE_WRITE|FMODE_READ);
+out_unlock:
 	mutex_unlock(&subsys->lock);
 }
 
@@ -617,7 +619,7 @@ u16 nvmet_ctrl_find_get(const char *subsysnqn, const char *hostnqn, u16 cntlid,
 	if (!subsys) {
 		pr_warn("connect request for invalid subsystem %s!\n",
 			subsysnqn);
-		req->rsp->result = IPO_IATTR_CONNECT_DATA(subsysnqn);
+		req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn);
 		return NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
 	}
 
@@ -638,7 +640,7 @@ u16 nvmet_ctrl_find_get(const char *subsysnqn, const char *hostnqn, u16 cntlid,
 
 	pr_warn("could not find controller %d for subsys %s / host %s\n",
 		cntlid, subsysnqn, hostnqn);
-	req->rsp->result = IPO_IATTR_CONNECT_DATA(cntlid);
+	req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(cntlid);
 	status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
 
 out:
@@ -700,7 +702,7 @@ u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn,
 	if (!subsys) {
 		pr_warn("connect request for invalid subsystem %s!\n",
 			subsysnqn);
-		req->rsp->result = IPO_IATTR_CONNECT_DATA(subsysnqn);
+		req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn);
 		goto out;
 	}
 
@@ -709,7 +711,7 @@ u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn,
 	if (!nvmet_host_allowed(req, subsys, hostnqn)) {
 		pr_info("connect by host %s for subsystem %s not allowed\n",
 			hostnqn, subsysnqn);
-		req->rsp->result = IPO_IATTR_CONNECT_DATA(hostnqn);
+		req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(hostnqn);
 		up_read(&nvmet_config_sem);
 		goto out_put_subsystem;
 	}
diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c
index 9a97ae67e656..f4088198cd0d 100644
--- a/drivers/nvme/target/fabrics-cmd.c
+++ b/drivers/nvme/target/fabrics-cmd.c
@@ -69,7 +69,7 @@ static void nvmet_execute_prop_get(struct nvmet_req *req)
 		}
 	}
 
-	req->rsp->result64 = cpu_to_le64(val);
+	req->rsp->result.u64 = cpu_to_le64(val);
 	nvmet_req_complete(req, status);
 }
 
@@ -125,7 +125,7 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req)
 	d = kmap(sg_page(req->sg)) + req->sg->offset;
 
 	/* zero out initial completion result, assign values as needed */
-	req->rsp->result = 0;
+	req->rsp->result.u32 = 0;
 
 	if (c->recfmt != 0) {
 		pr_warn("invalid connect version (%d).\n",
@@ -138,7 +138,7 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req)
 		pr_warn("connect attempt for invalid controller ID %#x\n",
 			d->cntlid);
 		status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
-		req->rsp->result = IPO_IATTR_CONNECT_DATA(cntlid);
+		req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(cntlid);
 		goto out;
 	}
 
@@ -155,7 +155,7 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req)
 
 	pr_info("creating controller %d for NQN %s.\n",
 			ctrl->cntlid, ctrl->hostnqn);
-	req->rsp->result16 = cpu_to_le16(ctrl->cntlid);
+	req->rsp->result.u16 = cpu_to_le16(ctrl->cntlid);
 
 out:
 	kunmap(sg_page(req->sg));
@@ -173,7 +173,7 @@ static void nvmet_execute_io_connect(struct nvmet_req *req)
 	d = kmap(sg_page(req->sg)) + req->sg->offset;
 
 	/* zero out initial completion result, assign values as needed */
-	req->rsp->result = 0;
+	req->rsp->result.u32 = 0;
 
 	if (c->recfmt != 0) {
 		pr_warn("invalid connect version (%d).\n",
@@ -191,14 +191,14 @@ static void nvmet_execute_io_connect(struct nvmet_req *req)
 	if (unlikely(qid > ctrl->subsys->max_qid)) {
 		pr_warn("invalid queue id (%d)\n", qid);
 		status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR;
-		req->rsp->result = IPO_IATTR_CONNECT_SQE(qid);
+		req->rsp->result.u32 = IPO_IATTR_CONNECT_SQE(qid);
 		goto out_ctrl_put;
 	}
 
 	status = nvmet_install_queue(ctrl, req);
 	if (status) {
 		/* pass back cntlid that had the issue of installing queue */
-		req->rsp->result16 = cpu_to_le16(ctrl->cntlid);
+		req->rsp->result.u16 = cpu_to_le16(ctrl->cntlid);
 		goto out_ctrl_put;
 	}
 
diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c
new file mode 100644
index 000000000000..173e842f19c9
--- /dev/null
+++ b/drivers/nvme/target/fc.c
@@ -0,0 +1,2288 @@
+/*
+ * Copyright (c) 2016 Avago Technologies.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful.
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
+ * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
+ * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO
+ * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
+ * See the GNU General Public License for more details, a copy of which
+ * can be found in the file COPYING included with this package
+ *
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/blk-mq.h>
+#include <linux/parser.h>
+#include <linux/random.h>
+#include <uapi/scsi/fc/fc_fs.h>
+#include <uapi/scsi/fc/fc_els.h>
+
+#include "nvmet.h"
+#include <linux/nvme-fc-driver.h>
+#include <linux/nvme-fc.h>
+
+
+/* *************************** Data Structures/Defines ****************** */
+
+
+#define NVMET_LS_CTX_COUNT		4
+
+/* for this implementation, assume small single frame rqst/rsp */
+#define NVME_FC_MAX_LS_BUFFER_SIZE		2048
+
+struct nvmet_fc_tgtport;
+struct nvmet_fc_tgt_assoc;
+
+struct nvmet_fc_ls_iod {
+	struct nvmefc_tgt_ls_req	*lsreq;
+	struct nvmefc_tgt_fcp_req	*fcpreq;	/* only if RS */
+
+	struct list_head		ls_list;	/* tgtport->ls_list */
+
+	struct nvmet_fc_tgtport		*tgtport;
+	struct nvmet_fc_tgt_assoc	*assoc;
+
+	u8				*rqstbuf;
+	u8				*rspbuf;
+	u16				rqstdatalen;
+	dma_addr_t			rspdma;
+
+	struct scatterlist		sg[2];
+
+	struct work_struct		work;
+} __aligned(sizeof(unsigned long long));
+
+#define NVMET_FC_MAX_KB_PER_XFR		256
+
+enum nvmet_fcp_datadir {
+	NVMET_FCP_NODATA,
+	NVMET_FCP_WRITE,
+	NVMET_FCP_READ,
+	NVMET_FCP_ABORTED,
+};
+
+struct nvmet_fc_fcp_iod {
+	struct nvmefc_tgt_fcp_req	*fcpreq;
+
+	struct nvme_fc_cmd_iu		cmdiubuf;
+	struct nvme_fc_ersp_iu		rspiubuf;
+	dma_addr_t			rspdma;
+	struct scatterlist		*data_sg;
+	struct scatterlist		*next_sg;
+	int				data_sg_cnt;
+	u32				next_sg_offset;
+	u32				total_length;
+	u32				offset;
+	enum nvmet_fcp_datadir		io_dir;
+	bool				active;
+	bool				abort;
+	spinlock_t			flock;
+
+	struct nvmet_req		req;
+	struct work_struct		work;
+
+	struct nvmet_fc_tgtport		*tgtport;
+	struct nvmet_fc_tgt_queue	*queue;
+
+	struct list_head		fcp_list;	/* tgtport->fcp_list */
+};
+
+struct nvmet_fc_tgtport {
+
+	struct nvmet_fc_target_port	fc_target_port;
+
+	struct list_head		tgt_list; /* nvmet_fc_target_list */
+	struct device			*dev;	/* dev for dma mapping */
+	struct nvmet_fc_target_template	*ops;
+
+	struct nvmet_fc_ls_iod		*iod;
+	spinlock_t			lock;
+	struct list_head		ls_list;
+	struct list_head		ls_busylist;
+	struct list_head		assoc_list;
+	struct ida			assoc_cnt;
+	struct nvmet_port		*port;
+	struct kref			ref;
+};
+
+struct nvmet_fc_tgt_queue {
+	bool				ninetypercent;
+	u16				qid;
+	u16				sqsize;
+	u16				ersp_ratio;
+	u16				sqhd;
+	int				cpu;
+	atomic_t			connected;
+	atomic_t			sqtail;
+	atomic_t			zrspcnt;
+	atomic_t			rsn;
+	spinlock_t			qlock;
+	struct nvmet_port		*port;
+	struct nvmet_cq			nvme_cq;
+	struct nvmet_sq			nvme_sq;
+	struct nvmet_fc_tgt_assoc	*assoc;
+	struct nvmet_fc_fcp_iod		*fod;		/* array of fcp_iods */
+	struct list_head		fod_list;
+	struct workqueue_struct		*work_q;
+	struct kref			ref;
+} __aligned(sizeof(unsigned long long));
+
+struct nvmet_fc_tgt_assoc {
+	u64				association_id;
+	u32				a_id;
+	struct nvmet_fc_tgtport		*tgtport;
+	struct list_head		a_list;
+	struct nvmet_fc_tgt_queue	*queues[NVMET_NR_QUEUES];
+	struct kref			ref;
+};
+
+
+static inline int
+nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr)
+{
+	return (iodptr - iodptr->tgtport->iod);
+}
+
+static inline int
+nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr)
+{
+	return (fodptr - fodptr->queue->fod);
+}
+
+
+/*
+ * Association and Connection IDs:
+ *
+ * Association ID will have random number in upper 6 bytes and zero
+ *   in lower 2 bytes
+ *
+ * Connection IDs will be Association ID with QID or'd in lower 2 bytes
+ *
+ * note: Association ID = Connection ID for queue 0
+ */
+#define BYTES_FOR_QID			sizeof(u16)
+#define BYTES_FOR_QID_SHIFT		(BYTES_FOR_QID * 8)
+#define NVMET_FC_QUEUEID_MASK		((u64)((1 << BYTES_FOR_QID_SHIFT) - 1))
+
+static inline u64
+nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid)
+{
+	return (assoc->association_id | qid);
+}
+
+static inline u64
+nvmet_fc_getassociationid(u64 connectionid)
+{
+	return connectionid & ~NVMET_FC_QUEUEID_MASK;
+}
+
+static inline u16
+nvmet_fc_getqueueid(u64 connectionid)
+{
+	return (u16)(connectionid & NVMET_FC_QUEUEID_MASK);
+}
+
+static inline struct nvmet_fc_tgtport *
+targetport_to_tgtport(struct nvmet_fc_target_port *targetport)
+{
+	return container_of(targetport, struct nvmet_fc_tgtport,
+				 fc_target_port);
+}
+
+static inline struct nvmet_fc_fcp_iod *
+nvmet_req_to_fod(struct nvmet_req *nvme_req)
+{
+	return container_of(nvme_req, struct nvmet_fc_fcp_iod, req);
+}
+
+
+/* *************************** Globals **************************** */
+
+
+static DEFINE_SPINLOCK(nvmet_fc_tgtlock);
+
+static LIST_HEAD(nvmet_fc_target_list);
+static DEFINE_IDA(nvmet_fc_tgtport_cnt);
+
+
+static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work);
+static void nvmet_fc_handle_fcp_rqst_work(struct work_struct *work);
+static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc);
+static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc);
+static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue);
+static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue);
+static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport);
+static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport);
+
+
+/* *********************** FC-NVME DMA Handling **************************** */
+
+/*
+ * The fcloop device passes in a NULL device pointer. Real LLD's will
+ * pass in a valid device pointer. If NULL is passed to the dma mapping
+ * routines, depending on the platform, it may or may not succeed, and
+ * may crash.
+ *
+ * As such:
+ * Wrapper all the dma routines and check the dev pointer.
+ *
+ * If simple mappings (return just a dma address, we'll noop them,
+ * returning a dma address of 0.
+ *
+ * On more complex mappings (dma_map_sg), a pseudo routine fills
+ * in the scatter list, setting all dma addresses to 0.
+ */
+
+static inline dma_addr_t
+fc_dma_map_single(struct device *dev, void *ptr, size_t size,
+		enum dma_data_direction dir)
+{
+	return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
+}
+
+static inline int
+fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
+{
+	return dev ? dma_mapping_error(dev, dma_addr) : 0;
+}
+
+static inline void
+fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
+	enum dma_data_direction dir)
+{
+	if (dev)
+		dma_unmap_single(dev, addr, size, dir);
+}
+
+static inline void
+fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
+		enum dma_data_direction dir)
+{
+	if (dev)
+		dma_sync_single_for_cpu(dev, addr, size, dir);
+}
+
+static inline void
+fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
+		enum dma_data_direction dir)
+{
+	if (dev)
+		dma_sync_single_for_device(dev, addr, size, dir);
+}
+
+/* pseudo dma_map_sg call */
+static int
+fc_map_sg(struct scatterlist *sg, int nents)
+{
+	struct scatterlist *s;
+	int i;
+
+	WARN_ON(nents == 0 || sg[0].length == 0);
+
+	for_each_sg(sg, s, nents, i) {
+		s->dma_address = 0L;
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+		s->dma_length = s->length;
+#endif
+	}
+	return nents;
+}
+
+static inline int
+fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
+		enum dma_data_direction dir)
+{
+	return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
+}
+
+static inline void
+fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
+		enum dma_data_direction dir)
+{
+	if (dev)
+		dma_unmap_sg(dev, sg, nents, dir);
+}
+
+
+/* *********************** FC-NVME Port Management ************************ */
+
+
+static int
+nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
+{
+	struct nvmet_fc_ls_iod *iod;
+	int i;
+
+	iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod),
+			GFP_KERNEL);
+	if (!iod)
+		return -ENOMEM;
+
+	tgtport->iod = iod;
+
+	for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
+		INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work);
+		iod->tgtport = tgtport;
+		list_add_tail(&iod->ls_list, &tgtport->ls_list);
+
+		iod->rqstbuf = kcalloc(2, NVME_FC_MAX_LS_BUFFER_SIZE,
+			GFP_KERNEL);
+		if (!iod->rqstbuf)
+			goto out_fail;
+
+		iod->rspbuf = iod->rqstbuf + NVME_FC_MAX_LS_BUFFER_SIZE;
+
+		iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf,
+						NVME_FC_MAX_LS_BUFFER_SIZE,
+						DMA_TO_DEVICE);
+		if (fc_dma_mapping_error(tgtport->dev, iod->rspdma))
+			goto out_fail;
+	}
+
+	return 0;
+
+out_fail:
+	kfree(iod->rqstbuf);
+	list_del(&iod->ls_list);
+	for (iod--, i--; i >= 0; iod--, i--) {
+		fc_dma_unmap_single(tgtport->dev, iod->rspdma,
+				NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
+		kfree(iod->rqstbuf);
+		list_del(&iod->ls_list);
+	}
+
+	kfree(iod);
+
+	return -EFAULT;
+}
+
+static void
+nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
+{
+	struct nvmet_fc_ls_iod *iod = tgtport->iod;
+	int i;
+
+	for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
+		fc_dma_unmap_single(tgtport->dev,
+				iod->rspdma, NVME_FC_MAX_LS_BUFFER_SIZE,
+				DMA_TO_DEVICE);
+		kfree(iod->rqstbuf);
+		list_del(&iod->ls_list);
+	}
+	kfree(tgtport->iod);
+}
+
+static struct nvmet_fc_ls_iod *
+nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport)
+{
+	static struct nvmet_fc_ls_iod *iod;
+	unsigned long flags;
+
+	spin_lock_irqsave(&tgtport->lock, flags);
+	iod = list_first_entry_or_null(&tgtport->ls_list,
+					struct nvmet_fc_ls_iod, ls_list);
+	if (iod)
+		list_move_tail(&iod->ls_list, &tgtport->ls_busylist);
+	spin_unlock_irqrestore(&tgtport->lock, flags);
+	return iod;
+}
+
+
+static void
+nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport,
+			struct nvmet_fc_ls_iod *iod)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&tgtport->lock, flags);
+	list_move(&iod->ls_list, &tgtport->ls_list);
+	spin_unlock_irqrestore(&tgtport->lock, flags);
+}
+
+static void
+nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
+				struct nvmet_fc_tgt_queue *queue)
+{
+	struct nvmet_fc_fcp_iod *fod = queue->fod;
+	int i;
+
+	for (i = 0; i < queue->sqsize; fod++, i++) {
+		INIT_WORK(&fod->work, nvmet_fc_handle_fcp_rqst_work);
+		fod->tgtport = tgtport;
+		fod->queue = queue;
+		fod->active = false;
+		list_add_tail(&fod->fcp_list, &queue->fod_list);
+		spin_lock_init(&fod->flock);
+
+		fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf,
+					sizeof(fod->rspiubuf), DMA_TO_DEVICE);
+		if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) {
+			list_del(&fod->fcp_list);
+			for (fod--, i--; i >= 0; fod--, i--) {
+				fc_dma_unmap_single(tgtport->dev, fod->rspdma,
+						sizeof(fod->rspiubuf),
+						DMA_TO_DEVICE);
+				fod->rspdma = 0L;
+				list_del(&fod->fcp_list);
+			}
+
+			return;
+		}
+	}
+}
+
+static void
+nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
+				struct nvmet_fc_tgt_queue *queue)
+{
+	struct nvmet_fc_fcp_iod *fod = queue->fod;
+	int i;
+
+	for (i = 0; i < queue->sqsize; fod++, i++) {
+		if (fod->rspdma)
+			fc_dma_unmap_single(tgtport->dev, fod->rspdma,
+				sizeof(fod->rspiubuf), DMA_TO_DEVICE);
+	}
+}
+
+static struct nvmet_fc_fcp_iod *
+nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue)
+{
+	static struct nvmet_fc_fcp_iod *fod;
+	unsigned long flags;
+
+	spin_lock_irqsave(&queue->qlock, flags);
+	fod = list_first_entry_or_null(&queue->fod_list,
+					struct nvmet_fc_fcp_iod, fcp_list);
+	if (fod) {
+		list_del(&fod->fcp_list);
+		fod->active = true;
+		fod->abort = false;
+		/*
+		 * no queue reference is taken, as it was taken by the
+		 * queue lookup just prior to the allocation. The iod
+		 * will "inherit" that reference.
+		 */
+	}
+	spin_unlock_irqrestore(&queue->qlock, flags);
+	return fod;
+}
+
+
+static void
+nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue,
+			struct nvmet_fc_fcp_iod *fod)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&queue->qlock, flags);
+	list_add_tail(&fod->fcp_list, &fod->queue->fod_list);
+	fod->active = false;
+	spin_unlock_irqrestore(&queue->qlock, flags);
+
+	/*
+	 * release the reference taken at queue lookup and fod allocation
+	 */
+	nvmet_fc_tgt_q_put(queue);
+}
+
+static int
+nvmet_fc_queue_to_cpu(struct nvmet_fc_tgtport *tgtport, int qid)
+{
+	int cpu, idx, cnt;
+
+	if (!(tgtport->ops->target_features &
+			NVMET_FCTGTFEAT_NEEDS_CMD_CPUSCHED) ||
+	    tgtport->ops->max_hw_queues == 1)
+		return WORK_CPU_UNBOUND;
+
+	/* Simple cpu selection based on qid modulo active cpu count */
+	idx = !qid ? 0 : (qid - 1) % num_active_cpus();
+
+	/* find the n'th active cpu */
+	for (cpu = 0, cnt = 0; ; ) {
+		if (cpu_active(cpu)) {
+			if (cnt == idx)
+				break;
+			cnt++;
+		}
+		cpu = (cpu + 1) % num_possible_cpus();
+	}
+
+	return cpu;
+}
+
+static struct nvmet_fc_tgt_queue *
+nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,
+			u16 qid, u16 sqsize)
+{
+	struct nvmet_fc_tgt_queue *queue;
+	unsigned long flags;
+	int ret;
+
+	if (qid >= NVMET_NR_QUEUES)
+		return NULL;
+
+	queue = kzalloc((sizeof(*queue) +
+				(sizeof(struct nvmet_fc_fcp_iod) * sqsize)),
+				GFP_KERNEL);
+	if (!queue)
+		return NULL;
+
+	if (!nvmet_fc_tgt_a_get(assoc))
+		goto out_free_queue;
+
+	queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0,
+				assoc->tgtport->fc_target_port.port_num,
+				assoc->a_id, qid);
+	if (!queue->work_q)
+		goto out_a_put;
+
+	queue->fod = (struct nvmet_fc_fcp_iod *)&queue[1];
+	queue->qid = qid;
+	queue->sqsize = sqsize;
+	queue->assoc = assoc;
+	queue->port = assoc->tgtport->port;
+	queue->cpu = nvmet_fc_queue_to_cpu(assoc->tgtport, qid);
+	INIT_LIST_HEAD(&queue->fod_list);
+	atomic_set(&queue->connected, 0);
+	atomic_set(&queue->sqtail, 0);
+	atomic_set(&queue->rsn, 1);
+	atomic_set(&queue->zrspcnt, 0);
+	spin_lock_init(&queue->qlock);
+	kref_init(&queue->ref);
+
+	nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue);
+
+	ret = nvmet_sq_init(&queue->nvme_sq);
+	if (ret)
+		goto out_fail_iodlist;
+
+	WARN_ON(assoc->queues[qid]);
+	spin_lock_irqsave(&assoc->tgtport->lock, flags);
+	assoc->queues[qid] = queue;
+	spin_unlock_irqrestore(&assoc->tgtport->lock, flags);
+
+	return queue;
+
+out_fail_iodlist:
+	nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue);
+	destroy_workqueue(queue->work_q);
+out_a_put:
+	nvmet_fc_tgt_a_put(assoc);
+out_free_queue:
+	kfree(queue);
+	return NULL;
+}
+
+
+static void
+nvmet_fc_tgt_queue_free(struct kref *ref)
+{
+	struct nvmet_fc_tgt_queue *queue =
+		container_of(ref, struct nvmet_fc_tgt_queue, ref);
+	unsigned long flags;
+
+	spin_lock_irqsave(&queue->assoc->tgtport->lock, flags);
+	queue->assoc->queues[queue->qid] = NULL;
+	spin_unlock_irqrestore(&queue->assoc->tgtport->lock, flags);
+
+	nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue);
+
+	nvmet_fc_tgt_a_put(queue->assoc);
+
+	destroy_workqueue(queue->work_q);
+
+	kfree(queue);
+}
+
+static void
+nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue)
+{
+	kref_put(&queue->ref, nvmet_fc_tgt_queue_free);
+}
+
+static int
+nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue)
+{
+	return kref_get_unless_zero(&queue->ref);
+}
+
+
+static void
+nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport,
+				struct nvmefc_tgt_fcp_req *fcpreq)
+{
+	int ret;
+
+	fcpreq->op = NVMET_FCOP_ABORT;
+	fcpreq->offset = 0;
+	fcpreq->timeout = 0;
+	fcpreq->transfer_length = 0;
+	fcpreq->transferred_length = 0;
+	fcpreq->fcp_error = 0;
+	fcpreq->sg_cnt = 0;
+
+	ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fcpreq);
+	if (ret)
+		/* should never reach here !! */
+		WARN_ON(1);
+}
+
+
+static void
+nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
+{
+	struct nvmet_fc_fcp_iod *fod = queue->fod;
+	unsigned long flags;
+	int i;
+	bool disconnect;
+
+	disconnect = atomic_xchg(&queue->connected, 0);
+
+	spin_lock_irqsave(&queue->qlock, flags);
+	/* about outstanding io's */
+	for (i = 0; i < queue->sqsize; fod++, i++) {
+		if (fod->active) {
+			spin_lock(&fod->flock);
+			fod->abort = true;
+			spin_unlock(&fod->flock);
+		}
+	}
+	spin_unlock_irqrestore(&queue->qlock, flags);
+
+	flush_workqueue(queue->work_q);
+
+	if (disconnect)
+		nvmet_sq_destroy(&queue->nvme_sq);
+
+	nvmet_fc_tgt_q_put(queue);
+}
+
+static struct nvmet_fc_tgt_queue *
+nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport,
+				u64 connection_id)
+{
+	struct nvmet_fc_tgt_assoc *assoc;
+	struct nvmet_fc_tgt_queue *queue;
+	u64 association_id = nvmet_fc_getassociationid(connection_id);
+	u16 qid = nvmet_fc_getqueueid(connection_id);
+	unsigned long flags;
+
+	spin_lock_irqsave(&tgtport->lock, flags);
+	list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
+		if (association_id == assoc->association_id) {
+			queue = assoc->queues[qid];
+			if (queue &&
+			    (!atomic_read(&queue->connected) ||
+			     !nvmet_fc_tgt_q_get(queue)))
+				queue = NULL;
+			spin_unlock_irqrestore(&tgtport->lock, flags);
+			return queue;
+		}
+	}
+	spin_unlock_irqrestore(&tgtport->lock, flags);
+	return NULL;
+}
+
+static struct nvmet_fc_tgt_assoc *
+nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport)
+{
+	struct nvmet_fc_tgt_assoc *assoc, *tmpassoc;
+	unsigned long flags;
+	u64 ran;
+	int idx;
+	bool needrandom = true;
+
+	assoc = kzalloc(sizeof(*assoc), GFP_KERNEL);
+	if (!assoc)
+		return NULL;
+
+	idx = ida_simple_get(&tgtport->assoc_cnt, 0, 0, GFP_KERNEL);
+	if (idx < 0)
+		goto out_free_assoc;
+
+	if (!nvmet_fc_tgtport_get(tgtport))
+		goto out_ida_put;
+
+	assoc->tgtport = tgtport;
+	assoc->a_id = idx;
+	INIT_LIST_HEAD(&assoc->a_list);
+	kref_init(&assoc->ref);
+
+	while (needrandom) {
+		get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID);
+		ran = ran << BYTES_FOR_QID_SHIFT;
+
+		spin_lock_irqsave(&tgtport->lock, flags);
+		needrandom = false;
+		list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list)
+			if (ran == tmpassoc->association_id) {
+				needrandom = true;
+				break;
+			}
+		if (!needrandom) {
+			assoc->association_id = ran;
+			list_add_tail(&assoc->a_list, &tgtport->assoc_list);
+		}
+		spin_unlock_irqrestore(&tgtport->lock, flags);
+	}
+
+	return assoc;
+
+out_ida_put:
+	ida_simple_remove(&tgtport->assoc_cnt, idx);
+out_free_assoc:
+	kfree(assoc);
+	return NULL;
+}
+
+static void
+nvmet_fc_target_assoc_free(struct kref *ref)
+{
+	struct nvmet_fc_tgt_assoc *assoc =
+		container_of(ref, struct nvmet_fc_tgt_assoc, ref);
+	struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
+	unsigned long flags;
+
+	spin_lock_irqsave(&tgtport->lock, flags);
+	list_del(&assoc->a_list);
+	spin_unlock_irqrestore(&tgtport->lock, flags);
+	ida_simple_remove(&tgtport->assoc_cnt, assoc->a_id);
+	kfree(assoc);
+	nvmet_fc_tgtport_put(tgtport);
+}
+
+static void
+nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc)
+{
+	kref_put(&assoc->ref, nvmet_fc_target_assoc_free);
+}
+
+static int
+nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc)
+{
+	return kref_get_unless_zero(&assoc->ref);
+}
+
+static void
+nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc)
+{
+	struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
+	struct nvmet_fc_tgt_queue *queue;
+	unsigned long flags;
+	int i;
+
+	spin_lock_irqsave(&tgtport->lock, flags);
+	for (i = NVMET_NR_QUEUES - 1; i >= 0; i--) {
+		queue = assoc->queues[i];
+		if (queue) {
+			if (!nvmet_fc_tgt_q_get(queue))
+				continue;
+			spin_unlock_irqrestore(&tgtport->lock, flags);
+			nvmet_fc_delete_target_queue(queue);
+			nvmet_fc_tgt_q_put(queue);
+			spin_lock_irqsave(&tgtport->lock, flags);
+		}
+	}
+	spin_unlock_irqrestore(&tgtport->lock, flags);
+
+	nvmet_fc_tgt_a_put(assoc);
+}
+
+static struct nvmet_fc_tgt_assoc *
+nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport,
+				u64 association_id)
+{
+	struct nvmet_fc_tgt_assoc *assoc;
+	struct nvmet_fc_tgt_assoc *ret = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&tgtport->lock, flags);
+	list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
+		if (association_id == assoc->association_id) {
+			ret = assoc;
+			nvmet_fc_tgt_a_get(assoc);
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&tgtport->lock, flags);
+
+	return ret;
+}
+
+
+/**
+ * nvme_fc_register_targetport - transport entry point called by an
+ *                              LLDD to register the existence of a local
+ *                              NVME subystem FC port.
+ * @pinfo:     pointer to information about the port to be registered
+ * @template:  LLDD entrypoints and operational parameters for the port
+ * @dev:       physical hardware device node port corresponds to. Will be
+ *             used for DMA mappings
+ * @portptr:   pointer to a local port pointer. Upon success, the routine
+ *             will allocate a nvme_fc_local_port structure and place its
+ *             address in the local port pointer. Upon failure, local port
+ *             pointer will be set to NULL.
+ *
+ * Returns:
+ * a completion status. Must be 0 upon success; a negative errno
+ * (ex: -ENXIO) upon failure.
+ */
+int
+nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,
+			struct nvmet_fc_target_template *template,
+			struct device *dev,
+			struct nvmet_fc_target_port **portptr)
+{
+	struct nvmet_fc_tgtport *newrec;
+	unsigned long flags;
+	int ret, idx;
+
+	if (!template->xmt_ls_rsp || !template->fcp_op ||
+	    !template->targetport_delete ||
+	    !template->max_hw_queues || !template->max_sgl_segments ||
+	    !template->max_dif_sgl_segments || !template->dma_boundary) {
+		ret = -EINVAL;
+		goto out_regtgt_failed;
+	}
+
+	newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz),
+			 GFP_KERNEL);
+	if (!newrec) {
+		ret = -ENOMEM;
+		goto out_regtgt_failed;
+	}
+
+	idx = ida_simple_get(&nvmet_fc_tgtport_cnt, 0, 0, GFP_KERNEL);
+	if (idx < 0) {
+		ret = -ENOSPC;
+		goto out_fail_kfree;
+	}
+
+	if (!get_device(dev) && dev) {
+		ret = -ENODEV;
+		goto out_ida_put;
+	}
+
+	newrec->fc_target_port.node_name = pinfo->node_name;
+	newrec->fc_target_port.port_name = pinfo->port_name;
+	newrec->fc_target_port.private = &newrec[1];
+	newrec->fc_target_port.port_id = pinfo->port_id;
+	newrec->fc_target_port.port_num = idx;
+	INIT_LIST_HEAD(&newrec->tgt_list);
+	newrec->dev = dev;
+	newrec->ops = template;
+	spin_lock_init(&newrec->lock);
+	INIT_LIST_HEAD(&newrec->ls_list);
+	INIT_LIST_HEAD(&newrec->ls_busylist);
+	INIT_LIST_HEAD(&newrec->assoc_list);
+	kref_init(&newrec->ref);
+	ida_init(&newrec->assoc_cnt);
+
+	ret = nvmet_fc_alloc_ls_iodlist(newrec);
+	if (ret) {
+		ret = -ENOMEM;
+		goto out_free_newrec;
+	}
+
+	spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
+	list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list);
+	spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
+
+	*portptr = &newrec->fc_target_port;
+	return 0;
+
+out_free_newrec:
+	put_device(dev);
+out_ida_put:
+	ida_simple_remove(&nvmet_fc_tgtport_cnt, idx);
+out_fail_kfree:
+	kfree(newrec);
+out_regtgt_failed:
+	*portptr = NULL;
+	return ret;
+}
+EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport);
+
+
+static void
+nvmet_fc_free_tgtport(struct kref *ref)
+{
+	struct nvmet_fc_tgtport *tgtport =
+		container_of(ref, struct nvmet_fc_tgtport, ref);
+	struct device *dev = tgtport->dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
+	list_del(&tgtport->tgt_list);
+	spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
+
+	nvmet_fc_free_ls_iodlist(tgtport);
+
+	/* let the LLDD know we've finished tearing it down */
+	tgtport->ops->targetport_delete(&tgtport->fc_target_port);
+
+	ida_simple_remove(&nvmet_fc_tgtport_cnt,
+			tgtport->fc_target_port.port_num);
+
+	ida_destroy(&tgtport->assoc_cnt);
+
+	kfree(tgtport);
+
+	put_device(dev);
+}
+
+static void
+nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport)
+{
+	kref_put(&tgtport->ref, nvmet_fc_free_tgtport);
+}
+
+static int
+nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport)
+{
+	return kref_get_unless_zero(&tgtport->ref);
+}
+
+static void
+__nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)
+{
+	struct nvmet_fc_tgt_assoc *assoc, *next;
+	unsigned long flags;
+
+	spin_lock_irqsave(&tgtport->lock, flags);
+	list_for_each_entry_safe(assoc, next,
+				&tgtport->assoc_list, a_list) {
+		if (!nvmet_fc_tgt_a_get(assoc))
+			continue;
+		spin_unlock_irqrestore(&tgtport->lock, flags);
+		nvmet_fc_delete_target_assoc(assoc);
+		nvmet_fc_tgt_a_put(assoc);
+		spin_lock_irqsave(&tgtport->lock, flags);
+	}
+	spin_unlock_irqrestore(&tgtport->lock, flags);
+}
+
+/*
+ * nvmet layer has called to terminate an association
+ */
+static void
+nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
+{
+	struct nvmet_fc_tgtport *tgtport, *next;
+	struct nvmet_fc_tgt_assoc *assoc;
+	struct nvmet_fc_tgt_queue *queue;
+	unsigned long flags;
+	bool found_ctrl = false;
+
+	/* this is a bit ugly, but don't want to make locks layered */
+	spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
+	list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list,
+			tgt_list) {
+		if (!nvmet_fc_tgtport_get(tgtport))
+			continue;
+		spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
+
+		spin_lock_irqsave(&tgtport->lock, flags);
+		list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
+			queue = assoc->queues[0];
+			if (queue && queue->nvme_sq.ctrl == ctrl) {
+				if (nvmet_fc_tgt_a_get(assoc))
+					found_ctrl = true;
+				break;
+			}
+		}
+		spin_unlock_irqrestore(&tgtport->lock, flags);
+
+		nvmet_fc_tgtport_put(tgtport);
+
+		if (found_ctrl) {
+			nvmet_fc_delete_target_assoc(assoc);
+			nvmet_fc_tgt_a_put(assoc);
+			return;
+		}
+
+		spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
+	}
+	spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
+}
+
+/**
+ * nvme_fc_unregister_targetport - transport entry point called by an
+ *                              LLDD to deregister/remove a previously
+ *                              registered a local NVME subsystem FC port.
+ * @tgtport: pointer to the (registered) target port that is to be
+ *           deregistered.
+ *
+ * Returns:
+ * a completion status. Must be 0 upon success; a negative errno
+ * (ex: -ENXIO) upon failure.
+ */
+int
+nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port)
+{
+	struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
+
+	/* terminate any outstanding associations */
+	__nvmet_fc_free_assocs(tgtport);
+
+	nvmet_fc_tgtport_put(tgtport);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport);
+
+
+/* *********************** FC-NVME LS Handling **************************** */
+
+
+static void
+nvmet_fc_format_rsp_hdr(void *buf, u8 ls_cmd, u32 desc_len, u8 rqst_ls_cmd)
+{
+	struct fcnvme_ls_acc_hdr *acc = buf;
+
+	acc->w0.ls_cmd = ls_cmd;
+	acc->desc_list_len = desc_len;
+	acc->rqst.desc_tag = cpu_to_be32(FCNVME_LSDESC_RQST);
+	acc->rqst.desc_len =
+			fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst));
+	acc->rqst.w0.ls_cmd = rqst_ls_cmd;
+}
+
+static int
+nvmet_fc_format_rjt(void *buf, u16 buflen, u8 ls_cmd,
+			u8 reason, u8 explanation, u8 vendor)
+{
+	struct fcnvme_ls_rjt *rjt = buf;
+
+	nvmet_fc_format_rsp_hdr(buf, FCNVME_LSDESC_RQST,
+			fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_rjt)),
+			ls_cmd);
+	rjt->rjt.desc_tag = cpu_to_be32(FCNVME_LSDESC_RJT);
+	rjt->rjt.desc_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rjt));
+	rjt->rjt.reason_code = reason;
+	rjt->rjt.reason_explanation = explanation;
+	rjt->rjt.vendor = vendor;
+
+	return sizeof(struct fcnvme_ls_rjt);
+}
+
+/* Validation Error indexes into the string table below */
+enum {
+	VERR_NO_ERROR		= 0,
+	VERR_CR_ASSOC_LEN	= 1,
+	VERR_CR_ASSOC_RQST_LEN	= 2,
+	VERR_CR_ASSOC_CMD	= 3,
+	VERR_CR_ASSOC_CMD_LEN	= 4,
+	VERR_ERSP_RATIO		= 5,
+	VERR_ASSOC_ALLOC_FAIL	= 6,
+	VERR_QUEUE_ALLOC_FAIL	= 7,
+	VERR_CR_CONN_LEN	= 8,
+	VERR_CR_CONN_RQST_LEN	= 9,
+	VERR_ASSOC_ID		= 10,
+	VERR_ASSOC_ID_LEN	= 11,
+	VERR_NO_ASSOC		= 12,
+	VERR_CONN_ID		= 13,
+	VERR_CONN_ID_LEN	= 14,
+	VERR_NO_CONN		= 15,
+	VERR_CR_CONN_CMD	= 16,
+	VERR_CR_CONN_CMD_LEN	= 17,
+	VERR_DISCONN_LEN	= 18,
+	VERR_DISCONN_RQST_LEN	= 19,
+	VERR_DISCONN_CMD	= 20,
+	VERR_DISCONN_CMD_LEN	= 21,
+	VERR_DISCONN_SCOPE	= 22,
+	VERR_RS_LEN		= 23,
+	VERR_RS_RQST_LEN	= 24,
+	VERR_RS_CMD		= 25,
+	VERR_RS_CMD_LEN		= 26,
+	VERR_RS_RCTL		= 27,
+	VERR_RS_RO		= 28,
+};
+
+static char *validation_errors[] = {
+	"OK",
+	"Bad CR_ASSOC Length",
+	"Bad CR_ASSOC Rqst Length",
+	"Not CR_ASSOC Cmd",
+	"Bad CR_ASSOC Cmd Length",
+	"Bad Ersp Ratio",
+	"Association Allocation Failed",
+	"Queue Allocation Failed",
+	"Bad CR_CONN Length",
+	"Bad CR_CONN Rqst Length",
+	"Not Association ID",
+	"Bad Association ID Length",
+	"No Association",
+	"Not Connection ID",
+	"Bad Connection ID Length",
+	"No Connection",
+	"Not CR_CONN Cmd",
+	"Bad CR_CONN Cmd Length",
+	"Bad DISCONN Length",
+	"Bad DISCONN Rqst Length",
+	"Not DISCONN Cmd",
+	"Bad DISCONN Cmd Length",
+	"Bad Disconnect Scope",
+	"Bad RS Length",
+	"Bad RS Rqst Length",
+	"Not RS Cmd",
+	"Bad RS Cmd Length",
+	"Bad RS R_CTL",
+	"Bad RS Relative Offset",
+};
+
+static void
+nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport,
+			struct nvmet_fc_ls_iod *iod)
+{
+	struct fcnvme_ls_cr_assoc_rqst *rqst =
+				(struct fcnvme_ls_cr_assoc_rqst *)iod->rqstbuf;
+	struct fcnvme_ls_cr_assoc_acc *acc =
+				(struct fcnvme_ls_cr_assoc_acc *)iod->rspbuf;
+	struct nvmet_fc_tgt_queue *queue;
+	int ret = 0;
+
+	memset(acc, 0, sizeof(*acc));
+
+	if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_assoc_rqst))
+		ret = VERR_CR_ASSOC_LEN;
+	else if (rqst->desc_list_len !=
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_ls_cr_assoc_rqst)))
+		ret = VERR_CR_ASSOC_RQST_LEN;
+	else if (rqst->assoc_cmd.desc_tag !=
+			cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD))
+		ret = VERR_CR_ASSOC_CMD;
+	else if (rqst->assoc_cmd.desc_len !=
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_cr_assoc_cmd)))
+		ret = VERR_CR_ASSOC_CMD_LEN;
+	else if (!rqst->assoc_cmd.ersp_ratio ||
+		 (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >=
+				be16_to_cpu(rqst->assoc_cmd.sqsize)))
+		ret = VERR_ERSP_RATIO;
+
+	else {
+		/* new association w/ admin queue */
+		iod->assoc = nvmet_fc_alloc_target_assoc(tgtport);
+		if (!iod->assoc)
+			ret = VERR_ASSOC_ALLOC_FAIL;
+		else {
+			queue = nvmet_fc_alloc_target_queue(iod->assoc, 0,
+					be16_to_cpu(rqst->assoc_cmd.sqsize));
+			if (!queue)
+				ret = VERR_QUEUE_ALLOC_FAIL;
+		}
+	}
+
+	if (ret) {
+		dev_err(tgtport->dev,
+			"Create Association LS failed: %s\n",
+			validation_errors[ret]);
+		iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
+				NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
+				ELS_RJT_LOGIC,
+				ELS_EXPL_NONE, 0);
+		return;
+	}
+
+	queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio);
+	atomic_set(&queue->connected, 1);
+	queue->sqhd = 0;	/* best place to init value */
+
+	/* format a response */
+
+	iod->lsreq->rsplen = sizeof(*acc);
+
+	nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_ls_cr_assoc_acc)),
+			FCNVME_LS_CREATE_ASSOCIATION);
+	acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
+	acc->associd.desc_len =
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_assoc_id));
+	acc->associd.association_id =
+			cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0));
+	acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
+	acc->connectid.desc_len =
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_conn_id));
+	acc->connectid.connection_id = acc->associd.association_id;
+}
+
+static void
+nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
+			struct nvmet_fc_ls_iod *iod)
+{
+	struct fcnvme_ls_cr_conn_rqst *rqst =
+				(struct fcnvme_ls_cr_conn_rqst *)iod->rqstbuf;
+	struct fcnvme_ls_cr_conn_acc *acc =
+				(struct fcnvme_ls_cr_conn_acc *)iod->rspbuf;
+	struct nvmet_fc_tgt_queue *queue;
+	int ret = 0;
+
+	memset(acc, 0, sizeof(*acc));
+
+	if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst))
+		ret = VERR_CR_CONN_LEN;
+	else if (rqst->desc_list_len !=
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_ls_cr_conn_rqst)))
+		ret = VERR_CR_CONN_RQST_LEN;
+	else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
+		ret = VERR_ASSOC_ID;
+	else if (rqst->associd.desc_len !=
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_assoc_id)))
+		ret = VERR_ASSOC_ID_LEN;
+	else if (rqst->connect_cmd.desc_tag !=
+			cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD))
+		ret = VERR_CR_CONN_CMD;
+	else if (rqst->connect_cmd.desc_len !=
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_cr_conn_cmd)))
+		ret = VERR_CR_CONN_CMD_LEN;
+	else if (!rqst->connect_cmd.ersp_ratio ||
+		 (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >=
+				be16_to_cpu(rqst->connect_cmd.sqsize)))
+		ret = VERR_ERSP_RATIO;
+
+	else {
+		/* new io queue */
+		iod->assoc = nvmet_fc_find_target_assoc(tgtport,
+				be64_to_cpu(rqst->associd.association_id));
+		if (!iod->assoc)
+			ret = VERR_NO_ASSOC;
+		else {
+			queue = nvmet_fc_alloc_target_queue(iod->assoc,
+					be16_to_cpu(rqst->connect_cmd.qid),
+					be16_to_cpu(rqst->connect_cmd.sqsize));
+			if (!queue)
+				ret = VERR_QUEUE_ALLOC_FAIL;
+
+			/* release get taken in nvmet_fc_find_target_assoc */
+			nvmet_fc_tgt_a_put(iod->assoc);
+		}
+	}
+
+	if (ret) {
+		dev_err(tgtport->dev,
+			"Create Connection LS failed: %s\n",
+			validation_errors[ret]);
+		iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
+				NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
+				(ret == VERR_NO_ASSOC) ?
+						ELS_RJT_PROT : ELS_RJT_LOGIC,
+				ELS_EXPL_NONE, 0);
+		return;
+	}
+
+	queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio);
+	atomic_set(&queue->connected, 1);
+	queue->sqhd = 0;	/* best place to init value */
+
+	/* format a response */
+
+	iod->lsreq->rsplen = sizeof(*acc);
+
+	nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
+			fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)),
+			FCNVME_LS_CREATE_CONNECTION);
+	acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
+	acc->connectid.desc_len =
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_conn_id));
+	acc->connectid.connection_id =
+			cpu_to_be64(nvmet_fc_makeconnid(iod->assoc,
+				be16_to_cpu(rqst->connect_cmd.qid)));
+}
+
+static void
+nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,
+			struct nvmet_fc_ls_iod *iod)
+{
+	struct fcnvme_ls_disconnect_rqst *rqst =
+			(struct fcnvme_ls_disconnect_rqst *)iod->rqstbuf;
+	struct fcnvme_ls_disconnect_acc *acc =
+			(struct fcnvme_ls_disconnect_acc *)iod->rspbuf;
+	struct nvmet_fc_tgt_queue *queue;
+	struct nvmet_fc_tgt_assoc *assoc;
+	int ret = 0;
+	bool del_assoc = false;
+
+	memset(acc, 0, sizeof(*acc));
+
+	if (iod->rqstdatalen < sizeof(struct fcnvme_ls_disconnect_rqst))
+		ret = VERR_DISCONN_LEN;
+	else if (rqst->desc_list_len !=
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_ls_disconnect_rqst)))
+		ret = VERR_DISCONN_RQST_LEN;
+	else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
+		ret = VERR_ASSOC_ID;
+	else if (rqst->associd.desc_len !=
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_assoc_id)))
+		ret = VERR_ASSOC_ID_LEN;
+	else if (rqst->discon_cmd.desc_tag !=
+			cpu_to_be32(FCNVME_LSDESC_DISCONN_CMD))
+		ret = VERR_DISCONN_CMD;
+	else if (rqst->discon_cmd.desc_len !=
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_lsdesc_disconn_cmd)))
+		ret = VERR_DISCONN_CMD_LEN;
+	else if ((rqst->discon_cmd.scope != FCNVME_DISCONN_ASSOCIATION) &&
+			(rqst->discon_cmd.scope != FCNVME_DISCONN_CONNECTION))
+		ret = VERR_DISCONN_SCOPE;
+	else {
+		/* match an active association */
+		assoc = nvmet_fc_find_target_assoc(tgtport,
+				be64_to_cpu(rqst->associd.association_id));
+		iod->assoc = assoc;
+		if (!assoc)
+			ret = VERR_NO_ASSOC;
+	}
+
+	if (ret) {
+		dev_err(tgtport->dev,
+			"Disconnect LS failed: %s\n",
+			validation_errors[ret]);
+		iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
+				NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
+				(ret == 8) ? ELS_RJT_PROT : ELS_RJT_LOGIC,
+				ELS_EXPL_NONE, 0);
+		return;
+	}
+
+	/* format a response */
+
+	iod->lsreq->rsplen = sizeof(*acc);
+
+	nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
+			fcnvme_lsdesc_len(
+				sizeof(struct fcnvme_ls_disconnect_acc)),
+			FCNVME_LS_DISCONNECT);
+
+
+	if (rqst->discon_cmd.scope == FCNVME_DISCONN_CONNECTION) {
+		queue = nvmet_fc_find_target_queue(tgtport,
+					be64_to_cpu(rqst->discon_cmd.id));
+		if (queue) {
+			int qid = queue->qid;
+
+			nvmet_fc_delete_target_queue(queue);
+
+			/* release the get taken by find_target_queue */
+			nvmet_fc_tgt_q_put(queue);
+
+			/* tear association down if io queue terminated */
+			if (!qid)
+				del_assoc = true;
+		}
+	}
+
+	/* release get taken in nvmet_fc_find_target_assoc */
+	nvmet_fc_tgt_a_put(iod->assoc);
+
+	if (del_assoc)
+		nvmet_fc_delete_target_assoc(iod->assoc);
+}
+
+
+/* *********************** NVME Ctrl Routines **************************** */
+
+
+static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req);
+
+static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
+
+static void
+nvmet_fc_xmt_ls_rsp_done(struct nvmefc_tgt_ls_req *lsreq)
+{
+	struct nvmet_fc_ls_iod *iod = lsreq->nvmet_fc_private;
+	struct nvmet_fc_tgtport *tgtport = iod->tgtport;
+
+	fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma,
+				NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
+	nvmet_fc_free_ls_iod(tgtport, iod);
+	nvmet_fc_tgtport_put(tgtport);
+}
+
+static void
+nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
+				struct nvmet_fc_ls_iod *iod)
+{
+	int ret;
+
+	fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma,
+				  NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
+
+	ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsreq);
+	if (ret)
+		nvmet_fc_xmt_ls_rsp_done(iod->lsreq);
+}
+
+/*
+ * Actual processing routine for received FC-NVME LS Requests from the LLD
+ */
+static void
+nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport,
+			struct nvmet_fc_ls_iod *iod)
+{
+	struct fcnvme_ls_rqst_w0 *w0 =
+			(struct fcnvme_ls_rqst_w0 *)iod->rqstbuf;
+
+	iod->lsreq->nvmet_fc_private = iod;
+	iod->lsreq->rspbuf = iod->rspbuf;
+	iod->lsreq->rspdma = iod->rspdma;
+	iod->lsreq->done = nvmet_fc_xmt_ls_rsp_done;
+	/* Be preventative. handlers will later set to valid length */
+	iod->lsreq->rsplen = 0;
+
+	iod->assoc = NULL;
+
+	/*
+	 * handlers:
+	 *   parse request input, execute the request, and format the
+	 *   LS response
+	 */
+	switch (w0->ls_cmd) {
+	case FCNVME_LS_CREATE_ASSOCIATION:
+		/* Creates Association and initial Admin Queue/Connection */
+		nvmet_fc_ls_create_association(tgtport, iod);
+		break;
+	case FCNVME_LS_CREATE_CONNECTION:
+		/* Creates an IO Queue/Connection */
+		nvmet_fc_ls_create_connection(tgtport, iod);
+		break;
+	case FCNVME_LS_DISCONNECT:
+		/* Terminate a Queue/Connection or the Association */
+		nvmet_fc_ls_disconnect(tgtport, iod);
+		break;
+	default:
+		iod->lsreq->rsplen = nvmet_fc_format_rjt(iod->rspbuf,
+				NVME_FC_MAX_LS_BUFFER_SIZE, w0->ls_cmd,
+				ELS_RJT_INVAL, ELS_EXPL_NONE, 0);
+	}
+
+	nvmet_fc_xmt_ls_rsp(tgtport, iod);
+}
+
+/*
+ * Actual processing routine for received FC-NVME LS Requests from the LLD
+ */
+static void
+nvmet_fc_handle_ls_rqst_work(struct work_struct *work)
+{
+	struct nvmet_fc_ls_iod *iod =
+		container_of(work, struct nvmet_fc_ls_iod, work);
+	struct nvmet_fc_tgtport *tgtport = iod->tgtport;
+
+	nvmet_fc_handle_ls_rqst(tgtport, iod);
+}
+
+
+/**
+ * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD
+ *                       upon the reception of a NVME LS request.
+ *
+ * The nvmet-fc layer will copy payload to an internal structure for
+ * processing.  As such, upon completion of the routine, the LLDD may
+ * immediately free/reuse the LS request buffer passed in the call.
+ *
+ * If this routine returns error, the LLDD should abort the exchange.
+ *
+ * @tgtport:    pointer to the (registered) target port the LS was
+ *              received on.
+ * @lsreq:      pointer to a lsreq request structure to be used to reference
+ *              the exchange corresponding to the LS.
+ * @lsreqbuf:   pointer to the buffer containing the LS Request
+ * @lsreqbuf_len: length, in bytes, of the received LS request
+ */
+int
+nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
+			struct nvmefc_tgt_ls_req *lsreq,
+			void *lsreqbuf, u32 lsreqbuf_len)
+{
+	struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
+	struct nvmet_fc_ls_iod *iod;
+
+	if (lsreqbuf_len > NVME_FC_MAX_LS_BUFFER_SIZE)
+		return -E2BIG;
+
+	if (!nvmet_fc_tgtport_get(tgtport))
+		return -ESHUTDOWN;
+
+	iod = nvmet_fc_alloc_ls_iod(tgtport);
+	if (!iod) {
+		nvmet_fc_tgtport_put(tgtport);
+		return -ENOENT;
+	}
+
+	iod->lsreq = lsreq;
+	iod->fcpreq = NULL;
+	memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len);
+	iod->rqstdatalen = lsreqbuf_len;
+
+	schedule_work(&iod->work);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req);
+
+
+/*
+ * **********************
+ * Start of FCP handling
+ * **********************
+ */
+
+static int
+nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
+{
+	struct scatterlist *sg;
+	struct page *page;
+	unsigned int nent;
+	u32 page_len, length;
+	int i = 0;
+
+	length = fod->total_length;
+	nent = DIV_ROUND_UP(length, PAGE_SIZE);
+	sg = kmalloc_array(nent, sizeof(struct scatterlist), GFP_KERNEL);
+	if (!sg)
+		goto out;
+
+	sg_init_table(sg, nent);
+
+	while (length) {
+		page_len = min_t(u32, length, PAGE_SIZE);
+
+		page = alloc_page(GFP_KERNEL);
+		if (!page)
+			goto out_free_pages;
+
+		sg_set_page(&sg[i], page, page_len, 0);
+		length -= page_len;
+		i++;
+	}
+
+	fod->data_sg = sg;
+	fod->data_sg_cnt = nent;
+	fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent,
+				((fod->io_dir == NVMET_FCP_WRITE) ?
+					DMA_FROM_DEVICE : DMA_TO_DEVICE));
+				/* note: write from initiator perspective */
+
+	return 0;
+
+out_free_pages:
+	while (i > 0) {
+		i--;
+		__free_page(sg_page(&sg[i]));
+	}
+	kfree(sg);
+	fod->data_sg = NULL;
+	fod->data_sg_cnt = 0;
+out:
+	return NVME_SC_INTERNAL;
+}
+
+static void
+nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
+{
+	struct scatterlist *sg;
+	int count;
+
+	if (!fod->data_sg || !fod->data_sg_cnt)
+		return;
+
+	fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt,
+				((fod->io_dir == NVMET_FCP_WRITE) ?
+					DMA_FROM_DEVICE : DMA_TO_DEVICE));
+	for_each_sg(fod->data_sg, sg, fod->data_sg_cnt, count)
+		__free_page(sg_page(sg));
+	kfree(fod->data_sg);
+}
+
+
+static bool
+queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd)
+{
+	u32 sqtail, used;
+
+	/* egad, this is ugly. And sqtail is just a best guess */
+	sqtail = atomic_read(&q->sqtail) % q->sqsize;
+
+	used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd);
+	return ((used * 10) >= (((u32)(q->sqsize - 1) * 9)));
+}
+
+/*
+ * Prep RSP payload.
+ * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op
+ */
+static void
+nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
+				struct nvmet_fc_fcp_iod *fod)
+{
+	struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf;
+	struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
+	struct nvme_completion *cqe = &ersp->cqe;
+	u32 *cqewd = (u32 *)cqe;
+	bool send_ersp = false;
+	u32 rsn, rspcnt, xfr_length;
+
+	if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP)
+		xfr_length = fod->total_length;
+	else
+		xfr_length = fod->offset;
+
+	/*
+	 * check to see if we can send a 0's rsp.
+	 *   Note: to send a 0's response, the NVME-FC host transport will
+	 *   recreate the CQE. The host transport knows: sq id, SQHD (last
+	 *   seen in an ersp), and command_id. Thus it will create a
+	 *   zero-filled CQE with those known fields filled in. Transport
+	 *   must send an ersp for any condition where the cqe won't match
+	 *   this.
+	 *
+	 * Here are the FC-NVME mandated cases where we must send an ersp:
+	 *  every N responses, where N=ersp_ratio
+	 *  force fabric commands to send ersp's (not in FC-NVME but good
+	 *    practice)
+	 *  normal cmds: any time status is non-zero, or status is zero
+	 *     but words 0 or 1 are non-zero.
+	 *  the SQ is 90% or more full
+	 *  the cmd is a fused command
+	 *  transferred data length not equal to cmd iu length
+	 */
+	rspcnt = atomic_inc_return(&fod->queue->zrspcnt);
+	if (!(rspcnt % fod->queue->ersp_ratio) ||
+	    sqe->opcode == nvme_fabrics_command ||
+	    xfr_length != fod->total_length ||
+	    (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] ||
+	    (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) ||
+	    queue_90percent_full(fod->queue, cqe->sq_head))
+		send_ersp = true;
+
+	/* re-set the fields */
+	fod->fcpreq->rspaddr = ersp;
+	fod->fcpreq->rspdma = fod->rspdma;
+
+	if (!send_ersp) {
+		memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP);
+		fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP;
+	} else {
+		ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32));
+		rsn = atomic_inc_return(&fod->queue->rsn);
+		ersp->rsn = cpu_to_be32(rsn);
+		ersp->xfrd_len = cpu_to_be32(xfr_length);
+		fod->fcpreq->rsplen = sizeof(*ersp);
+	}
+
+	fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma,
+				  sizeof(fod->rspiubuf), DMA_TO_DEVICE);
+}
+
+static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq);
+
+static void
+nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
+				struct nvmet_fc_fcp_iod *fod)
+{
+	int ret;
+
+	fod->fcpreq->op = NVMET_FCOP_RSP;
+	fod->fcpreq->timeout = 0;
+
+	nvmet_fc_prep_fcp_rsp(tgtport, fod);
+
+	ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
+	if (ret)
+		nvmet_fc_abort_op(tgtport, fod->fcpreq);
+}
+
+static void
+nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport,
+				struct nvmet_fc_fcp_iod *fod, u8 op)
+{
+	struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
+	struct scatterlist *sg, *datasg;
+	u32 tlen, sg_off;
+	int ret;
+
+	fcpreq->op = op;
+	fcpreq->offset = fod->offset;
+	fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC;
+	tlen = min_t(u32, (NVMET_FC_MAX_KB_PER_XFR * 1024),
+			(fod->total_length - fod->offset));
+	tlen = min_t(u32, tlen, NVME_FC_MAX_SEGMENTS * PAGE_SIZE);
+	tlen = min_t(u32, tlen, fod->tgtport->ops->max_sgl_segments
+					* PAGE_SIZE);
+	fcpreq->transfer_length = tlen;
+	fcpreq->transferred_length = 0;
+	fcpreq->fcp_error = 0;
+	fcpreq->rsplen = 0;
+
+	fcpreq->sg_cnt = 0;
+
+	datasg = fod->next_sg;
+	sg_off = fod->next_sg_offset;
+
+	for (sg = fcpreq->sg ; tlen; sg++) {
+		*sg = *datasg;
+		if (sg_off) {
+			sg->offset += sg_off;
+			sg->length -= sg_off;
+			sg->dma_address += sg_off;
+			sg_off = 0;
+		}
+		if (tlen < sg->length) {
+			sg->length = tlen;
+			fod->next_sg = datasg;
+			fod->next_sg_offset += tlen;
+		} else if (tlen == sg->length) {
+			fod->next_sg_offset = 0;
+			fod->next_sg = sg_next(datasg);
+		} else {
+			fod->next_sg_offset = 0;
+			datasg = sg_next(datasg);
+		}
+		tlen -= sg->length;
+		fcpreq->sg_cnt++;
+	}
+
+	/*
+	 * If the last READDATA request: check if LLDD supports
+	 * combined xfr with response.
+	 */
+	if ((op == NVMET_FCOP_READDATA) &&
+	    ((fod->offset + fcpreq->transfer_length) == fod->total_length) &&
+	    (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) {
+		fcpreq->op = NVMET_FCOP_READDATA_RSP;
+		nvmet_fc_prep_fcp_rsp(tgtport, fod);
+	}
+
+	ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
+	if (ret) {
+		/*
+		 * should be ok to set w/o lock as its in the thread of
+		 * execution (not an async timer routine) and doesn't
+		 * contend with any clearing action
+		 */
+		fod->abort = true;
+
+		if (op == NVMET_FCOP_WRITEDATA)
+			nvmet_req_complete(&fod->req,
+					NVME_SC_FC_TRANSPORT_ERROR);
+		else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ {
+			fcpreq->fcp_error = ret;
+			fcpreq->transferred_length = 0;
+			nvmet_fc_xmt_fcp_op_done(fod->fcpreq);
+		}
+	}
+}
+
+static void
+nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq)
+{
+	struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
+	struct nvmet_fc_tgtport *tgtport = fod->tgtport;
+	unsigned long flags;
+	bool abort;
+
+	spin_lock_irqsave(&fod->flock, flags);
+	abort = fod->abort;
+	spin_unlock_irqrestore(&fod->flock, flags);
+
+	/* if in the middle of an io and we need to tear down */
+	if (abort && fcpreq->op != NVMET_FCOP_ABORT) {
+		/* data no longer needed */
+		nvmet_fc_free_tgt_pgs(fod);
+
+		if (fcpreq->fcp_error || abort)
+			nvmet_req_complete(&fod->req, fcpreq->fcp_error);
+
+		return;
+	}
+
+	switch (fcpreq->op) {
+
+	case NVMET_FCOP_WRITEDATA:
+		if (abort || fcpreq->fcp_error ||
+		    fcpreq->transferred_length != fcpreq->transfer_length) {
+			nvmet_req_complete(&fod->req,
+					NVME_SC_FC_TRANSPORT_ERROR);
+			return;
+		}
+
+		fod->offset += fcpreq->transferred_length;
+		if (fod->offset != fod->total_length) {
+			/* transfer the next chunk */
+			nvmet_fc_transfer_fcp_data(tgtport, fod,
+						NVMET_FCOP_WRITEDATA);
+			return;
+		}
+
+		/* data transfer complete, resume with nvmet layer */
+
+		fod->req.execute(&fod->req);
+
+		break;
+
+	case NVMET_FCOP_READDATA:
+	case NVMET_FCOP_READDATA_RSP:
+		if (abort || fcpreq->fcp_error ||
+		    fcpreq->transferred_length != fcpreq->transfer_length) {
+			/* data no longer needed */
+			nvmet_fc_free_tgt_pgs(fod);
+
+			nvmet_fc_abort_op(tgtport, fod->fcpreq);
+			return;
+		}
+
+		/* success */
+
+		if (fcpreq->op == NVMET_FCOP_READDATA_RSP) {
+			/* data no longer needed */
+			nvmet_fc_free_tgt_pgs(fod);
+			fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma,
+					sizeof(fod->rspiubuf), DMA_TO_DEVICE);
+			nvmet_fc_free_fcp_iod(fod->queue, fod);
+			return;
+		}
+
+		fod->offset += fcpreq->transferred_length;
+		if (fod->offset != fod->total_length) {
+			/* transfer the next chunk */
+			nvmet_fc_transfer_fcp_data(tgtport, fod,
+						NVMET_FCOP_READDATA);
+			return;
+		}
+
+		/* data transfer complete, send response */
+
+		/* data no longer needed */
+		nvmet_fc_free_tgt_pgs(fod);
+
+		nvmet_fc_xmt_fcp_rsp(tgtport, fod);
+
+		break;
+
+	case NVMET_FCOP_RSP:
+	case NVMET_FCOP_ABORT:
+		fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma,
+				sizeof(fod->rspiubuf), DMA_TO_DEVICE);
+		nvmet_fc_free_fcp_iod(fod->queue, fod);
+		break;
+
+	default:
+		nvmet_fc_free_tgt_pgs(fod);
+		nvmet_fc_abort_op(tgtport, fod->fcpreq);
+		break;
+	}
+}
+
+/*
+ * actual completion handler after execution by the nvmet layer
+ */
+static void
+__nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport,
+			struct nvmet_fc_fcp_iod *fod, int status)
+{
+	struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
+	struct nvme_completion *cqe = &fod->rspiubuf.cqe;
+	unsigned long flags;
+	bool abort;
+
+	spin_lock_irqsave(&fod->flock, flags);
+	abort = fod->abort;
+	spin_unlock_irqrestore(&fod->flock, flags);
+
+	/* if we have a CQE, snoop the last sq_head value */
+	if (!status)
+		fod->queue->sqhd = cqe->sq_head;
+
+	if (abort) {
+		/* data no longer needed */
+		nvmet_fc_free_tgt_pgs(fod);
+
+		nvmet_fc_abort_op(tgtport, fod->fcpreq);
+		return;
+	}
+
+	/* if an error handling the cmd post initial parsing */
+	if (status) {
+		/* fudge up a failed CQE status for our transport error */
+		memset(cqe, 0, sizeof(*cqe));
+		cqe->sq_head = fod->queue->sqhd;	/* echo last cqe sqhd */
+		cqe->sq_id = cpu_to_le16(fod->queue->qid);
+		cqe->command_id = sqe->command_id;
+		cqe->status = cpu_to_le16(status);
+	} else {
+
+		/*
+		 * try to push the data even if the SQE status is non-zero.
+		 * There may be a status where data still was intended to
+		 * be moved
+		 */
+		if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) {
+			/* push the data over before sending rsp */
+			nvmet_fc_transfer_fcp_data(tgtport, fod,
+						NVMET_FCOP_READDATA);
+			return;
+		}
+
+		/* writes & no data - fall thru */
+	}
+
+	/* data no longer needed */
+	nvmet_fc_free_tgt_pgs(fod);
+
+	nvmet_fc_xmt_fcp_rsp(tgtport, fod);
+}
+
+
+static void
+nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req)
+{
+	struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req);
+	struct nvmet_fc_tgtport *tgtport = fod->tgtport;
+
+	__nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0);
+}
+
+
+/*
+ * Actual processing routine for received FC-NVME LS Requests from the LLD
+ */
+void
+nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
+			struct nvmet_fc_fcp_iod *fod)
+{
+	struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf;
+	int ret;
+
+	/*
+	 * Fused commands are currently not supported in the linux
+	 * implementation.
+	 *
+	 * As such, the implementation of the FC transport does not
+	 * look at the fused commands and order delivery to the upper
+	 * layer until we have both based on csn.
+	 */
+
+	fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done;
+
+	fod->total_length = be32_to_cpu(cmdiu->data_len);
+	if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) {
+		fod->io_dir = NVMET_FCP_WRITE;
+		if (!nvme_is_write(&cmdiu->sqe))
+			goto transport_error;
+	} else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) {
+		fod->io_dir = NVMET_FCP_READ;
+		if (nvme_is_write(&cmdiu->sqe))
+			goto transport_error;
+	} else {
+		fod->io_dir = NVMET_FCP_NODATA;
+		if (fod->total_length)
+			goto transport_error;
+	}
+
+	fod->req.cmd = &fod->cmdiubuf.sqe;
+	fod->req.rsp = &fod->rspiubuf.cqe;
+	fod->req.port = fod->queue->port;
+
+	/* ensure nvmet handlers will set cmd handler callback */
+	fod->req.execute = NULL;
+
+	/* clear any response payload */
+	memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf));
+
+	ret = nvmet_req_init(&fod->req,
+				&fod->queue->nvme_cq,
+				&fod->queue->nvme_sq,
+				&nvmet_fc_tgt_fcp_ops);
+	if (!ret) {	/* bad SQE content */
+		nvmet_fc_abort_op(tgtport, fod->fcpreq);
+		return;
+	}
+
+	/* keep a running counter of tail position */
+	atomic_inc(&fod->queue->sqtail);
+
+	fod->data_sg = NULL;
+	fod->data_sg_cnt = 0;
+	if (fod->total_length) {
+		ret = nvmet_fc_alloc_tgt_pgs(fod);
+		if (ret) {
+			nvmet_req_complete(&fod->req, ret);
+			return;
+		}
+	}
+	fod->req.sg = fod->data_sg;
+	fod->req.sg_cnt = fod->data_sg_cnt;
+	fod->offset = 0;
+	fod->next_sg = fod->data_sg;
+	fod->next_sg_offset = 0;
+
+	if (fod->io_dir == NVMET_FCP_WRITE) {
+		/* pull the data over before invoking nvmet layer */
+		nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA);
+		return;
+	}
+
+	/*
+	 * Reads or no data:
+	 *
+	 * can invoke the nvmet_layer now. If read data, cmd completion will
+	 * push the data
+	 */
+
+	fod->req.execute(&fod->req);
+
+	return;
+
+transport_error:
+	nvmet_fc_abort_op(tgtport, fod->fcpreq);
+}
+
+/*
+ * Actual processing routine for received FC-NVME LS Requests from the LLD
+ */
+static void
+nvmet_fc_handle_fcp_rqst_work(struct work_struct *work)
+{
+	struct nvmet_fc_fcp_iod *fod =
+		container_of(work, struct nvmet_fc_fcp_iod, work);
+	struct nvmet_fc_tgtport *tgtport = fod->tgtport;
+
+	nvmet_fc_handle_fcp_rqst(tgtport, fod);
+}
+
+/**
+ * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD
+ *                       upon the reception of a NVME FCP CMD IU.
+ *
+ * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc
+ * layer for processing.
+ *
+ * The nvmet-fc layer will copy cmd payload to an internal structure for
+ * processing.  As such, upon completion of the routine, the LLDD may
+ * immediately free/reuse the CMD IU buffer passed in the call.
+ *
+ * If this routine returns error, the lldd should abort the exchange.
+ *
+ * @target_port: pointer to the (registered) target port the FCP CMD IU
+ *              was receive on.
+ * @fcpreq:     pointer to a fcpreq request structure to be used to reference
+ *              the exchange corresponding to the FCP Exchange.
+ * @cmdiubuf:   pointer to the buffer containing the FCP CMD IU
+ * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU
+ */
+int
+nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port,
+			struct nvmefc_tgt_fcp_req *fcpreq,
+			void *cmdiubuf, u32 cmdiubuf_len)
+{
+	struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
+	struct nvme_fc_cmd_iu *cmdiu = cmdiubuf;
+	struct nvmet_fc_tgt_queue *queue;
+	struct nvmet_fc_fcp_iod *fod;
+
+	/* validate iu, so the connection id can be used to find the queue */
+	if ((cmdiubuf_len != sizeof(*cmdiu)) ||
+			(cmdiu->scsi_id != NVME_CMD_SCSI_ID) ||
+			(cmdiu->fc_id != NVME_CMD_FC_ID) ||
+			(be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4)))
+		return -EIO;
+
+
+	queue = nvmet_fc_find_target_queue(tgtport,
+				be64_to_cpu(cmdiu->connection_id));
+	if (!queue)
+		return -ENOTCONN;
+
+	/*
+	 * note: reference taken by find_target_queue
+	 * After successful fod allocation, the fod will inherit the
+	 * ownership of that reference and will remove the reference
+	 * when the fod is freed.
+	 */
+
+	fod = nvmet_fc_alloc_fcp_iod(queue);
+	if (!fod) {
+		/* release the queue lookup reference */
+		nvmet_fc_tgt_q_put(queue);
+		return -ENOENT;
+	}
+
+	fcpreq->nvmet_fc_private = fod;
+	fod->fcpreq = fcpreq;
+	/*
+	 * put all admin cmds on hw queue id 0. All io commands go to
+	 * the respective hw queue based on a modulo basis
+	 */
+	fcpreq->hwqid = queue->qid ?
+			((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0;
+	memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len);
+
+	queue_work_on(queue->cpu, queue->work_q, &fod->work);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req);
+
+enum {
+	FCT_TRADDR_ERR		= 0,
+	FCT_TRADDR_WWNN		= 1 << 0,
+	FCT_TRADDR_WWPN		= 1 << 1,
+};
+
+struct nvmet_fc_traddr {
+	u64	nn;
+	u64	pn;
+};
+
+static const match_table_t traddr_opt_tokens = {
+	{ FCT_TRADDR_WWNN,	"nn-%s"		},
+	{ FCT_TRADDR_WWPN,	"pn-%s"		},
+	{ FCT_TRADDR_ERR,	NULL		}
+};
+
+static int
+nvmet_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf)
+{
+	substring_t args[MAX_OPT_ARGS];
+	char *options, *o, *p;
+	int token, ret = 0;
+	u64 token64;
+
+	options = o = kstrdup(buf, GFP_KERNEL);
+	if (!options)
+		return -ENOMEM;
+
+	while ((p = strsep(&o, ",\n")) != NULL) {
+		if (!*p)
+			continue;
+
+		token = match_token(p, traddr_opt_tokens, args);
+		switch (token) {
+		case FCT_TRADDR_WWNN:
+			if (match_u64(args, &token64)) {
+				ret = -EINVAL;
+				goto out;
+			}
+			traddr->nn = token64;
+			break;
+		case FCT_TRADDR_WWPN:
+			if (match_u64(args, &token64)) {
+				ret = -EINVAL;
+				goto out;
+			}
+			traddr->pn = token64;
+			break;
+		default:
+			pr_warn("unknown traddr token or missing value '%s'\n",
+					p);
+			ret = -EINVAL;
+			goto out;
+		}
+	}
+
+out:
+	kfree(options);
+	return ret;
+}
+
+static int
+nvmet_fc_add_port(struct nvmet_port *port)
+{
+	struct nvmet_fc_tgtport *tgtport;
+	struct nvmet_fc_traddr traddr = { 0L, 0L };
+	unsigned long flags;
+	int ret;
+
+	/* validate the address info */
+	if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) ||
+	    (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC))
+		return -EINVAL;
+
+	/* map the traddr address info to a target port */
+
+	ret = nvmet_fc_parse_traddr(&traddr, port->disc_addr.traddr);
+	if (ret)
+		return ret;
+
+	ret = -ENXIO;
+	spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
+	list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) {
+		if ((tgtport->fc_target_port.node_name == traddr.nn) &&
+		    (tgtport->fc_target_port.port_name == traddr.pn)) {
+			/* a FC port can only be 1 nvmet port id */
+			if (!tgtport->port) {
+				tgtport->port = port;
+				port->priv = tgtport;
+				ret = 0;
+			} else
+				ret = -EALREADY;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
+	return ret;
+}
+
+static void
+nvmet_fc_remove_port(struct nvmet_port *port)
+{
+	struct nvmet_fc_tgtport *tgtport = port->priv;
+	unsigned long flags;
+
+	spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
+	if (tgtport->port == port) {
+		nvmet_fc_tgtport_put(tgtport);
+		tgtport->port = NULL;
+	}
+	spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
+}
+
+static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = {
+	.owner			= THIS_MODULE,
+	.type			= NVMF_TRTYPE_FC,
+	.msdbd			= 1,
+	.add_port		= nvmet_fc_add_port,
+	.remove_port		= nvmet_fc_remove_port,
+	.queue_response		= nvmet_fc_fcp_nvme_cmd_done,
+	.delete_ctrl		= nvmet_fc_delete_ctrl,
+};
+
+static int __init nvmet_fc_init_module(void)
+{
+	return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops);
+}
+
+static void __exit nvmet_fc_exit_module(void)
+{
+	/* sanity check - all lports should be removed */
+	if (!list_empty(&nvmet_fc_target_list))
+		pr_warn("%s: targetport list not empty\n", __func__);
+
+	nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops);
+
+	ida_destroy(&nvmet_fc_tgtport_cnt);
+}
+
+module_init(nvmet_fc_init_module);
+module_exit(nvmet_fc_exit_module);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c
new file mode 100644
index 000000000000..bcb8ebeb01c5
--- /dev/null
+++ b/drivers/nvme/target/fcloop.c
@@ -0,0 +1,1148 @@
+/*
+ * Copyright (c) 2016 Avago Technologies.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful.
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
+ * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
+ * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO
+ * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
+ * See the GNU General Public License for more details, a copy of which
+ * can be found in the file COPYING included with this package
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/module.h>
+#include <linux/parser.h>
+#include <uapi/scsi/fc/fc_fs.h>
+
+#include "../host/nvme.h"
+#include "../target/nvmet.h"
+#include <linux/nvme-fc-driver.h>
+#include <linux/nvme-fc.h>
+
+
+enum {
+	NVMF_OPT_ERR		= 0,
+	NVMF_OPT_WWNN		= 1 << 0,
+	NVMF_OPT_WWPN		= 1 << 1,
+	NVMF_OPT_ROLES		= 1 << 2,
+	NVMF_OPT_FCADDR		= 1 << 3,
+	NVMF_OPT_LPWWNN		= 1 << 4,
+	NVMF_OPT_LPWWPN		= 1 << 5,
+};
+
+struct fcloop_ctrl_options {
+	int			mask;
+	u64			wwnn;
+	u64			wwpn;
+	u32			roles;
+	u32			fcaddr;
+	u64			lpwwnn;
+	u64			lpwwpn;
+};
+
+static const match_table_t opt_tokens = {
+	{ NVMF_OPT_WWNN,	"wwnn=%s"	},
+	{ NVMF_OPT_WWPN,	"wwpn=%s"	},
+	{ NVMF_OPT_ROLES,	"roles=%d"	},
+	{ NVMF_OPT_FCADDR,	"fcaddr=%x"	},
+	{ NVMF_OPT_LPWWNN,	"lpwwnn=%s"	},
+	{ NVMF_OPT_LPWWPN,	"lpwwpn=%s"	},
+	{ NVMF_OPT_ERR,		NULL		}
+};
+
+static int
+fcloop_parse_options(struct fcloop_ctrl_options *opts,
+		const char *buf)
+{
+	substring_t args[MAX_OPT_ARGS];
+	char *options, *o, *p;
+	int token, ret = 0;
+	u64 token64;
+
+	options = o = kstrdup(buf, GFP_KERNEL);
+	if (!options)
+		return -ENOMEM;
+
+	while ((p = strsep(&o, ",\n")) != NULL) {
+		if (!*p)
+			continue;
+
+		token = match_token(p, opt_tokens, args);
+		opts->mask |= token;
+		switch (token) {
+		case NVMF_OPT_WWNN:
+			if (match_u64(args, &token64)) {
+				ret = -EINVAL;
+				goto out_free_options;
+			}
+			opts->wwnn = token64;
+			break;
+		case NVMF_OPT_WWPN:
+			if (match_u64(args, &token64)) {
+				ret = -EINVAL;
+				goto out_free_options;
+			}
+			opts->wwpn = token64;
+			break;
+		case NVMF_OPT_ROLES:
+			if (match_int(args, &token)) {
+				ret = -EINVAL;
+				goto out_free_options;
+			}
+			opts->roles = token;
+			break;
+		case NVMF_OPT_FCADDR:
+			if (match_hex(args, &token)) {
+				ret = -EINVAL;
+				goto out_free_options;
+			}
+			opts->fcaddr = token;
+			break;
+		case NVMF_OPT_LPWWNN:
+			if (match_u64(args, &token64)) {
+				ret = -EINVAL;
+				goto out_free_options;
+			}
+			opts->lpwwnn = token64;
+			break;
+		case NVMF_OPT_LPWWPN:
+			if (match_u64(args, &token64)) {
+				ret = -EINVAL;
+				goto out_free_options;
+			}
+			opts->lpwwpn = token64;
+			break;
+		default:
+			pr_warn("unknown parameter or missing value '%s'\n", p);
+			ret = -EINVAL;
+			goto out_free_options;
+		}
+	}
+
+out_free_options:
+	kfree(options);
+	return ret;
+}
+
+
+static int
+fcloop_parse_nm_options(struct device *dev, u64 *nname, u64 *pname,
+		const char *buf)
+{
+	substring_t args[MAX_OPT_ARGS];
+	char *options, *o, *p;
+	int token, ret = 0;
+	u64 token64;
+
+	*nname = -1;
+	*pname = -1;
+
+	options = o = kstrdup(buf, GFP_KERNEL);
+	if (!options)
+		return -ENOMEM;
+
+	while ((p = strsep(&o, ",\n")) != NULL) {
+		if (!*p)
+			continue;
+
+		token = match_token(p, opt_tokens, args);
+		switch (token) {
+		case NVMF_OPT_WWNN:
+			if (match_u64(args, &token64)) {
+				ret = -EINVAL;
+				goto out_free_options;
+			}
+			*nname = token64;
+			break;
+		case NVMF_OPT_WWPN:
+			if (match_u64(args, &token64)) {
+				ret = -EINVAL;
+				goto out_free_options;
+			}
+			*pname = token64;
+			break;
+		default:
+			pr_warn("unknown parameter or missing value '%s'\n", p);
+			ret = -EINVAL;
+			goto out_free_options;
+		}
+	}
+
+out_free_options:
+	kfree(options);
+
+	if (!ret) {
+		if (*nname == -1)
+			return -EINVAL;
+		if (*pname == -1)
+			return -EINVAL;
+	}
+
+	return ret;
+}
+
+
+#define LPORT_OPTS	(NVMF_OPT_WWNN | NVMF_OPT_WWPN)
+
+#define RPORT_OPTS	(NVMF_OPT_WWNN | NVMF_OPT_WWPN |  \
+			 NVMF_OPT_LPWWNN | NVMF_OPT_LPWWPN)
+
+#define TGTPORT_OPTS	(NVMF_OPT_WWNN | NVMF_OPT_WWPN)
+
+#define ALL_OPTS	(NVMF_OPT_WWNN | NVMF_OPT_WWPN | NVMF_OPT_ROLES | \
+			 NVMF_OPT_FCADDR | NVMF_OPT_LPWWNN | NVMF_OPT_LPWWPN)
+
+
+static DEFINE_SPINLOCK(fcloop_lock);
+static LIST_HEAD(fcloop_lports);
+static LIST_HEAD(fcloop_nports);
+
+struct fcloop_lport {
+	struct nvme_fc_local_port *localport;
+	struct list_head lport_list;
+	struct completion unreg_done;
+};
+
+struct fcloop_rport {
+	struct nvme_fc_remote_port *remoteport;
+	struct nvmet_fc_target_port *targetport;
+	struct fcloop_nport *nport;
+	struct fcloop_lport *lport;
+};
+
+struct fcloop_tport {
+	struct nvmet_fc_target_port *targetport;
+	struct nvme_fc_remote_port *remoteport;
+	struct fcloop_nport *nport;
+	struct fcloop_lport *lport;
+};
+
+struct fcloop_nport {
+	struct fcloop_rport *rport;
+	struct fcloop_tport *tport;
+	struct fcloop_lport *lport;
+	struct list_head nport_list;
+	struct kref ref;
+	struct completion rport_unreg_done;
+	struct completion tport_unreg_done;
+	u64 node_name;
+	u64 port_name;
+	u32 port_role;
+	u32 port_id;
+};
+
+struct fcloop_lsreq {
+	struct fcloop_tport		*tport;
+	struct nvmefc_ls_req		*lsreq;
+	struct work_struct		work;
+	struct nvmefc_tgt_ls_req	tgt_ls_req;
+	int				status;
+};
+
+struct fcloop_fcpreq {
+	struct fcloop_tport		*tport;
+	struct nvmefc_fcp_req		*fcpreq;
+	u16				status;
+	struct work_struct		work;
+	struct nvmefc_tgt_fcp_req	tgt_fcp_req;
+};
+
+
+static inline struct fcloop_lsreq *
+tgt_ls_req_to_lsreq(struct nvmefc_tgt_ls_req *tgt_lsreq)
+{
+	return container_of(tgt_lsreq, struct fcloop_lsreq, tgt_ls_req);
+}
+
+static inline struct fcloop_fcpreq *
+tgt_fcp_req_to_fcpreq(struct nvmefc_tgt_fcp_req *tgt_fcpreq)
+{
+	return container_of(tgt_fcpreq, struct fcloop_fcpreq, tgt_fcp_req);
+}
+
+
+static int
+fcloop_create_queue(struct nvme_fc_local_port *localport,
+			unsigned int qidx, u16 qsize,
+			void **handle)
+{
+	*handle = localport;
+	return 0;
+}
+
+static void
+fcloop_delete_queue(struct nvme_fc_local_port *localport,
+			unsigned int idx, void *handle)
+{
+}
+
+
+/*
+ * Transmit of LS RSP done (e.g. buffers all set). call back up
+ * initiator "done" flows.
+ */
+static void
+fcloop_tgt_lsrqst_done_work(struct work_struct *work)
+{
+	struct fcloop_lsreq *tls_req =
+		container_of(work, struct fcloop_lsreq, work);
+	struct fcloop_tport *tport = tls_req->tport;
+	struct nvmefc_ls_req *lsreq = tls_req->lsreq;
+
+	if (tport->remoteport)
+		lsreq->done(lsreq, tls_req->status);
+}
+
+static int
+fcloop_ls_req(struct nvme_fc_local_port *localport,
+			struct nvme_fc_remote_port *remoteport,
+			struct nvmefc_ls_req *lsreq)
+{
+	struct fcloop_lsreq *tls_req = lsreq->private;
+	struct fcloop_rport *rport = remoteport->private;
+	int ret = 0;
+
+	tls_req->lsreq = lsreq;
+	INIT_WORK(&tls_req->work, fcloop_tgt_lsrqst_done_work);
+
+	if (!rport->targetport) {
+		tls_req->status = -ECONNREFUSED;
+		schedule_work(&tls_req->work);
+		return ret;
+	}
+
+	tls_req->status = 0;
+	tls_req->tport = rport->targetport->private;
+	ret = nvmet_fc_rcv_ls_req(rport->targetport, &tls_req->tgt_ls_req,
+				 lsreq->rqstaddr, lsreq->rqstlen);
+
+	return ret;
+}
+
+static int
+fcloop_xmt_ls_rsp(struct nvmet_fc_target_port *tport,
+			struct nvmefc_tgt_ls_req *tgt_lsreq)
+{
+	struct fcloop_lsreq *tls_req = tgt_ls_req_to_lsreq(tgt_lsreq);
+	struct nvmefc_ls_req *lsreq = tls_req->lsreq;
+
+	memcpy(lsreq->rspaddr, tgt_lsreq->rspbuf,
+		((lsreq->rsplen < tgt_lsreq->rsplen) ?
+				lsreq->rsplen : tgt_lsreq->rsplen));
+	tgt_lsreq->done(tgt_lsreq);
+
+	schedule_work(&tls_req->work);
+
+	return 0;
+}
+
+/*
+ * FCP IO operation done. call back up initiator "done" flows.
+ */
+static void
+fcloop_tgt_fcprqst_done_work(struct work_struct *work)
+{
+	struct fcloop_fcpreq *tfcp_req =
+		container_of(work, struct fcloop_fcpreq, work);
+	struct fcloop_tport *tport = tfcp_req->tport;
+	struct nvmefc_fcp_req *fcpreq = tfcp_req->fcpreq;
+
+	if (tport->remoteport) {
+		fcpreq->status = tfcp_req->status;
+		fcpreq->done(fcpreq);
+	}
+}
+
+
+static int
+fcloop_fcp_req(struct nvme_fc_local_port *localport,
+			struct nvme_fc_remote_port *remoteport,
+			void *hw_queue_handle,
+			struct nvmefc_fcp_req *fcpreq)
+{
+	struct fcloop_fcpreq *tfcp_req = fcpreq->private;
+	struct fcloop_rport *rport = remoteport->private;
+	int ret = 0;
+
+	INIT_WORK(&tfcp_req->work, fcloop_tgt_fcprqst_done_work);
+
+	if (!rport->targetport) {
+		tfcp_req->status = NVME_SC_FC_TRANSPORT_ERROR;
+		schedule_work(&tfcp_req->work);
+		return ret;
+	}
+
+	tfcp_req->fcpreq = fcpreq;
+	tfcp_req->tport = rport->targetport->private;
+
+	ret = nvmet_fc_rcv_fcp_req(rport->targetport, &tfcp_req->tgt_fcp_req,
+				 fcpreq->cmdaddr, fcpreq->cmdlen);
+
+	return ret;
+}
+
+static void
+fcloop_fcp_copy_data(u8 op, struct scatterlist *data_sg,
+			struct scatterlist *io_sg, u32 offset, u32 length)
+{
+	void *data_p, *io_p;
+	u32 data_len, io_len, tlen;
+
+	io_p = sg_virt(io_sg);
+	io_len = io_sg->length;
+
+	for ( ; offset; ) {
+		tlen = min_t(u32, offset, io_len);
+		offset -= tlen;
+		io_len -= tlen;
+		if (!io_len) {
+			io_sg = sg_next(io_sg);
+			io_p = sg_virt(io_sg);
+			io_len = io_sg->length;
+		} else
+			io_p += tlen;
+	}
+
+	data_p = sg_virt(data_sg);
+	data_len = data_sg->length;
+
+	for ( ; length; ) {
+		tlen = min_t(u32, io_len, data_len);
+		tlen = min_t(u32, tlen, length);
+
+		if (op == NVMET_FCOP_WRITEDATA)
+			memcpy(data_p, io_p, tlen);
+		else
+			memcpy(io_p, data_p, tlen);
+
+		length -= tlen;
+
+		io_len -= tlen;
+		if ((!io_len) && (length)) {
+			io_sg = sg_next(io_sg);
+			io_p = sg_virt(io_sg);
+			io_len = io_sg->length;
+		} else
+			io_p += tlen;
+
+		data_len -= tlen;
+		if ((!data_len) && (length)) {
+			data_sg = sg_next(data_sg);
+			data_p = sg_virt(data_sg);
+			data_len = data_sg->length;
+		} else
+			data_p += tlen;
+	}
+}
+
+static int
+fcloop_fcp_op(struct nvmet_fc_target_port *tgtport,
+			struct nvmefc_tgt_fcp_req *tgt_fcpreq)
+{
+	struct fcloop_fcpreq *tfcp_req = tgt_fcp_req_to_fcpreq(tgt_fcpreq);
+	struct nvmefc_fcp_req *fcpreq = tfcp_req->fcpreq;
+	u32 rsplen = 0, xfrlen = 0;
+	int fcp_err = 0;
+	u8 op = tgt_fcpreq->op;
+
+	switch (op) {
+	case NVMET_FCOP_WRITEDATA:
+		xfrlen = tgt_fcpreq->transfer_length;
+		fcloop_fcp_copy_data(op, tgt_fcpreq->sg, fcpreq->first_sgl,
+					tgt_fcpreq->offset, xfrlen);
+		fcpreq->transferred_length += xfrlen;
+		break;
+
+	case NVMET_FCOP_READDATA:
+	case NVMET_FCOP_READDATA_RSP:
+		xfrlen = tgt_fcpreq->transfer_length;
+		fcloop_fcp_copy_data(op, tgt_fcpreq->sg, fcpreq->first_sgl,
+					tgt_fcpreq->offset, xfrlen);
+		fcpreq->transferred_length += xfrlen;
+		if (op == NVMET_FCOP_READDATA)
+			break;
+
+		/* Fall-Thru to RSP handling */
+
+	case NVMET_FCOP_RSP:
+		rsplen = ((fcpreq->rsplen < tgt_fcpreq->rsplen) ?
+				fcpreq->rsplen : tgt_fcpreq->rsplen);
+		memcpy(fcpreq->rspaddr, tgt_fcpreq->rspaddr, rsplen);
+		if (rsplen < tgt_fcpreq->rsplen)
+			fcp_err = -E2BIG;
+		fcpreq->rcv_rsplen = rsplen;
+		fcpreq->status = 0;
+		tfcp_req->status = 0;
+		break;
+
+	case NVMET_FCOP_ABORT:
+		tfcp_req->status = NVME_SC_FC_TRANSPORT_ABORTED;
+		break;
+
+	default:
+		fcp_err = -EINVAL;
+		break;
+	}
+
+	tgt_fcpreq->transferred_length = xfrlen;
+	tgt_fcpreq->fcp_error = fcp_err;
+	tgt_fcpreq->done(tgt_fcpreq);
+
+	if ((!fcp_err) && (op == NVMET_FCOP_RSP ||
+			op == NVMET_FCOP_READDATA_RSP ||
+			op == NVMET_FCOP_ABORT))
+		schedule_work(&tfcp_req->work);
+
+	return 0;
+}
+
+static void
+fcloop_ls_abort(struct nvme_fc_local_port *localport,
+			struct nvme_fc_remote_port *remoteport,
+				struct nvmefc_ls_req *lsreq)
+{
+}
+
+static void
+fcloop_fcp_abort(struct nvme_fc_local_port *localport,
+			struct nvme_fc_remote_port *remoteport,
+			void *hw_queue_handle,
+			struct nvmefc_fcp_req *fcpreq)
+{
+}
+
+static void
+fcloop_localport_delete(struct nvme_fc_local_port *localport)
+{
+	struct fcloop_lport *lport = localport->private;
+
+	/* release any threads waiting for the unreg to complete */
+	complete(&lport->unreg_done);
+}
+
+static void
+fcloop_remoteport_delete(struct nvme_fc_remote_port *remoteport)
+{
+	struct fcloop_rport *rport = remoteport->private;
+
+	/* release any threads waiting for the unreg to complete */
+	complete(&rport->nport->rport_unreg_done);
+}
+
+static void
+fcloop_targetport_delete(struct nvmet_fc_target_port *targetport)
+{
+	struct fcloop_tport *tport = targetport->private;
+
+	/* release any threads waiting for the unreg to complete */
+	complete(&tport->nport->tport_unreg_done);
+}
+
+#define	FCLOOP_HW_QUEUES		4
+#define	FCLOOP_SGL_SEGS			256
+#define FCLOOP_DMABOUND_4G		0xFFFFFFFF
+
+struct nvme_fc_port_template fctemplate = {
+	.localport_delete	= fcloop_localport_delete,
+	.remoteport_delete	= fcloop_remoteport_delete,
+	.create_queue		= fcloop_create_queue,
+	.delete_queue		= fcloop_delete_queue,
+	.ls_req			= fcloop_ls_req,
+	.fcp_io			= fcloop_fcp_req,
+	.ls_abort		= fcloop_ls_abort,
+	.fcp_abort		= fcloop_fcp_abort,
+	.max_hw_queues		= FCLOOP_HW_QUEUES,
+	.max_sgl_segments	= FCLOOP_SGL_SEGS,
+	.max_dif_sgl_segments	= FCLOOP_SGL_SEGS,
+	.dma_boundary		= FCLOOP_DMABOUND_4G,
+	/* sizes of additional private data for data structures */
+	.local_priv_sz		= sizeof(struct fcloop_lport),
+	.remote_priv_sz		= sizeof(struct fcloop_rport),
+	.lsrqst_priv_sz		= sizeof(struct fcloop_lsreq),
+	.fcprqst_priv_sz	= sizeof(struct fcloop_fcpreq),
+};
+
+struct nvmet_fc_target_template tgttemplate = {
+	.targetport_delete	= fcloop_targetport_delete,
+	.xmt_ls_rsp		= fcloop_xmt_ls_rsp,
+	.fcp_op			= fcloop_fcp_op,
+	.max_hw_queues		= FCLOOP_HW_QUEUES,
+	.max_sgl_segments	= FCLOOP_SGL_SEGS,
+	.max_dif_sgl_segments	= FCLOOP_SGL_SEGS,
+	.dma_boundary		= FCLOOP_DMABOUND_4G,
+	/* optional features */
+	.target_features	= NVMET_FCTGTFEAT_READDATA_RSP |
+				  NVMET_FCTGTFEAT_NEEDS_CMD_CPUSCHED,
+	/* sizes of additional private data for data structures */
+	.target_priv_sz		= sizeof(struct fcloop_tport),
+};
+
+static ssize_t
+fcloop_create_local_port(struct device *dev, struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct nvme_fc_port_info pinfo;
+	struct fcloop_ctrl_options *opts;
+	struct nvme_fc_local_port *localport;
+	struct fcloop_lport *lport;
+	int ret;
+
+	opts = kzalloc(sizeof(*opts), GFP_KERNEL);
+	if (!opts)
+		return -ENOMEM;
+
+	ret = fcloop_parse_options(opts, buf);
+	if (ret)
+		goto out_free_opts;
+
+	/* everything there ? */
+	if ((opts->mask & LPORT_OPTS) != LPORT_OPTS) {
+		ret = -EINVAL;
+		goto out_free_opts;
+	}
+
+	pinfo.node_name = opts->wwnn;
+	pinfo.port_name = opts->wwpn;
+	pinfo.port_role = opts->roles;
+	pinfo.port_id = opts->fcaddr;
+
+	ret = nvme_fc_register_localport(&pinfo, &fctemplate, NULL, &localport);
+	if (!ret) {
+		unsigned long flags;
+
+		/* success */
+		lport = localport->private;
+		lport->localport = localport;
+		INIT_LIST_HEAD(&lport->lport_list);
+
+		spin_lock_irqsave(&fcloop_lock, flags);
+		list_add_tail(&lport->lport_list, &fcloop_lports);
+		spin_unlock_irqrestore(&fcloop_lock, flags);
+
+		/* mark all of the input buffer consumed */
+		ret = count;
+	}
+
+out_free_opts:
+	kfree(opts);
+	return ret ? ret : count;
+}
+
+
+static void
+__unlink_local_port(struct fcloop_lport *lport)
+{
+	list_del(&lport->lport_list);
+}
+
+static int
+__wait_localport_unreg(struct fcloop_lport *lport)
+{
+	int ret;
+
+	init_completion(&lport->unreg_done);
+
+	ret = nvme_fc_unregister_localport(lport->localport);
+
+	wait_for_completion(&lport->unreg_done);
+
+	return ret;
+}
+
+
+static ssize_t
+fcloop_delete_local_port(struct device *dev, struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct fcloop_lport *tlport, *lport = NULL;
+	u64 nodename, portname;
+	unsigned long flags;
+	int ret;
+
+	ret = fcloop_parse_nm_options(dev, &nodename, &portname, buf);
+	if (ret)
+		return ret;
+
+	spin_lock_irqsave(&fcloop_lock, flags);
+
+	list_for_each_entry(tlport, &fcloop_lports, lport_list) {
+		if (tlport->localport->node_name == nodename &&
+		    tlport->localport->port_name == portname) {
+			lport = tlport;
+			__unlink_local_port(lport);
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&fcloop_lock, flags);
+
+	if (!lport)
+		return -ENOENT;
+
+	ret = __wait_localport_unreg(lport);
+
+	return ret ? ret : count;
+}
+
+static void
+fcloop_nport_free(struct kref *ref)
+{
+	struct fcloop_nport *nport =
+		container_of(ref, struct fcloop_nport, ref);
+	unsigned long flags;
+
+	spin_lock_irqsave(&fcloop_lock, flags);
+	list_del(&nport->nport_list);
+	spin_unlock_irqrestore(&fcloop_lock, flags);
+
+	kfree(nport);
+}
+
+static void
+fcloop_nport_put(struct fcloop_nport *nport)
+{
+	kref_put(&nport->ref, fcloop_nport_free);
+}
+
+static int
+fcloop_nport_get(struct fcloop_nport *nport)
+{
+	return kref_get_unless_zero(&nport->ref);
+}
+
+static struct fcloop_nport *
+fcloop_alloc_nport(const char *buf, size_t count, bool remoteport)
+{
+	struct fcloop_nport *newnport, *nport = NULL;
+	struct fcloop_lport *tmplport, *lport = NULL;
+	struct fcloop_ctrl_options *opts;
+	unsigned long flags;
+	u32 opts_mask = (remoteport) ? RPORT_OPTS : TGTPORT_OPTS;
+	int ret;
+
+	opts = kzalloc(sizeof(*opts), GFP_KERNEL);
+	if (!opts)
+		return NULL;
+
+	ret = fcloop_parse_options(opts, buf);
+	if (ret)
+		goto out_free_opts;
+
+	/* everything there ? */
+	if ((opts->mask & opts_mask) != opts_mask) {
+		ret = -EINVAL;
+		goto out_free_opts;
+	}
+
+	newnport = kzalloc(sizeof(*newnport), GFP_KERNEL);
+	if (!newnport)
+		goto out_free_opts;
+
+	INIT_LIST_HEAD(&newnport->nport_list);
+	newnport->node_name = opts->wwnn;
+	newnport->port_name = opts->wwpn;
+	if (opts->mask & NVMF_OPT_ROLES)
+		newnport->port_role = opts->roles;
+	if (opts->mask & NVMF_OPT_FCADDR)
+		newnport->port_id = opts->fcaddr;
+	kref_init(&newnport->ref);
+
+	spin_lock_irqsave(&fcloop_lock, flags);
+
+	list_for_each_entry(tmplport, &fcloop_lports, lport_list) {
+		if (tmplport->localport->node_name == opts->wwnn &&
+		    tmplport->localport->port_name == opts->wwpn)
+			goto out_invalid_opts;
+
+		if (tmplport->localport->node_name == opts->lpwwnn &&
+		    tmplport->localport->port_name == opts->lpwwpn)
+			lport = tmplport;
+	}
+
+	if (remoteport) {
+		if (!lport)
+			goto out_invalid_opts;
+		newnport->lport = lport;
+	}
+
+	list_for_each_entry(nport, &fcloop_nports, nport_list) {
+		if (nport->node_name == opts->wwnn &&
+		    nport->port_name == opts->wwpn) {
+			if ((remoteport && nport->rport) ||
+			    (!remoteport && nport->tport)) {
+				nport = NULL;
+				goto out_invalid_opts;
+			}
+
+			fcloop_nport_get(nport);
+
+			spin_unlock_irqrestore(&fcloop_lock, flags);
+
+			if (remoteport)
+				nport->lport = lport;
+			if (opts->mask & NVMF_OPT_ROLES)
+				nport->port_role = opts->roles;
+			if (opts->mask & NVMF_OPT_FCADDR)
+				nport->port_id = opts->fcaddr;
+			goto out_free_newnport;
+		}
+	}
+
+	list_add_tail(&newnport->nport_list, &fcloop_nports);
+
+	spin_unlock_irqrestore(&fcloop_lock, flags);
+
+	kfree(opts);
+	return newnport;
+
+out_invalid_opts:
+	spin_unlock_irqrestore(&fcloop_lock, flags);
+out_free_newnport:
+	kfree(newnport);
+out_free_opts:
+	kfree(opts);
+	return nport;
+}
+
+static ssize_t
+fcloop_create_remote_port(struct device *dev, struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct nvme_fc_remote_port *remoteport;
+	struct fcloop_nport *nport;
+	struct fcloop_rport *rport;
+	struct nvme_fc_port_info pinfo;
+	int ret;
+
+	nport = fcloop_alloc_nport(buf, count, true);
+	if (!nport)
+		return -EIO;
+
+	pinfo.node_name = nport->node_name;
+	pinfo.port_name = nport->port_name;
+	pinfo.port_role = nport->port_role;
+	pinfo.port_id = nport->port_id;
+
+	ret = nvme_fc_register_remoteport(nport->lport->localport,
+						&pinfo, &remoteport);
+	if (ret || !remoteport) {
+		fcloop_nport_put(nport);
+		return ret;
+	}
+
+	/* success */
+	rport = remoteport->private;
+	rport->remoteport = remoteport;
+	rport->targetport = (nport->tport) ?  nport->tport->targetport : NULL;
+	if (nport->tport) {
+		nport->tport->remoteport = remoteport;
+		nport->tport->lport = nport->lport;
+	}
+	rport->nport = nport;
+	rport->lport = nport->lport;
+	nport->rport = rport;
+
+	return ret ? ret : count;
+}
+
+
+static struct fcloop_rport *
+__unlink_remote_port(struct fcloop_nport *nport)
+{
+	struct fcloop_rport *rport = nport->rport;
+
+	if (rport && nport->tport)
+		nport->tport->remoteport = NULL;
+	nport->rport = NULL;
+
+	return rport;
+}
+
+static int
+__wait_remoteport_unreg(struct fcloop_nport *nport, struct fcloop_rport *rport)
+{
+	int ret;
+
+	if (!rport)
+		return -EALREADY;
+
+	init_completion(&nport->rport_unreg_done);
+
+	ret = nvme_fc_unregister_remoteport(rport->remoteport);
+	if (ret)
+		return ret;
+
+	wait_for_completion(&nport->rport_unreg_done);
+
+	fcloop_nport_put(nport);
+
+	return ret;
+}
+
+static ssize_t
+fcloop_delete_remote_port(struct device *dev, struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct fcloop_nport *nport = NULL, *tmpport;
+	static struct fcloop_rport *rport;
+	u64 nodename, portname;
+	unsigned long flags;
+	int ret;
+
+	ret = fcloop_parse_nm_options(dev, &nodename, &portname, buf);
+	if (ret)
+		return ret;
+
+	spin_lock_irqsave(&fcloop_lock, flags);
+
+	list_for_each_entry(tmpport, &fcloop_nports, nport_list) {
+		if (tmpport->node_name == nodename &&
+		    tmpport->port_name == portname && tmpport->rport) {
+			nport = tmpport;
+			rport = __unlink_remote_port(nport);
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&fcloop_lock, flags);
+
+	if (!nport)
+		return -ENOENT;
+
+	ret = __wait_remoteport_unreg(nport, rport);
+
+	return ret ? ret : count;
+}
+
+static ssize_t
+fcloop_create_target_port(struct device *dev, struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct nvmet_fc_target_port *targetport;
+	struct fcloop_nport *nport;
+	struct fcloop_tport *tport;
+	struct nvmet_fc_port_info tinfo;
+	int ret;
+
+	nport = fcloop_alloc_nport(buf, count, false);
+	if (!nport)
+		return -EIO;
+
+	tinfo.node_name = nport->node_name;
+	tinfo.port_name = nport->port_name;
+	tinfo.port_id = nport->port_id;
+
+	ret = nvmet_fc_register_targetport(&tinfo, &tgttemplate, NULL,
+						&targetport);
+	if (ret) {
+		fcloop_nport_put(nport);
+		return ret;
+	}
+
+	/* success */
+	tport = targetport->private;
+	tport->targetport = targetport;
+	tport->remoteport = (nport->rport) ?  nport->rport->remoteport : NULL;
+	if (nport->rport)
+		nport->rport->targetport = targetport;
+	tport->nport = nport;
+	tport->lport = nport->lport;
+	nport->tport = tport;
+
+	return ret ? ret : count;
+}
+
+
+static struct fcloop_tport *
+__unlink_target_port(struct fcloop_nport *nport)
+{
+	struct fcloop_tport *tport = nport->tport;
+
+	if (tport && nport->rport)
+		nport->rport->targetport = NULL;
+	nport->tport = NULL;
+
+	return tport;
+}
+
+static int
+__wait_targetport_unreg(struct fcloop_nport *nport, struct fcloop_tport *tport)
+{
+	int ret;
+
+	if (!tport)
+		return -EALREADY;
+
+	init_completion(&nport->tport_unreg_done);
+
+	ret = nvmet_fc_unregister_targetport(tport->targetport);
+	if (ret)
+		return ret;
+
+	wait_for_completion(&nport->tport_unreg_done);
+
+	fcloop_nport_put(nport);
+
+	return ret;
+}
+
+static ssize_t
+fcloop_delete_target_port(struct device *dev, struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct fcloop_nport *nport = NULL, *tmpport;
+	struct fcloop_tport *tport;
+	u64 nodename, portname;
+	unsigned long flags;
+	int ret;
+
+	ret = fcloop_parse_nm_options(dev, &nodename, &portname, buf);
+	if (ret)
+		return ret;
+
+	spin_lock_irqsave(&fcloop_lock, flags);
+
+	list_for_each_entry(tmpport, &fcloop_nports, nport_list) {
+		if (tmpport->node_name == nodename &&
+		    tmpport->port_name == portname && tmpport->tport) {
+			nport = tmpport;
+			tport = __unlink_target_port(nport);
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&fcloop_lock, flags);
+
+	if (!nport)
+		return -ENOENT;
+
+	ret = __wait_targetport_unreg(nport, tport);
+
+	return ret ? ret : count;
+}
+
+
+static DEVICE_ATTR(add_local_port, 0200, NULL, fcloop_create_local_port);
+static DEVICE_ATTR(del_local_port, 0200, NULL, fcloop_delete_local_port);
+static DEVICE_ATTR(add_remote_port, 0200, NULL, fcloop_create_remote_port);
+static DEVICE_ATTR(del_remote_port, 0200, NULL, fcloop_delete_remote_port);
+static DEVICE_ATTR(add_target_port, 0200, NULL, fcloop_create_target_port);
+static DEVICE_ATTR(del_target_port, 0200, NULL, fcloop_delete_target_port);
+
+static struct attribute *fcloop_dev_attrs[] = {
+	&dev_attr_add_local_port.attr,
+	&dev_attr_del_local_port.attr,
+	&dev_attr_add_remote_port.attr,
+	&dev_attr_del_remote_port.attr,
+	&dev_attr_add_target_port.attr,
+	&dev_attr_del_target_port.attr,
+	NULL
+};
+
+static struct attribute_group fclopp_dev_attrs_group = {
+	.attrs		= fcloop_dev_attrs,
+};
+
+static const struct attribute_group *fcloop_dev_attr_groups[] = {
+	&fclopp_dev_attrs_group,
+	NULL,
+};
+
+static struct class *fcloop_class;
+static struct device *fcloop_device;
+
+
+static int __init fcloop_init(void)
+{
+	int ret;
+
+	fcloop_class = class_create(THIS_MODULE, "fcloop");
+	if (IS_ERR(fcloop_class)) {
+		pr_err("couldn't register class fcloop\n");
+		ret = PTR_ERR(fcloop_class);
+		return ret;
+	}
+
+	fcloop_device = device_create_with_groups(
+				fcloop_class, NULL, MKDEV(0, 0), NULL,
+				fcloop_dev_attr_groups, "ctl");
+	if (IS_ERR(fcloop_device)) {
+		pr_err("couldn't create ctl device!\n");
+		ret = PTR_ERR(fcloop_device);
+		goto out_destroy_class;
+	}
+
+	get_device(fcloop_device);
+
+	return 0;
+
+out_destroy_class:
+	class_destroy(fcloop_class);
+	return ret;
+}
+
+static void __exit fcloop_exit(void)
+{
+	struct fcloop_lport *lport;
+	struct fcloop_nport *nport;
+	struct fcloop_tport *tport;
+	struct fcloop_rport *rport;
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&fcloop_lock, flags);
+
+	for (;;) {
+		nport = list_first_entry_or_null(&fcloop_nports,
+						typeof(*nport), nport_list);
+		if (!nport)
+			break;
+
+		tport = __unlink_target_port(nport);
+		rport = __unlink_remote_port(nport);
+
+		spin_unlock_irqrestore(&fcloop_lock, flags);
+
+		ret = __wait_targetport_unreg(nport, tport);
+		if (ret)
+			pr_warn("%s: Failed deleting target port\n", __func__);
+
+		ret = __wait_remoteport_unreg(nport, rport);
+		if (ret)
+			pr_warn("%s: Failed deleting remote port\n", __func__);
+
+		spin_lock_irqsave(&fcloop_lock, flags);
+	}
+
+	for (;;) {
+		lport = list_first_entry_or_null(&fcloop_lports,
+						typeof(*lport), lport_list);
+		if (!lport)
+			break;
+
+		__unlink_local_port(lport);
+
+		spin_unlock_irqrestore(&fcloop_lock, flags);
+
+		ret = __wait_localport_unreg(lport);
+		if (ret)
+			pr_warn("%s: Failed deleting local port\n", __func__);
+
+		spin_lock_irqsave(&fcloop_lock, flags);
+	}
+
+	spin_unlock_irqrestore(&fcloop_lock, flags);
+
+	put_device(fcloop_device);
+
+	device_destroy(fcloop_class, MKDEV(0, 0));
+	class_destroy(fcloop_class);
+}
+
+module_init(fcloop_init);
+module_exit(fcloop_exit);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/nvme/target/io-cmd.c b/drivers/nvme/target/io-cmd.c
index 4a96c2049b7b..4195115c7e54 100644
--- a/drivers/nvme/target/io-cmd.c
+++ b/drivers/nvme/target/io-cmd.c
@@ -37,9 +37,7 @@ static void nvmet_inline_bio_init(struct nvmet_req *req)
 {
 	struct bio *bio = &req->inline_bio;
 
-	bio_init(bio);
-	bio->bi_max_vecs = NVMET_MAX_INLINE_BIOVEC;
-	bio->bi_io_vec = req->inline_bvec;
+	bio_init(bio, req->inline_bvec, NVMET_MAX_INLINE_BIOVEC);
 }
 
 static void nvmet_execute_rw(struct nvmet_req *req)
@@ -58,7 +56,7 @@ static void nvmet_execute_rw(struct nvmet_req *req)
 
 	if (req->cmd->rw.opcode == nvme_cmd_write) {
 		op = REQ_OP_WRITE;
-		op_flags = WRITE_ODIRECT;
+		op_flags = REQ_SYNC | REQ_IDLE;
 		if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
 			op_flags |= REQ_FUA;
 	} else {
@@ -96,7 +94,7 @@ static void nvmet_execute_rw(struct nvmet_req *req)
 
 	cookie = submit_bio(bio);
 
-	blk_poll(bdev_get_queue(req->ns->bdev), cookie);
+	blk_mq_poll(bdev_get_queue(req->ns->bdev), cookie);
 }
 
 static void nvmet_execute_flush(struct nvmet_req *req)
@@ -109,7 +107,7 @@ static void nvmet_execute_flush(struct nvmet_req *req)
 	bio->bi_bdev = req->ns->bdev;
 	bio->bi_private = req;
 	bio->bi_end_io = nvmet_bio_done;
-	bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_FLUSH);
+	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 
 	submit_bio(bio);
 }
@@ -172,6 +170,32 @@ static void nvmet_execute_dsm(struct nvmet_req *req)
 	}
 }
 
+static void nvmet_execute_write_zeroes(struct nvmet_req *req)
+{
+	struct nvme_write_zeroes_cmd *write_zeroes = &req->cmd->write_zeroes;
+	struct bio *bio = NULL;
+	u16 status = NVME_SC_SUCCESS;
+	sector_t sector;
+	sector_t nr_sector;
+
+	sector = le64_to_cpu(write_zeroes->slba) <<
+		(req->ns->blksize_shift - 9);
+	nr_sector = (((sector_t)le32_to_cpu(write_zeroes->length)) <<
+		(req->ns->blksize_shift - 9)) + 1;
+
+	if (__blkdev_issue_zeroout(req->ns->bdev, sector, nr_sector,
+				GFP_KERNEL, &bio, true))
+		status = NVME_SC_INTERNAL | NVME_SC_DNR;
+
+	if (bio) {
+		bio->bi_private = req;
+		bio->bi_end_io = nvmet_bio_done;
+		submit_bio(bio);
+	} else {
+		nvmet_req_complete(req, status);
+	}
+}
+
 int nvmet_parse_io_cmd(struct nvmet_req *req)
 {
 	struct nvme_command *cmd = req->cmd;
@@ -209,6 +233,9 @@ int nvmet_parse_io_cmd(struct nvmet_req *req)
 		req->data_len = le32_to_cpu(cmd->dsm.nr + 1) *
 			sizeof(struct nvme_dsm_range);
 		return 0;
+	case nvme_cmd_write_zeroes:
+		req->execute = nvmet_execute_write_zeroes;
+		return 0;
 	default:
 		pr_err("nvmet: unhandled cmd %d\n", cmd->common.opcode);
 		return NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c
index d5df77d686b2..9aaa70071ae5 100644
--- a/drivers/nvme/target/loop.c
+++ b/drivers/nvme/target/loop.c
@@ -36,6 +36,7 @@
 	(NVME_LOOP_AQ_DEPTH - NVME_LOOP_NR_AEN_COMMANDS)
 
 struct nvme_loop_iod {
+	struct nvme_request	nvme_req;
 	struct nvme_command	cmd;
 	struct nvme_completion	rsp;
 	struct nvmet_req	req;
@@ -112,10 +113,10 @@ static void nvme_loop_complete_rq(struct request *req)
 	blk_mq_end_request(req, error);
 }
 
-static void nvme_loop_queue_response(struct nvmet_req *nvme_req)
+static void nvme_loop_queue_response(struct nvmet_req *req)
 {
 	struct nvme_loop_iod *iod =
-		container_of(nvme_req, struct nvme_loop_iod, req);
+		container_of(req, struct nvme_loop_iod, req);
 	struct nvme_completion *cqe = &iod->rsp;
 
 	/*
@@ -126,13 +127,13 @@ static void nvme_loop_queue_response(struct nvmet_req *nvme_req)
 	 */
 	if (unlikely(nvme_loop_queue_idx(iod->queue) == 0 &&
 			cqe->command_id >= NVME_LOOP_AQ_BLKMQ_DEPTH)) {
-		nvme_complete_async_event(&iod->queue->ctrl->ctrl, cqe);
+		nvme_complete_async_event(&iod->queue->ctrl->ctrl, cqe->status,
+				&cqe->result);
 	} else {
-		struct request *req = blk_mq_rq_from_pdu(iod);
+		struct request *rq = blk_mq_rq_from_pdu(iod);
 
-		if (req->cmd_type == REQ_TYPE_DRV_PRIV && req->special)
-			memcpy(req->special, cqe, sizeof(*cqe));
-		blk_mq_complete_request(req, le16_to_cpu(cqe->status) >> 1);
+		iod->nvme_req.result = cqe->result;
+		blk_mq_complete_request(rq, le16_to_cpu(cqe->status) >> 1);
 	}
 }
 
@@ -168,7 +169,7 @@ static int nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
 	int ret;
 
 	ret = nvme_setup_cmd(ns, req, &iod->cmd);
-	if (ret)
+	if (ret != BLK_MQ_RQ_QUEUE_OK)
 		return ret;
 
 	iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
@@ -178,26 +179,25 @@ static int nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
 		nvme_cleanup_cmd(req);
 		blk_mq_start_request(req);
 		nvme_loop_queue_response(&iod->req);
-		return 0;
+		return BLK_MQ_RQ_QUEUE_OK;
 	}
 
 	if (blk_rq_bytes(req)) {
 		iod->sg_table.sgl = iod->first_sgl;
 		ret = sg_alloc_table_chained(&iod->sg_table,
-			req->nr_phys_segments, iod->sg_table.sgl);
+				blk_rq_nr_phys_segments(req),
+				iod->sg_table.sgl);
 		if (ret)
 			return BLK_MQ_RQ_QUEUE_BUSY;
 
 		iod->req.sg = iod->sg_table.sgl;
 		iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl);
-		BUG_ON(iod->req.sg_cnt > req->nr_phys_segments);
 	}
 
-	iod->cmd.common.command_id = req->tag;
 	blk_mq_start_request(req);
 
 	schedule_work(&iod->work);
-	return 0;
+	return BLK_MQ_RQ_QUEUE_OK;
 }
 
 static void nvme_loop_submit_async_event(struct nvme_ctrl *arg, int aer_idx)
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 76b6eedccaf9..23d5eb1c944f 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -47,6 +47,7 @@ struct nvmet_ns {
 	loff_t			size;
 	u8			nguid[16];
 
+	bool			enabled;
 	struct nvmet_subsys	*subsys;
 	const char		*device_path;
 
@@ -61,11 +62,6 @@ static inline struct nvmet_ns *to_nvmet_ns(struct config_item *item)
 	return container_of(to_config_group(item), struct nvmet_ns, group);
 }
 
-static inline bool nvmet_ns_enabled(struct nvmet_ns *ns)
-{
-	return !list_empty_careful(&ns->dev_link);
-}
-
 struct nvmet_cq {
 	u16			qid;
 	u16			size;
@@ -238,7 +234,7 @@ static inline void nvmet_set_status(struct nvmet_req *req, u16 status)
 
 static inline void nvmet_set_result(struct nvmet_req *req, u32 result)
 {
-	req->rsp->result = cpu_to_le32(result);
+	req->rsp->result.u32 = cpu_to_le32(result);
 }
 
 /*
diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c
index 005ef5d17a19..8c3760a78ac0 100644
--- a/drivers/nvme/target/rdma.c
+++ b/drivers/nvme/target/rdma.c
@@ -1045,8 +1045,10 @@ nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev,
 	}
 
 	ret = nvmet_sq_init(&queue->nvme_sq);
-	if (ret)
+	if (ret) {
+		ret = NVME_RDMA_CM_NO_RSC;
 		goto out_free_queue;
+	}
 
 	ret = nvmet_rdma_parse_cm_connect_req(&event->param.conn, queue);
 	if (ret)
@@ -1116,6 +1118,7 @@ out_destroy_sq:
 out_free_queue:
 	kfree(queue);
 out_reject:
+	pr_debug("rejecting connect request with status code %d\n", ret);
 	nvmet_rdma_cm_reject(cm_id, ret);
 	return NULL;
 }
@@ -1129,7 +1132,8 @@ static void nvmet_rdma_qp_event(struct ib_event *event, void *priv)
 		rdma_notify(queue->cm_id, event->event);
 		break;
 	default:
-		pr_err("received unrecognized IB QP event %d\n", event->event);
+		pr_err("received IB QP event: %s (%d)\n",
+		       ib_event_msg(event->event), event->event);
 		break;
 	}
 }
@@ -1370,6 +1374,9 @@ static int nvmet_rdma_cm_handler(struct rdma_cm_id *cm_id,
 		ret = nvmet_rdma_device_removal(cm_id, queue);
 		break;
 	case RDMA_CM_EVENT_REJECTED:
+		pr_debug("Connection rejected: %s\n",
+			 rdma_reject_msg(cm_id, event->status));
+		/* FALLTHROUGH */
 	case RDMA_CM_EVENT_UNREACHABLE:
 	case RDMA_CM_EVENT_CONNECT_ERROR:
 		nvmet_rdma_queue_connect_fail(cm_id, queue);
diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig
index ba140eaee5c8..650f1b1797ad 100644
--- a/drivers/nvmem/Kconfig
+++ b/drivers/nvmem/Kconfig
@@ -35,6 +35,16 @@ config NVMEM_LPC18XX_EEPROM
 	  To compile this driver as a module, choose M here: the module
 	  will be called nvmem_lpc18xx_eeprom.
 
+config NVMEM_LPC18XX_OTP
+	tristate "NXP LPC18XX OTP Memory Support"
+	depends on ARCH_LPC18XX || COMPILE_TEST
+	depends on HAS_IOMEM
+	help
+	  Say Y here to include support for NXP LPC18xx OTP memory found on
+	  all LPC18xx and LPC43xx devices.
+	  To compile this driver as a module, choose M here: the module
+	  will be called nvmem_lpc18xx_otp.
+
 config NVMEM_MXS_OCOTP
 	tristate "Freescale MXS On-Chip OTP Memory Support"
 	depends on ARCH_MXS || COMPILE_TEST
@@ -80,6 +90,18 @@ config ROCKCHIP_EFUSE
 	  This driver can also be built as a module. If so, the module
 	  will be called nvmem_rockchip_efuse.
 
+config NVMEM_BCM_OCOTP
+	tristate "Broadcom On-Chip OTP Controller support"
+	depends on ARCH_BCM_IPROC || COMPILE_TEST
+	depends on HAS_IOMEM
+	default ARCH_BCM_IPROC
+	help
+	  Say y here to enable read/write access to the Broadcom OTP
+	  controller.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called nvmem-bcm-ocotp.
+
 config NVMEM_SUNXI_SID
 	tristate "Allwinner SoCs SID support"
 	depends on ARCH_SUNXI
diff --git a/drivers/nvmem/Makefile b/drivers/nvmem/Makefile
index 8f942a0cdaec..86e45995fdad 100644
--- a/drivers/nvmem/Makefile
+++ b/drivers/nvmem/Makefile
@@ -6,10 +6,14 @@ obj-$(CONFIG_NVMEM)		+= nvmem_core.o
 nvmem_core-y			:= core.o
 
 # Devices
+obj-$(CONFIG_NVMEM_BCM_OCOTP)	+= nvmem-bcm-ocotp.o
+nvmem-bcm-ocotp-y		:= bcm-ocotp.o
 obj-$(CONFIG_NVMEM_IMX_OCOTP)	+= nvmem-imx-ocotp.o
 nvmem-imx-ocotp-y		:= imx-ocotp.o
 obj-$(CONFIG_NVMEM_LPC18XX_EEPROM)	+= nvmem_lpc18xx_eeprom.o
 nvmem_lpc18xx_eeprom-y	:= lpc18xx_eeprom.o
+obj-$(CONFIG_NVMEM_LPC18XX_OTP)	+= nvmem_lpc18xx_otp.o
+nvmem_lpc18xx_otp-y		:= lpc18xx_otp.o
 obj-$(CONFIG_NVMEM_MXS_OCOTP)	+= nvmem-mxs-ocotp.o
 nvmem-mxs-ocotp-y		:= mxs-ocotp.o
 obj-$(CONFIG_MTK_EFUSE)		+= nvmem_mtk-efuse.o
diff --git a/drivers/nvmem/bcm-ocotp.c b/drivers/nvmem/bcm-ocotp.c
new file mode 100644
index 000000000000..646cadbf1f93
--- /dev/null
+++ b/drivers/nvmem/bcm-ocotp.c
@@ -0,0 +1,335 @@
+/*
+ * Copyright (C) 2016 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/nvmem-provider.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+
+/*
+ * # of tries for OTP Status. The time to execute a command varies. The slowest
+ * commands are writes which also vary based on the # of bits turned on. Writing
+ * 0xffffffff takes ~3800 us.
+ */
+#define OTPC_RETRIES                 5000
+
+/* Sequence to enable OTP program */
+#define OTPC_PROG_EN_SEQ             { 0xf, 0x4, 0x8, 0xd }
+
+/* OTPC Commands */
+#define OTPC_CMD_READ                0x0
+#define OTPC_CMD_OTP_PROG_ENABLE     0x2
+#define OTPC_CMD_OTP_PROG_DISABLE    0x3
+#define OTPC_CMD_PROGRAM             0xA
+
+/* OTPC Status Bits */
+#define OTPC_STAT_CMD_DONE           BIT(1)
+#define OTPC_STAT_PROG_OK            BIT(2)
+
+/* OTPC register definition */
+#define OTPC_MODE_REG_OFFSET         0x0
+#define OTPC_MODE_REG_OTPC_MODE      0
+#define OTPC_COMMAND_OFFSET          0x4
+#define OTPC_COMMAND_COMMAND_WIDTH   6
+#define OTPC_CMD_START_OFFSET        0x8
+#define OTPC_CMD_START_START         0
+#define OTPC_CPU_STATUS_OFFSET       0xc
+#define OTPC_CPUADDR_REG_OFFSET      0x28
+#define OTPC_CPUADDR_REG_OTPC_CPU_ADDRESS_WIDTH 16
+#define OTPC_CPU_WRITE_REG_OFFSET    0x2c
+
+#define OTPC_CMD_MASK  (BIT(OTPC_COMMAND_COMMAND_WIDTH) - 1)
+#define OTPC_ADDR_MASK (BIT(OTPC_CPUADDR_REG_OTPC_CPU_ADDRESS_WIDTH) - 1)
+
+
+struct otpc_map {
+	/* in words. */
+	u32 otpc_row_size;
+	/* 128 bit row / 4 words support. */
+	u16 data_r_offset[4];
+	/* 128 bit row / 4 words support. */
+	u16 data_w_offset[4];
+};
+
+static struct otpc_map otp_map = {
+	.otpc_row_size = 1,
+	.data_r_offset = {0x10},
+	.data_w_offset = {0x2c},
+};
+
+static struct otpc_map otp_map_v2 = {
+	.otpc_row_size = 2,
+	.data_r_offset = {0x10, 0x5c},
+	.data_w_offset = {0x2c, 0x64},
+};
+
+struct otpc_priv {
+	struct device       *dev;
+	void __iomem        *base;
+	struct otpc_map     *map;
+	struct nvmem_config *config;
+};
+
+static inline void set_command(void __iomem *base, u32 command)
+{
+	writel(command & OTPC_CMD_MASK, base + OTPC_COMMAND_OFFSET);
+}
+
+static inline void set_cpu_address(void __iomem *base, u32 addr)
+{
+	writel(addr & OTPC_ADDR_MASK, base + OTPC_CPUADDR_REG_OFFSET);
+}
+
+static inline void set_start_bit(void __iomem *base)
+{
+	writel(1 << OTPC_CMD_START_START, base + OTPC_CMD_START_OFFSET);
+}
+
+static inline void reset_start_bit(void __iomem *base)
+{
+	writel(0, base + OTPC_CMD_START_OFFSET);
+}
+
+static inline void write_cpu_data(void __iomem *base, u32 value)
+{
+	writel(value, base + OTPC_CPU_WRITE_REG_OFFSET);
+}
+
+static int poll_cpu_status(void __iomem *base, u32 value)
+{
+	u32 status;
+	u32 retries;
+
+	for (retries = 0; retries < OTPC_RETRIES; retries++) {
+		status = readl(base + OTPC_CPU_STATUS_OFFSET);
+		if (status & value)
+			break;
+		udelay(1);
+	}
+	if (retries == OTPC_RETRIES)
+		return -EAGAIN;
+
+	return 0;
+}
+
+static int enable_ocotp_program(void __iomem *base)
+{
+	static const u32 vals[] = OTPC_PROG_EN_SEQ;
+	int i;
+	int ret;
+
+	/* Write the magic sequence to enable programming */
+	set_command(base, OTPC_CMD_OTP_PROG_ENABLE);
+	for (i = 0; i < ARRAY_SIZE(vals); i++) {
+		write_cpu_data(base, vals[i]);
+		set_start_bit(base);
+		ret = poll_cpu_status(base, OTPC_STAT_CMD_DONE);
+		reset_start_bit(base);
+		if (ret)
+			return ret;
+	}
+
+	return poll_cpu_status(base, OTPC_STAT_PROG_OK);
+}
+
+static int disable_ocotp_program(void __iomem *base)
+{
+	int ret;
+
+	set_command(base, OTPC_CMD_OTP_PROG_DISABLE);
+	set_start_bit(base);
+	ret = poll_cpu_status(base, OTPC_STAT_PROG_OK);
+	reset_start_bit(base);
+
+	return ret;
+}
+
+static int bcm_otpc_read(void *context, unsigned int offset, void *val,
+	size_t bytes)
+{
+	struct otpc_priv *priv = context;
+	u32 *buf = val;
+	u32 bytes_read;
+	u32 address = offset / priv->config->word_size;
+	int i, ret;
+
+	for (bytes_read = 0; bytes_read < bytes;) {
+		set_command(priv->base, OTPC_CMD_READ);
+		set_cpu_address(priv->base, address++);
+		set_start_bit(priv->base);
+		ret = poll_cpu_status(priv->base, OTPC_STAT_CMD_DONE);
+		if (ret) {
+			dev_err(priv->dev, "otp read error: 0x%x", ret);
+			return -EIO;
+		}
+
+		for (i = 0; i < priv->map->otpc_row_size; i++) {
+			*buf++ = readl(priv->base +
+					priv->map->data_r_offset[i]);
+			bytes_read += sizeof(*buf);
+		}
+
+		reset_start_bit(priv->base);
+	}
+
+	return 0;
+}
+
+static int bcm_otpc_write(void *context, unsigned int offset, void *val,
+	size_t bytes)
+{
+	struct otpc_priv *priv = context;
+	u32 *buf = val;
+	u32 bytes_written;
+	u32 address = offset / priv->config->word_size;
+	int i, ret;
+
+	if (offset % priv->config->word_size)
+		return -EINVAL;
+
+	ret = enable_ocotp_program(priv->base);
+	if (ret)
+		return -EIO;
+
+	for (bytes_written = 0; bytes_written < bytes;) {
+		set_command(priv->base, OTPC_CMD_PROGRAM);
+		set_cpu_address(priv->base, address++);
+		for (i = 0; i < priv->map->otpc_row_size; i++) {
+			writel(*buf, priv->base + priv->map->data_r_offset[i]);
+			buf++;
+			bytes_written += sizeof(*buf);
+		}
+		set_start_bit(priv->base);
+		ret = poll_cpu_status(priv->base, OTPC_STAT_CMD_DONE);
+		reset_start_bit(priv->base);
+		if (ret) {
+			dev_err(priv->dev, "otp write error: 0x%x", ret);
+			return -EIO;
+		}
+	}
+
+	disable_ocotp_program(priv->base);
+
+	return 0;
+}
+
+static struct nvmem_config bcm_otpc_nvmem_config = {
+	.name = "bcm-ocotp",
+	.read_only = false,
+	.word_size = 4,
+	.stride = 4,
+	.owner = THIS_MODULE,
+	.reg_read = bcm_otpc_read,
+	.reg_write = bcm_otpc_write,
+};
+
+static const struct of_device_id bcm_otpc_dt_ids[] = {
+	{ .compatible = "brcm,ocotp" },
+	{ .compatible = "brcm,ocotp-v2" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, bcm_otpc_dt_ids);
+
+static int bcm_otpc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *dn = dev->of_node;
+	struct resource *res;
+	struct otpc_priv *priv;
+	struct nvmem_device *nvmem;
+	int err;
+	u32 num_words;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	if (of_device_is_compatible(dev->of_node, "brcm,ocotp"))
+		priv->map = &otp_map;
+	else if (of_device_is_compatible(dev->of_node, "brcm,ocotp-v2"))
+		priv->map = &otp_map_v2;
+	else {
+		dev_err(&pdev->dev,
+			"%s otpc config map not defined\n", __func__);
+		return -EINVAL;
+	}
+
+	/* Get OTP base address register. */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(priv->base)) {
+		dev_err(dev, "unable to map I/O memory\n");
+		return PTR_ERR(priv->base);
+	}
+
+	/* Enable CPU access to OTPC. */
+	writel(readl(priv->base + OTPC_MODE_REG_OFFSET) |
+		BIT(OTPC_MODE_REG_OTPC_MODE),
+		priv->base + OTPC_MODE_REG_OFFSET);
+	reset_start_bit(priv->base);
+
+	/* Read size of memory in words. */
+	err = of_property_read_u32(dn, "brcm,ocotp-size", &num_words);
+	if (err) {
+		dev_err(dev, "size parameter not specified\n");
+		return -EINVAL;
+	} else if (num_words == 0) {
+		dev_err(dev, "size must be > 0\n");
+		return -EINVAL;
+	}
+
+	bcm_otpc_nvmem_config.size = 4 * num_words;
+	bcm_otpc_nvmem_config.dev = dev;
+	bcm_otpc_nvmem_config.priv = priv;
+
+	if (of_device_is_compatible(dev->of_node, "brcm,ocotp-v2")) {
+		bcm_otpc_nvmem_config.word_size = 8;
+		bcm_otpc_nvmem_config.stride = 8;
+	}
+
+	priv->config = &bcm_otpc_nvmem_config;
+
+	nvmem = nvmem_register(&bcm_otpc_nvmem_config);
+	if (IS_ERR(nvmem)) {
+		dev_err(dev, "error registering nvmem config\n");
+		return PTR_ERR(nvmem);
+	}
+
+	platform_set_drvdata(pdev, nvmem);
+
+	return 0;
+}
+
+static int bcm_otpc_remove(struct platform_device *pdev)
+{
+	struct nvmem_device *nvmem = platform_get_drvdata(pdev);
+
+	return nvmem_unregister(nvmem);
+}
+
+static struct platform_driver bcm_otpc_driver = {
+	.probe	= bcm_otpc_probe,
+	.remove	= bcm_otpc_remove,
+	.driver = {
+		.name	= "brcm-otpc",
+		.of_match_table = bcm_otpc_dt_ids,
+	},
+};
+module_platform_driver(bcm_otpc_driver);
+
+MODULE_DESCRIPTION("Broadcom OTPC driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/nvmem/lpc18xx_otp.c b/drivers/nvmem/lpc18xx_otp.c
new file mode 100644
index 000000000000..be8d07403ffc
--- /dev/null
+++ b/drivers/nvmem/lpc18xx_otp.c
@@ -0,0 +1,124 @@
+/*
+ * NXP LPC18xx/43xx OTP memory NVMEM driver
+ *
+ * Copyright (c) 2016 Joachim Eastwood <manabian@gmail.com>
+ *
+ * Based on the imx ocotp driver,
+ * Copyright (c) 2015 Pengutronix, Philipp Zabel <p.zabel@pengutronix.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ *
+ * TODO: add support for writing OTP register via API in boot ROM.
+ */
+
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/nvmem-provider.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+/*
+ * LPC18xx OTP memory contains 4 banks with 4 32-bit words. Bank 0 starts
+ * at offset 0 from the base.
+ *
+ * Bank 0 contains the part ID for Flashless devices and is reseverd for
+ * devices with Flash.
+ * Bank 1/2 is generale purpose or AES key storage for secure devices.
+ * Bank 3 contains control data, USB ID and generale purpose words.
+ */
+#define LPC18XX_OTP_NUM_BANKS		4
+#define LPC18XX_OTP_WORDS_PER_BANK	4
+#define LPC18XX_OTP_WORD_SIZE		sizeof(u32)
+#define LPC18XX_OTP_SIZE		(LPC18XX_OTP_NUM_BANKS * \
+					 LPC18XX_OTP_WORDS_PER_BANK * \
+					 LPC18XX_OTP_WORD_SIZE)
+
+struct lpc18xx_otp {
+	void __iomem *base;
+};
+
+static int lpc18xx_otp_read(void *context, unsigned int offset,
+			    void *val, size_t bytes)
+{
+	struct lpc18xx_otp *otp = context;
+	unsigned int count = bytes >> 2;
+	u32 index = offset >> 2;
+	u32 *buf = val;
+	int i;
+
+	if (count > (LPC18XX_OTP_SIZE - index))
+		count = LPC18XX_OTP_SIZE - index;
+
+	for (i = index; i < (index + count); i++)
+		*buf++ = readl(otp->base + i * LPC18XX_OTP_WORD_SIZE);
+
+	return 0;
+}
+
+static struct nvmem_config lpc18xx_otp_nvmem_config = {
+	.name = "lpc18xx-otp",
+	.read_only = true,
+	.word_size = LPC18XX_OTP_WORD_SIZE,
+	.stride = LPC18XX_OTP_WORD_SIZE,
+	.owner = THIS_MODULE,
+	.reg_read = lpc18xx_otp_read,
+};
+
+static int lpc18xx_otp_probe(struct platform_device *pdev)
+{
+	struct nvmem_device *nvmem;
+	struct lpc18xx_otp *otp;
+	struct resource *res;
+
+	otp = devm_kzalloc(&pdev->dev, sizeof(*otp), GFP_KERNEL);
+	if (!otp)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	otp->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(otp->base))
+		return PTR_ERR(otp->base);
+
+	lpc18xx_otp_nvmem_config.size = LPC18XX_OTP_SIZE;
+	lpc18xx_otp_nvmem_config.dev = &pdev->dev;
+	lpc18xx_otp_nvmem_config.priv = otp;
+
+	nvmem = nvmem_register(&lpc18xx_otp_nvmem_config);
+	if (IS_ERR(nvmem))
+		return PTR_ERR(nvmem);
+
+	platform_set_drvdata(pdev, nvmem);
+
+	return 0;
+}
+
+static int lpc18xx_otp_remove(struct platform_device *pdev)
+{
+	struct nvmem_device *nvmem = platform_get_drvdata(pdev);
+
+	return nvmem_unregister(nvmem);
+}
+
+static const struct of_device_id lpc18xx_otp_dt_ids[] = {
+	{ .compatible = "nxp,lpc1850-otp" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, lpc18xx_otp_dt_ids);
+
+static struct platform_driver lpc18xx_otp_driver = {
+	.probe	= lpc18xx_otp_probe,
+	.remove	= lpc18xx_otp_remove,
+	.driver = {
+		.name	= "lpc18xx_otp",
+		.of_match_table = lpc18xx_otp_dt_ids,
+	},
+};
+module_platform_driver(lpc18xx_otp_driver);
+
+MODULE_AUTHOR("Joachim Eastwoood <manabian@gmail.com>");
+MODULE_DESCRIPTION("NXP LPC18xx OTP NVMEM driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/of/base.c b/drivers/of/base.c
index a0bccb54a9bd..d4bea3c797d6 100644
--- a/drivers/of/base.c
+++ b/drivers/of/base.c
@@ -1534,9 +1534,12 @@ void of_print_phandle_args(const char *msg, const struct of_phandle_args *args)
 {
 	int i;
 	printk("%s %s", msg, of_node_full_name(args->np));
-	for (i = 0; i < args->args_count; i++)
-		printk(i ? ",%08x" : ":%08x", args->args[i]);
-	printk("\n");
+	for (i = 0; i < args->args_count; i++) {
+		const char delim = i ? ',' : ':';
+
+		pr_cont("%c%08x", delim, args->args[i]);
+	}
+	pr_cont("\n");
 }
 
 int of_phandle_iterator_init(struct of_phandle_iterator *it,
diff --git a/drivers/of/irq.c b/drivers/of/irq.c
index 393fea85eb4e..3fda9a32defb 100644
--- a/drivers/of/irq.c
+++ b/drivers/of/irq.c
@@ -697,3 +697,4 @@ void of_msi_configure(struct device *dev, struct device_node *np)
 	dev_set_msi_domain(dev,
 			   of_msi_get_domain(dev, np, DOMAIN_BUS_PLATFORM_MSI));
 }
+EXPORT_SYMBOL_GPL(of_msi_configure);
diff --git a/drivers/of/of_numa.c b/drivers/of/of_numa.c
index f63d4b0deff0..a53982a330ea 100644
--- a/drivers/of/of_numa.c
+++ b/drivers/of/of_numa.c
@@ -176,7 +176,12 @@ int of_node_to_nid(struct device_node *device)
 			np->name);
 	of_node_put(np);
 
-	if (!r)
+	/*
+	 * If numa=off passed on command line, or with a defective
+	 * device tree, the nid may not be in the set of possible
+	 * nodes.  Check for this case and return NUMA_NO_NODE.
+	 */
+	if (!r && nid < MAX_NUMNODES && node_possible(nid))
 		return nid;
 
 	return NUMA_NO_NODE;
diff --git a/drivers/of/of_pci.c b/drivers/of/of_pci.c
index b58be12ab277..0ee42c3e66a1 100644
--- a/drivers/of/of_pci.c
+++ b/drivers/of/of_pci.c
@@ -120,6 +120,27 @@ int of_get_pci_domain_nr(struct device_node *node)
 EXPORT_SYMBOL_GPL(of_get_pci_domain_nr);
 
 /**
+ * This function will try to find the limitation of link speed by finding
+ * a property called "max-link-speed" of the given device node.
+ *
+ * @node: device tree node with the max link speed information
+ *
+ * Returns the associated max link speed from DT, or a negative value if the
+ * required property is not found or is invalid.
+ */
+int of_pci_get_max_link_speed(struct device_node *node)
+{
+	u32 max_link_speed;
+
+	if (of_property_read_u32(node, "max-link-speed", &max_link_speed) ||
+	    max_link_speed > 4)
+		return -EINVAL;
+
+	return max_link_speed;
+}
+EXPORT_SYMBOL_GPL(of_pci_get_max_link_speed);
+
+/**
  * of_pci_check_probe_only - Setup probe only mode if linux,pci-probe-only
  *                           is present and valid
  */
diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c
index 318dbb51e7a2..0d4cda7050e0 100644
--- a/drivers/of/overlay.c
+++ b/drivers/of/overlay.c
@@ -58,6 +58,41 @@ struct of_overlay {
 static int of_overlay_apply_one(struct of_overlay *ov,
 		struct device_node *target, const struct device_node *overlay);
 
+static BLOCKING_NOTIFIER_HEAD(of_overlay_chain);
+
+int of_overlay_notifier_register(struct notifier_block *nb)
+{
+	return blocking_notifier_chain_register(&of_overlay_chain, nb);
+}
+EXPORT_SYMBOL_GPL(of_overlay_notifier_register);
+
+int of_overlay_notifier_unregister(struct notifier_block *nb)
+{
+	return blocking_notifier_chain_unregister(&of_overlay_chain, nb);
+}
+EXPORT_SYMBOL_GPL(of_overlay_notifier_unregister);
+
+static int of_overlay_notify(struct of_overlay *ov,
+			     enum of_overlay_notify_action action)
+{
+	struct of_overlay_notify_data nd;
+	int i, ret;
+
+	for (i = 0; i < ov->count; i++) {
+		struct of_overlay_info *ovinfo = &ov->ovinfo_tab[i];
+
+		nd.target = ovinfo->target;
+		nd.overlay = ovinfo->overlay;
+
+		ret = blocking_notifier_call_chain(&of_overlay_chain,
+						   action, &nd);
+		if (ret)
+			return notifier_to_errno(ret);
+	}
+
+	return 0;
+}
+
 static int of_overlay_apply_single_property(struct of_overlay *ov,
 		struct device_node *target, struct property *prop)
 {
@@ -368,6 +403,13 @@ int of_overlay_create(struct device_node *tree)
 		goto err_free_idr;
 	}
 
+	err = of_overlay_notify(ov, OF_OVERLAY_PRE_APPLY);
+	if (err < 0) {
+		pr_err("%s: Pre-apply notifier failed (err=%d)\n",
+		       __func__, err);
+		goto err_free_idr;
+	}
+
 	/* apply the overlay */
 	err = of_overlay_apply(ov);
 	if (err)
@@ -382,6 +424,8 @@ int of_overlay_create(struct device_node *tree)
 	/* add to the tail of the overlay list */
 	list_add_tail(&ov->node, &ov_list);
 
+	of_overlay_notify(ov, OF_OVERLAY_POST_APPLY);
+
 	mutex_unlock(&of_mutex);
 
 	return id;
@@ -498,9 +542,10 @@ int of_overlay_destroy(int id)
 		goto out;
 	}
 
-
+	of_overlay_notify(ov, OF_OVERLAY_PRE_REMOVE);
 	list_del(&ov->node);
 	__of_changeset_revert(&ov->cset);
+	of_overlay_notify(ov, OF_OVERLAY_POST_REMOVE);
 	of_free_overlay_info(ov);
 	idr_remove(&ov_idr, id);
 	of_changeset_destroy(&ov->cset);
diff --git a/drivers/of/platform.c b/drivers/of/platform.c
index e4bf07d20f9b..b8064bc2b6eb 100644
--- a/drivers/of/platform.c
+++ b/drivers/of/platform.c
@@ -45,6 +45,9 @@ static int of_dev_node_match(struct device *dev, void *data)
  * of_find_device_by_node - Find the platform_device associated with a node
  * @np: Pointer to device tree node
  *
+ * Takes a reference to the embedded struct device which needs to be dropped
+ * after use.
+ *
  * Returns platform_device pointer, or NULL if not found
  */
 struct platform_device *of_find_device_by_node(struct device_node *np)
@@ -558,9 +561,6 @@ static int of_platform_device_destroy(struct device *dev, void *data)
  * of the given device (and, recurrently, their children) that have been
  * created from their respective device tree nodes (and only those,
  * leaving others - eg. manually created - unharmed).
- *
- * Returns 0 when all children devices have been removed or
- * -EBUSY when some children remained.
  */
 void of_platform_depopulate(struct device *parent)
 {
diff --git a/drivers/of/resolver.c b/drivers/of/resolver.c
index 46325d6394cf..8bf12e904fd2 100644
--- a/drivers/of/resolver.c
+++ b/drivers/of/resolver.c
@@ -28,20 +28,19 @@
  * Find a node with the give full name by recursively following any of
  * the child node links.
  */
-static struct device_node *__of_find_node_by_full_name(struct device_node *node,
+static struct device_node *find_node_by_full_name(struct device_node *node,
 		const char *full_name)
 {
 	struct device_node *child, *found;
 
-	if (node == NULL)
+	if (!node)
 		return NULL;
 
-	/* check */
-	if (of_node_cmp(node->full_name, full_name) == 0)
+	if (!of_node_cmp(node->full_name, full_name))
 		return of_node_get(node);
 
 	for_each_child_of_node(node, child) {
-		found = __of_find_node_by_full_name(child, full_name);
+		found = find_node_by_full_name(child, full_name);
 		if (found != NULL) {
 			of_node_put(child);
 			return found;
@@ -51,16 +50,12 @@ static struct device_node *__of_find_node_by_full_name(struct device_node *node,
 	return NULL;
 }
 
-/*
- * Find live tree's maximum phandle value.
- */
-static phandle of_get_tree_max_phandle(void)
+static phandle live_tree_max_phandle(void)
 {
 	struct device_node *node;
 	phandle phandle;
 	unsigned long flags;
 
-	/* now search recursively */
 	raw_spin_lock_irqsave(&devtree_lock, flags);
 	phandle = 0;
 	for_each_of_allnodes(node) {
@@ -73,131 +68,102 @@ static phandle of_get_tree_max_phandle(void)
 	return phandle;
 }
 
-/*
- * Adjust a subtree's phandle values by a given delta.
- * Makes sure not to just adjust the device node's phandle value,
- * but modify the phandle properties values as well.
- */
-static void __of_adjust_tree_phandles(struct device_node *node,
+static void adjust_overlay_phandles(struct device_node *overlay,
 		int phandle_delta)
 {
 	struct device_node *child;
 	struct property *prop;
 	phandle phandle;
 
-	/* first adjust the node's phandle direct value */
-	if (node->phandle != 0 && node->phandle != OF_PHANDLE_ILLEGAL)
-		node->phandle += phandle_delta;
+	/* adjust node's phandle in node */
+	if (overlay->phandle != 0 && overlay->phandle != OF_PHANDLE_ILLEGAL)
+		overlay->phandle += phandle_delta;
 
-	/* now adjust phandle & linux,phandle values */
-	for_each_property_of_node(node, prop) {
+	/* copy adjusted phandle into *phandle properties */
+	for_each_property_of_node(overlay, prop) {
 
-		/* only look for these two */
-		if (of_prop_cmp(prop->name, "phandle") != 0 &&
-		    of_prop_cmp(prop->name, "linux,phandle") != 0)
+		if (of_prop_cmp(prop->name, "phandle") &&
+		    of_prop_cmp(prop->name, "linux,phandle"))
 			continue;
 
-		/* must be big enough */
 		if (prop->length < 4)
 			continue;
 
-		/* read phandle value */
 		phandle = be32_to_cpup(prop->value);
-		if (phandle == OF_PHANDLE_ILLEGAL)	/* unresolved */
+		if (phandle == OF_PHANDLE_ILLEGAL)
 			continue;
 
-		/* adjust */
-		*(uint32_t *)prop->value = cpu_to_be32(node->phandle);
+		*(uint32_t *)prop->value = cpu_to_be32(overlay->phandle);
 	}
 
-	/* now do the children recursively */
-	for_each_child_of_node(node, child)
-		__of_adjust_tree_phandles(child, phandle_delta);
+	for_each_child_of_node(overlay, child)
+		adjust_overlay_phandles(child, phandle_delta);
 }
 
-static int __of_adjust_phandle_ref(struct device_node *node,
-		struct property *rprop, int value)
+static int update_usages_of_a_phandle_reference(struct device_node *overlay,
+		struct property *prop_fixup, phandle phandle)
 {
-	phandle phandle;
 	struct device_node *refnode;
-	struct property *sprop;
-	char *propval, *propcur, *propend, *nodestr, *propstr, *s;
-	int offset, propcurlen;
+	struct property *prop;
+	char *value, *cur, *end, *node_path, *prop_name, *s;
+	int offset, len;
 	int err = 0;
 
-	/* make a copy */
-	propval = kmalloc(rprop->length, GFP_KERNEL);
-	if (!propval) {
-		pr_err("%s: Could not copy value of '%s'\n",
-				__func__, rprop->name);
+	value = kmalloc(prop_fixup->length, GFP_KERNEL);
+	if (!value)
 		return -ENOMEM;
-	}
-	memcpy(propval, rprop->value, rprop->length);
+	memcpy(value, prop_fixup->value, prop_fixup->length);
 
-	propend = propval + rprop->length;
-	for (propcur = propval; propcur < propend; propcur += propcurlen + 1) {
-		propcurlen = strlen(propcur);
+	/* prop_fixup contains a list of tuples of path:property_name:offset */
+	end = value + prop_fixup->length;
+	for (cur = value; cur < end; cur += len + 1) {
+		len = strlen(cur);
 
-		nodestr = propcur;
-		s = strchr(propcur, ':');
+		node_path = cur;
+		s = strchr(cur, ':');
 		if (!s) {
-			pr_err("%s: Illegal symbol entry '%s' (1)\n",
-				__func__, propcur);
 			err = -EINVAL;
 			goto err_fail;
 		}
 		*s++ = '\0';
 
-		propstr = s;
+		prop_name = s;
 		s = strchr(s, ':');
 		if (!s) {
-			pr_err("%s: Illegal symbol entry '%s' (2)\n",
-				__func__, (char *)rprop->value);
 			err = -EINVAL;
 			goto err_fail;
 		}
-
 		*s++ = '\0';
+
 		err = kstrtoint(s, 10, &offset);
-		if (err != 0) {
-			pr_err("%s: Could get offset '%s'\n",
-				__func__, (char *)rprop->value);
+		if (err)
 			goto err_fail;
-		}
 
-		/* look into the resolve node for the full path */
-		refnode = __of_find_node_by_full_name(node, nodestr);
-		if (!refnode) {
-			pr_warn("%s: Could not find refnode '%s'\n",
-				__func__, (char *)rprop->value);
+		refnode = find_node_by_full_name(overlay, node_path);
+		if (!refnode)
 			continue;
-		}
 
-		/* now find the property */
-		for_each_property_of_node(refnode, sprop) {
-			if (of_prop_cmp(sprop->name, propstr) == 0)
+		for_each_property_of_node(refnode, prop) {
+			if (!of_prop_cmp(prop->name, prop_name))
 				break;
 		}
 		of_node_put(refnode);
 
-		if (!sprop) {
-			pr_err("%s: Could not find property '%s'\n",
-				__func__, (char *)rprop->value);
+		if (!prop) {
 			err = -ENOENT;
 			goto err_fail;
 		}
 
-		phandle = value;
-		*(__be32 *)(sprop->value + offset) = cpu_to_be32(phandle);
+		*(__be32 *)(prop->value + offset) = cpu_to_be32(phandle);
 	}
 
 err_fail:
-	kfree(propval);
+	kfree(value);
 	return err;
 }
 
 /* compare nodes taking into account that 'name' strips out the @ part */
-static int __of_node_name_cmp(const struct device_node *dn1,
+static int node_name_cmp(const struct device_node *dn1,
 		const struct device_node *dn2)
 {
 	const char *n1 = strrchr(dn1->full_name, '/') ? : "/";
@@ -208,85 +174,77 @@ static int __of_node_name_cmp(const struct device_node *dn1,
 
 /*
  * Adjust the local phandle references by the given phandle delta.
- * Assumes the existances of a __local_fixups__ node at the root.
- * Assumes that __of_verify_tree_phandle_references has been called.
- * Does not take any devtree locks so make sure you call this on a tree
- * which is at the detached state.
+ *
+ * Subtree @local_fixups, which is overlay node __local_fixups__,
+ * mirrors the fragment node structure at the root of the overlay.
+ *
+ * For each property in the fragments that contains a phandle reference,
+ * @local_fixups has a property of the same name that contains a list
+ * of offsets of the phandle reference(s) within the respective property
+ * value(s).  The values at these offsets will be fixed up.
  */
-static int __of_adjust_tree_phandle_references(struct device_node *node,
-		struct device_node *target, int phandle_delta)
+static int adjust_local_phandle_references(struct device_node *local_fixups,
+		struct device_node *overlay, int phandle_delta)
 {
-	struct device_node *child, *childtarget;
-	struct property *rprop, *sprop;
+	struct device_node *child, *overlay_child;
+	struct property *prop_fix, *prop;
 	int err, i, count;
 	unsigned int off;
 	phandle phandle;
 
-	if (node == NULL)
+	if (!local_fixups)
 		return 0;
 
-	for_each_property_of_node(node, rprop) {
+	for_each_property_of_node(local_fixups, prop_fix) {
 
 		/* skip properties added automatically */
-		if (of_prop_cmp(rprop->name, "name") == 0 ||
-		    of_prop_cmp(rprop->name, "phandle") == 0 ||
-		    of_prop_cmp(rprop->name, "linux,phandle") == 0)
+		if (!of_prop_cmp(prop_fix->name, "name") ||
+		    !of_prop_cmp(prop_fix->name, "phandle") ||
+		    !of_prop_cmp(prop_fix->name, "linux,phandle"))
 			continue;
 
-		if ((rprop->length % 4) != 0 || rprop->length == 0) {
-			pr_err("%s: Illegal property (size) '%s' @%s\n",
-					__func__, rprop->name, node->full_name);
+		if ((prop_fix->length % 4) != 0 || prop_fix->length == 0)
 			return -EINVAL;
-		}
-		count = rprop->length / sizeof(__be32);
+		count = prop_fix->length / sizeof(__be32);
 
-		/* now find the target property */
-		for_each_property_of_node(target, sprop) {
-			if (of_prop_cmp(sprop->name, rprop->name) == 0)
+		for_each_property_of_node(overlay, prop) {
+			if (!of_prop_cmp(prop->name, prop_fix->name))
 				break;
 		}
 
-		if (sprop == NULL) {
-			pr_err("%s: Could not find target property '%s' @%s\n",
-					__func__, rprop->name, node->full_name);
+		if (!prop)
 			return -EINVAL;
-		}
 
 		for (i = 0; i < count; i++) {
-			off = be32_to_cpu(((__be32 *)rprop->value)[i]);
-			/* make sure the offset doesn't overstep (even wrap) */
-			if (off >= sprop->length ||
-					(off + 4) > sprop->length) {
-				pr_err("%s: Illegal property '%s' @%s\n",
-						__func__, rprop->name,
-						node->full_name);
+			off = be32_to_cpu(((__be32 *)prop_fix->value)[i]);
+			if ((off + 4) > prop->length)
 				return -EINVAL;
-			}
-
-			if (phandle_delta) {
-				/* adjust */
-				phandle = be32_to_cpu(*(__be32 *)(sprop->value + off));
-				phandle += phandle_delta;
-				*(__be32 *)(sprop->value + off) = cpu_to_be32(phandle);
-			}
+
+			phandle = be32_to_cpu(*(__be32 *)(prop->value + off));
+			phandle += phandle_delta;
+			*(__be32 *)(prop->value + off) = cpu_to_be32(phandle);
 		}
 	}
 
-	for_each_child_of_node(node, child) {
-
-		for_each_child_of_node(target, childtarget)
-			if (__of_node_name_cmp(child, childtarget) == 0)
+	/*
+	 * These nested loops recurse down two subtrees in parallel, where the
+	 * node names in the two subtrees match.
+	 *
+	 * The roots of the subtrees are the overlay's __local_fixups__ node
+	 * and the overlay's root node.
+	 */
+	for_each_child_of_node(local_fixups, child) {
+
+		for_each_child_of_node(overlay, overlay_child)
+			if (!node_name_cmp(child, overlay_child))
 				break;
 
-		if (!childtarget) {
-			pr_err("%s: Could not find target child '%s' @%s\n",
-					__func__, child->name, node->full_name);
+		if (!overlay_child)
 			return -EINVAL;
-		}
 
-		err = __of_adjust_tree_phandle_references(child, childtarget,
+		err = adjust_local_phandle_references(child, overlay_child,
 				phandle_delta);
-		if (err != 0)
+		if (err)
 			return err;
 	}
 
@@ -294,111 +252,103 @@ static int __of_adjust_tree_phandle_references(struct device_node *node,
 }
 
 /**
- * of_resolve	- Resolve the given node against the live tree.
+ * of_resolve_phandles - Relocate and resolve overlay against live tree
  *
- * @resolve:	Node to resolve
+ * @overlay:	Pointer to devicetree overlay to relocate and resolve
  *
- * Perform dynamic Device Tree resolution against the live tree
- * to the given node to resolve. This depends on the live tree
- * having a __symbols__ node, and the resolve node the __fixups__ &
- * __local_fixups__ nodes (if needed).
- * The result of the operation is a resolve node that it's contents
- * are fit to be inserted or operate upon the live tree.
- * Returns 0 on success or a negative error value on error.
+ * Modify (relocate) values of local phandles in @overlay to a range that
+ * does not conflict with the live expanded devicetree.  Update references
+ * to the local phandles in @overlay.  Update (resolve) phandle references
+ * in @overlay that refer to the live expanded devicetree.
+ *
+ * Phandle values in the live tree are in the range of
+ * 1 .. live_tree_max_phandle().  The range of phandle values in the overlay
+ * also begin with at 1.  Adjust the phandle values in the overlay to begin
+ * at live_tree_max_phandle() + 1.  Update references to the phandles to
+ * the adjusted phandle values.
+ *
+ * The name of each property in the "__fixups__" node in the overlay matches
+ * the name of a symbol (a label) in the live tree.  The values of each
+ * property in the "__fixups__" node is a list of the property values in the
+ * overlay that need to be updated to contain the phandle reference
+ * corresponding to that symbol in the live tree.  Update the references in
+ * the overlay with the phandle values in the live tree.
+ *
+ * @overlay must be detached.
+ *
+ * Resolving and applying @overlay to the live expanded devicetree must be
+ * protected by a mechanism to ensure that multiple overlays are processed
+ * in a single threaded manner so that multiple overlays will not relocate
+ * phandles to overlapping ranges.  The mechanism to enforce this is not
+ * yet implemented.
+ *
+ * Return: %0 on success or a negative error value on error.
  */
-int of_resolve_phandles(struct device_node *resolve)
+int of_resolve_phandles(struct device_node *overlay)
 {
-	struct device_node *child, *childroot, *refnode;
-	struct device_node *root_sym, *resolve_sym, *resolve_fix;
-	struct property *rprop;
+	struct device_node *child, *local_fixups, *refnode;
+	struct device_node *tree_symbols, *overlay_fixups;
+	struct property *prop;
 	const char *refpath;
 	phandle phandle, phandle_delta;
 	int err;
 
-	if (!resolve)
-		pr_err("%s: null node\n", __func__);
-	if (resolve && !of_node_check_flag(resolve, OF_DETACHED))
-		pr_err("%s: node %s not detached\n", __func__,
-			 resolve->full_name);
-	/* the resolve node must exist, and be detached */
-	if (!resolve || !of_node_check_flag(resolve, OF_DETACHED))
-		return -EINVAL;
-
-	/* first we need to adjust the phandles */
-	phandle_delta = of_get_tree_max_phandle() + 1;
-	__of_adjust_tree_phandles(resolve, phandle_delta);
-
-	/* locate the local fixups */
-	childroot = NULL;
-	for_each_child_of_node(resolve, childroot)
-		if (of_node_cmp(childroot->name, "__local_fixups__") == 0)
-			break;
-
-	if (childroot != NULL) {
-		/* resolve root is guaranteed to be the '/' */
-		err = __of_adjust_tree_phandle_references(childroot,
-				resolve, 0);
-		if (err != 0)
-			return err;
+	tree_symbols = NULL;
 
-		BUG_ON(__of_adjust_tree_phandle_references(childroot,
-				resolve, phandle_delta));
+	if (!overlay) {
+		pr_err("null overlay\n");
+		err = -EINVAL;
+		goto out;
+	}
+	if (!of_node_check_flag(overlay, OF_DETACHED)) {
+		pr_err("overlay not detached\n");
+		err = -EINVAL;
+		goto out;
 	}
 
-	root_sym = NULL;
-	resolve_sym = NULL;
-	resolve_fix = NULL;
-
-	/* this may fail (if no fixups are required) */
-	root_sym = of_find_node_by_path("/__symbols__");
+	phandle_delta = live_tree_max_phandle() + 1;
+	adjust_overlay_phandles(overlay, phandle_delta);
 
-	/* locate the symbols & fixups nodes on resolve */
-	for_each_child_of_node(resolve, child) {
+	for_each_child_of_node(overlay, local_fixups)
+		if (!of_node_cmp(local_fixups->name, "__local_fixups__"))
+			break;
 
-		if (!resolve_sym &&
-				of_node_cmp(child->name, "__symbols__") == 0)
-			resolve_sym = child;
+	err = adjust_local_phandle_references(local_fixups, overlay, phandle_delta);
+	if (err)
+		goto out;
 
-		if (!resolve_fix &&
-				of_node_cmp(child->name, "__fixups__") == 0)
-			resolve_fix = child;
+	overlay_fixups = NULL;
 
-		/* both found, don't bother anymore */
-		if (resolve_sym && resolve_fix)
-			break;
+	for_each_child_of_node(overlay, child) {
+		if (!of_node_cmp(child->name, "__fixups__"))
+			overlay_fixups = child;
 	}
 
-	/* we do allow for the case where no fixups are needed */
-	if (!resolve_fix) {
-		err = 0;	/* no error */
+	if (!overlay_fixups) {
+		err = 0;
 		goto out;
 	}
 
-	/* we need to fixup, but no root symbols... */
-	if (!root_sym) {
-		pr_err("%s: no symbols in root of device tree.\n", __func__);
+	tree_symbols = of_find_node_by_path("/__symbols__");
+	if (!tree_symbols) {
+		pr_err("no symbols in root of device tree.\n");
 		err = -EINVAL;
 		goto out;
 	}
 
-	for_each_property_of_node(resolve_fix, rprop) {
+	for_each_property_of_node(overlay_fixups, prop) {
 
 		/* skip properties added automatically */
-		if (of_prop_cmp(rprop->name, "name") == 0)
+		if (!of_prop_cmp(prop->name, "name"))
 			continue;
 
-		err = of_property_read_string(root_sym,
-				rprop->name, &refpath);
-		if (err != 0) {
-			pr_err("%s: Could not find symbol '%s'\n",
-					__func__, rprop->name);
+		err = of_property_read_string(tree_symbols,
+				prop->name, &refpath);
+		if (err)
 			goto out;
-		}
 
 		refnode = of_find_node_by_path(refpath);
 		if (!refnode) {
-			pr_err("%s: Could not find node by path '%s'\n",
-					__func__, refpath);
 			err = -ENOENT;
 			goto out;
 		}
@@ -406,17 +356,15 @@ int of_resolve_phandles(struct device_node *resolve)
 		phandle = refnode->phandle;
 		of_node_put(refnode);
 
-		pr_debug("%s: %s phandle is 0x%08x\n",
-				__func__, rprop->name, phandle);
-
-		err = __of_adjust_phandle_ref(resolve, rprop, phandle);
+		err = update_usages_of_a_phandle_reference(overlay, prop, phandle);
 		if (err)
 			break;
 	}
 
 out:
-	/* NULL is handled by of_node_put as NOP */
-	of_node_put(root_sym);
+	if (err)
+		pr_err("overlay phandle fixup failed: %d\n", err);
+	of_node_put(tree_symbols);
 
 	return err;
 }
diff --git a/drivers/pci/access.c b/drivers/pci/access.c
index d11cdbb8fba3..db239547fefd 100644
--- a/drivers/pci/access.c
+++ b/drivers/pci/access.c
@@ -142,10 +142,22 @@ int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn,
 	if (size == 4) {
 		writel(val, addr);
 		return PCIBIOS_SUCCESSFUL;
-	} else {
-		mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
 	}
 
+	/*
+	 * In general, hardware that supports only 32-bit writes on PCI is
+	 * not spec-compliant.  For example, software may perform a 16-bit
+	 * write.  If the hardware only supports 32-bit accesses, we must
+	 * do a 32-bit read, merge in the 16 bits we intend to write,
+	 * followed by a 32-bit write.  If the 16 bits we *don't* intend to
+	 * write happen to have any RW1C (write-one-to-clear) bits set, we
+	 * just inadvertently cleared something we shouldn't have.
+	 */
+	dev_warn_ratelimited(&bus->dev, "%d-byte config write to %04x:%02x:%02x.%d offset %#x may corrupt adjacent RW1C bits\n",
+			     size, pci_domain_nr(bus), bus->number,
+			     PCI_SLOT(devfn), PCI_FUNC(devfn), where);
+
+	mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
 	tmp = readl(addr) & mask;
 	tmp |= val << ((where & 0x3) * 8);
 	writel(tmp, addr);
diff --git a/drivers/pci/bus.c b/drivers/pci/bus.c
index c288e5a52575..bc56cf19afd3 100644
--- a/drivers/pci/bus.c
+++ b/drivers/pci/bus.c
@@ -320,7 +320,7 @@ void pci_bus_add_device(struct pci_dev *dev)
 	pci_fixup_device(pci_fixup_final, dev);
 	pci_create_sysfs_dev_files(dev);
 	pci_proc_attach_device(dev);
-	pci_bridge_d3_device_changed(dev);
+	pci_bridge_d3_update(dev);
 
 	dev->match_driver = true;
 	retval = device_attach(&dev->dev);
diff --git a/drivers/pci/ecam.c b/drivers/pci/ecam.c
index 43ed08dd8b01..2fee61bb6559 100644
--- a/drivers/pci/ecam.c
+++ b/drivers/pci/ecam.c
@@ -162,3 +162,15 @@ struct pci_ecam_ops pci_generic_ecam_ops = {
 		.write		= pci_generic_config_write,
 	}
 };
+
+#if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)
+/* ECAM ops for 32-bit access only (non-compliant) */
+struct pci_ecam_ops pci_32b_ops = {
+	.bus_shift	= 20,
+	.pci_ops	= {
+		.map_bus	= pci_ecam_map_bus,
+		.read		= pci_generic_config_read32,
+		.write		= pci_generic_config_write32,
+	}
+};
+#endif
diff --git a/drivers/pci/host/Kconfig b/drivers/pci/host/Kconfig
index d7e7c0a827c3..898d2c48239c 100644
--- a/drivers/pci/host/Kconfig
+++ b/drivers/pci/host/Kconfig
@@ -69,7 +69,7 @@ config PCI_IMX6
 
 config PCI_TEGRA
 	bool "NVIDIA Tegra PCIe controller"
-	depends on ARCH_TEGRA && !ARM64
+	depends on ARCH_TEGRA
 	help
 	  Say Y here if you want support for the PCIe host controller found
 	  on NVIDIA Tegra SoCs.
@@ -133,8 +133,8 @@ config PCIE_XILINX
 
 config PCI_XGENE
 	bool "X-Gene PCIe controller"
-	depends on ARCH_XGENE
-	depends on OF
+	depends on ARM64
+	depends on OF || (ACPI && PCI_QUIRKS)
 	select PCIEPORTBUS
 	help
 	  Say Y here if you want internal PCI support on APM X-Gene SoC.
@@ -240,14 +240,16 @@ config PCIE_QCOM
 
 config PCI_HOST_THUNDER_PEM
 	bool "Cavium Thunder PCIe controller to off-chip devices"
-	depends on OF && ARM64
+	depends on ARM64
+	depends on OF || (ACPI && PCI_QUIRKS)
 	select PCI_HOST_COMMON
 	help
 	  Say Y here if you want PCIe support for CN88XX Cavium Thunder SoCs.
 
 config PCI_HOST_THUNDER_ECAM
 	bool "Cavium Thunder ECAM controller to on-chip devices on pass-1.x silicon"
-	depends on OF && ARM64
+	depends on ARM64
+	depends on OF || (ACPI && PCI_QUIRKS)
 	select PCI_HOST_COMMON
 	help
 	  Say Y here if you want ECAM support for CN88XX-Pass-1.x Cavium Thunder SoCs.
@@ -276,7 +278,7 @@ config PCIE_ARTPEC6
 
 config PCIE_ROCKCHIP
 	bool "Rockchip PCIe controller"
-	depends on ARCH_ROCKCHIP
+	depends on ARCH_ROCKCHIP || COMPILE_TEST
 	depends on OF
 	depends on PCI_MSI_IRQ_DOMAIN
 	select MFD_SYSCON
@@ -286,7 +288,7 @@ config PCIE_ROCKCHIP
 	  4 slots.
 
 config VMD
-	depends on PCI_MSI && X86_64
+	depends on PCI_MSI && X86_64 && SRCU
 	tristate "Intel Volume Management Device Driver"
 	default N
 	---help---
diff --git a/drivers/pci/host/Makefile b/drivers/pci/host/Makefile
index 084cb4983645..bfe3179ae74c 100644
--- a/drivers/pci/host/Makefile
+++ b/drivers/pci/host/Makefile
@@ -15,7 +15,6 @@ obj-$(CONFIG_PCIE_SPEAR13XX) += pcie-spear13xx.o
 obj-$(CONFIG_PCI_KEYSTONE) += pci-keystone-dw.o pci-keystone.o
 obj-$(CONFIG_PCIE_XILINX) += pcie-xilinx.o
 obj-$(CONFIG_PCIE_XILINX_NWL) += pcie-xilinx-nwl.o
-obj-$(CONFIG_PCI_XGENE) += pci-xgene.o
 obj-$(CONFIG_PCI_XGENE_MSI) += pci-xgene-msi.o
 obj-$(CONFIG_PCI_LAYERSCAPE) += pci-layerscape.o
 obj-$(CONFIG_PCI_VERSATILE) += pci-versatile.o
@@ -25,11 +24,23 @@ obj-$(CONFIG_PCIE_IPROC_PLATFORM) += pcie-iproc-platform.o
 obj-$(CONFIG_PCIE_IPROC_BCMA) += pcie-iproc-bcma.o
 obj-$(CONFIG_PCIE_ALTERA) += pcie-altera.o
 obj-$(CONFIG_PCIE_ALTERA_MSI) += pcie-altera-msi.o
-obj-$(CONFIG_PCI_HISI) += pcie-hisi.o
 obj-$(CONFIG_PCIE_QCOM) += pcie-qcom.o
-obj-$(CONFIG_PCI_HOST_THUNDER_ECAM) += pci-thunder-ecam.o
-obj-$(CONFIG_PCI_HOST_THUNDER_PEM) += pci-thunder-pem.o
 obj-$(CONFIG_PCIE_ARMADA_8K) += pcie-armada8k.o
 obj-$(CONFIG_PCIE_ARTPEC6) += pcie-artpec6.o
 obj-$(CONFIG_PCIE_ROCKCHIP) += pcie-rockchip.o
 obj-$(CONFIG_VMD) += vmd.o
+
+# The following drivers are for devices that use the generic ACPI
+# pci_root.c driver but don't support standard ECAM config access.
+# They contain MCFG quirks to replace the generic ECAM accessors with
+# device-specific ones that are shared with the DT driver.
+
+# The ACPI driver is generic and should not require driver-specific
+# config options to be enabled, so we always build these drivers on
+# ARM64 and use internal ifdefs to only build the pieces we need
+# depending on whether ACPI, the DT driver, or both are enabled.
+
+obj-$(CONFIG_ARM64) += pcie-hisi.o
+obj-$(CONFIG_ARM64) += pci-thunder-ecam.o
+obj-$(CONFIG_ARM64) += pci-thunder-pem.o
+obj-$(CONFIG_ARM64) += pci-xgene.o
diff --git a/drivers/pci/host/pci-hyperv.c b/drivers/pci/host/pci-hyperv.c
index 763ff8745828..3efcc7bdc5fb 100644
--- a/drivers/pci/host/pci-hyperv.c
+++ b/drivers/pci/host/pci-hyperv.c
@@ -378,6 +378,8 @@ struct hv_pcibus_device {
 	struct msi_domain_info msi_info;
 	struct msi_controller msi_chip;
 	struct irq_domain *irq_domain;
+	struct retarget_msi_interrupt retarget_msi_interrupt_params;
+	spinlock_t retarget_msi_interrupt_lock;
 };
 
 /*
@@ -755,7 +757,7 @@ static int hv_set_affinity(struct irq_data *data, const struct cpumask *dest,
 	return parent->chip->irq_set_affinity(parent, dest, force);
 }
 
-void hv_irq_mask(struct irq_data *data)
+static void hv_irq_mask(struct irq_data *data)
 {
 	pci_msi_mask_irq(data);
 }
@@ -770,38 +772,44 @@ void hv_irq_mask(struct irq_data *data)
  * is built out of this PCI bus's instance GUID and the function
  * number of the device.
  */
-void hv_irq_unmask(struct irq_data *data)
+static void hv_irq_unmask(struct irq_data *data)
 {
 	struct msi_desc *msi_desc = irq_data_get_msi_desc(data);
 	struct irq_cfg *cfg = irqd_cfg(data);
-	struct retarget_msi_interrupt params;
+	struct retarget_msi_interrupt *params;
 	struct hv_pcibus_device *hbus;
 	struct cpumask *dest;
 	struct pci_bus *pbus;
 	struct pci_dev *pdev;
 	int cpu;
+	unsigned long flags;
 
 	dest = irq_data_get_affinity_mask(data);
 	pdev = msi_desc_to_pci_dev(msi_desc);
 	pbus = pdev->bus;
 	hbus = container_of(pbus->sysdata, struct hv_pcibus_device, sysdata);
 
-	memset(&params, 0, sizeof(params));
-	params.partition_id = HV_PARTITION_ID_SELF;
-	params.source = 1; /* MSI(-X) */
-	params.address = msi_desc->msg.address_lo;
-	params.data = msi_desc->msg.data;
-	params.device_id = (hbus->hdev->dev_instance.b[5] << 24) |
+	spin_lock_irqsave(&hbus->retarget_msi_interrupt_lock, flags);
+
+	params = &hbus->retarget_msi_interrupt_params;
+	memset(params, 0, sizeof(*params));
+	params->partition_id = HV_PARTITION_ID_SELF;
+	params->source = 1; /* MSI(-X) */
+	params->address = msi_desc->msg.address_lo;
+	params->data = msi_desc->msg.data;
+	params->device_id = (hbus->hdev->dev_instance.b[5] << 24) |
 			   (hbus->hdev->dev_instance.b[4] << 16) |
 			   (hbus->hdev->dev_instance.b[7] << 8) |
 			   (hbus->hdev->dev_instance.b[6] & 0xf8) |
 			   PCI_FUNC(pdev->devfn);
-	params.vector = cfg->vector;
+	params->vector = cfg->vector;
 
 	for_each_cpu_and(cpu, dest, cpu_online_mask)
-		params.vp_mask |= (1ULL << vmbus_cpu_number_to_vp_number(cpu));
+		params->vp_mask |= (1ULL << vmbus_cpu_number_to_vp_number(cpu));
 
-	hv_do_hypercall(HVCALL_RETARGET_INTERRUPT, &params, NULL);
+	hv_do_hypercall(HVCALL_RETARGET_INTERRUPT, params, NULL);
+
+	spin_unlock_irqrestore(&hbus->retarget_msi_interrupt_lock, flags);
 
 	pci_msi_unmask_irq(data);
 }
@@ -1271,9 +1279,9 @@ static struct hv_pci_dev *new_pcichild_device(struct hv_pcibus_device *hbus,
 	struct hv_pci_dev *hpdev;
 	struct pci_child_message *res_req;
 	struct q_res_req_compl comp_pkt;
-	union {
-	struct pci_packet init_packet;
-		u8 buffer[0x100];
+	struct {
+		struct pci_packet init_packet;
+		u8 buffer[sizeof(struct pci_child_message)];
 	} pkt;
 	unsigned long flags;
 	int ret;
@@ -1582,6 +1590,10 @@ static void hv_eject_device_work(struct work_struct *work)
 		pci_dev_put(pdev);
 	}
 
+	spin_lock_irqsave(&hpdev->hbus->device_list_lock, flags);
+	list_del(&hpdev->list_entry);
+	spin_unlock_irqrestore(&hpdev->hbus->device_list_lock, flags);
+
 	memset(&ctxt, 0, sizeof(ctxt));
 	ejct_pkt = (struct pci_eject_response *)&ctxt.pkt.message;
 	ejct_pkt->message_type.type = PCI_EJECTION_COMPLETE;
@@ -1590,10 +1602,6 @@ static void hv_eject_device_work(struct work_struct *work)
 			 sizeof(*ejct_pkt), (unsigned long)&ctxt.pkt,
 			 VM_PKT_DATA_INBAND, 0);
 
-	spin_lock_irqsave(&hpdev->hbus->device_list_lock, flags);
-	list_del(&hpdev->list_entry);
-	spin_unlock_irqrestore(&hpdev->hbus->device_list_lock, flags);
-
 	put_pcichild(hpdev, hv_pcidev_ref_childlist);
 	put_pcichild(hpdev, hv_pcidev_ref_pnp);
 	put_hvpcibus(hpdev->hbus);
@@ -2186,6 +2194,7 @@ static int hv_pci_probe(struct hv_device *hdev,
 	INIT_LIST_HEAD(&hbus->resources_for_children);
 	spin_lock_init(&hbus->config_lock);
 	spin_lock_init(&hbus->device_list_lock);
+	spin_lock_init(&hbus->retarget_msi_interrupt_lock);
 	sema_init(&hbus->enum_sem, 1);
 	init_completion(&hbus->remove_event);
 
@@ -2266,24 +2275,32 @@ free_bus:
 	return ret;
 }
 
-/**
- * hv_pci_remove() - Remove routine for this VMBus channel
- * @hdev:	VMBus's tracking struct for this root PCI bus
- *
- * Return: 0 on success, -errno on failure
- */
-static int hv_pci_remove(struct hv_device *hdev)
+static void hv_pci_bus_exit(struct hv_device *hdev)
 {
-	int ret;
-	struct hv_pcibus_device *hbus;
-	union {
+	struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
+	struct {
 		struct pci_packet teardown_packet;
-		u8 buffer[0x100];
+		u8 buffer[sizeof(struct pci_message)];
 	} pkt;
 	struct pci_bus_relations relations;
 	struct hv_pci_compl comp_pkt;
+	int ret;
 
-	hbus = hv_get_drvdata(hdev);
+	/*
+	 * After the host sends the RESCIND_CHANNEL message, it doesn't
+	 * access the per-channel ringbuffer any longer.
+	 */
+	if (hdev->channel->rescind)
+		return;
+
+	/* Delete any children which might still exist. */
+	memset(&relations, 0, sizeof(relations));
+	hv_pci_devices_present(hbus, &relations);
+
+	ret = hv_send_resources_released(hdev);
+	if (ret)
+		dev_err(&hdev->device,
+			"Couldn't send resources released packet(s)\n");
 
 	memset(&pkt.teardown_packet, 0, sizeof(pkt.teardown_packet));
 	init_completion(&comp_pkt.host_event);
@@ -2298,7 +2315,19 @@ static int hv_pci_remove(struct hv_device *hdev)
 			       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 	if (!ret)
 		wait_for_completion_timeout(&comp_pkt.host_event, 10 * HZ);
+}
+
+/**
+ * hv_pci_remove() - Remove routine for this VMBus channel
+ * @hdev:	VMBus's tracking struct for this root PCI bus
+ *
+ * Return: 0 on success, -errno on failure
+ */
+static int hv_pci_remove(struct hv_device *hdev)
+{
+	struct hv_pcibus_device *hbus;
 
+	hbus = hv_get_drvdata(hdev);
 	if (hbus->state == hv_pcibus_installed) {
 		/* Remove the bus from PCI's point of view. */
 		pci_lock_rescan_remove();
@@ -2307,17 +2336,10 @@ static int hv_pci_remove(struct hv_device *hdev)
 		pci_unlock_rescan_remove();
 	}
 
-	ret = hv_send_resources_released(hdev);
-	if (ret)
-		dev_err(&hdev->device,
-			"Couldn't send resources released packet(s)\n");
+	hv_pci_bus_exit(hdev);
 
 	vmbus_close(hdev->channel);
 
-	/* Delete any children which might still exist. */
-	memset(&relations, 0, sizeof(relations));
-	hv_pci_devices_present(hbus, &relations);
-
 	iounmap(hbus->cfg_addr);
 	hv_free_config_window(hbus);
 	pci_free_resource_list(&hbus->resources_for_children);
diff --git a/drivers/pci/host/pci-layerscape.c b/drivers/pci/host/pci-layerscape.c
index 653707996342..ea789138531b 100644
--- a/drivers/pci/host/pci-layerscape.c
+++ b/drivers/pci/host/pci-layerscape.c
@@ -35,12 +35,10 @@
 #define PCIE_STRFMR1		0x71c /* Symbol Timer & Filter Mask Register1 */
 #define PCIE_DBI_RO_WR_EN	0x8bc /* DBI Read-Only Write Enable Register */
 
-/* PEX LUT registers */
-#define PCIE_LUT_DBG		0x7FC /* PEX LUT Debug Register */
-
 struct ls_pcie_drvdata {
 	u32 lut_offset;
 	u32 ltssm_shift;
+	u32 lut_dbg;
 	struct pcie_host_ops *ops;
 };
 
@@ -134,7 +132,7 @@ static int ls_pcie_link_up(struct pcie_port *pp)
 	struct ls_pcie *pcie = to_ls_pcie(pp);
 	u32 state;
 
-	state = (ioread32(pcie->lut + PCIE_LUT_DBG) >>
+	state = (ioread32(pcie->lut + pcie->drvdata->lut_dbg) >>
 		 pcie->drvdata->ltssm_shift) &
 		 LTSSM_STATE_MASK;
 
@@ -196,18 +194,28 @@ static struct ls_pcie_drvdata ls1021_drvdata = {
 static struct ls_pcie_drvdata ls1043_drvdata = {
 	.lut_offset = 0x10000,
 	.ltssm_shift = 24,
+	.lut_dbg = 0x7fc,
+	.ops = &ls_pcie_host_ops,
+};
+
+static struct ls_pcie_drvdata ls1046_drvdata = {
+	.lut_offset = 0x80000,
+	.ltssm_shift = 24,
+	.lut_dbg = 0x407fc,
 	.ops = &ls_pcie_host_ops,
 };
 
 static struct ls_pcie_drvdata ls2080_drvdata = {
 	.lut_offset = 0x80000,
 	.ltssm_shift = 0,
+	.lut_dbg = 0x7fc,
 	.ops = &ls_pcie_host_ops,
 };
 
 static const struct of_device_id ls_pcie_of_match[] = {
 	{ .compatible = "fsl,ls1021a-pcie", .data = &ls1021_drvdata },
 	{ .compatible = "fsl,ls1043a-pcie", .data = &ls1043_drvdata },
+	{ .compatible = "fsl,ls1046a-pcie", .data = &ls1046_drvdata },
 	{ .compatible = "fsl,ls2080a-pcie", .data = &ls2080_drvdata },
 	{ .compatible = "fsl,ls2085a-pcie", .data = &ls2080_drvdata },
 	{ },
@@ -252,10 +260,8 @@ static int __init ls_pcie_probe(struct platform_device *pdev)
 
 	dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs");
 	pcie->pp.dbi_base = devm_ioremap_resource(dev, dbi_base);
-	if (IS_ERR(pcie->pp.dbi_base)) {
-		dev_err(dev, "missing *regs* space\n");
+	if (IS_ERR(pcie->pp.dbi_base))
 		return PTR_ERR(pcie->pp.dbi_base);
-	}
 
 	pcie->lut = pcie->pp.dbi_base + pcie->drvdata->lut_offset;
 
diff --git a/drivers/pci/host/pci-rcar-gen2.c b/drivers/pci/host/pci-rcar-gen2.c
index 1eeefa4df64c..85348590848b 100644
--- a/drivers/pci/host/pci-rcar-gen2.c
+++ b/drivers/pci/host/pci-rcar-gen2.c
@@ -430,10 +430,10 @@ static int rcar_pci_probe(struct platform_device *pdev)
 }
 
 static struct of_device_id rcar_pci_of_match[] = {
-	{ .compatible = "renesas,pci-rcar-gen2", },
 	{ .compatible = "renesas,pci-r8a7790", },
 	{ .compatible = "renesas,pci-r8a7791", },
 	{ .compatible = "renesas,pci-r8a7794", },
+	{ .compatible = "renesas,pci-rcar-gen2", },
 	{ },
 };
 
diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index 8dfccf733241..ed8a93f2bfb5 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -51,10 +51,6 @@
 #include <soc/tegra/cpuidle.h>
 #include <soc/tegra/pmc.h>
 
-#include <asm/mach/irq.h>
-#include <asm/mach/map.h>
-#include <asm/mach/pci.h>
-
 #define INT_PCI_MSI_NR (8 * 32)
 
 /* register definitions */
@@ -188,6 +184,9 @@
 #define RP_VEND_XP	0x00000f00
 #define  RP_VEND_XP_DL_UP	(1 << 30)
 
+#define RP_VEND_CTL2 0x00000fa8
+#define  RP_VEND_CTL2_PCA_ENABLE (1 << 7)
+
 #define RP_PRIV_MISC	0x00000fe0
 #define  RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xe << 0)
 #define  RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xf << 0)
@@ -252,6 +251,7 @@ struct tegra_pcie_soc {
 	bool has_intr_prsnt_sense;
 	bool has_cml_clk;
 	bool has_gen2;
+	bool force_pca_enable;
 };
 
 static inline struct tegra_msi *to_tegra_msi(struct msi_controller *chip)
@@ -322,11 +322,6 @@ struct tegra_pcie_bus {
 	unsigned int nr;
 };
 
-static inline struct tegra_pcie *sys_to_pcie(struct pci_sys_data *sys)
-{
-	return sys->private_data;
-}
-
 static inline void afi_writel(struct tegra_pcie *pcie, u32 value,
 			      unsigned long offset)
 {
@@ -385,8 +380,7 @@ static struct tegra_pcie_bus *tegra_pcie_bus_alloc(struct tegra_pcie *pcie,
 						   unsigned int busnr)
 {
 	struct device *dev = pcie->dev;
-	pgprot_t prot = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
-				 L_PTE_XN | L_PTE_MT_DEV_SHARED | L_PTE_SHARED);
+	pgprot_t prot = pgprot_device(PAGE_KERNEL);
 	phys_addr_t cs = pcie->cs->start;
 	struct tegra_pcie_bus *bus;
 	unsigned int i;
@@ -430,7 +424,8 @@ free:
 
 static int tegra_pcie_add_bus(struct pci_bus *bus)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
+	struct pci_host_bridge *host = pci_find_host_bridge(bus);
+	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
 	struct tegra_pcie_bus *b;
 
 	b = tegra_pcie_bus_alloc(pcie, bus->number);
@@ -444,7 +439,8 @@ static int tegra_pcie_add_bus(struct pci_bus *bus)
 
 static void tegra_pcie_remove_bus(struct pci_bus *child)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(child->sysdata);
+	struct pci_host_bridge *host = pci_find_host_bridge(child);
+	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
 	struct tegra_pcie_bus *bus, *tmp;
 
 	list_for_each_entry_safe(bus, tmp, &pcie->buses, list) {
@@ -461,7 +457,8 @@ static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
 					unsigned int devfn,
 					int where)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(bus->sysdata);
+	struct pci_host_bridge *host = pci_find_host_bridge(bus);
+	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
 	struct device *dev = pcie->dev;
 	void __iomem *addr = NULL;
 
@@ -558,6 +555,12 @@ static void tegra_pcie_port_enable(struct tegra_pcie_port *port)
 	afi_writel(port->pcie, value, ctrl);
 
 	tegra_pcie_port_reset(port);
+
+	if (soc->force_pca_enable) {
+		value = readl(port->base + RP_VEND_CTL2);
+		value |= RP_VEND_CTL2_PCA_ENABLE;
+		writel(value, port->base + RP_VEND_CTL2);
+	}
 }
 
 static void tegra_pcie_port_disable(struct tegra_pcie_port *port)
@@ -610,39 +613,31 @@ static void tegra_pcie_relax_enable(struct pci_dev *dev)
 }
 DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, tegra_pcie_relax_enable);
 
-static int tegra_pcie_setup(int nr, struct pci_sys_data *sys)
+static int tegra_pcie_request_resources(struct tegra_pcie *pcie)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(sys);
+	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+	struct list_head *windows = &host->windows;
 	struct device *dev = pcie->dev;
 	int err;
 
-	sys->mem_offset = pcie->offset.mem;
-	sys->io_offset = pcie->offset.io;
+	pci_add_resource_offset(windows, &pcie->pio, pcie->offset.io);
+	pci_add_resource_offset(windows, &pcie->mem, pcie->offset.mem);
+	pci_add_resource_offset(windows, &pcie->prefetch, pcie->offset.mem);
+	pci_add_resource(windows, &pcie->busn);
 
-	err = devm_request_resource(dev, &iomem_resource, &pcie->io);
+	err = devm_request_pci_bus_resources(dev, windows);
 	if (err < 0)
 		return err;
 
-	err = pci_remap_iospace(&pcie->pio, pcie->io.start);
-	if (!err)
-		pci_add_resource_offset(&sys->resources, &pcie->pio,
-					sys->io_offset);
+	pci_remap_iospace(&pcie->pio, pcie->io.start);
 
-	pci_add_resource_offset(&sys->resources, &pcie->mem, sys->mem_offset);
-	pci_add_resource_offset(&sys->resources, &pcie->prefetch,
-				sys->mem_offset);
-	pci_add_resource(&sys->resources, &pcie->busn);
-
-	err = devm_request_pci_bus_resources(dev, &sys->resources);
-	if (err < 0)
-		return err;
-
-	return 1;
+	return 0;
 }
 
 static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
 {
-	struct tegra_pcie *pcie = sys_to_pcie(pdev->bus->sysdata);
+	struct pci_host_bridge *host = pci_find_host_bridge(pdev->bus);
+	struct tegra_pcie *pcie = pci_host_bridge_priv(host);
 	int irq;
 
 	tegra_cpuidle_pcie_irqs_in_use();
@@ -1499,10 +1494,11 @@ static const struct irq_domain_ops msi_domain_ops = {
 
 static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
 {
-	struct device *dev = pcie->dev;
-	struct platform_device *pdev = to_platform_device(dev);
+	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+	struct platform_device *pdev = to_platform_device(pcie->dev);
 	const struct tegra_pcie_soc *soc = pcie->soc;
 	struct tegra_msi *msi = &pcie->msi;
+	struct device *dev = pcie->dev;
 	unsigned long base;
 	int err;
 	u32 reg;
@@ -1559,6 +1555,8 @@ static int tegra_pcie_enable_msi(struct tegra_pcie *pcie)
 	reg |= AFI_INTR_MASK_MSI_MASK;
 	afi_writel(pcie, reg, AFI_INTR_MASK);
 
+	host->msi = &msi->chip;
+
 	return 0;
 
 err:
@@ -1609,7 +1607,8 @@ static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes,
 	struct device *dev = pcie->dev;
 	struct device_node *np = dev->of_node;
 
-	if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
+	if (of_device_is_compatible(np, "nvidia,tegra124-pcie") ||
+	    of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
 		switch (lanes) {
 		case 0x0000104:
 			dev_info(dev, "4x1, 1x1 configuration\n");
@@ -1730,7 +1729,22 @@ static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask)
 	struct device_node *np = dev->of_node;
 	unsigned int i = 0;
 
-	if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
+	if (of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
+		pcie->num_supplies = 6;
+
+		pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
+					      sizeof(*pcie->supplies),
+					      GFP_KERNEL);
+		if (!pcie->supplies)
+			return -ENOMEM;
+
+		pcie->supplies[i++].supply = "avdd-pll-uerefe";
+		pcie->supplies[i++].supply = "hvddio-pex";
+		pcie->supplies[i++].supply = "dvddio-pex";
+		pcie->supplies[i++].supply = "dvdd-pex-pll";
+		pcie->supplies[i++].supply = "hvdd-pex-pll-e";
+		pcie->supplies[i++].supply = "vddio-pex-ctl";
+	} else if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
 		pcie->num_supplies = 7;
 
 		pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
@@ -2021,11 +2035,10 @@ retry:
 	return false;
 }
 
-static int tegra_pcie_enable(struct tegra_pcie *pcie)
+static void tegra_pcie_enable_ports(struct tegra_pcie *pcie)
 {
 	struct device *dev = pcie->dev;
 	struct tegra_pcie_port *port, *tmp;
-	struct hw_pci hw;
 
 	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
 		dev_info(dev, "probing port %u, using %u lanes\n",
@@ -2041,21 +2054,6 @@ static int tegra_pcie_enable(struct tegra_pcie *pcie)
 		tegra_pcie_port_disable(port);
 		tegra_pcie_port_free(port);
 	}
-
-	memset(&hw, 0, sizeof(hw));
-
-#ifdef CONFIG_PCI_MSI
-	hw.msi_ctrl = &pcie->msi.chip;
-#endif
-
-	hw.nr_controllers = 1;
-	hw.private_data = (void **)&pcie;
-	hw.setup = tegra_pcie_setup;
-	hw.map_irq = tegra_pcie_map_irq;
-	hw.ops = &tegra_pcie_ops;
-
-	pci_common_init_dev(dev, &hw);
-	return 0;
 }
 
 static const struct tegra_pcie_soc tegra20_pcie = {
@@ -2069,6 +2067,7 @@ static const struct tegra_pcie_soc tegra20_pcie = {
 	.has_intr_prsnt_sense = false,
 	.has_cml_clk = false,
 	.has_gen2 = false,
+	.force_pca_enable = false,
 };
 
 static const struct tegra_pcie_soc tegra30_pcie = {
@@ -2083,6 +2082,7 @@ static const struct tegra_pcie_soc tegra30_pcie = {
 	.has_intr_prsnt_sense = true,
 	.has_cml_clk = true,
 	.has_gen2 = false,
+	.force_pca_enable = false,
 };
 
 static const struct tegra_pcie_soc tegra124_pcie = {
@@ -2096,9 +2096,25 @@ static const struct tegra_pcie_soc tegra124_pcie = {
 	.has_intr_prsnt_sense = true,
 	.has_cml_clk = true,
 	.has_gen2 = true,
+	.force_pca_enable = false,
+};
+
+static const struct tegra_pcie_soc tegra210_pcie = {
+	.num_ports = 2,
+	.msi_base_shift = 8,
+	.pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
+	.tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
+	.pads_refclk_cfg0 = 0x90b890b8,
+	.has_pex_clkreq_en = true,
+	.has_pex_bias_ctrl = true,
+	.has_intr_prsnt_sense = true,
+	.has_cml_clk = true,
+	.has_gen2 = true,
+	.force_pca_enable = true,
 };
 
 static const struct of_device_id tegra_pcie_of_match[] = {
+	{ .compatible = "nvidia,tegra210-pcie", .data = &tegra210_pcie },
 	{ .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie },
 	{ .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie },
 	{ .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie },
@@ -2217,13 +2233,17 @@ remove:
 static int tegra_pcie_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	struct pci_host_bridge *host;
 	struct tegra_pcie *pcie;
+	struct pci_bus *child;
 	int err;
 
-	pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
-	if (!pcie)
+	host = pci_alloc_host_bridge(sizeof(*pcie));
+	if (!host)
 		return -ENOMEM;
 
+	pcie = pci_host_bridge_priv(host);
+
 	pcie->soc = of_device_get_match_data(dev);
 	INIT_LIST_HEAD(&pcie->buses);
 	INIT_LIST_HEAD(&pcie->ports);
@@ -2243,6 +2263,10 @@ static int tegra_pcie_probe(struct platform_device *pdev)
 	if (err)
 		goto put_resources;
 
+	err = tegra_pcie_request_resources(pcie);
+	if (err)
+		goto put_resources;
+
 	/* setup the AFI address translations */
 	tegra_pcie_setup_translations(pcie);
 
@@ -2254,12 +2278,30 @@ static int tegra_pcie_probe(struct platform_device *pdev)
 		}
 	}
 
-	err = tegra_pcie_enable(pcie);
+	tegra_pcie_enable_ports(pcie);
+
+	pci_add_flags(PCI_REASSIGN_ALL_RSRC | PCI_REASSIGN_ALL_BUS);
+	host->busnr = pcie->busn.start;
+	host->dev.parent = &pdev->dev;
+	host->ops = &tegra_pcie_ops;
+
+	err = pci_register_host_bridge(host);
 	if (err < 0) {
-		dev_err(dev, "failed to enable PCIe ports: %d\n", err);
+		dev_err(dev, "failed to register host: %d\n", err);
 		goto disable_msi;
 	}
 
+	pci_scan_child_bus(host->bus);
+
+	pci_fixup_irqs(pci_common_swizzle, tegra_pcie_map_irq);
+	pci_bus_size_bridges(host->bus);
+	pci_bus_assign_resources(host->bus);
+
+	list_for_each_entry(child, &host->bus->children, node)
+		pcie_bus_configure_settings(child);
+
+	pci_bus_add_devices(host->bus);
+
 	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
 		err = tegra_pcie_debugfs_init(pcie);
 		if (err < 0)
diff --git a/drivers/pci/host/pci-thunder-ecam.c b/drivers/pci/host/pci-thunder-ecam.c
index d50a3dc2d8db..3f54a43bbbea 100644
--- a/drivers/pci/host/pci-thunder-ecam.c
+++ b/drivers/pci/host/pci-thunder-ecam.c
@@ -14,6 +14,8 @@
 #include <linux/pci-ecam.h>
 #include <linux/platform_device.h>
 
+#if defined(CONFIG_PCI_HOST_THUNDER_ECAM) || (defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS))
+
 static void set_val(u32 v, int where, int size, u32 *val)
 {
 	int shift = (where & 3) * 8;
@@ -346,7 +348,7 @@ static int thunder_ecam_config_write(struct pci_bus *bus, unsigned int devfn,
 	return pci_generic_config_write(bus, devfn, where, size, val);
 }
 
-static struct pci_ecam_ops pci_thunder_ecam_ops = {
+struct pci_ecam_ops pci_thunder_ecam_ops = {
 	.bus_shift	= 20,
 	.pci_ops	= {
 		.map_bus        = pci_ecam_map_bus,
@@ -355,6 +357,8 @@ static struct pci_ecam_ops pci_thunder_ecam_ops = {
 	}
 };
 
+#ifdef CONFIG_PCI_HOST_THUNDER_ECAM
+
 static const struct of_device_id thunder_ecam_of_match[] = {
 	{ .compatible = "cavium,pci-host-thunder-ecam" },
 	{ },
@@ -373,3 +377,6 @@ static struct platform_driver thunder_ecam_driver = {
 	.probe = thunder_ecam_probe,
 };
 builtin_platform_driver(thunder_ecam_driver);
+
+#endif
+#endif
diff --git a/drivers/pci/host/pci-thunder-pem.c b/drivers/pci/host/pci-thunder-pem.c
index 6abaf80ffb39..af722eb0ca75 100644
--- a/drivers/pci/host/pci-thunder-pem.c
+++ b/drivers/pci/host/pci-thunder-pem.c
@@ -18,8 +18,12 @@
 #include <linux/init.h>
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
+#include <linux/pci-acpi.h>
 #include <linux/pci-ecam.h>
 #include <linux/platform_device.h>
+#include "../pci.h"
+
+#if defined(CONFIG_PCI_HOST_THUNDER_PEM) || (defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS))
 
 #define PEM_CFG_WR 0x28
 #define PEM_CFG_RD 0x30
@@ -284,35 +288,16 @@ static int thunder_pem_config_write(struct pci_bus *bus, unsigned int devfn,
 	return pci_generic_config_write(bus, devfn, where, size, val);
 }
 
-static int thunder_pem_init(struct pci_config_window *cfg)
+static int thunder_pem_init(struct device *dev, struct pci_config_window *cfg,
+			    struct resource *res_pem)
 {
-	struct device *dev = cfg->parent;
-	resource_size_t bar4_start;
-	struct resource *res_pem;
 	struct thunder_pem_pci *pem_pci;
-	struct platform_device *pdev;
-
-	/* Only OF support for now */
-	if (!dev->of_node)
-		return -EINVAL;
+	resource_size_t bar4_start;
 
 	pem_pci = devm_kzalloc(dev, sizeof(*pem_pci), GFP_KERNEL);
 	if (!pem_pci)
 		return -ENOMEM;
 
-	pdev = to_platform_device(dev);
-
-	/*
-	 * The second register range is the PEM bridge to the PCIe
-	 * bus.  It has a different config access method than those
-	 * devices behind the bridge.
-	 */
-	res_pem = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-	if (!res_pem) {
-		dev_err(dev, "missing \"reg[1]\"property\n");
-		return -EINVAL;
-	}
-
 	pem_pci->pem_reg_base = devm_ioremap(dev, res_pem->start, 0x10000);
 	if (!pem_pci->pem_reg_base)
 		return -ENOMEM;
@@ -332,9 +317,69 @@ static int thunder_pem_init(struct pci_config_window *cfg)
 	return 0;
 }
 
+#if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)
+
+static int thunder_pem_acpi_init(struct pci_config_window *cfg)
+{
+	struct device *dev = cfg->parent;
+	struct acpi_device *adev = to_acpi_device(dev);
+	struct acpi_pci_root *root = acpi_driver_data(adev);
+	struct resource *res_pem;
+	int ret;
+
+	res_pem = devm_kzalloc(&adev->dev, sizeof(*res_pem), GFP_KERNEL);
+	if (!res_pem)
+		return -ENOMEM;
+
+	ret = acpi_get_rc_resources(dev, "THRX0002", root->segment, res_pem);
+	if (ret) {
+		dev_err(dev, "can't get rc base address\n");
+		return ret;
+	}
+
+	return thunder_pem_init(dev, cfg, res_pem);
+}
+
+struct pci_ecam_ops thunder_pem_ecam_ops = {
+	.bus_shift	= 24,
+	.init		= thunder_pem_acpi_init,
+	.pci_ops	= {
+		.map_bus	= pci_ecam_map_bus,
+		.read		= thunder_pem_config_read,
+		.write		= thunder_pem_config_write,
+	}
+};
+
+#endif
+
+#ifdef CONFIG_PCI_HOST_THUNDER_PEM
+
+static int thunder_pem_platform_init(struct pci_config_window *cfg)
+{
+	struct device *dev = cfg->parent;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct resource *res_pem;
+
+	if (!dev->of_node)
+		return -EINVAL;
+
+	/*
+	 * The second register range is the PEM bridge to the PCIe
+	 * bus.  It has a different config access method than those
+	 * devices behind the bridge.
+	 */
+	res_pem = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	if (!res_pem) {
+		dev_err(dev, "missing \"reg[1]\"property\n");
+		return -EINVAL;
+	}
+
+	return thunder_pem_init(dev, cfg, res_pem);
+}
+
 static struct pci_ecam_ops pci_thunder_pem_ops = {
 	.bus_shift	= 24,
-	.init		= thunder_pem_init,
+	.init		= thunder_pem_platform_init,
 	.pci_ops	= {
 		.map_bus	= pci_ecam_map_bus,
 		.read		= thunder_pem_config_read,
@@ -360,3 +405,6 @@ static struct platform_driver thunder_pem_driver = {
 	.probe = thunder_pem_probe,
 };
 builtin_platform_driver(thunder_pem_driver);
+
+#endif
+#endif
diff --git a/drivers/pci/host/pci-xgene.c b/drivers/pci/host/pci-xgene.c
index 1de23d74783f..7c3b54b9eb17 100644
--- a/drivers/pci/host/pci-xgene.c
+++ b/drivers/pci/host/pci-xgene.c
@@ -27,6 +27,8 @@
 #include <linux/of_irq.h>
 #include <linux/of_pci.h>
 #include <linux/pci.h>
+#include <linux/pci-acpi.h>
+#include <linux/pci-ecam.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 
@@ -64,7 +66,9 @@
 /* PCIe IP version */
 #define XGENE_PCIE_IP_VER_UNKN		0
 #define XGENE_PCIE_IP_VER_1		1
+#define XGENE_PCIE_IP_VER_2		2
 
+#if defined(CONFIG_PCI_XGENE) || (defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS))
 struct xgene_pcie_port {
 	struct device_node	*node;
 	struct device		*dev;
@@ -91,13 +95,24 @@ static inline u32 pcie_bar_low_val(u32 addr, u32 flags)
 	return (addr & PCI_BASE_ADDRESS_MEM_MASK) | flags;
 }
 
+static inline struct xgene_pcie_port *pcie_bus_to_port(struct pci_bus *bus)
+{
+	struct pci_config_window *cfg;
+
+	if (acpi_disabled)
+		return (struct xgene_pcie_port *)(bus->sysdata);
+
+	cfg = bus->sysdata;
+	return (struct xgene_pcie_port *)(cfg->priv);
+}
+
 /*
  * When the address bit [17:16] is 2'b01, the Configuration access will be
  * treated as Type 1 and it will be forwarded to external PCIe device.
  */
 static void __iomem *xgene_pcie_get_cfg_base(struct pci_bus *bus)
 {
-	struct xgene_pcie_port *port = bus->sysdata;
+	struct xgene_pcie_port *port = pcie_bus_to_port(bus);
 
 	if (bus->number >= (bus->primary + 1))
 		return port->cfg_base + AXI_EP_CFG_ACCESS;
@@ -111,7 +126,7 @@ static void __iomem *xgene_pcie_get_cfg_base(struct pci_bus *bus)
  */
 static void xgene_pcie_set_rtdid_reg(struct pci_bus *bus, uint devfn)
 {
-	struct xgene_pcie_port *port = bus->sysdata;
+	struct xgene_pcie_port *port = pcie_bus_to_port(bus);
 	unsigned int b, d, f;
 	u32 rtdid_val = 0;
 
@@ -158,7 +173,7 @@ static void __iomem *xgene_pcie_map_bus(struct pci_bus *bus, unsigned int devfn,
 static int xgene_pcie_config_read32(struct pci_bus *bus, unsigned int devfn,
 				    int where, int size, u32 *val)
 {
-	struct xgene_pcie_port *port = bus->sysdata;
+	struct xgene_pcie_port *port = pcie_bus_to_port(bus);
 
 	if (pci_generic_config_read32(bus, devfn, where & ~0x3, 4, val) !=
 	    PCIBIOS_SUCCESSFUL)
@@ -182,13 +197,103 @@ static int xgene_pcie_config_read32(struct pci_bus *bus, unsigned int devfn,
 
 	return PCIBIOS_SUCCESSFUL;
 }
+#endif
 
-static struct pci_ops xgene_pcie_ops = {
-	.map_bus = xgene_pcie_map_bus,
-	.read = xgene_pcie_config_read32,
-	.write = pci_generic_config_write32,
+#if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)
+static int xgene_get_csr_resource(struct acpi_device *adev,
+				  struct resource *res)
+{
+	struct device *dev = &adev->dev;
+	struct resource_entry *entry;
+	struct list_head list;
+	unsigned long flags;
+	int ret;
+
+	INIT_LIST_HEAD(&list);
+	flags = IORESOURCE_MEM;
+	ret = acpi_dev_get_resources(adev, &list,
+				     acpi_dev_filter_resource_type_cb,
+				     (void *) flags);
+	if (ret < 0) {
+		dev_err(dev, "failed to parse _CRS method, error code %d\n",
+			ret);
+		return ret;
+	}
+
+	if (ret == 0) {
+		dev_err(dev, "no IO and memory resources present in _CRS\n");
+		return -EINVAL;
+	}
+
+	entry = list_first_entry(&list, struct resource_entry, node);
+	*res = *entry->res;
+	acpi_dev_free_resource_list(&list);
+	return 0;
+}
+
+static int xgene_pcie_ecam_init(struct pci_config_window *cfg, u32 ipversion)
+{
+	struct device *dev = cfg->parent;
+	struct acpi_device *adev = to_acpi_device(dev);
+	struct xgene_pcie_port *port;
+	struct resource csr;
+	int ret;
+
+	port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
+	if (!port)
+		return -ENOMEM;
+
+	ret = xgene_get_csr_resource(adev, &csr);
+	if (ret) {
+		dev_err(dev, "can't get CSR resource\n");
+		kfree(port);
+		return ret;
+	}
+	port->csr_base = devm_ioremap_resource(dev, &csr);
+	if (IS_ERR(port->csr_base)) {
+		kfree(port);
+		return -ENOMEM;
+	}
+
+	port->cfg_base = cfg->win;
+	port->version = ipversion;
+
+	cfg->priv = port;
+	return 0;
+}
+
+static int xgene_v1_pcie_ecam_init(struct pci_config_window *cfg)
+{
+	return xgene_pcie_ecam_init(cfg, XGENE_PCIE_IP_VER_1);
+}
+
+struct pci_ecam_ops xgene_v1_pcie_ecam_ops = {
+	.bus_shift      = 16,
+	.init           = xgene_v1_pcie_ecam_init,
+	.pci_ops        = {
+		.map_bus        = xgene_pcie_map_bus,
+		.read           = xgene_pcie_config_read32,
+		.write          = pci_generic_config_write,
+	}
+};
+
+static int xgene_v2_pcie_ecam_init(struct pci_config_window *cfg)
+{
+	return xgene_pcie_ecam_init(cfg, XGENE_PCIE_IP_VER_2);
+}
+
+struct pci_ecam_ops xgene_v2_pcie_ecam_ops = {
+	.bus_shift      = 16,
+	.init           = xgene_v2_pcie_ecam_init,
+	.pci_ops        = {
+		.map_bus        = xgene_pcie_map_bus,
+		.read           = xgene_pcie_config_read32,
+		.write          = pci_generic_config_write,
+	}
 };
+#endif
 
+#if defined(CONFIG_PCI_XGENE)
 static u64 xgene_pcie_set_ib_mask(struct xgene_pcie_port *port, u32 addr,
 				  u32 flags, u64 size)
 {
@@ -521,6 +626,12 @@ static int xgene_pcie_setup(struct xgene_pcie_port *port,
 	return 0;
 }
 
+static struct pci_ops xgene_pcie_ops = {
+	.map_bus = xgene_pcie_map_bus,
+	.read = xgene_pcie_config_read32,
+	.write = pci_generic_config_write32,
+};
+
 static int xgene_pcie_probe_bridge(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -591,3 +702,4 @@ static struct platform_driver xgene_pcie_driver = {
 	.probe = xgene_pcie_probe_bridge,
 };
 builtin_platform_driver(xgene_pcie_driver);
+#endif
diff --git a/drivers/pci/host/pcie-altera.c b/drivers/pci/host/pcie-altera.c
index b0ac4dfafa0b..0c1540225ca3 100644
--- a/drivers/pci/host/pcie-altera.c
+++ b/drivers/pci/host/pcie-altera.c
@@ -550,10 +550,8 @@ static int altera_pcie_parse_dt(struct altera_pcie *pcie)
 
 	cra = platform_get_resource_byname(pdev, IORESOURCE_MEM, "Cra");
 	pcie->cra_base = devm_ioremap_resource(dev, cra);
-	if (IS_ERR(pcie->cra_base)) {
-		dev_err(dev, "failed to map cra memory\n");
+	if (IS_ERR(pcie->cra_base))
 		return PTR_ERR(pcie->cra_base);
-	}
 
 	/* setup IRQ */
 	pcie->irq = platform_get_irq(pdev, 0);
@@ -641,8 +639,4 @@ static struct platform_driver altera_pcie_driver = {
 	},
 };
 
-static int altera_pcie_init(void)
-{
-	return platform_driver_register(&altera_pcie_driver);
-}
-device_initcall(altera_pcie_init);
+builtin_platform_driver(altera_pcie_driver);
diff --git a/drivers/pci/host/pcie-hisi.c b/drivers/pci/host/pcie-hisi.c
index 56154c25980c..a301a7187b30 100644
--- a/drivers/pci/host/pcie-hisi.c
+++ b/drivers/pci/host/pcie-hisi.c
@@ -18,7 +18,106 @@
 #include <linux/of_pci.h>
 #include <linux/platform_device.h>
 #include <linux/of_device.h>
+#include <linux/pci.h>
+#include <linux/pci-acpi.h>
+#include <linux/pci-ecam.h>
 #include <linux/regmap.h>
+#include "../pci.h"
+
+#if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)
+
+static int hisi_pcie_acpi_rd_conf(struct pci_bus *bus, u32 devfn, int where,
+				  int size, u32 *val)
+{
+	struct pci_config_window *cfg = bus->sysdata;
+	int dev = PCI_SLOT(devfn);
+
+	if (bus->number == cfg->busr.start) {
+		/* access only one slot on each root port */
+		if (dev > 0)
+			return PCIBIOS_DEVICE_NOT_FOUND;
+		else
+			return pci_generic_config_read32(bus, devfn, where,
+							 size, val);
+	}
+
+	return pci_generic_config_read(bus, devfn, where, size, val);
+}
+
+static int hisi_pcie_acpi_wr_conf(struct pci_bus *bus, u32 devfn,
+				  int where, int size, u32 val)
+{
+	struct pci_config_window *cfg = bus->sysdata;
+	int dev = PCI_SLOT(devfn);
+
+	if (bus->number == cfg->busr.start) {
+		/* access only one slot on each root port */
+		if (dev > 0)
+			return PCIBIOS_DEVICE_NOT_FOUND;
+		else
+			return pci_generic_config_write32(bus, devfn, where,
+							  size, val);
+	}
+
+	return pci_generic_config_write(bus, devfn, where, size, val);
+}
+
+static void __iomem *hisi_pcie_map_bus(struct pci_bus *bus, unsigned int devfn,
+				       int where)
+{
+	struct pci_config_window *cfg = bus->sysdata;
+	void __iomem *reg_base = cfg->priv;
+
+	if (bus->number == cfg->busr.start)
+		return reg_base + where;
+	else
+		return pci_ecam_map_bus(bus, devfn, where);
+}
+
+static int hisi_pcie_init(struct pci_config_window *cfg)
+{
+	struct device *dev = cfg->parent;
+	struct acpi_device *adev = to_acpi_device(dev);
+	struct acpi_pci_root *root = acpi_driver_data(adev);
+	struct resource *res;
+	void __iomem *reg_base;
+	int ret;
+
+	/*
+	 * Retrieve RC base and size from a HISI0081 device with _UID
+	 * matching our segment.
+	 */
+	res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
+	if (!res)
+		return -ENOMEM;
+
+	ret = acpi_get_rc_resources(dev, "HISI0081", root->segment, res);
+	if (ret) {
+		dev_err(dev, "can't get rc base address\n");
+		return -ENOMEM;
+	}
+
+	reg_base = devm_ioremap(dev, res->start, resource_size(res));
+	if (!reg_base)
+		return -ENOMEM;
+
+	cfg->priv = reg_base;
+	return 0;
+}
+
+struct pci_ecam_ops hisi_pcie_ops = {
+	.bus_shift    = 20,
+	.init         =  hisi_pcie_init,
+	.pci_ops      = {
+		.map_bus    = hisi_pcie_map_bus,
+		.read       = hisi_pcie_acpi_rd_conf,
+		.write      = hisi_pcie_acpi_wr_conf,
+	}
+};
+
+#endif
+
+#ifdef CONFIG_PCI_HISI
 
 #include "pcie-designware.h"
 
@@ -185,17 +284,13 @@ static int hisi_pcie_probe(struct platform_device *pdev)
 
 	reg = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rc_dbi");
 	pp->dbi_base = devm_ioremap_resource(dev, reg);
-	if (IS_ERR(pp->dbi_base)) {
-		dev_err(dev, "cannot get rc_dbi base\n");
+	if (IS_ERR(pp->dbi_base))
 		return PTR_ERR(pp->dbi_base);
-	}
 
 	ret = hisi_add_pcie_port(hisi_pcie, pdev);
 	if (ret)
 		return ret;
 
-	dev_warn(dev, "only 32-bit config accesses supported; smaller writes may corrupt adjacent RW1C fields\n");
-
 	return 0;
 }
 
@@ -227,3 +322,5 @@ static struct platform_driver hisi_pcie_driver = {
 	},
 };
 builtin_platform_driver(hisi_pcie_driver);
+
+#endif
diff --git a/drivers/pci/host/pcie-iproc-bcma.c b/drivers/pci/host/pcie-iproc-bcma.c
index 8ce089043a27..bd4c9ec25edc 100644
--- a/drivers/pci/host/pcie-iproc-bcma.c
+++ b/drivers/pci/host/pcie-iproc-bcma.c
@@ -54,6 +54,7 @@ static int iproc_pcie_bcma_probe(struct bcma_device *bdev)
 
 	pcie->dev = dev;
 
+	pcie->type = IPROC_PCIE_PAXB_BCMA;
 	pcie->base = bdev->io_addr;
 	if (!pcie->base) {
 		dev_err(dev, "no controller registers\n");
diff --git a/drivers/pci/host/pcie-iproc-msi.c b/drivers/pci/host/pcie-iproc-msi.c
index 9a2973bdc78a..9fad7915f82a 100644
--- a/drivers/pci/host/pcie-iproc-msi.c
+++ b/drivers/pci/host/pcie-iproc-msi.c
@@ -563,6 +563,7 @@ int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node)
 	}
 
 	switch (pcie->type) {
+	case IPROC_PCIE_PAXB_BCMA:
 	case IPROC_PCIE_PAXB:
 		msi->reg_offsets = iproc_msi_reg_paxb;
 		msi->nr_eq_region = 1;
diff --git a/drivers/pci/host/pcie-iproc-platform.c b/drivers/pci/host/pcie-iproc-platform.c
index a3de087976b3..22d814a78a78 100644
--- a/drivers/pci/host/pcie-iproc-platform.c
+++ b/drivers/pci/host/pcie-iproc-platform.c
@@ -31,8 +31,14 @@ static const struct of_device_id iproc_pcie_of_match_table[] = {
 		.compatible = "brcm,iproc-pcie",
 		.data = (int *)IPROC_PCIE_PAXB,
 	}, {
+		.compatible = "brcm,iproc-pcie-paxb-v2",
+		.data = (int *)IPROC_PCIE_PAXB_V2,
+	}, {
 		.compatible = "brcm,iproc-pcie-paxc",
 		.data = (int *)IPROC_PCIE_PAXC,
+	}, {
+		.compatible = "brcm,iproc-pcie-paxc-v2",
+		.data = (int *)IPROC_PCIE_PAXC_V2,
 	},
 	{ /* sentinel */ }
 };
@@ -84,19 +90,6 @@ static int iproc_pcie_pltfm_probe(struct platform_device *pdev)
 			return ret;
 		}
 		pcie->ob.axi_offset = val;
-
-		ret = of_property_read_u32(np, "brcm,pcie-ob-window-size",
-					   &val);
-		if (ret) {
-			dev_err(dev,
-				"missing brcm,pcie-ob-window-size property\n");
-			return ret;
-		}
-		pcie->ob.window_size = (resource_size_t)val * SZ_1M;
-
-		if (of_property_read_bool(np, "brcm,pcie-ob-oarr-size"))
-			pcie->ob.set_oarr_size = true;
-
 		pcie->need_ob_cfg = true;
 	}
 
@@ -115,7 +108,14 @@ static int iproc_pcie_pltfm_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	pcie->map_irq = of_irq_parse_and_map_pci;
+	/* PAXC doesn't support legacy IRQs, skip mapping */
+	switch (pcie->type) {
+	case IPROC_PCIE_PAXC:
+	case IPROC_PCIE_PAXC_V2:
+		break;
+	default:
+		pcie->map_irq = of_irq_parse_and_map_pci;
+	}
 
 	ret = iproc_pcie_setup(pcie, &res);
 	if (ret)
diff --git a/drivers/pci/host/pcie-iproc.c b/drivers/pci/host/pcie-iproc.c
index 0b999a9fb843..3ebc025499b9 100644
--- a/drivers/pci/host/pcie-iproc.c
+++ b/drivers/pci/host/pcie-iproc.c
@@ -21,6 +21,7 @@
 #include <linux/slab.h>
 #include <linux/delay.h>
 #include <linux/interrupt.h>
+#include <linux/irqchip/arm-gic-v3.h>
 #include <linux/platform_device.h>
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
@@ -38,6 +39,12 @@
 #define RC_PCIE_RST_OUTPUT           BIT(RC_PCIE_RST_OUTPUT_SHIFT)
 #define PAXC_RESET_MASK              0x7f
 
+#define GIC_V3_CFG_SHIFT             0
+#define GIC_V3_CFG                   BIT(GIC_V3_CFG_SHIFT)
+
+#define MSI_ENABLE_CFG_SHIFT         0
+#define MSI_ENABLE_CFG               BIT(MSI_ENABLE_CFG_SHIFT)
+
 #define CFG_IND_ADDR_MASK            0x00001ffc
 
 #define CFG_ADDR_BUS_NUM_SHIFT       20
@@ -58,59 +65,319 @@
 #define PCIE_DL_ACTIVE_SHIFT         2
 #define PCIE_DL_ACTIVE               BIT(PCIE_DL_ACTIVE_SHIFT)
 
+#define APB_ERR_EN_SHIFT             0
+#define APB_ERR_EN                   BIT(APB_ERR_EN_SHIFT)
+
+/* derive the enum index of the outbound/inbound mapping registers */
+#define MAP_REG(base_reg, index)      ((base_reg) + (index) * 2)
+
+/*
+ * Maximum number of outbound mapping window sizes that can be supported by any
+ * OARR/OMAP mapping pair
+ */
+#define MAX_NUM_OB_WINDOW_SIZES      4
+
 #define OARR_VALID_SHIFT             0
 #define OARR_VALID                   BIT(OARR_VALID_SHIFT)
 #define OARR_SIZE_CFG_SHIFT          1
-#define OARR_SIZE_CFG                BIT(OARR_SIZE_CFG_SHIFT)
 
-#define PCI_EXP_CAP			0xac
+/*
+ * Maximum number of inbound mapping region sizes that can be supported by an
+ * IARR
+ */
+#define MAX_NUM_IB_REGION_SIZES      9
+
+#define IMAP_VALID_SHIFT             0
+#define IMAP_VALID                   BIT(IMAP_VALID_SHIFT)
 
-#define MAX_NUM_OB_WINDOWS           2
+#define PCI_EXP_CAP			0xac
 
 #define IPROC_PCIE_REG_INVALID 0xffff
 
+/**
+ * iProc PCIe outbound mapping controller specific parameters
+ *
+ * @window_sizes: list of supported outbound mapping window sizes in MB
+ * @nr_sizes: number of supported outbound mapping window sizes
+ */
+struct iproc_pcie_ob_map {
+	resource_size_t window_sizes[MAX_NUM_OB_WINDOW_SIZES];
+	unsigned int nr_sizes;
+};
+
+static const struct iproc_pcie_ob_map paxb_ob_map[] = {
+	{
+		/* OARR0/OMAP0 */
+		.window_sizes = { 128, 256 },
+		.nr_sizes = 2,
+	},
+	{
+		/* OARR1/OMAP1 */
+		.window_sizes = { 128, 256 },
+		.nr_sizes = 2,
+	},
+};
+
+static const struct iproc_pcie_ob_map paxb_v2_ob_map[] = {
+	{
+		/* OARR0/OMAP0 */
+		.window_sizes = { 128, 256 },
+		.nr_sizes = 2,
+	},
+	{
+		/* OARR1/OMAP1 */
+		.window_sizes = { 128, 256 },
+		.nr_sizes = 2,
+	},
+	{
+		/* OARR2/OMAP2 */
+		.window_sizes = { 128, 256, 512, 1024 },
+		.nr_sizes = 4,
+	},
+	{
+		/* OARR3/OMAP3 */
+		.window_sizes = { 128, 256, 512, 1024 },
+		.nr_sizes = 4,
+	},
+};
+
+/**
+ * iProc PCIe inbound mapping type
+ */
+enum iproc_pcie_ib_map_type {
+	/* for DDR memory */
+	IPROC_PCIE_IB_MAP_MEM = 0,
+
+	/* for device I/O memory */
+	IPROC_PCIE_IB_MAP_IO,
+
+	/* invalid or unused */
+	IPROC_PCIE_IB_MAP_INVALID
+};
+
+/**
+ * iProc PCIe inbound mapping controller specific parameters
+ *
+ * @type: inbound mapping region type
+ * @size_unit: inbound mapping region size unit, could be SZ_1K, SZ_1M, or
+ * SZ_1G
+ * @region_sizes: list of supported inbound mapping region sizes in KB, MB, or
+ * GB, depedning on the size unit
+ * @nr_sizes: number of supported inbound mapping region sizes
+ * @nr_windows: number of supported inbound mapping windows for the region
+ * @imap_addr_offset: register offset between the upper and lower 32-bit
+ * IMAP address registers
+ * @imap_window_offset: register offset between each IMAP window
+ */
+struct iproc_pcie_ib_map {
+	enum iproc_pcie_ib_map_type type;
+	unsigned int size_unit;
+	resource_size_t region_sizes[MAX_NUM_IB_REGION_SIZES];
+	unsigned int nr_sizes;
+	unsigned int nr_windows;
+	u16 imap_addr_offset;
+	u16 imap_window_offset;
+};
+
+static const struct iproc_pcie_ib_map paxb_v2_ib_map[] = {
+	{
+		/* IARR0/IMAP0 */
+		.type = IPROC_PCIE_IB_MAP_IO,
+		.size_unit = SZ_1K,
+		.region_sizes = { 32 },
+		.nr_sizes = 1,
+		.nr_windows = 8,
+		.imap_addr_offset = 0x40,
+		.imap_window_offset = 0x4,
+	},
+	{
+		/* IARR1/IMAP1 (currently unused) */
+		.type = IPROC_PCIE_IB_MAP_INVALID,
+	},
+	{
+		/* IARR2/IMAP2 */
+		.type = IPROC_PCIE_IB_MAP_MEM,
+		.size_unit = SZ_1M,
+		.region_sizes = { 64, 128, 256, 512, 1024, 2048, 4096, 8192,
+				  16384 },
+		.nr_sizes = 9,
+		.nr_windows = 1,
+		.imap_addr_offset = 0x4,
+		.imap_window_offset = 0x8,
+	},
+	{
+		/* IARR3/IMAP3 */
+		.type = IPROC_PCIE_IB_MAP_MEM,
+		.size_unit = SZ_1G,
+		.region_sizes = { 1, 2, 4, 8, 16, 32 },
+		.nr_sizes = 6,
+		.nr_windows = 8,
+		.imap_addr_offset = 0x4,
+		.imap_window_offset = 0x8,
+	},
+	{
+		/* IARR4/IMAP4 */
+		.type = IPROC_PCIE_IB_MAP_MEM,
+		.size_unit = SZ_1G,
+		.region_sizes = { 32, 64, 128, 256, 512 },
+		.nr_sizes = 5,
+		.nr_windows = 8,
+		.imap_addr_offset = 0x4,
+		.imap_window_offset = 0x8,
+	},
+};
+
+/*
+ * iProc PCIe host registers
+ */
 enum iproc_pcie_reg {
+	/* clock/reset signal control */
 	IPROC_PCIE_CLK_CTRL = 0,
+
+	/*
+	 * To allow MSI to be steered to an external MSI controller (e.g., ARM
+	 * GICv3 ITS)
+	 */
+	IPROC_PCIE_MSI_GIC_MODE,
+
+	/*
+	 * IPROC_PCIE_MSI_BASE_ADDR and IPROC_PCIE_MSI_WINDOW_SIZE define the
+	 * window where the MSI posted writes are written, for the writes to be
+	 * interpreted as MSI writes.
+	 */
+	IPROC_PCIE_MSI_BASE_ADDR,
+	IPROC_PCIE_MSI_WINDOW_SIZE,
+
+	/*
+	 * To hold the address of the register where the MSI writes are
+	 * programed.  When ARM GICv3 ITS is used, this should be programmed
+	 * with the address of the GITS_TRANSLATER register.
+	 */
+	IPROC_PCIE_MSI_ADDR_LO,
+	IPROC_PCIE_MSI_ADDR_HI,
+
+	/* enable MSI */
+	IPROC_PCIE_MSI_EN_CFG,
+
+	/* allow access to root complex configuration space */
 	IPROC_PCIE_CFG_IND_ADDR,
 	IPROC_PCIE_CFG_IND_DATA,
+
+	/* allow access to device configuration space */
 	IPROC_PCIE_CFG_ADDR,
 	IPROC_PCIE_CFG_DATA,
+
+	/* enable INTx */
 	IPROC_PCIE_INTX_EN,
-	IPROC_PCIE_OARR_LO,
-	IPROC_PCIE_OARR_HI,
-	IPROC_PCIE_OMAP_LO,
-	IPROC_PCIE_OMAP_HI,
+
+	/* outbound address mapping */
+	IPROC_PCIE_OARR0,
+	IPROC_PCIE_OMAP0,
+	IPROC_PCIE_OARR1,
+	IPROC_PCIE_OMAP1,
+	IPROC_PCIE_OARR2,
+	IPROC_PCIE_OMAP2,
+	IPROC_PCIE_OARR3,
+	IPROC_PCIE_OMAP3,
+
+	/* inbound address mapping */
+	IPROC_PCIE_IARR0,
+	IPROC_PCIE_IMAP0,
+	IPROC_PCIE_IARR1,
+	IPROC_PCIE_IMAP1,
+	IPROC_PCIE_IARR2,
+	IPROC_PCIE_IMAP2,
+	IPROC_PCIE_IARR3,
+	IPROC_PCIE_IMAP3,
+	IPROC_PCIE_IARR4,
+	IPROC_PCIE_IMAP4,
+
+	/* link status */
 	IPROC_PCIE_LINK_STATUS,
+
+	/* enable APB error for unsupported requests */
+	IPROC_PCIE_APB_ERR_EN,
+
+	/* total number of core registers */
+	IPROC_PCIE_MAX_NUM_REG,
+};
+
+/* iProc PCIe PAXB BCMA registers */
+static const u16 iproc_pcie_reg_paxb_bcma[] = {
+	[IPROC_PCIE_CLK_CTRL]         = 0x000,
+	[IPROC_PCIE_CFG_IND_ADDR]     = 0x120,
+	[IPROC_PCIE_CFG_IND_DATA]     = 0x124,
+	[IPROC_PCIE_CFG_ADDR]         = 0x1f8,
+	[IPROC_PCIE_CFG_DATA]         = 0x1fc,
+	[IPROC_PCIE_INTX_EN]          = 0x330,
+	[IPROC_PCIE_LINK_STATUS]      = 0xf0c,
 };
 
 /* iProc PCIe PAXB registers */
 static const u16 iproc_pcie_reg_paxb[] = {
-	[IPROC_PCIE_CLK_CTRL]     = 0x000,
-	[IPROC_PCIE_CFG_IND_ADDR] = 0x120,
-	[IPROC_PCIE_CFG_IND_DATA] = 0x124,
-	[IPROC_PCIE_CFG_ADDR]     = 0x1f8,
-	[IPROC_PCIE_CFG_DATA]     = 0x1fc,
-	[IPROC_PCIE_INTX_EN]      = 0x330,
-	[IPROC_PCIE_OARR_LO]      = 0xd20,
-	[IPROC_PCIE_OARR_HI]      = 0xd24,
-	[IPROC_PCIE_OMAP_LO]      = 0xd40,
-	[IPROC_PCIE_OMAP_HI]      = 0xd44,
-	[IPROC_PCIE_LINK_STATUS]  = 0xf0c,
+	[IPROC_PCIE_CLK_CTRL]         = 0x000,
+	[IPROC_PCIE_CFG_IND_ADDR]     = 0x120,
+	[IPROC_PCIE_CFG_IND_DATA]     = 0x124,
+	[IPROC_PCIE_CFG_ADDR]         = 0x1f8,
+	[IPROC_PCIE_CFG_DATA]         = 0x1fc,
+	[IPROC_PCIE_INTX_EN]          = 0x330,
+	[IPROC_PCIE_OARR0]            = 0xd20,
+	[IPROC_PCIE_OMAP0]            = 0xd40,
+	[IPROC_PCIE_OARR1]            = 0xd28,
+	[IPROC_PCIE_OMAP1]            = 0xd48,
+	[IPROC_PCIE_LINK_STATUS]      = 0xf0c,
+	[IPROC_PCIE_APB_ERR_EN]       = 0xf40,
+};
+
+/* iProc PCIe PAXB v2 registers */
+static const u16 iproc_pcie_reg_paxb_v2[] = {
+	[IPROC_PCIE_CLK_CTRL]         = 0x000,
+	[IPROC_PCIE_CFG_IND_ADDR]     = 0x120,
+	[IPROC_PCIE_CFG_IND_DATA]     = 0x124,
+	[IPROC_PCIE_CFG_ADDR]         = 0x1f8,
+	[IPROC_PCIE_CFG_DATA]         = 0x1fc,
+	[IPROC_PCIE_INTX_EN]          = 0x330,
+	[IPROC_PCIE_OARR0]            = 0xd20,
+	[IPROC_PCIE_OMAP0]            = 0xd40,
+	[IPROC_PCIE_OARR1]            = 0xd28,
+	[IPROC_PCIE_OMAP1]            = 0xd48,
+	[IPROC_PCIE_OARR2]            = 0xd60,
+	[IPROC_PCIE_OMAP2]            = 0xd68,
+	[IPROC_PCIE_OARR3]            = 0xdf0,
+	[IPROC_PCIE_OMAP3]            = 0xdf8,
+	[IPROC_PCIE_IARR0]            = 0xd00,
+	[IPROC_PCIE_IMAP0]            = 0xc00,
+	[IPROC_PCIE_IARR2]            = 0xd10,
+	[IPROC_PCIE_IMAP2]            = 0xcc0,
+	[IPROC_PCIE_IARR3]            = 0xe00,
+	[IPROC_PCIE_IMAP3]            = 0xe08,
+	[IPROC_PCIE_IARR4]            = 0xe68,
+	[IPROC_PCIE_IMAP4]            = 0xe70,
+	[IPROC_PCIE_LINK_STATUS]      = 0xf0c,
+	[IPROC_PCIE_APB_ERR_EN]       = 0xf40,
 };
 
 /* iProc PCIe PAXC v1 registers */
 static const u16 iproc_pcie_reg_paxc[] = {
-	[IPROC_PCIE_CLK_CTRL]     = 0x000,
-	[IPROC_PCIE_CFG_IND_ADDR] = 0x1f0,
-	[IPROC_PCIE_CFG_IND_DATA] = 0x1f4,
-	[IPROC_PCIE_CFG_ADDR]     = 0x1f8,
-	[IPROC_PCIE_CFG_DATA]     = 0x1fc,
-	[IPROC_PCIE_INTX_EN]      = IPROC_PCIE_REG_INVALID,
-	[IPROC_PCIE_OARR_LO]      = IPROC_PCIE_REG_INVALID,
-	[IPROC_PCIE_OARR_HI]      = IPROC_PCIE_REG_INVALID,
-	[IPROC_PCIE_OMAP_LO]      = IPROC_PCIE_REG_INVALID,
-	[IPROC_PCIE_OMAP_HI]      = IPROC_PCIE_REG_INVALID,
-	[IPROC_PCIE_LINK_STATUS]  = IPROC_PCIE_REG_INVALID,
+	[IPROC_PCIE_CLK_CTRL]         = 0x000,
+	[IPROC_PCIE_CFG_IND_ADDR]     = 0x1f0,
+	[IPROC_PCIE_CFG_IND_DATA]     = 0x1f4,
+	[IPROC_PCIE_CFG_ADDR]         = 0x1f8,
+	[IPROC_PCIE_CFG_DATA]         = 0x1fc,
+};
+
+/* iProc PCIe PAXC v2 registers */
+static const u16 iproc_pcie_reg_paxc_v2[] = {
+	[IPROC_PCIE_MSI_GIC_MODE]     = 0x050,
+	[IPROC_PCIE_MSI_BASE_ADDR]    = 0x074,
+	[IPROC_PCIE_MSI_WINDOW_SIZE]  = 0x078,
+	[IPROC_PCIE_MSI_ADDR_LO]      = 0x07c,
+	[IPROC_PCIE_MSI_ADDR_HI]      = 0x080,
+	[IPROC_PCIE_MSI_EN_CFG]       = 0x09c,
+	[IPROC_PCIE_CFG_IND_ADDR]     = 0x1f0,
+	[IPROC_PCIE_CFG_IND_DATA]     = 0x1f4,
+	[IPROC_PCIE_CFG_ADDR]         = 0x1f8,
+	[IPROC_PCIE_CFG_DATA]         = 0x1fc,
 };
 
 static inline struct iproc_pcie *iproc_data(struct pci_bus *bus)
@@ -159,16 +426,26 @@ static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie,
 	writel(val, pcie->base + offset);
 }
 
-static inline void iproc_pcie_ob_write(struct iproc_pcie *pcie,
-				       enum iproc_pcie_reg reg,
-				       unsigned window, u32 val)
+/**
+ * APB error forwarding can be disabled during access of configuration
+ * registers of the endpoint device, to prevent unsupported requests
+ * (typically seen during enumeration with multi-function devices) from
+ * triggering a system exception.
+ */
+static inline void iproc_pcie_apb_err_disable(struct pci_bus *bus,
+					      bool disable)
 {
-	u16 offset = iproc_pcie_reg_offset(pcie, reg);
-
-	if (iproc_pcie_reg_is_invalid(offset))
-		return;
+	struct iproc_pcie *pcie = iproc_data(bus);
+	u32 val;
 
-	writel(val, pcie->base + offset + (window * 8));
+	if (bus->number && pcie->has_apb_err_disable) {
+		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_APB_ERR_EN);
+		if (disable)
+			val &= ~APB_ERR_EN;
+		else
+			val |= APB_ERR_EN;
+		iproc_pcie_write_reg(pcie, IPROC_PCIE_APB_ERR_EN, val);
+	}
 }
 
 /**
@@ -204,7 +481,7 @@ static void __iomem *iproc_pcie_map_cfg_bus(struct pci_bus *bus,
 	 * PAXC is connected to an internally emulated EP within the SoC.  It
 	 * allows only one device.
 	 */
-	if (pcie->type == IPROC_PCIE_PAXC)
+	if (pcie->ep_is_internal)
 		if (slot > 0)
 			return NULL;
 
@@ -222,26 +499,47 @@ static void __iomem *iproc_pcie_map_cfg_bus(struct pci_bus *bus,
 		return (pcie->base + offset);
 }
 
+static int iproc_pcie_config_read32(struct pci_bus *bus, unsigned int devfn,
+				    int where, int size, u32 *val)
+{
+	int ret;
+
+	iproc_pcie_apb_err_disable(bus, true);
+	ret = pci_generic_config_read32(bus, devfn, where, size, val);
+	iproc_pcie_apb_err_disable(bus, false);
+
+	return ret;
+}
+
+static int iproc_pcie_config_write32(struct pci_bus *bus, unsigned int devfn,
+				     int where, int size, u32 val)
+{
+	int ret;
+
+	iproc_pcie_apb_err_disable(bus, true);
+	ret = pci_generic_config_write32(bus, devfn, where, size, val);
+	iproc_pcie_apb_err_disable(bus, false);
+
+	return ret;
+}
+
 static struct pci_ops iproc_pcie_ops = {
 	.map_bus = iproc_pcie_map_cfg_bus,
-	.read = pci_generic_config_read32,
-	.write = pci_generic_config_write32,
+	.read = iproc_pcie_config_read32,
+	.write = iproc_pcie_config_write32,
 };
 
 static void iproc_pcie_reset(struct iproc_pcie *pcie)
 {
 	u32 val;
 
-	if (pcie->type == IPROC_PCIE_PAXC) {
-		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);
-		val &= ~PAXC_RESET_MASK;
-		iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
-		udelay(100);
-		val |= PAXC_RESET_MASK;
-		iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
-		udelay(100);
+	/*
+	 * PAXC and the internal emulated endpoint device downstream should not
+	 * be reset.  If firmware has been loaded on the endpoint device at an
+	 * earlier boot stage, reset here causes issues.
+	 */
+	if (pcie->ep_is_internal)
 		return;
-	}
 
 	/*
 	 * Select perst_b signal as reset source. Put the device into reset,
@@ -270,7 +568,7 @@ static int iproc_pcie_check_link(struct iproc_pcie *pcie, struct pci_bus *bus)
 	 * PAXC connects to emulated endpoint devices directly and does not
 	 * have a Serdes.  Therefore skip the link detection logic here.
 	 */
-	if (pcie->type == IPROC_PCIE_PAXC)
+	if (pcie->ep_is_internal)
 		return 0;
 
 	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_LINK_STATUS);
@@ -334,6 +632,58 @@ static void iproc_pcie_enable(struct iproc_pcie *pcie)
 	iproc_pcie_write_reg(pcie, IPROC_PCIE_INTX_EN, SYS_RC_INTX_MASK);
 }
 
+static inline bool iproc_pcie_ob_is_valid(struct iproc_pcie *pcie,
+					  int window_idx)
+{
+	u32 val;
+
+	val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_OARR0, window_idx));
+
+	return !!(val & OARR_VALID);
+}
+
+static inline int iproc_pcie_ob_write(struct iproc_pcie *pcie, int window_idx,
+				      int size_idx, u64 axi_addr, u64 pci_addr)
+{
+	struct device *dev = pcie->dev;
+	u16 oarr_offset, omap_offset;
+
+	/*
+	 * Derive the OARR/OMAP offset from the first pair (OARR0/OMAP0) based
+	 * on window index.
+	 */
+	oarr_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OARR0,
+							  window_idx));
+	omap_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OMAP0,
+							  window_idx));
+	if (iproc_pcie_reg_is_invalid(oarr_offset) ||
+	    iproc_pcie_reg_is_invalid(omap_offset))
+		return -EINVAL;
+
+	/*
+	 * Program the OARR registers.  The upper 32-bit OARR register is
+	 * always right after the lower 32-bit OARR register.
+	 */
+	writel(lower_32_bits(axi_addr) | (size_idx << OARR_SIZE_CFG_SHIFT) |
+	       OARR_VALID, pcie->base + oarr_offset);
+	writel(upper_32_bits(axi_addr), pcie->base + oarr_offset + 4);
+
+	/* now program the OMAP registers */
+	writel(lower_32_bits(pci_addr), pcie->base + omap_offset);
+	writel(upper_32_bits(pci_addr), pcie->base + omap_offset + 4);
+
+	dev_info(dev, "ob window [%d]: offset 0x%x axi %pap pci %pap\n",
+		 window_idx, oarr_offset, &axi_addr, &pci_addr);
+	dev_info(dev, "oarr lo 0x%x oarr hi 0x%x\n",
+		 readl(pcie->base + oarr_offset),
+		 readl(pcie->base + oarr_offset + 4));
+	dev_info(dev, "omap lo 0x%x omap hi 0x%x\n",
+		 readl(pcie->base + omap_offset),
+		 readl(pcie->base + omap_offset + 4));
+
+	return 0;
+}
+
 /**
  * Some iProc SoCs require the SW to configure the outbound address mapping
  *
@@ -350,24 +700,7 @@ static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr,
 {
 	struct iproc_pcie_ob *ob = &pcie->ob;
 	struct device *dev = pcie->dev;
-	unsigned i;
-	u64 max_size = (u64)ob->window_size * MAX_NUM_OB_WINDOWS;
-	u64 remainder;
-
-	if (size > max_size) {
-		dev_err(dev,
-			"res size %pap exceeds max supported size 0x%llx\n",
-			&size, max_size);
-		return -EINVAL;
-	}
-
-	div64_u64_rem(size, ob->window_size, &remainder);
-	if (remainder) {
-		dev_err(dev,
-			"res size %pap needs to be multiple of window size %pap\n",
-			&size, &ob->window_size);
-		return -EINVAL;
-	}
+	int ret = -EINVAL, window_idx, size_idx;
 
 	if (axi_addr < ob->axi_offset) {
 		dev_err(dev, "axi address %pap less than offset %pap\n",
@@ -381,26 +714,70 @@ static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr,
 	 */
 	axi_addr -= ob->axi_offset;
 
-	for (i = 0; i < MAX_NUM_OB_WINDOWS; i++) {
-		iproc_pcie_ob_write(pcie, IPROC_PCIE_OARR_LO, i,
-				    lower_32_bits(axi_addr) | OARR_VALID |
-				    (ob->set_oarr_size ? 1 : 0));
-		iproc_pcie_ob_write(pcie, IPROC_PCIE_OARR_HI, i,
-				    upper_32_bits(axi_addr));
-		iproc_pcie_ob_write(pcie, IPROC_PCIE_OMAP_LO, i,
-				    lower_32_bits(pci_addr));
-		iproc_pcie_ob_write(pcie, IPROC_PCIE_OMAP_HI, i,
-				    upper_32_bits(pci_addr));
-
-		size -= ob->window_size;
-		if (size == 0)
+	/* iterate through all OARR/OMAP mapping windows */
+	for (window_idx = ob->nr_windows - 1; window_idx >= 0; window_idx--) {
+		const struct iproc_pcie_ob_map *ob_map =
+			&pcie->ob_map[window_idx];
+
+		/*
+		 * If current outbound window is already in use, move on to the
+		 * next one.
+		 */
+		if (iproc_pcie_ob_is_valid(pcie, window_idx))
+			continue;
+
+		/*
+		 * Iterate through all supported window sizes within the
+		 * OARR/OMAP pair to find a match.  Go through the window sizes
+		 * in a descending order.
+		 */
+		for (size_idx = ob_map->nr_sizes - 1; size_idx >= 0;
+		     size_idx--) {
+			resource_size_t window_size =
+				ob_map->window_sizes[size_idx] * SZ_1M;
+
+			if (size < window_size)
+				continue;
+
+			if (!IS_ALIGNED(axi_addr, window_size) ||
+			    !IS_ALIGNED(pci_addr, window_size)) {
+				dev_err(dev,
+					"axi %pap or pci %pap not aligned\n",
+					&axi_addr, &pci_addr);
+				return -EINVAL;
+			}
+
+			/*
+			 * Match found!  Program both OARR and OMAP and mark
+			 * them as a valid entry.
+			 */
+			ret = iproc_pcie_ob_write(pcie, window_idx, size_idx,
+						  axi_addr, pci_addr);
+			if (ret)
+				goto err_ob;
+
+			size -= window_size;
+			if (size == 0)
+				return 0;
+
+			/*
+			 * If we are here, we are done with the current window,
+			 * but not yet finished all mappings.  Need to move on
+			 * to the next window.
+			 */
+			axi_addr += window_size;
+			pci_addr += window_size;
 			break;
-
-		axi_addr += ob->window_size;
-		pci_addr += ob->window_size;
+		}
 	}
 
-	return 0;
+err_ob:
+	dev_err(dev, "unable to configure outbound mapping\n");
+	dev_err(dev,
+		"axi %pap, axi offset %pap, pci %pap, res size %pap\n",
+		&axi_addr, &ob->axi_offset, &pci_addr, &size);
+
+	return ret;
 }
 
 static int iproc_pcie_map_ranges(struct iproc_pcie *pcie,
@@ -434,13 +811,323 @@ static int iproc_pcie_map_ranges(struct iproc_pcie *pcie,
 	return 0;
 }
 
+static inline bool iproc_pcie_ib_is_in_use(struct iproc_pcie *pcie,
+					   int region_idx)
+{
+	const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx];
+	u32 val;
+
+	val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_IARR0, region_idx));
+
+	return !!(val & (BIT(ib_map->nr_sizes) - 1));
+}
+
+static inline bool iproc_pcie_ib_check_type(const struct iproc_pcie_ib_map *ib_map,
+					    enum iproc_pcie_ib_map_type type)
+{
+	return !!(ib_map->type == type);
+}
+
+static int iproc_pcie_ib_write(struct iproc_pcie *pcie, int region_idx,
+			       int size_idx, int nr_windows, u64 axi_addr,
+			       u64 pci_addr, resource_size_t size)
+{
+	struct device *dev = pcie->dev;
+	const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx];
+	u16 iarr_offset, imap_offset;
+	u32 val;
+	int window_idx;
+
+	iarr_offset = iproc_pcie_reg_offset(pcie,
+				MAP_REG(IPROC_PCIE_IARR0, region_idx));
+	imap_offset = iproc_pcie_reg_offset(pcie,
+				MAP_REG(IPROC_PCIE_IMAP0, region_idx));
+	if (iproc_pcie_reg_is_invalid(iarr_offset) ||
+	    iproc_pcie_reg_is_invalid(imap_offset))
+		return -EINVAL;
+
+	dev_info(dev, "ib region [%d]: offset 0x%x axi %pap pci %pap\n",
+		 region_idx, iarr_offset, &axi_addr, &pci_addr);
+
+	/*
+	 * Program the IARR registers.  The upper 32-bit IARR register is
+	 * always right after the lower 32-bit IARR register.
+	 */
+	writel(lower_32_bits(pci_addr) | BIT(size_idx),
+	       pcie->base + iarr_offset);
+	writel(upper_32_bits(pci_addr), pcie->base + iarr_offset + 4);
+
+	dev_info(dev, "iarr lo 0x%x iarr hi 0x%x\n",
+		 readl(pcie->base + iarr_offset),
+		 readl(pcie->base + iarr_offset + 4));
+
+	/*
+	 * Now program the IMAP registers.  Each IARR region may have one or
+	 * more IMAP windows.
+	 */
+	size >>= ilog2(nr_windows);
+	for (window_idx = 0; window_idx < nr_windows; window_idx++) {
+		val = readl(pcie->base + imap_offset);
+		val |= lower_32_bits(axi_addr) | IMAP_VALID;
+		writel(val, pcie->base + imap_offset);
+		writel(upper_32_bits(axi_addr),
+		       pcie->base + imap_offset + ib_map->imap_addr_offset);
+
+		dev_info(dev, "imap window [%d] lo 0x%x hi 0x%x\n",
+			 window_idx, readl(pcie->base + imap_offset),
+			 readl(pcie->base + imap_offset +
+			       ib_map->imap_addr_offset));
+
+		imap_offset += ib_map->imap_window_offset;
+		axi_addr += size;
+	}
+
+	return 0;
+}
+
+static int iproc_pcie_setup_ib(struct iproc_pcie *pcie,
+			       struct of_pci_range *range,
+			       enum iproc_pcie_ib_map_type type)
+{
+	struct device *dev = pcie->dev;
+	struct iproc_pcie_ib *ib = &pcie->ib;
+	int ret;
+	unsigned int region_idx, size_idx;
+	u64 axi_addr = range->cpu_addr, pci_addr = range->pci_addr;
+	resource_size_t size = range->size;
+
+	/* iterate through all IARR mapping regions */
+	for (region_idx = 0; region_idx < ib->nr_regions; region_idx++) {
+		const struct iproc_pcie_ib_map *ib_map =
+			&pcie->ib_map[region_idx];
+
+		/*
+		 * If current inbound region is already in use or not a
+		 * compatible type, move on to the next.
+		 */
+		if (iproc_pcie_ib_is_in_use(pcie, region_idx) ||
+		    !iproc_pcie_ib_check_type(ib_map, type))
+			continue;
+
+		/* iterate through all supported region sizes to find a match */
+		for (size_idx = 0; size_idx < ib_map->nr_sizes; size_idx++) {
+			resource_size_t region_size =
+			ib_map->region_sizes[size_idx] * ib_map->size_unit;
+
+			if (size != region_size)
+				continue;
+
+			if (!IS_ALIGNED(axi_addr, region_size) ||
+			    !IS_ALIGNED(pci_addr, region_size)) {
+				dev_err(dev,
+					"axi %pap or pci %pap not aligned\n",
+					&axi_addr, &pci_addr);
+				return -EINVAL;
+			}
+
+			/* Match found!  Program IARR and all IMAP windows. */
+			ret = iproc_pcie_ib_write(pcie, region_idx, size_idx,
+						  ib_map->nr_windows, axi_addr,
+						  pci_addr, size);
+			if (ret)
+				goto err_ib;
+			else
+				return 0;
+
+		}
+	}
+	ret = -EINVAL;
+
+err_ib:
+	dev_err(dev, "unable to configure inbound mapping\n");
+	dev_err(dev, "axi %pap, pci %pap, res size %pap\n",
+		&axi_addr, &pci_addr, &size);
+
+	return ret;
+}
+
+static int pci_dma_range_parser_init(struct of_pci_range_parser *parser,
+				     struct device_node *node)
+{
+	const int na = 3, ns = 2;
+	int rlen;
+
+	parser->node = node;
+	parser->pna = of_n_addr_cells(node);
+	parser->np = parser->pna + na + ns;
+
+	parser->range = of_get_property(node, "dma-ranges", &rlen);
+	if (!parser->range)
+		return -ENOENT;
+
+	parser->end = parser->range + rlen / sizeof(__be32);
+	return 0;
+}
+
+static int iproc_pcie_map_dma_ranges(struct iproc_pcie *pcie)
+{
+	struct of_pci_range range;
+	struct of_pci_range_parser parser;
+	int ret;
+
+	/* Get the dma-ranges from DT */
+	ret = pci_dma_range_parser_init(&parser, pcie->dev->of_node);
+	if (ret)
+		return ret;
+
+	for_each_of_pci_range(&parser, &range) {
+		/* Each range entry corresponds to an inbound mapping region */
+		ret = iproc_pcie_setup_ib(pcie, &range, IPROC_PCIE_IB_MAP_MEM);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int iproce_pcie_get_msi(struct iproc_pcie *pcie,
+			       struct device_node *msi_node,
+			       u64 *msi_addr)
+{
+	struct device *dev = pcie->dev;
+	int ret;
+	struct resource res;
+
+	/*
+	 * Check if 'msi-map' points to ARM GICv3 ITS, which is the only
+	 * supported external MSI controller that requires steering.
+	 */
+	if (!of_device_is_compatible(msi_node, "arm,gic-v3-its")) {
+		dev_err(dev, "unable to find compatible MSI controller\n");
+		return -ENODEV;
+	}
+
+	/* derive GITS_TRANSLATER address from GICv3 */
+	ret = of_address_to_resource(msi_node, 0, &res);
+	if (ret < 0) {
+		dev_err(dev, "unable to obtain MSI controller resources\n");
+		return ret;
+	}
+
+	*msi_addr = res.start + GITS_TRANSLATER;
+	return 0;
+}
+
+static int iproc_pcie_paxb_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr)
+{
+	int ret;
+	struct of_pci_range range;
+
+	memset(&range, 0, sizeof(range));
+	range.size = SZ_32K;
+	range.pci_addr = range.cpu_addr = msi_addr & ~(range.size - 1);
+
+	ret = iproc_pcie_setup_ib(pcie, &range, IPROC_PCIE_IB_MAP_IO);
+	return ret;
+}
+
+static void iproc_pcie_paxc_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr)
+{
+	u32 val;
+
+	/*
+	 * Program bits [43:13] of address of GITS_TRANSLATER register into
+	 * bits [30:0] of the MSI base address register.  In fact, in all iProc
+	 * based SoCs, all I/O register bases are well below the 32-bit
+	 * boundary, so we can safely assume bits [43:32] are always zeros.
+	 */
+	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_BASE_ADDR,
+			     (u32)(msi_addr >> 13));
+
+	/* use a default 8K window size */
+	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_WINDOW_SIZE, 0);
+
+	/* steering MSI to GICv3 ITS */
+	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_GIC_MODE);
+	val |= GIC_V3_CFG;
+	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_GIC_MODE, val);
+
+	/*
+	 * Program bits [43:2] of address of GITS_TRANSLATER register into the
+	 * iProc MSI address registers.
+	 */
+	msi_addr >>= 2;
+	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_HI,
+			     upper_32_bits(msi_addr));
+	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_LO,
+			     lower_32_bits(msi_addr));
+
+	/* enable MSI */
+	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG);
+	val |= MSI_ENABLE_CFG;
+	iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val);
+}
+
+static int iproc_pcie_msi_steer(struct iproc_pcie *pcie,
+				struct device_node *msi_node)
+{
+	struct device *dev = pcie->dev;
+	int ret;
+	u64 msi_addr;
+
+	ret = iproce_pcie_get_msi(pcie, msi_node, &msi_addr);
+	if (ret < 0) {
+		dev_err(dev, "msi steering failed\n");
+		return ret;
+	}
+
+	switch (pcie->type) {
+	case IPROC_PCIE_PAXB_V2:
+		ret = iproc_pcie_paxb_v2_msi_steer(pcie, msi_addr);
+		if (ret)
+			return ret;
+		break;
+	case IPROC_PCIE_PAXC_V2:
+		iproc_pcie_paxc_v2_msi_steer(pcie, msi_addr);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int iproc_pcie_msi_enable(struct iproc_pcie *pcie)
 {
 	struct device_node *msi_node;
+	int ret;
+
+	/*
+	 * Either the "msi-parent" or the "msi-map" phandle needs to exist
+	 * for us to obtain the MSI node.
+	 */
 
 	msi_node = of_parse_phandle(pcie->dev->of_node, "msi-parent", 0);
-	if (!msi_node)
-		return -ENODEV;
+	if (!msi_node) {
+		const __be32 *msi_map = NULL;
+		int len;
+		u32 phandle;
+
+		msi_map = of_get_property(pcie->dev->of_node, "msi-map", &len);
+		if (!msi_map)
+			return -ENODEV;
+
+		phandle = be32_to_cpup(msi_map + 1);
+		msi_node = of_find_node_by_phandle(phandle);
+		if (!msi_node)
+			return -ENODEV;
+	}
+
+	/*
+	 * Certain revisions of the iProc PCIe controller require additional
+	 * configurations to steer the MSI writes towards an external MSI
+	 * controller.
+	 */
+	if (pcie->need_msi_steer) {
+		ret = iproc_pcie_msi_steer(pcie, msi_node);
+		if (ret)
+			return ret;
+	}
 
 	/*
 	 * If another MSI controller is being used, the call below should fail
@@ -454,6 +1141,65 @@ static void iproc_pcie_msi_disable(struct iproc_pcie *pcie)
 	iproc_msi_exit(pcie);
 }
 
+static int iproc_pcie_rev_init(struct iproc_pcie *pcie)
+{
+	struct device *dev = pcie->dev;
+	unsigned int reg_idx;
+	const u16 *regs;
+
+	switch (pcie->type) {
+	case IPROC_PCIE_PAXB_BCMA:
+		regs = iproc_pcie_reg_paxb_bcma;
+		break;
+	case IPROC_PCIE_PAXB:
+		regs = iproc_pcie_reg_paxb;
+		pcie->has_apb_err_disable = true;
+		if (pcie->need_ob_cfg) {
+			pcie->ob_map = paxb_ob_map;
+			pcie->ob.nr_windows = ARRAY_SIZE(paxb_ob_map);
+		}
+		break;
+	case IPROC_PCIE_PAXB_V2:
+		regs = iproc_pcie_reg_paxb_v2;
+		pcie->has_apb_err_disable = true;
+		if (pcie->need_ob_cfg) {
+			pcie->ob_map = paxb_v2_ob_map;
+			pcie->ob.nr_windows = ARRAY_SIZE(paxb_v2_ob_map);
+		}
+		pcie->ib.nr_regions = ARRAY_SIZE(paxb_v2_ib_map);
+		pcie->ib_map = paxb_v2_ib_map;
+		pcie->need_msi_steer = true;
+		break;
+	case IPROC_PCIE_PAXC:
+		regs = iproc_pcie_reg_paxc;
+		pcie->ep_is_internal = true;
+		break;
+	case IPROC_PCIE_PAXC_V2:
+		regs = iproc_pcie_reg_paxc_v2;
+		pcie->ep_is_internal = true;
+		pcie->need_msi_steer = true;
+		break;
+	default:
+		dev_err(dev, "incompatible iProc PCIe interface\n");
+		return -EINVAL;
+	}
+
+	pcie->reg_offsets = devm_kcalloc(dev, IPROC_PCIE_MAX_NUM_REG,
+					 sizeof(*pcie->reg_offsets),
+					 GFP_KERNEL);
+	if (!pcie->reg_offsets)
+		return -ENOMEM;
+
+	/* go through the register table and populate all valid registers */
+	pcie->reg_offsets[0] = (pcie->type == IPROC_PCIE_PAXC_V2) ?
+		IPROC_PCIE_REG_INVALID : regs[0];
+	for (reg_idx = 1; reg_idx < IPROC_PCIE_MAX_NUM_REG; reg_idx++)
+		pcie->reg_offsets[reg_idx] = regs[reg_idx] ?
+			regs[reg_idx] : IPROC_PCIE_REG_INVALID;
+
+	return 0;
+}
+
 int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
 {
 	struct device *dev;
@@ -462,6 +1208,13 @@ int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
 	struct pci_bus *bus;
 
 	dev = pcie->dev;
+
+	ret = iproc_pcie_rev_init(pcie);
+	if (ret) {
+		dev_err(dev, "unable to initialize controller parameters\n");
+		return ret;
+	}
+
 	ret = devm_request_pci_bus_resources(dev, res);
 	if (ret)
 		return ret;
@@ -478,19 +1231,6 @@ int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
 		goto err_exit_phy;
 	}
 
-	switch (pcie->type) {
-	case IPROC_PCIE_PAXB:
-		pcie->reg_offsets = iproc_pcie_reg_paxb;
-		break;
-	case IPROC_PCIE_PAXC:
-		pcie->reg_offsets = iproc_pcie_reg_paxc;
-		break;
-	default:
-		dev_err(dev, "incompatible iProc PCIe interface\n");
-		ret = -EINVAL;
-		goto err_power_off_phy;
-	}
-
 	iproc_pcie_reset(pcie);
 
 	if (pcie->need_ob_cfg) {
@@ -501,6 +1241,10 @@ int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
 		}
 	}
 
+	ret = iproc_pcie_map_dma_ranges(pcie);
+	if (ret && ret != -ENOENT)
+		goto err_power_off_phy;
+
 #ifdef CONFIG_ARM
 	pcie->sysdata.private_data = pcie;
 	sysdata = &pcie->sysdata;
@@ -530,7 +1274,10 @@ int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
 
 	pci_scan_child_bus(bus);
 	pci_assign_unassigned_bus_resources(bus);
-	pci_fixup_irqs(pci_common_swizzle, pcie->map_irq);
+
+	if (pcie->map_irq)
+		pci_fixup_irqs(pci_common_swizzle, pcie->map_irq);
+
 	pci_bus_add_devices(bus);
 
 	return 0;
diff --git a/drivers/pci/host/pcie-iproc.h b/drivers/pci/host/pcie-iproc.h
index e84d93c53c7b..04fed8e907f1 100644
--- a/drivers/pci/host/pcie-iproc.h
+++ b/drivers/pci/host/pcie-iproc.h
@@ -24,23 +24,34 @@
  * endpoint devices.
  */
 enum iproc_pcie_type {
-	IPROC_PCIE_PAXB = 0,
+	IPROC_PCIE_PAXB_BCMA = 0,
+	IPROC_PCIE_PAXB,
+	IPROC_PCIE_PAXB_V2,
 	IPROC_PCIE_PAXC,
+	IPROC_PCIE_PAXC_V2,
 };
 
 /**
  * iProc PCIe outbound mapping
- * @set_oarr_size: indicates the OARR size bit needs to be set
  * @axi_offset: offset from the AXI address to the internal address used by
  * the iProc PCIe core
- * @window_size: outbound window size
+ * @nr_windows: total number of supported outbound mapping windows
  */
 struct iproc_pcie_ob {
-	bool set_oarr_size;
 	resource_size_t axi_offset;
-	resource_size_t window_size;
+	unsigned int nr_windows;
 };
 
+/**
+ * iProc PCIe inbound mapping
+ * @nr_regions: total number of supported inbound mapping regions
+ */
+struct iproc_pcie_ib {
+	unsigned int nr_regions;
+};
+
+struct iproc_pcie_ob_map;
+struct iproc_pcie_ib_map;
 struct iproc_msi;
 
 /**
@@ -55,14 +66,25 @@ struct iproc_msi;
  * @root_bus: pointer to root bus
  * @phy: optional PHY device that controls the Serdes
  * @map_irq: function callback to map interrupts
+ * @ep_is_internal: indicates an internal emulated endpoint device is connected
+ * @has_apb_err_disable: indicates the controller can be configured to prevent
+ * unsupported request from being forwarded as an APB bus error
+ *
  * @need_ob_cfg: indicates SW needs to configure the outbound mapping window
- * @ob: outbound mapping parameters
+ * @ob: outbound mapping related parameters
+ * @ob_map: outbound mapping related parameters specific to the controller
+ *
+ * @ib: inbound mapping related parameters
+ * @ib_map: outbound mapping region related parameters
+ *
+ * @need_msi_steer: indicates additional configuration of the iProc PCIe
+ * controller is required to steer MSI writes to external interrupt controller
  * @msi: MSI data
  */
 struct iproc_pcie {
 	struct device *dev;
 	enum iproc_pcie_type type;
-	const u16 *reg_offsets;
+	u16 *reg_offsets;
 	void __iomem *base;
 	phys_addr_t base_addr;
 #ifdef CONFIG_ARM
@@ -71,8 +93,17 @@ struct iproc_pcie {
 	struct pci_bus *root_bus;
 	struct phy *phy;
 	int (*map_irq)(const struct pci_dev *, u8, u8);
+	bool ep_is_internal;
+	bool has_apb_err_disable;
+
 	bool need_ob_cfg;
 	struct iproc_pcie_ob ob;
+	const struct iproc_pcie_ob_map *ob_map;
+
+	struct iproc_pcie_ib ib;
+	const struct iproc_pcie_ib_map *ib_map;
+
+	bool need_msi_steer;
 	struct iproc_msi *msi;
 };
 
diff --git a/drivers/pci/host/pcie-qcom.c b/drivers/pci/host/pcie-qcom.c
index 35936409b2d4..734ba0d4a5c8 100644
--- a/drivers/pci/host/pcie-qcom.c
+++ b/drivers/pci/host/pcie-qcom.c
@@ -36,11 +36,17 @@
 
 #include "pcie-designware.h"
 
+#define PCIE20_PARF_SYS_CTRL			0x00
 #define PCIE20_PARF_PHY_CTRL			0x40
 #define PCIE20_PARF_PHY_REFCLK			0x4C
 #define PCIE20_PARF_DBI_BASE_ADDR		0x168
-#define PCIE20_PARF_SLV_ADDR_SPACE_SIZE		0x16c
+#define PCIE20_PARF_SLV_ADDR_SPACE_SIZE		0x16C
+#define PCIE20_PARF_MHI_CLOCK_RESET_CTRL	0x174
 #define PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT	0x178
+#define PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT_V2	0x1A8
+#define PCIE20_PARF_LTSSM			0x1B0
+#define PCIE20_PARF_SID_OFFSET			0x234
+#define PCIE20_PARF_BDF_TRANSLATE_CFG		0x24C
 
 #define PCIE20_ELBI_SYS_CTRL			0x04
 #define PCIE20_ELBI_SYS_CTRL_LT_ENABLE		BIT(0)
@@ -72,9 +78,18 @@ struct qcom_pcie_resources_v1 {
 	struct regulator *vdda;
 };
 
+struct qcom_pcie_resources_v2 {
+	struct clk *aux_clk;
+	struct clk *master_clk;
+	struct clk *slave_clk;
+	struct clk *cfg_clk;
+	struct clk *pipe_clk;
+};
+
 union qcom_pcie_resources {
 	struct qcom_pcie_resources_v0 v0;
 	struct qcom_pcie_resources_v1 v1;
+	struct qcom_pcie_resources_v2 v2;
 };
 
 struct qcom_pcie;
@@ -82,7 +97,9 @@ struct qcom_pcie;
 struct qcom_pcie_ops {
 	int (*get_resources)(struct qcom_pcie *pcie);
 	int (*init)(struct qcom_pcie *pcie);
+	int (*post_init)(struct qcom_pcie *pcie);
 	void (*deinit)(struct qcom_pcie *pcie);
+	void (*ltssm_enable)(struct qcom_pcie *pcie);
 };
 
 struct qcom_pcie {
@@ -116,17 +133,35 @@ static irqreturn_t qcom_pcie_msi_irq_handler(int irq, void *arg)
 	return dw_handle_msi_irq(pp);
 }
 
-static int qcom_pcie_establish_link(struct qcom_pcie *pcie)
+static void qcom_pcie_v0_v1_ltssm_enable(struct qcom_pcie *pcie)
 {
 	u32 val;
 
-	if (dw_pcie_link_up(&pcie->pp))
-		return 0;
-
 	/* enable link training */
 	val = readl(pcie->elbi + PCIE20_ELBI_SYS_CTRL);
 	val |= PCIE20_ELBI_SYS_CTRL_LT_ENABLE;
 	writel(val, pcie->elbi + PCIE20_ELBI_SYS_CTRL);
+}
+
+static void qcom_pcie_v2_ltssm_enable(struct qcom_pcie *pcie)
+{
+	u32 val;
+
+	/* enable link training */
+	val = readl(pcie->parf + PCIE20_PARF_LTSSM);
+	val |= BIT(8);
+	writel(val, pcie->parf + PCIE20_PARF_LTSSM);
+}
+
+static int qcom_pcie_establish_link(struct qcom_pcie *pcie)
+{
+
+	if (dw_pcie_link_up(&pcie->pp))
+		return 0;
+
+	/* Enable Link Training state machine */
+	if (pcie->ops->ltssm_enable)
+		pcie->ops->ltssm_enable(pcie);
 
 	return dw_pcie_wait_for_link(&pcie->pp);
 }
@@ -421,6 +456,113 @@ err_res:
 	return ret;
 }
 
+static int qcom_pcie_get_resources_v2(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_v2 *res = &pcie->res.v2;
+	struct device *dev = pcie->pp.dev;
+
+	res->aux_clk = devm_clk_get(dev, "aux");
+	if (IS_ERR(res->aux_clk))
+		return PTR_ERR(res->aux_clk);
+
+	res->cfg_clk = devm_clk_get(dev, "cfg");
+	if (IS_ERR(res->cfg_clk))
+		return PTR_ERR(res->cfg_clk);
+
+	res->master_clk = devm_clk_get(dev, "bus_master");
+	if (IS_ERR(res->master_clk))
+		return PTR_ERR(res->master_clk);
+
+	res->slave_clk = devm_clk_get(dev, "bus_slave");
+	if (IS_ERR(res->slave_clk))
+		return PTR_ERR(res->slave_clk);
+
+	res->pipe_clk = devm_clk_get(dev, "pipe");
+	if (IS_ERR(res->pipe_clk))
+		return PTR_ERR(res->pipe_clk);
+
+	return 0;
+}
+
+static int qcom_pcie_init_v2(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_v2 *res = &pcie->res.v2;
+	struct device *dev = pcie->pp.dev;
+	u32 val;
+	int ret;
+
+	ret = clk_prepare_enable(res->aux_clk);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable aux clock\n");
+		return ret;
+	}
+
+	ret = clk_prepare_enable(res->cfg_clk);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable cfg clock\n");
+		goto err_cfg_clk;
+	}
+
+	ret = clk_prepare_enable(res->master_clk);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable master clock\n");
+		goto err_master_clk;
+	}
+
+	ret = clk_prepare_enable(res->slave_clk);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable slave clock\n");
+		goto err_slave_clk;
+	}
+
+	/* enable PCIe clocks and resets */
+	val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL);
+	val &= ~BIT(0);
+	writel(val, pcie->parf + PCIE20_PARF_PHY_CTRL);
+
+	/* change DBI base address */
+	writel(0, pcie->parf + PCIE20_PARF_DBI_BASE_ADDR);
+
+	/* MAC PHY_POWERDOWN MUX DISABLE  */
+	val = readl(pcie->parf + PCIE20_PARF_SYS_CTRL);
+	val &= ~BIT(29);
+	writel(val, pcie->parf + PCIE20_PARF_SYS_CTRL);
+
+	val = readl(pcie->parf + PCIE20_PARF_MHI_CLOCK_RESET_CTRL);
+	val |= BIT(4);
+	writel(val, pcie->parf + PCIE20_PARF_MHI_CLOCK_RESET_CTRL);
+
+	val = readl(pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT_V2);
+	val |= BIT(31);
+	writel(val, pcie->parf + PCIE20_PARF_AXI_MSTR_WR_ADDR_HALT_V2);
+
+	return 0;
+
+err_slave_clk:
+	clk_disable_unprepare(res->master_clk);
+err_master_clk:
+	clk_disable_unprepare(res->cfg_clk);
+err_cfg_clk:
+	clk_disable_unprepare(res->aux_clk);
+
+	return ret;
+}
+
+static int qcom_pcie_post_init_v2(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_v2 *res = &pcie->res.v2;
+	struct device *dev = pcie->pp.dev;
+	int ret;
+
+	ret = clk_prepare_enable(res->pipe_clk);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable pipe clock\n");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int qcom_pcie_link_up(struct pcie_port *pp)
 {
 	struct qcom_pcie *pcie = to_qcom_pcie(pp);
@@ -429,6 +571,17 @@ static int qcom_pcie_link_up(struct pcie_port *pp)
 	return !!(val & PCI_EXP_LNKSTA_DLLLA);
 }
 
+static void qcom_pcie_deinit_v2(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_v2 *res = &pcie->res.v2;
+
+	clk_disable_unprepare(res->pipe_clk);
+	clk_disable_unprepare(res->slave_clk);
+	clk_disable_unprepare(res->master_clk);
+	clk_disable_unprepare(res->cfg_clk);
+	clk_disable_unprepare(res->aux_clk);
+}
+
 static void qcom_pcie_host_init(struct pcie_port *pp)
 {
 	struct qcom_pcie *pcie = to_qcom_pcie(pp);
@@ -444,6 +597,9 @@ static void qcom_pcie_host_init(struct pcie_port *pp)
 	if (ret)
 		goto err_deinit;
 
+	if (pcie->ops->post_init)
+		pcie->ops->post_init(pcie);
+
 	dw_pcie_setup_rc(pp);
 
 	if (IS_ENABLED(CONFIG_PCI_MSI))
@@ -487,12 +643,22 @@ static const struct qcom_pcie_ops ops_v0 = {
 	.get_resources = qcom_pcie_get_resources_v0,
 	.init = qcom_pcie_init_v0,
 	.deinit = qcom_pcie_deinit_v0,
+	.ltssm_enable = qcom_pcie_v0_v1_ltssm_enable,
 };
 
 static const struct qcom_pcie_ops ops_v1 = {
 	.get_resources = qcom_pcie_get_resources_v1,
 	.init = qcom_pcie_init_v1,
 	.deinit = qcom_pcie_deinit_v1,
+	.ltssm_enable = qcom_pcie_v0_v1_ltssm_enable,
+};
+
+static const struct qcom_pcie_ops ops_v2 = {
+	.get_resources = qcom_pcie_get_resources_v2,
+	.init = qcom_pcie_init_v2,
+	.post_init = qcom_pcie_post_init_v2,
+	.deinit = qcom_pcie_deinit_v2,
+	.ltssm_enable = qcom_pcie_v2_ltssm_enable,
 };
 
 static int qcom_pcie_probe(struct platform_device *pdev)
@@ -572,6 +738,7 @@ static const struct of_device_id qcom_pcie_match[] = {
 	{ .compatible = "qcom,pcie-ipq8064", .data = &ops_v0 },
 	{ .compatible = "qcom,pcie-apq8064", .data = &ops_v0 },
 	{ .compatible = "qcom,pcie-apq8084", .data = &ops_v1 },
+	{ .compatible = "qcom,pcie-msm8996", .data = &ops_v2 },
 	{ }
 };
 
diff --git a/drivers/pci/host/pcie-rcar.c b/drivers/pci/host/pcie-rcar.c
index 62700d1896f4..aca85be101f8 100644
--- a/drivers/pci/host/pcie-rcar.c
+++ b/drivers/pci/host/pcie-rcar.c
@@ -1071,13 +1071,14 @@ static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie *pcie,
 
 static const struct of_device_id rcar_pcie_of_match[] = {
 	{ .compatible = "renesas,pcie-r8a7779", .data = rcar_pcie_hw_init_h1 },
-	{ .compatible = "renesas,pcie-rcar-gen2",
-	  .data = rcar_pcie_hw_init_gen2 },
 	{ .compatible = "renesas,pcie-r8a7790",
 	  .data = rcar_pcie_hw_init_gen2 },
 	{ .compatible = "renesas,pcie-r8a7791",
 	  .data = rcar_pcie_hw_init_gen2 },
+	{ .compatible = "renesas,pcie-rcar-gen2",
+	  .data = rcar_pcie_hw_init_gen2 },
 	{ .compatible = "renesas,pcie-r8a7795", .data = rcar_pcie_hw_init },
+	{ .compatible = "renesas,pcie-rcar-gen3", .data = rcar_pcie_hw_init },
 	{},
 };
 
diff --git a/drivers/pci/host/pcie-rockchip.c b/drivers/pci/host/pcie-rockchip.c
index e04f69beb42d..f2dca7bb0b39 100644
--- a/drivers/pci/host/pcie-rockchip.c
+++ b/drivers/pci/host/pcie-rockchip.c
@@ -53,6 +53,7 @@
 #define   PCIE_CLIENT_ARI_ENABLE	  HIWORD_UPDATE_BIT(0x0008)
 #define   PCIE_CLIENT_CONF_LANE_NUM(x)	  HIWORD_UPDATE(0x0030, ENCODE_LANES(x))
 #define   PCIE_CLIENT_MODE_RC		  HIWORD_UPDATE_BIT(0x0040)
+#define   PCIE_CLIENT_GEN_SEL_1		  HIWORD_UPDATE(0x0080, 0)
 #define   PCIE_CLIENT_GEN_SEL_2		  HIWORD_UPDATE_BIT(0x0080)
 #define PCIE_CLIENT_BASIC_STATUS1	(PCIE_CLIENT_BASE + 0x48)
 #define   PCIE_CLIENT_LINK_STATUS_UP		0x00300000
@@ -135,13 +136,14 @@
 #define PCIE_RC_CONFIG_VENDOR		(PCIE_RC_CONFIG_BASE + 0x00)
 #define PCIE_RC_CONFIG_RID_CCR		(PCIE_RC_CONFIG_BASE + 0x08)
 #define   PCIE_RC_CONFIG_SCC_SHIFT		16
+#define PCIE_RC_CONFIG_DCR		(PCIE_RC_CONFIG_BASE + 0xc4)
+#define   PCIE_RC_CONFIG_DCR_CSPL_SHIFT		18
+#define   PCIE_RC_CONFIG_DCR_CSPL_LIMIT		0xff
+#define   PCIE_RC_CONFIG_DCR_CPLS_SHIFT		26
 #define PCIE_RC_CONFIG_LCS		(PCIE_RC_CONFIG_BASE + 0xd0)
-#define   PCIE_RC_CONFIG_LCS_RETRAIN_LINK	BIT(5)
-#define   PCIE_RC_CONFIG_LCS_LBMIE		BIT(10)
-#define   PCIE_RC_CONFIG_LCS_LABIE		BIT(11)
-#define   PCIE_RC_CONFIG_LCS_LBMS		BIT(30)
-#define   PCIE_RC_CONFIG_LCS_LAMS		BIT(31)
 #define PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 (PCIE_RC_CONFIG_BASE + 0x90c)
+#define PCIE_RC_CONFIG_THP_CAP		(PCIE_RC_CONFIG_BASE + 0x274)
+#define   PCIE_RC_CONFIG_THP_CAP_NEXT_MASK	GENMASK(31, 20)
 
 #define PCIE_CORE_AXI_CONF_BASE		0xc00000
 #define PCIE_CORE_OB_REGION_ADDR0	(PCIE_CORE_AXI_CONF_BASE + 0x0)
@@ -203,8 +205,14 @@ struct rockchip_pcie {
 	struct	gpio_desc *ep_gpio;
 	u32	lanes;
 	u8	root_bus_nr;
+	int	link_gen;
 	struct	device *dev;
 	struct	irq_domain *irq_domain;
+	u32     io_size;
+	int     offset;
+	phys_addr_t io_bus_addr;
+	u32     mem_size;
+	phys_addr_t mem_bus_addr;
 };
 
 static u32 rockchip_pcie_read(struct rockchip_pcie *rockchip, u32 reg)
@@ -223,7 +231,7 @@ static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)
 	u32 status;
 
 	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
-	status |= (PCIE_RC_CONFIG_LCS_LBMIE | PCIE_RC_CONFIG_LCS_LABIE);
+	status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE);
 	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
 }
 
@@ -232,7 +240,7 @@ static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)
 	u32 status;
 
 	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
-	status |= (PCIE_RC_CONFIG_LCS_LBMS | PCIE_RC_CONFIG_LCS_LAMS);
+	status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16;
 	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
 }
 
@@ -398,6 +406,40 @@ static struct pci_ops rockchip_pcie_ops = {
 	.write = rockchip_pcie_wr_conf,
 };
 
+static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)
+{
+	u32 status, curr, scale, power;
+
+	if (IS_ERR(rockchip->vpcie3v3))
+		return;
+
+	/*
+	 * Set RC's captured slot power limit and scale if
+	 * vpcie3v3 available. The default values are both zero
+	 * which means the software should set these two according
+	 * to the actual power supply.
+	 */
+	curr = regulator_get_current_limit(rockchip->vpcie3v3);
+	if (curr > 0) {
+		scale = 3; /* 0.001x */
+		curr = curr / 1000; /* convert to mA */
+		power = (curr * 3300) / 1000; /* milliwatt */
+		while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) {
+			if (!scale) {
+				dev_warn(rockchip->dev, "invalid power supply\n");
+				return;
+			}
+			scale--;
+			power = power / 10;
+		}
+
+		status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR);
+		status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) |
+			  (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT);
+		rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR);
+	}
+}
+
 /**
  * rockchip_pcie_init_port - Initialize hardware
  * @rockchip: PCIe port information
@@ -429,26 +471,6 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 		return err;
 	}
 
-	udelay(10);
-
-	err = reset_control_deassert(rockchip->pm_rst);
-	if (err) {
-		dev_err(dev, "deassert pm_rst err %d\n", err);
-		return err;
-	}
-
-	err = reset_control_deassert(rockchip->aclk_rst);
-	if (err) {
-		dev_err(dev, "deassert mgmt_sticky_rst err %d\n", err);
-		return err;
-	}
-
-	err = reset_control_deassert(rockchip->pclk_rst);
-	if (err) {
-		dev_err(dev, "deassert mgmt_sticky_rst err %d\n", err);
-		return err;
-	}
-
 	err = phy_init(rockchip->phy);
 	if (err < 0) {
 		dev_err(dev, "fail to init phy, err %d\n", err);
@@ -479,14 +501,40 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 		return err;
 	}
 
+	udelay(10);
+
+	err = reset_control_deassert(rockchip->pm_rst);
+	if (err) {
+		dev_err(dev, "deassert pm_rst err %d\n", err);
+		return err;
+	}
+
+	err = reset_control_deassert(rockchip->aclk_rst);
+	if (err) {
+		dev_err(dev, "deassert aclk_rst err %d\n", err);
+		return err;
+	}
+
+	err = reset_control_deassert(rockchip->pclk_rst);
+	if (err) {
+		dev_err(dev, "deassert pclk_rst err %d\n", err);
+		return err;
+	}
+
+	if (rockchip->link_gen == 2)
+		rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_2,
+				    PCIE_CLIENT_CONFIG);
+	else
+		rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_1,
+				    PCIE_CLIENT_CONFIG);
+
 	rockchip_pcie_write(rockchip,
 			    PCIE_CLIENT_CONF_ENABLE |
 			    PCIE_CLIENT_LINK_TRAIN_ENABLE |
 			    PCIE_CLIENT_ARI_ENABLE |
 			    PCIE_CLIENT_CONF_LANE_NUM(rockchip->lanes) |
-			    PCIE_CLIENT_MODE_RC |
-			    PCIE_CLIENT_GEN_SEL_2,
-				PCIE_CLIENT_CONFIG);
+			    PCIE_CLIENT_MODE_RC,
+			    PCIE_CLIENT_CONFIG);
 
 	err = phy_power_on(rockchip->phy);
 	if (err) {
@@ -522,21 +570,19 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 		return err;
 	}
 
-	/*
-	 * We need to read/write PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 before
-	 * enabling ASPM.  Otherwise L1PwrOnSc and L1PwrOnVal isn't
-	 * reliable and enabling ASPM doesn't work.  This is a controller
-	 * bug we need to work around.
-	 */
-	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2);
-	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2);
-
 	/* Fix the transmitted FTS count desired to exit from L0s. */
 	status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);
-	status = (status & PCIE_CORE_CTRL_PLC1_FTS_MASK) |
+	status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) |
 		 (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);
 	rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);
 
+	rockchip_pcie_set_power_limit(rockchip);
+
+	/* Set RC's clock architecture as common clock */
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+	status |= PCI_EXP_LNKCTL_CCC;
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+
 	/* Enable Gen1 training */
 	rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
 			    PCIE_CLIENT_CONFIG);
@@ -563,35 +609,37 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 		msleep(20);
 	}
 
-	/*
-	 * Enable retrain for gen2. This should be configured only after
-	 * gen1 finished.
-	 */
-	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
-	status |= PCIE_RC_CONFIG_LCS_RETRAIN_LINK;
-	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+	if (rockchip->link_gen == 2) {
+		/*
+		 * Enable retrain for gen2. This should be configured only after
+		 * gen1 finished.
+		 */
+		status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
+		status |= PCI_EXP_LNKCTL_RL;
+		rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
+
+		timeout = jiffies + msecs_to_jiffies(500);
+		for (;;) {
+			status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
+			if ((status & PCIE_CORE_PL_CONF_SPEED_MASK) ==
+			    PCIE_CORE_PL_CONF_SPEED_5G) {
+				dev_dbg(dev, "PCIe link training gen2 pass!\n");
+				break;
+			}
 
-	timeout = jiffies + msecs_to_jiffies(500);
-	for (;;) {
-		status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
-		if ((status & PCIE_CORE_PL_CONF_SPEED_MASK) ==
-		    PCIE_CORE_PL_CONF_SPEED_5G) {
-			dev_dbg(dev, "PCIe link training gen2 pass!\n");
-			break;
-		}
+			if (time_after(jiffies, timeout)) {
+				dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
+				break;
+			}
 
-		if (time_after(jiffies, timeout)) {
-			dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
-			break;
+			msleep(20);
 		}
-
-		msleep(20);
 	}
 
 	/* Check the final link width from negotiated lane counter from MGMT */
 	status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
-	status =  0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
-			  PCIE_CORE_PL_CONF_LANE_MASK);
+	status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
+			  PCIE_CORE_PL_CONF_LANE_SHIFT);
 	dev_dbg(dev, "current link width is x%d\n", status);
 
 	rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID,
@@ -599,6 +647,12 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 	rockchip_pcie_write(rockchip,
 			    PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT,
 			    PCIE_RC_CONFIG_RID_CCR);
+
+	/* Clear THP cap's next cap pointer to remove L1 substate cap */
+	status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP);
+	status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
+	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP);
+
 	rockchip_pcie_write(rockchip, 0x0, PCIE_RC_BAR_CONF);
 
 	rockchip_pcie_write(rockchip,
@@ -794,6 +848,10 @@ static int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip)
 		rockchip->lanes = 1;
 	}
 
+	rockchip->link_gen = of_pci_get_max_link_speed(node);
+	if (rockchip->link_gen < 0 || rockchip->link_gen > 2)
+		rockchip->link_gen = 2;
+
 	rockchip->core_rst = devm_reset_control_get(dev, "core");
 	if (IS_ERR(rockchip->core_rst)) {
 		if (PTR_ERR(rockchip->core_rst) != -EPROBE_DEFER)
@@ -1087,6 +1145,50 @@ static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,
 	return 0;
 }
 
+static int rockchip_cfg_atu(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	int offset;
+	int err;
+	int reg_no;
+
+	for (reg_no = 0; reg_no < (rockchip->mem_size >> 20); reg_no++) {
+		err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
+						AXI_WRAPPER_MEM_WRITE,
+						20 - 1,
+						rockchip->mem_bus_addr +
+						(reg_no << 20),
+						0);
+		if (err) {
+			dev_err(dev, "program RC mem outbound ATU failed\n");
+			return err;
+		}
+	}
+
+	err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
+	if (err) {
+		dev_err(dev, "program RC mem inbound ATU failed\n");
+		return err;
+	}
+
+	offset = rockchip->mem_size >> 20;
+	for (reg_no = 0; reg_no < (rockchip->io_size >> 20); reg_no++) {
+		err = rockchip_pcie_prog_ob_atu(rockchip,
+						reg_no + 1 + offset,
+						AXI_WRAPPER_IO_WRITE,
+						20 - 1,
+						rockchip->io_bus_addr +
+						(reg_no << 20),
+						0);
+		if (err) {
+			dev_err(dev, "program RC io outbound ATU failed\n");
+			return err;
+		}
+	}
+
+	return 0;
+}
+
 static int rockchip_pcie_probe(struct platform_device *pdev)
 {
 	struct rockchip_pcie *rockchip;
@@ -1096,13 +1198,7 @@ static int rockchip_pcie_probe(struct platform_device *pdev)
 	resource_size_t io_base;
 	struct resource	*mem;
 	struct resource	*io;
-	phys_addr_t io_bus_addr = 0;
-	u32 io_size;
-	phys_addr_t mem_bus_addr = 0;
-	u32 mem_size = 0;
-	int reg_no;
 	int err;
-	int offset;
 
 	LIST_HEAD(res);
 
@@ -1169,14 +1265,13 @@ static int rockchip_pcie_probe(struct platform_device *pdev)
 		goto err_vpcie;
 
 	/* Get the I/O and memory ranges from DT */
-	io_size = 0;
 	resource_list_for_each_entry(win, &res) {
 		switch (resource_type(win->res)) {
 		case IORESOURCE_IO:
 			io = win->res;
 			io->name = "I/O";
-			io_size = resource_size(io);
-			io_bus_addr = io->start - win->offset;
+			rockchip->io_size = resource_size(io);
+			rockchip->io_bus_addr = io->start - win->offset;
 			err = pci_remap_iospace(io, io_base);
 			if (err) {
 				dev_warn(dev, "error %d: failed to map resource %pR\n",
@@ -1187,8 +1282,8 @@ static int rockchip_pcie_probe(struct platform_device *pdev)
 		case IORESOURCE_MEM:
 			mem = win->res;
 			mem->name = "MEM";
-			mem_size = resource_size(mem);
-			mem_bus_addr = mem->start - win->offset;
+			rockchip->mem_size = resource_size(mem);
+			rockchip->mem_bus_addr = mem->start - win->offset;
 			break;
 		case IORESOURCE_BUS:
 			rockchip->root_bus_nr = win->res->start;
@@ -1198,45 +1293,9 @@ static int rockchip_pcie_probe(struct platform_device *pdev)
 		}
 	}
 
-	if (mem_size) {
-		for (reg_no = 0; reg_no < (mem_size >> 20); reg_no++) {
-			err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
-							AXI_WRAPPER_MEM_WRITE,
-							20 - 1,
-							mem_bus_addr +
-							(reg_no << 20),
-							0);
-			if (err) {
-				dev_err(dev, "program RC mem outbound ATU failed\n");
-				goto err_vpcie;
-			}
-		}
-	}
-
-	err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
-	if (err) {
-		dev_err(dev, "program RC mem inbound ATU failed\n");
+	err = rockchip_cfg_atu(rockchip);
+	if (err)
 		goto err_vpcie;
-	}
-
-	offset = mem_size >> 20;
-
-	if (io_size) {
-		for (reg_no = 0; reg_no < (io_size >> 20); reg_no++) {
-			err = rockchip_pcie_prog_ob_atu(rockchip,
-							reg_no + 1 + offset,
-							AXI_WRAPPER_IO_WRITE,
-							20 - 1,
-							io_bus_addr +
-							(reg_no << 20),
-							0);
-			if (err) {
-				dev_err(dev, "program RC io outbound ATU failed\n");
-				goto err_vpcie;
-			}
-		}
-	}
-
 	bus = pci_scan_root_bus(&pdev->dev, 0, &rockchip_pcie_ops, rockchip, &res);
 	if (!bus) {
 		err = -ENOMEM;
@@ -1249,9 +1308,6 @@ static int rockchip_pcie_probe(struct platform_device *pdev)
 		pcie_bus_configure_settings(child);
 
 	pci_bus_add_devices(bus);
-
-	dev_warn(dev, "only 32-bit config accesses supported; smaller writes may corrupt adjacent RW1C fields\n");
-
 	return err;
 
 err_vpcie:
diff --git a/drivers/pci/host/pcie-spear13xx.c b/drivers/pci/host/pcie-spear13xx.c
index 3cf197ba7f37..dafe8b88d97d 100644
--- a/drivers/pci/host/pcie-spear13xx.c
+++ b/drivers/pci/host/pcie-spear13xx.c
@@ -296,8 +296,4 @@ static struct platform_driver spear13xx_pcie_driver = {
 	},
 };
 
-static int __init spear13xx_pcie_init(void)
-{
-	return platform_driver_register(&spear13xx_pcie_driver);
-}
-device_initcall(spear13xx_pcie_init);
+builtin_platform_driver(spear13xx_pcie_driver);
diff --git a/drivers/pci/host/vmd.c b/drivers/pci/host/vmd.c
index 37e29b580be3..18ef1a93c10a 100644
--- a/drivers/pci/host/vmd.c
+++ b/drivers/pci/host/vmd.c
@@ -19,6 +19,7 @@
 #include <linux/module.h>
 #include <linux/msi.h>
 #include <linux/pci.h>
+#include <linux/srcu.h>
 #include <linux/rculist.h>
 #include <linux/rcupdate.h>
 
@@ -39,7 +40,6 @@ static DEFINE_RAW_SPINLOCK(list_lock);
 /**
  * struct vmd_irq - private data to map driver IRQ to the VMD shared vector
  * @node:	list item for parent traversal.
- * @rcu:	RCU callback item for freeing.
  * @irq:	back pointer to parent.
  * @enabled:	true if driver enabled IRQ
  * @virq:	the virtual IRQ value provided to the requesting driver.
@@ -49,7 +49,6 @@ static DEFINE_RAW_SPINLOCK(list_lock);
  */
 struct vmd_irq {
 	struct list_head	node;
-	struct rcu_head		rcu;
 	struct vmd_irq_list	*irq;
 	bool			enabled;
 	unsigned int		virq;
@@ -58,11 +57,13 @@ struct vmd_irq {
 /**
  * struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector
  * @irq_list:	the list of irq's the VMD one demuxes to.
+ * @srcu:	SRCU struct for local synchronization.
  * @count:	number of child IRQs assigned to this vector; used to track
  *		sharing.
  */
 struct vmd_irq_list {
 	struct list_head	irq_list;
+	struct srcu_struct	srcu;
 	unsigned int		count;
 };
 
@@ -224,14 +225,14 @@ static void vmd_msi_free(struct irq_domain *domain,
 	struct vmd_irq *vmdirq = irq_get_chip_data(virq);
 	unsigned long flags;
 
-	synchronize_rcu();
+	synchronize_srcu(&vmdirq->irq->srcu);
 
 	/* XXX: Potential optimization to rebalance */
 	raw_spin_lock_irqsave(&list_lock, flags);
 	vmdirq->irq->count--;
 	raw_spin_unlock_irqrestore(&list_lock, flags);
 
-	kfree_rcu(vmdirq, rcu);
+	kfree(vmdirq);
 }
 
 static int vmd_msi_prepare(struct irq_domain *domain, struct device *dev,
@@ -646,11 +647,12 @@ static irqreturn_t vmd_irq(int irq, void *data)
 {
 	struct vmd_irq_list *irqs = data;
 	struct vmd_irq *vmdirq;
+	int idx;
 
-	rcu_read_lock();
+	idx = srcu_read_lock(&irqs->srcu);
 	list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node)
 		generic_handle_irq(vmdirq->virq);
-	rcu_read_unlock();
+	srcu_read_unlock(&irqs->srcu, idx);
 
 	return IRQ_HANDLED;
 }
@@ -696,6 +698,10 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
 		return -ENOMEM;
 
 	for (i = 0; i < vmd->msix_count; i++) {
+		err = init_srcu_struct(&vmd->irqs[i].srcu);
+		if (err)
+			return err;
+
 		INIT_LIST_HEAD(&vmd->irqs[i].irq_list);
 		err = devm_request_irq(&dev->dev, pci_irq_vector(dev, i),
 				       vmd_irq, 0, "vmd", &vmd->irqs[i]);
@@ -714,12 +720,20 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	return 0;
 }
 
+static void vmd_cleanup_srcu(struct vmd_dev *vmd)
+{
+	int i;
+
+	for (i = 0; i < vmd->msix_count; i++)
+		cleanup_srcu_struct(&vmd->irqs[i].srcu);
+}
+
 static void vmd_remove(struct pci_dev *dev)
 {
 	struct vmd_dev *vmd = pci_get_drvdata(dev);
 
 	vmd_detach_resources(vmd);
-	pci_set_drvdata(dev, NULL);
+	vmd_cleanup_srcu(vmd);
 	sysfs_remove_link(&vmd->dev->dev.kobj, "domain");
 	pci_stop_root_bus(vmd->bus);
 	pci_remove_root_bus(vmd->bus);
@@ -727,7 +741,7 @@ static void vmd_remove(struct pci_dev *dev)
 	irq_domain_remove(vmd->irq_domain);
 }
 
-#ifdef CONFIG_PM
+#ifdef CONFIG_PM_SLEEP
 static int vmd_suspend(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
diff --git a/drivers/pci/hotplug/acpiphp_glue.c b/drivers/pci/hotplug/acpiphp_glue.c
index a46b585fae31..5ed2dcaa8e27 100644
--- a/drivers/pci/hotplug/acpiphp_glue.c
+++ b/drivers/pci/hotplug/acpiphp_glue.c
@@ -222,35 +222,6 @@ static void acpiphp_post_dock_fixup(struct acpi_device *adev)
 	acpiphp_let_context_go(context);
 }
 
-/* Check whether the PCI device is managed by native PCIe hotplug driver */
-static bool device_is_managed_by_native_pciehp(struct pci_dev *pdev)
-{
-	u32 reg32;
-	acpi_handle tmp;
-	struct acpi_pci_root *root;
-
-	/* Check whether the PCIe port supports native PCIe hotplug */
-	if (pcie_capability_read_dword(pdev, PCI_EXP_SLTCAP, &reg32))
-		return false;
-	if (!(reg32 & PCI_EXP_SLTCAP_HPC))
-		return false;
-
-	/*
-	 * Check whether native PCIe hotplug has been enabled for
-	 * this PCIe hierarchy.
-	 */
-	tmp = acpi_find_root_bridge_handle(pdev);
-	if (!tmp)
-		return false;
-	root = acpi_pci_find_root(tmp);
-	if (!root)
-		return false;
-	if (!(root->osc_control_set & OSC_PCI_EXPRESS_NATIVE_HP_CONTROL))
-		return false;
-
-	return true;
-}
-
 /**
  * acpiphp_add_context - Add ACPIPHP context to an ACPI device object.
  * @handle: ACPI handle of the object to add a context to.
@@ -334,7 +305,7 @@ static acpi_status acpiphp_add_context(acpi_handle handle, u32 lvl, void *data,
 	 * expose slots to user space in those cases.
 	 */
 	if ((acpi_pci_check_ejectable(pbus, handle) || is_dock_device(adev))
-	    && !(pdev && device_is_managed_by_native_pciehp(pdev))) {
+	    && !(pdev && pdev->is_hotplug_bridge && pciehp_is_native(pdev))) {
 		unsigned long long sun;
 		int retval;
 
diff --git a/drivers/pci/hotplug/cpqphp_core.c b/drivers/pci/hotplug/cpqphp_core.c
index 74f3a0695b43..ec009a7dba20 100644
--- a/drivers/pci/hotplug/cpqphp_core.c
+++ b/drivers/pci/hotplug/cpqphp_core.c
@@ -867,7 +867,8 @@ static int cpqhpc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	 */
 	if ((pdev->revision <= 2) && (vendor_id != PCI_VENDOR_ID_INTEL)) {
 		err(msg_HPC_not_supported);
-		return -ENODEV;
+		rc = -ENODEV;
+		goto err_disable_device;
 	}
 
 	/* TODO: This code can be made to support non-Compaq or Intel
diff --git a/drivers/pci/hotplug/pci_hotplug_core.c b/drivers/pci/hotplug/pci_hotplug_core.c
index fea0b8b33589..56013d0daf7f 100644
--- a/drivers/pci/hotplug/pci_hotplug_core.c
+++ b/drivers/pci/hotplug/pci_hotplug_core.c
@@ -23,6 +23,9 @@
  *
  * Send feedback to <kristen.c.accardi@intel.com>
  *
+ * Authors:
+ *   Greg Kroah-Hartman <greg@kroah.com>
+ *   Scott Murray <scottm@somanetworks.com>
  */
 
 #include <linux/module.h>	/* try_module_get & module_put */
@@ -50,15 +53,9 @@
 #define info(format, arg...) printk(KERN_INFO "%s: " format, MY_NAME, ## arg)
 #define warn(format, arg...) printk(KERN_WARNING "%s: " format, MY_NAME, ## arg)
 
-
 /* local variables */
 static bool debug;
 
-#define DRIVER_VERSION	"0.5"
-#define DRIVER_AUTHOR	"Greg Kroah-Hartman <greg@kroah.com>, Scott Murray <scottm@somanetworks.com>"
-#define DRIVER_DESC	"PCI Hot Plug PCI Core"
-
-
 static LIST_HEAD(pci_hotplug_slot_list);
 static DEFINE_MUTEX(pci_hp_mutex);
 
@@ -534,7 +531,6 @@ static int __init pci_hotplug_init(void)
 		return result;
 	}
 
-	info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
 	return result;
 }
 device_initcall(pci_hotplug_init);
diff --git a/drivers/pci/hotplug/pciehp_core.c b/drivers/pci/hotplug/pciehp_core.c
index 7d32fa33dcef..35d84845d5af 100644
--- a/drivers/pci/hotplug/pciehp_core.c
+++ b/drivers/pci/hotplug/pciehp_core.c
@@ -25,6 +25,10 @@
  *
  * Send feedback to <greg@kroah.com>, <kristen.c.accardi@intel.com>
  *
+ * Authors:
+ *   Dan Zink <dan.zink@compaq.com>
+ *   Greg Kroah-Hartman <greg@kroah.com>
+ *   Dely Sy <dely.l.sy@intel.com>"
  */
 
 #include <linux/moduleparam.h>
@@ -42,10 +46,6 @@ bool pciehp_poll_mode;
 int pciehp_poll_time;
 static bool pciehp_force;
 
-#define DRIVER_VERSION	"0.4"
-#define DRIVER_AUTHOR	"Dan Zink <dan.zink@compaq.com>, Greg Kroah-Hartman <greg@kroah.com>, Dely Sy <dely.l.sy@intel.com>"
-#define DRIVER_DESC	"PCI Express Hot Plug Controller Driver"
-
 /*
  * not really modular, but the easiest way to keep compat with existing
  * bootargs behaviour is to continue using module_param here.
@@ -333,7 +333,6 @@ static int __init pcied_init(void)
 
 	retval = pcie_port_service_register(&hpdriver_portdrv);
 	dbg("pcie_port_service_register = %d\n", retval);
-	info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
 	if (retval)
 		dbg("Failure to register service\n");
 
diff --git a/drivers/pci/hotplug/pciehp_ctrl.c b/drivers/pci/hotplug/pciehp_ctrl.c
index efe69e879455..10c9c0ba8ff2 100644
--- a/drivers/pci/hotplug/pciehp_ctrl.c
+++ b/drivers/pci/hotplug/pciehp_ctrl.c
@@ -31,6 +31,7 @@
 #include <linux/kernel.h>
 #include <linux/types.h>
 #include <linux/slab.h>
+#include <linux/pm_runtime.h>
 #include <linux/pci.h>
 #include "../pci.h"
 #include "pciehp.h"
@@ -98,6 +99,7 @@ static int board_added(struct slot *p_slot)
 	pciehp_green_led_blink(p_slot);
 
 	/* Check link training status */
+	pm_runtime_get_sync(&ctrl->pcie->port->dev);
 	retval = pciehp_check_link_status(ctrl);
 	if (retval) {
 		ctrl_err(ctrl, "Failed to check link status\n");
@@ -118,12 +120,14 @@ static int board_added(struct slot *p_slot)
 		if (retval != -EEXIST)
 			goto err_exit;
 	}
+	pm_runtime_put(&ctrl->pcie->port->dev);
 
 	pciehp_green_led_on(p_slot);
 	pciehp_set_attention_status(p_slot, 0);
 	return 0;
 
 err_exit:
+	pm_runtime_put(&ctrl->pcie->port->dev);
 	set_slot_off(ctrl, p_slot);
 	return retval;
 }
@@ -137,7 +141,9 @@ static int remove_board(struct slot *p_slot)
 	int retval;
 	struct controller *ctrl = p_slot->ctrl;
 
+	pm_runtime_get_sync(&ctrl->pcie->port->dev);
 	retval = pciehp_unconfigure_device(p_slot);
+	pm_runtime_put(&ctrl->pcie->port->dev);
 	if (retval)
 		return retval;
 
@@ -410,7 +416,7 @@ int pciehp_enable_slot(struct slot *p_slot)
 		if (getstatus) {
 			ctrl_info(ctrl, "Slot(%s): Already enabled\n",
 				  slot_name(p_slot));
-			return -EINVAL;
+			return 0;
 		}
 	}
 
diff --git a/drivers/pci/hotplug/pciehp_hpc.c b/drivers/pci/hotplug/pciehp_hpc.c
index b57fc6d6e28a..026830a138ae 100644
--- a/drivers/pci/hotplug/pciehp_hpc.c
+++ b/drivers/pci/hotplug/pciehp_hpc.c
@@ -620,8 +620,18 @@ static irqreturn_t pciehp_isr(int irq, void *dev_id)
 		pciehp_queue_interrupt_event(slot, INT_BUTTON_PRESS);
 	}
 
-	/* Check Presence Detect Changed */
-	if (events & PCI_EXP_SLTSTA_PDC) {
+	/*
+	 * Check Link Status Changed at higher precedence than Presence
+	 * Detect Changed.  The PDS value may be set to "card present" from
+	 * out-of-band detection, which may be in conflict with a Link Down
+	 * and cause the wrong event to queue.
+	 */
+	if (events & PCI_EXP_SLTSTA_DLLSC) {
+		ctrl_info(ctrl, "Slot(%s): Link %s\n", slot_name(slot),
+			  link ? "Up" : "Down");
+		pciehp_queue_interrupt_event(slot, link ? INT_LINK_UP :
+					     INT_LINK_DOWN);
+	} else if (events & PCI_EXP_SLTSTA_PDC) {
 		present = !!(status & PCI_EXP_SLTSTA_PDS);
 		ctrl_info(ctrl, "Slot(%s): Card %spresent\n", slot_name(slot),
 			  present ? "" : "not ");
@@ -636,13 +646,6 @@ static irqreturn_t pciehp_isr(int irq, void *dev_id)
 		pciehp_queue_interrupt_event(slot, INT_POWER_FAULT);
 	}
 
-	if (events & PCI_EXP_SLTSTA_DLLSC) {
-		ctrl_info(ctrl, "Slot(%s): Link %s\n", slot_name(slot),
-			  link ? "Up" : "Down");
-		pciehp_queue_interrupt_event(slot, link ? INT_LINK_UP :
-					     INT_LINK_DOWN);
-	}
-
 	return IRQ_HANDLED;
 }
 
diff --git a/drivers/pci/hotplug/rpadlpar_core.c b/drivers/pci/hotplug/rpadlpar_core.c
index dc67f39779ec..c614ff7c3bc3 100644
--- a/drivers/pci/hotplug/rpadlpar_core.c
+++ b/drivers/pci/hotplug/rpadlpar_core.c
@@ -257,8 +257,13 @@ static int dlpar_add_phb(char *drc_name, struct device_node *dn)
 
 static int dlpar_add_vio_slot(char *drc_name, struct device_node *dn)
 {
-	if (vio_find_node(dn))
+	struct vio_dev *vio_dev;
+
+	vio_dev = vio_find_node(dn);
+	if (vio_dev) {
+		put_device(&vio_dev->dev);
 		return -EINVAL;
+	}
 
 	if (!vio_register_device_node(dn)) {
 		printk(KERN_ERR
@@ -334,6 +339,9 @@ static int dlpar_remove_vio_slot(char *drc_name, struct device_node *dn)
 		return -EINVAL;
 
 	vio_unregister_device(vio_dev);
+
+	put_device(&vio_dev->dev);
+
 	return 0;
 }
 
diff --git a/drivers/pci/hotplug/s390_pci_hpc.c b/drivers/pci/hotplug/s390_pci_hpc.c
index 50b8b7d54416..530d0e49f2ed 100644
--- a/drivers/pci/hotplug/s390_pci_hpc.c
+++ b/drivers/pci/hotplug/s390_pci_hpc.c
@@ -5,12 +5,13 @@
  *
  * Author(s):
  *   Jan Glauber <jang@linux.vnet.ibm.com>
+ *
+ * License: GPL
  */
 
 #define KMSG_COMPONENT "zpci"
 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 
-#include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/pci.h>
@@ -21,10 +22,6 @@
 #define SLOT_NAME_SIZE	10
 static LIST_HEAD(s390_hotplug_slot_list);
 
-MODULE_AUTHOR("Jan Glauber <jang@linux.vnet.ibm.com");
-MODULE_DESCRIPTION("Hot Plug PCI Controller for System z");
-MODULE_LICENSE("GPL");
-
 static int zpci_fn_configured(enum zpci_state state)
 {
 	return state == ZPCI_FN_STATE_CONFIGURED ||
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index e30f05c8517f..47227820406d 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -306,13 +306,6 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
 			return rc;
 	}
 
-	pci_iov_set_numvfs(dev, nr_virtfn);
-	iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE;
-	pci_cfg_access_lock(dev);
-	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
-	msleep(100);
-	pci_cfg_access_unlock(dev);
-
 	iov->initial_VFs = initial;
 	if (nr_virtfn < initial)
 		initial = nr_virtfn;
@@ -323,6 +316,13 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
 		goto err_pcibios;
 	}
 
+	pci_iov_set_numvfs(dev, nr_virtfn);
+	iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE;
+	pci_cfg_access_lock(dev);
+	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
+	msleep(100);
+	pci_cfg_access_unlock(dev);
+
 	for (i = 0; i < initial; i++) {
 		rc = pci_iov_add_virtfn(dev, i, 0);
 		if (rc)
@@ -554,21 +554,61 @@ void pci_iov_release(struct pci_dev *dev)
 }
 
 /**
- * pci_iov_resource_bar - get position of the SR-IOV BAR
+ * pci_iov_update_resource - update a VF BAR
  * @dev: the PCI device
  * @resno: the resource number
  *
- * Returns position of the BAR encapsulated in the SR-IOV capability.
+ * Update a VF BAR in the SR-IOV capability of a PF.
  */
-int pci_iov_resource_bar(struct pci_dev *dev, int resno)
+void pci_iov_update_resource(struct pci_dev *dev, int resno)
 {
-	if (resno < PCI_IOV_RESOURCES || resno > PCI_IOV_RESOURCE_END)
-		return 0;
+	struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL;
+	struct resource *res = dev->resource + resno;
+	int vf_bar = resno - PCI_IOV_RESOURCES;
+	struct pci_bus_region region;
+	u16 cmd;
+	u32 new;
+	int reg;
+
+	/*
+	 * The generic pci_restore_bars() path calls this for all devices,
+	 * including VFs and non-SR-IOV devices.  If this is not a PF, we
+	 * have nothing to do.
+	 */
+	if (!iov)
+		return;
+
+	pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &cmd);
+	if ((cmd & PCI_SRIOV_CTRL_VFE) && (cmd & PCI_SRIOV_CTRL_MSE)) {
+		dev_WARN(&dev->dev, "can't update enabled VF BAR%d %pR\n",
+			 vf_bar, res);
+		return;
+	}
+
+	/*
+	 * Ignore unimplemented BARs, unused resource slots for 64-bit
+	 * BARs, and non-movable resources, e.g., those described via
+	 * Enhanced Allocation.
+	 */
+	if (!res->flags)
+		return;
+
+	if (res->flags & IORESOURCE_UNSET)
+		return;
+
+	if (res->flags & IORESOURCE_PCI_FIXED)
+		return;
 
-	BUG_ON(!dev->is_physfn);
+	pcibios_resource_to_bus(dev->bus, &region, res);
+	new = region.start;
+	new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK;
 
-	return dev->sriov->pos + PCI_SRIOV_BAR +
-		4 * (resno - PCI_IOV_RESOURCES);
+	reg = iov->pos + PCI_SRIOV_BAR + 4 * vf_bar;
+	pci_write_config_dword(dev, reg, new);
+	if (res->flags & IORESOURCE_MEM_64) {
+		new = region.start >> 16 >> 16;
+		pci_write_config_dword(dev, reg + 4, new);
+	}
 }
 
 resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev,
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index dd27f73a45fc..50c5003295ca 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -1302,7 +1302,8 @@ const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
 	} else if (dev->msi_enabled) {
 		struct msi_desc *entry = first_pci_msi_entry(dev);
 
-		if (WARN_ON_ONCE(!entry || nr >= entry->nvec_used))
+		if (WARN_ON_ONCE(!entry || !entry->affinity ||
+				 nr >= entry->nvec_used))
 			return NULL;
 
 		return &entry->affinity[nr];
diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c
index d966d47c9e80..001860361434 100644
--- a/drivers/pci/pci-acpi.c
+++ b/drivers/pci/pci-acpi.c
@@ -29,6 +29,82 @@ const u8 pci_acpi_dsm_uuid[] = {
 	0x91, 0x17, 0xea, 0x4d, 0x19, 0xc3, 0x43, 0x4d
 };
 
+#if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_ARM64)
+static int acpi_get_rc_addr(struct acpi_device *adev, struct resource *res)
+{
+	struct device *dev = &adev->dev;
+	struct resource_entry *entry;
+	struct list_head list;
+	unsigned long flags;
+	int ret;
+
+	INIT_LIST_HEAD(&list);
+	flags = IORESOURCE_MEM;
+	ret = acpi_dev_get_resources(adev, &list,
+				     acpi_dev_filter_resource_type_cb,
+				     (void *) flags);
+	if (ret < 0) {
+		dev_err(dev, "failed to parse _CRS method, error code %d\n",
+			ret);
+		return ret;
+	}
+
+	if (ret == 0) {
+		dev_err(dev, "no IO and memory resources present in _CRS\n");
+		return -EINVAL;
+	}
+
+	entry = list_first_entry(&list, struct resource_entry, node);
+	*res = *entry->res;
+	acpi_dev_free_resource_list(&list);
+	return 0;
+}
+
+static acpi_status acpi_match_rc(acpi_handle handle, u32 lvl, void *context,
+				 void **retval)
+{
+	u16 *segment = context;
+	unsigned long long uid;
+	acpi_status status;
+
+	status = acpi_evaluate_integer(handle, "_UID", NULL, &uid);
+	if (ACPI_FAILURE(status) || uid != *segment)
+		return AE_CTRL_DEPTH;
+
+	*(acpi_handle *)retval = handle;
+	return AE_CTRL_TERMINATE;
+}
+
+int acpi_get_rc_resources(struct device *dev, const char *hid, u16 segment,
+			  struct resource *res)
+{
+	struct acpi_device *adev;
+	acpi_status status;
+	acpi_handle handle;
+	int ret;
+
+	status = acpi_get_devices(hid, acpi_match_rc, &segment, &handle);
+	if (ACPI_FAILURE(status)) {
+		dev_err(dev, "can't find _HID %s device to locate resources\n",
+			hid);
+		return -ENODEV;
+	}
+
+	ret = acpi_bus_get_device(handle, &adev);
+	if (ret)
+		return ret;
+
+	ret = acpi_get_rc_addr(adev, res);
+	if (ret) {
+		dev_err(dev, "can't get resource from %s\n",
+			dev_name(&adev->dev));
+		return ret;
+	}
+
+	return 0;
+}
+#endif
+
 phys_addr_t acpi_pci_root_get_mcfg_addr(acpi_handle handle)
 {
 	acpi_status status = AE_NOT_EXIST;
@@ -294,6 +370,30 @@ int pci_get_hp_params(struct pci_dev *dev, struct hotplug_params *hpp)
 EXPORT_SYMBOL_GPL(pci_get_hp_params);
 
 /**
+ * pciehp_is_native - Check whether a hotplug port is handled by the OS
+ * @pdev: Hotplug port to check
+ *
+ * Walk up from @pdev to the host bridge, obtain its cached _OSC Control Field
+ * and return the value of the "PCI Express Native Hot Plug control" bit.
+ * On failure to obtain the _OSC Control Field return %false.
+ */
+bool pciehp_is_native(struct pci_dev *pdev)
+{
+	struct acpi_pci_root *root;
+	acpi_handle handle;
+
+	handle = acpi_find_root_bridge_handle(pdev);
+	if (!handle)
+		return false;
+
+	root = acpi_pci_find_root(handle);
+	if (!root)
+		return false;
+
+	return root->osc_control_set & OSC_PCI_EXPRESS_NATIVE_HP_CONTROL;
+}
+
+/**
  * pci_acpi_wake_bus - Root bus wakeup notification fork function.
  * @work: Work item to handle.
  */
diff --git a/drivers/pci/pci-mid.c b/drivers/pci/pci-mid.c
index c7f3408e3148..1c4af7227bca 100644
--- a/drivers/pci/pci-mid.c
+++ b/drivers/pci/pci-mid.c
@@ -54,7 +54,7 @@ static bool mid_pci_need_resume(struct pci_dev *dev)
 	return false;
 }
 
-static struct pci_platform_pm_ops mid_pci_platform_pm = {
+static const struct pci_platform_pm_ops mid_pci_platform_pm = {
 	.is_manageable	= mid_pci_power_manageable,
 	.set_state	= mid_pci_set_power_state,
 	.get_state	= mid_pci_get_power_state,
diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index bcd10c795284..066628776e1b 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -50,6 +50,7 @@ pci_config_attr(vendor, "0x%04x\n");
 pci_config_attr(device, "0x%04x\n");
 pci_config_attr(subsystem_vendor, "0x%04x\n");
 pci_config_attr(subsystem_device, "0x%04x\n");
+pci_config_attr(revision, "0x%02x\n");
 pci_config_attr(class, "0x%06x\n");
 pci_config_attr(irq, "%u\n");
 
@@ -568,6 +569,7 @@ static struct attribute *pci_dev_attrs[] = {
 	&dev_attr_device.attr,
 	&dev_attr_subsystem_vendor.attr,
 	&dev_attr_subsystem_device.attr,
+	&dev_attr_revision.attr,
 	&dev_attr_class.attr,
 	&dev_attr_irq.attr,
 	&dev_attr_local_cpus.attr,
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index ba34907538f6..a881c0d3d2e8 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -564,10 +564,6 @@ static void pci_restore_bars(struct pci_dev *dev)
 {
 	int i;
 
-	/* Per SR-IOV spec 3.4.1.11, VF BARs are RO zero */
-	if (dev->is_virtfn)
-		return;
-
 	for (i = 0; i < PCI_BRIDGE_RESOURCES; i++)
 		pci_update_resource(dev, i);
 }
@@ -2106,6 +2102,10 @@ bool pci_dev_run_wake(struct pci_dev *dev)
 	if (!dev->pme_support)
 		return false;
 
+	/* PME-capable in principle, but not from the intended sleep state */
+	if (!pci_pme_capable(dev, pci_target_state(dev)))
+		return false;
+
 	while (bus->parent) {
 		struct pci_dev *bridge = bus->self;
 
@@ -2226,7 +2226,7 @@ void pci_config_pm_runtime_put(struct pci_dev *pdev)
  * This function checks if it is possible to move the bridge to D3.
  * Currently we only allow D3 for recent enough PCIe ports.
  */
-static bool pci_bridge_d3_possible(struct pci_dev *bridge)
+bool pci_bridge_d3_possible(struct pci_dev *bridge)
 {
 	unsigned int year;
 
@@ -2239,6 +2239,14 @@ static bool pci_bridge_d3_possible(struct pci_dev *bridge)
 	case PCI_EXP_TYPE_DOWNSTREAM:
 		if (pci_bridge_d3_disable)
 			return false;
+
+		/*
+		 * Hotplug ports handled by firmware in System Management Mode
+		 * may not be put into D3 by the OS (Thunderbolt on non-Macs).
+		 */
+		if (bridge->is_hotplug_bridge && !pciehp_is_native(bridge))
+			return false;
+
 		if (pci_bridge_d3_force)
 			return true;
 
@@ -2259,32 +2267,36 @@ static bool pci_bridge_d3_possible(struct pci_dev *bridge)
 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
 {
 	bool *d3cold_ok = data;
-	bool no_d3cold;
 
-	/*
-	 * The device needs to be allowed to go D3cold and if it is wake
-	 * capable to do so from D3cold.
-	 */
-	no_d3cold = dev->no_d3cold || !dev->d3cold_allowed ||
-		(device_may_wakeup(&dev->dev) && !pci_pme_capable(dev, PCI_D3cold)) ||
-		!pci_power_manageable(dev);
+	if (/* The device needs to be allowed to go D3cold ... */
+	    dev->no_d3cold || !dev->d3cold_allowed ||
 
-	*d3cold_ok = !no_d3cold;
+	    /* ... and if it is wakeup capable to do so from D3cold. */
+	    (device_may_wakeup(&dev->dev) &&
+	     !pci_pme_capable(dev, PCI_D3cold)) ||
 
-	return no_d3cold;
+	    /* If it is a bridge it must be allowed to go to D3. */
+	    !pci_power_manageable(dev) ||
+
+	    /* Hotplug interrupts cannot be delivered if the link is down. */
+	    dev->is_hotplug_bridge)
+
+		*d3cold_ok = false;
+
+	return !*d3cold_ok;
 }
 
 /*
  * pci_bridge_d3_update - Update bridge D3 capabilities
  * @dev: PCI device which is changed
- * @remove: Is the device being removed
  *
  * Update upstream bridge PM capabilities accordingly depending on if the
  * device PM configuration was changed or the device is being removed.  The
  * change is also propagated upstream.
  */
-static void pci_bridge_d3_update(struct pci_dev *dev, bool remove)
+void pci_bridge_d3_update(struct pci_dev *dev)
 {
+	bool remove = !device_is_registered(&dev->dev);
 	struct pci_dev *bridge;
 	bool d3cold_ok = true;
 
@@ -2292,55 +2304,39 @@ static void pci_bridge_d3_update(struct pci_dev *dev, bool remove)
 	if (!bridge || !pci_bridge_d3_possible(bridge))
 		return;
 
-	pci_dev_get(bridge);
 	/*
-	 * If the device is removed we do not care about its D3cold
-	 * capabilities.
+	 * If D3 is currently allowed for the bridge, removing one of its
+	 * children won't change that.
+	 */
+	if (remove && bridge->bridge_d3)
+		return;
+
+	/*
+	 * If D3 is currently allowed for the bridge and a child is added or
+	 * changed, disallowance of D3 can only be caused by that child, so
+	 * we only need to check that single device, not any of its siblings.
+	 *
+	 * If D3 is currently not allowed for the bridge, checking the device
+	 * first may allow us to skip checking its siblings.
 	 */
 	if (!remove)
 		pci_dev_check_d3cold(dev, &d3cold_ok);
 
-	if (d3cold_ok) {
-		/*
-		 * We need to go through all children to find out if all of
-		 * them can still go to D3cold.
-		 */
+	/*
+	 * If D3 is currently not allowed for the bridge, this may be caused
+	 * either by the device being changed/removed or any of its siblings,
+	 * so we need to go through all children to find out if one of them
+	 * continues to block D3.
+	 */
+	if (d3cold_ok && !bridge->bridge_d3)
 		pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold,
 			     &d3cold_ok);
-	}
 
 	if (bridge->bridge_d3 != d3cold_ok) {
 		bridge->bridge_d3 = d3cold_ok;
 		/* Propagate change to upstream bridges */
-		pci_bridge_d3_update(bridge, false);
+		pci_bridge_d3_update(bridge);
 	}
-
-	pci_dev_put(bridge);
-}
-
-/**
- * pci_bridge_d3_device_changed - Update bridge D3 capabilities on change
- * @dev: PCI device that was changed
- *
- * If a device is added or its PM configuration, such as is it allowed to
- * enter D3cold, is changed this function updates upstream bridge PM
- * capabilities accordingly.
- */
-void pci_bridge_d3_device_changed(struct pci_dev *dev)
-{
-	pci_bridge_d3_update(dev, false);
-}
-
-/**
- * pci_bridge_d3_device_removed - Update bridge D3 capabilities on remove
- * @dev: PCI device being removed
- *
- * Function updates upstream bridge PM capabilities based on other devices
- * still left on the bus.
- */
-void pci_bridge_d3_device_removed(struct pci_dev *dev)
-{
-	pci_bridge_d3_update(dev, true);
 }
 
 /**
@@ -2355,7 +2351,7 @@ void pci_d3cold_enable(struct pci_dev *dev)
 {
 	if (dev->no_d3cold) {
 		dev->no_d3cold = false;
-		pci_bridge_d3_device_changed(dev);
+		pci_bridge_d3_update(dev);
 	}
 }
 EXPORT_SYMBOL_GPL(pci_d3cold_enable);
@@ -2372,7 +2368,7 @@ void pci_d3cold_disable(struct pci_dev *dev)
 {
 	if (!dev->no_d3cold) {
 		dev->no_d3cold = true;
-		pci_bridge_d3_device_changed(dev);
+		pci_bridge_d3_update(dev);
 	}
 }
 EXPORT_SYMBOL_GPL(pci_d3cold_disable);
@@ -4831,36 +4827,6 @@ int pci_select_bars(struct pci_dev *dev, unsigned long flags)
 }
 EXPORT_SYMBOL(pci_select_bars);
 
-/**
- * pci_resource_bar - get position of the BAR associated with a resource
- * @dev: the PCI device
- * @resno: the resource number
- * @type: the BAR type to be filled in
- *
- * Returns BAR position in config space, or 0 if the BAR is invalid.
- */
-int pci_resource_bar(struct pci_dev *dev, int resno, enum pci_bar_type *type)
-{
-	int reg;
-
-	if (resno < PCI_ROM_RESOURCE) {
-		*type = pci_bar_unknown;
-		return PCI_BASE_ADDRESS_0 + 4 * resno;
-	} else if (resno == PCI_ROM_RESOURCE) {
-		*type = pci_bar_mem32;
-		return dev->rom_base_reg;
-	} else if (resno < PCI_BRIDGE_RESOURCES) {
-		/* device specific resource */
-		*type = pci_bar_unknown;
-		reg = pci_iov_resource_bar(dev, resno);
-		if (reg)
-			return reg;
-	}
-
-	dev_err(&dev->dev, "BAR %d: invalid resource\n", resno);
-	return 0;
-}
-
 /* Some architectures require additional programming to enable VGA */
 static arch_set_vga_state_t arch_set_vga_state;
 
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 451856210e18..cb17db242f30 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -1,9 +1,6 @@
 #ifndef DRIVERS_PCI_H
 #define DRIVERS_PCI_H
 
-#define PCI_CFG_SPACE_SIZE	256
-#define PCI_CFG_SPACE_EXP_SIZE	4096
-
 #define PCI_FIND_CAP_TTL	48
 
 extern const unsigned char pcie_link_speed[];
@@ -85,8 +82,8 @@ void pci_pm_init(struct pci_dev *dev);
 void pci_ea_init(struct pci_dev *dev);
 void pci_allocate_cap_save_buffers(struct pci_dev *dev);
 void pci_free_cap_save_buffers(struct pci_dev *dev);
-void pci_bridge_d3_device_changed(struct pci_dev *dev);
-void pci_bridge_d3_device_removed(struct pci_dev *dev);
+bool pci_bridge_d3_possible(struct pci_dev *dev);
+void pci_bridge_d3_update(struct pci_dev *dev);
 
 static inline void pci_wakeup_event(struct pci_dev *dev)
 {
@@ -245,7 +242,6 @@ bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *pl,
 int pci_setup_device(struct pci_dev *dev);
 int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
 		    struct resource *res, unsigned int reg);
-int pci_resource_bar(struct pci_dev *dev, int resno, enum pci_bar_type *type);
 void pci_configure_ari(struct pci_dev *dev);
 void __pci_bus_size_bridges(struct pci_bus *bus,
 			struct list_head *realloc_head);
@@ -289,7 +285,7 @@ static inline void pci_restore_ats_state(struct pci_dev *dev)
 #ifdef CONFIG_PCI_IOV
 int pci_iov_init(struct pci_dev *dev);
 void pci_iov_release(struct pci_dev *dev);
-int pci_iov_resource_bar(struct pci_dev *dev, int resno);
+void pci_iov_update_resource(struct pci_dev *dev, int resno);
 resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno);
 void pci_restore_iov_state(struct pci_dev *dev);
 int pci_iov_bus_range(struct pci_bus *bus);
@@ -303,10 +299,6 @@ static inline void pci_iov_release(struct pci_dev *dev)
 
 {
 }
-static inline int pci_iov_resource_bar(struct pci_dev *dev, int resno)
-{
-	return 0;
-}
 static inline void pci_restore_iov_state(struct pci_dev *dev)
 {
 }
@@ -356,4 +348,9 @@ static inline int pci_dev_specific_reset(struct pci_dev *dev, int probe)
 }
 #endif
 
+#if defined(CONFIG_PCI_QUIRKS) && defined(CONFIG_ARM64)
+int acpi_get_rc_resources(struct device *dev, const char *hid, u16 segment,
+			  struct resource *res);
+#endif
+
 #endif /* DRIVERS_PCI_H */
diff --git a/drivers/pci/pcie/aer/aerdrv.c b/drivers/pci/pcie/aer/aerdrv.c
index 139150b2bdfd..dea186a9d6b6 100644
--- a/drivers/pci/pcie/aer/aerdrv.c
+++ b/drivers/pci/pcie/aer/aerdrv.c
@@ -30,13 +30,6 @@
 #include "aerdrv.h"
 #include "../../pci.h"
 
-/*
- * Version Information
- */
-#define DRIVER_VERSION "v1.0"
-#define DRIVER_AUTHOR "tom.l.nguyen@intel.com"
-#define DRIVER_DESC "Root Port Advanced Error Reporting Driver"
-
 static int aer_probe(struct pcie_device *dev);
 static void aer_remove(struct pcie_device *dev);
 static pci_ers_result_t aer_error_detected(struct pci_dev *dev,
@@ -297,12 +290,12 @@ static int aer_probe(struct pcie_device *dev)
 {
 	int status;
 	struct aer_rpc *rpc;
-	struct device *device = &dev->device;
+	struct device *device = &dev->port->dev;
 
 	/* Alloc rpc data structure */
 	rpc = aer_alloc_rpc(dev);
 	if (!rpc) {
-		dev_printk(KERN_DEBUG, device, "alloc rpc failed\n");
+		dev_printk(KERN_DEBUG, device, "alloc AER rpc failed\n");
 		aer_remove(dev);
 		return -ENOMEM;
 	}
@@ -310,7 +303,8 @@ static int aer_probe(struct pcie_device *dev)
 	/* Request IRQ ISR */
 	status = request_irq(dev->irq, aer_irq, IRQF_SHARED, "aerdrv", dev);
 	if (status) {
-		dev_printk(KERN_DEBUG, device, "request IRQ failed\n");
+		dev_printk(KERN_DEBUG, device, "request AER IRQ %d failed\n",
+			   dev->irq);
 		aer_remove(dev);
 		return status;
 	}
@@ -318,8 +312,8 @@ static int aer_probe(struct pcie_device *dev)
 	rpc->isr = 1;
 
 	aer_enable_rootport(rpc);
-
-	return status;
+	dev_info(device, "AER enabled with IRQ %d\n", dev->irq);
+	return 0;
 }
 
 /**
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index 0ec649d961d7..17ac1dce3286 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -351,12 +351,26 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
 		return;
 	}
 
+	/* Get upstream/downstream components' register state */
+	pcie_get_aspm_reg(parent, &upreg);
+	child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
+	pcie_get_aspm_reg(child, &dwreg);
+
+	/*
+	 * If ASPM not supported, don't mess with the clocks and link,
+	 * bail out now.
+	 */
+	if (!(upreg.support & dwreg.support))
+		return;
+
 	/* Configure common clock before checking latencies */
 	pcie_aspm_configure_common_clock(link);
 
-	/* Get upstream/downstream components' register state */
+	/*
+	 * Re-read upstream/downstream components' register state
+	 * after clock configuration
+	 */
 	pcie_get_aspm_reg(parent, &upreg);
-	child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
 	pcie_get_aspm_reg(child, &dwreg);
 
 	/*
@@ -886,8 +900,8 @@ static ssize_t clk_ctl_store(struct device *dev,
 	return n;
 }
 
-static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
-static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
+static DEVICE_ATTR_RW(link_state);
+static DEVICE_ATTR_RW(clk_ctl);
 
 static char power_group[] = "power";
 void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
diff --git a/drivers/pci/pcie/pme.c b/drivers/pci/pcie/pme.c
index 884bad5320f8..717529331dac 100644
--- a/drivers/pci/pcie/pme.c
+++ b/drivers/pci/pcie/pme.c
@@ -300,8 +300,6 @@ static irqreturn_t pcie_pme_irq(int irq, void *context)
  */
 static int pcie_pme_set_native(struct pci_dev *dev, void *ign)
 {
-	dev_info(&dev->dev, "Signaling PME through PCIe PME interrupt\n");
-
 	device_set_run_wake(&dev->dev, true);
 	dev->pme_interrupt = true;
 	return 0;
@@ -319,23 +317,8 @@ static int pcie_pme_set_native(struct pci_dev *dev, void *ign)
 static void pcie_pme_mark_devices(struct pci_dev *port)
 {
 	pcie_pme_set_native(port, NULL);
-	if (port->subordinate) {
+	if (port->subordinate)
 		pci_walk_bus(port->subordinate, pcie_pme_set_native, NULL);
-	} else {
-		struct pci_bus *bus = port->bus;
-		struct pci_dev *dev;
-
-		/* Check if this is a root port event collector. */
-		if (pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC || !bus)
-			return;
-
-		down_read(&pci_bus_sem);
-		list_for_each_entry(dev, &bus->devices, bus_list)
-			if (pci_is_pcie(dev)
-			    && pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END)
-				pcie_pme_set_native(dev, NULL);
-		up_read(&pci_bus_sem);
-	}
 }
 
 /**
@@ -364,12 +347,14 @@ static int pcie_pme_probe(struct pcie_device *srv)
 	ret = request_irq(srv->irq, pcie_pme_irq, IRQF_SHARED, "PCIe PME", srv);
 	if (ret) {
 		kfree(data);
-	} else {
-		pcie_pme_mark_devices(port);
-		pcie_pme_interrupt_enable(port, true);
+		return ret;
 	}
 
-	return ret;
+	dev_info(&port->dev, "Signaling PME with IRQ %d\n", srv->irq);
+
+	pcie_pme_mark_devices(port);
+	pcie_pme_interrupt_enable(port, true);
+	return 0;
 }
 
 static bool pcie_pme_check_wakeup(struct pci_bus *bus)
diff --git a/drivers/pci/pcie/portdrv_core.c b/drivers/pci/pcie/portdrv_core.c
index e9270b4026f3..9698289f105c 100644
--- a/drivers/pci/pcie/portdrv_core.c
+++ b/drivers/pci/pcie/portdrv_core.c
@@ -499,7 +499,6 @@ static int pcie_port_probe_service(struct device *dev)
 	if (status)
 		return status;
 
-	dev_printk(KERN_DEBUG, dev, "service driver %s loaded\n", driver->name);
 	get_device(dev);
 	return 0;
 }
@@ -524,8 +523,6 @@ static int pcie_port_remove_service(struct device *dev)
 	pciedev = to_pcie_device(dev);
 	driver = to_service_driver(dev->driver);
 	if (driver && driver->remove) {
-		dev_printk(KERN_DEBUG, dev, "unloading service driver %s\n",
-			driver->name);
 		driver->remove(pciedev);
 		put_device(dev);
 	}
diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c
index 79327cc14e7d..8aa3f14bc87d 100644
--- a/drivers/pci/pcie/portdrv_pci.c
+++ b/drivers/pci/pcie/portdrv_pci.c
@@ -19,6 +19,7 @@
 #include <linux/dmi.h>
 #include <linux/pci-aspm.h>
 
+#include "../pci.h"
 #include "portdrv.h"
 #include "aer/aerdrv.h"
 
@@ -149,15 +150,7 @@ static int pcie_portdrv_probe(struct pci_dev *dev,
 
 	pci_save_state(dev);
 
-	/*
-	 * Prevent runtime PM if the port is advertising support for PCIe
-	 * hotplug.  Otherwise the BIOS hotplug SMI code might not be able
-	 * to enumerate devices behind this port properly (the port is
-	 * powered down preventing all config space accesses to the
-	 * subordinate devices).  We can't be sure for native PCIe hotplug
-	 * either so prevent that as well.
-	 */
-	if (!dev->is_hotplug_bridge) {
+	if (pci_bridge_d3_possible(dev)) {
 		/*
 		 * Keep the port resumed 100ms to make sure things like
 		 * config space accesses from userspace (lspci) will not
@@ -175,7 +168,7 @@ static int pcie_portdrv_probe(struct pci_dev *dev,
 
 static void pcie_portdrv_remove(struct pci_dev *dev)
 {
-	if (!dev->is_hotplug_bridge) {
+	if (pci_bridge_d3_possible(dev)) {
 		pm_runtime_forbid(&dev->dev);
 		pm_runtime_get_noresume(&dev->dev);
 		pm_runtime_dont_use_autosuspend(&dev->dev);
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 104c46d53121..e164b5c9f0f0 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -227,7 +227,8 @@ int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
 			mask64 = (u32)PCI_BASE_ADDRESS_MEM_MASK;
 		}
 	} else {
-		res->flags |= (l & IORESOURCE_ROM_ENABLE);
+		if (l & PCI_ROM_ADDRESS_ENABLE)
+			res->flags |= IORESOURCE_ROM_ENABLE;
 		l64 = l & PCI_ROM_ADDRESS_MASK;
 		sz64 = sz & PCI_ROM_ADDRESS_MASK;
 		mask64 = (u32)PCI_ROM_ADDRESS_MASK;
@@ -521,18 +522,19 @@ static void pci_release_host_bridge_dev(struct device *dev)
 	kfree(bridge);
 }
 
-static struct pci_host_bridge *pci_alloc_host_bridge(struct pci_bus *b)
+struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
 {
 	struct pci_host_bridge *bridge;
 
-	bridge = kzalloc(sizeof(*bridge), GFP_KERNEL);
+	bridge = kzalloc(sizeof(*bridge) + priv, GFP_KERNEL);
 	if (!bridge)
 		return NULL;
 
 	INIT_LIST_HEAD(&bridge->windows);
-	bridge->bus = b;
+
 	return bridge;
 }
+EXPORT_SYMBOL(pci_alloc_host_bridge);
 
 static const unsigned char pcix_bus_speed[] = {
 	PCI_SPEED_UNKNOWN,		/* 0 */
@@ -717,6 +719,123 @@ static void pci_set_bus_msi_domain(struct pci_bus *bus)
 	dev_set_msi_domain(&bus->dev, d);
 }
 
+int pci_register_host_bridge(struct pci_host_bridge *bridge)
+{
+	struct device *parent = bridge->dev.parent;
+	struct resource_entry *window, *n;
+	struct pci_bus *bus, *b;
+	resource_size_t offset;
+	LIST_HEAD(resources);
+	struct resource *res;
+	char addr[64], *fmt;
+	const char *name;
+	int err;
+
+	bus = pci_alloc_bus(NULL);
+	if (!bus)
+		return -ENOMEM;
+
+	bridge->bus = bus;
+
+	/* temporarily move resources off the list */
+	list_splice_init(&bridge->windows, &resources);
+	bus->sysdata = bridge->sysdata;
+	bus->msi = bridge->msi;
+	bus->ops = bridge->ops;
+	bus->number = bus->busn_res.start = bridge->busnr;
+#ifdef CONFIG_PCI_DOMAINS_GENERIC
+	bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
+#endif
+
+	b = pci_find_bus(pci_domain_nr(bus), bridge->busnr);
+	if (b) {
+		/* If we already got to this bus through a different bridge, ignore it */
+		dev_dbg(&b->dev, "bus already known\n");
+		err = -EEXIST;
+		goto free;
+	}
+
+	dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(bus),
+		     bridge->busnr);
+
+	err = pcibios_root_bridge_prepare(bridge);
+	if (err)
+		goto free;
+
+	err = device_register(&bridge->dev);
+	if (err)
+		put_device(&bridge->dev);
+
+	bus->bridge = get_device(&bridge->dev);
+	device_enable_async_suspend(bus->bridge);
+	pci_set_bus_of_node(bus);
+	pci_set_bus_msi_domain(bus);
+
+	if (!parent)
+		set_dev_node(bus->bridge, pcibus_to_node(bus));
+
+	bus->dev.class = &pcibus_class;
+	bus->dev.parent = bus->bridge;
+
+	dev_set_name(&bus->dev, "%04x:%02x", pci_domain_nr(bus), bus->number);
+	name = dev_name(&bus->dev);
+
+	err = device_register(&bus->dev);
+	if (err)
+		goto unregister;
+
+	pcibios_add_bus(bus);
+
+	/* Create legacy_io and legacy_mem files for this bus */
+	pci_create_legacy_files(bus);
+
+	if (parent)
+		dev_info(parent, "PCI host bridge to bus %s\n", name);
+	else
+		pr_info("PCI host bridge to bus %s\n", name);
+
+	/* Add initial resources to the bus */
+	resource_list_for_each_entry_safe(window, n, &resources) {
+		list_move_tail(&window->node, &bridge->windows);
+		offset = window->offset;
+		res = window->res;
+
+		if (res->flags & IORESOURCE_BUS)
+			pci_bus_insert_busn_res(bus, bus->number, res->end);
+		else
+			pci_bus_add_resource(bus, res, 0);
+
+		if (offset) {
+			if (resource_type(res) == IORESOURCE_IO)
+				fmt = " (bus address [%#06llx-%#06llx])";
+			else
+				fmt = " (bus address [%#010llx-%#010llx])";
+
+			snprintf(addr, sizeof(addr), fmt,
+				 (unsigned long long)(res->start - offset),
+				 (unsigned long long)(res->end - offset));
+		} else
+			addr[0] = '\0';
+
+		dev_info(&bus->dev, "root bus resource %pR%s\n", res, addr);
+	}
+
+	down_write(&pci_bus_sem);
+	list_add_tail(&bus->node, &pci_root_buses);
+	up_write(&pci_bus_sem);
+
+	return 0;
+
+unregister:
+	put_device(&bridge->dev);
+	device_unregister(&bridge->dev);
+
+free:
+	kfree(bus);
+	return err;
+}
+EXPORT_SYMBOL(pci_register_host_bridge);
+
 static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
 					   struct pci_dev *bridge, int busnr)
 {
@@ -1764,8 +1883,7 @@ static void pci_dma_configure(struct pci_dev *dev)
 		if (attr == DEV_DMA_NOT_SUPPORTED)
 			dev_warn(&dev->dev, "DMA not supported.\n");
 		else
-			arch_setup_dma_ops(&dev->dev, 0, 0, NULL,
-					   attr == DEV_DMA_COHERENT);
+			acpi_dma_configure(&dev->dev, attr);
 	}
 
 	pci_put_host_bridge_device(bridge);
@@ -2156,113 +2274,43 @@ void __weak pcibios_remove_bus(struct pci_bus *bus)
 {
 }
 
-struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
-		struct pci_ops *ops, void *sysdata, struct list_head *resources)
+static struct pci_bus *pci_create_root_bus_msi(struct device *parent,
+		int bus, struct pci_ops *ops, void *sysdata,
+		struct list_head *resources, struct msi_controller *msi)
 {
 	int error;
 	struct pci_host_bridge *bridge;
-	struct pci_bus *b, *b2;
-	struct resource_entry *window, *n;
-	struct resource *res;
-	resource_size_t offset;
-	char bus_addr[64];
-	char *fmt;
-
-	b = pci_alloc_bus(NULL);
-	if (!b)
-		return NULL;
 
-	b->sysdata = sysdata;
-	b->ops = ops;
-	b->number = b->busn_res.start = bus;
-#ifdef CONFIG_PCI_DOMAINS_GENERIC
-	b->domain_nr = pci_bus_find_domain_nr(b, parent);
-#endif
-	b2 = pci_find_bus(pci_domain_nr(b), bus);
-	if (b2) {
-		/* If we already got to this bus through a different bridge, ignore it */
-		dev_dbg(&b2->dev, "bus already known\n");
-		goto err_out;
-	}
-
-	bridge = pci_alloc_host_bridge(b);
+	bridge = pci_alloc_host_bridge(0);
 	if (!bridge)
-		goto err_out;
+		return NULL;
 
 	bridge->dev.parent = parent;
 	bridge->dev.release = pci_release_host_bridge_dev;
-	dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(b), bus);
-	error = pcibios_root_bridge_prepare(bridge);
-	if (error) {
-		kfree(bridge);
-		goto err_out;
-	}
-
-	error = device_register(&bridge->dev);
-	if (error) {
-		put_device(&bridge->dev);
-		goto err_out;
-	}
-	b->bridge = get_device(&bridge->dev);
-	device_enable_async_suspend(b->bridge);
-	pci_set_bus_of_node(b);
-	pci_set_bus_msi_domain(b);
 
-	if (!parent)
-		set_dev_node(b->bridge, pcibus_to_node(b));
-
-	b->dev.class = &pcibus_class;
-	b->dev.parent = b->bridge;
-	dev_set_name(&b->dev, "%04x:%02x", pci_domain_nr(b), bus);
-	error = device_register(&b->dev);
-	if (error)
-		goto class_dev_reg_err;
+	list_splice_init(resources, &bridge->windows);
+	bridge->sysdata = sysdata;
+	bridge->busnr = bus;
+	bridge->ops = ops;
+	bridge->msi = msi;
 
-	pcibios_add_bus(b);
-
-	/* Create legacy_io and legacy_mem files for this bus */
-	pci_create_legacy_files(b);
-
-	if (parent)
-		dev_info(parent, "PCI host bridge to bus %s\n", dev_name(&b->dev));
-	else
-		printk(KERN_INFO "PCI host bridge to bus %s\n", dev_name(&b->dev));
-
-	/* Add initial resources to the bus */
-	resource_list_for_each_entry_safe(window, n, resources) {
-		list_move_tail(&window->node, &bridge->windows);
-		res = window->res;
-		offset = window->offset;
-		if (res->flags & IORESOURCE_BUS)
-			pci_bus_insert_busn_res(b, bus, res->end);
-		else
-			pci_bus_add_resource(b, res, 0);
-		if (offset) {
-			if (resource_type(res) == IORESOURCE_IO)
-				fmt = " (bus address [%#06llx-%#06llx])";
-			else
-				fmt = " (bus address [%#010llx-%#010llx])";
-			snprintf(bus_addr, sizeof(bus_addr), fmt,
-				 (unsigned long long) (res->start - offset),
-				 (unsigned long long) (res->end - offset));
-		} else
-			bus_addr[0] = '\0';
-		dev_info(&b->dev, "root bus resource %pR%s\n", res, bus_addr);
-	}
+	error = pci_register_host_bridge(bridge);
+	if (error < 0)
+		goto err_out;
 
-	down_write(&pci_bus_sem);
-	list_add_tail(&b->node, &pci_root_buses);
-	up_write(&pci_bus_sem);
+	return bridge->bus;
 
-	return b;
-
-class_dev_reg_err:
-	put_device(&bridge->dev);
-	device_unregister(&bridge->dev);
 err_out:
-	kfree(b);
+	kfree(bridge);
 	return NULL;
 }
+
+struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
+		struct pci_ops *ops, void *sysdata, struct list_head *resources)
+{
+	return pci_create_root_bus_msi(parent, bus, ops, sysdata, resources,
+				       NULL);
+}
 EXPORT_SYMBOL_GPL(pci_create_root_bus);
 
 int pci_bus_insert_busn_res(struct pci_bus *b, int bus, int bus_max)
@@ -2343,12 +2391,10 @@ struct pci_bus *pci_scan_root_bus_msi(struct device *parent, int bus,
 			break;
 		}
 
-	b = pci_create_root_bus(parent, bus, ops, sysdata, resources);
+	b = pci_create_root_bus_msi(parent, bus, ops, sysdata, resources, msi);
 	if (!b)
 		return NULL;
 
-	b->msi = msi;
-
 	if (!found) {
 		dev_info(&b->dev,
 		 "No busn resource found for root bus, will use [bus %02x-ff]\n",
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index c232729f5b1b..9236e40ac055 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -2156,7 +2156,7 @@ static void quirk_blacklist_vpd(struct pci_dev *dev)
 {
 	if (dev->vpd) {
 		dev->vpd->len = 0;
-		dev_warn(&dev->dev, FW_BUG "VPD access disabled\n");
+		dev_warn(&dev->dev, FW_BUG "disabling VPD access (can't determine size of non-standard VPD format)\n");
 	}
 }
 
@@ -3137,8 +3137,9 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b5, quirk_remove_d3_delay);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x22b7, quirk_remove_d3_delay);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x2298, quirk_remove_d3_delay);
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x229c, quirk_remove_d3_delay);
+
 /*
- * Some devices may pass our check in pci_intx_mask_supported if
+ * Some devices may pass our check in pci_intx_mask_supported() if
  * PCI_COMMAND_INTX_DISABLE works though they actually do not properly
  * support this feature.
  */
@@ -3146,53 +3147,139 @@ static void quirk_broken_intx_masking(struct pci_dev *dev)
 {
 	dev->broken_intx_masking = 1;
 }
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_CHELSIO, 0x0030,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(0x1814, 0x0601, /* Ralink RT2800 802.11n PCI */
-			 quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x0030,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(0x1814, 0x0601, /* Ralink RT2800 802.11n PCI */
+			quirk_broken_intx_masking);
+
 /*
  * Realtek RTL8169 PCI Gigabit Ethernet Controller (rev 10)
  * Subsystem: Realtek RTL8169/8110 Family PCI Gigabit Ethernet NIC
  *
  * RTL8110SC - Fails under PCI device assignment using DisINTx masking.
  */
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_REALTEK, 0x8169,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MELLANOX, PCI_ANY_ID,
-			 quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_REALTEK, 0x8169,
+			quirk_broken_intx_masking);
 
 /*
  * Intel i40e (XL710/X710) 10/20/40GbE NICs all have broken INTx masking,
  * DisINTx can be set but the interrupt status bit is non-functional.
  */
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1572,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1574,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1580,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1581,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1583,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1584,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1585,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1586,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1587,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1588,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1589,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x37d0,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x37d1,
-			 quirk_broken_intx_masking);
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x37d2,
-			 quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1572,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1574,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1580,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1581,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1583,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1584,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1585,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1586,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1587,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1588,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x1589,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d0,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d1,
+			quirk_broken_intx_masking);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x37d2,
+			quirk_broken_intx_masking);
+
+static u16 mellanox_broken_intx_devs[] = {
+	PCI_DEVICE_ID_MELLANOX_HERMON_SDR,
+	PCI_DEVICE_ID_MELLANOX_HERMON_DDR,
+	PCI_DEVICE_ID_MELLANOX_HERMON_QDR,
+	PCI_DEVICE_ID_MELLANOX_HERMON_DDR_GEN2,
+	PCI_DEVICE_ID_MELLANOX_HERMON_QDR_GEN2,
+	PCI_DEVICE_ID_MELLANOX_HERMON_EN,
+	PCI_DEVICE_ID_MELLANOX_HERMON_EN_GEN2,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX_EN,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_T_GEN2,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_GEN2,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_5_GEN2,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX2,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX3,
+	PCI_DEVICE_ID_MELLANOX_CONNECTX3_PRO,
+};
+
+#define CONNECTX_4_CURR_MAX_MINOR 99
+#define CONNECTX_4_INTX_SUPPORT_MINOR 14
+
+/*
+ * Check ConnectX-4/LX FW version to see if it supports legacy interrupts.
+ * If so, don't mark it as broken.
+ * FW minor > 99 means older FW version format and no INTx masking support.
+ * FW minor < 14 means new FW version format and no INTx masking support.
+ */
+static void mellanox_check_broken_intx_masking(struct pci_dev *pdev)
+{
+	__be32 __iomem *fw_ver;
+	u16 fw_major;
+	u16 fw_minor;
+	u16 fw_subminor;
+	u32 fw_maj_min;
+	u32 fw_sub_min;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(mellanox_broken_intx_devs); i++) {
+		if (pdev->device == mellanox_broken_intx_devs[i]) {
+			pdev->broken_intx_masking = 1;
+			return;
+		}
+	}
+
+	/* Getting here means Connect-IB cards and up. Connect-IB has no INTx
+	 * support so shouldn't be checked further
+	 */
+	if (pdev->device == PCI_DEVICE_ID_MELLANOX_CONNECTIB)
+		return;
+
+	if (pdev->device != PCI_DEVICE_ID_MELLANOX_CONNECTX4 &&
+	    pdev->device != PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX)
+		return;
+
+	/* For ConnectX-4 and ConnectX-4LX, need to check FW support */
+	if (pci_enable_device_mem(pdev)) {
+		dev_warn(&pdev->dev, "Can't enable device memory\n");
+		return;
+	}
+
+	fw_ver = ioremap(pci_resource_start(pdev, 0), 4);
+	if (!fw_ver) {
+		dev_warn(&pdev->dev, "Can't map ConnectX-4 initialization segment\n");
+		goto out;
+	}
+
+	/* Reading from resource space should be 32b aligned */
+	fw_maj_min = ioread32be(fw_ver);
+	fw_sub_min = ioread32be(fw_ver + 1);
+	fw_major = fw_maj_min & 0xffff;
+	fw_minor = fw_maj_min >> 16;
+	fw_subminor = fw_sub_min & 0xffff;
+	if (fw_minor > CONNECTX_4_CURR_MAX_MINOR ||
+	    fw_minor < CONNECTX_4_INTX_SUPPORT_MINOR) {
+		dev_warn(&pdev->dev, "ConnectX-4: FW %u.%u.%u doesn't support INTx masking, disabling. Please upgrade FW to %d.14.1100 and up for INTx support\n",
+			 fw_major, fw_minor, fw_subminor, pdev->device ==
+			 PCI_DEVICE_ID_MELLANOX_CONNECTX4 ? 12 : 14);
+		pdev->broken_intx_masking = 1;
+	}
+
+	iounmap(fw_ver);
+
+out:
+	pci_disable_device(pdev);
+}
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_MELLANOX, PCI_ANY_ID,
+			mellanox_check_broken_intx_masking);
 
 static void quirk_no_bus_reset(struct pci_dev *dev)
 {
@@ -3255,6 +3342,25 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PORT_RIDGE,
 			quirk_thunderbolt_hotplug_msi);
 
+static void quirk_chelsio_extend_vpd(struct pci_dev *dev)
+{
+	pci_set_vpd_size(dev, 8192);
+}
+
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x20, quirk_chelsio_extend_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x21, quirk_chelsio_extend_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x22, quirk_chelsio_extend_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x23, quirk_chelsio_extend_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x24, quirk_chelsio_extend_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x25, quirk_chelsio_extend_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x26, quirk_chelsio_extend_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x30, quirk_chelsio_extend_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x31, quirk_chelsio_extend_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x32, quirk_chelsio_extend_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x35, quirk_chelsio_extend_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x36, quirk_chelsio_extend_vpd);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_CHELSIO, 0x37, quirk_chelsio_extend_vpd);
+
 #ifdef CONFIG_ACPI
 /*
  * Apple: Shutdown Cactus Ridge Thunderbolt controller.
diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c
index f9357e09e9b3..73a03d382590 100644
--- a/drivers/pci/remove.c
+++ b/drivers/pci/remove.c
@@ -40,7 +40,7 @@ static void pci_destroy_dev(struct pci_dev *dev)
 	list_del(&dev->bus_list);
 	up_write(&pci_bus_sem);
 
-	pci_bridge_d3_device_removed(dev);
+	pci_bridge_d3_update(dev);
 	pci_free_resources(dev);
 	put_device(&dev->dev);
 }
diff --git a/drivers/pci/rom.c b/drivers/pci/rom.c
index 06663d391b39..b6edb187d160 100644
--- a/drivers/pci/rom.c
+++ b/drivers/pci/rom.c
@@ -35,6 +35,11 @@ int pci_enable_rom(struct pci_dev *pdev)
 	if (res->flags & IORESOURCE_ROM_SHADOW)
 		return 0;
 
+	/*
+	 * Ideally pci_update_resource() would update the ROM BAR address,
+	 * and we would only set the enable bit here.  But apparently some
+	 * devices have buggy ROM BARs that read as zero when disabled.
+	 */
 	pcibios_resource_to_bus(pdev->bus, &region, res);
 	pci_read_config_dword(pdev, pdev->rom_base_reg, &rom_addr);
 	rom_addr &= ~PCI_ROM_ADDRESS_MASK;
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c
index 9526e341988b..4bc589ee78d0 100644
--- a/drivers/pci/setup-res.c
+++ b/drivers/pci/setup-res.c
@@ -25,21 +25,18 @@
 #include <linux/slab.h>
 #include "pci.h"
 
-
-void pci_update_resource(struct pci_dev *dev, int resno)
+static void pci_std_update_resource(struct pci_dev *dev, int resno)
 {
 	struct pci_bus_region region;
 	bool disable;
 	u16 cmd;
 	u32 new, check, mask;
 	int reg;
-	enum pci_bar_type type;
 	struct resource *res = dev->resource + resno;
 
-	if (dev->is_virtfn) {
-		dev_warn(&dev->dev, "can't update VF BAR%d\n", resno);
+	/* Per SR-IOV spec 3.4.1.11, VF BARs are RO zero */
+	if (dev->is_virtfn)
 		return;
-	}
 
 	/*
 	 * Ignore resources for unimplemented BARs and unused resource slots
@@ -60,21 +57,34 @@ void pci_update_resource(struct pci_dev *dev, int resno)
 		return;
 
 	pcibios_resource_to_bus(dev->bus, &region, res);
+	new = region.start;
 
-	new = region.start | (res->flags & PCI_REGION_FLAG_MASK);
-	if (res->flags & IORESOURCE_IO)
+	if (res->flags & IORESOURCE_IO) {
 		mask = (u32)PCI_BASE_ADDRESS_IO_MASK;
-	else
+		new |= res->flags & ~PCI_BASE_ADDRESS_IO_MASK;
+	} else if (resno == PCI_ROM_RESOURCE) {
+		mask = (u32)PCI_ROM_ADDRESS_MASK;
+	} else {
 		mask = (u32)PCI_BASE_ADDRESS_MEM_MASK;
+		new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK;
+	}
 
-	reg = pci_resource_bar(dev, resno, &type);
-	if (!reg)
-		return;
-	if (type != pci_bar_unknown) {
+	if (resno < PCI_ROM_RESOURCE) {
+		reg = PCI_BASE_ADDRESS_0 + 4 * resno;
+	} else if (resno == PCI_ROM_RESOURCE) {
+
+		/*
+		 * Apparently some Matrox devices have ROM BARs that read
+		 * as zero when disabled, so don't update ROM BARs unless
+		 * they're enabled.  See https://lkml.org/lkml/2005/8/30/138.
+		 */
 		if (!(res->flags & IORESOURCE_ROM_ENABLE))
 			return;
+
+		reg = dev->rom_base_reg;
 		new |= PCI_ROM_ADDRESS_ENABLE;
-	}
+	} else
+		return;
 
 	/*
 	 * We can't update a 64-bit BAR atomically, so when possible,
@@ -110,6 +120,16 @@ void pci_update_resource(struct pci_dev *dev, int resno)
 		pci_write_config_word(dev, PCI_COMMAND, cmd);
 }
 
+void pci_update_resource(struct pci_dev *dev, int resno)
+{
+	if (resno <= PCI_ROM_RESOURCE)
+		pci_std_update_resource(dev, resno);
+#ifdef CONFIG_PCI_IOV
+	else if (resno >= PCI_IOV_RESOURCES && resno <= PCI_IOV_RESOURCE_END)
+		pci_iov_update_resource(dev, resno);
+#endif
+}
+
 int pci_claim_resource(struct pci_dev *dev, int resource)
 {
 	struct resource *res = &dev->resource[resource];
diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c
index d6ff5e82377d..8fc2e9532575 100644
--- a/drivers/pci/xen-pcifront.c
+++ b/drivers/pci/xen-pcifront.c
@@ -1038,10 +1038,8 @@ static int pcifront_detach_devices(struct pcifront_device *pdev)
 			err = -ENOMEM;
 			goto out;
 		}
-		err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, str, "%d",
-				   &state);
-		if (err != 1)
-			state = XenbusStateUnknown;
+		state = xenbus_read_unsigned(pdev->xdev->otherend, str,
+					     XenbusStateUnknown);
 
 		if (state != XenbusStateClosing)
 			continue;
diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig
index fe00f9134d51..e8eb7f225a88 100644
--- a/drivers/phy/Kconfig
+++ b/drivers/phy/Kconfig
@@ -129,16 +129,6 @@ config PHY_MIPHY28LP
 	  Enable this to support the miphy transceiver (for SATA/PCIE/USB3)
 	  that is part of STMicroelectronics STiH407 SoC.
 
-config PHY_MIPHY365X
-	tristate "STMicroelectronics MIPHY365X PHY driver for STiH41x series"
-	depends on ARCH_STI
-	depends on HAS_IOMEM
-	depends on OF
-	select GENERIC_PHY
-	help
-	  Enable this to support the miphy transceiver (for SATA/PCIE)
-	  that is part of STMicroelectronics STiH41x SoC series.
-
 config PHY_RCAR_GEN2
 	tristate "Renesas R-Car generation 2 USB PHY driver"
 	depends on ARCH_RENESAS
@@ -373,7 +363,9 @@ config PHY_ROCKCHIP_INNO_USB2
 	tristate "Rockchip INNO USB2PHY Driver"
 	depends on (ARCH_ROCKCHIP || COMPILE_TEST) && OF
 	depends on COMMON_CLK
+	depends on USB_SUPPORT
 	select GENERIC_PHY
+	select USB_COMMON
 	help
 	  Support for Rockchip USB2.0 PHY with Innosilicon IP block.
 
@@ -438,14 +430,6 @@ config PHY_STIH407_USB
 	  Enable this support to enable the picoPHY device used by USB2
 	  and USB3 controllers on STMicroelectronics STiH407 SoC families.
 
-config PHY_STIH41X_USB
-	tristate "STMicroelectronics USB2 PHY driver for STiH41x series"
-	depends on ARCH_STI
-	select GENERIC_PHY
-	help
-	  Enable this to support the USB transceiver that is part of
-	  STMicroelectronics STiH41x SoC series.
-
 config PHY_QCOM_UFS
 	tristate "Qualcomm UFS PHY driver"
 	depends on OF && ARCH_QCOM
@@ -489,4 +473,17 @@ config PHY_NS2_PCIE
 	help
 	  Enable this to support the Broadcom Northstar2 PCIe PHY.
 	  If unsure, say N.
+
+config PHY_MESON8B_USB2
+	tristate "Meson8b and GXBB USB2 PHY driver"
+	default ARCH_MESON
+	depends on OF && (ARCH_MESON || COMPILE_TEST)
+	depends on USB_SUPPORT
+	select USB_COMMON
+	select GENERIC_PHY
+	help
+	  Enable this to support the Meson USB2 PHYs found in Meson8b
+	  and GXBB SoCs.
+	  If unsure, say N.
+
 endmenu
diff --git a/drivers/phy/Makefile b/drivers/phy/Makefile
index a534cf5be07d..65eb2f436a41 100644
--- a/drivers/phy/Makefile
+++ b/drivers/phy/Makefile
@@ -18,7 +18,6 @@ obj-$(CONFIG_PHY_PXA_28NM_USB2)		+= phy-pxa-28nm-usb2.o
 obj-$(CONFIG_PHY_PXA_28NM_HSIC)		+= phy-pxa-28nm-hsic.o
 obj-$(CONFIG_PHY_MVEBU_SATA)		+= phy-mvebu-sata.o
 obj-$(CONFIG_PHY_MIPHY28LP) 		+= phy-miphy28lp.o
-obj-$(CONFIG_PHY_MIPHY365X)		+= phy-miphy365x.o
 obj-$(CONFIG_PHY_RCAR_GEN2)		+= phy-rcar-gen2.o
 obj-$(CONFIG_PHY_RCAR_GEN3_USB2)	+= phy-rcar-gen3-usb2.o
 obj-$(CONFIG_OMAP_CONTROL_PHY)		+= phy-omap-control.o
@@ -50,7 +49,6 @@ obj-$(CONFIG_PHY_ST_SPEAR1310_MIPHY)	+= phy-spear1310-miphy.o
 obj-$(CONFIG_PHY_ST_SPEAR1340_MIPHY)	+= phy-spear1340-miphy.o
 obj-$(CONFIG_PHY_XGENE)			+= phy-xgene.o
 obj-$(CONFIG_PHY_STIH407_USB)		+= phy-stih407-usb.o
-obj-$(CONFIG_PHY_STIH41X_USB)		+= phy-stih41x-usb.o
 obj-$(CONFIG_PHY_QCOM_UFS) 	+= phy-qcom-ufs.o
 obj-$(CONFIG_PHY_QCOM_UFS) 	+= phy-qcom-ufs-qmp-20nm.o
 obj-$(CONFIG_PHY_QCOM_UFS) 	+= phy-qcom-ufs-qmp-14nm.o
@@ -60,3 +58,4 @@ obj-$(CONFIG_PHY_PISTACHIO_USB)		+= phy-pistachio-usb.o
 obj-$(CONFIG_PHY_CYGNUS_PCIE)		+= phy-bcm-cygnus-pcie.o
 obj-$(CONFIG_ARCH_TEGRA) += tegra/
 obj-$(CONFIG_PHY_NS2_PCIE)		+= phy-bcm-ns2-pcie.o
+obj-$(CONFIG_PHY_MESON8B_USB2)		+= phy-meson8b-usb2.o
diff --git a/drivers/phy/phy-berlin-sata.c b/drivers/phy/phy-berlin-sata.c
index f84a33a1bdd9..2c7a57f2d595 100644
--- a/drivers/phy/phy-berlin-sata.c
+++ b/drivers/phy/phy-berlin-sata.c
@@ -85,7 +85,6 @@ static int phy_berlin_sata_power_on(struct phy *phy)
 	struct phy_berlin_desc *desc = phy_get_drvdata(phy);
 	struct phy_berlin_priv *priv = dev_get_drvdata(phy->dev.parent);
 	void __iomem *ctrl_reg = priv->base + 0x60 + (desc->index * 0x80);
-	int ret = 0;
 	u32 regval;
 
 	clk_prepare_enable(priv->clk);
@@ -130,7 +129,7 @@ static int phy_berlin_sata_power_on(struct phy *phy)
 
 	clk_disable_unprepare(priv->clk);
 
-	return ret;
+	return 0;
 }
 
 static int phy_berlin_sata_power_off(struct phy *phy)
diff --git a/drivers/phy/phy-brcm-sata.c b/drivers/phy/phy-brcm-sata.c
index 8ffc44afdb75..ccbc3d994998 100644
--- a/drivers/phy/phy-brcm-sata.c
+++ b/drivers/phy/phy-brcm-sata.c
@@ -140,7 +140,7 @@ static inline void __iomem *brcm_sata_pcb_base(struct brcm_sata_port *port)
 	default:
 		dev_err(priv->dev, "invalid phy version\n");
 		break;
-	};
+	}
 
 	return priv->phy_base + (port->portnum * size);
 }
@@ -157,7 +157,7 @@ static inline void __iomem *brcm_sata_ctrl_base(struct brcm_sata_port *port)
 	default:
 		dev_err(priv->dev, "invalid phy version\n");
 		break;
-	};
+	}
 
 	return priv->ctrl_base + (port->portnum * size);
 }
@@ -365,7 +365,7 @@ static int brcm_sata_phy_init(struct phy *phy)
 		break;
 	default:
 		rc = -ENODEV;
-	};
+	}
 
 	return rc;
 }
diff --git a/drivers/phy/phy-da8xx-usb.c b/drivers/phy/phy-da8xx-usb.c
index c85fb0b59729..1b82bff6330f 100644
--- a/drivers/phy/phy-da8xx-usb.c
+++ b/drivers/phy/phy-da8xx-usb.c
@@ -23,6 +23,8 @@
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
+#define PHY_INIT_BITS	(CFGCHIP2_SESENDEN | CFGCHIP2_VBDTCTEN)
+
 struct da8xx_usb_phy {
 	struct phy_provider	*phy_provider;
 	struct phy		*usb11_phy;
@@ -208,6 +210,9 @@ static int da8xx_usb_phy_probe(struct platform_device *pdev)
 			dev_warn(dev, "Failed to create usb20 phy lookup\n");
 	}
 
+	regmap_write_bits(d_phy->regmap, CFGCHIP(2),
+			  PHY_INIT_BITS, PHY_INIT_BITS);
+
 	return 0;
 }
 
diff --git a/drivers/phy/phy-exynos-mipi-video.c b/drivers/phy/phy-exynos-mipi-video.c
index 8b851f718123..6bee04cc4d53 100644
--- a/drivers/phy/phy-exynos-mipi-video.c
+++ b/drivers/phy/phy-exynos-mipi-video.c
@@ -229,19 +229,6 @@ struct exynos_mipi_video_phy {
 	spinlock_t slock;
 };
 
-static inline int __is_running(const struct exynos_mipi_phy_desc *data,
-			struct exynos_mipi_video_phy *state)
-{
-	u32 val;
-	int ret;
-
-	ret = regmap_read(state->regmaps[data->resetn_map], data->resetn_reg, &val);
-	if (ret)
-		return 0;
-
-	return val & data->resetn_val;
-}
-
 static int __set_phy_state(const struct exynos_mipi_phy_desc *data,
 			   struct exynos_mipi_video_phy *state, unsigned int on)
 {
@@ -251,7 +238,7 @@ static int __set_phy_state(const struct exynos_mipi_phy_desc *data,
 
 	/* disable in PMU sysreg */
 	if (!on && data->coupled_phy_id >= 0 &&
-	    !__is_running(state->phys[data->coupled_phy_id].data, state)) {
+	    state->phys[data->coupled_phy_id].phy->power_count == 0) {
 		regmap_read(state->regmaps[data->enable_map], data->enable_reg,
 			    &val);
 		val &= ~data->enable_val;
diff --git a/drivers/phy/phy-exynos4210-usb2.c b/drivers/phy/phy-exynos4210-usb2.c
index f30bbb0fb3b2..1f50e1004828 100644
--- a/drivers/phy/phy-exynos4210-usb2.c
+++ b/drivers/phy/phy-exynos4210-usb2.c
@@ -141,7 +141,7 @@ static void exynos4210_isol(struct samsung_usb2_phy_instance *inst, bool on)
 		break;
 	default:
 		return;
-	};
+	}
 
 	regmap_update_bits(drv->reg_pmu, offset, mask, on ? 0 : mask);
 }
@@ -179,7 +179,7 @@ static void exynos4210_phy_pwr(struct samsung_usb2_phy_instance *inst, bool on)
 		rstbits =	EXYNOS_4210_URSTCON_PHY1_P1P2 |
 				EXYNOS_4210_URSTCON_HOST_LINK_P2;
 		break;
-	};
+	}
 
 	if (on) {
 		clk = readl(drv->reg_phy + EXYNOS_4210_UPHYCLK);
diff --git a/drivers/phy/phy-exynos4x12-usb2.c b/drivers/phy/phy-exynos4x12-usb2.c
index 765da90a536f..7f27a91acf87 100644
--- a/drivers/phy/phy-exynos4x12-usb2.c
+++ b/drivers/phy/phy-exynos4x12-usb2.c
@@ -187,7 +187,7 @@ static void exynos4x12_isol(struct samsung_usb2_phy_instance *inst, bool on)
 		break;
 	default:
 		return;
-	};
+	}
 
 	regmap_update_bits(drv->reg_pmu, offset, mask, on ? 0 : mask);
 }
@@ -237,7 +237,7 @@ static void exynos4x12_phy_pwr(struct samsung_usb2_phy_instance *inst, bool on)
 		rstbits =	EXYNOS_4x12_URSTCON_HSIC1 |
 				EXYNOS_4x12_URSTCON_HOST_LINK_P1;
 		break;
-	};
+	}
 
 	if (on) {
 		pwr = readl(drv->reg_phy + EXYNOS_4x12_UPHYPWR);
diff --git a/drivers/phy/phy-exynos5250-usb2.c b/drivers/phy/phy-exynos5250-usb2.c
index 2ed1735a076a..aad806272305 100644
--- a/drivers/phy/phy-exynos5250-usb2.c
+++ b/drivers/phy/phy-exynos5250-usb2.c
@@ -192,7 +192,7 @@ static void exynos5250_isol(struct samsung_usb2_phy_instance *inst, bool on)
 		break;
 	default:
 		return;
-	};
+	}
 
 	regmap_update_bits(drv->reg_pmu, offset, mask, on ? 0 : mask);
 }
diff --git a/drivers/phy/phy-meson8b-usb2.c b/drivers/phy/phy-meson8b-usb2.c
new file mode 100644
index 000000000000..33c9f4ba157d
--- /dev/null
+++ b/drivers/phy/phy-meson8b-usb2.c
@@ -0,0 +1,286 @@
+/*
+ * Meson8b and GXBB USB2 PHY driver
+ *
+ * Copyright (C) 2016 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/reset.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/usb/of.h>
+
+#define REG_CONFIG					0x00
+	#define REG_CONFIG_CLK_EN			BIT(0)
+	#define REG_CONFIG_CLK_SEL_MASK			GENMASK(3, 1)
+	#define REG_CONFIG_CLK_DIV_MASK			GENMASK(10, 4)
+	#define REG_CONFIG_CLK_32k_ALTSEL		BIT(15)
+	#define REG_CONFIG_TEST_TRIG			BIT(31)
+
+#define REG_CTRL					0x04
+	#define REG_CTRL_SOFT_PRST			BIT(0)
+	#define REG_CTRL_SOFT_HRESET			BIT(1)
+	#define REG_CTRL_SS_SCALEDOWN_MODE_MASK		GENMASK(3, 2)
+	#define REG_CTRL_CLK_DET_RST			BIT(4)
+	#define REG_CTRL_INTR_SEL			BIT(5)
+	#define REG_CTRL_CLK_DETECTED			BIT(8)
+	#define REG_CTRL_SOF_SENT_RCVD_TGL		BIT(9)
+	#define REG_CTRL_SOF_TOGGLE_OUT			BIT(10)
+	#define REG_CTRL_POWER_ON_RESET			BIT(15)
+	#define REG_CTRL_SLEEPM				BIT(16)
+	#define REG_CTRL_TX_BITSTUFF_ENN_H		BIT(17)
+	#define REG_CTRL_TX_BITSTUFF_ENN		BIT(18)
+	#define REG_CTRL_COMMON_ON			BIT(19)
+	#define REG_CTRL_REF_CLK_SEL_MASK		GENMASK(21, 20)
+	#define REG_CTRL_REF_CLK_SEL_SHIFT		20
+	#define REG_CTRL_FSEL_MASK			GENMASK(24, 22)
+	#define REG_CTRL_FSEL_SHIFT			22
+	#define REG_CTRL_PORT_RESET			BIT(25)
+	#define REG_CTRL_THREAD_ID_MASK			GENMASK(31, 26)
+
+#define REG_ENDP_INTR					0x08
+
+/* bits [31:26], [24:21] and [15:3] seem to be read-only */
+#define REG_ADP_BC					0x0c
+	#define REG_ADP_BC_VBUS_VLD_EXT_SEL		BIT(0)
+	#define REG_ADP_BC_VBUS_VLD_EXT			BIT(1)
+	#define REG_ADP_BC_OTG_DISABLE			BIT(2)
+	#define REG_ADP_BC_ID_PULLUP			BIT(3)
+	#define REG_ADP_BC_DRV_VBUS			BIT(4)
+	#define REG_ADP_BC_ADP_PRB_EN			BIT(5)
+	#define REG_ADP_BC_ADP_DISCHARGE		BIT(6)
+	#define REG_ADP_BC_ADP_CHARGE			BIT(7)
+	#define REG_ADP_BC_SESS_END			BIT(8)
+	#define REG_ADP_BC_DEVICE_SESS_VLD		BIT(9)
+	#define REG_ADP_BC_B_VALID			BIT(10)
+	#define REG_ADP_BC_A_VALID			BIT(11)
+	#define REG_ADP_BC_ID_DIG			BIT(12)
+	#define REG_ADP_BC_VBUS_VALID			BIT(13)
+	#define REG_ADP_BC_ADP_PROBE			BIT(14)
+	#define REG_ADP_BC_ADP_SENSE			BIT(15)
+	#define REG_ADP_BC_ACA_ENABLE			BIT(16)
+	#define REG_ADP_BC_DCD_ENABLE			BIT(17)
+	#define REG_ADP_BC_VDAT_DET_EN_B		BIT(18)
+	#define REG_ADP_BC_VDAT_SRC_EN_B		BIT(19)
+	#define REG_ADP_BC_CHARGE_SEL			BIT(20)
+	#define REG_ADP_BC_CHARGE_DETECT		BIT(21)
+	#define REG_ADP_BC_ACA_PIN_RANGE_C		BIT(22)
+	#define REG_ADP_BC_ACA_PIN_RANGE_B		BIT(23)
+	#define REG_ADP_BC_ACA_PIN_RANGE_A		BIT(24)
+	#define REG_ADP_BC_ACA_PIN_GND			BIT(25)
+	#define REG_ADP_BC_ACA_PIN_FLOAT		BIT(26)
+
+#define REG_DBG_UART					0x14
+
+#define REG_TEST					0x18
+	#define REG_TEST_DATA_IN_MASK			GENMASK(3, 0)
+	#define REG_TEST_EN_MASK			GENMASK(7, 4)
+	#define REG_TEST_ADDR_MASK			GENMASK(11, 8)
+	#define REG_TEST_DATA_OUT_SEL			BIT(12)
+	#define REG_TEST_CLK				BIT(13)
+	#define REG_TEST_VA_TEST_EN_B_MASK		GENMASK(15, 14)
+	#define REG_TEST_DATA_OUT_MASK			GENMASK(19, 16)
+	#define REG_TEST_DISABLE_ID_PULLUP		BIT(20)
+
+#define REG_TUNE					0x1c
+	#define REG_TUNE_TX_RES_TUNE_MASK		GENMASK(1, 0)
+	#define REG_TUNE_TX_HSXV_TUNE_MASK		GENMASK(3, 2)
+	#define REG_TUNE_TX_VREF_TUNE_MASK		GENMASK(7, 4)
+	#define REG_TUNE_TX_RISE_TUNE_MASK		GENMASK(9, 8)
+	#define REG_TUNE_TX_PREEMP_PULSE_TUNE		BIT(10)
+	#define REG_TUNE_TX_PREEMP_AMP_TUNE_MASK	GENMASK(12, 11)
+	#define REG_TUNE_TX_FSLS_TUNE_MASK		GENMASK(16, 13)
+	#define REG_TUNE_SQRX_TUNE_MASK			GENMASK(19, 17)
+	#define REG_TUNE_OTG_TUNE			GENMASK(22, 20)
+	#define REG_TUNE_COMP_DIS_TUNE			GENMASK(25, 23)
+	#define REG_TUNE_HOST_DM_PULLDOWN		BIT(26)
+	#define REG_TUNE_HOST_DP_PULLDOWN		BIT(27)
+
+#define RESET_COMPLETE_TIME				500
+#define ACA_ENABLE_COMPLETE_TIME			50
+
+struct phy_meson8b_usb2_priv {
+	void __iomem		*regs;
+	enum usb_dr_mode	dr_mode;
+	struct clk		*clk_usb_general;
+	struct clk		*clk_usb;
+	struct reset_control	*reset;
+};
+
+static u32 phy_meson8b_usb2_read(struct phy_meson8b_usb2_priv *phy_priv,
+				 u32 reg)
+{
+	return readl(phy_priv->regs + reg);
+}
+
+static void phy_meson8b_usb2_mask_bits(struct phy_meson8b_usb2_priv *phy_priv,
+				       u32 reg, u32 mask, u32 value)
+{
+	u32 data;
+
+	data = phy_meson8b_usb2_read(phy_priv, reg);
+	data &= ~mask;
+	data |= (value & mask);
+
+	writel(data, phy_priv->regs + reg);
+}
+
+static int phy_meson8b_usb2_power_on(struct phy *phy)
+{
+	struct phy_meson8b_usb2_priv *priv = phy_get_drvdata(phy);
+	int ret;
+
+	if (!IS_ERR_OR_NULL(priv->reset)) {
+		ret = reset_control_reset(priv->reset);
+		if (ret) {
+			dev_err(&phy->dev, "Failed to trigger USB reset\n");
+			return ret;
+		}
+	}
+
+	ret = clk_prepare_enable(priv->clk_usb_general);
+	if (ret) {
+		dev_err(&phy->dev, "Failed to enable USB general clock\n");
+		return ret;
+	}
+
+	ret = clk_prepare_enable(priv->clk_usb);
+	if (ret) {
+		dev_err(&phy->dev, "Failed to enable USB DDR clock\n");
+		clk_disable_unprepare(priv->clk_usb_general);
+		return ret;
+	}
+
+	phy_meson8b_usb2_mask_bits(priv, REG_CONFIG, REG_CONFIG_CLK_32k_ALTSEL,
+				   REG_CONFIG_CLK_32k_ALTSEL);
+
+	phy_meson8b_usb2_mask_bits(priv, REG_CTRL, REG_CTRL_REF_CLK_SEL_MASK,
+				   0x2 << REG_CTRL_REF_CLK_SEL_SHIFT);
+
+	phy_meson8b_usb2_mask_bits(priv, REG_CTRL, REG_CTRL_FSEL_MASK,
+				   0x5 << REG_CTRL_FSEL_SHIFT);
+
+	/* reset the PHY */
+	phy_meson8b_usb2_mask_bits(priv, REG_CTRL, REG_CTRL_POWER_ON_RESET,
+				   REG_CTRL_POWER_ON_RESET);
+	udelay(RESET_COMPLETE_TIME);
+	phy_meson8b_usb2_mask_bits(priv, REG_CTRL, REG_CTRL_POWER_ON_RESET, 0);
+	udelay(RESET_COMPLETE_TIME);
+
+	phy_meson8b_usb2_mask_bits(priv, REG_CTRL, REG_CTRL_SOF_TOGGLE_OUT,
+				   REG_CTRL_SOF_TOGGLE_OUT);
+
+	if (priv->dr_mode == USB_DR_MODE_HOST) {
+		phy_meson8b_usb2_mask_bits(priv, REG_ADP_BC,
+					   REG_ADP_BC_ACA_ENABLE,
+					   REG_ADP_BC_ACA_ENABLE);
+
+		udelay(ACA_ENABLE_COMPLETE_TIME);
+
+		if (phy_meson8b_usb2_read(priv, REG_ADP_BC) &
+			REG_ADP_BC_ACA_PIN_FLOAT) {
+			dev_warn(&phy->dev, "USB ID detect failed!\n");
+			clk_disable_unprepare(priv->clk_usb);
+			clk_disable_unprepare(priv->clk_usb_general);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int phy_meson8b_usb2_power_off(struct phy *phy)
+{
+	struct phy_meson8b_usb2_priv *priv = phy_get_drvdata(phy);
+
+	clk_disable_unprepare(priv->clk_usb);
+	clk_disable_unprepare(priv->clk_usb_general);
+
+	return 0;
+}
+
+static const struct phy_ops phy_meson8b_usb2_ops = {
+	.power_on	= phy_meson8b_usb2_power_on,
+	.power_off	= phy_meson8b_usb2_power_off,
+	.owner		= THIS_MODULE,
+};
+
+static int phy_meson8b_usb2_probe(struct platform_device *pdev)
+{
+	struct phy_meson8b_usb2_priv *priv;
+	struct resource *res;
+	struct phy *phy;
+	struct phy_provider *phy_provider;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->regs = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(priv->regs))
+		return PTR_ERR(priv->regs);
+
+	priv->clk_usb_general = devm_clk_get(&pdev->dev, "usb_general");
+	if (IS_ERR(priv->clk_usb_general))
+		return PTR_ERR(priv->clk_usb_general);
+
+	priv->clk_usb = devm_clk_get(&pdev->dev, "usb");
+	if (IS_ERR(priv->clk_usb))
+		return PTR_ERR(priv->clk_usb);
+
+	priv->reset = devm_reset_control_get_optional_shared(&pdev->dev, NULL);
+	if (PTR_ERR(priv->reset) == -EPROBE_DEFER)
+		return PTR_ERR(priv->reset);
+
+	priv->dr_mode = of_usb_get_dr_mode_by_phy(pdev->dev.of_node, -1);
+	if (priv->dr_mode == USB_DR_MODE_UNKNOWN) {
+		dev_err(&pdev->dev,
+			"missing dual role configuration of the controller\n");
+		return -EINVAL;
+	}
+
+	phy = devm_phy_create(&pdev->dev, NULL, &phy_meson8b_usb2_ops);
+	if (IS_ERR(phy)) {
+		dev_err(&pdev->dev, "failed to create PHY\n");
+		return PTR_ERR(phy);
+	}
+
+	phy_set_drvdata(phy, priv);
+
+	phy_provider =
+		devm_of_phy_provider_register(&pdev->dev, of_phy_simple_xlate);
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static const struct of_device_id phy_meson8b_usb2_of_match[] = {
+	{ .compatible = "amlogic,meson8b-usb2-phy", },
+	{ .compatible = "amlogic,meson-gxbb-usb2-phy", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, phy_meson8b_usb2_of_match);
+
+static struct platform_driver phy_meson8b_usb2_driver = {
+	.probe	= phy_meson8b_usb2_probe,
+	.driver	= {
+		.name		= "phy-meson-usb2",
+		.of_match_table	= phy_meson8b_usb2_of_match,
+	},
+};
+module_platform_driver(phy_meson8b_usb2_driver);
+
+MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
+MODULE_DESCRIPTION("Meson8b and GXBB USB2 PHY driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/phy/phy-miphy365x.c b/drivers/phy/phy-miphy365x.c
deleted file mode 100644
index e661f3b36eaa..000000000000
--- a/drivers/phy/phy-miphy365x.c
+++ /dev/null
@@ -1,625 +0,0 @@
-/*
- * Copyright (C) 2014 STMicroelectronics – All Rights Reserved
- *
- * STMicroelectronics PHY driver MiPHY365 (for SoC STiH416).
- *
- * Authors: Alexandre Torgue <alexandre.torgue@st.com>
- *          Lee Jones <lee.jones@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2, as
- * published by the Free Software Foundation.
- *
- */
-
-#include <linux/platform_device.h>
-#include <linux/io.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_platform.h>
-#include <linux/of_address.h>
-#include <linux/clk.h>
-#include <linux/phy/phy.h>
-#include <linux/delay.h>
-#include <linux/mfd/syscon.h>
-#include <linux/regmap.h>
-
-#include <dt-bindings/phy/phy.h>
-
-#define HFC_TIMEOUT		100
-
-#define SYSCFG_SELECT_SATA_MASK	BIT(1)
-#define SYSCFG_SELECT_SATA_POS	1
-
-/* MiPHY365x register definitions */
-#define RESET_REG		0x00
-#define RST_PLL			BIT(1)
-#define RST_PLL_CAL		BIT(2)
-#define RST_RX			BIT(4)
-#define RST_MACRO		BIT(7)
-
-#define STATUS_REG		0x01
-#define IDLL_RDY		BIT(0)
-#define PLL_RDY			BIT(1)
-#define DES_BIT_LOCK		BIT(2)
-#define DES_SYMBOL_LOCK		BIT(3)
-
-#define CTRL_REG		0x02
-#define TERM_EN			BIT(0)
-#define PCI_EN			BIT(2)
-#define DES_BIT_LOCK_EN		BIT(3)
-#define TX_POL			BIT(5)
-
-#define INT_CTRL_REG		0x03
-
-#define BOUNDARY1_REG		0x10
-#define SPDSEL_SEL		BIT(0)
-
-#define BOUNDARY3_REG		0x12
-#define TX_SPDSEL_GEN1_VAL	0
-#define TX_SPDSEL_GEN2_VAL	0x01
-#define TX_SPDSEL_GEN3_VAL	0x02
-#define RX_SPDSEL_GEN1_VAL	0
-#define RX_SPDSEL_GEN2_VAL	(0x01 << 3)
-#define RX_SPDSEL_GEN3_VAL	(0x02 << 3)
-
-#define PCIE_REG		0x16
-
-#define BUF_SEL_REG		0x20
-#define CONF_GEN_SEL_GEN3	0x02
-#define CONF_GEN_SEL_GEN2	0x01
-#define PD_VDDTFILTER		BIT(4)
-
-#define TXBUF1_REG		0x21
-#define SWING_VAL		0x04
-#define SWING_VAL_GEN1		0x03
-#define PREEMPH_VAL		(0x3 << 5)
-
-#define TXBUF2_REG		0x22
-#define TXSLEW_VAL		0x2
-#define TXSLEW_VAL_GEN1		0x4
-
-#define RXBUF_OFFSET_CTRL_REG	0x23
-
-#define RXBUF_REG		0x25
-#define SDTHRES_VAL		0x01
-#define EQ_ON3			(0x03 << 4)
-#define EQ_ON1			(0x01 << 4)
-
-#define COMP_CTRL1_REG		0x40
-#define START_COMSR		BIT(0)
-#define START_COMZC		BIT(1)
-#define COMSR_DONE		BIT(2)
-#define COMZC_DONE		BIT(3)
-#define COMP_AUTO_LOAD		BIT(4)
-
-#define COMP_CTRL2_REG		0x41
-#define COMP_2MHZ_RAT_GEN1	0x1e
-#define COMP_2MHZ_RAT		0xf
-
-#define COMP_CTRL3_REG		0x42
-#define COMSR_COMP_REF		0x33
-
-#define COMP_IDLL_REG		0x47
-#define COMZC_IDLL		0x2a
-
-#define PLL_CTRL1_REG		0x50
-#define PLL_START_CAL		BIT(0)
-#define BUF_EN			BIT(2)
-#define SYNCHRO_TX		BIT(3)
-#define SSC_EN			BIT(6)
-#define CONFIG_PLL		BIT(7)
-
-#define PLL_CTRL2_REG		0x51
-#define BYPASS_PLL_CAL		BIT(1)
-
-#define PLL_RAT_REG		0x52
-
-#define PLL_SSC_STEP_MSB_REG	0x56
-#define PLL_SSC_STEP_MSB_VAL	0x03
-
-#define PLL_SSC_STEP_LSB_REG	0x57
-#define PLL_SSC_STEP_LSB_VAL	0x63
-
-#define PLL_SSC_PER_MSB_REG	0x58
-#define PLL_SSC_PER_MSB_VAL	0
-
-#define PLL_SSC_PER_LSB_REG	0x59
-#define PLL_SSC_PER_LSB_VAL	0xf1
-
-#define IDLL_TEST_REG		0x72
-#define START_CLK_HF		BIT(6)
-
-#define DES_BITLOCK_REG		0x86
-#define BIT_LOCK_LEVEL		0x01
-#define BIT_LOCK_CNT_512	(0x03 << 5)
-
-struct miphy365x_phy {
-	struct phy *phy;
-	void __iomem *base;
-	bool pcie_tx_pol_inv;
-	bool sata_tx_pol_inv;
-	u32 sata_gen;
-	u32 ctrlreg;
-	u8 type;
-};
-
-struct miphy365x_dev {
-	struct device *dev;
-	struct regmap *regmap;
-	struct mutex miphy_mutex;
-	struct miphy365x_phy **phys;
-	int nphys;
-};
-
-/*
- * These values are represented in Device tree. They are considered to be ABI
- * and although they can be extended any existing values must not change.
- */
-enum miphy_sata_gen {
-	SATA_GEN1 = 1,
-	SATA_GEN2,
-	SATA_GEN3
-};
-
-static u8 rx_tx_spd[] = {
-	0, /* GEN0 doesn't exist. */
-	TX_SPDSEL_GEN1_VAL | RX_SPDSEL_GEN1_VAL,
-	TX_SPDSEL_GEN2_VAL | RX_SPDSEL_GEN2_VAL,
-	TX_SPDSEL_GEN3_VAL | RX_SPDSEL_GEN3_VAL
-};
-
-/*
- * This function selects the system configuration,
- * either two SATA, one SATA and one PCIe, or two PCIe lanes.
- */
-static int miphy365x_set_path(struct miphy365x_phy *miphy_phy,
-			      struct miphy365x_dev *miphy_dev)
-{
-	bool sata = (miphy_phy->type == PHY_TYPE_SATA);
-
-	return regmap_update_bits(miphy_dev->regmap,
-				  miphy_phy->ctrlreg,
-				  SYSCFG_SELECT_SATA_MASK,
-				  sata << SYSCFG_SELECT_SATA_POS);
-}
-
-static int miphy365x_init_pcie_port(struct miphy365x_phy *miphy_phy,
-				    struct miphy365x_dev *miphy_dev)
-{
-	u8 val;
-
-	if (miphy_phy->pcie_tx_pol_inv) {
-		/* Invert Tx polarity and clear pci_txdetect_pol bit */
-		val = TERM_EN | PCI_EN | DES_BIT_LOCK_EN | TX_POL;
-		writeb_relaxed(val, miphy_phy->base + CTRL_REG);
-		writeb_relaxed(0x00, miphy_phy->base + PCIE_REG);
-	}
-
-	return 0;
-}
-
-static inline int miphy365x_hfc_not_rdy(struct miphy365x_phy *miphy_phy,
-					struct miphy365x_dev *miphy_dev)
-{
-	unsigned long timeout = jiffies + msecs_to_jiffies(HFC_TIMEOUT);
-	u8 mask = IDLL_RDY | PLL_RDY;
-	u8 regval;
-
-	do {
-		regval = readb_relaxed(miphy_phy->base + STATUS_REG);
-		if (!(regval & mask))
-			return 0;
-
-		usleep_range(2000, 2500);
-	} while (time_before(jiffies, timeout));
-
-	dev_err(miphy_dev->dev, "HFC ready timeout!\n");
-	return -EBUSY;
-}
-
-static inline int miphy365x_rdy(struct miphy365x_phy *miphy_phy,
-				struct miphy365x_dev *miphy_dev)
-{
-	unsigned long timeout = jiffies + msecs_to_jiffies(HFC_TIMEOUT);
-	u8 mask = IDLL_RDY | PLL_RDY;
-	u8 regval;
-
-	do {
-		regval = readb_relaxed(miphy_phy->base + STATUS_REG);
-		if ((regval & mask) == mask)
-			return 0;
-
-		usleep_range(2000, 2500);
-	} while (time_before(jiffies, timeout));
-
-	dev_err(miphy_dev->dev, "PHY not ready timeout!\n");
-	return -EBUSY;
-}
-
-static inline void miphy365x_set_comp(struct miphy365x_phy *miphy_phy,
-				      struct miphy365x_dev *miphy_dev)
-{
-	u8 val, mask;
-
-	if (miphy_phy->sata_gen == SATA_GEN1)
-		writeb_relaxed(COMP_2MHZ_RAT_GEN1,
-			       miphy_phy->base + COMP_CTRL2_REG);
-	else
-		writeb_relaxed(COMP_2MHZ_RAT,
-			       miphy_phy->base + COMP_CTRL2_REG);
-
-	if (miphy_phy->sata_gen != SATA_GEN3) {
-		writeb_relaxed(COMSR_COMP_REF,
-			       miphy_phy->base + COMP_CTRL3_REG);
-		/*
-		 * Force VCO current to value defined by address 0x5A
-		 * and disable PCIe100Mref bit
-		 * Enable auto load compensation for pll_i_bias
-		 */
-		writeb_relaxed(BYPASS_PLL_CAL, miphy_phy->base + PLL_CTRL2_REG);
-		writeb_relaxed(COMZC_IDLL, miphy_phy->base + COMP_IDLL_REG);
-	}
-
-	/*
-	 * Force restart compensation and enable auto load
-	 * for Comzc_Tx, Comzc_Rx and Comsr on macro
-	 */
-	val = START_COMSR | START_COMZC | COMP_AUTO_LOAD;
-	writeb_relaxed(val, miphy_phy->base + COMP_CTRL1_REG);
-
-	mask = COMSR_DONE | COMZC_DONE;
-	while ((readb_relaxed(miphy_phy->base + COMP_CTRL1_REG) & mask)	!= mask)
-		cpu_relax();
-}
-
-static inline void miphy365x_set_ssc(struct miphy365x_phy *miphy_phy,
-				     struct miphy365x_dev *miphy_dev)
-{
-	u8 val;
-
-	/*
-	 * SSC Settings. SSC will be enabled through Link
-	 * SSC Ampl. = 0.4%
-	 * SSC Freq = 31KHz
-	 */
-	writeb_relaxed(PLL_SSC_STEP_MSB_VAL,
-		       miphy_phy->base + PLL_SSC_STEP_MSB_REG);
-	writeb_relaxed(PLL_SSC_STEP_LSB_VAL,
-		       miphy_phy->base + PLL_SSC_STEP_LSB_REG);
-	writeb_relaxed(PLL_SSC_PER_MSB_VAL,
-		       miphy_phy->base + PLL_SSC_PER_MSB_REG);
-	writeb_relaxed(PLL_SSC_PER_LSB_VAL,
-		       miphy_phy->base + PLL_SSC_PER_LSB_REG);
-
-	/* SSC Settings complete */
-	if (miphy_phy->sata_gen == SATA_GEN1) {
-		val = PLL_START_CAL | BUF_EN | SYNCHRO_TX | CONFIG_PLL;
-		writeb_relaxed(val, miphy_phy->base + PLL_CTRL1_REG);
-	} else {
-		val = SSC_EN | PLL_START_CAL | BUF_EN | SYNCHRO_TX | CONFIG_PLL;
-		writeb_relaxed(val, miphy_phy->base + PLL_CTRL1_REG);
-	}
-}
-
-static int miphy365x_init_sata_port(struct miphy365x_phy *miphy_phy,
-				    struct miphy365x_dev *miphy_dev)
-{
-	int ret;
-	u8 val;
-
-	/*
-	 * Force PHY macro reset, PLL calibration reset, PLL reset
-	 * and assert Deserializer Reset
-	 */
-	val = RST_PLL | RST_PLL_CAL | RST_RX | RST_MACRO;
-	writeb_relaxed(val, miphy_phy->base + RESET_REG);
-
-	if (miphy_phy->sata_tx_pol_inv)
-		writeb_relaxed(TX_POL, miphy_phy->base + CTRL_REG);
-
-	/*
-	 * Force macro1 to use rx_lspd, tx_lspd
-	 * Force Rx_Clock on first I-DLL phase
-	 * Force Des in HP mode on macro, rx_lspd, tx_lspd for Gen2/3
-	 */
-	writeb_relaxed(SPDSEL_SEL, miphy_phy->base + BOUNDARY1_REG);
-	writeb_relaxed(START_CLK_HF, miphy_phy->base + IDLL_TEST_REG);
-	val = rx_tx_spd[miphy_phy->sata_gen];
-	writeb_relaxed(val, miphy_phy->base + BOUNDARY3_REG);
-
-	/* Wait for HFC_READY = 0 */
-	ret = miphy365x_hfc_not_rdy(miphy_phy, miphy_dev);
-	if (ret)
-		return ret;
-
-	/* Compensation Recalibration */
-	miphy365x_set_comp(miphy_phy, miphy_dev);
-
-	switch (miphy_phy->sata_gen) {
-	case SATA_GEN3:
-		/*
-		 * TX Swing target 550-600mv peak to peak diff
-		 * Tx Slew target 90-110ps rising/falling time
-		 * Rx Eq ON3, Sigdet threshold SDTH1
-		 */
-		val = PD_VDDTFILTER | CONF_GEN_SEL_GEN3;
-		writeb_relaxed(val, miphy_phy->base + BUF_SEL_REG);
-		val = SWING_VAL | PREEMPH_VAL;
-		writeb_relaxed(val, miphy_phy->base + TXBUF1_REG);
-		writeb_relaxed(TXSLEW_VAL, miphy_phy->base + TXBUF2_REG);
-		writeb_relaxed(0x00, miphy_phy->base + RXBUF_OFFSET_CTRL_REG);
-		val = SDTHRES_VAL | EQ_ON3;
-		writeb_relaxed(val, miphy_phy->base + RXBUF_REG);
-		break;
-	case SATA_GEN2:
-		/*
-		 * conf gen sel=0x1 to program Gen2 banked registers
-		 * VDDT filter ON
-		 * Tx Swing target 550-600mV peak-to-peak diff
-		 * Tx Slew target 90-110 ps rising/falling time
-		 * RX Equalization ON1, Sigdet threshold SDTH1
-		 */
-		writeb_relaxed(CONF_GEN_SEL_GEN2,
-			       miphy_phy->base + BUF_SEL_REG);
-		writeb_relaxed(SWING_VAL, miphy_phy->base + TXBUF1_REG);
-		writeb_relaxed(TXSLEW_VAL, miphy_phy->base + TXBUF2_REG);
-		val = SDTHRES_VAL | EQ_ON1;
-		writeb_relaxed(val, miphy_phy->base + RXBUF_REG);
-		break;
-	case SATA_GEN1:
-		/*
-		 * conf gen sel = 00b to program Gen1 banked registers
-		 * VDDT filter ON
-		 * Tx Swing target 500-550mV peak-to-peak diff
-		 * Tx Slew target120-140 ps rising/falling time
-		 */
-		writeb_relaxed(PD_VDDTFILTER, miphy_phy->base + BUF_SEL_REG);
-		writeb_relaxed(SWING_VAL_GEN1, miphy_phy->base + TXBUF1_REG);
-		writeb_relaxed(TXSLEW_VAL_GEN1,	miphy_phy->base + TXBUF2_REG);
-		break;
-	default:
-		break;
-	}
-
-	/* Force Macro1 in partial mode & release pll cal reset */
-	writeb_relaxed(RST_RX, miphy_phy->base + RESET_REG);
-	usleep_range(100, 150);
-
-	miphy365x_set_ssc(miphy_phy, miphy_dev);
-
-	/* Wait for phy_ready */
-	ret = miphy365x_rdy(miphy_phy, miphy_dev);
-	if (ret)
-		return ret;
-
-	/*
-	 * Enable macro1 to use rx_lspd & tx_lspd
-	 * Release Rx_Clock on first I-DLL phase on macro1
-	 * Assert deserializer reset
-	 * des_bit_lock_en is set
-	 * bit lock detection strength
-	 * Deassert deserializer reset
-	 */
-	writeb_relaxed(0x00, miphy_phy->base + BOUNDARY1_REG);
-	writeb_relaxed(0x00, miphy_phy->base + IDLL_TEST_REG);
-	writeb_relaxed(RST_RX, miphy_phy->base + RESET_REG);
-	val = miphy_phy->sata_tx_pol_inv ?
-		(TX_POL | DES_BIT_LOCK_EN) : DES_BIT_LOCK_EN;
-	writeb_relaxed(val, miphy_phy->base + CTRL_REG);
-
-	val = BIT_LOCK_CNT_512 | BIT_LOCK_LEVEL;
-	writeb_relaxed(val, miphy_phy->base + DES_BITLOCK_REG);
-	writeb_relaxed(0x00, miphy_phy->base + RESET_REG);
-
-	return 0;
-}
-
-static int miphy365x_init(struct phy *phy)
-{
-	struct miphy365x_phy *miphy_phy = phy_get_drvdata(phy);
-	struct miphy365x_dev *miphy_dev = dev_get_drvdata(phy->dev.parent);
-	int ret = 0;
-
-	mutex_lock(&miphy_dev->miphy_mutex);
-
-	ret = miphy365x_set_path(miphy_phy, miphy_dev);
-	if (ret) {
-		mutex_unlock(&miphy_dev->miphy_mutex);
-		return ret;
-	}
-
-	/* Initialise Miphy for PCIe or SATA */
-	if (miphy_phy->type == PHY_TYPE_PCIE)
-		ret = miphy365x_init_pcie_port(miphy_phy, miphy_dev);
-	else
-		ret = miphy365x_init_sata_port(miphy_phy, miphy_dev);
-
-	mutex_unlock(&miphy_dev->miphy_mutex);
-
-	return ret;
-}
-
-static int miphy365x_get_addr(struct device *dev,
-		struct miphy365x_phy *miphy_phy, int index)
-{
-	struct device_node *phynode = miphy_phy->phy->dev.of_node;
-	const char *name;
-	int type = miphy_phy->type;
-	int ret;
-
-	ret = of_property_read_string_index(phynode, "reg-names", index, &name);
-	if (ret) {
-		dev_err(dev, "no reg-names property not found\n");
-		return ret;
-	}
-
-	if (!((!strncmp(name, "sata", 4) && type == PHY_TYPE_SATA) ||
-	      (!strncmp(name, "pcie", 4) && type == PHY_TYPE_PCIE)))
-		return 0;
-
-	miphy_phy->base = of_iomap(phynode, index);
-	if (!miphy_phy->base) {
-		dev_err(dev, "Failed to map %s\n", phynode->full_name);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static struct phy *miphy365x_xlate(struct device *dev,
-				   struct of_phandle_args *args)
-{
-	struct miphy365x_dev *miphy_dev = dev_get_drvdata(dev);
-	struct miphy365x_phy *miphy_phy = NULL;
-	struct device_node *phynode = args->np;
-	int ret, index;
-
-	if (args->args_count != 1) {
-		dev_err(dev, "Invalid number of cells in 'phy' property\n");
-		return ERR_PTR(-EINVAL);
-	}
-
-	for (index = 0; index < miphy_dev->nphys; index++)
-		if (phynode == miphy_dev->phys[index]->phy->dev.of_node) {
-			miphy_phy = miphy_dev->phys[index];
-			break;
-		}
-
-	if (!miphy_phy) {
-		dev_err(dev, "Failed to find appropriate phy\n");
-		return ERR_PTR(-EINVAL);
-	}
-
-	miphy_phy->type = args->args[0];
-
-	if (!(miphy_phy->type == PHY_TYPE_SATA ||
-	      miphy_phy->type == PHY_TYPE_PCIE)) {
-		dev_err(dev, "Unsupported device type: %d\n", miphy_phy->type);
-		return ERR_PTR(-EINVAL);
-	}
-
-	/* Each port handles SATA and PCIE - third entry is always sysconf. */
-	for (index = 0; index < 3; index++) {
-		ret = miphy365x_get_addr(dev, miphy_phy, index);
-		if (ret < 0)
-			return ERR_PTR(ret);
-	}
-
-	return miphy_phy->phy;
-}
-
-static const struct phy_ops miphy365x_ops = {
-	.init		= miphy365x_init,
-	.owner		= THIS_MODULE,
-};
-
-static int miphy365x_of_probe(struct device_node *phynode,
-			      struct miphy365x_phy *miphy_phy)
-{
-	of_property_read_u32(phynode, "st,sata-gen", &miphy_phy->sata_gen);
-	if (!miphy_phy->sata_gen)
-		miphy_phy->sata_gen = SATA_GEN1;
-
-	miphy_phy->pcie_tx_pol_inv =
-		of_property_read_bool(phynode, "st,pcie-tx-pol-inv");
-
-	miphy_phy->sata_tx_pol_inv =
-		of_property_read_bool(phynode, "st,sata-tx-pol-inv");
-
-	return 0;
-}
-
-static int miphy365x_probe(struct platform_device *pdev)
-{
-	struct device_node *child, *np = pdev->dev.of_node;
-	struct miphy365x_dev *miphy_dev;
-	struct phy_provider *provider;
-	struct phy *phy;
-	int ret, port = 0;
-
-	miphy_dev = devm_kzalloc(&pdev->dev, sizeof(*miphy_dev), GFP_KERNEL);
-	if (!miphy_dev)
-		return -ENOMEM;
-
-	miphy_dev->nphys = of_get_child_count(np);
-	miphy_dev->phys = devm_kcalloc(&pdev->dev, miphy_dev->nphys,
-				       sizeof(*miphy_dev->phys), GFP_KERNEL);
-	if (!miphy_dev->phys)
-		return -ENOMEM;
-
-	miphy_dev->regmap = syscon_regmap_lookup_by_phandle(np, "st,syscfg");
-	if (IS_ERR(miphy_dev->regmap)) {
-		dev_err(miphy_dev->dev, "No syscfg phandle specified\n");
-		return PTR_ERR(miphy_dev->regmap);
-	}
-
-	miphy_dev->dev = &pdev->dev;
-
-	dev_set_drvdata(&pdev->dev, miphy_dev);
-
-	mutex_init(&miphy_dev->miphy_mutex);
-
-	for_each_child_of_node(np, child) {
-		struct miphy365x_phy *miphy_phy;
-
-		miphy_phy = devm_kzalloc(&pdev->dev, sizeof(*miphy_phy),
-					 GFP_KERNEL);
-		if (!miphy_phy) {
-			ret = -ENOMEM;
-			goto put_child;
-		}
-
-		miphy_dev->phys[port] = miphy_phy;
-
-		phy = devm_phy_create(&pdev->dev, child, &miphy365x_ops);
-		if (IS_ERR(phy)) {
-			dev_err(&pdev->dev, "failed to create PHY\n");
-			ret = PTR_ERR(phy);
-			goto put_child;
-		}
-
-		miphy_dev->phys[port]->phy = phy;
-
-		ret = miphy365x_of_probe(child, miphy_phy);
-		if (ret)
-			goto put_child;
-
-		phy_set_drvdata(phy, miphy_dev->phys[port]);
-
-		port++;
-		/* sysconfig offsets are indexed from 1 */
-		ret = of_property_read_u32_index(np, "st,syscfg", port,
-					&miphy_phy->ctrlreg);
-		if (ret) {
-			dev_err(&pdev->dev, "No sysconfig offset found\n");
-			goto put_child;
-		}
-	}
-
-	provider = devm_of_phy_provider_register(&pdev->dev, miphy365x_xlate);
-	return PTR_ERR_OR_ZERO(provider);
-put_child:
-	of_node_put(child);
-	return ret;
-}
-
-static const struct of_device_id miphy365x_of_match[] = {
-	{ .compatible = "st,miphy365x-phy", },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, miphy365x_of_match);
-
-static struct platform_driver miphy365x_driver = {
-	.probe	= miphy365x_probe,
-	.driver = {
-		.name	= "miphy365x-phy",
-		.of_match_table	= miphy365x_of_match,
-	}
-};
-module_platform_driver(miphy365x_driver);
-
-MODULE_AUTHOR("Alexandre Torgue <alexandre.torgue@st.com>");
-MODULE_DESCRIPTION("STMicroelectronics miphy365x driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/phy-qcom-ufs-i.h b/drivers/phy/phy-qcom-ufs-i.h
index 2bd5ce43a724..d505d98cf5f8 100644
--- a/drivers/phy/phy-qcom-ufs-i.h
+++ b/drivers/phy/phy-qcom-ufs-i.h
@@ -141,11 +141,8 @@ struct ufs_qcom_phy_specific_ops {
 struct ufs_qcom_phy *get_ufs_qcom_phy(struct phy *generic_phy);
 int ufs_qcom_phy_power_on(struct phy *generic_phy);
 int ufs_qcom_phy_power_off(struct phy *generic_phy);
-int ufs_qcom_phy_exit(struct phy *generic_phy);
-int ufs_qcom_phy_init_clks(struct phy *generic_phy,
-			struct ufs_qcom_phy *phy_common);
-int ufs_qcom_phy_init_vregulators(struct phy *generic_phy,
-			struct ufs_qcom_phy *phy_common);
+int ufs_qcom_phy_init_clks(struct ufs_qcom_phy *phy_common);
+int ufs_qcom_phy_init_vregulators(struct ufs_qcom_phy *phy_common);
 int ufs_qcom_phy_remove(struct phy *generic_phy,
 		       struct ufs_qcom_phy *ufs_qcom_phy);
 struct phy *ufs_qcom_phy_generic_probe(struct platform_device *pdev,
diff --git a/drivers/phy/phy-qcom-ufs-qmp-14nm.c b/drivers/phy/phy-qcom-ufs-qmp-14nm.c
index 6ee51490f786..c71c84734916 100644
--- a/drivers/phy/phy-qcom-ufs-qmp-14nm.c
+++ b/drivers/phy/phy-qcom-ufs-qmp-14nm.c
@@ -44,30 +44,12 @@ void ufs_qcom_phy_qmp_14nm_advertise_quirks(struct ufs_qcom_phy *phy_common)
 
 static int ufs_qcom_phy_qmp_14nm_init(struct phy *generic_phy)
 {
-	struct ufs_qcom_phy_qmp_14nm *phy = phy_get_drvdata(generic_phy);
-	struct ufs_qcom_phy *phy_common = &phy->common_cfg;
-	int err;
-
-	err = ufs_qcom_phy_init_clks(generic_phy, phy_common);
-	if (err) {
-		dev_err(phy_common->dev, "%s: ufs_qcom_phy_init_clks() failed %d\n",
-			__func__, err);
-		goto out;
-	}
-
-	err = ufs_qcom_phy_init_vregulators(generic_phy, phy_common);
-	if (err) {
-		dev_err(phy_common->dev, "%s: ufs_qcom_phy_init_vregulators() failed %d\n",
-			__func__, err);
-		goto out;
-	}
-	phy_common->vdda_phy.max_uV = UFS_PHY_VDDA_PHY_UV;
-	phy_common->vdda_phy.min_uV = UFS_PHY_VDDA_PHY_UV;
-
-	ufs_qcom_phy_qmp_14nm_advertise_quirks(phy_common);
+	return 0;
+}
 
-out:
-	return err;
+static int ufs_qcom_phy_qmp_14nm_exit(struct phy *generic_phy)
+{
+	return 0;
 }
 
 static
@@ -117,7 +99,7 @@ static int ufs_qcom_phy_qmp_14nm_is_pcs_ready(struct ufs_qcom_phy *phy_common)
 
 static const struct phy_ops ufs_qcom_phy_qmp_14nm_phy_ops = {
 	.init		= ufs_qcom_phy_qmp_14nm_init,
-	.exit		= ufs_qcom_phy_exit,
+	.exit		= ufs_qcom_phy_qmp_14nm_exit,
 	.power_on	= ufs_qcom_phy_power_on,
 	.power_off	= ufs_qcom_phy_power_off,
 	.owner		= THIS_MODULE,
@@ -136,6 +118,7 @@ static int ufs_qcom_phy_qmp_14nm_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct phy *generic_phy;
 	struct ufs_qcom_phy_qmp_14nm *phy;
+	struct ufs_qcom_phy *phy_common;
 	int err = 0;
 
 	phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL);
@@ -143,8 +126,9 @@ static int ufs_qcom_phy_qmp_14nm_probe(struct platform_device *pdev)
 		err = -ENOMEM;
 		goto out;
 	}
+	phy_common = &phy->common_cfg;
 
-	generic_phy = ufs_qcom_phy_generic_probe(pdev, &phy->common_cfg,
+	generic_phy = ufs_qcom_phy_generic_probe(pdev, phy_common,
 				&ufs_qcom_phy_qmp_14nm_phy_ops, &phy_14nm_ops);
 
 	if (!generic_phy) {
@@ -154,39 +138,43 @@ static int ufs_qcom_phy_qmp_14nm_probe(struct platform_device *pdev)
 		goto out;
 	}
 
-	phy_set_drvdata(generic_phy, phy);
+	err = ufs_qcom_phy_init_clks(phy_common);
+	if (err) {
+		dev_err(phy_common->dev,
+			"%s: ufs_qcom_phy_init_clks() failed %d\n",
+			__func__, err);
+		goto out;
+	}
 
-	strlcpy(phy->common_cfg.name, UFS_PHY_NAME,
-		sizeof(phy->common_cfg.name));
+	err = ufs_qcom_phy_init_vregulators(phy_common);
+	if (err) {
+		dev_err(phy_common->dev,
+			"%s: ufs_qcom_phy_init_vregulators() failed %d\n",
+			__func__, err);
+		goto out;
+	}
+	phy_common->vdda_phy.max_uV = UFS_PHY_VDDA_PHY_UV;
+	phy_common->vdda_phy.min_uV = UFS_PHY_VDDA_PHY_UV;
 
-out:
-	return err;
-}
+	ufs_qcom_phy_qmp_14nm_advertise_quirks(phy_common);
 
-static int ufs_qcom_phy_qmp_14nm_remove(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct phy *generic_phy = to_phy(dev);
-	struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(generic_phy);
-	int err = 0;
+	phy_set_drvdata(generic_phy, phy);
 
-	err = ufs_qcom_phy_remove(generic_phy, ufs_qcom_phy);
-	if (err)
-		dev_err(dev, "%s: ufs_qcom_phy_remove failed = %d\n",
-			__func__, err);
+	strlcpy(phy_common->name, UFS_PHY_NAME, sizeof(phy_common->name));
 
+out:
 	return err;
 }
 
 static const struct of_device_id ufs_qcom_phy_qmp_14nm_of_match[] = {
 	{.compatible = "qcom,ufs-phy-qmp-14nm"},
+	{.compatible = "qcom,msm8996-ufs-phy-qmp-14nm"},
 	{},
 };
 MODULE_DEVICE_TABLE(of, ufs_qcom_phy_qmp_14nm_of_match);
 
 static struct platform_driver ufs_qcom_phy_qmp_14nm_driver = {
 	.probe = ufs_qcom_phy_qmp_14nm_probe,
-	.remove = ufs_qcom_phy_qmp_14nm_remove,
 	.driver = {
 		.of_match_table = ufs_qcom_phy_qmp_14nm_of_match,
 		.name = "ufs_qcom_phy_qmp_14nm",
diff --git a/drivers/phy/phy-qcom-ufs-qmp-20nm.c b/drivers/phy/phy-qcom-ufs-qmp-20nm.c
index 770087ab05e2..1a26a64e06d3 100644
--- a/drivers/phy/phy-qcom-ufs-qmp-20nm.c
+++ b/drivers/phy/phy-qcom-ufs-qmp-20nm.c
@@ -63,28 +63,12 @@ void ufs_qcom_phy_qmp_20nm_advertise_quirks(struct ufs_qcom_phy *phy_common)
 
 static int ufs_qcom_phy_qmp_20nm_init(struct phy *generic_phy)
 {
-	struct ufs_qcom_phy_qmp_20nm *phy = phy_get_drvdata(generic_phy);
-	struct ufs_qcom_phy *phy_common = &phy->common_cfg;
-	int err = 0;
-
-	err = ufs_qcom_phy_init_clks(generic_phy, phy_common);
-	if (err) {
-		dev_err(phy_common->dev, "%s: ufs_qcom_phy_init_clks() failed %d\n",
-			__func__, err);
-		goto out;
-	}
-
-	err = ufs_qcom_phy_init_vregulators(generic_phy, phy_common);
-	if (err) {
-		dev_err(phy_common->dev, "%s: ufs_qcom_phy_init_vregulators() failed %d\n",
-			__func__, err);
-		goto out;
-	}
-
-	ufs_qcom_phy_qmp_20nm_advertise_quirks(phy_common);
+	return 0;
+}
 
-out:
-	return err;
+static int ufs_qcom_phy_qmp_20nm_exit(struct phy *generic_phy)
+{
+	return 0;
 }
 
 static
@@ -173,7 +157,7 @@ static int ufs_qcom_phy_qmp_20nm_is_pcs_ready(struct ufs_qcom_phy *phy_common)
 
 static const struct phy_ops ufs_qcom_phy_qmp_20nm_phy_ops = {
 	.init		= ufs_qcom_phy_qmp_20nm_init,
-	.exit		= ufs_qcom_phy_exit,
+	.exit		= ufs_qcom_phy_qmp_20nm_exit,
 	.power_on	= ufs_qcom_phy_power_on,
 	.power_off	= ufs_qcom_phy_power_off,
 	.owner		= THIS_MODULE,
@@ -192,6 +176,7 @@ static int ufs_qcom_phy_qmp_20nm_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct phy *generic_phy;
 	struct ufs_qcom_phy_qmp_20nm *phy;
+	struct ufs_qcom_phy *phy_common;
 	int err = 0;
 
 	phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL);
@@ -199,8 +184,9 @@ static int ufs_qcom_phy_qmp_20nm_probe(struct platform_device *pdev)
 		err = -ENOMEM;
 		goto out;
 	}
+	phy_common = &phy->common_cfg;
 
-	generic_phy = ufs_qcom_phy_generic_probe(pdev, &phy->common_cfg,
+	generic_phy = ufs_qcom_phy_generic_probe(pdev, phy_common,
 				&ufs_qcom_phy_qmp_20nm_phy_ops, &phy_20nm_ops);
 
 	if (!generic_phy) {
@@ -210,27 +196,27 @@ static int ufs_qcom_phy_qmp_20nm_probe(struct platform_device *pdev)
 		goto out;
 	}
 
-	phy_set_drvdata(generic_phy, phy);
+	err = ufs_qcom_phy_init_clks(phy_common);
+	if (err) {
+		dev_err(phy_common->dev, "%s: ufs_qcom_phy_init_clks() failed %d\n",
+			__func__, err);
+		goto out;
+	}
 
-	strlcpy(phy->common_cfg.name, UFS_PHY_NAME,
-			sizeof(phy->common_cfg.name));
+	err = ufs_qcom_phy_init_vregulators(phy_common);
+	if (err) {
+		dev_err(phy_common->dev, "%s: ufs_qcom_phy_init_vregulators() failed %d\n",
+			__func__, err);
+		goto out;
+	}
 
-out:
-	return err;
-}
+	ufs_qcom_phy_qmp_20nm_advertise_quirks(phy_common);
 
-static int ufs_qcom_phy_qmp_20nm_remove(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct phy *generic_phy = to_phy(dev);
-	struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(generic_phy);
-	int err = 0;
+	phy_set_drvdata(generic_phy, phy);
 
-	err = ufs_qcom_phy_remove(generic_phy, ufs_qcom_phy);
-	if (err)
-		dev_err(dev, "%s: ufs_qcom_phy_remove failed = %d\n",
-			__func__, err);
+	strlcpy(phy_common->name, UFS_PHY_NAME, sizeof(phy_common->name));
 
+out:
 	return err;
 }
 
@@ -242,7 +228,6 @@ MODULE_DEVICE_TABLE(of, ufs_qcom_phy_qmp_20nm_of_match);
 
 static struct platform_driver ufs_qcom_phy_qmp_20nm_driver = {
 	.probe = ufs_qcom_phy_qmp_20nm_probe,
-	.remove = ufs_qcom_phy_qmp_20nm_remove,
 	.driver = {
 		.of_match_table = ufs_qcom_phy_qmp_20nm_of_match,
 		.name = "ufs_qcom_phy_qmp_20nm",
diff --git a/drivers/phy/phy-qcom-ufs.c b/drivers/phy/phy-qcom-ufs.c
index 18a5b495ad65..c69568b8543d 100644
--- a/drivers/phy/phy-qcom-ufs.c
+++ b/drivers/phy/phy-qcom-ufs.c
@@ -22,13 +22,6 @@
 #define VDDP_REF_CLK_MIN_UV        1200000
 #define VDDP_REF_CLK_MAX_UV        1200000
 
-static int __ufs_qcom_phy_init_vreg(struct phy *, struct ufs_qcom_phy_vreg *,
-				    const char *, bool);
-static int ufs_qcom_phy_init_vreg(struct phy *, struct ufs_qcom_phy_vreg *,
-				  const char *);
-static int ufs_qcom_phy_base_init(struct platform_device *pdev,
-				  struct ufs_qcom_phy *phy_common);
-
 int ufs_qcom_phy_calibrate(struct ufs_qcom_phy *ufs_qcom_phy,
 			   struct ufs_qcom_phy_calibration *tbl_A,
 			   int tbl_size_A,
@@ -75,45 +68,6 @@ out:
 }
 EXPORT_SYMBOL_GPL(ufs_qcom_phy_calibrate);
 
-struct phy *ufs_qcom_phy_generic_probe(struct platform_device *pdev,
-				struct ufs_qcom_phy *common_cfg,
-				const struct phy_ops *ufs_qcom_phy_gen_ops,
-				struct ufs_qcom_phy_specific_ops *phy_spec_ops)
-{
-	int err;
-	struct device *dev = &pdev->dev;
-	struct phy *generic_phy = NULL;
-	struct phy_provider *phy_provider;
-
-	err = ufs_qcom_phy_base_init(pdev, common_cfg);
-	if (err) {
-		dev_err(dev, "%s: phy base init failed %d\n", __func__, err);
-		goto out;
-	}
-
-	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
-	if (IS_ERR(phy_provider)) {
-		err = PTR_ERR(phy_provider);
-		dev_err(dev, "%s: failed to register phy %d\n", __func__, err);
-		goto out;
-	}
-
-	generic_phy = devm_phy_create(dev, NULL, ufs_qcom_phy_gen_ops);
-	if (IS_ERR(generic_phy)) {
-		err =  PTR_ERR(generic_phy);
-		dev_err(dev, "%s: failed to create phy %d\n", __func__, err);
-		generic_phy = NULL;
-		goto out;
-	}
-
-	common_cfg->phy_spec_ops = phy_spec_ops;
-	common_cfg->dev = dev;
-
-out:
-	return generic_phy;
-}
-EXPORT_SYMBOL_GPL(ufs_qcom_phy_generic_probe);
-
 /*
  * This assumes the embedded phy structure inside generic_phy is of type
  * struct ufs_qcom_phy. In order to function properly it's crucial
@@ -154,13 +108,50 @@ int ufs_qcom_phy_base_init(struct platform_device *pdev,
 	return 0;
 }
 
-static int __ufs_qcom_phy_clk_get(struct phy *phy,
+struct phy *ufs_qcom_phy_generic_probe(struct platform_device *pdev,
+				struct ufs_qcom_phy *common_cfg,
+				const struct phy_ops *ufs_qcom_phy_gen_ops,
+				struct ufs_qcom_phy_specific_ops *phy_spec_ops)
+{
+	int err;
+	struct device *dev = &pdev->dev;
+	struct phy *generic_phy = NULL;
+	struct phy_provider *phy_provider;
+
+	err = ufs_qcom_phy_base_init(pdev, common_cfg);
+	if (err) {
+		dev_err(dev, "%s: phy base init failed %d\n", __func__, err);
+		goto out;
+	}
+
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	if (IS_ERR(phy_provider)) {
+		err = PTR_ERR(phy_provider);
+		dev_err(dev, "%s: failed to register phy %d\n", __func__, err);
+		goto out;
+	}
+
+	generic_phy = devm_phy_create(dev, NULL, ufs_qcom_phy_gen_ops);
+	if (IS_ERR(generic_phy)) {
+		err =  PTR_ERR(generic_phy);
+		dev_err(dev, "%s: failed to create phy %d\n", __func__, err);
+		generic_phy = NULL;
+		goto out;
+	}
+
+	common_cfg->phy_spec_ops = phy_spec_ops;
+	common_cfg->dev = dev;
+
+out:
+	return generic_phy;
+}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_generic_probe);
+
+static int __ufs_qcom_phy_clk_get(struct device *dev,
 			 const char *name, struct clk **clk_out, bool err_print)
 {
 	struct clk *clk;
 	int err = 0;
-	struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(phy);
-	struct device *dev = ufs_qcom_phy->dev;
 
 	clk = devm_clk_get(dev, name);
 	if (IS_ERR(clk)) {
@@ -174,42 +165,44 @@ static int __ufs_qcom_phy_clk_get(struct phy *phy,
 	return err;
 }
 
-static
-int ufs_qcom_phy_clk_get(struct phy *phy,
+static int ufs_qcom_phy_clk_get(struct device *dev,
 			 const char *name, struct clk **clk_out)
 {
-	return __ufs_qcom_phy_clk_get(phy, name, clk_out, true);
+	return __ufs_qcom_phy_clk_get(dev, name, clk_out, true);
 }
 
-int
-ufs_qcom_phy_init_clks(struct phy *generic_phy,
-		       struct ufs_qcom_phy *phy_common)
+int ufs_qcom_phy_init_clks(struct ufs_qcom_phy *phy_common)
 {
 	int err;
 
-	err = ufs_qcom_phy_clk_get(generic_phy, "tx_iface_clk",
+	if (of_device_is_compatible(phy_common->dev->of_node,
+				"qcom,msm8996-ufs-phy-qmp-14nm"))
+		goto skip_txrx_clk;
+
+	err = ufs_qcom_phy_clk_get(phy_common->dev, "tx_iface_clk",
 				   &phy_common->tx_iface_clk);
 	if (err)
 		goto out;
 
-	err = ufs_qcom_phy_clk_get(generic_phy, "rx_iface_clk",
+	err = ufs_qcom_phy_clk_get(phy_common->dev, "rx_iface_clk",
 				   &phy_common->rx_iface_clk);
 	if (err)
 		goto out;
 
-	err = ufs_qcom_phy_clk_get(generic_phy, "ref_clk_src",
+	err = ufs_qcom_phy_clk_get(phy_common->dev, "ref_clk_src",
 				   &phy_common->ref_clk_src);
 	if (err)
 		goto out;
 
+skip_txrx_clk:
 	/*
 	 * "ref_clk_parent" is optional hence don't abort init if it's not
 	 * found.
 	 */
-	__ufs_qcom_phy_clk_get(generic_phy, "ref_clk_parent",
+	__ufs_qcom_phy_clk_get(phy_common->dev, "ref_clk_parent",
 				   &phy_common->ref_clk_parent, false);
 
-	err = ufs_qcom_phy_clk_get(generic_phy, "ref_clk",
+	err = ufs_qcom_phy_clk_get(phy_common->dev, "ref_clk",
 				   &phy_common->ref_clk);
 
 out:
@@ -217,41 +210,14 @@ out:
 }
 EXPORT_SYMBOL_GPL(ufs_qcom_phy_init_clks);
 
-int
-ufs_qcom_phy_init_vregulators(struct phy *generic_phy,
-			      struct ufs_qcom_phy *phy_common)
-{
-	int err;
-
-	err = ufs_qcom_phy_init_vreg(generic_phy, &phy_common->vdda_pll,
-		"vdda-pll");
-	if (err)
-		goto out;
-
-	err = ufs_qcom_phy_init_vreg(generic_phy, &phy_common->vdda_phy,
-		"vdda-phy");
-
-	if (err)
-		goto out;
-
-	/* vddp-ref-clk-* properties are optional */
-	__ufs_qcom_phy_init_vreg(generic_phy, &phy_common->vddp_ref_clk,
-				 "vddp-ref-clk", true);
-out:
-	return err;
-}
-EXPORT_SYMBOL_GPL(ufs_qcom_phy_init_vregulators);
-
-static int __ufs_qcom_phy_init_vreg(struct phy *phy,
+static int __ufs_qcom_phy_init_vreg(struct device *dev,
 		struct ufs_qcom_phy_vreg *vreg, const char *name, bool optional)
 {
 	int err = 0;
-	struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(phy);
-	struct device *dev = ufs_qcom_phy->dev;
 
 	char prop_name[MAX_PROP_NAME];
 
-	vreg->name = kstrdup(name, GFP_KERNEL);
+	vreg->name = devm_kstrdup(dev, name, GFP_KERNEL);
 	if (!vreg->name) {
 		err = -ENOMEM;
 		goto out;
@@ -304,14 +270,36 @@ out:
 	return err;
 }
 
-static int ufs_qcom_phy_init_vreg(struct phy *phy,
+static int ufs_qcom_phy_init_vreg(struct device *dev,
 			struct ufs_qcom_phy_vreg *vreg, const char *name)
 {
-	return __ufs_qcom_phy_init_vreg(phy, vreg, name, false);
+	return __ufs_qcom_phy_init_vreg(dev, vreg, name, false);
 }
 
-static
-int ufs_qcom_phy_cfg_vreg(struct phy *phy,
+int ufs_qcom_phy_init_vregulators(struct ufs_qcom_phy *phy_common)
+{
+	int err;
+
+	err = ufs_qcom_phy_init_vreg(phy_common->dev, &phy_common->vdda_pll,
+		"vdda-pll");
+	if (err)
+		goto out;
+
+	err = ufs_qcom_phy_init_vreg(phy_common->dev, &phy_common->vdda_phy,
+		"vdda-phy");
+
+	if (err)
+		goto out;
+
+	/* vddp-ref-clk-* properties are optional */
+	__ufs_qcom_phy_init_vreg(phy_common->dev, &phy_common->vddp_ref_clk,
+				 "vddp-ref-clk", true);
+out:
+	return err;
+}
+EXPORT_SYMBOL_GPL(ufs_qcom_phy_init_vregulators);
+
+static int ufs_qcom_phy_cfg_vreg(struct device *dev,
 			  struct ufs_qcom_phy_vreg *vreg, bool on)
 {
 	int ret = 0;
@@ -319,10 +307,6 @@ int ufs_qcom_phy_cfg_vreg(struct phy *phy,
 	const char *name = vreg->name;
 	int min_uV;
 	int uA_load;
-	struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(phy);
-	struct device *dev = ufs_qcom_phy->dev;
-
-	BUG_ON(!vreg);
 
 	if (regulator_count_voltages(reg) > 0) {
 		min_uV = on ? vreg->min_uV : 0;
@@ -350,18 +334,15 @@ out:
 	return ret;
 }
 
-static
-int ufs_qcom_phy_enable_vreg(struct phy *phy,
+static int ufs_qcom_phy_enable_vreg(struct device *dev,
 			     struct ufs_qcom_phy_vreg *vreg)
 {
-	struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(phy);
-	struct device *dev = ufs_qcom_phy->dev;
 	int ret = 0;
 
 	if (!vreg || vreg->enabled)
 		goto out;
 
-	ret = ufs_qcom_phy_cfg_vreg(phy, vreg, true);
+	ret = ufs_qcom_phy_cfg_vreg(dev, vreg, true);
 	if (ret) {
 		dev_err(dev, "%s: ufs_qcom_phy_cfg_vreg() failed, err=%d\n",
 			__func__, ret);
@@ -380,10 +361,9 @@ out:
 	return ret;
 }
 
-int ufs_qcom_phy_enable_ref_clk(struct phy *generic_phy)
+static int ufs_qcom_phy_enable_ref_clk(struct ufs_qcom_phy *phy)
 {
 	int ret = 0;
-	struct ufs_qcom_phy *phy = get_ufs_qcom_phy(generic_phy);
 
 	if (phy->is_ref_clk_enabled)
 		goto out;
@@ -430,14 +410,10 @@ out_disable_src:
 out:
 	return ret;
 }
-EXPORT_SYMBOL_GPL(ufs_qcom_phy_enable_ref_clk);
 
-static
-int ufs_qcom_phy_disable_vreg(struct phy *phy,
+static int ufs_qcom_phy_disable_vreg(struct device *dev,
 			      struct ufs_qcom_phy_vreg *vreg)
 {
-	struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(phy);
-	struct device *dev = ufs_qcom_phy->dev;
 	int ret = 0;
 
 	if (!vreg || !vreg->enabled || vreg->is_always_on)
@@ -447,7 +423,7 @@ int ufs_qcom_phy_disable_vreg(struct phy *phy,
 
 	if (!ret) {
 		/* ignore errors on applying disable config */
-		ufs_qcom_phy_cfg_vreg(phy, vreg, false);
+		ufs_qcom_phy_cfg_vreg(dev, vreg, false);
 		vreg->enabled = false;
 	} else {
 		dev_err(dev, "%s: %s disable failed, err=%d\n",
@@ -457,10 +433,8 @@ out:
 	return ret;
 }
 
-void ufs_qcom_phy_disable_ref_clk(struct phy *generic_phy)
+static void ufs_qcom_phy_disable_ref_clk(struct ufs_qcom_phy *phy)
 {
-	struct ufs_qcom_phy *phy = get_ufs_qcom_phy(generic_phy);
-
 	if (phy->is_ref_clk_enabled) {
 		clk_disable_unprepare(phy->ref_clk);
 		/*
@@ -473,7 +447,6 @@ void ufs_qcom_phy_disable_ref_clk(struct phy *generic_phy)
 		phy->is_ref_clk_enabled = false;
 	}
 }
-EXPORT_SYMBOL_GPL(ufs_qcom_phy_disable_ref_clk);
 
 #define UFS_REF_CLK_EN	(1 << 5)
 
@@ -526,9 +499,8 @@ void ufs_qcom_phy_disable_dev_ref_clk(struct phy *generic_phy)
 EXPORT_SYMBOL_GPL(ufs_qcom_phy_disable_dev_ref_clk);
 
 /* Turn ON M-PHY RMMI interface clocks */
-int ufs_qcom_phy_enable_iface_clk(struct phy *generic_phy)
+static int ufs_qcom_phy_enable_iface_clk(struct ufs_qcom_phy *phy)
 {
-	struct ufs_qcom_phy *phy = get_ufs_qcom_phy(generic_phy);
 	int ret = 0;
 
 	if (phy->is_iface_clk_enabled)
@@ -552,20 +524,16 @@ int ufs_qcom_phy_enable_iface_clk(struct phy *generic_phy)
 out:
 	return ret;
 }
-EXPORT_SYMBOL_GPL(ufs_qcom_phy_enable_iface_clk);
 
 /* Turn OFF M-PHY RMMI interface clocks */
-void ufs_qcom_phy_disable_iface_clk(struct phy *generic_phy)
+void ufs_qcom_phy_disable_iface_clk(struct ufs_qcom_phy *phy)
 {
-	struct ufs_qcom_phy *phy = get_ufs_qcom_phy(generic_phy);
-
 	if (phy->is_iface_clk_enabled) {
 		clk_disable_unprepare(phy->tx_iface_clk);
 		clk_disable_unprepare(phy->rx_iface_clk);
 		phy->is_iface_clk_enabled = false;
 	}
 }
-EXPORT_SYMBOL_GPL(ufs_qcom_phy_disable_iface_clk);
 
 int ufs_qcom_phy_start_serdes(struct phy *generic_phy)
 {
@@ -634,29 +602,6 @@ int ufs_qcom_phy_calibrate_phy(struct phy *generic_phy, bool is_rate_B)
 }
 EXPORT_SYMBOL_GPL(ufs_qcom_phy_calibrate_phy);
 
-int ufs_qcom_phy_remove(struct phy *generic_phy,
-			struct ufs_qcom_phy *ufs_qcom_phy)
-{
-	phy_power_off(generic_phy);
-
-	kfree(ufs_qcom_phy->vdda_pll.name);
-	kfree(ufs_qcom_phy->vdda_phy.name);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(ufs_qcom_phy_remove);
-
-int ufs_qcom_phy_exit(struct phy *generic_phy)
-{
-	struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(generic_phy);
-
-	if (ufs_qcom_phy->is_powered_on)
-		phy_power_off(generic_phy);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(ufs_qcom_phy_exit);
-
 int ufs_qcom_phy_is_pcs_ready(struct phy *generic_phy)
 {
 	struct ufs_qcom_phy *ufs_qcom_phy = get_ufs_qcom_phy(generic_phy);
@@ -678,7 +623,10 @@ int ufs_qcom_phy_power_on(struct phy *generic_phy)
 	struct device *dev = phy_common->dev;
 	int err;
 
-	err = ufs_qcom_phy_enable_vreg(generic_phy, &phy_common->vdda_phy);
+	if (phy_common->is_powered_on)
+		return 0;
+
+	err = ufs_qcom_phy_enable_vreg(dev, &phy_common->vdda_phy);
 	if (err) {
 		dev_err(dev, "%s enable vdda_phy failed, err=%d\n",
 			__func__, err);
@@ -688,23 +636,30 @@ int ufs_qcom_phy_power_on(struct phy *generic_phy)
 	phy_common->phy_spec_ops->power_control(phy_common, true);
 
 	/* vdda_pll also enables ref clock LDOs so enable it first */
-	err = ufs_qcom_phy_enable_vreg(generic_phy, &phy_common->vdda_pll);
+	err = ufs_qcom_phy_enable_vreg(dev, &phy_common->vdda_pll);
 	if (err) {
 		dev_err(dev, "%s enable vdda_pll failed, err=%d\n",
 			__func__, err);
 		goto out_disable_phy;
 	}
 
-	err = ufs_qcom_phy_enable_ref_clk(generic_phy);
+	err = ufs_qcom_phy_enable_iface_clk(phy_common);
 	if (err) {
-		dev_err(dev, "%s enable phy ref clock failed, err=%d\n",
+		dev_err(dev, "%s enable phy iface clock failed, err=%d\n",
 			__func__, err);
 		goto out_disable_pll;
 	}
 
+	err = ufs_qcom_phy_enable_ref_clk(phy_common);
+	if (err) {
+		dev_err(dev, "%s enable phy ref clock failed, err=%d\n",
+			__func__, err);
+		goto out_disable_iface_clk;
+	}
+
 	/* enable device PHY ref_clk pad rail */
 	if (phy_common->vddp_ref_clk.reg) {
-		err = ufs_qcom_phy_enable_vreg(generic_phy,
+		err = ufs_qcom_phy_enable_vreg(dev,
 					       &phy_common->vddp_ref_clk);
 		if (err) {
 			dev_err(dev, "%s enable vddp_ref_clk failed, err=%d\n",
@@ -717,11 +672,13 @@ int ufs_qcom_phy_power_on(struct phy *generic_phy)
 	goto out;
 
 out_disable_ref_clk:
-	ufs_qcom_phy_disable_ref_clk(generic_phy);
+	ufs_qcom_phy_disable_ref_clk(phy_common);
+out_disable_iface_clk:
+	ufs_qcom_phy_disable_iface_clk(phy_common);
 out_disable_pll:
-	ufs_qcom_phy_disable_vreg(generic_phy, &phy_common->vdda_pll);
+	ufs_qcom_phy_disable_vreg(dev, &phy_common->vdda_pll);
 out_disable_phy:
-	ufs_qcom_phy_disable_vreg(generic_phy, &phy_common->vdda_phy);
+	ufs_qcom_phy_disable_vreg(dev, &phy_common->vdda_phy);
 out:
 	return err;
 }
@@ -731,15 +688,19 @@ int ufs_qcom_phy_power_off(struct phy *generic_phy)
 {
 	struct ufs_qcom_phy *phy_common = get_ufs_qcom_phy(generic_phy);
 
+	if (!phy_common->is_powered_on)
+		return 0;
+
 	phy_common->phy_spec_ops->power_control(phy_common, false);
 
 	if (phy_common->vddp_ref_clk.reg)
-		ufs_qcom_phy_disable_vreg(generic_phy,
+		ufs_qcom_phy_disable_vreg(phy_common->dev,
 					  &phy_common->vddp_ref_clk);
-	ufs_qcom_phy_disable_ref_clk(generic_phy);
+	ufs_qcom_phy_disable_ref_clk(phy_common);
+	ufs_qcom_phy_disable_iface_clk(phy_common);
 
-	ufs_qcom_phy_disable_vreg(generic_phy, &phy_common->vdda_pll);
-	ufs_qcom_phy_disable_vreg(generic_phy, &phy_common->vdda_phy);
+	ufs_qcom_phy_disable_vreg(phy_common->dev, &phy_common->vdda_pll);
+	ufs_qcom_phy_disable_vreg(phy_common->dev, &phy_common->vdda_phy);
 	phy_common->is_powered_on = false;
 
 	return 0;
diff --git a/drivers/phy/phy-rcar-gen3-usb2.c b/drivers/phy/phy-rcar-gen3-usb2.c
index 3d97eadd247d..c63da1b955c1 100644
--- a/drivers/phy/phy-rcar-gen3-usb2.c
+++ b/drivers/phy/phy-rcar-gen3-usb2.c
@@ -70,6 +70,7 @@
 #define USB2_LINECTRL1_DP_RPD		BIT(18)
 #define USB2_LINECTRL1_DMRPD_EN		BIT(17)
 #define USB2_LINECTRL1_DM_RPD		BIT(16)
+#define USB2_LINECTRL1_OPMODE_NODRV	BIT(6)
 
 /* ADPCTRL */
 #define USB2_ADPCTRL_OTGSESSVLD		BIT(20)
@@ -161,6 +162,43 @@ static void rcar_gen3_init_for_peri(struct rcar_gen3_chan *ch)
 	schedule_work(&ch->work);
 }
 
+static void rcar_gen3_init_for_b_host(struct rcar_gen3_chan *ch)
+{
+	void __iomem *usb2_base = ch->base;
+	u32 val;
+
+	val = readl(usb2_base + USB2_LINECTRL1);
+	writel(val | USB2_LINECTRL1_OPMODE_NODRV, usb2_base + USB2_LINECTRL1);
+
+	rcar_gen3_set_linectrl(ch, 1, 1);
+	rcar_gen3_set_host_mode(ch, 1);
+	rcar_gen3_enable_vbus_ctrl(ch, 0);
+
+	val = readl(usb2_base + USB2_LINECTRL1);
+	writel(val & ~USB2_LINECTRL1_OPMODE_NODRV, usb2_base + USB2_LINECTRL1);
+}
+
+static void rcar_gen3_init_for_a_peri(struct rcar_gen3_chan *ch)
+{
+	rcar_gen3_set_linectrl(ch, 0, 1);
+	rcar_gen3_set_host_mode(ch, 0);
+	rcar_gen3_enable_vbus_ctrl(ch, 1);
+}
+
+static void rcar_gen3_init_from_a_peri_to_a_host(struct rcar_gen3_chan *ch)
+{
+	void __iomem *usb2_base = ch->base;
+	u32 val;
+
+	val = readl(usb2_base + USB2_OBINTEN);
+	writel(val & ~USB2_OBINT_BITS, usb2_base + USB2_OBINTEN);
+
+	rcar_gen3_enable_vbus_ctrl(ch, 0);
+	rcar_gen3_init_for_host(ch);
+
+	writel(val | USB2_OBINT_BITS, usb2_base + USB2_OBINTEN);
+}
+
 static bool rcar_gen3_check_id(struct rcar_gen3_chan *ch)
 {
 	return !!(readl(ch->base + USB2_ADPCTRL) & USB2_ADPCTRL_IDDIG);
@@ -174,6 +212,65 @@ static void rcar_gen3_device_recognition(struct rcar_gen3_chan *ch)
 		rcar_gen3_init_for_peri(ch);
 }
 
+static bool rcar_gen3_is_host(struct rcar_gen3_chan *ch)
+{
+	return !(readl(ch->base + USB2_COMMCTRL) & USB2_COMMCTRL_OTG_PERI);
+}
+
+static ssize_t role_store(struct device *dev, struct device_attribute *attr,
+			  const char *buf, size_t count)
+{
+	struct rcar_gen3_chan *ch = dev_get_drvdata(dev);
+	bool is_b_device, is_host, new_mode_is_host;
+
+	if (!ch->has_otg || !ch->phy->init_count)
+		return -EIO;
+
+	/*
+	 * is_b_device: true is B-Device. false is A-Device.
+	 * If {new_mode_}is_host: true is Host mode. false is Peripheral mode.
+	 */
+	is_b_device = rcar_gen3_check_id(ch);
+	is_host = rcar_gen3_is_host(ch);
+	if (!strncmp(buf, "host", strlen("host")))
+		new_mode_is_host = true;
+	else if (!strncmp(buf, "peripheral", strlen("peripheral")))
+		new_mode_is_host = false;
+	else
+		return -EINVAL;
+
+	/* If current and new mode is the same, this returns the error */
+	if (is_host == new_mode_is_host)
+		return -EINVAL;
+
+	if (new_mode_is_host) {		/* And is_host must be false */
+		if (!is_b_device)	/* A-Peripheral */
+			rcar_gen3_init_from_a_peri_to_a_host(ch);
+		else			/* B-Peripheral */
+			rcar_gen3_init_for_b_host(ch);
+	} else {			/* And is_host must be true */
+		if (!is_b_device)	/* A-Host */
+			rcar_gen3_init_for_a_peri(ch);
+		else			/* B-Host */
+			rcar_gen3_init_for_peri(ch);
+	}
+
+	return count;
+}
+
+static ssize_t role_show(struct device *dev, struct device_attribute *attr,
+			 char *buf)
+{
+	struct rcar_gen3_chan *ch = dev_get_drvdata(dev);
+
+	if (!ch->has_otg || !ch->phy->init_count)
+		return -EIO;
+
+	return sprintf(buf, "%s\n", rcar_gen3_is_host(ch) ? "host" :
+							    "peripheral");
+}
+static DEVICE_ATTR_RW(role);
+
 static void rcar_gen3_init_otg(struct rcar_gen3_chan *ch)
 {
 	void __iomem *usb2_base = ch->base;
@@ -351,21 +448,40 @@ static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev)
 		channel->vbus = NULL;
 	}
 
+	platform_set_drvdata(pdev, channel);
 	phy_set_drvdata(channel->phy, channel);
 
 	provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
-	if (IS_ERR(provider))
+	if (IS_ERR(provider)) {
 		dev_err(dev, "Failed to register PHY provider\n");
+	} else if (channel->has_otg) {
+		int ret;
+
+		ret = device_create_file(dev, &dev_attr_role);
+		if (ret < 0)
+			return ret;
+	}
 
 	return PTR_ERR_OR_ZERO(provider);
 }
 
+static int rcar_gen3_phy_usb2_remove(struct platform_device *pdev)
+{
+	struct rcar_gen3_chan *channel = platform_get_drvdata(pdev);
+
+	if (channel->has_otg)
+		device_remove_file(&pdev->dev, &dev_attr_role);
+
+	return 0;
+};
+
 static struct platform_driver rcar_gen3_phy_usb2_driver = {
 	.driver = {
 		.name		= "phy_rcar_gen3_usb2",
 		.of_match_table	= rcar_gen3_phy_usb2_match_table,
 	},
 	.probe	= rcar_gen3_phy_usb2_probe,
+	.remove = rcar_gen3_phy_usb2_remove,
 };
 module_platform_driver(rcar_gen3_phy_usb2_driver);
 
diff --git a/drivers/phy/phy-rockchip-emmc.c b/drivers/phy/phy-rockchip-emmc.c
index fd57345ffed2..f1b24f18e9b2 100644
--- a/drivers/phy/phy-rockchip-emmc.c
+++ b/drivers/phy/phy-rockchip-emmc.c
@@ -132,7 +132,7 @@ static int rockchip_emmc_phy_power(struct phy *phy, bool on_off)
 		default:
 			ideal_rate = 200000000;
 			break;
-		};
+		}
 
 		diff = (rate > ideal_rate) ?
 			rate - ideal_rate : ideal_rate - rate;
diff --git a/drivers/phy/phy-rockchip-inno-usb2.c b/drivers/phy/phy-rockchip-inno-usb2.c
index ac203107b071..2f99ec95079c 100644
--- a/drivers/phy/phy-rockchip-inno-usb2.c
+++ b/drivers/phy/phy-rockchip-inno-usb2.c
@@ -17,6 +17,7 @@
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/delay.h>
+#include <linux/extcon.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/gpio/consumer.h>
@@ -30,11 +31,15 @@
 #include <linux/of_platform.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
+#include <linux/power_supply.h>
 #include <linux/regmap.h>
 #include <linux/mfd/syscon.h>
+#include <linux/usb/of.h>
+#include <linux/usb/otg.h>
 
 #define BIT_WRITEABLE_SHIFT	16
-#define SCHEDULE_DELAY	(60 * HZ)
+#define SCHEDULE_DELAY		(60 * HZ)
+#define OTG_SCHEDULE_DELAY	(2 * HZ)
 
 enum rockchip_usb2phy_port_id {
 	USB2PHY_PORT_OTG,
@@ -49,6 +54,37 @@ enum rockchip_usb2phy_host_state {
 	PHY_STATE_FS_LS_ONLINE	= 4,
 };
 
+/**
+ * Different states involved in USB charger detection.
+ * USB_CHG_STATE_UNDEFINED	USB charger is not connected or detection
+ *				process is not yet started.
+ * USB_CHG_STATE_WAIT_FOR_DCD	Waiting for Data pins contact.
+ * USB_CHG_STATE_DCD_DONE	Data pin contact is detected.
+ * USB_CHG_STATE_PRIMARY_DONE	Primary detection is completed (Detects
+ *				between SDP and DCP/CDP).
+ * USB_CHG_STATE_SECONDARY_DONE	Secondary detection is completed (Detects
+ *				between DCP and CDP).
+ * USB_CHG_STATE_DETECTED	USB charger type is determined.
+ */
+enum usb_chg_state {
+	USB_CHG_STATE_UNDEFINED = 0,
+	USB_CHG_STATE_WAIT_FOR_DCD,
+	USB_CHG_STATE_DCD_DONE,
+	USB_CHG_STATE_PRIMARY_DONE,
+	USB_CHG_STATE_SECONDARY_DONE,
+	USB_CHG_STATE_DETECTED,
+};
+
+static const unsigned int rockchip_usb2phy_extcon_cable[] = {
+	EXTCON_USB,
+	EXTCON_USB_HOST,
+	EXTCON_CHG_USB_SDP,
+	EXTCON_CHG_USB_CDP,
+	EXTCON_CHG_USB_DCP,
+	EXTCON_CHG_USB_SLOW,
+	EXTCON_NONE,
+};
+
 struct usb2phy_reg {
 	unsigned int	offset;
 	unsigned int	bitend;
@@ -58,19 +94,55 @@ struct usb2phy_reg {
 };
 
 /**
+ * struct rockchip_chg_det_reg: usb charger detect registers
+ * @cp_det: charging port detected successfully.
+ * @dcp_det: dedicated charging port detected successfully.
+ * @dp_det: assert data pin connect successfully.
+ * @idm_sink_en: open dm sink curren.
+ * @idp_sink_en: open dp sink current.
+ * @idp_src_en: open dm source current.
+ * @rdm_pdwn_en: open dm pull down resistor.
+ * @vdm_src_en: open dm voltage source.
+ * @vdp_src_en: open dp voltage source.
+ * @opmode: utmi operational mode.
+ */
+struct rockchip_chg_det_reg {
+	struct usb2phy_reg	cp_det;
+	struct usb2phy_reg	dcp_det;
+	struct usb2phy_reg	dp_det;
+	struct usb2phy_reg	idm_sink_en;
+	struct usb2phy_reg	idp_sink_en;
+	struct usb2phy_reg	idp_src_en;
+	struct usb2phy_reg	rdm_pdwn_en;
+	struct usb2phy_reg	vdm_src_en;
+	struct usb2phy_reg	vdp_src_en;
+	struct usb2phy_reg	opmode;
+};
+
+/**
  * struct rockchip_usb2phy_port_cfg: usb-phy port configuration.
  * @phy_sus: phy suspend register.
+ * @bvalid_det_en: vbus valid rise detection enable register.
+ * @bvalid_det_st: vbus valid rise detection status register.
+ * @bvalid_det_clr: vbus valid rise detection clear register.
  * @ls_det_en: linestate detection enable register.
  * @ls_det_st: linestate detection state register.
  * @ls_det_clr: linestate detection clear register.
+ * @utmi_avalid: utmi vbus avalid status register.
+ * @utmi_bvalid: utmi vbus bvalid status register.
  * @utmi_ls: utmi linestate state register.
  * @utmi_hstdet: utmi host disconnect register.
  */
 struct rockchip_usb2phy_port_cfg {
 	struct usb2phy_reg	phy_sus;
+	struct usb2phy_reg	bvalid_det_en;
+	struct usb2phy_reg	bvalid_det_st;
+	struct usb2phy_reg	bvalid_det_clr;
 	struct usb2phy_reg	ls_det_en;
 	struct usb2phy_reg	ls_det_st;
 	struct usb2phy_reg	ls_det_clr;
+	struct usb2phy_reg	utmi_avalid;
+	struct usb2phy_reg	utmi_bvalid;
 	struct usb2phy_reg	utmi_ls;
 	struct usb2phy_reg	utmi_hstdet;
 };
@@ -80,31 +152,51 @@ struct rockchip_usb2phy_port_cfg {
  * @reg: the address offset of grf for usb-phy config.
  * @num_ports: specify how many ports that the phy has.
  * @clkout_ctl: keep on/turn off output clk of phy.
+ * @chg_det: charger detection registers.
  */
 struct rockchip_usb2phy_cfg {
 	unsigned int	reg;
 	unsigned int	num_ports;
 	struct usb2phy_reg	clkout_ctl;
 	const struct rockchip_usb2phy_port_cfg	port_cfgs[USB2PHY_NUM_PORTS];
+	const struct rockchip_chg_det_reg	chg_det;
 };
 
 /**
  * struct rockchip_usb2phy_port: usb-phy port data.
  * @port_id: flag for otg port or host port.
  * @suspended: phy suspended flag.
+ * @utmi_avalid: utmi avalid status usage flag.
+ *	true	- use avalid to get vbus status
+ *	flase	- use bvalid to get vbus status
+ * @vbus_attached: otg device vbus status.
+ * @bvalid_irq: IRQ number assigned for vbus valid rise detection.
  * @ls_irq: IRQ number assigned for linestate detection.
  * @mutex: for register updating in sm_work.
- * @sm_work: OTG state machine work.
+ * @chg_work: charge detect work.
+ * @otg_sm_work: OTG state machine work.
+ * @sm_work: HOST state machine work.
  * @phy_cfg: port register configuration, assigned by driver data.
+ * @event_nb: hold event notification callback.
+ * @state: define OTG enumeration states before device reset.
+ * @mode: the dr_mode of the controller.
  */
 struct rockchip_usb2phy_port {
 	struct phy	*phy;
 	unsigned int	port_id;
 	bool		suspended;
+	bool		utmi_avalid;
+	bool		vbus_attached;
+	int		bvalid_irq;
 	int		ls_irq;
 	struct mutex	mutex;
+	struct		delayed_work chg_work;
+	struct		delayed_work otg_sm_work;
 	struct		delayed_work sm_work;
 	const struct	rockchip_usb2phy_port_cfg *port_cfg;
+	struct notifier_block	event_nb;
+	enum usb_otg_state	state;
+	enum usb_dr_mode	mode;
 };
 
 /**
@@ -113,6 +205,11 @@ struct rockchip_usb2phy_port {
  * @clk: clock struct of phy input clk.
  * @clk480m: clock struct of phy output clk.
  * @clk_hw: clock struct of phy output clk management.
+ * @chg_state: states involved in USB charger detection.
+ * @chg_type: USB charger types.
+ * @dcd_retries: The retry count used to track Data contact
+ *		 detection process.
+ * @edev: extcon device for notification registration
  * @phy_cfg: phy register configuration, assigned by driver data.
  * @ports: phy port instance.
  */
@@ -122,6 +219,10 @@ struct rockchip_usb2phy {
 	struct clk	*clk;
 	struct clk	*clk480m;
 	struct clk_hw	clk480m_hw;
+	enum usb_chg_state	chg_state;
+	enum power_supply_type	chg_type;
+	u8			dcd_retries;
+	struct extcon_dev	*edev;
 	const struct rockchip_usb2phy_cfg	*phy_cfg;
 	struct rockchip_usb2phy_port	ports[USB2PHY_NUM_PORTS];
 };
@@ -153,7 +254,7 @@ static inline bool property_enabled(struct rockchip_usb2phy *rphy,
 	return tmp == reg->enable;
 }
 
-static int rockchip_usb2phy_clk480m_enable(struct clk_hw *hw)
+static int rockchip_usb2phy_clk480m_prepare(struct clk_hw *hw)
 {
 	struct rockchip_usb2phy *rphy =
 		container_of(hw, struct rockchip_usb2phy, clk480m_hw);
@@ -165,14 +266,14 @@ static int rockchip_usb2phy_clk480m_enable(struct clk_hw *hw)
 		if (ret)
 			return ret;
 
-		/* waitting for the clk become stable */
-		mdelay(1);
+		/* waiting for the clk become stable */
+		usleep_range(1200, 1300);
 	}
 
 	return 0;
 }
 
-static void rockchip_usb2phy_clk480m_disable(struct clk_hw *hw)
+static void rockchip_usb2phy_clk480m_unprepare(struct clk_hw *hw)
 {
 	struct rockchip_usb2phy *rphy =
 		container_of(hw, struct rockchip_usb2phy, clk480m_hw);
@@ -181,7 +282,7 @@ static void rockchip_usb2phy_clk480m_disable(struct clk_hw *hw)
 	property_enable(rphy, &rphy->phy_cfg->clkout_ctl, false);
 }
 
-static int rockchip_usb2phy_clk480m_enabled(struct clk_hw *hw)
+static int rockchip_usb2phy_clk480m_prepared(struct clk_hw *hw)
 {
 	struct rockchip_usb2phy *rphy =
 		container_of(hw, struct rockchip_usb2phy, clk480m_hw);
@@ -197,9 +298,9 @@ rockchip_usb2phy_clk480m_recalc_rate(struct clk_hw *hw,
 }
 
 static const struct clk_ops rockchip_usb2phy_clkout_ops = {
-	.enable = rockchip_usb2phy_clk480m_enable,
-	.disable = rockchip_usb2phy_clk480m_disable,
-	.is_enabled = rockchip_usb2phy_clk480m_enabled,
+	.prepare = rockchip_usb2phy_clk480m_prepare,
+	.unprepare = rockchip_usb2phy_clk480m_unprepare,
+	.is_prepared = rockchip_usb2phy_clk480m_prepared,
 	.recalc_rate = rockchip_usb2phy_clk480m_recalc_rate,
 };
 
@@ -263,33 +364,84 @@ err_ret:
 	return ret;
 }
 
-static int rockchip_usb2phy_init(struct phy *phy)
+static int rockchip_usb2phy_extcon_register(struct rockchip_usb2phy *rphy)
 {
-	struct rockchip_usb2phy_port *rport = phy_get_drvdata(phy);
-	struct rockchip_usb2phy *rphy = dev_get_drvdata(phy->dev.parent);
 	int ret;
+	struct device_node *node = rphy->dev->of_node;
+	struct extcon_dev *edev;
+
+	if (of_property_read_bool(node, "extcon")) {
+		edev = extcon_get_edev_by_phandle(rphy->dev, 0);
+		if (IS_ERR(edev)) {
+			if (PTR_ERR(edev) != -EPROBE_DEFER)
+				dev_err(rphy->dev, "Invalid or missing extcon\n");
+			return PTR_ERR(edev);
+		}
+	} else {
+		/* Initialize extcon device */
+		edev = devm_extcon_dev_allocate(rphy->dev,
+						rockchip_usb2phy_extcon_cable);
 
-	if (rport->port_id == USB2PHY_PORT_HOST) {
-		/* clear linestate and enable linestate detect irq */
-		mutex_lock(&rport->mutex);
+		if (IS_ERR(edev))
+			return -ENOMEM;
 
-		ret = property_enable(rphy, &rport->port_cfg->ls_det_clr, true);
+		ret = devm_extcon_dev_register(rphy->dev, edev);
 		if (ret) {
-			mutex_unlock(&rport->mutex);
+			dev_err(rphy->dev, "failed to register extcon device\n");
 			return ret;
 		}
+	}
 
-		ret = property_enable(rphy, &rport->port_cfg->ls_det_en, true);
-		if (ret) {
-			mutex_unlock(&rport->mutex);
-			return ret;
+	rphy->edev = edev;
+
+	return 0;
+}
+
+static int rockchip_usb2phy_init(struct phy *phy)
+{
+	struct rockchip_usb2phy_port *rport = phy_get_drvdata(phy);
+	struct rockchip_usb2phy *rphy = dev_get_drvdata(phy->dev.parent);
+	int ret = 0;
+
+	mutex_lock(&rport->mutex);
+
+	if (rport->port_id == USB2PHY_PORT_OTG) {
+		if (rport->mode != USB_DR_MODE_HOST) {
+			/* clear bvalid status and enable bvalid detect irq */
+			ret = property_enable(rphy,
+					      &rport->port_cfg->bvalid_det_clr,
+					      true);
+			if (ret)
+				goto out;
+
+			ret = property_enable(rphy,
+					      &rport->port_cfg->bvalid_det_en,
+					      true);
+			if (ret)
+				goto out;
+
+			schedule_delayed_work(&rport->otg_sm_work,
+					      OTG_SCHEDULE_DELAY);
+		} else {
+			/* If OTG works in host only mode, do nothing. */
+			dev_dbg(&rport->phy->dev, "mode %d\n", rport->mode);
 		}
+	} else if (rport->port_id == USB2PHY_PORT_HOST) {
+		/* clear linestate and enable linestate detect irq */
+		ret = property_enable(rphy, &rport->port_cfg->ls_det_clr, true);
+		if (ret)
+			goto out;
+
+		ret = property_enable(rphy, &rport->port_cfg->ls_det_en, true);
+		if (ret)
+			goto out;
 
-		mutex_unlock(&rport->mutex);
 		schedule_delayed_work(&rport->sm_work, SCHEDULE_DELAY);
 	}
 
-	return 0;
+out:
+	mutex_unlock(&rport->mutex);
+	return ret;
 }
 
 static int rockchip_usb2phy_power_on(struct phy *phy)
@@ -340,7 +492,11 @@ static int rockchip_usb2phy_exit(struct phy *phy)
 {
 	struct rockchip_usb2phy_port *rport = phy_get_drvdata(phy);
 
-	if (rport->port_id == USB2PHY_PORT_HOST)
+	if (rport->port_id == USB2PHY_PORT_OTG &&
+	    rport->mode != USB_DR_MODE_HOST) {
+		cancel_delayed_work_sync(&rport->otg_sm_work);
+		cancel_delayed_work_sync(&rport->chg_work);
+	} else if (rport->port_id == USB2PHY_PORT_HOST)
 		cancel_delayed_work_sync(&rport->sm_work);
 
 	return 0;
@@ -354,6 +510,249 @@ static const struct phy_ops rockchip_usb2phy_ops = {
 	.owner		= THIS_MODULE,
 };
 
+static void rockchip_usb2phy_otg_sm_work(struct work_struct *work)
+{
+	struct rockchip_usb2phy_port *rport =
+		container_of(work, struct rockchip_usb2phy_port,
+			     otg_sm_work.work);
+	struct rockchip_usb2phy *rphy = dev_get_drvdata(rport->phy->dev.parent);
+	static unsigned int cable;
+	unsigned long delay;
+	bool vbus_attach, sch_work, notify_charger;
+
+	if (rport->utmi_avalid)
+		vbus_attach =
+			property_enabled(rphy, &rport->port_cfg->utmi_avalid);
+	else
+		vbus_attach =
+			property_enabled(rphy, &rport->port_cfg->utmi_bvalid);
+
+	sch_work = false;
+	notify_charger = false;
+	delay = OTG_SCHEDULE_DELAY;
+	dev_dbg(&rport->phy->dev, "%s otg sm work\n",
+		usb_otg_state_string(rport->state));
+
+	switch (rport->state) {
+	case OTG_STATE_UNDEFINED:
+		rport->state = OTG_STATE_B_IDLE;
+		if (!vbus_attach)
+			rockchip_usb2phy_power_off(rport->phy);
+		/* fall through */
+	case OTG_STATE_B_IDLE:
+		if (extcon_get_cable_state_(rphy->edev, EXTCON_USB_HOST) > 0) {
+			dev_dbg(&rport->phy->dev, "usb otg host connect\n");
+			rport->state = OTG_STATE_A_HOST;
+			rockchip_usb2phy_power_on(rport->phy);
+			return;
+		} else if (vbus_attach) {
+			dev_dbg(&rport->phy->dev, "vbus_attach\n");
+			switch (rphy->chg_state) {
+			case USB_CHG_STATE_UNDEFINED:
+				schedule_delayed_work(&rport->chg_work, 0);
+				return;
+			case USB_CHG_STATE_DETECTED:
+				switch (rphy->chg_type) {
+				case POWER_SUPPLY_TYPE_USB:
+					dev_dbg(&rport->phy->dev,
+						"sdp cable is connecetd\n");
+					rockchip_usb2phy_power_on(rport->phy);
+					rport->state = OTG_STATE_B_PERIPHERAL;
+					notify_charger = true;
+					sch_work = true;
+					cable = EXTCON_CHG_USB_SDP;
+					break;
+				case POWER_SUPPLY_TYPE_USB_DCP:
+					dev_dbg(&rport->phy->dev,
+						"dcp cable is connecetd\n");
+					rockchip_usb2phy_power_off(rport->phy);
+					notify_charger = true;
+					sch_work = true;
+					cable = EXTCON_CHG_USB_DCP;
+					break;
+				case POWER_SUPPLY_TYPE_USB_CDP:
+					dev_dbg(&rport->phy->dev,
+						"cdp cable is connecetd\n");
+					rockchip_usb2phy_power_on(rport->phy);
+					rport->state = OTG_STATE_B_PERIPHERAL;
+					notify_charger = true;
+					sch_work = true;
+					cable = EXTCON_CHG_USB_CDP;
+					break;
+				default:
+					break;
+				}
+				break;
+			default:
+				break;
+			}
+		} else {
+			notify_charger = true;
+			rphy->chg_state = USB_CHG_STATE_UNDEFINED;
+			rphy->chg_type = POWER_SUPPLY_TYPE_UNKNOWN;
+		}
+
+		if (rport->vbus_attached != vbus_attach) {
+			rport->vbus_attached = vbus_attach;
+
+			if (notify_charger && rphy->edev)
+				extcon_set_cable_state_(rphy->edev,
+							cable, vbus_attach);
+		}
+		break;
+	case OTG_STATE_B_PERIPHERAL:
+		if (!vbus_attach) {
+			dev_dbg(&rport->phy->dev, "usb disconnect\n");
+			rphy->chg_state = USB_CHG_STATE_UNDEFINED;
+			rphy->chg_type = POWER_SUPPLY_TYPE_UNKNOWN;
+			rport->state = OTG_STATE_B_IDLE;
+			delay = 0;
+			rockchip_usb2phy_power_off(rport->phy);
+		}
+		sch_work = true;
+		break;
+	case OTG_STATE_A_HOST:
+		if (extcon_get_cable_state_(rphy->edev, EXTCON_USB_HOST) == 0) {
+			dev_dbg(&rport->phy->dev, "usb otg host disconnect\n");
+			rport->state = OTG_STATE_B_IDLE;
+			rockchip_usb2phy_power_off(rport->phy);
+		}
+		break;
+	default:
+		break;
+	}
+
+	if (sch_work)
+		schedule_delayed_work(&rport->otg_sm_work, delay);
+}
+
+static const char *chg_to_string(enum power_supply_type chg_type)
+{
+	switch (chg_type) {
+	case POWER_SUPPLY_TYPE_USB:
+		return "USB_SDP_CHARGER";
+	case POWER_SUPPLY_TYPE_USB_DCP:
+		return "USB_DCP_CHARGER";
+	case POWER_SUPPLY_TYPE_USB_CDP:
+		return "USB_CDP_CHARGER";
+	default:
+		return "INVALID_CHARGER";
+	}
+}
+
+static void rockchip_chg_enable_dcd(struct rockchip_usb2phy *rphy,
+				    bool en)
+{
+	property_enable(rphy, &rphy->phy_cfg->chg_det.rdm_pdwn_en, en);
+	property_enable(rphy, &rphy->phy_cfg->chg_det.idp_src_en, en);
+}
+
+static void rockchip_chg_enable_primary_det(struct rockchip_usb2phy *rphy,
+					    bool en)
+{
+	property_enable(rphy, &rphy->phy_cfg->chg_det.vdp_src_en, en);
+	property_enable(rphy, &rphy->phy_cfg->chg_det.idm_sink_en, en);
+}
+
+static void rockchip_chg_enable_secondary_det(struct rockchip_usb2phy *rphy,
+					      bool en)
+{
+	property_enable(rphy, &rphy->phy_cfg->chg_det.vdm_src_en, en);
+	property_enable(rphy, &rphy->phy_cfg->chg_det.idp_sink_en, en);
+}
+
+#define CHG_DCD_POLL_TIME	(100 * HZ / 1000)
+#define CHG_DCD_MAX_RETRIES	6
+#define CHG_PRIMARY_DET_TIME	(40 * HZ / 1000)
+#define CHG_SECONDARY_DET_TIME	(40 * HZ / 1000)
+static void rockchip_chg_detect_work(struct work_struct *work)
+{
+	struct rockchip_usb2phy_port *rport =
+		container_of(work, struct rockchip_usb2phy_port, chg_work.work);
+	struct rockchip_usb2phy *rphy = dev_get_drvdata(rport->phy->dev.parent);
+	bool is_dcd, tmout, vout;
+	unsigned long delay;
+
+	dev_dbg(&rport->phy->dev, "chg detection work state = %d\n",
+		rphy->chg_state);
+	switch (rphy->chg_state) {
+	case USB_CHG_STATE_UNDEFINED:
+		if (!rport->suspended)
+			rockchip_usb2phy_power_off(rport->phy);
+		/* put the controller in non-driving mode */
+		property_enable(rphy, &rphy->phy_cfg->chg_det.opmode, false);
+		/* Start DCD processing stage 1 */
+		rockchip_chg_enable_dcd(rphy, true);
+		rphy->chg_state = USB_CHG_STATE_WAIT_FOR_DCD;
+		rphy->dcd_retries = 0;
+		delay = CHG_DCD_POLL_TIME;
+		break;
+	case USB_CHG_STATE_WAIT_FOR_DCD:
+		/* get data contact detection status */
+		is_dcd = property_enabled(rphy, &rphy->phy_cfg->chg_det.dp_det);
+		tmout = ++rphy->dcd_retries == CHG_DCD_MAX_RETRIES;
+		/* stage 2 */
+		if (is_dcd || tmout) {
+			/* stage 4 */
+			/* Turn off DCD circuitry */
+			rockchip_chg_enable_dcd(rphy, false);
+			/* Voltage Source on DP, Probe on DM */
+			rockchip_chg_enable_primary_det(rphy, true);
+			delay = CHG_PRIMARY_DET_TIME;
+			rphy->chg_state = USB_CHG_STATE_DCD_DONE;
+		} else {
+			/* stage 3 */
+			delay = CHG_DCD_POLL_TIME;
+		}
+		break;
+	case USB_CHG_STATE_DCD_DONE:
+		vout = property_enabled(rphy, &rphy->phy_cfg->chg_det.cp_det);
+		rockchip_chg_enable_primary_det(rphy, false);
+		if (vout) {
+			/* Voltage Source on DM, Probe on DP  */
+			rockchip_chg_enable_secondary_det(rphy, true);
+			delay = CHG_SECONDARY_DET_TIME;
+			rphy->chg_state = USB_CHG_STATE_PRIMARY_DONE;
+		} else {
+			if (rphy->dcd_retries == CHG_DCD_MAX_RETRIES) {
+				/* floating charger found */
+				rphy->chg_type = POWER_SUPPLY_TYPE_USB_DCP;
+				rphy->chg_state = USB_CHG_STATE_DETECTED;
+				delay = 0;
+			} else {
+				rphy->chg_type = POWER_SUPPLY_TYPE_USB;
+				rphy->chg_state = USB_CHG_STATE_DETECTED;
+				delay = 0;
+			}
+		}
+		break;
+	case USB_CHG_STATE_PRIMARY_DONE:
+		vout = property_enabled(rphy, &rphy->phy_cfg->chg_det.dcp_det);
+		/* Turn off voltage source */
+		rockchip_chg_enable_secondary_det(rphy, false);
+		if (vout)
+			rphy->chg_type = POWER_SUPPLY_TYPE_USB_DCP;
+		else
+			rphy->chg_type = POWER_SUPPLY_TYPE_USB_CDP;
+		/* fall through */
+	case USB_CHG_STATE_SECONDARY_DONE:
+		rphy->chg_state = USB_CHG_STATE_DETECTED;
+		delay = 0;
+		/* fall through */
+	case USB_CHG_STATE_DETECTED:
+		/* put the controller in normal mode */
+		property_enable(rphy, &rphy->phy_cfg->chg_det.opmode, true);
+		rockchip_usb2phy_otg_sm_work(&rport->otg_sm_work.work);
+		dev_info(&rport->phy->dev, "charger = %s\n",
+			 chg_to_string(rphy->chg_type));
+		return;
+	default:
+		return;
+	}
+
+	schedule_delayed_work(&rport->chg_work, delay);
+}
+
 /*
  * The function manage host-phy port state and suspend/resume phy port
  * to save power.
@@ -485,6 +884,26 @@ static irqreturn_t rockchip_usb2phy_linestate_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static irqreturn_t rockchip_usb2phy_bvalid_irq(int irq, void *data)
+{
+	struct rockchip_usb2phy_port *rport = data;
+	struct rockchip_usb2phy *rphy = dev_get_drvdata(rport->phy->dev.parent);
+
+	if (!property_enabled(rphy, &rport->port_cfg->bvalid_det_st))
+		return IRQ_NONE;
+
+	mutex_lock(&rport->mutex);
+
+	/* clear bvalid detect irq pending status */
+	property_enable(rphy, &rport->port_cfg->bvalid_det_clr, true);
+
+	mutex_unlock(&rport->mutex);
+
+	rockchip_usb2phy_otg_sm_work(&rport->otg_sm_work.work);
+
+	return IRQ_HANDLED;
+}
+
 static int rockchip_usb2phy_host_port_init(struct rockchip_usb2phy *rphy,
 					   struct rockchip_usb2phy_port *rport,
 					   struct device_node *child_np)
@@ -509,13 +928,86 @@ static int rockchip_usb2phy_host_port_init(struct rockchip_usb2phy *rphy,
 					IRQF_ONESHOT,
 					"rockchip_usb2phy", rport);
 	if (ret) {
-		dev_err(rphy->dev, "failed to request irq handle\n");
+		dev_err(rphy->dev, "failed to request linestate irq handle\n");
 		return ret;
 	}
 
 	return 0;
 }
 
+static int rockchip_otg_event(struct notifier_block *nb,
+			      unsigned long event, void *ptr)
+{
+	struct rockchip_usb2phy_port *rport =
+		container_of(nb, struct rockchip_usb2phy_port, event_nb);
+
+	schedule_delayed_work(&rport->otg_sm_work, OTG_SCHEDULE_DELAY);
+
+	return NOTIFY_DONE;
+}
+
+static int rockchip_usb2phy_otg_port_init(struct rockchip_usb2phy *rphy,
+					  struct rockchip_usb2phy_port *rport,
+					  struct device_node *child_np)
+{
+	int ret;
+
+	rport->port_id = USB2PHY_PORT_OTG;
+	rport->port_cfg = &rphy->phy_cfg->port_cfgs[USB2PHY_PORT_OTG];
+	rport->state = OTG_STATE_UNDEFINED;
+
+	/*
+	 * set suspended flag to true, but actually don't
+	 * put phy in suspend mode, it aims to enable usb
+	 * phy and clock in power_on() called by usb controller
+	 * driver during probe.
+	 */
+	rport->suspended = true;
+	rport->vbus_attached = false;
+
+	mutex_init(&rport->mutex);
+
+	rport->mode = of_usb_get_dr_mode_by_phy(child_np, -1);
+	if (rport->mode == USB_DR_MODE_HOST) {
+		ret = 0;
+		goto out;
+	}
+
+	INIT_DELAYED_WORK(&rport->chg_work, rockchip_chg_detect_work);
+	INIT_DELAYED_WORK(&rport->otg_sm_work, rockchip_usb2phy_otg_sm_work);
+
+	rport->utmi_avalid =
+		of_property_read_bool(child_np, "rockchip,utmi-avalid");
+
+	rport->bvalid_irq = of_irq_get_byname(child_np, "otg-bvalid");
+	if (rport->bvalid_irq < 0) {
+		dev_err(rphy->dev, "no vbus valid irq provided\n");
+		ret = rport->bvalid_irq;
+		goto out;
+	}
+
+	ret = devm_request_threaded_irq(rphy->dev, rport->bvalid_irq, NULL,
+					rockchip_usb2phy_bvalid_irq,
+					IRQF_ONESHOT,
+					"rockchip_usb2phy_bvalid", rport);
+	if (ret) {
+		dev_err(rphy->dev, "failed to request otg-bvalid irq handle\n");
+		goto out;
+	}
+
+	if (!IS_ERR(rphy->edev)) {
+		rport->event_nb.notifier_call = rockchip_otg_event;
+
+		ret = extcon_register_notifier(rphy->edev, EXTCON_USB_HOST,
+					       &rport->event_nb);
+		if (ret)
+			dev_err(rphy->dev, "register USB HOST notifier failed\n");
+	}
+
+out:
+	return ret;
+}
+
 static int rockchip_usb2phy_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -553,8 +1045,14 @@ static int rockchip_usb2phy_probe(struct platform_device *pdev)
 
 	rphy->dev = dev;
 	phy_cfgs = match->data;
+	rphy->chg_state = USB_CHG_STATE_UNDEFINED;
+	rphy->chg_type = POWER_SUPPLY_TYPE_UNKNOWN;
 	platform_set_drvdata(pdev, rphy);
 
+	ret = rockchip_usb2phy_extcon_register(rphy);
+	if (ret)
+		return ret;
+
 	/* find out a proper config which can be matched with dt. */
 	index = 0;
 	while (phy_cfgs[index].reg) {
@@ -591,13 +1089,9 @@ static int rockchip_usb2phy_probe(struct platform_device *pdev)
 		struct rockchip_usb2phy_port *rport = &rphy->ports[index];
 		struct phy *phy;
 
-		/*
-		 * This driver aim to support both otg-port and host-port,
-		 * but unfortunately, the otg part is not ready in current,
-		 * so this comments and below codes are interim, which should
-		 * be changed after otg-port is supplied soon.
-		 */
-		if (of_node_cmp(child_np->name, "host-port"))
+		/* This driver aims to support both otg-port and host-port */
+		if (of_node_cmp(child_np->name, "host-port") &&
+		    of_node_cmp(child_np->name, "otg-port"))
 			goto next_child;
 
 		phy = devm_phy_create(dev, child_np, &rockchip_usb2phy_ops);
@@ -610,9 +1104,18 @@ static int rockchip_usb2phy_probe(struct platform_device *pdev)
 		rport->phy = phy;
 		phy_set_drvdata(rport->phy, rport);
 
-		ret = rockchip_usb2phy_host_port_init(rphy, rport, child_np);
-		if (ret)
-			goto put_child;
+		/* initialize otg/host port separately */
+		if (!of_node_cmp(child_np->name, "host-port")) {
+			ret = rockchip_usb2phy_host_port_init(rphy, rport,
+							      child_np);
+			if (ret)
+				goto put_child;
+		} else {
+			ret = rockchip_usb2phy_otg_port_init(rphy, rport,
+							     child_np);
+			if (ret)
+				goto put_child;
+		}
 
 next_child:
 		/* to prevent out of boundary */
@@ -654,10 +1157,18 @@ static const struct rockchip_usb2phy_cfg rk3366_phy_cfgs[] = {
 
 static const struct rockchip_usb2phy_cfg rk3399_phy_cfgs[] = {
 	{
-		.reg = 0xe450,
+		.reg		= 0xe450,
 		.num_ports	= 2,
 		.clkout_ctl	= { 0xe450, 4, 4, 1, 0 },
 		.port_cfgs	= {
+			[USB2PHY_PORT_OTG] = {
+				.phy_sus	= { 0xe454, 1, 0, 2, 1 },
+				.bvalid_det_en	= { 0xe3c0, 3, 3, 0, 1 },
+				.bvalid_det_st	= { 0xe3e0, 3, 3, 0, 1 },
+				.bvalid_det_clr	= { 0xe3d0, 3, 3, 0, 1 },
+				.utmi_avalid	= { 0xe2ac, 7, 7, 0, 1 },
+				.utmi_bvalid	= { 0xe2ac, 12, 12, 0, 1 },
+			},
 			[USB2PHY_PORT_HOST] = {
 				.phy_sus	= { 0xe458, 1, 0, 0x2, 0x1 },
 				.ls_det_en	= { 0xe3c0, 6, 6, 0, 1 },
@@ -667,12 +1178,32 @@ static const struct rockchip_usb2phy_cfg rk3399_phy_cfgs[] = {
 				.utmi_hstdet	= { 0xe2ac, 23, 23, 0, 1 }
 			}
 		},
+		.chg_det = {
+			.opmode		= { 0xe454, 3, 0, 5, 1 },
+			.cp_det		= { 0xe2ac, 2, 2, 0, 1 },
+			.dcp_det	= { 0xe2ac, 1, 1, 0, 1 },
+			.dp_det		= { 0xe2ac, 0, 0, 0, 1 },
+			.idm_sink_en	= { 0xe450, 8, 8, 0, 1 },
+			.idp_sink_en	= { 0xe450, 7, 7, 0, 1 },
+			.idp_src_en	= { 0xe450, 9, 9, 0, 1 },
+			.rdm_pdwn_en	= { 0xe450, 10, 10, 0, 1 },
+			.vdm_src_en	= { 0xe450, 12, 12, 0, 1 },
+			.vdp_src_en	= { 0xe450, 11, 11, 0, 1 },
+		},
 	},
 	{
-		.reg = 0xe460,
+		.reg		= 0xe460,
 		.num_ports	= 2,
 		.clkout_ctl	= { 0xe460, 4, 4, 1, 0 },
 		.port_cfgs	= {
+			[USB2PHY_PORT_OTG] = {
+				.phy_sus        = { 0xe464, 1, 0, 2, 1 },
+				.bvalid_det_en  = { 0xe3c0, 8, 8, 0, 1 },
+				.bvalid_det_st  = { 0xe3e0, 8, 8, 0, 1 },
+				.bvalid_det_clr = { 0xe3d0, 8, 8, 0, 1 },
+				.utmi_avalid	= { 0xe2ac, 10, 10, 0, 1 },
+				.utmi_bvalid    = { 0xe2ac, 16, 16, 0, 1 },
+			},
 			[USB2PHY_PORT_HOST] = {
 				.phy_sus	= { 0xe468, 1, 0, 0x2, 0x1 },
 				.ls_det_en	= { 0xe3c0, 11, 11, 0, 1 },
diff --git a/drivers/phy/phy-s5pv210-usb2.c b/drivers/phy/phy-s5pv210-usb2.c
index 004d320767e4..f6f72339bbc3 100644
--- a/drivers/phy/phy-s5pv210-usb2.c
+++ b/drivers/phy/phy-s5pv210-usb2.c
@@ -103,7 +103,7 @@ static void s5pv210_isol(struct samsung_usb2_phy_instance *inst, bool on)
 		break;
 	default:
 		return;
-	};
+	}
 
 	regmap_update_bits(drv->reg_pmu, S5PV210_USB_ISOL_OFFSET,
 							mask, on ? 0 : mask);
@@ -127,7 +127,7 @@ static void s5pv210_phy_pwr(struct samsung_usb2_phy_instance *inst, bool on)
 		rstbits =	S5PV210_URSTCON_PHY1_ALL |
 				S5PV210_URSTCON_HOST_LINK_ALL;
 		break;
-	};
+	}
 
 	if (on) {
 		writel(drv->ref_reg_val, drv->reg_phy + S5PV210_UPHYCLK);
diff --git a/drivers/phy/phy-stih41x-usb.c b/drivers/phy/phy-stih41x-usb.c
deleted file mode 100644
index 0ac74639ad02..000000000000
--- a/drivers/phy/phy-stih41x-usb.c
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Copyright (C) 2014 STMicroelectronics
- *
- * STMicroelectronics PHY driver for STiH41x USB.
- *
- * Author: Maxime Coquelin <maxime.coquelin@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2, as
- * published by the Free Software Foundation.
- *
- */
-
-#include <linux/platform_device.h>
-#include <linux/io.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_platform.h>
-#include <linux/clk.h>
-#include <linux/phy/phy.h>
-#include <linux/regmap.h>
-#include <linux/mfd/syscon.h>
-
-#define SYSCFG332  0x80
-#define SYSCFG2520 0x820
-
-/**
- * struct stih41x_usb_cfg - SoC specific PHY register mapping
- * @syscfg: Offset in syscfg registers bank
- * @cfg_mask: Bits mask for PHY configuration
- * @cfg: Static configuration value for PHY
- * @oscok: Notify the PHY oscillator clock is ready
- *	   Setting this bit enable the PHY
- */
-struct stih41x_usb_cfg {
-	u32 syscfg;
-	u32 cfg_mask;
-	u32 cfg;
-	u32 oscok;
-};
-
-/**
- * struct stih41x_usb_phy - Private data for the PHY
- * @dev: device for this controller
- * @regmap: Syscfg registers bank in which PHY is configured
- * @cfg: SoC specific PHY register mapping
- * @clk: Oscillator used by the PHY
- */
-struct stih41x_usb_phy {
-	struct device *dev;
-	struct regmap *regmap;
-	const struct stih41x_usb_cfg *cfg;
-	struct clk *clk;
-};
-
-static struct stih41x_usb_cfg stih415_usb_phy_cfg = {
-	.syscfg = SYSCFG332,
-	.cfg_mask = 0x3f,
-	.cfg = 0x38,
-	.oscok = BIT(6),
-};
-
-static struct stih41x_usb_cfg stih416_usb_phy_cfg = {
-	.syscfg = SYSCFG2520,
-	.cfg_mask = 0x33f,
-	.cfg = 0x238,
-	.oscok = BIT(6),
-};
-
-static int stih41x_usb_phy_init(struct phy *phy)
-{
-	struct stih41x_usb_phy *phy_dev = phy_get_drvdata(phy);
-
-	return regmap_update_bits(phy_dev->regmap, phy_dev->cfg->syscfg,
-			   phy_dev->cfg->cfg_mask, phy_dev->cfg->cfg);
-}
-
-static int stih41x_usb_phy_power_on(struct phy *phy)
-{
-	struct stih41x_usb_phy *phy_dev = phy_get_drvdata(phy);
-	int ret;
-
-	ret = clk_prepare_enable(phy_dev->clk);
-	if (ret) {
-		dev_err(phy_dev->dev, "Failed to enable osc_phy clock\n");
-		return ret;
-	}
-
-	ret = regmap_update_bits(phy_dev->regmap, phy_dev->cfg->syscfg,
-				 phy_dev->cfg->oscok, phy_dev->cfg->oscok);
-	if (ret)
-		clk_disable_unprepare(phy_dev->clk);
-
-	return ret;
-}
-
-static int stih41x_usb_phy_power_off(struct phy *phy)
-{
-	struct stih41x_usb_phy *phy_dev = phy_get_drvdata(phy);
-	int ret;
-
-	ret = regmap_update_bits(phy_dev->regmap, phy_dev->cfg->syscfg,
-			phy_dev->cfg->oscok, 0);
-	if (ret) {
-		dev_err(phy_dev->dev, "Failed to clear oscok bit\n");
-		return ret;
-	}
-
-	clk_disable_unprepare(phy_dev->clk);
-
-	return 0;
-}
-
-static const struct phy_ops stih41x_usb_phy_ops = {
-	.init		= stih41x_usb_phy_init,
-	.power_on	= stih41x_usb_phy_power_on,
-	.power_off	= stih41x_usb_phy_power_off,
-	.owner		= THIS_MODULE,
-};
-
-static const struct of_device_id stih41x_usb_phy_of_match[];
-
-static int stih41x_usb_phy_probe(struct platform_device *pdev)
-{
-	struct device_node *np = pdev->dev.of_node;
-	const struct of_device_id *match;
-	struct stih41x_usb_phy *phy_dev;
-	struct device *dev = &pdev->dev;
-	struct phy_provider *phy_provider;
-	struct phy *phy;
-
-	phy_dev = devm_kzalloc(dev, sizeof(*phy_dev), GFP_KERNEL);
-	if (!phy_dev)
-		return -ENOMEM;
-
-	match = of_match_device(stih41x_usb_phy_of_match, &pdev->dev);
-	if (!match)
-		return -ENODEV;
-
-	phy_dev->cfg = match->data;
-
-	phy_dev->regmap = syscon_regmap_lookup_by_phandle(np, "st,syscfg");
-	if (IS_ERR(phy_dev->regmap)) {
-		dev_err(dev, "No syscfg phandle specified\n");
-		return PTR_ERR(phy_dev->regmap);
-	}
-
-	phy_dev->clk = devm_clk_get(dev, "osc_phy");
-	if (IS_ERR(phy_dev->clk)) {
-		dev_err(dev, "osc_phy clk not found\n");
-		return PTR_ERR(phy_dev->clk);
-	}
-
-	phy = devm_phy_create(dev, NULL, &stih41x_usb_phy_ops);
-
-	if (IS_ERR(phy)) {
-		dev_err(dev, "failed to create phy\n");
-		return PTR_ERR(phy);
-	}
-
-	phy_dev->dev = dev;
-
-	phy_set_drvdata(phy, phy_dev);
-
-	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
-	return PTR_ERR_OR_ZERO(phy_provider);
-}
-
-static const struct of_device_id stih41x_usb_phy_of_match[] = {
-	{ .compatible = "st,stih415-usb-phy", .data = &stih415_usb_phy_cfg },
-	{ .compatible = "st,stih416-usb-phy", .data = &stih416_usb_phy_cfg },
-	{ /* sentinel */ },
-};
-MODULE_DEVICE_TABLE(of, stih41x_usb_phy_of_match);
-
-static struct platform_driver stih41x_usb_phy_driver = {
-	.probe	= stih41x_usb_phy_probe,
-	.driver = {
-		.name	= "stih41x-usb-phy",
-		.of_match_table	= stih41x_usb_phy_of_match,
-	}
-};
-module_platform_driver(stih41x_usb_phy_driver);
-
-MODULE_AUTHOR("Maxime Coquelin <maxime.coquelin@st.com>");
-MODULE_DESCRIPTION("STMicroelectronics USB PHY driver for STiH41x series");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/phy-sun4i-usb.c b/drivers/phy/phy-sun4i-usb.c
index fec34f5213c4..bf28a0fdd569 100644
--- a/drivers/phy/phy-sun4i-usb.c
+++ b/drivers/phy/phy-sun4i-usb.c
@@ -436,25 +436,31 @@ static int sun4i_usb_phy_set_mode(struct phy *_phy, enum phy_mode mode)
 {
 	struct sun4i_usb_phy *phy = phy_get_drvdata(_phy);
 	struct sun4i_usb_phy_data *data = to_sun4i_usb_phy_data(phy);
+	int new_mode;
 
 	if (phy->index != 0)
 		return -EINVAL;
 
 	switch (mode) {
 	case PHY_MODE_USB_HOST:
-		data->dr_mode = USB_DR_MODE_HOST;
+		new_mode = USB_DR_MODE_HOST;
 		break;
 	case PHY_MODE_USB_DEVICE:
-		data->dr_mode = USB_DR_MODE_PERIPHERAL;
+		new_mode = USB_DR_MODE_PERIPHERAL;
 		break;
 	case PHY_MODE_USB_OTG:
-		data->dr_mode = USB_DR_MODE_OTG;
+		new_mode = USB_DR_MODE_OTG;
 		break;
 	default:
 		return -EINVAL;
 	}
 
-	dev_info(&_phy->dev, "Changing dr_mode to %d\n", (int)data->dr_mode);
+	if (new_mode != data->dr_mode) {
+		dev_info(&_phy->dev, "Changing dr_mode to %d\n", new_mode);
+		data->dr_mode = new_mode;
+	}
+
+	data->id_det = -1; /* Force reprocessing of id */
 	data->force_session_end = true;
 	queue_delayed_work(system_wq, &data->detect, 0);
 
diff --git a/drivers/phy/phy-ti-pipe3.c b/drivers/phy/phy-ti-pipe3.c
index bf46844dc387..9c84d32c6f60 100644
--- a/drivers/phy/phy-ti-pipe3.c
+++ b/drivers/phy/phy-ti-pipe3.c
@@ -537,10 +537,7 @@ static int ti_pipe3_get_pll_base(struct ti_pipe3 *phy)
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 					   "pll_ctrl");
 	phy->pll_ctrl_base = devm_ioremap_resource(dev, res);
-	if (IS_ERR(phy->pll_ctrl_base))
-		return PTR_ERR(phy->pll_ctrl_base);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(phy->pll_ctrl_base);
 }
 
 static int ti_pipe3_probe(struct platform_device *pdev)
@@ -592,10 +589,7 @@ static int ti_pipe3_probe(struct platform_device *pdev)
 	ti_pipe3_power_off(generic_phy);
 
 	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
-	if (IS_ERR(phy_provider))
-		return PTR_ERR(phy_provider);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(phy_provider);
 }
 
 static int ti_pipe3_remove(struct platform_device *pdev)
diff --git a/drivers/phy/phy-twl4030-usb.c b/drivers/phy/phy-twl4030-usb.c
index 547ca7b3f098..2990b3965460 100644
--- a/drivers/phy/phy-twl4030-usb.c
+++ b/drivers/phy/phy-twl4030-usb.c
@@ -317,6 +317,9 @@ static enum musb_vbus_id_status
 			linkstat = MUSB_VBUS_OFF;
 	}
 
+	kobject_uevent(&twl->dev->kobj, linkstat == MUSB_VBUS_VALID
+					? KOBJ_ONLINE : KOBJ_OFFLINE);
+
 	dev_dbg(twl->dev, "HW_CONDITIONS 0x%02x/%d; link %d\n",
 			status, status, linkstat);
 
diff --git a/drivers/phy/tegra/xusb-tegra124.c b/drivers/phy/tegra/xusb-tegra124.c
index 119957249a51..c45cbedc6634 100644
--- a/drivers/phy/tegra/xusb-tegra124.c
+++ b/drivers/phy/tegra/xusb-tegra124.c
@@ -1483,7 +1483,6 @@ static int tegra124_usb3_port_enable(struct tegra_xusb_port *port)
 	struct tegra_xusb_padctl *padctl = port->padctl;
 	struct tegra_xusb_lane *lane = usb3->base.lane;
 	unsigned int index = port->index, offset;
-	int ret = 0;
 	u32 value;
 
 	value = padctl_readl(padctl, XUSB_PADCTL_SS_PORT_MAP);
@@ -1612,7 +1611,7 @@ static int tegra124_usb3_port_enable(struct tegra_xusb_port *port)
 	value &= ~XUSB_PADCTL_ELPG_PROGRAM_SSPX_ELPG_CLAMP_EN(index);
 	padctl_writel(padctl, value, XUSB_PADCTL_ELPG_PROGRAM);
 
-	return ret;
+	return 0;
 }
 
 static void tegra124_usb3_port_disable(struct tegra_xusb_port *port)
diff --git a/drivers/phy/tegra/xusb.c b/drivers/phy/tegra/xusb.c
index 873424ab0e32..3cbcb2537657 100644
--- a/drivers/phy/tegra/xusb.c
+++ b/drivers/phy/tegra/xusb.c
@@ -561,10 +561,7 @@ static int tegra_xusb_usb2_port_parse_dt(struct tegra_xusb_usb2_port *usb2)
 	usb2->internal = of_property_read_bool(np, "nvidia,internal");
 
 	usb2->supply = devm_regulator_get(&port->dev, "vbus");
-	if (IS_ERR(usb2->supply))
-		return PTR_ERR(usb2->supply);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(usb2->supply);
 }
 
 static int tegra_xusb_add_usb2_port(struct tegra_xusb_padctl *padctl,
@@ -731,10 +728,7 @@ static int tegra_xusb_usb3_port_parse_dt(struct tegra_xusb_usb3_port *usb3)
 	usb3->internal = of_property_read_bool(np, "nvidia,internal");
 
 	usb3->supply = devm_regulator_get(&port->dev, "vbus");
-	if (IS_ERR(usb3->supply))
-		return PTR_ERR(usb3->supply);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(usb3->supply);
 }
 
 static int tegra_xusb_add_usb3_port(struct tegra_xusb_padctl *padctl,
diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig
index 0e75d94972ba..54044a8ecbd7 100644
--- a/drivers/pinctrl/Kconfig
+++ b/drivers/pinctrl/Kconfig
@@ -93,6 +93,15 @@ config PINCTRL_AMD
 	  Requires ACPI/FDT device enumeration code to set up a platform
 	  device.
 
+config PINCTRL_DA850_PUPD
+	tristate "TI DA850/OMAP-L138/AM18XX pullup/pulldown groups"
+	depends on OF && (ARCH_DAVINCI_DA850 || COMPILE_TEST)
+	select PINCONF
+	select GENERIC_PINCONF
+	help
+	  Driver for TI DA850/OMAP-L138/AM18XX pinconf. Used to control
+	  pullup/pulldown pin groups.
+
 config PINCTRL_DIGICOLOR
 	bool
 	depends on OF && (ARCH_DIGICOLOR || COMPILE_TEST)
@@ -164,6 +173,21 @@ config PINCTRL_SIRF
 	select GENERIC_PINCONF
 	select GPIOLIB_IRQCHIP
 
+config PINCTRL_SX150X
+	bool "Semtech SX150x I2C GPIO expander pinctrl driver"
+	depends on GPIOLIB && I2C=y
+	select PINMUX
+	select PINCONF
+	select GENERIC_PINCONF
+	select GPIOLIB_IRQCHIP
+	select REGMAP
+	help
+	  Say yes here to provide support for Semtech SX150x-series I2C
+	  GPIO expanders as pinctrl module.
+	  Compatible models include:
+	  - 8 bits:  sx1508q, sx1502q
+	  - 16 bits: sx1509q, sx1506q
+
 config PINCTRL_PISTACHIO
 	def_bool y if MACH_PISTACHIO
 	depends on GPIOLIB
@@ -209,7 +233,7 @@ config PINCTRL_COH901
 
 config PINCTRL_MAX77620
 	tristate "MAX77620/MAX20024 Pincontrol support"
-	depends on MFD_MAX77620
+	depends on MFD_MAX77620 && OF
 	select PINMUX
 	select GENERIC_PINCONF
 	help
diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile
index 11bad373dfe0..25d50a86981d 100644
--- a/drivers/pinctrl/Makefile
+++ b/drivers/pinctrl/Makefile
@@ -14,6 +14,7 @@ obj-$(CONFIG_PINCTRL_BF60x)	+= pinctrl-adi2-bf60x.o
 obj-$(CONFIG_PINCTRL_AT91)	+= pinctrl-at91.o
 obj-$(CONFIG_PINCTRL_AT91PIO4)	+= pinctrl-at91-pio4.o
 obj-$(CONFIG_PINCTRL_AMD)	+= pinctrl-amd.o
+obj-$(CONFIG_PINCTRL_DA850_PUPD) += pinctrl-da850-pupd.o
 obj-$(CONFIG_PINCTRL_DIGICOLOR)	+= pinctrl-digicolor.o
 obj-$(CONFIG_PINCTRL_FALCON)	+= pinctrl-falcon.o
 obj-$(CONFIG_PINCTRL_MAX77620)	+= pinctrl-max77620.o
@@ -25,6 +26,7 @@ obj-$(CONFIG_PINCTRL_PISTACHIO)	+= pinctrl-pistachio.o
 obj-$(CONFIG_PINCTRL_ROCKCHIP)	+= pinctrl-rockchip.o
 obj-$(CONFIG_PINCTRL_SINGLE)	+= pinctrl-single.o
 obj-$(CONFIG_PINCTRL_SIRF)	+= sirf/
+obj-$(CONFIG_PINCTRL_SX150X)	+= pinctrl-sx150x.o
 obj-$(CONFIG_ARCH_TEGRA)	+= tegra/
 obj-$(CONFIG_PINCTRL_TZ1090)	+= pinctrl-tz1090.o
 obj-$(CONFIG_PINCTRL_TZ1090_PDC)	+= pinctrl-tz1090-pdc.o
diff --git a/drivers/pinctrl/bcm/Kconfig b/drivers/pinctrl/bcm/Kconfig
index 63246770bd74..8968dd7aebed 100644
--- a/drivers/pinctrl/bcm/Kconfig
+++ b/drivers/pinctrl/bcm/Kconfig
@@ -20,6 +20,7 @@ config PINCTRL_BCM2835
 	bool
 	select PINMUX
 	select PINCONF
+	select GPIOLIB_IRQCHIP
 
 config PINCTRL_IPROC_GPIO
 	bool "Broadcom iProc GPIO (with PINCONF) driver"
diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
index fa77165fab2c..85d009112864 100644
--- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c
+++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
@@ -24,11 +24,9 @@
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/gpio/driver.h>
-#include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/irq.h>
 #include <linux/irqdesc.h>
-#include <linux/irqdomain.h>
 #include <linux/module.h>
 #include <linux/of_address.h>
 #include <linux/of.h>
@@ -47,6 +45,7 @@
 #define MODULE_NAME "pinctrl-bcm2835"
 #define BCM2835_NUM_GPIOS 54
 #define BCM2835_NUM_BANKS 2
+#define BCM2835_NUM_IRQS  3
 
 #define BCM2835_PIN_BITMAP_SZ \
 	DIV_ROUND_UP(BCM2835_NUM_GPIOS, sizeof(unsigned long) * 8)
@@ -76,41 +75,27 @@ enum bcm2835_pinconf_param {
 	BCM2835_PINCONF_PARAM_PULL,
 };
 
-enum bcm2835_pinconf_pull {
-	BCM2835_PINCONFIG_PULL_NONE,
-	BCM2835_PINCONFIG_PULL_DOWN,
-	BCM2835_PINCONFIG_PULL_UP,
-};
-
 #define BCM2835_PINCONF_PACK(_param_, _arg_) ((_param_) << 16 | (_arg_))
 #define BCM2835_PINCONF_UNPACK_PARAM(_conf_) ((_conf_) >> 16)
 #define BCM2835_PINCONF_UNPACK_ARG(_conf_) ((_conf_) & 0xffff)
 
-struct bcm2835_gpio_irqdata {
-	struct bcm2835_pinctrl *pc;
-	int bank;
-};
-
 struct bcm2835_pinctrl {
 	struct device *dev;
 	void __iomem *base;
-	int irq[BCM2835_NUM_BANKS];
+	int irq[BCM2835_NUM_IRQS];
 
 	/* note: locking assumes each bank will have its own unsigned long */
 	unsigned long enabled_irq_map[BCM2835_NUM_BANKS];
 	unsigned int irq_type[BCM2835_NUM_GPIOS];
 
 	struct pinctrl_dev *pctl_dev;
-	struct irq_domain *irq_domain;
 	struct gpio_chip gpio_chip;
 	struct pinctrl_gpio_range gpio_range;
 
-	struct bcm2835_gpio_irqdata irq_data[BCM2835_NUM_BANKS];
+	int irq_group[BCM2835_NUM_IRQS];
 	spinlock_t irq_lock[BCM2835_NUM_BANKS];
 };
 
-static struct lock_class_key gpio_lock_class;
-
 /* pins are just named GPIO0..GPIO53 */
 #define BCM2835_GPIO_PIN(a) PINCTRL_PIN(a, "gpio" #a)
 static struct pinctrl_pin_desc bcm2835_gpio_pins[] = {
@@ -368,13 +353,6 @@ static int bcm2835_gpio_direction_output(struct gpio_chip *chip,
 	return pinctrl_gpio_direction_output(chip->base + offset);
 }
 
-static int bcm2835_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
-{
-	struct bcm2835_pinctrl *pc = gpiochip_get_data(chip);
-
-	return irq_linear_revmap(pc->irq_domain, offset);
-}
-
 static struct gpio_chip bcm2835_gpio_chip = {
 	.label = MODULE_NAME,
 	.owner = THIS_MODULE,
@@ -385,31 +363,67 @@ static struct gpio_chip bcm2835_gpio_chip = {
 	.get_direction = bcm2835_gpio_get_direction,
 	.get = bcm2835_gpio_get,
 	.set = bcm2835_gpio_set,
-	.to_irq = bcm2835_gpio_to_irq,
 	.base = -1,
 	.ngpio = BCM2835_NUM_GPIOS,
 	.can_sleep = false,
 };
 
-static irqreturn_t bcm2835_gpio_irq_handler(int irq, void *dev_id)
+static void bcm2835_gpio_irq_handle_bank(struct bcm2835_pinctrl *pc,
+					 unsigned int bank, u32 mask)
 {
-	struct bcm2835_gpio_irqdata *irqdata = dev_id;
-	struct bcm2835_pinctrl *pc = irqdata->pc;
-	int bank = irqdata->bank;
 	unsigned long events;
 	unsigned offset;
 	unsigned gpio;
 	unsigned int type;
 
 	events = bcm2835_gpio_rd(pc, GPEDS0 + bank * 4);
+	events &= mask;
 	events &= pc->enabled_irq_map[bank];
 	for_each_set_bit(offset, &events, 32) {
 		gpio = (32 * bank) + offset;
+		/* FIXME: no clue why the code looks up the type here */
 		type = pc->irq_type[gpio];
 
-		generic_handle_irq(irq_linear_revmap(pc->irq_domain, gpio));
+		generic_handle_irq(irq_linear_revmap(pc->gpio_chip.irqdomain,
+						     gpio));
+	}
+}
+
+static void bcm2835_gpio_irq_handler(struct irq_desc *desc)
+{
+	struct gpio_chip *chip = irq_desc_get_handler_data(desc);
+	struct bcm2835_pinctrl *pc = gpiochip_get_data(chip);
+	struct irq_chip *host_chip = irq_desc_get_chip(desc);
+	int irq = irq_desc_get_irq(desc);
+	int group;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(pc->irq); i++) {
+		if (pc->irq[i] == irq) {
+			group = pc->irq_group[i];
+			break;
+		}
+	}
+	/* This should not happen, every IRQ has a bank */
+	if (i == ARRAY_SIZE(pc->irq))
+		BUG();
+
+	chained_irq_enter(host_chip, desc);
+
+	switch (group) {
+	case 0: /* IRQ0 covers GPIOs 0-27 */
+		bcm2835_gpio_irq_handle_bank(pc, 0, 0x0fffffff);
+		break;
+	case 1: /* IRQ1 covers GPIOs 28-45 */
+		bcm2835_gpio_irq_handle_bank(pc, 0, 0xf0000000);
+		bcm2835_gpio_irq_handle_bank(pc, 1, 0x00003fff);
+		break;
+	case 2: /* IRQ2 covers GPIOs 46-53 */
+		bcm2835_gpio_irq_handle_bank(pc, 1, 0x003fc000);
+		break;
 	}
-	return events ? IRQ_HANDLED : IRQ_NONE;
+
+	chained_irq_exit(host_chip, desc);
 }
 
 static inline void __bcm2835_gpio_irq_config(struct bcm2835_pinctrl *pc,
@@ -455,7 +469,8 @@ static void bcm2835_gpio_irq_config(struct bcm2835_pinctrl *pc,
 
 static void bcm2835_gpio_irq_enable(struct irq_data *data)
 {
-	struct bcm2835_pinctrl *pc = irq_data_get_irq_chip_data(data);
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct bcm2835_pinctrl *pc = gpiochip_get_data(chip);
 	unsigned gpio = irqd_to_hwirq(data);
 	unsigned offset = GPIO_REG_SHIFT(gpio);
 	unsigned bank = GPIO_REG_OFFSET(gpio);
@@ -469,7 +484,8 @@ static void bcm2835_gpio_irq_enable(struct irq_data *data)
 
 static void bcm2835_gpio_irq_disable(struct irq_data *data)
 {
-	struct bcm2835_pinctrl *pc = irq_data_get_irq_chip_data(data);
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct bcm2835_pinctrl *pc = gpiochip_get_data(chip);
 	unsigned gpio = irqd_to_hwirq(data);
 	unsigned offset = GPIO_REG_SHIFT(gpio);
 	unsigned bank = GPIO_REG_OFFSET(gpio);
@@ -575,7 +591,8 @@ static int __bcm2835_gpio_irq_set_type_enabled(struct bcm2835_pinctrl *pc,
 
 static int bcm2835_gpio_irq_set_type(struct irq_data *data, unsigned int type)
 {
-	struct bcm2835_pinctrl *pc = irq_data_get_irq_chip_data(data);
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct bcm2835_pinctrl *pc = gpiochip_get_data(chip);
 	unsigned gpio = irqd_to_hwirq(data);
 	unsigned offset = GPIO_REG_SHIFT(gpio);
 	unsigned bank = GPIO_REG_OFFSET(gpio);
@@ -601,7 +618,8 @@ static int bcm2835_gpio_irq_set_type(struct irq_data *data, unsigned int type)
 
 static void bcm2835_gpio_irq_ack(struct irq_data *data)
 {
-	struct bcm2835_pinctrl *pc = irq_data_get_irq_chip_data(data);
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct bcm2835_pinctrl *pc = gpiochip_get_data(chip);
 	unsigned gpio = irqd_to_hwirq(data);
 
 	bcm2835_gpio_set_bit(pc, GPEDS0, gpio);
@@ -644,10 +662,11 @@ static void bcm2835_pctl_pin_dbg_show(struct pinctrl_dev *pctldev,
 		unsigned offset)
 {
 	struct bcm2835_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev);
+	struct gpio_chip *chip = &pc->gpio_chip;
 	enum bcm2835_fsel fsel = bcm2835_pinctrl_fsel_get(pc, offset);
 	const char *fname = bcm2835_functions[fsel];
 	int value = bcm2835_gpio_get_bit(pc, GPLEV0, offset);
-	int irq = irq_find_mapping(pc->irq_domain, offset);
+	int irq = irq_find_mapping(chip->irqdomain, offset);
 
 	seq_printf(s, "function %s in %s; irq %d (%s)",
 		fname, value ? "hi" : "lo",
@@ -821,6 +840,16 @@ static const struct pinctrl_ops bcm2835_pctl_ops = {
 	.dt_free_map = bcm2835_pctl_dt_free_map,
 };
 
+static int bcm2835_pmx_free(struct pinctrl_dev *pctldev,
+		unsigned offset)
+{
+	struct bcm2835_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev);
+
+	/* disable by setting to GPIO_IN */
+	bcm2835_pinctrl_fsel_set(pc, offset, BCM2835_FSEL_GPIO_IN);
+	return 0;
+}
+
 static int bcm2835_pmx_get_functions_count(struct pinctrl_dev *pctldev)
 {
 	return BCM2835_FSEL_COUNT;
@@ -880,6 +909,7 @@ static int bcm2835_pmx_gpio_set_direction(struct pinctrl_dev *pctldev,
 }
 
 static const struct pinmux_ops bcm2835_pmx_ops = {
+	.free = bcm2835_pmx_free,
 	.get_functions_count = bcm2835_pmx_get_functions_count,
 	.get_function_name = bcm2835_pmx_get_function_name,
 	.get_function_groups = bcm2835_pmx_get_function_groups,
@@ -917,12 +947,14 @@ static int bcm2835_pinconf_set(struct pinctrl_dev *pctldev,
 
 		bcm2835_gpio_wr(pc, GPPUD, arg & 3);
 		/*
-		 * Docs say to wait 150 cycles, but not of what. We assume a
-		 * 1 MHz clock here, which is pretty slow...
+		 * BCM2835 datasheet say to wait 150 cycles, but not of what.
+		 * But the VideoCore firmware delay for this operation
+		 * based nearly on the same amount of VPU cycles and this clock
+		 * runs at 250 MHz.
 		 */
-		udelay(150);
+		udelay(1);
 		bcm2835_gpio_wr(pc, GPPUDCLK0 + (off * 4), BIT(bit));
-		udelay(150);
+		udelay(1);
 		bcm2835_gpio_wr(pc, GPPUDCLK0 + (off * 4), 0);
 	} /* for each config */
 
@@ -980,26 +1012,9 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
 	pc->gpio_chip.parent = dev;
 	pc->gpio_chip.of_node = np;
 
-	pc->irq_domain = irq_domain_add_linear(np, BCM2835_NUM_GPIOS,
-			&irq_domain_simple_ops, NULL);
-	if (!pc->irq_domain) {
-		dev_err(dev, "could not create IRQ domain\n");
-		return -ENOMEM;
-	}
-
-	for (i = 0; i < BCM2835_NUM_GPIOS; i++) {
-		int irq = irq_create_mapping(pc->irq_domain, i);
-		irq_set_lockdep_class(irq, &gpio_lock_class);
-		irq_set_chip_and_handler(irq, &bcm2835_gpio_irq_chip,
-				handle_level_irq);
-		irq_set_chip_data(irq, pc);
-	}
-
 	for (i = 0; i < BCM2835_NUM_BANKS; i++) {
 		unsigned long events;
 		unsigned offset;
-		int len;
-		char *name;
 
 		/* clear event detection flags */
 		bcm2835_gpio_wr(pc, GPREN0 + i * 4, 0);
@@ -1014,24 +1029,7 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
 		for_each_set_bit(offset, &events, 32)
 			bcm2835_gpio_wr(pc, GPEDS0 + i * 4, BIT(offset));
 
-		pc->irq[i] = irq_of_parse_and_map(np, i);
-		pc->irq_data[i].pc = pc;
-		pc->irq_data[i].bank = i;
 		spin_lock_init(&pc->irq_lock[i]);
-
-		len = strlen(dev_name(pc->dev)) + 16;
-		name = devm_kzalloc(pc->dev, len, GFP_KERNEL);
-		if (!name)
-			return -ENOMEM;
-		snprintf(name, len, "%s:bank%d", dev_name(pc->dev), i);
-
-		err = devm_request_irq(dev, pc->irq[i],
-			bcm2835_gpio_irq_handler, IRQF_SHARED,
-			name, &pc->irq_data[i]);
-		if (err) {
-			dev_err(dev, "unable to request IRQ %d\n", pc->irq[i]);
-			return err;
-		}
 	}
 
 	err = gpiochip_add_data(&pc->gpio_chip, pc);
@@ -1040,6 +1038,29 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
 		return err;
 	}
 
+	err = gpiochip_irqchip_add(&pc->gpio_chip, &bcm2835_gpio_irq_chip,
+				   0, handle_level_irq, IRQ_TYPE_NONE);
+	if (err) {
+		dev_info(dev, "could not add irqchip\n");
+		return err;
+	}
+
+	for (i = 0; i < BCM2835_NUM_IRQS; i++) {
+		pc->irq[i] = irq_of_parse_and_map(np, i);
+		pc->irq_group[i] = i;
+		/*
+		 * Use the same handler for all groups: this is necessary
+		 * since we use one gpiochip to cover all lines - the
+		 * irq handler then needs to figure out which group and
+		 * bank that was firing the IRQ and look up the per-group
+		 * and bank data.
+		 */
+		gpiochip_set_chained_irqchip(&pc->gpio_chip,
+					     &bcm2835_gpio_irq_chip,
+					     pc->irq[i],
+					     bcm2835_gpio_irq_handler);
+	}
+
 	pc->pctl_dev = devm_pinctrl_register(dev, &bcm2835_pinctrl_desc, pc);
 	if (IS_ERR(pc->pctl_dev)) {
 		gpiochip_remove(&pc->gpio_chip);
diff --git a/drivers/pinctrl/devicetree.c b/drivers/pinctrl/devicetree.c
index 54dad89fc9bf..260908480075 100644
--- a/drivers/pinctrl/devicetree.c
+++ b/drivers/pinctrl/devicetree.c
@@ -253,3 +253,147 @@ err:
 	pinctrl_dt_free_maps(p);
 	return ret;
 }
+
+/*
+ * For pinctrl binding, typically #pinctrl-cells is for the pin controller
+ * device, so either parent or grandparent. See pinctrl-bindings.txt.
+ */
+static int pinctrl_find_cells_size(const struct device_node *np)
+{
+	const char *cells_name = "#pinctrl-cells";
+	int cells_size, error;
+
+	error = of_property_read_u32(np->parent, cells_name, &cells_size);
+	if (error) {
+		error = of_property_read_u32(np->parent->parent,
+					     cells_name, &cells_size);
+		if (error)
+			return -ENOENT;
+	}
+
+	return cells_size;
+}
+
+/**
+ * pinctrl_get_list_and_count - Gets the list and it's cell size and number
+ * @np: pointer to device node with the property
+ * @list_name: property that contains the list
+ * @list: pointer for the list found
+ * @cells_size: pointer for the cell size found
+ * @nr_elements: pointer for the number of elements found
+ *
+ * Typically np is a single pinctrl entry containing the list.
+ */
+static int pinctrl_get_list_and_count(const struct device_node *np,
+				      const char *list_name,
+				      const __be32 **list,
+				      int *cells_size,
+				      int *nr_elements)
+{
+	int size;
+
+	*cells_size = 0;
+	*nr_elements = 0;
+
+	*list = of_get_property(np, list_name, &size);
+	if (!*list)
+		return -ENOENT;
+
+	*cells_size = pinctrl_find_cells_size(np);
+	if (*cells_size < 0)
+		return -ENOENT;
+
+	/* First element is always the index within the pinctrl device */
+	*nr_elements = (size / sizeof(**list)) / (*cells_size + 1);
+
+	return 0;
+}
+
+/**
+ * pinctrl_count_index_with_args - Count number of elements in a pinctrl entry
+ * @np: pointer to device node with the property
+ * @list_name: property that contains the list
+ *
+ * Counts the number of elements in a pinctrl array consisting of an index
+ * within the controller and a number of u32 entries specified for each
+ * entry. Note that device_node is always for the parent pin controller device.
+ */
+int pinctrl_count_index_with_args(const struct device_node *np,
+				  const char *list_name)
+{
+	const __be32 *list;
+	int size, nr_cells, error;
+
+	error = pinctrl_get_list_and_count(np, list_name, &list,
+					   &nr_cells, &size);
+	if (error)
+		return error;
+
+	return size;
+}
+EXPORT_SYMBOL_GPL(pinctrl_count_index_with_args);
+
+/**
+ * pinctrl_copy_args - Populates of_phandle_args based on index
+ * @np: pointer to device node with the property
+ * @list: pointer to a list with the elements
+ * @index: entry within the list of elements
+ * @nr_cells: number of cells in the list
+ * @nr_elem: number of elements for each entry in the list
+ * @out_args: returned values
+ *
+ * Populates the of_phandle_args based on the index in the list.
+ */
+static int pinctrl_copy_args(const struct device_node *np,
+			     const __be32 *list,
+			     int index, int nr_cells, int nr_elem,
+			     struct of_phandle_args *out_args)
+{
+	int i;
+
+	memset(out_args, 0, sizeof(*out_args));
+	out_args->np = (struct device_node *)np;
+	out_args->args_count = nr_cells + 1;
+
+	if (index >= nr_elem)
+		return -EINVAL;
+
+	list += index * (nr_cells + 1);
+
+	for (i = 0; i < nr_cells + 1; i++)
+		out_args->args[i] = be32_to_cpup(list++);
+
+	return 0;
+}
+
+/**
+ * pinctrl_parse_index_with_args - Find a node pointed by index in a list
+ * @np: pointer to device node with the property
+ * @list_name: property that contains the list
+ * @index: index within the list
+ * @out_arts: entries in the list pointed by index
+ *
+ * Finds the selected element in a pinctrl array consisting of an index
+ * within the controller and a number of u32 entries specified for each
+ * entry. Note that device_node is always for the parent pin controller device.
+ */
+int pinctrl_parse_index_with_args(const struct device_node *np,
+				  const char *list_name, int index,
+				  struct of_phandle_args *out_args)
+{
+	const __be32 *list;
+	int nr_elem, nr_cells, error;
+
+	error = pinctrl_get_list_and_count(np, list_name, &list,
+					   &nr_cells, &nr_elem);
+	if (error || !nr_cells)
+		return error;
+
+	error = pinctrl_copy_args(np, list, index, nr_cells, nr_elem,
+				  out_args);
+	if (error)
+		return error;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pinctrl_parse_index_with_args);
diff --git a/drivers/pinctrl/devicetree.h b/drivers/pinctrl/devicetree.h
index 760bc4960f58..c2d1a5505850 100644
--- a/drivers/pinctrl/devicetree.h
+++ b/drivers/pinctrl/devicetree.h
@@ -16,11 +16,20 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+struct of_phandle_args;
+
 #ifdef CONFIG_OF
 
 void pinctrl_dt_free_maps(struct pinctrl *p);
 int pinctrl_dt_to_map(struct pinctrl *p);
 
+int pinctrl_count_index_with_args(const struct device_node *np,
+				  const char *list_name);
+
+int pinctrl_parse_index_with_args(const struct device_node *np,
+				  const char *list_name, int index,
+				  struct of_phandle_args *out_args);
+
 #else
 
 static inline int pinctrl_dt_to_map(struct pinctrl *p)
@@ -32,4 +41,18 @@ static inline void pinctrl_dt_free_maps(struct pinctrl *p)
 {
 }
 
+static inline int pinctrl_count_index_with_args(const struct device_node *np,
+						const char *list_name)
+{
+	return -ENODEV;
+}
+
+static inline int
+pinctrl_parse_index_with_args(const struct device_node *np,
+			      const char *list_name, int index,
+			      struct of_phandle_args *out_args)
+{
+	return -ENODEV;
+}
+
 #endif
diff --git a/drivers/pinctrl/freescale/pinctrl-imx.c b/drivers/pinctrl/freescale/pinctrl-imx.c
index 79c4e14a5a75..5ef7e875b50e 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx.c
@@ -778,10 +778,10 @@ int imx_pinctrl_probe(struct platform_device *pdev,
 	imx_pinctrl_desc->name = dev_name(&pdev->dev);
 	imx_pinctrl_desc->pins = info->pins;
 	imx_pinctrl_desc->npins = info->npins;
-	imx_pinctrl_desc->pctlops = &imx_pctrl_ops,
-	imx_pinctrl_desc->pmxops = &imx_pmx_ops,
-	imx_pinctrl_desc->confops = &imx_pinconf_ops,
-	imx_pinctrl_desc->owner = THIS_MODULE,
+	imx_pinctrl_desc->pctlops = &imx_pctrl_ops;
+	imx_pinctrl_desc->pmxops = &imx_pmx_ops;
+	imx_pinctrl_desc->confops = &imx_pinconf_ops;
+	imx_pinctrl_desc->owner = THIS_MODULE;
 
 	ret = imx_pinctrl_probe_dt(pdev, info);
 	if (ret) {
diff --git a/drivers/pinctrl/intel/pinctrl-baytrail.c b/drivers/pinctrl/intel/pinctrl-baytrail.c
index 71bbeb9321ba..37300634b7d2 100644
--- a/drivers/pinctrl/intel/pinctrl-baytrail.c
+++ b/drivers/pinctrl/intel/pinctrl-baytrail.c
@@ -1703,7 +1703,7 @@ static int byt_gpio_probe(struct byt_gpio *vg)
 	if (irq_rc && irq_rc->start) {
 		byt_gpio_irq_init_hw(vg);
 		ret = gpiochip_irqchip_add(gc, &byt_irqchip, 0,
-					   handle_simple_irq, IRQ_TYPE_NONE);
+					   handle_bad_irq, IRQ_TYPE_NONE);
 		if (ret) {
 			dev_err(&vg->pdev->dev, "failed to add irqchip\n");
 			goto fail;
diff --git a/drivers/pinctrl/intel/pinctrl-cherryview.c b/drivers/pinctrl/intel/pinctrl-cherryview.c
index c43b1e9a06af..5e66860a5e67 100644
--- a/drivers/pinctrl/intel/pinctrl-cherryview.c
+++ b/drivers/pinctrl/intel/pinctrl-cherryview.c
@@ -762,7 +762,7 @@ static void chv_pin_dbg_show(struct pinctrl_dev *pctldev, struct seq_file *s,
 		seq_printf(s, "mode %d ", mode);
 	}
 
-	seq_printf(s, "ctrl0 0x%08x ctrl1 0x%08x", ctrl0, ctrl1);
+	seq_printf(s, "0x%08x 0x%08x", ctrl0, ctrl1);
 
 	if (locked)
 		seq_puts(s, " [LOCKED]");
diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c
index 01443762e570..1e139672f1af 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.c
+++ b/drivers/pinctrl/intel/pinctrl-intel.c
@@ -911,7 +911,7 @@ static int intel_gpio_probe(struct intel_pinctrl *pctrl, int irq)
 	}
 
 	ret = gpiochip_irqchip_add(&pctrl->chip, &intel_gpio_irqchip, 0,
-				   handle_simple_irq, IRQ_TYPE_NONE);
+				   handle_bad_irq, IRQ_TYPE_NONE);
 	if (ret) {
 		dev_err(pctrl->dev, "failed to add irqchip\n");
 		goto fail;
diff --git a/drivers/pinctrl/intel/pinctrl-merrifield.c b/drivers/pinctrl/intel/pinctrl-merrifield.c
index 7826c7f0cb7c..b21896126f76 100644
--- a/drivers/pinctrl/intel/pinctrl-merrifield.c
+++ b/drivers/pinctrl/intel/pinctrl-merrifield.c
@@ -814,10 +814,51 @@ static int mrfld_config_set(struct pinctrl_dev *pctldev, unsigned int pin,
 	return 0;
 }
 
+static int mrfld_config_group_get(struct pinctrl_dev *pctldev,
+				  unsigned int group, unsigned long *config)
+{
+	const unsigned int *pins;
+	unsigned int npins;
+	int ret;
+
+	ret = mrfld_get_group_pins(pctldev, group, &pins, &npins);
+	if (ret)
+		return ret;
+
+	ret = mrfld_config_get(pctldev, pins[0], config);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int mrfld_config_group_set(struct pinctrl_dev *pctldev,
+				  unsigned int group, unsigned long *configs,
+				  unsigned int num_configs)
+{
+	const unsigned int *pins;
+	unsigned int npins;
+	int i, ret;
+
+	ret = mrfld_get_group_pins(pctldev, group, &pins, &npins);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < npins; i++) {
+		ret = mrfld_config_set(pctldev, pins[i], configs, num_configs);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static const struct pinconf_ops mrfld_pinconf_ops = {
 	.is_generic = true,
 	.pin_config_get = mrfld_config_get,
 	.pin_config_set = mrfld_config_set,
+	.pin_config_group_get = mrfld_config_group_get,
+	.pin_config_group_set = mrfld_config_group_set,
 };
 
 static const struct pinctrl_desc mrfld_pinctrl_desc = {
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt6397.c b/drivers/pinctrl/mediatek/pinctrl-mt6397.c
index 6eccb85c02cd..afcede7e2222 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mt6397.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mt6397.c
@@ -64,8 +64,4 @@ static struct platform_driver mtk_pinctrl_driver = {
 	},
 };
 
-static int __init mtk_pinctrl_init(void)
-{
-	return platform_driver_register(&mtk_pinctrl_driver);
-}
-device_initcall(mtk_pinctrl_init);
+builtin_platform_driver(mtk_pinctrl_driver);
diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-mt8173.h b/drivers/pinctrl/mediatek/pinctrl-mtk-mt8173.h
index 13e5b68bfe1b..9b018fdbeb51 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mtk-mt8173.h
+++ b/drivers/pinctrl/mediatek/pinctrl-mtk-mt8173.h
@@ -201,7 +201,7 @@ static const struct mtk_desc_pin mtk_pins_mt8173[] = {
 	MTK_PIN(
 		PINCTRL_PIN(16, "IDDIG"),
 		NULL, "mt8173",
-		MTK_EINT_FUNCTION(0, 16),
+		MTK_EINT_FUNCTION(1, 16),
 		MTK_FUNCTION(0, "GPIO16"),
 		MTK_FUNCTION(1, "IDDIG"),
 		MTK_FUNCTION(2, "CMFLASH"),
diff --git a/drivers/pinctrl/meson/Makefile b/drivers/pinctrl/meson/Makefile
index 24434f139947..27c5b5126008 100644
--- a/drivers/pinctrl/meson/Makefile
+++ b/drivers/pinctrl/meson/Makefile
@@ -1,2 +1,3 @@
-obj-y	+= pinctrl-meson8.o pinctrl-meson8b.o pinctrl-meson-gxbb.o
+obj-y	+= pinctrl-meson8.o pinctrl-meson8b.o
+obj-y	+= pinctrl-meson-gxbb.o pinctrl-meson-gxl.o
 obj-y	+= pinctrl-meson.o
diff --git a/drivers/pinctrl/meson/pinctrl-meson-gxl.c b/drivers/pinctrl/meson/pinctrl-meson-gxl.c
new file mode 100644
index 000000000000..25694f7094c7
--- /dev/null
+++ b/drivers/pinctrl/meson/pinctrl-meson-gxl.c
@@ -0,0 +1,589 @@
+/*
+ * Pin controller and GPIO driver for Amlogic Meson GXL.
+ *
+ * Copyright (C) 2016 Endless Mobile, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <dt-bindings/gpio/meson-gxl-gpio.h>
+#include "pinctrl-meson.h"
+
+#define EE_OFF	10
+
+static const struct pinctrl_pin_desc meson_gxl_periphs_pins[] = {
+	MESON_PIN(GPIOZ_0, EE_OFF),
+	MESON_PIN(GPIOZ_1, EE_OFF),
+	MESON_PIN(GPIOZ_2, EE_OFF),
+	MESON_PIN(GPIOZ_3, EE_OFF),
+	MESON_PIN(GPIOZ_4, EE_OFF),
+	MESON_PIN(GPIOZ_5, EE_OFF),
+	MESON_PIN(GPIOZ_6, EE_OFF),
+	MESON_PIN(GPIOZ_7, EE_OFF),
+	MESON_PIN(GPIOZ_8, EE_OFF),
+	MESON_PIN(GPIOZ_9, EE_OFF),
+	MESON_PIN(GPIOZ_10, EE_OFF),
+	MESON_PIN(GPIOZ_11, EE_OFF),
+	MESON_PIN(GPIOZ_12, EE_OFF),
+	MESON_PIN(GPIOZ_13, EE_OFF),
+	MESON_PIN(GPIOZ_14, EE_OFF),
+	MESON_PIN(GPIOZ_15, EE_OFF),
+
+	MESON_PIN(GPIOH_0, EE_OFF),
+	MESON_PIN(GPIOH_1, EE_OFF),
+	MESON_PIN(GPIOH_2, EE_OFF),
+	MESON_PIN(GPIOH_3, EE_OFF),
+	MESON_PIN(GPIOH_4, EE_OFF),
+	MESON_PIN(GPIOH_5, EE_OFF),
+	MESON_PIN(GPIOH_6, EE_OFF),
+	MESON_PIN(GPIOH_7, EE_OFF),
+	MESON_PIN(GPIOH_8, EE_OFF),
+	MESON_PIN(GPIOH_9, EE_OFF),
+
+	MESON_PIN(BOOT_0, EE_OFF),
+	MESON_PIN(BOOT_1, EE_OFF),
+	MESON_PIN(BOOT_2, EE_OFF),
+	MESON_PIN(BOOT_3, EE_OFF),
+	MESON_PIN(BOOT_4, EE_OFF),
+	MESON_PIN(BOOT_5, EE_OFF),
+	MESON_PIN(BOOT_6, EE_OFF),
+	MESON_PIN(BOOT_7, EE_OFF),
+	MESON_PIN(BOOT_8, EE_OFF),
+	MESON_PIN(BOOT_9, EE_OFF),
+	MESON_PIN(BOOT_10, EE_OFF),
+	MESON_PIN(BOOT_11, EE_OFF),
+	MESON_PIN(BOOT_12, EE_OFF),
+	MESON_PIN(BOOT_13, EE_OFF),
+	MESON_PIN(BOOT_14, EE_OFF),
+	MESON_PIN(BOOT_15, EE_OFF),
+
+	MESON_PIN(CARD_0, EE_OFF),
+	MESON_PIN(CARD_1, EE_OFF),
+	MESON_PIN(CARD_2, EE_OFF),
+	MESON_PIN(CARD_3, EE_OFF),
+	MESON_PIN(CARD_4, EE_OFF),
+	MESON_PIN(CARD_5, EE_OFF),
+	MESON_PIN(CARD_6, EE_OFF),
+
+	MESON_PIN(GPIODV_0, EE_OFF),
+	MESON_PIN(GPIODV_1, EE_OFF),
+	MESON_PIN(GPIODV_2, EE_OFF),
+	MESON_PIN(GPIODV_3, EE_OFF),
+	MESON_PIN(GPIODV_4, EE_OFF),
+	MESON_PIN(GPIODV_5, EE_OFF),
+	MESON_PIN(GPIODV_6, EE_OFF),
+	MESON_PIN(GPIODV_7, EE_OFF),
+	MESON_PIN(GPIODV_8, EE_OFF),
+	MESON_PIN(GPIODV_9, EE_OFF),
+	MESON_PIN(GPIODV_10, EE_OFF),
+	MESON_PIN(GPIODV_11, EE_OFF),
+	MESON_PIN(GPIODV_12, EE_OFF),
+	MESON_PIN(GPIODV_13, EE_OFF),
+	MESON_PIN(GPIODV_14, EE_OFF),
+	MESON_PIN(GPIODV_15, EE_OFF),
+	MESON_PIN(GPIODV_16, EE_OFF),
+	MESON_PIN(GPIODV_17, EE_OFF),
+	MESON_PIN(GPIODV_19, EE_OFF),
+	MESON_PIN(GPIODV_20, EE_OFF),
+	MESON_PIN(GPIODV_21, EE_OFF),
+	MESON_PIN(GPIODV_22, EE_OFF),
+	MESON_PIN(GPIODV_23, EE_OFF),
+	MESON_PIN(GPIODV_24, EE_OFF),
+	MESON_PIN(GPIODV_25, EE_OFF),
+	MESON_PIN(GPIODV_26, EE_OFF),
+	MESON_PIN(GPIODV_27, EE_OFF),
+	MESON_PIN(GPIODV_28, EE_OFF),
+	MESON_PIN(GPIODV_29, EE_OFF),
+
+	MESON_PIN(GPIOX_0, EE_OFF),
+	MESON_PIN(GPIOX_1, EE_OFF),
+	MESON_PIN(GPIOX_2, EE_OFF),
+	MESON_PIN(GPIOX_3, EE_OFF),
+	MESON_PIN(GPIOX_4, EE_OFF),
+	MESON_PIN(GPIOX_5, EE_OFF),
+	MESON_PIN(GPIOX_6, EE_OFF),
+	MESON_PIN(GPIOX_7, EE_OFF),
+	MESON_PIN(GPIOX_8, EE_OFF),
+	MESON_PIN(GPIOX_9, EE_OFF),
+	MESON_PIN(GPIOX_10, EE_OFF),
+	MESON_PIN(GPIOX_11, EE_OFF),
+	MESON_PIN(GPIOX_12, EE_OFF),
+	MESON_PIN(GPIOX_13, EE_OFF),
+	MESON_PIN(GPIOX_14, EE_OFF),
+	MESON_PIN(GPIOX_15, EE_OFF),
+	MESON_PIN(GPIOX_16, EE_OFF),
+	MESON_PIN(GPIOX_17, EE_OFF),
+	MESON_PIN(GPIOX_18, EE_OFF),
+
+	MESON_PIN(GPIOCLK_0, EE_OFF),
+	MESON_PIN(GPIOCLK_1, EE_OFF),
+
+	MESON_PIN(GPIO_TEST_N, EE_OFF),
+};
+
+static const unsigned int emmc_nand_d07_pins[] = {
+	PIN(BOOT_0, EE_OFF), PIN(BOOT_1, EE_OFF), PIN(BOOT_2, EE_OFF),
+	PIN(BOOT_3, EE_OFF), PIN(BOOT_4, EE_OFF), PIN(BOOT_5, EE_OFF),
+	PIN(BOOT_6, EE_OFF), PIN(BOOT_7, EE_OFF),
+};
+static const unsigned int emmc_clk_pins[] = { PIN(BOOT_8, EE_OFF) };
+static const unsigned int emmc_cmd_pins[] = { PIN(BOOT_10, EE_OFF) };
+static const unsigned int emmc_ds_pins[] = { PIN(BOOT_15, EE_OFF) };
+
+static const unsigned int sdcard_d0_pins[] = { PIN(CARD_1, EE_OFF) };
+static const unsigned int sdcard_d1_pins[] = { PIN(CARD_0, EE_OFF) };
+static const unsigned int sdcard_d2_pins[] = { PIN(CARD_5, EE_OFF) };
+static const unsigned int sdcard_d3_pins[] = { PIN(CARD_4, EE_OFF) };
+static const unsigned int sdcard_cmd_pins[] = { PIN(CARD_3, EE_OFF) };
+static const unsigned int sdcard_clk_pins[] = { PIN(CARD_2, EE_OFF) };
+
+static const unsigned int sdio_d0_pins[] = { PIN(GPIOX_0, EE_OFF) };
+static const unsigned int sdio_d1_pins[] = { PIN(GPIOX_1, EE_OFF) };
+static const unsigned int sdio_d2_pins[] = { PIN(GPIOX_2, EE_OFF) };
+static const unsigned int sdio_d3_pins[] = { PIN(GPIOX_3, EE_OFF) };
+static const unsigned int sdio_cmd_pins[] = { PIN(GPIOX_4, EE_OFF) };
+static const unsigned int sdio_clk_pins[] = { PIN(GPIOX_5, EE_OFF) };
+static const unsigned int sdio_irq_pins[] = { PIN(GPIOX_7, EE_OFF) };
+
+static const unsigned int nand_ce0_pins[]	= { PIN(BOOT_8, EE_OFF) };
+static const unsigned int nand_ce1_pins[]	= { PIN(BOOT_9, EE_OFF) };
+static const unsigned int nand_rb0_pins[]	= { PIN(BOOT_10, EE_OFF) };
+static const unsigned int nand_ale_pins[]	= { PIN(BOOT_11, EE_OFF) };
+static const unsigned int nand_cle_pins[]	= { PIN(BOOT_12, EE_OFF) };
+static const unsigned int nand_wen_clk_pins[]	= { PIN(BOOT_13, EE_OFF) };
+static const unsigned int nand_ren_wr_pins[]	= { PIN(BOOT_14, EE_OFF) };
+static const unsigned int nand_dqs_pins[]	= { PIN(BOOT_15, EE_OFF) };
+
+static const unsigned int uart_tx_a_pins[]	= { PIN(GPIOX_12, EE_OFF) };
+static const unsigned int uart_rx_a_pins[]	= { PIN(GPIOX_13, EE_OFF) };
+static const unsigned int uart_cts_a_pins[]	= { PIN(GPIOX_14, EE_OFF) };
+static const unsigned int uart_rts_a_pins[]	= { PIN(GPIOX_15, EE_OFF) };
+
+static const unsigned int uart_tx_b_pins[]	= { PIN(GPIODV_24, EE_OFF) };
+static const unsigned int uart_rx_b_pins[]	= { PIN(GPIODV_25, EE_OFF) };
+
+static const unsigned int uart_tx_c_pins[]	= { PIN(GPIOX_8, EE_OFF) };
+static const unsigned int uart_rx_c_pins[]	= { PIN(GPIOX_9, EE_OFF) };
+
+static const unsigned int i2c_sck_a_pins[]	= { PIN(GPIODV_25, EE_OFF) };
+static const unsigned int i2c_sda_a_pins[]	= { PIN(GPIODV_24, EE_OFF) };
+
+static const unsigned int i2c_sck_b_pins[]	= { PIN(GPIODV_27, EE_OFF) };
+static const unsigned int i2c_sda_b_pins[]	= { PIN(GPIODV_26, EE_OFF) };
+
+static const unsigned int i2c_sck_c_pins[]	= { PIN(GPIODV_29, EE_OFF) };
+static const unsigned int i2c_sda_c_pins[]	= { PIN(GPIODV_28, EE_OFF) };
+
+static const unsigned int eth_mdio_pins[]	= { PIN(GPIOZ_0, EE_OFF) };
+static const unsigned int eth_mdc_pins[]	= { PIN(GPIOZ_1, EE_OFF) };
+static const unsigned int eth_clk_rx_clk_pins[]	= { PIN(GPIOZ_2, EE_OFF) };
+static const unsigned int eth_rx_dv_pins[]	= { PIN(GPIOZ_3, EE_OFF) };
+static const unsigned int eth_rxd0_pins[]	= { PIN(GPIOZ_4, EE_OFF) };
+static const unsigned int eth_rxd1_pins[]	= { PIN(GPIOZ_5, EE_OFF) };
+static const unsigned int eth_rxd2_pins[]	= { PIN(GPIOZ_6, EE_OFF) };
+static const unsigned int eth_rxd3_pins[]	= { PIN(GPIOZ_7, EE_OFF) };
+static const unsigned int eth_rgmii_tx_clk_pins[] = { PIN(GPIOZ_8, EE_OFF) };
+static const unsigned int eth_tx_en_pins[]	= { PIN(GPIOZ_9, EE_OFF) };
+static const unsigned int eth_txd0_pins[]	= { PIN(GPIOZ_10, EE_OFF) };
+static const unsigned int eth_txd1_pins[]	= { PIN(GPIOZ_11, EE_OFF) };
+static const unsigned int eth_txd2_pins[]	= { PIN(GPIOZ_12, EE_OFF) };
+static const unsigned int eth_txd3_pins[]	= { PIN(GPIOZ_13, EE_OFF) };
+
+static const unsigned int pwm_e_pins[]		= { PIN(GPIOX_16, EE_OFF) };
+
+static const struct pinctrl_pin_desc meson_gxl_aobus_pins[] = {
+	MESON_PIN(GPIOAO_0, 0),
+	MESON_PIN(GPIOAO_1, 0),
+	MESON_PIN(GPIOAO_2, 0),
+	MESON_PIN(GPIOAO_3, 0),
+	MESON_PIN(GPIOAO_4, 0),
+	MESON_PIN(GPIOAO_5, 0),
+	MESON_PIN(GPIOAO_6, 0),
+	MESON_PIN(GPIOAO_7, 0),
+	MESON_PIN(GPIOAO_8, 0),
+	MESON_PIN(GPIOAO_9, 0),
+};
+
+static const unsigned int uart_tx_ao_a_pins[]	= { PIN(GPIOAO_0, 0) };
+static const unsigned int uart_rx_ao_a_pins[]	= { PIN(GPIOAO_1, 0) };
+static const unsigned int uart_cts_ao_a_pins[]	= { PIN(GPIOAO_2, 0) };
+static const unsigned int uart_rts_ao_a_pins[]	= { PIN(GPIOAO_3, 0) };
+static const unsigned int uart_tx_ao_b_pins[]	= { PIN(GPIOAO_0, 0) };
+static const unsigned int uart_rx_ao_b_pins[]	= { PIN(GPIOAO_1, 0),
+						    PIN(GPIOAO_5, 0) };
+static const unsigned int uart_cts_ao_b_pins[]	= { PIN(GPIOAO_2, 0) };
+static const unsigned int uart_rts_ao_b_pins[]	= { PIN(GPIOAO_3, 0) };
+
+static const unsigned int remote_input_ao_pins[] = {PIN(GPIOAO_7, 0) };
+
+static struct meson_pmx_group meson_gxl_periphs_groups[] = {
+	GPIO_GROUP(GPIOZ_0, EE_OFF),
+	GPIO_GROUP(GPIOZ_1, EE_OFF),
+	GPIO_GROUP(GPIOZ_2, EE_OFF),
+	GPIO_GROUP(GPIOZ_3, EE_OFF),
+	GPIO_GROUP(GPIOZ_4, EE_OFF),
+	GPIO_GROUP(GPIOZ_5, EE_OFF),
+	GPIO_GROUP(GPIOZ_6, EE_OFF),
+	GPIO_GROUP(GPIOZ_7, EE_OFF),
+	GPIO_GROUP(GPIOZ_8, EE_OFF),
+	GPIO_GROUP(GPIOZ_9, EE_OFF),
+	GPIO_GROUP(GPIOZ_10, EE_OFF),
+	GPIO_GROUP(GPIOZ_11, EE_OFF),
+	GPIO_GROUP(GPIOZ_12, EE_OFF),
+	GPIO_GROUP(GPIOZ_13, EE_OFF),
+	GPIO_GROUP(GPIOZ_14, EE_OFF),
+	GPIO_GROUP(GPIOZ_15, EE_OFF),
+
+	GPIO_GROUP(GPIOH_0, EE_OFF),
+	GPIO_GROUP(GPIOH_1, EE_OFF),
+	GPIO_GROUP(GPIOH_2, EE_OFF),
+	GPIO_GROUP(GPIOH_3, EE_OFF),
+	GPIO_GROUP(GPIOH_4, EE_OFF),
+	GPIO_GROUP(GPIOH_5, EE_OFF),
+	GPIO_GROUP(GPIOH_6, EE_OFF),
+	GPIO_GROUP(GPIOH_7, EE_OFF),
+	GPIO_GROUP(GPIOH_8, EE_OFF),
+	GPIO_GROUP(GPIOH_9, EE_OFF),
+
+	GPIO_GROUP(BOOT_0, EE_OFF),
+	GPIO_GROUP(BOOT_1, EE_OFF),
+	GPIO_GROUP(BOOT_2, EE_OFF),
+	GPIO_GROUP(BOOT_3, EE_OFF),
+	GPIO_GROUP(BOOT_4, EE_OFF),
+	GPIO_GROUP(BOOT_5, EE_OFF),
+	GPIO_GROUP(BOOT_6, EE_OFF),
+	GPIO_GROUP(BOOT_7, EE_OFF),
+	GPIO_GROUP(BOOT_8, EE_OFF),
+	GPIO_GROUP(BOOT_9, EE_OFF),
+	GPIO_GROUP(BOOT_10, EE_OFF),
+	GPIO_GROUP(BOOT_11, EE_OFF),
+	GPIO_GROUP(BOOT_12, EE_OFF),
+	GPIO_GROUP(BOOT_13, EE_OFF),
+	GPIO_GROUP(BOOT_14, EE_OFF),
+	GPIO_GROUP(BOOT_15, EE_OFF),
+
+	GPIO_GROUP(CARD_0, EE_OFF),
+	GPIO_GROUP(CARD_1, EE_OFF),
+	GPIO_GROUP(CARD_2, EE_OFF),
+	GPIO_GROUP(CARD_3, EE_OFF),
+	GPIO_GROUP(CARD_4, EE_OFF),
+	GPIO_GROUP(CARD_5, EE_OFF),
+	GPIO_GROUP(CARD_6, EE_OFF),
+
+	GPIO_GROUP(GPIODV_0, EE_OFF),
+	GPIO_GROUP(GPIODV_1, EE_OFF),
+	GPIO_GROUP(GPIODV_2, EE_OFF),
+	GPIO_GROUP(GPIODV_3, EE_OFF),
+	GPIO_GROUP(GPIODV_4, EE_OFF),
+	GPIO_GROUP(GPIODV_5, EE_OFF),
+	GPIO_GROUP(GPIODV_6, EE_OFF),
+	GPIO_GROUP(GPIODV_7, EE_OFF),
+	GPIO_GROUP(GPIODV_8, EE_OFF),
+	GPIO_GROUP(GPIODV_9, EE_OFF),
+	GPIO_GROUP(GPIODV_10, EE_OFF),
+	GPIO_GROUP(GPIODV_11, EE_OFF),
+	GPIO_GROUP(GPIODV_12, EE_OFF),
+	GPIO_GROUP(GPIODV_13, EE_OFF),
+	GPIO_GROUP(GPIODV_14, EE_OFF),
+	GPIO_GROUP(GPIODV_15, EE_OFF),
+	GPIO_GROUP(GPIODV_16, EE_OFF),
+	GPIO_GROUP(GPIODV_17, EE_OFF),
+	GPIO_GROUP(GPIODV_19, EE_OFF),
+	GPIO_GROUP(GPIODV_20, EE_OFF),
+	GPIO_GROUP(GPIODV_21, EE_OFF),
+	GPIO_GROUP(GPIODV_22, EE_OFF),
+	GPIO_GROUP(GPIODV_23, EE_OFF),
+	GPIO_GROUP(GPIODV_24, EE_OFF),
+	GPIO_GROUP(GPIODV_25, EE_OFF),
+	GPIO_GROUP(GPIODV_26, EE_OFF),
+	GPIO_GROUP(GPIODV_27, EE_OFF),
+	GPIO_GROUP(GPIODV_28, EE_OFF),
+	GPIO_GROUP(GPIODV_29, EE_OFF),
+
+	GPIO_GROUP(GPIOX_0, EE_OFF),
+	GPIO_GROUP(GPIOX_1, EE_OFF),
+	GPIO_GROUP(GPIOX_2, EE_OFF),
+	GPIO_GROUP(GPIOX_3, EE_OFF),
+	GPIO_GROUP(GPIOX_4, EE_OFF),
+	GPIO_GROUP(GPIOX_5, EE_OFF),
+	GPIO_GROUP(GPIOX_6, EE_OFF),
+	GPIO_GROUP(GPIOX_7, EE_OFF),
+	GPIO_GROUP(GPIOX_8, EE_OFF),
+	GPIO_GROUP(GPIOX_9, EE_OFF),
+	GPIO_GROUP(GPIOX_10, EE_OFF),
+	GPIO_GROUP(GPIOX_11, EE_OFF),
+	GPIO_GROUP(GPIOX_12, EE_OFF),
+	GPIO_GROUP(GPIOX_13, EE_OFF),
+	GPIO_GROUP(GPIOX_14, EE_OFF),
+	GPIO_GROUP(GPIOX_15, EE_OFF),
+	GPIO_GROUP(GPIOX_16, EE_OFF),
+	GPIO_GROUP(GPIOX_17, EE_OFF),
+	GPIO_GROUP(GPIOX_18, EE_OFF),
+
+	GPIO_GROUP(GPIOCLK_0, EE_OFF),
+	GPIO_GROUP(GPIOCLK_1, EE_OFF),
+
+	GPIO_GROUP(GPIO_TEST_N, EE_OFF),
+
+	/* Bank X */
+	GROUP(sdio_d0,		5,	31),
+	GROUP(sdio_d1,		5,	30),
+	GROUP(sdio_d2,		5,	29),
+	GROUP(sdio_d3,		5,	28),
+	GROUP(sdio_cmd,		5,	27),
+	GROUP(sdio_clk,		5,	26),
+	GROUP(sdio_irq,		5,	24),
+	GROUP(uart_tx_a,	5,	19),
+	GROUP(uart_rx_a,	5,	18),
+	GROUP(uart_cts_a,	5,	17),
+	GROUP(uart_rts_a,	5,	16),
+	GROUP(uart_tx_c,	5,	13),
+	GROUP(uart_rx_c,	5,	12),
+	GROUP(pwm_e,		5,	15),
+
+	/* Bank Z */
+	GROUP(eth_mdio,		4,	22),
+	GROUP(eth_mdc,		4,	23),
+	GROUP(eth_clk_rx_clk,	4,	21),
+	GROUP(eth_rx_dv,	4,	20),
+	GROUP(eth_rxd0,		4,	19),
+	GROUP(eth_rxd1,		4,	18),
+	GROUP(eth_rxd2,		4,	17),
+	GROUP(eth_rxd3,		4,	16),
+	GROUP(eth_rgmii_tx_clk,	4,	15),
+	GROUP(eth_tx_en,	4,	14),
+	GROUP(eth_txd0,		4,	13),
+	GROUP(eth_txd1,		4,	12),
+	GROUP(eth_txd2,		4,	11),
+	GROUP(eth_txd3,		4,	10),
+
+	/* Bank DV */
+	GROUP(uart_tx_b,	2,	16),
+	GROUP(uart_rx_b,	2,	15),
+	GROUP(i2c_sck_a,	1,	15),
+	GROUP(i2c_sda_a,	1,	14),
+	GROUP(i2c_sck_b,	1,	13),
+	GROUP(i2c_sda_b,	1,	12),
+	GROUP(i2c_sck_c,	1,	11),
+	GROUP(i2c_sda_c,	1,	10),
+
+	/* Bank BOOT */
+	GROUP(emmc_nand_d07,	7,	31),
+	GROUP(emmc_clk,		7,	30),
+	GROUP(emmc_cmd,		7,	29),
+	GROUP(emmc_ds,		7,	28),
+	GROUP(nand_ce0,		7,	7),
+	GROUP(nand_ce1,		7,	6),
+	GROUP(nand_rb0,		7,	5),
+	GROUP(nand_ale,		7,	4),
+	GROUP(nand_cle,		7,	3),
+	GROUP(nand_wen_clk,	7,	2),
+	GROUP(nand_ren_wr,	7,	1),
+	GROUP(nand_dqs,		7,	0),
+
+	/* Bank CARD */
+	GROUP(sdcard_d1,	6,	5),
+	GROUP(sdcard_d0,	6,	4),
+	GROUP(sdcard_d3,	6,	1),
+	GROUP(sdcard_d2,	6,	0),
+	GROUP(sdcard_cmd,	6,	2),
+	GROUP(sdcard_clk,	6,	3),
+};
+
+static struct meson_pmx_group meson_gxl_aobus_groups[] = {
+	GPIO_GROUP(GPIOAO_0, 0),
+	GPIO_GROUP(GPIOAO_1, 0),
+	GPIO_GROUP(GPIOAO_2, 0),
+	GPIO_GROUP(GPIOAO_3, 0),
+	GPIO_GROUP(GPIOAO_4, 0),
+	GPIO_GROUP(GPIOAO_5, 0),
+	GPIO_GROUP(GPIOAO_6, 0),
+	GPIO_GROUP(GPIOAO_7, 0),
+	GPIO_GROUP(GPIOAO_8, 0),
+	GPIO_GROUP(GPIOAO_9, 0),
+
+	/* bank AO */
+	GROUP(uart_tx_ao_b,	0,	26),
+	GROUP(uart_rx_ao_b,	0,	25),
+	GROUP(uart_tx_ao_a,	0,	12),
+	GROUP(uart_rx_ao_a,	0,	11),
+	GROUP(uart_cts_ao_a,	0,	10),
+	GROUP(uart_rts_ao_a,	0,	9),
+	GROUP(uart_cts_ao_b,	0,	8),
+	GROUP(uart_rts_ao_b,	0,	7),
+	GROUP(remote_input_ao,	0,	0),
+};
+
+static const char * const gpio_periphs_groups[] = {
+	"GPIOZ_0", "GPIOZ_1", "GPIOZ_2", "GPIOZ_3", "GPIOZ_4",
+	"GPIOZ_5", "GPIOZ_6", "GPIOZ_7", "GPIOZ_8", "GPIOZ_9",
+	"GPIOZ_10", "GPIOZ_11", "GPIOZ_12", "GPIOZ_13", "GPIOZ_14",
+	"GPIOZ_15",
+
+	"GPIOH_0", "GPIOH_1", "GPIOH_2", "GPIOH_3", "GPIOH_4",
+	"GPIOH_5", "GPIOH_6", "GPIOH_7", "GPIOH_8", "GPIOH_9",
+
+	"BOOT_0", "BOOT_1", "BOOT_2", "BOOT_3", "BOOT_4",
+	"BOOT_5", "BOOT_6", "BOOT_7", "BOOT_8", "BOOT_9",
+	"BOOT_10", "BOOT_11", "BOOT_12", "BOOT_13", "BOOT_14",
+	"BOOT_15",
+
+	"CARD_0", "CARD_1", "CARD_2", "CARD_3", "CARD_4",
+	"CARD_5", "CARD_6",
+
+	"GPIODV_0", "GPIODV_1", "GPIODV_2", "GPIODV_3", "GPIODV_4",
+	"GPIODV_5", "GPIODV_6", "GPIODV_7", "GPIODV_8", "GPIODV_9",
+	"GPIODV_10", "GPIODV_11", "GPIODV_12", "GPIODV_13", "GPIODV_14",
+	"GPIODV_15", "GPIODV_16", "GPIODV_17", "GPIODV_18", "GPIODV_19",
+	"GPIODV_20", "GPIODV_21", "GPIODV_22", "GPIODV_23", "GPIODV_24",
+	"GPIODV_25", "GPIODV_26", "GPIODV_27", "GPIODV_28", "GPIODV_29",
+
+	"GPIOX_0", "GPIOX_1", "GPIOX_2", "GPIOX_3", "GPIOX_4",
+	"GPIOX_5", "GPIOX_6", "GPIOX_7", "GPIOX_8", "GPIOX_9",
+	"GPIOX_10", "GPIOX_11", "GPIOX_12", "GPIOX_13", "GPIOX_14",
+	"GPIOX_15", "GPIOX_16", "GPIOX_17", "GPIOX_18",
+
+	"GPIO_TEST_N",
+};
+
+static const char * const emmc_groups[] = {
+	"emmc_nand_d07", "emmc_clk", "emmc_cmd", "emmc_ds",
+};
+
+static const char * const sdcard_groups[] = {
+	"sdcard_d0", "sdcard_d1", "sdcard_d2", "sdcard_d3",
+	"sdcard_cmd", "sdcard_clk",
+};
+
+static const char * const sdio_groups[] = {
+	"sdio_d0", "sdio_d1", "sdio_d2", "sdio_d3",
+	"sdio_cmd", "sdio_clk", "sdio_irq",
+};
+
+static const char * const nand_groups[] = {
+	"nand_ce0", "nand_ce1", "nand_rb0", "nand_ale", "nand_cle",
+	"nand_wen_clk", "nand_ren_wr", "nand_dqs",
+};
+
+static const char * const uart_a_groups[] = {
+	"uart_tx_a", "uart_rx_a", "uart_cts_a", "uart_rts_a",
+};
+
+static const char * const uart_b_groups[] = {
+	"uart_tx_b", "uart_rx_b",
+};
+
+static const char * const uart_c_groups[] = {
+	"uart_tx_c", "uart_rx_c",
+};
+
+static const char * const i2c_a_groups[] = {
+	"i2c_sck_a", "i2c_sda_a",
+};
+
+static const char * const i2c_b_groups[] = {
+	"i2c_sck_b", "i2c_sda_b",
+};
+
+static const char * const i2c_c_groups[] = {
+	"i2c_sck_c", "i2c_sda_c",
+};
+
+static const char * const eth_groups[] = {
+	"eth_mdio", "eth_mdc", "eth_clk_rx_clk", "eth_rx_dv",
+	"eth_rxd0", "eth_rxd1", "eth_rxd2", "eth_rxd3",
+	"eth_rgmii_tx_clk", "eth_tx_en",
+	"eth_txd0", "eth_txd1", "eth_txd2", "eth_txd3",
+};
+
+static const char * const pwm_e_groups[] = {
+	"pwm_e",
+};
+
+static const char * const gpio_aobus_groups[] = {
+	"GPIOAO_0", "GPIOAO_1", "GPIOAO_2", "GPIOAO_3", "GPIOAO_4",
+	"GPIOAO_5", "GPIOAO_6", "GPIOAO_7", "GPIOAO_8", "GPIOAO_9",
+};
+
+static const char * const uart_ao_groups[] = {
+	"uart_tx_ao_a", "uart_rx_ao_a", "uart_cts_ao_a", "uart_rts_ao_a",
+};
+
+static const char * const uart_ao_b_groups[] = {
+	"uart_tx_ao_b", "uart_rx_ao_b", "uart_cts_ao_b", "uart_rts_ao_b",
+};
+
+static const char * const remote_input_ao_groups[] = {
+	"remote_input_ao",
+};
+
+static struct meson_pmx_func meson_gxl_periphs_functions[] = {
+	FUNCTION(gpio_periphs),
+	FUNCTION(emmc),
+	FUNCTION(sdcard),
+	FUNCTION(sdio),
+	FUNCTION(nand),
+	FUNCTION(uart_a),
+	FUNCTION(uart_b),
+	FUNCTION(uart_c),
+	FUNCTION(i2c_a),
+	FUNCTION(i2c_b),
+	FUNCTION(i2c_c),
+	FUNCTION(eth),
+	FUNCTION(pwm_e),
+};
+
+static struct meson_pmx_func meson_gxl_aobus_functions[] = {
+	FUNCTION(gpio_aobus),
+	FUNCTION(uart_ao),
+	FUNCTION(uart_ao_b),
+	FUNCTION(remote_input_ao),
+};
+
+static struct meson_bank meson_gxl_periphs_banks[] = {
+	/*   name    first                      last                    pullen  pull    dir     out     in  */
+	BANK("X",    PIN(GPIOX_0, EE_OFF),	PIN(GPIOX_18, EE_OFF),  4,  0,  4,  0,  12, 0,  13, 0,  14, 0),
+	BANK("DV",   PIN(GPIODV_0, EE_OFF),	PIN(GPIODV_29, EE_OFF), 0,  0,  0,  0,  0,  0,  1,  0,  2,  0),
+	BANK("H",    PIN(GPIOH_0, EE_OFF),	PIN(GPIOH_9, EE_OFF),   1, 20,  1, 20,  3, 20,  4, 20,  5, 20),
+	BANK("Z",    PIN(GPIOZ_0, EE_OFF),	PIN(GPIOZ_15, EE_OFF),  3,  0,  3,  0,  9,  0,  10, 0, 11,  0),
+	BANK("CARD", PIN(CARD_0, EE_OFF),	PIN(CARD_6, EE_OFF),    2, 20,  2, 20,  6, 20,  7, 20,  8, 20),
+	BANK("BOOT", PIN(BOOT_0, EE_OFF),	PIN(BOOT_15, EE_OFF),   2,  0,  2,  0,  6,  0,  7,  0,  8,  0),
+	BANK("CLK",  PIN(GPIOCLK_0, EE_OFF),	PIN(GPIOCLK_1, EE_OFF), 3, 28,  3, 28,  9, 28, 10, 28, 11, 28),
+};
+
+static struct meson_bank meson_gxl_aobus_banks[] = {
+	/*   name    first              last              pullen  pull    dir     out     in  */
+	BANK("AO",   PIN(GPIOAO_0, 0),  PIN(GPIOAO_9, 0), 0,  0,  0, 16,  0,  0,  0, 16,  1,  0),
+};
+
+struct meson_pinctrl_data meson_gxl_periphs_pinctrl_data = {
+	.name		= "periphs-banks",
+	.pin_base	= 10,
+	.pins		= meson_gxl_periphs_pins,
+	.groups		= meson_gxl_periphs_groups,
+	.funcs		= meson_gxl_periphs_functions,
+	.banks		= meson_gxl_periphs_banks,
+	.num_pins	= ARRAY_SIZE(meson_gxl_periphs_pins),
+	.num_groups	= ARRAY_SIZE(meson_gxl_periphs_groups),
+	.num_funcs	= ARRAY_SIZE(meson_gxl_periphs_functions),
+	.num_banks	= ARRAY_SIZE(meson_gxl_periphs_banks),
+};
+
+struct meson_pinctrl_data meson_gxl_aobus_pinctrl_data = {
+	.name		= "aobus-banks",
+	.pin_base	= 0,
+	.pins		= meson_gxl_aobus_pins,
+	.groups		= meson_gxl_aobus_groups,
+	.funcs		= meson_gxl_aobus_functions,
+	.banks		= meson_gxl_aobus_banks,
+	.num_pins	= ARRAY_SIZE(meson_gxl_aobus_pins),
+	.num_groups	= ARRAY_SIZE(meson_gxl_aobus_groups),
+	.num_funcs	= ARRAY_SIZE(meson_gxl_aobus_functions),
+	.num_banks	= ARRAY_SIZE(meson_gxl_aobus_banks),
+};
diff --git a/drivers/pinctrl/meson/pinctrl-meson.c b/drivers/pinctrl/meson/pinctrl-meson.c
index 57122eda155a..a579126832af 100644
--- a/drivers/pinctrl/meson/pinctrl-meson.c
+++ b/drivers/pinctrl/meson/pinctrl-meson.c
@@ -524,6 +524,14 @@ static const struct of_device_id meson_pinctrl_dt_match[] = {
 		.compatible = "amlogic,meson-gxbb-aobus-pinctrl",
 		.data = &meson_gxbb_aobus_pinctrl_data,
 	},
+	{
+		.compatible = "amlogic,meson-gxl-periphs-pinctrl",
+		.data = &meson_gxl_periphs_pinctrl_data,
+	},
+	{
+		.compatible = "amlogic,meson-gxl-aobus-pinctrl",
+		.data = &meson_gxl_aobus_pinctrl_data,
+	},
 	{ },
 };
 
diff --git a/drivers/pinctrl/meson/pinctrl-meson.h b/drivers/pinctrl/meson/pinctrl-meson.h
index 98b5080650c1..1aa871d5431e 100644
--- a/drivers/pinctrl/meson/pinctrl-meson.h
+++ b/drivers/pinctrl/meson/pinctrl-meson.h
@@ -169,3 +169,5 @@ extern struct meson_pinctrl_data meson8b_cbus_pinctrl_data;
 extern struct meson_pinctrl_data meson8b_aobus_pinctrl_data;
 extern struct meson_pinctrl_data meson_gxbb_periphs_pinctrl_data;
 extern struct meson_pinctrl_data meson_gxbb_aobus_pinctrl_data;
+extern struct meson_pinctrl_data meson_gxl_periphs_pinctrl_data;
+extern struct meson_pinctrl_data meson_gxl_aobus_pinctrl_data;
diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c b/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c
index 8392083514fb..af4814479eb0 100644
--- a/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c
+++ b/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c
@@ -379,13 +379,24 @@ static const unsigned msp0txrx_a_1_pins[] = { DB8500_PIN_AC4, DB8500_PIN_AC3 };
 static const unsigned msp0tfstck_a_1_pins[] = { DB8500_PIN_AF3, DB8500_PIN_AE3 };
 static const unsigned msp0rfsrck_a_1_pins[] = { DB8500_PIN_AD3, DB8500_PIN_AD4 };
 /* Basic pins of the MMC/SD card 0 interface */
-static const unsigned mc0_a_1_pins[] = { DB8500_PIN_AC2, DB8500_PIN_AC1,
-	DB8500_PIN_AB4, DB8500_PIN_AA3, DB8500_PIN_AA4, DB8500_PIN_AB2,
-	DB8500_PIN_Y4, DB8500_PIN_Y2, DB8500_PIN_AA2, DB8500_PIN_AA1 };
+static const unsigned mc0_a_1_pins[] = { DB8500_PIN_AC2, /* MC0_CMDDIR */
+					 DB8500_PIN_AC1, /* MC0_DAT0DIR */
+					 DB8500_PIN_AB4, /* MC0_DAT2DIR */
+					 DB8500_PIN_AA3, /* MC0_FBCLK */
+					 DB8500_PIN_AA4, /* MC0_CLK */
+					 DB8500_PIN_AB2, /* MC0_CMD */
+					 DB8500_PIN_Y4,  /* MC0_DAT0 */
+					 DB8500_PIN_Y2,  /* MC0_DAT1 */
+					 DB8500_PIN_AA2, /* MC0_DAT2 */
+					 DB8500_PIN_AA1  /* MC0_DAT3 */
+};
 /* Often only 4 bits are used, then these are not needed (only used for MMC) */
-static const unsigned mc0_dat47_a_1_pins[] = { DB8500_PIN_W2, DB8500_PIN_W3,
-	DB8500_PIN_V3, DB8500_PIN_V2};
-static const unsigned mc0dat31dir_a_1_pins[] = { DB8500_PIN_AB3 };
+static const unsigned mc0_dat47_a_1_pins[] = { DB8500_PIN_W2, /* MC0_DAT4 */
+					       DB8500_PIN_W3, /* MC0_DAT5 */
+					       DB8500_PIN_V3, /* MC0_DAT6 */
+					       DB8500_PIN_V2  /* MC0_DAT7 */
+};
+static const unsigned mc0dat31dir_a_1_pins[] = { DB8500_PIN_AB3 }; /* MC0_DAT31DIR */
 /* MSP1 can only be on these pins, but TXD and RXD can be flipped */
 static const unsigned msp1txrx_a_1_pins[] = { DB8500_PIN_AF2, DB8500_PIN_AG2 };
 static const unsigned msp1_a_1_pins[] = { DB8500_PIN_AE1, DB8500_PIN_AE2 };
diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c
index 5020ae534479..ce3335accb5b 100644
--- a/drivers/pinctrl/pinconf-generic.c
+++ b/drivers/pinctrl/pinconf-generic.c
@@ -381,7 +381,7 @@ int pinconf_generic_dt_node_to_map(struct pinctrl_dev *pctldev,
 	if (ret < 0)
 		goto exit;
 
-	for_each_child_of_node(np_config, np) {
+	for_each_available_child_of_node(np_config, np) {
 		ret = pinconf_generic_dt_subnode_to_map(pctldev, np, map,
 					&reserved_maps, num_maps, type);
 		if (ret < 0)
diff --git a/drivers/pinctrl/pinctrl-at91.c b/drivers/pinctrl/pinctrl-at91.c
index 9f0904185909..569bc28cb909 100644
--- a/drivers/pinctrl/pinctrl-at91.c
+++ b/drivers/pinctrl/pinctrl-at91.c
@@ -56,6 +56,9 @@ static int gpio_banks;
 #define DRIVE_STRENGTH_SHIFT	5
 #define DRIVE_STRENGTH_MASK		0x3
 #define DRIVE_STRENGTH   (DRIVE_STRENGTH_MASK << DRIVE_STRENGTH_SHIFT)
+#define OUTPUT		(1 << 7)
+#define OUTPUT_VAL_SHIFT	8
+#define OUTPUT_VAL	(0x1 << OUTPUT_VAL_SHIFT)
 #define DEBOUNCE	(1 << 16)
 #define DEBOUNCE_VAL_SHIFT	17
 #define DEBOUNCE_VAL	(0x3fff << DEBOUNCE_VAL_SHIFT)
@@ -375,6 +378,19 @@ static void at91_mux_set_pullup(void __iomem *pio, unsigned mask, bool on)
 	writel_relaxed(mask, pio + (on ? PIO_PUER : PIO_PUDR));
 }
 
+static bool at91_mux_get_output(void __iomem *pio, unsigned int pin, bool *val)
+{
+	*val = (readl_relaxed(pio + PIO_ODSR) >> pin) & 0x1;
+	return (readl_relaxed(pio + PIO_OSR) >> pin) & 0x1;
+}
+
+static void at91_mux_set_output(void __iomem *pio, unsigned int mask,
+				bool is_on, bool val)
+{
+	writel_relaxed(mask, pio + (val ? PIO_SODR : PIO_CODR));
+	writel_relaxed(mask, pio + (is_on ? PIO_OER : PIO_ODR));
+}
+
 static unsigned at91_mux_get_multidrive(void __iomem *pio, unsigned pin)
 {
 	return (readl_relaxed(pio + PIO_MDSR) >> pin) & 0x1;
@@ -848,6 +864,7 @@ static int at91_pinconf_get(struct pinctrl_dev *pctldev,
 	void __iomem *pio;
 	unsigned pin;
 	int div;
+	bool out;
 
 	*config = 0;
 	dev_dbg(info->dev, "%s:%d, pin_id=%d", __func__, __LINE__, pin_id);
@@ -875,6 +892,8 @@ static int at91_pinconf_get(struct pinctrl_dev *pctldev,
 	if (info->ops->get_drivestrength)
 		*config |= (info->ops->get_drivestrength(pio, pin)
 				<< DRIVE_STRENGTH_SHIFT);
+	if (at91_mux_get_output(pio, pin, &out))
+		*config |= OUTPUT | (out << OUTPUT_VAL_SHIFT);
 
 	return 0;
 }
@@ -907,6 +926,8 @@ static int at91_pinconf_set(struct pinctrl_dev *pctldev,
 		if (config & PULL_UP && config & PULL_DOWN)
 			return -EINVAL;
 
+		at91_mux_set_output(pio, mask, config & OUTPUT,
+				    (config & OUTPUT_VAL) >> OUTPUT_VAL_SHIFT);
 		at91_mux_set_pullup(pio, mask, config & PULL_UP);
 		at91_mux_set_multidrive(pio, mask, config & MULTI_DRIVE);
 		if (info->ops->set_deglitch)
diff --git a/drivers/pinctrl/pinctrl-da850-pupd.c b/drivers/pinctrl/pinctrl-da850-pupd.c
new file mode 100644
index 000000000000..b36a90a3f3e4
--- /dev/null
+++ b/drivers/pinctrl/pinctrl-da850-pupd.c
@@ -0,0 +1,210 @@
+/*
+ * Pinconf driver for TI DA850/OMAP-L138/AM18XX pullup/pulldown groups
+ *
+ * Copyright (C) 2016  David Lechner
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/platform_device.h>
+
+#define DA850_PUPD_ENA		0x00
+#define DA850_PUPD_SEL		0x04
+
+struct da850_pupd_data {
+	void __iomem *base;
+	struct pinctrl_desc desc;
+	struct pinctrl_dev *pinctrl;
+};
+
+static const char * const da850_pupd_group_names[] = {
+	"cp0", "cp1", "cp2", "cp3", "cp4", "cp5", "cp6", "cp7",
+	"cp8", "cp9", "cp10", "cp11", "cp12", "cp13", "cp14", "cp15",
+	"cp16", "cp17", "cp18", "cp19", "cp20", "cp21", "cp22", "cp23",
+	"cp24", "cp25", "cp26", "cp27", "cp28", "cp29", "cp30", "cp31",
+};
+
+static int da850_pupd_get_groups_count(struct pinctrl_dev *pctldev)
+{
+	return ARRAY_SIZE(da850_pupd_group_names);
+}
+
+static const char *da850_pupd_get_group_name(struct pinctrl_dev *pctldev,
+					     unsigned int selector)
+{
+	return da850_pupd_group_names[selector];
+}
+
+static int da850_pupd_get_group_pins(struct pinctrl_dev *pctldev,
+				     unsigned int selector,
+				     const unsigned int **pins,
+				     unsigned int *num_pins)
+{
+	*num_pins = 0;
+
+	return 0;
+}
+
+static const struct pinctrl_ops da850_pupd_pctlops = {
+	.get_groups_count	= da850_pupd_get_groups_count,
+	.get_group_name		= da850_pupd_get_group_name,
+	.get_group_pins		= da850_pupd_get_group_pins,
+	.dt_node_to_map		= pinconf_generic_dt_node_to_map_group,
+	.dt_free_map		= pinconf_generic_dt_free_map,
+};
+
+static int da850_pupd_pin_config_group_get(struct pinctrl_dev *pctldev,
+					   unsigned int selector,
+					   unsigned long *config)
+{
+	struct da850_pupd_data *data = pinctrl_dev_get_drvdata(pctldev);
+	enum pin_config_param param = pinconf_to_config_param(*config);
+	u32 val;
+	u16 arg;
+
+	val = readl(data->base + DA850_PUPD_ENA);
+	arg = !!(~val & BIT(selector));
+
+	switch (param) {
+	case PIN_CONFIG_BIAS_DISABLE:
+		break;
+	case PIN_CONFIG_BIAS_PULL_UP:
+	case PIN_CONFIG_BIAS_PULL_DOWN:
+		if (arg) {
+			/* bias is disabled */
+			arg = 0;
+			break;
+		}
+		val = readl(data->base + DA850_PUPD_SEL);
+		if (param == PIN_CONFIG_BIAS_PULL_DOWN)
+			val = ~val;
+		arg = !!(val & BIT(selector));
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	*config = pinconf_to_config_packed(param, arg);
+
+	return 0;
+}
+
+static int da850_pupd_pin_config_group_set(struct pinctrl_dev *pctldev,
+					   unsigned int selector,
+					   unsigned long *configs,
+					   unsigned int num_configs)
+{
+	struct da850_pupd_data *data = pinctrl_dev_get_drvdata(pctldev);
+	u32 ena, sel;
+	enum pin_config_param param;
+	u16 arg;
+	int i;
+
+	ena = readl(data->base + DA850_PUPD_ENA);
+	sel = readl(data->base + DA850_PUPD_SEL);
+
+	for (i = 0; i < num_configs; i++) {
+		param = pinconf_to_config_param(configs[i]);
+		arg = pinconf_to_config_argument(configs[i]);
+
+		switch (param) {
+		case PIN_CONFIG_BIAS_DISABLE:
+			ena &= ~BIT(selector);
+			break;
+		case PIN_CONFIG_BIAS_PULL_UP:
+			ena |= BIT(selector);
+			sel |= BIT(selector);
+			break;
+		case PIN_CONFIG_BIAS_PULL_DOWN:
+			ena |= BIT(selector);
+			sel &= ~BIT(selector);
+			break;
+		default:
+			return -EINVAL;
+		}
+	}
+
+	writel(sel, data->base + DA850_PUPD_SEL);
+	writel(ena, data->base + DA850_PUPD_ENA);
+
+	return 0;
+}
+
+static const struct pinconf_ops da850_pupd_confops = {
+	.is_generic		= true,
+	.pin_config_group_get	= da850_pupd_pin_config_group_get,
+	.pin_config_group_set	= da850_pupd_pin_config_group_set,
+};
+
+static int da850_pupd_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct da850_pupd_data *data;
+	struct resource *res;
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	data->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(data->base)) {
+		dev_err(dev, "Could not map resource\n");
+		return PTR_ERR(data->base);
+	}
+
+	data->desc.name = dev_name(dev);
+	data->desc.pctlops = &da850_pupd_pctlops;
+	data->desc.confops = &da850_pupd_confops;
+	data->desc.owner = THIS_MODULE;
+
+	data->pinctrl = devm_pinctrl_register(dev, &data->desc, data);
+	if (IS_ERR(data->pinctrl)) {
+		dev_err(dev, "Failed to register pinctrl\n");
+		return PTR_ERR(data->pinctrl);
+	}
+
+	platform_set_drvdata(pdev, data);
+
+	return 0;
+}
+
+static int da850_pupd_remove(struct platform_device *pdev)
+{
+	return 0;
+}
+
+static const struct of_device_id da850_pupd_of_match[] = {
+	{ .compatible = "ti,da850-pupd" },
+	{ }
+};
+
+static struct platform_driver da850_pupd_driver = {
+	.driver	= {
+		.name		= "ti-da850-pupd",
+		.of_match_table	= da850_pupd_of_match,
+	},
+	.probe	= da850_pupd_probe,
+	.remove	= da850_pupd_remove,
+};
+module_platform_driver(da850_pupd_driver);
+
+MODULE_AUTHOR("David Lechner <david@lechnology.com>");
+MODULE_DESCRIPTION("TI DA850/OMAP-L138/AM18XX pullup/pulldown configuration");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/pinctrl-oxnas.c b/drivers/pinctrl/pinctrl-oxnas.c
index 917a7d2535d7..494ec9a7573a 100644
--- a/drivers/pinctrl/pinctrl-oxnas.c
+++ b/drivers/pinctrl/pinctrl-oxnas.c
@@ -37,15 +37,24 @@
 
 #define GPIO_BANK_START(bank)		((bank) * PINS_PER_BANK)
 
-/* Regmap Offsets */
-#define PINMUX_PRIMARY_SEL0	0x0c
-#define PINMUX_SECONDARY_SEL0	0x14
-#define PINMUX_TERTIARY_SEL0	0x8c
-#define PINMUX_PRIMARY_SEL1	0x10
-#define PINMUX_SECONDARY_SEL1	0x18
-#define PINMUX_TERTIARY_SEL1	0x90
-#define PINMUX_PULLUP_CTRL0	0xac
-#define PINMUX_PULLUP_CTRL1	0xb0
+/* OX810 Regmap Offsets */
+#define PINMUX_810_PRIMARY_SEL0		0x0c
+#define PINMUX_810_SECONDARY_SEL0	0x14
+#define PINMUX_810_TERTIARY_SEL0	0x8c
+#define PINMUX_810_PRIMARY_SEL1		0x10
+#define PINMUX_810_SECONDARY_SEL1	0x18
+#define PINMUX_810_TERTIARY_SEL1	0x90
+#define PINMUX_810_PULLUP_CTRL0		0xac
+#define PINMUX_810_PULLUP_CTRL1		0xb0
+
+/* OX820 Regmap Offsets */
+#define PINMUX_820_BANK_OFFSET		0x100000
+#define PINMUX_820_SECONDARY_SEL	0x14
+#define PINMUX_820_TERTIARY_SEL		0x8c
+#define PINMUX_820_QUATERNARY_SEL	0x94
+#define PINMUX_820_DEBUG_SEL		0x9c
+#define PINMUX_820_ALTERNATIVE_SEL	0xa4
+#define PINMUX_820_PULLUP_CTRL		0xac
 
 /* GPIO Registers */
 #define INPUT_VALUE	0x00
@@ -87,8 +96,6 @@ struct oxnas_pinctrl {
 	struct regmap *regmap;
 	struct device *dev;
 	struct pinctrl_dev *pctldev;
-	const struct pinctrl_pin_desc *pins;
-	unsigned int npins;
 	const struct oxnas_function *functions;
 	unsigned int nfunctions;
 	const struct oxnas_pin_group *groups;
@@ -97,7 +104,50 @@ struct oxnas_pinctrl {
 	unsigned int nbanks;
 };
 
-static const struct pinctrl_pin_desc oxnas_pins[] = {
+struct oxnas_pinctrl_data {
+	struct pinctrl_desc *desc;
+	struct oxnas_pinctrl *pctl;
+};
+
+static const struct pinctrl_pin_desc oxnas_ox810se_pins[] = {
+	PINCTRL_PIN(0, "gpio0"),
+	PINCTRL_PIN(1, "gpio1"),
+	PINCTRL_PIN(2, "gpio2"),
+	PINCTRL_PIN(3, "gpio3"),
+	PINCTRL_PIN(4, "gpio4"),
+	PINCTRL_PIN(5, "gpio5"),
+	PINCTRL_PIN(6, "gpio6"),
+	PINCTRL_PIN(7, "gpio7"),
+	PINCTRL_PIN(8, "gpio8"),
+	PINCTRL_PIN(9, "gpio9"),
+	PINCTRL_PIN(10, "gpio10"),
+	PINCTRL_PIN(11, "gpio11"),
+	PINCTRL_PIN(12, "gpio12"),
+	PINCTRL_PIN(13, "gpio13"),
+	PINCTRL_PIN(14, "gpio14"),
+	PINCTRL_PIN(15, "gpio15"),
+	PINCTRL_PIN(16, "gpio16"),
+	PINCTRL_PIN(17, "gpio17"),
+	PINCTRL_PIN(18, "gpio18"),
+	PINCTRL_PIN(19, "gpio19"),
+	PINCTRL_PIN(20, "gpio20"),
+	PINCTRL_PIN(21, "gpio21"),
+	PINCTRL_PIN(22, "gpio22"),
+	PINCTRL_PIN(23, "gpio23"),
+	PINCTRL_PIN(24, "gpio24"),
+	PINCTRL_PIN(25, "gpio25"),
+	PINCTRL_PIN(26, "gpio26"),
+	PINCTRL_PIN(27, "gpio27"),
+	PINCTRL_PIN(28, "gpio28"),
+	PINCTRL_PIN(29, "gpio29"),
+	PINCTRL_PIN(30, "gpio30"),
+	PINCTRL_PIN(31, "gpio31"),
+	PINCTRL_PIN(32, "gpio32"),
+	PINCTRL_PIN(33, "gpio33"),
+	PINCTRL_PIN(34, "gpio34"),
+};
+
+static const struct pinctrl_pin_desc oxnas_ox820_pins[] = {
 	PINCTRL_PIN(0, "gpio0"),
 	PINCTRL_PIN(1, "gpio1"),
 	PINCTRL_PIN(2, "gpio2"),
@@ -133,9 +183,24 @@ static const struct pinctrl_pin_desc oxnas_pins[] = {
 	PINCTRL_PIN(32, "gpio32"),
 	PINCTRL_PIN(33, "gpio33"),
 	PINCTRL_PIN(34, "gpio34"),
+	PINCTRL_PIN(35, "gpio35"),
+	PINCTRL_PIN(36, "gpio36"),
+	PINCTRL_PIN(37, "gpio37"),
+	PINCTRL_PIN(38, "gpio38"),
+	PINCTRL_PIN(39, "gpio39"),
+	PINCTRL_PIN(40, "gpio40"),
+	PINCTRL_PIN(41, "gpio41"),
+	PINCTRL_PIN(42, "gpio42"),
+	PINCTRL_PIN(43, "gpio43"),
+	PINCTRL_PIN(44, "gpio44"),
+	PINCTRL_PIN(45, "gpio45"),
+	PINCTRL_PIN(46, "gpio46"),
+	PINCTRL_PIN(47, "gpio47"),
+	PINCTRL_PIN(48, "gpio48"),
+	PINCTRL_PIN(49, "gpio49"),
 };
 
-static const char * const oxnas_fct0_group[] = {
+static const char * const oxnas_ox810se_fct0_group[] = {
 	"gpio0",  "gpio1",  "gpio2",  "gpio3",
 	"gpio4",  "gpio5",  "gpio6",  "gpio7",
 	"gpio8",  "gpio9",  "gpio10", "gpio11",
@@ -147,7 +212,7 @@ static const char * const oxnas_fct0_group[] = {
 	"gpio32", "gpio33", "gpio34"
 };
 
-static const char * const oxnas_fct3_group[] = {
+static const char * const oxnas_ox810se_fct3_group[] = {
 	"gpio0",  "gpio1",  "gpio2",  "gpio3",
 	"gpio4",  "gpio5",  "gpio6",  "gpio7",
 	"gpio8",  "gpio9",
@@ -158,6 +223,40 @@ static const char * const oxnas_fct3_group[] = {
 	"gpio34"
 };
 
+static const char * const oxnas_ox820_fct0_group[] = {
+	"gpio0",  "gpio1",  "gpio2",  "gpio3",
+	"gpio4",  "gpio5",  "gpio6",  "gpio7",
+	"gpio8",  "gpio9",  "gpio10", "gpio11",
+	"gpio12", "gpio13", "gpio14", "gpio15",
+	"gpio16", "gpio17", "gpio18", "gpio19",
+	"gpio20", "gpio21", "gpio22", "gpio23",
+	"gpio24", "gpio25", "gpio26", "gpio27",
+	"gpio28", "gpio29", "gpio30", "gpio31",
+	"gpio32", "gpio33", "gpio34", "gpio35",
+	"gpio36", "gpio37", "gpio38", "gpio39",
+	"gpio40", "gpio41", "gpio42", "gpio43",
+	"gpio44", "gpio45", "gpio46", "gpio47",
+	"gpio48", "gpio49"
+};
+
+static const char * const oxnas_ox820_fct1_group[] = {
+	"gpio3", "gpio4",
+	"gpio12", "gpio13", "gpio14", "gpio15",
+	"gpio16", "gpio17", "gpio18", "gpio19",
+	"gpio20", "gpio21", "gpio22", "gpio23",
+	"gpio24"
+};
+
+static const char * const oxnas_ox820_fct4_group[] = {
+	"gpio5", "gpio6", "gpio7", "gpio8",
+	"gpio24", "gpio25", "gpio26", "gpio27",
+	"gpio40", "gpio41", "gpio42", "gpio43"
+};
+
+static const char * const oxnas_ox820_fct5_group[] = {
+	"gpio28", "gpio29", "gpio30", "gpio31"
+};
+
 #define FUNCTION(_name, _gr)					\
 	{							\
 		.name = #_name,					\
@@ -165,9 +264,16 @@ static const char * const oxnas_fct3_group[] = {
 		.ngroups = ARRAY_SIZE(oxnas_##_gr##_group),	\
 	}
 
-static const struct oxnas_function oxnas_functions[] = {
-	FUNCTION(gpio, fct0),
-	FUNCTION(fct3, fct3),
+static const struct oxnas_function oxnas_ox810se_functions[] = {
+	FUNCTION(gpio, ox810se_fct0),
+	FUNCTION(fct3, ox810se_fct3),
+};
+
+static const struct oxnas_function oxnas_ox820_functions[] = {
+	FUNCTION(gpio, ox820_fct0),
+	FUNCTION(fct1, ox820_fct1),
+	FUNCTION(fct4, ox820_fct4),
+	FUNCTION(fct5, ox820_fct5),
 };
 
 #define OXNAS_PINCTRL_GROUP(_pin, _name, ...)				\
@@ -185,7 +291,7 @@ static const struct oxnas_function oxnas_functions[] = {
 		.fct = _fct,				\
 	}
 
-static const struct oxnas_pin_group oxnas_groups[] = {
+static const struct oxnas_pin_group oxnas_ox810se_groups[] = {
 	OXNAS_PINCTRL_GROUP(0, gpio0,
 			OXNAS_PINCTRL_FUNCTION(gpio, 0),
 			OXNAS_PINCTRL_FUNCTION(fct3, 3)),
@@ -282,6 +388,140 @@ static const struct oxnas_pin_group oxnas_groups[] = {
 			OXNAS_PINCTRL_FUNCTION(fct3, 3)),
 };
 
+static const struct oxnas_pin_group oxnas_ox820_groups[] = {
+	OXNAS_PINCTRL_GROUP(0, gpio0,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(1, gpio1,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(2, gpio2,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(3, gpio3,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(4, gpio4,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(5, gpio5,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct4, 4)),
+	OXNAS_PINCTRL_GROUP(6, gpio6,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct4, 4)),
+	OXNAS_PINCTRL_GROUP(7, gpio7,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct4, 4)),
+	OXNAS_PINCTRL_GROUP(8, gpio8,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct4, 4)),
+	OXNAS_PINCTRL_GROUP(9, gpio9,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(10, gpio10,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(11, gpio11,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(12, gpio12,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(13, gpio13,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(14, gpio14,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(15, gpio15,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(16, gpio16,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(17, gpio17,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(18, gpio18,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(19, gpio19,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(20, gpio20,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(21, gpio21,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(22, gpio22,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(23, gpio23,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1)),
+	OXNAS_PINCTRL_GROUP(24, gpio24,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct1, 1),
+			OXNAS_PINCTRL_FUNCTION(fct4, 5)),
+	OXNAS_PINCTRL_GROUP(25, gpio25,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct4, 4)),
+	OXNAS_PINCTRL_GROUP(26, gpio26,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct4, 4)),
+	OXNAS_PINCTRL_GROUP(27, gpio27,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct4, 4)),
+	OXNAS_PINCTRL_GROUP(28, gpio28,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct5, 5)),
+	OXNAS_PINCTRL_GROUP(29, gpio29,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct5, 5)),
+	OXNAS_PINCTRL_GROUP(30, gpio30,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct5, 5)),
+	OXNAS_PINCTRL_GROUP(31, gpio31,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct5, 5)),
+	OXNAS_PINCTRL_GROUP(32, gpio32,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(33, gpio33,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(34, gpio34,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(35, gpio35,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(36, gpio36,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(37, gpio37,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(38, gpio38,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(39, gpio39,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(40, gpio40,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct4, 4)),
+	OXNAS_PINCTRL_GROUP(41, gpio41,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct4, 4)),
+	OXNAS_PINCTRL_GROUP(42, gpio42,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct4, 4)),
+	OXNAS_PINCTRL_GROUP(43, gpio43,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0),
+			OXNAS_PINCTRL_FUNCTION(fct4, 4)),
+	OXNAS_PINCTRL_GROUP(44, gpio44,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(45, gpio45,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(46, gpio46,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(47, gpio47,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(48, gpio48,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+	OXNAS_PINCTRL_GROUP(49, gpio49,
+			OXNAS_PINCTRL_FUNCTION(gpio, 0)),
+};
+
 static inline struct oxnas_gpio_bank *pctl_to_bank(struct oxnas_pinctrl *pctl,
 						   unsigned int pin)
 {
@@ -352,8 +592,8 @@ static int oxnas_pinmux_get_function_groups(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
-static int oxnas_pinmux_enable(struct pinctrl_dev *pctldev,
-			       unsigned int func, unsigned int group)
+static int oxnas_ox810se_pinmux_enable(struct pinctrl_dev *pctldev,
+				       unsigned int func, unsigned int group)
 {
 	struct oxnas_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 	const struct oxnas_pin_group *pg = &pctl->groups[group];
@@ -371,22 +611,22 @@ static int oxnas_pinmux_enable(struct pinctrl_dev *pctldev,
 
 			regmap_write_bits(pctl->regmap,
 					  (pg->bank ?
-						PINMUX_PRIMARY_SEL1 :
-						PINMUX_PRIMARY_SEL0),
+						PINMUX_810_PRIMARY_SEL1 :
+						PINMUX_810_PRIMARY_SEL0),
 					  mask,
 					  (functions->fct == 1 ?
 						mask : 0));
 			regmap_write_bits(pctl->regmap,
 					  (pg->bank ?
-						PINMUX_SECONDARY_SEL1 :
-						PINMUX_SECONDARY_SEL0),
+						PINMUX_810_SECONDARY_SEL1 :
+						PINMUX_810_SECONDARY_SEL0),
 					  mask,
 					  (functions->fct == 2 ?
 						mask : 0));
 			regmap_write_bits(pctl->regmap,
 					  (pg->bank ?
-						PINMUX_TERTIARY_SEL1 :
-						PINMUX_TERTIARY_SEL0),
+						PINMUX_810_TERTIARY_SEL1 :
+						PINMUX_810_TERTIARY_SEL0),
 					  mask,
 					  (functions->fct == 3 ?
 						mask : 0));
@@ -402,9 +642,64 @@ static int oxnas_pinmux_enable(struct pinctrl_dev *pctldev,
 	return -EINVAL;
 }
 
-static int oxnas_gpio_request_enable(struct pinctrl_dev *pctldev,
-				     struct pinctrl_gpio_range *range,
-				     unsigned int offset)
+static int oxnas_ox820_pinmux_enable(struct pinctrl_dev *pctldev,
+				     unsigned int func, unsigned int group)
+{
+	struct oxnas_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	const struct oxnas_pin_group *pg = &pctl->groups[group];
+	const struct oxnas_function *pf = &pctl->functions[func];
+	const char *fname = pf->name;
+	struct oxnas_desc_function *functions = pg->functions;
+	unsigned int offset = (pg->bank ? PINMUX_820_BANK_OFFSET : 0);
+	u32 mask = BIT(pg->pin);
+
+	while (functions->name) {
+		if (!strcmp(functions->name, fname)) {
+			dev_dbg(pctl->dev,
+				"setting function %s bank %d pin %d fct %d mask %x\n",
+				fname, pg->bank, pg->pin,
+				functions->fct, mask);
+
+			regmap_write_bits(pctl->regmap,
+					  offset + PINMUX_820_SECONDARY_SEL,
+					  mask,
+					  (functions->fct == 1 ?
+						mask : 0));
+			regmap_write_bits(pctl->regmap,
+					  offset + PINMUX_820_TERTIARY_SEL,
+					  mask,
+					  (functions->fct == 2 ?
+						mask : 0));
+			regmap_write_bits(pctl->regmap,
+					  offset + PINMUX_820_QUATERNARY_SEL,
+					  mask,
+					  (functions->fct == 3 ?
+						mask : 0));
+			regmap_write_bits(pctl->regmap,
+					  offset + PINMUX_820_DEBUG_SEL,
+					  mask,
+					  (functions->fct == 4 ?
+						mask : 0));
+			regmap_write_bits(pctl->regmap,
+					  offset + PINMUX_820_ALTERNATIVE_SEL,
+					  mask,
+					  (functions->fct == 5 ?
+						mask : 0));
+
+			return 0;
+		}
+
+		functions++;
+	}
+
+	dev_err(pctl->dev, "cannot mux pin %u to function %u\n", group, func);
+
+	return -EINVAL;
+}
+
+static int oxnas_ox810se_gpio_request_enable(struct pinctrl_dev *pctldev,
+					     struct pinctrl_gpio_range *range,
+					     unsigned int offset)
 {
 	struct oxnas_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 	struct oxnas_gpio_bank *bank = gpiochip_get_data(range->gc);
@@ -415,18 +710,49 @@ static int oxnas_gpio_request_enable(struct pinctrl_dev *pctldev,
 
 	regmap_write_bits(pctl->regmap,
 			  (bank->id ?
-				PINMUX_PRIMARY_SEL1 :
-				PINMUX_PRIMARY_SEL0),
+				PINMUX_810_PRIMARY_SEL1 :
+				PINMUX_810_PRIMARY_SEL0),
 			  mask, 0);
 	regmap_write_bits(pctl->regmap,
 			  (bank->id ?
-				PINMUX_SECONDARY_SEL1 :
-				PINMUX_SECONDARY_SEL0),
+				PINMUX_810_SECONDARY_SEL1 :
+				PINMUX_810_SECONDARY_SEL0),
 			  mask, 0);
 	regmap_write_bits(pctl->regmap,
 			  (bank->id ?
-				PINMUX_TERTIARY_SEL1 :
-				PINMUX_TERTIARY_SEL0),
+				PINMUX_810_TERTIARY_SEL1 :
+				PINMUX_810_TERTIARY_SEL0),
+			  mask, 0);
+
+	return 0;
+}
+
+static int oxnas_ox820_gpio_request_enable(struct pinctrl_dev *pctldev,
+					   struct pinctrl_gpio_range *range,
+					   unsigned int offset)
+{
+	struct oxnas_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct oxnas_gpio_bank *bank = gpiochip_get_data(range->gc);
+	unsigned int bank_offset = (bank->id ? PINMUX_820_BANK_OFFSET : 0);
+	u32 mask = BIT(offset - bank->gpio_chip.base);
+
+	dev_dbg(pctl->dev, "requesting gpio %d in bank %d (id %d) with mask 0x%x\n",
+		offset, bank->gpio_chip.base, bank->id, mask);
+
+	regmap_write_bits(pctl->regmap,
+			  bank_offset + PINMUX_820_SECONDARY_SEL,
+			  mask, 0);
+	regmap_write_bits(pctl->regmap,
+			  bank_offset + PINMUX_820_TERTIARY_SEL,
+			  mask, 0);
+	regmap_write_bits(pctl->regmap,
+			  bank_offset + PINMUX_820_QUATERNARY_SEL,
+			  mask, 0);
+	regmap_write_bits(pctl->regmap,
+			  bank_offset + PINMUX_820_DEBUG_SEL,
+			  mask, 0);
+	regmap_write_bits(pctl->regmap,
+			  bank_offset + PINMUX_820_ALTERNATIVE_SEL,
 			  mask, 0);
 
 	return 0;
@@ -498,17 +824,26 @@ static int oxnas_gpio_set_direction(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
-static const struct pinmux_ops oxnas_pinmux_ops = {
+static const struct pinmux_ops oxnas_ox810se_pinmux_ops = {
 	.get_functions_count = oxnas_pinmux_get_functions_count,
 	.get_function_name = oxnas_pinmux_get_function_name,
 	.get_function_groups = oxnas_pinmux_get_function_groups,
-	.set_mux = oxnas_pinmux_enable,
-	.gpio_request_enable = oxnas_gpio_request_enable,
+	.set_mux = oxnas_ox810se_pinmux_enable,
+	.gpio_request_enable = oxnas_ox810se_gpio_request_enable,
 	.gpio_set_direction = oxnas_gpio_set_direction,
 };
 
-static int oxnas_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin,
-			     unsigned long *config)
+static const struct pinmux_ops oxnas_ox820_pinmux_ops = {
+	.get_functions_count = oxnas_pinmux_get_functions_count,
+	.get_function_name = oxnas_pinmux_get_function_name,
+	.get_function_groups = oxnas_pinmux_get_function_groups,
+	.set_mux = oxnas_ox820_pinmux_enable,
+	.gpio_request_enable = oxnas_ox820_gpio_request_enable,
+	.gpio_set_direction = oxnas_gpio_set_direction,
+};
+
+static int oxnas_ox810se_pinconf_get(struct pinctrl_dev *pctldev,
+				     unsigned int pin, unsigned long *config)
 {
 	struct oxnas_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 	struct oxnas_gpio_bank *bank = pctl_to_bank(pctl, pin);
@@ -521,8 +856,38 @@ static int oxnas_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin,
 	case PIN_CONFIG_BIAS_PULL_UP:
 		ret = regmap_read(pctl->regmap,
 				  (bank->id ?
-					PINMUX_PULLUP_CTRL1 :
-					PINMUX_PULLUP_CTRL0),
+					PINMUX_810_PULLUP_CTRL1 :
+					PINMUX_810_PULLUP_CTRL0),
+				  &arg);
+		if (ret)
+			return ret;
+
+		arg = !!(arg & mask);
+		break;
+	default:
+		return -ENOTSUPP;
+	}
+
+	*config = pinconf_to_config_packed(param, arg);
+
+	return 0;
+}
+
+static int oxnas_ox820_pinconf_get(struct pinctrl_dev *pctldev,
+				   unsigned int pin, unsigned long *config)
+{
+	struct oxnas_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct oxnas_gpio_bank *bank = pctl_to_bank(pctl, pin);
+	unsigned int param = pinconf_to_config_param(*config);
+	unsigned int bank_offset = (bank->id ? PINMUX_820_BANK_OFFSET : 0);
+	u32 mask = BIT(pin - bank->gpio_chip.base);
+	int ret;
+	u32 arg;
+
+	switch (param) {
+	case PIN_CONFIG_BIAS_PULL_UP:
+		ret = regmap_read(pctl->regmap,
+				  bank_offset + PINMUX_820_PULLUP_CTRL,
 				  &arg);
 		if (ret)
 			return ret;
@@ -538,8 +903,9 @@ static int oxnas_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin,
 	return 0;
 }
 
-static int oxnas_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
-			     unsigned long *configs, unsigned int num_configs)
+static int oxnas_ox810se_pinconf_set(struct pinctrl_dev *pctldev,
+				     unsigned int pin, unsigned long *configs,
+				     unsigned int num_configs)
 {
 	struct oxnas_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 	struct oxnas_gpio_bank *bank = pctl_to_bank(pctl, pin);
@@ -561,8 +927,8 @@ static int oxnas_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
 			dev_dbg(pctl->dev, "   pullup\n");
 			regmap_write_bits(pctl->regmap,
 					  (bank->id ?
-						PINMUX_PULLUP_CTRL1 :
-						PINMUX_PULLUP_CTRL0),
+						PINMUX_810_PULLUP_CTRL1 :
+						PINMUX_810_PULLUP_CTRL0),
 					  mask, mask);
 			break;
 		default:
@@ -575,18 +941,53 @@ static int oxnas_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
 	return 0;
 }
 
-static const struct pinconf_ops oxnas_pinconf_ops = {
-	.pin_config_get = oxnas_pinconf_get,
-	.pin_config_set = oxnas_pinconf_set,
+static int oxnas_ox820_pinconf_set(struct pinctrl_dev *pctldev,
+				   unsigned int pin, unsigned long *configs,
+				   unsigned int num_configs)
+{
+	struct oxnas_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct oxnas_gpio_bank *bank = pctl_to_bank(pctl, pin);
+	unsigned int bank_offset = (bank->id ? PINMUX_820_BANK_OFFSET : 0);
+	unsigned int param;
+	u32 arg;
+	unsigned int i;
+	u32 offset = pin - bank->gpio_chip.base;
+	u32 mask = BIT(offset);
+
+	dev_dbg(pctl->dev, "setting pin %d bank %d mask 0x%x\n",
+		pin, bank->gpio_chip.base, mask);
+
+	for (i = 0; i < num_configs; i++) {
+		param = pinconf_to_config_param(configs[i]);
+		arg = pinconf_to_config_argument(configs[i]);
+
+		switch (param) {
+		case PIN_CONFIG_BIAS_PULL_UP:
+			dev_dbg(pctl->dev, "   pullup\n");
+			regmap_write_bits(pctl->regmap,
+					  bank_offset + PINMUX_820_PULLUP_CTRL,
+					  mask, mask);
+			break;
+		default:
+			dev_err(pctl->dev, "Property %u not supported\n",
+				param);
+			return -ENOTSUPP;
+		}
+	}
+
+	return 0;
+}
+
+static const struct pinconf_ops oxnas_ox810se_pinconf_ops = {
+	.pin_config_get = oxnas_ox810se_pinconf_get,
+	.pin_config_set = oxnas_ox810se_pinconf_set,
 	.is_generic = true,
 };
 
-static struct pinctrl_desc oxnas_pinctrl_desc = {
-	.name = "oxnas-pinctrl",
-	.pctlops = &oxnas_pinctrl_ops,
-	.pmxops = &oxnas_pinmux_ops,
-	.confops = &oxnas_pinconf_ops,
-	.owner = THIS_MODULE,
+static const struct pinconf_ops oxnas_ox820_pinconf_ops = {
+	.pin_config_get = oxnas_ox820_pinconf_get,
+	.pin_config_set = oxnas_ox820_pinconf_set,
+	.is_generic = true,
 };
 
 static void oxnas_gpio_irq_ack(struct irq_data *data)
@@ -699,10 +1100,78 @@ static struct oxnas_gpio_bank oxnas_gpio_banks[] = {
 	GPIO_BANK(1),
 };
 
+static struct oxnas_pinctrl ox810se_pinctrl = {
+	.functions = oxnas_ox810se_functions,
+	.nfunctions = ARRAY_SIZE(oxnas_ox810se_functions),
+	.groups = oxnas_ox810se_groups,
+	.ngroups = ARRAY_SIZE(oxnas_ox810se_groups),
+	.gpio_banks = oxnas_gpio_banks,
+	.nbanks = ARRAY_SIZE(oxnas_gpio_banks),
+};
+
+static struct pinctrl_desc oxnas_ox810se_pinctrl_desc = {
+	.name = "oxnas-pinctrl",
+	.pins = oxnas_ox810se_pins,
+	.npins = ARRAY_SIZE(oxnas_ox810se_pins),
+	.pctlops = &oxnas_pinctrl_ops,
+	.pmxops = &oxnas_ox810se_pinmux_ops,
+	.confops = &oxnas_ox810se_pinconf_ops,
+	.owner = THIS_MODULE,
+};
+
+static struct oxnas_pinctrl ox820_pinctrl = {
+	.functions = oxnas_ox820_functions,
+	.nfunctions = ARRAY_SIZE(oxnas_ox820_functions),
+	.groups = oxnas_ox820_groups,
+	.ngroups = ARRAY_SIZE(oxnas_ox820_groups),
+	.gpio_banks = oxnas_gpio_banks,
+	.nbanks = ARRAY_SIZE(oxnas_gpio_banks),
+};
+
+static struct pinctrl_desc oxnas_ox820_pinctrl_desc = {
+	.name = "oxnas-pinctrl",
+	.pins = oxnas_ox820_pins,
+	.npins = ARRAY_SIZE(oxnas_ox820_pins),
+	.pctlops = &oxnas_pinctrl_ops,
+	.pmxops = &oxnas_ox820_pinmux_ops,
+	.confops = &oxnas_ox820_pinconf_ops,
+	.owner = THIS_MODULE,
+};
+
+static struct oxnas_pinctrl_data oxnas_ox810se_pinctrl_data = {
+	.desc = &oxnas_ox810se_pinctrl_desc,
+	.pctl = &ox810se_pinctrl,
+};
+
+static struct oxnas_pinctrl_data oxnas_ox820_pinctrl_data = {
+	.desc = &oxnas_ox820_pinctrl_desc,
+	.pctl = &ox820_pinctrl,
+};
+
+static const struct of_device_id oxnas_pinctrl_of_match[] = {
+	{ .compatible = "oxsemi,ox810se-pinctrl",
+	  .data = &oxnas_ox810se_pinctrl_data
+	},
+	{ .compatible = "oxsemi,ox820-pinctrl",
+	  .data = &oxnas_ox820_pinctrl_data,
+	},
+	{ },
+};
+
 static int oxnas_pinctrl_probe(struct platform_device *pdev)
 {
+	const struct of_device_id *id;
+	const struct oxnas_pinctrl_data *data;
 	struct oxnas_pinctrl *pctl;
 
+	id = of_match_node(oxnas_pinctrl_of_match, pdev->dev.of_node);
+	if (!id)
+		return -ENODEV;
+
+	data = id->data;
+	if (!data || !data->pctl || !data->desc)
+		return -EINVAL;
+
 	pctl = devm_kzalloc(&pdev->dev, sizeof(*pctl), GFP_KERNEL);
 	if (!pctl)
 		return -ENOMEM;
@@ -716,20 +1185,14 @@ static int oxnas_pinctrl_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
-	pctl->pins = oxnas_pins;
-	pctl->npins = ARRAY_SIZE(oxnas_pins);
-	pctl->functions = oxnas_functions;
-	pctl->nfunctions = ARRAY_SIZE(oxnas_functions);
-	pctl->groups = oxnas_groups;
-	pctl->ngroups = ARRAY_SIZE(oxnas_groups);
-	pctl->gpio_banks = oxnas_gpio_banks;
-	pctl->nbanks = ARRAY_SIZE(oxnas_gpio_banks);
-
-	oxnas_pinctrl_desc.pins = pctl->pins;
-	oxnas_pinctrl_desc.npins = pctl->npins;
+	pctl->functions = data->pctl->functions;
+	pctl->nfunctions = data->pctl->nfunctions;
+	pctl->groups = data->pctl->groups;
+	pctl->ngroups = data->pctl->ngroups;
+	pctl->gpio_banks = data->pctl->gpio_banks;
+	pctl->nbanks = data->pctl->nbanks;
 
-	pctl->pctldev = pinctrl_register(&oxnas_pinctrl_desc,
-					 &pdev->dev, pctl);
+	pctl->pctldev = pinctrl_register(data->desc, &pdev->dev, pctl);
 	if (IS_ERR(pctl->pctldev)) {
 		dev_err(&pdev->dev, "Failed to register pinctrl device\n");
 		return PTR_ERR(pctl->pctldev);
@@ -805,11 +1268,6 @@ static int oxnas_gpio_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static const struct of_device_id oxnas_pinctrl_of_match[] = {
-	{ .compatible = "oxsemi,ox810se-pinctrl", },
-	{ },
-};
-
 static struct platform_driver oxnas_pinctrl_driver = {
 	.driver = {
 		.name = "oxnas-pinctrl",
@@ -821,6 +1279,7 @@ static struct platform_driver oxnas_pinctrl_driver = {
 
 static const struct of_device_id oxnas_gpio_of_match[] = {
 	{ .compatible = "oxsemi,ox810se-gpio", },
+	{ .compatible = "oxsemi,ox820-gpio", },
 	{ },
 };
 
diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c
index 49bf7dcb7ed8..08765f58253c 100644
--- a/drivers/pinctrl/pinctrl-rockchip.c
+++ b/drivers/pinctrl/pinctrl-rockchip.c
@@ -59,6 +59,7 @@
 #define GPIO_LS_SYNC		0x60
 
 enum rockchip_pinctrl_type {
+	RK1108,
 	RK2928,
 	RK3066B,
 	RK3188,
@@ -624,6 +625,65 @@ static int rockchip_set_mux(struct rockchip_pin_bank *bank, int pin, int mux)
 	return ret;
 }
 
+#define RK1108_PULL_PMU_OFFSET		0x10
+#define RK1108_PULL_OFFSET		0x110
+#define RK1108_PULL_PINS_PER_REG	8
+#define RK1108_PULL_BITS_PER_PIN	2
+#define RK1108_PULL_BANK_STRIDE		16
+
+static void rk1108_calc_pull_reg_and_bit(struct rockchip_pin_bank *bank,
+					 int pin_num, struct regmap **regmap,
+					 int *reg, u8 *bit)
+{
+	struct rockchip_pinctrl *info = bank->drvdata;
+
+	/* The first 24 pins of the first bank are located in PMU */
+	if (bank->bank_num == 0) {
+		*regmap = info->regmap_pmu;
+		*reg = RK1108_PULL_PMU_OFFSET;
+	} else {
+		*reg = RK1108_PULL_OFFSET;
+		*regmap = info->regmap_base;
+		/* correct the offset, as we're starting with the 2nd bank */
+		*reg -= 0x10;
+		*reg += bank->bank_num * RK1108_PULL_BANK_STRIDE;
+	}
+
+	*reg += ((pin_num / RK1108_PULL_PINS_PER_REG) * 4);
+	*bit = (pin_num % RK1108_PULL_PINS_PER_REG);
+	*bit *= RK1108_PULL_BITS_PER_PIN;
+}
+
+#define RK1108_DRV_PMU_OFFSET		0x20
+#define RK1108_DRV_GRF_OFFSET		0x210
+#define RK1108_DRV_BITS_PER_PIN		2
+#define RK1108_DRV_PINS_PER_REG		8
+#define RK1108_DRV_BANK_STRIDE		16
+
+static void rk1108_calc_drv_reg_and_bit(struct rockchip_pin_bank *bank,
+					int pin_num, struct regmap **regmap,
+					int *reg, u8 *bit)
+{
+	struct rockchip_pinctrl *info = bank->drvdata;
+
+	/* The first 24 pins of the first bank are located in PMU */
+	if (bank->bank_num == 0) {
+		*regmap = info->regmap_pmu;
+		*reg = RK1108_DRV_PMU_OFFSET;
+	} else {
+		*regmap = info->regmap_base;
+		*reg = RK1108_DRV_GRF_OFFSET;
+
+		/* correct the offset, as we're starting with the 2nd bank */
+		*reg -= 0x10;
+		*reg += bank->bank_num * RK1108_DRV_BANK_STRIDE;
+	}
+
+	*reg += ((pin_num / RK1108_DRV_PINS_PER_REG) * 4);
+	*bit = pin_num % RK1108_DRV_PINS_PER_REG;
+	*bit *= RK1108_DRV_BITS_PER_PIN;
+}
+
 #define RK2928_PULL_OFFSET		0x118
 #define RK2928_PULL_PINS_PER_REG	16
 #define RK2928_PULL_BANK_STRIDE		8
@@ -1123,6 +1183,7 @@ static int rockchip_get_pull(struct rockchip_pin_bank *bank, int pin_num)
 		return !(data & BIT(bit))
 				? PIN_CONFIG_BIAS_PULL_PIN_DEFAULT
 				: PIN_CONFIG_BIAS_DISABLE;
+	case RK1108:
 	case RK3188:
 	case RK3288:
 	case RK3368:
@@ -1169,6 +1230,7 @@ static int rockchip_set_pull(struct rockchip_pin_bank *bank,
 
 		spin_unlock_irqrestore(&bank->slock, flags);
 		break;
+	case RK1108:
 	case RK3188:
 	case RK3288:
 	case RK3368:
@@ -1358,6 +1420,7 @@ static bool rockchip_pinconf_pull_valid(struct rockchip_pin_ctrl *ctrl,
 					pull == PIN_CONFIG_BIAS_DISABLE);
 	case RK3066B:
 		return pull ? false : true;
+	case RK1108:
 	case RK3188:
 	case RK3288:
 	case RK3368:
@@ -2455,6 +2518,27 @@ static int rockchip_pinctrl_probe(struct platform_device *pdev)
 	return 0;
 }
 
+static struct rockchip_pin_bank rk1108_pin_banks[] = {
+	PIN_BANK_IOMUX_FLAGS(0, 32, "gpio0", IOMUX_SOURCE_PMU,
+					     IOMUX_SOURCE_PMU,
+					     IOMUX_SOURCE_PMU,
+					     IOMUX_SOURCE_PMU),
+	PIN_BANK_IOMUX_FLAGS(1, 32, "gpio1", 0, 0, 0, 0),
+	PIN_BANK_IOMUX_FLAGS(2, 32, "gpio2", 0, 0, 0, 0),
+	PIN_BANK_IOMUX_FLAGS(3, 32, "gpio3", 0, 0, 0, 0),
+};
+
+static struct rockchip_pin_ctrl rk1108_pin_ctrl = {
+	.pin_banks		= rk1108_pin_banks,
+	.nr_banks		= ARRAY_SIZE(rk1108_pin_banks),
+	.label			= "RK1108-GPIO",
+	.type			= RK1108,
+	.grf_mux_offset		= 0x10,
+	.pmu_mux_offset		= 0x0,
+	.pull_calc_reg		= rk1108_calc_pull_reg_and_bit,
+	.drv_calc_reg		= rk1108_calc_drv_reg_and_bit,
+};
+
 static struct rockchip_pin_bank rk2928_pin_banks[] = {
 	PIN_BANK(0, 32, "gpio0"),
 	PIN_BANK(1, 32, "gpio1"),
@@ -2684,6 +2768,8 @@ static struct rockchip_pin_ctrl rk3399_pin_ctrl = {
 };
 
 static const struct of_device_id rockchip_pinctrl_dt_match[] = {
+	{ .compatible = "rockchip,rk1108-pinctrl",
+		.data = (void *)&rk1108_pin_ctrl },
 	{ .compatible = "rockchip,rk2928-pinctrl",
 		.data = (void *)&rk2928_pin_ctrl },
 	{ .compatible = "rockchip,rk3036-pinctrl",
diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c
index bfdf720db270..a5a0392ab817 100644
--- a/drivers/pinctrl/pinctrl-single.c
+++ b/drivers/pinctrl/pinctrl-single.c
@@ -31,12 +31,10 @@
 #include <linux/platform_data/pinctrl-single.h>
 
 #include "core.h"
+#include "devicetree.h"
 #include "pinconf.h"
 
 #define DRIVER_NAME			"pinctrl-single"
-#define PCS_MUX_PINS_NAME		"pinctrl-single,pins"
-#define PCS_MUX_BITS_NAME		"pinctrl-single,bits"
-#define PCS_REG_NAME_LEN		((sizeof(unsigned long) * 2) + 3)
 #define PCS_OFF_DISABLED		~0U
 
 /**
@@ -142,20 +140,6 @@ struct pcs_data {
 };
 
 /**
- * struct pcs_name - register name for a pin
- * @name:	name of the pinctrl register
- *
- * REVISIT: We may want to make names optional in the pinctrl
- * framework as some drivers may not care about pin names to
- * avoid kernel bloat. The pin names can be deciphered by user
- * space tools using debugfs based on the register address and
- * SoC packaging information.
- */
-struct pcs_name {
-	char name[PCS_REG_NAME_LEN];
-};
-
-/**
  * struct pcs_soc_data - SoC specific settings
  * @flags:	initial SoC specific PCS_FEAT_xxx values
  * @irq:	optional interrupt for the controller
@@ -177,8 +161,11 @@ struct pcs_soc_data {
  * @base:	virtual address of the controller
  * @size:	size of the ioremapped area
  * @dev:	device entry
+ * @np:		device tree node
  * @pctl:	pin controller device
  * @flags:	mask of PCS_FEAT_xxx values
+ * @missing_nr_pinctrl_cells: for legacy binding, may go away
+ * @socdata:	soc specific data
  * @lock:	spinlock for register access
  * @mutex:	mutex protecting the lists
  * @width:	bits per mux register
@@ -186,8 +173,8 @@ struct pcs_soc_data {
  * @fshift:	function register shift
  * @foff:	value to turn mux off
  * @fmax:	max number of functions in fmask
- * @bits_per_pin:number of bits per pin
- * @names:	array of register names for pins
+ * @bits_per_mux: number of bits per mux
+ * @bits_per_pin: number of bits per pin
  * @pins:	physical pins on the SoC
  * @pgtree:	pingroup index radix tree
  * @ftree:	function index radix tree
@@ -208,11 +195,13 @@ struct pcs_device {
 	void __iomem *base;
 	unsigned size;
 	struct device *dev;
+	struct device_node *np;
 	struct pinctrl_dev *pctl;
 	unsigned flags;
 #define PCS_QUIRK_SHARED_IRQ	(1 << 2)
 #define PCS_FEAT_IRQ		(1 << 1)
 #define PCS_FEAT_PINCONF	(1 << 0)
+	struct property *missing_nr_pinctrl_cells;
 	struct pcs_soc_data socdata;
 	raw_spinlock_t lock;
 	struct mutex mutex;
@@ -223,7 +212,6 @@ struct pcs_device {
 	unsigned fmax;
 	bool bits_per_mux;
 	unsigned bits_per_pin;
-	struct pcs_name *names;
 	struct pcs_data pins;
 	struct radix_tree_root pgtree;
 	struct radix_tree_root ftree;
@@ -354,13 +342,17 @@ static void pcs_pin_dbg_show(struct pinctrl_dev *pctldev,
 {
 	struct pcs_device *pcs;
 	unsigned val, mux_bytes;
+	unsigned long offset;
+	size_t pa;
 
 	pcs = pinctrl_dev_get_drvdata(pctldev);
 
 	mux_bytes = pcs->width / BITS_PER_BYTE;
-	val = pcs->read(pcs->base + pin * mux_bytes);
+	offset = pin * mux_bytes;
+	val = pcs->read(pcs->base + offset);
+	pa = pcs->res->start + offset;
 
-	seq_printf(s, "%08x %s " , val, DRIVER_NAME);
+	seq_printf(s, "%zx %08x %s ", pa, val, DRIVER_NAME);
 }
 
 static void pcs_dt_free_map(struct pinctrl_dev *pctldev,
@@ -763,7 +755,6 @@ static int pcs_add_pin(struct pcs_device *pcs, unsigned offset,
 {
 	struct pcs_soc_data *pcs_soc = &pcs->socdata;
 	struct pinctrl_pin_desc *pin;
-	struct pcs_name *pn;
 	int i;
 
 	i = pcs->pins.cur;
@@ -786,10 +777,6 @@ static int pcs_add_pin(struct pcs_device *pcs, unsigned offset,
 	}
 
 	pin = &pcs->pins.pa[i];
-	pn = &pcs->names[i];
-	sprintf(pn->name, "%lx.%u",
-		(unsigned long)pcs->res->start + offset, pin_pos);
-	pin->name = pn->name;
 	pin->number = i;
 	pcs->pins.cur++;
 
@@ -827,12 +814,6 @@ static int pcs_allocate_pin_table(struct pcs_device *pcs)
 	if (!pcs->pins.pa)
 		return -ENOMEM;
 
-	pcs->names = devm_kzalloc(pcs->dev,
-				sizeof(struct pcs_name) * nr_pins,
-				GFP_KERNEL);
-	if (!pcs->names)
-		return -ENOMEM;
-
 	pcs->desc.pins = pcs->pins.pa;
 	pcs->desc.npins = nr_pins;
 
@@ -1146,21 +1127,17 @@ static int pcs_parse_one_pinctrl_entry(struct pcs_device *pcs,
 						unsigned *num_maps,
 						const char **pgnames)
 {
+	const char *name = "pinctrl-single,pins";
 	struct pcs_func_vals *vals;
-	const __be32 *mux;
-	int size, rows, *pins, index = 0, found = 0, res = -ENOMEM;
+	int rows, *pins, found = 0, res = -ENOMEM, i;
 	struct pcs_function *function;
 
-	mux = of_get_property(np, PCS_MUX_PINS_NAME, &size);
-	if ((!mux) || (size < sizeof(*mux) * 2)) {
-		dev_err(pcs->dev, "bad data for mux %s\n",
-			np->name);
+	rows = pinctrl_count_index_with_args(np, name);
+	if (rows <= 0) {
+		dev_err(pcs->dev, "Ivalid number of rows: %d\n", rows);
 		return -EINVAL;
 	}
 
-	size /= sizeof(*mux);	/* Number of elements in array */
-	rows = size / 2;
-
 	vals = devm_kzalloc(pcs->dev, sizeof(*vals) * rows, GFP_KERNEL);
 	if (!vals)
 		return -ENOMEM;
@@ -1169,14 +1146,28 @@ static int pcs_parse_one_pinctrl_entry(struct pcs_device *pcs,
 	if (!pins)
 		goto free_vals;
 
-	while (index < size) {
-		unsigned offset, val;
+	for (i = 0; i < rows; i++) {
+		struct of_phandle_args pinctrl_spec;
+		unsigned int offset;
 		int pin;
 
-		offset = be32_to_cpup(mux + index++);
-		val = be32_to_cpup(mux + index++);
+		res = pinctrl_parse_index_with_args(np, name, i, &pinctrl_spec);
+		if (res)
+			return res;
+
+		if (pinctrl_spec.args_count < 2) {
+			dev_err(pcs->dev, "invalid args_count for spec: %i\n",
+				pinctrl_spec.args_count);
+			break;
+		}
+
+		/* Index plus one value cell */
+		offset = pinctrl_spec.args[0];
 		vals[found].reg = pcs->base + offset;
-		vals[found].val = val;
+		vals[found].val = pinctrl_spec.args[1];
+
+		dev_dbg(pcs->dev, "%s index: 0x%x value: 0x%x\n",
+			pinctrl_spec.np->name, offset, pinctrl_spec.args[1]);
 
 		pin = pcs_get_pin_by_offset(pcs, offset);
 		if (pin < 0) {
@@ -1190,8 +1181,10 @@ static int pcs_parse_one_pinctrl_entry(struct pcs_device *pcs,
 
 	pgnames[0] = np->name;
 	function = pcs_add_function(pcs, np, np->name, vals, found, pgnames, 1);
-	if (!function)
+	if (!function) {
+		res = -ENOMEM;
 		goto free_pins;
+	}
 
 	res = pcs_add_pingroup(pcs, np, np->name, pins, found);
 	if (res < 0)
@@ -1226,36 +1219,24 @@ free_vals:
 	return res;
 }
 
-#define PARAMS_FOR_BITS_PER_MUX 3
-
 static int pcs_parse_bits_in_pinctrl_entry(struct pcs_device *pcs,
 						struct device_node *np,
 						struct pinctrl_map **map,
 						unsigned *num_maps,
 						const char **pgnames)
 {
+	const char *name = "pinctrl-single,bits";
 	struct pcs_func_vals *vals;
-	const __be32 *mux;
-	int size, rows, *pins, index = 0, found = 0, res = -ENOMEM;
+	int rows, *pins, found = 0, res = -ENOMEM, i;
 	int npins_in_row;
 	struct pcs_function *function;
 
-	mux = of_get_property(np, PCS_MUX_BITS_NAME, &size);
-
-	if (!mux) {
-		dev_err(pcs->dev, "no valid property for %s\n", np->name);
-		return -EINVAL;
-	}
-
-	if (size < (sizeof(*mux) * PARAMS_FOR_BITS_PER_MUX)) {
-		dev_err(pcs->dev, "bad data for %s\n", np->name);
+	rows = pinctrl_count_index_with_args(np, name);
+	if (rows <= 0) {
+		dev_err(pcs->dev, "Invalid number of rows: %d\n", rows);
 		return -EINVAL;
 	}
 
-	/* Number of elements in array */
-	size /= sizeof(*mux);
-
-	rows = size / PARAMS_FOR_BITS_PER_MUX;
 	npins_in_row = pcs->width / pcs->bits_per_pin;
 
 	vals = devm_kzalloc(pcs->dev, sizeof(*vals) * rows * npins_in_row,
@@ -1268,15 +1249,30 @@ static int pcs_parse_bits_in_pinctrl_entry(struct pcs_device *pcs,
 	if (!pins)
 		goto free_vals;
 
-	while (index < size) {
+	for (i = 0; i < rows; i++) {
+		struct of_phandle_args pinctrl_spec;
 		unsigned offset, val;
 		unsigned mask, bit_pos, val_pos, mask_pos, submask;
 		unsigned pin_num_from_lsb;
 		int pin;
 
-		offset = be32_to_cpup(mux + index++);
-		val = be32_to_cpup(mux + index++);
-		mask = be32_to_cpup(mux + index++);
+		res = pinctrl_parse_index_with_args(np, name, i, &pinctrl_spec);
+		if (res)
+			return res;
+
+		if (pinctrl_spec.args_count < 3) {
+			dev_err(pcs->dev, "invalid args_count for spec: %i\n",
+				pinctrl_spec.args_count);
+			break;
+		}
+
+		/* Index plus two value cells */
+		offset = pinctrl_spec.args[0];
+		val = pinctrl_spec.args[1];
+		mask = pinctrl_spec.args[2];
+
+		dev_dbg(pcs->dev, "%s index: 0x%x value: 0x%x mask: 0x%x\n",
+			pinctrl_spec.np->name, offset, val, mask);
 
 		/* Parse pins in each row from LSB */
 		while (mask) {
@@ -1319,8 +1315,10 @@ static int pcs_parse_bits_in_pinctrl_entry(struct pcs_device *pcs,
 
 	pgnames[0] = np->name;
 	function = pcs_add_function(pcs, np, np->name, vals, found, pgnames, 1);
-	if (!function)
+	if (!function) {
+		res = -ENOMEM;
 		goto free_pins;
+	}
 
 	res = pcs_add_pingroup(pcs, np, np->name, pins, found);
 	if (res < 0)
@@ -1494,17 +1492,12 @@ static void pcs_free_resources(struct pcs_device *pcs)
 	pinctrl_unregister(pcs->pctl);
 	pcs_free_funcs(pcs);
 	pcs_free_pingroups(pcs);
+#if IS_BUILTIN(CONFIG_PINCTRL_SINGLE)
+	if (pcs->missing_nr_pinctrl_cells)
+		of_remove_property(pcs->np, pcs->missing_nr_pinctrl_cells);
+#endif
 }
 
-#define PCS_GET_PROP_U32(name, reg, err)				\
-	do {								\
-		ret = of_property_read_u32(np, name, reg);		\
-		if (ret) {						\
-			dev_err(pcs->dev, err);				\
-			return ret;					\
-		}							\
-	} while (0);
-
 static const struct of_device_id pcs_of_match[];
 
 static int pcs_add_gpio_func(struct device_node *node, struct pcs_device *pcs)
@@ -1820,6 +1813,55 @@ static int pinctrl_single_resume(struct platform_device *pdev)
 }
 #endif
 
+/**
+ * pcs_quirk_missing_pinctrl_cells - handle legacy binding
+ * @pcs: pinctrl driver instance
+ * @np: device tree node
+ * @cells: number of cells
+ *
+ * Handle legacy binding with no #pinctrl-cells. This should be
+ * always two pinctrl-single,bit-per-mux and one for others.
+ * At some point we may want to consider removing this.
+ */
+static int pcs_quirk_missing_pinctrl_cells(struct pcs_device *pcs,
+					   struct device_node *np,
+					   int cells)
+{
+	struct property *p;
+	const char *name = "#pinctrl-cells";
+	int error;
+	u32 val;
+
+	error = of_property_read_u32(np, name, &val);
+	if (!error)
+		return 0;
+
+	dev_warn(pcs->dev, "please update dts to use %s = <%i>\n",
+		 name, cells);
+
+	p = devm_kzalloc(pcs->dev, sizeof(*p), GFP_KERNEL);
+	if (!p)
+		return -ENOMEM;
+
+	p->length = sizeof(__be32);
+	p->value = devm_kzalloc(pcs->dev, sizeof(__be32), GFP_KERNEL);
+	if (!p->value)
+		return -ENOMEM;
+	*(__be32 *)p->value = cpu_to_be32(cells);
+
+	p->name = devm_kstrdup(pcs->dev, name, GFP_KERNEL);
+	if (!p->name)
+		return -ENOMEM;
+
+	pcs->missing_nr_pinctrl_cells = p;
+
+#if IS_BUILTIN(CONFIG_PINCTRL_SINGLE)
+	error = of_add_property(np, pcs->missing_nr_pinctrl_cells);
+#endif
+
+	return error;
+}
+
 static int pcs_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
@@ -1840,6 +1882,7 @@ static int pcs_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	}
 	pcs->dev = &pdev->dev;
+	pcs->np = np;
 	raw_spin_lock_init(&pcs->lock);
 	mutex_init(&pcs->mutex);
 	INIT_LIST_HEAD(&pcs->pingroups);
@@ -1849,8 +1892,13 @@ static int pcs_probe(struct platform_device *pdev)
 	pcs->flags = soc->flags;
 	memcpy(&pcs->socdata, soc, sizeof(*soc));
 
-	PCS_GET_PROP_U32("pinctrl-single,register-width", &pcs->width,
-			 "register width not specified\n");
+	ret = of_property_read_u32(np, "pinctrl-single,register-width",
+				   &pcs->width);
+	if (ret) {
+		dev_err(pcs->dev, "register width not specified\n");
+
+		return ret;
+	}
 
 	ret = of_property_read_u32(np, "pinctrl-single,function-mask",
 				   &pcs->fmask);
@@ -1871,6 +1919,13 @@ static int pcs_probe(struct platform_device *pdev)
 
 	pcs->bits_per_mux = of_property_read_bool(np,
 						  "pinctrl-single,bit-per-mux");
+	ret = pcs_quirk_missing_pinctrl_cells(pcs, np,
+					      pcs->bits_per_mux ? 2 : 1);
+	if (ret) {
+		dev_err(&pdev->dev, "unable to patch #pinctrl-cells\n");
+
+		return ret;
+	}
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!res) {
diff --git a/drivers/pinctrl/pinctrl-st.c b/drivers/pinctrl/pinctrl-st.c
index b7bb37167969..676efcc032d2 100644
--- a/drivers/pinctrl/pinctrl-st.c
+++ b/drivers/pinctrl/pinctrl-st.c
@@ -1006,7 +1006,7 @@ static void st_pinconf_dbg_show(struct pinctrl_dev *pctldev,
 
 	function = st_pctl_get_pin_function(pc, offset);
 	if (function)
-		snprintf(f, 10, "Alt Fn %d", function);
+		snprintf(f, 10, "Alt Fn %u", function);
 	else
 		snprintf(f, 5, "GPIO");
 
@@ -1181,7 +1181,7 @@ static int st_pctl_dt_parse_groups(struct device_node *np,
 		if (!strcmp(pp->name, "name"))
 			continue;
 
-		if (pp  && (pp->length/sizeof(__be32)) >= OF_GPIO_ARGS_MIN) {
+		if (pp->length / sizeof(__be32) >= OF_GPIO_ARGS_MIN) {
 			npins++;
 		} else {
 			pr_warn("Invalid st,pins in %s node\n", np->name);
diff --git a/drivers/pinctrl/pinctrl-sx150x.c b/drivers/pinctrl/pinctrl-sx150x.c
new file mode 100644
index 000000000000..29fb7403d24e
--- /dev/null
+++ b/drivers/pinctrl/pinctrl-sx150x.c
@@ -0,0 +1,1275 @@
+/*
+ * Copyright (c) 2016, BayLibre, SAS. All rights reserved.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * Copyright (c) 2010, Code Aurora Forum. All rights reserved.
+ *
+ * Driver for Semtech SX150X I2C GPIO Expanders
+ * The handling of the 4-bit chips (SX1501/SX1504/SX1507) is untested.
+ *
+ * Author: Gregory Bean <gbean@codeaurora.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/regmap.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/gpio/driver.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/pinctrl/pinconf-generic.h>
+
+#include "core.h"
+#include "pinconf.h"
+#include "pinctrl-utils.h"
+
+/* The chip models of sx150x */
+enum {
+	SX150X_123 = 0,
+	SX150X_456,
+	SX150X_789,
+};
+enum {
+	SX150X_789_REG_MISC_AUTOCLEAR_OFF = 1 << 0,
+	SX150X_MAX_REGISTER = 0xad,
+	SX150X_IRQ_TYPE_EDGE_RISING = 0x1,
+	SX150X_IRQ_TYPE_EDGE_FALLING = 0x2,
+	SX150X_789_RESET_KEY1 = 0x12,
+	SX150X_789_RESET_KEY2 = 0x34,
+};
+
+struct sx150x_123_pri {
+	u8 reg_pld_mode;
+	u8 reg_pld_table0;
+	u8 reg_pld_table1;
+	u8 reg_pld_table2;
+	u8 reg_pld_table3;
+	u8 reg_pld_table4;
+	u8 reg_advanced;
+};
+
+struct sx150x_456_pri {
+	u8 reg_pld_mode;
+	u8 reg_pld_table0;
+	u8 reg_pld_table1;
+	u8 reg_pld_table2;
+	u8 reg_pld_table3;
+	u8 reg_pld_table4;
+	u8 reg_advanced;
+};
+
+struct sx150x_789_pri {
+	u8 reg_drain;
+	u8 reg_polarity;
+	u8 reg_clock;
+	u8 reg_misc;
+	u8 reg_reset;
+	u8 ngpios;
+};
+
+struct sx150x_device_data {
+	u8 model;
+	u8 reg_pullup;
+	u8 reg_pulldn;
+	u8 reg_dir;
+	u8 reg_data;
+	u8 reg_irq_mask;
+	u8 reg_irq_src;
+	u8 reg_sense;
+	u8 ngpios;
+	union {
+		struct sx150x_123_pri x123;
+		struct sx150x_456_pri x456;
+		struct sx150x_789_pri x789;
+	} pri;
+	const struct pinctrl_pin_desc *pins;
+	unsigned int npins;
+};
+
+struct sx150x_pinctrl {
+	struct device *dev;
+	struct i2c_client *client;
+	struct pinctrl_dev *pctldev;
+	struct pinctrl_desc pinctrl_desc;
+	struct gpio_chip gpio;
+	struct irq_chip irq_chip;
+	struct regmap *regmap;
+	struct {
+		u32 sense;
+		u32 masked;
+	} irq;
+	struct mutex lock;
+	const struct sx150x_device_data *data;
+};
+
+static const struct pinctrl_pin_desc sx150x_4_pins[] = {
+	PINCTRL_PIN(0, "gpio0"),
+	PINCTRL_PIN(1, "gpio1"),
+	PINCTRL_PIN(2, "gpio2"),
+	PINCTRL_PIN(3, "gpio3"),
+	PINCTRL_PIN(4, "oscio"),
+};
+
+static const struct pinctrl_pin_desc sx150x_8_pins[] = {
+	PINCTRL_PIN(0, "gpio0"),
+	PINCTRL_PIN(1, "gpio1"),
+	PINCTRL_PIN(2, "gpio2"),
+	PINCTRL_PIN(3, "gpio3"),
+	PINCTRL_PIN(4, "gpio4"),
+	PINCTRL_PIN(5, "gpio5"),
+	PINCTRL_PIN(6, "gpio6"),
+	PINCTRL_PIN(7, "gpio7"),
+	PINCTRL_PIN(8, "oscio"),
+};
+
+static const struct pinctrl_pin_desc sx150x_16_pins[] = {
+	PINCTRL_PIN(0, "gpio0"),
+	PINCTRL_PIN(1, "gpio1"),
+	PINCTRL_PIN(2, "gpio2"),
+	PINCTRL_PIN(3, "gpio3"),
+	PINCTRL_PIN(4, "gpio4"),
+	PINCTRL_PIN(5, "gpio5"),
+	PINCTRL_PIN(6, "gpio6"),
+	PINCTRL_PIN(7, "gpio7"),
+	PINCTRL_PIN(8, "gpio8"),
+	PINCTRL_PIN(9, "gpio9"),
+	PINCTRL_PIN(10, "gpio10"),
+	PINCTRL_PIN(11, "gpio11"),
+	PINCTRL_PIN(12, "gpio12"),
+	PINCTRL_PIN(13, "gpio13"),
+	PINCTRL_PIN(14, "gpio14"),
+	PINCTRL_PIN(15, "gpio15"),
+	PINCTRL_PIN(16, "oscio"),
+};
+
+static const struct sx150x_device_data sx1501q_device_data = {
+	.model = SX150X_123,
+	.reg_pullup	= 0x02,
+	.reg_pulldn	= 0x03,
+	.reg_dir	= 0x01,
+	.reg_data	= 0x00,
+	.reg_irq_mask	= 0x05,
+	.reg_irq_src	= 0x08,
+	.reg_sense	= 0x07,
+	.pri.x123 = {
+		.reg_pld_mode	= 0x10,
+		.reg_pld_table0	= 0x11,
+		.reg_pld_table2	= 0x13,
+		.reg_advanced	= 0xad,
+	},
+	.ngpios	= 4,
+	.pins = sx150x_4_pins,
+	.npins = 4, /* oscio not available */
+};
+
+static const struct sx150x_device_data sx1502q_device_data = {
+	.model = SX150X_123,
+	.reg_pullup	= 0x02,
+	.reg_pulldn	= 0x03,
+	.reg_dir	= 0x01,
+	.reg_data	= 0x00,
+	.reg_irq_mask	= 0x05,
+	.reg_irq_src	= 0x08,
+	.reg_sense	= 0x06,
+	.pri.x123 = {
+		.reg_pld_mode	= 0x10,
+		.reg_pld_table0	= 0x11,
+		.reg_pld_table1	= 0x12,
+		.reg_pld_table2	= 0x13,
+		.reg_pld_table3	= 0x14,
+		.reg_pld_table4	= 0x15,
+		.reg_advanced	= 0xad,
+	},
+	.ngpios	= 8,
+	.pins = sx150x_8_pins,
+	.npins = 8, /* oscio not available */
+};
+
+static const struct sx150x_device_data sx1503q_device_data = {
+	.model = SX150X_123,
+	.reg_pullup	= 0x04,
+	.reg_pulldn	= 0x06,
+	.reg_dir	= 0x02,
+	.reg_data	= 0x00,
+	.reg_irq_mask	= 0x08,
+	.reg_irq_src	= 0x0e,
+	.reg_sense	= 0x0a,
+	.pri.x123 = {
+		.reg_pld_mode	= 0x20,
+		.reg_pld_table0	= 0x22,
+		.reg_pld_table1	= 0x24,
+		.reg_pld_table2	= 0x26,
+		.reg_pld_table3	= 0x28,
+		.reg_pld_table4	= 0x2a,
+		.reg_advanced	= 0xad,
+	},
+	.ngpios	= 16,
+	.pins = sx150x_16_pins,
+	.npins  = 16, /* oscio not available */
+};
+
+static const struct sx150x_device_data sx1504q_device_data = {
+	.model = SX150X_456,
+	.reg_pullup	= 0x02,
+	.reg_pulldn	= 0x03,
+	.reg_dir	= 0x01,
+	.reg_data	= 0x00,
+	.reg_irq_mask	= 0x05,
+	.reg_irq_src	= 0x08,
+	.reg_sense	= 0x07,
+	.pri.x456 = {
+		.reg_pld_mode	= 0x10,
+		.reg_pld_table0	= 0x11,
+		.reg_pld_table2	= 0x13,
+	},
+	.ngpios	= 4,
+	.pins = sx150x_4_pins,
+	.npins = 4, /* oscio not available */
+};
+
+static const struct sx150x_device_data sx1505q_device_data = {
+	.model = SX150X_456,
+	.reg_pullup	= 0x02,
+	.reg_pulldn	= 0x03,
+	.reg_dir	= 0x01,
+	.reg_data	= 0x00,
+	.reg_irq_mask	= 0x05,
+	.reg_irq_src	= 0x08,
+	.reg_sense	= 0x06,
+	.pri.x456 = {
+		.reg_pld_mode	= 0x10,
+		.reg_pld_table0	= 0x11,
+		.reg_pld_table1	= 0x12,
+		.reg_pld_table2	= 0x13,
+		.reg_pld_table3	= 0x14,
+		.reg_pld_table4	= 0x15,
+	},
+	.ngpios	= 8,
+	.pins = sx150x_8_pins,
+	.npins = 8, /* oscio not available */
+};
+
+static const struct sx150x_device_data sx1506q_device_data = {
+	.model = SX150X_456,
+	.reg_pullup	= 0x04,
+	.reg_pulldn	= 0x06,
+	.reg_dir	= 0x02,
+	.reg_data	= 0x00,
+	.reg_irq_mask	= 0x08,
+	.reg_irq_src	= 0x0e,
+	.reg_sense	= 0x0a,
+	.pri.x456 = {
+		.reg_pld_mode	= 0x20,
+		.reg_pld_table0	= 0x22,
+		.reg_pld_table1	= 0x24,
+		.reg_pld_table2	= 0x26,
+		.reg_pld_table3	= 0x28,
+		.reg_pld_table4	= 0x2a,
+		.reg_advanced	= 0xad,
+	},
+	.ngpios	= 16,
+	.pins = sx150x_16_pins,
+	.npins = 16, /* oscio not available */
+};
+
+static const struct sx150x_device_data sx1507q_device_data = {
+	.model = SX150X_789,
+	.reg_pullup	= 0x03,
+	.reg_pulldn	= 0x04,
+	.reg_dir	= 0x07,
+	.reg_data	= 0x08,
+	.reg_irq_mask	= 0x09,
+	.reg_irq_src	= 0x0b,
+	.reg_sense	= 0x0a,
+	.pri.x789 = {
+		.reg_drain	= 0x05,
+		.reg_polarity	= 0x06,
+		.reg_clock	= 0x0d,
+		.reg_misc	= 0x0e,
+		.reg_reset	= 0x7d,
+	},
+	.ngpios = 4,
+	.pins = sx150x_4_pins,
+	.npins = ARRAY_SIZE(sx150x_4_pins),
+};
+
+static const struct sx150x_device_data sx1508q_device_data = {
+	.model = SX150X_789,
+	.reg_pullup	= 0x03,
+	.reg_pulldn	= 0x04,
+	.reg_dir	= 0x07,
+	.reg_data	= 0x08,
+	.reg_irq_mask	= 0x09,
+	.reg_irq_src	= 0x0c,
+	.reg_sense	= 0x0a,
+	.pri.x789 = {
+		.reg_drain	= 0x05,
+		.reg_polarity	= 0x06,
+		.reg_clock	= 0x0f,
+		.reg_misc	= 0x10,
+		.reg_reset	= 0x7d,
+	},
+	.ngpios = 8,
+	.pins = sx150x_8_pins,
+	.npins = ARRAY_SIZE(sx150x_8_pins),
+};
+
+static const struct sx150x_device_data sx1509q_device_data = {
+	.model = SX150X_789,
+	.reg_pullup	= 0x06,
+	.reg_pulldn	= 0x08,
+	.reg_dir	= 0x0e,
+	.reg_data	= 0x10,
+	.reg_irq_mask	= 0x12,
+	.reg_irq_src	= 0x18,
+	.reg_sense	= 0x14,
+	.pri.x789 = {
+		.reg_drain	= 0x0a,
+		.reg_polarity	= 0x0c,
+		.reg_clock	= 0x1e,
+		.reg_misc	= 0x1f,
+		.reg_reset	= 0x7d,
+	},
+	.ngpios	= 16,
+	.pins = sx150x_16_pins,
+	.npins = ARRAY_SIZE(sx150x_16_pins),
+};
+
+static int sx150x_pinctrl_get_groups_count(struct pinctrl_dev *pctldev)
+{
+	return 0;
+}
+
+static const char *sx150x_pinctrl_get_group_name(struct pinctrl_dev *pctldev,
+						unsigned int group)
+{
+	return NULL;
+}
+
+static int sx150x_pinctrl_get_group_pins(struct pinctrl_dev *pctldev,
+					unsigned int group,
+					const unsigned int **pins,
+					unsigned int *num_pins)
+{
+	return -ENOTSUPP;
+}
+
+static const struct pinctrl_ops sx150x_pinctrl_ops = {
+	.get_groups_count = sx150x_pinctrl_get_groups_count,
+	.get_group_name = sx150x_pinctrl_get_group_name,
+	.get_group_pins = sx150x_pinctrl_get_group_pins,
+#ifdef CONFIG_OF
+	.dt_node_to_map = pinconf_generic_dt_node_to_map_pin,
+	.dt_free_map = pinctrl_utils_free_map,
+#endif
+};
+
+static bool sx150x_pin_is_oscio(struct sx150x_pinctrl *pctl, unsigned int pin)
+{
+	if (pin >= pctl->data->npins)
+		return false;
+
+	/* OSCIO pin is only present in 789 devices */
+	if (pctl->data->model != SX150X_789)
+		return false;
+
+	return !strcmp(pctl->data->pins[pin].name, "oscio");
+}
+
+static int sx150x_gpio_get_direction(struct gpio_chip *chip,
+				      unsigned int offset)
+{
+	struct sx150x_pinctrl *pctl = gpiochip_get_data(chip);
+	unsigned int value;
+	int ret;
+
+	if (sx150x_pin_is_oscio(pctl, offset))
+		return false;
+
+	ret = regmap_read(pctl->regmap, pctl->data->reg_dir, &value);
+	if (ret < 0)
+		return ret;
+
+	return !!(value & BIT(offset));
+}
+
+static int sx150x_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+	struct sx150x_pinctrl *pctl = gpiochip_get_data(chip);
+	unsigned int value;
+	int ret;
+
+	if (sx150x_pin_is_oscio(pctl, offset))
+		return -EINVAL;
+
+	ret = regmap_read(pctl->regmap, pctl->data->reg_data, &value);
+	if (ret < 0)
+		return ret;
+
+	return !!(value & BIT(offset));
+}
+
+static int sx150x_gpio_set_single_ended(struct gpio_chip *chip,
+					unsigned int offset,
+					enum single_ended_mode mode)
+{
+	struct sx150x_pinctrl *pctl = gpiochip_get_data(chip);
+	int ret;
+
+	switch (mode) {
+	case LINE_MODE_PUSH_PULL:
+		if (pctl->data->model != SX150X_789 ||
+		    sx150x_pin_is_oscio(pctl, offset))
+			return 0;
+
+		ret = regmap_write_bits(pctl->regmap,
+					pctl->data->pri.x789.reg_drain,
+					BIT(offset), 0);
+		break;
+
+	case LINE_MODE_OPEN_DRAIN:
+		if (pctl->data->model != SX150X_789 ||
+		    sx150x_pin_is_oscio(pctl, offset))
+			return -ENOTSUPP;
+
+		ret = regmap_write_bits(pctl->regmap,
+					pctl->data->pri.x789.reg_drain,
+					BIT(offset), BIT(offset));
+		break;
+	default:
+		ret = -ENOTSUPP;
+		break;
+	}
+
+	return ret;
+}
+
+static int __sx150x_gpio_set(struct sx150x_pinctrl *pctl, unsigned int offset,
+			     int value)
+{
+	return regmap_write_bits(pctl->regmap, pctl->data->reg_data,
+				 BIT(offset), value ? BIT(offset) : 0);
+}
+
+static int sx150x_gpio_oscio_set(struct sx150x_pinctrl *pctl,
+				 int value)
+{
+	return regmap_write(pctl->regmap,
+			    pctl->data->pri.x789.reg_clock,
+			    (value ? 0x1f : 0x10));
+}
+
+static void sx150x_gpio_set(struct gpio_chip *chip, unsigned int offset,
+			    int value)
+{
+	struct sx150x_pinctrl *pctl = gpiochip_get_data(chip);
+
+	if (sx150x_pin_is_oscio(pctl, offset))
+		sx150x_gpio_oscio_set(pctl, value);
+	else
+		__sx150x_gpio_set(pctl, offset, value);
+
+}
+
+static void sx150x_gpio_set_multiple(struct gpio_chip *chip,
+				     unsigned long *mask,
+				     unsigned long *bits)
+{
+	struct sx150x_pinctrl *pctl = gpiochip_get_data(chip);
+
+	regmap_write_bits(pctl->regmap, pctl->data->reg_data, *mask, *bits);
+}
+
+static int sx150x_gpio_direction_input(struct gpio_chip *chip,
+				       unsigned int offset)
+{
+	struct sx150x_pinctrl *pctl = gpiochip_get_data(chip);
+
+	if (sx150x_pin_is_oscio(pctl, offset))
+		return -EINVAL;
+
+	return regmap_write_bits(pctl->regmap,
+				 pctl->data->reg_dir,
+				 BIT(offset), BIT(offset));
+}
+
+static int sx150x_gpio_direction_output(struct gpio_chip *chip,
+					unsigned int offset, int value)
+{
+	struct sx150x_pinctrl *pctl = gpiochip_get_data(chip);
+	int ret;
+
+	if (sx150x_pin_is_oscio(pctl, offset))
+		return sx150x_gpio_oscio_set(pctl, value);
+
+	ret = __sx150x_gpio_set(pctl, offset, value);
+	if (ret < 0)
+		return ret;
+
+	return regmap_write_bits(pctl->regmap,
+				 pctl->data->reg_dir,
+				 BIT(offset), 0);
+}
+
+static void sx150x_irq_mask(struct irq_data *d)
+{
+	struct sx150x_pinctrl *pctl =
+			gpiochip_get_data(irq_data_get_irq_chip_data(d));
+	unsigned int n = d->hwirq;
+
+	pctl->irq.masked |= BIT(n);
+}
+
+static void sx150x_irq_unmask(struct irq_data *d)
+{
+	struct sx150x_pinctrl *pctl =
+			gpiochip_get_data(irq_data_get_irq_chip_data(d));
+	unsigned int n = d->hwirq;
+
+	pctl->irq.masked &= ~BIT(n);
+}
+
+static void sx150x_irq_set_sense(struct sx150x_pinctrl *pctl,
+				 unsigned int line, unsigned int sense)
+{
+	/*
+	 * Every interrupt line is represented by two bits shifted
+	 * proportionally to the line number
+	 */
+	const unsigned int n = line * 2;
+	const unsigned int mask = ~((SX150X_IRQ_TYPE_EDGE_RISING |
+				     SX150X_IRQ_TYPE_EDGE_FALLING) << n);
+
+	pctl->irq.sense &= mask;
+	pctl->irq.sense |= sense << n;
+}
+
+static int sx150x_irq_set_type(struct irq_data *d, unsigned int flow_type)
+{
+	struct sx150x_pinctrl *pctl =
+			gpiochip_get_data(irq_data_get_irq_chip_data(d));
+	unsigned int n, val = 0;
+
+	if (flow_type & (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW))
+		return -EINVAL;
+
+	n = d->hwirq;
+
+	if (flow_type & IRQ_TYPE_EDGE_RISING)
+		val |= SX150X_IRQ_TYPE_EDGE_RISING;
+	if (flow_type & IRQ_TYPE_EDGE_FALLING)
+		val |= SX150X_IRQ_TYPE_EDGE_FALLING;
+
+	sx150x_irq_set_sense(pctl, n, val);
+	return 0;
+}
+
+static irqreturn_t sx150x_irq_thread_fn(int irq, void *dev_id)
+{
+	struct sx150x_pinctrl *pctl = (struct sx150x_pinctrl *)dev_id;
+	unsigned long n, status;
+	unsigned int val;
+	int err;
+
+	err = regmap_read(pctl->regmap, pctl->data->reg_irq_src, &val);
+	if (err < 0)
+		return IRQ_NONE;
+
+	err = regmap_write(pctl->regmap, pctl->data->reg_irq_src, val);
+	if (err < 0)
+		return IRQ_NONE;
+
+	status = val;
+	for_each_set_bit(n, &status, pctl->data->ngpios)
+		handle_nested_irq(irq_find_mapping(pctl->gpio.irqdomain, n));
+
+	return IRQ_HANDLED;
+}
+
+static void sx150x_irq_bus_lock(struct irq_data *d)
+{
+	struct sx150x_pinctrl *pctl =
+			gpiochip_get_data(irq_data_get_irq_chip_data(d));
+
+	mutex_lock(&pctl->lock);
+}
+
+static void sx150x_irq_bus_sync_unlock(struct irq_data *d)
+{
+	struct sx150x_pinctrl *pctl =
+			gpiochip_get_data(irq_data_get_irq_chip_data(d));
+
+	regmap_write(pctl->regmap, pctl->data->reg_irq_mask, pctl->irq.masked);
+	regmap_write(pctl->regmap, pctl->data->reg_sense, pctl->irq.sense);
+	mutex_unlock(&pctl->lock);
+}
+
+static int sx150x_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin,
+			      unsigned long *config)
+{
+	struct sx150x_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	unsigned int param = pinconf_to_config_param(*config);
+	int ret;
+	u32 arg;
+	unsigned int data;
+
+	if (sx150x_pin_is_oscio(pctl, pin)) {
+		switch (param) {
+		case PIN_CONFIG_DRIVE_PUSH_PULL:
+		case PIN_CONFIG_OUTPUT:
+			ret = regmap_read(pctl->regmap,
+					  pctl->data->pri.x789.reg_clock,
+					  &data);
+			if (ret < 0)
+				return ret;
+
+			if (param == PIN_CONFIG_DRIVE_PUSH_PULL)
+				arg = (data & 0x1f) ? 1 : 0;
+			else {
+				if ((data & 0x1f) == 0x1f)
+					arg = 1;
+				else if ((data & 0x1f) == 0x10)
+					arg = 0;
+				else
+					return -EINVAL;
+			}
+
+			break;
+		default:
+			return -ENOTSUPP;
+		}
+
+		goto out;
+	}
+
+	switch (param) {
+	case PIN_CONFIG_BIAS_PULL_DOWN:
+		ret = regmap_read(pctl->regmap,
+				  pctl->data->reg_pulldn,
+				  &data);
+		data &= BIT(pin);
+
+		if (ret < 0)
+			return ret;
+
+		if (!ret)
+			return -EINVAL;
+
+		arg = 1;
+		break;
+
+	case PIN_CONFIG_BIAS_PULL_UP:
+		ret = regmap_read(pctl->regmap,
+				  pctl->data->reg_pullup,
+				  &data);
+		data &= BIT(pin);
+
+		if (ret < 0)
+			return ret;
+
+		if (!ret)
+			return -EINVAL;
+
+		arg = 1;
+		break;
+
+	case PIN_CONFIG_DRIVE_OPEN_DRAIN:
+		if (pctl->data->model != SX150X_789)
+			return -ENOTSUPP;
+
+		ret = regmap_read(pctl->regmap,
+				  pctl->data->pri.x789.reg_drain,
+				  &data);
+		data &= BIT(pin);
+
+		if (ret < 0)
+			return ret;
+
+		if (!data)
+			return -EINVAL;
+
+		arg = 1;
+		break;
+
+	case PIN_CONFIG_DRIVE_PUSH_PULL:
+		if (pctl->data->model != SX150X_789)
+			arg = true;
+		else {
+			ret = regmap_read(pctl->regmap,
+					  pctl->data->pri.x789.reg_drain,
+					  &data);
+			data &= BIT(pin);
+
+			if (ret < 0)
+				return ret;
+
+			if (data)
+				return -EINVAL;
+
+			arg = 1;
+		}
+		break;
+
+	case PIN_CONFIG_OUTPUT:
+		ret = sx150x_gpio_get_direction(&pctl->gpio, pin);
+		if (ret < 0)
+			return ret;
+
+		if (ret)
+			return -EINVAL;
+
+		ret = sx150x_gpio_get(&pctl->gpio, pin);
+		if (ret < 0)
+			return ret;
+
+		arg = ret;
+		break;
+
+	default:
+		return -ENOTSUPP;
+	}
+
+out:
+	*config = pinconf_to_config_packed(param, arg);
+
+	return 0;
+}
+
+static int sx150x_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
+			      unsigned long *configs, unsigned int num_configs)
+{
+	struct sx150x_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	enum pin_config_param param;
+	u32 arg;
+	int i;
+	int ret;
+
+	for (i = 0; i < num_configs; i++) {
+		param = pinconf_to_config_param(configs[i]);
+		arg = pinconf_to_config_argument(configs[i]);
+
+		if (sx150x_pin_is_oscio(pctl, pin)) {
+			if (param == PIN_CONFIG_OUTPUT) {
+				ret = sx150x_gpio_direction_output(&pctl->gpio,
+								   pin, arg);
+				if (ret < 0)
+					return ret;
+
+				continue;
+			} else
+				return -ENOTSUPP;
+		}
+
+		switch (param) {
+		case PIN_CONFIG_BIAS_PULL_PIN_DEFAULT:
+		case PIN_CONFIG_BIAS_DISABLE:
+			ret = regmap_write_bits(pctl->regmap,
+						pctl->data->reg_pulldn,
+						BIT(pin), 0);
+			if (ret < 0)
+				return ret;
+
+			ret = regmap_write_bits(pctl->regmap,
+						pctl->data->reg_pullup,
+						BIT(pin), 0);
+			if (ret < 0)
+				return ret;
+
+			break;
+
+		case PIN_CONFIG_BIAS_PULL_UP:
+			ret = regmap_write_bits(pctl->regmap,
+						pctl->data->reg_pullup,
+						BIT(pin), BIT(pin));
+			if (ret < 0)
+				return ret;
+
+			break;
+
+		case PIN_CONFIG_BIAS_PULL_DOWN:
+			ret = regmap_write_bits(pctl->regmap,
+						pctl->data->reg_pulldn,
+						BIT(pin), BIT(pin));
+			if (ret < 0)
+				return ret;
+
+			break;
+
+		case PIN_CONFIG_DRIVE_OPEN_DRAIN:
+			ret = sx150x_gpio_set_single_ended(&pctl->gpio,
+						pin, LINE_MODE_OPEN_DRAIN);
+			if (ret < 0)
+				return ret;
+
+			break;
+
+		case PIN_CONFIG_DRIVE_PUSH_PULL:
+			ret = sx150x_gpio_set_single_ended(&pctl->gpio,
+						pin, LINE_MODE_PUSH_PULL);
+			if (ret < 0)
+				return ret;
+
+			break;
+
+		case PIN_CONFIG_OUTPUT:
+			ret = sx150x_gpio_direction_output(&pctl->gpio,
+							   pin, arg);
+			if (ret < 0)
+				return ret;
+
+			break;
+
+		default:
+			return -ENOTSUPP;
+		}
+	} /* for each config */
+
+	return 0;
+}
+
+static const struct pinconf_ops sx150x_pinconf_ops = {
+	.pin_config_get = sx150x_pinconf_get,
+	.pin_config_set = sx150x_pinconf_set,
+	.is_generic = true,
+};
+
+static const struct i2c_device_id sx150x_id[] = {
+	{"sx1501q", (kernel_ulong_t) &sx1501q_device_data },
+	{"sx1502q", (kernel_ulong_t) &sx1502q_device_data },
+	{"sx1503q", (kernel_ulong_t) &sx1503q_device_data },
+	{"sx1504q", (kernel_ulong_t) &sx1504q_device_data },
+	{"sx1505q", (kernel_ulong_t) &sx1505q_device_data },
+	{"sx1506q", (kernel_ulong_t) &sx1506q_device_data },
+	{"sx1507q", (kernel_ulong_t) &sx1507q_device_data },
+	{"sx1508q", (kernel_ulong_t) &sx1508q_device_data },
+	{"sx1509q", (kernel_ulong_t) &sx1509q_device_data },
+	{}
+};
+
+static const struct of_device_id sx150x_of_match[] = {
+	{ .compatible = "semtech,sx1501q", .data = &sx1501q_device_data },
+	{ .compatible = "semtech,sx1502q", .data = &sx1502q_device_data },
+	{ .compatible = "semtech,sx1503q", .data = &sx1503q_device_data },
+	{ .compatible = "semtech,sx1504q", .data = &sx1504q_device_data },
+	{ .compatible = "semtech,sx1505q", .data = &sx1505q_device_data },
+	{ .compatible = "semtech,sx1506q", .data = &sx1506q_device_data },
+	{ .compatible = "semtech,sx1507q", .data = &sx1507q_device_data },
+	{ .compatible = "semtech,sx1508q", .data = &sx1508q_device_data },
+	{ .compatible = "semtech,sx1509q", .data = &sx1509q_device_data },
+	{},
+};
+
+static int sx150x_reset(struct sx150x_pinctrl *pctl)
+{
+	int err;
+
+	err = i2c_smbus_write_byte_data(pctl->client,
+					pctl->data->pri.x789.reg_reset,
+					SX150X_789_RESET_KEY1);
+	if (err < 0)
+		return err;
+
+	err = i2c_smbus_write_byte_data(pctl->client,
+					pctl->data->pri.x789.reg_reset,
+					SX150X_789_RESET_KEY2);
+	return err;
+}
+
+static int sx150x_init_misc(struct sx150x_pinctrl *pctl)
+{
+	u8 reg, value;
+
+	switch (pctl->data->model) {
+	case SX150X_789:
+		reg   = pctl->data->pri.x789.reg_misc;
+		value = SX150X_789_REG_MISC_AUTOCLEAR_OFF;
+		break;
+	case SX150X_456:
+		reg   = pctl->data->pri.x456.reg_advanced;
+		value = 0x00;
+
+		/*
+		 * Only SX1506 has RegAdvanced, SX1504/5 are expected
+		 * to initialize this offset to zero
+		 */
+		if (!reg)
+			return 0;
+		break;
+	case SX150X_123:
+		reg   = pctl->data->pri.x123.reg_advanced;
+		value = 0x00;
+		break;
+	default:
+		WARN(1, "Unknown chip model %d\n", pctl->data->model);
+		return -EINVAL;
+	}
+
+	return regmap_write(pctl->regmap, reg, value);
+}
+
+static int sx150x_init_hw(struct sx150x_pinctrl *pctl)
+{
+	const u8 reg[] = {
+		[SX150X_789] = pctl->data->pri.x789.reg_polarity,
+		[SX150X_456] = pctl->data->pri.x456.reg_pld_mode,
+		[SX150X_123] = pctl->data->pri.x123.reg_pld_mode,
+	};
+	int err;
+
+	if (pctl->data->model == SX150X_789 &&
+	    of_property_read_bool(pctl->dev->of_node, "semtech,probe-reset")) {
+		err = sx150x_reset(pctl);
+		if (err < 0)
+			return err;
+	}
+
+	err = sx150x_init_misc(pctl);
+	if (err < 0)
+		return err;
+
+	/* Set all pins to work in normal mode */
+	return regmap_write(pctl->regmap, reg[pctl->data->model], 0);
+}
+
+static int sx150x_regmap_reg_width(struct sx150x_pinctrl *pctl,
+				   unsigned int reg)
+{
+	const struct sx150x_device_data *data = pctl->data;
+
+	if (reg == data->reg_sense) {
+		/*
+		 * RegSense packs two bits of configuration per GPIO,
+		 * so we'd need to read twice as many bits as there
+		 * are GPIO in our chip
+		 */
+		return 2 * data->ngpios;
+	} else if ((data->model == SX150X_789 &&
+		    (reg == data->pri.x789.reg_misc ||
+		     reg == data->pri.x789.reg_clock ||
+		     reg == data->pri.x789.reg_reset))
+		   ||
+		   (data->model == SX150X_123 &&
+		    reg == data->pri.x123.reg_advanced)
+		   ||
+		   (data->model == SX150X_456 &&
+		    data->pri.x456.reg_advanced &&
+		    reg == data->pri.x456.reg_advanced)) {
+		return 8;
+	} else {
+		return data->ngpios;
+	}
+}
+
+static unsigned int sx150x_maybe_swizzle(struct sx150x_pinctrl *pctl,
+					 unsigned int reg, unsigned int val)
+{
+	unsigned int a, b;
+	const struct sx150x_device_data *data = pctl->data;
+
+	/*
+	 * Whereas SX1509 presents RegSense in a simple layout as such:
+	 *	reg     [ f f e e d d c c ]
+	 *	reg + 1 [ b b a a 9 9 8 8 ]
+	 *	reg + 2 [ 7 7 6 6 5 5 4 4 ]
+	 *	reg + 3 [ 3 3 2 2 1 1 0 0 ]
+	 *
+	 * SX1503 and SX1506 deviate from that data layout, instead storing
+	 * their contents as follows:
+	 *
+	 *	reg     [ f f e e d d c c ]
+	 *	reg + 1 [ 7 7 6 6 5 5 4 4 ]
+	 *	reg + 2 [ b b a a 9 9 8 8 ]
+	 *	reg + 3 [ 3 3 2 2 1 1 0 0 ]
+	 *
+	 * so, taking that into account, we swap two
+	 * inner bytes of a 4-byte result
+	 */
+
+	if (reg == data->reg_sense &&
+	    data->ngpios == 16 &&
+	    (data->model == SX150X_123 ||
+	     data->model == SX150X_456)) {
+		a = val & 0x00ff0000;
+		b = val & 0x0000ff00;
+
+		val &= 0xff0000ff;
+		val |= b << 8;
+		val |= a >> 8;
+	}
+
+	return val;
+}
+
+/*
+ * In order to mask the differences between 16 and 8 bit expander
+ * devices we set up a sligthly ficticious regmap that pretends to be
+ * a set of 32-bit (to accomodate RegSenseLow/RegSenseHigh
+ * pair/quartet) registers and transparently reconstructs those
+ * registers via multiple I2C/SMBus reads
+ *
+ * This way the rest of the driver code, interfacing with the chip via
+ * regmap API, can work assuming that each GPIO pin is represented by
+ * a group of bits at an offset proportional to GPIO number within a
+ * given register.
+ */
+static int sx150x_regmap_reg_read(void *context, unsigned int reg,
+				  unsigned int *result)
+{
+	int ret, n;
+	struct sx150x_pinctrl *pctl = context;
+	struct i2c_client *i2c = pctl->client;
+	const int width = sx150x_regmap_reg_width(pctl, reg);
+	unsigned int idx, val;
+
+	/*
+	 * There are four potential cases covered by this function:
+	 *
+	 * 1) 8-pin chip, single configuration bit register
+	 *
+	 *	This is trivial the code below just needs to read:
+	 *		reg  [ 7 6 5 4 3 2 1 0 ]
+	 *
+	 * 2) 8-pin chip, double configuration bit register (RegSense)
+	 *
+	 *	The read will be done as follows:
+	 *		reg      [ 7 7 6 6 5 5 4 4 ]
+	 *		reg + 1  [ 3 3 2 2 1 1 0 0 ]
+	 *
+	 * 3) 16-pin chip, single configuration bit register
+	 *
+	 *	The read will be done as follows:
+	 *		reg     [ f e d c b a 9 8 ]
+	 *		reg + 1 [ 7 6 5 4 3 2 1 0 ]
+	 *
+	 * 4) 16-pin chip, double configuration bit register (RegSense)
+	 *
+	 *	The read will be done as follows:
+	 *		reg     [ f f e e d d c c ]
+	 *		reg + 1 [ b b a a 9 9 8 8 ]
+	 *		reg + 2 [ 7 7 6 6 5 5 4 4 ]
+	 *		reg + 3 [ 3 3 2 2 1 1 0 0 ]
+	 */
+
+	for (n = width, val = 0, idx = reg; n > 0; n -= 8, idx++) {
+		val <<= 8;
+
+		ret = i2c_smbus_read_byte_data(i2c, idx);
+		if (ret < 0)
+			return ret;
+
+		val |= ret;
+	}
+
+	*result = sx150x_maybe_swizzle(pctl, reg, val);
+
+	return 0;
+}
+
+static int sx150x_regmap_reg_write(void *context, unsigned int reg,
+				   unsigned int val)
+{
+	int ret, n;
+	struct sx150x_pinctrl *pctl = context;
+	struct i2c_client *i2c = pctl->client;
+	const int width = sx150x_regmap_reg_width(pctl, reg);
+
+	val = sx150x_maybe_swizzle(pctl, reg, val);
+
+	n = (width - 1) & ~7;
+	do {
+		const u8 byte = (val >> n) & 0xff;
+
+		ret = i2c_smbus_write_byte_data(i2c, reg, byte);
+		if (ret < 0)
+			return ret;
+
+		reg++;
+		n -= 8;
+	} while (n >= 0);
+
+	return 0;
+}
+
+static bool sx150x_reg_volatile(struct device *dev, unsigned int reg)
+{
+	struct sx150x_pinctrl *pctl = i2c_get_clientdata(to_i2c_client(dev));
+
+	return reg == pctl->data->reg_irq_src || reg == pctl->data->reg_data;
+}
+
+const struct regmap_config sx150x_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 32,
+
+	.cache_type = REGCACHE_RBTREE,
+
+	.reg_read = sx150x_regmap_reg_read,
+	.reg_write = sx150x_regmap_reg_write,
+
+	.max_register = SX150X_MAX_REGISTER,
+	.volatile_reg = sx150x_reg_volatile,
+};
+
+static int sx150x_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	static const u32 i2c_funcs = I2C_FUNC_SMBUS_BYTE_DATA |
+				     I2C_FUNC_SMBUS_WRITE_WORD_DATA;
+	struct device *dev = &client->dev;
+	struct sx150x_pinctrl *pctl;
+	int ret;
+
+	if (!i2c_check_functionality(client->adapter, i2c_funcs))
+		return -ENOSYS;
+
+	pctl = devm_kzalloc(dev, sizeof(*pctl), GFP_KERNEL);
+	if (!pctl)
+		return -ENOMEM;
+
+	i2c_set_clientdata(client, pctl);
+
+	pctl->dev = dev;
+	pctl->client = client;
+
+	if (dev->of_node)
+		pctl->data = of_device_get_match_data(dev);
+	else
+		pctl->data = (struct sx150x_device_data *)id->driver_data;
+
+	if (!pctl->data)
+		return -EINVAL;
+
+	pctl->regmap = devm_regmap_init(dev, NULL, pctl,
+					&sx150x_regmap_config);
+	if (IS_ERR(pctl->regmap)) {
+		ret = PTR_ERR(pctl->regmap);
+		dev_err(dev, "Failed to allocate register map: %d\n",
+			ret);
+		return ret;
+	}
+
+	mutex_init(&pctl->lock);
+
+	ret = sx150x_init_hw(pctl);
+	if (ret)
+		return ret;
+
+	/* Register GPIO controller */
+	pctl->gpio.label = devm_kstrdup(dev, client->name, GFP_KERNEL);
+	pctl->gpio.base = -1;
+	pctl->gpio.ngpio = pctl->data->npins;
+	pctl->gpio.get_direction = sx150x_gpio_get_direction;
+	pctl->gpio.direction_input = sx150x_gpio_direction_input;
+	pctl->gpio.direction_output = sx150x_gpio_direction_output;
+	pctl->gpio.get = sx150x_gpio_get;
+	pctl->gpio.set = sx150x_gpio_set;
+	pctl->gpio.set_single_ended = sx150x_gpio_set_single_ended;
+	pctl->gpio.parent = dev;
+#ifdef CONFIG_OF_GPIO
+	pctl->gpio.of_node = dev->of_node;
+#endif
+	pctl->gpio.can_sleep = true;
+	/*
+	 * Setting multiple pins is not safe when all pins are not
+	 * handled by the same regmap register. The oscio pin (present
+	 * on the SX150X_789 chips) lives in its own register, so
+	 * would require locking that is not in place at this time.
+	 */
+	if (pctl->data->model != SX150X_789)
+		pctl->gpio.set_multiple = sx150x_gpio_set_multiple;
+
+	ret = devm_gpiochip_add_data(dev, &pctl->gpio, pctl);
+	if (ret)
+		return ret;
+
+	/* Add Interrupt support if an irq is specified */
+	if (client->irq > 0) {
+		pctl->irq_chip.name = devm_kstrdup(dev, client->name,
+						   GFP_KERNEL);
+		pctl->irq_chip.irq_mask = sx150x_irq_mask;
+		pctl->irq_chip.irq_unmask = sx150x_irq_unmask;
+		pctl->irq_chip.irq_set_type = sx150x_irq_set_type;
+		pctl->irq_chip.irq_bus_lock = sx150x_irq_bus_lock;
+		pctl->irq_chip.irq_bus_sync_unlock = sx150x_irq_bus_sync_unlock;
+
+		pctl->irq.masked = ~0;
+		pctl->irq.sense = 0;
+
+		/*
+		 * Because sx150x_irq_threaded_fn invokes all of the
+		 * nested interrrupt handlers via handle_nested_irq,
+		 * any "handler" passed to gpiochip_irqchip_add()
+		 * below is going to be ignored, so the choice of the
+		 * function does not matter that much.
+		 *
+		 * We set it to handle_bad_irq to avoid confusion,
+		 * plus it will be instantly noticeable if it is ever
+		 * called (should not happen)
+		 */
+		ret = gpiochip_irqchip_add_nested(&pctl->gpio,
+					&pctl->irq_chip, 0,
+					handle_bad_irq, IRQ_TYPE_NONE);
+		if (ret) {
+			dev_err(dev, "could not connect irqchip to gpiochip\n");
+			return ret;
+		}
+
+		ret = devm_request_threaded_irq(dev, client->irq, NULL,
+						sx150x_irq_thread_fn,
+						IRQF_ONESHOT | IRQF_SHARED |
+						IRQF_TRIGGER_FALLING,
+						pctl->irq_chip.name, pctl);
+		if (ret < 0)
+			return ret;
+
+		gpiochip_set_nested_irqchip(&pctl->gpio,
+					    &pctl->irq_chip,
+					    client->irq);
+	}
+
+	/* Pinctrl_desc */
+	pctl->pinctrl_desc.name = "sx150x-pinctrl";
+	pctl->pinctrl_desc.pctlops = &sx150x_pinctrl_ops;
+	pctl->pinctrl_desc.confops = &sx150x_pinconf_ops;
+	pctl->pinctrl_desc.pins = pctl->data->pins;
+	pctl->pinctrl_desc.npins = pctl->data->npins;
+	pctl->pinctrl_desc.owner = THIS_MODULE;
+
+	pctl->pctldev = pinctrl_register(&pctl->pinctrl_desc, dev, pctl);
+	if (IS_ERR(pctl->pctldev)) {
+		dev_err(dev, "Failed to register pinctrl device\n");
+		return PTR_ERR(pctl->pctldev);
+	}
+
+	return 0;
+}
+
+static struct i2c_driver sx150x_driver = {
+	.driver = {
+		.name = "sx150x-pinctrl",
+		.of_match_table = of_match_ptr(sx150x_of_match),
+	},
+	.probe    = sx150x_probe,
+	.id_table = sx150x_id,
+};
+
+static int __init sx150x_init(void)
+{
+	return i2c_add_driver(&sx150x_driver);
+}
+subsys_initcall(sx150x_init);
diff --git a/drivers/pinctrl/pinctrl-zynq.c b/drivers/pinctrl/pinctrl-zynq.c
index e0ecffcbe11f..b51a46dfdcc3 100644
--- a/drivers/pinctrl/pinctrl-zynq.c
+++ b/drivers/pinctrl/pinctrl-zynq.c
@@ -247,6 +247,8 @@ static const unsigned int smc0_nor_addr25_pins[] = {1};
 static const unsigned int smc0_nand_pins[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
 					      12, 13, 14, 16, 17, 18, 19, 20,
 					      21, 22, 23};
+static const unsigned int smc0_nand8_pins[] = {0, 2, 3,  4,  5,  6,  7,
+					       8, 9, 10, 11, 12, 13, 14};
 /* Note: CAN MIO clock inputs are modeled in the clock framework */
 static const unsigned int can0_0_pins[] = {10, 11};
 static const unsigned int can0_1_pins[] = {14, 15};
@@ -445,6 +447,7 @@ static const struct zynq_pctrl_group zynq_pctrl_groups[] = {
 	DEFINE_ZYNQ_PINCTRL_GRP(smc0_nor_cs1),
 	DEFINE_ZYNQ_PINCTRL_GRP(smc0_nor_addr25),
 	DEFINE_ZYNQ_PINCTRL_GRP(smc0_nand),
+	DEFINE_ZYNQ_PINCTRL_GRP(smc0_nand8),
 	DEFINE_ZYNQ_PINCTRL_GRP(can0_0),
 	DEFINE_ZYNQ_PINCTRL_GRP(can0_1),
 	DEFINE_ZYNQ_PINCTRL_GRP(can0_2),
@@ -709,7 +712,8 @@ static const char * const sdio1_wp_groups[] = {"gpio0_0_grp",
 static const char * const smc0_nor_groups[] = {"smc0_nor_grp"};
 static const char * const smc0_nor_cs1_groups[] = {"smc0_nor_cs1_grp"};
 static const char * const smc0_nor_addr25_groups[] = {"smc0_nor_addr25_grp"};
-static const char * const smc0_nand_groups[] = {"smc0_nand_grp"};
+static const char * const smc0_nand_groups[] = {"smc0_nand_grp",
+		"smc0_nand8_grp"};
 static const char * const can0_groups[] = {"can0_0_grp", "can0_1_grp",
 		"can0_2_grp", "can0_3_grp", "can0_4_grp", "can0_5_grp",
 		"can0_6_grp", "can0_7_grp", "can0_8_grp", "can0_9_grp",
diff --git a/drivers/pinctrl/qcom/Kconfig b/drivers/pinctrl/qcom/Kconfig
index 93ef268d5ccd..3ebdc01f53c0 100644
--- a/drivers/pinctrl/qcom/Kconfig
+++ b/drivers/pinctrl/qcom/Kconfig
@@ -79,6 +79,15 @@ config PINCTRL_MSM8916
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found on the Qualcomm 8916 platform.
 
+config PINCTRL_MSM8994
+	tristate "Qualcomm 8994 pin controller driver"
+	depends on GPIOLIB && OF
+	select PINCTRL_MSM
+	help
+	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+	  Qualcomm TLMM block found in the Qualcomm 8994 platform. The
+	  Qualcomm 8992 platform is also supported by this driver.
+
 config PINCTRL_MSM8996
 	tristate "Qualcomm MSM8996 pin controller driver"
 	depends on GPIOLIB && OF
diff --git a/drivers/pinctrl/qcom/Makefile b/drivers/pinctrl/qcom/Makefile
index 8319e11cecb5..ab47764dbc5c 100644
--- a/drivers/pinctrl/qcom/Makefile
+++ b/drivers/pinctrl/qcom/Makefile
@@ -8,6 +8,7 @@ obj-$(CONFIG_PINCTRL_MSM8660)	+= pinctrl-msm8660.o
 obj-$(CONFIG_PINCTRL_MSM8960)	+= pinctrl-msm8960.o
 obj-$(CONFIG_PINCTRL_MSM8X74)	+= pinctrl-msm8x74.o
 obj-$(CONFIG_PINCTRL_MSM8916)	+= pinctrl-msm8916.o
+obj-$(CONFIG_PINCTRL_MSM8994)   += pinctrl-msm8994.o
 obj-$(CONFIG_PINCTRL_MSM8996)   += pinctrl-msm8996.o
 obj-$(CONFIG_PINCTRL_QDF2XXX)	+= pinctrl-qdf2xxx.o
 obj-$(CONFIG_PINCTRL_MDM9615)	+= pinctrl-mdm9615.o
diff --git a/drivers/pinctrl/qcom/pinctrl-msm8994.c b/drivers/pinctrl/qcom/pinctrl-msm8994.c
new file mode 100644
index 000000000000..8e16d9ae0c39
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-msm8994.c
@@ -0,0 +1,1379 @@
+/*
+ * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pinctrl/pinctrl.h>
+
+#include "pinctrl-msm.h"
+
+#define FUNCTION(fname)					\
+	[MSM_MUX_##fname] = {				\
+		.name = #fname,				\
+		.groups = fname##_groups,		\
+		.ngroups = ARRAY_SIZE(fname##_groups),	\
+	}
+
+#define PINGROUP(id, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11)	\
+	{						\
+		.name = "gpio" #id,			\
+		.pins = gpio##id##_pins,		\
+		.npins = ARRAY_SIZE(gpio##id##_pins),	\
+		.funcs = (int[]){			\
+			MSM_MUX_gpio,			\
+			MSM_MUX_##f1,			\
+			MSM_MUX_##f2,			\
+			MSM_MUX_##f3,			\
+			MSM_MUX_##f4,			\
+			MSM_MUX_##f5,			\
+			MSM_MUX_##f6,			\
+			MSM_MUX_##f7,			\
+			MSM_MUX_##f8,			\
+			MSM_MUX_##f9,			\
+			MSM_MUX_##f10,			\
+			MSM_MUX_##f11			\
+		},					\
+		.nfuncs = 12,				\
+		.ctl_reg = 0x1000 + 0x10 * id,		\
+		.io_reg = 0x1004 + 0x10 * id,		\
+		.intr_cfg_reg = 0x1008 + 0x10 * id,	\
+		.intr_status_reg = 0x100c + 0x10 * id,	\
+		.intr_target_reg = 0x1008 + 0x10 * id,	\
+		.mux_bit = 2,				\
+		.pull_bit = 0,				\
+		.drv_bit = 6,				\
+		.oe_bit = 9,				\
+		.in_bit = 0,				\
+		.out_bit = 1,				\
+		.intr_enable_bit = 0,			\
+		.intr_status_bit = 0,			\
+		.intr_target_bit = 5,			\
+		.intr_target_kpss_val = 4,		\
+		.intr_raw_status_bit = 4,		\
+		.intr_polarity_bit = 1,			\
+		.intr_detection_bit = 2,		\
+		.intr_detection_width = 2,		\
+	}
+
+#define SDC_PINGROUP(pg_name, ctl, pull, drv)		\
+	{						\
+		.name = #pg_name,			\
+		.pins = pg_name##_pins,			\
+		.npins = ARRAY_SIZE(pg_name##_pins),	\
+		.ctl_reg = ctl,				\
+		.io_reg = 0,				\
+		.intr_cfg_reg = 0,			\
+		.intr_status_reg = 0,			\
+		.intr_target_reg = 0,			\
+		.mux_bit = -1,				\
+		.pull_bit = pull,			\
+		.drv_bit = drv,				\
+		.oe_bit = -1,				\
+		.in_bit = -1,				\
+		.out_bit = -1,				\
+		.intr_enable_bit = -1,			\
+		.intr_status_bit = -1,			\
+		.intr_target_bit = -1,			\
+		.intr_target_kpss_val = -1,		\
+		.intr_raw_status_bit = -1,		\
+		.intr_polarity_bit = -1,		\
+		.intr_detection_bit = -1,		\
+		.intr_detection_width = -1,		\
+	}
+static const struct pinctrl_pin_desc msm8994_pins[] = {
+	PINCTRL_PIN(0, "GPIO_0"),
+	PINCTRL_PIN(1, "GPIO_1"),
+	PINCTRL_PIN(2, "GPIO_2"),
+	PINCTRL_PIN(3, "GPIO_3"),
+	PINCTRL_PIN(4, "GPIO_4"),
+	PINCTRL_PIN(5, "GPIO_5"),
+	PINCTRL_PIN(6, "GPIO_6"),
+	PINCTRL_PIN(7, "GPIO_7"),
+	PINCTRL_PIN(8, "GPIO_8"),
+	PINCTRL_PIN(9, "GPIO_9"),
+	PINCTRL_PIN(10, "GPIO_10"),
+	PINCTRL_PIN(11, "GPIO_11"),
+	PINCTRL_PIN(12, "GPIO_12"),
+	PINCTRL_PIN(13, "GPIO_13"),
+	PINCTRL_PIN(14, "GPIO_14"),
+	PINCTRL_PIN(15, "GPIO_15"),
+	PINCTRL_PIN(16, "GPIO_16"),
+	PINCTRL_PIN(17, "GPIO_17"),
+	PINCTRL_PIN(18, "GPIO_18"),
+	PINCTRL_PIN(19, "GPIO_19"),
+	PINCTRL_PIN(20, "GPIO_20"),
+	PINCTRL_PIN(21, "GPIO_21"),
+	PINCTRL_PIN(22, "GPIO_22"),
+	PINCTRL_PIN(23, "GPIO_23"),
+	PINCTRL_PIN(24, "GPIO_24"),
+	PINCTRL_PIN(25, "GPIO_25"),
+	PINCTRL_PIN(26, "GPIO_26"),
+	PINCTRL_PIN(27, "GPIO_27"),
+	PINCTRL_PIN(28, "GPIO_28"),
+	PINCTRL_PIN(29, "GPIO_29"),
+	PINCTRL_PIN(30, "GPIO_30"),
+	PINCTRL_PIN(31, "GPIO_31"),
+	PINCTRL_PIN(32, "GPIO_32"),
+	PINCTRL_PIN(33, "GPIO_33"),
+	PINCTRL_PIN(34, "GPIO_34"),
+	PINCTRL_PIN(35, "GPIO_35"),
+	PINCTRL_PIN(36, "GPIO_36"),
+	PINCTRL_PIN(37, "GPIO_37"),
+	PINCTRL_PIN(38, "GPIO_38"),
+	PINCTRL_PIN(39, "GPIO_39"),
+	PINCTRL_PIN(40, "GPIO_40"),
+	PINCTRL_PIN(41, "GPIO_41"),
+	PINCTRL_PIN(42, "GPIO_42"),
+	PINCTRL_PIN(43, "GPIO_43"),
+	PINCTRL_PIN(44, "GPIO_44"),
+	PINCTRL_PIN(45, "GPIO_45"),
+	PINCTRL_PIN(46, "GPIO_46"),
+	PINCTRL_PIN(47, "GPIO_47"),
+	PINCTRL_PIN(48, "GPIO_48"),
+	PINCTRL_PIN(49, "GPIO_49"),
+	PINCTRL_PIN(50, "GPIO_50"),
+	PINCTRL_PIN(51, "GPIO_51"),
+	PINCTRL_PIN(52, "GPIO_52"),
+	PINCTRL_PIN(53, "GPIO_53"),
+	PINCTRL_PIN(54, "GPIO_54"),
+	PINCTRL_PIN(55, "GPIO_55"),
+	PINCTRL_PIN(56, "GPIO_56"),
+	PINCTRL_PIN(57, "GPIO_57"),
+	PINCTRL_PIN(58, "GPIO_58"),
+	PINCTRL_PIN(59, "GPIO_59"),
+	PINCTRL_PIN(60, "GPIO_60"),
+	PINCTRL_PIN(61, "GPIO_61"),
+	PINCTRL_PIN(62, "GPIO_62"),
+	PINCTRL_PIN(63, "GPIO_63"),
+	PINCTRL_PIN(64, "GPIO_64"),
+	PINCTRL_PIN(65, "GPIO_65"),
+	PINCTRL_PIN(66, "GPIO_66"),
+	PINCTRL_PIN(67, "GPIO_67"),
+	PINCTRL_PIN(68, "GPIO_68"),
+	PINCTRL_PIN(69, "GPIO_69"),
+	PINCTRL_PIN(70, "GPIO_70"),
+	PINCTRL_PIN(71, "GPIO_71"),
+	PINCTRL_PIN(72, "GPIO_72"),
+	PINCTRL_PIN(73, "GPIO_73"),
+	PINCTRL_PIN(74, "GPIO_74"),
+	PINCTRL_PIN(75, "GPIO_75"),
+	PINCTRL_PIN(76, "GPIO_76"),
+	PINCTRL_PIN(77, "GPIO_77"),
+	PINCTRL_PIN(78, "GPIO_78"),
+	PINCTRL_PIN(79, "GPIO_79"),
+	PINCTRL_PIN(80, "GPIO_80"),
+	PINCTRL_PIN(81, "GPIO_81"),
+	PINCTRL_PIN(82, "GPIO_82"),
+	PINCTRL_PIN(83, "GPIO_83"),
+	PINCTRL_PIN(84, "GPIO_84"),
+	PINCTRL_PIN(85, "GPIO_85"),
+	PINCTRL_PIN(86, "GPIO_86"),
+	PINCTRL_PIN(87, "GPIO_87"),
+	PINCTRL_PIN(88, "GPIO_88"),
+	PINCTRL_PIN(89, "GPIO_89"),
+	PINCTRL_PIN(90, "GPIO_90"),
+	PINCTRL_PIN(91, "GPIO_91"),
+	PINCTRL_PIN(92, "GPIO_92"),
+	PINCTRL_PIN(93, "GPIO_93"),
+	PINCTRL_PIN(94, "GPIO_94"),
+	PINCTRL_PIN(95, "GPIO_95"),
+	PINCTRL_PIN(96, "GPIO_96"),
+	PINCTRL_PIN(97, "GPIO_97"),
+	PINCTRL_PIN(98, "GPIO_98"),
+	PINCTRL_PIN(99, "GPIO_99"),
+	PINCTRL_PIN(100, "GPIO_100"),
+	PINCTRL_PIN(101, "GPIO_101"),
+	PINCTRL_PIN(102, "GPIO_102"),
+	PINCTRL_PIN(103, "GPIO_103"),
+	PINCTRL_PIN(104, "GPIO_104"),
+	PINCTRL_PIN(105, "GPIO_105"),
+	PINCTRL_PIN(106, "GPIO_106"),
+	PINCTRL_PIN(107, "GPIO_107"),
+	PINCTRL_PIN(108, "GPIO_108"),
+	PINCTRL_PIN(109, "GPIO_109"),
+	PINCTRL_PIN(110, "GPIO_110"),
+	PINCTRL_PIN(111, "GPIO_111"),
+	PINCTRL_PIN(112, "GPIO_112"),
+	PINCTRL_PIN(113, "GPIO_113"),
+	PINCTRL_PIN(114, "GPIO_114"),
+	PINCTRL_PIN(115, "GPIO_115"),
+	PINCTRL_PIN(116, "GPIO_116"),
+	PINCTRL_PIN(117, "GPIO_117"),
+	PINCTRL_PIN(118, "GPIO_118"),
+	PINCTRL_PIN(119, "GPIO_119"),
+	PINCTRL_PIN(120, "GPIO_120"),
+	PINCTRL_PIN(121, "GPIO_121"),
+	PINCTRL_PIN(122, "GPIO_122"),
+	PINCTRL_PIN(123, "GPIO_123"),
+	PINCTRL_PIN(124, "GPIO_124"),
+	PINCTRL_PIN(125, "GPIO_125"),
+	PINCTRL_PIN(126, "GPIO_126"),
+	PINCTRL_PIN(127, "GPIO_127"),
+	PINCTRL_PIN(128, "GPIO_128"),
+	PINCTRL_PIN(129, "GPIO_129"),
+	PINCTRL_PIN(130, "GPIO_130"),
+	PINCTRL_PIN(131, "GPIO_131"),
+	PINCTRL_PIN(132, "GPIO_132"),
+	PINCTRL_PIN(133, "GPIO_133"),
+	PINCTRL_PIN(134, "GPIO_134"),
+	PINCTRL_PIN(135, "GPIO_135"),
+	PINCTRL_PIN(136, "GPIO_136"),
+	PINCTRL_PIN(137, "GPIO_137"),
+	PINCTRL_PIN(138, "GPIO_138"),
+	PINCTRL_PIN(139, "GPIO_139"),
+	PINCTRL_PIN(140, "GPIO_140"),
+	PINCTRL_PIN(141, "GPIO_141"),
+	PINCTRL_PIN(142, "GPIO_142"),
+	PINCTRL_PIN(143, "GPIO_143"),
+	PINCTRL_PIN(144, "GPIO_144"),
+	PINCTRL_PIN(145, "GPIO_145"),
+	PINCTRL_PIN(146, "SDC1_RCLK"),
+	PINCTRL_PIN(147, "SDC1_CLK"),
+	PINCTRL_PIN(148, "SDC1_CMD"),
+	PINCTRL_PIN(149, "SDC1_DATA"),
+	PINCTRL_PIN(150, "SDC2_CLK"),
+	PINCTRL_PIN(151, "SDC2_CMD"),
+	PINCTRL_PIN(152, "SDC2_DATA"),
+	PINCTRL_PIN(153, "SDC3_CLK"),
+	PINCTRL_PIN(154, "SDC3_CMD"),
+	PINCTRL_PIN(155, "SDC3_DATA"),
+};
+
+#define DECLARE_MSM_GPIO_PINS(pin) \
+	static const unsigned int gpio##pin##_pins[] = { pin }
+DECLARE_MSM_GPIO_PINS(0);
+DECLARE_MSM_GPIO_PINS(1);
+DECLARE_MSM_GPIO_PINS(2);
+DECLARE_MSM_GPIO_PINS(3);
+DECLARE_MSM_GPIO_PINS(4);
+DECLARE_MSM_GPIO_PINS(5);
+DECLARE_MSM_GPIO_PINS(6);
+DECLARE_MSM_GPIO_PINS(7);
+DECLARE_MSM_GPIO_PINS(8);
+DECLARE_MSM_GPIO_PINS(9);
+DECLARE_MSM_GPIO_PINS(10);
+DECLARE_MSM_GPIO_PINS(11);
+DECLARE_MSM_GPIO_PINS(12);
+DECLARE_MSM_GPIO_PINS(13);
+DECLARE_MSM_GPIO_PINS(14);
+DECLARE_MSM_GPIO_PINS(15);
+DECLARE_MSM_GPIO_PINS(16);
+DECLARE_MSM_GPIO_PINS(17);
+DECLARE_MSM_GPIO_PINS(18);
+DECLARE_MSM_GPIO_PINS(19);
+DECLARE_MSM_GPIO_PINS(20);
+DECLARE_MSM_GPIO_PINS(21);
+DECLARE_MSM_GPIO_PINS(22);
+DECLARE_MSM_GPIO_PINS(23);
+DECLARE_MSM_GPIO_PINS(24);
+DECLARE_MSM_GPIO_PINS(25);
+DECLARE_MSM_GPIO_PINS(26);
+DECLARE_MSM_GPIO_PINS(27);
+DECLARE_MSM_GPIO_PINS(28);
+DECLARE_MSM_GPIO_PINS(29);
+DECLARE_MSM_GPIO_PINS(30);
+DECLARE_MSM_GPIO_PINS(31);
+DECLARE_MSM_GPIO_PINS(32);
+DECLARE_MSM_GPIO_PINS(33);
+DECLARE_MSM_GPIO_PINS(34);
+DECLARE_MSM_GPIO_PINS(35);
+DECLARE_MSM_GPIO_PINS(36);
+DECLARE_MSM_GPIO_PINS(37);
+DECLARE_MSM_GPIO_PINS(38);
+DECLARE_MSM_GPIO_PINS(39);
+DECLARE_MSM_GPIO_PINS(40);
+DECLARE_MSM_GPIO_PINS(41);
+DECLARE_MSM_GPIO_PINS(42);
+DECLARE_MSM_GPIO_PINS(43);
+DECLARE_MSM_GPIO_PINS(44);
+DECLARE_MSM_GPIO_PINS(45);
+DECLARE_MSM_GPIO_PINS(46);
+DECLARE_MSM_GPIO_PINS(47);
+DECLARE_MSM_GPIO_PINS(48);
+DECLARE_MSM_GPIO_PINS(49);
+DECLARE_MSM_GPIO_PINS(50);
+DECLARE_MSM_GPIO_PINS(51);
+DECLARE_MSM_GPIO_PINS(52);
+DECLARE_MSM_GPIO_PINS(53);
+DECLARE_MSM_GPIO_PINS(54);
+DECLARE_MSM_GPIO_PINS(55);
+DECLARE_MSM_GPIO_PINS(56);
+DECLARE_MSM_GPIO_PINS(57);
+DECLARE_MSM_GPIO_PINS(58);
+DECLARE_MSM_GPIO_PINS(59);
+DECLARE_MSM_GPIO_PINS(60);
+DECLARE_MSM_GPIO_PINS(61);
+DECLARE_MSM_GPIO_PINS(62);
+DECLARE_MSM_GPIO_PINS(63);
+DECLARE_MSM_GPIO_PINS(64);
+DECLARE_MSM_GPIO_PINS(65);
+DECLARE_MSM_GPIO_PINS(66);
+DECLARE_MSM_GPIO_PINS(67);
+DECLARE_MSM_GPIO_PINS(68);
+DECLARE_MSM_GPIO_PINS(69);
+DECLARE_MSM_GPIO_PINS(70);
+DECLARE_MSM_GPIO_PINS(71);
+DECLARE_MSM_GPIO_PINS(72);
+DECLARE_MSM_GPIO_PINS(73);
+DECLARE_MSM_GPIO_PINS(74);
+DECLARE_MSM_GPIO_PINS(75);
+DECLARE_MSM_GPIO_PINS(76);
+DECLARE_MSM_GPIO_PINS(77);
+DECLARE_MSM_GPIO_PINS(78);
+DECLARE_MSM_GPIO_PINS(79);
+DECLARE_MSM_GPIO_PINS(80);
+DECLARE_MSM_GPIO_PINS(81);
+DECLARE_MSM_GPIO_PINS(82);
+DECLARE_MSM_GPIO_PINS(83);
+DECLARE_MSM_GPIO_PINS(84);
+DECLARE_MSM_GPIO_PINS(85);
+DECLARE_MSM_GPIO_PINS(86);
+DECLARE_MSM_GPIO_PINS(87);
+DECLARE_MSM_GPIO_PINS(88);
+DECLARE_MSM_GPIO_PINS(89);
+DECLARE_MSM_GPIO_PINS(90);
+DECLARE_MSM_GPIO_PINS(91);
+DECLARE_MSM_GPIO_PINS(92);
+DECLARE_MSM_GPIO_PINS(93);
+DECLARE_MSM_GPIO_PINS(94);
+DECLARE_MSM_GPIO_PINS(95);
+DECLARE_MSM_GPIO_PINS(96);
+DECLARE_MSM_GPIO_PINS(97);
+DECLARE_MSM_GPIO_PINS(98);
+DECLARE_MSM_GPIO_PINS(99);
+DECLARE_MSM_GPIO_PINS(100);
+DECLARE_MSM_GPIO_PINS(101);
+DECLARE_MSM_GPIO_PINS(102);
+DECLARE_MSM_GPIO_PINS(103);
+DECLARE_MSM_GPIO_PINS(104);
+DECLARE_MSM_GPIO_PINS(105);
+DECLARE_MSM_GPIO_PINS(106);
+DECLARE_MSM_GPIO_PINS(107);
+DECLARE_MSM_GPIO_PINS(108);
+DECLARE_MSM_GPIO_PINS(109);
+DECLARE_MSM_GPIO_PINS(110);
+DECLARE_MSM_GPIO_PINS(111);
+DECLARE_MSM_GPIO_PINS(112);
+DECLARE_MSM_GPIO_PINS(113);
+DECLARE_MSM_GPIO_PINS(114);
+DECLARE_MSM_GPIO_PINS(115);
+DECLARE_MSM_GPIO_PINS(116);
+DECLARE_MSM_GPIO_PINS(117);
+DECLARE_MSM_GPIO_PINS(118);
+DECLARE_MSM_GPIO_PINS(119);
+DECLARE_MSM_GPIO_PINS(120);
+DECLARE_MSM_GPIO_PINS(121);
+DECLARE_MSM_GPIO_PINS(122);
+DECLARE_MSM_GPIO_PINS(123);
+DECLARE_MSM_GPIO_PINS(124);
+DECLARE_MSM_GPIO_PINS(125);
+DECLARE_MSM_GPIO_PINS(126);
+DECLARE_MSM_GPIO_PINS(127);
+DECLARE_MSM_GPIO_PINS(128);
+DECLARE_MSM_GPIO_PINS(129);
+DECLARE_MSM_GPIO_PINS(130);
+DECLARE_MSM_GPIO_PINS(131);
+DECLARE_MSM_GPIO_PINS(132);
+DECLARE_MSM_GPIO_PINS(133);
+DECLARE_MSM_GPIO_PINS(134);
+DECLARE_MSM_GPIO_PINS(135);
+DECLARE_MSM_GPIO_PINS(136);
+DECLARE_MSM_GPIO_PINS(137);
+DECLARE_MSM_GPIO_PINS(138);
+DECLARE_MSM_GPIO_PINS(139);
+DECLARE_MSM_GPIO_PINS(140);
+DECLARE_MSM_GPIO_PINS(141);
+DECLARE_MSM_GPIO_PINS(142);
+DECLARE_MSM_GPIO_PINS(143);
+DECLARE_MSM_GPIO_PINS(144);
+DECLARE_MSM_GPIO_PINS(145);
+
+static const unsigned int sdc1_rclk_pins[] = { 146 };
+static const unsigned int sdc1_clk_pins[] = { 147 };
+static const unsigned int sdc1_cmd_pins[] = { 148 };
+static const unsigned int sdc1_data_pins[] = { 149 };
+static const unsigned int sdc2_clk_pins[] = { 150 };
+static const unsigned int sdc2_cmd_pins[] = { 151 };
+static const unsigned int sdc2_data_pins[] = { 152 };
+static const unsigned int sdc3_clk_pins[] = { 153 };
+static const unsigned int sdc3_cmd_pins[] = { 154 };
+static const unsigned int sdc3_data_pins[] = { 155 };
+
+enum msm8994_functions {
+	MSM_MUX_audio_ref_clk,
+	MSM_MUX_blsp_i2c1,
+	MSM_MUX_blsp_i2c2,
+	MSM_MUX_blsp_i2c3,
+	MSM_MUX_blsp_i2c4,
+	MSM_MUX_blsp_i2c5,
+	MSM_MUX_blsp_i2c6,
+	MSM_MUX_blsp_i2c7,
+	MSM_MUX_blsp_i2c8,
+	MSM_MUX_blsp_i2c9,
+	MSM_MUX_blsp_i2c10,
+	MSM_MUX_blsp_i2c11,
+	MSM_MUX_blsp_i2c12,
+	MSM_MUX_blsp_spi1,
+	MSM_MUX_blsp_spi1_cs1,
+	MSM_MUX_blsp_spi1_cs2,
+	MSM_MUX_blsp_spi1_cs3,
+	MSM_MUX_blsp_spi2,
+	MSM_MUX_blsp_spi2_cs1,
+	MSM_MUX_blsp_spi2_cs2,
+	MSM_MUX_blsp_spi2_cs3,
+	MSM_MUX_blsp_spi3,
+	MSM_MUX_blsp_spi4,
+	MSM_MUX_blsp_spi5,
+	MSM_MUX_blsp_spi6,
+	MSM_MUX_blsp_spi7,
+	MSM_MUX_blsp_spi8,
+	MSM_MUX_blsp_spi9,
+	MSM_MUX_blsp_spi10,
+	MSM_MUX_blsp_spi10_cs1,
+	MSM_MUX_blsp_spi10_cs2,
+	MSM_MUX_blsp_spi10_cs3,
+	MSM_MUX_blsp_spi11,
+	MSM_MUX_blsp_spi12,
+	MSM_MUX_blsp_uart1,
+	MSM_MUX_blsp_uart2,
+	MSM_MUX_blsp_uart3,
+	MSM_MUX_blsp_uart4,
+	MSM_MUX_blsp_uart5,
+	MSM_MUX_blsp_uart6,
+	MSM_MUX_blsp_uart7,
+	MSM_MUX_blsp_uart8,
+	MSM_MUX_blsp_uart9,
+	MSM_MUX_blsp_uart10,
+	MSM_MUX_blsp_uart11,
+	MSM_MUX_blsp_uart12,
+	MSM_MUX_blsp_uim1,
+	MSM_MUX_blsp_uim2,
+	MSM_MUX_blsp_uim3,
+	MSM_MUX_blsp_uim4,
+	MSM_MUX_blsp_uim5,
+	MSM_MUX_blsp_uim6,
+	MSM_MUX_blsp_uim7,
+	MSM_MUX_blsp_uim8,
+	MSM_MUX_blsp_uim9,
+	MSM_MUX_blsp_uim10,
+	MSM_MUX_blsp_uim11,
+	MSM_MUX_blsp_uim12,
+	MSM_MUX_blsp11_i2c_scl_b,
+	MSM_MUX_blsp11_i2c_sda_b,
+	MSM_MUX_blsp11_uart_rx_b,
+	MSM_MUX_blsp11_uart_tx_b,
+	MSM_MUX_cam_mclk0,
+	MSM_MUX_cam_mclk1,
+	MSM_MUX_cam_mclk2,
+	MSM_MUX_cam_mclk3,
+	MSM_MUX_cci_async_in0,
+	MSM_MUX_cci_async_in1,
+	MSM_MUX_cci_async_in2,
+	MSM_MUX_cci_i2c0,
+	MSM_MUX_cci_i2c1,
+	MSM_MUX_cci_timer0,
+	MSM_MUX_cci_timer1,
+	MSM_MUX_cci_timer2,
+	MSM_MUX_cci_timer3,
+	MSM_MUX_cci_timer4,
+	MSM_MUX_gcc_gp1_clk_a,
+	MSM_MUX_gcc_gp1_clk_b,
+	MSM_MUX_gcc_gp2_clk_a,
+	MSM_MUX_gcc_gp2_clk_b,
+	MSM_MUX_gcc_gp3_clk_a,
+	MSM_MUX_gcc_gp3_clk_b,
+	MSM_MUX_gp_mn,
+	MSM_MUX_gp_pdm0,
+	MSM_MUX_gp_pdm1,
+	MSM_MUX_gp_pdm2,
+	MSM_MUX_gp0_clk,
+	MSM_MUX_gp1_clk,
+	MSM_MUX_gps_tx,
+	MSM_MUX_gsm_tx,
+	MSM_MUX_hdmi_cec,
+	MSM_MUX_hdmi_ddc,
+	MSM_MUX_hdmi_hpd,
+	MSM_MUX_hdmi_rcv,
+	MSM_MUX_mdp_vsync,
+	MSM_MUX_mss_lte,
+	MSM_MUX_nav_pps,
+	MSM_MUX_nav_tsync,
+	MSM_MUX_qdss_cti_trig_in_a,
+	MSM_MUX_qdss_cti_trig_in_b,
+	MSM_MUX_qdss_cti_trig_in_c,
+	MSM_MUX_qdss_cti_trig_in_d,
+	MSM_MUX_qdss_cti_trig_out_a,
+	MSM_MUX_qdss_cti_trig_out_b,
+	MSM_MUX_qdss_cti_trig_out_c,
+	MSM_MUX_qdss_cti_trig_out_d,
+	MSM_MUX_qdss_traceclk_a,
+	MSM_MUX_qdss_traceclk_b,
+	MSM_MUX_qdss_tracectl_a,
+	MSM_MUX_qdss_tracectl_b,
+	MSM_MUX_qdss_tracedata_a,
+	MSM_MUX_qdss_tracedata_b,
+	MSM_MUX_qua_mi2s,
+	MSM_MUX_pci_e0,
+	MSM_MUX_pci_e1,
+	MSM_MUX_pri_mi2s,
+	MSM_MUX_sdc4,
+	MSM_MUX_sec_mi2s,
+	MSM_MUX_slimbus,
+	MSM_MUX_spkr_i2s,
+	MSM_MUX_ter_mi2s,
+	MSM_MUX_tsif1,
+	MSM_MUX_tsif2,
+	MSM_MUX_uim1,
+	MSM_MUX_uim2,
+	MSM_MUX_uim3,
+	MSM_MUX_uim4,
+	MSM_MUX_uim_batt_alarm,
+	MSM_MUX_gpio,
+	MSM_MUX_NA,
+};
+
+static const char * const gpio_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", "gpio6", "gpio7",
+	"gpio8", "gpio9", "gpio10", "gpio11", "gpio12", "gpio13", "gpio14",
+	"gpio15", "gpio16", "gpio17", "gpio18", "gpio19", "gpio20", "gpio21",
+	"gpio22", "gpio23", "gpio24", "gpio25", "gpio26", "gpio27", "gpio28",
+	"gpio29", "gpio30", "gpio31", "gpio32", "gpio33", "gpio34", "gpio35",
+	"gpio36", "gpio37", "gpio38", "gpio39", "gpio40", "gpio41", "gpio42",
+	"gpio43", "gpio44", "gpio45", "gpio46", "gpio47", "gpio48", "gpio49",
+	"gpio50", "gpio51", "gpio52", "gpio53", "gpio54", "gpio55", "gpio56",
+	"gpio57", "gpio58", "gpio59", "gpio60", "gpio61", "gpio62", "gpio63",
+	"gpio64", "gpio65", "gpio66", "gpio67", "gpio68", "gpio69", "gpio70",
+	"gpio71", "gpio72", "gpio73", "gpio74", "gpio75", "gpio76", "gpio77",
+	"gpio78", "gpio79", "gpio80", "gpio81", "gpio82", "gpio83", "gpio84",
+	"gpio85", "gpio86", "gpio87", "gpio88", "gpio89", "gpio90", "gpio91",
+	"gpio92", "gpio93", "gpio94", "gpio95", "gpio96", "gpio97", "gpio98",
+	"gpio99", "gpio100", "gpio101", "gpio102", "gpio103", "gpio104",
+	"gpio105", "gpio106", "gpio107", "gpio108", "gpio109", "gpio110",
+	"gpio111", "gpio112", "gpio113", "gpio114", "gpio115", "gpio116",
+	"gpio117", "gpio118", "gpio119", "gpio120", "gpio121", "gpio122",
+	"gpio123", "gpio124", "gpio125", "gpio126", "gpio127", "gpio128",
+	"gpio129", "gpio130", "gpio131", "gpio132", "gpio133", "gpio134",
+	"gpio135", "gpio136", "gpio137", "gpio138", "gpio139", "gpio140",
+	"gpio141", "gpio142", "gpio143", "gpio144", "gpio145",
+};
+
+static const char * const blsp_spi1_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3"
+};
+static const char * const blsp_uart1_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3"
+};
+static const char * const blsp_uim1_groups[] = {
+	"gpio0", "gpio1"
+};
+static const char * const hdmi_rcv_groups[] = {
+	"gpio0"
+};
+static const char * const blsp_i2c1_groups[] = {
+	"gpio2", "gpio3"
+};
+static const char * const blsp_spi2_groups[] = {
+	"gpio4", "gpio5", "gpio6", "gpio7"
+};
+static const char * const blsp_uart2_groups[] = {
+	"gpio4", "gpio5", "gpio6", "gpio7"
+};
+static const char * const blsp_uim2_groups[] = {
+	"gpio4", "gpio5"
+};
+static const char * const qdss_cti_trig_out_b_groups[] = {
+	"gpio4",
+};
+static const char * const qdss_cti_trig_in_b_groups[] = {
+	"gpio5",
+};
+static const char * const blsp_i2c2_groups[] = {
+	"gpio6", "gpio7"
+};
+static const char * const blsp_spi3_groups[] = {
+	"gpio8", "gpio9", "gpio10", "gpio11"
+};
+static const char * const blsp_uart3_groups[] = {
+	"gpio8", "gpio9", "gpio10", "gpio11"
+};
+static const char * const blsp_uim3_groups[] = {
+	"gpio8", "gpio9"
+};
+static const char * const blsp_spi1_cs1_groups[] = {
+	"gpio8"
+};
+static const char * const blsp_spi1_cs2_groups[] = {
+	"gpio9", "gpio11"
+};
+static const char * const mdp_vsync_groups[] = {
+	"gpio10", "gpio11", "gpio12"
+};
+static const char * const blsp_i2c3_groups[] = {
+	"gpio10", "gpio11"
+};
+static const char * const blsp_spi1_cs3_groups[] = {
+	"gpio10"
+};
+static const char * const qdss_tracedata_b_groups[] = {
+	"gpio13", "gpio14", "gpio15", "gpio16", "gpio17", "gpio18",
+	"gpio19", "gpio21", "gpio22", "gpio23", "gpio25", "gpio26",
+	"gpio57", "gpio58", "gpio92", "gpio93",
+};
+static const char * const cam_mclk0_groups[] = {
+	"gpio13"
+};
+static const char * const cam_mclk1_groups[] = {
+	"gpio14"
+};
+static const char * const cam_mclk2_groups[] = {
+	"gpio15"
+};
+static const char * const cam_mclk3_groups[] = {
+	"gpio16"
+};
+static const char * const cci_i2c0_groups[] = {
+	"gpio17", "gpio18"
+};
+static const char * const blsp_spi4_groups[] = {
+	"gpio17", "gpio18", "gpio19", "gpio20"
+};
+static const char * const blsp_uart4_groups[] = {
+	"gpio17", "gpio18", "gpio19", "gpio20"
+};
+static const char * const blsp_uim4_groups[] = {
+	"gpio17", "gpio18"
+};
+static const char * const cci_i2c1_groups[] = {
+	"gpio19", "gpio20"
+};
+static const char * const blsp_i2c4_groups[] = {
+	"gpio19", "gpio20"
+};
+static const char * const cci_timer0_groups[] = {
+	"gpio21"
+};
+static const char * const blsp_spi5_groups[] = {
+	"gpio21", "gpio22", "gpio23", "gpio24"
+};
+static const char * const blsp_uart5_groups[] = {
+	"gpio21", "gpio22", "gpio23", "gpio24"
+};
+static const char * const blsp_uim5_groups[] = {
+	"gpio21", "gpio22"
+};
+static const char * const cci_timer1_groups[] = {
+	"gpio22"
+};
+static const char * const cci_timer2_groups[] = {
+	"gpio23"
+};
+static const char * const blsp_i2c5_groups[] = {
+	"gpio23", "gpio24"
+};
+static const char * const cci_timer3_groups[] = {
+	"gpio24"
+};
+static const char * const cci_async_in1_groups[] = {
+	"gpio24"
+};
+static const char * const cci_timer4_groups[] = {
+	"gpio25"
+};
+static const char * const cci_async_in2_groups[] = {
+	"gpio25"
+};
+static const char * const blsp_spi6_groups[] = {
+	"gpio25", "gpio26", "gpio27", "gpio28"
+};
+static const char * const blsp_uart6_groups[] = {
+	"gpio25", "gpio26", "gpio27", "gpio28"
+};
+static const char * const blsp_uim6_groups[] = {
+	"gpio25", "gpio26"
+};
+static const char * const cci_async_in0_groups[] = {
+	"gpio26"
+};
+static const char * const gp0_clk_groups[] = {
+	"gpio26"
+};
+static const char * const gp1_clk_groups[] = {
+	"gpio27", "gpio57", "gpio78"
+};
+static const char * const blsp_i2c6_groups[] = {
+	"gpio27", "gpio28"
+};
+static const char * const qdss_tracectl_a_groups[] = {
+	"gpio27",
+};
+static const char * const qdss_traceclk_a_groups[] = {
+	"gpio28",
+};
+static const char * const gp_mn_groups[] = {
+	"gpio29"
+};
+static const char * const hdmi_cec_groups[] = {
+	"gpio31"
+};
+static const char * const hdmi_ddc_groups[] = {
+	"gpio32", "gpio33"
+};
+static const char * const hdmi_hpd_groups[] = {
+	"gpio34"
+};
+static const char * const uim3_groups[] = {
+	"gpio35", "gpio36", "gpio37", "gpio38"
+};
+static const char * const pci_e1_groups[] = {
+	"gpio35", "gpio36",
+};
+static const char * const blsp_spi7_groups[] = {
+	"gpio41", "gpio42", "gpio43", "gpio44"
+};
+static const char * const blsp_uart7_groups[] = {
+	"gpio41", "gpio42", "gpio43", "gpio44"
+};
+static const char * const blsp_uim7_groups[] = {
+	"gpio41", "gpio42"
+};
+static const char * const qdss_cti_trig_out_c_groups[] = {
+	"gpio41",
+};
+static const char * const qdss_cti_trig_in_c_groups[] = {
+	"gpio42",
+};
+static const char * const blsp_i2c7_groups[] = {
+	"gpio43", "gpio44"
+};
+static const char * const blsp_spi8_groups[] = {
+	"gpio45", "gpio46", "gpio47", "gpio48"
+};
+static const char * const blsp_uart8_groups[] = {
+	"gpio45", "gpio46", "gpio47", "gpio48"
+};
+static const char * const blsp_uim8_groups[] = {
+	"gpio45", "gpio46"
+};
+static const char * const blsp_i2c8_groups[] = {
+	"gpio47", "gpio48"
+};
+static const char * const blsp_spi10_cs1_groups[] = {
+	"gpio47", "gpio67"
+};
+static const char * const blsp_spi10_cs2_groups[] = {
+	"gpio48", "gpio68"
+};
+static const char * const uim2_groups[] = {
+	"gpio49", "gpio50", "gpio51", "gpio52"
+};
+static const char * const blsp_spi9_groups[] = {
+	"gpio49", "gpio50", "gpio51", "gpio52"
+};
+static const char * const blsp_uart9_groups[] = {
+	"gpio49", "gpio50", "gpio51", "gpio52"
+};
+static const char * const blsp_uim9_groups[] = {
+	"gpio49", "gpio50"
+};
+static const char * const blsp_i2c9_groups[] = {
+	"gpio51", "gpio52"
+};
+static const char * const pci_e0_groups[] = {
+	"gpio53", "gpio54",
+};
+static const char * const uim4_groups[] = {
+	"gpio53", "gpio54", "gpio55", "gpio56"
+};
+static const char * const blsp_spi10_groups[] = {
+	"gpio53", "gpio54", "gpio55", "gpio56"
+};
+static const char * const blsp_uart10_groups[] = {
+	"gpio53", "gpio54", "gpio55", "gpio56"
+};
+static const char * const blsp_uim10_groups[] = {
+	"gpio53", "gpio54"
+};
+static const char * const qdss_tracedata_a_groups[] = {
+	"gpio53", "gpio54", "gpio63", "gpio64", "gpio65",
+	"gpio66", "gpio67", "gpio74", "gpio75", "gpio76",
+	"gpio77", "gpio85", "gpio86", "gpio87", "gpio89",
+	"gpio90"
+};
+static const char * const gp_pdm0_groups[] = {
+	"gpio54", "gpio95"
+};
+static const char * const blsp_i2c10_groups[] = {
+	"gpio55", "gpio56"
+};
+static const char * const qdss_cti_trig_in_a_groups[] = {
+	"gpio55",
+};
+static const char * const qdss_cti_trig_out_a_groups[] = {
+	"gpio56",
+};
+static const char * const qua_mi2s_groups[] = {
+	"gpio57", "gpio58", "gpio59", "gpio60", "gpio61", "gpio62", "gpio63",
+};
+static const char * const gcc_gp1_clk_a_groups[] = {
+	"gpio57"
+};
+static const char * const gcc_gp2_clk_a_groups[] = {
+	"gpio58"
+};
+static const char * const gcc_gp3_clk_a_groups[] = {
+	"gpio59"
+};
+static const char * const blsp_spi2_cs1_groups[] = {
+	"gpio62"
+};
+static const char * const blsp_spi2_cs2_groups[] = {
+	"gpio63"
+};
+static const char * const gp_pdm2_groups[] = {
+	"gpio63", "gpio79"
+};
+static const char * const pri_mi2s_groups[] = {
+	"gpio64", "gpio65", "gpio66", "gpio67", "gpio68"
+};
+static const char * const blsp_spi2_cs3_groups[] = {
+	"gpio66"
+};
+static const char * const spkr_i2s_groups[] = {
+	"gpio69", "gpio70", "gpio71", "gpio72"
+};
+static const char * const audio_ref_clk_groups[] = {
+	"gpio69"
+};
+static const char * const slimbus_groups[] = {
+	"gpio70", "gpio71"
+};
+static const char * const ter_mi2s_groups[] = {
+	"gpio73", "gpio74", "gpio75", "gpio76", "gpio77"
+};
+static const char * const gp_pdm1_groups[] = {
+	"gpio74", "gpio86"
+};
+static const char * const sec_mi2s_groups[] = {
+	"gpio78", "gpio79", "gpio80", "gpio81", "gpio82"
+};
+static const char * const gcc_gp1_clk_b_groups[] = {
+	"gpio78"
+};
+static const char * const blsp_spi11_groups[] = {
+	"gpio81", "gpio82", "gpio83", "gpio84"
+};
+static const char * const blsp_uart11_groups[] = {
+	"gpio81", "gpio82", "gpio83", "gpio84"
+};
+static const char * const blsp_uim11_groups[] = {
+	"gpio81", "gpio82"
+};
+static const char * const gcc_gp2_clk_b_groups[] = {
+	"gpio81"
+};
+static const char * const gcc_gp3_clk_b_groups[] = {
+	"gpio82"
+};
+static const char * const blsp_i2c11_groups[] = {
+	"gpio83", "gpio84"
+};
+static const char * const blsp_uart12_groups[] = {
+	"gpio85", "gpio86", "gpio87", "gpio88"
+};
+static const char * const blsp_uim12_groups[] = {
+	"gpio85", "gpio86"
+};
+static const char * const blsp_i2c12_groups[] = {
+	"gpio87", "gpio88"
+};
+static const char * const blsp_spi12_groups[] = {
+	"gpio85", "gpio86", "gpio87", "gpio88"
+};
+static const char * const tsif1_groups[] = {
+	"gpio89", "gpio90", "gpio91", "gpio110", "gpio111"
+};
+static const char * const blsp_spi10_cs3_groups[] = {
+	"gpio90"
+};
+static const char * const sdc4_groups[] = {
+	"gpio91", "gpio92", "gpio93", "gpio94", "gpio95", "gpio96"
+};
+static const char * const qdss_traceclk_b_groups[] = {
+	"gpio91",
+};
+static const char * const tsif2_groups[] = {
+	"gpio92", "gpio93", "gpio94", "gpio95", "gpio96"
+};
+static const char * const qdss_tracectl_b_groups[] = {
+	"gpio94",
+};
+static const char * const qdss_cti_trig_out_d_groups[] = {
+	"gpio95",
+};
+static const char * const qdss_cti_trig_in_d_groups[] = {
+	"gpio96",
+};
+static const char * const uim1_groups[] = {
+	"gpio97", "gpio98", "gpio99", "gpio100"
+};
+static const char * const uim_batt_alarm_groups[] = {
+	"gpio101"
+};
+static const char * const blsp11_uart_tx_b_groups[] = {
+	"gpio111"
+};
+static const char * const blsp11_uart_rx_b_groups[] = {
+	"gpio112"
+};
+static const char * const blsp11_i2c_sda_b_groups[] = {
+	"gpio113"
+};
+static const char * const blsp11_i2c_scl_b_groups[] = {
+	"gpio114"
+};
+static const char * const gsm_tx_groups[] = {
+	"gpio126", "gpio131", "gpio132", "gpio133"
+};
+static const char * const nav_tsync_groups[] = {
+	"gpio127"
+};
+static const char * const nav_pps_groups[] = {
+	"gpio127"
+};
+static const char * const gps_tx_groups[] = {
+	"gpio130"
+};
+static const char * const mss_lte_groups[] = {
+	"gpio134", "gpio135"
+};
+
+static const struct msm_function msm8994_functions[] = {
+	FUNCTION(audio_ref_clk),
+	FUNCTION(blsp_i2c1),
+	FUNCTION(blsp_i2c2),
+	FUNCTION(blsp_i2c3),
+	FUNCTION(blsp_i2c4),
+	FUNCTION(blsp_i2c5),
+	FUNCTION(blsp_i2c6),
+	FUNCTION(blsp_i2c7),
+	FUNCTION(blsp_i2c8),
+	FUNCTION(blsp_i2c9),
+	FUNCTION(blsp_i2c10),
+	FUNCTION(blsp_i2c11),
+	FUNCTION(blsp_i2c12),
+	FUNCTION(blsp_spi1),
+	FUNCTION(blsp_spi1_cs1),
+	FUNCTION(blsp_spi1_cs2),
+	FUNCTION(blsp_spi1_cs3),
+	FUNCTION(blsp_spi2),
+	FUNCTION(blsp_spi2_cs1),
+	FUNCTION(blsp_spi2_cs2),
+	FUNCTION(blsp_spi2_cs3),
+	FUNCTION(blsp_spi3),
+	FUNCTION(blsp_spi4),
+	FUNCTION(blsp_spi5),
+	FUNCTION(blsp_spi6),
+	FUNCTION(blsp_spi7),
+	FUNCTION(blsp_spi8),
+	FUNCTION(blsp_spi9),
+	FUNCTION(blsp_spi10),
+	FUNCTION(blsp_spi10_cs1),
+	FUNCTION(blsp_spi10_cs2),
+	FUNCTION(blsp_spi10_cs3),
+	FUNCTION(blsp_spi11),
+	FUNCTION(blsp_spi12),
+	FUNCTION(blsp_uart1),
+	FUNCTION(blsp_uart2),
+	FUNCTION(blsp_uart3),
+	FUNCTION(blsp_uart4),
+	FUNCTION(blsp_uart5),
+	FUNCTION(blsp_uart6),
+	FUNCTION(blsp_uart7),
+	FUNCTION(blsp_uart8),
+	FUNCTION(blsp_uart9),
+	FUNCTION(blsp_uart10),
+	FUNCTION(blsp_uart11),
+	FUNCTION(blsp_uart12),
+	FUNCTION(blsp_uim1),
+	FUNCTION(blsp_uim2),
+	FUNCTION(blsp_uim3),
+	FUNCTION(blsp_uim4),
+	FUNCTION(blsp_uim5),
+	FUNCTION(blsp_uim6),
+	FUNCTION(blsp_uim7),
+	FUNCTION(blsp_uim8),
+	FUNCTION(blsp_uim9),
+	FUNCTION(blsp_uim10),
+	FUNCTION(blsp_uim11),
+	FUNCTION(blsp_uim12),
+	FUNCTION(blsp11_i2c_scl_b),
+	FUNCTION(blsp11_i2c_sda_b),
+	FUNCTION(blsp11_uart_rx_b),
+	FUNCTION(blsp11_uart_tx_b),
+	FUNCTION(cam_mclk0),
+	FUNCTION(cam_mclk1),
+	FUNCTION(cam_mclk2),
+	FUNCTION(cam_mclk3),
+	FUNCTION(cci_async_in0),
+	FUNCTION(cci_async_in1),
+	FUNCTION(cci_async_in2),
+	FUNCTION(cci_i2c0),
+	FUNCTION(cci_i2c1),
+	FUNCTION(cci_timer0),
+	FUNCTION(cci_timer1),
+	FUNCTION(cci_timer2),
+	FUNCTION(cci_timer3),
+	FUNCTION(cci_timer4),
+	FUNCTION(gcc_gp1_clk_a),
+	FUNCTION(gcc_gp1_clk_b),
+	FUNCTION(gcc_gp2_clk_a),
+	FUNCTION(gcc_gp2_clk_b),
+	FUNCTION(gcc_gp3_clk_a),
+	FUNCTION(gcc_gp3_clk_b),
+	FUNCTION(gp_mn),
+	FUNCTION(gp_pdm0),
+	FUNCTION(gp_pdm1),
+	FUNCTION(gp_pdm2),
+	FUNCTION(gp0_clk),
+	FUNCTION(gp1_clk),
+	FUNCTION(gps_tx),
+	FUNCTION(gsm_tx),
+	FUNCTION(hdmi_cec),
+	FUNCTION(hdmi_ddc),
+	FUNCTION(hdmi_hpd),
+	FUNCTION(hdmi_rcv),
+	FUNCTION(mdp_vsync),
+	FUNCTION(mss_lte),
+	FUNCTION(nav_pps),
+	FUNCTION(nav_tsync),
+	FUNCTION(qdss_cti_trig_in_a),
+	FUNCTION(qdss_cti_trig_in_b),
+	FUNCTION(qdss_cti_trig_in_c),
+	FUNCTION(qdss_cti_trig_in_d),
+	FUNCTION(qdss_cti_trig_out_a),
+	FUNCTION(qdss_cti_trig_out_b),
+	FUNCTION(qdss_cti_trig_out_c),
+	FUNCTION(qdss_cti_trig_out_d),
+	FUNCTION(qdss_traceclk_a),
+	FUNCTION(qdss_traceclk_b),
+	FUNCTION(qdss_tracectl_a),
+	FUNCTION(qdss_tracectl_b),
+	FUNCTION(qdss_tracedata_a),
+	FUNCTION(qdss_tracedata_b),
+	FUNCTION(qua_mi2s),
+	FUNCTION(pci_e0),
+	FUNCTION(pci_e1),
+	FUNCTION(pri_mi2s),
+	FUNCTION(sdc4),
+	FUNCTION(sec_mi2s),
+	FUNCTION(slimbus),
+	FUNCTION(spkr_i2s),
+	FUNCTION(ter_mi2s),
+	FUNCTION(tsif1),
+	FUNCTION(tsif2),
+	FUNCTION(uim_batt_alarm),
+	FUNCTION(uim1),
+	FUNCTION(uim2),
+	FUNCTION(uim3),
+	FUNCTION(uim4),
+	FUNCTION(gpio),
+};
+
+static const struct msm_pingroup msm8994_groups[] = {
+	PINGROUP(0,   blsp_spi1, blsp_uart1, blsp_uim1, hdmi_rcv, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(1,   blsp_spi1, blsp_uart1, blsp_uim1, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(2,   blsp_spi1, blsp_uart1, blsp_i2c1, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(3,   blsp_spi1, blsp_uart1, blsp_i2c1, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(4,   blsp_spi2, blsp_uart2, blsp_uim2, NA, qdss_cti_trig_out_b,
+		 NA, NA, NA, NA, NA, NA),
+	PINGROUP(5,   blsp_spi2, blsp_uart2, blsp_uim2, NA, qdss_cti_trig_in_b,
+		 NA, NA, NA, NA, NA, NA),
+	PINGROUP(6,   blsp_spi2, blsp_uart2, blsp_i2c2, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(7,   blsp_spi2, blsp_uart2, blsp_i2c2, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(8,   blsp_spi3, blsp_uart3, blsp_uim3, blsp_spi1_cs1, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(9,   blsp_spi3, blsp_uart3, blsp_uim3, blsp_spi1_cs2, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(10,  mdp_vsync, blsp_spi3, blsp_uart3, blsp_i2c3,
+		 blsp_spi1_cs3, NA, NA, NA, NA, NA, NA),
+	PINGROUP(11,  mdp_vsync, blsp_spi3, blsp_uart3, blsp_i2c3,
+		 blsp_spi1_cs2, NA, NA, NA, NA, NA, NA),
+	PINGROUP(12,  mdp_vsync, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(13,  cam_mclk0, NA, NA, qdss_tracedata_b, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(14,  cam_mclk1, NA, NA, qdss_tracedata_b, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(15,  cam_mclk2, NA, qdss_tracedata_b, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(16,  cam_mclk3, NA, qdss_tracedata_b, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(17,  cci_i2c0, blsp_spi4, blsp_uart4, blsp_uim4, NA,
+		 qdss_tracedata_b, NA, NA, NA, NA, NA),
+	PINGROUP(18,  cci_i2c0, blsp_spi4, blsp_uart4, blsp_uim4, NA,
+		 qdss_tracedata_b, NA, NA, NA, NA, NA),
+	PINGROUP(19,  cci_i2c1, blsp_spi4, blsp_uart4, blsp_i2c4, NA,
+		 qdss_tracedata_b, NA, NA, NA, NA, NA),
+	PINGROUP(20,  cci_i2c1, blsp_spi4, blsp_uart4, blsp_i2c4, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(21,  cci_timer0, blsp_spi5, blsp_uart5, blsp_uim5, NA,
+		 qdss_tracedata_b, NA, NA, NA, NA, NA),
+	PINGROUP(22,  cci_timer1, blsp_spi5, blsp_uart5, blsp_uim5, NA,
+		 qdss_tracedata_b, NA, NA, NA, NA, NA),
+	PINGROUP(23,  cci_timer2, blsp_spi5, blsp_uart5, blsp_i2c5, NA, NA,
+		 qdss_tracedata_b, NA, NA, NA, NA),
+	PINGROUP(24,  cci_timer3, cci_async_in1, blsp_spi5, blsp_uart5,
+		 blsp_i2c5, NA, NA, NA, NA, NA, NA),
+	PINGROUP(25,  cci_timer4, cci_async_in2, blsp_spi6, blsp_uart6,
+		 blsp_uim6, NA, NA, qdss_tracedata_b, NA, NA, NA),
+	PINGROUP(26,  cci_async_in0, blsp_spi6, blsp_uart6, blsp_uim6, gp0_clk,
+		 NA, qdss_tracedata_b, NA, NA, NA, NA),
+	PINGROUP(27,  blsp_spi6, blsp_uart6, blsp_i2c6, gp1_clk,
+		 qdss_tracectl_a, NA, NA, NA, NA, NA, NA),
+	PINGROUP(28,  blsp_spi6, blsp_uart6, blsp_i2c6, qdss_traceclk_a, NA,
+		 NA, NA, NA, NA, NA, NA),
+	PINGROUP(29,  gp_mn, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(30,  NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(31,  hdmi_cec, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(32,  hdmi_ddc, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(33,  hdmi_ddc, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(34,  hdmi_hpd, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(35,  uim3, pci_e1, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(36,  uim3, pci_e1, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(37,  uim3, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(38,  uim3, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(39,  NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(40,  NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(41,  blsp_spi7, blsp_uart7, blsp_uim7, qdss_cti_trig_out_c,
+		 NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(42,  blsp_spi7, blsp_uart7, blsp_uim7, qdss_cti_trig_in_c, NA,
+		 NA, NA, NA, NA, NA, NA),
+	PINGROUP(43,  blsp_spi7, blsp_uart7, blsp_i2c7, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(44,  blsp_spi7, blsp_uart7, blsp_i2c7, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(45,  blsp_spi8, blsp_uart8, blsp_uim8, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(46,  blsp_spi8, blsp_uart8, blsp_uim8, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(47,  blsp_spi8, blsp_uart8, blsp_i2c8, blsp_spi10_cs1, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(48,  blsp_spi8, blsp_uart8, blsp_i2c8, blsp_spi10_cs2, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(49,  uim2, blsp_spi9, blsp_uart9, blsp_uim9, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(50,  uim2, blsp_spi9, blsp_uart9, blsp_uim9, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(51,  uim2, blsp_spi9, blsp_uart9, blsp_i2c9, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(52,  uim2, blsp_spi9, blsp_uart9, blsp_i2c9, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(53,  uim4, pci_e0, blsp_spi10, blsp_uart10, blsp_uim10, NA,
+		 NA, qdss_tracedata_a, NA, NA, NA),
+	PINGROUP(54,  uim4, pci_e0, blsp_spi10, blsp_uart10, blsp_uim10,
+		 gp_pdm0, NA, NA, qdss_tracedata_a, NA, NA),
+	PINGROUP(55,  uim4, blsp_spi10, blsp_uart10, blsp_i2c10, NA, NA, NA,
+		 qdss_cti_trig_in_a, NA, NA, NA),
+	PINGROUP(56,  uim4, blsp_spi10, blsp_uart10, blsp_i2c10, NA, NA,
+		 qdss_cti_trig_out_a, NA, NA, NA, NA),
+	PINGROUP(57,  qua_mi2s, gcc_gp1_clk_a, NA, NA, qdss_tracedata_b, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(58,  qua_mi2s, gcc_gp2_clk_a, NA, NA, qdss_tracedata_b, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(59,  qua_mi2s, gcc_gp3_clk_a, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(60,  qua_mi2s, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(61,  qua_mi2s, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(62,  qua_mi2s, blsp_spi2_cs1, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(63,  qua_mi2s, blsp_spi2_cs2, gp_pdm2, NA, NA, NA, NA, NA,
+		 qdss_tracedata_a, NA, NA),
+	PINGROUP(64,  pri_mi2s, NA, NA, NA, qdss_tracedata_a, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(65,  pri_mi2s, NA, NA, NA, qdss_tracedata_a, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(66,  pri_mi2s, blsp_spi2_cs3, NA, NA, NA, qdss_tracedata_a,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(67,  pri_mi2s, blsp_spi10_cs1, NA, NA, NA, qdss_tracedata_a,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(68,  pri_mi2s, blsp_spi10_cs2, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(69,  spkr_i2s, audio_ref_clk, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(70,  slimbus, spkr_i2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(71,  slimbus, spkr_i2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(72,  spkr_i2s, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(73,  ter_mi2s, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(74,  ter_mi2s, gp_pdm1, NA, NA, NA, qdss_tracedata_a, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(75,  ter_mi2s, NA, NA, qdss_tracedata_a, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(76,  ter_mi2s, NA, NA, qdss_tracedata_a, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(77,  ter_mi2s, NA, NA, qdss_tracedata_a, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(78,  sec_mi2s, gcc_gp1_clk_b, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(79,  sec_mi2s, gp_pdm2, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(80,  sec_mi2s, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(81,  sec_mi2s, blsp_spi11, blsp_uart11, blsp_uim11,
+		 gcc_gp2_clk_b, NA, NA, NA, NA, NA, NA),
+	PINGROUP(82,  sec_mi2s, blsp_spi11, blsp_uart11, blsp_uim11,
+		 gcc_gp3_clk_b, NA, NA, NA, NA, NA, NA),
+	PINGROUP(83,  blsp_spi11, blsp_uart11, blsp_i2c11, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(84,  blsp_spi11, blsp_uart11, blsp_i2c11, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(85,  blsp_spi12, blsp_uart12, blsp_uim12, NA, NA,
+		 qdss_tracedata_a, NA, NA, NA, NA, NA),
+	PINGROUP(86,  blsp_spi12, blsp_uart12, blsp_uim12, gp_pdm1, NA,
+		 qdss_tracedata_a, NA, NA, NA, NA, NA),
+	PINGROUP(87,  blsp_spi12, blsp_uart12, blsp_i2c12, NA,
+		 qdss_tracedata_a, NA, NA, NA, NA, NA, NA),
+	PINGROUP(88,  blsp_spi12, blsp_uart12, blsp_i2c12, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(89,  tsif1, NA, qdss_tracedata_a, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(90,  tsif1, blsp_spi10_cs3, qdss_tracedata_a, NA, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(91,  tsif1, sdc4, NA, NA, NA, NA, qdss_traceclk_b, NA, NA, NA,
+		 NA),
+	PINGROUP(92,  tsif2, sdc4, NA, NA, qdss_tracedata_b, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(93,  tsif2, sdc4, NA, NA, NA, NA, qdss_tracedata_b, NA, NA,
+		 NA, NA),
+	PINGROUP(94,  tsif2, sdc4, NA, NA, NA, NA, qdss_tracectl_b, NA, NA, NA,
+		 NA),
+	PINGROUP(95,  tsif2, sdc4, gp_pdm0, NA, NA, NA, qdss_cti_trig_out_d,
+		 NA, NA, NA, NA),
+	PINGROUP(96,  tsif2, sdc4, qdss_cti_trig_in_d, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(97,  uim1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(98,  uim1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(99,  uim1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(100, uim1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(101, uim_batt_alarm, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(102, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(103, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(104, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(105, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(106, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(107, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(108, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(109, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(110, tsif1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(111, tsif1, blsp11_uart_tx_b, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(112, blsp11_uart_rx_b, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(113, blsp11_i2c_sda_b, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(114, blsp11_i2c_scl_b, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(115, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(116, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(117, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(118, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(119, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(120, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(121, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(122, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(123, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(124, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(125, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(126, NA, gsm_tx, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(127, NA, nav_tsync, nav_pps, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(128, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(129, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(130, gps_tx, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(131, gsm_tx, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(132, gsm_tx, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(133, gsm_tx, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(134, mss_lte, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(135, mss_lte, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(136, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(137, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(138, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(139, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(140, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(141, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(142, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(143, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(144, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(145, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	SDC_PINGROUP(sdc1_rclk, 0x2044, 15, 0),
+	SDC_PINGROUP(sdc1_clk, 0x2044, 13, 6),
+	SDC_PINGROUP(sdc1_cmd, 0x2044, 11, 3),
+	SDC_PINGROUP(sdc1_data, 0x2044, 9, 0),
+	SDC_PINGROUP(sdc2_clk, 0x2048, 14, 6),
+	SDC_PINGROUP(sdc2_cmd, 0x2048, 11, 3),
+	SDC_PINGROUP(sdc2_data, 0x2048, 9, 0),
+	SDC_PINGROUP(sdc3_clk, 0x206c, 14, 6),
+	SDC_PINGROUP(sdc3_cmd, 0x206c, 11, 3),
+	SDC_PINGROUP(sdc3_data, 0x206c, 9, 0),
+};
+
+#define NUM_GPIO_PINGROUPS 146
+
+static const struct msm_pinctrl_soc_data msm8994_pinctrl = {
+	.pins = msm8994_pins,
+	.npins = ARRAY_SIZE(msm8994_pins),
+	.functions = msm8994_functions,
+	.nfunctions = ARRAY_SIZE(msm8994_functions),
+	.groups = msm8994_groups,
+	.ngroups = ARRAY_SIZE(msm8994_groups),
+	.ngpios = NUM_GPIO_PINGROUPS,
+};
+
+static int msm8994_pinctrl_probe(struct platform_device *pdev)
+{
+	return msm_pinctrl_probe(pdev, &msm8994_pinctrl);
+}
+
+static const struct of_device_id msm8994_pinctrl_of_match[] = {
+	{ .compatible = "qcom,msm8992-pinctrl", },
+	{ .compatible = "qcom,msm8994-pinctrl", },
+	{ }
+};
+
+static struct platform_driver msm8994_pinctrl_driver = {
+	.driver = {
+		.name = "msm8994-pinctrl",
+		.of_match_table = msm8994_pinctrl_of_match,
+	},
+	.probe = msm8994_pinctrl_probe,
+	.remove = msm_pinctrl_remove,
+};
+
+static int __init msm8994_pinctrl_init(void)
+{
+	return platform_driver_register(&msm8994_pinctrl_driver);
+}
+arch_initcall(msm8994_pinctrl_init);
+
+static void __exit msm8994_pinctrl_exit(void)
+{
+	platform_driver_unregister(&msm8994_pinctrl_driver);
+}
+module_exit(msm8994_pinctrl_exit);
+
+MODULE_DESCRIPTION("Qualcomm MSM8994 pinctrl driver");
+MODULE_LICENSE("GPL v2");
+MODULE_DEVICE_TABLE(of, msm8994_pinctrl_of_match);
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos.c b/drivers/pinctrl/samsung/pinctrl-exynos.c
index d32fa2b5ff82..12f7d1eb65bc 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos.c
+++ b/drivers/pinctrl/samsung/pinctrl-exynos.c
@@ -61,16 +61,15 @@ static void exynos_irq_mask(struct irq_data *irqd)
 	struct irq_chip *chip = irq_data_get_irq_chip(irqd);
 	struct exynos_irq_chip *our_chip = to_exynos_irq_chip(chip);
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned long reg_mask = our_chip->eint_mask + bank->eint_offset;
 	unsigned long mask;
 	unsigned long flags;
 
 	spin_lock_irqsave(&bank->slock, flags);
 
-	mask = readl(d->virt_base + reg_mask);
+	mask = readl(bank->eint_base + reg_mask);
 	mask |= 1 << irqd->hwirq;
-	writel(mask, d->virt_base + reg_mask);
+	writel(mask, bank->eint_base + reg_mask);
 
 	spin_unlock_irqrestore(&bank->slock, flags);
 }
@@ -80,10 +79,9 @@ static void exynos_irq_ack(struct irq_data *irqd)
 	struct irq_chip *chip = irq_data_get_irq_chip(irqd);
 	struct exynos_irq_chip *our_chip = to_exynos_irq_chip(chip);
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned long reg_pend = our_chip->eint_pend + bank->eint_offset;
 
-	writel(1 << irqd->hwirq, d->virt_base + reg_pend);
+	writel(1 << irqd->hwirq, bank->eint_base + reg_pend);
 }
 
 static void exynos_irq_unmask(struct irq_data *irqd)
@@ -91,7 +89,6 @@ static void exynos_irq_unmask(struct irq_data *irqd)
 	struct irq_chip *chip = irq_data_get_irq_chip(irqd);
 	struct exynos_irq_chip *our_chip = to_exynos_irq_chip(chip);
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned long reg_mask = our_chip->eint_mask + bank->eint_offset;
 	unsigned long mask;
 	unsigned long flags;
@@ -109,9 +106,9 @@ static void exynos_irq_unmask(struct irq_data *irqd)
 
 	spin_lock_irqsave(&bank->slock, flags);
 
-	mask = readl(d->virt_base + reg_mask);
+	mask = readl(bank->eint_base + reg_mask);
 	mask &= ~(1 << irqd->hwirq);
-	writel(mask, d->virt_base + reg_mask);
+	writel(mask, bank->eint_base + reg_mask);
 
 	spin_unlock_irqrestore(&bank->slock, flags);
 }
@@ -121,7 +118,6 @@ static int exynos_irq_set_type(struct irq_data *irqd, unsigned int type)
 	struct irq_chip *chip = irq_data_get_irq_chip(irqd);
 	struct exynos_irq_chip *our_chip = to_exynos_irq_chip(chip);
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned int shift = EXYNOS_EINT_CON_LEN * irqd->hwirq;
 	unsigned int con, trig_type;
 	unsigned long reg_con = our_chip->eint_con + bank->eint_offset;
@@ -152,10 +148,10 @@ static int exynos_irq_set_type(struct irq_data *irqd, unsigned int type)
 	else
 		irq_set_handler_locked(irqd, handle_level_irq);
 
-	con = readl(d->virt_base + reg_con);
+	con = readl(bank->eint_base + reg_con);
 	con &= ~(EXYNOS_EINT_CON_MASK << shift);
 	con |= trig_type << shift;
-	writel(con, d->virt_base + reg_con);
+	writel(con, bank->eint_base + reg_con);
 
 	return 0;
 }
@@ -166,7 +162,6 @@ static int exynos_irq_request_resources(struct irq_data *irqd)
 	struct exynos_irq_chip *our_chip = to_exynos_irq_chip(chip);
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
 	const struct samsung_pin_bank_type *bank_type = bank->type;
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned int shift = EXYNOS_EINT_CON_LEN * irqd->hwirq;
 	unsigned long reg_con = our_chip->eint_con + bank->eint_offset;
 	unsigned long flags;
@@ -188,10 +183,10 @@ static int exynos_irq_request_resources(struct irq_data *irqd)
 
 	spin_lock_irqsave(&bank->slock, flags);
 
-	con = readl(d->virt_base + reg_con);
+	con = readl(bank->eint_base + reg_con);
 	con &= ~(mask << shift);
 	con |= EXYNOS_EINT_FUNC << shift;
-	writel(con, d->virt_base + reg_con);
+	writel(con, bank->eint_base + reg_con);
 
 	spin_unlock_irqrestore(&bank->slock, flags);
 
@@ -206,7 +201,6 @@ static void exynos_irq_release_resources(struct irq_data *irqd)
 	struct exynos_irq_chip *our_chip = to_exynos_irq_chip(chip);
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
 	const struct samsung_pin_bank_type *bank_type = bank->type;
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned int shift = EXYNOS_EINT_CON_LEN * irqd->hwirq;
 	unsigned long reg_con = our_chip->eint_con + bank->eint_offset;
 	unsigned long flags;
@@ -221,10 +215,10 @@ static void exynos_irq_release_resources(struct irq_data *irqd)
 
 	spin_lock_irqsave(&bank->slock, flags);
 
-	con = readl(d->virt_base + reg_con);
+	con = readl(bank->eint_base + reg_con);
 	con &= ~(mask << shift);
 	con |= FUNC_INPUT << shift;
-	writel(con, d->virt_base + reg_con);
+	writel(con, bank->eint_base + reg_con);
 
 	spin_unlock_irqrestore(&bank->slock, flags);
 
@@ -274,7 +268,7 @@ static irqreturn_t exynos_eint_gpio_irq(int irq, void *data)
 	struct samsung_pin_bank *bank = d->pin_banks;
 	unsigned int svc, group, pin, virq;
 
-	svc = readl(d->virt_base + EXYNOS_SVC_OFFSET);
+	svc = readl(bank->eint_base + EXYNOS_SVC_OFFSET);
 	group = EXYNOS_SVC_GROUP(svc);
 	pin = svc & EXYNOS_SVC_NUM_MASK;
 
@@ -452,7 +446,6 @@ static void exynos_irq_demux_eint16_31(struct irq_desc *desc)
 {
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	struct exynos_muxed_weint_data *eintd = irq_desc_get_handler_data(desc);
-	struct samsung_pinctrl_drv_data *d = eintd->banks[0]->drvdata;
 	unsigned long pend;
 	unsigned long mask;
 	int i;
@@ -461,9 +454,9 @@ static void exynos_irq_demux_eint16_31(struct irq_desc *desc)
 
 	for (i = 0; i < eintd->nr_banks; ++i) {
 		struct samsung_pin_bank *b = eintd->banks[i];
-		pend = readl(d->virt_base + b->irq_chip->eint_pend
+		pend = readl(b->eint_base + b->irq_chip->eint_pend
 				+ b->eint_offset);
-		mask = readl(d->virt_base + b->irq_chip->eint_mask
+		mask = readl(b->eint_base + b->irq_chip->eint_mask
 				+ b->eint_offset);
 		exynos_irq_demux_eint(pend & ~mask, b->irq_domain);
 	}
@@ -581,7 +574,7 @@ static void exynos_pinctrl_suspend_bank(
 				struct samsung_pin_bank *bank)
 {
 	struct exynos_eint_gpio_save *save = bank->soc_priv;
-	void __iomem *regs = drvdata->virt_base;
+	void __iomem *regs = bank->eint_base;
 
 	save->eint_con = readl(regs + EXYNOS_GPIO_ECON_OFFSET
 						+ bank->eint_offset);
@@ -610,7 +603,7 @@ static void exynos_pinctrl_resume_bank(
 				struct samsung_pin_bank *bank)
 {
 	struct exynos_eint_gpio_save *save = bank->soc_priv;
-	void __iomem *regs = drvdata->virt_base;
+	void __iomem *regs = bank->eint_base;
 
 	pr_debug("%s:     con %#010x => %#010x\n", bank->name,
 			readl(regs + EXYNOS_GPIO_ECON_OFFSET
@@ -1346,6 +1339,11 @@ static const struct samsung_pin_bank_data exynos5433_pin_banks0[] = {
 	EXYNOS_PIN_BANK_EINTW(8, 0x020, "gpa1", 0x04),
 	EXYNOS_PIN_BANK_EINTW(8, 0x040, "gpa2", 0x08),
 	EXYNOS_PIN_BANK_EINTW(8, 0x060, "gpa3", 0x0c),
+	EXYNOS_PIN_BANK_EINTW_EXT(8, 0x020, "gpf1", 0x1004, 1),
+	EXYNOS_PIN_BANK_EINTW_EXT(4, 0x040, "gpf2", 0x1008, 1),
+	EXYNOS_PIN_BANK_EINTW_EXT(4, 0x060, "gpf3", 0x100c, 1),
+	EXYNOS_PIN_BANK_EINTW_EXT(8, 0x080, "gpf4", 0x1010, 1),
+	EXYNOS_PIN_BANK_EINTW_EXT(8, 0x0a0, "gpf5", 0x1014, 1),
 };
 
 /* pin banks of exynos5433 pin-controller - AUD */
@@ -1427,6 +1425,7 @@ const struct samsung_pin_ctrl exynos5433_pin_ctrl[] = {
 		.eint_wkup_init = exynos_eint_wkup_init,
 		.suspend	= exynos_pinctrl_suspend,
 		.resume		= exynos_pinctrl_resume,
+		.nr_ext_resources = 1,
 	}, {
 		/* pin-controller instance 1 data */
 		.pin_banks	= exynos5433_pin_banks1,
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos.h b/drivers/pinctrl/samsung/pinctrl-exynos.h
index 0f0f7cedb2dc..5821525a2c84 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos.h
+++ b/drivers/pinctrl/samsung/pinctrl-exynos.h
@@ -79,6 +79,17 @@
 		.name		= id			\
 	}
 
+#define EXYNOS_PIN_BANK_EINTW_EXT(pins, reg, id, offs, pctl_idx) \
+	{						\
+		.type           = &bank_type_alive,	\
+		.pctl_offset    = reg,                  \
+		.nr_pins        = pins,                 \
+		.eint_type      = EINT_TYPE_WKUP,       \
+		.eint_offset    = offs,                 \
+		.name           = id,                   \
+		.pctl_res_idx   = pctl_idx,             \
+	}						\
+
 /**
  * struct exynos_weint_data: irq specific data for all the wakeup interrupts
  * generated by the external wakeup interrupt controller.
diff --git a/drivers/pinctrl/samsung/pinctrl-s3c24xx.c b/drivers/pinctrl/samsung/pinctrl-s3c24xx.c
index 3d92f827da7a..b82a003546ae 100644
--- a/drivers/pinctrl/samsung/pinctrl-s3c24xx.c
+++ b/drivers/pinctrl/samsung/pinctrl-s3c24xx.c
@@ -151,7 +151,7 @@ static void s3c24xx_eint_set_function(struct samsung_pinctrl_drv_data *d,
 	u32 val;
 
 	/* Make sure that pin is configured as interrupt */
-	reg = d->virt_base + bank->pctl_offset;
+	reg = bank->pctl_base + bank->pctl_offset;
 	shift = pin * bank_type->fld_width[PINCFG_TYPE_FUNC];
 	mask = (1 << bank_type->fld_width[PINCFG_TYPE_FUNC]) - 1;
 
@@ -184,7 +184,7 @@ static int s3c24xx_eint_type(struct irq_data *data, unsigned int type)
 	s3c24xx_eint_set_handler(data, type);
 
 	/* Set up interrupt trigger */
-	reg = d->virt_base + EINT_REG(index);
+	reg = bank->eint_base + EINT_REG(index);
 	shift = EINT_OFFS(index);
 
 	val = readl(reg);
@@ -259,32 +259,29 @@ static void s3c2410_demux_eint0_3(struct irq_desc *desc)
 static void s3c2412_eint0_3_ack(struct irq_data *data)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(data);
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 
 	unsigned long bitval = 1UL << data->hwirq;
-	writel(bitval, d->virt_base + EINTPEND_REG);
+	writel(bitval, bank->eint_base + EINTPEND_REG);
 }
 
 static void s3c2412_eint0_3_mask(struct irq_data *data)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(data);
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned long mask;
 
-	mask = readl(d->virt_base + EINTMASK_REG);
+	mask = readl(bank->eint_base + EINTMASK_REG);
 	mask |= (1UL << data->hwirq);
-	writel(mask, d->virt_base + EINTMASK_REG);
+	writel(mask, bank->eint_base + EINTMASK_REG);
 }
 
 static void s3c2412_eint0_3_unmask(struct irq_data *data)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(data);
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned long mask;
 
-	mask = readl(d->virt_base + EINTMASK_REG);
+	mask = readl(bank->eint_base + EINTMASK_REG);
 	mask &= ~(1UL << data->hwirq);
-	writel(mask, d->virt_base + EINTMASK_REG);
+	writel(mask, bank->eint_base + EINTMASK_REG);
 }
 
 static struct irq_chip s3c2412_eint0_3_chip = {
@@ -319,34 +316,31 @@ static void s3c2412_demux_eint0_3(struct irq_desc *desc)
 static void s3c24xx_eint_ack(struct irq_data *data)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(data);
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned char index = bank->eint_offset + data->hwirq;
 
-	writel(1UL << index, d->virt_base + EINTPEND_REG);
+	writel(1UL << index, bank->eint_base + EINTPEND_REG);
 }
 
 static void s3c24xx_eint_mask(struct irq_data *data)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(data);
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned char index = bank->eint_offset + data->hwirq;
 	unsigned long mask;
 
-	mask = readl(d->virt_base + EINTMASK_REG);
+	mask = readl(bank->eint_base + EINTMASK_REG);
 	mask |= (1UL << index);
-	writel(mask, d->virt_base + EINTMASK_REG);
+	writel(mask, bank->eint_base + EINTMASK_REG);
 }
 
 static void s3c24xx_eint_unmask(struct irq_data *data)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(data);
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned char index = bank->eint_offset + data->hwirq;
 	unsigned long mask;
 
-	mask = readl(d->virt_base + EINTMASK_REG);
+	mask = readl(bank->eint_base + EINTMASK_REG);
 	mask &= ~(1UL << index);
-	writel(mask, d->virt_base + EINTMASK_REG);
+	writel(mask, bank->eint_base + EINTMASK_REG);
 }
 
 static struct irq_chip s3c24xx_eint_chip = {
@@ -362,13 +356,14 @@ static inline void s3c24xx_demux_eint(struct irq_desc *desc,
 {
 	struct s3c24xx_eint_data *data = irq_desc_get_handler_data(desc);
 	struct irq_chip *chip = irq_desc_get_chip(desc);
-	struct samsung_pinctrl_drv_data *d = data->drvdata;
+	struct irq_data *irqd = irq_desc_get_irq_data(desc);
+	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
 	unsigned int pend, mask;
 
 	chained_irq_enter(chip, desc);
 
-	pend = readl(d->virt_base + EINTPEND_REG);
-	mask = readl(d->virt_base + EINTMASK_REG);
+	pend = readl(bank->eint_base + EINTPEND_REG);
+	mask = readl(bank->eint_base + EINTMASK_REG);
 
 	pend &= ~mask;
 	pend &= range;
diff --git a/drivers/pinctrl/samsung/pinctrl-s3c64xx.c b/drivers/pinctrl/samsung/pinctrl-s3c64xx.c
index 43407ab248f5..4c632812ccff 100644
--- a/drivers/pinctrl/samsung/pinctrl-s3c64xx.c
+++ b/drivers/pinctrl/samsung/pinctrl-s3c64xx.c
@@ -280,7 +280,7 @@ static void s3c64xx_irq_set_function(struct samsung_pinctrl_drv_data *d,
 	u32 val;
 
 	/* Make sure that pin is configured as interrupt */
-	reg = d->virt_base + bank->pctl_offset;
+	reg = bank->pctl_base + bank->pctl_offset;
 	shift = pin;
 	if (bank_type->fld_width[PINCFG_TYPE_FUNC] * shift >= 32) {
 		/* 4-bit bank type with 2 con regs */
@@ -308,9 +308,8 @@ static void s3c64xx_irq_set_function(struct samsung_pinctrl_drv_data *d,
 static inline void s3c64xx_gpio_irq_set_mask(struct irq_data *irqd, bool mask)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned char index = EINT_OFFS(bank->eint_offset) + irqd->hwirq;
-	void __iomem *reg = d->virt_base + EINTMASK_REG(bank->eint_offset);
+	void __iomem *reg = bank->eint_base + EINTMASK_REG(bank->eint_offset);
 	u32 val;
 
 	val = readl(reg);
@@ -334,9 +333,8 @@ static void s3c64xx_gpio_irq_mask(struct irq_data *irqd)
 static void s3c64xx_gpio_irq_ack(struct irq_data *irqd)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned char index = EINT_OFFS(bank->eint_offset) + irqd->hwirq;
-	void __iomem *reg = d->virt_base + EINTPEND_REG(bank->eint_offset);
+	void __iomem *reg = bank->eint_base + EINTPEND_REG(bank->eint_offset);
 
 	writel(1 << index, reg);
 }
@@ -359,7 +357,7 @@ static int s3c64xx_gpio_irq_set_type(struct irq_data *irqd, unsigned int type)
 	s3c64xx_irq_set_handler(irqd, type);
 
 	/* Set up interrupt trigger */
-	reg = d->virt_base + EINTCON_REG(bank->eint_offset);
+	reg = bank->eint_base + EINTCON_REG(bank->eint_offset);
 	shift = EINT_OFFS(bank->eint_offset) + irqd->hwirq;
 	shift = 4 * (shift / 4); /* 4 EINTs per trigger selector */
 
@@ -411,7 +409,8 @@ static void s3c64xx_eint_gpio_irq(struct irq_desc *desc)
 {
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	struct s3c64xx_eint_gpio_data *data = irq_desc_get_handler_data(desc);
-	struct samsung_pinctrl_drv_data *drvdata = data->drvdata;
+	struct irq_data *irqd = irq_desc_get_irq_data(desc);
+	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
 
 	chained_irq_enter(chip, desc);
 
@@ -421,7 +420,7 @@ static void s3c64xx_eint_gpio_irq(struct irq_desc *desc)
 		unsigned int pin;
 		unsigned int virq;
 
-		svc = readl(drvdata->virt_base + SERVICE_REG);
+		svc = readl(bank->eint_base + SERVICE_REG);
 		group = SVC_GROUP(svc);
 		pin = svc & SVC_NUM_MASK;
 
@@ -518,15 +517,15 @@ static inline void s3c64xx_eint0_irq_set_mask(struct irq_data *irqd, bool mask)
 {
 	struct s3c64xx_eint0_domain_data *ddata =
 					irq_data_get_irq_chip_data(irqd);
-	struct samsung_pinctrl_drv_data *d = ddata->bank->drvdata;
+	struct samsung_pin_bank *bank = ddata->bank;
 	u32 val;
 
-	val = readl(d->virt_base + EINT0MASK_REG);
+	val = readl(bank->eint_base + EINT0MASK_REG);
 	if (mask)
 		val |= 1 << ddata->eints[irqd->hwirq];
 	else
 		val &= ~(1 << ddata->eints[irqd->hwirq]);
-	writel(val, d->virt_base + EINT0MASK_REG);
+	writel(val, bank->eint_base + EINT0MASK_REG);
 }
 
 static void s3c64xx_eint0_irq_unmask(struct irq_data *irqd)
@@ -543,10 +542,10 @@ static void s3c64xx_eint0_irq_ack(struct irq_data *irqd)
 {
 	struct s3c64xx_eint0_domain_data *ddata =
 					irq_data_get_irq_chip_data(irqd);
-	struct samsung_pinctrl_drv_data *d = ddata->bank->drvdata;
+	struct samsung_pin_bank *bank = ddata->bank;
 
 	writel(1 << ddata->eints[irqd->hwirq],
-					d->virt_base + EINT0PEND_REG);
+					bank->eint_base + EINT0PEND_REG);
 }
 
 static int s3c64xx_eint0_irq_set_type(struct irq_data *irqd, unsigned int type)
@@ -554,7 +553,7 @@ static int s3c64xx_eint0_irq_set_type(struct irq_data *irqd, unsigned int type)
 	struct s3c64xx_eint0_domain_data *ddata =
 					irq_data_get_irq_chip_data(irqd);
 	struct samsung_pin_bank *bank = ddata->bank;
-	struct samsung_pinctrl_drv_data *d = bank->drvdata;
+	struct samsung_pinctrl_drv_data *d = ddata->bank->drvdata;
 	void __iomem *reg;
 	int trigger;
 	u8 shift;
@@ -569,7 +568,7 @@ static int s3c64xx_eint0_irq_set_type(struct irq_data *irqd, unsigned int type)
 	s3c64xx_irq_set_handler(irqd, type);
 
 	/* Set up interrupt trigger */
-	reg = d->virt_base + EINT0CON0_REG;
+	reg = bank->eint_base + EINT0CON0_REG;
 	shift = ddata->eints[irqd->hwirq];
 	if (shift >= EINT_MAX_PER_REG) {
 		reg += 4;
@@ -601,14 +600,19 @@ static struct irq_chip s3c64xx_eint0_irq_chip = {
 static inline void s3c64xx_irq_demux_eint(struct irq_desc *desc, u32 range)
 {
 	struct irq_chip *chip = irq_desc_get_chip(desc);
+	struct irq_data *irqd = irq_desc_get_irq_data(desc);
+	struct s3c64xx_eint0_domain_data *ddata =
+					irq_data_get_irq_chip_data(irqd);
+	struct samsung_pin_bank *bank = ddata->bank;
+
 	struct s3c64xx_eint0_data *data = irq_desc_get_handler_data(desc);
-	struct samsung_pinctrl_drv_data *drvdata = data->drvdata;
+
 	unsigned int pend, mask;
 
 	chained_irq_enter(chip, desc);
 
-	pend = readl(drvdata->virt_base + EINT0PEND_REG);
-	mask = readl(drvdata->virt_base + EINT0MASK_REG);
+	pend = readl(bank->eint_base + EINT0PEND_REG);
+	mask = readl(bank->eint_base + EINT0MASK_REG);
 
 	pend = pend & range & ~mask;
 	pend &= range;
diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c
index 620727fabe64..41e62391c33c 100644
--- a/drivers/pinctrl/samsung/pinctrl-samsung.c
+++ b/drivers/pinctrl/samsung/pinctrl-samsung.c
@@ -33,6 +33,9 @@
 #include "../core.h"
 #include "pinctrl-samsung.h"
 
+/* maximum number of the memory resources */
+#define	SAMSUNG_PINCTRL_NUM_RESOURCES	2
+
 /* list of all possible config options supported */
 static struct pin_config {
 	const char *property;
@@ -345,7 +348,7 @@ static void pin_to_reg_bank(struct samsung_pinctrl_drv_data *drvdata,
 			((b->pin_base + b->nr_pins - 1) < pin))
 		b++;
 
-	*reg = drvdata->virt_base + b->pctl_offset;
+	*reg = b->pctl_base + b->pctl_offset;
 	*offset = pin - b->pin_base;
 	if (bank)
 		*bank = b;
@@ -526,7 +529,7 @@ static void samsung_gpio_set_value(struct gpio_chip *gc,
 	void __iomem *reg;
 	u32 data;
 
-	reg = bank->drvdata->virt_base + bank->pctl_offset;
+	reg = bank->pctl_base + bank->pctl_offset;
 
 	data = readl(reg + type->reg_offset[PINCFG_TYPE_DAT]);
 	data &= ~(1 << offset);
@@ -554,7 +557,7 @@ static int samsung_gpio_get(struct gpio_chip *gc, unsigned offset)
 	struct samsung_pin_bank *bank = gpiochip_get_data(gc);
 	const struct samsung_pin_bank_type *type = bank->type;
 
-	reg = bank->drvdata->virt_base + bank->pctl_offset;
+	reg = bank->pctl_base + bank->pctl_offset;
 
 	data = readl(reg + type->reg_offset[PINCFG_TYPE_DAT]);
 	data >>= offset;
@@ -581,8 +584,8 @@ static int samsung_gpio_set_direction(struct gpio_chip *gc,
 	type = bank->type;
 	drvdata = bank->drvdata;
 
-	reg = drvdata->virt_base + bank->pctl_offset +
-					type->reg_offset[PINCFG_TYPE_FUNC];
+	reg = bank->pctl_base + bank->pctl_offset
+			+ type->reg_offset[PINCFG_TYPE_FUNC];
 
 	mask = (1 << type->fld_width[PINCFG_TYPE_FUNC]) - 1;
 	shift = offset * type->fld_width[PINCFG_TYPE_FUNC];
@@ -979,6 +982,8 @@ samsung_pinctrl_get_soc_data(struct samsung_pinctrl_drv_data *d,
 	const struct samsung_pin_bank_data *bdata;
 	const struct samsung_pin_ctrl *ctrl;
 	struct samsung_pin_bank *bank;
+	struct resource *res;
+	void __iomem *virt_base[SAMSUNG_PINCTRL_NUM_RESOURCES];
 	int i;
 
 	id = of_alias_get_id(node, "pinctrl");
@@ -997,6 +1002,17 @@ samsung_pinctrl_get_soc_data(struct samsung_pinctrl_drv_data *d,
 	if (!d->pin_banks)
 		return ERR_PTR(-ENOMEM);
 
+	if (ctrl->nr_ext_resources + 1 > SAMSUNG_PINCTRL_NUM_RESOURCES)
+		return ERR_PTR(-EINVAL);
+
+	for (i = 0; i < ctrl->nr_ext_resources + 1; i++) {
+		res = platform_get_resource(pdev, IORESOURCE_MEM, i);
+		virt_base[i] = devm_ioremap(&pdev->dev, res->start,
+						resource_size(res));
+		if (IS_ERR(virt_base[i]))
+			return ERR_PTR(-EIO);
+	}
+
 	bank = d->pin_banks;
 	bdata = ctrl->pin_banks;
 	for (i = 0; i < ctrl->nr_banks; ++i, ++bdata, ++bank) {
@@ -1013,6 +1029,9 @@ samsung_pinctrl_get_soc_data(struct samsung_pinctrl_drv_data *d,
 		bank->drvdata = d;
 		bank->pin_base = d->nr_pins;
 		d->nr_pins += bank->nr_pins;
+
+		bank->eint_base = virt_base[0];
+		bank->pctl_base = virt_base[bdata->pctl_res_idx];
 	}
 
 	for_each_child_of_node(node, np) {
@@ -1052,11 +1071,6 @@ static int samsung_pinctrl_probe(struct platform_device *pdev)
 	}
 	drvdata->dev = dev;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	drvdata->virt_base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(drvdata->virt_base))
-		return PTR_ERR(drvdata->virt_base);
-
 	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
 	if (res)
 		drvdata->irq = res->start;
@@ -1094,12 +1108,11 @@ static int samsung_pinctrl_probe(struct platform_device *pdev)
 static void samsung_pinctrl_suspend_dev(
 	struct samsung_pinctrl_drv_data *drvdata)
 {
-	void __iomem *virt_base = drvdata->virt_base;
 	int i;
 
 	for (i = 0; i < drvdata->nr_banks; i++) {
 		struct samsung_pin_bank *bank = &drvdata->pin_banks[i];
-		void __iomem *reg = virt_base + bank->pctl_offset;
+		void __iomem *reg = bank->pctl_base + bank->pctl_offset;
 		const u8 *offs = bank->type->reg_offset;
 		const u8 *widths = bank->type->fld_width;
 		enum pincfg_type type;
@@ -1140,7 +1153,6 @@ static void samsung_pinctrl_suspend_dev(
  */
 static void samsung_pinctrl_resume_dev(struct samsung_pinctrl_drv_data *drvdata)
 {
-	void __iomem *virt_base = drvdata->virt_base;
 	int i;
 
 	if (drvdata->resume)
@@ -1148,7 +1160,7 @@ static void samsung_pinctrl_resume_dev(struct samsung_pinctrl_drv_data *drvdata)
 
 	for (i = 0; i < drvdata->nr_banks; i++) {
 		struct samsung_pin_bank *bank = &drvdata->pin_banks[i];
-		void __iomem *reg = virt_base + bank->pctl_offset;
+		void __iomem *reg = bank->pctl_base + bank->pctl_offset;
 		const u8 *offs = bank->type->reg_offset;
 		const u8 *widths = bank->type->fld_width;
 		enum pincfg_type type;
diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.h b/drivers/pinctrl/samsung/pinctrl-samsung.h
index cd31bfaf62cb..043cb6c11180 100644
--- a/drivers/pinctrl/samsung/pinctrl-samsung.h
+++ b/drivers/pinctrl/samsung/pinctrl-samsung.h
@@ -116,6 +116,7 @@ struct samsung_pin_bank_type {
  * struct samsung_pin_bank_data: represent a controller pin-bank (init data).
  * @type: type of the bank (register offsets and bitfield widths)
  * @pctl_offset: starting offset of the pin-bank registers.
+ * @pctl_res_idx: index of base address for pin-bank registers.
  * @nr_pins: number of pins included in this bank.
  * @eint_func: function to set in CON register to configure pin as EINT.
  * @eint_type: type of the external interrupt supported by the bank.
@@ -126,6 +127,7 @@ struct samsung_pin_bank_type {
 struct samsung_pin_bank_data {
 	const struct samsung_pin_bank_type *type;
 	u32		pctl_offset;
+	u8		pctl_res_idx;
 	u8		nr_pins;
 	u8		eint_func;
 	enum eint_type	eint_type;
@@ -137,8 +139,10 @@ struct samsung_pin_bank_data {
 /**
  * struct samsung_pin_bank: represent a controller pin-bank.
  * @type: type of the bank (register offsets and bitfield widths)
+ * @pctl_base: base address of the pin-bank registers
  * @pctl_offset: starting offset of the pin-bank registers.
  * @nr_pins: number of pins included in this bank.
+ * @eint_base: base address of the pin-bank EINT registers.
  * @eint_func: function to set in CON register to configure pin as EINT.
  * @eint_type: type of the external interrupt supported by the bank.
  * @eint_mask: bit mask of pins which support EINT function.
@@ -157,8 +161,10 @@ struct samsung_pin_bank_data {
  */
 struct samsung_pin_bank {
 	const struct samsung_pin_bank_type *type;
+	void __iomem	*pctl_base;
 	u32		pctl_offset;
 	u8		nr_pins;
+	void __iomem	*eint_base;
 	u8		eint_func;
 	enum eint_type	eint_type;
 	u32		eint_mask;
@@ -182,6 +188,7 @@ struct samsung_pin_bank {
  * struct samsung_pin_ctrl: represent a pin controller.
  * @pin_banks: list of pin banks included in this controller.
  * @nr_banks: number of pin banks.
+ * @nr_ext_resources: number of the extra base address for pin banks.
  * @eint_gpio_init: platform specific callback to setup the external gpio
  *	interrupts for the controller.
  * @eint_wkup_init: platform specific callback to setup the external wakeup
@@ -190,6 +197,7 @@ struct samsung_pin_bank {
 struct samsung_pin_ctrl {
 	const struct samsung_pin_bank_data *pin_banks;
 	u32		nr_banks;
+	int		nr_ext_resources;
 
 	int		(*eint_gpio_init)(struct samsung_pinctrl_drv_data *);
 	int		(*eint_wkup_init)(struct samsung_pinctrl_drv_data *);
@@ -200,7 +208,6 @@ struct samsung_pin_ctrl {
 /**
  * struct samsung_pinctrl_drv_data: wrapper for holding driver data together.
  * @node: global list node
- * @virt_base: register base address of the controller.
  * @dev: device instance representing the controller.
  * @irq: interrpt number used by the controller to notify gpio interrupts.
  * @ctrl: pin controller instance managed by the driver.
@@ -215,7 +222,6 @@ struct samsung_pin_ctrl {
  */
 struct samsung_pinctrl_drv_data {
 	struct list_head		node;
-	void __iomem			*virt_base;
 	struct device			*dev;
 	int				irq;
 
diff --git a/drivers/pinctrl/sh-pfc/core.c b/drivers/pinctrl/sh-pfc/core.c
index f3a8897d4e8f..cf80ce1dd7ce 100644
--- a/drivers/pinctrl/sh-pfc/core.c
+++ b/drivers/pinctrl/sh-pfc/core.c
@@ -389,6 +389,21 @@ int sh_pfc_config_mux(struct sh_pfc *pfc, unsigned mark, int pinmux_type)
 	return 0;
 }
 
+const struct sh_pfc_bias_info *
+sh_pfc_pin_to_bias_info(const struct sh_pfc_bias_info *info,
+			unsigned int num, unsigned int pin)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++)
+		if (info[i].pin == pin)
+			return &info[i];
+
+	WARN_ONCE(1, "Pin %u is not in bias info list\n", pin);
+
+	return NULL;
+}
+
 static int sh_pfc_init_ranges(struct sh_pfc *pfc)
 {
 	struct sh_pfc_pin_range *range;
diff --git a/drivers/pinctrl/sh-pfc/core.h b/drivers/pinctrl/sh-pfc/core.h
index 0bbdea5849f4..6d598dd63720 100644
--- a/drivers/pinctrl/sh-pfc/core.h
+++ b/drivers/pinctrl/sh-pfc/core.h
@@ -33,4 +33,8 @@ void sh_pfc_write_reg(struct sh_pfc *pfc, u32 reg, unsigned int width,
 int sh_pfc_get_pin_index(struct sh_pfc *pfc, unsigned int pin);
 int sh_pfc_config_mux(struct sh_pfc *pfc, unsigned mark, int pinmux_type);
 
+const struct sh_pfc_bias_info *
+sh_pfc_pin_to_bias_info(const struct sh_pfc_bias_info *info,
+			unsigned int num, unsigned int pin);
+
 #endif /* __SH_PFC_CORE_H__ */
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7778.c b/drivers/pinctrl/sh-pfc/pfc-r8a7778.c
index 18ef7042b3d1..c3af9ebee4af 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7778.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7778.c
@@ -24,6 +24,7 @@
 #include <linux/kernel.h>
 #include <linux/pinctrl/pinconf-generic.h>
 
+#include "core.h"
 #include "sh_pfc.h"
 
 #define PORT_GP_PUP_1(bank, pin, fn, sfx)	\
@@ -2918,183 +2919,182 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define PUPR4	0x110
 #define PUPR5	0x114
 
-static const struct {
-	u16 reg : 11;
-	u16 bit : 5;
-} pullups[] = {
-	[RCAR_GP_PIN(0,  6)] = { PUPR0,  0 },	/* A0 */
-	[RCAR_GP_PIN(0,  7)] = { PUPR0,  1 },	/* A1 */
-	[RCAR_GP_PIN(0,  8)] = { PUPR0,  2 },	/* A2 */
-	[RCAR_GP_PIN(0,  9)] = { PUPR0,  3 },	/* A3 */
-	[RCAR_GP_PIN(0, 10)] = { PUPR0,  4 },	/* A4 */
-	[RCAR_GP_PIN(0, 11)] = { PUPR0,  5 },	/* A5 */
-	[RCAR_GP_PIN(0, 12)] = { PUPR0,  6 },	/* A6 */
-	[RCAR_GP_PIN(0, 13)] = { PUPR0,  7 },	/* A7 */
-	[RCAR_GP_PIN(0, 14)] = { PUPR0,  8 },	/* A8 */
-	[RCAR_GP_PIN(0, 15)] = { PUPR0,  9 },	/* A9 */
-	[RCAR_GP_PIN(0, 16)] = { PUPR0, 10 },	/* A10 */
-	[RCAR_GP_PIN(0, 17)] = { PUPR0, 11 },	/* A11 */
-	[RCAR_GP_PIN(0, 18)] = { PUPR0, 12 },	/* A12 */
-	[RCAR_GP_PIN(0, 19)] = { PUPR0, 13 },	/* A13 */
-	[RCAR_GP_PIN(0, 20)] = { PUPR0, 14 },	/* A14 */
-	[RCAR_GP_PIN(0, 21)] = { PUPR0, 15 },	/* A15 */
-	[RCAR_GP_PIN(0, 22)] = { PUPR0, 16 },	/* A16 */
-	[RCAR_GP_PIN(0, 23)] = { PUPR0, 17 },	/* A17 */
-	[RCAR_GP_PIN(0, 24)] = { PUPR0, 18 },	/* A18 */
-	[RCAR_GP_PIN(0, 25)] = { PUPR0, 19 },	/* A19 */
-	[RCAR_GP_PIN(0, 26)] = { PUPR0, 20 },	/* A20 */
-	[RCAR_GP_PIN(0, 27)] = { PUPR0, 21 },	/* A21 */
-	[RCAR_GP_PIN(0, 28)] = { PUPR0, 22 },	/* A22 */
-	[RCAR_GP_PIN(0, 29)] = { PUPR0, 23 },	/* A23 */
-	[RCAR_GP_PIN(0, 30)] = { PUPR0, 24 },	/* A24 */
-	[RCAR_GP_PIN(0, 31)] = { PUPR0, 25 },	/* A25 */
-	[RCAR_GP_PIN(1,  3)] = { PUPR0, 26 },	/* /EX_CS0 */
-	[RCAR_GP_PIN(1,  4)] = { PUPR0, 27 },	/* /EX_CS1 */
-	[RCAR_GP_PIN(1,  5)] = { PUPR0, 28 },	/* /EX_CS2 */
-	[RCAR_GP_PIN(1,  6)] = { PUPR0, 29 },	/* /EX_CS3 */
-	[RCAR_GP_PIN(1,  7)] = { PUPR0, 30 },	/* /EX_CS4 */
-	[RCAR_GP_PIN(1,  8)] = { PUPR0, 31 },	/* /EX_CS5 */
-
-	[RCAR_GP_PIN(0,  0)] = { PUPR1,  0 },	/* /PRESETOUT	*/
-	[RCAR_GP_PIN(0,  5)] = { PUPR1,  1 },	/* /BS		*/
-	[RCAR_GP_PIN(1,  0)] = { PUPR1,  2 },	/* RD//WR	*/
-	[RCAR_GP_PIN(1,  1)] = { PUPR1,  3 },	/* /WE0		*/
-	[RCAR_GP_PIN(1,  2)] = { PUPR1,  4 },	/* /WE1		*/
-	[RCAR_GP_PIN(1, 11)] = { PUPR1,  5 },	/* EX_WAIT0	*/
-	[RCAR_GP_PIN(1,  9)] = { PUPR1,  6 },	/* DREQ0	*/
-	[RCAR_GP_PIN(1, 10)] = { PUPR1,  7 },	/* DACK0	*/
-	[RCAR_GP_PIN(1, 12)] = { PUPR1,  8 },	/* IRQ0		*/
-	[RCAR_GP_PIN(1, 13)] = { PUPR1,  9 },	/* IRQ1		*/
-
-	[RCAR_GP_PIN(1, 22)] = { PUPR2,  0 },	/* DU0_DR0	*/
-	[RCAR_GP_PIN(1, 23)] = { PUPR2,  1 },	/* DU0_DR1	*/
-	[RCAR_GP_PIN(1, 24)] = { PUPR2,  2 },	/* DU0_DR2	*/
-	[RCAR_GP_PIN(1, 25)] = { PUPR2,  3 },	/* DU0_DR3	*/
-	[RCAR_GP_PIN(1, 26)] = { PUPR2,  4 },	/* DU0_DR4	*/
-	[RCAR_GP_PIN(1, 27)] = { PUPR2,  5 },	/* DU0_DR5	*/
-	[RCAR_GP_PIN(1, 28)] = { PUPR2,  6 },	/* DU0_DR6	*/
-	[RCAR_GP_PIN(1, 29)] = { PUPR2,  7 },	/* DU0_DR7	*/
-	[RCAR_GP_PIN(1, 30)] = { PUPR2,  8 },	/* DU0_DG0	*/
-	[RCAR_GP_PIN(1, 31)] = { PUPR2,  9 },	/* DU0_DG1	*/
-	[RCAR_GP_PIN(2,  0)] = { PUPR2, 10 },	/* DU0_DG2	*/
-	[RCAR_GP_PIN(2,  1)] = { PUPR2, 11 },	/* DU0_DG3	*/
-	[RCAR_GP_PIN(2,  2)] = { PUPR2, 12 },	/* DU0_DG4	*/
-	[RCAR_GP_PIN(2,  3)] = { PUPR2, 13 },	/* DU0_DG5	*/
-	[RCAR_GP_PIN(2,  4)] = { PUPR2, 14 },	/* DU0_DG6	*/
-	[RCAR_GP_PIN(2,  5)] = { PUPR2, 15 },	/* DU0_DG7	*/
-	[RCAR_GP_PIN(2,  6)] = { PUPR2, 16 },	/* DU0_DB0	*/
-	[RCAR_GP_PIN(2,  7)] = { PUPR2, 17 },	/* DU0_DB1	*/
-	[RCAR_GP_PIN(2,  8)] = { PUPR2, 18 },	/* DU0_DB2	*/
-	[RCAR_GP_PIN(2,  9)] = { PUPR2, 19 },	/* DU0_DB3	*/
-	[RCAR_GP_PIN(2, 10)] = { PUPR2, 20 },	/* DU0_DB4	*/
-	[RCAR_GP_PIN(2, 11)] = { PUPR2, 21 },	/* DU0_DB5	*/
-	[RCAR_GP_PIN(2, 12)] = { PUPR2, 22 },	/* DU0_DB6	*/
-	[RCAR_GP_PIN(2, 13)] = { PUPR2, 23 },	/* DU0_DB7	*/
-	[RCAR_GP_PIN(2, 14)] = { PUPR2, 24 },	/* DU0_DOTCLKIN	*/
-	[RCAR_GP_PIN(2, 15)] = { PUPR2, 25 },	/* DU0_DOTCLKOUT0 */
-	[RCAR_GP_PIN(2, 17)] = { PUPR2, 26 },	/* DU0_HSYNC	*/
-	[RCAR_GP_PIN(2, 18)] = { PUPR2, 27 },	/* DU0_VSYNC	*/
-	[RCAR_GP_PIN(2, 19)] = { PUPR2, 28 },	/* DU0_EXODDF	*/
-	[RCAR_GP_PIN(2, 20)] = { PUPR2, 29 },	/* DU0_DISP	*/
-	[RCAR_GP_PIN(2, 21)] = { PUPR2, 30 },	/* DU0_CDE	*/
-	[RCAR_GP_PIN(2, 16)] = { PUPR2, 31 },	/* DU0_DOTCLKOUT1 */
-
-	[RCAR_GP_PIN(3, 24)] = { PUPR3,  0 },	/* VI0_CLK	*/
-	[RCAR_GP_PIN(3, 25)] = { PUPR3,  1 },	/* VI0_CLKENB	*/
-	[RCAR_GP_PIN(3, 26)] = { PUPR3,  2 },	/* VI0_FIELD	*/
-	[RCAR_GP_PIN(3, 27)] = { PUPR3,  3 },	/* /VI0_HSYNC	*/
-	[RCAR_GP_PIN(3, 28)] = { PUPR3,  4 },	/* /VI0_VSYNC	*/
-	[RCAR_GP_PIN(3, 29)] = { PUPR3,  5 },	/* VI0_DATA0	*/
-	[RCAR_GP_PIN(3, 30)] = { PUPR3,  6 },	/* VI0_DATA1	*/
-	[RCAR_GP_PIN(3, 31)] = { PUPR3,  7 },	/* VI0_DATA2	*/
-	[RCAR_GP_PIN(4,  0)] = { PUPR3,  8 },	/* VI0_DATA3	*/
-	[RCAR_GP_PIN(4,  1)] = { PUPR3,  9 },	/* VI0_DATA4	*/
-	[RCAR_GP_PIN(4,  2)] = { PUPR3, 10 },	/* VI0_DATA5	*/
-	[RCAR_GP_PIN(4,  3)] = { PUPR3, 11 },	/* VI0_DATA6	*/
-	[RCAR_GP_PIN(4,  4)] = { PUPR3, 12 },	/* VI0_DATA7	*/
-	[RCAR_GP_PIN(4,  5)] = { PUPR3, 13 },	/* VI0_G2	*/
-	[RCAR_GP_PIN(4,  6)] = { PUPR3, 14 },	/* VI0_G3	*/
-	[RCAR_GP_PIN(4,  7)] = { PUPR3, 15 },	/* VI0_G4	*/
-	[RCAR_GP_PIN(4,  8)] = { PUPR3, 16 },	/* VI0_G5	*/
-	[RCAR_GP_PIN(4, 21)] = { PUPR3, 17 },	/* VI1_DATA12	*/
-	[RCAR_GP_PIN(4, 22)] = { PUPR3, 18 },	/* VI1_DATA13	*/
-	[RCAR_GP_PIN(4, 23)] = { PUPR3, 19 },	/* VI1_DATA14	*/
-	[RCAR_GP_PIN(4, 24)] = { PUPR3, 20 },	/* VI1_DATA15	*/
-	[RCAR_GP_PIN(4,  9)] = { PUPR3, 21 },	/* ETH_REF_CLK	*/
-	[RCAR_GP_PIN(4, 10)] = { PUPR3, 22 },	/* ETH_TXD0	*/
-	[RCAR_GP_PIN(4, 11)] = { PUPR3, 23 },	/* ETH_TXD1	*/
-	[RCAR_GP_PIN(4, 12)] = { PUPR3, 24 },	/* ETH_CRS_DV	*/
-	[RCAR_GP_PIN(4, 13)] = { PUPR3, 25 },	/* ETH_TX_EN	*/
-	[RCAR_GP_PIN(4, 14)] = { PUPR3, 26 },	/* ETH_RX_ER	*/
-	[RCAR_GP_PIN(4, 15)] = { PUPR3, 27 },	/* ETH_RXD0	*/
-	[RCAR_GP_PIN(4, 16)] = { PUPR3, 28 },	/* ETH_RXD1	*/
-	[RCAR_GP_PIN(4, 17)] = { PUPR3, 29 },	/* ETH_MDC	*/
-	[RCAR_GP_PIN(4, 18)] = { PUPR3, 30 },	/* ETH_MDIO	*/
-	[RCAR_GP_PIN(4, 19)] = { PUPR3, 31 },	/* ETH_LINK	*/
-
-	[RCAR_GP_PIN(3,  6)] = { PUPR4,  0 },	/* SSI_SCK012	*/
-	[RCAR_GP_PIN(3,  7)] = { PUPR4,  1 },	/* SSI_WS012	*/
-	[RCAR_GP_PIN(3, 10)] = { PUPR4,  2 },	/* SSI_SDATA0	*/
-	[RCAR_GP_PIN(3,  9)] = { PUPR4,  3 },	/* SSI_SDATA1	*/
-	[RCAR_GP_PIN(3,  8)] = { PUPR4,  4 },	/* SSI_SDATA2	*/
-	[RCAR_GP_PIN(3,  2)] = { PUPR4,  5 },	/* SSI_SCK34	*/
-	[RCAR_GP_PIN(3,  3)] = { PUPR4,  6 },	/* SSI_WS34	*/
-	[RCAR_GP_PIN(3,  5)] = { PUPR4,  7 },	/* SSI_SDATA3	*/
-	[RCAR_GP_PIN(3,  4)] = { PUPR4,  8 },	/* SSI_SDATA4	*/
-	[RCAR_GP_PIN(2, 31)] = { PUPR4,  9 },	/* SSI_SCK5	*/
-	[RCAR_GP_PIN(3,  0)] = { PUPR4, 10 },	/* SSI_WS5	*/
-	[RCAR_GP_PIN(3,  1)] = { PUPR4, 11 },	/* SSI_SDATA5	*/
-	[RCAR_GP_PIN(2, 28)] = { PUPR4, 12 },	/* SSI_SCK6	*/
-	[RCAR_GP_PIN(2, 29)] = { PUPR4, 13 },	/* SSI_WS6	*/
-	[RCAR_GP_PIN(2, 30)] = { PUPR4, 14 },	/* SSI_SDATA6	*/
-	[RCAR_GP_PIN(2, 24)] = { PUPR4, 15 },	/* SSI_SCK78	*/
-	[RCAR_GP_PIN(2, 25)] = { PUPR4, 16 },	/* SSI_WS78	*/
-	[RCAR_GP_PIN(2, 27)] = { PUPR4, 17 },	/* SSI_SDATA7	*/
-	[RCAR_GP_PIN(2, 26)] = { PUPR4, 18 },	/* SSI_SDATA8	*/
-	[RCAR_GP_PIN(3, 23)] = { PUPR4, 19 },	/* TCLK0	*/
-	[RCAR_GP_PIN(3, 11)] = { PUPR4, 20 },	/* SD0_CLK	*/
-	[RCAR_GP_PIN(3, 12)] = { PUPR4, 21 },	/* SD0_CMD	*/
-	[RCAR_GP_PIN(3, 13)] = { PUPR4, 22 },	/* SD0_DAT0	*/
-	[RCAR_GP_PIN(3, 14)] = { PUPR4, 23 },	/* SD0_DAT1	*/
-	[RCAR_GP_PIN(3, 15)] = { PUPR4, 24 },	/* SD0_DAT2	*/
-	[RCAR_GP_PIN(3, 16)] = { PUPR4, 25 },	/* SD0_DAT3	*/
-	[RCAR_GP_PIN(3, 17)] = { PUPR4, 26 },	/* SD0_CD	*/
-	[RCAR_GP_PIN(3, 18)] = { PUPR4, 27 },	/* SD0_WP	*/
-	[RCAR_GP_PIN(2, 22)] = { PUPR4, 28 },	/* AUDIO_CLKA	*/
-	[RCAR_GP_PIN(2, 23)] = { PUPR4, 29 },	/* AUDIO_CLKB	*/
-	[RCAR_GP_PIN(1, 14)] = { PUPR4, 30 },	/* IRQ2		*/
-	[RCAR_GP_PIN(1, 15)] = { PUPR4, 31 },	/* IRQ3		*/
-
-	[RCAR_GP_PIN(0,  1)] = { PUPR5,  0 },	/* PENC0	*/
-	[RCAR_GP_PIN(0,  2)] = { PUPR5,  1 },	/* PENC1	*/
-	[RCAR_GP_PIN(0,  3)] = { PUPR5,  2 },	/* USB_OVC0	*/
-	[RCAR_GP_PIN(0,  4)] = { PUPR5,  3 },	/* USB_OVC1	*/
-	[RCAR_GP_PIN(1, 16)] = { PUPR5,  4 },	/* SCIF_CLK	*/
-	[RCAR_GP_PIN(1, 17)] = { PUPR5,  5 },	/* TX0		*/
-	[RCAR_GP_PIN(1, 18)] = { PUPR5,  6 },	/* RX0		*/
-	[RCAR_GP_PIN(1, 19)] = { PUPR5,  7 },	/* SCK0		*/
-	[RCAR_GP_PIN(1, 20)] = { PUPR5,  8 },	/* /CTS0	*/
-	[RCAR_GP_PIN(1, 21)] = { PUPR5,  9 },	/* /RTS0	*/
-	[RCAR_GP_PIN(3, 19)] = { PUPR5, 10 },	/* HSPI_CLK0	*/
-	[RCAR_GP_PIN(3, 20)] = { PUPR5, 11 },	/* /HSPI_CS0	*/
-	[RCAR_GP_PIN(3, 21)] = { PUPR5, 12 },	/* HSPI_RX0	*/
-	[RCAR_GP_PIN(3, 22)] = { PUPR5, 13 },	/* HSPI_TX0	*/
-	[RCAR_GP_PIN(4, 20)] = { PUPR5, 14 },	/* ETH_MAGIC	*/
-	[RCAR_GP_PIN(4, 25)] = { PUPR5, 15 },	/* AVS1		*/
-	[RCAR_GP_PIN(4, 26)] = { PUPR5, 16 },	/* AVS2		*/
+static const struct sh_pfc_bias_info bias_info[] = {
+	{ RCAR_GP_PIN(0,  6), PUPR0,  0 },	/* A0 */
+	{ RCAR_GP_PIN(0,  7), PUPR0,  1 },	/* A1 */
+	{ RCAR_GP_PIN(0,  8), PUPR0,  2 },	/* A2 */
+	{ RCAR_GP_PIN(0,  9), PUPR0,  3 },	/* A3 */
+	{ RCAR_GP_PIN(0, 10), PUPR0,  4 },	/* A4 */
+	{ RCAR_GP_PIN(0, 11), PUPR0,  5 },	/* A5 */
+	{ RCAR_GP_PIN(0, 12), PUPR0,  6 },	/* A6 */
+	{ RCAR_GP_PIN(0, 13), PUPR0,  7 },	/* A7 */
+	{ RCAR_GP_PIN(0, 14), PUPR0,  8 },	/* A8 */
+	{ RCAR_GP_PIN(0, 15), PUPR0,  9 },	/* A9 */
+	{ RCAR_GP_PIN(0, 16), PUPR0, 10 },	/* A10 */
+	{ RCAR_GP_PIN(0, 17), PUPR0, 11 },	/* A11 */
+	{ RCAR_GP_PIN(0, 18), PUPR0, 12 },	/* A12 */
+	{ RCAR_GP_PIN(0, 19), PUPR0, 13 },	/* A13 */
+	{ RCAR_GP_PIN(0, 20), PUPR0, 14 },	/* A14 */
+	{ RCAR_GP_PIN(0, 21), PUPR0, 15 },	/* A15 */
+	{ RCAR_GP_PIN(0, 22), PUPR0, 16 },	/* A16 */
+	{ RCAR_GP_PIN(0, 23), PUPR0, 17 },	/* A17 */
+	{ RCAR_GP_PIN(0, 24), PUPR0, 18 },	/* A18 */
+	{ RCAR_GP_PIN(0, 25), PUPR0, 19 },	/* A19 */
+	{ RCAR_GP_PIN(0, 26), PUPR0, 20 },	/* A20 */
+	{ RCAR_GP_PIN(0, 27), PUPR0, 21 },	/* A21 */
+	{ RCAR_GP_PIN(0, 28), PUPR0, 22 },	/* A22 */
+	{ RCAR_GP_PIN(0, 29), PUPR0, 23 },	/* A23 */
+	{ RCAR_GP_PIN(0, 30), PUPR0, 24 },	/* A24 */
+	{ RCAR_GP_PIN(0, 31), PUPR0, 25 },	/* A25 */
+	{ RCAR_GP_PIN(1,  3), PUPR0, 26 },	/* /EX_CS0 */
+	{ RCAR_GP_PIN(1,  4), PUPR0, 27 },	/* /EX_CS1 */
+	{ RCAR_GP_PIN(1,  5), PUPR0, 28 },	/* /EX_CS2 */
+	{ RCAR_GP_PIN(1,  6), PUPR0, 29 },	/* /EX_CS3 */
+	{ RCAR_GP_PIN(1,  7), PUPR0, 30 },	/* /EX_CS4 */
+	{ RCAR_GP_PIN(1,  8), PUPR0, 31 },	/* /EX_CS5 */
+
+	{ RCAR_GP_PIN(0,  0), PUPR1,  0 },	/* /PRESETOUT	*/
+	{ RCAR_GP_PIN(0,  5), PUPR1,  1 },	/* /BS		*/
+	{ RCAR_GP_PIN(1,  0), PUPR1,  2 },	/* RD//WR	*/
+	{ RCAR_GP_PIN(1,  1), PUPR1,  3 },	/* /WE0		*/
+	{ RCAR_GP_PIN(1,  2), PUPR1,  4 },	/* /WE1		*/
+	{ RCAR_GP_PIN(1, 11), PUPR1,  5 },	/* EX_WAIT0	*/
+	{ RCAR_GP_PIN(1,  9), PUPR1,  6 },	/* DREQ0	*/
+	{ RCAR_GP_PIN(1, 10), PUPR1,  7 },	/* DACK0	*/
+	{ RCAR_GP_PIN(1, 12), PUPR1,  8 },	/* IRQ0		*/
+	{ RCAR_GP_PIN(1, 13), PUPR1,  9 },	/* IRQ1		*/
+
+	{ RCAR_GP_PIN(1, 22), PUPR2,  0 },	/* DU0_DR0	*/
+	{ RCAR_GP_PIN(1, 23), PUPR2,  1 },	/* DU0_DR1	*/
+	{ RCAR_GP_PIN(1, 24), PUPR2,  2 },	/* DU0_DR2	*/
+	{ RCAR_GP_PIN(1, 25), PUPR2,  3 },	/* DU0_DR3	*/
+	{ RCAR_GP_PIN(1, 26), PUPR2,  4 },	/* DU0_DR4	*/
+	{ RCAR_GP_PIN(1, 27), PUPR2,  5 },	/* DU0_DR5	*/
+	{ RCAR_GP_PIN(1, 28), PUPR2,  6 },	/* DU0_DR6	*/
+	{ RCAR_GP_PIN(1, 29), PUPR2,  7 },	/* DU0_DR7	*/
+	{ RCAR_GP_PIN(1, 30), PUPR2,  8 },	/* DU0_DG0	*/
+	{ RCAR_GP_PIN(1, 31), PUPR2,  9 },	/* DU0_DG1	*/
+	{ RCAR_GP_PIN(2,  0), PUPR2, 10 },	/* DU0_DG2	*/
+	{ RCAR_GP_PIN(2,  1), PUPR2, 11 },	/* DU0_DG3	*/
+	{ RCAR_GP_PIN(2,  2), PUPR2, 12 },	/* DU0_DG4	*/
+	{ RCAR_GP_PIN(2,  3), PUPR2, 13 },	/* DU0_DG5	*/
+	{ RCAR_GP_PIN(2,  4), PUPR2, 14 },	/* DU0_DG6	*/
+	{ RCAR_GP_PIN(2,  5), PUPR2, 15 },	/* DU0_DG7	*/
+	{ RCAR_GP_PIN(2,  6), PUPR2, 16 },	/* DU0_DB0	*/
+	{ RCAR_GP_PIN(2,  7), PUPR2, 17 },	/* DU0_DB1	*/
+	{ RCAR_GP_PIN(2,  8), PUPR2, 18 },	/* DU0_DB2	*/
+	{ RCAR_GP_PIN(2,  9), PUPR2, 19 },	/* DU0_DB3	*/
+	{ RCAR_GP_PIN(2, 10), PUPR2, 20 },	/* DU0_DB4	*/
+	{ RCAR_GP_PIN(2, 11), PUPR2, 21 },	/* DU0_DB5	*/
+	{ RCAR_GP_PIN(2, 12), PUPR2, 22 },	/* DU0_DB6	*/
+	{ RCAR_GP_PIN(2, 13), PUPR2, 23 },	/* DU0_DB7	*/
+	{ RCAR_GP_PIN(2, 14), PUPR2, 24 },	/* DU0_DOTCLKIN	*/
+	{ RCAR_GP_PIN(2, 15), PUPR2, 25 },	/* DU0_DOTCLKOUT0 */
+	{ RCAR_GP_PIN(2, 17), PUPR2, 26 },	/* DU0_HSYNC	*/
+	{ RCAR_GP_PIN(2, 18), PUPR2, 27 },	/* DU0_VSYNC	*/
+	{ RCAR_GP_PIN(2, 19), PUPR2, 28 },	/* DU0_EXODDF	*/
+	{ RCAR_GP_PIN(2, 20), PUPR2, 29 },	/* DU0_DISP	*/
+	{ RCAR_GP_PIN(2, 21), PUPR2, 30 },	/* DU0_CDE	*/
+	{ RCAR_GP_PIN(2, 16), PUPR2, 31 },	/* DU0_DOTCLKOUT1 */
+
+	{ RCAR_GP_PIN(3, 24), PUPR3,  0 },	/* VI0_CLK	*/
+	{ RCAR_GP_PIN(3, 25), PUPR3,  1 },	/* VI0_CLKENB	*/
+	{ RCAR_GP_PIN(3, 26), PUPR3,  2 },	/* VI0_FIELD	*/
+	{ RCAR_GP_PIN(3, 27), PUPR3,  3 },	/* /VI0_HSYNC	*/
+	{ RCAR_GP_PIN(3, 28), PUPR3,  4 },	/* /VI0_VSYNC	*/
+	{ RCAR_GP_PIN(3, 29), PUPR3,  5 },	/* VI0_DATA0	*/
+	{ RCAR_GP_PIN(3, 30), PUPR3,  6 },	/* VI0_DATA1	*/
+	{ RCAR_GP_PIN(3, 31), PUPR3,  7 },	/* VI0_DATA2	*/
+	{ RCAR_GP_PIN(4,  0), PUPR3,  8 },	/* VI0_DATA3	*/
+	{ RCAR_GP_PIN(4,  1), PUPR3,  9 },	/* VI0_DATA4	*/
+	{ RCAR_GP_PIN(4,  2), PUPR3, 10 },	/* VI0_DATA5	*/
+	{ RCAR_GP_PIN(4,  3), PUPR3, 11 },	/* VI0_DATA6	*/
+	{ RCAR_GP_PIN(4,  4), PUPR3, 12 },	/* VI0_DATA7	*/
+	{ RCAR_GP_PIN(4,  5), PUPR3, 13 },	/* VI0_G2	*/
+	{ RCAR_GP_PIN(4,  6), PUPR3, 14 },	/* VI0_G3	*/
+	{ RCAR_GP_PIN(4,  7), PUPR3, 15 },	/* VI0_G4	*/
+	{ RCAR_GP_PIN(4,  8), PUPR3, 16 },	/* VI0_G5	*/
+	{ RCAR_GP_PIN(4, 21), PUPR3, 17 },	/* VI1_DATA12	*/
+	{ RCAR_GP_PIN(4, 22), PUPR3, 18 },	/* VI1_DATA13	*/
+	{ RCAR_GP_PIN(4, 23), PUPR3, 19 },	/* VI1_DATA14	*/
+	{ RCAR_GP_PIN(4, 24), PUPR3, 20 },	/* VI1_DATA15	*/
+	{ RCAR_GP_PIN(4,  9), PUPR3, 21 },	/* ETH_REF_CLK	*/
+	{ RCAR_GP_PIN(4, 10), PUPR3, 22 },	/* ETH_TXD0	*/
+	{ RCAR_GP_PIN(4, 11), PUPR3, 23 },	/* ETH_TXD1	*/
+	{ RCAR_GP_PIN(4, 12), PUPR3, 24 },	/* ETH_CRS_DV	*/
+	{ RCAR_GP_PIN(4, 13), PUPR3, 25 },	/* ETH_TX_EN	*/
+	{ RCAR_GP_PIN(4, 14), PUPR3, 26 },	/* ETH_RX_ER	*/
+	{ RCAR_GP_PIN(4, 15), PUPR3, 27 },	/* ETH_RXD0	*/
+	{ RCAR_GP_PIN(4, 16), PUPR3, 28 },	/* ETH_RXD1	*/
+	{ RCAR_GP_PIN(4, 17), PUPR3, 29 },	/* ETH_MDC	*/
+	{ RCAR_GP_PIN(4, 18), PUPR3, 30 },	/* ETH_MDIO	*/
+	{ RCAR_GP_PIN(4, 19), PUPR3, 31 },	/* ETH_LINK	*/
+
+	{ RCAR_GP_PIN(3,  6), PUPR4,  0 },	/* SSI_SCK012	*/
+	{ RCAR_GP_PIN(3,  7), PUPR4,  1 },	/* SSI_WS012	*/
+	{ RCAR_GP_PIN(3, 10), PUPR4,  2 },	/* SSI_SDATA0	*/
+	{ RCAR_GP_PIN(3,  9), PUPR4,  3 },	/* SSI_SDATA1	*/
+	{ RCAR_GP_PIN(3,  8), PUPR4,  4 },	/* SSI_SDATA2	*/
+	{ RCAR_GP_PIN(3,  2), PUPR4,  5 },	/* SSI_SCK34	*/
+	{ RCAR_GP_PIN(3,  3), PUPR4,  6 },	/* SSI_WS34	*/
+	{ RCAR_GP_PIN(3,  5), PUPR4,  7 },	/* SSI_SDATA3	*/
+	{ RCAR_GP_PIN(3,  4), PUPR4,  8 },	/* SSI_SDATA4	*/
+	{ RCAR_GP_PIN(2, 31), PUPR4,  9 },	/* SSI_SCK5	*/
+	{ RCAR_GP_PIN(3,  0), PUPR4, 10 },	/* SSI_WS5	*/
+	{ RCAR_GP_PIN(3,  1), PUPR4, 11 },	/* SSI_SDATA5	*/
+	{ RCAR_GP_PIN(2, 28), PUPR4, 12 },	/* SSI_SCK6	*/
+	{ RCAR_GP_PIN(2, 29), PUPR4, 13 },	/* SSI_WS6	*/
+	{ RCAR_GP_PIN(2, 30), PUPR4, 14 },	/* SSI_SDATA6	*/
+	{ RCAR_GP_PIN(2, 24), PUPR4, 15 },	/* SSI_SCK78	*/
+	{ RCAR_GP_PIN(2, 25), PUPR4, 16 },	/* SSI_WS78	*/
+	{ RCAR_GP_PIN(2, 27), PUPR4, 17 },	/* SSI_SDATA7	*/
+	{ RCAR_GP_PIN(2, 26), PUPR4, 18 },	/* SSI_SDATA8	*/
+	{ RCAR_GP_PIN(3, 23), PUPR4, 19 },	/* TCLK0	*/
+	{ RCAR_GP_PIN(3, 11), PUPR4, 20 },	/* SD0_CLK	*/
+	{ RCAR_GP_PIN(3, 12), PUPR4, 21 },	/* SD0_CMD	*/
+	{ RCAR_GP_PIN(3, 13), PUPR4, 22 },	/* SD0_DAT0	*/
+	{ RCAR_GP_PIN(3, 14), PUPR4, 23 },	/* SD0_DAT1	*/
+	{ RCAR_GP_PIN(3, 15), PUPR4, 24 },	/* SD0_DAT2	*/
+	{ RCAR_GP_PIN(3, 16), PUPR4, 25 },	/* SD0_DAT3	*/
+	{ RCAR_GP_PIN(3, 17), PUPR4, 26 },	/* SD0_CD	*/
+	{ RCAR_GP_PIN(3, 18), PUPR4, 27 },	/* SD0_WP	*/
+	{ RCAR_GP_PIN(2, 22), PUPR4, 28 },	/* AUDIO_CLKA	*/
+	{ RCAR_GP_PIN(2, 23), PUPR4, 29 },	/* AUDIO_CLKB	*/
+	{ RCAR_GP_PIN(1, 14), PUPR4, 30 },	/* IRQ2		*/
+	{ RCAR_GP_PIN(1, 15), PUPR4, 31 },	/* IRQ3		*/
+
+	{ RCAR_GP_PIN(0,  1), PUPR5,  0 },	/* PENC0	*/
+	{ RCAR_GP_PIN(0,  2), PUPR5,  1 },	/* PENC1	*/
+	{ RCAR_GP_PIN(0,  3), PUPR5,  2 },	/* USB_OVC0	*/
+	{ RCAR_GP_PIN(0,  4), PUPR5,  3 },	/* USB_OVC1	*/
+	{ RCAR_GP_PIN(1, 16), PUPR5,  4 },	/* SCIF_CLK	*/
+	{ RCAR_GP_PIN(1, 17), PUPR5,  5 },	/* TX0		*/
+	{ RCAR_GP_PIN(1, 18), PUPR5,  6 },	/* RX0		*/
+	{ RCAR_GP_PIN(1, 19), PUPR5,  7 },	/* SCK0		*/
+	{ RCAR_GP_PIN(1, 20), PUPR5,  8 },	/* /CTS0	*/
+	{ RCAR_GP_PIN(1, 21), PUPR5,  9 },	/* /RTS0	*/
+	{ RCAR_GP_PIN(3, 19), PUPR5, 10 },	/* HSPI_CLK0	*/
+	{ RCAR_GP_PIN(3, 20), PUPR5, 11 },	/* /HSPI_CS0	*/
+	{ RCAR_GP_PIN(3, 21), PUPR5, 12 },	/* HSPI_RX0	*/
+	{ RCAR_GP_PIN(3, 22), PUPR5, 13 },	/* HSPI_TX0	*/
+	{ RCAR_GP_PIN(4, 20), PUPR5, 14 },	/* ETH_MAGIC	*/
+	{ RCAR_GP_PIN(4, 25), PUPR5, 15 },	/* AVS1		*/
+	{ RCAR_GP_PIN(4, 26), PUPR5, 16 },	/* AVS2		*/
 };
 
 static unsigned int r8a7778_pinmux_get_bias(struct sh_pfc *pfc,
 					    unsigned int pin)
 {
+	const struct sh_pfc_bias_info *info;
 	void __iomem *addr;
 
-	if (WARN_ON_ONCE(!pullups[pin].reg))
+	info = sh_pfc_pin_to_bias_info(bias_info, ARRAY_SIZE(bias_info), pin);
+	if (!info)
 		return PIN_CONFIG_BIAS_DISABLE;
 
-	addr = pfc->windows->virt + pullups[pin].reg;
+	addr = pfc->windows->virt + info->reg;
 
-	if (ioread32(addr) & BIT(pullups[pin].bit))
+	if (ioread32(addr) & BIT(info->bit))
 		return PIN_CONFIG_BIAS_PULL_UP;
 	else
 		return PIN_CONFIG_BIAS_DISABLE;
@@ -3103,15 +3103,17 @@ static unsigned int r8a7778_pinmux_get_bias(struct sh_pfc *pfc,
 static void r8a7778_pinmux_set_bias(struct sh_pfc *pfc, unsigned int pin,
 				   unsigned int bias)
 {
+	const struct sh_pfc_bias_info *info;
 	void __iomem *addr;
 	u32 value;
 	u32 bit;
 
-	if (WARN_ON_ONCE(!pullups[pin].reg))
+	info = sh_pfc_pin_to_bias_info(bias_info, ARRAY_SIZE(bias_info), pin);
+	if (!info)
 		return;
 
-	addr = pfc->windows->virt + pullups[pin].reg;
-	bit = BIT(pullups[pin].bit);
+	addr = pfc->windows->virt + info->reg;
+	bit = BIT(info->bit);
 
 	value = ioread32(addr) & ~bit;
 	if (bias == PIN_CONFIG_BIAS_PULL_UP)
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7795.c b/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
index 2e8cc2adbed7..135ed5cbeb44 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
@@ -523,6 +523,22 @@ MOD_SEL0_2_1		MOD_SEL1_2 \
 			MOD_SEL1_1 \
 			MOD_SEL1_0		MOD_SEL2_0
 
+/*
+ * These pins are not able to be muxed but have other properties
+ * that can be set, such as drive-strength or pull-up/pull-down enable.
+ */
+#define PINMUX_STATIC \
+	FM(QSPI0_SPCLK) FM(QSPI0_SSL) FM(QSPI0_MOSI_IO0) FM(QSPI0_MISO_IO1) \
+	FM(QSPI0_IO2) FM(QSPI0_IO3) \
+	FM(QSPI1_SPCLK) FM(QSPI1_SSL) FM(QSPI1_MOSI_IO0) FM(QSPI1_MISO_IO1) \
+	FM(QSPI1_IO2) FM(QSPI1_IO3) \
+	FM(RPC_INT) FM(RPC_WP) FM(RPC_RESET) \
+	FM(AVB_TX_CTL) FM(AVB_TXC) FM(AVB_TD0) FM(AVB_TD1) FM(AVB_TD2) FM(AVB_TD3) \
+	FM(AVB_RX_CTL) FM(AVB_RXC) FM(AVB_RD0) FM(AVB_RD1) FM(AVB_RD2) FM(AVB_RD3) \
+	FM(AVB_TXCREFCLK) FM(AVB_MDIO) \
+	FM(CLKOUT) FM(PRESETOUT) \
+	FM(DU_DOTCLKIN0) FM(DU_DOTCLKIN1) FM(DU_DOTCLKIN2) FM(DU_DOTCLKIN3) \
+	FM(TMS) FM(TDO) FM(ASEBRK) FM(MLB_REF)
 
 enum {
 	PINMUX_RESERVED = 0,
@@ -548,6 +564,7 @@ enum {
 	PINMUX_GPSR
 	PINMUX_IPSR
 	PINMUX_MOD_SELS
+	PINMUX_STATIC
 	PINMUX_MARK_END,
 #undef F_
 #undef FM
@@ -1412,10 +1429,78 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP17_7_4,	STP_ISSYNC_0_E,		SEL_SSP1_0_4),
 	PINMUX_IPSR_MSEL(IP17_7_4,	RIF2_D1_B,		SEL_DRIF2_1),
 	PINMUX_IPSR_GPSR(IP17_7_4,	TPU0TO3),
+
+/*
+ * Static pins can not be muxed between different functions but
+ * still needs a mark entry in the pinmux list. Add each static
+ * pin to the list without an associated function. The sh-pfc
+ * core will do the right thing and skip trying to mux then pin
+ * while still applying configuration to it
+ */
+#define FM(x)	PINMUX_DATA(x##_MARK, 0),
+	PINMUX_STATIC
+#undef FM
 };
 
+/*
+ * R8A7795 has 8 banks with 32 PGIOS in each => 256 GPIOs.
+ * Physical layout rows: A - AW, cols: 1 - 39.
+ */
+#define ROW_GROUP_A(r) ('Z' - 'A' + 1 + (r))
+#define PIN_NUMBER(r, c) (((r) - 'A') * 39 + (c) + 300)
+#define PIN_A_NUMBER(r, c) PIN_NUMBER(ROW_GROUP_A(r), c)
+
 static const struct sh_pfc_pin pinmux_pins[] = {
 	PINMUX_GPIO_GP_ALL(),
+
+	/*
+	 * Pins not associated with a GPIO port.
+	 *
+	 * The pin positions are different between different r8a7795
+	 * packages, all that is needed for the pfc driver is a unique
+	 * number for each pin. To this end use the pin layout from
+	 * R-Car H3SiP to calculate a unique number for each pin.
+	 */
+	SH_PFC_PIN_NAMED_CFG('A',  8, AVB_TX_CTL, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('A',  9, AVB_MDIO, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('A', 12, AVB_TXCREFCLK, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('A', 13, AVB_RD0, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('A', 14, AVB_RD2, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('A', 16, AVB_RX_CTL, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('A', 17, AVB_TD2, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('A', 18, AVB_TD0, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('A', 19, AVB_TXC, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('B', 13, AVB_RD1, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('B', 14, AVB_RD3, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('B', 17, AVB_TD3, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('B', 18, AVB_TD1, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('B', 19, AVB_RXC, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('C',  1, PRESETOUT#, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('F',  1, CLKOUT, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('H', 37, MLB_REF, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('V',  3, QSPI1_SPCLK, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('V',  5, QSPI1_SSL, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('V',  6, RPC_WP#, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('V',  7, RPC_RESET#, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('W',  3, QSPI0_SPCLK, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('Y',  3, QSPI0_SSL, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('Y',  6, QSPI0_IO2, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG('Y',  7, RPC_INT#, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('B'),  4, QSPI0_MISO_IO1, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('B'),  6, QSPI0_IO3, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('C'),  3, QSPI1_IO3, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('C'),  5, QSPI0_MOSI_IO0, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('C'),  7, QSPI1_MOSI_IO0, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('D'), 38, FSCLKST#, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('E'),  4, QSPI1_IO2, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('E'),  5, QSPI1_MISO_IO1, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('P'),  7, DU_DOTCLKIN0, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('P'),  8, DU_DOTCLKIN1, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('R'),  7, DU_DOTCLKIN2, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('R'),  8, DU_DOTCLKIN3, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('R'), 30, TMS, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('T'), 28, TDO, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
+	SH_PFC_PIN_NAMED_CFG(ROW_GROUP_A('T'), 30, ASEBRK, SH_PFC_PIN_CFG_DRIVE_STRENGTH),
 };
 
 /* - AUDIO CLOCK ------------------------------------------------------------ */
@@ -1563,11 +1648,33 @@ static const unsigned int avb_phy_int_mux[] = {
 	AVB_PHY_INT_MARK,
 };
 static const unsigned int avb_mdc_pins[] = {
-	/* AVB_MDC */
-	RCAR_GP_PIN(2, 9),
+	/* AVB_MDC, AVB_MDIO */
+	RCAR_GP_PIN(2, 9), PIN_NUMBER('A', 9),
 };
 static const unsigned int avb_mdc_mux[] = {
-	AVB_MDC_MARK,
+	AVB_MDC_MARK, AVB_MDIO_MARK,
+};
+static const unsigned int avb_mii_pins[] = {
+	/*
+	 * AVB_TX_CTL, AVB_TXC, AVB_TD0,
+	 * AVB_TD1, AVB_TD2, AVB_TD3,
+	 * AVB_RX_CTL, AVB_RXC, AVB_RD0,
+	 * AVB_RD1, AVB_RD2, AVB_RD3,
+	 * AVB_TXCREFCLK
+	 */
+	PIN_NUMBER('A', 8), PIN_NUMBER('A', 19), PIN_NUMBER('A', 18),
+	PIN_NUMBER('B', 18), PIN_NUMBER('A', 17), PIN_NUMBER('B', 17),
+	PIN_NUMBER('A', 16), PIN_NUMBER('B', 19), PIN_NUMBER('A', 13),
+	PIN_NUMBER('B', 13), PIN_NUMBER('A', 14), PIN_NUMBER('B', 14),
+	PIN_NUMBER('A', 12),
+
+};
+static const unsigned int avb_mii_mux[] = {
+	AVB_TX_CTL_MARK, AVB_TXC_MARK, AVB_TD0_MARK,
+	AVB_TD1_MARK, AVB_TD2_MARK, AVB_TD3_MARK,
+	AVB_RX_CTL_MARK, AVB_RXC_MARK, AVB_RD0_MARK,
+	AVB_RD1_MARK, AVB_RD2_MARK, AVB_RD3_MARK,
+	AVB_TXCREFCLK_MARK,
 };
 static const unsigned int avb_avtp_pps_pins[] = {
 	/* AVB_AVTP_PPS */
@@ -3613,6 +3720,55 @@ static const unsigned int usb2_mux[] = {
 	USB2_PWEN_MARK, USB2_OVC_MARK,
 };
 
+/* - QSPI0 ------------------------------------------------------------------ */
+static const unsigned int qspi0_ctrl_pins[] = {
+	/* QSPI0_SPCLK, QSPI0_SSL */
+	PIN_NUMBER('W', 3), PIN_NUMBER('Y', 3),
+};
+static const unsigned int qspi0_ctrl_mux[] = {
+	QSPI0_SPCLK_MARK, QSPI0_SSL_MARK,
+};
+static const unsigned int qspi0_data2_pins[] = {
+	/* QSPI0_MOSI_IO0, QSPI0_MISO_IO1 */
+	PIN_A_NUMBER('C', 5), PIN_A_NUMBER('B', 4),
+};
+static const unsigned int qspi0_data2_mux[] = {
+	QSPI0_MOSI_IO0_MARK, QSPI0_MISO_IO1_MARK,
+};
+static const unsigned int qspi0_data4_pins[] = {
+	/* QSPI0_MOSI_IO0, QSPI0_MISO_IO1, QSPI0_IO2, QSPI0_IO3 */
+	PIN_A_NUMBER('C', 5), PIN_A_NUMBER('B', 4),
+	PIN_NUMBER('Y', 6), PIN_A_NUMBER('B', 6),
+};
+static const unsigned int qspi0_data4_mux[] = {
+	QSPI0_MOSI_IO0_MARK, QSPI0_MISO_IO1_MARK,
+	QSPI0_IO2_MARK, QSPI0_IO3_MARK,
+};
+/* - QSPI1 ------------------------------------------------------------------ */
+static const unsigned int qspi1_ctrl_pins[] = {
+	/* QSPI1_SPCLK, QSPI1_SSL */
+	PIN_NUMBER('V', 3), PIN_NUMBER('V', 5),
+};
+static const unsigned int qspi1_ctrl_mux[] = {
+	QSPI1_SPCLK_MARK, QSPI1_SSL_MARK,
+};
+static const unsigned int qspi1_data2_pins[] = {
+	/* QSPI1_MOSI_IO0, QSPI1_MISO_IO1 */
+	PIN_A_NUMBER('C', 7), PIN_A_NUMBER('E', 5),
+};
+static const unsigned int qspi1_data2_mux[] = {
+	QSPI1_MOSI_IO0_MARK, QSPI1_MISO_IO1_MARK,
+};
+static const unsigned int qspi1_data4_pins[] = {
+	/* QSPI1_MOSI_IO0, QSPI1_MISO_IO1, QSPI1_IO2, QSPI1_IO3 */
+	PIN_A_NUMBER('C', 7), PIN_A_NUMBER('E', 5),
+	PIN_A_NUMBER('E', 4), PIN_A_NUMBER('C', 3),
+};
+static const unsigned int qspi1_data4_mux[] = {
+	QSPI1_MOSI_IO0_MARK, QSPI1_MISO_IO1_MARK,
+	QSPI1_IO2_MARK, QSPI1_IO3_MARK,
+};
+
 static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(audio_clk_a_a),
 	SH_PFC_PIN_GROUP(audio_clk_a_b),
@@ -3635,6 +3791,7 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(avb_magic),
 	SH_PFC_PIN_GROUP(avb_phy_int),
 	SH_PFC_PIN_GROUP(avb_mdc),
+	SH_PFC_PIN_GROUP(avb_mii),
 	SH_PFC_PIN_GROUP(avb_avtp_pps),
 	SH_PFC_PIN_GROUP(avb_avtp_match_a),
 	SH_PFC_PIN_GROUP(avb_avtp_capture_a),
@@ -3912,6 +4069,12 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(usb0),
 	SH_PFC_PIN_GROUP(usb1),
 	SH_PFC_PIN_GROUP(usb2),
+	SH_PFC_PIN_GROUP(qspi0_ctrl),
+	SH_PFC_PIN_GROUP(qspi0_data2),
+	SH_PFC_PIN_GROUP(qspi0_data4),
+	SH_PFC_PIN_GROUP(qspi1_ctrl),
+	SH_PFC_PIN_GROUP(qspi1_data2),
+	SH_PFC_PIN_GROUP(qspi1_data4),
 };
 
 static const char * const audio_clk_groups[] = {
@@ -3939,6 +4102,7 @@ static const char * const avb_groups[] = {
 	"avb_magic",
 	"avb_phy_int",
 	"avb_mdc",
+	"avb_mii",
 	"avb_avtp_pps",
 	"avb_avtp_match_a",
 	"avb_avtp_capture_a",
@@ -4356,6 +4520,18 @@ static const char * const usb2_groups[] = {
 	"usb2",
 };
 
+static const char * const qspi0_groups[] = {
+	"qspi0_ctrl",
+	"qspi0_data2",
+	"qspi0_data4",
+};
+
+static const char * const qspi1_groups[] = {
+	"qspi1_ctrl",
+	"qspi1_data2",
+	"qspi1_data4",
+};
+
 static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(audio_clk),
 	SH_PFC_FUNCTION(avb),
@@ -4405,6 +4581,8 @@ static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(usb0),
 	SH_PFC_FUNCTION(usb1),
 	SH_PFC_FUNCTION(usb2),
+	SH_PFC_FUNCTION(qspi0),
+	SH_PFC_FUNCTION(qspi1),
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
@@ -4962,10 +5140,45 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 };
 
 static const struct pinmux_drive_reg pinmux_drive_regs[] = {
+	{ PINMUX_DRIVE_REG("DRVCTRL0", 0xe6060300) {
+		{ PIN_NUMBER('W', 3),   28, 2 },	/* QSPI0_SPCLK */
+		{ PIN_A_NUMBER('C', 5), 24, 2 },	/* QSPI0_MOSI_IO0 */
+		{ PIN_A_NUMBER('B', 4), 20, 2 },	/* QSPI0_MISO_IO1 */
+		{ PIN_NUMBER('Y', 6),   16, 2 },	/* QSPI0_IO2 */
+		{ PIN_A_NUMBER('B', 6), 12, 2 },	/* QSPI0_IO3 */
+		{ PIN_NUMBER('Y', 3),    8, 2 },	/* QSPI0_SSL */
+		{ PIN_NUMBER('V', 3),    4, 2 },	/* QSPI1_SPCLK */
+		{ PIN_A_NUMBER('C', 7),  0, 2 },	/* QSPI1_MOSI_IO0 */
+	} },
+	{ PINMUX_DRIVE_REG("DRVCTRL1", 0xe6060304) {
+		{ PIN_A_NUMBER('E', 5), 28, 2 },	/* QSPI1_MISO_IO1 */
+		{ PIN_A_NUMBER('E', 4), 24, 2 },	/* QSPI1_IO2 */
+		{ PIN_A_NUMBER('C', 3), 20, 2 },	/* QSPI1_IO3 */
+		{ PIN_NUMBER('V', 5),   16, 2 },	/* QSPI1_SSL */
+		{ PIN_NUMBER('Y', 7),   12, 2 },	/* RPC_INT# */
+		{ PIN_NUMBER('V', 6),    8, 2 },	/* RPC_WP# */
+		{ PIN_NUMBER('V', 7),    4, 2 },	/* RPC_RESET# */
+		{ PIN_NUMBER('A', 16),   0, 3 },	/* AVB_RX_CTL */
+	} },
+	{ PINMUX_DRIVE_REG("DRVCTRL2", 0xe6060308) {
+		{ PIN_NUMBER('B', 19),  28, 3 },	/* AVB_RXC */
+		{ PIN_NUMBER('A', 13),  24, 3 },	/* AVB_RD0 */
+		{ PIN_NUMBER('B', 13),  20, 3 },	/* AVB_RD1 */
+		{ PIN_NUMBER('A', 14),  16, 3 },	/* AVB_RD2 */
+		{ PIN_NUMBER('B', 14),  12, 3 },	/* AVB_RD3 */
+		{ PIN_NUMBER('A', 8),    8, 3 },	/* AVB_TX_CTL */
+		{ PIN_NUMBER('A', 19),   4, 3 },	/* AVB_TXC */
+		{ PIN_NUMBER('A', 18),   0, 3 },	/* AVB_TD0 */
+	} },
 	{ PINMUX_DRIVE_REG("DRVCTRL3", 0xe606030c) {
-		{ RCAR_GP_PIN(2,  9),  8, 3 },	/* AVB_MDC */
-		{ RCAR_GP_PIN(2, 10),  4, 3 },	/* AVB_MAGIC */
-		{ RCAR_GP_PIN(2, 11),  0, 3 },	/* AVB_PHY_INT */
+		{ PIN_NUMBER('B', 18),  28, 3 },	/* AVB_TD1 */
+		{ PIN_NUMBER('A', 17),  24, 3 },	/* AVB_TD2 */
+		{ PIN_NUMBER('B', 17),  20, 3 },	/* AVB_TD3 */
+		{ PIN_NUMBER('A', 12),  16, 3 },	/* AVB_TXCREFCLK */
+		{ PIN_NUMBER('A', 9),   12, 3 },	/* AVB_MDIO */
+		{ RCAR_GP_PIN(2,  9),    8, 3 },	/* AVB_MDC */
+		{ RCAR_GP_PIN(2, 10),    4, 3 },	/* AVB_MAGIC */
+		{ RCAR_GP_PIN(2, 11),    0, 3 },	/* AVB_PHY_INT */
 	} },
 	{ PINMUX_DRIVE_REG("DRVCTRL4", 0xe6060310) {
 		{ RCAR_GP_PIN(2, 12), 28, 3 },	/* AVB_LINK */
@@ -5008,6 +5221,7 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 		{ RCAR_GP_PIN(1, 19),  0, 3 },	/* A19 */
 	} },
 	{ PINMUX_DRIVE_REG("DRVCTRL8", 0xe6060320) {
+		{ PIN_NUMBER('F', 1), 28, 3 },	/* CLKOUT */
 		{ RCAR_GP_PIN(1, 20), 24, 3 },	/* CS0 */
 		{ RCAR_GP_PIN(1, 21), 20, 3 },	/* CS1_A26 */
 		{ RCAR_GP_PIN(1, 22), 16, 3 },	/* BS */
@@ -5018,6 +5232,7 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 	} },
 	{ PINMUX_DRIVE_REG("DRVCTRL9", 0xe6060324) {
 		{ RCAR_GP_PIN(1, 27), 28, 3 },	/* EX_WAIT0 */
+		{ PIN_NUMBER('C', 1), 24, 3 },	/* PRESETOUT# */
 		{ RCAR_GP_PIN(0,  0), 20, 3 },	/* D0 */
 		{ RCAR_GP_PIN(0,  1), 16, 3 },	/* D1 */
 		{ RCAR_GP_PIN(0,  2), 12, 3 },	/* D2 */
@@ -5036,20 +5251,30 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 		{ RCAR_GP_PIN(0, 13),  0, 3 },	/* D13 */
 	} },
 	{ PINMUX_DRIVE_REG("DRVCTRL11", 0xe606032c) {
-		{ RCAR_GP_PIN(0, 14), 28, 3 },	/* D14 */
-		{ RCAR_GP_PIN(0, 15), 24, 3 },	/* D15 */
-		{ RCAR_GP_PIN(7,  0), 20, 3 },	/* AVS1 */
-		{ RCAR_GP_PIN(7,  1), 16, 3 },	/* AVS2 */
-		{ RCAR_GP_PIN(7,  2), 12, 3 },	/* HDMI0_CEC */
-		{ RCAR_GP_PIN(7,  3),  8, 3 },	/* HDMI1_CEC */
+		{ RCAR_GP_PIN(0, 14),   28, 3 },	/* D14 */
+		{ RCAR_GP_PIN(0, 15),   24, 3 },	/* D15 */
+		{ RCAR_GP_PIN(7,  0),   20, 3 },	/* AVS1 */
+		{ RCAR_GP_PIN(7,  1),   16, 3 },	/* AVS2 */
+		{ RCAR_GP_PIN(7,  2),   12, 3 },	/* HDMI0_CEC */
+		{ RCAR_GP_PIN(7,  3),    8, 3 },	/* HDMI1_CEC */
+		{ PIN_A_NUMBER('P', 7),  4, 2 },	/* DU_DOTCLKIN0 */
+		{ PIN_A_NUMBER('P', 8),  0, 2 },	/* DU_DOTCLKIN1 */
+	} },
+	{ PINMUX_DRIVE_REG("DRVCTRL12", 0xe6060330) {
+		{ PIN_A_NUMBER('R', 7),  28, 2 },	/* DU_DOTCLKIN2 */
+		{ PIN_A_NUMBER('R', 8),  24, 2 },	/* DU_DOTCLKIN3 */
+		{ PIN_A_NUMBER('D', 38), 20, 2 },	/* FSCLKST# */
+		{ PIN_A_NUMBER('R', 30),  4, 2 },	/* TMS */
 	} },
 	{ PINMUX_DRIVE_REG("DRVCTRL13", 0xe6060334) {
-		{ RCAR_GP_PIN(3,  0), 20, 3 },	/* SD0_CLK */
-		{ RCAR_GP_PIN(3,  1), 16, 3 },	/* SD0_CMD */
-		{ RCAR_GP_PIN(3,  2), 12, 3 },	/* SD0_DAT0 */
-		{ RCAR_GP_PIN(3,  3),  8, 3 },	/* SD0_DAT1 */
-		{ RCAR_GP_PIN(3,  4),  4, 3 },	/* SD0_DAT2 */
-		{ RCAR_GP_PIN(3,  5),  0, 3 },	/* SD0_DAT3 */
+		{ PIN_A_NUMBER('T', 28), 28, 2 },	/* TDO */
+		{ PIN_A_NUMBER('T', 30), 24, 2 },	/* ASEBRK */
+		{ RCAR_GP_PIN(3,  0),    20, 3 },	/* SD0_CLK */
+		{ RCAR_GP_PIN(3,  1),    16, 3 },	/* SD0_CMD */
+		{ RCAR_GP_PIN(3,  2),    12, 3 },	/* SD0_DAT0 */
+		{ RCAR_GP_PIN(3,  3),     8, 3 },	/* SD0_DAT1 */
+		{ RCAR_GP_PIN(3,  4),     4, 3 },	/* SD0_DAT2 */
+		{ RCAR_GP_PIN(3,  5),     0, 3 },	/* SD0_DAT3 */
 	} },
 	{ PINMUX_DRIVE_REG("DRVCTRL14", 0xe6060338) {
 		{ RCAR_GP_PIN(3,  6), 28, 3 },	/* SD1_CLK */
@@ -5118,6 +5343,7 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 		{ RCAR_GP_PIN(5, 23), 16, 3 },	/* MLB_CLK */
 		{ RCAR_GP_PIN(5, 24), 12, 3 },	/* MLB_SIG */
 		{ RCAR_GP_PIN(5, 25),  8, 3 },	/* MLB_DAT */
+		{ PIN_NUMBER('H', 37),  4, 3 },	/* MLB_REF */
 		{ RCAR_GP_PIN(6,  0),  0, 3 },	/* SSI_SCK01239 */
 	} },
 	{ PINMUX_DRIVE_REG("DRVCTRL21", 0xe6060354) {
@@ -5188,206 +5414,206 @@ static int r8a7795_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin, u32 *poc
 #define PU5	0x14
 #define PU6	0x18
 
-static const struct {
-	u16 reg : 11;
-	u16 bit : 5;
-} pullups[] = {
-	[RCAR_GP_PIN(2, 11)] = { PU0, 31 },	/* AVB_PHY_INT */
-	[RCAR_GP_PIN(2, 10)] = { PU0, 30 },	/* AVB_MAGIC */
-	[RCAR_GP_PIN(2,  9)] = { PU0, 29 },	/* AVB_MDC */
-
-	[RCAR_GP_PIN(1, 19)] = { PU1, 31 },	/* A19 */
-	[RCAR_GP_PIN(1, 18)] = { PU1, 30 },	/* A18 */
-	[RCAR_GP_PIN(1, 17)] = { PU1, 29 },	/* A17 */
-	[RCAR_GP_PIN(1, 16)] = { PU1, 28 },	/* A16 */
-	[RCAR_GP_PIN(1, 15)] = { PU1, 27 },	/* A15 */
-	[RCAR_GP_PIN(1, 14)] = { PU1, 26 },	/* A14 */
-	[RCAR_GP_PIN(1, 13)] = { PU1, 25 },	/* A13 */
-	[RCAR_GP_PIN(1, 12)] = { PU1, 24 },	/* A12 */
-	[RCAR_GP_PIN(1, 11)] = { PU1, 23 },	/* A11 */
-	[RCAR_GP_PIN(1, 10)] = { PU1, 22 },	/* A10 */
-	[RCAR_GP_PIN(1,  9)] = { PU1, 21 },	/* A9 */
-	[RCAR_GP_PIN(1,  8)] = { PU1, 20 },	/* A8 */
-	[RCAR_GP_PIN(1,  7)] = { PU1, 19 },	/* A7 */
-	[RCAR_GP_PIN(1,  6)] = { PU1, 18 },	/* A6 */
-	[RCAR_GP_PIN(1,  5)] = { PU1, 17 },	/* A5 */
-	[RCAR_GP_PIN(1,  4)] = { PU1, 16 },	/* A4 */
-	[RCAR_GP_PIN(1,  3)] = { PU1, 15 },	/* A3 */
-	[RCAR_GP_PIN(1,  2)] = { PU1, 14 },	/* A2 */
-	[RCAR_GP_PIN(1,  1)] = { PU1, 13 },	/* A1 */
-	[RCAR_GP_PIN(1,  0)] = { PU1, 12 },	/* A0 */
-	[RCAR_GP_PIN(2,  8)] = { PU1, 11 },	/* PWM2_A */
-	[RCAR_GP_PIN(2,  7)] = { PU1, 10 },	/* PWM1_A */
-	[RCAR_GP_PIN(2,  6)] = { PU1,  9 },	/* PWM0 */
-	[RCAR_GP_PIN(2,  5)] = { PU1,  8 },	/* IRQ5 */
-	[RCAR_GP_PIN(2,  4)] = { PU1,  7 },	/* IRQ4 */
-	[RCAR_GP_PIN(2,  3)] = { PU1,  6 },	/* IRQ3 */
-	[RCAR_GP_PIN(2,  2)] = { PU1,  5 },	/* IRQ2 */
-	[RCAR_GP_PIN(2,  1)] = { PU1,  4 },	/* IRQ1 */
-	[RCAR_GP_PIN(2,  0)] = { PU1,  3 },	/* IRQ0 */
-	[RCAR_GP_PIN(2, 14)] = { PU1,  2 },	/* AVB_AVTP_CAPTURE_A */
-	[RCAR_GP_PIN(2, 13)] = { PU1,  1 },	/* AVB_AVTP_MATCH_A */
-	[RCAR_GP_PIN(2, 12)] = { PU1,  0 },	/* AVB_LINK */
-
-	[RCAR_GP_PIN(7,  3)] = { PU2, 29 },	/* HDMI1_CEC */
-	[RCAR_GP_PIN(7,  2)] = { PU2, 28 },	/* HDMI0_CEC */
-	[RCAR_GP_PIN(7,  1)] = { PU2, 27 },	/* AVS2 */
-	[RCAR_GP_PIN(7,  0)] = { PU2, 26 },	/* AVS1 */
-	[RCAR_GP_PIN(0, 15)] = { PU2, 25 },	/* D15 */
-	[RCAR_GP_PIN(0, 14)] = { PU2, 24 },	/* D14 */
-	[RCAR_GP_PIN(0, 13)] = { PU2, 23 },	/* D13 */
-	[RCAR_GP_PIN(0, 12)] = { PU2, 22 },	/* D12 */
-	[RCAR_GP_PIN(0, 11)] = { PU2, 21 },	/* D11 */
-	[RCAR_GP_PIN(0, 10)] = { PU2, 20 },	/* D10 */
-	[RCAR_GP_PIN(0,  9)] = { PU2, 19 },	/* D9 */
-	[RCAR_GP_PIN(0,  8)] = { PU2, 18 },	/* D8 */
-	[RCAR_GP_PIN(0,  7)] = { PU2, 17 },	/* D7 */
-	[RCAR_GP_PIN(0,  6)] = { PU2, 16 },	/* D6 */
-	[RCAR_GP_PIN(0,  5)] = { PU2, 15 },	/* D5 */
-	[RCAR_GP_PIN(0,  4)] = { PU2, 14 },	/* D4 */
-	[RCAR_GP_PIN(0,  3)] = { PU2, 13 },	/* D3 */
-	[RCAR_GP_PIN(0,  2)] = { PU2, 12 },	/* D2 */
-	[RCAR_GP_PIN(0,  1)] = { PU2, 11 },	/* D1 */
-	[RCAR_GP_PIN(0,  0)] = { PU2, 10 },	/* D0 */
-	[RCAR_GP_PIN(1, 27)] = { PU2,  8 },	/* EX_WAIT0_A */
-	[RCAR_GP_PIN(1, 26)] = { PU2,  7 },	/* WE1_N */
-	[RCAR_GP_PIN(1, 25)] = { PU2,  6 },	/* WE0_N */
-	[RCAR_GP_PIN(1, 24)] = { PU2,  5 },	/* RD_WR_N */
-	[RCAR_GP_PIN(1, 23)] = { PU2,  4 },	/* RD_N */
-	[RCAR_GP_PIN(1, 22)] = { PU2,  3 },	/* BS_N */
-	[RCAR_GP_PIN(1, 21)] = { PU2,  2 },	/* CS1_N_A26 */
-	[RCAR_GP_PIN(1, 20)] = { PU2,  1 },	/* CS0_N */
-
-	[RCAR_GP_PIN(4,  9)] = { PU3, 31 },	/* SD3_DAT0 */
-	[RCAR_GP_PIN(4,  8)] = { PU3, 30 },	/* SD3_CMD */
-	[RCAR_GP_PIN(4,  7)] = { PU3, 29 },	/* SD3_CLK */
-	[RCAR_GP_PIN(4,  6)] = { PU3, 28 },	/* SD2_DS */
-	[RCAR_GP_PIN(4,  5)] = { PU3, 27 },	/* SD2_DAT3 */
-	[RCAR_GP_PIN(4,  4)] = { PU3, 26 },	/* SD2_DAT2 */
-	[RCAR_GP_PIN(4,  3)] = { PU3, 25 },	/* SD2_DAT1 */
-	[RCAR_GP_PIN(4,  2)] = { PU3, 24 },	/* SD2_DAT0 */
-	[RCAR_GP_PIN(4,  1)] = { PU3, 23 },	/* SD2_CMD */
-	[RCAR_GP_PIN(4,  0)] = { PU3, 22 },	/* SD2_CLK */
-	[RCAR_GP_PIN(3, 11)] = { PU3, 21 },	/* SD1_DAT3 */
-	[RCAR_GP_PIN(3, 10)] = { PU3, 20 },	/* SD1_DAT2 */
-	[RCAR_GP_PIN(3,  9)] = { PU3, 19 },	/* SD1_DAT1 */
-	[RCAR_GP_PIN(3,  8)] = { PU3, 18 },	/* SD1_DAT0 */
-	[RCAR_GP_PIN(3,  7)] = { PU3, 17 },	/* SD1_CMD */
-	[RCAR_GP_PIN(3,  6)] = { PU3, 16 },	/* SD1_CLK */
-	[RCAR_GP_PIN(3,  5)] = { PU3, 15 },	/* SD0_DAT3 */
-	[RCAR_GP_PIN(3,  4)] = { PU3, 14 },	/* SD0_DAT2 */
-	[RCAR_GP_PIN(3,  3)] = { PU3, 13 },	/* SD0_DAT1 */
-	[RCAR_GP_PIN(3,  2)] = { PU3, 12 },	/* SD0_DAT0 */
-	[RCAR_GP_PIN(3,  1)] = { PU3, 11 },	/* SD0_CMD */
-	[RCAR_GP_PIN(3,  0)] = { PU3, 10 },	/* SD0_CLK */
-
-	[RCAR_GP_PIN(5, 19)] = { PU4, 31 },	/* MSIOF0_SS1 */
-	[RCAR_GP_PIN(5, 18)] = { PU4, 30 },	/* MSIOF0_SYNC */
-	[RCAR_GP_PIN(5, 17)] = { PU4, 29 },	/* MSIOF0_SCK */
-	[RCAR_GP_PIN(5, 16)] = { PU4, 28 },	/* HRTS0_N */
-	[RCAR_GP_PIN(5, 15)] = { PU4, 27 },	/* HCTS0_N */
-	[RCAR_GP_PIN(5, 14)] = { PU4, 26 },	/* HTX0 */
-	[RCAR_GP_PIN(5, 13)] = { PU4, 25 },	/* HRX0 */
-	[RCAR_GP_PIN(5, 12)] = { PU4, 24 },	/* HSCK0 */
-	[RCAR_GP_PIN(5, 11)] = { PU4, 23 },	/* RX2_A */
-	[RCAR_GP_PIN(5, 10)] = { PU4, 22 },	/* TX2_A */
-	[RCAR_GP_PIN(5,  9)] = { PU4, 21 },	/* SCK2 */
-	[RCAR_GP_PIN(5,  8)] = { PU4, 20 },	/* RTS1_N_TANS */
-	[RCAR_GP_PIN(5,  7)] = { PU4, 19 },	/* CTS1_N */
-	[RCAR_GP_PIN(5,  6)] = { PU4, 18 },	/* TX1_A */
-	[RCAR_GP_PIN(5,  5)] = { PU4, 17 },	/* RX1_A */
-	[RCAR_GP_PIN(5,  4)] = { PU4, 16 },	/* RTS0_N_TANS */
-	[RCAR_GP_PIN(5,  3)] = { PU4, 15 },	/* CTS0_N */
-	[RCAR_GP_PIN(5,  2)] = { PU4, 14 },	/* TX0 */
-	[RCAR_GP_PIN(5,  1)] = { PU4, 13 },	/* RX0 */
-	[RCAR_GP_PIN(5,  0)] = { PU4, 12 },	/* SCK0 */
-	[RCAR_GP_PIN(3, 15)] = { PU4, 11 },	/* SD1_WP */
-	[RCAR_GP_PIN(3, 14)] = { PU4, 10 },	/* SD1_CD */
-	[RCAR_GP_PIN(3, 13)] = { PU4,  9 },	/* SD0_WP */
-	[RCAR_GP_PIN(3, 12)] = { PU4,  8 },	/* SD0_CD */
-	[RCAR_GP_PIN(4, 17)] = { PU4,  7 },	/* SD3_DS */
-	[RCAR_GP_PIN(4, 16)] = { PU4,  6 },	/* SD3_DAT7 */
-	[RCAR_GP_PIN(4, 15)] = { PU4,  5 },	/* SD3_DAT6 */
-	[RCAR_GP_PIN(4, 14)] = { PU4,  4 },	/* SD3_DAT5 */
-	[RCAR_GP_PIN(4, 13)] = { PU4,  3 },	/* SD3_DAT4 */
-	[RCAR_GP_PIN(4, 12)] = { PU4,  2 },	/* SD3_DAT3 */
-	[RCAR_GP_PIN(4, 11)] = { PU4,  1 },	/* SD3_DAT2 */
-	[RCAR_GP_PIN(4, 10)] = { PU4,  0 },	/* SD3_DAT1 */
-
-	[RCAR_GP_PIN(6, 24)] = { PU5, 31 },	/* USB0_PWEN */
-	[RCAR_GP_PIN(6, 23)] = { PU5, 30 },	/* AUDIO_CLKB_B */
-	[RCAR_GP_PIN(6, 22)] = { PU5, 29 },	/* AUDIO_CLKA_A */
-	[RCAR_GP_PIN(6, 21)] = { PU5, 28 },	/* SSI_SDATA9_A */
-	[RCAR_GP_PIN(6, 20)] = { PU5, 27 },	/* SSI_SDATA8 */
-	[RCAR_GP_PIN(6, 19)] = { PU5, 26 },	/* SSI_SDATA7 */
-	[RCAR_GP_PIN(6, 18)] = { PU5, 25 },	/* SSI_WS78 */
-	[RCAR_GP_PIN(6, 17)] = { PU5, 24 },	/* SSI_SCK78 */
-	[RCAR_GP_PIN(6, 16)] = { PU5, 23 },	/* SSI_SDATA6 */
-	[RCAR_GP_PIN(6, 15)] = { PU5, 22 },	/* SSI_WS6 */
-	[RCAR_GP_PIN(6, 14)] = { PU5, 21 },	/* SSI_SCK6 */
-	[RCAR_GP_PIN(6, 13)] = { PU5, 20 },	/* SSI_SDATA5 */
-	[RCAR_GP_PIN(6, 12)] = { PU5, 19 },	/* SSI_WS5 */
-	[RCAR_GP_PIN(6, 11)] = { PU5, 18 },	/* SSI_SCK5 */
-	[RCAR_GP_PIN(6, 10)] = { PU5, 17 },	/* SSI_SDATA4 */
-	[RCAR_GP_PIN(6,  9)] = { PU5, 16 },	/* SSI_WS4 */
-	[RCAR_GP_PIN(6,  8)] = { PU5, 15 },	/* SSI_SCK4 */
-	[RCAR_GP_PIN(6,  7)] = { PU5, 14 },	/* SSI_SDATA3 */
-	[RCAR_GP_PIN(6,  6)] = { PU5, 13 },	/* SSI_WS34 */
-	[RCAR_GP_PIN(6,  5)] = { PU5, 12 },	/* SSI_SCK34 */
-	[RCAR_GP_PIN(6,  4)] = { PU5, 11 },	/* SSI_SDATA2_A */
-	[RCAR_GP_PIN(6,  3)] = { PU5, 10 },	/* SSI_SDATA1_A */
-	[RCAR_GP_PIN(6,  2)] = { PU5,  9 },	/* SSI_SDATA0 */
-	[RCAR_GP_PIN(6,  1)] = { PU5,  8 },	/* SSI_WS01239 */
-	[RCAR_GP_PIN(6,  0)] = { PU5,  7 },	/* SSI_SCK01239 */
-	[RCAR_GP_PIN(5, 25)] = { PU5,  5 },	/* MLB_DAT */
-	[RCAR_GP_PIN(5, 24)] = { PU5,  4 },	/* MLB_SIG */
-	[RCAR_GP_PIN(5, 23)] = { PU5,  3 },	/* MLB_CLK */
-	[RCAR_GP_PIN(5, 22)] = { PU5,  2 },	/* MSIOF0_RXD */
-	[RCAR_GP_PIN(5, 21)] = { PU5,  1 },	/* MSIOF0_SS2 */
-	[RCAR_GP_PIN(5, 20)] = { PU5,  0 },	/* MSIOF0_TXD */
-
-	[RCAR_GP_PIN(6, 31)] = { PU6,  6 },	/* USB31_OVC */
-	[RCAR_GP_PIN(6, 30)] = { PU6,  5 },	/* USB31_PWEN */
-	[RCAR_GP_PIN(6, 29)] = { PU6,  4 },	/* USB30_OVC */
-	[RCAR_GP_PIN(6, 28)] = { PU6,  3 },	/* USB30_PWEN */
-	[RCAR_GP_PIN(6, 27)] = { PU6,  2 },	/* USB1_OVC */
-	[RCAR_GP_PIN(6, 26)] = { PU6,  1 },	/* USB1_PWEN */
-	[RCAR_GP_PIN(6, 25)] = { PU6,  0 },	/* USB0_OVC */
+static const struct sh_pfc_bias_info bias_info[] = {
+	{ RCAR_GP_PIN(2, 11), PU0, 31 },	/* AVB_PHY_INT */
+	{ RCAR_GP_PIN(2, 10), PU0, 30 },	/* AVB_MAGIC */
+	{ RCAR_GP_PIN(2,  9), PU0, 29 },	/* AVB_MDC */
+
+	{ RCAR_GP_PIN(1, 19), PU1, 31 },	/* A19 */
+	{ RCAR_GP_PIN(1, 18), PU1, 30 },	/* A18 */
+	{ RCAR_GP_PIN(1, 17), PU1, 29 },	/* A17 */
+	{ RCAR_GP_PIN(1, 16), PU1, 28 },	/* A16 */
+	{ RCAR_GP_PIN(1, 15), PU1, 27 },	/* A15 */
+	{ RCAR_GP_PIN(1, 14), PU1, 26 },	/* A14 */
+	{ RCAR_GP_PIN(1, 13), PU1, 25 },	/* A13 */
+	{ RCAR_GP_PIN(1, 12), PU1, 24 },	/* A12 */
+	{ RCAR_GP_PIN(1, 11), PU1, 23 },	/* A11 */
+	{ RCAR_GP_PIN(1, 10), PU1, 22 },	/* A10 */
+	{ RCAR_GP_PIN(1,  9), PU1, 21 },	/* A9 */
+	{ RCAR_GP_PIN(1,  8), PU1, 20 },	/* A8 */
+	{ RCAR_GP_PIN(1,  7), PU1, 19 },	/* A7 */
+	{ RCAR_GP_PIN(1,  6), PU1, 18 },	/* A6 */
+	{ RCAR_GP_PIN(1,  5), PU1, 17 },	/* A5 */
+	{ RCAR_GP_PIN(1,  4), PU1, 16 },	/* A4 */
+	{ RCAR_GP_PIN(1,  3), PU1, 15 },	/* A3 */
+	{ RCAR_GP_PIN(1,  2), PU1, 14 },	/* A2 */
+	{ RCAR_GP_PIN(1,  1), PU1, 13 },	/* A1 */
+	{ RCAR_GP_PIN(1,  0), PU1, 12 },	/* A0 */
+	{ RCAR_GP_PIN(2,  8), PU1, 11 },	/* PWM2_A */
+	{ RCAR_GP_PIN(2,  7), PU1, 10 },	/* PWM1_A */
+	{ RCAR_GP_PIN(2,  6), PU1,  9 },	/* PWM0 */
+	{ RCAR_GP_PIN(2,  5), PU1,  8 },	/* IRQ5 */
+	{ RCAR_GP_PIN(2,  4), PU1,  7 },	/* IRQ4 */
+	{ RCAR_GP_PIN(2,  3), PU1,  6 },	/* IRQ3 */
+	{ RCAR_GP_PIN(2,  2), PU1,  5 },	/* IRQ2 */
+	{ RCAR_GP_PIN(2,  1), PU1,  4 },	/* IRQ1 */
+	{ RCAR_GP_PIN(2,  0), PU1,  3 },	/* IRQ0 */
+	{ RCAR_GP_PIN(2, 14), PU1,  2 },	/* AVB_AVTP_CAPTURE_A */
+	{ RCAR_GP_PIN(2, 13), PU1,  1 },	/* AVB_AVTP_MATCH_A */
+	{ RCAR_GP_PIN(2, 12), PU1,  0 },	/* AVB_LINK */
+
+	{ RCAR_GP_PIN(7,  3), PU2, 29 },	/* HDMI1_CEC */
+	{ RCAR_GP_PIN(7,  2), PU2, 28 },	/* HDMI0_CEC */
+	{ RCAR_GP_PIN(7,  1), PU2, 27 },	/* AVS2 */
+	{ RCAR_GP_PIN(7,  0), PU2, 26 },	/* AVS1 */
+	{ RCAR_GP_PIN(0, 15), PU2, 25 },	/* D15 */
+	{ RCAR_GP_PIN(0, 14), PU2, 24 },	/* D14 */
+	{ RCAR_GP_PIN(0, 13), PU2, 23 },	/* D13 */
+	{ RCAR_GP_PIN(0, 12), PU2, 22 },	/* D12 */
+	{ RCAR_GP_PIN(0, 11), PU2, 21 },	/* D11 */
+	{ RCAR_GP_PIN(0, 10), PU2, 20 },	/* D10 */
+	{ RCAR_GP_PIN(0,  9), PU2, 19 },	/* D9 */
+	{ RCAR_GP_PIN(0,  8), PU2, 18 },	/* D8 */
+	{ RCAR_GP_PIN(0,  7), PU2, 17 },	/* D7 */
+	{ RCAR_GP_PIN(0,  6), PU2, 16 },	/* D6 */
+	{ RCAR_GP_PIN(0,  5), PU2, 15 },	/* D5 */
+	{ RCAR_GP_PIN(0,  4), PU2, 14 },	/* D4 */
+	{ RCAR_GP_PIN(0,  3), PU2, 13 },	/* D3 */
+	{ RCAR_GP_PIN(0,  2), PU2, 12 },	/* D2 */
+	{ RCAR_GP_PIN(0,  1), PU2, 11 },	/* D1 */
+	{ RCAR_GP_PIN(0,  0), PU2, 10 },	/* D0 */
+	{ RCAR_GP_PIN(1, 27), PU2,  8 },	/* EX_WAIT0_A */
+	{ RCAR_GP_PIN(1, 26), PU2,  7 },	/* WE1_N */
+	{ RCAR_GP_PIN(1, 25), PU2,  6 },	/* WE0_N */
+	{ RCAR_GP_PIN(1, 24), PU2,  5 },	/* RD_WR_N */
+	{ RCAR_GP_PIN(1, 23), PU2,  4 },	/* RD_N */
+	{ RCAR_GP_PIN(1, 22), PU2,  3 },	/* BS_N */
+	{ RCAR_GP_PIN(1, 21), PU2,  2 },	/* CS1_N_A26 */
+	{ RCAR_GP_PIN(1, 20), PU2,  1 },	/* CS0_N */
+
+	{ RCAR_GP_PIN(4,  9), PU3, 31 },	/* SD3_DAT0 */
+	{ RCAR_GP_PIN(4,  8), PU3, 30 },	/* SD3_CMD */
+	{ RCAR_GP_PIN(4,  7), PU3, 29 },	/* SD3_CLK */
+	{ RCAR_GP_PIN(4,  6), PU3, 28 },	/* SD2_DS */
+	{ RCAR_GP_PIN(4,  5), PU3, 27 },	/* SD2_DAT3 */
+	{ RCAR_GP_PIN(4,  4), PU3, 26 },	/* SD2_DAT2 */
+	{ RCAR_GP_PIN(4,  3), PU3, 25 },	/* SD2_DAT1 */
+	{ RCAR_GP_PIN(4,  2), PU3, 24 },	/* SD2_DAT0 */
+	{ RCAR_GP_PIN(4,  1), PU3, 23 },	/* SD2_CMD */
+	{ RCAR_GP_PIN(4,  0), PU3, 22 },	/* SD2_CLK */
+	{ RCAR_GP_PIN(3, 11), PU3, 21 },	/* SD1_DAT3 */
+	{ RCAR_GP_PIN(3, 10), PU3, 20 },	/* SD1_DAT2 */
+	{ RCAR_GP_PIN(3,  9), PU3, 19 },	/* SD1_DAT1 */
+	{ RCAR_GP_PIN(3,  8), PU3, 18 },	/* SD1_DAT0 */
+	{ RCAR_GP_PIN(3,  7), PU3, 17 },	/* SD1_CMD */
+	{ RCAR_GP_PIN(3,  6), PU3, 16 },	/* SD1_CLK */
+	{ RCAR_GP_PIN(3,  5), PU3, 15 },	/* SD0_DAT3 */
+	{ RCAR_GP_PIN(3,  4), PU3, 14 },	/* SD0_DAT2 */
+	{ RCAR_GP_PIN(3,  3), PU3, 13 },	/* SD0_DAT1 */
+	{ RCAR_GP_PIN(3,  2), PU3, 12 },	/* SD0_DAT0 */
+	{ RCAR_GP_PIN(3,  1), PU3, 11 },	/* SD0_CMD */
+	{ RCAR_GP_PIN(3,  0), PU3, 10 },	/* SD0_CLK */
+
+	{ RCAR_GP_PIN(5, 19), PU4, 31 },	/* MSIOF0_SS1 */
+	{ RCAR_GP_PIN(5, 18), PU4, 30 },	/* MSIOF0_SYNC */
+	{ RCAR_GP_PIN(5, 17), PU4, 29 },	/* MSIOF0_SCK */
+	{ RCAR_GP_PIN(5, 16), PU4, 28 },	/* HRTS0_N */
+	{ RCAR_GP_PIN(5, 15), PU4, 27 },	/* HCTS0_N */
+	{ RCAR_GP_PIN(5, 14), PU4, 26 },	/* HTX0 */
+	{ RCAR_GP_PIN(5, 13), PU4, 25 },	/* HRX0 */
+	{ RCAR_GP_PIN(5, 12), PU4, 24 },	/* HSCK0 */
+	{ RCAR_GP_PIN(5, 11), PU4, 23 },	/* RX2_A */
+	{ RCAR_GP_PIN(5, 10), PU4, 22 },	/* TX2_A */
+	{ RCAR_GP_PIN(5,  9), PU4, 21 },	/* SCK2 */
+	{ RCAR_GP_PIN(5,  8), PU4, 20 },	/* RTS1_N_TANS */
+	{ RCAR_GP_PIN(5,  7), PU4, 19 },	/* CTS1_N */
+	{ RCAR_GP_PIN(5,  6), PU4, 18 },	/* TX1_A */
+	{ RCAR_GP_PIN(5,  5), PU4, 17 },	/* RX1_A */
+	{ RCAR_GP_PIN(5,  4), PU4, 16 },	/* RTS0_N_TANS */
+	{ RCAR_GP_PIN(5,  3), PU4, 15 },	/* CTS0_N */
+	{ RCAR_GP_PIN(5,  2), PU4, 14 },	/* TX0 */
+	{ RCAR_GP_PIN(5,  1), PU4, 13 },	/* RX0 */
+	{ RCAR_GP_PIN(5,  0), PU4, 12 },	/* SCK0 */
+	{ RCAR_GP_PIN(3, 15), PU4, 11 },	/* SD1_WP */
+	{ RCAR_GP_PIN(3, 14), PU4, 10 },	/* SD1_CD */
+	{ RCAR_GP_PIN(3, 13), PU4,  9 },	/* SD0_WP */
+	{ RCAR_GP_PIN(3, 12), PU4,  8 },	/* SD0_CD */
+	{ RCAR_GP_PIN(4, 17), PU4,  7 },	/* SD3_DS */
+	{ RCAR_GP_PIN(4, 16), PU4,  6 },	/* SD3_DAT7 */
+	{ RCAR_GP_PIN(4, 15), PU4,  5 },	/* SD3_DAT6 */
+	{ RCAR_GP_PIN(4, 14), PU4,  4 },	/* SD3_DAT5 */
+	{ RCAR_GP_PIN(4, 13), PU4,  3 },	/* SD3_DAT4 */
+	{ RCAR_GP_PIN(4, 12), PU4,  2 },	/* SD3_DAT3 */
+	{ RCAR_GP_PIN(4, 11), PU4,  1 },	/* SD3_DAT2 */
+	{ RCAR_GP_PIN(4, 10), PU4,  0 },	/* SD3_DAT1 */
+
+	{ RCAR_GP_PIN(6, 24), PU5, 31 },	/* USB0_PWEN */
+	{ RCAR_GP_PIN(6, 23), PU5, 30 },	/* AUDIO_CLKB_B */
+	{ RCAR_GP_PIN(6, 22), PU5, 29 },	/* AUDIO_CLKA_A */
+	{ RCAR_GP_PIN(6, 21), PU5, 28 },	/* SSI_SDATA9_A */
+	{ RCAR_GP_PIN(6, 20), PU5, 27 },	/* SSI_SDATA8 */
+	{ RCAR_GP_PIN(6, 19), PU5, 26 },	/* SSI_SDATA7 */
+	{ RCAR_GP_PIN(6, 18), PU5, 25 },	/* SSI_WS78 */
+	{ RCAR_GP_PIN(6, 17), PU5, 24 },	/* SSI_SCK78 */
+	{ RCAR_GP_PIN(6, 16), PU5, 23 },	/* SSI_SDATA6 */
+	{ RCAR_GP_PIN(6, 15), PU5, 22 },	/* SSI_WS6 */
+	{ RCAR_GP_PIN(6, 14), PU5, 21 },	/* SSI_SCK6 */
+	{ RCAR_GP_PIN(6, 13), PU5, 20 },	/* SSI_SDATA5 */
+	{ RCAR_GP_PIN(6, 12), PU5, 19 },	/* SSI_WS5 */
+	{ RCAR_GP_PIN(6, 11), PU5, 18 },	/* SSI_SCK5 */
+	{ RCAR_GP_PIN(6, 10), PU5, 17 },	/* SSI_SDATA4 */
+	{ RCAR_GP_PIN(6,  9), PU5, 16 },	/* SSI_WS4 */
+	{ RCAR_GP_PIN(6,  8), PU5, 15 },	/* SSI_SCK4 */
+	{ RCAR_GP_PIN(6,  7), PU5, 14 },	/* SSI_SDATA3 */
+	{ RCAR_GP_PIN(6,  6), PU5, 13 },	/* SSI_WS34 */
+	{ RCAR_GP_PIN(6,  5), PU5, 12 },	/* SSI_SCK34 */
+	{ RCAR_GP_PIN(6,  4), PU5, 11 },	/* SSI_SDATA2_A */
+	{ RCAR_GP_PIN(6,  3), PU5, 10 },	/* SSI_SDATA1_A */
+	{ RCAR_GP_PIN(6,  2), PU5,  9 },	/* SSI_SDATA0 */
+	{ RCAR_GP_PIN(6,  1), PU5,  8 },	/* SSI_WS01239 */
+	{ RCAR_GP_PIN(6,  0), PU5,  7 },	/* SSI_SCK01239 */
+	{ RCAR_GP_PIN(5, 25), PU5,  5 },	/* MLB_DAT */
+	{ RCAR_GP_PIN(5, 24), PU5,  4 },	/* MLB_SIG */
+	{ RCAR_GP_PIN(5, 23), PU5,  3 },	/* MLB_CLK */
+	{ RCAR_GP_PIN(5, 22), PU5,  2 },	/* MSIOF0_RXD */
+	{ RCAR_GP_PIN(5, 21), PU5,  1 },	/* MSIOF0_SS2 */
+	{ RCAR_GP_PIN(5, 20), PU5,  0 },	/* MSIOF0_TXD */
+
+	{ RCAR_GP_PIN(6, 31), PU6,  6 },	/* USB31_OVC */
+	{ RCAR_GP_PIN(6, 30), PU6,  5 },	/* USB31_PWEN */
+	{ RCAR_GP_PIN(6, 29), PU6,  4 },	/* USB30_OVC */
+	{ RCAR_GP_PIN(6, 28), PU6,  3 },	/* USB30_PWEN */
+	{ RCAR_GP_PIN(6, 27), PU6,  2 },	/* USB1_OVC */
+	{ RCAR_GP_PIN(6, 26), PU6,  1 },	/* USB1_PWEN */
+	{ RCAR_GP_PIN(6, 25), PU6,  0 },	/* USB0_OVC */
 };
 
 static unsigned int r8a7795_pinmux_get_bias(struct sh_pfc *pfc,
 					    unsigned int pin)
 {
+	const struct sh_pfc_bias_info *info;
 	u32 reg;
 	u32 bit;
 
-	if (WARN_ON_ONCE(!pullups[pin].reg))
+	info = sh_pfc_pin_to_bias_info(bias_info, ARRAY_SIZE(bias_info), pin);
+	if (!info)
 		return PIN_CONFIG_BIAS_DISABLE;
 
-	reg = pullups[pin].reg;
-	bit = BIT(pullups[pin].bit);
+	reg = info->reg;
+	bit = BIT(info->bit);
 
-	if (sh_pfc_read_reg(pfc, PUEN + reg, 32) & bit) {
-		if (sh_pfc_read_reg(pfc, PUD + reg, 32) & bit)
-			return PIN_CONFIG_BIAS_PULL_UP;
-		else
-			return PIN_CONFIG_BIAS_PULL_DOWN;
-	} else
+	if (!(sh_pfc_read_reg(pfc, PUEN + reg, 32) & bit))
 		return PIN_CONFIG_BIAS_DISABLE;
+	else if (sh_pfc_read_reg(pfc, PUD + reg, 32) & bit)
+		return PIN_CONFIG_BIAS_PULL_UP;
+	else
+		return PIN_CONFIG_BIAS_PULL_DOWN;
 }
 
 static void r8a7795_pinmux_set_bias(struct sh_pfc *pfc, unsigned int pin,
 				   unsigned int bias)
 {
+	const struct sh_pfc_bias_info *info;
 	u32 enable, updown;
 	u32 reg;
 	u32 bit;
 
-	if (WARN_ON_ONCE(!pullups[pin].reg))
+	info = sh_pfc_pin_to_bias_info(bias_info, ARRAY_SIZE(bias_info), pin);
+	if (!info)
 		return;
 
-	reg = pullups[pin].reg;
-	bit = BIT(pullups[pin].bit);
+	reg = info->reg;
+	bit = BIT(info->bit);
 
 	enable = sh_pfc_read_reg(pfc, PUEN + reg, 32) & ~bit;
 	if (bias != PIN_CONFIG_BIAS_DISABLE)
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7796.c b/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
index dc9b671ccf2e..7e16545a2c3c 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
@@ -122,22 +122,22 @@
 #define GPSR3_0		F_(SD0_CLK,		IP7_19_16)
 
 /* GPSR4 */
-#define GPSR4_17	F_(SD3_DS,		IP11_11_8)
-#define GPSR4_16	F_(SD3_DAT7,		IP10_7_4)
-#define GPSR4_15	F_(SD3_DAT6,		IP10_3_0)
-#define GPSR4_14	F_(SD3_DAT5,		IP9_31_28)
-#define GPSR4_13	F_(SD3_DAT4,		IP9_27_24)
+#define GPSR4_17	F_(SD3_DS,		IP11_7_4)
+#define GPSR4_16	F_(SD3_DAT7,		IP11_3_0)
+#define GPSR4_15	F_(SD3_DAT6,		IP10_31_28)
+#define GPSR4_14	F_(SD3_DAT5,		IP10_27_24)
+#define GPSR4_13	F_(SD3_DAT4,		IP10_23_20)
 #define GPSR4_12	F_(SD3_DAT3,		IP10_19_16)
 #define GPSR4_11	F_(SD3_DAT2,		IP10_15_12)
 #define GPSR4_10	F_(SD3_DAT1,		IP10_11_8)
 #define GPSR4_9		F_(SD3_DAT0,		IP10_7_4)
 #define GPSR4_8		F_(SD3_CMD,		IP10_3_0)
 #define GPSR4_7		F_(SD3_CLK,		IP9_31_28)
-#define GPSR4_6		F_(SD2_DS,		IP9_23_20)
-#define GPSR4_5		F_(SD2_DAT3,		IP9_19_16)
-#define GPSR4_4		F_(SD2_DAT2,		IP9_15_12)
-#define GPSR4_3		F_(SD2_DAT1,		IP9_11_8)
-#define GPSR4_2		F_(SD2_DAT0,		IP9_7_4)
+#define GPSR4_6		F_(SD2_DS,		IP9_27_24)
+#define GPSR4_5		F_(SD2_DAT3,		IP9_23_20)
+#define GPSR4_4		F_(SD2_DAT2,		IP9_19_16)
+#define GPSR4_3		F_(SD2_DAT1,		IP9_15_12)
+#define GPSR4_2		F_(SD2_DAT0,		IP9_11_8)
 #define GPSR4_1		F_(SD2_CMD,		IP9_7_4)
 #define GPSR4_0		F_(SD2_CLK,		IP9_3_0)
 
@@ -1490,6 +1490,418 @@ static const struct sh_pfc_pin pinmux_pins[] = {
 	PINMUX_GPIO_GP_ALL(),
 };
 
+/* - EtherAVB --------------------------------------------------------------- */
+static const unsigned int avb_link_pins[] = {
+	/* AVB_LINK */
+	RCAR_GP_PIN(2, 12),
+};
+static const unsigned int avb_link_mux[] = {
+	AVB_LINK_MARK,
+};
+static const unsigned int avb_magic_pins[] = {
+	/* AVB_MAGIC_ */
+	RCAR_GP_PIN(2, 10),
+};
+static const unsigned int avb_magic_mux[] = {
+	AVB_MAGIC_MARK,
+};
+static const unsigned int avb_phy_int_pins[] = {
+	/* AVB_PHY_INT */
+	RCAR_GP_PIN(2, 11),
+};
+static const unsigned int avb_phy_int_mux[] = {
+	AVB_PHY_INT_MARK,
+};
+static const unsigned int avb_mdc_pins[] = {
+	/* AVB_MDC */
+	RCAR_GP_PIN(2, 9),
+};
+static const unsigned int avb_mdc_mux[] = {
+	AVB_MDC_MARK,
+};
+static const unsigned int avb_avtp_pps_pins[] = {
+	/* AVB_AVTP_PPS */
+	RCAR_GP_PIN(2, 6),
+};
+static const unsigned int avb_avtp_pps_mux[] = {
+	AVB_AVTP_PPS_MARK,
+};
+static const unsigned int avb_avtp_match_a_pins[] = {
+	/* AVB_AVTP_MATCH_A */
+	RCAR_GP_PIN(2, 13),
+};
+static const unsigned int avb_avtp_match_a_mux[] = {
+	AVB_AVTP_MATCH_A_MARK,
+};
+static const unsigned int avb_avtp_capture_a_pins[] = {
+	/* AVB_AVTP_CAPTURE_A */
+	RCAR_GP_PIN(2, 14),
+};
+static const unsigned int avb_avtp_capture_a_mux[] = {
+	AVB_AVTP_CAPTURE_A_MARK,
+};
+static const unsigned int avb_avtp_match_b_pins[] = {
+	/*  AVB_AVTP_MATCH_B */
+	RCAR_GP_PIN(1, 8),
+};
+static const unsigned int avb_avtp_match_b_mux[] = {
+	AVB_AVTP_MATCH_B_MARK,
+};
+static const unsigned int avb_avtp_capture_b_pins[] = {
+	/* AVB_AVTP_CAPTURE_B */
+	RCAR_GP_PIN(1, 11),
+};
+static const unsigned int avb_avtp_capture_b_mux[] = {
+	AVB_AVTP_CAPTURE_B_MARK,
+};
+
+/* - DRIF0 --------------------------------------------------------------- */
+static const unsigned int drif0_ctrl_a_pins[] = {
+	/* CLK, SYNC */
+	RCAR_GP_PIN(6, 8), RCAR_GP_PIN(6, 9),
+};
+static const unsigned int drif0_ctrl_a_mux[] = {
+	RIF0_CLK_A_MARK, RIF0_SYNC_A_MARK,
+};
+static const unsigned int drif0_data0_a_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(6, 10),
+};
+static const unsigned int drif0_data0_a_mux[] = {
+	RIF0_D0_A_MARK,
+};
+static const unsigned int drif0_data1_a_pins[] = {
+	/* D1 */
+	RCAR_GP_PIN(6, 7),
+};
+static const unsigned int drif0_data1_a_mux[] = {
+	RIF0_D1_A_MARK,
+};
+static const unsigned int drif0_ctrl_b_pins[] = {
+	/* CLK, SYNC */
+	RCAR_GP_PIN(5, 0), RCAR_GP_PIN(5, 4),
+};
+static const unsigned int drif0_ctrl_b_mux[] = {
+	RIF0_CLK_B_MARK, RIF0_SYNC_B_MARK,
+};
+static const unsigned int drif0_data0_b_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(5, 1),
+};
+static const unsigned int drif0_data0_b_mux[] = {
+	RIF0_D0_B_MARK,
+};
+static const unsigned int drif0_data1_b_pins[] = {
+	/* D1 */
+	RCAR_GP_PIN(5, 2),
+};
+static const unsigned int drif0_data1_b_mux[] = {
+	RIF0_D1_B_MARK,
+};
+static const unsigned int drif0_ctrl_c_pins[] = {
+	/* CLK, SYNC */
+	RCAR_GP_PIN(5, 12), RCAR_GP_PIN(5, 15),
+};
+static const unsigned int drif0_ctrl_c_mux[] = {
+	RIF0_CLK_C_MARK, RIF0_SYNC_C_MARK,
+};
+static const unsigned int drif0_data0_c_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(5, 13),
+};
+static const unsigned int drif0_data0_c_mux[] = {
+	RIF0_D0_C_MARK,
+};
+static const unsigned int drif0_data1_c_pins[] = {
+	/* D1 */
+	RCAR_GP_PIN(5, 14),
+};
+static const unsigned int drif0_data1_c_mux[] = {
+	RIF0_D1_C_MARK,
+};
+/* - DRIF1 --------------------------------------------------------------- */
+static const unsigned int drif1_ctrl_a_pins[] = {
+	/* CLK, SYNC */
+	RCAR_GP_PIN(6, 17), RCAR_GP_PIN(6, 18),
+};
+static const unsigned int drif1_ctrl_a_mux[] = {
+	RIF1_CLK_A_MARK, RIF1_SYNC_A_MARK,
+};
+static const unsigned int drif1_data0_a_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(6, 19),
+};
+static const unsigned int drif1_data0_a_mux[] = {
+	RIF1_D0_A_MARK,
+};
+static const unsigned int drif1_data1_a_pins[] = {
+	/* D1 */
+	RCAR_GP_PIN(6, 20),
+};
+static const unsigned int drif1_data1_a_mux[] = {
+	RIF1_D1_A_MARK,
+};
+static const unsigned int drif1_ctrl_b_pins[] = {
+	/* CLK, SYNC */
+	RCAR_GP_PIN(5, 9), RCAR_GP_PIN(5, 3),
+};
+static const unsigned int drif1_ctrl_b_mux[] = {
+	RIF1_CLK_B_MARK, RIF1_SYNC_B_MARK,
+};
+static const unsigned int drif1_data0_b_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(5, 7),
+};
+static const unsigned int drif1_data0_b_mux[] = {
+	RIF1_D0_B_MARK,
+};
+static const unsigned int drif1_data1_b_pins[] = {
+	/* D1 */
+	RCAR_GP_PIN(5, 8),
+};
+static const unsigned int drif1_data1_b_mux[] = {
+	RIF1_D1_B_MARK,
+};
+static const unsigned int drif1_ctrl_c_pins[] = {
+	/* CLK, SYNC */
+	RCAR_GP_PIN(5, 5), RCAR_GP_PIN(5, 11),
+};
+static const unsigned int drif1_ctrl_c_mux[] = {
+	RIF1_CLK_C_MARK, RIF1_SYNC_C_MARK,
+};
+static const unsigned int drif1_data0_c_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(5, 6),
+};
+static const unsigned int drif1_data0_c_mux[] = {
+	RIF1_D0_C_MARK,
+};
+static const unsigned int drif1_data1_c_pins[] = {
+	/* D1 */
+	RCAR_GP_PIN(5, 10),
+};
+static const unsigned int drif1_data1_c_mux[] = {
+	RIF1_D1_C_MARK,
+};
+/* - DRIF2 --------------------------------------------------------------- */
+static const unsigned int drif2_ctrl_a_pins[] = {
+	/* CLK, SYNC */
+	RCAR_GP_PIN(6, 8), RCAR_GP_PIN(6, 9),
+};
+static const unsigned int drif2_ctrl_a_mux[] = {
+	RIF2_CLK_A_MARK, RIF2_SYNC_A_MARK,
+};
+static const unsigned int drif2_data0_a_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(6, 7),
+};
+static const unsigned int drif2_data0_a_mux[] = {
+	RIF2_D0_A_MARK,
+};
+static const unsigned int drif2_data1_a_pins[] = {
+	/* D1 */
+	RCAR_GP_PIN(6, 10),
+};
+static const unsigned int drif2_data1_a_mux[] = {
+	RIF2_D1_A_MARK,
+};
+static const unsigned int drif2_ctrl_b_pins[] = {
+	/* CLK, SYNC */
+	RCAR_GP_PIN(6, 26), RCAR_GP_PIN(6, 27),
+};
+static const unsigned int drif2_ctrl_b_mux[] = {
+	RIF2_CLK_B_MARK, RIF2_SYNC_B_MARK,
+};
+static const unsigned int drif2_data0_b_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(6, 30),
+};
+static const unsigned int drif2_data0_b_mux[] = {
+	RIF2_D0_B_MARK,
+};
+static const unsigned int drif2_data1_b_pins[] = {
+	/* D1 */
+	RCAR_GP_PIN(6, 31),
+};
+static const unsigned int drif2_data1_b_mux[] = {
+	RIF2_D1_B_MARK,
+};
+/* - DRIF3 --------------------------------------------------------------- */
+static const unsigned int drif3_ctrl_a_pins[] = {
+	/* CLK, SYNC */
+	RCAR_GP_PIN(6, 17), RCAR_GP_PIN(6, 18),
+};
+static const unsigned int drif3_ctrl_a_mux[] = {
+	RIF3_CLK_A_MARK, RIF3_SYNC_A_MARK,
+};
+static const unsigned int drif3_data0_a_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(6, 19),
+};
+static const unsigned int drif3_data0_a_mux[] = {
+	RIF3_D0_A_MARK,
+};
+static const unsigned int drif3_data1_a_pins[] = {
+	/* D1 */
+	RCAR_GP_PIN(6, 20),
+};
+static const unsigned int drif3_data1_a_mux[] = {
+	RIF3_D1_A_MARK,
+};
+static const unsigned int drif3_ctrl_b_pins[] = {
+	/* CLK, SYNC */
+	RCAR_GP_PIN(6, 24), RCAR_GP_PIN(6, 25),
+};
+static const unsigned int drif3_ctrl_b_mux[] = {
+	RIF3_CLK_B_MARK, RIF3_SYNC_B_MARK,
+};
+static const unsigned int drif3_data0_b_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(6, 28),
+};
+static const unsigned int drif3_data0_b_mux[] = {
+	RIF3_D0_B_MARK,
+};
+static const unsigned int drif3_data1_b_pins[] = {
+	/* D1 */
+	RCAR_GP_PIN(6, 29),
+};
+static const unsigned int drif3_data1_b_mux[] = {
+	RIF3_D1_B_MARK,
+};
+
+/* - DU --------------------------------------------------------------------- */
+static const unsigned int du_rgb666_pins[] = {
+	/* R[7:2], G[7:2], B[7:2] */
+	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 14), RCAR_GP_PIN(0, 13),
+	RCAR_GP_PIN(0, 12), RCAR_GP_PIN(0, 11), RCAR_GP_PIN(0, 10),
+	RCAR_GP_PIN(1, 15), RCAR_GP_PIN(1, 14), RCAR_GP_PIN(1, 13),
+	RCAR_GP_PIN(1, 12), RCAR_GP_PIN(1, 19), RCAR_GP_PIN(1, 18),
+	RCAR_GP_PIN(1, 7),  RCAR_GP_PIN(1, 6),  RCAR_GP_PIN(1, 5),
+	RCAR_GP_PIN(1, 4),  RCAR_GP_PIN(1, 3),  RCAR_GP_PIN(1, 2),
+};
+static const unsigned int du_rgb666_mux[] = {
+	DU_DR7_MARK, DU_DR6_MARK, DU_DR5_MARK, DU_DR4_MARK,
+	DU_DR3_MARK, DU_DR2_MARK,
+	DU_DG7_MARK, DU_DG6_MARK, DU_DG5_MARK, DU_DG4_MARK,
+	DU_DG3_MARK, DU_DG2_MARK,
+	DU_DB7_MARK, DU_DB6_MARK, DU_DB5_MARK, DU_DB4_MARK,
+	DU_DB3_MARK, DU_DB2_MARK,
+};
+static const unsigned int du_rgb888_pins[] = {
+	/* R[7:0], G[7:0], B[7:0] */
+	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 14), RCAR_GP_PIN(0, 13),
+	RCAR_GP_PIN(0, 12), RCAR_GP_PIN(0, 11), RCAR_GP_PIN(0, 10),
+	RCAR_GP_PIN(0, 9),  RCAR_GP_PIN(0, 8),
+	RCAR_GP_PIN(1, 15), RCAR_GP_PIN(1, 14), RCAR_GP_PIN(1, 13),
+	RCAR_GP_PIN(1, 12), RCAR_GP_PIN(1, 19), RCAR_GP_PIN(1, 18),
+	RCAR_GP_PIN(1, 17), RCAR_GP_PIN(1, 16),
+	RCAR_GP_PIN(1, 7),  RCAR_GP_PIN(1, 6),  RCAR_GP_PIN(1, 5),
+	RCAR_GP_PIN(1, 4),  RCAR_GP_PIN(1, 3),  RCAR_GP_PIN(1, 2),
+	RCAR_GP_PIN(1, 1),  RCAR_GP_PIN(1, 0),
+};
+static const unsigned int du_rgb888_mux[] = {
+	DU_DR7_MARK, DU_DR6_MARK, DU_DR5_MARK, DU_DR4_MARK,
+	DU_DR3_MARK, DU_DR2_MARK, DU_DR1_MARK, DU_DR0_MARK,
+	DU_DG7_MARK, DU_DG6_MARK, DU_DG5_MARK, DU_DG4_MARK,
+	DU_DG3_MARK, DU_DG2_MARK, DU_DG1_MARK, DU_DG0_MARK,
+	DU_DB7_MARK, DU_DB6_MARK, DU_DB5_MARK, DU_DB4_MARK,
+	DU_DB3_MARK, DU_DB2_MARK, DU_DB1_MARK, DU_DB0_MARK,
+};
+static const unsigned int du_clk_out_0_pins[] = {
+	/* CLKOUT */
+	RCAR_GP_PIN(1, 27),
+};
+static const unsigned int du_clk_out_0_mux[] = {
+	DU_DOTCLKOUT0_MARK
+};
+static const unsigned int du_clk_out_1_pins[] = {
+	/* CLKOUT */
+	RCAR_GP_PIN(2, 3),
+};
+static const unsigned int du_clk_out_1_mux[] = {
+	DU_DOTCLKOUT1_MARK
+};
+static const unsigned int du_sync_pins[] = {
+	/* EXVSYNC/VSYNC, EXHSYNC/HSYNC */
+	RCAR_GP_PIN(2, 5), RCAR_GP_PIN(2, 4),
+};
+static const unsigned int du_sync_mux[] = {
+	DU_EXVSYNC_DU_VSYNC_MARK, DU_EXHSYNC_DU_HSYNC_MARK
+};
+static const unsigned int du_oddf_pins[] = {
+	/* EXDISP/EXODDF/EXCDE */
+	RCAR_GP_PIN(2, 2),
+};
+static const unsigned int du_oddf_mux[] = {
+	DU_EXODDF_DU_ODDF_DISP_CDE_MARK,
+};
+static const unsigned int du_cde_pins[] = {
+	/* CDE */
+	RCAR_GP_PIN(2, 0),
+};
+static const unsigned int du_cde_mux[] = {
+	DU_CDE_MARK,
+};
+static const unsigned int du_disp_pins[] = {
+	/* DISP */
+	RCAR_GP_PIN(2, 1),
+};
+static const unsigned int du_disp_mux[] = {
+	DU_DISP_MARK,
+};
+
+/* - I2C -------------------------------------------------------------------- */
+static const unsigned int i2c1_a_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(5, 11), RCAR_GP_PIN(5, 10),
+};
+static const unsigned int i2c1_a_mux[] = {
+	SDA1_A_MARK, SCL1_A_MARK,
+};
+static const unsigned int i2c1_b_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(5, 24), RCAR_GP_PIN(5, 23),
+};
+static const unsigned int i2c1_b_mux[] = {
+	SDA1_B_MARK, SCL1_B_MARK,
+};
+static const unsigned int i2c2_a_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(5, 0), RCAR_GP_PIN(5, 4),
+};
+static const unsigned int i2c2_a_mux[] = {
+	SDA2_A_MARK, SCL2_A_MARK,
+};
+static const unsigned int i2c2_b_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(3, 13), RCAR_GP_PIN(3, 12),
+};
+static const unsigned int i2c2_b_mux[] = {
+	SDA2_B_MARK, SCL2_B_MARK,
+};
+static const unsigned int i2c6_a_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(1, 8), RCAR_GP_PIN(1, 11),
+};
+static const unsigned int i2c6_a_mux[] = {
+	SDA6_A_MARK, SCL6_A_MARK,
+};
+static const unsigned int i2c6_b_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(1, 26), RCAR_GP_PIN(1, 25),
+};
+static const unsigned int i2c6_b_mux[] = {
+	SDA6_B_MARK, SCL6_B_MARK,
+};
+static const unsigned int i2c6_c_pins[] = {
+	/* SDA, SCL */
+	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 14),
+};
+static const unsigned int i2c6_c_mux[] = {
+	SDA6_C_MARK, SCL6_C_MARK,
+};
+
 /* - SCIF0 ------------------------------------------------------------------ */
 static const unsigned int scif0_data_pins[] = {
 	/* RX, TX */
@@ -1912,6 +2324,60 @@ static const unsigned int sdhi3_ds_mux[] = {
 };
 
 static const struct sh_pfc_pin_group pinmux_groups[] = {
+	SH_PFC_PIN_GROUP(avb_link),
+	SH_PFC_PIN_GROUP(avb_magic),
+	SH_PFC_PIN_GROUP(avb_phy_int),
+	SH_PFC_PIN_GROUP(avb_mdc),
+	SH_PFC_PIN_GROUP(avb_avtp_pps),
+	SH_PFC_PIN_GROUP(avb_avtp_match_a),
+	SH_PFC_PIN_GROUP(avb_avtp_capture_a),
+	SH_PFC_PIN_GROUP(avb_avtp_match_b),
+	SH_PFC_PIN_GROUP(avb_avtp_capture_b),
+	SH_PFC_PIN_GROUP(drif0_ctrl_a),
+	SH_PFC_PIN_GROUP(drif0_data0_a),
+	SH_PFC_PIN_GROUP(drif0_data1_a),
+	SH_PFC_PIN_GROUP(drif0_ctrl_b),
+	SH_PFC_PIN_GROUP(drif0_data0_b),
+	SH_PFC_PIN_GROUP(drif0_data1_b),
+	SH_PFC_PIN_GROUP(drif0_ctrl_c),
+	SH_PFC_PIN_GROUP(drif0_data0_c),
+	SH_PFC_PIN_GROUP(drif0_data1_c),
+	SH_PFC_PIN_GROUP(drif1_ctrl_a),
+	SH_PFC_PIN_GROUP(drif1_data0_a),
+	SH_PFC_PIN_GROUP(drif1_data1_a),
+	SH_PFC_PIN_GROUP(drif1_ctrl_b),
+	SH_PFC_PIN_GROUP(drif1_data0_b),
+	SH_PFC_PIN_GROUP(drif1_data1_b),
+	SH_PFC_PIN_GROUP(drif1_ctrl_c),
+	SH_PFC_PIN_GROUP(drif1_data0_c),
+	SH_PFC_PIN_GROUP(drif1_data1_c),
+	SH_PFC_PIN_GROUP(drif2_ctrl_a),
+	SH_PFC_PIN_GROUP(drif2_data0_a),
+	SH_PFC_PIN_GROUP(drif2_data1_a),
+	SH_PFC_PIN_GROUP(drif2_ctrl_b),
+	SH_PFC_PIN_GROUP(drif2_data0_b),
+	SH_PFC_PIN_GROUP(drif2_data1_b),
+	SH_PFC_PIN_GROUP(drif3_ctrl_a),
+	SH_PFC_PIN_GROUP(drif3_data0_a),
+	SH_PFC_PIN_GROUP(drif3_data1_a),
+	SH_PFC_PIN_GROUP(drif3_ctrl_b),
+	SH_PFC_PIN_GROUP(drif3_data0_b),
+	SH_PFC_PIN_GROUP(drif3_data1_b),
+	SH_PFC_PIN_GROUP(du_rgb666),
+	SH_PFC_PIN_GROUP(du_rgb888),
+	SH_PFC_PIN_GROUP(du_clk_out_0),
+	SH_PFC_PIN_GROUP(du_clk_out_1),
+	SH_PFC_PIN_GROUP(du_sync),
+	SH_PFC_PIN_GROUP(du_oddf),
+	SH_PFC_PIN_GROUP(du_cde),
+	SH_PFC_PIN_GROUP(du_disp),
+	SH_PFC_PIN_GROUP(i2c1_a),
+	SH_PFC_PIN_GROUP(i2c1_b),
+	SH_PFC_PIN_GROUP(i2c2_a),
+	SH_PFC_PIN_GROUP(i2c2_b),
+	SH_PFC_PIN_GROUP(i2c6_a),
+	SH_PFC_PIN_GROUP(i2c6_b),
+	SH_PFC_PIN_GROUP(i2c6_c),
 	SH_PFC_PIN_GROUP(scif0_data),
 	SH_PFC_PIN_GROUP(scif0_clk),
 	SH_PFC_PIN_GROUP(scif0_ctrl),
@@ -1969,6 +2435,87 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(sdhi3_ds),
 };
 
+static const char * const avb_groups[] = {
+	"avb_link",
+	"avb_magic",
+	"avb_phy_int",
+	"avb_mdc",
+	"avb_avtp_pps",
+	"avb_avtp_match_a",
+	"avb_avtp_capture_a",
+	"avb_avtp_match_b",
+	"avb_avtp_capture_b",
+};
+
+static const char * const drif0_groups[] = {
+	"drif0_ctrl_a",
+	"drif0_data0_a",
+	"drif0_data1_a",
+	"drif0_ctrl_b",
+	"drif0_data0_b",
+	"drif0_data1_b",
+	"drif0_ctrl_c",
+	"drif0_data0_c",
+	"drif0_data1_c",
+};
+
+static const char * const drif1_groups[] = {
+	"drif1_ctrl_a",
+	"drif1_data0_a",
+	"drif1_data1_a",
+	"drif1_ctrl_b",
+	"drif1_data0_b",
+	"drif1_data1_b",
+	"drif1_ctrl_c",
+	"drif1_data0_c",
+	"drif1_data1_c",
+};
+
+static const char * const drif2_groups[] = {
+	"drif2_ctrl_a",
+	"drif2_data0_a",
+	"drif2_data1_a",
+	"drif2_ctrl_b",
+	"drif2_data0_b",
+	"drif2_data1_b",
+};
+
+static const char * const drif3_groups[] = {
+	"drif3_ctrl_a",
+	"drif3_data0_a",
+	"drif3_data1_a",
+	"drif3_ctrl_b",
+	"drif3_data0_b",
+	"drif3_data1_b",
+};
+
+static const char * const du_groups[] = {
+	"du_rgb666",
+	"du_rgb888",
+	"du_clk_out_0",
+	"du_clk_out_1",
+	"du_sync",
+	"du_oddf",
+	"du_cde",
+	"du_disp",
+};
+
+static const char * const i2c1_groups[] = {
+	"i2c1_a",
+	"i2c1_b",
+};
+
+static const char * const i2c2_groups[] = {
+	"i2c2_a",
+	"i2c2_b",
+};
+
+static const char * const i2c6_groups[] = {
+	"i2c6_a",
+	"i2c6_b",
+	"i2c6_c",
+};
+
 static const char * const scif0_groups[] = {
 	"scif0_data",
 	"scif0_clk",
@@ -2058,6 +2605,15 @@ static const char * const sdhi3_groups[] = {
 };
 
 static const struct sh_pfc_function pinmux_functions[] = {
+	SH_PFC_FUNCTION(avb),
+	SH_PFC_FUNCTION(drif0),
+	SH_PFC_FUNCTION(drif1),
+	SH_PFC_FUNCTION(drif2),
+	SH_PFC_FUNCTION(drif3),
+	SH_PFC_FUNCTION(du),
+	SH_PFC_FUNCTION(i2c1),
+	SH_PFC_FUNCTION(i2c2),
+	SH_PFC_FUNCTION(i2c6),
 	SH_PFC_FUNCTION(scif0),
 	SH_PFC_FUNCTION(scif1),
 	SH_PFC_FUNCTION(scif2),
diff --git a/drivers/pinctrl/sh-pfc/pinctrl.c b/drivers/pinctrl/sh-pfc/pinctrl.c
index c5772584594c..fcacfa73ef6e 100644
--- a/drivers/pinctrl/sh-pfc/pinctrl.c
+++ b/drivers/pinctrl/sh-pfc/pinctrl.c
@@ -570,7 +570,8 @@ static bool sh_pfc_pinconf_validate(struct sh_pfc *pfc, unsigned int _pin,
 
 	switch (param) {
 	case PIN_CONFIG_BIAS_DISABLE:
-		return true;
+		return pin->configs &
+			(SH_PFC_PIN_CFG_PULL_UP | SH_PFC_PIN_CFG_PULL_DOWN);
 
 	case PIN_CONFIG_BIAS_PULL_UP:
 		return pin->configs & SH_PFC_PIN_CFG_PULL_UP;
diff --git a/drivers/pinctrl/sh-pfc/sh_pfc.h b/drivers/pinctrl/sh-pfc/sh_pfc.h
index 2345421103db..e42cc7a8d10e 100644
--- a/drivers/pinctrl/sh-pfc/sh_pfc.h
+++ b/drivers/pinctrl/sh-pfc/sh_pfc.h
@@ -189,6 +189,12 @@ struct sh_pfc_window {
 	unsigned long size;
 };
 
+struct sh_pfc_bias_info {
+	u16 pin;
+	u16 reg : 11;
+	u16 bit : 5;
+};
+
 struct sh_pfc_pin_range;
 
 struct sh_pfc {
@@ -540,6 +546,14 @@ extern const struct sh_pfc_soc_info shx3_pinmux_info;
 		.configs = SH_PFC_PIN_CFG_NO_GPIO,			\
 	}
 
+/* SH_PFC_PIN_NAMED_CFG - Expand to a sh_pfc_pin entry with the given name */
+#define SH_PFC_PIN_NAMED_CFG(row, col, _name, cfgs)			\
+	{								\
+		.pin = PIN_NUMBER(row, col),				\
+		.name = __stringify(PIN_##_name),			\
+		.configs = SH_PFC_PIN_CFG_NO_GPIO | cfgs,		\
+	}
+
 /* PINMUX_DATA_ALL - Expand to a list of PORT_name_DATA, PORT_name_FN0,
  *		     PORT_name_OUT, PORT_name_IN marks
  */
diff --git a/drivers/pinctrl/stm32/pinctrl-stm32f429.c b/drivers/pinctrl/stm32/pinctrl-stm32f429.c
index e9b15dc0654b..990b867b9625 100644
--- a/drivers/pinctrl/stm32/pinctrl-stm32f429.c
+++ b/drivers/pinctrl/stm32/pinctrl-stm32f429.c
@@ -1584,8 +1584,4 @@ static struct platform_driver stm32f429_pinctrl_driver = {
 	},
 };
 
-static int __init stm32f429_pinctrl_init(void)
-{
-	return platform_driver_register(&stm32f429_pinctrl_driver);
-}
-device_initcall(stm32f429_pinctrl_init);
+builtin_platform_driver(stm32f429_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-gr8.c b/drivers/pinctrl/sunxi/pinctrl-gr8.c
index 2904d2b7378b..2f232c3a0579 100644
--- a/drivers/pinctrl/sunxi/pinctrl-gr8.c
+++ b/drivers/pinctrl/sunxi/pinctrl-gr8.c
@@ -12,7 +12,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -525,7 +525,6 @@ static const struct of_device_id sun5i_gr8_pinctrl_match[] = {
 	{ .compatible = "nextthing,gr8-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun5i_gr8_pinctrl_match);
 
 static struct platform_driver sun5i_gr8_pinctrl_driver = {
 	.probe	= sun5i_gr8_pinctrl_probe,
@@ -534,8 +533,4 @@ static struct platform_driver sun5i_gr8_pinctrl_driver = {
 		.of_match_table	= sun5i_gr8_pinctrl_match,
 	},
 };
-module_platform_driver(sun5i_gr8_pinctrl_driver);
-
-MODULE_AUTHOR("Mylene Josserand <mylene.josserand@free-electrons.com");
-MODULE_DESCRIPTION("NextThing GR8 pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun5i_gr8_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun4i-a10.c b/drivers/pinctrl/sunxi/pinctrl-sun4i-a10.c
index 862a096c5dba..fb30b86a97ee 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun4i-a10.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun4i-a10.c
@@ -10,7 +10,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -1035,7 +1035,6 @@ static const struct of_device_id sun4i_a10_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun4i-a10-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun4i_a10_pinctrl_match);
 
 static struct platform_driver sun4i_a10_pinctrl_driver = {
 	.probe	= sun4i_a10_pinctrl_probe,
@@ -1044,8 +1043,4 @@ static struct platform_driver sun4i_a10_pinctrl_driver = {
 		.of_match_table	= sun4i_a10_pinctrl_match,
 	},
 };
-module_platform_driver(sun4i_a10_pinctrl_driver);
-
-MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com");
-MODULE_DESCRIPTION("Allwinner A10 pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun4i_a10_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun5i-a10s.c b/drivers/pinctrl/sunxi/pinctrl-sun5i-a10s.c
index f9a3f8f446f7..a5b57fdff9e1 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun5i-a10s.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun5i-a10s.c
@@ -10,7 +10,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -674,7 +674,6 @@ static const struct of_device_id sun5i_a10s_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun5i-a10s-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun5i_a10s_pinctrl_match);
 
 static struct platform_driver sun5i_a10s_pinctrl_driver = {
 	.probe	= sun5i_a10s_pinctrl_probe,
@@ -683,8 +682,4 @@ static struct platform_driver sun5i_a10s_pinctrl_driver = {
 		.of_match_table	= sun5i_a10s_pinctrl_match,
 	},
 };
-module_platform_driver(sun5i_a10s_pinctrl_driver);
-
-MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com");
-MODULE_DESCRIPTION("Allwinner A10s pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun5i_a10s_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun5i-a13.c b/drivers/pinctrl/sunxi/pinctrl-sun5i-a13.c
index 2bb07b38834f..8575f3f6d3dd 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun5i-a13.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun5i-a13.c
@@ -10,7 +10,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -392,7 +392,6 @@ static const struct of_device_id sun5i_a13_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun5i-a13-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun5i_a13_pinctrl_match);
 
 static struct platform_driver sun5i_a13_pinctrl_driver = {
 	.probe	= sun5i_a13_pinctrl_probe,
@@ -401,8 +400,4 @@ static struct platform_driver sun5i_a13_pinctrl_driver = {
 		.of_match_table	= sun5i_a13_pinctrl_match,
 	},
 };
-module_platform_driver(sun5i_a13_pinctrl_driver);
-
-MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com");
-MODULE_DESCRIPTION("Allwinner A13 pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun5i_a13_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun6i-a31-r.c b/drivers/pinctrl/sunxi/pinctrl-sun6i-a31-r.c
index d4bc4f0e8be0..a22bd88a1f03 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun6i-a31-r.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun6i-a31-r.c
@@ -12,7 +12,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -136,7 +136,6 @@ static const struct of_device_id sun6i_a31_r_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun6i-a31-r-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun6i_a31_r_pinctrl_match);
 
 static struct platform_driver sun6i_a31_r_pinctrl_driver = {
 	.probe	= sun6i_a31_r_pinctrl_probe,
@@ -145,9 +144,4 @@ static struct platform_driver sun6i_a31_r_pinctrl_driver = {
 		.of_match_table	= sun6i_a31_r_pinctrl_match,
 	},
 };
-module_platform_driver(sun6i_a31_r_pinctrl_driver);
-
-MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com");
-MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com");
-MODULE_DESCRIPTION("Allwinner A31 R_PIO pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun6i_a31_r_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun6i-a31.c b/drivers/pinctrl/sunxi/pinctrl-sun6i-a31.c
index a70b52957e24..9e58926bef37 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun6i-a31.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun6i-a31.c
@@ -10,7 +10,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -934,7 +934,6 @@ static const struct of_device_id sun6i_a31_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun6i-a31-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun6i_a31_pinctrl_match);
 
 static struct platform_driver sun6i_a31_pinctrl_driver = {
 	.probe	= sun6i_a31_pinctrl_probe,
@@ -943,8 +942,4 @@ static struct platform_driver sun6i_a31_pinctrl_driver = {
 		.of_match_table	= sun6i_a31_pinctrl_match,
 	},
 };
-module_platform_driver(sun6i_a31_pinctrl_driver);
-
-MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com");
-MODULE_DESCRIPTION("Allwinner A31 pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun6i_a31_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun6i-a31s.c b/drivers/pinctrl/sunxi/pinctrl-sun6i-a31s.c
index e570d5c93ecc..231a746a5356 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun6i-a31s.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun6i-a31s.c
@@ -11,7 +11,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -798,7 +798,6 @@ static const struct of_device_id sun6i_a31s_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun6i-a31s-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun6i_a31s_pinctrl_match);
 
 static struct platform_driver sun6i_a31s_pinctrl_driver = {
 	.probe	= sun6i_a31s_pinctrl_probe,
@@ -807,8 +806,4 @@ static struct platform_driver sun6i_a31s_pinctrl_driver = {
 		.of_match_table	= sun6i_a31s_pinctrl_match,
 	},
 };
-module_platform_driver(sun6i_a31s_pinctrl_driver);
-
-MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
-MODULE_DESCRIPTION("Allwinner A31s pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun6i_a31s_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun7i-a20.c b/drivers/pinctrl/sunxi/pinctrl-sun7i-a20.c
index 435ad30f45db..b6f4c68ffb39 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun7i-a20.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun7i-a20.c
@@ -10,7 +10,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -1045,7 +1045,6 @@ static const struct of_device_id sun7i_a20_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun7i-a20-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun7i_a20_pinctrl_match);
 
 static struct platform_driver sun7i_a20_pinctrl_driver = {
 	.probe	= sun7i_a20_pinctrl_probe,
@@ -1054,8 +1053,4 @@ static struct platform_driver sun7i_a20_pinctrl_driver = {
 		.of_match_table	= sun7i_a20_pinctrl_match,
 	},
 };
-module_platform_driver(sun7i_a20_pinctrl_driver);
-
-MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com");
-MODULE_DESCRIPTION("Allwinner A20 pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun7i_a20_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun8i-a23-r.c b/drivers/pinctrl/sunxi/pinctrl-sun8i-a23-r.c
index 056287635873..2292e05a397b 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun8i-a23-r.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun8i-a23-r.c
@@ -15,7 +15,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -123,7 +123,6 @@ static const struct of_device_id sun8i_a23_r_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun8i-a23-r-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun8i_a23_r_pinctrl_match);
 
 static struct platform_driver sun8i_a23_r_pinctrl_driver = {
 	.probe	= sun8i_a23_r_pinctrl_probe,
@@ -132,10 +131,4 @@ static struct platform_driver sun8i_a23_r_pinctrl_driver = {
 		.of_match_table	= sun8i_a23_r_pinctrl_match,
 	},
 };
-module_platform_driver(sun8i_a23_r_pinctrl_driver);
-
-MODULE_AUTHOR("Chen-Yu Tsai <wens@csie.org>");
-MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com");
-MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com");
-MODULE_DESCRIPTION("Allwinner A23 R_PIO pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun8i_a23_r_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun8i-a23.c b/drivers/pinctrl/sunxi/pinctrl-sun8i-a23.c
index f9d661e5c14a..721b6935baf3 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun8i-a23.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun8i-a23.c
@@ -14,7 +14,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -575,7 +575,6 @@ static const struct of_device_id sun8i_a23_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun8i-a23-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun8i_a23_pinctrl_match);
 
 static struct platform_driver sun8i_a23_pinctrl_driver = {
 	.probe	= sun8i_a23_pinctrl_probe,
@@ -584,9 +583,4 @@ static struct platform_driver sun8i_a23_pinctrl_driver = {
 		.of_match_table	= sun8i_a23_pinctrl_match,
 	},
 };
-module_platform_driver(sun8i_a23_pinctrl_driver);
-
-MODULE_AUTHOR("Chen-Yu Tsai <wens@csie.org>");
-MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com");
-MODULE_DESCRIPTION("Allwinner A23 pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun8i_a23_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun8i-a33.c b/drivers/pinctrl/sunxi/pinctrl-sun8i-a33.c
index 3131cac2b76f..ef1e0bef4099 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun8i-a33.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun8i-a33.c
@@ -12,7 +12,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -498,7 +498,6 @@ static const struct of_device_id sun8i_a33_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun8i-a33-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun8i_a33_pinctrl_match);
 
 static struct platform_driver sun8i_a33_pinctrl_driver = {
 	.probe	= sun8i_a33_pinctrl_probe,
@@ -507,8 +506,4 @@ static struct platform_driver sun8i_a33_pinctrl_driver = {
 		.of_match_table	= sun8i_a33_pinctrl_match,
 	},
 };
-module_platform_driver(sun8i_a33_pinctrl_driver);
-
-MODULE_AUTHOR("Vishnu Patekar <vishnupatekar0510@gmail.com>");
-MODULE_DESCRIPTION("Allwinner a33 pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun8i_a33_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun8i-a83t.c b/drivers/pinctrl/sunxi/pinctrl-sun8i-a83t.c
index 90b973e15982..9aec1d2232dd 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun8i-a83t.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun8i-a83t.c
@@ -12,7 +12,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -587,7 +587,6 @@ static const struct of_device_id sun8i_a83t_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun8i-a83t-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun8i_a83t_pinctrl_match);
 
 static struct platform_driver sun8i_a83t_pinctrl_driver = {
 	.probe	= sun8i_a83t_pinctrl_probe,
@@ -596,8 +595,4 @@ static struct platform_driver sun8i_a83t_pinctrl_driver = {
 		.of_match_table	= sun8i_a83t_pinctrl_match,
 	},
 };
-module_platform_driver(sun8i_a83t_pinctrl_driver);
-
-MODULE_AUTHOR("Vishnu Patekar <vishnupatekar0510@gmail.com>");
-MODULE_DESCRIPTION("Allwinner a83t pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun8i_a83t_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun9i-a80.c b/drivers/pinctrl/sunxi/pinctrl-sun9i-a80.c
index 1b580ba76453..bc14e954d7a2 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun9i-a80.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun9i-a80.c
@@ -10,7 +10,7 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -733,7 +733,6 @@ static const struct of_device_id sun9i_a80_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun9i-a80-pinctrl", },
 	{}
 };
-MODULE_DEVICE_TABLE(of, sun9i_a80_pinctrl_match);
 
 static struct platform_driver sun9i_a80_pinctrl_driver = {
 	.probe	= sun9i_a80_pinctrl_probe,
@@ -742,8 +741,4 @@ static struct platform_driver sun9i_a80_pinctrl_driver = {
 		.of_match_table	= sun9i_a80_pinctrl_match,
 	},
 };
-module_platform_driver(sun9i_a80_pinctrl_driver);
-
-MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>");
-MODULE_DESCRIPTION("Allwinner A80 pinctrl driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sun9i_a80_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
index 0facbea5f465..0eb51e33cb1b 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
@@ -28,6 +28,8 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 
+#include <dt-bindings/pinctrl/sun4i-a10.h>
+
 #include "../core.h"
 #include "pinctrl-sunxi.h"
 
@@ -145,6 +147,171 @@ static int sunxi_pctrl_get_group_pins(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
+static bool sunxi_pctrl_has_bias_prop(struct device_node *node)
+{
+	return of_find_property(node, "bias-pull-up", NULL) ||
+		of_find_property(node, "bias-pull-down", NULL) ||
+		of_find_property(node, "bias-disable", NULL) ||
+		of_find_property(node, "allwinner,pull", NULL);
+}
+
+static bool sunxi_pctrl_has_drive_prop(struct device_node *node)
+{
+	return of_find_property(node, "drive-strength", NULL) ||
+		of_find_property(node, "allwinner,drive", NULL);
+}
+
+static int sunxi_pctrl_parse_bias_prop(struct device_node *node)
+{
+	u32 val;
+
+	/* Try the new style binding */
+	if (of_find_property(node, "bias-pull-up", NULL))
+		return PIN_CONFIG_BIAS_PULL_UP;
+
+	if (of_find_property(node, "bias-pull-down", NULL))
+		return PIN_CONFIG_BIAS_PULL_DOWN;
+
+	if (of_find_property(node, "bias-disable", NULL))
+		return PIN_CONFIG_BIAS_DISABLE;
+
+	/* And fall back to the old binding */
+	if (of_property_read_u32(node, "allwinner,pull", &val))
+		return -EINVAL;
+
+	switch (val) {
+	case SUN4I_PINCTRL_NO_PULL:
+		return PIN_CONFIG_BIAS_DISABLE;
+	case SUN4I_PINCTRL_PULL_UP:
+		return PIN_CONFIG_BIAS_PULL_UP;
+	case SUN4I_PINCTRL_PULL_DOWN:
+		return PIN_CONFIG_BIAS_PULL_DOWN;
+	}
+
+	return -EINVAL;
+}
+
+static int sunxi_pctrl_parse_drive_prop(struct device_node *node)
+{
+	u32 val;
+
+	/* Try the new style binding */
+	if (!of_property_read_u32(node, "drive-strength", &val)) {
+		/* We can't go below 10mA ... */
+		if (val < 10)
+			return -EINVAL;
+
+		/* ... and only up to 40 mA ... */
+		if (val > 40)
+			val = 40;
+
+		/* by steps of 10 mA */
+		return rounddown(val, 10);
+	}
+
+	/* And then fall back to the old binding */
+	if (of_property_read_u32(node, "allwinner,drive", &val))
+		return -EINVAL;
+
+	return (val + 1) * 10;
+}
+
+static const char *sunxi_pctrl_parse_function_prop(struct device_node *node)
+{
+	const char *function;
+	int ret;
+
+	/* Try the generic binding */
+	ret = of_property_read_string(node, "function", &function);
+	if (!ret)
+		return function;
+
+	/* And fall back to our legacy one */
+	ret = of_property_read_string(node, "allwinner,function", &function);
+	if (!ret)
+		return function;
+
+	return NULL;
+}
+
+static const char *sunxi_pctrl_find_pins_prop(struct device_node *node,
+					      int *npins)
+{
+	int count;
+
+	/* Try the generic binding */
+	count = of_property_count_strings(node, "pins");
+	if (count > 0) {
+		*npins = count;
+		return "pins";
+	}
+
+	/* And fall back to our legacy one */
+	count = of_property_count_strings(node, "allwinner,pins");
+	if (count > 0) {
+		*npins = count;
+		return "allwinner,pins";
+	}
+
+	return NULL;
+}
+
+static unsigned long *sunxi_pctrl_build_pin_config(struct device_node *node,
+						   unsigned int *len)
+{
+	unsigned long *pinconfig;
+	unsigned int configlen = 0, idx = 0;
+	int ret;
+
+	if (sunxi_pctrl_has_drive_prop(node))
+		configlen++;
+	if (sunxi_pctrl_has_bias_prop(node))
+		configlen++;
+
+	/*
+	 * If we don't have any configuration, bail out
+	 */
+	if (!configlen)
+		return NULL;
+
+	pinconfig = kzalloc(configlen * sizeof(*pinconfig), GFP_KERNEL);
+	if (!pinconfig)
+		return ERR_PTR(-ENOMEM);
+
+	if (sunxi_pctrl_has_drive_prop(node)) {
+		int drive = sunxi_pctrl_parse_drive_prop(node);
+		if (drive < 0) {
+			ret = drive;
+			goto err_free;
+		}
+
+		pinconfig[idx++] = pinconf_to_config_packed(PIN_CONFIG_DRIVE_STRENGTH,
+							  drive);
+	}
+
+	if (sunxi_pctrl_has_bias_prop(node)) {
+		int pull = sunxi_pctrl_parse_bias_prop(node);
+		int arg = 0;
+		if (pull < 0) {
+			ret = pull;
+			goto err_free;
+		}
+
+		if (pull != PIN_CONFIG_BIAS_DISABLE)
+			arg = 1; /* hardware uses weak pull resistors */
+
+		pinconfig[idx++] = pinconf_to_config_packed(pull, arg);
+	}
+
+
+	*len = configlen;
+	return pinconfig;
+
+err_free:
+	kfree(pinconfig);
+	return ERR_PTR(ret);
+}
+
 static int sunxi_pctrl_dt_node_to_map(struct pinctrl_dev *pctldev,
 				      struct device_node *node,
 				      struct pinctrl_map **map,
@@ -153,38 +320,48 @@ static int sunxi_pctrl_dt_node_to_map(struct pinctrl_dev *pctldev,
 	struct sunxi_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 	unsigned long *pinconfig;
 	struct property *prop;
-	const char *function;
+	const char *function, *pin_prop;
 	const char *group;
-	int ret, nmaps, i = 0;
-	u32 val;
+	int ret, npins, nmaps, configlen = 0, i = 0;
 
 	*map = NULL;
 	*num_maps = 0;
 
-	ret = of_property_read_string(node, "allwinner,function", &function);
-	if (ret) {
-		dev_err(pctl->dev,
-			"missing allwinner,function property in node %s\n",
+	function = sunxi_pctrl_parse_function_prop(node);
+	if (!function) {
+		dev_err(pctl->dev, "missing function property in node %s\n",
 			node->name);
 		return -EINVAL;
 	}
 
-	nmaps = of_property_count_strings(node, "allwinner,pins") * 2;
-	if (nmaps < 0) {
-		dev_err(pctl->dev,
-			"missing allwinner,pins property in node %s\n",
+	pin_prop = sunxi_pctrl_find_pins_prop(node, &npins);
+	if (!pin_prop) {
+		dev_err(pctl->dev, "missing pins property in node %s\n",
 			node->name);
 		return -EINVAL;
 	}
 
+	/*
+	 * We have two maps for each pin: one for the function, one
+	 * for the configuration (bias, strength, etc).
+	 *
+	 * We might be slightly overshooting, since we might not have
+	 * any configuration.
+	 */
+	nmaps = npins * 2;
 	*map = kmalloc(nmaps * sizeof(struct pinctrl_map), GFP_KERNEL);
 	if (!*map)
 		return -ENOMEM;
 
-	of_property_for_each_string(node, "allwinner,pins", prop, group) {
+	pinconfig = sunxi_pctrl_build_pin_config(node, &configlen);
+	if (IS_ERR(pinconfig)) {
+		ret = PTR_ERR(pinconfig);
+		goto err_free_map;
+	}
+
+	of_property_for_each_string(node, pin_prop, prop, group) {
 		struct sunxi_pinctrl_group *grp =
 			sunxi_pinctrl_find_group_by_name(pctl, group);
-		int j = 0, configlen = 0;
 
 		if (!grp) {
 			dev_err(pctl->dev, "unknown pin %s", group);
@@ -205,45 +382,31 @@ static int sunxi_pctrl_dt_node_to_map(struct pinctrl_dev *pctldev,
 
 		i++;
 
-		(*map)[i].type = PIN_MAP_TYPE_CONFIGS_GROUP;
-		(*map)[i].data.configs.group_or_pin = group;
-
-		if (of_find_property(node, "allwinner,drive", NULL))
-			configlen++;
-		if (of_find_property(node, "allwinner,pull", NULL))
-			configlen++;
-
-		pinconfig = kzalloc(configlen * sizeof(*pinconfig), GFP_KERNEL);
-		if (!pinconfig) {
-			kfree(*map);
-			return -ENOMEM;
+		if (pinconfig) {
+			(*map)[i].type = PIN_MAP_TYPE_CONFIGS_GROUP;
+			(*map)[i].data.configs.group_or_pin = group;
+			(*map)[i].data.configs.configs = pinconfig;
+			(*map)[i].data.configs.num_configs = configlen;
+			i++;
 		}
-
-		if (!of_property_read_u32(node, "allwinner,drive", &val)) {
-			u16 strength = (val + 1) * 10;
-			pinconfig[j++] =
-				pinconf_to_config_packed(PIN_CONFIG_DRIVE_STRENGTH,
-							 strength);
-		}
-
-		if (!of_property_read_u32(node, "allwinner,pull", &val)) {
-			enum pin_config_param pull = PIN_CONFIG_END;
-			if (val == 1)
-				pull = PIN_CONFIG_BIAS_PULL_UP;
-			else if (val == 2)
-				pull = PIN_CONFIG_BIAS_PULL_DOWN;
-			pinconfig[j++] = pinconf_to_config_packed(pull, 0);
-		}
-
-		(*map)[i].data.configs.configs = pinconfig;
-		(*map)[i].data.configs.num_configs = configlen;
-
-		i++;
 	}
 
-	*num_maps = nmaps;
+	*num_maps = i;
+
+	/*
+	 * We know have the number of maps we need, we can resize our
+	 * map array
+	 */
+	*map = krealloc(*map, i * sizeof(struct pinctrl_map), GFP_KERNEL);
+	if (!*map)
+		return -ENOMEM;
 
 	return 0;
+
+err_free_map:
+	kfree(*map);
+	*map = NULL;
+	return ret;
 }
 
 static void sunxi_pctrl_dt_free_map(struct pinctrl_dev *pctldev,
@@ -252,9 +415,17 @@ static void sunxi_pctrl_dt_free_map(struct pinctrl_dev *pctldev,
 {
 	int i;
 
-	for (i = 0; i < num_maps; i++) {
-		if (map[i].type == PIN_MAP_TYPE_CONFIGS_GROUP)
-			kfree(map[i].data.configs.configs);
+	/* pin config is never in the first map */
+	for (i = 1; i < num_maps; i++) {
+		if (map[i].type != PIN_MAP_TYPE_CONFIGS_GROUP)
+			continue;
+
+		/*
+		 * All the maps share the same pin config,
+		 * free only the first one we find.
+		 */
+		kfree(map[i].data.configs.configs);
+		break;
 	}
 
 	kfree(map);
@@ -268,15 +439,91 @@ static const struct pinctrl_ops sunxi_pctrl_ops = {
 	.get_group_pins		= sunxi_pctrl_get_group_pins,
 };
 
+static int sunxi_pconf_reg(unsigned pin, enum pin_config_param param,
+			   u32 *offset, u32 *shift, u32 *mask)
+{
+	switch (param) {
+	case PIN_CONFIG_DRIVE_STRENGTH:
+		*offset = sunxi_dlevel_reg(pin);
+		*shift = sunxi_dlevel_offset(pin);
+		*mask = DLEVEL_PINS_MASK;
+		break;
+
+	case PIN_CONFIG_BIAS_PULL_UP:
+	case PIN_CONFIG_BIAS_PULL_DOWN:
+	case PIN_CONFIG_BIAS_DISABLE:
+		*offset = sunxi_pull_reg(pin);
+		*shift = sunxi_pull_offset(pin);
+		*mask = PULL_PINS_MASK;
+		break;
+
+	default:
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static int sunxi_pconf_get(struct pinctrl_dev *pctldev, unsigned pin,
+			   unsigned long *config)
+{
+	struct sunxi_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	enum pin_config_param param = pinconf_to_config_param(*config);
+	u32 offset, shift, mask, val;
+	u16 arg;
+	int ret;
+
+	pin -= pctl->desc->pin_base;
+
+	ret = sunxi_pconf_reg(pin, param, &offset, &shift, &mask);
+	if (ret < 0)
+		return ret;
+
+	val = (readl(pctl->membase + offset) >> shift) & mask;
+
+	switch (pinconf_to_config_param(*config)) {
+	case PIN_CONFIG_DRIVE_STRENGTH:
+		arg = (val + 1) * 10;
+		break;
+
+	case PIN_CONFIG_BIAS_PULL_UP:
+		if (val != SUN4I_PINCTRL_PULL_UP)
+			return -EINVAL;
+		arg = 1; /* hardware is weak pull-up */
+		break;
+
+	case PIN_CONFIG_BIAS_PULL_DOWN:
+		if (val != SUN4I_PINCTRL_PULL_DOWN)
+			return -EINVAL;
+		arg = 1; /* hardware is weak pull-down */
+		break;
+
+	case PIN_CONFIG_BIAS_DISABLE:
+		if (val != SUN4I_PINCTRL_NO_PULL)
+			return -EINVAL;
+		arg = 0;
+		break;
+
+	default:
+		/* sunxi_pconf_reg should catch anything unsupported */
+		WARN_ON(1);
+		return -ENOTSUPP;
+	}
+
+	*config = pinconf_to_config_packed(param, arg);
+
+	return 0;
+}
+
 static int sunxi_pconf_group_get(struct pinctrl_dev *pctldev,
 				 unsigned group,
 				 unsigned long *config)
 {
 	struct sunxi_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sunxi_pinctrl_group *g = &pctl->groups[group];
 
-	*config = pctl->groups[group].config;
-
-	return 0;
+	/* We only support 1 pin per group. Chain it to the pin callback */
+	return sunxi_pconf_get(pctldev, g->pin, config);
 }
 
 static int sunxi_pconf_group_set(struct pinctrl_dev *pctldev,
@@ -286,23 +533,27 @@ static int sunxi_pconf_group_set(struct pinctrl_dev *pctldev,
 {
 	struct sunxi_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
 	struct sunxi_pinctrl_group *g = &pctl->groups[group];
-	unsigned long flags;
 	unsigned pin = g->pin - pctl->desc->pin_base;
-	u32 val, mask;
-	u16 strength;
-	u8 dlevel;
 	int i;
 
-	spin_lock_irqsave(&pctl->lock, flags);
-
 	for (i = 0; i < num_configs; i++) {
-		switch (pinconf_to_config_param(configs[i])) {
+		enum pin_config_param param;
+		unsigned long flags;
+		u32 offset, shift, mask, reg;
+		u16 arg, val;
+		int ret;
+
+		param = pinconf_to_config_param(configs[i]);
+		arg = pinconf_to_config_argument(configs[i]);
+
+		ret = sunxi_pconf_reg(pin, param, &offset, &shift, &mask);
+		if (ret < 0)
+			return ret;
+
+		switch (param) {
 		case PIN_CONFIG_DRIVE_STRENGTH:
-			strength = pinconf_to_config_argument(configs[i]);
-			if (strength > 40) {
-				spin_unlock_irqrestore(&pctl->lock, flags);
+			if (arg < 10 || arg > 40)
 				return -EINVAL;
-			}
 			/*
 			 * We convert from mA to what the register expects:
 			 *   0: 10mA
@@ -310,38 +561,40 @@ static int sunxi_pconf_group_set(struct pinctrl_dev *pctldev,
 			 *   2: 30mA
 			 *   3: 40mA
 			 */
-			dlevel = strength / 10 - 1;
-			val = readl(pctl->membase + sunxi_dlevel_reg(pin));
-			mask = DLEVEL_PINS_MASK << sunxi_dlevel_offset(pin);
-			writel((val & ~mask)
-				| dlevel << sunxi_dlevel_offset(pin),
-				pctl->membase + sunxi_dlevel_reg(pin));
+			val = arg / 10 - 1;
+			break;
+		case PIN_CONFIG_BIAS_DISABLE:
+			val = 0;
 			break;
 		case PIN_CONFIG_BIAS_PULL_UP:
-			val = readl(pctl->membase + sunxi_pull_reg(pin));
-			mask = PULL_PINS_MASK << sunxi_pull_offset(pin);
-			writel((val & ~mask) | 1 << sunxi_pull_offset(pin),
-				pctl->membase + sunxi_pull_reg(pin));
+			if (arg == 0)
+				return -EINVAL;
+			val = 1;
 			break;
 		case PIN_CONFIG_BIAS_PULL_DOWN:
-			val = readl(pctl->membase + sunxi_pull_reg(pin));
-			mask = PULL_PINS_MASK << sunxi_pull_offset(pin);
-			writel((val & ~mask) | 2 << sunxi_pull_offset(pin),
-				pctl->membase + sunxi_pull_reg(pin));
+			if (arg == 0)
+				return -EINVAL;
+			val = 2;
 			break;
 		default:
-			break;
+			/* sunxi_pconf_reg should catch anything unsupported */
+			WARN_ON(1);
+			return -ENOTSUPP;
 		}
-		/* cache the config value */
-		g->config = configs[i];
-	} /* for each config */
 
-	spin_unlock_irqrestore(&pctl->lock, flags);
+		spin_lock_irqsave(&pctl->lock, flags);
+		reg = readl(pctl->membase + offset);
+		reg &= ~(mask << shift);
+		writel(reg | val << shift, pctl->membase + offset);
+		spin_unlock_irqrestore(&pctl->lock, flags);
+	} /* for each config */
 
 	return 0;
 }
 
 static const struct pinconf_ops sunxi_pconf_ops = {
+	.is_generic		= true,
+	.pin_config_get		= sunxi_pconf_get,
 	.pin_config_group_get	= sunxi_pconf_group_get,
 	.pin_config_group_set	= sunxi_pconf_group_set,
 };
@@ -870,6 +1123,91 @@ static int sunxi_pinctrl_build_state(struct platform_device *pdev)
 	return 0;
 }
 
+static int sunxi_pinctrl_get_debounce_div(struct clk *clk, int freq, int *diff)
+{
+	unsigned long clock = clk_get_rate(clk);
+	unsigned int best_diff, best_div;
+	int i;
+
+	best_diff = abs(freq - clock);
+	best_div = 0;
+
+	for (i = 1; i < 8; i++) {
+		int cur_diff = abs(freq - (clock >> i));
+
+		if (cur_diff < best_diff) {
+			best_diff = cur_diff;
+			best_div = i;
+		}
+	}
+
+	*diff = best_diff;
+	return best_div;
+}
+
+static int sunxi_pinctrl_setup_debounce(struct sunxi_pinctrl *pctl,
+					struct device_node *node)
+{
+	unsigned int hosc_diff, losc_diff;
+	unsigned int hosc_div, losc_div;
+	struct clk *hosc, *losc;
+	u8 div, src;
+	int i, ret;
+
+	/* Deal with old DTs that didn't have the oscillators */
+	if (of_count_phandle_with_args(node, "clocks", "#clock-cells") != 3)
+		return 0;
+
+	/* If we don't have any setup, bail out */
+	if (!of_find_property(node, "input-debounce", NULL))
+		return 0;
+
+	losc = devm_clk_get(pctl->dev, "losc");
+	if (IS_ERR(losc))
+		return PTR_ERR(losc);
+
+	hosc = devm_clk_get(pctl->dev, "hosc");
+	if (IS_ERR(hosc))
+		return PTR_ERR(hosc);
+
+	for (i = 0; i < pctl->desc->irq_banks; i++) {
+		unsigned long debounce_freq;
+		u32 debounce;
+
+		ret = of_property_read_u32_index(node, "input-debounce",
+						 i, &debounce);
+		if (ret)
+			return ret;
+
+		if (!debounce)
+			continue;
+
+		debounce_freq = DIV_ROUND_CLOSEST(USEC_PER_SEC, debounce);
+		losc_div = sunxi_pinctrl_get_debounce_div(losc,
+							  debounce_freq,
+							  &losc_diff);
+
+		hosc_div = sunxi_pinctrl_get_debounce_div(hosc,
+							  debounce_freq,
+							  &hosc_diff);
+
+		if (hosc_diff < losc_diff) {
+			div = hosc_div;
+			src = 1;
+		} else {
+			div = losc_div;
+			src = 0;
+		}
+
+		writel(src | div << 4,
+		       pctl->membase +
+		       sunxi_irq_debounce_reg_from_bank(i,
+							pctl->desc->irq_bank_base));
+	}
+
+	return 0;
+}
+
 int sunxi_pinctrl_init(struct platform_device *pdev,
 		       const struct sunxi_pinctrl_desc *desc)
 {
@@ -1032,6 +1370,8 @@ int sunxi_pinctrl_init(struct platform_device *pdev,
 						 pctl);
 	}
 
+	sunxi_pinctrl_setup_debounce(pctl, node);
+
 	dev_info(&pdev->dev, "initialized sunXi PIO driver\n");
 
 	return 0;
diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.h b/drivers/pinctrl/sunxi/pinctrl-sunxi.h
index 0afce1ab12d0..f78a44a03189 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sunxi.h
+++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.h
@@ -69,6 +69,8 @@
 #define IRQ_STATUS_IRQ_BITS		1
 #define IRQ_STATUS_IRQ_MASK		((1 << IRQ_STATUS_IRQ_BITS) - 1)
 
+#define IRQ_DEBOUNCE_REG	0x218
+
 #define IRQ_MEM_SIZE		0x20
 
 #define IRQ_EDGE_RISING		0x00
@@ -109,7 +111,6 @@ struct sunxi_pinctrl_function {
 
 struct sunxi_pinctrl_group {
 	const char	*name;
-	unsigned long	config;
 	unsigned	pin;
 };
 
@@ -266,6 +267,11 @@ static inline u32 sunxi_irq_ctrl_offset(u16 irq)
 	return irq_num * IRQ_CTRL_IRQ_BITS;
 }
 
+static inline u32 sunxi_irq_debounce_reg_from_bank(u8 bank, unsigned bank_base)
+{
+	return IRQ_DEBOUNCE_REG + (bank_base + bank) * IRQ_MEM_SIZE;
+}
+
 static inline u32 sunxi_irq_status_reg_from_bank(u8 bank, unsigned bank_base)
 {
 	return IRQ_STATUS_REG + (bank_base + bank) * IRQ_MEM_SIZE;
diff --git a/drivers/pinctrl/vt8500/pinctrl-vt8500.c b/drivers/pinctrl/vt8500/pinctrl-vt8500.c
index ca946b3dbdb4..767f340d6b11 100644
--- a/drivers/pinctrl/vt8500/pinctrl-vt8500.c
+++ b/drivers/pinctrl/vt8500/pinctrl-vt8500.c
@@ -14,7 +14,7 @@
  */
 
 #include <linux/io.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
@@ -473,11 +473,6 @@ static int vt8500_pinctrl_probe(struct platform_device *pdev)
 	return wmt_pinctrl_probe(pdev, data);
 }
 
-static int vt8500_pinctrl_remove(struct platform_device *pdev)
-{
-	return wmt_pinctrl_remove(pdev);
-}
-
 static const struct of_device_id wmt_pinctrl_of_match[] = {
 	{ .compatible = "via,vt8500-pinctrl" },
 	{ /* sentinel */ },
@@ -485,16 +480,10 @@ static const struct of_device_id wmt_pinctrl_of_match[] = {
 
 static struct platform_driver wmt_pinctrl_driver = {
 	.probe	= vt8500_pinctrl_probe,
-	.remove	= vt8500_pinctrl_remove,
 	.driver = {
 		.name	= "pinctrl-vt8500",
 		.of_match_table	= wmt_pinctrl_of_match,
+		.suppress_bind_attrs = true,
 	},
 };
-
-module_platform_driver(wmt_pinctrl_driver);
-
-MODULE_AUTHOR("Tony Prisk <linux@prisktech.co.nz>");
-MODULE_DESCRIPTION("VIA VT8500 Pincontrol driver");
-MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, wmt_pinctrl_of_match);
+builtin_platform_driver(wmt_pinctrl_driver);
diff --git a/drivers/pinctrl/vt8500/pinctrl-wm8505.c b/drivers/pinctrl/vt8500/pinctrl-wm8505.c
index 626fc7ec0174..a56fdbd87e42 100644
--- a/drivers/pinctrl/vt8500/pinctrl-wm8505.c
+++ b/drivers/pinctrl/vt8500/pinctrl-wm8505.c
@@ -14,7 +14,7 @@
  */
 
 #include <linux/io.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
@@ -504,11 +504,6 @@ static int wm8505_pinctrl_probe(struct platform_device *pdev)
 	return wmt_pinctrl_probe(pdev, data);
 }
 
-static int wm8505_pinctrl_remove(struct platform_device *pdev)
-{
-	return wmt_pinctrl_remove(pdev);
-}
-
 static const struct of_device_id wmt_pinctrl_of_match[] = {
 	{ .compatible = "wm,wm8505-pinctrl" },
 	{ /* sentinel */ },
@@ -516,16 +511,10 @@ static const struct of_device_id wmt_pinctrl_of_match[] = {
 
 static struct platform_driver wmt_pinctrl_driver = {
 	.probe	= wm8505_pinctrl_probe,
-	.remove	= wm8505_pinctrl_remove,
 	.driver = {
 		.name	= "pinctrl-wm8505",
 		.of_match_table	= wmt_pinctrl_of_match,
+		.suppress_bind_attrs = true,
 	},
 };
-
-module_platform_driver(wmt_pinctrl_driver);
-
-MODULE_AUTHOR("Tony Prisk <linux@prisktech.co.nz>");
-MODULE_DESCRIPTION("Wondermedia WM8505 Pincontrol driver");
-MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, wmt_pinctrl_of_match);
+builtin_platform_driver(wmt_pinctrl_driver);
diff --git a/drivers/pinctrl/vt8500/pinctrl-wm8650.c b/drivers/pinctrl/vt8500/pinctrl-wm8650.c
index 8953aba8bfc2..270dd491f5a1 100644
--- a/drivers/pinctrl/vt8500/pinctrl-wm8650.c
+++ b/drivers/pinctrl/vt8500/pinctrl-wm8650.c
@@ -14,7 +14,7 @@
  */
 
 #include <linux/io.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
@@ -342,11 +342,6 @@ static int wm8650_pinctrl_probe(struct platform_device *pdev)
 	return wmt_pinctrl_probe(pdev, data);
 }
 
-static int wm8650_pinctrl_remove(struct platform_device *pdev)
-{
-	return wmt_pinctrl_remove(pdev);
-}
-
 static const struct of_device_id wmt_pinctrl_of_match[] = {
 	{ .compatible = "wm,wm8650-pinctrl" },
 	{ /* sentinel */ },
@@ -354,16 +349,10 @@ static const struct of_device_id wmt_pinctrl_of_match[] = {
 
 static struct platform_driver wmt_pinctrl_driver = {
 	.probe	= wm8650_pinctrl_probe,
-	.remove	= wm8650_pinctrl_remove,
 	.driver = {
 		.name	= "pinctrl-wm8650",
 		.of_match_table	= wmt_pinctrl_of_match,
+		.suppress_bind_attrs = true,
 	},
 };
-
-module_platform_driver(wmt_pinctrl_driver);
-
-MODULE_AUTHOR("Tony Prisk <linux@prisktech.co.nz>");
-MODULE_DESCRIPTION("Wondermedia WM8650 Pincontrol driver");
-MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, wmt_pinctrl_of_match);
+builtin_platform_driver(wmt_pinctrl_driver);
diff --git a/drivers/pinctrl/vt8500/pinctrl-wm8750.c b/drivers/pinctrl/vt8500/pinctrl-wm8750.c
index c79053d430db..74f7b3a18f3a 100644
--- a/drivers/pinctrl/vt8500/pinctrl-wm8750.c
+++ b/drivers/pinctrl/vt8500/pinctrl-wm8750.c
@@ -14,7 +14,7 @@
  */
 
 #include <linux/io.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
@@ -381,11 +381,6 @@ static int wm8750_pinctrl_probe(struct platform_device *pdev)
 	return wmt_pinctrl_probe(pdev, data);
 }
 
-static int wm8750_pinctrl_remove(struct platform_device *pdev)
-{
-	return wmt_pinctrl_remove(pdev);
-}
-
 static const struct of_device_id wmt_pinctrl_of_match[] = {
 	{ .compatible = "wm,wm8750-pinctrl" },
 	{ /* sentinel */ },
@@ -393,16 +388,10 @@ static const struct of_device_id wmt_pinctrl_of_match[] = {
 
 static struct platform_driver wmt_pinctrl_driver = {
 	.probe	= wm8750_pinctrl_probe,
-	.remove	= wm8750_pinctrl_remove,
 	.driver = {
 		.name	= "pinctrl-wm8750",
 		.of_match_table	= wmt_pinctrl_of_match,
+		.suppress_bind_attrs = true,
 	},
 };
-
-module_platform_driver(wmt_pinctrl_driver);
-
-MODULE_AUTHOR("Tony Prisk <linux@prisktech.co.nz>");
-MODULE_DESCRIPTION("Wondermedia WM8750 Pincontrol driver");
-MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, wmt_pinctrl_of_match);
+builtin_platform_driver(wmt_pinctrl_driver);
diff --git a/drivers/pinctrl/vt8500/pinctrl-wm8850.c b/drivers/pinctrl/vt8500/pinctrl-wm8850.c
index f232b163c735..45792aa7a06e 100644
--- a/drivers/pinctrl/vt8500/pinctrl-wm8850.c
+++ b/drivers/pinctrl/vt8500/pinctrl-wm8850.c
@@ -14,7 +14,7 @@
  */
 
 #include <linux/io.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
@@ -360,11 +360,6 @@ static int wm8850_pinctrl_probe(struct platform_device *pdev)
 	return wmt_pinctrl_probe(pdev, data);
 }
 
-static int wm8850_pinctrl_remove(struct platform_device *pdev)
-{
-	return wmt_pinctrl_remove(pdev);
-}
-
 static const struct of_device_id wmt_pinctrl_of_match[] = {
 	{ .compatible = "wm,wm8850-pinctrl" },
 	{ /* sentinel */ },
@@ -372,16 +367,10 @@ static const struct of_device_id wmt_pinctrl_of_match[] = {
 
 static struct platform_driver wmt_pinctrl_driver = {
 	.probe	= wm8850_pinctrl_probe,
-	.remove	= wm8850_pinctrl_remove,
 	.driver = {
 		.name	= "pinctrl-wm8850",
 		.of_match_table	= wmt_pinctrl_of_match,
+		.suppress_bind_attrs = true,
 	},
 };
-
-module_platform_driver(wmt_pinctrl_driver);
-
-MODULE_AUTHOR("Tony Prisk <linux@prisktech.co.nz>");
-MODULE_DESCRIPTION("Wondermedia WM8850 Pincontrol driver");
-MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, wmt_pinctrl_of_match);
+builtin_platform_driver(wmt_pinctrl_driver);
diff --git a/drivers/pinctrl/vt8500/pinctrl-wmt.c b/drivers/pinctrl/vt8500/pinctrl-wmt.c
index cbc638631678..270ca2a47a8c 100644
--- a/drivers/pinctrl/vt8500/pinctrl-wmt.c
+++ b/drivers/pinctrl/vt8500/pinctrl-wmt.c
@@ -18,7 +18,6 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/irq.h>
-#include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
 #include <linux/pinctrl/consumer.h>
@@ -608,12 +607,3 @@ fail_range:
 	gpiochip_remove(&data->gpio_chip);
 	return err;
 }
-
-int wmt_pinctrl_remove(struct platform_device *pdev)
-{
-	struct wmt_pinctrl_data *data = platform_get_drvdata(pdev);
-
-	gpiochip_remove(&data->gpio_chip);
-
-	return 0;
-}
diff --git a/drivers/pinctrl/vt8500/pinctrl-wmt.h b/drivers/pinctrl/vt8500/pinctrl-wmt.h
index 41f5f2deb5d6..885613396fe7 100644
--- a/drivers/pinctrl/vt8500/pinctrl-wmt.h
+++ b/drivers/pinctrl/vt8500/pinctrl-wmt.h
@@ -76,4 +76,3 @@ struct wmt_pinctrl_data {
 
 int wmt_pinctrl_probe(struct platform_device *pdev,
 		      struct wmt_pinctrl_data *data);
-int wmt_pinctrl_remove(struct platform_device *pdev);
diff --git a/drivers/platform/chrome/cros_ec_dev.c b/drivers/platform/chrome/cros_ec_dev.c
index 8abd80dbcbed..47268ecedc4d 100644
--- a/drivers/platform/chrome/cros_ec_dev.c
+++ b/drivers/platform/chrome/cros_ec_dev.c
@@ -18,6 +18,7 @@
  */
 
 #include <linux/fs.h>
+#include <linux/mfd/core.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
@@ -87,6 +88,41 @@ exit:
 	return ret;
 }
 
+static int cros_ec_check_features(struct cros_ec_dev *ec, int feature)
+{
+	struct cros_ec_command *msg;
+	int ret;
+
+	if (ec->features[0] == -1U && ec->features[1] == -1U) {
+		/* features bitmap not read yet */
+
+		msg = kmalloc(sizeof(*msg) + sizeof(ec->features), GFP_KERNEL);
+		if (!msg)
+			return -ENOMEM;
+
+		msg->version = 0;
+		msg->command = EC_CMD_GET_FEATURES + ec->cmd_offset;
+		msg->insize = sizeof(ec->features);
+		msg->outsize = 0;
+
+		ret = cros_ec_cmd_xfer(ec->ec_dev, msg);
+		if (ret < 0 || msg->result != EC_RES_SUCCESS) {
+			dev_warn(ec->dev, "cannot get EC features: %d/%d\n",
+				 ret, msg->result);
+			memset(ec->features, 0, sizeof(ec->features));
+		}
+
+		memcpy(ec->features, msg->data, sizeof(ec->features));
+
+		dev_dbg(ec->dev, "EC features %08x %08x\n",
+			ec->features[0], ec->features[1]);
+
+		kfree(msg);
+	}
+
+	return ec->features[feature / 32] & EC_FEATURE_MASK_0(feature);
+}
+
 /* Device file ops */
 static int ec_device_open(struct inode *inode, struct file *filp)
 {
@@ -230,6 +266,123 @@ static void __remove(struct device *dev)
 	kfree(ec);
 }
 
+static void cros_ec_sensors_register(struct cros_ec_dev *ec)
+{
+	/*
+	 * Issue a command to get the number of sensor reported.
+	 * Build an array of sensors driver and register them all.
+	 */
+	int ret, i, id, sensor_num;
+	struct mfd_cell *sensor_cells;
+	struct cros_ec_sensor_platform *sensor_platforms;
+	int sensor_type[MOTIONSENSE_TYPE_MAX];
+	struct ec_params_motion_sense *params;
+	struct ec_response_motion_sense *resp;
+	struct cros_ec_command *msg;
+
+	msg = kzalloc(sizeof(struct cros_ec_command) +
+		      max(sizeof(*params), sizeof(*resp)), GFP_KERNEL);
+	if (msg == NULL)
+		return;
+
+	msg->version = 2;
+	msg->command = EC_CMD_MOTION_SENSE_CMD + ec->cmd_offset;
+	msg->outsize = sizeof(*params);
+	msg->insize = sizeof(*resp);
+
+	params = (struct ec_params_motion_sense *)msg->data;
+	params->cmd = MOTIONSENSE_CMD_DUMP;
+
+	ret = cros_ec_cmd_xfer(ec->ec_dev, msg);
+	if (ret < 0 || msg->result != EC_RES_SUCCESS) {
+		dev_warn(ec->dev, "cannot get EC sensor information: %d/%d\n",
+			 ret, msg->result);
+		goto error;
+	}
+
+	resp = (struct ec_response_motion_sense *)msg->data;
+	sensor_num = resp->dump.sensor_count;
+	/* Allocate 2 extra sensors in case lid angle or FIFO are needed */
+	sensor_cells = kzalloc(sizeof(struct mfd_cell) * (sensor_num + 2),
+			       GFP_KERNEL);
+	if (sensor_cells == NULL)
+		goto error;
+
+	sensor_platforms = kzalloc(sizeof(struct cros_ec_sensor_platform) *
+		  (sensor_num + 1), GFP_KERNEL);
+	if (sensor_platforms == NULL)
+		goto error_platforms;
+
+	memset(sensor_type, 0, sizeof(sensor_type));
+	id = 0;
+	for (i = 0; i < sensor_num; i++) {
+		params->cmd = MOTIONSENSE_CMD_INFO;
+		params->info.sensor_num = i;
+		ret = cros_ec_cmd_xfer(ec->ec_dev, msg);
+		if (ret < 0 || msg->result != EC_RES_SUCCESS) {
+			dev_warn(ec->dev, "no info for EC sensor %d : %d/%d\n",
+				 i, ret, msg->result);
+			continue;
+		}
+		switch (resp->info.type) {
+		case MOTIONSENSE_TYPE_ACCEL:
+			sensor_cells[id].name = "cros-ec-accel";
+			break;
+		case MOTIONSENSE_TYPE_GYRO:
+			sensor_cells[id].name = "cros-ec-gyro";
+			break;
+		case MOTIONSENSE_TYPE_MAG:
+			sensor_cells[id].name = "cros-ec-mag";
+			break;
+		case MOTIONSENSE_TYPE_PROX:
+			sensor_cells[id].name = "cros-ec-prox";
+			break;
+		case MOTIONSENSE_TYPE_LIGHT:
+			sensor_cells[id].name = "cros-ec-light";
+			break;
+		case MOTIONSENSE_TYPE_ACTIVITY:
+			sensor_cells[id].name = "cros-ec-activity";
+			break;
+		default:
+			dev_warn(ec->dev, "unknown type %d\n", resp->info.type);
+			continue;
+		}
+		sensor_platforms[id].sensor_num = i;
+		sensor_cells[id].id = sensor_type[resp->info.type];
+		sensor_cells[id].platform_data = &sensor_platforms[id];
+		sensor_cells[id].pdata_size =
+			sizeof(struct cros_ec_sensor_platform);
+
+		sensor_type[resp->info.type]++;
+		id++;
+	}
+	if (sensor_type[MOTIONSENSE_TYPE_ACCEL] >= 2) {
+		sensor_platforms[id].sensor_num = sensor_num;
+
+		sensor_cells[id].name = "cros-ec-angle";
+		sensor_cells[id].id = 0;
+		sensor_cells[id].platform_data = &sensor_platforms[id];
+		sensor_cells[id].pdata_size =
+			sizeof(struct cros_ec_sensor_platform);
+		id++;
+	}
+	if (cros_ec_check_features(ec, EC_FEATURE_MOTION_SENSE_FIFO)) {
+		sensor_cells[id].name = "cros-ec-ring";
+		id++;
+	}
+
+	ret = mfd_add_devices(ec->dev, 0, sensor_cells, id,
+			      NULL, 0, NULL);
+	if (ret)
+		dev_err(ec->dev, "failed to add EC sensors\n");
+
+	kfree(sensor_platforms);
+error_platforms:
+	kfree(sensor_cells);
+error:
+	kfree(msg);
+}
+
 static int ec_device_probe(struct platform_device *pdev)
 {
 	int retval = -ENOMEM;
@@ -245,6 +398,8 @@ static int ec_device_probe(struct platform_device *pdev)
 	ec->ec_dev = dev_get_drvdata(dev->parent);
 	ec->dev = dev;
 	ec->cmd_offset = ec_platform->cmd_offset;
+	ec->features[0] = -1U; /* Not cached yet */
+	ec->features[1] = -1U; /* Not cached yet */
 	device_initialize(&ec->class_dev);
 	cdev_init(&ec->cdev, &fops);
 
@@ -282,6 +437,10 @@ static int ec_device_probe(struct platform_device *pdev)
 		goto dev_reg_failed;
 	}
 
+	/* check whether this EC is a sensor hub. */
+	if (cros_ec_check_features(ec, EC_FEATURE_MOTION_SENSE))
+		cros_ec_sensors_register(ec);
+
 	return 0;
 
 dev_reg_failed:
diff --git a/drivers/platform/goldfish/goldfish_pipe.c b/drivers/platform/goldfish/goldfish_pipe.c
index 1aba2c74160e..2b21033f11f0 100644
--- a/drivers/platform/goldfish/goldfish_pipe.c
+++ b/drivers/platform/goldfish/goldfish_pipe.c
@@ -308,10 +308,8 @@ static ssize_t goldfish_pipe_read_write(struct file *filp, char __user *buffer,
 		 * returns a small amount, then there's no need to pin that
 		 * much memory to the process.
 		 */
-		down_read(&current->mm->mmap_sem);
-		ret = get_user_pages(address, 1, is_write ? 0 : FOLL_WRITE,
-				&page, NULL);
-		up_read(&current->mm->mmap_sem);
+		ret = get_user_pages_unlocked(address, 1, &page,
+				is_write ? 0 : FOLL_WRITE);
 		if (ret < 0)
 			break;
 
diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig
index b8a21d7b25d4..185376901d9c 100644
--- a/drivers/platform/x86/Kconfig
+++ b/drivers/platform/x86/Kconfig
@@ -1027,4 +1027,15 @@ config INTEL_TELEMETRY
 	  used to get various SoC events and parameters
 	  directly via debugfs files. Various tools may use
 	  this interface for SoC state monitoring.
+
+config MLX_CPLD_PLATFORM
+	tristate "Mellanox platform hotplug driver support"
+	default n
+	depends on MLX_PLATFORM
+	select HWMON
+	select I2C
+	---help---
+	  This driver handles hot-plug events for the power suppliers, power
+	  cables and fans on the wide range Mellanox IB and Ethernet systems.
+
 endif # X86_PLATFORM_DEVICES
diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile
index 2efa86d2a1a7..1f06b6339cf7 100644
--- a/drivers/platform/x86/Makefile
+++ b/drivers/platform/x86/Makefile
@@ -71,3 +71,4 @@ obj-$(CONFIG_INTEL_TELEMETRY)	+= intel_telemetry_core.o \
 				   intel_telemetry_pltdrv.o \
 				   intel_telemetry_debugfs.o
 obj-$(CONFIG_INTEL_PMC_CORE)    += intel_pmc_core.o
+obj-$(CONFIG_MLX_CPLD_PLATFORM)	+= mlxcpld-hotplug.o
diff --git a/drivers/platform/x86/acer-wmi.c b/drivers/platform/x86/acer-wmi.c
index 79d64ea00bfb..a66192f692e3 100644
--- a/drivers/platform/x86/acer-wmi.c
+++ b/drivers/platform/x86/acer-wmi.c
@@ -355,6 +355,32 @@ static const struct dmi_system_id acer_blacklist[] __initconst = {
 	{}
 };
 
+static const struct dmi_system_id amw0_whitelist[] __initconst = {
+	{
+		.ident = "Acer",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+		},
+	},
+	{
+		.ident = "Gateway",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Gateway"),
+		},
+	},
+	{
+		.ident = "Packard Bell",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Packard Bell"),
+		},
+	},
+	{}
+};
+
+/*
+ * This quirk table is only for Acer/Gateway/Packard Bell family
+ * that those machines are supported by acer-wmi driver.
+ */
 static const struct dmi_system_id acer_quirks[] __initconst = {
 	{
 		.callback = dmi_matched,
@@ -464,6 +490,17 @@ static const struct dmi_system_id acer_quirks[] __initconst = {
 		},
 		.driver_data = &quirk_acer_travelmate_2490,
 	},
+	{}
+};
+
+/*
+ * This quirk list is for those non-acer machines that have AMW0_GUID1
+ * but supported by acer-wmi in past days. Keeping this quirk list here
+ * is only for backward compatible. Please do not add new machine to
+ * here anymore. Those non-acer machines should be supported by
+ * appropriate wmi drivers.
+ */
+static const struct dmi_system_id non_acer_quirks[] __initconst = {
 	{
 		.callback = dmi_matched,
 		.ident = "Fujitsu Siemens Amilo Li 1718",
@@ -598,6 +635,7 @@ static void __init find_quirks(void)
 {
 	if (!force_series) {
 		dmi_check_system(acer_quirks);
+		dmi_check_system(non_acer_quirks);
 	} else if (force_series == 2490) {
 		quirks = &quirk_acer_travelmate_2490;
 	}
@@ -2108,6 +2146,24 @@ static int __init acer_wmi_init(void)
 	find_quirks();
 
 	/*
+	 * The AMW0_GUID1 wmi is not only found on Acer family but also other
+	 * machines like Lenovo, Fujitsu and Medion. In the past days,
+	 * acer-wmi driver handled those non-Acer machines by quirks list.
+	 * But actually acer-wmi driver was loaded on any machines that have
+	 * AMW0_GUID1. This behavior is strange because those machines should
+	 * be supported by appropriate wmi drivers. e.g. fujitsu-laptop,
+	 * ideapad-laptop. So, here checks the machine that has AMW0_GUID1
+	 * should be in Acer/Gateway/Packard Bell white list, or it's already
+	 * in the past quirk list.
+	 */
+	if (wmi_has_guid(AMW0_GUID1) &&
+	    !dmi_check_system(amw0_whitelist) &&
+	    quirks == &quirk_unknown) {
+		pr_err("Unsupported machine has AMW0_GUID1, unable to load\n");
+		return -ENODEV;
+	}
+
+	/*
 	 * Detect which ACPI-WMI interface we're using.
 	 */
 	if (wmi_has_guid(AMW0_GUID1) && wmi_has_guid(WMID_GUID1))
diff --git a/drivers/platform/x86/asus-nb-wmi.c b/drivers/platform/x86/asus-nb-wmi.c
index 26e4cbc34db8..5be4783e40d4 100644
--- a/drivers/platform/x86/asus-nb-wmi.c
+++ b/drivers/platform/x86/asus-nb-wmi.c
@@ -116,8 +116,13 @@ static struct quirk_entry quirk_asus_ux303ub = {
 	.wmi_backlight_native = true,
 };
 
+static struct quirk_entry quirk_asus_x550lb = {
+	.xusb2pr = 0x01D9,
+};
+
 static int dmi_matched(const struct dmi_system_id *dmi)
 {
+	pr_info("Identified laptop model '%s'\n", dmi->ident);
 	quirks = dmi->driver_data;
 	return 1;
 }
@@ -175,6 +180,15 @@ static const struct dmi_system_id asus_quirks[] = {
 	},
 	{
 		.callback = dmi_matched,
+		.ident = "ASUSTeK COMPUTER INC. X45U",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "X45U"),
+		},
+		.driver_data = &quirk_asus_wapf4,
+	},
+	{
+		.callback = dmi_matched,
 		.ident = "ASUSTeK COMPUTER INC. X456UA",
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
@@ -398,6 +412,15 @@ static const struct dmi_system_id asus_quirks[] = {
 		},
 		.driver_data = &quirk_asus_ux303ub,
 	},
+	{
+		.callback = dmi_matched,
+		.ident = "ASUSTeK COMPUTER INC. X550LB",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "X550LB"),
+		},
+		.driver_data = &quirk_asus_x550lb,
+	},
 	{},
 };
 
diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c
index ce6ca31a2d09..43cb680adbb4 100644
--- a/drivers/platform/x86/asus-wmi.c
+++ b/drivers/platform/x86/asus-wmi.c
@@ -156,6 +156,9 @@ MODULE_LICENSE("GPL");
 #define ASUS_FAN_CTRL_MANUAL		1
 #define ASUS_FAN_CTRL_AUTO		2
 
+#define USB_INTEL_XUSB2PR		0xD0
+#define PCI_DEVICE_ID_INTEL_LYNXPOINT_LP_XHCI	0x9c31
+
 struct bios_args {
 	u32 arg0;
 	u32 arg1;
@@ -1080,6 +1083,29 @@ exit:
 	return result;
 }
 
+static void asus_wmi_set_xusb2pr(struct asus_wmi *asus)
+{
+	struct pci_dev *xhci_pdev;
+	u32 orig_ports_available;
+	u32 ports_available = asus->driver->quirks->xusb2pr;
+
+	xhci_pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
+			PCI_DEVICE_ID_INTEL_LYNXPOINT_LP_XHCI,
+			NULL);
+
+	if (!xhci_pdev)
+		return;
+
+	pci_read_config_dword(xhci_pdev, USB_INTEL_XUSB2PR,
+				&orig_ports_available);
+
+	pci_write_config_dword(xhci_pdev, USB_INTEL_XUSB2PR,
+				cpu_to_le32(ports_available));
+
+	pr_info("set USB_INTEL_XUSB2PR old: 0x%04x, new: 0x%04x\n",
+			orig_ports_available, ports_available);
+}
+
 /*
  * Hwmon device
  */
@@ -2087,6 +2113,9 @@ static int asus_wmi_add(struct platform_device *pdev)
 	if (asus->driver->quirks->wmi_backlight_native)
 		acpi_video_set_dmi_backlight_type(acpi_backlight_native);
 
+	if (asus->driver->quirks->xusb2pr)
+		asus_wmi_set_xusb2pr(asus);
+
 	if (acpi_video_get_backlight_type() == acpi_backlight_vendor) {
 		err = asus_wmi_backlight_init(asus);
 		if (err && err != -ENODEV)
diff --git a/drivers/platform/x86/asus-wmi.h b/drivers/platform/x86/asus-wmi.h
index 0e19014e9f54..fdff626c3b51 100644
--- a/drivers/platform/x86/asus-wmi.h
+++ b/drivers/platform/x86/asus-wmi.h
@@ -53,6 +53,7 @@ struct quirk_entry {
 	 * and let the ACPI interrupt to send out the key event.
 	 */
 	int no_display_toggle;
+	u32 xusb2pr;
 
 	bool (*i8042_filter)(unsigned char data, unsigned char str,
 			     struct serio *serio);
diff --git a/drivers/platform/x86/dell-laptop.c b/drivers/platform/x86/dell-laptop.c
index 2c2f02b2e08a..14392a01ab36 100644
--- a/drivers/platform/x86/dell-laptop.c
+++ b/drivers/platform/x86/dell-laptop.c
@@ -1904,38 +1904,40 @@ static enum led_brightness kbd_led_level_get(struct led_classdev *led_cdev)
 	return 0;
 }
 
-static void kbd_led_level_set(struct led_classdev *led_cdev,
-			      enum led_brightness value)
+static int kbd_led_level_set(struct led_classdev *led_cdev,
+			     enum led_brightness value)
 {
 	struct kbd_state state;
 	struct kbd_state new_state;
 	u16 num;
+	int ret;
 
 	if (kbd_get_max_level()) {
-		if (kbd_get_state(&state))
-			return;
+		ret = kbd_get_state(&state);
+		if (ret)
+			return ret;
 		new_state = state;
-		if (kbd_set_level(&new_state, value))
-			return;
-		kbd_set_state_safe(&new_state, &state);
-		return;
+		ret = kbd_set_level(&new_state, value);
+		if (ret)
+			return ret;
+		return kbd_set_state_safe(&new_state, &state);
 	}
 
 	if (kbd_get_valid_token_counts()) {
 		for (num = kbd_token_bits; num != 0 && value > 0; --value)
 			num &= num - 1; /* clear the first bit set */
 		if (num == 0)
-			return;
-		kbd_set_token_bit(ffs(num) - 1);
-		return;
+			return 0;
+		return kbd_set_token_bit(ffs(num) - 1);
 	}
 
 	pr_warn("Keyboard brightness level control not supported\n");
+	return -ENXIO;
 }
 
 static struct led_classdev kbd_led = {
 	.name           = "dell::kbd_backlight",
-	.brightness_set = kbd_led_level_set,
+	.brightness_set_blocking = kbd_led_level_set,
 	.brightness_get = kbd_led_level_get,
 	.groups         = kbd_led_groups,
 };
diff --git a/drivers/platform/x86/dell-wmi.c b/drivers/platform/x86/dell-wmi.c
index da2fe18162e1..75e637047d36 100644
--- a/drivers/platform/x86/dell-wmi.c
+++ b/drivers/platform/x86/dell-wmi.c
@@ -114,7 +114,7 @@ static const struct key_entry dell_wmi_keymap_type_0000[] __initconst = {
 	{ KE_IGNORE, 0xe00e, { KEY_RESERVED } },
 
 	/* Wifi Catcher */
-	{ KE_KEY,    0xe011, { KEY_PROG2 } },
+	{ KE_KEY,    0xe011, { KEY_WLAN } },
 
 	/* Ambient light sensor toggle */
 	{ KE_IGNORE, 0xe013, { KEY_RESERVED } },
@@ -274,6 +274,16 @@ static const struct key_entry dell_wmi_keymap_type_0010[] __initconst = {
 
 	/* Stealth mode toggle */
 	{ KE_IGNORE, 0x155, { KEY_RESERVED } },
+
+	/* Rugged magnetic dock attach/detach events */
+	{ KE_IGNORE, 0x156, { KEY_RESERVED } },
+	{ KE_IGNORE, 0x157, { KEY_RESERVED } },
+
+	/* Rugged programmable (P1/P2/P3 keys) */
+	{ KE_KEY,    0x850, { KEY_PROG1 } },
+	{ KE_KEY,    0x851, { KEY_PROG2 } },
+	{ KE_KEY,    0x852, { KEY_PROG3 } },
+
 };
 
 /*
diff --git a/drivers/platform/x86/intel-hid.c b/drivers/platform/x86/intel-hid.c
index 12dbb5063376..cb3ab2b212b1 100644
--- a/drivers/platform/x86/intel-hid.c
+++ b/drivers/platform/x86/intel-hid.c
@@ -69,7 +69,7 @@ static int intel_hid_set_enable(struct device *device, int enable)
 
 	arg0.integer.value = enable;
 	status = acpi_evaluate_object(ACPI_HANDLE(device), "HDSM", &args, NULL);
-	if (!ACPI_SUCCESS(status)) {
+	if (ACPI_FAILURE(status)) {
 		dev_warn(device, "failed to %sable hotkeys\n",
 			 enable ? "en" : "dis");
 		return -EIO;
@@ -148,7 +148,7 @@ static void notify_handler(acpi_handle handle, u32 event, void *context)
 	}
 
 	status = acpi_evaluate_integer(handle, "HDEM", NULL, &ev_index);
-	if (!ACPI_SUCCESS(status)) {
+	if (ACPI_FAILURE(status)) {
 		dev_warn(&device->dev, "failed to get event index\n");
 		return;
 	}
@@ -167,7 +167,7 @@ static int intel_hid_probe(struct platform_device *device)
 	int err;
 
 	status = acpi_evaluate_integer(handle, "HDMM", NULL, &mode);
-	if (!ACPI_SUCCESS(status)) {
+	if (ACPI_FAILURE(status)) {
 		dev_warn(&device->dev, "failed to read mode\n");
 		return -ENODEV;
 	}
diff --git a/drivers/platform/x86/intel-smartconnect.c b/drivers/platform/x86/intel-smartconnect.c
index 04cf5dffdfd9..bbe4c06c769f 100644
--- a/drivers/platform/x86/intel-smartconnect.c
+++ b/drivers/platform/x86/intel-smartconnect.c
@@ -29,7 +29,7 @@ static int smartconnect_acpi_init(struct acpi_device *acpi)
 	acpi_status status;
 
 	status = acpi_evaluate_integer(acpi->handle, "GAOS", NULL, &value);
-	if (!ACPI_SUCCESS(status))
+	if (ACPI_FAILURE(status))
 		return -EINVAL;
 
 	if (value & 0x1) {
diff --git a/drivers/platform/x86/intel-vbtn.c b/drivers/platform/x86/intel-vbtn.c
index 78080763df51..554e82ebe83c 100644
--- a/drivers/platform/x86/intel-vbtn.c
+++ b/drivers/platform/x86/intel-vbtn.c
@@ -49,34 +49,19 @@ static int intel_vbtn_input_setup(struct platform_device *device)
 	struct intel_vbtn_priv *priv = dev_get_drvdata(&device->dev);
 	int ret;
 
-	priv->input_dev = input_allocate_device();
+	priv->input_dev = devm_input_allocate_device(&device->dev);
 	if (!priv->input_dev)
 		return -ENOMEM;
 
 	ret = sparse_keymap_setup(priv->input_dev, intel_vbtn_keymap, NULL);
 	if (ret)
-		goto err_free_device;
+		return ret;
 
 	priv->input_dev->dev.parent = &device->dev;
 	priv->input_dev->name = "Intel Virtual Button driver";
 	priv->input_dev->id.bustype = BUS_HOST;
 
-	ret = input_register_device(priv->input_dev);
-	if (ret)
-		goto err_free_device;
-
-	return 0;
-
-err_free_device:
-	input_free_device(priv->input_dev);
-	return ret;
-}
-
-static void intel_vbtn_input_destroy(struct platform_device *device)
-{
-	struct intel_vbtn_priv *priv = dev_get_drvdata(&device->dev);
-
-	input_unregister_device(priv->input_dev);
+	return input_register_device(priv->input_dev);
 }
 
 static void notify_handler(acpi_handle handle, u32 event, void *context)
@@ -97,7 +82,7 @@ static int intel_vbtn_probe(struct platform_device *device)
 	int err;
 
 	status = acpi_evaluate_object(handle, "VBDL", NULL, NULL);
-	if (!ACPI_SUCCESS(status)) {
+	if (ACPI_FAILURE(status)) {
 		dev_warn(&device->dev, "failed to read Intel Virtual Button driver\n");
 		return -ENODEV;
 	}
@@ -117,24 +102,16 @@ static int intel_vbtn_probe(struct platform_device *device)
 					     ACPI_DEVICE_NOTIFY,
 					     notify_handler,
 					     device);
-	if (ACPI_FAILURE(status)) {
-		err = -EBUSY;
-		goto err_remove_input;
-	}
+	if (ACPI_FAILURE(status))
+		return -EBUSY;
 
 	return 0;
-
-err_remove_input:
-	intel_vbtn_input_destroy(device);
-
-	return err;
 }
 
 static int intel_vbtn_remove(struct platform_device *device)
 {
 	acpi_handle handle = ACPI_HANDLE(&device->dev);
 
-	intel_vbtn_input_destroy(device);
 	acpi_remove_notify_handler(handle, ACPI_DEVICE_NOTIFY, notify_handler);
 
 	/*
diff --git a/drivers/platform/x86/intel_mid_thermal.c b/drivers/platform/x86/intel_mid_thermal.c
index 9f713b832ba3..0df3c9d37509 100644
--- a/drivers/platform/x86/intel_mid_thermal.c
+++ b/drivers/platform/x86/intel_mid_thermal.c
@@ -415,6 +415,7 @@ static struct thermal_device_info *initialize_sensor(int index)
 	return td_info;
 }
 
+#ifdef CONFIG_PM_SLEEP
 /**
  * mid_thermal_resume - resume routine
  * @dev: device structure
@@ -442,6 +443,7 @@ static int mid_thermal_suspend(struct device *dev)
 	 */
 	return configure_adc(0);
 }
+#endif
 
 static SIMPLE_DEV_PM_OPS(mid_thermal_pm,
 			 mid_thermal_suspend, mid_thermal_resume);
diff --git a/drivers/platform/x86/intel_pmc_core.c b/drivers/platform/x86/intel_pmc_core.c
index e8b1b836ca2d..b130b8c9b9d7 100644
--- a/drivers/platform/x86/intel_pmc_core.c
+++ b/drivers/platform/x86/intel_pmc_core.c
@@ -19,10 +19,12 @@
  */
 
 #include <linux/debugfs.h>
+#include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/pci.h>
+#include <linux/uaccess.h>
 
 #include <asm/cpu_device_id.h>
 #include <asm/intel-family.h>
@@ -32,16 +34,106 @@
 
 static struct pmc_dev pmc;
 
+static const struct pmc_bit_map spt_pll_map[] = {
+	{"MIPI PLL",			SPT_PMC_BIT_MPHY_CMN_LANE0},
+	{"GEN2 USB2PCIE2 PLL",		SPT_PMC_BIT_MPHY_CMN_LANE1},
+	{"DMIPCIE3 PLL",		SPT_PMC_BIT_MPHY_CMN_LANE2},
+	{"SATA PLL",			SPT_PMC_BIT_MPHY_CMN_LANE3},
+	{},
+};
+
+static const struct pmc_bit_map spt_mphy_map[] = {
+	{"MPHY CORE LANE 0",           SPT_PMC_BIT_MPHY_LANE0},
+	{"MPHY CORE LANE 1",           SPT_PMC_BIT_MPHY_LANE1},
+	{"MPHY CORE LANE 2",           SPT_PMC_BIT_MPHY_LANE2},
+	{"MPHY CORE LANE 3",           SPT_PMC_BIT_MPHY_LANE3},
+	{"MPHY CORE LANE 4",           SPT_PMC_BIT_MPHY_LANE4},
+	{"MPHY CORE LANE 5",           SPT_PMC_BIT_MPHY_LANE5},
+	{"MPHY CORE LANE 6",           SPT_PMC_BIT_MPHY_LANE6},
+	{"MPHY CORE LANE 7",           SPT_PMC_BIT_MPHY_LANE7},
+	{"MPHY CORE LANE 8",           SPT_PMC_BIT_MPHY_LANE8},
+	{"MPHY CORE LANE 9",           SPT_PMC_BIT_MPHY_LANE9},
+	{"MPHY CORE LANE 10",          SPT_PMC_BIT_MPHY_LANE10},
+	{"MPHY CORE LANE 11",          SPT_PMC_BIT_MPHY_LANE11},
+	{"MPHY CORE LANE 12",          SPT_PMC_BIT_MPHY_LANE12},
+	{"MPHY CORE LANE 13",          SPT_PMC_BIT_MPHY_LANE13},
+	{"MPHY CORE LANE 14",          SPT_PMC_BIT_MPHY_LANE14},
+	{"MPHY CORE LANE 15",          SPT_PMC_BIT_MPHY_LANE15},
+	{},
+};
+
+static const struct pmc_bit_map spt_pfear_map[] = {
+	{"PMC",				SPT_PMC_BIT_PMC},
+	{"OPI-DMI",			SPT_PMC_BIT_OPI},
+	{"SPI / eSPI",			SPT_PMC_BIT_SPI},
+	{"XHCI",			SPT_PMC_BIT_XHCI},
+	{"SPA",				SPT_PMC_BIT_SPA},
+	{"SPB",				SPT_PMC_BIT_SPB},
+	{"SPC",				SPT_PMC_BIT_SPC},
+	{"GBE",				SPT_PMC_BIT_GBE},
+	{"SATA",			SPT_PMC_BIT_SATA},
+	{"HDA-PGD0",			SPT_PMC_BIT_HDA_PGD0},
+	{"HDA-PGD1",			SPT_PMC_BIT_HDA_PGD1},
+	{"HDA-PGD2",			SPT_PMC_BIT_HDA_PGD2},
+	{"HDA-PGD3",			SPT_PMC_BIT_HDA_PGD3},
+	{"RSVD",			SPT_PMC_BIT_RSVD_0B},
+	{"LPSS",			SPT_PMC_BIT_LPSS},
+	{"LPC",				SPT_PMC_BIT_LPC},
+	{"SMB",				SPT_PMC_BIT_SMB},
+	{"ISH",				SPT_PMC_BIT_ISH},
+	{"P2SB",			SPT_PMC_BIT_P2SB},
+	{"DFX",				SPT_PMC_BIT_DFX},
+	{"SCC",				SPT_PMC_BIT_SCC},
+	{"RSVD",			SPT_PMC_BIT_RSVD_0C},
+	{"FUSE",			SPT_PMC_BIT_FUSE},
+	{"CAMERA",			SPT_PMC_BIT_CAMREA},
+	{"RSVD",			SPT_PMC_BIT_RSVD_0D},
+	{"USB3-OTG",			SPT_PMC_BIT_USB3_OTG},
+	{"EXI",				SPT_PMC_BIT_EXI},
+	{"CSE",				SPT_PMC_BIT_CSE},
+	{"CSME_KVM",			SPT_PMC_BIT_CSME_KVM},
+	{"CSME_PMT",			SPT_PMC_BIT_CSME_PMT},
+	{"CSME_CLINK",			SPT_PMC_BIT_CSME_CLINK},
+	{"CSME_PTIO",			SPT_PMC_BIT_CSME_PTIO},
+	{"CSME_USBR",			SPT_PMC_BIT_CSME_USBR},
+	{"CSME_SUSRAM",			SPT_PMC_BIT_CSME_SUSRAM},
+	{"CSME_SMT",			SPT_PMC_BIT_CSME_SMT},
+	{"RSVD",			SPT_PMC_BIT_RSVD_1A},
+	{"CSME_SMS2",			SPT_PMC_BIT_CSME_SMS2},
+	{"CSME_SMS1",			SPT_PMC_BIT_CSME_SMS1},
+	{"CSME_RTC",			SPT_PMC_BIT_CSME_RTC},
+	{"CSME_PSF",			SPT_PMC_BIT_CSME_PSF},
+	{},
+};
+
+static const struct pmc_reg_map spt_reg_map = {
+	.pfear_sts = spt_pfear_map,
+	.mphy_sts = spt_mphy_map,
+	.pll_sts = spt_pll_map,
+};
+
 static const struct pci_device_id pmc_pci_ids[] = {
-	{ PCI_VDEVICE(INTEL, SPT_PMC_PCI_DEVICE_ID), (kernel_ulong_t)NULL },
+	{ PCI_VDEVICE(INTEL, SPT_PMC_PCI_DEVICE_ID),
+					(kernel_ulong_t)&spt_reg_map },
 	{ 0, },
 };
 
+static inline u8 pmc_core_reg_read_byte(struct pmc_dev *pmcdev, int offset)
+{
+	return readb(pmcdev->regbase + offset);
+}
+
 static inline u32 pmc_core_reg_read(struct pmc_dev *pmcdev, int reg_offset)
 {
 	return readl(pmcdev->regbase + reg_offset);
 }
 
+static inline void pmc_core_reg_write(struct pmc_dev *pmcdev, int
+							reg_offset, u32 val)
+{
+	writel(val, pmcdev->regbase + reg_offset);
+}
+
 static inline u32 pmc_core_adjust_slp_s0_step(u32 value)
 {
 	return value * SPT_PMC_SLP_S0_RES_COUNTER_STEP;
@@ -90,6 +182,245 @@ static int pmc_core_dev_state_get(void *data, u64 *val)
 
 DEFINE_DEBUGFS_ATTRIBUTE(pmc_core_dev_state, pmc_core_dev_state_get, NULL, "%llu\n");
 
+static int pmc_core_check_read_lock_bit(void)
+{
+	struct pmc_dev *pmcdev = &pmc;
+	u32 value;
+
+	value = pmc_core_reg_read(pmcdev, SPT_PMC_PM_CFG_OFFSET);
+	return test_bit(SPT_PMC_READ_DISABLE_BIT,
+			(unsigned long *)&value);
+}
+
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+static void pmc_core_display_map(struct seq_file *s, int index,
+				 u8 pf_reg, const struct pmc_bit_map *pf_map)
+{
+	seq_printf(s, "PCH IP: %-2d - %-32s\tState: %s\n",
+		   index, pf_map[index].name,
+		   pf_map[index].bit_mask & pf_reg ? "Off" : "On");
+}
+
+static int pmc_core_ppfear_sts_show(struct seq_file *s, void *unused)
+{
+	struct pmc_dev *pmcdev = s->private;
+	const struct pmc_bit_map *map = pmcdev->map->pfear_sts;
+	u8 pf_regs[NUM_ENTRIES];
+	int index, iter;
+
+	iter = SPT_PMC_XRAM_PPFEAR0A;
+
+	for (index = 0; index < NUM_ENTRIES; index++, iter++)
+		pf_regs[index] = pmc_core_reg_read_byte(pmcdev, iter);
+
+	for (index = 0; map[index].name; index++)
+		pmc_core_display_map(s, index, pf_regs[index / 8], map);
+
+	return 0;
+}
+
+static int pmc_core_ppfear_sts_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, pmc_core_ppfear_sts_show, inode->i_private);
+}
+
+static const struct file_operations pmc_core_ppfear_ops = {
+	.open           = pmc_core_ppfear_sts_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = single_release,
+};
+
+/* This function should return link status, 0 means ready */
+static int pmc_core_mtpmc_link_status(void)
+{
+	struct pmc_dev *pmcdev = &pmc;
+	u32 value;
+
+	value = pmc_core_reg_read(pmcdev, SPT_PMC_PM_STS_OFFSET);
+	return test_bit(SPT_PMC_MSG_FULL_STS_BIT,
+			(unsigned long *)&value);
+}
+
+static int pmc_core_send_msg(u32 *addr_xram)
+{
+	struct pmc_dev *pmcdev = &pmc;
+	u32 dest;
+	int timeout;
+
+	for (timeout = NUM_RETRIES; timeout > 0; timeout--) {
+		if (pmc_core_mtpmc_link_status() == 0)
+			break;
+		msleep(5);
+	}
+
+	if (timeout <= 0 && pmc_core_mtpmc_link_status())
+		return -EBUSY;
+
+	dest = (*addr_xram & MTPMC_MASK) | (1U << 1);
+	pmc_core_reg_write(pmcdev, SPT_PMC_MTPMC_OFFSET, dest);
+	return 0;
+}
+
+static int pmc_core_mphy_pg_sts_show(struct seq_file *s, void *unused)
+{
+	struct pmc_dev *pmcdev = s->private;
+	const struct pmc_bit_map *map = pmcdev->map->mphy_sts;
+	u32 mphy_core_reg_low, mphy_core_reg_high;
+	u32 val_low, val_high;
+	int index, err = 0;
+
+	if (pmcdev->pmc_xram_read_bit) {
+		seq_puts(s, "Access denied: please disable PMC_READ_DISABLE setting in BIOS.");
+		return 0;
+	}
+
+	mphy_core_reg_low  = (SPT_PMC_MPHY_CORE_STS_0 << 16);
+	mphy_core_reg_high = (SPT_PMC_MPHY_CORE_STS_1 << 16);
+
+	mutex_lock(&pmcdev->lock);
+
+	if (pmc_core_send_msg(&mphy_core_reg_low) != 0) {
+		err = -EBUSY;
+		goto out_unlock;
+	}
+
+	msleep(10);
+	val_low = pmc_core_reg_read(pmcdev, SPT_PMC_MFPMC_OFFSET);
+
+	if (pmc_core_send_msg(&mphy_core_reg_high) != 0) {
+		err = -EBUSY;
+		goto out_unlock;
+	}
+
+	msleep(10);
+	val_high = pmc_core_reg_read(pmcdev, SPT_PMC_MFPMC_OFFSET);
+
+	for (index = 0; map[index].name && index < 8; index++) {
+		seq_printf(s, "%-32s\tState: %s\n",
+			   map[index].name,
+			   map[index].bit_mask & val_low ? "Not power gated" :
+			   "Power gated");
+	}
+
+	for (index = 8; map[index].name; index++) {
+		seq_printf(s, "%-32s\tState: %s\n",
+			   map[index].name,
+			   map[index].bit_mask & val_high ? "Not power gated" :
+			   "Power gated");
+	}
+
+out_unlock:
+	mutex_unlock(&pmcdev->lock);
+	return err;
+}
+
+static int pmc_core_mphy_pg_sts_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, pmc_core_mphy_pg_sts_show, inode->i_private);
+}
+
+static const struct file_operations pmc_core_mphy_pg_ops = {
+	.open           = pmc_core_mphy_pg_sts_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = single_release,
+};
+
+static int pmc_core_pll_show(struct seq_file *s, void *unused)
+{
+	struct pmc_dev *pmcdev = s->private;
+	const struct pmc_bit_map *map = pmcdev->map->pll_sts;
+	u32 mphy_common_reg, val;
+	int index, err = 0;
+
+	if (pmcdev->pmc_xram_read_bit) {
+		seq_puts(s, "Access denied: please disable PMC_READ_DISABLE setting in BIOS.");
+		return 0;
+	}
+
+	mphy_common_reg  = (SPT_PMC_MPHY_COM_STS_0 << 16);
+	mutex_lock(&pmcdev->lock);
+
+	if (pmc_core_send_msg(&mphy_common_reg) != 0) {
+		err = -EBUSY;
+		goto out_unlock;
+	}
+
+	/* Observed PMC HW response latency for MTPMC-MFPMC is ~10 ms */
+	msleep(10);
+	val = pmc_core_reg_read(pmcdev, SPT_PMC_MFPMC_OFFSET);
+
+	for (index = 0; map[index].name ; index++) {
+		seq_printf(s, "%-32s\tState: %s\n",
+			   map[index].name,
+			   map[index].bit_mask & val ? "Active" : "Idle");
+	}
+
+out_unlock:
+	mutex_unlock(&pmcdev->lock);
+	return err;
+}
+
+static int pmc_core_pll_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, pmc_core_pll_show, inode->i_private);
+}
+
+static const struct file_operations pmc_core_pll_ops = {
+	.open           = pmc_core_pll_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = single_release,
+};
+
+static ssize_t pmc_core_ltr_ignore_write(struct file *file, const char __user
+*userbuf, size_t count, loff_t *ppos)
+{
+	struct pmc_dev *pmcdev = &pmc;
+	u32 val, buf_size, fd;
+	int err = 0;
+
+	buf_size = count < 64 ? count : 64;
+	mutex_lock(&pmcdev->lock);
+
+	if (kstrtou32_from_user(userbuf, buf_size, 10, &val)) {
+		err = -EFAULT;
+		goto out_unlock;
+	}
+
+	if (val > NUM_IP_IGN_ALLOWED) {
+		err = -EINVAL;
+		goto out_unlock;
+	}
+
+	fd = pmc_core_reg_read(pmcdev, SPT_PMC_LTR_IGNORE_OFFSET);
+	fd |= (1U << val);
+	pmc_core_reg_write(pmcdev, SPT_PMC_LTR_IGNORE_OFFSET, fd);
+
+out_unlock:
+	mutex_unlock(&pmcdev->lock);
+	return err == 0 ? count : err;
+}
+
+static int pmc_core_ltr_ignore_show(struct seq_file *s, void *unused)
+{
+	return 0;
+}
+
+static int pmc_core_ltr_ignore_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, pmc_core_ltr_ignore_show, inode->i_private);
+}
+
+static const struct file_operations pmc_core_ltr_ignore_ops = {
+	.open           = pmc_core_ltr_ignore_open,
+	.read           = seq_read,
+	.write          = pmc_core_ltr_ignore_write,
+	.llseek         = seq_lseek,
+	.release        = single_release,
+};
+
 static void pmc_core_dbgfs_unregister(struct pmc_dev *pmcdev)
 {
 	debugfs_remove_recursive(pmcdev->dbgfs_dir);
@@ -106,20 +437,59 @@ static int pmc_core_dbgfs_register(struct pmc_dev *pmcdev)
 	pmcdev->dbgfs_dir = dir;
 	file = debugfs_create_file("slp_s0_residency_usec", S_IFREG | S_IRUGO,
 				   dir, pmcdev, &pmc_core_dev_state);
+	if (!file)
+		goto err;
 
-	if (!file) {
-		pmc_core_dbgfs_unregister(pmcdev);
-		return -ENODEV;
-	}
+	file = debugfs_create_file("pch_ip_power_gating_status",
+				   S_IFREG | S_IRUGO, dir, pmcdev,
+				   &pmc_core_ppfear_ops);
+	if (!file)
+		goto err;
+
+	file = debugfs_create_file("mphy_core_lanes_power_gating_status",
+				   S_IFREG | S_IRUGO, dir, pmcdev,
+				   &pmc_core_mphy_pg_ops);
+	if (!file)
+		goto err;
+
+	file = debugfs_create_file("pll_status",
+				   S_IFREG | S_IRUGO, dir, pmcdev,
+				   &pmc_core_pll_ops);
+	if (!file)
+		goto err;
+
+	file = debugfs_create_file("ltr_ignore",
+				   S_IFREG | S_IRUGO, dir, pmcdev,
+				   &pmc_core_ltr_ignore_ops);
+
+	if (!file)
+		goto err;
 
 	return 0;
+err:
+	pmc_core_dbgfs_unregister(pmcdev);
+	return -ENODEV;
 }
+#else
+static inline int pmc_core_dbgfs_register(struct pmc_dev *pmcdev)
+{
+	return 0;
+}
+
+static inline void pmc_core_dbgfs_unregister(struct pmc_dev *pmcdev)
+{
+}
+#endif /* CONFIG_DEBUG_FS */
 
 static const struct x86_cpu_id intel_pmc_core_ids[] = {
 	{ X86_VENDOR_INTEL, 6, INTEL_FAM6_SKYLAKE_MOBILE, X86_FEATURE_MWAIT,
 		(kernel_ulong_t)NULL},
 	{ X86_VENDOR_INTEL, 6, INTEL_FAM6_SKYLAKE_DESKTOP, X86_FEATURE_MWAIT,
 		(kernel_ulong_t)NULL},
+	{ X86_VENDOR_INTEL, 6, INTEL_FAM6_KABYLAKE_MOBILE, X86_FEATURE_MWAIT,
+		(kernel_ulong_t)NULL},
+	{ X86_VENDOR_INTEL, 6, INTEL_FAM6_KABYLAKE_DESKTOP, X86_FEATURE_MWAIT,
+		(kernel_ulong_t)NULL},
 	{}
 };
 
@@ -128,6 +498,7 @@ static int pmc_core_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	struct device *ptr_dev = &dev->dev;
 	struct pmc_dev *pmcdev = &pmc;
 	const struct x86_cpu_id *cpu_id;
+	const struct pmc_reg_map *map = (struct pmc_reg_map *)id->driver_data;
 	int err;
 
 	cpu_id = x86_match_cpu(intel_pmc_core_ids);
@@ -149,6 +520,7 @@ static int pmc_core_probe(struct pci_dev *dev, const struct pci_device_id *id)
 		dev_dbg(&dev->dev, "PMC Core: failed to read PCI config space.\n");
 		return err;
 	}
+	pmcdev->base_addr &= PMC_BASE_ADDR_MASK;
 	dev_dbg(&dev->dev, "PMC Core: PWRMBASE is %#x\n", pmcdev->base_addr);
 
 	pmcdev->regbase = devm_ioremap_nocache(ptr_dev,
@@ -159,6 +531,10 @@ static int pmc_core_probe(struct pci_dev *dev, const struct pci_device_id *id)
 		return -ENOMEM;
 	}
 
+	mutex_init(&pmcdev->lock);
+	pmcdev->pmc_xram_read_bit = pmc_core_check_read_lock_bit();
+	pmcdev->map = map;
+
 	err = pmc_core_dbgfs_register(pmcdev);
 	if (err < 0)
 		dev_warn(&dev->dev, "PMC Core: debugfs register failed.\n");
diff --git a/drivers/platform/x86/intel_pmc_core.h b/drivers/platform/x86/intel_pmc_core.h
index e3f671f4d122..5a48e7728479 100644
--- a/drivers/platform/x86/intel_pmc_core.h
+++ b/drivers/platform/x86/intel_pmc_core.h
@@ -26,8 +26,111 @@
 
 #define SPT_PMC_BASE_ADDR_OFFSET		0x48
 #define SPT_PMC_SLP_S0_RES_COUNTER_OFFSET	0x13c
-#define SPT_PMC_MMIO_REG_LEN			0x100
+#define SPT_PMC_PM_CFG_OFFSET			0x18
+#define SPT_PMC_PM_STS_OFFSET			0x1c
+#define SPT_PMC_MTPMC_OFFSET			0x20
+#define SPT_PMC_MFPMC_OFFSET			0x38
+#define SPT_PMC_LTR_IGNORE_OFFSET		0x30C
+#define SPT_PMC_MPHY_CORE_STS_0			0x1143
+#define SPT_PMC_MPHY_CORE_STS_1			0x1142
+#define SPT_PMC_MPHY_COM_STS_0			0x1155
+#define SPT_PMC_MMIO_REG_LEN			0x1000
 #define SPT_PMC_SLP_S0_RES_COUNTER_STEP		0x64
+#define PMC_BASE_ADDR_MASK			~(SPT_PMC_MMIO_REG_LEN - 1)
+#define MTPMC_MASK				0xffff0000
+#define NUM_ENTRIES				5
+#define SPT_PMC_READ_DISABLE_BIT		0x16
+#define SPT_PMC_MSG_FULL_STS_BIT		0x18
+#define NUM_RETRIES				100
+#define NUM_IP_IGN_ALLOWED			17
+
+/* Sunrise Point: PGD PFET Enable Ack Status Registers */
+enum ppfear_regs {
+	SPT_PMC_XRAM_PPFEAR0A = 0x590,
+	SPT_PMC_XRAM_PPFEAR0B,
+	SPT_PMC_XRAM_PPFEAR0C,
+	SPT_PMC_XRAM_PPFEAR0D,
+	SPT_PMC_XRAM_PPFEAR1A,
+};
+
+#define SPT_PMC_BIT_PMC				BIT(0)
+#define SPT_PMC_BIT_OPI				BIT(1)
+#define SPT_PMC_BIT_SPI				BIT(2)
+#define SPT_PMC_BIT_XHCI			BIT(3)
+#define SPT_PMC_BIT_SPA				BIT(4)
+#define SPT_PMC_BIT_SPB				BIT(5)
+#define SPT_PMC_BIT_SPC				BIT(6)
+#define SPT_PMC_BIT_GBE				BIT(7)
+
+#define SPT_PMC_BIT_SATA			BIT(0)
+#define SPT_PMC_BIT_HDA_PGD0			BIT(1)
+#define SPT_PMC_BIT_HDA_PGD1			BIT(2)
+#define SPT_PMC_BIT_HDA_PGD2			BIT(3)
+#define SPT_PMC_BIT_HDA_PGD3			BIT(4)
+#define SPT_PMC_BIT_RSVD_0B			BIT(5)
+#define SPT_PMC_BIT_LPSS			BIT(6)
+#define SPT_PMC_BIT_LPC				BIT(7)
+
+#define SPT_PMC_BIT_SMB				BIT(0)
+#define SPT_PMC_BIT_ISH				BIT(1)
+#define SPT_PMC_BIT_P2SB			BIT(2)
+#define SPT_PMC_BIT_DFX				BIT(3)
+#define SPT_PMC_BIT_SCC				BIT(4)
+#define SPT_PMC_BIT_RSVD_0C			BIT(5)
+#define SPT_PMC_BIT_FUSE			BIT(6)
+#define SPT_PMC_BIT_CAMREA			BIT(7)
+
+#define SPT_PMC_BIT_RSVD_0D			BIT(0)
+#define SPT_PMC_BIT_USB3_OTG			BIT(1)
+#define SPT_PMC_BIT_EXI				BIT(2)
+#define SPT_PMC_BIT_CSE				BIT(3)
+#define SPT_PMC_BIT_CSME_KVM			BIT(4)
+#define SPT_PMC_BIT_CSME_PMT			BIT(5)
+#define SPT_PMC_BIT_CSME_CLINK			BIT(6)
+#define SPT_PMC_BIT_CSME_PTIO			BIT(7)
+
+#define SPT_PMC_BIT_CSME_USBR			BIT(0)
+#define SPT_PMC_BIT_CSME_SUSRAM			BIT(1)
+#define SPT_PMC_BIT_CSME_SMT			BIT(2)
+#define SPT_PMC_BIT_RSVD_1A			BIT(3)
+#define SPT_PMC_BIT_CSME_SMS2			BIT(4)
+#define SPT_PMC_BIT_CSME_SMS1			BIT(5)
+#define SPT_PMC_BIT_CSME_RTC			BIT(6)
+#define SPT_PMC_BIT_CSME_PSF			BIT(7)
+
+#define SPT_PMC_BIT_MPHY_LANE0			BIT(0)
+#define SPT_PMC_BIT_MPHY_LANE1			BIT(1)
+#define SPT_PMC_BIT_MPHY_LANE2			BIT(2)
+#define SPT_PMC_BIT_MPHY_LANE3			BIT(3)
+#define SPT_PMC_BIT_MPHY_LANE4			BIT(4)
+#define SPT_PMC_BIT_MPHY_LANE5			BIT(5)
+#define SPT_PMC_BIT_MPHY_LANE6			BIT(6)
+#define SPT_PMC_BIT_MPHY_LANE7			BIT(7)
+
+#define SPT_PMC_BIT_MPHY_LANE8			BIT(0)
+#define SPT_PMC_BIT_MPHY_LANE9			BIT(1)
+#define SPT_PMC_BIT_MPHY_LANE10			BIT(2)
+#define SPT_PMC_BIT_MPHY_LANE11			BIT(3)
+#define SPT_PMC_BIT_MPHY_LANE12			BIT(4)
+#define SPT_PMC_BIT_MPHY_LANE13			BIT(5)
+#define SPT_PMC_BIT_MPHY_LANE14			BIT(6)
+#define SPT_PMC_BIT_MPHY_LANE15			BIT(7)
+
+#define SPT_PMC_BIT_MPHY_CMN_LANE0		BIT(0)
+#define SPT_PMC_BIT_MPHY_CMN_LANE1		BIT(1)
+#define SPT_PMC_BIT_MPHY_CMN_LANE2		BIT(2)
+#define SPT_PMC_BIT_MPHY_CMN_LANE3		BIT(3)
+
+struct pmc_bit_map {
+	const char *name;
+	u32 bit_mask;
+};
+
+struct pmc_reg_map {
+	const struct pmc_bit_map *pfear_sts;
+	const struct pmc_bit_map *mphy_sts;
+	const struct pmc_bit_map *pll_sts;
+};
 
 /**
  * struct pmc_dev - pmc device structure
@@ -43,8 +146,13 @@
 struct pmc_dev {
 	u32 base_addr;
 	void __iomem *regbase;
+	const struct pmc_reg_map *map;
+#if IS_ENABLED(CONFIG_DEBUG_FS)
 	struct dentry *dbgfs_dir;
+#endif /* CONFIG_DEBUG_FS */
 	bool has_slp_s0_res;
+	int pmc_xram_read_bit;
+	struct mutex lock; /* generic mutex lock for PMC Core */
 };
 
 #endif /* PMC_CORE_H */
diff --git a/drivers/platform/x86/mlxcpld-hotplug.c b/drivers/platform/x86/mlxcpld-hotplug.c
new file mode 100644
index 000000000000..aff3686b3b37
--- /dev/null
+++ b/drivers/platform/x86/mlxcpld-hotplug.c
@@ -0,0 +1,515 @@
+/*
+ * drivers/platform/x86/mlxcpld-hotplug.c
+ * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2016 Vadim Pasternak <vadimp@mellanox.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the names of the copyright holders nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_data/mlxcpld-hotplug.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+
+/* Offset of event and mask registers from status register */
+#define MLXCPLD_HOTPLUG_EVENT_OFF	1
+#define MLXCPLD_HOTPLUG_MASK_OFF	2
+#define MLXCPLD_HOTPLUG_AGGR_MASK_OFF	1
+
+#define MLXCPLD_HOTPLUG_ATTRS_NUM	8
+
+/**
+ * enum mlxcpld_hotplug_attr_type - sysfs attributes for hotplug events:
+ * @MLXCPLD_HOTPLUG_ATTR_TYPE_PSU: power supply unit attribute;
+ * @MLXCPLD_HOTPLUG_ATTR_TYPE_PWR: power cable attribute;
+ * @MLXCPLD_HOTPLUG_ATTR_TYPE_FAN: FAN drawer attribute;
+ */
+enum mlxcpld_hotplug_attr_type {
+	MLXCPLD_HOTPLUG_ATTR_TYPE_PSU,
+	MLXCPLD_HOTPLUG_ATTR_TYPE_PWR,
+	MLXCPLD_HOTPLUG_ATTR_TYPE_FAN,
+};
+
+/**
+ * struct mlxcpld_hotplug_priv_data - platform private data:
+ * @irq: platform interrupt number;
+ * @pdev: platform device;
+ * @plat: platform data;
+ * @hwmon: hwmon device;
+ * @mlxcpld_hotplug_attr: sysfs attributes array;
+ * @mlxcpld_hotplug_dev_attr: sysfs sensor device attribute array;
+ * @group: sysfs attribute group;
+ * @groups: list of sysfs attribute group for hwmon registration;
+ * @dwork: delayed work template;
+ * @lock: spin lock;
+ * @aggr_cache: last value of aggregation register status;
+ * @psu_cache: last value of PSU register status;
+ * @pwr_cache: last value of power register status;
+ * @fan_cache: last value of FAN register status;
+ */
+struct mlxcpld_hotplug_priv_data {
+	int irq;
+	struct platform_device *pdev;
+	struct mlxcpld_hotplug_platform_data *plat;
+	struct device *hwmon;
+	struct attribute *mlxcpld_hotplug_attr[MLXCPLD_HOTPLUG_ATTRS_NUM + 1];
+	struct sensor_device_attribute_2
+			mlxcpld_hotplug_dev_attr[MLXCPLD_HOTPLUG_ATTRS_NUM];
+	struct attribute_group group;
+	const struct attribute_group *groups[2];
+	struct delayed_work dwork;
+	spinlock_t lock;
+	u8 aggr_cache;
+	u8 psu_cache;
+	u8 pwr_cache;
+	u8 fan_cache;
+};
+
+static ssize_t mlxcpld_hotplug_attr_show(struct device *dev,
+					 struct device_attribute *attr,
+					 char *buf)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct mlxcpld_hotplug_priv_data *priv = platform_get_drvdata(pdev);
+	int index = to_sensor_dev_attr_2(attr)->index;
+	int nr = to_sensor_dev_attr_2(attr)->nr;
+	u8 reg_val = 0;
+
+	switch (nr) {
+	case MLXCPLD_HOTPLUG_ATTR_TYPE_PSU:
+		/* Bit = 0 : PSU is present. */
+		reg_val = !!!(inb(priv->plat->psu_reg_offset) & BIT(index));
+		break;
+
+	case MLXCPLD_HOTPLUG_ATTR_TYPE_PWR:
+		/* Bit = 1 : power cable is attached. */
+		reg_val = !!(inb(priv->plat->pwr_reg_offset) & BIT(index %
+						priv->plat->pwr_count));
+		break;
+
+	case MLXCPLD_HOTPLUG_ATTR_TYPE_FAN:
+		/* Bit = 0 : FAN is present. */
+		reg_val = !!!(inb(priv->plat->fan_reg_offset) & BIT(index %
+						priv->plat->fan_count));
+		break;
+	}
+
+	return sprintf(buf, "%u\n", reg_val);
+}
+
+#define PRIV_ATTR(i) priv->mlxcpld_hotplug_attr[i]
+#define PRIV_DEV_ATTR(i) priv->mlxcpld_hotplug_dev_attr[i]
+static int mlxcpld_hotplug_attr_init(struct mlxcpld_hotplug_priv_data *priv)
+{
+	int num_attrs = priv->plat->psu_count + priv->plat->pwr_count +
+			priv->plat->fan_count;
+	int i;
+
+	priv->group.attrs = devm_kzalloc(&priv->pdev->dev, num_attrs *
+					 sizeof(struct attribute *),
+					 GFP_KERNEL);
+	if (!priv->group.attrs)
+		return -ENOMEM;
+
+	for (i = 0; i < num_attrs; i++) {
+		PRIV_ATTR(i) = &PRIV_DEV_ATTR(i).dev_attr.attr;
+
+		if (i < priv->plat->psu_count) {
+			PRIV_ATTR(i)->name = devm_kasprintf(&priv->pdev->dev,
+						GFP_KERNEL, "psu%u", i + 1);
+			PRIV_DEV_ATTR(i).nr = MLXCPLD_HOTPLUG_ATTR_TYPE_PSU;
+		} else if (i < priv->plat->psu_count + priv->plat->pwr_count) {
+			PRIV_ATTR(i)->name = devm_kasprintf(&priv->pdev->dev,
+						GFP_KERNEL, "pwr%u", i %
+						priv->plat->pwr_count + 1);
+			PRIV_DEV_ATTR(i).nr = MLXCPLD_HOTPLUG_ATTR_TYPE_PWR;
+		} else {
+			PRIV_ATTR(i)->name = devm_kasprintf(&priv->pdev->dev,
+						GFP_KERNEL, "fan%u", i %
+						priv->plat->fan_count + 1);
+			PRIV_DEV_ATTR(i).nr = MLXCPLD_HOTPLUG_ATTR_TYPE_FAN;
+		}
+
+		if (!PRIV_ATTR(i)->name) {
+			dev_err(&priv->pdev->dev, "Memory allocation failed for sysfs attribute %d.\n",
+				i + 1);
+			return -ENOMEM;
+		}
+
+		PRIV_DEV_ATTR(i).dev_attr.attr.name = PRIV_ATTR(i)->name;
+		PRIV_DEV_ATTR(i).dev_attr.attr.mode = S_IRUGO;
+		PRIV_DEV_ATTR(i).dev_attr.show = mlxcpld_hotplug_attr_show;
+		PRIV_DEV_ATTR(i).index = i;
+		sysfs_attr_init(&PRIV_DEV_ATTR(i).dev_attr.attr);
+	}
+
+	priv->group.attrs = priv->mlxcpld_hotplug_attr;
+	priv->groups[0] = &priv->group;
+	priv->groups[1] = NULL;
+
+	return 0;
+}
+
+static int mlxcpld_hotplug_device_create(struct device *dev,
+					 struct mlxcpld_hotplug_device *item)
+{
+	item->adapter = i2c_get_adapter(item->bus);
+	if (!item->adapter) {
+		dev_err(dev, "Failed to get adapter for bus %d\n",
+			item->bus);
+		return -EFAULT;
+	}
+
+	item->client = i2c_new_device(item->adapter, &item->brdinfo);
+	if (!item->client) {
+		dev_err(dev, "Failed to create client %s at bus %d at addr 0x%02x\n",
+			item->brdinfo.type, item->bus, item->brdinfo.addr);
+		i2c_put_adapter(item->adapter);
+		item->adapter = NULL;
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+static void mlxcpld_hotplug_device_destroy(struct mlxcpld_hotplug_device *item)
+{
+	if (item->client) {
+		i2c_unregister_device(item->client);
+		item->client = NULL;
+	}
+
+	if (item->adapter) {
+		i2c_put_adapter(item->adapter);
+		item->adapter = NULL;
+	}
+}
+
+static inline void
+mlxcpld_hotplug_work_helper(struct device *dev,
+			    struct mlxcpld_hotplug_device *item, u8 is_inverse,
+			    u16 offset, u8 mask, u8 *cache)
+{
+	u8 val, asserted;
+	int bit;
+
+	/* Mask event. */
+	outb(0, offset + MLXCPLD_HOTPLUG_MASK_OFF);
+	/* Read status. */
+	val = inb(offset) & mask;
+	asserted = *cache ^ val;
+	*cache = val;
+
+	/*
+	 * Validate if item related to received signal type is valid.
+	 * It should never happen, excepted the situation when some
+	 * piece of hardware is broken. In such situation just produce
+	 * error message and return. Caller must continue to handle the
+	 * signals from other devices if any.
+	 */
+	if (unlikely(!item)) {
+		dev_err(dev, "False signal is received: register at offset 0x%02x, mask 0x%02x.\n",
+			offset, mask);
+		return;
+	}
+
+	for_each_set_bit(bit, (unsigned long *)&asserted, 8) {
+		if (val & BIT(bit)) {
+			if (is_inverse)
+				mlxcpld_hotplug_device_destroy(item + bit);
+			else
+				mlxcpld_hotplug_device_create(dev, item + bit);
+		} else {
+			if (is_inverse)
+				mlxcpld_hotplug_device_create(dev, item + bit);
+			else
+				mlxcpld_hotplug_device_destroy(item + bit);
+		}
+	}
+
+	/* Acknowledge event. */
+	outb(0, offset + MLXCPLD_HOTPLUG_EVENT_OFF);
+	/* Unmask event. */
+	outb(mask, offset + MLXCPLD_HOTPLUG_MASK_OFF);
+}
+
+/*
+ * mlxcpld_hotplug_work_handler - performs traversing of CPLD interrupt
+ * registers according to the below hierarchy schema:
+ *
+ *                   Aggregation registers (status/mask)
+ * PSU registers:           *---*
+ * *-----------------*      |   |
+ * |status/event/mask|----->| * |
+ * *-----------------*      |   |
+ * Power registers:         |   |
+ * *-----------------*      |   |
+ * |status/event/mask|----->| * |---> CPU
+ * *-----------------*      |   |
+ * FAN registers:
+ * *-----------------*      |   |
+ * |status/event/mask|----->| * |
+ * *-----------------*      |   |
+ *                          *---*
+ * In case some system changed are detected: FAN in/out, PSU in/out, power
+ * cable attached/detached, relevant device is created or destroyed.
+ */
+static void mlxcpld_hotplug_work_handler(struct work_struct *work)
+{
+	struct mlxcpld_hotplug_priv_data *priv = container_of(work,
+				struct mlxcpld_hotplug_priv_data, dwork.work);
+	u8 val, aggr_asserted;
+	unsigned long flags;
+
+	/* Mask aggregation event. */
+	outb(0, priv->plat->top_aggr_offset + MLXCPLD_HOTPLUG_AGGR_MASK_OFF);
+	/* Read aggregation status. */
+	val = inb(priv->plat->top_aggr_offset) & priv->plat->top_aggr_mask;
+	aggr_asserted = priv->aggr_cache ^ val;
+	priv->aggr_cache = val;
+
+	/* Handle PSU configuration changes. */
+	if (aggr_asserted & priv->plat->top_aggr_psu_mask)
+		mlxcpld_hotplug_work_helper(&priv->pdev->dev, priv->plat->psu,
+					    1, priv->plat->psu_reg_offset,
+					    priv->plat->psu_mask,
+					    &priv->psu_cache);
+
+	/* Handle power cable configuration changes. */
+	if (aggr_asserted & priv->plat->top_aggr_pwr_mask)
+		mlxcpld_hotplug_work_helper(&priv->pdev->dev, priv->plat->pwr,
+					    0, priv->plat->pwr_reg_offset,
+					    priv->plat->pwr_mask,
+					    &priv->pwr_cache);
+
+	/* Handle FAN configuration changes. */
+	if (aggr_asserted & priv->plat->top_aggr_fan_mask)
+		mlxcpld_hotplug_work_helper(&priv->pdev->dev, priv->plat->fan,
+					    1, priv->plat->fan_reg_offset,
+					    priv->plat->fan_mask,
+					    &priv->fan_cache);
+
+	if (aggr_asserted) {
+		spin_lock_irqsave(&priv->lock, flags);
+
+		/*
+		 * It is possible, that some signals have been inserted, while
+		 * interrupt has been masked by mlxcpld_hotplug_work_handler.
+		 * In this case such signals will be missed. In order to handle
+		 * these signals delayed work is canceled and work task
+		 * re-scheduled for immediate execution. It allows to handle
+		 * missed signals, if any. In other case work handler just
+		 * validates that no new signals have been received during
+		 * masking.
+		 */
+		cancel_delayed_work(&priv->dwork);
+		schedule_delayed_work(&priv->dwork, 0);
+
+		spin_unlock_irqrestore(&priv->lock, flags);
+
+		return;
+	}
+
+	/* Unmask aggregation event (no need acknowledge). */
+	outb(priv->plat->top_aggr_mask, priv->plat->top_aggr_offset +
+						MLXCPLD_HOTPLUG_AGGR_MASK_OFF);
+}
+
+static void mlxcpld_hotplug_set_irq(struct mlxcpld_hotplug_priv_data *priv)
+{
+	/* Clear psu presense event. */
+	outb(0, priv->plat->psu_reg_offset + MLXCPLD_HOTPLUG_EVENT_OFF);
+	/* Set psu initial status as mask and unmask psu event. */
+	priv->psu_cache = priv->plat->psu_mask;
+	outb(priv->plat->psu_mask, priv->plat->psu_reg_offset +
+						MLXCPLD_HOTPLUG_MASK_OFF);
+
+	/* Clear power cable event. */
+	outb(0, priv->plat->pwr_reg_offset + MLXCPLD_HOTPLUG_EVENT_OFF);
+	/* Keep power initial status as zero and unmask power event. */
+	outb(priv->plat->pwr_mask, priv->plat->pwr_reg_offset +
+						MLXCPLD_HOTPLUG_MASK_OFF);
+
+	/* Clear fan presense event. */
+	outb(0, priv->plat->fan_reg_offset + MLXCPLD_HOTPLUG_EVENT_OFF);
+	/* Set fan initial status as mask and unmask fan event. */
+	priv->fan_cache = priv->plat->fan_mask;
+	outb(priv->plat->fan_mask, priv->plat->fan_reg_offset +
+						MLXCPLD_HOTPLUG_MASK_OFF);
+
+	/* Keep aggregation initial status as zero and unmask events. */
+	outb(priv->plat->top_aggr_mask, priv->plat->top_aggr_offset +
+						MLXCPLD_HOTPLUG_AGGR_MASK_OFF);
+
+	/* Invoke work handler for initializing hot plug devices setting. */
+	mlxcpld_hotplug_work_handler(&priv->dwork.work);
+
+	enable_irq(priv->irq);
+}
+
+static void mlxcpld_hotplug_unset_irq(struct mlxcpld_hotplug_priv_data *priv)
+{
+	int i;
+
+	disable_irq(priv->irq);
+	cancel_delayed_work_sync(&priv->dwork);
+
+	/* Mask aggregation event. */
+	outb(0, priv->plat->top_aggr_offset + MLXCPLD_HOTPLUG_AGGR_MASK_OFF);
+
+	/* Mask psu presense event. */
+	outb(0, priv->plat->psu_reg_offset + MLXCPLD_HOTPLUG_MASK_OFF);
+	/* Clear psu presense event. */
+	outb(0, priv->plat->psu_reg_offset + MLXCPLD_HOTPLUG_EVENT_OFF);
+
+	/* Mask power cable event. */
+	outb(0, priv->plat->pwr_reg_offset + MLXCPLD_HOTPLUG_MASK_OFF);
+	/* Clear power cable event. */
+	outb(0, priv->plat->pwr_reg_offset + MLXCPLD_HOTPLUG_EVENT_OFF);
+
+	/* Mask fan presense event. */
+	outb(0, priv->plat->fan_reg_offset + MLXCPLD_HOTPLUG_MASK_OFF);
+	/* Clear fan presense event. */
+	outb(0, priv->plat->fan_reg_offset + MLXCPLD_HOTPLUG_EVENT_OFF);
+
+	/* Remove all the attached devices. */
+	for (i = 0; i < priv->plat->psu_count; i++)
+		mlxcpld_hotplug_device_destroy(priv->plat->psu + i);
+
+	for (i = 0; i < priv->plat->pwr_count; i++)
+		mlxcpld_hotplug_device_destroy(priv->plat->pwr + i);
+
+	for (i = 0; i < priv->plat->fan_count; i++)
+		mlxcpld_hotplug_device_destroy(priv->plat->fan + i);
+}
+
+static irqreturn_t mlxcpld_hotplug_irq_handler(int irq, void *dev)
+{
+	struct mlxcpld_hotplug_priv_data *priv =
+				(struct mlxcpld_hotplug_priv_data *)dev;
+
+	/* Schedule work task for immediate execution.*/
+	schedule_delayed_work(&priv->dwork, 0);
+
+	return IRQ_HANDLED;
+}
+
+static int mlxcpld_hotplug_probe(struct platform_device *pdev)
+{
+	struct mlxcpld_hotplug_platform_data *pdata;
+	struct mlxcpld_hotplug_priv_data *priv;
+	int err;
+
+	pdata = dev_get_platdata(&pdev->dev);
+	if (!pdata) {
+		dev_err(&pdev->dev, "Failed to get platform data.\n");
+		return -EINVAL;
+	}
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->pdev = pdev;
+	priv->plat = pdata;
+
+	priv->irq = platform_get_irq(pdev, 0);
+	if (priv->irq < 0) {
+		dev_err(&pdev->dev, "Failed to get platform irq: %d\n",
+			priv->irq);
+		return priv->irq;
+	}
+
+	err = devm_request_irq(&pdev->dev, priv->irq,
+				mlxcpld_hotplug_irq_handler, 0, pdev->name,
+				priv);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to request irq: %d\n", err);
+		return err;
+	}
+	disable_irq(priv->irq);
+
+	INIT_DELAYED_WORK(&priv->dwork, mlxcpld_hotplug_work_handler);
+	spin_lock_init(&priv->lock);
+
+	err = mlxcpld_hotplug_attr_init(priv);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate attributes: %d\n", err);
+		return err;
+	}
+
+	priv->hwmon = devm_hwmon_device_register_with_groups(&pdev->dev,
+					"mlxcpld_hotplug", priv, priv->groups);
+	if (IS_ERR(priv->hwmon)) {
+		dev_err(&pdev->dev, "Failed to register hwmon device %ld\n",
+			PTR_ERR(priv->hwmon));
+		return PTR_ERR(priv->hwmon);
+	}
+
+	platform_set_drvdata(pdev, priv);
+
+	/* Perform initial interrupts setup. */
+	mlxcpld_hotplug_set_irq(priv);
+
+	return 0;
+}
+
+static int mlxcpld_hotplug_remove(struct platform_device *pdev)
+{
+	struct mlxcpld_hotplug_priv_data *priv = platform_get_drvdata(pdev);
+
+	/* Clean interrupts setup. */
+	mlxcpld_hotplug_unset_irq(priv);
+
+	return 0;
+}
+
+static struct platform_driver mlxcpld_hotplug_driver = {
+	.driver = {
+		.name = "mlxcpld-hotplug",
+	},
+	.probe = mlxcpld_hotplug_probe,
+	.remove = mlxcpld_hotplug_remove,
+};
+
+module_platform_driver(mlxcpld_hotplug_driver);
+
+MODULE_AUTHOR("Vadim Pasternak <vadimp@mellanox.com>");
+MODULE_DESCRIPTION("Mellanox CPLD hotplug platform driver");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_ALIAS("platform:mlxcpld-hotplug");
diff --git a/drivers/platform/x86/panasonic-laptop.c b/drivers/platform/x86/panasonic-laptop.c
index 3f870972247c..59b8eb626dcc 100644
--- a/drivers/platform/x86/panasonic-laptop.c
+++ b/drivers/platform/x86/panasonic-laptop.c
@@ -458,7 +458,7 @@ static void acpi_pcc_generate_keyinput(struct pcc_acpi *pcc)
 
 	rc = acpi_evaluate_integer(pcc->handle, METHOD_HKEY_QUERY,
 				   NULL, &result);
-	if (!ACPI_SUCCESS(rc)) {
+	if (ACPI_FAILURE(rc)) {
 		ACPI_DEBUG_PRINT((ACPI_DB_ERROR,
 				 "error getting hotkey status\n"));
 		return;
diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c
index b65ce7519411..aa65a857a6b1 100644
--- a/drivers/platform/x86/thinkpad_acpi.c
+++ b/drivers/platform/x86/thinkpad_acpi.c
@@ -128,6 +128,7 @@ enum {
 /* ACPI HIDs */
 #define TPACPI_ACPI_IBM_HKEY_HID	"IBM0068"
 #define TPACPI_ACPI_LENOVO_HKEY_HID	"LEN0068"
+#define TPACPI_ACPI_LENOVO_HKEY_V2_HID	"LEN0268"
 #define TPACPI_ACPI_EC_HID		"PNP0C09"
 
 /* Input IDs */
@@ -190,6 +191,9 @@ enum tpacpi_hkey_event_t {
 	TP_HKEY_EV_LID_OPEN		= 0x5002, /* laptop lid opened */
 	TP_HKEY_EV_TABLET_TABLET	= 0x5009, /* tablet swivel up */
 	TP_HKEY_EV_TABLET_NOTEBOOK	= 0x500a, /* tablet swivel down */
+	TP_HKEY_EV_TABLET_CHANGED	= 0x60c0, /* X1 Yoga (2016):
+						   * enter/leave tablet mode
+						   */
 	TP_HKEY_EV_PEN_INSERTED		= 0x500b, /* tablet pen inserted */
 	TP_HKEY_EV_PEN_REMOVED		= 0x500c, /* tablet pen removed */
 	TP_HKEY_EV_BRGHT_CHANGED	= 0x5010, /* backlight control event */
@@ -302,7 +306,12 @@ static struct {
 	u32 hotkey:1;
 	u32 hotkey_mask:1;
 	u32 hotkey_wlsw:1;
-	u32 hotkey_tablet:1;
+	enum {
+		TP_HOTKEY_TABLET_NONE = 0,
+		TP_HOTKEY_TABLET_USES_MHKG,
+		/* X1 Yoga 2016, seen on BIOS N1FET44W */
+		TP_HOTKEY_TABLET_USES_CMMD,
+	} hotkey_tablet;
 	u32 kbdlight:1;
 	u32 light:1;
 	u32 light_status:1;
@@ -2059,6 +2068,8 @@ static void hotkey_poll_setup(const bool may_warn);
 
 /* HKEY.MHKG() return bits */
 #define TP_HOTKEY_TABLET_MASK (1 << 3)
+/* ThinkPad X1 Yoga (2016) */
+#define TP_EC_CMMD_TABLET_MODE 0x6
 
 static int hotkey_get_wlsw(void)
 {
@@ -2083,10 +2094,23 @@ static int hotkey_get_tablet_mode(int *status)
 {
 	int s;
 
-	if (!acpi_evalf(hkey_handle, &s, "MHKG", "d"))
-		return -EIO;
+	switch (tp_features.hotkey_tablet) {
+	case TP_HOTKEY_TABLET_USES_MHKG:
+		if (!acpi_evalf(hkey_handle, &s, "MHKG", "d"))
+			return -EIO;
+
+		*status = ((s & TP_HOTKEY_TABLET_MASK) != 0);
+		break;
+	case TP_HOTKEY_TABLET_USES_CMMD:
+		if (!acpi_evalf(ec_handle, &s, "CMMD", "d"))
+			return -EIO;
+
+		*status = (s == TP_EC_CMMD_TABLET_MODE);
+		break;
+	default:
+		break;
+	}
 
-	*status = ((s & TP_HOTKEY_TABLET_MASK) != 0);
 	return 0;
 }
 
@@ -3117,6 +3141,37 @@ static const struct tpacpi_quirk tpacpi_hotkey_qtable[] __initconst = {
 typedef u16 tpacpi_keymap_entry_t;
 typedef tpacpi_keymap_entry_t tpacpi_keymap_t[TPACPI_HOTKEY_MAP_LEN];
 
+static int hotkey_init_tablet_mode(void)
+{
+	int in_tablet_mode = 0, res;
+	char *type = NULL;
+
+	if (acpi_evalf(hkey_handle, &res, "MHKG", "qd")) {
+		/* For X41t, X60t, X61t Tablets... */
+		tp_features.hotkey_tablet = TP_HOTKEY_TABLET_USES_MHKG;
+		in_tablet_mode = !!(res & TP_HOTKEY_TABLET_MASK);
+		type = "MHKG";
+	} else if (acpi_evalf(ec_handle, &res, "CMMD", "qd")) {
+		/* For X1 Yoga (2016) */
+		tp_features.hotkey_tablet = TP_HOTKEY_TABLET_USES_CMMD;
+		in_tablet_mode = res == TP_EC_CMMD_TABLET_MODE;
+		type = "CMMD";
+	}
+
+	if (!tp_features.hotkey_tablet)
+		return 0;
+
+	pr_info("Tablet mode switch found (type: %s), currently in %s mode\n",
+		type, in_tablet_mode ? "tablet" : "laptop");
+
+	res = add_to_attr_set(hotkey_dev_attributes,
+			      &dev_attr_hotkey_tablet_mode.attr);
+	if (res)
+		return -1;
+
+	return in_tablet_mode;
+}
+
 static int __init hotkey_init(struct ibm_init_struct *iibm)
 {
 	/* Requirements for changing the default keymaps:
@@ -3464,21 +3519,14 @@ static int __init hotkey_init(struct ibm_init_struct *iibm)
 		res = add_to_attr_set(hotkey_dev_attributes,
 				&dev_attr_hotkey_radio_sw.attr);
 
-	/* For X41t, X60t, X61t Tablets... */
-	if (!res && acpi_evalf(hkey_handle, &status, "MHKG", "qd")) {
-		tp_features.hotkey_tablet = 1;
-		tabletsw_state = !!(status & TP_HOTKEY_TABLET_MASK);
-		pr_info("possible tablet mode switch found; "
-			"ThinkPad in %s mode\n",
-			(tabletsw_state) ? "tablet" : "laptop");
-		res = add_to_attr_set(hotkey_dev_attributes,
-				&dev_attr_hotkey_tablet_mode.attr);
-	}
+	res = hotkey_init_tablet_mode();
+	if (res < 0)
+		goto err_exit;
 
-	if (!res)
-		res = register_attr_set_with_sysfs(
-				hotkey_dev_attributes,
-				&tpacpi_pdev->dev.kobj);
+	tabletsw_state = res;
+
+	res = register_attr_set_with_sysfs(hotkey_dev_attributes,
+					   &tpacpi_pdev->dev.kobj);
 	if (res)
 		goto err_exit;
 
@@ -3899,6 +3947,12 @@ static bool hotkey_notify_6xxx(const u32 hkey,
 		*ignore_acpi_ev = true;
 		return true;
 
+	case TP_HKEY_EV_TABLET_CHANGED:
+		tpacpi_input_send_tabletsw();
+		hotkey_tablet_mode_notify_change();
+		*send_acpi_ev = false;
+		break;
+
 	default:
 		pr_warn("unknown possible thermal alarm or keyboard event received\n");
 		known = false;
@@ -4143,6 +4197,7 @@ errexit:
 static const struct acpi_device_id ibm_htk_device_ids[] = {
 	{TPACPI_ACPI_IBM_HKEY_HID, 0},
 	{TPACPI_ACPI_LENOVO_HKEY_HID, 0},
+	{TPACPI_ACPI_LENOVO_HKEY_V2_HID, 0},
 	{"", 0},
 };
 
@@ -7716,7 +7771,7 @@ static struct ibm_struct volume_driver_data = {
 
 #define alsa_card NULL
 
-static void inline volume_alsa_notify_change(void)
+static inline void volume_alsa_notify_change(void)
 {
 }
 
@@ -9018,7 +9073,7 @@ static int mute_led_on_off(struct tp_led_table *t, bool state)
 	acpi_handle temp;
 	int output;
 
-	if (!ACPI_SUCCESS(acpi_get_handle(hkey_handle, t->name, &temp))) {
+	if (ACPI_FAILURE(acpi_get_handle(hkey_handle, t->name, &temp))) {
 		pr_warn("Thinkpad ACPI has no %s interface.\n", t->name);
 		return -EIO;
 	}
diff --git a/drivers/power/avs/rockchip-io-domain.c b/drivers/power/avs/rockchip-io-domain.c
index 01b6d3f9b8fb..56bce1908be2 100644
--- a/drivers/power/avs/rockchip-io-domain.c
+++ b/drivers/power/avs/rockchip-io-domain.c
@@ -143,7 +143,7 @@ static int rockchip_iodomain_notify(struct notifier_block *nb,
 	if (ret && event == REGULATOR_EVENT_PRE_VOLTAGE_CHANGE)
 		return NOTIFY_BAD;
 
-	dev_info(supply->iod->dev, "Setting to %d done\n", uV);
+	dev_dbg(supply->iod->dev, "Setting to %d done\n", uV);
 	return NOTIFY_OK;
 }
 
diff --git a/drivers/power/reset/Kconfig b/drivers/power/reset/Kconfig
index c74c3f67b8da..abeb77217a21 100644
--- a/drivers/power/reset/Kconfig
+++ b/drivers/power/reset/Kconfig
@@ -104,6 +104,16 @@ config POWER_RESET_MSM
 	help
 	  Power off and restart support for Qualcomm boards.
 
+config POWER_RESET_PIIX4_POWEROFF
+	tristate "Intel PIIX4 power-off driver"
+	depends on PCI
+	depends on MIPS || COMPILE_TEST
+	help
+	  This driver supports powering off a system using the Intel PIIX4
+	  southbridge, for example the MIPS Malta development board. The
+	  southbridge SOff state is entered in response to a request to
+	  power off the system.
+
 config POWER_RESET_LTC2952
 	bool "LTC2952 PowerPath power-off driver"
 	depends on OF_GPIO
diff --git a/drivers/power/reset/Makefile b/drivers/power/reset/Makefile
index 1be307c7fc25..11dae3b56ff9 100644
--- a/drivers/power/reset/Makefile
+++ b/drivers/power/reset/Makefile
@@ -10,6 +10,7 @@ obj-$(CONFIG_POWER_RESET_GPIO_RESTART) += gpio-restart.o
 obj-$(CONFIG_POWER_RESET_HISI) += hisi-reboot.o
 obj-$(CONFIG_POWER_RESET_IMX) += imx-snvs-poweroff.o
 obj-$(CONFIG_POWER_RESET_MSM) += msm-poweroff.o
+obj-$(CONFIG_POWER_RESET_PIIX4_POWEROFF) += piix4-poweroff.o
 obj-$(CONFIG_POWER_RESET_LTC2952) += ltc2952-poweroff.o
 obj-$(CONFIG_POWER_RESET_QNAP) += qnap-poweroff.o
 obj-$(CONFIG_POWER_RESET_RESTART) += restart-poweroff.o
diff --git a/drivers/power/reset/at91-poweroff.c b/drivers/power/reset/at91-poweroff.c
index e9e24df35f26..a85dd4d233af 100644
--- a/drivers/power/reset/at91-poweroff.c
+++ b/drivers/power/reset/at91-poweroff.c
@@ -169,6 +169,7 @@ static const struct of_device_id at91_poweroff_of_match[] = {
 	{ .compatible = "atmel,at91sam9x5-shdwc", },
 	{ /*sentinel*/ }
 };
+MODULE_DEVICE_TABLE(of, at91_poweroff_of_match);
 
 static struct platform_driver at91_poweroff_driver = {
 	.remove = __exit_p(at91_poweroff_remove),
diff --git a/drivers/power/reset/at91-reset.c b/drivers/power/reset/at91-reset.c
index 1b5d450586d1..568580cf0655 100644
--- a/drivers/power/reset/at91-reset.c
+++ b/drivers/power/reset/at91-reset.c
@@ -175,6 +175,7 @@ static const struct of_device_id at91_reset_of_match[] = {
 	{ .compatible = "atmel,sama5d3-rstc", .data = sama5d3_restart },
 	{ /* sentinel */ }
 };
+MODULE_DEVICE_TABLE(of, at91_reset_of_match);
 
 static struct notifier_block at91_restart_nb = {
 	.priority = 192,
@@ -242,6 +243,7 @@ static const struct platform_device_id at91_reset_plat_match[] = {
 	{ "at91-sam9g45-reset", (unsigned long)at91sam9g45_restart },
 	{ /* sentinel */ }
 };
+MODULE_DEVICE_TABLE(platform, at91_reset_plat_match);
 
 static struct platform_driver at91_reset_driver = {
 	.remove = __exit_p(at91_reset_remove),
diff --git a/drivers/power/reset/piix4-poweroff.c b/drivers/power/reset/piix4-poweroff.c
new file mode 100644
index 000000000000..bacfc95783f0
--- /dev/null
+++ b/drivers/power/reset/piix4-poweroff.c
@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2016 Imagination Technologies
+ * Author: Paul Burton <paul.burton@imgtec.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/pm.h>
+
+static struct pci_dev *pm_dev;
+static resource_size_t io_offset;
+
+enum piix4_pm_io_reg {
+	PIIX4_FUNC3IO_PMSTS			= 0x00,
+#define PIIX4_FUNC3IO_PMSTS_PWRBTN_STS		BIT(8)
+	PIIX4_FUNC3IO_PMCNTRL			= 0x04,
+#define PIIX4_FUNC3IO_PMCNTRL_SUS_EN		BIT(13)
+#define PIIX4_FUNC3IO_PMCNTRL_SUS_TYP_SOFF	(0x0 << 10)
+};
+
+#define PIIX4_SUSPEND_MAGIC			0x00120002
+
+static const int piix4_pm_io_region = PCI_BRIDGE_RESOURCES;
+
+static void piix4_poweroff(void)
+{
+	int spec_devid;
+	u16 sts;
+
+	/* Ensure the power button status is clear */
+	while (1) {
+		sts = inw(io_offset + PIIX4_FUNC3IO_PMSTS);
+		if (!(sts & PIIX4_FUNC3IO_PMSTS_PWRBTN_STS))
+			break;
+		outw(sts, io_offset + PIIX4_FUNC3IO_PMSTS);
+	}
+
+	/* Enable entry to suspend */
+	outw(PIIX4_FUNC3IO_PMCNTRL_SUS_TYP_SOFF | PIIX4_FUNC3IO_PMCNTRL_SUS_EN,
+	     io_offset + PIIX4_FUNC3IO_PMCNTRL);
+
+	/* If the special cycle occurs too soon this doesn't work... */
+	mdelay(10);
+
+	/*
+	 * The PIIX4 will enter the suspend state only after seeing a special
+	 * cycle with the correct magic data on the PCI bus. Generate that
+	 * cycle now.
+	 */
+	spec_devid = PCI_DEVID(0, PCI_DEVFN(0x1f, 0x7));
+	pci_bus_write_config_dword(pm_dev->bus, spec_devid, 0,
+				   PIIX4_SUSPEND_MAGIC);
+
+	/* Give the system some time to power down, then error */
+	mdelay(1000);
+	pr_emerg("Unable to poweroff system\n");
+}
+
+static int piix4_poweroff_probe(struct pci_dev *dev,
+				const struct pci_device_id *id)
+{
+	int res;
+
+	if (pm_dev)
+		return -EINVAL;
+
+	/* Request access to the PIIX4 PM IO registers */
+	res = pci_request_region(dev, piix4_pm_io_region,
+				 "PIIX4 PM IO registers");
+	if (res) {
+		dev_err(&dev->dev, "failed to request PM IO registers: %d\n",
+			res);
+		return res;
+	}
+
+	pm_dev = dev;
+	io_offset = pci_resource_start(dev, piix4_pm_io_region);
+	pm_power_off = piix4_poweroff;
+
+	return 0;
+}
+
+static void piix4_poweroff_remove(struct pci_dev *dev)
+{
+	if (pm_power_off == piix4_poweroff)
+		pm_power_off = NULL;
+
+	pci_release_region(dev, piix4_pm_io_region);
+	pm_dev = NULL;
+}
+
+static const struct pci_device_id piix4_poweroff_ids[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB_3) },
+	{ 0 },
+};
+
+static struct pci_driver piix4_poweroff_driver = {
+	.name		= "piix4-poweroff",
+	.id_table	= piix4_poweroff_ids,
+	.probe		= piix4_poweroff_probe,
+	.remove		= piix4_poweroff_remove,
+};
+
+module_pci_driver(piix4_poweroff_driver);
+MODULE_AUTHOR("Paul Burton <paul.burton@imgtec.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/power/reset/syscon-reboot-mode.c b/drivers/power/reset/syscon-reboot-mode.c
index 1ecb51d67149..c8c371b285b1 100644
--- a/drivers/power/reset/syscon-reboot-mode.c
+++ b/drivers/power/reset/syscon-reboot-mode.c
@@ -74,6 +74,7 @@ static const struct of_device_id syscon_reboot_mode_of_match[] = {
 	{ .compatible = "syscon-reboot-mode" },
 	{}
 };
+MODULE_DEVICE_TABLE(of, syscon_reboot_mode_of_match);
 
 static struct platform_driver syscon_reboot_mode_driver = {
 	.probe = syscon_reboot_mode_probe,
diff --git a/drivers/power/reset/zx-reboot.c b/drivers/power/reset/zx-reboot.c
index b0b1eb3a78c2..7549c7f74a3c 100644
--- a/drivers/power/reset/zx-reboot.c
+++ b/drivers/power/reset/zx-reboot.c
@@ -72,6 +72,7 @@ static const struct of_device_id zx_reboot_of_match[] = {
 	{ .compatible = "zte,sysctrl" },
 	{}
 };
+MODULE_DEVICE_TABLE(of, zx_reboot_of_match);
 
 static struct platform_driver zx_reboot_driver = {
 	.probe = zx_reboot_probe,
diff --git a/drivers/power/supply/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c
index 2199f673118c..c569f82a0071 100644
--- a/drivers/power/supply/ab8500_fg.c
+++ b/drivers/power/supply/ab8500_fg.c
@@ -1900,7 +1900,7 @@ static void ab8500_fg_low_bat_work(struct work_struct *work)
  * ab8500_fg_battok_calc - calculate the bit pattern corresponding
  * to the target voltage.
  * @di:       pointer to the ab8500_fg structure
- * @target    target voltage
+ * @target:   target voltage
  *
  * Returns bit pattern closest to the target voltage
  * valid return values are 0-14. (0-BATT_OK_MAX_NR_INCREMENTS)
@@ -2391,7 +2391,7 @@ static void ab8500_fg_external_power_changed(struct power_supply *psy)
 }
 
 /**
- * abab8500_fg_reinit_work() - work to reset the FG algorithm
+ * ab8500_fg_reinit_work() - work to reset the FG algorithm
  * @work:	pointer to the work_struct structure
  *
  * Used to reset the current battery capacity to be able to
@@ -2528,7 +2528,7 @@ static struct kobj_type ab8500_fg_ktype = {
 };
 
 /**
- * ab8500_chargalg_sysfs_exit() - de-init of sysfs entry
+ * ab8500_fg_sysfs_exit() - de-init of sysfs entry
  * @di:                pointer to the struct ab8500_chargalg
  *
  * This function removes the entry in sysfs.
@@ -2539,7 +2539,7 @@ static void ab8500_fg_sysfs_exit(struct ab8500_fg *di)
 }
 
 /**
- * ab8500_chargalg_sysfs_init() - init of sysfs entry
+ * ab8500_fg_sysfs_init() - init of sysfs entry
  * @di:                pointer to the struct ab8500_chargalg
  *
  * This function adds an entry in sysfs.
diff --git a/drivers/power/supply/axp288_fuel_gauge.c b/drivers/power/supply/axp288_fuel_gauge.c
index 5bdde692f724..539eb41504bb 100644
--- a/drivers/power/supply/axp288_fuel_gauge.c
+++ b/drivers/power/supply/axp288_fuel_gauge.c
@@ -1120,6 +1120,7 @@ static const struct platform_device_id axp288_fg_id_table[] = {
 	{ .name = DEV_NAME },
 	{},
 };
+MODULE_DEVICE_TABLE(platform, axp288_fg_id_table);
 
 static int axp288_fuel_gauge_remove(struct platform_device *pdev)
 {
diff --git a/drivers/power/supply/bq24190_charger.c b/drivers/power/supply/bq24190_charger.c
index f5746b9f4e83..e9584330aeed 100644
--- a/drivers/power/supply/bq24190_charger.c
+++ b/drivers/power/supply/bq24190_charger.c
@@ -1141,7 +1141,7 @@ static int bq24190_battery_set_property(struct power_supply *psy,
 
 	dev_dbg(bdi->dev, "prop: %d\n", psp);
 
-	pm_runtime_put_sync(bdi->dev);
+	pm_runtime_get_sync(bdi->dev);
 
 	switch (psp) {
 	case POWER_SUPPLY_PROP_ONLINE:
diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c
index 3b0dbc689d72..08c36b8e04bd 100644
--- a/drivers/power/supply/bq27xxx_battery.c
+++ b/drivers/power/supply/bq27xxx_battery.c
@@ -164,6 +164,25 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_DCAP] = 0x3c,
 		[BQ27XXX_REG_AP] = INVALID_REG_ADDR,
 	},
+	[BQ27510] = {
+		[BQ27XXX_REG_CTRL] = 0x00,
+		[BQ27XXX_REG_TEMP] = 0x06,
+		[BQ27XXX_REG_INT_TEMP] = 0x28,
+		[BQ27XXX_REG_VOLT] = 0x08,
+		[BQ27XXX_REG_AI] = 0x14,
+		[BQ27XXX_REG_FLAGS] = 0x0a,
+		[BQ27XXX_REG_TTE] = 0x16,
+		[BQ27XXX_REG_TTF] = INVALID_REG_ADDR,
+		[BQ27XXX_REG_TTES] = 0x1a,
+		[BQ27XXX_REG_TTECP] = INVALID_REG_ADDR,
+		[BQ27XXX_REG_NAC] = 0x0c,
+		[BQ27XXX_REG_FCC] = 0x12,
+		[BQ27XXX_REG_CYCT] = 0x1e,
+		[BQ27XXX_REG_AE] = INVALID_REG_ADDR,
+		[BQ27XXX_REG_SOC] = 0x20,
+		[BQ27XXX_REG_DCAP] = 0x2e,
+		[BQ27XXX_REG_AP] = INVALID_REG_ADDR,
+	},
 	[BQ27530] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
@@ -302,6 +321,24 @@ static enum power_supply_property bq27500_battery_props[] = {
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
 
+static enum power_supply_property bq27510_battery_props[] = {
+	POWER_SUPPLY_PROP_STATUS,
+	POWER_SUPPLY_PROP_PRESENT,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_CURRENT_NOW,
+	POWER_SUPPLY_PROP_CAPACITY,
+	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
+	POWER_SUPPLY_PROP_TEMP,
+	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
+	POWER_SUPPLY_PROP_TECHNOLOGY,
+	POWER_SUPPLY_PROP_CHARGE_FULL,
+	POWER_SUPPLY_PROP_CHARGE_NOW,
+	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
+	POWER_SUPPLY_PROP_CYCLE_COUNT,
+	POWER_SUPPLY_PROP_HEALTH,
+	POWER_SUPPLY_PROP_MANUFACTURER,
+};
+
 static enum power_supply_property bq27530_battery_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
@@ -385,6 +422,7 @@ static struct {
 	BQ27XXX_PROP(BQ27000, bq27000_battery_props),
 	BQ27XXX_PROP(BQ27010, bq27010_battery_props),
 	BQ27XXX_PROP(BQ27500, bq27500_battery_props),
+	BQ27XXX_PROP(BQ27510, bq27510_battery_props),
 	BQ27XXX_PROP(BQ27530, bq27530_battery_props),
 	BQ27XXX_PROP(BQ27541, bq27541_battery_props),
 	BQ27XXX_PROP(BQ27545, bq27545_battery_props),
@@ -397,10 +435,11 @@ static LIST_HEAD(bq27xxx_battery_devices);
 static int poll_interval_param_set(const char *val, const struct kernel_param *kp)
 {
 	struct bq27xxx_device_info *di;
+	unsigned int prev_val = *(unsigned int *) kp->arg;
 	int ret;
 
 	ret = param_set_uint(val, kp);
-	if (ret < 0)
+	if (ret < 0 || prev_val == *(unsigned int *) kp->arg)
 		return ret;
 
 	mutex_lock(&bq27xxx_list_lock);
@@ -635,7 +674,8 @@ static int bq27xxx_battery_read_pwr_avg(struct bq27xxx_device_info *di)
  */
 static bool bq27xxx_battery_overtemp(struct bq27xxx_device_info *di, u16 flags)
 {
-	if (di->chip == BQ27500 || di->chip == BQ27541 || di->chip == BQ27545)
+	if (di->chip == BQ27500 || di->chip == BQ27510 ||
+	    di->chip == BQ27541 || di->chip == BQ27545)
 		return flags & (BQ27XXX_FLAG_OTC | BQ27XXX_FLAG_OTD);
 	if (di->chip == BQ27530 || di->chip == BQ27421)
 		return flags & BQ27XXX_FLAG_OT;
diff --git a/drivers/power/supply/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c
index 85d4ea2a9c20..5c5c3a6f9923 100644
--- a/drivers/power/supply/bq27xxx_battery_i2c.c
+++ b/drivers/power/supply/bq27xxx_battery_i2c.c
@@ -149,8 +149,8 @@ static const struct i2c_device_id bq27xxx_i2c_id_table[] = {
 	{ "bq27200", BQ27000 },
 	{ "bq27210", BQ27010 },
 	{ "bq27500", BQ27500 },
-	{ "bq27510", BQ27500 },
-	{ "bq27520", BQ27500 },
+	{ "bq27510", BQ27510 },
+	{ "bq27520", BQ27510 },
 	{ "bq27530", BQ27530 },
 	{ "bq27531", BQ27530 },
 	{ "bq27541", BQ27541 },
diff --git a/drivers/power/supply/ipaq_micro_battery.c b/drivers/power/supply/ipaq_micro_battery.c
index 4af7b770f293..2fa6edd6e8b1 100644
--- a/drivers/power/supply/ipaq_micro_battery.c
+++ b/drivers/power/supply/ipaq_micro_battery.c
@@ -313,4 +313,4 @@ module_platform_driver(micro_batt_device_driver);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("driver for iPAQ Atmel micro battery");
-MODULE_ALIAS("platform:battery-ipaq-micro");
+MODULE_ALIAS("platform:ipaq-micro-battery");
diff --git a/drivers/power/supply/lp8788-charger.c b/drivers/power/supply/lp8788-charger.c
index 7321b727d484..509e2b341bd6 100644
--- a/drivers/power/supply/lp8788-charger.c
+++ b/drivers/power/supply/lp8788-charger.c
@@ -384,9 +384,6 @@ static int lp8788_update_charger_params(struct platform_device *pdev,
 	for (i = 0; i < pdata->num_chg_params; i++) {
 		param = pdata->chg_params + i;
 
-		if (!param)
-			continue;
-
 		if (lp8788_is_valid_charger_register(param->addr)) {
 			ret = lp8788_write_byte(lp, param->addr, param->val);
 			if (ret)
diff --git a/drivers/power/supply/max17040_battery.c b/drivers/power/supply/max17040_battery.c
index 8689c80202b5..e7c3649b31a0 100644
--- a/drivers/power/supply/max17040_battery.c
+++ b/drivers/power/supply/max17040_battery.c
@@ -21,18 +21,13 @@
 #include <linux/max17040_battery.h>
 #include <linux/slab.h>
 
-#define MAX17040_VCELL_MSB	0x02
-#define MAX17040_VCELL_LSB	0x03
-#define MAX17040_SOC_MSB	0x04
-#define MAX17040_SOC_LSB	0x05
-#define MAX17040_MODE_MSB	0x06
-#define MAX17040_MODE_LSB	0x07
-#define MAX17040_VER_MSB	0x08
-#define MAX17040_VER_LSB	0x09
-#define MAX17040_RCOMP_MSB	0x0C
-#define MAX17040_RCOMP_LSB	0x0D
-#define MAX17040_CMD_MSB	0xFE
-#define MAX17040_CMD_LSB	0xFF
+#define MAX17040_VCELL	0x02
+#define MAX17040_SOC	0x04
+#define MAX17040_MODE	0x06
+#define MAX17040_VER	0x08
+#define MAX17040_RCOMP	0x0C
+#define MAX17040_CMD	0xFE
+
 
 #define MAX17040_DELAY		1000
 #define MAX17040_BATTERY_FULL	95
@@ -78,11 +73,11 @@ static int max17040_get_property(struct power_supply *psy,
 	return 0;
 }
 
-static int max17040_write_reg(struct i2c_client *client, int reg, u8 value)
+static int max17040_write_reg(struct i2c_client *client, int reg, u16 value)
 {
 	int ret;
 
-	ret = i2c_smbus_write_byte_data(client, reg, value);
+	ret = i2c_smbus_write_word_swapped(client, reg, value);
 
 	if (ret < 0)
 		dev_err(&client->dev, "%s: err %d\n", __func__, ret);
@@ -94,7 +89,7 @@ static int max17040_read_reg(struct i2c_client *client, int reg)
 {
 	int ret;
 
-	ret = i2c_smbus_read_byte_data(client, reg);
+	ret = i2c_smbus_read_word_swapped(client, reg);
 
 	if (ret < 0)
 		dev_err(&client->dev, "%s: err %d\n", __func__, ret);
@@ -104,43 +99,36 @@ static int max17040_read_reg(struct i2c_client *client, int reg)
 
 static void max17040_reset(struct i2c_client *client)
 {
-	max17040_write_reg(client, MAX17040_CMD_MSB, 0x54);
-	max17040_write_reg(client, MAX17040_CMD_LSB, 0x00);
+	max17040_write_reg(client, MAX17040_CMD, 0x0054);
 }
 
 static void max17040_get_vcell(struct i2c_client *client)
 {
 	struct max17040_chip *chip = i2c_get_clientdata(client);
-	u8 msb;
-	u8 lsb;
+	u16 vcell;
 
-	msb = max17040_read_reg(client, MAX17040_VCELL_MSB);
-	lsb = max17040_read_reg(client, MAX17040_VCELL_LSB);
+	vcell = max17040_read_reg(client, MAX17040_VCELL);
 
-	chip->vcell = (msb << 4) + (lsb >> 4);
+	chip->vcell = vcell;
 }
 
 static void max17040_get_soc(struct i2c_client *client)
 {
 	struct max17040_chip *chip = i2c_get_clientdata(client);
-	u8 msb;
-	u8 lsb;
+	u16 soc;
 
-	msb = max17040_read_reg(client, MAX17040_SOC_MSB);
-	lsb = max17040_read_reg(client, MAX17040_SOC_LSB);
+	soc = max17040_read_reg(client, MAX17040_SOC);
 
-	chip->soc = msb;
+	chip->soc = (soc >> 8);
 }
 
 static void max17040_get_version(struct i2c_client *client)
 {
-	u8 msb;
-	u8 lsb;
+	u16 version;
 
-	msb = max17040_read_reg(client, MAX17040_VER_MSB);
-	lsb = max17040_read_reg(client, MAX17040_VER_LSB);
+	version = max17040_read_reg(client, MAX17040_VER);
 
-	dev_info(&client->dev, "MAX17040 Fuel-Gauge Ver %d%d\n", msb, lsb);
+	dev_info(&client->dev, "MAX17040 Fuel-Gauge Ver 0x%x\n", version);
 }
 
 static void max17040_get_online(struct i2c_client *client)
diff --git a/drivers/power/supply/max8997_charger.c b/drivers/power/supply/max8997_charger.c
index 0b2eab571528..290ddc12b040 100644
--- a/drivers/power/supply/max8997_charger.c
+++ b/drivers/power/supply/max8997_charger.c
@@ -184,6 +184,7 @@ static const struct platform_device_id max8997_battery_id[] = {
 	{ "max8997-battery", 0 },
 	{ }
 };
+MODULE_DEVICE_TABLE(platform, max8997_battery_id);
 
 static struct platform_driver max8997_battery_driver = {
 	.driver = {
diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c
index a74d8ca383a1..1e0960b646e8 100644
--- a/drivers/power/supply/power_supply_core.c
+++ b/drivers/power/supply/power_supply_core.c
@@ -413,7 +413,7 @@ static int power_supply_match_device_node(struct device *dev, const void *data)
 /**
  * power_supply_get_by_phandle() - Search for a power supply and returns its ref
  * @np: Pointer to device node holding phandle property
- * @phandle_name: Name of property holding a power supply name
+ * @property: Name of property holding a power supply name
  *
  * If power supply was found, it increases reference count for the
  * internal power supply's device. The user should power_supply_put()
@@ -458,7 +458,7 @@ static void devm_power_supply_put(struct device *dev, void *res)
  * devm_power_supply_get_by_phandle() - Resource managed version of
  *  power_supply_get_by_phandle()
  * @dev: Pointer to device holding phandle property
- * @phandle_name: Name of property holding a power supply phandle
+ * @property: Name of property holding a power supply phandle
  *
  * Return: On success returns a reference to a power supply with
  * matching name equals to value under @property, NULL or ERR_PTR otherwise.
diff --git a/drivers/power/supply/wm8350_power.c b/drivers/power/supply/wm8350_power.c
index 5c5880664e09..a2740cf57ad3 100644
--- a/drivers/power/supply/wm8350_power.c
+++ b/drivers/power/supply/wm8350_power.c
@@ -182,7 +182,7 @@ static ssize_t charger_state_show(struct device *dev,
 	return sprintf(buf, "%s\n", charge);
 }
 
-static DEVICE_ATTR(charger_state, 0444, charger_state_show, NULL);
+static DEVICE_ATTR_RO(charger_state);
 
 static irqreturn_t wm8350_charger_handler(int irq, void *data)
 {
diff --git a/drivers/power/supply/wm97xx_battery.c b/drivers/power/supply/wm97xx_battery.c
index 6285626d142a..e3edb31ac880 100644
--- a/drivers/power/supply/wm97xx_battery.c
+++ b/drivers/power/supply/wm97xx_battery.c
@@ -30,8 +30,7 @@ static enum power_supply_property *prop;
 
 static unsigned long wm97xx_read_bat(struct power_supply *bat_ps)
 {
-	struct wm97xx_pdata *wmdata = bat_ps->dev.parent->platform_data;
-	struct wm97xx_batt_pdata *pdata = wmdata->batt_pdata;
+	struct wm97xx_batt_pdata *pdata = power_supply_get_drvdata(bat_ps);
 
 	return wm97xx_read_aux_adc(dev_get_drvdata(bat_ps->dev.parent),
 					pdata->batt_aux) * pdata->batt_mult /
@@ -40,8 +39,7 @@ static unsigned long wm97xx_read_bat(struct power_supply *bat_ps)
 
 static unsigned long wm97xx_read_temp(struct power_supply *bat_ps)
 {
-	struct wm97xx_pdata *wmdata = bat_ps->dev.parent->platform_data;
-	struct wm97xx_batt_pdata *pdata = wmdata->batt_pdata;
+	struct wm97xx_batt_pdata *pdata = power_supply_get_drvdata(bat_ps);
 
 	return wm97xx_read_aux_adc(dev_get_drvdata(bat_ps->dev.parent),
 					pdata->temp_aux) * pdata->temp_mult /
@@ -52,8 +50,7 @@ static int wm97xx_bat_get_property(struct power_supply *bat_ps,
 			    enum power_supply_property psp,
 			    union power_supply_propval *val)
 {
-	struct wm97xx_pdata *wmdata = bat_ps->dev.parent->platform_data;
-	struct wm97xx_batt_pdata *pdata = wmdata->batt_pdata;
+	struct wm97xx_batt_pdata *pdata = power_supply_get_drvdata(bat_ps);
 
 	switch (psp) {
 	case POWER_SUPPLY_PROP_STATUS:
@@ -103,8 +100,7 @@ static void wm97xx_bat_external_power_changed(struct power_supply *bat_ps)
 static void wm97xx_bat_update(struct power_supply *bat_ps)
 {
 	int old_status = bat_status;
-	struct wm97xx_pdata *wmdata = bat_ps->dev.parent->platform_data;
-	struct wm97xx_batt_pdata *pdata = wmdata->batt_pdata;
+	struct wm97xx_batt_pdata *pdata = power_supply_get_drvdata(bat_ps);
 
 	mutex_lock(&work_lock);
 
@@ -166,15 +162,15 @@ static int wm97xx_bat_probe(struct platform_device *dev)
 	int ret = 0;
 	int props = 1;	/* POWER_SUPPLY_PROP_PRESENT */
 	int i = 0;
-	struct wm97xx_pdata *wmdata = dev->dev.platform_data;
-	struct wm97xx_batt_pdata *pdata;
+	struct wm97xx_batt_pdata *pdata = dev->dev.platform_data;
+	struct power_supply_config cfg = {};
 
-	if (!wmdata) {
+	if (!pdata) {
 		dev_err(&dev->dev, "No platform data supplied\n");
 		return -EINVAL;
 	}
 
-	pdata = wmdata->batt_pdata;
+	cfg.drv_data = pdata;
 
 	if (dev->id != -1)
 		return -EINVAL;
@@ -243,7 +239,7 @@ static int wm97xx_bat_probe(struct platform_device *dev)
 	bat_psy_desc.properties = prop;
 	bat_psy_desc.num_properties = props;
 
-	bat_psy = power_supply_register(&dev->dev, &bat_psy_desc, NULL);
+	bat_psy = power_supply_register(&dev->dev, &bat_psy_desc, &cfg);
 	if (!IS_ERR(bat_psy)) {
 		schedule_work(&bat_work);
 	} else {
@@ -266,8 +262,7 @@ err:
 
 static int wm97xx_bat_remove(struct platform_device *dev)
 {
-	struct wm97xx_pdata *wmdata = dev->dev.platform_data;
-	struct wm97xx_batt_pdata *pdata = wmdata->batt_pdata;
+	struct wm97xx_batt_pdata *pdata = dev->dev.platform_data;
 
 	if (pdata && gpio_is_valid(pdata->charge_gpio)) {
 		free_irq(gpio_to_irq(pdata->charge_gpio), dev);
diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c
index 243b233ff31b..9a25110c4a46 100644
--- a/drivers/powercap/intel_rapl.c
+++ b/drivers/powercap/intel_rapl.c
@@ -189,14 +189,13 @@ struct rapl_package {
 	unsigned int time_unit;
 	struct rapl_domain *domains; /* array of domains, sized at runtime */
 	struct powercap_zone *power_zone; /* keep track of parent zone */
-	int nr_cpus; /* active cpus on the package, topology info is lost during
-		      * cpu hotplug. so we have to track ourselves.
-		      */
 	unsigned long power_limit_irq; /* keep track of package power limit
 					* notify interrupt enable status.
 					*/
 	struct list_head plist;
 	int lead_cpu; /* one active cpu per package for access */
+	/* Track active cpus */
+	struct cpumask cpumask;
 };
 
 struct rapl_defaults {
@@ -275,18 +274,6 @@ static struct rapl_package *find_package_by_id(int id)
 	return NULL;
 }
 
-/* caller must hold cpu hotplug lock */
-static void rapl_cleanup_data(void)
-{
-	struct rapl_package *p, *tmp;
-
-	list_for_each_entry_safe(p, tmp, &rapl_packages, plist) {
-		kfree(p->domains);
-		list_del(&p->plist);
-		kfree(p);
-	}
-}
-
 static int get_energy_counter(struct powercap_zone *power_zone, u64 *energy_raw)
 {
 	struct rapl_domain *rd;
@@ -442,6 +429,7 @@ static int contraint_to_pl(struct rapl_domain *rd, int cid)
 			return i;
 		}
 	}
+	pr_err("Cannot find matching power limit for constraint %d\n", cid);
 
 	return -EINVAL;
 }
@@ -457,6 +445,10 @@ static int set_power_limit(struct powercap_zone *power_zone, int cid,
 	get_online_cpus();
 	rd = power_zone_to_rapl_domain(power_zone);
 	id = contraint_to_pl(rd, cid);
+	if (id < 0) {
+		ret = id;
+		goto set_exit;
+	}
 
 	rp = rd->rp;
 
@@ -496,6 +488,11 @@ static int get_current_power_limit(struct powercap_zone *power_zone, int cid,
 	get_online_cpus();
 	rd = power_zone_to_rapl_domain(power_zone);
 	id = contraint_to_pl(rd, cid);
+	if (id < 0) {
+		ret = id;
+		goto get_exit;
+	}
+
 	switch (rd->rpl[id].prim_id) {
 	case PL1_ENABLE:
 		prim = POWER_LIMIT1;
@@ -512,6 +509,7 @@ static int get_current_power_limit(struct powercap_zone *power_zone, int cid,
 	else
 		*data = val;
 
+get_exit:
 	put_online_cpus();
 
 	return ret;
@@ -527,6 +525,10 @@ static int set_time_window(struct powercap_zone *power_zone, int cid,
 	get_online_cpus();
 	rd = power_zone_to_rapl_domain(power_zone);
 	id = contraint_to_pl(rd, cid);
+	if (id < 0) {
+		ret = id;
+		goto set_time_exit;
+	}
 
 	switch (rd->rpl[id].prim_id) {
 	case PL1_ENABLE:
@@ -538,6 +540,8 @@ static int set_time_window(struct powercap_zone *power_zone, int cid,
 	default:
 		ret = -EINVAL;
 	}
+
+set_time_exit:
 	put_online_cpus();
 	return ret;
 }
@@ -552,6 +556,10 @@ static int get_time_window(struct powercap_zone *power_zone, int cid, u64 *data)
 	get_online_cpus();
 	rd = power_zone_to_rapl_domain(power_zone);
 	id = contraint_to_pl(rd, cid);
+	if (id < 0) {
+		ret = id;
+		goto get_time_exit;
+	}
 
 	switch (rd->rpl[id].prim_id) {
 	case PL1_ENABLE:
@@ -566,6 +574,8 @@ static int get_time_window(struct powercap_zone *power_zone, int cid, u64 *data)
 	}
 	if (!ret)
 		*data = val;
+
+get_time_exit:
 	put_online_cpus();
 
 	return ret;
@@ -707,7 +717,7 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type,
 	case ENERGY_UNIT:
 		scale = ENERGY_UNIT_SCALE;
 		/* per domain unit takes precedence */
-		if (rd && rd->domain_energy_unit)
+		if (rd->domain_energy_unit)
 			units = rd->domain_energy_unit;
 		else
 			units = rp->energy_unit;
@@ -976,10 +986,20 @@ static void package_power_limit_irq_save(struct rapl_package *rp)
 	smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1);
 }
 
-static void power_limit_irq_restore_cpu(void *info)
+/*
+ * Restore per package power limit interrupt enable state. Called from cpu
+ * hotplug code on package removal.
+ */
+static void package_power_limit_irq_restore(struct rapl_package *rp)
 {
-	u32 l, h = 0;
-	struct rapl_package *rp = (struct rapl_package *)info;
+	u32 l, h;
+
+	if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
+		return;
+
+	/* irq enable state not saved, nothing to restore */
+	if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))
+		return;
 
 	rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
 
@@ -991,19 +1011,6 @@ static void power_limit_irq_restore_cpu(void *info)
 	wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
 }
 
-/* restore per package power limit interrupt enable state */
-static void package_power_limit_irq_restore(struct rapl_package *rp)
-{
-	if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
-		return;
-
-	/* irq enable state not saved, nothing to restore */
-	if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))
-		return;
-
-	smp_call_function_single(rp->lead_cpu, power_limit_irq_restore_cpu, rp, 1);
-}
-
 static void set_floor_freq_default(struct rapl_domain *rd, bool mode)
 {
 	int nr_powerlimit = find_nr_power_limit(rd);
@@ -1160,84 +1167,49 @@ static const struct x86_cpu_id rapl_ids[] __initconst = {
 	RAPL_CPU(INTEL_FAM6_ATOM_DENVERTON,	rapl_defaults_core),
 
 	RAPL_CPU(INTEL_FAM6_XEON_PHI_KNL,	rapl_defaults_hsw_server),
+	RAPL_CPU(INTEL_FAM6_XEON_PHI_KNM,	rapl_defaults_hsw_server),
 	{}
 };
 MODULE_DEVICE_TABLE(x86cpu, rapl_ids);
 
-/* read once for all raw primitive data for all packages, domains */
-static void rapl_update_domain_data(void)
+/* Read once for all raw primitive data for domains */
+static void rapl_update_domain_data(struct rapl_package *rp)
 {
 	int dmn, prim;
 	u64 val;
-	struct rapl_package *rp;
 
-	list_for_each_entry(rp, &rapl_packages, plist) {
-		for (dmn = 0; dmn < rp->nr_domains; dmn++) {
-			pr_debug("update package %d domain %s data\n", rp->id,
-				rp->domains[dmn].name);
-			/* exclude non-raw primitives */
-			for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++)
-				if (!rapl_read_data_raw(&rp->domains[dmn], prim,
-								rpi[prim].unit,
-								&val))
-					rp->domains[dmn].rdd.primitives[prim] =
-									val;
+	for (dmn = 0; dmn < rp->nr_domains; dmn++) {
+		pr_debug("update package %d domain %s data\n", rp->id,
+			 rp->domains[dmn].name);
+		/* exclude non-raw primitives */
+		for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) {
+			if (!rapl_read_data_raw(&rp->domains[dmn], prim,
+						rpi[prim].unit, &val))
+				rp->domains[dmn].rdd.primitives[prim] =	val;
 		}
 	}
 
 }
 
-static int rapl_unregister_powercap(void)
+static void rapl_unregister_powercap(void)
 {
-	struct rapl_package *rp;
-	struct rapl_domain *rd, *rd_package = NULL;
-
-	/* unregister all active rapl packages from the powercap layer,
-	 * hotplug lock held
-	 */
-	list_for_each_entry(rp, &rapl_packages, plist) {
-		package_power_limit_irq_restore(rp);
-
-		for (rd = rp->domains; rd < rp->domains + rp->nr_domains;
-		     rd++) {
-			pr_debug("remove package, undo power limit on %d: %s\n",
-				rp->id, rd->name);
-			rapl_write_data_raw(rd, PL1_ENABLE, 0);
-			rapl_write_data_raw(rd, PL1_CLAMP, 0);
-			if (find_nr_power_limit(rd) > 1) {
-				rapl_write_data_raw(rd, PL2_ENABLE, 0);
-				rapl_write_data_raw(rd, PL2_CLAMP, 0);
-			}
-			if (rd->id == RAPL_DOMAIN_PACKAGE) {
-				rd_package = rd;
-				continue;
-			}
-			powercap_unregister_zone(control_type, &rd->power_zone);
-		}
-		/* do the package zone last */
-		if (rd_package)
-			powercap_unregister_zone(control_type,
-						&rd_package->power_zone);
-	}
-
 	if (platform_rapl_domain) {
 		powercap_unregister_zone(control_type,
 					 &platform_rapl_domain->power_zone);
 		kfree(platform_rapl_domain);
 	}
-
 	powercap_unregister_control_type(control_type);
-
-	return 0;
 }
 
 static int rapl_package_register_powercap(struct rapl_package *rp)
 {
 	struct rapl_domain *rd;
-	int ret = 0;
 	char dev_name[17]; /* max domain name = 7 + 1 + 8 for int + 1 for null*/
 	struct powercap_zone *power_zone = NULL;
-	int nr_pl;
+	int nr_pl, ret;;
+
+	/* Update the domain data of the new package */
+	rapl_update_domain_data(rp);
 
 	/* first we register package domain as the parent zone*/
 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
@@ -1257,8 +1229,7 @@ static int rapl_package_register_powercap(struct rapl_package *rp)
 			if (IS_ERR(power_zone)) {
 				pr_debug("failed to register package, %d\n",
 					rp->id);
-				ret = PTR_ERR(power_zone);
-				goto exit_package;
+				return PTR_ERR(power_zone);
 			}
 			/* track parent zone in per package/socket data */
 			rp->power_zone = power_zone;
@@ -1268,8 +1239,7 @@ static int rapl_package_register_powercap(struct rapl_package *rp)
 	}
 	if (!power_zone) {
 		pr_err("no package domain found, unknown topology!\n");
-		ret = -ENODEV;
-		goto exit_package;
+		return -ENODEV;
 	}
 	/* now register domains as children of the socket/package*/
 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
@@ -1290,11 +1260,11 @@ static int rapl_package_register_powercap(struct rapl_package *rp)
 			goto err_cleanup;
 		}
 	}
+	return 0;
 
-exit_package:
-	return ret;
 err_cleanup:
-	/* clean up previously initialized domains within the package if we
+	/*
+	 * Clean up previously initialized domains within the package if we
 	 * failed after the first domain setup.
 	 */
 	while (--rd >= rp->domains) {
@@ -1305,7 +1275,7 @@ err_cleanup:
 	return ret;
 }
 
-static int rapl_register_psys(void)
+static int __init rapl_register_psys(void)
 {
 	struct rapl_domain *rd;
 	struct powercap_zone *power_zone;
@@ -1346,40 +1316,14 @@ static int rapl_register_psys(void)
 	return 0;
 }
 
-static int rapl_register_powercap(void)
+static int __init rapl_register_powercap(void)
 {
-	struct rapl_domain *rd;
-	struct rapl_package *rp;
-	int ret = 0;
-
 	control_type = powercap_register_control_type(NULL, "intel-rapl", NULL);
 	if (IS_ERR(control_type)) {
 		pr_debug("failed to register powercap control_type.\n");
 		return PTR_ERR(control_type);
 	}
-	/* read the initial data */
-	rapl_update_domain_data();
-	list_for_each_entry(rp, &rapl_packages, plist)
-		if (rapl_package_register_powercap(rp))
-			goto err_cleanup_package;
-
-	/* Don't bail out if PSys is not supported */
-	rapl_register_psys();
-
-	return ret;
-
-err_cleanup_package:
-	/* clean up previously initialized packages */
-	list_for_each_entry_continue_reverse(rp, &rapl_packages, plist) {
-		for (rd = rp->domains; rd < rp->domains + rp->nr_domains;
-		     rd++) {
-			pr_debug("unregister zone/package %d, %s domain\n",
-				rp->id, rd->name);
-			powercap_unregister_zone(control_type, &rd->power_zone);
-		}
-	}
-
-	return ret;
+	return 0;
 }
 
 static int rapl_check_domain(int cpu, int domain)
@@ -1452,9 +1396,8 @@ static void rapl_detect_powerlimit(struct rapl_domain *rd)
  */
 static int rapl_detect_domains(struct rapl_package *rp, int cpu)
 {
-	int i;
-	int ret = 0;
 	struct rapl_domain *rd;
+	int i;
 
 	for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
 		/* use physical package id to read counters */
@@ -1466,84 +1409,20 @@ static int rapl_detect_domains(struct rapl_package *rp, int cpu)
 	rp->nr_domains = bitmap_weight(&rp->domain_map,	RAPL_DOMAIN_MAX);
 	if (!rp->nr_domains) {
 		pr_debug("no valid rapl domains found in package %d\n", rp->id);
-		ret = -ENODEV;
-		goto done;
+		return -ENODEV;
 	}
 	pr_debug("found %d domains on package %d\n", rp->nr_domains, rp->id);
 
 	rp->domains = kcalloc(rp->nr_domains + 1, sizeof(struct rapl_domain),
 			GFP_KERNEL);
-	if (!rp->domains) {
-		ret = -ENOMEM;
-		goto done;
-	}
+	if (!rp->domains)
+		return -ENOMEM;
+
 	rapl_init_domains(rp);
 
 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++)
 		rapl_detect_powerlimit(rd);
 
-
-
-done:
-	return ret;
-}
-
-static bool is_package_new(int package)
-{
-	struct rapl_package *rp;
-
-	/* caller prevents cpu hotplug, there will be no new packages added
-	 * or deleted while traversing the package list, no need for locking.
-	 */
-	list_for_each_entry(rp, &rapl_packages, plist)
-		if (package == rp->id)
-			return false;
-
-	return true;
-}
-
-/* RAPL interface can be made of a two-level hierarchy: package level and domain
- * level. We first detect the number of packages then domains of each package.
- * We have to consider the possiblity of CPU online/offline due to hotplug and
- * other scenarios.
- */
-static int rapl_detect_topology(void)
-{
-	int i;
-	int phy_package_id;
-	struct rapl_package *new_package, *rp;
-
-	for_each_online_cpu(i) {
-		phy_package_id = topology_physical_package_id(i);
-		if (is_package_new(phy_package_id)) {
-			new_package = kzalloc(sizeof(*rp), GFP_KERNEL);
-			if (!new_package) {
-				rapl_cleanup_data();
-				return -ENOMEM;
-			}
-			/* add the new package to the list */
-			new_package->id = phy_package_id;
-			new_package->nr_cpus = 1;
-			/* use the first active cpu of the package to access */
-			new_package->lead_cpu = i;
-			/* check if the package contains valid domains */
-			if (rapl_detect_domains(new_package, i) ||
-				rapl_defaults->check_unit(new_package, i)) {
-				kfree(new_package->domains);
-				kfree(new_package);
-				/* free up the packages already initialized */
-				rapl_cleanup_data();
-				return -ENODEV;
-			}
-			INIT_LIST_HEAD(&new_package->plist);
-			list_add(&new_package->plist, &rapl_packages);
-		} else {
-			rp = find_package_by_id(phy_package_id);
-			if (rp)
-				++rp->nr_cpus;
-		}
-	}
-
 	return 0;
 }
 
@@ -1552,12 +1431,21 @@ static void rapl_remove_package(struct rapl_package *rp)
 {
 	struct rapl_domain *rd, *rd_package = NULL;
 
+	package_power_limit_irq_restore(rp);
+
 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
+		rapl_write_data_raw(rd, PL1_ENABLE, 0);
+		rapl_write_data_raw(rd, PL1_CLAMP, 0);
+		if (find_nr_power_limit(rd) > 1) {
+			rapl_write_data_raw(rd, PL2_ENABLE, 0);
+			rapl_write_data_raw(rd, PL2_CLAMP, 0);
+		}
 		if (rd->id == RAPL_DOMAIN_PACKAGE) {
 			rd_package = rd;
 			continue;
 		}
-		pr_debug("remove package %d, %s domain\n", rp->id, rd->name);
+		pr_debug("remove package, undo power limit on %d: %s\n",
+			 rp->id, rd->name);
 		powercap_unregister_zone(control_type, &rd->power_zone);
 	}
 	/* do parent zone last */
@@ -1567,20 +1455,17 @@ static void rapl_remove_package(struct rapl_package *rp)
 }
 
 /* called from CPU hotplug notifier, hotplug lock held */
-static int rapl_add_package(int cpu)
+static struct rapl_package *rapl_add_package(int cpu, int pkgid)
 {
-	int ret = 0;
-	int phy_package_id;
 	struct rapl_package *rp;
+	int ret;
 
-	phy_package_id = topology_physical_package_id(cpu);
 	rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL);
 	if (!rp)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	/* add the new package to the list */
-	rp->id = phy_package_id;
-	rp->nr_cpus = 1;
+	rp->id = pkgid;
 	rp->lead_cpu = cpu;
 
 	/* check if the package contains valid domains */
@@ -1589,17 +1474,17 @@ static int rapl_add_package(int cpu)
 		ret = -ENODEV;
 		goto err_free_package;
 	}
-	if (!rapl_package_register_powercap(rp)) {
+	ret = rapl_package_register_powercap(rp);
+	if (!ret) {
 		INIT_LIST_HEAD(&rp->plist);
 		list_add(&rp->plist, &rapl_packages);
-		return ret;
+		return rp;
 	}
 
 err_free_package:
 	kfree(rp->domains);
 	kfree(rp);
-
-	return ret;
+	return ERR_PTR(ret);
 }
 
 /* Handles CPU hotplug on multi-socket systems.
@@ -1609,55 +1494,46 @@ err_free_package:
  * associated domains. Cooling devices are handled accordingly at
  * per-domain level.
  */
-static int rapl_cpu_callback(struct notifier_block *nfb,
-				unsigned long action, void *hcpu)
+static int rapl_cpu_online(unsigned int cpu)
 {
-	unsigned long cpu = (unsigned long)hcpu;
-	int phy_package_id;
+	int pkgid = topology_physical_package_id(cpu);
 	struct rapl_package *rp;
-	int lead_cpu;
 
-	phy_package_id = topology_physical_package_id(cpu);
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-	case CPU_DOWN_FAILED:
-	case CPU_DOWN_FAILED_FROZEN:
-		rp = find_package_by_id(phy_package_id);
-		if (rp)
-			++rp->nr_cpus;
-		else
-			rapl_add_package(cpu);
-		break;
-	case CPU_DOWN_PREPARE:
-	case CPU_DOWN_PREPARE_FROZEN:
-		rp = find_package_by_id(phy_package_id);
-		if (!rp)
-			break;
-		if (--rp->nr_cpus == 0)
-			rapl_remove_package(rp);
-		else if (cpu == rp->lead_cpu) {
-			/* choose another active cpu in the package */
-			lead_cpu = cpumask_any_but(topology_core_cpumask(cpu), cpu);
-			if (lead_cpu < nr_cpu_ids)
-				rp->lead_cpu = lead_cpu;
-			else /* should never go here */
-				pr_err("no active cpu available for package %d\n",
-					phy_package_id);
-		}
+	rp = find_package_by_id(pkgid);
+	if (!rp) {
+		rp = rapl_add_package(cpu, pkgid);
+		if (IS_ERR(rp))
+			return PTR_ERR(rp);
 	}
+	cpumask_set_cpu(cpu, &rp->cpumask);
+	return 0;
+}
+
+static int rapl_cpu_down_prep(unsigned int cpu)
+{
+	int pkgid = topology_physical_package_id(cpu);
+	struct rapl_package *rp;
+	int lead_cpu;
+
+	rp = find_package_by_id(pkgid);
+	if (!rp)
+		return 0;
 
-	return NOTIFY_OK;
+	cpumask_clear_cpu(cpu, &rp->cpumask);
+	lead_cpu = cpumask_first(&rp->cpumask);
+	if (lead_cpu >= nr_cpu_ids)
+		rapl_remove_package(rp);
+	else if (rp->lead_cpu == cpu)
+		rp->lead_cpu = lead_cpu;
+	return 0;
 }
 
-static struct notifier_block rapl_cpu_notifier = {
-	.notifier_call = rapl_cpu_callback,
-};
+static enum cpuhp_state pcap_rapl_online;
 
 static int __init rapl_init(void)
 {
-	int ret = 0;
 	const struct x86_cpu_id *id;
+	int ret;
 
 	id = x86_match_cpu(rapl_ids);
 	if (!id) {
@@ -1669,36 +1545,29 @@ static int __init rapl_init(void)
 
 	rapl_defaults = (struct rapl_defaults *)id->driver_data;
 
-	cpu_notifier_register_begin();
-
-	/* prevent CPU hotplug during detection */
-	get_online_cpus();
-	ret = rapl_detect_topology();
+	ret = rapl_register_powercap();
 	if (ret)
-		goto done;
+		return ret;
 
-	if (rapl_register_powercap()) {
-		rapl_cleanup_data();
-		ret = -ENODEV;
-		goto done;
-	}
-	__register_hotcpu_notifier(&rapl_cpu_notifier);
-done:
-	put_online_cpus();
-	cpu_notifier_register_done();
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powercap/rapl:online",
+				rapl_cpu_online, rapl_cpu_down_prep);
+	if (ret < 0)
+		goto err_unreg;
+	pcap_rapl_online = ret;
+
+	/* Don't bail out if PSys is not supported */
+	rapl_register_psys();
+	return 0;
 
+err_unreg:
+	rapl_unregister_powercap();
 	return ret;
 }
 
 static void __exit rapl_exit(void)
 {
-	cpu_notifier_register_begin();
-	get_online_cpus();
-	__unregister_hotcpu_notifier(&rapl_cpu_notifier);
+	cpuhp_remove_state(pcap_rapl_online);
 	rapl_unregister_powercap();
-	rapl_cleanup_data();
-	put_online_cpus();
-	cpu_notifier_register_done();
 }
 
 module_init(rapl_init);
diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig
index bf0128899c09..f92dd41b0395 100644
--- a/drivers/pwm/Kconfig
+++ b/drivers/pwm/Kconfig
@@ -175,6 +175,15 @@ config PWM_FSL_FTM
 	  To compile this driver as a module, choose M here: the module
 	  will be called pwm-fsl-ftm.
 
+config PWM_HIBVT
+	tristate "HiSilicon BVT PWM support"
+	depends on ARCH_HISI || COMPILE_TEST
+	help
+	  Generic PWM framework driver for HiSilicon BVT SoCs.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called pwm-hibvt.
+
 config PWM_IMG
 	tristate "Imagination Technologies PWM driver"
 	depends on HAS_IOMEM
diff --git a/drivers/pwm/Makefile b/drivers/pwm/Makefile
index 1194c54efcc2..a48bdb517792 100644
--- a/drivers/pwm/Makefile
+++ b/drivers/pwm/Makefile
@@ -15,6 +15,7 @@ obj-$(CONFIG_PWM_CRC)		+= pwm-crc.o
 obj-$(CONFIG_PWM_CROS_EC)	+= pwm-cros-ec.o
 obj-$(CONFIG_PWM_EP93XX)	+= pwm-ep93xx.o
 obj-$(CONFIG_PWM_FSL_FTM)	+= pwm-fsl-ftm.o
+obj-$(CONFIG_PWM_HIBVT)		+= pwm-hibvt.o
 obj-$(CONFIG_PWM_IMG)		+= pwm-img.o
 obj-$(CONFIG_PWM_IMX)		+= pwm-imx.o
 obj-$(CONFIG_PWM_JZ4740)	+= pwm-jz4740.o
diff --git a/drivers/pwm/pwm-hibvt.c b/drivers/pwm/pwm-hibvt.c
new file mode 100644
index 000000000000..d0e8f8542626
--- /dev/null
+++ b/drivers/pwm/pwm-hibvt.c
@@ -0,0 +1,271 @@
+/*
+ * PWM Controller Driver for HiSilicon BVT SoCs
+ *
+ * Copyright (c) 2016 HiSilicon Technologies Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pwm.h>
+#include <linux/reset.h>
+
+#define PWM_CFG0_ADDR(x)    (((x) * 0x20) + 0x0)
+#define PWM_CFG1_ADDR(x)    (((x) * 0x20) + 0x4)
+#define PWM_CFG2_ADDR(x)    (((x) * 0x20) + 0x8)
+#define PWM_CTRL_ADDR(x)    (((x) * 0x20) + 0xC)
+
+#define PWM_ENABLE_SHIFT    0
+#define PWM_ENABLE_MASK     BIT(0)
+
+#define PWM_POLARITY_SHIFT  1
+#define PWM_POLARITY_MASK   BIT(1)
+
+#define PWM_KEEP_SHIFT      2
+#define PWM_KEEP_MASK       BIT(2)
+
+#define PWM_PERIOD_MASK     GENMASK(31, 0)
+#define PWM_DUTY_MASK       GENMASK(31, 0)
+
+struct hibvt_pwm_chip {
+	struct pwm_chip	chip;
+	struct clk *clk;
+	void __iomem *base;
+	struct reset_control *rstc;
+};
+
+struct hibvt_pwm_soc {
+	u32 num_pwms;
+};
+
+static const struct hibvt_pwm_soc pwm_soc[2] = {
+	{ .num_pwms = 4 },
+	{ .num_pwms = 8 },
+};
+
+static inline struct hibvt_pwm_chip *to_hibvt_pwm_chip(struct pwm_chip *chip)
+{
+	return container_of(chip, struct hibvt_pwm_chip, chip);
+}
+
+static void hibvt_pwm_set_bits(void __iomem *base, u32 offset,
+					u32 mask, u32 data)
+{
+	void __iomem *address = base + offset;
+	u32 value;
+
+	value = readl(address);
+	value &= ~mask;
+	value |= (data & mask);
+	writel(value, address);
+}
+
+static void hibvt_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
+{
+	struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip);
+
+	hibvt_pwm_set_bits(hi_pwm_chip->base, PWM_CTRL_ADDR(pwm->hwpwm),
+			PWM_ENABLE_MASK, 0x1);
+}
+
+static void hibvt_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
+{
+	struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip);
+
+	hibvt_pwm_set_bits(hi_pwm_chip->base, PWM_CTRL_ADDR(pwm->hwpwm),
+			PWM_ENABLE_MASK, 0x0);
+}
+
+static void hibvt_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
+					int duty_cycle_ns, int period_ns)
+{
+	struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip);
+	u32 freq, period, duty;
+
+	freq = div_u64(clk_get_rate(hi_pwm_chip->clk), 1000000);
+
+	period = div_u64(freq * period_ns, 1000);
+	duty = div_u64(period * duty_cycle_ns, period_ns);
+
+	hibvt_pwm_set_bits(hi_pwm_chip->base, PWM_CFG0_ADDR(pwm->hwpwm),
+			PWM_PERIOD_MASK, period);
+
+	hibvt_pwm_set_bits(hi_pwm_chip->base, PWM_CFG1_ADDR(pwm->hwpwm),
+			PWM_DUTY_MASK, duty);
+}
+
+static void hibvt_pwm_set_polarity(struct pwm_chip *chip,
+					struct pwm_device *pwm,
+					enum pwm_polarity polarity)
+{
+	struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip);
+
+	if (polarity == PWM_POLARITY_INVERSED)
+		hibvt_pwm_set_bits(hi_pwm_chip->base, PWM_CTRL_ADDR(pwm->hwpwm),
+				PWM_POLARITY_MASK, (0x1 << PWM_POLARITY_SHIFT));
+	else
+		hibvt_pwm_set_bits(hi_pwm_chip->base, PWM_CTRL_ADDR(pwm->hwpwm),
+				PWM_POLARITY_MASK, (0x0 << PWM_POLARITY_SHIFT));
+}
+
+static void hibvt_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm,
+				struct pwm_state *state)
+{
+	struct hibvt_pwm_chip *hi_pwm_chip = to_hibvt_pwm_chip(chip);
+	void __iomem *base;
+	u32 freq, value;
+
+	freq = div_u64(clk_get_rate(hi_pwm_chip->clk), 1000000);
+	base = hi_pwm_chip->base;
+
+	value = readl(base + PWM_CFG0_ADDR(pwm->hwpwm));
+	state->period = div_u64(value * 1000, freq);
+
+	value = readl(base + PWM_CFG1_ADDR(pwm->hwpwm));
+	state->duty_cycle = div_u64(value * 1000, freq);
+
+	value = readl(base + PWM_CTRL_ADDR(pwm->hwpwm));
+	state->enabled = (PWM_ENABLE_MASK & value);
+}
+
+static int hibvt_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
+				struct pwm_state *state)
+{
+	if (state->polarity != pwm->state.polarity)
+		hibvt_pwm_set_polarity(chip, pwm, state->polarity);
+
+	if (state->period != pwm->state.period ||
+		state->duty_cycle != pwm->state.duty_cycle)
+		hibvt_pwm_config(chip, pwm, state->duty_cycle, state->period);
+
+	if (state->enabled != pwm->state.enabled) {
+		if (state->enabled)
+			hibvt_pwm_enable(chip, pwm);
+		else
+			hibvt_pwm_disable(chip, pwm);
+	}
+
+	return 0;
+}
+
+static struct pwm_ops hibvt_pwm_ops = {
+	.get_state = hibvt_pwm_get_state,
+	.apply = hibvt_pwm_apply,
+
+	.owner = THIS_MODULE,
+};
+
+static int hibvt_pwm_probe(struct platform_device *pdev)
+{
+	const struct hibvt_pwm_soc *soc =
+				of_device_get_match_data(&pdev->dev);
+	struct hibvt_pwm_chip *pwm_chip;
+	struct resource *res;
+	int ret;
+	int i;
+
+	pwm_chip = devm_kzalloc(&pdev->dev, sizeof(*pwm_chip), GFP_KERNEL);
+	if (pwm_chip == NULL)
+		return -ENOMEM;
+
+	pwm_chip->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(pwm_chip->clk)) {
+		dev_err(&pdev->dev, "getting clock failed with %ld\n",
+				PTR_ERR(pwm_chip->clk));
+		return PTR_ERR(pwm_chip->clk);
+	}
+
+	pwm_chip->chip.ops = &hibvt_pwm_ops;
+	pwm_chip->chip.dev = &pdev->dev;
+	pwm_chip->chip.base = -1;
+	pwm_chip->chip.npwm = soc->num_pwms;
+	pwm_chip->chip.of_xlate = of_pwm_xlate_with_flags;
+	pwm_chip->chip.of_pwm_n_cells = 3;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	pwm_chip->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(pwm_chip->base))
+		return PTR_ERR(pwm_chip->base);
+
+	ret = clk_prepare_enable(pwm_chip->clk);
+	if (ret < 0)
+		return ret;
+
+	pwm_chip->rstc = devm_reset_control_get(&pdev->dev, NULL);
+	if (IS_ERR(pwm_chip->rstc)) {
+		clk_disable_unprepare(pwm_chip->clk);
+		return PTR_ERR(pwm_chip->rstc);
+	}
+
+	reset_control_assert(pwm_chip->rstc);
+	msleep(30);
+	reset_control_deassert(pwm_chip->rstc);
+
+	ret = pwmchip_add(&pwm_chip->chip);
+	if (ret < 0) {
+		clk_disable_unprepare(pwm_chip->clk);
+		return ret;
+	}
+
+	for (i = 0; i < pwm_chip->chip.npwm; i++) {
+		hibvt_pwm_set_bits(pwm_chip->base, PWM_CTRL_ADDR(i),
+				PWM_KEEP_MASK, (0x1 << PWM_KEEP_SHIFT));
+	}
+
+	platform_set_drvdata(pdev, pwm_chip);
+
+	return 0;
+}
+
+static int hibvt_pwm_remove(struct platform_device *pdev)
+{
+	struct hibvt_pwm_chip *pwm_chip;
+
+	pwm_chip = platform_get_drvdata(pdev);
+
+	reset_control_assert(pwm_chip->rstc);
+	msleep(30);
+	reset_control_deassert(pwm_chip->rstc);
+
+	clk_disable_unprepare(pwm_chip->clk);
+
+	return pwmchip_remove(&pwm_chip->chip);
+}
+
+static const struct of_device_id hibvt_pwm_of_match[] = {
+	{ .compatible = "hisilicon,hi3516cv300-pwm", .data = &pwm_soc[0] },
+	{ .compatible = "hisilicon,hi3519v100-pwm", .data = &pwm_soc[1] },
+	{  }
+};
+MODULE_DEVICE_TABLE(of, hibvt_pwm_of_match);
+
+static struct platform_driver hibvt_pwm_driver = {
+	.driver = {
+		.name = "hibvt-pwm",
+		.of_match_table = hibvt_pwm_of_match,
+	},
+	.probe = hibvt_pwm_probe,
+	.remove	= hibvt_pwm_remove,
+};
+module_platform_driver(hibvt_pwm_driver);
+
+MODULE_AUTHOR("Jian Yuan");
+MODULE_DESCRIPTION("HiSilicon BVT SoCs PWM driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pwm/pwm-meson.c b/drivers/pwm/pwm-meson.c
index 9d5bd7d5c610..045ef9fa6fe3 100644
--- a/drivers/pwm/pwm-meson.c
+++ b/drivers/pwm/pwm-meson.c
@@ -524,7 +524,6 @@ static struct platform_driver meson_pwm_driver = {
 };
 module_platform_driver(meson_pwm_driver);
 
-MODULE_ALIAS("platform:meson-pwm");
 MODULE_DESCRIPTION("Amlogic Meson PWM Generator driver");
 MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/regulator/Makefile b/drivers/regulator/Makefile
index 2142a5d3fc08..14294692beb9 100644
--- a/drivers/regulator/Makefile
+++ b/drivers/regulator/Makefile
@@ -104,7 +104,7 @@ obj-$(CONFIG_REGULATOR_TPS6586X) += tps6586x-regulator.o
 obj-$(CONFIG_REGULATOR_TPS65910) += tps65910-regulator.o
 obj-$(CONFIG_REGULATOR_TPS65912) += tps65912-regulator.o
 obj-$(CONFIG_REGULATOR_TPS80031) += tps80031-regulator.o
-obj-$(CONFIG_REGULATOR_TWL4030) += twl-regulator.o
+obj-$(CONFIG_REGULATOR_TWL4030) += twl-regulator.o twl6030-regulator.o
 obj-$(CONFIG_REGULATOR_VEXPRESS) += vexpress-regulator.o
 obj-$(CONFIG_REGULATOR_WM831X) += wm831x-dcdc.o
 obj-$(CONFIG_REGULATOR_WM831X) += wm831x-isink.o
diff --git a/drivers/regulator/arizona-ldo1.c b/drivers/regulator/arizona-ldo1.c
index f7c88ff90c43..302b57cb89c6 100644
--- a/drivers/regulator/arizona-ldo1.c
+++ b/drivers/regulator/arizona-ldo1.c
@@ -130,6 +130,7 @@ static const struct regulator_desc arizona_ldo1_hc = {
 	.uV_step = 50000,
 	.n_voltages = 8,
 	.enable_time = 1500,
+	.ramp_delay = 24000,
 
 	.owner = THIS_MODULE,
 };
@@ -153,6 +154,7 @@ static const struct regulator_desc arizona_ldo1 = {
 	.uV_step = 25000,
 	.n_voltages = 13,
 	.enable_time = 500,
+	.ramp_delay = 24000,
 
 	.owner = THIS_MODULE,
 };
diff --git a/drivers/regulator/axp20x-regulator.c b/drivers/regulator/axp20x-regulator.c
index 54382ef902c6..e6a512ebeae2 100644
--- a/drivers/regulator/axp20x-regulator.c
+++ b/drivers/regulator/axp20x-regulator.c
@@ -337,10 +337,18 @@ static const struct regulator_desc axp809_regulators[] = {
 		 AXP22X_ELDO2_V_OUT, 0x1f, AXP22X_PWR_OUT_CTRL2, BIT(1)),
 	AXP_DESC(AXP809, ELDO3, "eldo3", "eldoin", 700, 3300, 100,
 		 AXP22X_ELDO3_V_OUT, 0x1f, AXP22X_PWR_OUT_CTRL2, BIT(2)),
-	AXP_DESC_IO(AXP809, LDO_IO0, "ldo_io0", "ips", 700, 3300, 100,
+	/*
+	 * Note the datasheet only guarantees reliable operation up to
+	 * 3.3V, this needs to be enforced via dts provided constraints
+	 */
+	AXP_DESC_IO(AXP809, LDO_IO0, "ldo_io0", "ips", 700, 3800, 100,
 		    AXP22X_LDO_IO0_V_OUT, 0x1f, AXP20X_GPIO0_CTRL, 0x07,
 		    AXP22X_IO_ENABLED, AXP22X_IO_DISABLED),
-	AXP_DESC_IO(AXP809, LDO_IO1, "ldo_io1", "ips", 700, 3300, 100,
+	/*
+	 * Note the datasheet only guarantees reliable operation up to
+	 * 3.3V, this needs to be enforced via dts provided constraints
+	 */
+	AXP_DESC_IO(AXP809, LDO_IO1, "ldo_io1", "ips", 700, 3800, 100,
 		    AXP22X_LDO_IO1_V_OUT, 0x1f, AXP20X_GPIO1_CTRL, 0x07,
 		    AXP22X_IO_ENABLED, AXP22X_IO_DISABLED),
 	AXP_DESC_FIXED(AXP809, RTC_LDO, "rtc_ldo", "ips", 1800),
diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c
index 5c1519b229e0..04baac9a165b 100644
--- a/drivers/regulator/core.c
+++ b/drivers/regulator/core.c
@@ -204,7 +204,7 @@ static struct device_node *of_get_regulator(struct device *dev, const char *supp
 	regnode = of_parse_phandle(dev->of_node, prop_name, 0);
 
 	if (!regnode) {
-		dev_dbg(dev, "Looking up %s property in node %s failed",
+		dev_dbg(dev, "Looking up %s property in node %s failed\n",
 				prop_name, dev->of_node->full_name);
 		return NULL;
 	}
@@ -293,7 +293,8 @@ static int regulator_check_current_limit(struct regulator_dev *rdev,
 }
 
 /* operating mode constraint check */
-static int regulator_mode_constrain(struct regulator_dev *rdev, int *mode)
+static int regulator_mode_constrain(struct regulator_dev *rdev,
+				    unsigned int *mode)
 {
 	switch (*mode) {
 	case REGULATOR_MODE_FAST:
@@ -3359,6 +3360,39 @@ unsigned int regulator_get_mode(struct regulator *regulator)
 }
 EXPORT_SYMBOL_GPL(regulator_get_mode);
 
+static int _regulator_get_error_flags(struct regulator_dev *rdev,
+					unsigned int *flags)
+{
+	int ret;
+
+	mutex_lock(&rdev->mutex);
+
+	/* sanity check */
+	if (!rdev->desc->ops->get_error_flags) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ret = rdev->desc->ops->get_error_flags(rdev, flags);
+out:
+	mutex_unlock(&rdev->mutex);
+	return ret;
+}
+
+/**
+ * regulator_get_error_flags - get regulator error information
+ * @regulator: regulator source
+ * @flags: pointer to store error flags
+ *
+ * Get the current regulator error information.
+ */
+int regulator_get_error_flags(struct regulator *regulator,
+				unsigned int *flags)
+{
+	return _regulator_get_error_flags(regulator->rdev, flags);
+}
+EXPORT_SYMBOL_GPL(regulator_get_error_flags);
+
 /**
  * regulator_set_load - set regulator load
  * @regulator: regulator source
diff --git a/drivers/regulator/fixed.c b/drivers/regulator/fixed.c
index 988a7472c2ab..a43b0e8a438d 100644
--- a/drivers/regulator/fixed.c
+++ b/drivers/regulator/fixed.c
@@ -30,6 +30,9 @@
 #include <linux/of_gpio.h>
 #include <linux/regulator/of_regulator.h>
 #include <linux/regulator/machine.h>
+#include <linux/acpi.h>
+#include <linux/property.h>
+#include <linux/gpio/consumer.h>
 
 struct fixed_voltage_data {
 	struct regulator_desc desc;
@@ -94,6 +97,44 @@ of_get_fixed_voltage_config(struct device *dev,
 	return config;
 }
 
+/**
+ * acpi_get_fixed_voltage_config - extract fixed_voltage_config structure info
+ * @dev: device requesting for fixed_voltage_config
+ * @desc: regulator description
+ *
+ * Populates fixed_voltage_config structure by extracting data through ACPI
+ * interface, returns a pointer to the populated structure of NULL if memory
+ * alloc fails.
+ */
+static struct fixed_voltage_config *
+acpi_get_fixed_voltage_config(struct device *dev,
+			      const struct regulator_desc *desc)
+{
+	struct fixed_voltage_config *config;
+	const char *supply_name;
+	struct gpio_desc *gpiod;
+	int ret;
+
+	config = devm_kzalloc(dev, sizeof(*config), GFP_KERNEL);
+	if (!config)
+		return ERR_PTR(-ENOMEM);
+
+	ret = device_property_read_string(dev, "supply-name", &supply_name);
+	if (!ret)
+		config->supply_name = supply_name;
+
+	gpiod = gpiod_get(dev, "gpio", GPIOD_ASIS);
+	if (IS_ERR(gpiod))
+		return ERR_PTR(-ENODEV);
+
+	config->gpio = desc_to_gpio(gpiod);
+	config->enable_high = device_property_read_bool(dev,
+							"enable-active-high");
+	gpiod_put(gpiod);
+
+	return config;
+}
+
 static struct regulator_ops fixed_voltage_ops = {
 };
 
@@ -114,6 +155,11 @@ static int reg_fixed_voltage_probe(struct platform_device *pdev)
 						     &drvdata->desc);
 		if (IS_ERR(config))
 			return PTR_ERR(config);
+	} else if (ACPI_HANDLE(&pdev->dev)) {
+		config = acpi_get_fixed_voltage_config(&pdev->dev,
+						       &drvdata->desc);
+		if (IS_ERR(config))
+			return PTR_ERR(config);
 	} else {
 		config = dev_get_platdata(&pdev->dev);
 	}
diff --git a/drivers/regulator/gpio-regulator.c b/drivers/regulator/gpio-regulator.c
index 83e89e5d4752..0fce06acfaec 100644
--- a/drivers/regulator/gpio-regulator.c
+++ b/drivers/regulator/gpio-regulator.c
@@ -162,8 +162,8 @@ of_get_gpio_regulator_config(struct device *dev, struct device_node *np,
 	of_property_read_u32(np, "startup-delay-us", &config->startup_delay);
 
 	config->enable_gpio = of_get_named_gpio(np, "enable-gpio", 0);
-	if (config->enable_gpio == -EPROBE_DEFER)
-		return ERR_PTR(-EPROBE_DEFER);
+	if (config->enable_gpio < 0 && config->enable_gpio != -ENOENT)
+		return ERR_PTR(config->enable_gpio);
 
 	/* Fetch GPIOs. - optional property*/
 	ret = of_gpio_count(np);
@@ -190,8 +190,11 @@ of_get_gpio_regulator_config(struct device *dev, struct device_node *np,
 
 		for (i = 0; i < config->nr_gpios; i++) {
 			gpio = of_get_named_gpio(np, "gpios", i);
-			if (gpio < 0)
+			if (gpio < 0) {
+				if (gpio != -ENOENT)
+					return ERR_PTR(gpio);
 				break;
+			}
 			config->gpios[i].gpio = gpio;
 			if (proplen > 0) {
 				of_property_read_u32_index(np, "gpios-states",
diff --git a/drivers/regulator/helpers.c b/drivers/regulator/helpers.c
index bcf38fd5106a..379cdacc05d8 100644
--- a/drivers/regulator/helpers.c
+++ b/drivers/regulator/helpers.c
@@ -454,13 +454,17 @@ EXPORT_SYMBOL_GPL(regulator_set_bypass_regmap);
 int regulator_get_bypass_regmap(struct regulator_dev *rdev, bool *enable)
 {
 	unsigned int val;
+	unsigned int val_on = rdev->desc->bypass_val_on;
 	int ret;
 
 	ret = regmap_read(rdev->regmap, rdev->desc->bypass_reg, &val);
 	if (ret != 0)
 		return ret;
 
-	*enable = (val & rdev->desc->bypass_mask) == rdev->desc->bypass_val_on;
+	if (!val_on)
+		val_on = rdev->desc->bypass_mask;
+
+	*enable = (val & rdev->desc->bypass_mask) == val_on;
 
 	return 0;
 }
diff --git a/drivers/regulator/lp873x-regulator.c b/drivers/regulator/lp873x-regulator.c
index e504b9148226..70e3df653381 100644
--- a/drivers/regulator/lp873x-regulator.c
+++ b/drivers/regulator/lp873x-regulator.c
@@ -24,6 +24,7 @@
 	[_id] = {							\
 		.desc = {						\
 			.name			= _name,		\
+			.supply_name		= _of "-in",		\
 			.id			= _id,			\
 			.of_match		= of_match_ptr(_of),	\
 			.regulators_node	= of_match_ptr("regulators"),\
diff --git a/drivers/regulator/max77620-regulator.c b/drivers/regulator/max77620-regulator.c
index a1b49a6d538f..d088a7c79e60 100644
--- a/drivers/regulator/max77620-regulator.c
+++ b/drivers/regulator/max77620-regulator.c
@@ -73,7 +73,6 @@ struct max77620_regulator_info {
 };
 
 struct max77620_regulator_pdata {
-	struct regulator_init_data *reg_idata;
 	int active_fps_src;
 	int active_fps_pd_slot;
 	int active_fps_pu_slot;
@@ -81,6 +80,7 @@ struct max77620_regulator_pdata {
 	int suspend_fps_pd_slot;
 	int suspend_fps_pu_slot;
 	int current_mode;
+	int power_ok;
 	int ramp_rate_setting;
 };
 
@@ -351,11 +351,48 @@ static int max77620_set_slew_rate(struct max77620_regulator *pmic, int id,
 	return 0;
 }
 
+static int max77620_config_power_ok(struct max77620_regulator *pmic, int id)
+{
+	struct max77620_regulator_pdata *rpdata = &pmic->reg_pdata[id];
+	struct max77620_regulator_info *rinfo = pmic->rinfo[id];
+	struct max77620_chip *chip = dev_get_drvdata(pmic->dev->parent);
+	u8 val, mask;
+	int ret;
+
+	switch (chip->chip_id) {
+	case MAX20024:
+		if (rpdata->power_ok >= 0) {
+			if (rinfo->type == MAX77620_REGULATOR_TYPE_SD)
+				mask = MAX20024_SD_CFG1_MPOK_MASK;
+			else
+				mask = MAX20024_LDO_CFG2_MPOK_MASK;
+
+			val = rpdata->power_ok ? mask : 0;
+
+			ret = regmap_update_bits(pmic->rmap, rinfo->cfg_addr,
+						 mask, val);
+			if (ret < 0) {
+				dev_err(pmic->dev, "Reg 0x%02x update failed %d\n",
+					rinfo->cfg_addr, ret);
+				return ret;
+			}
+		}
+		break;
+
+	default:
+		break;
+	}
+
+	return 0;
+}
+
 static int max77620_init_pmic(struct max77620_regulator *pmic, int id)
 {
 	struct max77620_regulator_pdata *rpdata = &pmic->reg_pdata[id];
 	int ret;
 
+	max77620_config_power_ok(pmic, id);
+
 	/* Update power mode */
 	ret = max77620_regulator_get_power_mode(pmic, id);
 	if (ret < 0)
@@ -595,6 +632,12 @@ static int max77620_of_parse_cb(struct device_node *np,
 			np, "maxim,suspend-fps-power-down-slot", &pval);
 	rpdata->suspend_fps_pd_slot = (!ret) ? pval : -1;
 
+	ret = of_property_read_u32(np, "maxim,power-ok-control", &pval);
+	if (!ret)
+		rpdata->power_ok = pval;
+	else
+		rpdata->power_ok = -1;
+
 	ret = of_property_read_u32(np, "maxim,ramp-rate-setting", &pval);
 	rpdata->ramp_rate_setting = (!ret) ? pval : 0;
 
@@ -807,6 +850,8 @@ static int max77620_regulator_resume(struct device *dev)
 	for (id = 0; id < MAX77620_NUM_REGS; id++) {
 		reg_pdata = &pmic->reg_pdata[id];
 
+		max77620_config_power_ok(pmic, id);
+
 		max77620_regulator_set_fps_slots(pmic, id, false);
 		if (reg_pdata->active_fps_src < 0)
 			continue;
diff --git a/drivers/regulator/rk808-regulator.c b/drivers/regulator/rk808-regulator.c
index 3314bf299a51..fb44d5215e30 100644
--- a/drivers/regulator/rk808-regulator.c
+++ b/drivers/regulator/rk808-regulator.c
@@ -120,7 +120,7 @@ static const struct regulator_linear_range rk808_ldo3_voltage_ranges[] = {
 static int rk808_buck1_2_get_voltage_sel_regmap(struct regulator_dev *rdev)
 {
 	struct rk808_regulator_data *pdata = rdev_get_drvdata(rdev);
-	int id = rdev->desc->id - RK808_ID_DCDC1;
+	int id = rdev_get_id(rdev);
 	struct gpio_desc *gpio = pdata->dvs_gpio[id];
 	unsigned int val;
 	int ret;
@@ -193,7 +193,7 @@ static int rk808_buck1_2_set_voltage_sel(struct regulator_dev *rdev,
 					 unsigned sel)
 {
 	struct rk808_regulator_data *pdata = rdev_get_drvdata(rdev);
-	int id = rdev->desc->id - RK808_ID_DCDC1;
+	int id = rdev_get_id(rdev);
 	struct gpio_desc *gpio = pdata->dvs_gpio[id];
 	unsigned int reg = rdev->desc->vsel_reg;
 	unsigned old_sel;
@@ -232,7 +232,7 @@ static int rk808_buck1_2_set_voltage_time_sel(struct regulator_dev *rdev,
 				       unsigned int new_selector)
 {
 	struct rk808_regulator_data *pdata = rdev_get_drvdata(rdev);
-	int id = rdev->desc->id - RK808_ID_DCDC1;
+	int id = rdev_get_id(rdev);
 	struct gpio_desc *gpio = pdata->dvs_gpio[id];
 
 	/* if there is no dvs1/2 pin, we don't need wait extra time here. */
@@ -245,8 +245,7 @@ static int rk808_buck1_2_set_voltage_time_sel(struct regulator_dev *rdev,
 static int rk808_set_ramp_delay(struct regulator_dev *rdev, int ramp_delay)
 {
 	unsigned int ramp_value = RK808_RAMP_RATE_10MV_PER_US;
-	unsigned int reg = rk808_buck_config_regs[rdev->desc->id -
-						  RK808_ID_DCDC1];
+	unsigned int reg = rk808_buck_config_regs[rdev_get_id(rdev)];
 
 	switch (ramp_delay) {
 	case 1 ... 2000:
diff --git a/drivers/regulator/stw481x-vmmc.c b/drivers/regulator/stw481x-vmmc.c
index 7d2ae3e9e942..342f5da79975 100644
--- a/drivers/regulator/stw481x-vmmc.c
+++ b/drivers/regulator/stw481x-vmmc.c
@@ -47,7 +47,8 @@ static struct regulator_desc vmmc_regulator = {
 	.volt_table = stw481x_vmmc_voltages,
 	.enable_time = 200, /* FIXME: look this up */
 	.enable_reg = STW_CONF1,
-	.enable_mask = STW_CONF1_PDN_VMMC,
+	.enable_mask = STW_CONF1_PDN_VMMC | STW_CONF1_MMC_LS_STATUS,
+	.enable_val = STW_CONF1_PDN_VMMC,
 	.vsel_reg = STW_CONF1,
 	.vsel_mask = STW_CONF1_VMMC_MASK,
 };
diff --git a/drivers/regulator/tps6507x-regulator.c b/drivers/regulator/tps6507x-regulator.c
index dad0bac09ecf..c179a3a221af 100644
--- a/drivers/regulator/tps6507x-regulator.c
+++ b/drivers/regulator/tps6507x-regulator.c
@@ -375,7 +375,7 @@ static struct tps6507x_board *tps6507x_parse_dt_reg_data(
 	struct device_node *np = pdev->dev.parent->of_node;
 	struct device_node *regulators;
 	struct of_regulator_match *matches;
-	static struct regulator_init_data *reg_data;
+	struct regulator_init_data *reg_data;
 	int idx = 0, count, ret;
 
 	tps_board = devm_kzalloc(&pdev->dev, sizeof(*tps_board),
diff --git a/drivers/regulator/tps65086-regulator.c b/drivers/regulator/tps65086-regulator.c
index 33f389d583ef..ecb0371780af 100644
--- a/drivers/regulator/tps65086-regulator.c
+++ b/drivers/regulator/tps65086-regulator.c
@@ -71,7 +71,7 @@ struct tps65086_regulator {
 	unsigned int decay_mask;
 };
 
-static const struct regulator_linear_range tps65086_buck126_10mv_ranges[] = {
+static const struct regulator_linear_range tps65086_10mv_ranges[] = {
 	REGULATOR_LINEAR_RANGE(0, 0x0, 0x0, 0),
 	REGULATOR_LINEAR_RANGE(410000, 0x1, 0x7F, 10000),
 };
@@ -82,7 +82,7 @@ static const struct regulator_linear_range tps65086_buck126_25mv_ranges[] = {
 	REGULATOR_LINEAR_RANGE(1025000, 0x19, 0x7F, 25000),
 };
 
-static const struct regulator_linear_range tps65086_buck345_ranges[] = {
+static const struct regulator_linear_range tps65086_buck345_25mv_ranges[] = {
 	REGULATOR_LINEAR_RANGE(0, 0x0, 0x0, 0),
 	REGULATOR_LINEAR_RANGE(425000, 0x1, 0x7F, 25000),
 };
@@ -125,27 +125,27 @@ static int tps65086_of_parse_cb(struct device_node *dev,
 static struct tps65086_regulator regulators[] = {
 	TPS65086_REGULATOR("BUCK1", "buck1", BUCK1, 0x80, TPS65086_BUCK1CTRL,
 			   BUCK_VID_MASK, TPS65086_BUCK123CTRL, BIT(0),
-			   tps65086_buck126_10mv_ranges, TPS65086_BUCK1CTRL,
+			   tps65086_10mv_ranges, TPS65086_BUCK1CTRL,
 			   BIT(0)),
 	TPS65086_REGULATOR("BUCK2", "buck2", BUCK2, 0x80, TPS65086_BUCK2CTRL,
 			   BUCK_VID_MASK, TPS65086_BUCK123CTRL, BIT(1),
-			   tps65086_buck126_10mv_ranges, TPS65086_BUCK2CTRL,
+			   tps65086_10mv_ranges, TPS65086_BUCK2CTRL,
 			   BIT(0)),
 	TPS65086_REGULATOR("BUCK3", "buck3", BUCK3, 0x80, TPS65086_BUCK3VID,
 			   BUCK_VID_MASK, TPS65086_BUCK123CTRL, BIT(2),
-			   tps65086_buck345_ranges, TPS65086_BUCK3DECAY,
+			   tps65086_10mv_ranges, TPS65086_BUCK3DECAY,
 			   BIT(0)),
 	TPS65086_REGULATOR("BUCK4", "buck4", BUCK4, 0x80, TPS65086_BUCK4VID,
 			   BUCK_VID_MASK, TPS65086_BUCK4CTRL, BIT(0),
-			   tps65086_buck345_ranges, TPS65086_BUCK4VID,
+			   tps65086_10mv_ranges, TPS65086_BUCK4VID,
 			   BIT(0)),
 	TPS65086_REGULATOR("BUCK5", "buck5", BUCK5, 0x80, TPS65086_BUCK5VID,
 			   BUCK_VID_MASK, TPS65086_BUCK5CTRL, BIT(0),
-			   tps65086_buck345_ranges, TPS65086_BUCK5CTRL,
+			   tps65086_10mv_ranges, TPS65086_BUCK5CTRL,
 			   BIT(0)),
 	TPS65086_REGULATOR("BUCK6", "buck6", BUCK6, 0x80, TPS65086_BUCK6VID,
 			   BUCK_VID_MASK, TPS65086_BUCK6CTRL, BIT(0),
-			   tps65086_buck126_10mv_ranges, TPS65086_BUCK6CTRL,
+			   tps65086_10mv_ranges, TPS65086_BUCK6CTRL,
 			   BIT(0)),
 	TPS65086_REGULATOR("LDOA1", "ldoa1", LDOA1, 0xF, TPS65086_LDOA1CTRL,
 			   VDOA1_VID_MASK, TPS65086_LDOA1CTRL, BIT(0),
@@ -162,18 +162,6 @@ static struct tps65086_regulator regulators[] = {
 	TPS65086_SWITCH("VTT", "vtt", VTT, TPS65086_SWVTT_EN, BIT(4)),
 };
 
-static inline bool has_25mv_mode(int id)
-{
-	switch (id) {
-	case BUCK1:
-	case BUCK2:
-	case BUCK6:
-		return true;
-	default:
-		return false;
-	}
-}
-
 static int tps65086_of_parse_cb(struct device_node *dev,
 				const struct regulator_desc *desc,
 				struct regulator_config *config)
@@ -181,12 +169,27 @@ static int tps65086_of_parse_cb(struct device_node *dev,
 	int ret;
 
 	/* Check for 25mV step mode */
-	if (has_25mv_mode(desc->id) &&
-			of_property_read_bool(config->of_node, "ti,regulator-step-size-25mv")) {
-		regulators[desc->id].desc.linear_ranges =
+	if (of_property_read_bool(config->of_node, "ti,regulator-step-size-25mv")) {
+		switch (desc->id) {
+		case BUCK1:
+		case BUCK2:
+		case BUCK6:
+			regulators[desc->id].desc.linear_ranges =
 				tps65086_buck126_25mv_ranges;
-		regulators[desc->id].desc.n_linear_ranges =
+			regulators[desc->id].desc.n_linear_ranges =
 				ARRAY_SIZE(tps65086_buck126_25mv_ranges);
+			break;
+		case BUCK3:
+		case BUCK4:
+		case BUCK5:
+			regulators[desc->id].desc.linear_ranges =
+				tps65086_buck345_25mv_ranges;
+			regulators[desc->id].desc.n_linear_ranges =
+				ARRAY_SIZE(tps65086_buck345_25mv_ranges);
+			break;
+		default:
+			dev_warn(config->dev, "25mV step mode only valid for BUCK regulators\n");
+		}
 	}
 
 	/* Check for decay mode */
diff --git a/drivers/regulator/twl-regulator.c b/drivers/regulator/twl-regulator.c
index 210681d6b743..6c9ec84121bd 100644
--- a/drivers/regulator/twl-regulator.c
+++ b/drivers/regulator/twl-regulator.c
@@ -24,7 +24,7 @@
 #include <linux/delay.h>
 
 /*
- * The TWL4030/TW5030/TPS659x0/TWL6030 family chips include power management, a
+ * The TWL4030/TW5030/TPS659x0 family chips include power management, a
  * USB OTG transceiver, an RTC, ADC, PWM, and lots more.  Some versions
  * include an audio codec, battery charger, and more voltage regulators.
  * These chips are often used in OMAP-based systems.
@@ -48,25 +48,12 @@ struct twlreg_info {
 	/* State REMAP default configuration */
 	u8			remap;
 
-	/* chip constraints on regulator behavior */
-	u16			min_mV;
-	u16			max_mV;
-
-	u8			flags;
-
 	/* used by regulator core */
 	struct regulator_desc	desc;
 
 	/* chip specific features */
 	unsigned long		features;
 
-	/*
-	 * optional override functions for voltage set/get
-	 * these are currently only used for SMPS regulators
-	 */
-	int			(*get_voltage)(void *data);
-	int			(*set_voltage)(void *data, int target_uV);
-
 	/* data passed from board for external get/set voltage */
 	void			*data;
 };
@@ -88,33 +75,6 @@ struct twlreg_info {
 #define VREG_STATE		2
 #define VREG_VOLTAGE		3
 #define VREG_VOLTAGE_SMPS	4
-/* TWL6030 Misc register offsets */
-#define VREG_BC_ALL		1
-#define VREG_BC_REF		2
-#define VREG_BC_PROC		3
-#define VREG_BC_CLK_RST		4
-
-/* TWL6030 LDO register values for CFG_STATE */
-#define TWL6030_CFG_STATE_OFF	0x00
-#define TWL6030_CFG_STATE_ON	0x01
-#define TWL6030_CFG_STATE_OFF2	0x02
-#define TWL6030_CFG_STATE_SLEEP	0x03
-#define TWL6030_CFG_STATE_GRP_SHIFT	5
-#define TWL6030_CFG_STATE_APP_SHIFT	2
-#define TWL6030_CFG_STATE_APP_MASK	(0x03 << TWL6030_CFG_STATE_APP_SHIFT)
-#define TWL6030_CFG_STATE_APP(v)	(((v) & TWL6030_CFG_STATE_APP_MASK) >>\
-						TWL6030_CFG_STATE_APP_SHIFT)
-
-/* Flags for SMPS Voltage reading */
-#define SMPS_OFFSET_EN		BIT(0)
-#define SMPS_EXTENDED_EN	BIT(1)
-
-/* twl6032 SMPS EPROM values */
-#define TWL6030_SMPS_OFFSET		0xB0
-#define TWL6030_SMPS_MULT		0xB3
-#define SMPS_MULTOFFSET_SMPS4	BIT(0)
-#define SMPS_MULTOFFSET_VIO	BIT(1)
-#define SMPS_MULTOFFSET_SMPS3	BIT(6)
 
 static inline int
 twlreg_read(struct twlreg_info *info, unsigned slave_subgp, unsigned offset)
@@ -168,26 +128,6 @@ static int twl4030reg_is_enabled(struct regulator_dev *rdev)
 	return state & P1_GRP_4030;
 }
 
-static int twl6030reg_is_enabled(struct regulator_dev *rdev)
-{
-	struct twlreg_info	*info = rdev_get_drvdata(rdev);
-	int			grp = 0, val;
-
-	if (!(twl_class_is_6030() && (info->features & TWL6032_SUBCLASS))) {
-		grp = twlreg_grp(rdev);
-		if (grp < 0)
-			return grp;
-		grp &= P1_GRP_6030;
-	} else {
-		grp = 1;
-	}
-
-	val = twlreg_read(info, TWL_MODULE_PM_RECEIVER, VREG_STATE);
-	val = TWL6030_CFG_STATE_APP(val);
-
-	return grp && (val == TWL6030_CFG_STATE_ON);
-}
-
 #define PB_I2C_BUSY	BIT(0)
 #define PB_I2C_BWEN	BIT(1)
 
@@ -273,23 +213,6 @@ static int twl4030reg_enable(struct regulator_dev *rdev)
 	return ret;
 }
 
-static int twl6030reg_enable(struct regulator_dev *rdev)
-{
-	struct twlreg_info	*info = rdev_get_drvdata(rdev);
-	int			grp = 0;
-	int			ret;
-
-	if (!(twl_class_is_6030() && (info->features & TWL6032_SUBCLASS)))
-		grp = twlreg_grp(rdev);
-	if (grp < 0)
-		return grp;
-
-	ret = twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_STATE,
-			grp << TWL6030_CFG_STATE_GRP_SHIFT |
-			TWL6030_CFG_STATE_ON);
-	return ret;
-}
-
 static int twl4030reg_disable(struct regulator_dev *rdev)
 {
 	struct twlreg_info	*info = rdev_get_drvdata(rdev);
@@ -307,23 +230,6 @@ static int twl4030reg_disable(struct regulator_dev *rdev)
 	return ret;
 }
 
-static int twl6030reg_disable(struct regulator_dev *rdev)
-{
-	struct twlreg_info	*info = rdev_get_drvdata(rdev);
-	int			grp = 0;
-	int			ret;
-
-	if (!(twl_class_is_6030() && (info->features & TWL6032_SUBCLASS)))
-		grp = P1_GRP_6030 | P2_GRP_6030 | P3_GRP_6030;
-
-	/* For 6030, set the off state for all grps enabled */
-	ret = twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_STATE,
-			(grp) << TWL6030_CFG_STATE_GRP_SHIFT |
-			TWL6030_CFG_STATE_OFF);
-
-	return ret;
-}
-
 static int twl4030reg_get_status(struct regulator_dev *rdev)
 {
 	int	state = twlreg_grp(rdev);
@@ -340,33 +246,6 @@ static int twl4030reg_get_status(struct regulator_dev *rdev)
 		: REGULATOR_STATUS_STANDBY;
 }
 
-static int twl6030reg_get_status(struct regulator_dev *rdev)
-{
-	struct twlreg_info	*info = rdev_get_drvdata(rdev);
-	int			val;
-
-	val = twlreg_grp(rdev);
-	if (val < 0)
-		return val;
-
-	val = twlreg_read(info, TWL_MODULE_PM_RECEIVER, VREG_STATE);
-
-	switch (TWL6030_CFG_STATE_APP(val)) {
-	case TWL6030_CFG_STATE_ON:
-		return REGULATOR_STATUS_NORMAL;
-
-	case TWL6030_CFG_STATE_SLEEP:
-		return REGULATOR_STATUS_STANDBY;
-
-	case TWL6030_CFG_STATE_OFF:
-	case TWL6030_CFG_STATE_OFF2:
-	default:
-		break;
-	}
-
-	return REGULATOR_STATUS_OFF;
-}
-
 static int twl4030reg_set_mode(struct regulator_dev *rdev, unsigned mode)
 {
 	struct twlreg_info	*info = rdev_get_drvdata(rdev);
@@ -399,36 +278,6 @@ static inline unsigned int twl4030reg_map_mode(unsigned int mode)
 	}
 }
 
-static int twl6030reg_set_mode(struct regulator_dev *rdev, unsigned mode)
-{
-	struct twlreg_info	*info = rdev_get_drvdata(rdev);
-	int grp = 0;
-	int val;
-
-	if (!(twl_class_is_6030() && (info->features & TWL6032_SUBCLASS)))
-		grp = twlreg_grp(rdev);
-
-	if (grp < 0)
-		return grp;
-
-	/* Compose the state register settings */
-	val = grp << TWL6030_CFG_STATE_GRP_SHIFT;
-	/* We can only set the mode through state machine commands... */
-	switch (mode) {
-	case REGULATOR_MODE_NORMAL:
-		val |= TWL6030_CFG_STATE_ON;
-		break;
-	case REGULATOR_MODE_STANDBY:
-		val |= TWL6030_CFG_STATE_SLEEP;
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	return twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_STATE, val);
-}
-
 /*----------------------------------------------------------------------*/
 
 /*
@@ -565,12 +414,7 @@ twl4030smps_set_voltage(struct regulator_dev *rdev, int min_uV, int max_uV,
 	struct twlreg_info *info = rdev_get_drvdata(rdev);
 	int vsel = DIV_ROUND_UP(min_uV - 600000, 12500);
 
-	if (info->set_voltage) {
-		return info->set_voltage(info->data, min_uV);
-	} else {
-		twlreg_write(info, TWL_MODULE_PM_RECEIVER,
-			VREG_VOLTAGE_SMPS_4030, vsel);
-	}
+	twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_VOLTAGE_SMPS_4030, vsel);
 
 	return 0;
 }
@@ -580,9 +424,6 @@ static int twl4030smps_get_voltage(struct regulator_dev *rdev)
 	struct twlreg_info *info = rdev_get_drvdata(rdev);
 	int vsel;
 
-	if (info->get_voltage)
-		return info->get_voltage(info->data);
-
 	vsel = twlreg_read(info, TWL_MODULE_PM_RECEIVER,
 		VREG_VOLTAGE_SMPS_4030);
 
@@ -594,85 +435,6 @@ static struct regulator_ops twl4030smps_ops = {
 	.get_voltage	= twl4030smps_get_voltage,
 };
 
-static int twl6030coresmps_set_voltage(struct regulator_dev *rdev, int min_uV,
-	int max_uV, unsigned *selector)
-{
-	struct twlreg_info *info = rdev_get_drvdata(rdev);
-
-	if (info->set_voltage)
-		return info->set_voltage(info->data, min_uV);
-
-	return -ENODEV;
-}
-
-static int twl6030coresmps_get_voltage(struct regulator_dev *rdev)
-{
-	struct twlreg_info *info = rdev_get_drvdata(rdev);
-
-	if (info->get_voltage)
-		return info->get_voltage(info->data);
-
-	return -ENODEV;
-}
-
-static struct regulator_ops twl6030coresmps_ops = {
-	.set_voltage	= twl6030coresmps_set_voltage,
-	.get_voltage	= twl6030coresmps_get_voltage,
-};
-
-static int twl6030ldo_list_voltage(struct regulator_dev *rdev, unsigned sel)
-{
-	struct twlreg_info *info = rdev_get_drvdata(rdev);
-
-	switch (sel) {
-	case 0:
-		return 0;
-	case 1 ... 24:
-		/* Linear mapping from 00000001 to 00011000:
-		 * Absolute voltage value = 1.0 V + 0.1 V × (sel – 00000001)
-		 */
-		return (info->min_mV + 100 * (sel - 1)) * 1000;
-	case 25 ... 30:
-		return -EINVAL;
-	case 31:
-		return 2750000;
-	default:
-		return -EINVAL;
-	}
-}
-
-static int
-twl6030ldo_set_voltage_sel(struct regulator_dev *rdev, unsigned selector)
-{
-	struct twlreg_info	*info = rdev_get_drvdata(rdev);
-
-	return twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_VOLTAGE,
-			    selector);
-}
-
-static int twl6030ldo_get_voltage_sel(struct regulator_dev *rdev)
-{
-	struct twlreg_info	*info = rdev_get_drvdata(rdev);
-	int vsel = twlreg_read(info, TWL_MODULE_PM_RECEIVER, VREG_VOLTAGE);
-
-	return vsel;
-}
-
-static struct regulator_ops twl6030ldo_ops = {
-	.list_voltage	= twl6030ldo_list_voltage,
-
-	.set_voltage_sel = twl6030ldo_set_voltage_sel,
-	.get_voltage_sel = twl6030ldo_get_voltage_sel,
-
-	.enable		= twl6030reg_enable,
-	.disable	= twl6030reg_disable,
-	.is_enabled	= twl6030reg_is_enabled,
-
-	.set_mode	= twl6030reg_set_mode,
-
-	.get_status	= twl6030reg_get_status,
-};
-
 /*----------------------------------------------------------------------*/
 
 static struct regulator_ops twl4030fixed_ops = {
@@ -687,226 +449,8 @@ static struct regulator_ops twl4030fixed_ops = {
 	.get_status	= twl4030reg_get_status,
 };
 
-static struct regulator_ops twl6030fixed_ops = {
-	.list_voltage	= regulator_list_voltage_linear,
-
-	.enable		= twl6030reg_enable,
-	.disable	= twl6030reg_disable,
-	.is_enabled	= twl6030reg_is_enabled,
-
-	.set_mode	= twl6030reg_set_mode,
-
-	.get_status	= twl6030reg_get_status,
-};
-
-/*
- * SMPS status and control
- */
-
-static int twl6030smps_list_voltage(struct regulator_dev *rdev, unsigned index)
-{
-	struct twlreg_info	*info = rdev_get_drvdata(rdev);
-
-	int voltage = 0;
-
-	switch (info->flags) {
-	case SMPS_OFFSET_EN:
-		voltage = 100000;
-		/* fall through */
-	case 0:
-		switch (index) {
-		case 0:
-			voltage = 0;
-			break;
-		case 58:
-			voltage = 1350 * 1000;
-			break;
-		case 59:
-			voltage = 1500 * 1000;
-			break;
-		case 60:
-			voltage = 1800 * 1000;
-			break;
-		case 61:
-			voltage = 1900 * 1000;
-			break;
-		case 62:
-			voltage = 2100 * 1000;
-			break;
-		default:
-			voltage += (600000 + (12500 * (index - 1)));
-		}
-		break;
-	case SMPS_EXTENDED_EN:
-		switch (index) {
-		case 0:
-			voltage = 0;
-			break;
-		case 58:
-			voltage = 2084 * 1000;
-			break;
-		case 59:
-			voltage = 2315 * 1000;
-			break;
-		case 60:
-			voltage = 2778 * 1000;
-			break;
-		case 61:
-			voltage = 2932 * 1000;
-			break;
-		case 62:
-			voltage = 3241 * 1000;
-			break;
-		default:
-			voltage = (1852000 + (38600 * (index - 1)));
-		}
-		break;
-	case SMPS_OFFSET_EN | SMPS_EXTENDED_EN:
-		switch (index) {
-		case 0:
-			voltage = 0;
-			break;
-		case 58:
-			voltage = 4167 * 1000;
-			break;
-		case 59:
-			voltage = 2315 * 1000;
-			break;
-		case 60:
-			voltage = 2778 * 1000;
-			break;
-		case 61:
-			voltage = 2932 * 1000;
-			break;
-		case 62:
-			voltage = 3241 * 1000;
-			break;
-		default:
-			voltage = (2161000 + (38600 * (index - 1)));
-		}
-		break;
-	}
-
-	return voltage;
-}
-
-static int twl6030smps_map_voltage(struct regulator_dev *rdev, int min_uV,
-				   int max_uV)
-{
-	struct twlreg_info *info = rdev_get_drvdata(rdev);
-	int vsel = 0;
-
-	switch (info->flags) {
-	case 0:
-		if (min_uV == 0)
-			vsel = 0;
-		else if ((min_uV >= 600000) && (min_uV <= 1300000)) {
-			vsel = DIV_ROUND_UP(min_uV - 600000, 12500);
-			vsel++;
-		}
-		/* Values 1..57 for vsel are linear and can be calculated
-		 * values 58..62 are non linear.
-		 */
-		else if ((min_uV > 1900000) && (min_uV <= 2100000))
-			vsel = 62;
-		else if ((min_uV > 1800000) && (min_uV <= 1900000))
-			vsel = 61;
-		else if ((min_uV > 1500000) && (min_uV <= 1800000))
-			vsel = 60;
-		else if ((min_uV > 1350000) && (min_uV <= 1500000))
-			vsel = 59;
-		else if ((min_uV > 1300000) && (min_uV <= 1350000))
-			vsel = 58;
-		else
-			return -EINVAL;
-		break;
-	case SMPS_OFFSET_EN:
-		if (min_uV == 0)
-			vsel = 0;
-		else if ((min_uV >= 700000) && (min_uV <= 1420000)) {
-			vsel = DIV_ROUND_UP(min_uV - 700000, 12500);
-			vsel++;
-		}
-		/* Values 1..57 for vsel are linear and can be calculated
-		 * values 58..62 are non linear.
-		 */
-		else if ((min_uV > 1900000) && (min_uV <= 2100000))
-			vsel = 62;
-		else if ((min_uV > 1800000) && (min_uV <= 1900000))
-			vsel = 61;
-		else if ((min_uV > 1350000) && (min_uV <= 1800000))
-			vsel = 60;
-		else if ((min_uV > 1350000) && (min_uV <= 1500000))
-			vsel = 59;
-		else if ((min_uV > 1300000) && (min_uV <= 1350000))
-			vsel = 58;
-		else
-			return -EINVAL;
-		break;
-	case SMPS_EXTENDED_EN:
-		if (min_uV == 0) {
-			vsel = 0;
-		} else if ((min_uV >= 1852000) && (max_uV <= 4013600)) {
-			vsel = DIV_ROUND_UP(min_uV - 1852000, 38600);
-			vsel++;
-		}
-		break;
-	case SMPS_OFFSET_EN|SMPS_EXTENDED_EN:
-		if (min_uV == 0) {
-			vsel = 0;
-		} else if ((min_uV >= 2161000) && (min_uV <= 4321000)) {
-			vsel = DIV_ROUND_UP(min_uV - 2161000, 38600);
-			vsel++;
-		}
-		break;
-	}
-
-	return vsel;
-}
-
-static int twl6030smps_set_voltage_sel(struct regulator_dev *rdev,
-				       unsigned int selector)
-{
-	struct twlreg_info *info = rdev_get_drvdata(rdev);
-
-	return twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_VOLTAGE_SMPS,
-			    selector);
-}
-
-static int twl6030smps_get_voltage_sel(struct regulator_dev *rdev)
-{
-	struct twlreg_info	*info = rdev_get_drvdata(rdev);
-
-	return twlreg_read(info, TWL_MODULE_PM_RECEIVER, VREG_VOLTAGE_SMPS);
-}
-
-static struct regulator_ops twlsmps_ops = {
-	.list_voltage		= twl6030smps_list_voltage,
-	.map_voltage		= twl6030smps_map_voltage,
-
-	.set_voltage_sel	= twl6030smps_set_voltage_sel,
-	.get_voltage_sel	= twl6030smps_get_voltage_sel,
-
-	.enable			= twl6030reg_enable,
-	.disable		= twl6030reg_disable,
-	.is_enabled		= twl6030reg_is_enabled,
-
-	.set_mode		= twl6030reg_set_mode,
-
-	.get_status		= twl6030reg_get_status,
-};
-
 /*----------------------------------------------------------------------*/
 
-#define TWL4030_FIXED_LDO(label, offset, mVolts, num, turnon_delay, \
-			remap_conf) \
-		TWL_FIXED_LDO(label, offset, mVolts, num, turnon_delay, \
-			remap_conf, TWL4030, twl4030fixed_ops, \
-			twl4030reg_map_mode)
-#define TWL6030_FIXED_LDO(label, offset, mVolts, turnon_delay) \
-		TWL_FIXED_LDO(label, offset, mVolts, 0x0, turnon_delay, \
-			0x0, TWL6030, twl6030fixed_ops, NULL)
-
 #define TWL4030_ADJUSTABLE_LDO(label, offset, num, turnon_delay, remap_conf) \
 static const struct twlreg_info TWL4030_INFO_##label = { \
 	.base = offset, \
@@ -942,79 +486,22 @@ static const struct twlreg_info TWL4030_INFO_##label = { \
 		}, \
 	}
 
-#define TWL6030_ADJUSTABLE_SMPS(label) \
-static const struct twlreg_info TWL6030_INFO_##label = { \
-	.desc = { \
-		.name = #label, \
-		.id = TWL6030_REG_##label, \
-		.ops = &twl6030coresmps_ops, \
-		.type = REGULATOR_VOLTAGE, \
-		.owner = THIS_MODULE, \
-		}, \
-	}
-
-#define TWL6030_ADJUSTABLE_LDO(label, offset, min_mVolts, max_mVolts) \
-static const struct twlreg_info TWL6030_INFO_##label = { \
-	.base = offset, \
-	.min_mV = min_mVolts, \
-	.max_mV = max_mVolts, \
-	.desc = { \
-		.name = #label, \
-		.id = TWL6030_REG_##label, \
-		.n_voltages = 32, \
-		.ops = &twl6030ldo_ops, \
-		.type = REGULATOR_VOLTAGE, \
-		.owner = THIS_MODULE, \
-		}, \
-	}
-
-#define TWL6032_ADJUSTABLE_LDO(label, offset, min_mVolts, max_mVolts) \
-static const struct twlreg_info TWL6032_INFO_##label = { \
-	.base = offset, \
-	.min_mV = min_mVolts, \
-	.max_mV = max_mVolts, \
-	.desc = { \
-		.name = #label, \
-		.id = TWL6032_REG_##label, \
-		.n_voltages = 32, \
-		.ops = &twl6030ldo_ops, \
-		.type = REGULATOR_VOLTAGE, \
-		.owner = THIS_MODULE, \
-		}, \
-	}
-
-#define TWL_FIXED_LDO(label, offset, mVolts, num, turnon_delay, remap_conf, \
-		family, operations, map_mode) \
+#define TWL4030_FIXED_LDO(label, offset, mVolts, num, turnon_delay, \
+			remap_conf) \
 static const struct twlreg_info TWLFIXED_INFO_##label = { \
 	.base = offset, \
 	.id = num, \
-	.min_mV = mVolts, \
 	.remap = remap_conf, \
 	.desc = { \
 		.name = #label, \
-		.id = family##_REG_##label, \
+		.id = TWL4030##_REG_##label, \
 		.n_voltages = 1, \
-		.ops = &operations, \
+		.ops = &twl4030fixed_ops, \
 		.type = REGULATOR_VOLTAGE, \
 		.owner = THIS_MODULE, \
 		.min_uV = mVolts * 1000, \
 		.enable_time = turnon_delay, \
-		.of_map_mode = map_mode, \
-		}, \
-	}
-
-#define TWL6032_ADJUSTABLE_SMPS(label, offset) \
-static const struct twlreg_info TWLSMPS_INFO_##label = { \
-	.base = offset, \
-	.min_mV = 600, \
-	.max_mV = 2100, \
-	.desc = { \
-		.name = #label, \
-		.id = TWL6032_REG_##label, \
-		.n_voltages = 63, \
-		.ops = &twlsmps_ops, \
-		.type = REGULATOR_VOLTAGE, \
-		.owner = THIS_MODULE, \
+		.of_map_mode = twl4030reg_map_mode, \
 		}, \
 	}
 
@@ -1038,60 +525,11 @@ TWL4030_ADJUSTABLE_LDO(VIO, 0x4b, 14, 1000, 0x08);
 TWL4030_ADJUSTABLE_SMPS(VDD1, 0x55, 15, 1000, 0x08);
 TWL4030_ADJUSTABLE_SMPS(VDD2, 0x63, 16, 1000, 0x08);
 /* VUSBCP is managed *only* by the USB subchip */
-/* 6030 REG with base as PMC Slave Misc : 0x0030 */
-/* Turnon-delay and remap configuration values for 6030 are not
-   verified since the specification is not public */
-TWL6030_ADJUSTABLE_SMPS(VDD1);
-TWL6030_ADJUSTABLE_SMPS(VDD2);
-TWL6030_ADJUSTABLE_SMPS(VDD3);
-TWL6030_ADJUSTABLE_LDO(VAUX1_6030, 0x54, 1000, 3300);
-TWL6030_ADJUSTABLE_LDO(VAUX2_6030, 0x58, 1000, 3300);
-TWL6030_ADJUSTABLE_LDO(VAUX3_6030, 0x5c, 1000, 3300);
-TWL6030_ADJUSTABLE_LDO(VMMC, 0x68, 1000, 3300);
-TWL6030_ADJUSTABLE_LDO(VPP, 0x6c, 1000, 3300);
-TWL6030_ADJUSTABLE_LDO(VUSIM, 0x74, 1000, 3300);
-/* 6025 are renamed compared to 6030 versions */
-TWL6032_ADJUSTABLE_LDO(LDO2, 0x54, 1000, 3300);
-TWL6032_ADJUSTABLE_LDO(LDO4, 0x58, 1000, 3300);
-TWL6032_ADJUSTABLE_LDO(LDO3, 0x5c, 1000, 3300);
-TWL6032_ADJUSTABLE_LDO(LDO5, 0x68, 1000, 3300);
-TWL6032_ADJUSTABLE_LDO(LDO1, 0x6c, 1000, 3300);
-TWL6032_ADJUSTABLE_LDO(LDO7, 0x74, 1000, 3300);
-TWL6032_ADJUSTABLE_LDO(LDO6, 0x60, 1000, 3300);
-TWL6032_ADJUSTABLE_LDO(LDOLN, 0x64, 1000, 3300);
-TWL6032_ADJUSTABLE_LDO(LDOUSB, 0x70, 1000, 3300);
 TWL4030_FIXED_LDO(VINTANA1, 0x3f, 1500, 11, 100, 0x08);
 TWL4030_FIXED_LDO(VINTDIG, 0x47, 1500, 13, 100, 0x08);
 TWL4030_FIXED_LDO(VUSB1V5, 0x71, 1500, 17, 100, 0x08);
 TWL4030_FIXED_LDO(VUSB1V8, 0x74, 1800, 18, 100, 0x08);
 TWL4030_FIXED_LDO(VUSB3V1, 0x77, 3100, 19, 150, 0x08);
-TWL6030_FIXED_LDO(VANA, 0x50, 2100, 0);
-TWL6030_FIXED_LDO(VCXIO, 0x60, 1800, 0);
-TWL6030_FIXED_LDO(VDAC, 0x64, 1800, 0);
-TWL6030_FIXED_LDO(VUSB, 0x70, 3300, 0);
-TWL6030_FIXED_LDO(V1V8, 0x16, 1800, 0);
-TWL6030_FIXED_LDO(V2V1, 0x1c, 2100, 0);
-TWL6032_ADJUSTABLE_SMPS(SMPS3, 0x34);
-TWL6032_ADJUSTABLE_SMPS(SMPS4, 0x10);
-TWL6032_ADJUSTABLE_SMPS(VIO, 0x16);
-
-static u8 twl_get_smps_offset(void)
-{
-	u8 value;
-
-	twl_i2c_read_u8(TWL_MODULE_PM_RECEIVER, &value,
-			TWL6030_SMPS_OFFSET);
-	return value;
-}
-
-static u8 twl_get_smps_mult(void)
-{
-	u8 value;
-
-	twl_i2c_read_u8(TWL_MODULE_PM_RECEIVER, &value,
-			TWL6030_SMPS_MULT);
-	return value;
-}
 
 #define TWL_OF_MATCH(comp, family, label) \
 	{ \
@@ -1121,81 +559,37 @@ static const struct of_device_id twl_of_match[] = {
 	TWL4030_OF_MATCH("ti,twl4030-vio", VIO),
 	TWL4030_OF_MATCH("ti,twl4030-vdd1", VDD1),
 	TWL4030_OF_MATCH("ti,twl4030-vdd2", VDD2),
-	TWL6030_OF_MATCH("ti,twl6030-vdd1", VDD1),
-	TWL6030_OF_MATCH("ti,twl6030-vdd2", VDD2),
-	TWL6030_OF_MATCH("ti,twl6030-vdd3", VDD3),
-	TWL6030_OF_MATCH("ti,twl6030-vaux1", VAUX1_6030),
-	TWL6030_OF_MATCH("ti,twl6030-vaux2", VAUX2_6030),
-	TWL6030_OF_MATCH("ti,twl6030-vaux3", VAUX3_6030),
-	TWL6030_OF_MATCH("ti,twl6030-vmmc", VMMC),
-	TWL6030_OF_MATCH("ti,twl6030-vpp", VPP),
-	TWL6030_OF_MATCH("ti,twl6030-vusim", VUSIM),
-	TWL6032_OF_MATCH("ti,twl6032-ldo2", LDO2),
-	TWL6032_OF_MATCH("ti,twl6032-ldo4", LDO4),
-	TWL6032_OF_MATCH("ti,twl6032-ldo3", LDO3),
-	TWL6032_OF_MATCH("ti,twl6032-ldo5", LDO5),
-	TWL6032_OF_MATCH("ti,twl6032-ldo1", LDO1),
-	TWL6032_OF_MATCH("ti,twl6032-ldo7", LDO7),
-	TWL6032_OF_MATCH("ti,twl6032-ldo6", LDO6),
-	TWL6032_OF_MATCH("ti,twl6032-ldoln", LDOLN),
-	TWL6032_OF_MATCH("ti,twl6032-ldousb", LDOUSB),
 	TWLFIXED_OF_MATCH("ti,twl4030-vintana1", VINTANA1),
 	TWLFIXED_OF_MATCH("ti,twl4030-vintdig", VINTDIG),
 	TWLFIXED_OF_MATCH("ti,twl4030-vusb1v5", VUSB1V5),
 	TWLFIXED_OF_MATCH("ti,twl4030-vusb1v8", VUSB1V8),
 	TWLFIXED_OF_MATCH("ti,twl4030-vusb3v1", VUSB3V1),
-	TWLFIXED_OF_MATCH("ti,twl6030-vana", VANA),
-	TWLFIXED_OF_MATCH("ti,twl6030-vcxio", VCXIO),
-	TWLFIXED_OF_MATCH("ti,twl6030-vdac", VDAC),
-	TWLFIXED_OF_MATCH("ti,twl6030-vusb", VUSB),
-	TWLFIXED_OF_MATCH("ti,twl6030-v1v8", V1V8),
-	TWLFIXED_OF_MATCH("ti,twl6030-v2v1", V2V1),
-	TWLSMPS_OF_MATCH("ti,twl6032-smps3", SMPS3),
-	TWLSMPS_OF_MATCH("ti,twl6032-smps4", SMPS4),
-	TWLSMPS_OF_MATCH("ti,twl6032-vio", VIO),
 	{},
 };
 MODULE_DEVICE_TABLE(of, twl_of_match);
 
 static int twlreg_probe(struct platform_device *pdev)
 {
-	int				i, id;
+	int id;
 	struct twlreg_info		*info;
 	const struct twlreg_info	*template;
 	struct regulator_init_data	*initdata;
 	struct regulation_constraints	*c;
 	struct regulator_dev		*rdev;
-	struct twl_regulator_driver_data	*drvdata;
 	const struct of_device_id	*match;
 	struct regulator_config		config = { };
 
 	match = of_match_device(twl_of_match, &pdev->dev);
-	if (match) {
-		template = match->data;
-		id = template->desc.id;
-		initdata = of_get_regulator_init_data(&pdev->dev,
-						      pdev->dev.of_node,
-						      &template->desc);
-		drvdata = NULL;
-	} else {
-		id = pdev->id;
-		initdata = dev_get_platdata(&pdev->dev);
-		for (i = 0, template = NULL; i < ARRAY_SIZE(twl_of_match); i++) {
-			template = twl_of_match[i].data;
-			if (template && template->desc.id == id)
-				break;
-		}
-		if (i == ARRAY_SIZE(twl_of_match))
-			return -ENODEV;
-
-		drvdata = initdata->driver_data;
-		if (!drvdata)
-			return -EINVAL;
-	}
+	if (!match)
+		return -ENODEV;
 
+	template = match->data;
 	if (!template)
 		return -ENODEV;
 
+	id = template->desc.id;
+	initdata = of_get_regulator_init_data(&pdev->dev, pdev->dev.of_node,
+						&template->desc);
 	if (!initdata)
 		return -EINVAL;
 
@@ -1203,14 +597,6 @@ static int twlreg_probe(struct platform_device *pdev)
 	if (!info)
 		return -ENOMEM;
 
-	if (drvdata) {
-		/* copy the driver data into regulator data */
-		info->features = drvdata->features;
-		info->data = drvdata->data;
-		info->set_voltage = drvdata->set_voltage;
-		info->get_voltage = drvdata->get_voltage;
-	}
-
 	/* Constrain board-specific capabilities according to what
 	 * this driver and the chip itself can actually do.
 	 */
@@ -1233,27 +619,6 @@ static int twlreg_probe(struct platform_device *pdev)
 		break;
 	}
 
-	switch (id) {
-	case TWL6032_REG_SMPS3:
-		if (twl_get_smps_mult() & SMPS_MULTOFFSET_SMPS3)
-			info->flags |= SMPS_EXTENDED_EN;
-		if (twl_get_smps_offset() & SMPS_MULTOFFSET_SMPS3)
-			info->flags |= SMPS_OFFSET_EN;
-		break;
-	case TWL6032_REG_SMPS4:
-		if (twl_get_smps_mult() & SMPS_MULTOFFSET_SMPS4)
-			info->flags |= SMPS_EXTENDED_EN;
-		if (twl_get_smps_offset() & SMPS_MULTOFFSET_SMPS4)
-			info->flags |= SMPS_OFFSET_EN;
-		break;
-	case TWL6032_REG_VIO:
-		if (twl_get_smps_mult() & SMPS_MULTOFFSET_VIO)
-			info->flags |= SMPS_EXTENDED_EN;
-		if (twl_get_smps_offset() & SMPS_MULTOFFSET_VIO)
-			info->flags |= SMPS_OFFSET_EN;
-		break;
-	}
-
 	config.dev = &pdev->dev;
 	config.init_data = initdata;
 	config.driver_data = info;
@@ -1267,9 +632,7 @@ static int twlreg_probe(struct platform_device *pdev)
 	}
 	platform_set_drvdata(pdev, rdev);
 
-	if (twl_class_is_4030())
-		twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_REMAP,
-						info->remap);
+	twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_REMAP, info->remap);
 
 	/* NOTE:  many regulators support short-circuit IRQs (presentable
 	 * as REGULATOR_OVER_CURRENT notifications?) configured via:
@@ -1282,7 +645,7 @@ static int twlreg_probe(struct platform_device *pdev)
 	return 0;
 }
 
-MODULE_ALIAS("platform:twl_reg");
+MODULE_ALIAS("platform:twl4030_reg");
 
 static struct platform_driver twlreg_driver = {
 	.probe		= twlreg_probe,
@@ -1290,7 +653,7 @@ static struct platform_driver twlreg_driver = {
 	 * "twl_regulator.12" (and friends) to "twl_regulator.1".
 	 */
 	.driver  = {
-		.name  = "twl_reg",
+		.name  = "twl4030_reg",
 		.of_match_table = of_match_ptr(twl_of_match),
 	},
 };
@@ -1307,5 +670,5 @@ static void __exit twlreg_exit(void)
 }
 module_exit(twlreg_exit)
 
-MODULE_DESCRIPTION("TWL regulator driver");
+MODULE_DESCRIPTION("TWL4030 regulator driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/twl6030-regulator.c b/drivers/regulator/twl6030-regulator.c
new file mode 100644
index 000000000000..4864b9d742c0
--- /dev/null
+++ b/drivers/regulator/twl6030-regulator.c
@@ -0,0 +1,793 @@
+/*
+ * Split TWL6030 logic from twl-regulator.c:
+ * Copyright (C) 2008 David Brownell
+ *
+ * Copyright (C) 2016 Nicolae Rosia <nicolae.rosia@gmail.com>
+ * 
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+#include <linux/regulator/of_regulator.h>
+#include <linux/i2c/twl.h>
+#include <linux/delay.h>
+
+struct twlreg_info {
+	/* start of regulator's PM_RECEIVER control register bank */
+	u8			base;
+
+	/* twl resource ID, for resource control state machine */
+	u8			id;
+
+	/* chip constraints on regulator behavior */
+	u16			min_mV;
+
+	u8			flags;
+
+	/* used by regulator core */
+	struct regulator_desc	desc;
+
+	/* chip specific features */
+	unsigned long		features;
+
+	/* data passed from board for external get/set voltage */
+	void			*data;
+};
+
+
+/* LDO control registers ... offset is from the base of its register bank.
+ * The first three registers of all power resource banks help hardware to
+ * manage the various resource groups.
+ */
+/* Common offset in TWL4030/6030 */
+#define VREG_GRP		0
+/* TWL6030 register offsets */
+#define VREG_TRANS		1
+#define VREG_STATE		2
+#define VREG_VOLTAGE		3
+#define VREG_VOLTAGE_SMPS	4
+/* TWL6030 Misc register offsets */
+#define VREG_BC_ALL		1
+#define VREG_BC_REF		2
+#define VREG_BC_PROC		3
+#define VREG_BC_CLK_RST		4
+
+/* TWL6030 LDO register values for CFG_STATE */
+#define TWL6030_CFG_STATE_OFF	0x00
+#define TWL6030_CFG_STATE_ON	0x01
+#define TWL6030_CFG_STATE_OFF2	0x02
+#define TWL6030_CFG_STATE_SLEEP	0x03
+#define TWL6030_CFG_STATE_GRP_SHIFT	5
+#define TWL6030_CFG_STATE_APP_SHIFT	2
+#define TWL6030_CFG_STATE_APP_MASK	(0x03 << TWL6030_CFG_STATE_APP_SHIFT)
+#define TWL6030_CFG_STATE_APP(v)	(((v) & TWL6030_CFG_STATE_APP_MASK) >>\
+						TWL6030_CFG_STATE_APP_SHIFT)
+
+/* Flags for SMPS Voltage reading */
+#define SMPS_OFFSET_EN		BIT(0)
+#define SMPS_EXTENDED_EN	BIT(1)
+
+/* twl6032 SMPS EPROM values */
+#define TWL6030_SMPS_OFFSET		0xB0
+#define TWL6030_SMPS_MULT		0xB3
+#define SMPS_MULTOFFSET_SMPS4	BIT(0)
+#define SMPS_MULTOFFSET_VIO	BIT(1)
+#define SMPS_MULTOFFSET_SMPS3	BIT(6)
+
+static inline int
+twlreg_read(struct twlreg_info *info, unsigned slave_subgp, unsigned offset)
+{
+	u8 value;
+	int status;
+
+	status = twl_i2c_read_u8(slave_subgp,
+			&value, info->base + offset);
+	return (status < 0) ? status : value;
+}
+
+static inline int
+twlreg_write(struct twlreg_info *info, unsigned slave_subgp, unsigned offset,
+						 u8 value)
+{
+	return twl_i2c_write_u8(slave_subgp,
+			value, info->base + offset);
+}
+
+/* generic power resource operations, which work on all regulators */
+static int twlreg_grp(struct regulator_dev *rdev)
+{
+	return twlreg_read(rdev_get_drvdata(rdev), TWL_MODULE_PM_RECEIVER,
+								 VREG_GRP);
+}
+
+/*
+ * Enable/disable regulators by joining/leaving the P1 (processor) group.
+ * We assume nobody else is updating the DEV_GRP registers.
+ */
+/* definition for 6030 family */
+#define P3_GRP_6030	BIT(2)		/* secondary processor, modem, etc */
+#define P2_GRP_6030	BIT(1)		/* "peripherals" */
+#define P1_GRP_6030	BIT(0)		/* CPU/Linux */
+
+static int twl6030reg_is_enabled(struct regulator_dev *rdev)
+{
+	struct twlreg_info	*info = rdev_get_drvdata(rdev);
+	int			grp = 0, val;
+
+	if (!(twl_class_is_6030() && (info->features & TWL6032_SUBCLASS))) {
+		grp = twlreg_grp(rdev);
+		if (grp < 0)
+			return grp;
+		grp &= P1_GRP_6030;
+	} else {
+		grp = 1;
+	}
+
+	val = twlreg_read(info, TWL_MODULE_PM_RECEIVER, VREG_STATE);
+	val = TWL6030_CFG_STATE_APP(val);
+
+	return grp && (val == TWL6030_CFG_STATE_ON);
+}
+
+#define PB_I2C_BUSY	BIT(0)
+#define PB_I2C_BWEN	BIT(1)
+
+
+static int twl6030reg_enable(struct regulator_dev *rdev)
+{
+	struct twlreg_info	*info = rdev_get_drvdata(rdev);
+	int			grp = 0;
+	int			ret;
+
+	if (!(twl_class_is_6030() && (info->features & TWL6032_SUBCLASS)))
+		grp = twlreg_grp(rdev);
+	if (grp < 0)
+		return grp;
+
+	ret = twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_STATE,
+			grp << TWL6030_CFG_STATE_GRP_SHIFT |
+			TWL6030_CFG_STATE_ON);
+	return ret;
+}
+
+static int twl6030reg_disable(struct regulator_dev *rdev)
+{
+	struct twlreg_info	*info = rdev_get_drvdata(rdev);
+	int			grp = 0;
+	int			ret;
+
+	if (!(twl_class_is_6030() && (info->features & TWL6032_SUBCLASS)))
+		grp = P1_GRP_6030 | P2_GRP_6030 | P3_GRP_6030;
+
+	/* For 6030, set the off state for all grps enabled */
+	ret = twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_STATE,
+			(grp) << TWL6030_CFG_STATE_GRP_SHIFT |
+			TWL6030_CFG_STATE_OFF);
+
+	return ret;
+}
+
+static int twl6030reg_get_status(struct regulator_dev *rdev)
+{
+	struct twlreg_info	*info = rdev_get_drvdata(rdev);
+	int			val;
+
+	val = twlreg_grp(rdev);
+	if (val < 0)
+		return val;
+
+	val = twlreg_read(info, TWL_MODULE_PM_RECEIVER, VREG_STATE);
+
+	switch (TWL6030_CFG_STATE_APP(val)) {
+	case TWL6030_CFG_STATE_ON:
+		return REGULATOR_STATUS_NORMAL;
+
+	case TWL6030_CFG_STATE_SLEEP:
+		return REGULATOR_STATUS_STANDBY;
+
+	case TWL6030_CFG_STATE_OFF:
+	case TWL6030_CFG_STATE_OFF2:
+	default:
+		break;
+	}
+
+	return REGULATOR_STATUS_OFF;
+}
+
+static int twl6030reg_set_mode(struct regulator_dev *rdev, unsigned mode)
+{
+	struct twlreg_info	*info = rdev_get_drvdata(rdev);
+	int grp = 0;
+	int val;
+
+	if (!(twl_class_is_6030() && (info->features & TWL6032_SUBCLASS)))
+		grp = twlreg_grp(rdev);
+
+	if (grp < 0)
+		return grp;
+
+	/* Compose the state register settings */
+	val = grp << TWL6030_CFG_STATE_GRP_SHIFT;
+	/* We can only set the mode through state machine commands... */
+	switch (mode) {
+	case REGULATOR_MODE_NORMAL:
+		val |= TWL6030_CFG_STATE_ON;
+		break;
+	case REGULATOR_MODE_STANDBY:
+		val |= TWL6030_CFG_STATE_SLEEP;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_STATE, val);
+}
+
+static int twl6030coresmps_set_voltage(struct regulator_dev *rdev, int min_uV,
+	int max_uV, unsigned *selector)
+{
+	return -ENODEV;
+}
+
+static int twl6030coresmps_get_voltage(struct regulator_dev *rdev)
+{
+	return -ENODEV;
+}
+
+static struct regulator_ops twl6030coresmps_ops = {
+	.set_voltage	= twl6030coresmps_set_voltage,
+	.get_voltage	= twl6030coresmps_get_voltage,
+};
+
+static int twl6030ldo_list_voltage(struct regulator_dev *rdev, unsigned sel)
+{
+	struct twlreg_info *info = rdev_get_drvdata(rdev);
+
+	switch (sel) {
+	case 0:
+		return 0;
+	case 1 ... 24:
+		/* Linear mapping from 00000001 to 00011000:
+		 * Absolute voltage value = 1.0 V + 0.1 V × (sel – 00000001)
+		 */
+		return (info->min_mV + 100 * (sel - 1)) * 1000;
+	case 25 ... 30:
+		return -EINVAL;
+	case 31:
+		return 2750000;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int
+twl6030ldo_set_voltage_sel(struct regulator_dev *rdev, unsigned selector)
+{
+	struct twlreg_info	*info = rdev_get_drvdata(rdev);
+
+	return twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_VOLTAGE,
+			    selector);
+}
+
+static int twl6030ldo_get_voltage_sel(struct regulator_dev *rdev)
+{
+	struct twlreg_info	*info = rdev_get_drvdata(rdev);
+	int vsel = twlreg_read(info, TWL_MODULE_PM_RECEIVER, VREG_VOLTAGE);
+
+	return vsel;
+}
+
+static struct regulator_ops twl6030ldo_ops = {
+	.list_voltage	= twl6030ldo_list_voltage,
+
+	.set_voltage_sel = twl6030ldo_set_voltage_sel,
+	.get_voltage_sel = twl6030ldo_get_voltage_sel,
+
+	.enable		= twl6030reg_enable,
+	.disable	= twl6030reg_disable,
+	.is_enabled	= twl6030reg_is_enabled,
+
+	.set_mode	= twl6030reg_set_mode,
+
+	.get_status	= twl6030reg_get_status,
+};
+
+static struct regulator_ops twl6030fixed_ops = {
+	.list_voltage	= regulator_list_voltage_linear,
+
+	.enable		= twl6030reg_enable,
+	.disable	= twl6030reg_disable,
+	.is_enabled	= twl6030reg_is_enabled,
+
+	.set_mode	= twl6030reg_set_mode,
+
+	.get_status	= twl6030reg_get_status,
+};
+
+/*
+ * SMPS status and control
+ */
+
+static int twl6030smps_list_voltage(struct regulator_dev *rdev, unsigned index)
+{
+	struct twlreg_info	*info = rdev_get_drvdata(rdev);
+
+	int voltage = 0;
+
+	switch (info->flags) {
+	case SMPS_OFFSET_EN:
+		voltage = 100000;
+		/* fall through */
+	case 0:
+		switch (index) {
+		case 0:
+			voltage = 0;
+			break;
+		case 58:
+			voltage = 1350 * 1000;
+			break;
+		case 59:
+			voltage = 1500 * 1000;
+			break;
+		case 60:
+			voltage = 1800 * 1000;
+			break;
+		case 61:
+			voltage = 1900 * 1000;
+			break;
+		case 62:
+			voltage = 2100 * 1000;
+			break;
+		default:
+			voltage += (600000 + (12500 * (index - 1)));
+		}
+		break;
+	case SMPS_EXTENDED_EN:
+		switch (index) {
+		case 0:
+			voltage = 0;
+			break;
+		case 58:
+			voltage = 2084 * 1000;
+			break;
+		case 59:
+			voltage = 2315 * 1000;
+			break;
+		case 60:
+			voltage = 2778 * 1000;
+			break;
+		case 61:
+			voltage = 2932 * 1000;
+			break;
+		case 62:
+			voltage = 3241 * 1000;
+			break;
+		default:
+			voltage = (1852000 + (38600 * (index - 1)));
+		}
+		break;
+	case SMPS_OFFSET_EN | SMPS_EXTENDED_EN:
+		switch (index) {
+		case 0:
+			voltage = 0;
+			break;
+		case 58:
+			voltage = 4167 * 1000;
+			break;
+		case 59:
+			voltage = 2315 * 1000;
+			break;
+		case 60:
+			voltage = 2778 * 1000;
+			break;
+		case 61:
+			voltage = 2932 * 1000;
+			break;
+		case 62:
+			voltage = 3241 * 1000;
+			break;
+		default:
+			voltage = (2161000 + (38600 * (index - 1)));
+		}
+		break;
+	}
+
+	return voltage;
+}
+
+static int twl6030smps_map_voltage(struct regulator_dev *rdev, int min_uV,
+				   int max_uV)
+{
+	struct twlreg_info *info = rdev_get_drvdata(rdev);
+	int vsel = 0;
+
+	switch (info->flags) {
+	case 0:
+		if (min_uV == 0)
+			vsel = 0;
+		else if ((min_uV >= 600000) && (min_uV <= 1300000)) {
+			vsel = DIV_ROUND_UP(min_uV - 600000, 12500);
+			vsel++;
+		}
+		/* Values 1..57 for vsel are linear and can be calculated
+		 * values 58..62 are non linear.
+		 */
+		else if ((min_uV > 1900000) && (min_uV <= 2100000))
+			vsel = 62;
+		else if ((min_uV > 1800000) && (min_uV <= 1900000))
+			vsel = 61;
+		else if ((min_uV > 1500000) && (min_uV <= 1800000))
+			vsel = 60;
+		else if ((min_uV > 1350000) && (min_uV <= 1500000))
+			vsel = 59;
+		else if ((min_uV > 1300000) && (min_uV <= 1350000))
+			vsel = 58;
+		else
+			return -EINVAL;
+		break;
+	case SMPS_OFFSET_EN:
+		if (min_uV == 0)
+			vsel = 0;
+		else if ((min_uV >= 700000) && (min_uV <= 1420000)) {
+			vsel = DIV_ROUND_UP(min_uV - 700000, 12500);
+			vsel++;
+		}
+		/* Values 1..57 for vsel are linear and can be calculated
+		 * values 58..62 are non linear.
+		 */
+		else if ((min_uV > 1900000) && (min_uV <= 2100000))
+			vsel = 62;
+		else if ((min_uV > 1800000) && (min_uV <= 1900000))
+			vsel = 61;
+		else if ((min_uV > 1350000) && (min_uV <= 1800000))
+			vsel = 60;
+		else if ((min_uV > 1350000) && (min_uV <= 1500000))
+			vsel = 59;
+		else if ((min_uV > 1300000) && (min_uV <= 1350000))
+			vsel = 58;
+		else
+			return -EINVAL;
+		break;
+	case SMPS_EXTENDED_EN:
+		if (min_uV == 0) {
+			vsel = 0;
+		} else if ((min_uV >= 1852000) && (max_uV <= 4013600)) {
+			vsel = DIV_ROUND_UP(min_uV - 1852000, 38600);
+			vsel++;
+		}
+		break;
+	case SMPS_OFFSET_EN|SMPS_EXTENDED_EN:
+		if (min_uV == 0) {
+			vsel = 0;
+		} else if ((min_uV >= 2161000) && (min_uV <= 4321000)) {
+			vsel = DIV_ROUND_UP(min_uV - 2161000, 38600);
+			vsel++;
+		}
+		break;
+	}
+
+	return vsel;
+}
+
+static int twl6030smps_set_voltage_sel(struct regulator_dev *rdev,
+				       unsigned int selector)
+{
+	struct twlreg_info *info = rdev_get_drvdata(rdev);
+
+	return twlreg_write(info, TWL_MODULE_PM_RECEIVER, VREG_VOLTAGE_SMPS,
+			    selector);
+}
+
+static int twl6030smps_get_voltage_sel(struct regulator_dev *rdev)
+{
+	struct twlreg_info	*info = rdev_get_drvdata(rdev);
+
+	return twlreg_read(info, TWL_MODULE_PM_RECEIVER, VREG_VOLTAGE_SMPS);
+}
+
+static struct regulator_ops twlsmps_ops = {
+	.list_voltage		= twl6030smps_list_voltage,
+	.map_voltage		= twl6030smps_map_voltage,
+
+	.set_voltage_sel	= twl6030smps_set_voltage_sel,
+	.get_voltage_sel	= twl6030smps_get_voltage_sel,
+
+	.enable			= twl6030reg_enable,
+	.disable		= twl6030reg_disable,
+	.is_enabled		= twl6030reg_is_enabled,
+
+	.set_mode		= twl6030reg_set_mode,
+
+	.get_status		= twl6030reg_get_status,
+};
+
+/*----------------------------------------------------------------------*/
+
+#define TWL6030_ADJUSTABLE_SMPS(label) \
+static const struct twlreg_info TWL6030_INFO_##label = { \
+	.desc = { \
+		.name = #label, \
+		.id = TWL6030_REG_##label, \
+		.ops = &twl6030coresmps_ops, \
+		.type = REGULATOR_VOLTAGE, \
+		.owner = THIS_MODULE, \
+		}, \
+	}
+
+#define TWL6030_ADJUSTABLE_LDO(label, offset, min_mVolts) \
+static const struct twlreg_info TWL6030_INFO_##label = { \
+	.base = offset, \
+	.min_mV = min_mVolts, \
+	.desc = { \
+		.name = #label, \
+		.id = TWL6030_REG_##label, \
+		.n_voltages = 32, \
+		.ops = &twl6030ldo_ops, \
+		.type = REGULATOR_VOLTAGE, \
+		.owner = THIS_MODULE, \
+		}, \
+	}
+
+#define TWL6032_ADJUSTABLE_LDO(label, offset, min_mVolts) \
+static const struct twlreg_info TWL6032_INFO_##label = { \
+	.base = offset, \
+	.min_mV = min_mVolts, \
+	.desc = { \
+		.name = #label, \
+		.id = TWL6032_REG_##label, \
+		.n_voltages = 32, \
+		.ops = &twl6030ldo_ops, \
+		.type = REGULATOR_VOLTAGE, \
+		.owner = THIS_MODULE, \
+		}, \
+	}
+
+#define TWL6030_FIXED_LDO(label, offset, mVolts, turnon_delay) \
+static const struct twlreg_info TWLFIXED_INFO_##label = { \
+	.base = offset, \
+	.id = 0, \
+	.min_mV = mVolts, \
+	.desc = { \
+		.name = #label, \
+		.id = TWL6030##_REG_##label, \
+		.n_voltages = 1, \
+		.ops = &twl6030fixed_ops, \
+		.type = REGULATOR_VOLTAGE, \
+		.owner = THIS_MODULE, \
+		.min_uV = mVolts * 1000, \
+		.enable_time = turnon_delay, \
+		.of_map_mode = NULL, \
+		}, \
+	}
+
+#define TWL6032_ADJUSTABLE_SMPS(label, offset) \
+static const struct twlreg_info TWLSMPS_INFO_##label = { \
+	.base = offset, \
+	.min_mV = 600, \
+	.desc = { \
+		.name = #label, \
+		.id = TWL6032_REG_##label, \
+		.n_voltages = 63, \
+		.ops = &twlsmps_ops, \
+		.type = REGULATOR_VOLTAGE, \
+		.owner = THIS_MODULE, \
+		}, \
+	}
+
+/* VUSBCP is managed *only* by the USB subchip */
+/* 6030 REG with base as PMC Slave Misc : 0x0030 */
+/* Turnon-delay and remap configuration values for 6030 are not
+   verified since the specification is not public */
+TWL6030_ADJUSTABLE_SMPS(VDD1);
+TWL6030_ADJUSTABLE_SMPS(VDD2);
+TWL6030_ADJUSTABLE_SMPS(VDD3);
+TWL6030_ADJUSTABLE_LDO(VAUX1_6030, 0x54, 1000);
+TWL6030_ADJUSTABLE_LDO(VAUX2_6030, 0x58, 1000);
+TWL6030_ADJUSTABLE_LDO(VAUX3_6030, 0x5c, 1000);
+TWL6030_ADJUSTABLE_LDO(VMMC, 0x68, 1000);
+TWL6030_ADJUSTABLE_LDO(VPP, 0x6c, 1000);
+TWL6030_ADJUSTABLE_LDO(VUSIM, 0x74, 1000);
+/* 6025 are renamed compared to 6030 versions */
+TWL6032_ADJUSTABLE_LDO(LDO2, 0x54, 1000);
+TWL6032_ADJUSTABLE_LDO(LDO4, 0x58, 1000);
+TWL6032_ADJUSTABLE_LDO(LDO3, 0x5c, 1000);
+TWL6032_ADJUSTABLE_LDO(LDO5, 0x68, 1000);
+TWL6032_ADJUSTABLE_LDO(LDO1, 0x6c, 1000);
+TWL6032_ADJUSTABLE_LDO(LDO7, 0x74, 1000);
+TWL6032_ADJUSTABLE_LDO(LDO6, 0x60, 1000);
+TWL6032_ADJUSTABLE_LDO(LDOLN, 0x64, 1000);
+TWL6032_ADJUSTABLE_LDO(LDOUSB, 0x70, 1000);
+TWL6030_FIXED_LDO(VANA, 0x50, 2100, 0);
+TWL6030_FIXED_LDO(VCXIO, 0x60, 1800, 0);
+TWL6030_FIXED_LDO(VDAC, 0x64, 1800, 0);
+TWL6030_FIXED_LDO(VUSB, 0x70, 3300, 0);
+TWL6030_FIXED_LDO(V1V8, 0x16, 1800, 0);
+TWL6030_FIXED_LDO(V2V1, 0x1c, 2100, 0);
+TWL6032_ADJUSTABLE_SMPS(SMPS3, 0x34);
+TWL6032_ADJUSTABLE_SMPS(SMPS4, 0x10);
+TWL6032_ADJUSTABLE_SMPS(VIO, 0x16);
+
+static u8 twl_get_smps_offset(void)
+{
+	u8 value;
+
+	twl_i2c_read_u8(TWL_MODULE_PM_RECEIVER, &value,
+			TWL6030_SMPS_OFFSET);
+	return value;
+}
+
+static u8 twl_get_smps_mult(void)
+{
+	u8 value;
+
+	twl_i2c_read_u8(TWL_MODULE_PM_RECEIVER, &value,
+			TWL6030_SMPS_MULT);
+	return value;
+}
+
+#define TWL_OF_MATCH(comp, family, label) \
+	{ \
+		.compatible = comp, \
+		.data = &family##_INFO_##label, \
+	}
+
+#define TWL6030_OF_MATCH(comp, label) TWL_OF_MATCH(comp, TWL6030, label)
+#define TWL6032_OF_MATCH(comp, label) TWL_OF_MATCH(comp, TWL6032, label)
+#define TWLFIXED_OF_MATCH(comp, label) TWL_OF_MATCH(comp, TWLFIXED, label)
+#define TWLSMPS_OF_MATCH(comp, label) TWL_OF_MATCH(comp, TWLSMPS, label)
+
+static const struct of_device_id twl_of_match[] = {
+	TWL6030_OF_MATCH("ti,twl6030-vdd1", VDD1),
+	TWL6030_OF_MATCH("ti,twl6030-vdd2", VDD2),
+	TWL6030_OF_MATCH("ti,twl6030-vdd3", VDD3),
+	TWL6030_OF_MATCH("ti,twl6030-vaux1", VAUX1_6030),
+	TWL6030_OF_MATCH("ti,twl6030-vaux2", VAUX2_6030),
+	TWL6030_OF_MATCH("ti,twl6030-vaux3", VAUX3_6030),
+	TWL6030_OF_MATCH("ti,twl6030-vmmc", VMMC),
+	TWL6030_OF_MATCH("ti,twl6030-vpp", VPP),
+	TWL6030_OF_MATCH("ti,twl6030-vusim", VUSIM),
+	TWL6032_OF_MATCH("ti,twl6032-ldo2", LDO2),
+	TWL6032_OF_MATCH("ti,twl6032-ldo4", LDO4),
+	TWL6032_OF_MATCH("ti,twl6032-ldo3", LDO3),
+	TWL6032_OF_MATCH("ti,twl6032-ldo5", LDO5),
+	TWL6032_OF_MATCH("ti,twl6032-ldo1", LDO1),
+	TWL6032_OF_MATCH("ti,twl6032-ldo7", LDO7),
+	TWL6032_OF_MATCH("ti,twl6032-ldo6", LDO6),
+	TWL6032_OF_MATCH("ti,twl6032-ldoln", LDOLN),
+	TWL6032_OF_MATCH("ti,twl6032-ldousb", LDOUSB),
+	TWLFIXED_OF_MATCH("ti,twl6030-vana", VANA),
+	TWLFIXED_OF_MATCH("ti,twl6030-vcxio", VCXIO),
+	TWLFIXED_OF_MATCH("ti,twl6030-vdac", VDAC),
+	TWLFIXED_OF_MATCH("ti,twl6030-vusb", VUSB),
+	TWLFIXED_OF_MATCH("ti,twl6030-v1v8", V1V8),
+	TWLFIXED_OF_MATCH("ti,twl6030-v2v1", V2V1),
+	TWLSMPS_OF_MATCH("ti,twl6032-smps3", SMPS3),
+	TWLSMPS_OF_MATCH("ti,twl6032-smps4", SMPS4),
+	TWLSMPS_OF_MATCH("ti,twl6032-vio", VIO),
+	{},
+};
+MODULE_DEVICE_TABLE(of, twl_of_match);
+
+static int twlreg_probe(struct platform_device *pdev)
+{
+	int id;
+	struct twlreg_info		*info;
+	const struct twlreg_info	*template;
+	struct regulator_init_data	*initdata;
+	struct regulation_constraints	*c;
+	struct regulator_dev		*rdev;
+	const struct of_device_id	*match;
+	struct regulator_config		config = { };
+
+	match = of_match_device(twl_of_match, &pdev->dev);
+	if (!match)
+		return -ENODEV;
+
+	template = match->data;
+	if (!template)
+		return -ENODEV;
+
+	id = template->desc.id;
+	initdata = of_get_regulator_init_data(&pdev->dev, pdev->dev.of_node,
+						&template->desc);
+	if (!initdata)
+		return -EINVAL;
+
+	info = devm_kmemdup(&pdev->dev, template, sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	/* Constrain board-specific capabilities according to what
+	 * this driver and the chip itself can actually do.
+	 */
+	c = &initdata->constraints;
+	c->valid_modes_mask &= REGULATOR_MODE_NORMAL | REGULATOR_MODE_STANDBY;
+	c->valid_ops_mask &= REGULATOR_CHANGE_VOLTAGE
+				| REGULATOR_CHANGE_MODE
+				| REGULATOR_CHANGE_STATUS;
+
+	switch (id) {
+	case TWL6032_REG_SMPS3:
+		if (twl_get_smps_mult() & SMPS_MULTOFFSET_SMPS3)
+			info->flags |= SMPS_EXTENDED_EN;
+		if (twl_get_smps_offset() & SMPS_MULTOFFSET_SMPS3)
+			info->flags |= SMPS_OFFSET_EN;
+		break;
+	case TWL6032_REG_SMPS4:
+		if (twl_get_smps_mult() & SMPS_MULTOFFSET_SMPS4)
+			info->flags |= SMPS_EXTENDED_EN;
+		if (twl_get_smps_offset() & SMPS_MULTOFFSET_SMPS4)
+			info->flags |= SMPS_OFFSET_EN;
+		break;
+	case TWL6032_REG_VIO:
+		if (twl_get_smps_mult() & SMPS_MULTOFFSET_VIO)
+			info->flags |= SMPS_EXTENDED_EN;
+		if (twl_get_smps_offset() & SMPS_MULTOFFSET_VIO)
+			info->flags |= SMPS_OFFSET_EN;
+		break;
+	}
+
+	config.dev = &pdev->dev;
+	config.init_data = initdata;
+	config.driver_data = info;
+	config.of_node = pdev->dev.of_node;
+
+	rdev = devm_regulator_register(&pdev->dev, &info->desc, &config);
+	if (IS_ERR(rdev)) {
+		dev_err(&pdev->dev, "can't register %s, %ld\n",
+				info->desc.name, PTR_ERR(rdev));
+		return PTR_ERR(rdev);
+	}
+	platform_set_drvdata(pdev, rdev);
+
+	/* NOTE:  many regulators support short-circuit IRQs (presentable
+	 * as REGULATOR_OVER_CURRENT notifications?) configured via:
+	 *  - SC_CONFIG
+	 *  - SC_DETECT1 (vintana2, vmmc1/2, vaux1/2/3/4)
+	 *  - SC_DETECT2 (vusb, vdac, vio, vdd1/2, vpll2)
+	 *  - IT_CONFIG
+	 */
+
+	return 0;
+}
+
+MODULE_ALIAS("platform:twl6030_reg");
+
+static struct platform_driver twlreg_driver = {
+	.probe		= twlreg_probe,
+	/* NOTE: short name, to work around driver model truncation of
+	 * "twl_regulator.12" (and friends) to "twl_regulator.1".
+	 */
+	.driver  = {
+		.name  = "twl6030_reg",
+		.of_match_table = of_match_ptr(twl_of_match),
+	},
+};
+
+static int __init twlreg_init(void)
+{
+	return platform_driver_register(&twlreg_driver);
+}
+subsys_initcall(twlreg_init);
+
+static void __exit twlreg_exit(void)
+{
+	platform_driver_unregister(&twlreg_driver);
+}
+module_exit(twlreg_exit)
+
+MODULE_DESCRIPTION("TWL6030 regulator driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
index f396bfef5d42..8f9cf0bc571c 100644
--- a/drivers/remoteproc/Kconfig
+++ b/drivers/remoteproc/Kconfig
@@ -1,20 +1,21 @@
 menu "Remoteproc drivers"
 
-# REMOTEPROC gets selected by whoever wants it
 config REMOTEPROC
-	tristate
+	tristate "Support for Remote Processor subsystem"
 	depends on HAS_DMA
 	select CRC32
 	select FW_LOADER
 	select VIRTIO
 	select VIRTUALIZATION
 
+if REMOTEPROC
+
 config OMAP_REMOTEPROC
 	tristate "OMAP remoteproc support"
 	depends on HAS_DMA
 	depends on ARCH_OMAP4 || SOC_OMAP5
 	depends on OMAP_IOMMU
-	select REMOTEPROC
+	depends on REMOTEPROC
 	select MAILBOX
 	select OMAP2PLUS_MBOX
 	select RPMSG_VIRTIO
@@ -31,20 +32,10 @@ config OMAP_REMOTEPROC
 	  It's safe to say n here if you're not interested in multimedia
 	  offloading or just want a bare minimum kernel.
 
-config STE_MODEM_RPROC
-	tristate "STE-Modem remoteproc support"
-	depends on HAS_DMA
-	select REMOTEPROC
-	default n
-	help
-	  Say y or m here to support STE-Modem shared memory driver.
-	  This can be either built-in or a loadable module.
-	  If unsure say N.
-
 config WKUP_M3_RPROC
 	tristate "AMx3xx Wakeup M3 remoteproc support"
 	depends on SOC_AM33XX || SOC_AM43XX
-	select REMOTEPROC
+	depends on REMOTEPROC
 	help
 	  Say y here to support Wakeup M3 remote processor on TI AM33xx
 	  and AM43xx family of SoCs.
@@ -57,8 +48,8 @@ config WKUP_M3_RPROC
 config DA8XX_REMOTEPROC
 	tristate "DA8xx/OMAP-L13x remoteproc support"
 	depends on ARCH_DAVINCI_DA8XX
+	depends on REMOTEPROC
 	select CMA if MMU
-	select REMOTEPROC
 	select RPMSG_VIRTIO
 	help
 	  Say y here to support DA8xx/OMAP-L13x remote processors via the
@@ -77,6 +68,18 @@ config DA8XX_REMOTEPROC
 	  It's safe to say n here if you're not interested in multimedia
 	  offloading.
 
+config QCOM_ADSP_PIL
+	tristate "Qualcomm ADSP Peripheral Image Loader"
+	depends on OF && ARCH_QCOM
+	depends on REMOTEPROC
+	depends on QCOM_SMEM
+	select MFD_SYSCON
+	select QCOM_MDT_LOADER
+	select QCOM_SCM
+	help
+	  Say y here to support the TrustZone based Peripherial Image Loader
+	  for the Qualcomm ADSP remote processors.
+
 config QCOM_MDT_LOADER
 	tristate
 
@@ -84,25 +87,22 @@ config QCOM_Q6V5_PIL
 	tristate "Qualcomm Hexagon V5 Peripherial Image Loader"
 	depends on OF && ARCH_QCOM
 	depends on QCOM_SMEM
+	depends on REMOTEPROC
 	select MFD_SYSCON
 	select QCOM_MDT_LOADER
-	select REMOTEPROC
+	select QCOM_SCM
 	help
 	  Say y here to support the Qualcomm Peripherial Image Loader for the
 	  Hexagon V5 based remote processors.
 
-config QCOM_WCNSS_IRIS
-	tristate
-	depends on OF && ARCH_QCOM
-
 config QCOM_WCNSS_PIL
 	tristate "Qualcomm WCNSS Peripheral Image Loader"
 	depends on OF && ARCH_QCOM
+	depends on QCOM_SMD || (COMPILE_TEST && QCOM_SMD=n)
 	depends on QCOM_SMEM
+	depends on REMOTEPROC
 	select QCOM_MDT_LOADER
 	select QCOM_SCM
-	select QCOM_WCNSS_IRIS
-	select REMOTEPROC
 	help
 	  Say y here to support the Peripheral Image Loader for the Qualcomm
 	  Wireless Connectivity Subsystem.
@@ -110,10 +110,16 @@ config QCOM_WCNSS_PIL
 config ST_REMOTEPROC
 	tristate "ST remoteproc support"
 	depends on ARCH_STI
-	select REMOTEPROC
+	depends on REMOTEPROC
 	help
 	  Say y here to support ST's adjunct processors via the remote
 	  processor framework.
 	  This can be either built-in or a loadable module.
 
+config ST_SLIM_REMOTEPROC
+	tristate
+	depends on REMOTEPROC
+
+endif # REMOTEPROC
+
 endmenu
diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile
index 6dfb62ed643f..0938ea3c41ba 100644
--- a/drivers/remoteproc/Makefile
+++ b/drivers/remoteproc/Makefile
@@ -5,14 +5,17 @@
 obj-$(CONFIG_REMOTEPROC)		+= remoteproc.o
 remoteproc-y				:= remoteproc_core.o
 remoteproc-y				+= remoteproc_debugfs.o
+remoteproc-y				+= remoteproc_sysfs.o
 remoteproc-y				+= remoteproc_virtio.o
 remoteproc-y				+= remoteproc_elf_loader.o
 obj-$(CONFIG_OMAP_REMOTEPROC)		+= omap_remoteproc.o
-obj-$(CONFIG_STE_MODEM_RPROC)	 	+= ste_modem_rproc.o
 obj-$(CONFIG_WKUP_M3_RPROC)		+= wkup_m3_rproc.o
 obj-$(CONFIG_DA8XX_REMOTEPROC)		+= da8xx_remoteproc.o
+obj-$(CONFIG_QCOM_ADSP_PIL)		+= qcom_adsp_pil.o
 obj-$(CONFIG_QCOM_MDT_LOADER)		+= qcom_mdt_loader.o
 obj-$(CONFIG_QCOM_Q6V5_PIL)		+= qcom_q6v5_pil.o
-obj-$(CONFIG_QCOM_WCNSS_IRIS)		+= qcom_wcnss_iris.o
-obj-$(CONFIG_QCOM_WCNSS_PIL)		+= qcom_wcnss.o
+obj-$(CONFIG_QCOM_WCNSS_PIL)		+= qcom_wcnss_pil.o
+qcom_wcnss_pil-y			+= qcom_wcnss.o
+qcom_wcnss_pil-y			+= qcom_wcnss_iris.o
 obj-$(CONFIG_ST_REMOTEPROC)		+= st_remoteproc.o
+obj-$(CONFIG_ST_SLIM_REMOTEPROC)	+= st_slim_rproc.o
diff --git a/drivers/remoteproc/qcom_adsp_pil.c b/drivers/remoteproc/qcom_adsp_pil.c
new file mode 100644
index 000000000000..43a4ed2f346c
--- /dev/null
+++ b/drivers/remoteproc/qcom_adsp_pil.c
@@ -0,0 +1,428 @@
+/*
+ * Qualcomm ADSP Peripheral Image Loader for MSM8974 and MSM8996
+ *
+ * Copyright (C) 2016 Linaro Ltd
+ * Copyright (C) 2014 Sony Mobile Communications AB
+ * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/firmware.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/qcom_scm.h>
+#include <linux/regulator/consumer.h>
+#include <linux/remoteproc.h>
+#include <linux/soc/qcom/smem.h>
+#include <linux/soc/qcom/smem_state.h>
+
+#include "qcom_mdt_loader.h"
+#include "remoteproc_internal.h"
+
+#define ADSP_CRASH_REASON_SMEM		423
+#define ADSP_FIRMWARE_NAME		"adsp.mdt"
+#define ADSP_PAS_ID			1
+
+struct qcom_adsp {
+	struct device *dev;
+	struct rproc *rproc;
+
+	int wdog_irq;
+	int fatal_irq;
+	int ready_irq;
+	int handover_irq;
+	int stop_ack_irq;
+
+	struct qcom_smem_state *state;
+	unsigned stop_bit;
+
+	struct clk *xo;
+
+	struct regulator *cx_supply;
+
+	struct completion start_done;
+	struct completion stop_done;
+
+	phys_addr_t mem_phys;
+	phys_addr_t mem_reloc;
+	void *mem_region;
+	size_t mem_size;
+};
+
+static int adsp_load(struct rproc *rproc, const struct firmware *fw)
+{
+	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	phys_addr_t fw_addr;
+	size_t fw_size;
+	bool relocate;
+	int ret;
+
+	ret = qcom_scm_pas_init_image(ADSP_PAS_ID, fw->data, fw->size);
+	if (ret) {
+		dev_err(&rproc->dev, "invalid firmware metadata\n");
+		return ret;
+	}
+
+	ret = qcom_mdt_parse(fw, &fw_addr, &fw_size, &relocate);
+	if (ret) {
+		dev_err(&rproc->dev, "failed to parse mdt header\n");
+		return ret;
+	}
+
+	if (relocate) {
+		adsp->mem_reloc = fw_addr;
+
+		ret = qcom_scm_pas_mem_setup(ADSP_PAS_ID, adsp->mem_phys, fw_size);
+		if (ret) {
+			dev_err(&rproc->dev, "unable to setup memory for image\n");
+			return ret;
+		}
+	}
+
+	return qcom_mdt_load(rproc, fw, rproc->firmware);
+}
+
+static const struct rproc_fw_ops adsp_fw_ops = {
+	.find_rsc_table = qcom_mdt_find_rsc_table,
+	.load = adsp_load,
+};
+
+static int adsp_start(struct rproc *rproc)
+{
+	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	int ret;
+
+	ret = clk_prepare_enable(adsp->xo);
+	if (ret)
+		return ret;
+
+	ret = regulator_enable(adsp->cx_supply);
+	if (ret)
+		goto disable_clocks;
+
+	ret = qcom_scm_pas_auth_and_reset(ADSP_PAS_ID);
+	if (ret) {
+		dev_err(adsp->dev,
+			"failed to authenticate image and release reset\n");
+		goto disable_regulators;
+	}
+
+	ret = wait_for_completion_timeout(&adsp->start_done,
+					  msecs_to_jiffies(5000));
+	if (!ret) {
+		dev_err(adsp->dev, "start timed out\n");
+		qcom_scm_pas_shutdown(ADSP_PAS_ID);
+		ret = -ETIMEDOUT;
+		goto disable_regulators;
+	}
+
+	ret = 0;
+
+disable_regulators:
+	regulator_disable(adsp->cx_supply);
+disable_clocks:
+	clk_disable_unprepare(adsp->xo);
+
+	return ret;
+}
+
+static int adsp_stop(struct rproc *rproc)
+{
+	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	int ret;
+
+	qcom_smem_state_update_bits(adsp->state,
+				    BIT(adsp->stop_bit),
+				    BIT(adsp->stop_bit));
+
+	ret = wait_for_completion_timeout(&adsp->stop_done,
+					  msecs_to_jiffies(5000));
+	if (ret == 0)
+		dev_err(adsp->dev, "timed out on wait\n");
+
+	qcom_smem_state_update_bits(adsp->state,
+				    BIT(adsp->stop_bit),
+				    0);
+
+	ret = qcom_scm_pas_shutdown(ADSP_PAS_ID);
+	if (ret)
+		dev_err(adsp->dev, "failed to shutdown: %d\n", ret);
+
+	return ret;
+}
+
+static void *adsp_da_to_va(struct rproc *rproc, u64 da, int len)
+{
+	struct qcom_adsp *adsp = (struct qcom_adsp *)rproc->priv;
+	int offset;
+
+	offset = da - adsp->mem_reloc;
+	if (offset < 0 || offset + len > adsp->mem_size)
+		return NULL;
+
+	return adsp->mem_region + offset;
+}
+
+static const struct rproc_ops adsp_ops = {
+	.start = adsp_start,
+	.stop = adsp_stop,
+	.da_to_va = adsp_da_to_va,
+};
+
+static irqreturn_t adsp_wdog_interrupt(int irq, void *dev)
+{
+	struct qcom_adsp *adsp = dev;
+
+	rproc_report_crash(adsp->rproc, RPROC_WATCHDOG);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t adsp_fatal_interrupt(int irq, void *dev)
+{
+	struct qcom_adsp *adsp = dev;
+	size_t len;
+	char *msg;
+
+	msg = qcom_smem_get(QCOM_SMEM_HOST_ANY, ADSP_CRASH_REASON_SMEM, &len);
+	if (!IS_ERR(msg) && len > 0 && msg[0])
+		dev_err(adsp->dev, "fatal error received: %s\n", msg);
+
+	rproc_report_crash(adsp->rproc, RPROC_FATAL_ERROR);
+
+	if (!IS_ERR(msg))
+		msg[0] = '\0';
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t adsp_ready_interrupt(int irq, void *dev)
+{
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t adsp_handover_interrupt(int irq, void *dev)
+{
+	struct qcom_adsp *adsp = dev;
+
+	complete(&adsp->start_done);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t adsp_stop_ack_interrupt(int irq, void *dev)
+{
+	struct qcom_adsp *adsp = dev;
+
+	complete(&adsp->stop_done);
+
+	return IRQ_HANDLED;
+}
+
+static int adsp_init_clock(struct qcom_adsp *adsp)
+{
+	int ret;
+
+	adsp->xo = devm_clk_get(adsp->dev, "xo");
+	if (IS_ERR(adsp->xo)) {
+		ret = PTR_ERR(adsp->xo);
+		if (ret != -EPROBE_DEFER)
+			dev_err(adsp->dev, "failed to get xo clock");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int adsp_init_regulator(struct qcom_adsp *adsp)
+{
+	adsp->cx_supply = devm_regulator_get(adsp->dev, "cx");
+	if (IS_ERR(adsp->cx_supply))
+		return PTR_ERR(adsp->cx_supply);
+
+	regulator_set_load(adsp->cx_supply, 100000);
+
+	return 0;
+}
+
+static int adsp_request_irq(struct qcom_adsp *adsp,
+			     struct platform_device *pdev,
+			     const char *name,
+			     irq_handler_t thread_fn)
+{
+	int ret;
+
+	ret = platform_get_irq_byname(pdev, name);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "no %s IRQ defined\n", name);
+		return ret;
+	}
+
+	ret = devm_request_threaded_irq(&pdev->dev, ret,
+					NULL, thread_fn,
+					IRQF_ONESHOT,
+					"adsp", adsp);
+	if (ret)
+		dev_err(&pdev->dev, "request %s IRQ failed\n", name);
+
+	return ret;
+}
+
+static int adsp_alloc_memory_region(struct qcom_adsp *adsp)
+{
+	struct device_node *node;
+	struct resource r;
+	int ret;
+
+	node = of_parse_phandle(adsp->dev->of_node, "memory-region", 0);
+	if (!node) {
+		dev_err(adsp->dev, "no memory-region specified\n");
+		return -EINVAL;
+	}
+
+	ret = of_address_to_resource(node, 0, &r);
+	if (ret)
+		return ret;
+
+	adsp->mem_phys = adsp->mem_reloc = r.start;
+	adsp->mem_size = resource_size(&r);
+	adsp->mem_region = devm_ioremap_wc(adsp->dev, adsp->mem_phys, adsp->mem_size);
+	if (!adsp->mem_region) {
+		dev_err(adsp->dev, "unable to map memory region: %pa+%zx\n",
+			&r.start, adsp->mem_size);
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+static int adsp_probe(struct platform_device *pdev)
+{
+	struct qcom_adsp *adsp;
+	struct rproc *rproc;
+	int ret;
+
+	if (!qcom_scm_is_available())
+		return -EPROBE_DEFER;
+
+	if (!qcom_scm_pas_supported(ADSP_PAS_ID)) {
+		dev_err(&pdev->dev, "PAS is not available for ADSP\n");
+		return -ENXIO;
+	}
+
+	rproc = rproc_alloc(&pdev->dev, pdev->name, &adsp_ops,
+			    ADSP_FIRMWARE_NAME, sizeof(*adsp));
+	if (!rproc) {
+		dev_err(&pdev->dev, "unable to allocate remoteproc\n");
+		return -ENOMEM;
+	}
+
+	rproc->fw_ops = &adsp_fw_ops;
+
+	adsp = (struct qcom_adsp *)rproc->priv;
+	adsp->dev = &pdev->dev;
+	adsp->rproc = rproc;
+	platform_set_drvdata(pdev, adsp);
+
+	init_completion(&adsp->start_done);
+	init_completion(&adsp->stop_done);
+
+	ret = adsp_alloc_memory_region(adsp);
+	if (ret)
+		goto free_rproc;
+
+	ret = adsp_init_clock(adsp);
+	if (ret)
+		goto free_rproc;
+
+	ret = adsp_init_regulator(adsp);
+	if (ret)
+		goto free_rproc;
+
+	ret = adsp_request_irq(adsp, pdev, "wdog", adsp_wdog_interrupt);
+	if (ret < 0)
+		goto free_rproc;
+	adsp->wdog_irq = ret;
+
+	ret = adsp_request_irq(adsp, pdev, "fatal", adsp_fatal_interrupt);
+	if (ret < 0)
+		goto free_rproc;
+	adsp->fatal_irq = ret;
+
+	ret = adsp_request_irq(adsp, pdev, "ready", adsp_ready_interrupt);
+	if (ret < 0)
+		goto free_rproc;
+	adsp->ready_irq = ret;
+
+	ret = adsp_request_irq(adsp, pdev, "handover", adsp_handover_interrupt);
+	if (ret < 0)
+		goto free_rproc;
+	adsp->handover_irq = ret;
+
+	ret = adsp_request_irq(adsp, pdev, "stop-ack", adsp_stop_ack_interrupt);
+	if (ret < 0)
+		goto free_rproc;
+	adsp->stop_ack_irq = ret;
+
+	adsp->state = qcom_smem_state_get(&pdev->dev, "stop",
+					  &adsp->stop_bit);
+	if (IS_ERR(adsp->state)) {
+		ret = PTR_ERR(adsp->state);
+		goto free_rproc;
+	}
+
+	ret = rproc_add(rproc);
+	if (ret)
+		goto free_rproc;
+
+	return 0;
+
+free_rproc:
+	rproc_free(rproc);
+
+	return ret;
+}
+
+static int adsp_remove(struct platform_device *pdev)
+{
+	struct qcom_adsp *adsp = platform_get_drvdata(pdev);
+
+	qcom_smem_state_put(adsp->state);
+	rproc_del(adsp->rproc);
+	rproc_free(adsp->rproc);
+
+	return 0;
+}
+
+static const struct of_device_id adsp_of_match[] = {
+	{ .compatible = "qcom,msm8974-adsp-pil" },
+	{ .compatible = "qcom,msm8996-adsp-pil" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, adsp_of_match);
+
+static struct platform_driver adsp_driver = {
+	.probe = adsp_probe,
+	.remove = adsp_remove,
+	.driver = {
+		.name = "qcom_adsp_pil",
+		.of_match_table = adsp_of_match,
+	},
+};
+
+module_platform_driver(adsp_driver);
+MODULE_DESCRIPTION("Qualcomm MSM8974/MSM8996 ADSP Peripherial Image Loader");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/remoteproc/qcom_mdt_loader.c b/drivers/remoteproc/qcom_mdt_loader.c
index 114e8e4cef67..2ff18cd6c096 100644
--- a/drivers/remoteproc/qcom_mdt_loader.c
+++ b/drivers/remoteproc/qcom_mdt_loader.c
@@ -20,6 +20,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/remoteproc.h>
+#include <linux/sizes.h>
 #include <linux/slab.h>
 
 #include "remoteproc_internal.h"
diff --git a/drivers/remoteproc/qcom_q6v5_pil.c b/drivers/remoteproc/qcom_q6v5_pil.c
index 2e0caaaa766a..b08989b48df7 100644
--- a/drivers/remoteproc/qcom_q6v5_pil.c
+++ b/drivers/remoteproc/qcom_q6v5_pil.c
@@ -894,6 +894,7 @@ static const struct of_device_id q6v5_of_match[] = {
 	{ .compatible = "qcom,q6v5-pil", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, q6v5_of_match);
 
 static struct platform_driver q6v5_driver = {
 	.probe = q6v5_probe,
diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c
index f5cedeaafba1..ebd61f5d18bb 100644
--- a/drivers/remoteproc/qcom_wcnss.c
+++ b/drivers/remoteproc/qcom_wcnss.c
@@ -30,6 +30,7 @@
 #include <linux/remoteproc.h>
 #include <linux/soc/qcom/smem.h>
 #include <linux/soc/qcom/smem_state.h>
+#include <linux/rpmsg/qcom_smd.h>
 
 #include "qcom_mdt_loader.h"
 #include "remoteproc_internal.h"
@@ -94,6 +95,10 @@ struct qcom_wcnss {
 	phys_addr_t mem_reloc;
 	void *mem_region;
 	size_t mem_size;
+
+	struct device_node *smd_node;
+	struct qcom_smd_edge *smd_edge;
+	struct rproc_subdev smd_subdev;
 };
 
 static const struct wcnss_data riva_data = {
@@ -143,7 +148,6 @@ void qcom_wcnss_assign_iris(struct qcom_wcnss *wcnss,
 
 	mutex_unlock(&wcnss->iris_lock);
 }
-EXPORT_SYMBOL_GPL(qcom_wcnss_assign_iris);
 
 static int wcnss_load(struct rproc *rproc, const struct firmware *fw)
 {
@@ -396,6 +400,23 @@ static irqreturn_t wcnss_stop_ack_interrupt(int irq, void *dev)
 	return IRQ_HANDLED;
 }
 
+static int wcnss_smd_probe(struct rproc_subdev *subdev)
+{
+	struct qcom_wcnss *wcnss = container_of(subdev, struct qcom_wcnss, smd_subdev);
+
+	wcnss->smd_edge = qcom_smd_register_edge(wcnss->dev, wcnss->smd_node);
+
+	return IS_ERR(wcnss->smd_edge) ? PTR_ERR(wcnss->smd_edge) : 0;
+}
+
+static void wcnss_smd_remove(struct rproc_subdev *subdev)
+{
+	struct qcom_wcnss *wcnss = container_of(subdev, struct qcom_wcnss, smd_subdev);
+
+	qcom_smd_unregister_edge(wcnss->smd_edge);
+	wcnss->smd_edge = NULL;
+}
+
 static int wcnss_init_regulators(struct qcom_wcnss *wcnss,
 				 const struct wcnss_vreg_info *info,
 				 int num_vregs)
@@ -578,6 +599,10 @@ static int wcnss_probe(struct platform_device *pdev)
 		}
 	}
 
+	wcnss->smd_node = of_get_child_by_name(pdev->dev.of_node, "smd-edge");
+	if (wcnss->smd_node)
+		rproc_add_subdev(rproc, &wcnss->smd_subdev, wcnss_smd_probe, wcnss_smd_remove);
+
 	ret = rproc_add(rproc);
 	if (ret)
 		goto free_rproc;
@@ -596,6 +621,7 @@ static int wcnss_remove(struct platform_device *pdev)
 
 	of_platform_depopulate(&pdev->dev);
 
+	of_node_put(wcnss->smd_node);
 	qcom_smem_state_put(wcnss->state);
 	rproc_del(wcnss->rproc);
 	rproc_free(wcnss->rproc);
@@ -609,6 +635,7 @@ static const struct of_device_id wcnss_of_match[] = {
 	{ .compatible = "qcom,pronto-v2-pil", &pronto_v2_data },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, wcnss_of_match);
 
 static struct platform_driver wcnss_driver = {
 	.probe = wcnss_probe,
@@ -619,6 +646,28 @@ static struct platform_driver wcnss_driver = {
 	},
 };
 
-module_platform_driver(wcnss_driver);
+static int __init wcnss_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&wcnss_driver);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&qcom_iris_driver);
+	if (ret)
+		platform_driver_unregister(&wcnss_driver);
+
+	return ret;
+}
+module_init(wcnss_init);
+
+static void __exit wcnss_exit(void)
+{
+	platform_driver_unregister(&qcom_iris_driver);
+	platform_driver_unregister(&wcnss_driver);
+}
+module_exit(wcnss_exit);
+
 MODULE_DESCRIPTION("Qualcomm Peripherial Image Loader for Wireless Subsystem");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/remoteproc/qcom_wcnss.h b/drivers/remoteproc/qcom_wcnss.h
index 9dc4a9fe41e1..25fb7f62a457 100644
--- a/drivers/remoteproc/qcom_wcnss.h
+++ b/drivers/remoteproc/qcom_wcnss.h
@@ -4,6 +4,8 @@
 struct qcom_iris;
 struct qcom_wcnss;
 
+extern struct platform_driver qcom_iris_driver;
+
 struct wcnss_vreg_info {
 	const char * const name;
 	int min_voltage;
diff --git a/drivers/remoteproc/qcom_wcnss_iris.c b/drivers/remoteproc/qcom_wcnss_iris.c
index f0ca24a8dd0b..e842be58e8c7 100644
--- a/drivers/remoteproc/qcom_wcnss_iris.c
+++ b/drivers/remoteproc/qcom_wcnss_iris.c
@@ -94,14 +94,12 @@ disable_regulators:
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(qcom_iris_enable);
 
 void qcom_iris_disable(struct qcom_iris *iris)
 {
 	clk_disable_unprepare(iris->xo_clk);
 	regulator_bulk_disable(iris->num_vregs, iris->vregs);
 }
-EXPORT_SYMBOL_GPL(qcom_iris_disable);
 
 static int qcom_iris_probe(struct platform_device *pdev)
 {
@@ -173,8 +171,9 @@ static const struct of_device_id iris_of_match[] = {
 	{ .compatible = "qcom,wcn3680", .data = &wcn3680_data },
 	{}
 };
+MODULE_DEVICE_TABLE(of, iris_of_match);
 
-static struct platform_driver wcnss_driver = {
+struct platform_driver qcom_iris_driver = {
 	.probe = qcom_iris_probe,
 	.remove = qcom_iris_remove,
 	.driver = {
@@ -182,7 +181,3 @@ static struct platform_driver wcnss_driver = {
 		.of_match_table = iris_of_match,
 	},
 };
-
-module_platform_driver(wcnss_driver);
-MODULE_DESCRIPTION("Qualcomm Wireless Subsystem Iris driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c
index c6bfb3496684..9a507e77eced 100644
--- a/drivers/remoteproc/remoteproc_core.c
+++ b/drivers/remoteproc/remoteproc_core.c
@@ -236,6 +236,10 @@ int rproc_alloc_vring(struct rproc_vdev *rvdev, int i)
 	}
 	notifyid = ret;
 
+	/* Potentially bump max_notifyid */
+	if (notifyid > rproc->max_notifyid)
+		rproc->max_notifyid = notifyid;
+
 	dev_dbg(dev, "vring%d: va %p dma %pad size 0x%x idr %d\n",
 		i, va, &dma, size, notifyid);
 
@@ -296,6 +300,20 @@ void rproc_free_vring(struct rproc_vring *rvring)
 	rsc->vring[idx].notifyid = -1;
 }
 
+static int rproc_vdev_do_probe(struct rproc_subdev *subdev)
+{
+	struct rproc_vdev *rvdev = container_of(subdev, struct rproc_vdev, subdev);
+
+	return rproc_add_virtio_dev(rvdev, rvdev->id);
+}
+
+static void rproc_vdev_do_remove(struct rproc_subdev *subdev)
+{
+	struct rproc_vdev *rvdev = container_of(subdev, struct rproc_vdev, subdev);
+
+	rproc_remove_virtio_dev(rvdev);
+}
+
 /**
  * rproc_handle_vdev() - handle a vdev fw resource
  * @rproc: the remote processor
@@ -356,6 +374,9 @@ static int rproc_handle_vdev(struct rproc *rproc, struct fw_rsc_vdev *rsc,
 	if (!rvdev)
 		return -ENOMEM;
 
+	kref_init(&rvdev->refcount);
+
+	rvdev->id = rsc->id;
 	rvdev->rproc = rproc;
 
 	/* parse the vrings */
@@ -368,22 +389,51 @@ static int rproc_handle_vdev(struct rproc *rproc, struct fw_rsc_vdev *rsc,
 	/* remember the resource offset*/
 	rvdev->rsc_offset = offset;
 
+	/* allocate the vring resources */
+	for (i = 0; i < rsc->num_of_vrings; i++) {
+		ret = rproc_alloc_vring(rvdev, i);
+		if (ret)
+			goto unwind_vring_allocations;
+	}
+
+	/* track the rvdevs list reference */
+	kref_get(&rvdev->refcount);
+
 	list_add_tail(&rvdev->node, &rproc->rvdevs);
 
-	/* it is now safe to add the virtio device */
-	ret = rproc_add_virtio_dev(rvdev, rsc->id);
-	if (ret)
-		goto remove_rvdev;
+	rproc_add_subdev(rproc, &rvdev->subdev,
+			 rproc_vdev_do_probe, rproc_vdev_do_remove);
 
 	return 0;
 
-remove_rvdev:
-	list_del(&rvdev->node);
+unwind_vring_allocations:
+	for (i--; i >= 0; i--)
+		rproc_free_vring(&rvdev->vring[i]);
 free_rvdev:
 	kfree(rvdev);
 	return ret;
 }
 
+void rproc_vdev_release(struct kref *ref)
+{
+	struct rproc_vdev *rvdev = container_of(ref, struct rproc_vdev, refcount);
+	struct rproc_vring *rvring;
+	struct rproc *rproc = rvdev->rproc;
+	int id;
+
+	for (id = 0; id < ARRAY_SIZE(rvdev->vring); id++) {
+		rvring = &rvdev->vring[id];
+		if (!rvring->va)
+			continue;
+
+		rproc_free_vring(rvring);
+	}
+
+	rproc_remove_subdev(rproc, &rvdev->subdev);
+	list_del(&rvdev->node);
+	kfree(rvdev);
+}
+
 /**
  * rproc_handle_trace() - handle a shared trace buffer resource
  * @rproc: the remote processor
@@ -673,15 +723,6 @@ free_carv:
 	return ret;
 }
 
-static int rproc_count_vrings(struct rproc *rproc, struct fw_rsc_vdev *rsc,
-			      int offset, int avail)
-{
-	/* Summarize the number of notification IDs */
-	rproc->max_notifyid += rsc->num_of_vrings;
-
-	return 0;
-}
-
 /*
  * A lookup table for resource handlers. The indices are defined in
  * enum fw_resource_type.
@@ -690,10 +731,6 @@ static rproc_handle_resource_t rproc_loading_handlers[RSC_LAST] = {
 	[RSC_CARVEOUT] = (rproc_handle_resource_t)rproc_handle_carveout,
 	[RSC_DEVMEM] = (rproc_handle_resource_t)rproc_handle_devmem,
 	[RSC_TRACE] = (rproc_handle_resource_t)rproc_handle_trace,
-	[RSC_VDEV] = (rproc_handle_resource_t)rproc_count_vrings,
-};
-
-static rproc_handle_resource_t rproc_vdev_handler[RSC_LAST] = {
 	[RSC_VDEV] = (rproc_handle_resource_t)rproc_handle_vdev,
 };
 
@@ -736,6 +773,34 @@ static int rproc_handle_resources(struct rproc *rproc, int len,
 	return ret;
 }
 
+static int rproc_probe_subdevices(struct rproc *rproc)
+{
+	struct rproc_subdev *subdev;
+	int ret;
+
+	list_for_each_entry(subdev, &rproc->subdevs, node) {
+		ret = subdev->probe(subdev);
+		if (ret)
+			goto unroll_registration;
+	}
+
+	return 0;
+
+unroll_registration:
+	list_for_each_entry_continue_reverse(subdev, &rproc->subdevs, node)
+		subdev->remove(subdev);
+
+	return ret;
+}
+
+static void rproc_remove_subdevices(struct rproc *rproc)
+{
+	struct rproc_subdev *subdev;
+
+	list_for_each_entry(subdev, &rproc->subdevs, node)
+		subdev->remove(subdev);
+}
+
 /**
  * rproc_resource_cleanup() - clean up and free all acquired resources
  * @rproc: rproc handle
@@ -782,7 +847,7 @@ static void rproc_resource_cleanup(struct rproc *rproc)
 
 	/* clean up remote vdev entries */
 	list_for_each_entry_safe(rvdev, rvtmp, &rproc->rvdevs, node)
-		rproc_remove_virtio_dev(rvdev);
+		kref_put(&rvdev->refcount, rproc_vdev_release);
 }
 
 /*
@@ -824,25 +889,16 @@ static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)
 	/*
 	 * Create a copy of the resource table. When a virtio device starts
 	 * and calls vring_new_virtqueue() the address of the allocated vring
-	 * will be stored in the cached_table. Before the device is started,
-	 * cached_table will be copied into device memory.
+	 * will be stored in the table_ptr. Before the device is started,
+	 * table_ptr will be copied into device memory.
 	 */
-	rproc->cached_table = kmemdup(table, tablesz, GFP_KERNEL);
-	if (!rproc->cached_table)
+	rproc->table_ptr = kmemdup(table, tablesz, GFP_KERNEL);
+	if (!rproc->table_ptr)
 		goto clean_up;
 
-	rproc->table_ptr = rproc->cached_table;
-
 	/* reset max_notifyid */
 	rproc->max_notifyid = -1;
 
-	/* look for virtio devices and register them */
-	ret = rproc_handle_resources(rproc, tablesz, rproc_vdev_handler);
-	if (ret) {
-		dev_err(dev, "Failed to handle vdev resources: %d\n", ret);
-		goto clean_up;
-	}
-
 	/* handle fw resources which are required to boot rproc */
 	ret = rproc_handle_resources(rproc, tablesz, rproc_loading_handlers);
 	if (ret) {
@@ -858,18 +914,16 @@ static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)
 	}
 
 	/*
-	 * The starting device has been given the rproc->cached_table as the
+	 * The starting device has been given the rproc->table_ptr as the
 	 * resource table. The address of the vring along with the other
-	 * allocated resources (carveouts etc) is stored in cached_table.
+	 * allocated resources (carveouts etc) is stored in table_ptr.
 	 * In order to pass this information to the remote device we must copy
 	 * this information to device memory. We also update the table_ptr so
 	 * that any subsequent changes will be applied to the loaded version.
 	 */
 	loaded_table = rproc_find_loaded_rsc_table(rproc, fw);
-	if (loaded_table) {
-		memcpy(loaded_table, rproc->cached_table, tablesz);
-		rproc->table_ptr = loaded_table;
-	}
+	if (loaded_table)
+		memcpy(loaded_table, rproc->table_ptr, tablesz);
 
 	/* power up the remote processor */
 	ret = rproc->ops->start(rproc);
@@ -878,17 +932,26 @@ static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)
 		goto clean_up_resources;
 	}
 
+	/* probe any subdevices for the remote processor */
+	ret = rproc_probe_subdevices(rproc);
+	if (ret) {
+		dev_err(dev, "failed to probe subdevices for %s: %d\n",
+			rproc->name, ret);
+		goto stop_rproc;
+	}
+
 	rproc->state = RPROC_RUNNING;
 
 	dev_info(dev, "remote processor %s is now up\n", rproc->name);
 
 	return 0;
 
+stop_rproc:
+	rproc->ops->stop(rproc);
 clean_up_resources:
 	rproc_resource_cleanup(rproc);
 clean_up:
-	kfree(rproc->cached_table);
-	rproc->cached_table = NULL;
+	kfree(rproc->table_ptr);
 	rproc->table_ptr = NULL;
 
 	rproc_disable_iommu(rproc);
@@ -909,7 +972,7 @@ static void rproc_fw_config_virtio(const struct firmware *fw, void *context)
 
 	/* if rproc is marked always-on, request it to boot */
 	if (rproc->auto_boot)
-		rproc_boot_nowait(rproc);
+		rproc_boot(rproc);
 
 	release_firmware(fw);
 	/* allow rproc_del() contexts, if any, to proceed */
@@ -1007,7 +1070,6 @@ static void rproc_crash_handler_work(struct work_struct *work)
 /**
  * __rproc_boot() - boot a remote processor
  * @rproc: handle of a remote processor
- * @wait: wait for rproc registration completion
  *
  * Boot a remote processor (i.e. load its firmware, power it on, ...).
  *
@@ -1016,7 +1078,7 @@ static void rproc_crash_handler_work(struct work_struct *work)
  *
  * Returns 0 on success, and an appropriate error value otherwise.
  */
-static int __rproc_boot(struct rproc *rproc, bool wait)
+static int __rproc_boot(struct rproc *rproc)
 {
 	const struct firmware *firmware_p;
 	struct device *dev;
@@ -1050,10 +1112,6 @@ static int __rproc_boot(struct rproc *rproc, bool wait)
 		goto downref_rproc;
 	}
 
-	/* if rproc virtio is not yet configured, wait */
-	if (wait)
-		wait_for_completion(&rproc->firmware_loading_complete);
-
 	ret = rproc_fw_boot(rproc, firmware_p);
 
 	release_firmware(firmware_p);
@@ -1072,22 +1130,11 @@ unlock_mutex:
  */
 int rproc_boot(struct rproc *rproc)
 {
-	return __rproc_boot(rproc, true);
+	return __rproc_boot(rproc);
 }
 EXPORT_SYMBOL(rproc_boot);
 
 /**
- * rproc_boot_nowait() - boot a remote processor
- * @rproc: handle of a remote processor
- *
- * Same as rproc_boot() but don't wait for rproc registration completion
- */
-int rproc_boot_nowait(struct rproc *rproc)
-{
-	return __rproc_boot(rproc, false);
-}
-
-/**
  * rproc_shutdown() - power off the remote processor
  * @rproc: the remote processor
  *
@@ -1121,6 +1168,9 @@ void rproc_shutdown(struct rproc *rproc)
 	if (!atomic_dec_and_test(&rproc->power))
 		goto out;
 
+	/* remove any subdevices for the remote processor */
+	rproc_remove_subdevices(rproc);
+
 	/* power off the remote processor */
 	ret = rproc->ops->stop(rproc);
 	if (ret) {
@@ -1135,8 +1185,7 @@ void rproc_shutdown(struct rproc *rproc)
 	rproc_disable_iommu(rproc);
 
 	/* Free the copy of the resource table */
-	kfree(rproc->cached_table);
-	rproc->cached_table = NULL;
+	kfree(rproc->table_ptr);
 	rproc->table_ptr = NULL;
 
 	/* if in crash state, unlock crash handler */
@@ -1233,9 +1282,6 @@ int rproc_add(struct rproc *rproc)
 
 	dev_info(dev, "%s is available\n", rproc->name);
 
-	dev_info(dev, "Note: remoteproc is still under development and considered experimental.\n");
-	dev_info(dev, "THE BINARY FORMAT IS NOT YET FINALIZED, and backward compatibility isn't yet guaranteed.\n");
-
 	/* create debugfs entries */
 	rproc_create_debug_dir(rproc);
 	ret = rproc_add_virtio_devices(rproc);
@@ -1273,6 +1319,7 @@ static void rproc_type_release(struct device *dev)
 	if (rproc->index >= 0)
 		ida_simple_remove(&rproc_dev_index, rproc->index);
 
+	kfree(rproc->firmware);
 	kfree(rproc);
 }
 
@@ -1310,31 +1357,31 @@ struct rproc *rproc_alloc(struct device *dev, const char *name,
 {
 	struct rproc *rproc;
 	char *p, *template = "rproc-%s-fw";
-	int name_len = 0;
+	int name_len;
 
 	if (!dev || !name || !ops)
 		return NULL;
 
-	if (!firmware)
+	if (!firmware) {
 		/*
-		 * Make room for default firmware name (minus %s plus '\0').
 		 * If the caller didn't pass in a firmware name then
-		 * construct a default name.  We're already glomming 'len'
-		 * bytes onto the end of the struct rproc allocation, so do
-		 * a few more for the default firmware name (but only if
-		 * the caller doesn't pass one).
+		 * construct a default name.
 		 */
 		name_len = strlen(name) + strlen(template) - 2 + 1;
-
-	rproc = kzalloc(sizeof(*rproc) + len + name_len, GFP_KERNEL);
-	if (!rproc)
-		return NULL;
-
-	if (!firmware) {
-		p = (char *)rproc + sizeof(struct rproc) + len;
+		p = kmalloc(name_len, GFP_KERNEL);
+		if (!p)
+			return NULL;
 		snprintf(p, name_len, template, name);
 	} else {
-		p = (char *)firmware;
+		p = kstrdup(firmware, GFP_KERNEL);
+		if (!p)
+			return NULL;
+	}
+
+	rproc = kzalloc(sizeof(struct rproc) + len, GFP_KERNEL);
+	if (!rproc) {
+		kfree(p);
+		return NULL;
 	}
 
 	rproc->firmware = p;
@@ -1346,6 +1393,7 @@ struct rproc *rproc_alloc(struct device *dev, const char *name,
 	device_initialize(&rproc->dev);
 	rproc->dev.parent = dev;
 	rproc->dev.type = &rproc_type;
+	rproc->dev.class = &rproc_class;
 
 	/* Assign a unique device index and name */
 	rproc->index = ida_simple_get(&rproc_dev_index, 0, 0, GFP_KERNEL);
@@ -1370,6 +1418,7 @@ struct rproc *rproc_alloc(struct device *dev, const char *name,
 	INIT_LIST_HEAD(&rproc->mappings);
 	INIT_LIST_HEAD(&rproc->traces);
 	INIT_LIST_HEAD(&rproc->rvdevs);
+	INIT_LIST_HEAD(&rproc->subdevs);
 
 	INIT_WORK(&rproc->crash_handler, rproc_crash_handler_work);
 	init_completion(&rproc->crash_comp);
@@ -1428,8 +1477,6 @@ EXPORT_SYMBOL(rproc_put);
  */
 int rproc_del(struct rproc *rproc)
 {
-	struct rproc_vdev *rvdev, *tmp;
-
 	if (!rproc)
 		return -EINVAL;
 
@@ -1441,10 +1488,6 @@ int rproc_del(struct rproc *rproc)
 	if (rproc->auto_boot)
 		rproc_shutdown(rproc);
 
-	/* clean up remote vdev entries */
-	list_for_each_entry_safe(rvdev, tmp, &rproc->rvdevs, node)
-		rproc_remove_virtio_dev(rvdev);
-
 	/* the rproc is downref'ed as soon as it's removed from the klist */
 	mutex_lock(&rproc_list_mutex);
 	list_del(&rproc->node);
@@ -1457,6 +1500,36 @@ int rproc_del(struct rproc *rproc)
 EXPORT_SYMBOL(rproc_del);
 
 /**
+ * rproc_add_subdev() - add a subdevice to a remoteproc
+ * @rproc: rproc handle to add the subdevice to
+ * @subdev: subdev handle to register
+ * @probe: function to call when the rproc boots
+ * @remove: function to call when the rproc shuts down
+ */
+void rproc_add_subdev(struct rproc *rproc,
+		      struct rproc_subdev *subdev,
+		      int (*probe)(struct rproc_subdev *subdev),
+		      void (*remove)(struct rproc_subdev *subdev))
+{
+	subdev->probe = probe;
+	subdev->remove = remove;
+
+	list_add_tail(&subdev->node, &rproc->subdevs);
+}
+EXPORT_SYMBOL(rproc_add_subdev);
+
+/**
+ * rproc_remove_subdev() - remove a subdevice from a remoteproc
+ * @rproc: rproc handle to remove the subdevice from
+ * @subdev: subdev handle, previously registered with rproc_add_subdev()
+ */
+void rproc_remove_subdev(struct rproc *rproc, struct rproc_subdev *subdev)
+{
+	list_del(&subdev->node);
+}
+EXPORT_SYMBOL(rproc_remove_subdev);
+
+/**
  * rproc_report_crash() - rproc crash reporter function
  * @rproc: remote processor
  * @type: crash type
@@ -1484,6 +1557,7 @@ EXPORT_SYMBOL(rproc_report_crash);
 
 static int __init remoteproc_init(void)
 {
+	rproc_init_sysfs();
 	rproc_init_debugfs();
 
 	return 0;
@@ -1495,6 +1569,7 @@ static void __exit remoteproc_exit(void)
 	ida_destroy(&rproc_dev_index);
 
 	rproc_exit_debugfs();
+	rproc_exit_sysfs();
 }
 module_exit(remoteproc_exit);
 
diff --git a/drivers/remoteproc/remoteproc_debugfs.c b/drivers/remoteproc/remoteproc_debugfs.c
index 374797206c79..1c122e230cec 100644
--- a/drivers/remoteproc/remoteproc_debugfs.c
+++ b/drivers/remoteproc/remoteproc_debugfs.c
@@ -59,75 +59,6 @@ static const struct file_operations trace_rproc_ops = {
 	.llseek	= generic_file_llseek,
 };
 
-/*
- * A state-to-string lookup table, for exposing a human readable state
- * via debugfs. Always keep in sync with enum rproc_state
- */
-static const char * const rproc_state_string[] = {
-	"offline",
-	"suspended",
-	"running",
-	"crashed",
-	"invalid",
-};
-
-/* expose the state of the remote processor via debugfs */
-static ssize_t rproc_state_read(struct file *filp, char __user *userbuf,
-				size_t count, loff_t *ppos)
-{
-	struct rproc *rproc = filp->private_data;
-	unsigned int state;
-	char buf[30];
-	int i;
-
-	state = rproc->state > RPROC_LAST ? RPROC_LAST : rproc->state;
-
-	i = scnprintf(buf, 30, "%.28s (%d)\n", rproc_state_string[state],
-		      rproc->state);
-
-	return simple_read_from_buffer(userbuf, count, ppos, buf, i);
-}
-
-static ssize_t rproc_state_write(struct file *filp, const char __user *userbuf,
-				 size_t count, loff_t *ppos)
-{
-	struct rproc *rproc = filp->private_data;
-	char buf[10];
-	int ret;
-
-	if (count > sizeof(buf) || count <= 0)
-		return -EINVAL;
-
-	ret = copy_from_user(buf, userbuf, count);
-	if (ret)
-		return -EFAULT;
-
-	if (buf[count - 1] == '\n')
-		buf[count - 1] = '\0';
-
-	if (!strncmp(buf, "start", count)) {
-		ret = rproc_boot(rproc);
-		if (ret) {
-			dev_err(&rproc->dev, "Boot failed: %d\n", ret);
-			return ret;
-		}
-	} else if (!strncmp(buf, "stop", count)) {
-		rproc_shutdown(rproc);
-	} else {
-		dev_err(&rproc->dev, "Unrecognised option: %s\n", buf);
-		return -EINVAL;
-	}
-
-	return count;
-}
-
-static const struct file_operations rproc_state_ops = {
-	.read = rproc_state_read,
-	.write = rproc_state_write,
-	.open = simple_open,
-	.llseek	= generic_file_llseek,
-};
-
 /* expose the name of the remote processor via debugfs */
 static ssize_t rproc_name_read(struct file *filp, char __user *userbuf,
 			       size_t count, loff_t *ppos)
@@ -265,8 +196,6 @@ void rproc_create_debug_dir(struct rproc *rproc)
 
 	debugfs_create_file("name", 0400, rproc->dbg_dir,
 			    rproc, &rproc_name_ops);
-	debugfs_create_file("state", 0400, rproc->dbg_dir,
-			    rproc, &rproc_state_ops);
 	debugfs_create_file("recovery", 0400, rproc->dbg_dir,
 			    rproc, &rproc_recovery_ops);
 }
diff --git a/drivers/remoteproc/remoteproc_internal.h b/drivers/remoteproc/remoteproc_internal.h
index 4cf93ca2816e..1e9e5b3f021c 100644
--- a/drivers/remoteproc/remoteproc_internal.h
+++ b/drivers/remoteproc/remoteproc_internal.h
@@ -49,6 +49,7 @@ struct rproc_fw_ops {
 void rproc_release(struct kref *kref);
 irqreturn_t rproc_vq_interrupt(struct rproc *rproc, int vq_id);
 int rproc_boot_nowait(struct rproc *rproc);
+void rproc_vdev_release(struct kref *ref);
 
 /* from remoteproc_virtio.c */
 int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id);
@@ -63,6 +64,11 @@ void rproc_create_debug_dir(struct rproc *rproc);
 void rproc_init_debugfs(void);
 void rproc_exit_debugfs(void);
 
+/* from remoteproc_sysfs.c */
+extern struct class rproc_class;
+int rproc_init_sysfs(void);
+void rproc_exit_sysfs(void);
+
 void rproc_free_vring(struct rproc_vring *rvring);
 int rproc_alloc_vring(struct rproc_vdev *rvdev, int i);
 
diff --git a/drivers/remoteproc/remoteproc_sysfs.c b/drivers/remoteproc/remoteproc_sysfs.c
new file mode 100644
index 000000000000..bc5b0e00efb1
--- /dev/null
+++ b/drivers/remoteproc/remoteproc_sysfs.c
@@ -0,0 +1,151 @@
+/*
+ * Remote Processor Framework
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/remoteproc.h>
+
+#include "remoteproc_internal.h"
+
+#define to_rproc(d) container_of(d, struct rproc, dev)
+
+/* Expose the loaded / running firmware name via sysfs */
+static ssize_t firmware_show(struct device *dev, struct device_attribute *attr,
+			  char *buf)
+{
+	struct rproc *rproc = to_rproc(dev);
+
+	return sprintf(buf, "%s\n", rproc->firmware);
+}
+
+/* Change firmware name via sysfs */
+static ssize_t firmware_store(struct device *dev,
+			      struct device_attribute *attr,
+			      const char *buf, size_t count)
+{
+	struct rproc *rproc = to_rproc(dev);
+	char *p;
+	int err, len = count;
+
+	err = mutex_lock_interruptible(&rproc->lock);
+	if (err) {
+		dev_err(dev, "can't lock rproc %s: %d\n", rproc->name, err);
+		return -EINVAL;
+	}
+
+	if (rproc->state != RPROC_OFFLINE) {
+		dev_err(dev, "can't change firmware while running\n");
+		err = -EBUSY;
+		goto out;
+	}
+
+	len = strcspn(buf, "\n");
+
+	p = kstrndup(buf, len, GFP_KERNEL);
+	if (!p) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	kfree(rproc->firmware);
+	rproc->firmware = p;
+out:
+	mutex_unlock(&rproc->lock);
+
+	return err ? err : count;
+}
+static DEVICE_ATTR_RW(firmware);
+
+/*
+ * A state-to-string lookup table, for exposing a human readable state
+ * via sysfs. Always keep in sync with enum rproc_state
+ */
+static const char * const rproc_state_string[] = {
+	[RPROC_OFFLINE]		= "offline",
+	[RPROC_SUSPENDED]	= "suspended",
+	[RPROC_RUNNING]		= "running",
+	[RPROC_CRASHED]		= "crashed",
+	[RPROC_LAST]		= "invalid",
+};
+
+/* Expose the state of the remote processor via sysfs */
+static ssize_t state_show(struct device *dev, struct device_attribute *attr,
+			  char *buf)
+{
+	struct rproc *rproc = to_rproc(dev);
+	unsigned int state;
+
+	state = rproc->state > RPROC_LAST ? RPROC_LAST : rproc->state;
+	return sprintf(buf, "%s\n", rproc_state_string[state]);
+}
+
+/* Change remote processor state via sysfs */
+static ssize_t state_store(struct device *dev,
+			      struct device_attribute *attr,
+			      const char *buf, size_t count)
+{
+	struct rproc *rproc = to_rproc(dev);
+	int ret = 0;
+
+	if (sysfs_streq(buf, "start")) {
+		if (rproc->state == RPROC_RUNNING)
+			return -EBUSY;
+
+		ret = rproc_boot(rproc);
+		if (ret)
+			dev_err(&rproc->dev, "Boot failed: %d\n", ret);
+	} else if (sysfs_streq(buf, "stop")) {
+		if (rproc->state != RPROC_RUNNING)
+			return -EINVAL;
+
+		rproc_shutdown(rproc);
+	} else {
+		dev_err(&rproc->dev, "Unrecognised option: %s\n", buf);
+		ret = -EINVAL;
+	}
+	return ret ? ret : count;
+}
+static DEVICE_ATTR_RW(state);
+
+static struct attribute *rproc_attrs[] = {
+	&dev_attr_firmware.attr,
+	&dev_attr_state.attr,
+	NULL
+};
+
+static const struct attribute_group rproc_devgroup = {
+	.attrs = rproc_attrs
+};
+
+static const struct attribute_group *rproc_devgroups[] = {
+	&rproc_devgroup,
+	NULL
+};
+
+struct class rproc_class = {
+	.name		= "remoteproc",
+	.dev_groups	= rproc_devgroups,
+};
+
+int __init rproc_init_sysfs(void)
+{
+	/* create remoteproc device class for sysfs */
+	int err = class_register(&rproc_class);
+
+	if (err)
+		pr_err("remoteproc: unable to register class\n");
+	return err;
+}
+
+void __exit rproc_exit_sysfs(void)
+{
+	class_unregister(&rproc_class);
+}
diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c
index 01870a16d6d2..364411fb7734 100644
--- a/drivers/remoteproc/remoteproc_virtio.c
+++ b/drivers/remoteproc/remoteproc_virtio.c
@@ -79,7 +79,7 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev,
 	struct rproc_vring *rvring;
 	struct virtqueue *vq;
 	void *addr;
-	int len, size, ret;
+	int len, size;
 
 	/* we're temporarily limited to two virtqueues per rvdev */
 	if (id >= ARRAY_SIZE(rvdev->vring))
@@ -88,10 +88,6 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev,
 	if (!name)
 		return NULL;
 
-	ret = rproc_alloc_vring(rvdev, id);
-	if (ret)
-		return ERR_PTR(ret);
-
 	rvring = &rvdev->vring[id];
 	addr = rvring->va;
 	len = rvring->len;
@@ -130,7 +126,6 @@ static void __rproc_virtio_del_vqs(struct virtio_device *vdev)
 		rvring = vq->priv;
 		rvring->vq = NULL;
 		vring_del_virtqueue(vq);
-		rproc_free_vring(rvring);
 	}
 }
 
@@ -282,14 +277,13 @@ static const struct virtio_config_ops rproc_virtio_config_ops = {
  * Never call this function directly; it will be called by the driver
  * core when needed.
  */
-static void rproc_vdev_release(struct device *dev)
+static void rproc_virtio_dev_release(struct device *dev)
 {
 	struct virtio_device *vdev = dev_to_virtio(dev);
 	struct rproc_vdev *rvdev = vdev_to_rvdev(vdev);
 	struct rproc *rproc = vdev_to_rproc(vdev);
 
-	list_del(&rvdev->node);
-	kfree(rvdev);
+	kref_put(&rvdev->refcount, rproc_vdev_release);
 
 	put_device(&rproc->dev);
 }
@@ -313,7 +307,7 @@ int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id)
 	vdev->id.device	= id,
 	vdev->config = &rproc_virtio_config_ops,
 	vdev->dev.parent = dev;
-	vdev->dev.release = rproc_vdev_release;
+	vdev->dev.release = rproc_virtio_dev_release;
 
 	/*
 	 * We're indirectly making a non-temporary copy of the rproc pointer
@@ -325,6 +319,9 @@ int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id)
 	 */
 	get_device(&rproc->dev);
 
+	/* Reference the vdev and vring allocations */
+	kref_get(&rvdev->refcount);
+
 	ret = register_virtio_device(vdev);
 	if (ret) {
 		put_device(&rproc->dev);
diff --git a/drivers/remoteproc/st_remoteproc.c b/drivers/remoteproc/st_remoteproc.c
index ae8963fcc8c8..da4e152e9733 100644
--- a/drivers/remoteproc/st_remoteproc.c
+++ b/drivers/remoteproc/st_remoteproc.c
@@ -245,8 +245,10 @@ static int st_rproc_probe(struct platform_device *pdev)
 		goto free_rproc;
 
 	enabled = st_rproc_state(pdev);
-	if (enabled < 0)
+	if (enabled < 0) {
+		ret = enabled;
 		goto free_rproc;
+	}
 
 	if (enabled) {
 		atomic_inc(&rproc->power);
diff --git a/drivers/remoteproc/st_slim_rproc.c b/drivers/remoteproc/st_slim_rproc.c
new file mode 100644
index 000000000000..507716c8721f
--- /dev/null
+++ b/drivers/remoteproc/st_slim_rproc.c
@@ -0,0 +1,364 @@
+/*
+ * SLIM core rproc driver
+ *
+ * Copyright (C) 2016 STMicroelectronics
+ *
+ * Author: Peter Griffin <peter.griffin@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/remoteproc.h>
+#include <linux/remoteproc/st_slim_rproc.h>
+#include "remoteproc_internal.h"
+
+/* SLIM core registers */
+#define SLIM_ID_OFST		0x0
+#define SLIM_VER_OFST		0x4
+
+#define SLIM_EN_OFST		0x8
+#define SLIM_EN_RUN			BIT(0)
+
+#define SLIM_CLK_GATE_OFST	0xC
+#define SLIM_CLK_GATE_DIS		BIT(0)
+#define SLIM_CLK_GATE_RESET		BIT(2)
+
+#define SLIM_SLIM_PC_OFST	0x20
+
+/* DMEM registers */
+#define SLIM_REV_ID_OFST	0x0
+#define SLIM_REV_ID_MIN_MASK		GENMASK(15, 8)
+#define SLIM_REV_ID_MIN(id)		((id & SLIM_REV_ID_MIN_MASK) >> 8)
+#define SLIM_REV_ID_MAJ_MASK		GENMASK(23, 16)
+#define SLIM_REV_ID_MAJ(id)		((id & SLIM_REV_ID_MAJ_MASK) >> 16)
+
+
+/* peripherals registers */
+#define SLIM_STBUS_SYNC_OFST	0xF88
+#define SLIM_STBUS_SYNC_DIS		BIT(0)
+
+#define SLIM_INT_SET_OFST	0xFD4
+#define SLIM_INT_CLR_OFST	0xFD8
+#define SLIM_INT_MASK_OFST	0xFDC
+
+#define SLIM_CMD_CLR_OFST	0xFC8
+#define SLIM_CMD_MASK_OFST	0xFCC
+
+static const char *mem_names[ST_SLIM_MEM_MAX] = {
+	[ST_SLIM_DMEM]	= "dmem",
+	[ST_SLIM_IMEM]	= "imem",
+};
+
+static int slim_clk_get(struct st_slim_rproc *slim_rproc, struct device *dev)
+{
+	int clk, err;
+
+	for (clk = 0; clk < ST_SLIM_MAX_CLK; clk++) {
+		slim_rproc->clks[clk] = of_clk_get(dev->of_node, clk);
+		if (IS_ERR(slim_rproc->clks[clk])) {
+			err = PTR_ERR(slim_rproc->clks[clk]);
+			if (err == -EPROBE_DEFER)
+				goto err_put_clks;
+			slim_rproc->clks[clk] = NULL;
+			break;
+		}
+	}
+
+	return 0;
+
+err_put_clks:
+	while (--clk >= 0)
+		clk_put(slim_rproc->clks[clk]);
+
+	return err;
+}
+
+static void slim_clk_disable(struct st_slim_rproc *slim_rproc)
+{
+	int clk;
+
+	for (clk = 0; clk < ST_SLIM_MAX_CLK && slim_rproc->clks[clk]; clk++)
+		clk_disable_unprepare(slim_rproc->clks[clk]);
+}
+
+static int slim_clk_enable(struct st_slim_rproc *slim_rproc)
+{
+	int clk, ret;
+
+	for (clk = 0; clk < ST_SLIM_MAX_CLK && slim_rproc->clks[clk]; clk++) {
+		ret = clk_prepare_enable(slim_rproc->clks[clk]);
+		if (ret)
+			goto err_disable_clks;
+	}
+
+	return 0;
+
+err_disable_clks:
+	while (--clk >= 0)
+		clk_disable_unprepare(slim_rproc->clks[clk]);
+
+	return ret;
+}
+
+/*
+ * Remoteproc slim specific device handlers
+ */
+static int slim_rproc_start(struct rproc *rproc)
+{
+	struct device *dev = &rproc->dev;
+	struct st_slim_rproc *slim_rproc = rproc->priv;
+	unsigned long hw_id, hw_ver, fw_rev;
+	u32 val;
+
+	/* disable CPU pipeline clock & reset CPU pipeline */
+	val = SLIM_CLK_GATE_DIS | SLIM_CLK_GATE_RESET;
+	writel(val, slim_rproc->slimcore + SLIM_CLK_GATE_OFST);
+
+	/* disable SLIM core STBus sync */
+	writel(SLIM_STBUS_SYNC_DIS, slim_rproc->peri + SLIM_STBUS_SYNC_OFST);
+
+	/* enable cpu pipeline clock */
+	writel(!SLIM_CLK_GATE_DIS,
+		slim_rproc->slimcore + SLIM_CLK_GATE_OFST);
+
+	/* clear int & cmd mailbox */
+	writel(~0U, slim_rproc->peri + SLIM_INT_CLR_OFST);
+	writel(~0U, slim_rproc->peri + SLIM_CMD_CLR_OFST);
+
+	/* enable all channels cmd & int */
+	writel(~0U, slim_rproc->peri + SLIM_INT_MASK_OFST);
+	writel(~0U, slim_rproc->peri + SLIM_CMD_MASK_OFST);
+
+	/* enable cpu */
+	writel(SLIM_EN_RUN, slim_rproc->slimcore + SLIM_EN_OFST);
+
+	hw_id = readl_relaxed(slim_rproc->slimcore + SLIM_ID_OFST);
+	hw_ver = readl_relaxed(slim_rproc->slimcore + SLIM_VER_OFST);
+
+	fw_rev = readl(slim_rproc->mem[ST_SLIM_DMEM].cpu_addr +
+			SLIM_REV_ID_OFST);
+
+	dev_info(dev, "fw rev:%ld.%ld on SLIM %ld.%ld\n",
+		 SLIM_REV_ID_MAJ(fw_rev), SLIM_REV_ID_MIN(fw_rev),
+		 hw_id, hw_ver);
+
+	return 0;
+}
+
+static int slim_rproc_stop(struct rproc *rproc)
+{
+	struct st_slim_rproc *slim_rproc = rproc->priv;
+	u32 val;
+
+	/* mask all (cmd & int) channels */
+	writel(0UL, slim_rproc->peri + SLIM_INT_MASK_OFST);
+	writel(0UL, slim_rproc->peri + SLIM_CMD_MASK_OFST);
+
+	/* disable cpu pipeline clock */
+	writel(SLIM_CLK_GATE_DIS, slim_rproc->slimcore + SLIM_CLK_GATE_OFST);
+
+	writel(!SLIM_EN_RUN, slim_rproc->slimcore + SLIM_EN_OFST);
+
+	val = readl(slim_rproc->slimcore + SLIM_EN_OFST);
+	if (val & SLIM_EN_RUN)
+		dev_warn(&rproc->dev, "Failed to disable SLIM");
+
+	dev_dbg(&rproc->dev, "slim stopped\n");
+
+	return 0;
+}
+
+static void *slim_rproc_da_to_va(struct rproc *rproc, u64 da, int len)
+{
+	struct st_slim_rproc *slim_rproc = rproc->priv;
+	void *va = NULL;
+	int i;
+
+	for (i = 0; i < ST_SLIM_MEM_MAX; i++) {
+		if (da != slim_rproc->mem[i].bus_addr)
+			continue;
+
+		if (len <= slim_rproc->mem[i].size) {
+			/* __force to make sparse happy with type conversion */
+			va = (__force void *)slim_rproc->mem[i].cpu_addr;
+			break;
+		}
+	}
+
+	dev_dbg(&rproc->dev, "da = 0x%llx len = 0x%x va = 0x%p\n", da, len, va);
+
+	return va;
+}
+
+static struct rproc_ops slim_rproc_ops = {
+	.start		= slim_rproc_start,
+	.stop		= slim_rproc_stop,
+	.da_to_va       = slim_rproc_da_to_va,
+};
+
+/*
+ * Firmware handler operations: sanity, boot address, load ...
+ */
+
+static struct resource_table empty_rsc_tbl = {
+	.ver = 1,
+	.num = 0,
+};
+
+static struct resource_table *slim_rproc_find_rsc_table(struct rproc *rproc,
+					       const struct firmware *fw,
+					       int *tablesz)
+{
+	*tablesz = sizeof(empty_rsc_tbl);
+	return &empty_rsc_tbl;
+}
+
+static struct rproc_fw_ops slim_rproc_fw_ops = {
+	.find_rsc_table = slim_rproc_find_rsc_table,
+};
+
+/**
+ * st_slim_rproc_alloc() - allocate and initialise slim rproc
+ * @pdev: Pointer to the platform_device struct
+ * @fw_name: Name of firmware for rproc to use
+ *
+ * Function for allocating and initialising a slim rproc for use by
+ * device drivers whose IP is based around the SLIM core. It
+ * obtains and enables any clocks required by the SLIM core and also
+ * ioremaps the various IO.
+ *
+ * Returns st_slim_rproc pointer or PTR_ERR() on error.
+ */
+
+struct st_slim_rproc *st_slim_rproc_alloc(struct platform_device *pdev,
+				char *fw_name)
+{
+	struct device *dev = &pdev->dev;
+	struct st_slim_rproc *slim_rproc;
+	struct device_node *np = dev->of_node;
+	struct rproc *rproc;
+	struct resource *res;
+	int err, i;
+	const struct rproc_fw_ops *elf_ops;
+
+	if (!fw_name)
+		return ERR_PTR(-EINVAL);
+
+	if (!of_device_is_compatible(np, "st,slim-rproc"))
+		return ERR_PTR(-EINVAL);
+
+	rproc = rproc_alloc(dev, np->name, &slim_rproc_ops,
+			fw_name, sizeof(*slim_rproc));
+	if (!rproc)
+		return ERR_PTR(-ENOMEM);
+
+	rproc->has_iommu = false;
+
+	slim_rproc = rproc->priv;
+	slim_rproc->rproc = rproc;
+
+	elf_ops = rproc->fw_ops;
+	/* Use some generic elf ops */
+	slim_rproc_fw_ops.load = elf_ops->load;
+	slim_rproc_fw_ops.sanity_check = elf_ops->sanity_check;
+
+	rproc->fw_ops = &slim_rproc_fw_ops;
+
+	/* get imem and dmem */
+	for (i = 0; i < ARRAY_SIZE(mem_names); i++) {
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						mem_names[i]);
+
+		slim_rproc->mem[i].cpu_addr = devm_ioremap_resource(dev, res);
+		if (IS_ERR(slim_rproc->mem[i].cpu_addr)) {
+			dev_err(&pdev->dev, "devm_ioremap_resource failed\n");
+			err = PTR_ERR(slim_rproc->mem[i].cpu_addr);
+			goto err;
+		}
+		slim_rproc->mem[i].bus_addr = res->start;
+		slim_rproc->mem[i].size = resource_size(res);
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "slimcore");
+	slim_rproc->slimcore = devm_ioremap_resource(dev, res);
+	if (IS_ERR(slim_rproc->slimcore)) {
+		dev_err(&pdev->dev, "failed to ioremap slimcore IO\n");
+		err = PTR_ERR(slim_rproc->slimcore);
+		goto err;
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "peripherals");
+	slim_rproc->peri = devm_ioremap_resource(dev, res);
+	if (IS_ERR(slim_rproc->peri)) {
+		dev_err(&pdev->dev, "failed to ioremap peripherals IO\n");
+		err = PTR_ERR(slim_rproc->peri);
+		goto err;
+	}
+
+	err = slim_clk_get(slim_rproc, dev);
+	if (err)
+		goto err;
+
+	err = slim_clk_enable(slim_rproc);
+	if (err) {
+		dev_err(dev, "Failed to enable clocks\n");
+		goto err_clk_put;
+	}
+
+	/* Register as a remoteproc device */
+	err = rproc_add(rproc);
+	if (err) {
+		dev_err(dev, "registration of slim remoteproc failed\n");
+		goto err_clk_dis;
+	}
+
+	return slim_rproc;
+
+err_clk_dis:
+	slim_clk_disable(slim_rproc);
+err_clk_put:
+	for (i = 0; i < ST_SLIM_MAX_CLK && slim_rproc->clks[i]; i++)
+		clk_put(slim_rproc->clks[i]);
+err:
+	rproc_free(rproc);
+	return ERR_PTR(err);
+}
+EXPORT_SYMBOL(st_slim_rproc_alloc);
+
+/**
+  * st_slim_rproc_put() - put slim rproc resources
+  * @slim_rproc: Pointer to the st_slim_rproc struct
+  *
+  * Function for calling respective _put() functions on slim_rproc resources.
+  *
+  */
+void st_slim_rproc_put(struct st_slim_rproc *slim_rproc)
+{
+	int clk;
+
+	if (!slim_rproc)
+		return;
+
+	slim_clk_disable(slim_rproc);
+
+	for (clk = 0; clk < ST_SLIM_MAX_CLK && slim_rproc->clks[clk]; clk++)
+		clk_put(slim_rproc->clks[clk]);
+
+	rproc_del(slim_rproc->rproc);
+	rproc_free(slim_rproc->rproc);
+}
+EXPORT_SYMBOL(st_slim_rproc_put);
+
+MODULE_AUTHOR("Peter Griffin <peter.griffin@linaro.org>");
+MODULE_DESCRIPTION("STMicroelectronics SLIM core rproc driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/remoteproc/ste_modem_rproc.c b/drivers/remoteproc/ste_modem_rproc.c
deleted file mode 100644
index 03d69a9a3c5b..000000000000
--- a/drivers/remoteproc/ste_modem_rproc.c
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Copyright (C) ST-Ericsson AB 2012
- * Author: Sjur Brændeland <sjur.brandeland@stericsson.com>
- * License terms:  GNU General Public License (GPL), version 2
- */
-
-#include <linux/module.h>
-#include <linux/dma-mapping.h>
-#include <linux/remoteproc.h>
-#include <linux/ste_modem_shm.h>
-#include "remoteproc_internal.h"
-
-#define SPROC_FW_SIZE (50 * 4096)
-#define SPROC_MAX_TOC_ENTRIES 32
-#define SPROC_MAX_NOTIFY_ID 14
-#define SPROC_RESOURCE_NAME "rsc-table"
-#define SPROC_MODEM_NAME "ste-modem"
-#define SPROC_MODEM_FIRMWARE SPROC_MODEM_NAME "-fw.bin"
-
-#define sproc_dbg(sproc, fmt, ...) \
-	dev_dbg(&sproc->mdev->pdev.dev, fmt, ##__VA_ARGS__)
-#define sproc_err(sproc, fmt, ...) \
-	dev_err(&sproc->mdev->pdev.dev, fmt, ##__VA_ARGS__)
-
-/* STE-modem control structure */
-struct sproc {
-	struct rproc *rproc;
-	struct ste_modem_device *mdev;
-	int error;
-	void *fw_addr;
-	size_t fw_size;
-	dma_addr_t fw_dma_addr;
-};
-
-/* STE-Modem firmware entry */
-struct ste_toc_entry {
-	__le32 start;
-	__le32 size;
-	__le32 flags;
-	__le32 entry_point;
-	__le32 load_addr;
-	char name[12];
-};
-
-/*
- * The Table Of Content is located at the start of the firmware image and
- * at offset zero in the shared memory region. The resource table typically
- * contains the initial boot image (boot strap) and other information elements
- * such as remoteproc resource table. Each entry is identified by a unique
- * name.
- */
-struct ste_toc {
-	struct ste_toc_entry table[SPROC_MAX_TOC_ENTRIES];
-};
-
-/* Loads the firmware to shared memory. */
-static int sproc_load_segments(struct rproc *rproc, const struct firmware *fw)
-{
-	struct sproc *sproc = rproc->priv;
-
-	memcpy(sproc->fw_addr, fw->data, fw->size);
-
-	return 0;
-}
-
-/* Find the entry for resource table in the Table of Content */
-static const struct ste_toc_entry *sproc_find_rsc_entry(const void *data)
-{
-	int i;
-	const struct ste_toc *toc = data;
-
-	/* Search the table for the resource table */
-	for (i = 0; i < SPROC_MAX_TOC_ENTRIES &&
-		    toc->table[i].start != 0xffffffff; i++) {
-		if (!strncmp(toc->table[i].name, SPROC_RESOURCE_NAME,
-			     sizeof(toc->table[i].name)))
-			return &toc->table[i];
-	}
-
-	return NULL;
-}
-
-/* Find the resource table inside the remote processor's firmware. */
-static struct resource_table *
-sproc_find_rsc_table(struct rproc *rproc, const struct firmware *fw,
-		     int *tablesz)
-{
-	struct sproc *sproc = rproc->priv;
-	struct resource_table *table;
-	const struct ste_toc_entry *entry;
-
-	if (!fw)
-		return NULL;
-
-	entry = sproc_find_rsc_entry(fw->data);
-	if (!entry) {
-		sproc_err(sproc, "resource table not found in fw\n");
-		return NULL;
-	}
-
-	table = (void *)(fw->data + entry->start);
-
-	/* sanity check size and offset of resource table */
-	if (entry->start > SPROC_FW_SIZE ||
-	    entry->size > SPROC_FW_SIZE ||
-	    fw->size > SPROC_FW_SIZE ||
-	    entry->start + entry->size > fw->size ||
-	    sizeof(struct resource_table) > entry->size) {
-		sproc_err(sproc, "bad size of fw or resource table\n");
-		return NULL;
-	}
-
-	/* we don't support any version beyond the first */
-	if (table->ver != 1) {
-		sproc_err(sproc, "unsupported fw ver: %d\n", table->ver);
-		return NULL;
-	}
-
-	/* make sure reserved bytes are zeroes */
-	if (table->reserved[0] || table->reserved[1]) {
-		sproc_err(sproc, "non zero reserved bytes\n");
-		return NULL;
-	}
-
-	/* make sure the offsets array isn't truncated */
-	if (table->num > SPROC_MAX_TOC_ENTRIES ||
-	    table->num * sizeof(table->offset[0]) +
-	    sizeof(struct resource_table) > entry->size) {
-		sproc_err(sproc, "resource table incomplete\n");
-		return NULL;
-	}
-
-	/* If the fw size has grown, release the previous fw allocation */
-	if (SPROC_FW_SIZE < fw->size) {
-		sproc_err(sproc, "Insufficient space for fw (%d < %zd)\n",
-			  SPROC_FW_SIZE, fw->size);
-		return NULL;
-	}
-
-	sproc->fw_size = fw->size;
-	*tablesz = entry->size;
-
-	return table;
-}
-
-/* Find the resource table inside the remote processor's firmware. */
-static struct resource_table *
-sproc_find_loaded_rsc_table(struct rproc *rproc, const struct firmware *fw)
-{
-	struct sproc *sproc = rproc->priv;
-	const struct ste_toc_entry *entry;
-
-	if (!fw || !sproc->fw_addr)
-		return NULL;
-
-	entry = sproc_find_rsc_entry(sproc->fw_addr);
-	if (!entry) {
-		sproc_err(sproc, "resource table not found in fw\n");
-		return NULL;
-	}
-
-	return sproc->fw_addr + entry->start;
-}
-
-/* STE modem firmware handler operations */
-static const struct rproc_fw_ops sproc_fw_ops = {
-	.load = sproc_load_segments,
-	.find_rsc_table = sproc_find_rsc_table,
-	.find_loaded_rsc_table = sproc_find_loaded_rsc_table,
-};
-
-/* Kick the modem with specified notification id */
-static void sproc_kick(struct rproc *rproc, int vqid)
-{
-	struct sproc *sproc = rproc->priv;
-
-	sproc_dbg(sproc, "kick vqid:%d\n", vqid);
-
-	/*
-	 * We need different notification IDs for RX and TX so add
-	 * an offset on TX notification IDs.
-	 */
-	sproc->mdev->ops.kick(sproc->mdev, vqid + SPROC_MAX_NOTIFY_ID);
-}
-
-/* Received a kick from a modem, kick the virtqueue */
-static void sproc_kick_callback(struct ste_modem_device *mdev, int vqid)
-{
-	struct sproc *sproc = mdev->drv_data;
-
-	if (rproc_vq_interrupt(sproc->rproc, vqid) == IRQ_NONE)
-		sproc_dbg(sproc, "no message was found in vqid %d\n", vqid);
-}
-
-static struct ste_modem_dev_cb sproc_dev_cb = {
-	.kick = sproc_kick_callback,
-};
-
-/* Start the STE modem */
-static int sproc_start(struct rproc *rproc)
-{
-	struct sproc *sproc = rproc->priv;
-	int i, err;
-
-	sproc_dbg(sproc, "start ste-modem\n");
-
-	/* Sanity test the max_notifyid */
-	if (rproc->max_notifyid > SPROC_MAX_NOTIFY_ID) {
-		sproc_err(sproc, "Notification IDs too high:%d\n",
-			  rproc->max_notifyid);
-		return -EINVAL;
-	}
-
-	/* Subscribe to notifications */
-	for (i = 0; i <= rproc->max_notifyid; i++) {
-		err = sproc->mdev->ops.kick_subscribe(sproc->mdev, i);
-		if (err) {
-			sproc_err(sproc,
-				  "subscription of kicks failed:%d\n", err);
-			return err;
-		}
-	}
-
-	/* Request modem start-up*/
-	return sproc->mdev->ops.power(sproc->mdev, true);
-}
-
-/* Stop the STE modem */
-static int sproc_stop(struct rproc *rproc)
-{
-	struct sproc *sproc = rproc->priv;
-
-	sproc_dbg(sproc, "stop ste-modem\n");
-
-	return sproc->mdev->ops.power(sproc->mdev, false);
-}
-
-static struct rproc_ops sproc_ops = {
-	.start		= sproc_start,
-	.stop		= sproc_stop,
-	.kick		= sproc_kick,
-};
-
-/* STE modem device is unregistered */
-static int sproc_drv_remove(struct platform_device *pdev)
-{
-	struct ste_modem_device *mdev =
-		container_of(pdev, struct ste_modem_device, pdev);
-	struct sproc *sproc = mdev->drv_data;
-
-	sproc_dbg(sproc, "remove ste-modem\n");
-
-	/* Reset device callback functions */
-	sproc->mdev->ops.setup(sproc->mdev, NULL);
-
-	/* Unregister as remoteproc device */
-	rproc_del(sproc->rproc);
-	dma_free_coherent(sproc->rproc->dev.parent, SPROC_FW_SIZE,
-			  sproc->fw_addr, sproc->fw_dma_addr);
-	rproc_free(sproc->rproc);
-
-	mdev->drv_data = NULL;
-
-	return 0;
-}
-
-/* Handle probe of a modem device */
-static int sproc_probe(struct platform_device *pdev)
-{
-	struct ste_modem_device *mdev =
-		container_of(pdev, struct ste_modem_device, pdev);
-	struct sproc *sproc;
-	struct rproc *rproc;
-	int err;
-
-	dev_dbg(&mdev->pdev.dev, "probe ste-modem\n");
-
-	if (!mdev->ops.setup || !mdev->ops.kick || !mdev->ops.kick_subscribe ||
-	    !mdev->ops.power) {
-		dev_err(&mdev->pdev.dev, "invalid mdev ops\n");
-		return -EINVAL;
-	}
-
-	rproc = rproc_alloc(&mdev->pdev.dev, mdev->pdev.name, &sproc_ops,
-			    SPROC_MODEM_FIRMWARE, sizeof(*sproc));
-	if (!rproc)
-		return -ENOMEM;
-
-	sproc = rproc->priv;
-	sproc->mdev = mdev;
-	sproc->rproc = rproc;
-	rproc->has_iommu = false;
-	mdev->drv_data = sproc;
-
-	/* Provide callback functions to modem device */
-	sproc->mdev->ops.setup(sproc->mdev, &sproc_dev_cb);
-
-	/* Set the STE-modem specific firmware handler */
-	rproc->fw_ops = &sproc_fw_ops;
-
-	/*
-	 * STE-modem requires the firmware to be located
-	 * at the start of the shared memory region. So we need to
-	 * reserve space for firmware at the start.
-	 */
-	sproc->fw_addr = dma_alloc_coherent(rproc->dev.parent, SPROC_FW_SIZE,
-					    &sproc->fw_dma_addr,
-					    GFP_KERNEL);
-	if (!sproc->fw_addr) {
-		sproc_err(sproc, "Cannot allocate memory for fw\n");
-		err = -ENOMEM;
-		goto free_rproc;
-	}
-
-	/* Register as a remoteproc device */
-	err = rproc_add(rproc);
-	if (err)
-		goto free_mem;
-
-	return 0;
-
-free_mem:
-	dma_free_coherent(rproc->dev.parent, SPROC_FW_SIZE,
-			  sproc->fw_addr, sproc->fw_dma_addr);
-free_rproc:
-	/* Reset device data upon error */
-	mdev->drv_data = NULL;
-	rproc_free(rproc);
-	return err;
-}
-
-static struct platform_driver sproc_driver = {
-	.driver	= {
-		.name	= SPROC_MODEM_NAME,
-	},
-	.probe	= sproc_probe,
-	.remove	= sproc_drv_remove,
-};
-
-module_platform_driver(sproc_driver);
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("STE Modem driver using the Remote Processor Framework");
diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig
index 06d9fa2f3bc0..172dc966a01f 100644
--- a/drivers/reset/Kconfig
+++ b/drivers/reset/Kconfig
@@ -94,5 +94,6 @@ config RESET_ZYNQ
 
 source "drivers/reset/sti/Kconfig"
 source "drivers/reset/hisilicon/Kconfig"
+source "drivers/reset/tegra/Kconfig"
 
 endif
diff --git a/drivers/reset/Makefile b/drivers/reset/Makefile
index bbe7026617fc..13b346e03d84 100644
--- a/drivers/reset/Makefile
+++ b/drivers/reset/Makefile
@@ -1,6 +1,7 @@
 obj-y += core.o
 obj-y += hisilicon/
 obj-$(CONFIG_ARCH_STI) += sti/
+obj-$(CONFIG_ARCH_TEGRA) += tegra/
 obj-$(CONFIG_RESET_ATH79) += reset-ath79.o
 obj-$(CONFIG_RESET_BERLIN) += reset-berlin.o
 obj-$(CONFIG_RESET_LPC18XX) += reset-lpc18xx.o
diff --git a/drivers/reset/core.c b/drivers/reset/core.c
index b8ae1dbd4c17..10368ed8fd13 100644
--- a/drivers/reset/core.c
+++ b/drivers/reset/core.c
@@ -32,6 +32,9 @@ static LIST_HEAD(reset_controller_list);
  * @refcnt: Number of gets of this reset_control
  * @shared: Is this a shared (1), or an exclusive (0) reset_control?
  * @deassert_cnt: Number of times this reset line has been deasserted
+ * @triggered_count: Number of times this reset line has been reset. Currently
+ *                   only used for shared resets, which means that the value
+ *                   will be either 0 or 1.
  */
 struct reset_control {
 	struct reset_controller_dev *rcdev;
@@ -40,6 +43,7 @@ struct reset_control {
 	unsigned int refcnt;
 	int shared;
 	atomic_t deassert_count;
+	atomic_t triggered_count;
 };
 
 /**
@@ -134,18 +138,35 @@ EXPORT_SYMBOL_GPL(devm_reset_controller_register);
  * reset_control_reset - reset the controlled device
  * @rstc: reset controller
  *
- * Calling this on a shared reset controller is an error.
+ * On a shared reset line the actual reset pulse is only triggered once for the
+ * lifetime of the reset_control instance: for all but the first caller this is
+ * a no-op.
+ * Consumers must not use reset_control_(de)assert on shared reset lines when
+ * reset_control_reset has been used.
  */
 int reset_control_reset(struct reset_control *rstc)
 {
-	if (WARN_ON(IS_ERR_OR_NULL(rstc)) ||
-	    WARN_ON(rstc->shared))
+	int ret;
+
+	if (WARN_ON(IS_ERR_OR_NULL(rstc)))
 		return -EINVAL;
 
-	if (rstc->rcdev->ops->reset)
-		return rstc->rcdev->ops->reset(rstc->rcdev, rstc->id);
+	if (!rstc->rcdev->ops->reset)
+		return -ENOTSUPP;
 
-	return -ENOTSUPP;
+	if (rstc->shared) {
+		if (WARN_ON(atomic_read(&rstc->deassert_count) != 0))
+			return -EINVAL;
+
+		if (atomic_inc_return(&rstc->triggered_count) != 1)
+			return 0;
+	}
+
+	ret = rstc->rcdev->ops->reset(rstc->rcdev, rstc->id);
+	if (rstc->shared && !ret)
+		atomic_dec(&rstc->triggered_count);
+
+	return ret;
 }
 EXPORT_SYMBOL_GPL(reset_control_reset);
 
@@ -159,6 +180,8 @@ EXPORT_SYMBOL_GPL(reset_control_reset);
  *
  * For shared reset controls a driver cannot expect the hw's registers and
  * internal state to be reset, but must be prepared for this to happen.
+ * Consumers must not use reset_control_reset on shared reset lines when
+ * reset_control_(de)assert has been used.
  */
 int reset_control_assert(struct reset_control *rstc)
 {
@@ -169,6 +192,9 @@ int reset_control_assert(struct reset_control *rstc)
 		return -ENOTSUPP;
 
 	if (rstc->shared) {
+		if (WARN_ON(atomic_read(&rstc->triggered_count) != 0))
+			return -EINVAL;
+
 		if (WARN_ON(atomic_read(&rstc->deassert_count) == 0))
 			return -EINVAL;
 
@@ -185,6 +211,8 @@ EXPORT_SYMBOL_GPL(reset_control_assert);
  * @rstc: reset controller
  *
  * After calling this function, the reset is guaranteed to be deasserted.
+ * Consumers must not use reset_control_reset on shared reset lines when
+ * reset_control_(de)assert has been used.
  */
 int reset_control_deassert(struct reset_control *rstc)
 {
@@ -195,6 +223,9 @@ int reset_control_deassert(struct reset_control *rstc)
 		return -ENOTSUPP;
 
 	if (rstc->shared) {
+		if (WARN_ON(atomic_read(&rstc->triggered_count) != 0))
+			return -EINVAL;
+
 		if (atomic_inc_return(&rstc->deassert_count) != 1)
 			return 0;
 	}
diff --git a/drivers/reset/reset-berlin.c b/drivers/reset/reset-berlin.c
index 369f3917fd8e..371197bbd055 100644
--- a/drivers/reset/reset-berlin.c
+++ b/drivers/reset/reset-berlin.c
@@ -1,6 +1,8 @@
 /*
  * Copyright (C) 2014 Marvell Technology Group Ltd.
  *
+ * Marvell Berlin reset driver
+ *
  * Antoine Tenart <antoine.tenart@free-electrons.com>
  * Sebastian Hesselbarth <sebastian.hesselbarth@gmail.com>
  *
@@ -12,7 +14,7 @@
 #include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/mfd/syscon.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
@@ -91,7 +93,6 @@ static const struct of_device_id berlin_reset_dt_match[] = {
 	{ .compatible = "marvell,berlin2-reset" },
 	{ },
 };
-MODULE_DEVICE_TABLE(of, berlin_reset_dt_match);
 
 static struct platform_driver berlin_reset_driver = {
 	.probe	= berlin2_reset_probe,
@@ -100,9 +101,4 @@ static struct platform_driver berlin_reset_driver = {
 		.of_match_table = berlin_reset_dt_match,
 	},
 };
-module_platform_driver(berlin_reset_driver);
-
-MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
-MODULE_AUTHOR("Sebastian Hesselbarth <sebastian.hesselbarth@gmail.com>");
-MODULE_DESCRIPTION("Marvell Berlin reset driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(berlin_reset_driver);
diff --git a/drivers/reset/reset-lpc18xx.c b/drivers/reset/reset-lpc18xx.c
index 54cca0055171..a62ad52e262b 100644
--- a/drivers/reset/reset-lpc18xx.c
+++ b/drivers/reset/reset-lpc18xx.c
@@ -13,7 +13,7 @@
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/io.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/reboot.h>
@@ -218,39 +218,17 @@ dis_clk_reg:
 	return ret;
 }
 
-static int lpc18xx_rgu_remove(struct platform_device *pdev)
-{
-	struct lpc18xx_rgu_data *rc = platform_get_drvdata(pdev);
-	int ret;
-
-	ret = unregister_restart_handler(&rc->restart_nb);
-	if (ret)
-		dev_warn(&pdev->dev, "failed to unregister restart handler\n");
-
-	reset_controller_unregister(&rc->rcdev);
-
-	clk_disable_unprepare(rc->clk_delay);
-	clk_disable_unprepare(rc->clk_reg);
-
-	return 0;
-}
-
 static const struct of_device_id lpc18xx_rgu_match[] = {
 	{ .compatible = "nxp,lpc1850-rgu" },
 	{ }
 };
-MODULE_DEVICE_TABLE(of, lpc18xx_rgu_match);
 
 static struct platform_driver lpc18xx_rgu_driver = {
 	.probe	= lpc18xx_rgu_probe,
-	.remove	= lpc18xx_rgu_remove,
 	.driver	= {
-		.name		= "lpc18xx-reset",
-		.of_match_table	= lpc18xx_rgu_match,
+		.name			= "lpc18xx-reset",
+		.of_match_table		= lpc18xx_rgu_match,
+		.suppress_bind_attrs	= true,
 	},
 };
-module_platform_driver(lpc18xx_rgu_driver);
-
-MODULE_AUTHOR("Joachim Eastwood <manabian@gmail.com>");
-MODULE_DESCRIPTION("Reset driver for LPC18xx/43xx RGU");
-MODULE_LICENSE("GPL v2");
+builtin_platform_driver(lpc18xx_rgu_driver);
diff --git a/drivers/reset/reset-oxnas.c b/drivers/reset/reset-oxnas.c
index 944980572f79..0d9036dea010 100644
--- a/drivers/reset/reset-oxnas.c
+++ b/drivers/reset/reset-oxnas.c
@@ -80,6 +80,7 @@ static const struct reset_control_ops oxnas_reset_ops = {
 
 static const struct of_device_id oxnas_reset_dt_ids[] = {
 	 { .compatible = "oxsemi,ox810se-reset", },
+	 { .compatible = "oxsemi,ox820-reset", },
 	 { /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, oxnas_reset_dt_ids);
diff --git a/drivers/reset/reset-socfpga.c b/drivers/reset/reset-socfpga.c
index 78ebf8424375..43e4a9f39b9b 100644
--- a/drivers/reset/reset-socfpga.c
+++ b/drivers/reset/reset-socfpga.c
@@ -1,4 +1,6 @@
 /*
+ * Socfpga Reset Controller Driver
+ *
  * Copyright 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de>
  *
  * based on
@@ -16,7 +18,7 @@
 
 #include <linux/err.h>
 #include <linux/io.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/reset-controller.h>
@@ -148,8 +150,4 @@ static struct platform_driver socfpga_reset_driver = {
 		.of_match_table	= socfpga_reset_dt_ids,
 	},
 };
-module_platform_driver(socfpga_reset_driver);
-
-MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de");
-MODULE_DESCRIPTION("Socfpga Reset Controller Driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(socfpga_reset_driver);
diff --git a/drivers/reset/reset-sunxi.c b/drivers/reset/reset-sunxi.c
index 3080190b3f90..b44f6b5f87b6 100644
--- a/drivers/reset/reset-sunxi.c
+++ b/drivers/reset/reset-sunxi.c
@@ -13,7 +13,7 @@
 
 #include <linux/err.h>
 #include <linux/io.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
@@ -142,7 +142,6 @@ static const struct of_device_id sunxi_reset_dt_ids[] = {
 	 { .compatible = "allwinner,sun6i-a31-clock-reset", },
 	 { /* sentinel */ },
 };
-MODULE_DEVICE_TABLE(of, sunxi_reset_dt_ids);
 
 static int sunxi_reset_probe(struct platform_device *pdev)
 {
@@ -175,8 +174,4 @@ static struct platform_driver sunxi_reset_driver = {
 		.of_match_table	= sunxi_reset_dt_ids,
 	},
 };
-module_platform_driver(sunxi_reset_driver);
-
-MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com");
-MODULE_DESCRIPTION("Allwinner SoCs Reset Controller Driver");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(sunxi_reset_driver);
diff --git a/drivers/reset/reset-zynq.c b/drivers/reset/reset-zynq.c
index 138f2f205662..87a4e355578f 100644
--- a/drivers/reset/reset-zynq.c
+++ b/drivers/reset/reset-zynq.c
@@ -3,6 +3,8 @@
  *
  * Xilinx Zynq Reset controller driver
  *
+ * Author: Moritz Fischer <moritz.fischer@ettus.com>
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; version 2 of the License.
@@ -15,7 +17,7 @@
 
 #include <linux/err.h>
 #include <linux/io.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/mfd/syscon.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
@@ -137,8 +139,4 @@ static struct platform_driver zynq_reset_driver = {
 		.of_match_table	= zynq_reset_dt_ids,
 	},
 };
-module_platform_driver(zynq_reset_driver);
-
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Moritz Fischer <moritz.fischer@ettus.com>");
-MODULE_DESCRIPTION("Zynq Reset Controller Driver");
+builtin_platform_driver(zynq_reset_driver);
diff --git a/drivers/reset/sti/Kconfig b/drivers/reset/sti/Kconfig
index 613178553612..71592b5bfd14 100644
--- a/drivers/reset/sti/Kconfig
+++ b/drivers/reset/sti/Kconfig
@@ -3,14 +3,6 @@ if ARCH_STI
 config STI_RESET_SYSCFG
 	bool
 
-config STIH415_RESET
-	bool
-	select STI_RESET_SYSCFG
-
-config STIH416_RESET
-	bool
-	select STI_RESET_SYSCFG
-
 config STIH407_RESET
 	bool
 	select STI_RESET_SYSCFG
diff --git a/drivers/reset/sti/Makefile b/drivers/reset/sti/Makefile
index dc85dfbe56a9..f9d82411f29e 100644
--- a/drivers/reset/sti/Makefile
+++ b/drivers/reset/sti/Makefile
@@ -1,5 +1,3 @@
 obj-$(CONFIG_STI_RESET_SYSCFG) += reset-syscfg.o
 
-obj-$(CONFIG_STIH415_RESET) += reset-stih415.o
-obj-$(CONFIG_STIH416_RESET) += reset-stih416.o
 obj-$(CONFIG_STIH407_RESET) += reset-stih407.o
diff --git a/drivers/reset/sti/reset-stih415.c b/drivers/reset/sti/reset-stih415.c
deleted file mode 100644
index 6f220cdbef46..000000000000
--- a/drivers/reset/sti/reset-stih415.c
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics (R&D) Limited
- * Author: Stephen Gallimore <stephen.gallimore@st.com>
- * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_platform.h>
-#include <linux/platform_device.h>
-
-#include <dt-bindings/reset/stih415-resets.h>
-
-#include "reset-syscfg.h"
-
-/*
- * STiH415 Peripheral powerdown definitions.
- */
-static const char stih415_front[] = "st,stih415-front-syscfg";
-static const char stih415_rear[] = "st,stih415-rear-syscfg";
-static const char stih415_sbc[] = "st,stih415-sbc-syscfg";
-static const char stih415_lpm[] = "st,stih415-lpm-syscfg";
-
-#define STIH415_PDN_FRONT(_bit) \
-	_SYSCFG_RST_CH(stih415_front, SYSCFG_114, _bit, SYSSTAT_187, _bit)
-
-#define STIH415_PDN_REAR(_cntl, _stat) \
-	_SYSCFG_RST_CH(stih415_rear, SYSCFG_336, _cntl, SYSSTAT_384, _stat)
-
-#define STIH415_SRST_REAR(_reg, _bit) \
-	_SYSCFG_RST_CH_NO_ACK(stih415_rear, _reg, _bit)
-
-#define STIH415_SRST_SBC(_reg, _bit) \
-	_SYSCFG_RST_CH_NO_ACK(stih415_sbc, _reg, _bit)
-
-#define STIH415_SRST_FRONT(_reg, _bit) \
-	_SYSCFG_RST_CH_NO_ACK(stih415_front, _reg, _bit)
-
-#define STIH415_SRST_LPM(_reg, _bit) \
-	_SYSCFG_RST_CH_NO_ACK(stih415_lpm, _reg, _bit)
-
-#define SYSCFG_114	0x38 /* Powerdown request EMI/NAND/Keyscan */
-#define SYSSTAT_187	0x15c /* Powerdown status EMI/NAND/Keyscan */
-
-#define SYSCFG_336	0x90 /* Powerdown request USB/SATA/PCIe */
-#define SYSSTAT_384	0x150 /* Powerdown status USB/SATA/PCIe */
-
-#define SYSCFG_376	0x130 /* Reset generator 0 control 0 */
-#define SYSCFG_166	0x108 /* Softreset Ethernet 0 */
-#define SYSCFG_31	0x7c /* Softreset Ethernet 1 */
-#define LPM_SYSCFG_1	0x4 /* Softreset IRB */
-
-static const struct syscfg_reset_channel_data stih415_powerdowns[] = {
-	[STIH415_EMISS_POWERDOWN]	= STIH415_PDN_FRONT(0),
-	[STIH415_NAND_POWERDOWN]	= STIH415_PDN_FRONT(1),
-	[STIH415_KEYSCAN_POWERDOWN]	= STIH415_PDN_FRONT(2),
-	[STIH415_USB0_POWERDOWN]	= STIH415_PDN_REAR(0, 0),
-	[STIH415_USB1_POWERDOWN]	= STIH415_PDN_REAR(1, 1),
-	[STIH415_USB2_POWERDOWN]	= STIH415_PDN_REAR(2, 2),
-	[STIH415_SATA0_POWERDOWN]	= STIH415_PDN_REAR(3, 3),
-	[STIH415_SATA1_POWERDOWN]	= STIH415_PDN_REAR(4, 4),
-	[STIH415_PCIE_POWERDOWN]	= STIH415_PDN_REAR(5, 8),
-};
-
-static const struct syscfg_reset_channel_data stih415_softresets[] = {
-	[STIH415_ETH0_SOFTRESET] = STIH415_SRST_FRONT(SYSCFG_166, 0),
-	[STIH415_ETH1_SOFTRESET] = STIH415_SRST_SBC(SYSCFG_31, 0),
-	[STIH415_IRB_SOFTRESET]	 = STIH415_SRST_LPM(LPM_SYSCFG_1, 6),
-	[STIH415_USB0_SOFTRESET] = STIH415_SRST_REAR(SYSCFG_376, 9),
-	[STIH415_USB1_SOFTRESET] = STIH415_SRST_REAR(SYSCFG_376, 10),
-	[STIH415_USB2_SOFTRESET] = STIH415_SRST_REAR(SYSCFG_376, 11),
-	[STIH415_KEYSCAN_SOFTRESET] = STIH415_SRST_LPM(LPM_SYSCFG_1, 8),
-};
-
-static struct syscfg_reset_controller_data stih415_powerdown_controller = {
-	.wait_for_ack = true,
-	.nr_channels = ARRAY_SIZE(stih415_powerdowns),
-	.channels = stih415_powerdowns,
-};
-
-static struct syscfg_reset_controller_data stih415_softreset_controller = {
-	.wait_for_ack = false,
-	.active_low = true,
-	.nr_channels = ARRAY_SIZE(stih415_softresets),
-	.channels = stih415_softresets,
-};
-
-static const struct of_device_id stih415_reset_match[] = {
-	{ .compatible = "st,stih415-powerdown",
-	  .data = &stih415_powerdown_controller, },
-	{ .compatible = "st,stih415-softreset",
-	  .data = &stih415_softreset_controller, },
-	{},
-};
-
-static struct platform_driver stih415_reset_driver = {
-	.probe = syscfg_reset_probe,
-	.driver = {
-		.name = "reset-stih415",
-		.of_match_table = stih415_reset_match,
-	},
-};
-
-static int __init stih415_reset_init(void)
-{
-	return platform_driver_register(&stih415_reset_driver);
-}
-arch_initcall(stih415_reset_init);
diff --git a/drivers/reset/sti/reset-stih416.c b/drivers/reset/sti/reset-stih416.c
deleted file mode 100644
index c581d606ef0f..000000000000
--- a/drivers/reset/sti/reset-stih416.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Copyright (C) 2013 STMicroelectronics (R&D) Limited
- * Author: Stephen Gallimore <stephen.gallimore@st.com>
- * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_platform.h>
-#include <linux/platform_device.h>
-
-#include <dt-bindings/reset/stih416-resets.h>
-
-#include "reset-syscfg.h"
-
-/*
- * STiH416 Peripheral powerdown definitions.
- */
-static const char stih416_front[] = "st,stih416-front-syscfg";
-static const char stih416_rear[] = "st,stih416-rear-syscfg";
-static const char stih416_sbc[] = "st,stih416-sbc-syscfg";
-static const char stih416_lpm[] = "st,stih416-lpm-syscfg";
-static const char stih416_cpu[] = "st,stih416-cpu-syscfg";
-
-#define STIH416_PDN_FRONT(_bit) \
-	_SYSCFG_RST_CH(stih416_front, SYSCFG_1500, _bit, SYSSTAT_1578, _bit)
-
-#define STIH416_PDN_REAR(_cntl, _stat) \
-	_SYSCFG_RST_CH(stih416_rear, SYSCFG_2525, _cntl, SYSSTAT_2583, _stat)
-
-#define SYSCFG_1500	0x7d0 /* Powerdown request EMI/NAND/Keyscan */
-#define SYSSTAT_1578	0x908 /* Powerdown status EMI/NAND/Keyscan */
-
-#define SYSCFG_2525	0x834 /* Powerdown request USB/SATA/PCIe */
-#define SYSSTAT_2583	0x91c /* Powerdown status USB/SATA/PCIe */
-
-#define SYSCFG_2552	0x8A0 /* Reset Generator control 0 */
-#define SYSCFG_1539	0x86c /* Softreset Ethernet 0 */
-#define SYSCFG_510	0x7f8 /* Softreset Ethernet 1 */
-#define LPM_SYSCFG_1	0x4 /* Softreset IRB */
-#define SYSCFG_2553	0x8a4 /* Softreset SATA0/1, PCIE0/1 */
-#define SYSCFG_7563	0x8cc /* MPE softresets 0 */
-#define SYSCFG_7564	0x8d0 /* MPE softresets 1 */
-
-#define STIH416_SRST_CPU(_reg, _bit) \
-	 _SYSCFG_RST_CH_NO_ACK(stih416_cpu, _reg, _bit)
-
-#define STIH416_SRST_FRONT(_reg, _bit) \
-	 _SYSCFG_RST_CH_NO_ACK(stih416_front, _reg, _bit)
-
-#define STIH416_SRST_REAR(_reg, _bit) \
-	 _SYSCFG_RST_CH_NO_ACK(stih416_rear, _reg, _bit)
-
-#define STIH416_SRST_LPM(_reg, _bit) \
-	 _SYSCFG_RST_CH_NO_ACK(stih416_lpm, _reg, _bit)
-
-#define STIH416_SRST_SBC(_reg, _bit) \
-	 _SYSCFG_RST_CH_NO_ACK(stih416_sbc, _reg, _bit)
-
-static const struct syscfg_reset_channel_data stih416_powerdowns[] = {
-	[STIH416_EMISS_POWERDOWN]	= STIH416_PDN_FRONT(0),
-	[STIH416_NAND_POWERDOWN]	= STIH416_PDN_FRONT(1),
-	[STIH416_KEYSCAN_POWERDOWN]	= STIH416_PDN_FRONT(2),
-	[STIH416_USB0_POWERDOWN]	= STIH416_PDN_REAR(0, 0),
-	[STIH416_USB1_POWERDOWN]	= STIH416_PDN_REAR(1, 1),
-	[STIH416_USB2_POWERDOWN]	= STIH416_PDN_REAR(2, 2),
-	[STIH416_USB3_POWERDOWN]	= STIH416_PDN_REAR(6, 5),
-	[STIH416_SATA0_POWERDOWN]	= STIH416_PDN_REAR(3, 3),
-	[STIH416_SATA1_POWERDOWN]	= STIH416_PDN_REAR(4, 4),
-	[STIH416_PCIE0_POWERDOWN]	= STIH416_PDN_REAR(7, 9),
-	[STIH416_PCIE1_POWERDOWN]	= STIH416_PDN_REAR(5, 8),
-};
-
-static const struct syscfg_reset_channel_data stih416_softresets[] = {
-	[STIH416_ETH0_SOFTRESET] = STIH416_SRST_FRONT(SYSCFG_1539, 0),
-	[STIH416_ETH1_SOFTRESET] = STIH416_SRST_SBC(SYSCFG_510, 0),
-	[STIH416_IRB_SOFTRESET]	 = STIH416_SRST_LPM(LPM_SYSCFG_1, 6),
-	[STIH416_USB0_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2552, 9),
-	[STIH416_USB1_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2552, 10),
-	[STIH416_USB2_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2552, 11),
-	[STIH416_USB3_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2552, 28),
-	[STIH416_SATA0_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2553, 7),
-	[STIH416_SATA1_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2553, 3),
-	[STIH416_PCIE0_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2553, 15),
-	[STIH416_PCIE1_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2553, 2),
-	[STIH416_AUD_DAC_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2553, 14),
-	[STIH416_HDTVOUT_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2552, 5),
-	[STIH416_VTAC_M_RX_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2552, 25),
-	[STIH416_VTAC_A_RX_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2552, 26),
-	[STIH416_SYNC_HD_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2553, 5),
-	[STIH416_SYNC_SD_SOFTRESET] = STIH416_SRST_REAR(SYSCFG_2553, 6),
-	[STIH416_BLITTER_SOFTRESET] = STIH416_SRST_CPU(SYSCFG_7563, 10),
-	[STIH416_GPU_SOFTRESET] = STIH416_SRST_CPU(SYSCFG_7563, 11),
-	[STIH416_VTAC_M_TX_SOFTRESET] = STIH416_SRST_CPU(SYSCFG_7563, 18),
-	[STIH416_VTAC_A_TX_SOFTRESET] = STIH416_SRST_CPU(SYSCFG_7563, 19),
-	[STIH416_VTG_AUX_SOFTRESET] = STIH416_SRST_CPU(SYSCFG_7563, 21),
-	[STIH416_JPEG_DEC_SOFTRESET] = STIH416_SRST_CPU(SYSCFG_7563, 23),
-	[STIH416_HVA_SOFTRESET] = STIH416_SRST_CPU(SYSCFG_7564, 2),
-	[STIH416_COMPO_M_SOFTRESET] = STIH416_SRST_CPU(SYSCFG_7564, 3),
-	[STIH416_COMPO_A_SOFTRESET] = STIH416_SRST_CPU(SYSCFG_7564, 4),
-	[STIH416_VP8_DEC_SOFTRESET] = STIH416_SRST_CPU(SYSCFG_7564, 10),
-	[STIH416_VTG_MAIN_SOFTRESET] = STIH416_SRST_CPU(SYSCFG_7564, 16),
-	[STIH416_KEYSCAN_SOFTRESET] = STIH416_SRST_LPM(LPM_SYSCFG_1, 8),
-};
-
-static struct syscfg_reset_controller_data stih416_powerdown_controller = {
-	.wait_for_ack	= true,
-	.nr_channels	= ARRAY_SIZE(stih416_powerdowns),
-	.channels	= stih416_powerdowns,
-};
-
-static struct syscfg_reset_controller_data stih416_softreset_controller = {
-	.wait_for_ack = false,
-	.active_low = true,
-	.nr_channels = ARRAY_SIZE(stih416_softresets),
-	.channels = stih416_softresets,
-};
-
-static const struct of_device_id stih416_reset_match[] = {
-	{ .compatible = "st,stih416-powerdown",
-	  .data = &stih416_powerdown_controller, },
-	{ .compatible = "st,stih416-softreset",
-	  .data = &stih416_softreset_controller, },
-	{},
-};
-
-static struct platform_driver stih416_reset_driver = {
-	.probe = syscfg_reset_probe,
-	.driver = {
-		.name = "reset-stih416",
-		.of_match_table = stih416_reset_match,
-	},
-};
-
-static int __init stih416_reset_init(void)
-{
-	return platform_driver_register(&stih416_reset_driver);
-}
-arch_initcall(stih416_reset_init);
diff --git a/drivers/reset/tegra/Kconfig b/drivers/reset/tegra/Kconfig
new file mode 100644
index 000000000000..d2afa293df7d
--- /dev/null
+++ b/drivers/reset/tegra/Kconfig
@@ -0,0 +1,2 @@
+config RESET_TEGRA_BPMP
+	def_bool TEGRA_BPMP
diff --git a/drivers/reset/tegra/Makefile b/drivers/reset/tegra/Makefile
new file mode 100644
index 000000000000..775243ab7383
--- /dev/null
+++ b/drivers/reset/tegra/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_RESET_TEGRA_BPMP) += reset-bpmp.o
diff --git a/drivers/reset/tegra/reset-bpmp.c b/drivers/reset/tegra/reset-bpmp.c
new file mode 100644
index 000000000000..5daf2ee1a396
--- /dev/null
+++ b/drivers/reset/tegra/reset-bpmp.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2016 NVIDIA Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/reset-controller.h>
+
+#include <soc/tegra/bpmp.h>
+#include <soc/tegra/bpmp-abi.h>
+
+static struct tegra_bpmp *to_tegra_bpmp(struct reset_controller_dev *rstc)
+{
+	return container_of(rstc, struct tegra_bpmp, rstc);
+}
+
+static int tegra_bpmp_reset_common(struct reset_controller_dev *rstc,
+				   enum mrq_reset_commands command,
+				   unsigned int id)
+{
+	struct tegra_bpmp *bpmp = to_tegra_bpmp(rstc);
+	struct mrq_reset_request request;
+	struct tegra_bpmp_message msg;
+
+	memset(&request, 0, sizeof(request));
+	request.cmd = command;
+	request.reset_id = id;
+
+	memset(&msg, 0, sizeof(msg));
+	msg.mrq = MRQ_RESET;
+	msg.tx.data = &request;
+	msg.tx.size = sizeof(request);
+
+	return tegra_bpmp_transfer(bpmp, &msg);
+}
+
+static int tegra_bpmp_reset_module(struct reset_controller_dev *rstc,
+				   unsigned long id)
+{
+	return tegra_bpmp_reset_common(rstc, CMD_RESET_MODULE, id);
+}
+
+static int tegra_bpmp_reset_assert(struct reset_controller_dev *rstc,
+				   unsigned long id)
+{
+	return tegra_bpmp_reset_common(rstc, CMD_RESET_ASSERT, id);
+}
+
+static int tegra_bpmp_reset_deassert(struct reset_controller_dev *rstc,
+				     unsigned long id)
+{
+	return tegra_bpmp_reset_common(rstc, CMD_RESET_DEASSERT, id);
+}
+
+static const struct reset_control_ops tegra_bpmp_reset_ops = {
+	.reset = tegra_bpmp_reset_module,
+	.assert = tegra_bpmp_reset_assert,
+	.deassert = tegra_bpmp_reset_deassert,
+};
+
+int tegra_bpmp_init_resets(struct tegra_bpmp *bpmp)
+{
+	bpmp->rstc.ops = &tegra_bpmp_reset_ops;
+	bpmp->rstc.owner = THIS_MODULE;
+	bpmp->rstc.of_node = bpmp->dev->of_node;
+	bpmp->rstc.nr_resets = bpmp->soc->num_resets;
+
+	return devm_reset_controller_register(bpmp->dev, &bpmp->rstc);
+}
diff --git a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c
index 06fef2b4c814..0fae48116a0d 100644
--- a/drivers/rpmsg/qcom_smd.c
+++ b/drivers/rpmsg/qcom_smd.c
@@ -25,6 +25,7 @@
 #include <linux/soc/qcom/smem.h>
 #include <linux/wait.h>
 #include <linux/rpmsg.h>
+#include <linux/rpmsg/qcom_smd.h>
 
 #include "rpmsg_internal.h"
 
@@ -739,7 +740,7 @@ static int __qcom_smd_send(struct qcom_smd_channel *channel, const void *data,
 
 	while (qcom_smd_get_tx_avail(channel) < tlen) {
 		if (!wait) {
-			ret = -ENOMEM;
+			ret = -EAGAIN;
 			goto out;
 		}
 
@@ -820,20 +821,13 @@ qcom_smd_find_channel(struct qcom_smd_edge *edge, const char *name)
 	struct qcom_smd_channel *channel;
 	struct qcom_smd_channel *ret = NULL;
 	unsigned long flags;
-	unsigned state;
 
 	spin_lock_irqsave(&edge->channels_lock, flags);
 	list_for_each_entry(channel, &edge->channels, list) {
-		if (strcmp(channel->name, name))
-			continue;
-
-		state = GET_RX_CHANNEL_INFO(channel, state);
-		if (state != SMD_CHANNEL_OPENING &&
-		    state != SMD_CHANNEL_OPENED)
-			continue;
-
-		ret = channel;
-		break;
+		if (!strcmp(channel->name, name)) {
+			ret = channel;
+			break;
+		}
 	}
 	spin_unlock_irqrestore(&edge->channels_lock, flags);
 
diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c
index b6ea9ffa7381..a79cb5a9e5f2 100644
--- a/drivers/rpmsg/rpmsg_core.c
+++ b/drivers/rpmsg/rpmsg_core.c
@@ -71,6 +71,9 @@ struct rpmsg_endpoint *rpmsg_create_ept(struct rpmsg_device *rpdev,
 					rpmsg_rx_cb_t cb, void *priv,
 					struct rpmsg_channel_info chinfo)
 {
+	if (WARN_ON(!rpdev))
+		return ERR_PTR(-EINVAL);
+
 	return rpdev->ops->create_ept(rpdev, cb, priv, chinfo);
 }
 EXPORT_SYMBOL(rpmsg_create_ept);
@@ -80,11 +83,13 @@ EXPORT_SYMBOL(rpmsg_create_ept);
  * @ept: endpoing to destroy
  *
  * Should be used by drivers to destroy an rpmsg endpoint previously
- * created with rpmsg_create_ept().
+ * created with rpmsg_create_ept(). As with other types of "free" NULL
+ * is a valid parameter.
  */
 void rpmsg_destroy_ept(struct rpmsg_endpoint *ept)
 {
-	ept->ops->destroy_ept(ept);
+	if (ept)
+		ept->ops->destroy_ept(ept);
 }
 EXPORT_SYMBOL(rpmsg_destroy_ept);
 
@@ -108,6 +113,11 @@ EXPORT_SYMBOL(rpmsg_destroy_ept);
  */
 int rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len)
 {
+	if (WARN_ON(!ept))
+		return -EINVAL;
+	if (!ept->ops->send)
+		return -ENXIO;
+
 	return ept->ops->send(ept, data, len);
 }
 EXPORT_SYMBOL(rpmsg_send);
@@ -132,6 +142,11 @@ EXPORT_SYMBOL(rpmsg_send);
  */
 int rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
 {
+	if (WARN_ON(!ept))
+		return -EINVAL;
+	if (!ept->ops->sendto)
+		return -ENXIO;
+
 	return ept->ops->sendto(ept, data, len, dst);
 }
 EXPORT_SYMBOL(rpmsg_sendto);
@@ -159,6 +174,11 @@ EXPORT_SYMBOL(rpmsg_sendto);
 int rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst,
 			  void *data, int len)
 {
+	if (WARN_ON(!ept))
+		return -EINVAL;
+	if (!ept->ops->send_offchannel)
+		return -ENXIO;
+
 	return ept->ops->send_offchannel(ept, src, dst, data, len);
 }
 EXPORT_SYMBOL(rpmsg_send_offchannel);
@@ -182,6 +202,11 @@ EXPORT_SYMBOL(rpmsg_send_offchannel);
  */
 int rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len)
 {
+	if (WARN_ON(!ept))
+		return -EINVAL;
+	if (!ept->ops->trysend)
+		return -ENXIO;
+
 	return ept->ops->trysend(ept, data, len);
 }
 EXPORT_SYMBOL(rpmsg_trysend);
@@ -205,6 +230,11 @@ EXPORT_SYMBOL(rpmsg_trysend);
  */
 int rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
 {
+	if (WARN_ON(!ept))
+		return -EINVAL;
+	if (!ept->ops->trysendto)
+		return -ENXIO;
+
 	return ept->ops->trysendto(ept, data, len, dst);
 }
 EXPORT_SYMBOL(rpmsg_trysendto);
@@ -231,6 +261,11 @@ EXPORT_SYMBOL(rpmsg_trysendto);
 int rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst,
 			     void *data, int len)
 {
+	if (WARN_ON(!ept))
+		return -EINVAL;
+	if (!ept->ops->trysend_offchannel)
+		return -ENXIO;
+
 	return ept->ops->trysend_offchannel(ept, src, dst, data, len);
 }
 EXPORT_SYMBOL(rpmsg_trysend_offchannel);
@@ -315,6 +350,9 @@ static int rpmsg_dev_match(struct device *dev, struct device_driver *drv)
 	const struct rpmsg_device_id *ids = rpdrv->id_table;
 	unsigned int i;
 
+	if (rpdev->driver_override)
+		return !strcmp(rpdev->driver_override, drv->name);
+
 	if (ids)
 		for (i = 0; ids[i].name[0]; i++)
 			if (rpmsg_id_match(rpdev, &ids[i]))
@@ -344,27 +382,30 @@ static int rpmsg_dev_probe(struct device *dev)
 	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
 	struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver);
 	struct rpmsg_channel_info chinfo = {};
-	struct rpmsg_endpoint *ept;
+	struct rpmsg_endpoint *ept = NULL;
 	int err;
 
-	strncpy(chinfo.name, rpdev->id.name, RPMSG_NAME_SIZE);
-	chinfo.src = rpdev->src;
-	chinfo.dst = RPMSG_ADDR_ANY;
+	if (rpdrv->callback) {
+		strncpy(chinfo.name, rpdev->id.name, RPMSG_NAME_SIZE);
+		chinfo.src = rpdev->src;
+		chinfo.dst = RPMSG_ADDR_ANY;
 
-	ept = rpmsg_create_ept(rpdev, rpdrv->callback, NULL, chinfo);
-	if (!ept) {
-		dev_err(dev, "failed to create endpoint\n");
-		err = -ENOMEM;
-		goto out;
-	}
+		ept = rpmsg_create_ept(rpdev, rpdrv->callback, NULL, chinfo);
+		if (!ept) {
+			dev_err(dev, "failed to create endpoint\n");
+			err = -ENOMEM;
+			goto out;
+		}
 
-	rpdev->ept = ept;
-	rpdev->src = ept->addr;
+		rpdev->ept = ept;
+		rpdev->src = ept->addr;
+	}
 
 	err = rpdrv->probe(rpdev);
 	if (err) {
 		dev_err(dev, "%s: failed: %d\n", __func__, err);
-		rpmsg_destroy_ept(ept);
+		if (ept)
+			rpmsg_destroy_ept(ept);
 		goto out;
 	}
 
@@ -385,7 +426,8 @@ static int rpmsg_dev_remove(struct device *dev)
 
 	rpdrv->remove(rpdev);
 
-	rpmsg_destroy_ept(rpdev->ept);
+	if (rpdev->ept)
+		rpmsg_destroy_ept(rpdev->ept);
 
 	return err;
 }
diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c
index 1de089019268..0e3fdfdbd098 100644
--- a/drivers/s390/block/dasd.c
+++ b/drivers/s390/block/dasd.c
@@ -69,6 +69,7 @@ static void dasd_block_tasklet(struct dasd_block *);
 static void do_kick_device(struct work_struct *);
 static void do_restore_device(struct work_struct *);
 static void do_reload_device(struct work_struct *);
+static void do_requeue_requests(struct work_struct *);
 static void dasd_return_cqr_cb(struct dasd_ccw_req *, void *);
 static void dasd_device_timeout(unsigned long);
 static void dasd_block_timeout(unsigned long);
@@ -125,6 +126,7 @@ struct dasd_device *dasd_alloc_device(void)
 	INIT_WORK(&device->kick_work, do_kick_device);
 	INIT_WORK(&device->restore_device, do_restore_device);
 	INIT_WORK(&device->reload_device, do_reload_device);
+	INIT_WORK(&device->requeue_requests, do_requeue_requests);
 	device->state = DASD_STATE_NEW;
 	device->target = DASD_STATE_NEW;
 	mutex_init(&device->state_mutex);
@@ -1448,9 +1450,9 @@ int dasd_start_IO(struct dasd_ccw_req *cqr)
 	cqr->starttime = jiffies;
 	cqr->retries--;
 	if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) {
-		cqr->lpm &= device->path_data.opm;
+		cqr->lpm &= dasd_path_get_opm(device);
 		if (!cqr->lpm)
-			cqr->lpm = device->path_data.opm;
+			cqr->lpm = dasd_path_get_opm(device);
 	}
 	if (cqr->cpmode == 1) {
 		rc = ccw_device_tm_start(device->cdev, cqr->cpaddr,
@@ -1483,8 +1485,8 @@ int dasd_start_IO(struct dasd_ccw_req *cqr)
 			DBF_DEV_EVENT(DBF_WARNING, device,
 				      "start_IO: selected paths gone (%x)",
 				      cqr->lpm);
-		} else if (cqr->lpm != device->path_data.opm) {
-			cqr->lpm = device->path_data.opm;
+		} else if (cqr->lpm != dasd_path_get_opm(device)) {
+			cqr->lpm = dasd_path_get_opm(device);
 			DBF_DEV_EVENT(DBF_DEBUG, device, "%s",
 				      "start_IO: selected paths gone,"
 				      " retry on all paths");
@@ -1493,11 +1495,10 @@ int dasd_start_IO(struct dasd_ccw_req *cqr)
 				      "start_IO: all paths in opm gone,"
 				      " do path verification");
 			dasd_generic_last_path_gone(device);
-			device->path_data.opm = 0;
-			device->path_data.ppm = 0;
-			device->path_data.npm = 0;
-			device->path_data.tbvpm =
-				ccw_device_get_path_mask(device->cdev);
+			dasd_path_no_path(device);
+			dasd_path_set_tbvpm(device,
+					  ccw_device_get_path_mask(
+						  device->cdev));
 		}
 		break;
 	case -ENODEV:
@@ -1623,6 +1624,13 @@ void dasd_generic_handle_state_change(struct dasd_device *device)
 }
 EXPORT_SYMBOL_GPL(dasd_generic_handle_state_change);
 
+static int dasd_check_hpf_error(struct irb *irb)
+{
+	return (scsw_tm_is_valid_schxs(&irb->scsw) &&
+	    (irb->scsw.tm.sesq == SCSW_SESQ_DEV_NOFCX ||
+	     irb->scsw.tm.sesq == SCSW_SESQ_PATH_NOFCX));
+}
+
 /*
  * Interrupt handler for "normal" ssch-io based dasd devices.
  */
@@ -1642,7 +1650,7 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm,
 		switch (PTR_ERR(irb)) {
 		case -EIO:
 			if (cqr && cqr->status == DASD_CQR_CLEAR_PENDING) {
-				device = (struct dasd_device *) cqr->startdev;
+				device = cqr->startdev;
 				cqr->status = DASD_CQR_CLEARED;
 				dasd_device_clear_timer(device);
 				wake_up(&dasd_flush_wq);
@@ -1749,19 +1757,26 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm,
 					  struct dasd_ccw_req, devlist);
 		}
 	} else {  /* error */
+		/* check for HPF error
+		 * call discipline function to requeue all requests
+		 * and disable HPF accordingly
+		 */
+		if (cqr->cpmode && dasd_check_hpf_error(irb) &&
+		    device->discipline->handle_hpf_error)
+			device->discipline->handle_hpf_error(device, irb);
 		/*
 		 * If we don't want complex ERP for this request, then just
 		 * reset this and retry it in the fastpath
 		 */
 		if (!test_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags) &&
 		    cqr->retries > 0) {
-			if (cqr->lpm == device->path_data.opm)
+			if (cqr->lpm == dasd_path_get_opm(device))
 				DBF_DEV_EVENT(DBF_DEBUG, device,
 					      "default ERP in fastpath "
 					      "(%i retries left)",
 					      cqr->retries);
 			if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags))
-				cqr->lpm = device->path_data.opm;
+				cqr->lpm = dasd_path_get_opm(device);
 			cqr->status = DASD_CQR_QUEUED;
 			next = cqr;
 		} else
@@ -2002,17 +2017,18 @@ static void __dasd_device_check_path_events(struct dasd_device *device)
 {
 	int rc;
 
-	if (device->path_data.tbvpm) {
-		if (device->stopped & ~(DASD_STOPPED_DC_WAIT |
-					DASD_UNRESUMED_PM))
-			return;
-		rc = device->discipline->verify_path(
-			device, device->path_data.tbvpm);
-		if (rc)
-			dasd_device_set_timer(device, 50);
-		else
-			device->path_data.tbvpm = 0;
-	}
+	if (!dasd_path_get_tbvpm(device))
+		return;
+
+	if (device->stopped &
+	    ~(DASD_STOPPED_DC_WAIT | DASD_UNRESUMED_PM))
+		return;
+	rc = device->discipline->verify_path(device,
+					     dasd_path_get_tbvpm(device));
+	if (rc)
+		dasd_device_set_timer(device, 50);
+	else
+		dasd_path_clear_all_verify(device);
 };
 
 /*
@@ -2924,10 +2940,10 @@ static int _dasd_requeue_request(struct dasd_ccw_req *cqr)
 
 	if (!block)
 		return -EINVAL;
-	spin_lock_irqsave(&block->queue_lock, flags);
+	spin_lock_irqsave(&block->request_queue_lock, flags);
 	req = (struct request *) cqr->callback_data;
 	blk_requeue_request(block->request_queue, req);
-	spin_unlock_irqrestore(&block->queue_lock, flags);
+	spin_unlock_irqrestore(&block->request_queue_lock, flags);
 
 	return 0;
 }
@@ -3121,6 +3137,7 @@ static int dasd_alloc_queue(struct dasd_block *block)
  */
 static void dasd_setup_queue(struct dasd_block *block)
 {
+	struct request_queue *q = block->request_queue;
 	int max;
 
 	if (block->base->features & DASD_FEATURE_USERAW) {
@@ -3135,17 +3152,16 @@ static void dasd_setup_queue(struct dasd_block *block)
 	} else {
 		max = block->base->discipline->max_blocks << block->s2b_shift;
 	}
-	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, block->request_queue);
-	block->request_queue->limits.max_dev_sectors = max;
-	blk_queue_logical_block_size(block->request_queue,
-				     block->bp_block);
-	blk_queue_max_hw_sectors(block->request_queue, max);
-	blk_queue_max_segments(block->request_queue, -1L);
+	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
+	q->limits.max_dev_sectors = max;
+	blk_queue_logical_block_size(q, block->bp_block);
+	blk_queue_max_hw_sectors(q, max);
+	blk_queue_max_segments(q, USHRT_MAX);
 	/* with page sized segments we can translate each segement into
 	 * one idaw/tidaw
 	 */
-	blk_queue_max_segment_size(block->request_queue, PAGE_SIZE);
-	blk_queue_segment_boundary(block->request_queue, PAGE_SIZE - 1);
+	blk_queue_max_segment_size(q, PAGE_SIZE);
+	blk_queue_segment_boundary(q, PAGE_SIZE - 1);
 }
 
 /*
@@ -3517,11 +3533,15 @@ int dasd_generic_set_offline(struct ccw_device *cdev)
 	struct dasd_device *device;
 	struct dasd_block *block;
 	int max_count, open_count, rc;
+	unsigned long flags;
 
 	rc = 0;
-	device = dasd_device_from_cdev(cdev);
-	if (IS_ERR(device))
+	spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
+	device = dasd_device_from_cdev_locked(cdev);
+	if (IS_ERR(device)) {
+		spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
 		return PTR_ERR(device);
+	}
 
 	/*
 	 * We must make sure that this device is currently not in use.
@@ -3540,8 +3560,7 @@ int dasd_generic_set_offline(struct ccw_device *cdev)
 				pr_warn("%s: The DASD cannot be set offline while it is in use\n",
 					dev_name(&cdev->dev));
 			clear_bit(DASD_FLAG_OFFLINE, &device->flags);
-			dasd_put_device(device);
-			return -EBUSY;
+			goto out_busy;
 		}
 	}
 
@@ -3551,19 +3570,19 @@ int dasd_generic_set_offline(struct ccw_device *cdev)
 		 * could only be called by normal offline so safe_offline flag
 		 * needs to be removed to run normal offline and kill all I/O
 		 */
-		if (test_and_set_bit(DASD_FLAG_OFFLINE, &device->flags)) {
+		if (test_and_set_bit(DASD_FLAG_OFFLINE, &device->flags))
 			/* Already doing normal offline processing */
-			dasd_put_device(device);
-			return -EBUSY;
-		} else
+			goto out_busy;
+		else
 			clear_bit(DASD_FLAG_SAFE_OFFLINE, &device->flags);
-
-	} else
-		if (test_bit(DASD_FLAG_OFFLINE, &device->flags)) {
+	} else {
+		if (test_bit(DASD_FLAG_OFFLINE, &device->flags))
 			/* Already doing offline processing */
-			dasd_put_device(device);
-			return -EBUSY;
-		}
+			goto out_busy;
+	}
+
+	set_bit(DASD_FLAG_OFFLINE, &device->flags);
+	spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
 
 	/*
 	 * if safe_offline called set safe_offline_running flag and
@@ -3591,7 +3610,6 @@ int dasd_generic_set_offline(struct ccw_device *cdev)
 			goto interrupted;
 	}
 
-	set_bit(DASD_FLAG_OFFLINE, &device->flags);
 	dasd_set_target_state(device, DASD_STATE_NEW);
 	/* dasd_delete_device destroys the device reference. */
 	block = device->block;
@@ -3610,7 +3628,14 @@ interrupted:
 	clear_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags);
 	clear_bit(DASD_FLAG_OFFLINE, &device->flags);
 	dasd_put_device(device);
+
 	return rc;
+
+out_busy:
+	dasd_put_device(device);
+	spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
+
+	return -EBUSY;
 }
 EXPORT_SYMBOL_GPL(dasd_generic_set_offline);
 
@@ -3675,14 +3700,12 @@ int dasd_generic_notify(struct ccw_device *cdev, int event)
 	case CIO_GONE:
 	case CIO_BOXED:
 	case CIO_NO_PATH:
-		device->path_data.opm = 0;
-		device->path_data.ppm = 0;
-		device->path_data.npm = 0;
+		dasd_path_no_path(device);
 		ret = dasd_generic_last_path_gone(device);
 		break;
 	case CIO_OPER:
 		ret = 1;
-		if (device->path_data.opm)
+		if (dasd_path_get_opm(device))
 			ret = dasd_generic_path_operational(device);
 		break;
 	}
@@ -3693,48 +3716,32 @@ EXPORT_SYMBOL_GPL(dasd_generic_notify);
 
 void dasd_generic_path_event(struct ccw_device *cdev, int *path_event)
 {
-	int chp;
-	__u8 oldopm, eventlpm;
 	struct dasd_device *device;
+	int chp, oldopm, hpfpm, ifccpm;
 
 	device = dasd_device_from_cdev_locked(cdev);
 	if (IS_ERR(device))
 		return;
+
+	oldopm = dasd_path_get_opm(device);
 	for (chp = 0; chp < 8; chp++) {
-		eventlpm = 0x80 >> chp;
 		if (path_event[chp] & PE_PATH_GONE) {
-			oldopm = device->path_data.opm;
-			device->path_data.opm &= ~eventlpm;
-			device->path_data.ppm &= ~eventlpm;
-			device->path_data.npm &= ~eventlpm;
-			if (oldopm && !device->path_data.opm) {
-				dev_warn(&device->cdev->dev,
-					 "No verified channel paths remain "
-					 "for the device\n");
-				DBF_DEV_EVENT(DBF_WARNING, device,
-					      "%s", "last verified path gone");
-				dasd_eer_write(device, NULL, DASD_EER_NOPATH);
-				dasd_device_set_stop_bits(device,
-							  DASD_STOPPED_DC_WAIT);
-			}
+			dasd_path_notoper(device, chp);
 		}
 		if (path_event[chp] & PE_PATH_AVAILABLE) {
-			device->path_data.opm &= ~eventlpm;
-			device->path_data.ppm &= ~eventlpm;
-			device->path_data.npm &= ~eventlpm;
-			device->path_data.tbvpm |= eventlpm;
+			dasd_path_available(device, chp);
 			dasd_schedule_device_bh(device);
 		}
 		if (path_event[chp] & PE_PATHGROUP_ESTABLISHED) {
-			if (!(device->path_data.opm & eventlpm) &&
-			    !(device->path_data.tbvpm & eventlpm)) {
+			if (!dasd_path_is_operational(device, chp) &&
+			    !dasd_path_need_verify(device, chp)) {
 				/*
 				 * we can not establish a pathgroup on an
 				 * unavailable path, so trigger a path
 				 * verification first
 				 */
-				device->path_data.tbvpm |= eventlpm;
-				dasd_schedule_device_bh(device);
+			dasd_path_available(device, chp);
+			dasd_schedule_device_bh(device);
 			}
 			DBF_DEV_EVENT(DBF_WARNING, device, "%s",
 				      "Pathgroup re-established\n");
@@ -3742,45 +3749,65 @@ void dasd_generic_path_event(struct ccw_device *cdev, int *path_event)
 				device->discipline->kick_validate(device);
 		}
 	}
+	hpfpm = dasd_path_get_hpfpm(device);
+	ifccpm = dasd_path_get_ifccpm(device);
+	if (!dasd_path_get_opm(device) && hpfpm) {
+		/*
+		 * device has no operational paths but at least one path is
+		 * disabled due to HPF errors
+		 * disable HPF at all and use the path(s) again
+		 */
+		if (device->discipline->disable_hpf)
+			device->discipline->disable_hpf(device);
+		dasd_device_set_stop_bits(device, DASD_STOPPED_NOT_ACC);
+		dasd_path_set_tbvpm(device, hpfpm);
+		dasd_schedule_device_bh(device);
+		dasd_schedule_requeue(device);
+	} else if (!dasd_path_get_opm(device) && ifccpm) {
+		/*
+		 * device has no operational paths but at least one path is
+		 * disabled due to IFCC errors
+		 * trigger path verification on paths with IFCC errors
+		 */
+		dasd_path_set_tbvpm(device, ifccpm);
+		dasd_schedule_device_bh(device);
+	}
+	if (oldopm && !dasd_path_get_opm(device) && !hpfpm && !ifccpm) {
+		dev_warn(&device->cdev->dev,
+			 "No verified channel paths remain for the device\n");
+		DBF_DEV_EVENT(DBF_WARNING, device,
+			      "%s", "last verified path gone");
+		dasd_eer_write(device, NULL, DASD_EER_NOPATH);
+		dasd_device_set_stop_bits(device,
+					  DASD_STOPPED_DC_WAIT);
+	}
 	dasd_put_device(device);
 }
 EXPORT_SYMBOL_GPL(dasd_generic_path_event);
 
 int dasd_generic_verify_path(struct dasd_device *device, __u8 lpm)
 {
-	if (!device->path_data.opm && lpm) {
-		device->path_data.opm = lpm;
+	if (!dasd_path_get_opm(device) && lpm) {
+		dasd_path_set_opm(device, lpm);
 		dasd_generic_path_operational(device);
 	} else
-		device->path_data.opm |= lpm;
+		dasd_path_add_opm(device, lpm);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(dasd_generic_verify_path);
 
-
-int dasd_generic_pm_freeze(struct ccw_device *cdev)
+/*
+ * clear active requests and requeue them to block layer if possible
+ */
+static int dasd_generic_requeue_all_requests(struct dasd_device *device)
 {
-	struct dasd_device *device = dasd_device_from_cdev(cdev);
-	struct list_head freeze_queue;
+	struct list_head requeue_queue;
 	struct dasd_ccw_req *cqr, *n;
 	struct dasd_ccw_req *refers;
 	int rc;
 
-	if (IS_ERR(device))
-		return PTR_ERR(device);
-
-	/* mark device as suspended */
-	set_bit(DASD_FLAG_SUSPENDED, &device->flags);
-
-	if (device->discipline->freeze)
-		rc = device->discipline->freeze(device);
-
-	/* disallow new I/O  */
-	dasd_device_set_stop_bits(device, DASD_STOPPED_PM);
-
-	/* clear active requests and requeue them to block layer if possible */
-	INIT_LIST_HEAD(&freeze_queue);
-	spin_lock_irq(get_ccwdev_lock(cdev));
+	INIT_LIST_HEAD(&requeue_queue);
+	spin_lock_irq(get_ccwdev_lock(device->cdev));
 	rc = 0;
 	list_for_each_entry_safe(cqr, n, &device->ccw_queue, devlist) {
 		/* Check status and move request to flush_queue */
@@ -3791,25 +3818,22 @@ int dasd_generic_pm_freeze(struct ccw_device *cdev)
 				dev_err(&device->cdev->dev,
 					"Unable to terminate request %p "
 					"on suspend\n", cqr);
-				spin_unlock_irq(get_ccwdev_lock(cdev));
+				spin_unlock_irq(get_ccwdev_lock(device->cdev));
 				dasd_put_device(device);
 				return rc;
 			}
 		}
-		list_move_tail(&cqr->devlist, &freeze_queue);
+		list_move_tail(&cqr->devlist, &requeue_queue);
 	}
-	spin_unlock_irq(get_ccwdev_lock(cdev));
+	spin_unlock_irq(get_ccwdev_lock(device->cdev));
 
-	list_for_each_entry_safe(cqr, n, &freeze_queue, devlist) {
+	list_for_each_entry_safe(cqr, n, &requeue_queue, devlist) {
 		wait_event(dasd_flush_wq,
 			   (cqr->status != DASD_CQR_CLEAR_PENDING));
-		if (cqr->status == DASD_CQR_CLEARED)
-			cqr->status = DASD_CQR_QUEUED;
 
-		/* requeue requests to blocklayer will only work for
-		   block device requests */
-		if (_dasd_requeue_request(cqr))
-			continue;
+		/* mark sleepon requests as ended */
+		if (cqr->callback_data == DASD_SLEEPON_START_TAG)
+			cqr->callback_data = DASD_SLEEPON_END_TAG;
 
 		/* remove requests from device and block queue */
 		list_del_init(&cqr->devlist);
@@ -3821,6 +3845,14 @@ int dasd_generic_pm_freeze(struct ccw_device *cdev)
 			dasd_free_erp_request(cqr, cqr->memdev);
 			cqr = refers;
 		}
+
+		/*
+		 * requeue requests to blocklayer will only work
+		 * for block device requests
+		 */
+		if (_dasd_requeue_request(cqr))
+			continue;
+
 		if (cqr->block)
 			list_del_init(&cqr->blocklist);
 		cqr->block->base->discipline->free_cp(
@@ -3831,15 +3863,56 @@ int dasd_generic_pm_freeze(struct ccw_device *cdev)
 	 * if requests remain then they are internal request
 	 * and go back to the device queue
 	 */
-	if (!list_empty(&freeze_queue)) {
+	if (!list_empty(&requeue_queue)) {
 		/* move freeze_queue to start of the ccw_queue */
-		spin_lock_irq(get_ccwdev_lock(cdev));
-		list_splice_tail(&freeze_queue, &device->ccw_queue);
-		spin_unlock_irq(get_ccwdev_lock(cdev));
+		spin_lock_irq(get_ccwdev_lock(device->cdev));
+		list_splice_tail(&requeue_queue, &device->ccw_queue);
+		spin_unlock_irq(get_ccwdev_lock(device->cdev));
 	}
-	dasd_put_device(device);
+	/* wake up generic waitqueue for eventually ended sleepon requests */
+	wake_up(&generic_waitq);
 	return rc;
 }
+
+static void do_requeue_requests(struct work_struct *work)
+{
+	struct dasd_device *device = container_of(work, struct dasd_device,
+						  requeue_requests);
+	dasd_generic_requeue_all_requests(device);
+	dasd_device_remove_stop_bits(device, DASD_STOPPED_NOT_ACC);
+	if (device->block)
+		dasd_schedule_block_bh(device->block);
+	dasd_put_device(device);
+}
+
+void dasd_schedule_requeue(struct dasd_device *device)
+{
+	dasd_get_device(device);
+	/* queue call to dasd_reload_device to the kernel event daemon. */
+	if (!schedule_work(&device->requeue_requests))
+		dasd_put_device(device);
+}
+EXPORT_SYMBOL(dasd_schedule_requeue);
+
+int dasd_generic_pm_freeze(struct ccw_device *cdev)
+{
+	struct dasd_device *device = dasd_device_from_cdev(cdev);
+	int rc;
+
+	if (IS_ERR(device))
+		return PTR_ERR(device);
+
+	/* mark device as suspended */
+	set_bit(DASD_FLAG_SUSPENDED, &device->flags);
+
+	if (device->discipline->freeze)
+		rc = device->discipline->freeze(device);
+
+	/* disallow new I/O  */
+	dasd_device_set_stop_bits(device, DASD_STOPPED_PM);
+
+	return dasd_generic_requeue_all_requests(device);
+}
 EXPORT_SYMBOL_GPL(dasd_generic_pm_freeze);
 
 int dasd_generic_restore_device(struct ccw_device *cdev)
diff --git a/drivers/s390/block/dasd_3990_erp.c b/drivers/s390/block/dasd_3990_erp.c
index 8305ab688d57..774da20ceb58 100644
--- a/drivers/s390/block/dasd_3990_erp.c
+++ b/drivers/s390/block/dasd_3990_erp.c
@@ -152,7 +152,7 @@ dasd_3990_erp_alternate_path(struct dasd_ccw_req * erp)
 	opm = ccw_device_get_path_mask(device->cdev);
 	spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
 	if (erp->lpm == 0)
-		erp->lpm = device->path_data.opm &
+		erp->lpm = dasd_path_get_opm(device) &
 			~(erp->irb.esw.esw0.sublog.lpum);
 	else
 		erp->lpm &= ~(erp->irb.esw.esw0.sublog.lpum);
@@ -273,7 +273,7 @@ static struct dasd_ccw_req *dasd_3990_erp_action_1(struct dasd_ccw_req *erp)
 	    !test_bit(DASD_CQR_VERIFY_PATH, &erp->flags)) {
 		erp->status = DASD_CQR_FILLED;
 		erp->retries = 10;
-		erp->lpm = erp->startdev->path_data.opm;
+		erp->lpm = dasd_path_get_opm(erp->startdev);
 		erp->function = dasd_3990_erp_action_1_sec;
 	}
 	return erp;
@@ -674,7 +674,7 @@ dasd_3990_handle_env_data(struct dasd_ccw_req * erp, char *sense)
 			break;
 		case 0x0D:
 			dev_warn(&device->cdev->dev,
-				    "FORMAT 4 - No syn byte in count "
+				    "FORMAT 4 - No sync byte in count "
 				    "address area; offset active\n");
 			break;
 		case 0x0E:
@@ -684,7 +684,7 @@ dasd_3990_handle_env_data(struct dasd_ccw_req * erp, char *sense)
 			break;
 		case 0x0F:
 			dev_warn(&device->cdev->dev,
-				    "FORMAT 4 - No syn byte in data area; "
+				    "FORMAT 4 - No sync byte in data area; "
 				    "offset active\n");
 			break;
 		default:
@@ -999,7 +999,7 @@ dasd_3990_handle_env_data(struct dasd_ccw_req * erp, char *sense)
 			break;
 		default:
 			dev_warn(&device->cdev->dev,
-				    "FORMAT D - Reserved\n");
+				    "FORMAT F - Reserved\n");
 		}
 		break;
 
@@ -1926,7 +1926,7 @@ dasd_3990_erp_compound_path(struct dasd_ccw_req * erp, char *sense)
 		    !test_bit(DASD_CQR_VERIFY_PATH, &erp->flags)) {
 			/* reset the lpm and the status to be able to
 			 * try further actions. */
-			erp->lpm = erp->startdev->path_data.opm;
+			erp->lpm = dasd_path_get_opm(erp->startdev);
 			erp->status = DASD_CQR_NEED_ERP;
 		}
 	}
@@ -2208,6 +2208,51 @@ dasd_3990_erp_inspect_32(struct dasd_ccw_req * erp, char *sense)
 
 }				/* end dasd_3990_erp_inspect_32 */
 
+static void dasd_3990_erp_disable_path(struct dasd_device *device, __u8 lpum)
+{
+	int pos = pathmask_to_pos(lpum);
+
+	/* no remaining path, cannot disable */
+	if (!(dasd_path_get_opm(device) & ~lpum))
+		return;
+
+	dev_err(&device->cdev->dev,
+		"Path %x.%02x (pathmask %02x) is disabled - IFCC threshold exceeded\n",
+		device->path[pos].cssid, device->path[pos].chpid, lpum);
+	dasd_path_remove_opm(device, lpum);
+	dasd_path_add_ifccpm(device, lpum);
+	device->path[pos].errorclk = 0;
+	atomic_set(&device->path[pos].error_count, 0);
+}
+
+static void dasd_3990_erp_account_error(struct dasd_ccw_req *erp)
+{
+	struct dasd_device *device = erp->startdev;
+	__u8 lpum = erp->refers->irb.esw.esw1.lpum;
+	int pos = pathmask_to_pos(lpum);
+	unsigned long long clk;
+
+	if (!device->path_thrhld)
+		return;
+
+	clk = get_tod_clock();
+	/*
+	 * check if the last error is longer ago than the timeout,
+	 * if so reset error state
+	 */
+	if ((tod_to_ns(clk - device->path[pos].errorclk) / NSEC_PER_SEC)
+	    >= device->path_interval) {
+		atomic_set(&device->path[pos].error_count, 0);
+		device->path[pos].errorclk = 0;
+	}
+	atomic_inc(&device->path[pos].error_count);
+	device->path[pos].errorclk = clk;
+	/* threshold exceeded disable path if possible */
+	if (atomic_read(&device->path[pos].error_count) >=
+	    device->path_thrhld)
+		dasd_3990_erp_disable_path(device, lpum);
+}
+
 /*
  *****************************************************************************
  * main ERP control functions (24 and 32 byte sense)
@@ -2237,6 +2282,7 @@ dasd_3990_erp_control_check(struct dasd_ccw_req *erp)
 					   | SCHN_STAT_CHN_CTRL_CHK)) {
 		DBF_DEV_EVENT(DBF_WARNING, device, "%s",
 			    "channel or interface control check");
+		dasd_3990_erp_account_error(erp);
 		erp = dasd_3990_erp_action_4(erp, NULL);
 	}
 	return erp;
diff --git a/drivers/s390/block/dasd_devmap.c b/drivers/s390/block/dasd_devmap.c
index 15a1a70cace9..84ca314c87e3 100644
--- a/drivers/s390/block/dasd_devmap.c
+++ b/drivers/s390/block/dasd_devmap.c
@@ -725,27 +725,15 @@ static ssize_t dasd_ff_show(struct device *dev, struct device_attribute *attr,
 static ssize_t dasd_ff_store(struct device *dev, struct device_attribute *attr,
 	      const char *buf, size_t count)
 {
-	struct dasd_devmap *devmap;
-	int val;
-	char *endp;
-
-	devmap = dasd_devmap_from_cdev(to_ccwdev(dev));
-	if (IS_ERR(devmap))
-		return PTR_ERR(devmap);
+	unsigned int val;
+	int rc;
 
-	val = simple_strtoul(buf, &endp, 0);
-	if (((endp + 1) < (buf + count)) || (val > 1))
+	if (kstrtouint(buf, 0, &val) || val > 1)
 		return -EINVAL;
 
-	spin_lock(&dasd_devmap_lock);
-	if (val)
-		devmap->features |= DASD_FEATURE_FAILFAST;
-	else
-		devmap->features &= ~DASD_FEATURE_FAILFAST;
-	if (devmap->device)
-		devmap->device->features = devmap->features;
-	spin_unlock(&dasd_devmap_lock);
-	return count;
+	rc = dasd_set_feature(to_ccwdev(dev), DASD_FEATURE_FAILFAST, val);
+
+	return rc ? : count;
 }
 
 static DEVICE_ATTR(failfast, 0644, dasd_ff_show, dasd_ff_store);
@@ -771,32 +759,41 @@ static ssize_t
 dasd_ro_store(struct device *dev, struct device_attribute *attr,
 	      const char *buf, size_t count)
 {
-	struct dasd_devmap *devmap;
+	struct ccw_device *cdev = to_ccwdev(dev);
 	struct dasd_device *device;
-	int val;
-	char *endp;
-
-	devmap = dasd_devmap_from_cdev(to_ccwdev(dev));
-	if (IS_ERR(devmap))
-		return PTR_ERR(devmap);
+	unsigned long flags;
+	unsigned int val;
+	int rc;
 
-	val = simple_strtoul(buf, &endp, 0);
-	if (((endp + 1) < (buf + count)) || (val > 1))
+	if (kstrtouint(buf, 0, &val) || val > 1)
 		return -EINVAL;
 
-	spin_lock(&dasd_devmap_lock);
-	if (val)
-		devmap->features |= DASD_FEATURE_READONLY;
-	else
-		devmap->features &= ~DASD_FEATURE_READONLY;
-	device = devmap->device;
-	if (device) {
-		device->features = devmap->features;
-		val = val || test_bit(DASD_FLAG_DEVICE_RO, &device->flags);
+	rc = dasd_set_feature(cdev, DASD_FEATURE_READONLY, val);
+	if (rc)
+		return rc;
+
+	device = dasd_device_from_cdev(cdev);
+	if (IS_ERR(device))
+		return PTR_ERR(device);
+
+	spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
+	val = val || test_bit(DASD_FLAG_DEVICE_RO, &device->flags);
+
+	if (!device->block || !device->block->gdp ||
+	    test_bit(DASD_FLAG_OFFLINE, &device->flags)) {
+		spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
+		goto out;
 	}
-	spin_unlock(&dasd_devmap_lock);
-	if (device && device->block && device->block->gdp)
-		set_disk_ro(device->block->gdp, val);
+	/* Increase open_count to avoid losing the block device */
+	atomic_inc(&device->block->open_count);
+	spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
+
+	set_disk_ro(device->block->gdp, val);
+	atomic_dec(&device->block->open_count);
+
+out:
+	dasd_put_device(device);
+
 	return count;
 }
 
@@ -823,27 +820,15 @@ static ssize_t
 dasd_erplog_store(struct device *dev, struct device_attribute *attr,
 	      const char *buf, size_t count)
 {
-	struct dasd_devmap *devmap;
-	int val;
-	char *endp;
-
-	devmap = dasd_devmap_from_cdev(to_ccwdev(dev));
-	if (IS_ERR(devmap))
-		return PTR_ERR(devmap);
+	unsigned int val;
+	int rc;
 
-	val = simple_strtoul(buf, &endp, 0);
-	if (((endp + 1) < (buf + count)) || (val > 1))
+	if (kstrtouint(buf, 0, &val) || val > 1)
 		return -EINVAL;
 
-	spin_lock(&dasd_devmap_lock);
-	if (val)
-		devmap->features |= DASD_FEATURE_ERPLOG;
-	else
-		devmap->features &= ~DASD_FEATURE_ERPLOG;
-	if (devmap->device)
-		devmap->device->features = devmap->features;
-	spin_unlock(&dasd_devmap_lock);
-	return count;
+	rc = dasd_set_feature(to_ccwdev(dev), DASD_FEATURE_ERPLOG, val);
+
+	return rc ? : count;
 }
 
 static DEVICE_ATTR(erplog, 0644, dasd_erplog_show, dasd_erplog_store);
@@ -871,16 +856,14 @@ dasd_use_diag_store(struct device *dev, struct device_attribute *attr,
 		    const char *buf, size_t count)
 {
 	struct dasd_devmap *devmap;
+	unsigned int val;
 	ssize_t rc;
-	int val;
-	char *endp;
 
 	devmap = dasd_devmap_from_cdev(to_ccwdev(dev));
 	if (IS_ERR(devmap))
 		return PTR_ERR(devmap);
 
-	val = simple_strtoul(buf, &endp, 0);
-	if (((endp + 1) < (buf + count)) || (val > 1))
+	if (kstrtouint(buf, 0, &val) || val > 1)
 		return -EINVAL;
 
 	spin_lock(&dasd_devmap_lock);
@@ -994,10 +977,12 @@ dasd_access_show(struct device *dev, struct device_attribute *attr,
 	if (IS_ERR(device))
 		return PTR_ERR(device);
 
-	if (device->discipline->host_access_count)
-		count = device->discipline->host_access_count(device);
-	else
+	if (!device->discipline)
+		count = -ENODEV;
+	else if (!device->discipline->host_access_count)
 		count = -EOPNOTSUPP;
+	else
+		count = device->discipline->host_access_count(device);
 
 	dasd_put_device(device);
 	if (count < 0)
@@ -1197,27 +1182,25 @@ static ssize_t
 dasd_eer_store(struct device *dev, struct device_attribute *attr,
 	       const char *buf, size_t count)
 {
-	struct dasd_devmap *devmap;
-	int val, rc;
-	char *endp;
+	struct dasd_device *device;
+	unsigned int val;
+	int rc = 0;
 
-	devmap = dasd_devmap_from_cdev(to_ccwdev(dev));
-	if (IS_ERR(devmap))
-		return PTR_ERR(devmap);
-	if (!devmap->device)
-		return -ENODEV;
+	device = dasd_device_from_cdev(to_ccwdev(dev));
+	if (IS_ERR(device))
+		return PTR_ERR(device);
 
-	val = simple_strtoul(buf, &endp, 0);
-	if (((endp + 1) < (buf + count)) || (val > 1))
+	if (kstrtouint(buf, 0, &val) || val > 1)
 		return -EINVAL;
 
-	if (val) {
-		rc = dasd_eer_enable(devmap->device);
-		if (rc)
-			return rc;
-	} else
-		dasd_eer_disable(devmap->device);
-	return count;
+	if (val)
+		rc = dasd_eer_enable(device);
+	else
+		dasd_eer_disable(device);
+
+	dasd_put_device(device);
+
+	return rc ? : count;
 }
 
 static DEVICE_ATTR(eer_enabled, 0644, dasd_eer_show, dasd_eer_store);
@@ -1360,6 +1343,50 @@ dasd_timeout_store(struct device *dev, struct device_attribute *attr,
 static DEVICE_ATTR(timeout, 0644,
 		   dasd_timeout_show, dasd_timeout_store);
 
+
+static ssize_t
+dasd_path_reset_store(struct device *dev, struct device_attribute *attr,
+		      const char *buf, size_t count)
+{
+	struct dasd_device *device;
+	unsigned int val;
+
+	device = dasd_device_from_cdev(to_ccwdev(dev));
+	if (IS_ERR(device))
+		return -ENODEV;
+
+	if ((kstrtouint(buf, 16, &val) != 0) || val > 0xff)
+		val = 0;
+
+	if (device->discipline && device->discipline->reset_path)
+		device->discipline->reset_path(device, (__u8) val);
+
+	dasd_put_device(device);
+	return count;
+}
+
+static DEVICE_ATTR(path_reset, 0200, NULL, dasd_path_reset_store);
+
+static ssize_t dasd_hpf_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	struct dasd_device *device;
+	int hpf;
+
+	device = dasd_device_from_cdev(to_ccwdev(dev));
+	if (IS_ERR(device))
+		return -ENODEV;
+	if (!device->discipline || !device->discipline->hpf_enabled) {
+		dasd_put_device(device);
+		return snprintf(buf, PAGE_SIZE, "%d\n", dasd_nofcx);
+	}
+	hpf = device->discipline->hpf_enabled(device);
+	dasd_put_device(device);
+	return snprintf(buf, PAGE_SIZE, "%d\n", hpf);
+}
+
+static DEVICE_ATTR(hpf, 0444, dasd_hpf_show, NULL);
+
 static ssize_t dasd_reservation_policy_show(struct device *dev,
 					    struct device_attribute *attr,
 					    char *buf)
@@ -1385,27 +1412,17 @@ static ssize_t dasd_reservation_policy_store(struct device *dev,
 					     struct device_attribute *attr,
 					     const char *buf, size_t count)
 {
-	struct dasd_devmap *devmap;
+	struct ccw_device *cdev = to_ccwdev(dev);
 	int rc;
 
-	devmap = dasd_devmap_from_cdev(to_ccwdev(dev));
-	if (IS_ERR(devmap))
-		return PTR_ERR(devmap);
-	rc = 0;
-	spin_lock(&dasd_devmap_lock);
 	if (sysfs_streq("ignore", buf))
-		devmap->features &= ~DASD_FEATURE_FAILONSLCK;
+		rc = dasd_set_feature(cdev, DASD_FEATURE_FAILONSLCK, 0);
 	else if (sysfs_streq("fail", buf))
-		devmap->features |= DASD_FEATURE_FAILONSLCK;
+		rc = dasd_set_feature(cdev, DASD_FEATURE_FAILONSLCK, 1);
 	else
 		rc = -EINVAL;
-	if (devmap->device)
-		devmap->device->features = devmap->features;
-	spin_unlock(&dasd_devmap_lock);
-	if (rc)
-		return rc;
-	else
-		return count;
+
+	return rc ? : count;
 }
 
 static DEVICE_ATTR(reservation_policy, 0644,
@@ -1461,25 +1478,120 @@ static ssize_t dasd_pm_show(struct device *dev,
 			      struct device_attribute *attr, char *buf)
 {
 	struct dasd_device *device;
-	u8 opm, nppm, cablepm, cuirpm, hpfpm;
+	u8 opm, nppm, cablepm, cuirpm, hpfpm, ifccpm;
 
 	device = dasd_device_from_cdev(to_ccwdev(dev));
 	if (IS_ERR(device))
 		return sprintf(buf, "0\n");
 
-	opm = device->path_data.opm;
-	nppm = device->path_data.npm;
-	cablepm = device->path_data.cablepm;
-	cuirpm = device->path_data.cuirpm;
-	hpfpm = device->path_data.hpfpm;
+	opm = dasd_path_get_opm(device);
+	nppm = dasd_path_get_nppm(device);
+	cablepm = dasd_path_get_cablepm(device);
+	cuirpm = dasd_path_get_cuirpm(device);
+	hpfpm = dasd_path_get_hpfpm(device);
+	ifccpm = dasd_path_get_ifccpm(device);
 	dasd_put_device(device);
 
-	return sprintf(buf, "%02x %02x %02x %02x %02x\n", opm, nppm,
-		       cablepm, cuirpm, hpfpm);
+	return sprintf(buf, "%02x %02x %02x %02x %02x %02x\n", opm, nppm,
+		       cablepm, cuirpm, hpfpm, ifccpm);
 }
 
 static DEVICE_ATTR(path_masks, 0444, dasd_pm_show, NULL);
 
+/*
+ * threshold value for IFCC/CCC errors
+ */
+static ssize_t
+dasd_path_threshold_show(struct device *dev,
+			  struct device_attribute *attr, char *buf)
+{
+	struct dasd_device *device;
+	int len;
+
+	device = dasd_device_from_cdev(to_ccwdev(dev));
+	if (IS_ERR(device))
+		return -ENODEV;
+	len = snprintf(buf, PAGE_SIZE, "%lu\n", device->path_thrhld);
+	dasd_put_device(device);
+	return len;
+}
+
+static ssize_t
+dasd_path_threshold_store(struct device *dev, struct device_attribute *attr,
+			   const char *buf, size_t count)
+{
+	struct dasd_device *device;
+	unsigned long flags;
+	unsigned long val;
+
+	device = dasd_device_from_cdev(to_ccwdev(dev));
+	if (IS_ERR(device))
+		return -ENODEV;
+
+	if ((kstrtoul(buf, 10, &val) != 0) ||
+	    (val > DASD_THRHLD_MAX) || val == 0) {
+		dasd_put_device(device);
+		return -EINVAL;
+	}
+	spin_lock_irqsave(get_ccwdev_lock(to_ccwdev(dev)), flags);
+	if (val)
+		device->path_thrhld = val;
+	spin_unlock_irqrestore(get_ccwdev_lock(to_ccwdev(dev)), flags);
+	dasd_put_device(device);
+	return count;
+}
+
+static DEVICE_ATTR(path_threshold, 0644, dasd_path_threshold_show,
+		   dasd_path_threshold_store);
+/*
+ * interval for IFCC/CCC checks
+ * meaning time with no IFCC/CCC error before the error counter
+ * gets reset
+ */
+static ssize_t
+dasd_path_interval_show(struct device *dev,
+			struct device_attribute *attr, char *buf)
+{
+	struct dasd_device *device;
+	int len;
+
+	device = dasd_device_from_cdev(to_ccwdev(dev));
+	if (IS_ERR(device))
+		return -ENODEV;
+	len = snprintf(buf, PAGE_SIZE, "%lu\n", device->path_interval);
+	dasd_put_device(device);
+	return len;
+}
+
+static ssize_t
+dasd_path_interval_store(struct device *dev, struct device_attribute *attr,
+	       const char *buf, size_t count)
+{
+	struct dasd_device *device;
+	unsigned long flags;
+	unsigned long val;
+
+	device = dasd_device_from_cdev(to_ccwdev(dev));
+	if (IS_ERR(device))
+		return -ENODEV;
+
+	if ((kstrtoul(buf, 10, &val) != 0) ||
+	    (val > DASD_INTERVAL_MAX) || val == 0) {
+		dasd_put_device(device);
+		return -EINVAL;
+	}
+	spin_lock_irqsave(get_ccwdev_lock(to_ccwdev(dev)), flags);
+	if (val)
+		device->path_interval = val;
+	spin_unlock_irqrestore(get_ccwdev_lock(to_ccwdev(dev)), flags);
+	dasd_put_device(device);
+	return count;
+}
+
+static DEVICE_ATTR(path_interval, 0644, dasd_path_interval_show,
+		   dasd_path_interval_store);
+
+
 static struct attribute * dasd_attrs[] = {
 	&dev_attr_readonly.attr,
 	&dev_attr_discipline.attr,
@@ -1500,6 +1612,10 @@ static struct attribute * dasd_attrs[] = {
 	&dev_attr_safe_offline.attr,
 	&dev_attr_host_access_count.attr,
 	&dev_attr_path_masks.attr,
+	&dev_attr_path_threshold.attr,
+	&dev_attr_path_interval.attr,
+	&dev_attr_path_reset.attr,
+	&dev_attr_hpf.attr,
 	NULL,
 };
 
@@ -1531,7 +1647,7 @@ dasd_set_feature(struct ccw_device *cdev, int feature, int flag)
 {
 	struct dasd_devmap *devmap;
 
-	devmap = dasd_find_busid(dev_name(&cdev->dev));
+	devmap = dasd_devmap_from_cdev(cdev);
 	if (IS_ERR(devmap))
 		return PTR_ERR(devmap);
 
diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c
index a7a88476e215..67bf50c9946f 100644
--- a/drivers/s390/block/dasd_eckd.c
+++ b/drivers/s390/block/dasd_eckd.c
@@ -1042,8 +1042,11 @@ static void dasd_eckd_clear_conf_data(struct dasd_device *device)
 	private->conf_data = NULL;
 	private->conf_len = 0;
 	for (i = 0; i < 8; i++) {
-		kfree(private->path_conf_data[i]);
-		private->path_conf_data[i] = NULL;
+		kfree(device->path[i].conf_data);
+		device->path[i].conf_data = NULL;
+		device->path[i].cssid = 0;
+		device->path[i].ssid = 0;
+		device->path[i].chpid = 0;
 	}
 }
 
@@ -1055,13 +1058,14 @@ static int dasd_eckd_read_conf(struct dasd_device *device)
 	int rc, path_err, pos;
 	__u8 lpm, opm;
 	struct dasd_eckd_private *private, path_private;
-	struct dasd_path *path_data;
 	struct dasd_uid *uid;
 	char print_path_uid[60], print_device_uid[60];
+	struct channel_path_desc *chp_desc;
+	struct subchannel_id sch_id;
 
 	private = device->private;
-	path_data = &device->path_data;
 	opm = ccw_device_get_path_mask(device->cdev);
+	ccw_device_get_schid(device->cdev, &sch_id);
 	conf_data_saved = 0;
 	path_err = 0;
 	/* get configuration data per operational path */
@@ -1081,7 +1085,7 @@ static int dasd_eckd_read_conf(struct dasd_device *device)
 					"No configuration data "
 					"retrieved");
 			/* no further analysis possible */
-			path_data->opm |= lpm;
+			dasd_path_add_opm(device, opm);
 			continue;	/* no error */
 		}
 		/* save first valid configuration data */
@@ -1098,8 +1102,13 @@ static int dasd_eckd_read_conf(struct dasd_device *device)
 			}
 			pos = pathmask_to_pos(lpm);
 			/* store per path conf_data */
-			private->path_conf_data[pos] =
-				(struct dasd_conf_data *) conf_data;
+			device->path[pos].conf_data = conf_data;
+			device->path[pos].cssid = sch_id.cssid;
+			device->path[pos].ssid = sch_id.ssid;
+			chp_desc = ccw_device_get_chp_desc(device->cdev, pos);
+			if (chp_desc)
+				device->path[pos].chpid = chp_desc->chpid;
+			kfree(chp_desc);
 			/*
 			 * build device UID that other path data
 			 * can be compared to it
@@ -1154,42 +1163,66 @@ static int dasd_eckd_read_conf(struct dasd_device *device)
 					"device %s instead of %s\n", lpm,
 					print_path_uid, print_device_uid);
 				path_err = -EINVAL;
-				path_data->cablepm |= lpm;
+				dasd_path_add_cablepm(device, lpm);
 				continue;
 			}
 			pos = pathmask_to_pos(lpm);
 			/* store per path conf_data */
-			private->path_conf_data[pos] =
-				(struct dasd_conf_data *) conf_data;
+			device->path[pos].conf_data = conf_data;
+			device->path[pos].cssid = sch_id.cssid;
+			device->path[pos].ssid = sch_id.ssid;
+			chp_desc = ccw_device_get_chp_desc(device->cdev, pos);
+			if (chp_desc)
+				device->path[pos].chpid = chp_desc->chpid;
+			kfree(chp_desc);
 			path_private.conf_data = NULL;
 			path_private.conf_len = 0;
 		}
 		switch (dasd_eckd_path_access(conf_data, conf_len)) {
 		case 0x02:
-			path_data->npm |= lpm;
+			dasd_path_add_nppm(device, lpm);
 			break;
 		case 0x03:
-			path_data->ppm |= lpm;
+			dasd_path_add_ppm(device, lpm);
 			break;
 		}
-		if (!path_data->opm) {
-			path_data->opm = lpm;
+		if (!dasd_path_get_opm(device)) {
+			dasd_path_set_opm(device, lpm);
 			dasd_generic_path_operational(device);
 		} else {
-			path_data->opm |= lpm;
+			dasd_path_add_opm(device, lpm);
 		}
-		/*
-		 * if the path is used
-		 * it should not be in one of the negative lists
-		 */
-		path_data->cablepm &= ~lpm;
-		path_data->hpfpm &= ~lpm;
-		path_data->cuirpm &= ~lpm;
 	}
 
 	return path_err;
 }
 
+static u32 get_fcx_max_data(struct dasd_device *device)
+{
+	struct dasd_eckd_private *private = device->private;
+	int fcx_in_css, fcx_in_gneq, fcx_in_features;
+	int tpm, mdc;
+
+	if (dasd_nofcx)
+		return 0;
+	/* is transport mode supported? */
+	fcx_in_css = css_general_characteristics.fcx;
+	fcx_in_gneq = private->gneq->reserved2[7] & 0x04;
+	fcx_in_features = private->features.feature[40] & 0x80;
+	tpm = fcx_in_css && fcx_in_gneq && fcx_in_features;
+
+	if (!tpm)
+		return 0;
+
+	mdc = ccw_device_get_mdc(device->cdev, 0);
+	if (mdc < 0) {
+		dev_warn(&device->cdev->dev, "Detecting the maximum supported data size for zHPF requests failed\n");
+		return 0;
+	} else {
+		return (u32)mdc * FCX_MAX_DATA_FACTOR;
+	}
+}
+
 static int verify_fcx_max_data(struct dasd_device *device, __u8 lpm)
 {
 	struct dasd_eckd_private *private = device->private;
@@ -1222,8 +1255,7 @@ static int rebuild_device_uid(struct dasd_device *device,
 			      struct path_verification_work_data *data)
 {
 	struct dasd_eckd_private *private = device->private;
-	struct dasd_path *path_data = &device->path_data;
-	__u8 lpm, opm = path_data->opm;
+	__u8 lpm, opm = dasd_path_get_opm(device);
 	int rc = -ENODEV;
 
 	for (lpm = 0x80; lpm; lpm >>= 1) {
@@ -1356,7 +1388,7 @@ static void do_path_verification_work(struct work_struct *work)
 		 * in other case the device UID may have changed and
 		 * the first working path UID will be used as device UID
 		 */
-		if (device->path_data.opm &&
+		if (dasd_path_get_opm(device) &&
 		    dasd_eckd_compare_path_uid(device, &path_private)) {
 			/*
 			 * the comparison was not successful
@@ -1406,23 +1438,17 @@ static void do_path_verification_work(struct work_struct *work)
 		 * situation in dasd_start_IO.
 		 */
 		spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
-		if (!device->path_data.opm && opm) {
-			device->path_data.opm = opm;
-			device->path_data.cablepm &= ~opm;
-			device->path_data.cuirpm &= ~opm;
-			device->path_data.hpfpm &= ~opm;
+		if (!dasd_path_get_opm(device) && opm) {
+			dasd_path_set_opm(device, opm);
 			dasd_generic_path_operational(device);
 		} else {
-			device->path_data.opm |= opm;
-			device->path_data.cablepm &= ~opm;
-			device->path_data.cuirpm &= ~opm;
-			device->path_data.hpfpm &= ~opm;
+			dasd_path_add_opm(device, opm);
 		}
-		device->path_data.npm |= npm;
-		device->path_data.ppm |= ppm;
-		device->path_data.tbvpm |= epm;
-		device->path_data.cablepm |= cablepm;
-		device->path_data.hpfpm |= hpfpm;
+		dasd_path_add_nppm(device, npm);
+		dasd_path_add_ppm(device, ppm);
+		dasd_path_add_tbvpm(device, epm);
+		dasd_path_add_cablepm(device, cablepm);
+		dasd_path_add_nohpfpm(device, hpfpm);
 		spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
 	}
 	clear_bit(DASD_FLAG_PATH_VERIFY, &device->flags);
@@ -1456,6 +1482,19 @@ static int dasd_eckd_verify_path(struct dasd_device *device, __u8 lpm)
 	return 0;
 }
 
+static void dasd_eckd_reset_path(struct dasd_device *device, __u8 pm)
+{
+	struct dasd_eckd_private *private = device->private;
+	unsigned long flags;
+
+	if (!private->fcx_max_data)
+		private->fcx_max_data = get_fcx_max_data(device);
+	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
+	dasd_path_set_tbvpm(device, pm ? : dasd_path_get_notoperpm(device));
+	dasd_schedule_device_bh(device);
+	spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
+}
+
 static int dasd_eckd_read_features(struct dasd_device *device)
 {
 	struct dasd_eckd_private *private = device->private;
@@ -1652,32 +1691,6 @@ static void dasd_eckd_kick_validate_server(struct dasd_device *device)
 		dasd_put_device(device);
 }
 
-static u32 get_fcx_max_data(struct dasd_device *device)
-{
-	struct dasd_eckd_private *private = device->private;
-	int fcx_in_css, fcx_in_gneq, fcx_in_features;
-	int tpm, mdc;
-
-	if (dasd_nofcx)
-		return 0;
-	/* is transport mode supported? */
-	fcx_in_css = css_general_characteristics.fcx;
-	fcx_in_gneq = private->gneq->reserved2[7] & 0x04;
-	fcx_in_features = private->features.feature[40] & 0x80;
-	tpm = fcx_in_css && fcx_in_gneq && fcx_in_features;
-
-	if (!tpm)
-		return 0;
-
-	mdc = ccw_device_get_mdc(device->cdev, 0);
-	if (mdc < 0) {
-		dev_warn(&device->cdev->dev, "Detecting the maximum supported"
-			 " data size for zHPF requests failed\n");
-		return 0;
-	} else
-		return (u32)mdc * FCX_MAX_DATA_FACTOR;
-}
-
 /*
  * Check device characteristics.
  * If the device is accessible using ECKD discipline, the device is enabled.
@@ -1729,10 +1742,11 @@ dasd_eckd_check_characteristics(struct dasd_device *device)
 	if (rc)
 		goto out_err1;
 
-	/* set default timeout */
+	/* set some default values */
 	device->default_expires = DASD_EXPIRES;
-	/* set default retry count */
 	device->default_retries = DASD_RETRIES;
+	device->path_thrhld = DASD_ECKD_PATH_THRHLD;
+	device->path_interval = DASD_ECKD_PATH_INTERVAL;
 
 	if (private->gneq) {
 		value = 1;
@@ -1839,13 +1853,16 @@ static void dasd_eckd_uncheck_device(struct dasd_device *device)
 	private->gneq = NULL;
 	private->conf_len = 0;
 	for (i = 0; i < 8; i++) {
-		kfree(private->path_conf_data[i]);
-		if ((__u8 *)private->path_conf_data[i] ==
+		kfree(device->path[i].conf_data);
+		if ((__u8 *)device->path[i].conf_data ==
 		    private->conf_data) {
 			private->conf_data = NULL;
 			private->conf_len = 0;
 		}
-		private->path_conf_data[i] = NULL;
+		device->path[i].conf_data = NULL;
+		device->path[i].cssid = 0;
+		device->path[i].ssid = 0;
+		device->path[i].chpid = 0;
 	}
 	kfree(private->conf_data);
 	private->conf_data = NULL;
@@ -2966,7 +2983,7 @@ static void dasd_eckd_handle_terminated_request(struct dasd_ccw_req *cqr)
 	if (cqr->block && (cqr->startdev != cqr->block->base)) {
 		dasd_eckd_reset_ccw_to_base_io(cqr);
 		cqr->startdev = cqr->block->base;
-		cqr->lpm = cqr->block->base->path_data.opm;
+		cqr->lpm = dasd_path_get_opm(cqr->block->base);
 	}
 };
 
@@ -3251,7 +3268,7 @@ static struct dasd_ccw_req *dasd_eckd_build_cp_cmd_single(
 	cqr->memdev = startdev;
 	cqr->block = block;
 	cqr->expires = startdev->default_expires * HZ;	/* default 5 minutes */
-	cqr->lpm = startdev->path_data.ppm;
+	cqr->lpm = dasd_path_get_ppm(startdev);
 	cqr->retries = startdev->default_retries;
 	cqr->buildclk = get_tod_clock();
 	cqr->status = DASD_CQR_FILLED;
@@ -3426,7 +3443,7 @@ static struct dasd_ccw_req *dasd_eckd_build_cp_cmd_track(
 	cqr->memdev = startdev;
 	cqr->block = block;
 	cqr->expires = startdev->default_expires * HZ;	/* default 5 minutes */
-	cqr->lpm = startdev->path_data.ppm;
+	cqr->lpm = dasd_path_get_ppm(startdev);
 	cqr->retries = startdev->default_retries;
 	cqr->buildclk = get_tod_clock();
 	cqr->status = DASD_CQR_FILLED;
@@ -3735,7 +3752,7 @@ static struct dasd_ccw_req *dasd_eckd_build_cp_tpm_track(
 	cqr->memdev = startdev;
 	cqr->block = block;
 	cqr->expires = startdev->default_expires * HZ;	/* default 5 minutes */
-	cqr->lpm = startdev->path_data.ppm;
+	cqr->lpm = dasd_path_get_ppm(startdev);
 	cqr->retries = startdev->default_retries;
 	cqr->buildclk = get_tod_clock();
 	cqr->status = DASD_CQR_FILLED;
@@ -3962,7 +3979,7 @@ static struct dasd_ccw_req *dasd_raw_build_cp(struct dasd_device *startdev,
 	cqr->memdev = startdev;
 	cqr->block = block;
 	cqr->expires = startdev->default_expires * HZ;
-	cqr->lpm = startdev->path_data.ppm;
+	cqr->lpm = dasd_path_get_ppm(startdev);
 	cqr->retries = startdev->default_retries;
 	cqr->buildclk = get_tod_clock();
 	cqr->status = DASD_CQR_FILLED;
@@ -4783,7 +4800,8 @@ static void dasd_eckd_dump_sense_tcw(struct dasd_device *device,
 		       req, scsw_cc(&irb->scsw), scsw_fctl(&irb->scsw),
 		       scsw_actl(&irb->scsw), scsw_stctl(&irb->scsw),
 		       scsw_dstat(&irb->scsw), scsw_cstat(&irb->scsw),
-		       irb->scsw.tm.fcxs, irb->scsw.tm.schxs,
+		       irb->scsw.tm.fcxs,
+		       (irb->scsw.tm.ifob << 7) | irb->scsw.tm.sesq,
 		       req ? req->intrc : 0);
 	len += sprintf(page + len, PRINTK_HEADER
 		       " device %s: Failing TCW: %p\n",
@@ -5306,11 +5324,10 @@ static int dasd_hosts_print(struct dasd_device *device, struct seq_file *m)
  */
 static int
 dasd_eckd_psf_cuir_response(struct dasd_device *device, int response,
-			    __u32 message_id,
-			    struct channel_path_desc *desc,
-			    struct subchannel_id sch_id)
+			    __u32 message_id, __u8 lpum)
 {
 	struct dasd_psf_cuir_response *psf_cuir;
+	int pos = pathmask_to_pos(lpum);
 	struct dasd_ccw_req *cqr;
 	struct ccw1 *ccw;
 	int rc;
@@ -5328,11 +5345,10 @@ dasd_eckd_psf_cuir_response(struct dasd_device *device, int response,
 	psf_cuir = (struct dasd_psf_cuir_response *)cqr->data;
 	psf_cuir->order = PSF_ORDER_CUIR_RESPONSE;
 	psf_cuir->cc = response;
-	if (desc)
-		psf_cuir->chpid = desc->chpid;
+	psf_cuir->chpid = device->path[pos].chpid;
 	psf_cuir->message_id = message_id;
-	psf_cuir->cssid = sch_id.cssid;
-	psf_cuir->ssid = sch_id.ssid;
+	psf_cuir->cssid = device->path[pos].cssid;
+	psf_cuir->ssid = device->path[pos].ssid;
 	ccw = cqr->cpaddr;
 	ccw->cmd_code = DASD_ECKD_CCW_PSF;
 	ccw->cda = (__u32)(addr_t)psf_cuir;
@@ -5363,20 +5379,19 @@ static struct dasd_conf_data *dasd_eckd_get_ref_conf(struct dasd_device *device,
 						     __u8 lpum,
 						     struct dasd_cuir_message *cuir)
 {
-	struct dasd_eckd_private *private = device->private;
 	struct dasd_conf_data *conf_data;
 	int path, pos;
 
 	if (cuir->record_selector == 0)
 		goto out;
 	for (path = 0x80, pos = 0; path; path >>= 1, pos++) {
-		conf_data = private->path_conf_data[pos];
+		conf_data = device->path[pos].conf_data;
 		if (conf_data->gneq.record_selector ==
 		    cuir->record_selector)
 			return conf_data;
 	}
 out:
-	return private->path_conf_data[pathmask_to_pos(lpum)];
+	return device->path[pathmask_to_pos(lpum)].conf_data;
 }
 
 /*
@@ -5391,7 +5406,6 @@ out:
 static int dasd_eckd_cuir_scope(struct dasd_device *device, __u8 lpum,
 				struct dasd_cuir_message *cuir)
 {
-	struct dasd_eckd_private *private = device->private;
 	struct dasd_conf_data *ref_conf_data;
 	unsigned long bitmask = 0, mask = 0;
 	struct dasd_conf_data *conf_data;
@@ -5417,11 +5431,10 @@ static int dasd_eckd_cuir_scope(struct dasd_device *device, __u8 lpum,
 	mask |= cuir->neq_map[1] << 8;
 	mask |= cuir->neq_map[0] << 16;
 
-	for (path = 0x80; path; path >>= 1) {
+	for (path = 0; path < 8; path++) {
 		/* initialise data per path */
 		bitmask = mask;
-		pos = pathmask_to_pos(path);
-		conf_data = private->path_conf_data[pos];
+		conf_data = device->path[path].conf_data;
 		pos = 8 - ffs(cuir->ned_map);
 		ned = (char *) &conf_data->neds[pos];
 		/* compare reference ned and per path ned */
@@ -5442,33 +5455,29 @@ static int dasd_eckd_cuir_scope(struct dasd_device *device, __u8 lpum,
 			continue;
 		/* device and path match the reference values
 		   add path to CUIR scope */
-		tbcpm |= path;
+		tbcpm |= 0x80 >> path;
 	}
 	return tbcpm;
 }
 
 static void dasd_eckd_cuir_notify_user(struct dasd_device *device,
-				       unsigned long paths,
-				       struct subchannel_id sch_id, int action)
+				       unsigned long paths, int action)
 {
-	struct channel_path_desc *desc;
 	int pos;
 
 	while (paths) {
 		/* get position of bit in mask */
-		pos = ffs(paths) - 1;
+		pos = 8 - ffs(paths);
 		/* get channel path descriptor from this position */
-		desc = ccw_device_get_chp_desc(device->cdev, 7 - pos);
 		if (action == CUIR_QUIESCE)
-			pr_warn("Service on the storage server caused path "
-				"%x.%02x to go offline", sch_id.cssid,
-				desc ? desc->chpid : 0);
+			pr_warn("Service on the storage server caused path %x.%02x to go offline",
+				device->path[pos].cssid,
+				device->path[pos].chpid);
 		else if (action == CUIR_RESUME)
-			pr_info("Path %x.%02x is back online after service "
-				"on the storage server", sch_id.cssid,
-				desc ? desc->chpid : 0);
-		kfree(desc);
-		clear_bit(pos, &paths);
+			pr_info("Path %x.%02x is back online after service on the storage server",
+				device->path[pos].cssid,
+				device->path[pos].chpid);
+		clear_bit(7 - pos, &paths);
 	}
 }
 
@@ -5479,16 +5488,16 @@ static int dasd_eckd_cuir_remove_path(struct dasd_device *device, __u8 lpum,
 
 	tbcpm = dasd_eckd_cuir_scope(device, lpum, cuir);
 	/* nothing to do if path is not in use */
-	if (!(device->path_data.opm & tbcpm))
+	if (!(dasd_path_get_opm(device) & tbcpm))
 		return 0;
-	if (!(device->path_data.opm & ~tbcpm)) {
+	if (!(dasd_path_get_opm(device) & ~tbcpm)) {
 		/* no path would be left if the CUIR action is taken
 		   return error */
 		return -EINVAL;
 	}
 	/* remove device from operational path mask */
-	device->path_data.opm &= ~tbcpm;
-	device->path_data.cuirpm |= tbcpm;
+	dasd_path_remove_opm(device, tbcpm);
+	dasd_path_add_cuirpm(device, tbcpm);
 	return tbcpm;
 }
 
@@ -5501,7 +5510,6 @@ static int dasd_eckd_cuir_remove_path(struct dasd_device *device, __u8 lpum,
  * notify the already set offline devices again
  */
 static int dasd_eckd_cuir_quiesce(struct dasd_device *device, __u8 lpum,
-				  struct subchannel_id sch_id,
 				  struct dasd_cuir_message *cuir)
 {
 	struct dasd_eckd_private *private = device->private;
@@ -5556,14 +5564,13 @@ static int dasd_eckd_cuir_quiesce(struct dasd_device *device, __u8 lpum,
 		}
 	}
 	/* notify user about all paths affected by CUIR action */
-	dasd_eckd_cuir_notify_user(device, paths, sch_id, CUIR_QUIESCE);
+	dasd_eckd_cuir_notify_user(device, paths, CUIR_QUIESCE);
 	return 0;
 out_err:
 	return tbcpm;
 }
 
 static int dasd_eckd_cuir_resume(struct dasd_device *device, __u8 lpum,
-				 struct subchannel_id sch_id,
 				 struct dasd_cuir_message *cuir)
 {
 	struct dasd_eckd_private *private = device->private;
@@ -5581,8 +5588,8 @@ static int dasd_eckd_cuir_resume(struct dasd_device *device, __u8 lpum,
 				 alias_list) {
 		tbcpm = dasd_eckd_cuir_scope(dev, lpum, cuir);
 		paths |= tbcpm;
-		if (!(dev->path_data.opm & tbcpm)) {
-			dev->path_data.tbvpm |= tbcpm;
+		if (!(dasd_path_get_opm(dev) & tbcpm)) {
+			dasd_path_add_tbvpm(dev, tbcpm);
 			dasd_schedule_device_bh(dev);
 		}
 	}
@@ -5591,8 +5598,8 @@ static int dasd_eckd_cuir_resume(struct dasd_device *device, __u8 lpum,
 				 alias_list) {
 		tbcpm = dasd_eckd_cuir_scope(dev, lpum, cuir);
 		paths |= tbcpm;
-		if (!(dev->path_data.opm & tbcpm)) {
-			dev->path_data.tbvpm |= tbcpm;
+		if (!(dasd_path_get_opm(dev) & tbcpm)) {
+			dasd_path_add_tbvpm(dev, tbcpm);
 			dasd_schedule_device_bh(dev);
 		}
 	}
@@ -5605,8 +5612,8 @@ static int dasd_eckd_cuir_resume(struct dasd_device *device, __u8 lpum,
 					 alias_list) {
 			tbcpm = dasd_eckd_cuir_scope(dev, lpum, cuir);
 			paths |= tbcpm;
-			if (!(dev->path_data.opm & tbcpm)) {
-				dev->path_data.tbvpm |= tbcpm;
+			if (!(dasd_path_get_opm(dev) & tbcpm)) {
+				dasd_path_add_tbvpm(dev, tbcpm);
 				dasd_schedule_device_bh(dev);
 			}
 		}
@@ -5615,14 +5622,14 @@ static int dasd_eckd_cuir_resume(struct dasd_device *device, __u8 lpum,
 					 alias_list) {
 			tbcpm = dasd_eckd_cuir_scope(dev, lpum, cuir);
 			paths |= tbcpm;
-			if (!(dev->path_data.opm & tbcpm)) {
-				dev->path_data.tbvpm |= tbcpm;
+			if (!(dasd_path_get_opm(dev) & tbcpm)) {
+				dasd_path_add_tbvpm(dev, tbcpm);
 				dasd_schedule_device_bh(dev);
 			}
 		}
 	}
 	/* notify user about all paths affected by CUIR action */
-	dasd_eckd_cuir_notify_user(device, paths, sch_id, CUIR_RESUME);
+	dasd_eckd_cuir_notify_user(device, paths, CUIR_RESUME);
 	return 0;
 }
 
@@ -5630,38 +5637,31 @@ static void dasd_eckd_handle_cuir(struct dasd_device *device, void *messages,
 				 __u8 lpum)
 {
 	struct dasd_cuir_message *cuir = messages;
-	struct channel_path_desc *desc;
-	struct subchannel_id sch_id;
-	int pos, response;
+	int response;
 
 	DBF_DEV_EVENT(DBF_WARNING, device,
 		      "CUIR request: %016llx %016llx %016llx %08x",
 		      ((u64 *)cuir)[0], ((u64 *)cuir)[1], ((u64 *)cuir)[2],
 		      ((u32 *)cuir)[3]);
-	ccw_device_get_schid(device->cdev, &sch_id);
-	pos = pathmask_to_pos(lpum);
-	desc = ccw_device_get_chp_desc(device->cdev, pos);
 
 	if (cuir->code == CUIR_QUIESCE) {
 		/* quiesce */
-		if (dasd_eckd_cuir_quiesce(device, lpum, sch_id, cuir))
+		if (dasd_eckd_cuir_quiesce(device, lpum, cuir))
 			response = PSF_CUIR_LAST_PATH;
 		else
 			response = PSF_CUIR_COMPLETED;
 	} else if (cuir->code == CUIR_RESUME) {
 		/* resume */
-		dasd_eckd_cuir_resume(device, lpum, sch_id, cuir);
+		dasd_eckd_cuir_resume(device, lpum, cuir);
 		response = PSF_CUIR_COMPLETED;
 	} else
 		response = PSF_CUIR_NOT_SUPPORTED;
 
 	dasd_eckd_psf_cuir_response(device, response,
-				    cuir->message_id, desc, sch_id);
+				    cuir->message_id, lpum);
 	DBF_DEV_EVENT(DBF_WARNING, device,
 		      "CUIR response: %d on message ID %08x", response,
 		      cuir->message_id);
-	/* free descriptor copy */
-	kfree(desc);
 	/* to make sure there is no attention left schedule work again */
 	device->discipline->check_attention(device, lpum);
 }
@@ -5708,6 +5708,63 @@ static int dasd_eckd_check_attention(struct dasd_device *device, __u8 lpum)
 	return 0;
 }
 
+static int dasd_eckd_disable_hpf_path(struct dasd_device *device, __u8 lpum)
+{
+	if (~lpum & dasd_path_get_opm(device)) {
+		dasd_path_add_nohpfpm(device, lpum);
+		dasd_path_remove_opm(device, lpum);
+		dev_err(&device->cdev->dev,
+			"Channel path %02X lost HPF functionality and is disabled\n",
+			lpum);
+		return 1;
+	}
+	return 0;
+}
+
+static void dasd_eckd_disable_hpf_device(struct dasd_device *device)
+{
+	struct dasd_eckd_private *private = device->private;
+
+	dev_err(&device->cdev->dev,
+		"High Performance FICON disabled\n");
+	private->fcx_max_data = 0;
+}
+
+static int dasd_eckd_hpf_enabled(struct dasd_device *device)
+{
+	struct dasd_eckd_private *private = device->private;
+
+	return private->fcx_max_data ? 1 : 0;
+}
+
+static void dasd_eckd_handle_hpf_error(struct dasd_device *device,
+				       struct irb *irb)
+{
+	struct dasd_eckd_private *private = device->private;
+
+	if (!private->fcx_max_data) {
+		/* sanity check for no HPF, the error makes no sense */
+		DBF_DEV_EVENT(DBF_WARNING, device, "%s",
+			      "Trying to disable HPF for a non HPF device");
+		return;
+	}
+	if (irb->scsw.tm.sesq == SCSW_SESQ_DEV_NOFCX) {
+		dasd_eckd_disable_hpf_device(device);
+	} else if (irb->scsw.tm.sesq == SCSW_SESQ_PATH_NOFCX) {
+		if (dasd_eckd_disable_hpf_path(device, irb->esw.esw1.lpum))
+			return;
+		dasd_eckd_disable_hpf_device(device);
+		dasd_path_set_tbvpm(device,
+				  dasd_path_get_hpfpm(device));
+	}
+	/*
+	 * prevent that any new I/O ist started on the device and schedule a
+	 * requeue of existing requests
+	 */
+	dasd_device_set_stop_bits(device, DASD_STOPPED_NOT_ACC);
+	dasd_schedule_requeue(device);
+}
+
 static struct ccw_driver dasd_eckd_driver = {
 	.driver = {
 		.name	= "dasd-eckd",
@@ -5776,6 +5833,10 @@ static struct dasd_discipline dasd_eckd_discipline = {
 	.check_attention = dasd_eckd_check_attention,
 	.host_access_count = dasd_eckd_host_access_count,
 	.hosts_print = dasd_hosts_print,
+	.handle_hpf_error = dasd_eckd_handle_hpf_error,
+	.disable_hpf = dasd_eckd_disable_hpf_device,
+	.hpf_enabled = dasd_eckd_hpf_enabled,
+	.reset_path = dasd_eckd_reset_path,
 };
 
 static int __init
diff --git a/drivers/s390/block/dasd_eckd.h b/drivers/s390/block/dasd_eckd.h
index 59803626ea36..e2a710c250a5 100644
--- a/drivers/s390/block/dasd_eckd.h
+++ b/drivers/s390/block/dasd_eckd.h
@@ -94,6 +94,8 @@
 #define FCX_MAX_DATA_FACTOR 65536
 #define DASD_ECKD_RCD_DATA_SIZE 256
 
+#define DASD_ECKD_PATH_THRHLD		 256
+#define DASD_ECKD_PATH_INTERVAL		 300
 
 /*****************************************************************************
  * SECTION: Type Definitions
@@ -535,8 +537,7 @@ struct dasd_eckd_private {
 	struct dasd_eckd_characteristics rdc_data;
 	u8 *conf_data;
 	int conf_len;
-	/* per path configuration data */
-	struct dasd_conf_data *path_conf_data[8];
+
 	/* pointers to specific parts in the conf_data */
 	struct dasd_ned *ned;
 	struct dasd_sneq *sneq;
diff --git a/drivers/s390/block/dasd_eer.c b/drivers/s390/block/dasd_eer.c
index 21ef63cf0960..6c5d671304b4 100644
--- a/drivers/s390/block/dasd_eer.c
+++ b/drivers/s390/block/dasd_eer.c
@@ -454,20 +454,30 @@ static void dasd_eer_snss_cb(struct dasd_ccw_req *cqr, void *data)
  */
 int dasd_eer_enable(struct dasd_device *device)
 {
-	struct dasd_ccw_req *cqr;
+	struct dasd_ccw_req *cqr = NULL;
 	unsigned long flags;
 	struct ccw1 *ccw;
+	int rc = 0;
 
+	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
 	if (device->eer_cqr)
-		return 0;
+		goto out;
+	else if (!device->discipline ||
+		 strcmp(device->discipline->name, "ECKD"))
+		rc = -EMEDIUMTYPE;
+	else if (test_bit(DASD_FLAG_OFFLINE, &device->flags))
+		rc = -EBUSY;
 
-	if (!device->discipline || strcmp(device->discipline->name, "ECKD"))
-		return -EPERM;	/* FIXME: -EMEDIUMTYPE ? */
+	if (rc)
+		goto out;
 
 	cqr = dasd_kmalloc_request(DASD_ECKD_MAGIC, 1 /* SNSS */,
 				   SNSS_DATA_SIZE, device);
-	if (IS_ERR(cqr))
-		return -ENOMEM;
+	if (IS_ERR(cqr)) {
+		rc = -ENOMEM;
+		cqr = NULL;
+		goto out;
+	}
 
 	cqr->startdev = device;
 	cqr->retries = 255;
@@ -485,15 +495,18 @@ int dasd_eer_enable(struct dasd_device *device)
 	cqr->status = DASD_CQR_FILLED;
 	cqr->callback = dasd_eer_snss_cb;
 
-	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
 	if (!device->eer_cqr) {
 		device->eer_cqr = cqr;
 		cqr = NULL;
 	}
+
+out:
 	spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
+
 	if (cqr)
 		dasd_kfree_request(cqr, device);
-	return 0;
+
+	return rc;
 }
 
 /*
diff --git a/drivers/s390/block/dasd_erp.c b/drivers/s390/block/dasd_erp.c
index d138d0116734..113c1c1fa1af 100644
--- a/drivers/s390/block/dasd_erp.c
+++ b/drivers/s390/block/dasd_erp.c
@@ -96,7 +96,7 @@ dasd_default_erp_action(struct dasd_ccw_req *cqr)
                              "default ERP called (%i retries left)",
                              cqr->retries);
 		if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags))
-			cqr->lpm = device->path_data.opm;
+			cqr->lpm = dasd_path_get_opm(device);
 		cqr->status = DASD_CQR_FILLED;
         } else {
 		pr_err("%s: default ERP has run out of retries and failed\n",
diff --git a/drivers/s390/block/dasd_fba.c b/drivers/s390/block/dasd_fba.c
index d7b5b550364b..462cab5d4302 100644
--- a/drivers/s390/block/dasd_fba.c
+++ b/drivers/s390/block/dasd_fba.c
@@ -168,7 +168,7 @@ dasd_fba_check_characteristics(struct dasd_device *device)
 
 	device->default_expires = DASD_EXPIRES;
 	device->default_retries = FBA_DEFAULT_RETRIES;
-	device->path_data.opm = LPM_ANYPATH;
+	dasd_path_set_opm(device, LPM_ANYPATH);
 
 	readonly = dasd_device_is_ro(device);
 	if (readonly)
diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h
index 87ff6cef872f..24be210c10e5 100644
--- a/drivers/s390/block/dasd_int.h
+++ b/drivers/s390/block/dasd_int.h
@@ -55,6 +55,7 @@
 #include <asm/debug.h>
 #include <asm/dasd.h>
 #include <asm/idals.h>
+#include <linux/bitops.h>
 
 /* DASD discipline magic */
 #define DASD_ECKD_MAGIC 0xC5C3D2C4
@@ -377,6 +378,10 @@ struct dasd_discipline {
 	int (*check_attention)(struct dasd_device *, __u8);
 	int (*host_access_count)(struct dasd_device *);
 	int (*hosts_print)(struct dasd_device *, struct seq_file *);
+	void (*handle_hpf_error)(struct dasd_device *, struct irb *);
+	void (*disable_hpf)(struct dasd_device *);
+	int (*hpf_enabled)(struct dasd_device *);
+	void (*reset_path)(struct dasd_device *, __u8);
 };
 
 extern struct dasd_discipline *dasd_diag_discipline_pointer;
@@ -397,17 +402,31 @@ extern struct dasd_discipline *dasd_diag_discipline_pointer;
 #define DASD_EER_STATECHANGE 3
 #define DASD_EER_PPRCSUSPEND 4
 
+/* DASD path handling */
+
+#define DASD_PATH_OPERATIONAL  1
+#define DASD_PATH_TBV	       2
+#define DASD_PATH_PP	       3
+#define DASD_PATH_NPP	       4
+#define DASD_PATH_MISCABLED    5
+#define DASD_PATH_NOHPF        6
+#define DASD_PATH_CUIR	       7
+#define DASD_PATH_IFCC	       8
+
+#define DASD_THRHLD_MAX		4294967295U
+#define DASD_INTERVAL_MAX	4294967295U
+
 struct dasd_path {
-	__u8 opm;
-	__u8 tbvpm;
-	__u8 ppm;
-	__u8 npm;
-	/* paths that are not used because of a special condition */
-	__u8 cablepm; /* miss-cabled */
-	__u8 hpfpm;   /* the HPF requirements of the other paths are not met */
-	__u8 cuirpm;  /* CUIR varied offline */
+	unsigned long flags;
+	u8 cssid;
+	u8 ssid;
+	u8 chpid;
+	struct dasd_conf_data *conf_data;
+	atomic_t error_count;
+	unsigned long long errorclk;
 };
 
+
 struct dasd_profile_info {
 	/* legacy part of profile data, as in dasd_profile_info_t */
 	unsigned int dasd_io_reqs;	 /* number of requests processed */
@@ -458,7 +477,8 @@ struct dasd_device {
 	struct dasd_discipline *discipline;
 	struct dasd_discipline *base_discipline;
 	void *private;
-	struct dasd_path path_data;
+	struct dasd_path path[8];
+	__u8 opm;
 
 	/* Device state and target state. */
 	int state, target;
@@ -483,6 +503,7 @@ struct dasd_device {
 	struct work_struct reload_device;
 	struct work_struct kick_validate;
 	struct work_struct suc_work;
+	struct work_struct requeue_requests;
 	struct timer_list timer;
 
 	debug_info_t *debug_area;
@@ -498,6 +519,9 @@ struct dasd_device {
 
 	unsigned long blk_timeout;
 
+	unsigned long path_thrhld;
+	unsigned long path_interval;
+
 	struct dentry *debugfs_dentry;
 	struct dentry *hosts_dentry;
 	struct dasd_profile profile;
@@ -707,6 +731,7 @@ void dasd_set_target_state(struct dasd_device *, int);
 void dasd_kick_device(struct dasd_device *);
 void dasd_restore_device(struct dasd_device *);
 void dasd_reload_device(struct dasd_device *);
+void dasd_schedule_requeue(struct dasd_device *);
 
 void dasd_add_request_head(struct dasd_ccw_req *);
 void dasd_add_request_tail(struct dasd_ccw_req *);
@@ -835,4 +860,410 @@ static inline int dasd_eer_enabled(struct dasd_device *device)
 #define dasd_eer_enabled(d)	(0)
 #endif	/* CONFIG_DASD_ERR */
 
+
+/* DASD path handling functions */
+
+/*
+ * helper functions to modify bit masks for a given channel path for a device
+ */
+static inline int dasd_path_is_operational(struct dasd_device *device, int chp)
+{
+	return test_bit(DASD_PATH_OPERATIONAL, &device->path[chp].flags);
+}
+
+static inline int dasd_path_need_verify(struct dasd_device *device, int chp)
+{
+	return test_bit(DASD_PATH_TBV, &device->path[chp].flags);
+}
+
+static inline void dasd_path_verify(struct dasd_device *device, int chp)
+{
+	__set_bit(DASD_PATH_TBV, &device->path[chp].flags);
+}
+
+static inline void dasd_path_clear_verify(struct dasd_device *device, int chp)
+{
+	__clear_bit(DASD_PATH_TBV, &device->path[chp].flags);
+}
+
+static inline void dasd_path_clear_all_verify(struct dasd_device *device)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++)
+		dasd_path_clear_verify(device, chp);
+}
+
+static inline void dasd_path_operational(struct dasd_device *device, int chp)
+{
+	__set_bit(DASD_PATH_OPERATIONAL, &device->path[chp].flags);
+	device->opm |= (0x80 >> chp);
+}
+
+static inline void dasd_path_nonpreferred(struct dasd_device *device, int chp)
+{
+	__set_bit(DASD_PATH_NPP, &device->path[chp].flags);
+}
+
+static inline int dasd_path_is_nonpreferred(struct dasd_device *device, int chp)
+{
+	return test_bit(DASD_PATH_NPP, &device->path[chp].flags);
+}
+
+static inline void dasd_path_clear_nonpreferred(struct dasd_device *device,
+						int chp)
+{
+	__clear_bit(DASD_PATH_NPP, &device->path[chp].flags);
+}
+
+static inline void dasd_path_preferred(struct dasd_device *device, int chp)
+{
+	__set_bit(DASD_PATH_PP, &device->path[chp].flags);
+}
+
+static inline int dasd_path_is_preferred(struct dasd_device *device, int chp)
+{
+	return test_bit(DASD_PATH_PP, &device->path[chp].flags);
+}
+
+static inline void dasd_path_clear_preferred(struct dasd_device *device,
+					     int chp)
+{
+	__clear_bit(DASD_PATH_PP, &device->path[chp].flags);
+}
+
+static inline void dasd_path_clear_oper(struct dasd_device *device, int chp)
+{
+	__clear_bit(DASD_PATH_OPERATIONAL, &device->path[chp].flags);
+	device->opm &= ~(0x80 >> chp);
+}
+
+static inline void dasd_path_clear_cable(struct dasd_device *device, int chp)
+{
+	__clear_bit(DASD_PATH_MISCABLED, &device->path[chp].flags);
+}
+
+static inline void dasd_path_cuir(struct dasd_device *device, int chp)
+{
+	__set_bit(DASD_PATH_CUIR, &device->path[chp].flags);
+}
+
+static inline int dasd_path_is_cuir(struct dasd_device *device, int chp)
+{
+	return test_bit(DASD_PATH_CUIR, &device->path[chp].flags);
+}
+
+static inline void dasd_path_clear_cuir(struct dasd_device *device, int chp)
+{
+	__clear_bit(DASD_PATH_CUIR, &device->path[chp].flags);
+}
+
+static inline void dasd_path_ifcc(struct dasd_device *device, int chp)
+{
+	set_bit(DASD_PATH_IFCC, &device->path[chp].flags);
+}
+
+static inline int dasd_path_is_ifcc(struct dasd_device *device, int chp)
+{
+	return test_bit(DASD_PATH_IFCC, &device->path[chp].flags);
+}
+
+static inline void dasd_path_clear_ifcc(struct dasd_device *device, int chp)
+{
+	clear_bit(DASD_PATH_IFCC, &device->path[chp].flags);
+}
+
+static inline void dasd_path_clear_nohpf(struct dasd_device *device, int chp)
+{
+	__clear_bit(DASD_PATH_NOHPF, &device->path[chp].flags);
+}
+
+static inline void dasd_path_miscabled(struct dasd_device *device, int chp)
+{
+	__set_bit(DASD_PATH_MISCABLED, &device->path[chp].flags);
+}
+
+static inline int dasd_path_is_miscabled(struct dasd_device *device, int chp)
+{
+	return test_bit(DASD_PATH_MISCABLED, &device->path[chp].flags);
+}
+
+static inline void dasd_path_nohpf(struct dasd_device *device, int chp)
+{
+	__set_bit(DASD_PATH_NOHPF, &device->path[chp].flags);
+}
+
+static inline int dasd_path_is_nohpf(struct dasd_device *device, int chp)
+{
+	return test_bit(DASD_PATH_NOHPF, &device->path[chp].flags);
+}
+
+/*
+ * get functions for path masks
+ * will return a path masks for the given device
+ */
+
+static inline __u8 dasd_path_get_opm(struct dasd_device *device)
+{
+	return device->opm;
+}
+
+static inline __u8 dasd_path_get_tbvpm(struct dasd_device *device)
+{
+	int chp;
+	__u8 tbvpm = 0x00;
+
+	for (chp = 0; chp < 8; chp++)
+		if (dasd_path_need_verify(device, chp))
+			tbvpm |= 0x80 >> chp;
+	return tbvpm;
+}
+
+static inline __u8 dasd_path_get_nppm(struct dasd_device *device)
+{
+	int chp;
+	__u8 npm = 0x00;
+
+	for (chp = 0; chp < 8; chp++) {
+		if (dasd_path_is_nonpreferred(device, chp))
+			npm |= 0x80 >> chp;
+	}
+	return npm;
+}
+
+static inline __u8 dasd_path_get_ppm(struct dasd_device *device)
+{
+	int chp;
+	__u8 ppm = 0x00;
+
+	for (chp = 0; chp < 8; chp++)
+		if (dasd_path_is_preferred(device, chp))
+			ppm |= 0x80 >> chp;
+	return ppm;
+}
+
+static inline __u8 dasd_path_get_cablepm(struct dasd_device *device)
+{
+	int chp;
+	__u8 cablepm = 0x00;
+
+	for (chp = 0; chp < 8; chp++)
+		if (dasd_path_is_miscabled(device, chp))
+			cablepm |= 0x80 >> chp;
+	return cablepm;
+}
+
+static inline __u8 dasd_path_get_cuirpm(struct dasd_device *device)
+{
+	int chp;
+	__u8 cuirpm = 0x00;
+
+	for (chp = 0; chp < 8; chp++)
+		if (dasd_path_is_cuir(device, chp))
+			cuirpm |= 0x80 >> chp;
+	return cuirpm;
+}
+
+static inline __u8 dasd_path_get_ifccpm(struct dasd_device *device)
+{
+	int chp;
+	__u8 ifccpm = 0x00;
+
+	for (chp = 0; chp < 8; chp++)
+		if (dasd_path_is_ifcc(device, chp))
+			ifccpm |= 0x80 >> chp;
+	return ifccpm;
+}
+
+static inline __u8 dasd_path_get_hpfpm(struct dasd_device *device)
+{
+	int chp;
+	__u8 hpfpm = 0x00;
+
+	for (chp = 0; chp < 8; chp++)
+		if (dasd_path_is_nohpf(device, chp))
+			hpfpm |= 0x80 >> chp;
+	return hpfpm;
+}
+
+/*
+ * add functions for path masks
+ * the existing path mask will be extended by the given path mask
+ */
+static inline void dasd_path_add_tbvpm(struct dasd_device *device, __u8 pm)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++)
+		if (pm & (0x80 >> chp))
+			dasd_path_verify(device, chp);
+}
+
+static inline __u8 dasd_path_get_notoperpm(struct dasd_device *device)
+{
+	int chp;
+	__u8 nopm = 0x00;
+
+	for (chp = 0; chp < 8; chp++)
+		if (dasd_path_is_nohpf(device, chp) ||
+		    dasd_path_is_ifcc(device, chp) ||
+		    dasd_path_is_cuir(device, chp) ||
+		    dasd_path_is_miscabled(device, chp))
+			nopm |= 0x80 >> chp;
+	return nopm;
+}
+
+static inline void dasd_path_add_opm(struct dasd_device *device, __u8 pm)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++)
+		if (pm & (0x80 >> chp)) {
+			dasd_path_operational(device, chp);
+			/*
+			 * if the path is used
+			 * it should not be in one of the negative lists
+			 */
+			dasd_path_clear_nohpf(device, chp);
+			dasd_path_clear_cuir(device, chp);
+			dasd_path_clear_cable(device, chp);
+			dasd_path_clear_ifcc(device, chp);
+		}
+}
+
+static inline void dasd_path_add_cablepm(struct dasd_device *device, __u8 pm)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++)
+		if (pm & (0x80 >> chp))
+			dasd_path_miscabled(device, chp);
+}
+
+static inline void dasd_path_add_cuirpm(struct dasd_device *device, __u8 pm)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++)
+		if (pm & (0x80 >> chp))
+			dasd_path_cuir(device, chp);
+}
+
+static inline void dasd_path_add_ifccpm(struct dasd_device *device, __u8 pm)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++)
+		if (pm & (0x80 >> chp))
+			dasd_path_ifcc(device, chp);
+}
+
+static inline void dasd_path_add_nppm(struct dasd_device *device, __u8 pm)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++)
+		if (pm & (0x80 >> chp))
+			dasd_path_nonpreferred(device, chp);
+}
+
+static inline void dasd_path_add_nohpfpm(struct dasd_device *device, __u8 pm)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++)
+		if (pm & (0x80 >> chp))
+			dasd_path_nohpf(device, chp);
+}
+
+static inline void dasd_path_add_ppm(struct dasd_device *device, __u8 pm)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++)
+		if (pm & (0x80 >> chp))
+			dasd_path_preferred(device, chp);
+}
+
+/*
+ * set functions for path masks
+ * the existing path mask will be replaced by the given path mask
+ */
+static inline void dasd_path_set_tbvpm(struct dasd_device *device, __u8 pm)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++)
+		if (pm & (0x80 >> chp))
+			dasd_path_verify(device, chp);
+		else
+			dasd_path_clear_verify(device, chp);
+}
+
+static inline void dasd_path_set_opm(struct dasd_device *device, __u8 pm)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++) {
+		dasd_path_clear_oper(device, chp);
+		if (pm & (0x80 >> chp)) {
+			dasd_path_operational(device, chp);
+			/*
+			 * if the path is used
+			 * it should not be in one of the negative lists
+			 */
+			dasd_path_clear_nohpf(device, chp);
+			dasd_path_clear_cuir(device, chp);
+			dasd_path_clear_cable(device, chp);
+			dasd_path_clear_ifcc(device, chp);
+		}
+	}
+}
+
+/*
+ * remove functions for path masks
+ * the existing path mask will be cleared with the given path mask
+ */
+static inline void dasd_path_remove_opm(struct dasd_device *device, __u8 pm)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++) {
+		if (pm & (0x80 >> chp))
+			dasd_path_clear_oper(device, chp);
+	}
+}
+
+/*
+ * add the newly available path to the to be verified pm and remove it from
+ * normal operation until it is verified
+ */
+static inline void dasd_path_available(struct dasd_device *device, int chp)
+{
+	dasd_path_clear_oper(device, chp);
+	dasd_path_verify(device, chp);
+}
+
+static inline void dasd_path_notoper(struct dasd_device *device, int chp)
+{
+	dasd_path_clear_oper(device, chp);
+	dasd_path_clear_preferred(device, chp);
+	dasd_path_clear_nonpreferred(device, chp);
+}
+
+/*
+ * remove all paths from normal operation
+ */
+static inline void dasd_path_no_path(struct dasd_device *device)
+{
+	int chp;
+
+	for (chp = 0; chp < 8; chp++)
+		dasd_path_notoper(device, chp);
+
+	dasd_path_clear_all_verify(device);
+}
+
+/* end - path handling */
+
 #endif				/* DASD_H */
diff --git a/drivers/s390/char/con3215.c b/drivers/s390/char/con3215.c
index 931d10e86837..1b8d825623bd 100644
--- a/drivers/s390/char/con3215.c
+++ b/drivers/s390/char/con3215.c
@@ -9,7 +9,6 @@
  *	      Dan Morrison, IBM Corporation <dmorriso@cse.buffalo.edu>
  */
 
-#include <linux/module.h>
 #include <linux/types.h>
 #include <linux/kdev_t.h>
 #include <linux/tty.h>
@@ -1215,13 +1214,4 @@ static int __init tty3215_init(void)
 	tty3215_driver = driver;
 	return 0;
 }
-
-static void __exit tty3215_exit(void)
-{
-	tty_unregister_driver(tty3215_driver);
-	put_tty_driver(tty3215_driver);
-	ccw_driver_unregister(&raw3215_ccw_driver);
-}
-
-module_init(tty3215_init);
-module_exit(tty3215_exit);
+device_initcall(tty3215_init);
diff --git a/drivers/s390/char/sclp.h b/drivers/s390/char/sclp.h
index 7a10c56334bb..e1fc7eb043d6 100644
--- a/drivers/s390/char/sclp.h
+++ b/drivers/s390/char/sclp.h
@@ -59,6 +59,7 @@
 
 typedef unsigned int sclp_cmdw_t;
 
+#define SCLP_CMDW_READ_CPU_INFO		0x00010001
 #define SCLP_CMDW_READ_EVENT_DATA	0x00770005
 #define SCLP_CMDW_WRITE_EVENT_DATA	0x00760005
 #define SCLP_CMDW_WRITE_EVENT_MASK	0x00780005
@@ -102,6 +103,28 @@ struct init_sccb {
 	sccb_mask_t sclp_send_mask;
 } __attribute__((packed));
 
+struct read_cpu_info_sccb {
+	struct	sccb_header header;
+	u16	nr_configured;
+	u16	offset_configured;
+	u16	nr_standby;
+	u16	offset_standby;
+	u8	reserved[4096 - 16];
+} __attribute__((packed, aligned(PAGE_SIZE)));
+
+static inline void sclp_fill_core_info(struct sclp_core_info *info,
+				       struct read_cpu_info_sccb *sccb)
+{
+	char *page = (char *) sccb;
+
+	memset(info, 0, sizeof(*info));
+	info->configured = sccb->nr_configured;
+	info->standby = sccb->nr_standby;
+	info->combined = sccb->nr_configured + sccb->nr_standby;
+	memcpy(&info->core, page + sccb->offset_configured,
+	       info->combined * sizeof(struct sclp_core_entry));
+}
+
 #define SCLP_HAS_CHP_INFO	(sclp.facilities & 0x8000000000000000ULL)
 #define SCLP_HAS_CHP_RECONFIG	(sclp.facilities & 0x2000000000000000ULL)
 #define SCLP_HAS_CPU_INFO	(sclp.facilities & 0x0800000000000000ULL)
diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c
index e3fc7539116b..b9c5522b8a68 100644
--- a/drivers/s390/char/sclp_cmd.c
+++ b/drivers/s390/char/sclp_cmd.c
@@ -80,33 +80,10 @@ out:
  * CPU configuration related functions.
  */
 
-#define SCLP_CMDW_READ_CPU_INFO		0x00010001
 #define SCLP_CMDW_CONFIGURE_CPU		0x00110001
 #define SCLP_CMDW_DECONFIGURE_CPU	0x00100001
 
-struct read_cpu_info_sccb {
-	struct	sccb_header header;
-	u16	nr_configured;
-	u16	offset_configured;
-	u16	nr_standby;
-	u16	offset_standby;
-	u8	reserved[4096 - 16];
-} __attribute__((packed, aligned(PAGE_SIZE)));
-
-static void sclp_fill_core_info(struct sclp_core_info *info,
-				struct read_cpu_info_sccb *sccb)
-{
-	char *page = (char *) sccb;
-
-	memset(info, 0, sizeof(*info));
-	info->configured = sccb->nr_configured;
-	info->standby = sccb->nr_standby;
-	info->combined = sccb->nr_configured + sccb->nr_standby;
-	memcpy(&info->core, page + sccb->offset_configured,
-	       info->combined * sizeof(struct sclp_core_entry));
-}
-
-int sclp_get_core_info(struct sclp_core_info *info)
+int _sclp_get_core_info(struct sclp_core_info *info)
 {
 	int rc;
 	struct read_cpu_info_sccb *sccb;
diff --git a/drivers/s390/char/sclp_ctl.c b/drivers/s390/char/sclp_ctl.c
index 554eaa1e347d..78a7e4f94721 100644
--- a/drivers/s390/char/sclp_ctl.c
+++ b/drivers/s390/char/sclp_ctl.c
@@ -10,7 +10,7 @@
 #include <linux/uaccess.h>
 #include <linux/miscdevice.h>
 #include <linux/gfp.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/ioctl.h>
 #include <linux/fs.h>
 #include <asm/compat.h>
@@ -126,4 +126,4 @@ static struct miscdevice sclp_ctl_device = {
 	.name = "sclp",
 	.fops = &sclp_ctl_fops,
 };
-module_misc_device(sclp_ctl_device);
+builtin_misc_device(sclp_ctl_device);
diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c
index c71df0c7dedc..f8e46c22e641 100644
--- a/drivers/s390/char/sclp_early.c
+++ b/drivers/s390/char/sclp_early.c
@@ -221,6 +221,36 @@ static int __init sclp_set_event_mask(struct init_sccb *sccb,
 	return sclp_cmd_early(SCLP_CMDW_WRITE_EVENT_MASK, sccb);
 }
 
+static struct sclp_core_info sclp_core_info_early __initdata;
+static int sclp_core_info_early_valid __initdata;
+
+static void __init sclp_init_core_info_early(struct read_cpu_info_sccb *sccb)
+{
+	int rc;
+
+	if (!SCLP_HAS_CPU_INFO)
+		return;
+	memset(sccb, 0, sizeof(*sccb));
+	sccb->header.length = sizeof(*sccb);
+	do {
+		rc = sclp_cmd_sync_early(SCLP_CMDW_READ_CPU_INFO, sccb);
+	} while (rc == -EBUSY);
+	if (rc)
+		return;
+	if (sccb->header.response_code != 0x0010)
+		return;
+	sclp_fill_core_info(&sclp_core_info_early, sccb);
+	sclp_core_info_early_valid = 1;
+}
+
+int __init _sclp_get_core_info_early(struct sclp_core_info *info)
+{
+	if (!sclp_core_info_early_valid)
+		return -EIO;
+	*info = sclp_core_info_early;
+	return 0;
+}
+
 static long __init sclp_hsa_size_init(struct sdias_sccb *sccb)
 {
 	sccb_init_eq_size(sccb);
@@ -293,6 +323,7 @@ void __init sclp_early_detect(void)
 	void *sccb = &sccb_early;
 
 	sclp_facilities_detect(sccb);
+	sclp_init_core_info_early(sccb);
 	sclp_hsa_size_detect(sccb);
 
 	/* Turn off SCLP event notifications.  Also save remote masks in the
diff --git a/drivers/s390/char/sclp_quiesce.c b/drivers/s390/char/sclp_quiesce.c
index 475e470d9768..e4958511168a 100644
--- a/drivers/s390/char/sclp_quiesce.c
+++ b/drivers/s390/char/sclp_quiesce.c
@@ -6,7 +6,6 @@
  *             Peter Oberparleiter <peter.oberparleiter@de.ibm.com>
  */
 
-#include <linux/module.h>
 #include <linux/types.h>
 #include <linux/cpumask.h>
 #include <linux/smp.h>
@@ -80,5 +79,4 @@ static int __init sclp_quiesce_init(void)
 {
 	return sclp_register(&sclp_quiesce_event);
 }
-
-module_init(sclp_quiesce_init);
+device_initcall(sclp_quiesce_init);
diff --git a/drivers/s390/char/sclp_tty.c b/drivers/s390/char/sclp_tty.c
index 3c6e174e19b6..9259017a1295 100644
--- a/drivers/s390/char/sclp_tty.c
+++ b/drivers/s390/char/sclp_tty.c
@@ -7,7 +7,6 @@
  *		 Martin Schwidefsky <schwidefsky@de.ibm.com>
  */
 
-#include <linux/module.h>
 #include <linux/kmod.h>
 #include <linux/tty.h>
 #include <linux/tty_driver.h>
@@ -573,4 +572,4 @@ sclp_tty_init(void)
 	sclp_tty_driver = driver;
 	return 0;
 }
-module_init(sclp_tty_init);
+device_initcall(sclp_tty_init);
diff --git a/drivers/s390/char/vmlogrdr.c b/drivers/s390/char/vmlogrdr.c
index e883063c7258..3167e8581994 100644
--- a/drivers/s390/char/vmlogrdr.c
+++ b/drivers/s390/char/vmlogrdr.c
@@ -870,7 +870,7 @@ static int __init vmlogrdr_init(void)
 		goto cleanup;
 
 	for (i=0; i < MAXMINOR; ++i ) {
-		sys_ser[i].buffer = (char *) get_zeroed_page(GFP_KERNEL);
+		sys_ser[i].buffer = (char *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
 		if (!sys_ser[i].buffer) {
 			rc = -ENOMEM;
 			break;
diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c
index 16992e2a40ad..f771e5e9e26b 100644
--- a/drivers/s390/char/zcore.c
+++ b/drivers/s390/char/zcore.c
@@ -7,6 +7,7 @@
  *
  * Copyright IBM Corp. 2003, 2008
  * Author(s): Michael Holzheu
+ * License: GPL
  */
 
 #define KMSG_COMPONENT "zdump"
@@ -16,7 +17,6 @@
 #include <linux/slab.h>
 #include <linux/miscdevice.h>
 #include <linux/debugfs.h>
-#include <linux/module.h>
 #include <linux/memblock.h>
 
 #include <asm/asm-offsets.h>
@@ -320,7 +320,7 @@ static int __init zcore_init(void)
 		goto fail;
 	}
 
-	pr_alert("DETECTED 'S390X (64 bit) OS'\n");
+	pr_alert("The dump process started for a 64-bit operating system\n");
 	rc = init_cpu_info();
 	if (rc)
 		goto fail;
@@ -364,22 +364,4 @@ fail:
 	diag308(DIAG308_REL_HSA, NULL);
 	return rc;
 }
-
-static void __exit zcore_exit(void)
-{
-	debug_unregister(zcore_dbf);
-	sclp_sdias_exit();
-	free_page((unsigned long) ipl_block);
-	debugfs_remove(zcore_hsa_file);
-	debugfs_remove(zcore_reipl_file);
-	debugfs_remove(zcore_memmap_file);
-	debugfs_remove(zcore_dir);
-	diag308(DIAG308_REL_HSA, NULL);
-}
-
-MODULE_AUTHOR("Copyright IBM Corp. 2003,2008");
-MODULE_DESCRIPTION("zcore module for zfcpdump support");
-MODULE_LICENSE("GPL");
-
 subsys_initcall(zcore_init);
-module_exit(zcore_exit);
diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c
index 268aa23afa01..6b6386e9a500 100644
--- a/drivers/s390/cio/cmf.c
+++ b/drivers/s390/cio/cmf.c
@@ -30,7 +30,7 @@
 #include <linux/device.h>
 #include <linux/init.h>
 #include <linux/list.h>
-#include <linux/module.h>
+#include <linux/export.h>
 #include <linux/moduleparam.h>
 #include <linux/slab.h>
 #include <linux/timex.h>	/* get_tod_clock() */
@@ -1389,13 +1389,7 @@ static int __init init_cmf(void)
 		"%s (mode %s)\n", format_string, detect_string);
 	return 0;
 }
-module_init(init_cmf);
-
-
-MODULE_AUTHOR("Arnd Bergmann <arndb@de.ibm.com>");
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("channel measurement facility base driver\n"
-		   "Copyright IBM Corp. 2003\n");
+device_initcall(init_cmf);
 
 EXPORT_SYMBOL_GPL(enable_cmf);
 EXPORT_SYMBOL_GPL(disable_cmf);
diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c
index 3d2b20ee613f..bc099b61394d 100644
--- a/drivers/s390/cio/css.c
+++ b/drivers/s390/cio/css.c
@@ -5,12 +5,14 @@
  *
  * Author(s): Arnd Bergmann (arndb@de.ibm.com)
  *	      Cornelia Huck (cornelia.huck@de.ibm.com)
+ *
+ * License: GPL
  */
 
 #define KMSG_COMPONENT "cio"
 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 
-#include <linux/module.h>
+#include <linux/export.h>
 #include <linux/init.h>
 #include <linux/device.h>
 #include <linux/slab.h>
@@ -1285,5 +1287,3 @@ void css_driver_unregister(struct css_driver *cdrv)
 	driver_unregister(&cdrv->drv);
 }
 EXPORT_SYMBOL_GPL(css_driver_unregister);
-
-MODULE_LICENSE("GPL");
diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c
index 6a58bc8f46e2..79823ee9c100 100644
--- a/drivers/s390/cio/device.c
+++ b/drivers/s390/cio/device.c
@@ -5,12 +5,14 @@
  *    Author(s): Arnd Bergmann (arndb@de.ibm.com)
  *		 Cornelia Huck (cornelia.huck@de.ibm.com)
  *		 Martin Schwidefsky (schwidefsky@de.ibm.com)
+ *
+ * License: GPL
  */
 
 #define KMSG_COMPONENT "cio"
 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 
-#include <linux/module.h>
+#include <linux/export.h>
 #include <linux/init.h>
 #include <linux/spinlock.h>
 #include <linux/errno.h>
@@ -145,7 +147,6 @@ static struct css_device_id io_subchannel_ids[] = {
 	{ .match_flags = 0x1, .type = SUBCHANNEL_TYPE_IO, },
 	{ /* end of list */ },
 };
-MODULE_DEVICE_TABLE(css, io_subchannel_ids);
 
 static int io_subchannel_prepare(struct subchannel *sch)
 {
@@ -2150,7 +2151,6 @@ int ccw_device_siosl(struct ccw_device *cdev)
 }
 EXPORT_SYMBOL_GPL(ccw_device_siosl);
 
-MODULE_LICENSE("GPL");
 EXPORT_SYMBOL(ccw_device_set_online);
 EXPORT_SYMBOL(ccw_device_set_offline);
 EXPORT_SYMBOL(ccw_driver_register);
diff --git a/drivers/s390/cio/device.h b/drivers/s390/cio/device.h
index 065b1be98e2c..ec497af99dd8 100644
--- a/drivers/s390/cio/device.h
+++ b/drivers/s390/cio/device.h
@@ -13,7 +13,6 @@
  */
 enum dev_state {
 	DEV_STATE_NOT_OPER,
-	DEV_STATE_SENSE_PGID,
 	DEV_STATE_SENSE_ID,
 	DEV_STATE_OFFLINE,
 	DEV_STATE_VERIFY,
diff --git a/drivers/s390/cio/device_fsm.c b/drivers/s390/cio/device_fsm.c
index 8327d47e08b6..9afb5ce13007 100644
--- a/drivers/s390/cio/device_fsm.c
+++ b/drivers/s390/cio/device_fsm.c
@@ -1058,12 +1058,6 @@ fsm_func_t *dev_jumptable[NR_DEV_STATES][NR_DEV_EVENTS] = {
 		[DEV_EVENT_TIMEOUT]	= ccw_device_nop,
 		[DEV_EVENT_VERIFY]	= ccw_device_nop,
 	},
-	[DEV_STATE_SENSE_PGID] = {
-		[DEV_EVENT_NOTOPER]	= ccw_device_request_event,
-		[DEV_EVENT_INTERRUPT]	= ccw_device_request_event,
-		[DEV_EVENT_TIMEOUT]	= ccw_device_request_event,
-		[DEV_EVENT_VERIFY]	= ccw_device_nop,
-	},
 	[DEV_STATE_SENSE_ID] = {
 		[DEV_EVENT_NOTOPER]	= ccw_device_request_event,
 		[DEV_EVENT_INTERRUPT]	= ccw_device_request_event,
diff --git a/drivers/s390/cio/device_ops.c b/drivers/s390/cio/device_ops.c
index 877d9f601e63..cf8c4ac6323a 100644
--- a/drivers/s390/cio/device_ops.c
+++ b/drivers/s390/cio/device_ops.c
@@ -3,8 +3,10 @@
  *
  * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
  *	      Cornelia Huck (cornelia.huck@de.ibm.com)
+ *
+ * License: GPL
  */
-#include <linux/module.h>
+#include <linux/export.h>
 #include <linux/init.h>
 #include <linux/errno.h>
 #include <linux/slab.h>
@@ -676,7 +678,6 @@ void ccw_device_get_schid(struct ccw_device *cdev, struct subchannel_id *schid)
 }
 EXPORT_SYMBOL_GPL(ccw_device_get_schid);
 
-MODULE_LICENSE("GPL");
 EXPORT_SYMBOL(ccw_device_set_options_mask);
 EXPORT_SYMBOL(ccw_device_set_options);
 EXPORT_SYMBOL(ccw_device_clear_options);
diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile
index b8ab18676e69..0a7fb83f35e5 100644
--- a/drivers/s390/crypto/Makefile
+++ b/drivers/s390/crypto/Makefile
@@ -2,10 +2,11 @@
 # S/390 crypto devices
 #
 
-ap-objs := ap_bus.o
-# zcrypt_api depends on ap
-obj-$(CONFIG_ZCRYPT) += ap.o zcrypt_api.o
-# msgtype* depend on zcrypt_api
-obj-$(CONFIG_ZCRYPT) += zcrypt_msgtype6.o zcrypt_msgtype50.o
-# adapter drivers depend on ap, zcrypt_api and msgtype*
+ap-objs := ap_bus.o ap_card.o ap_queue.o
+obj-$(subst m,y,$(CONFIG_ZCRYPT)) += ap.o
+# zcrypt_api.o and zcrypt_msgtype*.o depend on ap.o
+zcrypt-objs := zcrypt_api.o zcrypt_card.o zcrypt_queue.o
+zcrypt-objs += zcrypt_msgtype6.o zcrypt_msgtype50.o
+obj-$(CONFIG_ZCRYPT) += zcrypt.o
+# adapter drivers depend on ap.o and zcrypt.o
 obj-$(CONFIG_ZCRYPT) += zcrypt_pcixcc.o zcrypt_cex2a.o zcrypt_cex4.o
diff --git a/drivers/s390/crypto/ap_asm.h b/drivers/s390/crypto/ap_asm.h
new file mode 100644
index 000000000000..7a630047c372
--- /dev/null
+++ b/drivers/s390/crypto/ap_asm.h
@@ -0,0 +1,191 @@
+/*
+ * Copyright IBM Corp. 2016
+ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * Adjunct processor bus inline assemblies.
+ */
+
+#ifndef _AP_ASM_H_
+#define _AP_ASM_H_
+
+#include <asm/isc.h>
+
+/**
+ * ap_intructions_available() - Test if AP instructions are available.
+ *
+ * Returns 0 if the AP instructions are installed.
+ */
+static inline int ap_instructions_available(void)
+{
+	register unsigned long reg0 asm ("0") = AP_MKQID(0, 0);
+	register unsigned long reg1 asm ("1") = -ENODEV;
+	register unsigned long reg2 asm ("2") = 0UL;
+
+	asm volatile(
+		"   .long 0xb2af0000\n"		/* PQAP(TAPQ) */
+		"0: la    %1,0\n"
+		"1:\n"
+		EX_TABLE(0b, 1b)
+		: "+d" (reg0), "+d" (reg1), "+d" (reg2) : : "cc");
+	return reg1;
+}
+
+/**
+ * ap_tapq(): Test adjunct processor queue.
+ * @qid: The AP queue number
+ * @info: Pointer to queue descriptor
+ *
+ * Returns AP queue status structure.
+ */
+static inline struct ap_queue_status ap_tapq(ap_qid_t qid, unsigned long *info)
+{
+	register unsigned long reg0 asm ("0") = qid;
+	register struct ap_queue_status reg1 asm ("1");
+	register unsigned long reg2 asm ("2") = 0UL;
+
+	asm volatile(".long 0xb2af0000"		/* PQAP(TAPQ) */
+		     : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
+	if (info)
+		*info = reg2;
+	return reg1;
+}
+
+/**
+ * ap_pqap_rapq(): Reset adjunct processor queue.
+ * @qid: The AP queue number
+ *
+ * Returns AP queue status structure.
+ */
+static inline struct ap_queue_status ap_rapq(ap_qid_t qid)
+{
+	register unsigned long reg0 asm ("0") = qid | 0x01000000UL;
+	register struct ap_queue_status reg1 asm ("1");
+	register unsigned long reg2 asm ("2") = 0UL;
+
+	asm volatile(
+		".long 0xb2af0000"		/* PQAP(RAPQ) */
+		: "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
+	return reg1;
+}
+
+/**
+ * ap_aqic(): Enable interruption for a specific AP.
+ * @qid: The AP queue number
+ * @ind: The notification indicator byte
+ *
+ * Returns AP queue status.
+ */
+static inline struct ap_queue_status ap_aqic(ap_qid_t qid, void *ind)
+{
+	register unsigned long reg0 asm ("0") = qid | (3UL << 24);
+	register unsigned long reg1_in asm ("1") = (8UL << 44) | AP_ISC;
+	register struct ap_queue_status reg1_out asm ("1");
+	register void *reg2 asm ("2") = ind;
+
+	asm volatile(
+		".long 0xb2af0000"		/* PQAP(AQIC) */
+		: "+d" (reg0), "+d" (reg1_in), "=d" (reg1_out), "+d" (reg2)
+		:
+		: "cc");
+	return reg1_out;
+}
+
+/**
+ * ap_qci(): Get AP configuration data
+ *
+ * Returns 0 on success, or -EOPNOTSUPP.
+ */
+static inline int ap_qci(void *config)
+{
+	register unsigned long reg0 asm ("0") = 0x04000000UL;
+	register unsigned long reg1 asm ("1") = -EINVAL;
+	register void *reg2 asm ("2") = (void *) config;
+
+	asm volatile(
+		".long 0xb2af0000\n"		/* PQAP(QCI) */
+		"0: la    %1,0\n"
+		"1:\n"
+		EX_TABLE(0b, 1b)
+		: "+d" (reg0), "+d" (reg1), "+d" (reg2)
+		:
+		: "cc", "memory");
+
+	return reg1;
+}
+
+/**
+ * ap_nqap(): Send message to adjunct processor queue.
+ * @qid: The AP queue number
+ * @psmid: The program supplied message identifier
+ * @msg: The message text
+ * @length: The message length
+ *
+ * Returns AP queue status structure.
+ * Condition code 1 on NQAP can't happen because the L bit is 1.
+ * Condition code 2 on NQAP also means the send is incomplete,
+ * because a segment boundary was reached. The NQAP is repeated.
+ */
+static inline struct ap_queue_status ap_nqap(ap_qid_t qid,
+					     unsigned long long psmid,
+					     void *msg, size_t length)
+{
+	struct msgblock { char _[length]; };
+	register unsigned long reg0 asm ("0") = qid | 0x40000000UL;
+	register struct ap_queue_status reg1 asm ("1");
+	register unsigned long reg2 asm ("2") = (unsigned long) msg;
+	register unsigned long reg3 asm ("3") = (unsigned long) length;
+	register unsigned long reg4 asm ("4") = (unsigned int) (psmid >> 32);
+	register unsigned long reg5 asm ("5") = psmid & 0xffffffff;
+
+	asm volatile (
+		"0: .long 0xb2ad0042\n"		/* NQAP */
+		"   brc   2,0b"
+		: "+d" (reg0), "=d" (reg1), "+d" (reg2), "+d" (reg3)
+		: "d" (reg4), "d" (reg5), "m" (*(struct msgblock *) msg)
+		: "cc");
+	return reg1;
+}
+
+/**
+ * ap_dqap(): Receive message from adjunct processor queue.
+ * @qid: The AP queue number
+ * @psmid: Pointer to program supplied message identifier
+ * @msg: The message text
+ * @length: The message length
+ *
+ * Returns AP queue status structure.
+ * Condition code 1 on DQAP means the receive has taken place
+ * but only partially.	The response is incomplete, hence the
+ * DQAP is repeated.
+ * Condition code 2 on DQAP also means the receive is incomplete,
+ * this time because a segment boundary was reached. Again, the
+ * DQAP is repeated.
+ * Note that gpr2 is used by the DQAP instruction to keep track of
+ * any 'residual' length, in case the instruction gets interrupted.
+ * Hence it gets zeroed before the instruction.
+ */
+static inline struct ap_queue_status ap_dqap(ap_qid_t qid,
+					     unsigned long long *psmid,
+					     void *msg, size_t length)
+{
+	struct msgblock { char _[length]; };
+	register unsigned long reg0 asm("0") = qid | 0x80000000UL;
+	register struct ap_queue_status reg1 asm ("1");
+	register unsigned long reg2 asm("2") = 0UL;
+	register unsigned long reg4 asm("4") = (unsigned long) msg;
+	register unsigned long reg5 asm("5") = (unsigned long) length;
+	register unsigned long reg6 asm("6") = 0UL;
+	register unsigned long reg7 asm("7") = 0UL;
+
+
+	asm volatile(
+		"0: .long 0xb2ae0064\n"		/* DQAP */
+		"   brc   6,0b\n"
+		: "+d" (reg0), "=d" (reg1), "+d" (reg2),
+		"+d" (reg4), "+d" (reg5), "+d" (reg6), "+d" (reg7),
+		"=m" (*(struct msgblock *) msg) : : "cc");
+	*psmid = (((unsigned long long) reg6) << 32) + reg7;
+	return reg1;
+}
+
+#endif /* _AP_ASM_H_ */
diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
index ed92fb09fc8e..6d75984a3d85 100644
--- a/drivers/s390/crypto/ap_bus.c
+++ b/drivers/s390/crypto/ap_bus.c
@@ -46,8 +46,12 @@
 #include <linux/ktime.h>
 #include <asm/facility.h>
 #include <linux/crypto.h>
+#include <linux/mod_devicetable.h>
+#include <linux/debugfs.h>
 
 #include "ap_bus.h"
+#include "ap_asm.h"
+#include "ap_debug.h"
 
 /*
  * Module description.
@@ -62,6 +66,7 @@ MODULE_ALIAS_CRYPTO("z90crypt");
  * Module parameter
  */
 int ap_domain_index = -1;	/* Adjunct Processor Domain Index */
+static DEFINE_SPINLOCK(ap_domain_lock);
 module_param_named(domain, ap_domain_index, int, S_IRUSR|S_IRGRP);
 MODULE_PARM_DESC(domain, "domain index for ap devices");
 EXPORT_SYMBOL(ap_domain_index);
@@ -70,13 +75,21 @@ static int ap_thread_flag = 0;
 module_param_named(poll_thread, ap_thread_flag, int, S_IRUSR|S_IRGRP);
 MODULE_PARM_DESC(poll_thread, "Turn on/off poll thread, default is 0 (off).");
 
-static struct device *ap_root_device = NULL;
+static struct device *ap_root_device;
+
+DEFINE_SPINLOCK(ap_list_lock);
+LIST_HEAD(ap_card_list);
+
 static struct ap_config_info *ap_configuration;
-static DEFINE_SPINLOCK(ap_device_list_lock);
-static LIST_HEAD(ap_device_list);
 static bool initialised;
 
 /*
+ * AP bus related debug feature things.
+ */
+static struct dentry *ap_dbf_root;
+debug_info_t *ap_dbf_info;
+
+/*
  * Workqueue timer for bus rescan.
  */
 static struct timer_list ap_config_timer;
@@ -89,7 +102,6 @@ static DECLARE_WORK(ap_scan_work, ap_scan_bus);
  */
 static void ap_tasklet_fn(unsigned long);
 static DECLARE_TASKLET(ap_tasklet, ap_tasklet_fn, 0);
-static atomic_t ap_poll_requests = ATOMIC_INIT(0);
 static DECLARE_WAIT_QUEUE_HEAD(ap_poll_wait);
 static struct task_struct *ap_poll_kthread = NULL;
 static DEFINE_MUTEX(ap_poll_thread_mutex);
@@ -129,23 +141,17 @@ static inline int ap_using_interrupts(void)
 }
 
 /**
- * ap_intructions_available() - Test if AP instructions are available.
+ * ap_airq_ptr() - Get the address of the adapter interrupt indicator
  *
- * Returns 0 if the AP instructions are installed.
+ * Returns the address of the local-summary-indicator of the adapter
+ * interrupt handler for AP, or NULL if adapter interrupts are not
+ * available.
  */
-static inline int ap_instructions_available(void)
+void *ap_airq_ptr(void)
 {
-	register unsigned long reg0 asm ("0") = AP_MKQID(0,0);
-	register unsigned long reg1 asm ("1") = -ENODEV;
-	register unsigned long reg2 asm ("2") = 0UL;
-
-	asm volatile(
-		"   .long 0xb2af0000\n"		/* PQAP(TAPQ) */
-		"0: la    %1,0\n"
-		"1:\n"
-		EX_TABLE(0b, 1b)
-		: "+d" (reg0), "+d" (reg1), "+d" (reg2) : : "cc" );
-	return reg1;
+	if (ap_using_interrupts())
+		return ap_airq.lsi_ptr;
+	return NULL;
 }
 
 /**
@@ -169,19 +175,6 @@ static int ap_configuration_available(void)
 	return test_facility(12);
 }
 
-static inline struct ap_queue_status
-__pqap_tapq(ap_qid_t qid, unsigned long *info)
-{
-	register unsigned long reg0 asm ("0") = qid;
-	register struct ap_queue_status reg1 asm ("1");
-	register unsigned long reg2 asm ("2") = 0UL;
-
-	asm volatile(".long 0xb2af0000"		/* PQAP(TAPQ) */
-		     : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
-	*info = reg2;
-	return reg1;
-}
-
 /**
  * ap_test_queue(): Test adjunct processor queue.
  * @qid: The AP queue number
@@ -192,85 +185,16 @@ __pqap_tapq(ap_qid_t qid, unsigned long *info)
 static inline struct ap_queue_status
 ap_test_queue(ap_qid_t qid, unsigned long *info)
 {
-	struct ap_queue_status aqs;
-	unsigned long _info;
-
 	if (test_facility(15))
 		qid |= 1UL << 23;		/* set APFT T bit*/
-	aqs = __pqap_tapq(qid, &_info);
-	if (info)
-		*info = _info;
-	return aqs;
-}
-
-/**
- * ap_reset_queue(): Reset adjunct processor queue.
- * @qid: The AP queue number
- *
- * Returns AP queue status structure.
- */
-static inline struct ap_queue_status ap_reset_queue(ap_qid_t qid)
-{
-	register unsigned long reg0 asm ("0") = qid | 0x01000000UL;
-	register struct ap_queue_status reg1 asm ("1");
-	register unsigned long reg2 asm ("2") = 0UL;
-
-	asm volatile(
-		".long 0xb2af0000"		/* PQAP(RAPQ) */
-		: "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
-	return reg1;
-}
-
-/**
- * ap_queue_interruption_control(): Enable interruption for a specific AP.
- * @qid: The AP queue number
- * @ind: The notification indicator byte
- *
- * Returns AP queue status.
- */
-static inline struct ap_queue_status
-ap_queue_interruption_control(ap_qid_t qid, void *ind)
-{
-	register unsigned long reg0 asm ("0") = qid | 0x03000000UL;
-	register unsigned long reg1_in asm ("1") = 0x0000800000000000UL | AP_ISC;
-	register struct ap_queue_status reg1_out asm ("1");
-	register void *reg2 asm ("2") = ind;
-	asm volatile(
-		".long 0xb2af0000"		/* PQAP(AQIC) */
-		: "+d" (reg0), "+d" (reg1_in), "=d" (reg1_out), "+d" (reg2)
-		:
-		: "cc" );
-	return reg1_out;
-}
-
-/**
- * ap_query_configuration(): Get AP configuration data
- *
- * Returns 0 on success, or -EOPNOTSUPP.
- */
-static inline int __ap_query_configuration(void)
-{
-	register unsigned long reg0 asm ("0") = 0x04000000UL;
-	register unsigned long reg1 asm ("1") = -EINVAL;
-	register void *reg2 asm ("2") = (void *) ap_configuration;
-
-	asm volatile(
-		".long 0xb2af0000\n"		/* PQAP(QCI) */
-		"0: la    %1,0\n"
-		"1:\n"
-		EX_TABLE(0b, 1b)
-		: "+d" (reg0), "+d" (reg1), "+d" (reg2)
-		:
-		: "cc");
-
-	return reg1;
+	return ap_tapq(qid, info);
 }
 
 static inline int ap_query_configuration(void)
 {
 	if (!ap_configuration)
 		return -EOPNOTSUPP;
-	return __ap_query_configuration();
+	return ap_qci(ap_configuration);
 }
 
 /**
@@ -331,162 +255,6 @@ static inline int ap_test_config_domain(unsigned int domain)
 }
 
 /**
- * ap_queue_enable_interruption(): Enable interruption on an AP.
- * @qid: The AP queue number
- * @ind: the notification indicator byte
- *
- * Enables interruption on AP queue via ap_queue_interruption_control(). Based
- * on the return value it waits a while and tests the AP queue if interrupts
- * have been switched on using ap_test_queue().
- */
-static int ap_queue_enable_interruption(struct ap_device *ap_dev, void *ind)
-{
-	struct ap_queue_status status;
-
-	status = ap_queue_interruption_control(ap_dev->qid, ind);
-	switch (status.response_code) {
-	case AP_RESPONSE_NORMAL:
-	case AP_RESPONSE_OTHERWISE_CHANGED:
-		return 0;
-	case AP_RESPONSE_Q_NOT_AVAIL:
-	case AP_RESPONSE_DECONFIGURED:
-	case AP_RESPONSE_CHECKSTOPPED:
-	case AP_RESPONSE_INVALID_ADDRESS:
-		pr_err("Registering adapter interrupts for AP %d failed\n",
-		       AP_QID_DEVICE(ap_dev->qid));
-		return -EOPNOTSUPP;
-	case AP_RESPONSE_RESET_IN_PROGRESS:
-	case AP_RESPONSE_BUSY:
-	default:
-		return -EBUSY;
-	}
-}
-
-static inline struct ap_queue_status
-__nqap(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length)
-{
-	typedef struct { char _[length]; } msgblock;
-	register unsigned long reg0 asm ("0") = qid | 0x40000000UL;
-	register struct ap_queue_status reg1 asm ("1");
-	register unsigned long reg2 asm ("2") = (unsigned long) msg;
-	register unsigned long reg3 asm ("3") = (unsigned long) length;
-	register unsigned long reg4 asm ("4") = (unsigned int) (psmid >> 32);
-	register unsigned long reg5 asm ("5") = psmid & 0xffffffff;
-
-	asm volatile (
-		"0: .long 0xb2ad0042\n"		/* NQAP */
-		"   brc   2,0b"
-		: "+d" (reg0), "=d" (reg1), "+d" (reg2), "+d" (reg3)
-		: "d" (reg4), "d" (reg5), "m" (*(msgblock *) msg)
-		: "cc");
-	return reg1;
-}
-
-/**
- * __ap_send(): Send message to adjunct processor queue.
- * @qid: The AP queue number
- * @psmid: The program supplied message identifier
- * @msg: The message text
- * @length: The message length
- * @special: Special Bit
- *
- * Returns AP queue status structure.
- * Condition code 1 on NQAP can't happen because the L bit is 1.
- * Condition code 2 on NQAP also means the send is incomplete,
- * because a segment boundary was reached. The NQAP is repeated.
- */
-static inline struct ap_queue_status
-__ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length,
-	  unsigned int special)
-{
-	if (special == 1)
-		qid |= 0x400000UL;
-	return __nqap(qid, psmid, msg, length);
-}
-
-int ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length)
-{
-	struct ap_queue_status status;
-
-	status = __ap_send(qid, psmid, msg, length, 0);
-	switch (status.response_code) {
-	case AP_RESPONSE_NORMAL:
-		return 0;
-	case AP_RESPONSE_Q_FULL:
-	case AP_RESPONSE_RESET_IN_PROGRESS:
-		return -EBUSY;
-	case AP_RESPONSE_REQ_FAC_NOT_INST:
-		return -EINVAL;
-	default:	/* Device is gone. */
-		return -ENODEV;
-	}
-}
-EXPORT_SYMBOL(ap_send);
-
-/**
- * __ap_recv(): Receive message from adjunct processor queue.
- * @qid: The AP queue number
- * @psmid: Pointer to program supplied message identifier
- * @msg: The message text
- * @length: The message length
- *
- * Returns AP queue status structure.
- * Condition code 1 on DQAP means the receive has taken place
- * but only partially.	The response is incomplete, hence the
- * DQAP is repeated.
- * Condition code 2 on DQAP also means the receive is incomplete,
- * this time because a segment boundary was reached. Again, the
- * DQAP is repeated.
- * Note that gpr2 is used by the DQAP instruction to keep track of
- * any 'residual' length, in case the instruction gets interrupted.
- * Hence it gets zeroed before the instruction.
- */
-static inline struct ap_queue_status
-__ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
-{
-	typedef struct { char _[length]; } msgblock;
-	register unsigned long reg0 asm("0") = qid | 0x80000000UL;
-	register struct ap_queue_status reg1 asm ("1");
-	register unsigned long reg2 asm("2") = 0UL;
-	register unsigned long reg4 asm("4") = (unsigned long) msg;
-	register unsigned long reg5 asm("5") = (unsigned long) length;
-	register unsigned long reg6 asm("6") = 0UL;
-	register unsigned long reg7 asm("7") = 0UL;
-
-
-	asm volatile(
-		"0: .long 0xb2ae0064\n"		/* DQAP */
-		"   brc   6,0b\n"
-		: "+d" (reg0), "=d" (reg1), "+d" (reg2),
-		"+d" (reg4), "+d" (reg5), "+d" (reg6), "+d" (reg7),
-		"=m" (*(msgblock *) msg) : : "cc" );
-	*psmid = (((unsigned long long) reg6) << 32) + reg7;
-	return reg1;
-}
-
-int ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
-{
-	struct ap_queue_status status;
-
-	if (msg == NULL)
-		return -EINVAL;
-	status = __ap_recv(qid, psmid, msg, length);
-	switch (status.response_code) {
-	case AP_RESPONSE_NORMAL:
-		return 0;
-	case AP_RESPONSE_NO_PENDING_REPLY:
-		if (status.queue_empty)
-			return -ENOENT;
-		return -EBUSY;
-	case AP_RESPONSE_RESET_IN_PROGRESS:
-		return -EBUSY;
-	default:
-		return -ENODEV;
-	}
-}
-EXPORT_SYMBOL(ap_recv);
-
-/**
  * ap_query_queue(): Check if an AP queue is available.
  * @qid: The AP queue number
  * @queue_depth: Pointer to queue depth value
@@ -500,7 +268,7 @@ static int ap_query_queue(ap_qid_t qid, int *queue_depth, int *device_type,
 	unsigned long info;
 	int nd;
 
-	if (!ap_test_config_card_id(AP_QID_DEVICE(qid)))
+	if (!ap_test_config_card_id(AP_QID_CARD(qid)))
 		return -ENODEV;
 
 	status = ap_test_queue(qid, &info);
@@ -511,8 +279,28 @@ static int ap_query_queue(ap_qid_t qid, int *queue_depth, int *device_type,
 		*facilities = (unsigned int)(info >> 32);
 		/* Update maximum domain id */
 		nd = (info >> 16) & 0xff;
+		/* if N bit is available, z13 and newer */
 		if ((info & (1UL << 57)) && nd > 0)
 			ap_max_domain_id = nd;
+		else /* older machine types */
+			ap_max_domain_id = 15;
+		switch (*device_type) {
+			/* For CEX2 and CEX3 the available functions
+			 * are not refrected by the facilities bits.
+			 * Instead it is coded into the type. So here
+			 * modify the function bits based on the type.
+			 */
+		case AP_DEVICE_TYPE_CEX2A:
+		case AP_DEVICE_TYPE_CEX3A:
+			*facilities |= 0x08000000;
+			break;
+		case AP_DEVICE_TYPE_CEX2C:
+		case AP_DEVICE_TYPE_CEX3C:
+			*facilities |= 0x10000000;
+			break;
+		default:
+			break;
+		}
 		return 0;
 	case AP_RESPONSE_Q_NOT_AVAIL:
 	case AP_RESPONSE_DECONFIGURED:
@@ -528,9 +316,7 @@ static int ap_query_queue(ap_qid_t qid, int *queue_depth, int *device_type,
 	}
 }
 
-/* State machine definitions and helpers */
-
-static void ap_sm_wait(enum ap_wait wait)
+void ap_wait(enum ap_wait wait)
 {
 	ktime_t hr_time;
 
@@ -559,350 +345,21 @@ static void ap_sm_wait(enum ap_wait wait)
 	}
 }
 
-static enum ap_wait ap_sm_nop(struct ap_device *ap_dev)
-{
-	return AP_WAIT_NONE;
-}
-
-/**
- * ap_sm_recv(): Receive pending reply messages from an AP device but do
- *	not change the state of the device.
- * @ap_dev: pointer to the AP device
- *
- * Returns AP_WAIT_NONE, AP_WAIT_AGAIN, or AP_WAIT_INTERRUPT
- */
-static struct ap_queue_status ap_sm_recv(struct ap_device *ap_dev)
-{
-	struct ap_queue_status status;
-	struct ap_message *ap_msg;
-
-	status = __ap_recv(ap_dev->qid, &ap_dev->reply->psmid,
-			   ap_dev->reply->message, ap_dev->reply->length);
-	switch (status.response_code) {
-	case AP_RESPONSE_NORMAL:
-		atomic_dec(&ap_poll_requests);
-		ap_dev->queue_count--;
-		if (ap_dev->queue_count > 0)
-			mod_timer(&ap_dev->timeout,
-				  jiffies + ap_dev->drv->request_timeout);
-		list_for_each_entry(ap_msg, &ap_dev->pendingq, list) {
-			if (ap_msg->psmid != ap_dev->reply->psmid)
-				continue;
-			list_del_init(&ap_msg->list);
-			ap_dev->pendingq_count--;
-			ap_msg->receive(ap_dev, ap_msg, ap_dev->reply);
-			break;
-		}
-	case AP_RESPONSE_NO_PENDING_REPLY:
-		if (!status.queue_empty || ap_dev->queue_count <= 0)
-			break;
-		/* The card shouldn't forget requests but who knows. */
-		atomic_sub(ap_dev->queue_count, &ap_poll_requests);
-		ap_dev->queue_count = 0;
-		list_splice_init(&ap_dev->pendingq, &ap_dev->requestq);
-		ap_dev->requestq_count += ap_dev->pendingq_count;
-		ap_dev->pendingq_count = 0;
-		break;
-	default:
-		break;
-	}
-	return status;
-}
-
-/**
- * ap_sm_read(): Receive pending reply messages from an AP device.
- * @ap_dev: pointer to the AP device
- *
- * Returns AP_WAIT_NONE, AP_WAIT_AGAIN, or AP_WAIT_INTERRUPT
- */
-static enum ap_wait ap_sm_read(struct ap_device *ap_dev)
-{
-	struct ap_queue_status status;
-
-	if (!ap_dev->reply)
-		return AP_WAIT_NONE;
-	status = ap_sm_recv(ap_dev);
-	switch (status.response_code) {
-	case AP_RESPONSE_NORMAL:
-		if (ap_dev->queue_count > 0) {
-			ap_dev->state = AP_STATE_WORKING;
-			return AP_WAIT_AGAIN;
-		}
-		ap_dev->state = AP_STATE_IDLE;
-		return AP_WAIT_NONE;
-	case AP_RESPONSE_NO_PENDING_REPLY:
-		if (ap_dev->queue_count > 0)
-			return AP_WAIT_INTERRUPT;
-		ap_dev->state = AP_STATE_IDLE;
-		return AP_WAIT_NONE;
-	default:
-		ap_dev->state = AP_STATE_BORKED;
-		return AP_WAIT_NONE;
-	}
-}
-
-/**
- * ap_sm_suspend_read(): Receive pending reply messages from an AP device
- * without changing the device state in between. In suspend mode we don't
- * allow sending new requests, therefore just fetch pending replies.
- * @ap_dev: pointer to the AP device
- *
- * Returns AP_WAIT_NONE or AP_WAIT_AGAIN
- */
-static enum ap_wait ap_sm_suspend_read(struct ap_device *ap_dev)
-{
-	struct ap_queue_status status;
-
-	if (!ap_dev->reply)
-		return AP_WAIT_NONE;
-	status = ap_sm_recv(ap_dev);
-	switch (status.response_code) {
-	case AP_RESPONSE_NORMAL:
-		if (ap_dev->queue_count > 0)
-			return AP_WAIT_AGAIN;
-		/* fall through */
-	default:
-		return AP_WAIT_NONE;
-	}
-}
-
-/**
- * ap_sm_write(): Send messages from the request queue to an AP device.
- * @ap_dev: pointer to the AP device
- *
- * Returns AP_WAIT_NONE, AP_WAIT_AGAIN, or AP_WAIT_INTERRUPT
- */
-static enum ap_wait ap_sm_write(struct ap_device *ap_dev)
-{
-	struct ap_queue_status status;
-	struct ap_message *ap_msg;
-
-	if (ap_dev->requestq_count <= 0)
-		return AP_WAIT_NONE;
-	/* Start the next request on the queue. */
-	ap_msg = list_entry(ap_dev->requestq.next, struct ap_message, list);
-	status = __ap_send(ap_dev->qid, ap_msg->psmid,
-			   ap_msg->message, ap_msg->length, ap_msg->special);
-	switch (status.response_code) {
-	case AP_RESPONSE_NORMAL:
-		atomic_inc(&ap_poll_requests);
-		ap_dev->queue_count++;
-		if (ap_dev->queue_count == 1)
-			mod_timer(&ap_dev->timeout,
-				  jiffies + ap_dev->drv->request_timeout);
-		list_move_tail(&ap_msg->list, &ap_dev->pendingq);
-		ap_dev->requestq_count--;
-		ap_dev->pendingq_count++;
-		if (ap_dev->queue_count < ap_dev->queue_depth) {
-			ap_dev->state = AP_STATE_WORKING;
-			return AP_WAIT_AGAIN;
-		}
-		/* fall through */
-	case AP_RESPONSE_Q_FULL:
-		ap_dev->state = AP_STATE_QUEUE_FULL;
-		return AP_WAIT_INTERRUPT;
-	case AP_RESPONSE_RESET_IN_PROGRESS:
-		ap_dev->state = AP_STATE_RESET_WAIT;
-		return AP_WAIT_TIMEOUT;
-	case AP_RESPONSE_MESSAGE_TOO_BIG:
-	case AP_RESPONSE_REQ_FAC_NOT_INST:
-		list_del_init(&ap_msg->list);
-		ap_dev->requestq_count--;
-		ap_msg->rc = -EINVAL;
-		ap_msg->receive(ap_dev, ap_msg, NULL);
-		return AP_WAIT_AGAIN;
-	default:
-		ap_dev->state = AP_STATE_BORKED;
-		return AP_WAIT_NONE;
-	}
-}
-
-/**
- * ap_sm_read_write(): Send and receive messages to/from an AP device.
- * @ap_dev: pointer to the AP device
- *
- * Returns AP_WAIT_NONE, AP_WAIT_AGAIN, or AP_WAIT_INTERRUPT
- */
-static enum ap_wait ap_sm_read_write(struct ap_device *ap_dev)
-{
-	return min(ap_sm_read(ap_dev), ap_sm_write(ap_dev));
-}
-
-/**
- * ap_sm_reset(): Reset an AP queue.
- * @qid: The AP queue number
- *
- * Submit the Reset command to an AP queue.
- */
-static enum ap_wait ap_sm_reset(struct ap_device *ap_dev)
-{
-	struct ap_queue_status status;
-
-	status = ap_reset_queue(ap_dev->qid);
-	switch (status.response_code) {
-	case AP_RESPONSE_NORMAL:
-	case AP_RESPONSE_RESET_IN_PROGRESS:
-		ap_dev->state = AP_STATE_RESET_WAIT;
-		ap_dev->interrupt = AP_INTR_DISABLED;
-		return AP_WAIT_TIMEOUT;
-	case AP_RESPONSE_BUSY:
-		return AP_WAIT_TIMEOUT;
-	case AP_RESPONSE_Q_NOT_AVAIL:
-	case AP_RESPONSE_DECONFIGURED:
-	case AP_RESPONSE_CHECKSTOPPED:
-	default:
-		ap_dev->state = AP_STATE_BORKED;
-		return AP_WAIT_NONE;
-	}
-}
-
-/**
- * ap_sm_reset_wait(): Test queue for completion of the reset operation
- * @ap_dev: pointer to the AP device
- *
- * Returns AP_POLL_IMMEDIATELY, AP_POLL_AFTER_TIMEROUT or 0.
- */
-static enum ap_wait ap_sm_reset_wait(struct ap_device *ap_dev)
-{
-	struct ap_queue_status status;
-	unsigned long info;
-
-	if (ap_dev->queue_count > 0 && ap_dev->reply)
-		/* Try to read a completed message and get the status */
-		status = ap_sm_recv(ap_dev);
-	else
-		/* Get the status with TAPQ */
-		status = ap_test_queue(ap_dev->qid, &info);
-
-	switch (status.response_code) {
-	case AP_RESPONSE_NORMAL:
-		if (ap_using_interrupts() &&
-		    ap_queue_enable_interruption(ap_dev,
-						 ap_airq.lsi_ptr) == 0)
-			ap_dev->state = AP_STATE_SETIRQ_WAIT;
-		else
-			ap_dev->state = (ap_dev->queue_count > 0) ?
-				AP_STATE_WORKING : AP_STATE_IDLE;
-		return AP_WAIT_AGAIN;
-	case AP_RESPONSE_BUSY:
-	case AP_RESPONSE_RESET_IN_PROGRESS:
-		return AP_WAIT_TIMEOUT;
-	case AP_RESPONSE_Q_NOT_AVAIL:
-	case AP_RESPONSE_DECONFIGURED:
-	case AP_RESPONSE_CHECKSTOPPED:
-	default:
-		ap_dev->state = AP_STATE_BORKED;
-		return AP_WAIT_NONE;
-	}
-}
-
-/**
- * ap_sm_setirq_wait(): Test queue for completion of the irq enablement
- * @ap_dev: pointer to the AP device
- *
- * Returns AP_POLL_IMMEDIATELY, AP_POLL_AFTER_TIMEROUT or 0.
- */
-static enum ap_wait ap_sm_setirq_wait(struct ap_device *ap_dev)
-{
-	struct ap_queue_status status;
-	unsigned long info;
-
-	if (ap_dev->queue_count > 0 && ap_dev->reply)
-		/* Try to read a completed message and get the status */
-		status = ap_sm_recv(ap_dev);
-	else
-		/* Get the status with TAPQ */
-		status = ap_test_queue(ap_dev->qid, &info);
-
-	if (status.int_enabled == 1) {
-		/* Irqs are now enabled */
-		ap_dev->interrupt = AP_INTR_ENABLED;
-		ap_dev->state = (ap_dev->queue_count > 0) ?
-			AP_STATE_WORKING : AP_STATE_IDLE;
-	}
-
-	switch (status.response_code) {
-	case AP_RESPONSE_NORMAL:
-		if (ap_dev->queue_count > 0)
-			return AP_WAIT_AGAIN;
-		/* fallthrough */
-	case AP_RESPONSE_NO_PENDING_REPLY:
-		return AP_WAIT_TIMEOUT;
-	default:
-		ap_dev->state = AP_STATE_BORKED;
-		return AP_WAIT_NONE;
-	}
-}
-
-/*
- * AP state machine jump table
- */
-static ap_func_t *ap_jumptable[NR_AP_STATES][NR_AP_EVENTS] = {
-	[AP_STATE_RESET_START] = {
-		[AP_EVENT_POLL] = ap_sm_reset,
-		[AP_EVENT_TIMEOUT] = ap_sm_nop,
-	},
-	[AP_STATE_RESET_WAIT] = {
-		[AP_EVENT_POLL] = ap_sm_reset_wait,
-		[AP_EVENT_TIMEOUT] = ap_sm_nop,
-	},
-	[AP_STATE_SETIRQ_WAIT] = {
-		[AP_EVENT_POLL] = ap_sm_setirq_wait,
-		[AP_EVENT_TIMEOUT] = ap_sm_nop,
-	},
-	[AP_STATE_IDLE] = {
-		[AP_EVENT_POLL] = ap_sm_write,
-		[AP_EVENT_TIMEOUT] = ap_sm_nop,
-	},
-	[AP_STATE_WORKING] = {
-		[AP_EVENT_POLL] = ap_sm_read_write,
-		[AP_EVENT_TIMEOUT] = ap_sm_reset,
-	},
-	[AP_STATE_QUEUE_FULL] = {
-		[AP_EVENT_POLL] = ap_sm_read,
-		[AP_EVENT_TIMEOUT] = ap_sm_reset,
-	},
-	[AP_STATE_SUSPEND_WAIT] = {
-		[AP_EVENT_POLL] = ap_sm_suspend_read,
-		[AP_EVENT_TIMEOUT] = ap_sm_nop,
-	},
-	[AP_STATE_BORKED] = {
-		[AP_EVENT_POLL] = ap_sm_nop,
-		[AP_EVENT_TIMEOUT] = ap_sm_nop,
-	},
-};
-
-static inline enum ap_wait ap_sm_event(struct ap_device *ap_dev,
-				       enum ap_event event)
-{
-	return ap_jumptable[ap_dev->state][event](ap_dev);
-}
-
-static inline enum ap_wait ap_sm_event_loop(struct ap_device *ap_dev,
-					    enum ap_event event)
-{
-	enum ap_wait wait;
-
-	while ((wait = ap_sm_event(ap_dev, event)) == AP_WAIT_AGAIN)
-		;
-	return wait;
-}
-
 /**
  * ap_request_timeout(): Handling of request timeouts
  * @data: Holds the AP device.
  *
  * Handles request timeouts.
  */
-static void ap_request_timeout(unsigned long data)
+void ap_request_timeout(unsigned long data)
 {
-	struct ap_device *ap_dev = (struct ap_device *) data;
+	struct ap_queue *aq = (struct ap_queue *) data;
 
 	if (ap_suspend_flag)
 		return;
-	spin_lock_bh(&ap_dev->lock);
-	ap_sm_wait(ap_sm_event(ap_dev, AP_EVENT_TIMEOUT));
-	spin_unlock_bh(&ap_dev->lock);
+	spin_lock_bh(&aq->lock);
+	ap_wait(ap_sm_event(aq, AP_EVENT_TIMEOUT));
+	spin_unlock_bh(&aq->lock);
 }
 
 /**
@@ -937,7 +394,8 @@ static void ap_interrupt_handler(struct airq_struct *airq)
  */
 static void ap_tasklet_fn(unsigned long dummy)
 {
-	struct ap_device *ap_dev;
+	struct ap_card *ac;
+	struct ap_queue *aq;
 	enum ap_wait wait = AP_WAIT_NONE;
 
 	/* Reset the indicator if interrupts are used. Thus new interrupts can
@@ -947,14 +405,35 @@ static void ap_tasklet_fn(unsigned long dummy)
 	if (ap_using_interrupts())
 		xchg(ap_airq.lsi_ptr, 0);
 
-	spin_lock(&ap_device_list_lock);
-	list_for_each_entry(ap_dev, &ap_device_list, list) {
-		spin_lock_bh(&ap_dev->lock);
-		wait = min(wait, ap_sm_event_loop(ap_dev, AP_EVENT_POLL));
-		spin_unlock_bh(&ap_dev->lock);
+	spin_lock_bh(&ap_list_lock);
+	for_each_ap_card(ac) {
+		for_each_ap_queue(aq, ac) {
+			spin_lock_bh(&aq->lock);
+			wait = min(wait, ap_sm_event_loop(aq, AP_EVENT_POLL));
+			spin_unlock_bh(&aq->lock);
+		}
 	}
-	spin_unlock(&ap_device_list_lock);
-	ap_sm_wait(wait);
+	spin_unlock_bh(&ap_list_lock);
+
+	ap_wait(wait);
+}
+
+static int ap_pending_requests(void)
+{
+	struct ap_card *ac;
+	struct ap_queue *aq;
+
+	spin_lock_bh(&ap_list_lock);
+	for_each_ap_card(ac) {
+		for_each_ap_queue(aq, ac) {
+			if (aq->queue_count == 0)
+				continue;
+			spin_unlock_bh(&ap_list_lock);
+			return 1;
+		}
+	}
+	spin_unlock_bh(&ap_list_lock);
+	return 0;
 }
 
 /**
@@ -976,8 +455,7 @@ static int ap_poll_thread(void *data)
 	while (!kthread_should_stop()) {
 		add_wait_queue(&ap_poll_wait, &wait);
 		set_current_state(TASK_INTERRUPTIBLE);
-		if (ap_suspend_flag ||
-		    atomic_read(&ap_poll_requests) <= 0) {
+		if (ap_suspend_flag || !ap_pending_requests()) {
 			schedule();
 			try_to_freeze();
 		}
@@ -989,7 +467,8 @@ static int ap_poll_thread(void *data)
 			continue;
 		}
 		ap_tasklet_fn(0);
-	} while (!kthread_should_stop());
+	}
+
 	return 0;
 }
 
@@ -1018,207 +497,8 @@ static void ap_poll_thread_stop(void)
 	mutex_unlock(&ap_poll_thread_mutex);
 }
 
-/**
- * ap_queue_message(): Queue a request to an AP device.
- * @ap_dev: The AP device to queue the message to
- * @ap_msg: The message that is to be added
- */
-void ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
-{
-	/* For asynchronous message handling a valid receive-callback
-	 * is required. */
-	BUG_ON(!ap_msg->receive);
-
-	spin_lock_bh(&ap_dev->lock);
-	/* Queue the message. */
-	list_add_tail(&ap_msg->list, &ap_dev->requestq);
-	ap_dev->requestq_count++;
-	ap_dev->total_request_count++;
-	/* Send/receive as many request from the queue as possible. */
-	ap_sm_wait(ap_sm_event_loop(ap_dev, AP_EVENT_POLL));
-	spin_unlock_bh(&ap_dev->lock);
-}
-EXPORT_SYMBOL(ap_queue_message);
-
-/**
- * ap_cancel_message(): Cancel a crypto request.
- * @ap_dev: The AP device that has the message queued
- * @ap_msg: The message that is to be removed
- *
- * Cancel a crypto request. This is done by removing the request
- * from the device pending or request queue. Note that the
- * request stays on the AP queue. When it finishes the message
- * reply will be discarded because the psmid can't be found.
- */
-void ap_cancel_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
-{
-	struct ap_message *tmp;
-
-	spin_lock_bh(&ap_dev->lock);
-	if (!list_empty(&ap_msg->list)) {
-		list_for_each_entry(tmp, &ap_dev->pendingq, list)
-			if (tmp->psmid == ap_msg->psmid) {
-				ap_dev->pendingq_count--;
-				goto found;
-			}
-		ap_dev->requestq_count--;
-found:
-		list_del_init(&ap_msg->list);
-	}
-	spin_unlock_bh(&ap_dev->lock);
-}
-EXPORT_SYMBOL(ap_cancel_message);
-
-/*
- * AP device related attributes.
- */
-static ssize_t ap_hwtype_show(struct device *dev,
-			      struct device_attribute *attr, char *buf)
-{
-	struct ap_device *ap_dev = to_ap_dev(dev);
-	return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->device_type);
-}
-
-static DEVICE_ATTR(hwtype, 0444, ap_hwtype_show, NULL);
-
-static ssize_t ap_raw_hwtype_show(struct device *dev,
-			      struct device_attribute *attr, char *buf)
-{
-	struct ap_device *ap_dev = to_ap_dev(dev);
-
-	return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->raw_hwtype);
-}
-
-static DEVICE_ATTR(raw_hwtype, 0444, ap_raw_hwtype_show, NULL);
-
-static ssize_t ap_depth_show(struct device *dev, struct device_attribute *attr,
-			     char *buf)
-{
-	struct ap_device *ap_dev = to_ap_dev(dev);
-	return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->queue_depth);
-}
-
-static DEVICE_ATTR(depth, 0444, ap_depth_show, NULL);
-static ssize_t ap_request_count_show(struct device *dev,
-				     struct device_attribute *attr,
-				     char *buf)
-{
-	struct ap_device *ap_dev = to_ap_dev(dev);
-	int rc;
-
-	spin_lock_bh(&ap_dev->lock);
-	rc = snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->total_request_count);
-	spin_unlock_bh(&ap_dev->lock);
-	return rc;
-}
-
-static DEVICE_ATTR(request_count, 0444, ap_request_count_show, NULL);
-
-static ssize_t ap_requestq_count_show(struct device *dev,
-				      struct device_attribute *attr, char *buf)
-{
-	struct ap_device *ap_dev = to_ap_dev(dev);
-	int rc;
-
-	spin_lock_bh(&ap_dev->lock);
-	rc = snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->requestq_count);
-	spin_unlock_bh(&ap_dev->lock);
-	return rc;
-}
-
-static DEVICE_ATTR(requestq_count, 0444, ap_requestq_count_show, NULL);
-
-static ssize_t ap_pendingq_count_show(struct device *dev,
-				      struct device_attribute *attr, char *buf)
-{
-	struct ap_device *ap_dev = to_ap_dev(dev);
-	int rc;
-
-	spin_lock_bh(&ap_dev->lock);
-	rc = snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->pendingq_count);
-	spin_unlock_bh(&ap_dev->lock);
-	return rc;
-}
-
-static DEVICE_ATTR(pendingq_count, 0444, ap_pendingq_count_show, NULL);
-
-static ssize_t ap_reset_show(struct device *dev,
-				      struct device_attribute *attr, char *buf)
-{
-	struct ap_device *ap_dev = to_ap_dev(dev);
-	int rc = 0;
-
-	spin_lock_bh(&ap_dev->lock);
-	switch (ap_dev->state) {
-	case AP_STATE_RESET_START:
-	case AP_STATE_RESET_WAIT:
-		rc = snprintf(buf, PAGE_SIZE, "Reset in progress.\n");
-		break;
-	case AP_STATE_WORKING:
-	case AP_STATE_QUEUE_FULL:
-		rc = snprintf(buf, PAGE_SIZE, "Reset Timer armed.\n");
-		break;
-	default:
-		rc = snprintf(buf, PAGE_SIZE, "No Reset Timer set.\n");
-	}
-	spin_unlock_bh(&ap_dev->lock);
-	return rc;
-}
-
-static DEVICE_ATTR(reset, 0444, ap_reset_show, NULL);
-
-static ssize_t ap_interrupt_show(struct device *dev,
-				      struct device_attribute *attr, char *buf)
-{
-	struct ap_device *ap_dev = to_ap_dev(dev);
-	int rc = 0;
-
-	spin_lock_bh(&ap_dev->lock);
-	if (ap_dev->state == AP_STATE_SETIRQ_WAIT)
-		rc = snprintf(buf, PAGE_SIZE, "Enable Interrupt pending.\n");
-	else if (ap_dev->interrupt == AP_INTR_ENABLED)
-		rc = snprintf(buf, PAGE_SIZE, "Interrupts enabled.\n");
-	else
-		rc = snprintf(buf, PAGE_SIZE, "Interrupts disabled.\n");
-	spin_unlock_bh(&ap_dev->lock);
-	return rc;
-}
-
-static DEVICE_ATTR(interrupt, 0444, ap_interrupt_show, NULL);
-
-static ssize_t ap_modalias_show(struct device *dev,
-				struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "ap:t%02X\n", to_ap_dev(dev)->device_type);
-}
-
-static DEVICE_ATTR(modalias, 0444, ap_modalias_show, NULL);
-
-static ssize_t ap_functions_show(struct device *dev,
-				 struct device_attribute *attr, char *buf)
-{
-	struct ap_device *ap_dev = to_ap_dev(dev);
-	return snprintf(buf, PAGE_SIZE, "0x%08X\n", ap_dev->functions);
-}
-
-static DEVICE_ATTR(ap_functions, 0444, ap_functions_show, NULL);
-
-static struct attribute *ap_dev_attrs[] = {
-	&dev_attr_hwtype.attr,
-	&dev_attr_raw_hwtype.attr,
-	&dev_attr_depth.attr,
-	&dev_attr_request_count.attr,
-	&dev_attr_requestq_count.attr,
-	&dev_attr_pendingq_count.attr,
-	&dev_attr_reset.attr,
-	&dev_attr_interrupt.attr,
-	&dev_attr_modalias.attr,
-	&dev_attr_ap_functions.attr,
-	NULL
-};
-static struct attribute_group ap_dev_attr_group = {
-	.attrs = ap_dev_attrs
-};
+#define is_card_dev(x) ((x)->parent == ap_root_device)
+#define is_queue_dev(x) ((x)->parent != ap_root_device)
 
 /**
  * ap_bus_match()
@@ -1229,7 +509,6 @@ static struct attribute_group ap_dev_attr_group = {
  */
 static int ap_bus_match(struct device *dev, struct device_driver *drv)
 {
-	struct ap_device *ap_dev = to_ap_dev(dev);
 	struct ap_driver *ap_drv = to_ap_drv(drv);
 	struct ap_device_id *id;
 
@@ -1238,10 +517,14 @@ static int ap_bus_match(struct device *dev, struct device_driver *drv)
 	 * supported types of the device_driver.
 	 */
 	for (id = ap_drv->ids; id->match_flags; id++) {
-		if ((id->match_flags & AP_DEVICE_ID_MATCH_DEVICE_TYPE) &&
-		    (id->dev_type != ap_dev->device_type))
-			continue;
-		return 1;
+		if (is_card_dev(dev) &&
+		    id->match_flags & AP_DEVICE_ID_MATCH_CARD_TYPE &&
+		    id->dev_type == to_ap_dev(dev)->device_type)
+			return 1;
+		if (is_queue_dev(dev) &&
+		    id->match_flags & AP_DEVICE_ID_MATCH_QUEUE_TYPE &&
+		    id->dev_type == to_ap_dev(dev)->device_type)
+			return 1;
 	}
 	return 0;
 }
@@ -1273,27 +556,28 @@ static int ap_uevent (struct device *dev, struct kobj_uevent_env *env)
 	return retval;
 }
 
-static int ap_dev_suspend(struct device *dev, pm_message_t state)
+static int ap_dev_suspend(struct device *dev)
 {
 	struct ap_device *ap_dev = to_ap_dev(dev);
 
-	/* Poll on the device until all requests are finished. */
-	spin_lock_bh(&ap_dev->lock);
-	ap_dev->state = AP_STATE_SUSPEND_WAIT;
-	while (ap_sm_event(ap_dev, AP_EVENT_POLL) != AP_WAIT_NONE)
-		;
-	ap_dev->state = AP_STATE_BORKED;
-	spin_unlock_bh(&ap_dev->lock);
+	if (ap_dev->drv && ap_dev->drv->suspend)
+		ap_dev->drv->suspend(ap_dev);
 	return 0;
 }
 
 static int ap_dev_resume(struct device *dev)
 {
+	struct ap_device *ap_dev = to_ap_dev(dev);
+
+	if (ap_dev->drv && ap_dev->drv->resume)
+		ap_dev->drv->resume(ap_dev);
 	return 0;
 }
 
 static void ap_bus_suspend(void)
 {
+	AP_DBF(DBF_DEBUG, "ap_bus_suspend running\n");
+
 	ap_suspend_flag = 1;
 	/*
 	 * Disable scanning for devices, thus we do not want to scan
@@ -1303,9 +587,25 @@ static void ap_bus_suspend(void)
 	tasklet_disable(&ap_tasklet);
 }
 
-static int __ap_devices_unregister(struct device *dev, void *dummy)
+static int __ap_card_devices_unregister(struct device *dev, void *dummy)
+{
+	if (is_card_dev(dev))
+		device_unregister(dev);
+	return 0;
+}
+
+static int __ap_queue_devices_unregister(struct device *dev, void *dummy)
+{
+	if (is_queue_dev(dev))
+		device_unregister(dev);
+	return 0;
+}
+
+static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data)
 {
-	device_unregister(dev);
+	if (is_queue_dev(dev) &&
+	    AP_QID_CARD(to_ap_queue(dev)->qid) == (int)(long) data)
+		device_unregister(dev);
 	return 0;
 }
 
@@ -1313,8 +613,15 @@ static void ap_bus_resume(void)
 {
 	int rc;
 
-	/* Unconditionally remove all AP devices */
-	bus_for_each_dev(&ap_bus_type, NULL, NULL, __ap_devices_unregister);
+	AP_DBF(DBF_DEBUG, "ap_bus_resume running\n");
+
+	/* remove all queue devices */
+	bus_for_each_dev(&ap_bus_type, NULL, NULL,
+			 __ap_queue_devices_unregister);
+	/* remove all card devices */
+	bus_for_each_dev(&ap_bus_type, NULL, NULL,
+			 __ap_card_devices_unregister);
+
 	/* Reset thin interrupt setting */
 	if (ap_interrupts_available() && !ap_using_interrupts()) {
 		rc = register_adapter_interrupt(&ap_airq);
@@ -1356,25 +663,15 @@ static struct notifier_block ap_power_notifier = {
 	.notifier_call = ap_power_event,
 };
 
+static SIMPLE_DEV_PM_OPS(ap_bus_pm_ops, ap_dev_suspend, ap_dev_resume);
+
 static struct bus_type ap_bus_type = {
 	.name = "ap",
 	.match = &ap_bus_match,
 	.uevent = &ap_uevent,
-	.suspend = ap_dev_suspend,
-	.resume = ap_dev_resume,
+	.pm = &ap_bus_pm_ops,
 };
 
-void ap_device_init_reply(struct ap_device *ap_dev,
-			  struct ap_message *reply)
-{
-	ap_dev->reply = reply;
-
-	spin_lock_bh(&ap_dev->lock);
-	ap_sm_wait(ap_sm_event(ap_dev, AP_EVENT_POLL));
-	spin_unlock_bh(&ap_dev->lock);
-}
-EXPORT_SYMBOL(ap_device_init_reply);
-
 static int ap_device_probe(struct device *dev)
 {
 	struct ap_device *ap_dev = to_ap_dev(dev);
@@ -1388,61 +685,22 @@ static int ap_device_probe(struct device *dev)
 	return rc;
 }
 
-/**
- * __ap_flush_queue(): Flush requests.
- * @ap_dev: Pointer to the AP device
- *
- * Flush all requests from the request/pending queue of an AP device.
- */
-static void __ap_flush_queue(struct ap_device *ap_dev)
-{
-	struct ap_message *ap_msg, *next;
-
-	list_for_each_entry_safe(ap_msg, next, &ap_dev->pendingq, list) {
-		list_del_init(&ap_msg->list);
-		ap_dev->pendingq_count--;
-		ap_msg->rc = -EAGAIN;
-		ap_msg->receive(ap_dev, ap_msg, NULL);
-	}
-	list_for_each_entry_safe(ap_msg, next, &ap_dev->requestq, list) {
-		list_del_init(&ap_msg->list);
-		ap_dev->requestq_count--;
-		ap_msg->rc = -EAGAIN;
-		ap_msg->receive(ap_dev, ap_msg, NULL);
-	}
-}
-
-void ap_flush_queue(struct ap_device *ap_dev)
-{
-	spin_lock_bh(&ap_dev->lock);
-	__ap_flush_queue(ap_dev);
-	spin_unlock_bh(&ap_dev->lock);
-}
-EXPORT_SYMBOL(ap_flush_queue);
-
 static int ap_device_remove(struct device *dev)
 {
 	struct ap_device *ap_dev = to_ap_dev(dev);
 	struct ap_driver *ap_drv = ap_dev->drv;
 
-	ap_flush_queue(ap_dev);
-	del_timer_sync(&ap_dev->timeout);
-	spin_lock_bh(&ap_device_list_lock);
-	list_del_init(&ap_dev->list);
-	spin_unlock_bh(&ap_device_list_lock);
+	spin_lock_bh(&ap_list_lock);
+	if (is_card_dev(dev))
+		list_del_init(&to_ap_card(dev)->list);
+	else
+		list_del_init(&to_ap_queue(dev)->list);
+	spin_unlock_bh(&ap_list_lock);
 	if (ap_drv->remove)
 		ap_drv->remove(ap_dev);
-	spin_lock_bh(&ap_dev->lock);
-	atomic_sub(ap_dev->queue_count, &ap_poll_requests);
-	spin_unlock_bh(&ap_dev->lock);
 	return 0;
 }
 
-static void ap_device_release(struct device *dev)
-{
-	kfree(to_ap_dev(dev));
-}
-
 int ap_driver_register(struct ap_driver *ap_drv, struct module *owner,
 		       char *name)
 {
@@ -1485,18 +743,30 @@ static ssize_t ap_domain_show(struct bus_type *bus, char *buf)
 	return snprintf(buf, PAGE_SIZE, "%d\n", ap_domain_index);
 }
 
-static BUS_ATTR(ap_domain, 0444, ap_domain_show, NULL);
+static ssize_t ap_domain_store(struct bus_type *bus,
+			       const char *buf, size_t count)
+{
+	int domain;
+
+	if (sscanf(buf, "%i\n", &domain) != 1 ||
+	    domain < 0 || domain > ap_max_domain_id)
+		return -EINVAL;
+	spin_lock_bh(&ap_domain_lock);
+	ap_domain_index = domain;
+	spin_unlock_bh(&ap_domain_lock);
+
+	AP_DBF(DBF_DEBUG, "store new default domain=%d\n", domain);
+
+	return count;
+}
+
+static BUS_ATTR(ap_domain, 0644, ap_domain_show, ap_domain_store);
 
 static ssize_t ap_control_domain_mask_show(struct bus_type *bus, char *buf)
 {
 	if (!ap_configuration)	/* QCI not supported */
 		return snprintf(buf, PAGE_SIZE, "not supported\n");
-	if (!test_facility(76))
-		/* format 0 - 16 bit domain field */
-		return snprintf(buf, PAGE_SIZE, "%08x%08x\n",
-				ap_configuration->adm[0],
-				ap_configuration->adm[1]);
-	/* format 1 - 256 bit domain field */
+
 	return snprintf(buf, PAGE_SIZE,
 			"0x%08x%08x%08x%08x%08x%08x%08x%08x\n",
 			ap_configuration->adm[0], ap_configuration->adm[1],
@@ -1508,6 +778,22 @@ static ssize_t ap_control_domain_mask_show(struct bus_type *bus, char *buf)
 static BUS_ATTR(ap_control_domain_mask, 0444,
 		ap_control_domain_mask_show, NULL);
 
+static ssize_t ap_usage_domain_mask_show(struct bus_type *bus, char *buf)
+{
+	if (!ap_configuration)	/* QCI not supported */
+		return snprintf(buf, PAGE_SIZE, "not supported\n");
+
+	return snprintf(buf, PAGE_SIZE,
+			"0x%08x%08x%08x%08x%08x%08x%08x%08x\n",
+			ap_configuration->aqm[0], ap_configuration->aqm[1],
+			ap_configuration->aqm[2], ap_configuration->aqm[3],
+			ap_configuration->aqm[4], ap_configuration->aqm[5],
+			ap_configuration->aqm[6], ap_configuration->aqm[7]);
+}
+
+static BUS_ATTR(ap_usage_domain_mask, 0444,
+		ap_usage_domain_mask_show, NULL);
+
 static ssize_t ap_config_time_show(struct bus_type *bus, char *buf)
 {
 	return snprintf(buf, PAGE_SIZE, "%d\n", ap_config_time);
@@ -1603,6 +889,7 @@ static BUS_ATTR(ap_max_domain_id, 0444, ap_max_domain_id_show, NULL);
 static struct bus_attribute *const ap_bus_attrs[] = {
 	&bus_attr_ap_domain,
 	&bus_attr_ap_control_domain_mask,
+	&bus_attr_ap_usage_domain_mask,
 	&bus_attr_config_time,
 	&bus_attr_poll_thread,
 	&bus_attr_ap_interrupts,
@@ -1627,9 +914,12 @@ static int ap_select_domain(void)
 	 * the "domain=" parameter or the domain with the maximum number
 	 * of devices.
 	 */
-	if (ap_domain_index >= 0)
+	spin_lock_bh(&ap_domain_lock);
+	if (ap_domain_index >= 0) {
 		/* Domain has already been selected. */
+		spin_unlock_bh(&ap_domain_lock);
 		return 0;
+	}
 	best_domain = -1;
 	max_count = 0;
 	for (i = 0; i < AP_DOMAINS; i++) {
@@ -1651,109 +941,171 @@ static int ap_select_domain(void)
 	}
 	if (best_domain >= 0){
 		ap_domain_index = best_domain;
+		spin_unlock_bh(&ap_domain_lock);
 		return 0;
 	}
+	spin_unlock_bh(&ap_domain_lock);
 	return -ENODEV;
 }
 
-/**
- * __ap_scan_bus(): Scan the AP bus.
- * @dev: Pointer to device
- * @data: Pointer to data
- *
- * Scan the AP bus for new devices.
+/*
+ * helper function to be used with bus_find_dev
+ * matches for the card device with the given id
  */
-static int __ap_scan_bus(struct device *dev, void *data)
+static int __match_card_device_with_id(struct device *dev, void *data)
 {
-	return to_ap_dev(dev)->qid == (ap_qid_t)(unsigned long) data;
+	return is_card_dev(dev) && to_ap_card(dev)->id == (int)(long) data;
 }
 
+/* helper function to be used with bus_find_dev
+ * matches for the queue device with a given qid
+ */
+static int __match_queue_device_with_qid(struct device *dev, void *data)
+{
+	return is_queue_dev(dev) && to_ap_queue(dev)->qid == (int)(long) data;
+}
+
+/**
+ * ap_scan_bus(): Scan the AP bus for new devices
+ * Runs periodically, workqueue timer (ap_config_time)
+ */
 static void ap_scan_bus(struct work_struct *unused)
 {
-	struct ap_device *ap_dev;
+	struct ap_queue *aq;
+	struct ap_card *ac;
 	struct device *dev;
 	ap_qid_t qid;
-	int queue_depth = 0, device_type = 0;
-	unsigned int device_functions = 0;
-	int rc, i, borked;
+	int depth = 0, type = 0;
+	unsigned int functions = 0;
+	int rc, id, dom, borked, domains;
+
+	AP_DBF(DBF_DEBUG, "ap_scan_bus running\n");
 
 	ap_query_configuration();
 	if (ap_select_domain() != 0)
 		goto out;
 
-	for (i = 0; i < AP_DEVICES; i++) {
-		qid = AP_MKQID(i, ap_domain_index);
+	for (id = 0; id < AP_DEVICES; id++) {
+		/* check if device is registered */
 		dev = bus_find_device(&ap_bus_type, NULL,
-				      (void *)(unsigned long)qid,
-				      __ap_scan_bus);
-		rc = ap_query_queue(qid, &queue_depth, &device_type,
-				    &device_functions);
-		if (dev) {
-			ap_dev = to_ap_dev(dev);
-			spin_lock_bh(&ap_dev->lock);
-			if (rc == -ENODEV)
-				ap_dev->state = AP_STATE_BORKED;
-			borked = ap_dev->state == AP_STATE_BORKED;
-			spin_unlock_bh(&ap_dev->lock);
-			if (borked)	/* Remove broken device */
+				      (void *)(long) id,
+				      __match_card_device_with_id);
+		ac = dev ? to_ap_card(dev) : NULL;
+		if (!ap_test_config_card_id(id)) {
+			if (dev) {
+				/* Card device has been removed from
+				 * configuration, remove the belonging
+				 * queue devices.
+				 */
+				bus_for_each_dev(&ap_bus_type, NULL,
+					(void *)(long) id,
+					__ap_queue_devices_with_id_unregister);
+				/* now remove the card device */
 				device_unregister(dev);
-			put_device(dev);
-			if (!borked)
-				continue;
-		}
-		if (rc)
-			continue;
-		ap_dev = kzalloc(sizeof(*ap_dev), GFP_KERNEL);
-		if (!ap_dev)
-			break;
-		ap_dev->qid = qid;
-		ap_dev->state = AP_STATE_RESET_START;
-		ap_dev->interrupt = AP_INTR_DISABLED;
-		ap_dev->queue_depth = queue_depth;
-		ap_dev->raw_hwtype = device_type;
-		ap_dev->device_type = device_type;
-		ap_dev->functions = device_functions;
-		spin_lock_init(&ap_dev->lock);
-		INIT_LIST_HEAD(&ap_dev->pendingq);
-		INIT_LIST_HEAD(&ap_dev->requestq);
-		INIT_LIST_HEAD(&ap_dev->list);
-		setup_timer(&ap_dev->timeout, ap_request_timeout,
-			    (unsigned long) ap_dev);
-
-		ap_dev->device.bus = &ap_bus_type;
-		ap_dev->device.parent = ap_root_device;
-		rc = dev_set_name(&ap_dev->device, "card%02x",
-				  AP_QID_DEVICE(ap_dev->qid));
-		if (rc) {
-			kfree(ap_dev);
-			continue;
-		}
-		/* Add to list of devices */
-		spin_lock_bh(&ap_device_list_lock);
-		list_add(&ap_dev->list, &ap_device_list);
-		spin_unlock_bh(&ap_device_list_lock);
-		/* Start with a device reset */
-		spin_lock_bh(&ap_dev->lock);
-		ap_sm_wait(ap_sm_event(ap_dev, AP_EVENT_POLL));
-		spin_unlock_bh(&ap_dev->lock);
-		/* Register device */
-		ap_dev->device.release = ap_device_release;
-		rc = device_register(&ap_dev->device);
-		if (rc) {
-			spin_lock_bh(&ap_dev->lock);
-			list_del_init(&ap_dev->list);
-			spin_unlock_bh(&ap_dev->lock);
-			put_device(&ap_dev->device);
+				put_device(dev);
+			}
 			continue;
 		}
-		/* Add device attributes. */
-		rc = sysfs_create_group(&ap_dev->device.kobj,
-					&ap_dev_attr_group);
-		if (rc) {
-			device_unregister(&ap_dev->device);
-			continue;
+		/* According to the configuration there should be a card
+		 * device, so check if there is at least one valid queue
+		 * and maybe create queue devices and the card device.
+		 */
+		domains = 0;
+		for (dom = 0; dom < AP_DOMAINS; dom++) {
+			qid = AP_MKQID(id, dom);
+			dev = bus_find_device(&ap_bus_type, NULL,
+					      (void *)(long) qid,
+					      __match_queue_device_with_qid);
+			aq = dev ? to_ap_queue(dev) : NULL;
+			if (!ap_test_config_domain(dom)) {
+				if (dev) {
+					/* Queue device exists but has been
+					 * removed from configuration.
+					 */
+					device_unregister(dev);
+					put_device(dev);
+				}
+				continue;
+			}
+			rc = ap_query_queue(qid, &depth, &type, &functions);
+			if (dev) {
+				spin_lock_bh(&aq->lock);
+				if (rc == -ENODEV ||
+				    /* adapter reconfiguration */
+				    (ac && ac->functions != functions))
+					aq->state = AP_STATE_BORKED;
+				borked = aq->state == AP_STATE_BORKED;
+				spin_unlock_bh(&aq->lock);
+				if (borked)	/* Remove broken device */
+					device_unregister(dev);
+				put_device(dev);
+				if (!borked) {
+					domains++;
+					continue;
+				}
+			}
+			if (rc)
+				continue;
+			/* new queue device needed */
+			if (!ac) {
+				/* but first create the card device */
+				ac = ap_card_create(id, depth,
+						    type, functions);
+				if (!ac)
+					continue;
+				ac->ap_dev.device.bus = &ap_bus_type;
+				ac->ap_dev.device.parent = ap_root_device;
+				dev_set_name(&ac->ap_dev.device,
+					     "card%02x", id);
+				/* Register card with AP bus */
+				rc = device_register(&ac->ap_dev.device);
+				if (rc) {
+					put_device(&ac->ap_dev.device);
+					ac = NULL;
+					break;
+				}
+				/* get it and thus adjust reference counter */
+				get_device(&ac->ap_dev.device);
+				/* Add card device to card list */
+				spin_lock_bh(&ap_list_lock);
+				list_add(&ac->list, &ap_card_list);
+				spin_unlock_bh(&ap_list_lock);
+			}
+			/* now create the new queue device */
+			aq = ap_queue_create(qid, type);
+			if (!aq)
+				continue;
+			aq->card = ac;
+			aq->ap_dev.device.bus = &ap_bus_type;
+			aq->ap_dev.device.parent = &ac->ap_dev.device;
+			dev_set_name(&aq->ap_dev.device,
+				     "%02x.%04x", id, dom);
+			/* Add queue device to card queue list */
+			spin_lock_bh(&ap_list_lock);
+			list_add(&aq->list, &ac->queues);
+			spin_unlock_bh(&ap_list_lock);
+			/* Start with a device reset */
+			spin_lock_bh(&aq->lock);
+			ap_wait(ap_sm_event(aq, AP_EVENT_POLL));
+			spin_unlock_bh(&aq->lock);
+			/* Register device */
+			rc = device_register(&aq->ap_dev.device);
+			if (rc) {
+				spin_lock_bh(&ap_list_lock);
+				list_del_init(&aq->list);
+				spin_unlock_bh(&ap_list_lock);
+				put_device(&aq->ap_dev.device);
+				continue;
+			}
+			domains++;
+		} /* end domain loop */
+		if (ac) {
+			/* remove card dev if there are no queue devices */
+			if (!domains)
+				device_unregister(&ac->ap_dev.device);
+			put_device(&ac->ap_dev.device);
 		}
-	}
+	} /* end device loop */
 out:
 	mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ);
 }
@@ -1772,7 +1124,7 @@ static void ap_reset_domain(void)
 	if (ap_domain_index == -1 || !ap_test_config_domain(ap_domain_index))
 		return;
 	for (i = 0; i < AP_DEVICES; i++)
-		ap_reset_queue(AP_MKQID(i, ap_domain_index));
+		ap_rapq(AP_MKQID(i, ap_domain_index));
 }
 
 static void ap_reset_all(void)
@@ -1785,7 +1137,7 @@ static void ap_reset_all(void)
 		for (j = 0; j < AP_DEVICES; j++) {
 			if (!ap_test_config_card_id(j))
 				continue;
-			ap_reset_queue(AP_MKQID(j, i));
+			ap_rapq(AP_MKQID(j, i));
 		}
 	}
 }
@@ -1794,6 +1146,23 @@ static struct reset_call ap_reset_call = {
 	.fn = ap_reset_all,
 };
 
+int __init ap_debug_init(void)
+{
+	ap_dbf_root = debugfs_create_dir("ap", NULL);
+	ap_dbf_info = debug_register("ap", 1, 1,
+				     DBF_MAX_SPRINTF_ARGS * sizeof(long));
+	debug_register_view(ap_dbf_info, &debug_sprintf_view);
+	debug_set_level(ap_dbf_info, DBF_ERR);
+
+	return 0;
+}
+
+void ap_debug_exit(void)
+{
+	debugfs_remove(ap_dbf_root);
+	debug_unregister(ap_dbf_info);
+}
+
 /**
  * ap_module_init(): The module initialization code.
  *
@@ -1804,6 +1173,10 @@ int __init ap_module_init(void)
 	int max_domain_id;
 	int rc, i;
 
+	rc = ap_debug_init();
+	if (rc)
+		return rc;
+
 	if (ap_instructions_available() != 0) {
 		pr_warn("The hardware system does not support AP instructions\n");
 		return -ENODEV;
@@ -1913,7 +1286,15 @@ void ap_module_exit(void)
 	del_timer_sync(&ap_config_timer);
 	hrtimer_cancel(&ap_poll_timer);
 	tasklet_kill(&ap_tasklet);
-	bus_for_each_dev(&ap_bus_type, NULL, NULL, __ap_devices_unregister);
+
+	/* first remove queue devices */
+	bus_for_each_dev(&ap_bus_type, NULL, NULL,
+			 __ap_queue_devices_unregister);
+	/* now remove the card devices */
+	bus_for_each_dev(&ap_bus_type, NULL, NULL,
+			 __ap_card_devices_unregister);
+
+	/* remove bus attributes */
 	for (i = 0; ap_bus_attrs[i]; i++)
 		bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
 	unregister_pm_notifier(&ap_power_notifier);
@@ -1923,6 +1304,8 @@ void ap_module_exit(void)
 	unregister_reset_call(&ap_reset_call);
 	if (ap_using_interrupts())
 		unregister_adapter_interrupt(&ap_airq);
+
+	ap_debug_exit();
 }
 
 module_init(ap_module_init);
diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h
index d7fdf5c024d7..4dc7c88fb054 100644
--- a/drivers/s390/crypto/ap_bus.h
+++ b/drivers/s390/crypto/ap_bus.h
@@ -27,7 +27,6 @@
 #define _AP_BUS_H_
 
 #include <linux/device.h>
-#include <linux/mod_devicetable.h>
 #include <linux/types.h>
 
 #define AP_DEVICES 64		/* Number of AP devices. */
@@ -38,14 +37,17 @@
 
 extern int ap_domain_index;
 
+extern spinlock_t ap_list_lock;
+extern struct list_head ap_card_list;
+
 /**
  * The ap_qid_t identifier of an ap queue. It contains a
- * 6 bit device index and a 4 bit queue index (domain).
+ * 6 bit card index and a 4 bit queue index (domain).
  */
 typedef unsigned int ap_qid_t;
 
-#define AP_MKQID(_device, _queue) (((_device) & 63) << 8 | ((_queue) & 255))
-#define AP_QID_DEVICE(_qid) (((_qid) >> 8) & 63)
+#define AP_MKQID(_card, _queue) (((_card) & 63) << 8 | ((_queue) & 255))
+#define AP_QID_CARD(_qid) (((_qid) >> 8) & 63)
 #define AP_QID_QUEUE(_qid) ((_qid) & 255)
 
 /**
@@ -55,7 +57,7 @@ typedef unsigned int ap_qid_t;
  * @queue_full: Is 1 if the queue is full
  * @pad: A 4 bit pad
  * @int_enabled: Shows if interrupts are enabled for the AP
- * @response_conde: Holds the 8 bit response code
+ * @response_code: Holds the 8 bit response code
  * @pad2: A 16 bit pad
  *
  * The ap queue status word is returned by all three AP functions
@@ -105,6 +107,7 @@ static inline int ap_test_bit(unsigned int *ptr, unsigned int nr)
 #define AP_DEVICE_TYPE_CEX3C	9
 #define AP_DEVICE_TYPE_CEX4	10
 #define AP_DEVICE_TYPE_CEX5	11
+#define AP_DEVICE_TYPE_CEX6	12
 
 /*
  * Known function facilities
@@ -166,7 +169,8 @@ struct ap_driver {
 
 	int (*probe)(struct ap_device *);
 	void (*remove)(struct ap_device *);
-	int request_timeout;		/* request timeout in jiffies */
+	void (*suspend)(struct ap_device *);
+	void (*resume)(struct ap_device *);
 };
 
 #define to_ap_drv(x) container_of((x), struct ap_driver, driver)
@@ -174,38 +178,51 @@ struct ap_driver {
 int ap_driver_register(struct ap_driver *, struct module *, char *);
 void ap_driver_unregister(struct ap_driver *);
 
-typedef enum ap_wait (ap_func_t)(struct ap_device *ap_dev);
-
 struct ap_device {
 	struct device device;
 	struct ap_driver *drv;		/* Pointer to AP device driver. */
-	spinlock_t lock;		/* Per device lock. */
-	struct list_head list;		/* private list of all AP devices. */
+	int device_type;		/* AP device type. */
+};
 
-	enum ap_state state;		/* State of the AP device. */
+#define to_ap_dev(x) container_of((x), struct ap_device, device)
 
-	ap_qid_t qid;			/* AP queue id. */
-	int queue_depth;		/* AP queue depth.*/
-	int device_type;		/* AP device type. */
+struct ap_card {
+	struct ap_device ap_dev;
+	struct list_head list;		/* Private list of AP cards. */
+	struct list_head queues;	/* List of assoc. AP queues */
+	void *private;			/* ap driver private pointer. */
 	int raw_hwtype;			/* AP raw hardware type. */
 	unsigned int functions;		/* AP device function bitfield. */
-	struct timer_list timeout;	/* Timer for request timeouts. */
+	int queue_depth;		/* AP queue depth.*/
+	int id;				/* AP card number. */
+	atomic_t total_request_count;	/* # requests ever for this AP device.*/
+};
+
+#define to_ap_card(x) container_of((x), struct ap_card, ap_dev.device)
 
+struct ap_queue {
+	struct ap_device ap_dev;
+	struct list_head list;		/* Private list of AP queues. */
+	struct ap_card *card;		/* Ptr to assoc. AP card. */
+	spinlock_t lock;		/* Per device lock. */
+	void *private;			/* ap driver private pointer. */
+	ap_qid_t qid;			/* AP queue id. */
 	int interrupt;			/* indicate if interrupts are enabled */
 	int queue_count;		/* # messages currently on AP queue. */
-
-	struct list_head pendingq;	/* List of message sent to AP queue. */
+	enum ap_state state;		/* State of the AP device. */
 	int pendingq_count;		/* # requests on pendingq list. */
-	struct list_head requestq;	/* List of message yet to be sent. */
 	int requestq_count;		/* # requests on requestq list. */
-	int total_request_count;	/* # requests ever for this AP device. */
-
+	int total_request_count;	/* # requests ever for this AP device.*/
+	int request_timeout;		/* Request timout in jiffies. */
+	struct timer_list timeout;	/* Timer for request timeouts. */
+	struct list_head pendingq;	/* List of message sent to AP queue. */
+	struct list_head requestq;	/* List of message yet to be sent. */
 	struct ap_message *reply;	/* Per device reply message. */
-
-	void *private;			/* ap driver private pointer. */
 };
 
-#define to_ap_dev(x) container_of((x), struct ap_device, device)
+#define to_ap_queue(x) container_of((x), struct ap_queue, ap_dev.device)
+
+typedef enum ap_wait (ap_func_t)(struct ap_queue *queue);
 
 struct ap_message {
 	struct list_head list;		/* Request queueing. */
@@ -217,7 +234,7 @@ struct ap_message {
 	void *private;			/* ap driver private pointer. */
 	unsigned int special:1;		/* Used for special commands. */
 	/* receive is called from tasklet context */
-	void (*receive)(struct ap_device *, struct ap_message *,
+	void (*receive)(struct ap_queue *, struct ap_message *,
 			struct ap_message *);
 };
 
@@ -232,10 +249,6 @@ struct ap_config_info {
 	unsigned char reserved4[16];
 } __packed;
 
-#define AP_DEVICE(dt)					\
-	.dev_type=(dt),					\
-	.match_flags=AP_DEVICE_ID_MATCH_DEVICE_TYPE,
-
 /**
  * ap_init_message() - Initialize ap_message.
  * Initialize a message before using. Otherwise this might result in
@@ -250,6 +263,12 @@ static inline void ap_init_message(struct ap_message *ap_msg)
 	ap_msg->receive = NULL;
 }
 
+#define for_each_ap_card(_ac) \
+	list_for_each_entry(_ac, &ap_card_list, list)
+
+#define for_each_ap_queue(_aq, _ac) \
+	list_for_each_entry(_aq, &(_ac)->queues, list)
+
 /*
  * Note: don't use ap_send/ap_recv after using ap_queue_message
  * for the first time. Otherwise the ap message queue will get
@@ -258,11 +277,26 @@ static inline void ap_init_message(struct ap_message *ap_msg)
 int ap_send(ap_qid_t, unsigned long long, void *, size_t);
 int ap_recv(ap_qid_t, unsigned long long *, void *, size_t);
 
-void ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg);
-void ap_cancel_message(struct ap_device *ap_dev, struct ap_message *ap_msg);
-void ap_flush_queue(struct ap_device *ap_dev);
+enum ap_wait ap_sm_event(struct ap_queue *aq, enum ap_event event);
+enum ap_wait ap_sm_event_loop(struct ap_queue *aq, enum ap_event event);
+
+void ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg);
+void ap_cancel_message(struct ap_queue *aq, struct ap_message *ap_msg);
+void ap_flush_queue(struct ap_queue *aq);
+
+void *ap_airq_ptr(void);
+void ap_wait(enum ap_wait wait);
+void ap_request_timeout(unsigned long data);
 void ap_bus_force_rescan(void);
-void ap_device_init_reply(struct ap_device *ap_dev, struct ap_message *ap_msg);
+
+void ap_queue_init_reply(struct ap_queue *aq, struct ap_message *ap_msg);
+struct ap_queue *ap_queue_create(ap_qid_t qid, int device_type);
+void ap_queue_remove(struct ap_queue *aq);
+void ap_queue_suspend(struct ap_device *ap_dev);
+void ap_queue_resume(struct ap_device *ap_dev);
+
+struct ap_card *ap_card_create(int id, int queue_depth, int device_type,
+			       unsigned int device_functions);
 
 int ap_module_init(void);
 void ap_module_exit(void);
diff --git a/drivers/s390/crypto/ap_card.c b/drivers/s390/crypto/ap_card.c
new file mode 100644
index 000000000000..0110d44172a3
--- /dev/null
+++ b/drivers/s390/crypto/ap_card.c
@@ -0,0 +1,170 @@
+/*
+ * Copyright IBM Corp. 2016
+ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * Adjunct processor bus, card related code.
+ */
+
+#define KMSG_COMPONENT "ap"
+#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
+
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <asm/facility.h>
+
+#include "ap_bus.h"
+#include "ap_asm.h"
+
+/*
+ * AP card related attributes.
+ */
+static ssize_t ap_hwtype_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", ac->ap_dev.device_type);
+}
+
+static DEVICE_ATTR(hwtype, 0444, ap_hwtype_show, NULL);
+
+static ssize_t ap_raw_hwtype_show(struct device *dev,
+				  struct device_attribute *attr, char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", ac->raw_hwtype);
+}
+
+static DEVICE_ATTR(raw_hwtype, 0444, ap_raw_hwtype_show, NULL);
+
+static ssize_t ap_depth_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", ac->queue_depth);
+}
+
+static DEVICE_ATTR(depth, 0444, ap_depth_show, NULL);
+
+static ssize_t ap_functions_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+
+	return snprintf(buf, PAGE_SIZE, "0x%08X\n", ac->functions);
+}
+
+static DEVICE_ATTR(ap_functions, 0444, ap_functions_show, NULL);
+
+static ssize_t ap_request_count_show(struct device *dev,
+				     struct device_attribute *attr,
+				     char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+	unsigned int req_cnt;
+
+	req_cnt = 0;
+	spin_lock_bh(&ap_list_lock);
+	req_cnt = atomic_read(&ac->total_request_count);
+	spin_unlock_bh(&ap_list_lock);
+	return snprintf(buf, PAGE_SIZE, "%d\n", req_cnt);
+}
+
+static DEVICE_ATTR(request_count, 0444, ap_request_count_show, NULL);
+
+static ssize_t ap_requestq_count_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+	struct ap_queue *aq;
+	unsigned int reqq_cnt;
+
+	reqq_cnt = 0;
+	spin_lock_bh(&ap_list_lock);
+	for_each_ap_queue(aq, ac)
+		reqq_cnt += aq->requestq_count;
+	spin_unlock_bh(&ap_list_lock);
+	return snprintf(buf, PAGE_SIZE, "%d\n", reqq_cnt);
+}
+
+static DEVICE_ATTR(requestq_count, 0444, ap_requestq_count_show, NULL);
+
+static ssize_t ap_pendingq_count_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct ap_card *ac = to_ap_card(dev);
+	struct ap_queue *aq;
+	unsigned int penq_cnt;
+
+	penq_cnt = 0;
+	spin_lock_bh(&ap_list_lock);
+	for_each_ap_queue(aq, ac)
+		penq_cnt += aq->pendingq_count;
+	spin_unlock_bh(&ap_list_lock);
+	return snprintf(buf, PAGE_SIZE, "%d\n", penq_cnt);
+}
+
+static DEVICE_ATTR(pendingq_count, 0444, ap_pendingq_count_show, NULL);
+
+static ssize_t ap_modalias_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "ap:t%02X\n", to_ap_dev(dev)->device_type);
+}
+
+static DEVICE_ATTR(modalias, 0444, ap_modalias_show, NULL);
+
+static struct attribute *ap_card_dev_attrs[] = {
+	&dev_attr_hwtype.attr,
+	&dev_attr_raw_hwtype.attr,
+	&dev_attr_depth.attr,
+	&dev_attr_ap_functions.attr,
+	&dev_attr_request_count.attr,
+	&dev_attr_requestq_count.attr,
+	&dev_attr_pendingq_count.attr,
+	&dev_attr_modalias.attr,
+	NULL
+};
+
+static struct attribute_group ap_card_dev_attr_group = {
+	.attrs = ap_card_dev_attrs
+};
+
+static const struct attribute_group *ap_card_dev_attr_groups[] = {
+	&ap_card_dev_attr_group,
+	NULL
+};
+
+struct device_type ap_card_type = {
+	.name = "ap_card",
+	.groups = ap_card_dev_attr_groups,
+};
+
+static void ap_card_device_release(struct device *dev)
+{
+	kfree(to_ap_card(dev));
+}
+
+struct ap_card *ap_card_create(int id, int queue_depth, int device_type,
+			       unsigned int functions)
+{
+	struct ap_card *ac;
+
+	ac = kzalloc(sizeof(*ac), GFP_KERNEL);
+	if (!ac)
+		return NULL;
+	INIT_LIST_HEAD(&ac->queues);
+	ac->ap_dev.device.release = ap_card_device_release;
+	ac->ap_dev.device.type = &ap_card_type;
+	ac->ap_dev.device_type = device_type;
+	/* CEX6 toleration: map to CEX5 */
+	if (device_type == AP_DEVICE_TYPE_CEX6)
+		ac->ap_dev.device_type = AP_DEVICE_TYPE_CEX5;
+	ac->raw_hwtype = device_type;
+	ac->queue_depth = queue_depth;
+	ac->functions = functions;
+	ac->id = id;
+	return ac;
+}
diff --git a/drivers/s390/crypto/ap_debug.h b/drivers/s390/crypto/ap_debug.h
new file mode 100644
index 000000000000..78dbff842dae
--- /dev/null
+++ b/drivers/s390/crypto/ap_debug.h
@@ -0,0 +1,28 @@
+/*
+ *  Copyright IBM Corp. 2016
+ *  Author(s): Harald Freudenberger <freude@de.ibm.com>
+ */
+#ifndef AP_DEBUG_H
+#define AP_DEBUG_H
+
+#include <asm/debug.h>
+
+#define DBF_ERR		3	/* error conditions   */
+#define DBF_WARN	4	/* warning conditions */
+#define DBF_INFO	5	/* informational      */
+#define DBF_DEBUG	6	/* for debugging only */
+
+#define RC2ERR(rc) ((rc) ? DBF_ERR : DBF_INFO)
+#define RC2WARN(rc) ((rc) ? DBF_WARN : DBF_INFO)
+
+#define DBF_MAX_SPRINTF_ARGS 5
+
+#define AP_DBF(...)					\
+	debug_sprintf_event(ap_dbf_info, ##__VA_ARGS__)
+
+extern debug_info_t *ap_dbf_info;
+
+int ap_debug_init(void);
+void ap_debug_exit(void);
+
+#endif /* AP_DEBUG_H */
diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c
new file mode 100644
index 000000000000..b58a917dc510
--- /dev/null
+++ b/drivers/s390/crypto/ap_queue.c
@@ -0,0 +1,701 @@
+/*
+ * Copyright IBM Corp. 2016
+ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * Adjunct processor bus, queue related code.
+ */
+
+#define KMSG_COMPONENT "ap"
+#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
+
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <asm/facility.h>
+
+#include "ap_bus.h"
+#include "ap_asm.h"
+
+/**
+ * ap_queue_enable_interruption(): Enable interruption on an AP queue.
+ * @qid: The AP queue number
+ * @ind: the notification indicator byte
+ *
+ * Enables interruption on AP queue via ap_aqic(). Based on the return
+ * value it waits a while and tests the AP queue if interrupts
+ * have been switched on using ap_test_queue().
+ */
+static int ap_queue_enable_interruption(struct ap_queue *aq, void *ind)
+{
+	struct ap_queue_status status;
+
+	status = ap_aqic(aq->qid, ind);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+	case AP_RESPONSE_OTHERWISE_CHANGED:
+		return 0;
+	case AP_RESPONSE_Q_NOT_AVAIL:
+	case AP_RESPONSE_DECONFIGURED:
+	case AP_RESPONSE_CHECKSTOPPED:
+	case AP_RESPONSE_INVALID_ADDRESS:
+		pr_err("Registering adapter interrupts for AP device %02x.%04x failed\n",
+		       AP_QID_CARD(aq->qid),
+		       AP_QID_QUEUE(aq->qid));
+		return -EOPNOTSUPP;
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+	case AP_RESPONSE_BUSY:
+	default:
+		return -EBUSY;
+	}
+}
+
+/**
+ * __ap_send(): Send message to adjunct processor queue.
+ * @qid: The AP queue number
+ * @psmid: The program supplied message identifier
+ * @msg: The message text
+ * @length: The message length
+ * @special: Special Bit
+ *
+ * Returns AP queue status structure.
+ * Condition code 1 on NQAP can't happen because the L bit is 1.
+ * Condition code 2 on NQAP also means the send is incomplete,
+ * because a segment boundary was reached. The NQAP is repeated.
+ */
+static inline struct ap_queue_status
+__ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length,
+	  unsigned int special)
+{
+	if (special == 1)
+		qid |= 0x400000UL;
+	return ap_nqap(qid, psmid, msg, length);
+}
+
+int ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length)
+{
+	struct ap_queue_status status;
+
+	status = __ap_send(qid, psmid, msg, length, 0);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		return 0;
+	case AP_RESPONSE_Q_FULL:
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+		return -EBUSY;
+	case AP_RESPONSE_REQ_FAC_NOT_INST:
+		return -EINVAL;
+	default:	/* Device is gone. */
+		return -ENODEV;
+	}
+}
+EXPORT_SYMBOL(ap_send);
+
+int ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
+{
+	struct ap_queue_status status;
+
+	if (msg == NULL)
+		return -EINVAL;
+	status = ap_dqap(qid, psmid, msg, length);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		return 0;
+	case AP_RESPONSE_NO_PENDING_REPLY:
+		if (status.queue_empty)
+			return -ENOENT;
+		return -EBUSY;
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+		return -EBUSY;
+	default:
+		return -ENODEV;
+	}
+}
+EXPORT_SYMBOL(ap_recv);
+
+/* State machine definitions and helpers */
+
+static enum ap_wait ap_sm_nop(struct ap_queue *aq)
+{
+	return AP_WAIT_NONE;
+}
+
+/**
+ * ap_sm_recv(): Receive pending reply messages from an AP queue but do
+ *	not change the state of the device.
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_WAIT_NONE, AP_WAIT_AGAIN, or AP_WAIT_INTERRUPT
+ */
+static struct ap_queue_status ap_sm_recv(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+	struct ap_message *ap_msg;
+
+	status = ap_dqap(aq->qid, &aq->reply->psmid,
+			 aq->reply->message, aq->reply->length);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		aq->queue_count--;
+		if (aq->queue_count > 0)
+			mod_timer(&aq->timeout,
+				  jiffies + aq->request_timeout);
+		list_for_each_entry(ap_msg, &aq->pendingq, list) {
+			if (ap_msg->psmid != aq->reply->psmid)
+				continue;
+			list_del_init(&ap_msg->list);
+			aq->pendingq_count--;
+			ap_msg->receive(aq, ap_msg, aq->reply);
+			break;
+		}
+	case AP_RESPONSE_NO_PENDING_REPLY:
+		if (!status.queue_empty || aq->queue_count <= 0)
+			break;
+		/* The card shouldn't forget requests but who knows. */
+		aq->queue_count = 0;
+		list_splice_init(&aq->pendingq, &aq->requestq);
+		aq->requestq_count += aq->pendingq_count;
+		aq->pendingq_count = 0;
+		break;
+	default:
+		break;
+	}
+	return status;
+}
+
+/**
+ * ap_sm_read(): Receive pending reply messages from an AP queue.
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_WAIT_NONE, AP_WAIT_AGAIN, or AP_WAIT_INTERRUPT
+ */
+static enum ap_wait ap_sm_read(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+
+	if (!aq->reply)
+		return AP_WAIT_NONE;
+	status = ap_sm_recv(aq);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		if (aq->queue_count > 0) {
+			aq->state = AP_STATE_WORKING;
+			return AP_WAIT_AGAIN;
+		}
+		aq->state = AP_STATE_IDLE;
+		return AP_WAIT_NONE;
+	case AP_RESPONSE_NO_PENDING_REPLY:
+		if (aq->queue_count > 0)
+			return AP_WAIT_INTERRUPT;
+		aq->state = AP_STATE_IDLE;
+		return AP_WAIT_NONE;
+	default:
+		aq->state = AP_STATE_BORKED;
+		return AP_WAIT_NONE;
+	}
+}
+
+/**
+ * ap_sm_suspend_read(): Receive pending reply messages from an AP queue
+ * without changing the device state in between. In suspend mode we don't
+ * allow sending new requests, therefore just fetch pending replies.
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_WAIT_NONE or AP_WAIT_AGAIN
+ */
+static enum ap_wait ap_sm_suspend_read(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+
+	if (!aq->reply)
+		return AP_WAIT_NONE;
+	status = ap_sm_recv(aq);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		if (aq->queue_count > 0)
+			return AP_WAIT_AGAIN;
+		/* fall through */
+	default:
+		return AP_WAIT_NONE;
+	}
+}
+
+/**
+ * ap_sm_write(): Send messages from the request queue to an AP queue.
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_WAIT_NONE, AP_WAIT_AGAIN, or AP_WAIT_INTERRUPT
+ */
+static enum ap_wait ap_sm_write(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+	struct ap_message *ap_msg;
+
+	if (aq->requestq_count <= 0)
+		return AP_WAIT_NONE;
+	/* Start the next request on the queue. */
+	ap_msg = list_entry(aq->requestq.next, struct ap_message, list);
+	status = __ap_send(aq->qid, ap_msg->psmid,
+			   ap_msg->message, ap_msg->length, ap_msg->special);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		aq->queue_count++;
+		if (aq->queue_count == 1)
+			mod_timer(&aq->timeout, jiffies + aq->request_timeout);
+		list_move_tail(&ap_msg->list, &aq->pendingq);
+		aq->requestq_count--;
+		aq->pendingq_count++;
+		if (aq->queue_count < aq->card->queue_depth) {
+			aq->state = AP_STATE_WORKING;
+			return AP_WAIT_AGAIN;
+		}
+		/* fall through */
+	case AP_RESPONSE_Q_FULL:
+		aq->state = AP_STATE_QUEUE_FULL;
+		return AP_WAIT_INTERRUPT;
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+		aq->state = AP_STATE_RESET_WAIT;
+		return AP_WAIT_TIMEOUT;
+	case AP_RESPONSE_MESSAGE_TOO_BIG:
+	case AP_RESPONSE_REQ_FAC_NOT_INST:
+		list_del_init(&ap_msg->list);
+		aq->requestq_count--;
+		ap_msg->rc = -EINVAL;
+		ap_msg->receive(aq, ap_msg, NULL);
+		return AP_WAIT_AGAIN;
+	default:
+		aq->state = AP_STATE_BORKED;
+		return AP_WAIT_NONE;
+	}
+}
+
+/**
+ * ap_sm_read_write(): Send and receive messages to/from an AP queue.
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_WAIT_NONE, AP_WAIT_AGAIN, or AP_WAIT_INTERRUPT
+ */
+static enum ap_wait ap_sm_read_write(struct ap_queue *aq)
+{
+	return min(ap_sm_read(aq), ap_sm_write(aq));
+}
+
+/**
+ * ap_sm_reset(): Reset an AP queue.
+ * @qid: The AP queue number
+ *
+ * Submit the Reset command to an AP queue.
+ */
+static enum ap_wait ap_sm_reset(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+
+	status = ap_rapq(aq->qid);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+		aq->state = AP_STATE_RESET_WAIT;
+		aq->interrupt = AP_INTR_DISABLED;
+		return AP_WAIT_TIMEOUT;
+	case AP_RESPONSE_BUSY:
+		return AP_WAIT_TIMEOUT;
+	case AP_RESPONSE_Q_NOT_AVAIL:
+	case AP_RESPONSE_DECONFIGURED:
+	case AP_RESPONSE_CHECKSTOPPED:
+	default:
+		aq->state = AP_STATE_BORKED;
+		return AP_WAIT_NONE;
+	}
+}
+
+/**
+ * ap_sm_reset_wait(): Test queue for completion of the reset operation
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_POLL_IMMEDIATELY, AP_POLL_AFTER_TIMEROUT or 0.
+ */
+static enum ap_wait ap_sm_reset_wait(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+	void *lsi_ptr;
+
+	if (aq->queue_count > 0 && aq->reply)
+		/* Try to read a completed message and get the status */
+		status = ap_sm_recv(aq);
+	else
+		/* Get the status with TAPQ */
+		status = ap_tapq(aq->qid, NULL);
+
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		lsi_ptr = ap_airq_ptr();
+		if (lsi_ptr && ap_queue_enable_interruption(aq, lsi_ptr) == 0)
+			aq->state = AP_STATE_SETIRQ_WAIT;
+		else
+			aq->state = (aq->queue_count > 0) ?
+				AP_STATE_WORKING : AP_STATE_IDLE;
+		return AP_WAIT_AGAIN;
+	case AP_RESPONSE_BUSY:
+	case AP_RESPONSE_RESET_IN_PROGRESS:
+		return AP_WAIT_TIMEOUT;
+	case AP_RESPONSE_Q_NOT_AVAIL:
+	case AP_RESPONSE_DECONFIGURED:
+	case AP_RESPONSE_CHECKSTOPPED:
+	default:
+		aq->state = AP_STATE_BORKED;
+		return AP_WAIT_NONE;
+	}
+}
+
+/**
+ * ap_sm_setirq_wait(): Test queue for completion of the irq enablement
+ * @aq: pointer to the AP queue
+ *
+ * Returns AP_POLL_IMMEDIATELY, AP_POLL_AFTER_TIMEROUT or 0.
+ */
+static enum ap_wait ap_sm_setirq_wait(struct ap_queue *aq)
+{
+	struct ap_queue_status status;
+
+	if (aq->queue_count > 0 && aq->reply)
+		/* Try to read a completed message and get the status */
+		status = ap_sm_recv(aq);
+	else
+		/* Get the status with TAPQ */
+		status = ap_tapq(aq->qid, NULL);
+
+	if (status.int_enabled == 1) {
+		/* Irqs are now enabled */
+		aq->interrupt = AP_INTR_ENABLED;
+		aq->state = (aq->queue_count > 0) ?
+			AP_STATE_WORKING : AP_STATE_IDLE;
+	}
+
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		if (aq->queue_count > 0)
+			return AP_WAIT_AGAIN;
+		/* fallthrough */
+	case AP_RESPONSE_NO_PENDING_REPLY:
+		return AP_WAIT_TIMEOUT;
+	default:
+		aq->state = AP_STATE_BORKED;
+		return AP_WAIT_NONE;
+	}
+}
+
+/*
+ * AP state machine jump table
+ */
+static ap_func_t *ap_jumptable[NR_AP_STATES][NR_AP_EVENTS] = {
+	[AP_STATE_RESET_START] = {
+		[AP_EVENT_POLL] = ap_sm_reset,
+		[AP_EVENT_TIMEOUT] = ap_sm_nop,
+	},
+	[AP_STATE_RESET_WAIT] = {
+		[AP_EVENT_POLL] = ap_sm_reset_wait,
+		[AP_EVENT_TIMEOUT] = ap_sm_nop,
+	},
+	[AP_STATE_SETIRQ_WAIT] = {
+		[AP_EVENT_POLL] = ap_sm_setirq_wait,
+		[AP_EVENT_TIMEOUT] = ap_sm_nop,
+	},
+	[AP_STATE_IDLE] = {
+		[AP_EVENT_POLL] = ap_sm_write,
+		[AP_EVENT_TIMEOUT] = ap_sm_nop,
+	},
+	[AP_STATE_WORKING] = {
+		[AP_EVENT_POLL] = ap_sm_read_write,
+		[AP_EVENT_TIMEOUT] = ap_sm_reset,
+	},
+	[AP_STATE_QUEUE_FULL] = {
+		[AP_EVENT_POLL] = ap_sm_read,
+		[AP_EVENT_TIMEOUT] = ap_sm_reset,
+	},
+	[AP_STATE_SUSPEND_WAIT] = {
+		[AP_EVENT_POLL] = ap_sm_suspend_read,
+		[AP_EVENT_TIMEOUT] = ap_sm_nop,
+	},
+	[AP_STATE_BORKED] = {
+		[AP_EVENT_POLL] = ap_sm_nop,
+		[AP_EVENT_TIMEOUT] = ap_sm_nop,
+	},
+};
+
+enum ap_wait ap_sm_event(struct ap_queue *aq, enum ap_event event)
+{
+	return ap_jumptable[aq->state][event](aq);
+}
+
+enum ap_wait ap_sm_event_loop(struct ap_queue *aq, enum ap_event event)
+{
+	enum ap_wait wait;
+
+	while ((wait = ap_sm_event(aq, event)) == AP_WAIT_AGAIN)
+		;
+	return wait;
+}
+
+/*
+ * Power management for queue devices
+ */
+void ap_queue_suspend(struct ap_device *ap_dev)
+{
+	struct ap_queue *aq = to_ap_queue(&ap_dev->device);
+
+	/* Poll on the device until all requests are finished. */
+	spin_lock_bh(&aq->lock);
+	aq->state = AP_STATE_SUSPEND_WAIT;
+	while (ap_sm_event(aq, AP_EVENT_POLL) != AP_WAIT_NONE)
+		;
+	aq->state = AP_STATE_BORKED;
+	spin_unlock_bh(&aq->lock);
+}
+EXPORT_SYMBOL(ap_queue_suspend);
+
+void ap_queue_resume(struct ap_device *ap_dev)
+{
+}
+EXPORT_SYMBOL(ap_queue_resume);
+
+/*
+ * AP queue related attributes.
+ */
+static ssize_t ap_request_count_show(struct device *dev,
+				     struct device_attribute *attr,
+				     char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	unsigned int req_cnt;
+
+	spin_lock_bh(&aq->lock);
+	req_cnt = aq->total_request_count;
+	spin_unlock_bh(&aq->lock);
+	return snprintf(buf, PAGE_SIZE, "%d\n", req_cnt);
+}
+
+static DEVICE_ATTR(request_count, 0444, ap_request_count_show, NULL);
+
+static ssize_t ap_requestq_count_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	unsigned int reqq_cnt = 0;
+
+	spin_lock_bh(&aq->lock);
+	reqq_cnt = aq->requestq_count;
+	spin_unlock_bh(&aq->lock);
+	return snprintf(buf, PAGE_SIZE, "%d\n", reqq_cnt);
+}
+
+static DEVICE_ATTR(requestq_count, 0444, ap_requestq_count_show, NULL);
+
+static ssize_t ap_pendingq_count_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	unsigned int penq_cnt = 0;
+
+	spin_lock_bh(&aq->lock);
+	penq_cnt = aq->pendingq_count;
+	spin_unlock_bh(&aq->lock);
+	return snprintf(buf, PAGE_SIZE, "%d\n", penq_cnt);
+}
+
+static DEVICE_ATTR(pendingq_count, 0444, ap_pendingq_count_show, NULL);
+
+static ssize_t ap_reset_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	int rc = 0;
+
+	spin_lock_bh(&aq->lock);
+	switch (aq->state) {
+	case AP_STATE_RESET_START:
+	case AP_STATE_RESET_WAIT:
+		rc = snprintf(buf, PAGE_SIZE, "Reset in progress.\n");
+		break;
+	case AP_STATE_WORKING:
+	case AP_STATE_QUEUE_FULL:
+		rc = snprintf(buf, PAGE_SIZE, "Reset Timer armed.\n");
+		break;
+	default:
+		rc = snprintf(buf, PAGE_SIZE, "No Reset Timer set.\n");
+	}
+	spin_unlock_bh(&aq->lock);
+	return rc;
+}
+
+static DEVICE_ATTR(reset, 0444, ap_reset_show, NULL);
+
+static ssize_t ap_interrupt_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct ap_queue *aq = to_ap_queue(dev);
+	int rc = 0;
+
+	spin_lock_bh(&aq->lock);
+	if (aq->state == AP_STATE_SETIRQ_WAIT)
+		rc = snprintf(buf, PAGE_SIZE, "Enable Interrupt pending.\n");
+	else if (aq->interrupt == AP_INTR_ENABLED)
+		rc = snprintf(buf, PAGE_SIZE, "Interrupts enabled.\n");
+	else
+		rc = snprintf(buf, PAGE_SIZE, "Interrupts disabled.\n");
+	spin_unlock_bh(&aq->lock);
+	return rc;
+}
+
+static DEVICE_ATTR(interrupt, 0444, ap_interrupt_show, NULL);
+
+static struct attribute *ap_queue_dev_attrs[] = {
+	&dev_attr_request_count.attr,
+	&dev_attr_requestq_count.attr,
+	&dev_attr_pendingq_count.attr,
+	&dev_attr_reset.attr,
+	&dev_attr_interrupt.attr,
+	NULL
+};
+
+static struct attribute_group ap_queue_dev_attr_group = {
+	.attrs = ap_queue_dev_attrs
+};
+
+static const struct attribute_group *ap_queue_dev_attr_groups[] = {
+	&ap_queue_dev_attr_group,
+	NULL
+};
+
+struct device_type ap_queue_type = {
+	.name = "ap_queue",
+	.groups = ap_queue_dev_attr_groups,
+};
+
+static void ap_queue_device_release(struct device *dev)
+{
+	kfree(to_ap_queue(dev));
+}
+
+struct ap_queue *ap_queue_create(ap_qid_t qid, int device_type)
+{
+	struct ap_queue *aq;
+
+	aq = kzalloc(sizeof(*aq), GFP_KERNEL);
+	if (!aq)
+		return NULL;
+	aq->ap_dev.device.release = ap_queue_device_release;
+	aq->ap_dev.device.type = &ap_queue_type;
+	aq->ap_dev.device_type = device_type;
+	/* CEX6 toleration: map to CEX5 */
+	if (device_type == AP_DEVICE_TYPE_CEX6)
+		aq->ap_dev.device_type = AP_DEVICE_TYPE_CEX5;
+	aq->qid = qid;
+	aq->state = AP_STATE_RESET_START;
+	aq->interrupt = AP_INTR_DISABLED;
+	spin_lock_init(&aq->lock);
+	INIT_LIST_HEAD(&aq->pendingq);
+	INIT_LIST_HEAD(&aq->requestq);
+	setup_timer(&aq->timeout, ap_request_timeout, (unsigned long) aq);
+
+	return aq;
+}
+
+void ap_queue_init_reply(struct ap_queue *aq, struct ap_message *reply)
+{
+	aq->reply = reply;
+
+	spin_lock_bh(&aq->lock);
+	ap_wait(ap_sm_event(aq, AP_EVENT_POLL));
+	spin_unlock_bh(&aq->lock);
+}
+EXPORT_SYMBOL(ap_queue_init_reply);
+
+/**
+ * ap_queue_message(): Queue a request to an AP device.
+ * @aq: The AP device to queue the message to
+ * @ap_msg: The message that is to be added
+ */
+void ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg)
+{
+	/* For asynchronous message handling a valid receive-callback
+	 * is required.
+	 */
+	BUG_ON(!ap_msg->receive);
+
+	spin_lock_bh(&aq->lock);
+	/* Queue the message. */
+	list_add_tail(&ap_msg->list, &aq->requestq);
+	aq->requestq_count++;
+	aq->total_request_count++;
+	atomic_inc(&aq->card->total_request_count);
+	/* Send/receive as many request from the queue as possible. */
+	ap_wait(ap_sm_event_loop(aq, AP_EVENT_POLL));
+	spin_unlock_bh(&aq->lock);
+}
+EXPORT_SYMBOL(ap_queue_message);
+
+/**
+ * ap_cancel_message(): Cancel a crypto request.
+ * @aq: The AP device that has the message queued
+ * @ap_msg: The message that is to be removed
+ *
+ * Cancel a crypto request. This is done by removing the request
+ * from the device pending or request queue. Note that the
+ * request stays on the AP queue. When it finishes the message
+ * reply will be discarded because the psmid can't be found.
+ */
+void ap_cancel_message(struct ap_queue *aq, struct ap_message *ap_msg)
+{
+	struct ap_message *tmp;
+
+	spin_lock_bh(&aq->lock);
+	if (!list_empty(&ap_msg->list)) {
+		list_for_each_entry(tmp, &aq->pendingq, list)
+			if (tmp->psmid == ap_msg->psmid) {
+				aq->pendingq_count--;
+				goto found;
+			}
+		aq->requestq_count--;
+found:
+		list_del_init(&ap_msg->list);
+	}
+	spin_unlock_bh(&aq->lock);
+}
+EXPORT_SYMBOL(ap_cancel_message);
+
+/**
+ * __ap_flush_queue(): Flush requests.
+ * @aq: Pointer to the AP queue
+ *
+ * Flush all requests from the request/pending queue of an AP device.
+ */
+static void __ap_flush_queue(struct ap_queue *aq)
+{
+	struct ap_message *ap_msg, *next;
+
+	list_for_each_entry_safe(ap_msg, next, &aq->pendingq, list) {
+		list_del_init(&ap_msg->list);
+		aq->pendingq_count--;
+		ap_msg->rc = -EAGAIN;
+		ap_msg->receive(aq, ap_msg, NULL);
+	}
+	list_for_each_entry_safe(ap_msg, next, &aq->requestq, list) {
+		list_del_init(&ap_msg->list);
+		aq->requestq_count--;
+		ap_msg->rc = -EAGAIN;
+		ap_msg->receive(aq, ap_msg, NULL);
+	}
+}
+
+void ap_flush_queue(struct ap_queue *aq)
+{
+	spin_lock_bh(&aq->lock);
+	__ap_flush_queue(aq);
+	spin_unlock_bh(&aq->lock);
+}
+EXPORT_SYMBOL(ap_flush_queue);
+
+void ap_queue_remove(struct ap_queue *aq)
+{
+	ap_flush_queue(aq);
+	del_timer_sync(&aq->timeout);
+}
+EXPORT_SYMBOL(ap_queue_remove);
diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c
index 5d3d04c040c2..854a6e58dfea 100644
--- a/drivers/s390/crypto/zcrypt_api.c
+++ b/drivers/s390/crypto/zcrypt_api.c
@@ -41,10 +41,14 @@
 #include <linux/debugfs.h>
 #include <asm/debug.h>
 
-#include "zcrypt_debug.h"
+#define CREATE_TRACE_POINTS
+#include <asm/trace/zcrypt.h>
+
 #include "zcrypt_api.h"
+#include "zcrypt_debug.h"
 
 #include "zcrypt_msgtype6.h"
+#include "zcrypt_msgtype50.h"
 
 /*
  * Module description.
@@ -54,76 +58,31 @@ MODULE_DESCRIPTION("Cryptographic Coprocessor interface, " \
 		   "Copyright IBM Corp. 2001, 2012");
 MODULE_LICENSE("GPL");
 
+/*
+ * zcrypt tracepoint functions
+ */
+EXPORT_TRACEPOINT_SYMBOL(s390_zcrypt_req);
+EXPORT_TRACEPOINT_SYMBOL(s390_zcrypt_rep);
+
 static int zcrypt_hwrng_seed = 1;
 module_param_named(hwrng_seed, zcrypt_hwrng_seed, int, S_IRUSR|S_IRGRP);
 MODULE_PARM_DESC(hwrng_seed, "Turn on/off hwrng auto seed, default is 1 (on).");
 
-static DEFINE_SPINLOCK(zcrypt_device_lock);
-static LIST_HEAD(zcrypt_device_list);
-static int zcrypt_device_count = 0;
+DEFINE_SPINLOCK(zcrypt_list_lock);
+LIST_HEAD(zcrypt_card_list);
+int zcrypt_device_count;
+
 static atomic_t zcrypt_open_count = ATOMIC_INIT(0);
 static atomic_t zcrypt_rescan_count = ATOMIC_INIT(0);
 
 atomic_t zcrypt_rescan_req = ATOMIC_INIT(0);
 EXPORT_SYMBOL(zcrypt_rescan_req);
 
-static int zcrypt_rng_device_add(void);
-static void zcrypt_rng_device_remove(void);
-
-static DEFINE_SPINLOCK(zcrypt_ops_list_lock);
 static LIST_HEAD(zcrypt_ops_list);
 
-static debug_info_t *zcrypt_dbf_common;
-static debug_info_t *zcrypt_dbf_devices;
-static struct dentry *debugfs_root;
-
-/*
- * Device attributes common for all crypto devices.
- */
-static ssize_t zcrypt_type_show(struct device *dev,
-				struct device_attribute *attr, char *buf)
-{
-	struct zcrypt_device *zdev = to_ap_dev(dev)->private;
-	return snprintf(buf, PAGE_SIZE, "%s\n", zdev->type_string);
-}
-
-static DEVICE_ATTR(type, 0444, zcrypt_type_show, NULL);
-
-static ssize_t zcrypt_online_show(struct device *dev,
-				  struct device_attribute *attr, char *buf)
-{
-	struct zcrypt_device *zdev = to_ap_dev(dev)->private;
-	return snprintf(buf, PAGE_SIZE, "%d\n", zdev->online);
-}
-
-static ssize_t zcrypt_online_store(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf, size_t count)
-{
-	struct zcrypt_device *zdev = to_ap_dev(dev)->private;
-	int online;
-
-	if (sscanf(buf, "%d\n", &online) != 1 || online < 0 || online > 1)
-		return -EINVAL;
-	zdev->online = online;
-	ZCRYPT_DBF_DEV(DBF_INFO, zdev, "dev%04xo%dman", zdev->ap_dev->qid,
-		       zdev->online);
-	if (!online)
-		ap_flush_queue(zdev->ap_dev);
-	return count;
-}
-
-static DEVICE_ATTR(online, 0644, zcrypt_online_show, zcrypt_online_store);
-
-static struct attribute * zcrypt_device_attrs[] = {
-	&dev_attr_type.attr,
-	&dev_attr_online.attr,
-	NULL,
-};
-
-static struct attribute_group zcrypt_device_attr_group = {
-	.attrs = zcrypt_device_attrs,
-};
+/* Zcrypt related debug feature stuff. */
+static struct dentry *zcrypt_dbf_root;
+debug_info_t *zcrypt_dbf_info;
 
 /**
  * Process a rescan of the transport layer.
@@ -136,242 +95,34 @@ static inline int zcrypt_process_rescan(void)
 		atomic_set(&zcrypt_rescan_req, 0);
 		atomic_inc(&zcrypt_rescan_count);
 		ap_bus_force_rescan();
-		ZCRYPT_DBF_COMMON(DBF_INFO, "rescan%07d",
-				  atomic_inc_return(&zcrypt_rescan_count));
+		ZCRYPT_DBF(DBF_INFO, "rescan count=%07d",
+			   atomic_inc_return(&zcrypt_rescan_count));
 		return 1;
 	}
 	return 0;
 }
 
-/**
- * __zcrypt_increase_preference(): Increase preference of a crypto device.
- * @zdev: Pointer the crypto device
- *
- * Move the device towards the head of the device list.
- * Need to be called while holding the zcrypt device list lock.
- * Note: cards with speed_rating of 0 are kept at the end of the list.
- */
-static void __zcrypt_increase_preference(struct zcrypt_device *zdev)
-{
-	struct zcrypt_device *tmp;
-	struct list_head *l;
-
-	if (zdev->speed_rating == 0)
-		return;
-	for (l = zdev->list.prev; l != &zcrypt_device_list; l = l->prev) {
-		tmp = list_entry(l, struct zcrypt_device, list);
-		if ((tmp->request_count + 1) * tmp->speed_rating <=
-		    (zdev->request_count + 1) * zdev->speed_rating &&
-		    tmp->speed_rating != 0)
-			break;
-	}
-	if (l == zdev->list.prev)
-		return;
-	/* Move zdev behind l */
-	list_move(&zdev->list, l);
-}
-
-/**
- * __zcrypt_decrease_preference(): Decrease preference of a crypto device.
- * @zdev: Pointer to a crypto device.
- *
- * Move the device towards the tail of the device list.
- * Need to be called while holding the zcrypt device list lock.
- * Note: cards with speed_rating of 0 are kept at the end of the list.
- */
-static void __zcrypt_decrease_preference(struct zcrypt_device *zdev)
-{
-	struct zcrypt_device *tmp;
-	struct list_head *l;
-
-	if (zdev->speed_rating == 0)
-		return;
-	for (l = zdev->list.next; l != &zcrypt_device_list; l = l->next) {
-		tmp = list_entry(l, struct zcrypt_device, list);
-		if ((tmp->request_count + 1) * tmp->speed_rating >
-		    (zdev->request_count + 1) * zdev->speed_rating ||
-		    tmp->speed_rating == 0)
-			break;
-	}
-	if (l == zdev->list.next)
-		return;
-	/* Move zdev before l */
-	list_move_tail(&zdev->list, l);
-}
-
-static void zcrypt_device_release(struct kref *kref)
-{
-	struct zcrypt_device *zdev =
-		container_of(kref, struct zcrypt_device, refcount);
-	zcrypt_device_free(zdev);
-}
-
-void zcrypt_device_get(struct zcrypt_device *zdev)
-{
-	kref_get(&zdev->refcount);
-}
-EXPORT_SYMBOL(zcrypt_device_get);
-
-int zcrypt_device_put(struct zcrypt_device *zdev)
-{
-	return kref_put(&zdev->refcount, zcrypt_device_release);
-}
-EXPORT_SYMBOL(zcrypt_device_put);
-
-struct zcrypt_device *zcrypt_device_alloc(size_t max_response_size)
-{
-	struct zcrypt_device *zdev;
-
-	zdev = kzalloc(sizeof(struct zcrypt_device), GFP_KERNEL);
-	if (!zdev)
-		return NULL;
-	zdev->reply.message = kmalloc(max_response_size, GFP_KERNEL);
-	if (!zdev->reply.message)
-		goto out_free;
-	zdev->reply.length = max_response_size;
-	spin_lock_init(&zdev->lock);
-	INIT_LIST_HEAD(&zdev->list);
-	zdev->dbf_area = zcrypt_dbf_devices;
-	return zdev;
-
-out_free:
-	kfree(zdev);
-	return NULL;
-}
-EXPORT_SYMBOL(zcrypt_device_alloc);
-
-void zcrypt_device_free(struct zcrypt_device *zdev)
-{
-	kfree(zdev->reply.message);
-	kfree(zdev);
-}
-EXPORT_SYMBOL(zcrypt_device_free);
-
-/**
- * zcrypt_device_register() - Register a crypto device.
- * @zdev: Pointer to a crypto device
- *
- * Register a crypto device. Returns 0 if successful.
- */
-int zcrypt_device_register(struct zcrypt_device *zdev)
-{
-	int rc;
-
-	if (!zdev->ops)
-		return -ENODEV;
-	rc = sysfs_create_group(&zdev->ap_dev->device.kobj,
-				&zcrypt_device_attr_group);
-	if (rc)
-		goto out;
-	get_device(&zdev->ap_dev->device);
-	kref_init(&zdev->refcount);
-	spin_lock_bh(&zcrypt_device_lock);
-	zdev->online = 1;	/* New devices are online by default. */
-	ZCRYPT_DBF_DEV(DBF_INFO, zdev, "dev%04xo%dreg", zdev->ap_dev->qid,
-		       zdev->online);
-	list_add_tail(&zdev->list, &zcrypt_device_list);
-	__zcrypt_increase_preference(zdev);
-	zcrypt_device_count++;
-	spin_unlock_bh(&zcrypt_device_lock);
-	if (zdev->ops->rng) {
-		rc = zcrypt_rng_device_add();
-		if (rc)
-			goto out_unregister;
-	}
-	return 0;
-
-out_unregister:
-	spin_lock_bh(&zcrypt_device_lock);
-	zcrypt_device_count--;
-	list_del_init(&zdev->list);
-	spin_unlock_bh(&zcrypt_device_lock);
-	sysfs_remove_group(&zdev->ap_dev->device.kobj,
-			   &zcrypt_device_attr_group);
-	put_device(&zdev->ap_dev->device);
-	zcrypt_device_put(zdev);
-out:
-	return rc;
-}
-EXPORT_SYMBOL(zcrypt_device_register);
-
-/**
- * zcrypt_device_unregister(): Unregister a crypto device.
- * @zdev: Pointer to crypto device
- *
- * Unregister a crypto device.
- */
-void zcrypt_device_unregister(struct zcrypt_device *zdev)
-{
-	if (zdev->ops->rng)
-		zcrypt_rng_device_remove();
-	spin_lock_bh(&zcrypt_device_lock);
-	zcrypt_device_count--;
-	list_del_init(&zdev->list);
-	spin_unlock_bh(&zcrypt_device_lock);
-	sysfs_remove_group(&zdev->ap_dev->device.kobj,
-			   &zcrypt_device_attr_group);
-	put_device(&zdev->ap_dev->device);
-	zcrypt_device_put(zdev);
-}
-EXPORT_SYMBOL(zcrypt_device_unregister);
-
 void zcrypt_msgtype_register(struct zcrypt_ops *zops)
 {
-	spin_lock_bh(&zcrypt_ops_list_lock);
 	list_add_tail(&zops->list, &zcrypt_ops_list);
-	spin_unlock_bh(&zcrypt_ops_list_lock);
 }
-EXPORT_SYMBOL(zcrypt_msgtype_register);
 
 void zcrypt_msgtype_unregister(struct zcrypt_ops *zops)
 {
-	spin_lock_bh(&zcrypt_ops_list_lock);
 	list_del_init(&zops->list);
-	spin_unlock_bh(&zcrypt_ops_list_lock);
 }
-EXPORT_SYMBOL(zcrypt_msgtype_unregister);
 
-static inline
-struct zcrypt_ops *__ops_lookup(unsigned char *name, int variant)
+struct zcrypt_ops *zcrypt_msgtype(unsigned char *name, int variant)
 {
 	struct zcrypt_ops *zops;
-	int found = 0;
 
-	spin_lock_bh(&zcrypt_ops_list_lock);
-	list_for_each_entry(zops, &zcrypt_ops_list, list) {
+	list_for_each_entry(zops, &zcrypt_ops_list, list)
 		if ((zops->variant == variant) &&
-		    (!strncmp(zops->name, name, sizeof(zops->name)))) {
-			found = 1;
-			break;
-		}
-	}
-	if (!found || !try_module_get(zops->owner))
-		zops = NULL;
-
-	spin_unlock_bh(&zcrypt_ops_list_lock);
-
-	return zops;
-}
-
-struct zcrypt_ops *zcrypt_msgtype_request(unsigned char *name, int variant)
-{
-	struct zcrypt_ops *zops = NULL;
-
-	zops = __ops_lookup(name, variant);
-	if (!zops) {
-		request_module("%s", name);
-		zops = __ops_lookup(name, variant);
-	}
-	return zops;
-}
-EXPORT_SYMBOL(zcrypt_msgtype_request);
-
-void zcrypt_msgtype_release(struct zcrypt_ops *zops)
-{
-	if (zops)
-		module_put(zops->owner);
+		    (!strncmp(zops->name, name, sizeof(zops->name))))
+			return zops;
+	return NULL;
 }
-EXPORT_SYMBOL(zcrypt_msgtype_release);
+EXPORT_SYMBOL(zcrypt_msgtype);
 
 /**
  * zcrypt_read (): Not supported beyond zcrypt 1.3.1.
@@ -417,16 +168,80 @@ static int zcrypt_release(struct inode *inode, struct file *filp)
 	return 0;
 }
 
+static inline struct zcrypt_queue *zcrypt_pick_queue(struct zcrypt_card *zc,
+						     struct zcrypt_queue *zq,
+						     unsigned int weight)
+{
+	if (!zq || !try_module_get(zq->queue->ap_dev.drv->driver.owner))
+		return NULL;
+	zcrypt_queue_get(zq);
+	get_device(&zq->queue->ap_dev.device);
+	atomic_add(weight, &zc->load);
+	atomic_add(weight, &zq->load);
+	zq->request_count++;
+	return zq;
+}
+
+static inline void zcrypt_drop_queue(struct zcrypt_card *zc,
+				     struct zcrypt_queue *zq,
+				     unsigned int weight)
+{
+	struct module *mod = zq->queue->ap_dev.drv->driver.owner;
+
+	zq->request_count--;
+	atomic_sub(weight, &zc->load);
+	atomic_sub(weight, &zq->load);
+	put_device(&zq->queue->ap_dev.device);
+	zcrypt_queue_put(zq);
+	module_put(mod);
+}
+
+static inline bool zcrypt_card_compare(struct zcrypt_card *zc,
+				       struct zcrypt_card *pref_zc,
+				       unsigned weight, unsigned pref_weight)
+{
+	if (!pref_zc)
+		return 0;
+	weight += atomic_read(&zc->load);
+	pref_weight += atomic_read(&pref_zc->load);
+	if (weight == pref_weight)
+		return atomic_read(&zc->card->total_request_count) >
+			atomic_read(&pref_zc->card->total_request_count);
+	return weight > pref_weight;
+}
+
+static inline bool zcrypt_queue_compare(struct zcrypt_queue *zq,
+					struct zcrypt_queue *pref_zq,
+					unsigned weight, unsigned pref_weight)
+{
+	if (!pref_zq)
+		return 0;
+	weight += atomic_read(&zq->load);
+	pref_weight += atomic_read(&pref_zq->load);
+	if (weight == pref_weight)
+		return &zq->queue->total_request_count >
+			&pref_zq->queue->total_request_count;
+	return weight > pref_weight;
+}
+
 /*
  * zcrypt ioctls.
  */
 static long zcrypt_rsa_modexpo(struct ica_rsa_modexpo *mex)
 {
-	struct zcrypt_device *zdev;
-	int rc;
+	struct zcrypt_card *zc, *pref_zc;
+	struct zcrypt_queue *zq, *pref_zq;
+	unsigned int weight, pref_weight;
+	unsigned int func_code;
+	int qid = 0, rc = -ENODEV;
+
+	trace_s390_zcrypt_req(mex, TP_ICARSAMODEXPO);
+
+	if (mex->outputdatalength < mex->inputdatalength) {
+		rc = -EINVAL;
+		goto out;
+	}
 
-	if (mex->outputdatalength < mex->inputdatalength)
-		return -EINVAL;
 	/*
 	 * As long as outputdatalength is big enough, we can set the
 	 * outputdatalength equal to the inputdatalength, since that is the
@@ -434,44 +249,73 @@ static long zcrypt_rsa_modexpo(struct ica_rsa_modexpo *mex)
 	 */
 	mex->outputdatalength = mex->inputdatalength;
 
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list) {
-		if (!zdev->online ||
-		    !zdev->ops->rsa_modexpo ||
-		    zdev->min_mod_size > mex->inputdatalength ||
-		    zdev->max_mod_size < mex->inputdatalength)
+	rc = get_rsa_modex_fc(mex, &func_code);
+	if (rc)
+		goto out;
+
+	pref_zc = NULL;
+	pref_zq = NULL;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		/* Check for online accelarator and CCA cards */
+		if (!zc->online || !(zc->card->functions & 0x18000000))
+			continue;
+		/* Check for size limits */
+		if (zc->min_mod_size > mex->inputdatalength ||
+		    zc->max_mod_size < mex->inputdatalength)
+			continue;
+		/* get weight index of the card device	*/
+		weight = zc->speed_rating[func_code];
+		if (zcrypt_card_compare(zc, pref_zc, weight, pref_weight))
 			continue;
-		zcrypt_device_get(zdev);
-		get_device(&zdev->ap_dev->device);
-		zdev->request_count++;
-		__zcrypt_decrease_preference(zdev);
-		if (try_module_get(zdev->ap_dev->drv->driver.owner)) {
-			spin_unlock_bh(&zcrypt_device_lock);
-			rc = zdev->ops->rsa_modexpo(zdev, mex);
-			spin_lock_bh(&zcrypt_device_lock);
-			module_put(zdev->ap_dev->drv->driver.owner);
+		for_each_zcrypt_queue(zq, zc) {
+			/* check if device is online and eligible */
+			if (!zq->online || !zq->ops->rsa_modexpo)
+				continue;
+			if (zcrypt_queue_compare(zq, pref_zq,
+						 weight, pref_weight))
+				continue;
+			pref_zc = zc;
+			pref_zq = zq;
+			pref_weight = weight;
 		}
-		else
-			rc = -EAGAIN;
-		zdev->request_count--;
-		__zcrypt_increase_preference(zdev);
-		put_device(&zdev->ap_dev->device);
-		zcrypt_device_put(zdev);
-		spin_unlock_bh(&zcrypt_device_lock);
-		return rc;
 	}
-	spin_unlock_bh(&zcrypt_device_lock);
-	return -ENODEV;
+	pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, weight);
+	spin_unlock(&zcrypt_list_lock);
+
+	if (!pref_zq) {
+		rc = -ENODEV;
+		goto out;
+	}
+
+	qid = pref_zq->queue->qid;
+	rc = pref_zq->ops->rsa_modexpo(pref_zq, mex);
+
+	spin_lock(&zcrypt_list_lock);
+	zcrypt_drop_queue(pref_zc, pref_zq, weight);
+	spin_unlock(&zcrypt_list_lock);
+
+out:
+	trace_s390_zcrypt_rep(mex, func_code, rc,
+			      AP_QID_CARD(qid), AP_QID_QUEUE(qid));
+	return rc;
 }
 
 static long zcrypt_rsa_crt(struct ica_rsa_modexpo_crt *crt)
 {
-	struct zcrypt_device *zdev;
-	unsigned long long z1, z2, z3;
-	int rc, copied;
+	struct zcrypt_card *zc, *pref_zc;
+	struct zcrypt_queue *zq, *pref_zq;
+	unsigned int weight, pref_weight;
+	unsigned int func_code;
+	int qid = 0, rc = -ENODEV;
+
+	trace_s390_zcrypt_req(crt, TP_ICARSACRT);
+
+	if (crt->outputdatalength < crt->inputdatalength) {
+		rc = -EINVAL;
+		goto out;
+	}
 
-	if (crt->outputdatalength < crt->inputdatalength)
-		return -EINVAL;
 	/*
 	 * As long as outputdatalength is big enough, we can set the
 	 * outputdatalength equal to the inputdatalength, since that is the
@@ -479,308 +323,445 @@ static long zcrypt_rsa_crt(struct ica_rsa_modexpo_crt *crt)
 	 */
 	crt->outputdatalength = crt->inputdatalength;
 
-	copied = 0;
- restart:
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list) {
-		if (!zdev->online ||
-		    !zdev->ops->rsa_modexpo_crt ||
-		    zdev->min_mod_size > crt->inputdatalength ||
-		    zdev->max_mod_size < crt->inputdatalength)
+	rc = get_rsa_crt_fc(crt, &func_code);
+	if (rc)
+		goto out;
+
+	pref_zc = NULL;
+	pref_zq = NULL;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		/* Check for online accelarator and CCA cards */
+		if (!zc->online || !(zc->card->functions & 0x18000000))
+			continue;
+		/* Check for size limits */
+		if (zc->min_mod_size > crt->inputdatalength ||
+		    zc->max_mod_size < crt->inputdatalength)
 			continue;
-		if (zdev->short_crt && crt->inputdatalength > 240) {
-			/*
-			 * Check inputdata for leading zeros for cards
-			 * that can't handle np_prime, bp_key, or
-			 * u_mult_inv > 128 bytes.
-			 */
-			if (copied == 0) {
-				unsigned int len;
-				spin_unlock_bh(&zcrypt_device_lock);
-				/* len is max 256 / 2 - 120 = 8
-				 * For bigger device just assume len of leading
-				 * 0s is 8 as stated in the requirements for
-				 * ica_rsa_modexpo_crt struct in zcrypt.h.
-				 */
-				if (crt->inputdatalength <= 256)
-					len = crt->inputdatalength / 2 - 120;
-				else
-					len = 8;
-				if (len > sizeof(z1))
-					return -EFAULT;
-				z1 = z2 = z3 = 0;
-				if (copy_from_user(&z1, crt->np_prime, len) ||
-				    copy_from_user(&z2, crt->bp_key, len) ||
-				    copy_from_user(&z3, crt->u_mult_inv, len))
-					return -EFAULT;
-				z1 = z2 = z3 = 0;
-				copied = 1;
-				/*
-				 * We have to restart device lookup -
-				 * the device list may have changed by now.
-				 */
-				goto restart;
-			}
-			if (z1 != 0ULL || z2 != 0ULL || z3 != 0ULL)
-				/* The device can't handle this request. */
+		/* get weight index of the card device	*/
+		weight = zc->speed_rating[func_code];
+		if (zcrypt_card_compare(zc, pref_zc, weight, pref_weight))
+			continue;
+		for_each_zcrypt_queue(zq, zc) {
+			/* check if device is online and eligible */
+			if (!zq->online || !zq->ops->rsa_modexpo_crt)
 				continue;
+			if (zcrypt_queue_compare(zq, pref_zq,
+						 weight, pref_weight))
+				continue;
+			pref_zc = zc;
+			pref_zq = zq;
+			pref_weight = weight;
 		}
-		zcrypt_device_get(zdev);
-		get_device(&zdev->ap_dev->device);
-		zdev->request_count++;
-		__zcrypt_decrease_preference(zdev);
-		if (try_module_get(zdev->ap_dev->drv->driver.owner)) {
-			spin_unlock_bh(&zcrypt_device_lock);
-			rc = zdev->ops->rsa_modexpo_crt(zdev, crt);
-			spin_lock_bh(&zcrypt_device_lock);
-			module_put(zdev->ap_dev->drv->driver.owner);
-		}
-		else
-			rc = -EAGAIN;
-		zdev->request_count--;
-		__zcrypt_increase_preference(zdev);
-		put_device(&zdev->ap_dev->device);
-		zcrypt_device_put(zdev);
-		spin_unlock_bh(&zcrypt_device_lock);
-		return rc;
 	}
-	spin_unlock_bh(&zcrypt_device_lock);
-	return -ENODEV;
+	pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, weight);
+	spin_unlock(&zcrypt_list_lock);
+
+	if (!pref_zq) {
+		rc = -ENODEV;
+		goto out;
+	}
+
+	qid = pref_zq->queue->qid;
+	rc = pref_zq->ops->rsa_modexpo_crt(pref_zq, crt);
+
+	spin_lock(&zcrypt_list_lock);
+	zcrypt_drop_queue(pref_zc, pref_zq, weight);
+	spin_unlock(&zcrypt_list_lock);
+
+out:
+	trace_s390_zcrypt_rep(crt, func_code, rc,
+			      AP_QID_CARD(qid), AP_QID_QUEUE(qid));
+	return rc;
 }
 
 static long zcrypt_send_cprb(struct ica_xcRB *xcRB)
 {
-	struct zcrypt_device *zdev;
-	int rc;
+	struct zcrypt_card *zc, *pref_zc;
+	struct zcrypt_queue *zq, *pref_zq;
+	struct ap_message ap_msg;
+	unsigned int weight, pref_weight;
+	unsigned int func_code;
+	unsigned short *domain;
+	int qid = 0, rc = -ENODEV;
+
+	trace_s390_zcrypt_req(xcRB, TB_ZSECSENDCPRB);
 
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list) {
-		if (!zdev->online || !zdev->ops->send_cprb ||
-		   (zdev->ops->variant == MSGTYPE06_VARIANT_EP11) ||
-		   (xcRB->user_defined != AUTOSELECT &&
-		    AP_QID_DEVICE(zdev->ap_dev->qid) != xcRB->user_defined))
+	rc = get_cprb_fc(xcRB, &ap_msg, &func_code, &domain);
+	if (rc)
+		goto out;
+
+	pref_zc = NULL;
+	pref_zq = NULL;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		/* Check for online CCA cards */
+		if (!zc->online || !(zc->card->functions & 0x10000000))
+			continue;
+		/* Check for user selected CCA card */
+		if (xcRB->user_defined != AUTOSELECT &&
+		    xcRB->user_defined != zc->card->id)
 			continue;
-		zcrypt_device_get(zdev);
-		get_device(&zdev->ap_dev->device);
-		zdev->request_count++;
-		__zcrypt_decrease_preference(zdev);
-		if (try_module_get(zdev->ap_dev->drv->driver.owner)) {
-			spin_unlock_bh(&zcrypt_device_lock);
-			rc = zdev->ops->send_cprb(zdev, xcRB);
-			spin_lock_bh(&zcrypt_device_lock);
-			module_put(zdev->ap_dev->drv->driver.owner);
+		/* get weight index of the card device	*/
+		weight = speed_idx_cca(func_code) * zc->speed_rating[SECKEY];
+		if (zcrypt_card_compare(zc, pref_zc, weight, pref_weight))
+			continue;
+		for_each_zcrypt_queue(zq, zc) {
+			/* check if device is online and eligible */
+			if (!zq->online ||
+			    !zq->ops->send_cprb ||
+			    ((*domain != (unsigned short) AUTOSELECT) &&
+			     (*domain != AP_QID_QUEUE(zq->queue->qid))))
+				continue;
+			if (zcrypt_queue_compare(zq, pref_zq,
+						 weight, pref_weight))
+				continue;
+			pref_zc = zc;
+			pref_zq = zq;
+			pref_weight = weight;
 		}
-		else
-			rc = -EAGAIN;
-		zdev->request_count--;
-		__zcrypt_increase_preference(zdev);
-		put_device(&zdev->ap_dev->device);
-		zcrypt_device_put(zdev);
-		spin_unlock_bh(&zcrypt_device_lock);
-		return rc;
 	}
-	spin_unlock_bh(&zcrypt_device_lock);
-	return -ENODEV;
-}
+	pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, weight);
+	spin_unlock(&zcrypt_list_lock);
 
-struct ep11_target_dev_list {
-	unsigned short		targets_num;
-	struct ep11_target_dev	*targets;
-};
+	if (!pref_zq) {
+		rc = -ENODEV;
+		goto out;
+	}
+
+	/* in case of auto select, provide the correct domain */
+	qid = pref_zq->queue->qid;
+	if (*domain == (unsigned short) AUTOSELECT)
+		*domain = AP_QID_QUEUE(qid);
 
-static bool is_desired_ep11dev(unsigned int dev_qid,
-			       struct ep11_target_dev_list dev_list)
+	rc = pref_zq->ops->send_cprb(pref_zq, xcRB, &ap_msg);
+
+	spin_lock(&zcrypt_list_lock);
+	zcrypt_drop_queue(pref_zc, pref_zq, weight);
+	spin_unlock(&zcrypt_list_lock);
+
+out:
+	trace_s390_zcrypt_rep(xcRB, func_code, rc,
+			      AP_QID_CARD(qid), AP_QID_QUEUE(qid));
+	return rc;
+}
+
+static bool is_desired_ep11_card(unsigned int dev_id,
+				 unsigned short target_num,
+				 struct ep11_target_dev *targets)
 {
-	int n;
+	while (target_num-- > 0) {
+		if (dev_id == targets->ap_id)
+			return true;
+		targets++;
+	}
+	return false;
+}
 
-	for (n = 0; n < dev_list.targets_num; n++, dev_list.targets++) {
-		if ((AP_QID_DEVICE(dev_qid) == dev_list.targets->ap_id) &&
-		    (AP_QID_QUEUE(dev_qid) == dev_list.targets->dom_id)) {
+static bool is_desired_ep11_queue(unsigned int dev_qid,
+				  unsigned short target_num,
+				  struct ep11_target_dev *targets)
+{
+	while (target_num-- > 0) {
+		if (AP_MKQID(targets->ap_id, targets->dom_id) == dev_qid)
 			return true;
-		}
+		targets++;
 	}
 	return false;
 }
 
 static long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb)
 {
-	struct zcrypt_device *zdev;
-	bool autoselect = false;
-	int rc;
-	struct ep11_target_dev_list ep11_dev_list = {
-		.targets_num	=  0x00,
-		.targets	=  NULL,
-	};
+	struct zcrypt_card *zc, *pref_zc;
+	struct zcrypt_queue *zq, *pref_zq;
+	struct ep11_target_dev *targets;
+	unsigned short target_num;
+	unsigned int weight, pref_weight;
+	unsigned int func_code;
+	struct ap_message ap_msg;
+	int qid = 0, rc = -ENODEV;
+
+	trace_s390_zcrypt_req(xcrb, TP_ZSENDEP11CPRB);
 
-	ep11_dev_list.targets_num = (unsigned short) xcrb->targets_num;
+	target_num = (unsigned short) xcrb->targets_num;
 
 	/* empty list indicates autoselect (all available targets) */
-	if (ep11_dev_list.targets_num == 0)
-		autoselect = true;
-	else {
-		ep11_dev_list.targets = kcalloc((unsigned short)
-						xcrb->targets_num,
-						sizeof(struct ep11_target_dev),
-						GFP_KERNEL);
-		if (!ep11_dev_list.targets)
-			return -ENOMEM;
+	targets = NULL;
+	if (target_num != 0) {
+		struct ep11_target_dev __user *uptr;
 
-		if (copy_from_user(ep11_dev_list.targets,
-				   (struct ep11_target_dev __force __user *)
-				   xcrb->targets, xcrb->targets_num *
-				   sizeof(struct ep11_target_dev)))
-			return -EFAULT;
+		targets = kcalloc(target_num, sizeof(*targets), GFP_KERNEL);
+		if (!targets) {
+			rc = -ENOMEM;
+			goto out;
+		}
+
+		uptr = (struct ep11_target_dev __force __user *) xcrb->targets;
+		if (copy_from_user(targets, uptr,
+				   target_num * sizeof(*targets))) {
+			rc = -EFAULT;
+			goto out;
+		}
 	}
 
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list) {
-		/* check if device is eligible */
-		if (!zdev->online ||
-		    zdev->ops->variant != MSGTYPE06_VARIANT_EP11)
-			continue;
+	rc = get_ep11cprb_fc(xcrb, &ap_msg, &func_code);
+	if (rc)
+		goto out_free;
 
-		/* check if device is selected as valid target */
-		if (!is_desired_ep11dev(zdev->ap_dev->qid, ep11_dev_list) &&
-		    !autoselect)
+	pref_zc = NULL;
+	pref_zq = NULL;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		/* Check for online EP11 cards */
+		if (!zc->online || !(zc->card->functions & 0x04000000))
+			continue;
+		/* Check for user selected EP11 card */
+		if (targets &&
+		    !is_desired_ep11_card(zc->card->id, target_num, targets))
 			continue;
+		/* get weight index of the card device	*/
+		weight = speed_idx_ep11(func_code) * zc->speed_rating[SECKEY];
+		if (zcrypt_card_compare(zc, pref_zc, weight, pref_weight))
+			continue;
+		for_each_zcrypt_queue(zq, zc) {
+			/* check if device is online and eligible */
+			if (!zq->online ||
+			    !zq->ops->send_ep11_cprb ||
+			    (targets &&
+			     !is_desired_ep11_queue(zq->queue->qid,
+						    target_num, targets)))
+				continue;
+			if (zcrypt_queue_compare(zq, pref_zq,
+						 weight, pref_weight))
+				continue;
+			pref_zc = zc;
+			pref_zq = zq;
+			pref_weight = weight;
+		}
+	}
+	pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, weight);
+	spin_unlock(&zcrypt_list_lock);
 
-		zcrypt_device_get(zdev);
-		get_device(&zdev->ap_dev->device);
-		zdev->request_count++;
-		__zcrypt_decrease_preference(zdev);
-		if (try_module_get(zdev->ap_dev->drv->driver.owner)) {
-			spin_unlock_bh(&zcrypt_device_lock);
-			rc = zdev->ops->send_ep11_cprb(zdev, xcrb);
-			spin_lock_bh(&zcrypt_device_lock);
-			module_put(zdev->ap_dev->drv->driver.owner);
-		} else {
-			rc = -EAGAIN;
-		  }
-		zdev->request_count--;
-		__zcrypt_increase_preference(zdev);
-		put_device(&zdev->ap_dev->device);
-		zcrypt_device_put(zdev);
-		spin_unlock_bh(&zcrypt_device_lock);
-		return rc;
+	if (!pref_zq) {
+		rc = -ENODEV;
+		goto out_free;
 	}
-	spin_unlock_bh(&zcrypt_device_lock);
-	return -ENODEV;
+
+	qid = pref_zq->queue->qid;
+	rc = pref_zq->ops->send_ep11_cprb(pref_zq, xcrb, &ap_msg);
+
+	spin_lock(&zcrypt_list_lock);
+	zcrypt_drop_queue(pref_zc, pref_zq, weight);
+	spin_unlock(&zcrypt_list_lock);
+
+out_free:
+	kfree(targets);
+out:
+	trace_s390_zcrypt_rep(xcrb, func_code, rc,
+			      AP_QID_CARD(qid), AP_QID_QUEUE(qid));
+	return rc;
 }
 
 static long zcrypt_rng(char *buffer)
 {
-	struct zcrypt_device *zdev;
-	int rc;
+	struct zcrypt_card *zc, *pref_zc;
+	struct zcrypt_queue *zq, *pref_zq;
+	unsigned int weight, pref_weight;
+	unsigned int func_code;
+	struct ap_message ap_msg;
+	unsigned int domain;
+	int qid = 0, rc = -ENODEV;
+
+	trace_s390_zcrypt_req(buffer, TP_HWRNGCPRB);
 
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list) {
-		if (!zdev->online || !zdev->ops->rng)
+	rc = get_rng_fc(&ap_msg, &func_code, &domain);
+	if (rc)
+		goto out;
+
+	pref_zc = NULL;
+	pref_zq = NULL;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		/* Check for online CCA cards */
+		if (!zc->online || !(zc->card->functions & 0x10000000))
 			continue;
-		zcrypt_device_get(zdev);
-		get_device(&zdev->ap_dev->device);
-		zdev->request_count++;
-		__zcrypt_decrease_preference(zdev);
-		if (try_module_get(zdev->ap_dev->drv->driver.owner)) {
-			spin_unlock_bh(&zcrypt_device_lock);
-			rc = zdev->ops->rng(zdev, buffer);
-			spin_lock_bh(&zcrypt_device_lock);
-			module_put(zdev->ap_dev->drv->driver.owner);
-		} else
-			rc = -EAGAIN;
-		zdev->request_count--;
-		__zcrypt_increase_preference(zdev);
-		put_device(&zdev->ap_dev->device);
-		zcrypt_device_put(zdev);
-		spin_unlock_bh(&zcrypt_device_lock);
-		return rc;
+		/* get weight index of the card device	*/
+		weight = zc->speed_rating[func_code];
+		if (zcrypt_card_compare(zc, pref_zc, weight, pref_weight))
+			continue;
+		for_each_zcrypt_queue(zq, zc) {
+			/* check if device is online and eligible */
+			if (!zq->online || !zq->ops->rng)
+				continue;
+			if (zcrypt_queue_compare(zq, pref_zq,
+						 weight, pref_weight))
+				continue;
+			pref_zc = zc;
+			pref_zq = zq;
+			pref_weight = weight;
+		}
+	}
+	pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, weight);
+	spin_unlock(&zcrypt_list_lock);
+
+	if (!pref_zq)
+		return -ENODEV;
+
+	qid = pref_zq->queue->qid;
+	rc = pref_zq->ops->rng(pref_zq, buffer, &ap_msg);
+
+	spin_lock(&zcrypt_list_lock);
+	zcrypt_drop_queue(pref_zc, pref_zq, weight);
+	spin_unlock(&zcrypt_list_lock);
+
+out:
+	trace_s390_zcrypt_rep(buffer, func_code, rc,
+			      AP_QID_CARD(qid), AP_QID_QUEUE(qid));
+	return rc;
+}
+
+static void zcrypt_device_status_mask(struct zcrypt_device_matrix *matrix)
+{
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
+	struct zcrypt_device_status *stat;
+
+	memset(matrix, 0, sizeof(*matrix));
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			stat = matrix->device;
+			stat += AP_QID_CARD(zq->queue->qid) * MAX_ZDEV_DOMAINS;
+			stat += AP_QID_QUEUE(zq->queue->qid);
+			stat->hwtype = zc->card->ap_dev.device_type;
+			stat->functions = zc->card->functions >> 26;
+			stat->qid = zq->queue->qid;
+			stat->online = zq->online ? 0x01 : 0x00;
+		}
 	}
-	spin_unlock_bh(&zcrypt_device_lock);
-	return -ENODEV;
+	spin_unlock(&zcrypt_list_lock);
 }
+EXPORT_SYMBOL(zcrypt_device_status_mask);
 
 static void zcrypt_status_mask(char status[AP_DEVICES])
 {
-	struct zcrypt_device *zdev;
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
 
 	memset(status, 0, sizeof(char) * AP_DEVICES);
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list)
-		status[AP_QID_DEVICE(zdev->ap_dev->qid)] =
-			zdev->online ? zdev->user_space_type : 0x0d;
-	spin_unlock_bh(&zcrypt_device_lock);
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index)
+				continue;
+			status[AP_QID_CARD(zq->queue->qid)] =
+				zc->online ? zc->user_space_type : 0x0d;
+		}
+	}
+	spin_unlock(&zcrypt_list_lock);
 }
 
 static void zcrypt_qdepth_mask(char qdepth[AP_DEVICES])
 {
-	struct zcrypt_device *zdev;
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
 
 	memset(qdepth, 0, sizeof(char)	* AP_DEVICES);
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list) {
-		spin_lock(&zdev->ap_dev->lock);
-		qdepth[AP_QID_DEVICE(zdev->ap_dev->qid)] =
-			zdev->ap_dev->pendingq_count +
-			zdev->ap_dev->requestq_count;
-		spin_unlock(&zdev->ap_dev->lock);
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index)
+				continue;
+			spin_lock(&zq->queue->lock);
+			qdepth[AP_QID_CARD(zq->queue->qid)] =
+				zq->queue->pendingq_count +
+				zq->queue->requestq_count;
+			spin_unlock(&zq->queue->lock);
+		}
 	}
-	spin_unlock_bh(&zcrypt_device_lock);
+	spin_unlock(&zcrypt_list_lock);
 }
 
 static void zcrypt_perdev_reqcnt(int reqcnt[AP_DEVICES])
 {
-	struct zcrypt_device *zdev;
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
 
 	memset(reqcnt, 0, sizeof(int) * AP_DEVICES);
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list) {
-		spin_lock(&zdev->ap_dev->lock);
-		reqcnt[AP_QID_DEVICE(zdev->ap_dev->qid)] =
-			zdev->ap_dev->total_request_count;
-		spin_unlock(&zdev->ap_dev->lock);
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index)
+				continue;
+			spin_lock(&zq->queue->lock);
+			reqcnt[AP_QID_CARD(zq->queue->qid)] =
+				zq->queue->total_request_count;
+			spin_unlock(&zq->queue->lock);
+		}
 	}
-	spin_unlock_bh(&zcrypt_device_lock);
+	spin_unlock(&zcrypt_list_lock);
 }
 
 static int zcrypt_pendingq_count(void)
 {
-	struct zcrypt_device *zdev;
-	int pendingq_count = 0;
-
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list) {
-		spin_lock(&zdev->ap_dev->lock);
-		pendingq_count += zdev->ap_dev->pendingq_count;
-		spin_unlock(&zdev->ap_dev->lock);
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
+	int pendingq_count;
+
+	pendingq_count = 0;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index)
+				continue;
+			spin_lock(&zq->queue->lock);
+			pendingq_count += zq->queue->pendingq_count;
+			spin_unlock(&zq->queue->lock);
+		}
 	}
-	spin_unlock_bh(&zcrypt_device_lock);
+	spin_unlock(&zcrypt_list_lock);
 	return pendingq_count;
 }
 
 static int zcrypt_requestq_count(void)
 {
-	struct zcrypt_device *zdev;
-	int requestq_count = 0;
-
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list) {
-		spin_lock(&zdev->ap_dev->lock);
-		requestq_count += zdev->ap_dev->requestq_count;
-		spin_unlock(&zdev->ap_dev->lock);
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
+	int requestq_count;
+
+	requestq_count = 0;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index)
+				continue;
+			spin_lock(&zq->queue->lock);
+			requestq_count += zq->queue->requestq_count;
+			spin_unlock(&zq->queue->lock);
+		}
 	}
-	spin_unlock_bh(&zcrypt_device_lock);
+	spin_unlock(&zcrypt_list_lock);
 	return requestq_count;
 }
 
 static int zcrypt_count_type(int type)
 {
-	struct zcrypt_device *zdev;
-	int device_count = 0;
-
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list)
-		if (zdev->user_space_type == type)
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
+	int device_count;
+
+	device_count = 0;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		if (zc->card->id != type)
+			continue;
+		for_each_zcrypt_queue(zq, zc) {
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index)
+				continue;
 			device_count++;
-	spin_unlock_bh(&zcrypt_device_lock);
+		}
+	}
+	spin_unlock(&zcrypt_list_lock);
 	return device_count;
 }
 
@@ -887,6 +868,25 @@ static long zcrypt_unlocked_ioctl(struct file *filp, unsigned int cmd,
 			return -EFAULT;
 		return rc;
 	}
+	case ZDEVICESTATUS: {
+		struct zcrypt_device_matrix *device_status;
+
+		device_status = kzalloc(sizeof(struct zcrypt_device_matrix),
+					GFP_KERNEL);
+		if (!device_status)
+			return -ENOMEM;
+
+		zcrypt_device_status_mask(device_status);
+
+		if (copy_to_user((char __user *) arg, device_status,
+				 sizeof(struct zcrypt_device_matrix))) {
+			kfree(device_status);
+			return -EFAULT;
+		}
+
+		kfree(device_status);
+		return 0;
+	}
 	case Z90STAT_STATUS_MASK: {
 		char status[AP_DEVICES];
 		zcrypt_status_mask(status);
@@ -1249,29 +1249,36 @@ static int zcrypt_proc_open(struct inode *inode, struct file *file)
 
 static void zcrypt_disable_card(int index)
 {
-	struct zcrypt_device *zdev;
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
 
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list)
-		if (AP_QID_DEVICE(zdev->ap_dev->qid) == index) {
-			zdev->online = 0;
-			ap_flush_queue(zdev->ap_dev);
-			break;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index)
+				continue;
+			zq->online = 0;
+			ap_flush_queue(zq->queue);
 		}
-	spin_unlock_bh(&zcrypt_device_lock);
+	}
+	spin_unlock(&zcrypt_list_lock);
 }
 
 static void zcrypt_enable_card(int index)
 {
-	struct zcrypt_device *zdev;
+	struct zcrypt_card *zc;
+	struct zcrypt_queue *zq;
 
-	spin_lock_bh(&zcrypt_device_lock);
-	list_for_each_entry(zdev, &zcrypt_device_list, list)
-		if (AP_QID_DEVICE(zdev->ap_dev->qid) == index) {
-			zdev->online = 1;
-			break;
+	spin_lock(&zcrypt_list_lock);
+	for_each_zcrypt_card(zc) {
+		for_each_zcrypt_queue(zq, zc) {
+			if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index)
+				continue;
+			zq->online = 1;
+			ap_flush_queue(zq->queue);
 		}
-	spin_unlock_bh(&zcrypt_device_lock);
+	}
+	spin_unlock(&zcrypt_list_lock);
 }
 
 static ssize_t zcrypt_proc_write(struct file *file, const char __user *buffer,
@@ -1369,7 +1376,7 @@ static struct hwrng zcrypt_rng_dev = {
 	.quality	= 990,
 };
 
-static int zcrypt_rng_device_add(void)
+int zcrypt_rng_device_add(void)
 {
 	int rc = 0;
 
@@ -1399,7 +1406,7 @@ out:
 	return rc;
 }
 
-static void zcrypt_rng_device_remove(void)
+void zcrypt_rng_device_remove(void)
 {
 	mutex_lock(&zcrypt_rng_mutex);
 	zcrypt_rng_device_count--;
@@ -1412,24 +1419,19 @@ static void zcrypt_rng_device_remove(void)
 
 int __init zcrypt_debug_init(void)
 {
-	debugfs_root = debugfs_create_dir("zcrypt", NULL);
-
-	zcrypt_dbf_common = debug_register("zcrypt_common", 1, 1, 16);
-	debug_register_view(zcrypt_dbf_common, &debug_hex_ascii_view);
-	debug_set_level(zcrypt_dbf_common, DBF_ERR);
-
-	zcrypt_dbf_devices = debug_register("zcrypt_devices", 1, 1, 16);
-	debug_register_view(zcrypt_dbf_devices, &debug_hex_ascii_view);
-	debug_set_level(zcrypt_dbf_devices, DBF_ERR);
+	zcrypt_dbf_root = debugfs_create_dir("zcrypt", NULL);
+	zcrypt_dbf_info = debug_register("zcrypt", 1, 1,
+					 DBF_MAX_SPRINTF_ARGS * sizeof(long));
+	debug_register_view(zcrypt_dbf_info, &debug_sprintf_view);
+	debug_set_level(zcrypt_dbf_info, DBF_ERR);
 
 	return 0;
 }
 
 void zcrypt_debug_exit(void)
 {
-	debugfs_remove(debugfs_root);
-	debug_unregister(zcrypt_dbf_common);
-	debug_unregister(zcrypt_dbf_devices);
+	debugfs_remove(zcrypt_dbf_root);
+	debug_unregister(zcrypt_dbf_info);
 }
 
 /**
@@ -1453,12 +1455,15 @@ int __init zcrypt_api_init(void)
 		goto out;
 
 	/* Set up the proc file system */
-	zcrypt_entry = proc_create("driver/z90crypt", 0644, NULL, &zcrypt_proc_fops);
+	zcrypt_entry = proc_create("driver/z90crypt", 0644, NULL,
+				   &zcrypt_proc_fops);
 	if (!zcrypt_entry) {
 		rc = -ENOMEM;
 		goto out_misc;
 	}
 
+	zcrypt_msgtype6_init();
+	zcrypt_msgtype50_init();
 	return 0;
 
 out_misc:
@@ -1472,10 +1477,12 @@ out:
  *
  * The module termination code.
  */
-void zcrypt_api_exit(void)
+void __exit zcrypt_api_exit(void)
 {
 	remove_proc_entry("driver/z90crypt", NULL);
 	misc_deregister(&zcrypt_misc_device);
+	zcrypt_msgtype6_exit();
+	zcrypt_msgtype50_exit();
 	zcrypt_debug_exit();
 }
 
diff --git a/drivers/s390/crypto/zcrypt_api.h b/drivers/s390/crypto/zcrypt_api.h
index 38618f05ad92..274a59051534 100644
--- a/drivers/s390/crypto/zcrypt_api.h
+++ b/drivers/s390/crypto/zcrypt_api.h
@@ -84,57 +84,110 @@ struct ica_z90_status {
  */
 #define ZCRYPT_RNG_BUFFER_SIZE	4096
 
-struct zcrypt_device;
+/*
+ * Identifier for Crypto Request Performance Index
+ */
+enum crypto_ops {
+	MEX_1K,
+	MEX_2K,
+	MEX_4K,
+	CRT_1K,
+	CRT_2K,
+	CRT_4K,
+	HWRNG,
+	SECKEY,
+	NUM_OPS
+};
+
+struct zcrypt_queue;
 
 struct zcrypt_ops {
-	long (*rsa_modexpo)(struct zcrypt_device *, struct ica_rsa_modexpo *);
-	long (*rsa_modexpo_crt)(struct zcrypt_device *,
+	long (*rsa_modexpo)(struct zcrypt_queue *, struct ica_rsa_modexpo *);
+	long (*rsa_modexpo_crt)(struct zcrypt_queue *,
 				struct ica_rsa_modexpo_crt *);
-	long (*send_cprb)(struct zcrypt_device *, struct ica_xcRB *);
-	long (*send_ep11_cprb)(struct zcrypt_device *, struct ep11_urb *);
-	long (*rng)(struct zcrypt_device *, char *);
+	long (*send_cprb)(struct zcrypt_queue *, struct ica_xcRB *,
+			  struct ap_message *);
+	long (*send_ep11_cprb)(struct zcrypt_queue *, struct ep11_urb *,
+			       struct ap_message *);
+	long (*rng)(struct zcrypt_queue *, char *, struct ap_message *);
 	struct list_head list;		/* zcrypt ops list. */
 	struct module *owner;
 	int variant;
 	char name[128];
 };
 
-struct zcrypt_device {
+struct zcrypt_card {
 	struct list_head list;		/* Device list. */
-	spinlock_t lock;		/* Per device lock. */
+	struct list_head zqueues;	/* List of zcrypt queues */
 	struct kref refcount;		/* device refcounting */
-	struct ap_device *ap_dev;	/* The "real" ap device. */
-	struct zcrypt_ops *ops;		/* Crypto operations. */
+	struct ap_card *card;		/* The "real" ap card device. */
 	int online;			/* User online/offline */
 
 	int user_space_type;		/* User space device id. */
 	char *type_string;		/* User space device name. */
 	int min_mod_size;		/* Min number of bits. */
 	int max_mod_size;		/* Max number of bits. */
-	int short_crt;			/* Card has crt length restriction. */
-	int speed_rating;		/* Speed of the crypto device. */
+	int max_exp_bit_length;
+	int speed_rating[NUM_OPS];	/* Speed idx of crypto ops. */
+	atomic_t load;			/* Utilization of the crypto device */
 
 	int request_count;		/* # current requests. */
+};
 
-	struct ap_message reply;	/* Per-device reply structure. */
-	int max_exp_bit_length;
+struct zcrypt_queue {
+	struct list_head list;		/* Device list. */
+	struct kref refcount;		/* device refcounting */
+	struct zcrypt_card *zcard;
+	struct zcrypt_ops *ops;		/* Crypto operations. */
+	struct ap_queue *queue;		/* The "real" ap queue device. */
+	int online;			/* User online/offline */
+
+	atomic_t load;			/* Utilization of the crypto device */
 
-	debug_info_t *dbf_area;		/* debugging */
+	int request_count;		/* # current requests. */
+
+	struct ap_message reply;	/* Per-device reply structure. */
 };
 
 /* transport layer rescanning */
 extern atomic_t zcrypt_rescan_req;
 
-struct zcrypt_device *zcrypt_device_alloc(size_t);
-void zcrypt_device_free(struct zcrypt_device *);
-void zcrypt_device_get(struct zcrypt_device *);
-int zcrypt_device_put(struct zcrypt_device *);
-int zcrypt_device_register(struct zcrypt_device *);
-void zcrypt_device_unregister(struct zcrypt_device *);
+extern spinlock_t zcrypt_list_lock;
+extern int zcrypt_device_count;
+extern struct list_head zcrypt_card_list;
+
+#define for_each_zcrypt_card(_zc) \
+	list_for_each_entry(_zc, &zcrypt_card_list, list)
+
+#define for_each_zcrypt_queue(_zq, _zc) \
+	list_for_each_entry(_zq, &(_zc)->zqueues, list)
+
+struct zcrypt_card *zcrypt_card_alloc(void);
+void zcrypt_card_free(struct zcrypt_card *);
+void zcrypt_card_get(struct zcrypt_card *);
+int zcrypt_card_put(struct zcrypt_card *);
+int zcrypt_card_register(struct zcrypt_card *);
+void zcrypt_card_unregister(struct zcrypt_card *);
+struct zcrypt_card *zcrypt_card_get_best(unsigned int *,
+					 unsigned int, unsigned int);
+void zcrypt_card_put_best(struct zcrypt_card *, unsigned int);
+
+struct zcrypt_queue *zcrypt_queue_alloc(size_t);
+void zcrypt_queue_free(struct zcrypt_queue *);
+void zcrypt_queue_get(struct zcrypt_queue *);
+int zcrypt_queue_put(struct zcrypt_queue *);
+int zcrypt_queue_register(struct zcrypt_queue *);
+void zcrypt_queue_unregister(struct zcrypt_queue *);
+void zcrypt_queue_force_online(struct zcrypt_queue *, int);
+struct zcrypt_queue *zcrypt_queue_get_best(unsigned int, unsigned int);
+void  zcrypt_queue_put_best(struct zcrypt_queue *, unsigned int);
+
+int zcrypt_rng_device_add(void);
+void zcrypt_rng_device_remove(void);
+
 void zcrypt_msgtype_register(struct zcrypt_ops *);
 void zcrypt_msgtype_unregister(struct zcrypt_ops *);
-struct zcrypt_ops *zcrypt_msgtype_request(unsigned char *, int);
-void zcrypt_msgtype_release(struct zcrypt_ops *);
+struct zcrypt_ops *zcrypt_msgtype(unsigned char *, int);
 int zcrypt_api_init(void);
 void zcrypt_api_exit(void);
 
diff --git a/drivers/s390/crypto/zcrypt_card.c b/drivers/s390/crypto/zcrypt_card.c
new file mode 100644
index 000000000000..53436ea52230
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_card.c
@@ -0,0 +1,187 @@
+/*
+ *  zcrypt 2.1.0
+ *
+ *  Copyright IBM Corp. 2001, 2012
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *	       Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *  MSGTYPE restruct:		  Holger Dengler <hd@linux.vnet.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/compat.h>
+#include <linux/slab.h>
+#include <linux/atomic.h>
+#include <linux/uaccess.h>
+#include <linux/hw_random.h>
+#include <linux/debugfs.h>
+#include <asm/debug.h>
+
+#include "zcrypt_debug.h"
+#include "zcrypt_api.h"
+
+#include "zcrypt_msgtype6.h"
+#include "zcrypt_msgtype50.h"
+
+/*
+ * Device attributes common for all crypto card devices.
+ */
+
+static ssize_t zcrypt_card_type_show(struct device *dev,
+				     struct device_attribute *attr, char *buf)
+{
+	struct zcrypt_card *zc = to_ap_card(dev)->private;
+
+	return snprintf(buf, PAGE_SIZE, "%s\n", zc->type_string);
+}
+
+static DEVICE_ATTR(type, 0444, zcrypt_card_type_show, NULL);
+
+static ssize_t zcrypt_card_online_show(struct device *dev,
+				       struct device_attribute *attr,
+				       char *buf)
+{
+	struct zcrypt_card *zc = to_ap_card(dev)->private;
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", zc->online);
+}
+
+static ssize_t zcrypt_card_online_store(struct device *dev,
+					struct device_attribute *attr,
+					const char *buf, size_t count)
+{
+	struct zcrypt_card *zc = to_ap_card(dev)->private;
+	struct zcrypt_queue *zq;
+	int online, id;
+
+	if (sscanf(buf, "%d\n", &online) != 1 || online < 0 || online > 1)
+		return -EINVAL;
+
+	zc->online = online;
+	id = zc->card->id;
+
+	ZCRYPT_DBF(DBF_INFO, "card=%02x online=%d\n", id, online);
+
+	spin_lock(&zcrypt_list_lock);
+	list_for_each_entry(zq, &zc->zqueues, list)
+		zcrypt_queue_force_online(zq, online);
+	spin_unlock(&zcrypt_list_lock);
+	return count;
+}
+
+static DEVICE_ATTR(online, 0644, zcrypt_card_online_show,
+		   zcrypt_card_online_store);
+
+static struct attribute *zcrypt_card_attrs[] = {
+	&dev_attr_type.attr,
+	&dev_attr_online.attr,
+	NULL,
+};
+
+static struct attribute_group zcrypt_card_attr_group = {
+	.attrs = zcrypt_card_attrs,
+};
+
+struct zcrypt_card *zcrypt_card_alloc(void)
+{
+	struct zcrypt_card *zc;
+
+	zc = kzalloc(sizeof(struct zcrypt_card), GFP_KERNEL);
+	if (!zc)
+		return NULL;
+	INIT_LIST_HEAD(&zc->list);
+	INIT_LIST_HEAD(&zc->zqueues);
+	kref_init(&zc->refcount);
+	return zc;
+}
+EXPORT_SYMBOL(zcrypt_card_alloc);
+
+void zcrypt_card_free(struct zcrypt_card *zc)
+{
+	kfree(zc);
+}
+EXPORT_SYMBOL(zcrypt_card_free);
+
+static void zcrypt_card_release(struct kref *kref)
+{
+	struct zcrypt_card *zdev =
+		container_of(kref, struct zcrypt_card, refcount);
+	zcrypt_card_free(zdev);
+}
+
+void zcrypt_card_get(struct zcrypt_card *zc)
+{
+	kref_get(&zc->refcount);
+}
+EXPORT_SYMBOL(zcrypt_card_get);
+
+int zcrypt_card_put(struct zcrypt_card *zc)
+{
+	return kref_put(&zc->refcount, zcrypt_card_release);
+}
+EXPORT_SYMBOL(zcrypt_card_put);
+
+/**
+ * zcrypt_card_register() - Register a crypto card device.
+ * @zc: Pointer to a crypto card device
+ *
+ * Register a crypto card device. Returns 0 if successful.
+ */
+int zcrypt_card_register(struct zcrypt_card *zc)
+{
+	int rc;
+
+	rc = sysfs_create_group(&zc->card->ap_dev.device.kobj,
+				&zcrypt_card_attr_group);
+	if (rc)
+		return rc;
+
+	spin_lock(&zcrypt_list_lock);
+	list_add_tail(&zc->list, &zcrypt_card_list);
+	spin_unlock(&zcrypt_list_lock);
+
+	zc->online = 1;
+
+	ZCRYPT_DBF(DBF_INFO, "card=%02x register online=1\n", zc->card->id);
+
+	return rc;
+}
+EXPORT_SYMBOL(zcrypt_card_register);
+
+/**
+ * zcrypt_card_unregister(): Unregister a crypto card device.
+ * @zc: Pointer to crypto card device
+ *
+ * Unregister a crypto card device.
+ */
+void zcrypt_card_unregister(struct zcrypt_card *zc)
+{
+	ZCRYPT_DBF(DBF_INFO, "card=%02x unregister\n", zc->card->id);
+
+	spin_lock(&zcrypt_list_lock);
+	list_del_init(&zc->list);
+	spin_unlock(&zcrypt_list_lock);
+	sysfs_remove_group(&zc->card->ap_dev.device.kobj,
+			   &zcrypt_card_attr_group);
+}
+EXPORT_SYMBOL(zcrypt_card_unregister);
diff --git a/drivers/s390/crypto/zcrypt_cex2a.c b/drivers/s390/crypto/zcrypt_cex2a.c
index 15104aaa075a..c7d48a18199e 100644
--- a/drivers/s390/crypto/zcrypt_cex2a.c
+++ b/drivers/s390/crypto/zcrypt_cex2a.c
@@ -31,6 +31,7 @@
 #include <linux/err.h>
 #include <linux/atomic.h>
 #include <asm/uaccess.h>
+#include <linux/mod_devicetable.h>
 
 #include "ap_bus.h"
 #include "zcrypt_api.h"
@@ -43,9 +44,6 @@
 #define CEX3A_MIN_MOD_SIZE	CEX2A_MIN_MOD_SIZE
 #define CEX3A_MAX_MOD_SIZE	512	/* 4096 bits	*/
 
-#define CEX2A_SPEED_RATING	970
-#define CEX3A_SPEED_RATING	900 /* Fixme: Needs finetuning */
-
 #define CEX2A_MAX_MESSAGE_SIZE	0x390	/* sizeof(struct type50_crb2_msg)    */
 #define CEX2A_MAX_RESPONSE_SIZE 0x110	/* max outputdatalength + type80_hdr */
 
@@ -57,107 +55,195 @@
 #define CEX2A_CLEANUP_TIME	(15*HZ)
 #define CEX3A_CLEANUP_TIME	CEX2A_CLEANUP_TIME
 
-static struct ap_device_id zcrypt_cex2a_ids[] = {
-	{ AP_DEVICE(AP_DEVICE_TYPE_CEX2A) },
-	{ AP_DEVICE(AP_DEVICE_TYPE_CEX3A) },
-	{ /* end of list */ },
-};
-
-MODULE_DEVICE_TABLE(ap, zcrypt_cex2a_ids);
 MODULE_AUTHOR("IBM Corporation");
 MODULE_DESCRIPTION("CEX2A Cryptographic Coprocessor device driver, " \
 		   "Copyright IBM Corp. 2001, 2012");
 MODULE_LICENSE("GPL");
 
-static int zcrypt_cex2a_probe(struct ap_device *ap_dev);
-static void zcrypt_cex2a_remove(struct ap_device *ap_dev);
+static struct ap_device_id zcrypt_cex2a_card_ids[] = {
+	{ .dev_type = AP_DEVICE_TYPE_CEX2A,
+	  .match_flags = AP_DEVICE_ID_MATCH_CARD_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX3A,
+	  .match_flags = AP_DEVICE_ID_MATCH_CARD_TYPE },
+	{ /* end of list */ },
+};
+
+MODULE_DEVICE_TABLE(ap, zcrypt_cex2a_card_ids);
 
-static struct ap_driver zcrypt_cex2a_driver = {
-	.probe = zcrypt_cex2a_probe,
-	.remove = zcrypt_cex2a_remove,
-	.ids = zcrypt_cex2a_ids,
-	.request_timeout = CEX2A_CLEANUP_TIME,
+static struct ap_device_id zcrypt_cex2a_queue_ids[] = {
+	{ .dev_type = AP_DEVICE_TYPE_CEX2A,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX3A,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ /* end of list */ },
 };
 
+MODULE_DEVICE_TABLE(ap, zcrypt_cex2a_queue_ids);
+
 /**
- * Probe function for CEX2A cards. It always accepts the AP device
- * since the bus_match already checked the hardware type.
+ * Probe function for CEX2A card devices. It always accepts the AP device
+ * since the bus_match already checked the card type.
  * @ap_dev: pointer to the AP device.
  */
-static int zcrypt_cex2a_probe(struct ap_device *ap_dev)
+static int zcrypt_cex2a_card_probe(struct ap_device *ap_dev)
 {
-	struct zcrypt_device *zdev = NULL;
+	/*
+	 * Normalized speed ratings per crypto adapter
+	 * MEX_1k, MEX_2k, MEX_4k, CRT_1k, CRT_2k, CRT_4k, RNG, SECKEY
+	 */
+	static const int CEX2A_SPEED_IDX[] = {
+		800, 1000, 2000,  900, 1200, 2400, 0, 0};
+	static const int CEX3A_SPEED_IDX[] = {
+		400,  500, 1000,  450,	550, 1200, 0, 0};
+
+	struct ap_card *ac = to_ap_card(&ap_dev->device);
+	struct zcrypt_card *zc;
 	int rc = 0;
 
+	zc = zcrypt_card_alloc();
+	if (!zc)
+		return -ENOMEM;
+	zc->card = ac;
+	ac->private = zc;
+
+	if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX2A) {
+		zc->min_mod_size = CEX2A_MIN_MOD_SIZE;
+		zc->max_mod_size = CEX2A_MAX_MOD_SIZE;
+		memcpy(zc->speed_rating, CEX2A_SPEED_IDX,
+		       sizeof(CEX2A_SPEED_IDX));
+		zc->max_exp_bit_length = CEX2A_MAX_MOD_SIZE;
+		zc->type_string = "CEX2A";
+		zc->user_space_type = ZCRYPT_CEX2A;
+	} else if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX3A) {
+		zc->min_mod_size = CEX2A_MIN_MOD_SIZE;
+		zc->max_mod_size = CEX2A_MAX_MOD_SIZE;
+		zc->max_exp_bit_length = CEX2A_MAX_MOD_SIZE;
+		if (ap_test_bit(&ac->functions, AP_FUNC_MEX4K) &&
+		    ap_test_bit(&ac->functions, AP_FUNC_CRT4K)) {
+			zc->max_mod_size = CEX3A_MAX_MOD_SIZE;
+			zc->max_exp_bit_length = CEX3A_MAX_MOD_SIZE;
+		}
+		memcpy(zc->speed_rating, CEX3A_SPEED_IDX,
+		       sizeof(CEX3A_SPEED_IDX));
+		zc->type_string = "CEX3A";
+		zc->user_space_type = ZCRYPT_CEX3A;
+	} else {
+		zcrypt_card_free(zc);
+		return -ENODEV;
+	}
+	zc->online = 1;
+
+	rc = zcrypt_card_register(zc);
+	if (rc) {
+		ac->private = NULL;
+		zcrypt_card_free(zc);
+	}
+
+	return rc;
+}
+
+/**
+ * This is called to remove the CEX2A card driver information
+ * if an AP card device is removed.
+ */
+static void zcrypt_cex2a_card_remove(struct ap_device *ap_dev)
+{
+	struct zcrypt_card *zc = to_ap_card(&ap_dev->device)->private;
+
+	if (zc)
+		zcrypt_card_unregister(zc);
+}
+
+static struct ap_driver zcrypt_cex2a_card_driver = {
+	.probe = zcrypt_cex2a_card_probe,
+	.remove = zcrypt_cex2a_card_remove,
+	.ids = zcrypt_cex2a_card_ids,
+};
+
+/**
+ * Probe function for CEX2A queue devices. It always accepts the AP device
+ * since the bus_match already checked the queue type.
+ * @ap_dev: pointer to the AP device.
+ */
+static int zcrypt_cex2a_queue_probe(struct ap_device *ap_dev)
+{
+	struct ap_queue *aq = to_ap_queue(&ap_dev->device);
+	struct zcrypt_queue *zq = NULL;
+	int rc;
+
 	switch (ap_dev->device_type) {
 	case AP_DEVICE_TYPE_CEX2A:
-		zdev = zcrypt_device_alloc(CEX2A_MAX_RESPONSE_SIZE);
-		if (!zdev)
+		zq = zcrypt_queue_alloc(CEX2A_MAX_RESPONSE_SIZE);
+		if (!zq)
 			return -ENOMEM;
-		zdev->user_space_type = ZCRYPT_CEX2A;
-		zdev->type_string = "CEX2A";
-		zdev->min_mod_size = CEX2A_MIN_MOD_SIZE;
-		zdev->max_mod_size = CEX2A_MAX_MOD_SIZE;
-		zdev->short_crt = 1;
-		zdev->speed_rating = CEX2A_SPEED_RATING;
-		zdev->max_exp_bit_length = CEX2A_MAX_MOD_SIZE;
 		break;
 	case AP_DEVICE_TYPE_CEX3A:
-		zdev = zcrypt_device_alloc(CEX3A_MAX_RESPONSE_SIZE);
-		if (!zdev)
+		zq = zcrypt_queue_alloc(CEX3A_MAX_RESPONSE_SIZE);
+		if (!zq)
 			return -ENOMEM;
-		zdev->user_space_type = ZCRYPT_CEX3A;
-		zdev->type_string = "CEX3A";
-		zdev->min_mod_size = CEX2A_MIN_MOD_SIZE;
-		zdev->max_mod_size = CEX2A_MAX_MOD_SIZE;
-		zdev->max_exp_bit_length = CEX2A_MAX_MOD_SIZE;
-		if (ap_test_bit(&ap_dev->functions, AP_FUNC_MEX4K) &&
-		    ap_test_bit(&ap_dev->functions, AP_FUNC_CRT4K)) {
-			zdev->max_mod_size = CEX3A_MAX_MOD_SIZE;
-			zdev->max_exp_bit_length = CEX3A_MAX_MOD_SIZE;
-		}
-		zdev->short_crt = 1;
-		zdev->speed_rating = CEX3A_SPEED_RATING;
 		break;
 	}
-	if (!zdev)
+	if (!zq)
 		return -ENODEV;
-	zdev->ops = zcrypt_msgtype_request(MSGTYPE50_NAME,
-					   MSGTYPE50_VARIANT_DEFAULT);
-	zdev->ap_dev = ap_dev;
-	zdev->online = 1;
-	ap_device_init_reply(ap_dev, &zdev->reply);
-	ap_dev->private = zdev;
-	rc = zcrypt_device_register(zdev);
+	zq->ops = zcrypt_msgtype(MSGTYPE50_NAME, MSGTYPE50_VARIANT_DEFAULT);
+	zq->queue = aq;
+	zq->online = 1;
+	atomic_set(&zq->load, 0);
+	ap_queue_init_reply(aq, &zq->reply);
+	aq->request_timeout = CEX2A_CLEANUP_TIME,
+	aq->private = zq;
+	rc = zcrypt_queue_register(zq);
 	if (rc) {
-		ap_dev->private = NULL;
-		zcrypt_msgtype_release(zdev->ops);
-		zcrypt_device_free(zdev);
+		aq->private = NULL;
+		zcrypt_queue_free(zq);
 	}
+
 	return rc;
 }
 
 /**
- * This is called to remove the extended CEX2A driver information
- * if an AP device is removed.
+ * This is called to remove the CEX2A queue driver information
+ * if an AP queue device is removed.
  */
-static void zcrypt_cex2a_remove(struct ap_device *ap_dev)
+static void zcrypt_cex2a_queue_remove(struct ap_device *ap_dev)
 {
-	struct zcrypt_device *zdev = ap_dev->private;
-	struct zcrypt_ops *zops = zdev->ops;
+	struct ap_queue *aq = to_ap_queue(&ap_dev->device);
+	struct zcrypt_queue *zq = aq->private;
 
-	zcrypt_device_unregister(zdev);
-	zcrypt_msgtype_release(zops);
+	ap_queue_remove(aq);
+	if (zq)
+		zcrypt_queue_unregister(zq);
 }
 
+static struct ap_driver zcrypt_cex2a_queue_driver = {
+	.probe = zcrypt_cex2a_queue_probe,
+	.remove = zcrypt_cex2a_queue_remove,
+	.suspend = ap_queue_suspend,
+	.resume = ap_queue_resume,
+	.ids = zcrypt_cex2a_queue_ids,
+};
+
 int __init zcrypt_cex2a_init(void)
 {
-	return ap_driver_register(&zcrypt_cex2a_driver, THIS_MODULE, "cex2a");
+	int rc;
+
+	rc = ap_driver_register(&zcrypt_cex2a_card_driver,
+				THIS_MODULE, "cex2acard");
+	if (rc)
+		return rc;
+
+	rc = ap_driver_register(&zcrypt_cex2a_queue_driver,
+				THIS_MODULE, "cex2aqueue");
+	if (rc)
+		ap_driver_unregister(&zcrypt_cex2a_card_driver);
+
+	return rc;
 }
 
 void __exit zcrypt_cex2a_exit(void)
 {
-	ap_driver_unregister(&zcrypt_cex2a_driver);
+	ap_driver_unregister(&zcrypt_cex2a_queue_driver);
+	ap_driver_unregister(&zcrypt_cex2a_card_driver);
 }
 
 module_init(zcrypt_cex2a_init);
diff --git a/drivers/s390/crypto/zcrypt_cex4.c b/drivers/s390/crypto/zcrypt_cex4.c
index ccb2e78ebf0e..4e91163d70a6 100644
--- a/drivers/s390/crypto/zcrypt_cex4.c
+++ b/drivers/s390/crypto/zcrypt_cex4.c
@@ -9,6 +9,7 @@
 #include <linux/err.h>
 #include <linux/atomic.h>
 #include <linux/uaccess.h>
+#include <linux/mod_devicetable.h>
 
 #include "ap_bus.h"
 #include "zcrypt_api.h"
@@ -24,13 +25,6 @@
 #define CEX4C_MIN_MOD_SIZE	 16	/*  256 bits	*/
 #define CEX4C_MAX_MOD_SIZE	512	/* 4096 bits	*/
 
-#define CEX4A_SPEED_RATING	900	 /* TODO new card, new speed rating */
-#define CEX4C_SPEED_RATING	6500	 /* TODO new card, new speed rating */
-#define CEX4P_SPEED_RATING	7000	 /* TODO new card, new speed rating */
-#define CEX5A_SPEED_RATING	450	 /* TODO new card, new speed rating */
-#define CEX5C_SPEED_RATING	3250	 /* TODO new card, new speed rating */
-#define CEX5P_SPEED_RATING	3500	 /* TODO new card, new speed rating */
-
 #define CEX4A_MAX_MESSAGE_SIZE	MSGTYPE50_CRB3_MAX_MSG_SIZE
 #define CEX4C_MAX_MESSAGE_SIZE	MSGTYPE06_MAX_MSG_SIZE
 
@@ -41,147 +35,246 @@
  */
 #define CEX4_CLEANUP_TIME	(900*HZ)
 
-static struct ap_device_id zcrypt_cex4_ids[] = {
-	{ AP_DEVICE(AP_DEVICE_TYPE_CEX4)  },
-	{ AP_DEVICE(AP_DEVICE_TYPE_CEX5)  },
-	{ /* end of list */ },
-};
-
-MODULE_DEVICE_TABLE(ap, zcrypt_cex4_ids);
 MODULE_AUTHOR("IBM Corporation");
 MODULE_DESCRIPTION("CEX4 Cryptographic Card device driver, " \
 		   "Copyright IBM Corp. 2012");
 MODULE_LICENSE("GPL");
 
-static int zcrypt_cex4_probe(struct ap_device *ap_dev);
-static void zcrypt_cex4_remove(struct ap_device *ap_dev);
+static struct ap_device_id zcrypt_cex4_card_ids[] = {
+	{ .dev_type = AP_DEVICE_TYPE_CEX4,
+	  .match_flags = AP_DEVICE_ID_MATCH_CARD_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX5,
+	  .match_flags = AP_DEVICE_ID_MATCH_CARD_TYPE },
+	{ /* end of list */ },
+};
 
-static struct ap_driver zcrypt_cex4_driver = {
-	.probe = zcrypt_cex4_probe,
-	.remove = zcrypt_cex4_remove,
-	.ids = zcrypt_cex4_ids,
-	.request_timeout = CEX4_CLEANUP_TIME,
+MODULE_DEVICE_TABLE(ap, zcrypt_cex4_card_ids);
+
+static struct ap_device_id zcrypt_cex4_queue_ids[] = {
+	{ .dev_type = AP_DEVICE_TYPE_CEX4,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX5,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ /* end of list */ },
 };
 
+MODULE_DEVICE_TABLE(ap, zcrypt_cex4_queue_ids);
+
 /**
- * Probe function for CEX4 cards. It always accepts the AP device
+ * Probe function for CEX4 card device. It always accepts the AP device
  * since the bus_match already checked the hardware type.
  * @ap_dev: pointer to the AP device.
  */
-static int zcrypt_cex4_probe(struct ap_device *ap_dev)
+static int zcrypt_cex4_card_probe(struct ap_device *ap_dev)
 {
-	struct zcrypt_device *zdev = NULL;
+	/*
+	 * Normalized speed ratings per crypto adapter
+	 * MEX_1k, MEX_2k, MEX_4k, CRT_1k, CRT_2k, CRT_4k, RNG, SECKEY
+	 */
+	static const int CEX4A_SPEED_IDX[] = {
+		5,  6,	  59,  20, 115,  581,  0,  0};
+	static const int CEX5A_SPEED_IDX[] = {
+		3,  3,	   6,	8,  32,  218,  0,  0};
+	static const int CEX4C_SPEED_IDX[] = {
+		24,  25,   82,	41, 138, 1111, 79,  8};
+	static const int CEX5C_SPEED_IDX[] = {
+		10,  14,   23,	17,  45,  242, 63,  4};
+	static const int CEX4P_SPEED_IDX[] = {
+		142, 198, 1852, 203, 331, 1563,  0,  8};
+	static const int CEX5P_SPEED_IDX[] = {
+		49,  67,  131,	52,  85,  287,	0,  4};
+
+	struct ap_card *ac = to_ap_card(&ap_dev->device);
+	struct zcrypt_card *zc;
 	int rc = 0;
 
-	switch (ap_dev->device_type) {
-	case AP_DEVICE_TYPE_CEX4:
-	case AP_DEVICE_TYPE_CEX5:
-		if (ap_test_bit(&ap_dev->functions, AP_FUNC_ACCEL)) {
-			zdev = zcrypt_device_alloc(CEX4A_MAX_MESSAGE_SIZE);
-			if (!zdev)
-				return -ENOMEM;
-			if (ap_dev->device_type == AP_DEVICE_TYPE_CEX4) {
-				zdev->type_string = "CEX4A";
-				zdev->speed_rating = CEX4A_SPEED_RATING;
-			} else {
-				zdev->type_string = "CEX5A";
-				zdev->speed_rating = CEX5A_SPEED_RATING;
-			}
-			zdev->user_space_type = ZCRYPT_CEX3A;
-			zdev->min_mod_size = CEX4A_MIN_MOD_SIZE;
-			if (ap_test_bit(&ap_dev->functions, AP_FUNC_MEX4K) &&
-			    ap_test_bit(&ap_dev->functions, AP_FUNC_CRT4K)) {
-				zdev->max_mod_size =
-					CEX4A_MAX_MOD_SIZE_4K;
-				zdev->max_exp_bit_length =
-					CEX4A_MAX_MOD_SIZE_4K;
-			} else {
-				zdev->max_mod_size =
-					CEX4A_MAX_MOD_SIZE_2K;
-				zdev->max_exp_bit_length =
-					CEX4A_MAX_MOD_SIZE_2K;
-			}
-			zdev->short_crt = 1;
-			zdev->ops = zcrypt_msgtype_request(MSGTYPE50_NAME,
-							   MSGTYPE50_VARIANT_DEFAULT);
-		} else if (ap_test_bit(&ap_dev->functions, AP_FUNC_COPRO)) {
-			zdev = zcrypt_device_alloc(CEX4C_MAX_MESSAGE_SIZE);
-			if (!zdev)
-				return -ENOMEM;
-			if (ap_dev->device_type == AP_DEVICE_TYPE_CEX4) {
-				zdev->type_string = "CEX4C";
-				zdev->speed_rating = CEX4C_SPEED_RATING;
-			} else {
-				zdev->type_string = "CEX5C";
-				zdev->speed_rating = CEX5C_SPEED_RATING;
-			}
-			zdev->user_space_type = ZCRYPT_CEX3C;
-			zdev->min_mod_size = CEX4C_MIN_MOD_SIZE;
-			zdev->max_mod_size = CEX4C_MAX_MOD_SIZE;
-			zdev->max_exp_bit_length = CEX4C_MAX_MOD_SIZE;
-			zdev->short_crt = 0;
-			zdev->ops = zcrypt_msgtype_request(MSGTYPE06_NAME,
-							   MSGTYPE06_VARIANT_DEFAULT);
-		} else if (ap_test_bit(&ap_dev->functions, AP_FUNC_EP11)) {
-			zdev = zcrypt_device_alloc(CEX4C_MAX_MESSAGE_SIZE);
-			if (!zdev)
-				return -ENOMEM;
-			if (ap_dev->device_type == AP_DEVICE_TYPE_CEX4) {
-				zdev->type_string = "CEX4P";
-				zdev->speed_rating = CEX4P_SPEED_RATING;
-			} else {
-				zdev->type_string = "CEX5P";
-				zdev->speed_rating = CEX5P_SPEED_RATING;
-			}
-			zdev->user_space_type = ZCRYPT_CEX4;
-			zdev->min_mod_size = CEX4C_MIN_MOD_SIZE;
-			zdev->max_mod_size = CEX4C_MAX_MOD_SIZE;
-			zdev->max_exp_bit_length = CEX4C_MAX_MOD_SIZE;
-			zdev->short_crt = 0;
-			zdev->ops = zcrypt_msgtype_request(MSGTYPE06_NAME,
-							MSGTYPE06_VARIANT_EP11);
+	zc = zcrypt_card_alloc();
+	if (!zc)
+		return -ENOMEM;
+	zc->card = ac;
+	ac->private = zc;
+	if (ap_test_bit(&ac->functions, AP_FUNC_ACCEL)) {
+		if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX4) {
+			zc->type_string = "CEX4A";
+			zc->user_space_type = ZCRYPT_CEX4;
+			memcpy(zc->speed_rating, CEX4A_SPEED_IDX,
+			       sizeof(CEX4A_SPEED_IDX));
+		} else {
+			zc->type_string = "CEX5A";
+			zc->user_space_type = ZCRYPT_CEX5;
+			memcpy(zc->speed_rating, CEX5A_SPEED_IDX,
+			       sizeof(CEX5A_SPEED_IDX));
 		}
-		break;
-	}
-	if (!zdev)
+		zc->min_mod_size = CEX4A_MIN_MOD_SIZE;
+		if (ap_test_bit(&ac->functions, AP_FUNC_MEX4K) &&
+		    ap_test_bit(&ac->functions, AP_FUNC_CRT4K)) {
+			zc->max_mod_size = CEX4A_MAX_MOD_SIZE_4K;
+			zc->max_exp_bit_length =
+				CEX4A_MAX_MOD_SIZE_4K;
+		} else {
+			zc->max_mod_size = CEX4A_MAX_MOD_SIZE_2K;
+			zc->max_exp_bit_length =
+				CEX4A_MAX_MOD_SIZE_2K;
+		}
+	} else if (ap_test_bit(&ac->functions, AP_FUNC_COPRO)) {
+		if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX4) {
+			zc->type_string = "CEX4C";
+			/* wrong user space type, must be CEX4
+			 * just keep it for cca compatibility
+			 */
+			zc->user_space_type = ZCRYPT_CEX3C;
+			memcpy(zc->speed_rating, CEX4C_SPEED_IDX,
+			       sizeof(CEX4C_SPEED_IDX));
+		} else {
+			zc->type_string = "CEX5C";
+			/* wrong user space type, must be CEX5
+			 * just keep it for cca compatibility
+			 */
+			zc->user_space_type = ZCRYPT_CEX3C;
+			memcpy(zc->speed_rating, CEX5C_SPEED_IDX,
+			       sizeof(CEX5C_SPEED_IDX));
+		}
+		zc->min_mod_size = CEX4C_MIN_MOD_SIZE;
+		zc->max_mod_size = CEX4C_MAX_MOD_SIZE;
+		zc->max_exp_bit_length = CEX4C_MAX_MOD_SIZE;
+	} else if (ap_test_bit(&ac->functions, AP_FUNC_EP11)) {
+		if (ac->ap_dev.device_type == AP_DEVICE_TYPE_CEX4) {
+			zc->type_string = "CEX4P";
+			zc->user_space_type = ZCRYPT_CEX4;
+			memcpy(zc->speed_rating, CEX4P_SPEED_IDX,
+			       sizeof(CEX4P_SPEED_IDX));
+		} else {
+			zc->type_string = "CEX5P";
+			zc->user_space_type = ZCRYPT_CEX5;
+			memcpy(zc->speed_rating, CEX5P_SPEED_IDX,
+			       sizeof(CEX5P_SPEED_IDX));
+		}
+		zc->min_mod_size = CEX4C_MIN_MOD_SIZE;
+		zc->max_mod_size = CEX4C_MAX_MOD_SIZE;
+		zc->max_exp_bit_length = CEX4C_MAX_MOD_SIZE;
+	} else {
+		zcrypt_card_free(zc);
 		return -ENODEV;
-	zdev->ap_dev = ap_dev;
-	zdev->online = 1;
-	ap_device_init_reply(ap_dev, &zdev->reply);
-	ap_dev->private = zdev;
-	rc = zcrypt_device_register(zdev);
+	}
+	zc->online = 1;
+
+	rc = zcrypt_card_register(zc);
 	if (rc) {
-		zcrypt_msgtype_release(zdev->ops);
-		ap_dev->private = NULL;
-		zcrypt_device_free(zdev);
+		ac->private = NULL;
+		zcrypt_card_free(zc);
 	}
+
 	return rc;
 }
 
 /**
- * This is called to remove the extended CEX4 driver information
- * if an AP device is removed.
+ * This is called to remove the CEX4 card driver information
+ * if an AP card device is removed.
+ */
+static void zcrypt_cex4_card_remove(struct ap_device *ap_dev)
+{
+	struct zcrypt_card *zc = to_ap_card(&ap_dev->device)->private;
+
+	if (zc)
+		zcrypt_card_unregister(zc);
+}
+
+static struct ap_driver zcrypt_cex4_card_driver = {
+	.probe = zcrypt_cex4_card_probe,
+	.remove = zcrypt_cex4_card_remove,
+	.ids = zcrypt_cex4_card_ids,
+};
+
+/**
+ * Probe function for CEX4 queue device. It always accepts the AP device
+ * since the bus_match already checked the hardware type.
+ * @ap_dev: pointer to the AP device.
  */
-static void zcrypt_cex4_remove(struct ap_device *ap_dev)
+static int zcrypt_cex4_queue_probe(struct ap_device *ap_dev)
 {
-	struct zcrypt_device *zdev = ap_dev->private;
-	struct zcrypt_ops *zops;
+	struct ap_queue *aq = to_ap_queue(&ap_dev->device);
+	struct zcrypt_queue *zq;
+	int rc;
 
-	if (zdev) {
-		zops = zdev->ops;
-		zcrypt_device_unregister(zdev);
-		zcrypt_msgtype_release(zops);
+	if (ap_test_bit(&aq->card->functions, AP_FUNC_ACCEL)) {
+		zq = zcrypt_queue_alloc(CEX4A_MAX_MESSAGE_SIZE);
+		if (!zq)
+			return -ENOMEM;
+		zq->ops = zcrypt_msgtype(MSGTYPE50_NAME,
+					 MSGTYPE50_VARIANT_DEFAULT);
+	} else if (ap_test_bit(&aq->card->functions, AP_FUNC_COPRO)) {
+		zq = zcrypt_queue_alloc(CEX4C_MAX_MESSAGE_SIZE);
+		if (!zq)
+			return -ENOMEM;
+		zq->ops = zcrypt_msgtype(MSGTYPE06_NAME,
+					 MSGTYPE06_VARIANT_DEFAULT);
+	} else if (ap_test_bit(&aq->card->functions, AP_FUNC_EP11)) {
+		zq = zcrypt_queue_alloc(CEX4C_MAX_MESSAGE_SIZE);
+		if (!zq)
+			return -ENOMEM;
+		zq->ops = zcrypt_msgtype(MSGTYPE06_NAME,
+					 MSGTYPE06_VARIANT_EP11);
+	} else {
+		return -ENODEV;
 	}
+	zq->queue = aq;
+	zq->online = 1;
+	atomic_set(&zq->load, 0);
+	ap_queue_init_reply(aq, &zq->reply);
+	aq->request_timeout = CEX4_CLEANUP_TIME,
+	aq->private = zq;
+	rc = zcrypt_queue_register(zq);
+	if (rc) {
+		aq->private = NULL;
+		zcrypt_queue_free(zq);
+	}
+
+	return rc;
+}
+
+/**
+ * This is called to remove the CEX4 queue driver information
+ * if an AP queue device is removed.
+ */
+static void zcrypt_cex4_queue_remove(struct ap_device *ap_dev)
+{
+	struct ap_queue *aq = to_ap_queue(&ap_dev->device);
+	struct zcrypt_queue *zq = aq->private;
+
+	ap_queue_remove(aq);
+	if (zq)
+		zcrypt_queue_unregister(zq);
 }
 
+static struct ap_driver zcrypt_cex4_queue_driver = {
+	.probe = zcrypt_cex4_queue_probe,
+	.remove = zcrypt_cex4_queue_remove,
+	.suspend = ap_queue_suspend,
+	.resume = ap_queue_resume,
+	.ids = zcrypt_cex4_queue_ids,
+};
+
 int __init zcrypt_cex4_init(void)
 {
-	return ap_driver_register(&zcrypt_cex4_driver, THIS_MODULE, "cex4");
+	int rc;
+
+	rc = ap_driver_register(&zcrypt_cex4_card_driver,
+				THIS_MODULE, "cex4card");
+	if (rc)
+		return rc;
+
+	rc = ap_driver_register(&zcrypt_cex4_queue_driver,
+				THIS_MODULE, "cex4queue");
+	if (rc)
+		ap_driver_unregister(&zcrypt_cex4_card_driver);
+
+	return rc;
 }
 
 void __exit zcrypt_cex4_exit(void)
 {
-	ap_driver_unregister(&zcrypt_cex4_driver);
+	ap_driver_unregister(&zcrypt_cex4_queue_driver);
+	ap_driver_unregister(&zcrypt_cex4_card_driver);
 }
 
 module_init(zcrypt_cex4_init);
diff --git a/drivers/s390/crypto/zcrypt_debug.h b/drivers/s390/crypto/zcrypt_debug.h
index 28d9349de1ad..13e38defb6b8 100644
--- a/drivers/s390/crypto/zcrypt_debug.h
+++ b/drivers/s390/crypto/zcrypt_debug.h
@@ -1,51 +1,27 @@
 /*
- *  Copyright IBM Corp. 2012
+ *  Copyright IBM Corp. 2016
  *  Author(s): Holger Dengler (hd@linux.vnet.ibm.com)
+ *	       Harald Freudenberger <freude@de.ibm.com>
  */
 #ifndef ZCRYPT_DEBUG_H
 #define ZCRYPT_DEBUG_H
 
 #include <asm/debug.h>
-#include "zcrypt_api.h"
 
-/* that gives us 15 characters in the text event views */
-#define ZCRYPT_DBF_LEN	16
-
-#define DBF_ERR		3	/* error conditions	*/
-#define DBF_WARN	4	/* warning conditions	*/
-#define DBF_INFO	6	/* informational	*/
+#define DBF_ERR		3	/* error conditions   */
+#define DBF_WARN	4	/* warning conditions */
+#define DBF_INFO	5	/* informational      */
+#define DBF_DEBUG	6	/* for debugging only */
 
+#define RC2ERR(rc) ((rc) ? DBF_ERR : DBF_INFO)
 #define RC2WARN(rc) ((rc) ? DBF_WARN : DBF_INFO)
 
-#define ZCRYPT_DBF_COMMON(level, text...) \
-	do { \
-		if (debug_level_enabled(zcrypt_dbf_common, level)) { \
-			char debug_buffer[ZCRYPT_DBF_LEN]; \
-			snprintf(debug_buffer, ZCRYPT_DBF_LEN, text); \
-			debug_text_event(zcrypt_dbf_common, level, \
-					 debug_buffer); \
-		} \
-	} while (0)
-
-#define ZCRYPT_DBF_DEVICES(level, text...) \
-	do { \
-		if (debug_level_enabled(zcrypt_dbf_devices, level)) { \
-			char debug_buffer[ZCRYPT_DBF_LEN]; \
-			snprintf(debug_buffer, ZCRYPT_DBF_LEN, text); \
-			debug_text_event(zcrypt_dbf_devices, level, \
-					 debug_buffer); \
-		} \
-	} while (0)
-
-#define ZCRYPT_DBF_DEV(level, device, text...) \
-	do { \
-		if (debug_level_enabled(device->dbf_area, level)) { \
-			char debug_buffer[ZCRYPT_DBF_LEN]; \
-			snprintf(debug_buffer, ZCRYPT_DBF_LEN, text); \
-			debug_text_event(device->dbf_area, level, \
-					 debug_buffer); \
-		} \
-	} while (0)
+#define DBF_MAX_SPRINTF_ARGS 5
+
+#define ZCRYPT_DBF(...)					\
+	debug_sprintf_event(zcrypt_dbf_info, ##__VA_ARGS__)
+
+extern debug_info_t *zcrypt_dbf_info;
 
 int zcrypt_debug_init(void);
 void zcrypt_debug_exit(void);
diff --git a/drivers/s390/crypto/zcrypt_error.h b/drivers/s390/crypto/zcrypt_error.h
index de1b6c1d172c..13df60209ed3 100644
--- a/drivers/s390/crypto/zcrypt_error.h
+++ b/drivers/s390/crypto/zcrypt_error.h
@@ -55,52 +55,61 @@ struct error_hdr {
 #define TYPE82_RSP_CODE 0x82
 #define TYPE88_RSP_CODE 0x88
 
-#define REP82_ERROR_MACHINE_FAILURE  0x10
-#define REP82_ERROR_PREEMPT_FAILURE  0x12
-#define REP82_ERROR_CHECKPT_FAILURE  0x14
-#define REP82_ERROR_MESSAGE_TYPE     0x20
-#define REP82_ERROR_INVALID_COMM_CD  0x21	/* Type 84	*/
-#define REP82_ERROR_INVALID_MSG_LEN  0x23
-#define REP82_ERROR_RESERVD_FIELD    0x24	/* was 0x50	*/
-#define REP82_ERROR_FORMAT_FIELD     0x29
-#define REP82_ERROR_INVALID_COMMAND  0x30
-#define REP82_ERROR_MALFORMED_MSG    0x40
-#define REP82_ERROR_RESERVED_FIELDO  0x50	/* old value	*/
-#define REP82_ERROR_WORD_ALIGNMENT   0x60
-#define REP82_ERROR_MESSAGE_LENGTH   0x80
-#define REP82_ERROR_OPERAND_INVALID  0x82
-#define REP82_ERROR_OPERAND_SIZE     0x84
-#define REP82_ERROR_EVEN_MOD_IN_OPND 0x85
-#define REP82_ERROR_RESERVED_FIELD   0x88
-#define REP82_ERROR_TRANSPORT_FAIL   0x90
-#define REP82_ERROR_PACKET_TRUNCATED 0xA0
-#define REP82_ERROR_ZERO_BUFFER_LEN  0xB0
+#define REP82_ERROR_MACHINE_FAILURE	    0x10
+#define REP82_ERROR_PREEMPT_FAILURE	    0x12
+#define REP82_ERROR_CHECKPT_FAILURE	    0x14
+#define REP82_ERROR_MESSAGE_TYPE	    0x20
+#define REP82_ERROR_INVALID_COMM_CD	    0x21 /* Type 84	*/
+#define REP82_ERROR_INVALID_MSG_LEN	    0x23
+#define REP82_ERROR_RESERVD_FIELD	    0x24 /* was 0x50	*/
+#define REP82_ERROR_FORMAT_FIELD	    0x29
+#define REP82_ERROR_INVALID_COMMAND	    0x30
+#define REP82_ERROR_MALFORMED_MSG	    0x40
+#define REP82_ERROR_INVALID_DOMAIN_PRECHECK 0x42
+#define REP82_ERROR_RESERVED_FIELDO	    0x50 /* old value	*/
+#define REP82_ERROR_WORD_ALIGNMENT	    0x60
+#define REP82_ERROR_MESSAGE_LENGTH	    0x80
+#define REP82_ERROR_OPERAND_INVALID	    0x82
+#define REP82_ERROR_OPERAND_SIZE	    0x84
+#define REP82_ERROR_EVEN_MOD_IN_OPND	    0x85
+#define REP82_ERROR_RESERVED_FIELD	    0x88
+#define REP82_ERROR_INVALID_DOMAIN_PENDING  0x8A
+#define REP82_ERROR_TRANSPORT_FAIL	    0x90
+#define REP82_ERROR_PACKET_TRUNCATED	    0xA0
+#define REP82_ERROR_ZERO_BUFFER_LEN	    0xB0
 
-#define REP88_ERROR_MODULE_FAILURE   0x10
+#define REP88_ERROR_MODULE_FAILURE	    0x10
 
-#define REP88_ERROR_MESSAGE_TYPE     0x20
-#define REP88_ERROR_MESSAGE_MALFORMD 0x22
-#define REP88_ERROR_MESSAGE_LENGTH   0x23
-#define REP88_ERROR_RESERVED_FIELD   0x24
-#define REP88_ERROR_KEY_TYPE	     0x34
-#define REP88_ERROR_INVALID_KEY      0x82	/* CEX2A	*/
-#define REP88_ERROR_OPERAND	     0x84	/* CEX2A	*/
-#define REP88_ERROR_OPERAND_EVEN_MOD 0x85	/* CEX2A	*/
+#define REP88_ERROR_MESSAGE_TYPE	    0x20
+#define REP88_ERROR_MESSAGE_MALFORMD	    0x22
+#define REP88_ERROR_MESSAGE_LENGTH	    0x23
+#define REP88_ERROR_RESERVED_FIELD	    0x24
+#define REP88_ERROR_KEY_TYPE		    0x34
+#define REP88_ERROR_INVALID_KEY	    0x82 /* CEX2A	*/
+#define REP88_ERROR_OPERAND		    0x84 /* CEX2A	*/
+#define REP88_ERROR_OPERAND_EVEN_MOD	    0x85 /* CEX2A	*/
 
-static inline int convert_error(struct zcrypt_device *zdev,
+static inline int convert_error(struct zcrypt_queue *zq,
 				struct ap_message *reply)
 {
 	struct error_hdr *ehdr = reply->message;
+	int card = AP_QID_CARD(zq->queue->qid);
+	int queue = AP_QID_QUEUE(zq->queue->qid);
 
 	switch (ehdr->reply_code) {
 	case REP82_ERROR_OPERAND_INVALID:
 	case REP82_ERROR_OPERAND_SIZE:
 	case REP82_ERROR_EVEN_MOD_IN_OPND:
 	case REP88_ERROR_MESSAGE_MALFORMD:
+	case REP82_ERROR_INVALID_DOMAIN_PRECHECK:
+	case REP82_ERROR_INVALID_DOMAIN_PENDING:
 	//   REP88_ERROR_INVALID_KEY		// '82' CEX2A
 	//   REP88_ERROR_OPERAND		// '84' CEX2A
 	//   REP88_ERROR_OPERAND_EVEN_MOD	// '85' CEX2A
 		/* Invalid input data. */
+		ZCRYPT_DBF(DBF_WARN,
+			   "device=%02x.%04x reply=0x%02x => rc=EINVAL\n",
+			   card, queue, ehdr->reply_code);
 		return -EINVAL;
 	case REP82_ERROR_MESSAGE_TYPE:
 	//   REP88_ERROR_MESSAGE_TYPE		// '20' CEX2A
@@ -110,32 +119,32 @@ static inline int convert_error(struct zcrypt_device *zdev,
 		 * and then repeat the request.
 		 */
 		atomic_set(&zcrypt_rescan_req, 1);
-		zdev->online = 0;
-		pr_err("Cryptographic device %x failed and was set offline\n",
-		       AP_QID_DEVICE(zdev->ap_dev->qid));
-		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%drc%d",
-			AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online,
-			ehdr->reply_code);
+		zq->online = 0;
+		pr_err("Cryptographic device %02x.%04x failed and was set offline\n",
+		       card, queue);
+		ZCRYPT_DBF(DBF_ERR,
+			   "device=%02x.%04x reply=0x%02x => online=0 rc=EAGAIN\n",
+			   card, queue, ehdr->reply_code);
 		return -EAGAIN;
 	case REP82_ERROR_TRANSPORT_FAIL:
 	case REP82_ERROR_MACHINE_FAILURE:
 	//   REP88_ERROR_MODULE_FAILURE		// '10' CEX2A
 		/* If a card fails disable it and repeat the request. */
 		atomic_set(&zcrypt_rescan_req, 1);
-		zdev->online = 0;
-		pr_err("Cryptographic device %x failed and was set offline\n",
-		       AP_QID_DEVICE(zdev->ap_dev->qid));
-		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%drc%d",
-			AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online,
-			ehdr->reply_code);
+		zq->online = 0;
+		pr_err("Cryptographic device %02x.%04x failed and was set offline\n",
+		       card, queue);
+		ZCRYPT_DBF(DBF_ERR,
+			   "device=%02x.%04x reply=0x%02x => online=0 rc=EAGAIN\n",
+			   card, queue, ehdr->reply_code);
 		return -EAGAIN;
 	default:
-		zdev->online = 0;
-		pr_err("Cryptographic device %x failed and was set offline\n",
-		       AP_QID_DEVICE(zdev->ap_dev->qid));
-		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%drc%d",
-			AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online,
-			ehdr->reply_code);
+		zq->online = 0;
+		pr_err("Cryptographic device %02x.%04x failed and was set offline\n",
+		       card, queue);
+		ZCRYPT_DBF(DBF_ERR,
+			   "device=%02x.%04x reply=0x%02x => online=0 rc=EAGAIN\n",
+			   card, queue, ehdr->reply_code);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
 }
diff --git a/drivers/s390/crypto/zcrypt_msgtype50.c b/drivers/s390/crypto/zcrypt_msgtype50.c
index eedfaa2cf715..6dd5d7c58dd0 100644
--- a/drivers/s390/crypto/zcrypt_msgtype50.c
+++ b/drivers/s390/crypto/zcrypt_msgtype50.c
@@ -53,9 +53,6 @@ MODULE_DESCRIPTION("Cryptographic Accelerator (message type 50), " \
 		   "Copyright IBM Corp. 2001, 2012");
 MODULE_LICENSE("GPL");
 
-static void zcrypt_cex2a_receive(struct ap_device *, struct ap_message *,
-				 struct ap_message *);
-
 /**
  * The type 50 message family is associated with a CEX2A card.
  *
@@ -173,16 +170,48 @@ struct type80_hdr {
 	unsigned char	reserved3[8];
 } __packed;
 
+unsigned int get_rsa_modex_fc(struct ica_rsa_modexpo *mex, int *fcode)
+{
+
+	if (!mex->inputdatalength)
+		return -EINVAL;
+
+	if (mex->inputdatalength <= 128)	/* 1024 bit */
+		*fcode = MEX_1K;
+	else if (mex->inputdatalength <= 256)	/* 2048 bit */
+		*fcode = MEX_2K;
+	else					/* 4096 bit */
+		*fcode = MEX_4K;
+
+	return 0;
+}
+
+unsigned int get_rsa_crt_fc(struct ica_rsa_modexpo_crt *crt, int *fcode)
+{
+
+	if (!crt->inputdatalength)
+		return -EINVAL;
+
+	if (crt->inputdatalength <= 128)	/* 1024 bit */
+		*fcode = CRT_1K;
+	else if (crt->inputdatalength <= 256)	/* 2048 bit */
+		*fcode = CRT_2K;
+	else					/* 4096 bit */
+		*fcode = CRT_4K;
+
+	return 0;
+}
+
 /**
  * Convert a ICAMEX message to a type50 MEX message.
  *
- * @zdev: crypto device pointer
- * @zreq: crypto request pointer
+ * @zq: crypto queue pointer
+ * @ap_msg: crypto request pointer
  * @mex: pointer to user input data
  *
  * Returns 0 on success or -EFAULT.
  */
-static int ICAMEX_msg_to_type50MEX_msg(struct zcrypt_device *zdev,
+static int ICAMEX_msg_to_type50MEX_msg(struct zcrypt_queue *zq,
 				       struct ap_message *ap_msg,
 				       struct ica_rsa_modexpo *mex)
 {
@@ -234,13 +263,13 @@ static int ICAMEX_msg_to_type50MEX_msg(struct zcrypt_device *zdev,
 /**
  * Convert a ICACRT message to a type50 CRT message.
  *
- * @zdev: crypto device pointer
- * @zreq: crypto request pointer
+ * @zq: crypto queue pointer
+ * @ap_msg: crypto request pointer
  * @crt: pointer to user input data
  *
  * Returns 0 on success or -EFAULT.
  */
-static int ICACRT_msg_to_type50CRT_msg(struct zcrypt_device *zdev,
+static int ICACRT_msg_to_type50CRT_msg(struct zcrypt_queue *zq,
 				       struct ap_message *ap_msg,
 				       struct ica_rsa_modexpo_crt *crt)
 {
@@ -283,7 +312,7 @@ static int ICACRT_msg_to_type50CRT_msg(struct zcrypt_device *zdev,
 		u = crb2->u + sizeof(crb2->u) - short_len;
 		inp = crb2->message + sizeof(crb2->message) - mod_len;
 	} else if ((mod_len <= 512) &&	/* up to 4096 bit key size */
-		   (zdev->max_mod_size == CEX3A_MAX_MOD_SIZE)) { /* >= CEX3A */
+		   (zq->zcard->max_mod_size == CEX3A_MAX_MOD_SIZE)) {
 		struct type50_crb3_msg *crb3 = ap_msg->message;
 		memset(crb3, 0, sizeof(*crb3));
 		ap_msg->length = sizeof(*crb3);
@@ -317,14 +346,14 @@ static int ICACRT_msg_to_type50CRT_msg(struct zcrypt_device *zdev,
 /**
  * Copy results from a type 80 reply message back to user space.
  *
- * @zdev: crypto device pointer
+ * @zq: crypto device pointer
  * @reply: reply AP message.
  * @data: pointer to user output data
  * @length: size of user output data
  *
  * Returns 0 on success or -EFAULT.
  */
-static int convert_type80(struct zcrypt_device *zdev,
+static int convert_type80(struct zcrypt_queue *zq,
 			  struct ap_message *reply,
 			  char __user *outputdata,
 			  unsigned int outputdatalength)
@@ -334,16 +363,18 @@ static int convert_type80(struct zcrypt_device *zdev,
 
 	if (t80h->len < sizeof(*t80h) + outputdatalength) {
 		/* The result is too short, the CEX2A card may not do that.. */
-		zdev->online = 0;
-		pr_err("Cryptographic device %x failed and was set offline\n",
-		       AP_QID_DEVICE(zdev->ap_dev->qid));
-		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%drc%d",
-			       AP_QID_DEVICE(zdev->ap_dev->qid),
-			       zdev->online, t80h->code);
-
+		zq->online = 0;
+		pr_err("Cryptographic device %02x.%04x failed and was set offline\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid));
+		ZCRYPT_DBF(DBF_ERR,
+			   "device=%02x.%04x code=0x%02x => online=0 rc=EAGAIN\n",
+			   AP_QID_CARD(zq->queue->qid),
+			   AP_QID_QUEUE(zq->queue->qid),
+			   t80h->code);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
-	if (zdev->user_space_type == ZCRYPT_CEX2A)
+	if (zq->zcard->user_space_type == ZCRYPT_CEX2A)
 		BUG_ON(t80h->len > CEX2A_MAX_RESPONSE_SIZE);
 	else
 		BUG_ON(t80h->len > CEX3A_MAX_RESPONSE_SIZE);
@@ -353,25 +384,31 @@ static int convert_type80(struct zcrypt_device *zdev,
 	return 0;
 }
 
-static int convert_response(struct zcrypt_device *zdev,
+static int convert_response(struct zcrypt_queue *zq,
 			    struct ap_message *reply,
 			    char __user *outputdata,
 			    unsigned int outputdatalength)
 {
 	/* Response type byte is the second byte in the response. */
-	switch (((unsigned char *) reply->message)[1]) {
+	unsigned char rtype = ((unsigned char *) reply->message)[1];
+
+	switch (rtype) {
 	case TYPE82_RSP_CODE:
 	case TYPE88_RSP_CODE:
-		return convert_error(zdev, reply);
+		return convert_error(zq, reply);
 	case TYPE80_RSP_CODE:
-		return convert_type80(zdev, reply,
+		return convert_type80(zq, reply,
 				      outputdata, outputdatalength);
 	default: /* Unknown response type, this should NEVER EVER happen */
-		zdev->online = 0;
-		pr_err("Cryptographic device %x failed and was set offline\n",
-		       AP_QID_DEVICE(zdev->ap_dev->qid));
-		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%dfail",
-			       AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online);
+		zq->online = 0;
+		pr_err("Cryptographic device %02x.%04x failed and was set offline\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid));
+		ZCRYPT_DBF(DBF_ERR,
+			   "device=%02x.%04x rtype=0x%02x => online=0 rc=EAGAIN\n",
+			   AP_QID_CARD(zq->queue->qid),
+			   AP_QID_QUEUE(zq->queue->qid),
+			   (unsigned int) rtype);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
 }
@@ -380,11 +417,11 @@ static int convert_response(struct zcrypt_device *zdev,
  * This function is called from the AP bus code after a crypto request
  * "msg" has finished with the reply message "reply".
  * It is called from tasklet context.
- * @ap_dev: pointer to the AP device
+ * @aq: pointer to the AP device
  * @msg: pointer to the AP message
  * @reply: pointer to the AP reply message
  */
-static void zcrypt_cex2a_receive(struct ap_device *ap_dev,
+static void zcrypt_cex2a_receive(struct ap_queue *aq,
 				 struct ap_message *msg,
 				 struct ap_message *reply)
 {
@@ -400,7 +437,7 @@ static void zcrypt_cex2a_receive(struct ap_device *ap_dev,
 		goto out;	/* ap_msg->rc indicates the error */
 	t80h = reply->message;
 	if (t80h->type == TYPE80_RSP_CODE) {
-		if (ap_dev->device_type == AP_DEVICE_TYPE_CEX2A)
+		if (aq->ap_dev.device_type == AP_DEVICE_TYPE_CEX2A)
 			length = min_t(int,
 				       CEX2A_MAX_RESPONSE_SIZE, t80h->len);
 		else
@@ -418,11 +455,11 @@ static atomic_t zcrypt_step = ATOMIC_INIT(0);
 /**
  * The request distributor calls this function if it picked the CEX2A
  * device to handle a modexpo request.
- * @zdev: pointer to zcrypt_device structure that identifies the
+ * @zq: pointer to zcrypt_queue structure that identifies the
  *	  CEX2A device to the request distributor
  * @mex: pointer to the modexpo request buffer
  */
-static long zcrypt_cex2a_modexpo(struct zcrypt_device *zdev,
+static long zcrypt_cex2a_modexpo(struct zcrypt_queue *zq,
 				 struct ica_rsa_modexpo *mex)
 {
 	struct ap_message ap_msg;
@@ -430,7 +467,7 @@ static long zcrypt_cex2a_modexpo(struct zcrypt_device *zdev,
 	int rc;
 
 	ap_init_message(&ap_msg);
-	if (zdev->user_space_type == ZCRYPT_CEX2A)
+	if (zq->zcard->user_space_type == ZCRYPT_CEX2A)
 		ap_msg.message = kmalloc(MSGTYPE50_CRB2_MAX_MSG_SIZE,
 					 GFP_KERNEL);
 	else
@@ -442,20 +479,20 @@ static long zcrypt_cex2a_modexpo(struct zcrypt_device *zdev,
 	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
 				atomic_inc_return(&zcrypt_step);
 	ap_msg.private = &work;
-	rc = ICAMEX_msg_to_type50MEX_msg(zdev, &ap_msg, mex);
+	rc = ICAMEX_msg_to_type50MEX_msg(zq, &ap_msg, mex);
 	if (rc)
 		goto out_free;
 	init_completion(&work);
-	ap_queue_message(zdev->ap_dev, &ap_msg);
+	ap_queue_message(zq->queue, &ap_msg);
 	rc = wait_for_completion_interruptible(&work);
 	if (rc == 0) {
 		rc = ap_msg.rc;
 		if (rc == 0)
-			rc = convert_response(zdev, &ap_msg, mex->outputdata,
+			rc = convert_response(zq, &ap_msg, mex->outputdata,
 					      mex->outputdatalength);
 	} else
 		/* Signal pending. */
-		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		ap_cancel_message(zq->queue, &ap_msg);
 out_free:
 	kfree(ap_msg.message);
 	return rc;
@@ -464,11 +501,11 @@ out_free:
 /**
  * The request distributor calls this function if it picked the CEX2A
  * device to handle a modexpo_crt request.
- * @zdev: pointer to zcrypt_device structure that identifies the
+ * @zq: pointer to zcrypt_queue structure that identifies the
  *	  CEX2A device to the request distributor
  * @crt: pointer to the modexpoc_crt request buffer
  */
-static long zcrypt_cex2a_modexpo_crt(struct zcrypt_device *zdev,
+static long zcrypt_cex2a_modexpo_crt(struct zcrypt_queue *zq,
 				     struct ica_rsa_modexpo_crt *crt)
 {
 	struct ap_message ap_msg;
@@ -476,7 +513,7 @@ static long zcrypt_cex2a_modexpo_crt(struct zcrypt_device *zdev,
 	int rc;
 
 	ap_init_message(&ap_msg);
-	if (zdev->user_space_type == ZCRYPT_CEX2A)
+	if (zq->zcard->user_space_type == ZCRYPT_CEX2A)
 		ap_msg.message = kmalloc(MSGTYPE50_CRB2_MAX_MSG_SIZE,
 					 GFP_KERNEL);
 	else
@@ -488,20 +525,20 @@ static long zcrypt_cex2a_modexpo_crt(struct zcrypt_device *zdev,
 	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
 				atomic_inc_return(&zcrypt_step);
 	ap_msg.private = &work;
-	rc = ICACRT_msg_to_type50CRT_msg(zdev, &ap_msg, crt);
+	rc = ICACRT_msg_to_type50CRT_msg(zq, &ap_msg, crt);
 	if (rc)
 		goto out_free;
 	init_completion(&work);
-	ap_queue_message(zdev->ap_dev, &ap_msg);
+	ap_queue_message(zq->queue, &ap_msg);
 	rc = wait_for_completion_interruptible(&work);
 	if (rc == 0) {
 		rc = ap_msg.rc;
 		if (rc == 0)
-			rc = convert_response(zdev, &ap_msg, crt->outputdata,
+			rc = convert_response(zq, &ap_msg, crt->outputdata,
 					      crt->outputdatalength);
 	} else
 		/* Signal pending. */
-		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		ap_cancel_message(zq->queue, &ap_msg);
 out_free:
 	kfree(ap_msg.message);
 	return rc;
@@ -518,16 +555,12 @@ static struct zcrypt_ops zcrypt_msgtype50_ops = {
 	.variant = MSGTYPE50_VARIANT_DEFAULT,
 };
 
-int __init zcrypt_msgtype50_init(void)
+void __init zcrypt_msgtype50_init(void)
 {
 	zcrypt_msgtype_register(&zcrypt_msgtype50_ops);
-	return 0;
 }
 
 void __exit zcrypt_msgtype50_exit(void)
 {
 	zcrypt_msgtype_unregister(&zcrypt_msgtype50_ops);
 }
-
-module_init(zcrypt_msgtype50_init);
-module_exit(zcrypt_msgtype50_exit);
diff --git a/drivers/s390/crypto/zcrypt_msgtype50.h b/drivers/s390/crypto/zcrypt_msgtype50.h
index 0a66e4aeeb50..5cc280318ee7 100644
--- a/drivers/s390/crypto/zcrypt_msgtype50.h
+++ b/drivers/s390/crypto/zcrypt_msgtype50.h
@@ -35,7 +35,10 @@
 
 #define MSGTYPE_ADJUSTMENT		0x08  /*type04 extension (not needed in type50)*/
 
-int zcrypt_msgtype50_init(void);
+unsigned int get_rsa_modex_fc(struct ica_rsa_modexpo *, int *);
+unsigned int get_rsa_crt_fc(struct ica_rsa_modexpo_crt *, int *);
+
+void zcrypt_msgtype50_init(void);
 void zcrypt_msgtype50_exit(void);
 
 #endif /* _ZCRYPT_MSGTYPE50_H_ */
diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c
index 21959719daef..e5563ffeb839 100644
--- a/drivers/s390/crypto/zcrypt_msgtype6.c
+++ b/drivers/s390/crypto/zcrypt_msgtype6.c
@@ -60,9 +60,6 @@ MODULE_DESCRIPTION("Cryptographic Coprocessor (message type 6), " \
 		   "Copyright IBM Corp. 2001, 2012");
 MODULE_LICENSE("GPL");
 
-static void zcrypt_msgtype6_receive(struct ap_device *, struct ap_message *,
-				 struct ap_message *);
-
 /**
  * CPRB
  *	  Note that all shorts, ints and longs are little-endian.
@@ -149,16 +146,122 @@ static struct CPRBX static_cprbx = {
 	.func_id	= {0x54, 0x32},
 };
 
+int speed_idx_cca(int req_type)
+{
+	switch (req_type) {
+	case 0x4142:
+	case 0x4149:
+	case 0x414D:
+	case 0x4341:
+	case 0x4344:
+	case 0x4354:
+	case 0x4358:
+	case 0x444B:
+	case 0x4558:
+	case 0x4643:
+	case 0x4651:
+	case 0x4C47:
+	case 0x4C4B:
+	case 0x4C51:
+	case 0x4F48:
+	case 0x504F:
+	case 0x5053:
+	case 0x5058:
+	case 0x5343:
+	case 0x5344:
+	case 0x5345:
+	case 0x5350:
+		return LOW;
+	case 0x414B:
+	case 0x4345:
+	case 0x4349:
+	case 0x434D:
+	case 0x4847:
+	case 0x4849:
+	case 0x484D:
+	case 0x4850:
+	case 0x4851:
+	case 0x4954:
+	case 0x4958:
+	case 0x4B43:
+	case 0x4B44:
+	case 0x4B45:
+	case 0x4B47:
+	case 0x4B48:
+	case 0x4B49:
+	case 0x4B4E:
+	case 0x4B50:
+	case 0x4B52:
+	case 0x4B54:
+	case 0x4B58:
+	case 0x4D50:
+	case 0x4D53:
+	case 0x4D56:
+	case 0x4D58:
+	case 0x5044:
+	case 0x5045:
+	case 0x5046:
+	case 0x5047:
+	case 0x5049:
+	case 0x504B:
+	case 0x504D:
+	case 0x5254:
+	case 0x5347:
+	case 0x5349:
+	case 0x534B:
+	case 0x534D:
+	case 0x5356:
+	case 0x5358:
+	case 0x5443:
+	case 0x544B:
+	case 0x5647:
+		return HIGH;
+	default:
+		return MEDIUM;
+	}
+}
+
+int speed_idx_ep11(int req_type)
+{
+	switch (req_type) {
+	case  1:
+	case  2:
+	case 36:
+	case 37:
+	case 38:
+	case 39:
+	case 40:
+		return LOW;
+	case 17:
+	case 18:
+	case 19:
+	case 20:
+	case 21:
+	case 22:
+	case 26:
+	case 30:
+	case 31:
+	case 32:
+	case 33:
+	case 34:
+	case 35:
+		return HIGH;
+	default:
+		return MEDIUM;
+	}
+}
+
+
 /**
  * Convert a ICAMEX message to a type6 MEX message.
  *
- * @zdev: crypto device pointer
+ * @zq: crypto device pointer
  * @ap_msg: pointer to AP message
  * @mex: pointer to user input data
  *
  * Returns 0 on success or -EFAULT.
  */
-static int ICAMEX_msg_to_type6MEX_msgX(struct zcrypt_device *zdev,
+static int ICAMEX_msg_to_type6MEX_msgX(struct zcrypt_queue *zq,
 				       struct ap_message *ap_msg,
 				       struct ica_rsa_modexpo *mex)
 {
@@ -173,11 +276,6 @@ static int ICAMEX_msg_to_type6MEX_msgX(struct zcrypt_device *zdev,
 		.ulen		= 10,
 		.only_rule	= {'M', 'R', 'P', ' ', ' ', ' ', ' ', ' '}
 	};
-	static struct function_and_rules_block static_pke_fnr_MCL2 = {
-		.function_code	= {'P', 'K'},
-		.ulen		= 10,
-		.only_rule	= {'Z', 'E', 'R', 'O', '-', 'P', 'A', 'D'}
-	};
 	struct {
 		struct type6_hdr hdr;
 		struct CPRBX cprbx;
@@ -204,11 +302,10 @@ static int ICAMEX_msg_to_type6MEX_msgX(struct zcrypt_device *zdev,
 	msg->hdr.FromCardLen1 = PCIXCC_MAX_ICA_RESPONSE_SIZE - sizeof(msg->hdr);
 
 	msg->cprbx = static_cprbx;
-	msg->cprbx.domain = AP_QID_QUEUE(zdev->ap_dev->qid);
+	msg->cprbx.domain = AP_QID_QUEUE(zq->queue->qid);
 	msg->cprbx.rpl_msgbl = msg->hdr.FromCardLen1;
 
-	msg->fr = (zdev->user_space_type == ZCRYPT_PCIXCC_MCL2) ?
-		static_pke_fnr_MCL2 : static_pke_fnr;
+	msg->fr = static_pke_fnr;
 
 	msg->cprbx.req_parml = size - sizeof(msg->hdr) - sizeof(msg->cprbx);
 
@@ -219,13 +316,13 @@ static int ICAMEX_msg_to_type6MEX_msgX(struct zcrypt_device *zdev,
 /**
  * Convert a ICACRT message to a type6 CRT message.
  *
- * @zdev: crypto device pointer
+ * @zq: crypto device pointer
  * @ap_msg: pointer to AP message
  * @crt: pointer to user input data
  *
  * Returns 0 on success or -EFAULT.
  */
-static int ICACRT_msg_to_type6CRT_msgX(struct zcrypt_device *zdev,
+static int ICACRT_msg_to_type6CRT_msgX(struct zcrypt_queue *zq,
 				       struct ap_message *ap_msg,
 				       struct ica_rsa_modexpo_crt *crt)
 {
@@ -241,11 +338,6 @@ static int ICACRT_msg_to_type6CRT_msgX(struct zcrypt_device *zdev,
 		.only_rule	= {'Z', 'E', 'R', 'O', '-', 'P', 'A', 'D'}
 	};
 
-	static struct function_and_rules_block static_pkd_fnr_MCL2 = {
-		.function_code	= {'P', 'D'},
-		.ulen		= 10,
-		.only_rule	= {'P', 'K', 'C', 'S', '-', '1', '.', '2'}
-	};
 	struct {
 		struct type6_hdr hdr;
 		struct CPRBX cprbx;
@@ -272,12 +364,11 @@ static int ICACRT_msg_to_type6CRT_msgX(struct zcrypt_device *zdev,
 	msg->hdr.FromCardLen1 = PCIXCC_MAX_ICA_RESPONSE_SIZE - sizeof(msg->hdr);
 
 	msg->cprbx = static_cprbx;
-	msg->cprbx.domain = AP_QID_QUEUE(zdev->ap_dev->qid);
+	msg->cprbx.domain = AP_QID_QUEUE(zq->queue->qid);
 	msg->cprbx.req_parml = msg->cprbx.rpl_msgbl =
 		size - sizeof(msg->hdr) - sizeof(msg->cprbx);
 
-	msg->fr = (zdev->user_space_type == ZCRYPT_PCIXCC_MCL2) ?
-		static_pkd_fnr_MCL2 : static_pkd_fnr;
+	msg->fr = static_pkd_fnr;
 
 	ap_msg->length = size;
 	return 0;
@@ -286,7 +377,7 @@ static int ICACRT_msg_to_type6CRT_msgX(struct zcrypt_device *zdev,
 /**
  * Convert a XCRB message to a type6 CPRB message.
  *
- * @zdev: crypto device pointer
+ * @zq: crypto device pointer
  * @ap_msg: pointer to AP message
  * @xcRB: pointer to user input data
  *
@@ -297,9 +388,10 @@ struct type86_fmt2_msg {
 	struct type86_fmt2_ext fmt2;
 } __packed;
 
-static int XCRB_msg_to_type6CPRB_msgX(struct zcrypt_device *zdev,
-				       struct ap_message *ap_msg,
-				       struct ica_xcRB *xcRB)
+static int XCRB_msg_to_type6CPRB_msgX(struct ap_message *ap_msg,
+				      struct ica_xcRB *xcRB,
+				      unsigned int *fcode,
+				      unsigned short **dom)
 {
 	static struct type6_hdr static_type6_hdrX = {
 		.type		=  0x06,
@@ -379,6 +471,9 @@ static int XCRB_msg_to_type6CPRB_msgX(struct zcrypt_device *zdev,
 	memcpy(msg->hdr.function_code, function_code,
 	       sizeof(msg->hdr.function_code));
 
+	*fcode = (msg->hdr.function_code[0] << 8) | msg->hdr.function_code[1];
+	*dom = (unsigned short *)&msg->cprbx.domain;
+
 	if (memcmp(function_code, "US", 2) == 0)
 		ap_msg->special = 1;
 	else
@@ -389,15 +484,15 @@ static int XCRB_msg_to_type6CPRB_msgX(struct zcrypt_device *zdev,
 	    copy_from_user(req_data, xcRB->request_data_address,
 		xcRB->request_data_length))
 		return -EFAULT;
+
 	return 0;
 }
 
-static int xcrb_msg_to_type6_ep11cprb_msgx(struct zcrypt_device *zdev,
-				       struct ap_message *ap_msg,
-				       struct ep11_urb *xcRB)
+static int xcrb_msg_to_type6_ep11cprb_msgx(struct ap_message *ap_msg,
+				       struct ep11_urb *xcRB,
+				       unsigned int *fcode)
 {
 	unsigned int lfmt;
-
 	static struct type6_hdr static_type6_ep11_hdr = {
 		.type		=  0x06,
 		.rqid		= {0x00, 0x01},
@@ -421,7 +516,7 @@ static int xcrb_msg_to_type6_ep11cprb_msgx(struct zcrypt_device *zdev,
 		unsigned char	dom_tag;	/* fixed value 0x4 */
 		unsigned char	dom_len;	/* fixed value 0x4 */
 		unsigned int	dom_val;	/* domain id	   */
-	} __packed * payload_hdr;
+	} __packed * payload_hdr = NULL;
 
 	if (CEIL4(xcRB->req_len) < xcRB->req_len)
 		return -EINVAL; /* overflow after alignment*/
@@ -450,43 +545,30 @@ static int xcrb_msg_to_type6_ep11cprb_msgx(struct zcrypt_device *zdev,
 		return -EFAULT;
 	}
 
-	/*
-	 The target domain field within the cprb body/payload block will be
-	 replaced by the usage domain for non-management commands only.
-	 Therefore we check the first bit of the 'flags' parameter for
-	 management command indication.
-	   0 - non management command
-	   1 - management command
-	*/
-	if (!((msg->cprbx.flags & 0x80) == 0x80)) {
-		msg->cprbx.target_id = (unsigned int)
-					AP_QID_QUEUE(zdev->ap_dev->qid);
-
-		if ((msg->pld_lenfmt & 0x80) == 0x80) { /*ext.len.fmt 2 or 3*/
-			switch (msg->pld_lenfmt & 0x03) {
-			case 1:
-				lfmt = 2;
-				break;
-			case 2:
-				lfmt = 3;
-				break;
-			default:
-				return -EINVAL;
-			}
-		} else {
-			lfmt = 1; /* length format #1 */
-		  }
-		payload_hdr = (struct pld_hdr *)((&(msg->pld_lenfmt))+lfmt);
-		payload_hdr->dom_val = (unsigned int)
-					AP_QID_QUEUE(zdev->ap_dev->qid);
+	if ((msg->pld_lenfmt & 0x80) == 0x80) { /*ext.len.fmt 2 or 3*/
+		switch (msg->pld_lenfmt & 0x03) {
+		case 1:
+			lfmt = 2;
+			break;
+		case 2:
+			lfmt = 3;
+			break;
+		default:
+			return -EINVAL;
+		}
+	} else {
+		lfmt = 1; /* length format #1 */
 	}
+	payload_hdr = (struct pld_hdr *)((&(msg->pld_lenfmt))+lfmt);
+	*fcode = payload_hdr->func_val & 0xFFFF;
+
 	return 0;
 }
 
 /**
  * Copy results from a type 86 ICA reply message back to user space.
  *
- * @zdev: crypto device pointer
+ * @zq: crypto device pointer
  * @reply: reply AP message.
  * @data: pointer to user output data
  * @length: size of user output data
@@ -508,7 +590,7 @@ struct type86_ep11_reply {
 	struct ep11_cprb cprbx;
 } __packed;
 
-static int convert_type86_ica(struct zcrypt_device *zdev,
+static int convert_type86_ica(struct zcrypt_queue *zq,
 			  struct ap_message *reply,
 			  char __user *outputdata,
 			  unsigned int outputdatalength)
@@ -556,26 +638,37 @@ static int convert_type86_ica(struct zcrypt_device *zdev,
 	service_rc = msg->cprbx.ccp_rtcode;
 	if (unlikely(service_rc != 0)) {
 		service_rs = msg->cprbx.ccp_rscode;
-		if (service_rc == 8 && service_rs == 66)
-			return -EINVAL;
-		if (service_rc == 8 && service_rs == 65)
-			return -EINVAL;
-		if (service_rc == 8 && service_rs == 770)
+		if ((service_rc == 8 && service_rs == 66) ||
+		    (service_rc == 8 && service_rs == 65) ||
+		    (service_rc == 8 && service_rs == 72) ||
+		    (service_rc == 8 && service_rs == 770) ||
+		    (service_rc == 12 && service_rs == 769)) {
+			ZCRYPT_DBF(DBF_DEBUG,
+				   "device=%02x.%04x rc/rs=%d/%d => rc=EINVAL\n",
+				   AP_QID_CARD(zq->queue->qid),
+				   AP_QID_QUEUE(zq->queue->qid),
+				   (int) service_rc, (int) service_rs);
 			return -EINVAL;
+		}
 		if (service_rc == 8 && service_rs == 783) {
-			zdev->min_mod_size = PCIXCC_MIN_MOD_SIZE_OLD;
+			zq->zcard->min_mod_size =
+				PCIXCC_MIN_MOD_SIZE_OLD;
+			ZCRYPT_DBF(DBF_DEBUG,
+				   "device=%02x.%04x rc/rs=%d/%d => rc=EAGAIN\n",
+				   AP_QID_CARD(zq->queue->qid),
+				   AP_QID_QUEUE(zq->queue->qid),
+				   (int) service_rc, (int) service_rs);
 			return -EAGAIN;
 		}
-		if (service_rc == 12 && service_rs == 769)
-			return -EINVAL;
-		if (service_rc == 8 && service_rs == 72)
-			return -EINVAL;
-		zdev->online = 0;
-		pr_err("Cryptographic device %x failed and was set offline\n",
-		       AP_QID_DEVICE(zdev->ap_dev->qid));
-		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%drc%d",
-			       AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online,
-			       msg->hdr.reply_code);
+		zq->online = 0;
+		pr_err("Cryptographic device %02x.%04x failed and was set offline\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid));
+		ZCRYPT_DBF(DBF_ERR,
+			   "device=%02x.%04x rc/rs=%d/%d => online=0 rc=EAGAIN\n",
+			   AP_QID_CARD(zq->queue->qid),
+			   AP_QID_QUEUE(zq->queue->qid),
+			   (int) service_rc, (int) service_rs);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
 	data = msg->text;
@@ -611,13 +704,13 @@ static int convert_type86_ica(struct zcrypt_device *zdev,
 /**
  * Copy results from a type 86 XCRB reply message back to user space.
  *
- * @zdev: crypto device pointer
+ * @zq: crypto device pointer
  * @reply: reply AP message.
  * @xcRB: pointer to XCRB
  *
  * Returns 0 on success or -EINVAL, -EFAULT, -EAGAIN in case of an error.
  */
-static int convert_type86_xcrb(struct zcrypt_device *zdev,
+static int convert_type86_xcrb(struct zcrypt_queue *zq,
 			       struct ap_message *reply,
 			       struct ica_xcRB *xcRB)
 {
@@ -642,13 +735,13 @@ static int convert_type86_xcrb(struct zcrypt_device *zdev,
 /**
  * Copy results from a type 86 EP11 XCRB reply message back to user space.
  *
- * @zdev: crypto device pointer
+ * @zq: crypto device pointer
  * @reply: reply AP message.
  * @xcRB: pointer to EP11 user request block
  *
  * Returns 0 on success or -EINVAL, -EFAULT, -EAGAIN in case of an error.
  */
-static int convert_type86_ep11_xcrb(struct zcrypt_device *zdev,
+static int convert_type86_ep11_xcrb(struct zcrypt_queue *zq,
 				    struct ap_message *reply,
 				    struct ep11_urb *xcRB)
 {
@@ -666,7 +759,7 @@ static int convert_type86_ep11_xcrb(struct zcrypt_device *zdev,
 	return 0;
 }
 
-static int convert_type86_rng(struct zcrypt_device *zdev,
+static int convert_type86_rng(struct zcrypt_queue *zq,
 			  struct ap_message *reply,
 			  char *buffer)
 {
@@ -683,104 +776,113 @@ static int convert_type86_rng(struct zcrypt_device *zdev,
 	return msg->fmt2.count2;
 }
 
-static int convert_response_ica(struct zcrypt_device *zdev,
+static int convert_response_ica(struct zcrypt_queue *zq,
 			    struct ap_message *reply,
 			    char __user *outputdata,
 			    unsigned int outputdatalength)
 {
 	struct type86x_reply *msg = reply->message;
 
-	/* Response type byte is the second byte in the response. */
-	switch (((unsigned char *) reply->message)[1]) {
+	switch (msg->hdr.type) {
 	case TYPE82_RSP_CODE:
 	case TYPE88_RSP_CODE:
-		return convert_error(zdev, reply);
+		return convert_error(zq, reply);
 	case TYPE86_RSP_CODE:
 		if (msg->cprbx.ccp_rtcode &&
 		   (msg->cprbx.ccp_rscode == 0x14f) &&
 		   (outputdatalength > 256)) {
-			if (zdev->max_exp_bit_length <= 17) {
-				zdev->max_exp_bit_length = 17;
+			if (zq->zcard->max_exp_bit_length <= 17) {
+				zq->zcard->max_exp_bit_length = 17;
 				return -EAGAIN;
 			} else
 				return -EINVAL;
 		}
 		if (msg->hdr.reply_code)
-			return convert_error(zdev, reply);
+			return convert_error(zq, reply);
 		if (msg->cprbx.cprb_ver_id == 0x02)
-			return convert_type86_ica(zdev, reply,
+			return convert_type86_ica(zq, reply,
 						  outputdata, outputdatalength);
 		/* Fall through, no break, incorrect cprb version is an unknown
 		 * response */
 	default: /* Unknown response type, this should NEVER EVER happen */
-		zdev->online = 0;
-		pr_err("Cryptographic device %x failed and was set offline\n",
-		       AP_QID_DEVICE(zdev->ap_dev->qid));
-		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%dfail",
-			       AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online);
+		zq->online = 0;
+		pr_err("Cryptographic device %02x.%04x failed and was set offline\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid));
+		ZCRYPT_DBF(DBF_ERR,
+			   "device=%02x.%04x rtype=0x%02x => online=0 rc=EAGAIN\n",
+			   AP_QID_CARD(zq->queue->qid),
+			   AP_QID_QUEUE(zq->queue->qid),
+			   (int) msg->hdr.type);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
 }
 
-static int convert_response_xcrb(struct zcrypt_device *zdev,
+static int convert_response_xcrb(struct zcrypt_queue *zq,
 			    struct ap_message *reply,
 			    struct ica_xcRB *xcRB)
 {
 	struct type86x_reply *msg = reply->message;
 
-	/* Response type byte is the second byte in the response. */
-	switch (((unsigned char *) reply->message)[1]) {
+	switch (msg->hdr.type) {
 	case TYPE82_RSP_CODE:
 	case TYPE88_RSP_CODE:
 		xcRB->status = 0x0008044DL; /* HDD_InvalidParm */
-		return convert_error(zdev, reply);
+		return convert_error(zq, reply);
 	case TYPE86_RSP_CODE:
 		if (msg->hdr.reply_code) {
 			memcpy(&(xcRB->status), msg->fmt2.apfs, sizeof(u32));
-			return convert_error(zdev, reply);
+			return convert_error(zq, reply);
 		}
 		if (msg->cprbx.cprb_ver_id == 0x02)
-			return convert_type86_xcrb(zdev, reply, xcRB);
+			return convert_type86_xcrb(zq, reply, xcRB);
 		/* Fall through, no break, incorrect cprb version is an unknown
 		 * response */
 	default: /* Unknown response type, this should NEVER EVER happen */
 		xcRB->status = 0x0008044DL; /* HDD_InvalidParm */
-		zdev->online = 0;
-		pr_err("Cryptographic device %x failed and was set offline\n",
-		       AP_QID_DEVICE(zdev->ap_dev->qid));
-		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%dfail",
-			       AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online);
+		zq->online = 0;
+		pr_err("Cryptographic device %02x.%04x failed and was set offline\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid));
+		ZCRYPT_DBF(DBF_ERR,
+			   "device=%02x.%04x rtype=0x%02x => online=0 rc=EAGAIN\n",
+			   AP_QID_CARD(zq->queue->qid),
+			   AP_QID_QUEUE(zq->queue->qid),
+			   (int) msg->hdr.type);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
 }
 
-static int convert_response_ep11_xcrb(struct zcrypt_device *zdev,
+static int convert_response_ep11_xcrb(struct zcrypt_queue *zq,
 	struct ap_message *reply, struct ep11_urb *xcRB)
 {
 	struct type86_ep11_reply *msg = reply->message;
 
-	/* Response type byte is the second byte in the response. */
-	switch (((unsigned char *)reply->message)[1]) {
+	switch (msg->hdr.type) {
 	case TYPE82_RSP_CODE:
 	case TYPE87_RSP_CODE:
-		return convert_error(zdev, reply);
+		return convert_error(zq, reply);
 	case TYPE86_RSP_CODE:
 		if (msg->hdr.reply_code)
-			return convert_error(zdev, reply);
+			return convert_error(zq, reply);
 		if (msg->cprbx.cprb_ver_id == 0x04)
-			return convert_type86_ep11_xcrb(zdev, reply, xcRB);
+			return convert_type86_ep11_xcrb(zq, reply, xcRB);
 	/* Fall through, no break, incorrect cprb version is an unknown resp.*/
 	default: /* Unknown response type, this should NEVER EVER happen */
-		zdev->online = 0;
-		pr_err("Cryptographic device %x failed and was set offline\n",
-		       AP_QID_DEVICE(zdev->ap_dev->qid));
-		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%dfail",
-			       AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online);
+		zq->online = 0;
+		pr_err("Cryptographic device %02x.%04x failed and was set offline\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid));
+		ZCRYPT_DBF(DBF_ERR,
+			   "device=%02x.%04x rtype=0x%02x => online=0 rc=EAGAIN\n",
+			   AP_QID_CARD(zq->queue->qid),
+			   AP_QID_QUEUE(zq->queue->qid),
+			   (int) msg->hdr.type);
 		return -EAGAIN; /* repeat the request on a different device. */
 	}
 }
 
-static int convert_response_rng(struct zcrypt_device *zdev,
+static int convert_response_rng(struct zcrypt_queue *zq,
 				 struct ap_message *reply,
 				 char *data)
 {
@@ -794,15 +896,19 @@ static int convert_response_rng(struct zcrypt_device *zdev,
 		if (msg->hdr.reply_code)
 			return -EINVAL;
 		if (msg->cprbx.cprb_ver_id == 0x02)
-			return convert_type86_rng(zdev, reply, data);
+			return convert_type86_rng(zq, reply, data);
 		/* Fall through, no break, incorrect cprb version is an unknown
 		 * response */
 	default: /* Unknown response type, this should NEVER EVER happen */
-		zdev->online = 0;
-		pr_err("Cryptographic device %x failed and was set offline\n",
-		       AP_QID_DEVICE(zdev->ap_dev->qid));
-		ZCRYPT_DBF_DEV(DBF_ERR, zdev, "dev%04xo%dfail",
-			       AP_QID_DEVICE(zdev->ap_dev->qid), zdev->online);
+		zq->online = 0;
+		pr_err("Cryptographic device %02x.%04x failed and was set offline\n",
+		       AP_QID_CARD(zq->queue->qid),
+		       AP_QID_QUEUE(zq->queue->qid));
+		ZCRYPT_DBF(DBF_ERR,
+			   "device=%02x.%04x rtype=0x%02x => online=0 rc=EAGAIN\n",
+			   AP_QID_CARD(zq->queue->qid),
+			   AP_QID_QUEUE(zq->queue->qid),
+			   (int) msg->hdr.type);
 		return -EAGAIN;	/* repeat the request on a different device. */
 	}
 }
@@ -811,11 +917,11 @@ static int convert_response_rng(struct zcrypt_device *zdev,
  * This function is called from the AP bus code after a crypto request
  * "msg" has finished with the reply message "reply".
  * It is called from tasklet context.
- * @ap_dev: pointer to the AP device
+ * @aq: pointer to the AP queue
  * @msg: pointer to the AP message
  * @reply: pointer to the AP reply message
  */
-static void zcrypt_msgtype6_receive(struct ap_device *ap_dev,
+static void zcrypt_msgtype6_receive(struct ap_queue *aq,
 				  struct ap_message *msg,
 				  struct ap_message *reply)
 {
@@ -860,11 +966,11 @@ out:
  * This function is called from the AP bus code after a crypto request
  * "msg" has finished with the reply message "reply".
  * It is called from tasklet context.
- * @ap_dev: pointer to the AP device
+ * @aq: pointer to the AP queue
  * @msg: pointer to the AP message
  * @reply: pointer to the AP reply message
  */
-static void zcrypt_msgtype6_receive_ep11(struct ap_device *ap_dev,
+static void zcrypt_msgtype6_receive_ep11(struct ap_queue *aq,
 					 struct ap_message *msg,
 					 struct ap_message *reply)
 {
@@ -904,11 +1010,11 @@ static atomic_t zcrypt_step = ATOMIC_INIT(0);
 /**
  * The request distributor calls this function if it picked the PCIXCC/CEX2C
  * device to handle a modexpo request.
- * @zdev: pointer to zcrypt_device structure that identifies the
+ * @zq: pointer to zcrypt_queue structure that identifies the
  *	  PCIXCC/CEX2C device to the request distributor
  * @mex: pointer to the modexpo request buffer
  */
-static long zcrypt_msgtype6_modexpo(struct zcrypt_device *zdev,
+static long zcrypt_msgtype6_modexpo(struct zcrypt_queue *zq,
 				  struct ica_rsa_modexpo *mex)
 {
 	struct ap_message ap_msg;
@@ -925,21 +1031,21 @@ static long zcrypt_msgtype6_modexpo(struct zcrypt_device *zdev,
 	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
 				atomic_inc_return(&zcrypt_step);
 	ap_msg.private = &resp_type;
-	rc = ICAMEX_msg_to_type6MEX_msgX(zdev, &ap_msg, mex);
+	rc = ICAMEX_msg_to_type6MEX_msgX(zq, &ap_msg, mex);
 	if (rc)
 		goto out_free;
 	init_completion(&resp_type.work);
-	ap_queue_message(zdev->ap_dev, &ap_msg);
+	ap_queue_message(zq->queue, &ap_msg);
 	rc = wait_for_completion_interruptible(&resp_type.work);
 	if (rc == 0) {
 		rc = ap_msg.rc;
 		if (rc == 0)
-			rc = convert_response_ica(zdev, &ap_msg,
+			rc = convert_response_ica(zq, &ap_msg,
 						  mex->outputdata,
 						  mex->outputdatalength);
 	} else
 		/* Signal pending. */
-		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		ap_cancel_message(zq->queue, &ap_msg);
 out_free:
 	free_page((unsigned long) ap_msg.message);
 	return rc;
@@ -948,11 +1054,11 @@ out_free:
 /**
  * The request distributor calls this function if it picked the PCIXCC/CEX2C
  * device to handle a modexpo_crt request.
- * @zdev: pointer to zcrypt_device structure that identifies the
+ * @zq: pointer to zcrypt_queue structure that identifies the
  *	  PCIXCC/CEX2C device to the request distributor
  * @crt: pointer to the modexpoc_crt request buffer
  */
-static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_device *zdev,
+static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_queue *zq,
 				      struct ica_rsa_modexpo_crt *crt)
 {
 	struct ap_message ap_msg;
@@ -969,148 +1075,258 @@ static long zcrypt_msgtype6_modexpo_crt(struct zcrypt_device *zdev,
 	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
 				atomic_inc_return(&zcrypt_step);
 	ap_msg.private = &resp_type;
-	rc = ICACRT_msg_to_type6CRT_msgX(zdev, &ap_msg, crt);
+	rc = ICACRT_msg_to_type6CRT_msgX(zq, &ap_msg, crt);
 	if (rc)
 		goto out_free;
 	init_completion(&resp_type.work);
-	ap_queue_message(zdev->ap_dev, &ap_msg);
+	ap_queue_message(zq->queue, &ap_msg);
 	rc = wait_for_completion_interruptible(&resp_type.work);
 	if (rc == 0) {
 		rc = ap_msg.rc;
 		if (rc == 0)
-			rc = convert_response_ica(zdev, &ap_msg,
+			rc = convert_response_ica(zq, &ap_msg,
 						  crt->outputdata,
 						  crt->outputdatalength);
-	} else
+	} else {
 		/* Signal pending. */
-		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		ap_cancel_message(zq->queue, &ap_msg);
+	}
 out_free:
 	free_page((unsigned long) ap_msg.message);
 	return rc;
 }
 
+unsigned int get_cprb_fc(struct ica_xcRB *xcRB,
+				struct ap_message *ap_msg,
+				unsigned int *func_code, unsigned short **dom)
+{
+	struct response_type resp_type = {
+		.type = PCIXCC_RESPONSE_TYPE_XCRB,
+	};
+	int rc;
+
+	ap_init_message(ap_msg);
+	ap_msg->message = kmalloc(MSGTYPE06_MAX_MSG_SIZE, GFP_KERNEL);
+	if (!ap_msg->message)
+		return -ENOMEM;
+	ap_msg->receive = zcrypt_msgtype6_receive;
+	ap_msg->psmid = (((unsigned long long) current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	ap_msg->private = kmalloc(sizeof(resp_type), GFP_KERNEL);
+	if (!ap_msg->private) {
+		kzfree(ap_msg->message);
+		return -ENOMEM;
+	}
+	memcpy(ap_msg->private, &resp_type, sizeof(resp_type));
+	rc = XCRB_msg_to_type6CPRB_msgX(ap_msg, xcRB, func_code, dom);
+	if (rc) {
+		kzfree(ap_msg->message);
+		kzfree(ap_msg->private);
+	}
+	return rc;
+}
+
 /**
  * The request distributor calls this function if it picked the PCIXCC/CEX2C
  * device to handle a send_cprb request.
- * @zdev: pointer to zcrypt_device structure that identifies the
+ * @zq: pointer to zcrypt_queue structure that identifies the
  *	  PCIXCC/CEX2C device to the request distributor
  * @xcRB: pointer to the send_cprb request buffer
  */
-static long zcrypt_msgtype6_send_cprb(struct zcrypt_device *zdev,
-				    struct ica_xcRB *xcRB)
+static long zcrypt_msgtype6_send_cprb(struct zcrypt_queue *zq,
+				    struct ica_xcRB *xcRB,
+				    struct ap_message *ap_msg)
 {
-	struct ap_message ap_msg;
-	struct response_type resp_type = {
-		.type = PCIXCC_RESPONSE_TYPE_XCRB,
-	};
 	int rc;
+	struct response_type *rtype = (struct response_type *)(ap_msg->private);
 
-	ap_init_message(&ap_msg);
-	ap_msg.message = kmalloc(MSGTYPE06_MAX_MSG_SIZE, GFP_KERNEL);
-	if (!ap_msg.message)
-		return -ENOMEM;
-	ap_msg.receive = zcrypt_msgtype6_receive;
-	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
-				atomic_inc_return(&zcrypt_step);
-	ap_msg.private = &resp_type;
-	rc = XCRB_msg_to_type6CPRB_msgX(zdev, &ap_msg, xcRB);
-	if (rc)
-		goto out_free;
-	init_completion(&resp_type.work);
-	ap_queue_message(zdev->ap_dev, &ap_msg);
-	rc = wait_for_completion_interruptible(&resp_type.work);
+	init_completion(&rtype->work);
+	ap_queue_message(zq->queue, ap_msg);
+	rc = wait_for_completion_interruptible(&rtype->work);
 	if (rc == 0) {
-		rc = ap_msg.rc;
+		rc = ap_msg->rc;
 		if (rc == 0)
-			rc = convert_response_xcrb(zdev, &ap_msg, xcRB);
+			rc = convert_response_xcrb(zq, ap_msg, xcRB);
 	} else
 		/* Signal pending. */
-		ap_cancel_message(zdev->ap_dev, &ap_msg);
-out_free:
-	kzfree(ap_msg.message);
+		ap_cancel_message(zq->queue, ap_msg);
+
+	kzfree(ap_msg->message);
+	kzfree(ap_msg->private);
+	return rc;
+}
+
+unsigned int get_ep11cprb_fc(struct ep11_urb *xcrb,
+				    struct ap_message *ap_msg,
+				    unsigned int *func_code)
+{
+	struct response_type resp_type = {
+		.type = PCIXCC_RESPONSE_TYPE_EP11,
+	};
+	int rc;
+
+	ap_init_message(ap_msg);
+	ap_msg->message = kmalloc(MSGTYPE06_MAX_MSG_SIZE, GFP_KERNEL);
+	if (!ap_msg->message)
+		return -ENOMEM;
+	ap_msg->receive = zcrypt_msgtype6_receive_ep11;
+	ap_msg->psmid = (((unsigned long long) current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	ap_msg->private = kmalloc(sizeof(resp_type), GFP_KERNEL);
+	if (!ap_msg->private) {
+		kzfree(ap_msg->message);
+		return -ENOMEM;
+	}
+	memcpy(ap_msg->private, &resp_type, sizeof(resp_type));
+	rc = xcrb_msg_to_type6_ep11cprb_msgx(ap_msg, xcrb, func_code);
+	if (rc) {
+		kzfree(ap_msg->message);
+		kzfree(ap_msg->private);
+	}
 	return rc;
 }
 
 /**
  * The request distributor calls this function if it picked the CEX4P
  * device to handle a send_ep11_cprb request.
- * @zdev: pointer to zcrypt_device structure that identifies the
+ * @zq: pointer to zcrypt_queue structure that identifies the
  *	  CEX4P device to the request distributor
  * @xcRB: pointer to the ep11 user request block
  */
-static long zcrypt_msgtype6_send_ep11_cprb(struct zcrypt_device *zdev,
-						struct ep11_urb *xcrb)
+static long zcrypt_msgtype6_send_ep11_cprb(struct zcrypt_queue *zq,
+					   struct ep11_urb *xcrb,
+					   struct ap_message *ap_msg)
 {
-	struct ap_message ap_msg;
-	struct response_type resp_type = {
-		.type = PCIXCC_RESPONSE_TYPE_EP11,
-	};
 	int rc;
+	unsigned int lfmt;
+	struct response_type *rtype = (struct response_type *)(ap_msg->private);
+	struct {
+		struct type6_hdr hdr;
+		struct ep11_cprb cprbx;
+		unsigned char	pld_tag;	/* fixed value 0x30 */
+		unsigned char	pld_lenfmt;	/* payload length format */
+	} __packed * msg = ap_msg->message;
+	struct pld_hdr {
+		unsigned char	func_tag;	/* fixed value 0x4 */
+		unsigned char	func_len;	/* fixed value 0x4 */
+		unsigned int	func_val;	/* function ID	   */
+		unsigned char	dom_tag;	/* fixed value 0x4 */
+		unsigned char	dom_len;	/* fixed value 0x4 */
+		unsigned int	dom_val;	/* domain id	   */
+	} __packed * payload_hdr = NULL;
 
-	ap_init_message(&ap_msg);
-	ap_msg.message = kmalloc(MSGTYPE06_MAX_MSG_SIZE, GFP_KERNEL);
-	if (!ap_msg.message)
-		return -ENOMEM;
-	ap_msg.receive = zcrypt_msgtype6_receive_ep11;
-	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
-				atomic_inc_return(&zcrypt_step);
-	ap_msg.private = &resp_type;
-	rc = xcrb_msg_to_type6_ep11cprb_msgx(zdev, &ap_msg, xcrb);
-	if (rc)
-		goto out_free;
-	init_completion(&resp_type.work);
-	ap_queue_message(zdev->ap_dev, &ap_msg);
-	rc = wait_for_completion_interruptible(&resp_type.work);
+
+	/**
+	 * The target domain field within the cprb body/payload block will be
+	 * replaced by the usage domain for non-management commands only.
+	 * Therefore we check the first bit of the 'flags' parameter for
+	 * management command indication.
+	 *   0 - non management command
+	 *   1 - management command
+	 */
+	if (!((msg->cprbx.flags & 0x80) == 0x80)) {
+		msg->cprbx.target_id = (unsigned int)
+					AP_QID_QUEUE(zq->queue->qid);
+
+		if ((msg->pld_lenfmt & 0x80) == 0x80) { /*ext.len.fmt 2 or 3*/
+			switch (msg->pld_lenfmt & 0x03) {
+			case 1:
+				lfmt = 2;
+				break;
+			case 2:
+				lfmt = 3;
+				break;
+			default:
+				return -EINVAL;
+			}
+		} else {
+			lfmt = 1; /* length format #1 */
+		}
+		payload_hdr = (struct pld_hdr *)((&(msg->pld_lenfmt))+lfmt);
+		payload_hdr->dom_val = (unsigned int)
+					AP_QID_QUEUE(zq->queue->qid);
+	}
+
+	init_completion(&rtype->work);
+	ap_queue_message(zq->queue, ap_msg);
+	rc = wait_for_completion_interruptible(&rtype->work);
 	if (rc == 0) {
-		rc = ap_msg.rc;
+		rc = ap_msg->rc;
 		if (rc == 0)
-			rc = convert_response_ep11_xcrb(zdev, &ap_msg, xcrb);
+			rc = convert_response_ep11_xcrb(zq, ap_msg, xcrb);
 	} else
 		/* Signal pending. */
-		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		ap_cancel_message(zq->queue, ap_msg);
 
-out_free:
-	kzfree(ap_msg.message);
+	kzfree(ap_msg->message);
+	kzfree(ap_msg->private);
 	return rc;
 }
 
+unsigned int get_rng_fc(struct ap_message *ap_msg, int *func_code,
+						   unsigned int *domain)
+{
+	struct response_type resp_type = {
+		.type = PCIXCC_RESPONSE_TYPE_XCRB,
+	};
+
+	ap_init_message(ap_msg);
+	ap_msg->message = kmalloc(MSGTYPE06_MAX_MSG_SIZE, GFP_KERNEL);
+	if (!ap_msg->message)
+		return -ENOMEM;
+	ap_msg->receive = zcrypt_msgtype6_receive;
+	ap_msg->psmid = (((unsigned long long) current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	ap_msg->private = kmalloc(sizeof(resp_type), GFP_KERNEL);
+	if (!ap_msg->private) {
+		kzfree(ap_msg->message);
+		return -ENOMEM;
+	}
+	memcpy(ap_msg->private, &resp_type, sizeof(resp_type));
+
+	rng_type6CPRB_msgX(ap_msg, ZCRYPT_RNG_BUFFER_SIZE, domain);
+
+	*func_code = HWRNG;
+	return 0;
+}
+
 /**
  * The request distributor calls this function if it picked the PCIXCC/CEX2C
  * device to generate random data.
- * @zdev: pointer to zcrypt_device structure that identifies the
+ * @zq: pointer to zcrypt_queue structure that identifies the
  *	  PCIXCC/CEX2C device to the request distributor
  * @buffer: pointer to a memory page to return random data
  */
-
-static long zcrypt_msgtype6_rng(struct zcrypt_device *zdev,
-				    char *buffer)
+static long zcrypt_msgtype6_rng(struct zcrypt_queue *zq,
+				char *buffer, struct ap_message *ap_msg)
 {
-	struct ap_message ap_msg;
-	struct response_type resp_type = {
-		.type = PCIXCC_RESPONSE_TYPE_XCRB,
-	};
+	struct {
+		struct type6_hdr hdr;
+		struct CPRBX cprbx;
+		char function_code[2];
+		short int rule_length;
+		char rule[8];
+		short int verb_length;
+		short int key_length;
+	} __packed * msg = ap_msg->message;
+	struct response_type *rtype = (struct response_type *)(ap_msg->private);
 	int rc;
 
-	ap_init_message(&ap_msg);
-	ap_msg.message = kmalloc(MSGTYPE06_MAX_MSG_SIZE, GFP_KERNEL);
-	if (!ap_msg.message)
-		return -ENOMEM;
-	ap_msg.receive = zcrypt_msgtype6_receive;
-	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
-				atomic_inc_return(&zcrypt_step);
-	ap_msg.private = &resp_type;
-	rng_type6CPRB_msgX(zdev->ap_dev, &ap_msg, ZCRYPT_RNG_BUFFER_SIZE);
-	init_completion(&resp_type.work);
-	ap_queue_message(zdev->ap_dev, &ap_msg);
-	rc = wait_for_completion_interruptible(&resp_type.work);
+	msg->cprbx.domain = AP_QID_QUEUE(zq->queue->qid);
+
+	init_completion(&rtype->work);
+	ap_queue_message(zq->queue, ap_msg);
+	rc = wait_for_completion_interruptible(&rtype->work);
 	if (rc == 0) {
-		rc = ap_msg.rc;
+		rc = ap_msg->rc;
 		if (rc == 0)
-			rc = convert_response_rng(zdev, &ap_msg, buffer);
+			rc = convert_response_rng(zq, ap_msg, buffer);
 	} else
 		/* Signal pending. */
-		ap_cancel_message(zdev->ap_dev, &ap_msg);
-	kfree(ap_msg.message);
+		ap_cancel_message(zq->queue, ap_msg);
+
+	kzfree(ap_msg->message);
+	kzfree(ap_msg->private);
 	return rc;
 }
 
@@ -1145,12 +1361,11 @@ static struct zcrypt_ops zcrypt_msgtype6_ep11_ops = {
 	.send_ep11_cprb = zcrypt_msgtype6_send_ep11_cprb,
 };
 
-int __init zcrypt_msgtype6_init(void)
+void __init zcrypt_msgtype6_init(void)
 {
 	zcrypt_msgtype_register(&zcrypt_msgtype6_norng_ops);
 	zcrypt_msgtype_register(&zcrypt_msgtype6_ops);
 	zcrypt_msgtype_register(&zcrypt_msgtype6_ep11_ops);
-	return 0;
 }
 
 void __exit zcrypt_msgtype6_exit(void)
@@ -1159,6 +1374,3 @@ void __exit zcrypt_msgtype6_exit(void)
 	zcrypt_msgtype_unregister(&zcrypt_msgtype6_ops);
 	zcrypt_msgtype_unregister(&zcrypt_msgtype6_ep11_ops);
 }
-
-module_init(zcrypt_msgtype6_init);
-module_exit(zcrypt_msgtype6_exit);
diff --git a/drivers/s390/crypto/zcrypt_msgtype6.h b/drivers/s390/crypto/zcrypt_msgtype6.h
index 207247570623..7a0d5b57821f 100644
--- a/drivers/s390/crypto/zcrypt_msgtype6.h
+++ b/drivers/s390/crypto/zcrypt_msgtype6.h
@@ -116,15 +116,28 @@ struct type86_fmt2_ext {
 	unsigned int	  offset4;	/* 0x00000000			*/
 } __packed;
 
+unsigned int get_cprb_fc(struct ica_xcRB *, struct ap_message *,
+			 unsigned int *, unsigned short **);
+unsigned int get_ep11cprb_fc(struct ep11_urb *, struct ap_message *,
+			     unsigned int *);
+unsigned int get_rng_fc(struct ap_message *, int *, unsigned int *);
+
+#define LOW	10
+#define MEDIUM	100
+#define HIGH	500
+
+int speed_idx_cca(int);
+int speed_idx_ep11(int);
+
 /**
  * Prepare a type6 CPRB message for random number generation
  *
  * @ap_dev: AP device pointer
  * @ap_msg: pointer to AP message
  */
-static inline void rng_type6CPRB_msgX(struct ap_device *ap_dev,
-			       struct ap_message *ap_msg,
-			       unsigned random_number_length)
+static inline void rng_type6CPRB_msgX(struct ap_message *ap_msg,
+				      unsigned int random_number_length,
+				      unsigned int *domain)
 {
 	struct {
 		struct type6_hdr hdr;
@@ -156,16 +169,16 @@ static inline void rng_type6CPRB_msgX(struct ap_device *ap_dev,
 	msg->hdr.FromCardLen2 = random_number_length,
 	msg->cprbx = local_cprbx;
 	msg->cprbx.rpl_datal = random_number_length,
-	msg->cprbx.domain = AP_QID_QUEUE(ap_dev->qid);
 	memcpy(msg->function_code, msg->hdr.function_code, 0x02);
 	msg->rule_length = 0x0a;
 	memcpy(msg->rule, "RANDOM  ", 8);
 	msg->verb_length = 0x02;
 	msg->key_length = 0x02;
 	ap_msg->length = sizeof(*msg);
+	*domain = (unsigned short)msg->cprbx.domain;
 }
 
-int zcrypt_msgtype6_init(void);
+void zcrypt_msgtype6_init(void);
 void zcrypt_msgtype6_exit(void);
 
 #endif /* _ZCRYPT_MSGTYPE6_H_ */
diff --git a/drivers/s390/crypto/zcrypt_pcixcc.c b/drivers/s390/crypto/zcrypt_pcixcc.c
index df8f0c4dacb7..26ceaa696765 100644
--- a/drivers/s390/crypto/zcrypt_pcixcc.c
+++ b/drivers/s390/crypto/zcrypt_pcixcc.c
@@ -32,6 +32,7 @@
 #include <linux/slab.h>
 #include <linux/atomic.h>
 #include <asm/uaccess.h>
+#include <linux/mod_devicetable.h>
 
 #include "ap_bus.h"
 #include "zcrypt_api.h"
@@ -46,11 +47,6 @@
 #define CEX3C_MIN_MOD_SIZE	PCIXCC_MIN_MOD_SIZE
 #define CEX3C_MAX_MOD_SIZE	512	/* 4096 bits	*/
 
-#define PCIXCC_MCL2_SPEED_RATING	7870
-#define PCIXCC_MCL3_SPEED_RATING	7870
-#define CEX2C_SPEED_RATING		7000
-#define CEX3C_SPEED_RATING		6500
-
 #define PCIXCC_MAX_ICA_MESSAGE_SIZE 0x77c  /* max size type6 v2 crt message */
 #define PCIXCC_MAX_ICA_RESPONSE_SIZE 0x77c /* max size type86 v2 reply	    */
 
@@ -67,142 +63,34 @@ struct response_type {
 #define PCIXCC_RESPONSE_TYPE_ICA  0
 #define PCIXCC_RESPONSE_TYPE_XCRB 1
 
-static struct ap_device_id zcrypt_pcixcc_ids[] = {
-	{ AP_DEVICE(AP_DEVICE_TYPE_PCIXCC) },
-	{ AP_DEVICE(AP_DEVICE_TYPE_CEX2C) },
-	{ AP_DEVICE(AP_DEVICE_TYPE_CEX3C) },
-	{ /* end of list */ },
-};
-
-MODULE_DEVICE_TABLE(ap, zcrypt_pcixcc_ids);
 MODULE_AUTHOR("IBM Corporation");
 MODULE_DESCRIPTION("PCIXCC Cryptographic Coprocessor device driver, " \
 		   "Copyright IBM Corp. 2001, 2012");
 MODULE_LICENSE("GPL");
 
-static int zcrypt_pcixcc_probe(struct ap_device *ap_dev);
-static void zcrypt_pcixcc_remove(struct ap_device *ap_dev);
-
-static struct ap_driver zcrypt_pcixcc_driver = {
-	.probe = zcrypt_pcixcc_probe,
-	.remove = zcrypt_pcixcc_remove,
-	.ids = zcrypt_pcixcc_ids,
-	.request_timeout = PCIXCC_CLEANUP_TIME,
+static struct ap_device_id zcrypt_pcixcc_card_ids[] = {
+	{ .dev_type = AP_DEVICE_TYPE_PCIXCC,
+	  .match_flags = AP_DEVICE_ID_MATCH_CARD_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX2C,
+	  .match_flags = AP_DEVICE_ID_MATCH_CARD_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX3C,
+	  .match_flags = AP_DEVICE_ID_MATCH_CARD_TYPE },
+	{ /* end of list */ },
 };
 
-/**
- * Micro-code detection function. Its sends a message to a pcixcc card
- * to find out the microcode level.
- * @ap_dev: pointer to the AP device.
- */
-static int zcrypt_pcixcc_mcl(struct ap_device *ap_dev)
-{
-	static unsigned char msg[] = {
-		0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x43,0x41,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x50,0x4B,0x00,0x00,
-		0x00,0x00,0x01,0xC4,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x07,0x24,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0xDC,0x02,0x00,0x00,0x00,0x54,0x32,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xE8,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x24,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		0x00,0x00,0x00,0x00,0x50,0x4B,0x00,0x0A,
-		0x4D,0x52,0x50,0x20,0x20,0x20,0x20,0x20,
-		0x00,0x42,0x00,0x01,0x02,0x03,0x04,0x05,
-		0x06,0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,
-		0x0E,0x0F,0x00,0x11,0x22,0x33,0x44,0x55,
-		0x66,0x77,0x88,0x99,0xAA,0xBB,0xCC,0xDD,
-		0xEE,0xFF,0xFF,0xEE,0xDD,0xCC,0xBB,0xAA,
-		0x99,0x88,0x77,0x66,0x55,0x44,0x33,0x22,
-		0x11,0x00,0x01,0x23,0x45,0x67,0x89,0xAB,
-		0xCD,0xEF,0xFE,0xDC,0xBA,0x98,0x76,0x54,
-		0x32,0x10,0x00,0x9A,0x00,0x98,0x00,0x00,
-		0x1E,0x00,0x00,0x94,0x00,0x00,0x00,0x00,
-		0x04,0x00,0x00,0x8C,0x00,0x00,0x00,0x40,
-		0x02,0x00,0x00,0x40,0xBA,0xE8,0x23,0x3C,
-		0x75,0xF3,0x91,0x61,0xD6,0x73,0x39,0xCF,
-		0x7B,0x6D,0x8E,0x61,0x97,0x63,0x9E,0xD9,
-		0x60,0x55,0xD6,0xC7,0xEF,0xF8,0x1E,0x63,
-		0x95,0x17,0xCC,0x28,0x45,0x60,0x11,0xC5,
-		0xC4,0x4E,0x66,0xC6,0xE6,0xC3,0xDE,0x8A,
-		0x19,0x30,0xCF,0x0E,0xD7,0xAA,0xDB,0x01,
-		0xD8,0x00,0xBB,0x8F,0x39,0x9F,0x64,0x28,
-		0xF5,0x7A,0x77,0x49,0xCC,0x6B,0xA3,0x91,
-		0x97,0x70,0xE7,0x60,0x1E,0x39,0xE1,0xE5,
-		0x33,0xE1,0x15,0x63,0x69,0x08,0x80,0x4C,
-		0x67,0xC4,0x41,0x8F,0x48,0xDF,0x26,0x98,
-		0xF1,0xD5,0x8D,0x88,0xD9,0x6A,0xA4,0x96,
-		0xC5,0x84,0xD9,0x30,0x49,0x67,0x7D,0x19,
-		0xB1,0xB3,0x45,0x4D,0xB2,0x53,0x9A,0x47,
-		0x3C,0x7C,0x55,0xBF,0xCC,0x85,0x00,0x36,
-		0xF1,0x3D,0x93,0x53
-	};
-	unsigned long long psmid;
-	struct CPRBX *cprbx;
-	char *reply;
-	int rc, i;
-
-	reply = (void *) get_zeroed_page(GFP_KERNEL);
-	if (!reply)
-		return -ENOMEM;
+MODULE_DEVICE_TABLE(ap, zcrypt_pcixcc_card_ids);
 
-	rc = ap_send(ap_dev->qid, 0x0102030405060708ULL, msg, sizeof(msg));
-	if (rc)
-		goto out_free;
-
-	/* Wait for the test message to complete. */
-	for (i = 0; i < 6; i++) {
-		msleep(300);
-		rc = ap_recv(ap_dev->qid, &psmid, reply, 4096);
-		if (rc == 0 && psmid == 0x0102030405060708ULL)
-			break;
-	}
-
-	if (i >= 6) {
-		/* Got no answer. */
-		rc = -ENODEV;
-		goto out_free;
-	}
+static struct ap_device_id zcrypt_pcixcc_queue_ids[] = {
+	{ .dev_type = AP_DEVICE_TYPE_PCIXCC,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX2C,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ .dev_type = AP_DEVICE_TYPE_CEX3C,
+	  .match_flags = AP_DEVICE_ID_MATCH_QUEUE_TYPE },
+	{ /* end of list */ },
+};
 
-	cprbx = (struct CPRBX *) (reply + 48);
-	if (cprbx->ccp_rtcode == 8 && cprbx->ccp_rscode == 33)
-		rc = ZCRYPT_PCIXCC_MCL2;
-	else
-		rc = ZCRYPT_PCIXCC_MCL3;
-out_free:
-	free_page((unsigned long) reply);
-	return rc;
-}
+MODULE_DEVICE_TABLE(ap, zcrypt_pcixcc_queue_ids);
 
 /**
  * Large random number detection function. Its sends a message to a pcixcc
@@ -211,15 +99,25 @@ out_free:
  *
  * Returns 1 if large random numbers are supported, 0 if not and < 0 on error.
  */
-static int zcrypt_pcixcc_rng_supported(struct ap_device *ap_dev)
+static int zcrypt_pcixcc_rng_supported(struct ap_queue *aq)
 {
 	struct ap_message ap_msg;
 	unsigned long long psmid;
+	unsigned int domain;
 	struct {
 		struct type86_hdr hdr;
 		struct type86_fmt2_ext fmt2;
 		struct CPRBX cprbx;
 	} __attribute__((packed)) *reply;
+	struct {
+		struct type6_hdr hdr;
+		struct CPRBX cprbx;
+		char function_code[2];
+		short int rule_length;
+		char rule[8];
+		short int verb_length;
+		short int key_length;
+	} __packed * msg;
 	int rc, i;
 
 	ap_init_message(&ap_msg);
@@ -227,8 +125,12 @@ static int zcrypt_pcixcc_rng_supported(struct ap_device *ap_dev)
 	if (!ap_msg.message)
 		return -ENOMEM;
 
-	rng_type6CPRB_msgX(ap_dev, &ap_msg, 4);
-	rc = ap_send(ap_dev->qid, 0x0102030405060708ULL, ap_msg.message,
+	rng_type6CPRB_msgX(&ap_msg, 4, &domain);
+
+	msg = ap_msg.message;
+	msg->cprbx.domain = AP_QID_QUEUE(aq->qid);
+
+	rc = ap_send(aq->qid, 0x0102030405060708ULL, ap_msg.message,
 		     ap_msg.length);
 	if (rc)
 		goto out_free;
@@ -236,7 +138,7 @@ static int zcrypt_pcixcc_rng_supported(struct ap_device *ap_dev)
 	/* Wait for the test message to complete. */
 	for (i = 0; i < 2 * HZ; i++) {
 		msleep(1000 / HZ);
-		rc = ap_recv(ap_dev->qid, &psmid, ap_msg.message, 4096);
+		rc = ap_recv(aq->qid, &psmid, ap_msg.message, 4096);
 		if (rc == 0 && psmid == 0x0102030405060708ULL)
 			break;
 	}
@@ -258,110 +160,168 @@ out_free:
 }
 
 /**
- * Probe function for PCIXCC/CEX2C cards. It always accepts the AP device
- * since the bus_match already checked the hardware type. The PCIXCC
- * cards come in two flavours: micro code level 2 and micro code level 3.
- * This is checked by sending a test message to the device.
- * @ap_dev: pointer to the AP device.
+ * Probe function for PCIXCC/CEX2C card devices. It always accepts the
+ * AP device since the bus_match already checked the hardware type. The
+ * PCIXCC cards come in two flavours: micro code level 2 and micro code
+ * level 3. This is checked by sending a test message to the device.
+ * @ap_dev: pointer to the AP card device.
  */
-static int zcrypt_pcixcc_probe(struct ap_device *ap_dev)
+static int zcrypt_pcixcc_card_probe(struct ap_device *ap_dev)
 {
-	struct zcrypt_device *zdev;
+	/*
+	 * Normalized speed ratings per crypto adapter
+	 * MEX_1k, MEX_2k, MEX_4k, CRT_1k, CRT_2k, CRT_4k, RNG, SECKEY
+	 */
+	static const int CEX2C_SPEED_IDX[] = {
+		1000, 1400, 2400, 1100, 1500, 2600, 100, 12};
+	static const int CEX3C_SPEED_IDX[] = {
+		500,  700, 1400,  550,	800, 1500,  80, 10};
+
+	struct ap_card *ac = to_ap_card(&ap_dev->device);
+	struct zcrypt_card *zc;
 	int rc = 0;
 
-	zdev = zcrypt_device_alloc(PCIXCC_MAX_XCRB_MESSAGE_SIZE);
-	if (!zdev)
+	zc = zcrypt_card_alloc();
+	if (!zc)
 		return -ENOMEM;
-	zdev->ap_dev = ap_dev;
-	zdev->online = 1;
-	switch (ap_dev->device_type) {
-	case AP_DEVICE_TYPE_PCIXCC:
-		rc = zcrypt_pcixcc_mcl(ap_dev);
-		if (rc < 0) {
-			zcrypt_device_free(zdev);
-			return rc;
-		}
-		zdev->user_space_type = rc;
-		if (rc == ZCRYPT_PCIXCC_MCL2) {
-			zdev->type_string = "PCIXCC_MCL2";
-			zdev->speed_rating = PCIXCC_MCL2_SPEED_RATING;
-			zdev->min_mod_size = PCIXCC_MIN_MOD_SIZE_OLD;
-			zdev->max_mod_size = PCIXCC_MAX_MOD_SIZE;
-			zdev->max_exp_bit_length = PCIXCC_MAX_MOD_SIZE;
-		} else {
-			zdev->type_string = "PCIXCC_MCL3";
-			zdev->speed_rating = PCIXCC_MCL3_SPEED_RATING;
-			zdev->min_mod_size = PCIXCC_MIN_MOD_SIZE;
-			zdev->max_mod_size = PCIXCC_MAX_MOD_SIZE;
-			zdev->max_exp_bit_length = PCIXCC_MAX_MOD_SIZE;
-		}
-		break;
+	zc->card = ac;
+	ac->private = zc;
+	switch (ac->ap_dev.device_type) {
 	case AP_DEVICE_TYPE_CEX2C:
-		zdev->user_space_type = ZCRYPT_CEX2C;
-		zdev->type_string = "CEX2C";
-		zdev->speed_rating = CEX2C_SPEED_RATING;
-		zdev->min_mod_size = PCIXCC_MIN_MOD_SIZE;
-		zdev->max_mod_size = PCIXCC_MAX_MOD_SIZE;
-		zdev->max_exp_bit_length = PCIXCC_MAX_MOD_SIZE;
+		zc->user_space_type = ZCRYPT_CEX2C;
+		zc->type_string = "CEX2C";
+		memcpy(zc->speed_rating, CEX2C_SPEED_IDX,
+		       sizeof(CEX2C_SPEED_IDX));
+		zc->min_mod_size = PCIXCC_MIN_MOD_SIZE;
+		zc->max_mod_size = PCIXCC_MAX_MOD_SIZE;
+		zc->max_exp_bit_length = PCIXCC_MAX_MOD_SIZE;
 		break;
 	case AP_DEVICE_TYPE_CEX3C:
-		zdev->user_space_type = ZCRYPT_CEX3C;
-		zdev->type_string = "CEX3C";
-		zdev->speed_rating = CEX3C_SPEED_RATING;
-		zdev->min_mod_size = CEX3C_MIN_MOD_SIZE;
-		zdev->max_mod_size = CEX3C_MAX_MOD_SIZE;
-		zdev->max_exp_bit_length = CEX3C_MAX_MOD_SIZE;
+		zc->user_space_type = ZCRYPT_CEX3C;
+		zc->type_string = "CEX3C";
+		memcpy(zc->speed_rating, CEX3C_SPEED_IDX,
+		       sizeof(CEX3C_SPEED_IDX));
+		zc->min_mod_size = CEX3C_MIN_MOD_SIZE;
+		zc->max_mod_size = CEX3C_MAX_MOD_SIZE;
+		zc->max_exp_bit_length = CEX3C_MAX_MOD_SIZE;
 		break;
 	default:
-		goto out_free;
+		zcrypt_card_free(zc);
+		return -ENODEV;
 	}
+	zc->online = 1;
+
+	rc = zcrypt_card_register(zc);
+	if (rc) {
+		ac->private = NULL;
+		zcrypt_card_free(zc);
+	}
+
+	return rc;
+}
 
-	rc = zcrypt_pcixcc_rng_supported(ap_dev);
+/**
+ * This is called to remove the PCIXCC/CEX2C card driver information
+ * if an AP card device is removed.
+ */
+static void zcrypt_pcixcc_card_remove(struct ap_device *ap_dev)
+{
+	struct zcrypt_card *zc = to_ap_card(&ap_dev->device)->private;
+
+	if (zc)
+		zcrypt_card_unregister(zc);
+}
+
+static struct ap_driver zcrypt_pcixcc_card_driver = {
+	.probe = zcrypt_pcixcc_card_probe,
+	.remove = zcrypt_pcixcc_card_remove,
+	.ids = zcrypt_pcixcc_card_ids,
+};
+
+/**
+ * Probe function for PCIXCC/CEX2C queue devices. It always accepts the
+ * AP device since the bus_match already checked the hardware type. The
+ * PCIXCC cards come in two flavours: micro code level 2 and micro code
+ * level 3. This is checked by sending a test message to the device.
+ * @ap_dev: pointer to the AP card device.
+ */
+static int zcrypt_pcixcc_queue_probe(struct ap_device *ap_dev)
+{
+	struct ap_queue *aq = to_ap_queue(&ap_dev->device);
+	struct zcrypt_queue *zq;
+	int rc;
+
+	zq = zcrypt_queue_alloc(PCIXCC_MAX_XCRB_MESSAGE_SIZE);
+	if (!zq)
+		return -ENOMEM;
+	zq->queue = aq;
+	zq->online = 1;
+	atomic_set(&zq->load, 0);
+	rc = zcrypt_pcixcc_rng_supported(aq);
 	if (rc < 0) {
-		zcrypt_device_free(zdev);
+		zcrypt_queue_free(zq);
 		return rc;
 	}
 	if (rc)
-		zdev->ops = zcrypt_msgtype_request(MSGTYPE06_NAME,
-						   MSGTYPE06_VARIANT_DEFAULT);
+		zq->ops = zcrypt_msgtype(MSGTYPE06_NAME,
+					 MSGTYPE06_VARIANT_DEFAULT);
 	else
-		zdev->ops = zcrypt_msgtype_request(MSGTYPE06_NAME,
-						   MSGTYPE06_VARIANT_NORNG);
-	ap_device_init_reply(ap_dev, &zdev->reply);
-	ap_dev->private = zdev;
-	rc = zcrypt_device_register(zdev);
-	if (rc)
-		goto out_free;
-	return 0;
-
- out_free:
-	ap_dev->private = NULL;
-	zcrypt_msgtype_release(zdev->ops);
-	zcrypt_device_free(zdev);
+		zq->ops = zcrypt_msgtype(MSGTYPE06_NAME,
+					 MSGTYPE06_VARIANT_NORNG);
+	ap_queue_init_reply(aq, &zq->reply);
+	aq->request_timeout = PCIXCC_CLEANUP_TIME,
+	aq->private = zq;
+	rc = zcrypt_queue_register(zq);
+	if (rc) {
+		aq->private = NULL;
+		zcrypt_queue_free(zq);
+	}
 	return rc;
 }
 
 /**
- * This is called to remove the extended PCIXCC/CEX2C driver information
- * if an AP device is removed.
+ * This is called to remove the PCIXCC/CEX2C queue driver information
+ * if an AP queue device is removed.
  */
-static void zcrypt_pcixcc_remove(struct ap_device *ap_dev)
+static void zcrypt_pcixcc_queue_remove(struct ap_device *ap_dev)
 {
-	struct zcrypt_device *zdev = ap_dev->private;
-	struct zcrypt_ops *zops = zdev->ops;
+	struct ap_queue *aq = to_ap_queue(&ap_dev->device);
+	struct zcrypt_queue *zq = aq->private;
 
-	zcrypt_device_unregister(zdev);
-	zcrypt_msgtype_release(zops);
+	ap_queue_remove(aq);
+	if (zq)
+		zcrypt_queue_unregister(zq);
 }
 
+static struct ap_driver zcrypt_pcixcc_queue_driver = {
+	.probe = zcrypt_pcixcc_queue_probe,
+	.remove = zcrypt_pcixcc_queue_remove,
+	.suspend = ap_queue_suspend,
+	.resume = ap_queue_resume,
+	.ids = zcrypt_pcixcc_queue_ids,
+};
+
 int __init zcrypt_pcixcc_init(void)
 {
-	return ap_driver_register(&zcrypt_pcixcc_driver, THIS_MODULE, "pcixcc");
+	int rc;
+
+	rc = ap_driver_register(&zcrypt_pcixcc_card_driver,
+				THIS_MODULE, "pcixcccard");
+	if (rc)
+		return rc;
+
+	rc = ap_driver_register(&zcrypt_pcixcc_queue_driver,
+				THIS_MODULE, "pcixccqueue");
+	if (rc)
+		ap_driver_unregister(&zcrypt_pcixcc_card_driver);
+
+	return rc;
 }
 
 void zcrypt_pcixcc_exit(void)
 {
-	ap_driver_unregister(&zcrypt_pcixcc_driver);
+	ap_driver_unregister(&zcrypt_pcixcc_queue_driver);
+	ap_driver_unregister(&zcrypt_pcixcc_card_driver);
 }
 
 module_init(zcrypt_pcixcc_init);
diff --git a/drivers/s390/crypto/zcrypt_queue.c b/drivers/s390/crypto/zcrypt_queue.c
new file mode 100644
index 000000000000..a303f3b2c328
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_queue.c
@@ -0,0 +1,226 @@
+/*
+ *  zcrypt 2.1.0
+ *
+ *  Copyright IBM Corp. 2001, 2012
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *	       Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *  MSGTYPE restruct:		  Holger Dengler <hd@linux.vnet.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/compat.h>
+#include <linux/slab.h>
+#include <linux/atomic.h>
+#include <linux/uaccess.h>
+#include <linux/hw_random.h>
+#include <linux/debugfs.h>
+#include <asm/debug.h>
+
+#include "zcrypt_debug.h"
+#include "zcrypt_api.h"
+
+#include "zcrypt_msgtype6.h"
+#include "zcrypt_msgtype50.h"
+
+/*
+ * Device attributes common for all crypto queue devices.
+ */
+
+static ssize_t zcrypt_queue_online_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct zcrypt_queue *zq = to_ap_queue(dev)->private;
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", zq->online);
+}
+
+static ssize_t zcrypt_queue_online_store(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf, size_t count)
+{
+	struct zcrypt_queue *zq = to_ap_queue(dev)->private;
+	struct zcrypt_card *zc = zq->zcard;
+	int online;
+
+	if (sscanf(buf, "%d\n", &online) != 1 || online < 0 || online > 1)
+		return -EINVAL;
+
+	if (online && !zc->online)
+		return -EINVAL;
+	zq->online = online;
+
+	ZCRYPT_DBF(DBF_INFO, "queue=%02x.%04x online=%d\n",
+		   AP_QID_CARD(zq->queue->qid),
+		   AP_QID_QUEUE(zq->queue->qid),
+		   online);
+
+	if (!online)
+		ap_flush_queue(zq->queue);
+	return count;
+}
+
+static DEVICE_ATTR(online, 0644, zcrypt_queue_online_show,
+		   zcrypt_queue_online_store);
+
+static struct attribute *zcrypt_queue_attrs[] = {
+	&dev_attr_online.attr,
+	NULL,
+};
+
+static struct attribute_group zcrypt_queue_attr_group = {
+	.attrs = zcrypt_queue_attrs,
+};
+
+void zcrypt_queue_force_online(struct zcrypt_queue *zq, int online)
+{
+	zq->online = online;
+	if (!online)
+		ap_flush_queue(zq->queue);
+}
+
+struct zcrypt_queue *zcrypt_queue_alloc(size_t max_response_size)
+{
+	struct zcrypt_queue *zq;
+
+	zq = kzalloc(sizeof(struct zcrypt_queue), GFP_KERNEL);
+	if (!zq)
+		return NULL;
+	zq->reply.message = kmalloc(max_response_size, GFP_KERNEL);
+	if (!zq->reply.message)
+		goto out_free;
+	zq->reply.length = max_response_size;
+	INIT_LIST_HEAD(&zq->list);
+	kref_init(&zq->refcount);
+	return zq;
+
+out_free:
+	kfree(zq);
+	return NULL;
+}
+EXPORT_SYMBOL(zcrypt_queue_alloc);
+
+void zcrypt_queue_free(struct zcrypt_queue *zq)
+{
+	kfree(zq->reply.message);
+	kfree(zq);
+}
+EXPORT_SYMBOL(zcrypt_queue_free);
+
+static void zcrypt_queue_release(struct kref *kref)
+{
+	struct zcrypt_queue *zq =
+		container_of(kref, struct zcrypt_queue, refcount);
+	zcrypt_queue_free(zq);
+}
+
+void zcrypt_queue_get(struct zcrypt_queue *zq)
+{
+	kref_get(&zq->refcount);
+}
+EXPORT_SYMBOL(zcrypt_queue_get);
+
+int zcrypt_queue_put(struct zcrypt_queue *zq)
+{
+	return kref_put(&zq->refcount, zcrypt_queue_release);
+}
+EXPORT_SYMBOL(zcrypt_queue_put);
+
+/**
+ * zcrypt_queue_register() - Register a crypto queue device.
+ * @zq: Pointer to a crypto queue device
+ *
+ * Register a crypto queue device. Returns 0 if successful.
+ */
+int zcrypt_queue_register(struct zcrypt_queue *zq)
+{
+	struct zcrypt_card *zc;
+	int rc;
+
+	spin_lock(&zcrypt_list_lock);
+	zc = zq->queue->card->private;
+	zcrypt_card_get(zc);
+	zq->zcard = zc;
+	zq->online = 1;	/* New devices are online by default. */
+
+	ZCRYPT_DBF(DBF_INFO, "queue=%02x.%04x register online=1\n",
+		   AP_QID_CARD(zq->queue->qid), AP_QID_QUEUE(zq->queue->qid));
+
+	list_add_tail(&zq->list, &zc->zqueues);
+	zcrypt_device_count++;
+	spin_unlock(&zcrypt_list_lock);
+
+	rc = sysfs_create_group(&zq->queue->ap_dev.device.kobj,
+				&zcrypt_queue_attr_group);
+	if (rc)
+		goto out;
+	get_device(&zq->queue->ap_dev.device);
+
+	if (zq->ops->rng) {
+		rc = zcrypt_rng_device_add();
+		if (rc)
+			goto out_unregister;
+	}
+	return 0;
+
+out_unregister:
+	sysfs_remove_group(&zq->queue->ap_dev.device.kobj,
+			   &zcrypt_queue_attr_group);
+	put_device(&zq->queue->ap_dev.device);
+out:
+	spin_lock(&zcrypt_list_lock);
+	list_del_init(&zq->list);
+	spin_unlock(&zcrypt_list_lock);
+	zcrypt_card_put(zc);
+	return rc;
+}
+EXPORT_SYMBOL(zcrypt_queue_register);
+
+/**
+ * zcrypt_queue_unregister(): Unregister a crypto queue device.
+ * @zq: Pointer to crypto queue device
+ *
+ * Unregister a crypto queue device.
+ */
+void zcrypt_queue_unregister(struct zcrypt_queue *zq)
+{
+	struct zcrypt_card *zc;
+
+	ZCRYPT_DBF(DBF_INFO, "queue=%02x.%04x unregister\n",
+		   AP_QID_CARD(zq->queue->qid), AP_QID_QUEUE(zq->queue->qid));
+
+	zc = zq->zcard;
+	spin_lock(&zcrypt_list_lock);
+	list_del_init(&zq->list);
+	zcrypt_device_count--;
+	spin_unlock(&zcrypt_list_lock);
+	zcrypt_card_put(zc);
+	if (zq->ops->rng)
+		zcrypt_rng_device_remove();
+	sysfs_remove_group(&zq->queue->ap_dev.device.kobj,
+			   &zcrypt_queue_attr_group);
+	put_device(&zq->queue->ap_dev.device);
+	zcrypt_queue_put(zq);
+}
+EXPORT_SYMBOL(zcrypt_queue_unregister);
diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h
index c8fed9fa1cca..968a0ab4b398 100644
--- a/drivers/s390/scsi/zfcp_ext.h
+++ b/drivers/s390/scsi/zfcp_ext.h
@@ -84,8 +84,8 @@ extern void zfcp_fc_link_test_work(struct work_struct *);
 extern void zfcp_fc_wka_ports_force_offline(struct zfcp_fc_wka_ports *);
 extern int zfcp_fc_gs_setup(struct zfcp_adapter *);
 extern void zfcp_fc_gs_destroy(struct zfcp_adapter *);
-extern int zfcp_fc_exec_bsg_job(struct fc_bsg_job *);
-extern int zfcp_fc_timeout_bsg_job(struct fc_bsg_job *);
+extern int zfcp_fc_exec_bsg_job(struct bsg_job *);
+extern int zfcp_fc_timeout_bsg_job(struct bsg_job *);
 extern void zfcp_fc_sym_name_update(struct work_struct *);
 extern unsigned int zfcp_fc_port_scan_backoff(void);
 extern void zfcp_fc_conditional_port_scan(struct zfcp_adapter *);
diff --git a/drivers/s390/scsi/zfcp_fc.c b/drivers/s390/scsi/zfcp_fc.c
index 237688af179b..7331eea67435 100644
--- a/drivers/s390/scsi/zfcp_fc.c
+++ b/drivers/s390/scsi/zfcp_fc.c
@@ -13,6 +13,7 @@
 #include <linux/slab.h>
 #include <linux/utsname.h>
 #include <linux/random.h>
+#include <linux/bsg-lib.h>
 #include <scsi/fc/fc_els.h>
 #include <scsi/libfc.h>
 #include "zfcp_ext.h"
@@ -885,26 +886,30 @@ out_free:
 
 static void zfcp_fc_ct_els_job_handler(void *data)
 {
-	struct fc_bsg_job *job = data;
+	struct bsg_job *job = data;
 	struct zfcp_fsf_ct_els *zfcp_ct_els = job->dd_data;
 	struct fc_bsg_reply *jr = job->reply;
 
 	jr->reply_payload_rcv_len = job->reply_payload.payload_len;
 	jr->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK;
 	jr->result = zfcp_ct_els->status ? -EIO : 0;
-	job->job_done(job);
+	bsg_job_done(job, jr->result, jr->reply_payload_rcv_len);
 }
 
-static struct zfcp_fc_wka_port *zfcp_fc_job_wka_port(struct fc_bsg_job *job)
+static struct zfcp_fc_wka_port *zfcp_fc_job_wka_port(struct bsg_job *job)
 {
 	u32 preamble_word1;
 	u8 gs_type;
 	struct zfcp_adapter *adapter;
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_rport *rport = fc_bsg_to_rport(job);
+	struct Scsi_Host *shost;
 
-	preamble_word1 = job->request->rqst_data.r_ct.preamble_word1;
+	preamble_word1 = bsg_request->rqst_data.r_ct.preamble_word1;
 	gs_type = (preamble_word1 & 0xff000000) >> 24;
 
-	adapter = (struct zfcp_adapter *) job->shost->hostdata[0];
+	shost = rport ? rport_to_shost(rport) : fc_bsg_to_shost(job);
+	adapter = (struct zfcp_adapter *) shost->hostdata[0];
 
 	switch (gs_type) {
 	case FC_FST_ALIAS:
@@ -924,7 +929,7 @@ static struct zfcp_fc_wka_port *zfcp_fc_job_wka_port(struct fc_bsg_job *job)
 
 static void zfcp_fc_ct_job_handler(void *data)
 {
-	struct fc_bsg_job *job = data;
+	struct bsg_job *job = data;
 	struct zfcp_fc_wka_port *wka_port;
 
 	wka_port = zfcp_fc_job_wka_port(job);
@@ -933,11 +938,12 @@ static void zfcp_fc_ct_job_handler(void *data)
 	zfcp_fc_ct_els_job_handler(data);
 }
 
-static int zfcp_fc_exec_els_job(struct fc_bsg_job *job,
+static int zfcp_fc_exec_els_job(struct bsg_job *job,
 				struct zfcp_adapter *adapter)
 {
 	struct zfcp_fsf_ct_els *els = job->dd_data;
-	struct fc_rport *rport = job->rport;
+	struct fc_rport *rport = fc_bsg_to_rport(job);
+	struct fc_bsg_request *bsg_request = job->request;
 	struct zfcp_port *port;
 	u32 d_id;
 
@@ -949,13 +955,13 @@ static int zfcp_fc_exec_els_job(struct fc_bsg_job *job,
 		d_id = port->d_id;
 		put_device(&port->dev);
 	} else
-		d_id = ntoh24(job->request->rqst_data.h_els.port_id);
+		d_id = ntoh24(bsg_request->rqst_data.h_els.port_id);
 
 	els->handler = zfcp_fc_ct_els_job_handler;
 	return zfcp_fsf_send_els(adapter, d_id, els, job->req->timeout / HZ);
 }
 
-static int zfcp_fc_exec_ct_job(struct fc_bsg_job *job,
+static int zfcp_fc_exec_ct_job(struct bsg_job *job,
 			       struct zfcp_adapter *adapter)
 {
 	int ret;
@@ -978,13 +984,15 @@ static int zfcp_fc_exec_ct_job(struct fc_bsg_job *job,
 	return ret;
 }
 
-int zfcp_fc_exec_bsg_job(struct fc_bsg_job *job)
+int zfcp_fc_exec_bsg_job(struct bsg_job *job)
 {
 	struct Scsi_Host *shost;
 	struct zfcp_adapter *adapter;
 	struct zfcp_fsf_ct_els *ct_els = job->dd_data;
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_rport *rport = fc_bsg_to_rport(job);
 
-	shost = job->rport ? rport_to_shost(job->rport) : job->shost;
+	shost = rport ? rport_to_shost(rport) : fc_bsg_to_shost(job);
 	adapter = (struct zfcp_adapter *)shost->hostdata[0];
 
 	if (!(atomic_read(&adapter->status) & ZFCP_STATUS_COMMON_OPEN))
@@ -994,7 +1002,7 @@ int zfcp_fc_exec_bsg_job(struct fc_bsg_job *job)
 	ct_els->resp = job->reply_payload.sg_list;
 	ct_els->handler_data = job;
 
-	switch (job->request->msgcode) {
+	switch (bsg_request->msgcode) {
 	case FC_BSG_RPT_ELS:
 	case FC_BSG_HST_ELS_NOLOGIN:
 		return zfcp_fc_exec_els_job(job, adapter);
@@ -1006,7 +1014,7 @@ int zfcp_fc_exec_bsg_job(struct fc_bsg_job *job)
 	}
 }
 
-int zfcp_fc_timeout_bsg_job(struct fc_bsg_job *job)
+int zfcp_fc_timeout_bsg_job(struct bsg_job *job)
 {
 	/* hardware tracks timeout, reset bsg timeout to not interfere */
 	return -EAGAIN;
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
index 8688ad4c825f..639ed4e6afd1 100644
--- a/drivers/s390/virtio/virtio_ccw.c
+++ b/drivers/s390/virtio/virtio_ccw.c
@@ -24,7 +24,7 @@
 #include <linux/wait.h>
 #include <linux/list.h>
 #include <linux/bitops.h>
-#include <linux/module.h>
+#include <linux/moduleparam.h>
 #include <linux/io.h>
 #include <linux/kvm_para.h>
 #include <linux/notifier.h>
@@ -235,16 +235,6 @@ static struct airq_info *new_airq_info(void)
 	return info;
 }
 
-static void destroy_airq_info(struct airq_info *info)
-{
-	if (!info)
-		return;
-
-	unregister_adapter_interrupt(&info->airq);
-	airq_iv_release(info->aiv);
-	kfree(info);
-}
-
 static unsigned long get_airq_indicator(struct virtqueue *vqs[], int nvqs,
 					u64 *first, void **airq_info)
 {
@@ -1294,7 +1284,6 @@ static struct ccw_device_id virtio_ids[] = {
 	{ CCW_DEVICE(0x3832, 0) },
 	{},
 };
-MODULE_DEVICE_TABLE(ccw, virtio_ids);
 
 static struct ccw_driver virtio_ccw_driver = {
 	.driver = {
@@ -1406,14 +1395,4 @@ static int __init virtio_ccw_init(void)
 	no_auto_parse();
 	return ccw_driver_register(&virtio_ccw_driver);
 }
-module_init(virtio_ccw_init);
-
-static void __exit virtio_ccw_exit(void)
-{
-	int i;
-
-	ccw_driver_unregister(&virtio_ccw_driver);
-	for (i = 0; i < MAX_AIRQ_AREAS; i++)
-		destroy_airq_info(airq_areas[i]);
-}
-module_exit(virtio_ccw_exit);
+device_initcall(virtio_ccw_init);
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index 3e2bdb90813c..dfa93347c752 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -263,6 +263,7 @@ config SCSI_SPI_ATTRS
 config SCSI_FC_ATTRS
 	tristate "FiberChannel Transport Attributes"
 	depends on SCSI && NET
+	select BLK_DEV_BSGLIB
 	select SCSI_NETLINK
 	help
 	  If you wish to export transport-specific information about
@@ -743,40 +744,18 @@ config SCSI_ISCI
 	  control unit found in the Intel(R) C600 series chipset.
 
 config SCSI_GENERIC_NCR5380
-	tristate "Generic NCR5380/53c400 SCSI PIO support"
-	depends on ISA && SCSI
+	tristate "Generic NCR5380/53c400 SCSI ISA card support"
+	depends on ISA && SCSI && HAS_IOPORT_MAP
 	select SCSI_SPI_ATTRS
 	---help---
-	  This is a driver for the old NCR 53c80 series of SCSI controllers
-	  on boards using PIO. Most boards such as the Trantor T130 fit this
-	  category, along with a large number of ISA 8bit controllers shipped
-	  for free with SCSI scanners. If you have a PAS16, T128 or DMX3191
-	  you should select the specific driver for that card rather than
-	  generic 5380 support.
-
-	  It is explained in section 3.8 of the SCSI-HOWTO, available from
-	  <http://www.tldp.org/docs.html#howto>.  If it doesn't work out
-	  of the box, you may have to change some settings in
-	  <file:drivers/scsi/g_NCR5380.h>.
+	  This is a driver for old ISA card SCSI controllers based on a
+	  NCR 5380, 53C80, 53C400, 53C400A, or DTC 436 device.
+	  Most boards such as the Trantor T130 fit this category, as do
+	  various 8-bit and 16-bit ISA cards bundled with SCSI scanners.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called g_NCR5380.
 
-config SCSI_GENERIC_NCR5380_MMIO
-	tristate "Generic NCR5380/53c400 SCSI MMIO support"
-	depends on ISA && SCSI
-	select SCSI_SPI_ATTRS
-	---help---
-	  This is a driver for the old NCR 53c80 series of SCSI controllers
-	  on boards using memory mapped I/O. 
-	  It is explained in section 3.8 of the SCSI-HOWTO, available from
-	  <http://www.tldp.org/docs.html#howto>.  If it doesn't work out
-	  of the box, you may have to change some settings in
-	  <file:drivers/scsi/g_NCR5380.h>.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called g_NCR5380_mmio.
-
 config SCSI_IPS
 	tristate "IBM ServeRAID support"
 	depends on PCI && SCSI
diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile
index 38d938d7fe67..a2d03957cbe2 100644
--- a/drivers/scsi/Makefile
+++ b/drivers/scsi/Makefile
@@ -74,7 +74,6 @@ obj-$(CONFIG_SCSI_ISCI)		+= isci/
 obj-$(CONFIG_SCSI_IPS)		+= ips.o
 obj-$(CONFIG_SCSI_FUTURE_DOMAIN)+= fdomain.o
 obj-$(CONFIG_SCSI_GENERIC_NCR5380) += g_NCR5380.o
-obj-$(CONFIG_SCSI_GENERIC_NCR5380_MMIO) += g_NCR5380_mmio.o
 obj-$(CONFIG_SCSI_NCR53C406A)	+= NCR53c406a.o
 obj-$(CONFIG_SCSI_NCR_D700)	+= 53c700.o NCR_D700.o
 obj-$(CONFIG_SCSI_NCR_Q720)	+= NCR_Q720_mod.o
@@ -173,6 +172,7 @@ hv_storvsc-y			:= storvsc_drv.o
 
 sd_mod-objs	:= sd.o
 sd_mod-$(CONFIG_BLK_DEV_INTEGRITY) += sd_dif.o
+sd_mod-$(CONFIG_BLK_DEV_ZONED) += sd_zbc.o
 
 sr_mod-objs	:= sr.o sr_ioctl.o sr_vendor.o
 ncr53c8xx-flags-$(CONFIG_SCSI_ZALON) \
diff --git a/drivers/scsi/NCR5380.c b/drivers/scsi/NCR5380.c
index 790babc5ef66..d849ffa378b1 100644
--- a/drivers/scsi/NCR5380.c
+++ b/drivers/scsi/NCR5380.c
@@ -121,9 +121,10 @@
  *
  * Either real DMA *or* pseudo DMA may be implemented
  *
- * NCR5380_dma_write_setup(instance, src, count) - initialize
- * NCR5380_dma_read_setup(instance, dst, count) - initialize
- * NCR5380_dma_residual(instance); - residual count
+ * NCR5380_dma_xfer_len   - determine size of DMA/PDMA transfer
+ * NCR5380_dma_send_setup - execute DMA/PDMA from memory to 5380
+ * NCR5380_dma_recv_setup - execute DMA/PDMA from 5380 to memory
+ * NCR5380_dma_residual   - residual byte count
  *
  * The generic driver is initialized by calling NCR5380_init(instance),
  * after setting the appropriate host specific fields and ID.  If the
@@ -178,7 +179,7 @@ static inline void initialize_SCp(struct scsi_cmnd *cmd)
 
 /**
  * NCR5380_poll_politely2 - wait for two chip register values
- * @instance: controller to poll
+ * @hostdata: host private data
  * @reg1: 5380 register to poll
  * @bit1: Bitmask to check
  * @val1: Expected value
@@ -195,18 +196,14 @@ static inline void initialize_SCp(struct scsi_cmnd *cmd)
  * Returns 0 if either or both event(s) occurred otherwise -ETIMEDOUT.
  */
 
-static int NCR5380_poll_politely2(struct Scsi_Host *instance,
-                                  int reg1, int bit1, int val1,
-                                  int reg2, int bit2, int val2, int wait)
+static int NCR5380_poll_politely2(struct NCR5380_hostdata *hostdata,
+                                  unsigned int reg1, u8 bit1, u8 val1,
+                                  unsigned int reg2, u8 bit2, u8 val2,
+                                  unsigned long wait)
 {
-	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	unsigned long n = hostdata->poll_loops;
 	unsigned long deadline = jiffies + wait;
-	unsigned long n;
 
-	/* Busy-wait for up to 10 ms */
-	n = min(10000U, jiffies_to_usecs(wait));
-	n *= hostdata->accesses_per_ms;
-	n /= 2000;
 	do {
 		if ((NCR5380_read(reg1) & bit1) == val1)
 			return 0;
@@ -288,6 +285,7 @@ mrs[] = {
 
 static void NCR5380_print(struct Scsi_Host *instance)
 {
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	unsigned char status, data, basr, mr, icr, i;
 
 	data = NCR5380_read(CURRENT_SCSI_DATA_REG);
@@ -337,6 +335,7 @@ static struct {
 
 static void NCR5380_print_phase(struct Scsi_Host *instance)
 {
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	unsigned char status;
 	int i;
 
@@ -441,14 +440,14 @@ static void prepare_info(struct Scsi_Host *instance)
 	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 
 	snprintf(hostdata->info, sizeof(hostdata->info),
-	         "%s, io_port 0x%lx, n_io_port %d, "
-	         "base 0x%lx, irq %d, "
+	         "%s, irq %d, "
+		 "io_port 0x%lx, base 0x%lx, "
 	         "can_queue %d, cmd_per_lun %d, "
 	         "sg_tablesize %d, this_id %d, "
 	         "flags { %s%s%s}, "
 	         "options { %s} ",
-	         instance->hostt->name, instance->io_port, instance->n_io_port,
-	         instance->base, instance->irq,
+	         instance->hostt->name, instance->irq,
+		 hostdata->io_port, hostdata->base,
 	         instance->can_queue, instance->cmd_per_lun,
 	         instance->sg_tablesize, instance->this_id,
 	         hostdata->flags & FLAG_DMA_FIXUP     ? "DMA_FIXUP "     : "",
@@ -482,6 +481,7 @@ static int NCR5380_init(struct Scsi_Host *instance, int flags)
 	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	int i;
 	unsigned long deadline;
+	unsigned long accesses_per_ms;
 
 	instance->max_lun = 7;
 
@@ -530,7 +530,8 @@ static int NCR5380_init(struct Scsi_Host *instance, int flags)
 		++i;
 		cpu_relax();
 	} while (time_is_after_jiffies(deadline));
-	hostdata->accesses_per_ms = i / 256;
+	accesses_per_ms = i / 256;
+	hostdata->poll_loops = NCR5380_REG_POLL_TIME * accesses_per_ms / 2;
 
 	return 0;
 }
@@ -560,7 +561,7 @@ static int NCR5380_maybe_reset_bus(struct Scsi_Host *instance)
 		case 3:
 		case 5:
 			shost_printk(KERN_ERR, instance, "SCSI bus busy, waiting up to five seconds\n");
-			NCR5380_poll_politely(instance,
+			NCR5380_poll_politely(hostdata,
 			                      STATUS_REG, SR_BSY, 0, 5 * HZ);
 			break;
 		case 2:
@@ -871,7 +872,7 @@ static void NCR5380_dma_complete(struct Scsi_Host *instance)
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 	NCR5380_read(RESET_PARITY_INTERRUPT_REG);
 
-	transferred = hostdata->dma_len - NCR5380_dma_residual(instance);
+	transferred = hostdata->dma_len - NCR5380_dma_residual(hostdata);
 	hostdata->dma_len = 0;
 
 	data = (unsigned char **)&hostdata->connected->SCp.ptr;
@@ -994,7 +995,7 @@ static irqreturn_t __maybe_unused NCR5380_intr(int irq, void *dev_id)
 		}
 		handled = 1;
 	} else {
-		shost_printk(KERN_NOTICE, instance, "interrupt without IRQ bit\n");
+		dsprintk(NDEBUG_INTR, instance, "interrupt without IRQ bit\n");
 #ifdef SUN3_SCSI_VME
 		dregs->csr |= CSR_DMA_ENABLE;
 #endif
@@ -1075,7 +1076,7 @@ static struct scsi_cmnd *NCR5380_select(struct Scsi_Host *instance,
 	 */
 
 	spin_unlock_irq(&hostdata->lock);
-	err = NCR5380_poll_politely2(instance, MODE_REG, MR_ARBITRATE, 0,
+	err = NCR5380_poll_politely2(hostdata, MODE_REG, MR_ARBITRATE, 0,
 	                INITIATOR_COMMAND_REG, ICR_ARBITRATION_PROGRESS,
 	                                       ICR_ARBITRATION_PROGRESS, HZ);
 	spin_lock_irq(&hostdata->lock);
@@ -1201,7 +1202,7 @@ static struct scsi_cmnd *NCR5380_select(struct Scsi_Host *instance,
 	 * selection.
 	 */
 
-	err = NCR5380_poll_politely(instance, STATUS_REG, SR_BSY, SR_BSY,
+	err = NCR5380_poll_politely(hostdata, STATUS_REG, SR_BSY, SR_BSY,
 	                            msecs_to_jiffies(250));
 
 	if ((NCR5380_read(STATUS_REG) & (SR_SEL | SR_IO)) == (SR_SEL | SR_IO)) {
@@ -1247,7 +1248,7 @@ static struct scsi_cmnd *NCR5380_select(struct Scsi_Host *instance,
 
 	/* Wait for start of REQ/ACK handshake */
 
-	err = NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, HZ);
+	err = NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, SR_REQ, HZ);
 	spin_lock_irq(&hostdata->lock);
 	if (err < 0) {
 		shost_printk(KERN_ERR, instance, "select: REQ timeout\n");
@@ -1318,6 +1319,7 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance,
 				unsigned char *phase, int *count,
 				unsigned char **data)
 {
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	unsigned char p = *phase, tmp;
 	int c = *count;
 	unsigned char *d = *data;
@@ -1336,7 +1338,7 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance,
 		 * valid
 		 */
 
-		if (NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, HZ) < 0)
+		if (NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, SR_REQ, HZ) < 0)
 			break;
 
 		dsprintk(NDEBUG_HANDSHAKE, instance, "REQ asserted\n");
@@ -1381,7 +1383,7 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance,
 			NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ACK);
 		}
 
-		if (NCR5380_poll_politely(instance,
+		if (NCR5380_poll_politely(hostdata,
 		                          STATUS_REG, SR_REQ, 0, 5 * HZ) < 0)
 			break;
 
@@ -1440,6 +1442,7 @@ static int NCR5380_transfer_pio(struct Scsi_Host *instance,
 
 static void do_reset(struct Scsi_Host *instance)
 {
+	struct NCR5380_hostdata __maybe_unused *hostdata = shost_priv(instance);
 	unsigned long flags;
 
 	local_irq_save(flags);
@@ -1462,6 +1465,7 @@ static void do_reset(struct Scsi_Host *instance)
 
 static int do_abort(struct Scsi_Host *instance)
 {
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	unsigned char *msgptr, phase, tmp;
 	int len;
 	int rc;
@@ -1479,7 +1483,7 @@ static int do_abort(struct Scsi_Host *instance)
 	 * the target sees, so we just handshake.
 	 */
 
-	rc = NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, 10 * HZ);
+	rc = NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, SR_REQ, 10 * HZ);
 	if (rc < 0)
 		goto timeout;
 
@@ -1490,7 +1494,7 @@ static int do_abort(struct Scsi_Host *instance)
 	if (tmp != PHASE_MSGOUT) {
 		NCR5380_write(INITIATOR_COMMAND_REG,
 		              ICR_BASE | ICR_ASSERT_ATN | ICR_ASSERT_ACK);
-		rc = NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, 0, 3 * HZ);
+		rc = NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, 0, 3 * HZ);
 		if (rc < 0)
 			goto timeout;
 		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_ATN);
@@ -1575,9 +1579,9 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance,
 		 * starting the NCR. This is also the cleaner way for the TT.
 		 */
 		if (p & SR_IO)
-			result = NCR5380_dma_recv_setup(instance, d, c);
+			result = NCR5380_dma_recv_setup(hostdata, d, c);
 		else
-			result = NCR5380_dma_send_setup(instance, d, c);
+			result = NCR5380_dma_send_setup(hostdata, d, c);
 	}
 
 	/*
@@ -1609,9 +1613,9 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance,
 		 * NCR access, else the DMA setup gets trashed!
 		 */
 		if (p & SR_IO)
-			result = NCR5380_dma_recv_setup(instance, d, c);
+			result = NCR5380_dma_recv_setup(hostdata, d, c);
 		else
-			result = NCR5380_dma_send_setup(instance, d, c);
+			result = NCR5380_dma_send_setup(hostdata, d, c);
 	}
 
 	/* On failure, NCR5380_dma_xxxx_setup() returns a negative int. */
@@ -1678,12 +1682,12 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance,
 			 * byte.
 			 */
 
-			if (NCR5380_poll_politely(instance, BUS_AND_STATUS_REG,
+			if (NCR5380_poll_politely(hostdata, BUS_AND_STATUS_REG,
 			                          BASR_DRQ, BASR_DRQ, HZ) < 0) {
 				result = -1;
 				shost_printk(KERN_ERR, instance, "PDMA read: DRQ timeout\n");
 			}
-			if (NCR5380_poll_politely(instance, STATUS_REG,
+			if (NCR5380_poll_politely(hostdata, STATUS_REG,
 			                          SR_REQ, 0, HZ) < 0) {
 				result = -1;
 				shost_printk(KERN_ERR, instance, "PDMA read: !REQ timeout\n");
@@ -1694,7 +1698,7 @@ static int NCR5380_transfer_dma(struct Scsi_Host *instance,
 			 * Wait for the last byte to be sent.  If REQ is being asserted for
 			 * the byte we're interested, we'll ACK it and it will go false.
 			 */
-			if (NCR5380_poll_politely2(instance,
+			if (NCR5380_poll_politely2(hostdata,
 			     BUS_AND_STATUS_REG, BASR_DRQ, BASR_DRQ,
 			     BUS_AND_STATUS_REG, BASR_PHASE_MATCH, 0, HZ) < 0) {
 				result = -1;
@@ -1751,22 +1755,26 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 				NCR5380_dprint_phase(NDEBUG_INFORMATION, instance);
 			}
 #ifdef CONFIG_SUN3
-			if (phase == PHASE_CMDOUT) {
-				void *d;
-				unsigned long count;
+			if (phase == PHASE_CMDOUT &&
+			    sun3_dma_setup_done != cmd) {
+				int count;
 
 				if (!cmd->SCp.this_residual && cmd->SCp.buffers_residual) {
-					count = cmd->SCp.buffer->length;
-					d = sg_virt(cmd->SCp.buffer);
-				} else {
-					count = cmd->SCp.this_residual;
-					d = cmd->SCp.ptr;
+					++cmd->SCp.buffer;
+					--cmd->SCp.buffers_residual;
+					cmd->SCp.this_residual = cmd->SCp.buffer->length;
+					cmd->SCp.ptr = sg_virt(cmd->SCp.buffer);
 				}
 
-				if (sun3_dma_setup_done != cmd &&
-				    sun3scsi_dma_xfer_len(count, cmd) > 0) {
-					sun3scsi_dma_setup(instance, d, count,
-					                   rq_data_dir(cmd->request));
+				count = sun3scsi_dma_xfer_len(hostdata, cmd);
+
+				if (count > 0) {
+					if (rq_data_dir(cmd->request))
+						sun3scsi_dma_send_setup(hostdata,
+						                        cmd->SCp.ptr, count);
+					else
+						sun3scsi_dma_recv_setup(hostdata,
+						                        cmd->SCp.ptr, count);
 					sun3_dma_setup_done = cmd;
 				}
 #ifdef SUN3_SCSI_VME
@@ -1827,7 +1835,7 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 
 				transfersize = 0;
 				if (!cmd->device->borken)
-					transfersize = NCR5380_dma_xfer_len(instance, cmd, phase);
+					transfersize = NCR5380_dma_xfer_len(hostdata, cmd);
 
 				if (transfersize > 0) {
 					len = transfersize;
@@ -2073,7 +2081,7 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 			} /* switch(phase) */
 		} else {
 			spin_unlock_irq(&hostdata->lock);
-			NCR5380_poll_politely(instance, STATUS_REG, SR_REQ, SR_REQ, HZ);
+			NCR5380_poll_politely(hostdata, STATUS_REG, SR_REQ, SR_REQ, HZ);
 			spin_lock_irq(&hostdata->lock);
 		}
 	}
@@ -2119,7 +2127,7 @@ static void NCR5380_reselect(struct Scsi_Host *instance)
 	 */
 
 	NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE | ICR_ASSERT_BSY);
-	if (NCR5380_poll_politely(instance,
+	if (NCR5380_poll_politely(hostdata,
 	                          STATUS_REG, SR_SEL, 0, 2 * HZ) < 0) {
 		NCR5380_write(INITIATOR_COMMAND_REG, ICR_BASE);
 		return;
@@ -2130,7 +2138,7 @@ static void NCR5380_reselect(struct Scsi_Host *instance)
 	 * Wait for target to go into MSGIN.
 	 */
 
-	if (NCR5380_poll_politely(instance,
+	if (NCR5380_poll_politely(hostdata,
 	                          STATUS_REG, SR_REQ, SR_REQ, 2 * HZ) < 0) {
 		do_abort(instance);
 		return;
@@ -2204,22 +2212,25 @@ static void NCR5380_reselect(struct Scsi_Host *instance)
 	}
 
 #ifdef CONFIG_SUN3
-	{
-		void *d;
-		unsigned long count;
+	if (sun3_dma_setup_done != tmp) {
+		int count;
 
 		if (!tmp->SCp.this_residual && tmp->SCp.buffers_residual) {
-			count = tmp->SCp.buffer->length;
-			d = sg_virt(tmp->SCp.buffer);
-		} else {
-			count = tmp->SCp.this_residual;
-			d = tmp->SCp.ptr;
+			++tmp->SCp.buffer;
+			--tmp->SCp.buffers_residual;
+			tmp->SCp.this_residual = tmp->SCp.buffer->length;
+			tmp->SCp.ptr = sg_virt(tmp->SCp.buffer);
 		}
 
-		if (sun3_dma_setup_done != tmp &&
-		    sun3scsi_dma_xfer_len(count, tmp) > 0) {
-			sun3scsi_dma_setup(instance, d, count,
-			                   rq_data_dir(tmp->request));
+		count = sun3scsi_dma_xfer_len(hostdata, tmp);
+
+		if (count > 0) {
+			if (rq_data_dir(tmp->request))
+				sun3scsi_dma_send_setup(hostdata,
+				                        tmp->SCp.ptr, count);
+			else
+				sun3scsi_dma_recv_setup(hostdata,
+				                        tmp->SCp.ptr, count);
 			sun3_dma_setup_done = tmp;
 		}
 	}
diff --git a/drivers/scsi/NCR5380.h b/drivers/scsi/NCR5380.h
index 965d92339455..3c6ce5434449 100644
--- a/drivers/scsi/NCR5380.h
+++ b/drivers/scsi/NCR5380.h
@@ -219,27 +219,32 @@
 #define FLAG_TOSHIBA_DELAY		128	/* Allow for borken CD-ROMs */
 
 struct NCR5380_hostdata {
-	NCR5380_implementation_fields;		/* implementation specific */
-	struct Scsi_Host *host;			/* Host backpointer */
-	unsigned char id_mask, id_higher_mask;	/* 1 << id, all bits greater */
-	unsigned char busy[8];			/* index = target, bit = lun */
-	int dma_len;				/* requested length of DMA */
-	unsigned char last_message;		/* last message OUT */
-	struct scsi_cmnd *connected;		/* currently connected cmnd */
-	struct scsi_cmnd *selecting;		/* cmnd to be connected */
-	struct list_head unissued;		/* waiting to be issued */
-	struct list_head autosense;		/* priority issue queue */
-	struct list_head disconnected;		/* waiting for reconnect */
-	spinlock_t lock;			/* protects this struct */
-	int flags;
-	struct scsi_eh_save ses;
-	struct scsi_cmnd *sensing;
+	NCR5380_implementation_fields;		/* Board-specific data */
+	u8 __iomem *io;				/* Remapped 5380 address */
+	u8 __iomem *pdma_io;			/* Remapped PDMA address */
+	unsigned long poll_loops;		/* Register polling limit */
+	spinlock_t lock;			/* Protects this struct */
+	struct scsi_cmnd *connected;		/* Currently connected cmnd */
+	struct list_head disconnected;		/* Waiting for reconnect */
+	struct Scsi_Host *host;			/* SCSI host backpointer */
+	struct workqueue_struct *work_q;	/* SCSI host work queue */
+	struct work_struct main_task;		/* Work item for main loop */
+	int flags;				/* Board-specific quirks */
+	int dma_len;				/* Requested length of DMA */
+	int read_overruns;	/* Transfer size reduction for DMA erratum */
+	unsigned long io_port;			/* Device IO port */
+	unsigned long base;			/* Device base address */
+	struct list_head unissued;		/* Waiting to be issued */
+	struct scsi_cmnd *selecting;		/* Cmnd to be connected */
+	struct list_head autosense;		/* Priority cmnd queue */
+	struct scsi_cmnd *sensing;		/* Cmnd needing autosense */
+	struct scsi_eh_save ses;		/* Cmnd state saved for EH */
+	unsigned char busy[8];			/* Index = target, bit = lun */
+	unsigned char id_mask;			/* 1 << Host ID */
+	unsigned char id_higher_mask;		/* All bits above id_mask */
+	unsigned char last_message;		/* Last Message Out */
+	unsigned long region_size;		/* Size of address/port range */
 	char info[256];
-	int read_overruns;                /* number of bytes to cut from a
-	                                   * transfer to handle chip overruns */
-	struct work_struct main_task;
-	struct workqueue_struct *work_q;
-	unsigned long accesses_per_ms;	/* chip register accesses per ms */
 };
 
 #ifdef __KERNEL__
@@ -252,6 +257,9 @@ struct NCR5380_cmd {
 
 #define NCR5380_PIO_CHUNK_SIZE		256
 
+/* Time limit (ms) to poll registers when IRQs are disabled, e.g. during PDMA */
+#define NCR5380_REG_POLL_TIME		15
+
 static inline struct scsi_cmnd *NCR5380_to_scmd(struct NCR5380_cmd *ncmd_ptr)
 {
 	return ((struct scsi_cmnd *)ncmd_ptr) - 1;
@@ -294,14 +302,45 @@ static void NCR5380_reselect(struct Scsi_Host *instance);
 static struct scsi_cmnd *NCR5380_select(struct Scsi_Host *, struct scsi_cmnd *);
 static int NCR5380_transfer_dma(struct Scsi_Host *instance, unsigned char *phase, int *count, unsigned char **data);
 static int NCR5380_transfer_pio(struct Scsi_Host *instance, unsigned char *phase, int *count, unsigned char **data);
-static int NCR5380_poll_politely2(struct Scsi_Host *, int, int, int, int, int, int, int);
+static int NCR5380_poll_politely2(struct NCR5380_hostdata *,
+                                  unsigned int, u8, u8,
+                                  unsigned int, u8, u8, unsigned long);
 
-static inline int NCR5380_poll_politely(struct Scsi_Host *instance,
-					int reg, int bit, int val, int wait)
+static inline int NCR5380_poll_politely(struct NCR5380_hostdata *hostdata,
+                                        unsigned int reg, u8 bit, u8 val,
+                                        unsigned long wait)
 {
-	return NCR5380_poll_politely2(instance, reg, bit, val,
+	if ((NCR5380_read(reg) & bit) == val)
+		return 0;
+
+	return NCR5380_poll_politely2(hostdata, reg, bit, val,
 						reg, bit, val, wait);
 }
 
+static int NCR5380_dma_xfer_len(struct NCR5380_hostdata *,
+                                struct scsi_cmnd *);
+static int NCR5380_dma_send_setup(struct NCR5380_hostdata *,
+                                  unsigned char *, int);
+static int NCR5380_dma_recv_setup(struct NCR5380_hostdata *,
+                                  unsigned char *, int);
+static int NCR5380_dma_residual(struct NCR5380_hostdata *);
+
+static inline int NCR5380_dma_xfer_none(struct NCR5380_hostdata *hostdata,
+                                        struct scsi_cmnd *cmd)
+{
+	return 0;
+}
+
+static inline int NCR5380_dma_setup_none(struct NCR5380_hostdata *hostdata,
+                                         unsigned char *data, int count)
+{
+	return 0;
+}
+
+static inline int NCR5380_dma_residual_none(struct NCR5380_hostdata *hostdata)
+{
+	return 0;
+}
+
 #endif				/* __KERNEL__ */
 #endif				/* NCR5380_H */
diff --git a/drivers/scsi/aacraid/aacraid.h b/drivers/scsi/aacraid/aacraid.h
index 969c312de1be..f059c14efa0c 100644
--- a/drivers/scsi/aacraid/aacraid.h
+++ b/drivers/scsi/aacraid/aacraid.h
@@ -1246,7 +1246,6 @@ struct aac_dev
 	u32			max_msix;	/* max. MSI-X vectors */
 	u32			vector_cap;	/* MSI-X vector capab.*/
 	int			msi_enabled;	/* MSI/MSI-X enabled */
-	struct msix_entry	msixentry[AAC_MAX_MSIX];
 	struct aac_msix_ctx	aac_msix[AAC_MAX_MSIX]; /* context */
 	u8			adapter_shutdown;
 	u32			handle_pci_error;
diff --git a/drivers/scsi/aacraid/comminit.c b/drivers/scsi/aacraid/comminit.c
index 341ea327ae79..4f56b1003cc7 100644
--- a/drivers/scsi/aacraid/comminit.c
+++ b/drivers/scsi/aacraid/comminit.c
@@ -378,16 +378,12 @@ void aac_define_int_mode(struct aac_dev *dev)
 	if (msi_count > AAC_MAX_MSIX)
 		msi_count = AAC_MAX_MSIX;
 
-	for (i = 0; i < msi_count; i++)
-		dev->msixentry[i].entry = i;
-
 	if (msi_count > 1 &&
 	    pci_find_capability(dev->pdev, PCI_CAP_ID_MSIX)) {
 		min_msix = 2;
-		i = pci_enable_msix_range(dev->pdev,
-				    dev->msixentry,
-				    min_msix,
-				    msi_count);
+		i = pci_alloc_irq_vectors(dev->pdev,
+					  min_msix, msi_count,
+					  PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
 		if (i > 0) {
 			dev->msi_enabled = 1;
 			msi_count = i;
diff --git a/drivers/scsi/aacraid/commsup.c b/drivers/scsi/aacraid/commsup.c
index 0aeecec1f5ea..9e7551fe4b19 100644
--- a/drivers/scsi/aacraid/commsup.c
+++ b/drivers/scsi/aacraid/commsup.c
@@ -2043,30 +2043,22 @@ int aac_acquire_irq(struct aac_dev *dev)
 	int i;
 	int j;
 	int ret = 0;
-	int cpu;
 
-	cpu = cpumask_first(cpu_online_mask);
 	if (!dev->sync_mode && dev->msi_enabled && dev->max_msix > 1) {
 		for (i = 0; i < dev->max_msix; i++) {
 			dev->aac_msix[i].vector_no = i;
 			dev->aac_msix[i].dev = dev;
-			if (request_irq(dev->msixentry[i].vector,
+			if (request_irq(pci_irq_vector(dev->pdev, i),
 					dev->a_ops.adapter_intr,
 					0, "aacraid", &(dev->aac_msix[i]))) {
 				printk(KERN_ERR "%s%d: Failed to register IRQ for vector %d.\n",
 						dev->name, dev->id, i);
 				for (j = 0 ; j < i ; j++)
-					free_irq(dev->msixentry[j].vector,
+					free_irq(pci_irq_vector(dev->pdev, j),
 						 &(dev->aac_msix[j]));
 				pci_disable_msix(dev->pdev);
 				ret = -1;
 			}
-			if (irq_set_affinity_hint(dev->msixentry[i].vector,
-							get_cpu_mask(cpu))) {
-				printk(KERN_ERR "%s%d: Failed to set IRQ affinity for cpu %d\n",
-					    dev->name, dev->id, cpu);
-			}
-			cpu = cpumask_next(cpu, cpu_online_mask);
 		}
 	} else {
 		dev->aac_msix[0].vector_no = 0;
@@ -2096,16 +2088,9 @@ void aac_free_irq(struct aac_dev *dev)
 	    dev->pdev->device == PMC_DEVICE_S8 ||
 	    dev->pdev->device == PMC_DEVICE_S9) {
 		if (dev->max_msix > 1) {
-			for (i = 0; i < dev->max_msix; i++) {
-				if (irq_set_affinity_hint(
-					dev->msixentry[i].vector, NULL)) {
-					printk(KERN_ERR "%s%d: Failed to reset IRQ affinity for cpu %d\n",
-					    dev->name, dev->id, cpu);
-				}
-				cpu = cpumask_next(cpu, cpu_online_mask);
-				free_irq(dev->msixentry[i].vector,
-						&(dev->aac_msix[i]));
-			}
+			for (i = 0; i < dev->max_msix; i++)
+				free_irq(pci_irq_vector(dev->pdev, i),
+					 &(dev->aac_msix[i]));
 		} else {
 			free_irq(dev->pdev->irq, &(dev->aac_msix[0]));
 		}
diff --git a/drivers/scsi/aacraid/linit.c b/drivers/scsi/aacraid/linit.c
index 79871f3519ff..e4f3e22fcbd9 100644
--- a/drivers/scsi/aacraid/linit.c
+++ b/drivers/scsi/aacraid/linit.c
@@ -1071,7 +1071,6 @@ static struct scsi_host_template aac_driver_template = {
 static void __aac_shutdown(struct aac_dev * aac)
 {
 	int i;
-	int cpu;
 
 	aac_send_shutdown(aac);
 
@@ -1087,24 +1086,13 @@ static void __aac_shutdown(struct aac_dev * aac)
 		kthread_stop(aac->thread);
 	}
 	aac_adapter_disable_int(aac);
-	cpu = cpumask_first(cpu_online_mask);
 	if (aac->pdev->device == PMC_DEVICE_S6 ||
 	    aac->pdev->device == PMC_DEVICE_S7 ||
 	    aac->pdev->device == PMC_DEVICE_S8 ||
 	    aac->pdev->device == PMC_DEVICE_S9) {
 		if (aac->max_msix > 1) {
 			for (i = 0; i < aac->max_msix; i++) {
-				if (irq_set_affinity_hint(
-				    aac->msixentry[i].vector,
-				    NULL)) {
-					printk(KERN_ERR "%s%d: Failed to reset IRQ affinity for cpu %d\n",
-						aac->name,
-						aac->id,
-						cpu);
-				}
-				cpu = cpumask_next(cpu,
-						cpu_online_mask);
-				free_irq(aac->msixentry[i].vector,
+				free_irq(pci_irq_vector(aac->pdev, i),
 					 &(aac->aac_msix[i]));
 			}
 		} else {
@@ -1350,7 +1338,7 @@ static void aac_release_resources(struct aac_dev *aac)
 	    aac->pdev->device == PMC_DEVICE_S9) {
 		if (aac->max_msix > 1) {
 			for (i = 0; i < aac->max_msix; i++)
-				free_irq(aac->msixentry[i].vector,
+				free_irq(pci_irq_vector(aac->pdev, i),
 					&(aac->aac_msix[i]));
 		} else {
 			free_irq(aac->pdev->irq, &(aac->aac_msix[0]));
@@ -1396,13 +1384,13 @@ static int aac_acquire_resources(struct aac_dev *dev)
 			dev->aac_msix[i].vector_no = i;
 			dev->aac_msix[i].dev = dev;
 
-			if (request_irq(dev->msixentry[i].vector,
+			if (request_irq(pci_irq_vector(dev->pdev, i),
 					dev->a_ops.adapter_intr,
 					0, "aacraid", &(dev->aac_msix[i]))) {
 				printk(KERN_ERR "%s%d: Failed to register IRQ for vector %d.\n",
 						name, instance, i);
 				for (j = 0 ; j < i ; j++)
-					free_irq(dev->msixentry[j].vector,
+					free_irq(pci_irq_vector(dev->pdev, j),
 						 &(dev->aac_msix[j]));
 				pci_disable_msix(dev->pdev);
 				goto error_iounmap;
diff --git a/drivers/scsi/advansys.c b/drivers/scsi/advansys.c
index febbd83e2ecd..81dd0927246b 100644
--- a/drivers/scsi/advansys.c
+++ b/drivers/scsi/advansys.c
@@ -11030,6 +11030,9 @@ static int advansys_board_found(struct Scsi_Host *shost, unsigned int iop,
 		ASC_DBG(2, "AdvInitGetConfig()\n");
 
 		ret = AdvInitGetConfig(pdev, shost) ? -ENODEV : 0;
+#else
+		share_irq = 0;
+		ret = -ENODEV;
 #endif /* CONFIG_PCI */
 	}
 
diff --git a/drivers/scsi/aic7xxx/aicasm/aicasm.c b/drivers/scsi/aic7xxx/aicasm/aicasm.c
index 2e3117aa382f..21ac265280bf 100644
--- a/drivers/scsi/aic7xxx/aicasm/aicasm.c
+++ b/drivers/scsi/aic7xxx/aicasm/aicasm.c
@@ -254,7 +254,7 @@ main(int argc, char *argv[])
 	argv += optind;
 
 	if (argc != 1) {
-		fprintf(stderr, "%s: No input file specifiled\n", appname);
+		fprintf(stderr, "%s: No input file specified\n", appname);
 		usage();
 		/* NOTREACHED */
 	}
diff --git a/drivers/scsi/aic94xx/aic94xx_hwi.c b/drivers/scsi/aic94xx/aic94xx_hwi.c
index 7c713f797535..f2671a8fa7e3 100644
--- a/drivers/scsi/aic94xx/aic94xx_hwi.c
+++ b/drivers/scsi/aic94xx/aic94xx_hwi.c
@@ -228,8 +228,11 @@ static int asd_init_scbs(struct asd_ha_struct *asd_ha)
 	bitmap_bytes = (asd_ha->seq.tc_index_bitmap_bits+7)/8;
 	bitmap_bytes = BITS_TO_LONGS(bitmap_bytes*8)*sizeof(unsigned long);
 	asd_ha->seq.tc_index_bitmap = kzalloc(bitmap_bytes, GFP_KERNEL);
-	if (!asd_ha->seq.tc_index_bitmap)
+	if (!asd_ha->seq.tc_index_bitmap) {
+		kfree(asd_ha->seq.tc_index_array);
+		asd_ha->seq.tc_index_array = NULL;
 		return -ENOMEM;
+	}
 
 	spin_lock_init(&seq->tc_index_lock);
 
diff --git a/drivers/scsi/arcmsr/arcmsr.h b/drivers/scsi/arcmsr/arcmsr.h
index cf99f8cf4cdd..a254b32eba39 100644
--- a/drivers/scsi/arcmsr/arcmsr.h
+++ b/drivers/scsi/arcmsr/arcmsr.h
@@ -629,7 +629,6 @@ struct AdapterControlBlock
 	struct pci_dev *		pdev;
 	struct Scsi_Host *		host;
 	unsigned long			vir2phy_offset;
-	struct msix_entry	entries[ARCMST_NUM_MSIX_VECTORS];
 	/* Offset is used in making arc cdb physical to virtual calculations */
 	uint32_t			outbound_int_enable;
 	uint32_t			cdb_phyaddr_hi32;
@@ -671,8 +670,6 @@ struct AdapterControlBlock
 	/* iop init */
 	#define ACB_F_ABORT				0x0200
 	#define ACB_F_FIRMWARE_TRAP           		0x0400
-	#define ACB_F_MSI_ENABLED		0x1000
-	#define ACB_F_MSIX_ENABLED		0x2000
 	struct CommandControlBlock *			pccb_pool[ARCMSR_MAX_FREECCB_NUM];
 	/* used for memory free */
 	struct list_head		ccb_free_list;
@@ -725,7 +722,7 @@ struct AdapterControlBlock
 	atomic_t 			rq_map_token;
 	atomic_t			ante_token_value;
 	uint32_t	maxOutstanding;
-	int		msix_vector_count;
+	int		vector_count;
 };/* HW_DEVICE_EXTENSION */
 /*
 *******************************************************************************
diff --git a/drivers/scsi/arcmsr/arcmsr_hba.c b/drivers/scsi/arcmsr/arcmsr_hba.c
index f0cfb0451757..9e45749d55ed 100644
--- a/drivers/scsi/arcmsr/arcmsr_hba.c
+++ b/drivers/scsi/arcmsr/arcmsr_hba.c
@@ -720,51 +720,39 @@ static void arcmsr_message_isr_bh_fn(struct work_struct *work)
 static int
 arcmsr_request_irq(struct pci_dev *pdev, struct AdapterControlBlock *acb)
 {
-	int	i, j, r;
-	struct msix_entry entries[ARCMST_NUM_MSIX_VECTORS];
-
-	for (i = 0; i < ARCMST_NUM_MSIX_VECTORS; i++)
-		entries[i].entry = i;
-	r = pci_enable_msix_range(pdev, entries, 1, ARCMST_NUM_MSIX_VECTORS);
-	if (r < 0)
-		goto msi_int;
-	acb->msix_vector_count = r;
-	for (i = 0; i < r; i++) {
-		if (request_irq(entries[i].vector,
-			arcmsr_do_interrupt, 0, "arcmsr", acb)) {
+	unsigned long flags;
+	int nvec, i;
+
+	nvec = pci_alloc_irq_vectors(pdev, 1, ARCMST_NUM_MSIX_VECTORS,
+			PCI_IRQ_MSIX);
+	if (nvec > 0) {
+		pr_info("arcmsr%d: msi-x enabled\n", acb->host->host_no);
+		flags = 0;
+	} else {
+		nvec = pci_alloc_irq_vectors(pdev, 1, 1,
+				PCI_IRQ_MSI | PCI_IRQ_LEGACY);
+		if (nvec < 1)
+			return FAILED;
+
+		flags = IRQF_SHARED;
+	}
+
+	acb->vector_count = nvec;
+	for (i = 0; i < nvec; i++) {
+		if (request_irq(pci_irq_vector(pdev, i), arcmsr_do_interrupt,
+				flags, "arcmsr", acb)) {
 			pr_warn("arcmsr%d: request_irq =%d failed!\n",
-				acb->host->host_no, entries[i].vector);
-			for (j = 0 ; j < i ; j++)
-				free_irq(entries[j].vector, acb);
-			pci_disable_msix(pdev);
-			goto msi_int;
+				acb->host->host_no, pci_irq_vector(pdev, i));
+			goto out_free_irq;
 		}
-		acb->entries[i] = entries[i];
-	}
-	acb->acb_flags |= ACB_F_MSIX_ENABLED;
-	pr_info("arcmsr%d: msi-x enabled\n", acb->host->host_no);
-	return SUCCESS;
-msi_int:
-	if (pci_enable_msi_exact(pdev, 1) < 0)
-		goto legacy_int;
-	if (request_irq(pdev->irq, arcmsr_do_interrupt,
-		IRQF_SHARED, "arcmsr", acb)) {
-		pr_warn("arcmsr%d: request_irq =%d failed!\n",
-			acb->host->host_no, pdev->irq);
-		pci_disable_msi(pdev);
-		goto legacy_int;
-	}
-	acb->acb_flags |= ACB_F_MSI_ENABLED;
-	pr_info("arcmsr%d: msi enabled\n", acb->host->host_no);
-	return SUCCESS;
-legacy_int:
-	if (request_irq(pdev->irq, arcmsr_do_interrupt,
-		IRQF_SHARED, "arcmsr", acb)) {
-		pr_warn("arcmsr%d: request_irq = %d failed!\n",
-			acb->host->host_no, pdev->irq);
-		return FAILED;
 	}
+
 	return SUCCESS;
+out_free_irq:
+	while (--i >= 0)
+		free_irq(pci_irq_vector(pdev, i), acb);
+	pci_free_irq_vectors(pdev);
+	return FAILED;
 }
 
 static int arcmsr_probe(struct pci_dev *pdev, const struct pci_device_id *id)
@@ -886,15 +874,9 @@ static void arcmsr_free_irq(struct pci_dev *pdev,
 {
 	int i;
 
-	if (acb->acb_flags & ACB_F_MSI_ENABLED) {
-		free_irq(pdev->irq, acb);
-		pci_disable_msi(pdev);
-	} else if (acb->acb_flags & ACB_F_MSIX_ENABLED) {
-		for (i = 0; i < acb->msix_vector_count; i++)
-			free_irq(acb->entries[i].vector, acb);
-		pci_disable_msix(pdev);
-	} else
-		free_irq(pdev->irq, acb);
+	for (i = 0; i < acb->vector_count; i++)
+		free_irq(pci_irq_vector(pdev, i), acb);
+	pci_free_irq_vectors(pdev);
 }
 
 static int arcmsr_suspend(struct pci_dev *pdev, pm_message_t state)
diff --git a/drivers/scsi/arm/cumana_1.c b/drivers/scsi/arm/cumana_1.c
index 8e9cfe8f22f5..a87b99c7fb9a 100644
--- a/drivers/scsi/arm/cumana_1.c
+++ b/drivers/scsi/arm/cumana_1.c
@@ -14,49 +14,48 @@
 #include <scsi/scsi_host.h>
 
 #define priv(host)			((struct NCR5380_hostdata *)(host)->hostdata)
-#define NCR5380_read(reg)		cumanascsi_read(instance, reg)
-#define NCR5380_write(reg, value)	cumanascsi_write(instance, reg, value)
+#define NCR5380_read(reg)		cumanascsi_read(hostdata, reg)
+#define NCR5380_write(reg, value)	cumanascsi_write(hostdata, reg, value)
 
-#define NCR5380_dma_xfer_len(instance, cmd, phase)	(cmd->transfersize)
+#define NCR5380_dma_xfer_len		cumanascsi_dma_xfer_len
 #define NCR5380_dma_recv_setup		cumanascsi_pread
 #define NCR5380_dma_send_setup		cumanascsi_pwrite
-#define NCR5380_dma_residual(instance)	(0)
+#define NCR5380_dma_residual		NCR5380_dma_residual_none
 
 #define NCR5380_intr			cumanascsi_intr
 #define NCR5380_queue_command		cumanascsi_queue_command
 #define NCR5380_info			cumanascsi_info
 
 #define NCR5380_implementation_fields	\
-	unsigned ctrl;			\
-	void __iomem *base;		\
-	void __iomem *dma
+	unsigned ctrl
 
-#include "../NCR5380.h"
+struct NCR5380_hostdata;
+static u8 cumanascsi_read(struct NCR5380_hostdata *, unsigned int);
+static void cumanascsi_write(struct NCR5380_hostdata *, unsigned int, u8);
 
-void cumanascsi_setup(char *str, int *ints)
-{
-}
+#include "../NCR5380.h"
 
 #define CTRL	0x16fc
 #define STAT	0x2004
 #define L(v)	(((v)<<16)|((v) & 0x0000ffff))
 #define H(v)	(((v)>>16)|((v) & 0xffff0000))
 
-static inline int cumanascsi_pwrite(struct Scsi_Host *host,
+static inline int cumanascsi_pwrite(struct NCR5380_hostdata *hostdata,
                                     unsigned char *addr, int len)
 {
   unsigned long *laddr;
-  void __iomem *dma = priv(host)->dma + 0x2000;
+  u8 __iomem *base = hostdata->io;
+  u8 __iomem *dma = hostdata->pdma_io + 0x2000;
 
   if(!len) return 0;
 
-  writeb(0x02, priv(host)->base + CTRL);
+  writeb(0x02, base + CTRL);
   laddr = (unsigned long *)addr;
   while(len >= 32)
   {
     unsigned int status;
     unsigned long v;
-    status = readb(priv(host)->base + STAT);
+    status = readb(base + STAT);
     if(status & 0x80)
       goto end;
     if(!(status & 0x40))
@@ -75,12 +74,12 @@ static inline int cumanascsi_pwrite(struct Scsi_Host *host,
   }
 
   addr = (unsigned char *)laddr;
-  writeb(0x12, priv(host)->base + CTRL);
+  writeb(0x12, base + CTRL);
 
   while(len > 0)
   {
     unsigned int status;
-    status = readb(priv(host)->base + STAT);
+    status = readb(base + STAT);
     if(status & 0x80)
       goto end;
     if(status & 0x40)
@@ -90,7 +89,7 @@ static inline int cumanascsi_pwrite(struct Scsi_Host *host,
         break;
     }
 
-    status = readb(priv(host)->base + STAT);
+    status = readb(base + STAT);
     if(status & 0x80)
       goto end;
     if(status & 0x40)
@@ -101,27 +100,28 @@ static inline int cumanascsi_pwrite(struct Scsi_Host *host,
     }
   }
 end:
-  writeb(priv(host)->ctrl | 0x40, priv(host)->base + CTRL);
+  writeb(hostdata->ctrl | 0x40, base + CTRL);
 
 	if (len)
 		return -1;
 	return 0;
 }
 
-static inline int cumanascsi_pread(struct Scsi_Host *host,
+static inline int cumanascsi_pread(struct NCR5380_hostdata *hostdata,
                                    unsigned char *addr, int len)
 {
   unsigned long *laddr;
-  void __iomem *dma = priv(host)->dma + 0x2000;
+  u8 __iomem *base = hostdata->io;
+  u8 __iomem *dma = hostdata->pdma_io + 0x2000;
 
   if(!len) return 0;
 
-  writeb(0x00, priv(host)->base + CTRL);
+  writeb(0x00, base + CTRL);
   laddr = (unsigned long *)addr;
   while(len >= 32)
   {
     unsigned int status;
-    status = readb(priv(host)->base + STAT);
+    status = readb(base + STAT);
     if(status & 0x80)
       goto end;
     if(!(status & 0x40))
@@ -140,12 +140,12 @@ static inline int cumanascsi_pread(struct Scsi_Host *host,
   }
 
   addr = (unsigned char *)laddr;
-  writeb(0x10, priv(host)->base + CTRL);
+  writeb(0x10, base + CTRL);
 
   while(len > 0)
   {
     unsigned int status;
-    status = readb(priv(host)->base + STAT);
+    status = readb(base + STAT);
     if(status & 0x80)
       goto end;
     if(status & 0x40)
@@ -155,7 +155,7 @@ static inline int cumanascsi_pread(struct Scsi_Host *host,
         break;
     }
 
-    status = readb(priv(host)->base + STAT);
+    status = readb(base + STAT);
     if(status & 0x80)
       goto end;
     if(status & 0x40)
@@ -166,37 +166,45 @@ static inline int cumanascsi_pread(struct Scsi_Host *host,
     }
   }
 end:
-  writeb(priv(host)->ctrl | 0x40, priv(host)->base + CTRL);
+  writeb(hostdata->ctrl | 0x40, base + CTRL);
 
 	if (len)
 		return -1;
 	return 0;
 }
 
-static unsigned char cumanascsi_read(struct Scsi_Host *host, unsigned int reg)
+static int cumanascsi_dma_xfer_len(struct NCR5380_hostdata *hostdata,
+                                   struct scsi_cmnd *cmd)
+{
+	return cmd->transfersize;
+}
+
+static u8 cumanascsi_read(struct NCR5380_hostdata *hostdata,
+                          unsigned int reg)
 {
-	void __iomem *base = priv(host)->base;
-	unsigned char val;
+	u8 __iomem *base = hostdata->io;
+	u8 val;
 
 	writeb(0, base + CTRL);
 
 	val = readb(base + 0x2100 + (reg << 2));
 
-	priv(host)->ctrl = 0x40;
+	hostdata->ctrl = 0x40;
 	writeb(0x40, base + CTRL);
 
 	return val;
 }
 
-static void cumanascsi_write(struct Scsi_Host *host, unsigned int reg, unsigned int value)
+static void cumanascsi_write(struct NCR5380_hostdata *hostdata,
+                             unsigned int reg, u8 value)
 {
-	void __iomem *base = priv(host)->base;
+	u8 __iomem *base = hostdata->io;
 
 	writeb(0, base + CTRL);
 
 	writeb(value, base + 0x2100 + (reg << 2));
 
-	priv(host)->ctrl = 0x40;
+	hostdata->ctrl = 0x40;
 	writeb(0x40, base + CTRL);
 }
 
@@ -235,11 +243,11 @@ static int cumanascsi1_probe(struct expansion_card *ec,
 		goto out_release;
 	}
 
-	priv(host)->base = ioremap(ecard_resource_start(ec, ECARD_RES_IOCSLOW),
-				   ecard_resource_len(ec, ECARD_RES_IOCSLOW));
-	priv(host)->dma = ioremap(ecard_resource_start(ec, ECARD_RES_MEMC),
-				  ecard_resource_len(ec, ECARD_RES_MEMC));
-	if (!priv(host)->base || !priv(host)->dma) {
+	priv(host)->io = ioremap(ecard_resource_start(ec, ECARD_RES_IOCSLOW),
+	                         ecard_resource_len(ec, ECARD_RES_IOCSLOW));
+	priv(host)->pdma_io = ioremap(ecard_resource_start(ec, ECARD_RES_MEMC),
+	                              ecard_resource_len(ec, ECARD_RES_MEMC));
+	if (!priv(host)->io || !priv(host)->pdma_io) {
 		ret = -ENOMEM;
 		goto out_unmap;
 	}
@@ -253,7 +261,7 @@ static int cumanascsi1_probe(struct expansion_card *ec,
 	NCR5380_maybe_reset_bus(host);
 
         priv(host)->ctrl = 0;
-        writeb(0, priv(host)->base + CTRL);
+        writeb(0, priv(host)->io + CTRL);
 
 	ret = request_irq(host->irq, cumanascsi_intr, 0,
 			  "CumanaSCSI-1", host);
@@ -275,8 +283,8 @@ static int cumanascsi1_probe(struct expansion_card *ec,
  out_exit:
 	NCR5380_exit(host);
  out_unmap:
-	iounmap(priv(host)->base);
-	iounmap(priv(host)->dma);
+	iounmap(priv(host)->io);
+	iounmap(priv(host)->pdma_io);
 	scsi_host_put(host);
  out_release:
 	ecard_release_resources(ec);
@@ -287,15 +295,17 @@ static int cumanascsi1_probe(struct expansion_card *ec,
 static void cumanascsi1_remove(struct expansion_card *ec)
 {
 	struct Scsi_Host *host = ecard_get_drvdata(ec);
+	void __iomem *base = priv(host)->io;
+	void __iomem *dma = priv(host)->pdma_io;
 
 	ecard_set_drvdata(ec, NULL);
 
 	scsi_remove_host(host);
 	free_irq(host->irq, host);
 	NCR5380_exit(host);
-	iounmap(priv(host)->base);
-	iounmap(priv(host)->dma);
 	scsi_host_put(host);
+	iounmap(base);
+	iounmap(dma);
 	ecard_release_resources(ec);
 }
 
diff --git a/drivers/scsi/arm/oak.c b/drivers/scsi/arm/oak.c
index a396024a3cae..6be6666534d4 100644
--- a/drivers/scsi/arm/oak.c
+++ b/drivers/scsi/arm/oak.c
@@ -16,21 +16,18 @@
 
 #define priv(host)			((struct NCR5380_hostdata *)(host)->hostdata)
 
-#define NCR5380_read(reg) \
-	readb(priv(instance)->base + ((reg) << 2))
-#define NCR5380_write(reg, value) \
-	writeb(value, priv(instance)->base + ((reg) << 2))
+#define NCR5380_read(reg)           readb(hostdata->io + ((reg) << 2))
+#define NCR5380_write(reg, value)   writeb(value, hostdata->io + ((reg) << 2))
 
-#define NCR5380_dma_xfer_len(instance, cmd, phase)	(0)
+#define NCR5380_dma_xfer_len		NCR5380_dma_xfer_none
 #define NCR5380_dma_recv_setup		oakscsi_pread
 #define NCR5380_dma_send_setup		oakscsi_pwrite
-#define NCR5380_dma_residual(instance)	(0)
+#define NCR5380_dma_residual		NCR5380_dma_residual_none
 
 #define NCR5380_queue_command		oakscsi_queue_command
 #define NCR5380_info			oakscsi_info
 
-#define NCR5380_implementation_fields	\
-	void __iomem *base
+#define NCR5380_implementation_fields	/* none */
 
 #include "../NCR5380.h"
 
@@ -40,10 +37,10 @@
 #define STAT	((128 + 16) << 2)
 #define DATA	((128 + 8) << 2)
 
-static inline int oakscsi_pwrite(struct Scsi_Host *instance,
+static inline int oakscsi_pwrite(struct NCR5380_hostdata *hostdata,
                                  unsigned char *addr, int len)
 {
-  void __iomem *base = priv(instance)->base;
+  u8 __iomem *base = hostdata->io;
 
 printk("writing %p len %d\n",addr, len);
 
@@ -55,10 +52,11 @@ printk("writing %p len %d\n",addr, len);
   return 0;
 }
 
-static inline int oakscsi_pread(struct Scsi_Host *instance,
+static inline int oakscsi_pread(struct NCR5380_hostdata *hostdata,
                                 unsigned char *addr, int len)
 {
-  void __iomem *base = priv(instance)->base;
+  u8 __iomem *base = hostdata->io;
+
 printk("reading %p len %d\n", addr, len);
   while(len > 0)
   {
@@ -133,15 +131,14 @@ static int oakscsi_probe(struct expansion_card *ec, const struct ecard_id *id)
 		goto release;
 	}
 
-	priv(host)->base = ioremap(ecard_resource_start(ec, ECARD_RES_MEMC),
-				   ecard_resource_len(ec, ECARD_RES_MEMC));
-	if (!priv(host)->base) {
+	priv(host)->io = ioremap(ecard_resource_start(ec, ECARD_RES_MEMC),
+	                         ecard_resource_len(ec, ECARD_RES_MEMC));
+	if (!priv(host)->io) {
 		ret = -ENOMEM;
 		goto unreg;
 	}
 
 	host->irq = NO_IRQ;
-	host->n_io_port = 255;
 
 	ret = NCR5380_init(host, FLAG_DMA_FIXUP | FLAG_LATE_DMA_SETUP);
 	if (ret)
@@ -159,7 +156,7 @@ static int oakscsi_probe(struct expansion_card *ec, const struct ecard_id *id)
  out_exit:
 	NCR5380_exit(host);
  out_unmap:
-	iounmap(priv(host)->base);
+	iounmap(priv(host)->io);
  unreg:
 	scsi_host_put(host);
  release:
@@ -171,13 +168,14 @@ static int oakscsi_probe(struct expansion_card *ec, const struct ecard_id *id)
 static void oakscsi_remove(struct expansion_card *ec)
 {
 	struct Scsi_Host *host = ecard_get_drvdata(ec);
+	void __iomem *base = priv(host)->io;
 
 	ecard_set_drvdata(ec, NULL);
 	scsi_remove_host(host);
 
 	NCR5380_exit(host);
-	iounmap(priv(host)->base);
 	scsi_host_put(host);
+	iounmap(base);
 	ecard_release_resources(ec);
 }
 
diff --git a/drivers/scsi/atari_scsi.c b/drivers/scsi/atari_scsi.c
index a59ad94ea52b..105b35393ce9 100644
--- a/drivers/scsi/atari_scsi.c
+++ b/drivers/scsi/atari_scsi.c
@@ -57,6 +57,9 @@
 
 #define NCR5380_implementation_fields   /* none */
 
+static u8 (*atari_scsi_reg_read)(unsigned int);
+static void (*atari_scsi_reg_write)(unsigned int, u8);
+
 #define NCR5380_read(reg)               atari_scsi_reg_read(reg)
 #define NCR5380_write(reg, value)       atari_scsi_reg_write(reg, value)
 
@@ -64,14 +67,10 @@
 #define NCR5380_abort                   atari_scsi_abort
 #define NCR5380_info                    atari_scsi_info
 
-#define NCR5380_dma_recv_setup(instance, data, count) \
-        atari_scsi_dma_setup(instance, data, count, 0)
-#define NCR5380_dma_send_setup(instance, data, count) \
-        atari_scsi_dma_setup(instance, data, count, 1)
-#define NCR5380_dma_residual(instance) \
-        atari_scsi_dma_residual(instance)
-#define NCR5380_dma_xfer_len(instance, cmd, phase) \
-        atari_dma_xfer_len(cmd->SCp.this_residual, cmd, !((phase) & SR_IO))
+#define NCR5380_dma_xfer_len            atari_scsi_dma_xfer_len
+#define NCR5380_dma_recv_setup          atari_scsi_dma_recv_setup
+#define NCR5380_dma_send_setup          atari_scsi_dma_send_setup
+#define NCR5380_dma_residual            atari_scsi_dma_residual
 
 #define NCR5380_acquire_dma_irq(instance)      falcon_get_lock(instance)
 #define NCR5380_release_dma_irq(instance)      falcon_release_lock()
@@ -126,9 +125,6 @@ static inline unsigned long SCSI_DMA_GETADR(void)
 
 static void atari_scsi_fetch_restbytes(void);
 
-static unsigned char (*atari_scsi_reg_read)(unsigned char reg);
-static void (*atari_scsi_reg_write)(unsigned char reg, unsigned char value);
-
 static unsigned long	atari_dma_residual, atari_dma_startaddr;
 static short		atari_dma_active;
 /* pointer to the dribble buffer */
@@ -457,15 +453,14 @@ static int __init atari_scsi_setup(char *str)
 __setup("atascsi=", atari_scsi_setup);
 #endif /* !MODULE */
 
-
-static unsigned long atari_scsi_dma_setup(struct Scsi_Host *instance,
+static unsigned long atari_scsi_dma_setup(struct NCR5380_hostdata *hostdata,
 					  void *data, unsigned long count,
 					  int dir)
 {
 	unsigned long addr = virt_to_phys(data);
 
-	dprintk(NDEBUG_DMA, "scsi%d: setting up dma, data = %p, phys = %lx, count = %ld, "
-		   "dir = %d\n", instance->host_no, data, addr, count, dir);
+	dprintk(NDEBUG_DMA, "scsi%d: setting up dma, data = %p, phys = %lx, count = %ld, dir = %d\n",
+	        hostdata->host->host_no, data, addr, count, dir);
 
 	if (!IS_A_TT() && !STRAM_ADDR(addr)) {
 		/* If we have a non-DMAable address on a Falcon, use the dribble
@@ -522,8 +517,19 @@ static unsigned long atari_scsi_dma_setup(struct Scsi_Host *instance,
 	return count;
 }
 
+static inline int atari_scsi_dma_recv_setup(struct NCR5380_hostdata *hostdata,
+                                            unsigned char *data, int count)
+{
+	return atari_scsi_dma_setup(hostdata, data, count, 0);
+}
+
+static inline int atari_scsi_dma_send_setup(struct NCR5380_hostdata *hostdata,
+                                            unsigned char *data, int count)
+{
+	return atari_scsi_dma_setup(hostdata, data, count, 1);
+}
 
-static long atari_scsi_dma_residual(struct Scsi_Host *instance)
+static int atari_scsi_dma_residual(struct NCR5380_hostdata *hostdata)
 {
 	return atari_dma_residual;
 }
@@ -564,10 +570,11 @@ static int falcon_classify_cmd(struct scsi_cmnd *cmd)
  * the overrun problem, so this question is academic :-)
  */
 
-static unsigned long atari_dma_xfer_len(unsigned long wanted_len,
-					struct scsi_cmnd *cmd, int write_flag)
+static int atari_scsi_dma_xfer_len(struct NCR5380_hostdata *hostdata,
+                                   struct scsi_cmnd *cmd)
 {
-	unsigned long	possible_len, limit;
+	int wanted_len = cmd->SCp.this_residual;
+	int possible_len, limit;
 
 	if (wanted_len < DMA_MIN_SIZE)
 		return 0;
@@ -604,7 +611,7 @@ static unsigned long atari_dma_xfer_len(unsigned long wanted_len,
 	 * use the dribble buffer and thus can do only STRAM_BUFFER_SIZE bytes.
 	 */
 
-	if (write_flag) {
+	if (cmd->sc_data_direction == DMA_TO_DEVICE) {
 		/* Write operation can always use the DMA, but the transfer size must
 		 * be rounded up to the next multiple of 512 (atari_dma_setup() does
 		 * this).
@@ -644,8 +651,8 @@ static unsigned long atari_dma_xfer_len(unsigned long wanted_len,
 		possible_len = limit;
 
 	if (possible_len != wanted_len)
-		dprintk(NDEBUG_DMA, "Sorry, must cut DMA transfer size to %ld bytes "
-			   "instead of %ld\n", possible_len, wanted_len);
+		dprintk(NDEBUG_DMA, "DMA transfer now %d bytes instead of %d\n",
+		        possible_len, wanted_len);
 
 	return possible_len;
 }
@@ -658,26 +665,38 @@ static unsigned long atari_dma_xfer_len(unsigned long wanted_len,
  * NCR5380_write call these functions via function pointers.
  */
 
-static unsigned char atari_scsi_tt_reg_read(unsigned char reg)
+static u8 atari_scsi_tt_reg_read(unsigned int reg)
 {
 	return tt_scsi_regp[reg * 2];
 }
 
-static void atari_scsi_tt_reg_write(unsigned char reg, unsigned char value)
+static void atari_scsi_tt_reg_write(unsigned int reg, u8 value)
 {
 	tt_scsi_regp[reg * 2] = value;
 }
 
-static unsigned char atari_scsi_falcon_reg_read(unsigned char reg)
+static u8 atari_scsi_falcon_reg_read(unsigned int reg)
 {
-	dma_wd.dma_mode_status= (u_short)(0x88 + reg);
-	return (u_char)dma_wd.fdc_acces_seccount;
+	unsigned long flags;
+	u8 result;
+
+	reg += 0x88;
+	local_irq_save(flags);
+	dma_wd.dma_mode_status = (u_short)reg;
+	result = (u8)dma_wd.fdc_acces_seccount;
+	local_irq_restore(flags);
+	return result;
 }
 
-static void atari_scsi_falcon_reg_write(unsigned char reg, unsigned char value)
+static void atari_scsi_falcon_reg_write(unsigned int reg, u8 value)
 {
-	dma_wd.dma_mode_status = (u_short)(0x88 + reg);
+	unsigned long flags;
+
+	reg += 0x88;
+	local_irq_save(flags);
+	dma_wd.dma_mode_status = (u_short)reg;
 	dma_wd.fdc_acces_seccount = (u_short)value;
+	local_irq_restore(flags);
 }
 
 
diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c
index d9239c2d49b1..b5112d6d7e73 100644
--- a/drivers/scsi/be2iscsi/be_main.c
+++ b/drivers/scsi/be2iscsi/be_main.c
@@ -3049,8 +3049,10 @@ static int beiscsi_create_eqs(struct beiscsi_hba *phba,
 		eq_vaddress = pci_alloc_consistent(phba->pcidev,
 						   num_eq_pages * PAGE_SIZE,
 						   &paddr);
-		if (!eq_vaddress)
+		if (!eq_vaddress) {
+			ret = -ENOMEM;
 			goto create_eq_error;
+		}
 
 		mem->va = eq_vaddress;
 		ret = be_fill_queue(eq, phba->params.num_eq_entries,
@@ -3113,8 +3115,10 @@ static int beiscsi_create_cqs(struct beiscsi_hba *phba,
 		cq_vaddress = pci_alloc_consistent(phba->pcidev,
 						   num_cq_pages * PAGE_SIZE,
 						   &paddr);
-		if (!cq_vaddress)
+		if (!cq_vaddress) {
+			ret = -ENOMEM;
 			goto create_cq_error;
+		}
 
 		ret = be_fill_queue(cq, phba->params.num_cq_entries,
 				    sizeof(struct sol_cqe), cq_vaddress);
diff --git a/drivers/scsi/bfa/bfa_ioc.h b/drivers/scsi/bfa/bfa_ioc.h
index 713745da44c6..0f9fab770339 100644
--- a/drivers/scsi/bfa/bfa_ioc.h
+++ b/drivers/scsi/bfa/bfa_ioc.h
@@ -111,20 +111,24 @@ struct bfa_meminfo_s {
 	struct bfa_mem_kva_s kva_info;
 };
 
-/* BFA memory segment setup macros */
-#define bfa_mem_dma_setup(_meminfo, _dm_ptr, _seg_sz) do {	\
-	((bfa_mem_dma_t *)(_dm_ptr))->mem_len = (_seg_sz);	\
-	if (_seg_sz)						\
-		list_add_tail(&((bfa_mem_dma_t *)_dm_ptr)->qe,	\
-			      &(_meminfo)->dma_info.qe);	\
-} while (0)
+/* BFA memory segment setup helpers */
+static inline void bfa_mem_dma_setup(struct bfa_meminfo_s *meminfo,
+				     struct bfa_mem_dma_s *dm_ptr,
+				     size_t seg_sz)
+{
+	dm_ptr->mem_len = seg_sz;
+	if (seg_sz)
+		list_add_tail(&dm_ptr->qe, &meminfo->dma_info.qe);
+}
 
-#define bfa_mem_kva_setup(_meminfo, _kva_ptr, _seg_sz) do {	\
-	((bfa_mem_kva_t *)(_kva_ptr))->mem_len = (_seg_sz);	\
-	if (_seg_sz)						\
-		list_add_tail(&((bfa_mem_kva_t *)_kva_ptr)->qe,	\
-			      &(_meminfo)->kva_info.qe);	\
-} while (0)
+static inline void bfa_mem_kva_setup(struct bfa_meminfo_s *meminfo,
+				     struct bfa_mem_kva_s *kva_ptr,
+				     size_t seg_sz)
+{
+	kva_ptr->mem_len = seg_sz;
+	if (seg_sz)
+		list_add_tail(&kva_ptr->qe, &meminfo->kva_info.qe);
+}
 
 /* BFA dma memory segments iterator */
 #define bfa_mem_dma_sptr(_mod, _i)	(&(_mod)->dma_seg[(_i)])
diff --git a/drivers/scsi/bfa/bfad_bsg.c b/drivers/scsi/bfa/bfad_bsg.c
index d1ad0208dfe7..a9a00169ad91 100644
--- a/drivers/scsi/bfa/bfad_bsg.c
+++ b/drivers/scsi/bfa/bfad_bsg.c
@@ -3130,11 +3130,12 @@ bfad_iocmd_handler(struct bfad_s *bfad, unsigned int cmd, void *iocmd,
 }
 
 static int
-bfad_im_bsg_vendor_request(struct fc_bsg_job *job)
+bfad_im_bsg_vendor_request(struct bsg_job *job)
 {
-	uint32_t vendor_cmd = job->request->rqst_data.h_vendor.vendor_cmd[0];
-	struct bfad_im_port_s *im_port =
-			(struct bfad_im_port_s *) job->shost->hostdata[0];
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
+	uint32_t vendor_cmd = bsg_request->rqst_data.h_vendor.vendor_cmd[0];
+	struct bfad_im_port_s *im_port = shost_priv(fc_bsg_to_shost(job));
 	struct bfad_s *bfad = im_port->bfad;
 	struct request_queue *request_q = job->req->q;
 	void *payload_kbuf;
@@ -3175,18 +3176,19 @@ bfad_im_bsg_vendor_request(struct fc_bsg_job *job)
 
 	/* Fill the BSG job reply data */
 	job->reply_len = job->reply_payload.payload_len;
-	job->reply->reply_payload_rcv_len = job->reply_payload.payload_len;
-	job->reply->result = rc;
+	bsg_reply->reply_payload_rcv_len = job->reply_payload.payload_len;
+	bsg_reply->result = rc;
 
-	job->job_done(job);
+	bsg_job_done(job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	return rc;
 error:
 	/* free the command buffer */
 	kfree(payload_kbuf);
 out:
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 	job->reply_len = sizeof(uint32_t);
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 	return rc;
 }
 
@@ -3312,7 +3314,7 @@ bfad_fcxp_free_mem(struct bfad_s *bfad, struct bfad_buf_info *buf_base,
 }
 
 int
-bfad_fcxp_bsg_send(struct fc_bsg_job *job, struct bfad_fcxp *drv_fcxp,
+bfad_fcxp_bsg_send(struct bsg_job *job, struct bfad_fcxp *drv_fcxp,
 		   bfa_bsg_fcpt_t *bsg_fcpt)
 {
 	struct bfa_fcxp_s *hal_fcxp;
@@ -3352,28 +3354,29 @@ bfad_fcxp_bsg_send(struct fc_bsg_job *job, struct bfad_fcxp *drv_fcxp,
 }
 
 int
-bfad_im_bsg_els_ct_request(struct fc_bsg_job *job)
+bfad_im_bsg_els_ct_request(struct bsg_job *job)
 {
 	struct bfa_bsg_data *bsg_data;
-	struct bfad_im_port_s *im_port =
-			(struct bfad_im_port_s *) job->shost->hostdata[0];
+	struct bfad_im_port_s *im_port = shost_priv(fc_bsg_to_shost(job));
 	struct bfad_s *bfad = im_port->bfad;
 	bfa_bsg_fcpt_t *bsg_fcpt;
 	struct bfad_fcxp    *drv_fcxp;
 	struct bfa_fcs_lport_s *fcs_port;
 	struct bfa_fcs_rport_s *fcs_rport;
-	uint32_t command_type = job->request->msgcode;
+	struct fc_bsg_request *bsg_request = bsg_request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
+	uint32_t command_type = bsg_request->msgcode;
 	unsigned long flags;
 	struct bfad_buf_info *rsp_buf_info;
 	void *req_kbuf = NULL, *rsp_kbuf = NULL;
 	int rc = -EINVAL;
 
 	job->reply_len  = sizeof(uint32_t);	/* Atleast uint32_t reply_len */
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 
 	/* Get the payload passed in from userspace */
-	bsg_data = (struct bfa_bsg_data *) (((char *)job->request) +
-					sizeof(struct fc_bsg_request));
+	bsg_data = (struct bfa_bsg_data *) (((char *)bsg_request) +
+					    sizeof(struct fc_bsg_request));
 	if (bsg_data == NULL)
 		goto out;
 
@@ -3517,13 +3520,13 @@ bfad_im_bsg_els_ct_request(struct fc_bsg_job *job)
 	/* fill the job->reply data */
 	if (drv_fcxp->req_status == BFA_STATUS_OK) {
 		job->reply_len = drv_fcxp->rsp_len;
-		job->reply->reply_payload_rcv_len = drv_fcxp->rsp_len;
-		job->reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK;
+		bsg_reply->reply_payload_rcv_len = drv_fcxp->rsp_len;
+		bsg_reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK;
 	} else {
-		job->reply->reply_payload_rcv_len =
+		bsg_reply->reply_payload_rcv_len =
 					sizeof(struct fc_bsg_ctels_reply);
 		job->reply_len = sizeof(uint32_t);
-		job->reply->reply_data.ctels_reply.status =
+		bsg_reply->reply_data.ctels_reply.status =
 						FC_CTELS_STATUS_REJECT;
 	}
 
@@ -3549,20 +3552,23 @@ out_free_mem:
 	kfree(bsg_fcpt);
 	kfree(drv_fcxp);
 out:
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 
 	if (rc == BFA_STATUS_OK)
-		job->job_done(job);
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 
 	return rc;
 }
 
 int
-bfad_im_bsg_request(struct fc_bsg_job *job)
+bfad_im_bsg_request(struct bsg_job *job)
 {
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	uint32_t rc = BFA_STATUS_OK;
 
-	switch (job->request->msgcode) {
+	switch (bsg_request->msgcode) {
 	case FC_BSG_HST_VENDOR:
 		/* Process BSG HST Vendor requests */
 		rc = bfad_im_bsg_vendor_request(job);
@@ -3575,8 +3581,8 @@ bfad_im_bsg_request(struct fc_bsg_job *job)
 		rc = bfad_im_bsg_els_ct_request(job);
 		break;
 	default:
-		job->reply->result = rc = -EINVAL;
-		job->reply->reply_payload_rcv_len = 0;
+		bsg_reply->result = rc = -EINVAL;
+		bsg_reply->reply_payload_rcv_len = 0;
 		break;
 	}
 
@@ -3584,7 +3590,7 @@ bfad_im_bsg_request(struct fc_bsg_job *job)
 }
 
 int
-bfad_im_bsg_timeout(struct fc_bsg_job *job)
+bfad_im_bsg_timeout(struct bsg_job *job)
 {
 	/* Don't complete the BSG job request - return -EAGAIN
 	 * to reset bsg job timeout : for ELS/CT pass thru we
diff --git a/drivers/scsi/bfa/bfad_im.h b/drivers/scsi/bfa/bfad_im.h
index 836fdc221edd..c81ec2a77ef5 100644
--- a/drivers/scsi/bfa/bfad_im.h
+++ b/drivers/scsi/bfa/bfad_im.h
@@ -166,8 +166,8 @@ extern struct device_attribute *bfad_im_vport_attrs[];
 
 irqreturn_t bfad_intx(int irq, void *dev_id);
 
-int bfad_im_bsg_request(struct fc_bsg_job *job);
-int bfad_im_bsg_timeout(struct fc_bsg_job *job);
+int bfad_im_bsg_request(struct bsg_job *job);
+int bfad_im_bsg_timeout(struct bsg_job *job);
 
 /*
  * Macro to set the SCSI device sdev_bflags - sdev_bflags are used by the
diff --git a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
index f9ddb6156f14..0990130821fa 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
@@ -970,7 +970,6 @@ static int bnx2fc_libfc_config(struct fc_lport *lport)
 		sizeof(struct libfc_function_template));
 	fc_elsct_init(lport);
 	fc_exch_init(lport);
-	fc_rport_init(lport);
 	fc_disc_init(lport);
 	fc_disc_config(lport, lport);
 	return 0;
diff --git a/drivers/scsi/bnx2fc/bnx2fc_tgt.c b/drivers/scsi/bnx2fc/bnx2fc_tgt.c
index 08ec318afb99..739bfb62aff6 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_tgt.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_tgt.c
@@ -80,7 +80,6 @@ static void bnx2fc_offload_session(struct fcoe_port *port,
 					struct bnx2fc_rport *tgt,
 					struct fc_rport_priv *rdata)
 {
-	struct fc_lport *lport = rdata->local_port;
 	struct fc_rport *rport = rdata->rport;
 	struct bnx2fc_interface *interface = port->priv;
 	struct bnx2fc_hba *hba = interface->hba;
@@ -160,7 +159,7 @@ ofld_err:
 tgt_init_err:
 	if (tgt->fcoe_conn_id != -1)
 		bnx2fc_free_conn_id(hba, tgt->fcoe_conn_id);
-	lport->tt.rport_logoff(rdata);
+	fc_rport_logoff(rdata);
 }
 
 void bnx2fc_flush_active_ios(struct bnx2fc_rport *tgt)
diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
index 4655a9f9dcea..9e6f647ff1c1 100644
--- a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
+++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
@@ -1411,7 +1411,7 @@ static int init_act_open(struct cxgbi_sock *csk)
 	csk->atid = cxgb4_alloc_atid(lldi->tids, csk);
 	if (csk->atid < 0) {
 		pr_err("%s, NO atid available.\n", ndev->name);
-		return -EINVAL;
+		goto rel_resource_without_clip;
 	}
 	cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
 	cxgbi_sock_get(csk);
diff --git a/drivers/scsi/cxlflash/common.h b/drivers/scsi/cxlflash/common.h
index 6e6815545a71..0e9de5d62da2 100644
--- a/drivers/scsi/cxlflash/common.h
+++ b/drivers/scsi/cxlflash/common.h
@@ -19,6 +19,7 @@
 #include <linux/rwsem.h>
 #include <linux/types.h>
 #include <scsi/scsi.h>
+#include <scsi/scsi_cmnd.h>
 #include <scsi/scsi_device.h>
 
 extern const struct file_operations cxlflash_cxl_fops;
@@ -62,11 +63,6 @@ static inline void check_sizes(void)
 /* AFU defines a fixed size of 4K for command buffers (borrow 4K page define) */
 #define CMD_BUFSIZE     SIZE_4K
 
-/* flags in IOA status area for host use */
-#define B_DONE       0x01
-#define B_ERROR      0x02	/* set with B_DONE */
-#define B_TIMEOUT    0x04	/* set with B_DONE & B_ERROR */
-
 enum cxlflash_lr_state {
 	LINK_RESET_INVALID,
 	LINK_RESET_REQUIRED,
@@ -132,12 +128,9 @@ struct cxlflash_cfg {
 struct afu_cmd {
 	struct sisl_ioarcb rcb;	/* IOARCB (cache line aligned) */
 	struct sisl_ioasa sa;	/* IOASA must follow IOARCB */
-	spinlock_t slock;
-	struct completion cevent;
-	char *buf;		/* per command buffer */
 	struct afu *parent;
-	int slot;
-	atomic_t free;
+	struct scsi_cmnd *scp;
+	struct completion cevent;
 
 	u8 cmd_tmf:1;
 
@@ -147,19 +140,31 @@ struct afu_cmd {
 	 */
 } __aligned(cache_line_size());
 
+static inline struct afu_cmd *sc_to_afuc(struct scsi_cmnd *sc)
+{
+	return PTR_ALIGN(scsi_cmd_priv(sc), __alignof__(struct afu_cmd));
+}
+
+static inline struct afu_cmd *sc_to_afucz(struct scsi_cmnd *sc)
+{
+	struct afu_cmd *afuc = sc_to_afuc(sc);
+
+	memset(afuc, 0, sizeof(*afuc));
+	return afuc;
+}
+
 struct afu {
 	/* Stuff requiring alignment go first. */
 
 	u64 rrq_entry[NUM_RRQ_ENTRY];	/* 2K RRQ */
-	/*
-	 * Command & data for AFU commands.
-	 */
-	struct afu_cmd cmd[CXLFLASH_NUM_CMDS];
 
 	/* Beware of alignment till here. Preferably introduce new
 	 * fields after this point
 	 */
 
+	int (*send_cmd)(struct afu *, struct afu_cmd *);
+	void (*context_reset)(struct afu_cmd *);
+
 	/* AFU HW */
 	struct cxl_ioctl_start_work work;
 	struct cxlflash_afu_map __iomem *afu_map;	/* entire MMIO map */
@@ -173,10 +178,10 @@ struct afu {
 	u64 *hrrq_end;
 	u64 *hrrq_curr;
 	bool toggle;
-	bool read_room;
-	atomic64_t room;
+	atomic_t cmds_active;	/* Number of currently active AFU commands */
+	s64 room;
+	spinlock_t rrin_slock; /* Lock to rrin queuing and cmd_room updates */
 	u64 hb;
-	u32 cmd_couts;		/* Number of command checkouts */
 	u32 internal_lun;	/* User-desired LUN mode for this AFU */
 
 	char version[16];
diff --git a/drivers/scsi/cxlflash/lunmgt.c b/drivers/scsi/cxlflash/lunmgt.c
index a0923cade6f3..6c318db90c85 100644
--- a/drivers/scsi/cxlflash/lunmgt.c
+++ b/drivers/scsi/cxlflash/lunmgt.c
@@ -254,8 +254,14 @@ int cxlflash_manage_lun(struct scsi_device *sdev,
 		if (lli->parent->mode != MODE_NONE)
 			rc = -EBUSY;
 		else {
+			/*
+			 * Clean up local LUN for this port and reset table
+			 * tracking when no more references exist.
+			 */
 			sdev->hostdata = NULL;
 			lli->port_sel &= ~CHAN2PORT(chan);
+			if (lli->port_sel == 0U)
+				lli->in_table = false;
 		}
 	}
 
diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c
index b301655f91cd..b17ebf6d0a7e 100644
--- a/drivers/scsi/cxlflash/main.c
+++ b/drivers/scsi/cxlflash/main.c
@@ -35,67 +35,6 @@ MODULE_AUTHOR("Matthew R. Ochs <mrochs@linux.vnet.ibm.com>");
 MODULE_LICENSE("GPL");
 
 /**
- * cmd_checkout() - checks out an AFU command
- * @afu:	AFU to checkout from.
- *
- * Commands are checked out in a round-robin fashion. Note that since
- * the command pool is larger than the hardware queue, the majority of
- * times we will only loop once or twice before getting a command. The
- * buffer and CDB within the command are initialized (zeroed) prior to
- * returning.
- *
- * Return: The checked out command or NULL when command pool is empty.
- */
-static struct afu_cmd *cmd_checkout(struct afu *afu)
-{
-	int k, dec = CXLFLASH_NUM_CMDS;
-	struct afu_cmd *cmd;
-
-	while (dec--) {
-		k = (afu->cmd_couts++ & (CXLFLASH_NUM_CMDS - 1));
-
-		cmd = &afu->cmd[k];
-
-		if (!atomic_dec_if_positive(&cmd->free)) {
-			pr_devel("%s: returning found index=%d cmd=%p\n",
-				 __func__, cmd->slot, cmd);
-			memset(cmd->buf, 0, CMD_BUFSIZE);
-			memset(cmd->rcb.cdb, 0, sizeof(cmd->rcb.cdb));
-			return cmd;
-		}
-	}
-
-	return NULL;
-}
-
-/**
- * cmd_checkin() - checks in an AFU command
- * @cmd:	AFU command to checkin.
- *
- * Safe to pass commands that have already been checked in. Several
- * internal tracking fields are reset as part of the checkin. Note
- * that these are intentionally reset prior to toggling the free bit
- * to avoid clobbering values in the event that the command is checked
- * out right away.
- */
-static void cmd_checkin(struct afu_cmd *cmd)
-{
-	cmd->rcb.scp = NULL;
-	cmd->rcb.timeout = 0;
-	cmd->sa.ioasc = 0;
-	cmd->cmd_tmf = false;
-	cmd->sa.host_use[0] = 0; /* clears both completion and retry bytes */
-
-	if (unlikely(atomic_inc_return(&cmd->free) != 1)) {
-		pr_err("%s: Freeing cmd (%d) that is not in use!\n",
-		       __func__, cmd->slot);
-		return;
-	}
-
-	pr_devel("%s: released cmd %p index=%d\n", __func__, cmd, cmd->slot);
-}
-
-/**
  * process_cmd_err() - command error handler
  * @cmd:	AFU command that experienced the error.
  * @scp:	SCSI command associated with the AFU command in error.
@@ -212,7 +151,7 @@ static void process_cmd_err(struct afu_cmd *cmd, struct scsi_cmnd *scp)
  *
  * Prepares and submits command that has either completed or timed out to
  * the SCSI stack. Checks AFU command back into command pool for non-internal
- * (rcb.scp populated) commands.
+ * (cmd->scp populated) commands.
  */
 static void cmd_complete(struct afu_cmd *cmd)
 {
@@ -222,19 +161,14 @@ static void cmd_complete(struct afu_cmd *cmd)
 	struct cxlflash_cfg *cfg = afu->parent;
 	bool cmd_is_tmf;
 
-	spin_lock_irqsave(&cmd->slock, lock_flags);
-	cmd->sa.host_use_b[0] |= B_DONE;
-	spin_unlock_irqrestore(&cmd->slock, lock_flags);
-
-	if (cmd->rcb.scp) {
-		scp = cmd->rcb.scp;
+	if (cmd->scp) {
+		scp = cmd->scp;
 		if (unlikely(cmd->sa.ioasc))
 			process_cmd_err(cmd, scp);
 		else
 			scp->result = (DID_OK << 16);
 
 		cmd_is_tmf = cmd->cmd_tmf;
-		cmd_checkin(cmd); /* Don't use cmd after here */
 
 		pr_debug_ratelimited("%s: calling scsi_done scp=%p result=%X "
 				     "ioasc=%d\n", __func__, scp, scp->result,
@@ -254,49 +188,19 @@ static void cmd_complete(struct afu_cmd *cmd)
 }
 
 /**
- * context_reset() - timeout handler for AFU commands
+ * context_reset_ioarrin() - reset command owner context via IOARRIN register
  * @cmd:	AFU command that timed out.
- *
- * Sends a reset to the AFU.
  */
-static void context_reset(struct afu_cmd *cmd)
+static void context_reset_ioarrin(struct afu_cmd *cmd)
 {
 	int nretry = 0;
 	u64 rrin = 0x1;
-	u64 room = 0;
 	struct afu *afu = cmd->parent;
-	ulong lock_flags;
+	struct cxlflash_cfg *cfg = afu->parent;
+	struct device *dev = &cfg->dev->dev;
 
 	pr_debug("%s: cmd=%p\n", __func__, cmd);
 
-	spin_lock_irqsave(&cmd->slock, lock_flags);
-
-	/* Already completed? */
-	if (cmd->sa.host_use_b[0] & B_DONE) {
-		spin_unlock_irqrestore(&cmd->slock, lock_flags);
-		return;
-	}
-
-	cmd->sa.host_use_b[0] |= (B_DONE | B_ERROR | B_TIMEOUT);
-	spin_unlock_irqrestore(&cmd->slock, lock_flags);
-
-	/*
-	 * We really want to send this reset at all costs, so spread
-	 * out wait time on successive retries for available room.
-	 */
-	do {
-		room = readq_be(&afu->host_map->cmd_room);
-		atomic64_set(&afu->room, room);
-		if (room)
-			goto write_rrin;
-		udelay(1 << nretry);
-	} while (nretry++ < MC_ROOM_RETRY_CNT);
-
-	pr_err("%s: no cmd_room to send reset\n", __func__);
-	return;
-
-write_rrin:
-	nretry = 0;
 	writeq_be(rrin, &afu->host_map->ioarrin);
 	do {
 		rrin = readq_be(&afu->host_map->ioarrin);
@@ -305,93 +209,81 @@ write_rrin:
 		/* Double delay each time */
 		udelay(1 << nretry);
 	} while (nretry++ < MC_ROOM_RETRY_CNT);
+
+	dev_dbg(dev, "%s: returning rrin=0x%016llX nretry=%d\n",
+		__func__, rrin, nretry);
 }
 
 /**
- * send_cmd() - sends an AFU command
+ * send_cmd_ioarrin() - sends an AFU command via IOARRIN register
  * @afu:	AFU associated with the host.
  * @cmd:	AFU command to send.
  *
  * Return:
  *	0 on success, SCSI_MLQUEUE_HOST_BUSY on failure
  */
-static int send_cmd(struct afu *afu, struct afu_cmd *cmd)
+static int send_cmd_ioarrin(struct afu *afu, struct afu_cmd *cmd)
 {
 	struct cxlflash_cfg *cfg = afu->parent;
 	struct device *dev = &cfg->dev->dev;
-	int nretry = 0;
 	int rc = 0;
-	u64 room;
-	long newval;
+	s64 room;
+	ulong lock_flags;
 
 	/*
-	 * This routine is used by critical users such an AFU sync and to
-	 * send a task management function (TMF). Thus we want to retry a
-	 * bit before returning an error. To avoid the performance penalty
-	 * of MMIO, we spread the update of 'room' over multiple commands.
+	 * To avoid the performance penalty of MMIO, spread the update of
+	 * 'room' over multiple commands.
 	 */
-retry:
-	newval = atomic64_dec_if_positive(&afu->room);
-	if (!newval) {
-		do {
-			room = readq_be(&afu->host_map->cmd_room);
-			atomic64_set(&afu->room, room);
-			if (room)
-				goto write_ioarrin;
-			udelay(1 << nretry);
-		} while (nretry++ < MC_ROOM_RETRY_CNT);
-
-		dev_err(dev, "%s: no cmd_room to send 0x%X\n",
-		       __func__, cmd->rcb.cdb[0]);
-
-		goto no_room;
-	} else if (unlikely(newval < 0)) {
-		/* This should be rare. i.e. Only if two threads race and
-		 * decrement before the MMIO read is done. In this case
-		 * just benefit from the other thread having updated
-		 * afu->room.
-		 */
-		if (nretry++ < MC_ROOM_RETRY_CNT) {
-			udelay(1 << nretry);
-			goto retry;
+	spin_lock_irqsave(&afu->rrin_slock, lock_flags);
+	if (--afu->room < 0) {
+		room = readq_be(&afu->host_map->cmd_room);
+		if (room <= 0) {
+			dev_dbg_ratelimited(dev, "%s: no cmd_room to send "
+					    "0x%02X, room=0x%016llX\n",
+					    __func__, cmd->rcb.cdb[0], room);
+			afu->room = 0;
+			rc = SCSI_MLQUEUE_HOST_BUSY;
+			goto out;
 		}
-
-		goto no_room;
+		afu->room = room - 1;
 	}
 
-write_ioarrin:
 	writeq_be((u64)&cmd->rcb, &afu->host_map->ioarrin);
 out:
+	spin_unlock_irqrestore(&afu->rrin_slock, lock_flags);
 	pr_devel("%s: cmd=%p len=%d ea=%p rc=%d\n", __func__, cmd,
 		 cmd->rcb.data_len, (void *)cmd->rcb.data_ea, rc);
 	return rc;
-
-no_room:
-	afu->read_room = true;
-	kref_get(&cfg->afu->mapcount);
-	schedule_work(&cfg->work_q);
-	rc = SCSI_MLQUEUE_HOST_BUSY;
-	goto out;
 }
 
 /**
  * wait_resp() - polls for a response or timeout to a sent AFU command
  * @afu:	AFU associated with the host.
  * @cmd:	AFU command that was sent.
+ *
+ * Return:
+ *	0 on success, -1 on timeout/error
  */
-static void wait_resp(struct afu *afu, struct afu_cmd *cmd)
+static int wait_resp(struct afu *afu, struct afu_cmd *cmd)
 {
+	int rc = 0;
 	ulong timeout = msecs_to_jiffies(cmd->rcb.timeout * 2 * 1000);
 
 	timeout = wait_for_completion_timeout(&cmd->cevent, timeout);
-	if (!timeout)
-		context_reset(cmd);
+	if (!timeout) {
+		afu->context_reset(cmd);
+		rc = -1;
+	}
 
-	if (unlikely(cmd->sa.ioasc != 0))
+	if (unlikely(cmd->sa.ioasc != 0)) {
 		pr_err("%s: CMD 0x%X failed, IOASC: flags 0x%X, afu_rc 0x%X, "
 		       "scsi_rc 0x%X, fc_rc 0x%X\n", __func__, cmd->rcb.cdb[0],
 		       cmd->sa.rc.flags, cmd->sa.rc.afu_rc, cmd->sa.rc.scsi_rc,
 		       cmd->sa.rc.fc_rc);
+		rc = -1;
+	}
+
+	return rc;
 }
 
 /**
@@ -405,24 +297,15 @@ static void wait_resp(struct afu *afu, struct afu_cmd *cmd)
  */
 static int send_tmf(struct afu *afu, struct scsi_cmnd *scp, u64 tmfcmd)
 {
-	struct afu_cmd *cmd;
-
 	u32 port_sel = scp->device->channel + 1;
-	short lflag = 0;
 	struct Scsi_Host *host = scp->device->host;
 	struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)host->hostdata;
+	struct afu_cmd *cmd = sc_to_afucz(scp);
 	struct device *dev = &cfg->dev->dev;
 	ulong lock_flags;
 	int rc = 0;
 	ulong to;
 
-	cmd = cmd_checkout(afu);
-	if (unlikely(!cmd)) {
-		dev_err(dev, "%s: could not get a free command\n", __func__);
-		rc = SCSI_MLQUEUE_HOST_BUSY;
-		goto out;
-	}
-
 	/* When Task Management Function is active do not send another */
 	spin_lock_irqsave(&cfg->tmf_slock, lock_flags);
 	if (cfg->tmf_active)
@@ -430,28 +313,23 @@ static int send_tmf(struct afu *afu, struct scsi_cmnd *scp, u64 tmfcmd)
 						  !cfg->tmf_active,
 						  cfg->tmf_slock);
 	cfg->tmf_active = true;
-	cmd->cmd_tmf = true;
 	spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags);
 
+	cmd->scp = scp;
+	cmd->parent = afu;
+	cmd->cmd_tmf = true;
+
 	cmd->rcb.ctx_id = afu->ctx_hndl;
+	cmd->rcb.msi = SISL_MSI_RRQ_UPDATED;
 	cmd->rcb.port_sel = port_sel;
 	cmd->rcb.lun_id = lun_to_lunid(scp->device->lun);
-
-	lflag = SISL_REQ_FLAGS_TMF_CMD;
-
 	cmd->rcb.req_flags = (SISL_REQ_FLAGS_PORT_LUN_ID |
-			      SISL_REQ_FLAGS_SUP_UNDERRUN | lflag);
-
-	/* Stash the scp in the reserved field, for reuse during interrupt */
-	cmd->rcb.scp = scp;
-
-	/* Copy the CDB from the cmd passed in */
+			      SISL_REQ_FLAGS_SUP_UNDERRUN |
+			      SISL_REQ_FLAGS_TMF_CMD);
 	memcpy(cmd->rcb.cdb, &tmfcmd, sizeof(tmfcmd));
 
-	/* Send the command */
-	rc = send_cmd(afu, cmd);
+	rc = afu->send_cmd(afu, cmd);
 	if (unlikely(rc)) {
-		cmd_checkin(cmd);
 		spin_lock_irqsave(&cfg->tmf_slock, lock_flags);
 		cfg->tmf_active = false;
 		spin_unlock_irqrestore(&cfg->tmf_slock, lock_flags);
@@ -507,12 +385,12 @@ static int cxlflash_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scp)
 	struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)host->hostdata;
 	struct afu *afu = cfg->afu;
 	struct device *dev = &cfg->dev->dev;
-	struct afu_cmd *cmd;
+	struct afu_cmd *cmd = sc_to_afucz(scp);
+	struct scatterlist *sg = scsi_sglist(scp);
 	u32 port_sel = scp->device->channel + 1;
-	int nseg, i, ncount;
-	struct scatterlist *sg;
+	u16 req_flags = SISL_REQ_FLAGS_SUP_UNDERRUN;
 	ulong lock_flags;
-	short lflag = 0;
+	int nseg = 0;
 	int rc = 0;
 	int kref_got = 0;
 
@@ -552,55 +430,38 @@ static int cxlflash_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scp)
 		break;
 	}
 
-	cmd = cmd_checkout(afu);
-	if (unlikely(!cmd)) {
-		dev_err(dev, "%s: could not get a free command\n", __func__);
-		rc = SCSI_MLQUEUE_HOST_BUSY;
-		goto out;
-	}
-
 	kref_get(&cfg->afu->mapcount);
 	kref_got = 1;
 
+	if (likely(sg)) {
+		nseg = scsi_dma_map(scp);
+		if (unlikely(nseg < 0)) {
+			dev_err(dev, "%s: Fail DMA map!\n", __func__);
+			rc = SCSI_MLQUEUE_HOST_BUSY;
+			goto out;
+		}
+
+		cmd->rcb.data_len = sg_dma_len(sg);
+		cmd->rcb.data_ea = sg_dma_address(sg);
+	}
+
+	cmd->scp = scp;
+	cmd->parent = afu;
+
 	cmd->rcb.ctx_id = afu->ctx_hndl;
+	cmd->rcb.msi = SISL_MSI_RRQ_UPDATED;
 	cmd->rcb.port_sel = port_sel;
 	cmd->rcb.lun_id = lun_to_lunid(scp->device->lun);
 
 	if (scp->sc_data_direction == DMA_TO_DEVICE)
-		lflag = SISL_REQ_FLAGS_HOST_WRITE;
-	else
-		lflag = SISL_REQ_FLAGS_HOST_READ;
-
-	cmd->rcb.req_flags = (SISL_REQ_FLAGS_PORT_LUN_ID |
-			      SISL_REQ_FLAGS_SUP_UNDERRUN | lflag);
-
-	/* Stash the scp in the reserved field, for reuse during interrupt */
-	cmd->rcb.scp = scp;
-
-	nseg = scsi_dma_map(scp);
-	if (unlikely(nseg < 0)) {
-		dev_err(dev, "%s: Fail DMA map! nseg=%d\n",
-			__func__, nseg);
-		rc = SCSI_MLQUEUE_HOST_BUSY;
-		goto out;
-	}
+		req_flags |= SISL_REQ_FLAGS_HOST_WRITE;
 
-	ncount = scsi_sg_count(scp);
-	scsi_for_each_sg(scp, sg, ncount, i) {
-		cmd->rcb.data_len = sg_dma_len(sg);
-		cmd->rcb.data_ea = sg_dma_address(sg);
-	}
-
-	/* Copy the CDB from the scsi_cmnd passed in */
+	cmd->rcb.req_flags = req_flags;
 	memcpy(cmd->rcb.cdb, scp->cmnd, sizeof(cmd->rcb.cdb));
 
-	/* Send the command */
-	rc = send_cmd(afu, cmd);
-	if (unlikely(rc)) {
-		cmd_checkin(cmd);
+	rc = afu->send_cmd(afu, cmd);
+	if (unlikely(rc))
 		scsi_dma_unmap(scp);
-	}
-
 out:
 	if (kref_got)
 		kref_put(&afu->mapcount, afu_unmap);
@@ -628,17 +489,9 @@ static void cxlflash_wait_for_pci_err_recovery(struct cxlflash_cfg *cfg)
  */
 static void free_mem(struct cxlflash_cfg *cfg)
 {
-	int i;
-	char *buf = NULL;
 	struct afu *afu = cfg->afu;
 
 	if (cfg->afu) {
-		for (i = 0; i < CXLFLASH_NUM_CMDS; i++) {
-			buf = afu->cmd[i].buf;
-			if (!((u64)buf & (PAGE_SIZE - 1)))
-				free_page((ulong)buf);
-		}
-
 		free_pages((ulong)afu, get_order(sizeof(struct afu)));
 		cfg->afu = NULL;
 	}
@@ -650,30 +503,16 @@ static void free_mem(struct cxlflash_cfg *cfg)
  *
  * Safe to call with AFU in a partially allocated/initialized state.
  *
- * Cleans up all state associated with the command queue, and unmaps
+ * Waits for any active internal AFU commands to timeout and then unmaps
  * the MMIO space.
- *
- *  - complete() will take care of commands we initiated (they'll be checked
- *  in as part of the cleanup that occurs after the completion)
- *
- *  - cmd_checkin() will take care of entries that we did not initiate and that
- *  have not (and will not) complete because they are sitting on a [now stale]
- *  hardware queue
  */
 static void stop_afu(struct cxlflash_cfg *cfg)
 {
-	int i;
 	struct afu *afu = cfg->afu;
-	struct afu_cmd *cmd;
 
 	if (likely(afu)) {
-		for (i = 0; i < CXLFLASH_NUM_CMDS; i++) {
-			cmd = &afu->cmd[i];
-			complete(&cmd->cevent);
-			if (!atomic_read(&cmd->free))
-				cmd_checkin(cmd);
-		}
-
+		while (atomic_read(&afu->cmds_active))
+			ssleep(1);
 		if (likely(afu->afu_map)) {
 			cxl_psa_unmap((void __iomem *)afu->afu_map);
 			afu->afu_map = NULL;
@@ -886,8 +725,6 @@ static void cxlflash_remove(struct pci_dev *pdev)
 static int alloc_mem(struct cxlflash_cfg *cfg)
 {
 	int rc = 0;
-	int i;
-	char *buf = NULL;
 	struct device *dev = &cfg->dev->dev;
 
 	/* AFU is ~12k, i.e. only one 64k page or up to four 4k pages */
@@ -901,25 +738,6 @@ static int alloc_mem(struct cxlflash_cfg *cfg)
 	}
 	cfg->afu->parent = cfg;
 	cfg->afu->afu_map = NULL;
-
-	for (i = 0; i < CXLFLASH_NUM_CMDS; buf += CMD_BUFSIZE, i++) {
-		if (!((u64)buf & (PAGE_SIZE - 1))) {
-			buf = (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
-			if (unlikely(!buf)) {
-				dev_err(dev,
-					"%s: Allocate command buffers fail!\n",
-				       __func__);
-				rc = -ENOMEM;
-				free_mem(cfg);
-				goto out;
-			}
-		}
-
-		cfg->afu->cmd[i].buf = buf;
-		atomic_set(&cfg->afu->cmd[i].free, 1);
-		cfg->afu->cmd[i].slot = i;
-	}
-
 out:
 	return rc;
 }
@@ -1549,13 +1367,6 @@ static void init_pcr(struct cxlflash_cfg *cfg)
 
 	/* Program the Endian Control for the master context */
 	writeq_be(SISL_ENDIAN_CTRL, &afu->host_map->endian_ctrl);
-
-	/* Initialize cmd fields that never change */
-	for (i = 0; i < CXLFLASH_NUM_CMDS; i++) {
-		afu->cmd[i].rcb.ctx_id = afu->ctx_hndl;
-		afu->cmd[i].rcb.msi = SISL_MSI_RRQ_UPDATED;
-		afu->cmd[i].rcb.rrq = 0x0;
-	}
 }
 
 /**
@@ -1644,19 +1455,8 @@ out:
 static int start_afu(struct cxlflash_cfg *cfg)
 {
 	struct afu *afu = cfg->afu;
-	struct afu_cmd *cmd;
-
-	int i = 0;
 	int rc = 0;
 
-	for (i = 0; i < CXLFLASH_NUM_CMDS; i++) {
-		cmd = &afu->cmd[i];
-
-		init_completion(&cmd->cevent);
-		spin_lock_init(&cmd->slock);
-		cmd->parent = afu;
-	}
-
 	init_pcr(cfg);
 
 	/* After an AFU reset, RRQ entries are stale, clear them */
@@ -1829,6 +1629,9 @@ static int init_afu(struct cxlflash_cfg *cfg)
 		goto err2;
 	}
 
+	afu->send_cmd = send_cmd_ioarrin;
+	afu->context_reset = context_reset_ioarrin;
+
 	pr_debug("%s: afu version %s, interface version 0x%llX\n", __func__,
 		 afu->version, afu->interface_version);
 
@@ -1840,7 +1643,8 @@ static int init_afu(struct cxlflash_cfg *cfg)
 	}
 
 	afu_err_intr_init(cfg->afu);
-	atomic64_set(&afu->room, readq_be(&afu->host_map->cmd_room));
+	spin_lock_init(&afu->rrin_slock);
+	afu->room = readq_be(&afu->host_map->cmd_room);
 
 	/* Restore the LUN mappings */
 	cxlflash_restore_luntable(cfg);
@@ -1884,8 +1688,8 @@ int cxlflash_afu_sync(struct afu *afu, ctx_hndl_t ctx_hndl_u,
 	struct cxlflash_cfg *cfg = afu->parent;
 	struct device *dev = &cfg->dev->dev;
 	struct afu_cmd *cmd = NULL;
+	char *buf = NULL;
 	int rc = 0;
-	int retry_cnt = 0;
 	static DEFINE_MUTEX(sync_active);
 
 	if (cfg->state != STATE_NORMAL) {
@@ -1894,27 +1698,23 @@ int cxlflash_afu_sync(struct afu *afu, ctx_hndl_t ctx_hndl_u,
 	}
 
 	mutex_lock(&sync_active);
-retry:
-	cmd = cmd_checkout(afu);
-	if (unlikely(!cmd)) {
-		retry_cnt++;
-		udelay(1000 * retry_cnt);
-		if (retry_cnt < MC_RETRY_CNT)
-			goto retry;
-		dev_err(dev, "%s: could not get a free command\n", __func__);
+	atomic_inc(&afu->cmds_active);
+	buf = kzalloc(sizeof(*cmd) + __alignof__(*cmd) - 1, GFP_KERNEL);
+	if (unlikely(!buf)) {
+		dev_err(dev, "%s: no memory for command\n", __func__);
 		rc = -1;
 		goto out;
 	}
 
-	pr_debug("%s: afu=%p cmd=%p %d\n", __func__, afu, cmd, ctx_hndl_u);
+	cmd = (struct afu_cmd *)PTR_ALIGN(buf, __alignof__(*cmd));
+	init_completion(&cmd->cevent);
+	cmd->parent = afu;
 
-	memset(cmd->rcb.cdb, 0, sizeof(cmd->rcb.cdb));
+	pr_debug("%s: afu=%p cmd=%p %d\n", __func__, afu, cmd, ctx_hndl_u);
 
 	cmd->rcb.req_flags = SISL_REQ_FLAGS_AFU_CMD;
-	cmd->rcb.port_sel = 0x0;	/* NA */
-	cmd->rcb.lun_id = 0x0;	/* NA */
-	cmd->rcb.data_len = 0x0;
-	cmd->rcb.data_ea = 0x0;
+	cmd->rcb.ctx_id = afu->ctx_hndl;
+	cmd->rcb.msi = SISL_MSI_RRQ_UPDATED;
 	cmd->rcb.timeout = MC_AFU_SYNC_TIMEOUT;
 
 	cmd->rcb.cdb[0] = 0xC0;	/* AFU Sync */
@@ -1924,20 +1724,17 @@ retry:
 	*((__be16 *)&cmd->rcb.cdb[2]) = cpu_to_be16(ctx_hndl_u);
 	*((__be32 *)&cmd->rcb.cdb[4]) = cpu_to_be32(res_hndl_u);
 
-	rc = send_cmd(afu, cmd);
+	rc = afu->send_cmd(afu, cmd);
 	if (unlikely(rc))
 		goto out;
 
-	wait_resp(afu, cmd);
-
-	/* Set on timeout */
-	if (unlikely((cmd->sa.ioasc != 0) ||
-		     (cmd->sa.host_use_b[0] & B_ERROR)))
+	rc = wait_resp(afu, cmd);
+	if (unlikely(rc))
 		rc = -1;
 out:
+	atomic_dec(&afu->cmds_active);
 	mutex_unlock(&sync_active);
-	if (cmd)
-		cmd_checkin(cmd);
+	kfree(buf);
 	pr_debug("%s: returning rc=%d\n", __func__, rc);
 	return rc;
 }
@@ -2376,8 +2173,9 @@ static struct scsi_host_template driver_template = {
 	.change_queue_depth = cxlflash_change_queue_depth,
 	.cmd_per_lun = CXLFLASH_MAX_CMDS_PER_LUN,
 	.can_queue = CXLFLASH_MAX_CMDS,
+	.cmd_size = sizeof(struct afu_cmd) + __alignof__(struct afu_cmd) - 1,
 	.this_id = -1,
-	.sg_tablesize = SG_NONE,	/* No scatter gather support */
+	.sg_tablesize = 1,	/* No scatter gather support */
 	.max_sectors = CXLFLASH_MAX_SECTORS,
 	.use_clustering = ENABLE_CLUSTERING,
 	.shost_attrs = cxlflash_host_attrs,
@@ -2412,7 +2210,6 @@ MODULE_DEVICE_TABLE(pci, cxlflash_pci_table);
  * Handles the following events:
  * - Link reset which cannot be performed on interrupt context due to
  * blocking up to a few seconds
- * - Read AFU command room
  * - Rescan the host
  */
 static void cxlflash_worker_thread(struct work_struct *work)
@@ -2449,11 +2246,6 @@ static void cxlflash_worker_thread(struct work_struct *work)
 		cfg->lr_state = LINK_RESET_COMPLETE;
 	}
 
-	if (afu->read_room) {
-		atomic64_set(&afu->room, readq_be(&afu->host_map->cmd_room));
-		afu->read_room = false;
-	}
-
 	spin_unlock_irqrestore(cfg->host->host_lock, lock_flags);
 
 	if (atomic_dec_if_positive(&cfg->scan_host_needed) >= 0)
diff --git a/drivers/scsi/cxlflash/sislite.h b/drivers/scsi/cxlflash/sislite.h
index 347fc1671975..1a2d09c148b3 100644
--- a/drivers/scsi/cxlflash/sislite.h
+++ b/drivers/scsi/cxlflash/sislite.h
@@ -72,7 +72,7 @@ struct sisl_ioarcb {
 	u16 timeout;		/* in units specified by req_flags */
 	u32 rsvd1;
 	u8 cdb[16];		/* must be in big endian */
-	struct scsi_cmnd *scp;
+	u64 reserved;		/* Reserved area */
 } __packed;
 
 struct sisl_rc {
diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
index 7bb20684e9fa..d704752b6332 100644
--- a/drivers/scsi/device_handler/scsi_dh_alua.c
+++ b/drivers/scsi/device_handler/scsi_dh_alua.c
@@ -95,7 +95,7 @@ struct alua_port_group {
 
 struct alua_dh_data {
 	struct list_head	node;
-	struct alua_port_group	*pg;
+	struct alua_port_group __rcu *pg;
 	int			group_id;
 	spinlock_t		pg_lock;
 	struct scsi_device	*sdev;
@@ -154,7 +154,8 @@ static int submit_rtpg(struct scsi_device *sdev, unsigned char *buff,
 	return scsi_execute_req_flags(sdev, cdb, DMA_FROM_DEVICE,
 				      buff, bufflen, sshdr,
 				      ALUA_FAILOVER_TIMEOUT * HZ,
-				      ALUA_FAILOVER_RETRIES, NULL, req_flags);
+				      ALUA_FAILOVER_RETRIES, NULL,
+				      req_flags, 0);
 }
 
 /*
@@ -187,7 +188,8 @@ static int submit_stpg(struct scsi_device *sdev, int group_id,
 	return scsi_execute_req_flags(sdev, cdb, DMA_TO_DEVICE,
 				      stpg_data, stpg_len,
 				      sshdr, ALUA_FAILOVER_TIMEOUT * HZ,
-				      ALUA_FAILOVER_RETRIES, NULL, req_flags);
+				      ALUA_FAILOVER_RETRIES, NULL,
+				      req_flags, 0);
 }
 
 static struct alua_port_group *alua_find_get_pg(char *id_str, size_t id_size,
@@ -369,7 +371,7 @@ static int alua_check_vpd(struct scsi_device *sdev, struct alua_dh_data *h,
 
 	/* Check for existing port group references */
 	spin_lock(&h->pg_lock);
-	old_pg = h->pg;
+	old_pg = rcu_dereference_protected(h->pg, lockdep_is_held(&h->pg_lock));
 	if (old_pg != pg) {
 		/* port group has changed. Update to new port group */
 		if (h->pg) {
@@ -388,7 +390,9 @@ static int alua_check_vpd(struct scsi_device *sdev, struct alua_dh_data *h,
 		list_add_rcu(&h->node, &pg->dh_list);
 	spin_unlock_irqrestore(&pg->lock, flags);
 
-	alua_rtpg_queue(h->pg, sdev, NULL, true);
+	alua_rtpg_queue(rcu_dereference_protected(h->pg,
+						  lockdep_is_held(&h->pg_lock)),
+			sdev, NULL, true);
 	spin_unlock(&h->pg_lock);
 
 	if (old_pg)
@@ -940,7 +944,7 @@ static int alua_initialize(struct scsi_device *sdev, struct alua_dh_data *h)
 static int alua_set_params(struct scsi_device *sdev, const char *params)
 {
 	struct alua_dh_data *h = sdev->handler_data;
-	struct alua_port_group __rcu *pg = NULL;
+	struct alua_port_group *pg = NULL;
 	unsigned int optimize = 0, argc;
 	const char *p = params;
 	int result = SCSI_DH_OK;
@@ -987,7 +991,7 @@ static int alua_activate(struct scsi_device *sdev,
 	struct alua_dh_data *h = sdev->handler_data;
 	int err = SCSI_DH_OK;
 	struct alua_queue_data *qdata;
-	struct alua_port_group __rcu *pg;
+	struct alua_port_group *pg;
 
 	qdata = kzalloc(sizeof(*qdata), GFP_KERNEL);
 	if (!qdata) {
@@ -1051,7 +1055,7 @@ static void alua_check(struct scsi_device *sdev, bool force)
 static int alua_prep_fn(struct scsi_device *sdev, struct request *req)
 {
 	struct alua_dh_data *h = sdev->handler_data;
-	struct alua_port_group __rcu *pg;
+	struct alua_port_group *pg;
 	unsigned char state = SCSI_ACCESS_STATE_OPTIMAL;
 	int ret = BLKPREP_OK;
 
@@ -1066,7 +1070,7 @@ static int alua_prep_fn(struct scsi_device *sdev, struct request *req)
 		 state != SCSI_ACCESS_STATE_ACTIVE &&
 		 state != SCSI_ACCESS_STATE_LBA) {
 		ret = BLKPREP_KILL;
-		req->cmd_flags |= REQ_QUIET;
+		req->rq_flags |= RQF_QUIET;
 	}
 	return ret;
 
@@ -1121,7 +1125,7 @@ static void alua_bus_detach(struct scsi_device *sdev)
 	struct alua_port_group *pg;
 
 	spin_lock(&h->pg_lock);
-	pg = h->pg;
+	pg = rcu_dereference_protected(h->pg, lockdep_is_held(&h->pg_lock));
 	rcu_assign_pointer(h->pg, NULL);
 	h->sdev = NULL;
 	spin_unlock(&h->pg_lock);
diff --git a/drivers/scsi/device_handler/scsi_dh_emc.c b/drivers/scsi/device_handler/scsi_dh_emc.c
index 375d81850f15..5b80746980b8 100644
--- a/drivers/scsi/device_handler/scsi_dh_emc.c
+++ b/drivers/scsi/device_handler/scsi_dh_emc.c
@@ -452,7 +452,7 @@ static int clariion_prep_fn(struct scsi_device *sdev, struct request *req)
 
 	if (h->lun_state != CLARIION_LUN_OWNED) {
 		ret = BLKPREP_KILL;
-		req->cmd_flags |= REQ_QUIET;
+		req->rq_flags |= RQF_QUIET;
 	}
 	return ret;
 
diff --git a/drivers/scsi/device_handler/scsi_dh_hp_sw.c b/drivers/scsi/device_handler/scsi_dh_hp_sw.c
index 9406d5f4a3d3..308e87195dc1 100644
--- a/drivers/scsi/device_handler/scsi_dh_hp_sw.c
+++ b/drivers/scsi/device_handler/scsi_dh_hp_sw.c
@@ -266,7 +266,7 @@ static int hp_sw_prep_fn(struct scsi_device *sdev, struct request *req)
 
 	if (h->path_state != HP_SW_PATH_ACTIVE) {
 		ret = BLKPREP_KILL;
-		req->cmd_flags |= REQ_QUIET;
+		req->rq_flags |= RQF_QUIET;
 	}
 	return ret;
 
diff --git a/drivers/scsi/device_handler/scsi_dh_rdac.c b/drivers/scsi/device_handler/scsi_dh_rdac.c
index 06fbd0b0c68a..00d9c326158e 100644
--- a/drivers/scsi/device_handler/scsi_dh_rdac.c
+++ b/drivers/scsi/device_handler/scsi_dh_rdac.c
@@ -724,7 +724,7 @@ static int rdac_prep_fn(struct scsi_device *sdev, struct request *req)
 
 	if (h->state != RDAC_STATE_ACTIVE) {
 		ret = BLKPREP_KILL;
-		req->cmd_flags |= REQ_QUIET;
+		req->rq_flags |= RQF_QUIET;
 	}
 	return ret;
 
diff --git a/drivers/scsi/dmx3191d.c b/drivers/scsi/dmx3191d.c
index 9b5a457d4bca..6af3394d051d 100644
--- a/drivers/scsi/dmx3191d.c
+++ b/drivers/scsi/dmx3191d.c
@@ -34,13 +34,13 @@
  * Definitions for the generic 5380 driver.
  */
 
-#define NCR5380_read(reg)		inb(instance->io_port + reg)
-#define NCR5380_write(reg, value)	outb(value, instance->io_port + reg)
+#define NCR5380_read(reg)		inb(hostdata->base + (reg))
+#define NCR5380_write(reg, value)	outb(value, hostdata->base + (reg))
 
-#define NCR5380_dma_xfer_len(instance, cmd, phase)	(0)
-#define NCR5380_dma_recv_setup(instance, dst, len)	(0)
-#define NCR5380_dma_send_setup(instance, src, len)	(0)
-#define NCR5380_dma_residual(instance)			(0)
+#define NCR5380_dma_xfer_len		NCR5380_dma_xfer_none
+#define NCR5380_dma_recv_setup		NCR5380_dma_setup_none
+#define NCR5380_dma_send_setup		NCR5380_dma_setup_none
+#define NCR5380_dma_residual		NCR5380_dma_residual_none
 
 #define NCR5380_implementation_fields	/* none */
 
@@ -71,6 +71,7 @@ static int dmx3191d_probe_one(struct pci_dev *pdev,
 			      const struct pci_device_id *id)
 {
 	struct Scsi_Host *shost;
+	struct NCR5380_hostdata *hostdata;
 	unsigned long io;
 	int error = -ENODEV;
 
@@ -88,7 +89,9 @@ static int dmx3191d_probe_one(struct pci_dev *pdev,
 			sizeof(struct NCR5380_hostdata));
 	if (!shost)
 		goto out_release_region;       
-	shost->io_port = io;
+
+	hostdata = shost_priv(shost);
+	hostdata->base = io;
 
 	/* This card does not seem to raise an interrupt on pdev->irq.
 	 * Steam-powered SCSI controllers run without an IRQ anyway.
@@ -125,7 +128,8 @@ out_host_put:
 static void dmx3191d_remove_one(struct pci_dev *pdev)
 {
 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
-	unsigned long io = shost->io_port;
+	struct NCR5380_hostdata *hostdata = shost_priv(shost);
+	unsigned long io = hostdata->base;
 
 	scsi_remove_host(shost);
 
@@ -149,18 +153,7 @@ static struct pci_driver dmx3191d_pci_driver = {
 	.remove		= dmx3191d_remove_one,
 };
 
-static int __init dmx3191d_init(void)
-{
-	return pci_register_driver(&dmx3191d_pci_driver);
-}
-
-static void __exit dmx3191d_exit(void)
-{
-	pci_unregister_driver(&dmx3191d_pci_driver);
-}
-
-module_init(dmx3191d_init);
-module_exit(dmx3191d_exit);
+module_pci_driver(dmx3191d_pci_driver);
 
 MODULE_AUTHOR("Massimo Piccioni <dafastidio@libero.it>");
 MODULE_DESCRIPTION("Domex DMX3191D SCSI driver");
diff --git a/drivers/scsi/dpt_i2o.c b/drivers/scsi/dpt_i2o.c
index 21c8d210c456..27c0dce22e72 100644
--- a/drivers/scsi/dpt_i2o.c
+++ b/drivers/scsi/dpt_i2o.c
@@ -651,7 +651,6 @@ static u32 adpt_ioctl_to_context(adpt_hba * pHba, void *reply)
 	}
 	spin_unlock_irqrestore(pHba->host->host_lock, flags);
 	if (i >= nr) {
-		kfree (reply);
 		printk(KERN_WARNING"%s: Too many outstanding "
 				"ioctl commands\n", pHba->name);
 		return (u32)-1;
@@ -1754,8 +1753,10 @@ static int adpt_i2o_passthru(adpt_hba* pHba, u32 __user *arg)
 	sg_offset = (msg[0]>>4)&0xf;
 	msg[2] = 0x40000000; // IOCTL context
 	msg[3] = adpt_ioctl_to_context(pHba, reply);
-	if (msg[3] == (u32)-1)
+	if (msg[3] == (u32)-1) {
+		kfree(reply);
 		return -EBUSY;
+	}
 
 	memset(sg_list,0, sizeof(sg_list[0])*pHba->sg_tablesize);
 	if(sg_offset) {
@@ -3350,7 +3351,7 @@ static int adpt_i2o_query_scalar(adpt_hba* pHba, int tid,
 	if (opblk_va == NULL) {
 		dma_free_coherent(&pHba->pDev->dev, sizeof(u8) * (8+buflen),
 			resblk_va, resblk_pa);
-		printk(KERN_CRIT "%s: query operatio failed; Out of memory.\n",
+		printk(KERN_CRIT "%s: query operation failed; Out of memory.\n",
 			pHba->name);
 		return -ENOMEM;
 	}
diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c
index 9bd41a35a78a..59150cad0353 100644
--- a/drivers/scsi/fcoe/fcoe.c
+++ b/drivers/scsi/fcoe/fcoe.c
@@ -63,6 +63,14 @@ unsigned int fcoe_debug_logging;
 module_param_named(debug_logging, fcoe_debug_logging, int, S_IRUGO|S_IWUSR);
 MODULE_PARM_DESC(debug_logging, "a bit mask of logging levels");
 
+unsigned int fcoe_e_d_tov = 2 * 1000;
+module_param_named(e_d_tov, fcoe_e_d_tov, int, S_IRUGO|S_IWUSR);
+MODULE_PARM_DESC(e_d_tov, "E_D_TOV in ms, default 2000");
+
+unsigned int fcoe_r_a_tov = 2 * 2 * 1000;
+module_param_named(r_a_tov, fcoe_r_a_tov, int, S_IRUGO|S_IWUSR);
+MODULE_PARM_DESC(r_a_tov, "R_A_TOV in ms, default 4000");
+
 static DEFINE_MUTEX(fcoe_config_mutex);
 
 static struct workqueue_struct *fcoe_wq;
@@ -582,7 +590,8 @@ static void fcoe_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb)
 	 * Use default VLAN for FIP VLAN discovery protocol
 	 */
 	frame = (struct fip_frame *)skb->data;
-	if (frame->fip.fip_op == ntohs(FIP_OP_VLAN) &&
+	if (ntohs(frame->eth.h_proto) == ETH_P_FIP &&
+	    ntohs(frame->fip.fip_op) == FIP_OP_VLAN &&
 	    fcoe->realdev != fcoe->netdev)
 		skb->dev = fcoe->realdev;
 	else
@@ -633,8 +642,8 @@ static int fcoe_lport_config(struct fc_lport *lport)
 	lport->qfull = 0;
 	lport->max_retry_count = 3;
 	lport->max_rport_retry_count = 3;
-	lport->e_d_tov = 2 * 1000;	/* FC-FS default */
-	lport->r_a_tov = 2 * 2 * 1000;
+	lport->e_d_tov = fcoe_e_d_tov;
+	lport->r_a_tov = fcoe_r_a_tov;
 	lport->service_params = (FCP_SPPF_INIT_FCN | FCP_SPPF_RD_XRDY_DIS |
 				 FCP_SPPF_RETRY | FCP_SPPF_CONF_COMPL);
 	lport->does_npiv = 1;
@@ -2160,11 +2169,13 @@ static bool fcoe_match(struct net_device *netdev)
  */
 static void fcoe_dcb_create(struct fcoe_interface *fcoe)
 {
+	int ctlr_prio = TC_PRIO_BESTEFFORT;
+	int fcoe_prio = TC_PRIO_INTERACTIVE;
+	struct fcoe_ctlr *ctlr = fcoe_to_ctlr(fcoe);
 #ifdef CONFIG_DCB
 	int dcbx;
 	u8 fup, up;
 	struct net_device *netdev = fcoe->realdev;
-	struct fcoe_ctlr *ctlr = fcoe_to_ctlr(fcoe);
 	struct dcb_app app = {
 				.priority = 0,
 				.protocol = ETH_P_FCOE
@@ -2186,10 +2197,12 @@ static void fcoe_dcb_create(struct fcoe_interface *fcoe)
 			fup = dcb_getapp(netdev, &app);
 		}
 
-		fcoe->priority = ffs(up) ? ffs(up) - 1 : 0;
-		ctlr->priority = ffs(fup) ? ffs(fup) - 1 : fcoe->priority;
+		fcoe_prio = ffs(up) ? ffs(up) - 1 : 0;
+		ctlr_prio = ffs(fup) ? ffs(fup) - 1 : fcoe_prio;
 	}
 #endif
+	fcoe->priority = fcoe_prio;
+	ctlr->priority = ctlr_prio;
 }
 
 enum fcoe_create_link_state {
diff --git a/drivers/scsi/fcoe/fcoe_ctlr.c b/drivers/scsi/fcoe/fcoe_ctlr.c
index dcf36537a767..cea57e27e713 100644
--- a/drivers/scsi/fcoe/fcoe_ctlr.c
+++ b/drivers/scsi/fcoe/fcoe_ctlr.c
@@ -801,6 +801,8 @@ int fcoe_ctlr_els_send(struct fcoe_ctlr *fip, struct fc_lport *lport,
 	return -EINPROGRESS;
 drop:
 	kfree_skb(skb);
+	LIBFCOE_FIP_DBG(fip, "drop els_send op %u d_id %x\n",
+			op, ntoh24(fh->fh_d_id));
 	return -EINVAL;
 }
 EXPORT_SYMBOL(fcoe_ctlr_els_send);
@@ -1316,7 +1318,7 @@ drop:
  * The overall length has already been checked.
  */
 static void fcoe_ctlr_recv_clr_vlink(struct fcoe_ctlr *fip,
-				     struct fip_header *fh)
+				     struct sk_buff *skb)
 {
 	struct fip_desc *desc;
 	struct fip_mac_desc *mp;
@@ -1331,14 +1333,18 @@ static void fcoe_ctlr_recv_clr_vlink(struct fcoe_ctlr *fip,
 	int num_vlink_desc;
 	int reset_phys_port = 0;
 	struct fip_vn_desc **vlink_desc_arr = NULL;
+	struct fip_header *fh = (struct fip_header *)skb->data;
+	struct ethhdr *eh = eth_hdr(skb);
 
 	LIBFCOE_FIP_DBG(fip, "Clear Virtual Link received\n");
 
-	if (!fcf || !lport->port_id) {
+	if (!fcf) {
 		/*
 		 * We are yet to select best FCF, but we got CVL in the
 		 * meantime. reset the ctlr and let it rediscover the FCF
 		 */
+		LIBFCOE_FIP_DBG(fip, "Resetting fcoe_ctlr as FCF has not been "
+		    "selected yet\n");
 		mutex_lock(&fip->ctlr_mutex);
 		fcoe_ctlr_reset(fip);
 		mutex_unlock(&fip->ctlr_mutex);
@@ -1346,6 +1352,31 @@ static void fcoe_ctlr_recv_clr_vlink(struct fcoe_ctlr *fip,
 	}
 
 	/*
+	 * If we've selected an FCF check that the CVL is from there to avoid
+	 * processing CVLs from an unexpected source.  If it is from an
+	 * unexpected source drop it on the floor.
+	 */
+	if (!ether_addr_equal(eh->h_source, fcf->fcf_mac)) {
+		LIBFCOE_FIP_DBG(fip, "Dropping CVL due to source address "
+		    "mismatch with FCF src=%pM\n", eh->h_source);
+		return;
+	}
+
+	/*
+	 * If we haven't logged into the fabric but receive a CVL we should
+	 * reset everything and go back to solicitation.
+	 */
+	if (!lport->port_id) {
+		LIBFCOE_FIP_DBG(fip, "lport not logged in, resoliciting\n");
+		mutex_lock(&fip->ctlr_mutex);
+		fcoe_ctlr_reset(fip);
+		mutex_unlock(&fip->ctlr_mutex);
+		fc_lport_reset(fip->lp);
+		fcoe_ctlr_solicit(fip, NULL);
+		return;
+	}
+
+	/*
 	 * mask of required descriptors.  Validating each one clears its bit.
 	 */
 	desc_mask = BIT(FIP_DT_MAC) | BIT(FIP_DT_NAME);
@@ -1576,7 +1607,7 @@ static int fcoe_ctlr_recv_handler(struct fcoe_ctlr *fip, struct sk_buff *skb)
 	if (op == FIP_OP_DISC && sub == FIP_SC_ADV)
 		fcoe_ctlr_recv_adv(fip, skb);
 	else if (op == FIP_OP_CTRL && sub == FIP_SC_CLR_VLINK)
-		fcoe_ctlr_recv_clr_vlink(fip, fiph);
+		fcoe_ctlr_recv_clr_vlink(fip, skb);
 	kfree_skb(skb);
 	return 0;
 drop:
@@ -2122,7 +2153,7 @@ static void fcoe_ctlr_vn_rport_callback(struct fc_lport *lport,
 			LIBFCOE_FIP_DBG(fip,
 					"rport FLOGI limited port_id %6.6x\n",
 					rdata->ids.port_id);
-			lport->tt.rport_logoff(rdata);
+			fc_rport_logoff(rdata);
 		}
 		break;
 	default:
@@ -2145,9 +2176,15 @@ static void fcoe_ctlr_disc_stop_locked(struct fc_lport *lport)
 {
 	struct fc_rport_priv *rdata;
 
+	rcu_read_lock();
+	list_for_each_entry_rcu(rdata, &lport->disc.rports, peers) {
+		if (kref_get_unless_zero(&rdata->kref)) {
+			fc_rport_logoff(rdata);
+			kref_put(&rdata->kref, fc_rport_destroy);
+		}
+	}
+	rcu_read_unlock();
 	mutex_lock(&lport->disc.disc_mutex);
-	list_for_each_entry_rcu(rdata, &lport->disc.rports, peers)
-		lport->tt.rport_logoff(rdata);
 	lport->disc.disc_callback = NULL;
 	mutex_unlock(&lport->disc.disc_mutex);
 }
@@ -2178,7 +2215,7 @@ static void fcoe_ctlr_disc_stop(struct fc_lport *lport)
 static void fcoe_ctlr_disc_stop_final(struct fc_lport *lport)
 {
 	fcoe_ctlr_disc_stop(lport);
-	lport->tt.rport_flush_queue();
+	fc_rport_flush_queue();
 	synchronize_rcu();
 }
 
@@ -2393,6 +2430,8 @@ static void fcoe_ctlr_vn_probe_req(struct fcoe_ctlr *fip,
 	switch (fip->state) {
 	case FIP_ST_VNMP_CLAIM:
 	case FIP_ST_VNMP_UP:
+		LIBFCOE_FIP_DBG(fip, "vn_probe_req: send reply, state %x\n",
+				fip->state);
 		fcoe_ctlr_vn_send(fip, FIP_SC_VN_PROBE_REP,
 				  frport->enode_mac, 0);
 		break;
@@ -2407,15 +2446,21 @@ static void fcoe_ctlr_vn_probe_req(struct fcoe_ctlr *fip,
 		 */
 		if (fip->lp->wwpn > rdata->ids.port_name &&
 		    !(frport->flags & FIP_FL_REC_OR_P2P)) {
+			LIBFCOE_FIP_DBG(fip, "vn_probe_req: "
+					"port_id collision\n");
 			fcoe_ctlr_vn_send(fip, FIP_SC_VN_PROBE_REP,
 					  frport->enode_mac, 0);
 			break;
 		}
 		/* fall through */
 	case FIP_ST_VNMP_START:
+		LIBFCOE_FIP_DBG(fip, "vn_probe_req: "
+				"restart VN2VN negotiation\n");
 		fcoe_ctlr_vn_restart(fip);
 		break;
 	default:
+		LIBFCOE_FIP_DBG(fip, "vn_probe_req: ignore state %x\n",
+				fip->state);
 		break;
 	}
 }
@@ -2437,9 +2482,12 @@ static void fcoe_ctlr_vn_probe_reply(struct fcoe_ctlr *fip,
 	case FIP_ST_VNMP_PROBE1:
 	case FIP_ST_VNMP_PROBE2:
 	case FIP_ST_VNMP_CLAIM:
+		LIBFCOE_FIP_DBG(fip, "vn_probe_reply: restart state %x\n",
+				fip->state);
 		fcoe_ctlr_vn_restart(fip);
 		break;
 	case FIP_ST_VNMP_UP:
+		LIBFCOE_FIP_DBG(fip, "vn_probe_reply: send claim notify\n");
 		fcoe_ctlr_vn_send_claim(fip);
 		break;
 	default:
@@ -2467,26 +2515,33 @@ static void fcoe_ctlr_vn_add(struct fcoe_ctlr *fip, struct fc_rport_priv *new)
 		return;
 
 	mutex_lock(&lport->disc.disc_mutex);
-	rdata = lport->tt.rport_create(lport, port_id);
+	rdata = fc_rport_create(lport, port_id);
 	if (!rdata) {
 		mutex_unlock(&lport->disc.disc_mutex);
 		return;
 	}
+	mutex_lock(&rdata->rp_mutex);
+	mutex_unlock(&lport->disc.disc_mutex);
 
 	rdata->ops = &fcoe_ctlr_vn_rport_ops;
 	rdata->disc_id = lport->disc.disc_id;
 
 	ids = &rdata->ids;
 	if ((ids->port_name != -1 && ids->port_name != new->ids.port_name) ||
-	    (ids->node_name != -1 && ids->node_name != new->ids.node_name))
-		lport->tt.rport_logoff(rdata);
+	    (ids->node_name != -1 && ids->node_name != new->ids.node_name)) {
+		mutex_unlock(&rdata->rp_mutex);
+		LIBFCOE_FIP_DBG(fip, "vn_add rport logoff %6.6x\n", port_id);
+		fc_rport_logoff(rdata);
+		mutex_lock(&rdata->rp_mutex);
+	}
 	ids->port_name = new->ids.port_name;
 	ids->node_name = new->ids.node_name;
-	mutex_unlock(&lport->disc.disc_mutex);
+	mutex_unlock(&rdata->rp_mutex);
 
 	frport = fcoe_ctlr_rport(rdata);
-	LIBFCOE_FIP_DBG(fip, "vn_add rport %6.6x %s\n",
-			port_id, frport->fcoe_len ? "old" : "new");
+	LIBFCOE_FIP_DBG(fip, "vn_add rport %6.6x %s state %d\n",
+			port_id, frport->fcoe_len ? "old" : "new",
+			rdata->rp_state);
 	*frport = *fcoe_ctlr_rport(new);
 	frport->time = 0;
 }
@@ -2506,12 +2561,12 @@ static int fcoe_ctlr_vn_lookup(struct fcoe_ctlr *fip, u32 port_id, u8 *mac)
 	struct fcoe_rport *frport;
 	int ret = -1;
 
-	rdata = lport->tt.rport_lookup(lport, port_id);
+	rdata = fc_rport_lookup(lport, port_id);
 	if (rdata) {
 		frport = fcoe_ctlr_rport(rdata);
 		memcpy(mac, frport->enode_mac, ETH_ALEN);
 		ret = 0;
-		kref_put(&rdata->kref, lport->tt.rport_destroy);
+		kref_put(&rdata->kref, fc_rport_destroy);
 	}
 	return ret;
 }
@@ -2529,6 +2584,7 @@ static void fcoe_ctlr_vn_claim_notify(struct fcoe_ctlr *fip,
 	struct fcoe_rport *frport = fcoe_ctlr_rport(new);
 
 	if (frport->flags & FIP_FL_REC_OR_P2P) {
+		LIBFCOE_FIP_DBG(fip, "send probe req for P2P/REC\n");
 		fcoe_ctlr_vn_send(fip, FIP_SC_VN_PROBE_REQ, fcoe_all_vn2vn, 0);
 		return;
 	}
@@ -2536,25 +2592,37 @@ static void fcoe_ctlr_vn_claim_notify(struct fcoe_ctlr *fip,
 	case FIP_ST_VNMP_START:
 	case FIP_ST_VNMP_PROBE1:
 	case FIP_ST_VNMP_PROBE2:
-		if (new->ids.port_id == fip->port_id)
+		if (new->ids.port_id == fip->port_id) {
+			LIBFCOE_FIP_DBG(fip, "vn_claim_notify: "
+					"restart, state %d\n",
+					fip->state);
 			fcoe_ctlr_vn_restart(fip);
+		}
 		break;
 	case FIP_ST_VNMP_CLAIM:
 	case FIP_ST_VNMP_UP:
 		if (new->ids.port_id == fip->port_id) {
 			if (new->ids.port_name > fip->lp->wwpn) {
+				LIBFCOE_FIP_DBG(fip, "vn_claim_notify: "
+						"restart, port_id collision\n");
 				fcoe_ctlr_vn_restart(fip);
 				break;
 			}
+			LIBFCOE_FIP_DBG(fip, "vn_claim_notify: "
+					"send claim notify\n");
 			fcoe_ctlr_vn_send_claim(fip);
 			break;
 		}
+		LIBFCOE_FIP_DBG(fip, "vn_claim_notify: send reply to %x\n",
+				new->ids.port_id);
 		fcoe_ctlr_vn_send(fip, FIP_SC_VN_CLAIM_REP, frport->enode_mac,
 				  min((u32)frport->fcoe_len,
 				      fcoe_ctlr_fcoe_size(fip)));
 		fcoe_ctlr_vn_add(fip, new);
 		break;
 	default:
+		LIBFCOE_FIP_DBG(fip, "vn_claim_notify: "
+				"ignoring claim from %x\n", new->ids.port_id);
 		break;
 	}
 }
@@ -2591,19 +2659,26 @@ static void fcoe_ctlr_vn_beacon(struct fcoe_ctlr *fip,
 
 	frport = fcoe_ctlr_rport(new);
 	if (frport->flags & FIP_FL_REC_OR_P2P) {
+		LIBFCOE_FIP_DBG(fip, "p2p beacon while in vn2vn mode\n");
 		fcoe_ctlr_vn_send(fip, FIP_SC_VN_PROBE_REQ, fcoe_all_vn2vn, 0);
 		return;
 	}
-	rdata = lport->tt.rport_lookup(lport, new->ids.port_id);
+	rdata = fc_rport_lookup(lport, new->ids.port_id);
 	if (rdata) {
 		if (rdata->ids.node_name == new->ids.node_name &&
 		    rdata->ids.port_name == new->ids.port_name) {
 			frport = fcoe_ctlr_rport(rdata);
-			if (!frport->time && fip->state == FIP_ST_VNMP_UP)
-				lport->tt.rport_login(rdata);
+			LIBFCOE_FIP_DBG(fip, "beacon from rport %x\n",
+					rdata->ids.port_id);
+			if (!frport->time && fip->state == FIP_ST_VNMP_UP) {
+				LIBFCOE_FIP_DBG(fip, "beacon expired "
+						"for rport %x\n",
+						rdata->ids.port_id);
+				fc_rport_login(rdata);
+			}
 			frport->time = jiffies;
 		}
-		kref_put(&rdata->kref, lport->tt.rport_destroy);
+		kref_put(&rdata->kref, fc_rport_destroy);
 		return;
 	}
 	if (fip->state != FIP_ST_VNMP_UP)
@@ -2638,11 +2713,15 @@ static unsigned long fcoe_ctlr_vn_age(struct fcoe_ctlr *fip)
 	unsigned long deadline;
 
 	next_time = jiffies + msecs_to_jiffies(FIP_VN_BEACON_INT * 10);
-	mutex_lock(&lport->disc.disc_mutex);
+	rcu_read_lock();
 	list_for_each_entry_rcu(rdata, &lport->disc.rports, peers) {
+		if (!kref_get_unless_zero(&rdata->kref))
+			continue;
 		frport = fcoe_ctlr_rport(rdata);
-		if (!frport->time)
+		if (!frport->time) {
+			kref_put(&rdata->kref, fc_rport_destroy);
 			continue;
+		}
 		deadline = frport->time +
 			   msecs_to_jiffies(FIP_VN_BEACON_INT * 25 / 10);
 		if (time_after_eq(jiffies, deadline)) {
@@ -2650,11 +2729,12 @@ static unsigned long fcoe_ctlr_vn_age(struct fcoe_ctlr *fip)
 			LIBFCOE_FIP_DBG(fip,
 				"port %16.16llx fc_id %6.6x beacon expired\n",
 				rdata->ids.port_name, rdata->ids.port_id);
-			lport->tt.rport_logoff(rdata);
+			fc_rport_logoff(rdata);
 		} else if (time_before(deadline, next_time))
 			next_time = deadline;
+		kref_put(&rdata->kref, fc_rport_destroy);
 	}
-	mutex_unlock(&lport->disc.disc_mutex);
+	rcu_read_unlock();
 	return next_time;
 }
 
@@ -2674,11 +2754,21 @@ static int fcoe_ctlr_vn_recv(struct fcoe_ctlr *fip, struct sk_buff *skb)
 		struct fc_rport_priv rdata;
 		struct fcoe_rport frport;
 	} buf;
-	int rc;
+	int rc, vlan_id = 0;
 
 	fiph = (struct fip_header *)skb->data;
 	sub = fiph->fip_subcode;
 
+	if (fip->lp->vlan)
+		vlan_id = skb_vlan_tag_get_id(skb);
+
+	if (vlan_id && vlan_id != fip->lp->vlan) {
+		LIBFCOE_FIP_DBG(fip, "vn_recv drop frame sub %x vlan %d\n",
+				sub, vlan_id);
+		rc = -EAGAIN;
+		goto drop;
+	}
+
 	rc = fcoe_ctlr_vn_parse(fip, skb, &buf.rdata);
 	if (rc) {
 		LIBFCOE_FIP_DBG(fip, "vn_recv vn_parse error %d\n", rc);
@@ -2941,7 +3031,7 @@ static void fcoe_ctlr_disc_recv(struct fc_lport *lport, struct fc_frame *fp)
 
 	rjt_data.reason = ELS_RJT_UNSUP;
 	rjt_data.explan = ELS_EXPL_NONE;
-	lport->tt.seq_els_rsp_send(fp, ELS_LS_RJT, &rjt_data);
+	fc_seq_els_rsp_send(fp, ELS_LS_RJT, &rjt_data);
 	fc_frame_free(fp);
 }
 
@@ -2991,12 +3081,17 @@ static void fcoe_ctlr_vn_disc(struct fcoe_ctlr *fip)
 	mutex_lock(&disc->disc_mutex);
 	callback = disc->pending ? disc->disc_callback : NULL;
 	disc->pending = 0;
+	mutex_unlock(&disc->disc_mutex);
+	rcu_read_lock();
 	list_for_each_entry_rcu(rdata, &disc->rports, peers) {
+		if (!kref_get_unless_zero(&rdata->kref))
+			continue;
 		frport = fcoe_ctlr_rport(rdata);
 		if (frport->time)
-			lport->tt.rport_login(rdata);
+			fc_rport_login(rdata);
+		kref_put(&rdata->kref, fc_rport_destroy);
 	}
-	mutex_unlock(&disc->disc_mutex);
+	rcu_read_unlock();
 	if (callback)
 		callback(lport, DISC_EV_SUCCESS);
 }
@@ -3015,11 +3110,13 @@ static void fcoe_ctlr_vn_timeout(struct fcoe_ctlr *fip)
 	switch (fip->state) {
 	case FIP_ST_VNMP_START:
 		fcoe_ctlr_set_state(fip, FIP_ST_VNMP_PROBE1);
+		LIBFCOE_FIP_DBG(fip, "vn_timeout: send 1st probe request\n");
 		fcoe_ctlr_vn_send(fip, FIP_SC_VN_PROBE_REQ, fcoe_all_vn2vn, 0);
 		next_time = jiffies + msecs_to_jiffies(FIP_VN_PROBE_WAIT);
 		break;
 	case FIP_ST_VNMP_PROBE1:
 		fcoe_ctlr_set_state(fip, FIP_ST_VNMP_PROBE2);
+		LIBFCOE_FIP_DBG(fip, "vn_timeout: send 2nd probe request\n");
 		fcoe_ctlr_vn_send(fip, FIP_SC_VN_PROBE_REQ, fcoe_all_vn2vn, 0);
 		next_time = jiffies + msecs_to_jiffies(FIP_VN_ANN_WAIT);
 		break;
@@ -3030,6 +3127,7 @@ static void fcoe_ctlr_vn_timeout(struct fcoe_ctlr *fip)
 		hton24(mac + 3, new_port_id);
 		fcoe_ctlr_map_dest(fip);
 		fip->update_mac(fip->lp, mac);
+		LIBFCOE_FIP_DBG(fip, "vn_timeout: send claim notify\n");
 		fcoe_ctlr_vn_send_claim(fip);
 		next_time = jiffies + msecs_to_jiffies(FIP_VN_ANN_WAIT);
 		break;
@@ -3041,6 +3139,7 @@ static void fcoe_ctlr_vn_timeout(struct fcoe_ctlr *fip)
 		next_time = fip->sol_time + msecs_to_jiffies(FIP_VN_ANN_WAIT);
 		if (time_after_eq(jiffies, next_time)) {
 			fcoe_ctlr_set_state(fip, FIP_ST_VNMP_UP);
+			LIBFCOE_FIP_DBG(fip, "vn_timeout: send vn2vn beacon\n");
 			fcoe_ctlr_vn_send(fip, FIP_SC_VN_BEACON,
 					  fcoe_all_vn2vn, 0);
 			next_time = jiffies + msecs_to_jiffies(FIP_VN_ANN_WAIT);
@@ -3051,6 +3150,7 @@ static void fcoe_ctlr_vn_timeout(struct fcoe_ctlr *fip)
 	case FIP_ST_VNMP_UP:
 		next_time = fcoe_ctlr_vn_age(fip);
 		if (time_after_eq(jiffies, fip->port_ka_time)) {
+			LIBFCOE_FIP_DBG(fip, "vn_timeout: send vn2vn beacon\n");
 			fcoe_ctlr_vn_send(fip, FIP_SC_VN_BEACON,
 					  fcoe_all_vn2vn, 0);
 			fip->port_ka_time = jiffies +
@@ -3135,7 +3235,6 @@ int fcoe_libfc_config(struct fc_lport *lport, struct fcoe_ctlr *fip,
 	fc_exch_init(lport);
 	fc_elsct_init(lport);
 	fc_lport_init(lport);
-	fc_rport_init(lport);
 	fc_disc_init(lport);
 	fcoe_ctlr_mode_set(lport, fip, fip->mode);
 	return 0;
diff --git a/drivers/scsi/fcoe/fcoe_sysfs.c b/drivers/scsi/fcoe/fcoe_sysfs.c
index 0675fd128734..9cf3d56296ab 100644
--- a/drivers/scsi/fcoe/fcoe_sysfs.c
+++ b/drivers/scsi/fcoe/fcoe_sysfs.c
@@ -335,16 +335,24 @@ static ssize_t store_ctlr_enabled(struct device *dev,
 				  const char *buf, size_t count)
 {
 	struct fcoe_ctlr_device *ctlr = dev_to_ctlr(dev);
+	bool enabled;
 	int rc;
 
+	if (*buf == '1')
+		enabled = true;
+	else if (*buf == '0')
+		enabled = false;
+	else
+		return -EINVAL;
+
 	switch (ctlr->enabled) {
 	case FCOE_CTLR_ENABLED:
-		if (*buf == '1')
+		if (enabled)
 			return count;
 		ctlr->enabled = FCOE_CTLR_DISABLED;
 		break;
 	case FCOE_CTLR_DISABLED:
-		if (*buf == '0')
+		if (!enabled)
 			return count;
 		ctlr->enabled = FCOE_CTLR_ENABLED;
 		break;
@@ -424,6 +432,75 @@ static FCOE_DEVICE_ATTR(ctlr, fip_vlan_responder, S_IRUGO | S_IWUSR,
 			store_ctlr_fip_resp);
 
 static ssize_t
+fcoe_ctlr_var_store(u32 *var, const char *buf, size_t count)
+{
+	int err;
+	unsigned long v;
+
+	err = kstrtoul(buf, 10, &v);
+	if (err || v > UINT_MAX)
+		return -EINVAL;
+
+	*var = v;
+
+	return count;
+}
+
+static ssize_t store_ctlr_r_a_tov(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t count)
+{
+	struct fcoe_ctlr_device *ctlr_dev = dev_to_ctlr(dev);
+	struct fcoe_ctlr *ctlr = fcoe_ctlr_device_priv(ctlr_dev);
+
+	if (ctlr_dev->enabled == FCOE_CTLR_ENABLED)
+		return -EBUSY;
+	if (ctlr_dev->enabled == FCOE_CTLR_DISABLED)
+		return fcoe_ctlr_var_store(&ctlr->lp->r_a_tov, buf, count);
+	return -ENOTSUPP;
+}
+
+static ssize_t show_ctlr_r_a_tov(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct fcoe_ctlr_device *ctlr_dev = dev_to_ctlr(dev);
+	struct fcoe_ctlr *ctlr = fcoe_ctlr_device_priv(ctlr_dev);
+
+	return sprintf(buf, "%d\n", ctlr->lp->r_a_tov);
+}
+
+static FCOE_DEVICE_ATTR(ctlr, r_a_tov, S_IRUGO | S_IWUSR,
+			show_ctlr_r_a_tov, store_ctlr_r_a_tov);
+
+static ssize_t store_ctlr_e_d_tov(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t count)
+{
+	struct fcoe_ctlr_device *ctlr_dev = dev_to_ctlr(dev);
+	struct fcoe_ctlr *ctlr = fcoe_ctlr_device_priv(ctlr_dev);
+
+	if (ctlr_dev->enabled == FCOE_CTLR_ENABLED)
+		return -EBUSY;
+	if (ctlr_dev->enabled == FCOE_CTLR_DISABLED)
+		return fcoe_ctlr_var_store(&ctlr->lp->e_d_tov, buf, count);
+	return -ENOTSUPP;
+}
+
+static ssize_t show_ctlr_e_d_tov(struct device *dev,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct fcoe_ctlr_device *ctlr_dev = dev_to_ctlr(dev);
+	struct fcoe_ctlr *ctlr = fcoe_ctlr_device_priv(ctlr_dev);
+
+	return sprintf(buf, "%d\n", ctlr->lp->e_d_tov);
+}
+
+static FCOE_DEVICE_ATTR(ctlr, e_d_tov, S_IRUGO | S_IWUSR,
+			show_ctlr_e_d_tov, store_ctlr_e_d_tov);
+
+static ssize_t
 store_private_fcoe_ctlr_fcf_dev_loss_tmo(struct device *dev,
 					 struct device_attribute *attr,
 					 const char *buf, size_t count)
@@ -507,6 +584,8 @@ static struct attribute_group fcoe_ctlr_lesb_attr_group = {
 static struct attribute *fcoe_ctlr_attrs[] = {
 	&device_attr_fcoe_ctlr_fip_vlan_responder.attr,
 	&device_attr_fcoe_ctlr_fcf_dev_loss_tmo.attr,
+	&device_attr_fcoe_ctlr_r_a_tov.attr,
+	&device_attr_fcoe_ctlr_e_d_tov.attr,
 	&device_attr_fcoe_ctlr_enabled.attr,
 	&device_attr_fcoe_ctlr_mode.attr,
 	NULL,
diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c
index d9fd2f841585..2544a37ece0a 100644
--- a/drivers/scsi/fnic/fnic_scsi.c
+++ b/drivers/scsi/fnic/fnic_scsi.c
@@ -441,30 +441,38 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc, void (*done)(struct scsi_
 	unsigned long ptr;
 	spinlock_t *io_lock = NULL;
 	int io_lock_acquired = 0;
+	struct fc_rport_libfc_priv *rp;
 
 	if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED)))
 		return SCSI_MLQUEUE_HOST_BUSY;
 
 	rport = starget_to_rport(scsi_target(sc->device));
+	if (!rport) {
+		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,
+				"returning DID_NO_CONNECT for IO as rport is NULL\n");
+		sc->result = DID_NO_CONNECT << 16;
+		done(sc);
+		return 0;
+	}
+
 	ret = fc_remote_port_chkready(rport);
 	if (ret) {
+		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,
+				"rport is not ready\n");
 		atomic64_inc(&fnic_stats->misc_stats.rport_not_ready);
 		sc->result = ret;
 		done(sc);
 		return 0;
 	}
 
-	if (rport) {
-		struct fc_rport_libfc_priv *rp = rport->dd_data;
-
-		if (!rp || rp->rp_state != RPORT_ST_READY) {
-			FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,
+	rp = rport->dd_data;
+	if (!rp || rp->rp_state != RPORT_ST_READY) {
+		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,
 				"returning DID_NO_CONNECT for IO as rport is removed\n");
-			atomic64_inc(&fnic_stats->misc_stats.rport_not_ready);
-			sc->result = DID_NO_CONNECT<<16;
-			done(sc);
-			return 0;
-		}
+		atomic64_inc(&fnic_stats->misc_stats.rport_not_ready);
+		sc->result = DID_NO_CONNECT<<16;
+		done(sc);
+		return 0;
 	}
 
 	if (lp->state != LPORT_ST_READY || !(lp->link_up))
@@ -2543,7 +2551,7 @@ int fnic_reset(struct Scsi_Host *shost)
 	 * Reset local port, this will clean up libFC exchanges,
 	 * reset remote port sessions, and if link is up, begin flogi
 	 */
-	ret = lp->tt.lport_reset(lp);
+	ret = fc_lport_reset(lp);
 
 	FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,
 		      "Returning from fnic reset %s\n",
diff --git a/drivers/scsi/fnic/fnic_trace.c b/drivers/scsi/fnic/fnic_trace.c
index 4e15c4bf0795..5a5fa01576b7 100644
--- a/drivers/scsi/fnic/fnic_trace.c
+++ b/drivers/scsi/fnic/fnic_trace.c
@@ -613,7 +613,7 @@ int fnic_fc_trace_set_data(u32 host_no, u8 frame_type,
 			fc_trace_entries.rd_idx = 0;
 	}
 
-	fc_buf->time_stamp = CURRENT_TIME;
+	ktime_get_real_ts64(&fc_buf->time_stamp);
 	fc_buf->host_no = host_no;
 	fc_buf->frame_type = frame_type;
 
@@ -740,7 +740,7 @@ void copy_and_format_trace_data(struct fc_trace_hdr *tdata,
 
 	len = *orig_len;
 
-	time_to_tm(tdata->time_stamp.tv_sec, 0, &tm);
+	time64_to_tm(tdata->time_stamp.tv_sec, 0, &tm);
 
 	fmt = "%02d:%02d:%04ld %02d:%02d:%02d.%09lu ns%8x       %c%8x\t";
 	len += snprintf(fnic_dbgfs_prt->buffer + len,
diff --git a/drivers/scsi/fnic/fnic_trace.h b/drivers/scsi/fnic/fnic_trace.h
index a8aa0578fcb0..e375d0c2eaaf 100644
--- a/drivers/scsi/fnic/fnic_trace.h
+++ b/drivers/scsi/fnic/fnic_trace.h
@@ -72,7 +72,7 @@ struct fnic_trace_data {
 typedef struct fnic_trace_data fnic_trace_data_t;
 
 struct fc_trace_hdr {
-	struct timespec time_stamp;
+	struct timespec64 time_stamp;
 	u32 host_no;
 	u8 frame_type;
 	u8 frame_len;
diff --git a/drivers/scsi/fnic/vnic_dev.c b/drivers/scsi/fnic/vnic_dev.c
index 9795d6f3e197..ba69d6112fa1 100644
--- a/drivers/scsi/fnic/vnic_dev.c
+++ b/drivers/scsi/fnic/vnic_dev.c
@@ -499,10 +499,7 @@ void vnic_dev_add_addr(struct vnic_dev *vdev, u8 *addr)
 
 	err = vnic_dev_cmd(vdev, CMD_ADDR_ADD, &a0, &a1, wait);
 	if (err)
-		printk(KERN_ERR
-			"Can't add addr [%02x:%02x:%02x:%02x:%02x:%02x], %d\n",
-			addr[0], addr[1], addr[2], addr[3], addr[4], addr[5],
-			err);
+		pr_err("Can't add addr [%pM], %d\n", addr, err);
 }
 
 void vnic_dev_del_addr(struct vnic_dev *vdev, u8 *addr)
@@ -517,10 +514,7 @@ void vnic_dev_del_addr(struct vnic_dev *vdev, u8 *addr)
 
 	err = vnic_dev_cmd(vdev, CMD_ADDR_DEL, &a0, &a1, wait);
 	if (err)
-		printk(KERN_ERR
-			"Can't del addr [%02x:%02x:%02x:%02x:%02x:%02x], %d\n",
-			addr[0], addr[1], addr[2], addr[3], addr[4], addr[5],
-			err);
+		pr_err("Can't del addr [%pM], %d\n", addr, err);
 }
 
 int vnic_dev_notify_set(struct vnic_dev *vdev, u16 intr)
diff --git a/drivers/scsi/g_NCR5380.c b/drivers/scsi/g_NCR5380.c
index cbf010324c18..de5147a8c959 100644
--- a/drivers/scsi/g_NCR5380.c
+++ b/drivers/scsi/g_NCR5380.c
@@ -64,9 +64,9 @@ static int card[] = { -1, -1, -1, -1, -1, -1, -1, -1 };
 module_param_array(card, int, NULL, 0);
 MODULE_PARM_DESC(card, "card type (0=NCR5380, 1=NCR53C400, 2=NCR53C400A, 3=DTC3181E, 4=HP C2502)");
 
+MODULE_ALIAS("g_NCR5380_mmio");
 MODULE_LICENSE("GPL");
 
-#ifndef SCSI_G_NCR5380_MEM
 /*
  * Configure I/O address of 53C400A or DTC436 by writing magic numbers
  * to ports 0x779 and 0x379.
@@ -88,40 +88,35 @@ static void magic_configure(int idx, u8 irq, u8 magic[])
 		cfg = 0x80 | idx | (irq << 4);
 	outb(cfg, 0x379);
 }
-#endif
+
+static unsigned int ncr_53c400a_ports[] = {
+	0x280, 0x290, 0x300, 0x310, 0x330, 0x340, 0x348, 0x350, 0
+};
+static unsigned int dtc_3181e_ports[] = {
+	0x220, 0x240, 0x280, 0x2a0, 0x2c0, 0x300, 0x320, 0x340, 0
+};
+static u8 ncr_53c400a_magic[] = {	/* 53C400A & DTC436 */
+	0x59, 0xb9, 0xc5, 0xae, 0xa6
+};
+static u8 hp_c2502_magic[] = {	/* HP C2502 */
+	0x0f, 0x22, 0xf0, 0x20, 0x80
+};
 
 static int generic_NCR5380_init_one(struct scsi_host_template *tpnt,
 			struct device *pdev, int base, int irq, int board)
 {
-	unsigned int *ports;
+	bool is_pmio = base <= 0xffff;
+	int ret;
+	int flags = 0;
+	unsigned int *ports = NULL;
 	u8 *magic = NULL;
-#ifndef SCSI_G_NCR5380_MEM
 	int i;
 	int port_idx = -1;
 	unsigned long region_size;
-#endif
-	static unsigned int ncr_53c400a_ports[] = {
-		0x280, 0x290, 0x300, 0x310, 0x330, 0x340, 0x348, 0x350, 0
-	};
-	static unsigned int dtc_3181e_ports[] = {
-		0x220, 0x240, 0x280, 0x2a0, 0x2c0, 0x300, 0x320, 0x340, 0
-	};
-	static u8 ncr_53c400a_magic[] = {	/* 53C400A & DTC436 */
-		0x59, 0xb9, 0xc5, 0xae, 0xa6
-	};
-	static u8 hp_c2502_magic[] = {	/* HP C2502 */
-		0x0f, 0x22, 0xf0, 0x20, 0x80
-	};
-	int flags, ret;
 	struct Scsi_Host *instance;
 	struct NCR5380_hostdata *hostdata;
-#ifdef SCSI_G_NCR5380_MEM
-	void __iomem *iomem;
-	resource_size_t iomem_size;
-#endif
+	u8 __iomem *iomem;
 
-	ports = NULL;
-	flags = 0;
 	switch (board) {
 	case BOARD_NCR5380:
 		flags = FLAG_NO_PSEUDO_DMA | FLAG_DMA_FIXUP;
@@ -140,8 +135,7 @@ static int generic_NCR5380_init_one(struct scsi_host_template *tpnt,
 		break;
 	}
 
-#ifndef SCSI_G_NCR5380_MEM
-	if (ports && magic) {
+	if (is_pmio && ports && magic) {
 		/* wakeup sequence for the NCR53C400A and DTC3181E */
 
 		/* Disable the adapter and look for a free io port */
@@ -170,84 +164,89 @@ static int generic_NCR5380_init_one(struct scsi_host_template *tpnt,
 		if (ports[i]) {
 			/* At this point we have our region reserved */
 			magic_configure(i, 0, magic); /* no IRQ yet */
-			outb(0xc0, ports[i] + 9);
-			if (inb(ports[i] + 9) != 0x80) {
+			base = ports[i];
+			outb(0xc0, base + 9);
+			if (inb(base + 9) != 0x80) {
 				ret = -ENODEV;
 				goto out_release;
 			}
-			base = ports[i];
 			port_idx = i;
 		} else
 			return -EINVAL;
-	}
-	else
-	{
+	} else if (is_pmio) {
 		/* NCR5380 - no configuration, just grab */
 		region_size = 8;
 		if (!base || !request_region(base, region_size, "ncr5380"))
 			return -EBUSY;
+	} else {	/* MMIO */
+		region_size = NCR53C400_region_size;
+		if (!request_mem_region(base, region_size, "ncr5380"))
+			return -EBUSY;
 	}
-#else
-	iomem_size = NCR53C400_region_size;
-	if (!request_mem_region(base, iomem_size, "ncr5380"))
-		return -EBUSY;
-	iomem = ioremap(base, iomem_size);
+
+	if (is_pmio)
+		iomem = ioport_map(base, region_size);
+	else
+		iomem = ioremap(base, region_size);
+
 	if (!iomem) {
-		release_mem_region(base, iomem_size);
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto out_release;
 	}
-#endif
+
 	instance = scsi_host_alloc(tpnt, sizeof(struct NCR5380_hostdata));
 	if (instance == NULL) {
 		ret = -ENOMEM;
-		goto out_release;
+		goto out_unmap;
 	}
 	hostdata = shost_priv(instance);
 
-#ifndef SCSI_G_NCR5380_MEM
-	instance->io_port = base;
-	instance->n_io_port = region_size;
-	hostdata->io_width = 1; /* 8-bit PDMA by default */
-
-	/*
-	 * On NCR53C400 boards, NCR5380 registers are mapped 8 past
-	 * the base address.
-	 */
-	switch (board) {
-	case BOARD_NCR53C400:
-		instance->io_port += 8;
-		hostdata->c400_ctl_status = 0;
-		hostdata->c400_blk_cnt = 1;
-		hostdata->c400_host_buf = 4;
-		break;
-	case BOARD_DTC3181E:
-		hostdata->io_width = 2;	/* 16-bit PDMA */
-		/* fall through */
-	case BOARD_NCR53C400A:
-	case BOARD_HP_C2502:
-		hostdata->c400_ctl_status = 9;
-		hostdata->c400_blk_cnt = 10;
-		hostdata->c400_host_buf = 8;
-		break;
-	}
-#else
-	instance->base = base;
-	hostdata->iomem = iomem;
-	hostdata->iomem_size = iomem_size;
-	switch (board) {
-	case BOARD_NCR53C400:
-		hostdata->c400_ctl_status = 0x100;
-		hostdata->c400_blk_cnt = 0x101;
-		hostdata->c400_host_buf = 0x104;
-		break;
-	case BOARD_DTC3181E:
-	case BOARD_NCR53C400A:
-	case BOARD_HP_C2502:
-		pr_err(DRV_MODULE_NAME ": unknown register offsets\n");
-		ret = -EINVAL;
-		goto out_unregister;
+	hostdata->io = iomem;
+	hostdata->region_size = region_size;
+
+	if (is_pmio) {
+		hostdata->io_port = base;
+		hostdata->io_width = 1; /* 8-bit PDMA by default */
+		hostdata->offset = 0;
+
+		/*
+		 * On NCR53C400 boards, NCR5380 registers are mapped 8 past
+		 * the base address.
+		 */
+		switch (board) {
+		case BOARD_NCR53C400:
+			hostdata->io_port += 8;
+			hostdata->c400_ctl_status = 0;
+			hostdata->c400_blk_cnt = 1;
+			hostdata->c400_host_buf = 4;
+			break;
+		case BOARD_DTC3181E:
+			hostdata->io_width = 2;	/* 16-bit PDMA */
+			/* fall through */
+		case BOARD_NCR53C400A:
+		case BOARD_HP_C2502:
+			hostdata->c400_ctl_status = 9;
+			hostdata->c400_blk_cnt = 10;
+			hostdata->c400_host_buf = 8;
+			break;
+		}
+	} else {
+		hostdata->base = base;
+		hostdata->offset = NCR53C400_mem_base;
+		switch (board) {
+		case BOARD_NCR53C400:
+			hostdata->c400_ctl_status = 0x100;
+			hostdata->c400_blk_cnt = 0x101;
+			hostdata->c400_host_buf = 0x104;
+			break;
+		case BOARD_DTC3181E:
+		case BOARD_NCR53C400A:
+		case BOARD_HP_C2502:
+			pr_err(DRV_MODULE_NAME ": unknown register offsets\n");
+			ret = -EINVAL;
+			goto out_unregister;
+		}
 	}
-#endif
 
 	ret = NCR5380_init(instance, flags | FLAG_LATE_DMA_SETUP);
 	if (ret)
@@ -273,11 +272,9 @@ static int generic_NCR5380_init_one(struct scsi_host_template *tpnt,
 		instance->irq = NO_IRQ;
 
 	if (instance->irq != NO_IRQ) {
-#ifndef SCSI_G_NCR5380_MEM
 		/* set IRQ for HP C2502 */
 		if (board == BOARD_HP_C2502)
 			magic_configure(port_idx, instance->irq, magic);
-#endif
 		if (request_irq(instance->irq, generic_NCR5380_intr,
 				0, "NCR5380", instance)) {
 			printk(KERN_WARNING "scsi%d : IRQ%d not free, interrupts disabled\n", instance->host_no, instance->irq);
@@ -303,38 +300,39 @@ out_free_irq:
 	NCR5380_exit(instance);
 out_unregister:
 	scsi_host_put(instance);
-out_release:
-#ifndef SCSI_G_NCR5380_MEM
-	release_region(base, region_size);
-#else
+out_unmap:
 	iounmap(iomem);
-	release_mem_region(base, iomem_size);
-#endif
+out_release:
+	if (is_pmio)
+		release_region(base, region_size);
+	else
+		release_mem_region(base, region_size);
 	return ret;
 }
 
 static void generic_NCR5380_release_resources(struct Scsi_Host *instance)
 {
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	void __iomem *iomem = hostdata->io;
+	unsigned long io_port = hostdata->io_port;
+	unsigned long base = hostdata->base;
+	unsigned long region_size = hostdata->region_size;
+
 	scsi_remove_host(instance);
 	if (instance->irq != NO_IRQ)
 		free_irq(instance->irq, instance);
 	NCR5380_exit(instance);
-#ifndef SCSI_G_NCR5380_MEM
-	release_region(instance->io_port, instance->n_io_port);
-#else
-	{
-		struct NCR5380_hostdata *hostdata = shost_priv(instance);
-
-		iounmap(hostdata->iomem);
-		release_mem_region(instance->base, hostdata->iomem_size);
-	}
-#endif
 	scsi_host_put(instance);
+	iounmap(iomem);
+	if (io_port)
+		release_region(io_port, region_size);
+	else
+		release_mem_region(base, region_size);
 }
 
 /**
  *	generic_NCR5380_pread - pseudo DMA read
- *	@instance: adapter to read from
+ *	@hostdata: scsi host private data
  *	@dst: buffer to read into
  *	@len: buffer length
  *
@@ -342,10 +340,9 @@ static void generic_NCR5380_release_resources(struct Scsi_Host *instance)
  *	controller
  */
  
-static inline int generic_NCR5380_pread(struct Scsi_Host *instance,
+static inline int generic_NCR5380_pread(struct NCR5380_hostdata *hostdata,
                                         unsigned char *dst, int len)
 {
-	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	int blocks = len / 128;
 	int start = 0;
 
@@ -361,18 +358,16 @@ static inline int generic_NCR5380_pread(struct Scsi_Host *instance,
 		while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
 			; /* FIXME - no timeout */
 
-#ifndef SCSI_G_NCR5380_MEM
-		if (hostdata->io_width == 2)
-			insw(instance->io_port + hostdata->c400_host_buf,
+		if (hostdata->io_port && hostdata->io_width == 2)
+			insw(hostdata->io_port + hostdata->c400_host_buf,
 							dst + start, 64);
-		else
-			insb(instance->io_port + hostdata->c400_host_buf,
+		else if (hostdata->io_port)
+			insb(hostdata->io_port + hostdata->c400_host_buf,
 							dst + start, 128);
-#else
-		/* implies SCSI_G_NCR5380_MEM */
-		memcpy_fromio(dst + start,
-		              hostdata->iomem + NCR53C400_host_buffer, 128);
-#endif
+		else
+			memcpy_fromio(dst + start,
+				hostdata->io + NCR53C400_host_buffer, 128);
+
 		start += 128;
 		blocks--;
 	}
@@ -381,18 +376,16 @@ static inline int generic_NCR5380_pread(struct Scsi_Host *instance,
 		while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
 			; /* FIXME - no timeout */
 
-#ifndef SCSI_G_NCR5380_MEM
-		if (hostdata->io_width == 2)
-			insw(instance->io_port + hostdata->c400_host_buf,
+		if (hostdata->io_port && hostdata->io_width == 2)
+			insw(hostdata->io_port + hostdata->c400_host_buf,
 							dst + start, 64);
-		else
-			insb(instance->io_port + hostdata->c400_host_buf,
+		else if (hostdata->io_port)
+			insb(hostdata->io_port + hostdata->c400_host_buf,
 							dst + start, 128);
-#else
-		/* implies SCSI_G_NCR5380_MEM */
-		memcpy_fromio(dst + start,
-		              hostdata->iomem + NCR53C400_host_buffer, 128);
-#endif
+		else
+			memcpy_fromio(dst + start,
+				hostdata->io + NCR53C400_host_buffer, 128);
+
 		start += 128;
 		blocks--;
 	}
@@ -412,7 +405,7 @@ static inline int generic_NCR5380_pread(struct Scsi_Host *instance,
 
 /**
  *	generic_NCR5380_pwrite - pseudo DMA write
- *	@instance: adapter to read from
+ *	@hostdata: scsi host private data
  *	@dst: buffer to read into
  *	@len: buffer length
  *
@@ -420,10 +413,9 @@ static inline int generic_NCR5380_pread(struct Scsi_Host *instance,
  *	controller
  */
 
-static inline int generic_NCR5380_pwrite(struct Scsi_Host *instance,
+static inline int generic_NCR5380_pwrite(struct NCR5380_hostdata *hostdata,
                                          unsigned char *src, int len)
 {
-	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	int blocks = len / 128;
 	int start = 0;
 
@@ -439,18 +431,17 @@ static inline int generic_NCR5380_pwrite(struct Scsi_Host *instance,
 			break;
 		while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
 			; // FIXME - timeout
-#ifndef SCSI_G_NCR5380_MEM
-		if (hostdata->io_width == 2)
-			outsw(instance->io_port + hostdata->c400_host_buf,
+
+		if (hostdata->io_port && hostdata->io_width == 2)
+			outsw(hostdata->io_port + hostdata->c400_host_buf,
 							src + start, 64);
-		else
-			outsb(instance->io_port + hostdata->c400_host_buf,
+		else if (hostdata->io_port)
+			outsb(hostdata->io_port + hostdata->c400_host_buf,
 							src + start, 128);
-#else
-		/* implies SCSI_G_NCR5380_MEM */
-		memcpy_toio(hostdata->iomem + NCR53C400_host_buffer,
-		            src + start, 128);
-#endif
+		else
+			memcpy_toio(hostdata->io + NCR53C400_host_buffer,
+			            src + start, 128);
+
 		start += 128;
 		blocks--;
 	}
@@ -458,18 +449,16 @@ static inline int generic_NCR5380_pwrite(struct Scsi_Host *instance,
 		while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
 			; // FIXME - no timeout
 
-#ifndef SCSI_G_NCR5380_MEM
-		if (hostdata->io_width == 2)
-			outsw(instance->io_port + hostdata->c400_host_buf,
+		if (hostdata->io_port && hostdata->io_width == 2)
+			outsw(hostdata->io_port + hostdata->c400_host_buf,
 							src + start, 64);
-		else
-			outsb(instance->io_port + hostdata->c400_host_buf,
+		else if (hostdata->io_port)
+			outsb(hostdata->io_port + hostdata->c400_host_buf,
 							src + start, 128);
-#else
-		/* implies SCSI_G_NCR5380_MEM */
-		memcpy_toio(hostdata->iomem + NCR53C400_host_buffer,
-		            src + start, 128);
-#endif
+		else
+			memcpy_toio(hostdata->io + NCR53C400_host_buffer,
+			            src + start, 128);
+
 		start += 128;
 		blocks--;
 	}
@@ -489,10 +478,9 @@ static inline int generic_NCR5380_pwrite(struct Scsi_Host *instance,
 	return 0;
 }
 
-static int generic_NCR5380_dma_xfer_len(struct Scsi_Host *instance,
+static int generic_NCR5380_dma_xfer_len(struct NCR5380_hostdata *hostdata,
                                         struct scsi_cmnd *cmd)
 {
-	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	int transfersize = cmd->transfersize;
 
 	if (hostdata->flags & FLAG_NO_PSEUDO_DMA)
@@ -566,7 +554,7 @@ static struct isa_driver generic_NCR5380_isa_driver = {
 	},
 };
 
-#if !defined(SCSI_G_NCR5380_MEM) && defined(CONFIG_PNP)
+#ifdef CONFIG_PNP
 static struct pnp_device_id generic_NCR5380_pnp_ids[] = {
 	{ .id = "DTC436e", .driver_data = BOARD_DTC3181E },
 	{ .id = "" }
@@ -600,7 +588,7 @@ static struct pnp_driver generic_NCR5380_pnp_driver = {
 	.probe		= generic_NCR5380_pnp_probe,
 	.remove		= generic_NCR5380_pnp_remove,
 };
-#endif /* !defined(SCSI_G_NCR5380_MEM) && defined(CONFIG_PNP) */
+#endif /* defined(CONFIG_PNP) */
 
 static int pnp_registered, isa_registered;
 
@@ -624,7 +612,7 @@ static int __init generic_NCR5380_init(void)
 			card[0] = BOARD_HP_C2502;
 	}
 
-#if !defined(SCSI_G_NCR5380_MEM) && defined(CONFIG_PNP)
+#ifdef CONFIG_PNP
 	if (!pnp_register_driver(&generic_NCR5380_pnp_driver))
 		pnp_registered = 1;
 #endif
@@ -637,7 +625,7 @@ static int __init generic_NCR5380_init(void)
 
 static void __exit generic_NCR5380_exit(void)
 {
-#if !defined(SCSI_G_NCR5380_MEM) && defined(CONFIG_PNP)
+#ifdef CONFIG_PNP
 	if (pnp_registered)
 		pnp_unregister_driver(&generic_NCR5380_pnp_driver);
 #endif
diff --git a/drivers/scsi/g_NCR5380.h b/drivers/scsi/g_NCR5380.h
index b175b9234458..3ce5b65ccb00 100644
--- a/drivers/scsi/g_NCR5380.h
+++ b/drivers/scsi/g_NCR5380.h
@@ -14,49 +14,28 @@
 #ifndef GENERIC_NCR5380_H
 #define GENERIC_NCR5380_H
 
-#ifndef SCSI_G_NCR5380_MEM
 #define DRV_MODULE_NAME "g_NCR5380"
 
 #define NCR5380_read(reg) \
-	inb(instance->io_port + (reg))
+	ioread8(hostdata->io + hostdata->offset + (reg))
 #define NCR5380_write(reg, value) \
-	outb(value, instance->io_port + (reg))
+	iowrite8(value, hostdata->io + hostdata->offset + (reg))
 
 #define NCR5380_implementation_fields \
+	int offset; \
 	int c400_ctl_status; \
 	int c400_blk_cnt; \
 	int c400_host_buf; \
 	int io_width;
 
-#else 
-/* therefore SCSI_G_NCR5380_MEM */
-#define DRV_MODULE_NAME "g_NCR5380_mmio"
-
 #define NCR53C400_mem_base 0x3880
 #define NCR53C400_host_buffer 0x3900
 #define NCR53C400_region_size 0x3a00
 
-#define NCR5380_read(reg) \
-	readb(((struct NCR5380_hostdata *)shost_priv(instance))->iomem + \
-	      NCR53C400_mem_base + (reg))
-#define NCR5380_write(reg, value) \
-	writeb(value, ((struct NCR5380_hostdata *)shost_priv(instance))->iomem + \
-	       NCR53C400_mem_base + (reg))
-
-#define NCR5380_implementation_fields \
-	void __iomem *iomem; \
-	resource_size_t iomem_size; \
-	int c400_ctl_status; \
-	int c400_blk_cnt; \
-	int c400_host_buf;
-
-#endif
-
-#define NCR5380_dma_xfer_len(instance, cmd, phase) \
-        generic_NCR5380_dma_xfer_len(instance, cmd)
+#define NCR5380_dma_xfer_len		generic_NCR5380_dma_xfer_len
 #define NCR5380_dma_recv_setup		generic_NCR5380_pread
 #define NCR5380_dma_send_setup		generic_NCR5380_pwrite
-#define NCR5380_dma_residual(instance)	(0)
+#define NCR5380_dma_residual		NCR5380_dma_residual_none
 
 #define NCR5380_intr generic_NCR5380_intr
 #define NCR5380_queue_command generic_NCR5380_queue_command
@@ -73,4 +52,3 @@
 #define BOARD_HP_C2502	4
 
 #endif /* GENERIC_NCR5380_H */
-
diff --git a/drivers/scsi/g_NCR5380_mmio.c b/drivers/scsi/g_NCR5380_mmio.c
deleted file mode 100644
index 8cdde71ba0c8..000000000000
--- a/drivers/scsi/g_NCR5380_mmio.c
+++ /dev/null
@@ -1,10 +0,0 @@
-/*
- *	There is probably a nicer way to do this but this one makes
- *	pretty obvious what is happening. We rebuild the same file with
- *	different options for mmio versus pio.
- */
-
-#define SCSI_G_NCR5380_MEM
-
-#include "g_NCR5380.c"
-
diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h
index 72c98522bd26..c0cd505a9ef7 100644
--- a/drivers/scsi/hisi_sas/hisi_sas.h
+++ b/drivers/scsi/hisi_sas/hisi_sas.h
@@ -13,6 +13,7 @@
 #define _HISI_SAS_H_
 
 #include <linux/acpi.h>
+#include <linux/clk.h>
 #include <linux/dmapool.h>
 #include <linux/mfd/syscon.h>
 #include <linux/module.h>
@@ -110,7 +111,7 @@ struct hisi_sas_device {
 	struct domain_device	*sas_device;
 	u64 attached_phy;
 	u64 device_id;
-	u64 running_req;
+	atomic64_t running_req;
 	u8 dev_status;
 };
 
@@ -149,7 +150,8 @@ struct hisi_sas_hw {
 				struct domain_device *device);
 	struct hisi_sas_device *(*alloc_dev)(struct domain_device *device);
 	void (*sl_notify)(struct hisi_hba *hisi_hba, int phy_no);
-	int (*get_free_slot)(struct hisi_hba *hisi_hba, int *q, int *s);
+	int (*get_free_slot)(struct hisi_hba *hisi_hba, u32 dev_id,
+			int *q, int *s);
 	void (*start_delivery)(struct hisi_hba *hisi_hba);
 	int (*prep_ssp)(struct hisi_hba *hisi_hba,
 			struct hisi_sas_slot *slot, int is_tmf,
@@ -166,6 +168,9 @@ struct hisi_sas_hw {
 	void (*phy_enable)(struct hisi_hba *hisi_hba, int phy_no);
 	void (*phy_disable)(struct hisi_hba *hisi_hba, int phy_no);
 	void (*phy_hard_reset)(struct hisi_hba *hisi_hba, int phy_no);
+	void (*phy_set_linkrate)(struct hisi_hba *hisi_hba, int phy_no,
+			struct sas_phy_linkrates *linkrates);
+	enum sas_linkrate (*phy_get_max_linkrate)(void);
 	void (*free_device)(struct hisi_hba *hisi_hba,
 			    struct hisi_sas_device *dev);
 	int (*get_wideport_bitmap)(struct hisi_hba *hisi_hba, int port_id);
@@ -183,6 +188,7 @@ struct hisi_hba {
 	u32 ctrl_reset_reg;
 	u32 ctrl_reset_sts_reg;
 	u32 ctrl_clock_ena_reg;
+	u32 refclk_frequency_mhz;
 	u8 sas_addr[SAS_ADDR_SIZE];
 
 	int n_phy;
@@ -205,7 +211,6 @@ struct hisi_hba {
 	struct hisi_sas_port port[HISI_SAS_MAX_PHYS];
 
 	int	queue_count;
-	int	queue;
 	struct hisi_sas_slot	*slot_prep;
 
 	struct dma_pool *sge_page_pool;
diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c
index 2f872f784e10..d50e9cfefd24 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_main.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_main.c
@@ -162,8 +162,8 @@ out:
 	hisi_sas_slot_task_free(hisi_hba, task, abort_slot);
 	if (task->task_done)
 		task->task_done(task);
-	if (sas_dev && sas_dev->running_req)
-		sas_dev->running_req--;
+	if (sas_dev)
+		atomic64_dec(&sas_dev->running_req);
 }
 
 static int hisi_sas_task_prep(struct sas_task *task, struct hisi_hba *hisi_hba,
@@ -232,8 +232,8 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_hba *hisi_hba,
 		rc = hisi_sas_slot_index_alloc(hisi_hba, &slot_idx);
 	if (rc)
 		goto err_out;
-	rc = hisi_hba->hw->get_free_slot(hisi_hba, &dlvry_queue,
-					 &dlvry_queue_slot);
+	rc = hisi_hba->hw->get_free_slot(hisi_hba, sas_dev->device_id,
+					&dlvry_queue, &dlvry_queue_slot);
 	if (rc)
 		goto err_out_tag;
 
@@ -303,7 +303,7 @@ static int hisi_sas_task_prep(struct sas_task *task, struct hisi_hba *hisi_hba,
 
 	hisi_hba->slot_prep = slot;
 
-	sas_dev->running_req++;
+	atomic64_inc(&sas_dev->running_req);
 	++(*pass);
 
 	return 0;
@@ -369,9 +369,14 @@ static void hisi_sas_bytes_dmaed(struct hisi_hba *hisi_hba, int phy_no)
 		struct sas_phy *sphy = sas_phy->phy;
 
 		sphy->negotiated_linkrate = sas_phy->linkrate;
-		sphy->minimum_linkrate = phy->minimum_linkrate;
 		sphy->minimum_linkrate_hw = SAS_LINK_RATE_1_5_GBPS;
-		sphy->maximum_linkrate = phy->maximum_linkrate;
+		sphy->maximum_linkrate_hw =
+			hisi_hba->hw->phy_get_max_linkrate();
+		if (sphy->minimum_linkrate == SAS_LINK_RATE_UNKNOWN)
+			sphy->minimum_linkrate = phy->minimum_linkrate;
+
+		if (sphy->maximum_linkrate == SAS_LINK_RATE_UNKNOWN)
+			sphy->maximum_linkrate = phy->maximum_linkrate;
 	}
 
 	if (phy->phy_type & PORT_TYPE_SAS) {
@@ -537,7 +542,7 @@ static void hisi_sas_port_notify_formed(struct asd_sas_phy *sas_phy)
 	struct hisi_hba *hisi_hba = sas_ha->lldd_ha;
 	struct hisi_sas_phy *phy = sas_phy->lldd_phy;
 	struct asd_sas_port *sas_port = sas_phy->port;
-	struct hisi_sas_port *port = &hisi_hba->port[sas_phy->id];
+	struct hisi_sas_port *port = &hisi_hba->port[phy->port_id];
 	unsigned long flags;
 
 	if (!sas_port)
@@ -645,6 +650,9 @@ static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func,
 		break;
 
 	case PHY_FUNC_SET_LINK_RATE:
+		hisi_hba->hw->phy_set_linkrate(hisi_hba, phy_no, funcdata);
+		break;
+
 	case PHY_FUNC_RELEASE_SPINUP_HOLD:
 	default:
 		return -EOPNOTSUPP;
@@ -764,7 +772,8 @@ static int hisi_sas_exec_internal_tmf_task(struct domain_device *device,
 		task = NULL;
 	}
 ex_err:
-	WARN_ON(retry == TASK_RETRY);
+	if (retry == TASK_RETRY)
+		dev_warn(dev, "abort tmf: executing internal task failed!\n");
 	sas_free_task(task);
 	return res;
 }
@@ -960,6 +969,9 @@ static int hisi_sas_query_task(struct sas_task *task)
 		case TMF_RESP_FUNC_FAILED:
 		case TMF_RESP_FUNC_COMPLETE:
 			break;
+		default:
+			rc = TMF_RESP_FUNC_FAILED;
+			break;
 		}
 	}
 	return rc;
@@ -987,8 +999,8 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, u64 device_id,
 	rc = hisi_sas_slot_index_alloc(hisi_hba, &slot_idx);
 	if (rc)
 		goto err_out;
-	rc = hisi_hba->hw->get_free_slot(hisi_hba, &dlvry_queue,
-					 &dlvry_queue_slot);
+	rc = hisi_hba->hw->get_free_slot(hisi_hba, sas_dev->device_id,
+					&dlvry_queue, &dlvry_queue_slot);
 	if (rc)
 		goto err_out_tag;
 
@@ -1023,7 +1035,8 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, u64 device_id,
 
 	hisi_hba->slot_prep = slot;
 
-	sas_dev->running_req++;
+	atomic64_inc(&sas_dev->running_req);
+
 	/* send abort command to our chip */
 	hisi_hba->hw->start_delivery(hisi_hba);
 
@@ -1396,10 +1409,13 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev,
 	struct hisi_hba *hisi_hba;
 	struct device *dev = &pdev->dev;
 	struct device_node *np = pdev->dev.of_node;
+	struct clk *refclk;
 
 	shost = scsi_host_alloc(&hisi_sas_sht, sizeof(*hisi_hba));
-	if (!shost)
-		goto err_out;
+	if (!shost) {
+		dev_err(dev, "scsi host alloc failed\n");
+		return NULL;
+	}
 	hisi_hba = shost_priv(shost);
 
 	hisi_hba->hw = hw;
@@ -1432,6 +1448,12 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev,
 			goto err_out;
 	}
 
+	refclk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(refclk))
+		dev_info(dev, "no ref clk property\n");
+	else
+		hisi_hba->refclk_frequency_mhz = clk_get_rate(refclk) / 1000000;
+
 	if (device_property_read_u32(dev, "phy-count", &hisi_hba->n_phy))
 		goto err_out;
 
@@ -1457,6 +1479,7 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev,
 
 	return shost;
 err_out:
+	kfree(shost);
 	dev_err(dev, "shost alloc failed\n");
 	return NULL;
 }
@@ -1483,10 +1506,8 @@ int hisi_sas_probe(struct platform_device *pdev,
 	int rc, phy_nr, port_nr, i;
 
 	shost = hisi_sas_shost_alloc(pdev, hw);
-	if (!shost) {
-		rc = -ENOMEM;
-		goto err_out_ha;
-	}
+	if (!shost)
+		return -ENOMEM;
 
 	sha = SHOST_TO_SAS_HA(shost);
 	hisi_hba = shost_priv(shost);
@@ -1496,12 +1517,13 @@ int hisi_sas_probe(struct platform_device *pdev,
 
 	arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL);
 	arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL);
-	if (!arr_phy || !arr_port)
-		return -ENOMEM;
+	if (!arr_phy || !arr_port) {
+		rc = -ENOMEM;
+		goto err_out_ha;
+	}
 
 	sha->sas_phy = arr_phy;
 	sha->sas_port = arr_port;
-	sha->core.shost = shost;
 	sha->lldd_ha = hisi_hba;
 
 	shost->transportt = hisi_sas_stt;
@@ -1546,6 +1568,7 @@ int hisi_sas_probe(struct platform_device *pdev,
 err_out_register_ha:
 	scsi_remove_host(shost);
 err_out_ha:
+	hisi_sas_free(hisi_hba);
 	kfree(shost);
 	return rc;
 }
@@ -1555,12 +1578,14 @@ int hisi_sas_remove(struct platform_device *pdev)
 {
 	struct sas_ha_struct *sha = platform_get_drvdata(pdev);
 	struct hisi_hba *hisi_hba = sha->lldd_ha;
+	struct Scsi_Host *shost = sha->core.shost;
 
 	scsi_remove_host(sha->core.shost);
 	sas_unregister_ha(sha);
 	sas_remove_host(sha->core.shost);
 
 	hisi_sas_free(hisi_hba);
+	kfree(shost);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(hisi_sas_remove);
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
index c0ac49d8bc8d..8a1be0ba8a22 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
@@ -843,6 +843,49 @@ static void sl_notify_v1_hw(struct hisi_hba *hisi_hba, int phy_no)
 	hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
 }
 
+static enum sas_linkrate phy_get_max_linkrate_v1_hw(void)
+{
+	return SAS_LINK_RATE_6_0_GBPS;
+}
+
+static void phy_set_linkrate_v1_hw(struct hisi_hba *hisi_hba, int phy_no,
+		struct sas_phy_linkrates *r)
+{
+	u32 prog_phy_link_rate =
+		hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE);
+	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
+	struct asd_sas_phy *sas_phy = &phy->sas_phy;
+	int i;
+	enum sas_linkrate min, max;
+	u32 rate_mask = 0;
+
+	if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
+		max = sas_phy->phy->maximum_linkrate;
+		min = r->minimum_linkrate;
+	} else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
+		max = r->maximum_linkrate;
+		min = sas_phy->phy->minimum_linkrate;
+	} else
+		return;
+
+	sas_phy->phy->maximum_linkrate = max;
+	sas_phy->phy->minimum_linkrate = min;
+
+	min -= SAS_LINK_RATE_1_5_GBPS;
+	max -= SAS_LINK_RATE_1_5_GBPS;
+
+	for (i = 0; i <= max; i++)
+		rate_mask |= 1 << (i * 2);
+
+	prog_phy_link_rate &= ~0xff;
+	prog_phy_link_rate |= rate_mask;
+
+	hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
+			prog_phy_link_rate);
+
+	phy_hard_reset_v1_hw(hisi_hba, phy_no);
+}
+
 static int get_wideport_bitmap_v1_hw(struct hisi_hba *hisi_hba, int port_id)
 {
 	int i, bitmap = 0;
@@ -862,29 +905,23 @@ static int get_wideport_bitmap_v1_hw(struct hisi_hba *hisi_hba, int port_id)
  * The callpath to this function and upto writing the write
  * queue pointer should be safe from interruption.
  */
-static int get_free_slot_v1_hw(struct hisi_hba *hisi_hba, int *q, int *s)
+static int get_free_slot_v1_hw(struct hisi_hba *hisi_hba, u32 dev_id,
+				int *q, int *s)
 {
 	struct device *dev = &hisi_hba->pdev->dev;
 	struct hisi_sas_dq *dq;
 	u32 r, w;
-	int queue = hisi_hba->queue;
-
-	while (1) {
-		dq = &hisi_hba->dq[queue];
-		w = dq->wr_point;
-		r = hisi_sas_read32_relaxed(hisi_hba,
-				    DLVRY_Q_0_RD_PTR + (queue * 0x14));
-		if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
-			queue = (queue + 1) % hisi_hba->queue_count;
-			if (queue == hisi_hba->queue) {
-				dev_warn(dev, "could not find free slot\n");
-				return -EAGAIN;
-			}
-			continue;
-		}
-		break;
+	int queue = dev_id % hisi_hba->queue_count;
+
+	dq = &hisi_hba->dq[queue];
+	w = dq->wr_point;
+	r = hisi_sas_read32_relaxed(hisi_hba,
+				DLVRY_Q_0_RD_PTR + (queue * 0x14));
+	if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
+		dev_warn(dev, "could not find free slot\n");
+		return -EAGAIN;
 	}
-	hisi_hba->queue = (queue + 1) % hisi_hba->queue_count;
+
 	*q = queue;
 	*s = w;
 	return 0;
@@ -1372,8 +1409,8 @@ static int slot_complete_v1_hw(struct hisi_hba *hisi_hba,
 	}
 
 out:
-	if (sas_dev && sas_dev->running_req)
-		sas_dev->running_req--;
+	if (sas_dev)
+		atomic64_dec(&sas_dev->running_req);
 
 	hisi_sas_slot_task_free(hisi_hba, task, slot);
 	sts = ts->stat;
@@ -1824,6 +1861,8 @@ static const struct hisi_sas_hw hisi_sas_v1_hw = {
 	.phy_enable = enable_phy_v1_hw,
 	.phy_disable = disable_phy_v1_hw,
 	.phy_hard_reset = phy_hard_reset_v1_hw,
+	.phy_set_linkrate = phy_set_linkrate_v1_hw,
+	.phy_get_max_linkrate = phy_get_max_linkrate_v1_hw,
 	.get_wideport_bitmap = get_wideport_bitmap_v1_hw,
 	.max_command_entries = HISI_SAS_COMMAND_ENTRIES_V1_HW,
 	.complete_hdr_size = sizeof(struct hisi_sas_complete_v1_hdr),
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
index 9825a3f49f53..b934aec1eebb 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
@@ -55,10 +55,44 @@
 #define HGC_DFX_CFG2			0xc0
 #define HGC_IOMB_PROC1_STATUS	0x104
 #define CFG_1US_TIMER_TRSH		0xcc
+#define HGC_LM_DFX_STATUS2		0x128
+#define HGC_LM_DFX_STATUS2_IOSTLIST_OFF		0
+#define HGC_LM_DFX_STATUS2_IOSTLIST_MSK	(0xfff << \
+					 HGC_LM_DFX_STATUS2_IOSTLIST_OFF)
+#define HGC_LM_DFX_STATUS2_ITCTLIST_OFF		12
+#define HGC_LM_DFX_STATUS2_ITCTLIST_MSK	(0x7ff << \
+					 HGC_LM_DFX_STATUS2_ITCTLIST_OFF)
+#define HGC_CQE_ECC_ADDR		0x13c
+#define HGC_CQE_ECC_1B_ADDR_OFF	0
+#define HGC_CQE_ECC_1B_ADDR_MSK	(0x3f << HGC_CQE_ECC_1B_ADDR_OFF)
+#define HGC_CQE_ECC_MB_ADDR_OFF	8
+#define HGC_CQE_ECC_MB_ADDR_MSK (0x3f << HGC_CQE_ECC_MB_ADDR_OFF)
+#define HGC_IOST_ECC_ADDR		0x140
+#define HGC_IOST_ECC_1B_ADDR_OFF	0
+#define HGC_IOST_ECC_1B_ADDR_MSK	(0x3ff << HGC_IOST_ECC_1B_ADDR_OFF)
+#define HGC_IOST_ECC_MB_ADDR_OFF	16
+#define HGC_IOST_ECC_MB_ADDR_MSK	(0x3ff << HGC_IOST_ECC_MB_ADDR_OFF)
+#define HGC_DQE_ECC_ADDR		0x144
+#define HGC_DQE_ECC_1B_ADDR_OFF	0
+#define HGC_DQE_ECC_1B_ADDR_MSK	(0xfff << HGC_DQE_ECC_1B_ADDR_OFF)
+#define HGC_DQE_ECC_MB_ADDR_OFF	16
+#define HGC_DQE_ECC_MB_ADDR_MSK (0xfff << HGC_DQE_ECC_MB_ADDR_OFF)
 #define HGC_INVLD_DQE_INFO		0x148
 #define HGC_INVLD_DQE_INFO_FB_CH0_OFF	9
 #define HGC_INVLD_DQE_INFO_FB_CH0_MSK	(0x1 << HGC_INVLD_DQE_INFO_FB_CH0_OFF)
 #define HGC_INVLD_DQE_INFO_FB_CH3_OFF	18
+#define HGC_ITCT_ECC_ADDR		0x150
+#define HGC_ITCT_ECC_1B_ADDR_OFF		0
+#define HGC_ITCT_ECC_1B_ADDR_MSK		(0x3ff << \
+						 HGC_ITCT_ECC_1B_ADDR_OFF)
+#define HGC_ITCT_ECC_MB_ADDR_OFF		16
+#define HGC_ITCT_ECC_MB_ADDR_MSK		(0x3ff << \
+						 HGC_ITCT_ECC_MB_ADDR_OFF)
+#define HGC_AXI_FIFO_ERR_INFO	0x154
+#define AXI_ERR_INFO_OFF		0
+#define AXI_ERR_INFO_MSK		(0xff << AXI_ERR_INFO_OFF)
+#define FIFO_ERR_INFO_OFF		8
+#define FIFO_ERR_INFO_MSK		(0xff << FIFO_ERR_INFO_OFF)
 #define INT_COAL_EN			0x19c
 #define OQ_INT_COAL_TIME		0x1a0
 #define OQ_INT_COAL_CNT			0x1a4
@@ -73,13 +107,41 @@
 #define ENT_INT_SRC1_D2H_FIS_CH1_MSK	(0x1 << ENT_INT_SRC1_D2H_FIS_CH1_OFF)
 #define ENT_INT_SRC2			0x1bc
 #define ENT_INT_SRC3			0x1c0
+#define ENT_INT_SRC3_WP_DEPTH_OFF		8
+#define ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF	9
+#define ENT_INT_SRC3_RP_DEPTH_OFF		10
+#define ENT_INT_SRC3_AXI_OFF			11
+#define ENT_INT_SRC3_FIFO_OFF			12
+#define ENT_INT_SRC3_LM_OFF				14
 #define ENT_INT_SRC3_ITC_INT_OFF	15
 #define ENT_INT_SRC3_ITC_INT_MSK	(0x1 << ENT_INT_SRC3_ITC_INT_OFF)
+#define ENT_INT_SRC3_ABT_OFF		16
 #define ENT_INT_SRC_MSK1		0x1c4
 #define ENT_INT_SRC_MSK2		0x1c8
 #define ENT_INT_SRC_MSK3		0x1cc
 #define ENT_INT_SRC_MSK3_ENT95_MSK_OFF	31
 #define ENT_INT_SRC_MSK3_ENT95_MSK_MSK	(0x1 << ENT_INT_SRC_MSK3_ENT95_MSK_OFF)
+#define SAS_ECC_INTR			0x1e8
+#define SAS_ECC_INTR_DQE_ECC_1B_OFF		0
+#define SAS_ECC_INTR_DQE_ECC_MB_OFF		1
+#define SAS_ECC_INTR_IOST_ECC_1B_OFF	2
+#define SAS_ECC_INTR_IOST_ECC_MB_OFF	3
+#define SAS_ECC_INTR_ITCT_ECC_MB_OFF	4
+#define SAS_ECC_INTR_ITCT_ECC_1B_OFF	5
+#define SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF	6
+#define SAS_ECC_INTR_IOSTLIST_ECC_1B_OFF	7
+#define SAS_ECC_INTR_ITCTLIST_ECC_1B_OFF	8
+#define SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF	9
+#define SAS_ECC_INTR_CQE_ECC_1B_OFF		10
+#define SAS_ECC_INTR_CQE_ECC_MB_OFF		11
+#define SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF	12
+#define SAS_ECC_INTR_NCQ_MEM0_ECC_1B_OFF	13
+#define SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF	14
+#define SAS_ECC_INTR_NCQ_MEM1_ECC_1B_OFF	15
+#define SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF	16
+#define SAS_ECC_INTR_NCQ_MEM2_ECC_1B_OFF	17
+#define SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF	18
+#define SAS_ECC_INTR_NCQ_MEM3_ECC_1B_OFF	19
 #define SAS_ECC_INTR_MSK		0x1ec
 #define HGC_ERR_STAT_EN			0x238
 #define DLVRY_Q_0_BASE_ADDR_LO		0x260
@@ -94,7 +156,20 @@
 #define COMPL_Q_0_DEPTH			0x4e8
 #define COMPL_Q_0_WR_PTR		0x4ec
 #define COMPL_Q_0_RD_PTR		0x4f0
-
+#define HGC_RXM_DFX_STATUS14	0xae8
+#define HGC_RXM_DFX_STATUS14_MEM0_OFF		0
+#define HGC_RXM_DFX_STATUS14_MEM0_MSK		(0x1ff << \
+						 HGC_RXM_DFX_STATUS14_MEM0_OFF)
+#define HGC_RXM_DFX_STATUS14_MEM1_OFF		9
+#define HGC_RXM_DFX_STATUS14_MEM1_MSK		(0x1ff << \
+						 HGC_RXM_DFX_STATUS14_MEM1_OFF)
+#define HGC_RXM_DFX_STATUS14_MEM2_OFF		18
+#define HGC_RXM_DFX_STATUS14_MEM2_MSK		(0x1ff << \
+						 HGC_RXM_DFX_STATUS14_MEM2_OFF)
+#define HGC_RXM_DFX_STATUS15	0xaec
+#define HGC_RXM_DFX_STATUS15_MEM3_OFF		0
+#define HGC_RXM_DFX_STATUS15_MEM3_MSK		(0x1ff << \
+						 HGC_RXM_DFX_STATUS15_MEM3_OFF)
 /* phy registers need init */
 #define PORT_BASE			(0x2000)
 
@@ -119,6 +194,9 @@
 #define SL_CONTROL_NOTIFY_EN_MSK	(0x1 << SL_CONTROL_NOTIFY_EN_OFF)
 #define SL_CONTROL_CTA_OFF		17
 #define SL_CONTROL_CTA_MSK		(0x1 << SL_CONTROL_CTA_OFF)
+#define RX_PRIMS_STATUS         (PORT_BASE + 0x98)
+#define RX_BCAST_CHG_OFF        1
+#define RX_BCAST_CHG_MSK        (0x1 << RX_BCAST_CHG_OFF)
 #define TX_ID_DWORD0			(PORT_BASE + 0x9c)
 #define TX_ID_DWORD1			(PORT_BASE + 0xa0)
 #define TX_ID_DWORD2			(PORT_BASE + 0xa4)
@@ -267,6 +345,8 @@
 #define ITCT_HDR_RTOLT_OFF		48
 #define ITCT_HDR_RTOLT_MSK		(0xffffULL << ITCT_HDR_RTOLT_OFF)
 
+#define HISI_SAS_FATAL_INT_NR	2
+
 struct hisi_sas_complete_v2_hdr {
 	__le32 dw0;
 	__le32 dw1;
@@ -659,8 +739,6 @@ static void free_device_v2_hw(struct hisi_hba *hisi_hba,
 			qw0 &= ~(1 << ITCT_HDR_VALID_OFF);
 			hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
 					 ENT_INT_SRC3_ITC_INT_MSK);
-			hisi_hba->devices[dev_id].dev_type = SAS_PHY_UNUSED;
-			hisi_hba->devices[dev_id].dev_status = HISI_SAS_DEV_NORMAL;
 
 			/* clear the itct */
 			hisi_sas_write32(hisi_hba, ITCT_CLR, 0);
@@ -808,7 +886,7 @@ static void init_reg_v2_hw(struct hisi_hba *hisi_hba)
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0x7efefefe);
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0x7efefefe);
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0x7ffffffe);
-	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xfffff3c0);
+	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xfff00c30);
 	for (i = 0; i < hisi_hba->queue_count; i++)
 		hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0);
 
@@ -824,7 +902,7 @@ static void init_reg_v2_hw(struct hisi_hba *hisi_hba)
 		hisi_sas_phy_write32(hisi_hba, i, DONE_RECEIVED_TIME, 0x10);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
-		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff);
+		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xfff87fff);
 		hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xffffffff);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0x8ffffbff);
@@ -836,7 +914,9 @@ static void init_reg_v2_hw(struct hisi_hba *hisi_hba)
 		hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT_COAL_EN, 0x0);
 		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x0);
-		hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL, 0x199B694);
+		if (hisi_hba->refclk_frequency_mhz == 66)
+			hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL, 0x199B694);
+		/* else, do nothing -> leave it how you found it */
 	}
 
 	for (i = 0; i < hisi_hba->queue_count; i++) {
@@ -980,6 +1060,49 @@ static void sl_notify_v2_hw(struct hisi_hba *hisi_hba, int phy_no)
 	hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
 }
 
+static enum sas_linkrate phy_get_max_linkrate_v2_hw(void)
+{
+	return SAS_LINK_RATE_12_0_GBPS;
+}
+
+static void phy_set_linkrate_v2_hw(struct hisi_hba *hisi_hba, int phy_no,
+		struct sas_phy_linkrates *r)
+{
+	u32 prog_phy_link_rate =
+		hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE);
+	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
+	struct asd_sas_phy *sas_phy = &phy->sas_phy;
+	int i;
+	enum sas_linkrate min, max;
+	u32 rate_mask = 0;
+
+	if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
+		max = sas_phy->phy->maximum_linkrate;
+		min = r->minimum_linkrate;
+	} else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
+		max = r->maximum_linkrate;
+		min = sas_phy->phy->minimum_linkrate;
+	} else
+		return;
+
+	sas_phy->phy->maximum_linkrate = max;
+	sas_phy->phy->minimum_linkrate = min;
+
+	min -= SAS_LINK_RATE_1_5_GBPS;
+	max -= SAS_LINK_RATE_1_5_GBPS;
+
+	for (i = 0; i <= max; i++)
+		rate_mask |= 1 << (i * 2);
+
+	prog_phy_link_rate &= ~0xff;
+	prog_phy_link_rate |= rate_mask;
+
+	hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
+			prog_phy_link_rate);
+
+	phy_hard_reset_v2_hw(hisi_hba, phy_no);
+}
+
 static int get_wideport_bitmap_v2_hw(struct hisi_hba *hisi_hba, int port_id)
 {
 	int i, bitmap = 0;
@@ -1010,29 +1133,24 @@ static int get_wideport_bitmap_v2_hw(struct hisi_hba *hisi_hba, int port_id)
  * The callpath to this function and upto writing the write
  * queue pointer should be safe from interruption.
  */
-static int get_free_slot_v2_hw(struct hisi_hba *hisi_hba, int *q, int *s)
+static int get_free_slot_v2_hw(struct hisi_hba *hisi_hba, u32 dev_id,
+				int *q, int *s)
 {
 	struct device *dev = &hisi_hba->pdev->dev;
 	struct hisi_sas_dq *dq;
 	u32 r, w;
-	int queue = hisi_hba->queue;
-
-	while (1) {
-		dq = &hisi_hba->dq[queue];
-		w = dq->wr_point;
-		r = hisi_sas_read32_relaxed(hisi_hba,
-					    DLVRY_Q_0_RD_PTR + (queue * 0x14));
-		if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
-			queue = (queue + 1) % hisi_hba->queue_count;
-			if (queue == hisi_hba->queue) {
-				dev_warn(dev, "could not find free slot\n");
-				return -EAGAIN;
-			}
-			continue;
-		}
-		break;
+	int queue = dev_id % hisi_hba->queue_count;
+
+	dq = &hisi_hba->dq[queue];
+	w = dq->wr_point;
+	r = hisi_sas_read32_relaxed(hisi_hba,
+				DLVRY_Q_0_RD_PTR + (queue * 0x14));
+	if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
+		dev_warn(dev, "full queue=%d r=%d w=%d\n\n",
+				queue, r, w);
+		return -EAGAIN;
 	}
-	hisi_hba->queue = (queue + 1) % hisi_hba->queue_count;
+
 	*q = queue;
 	*s = w;
 	return 0;
@@ -1653,8 +1771,8 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot,
 	}
 
 out:
-	if (sas_dev && sas_dev->running_req)
-		sas_dev->running_req--;
+	if (sas_dev)
+		atomic64_dec(&sas_dev->running_req);
 
 	hisi_sas_slot_task_free(hisi_hba, task, slot);
 	sts = ts->stat;
@@ -1675,6 +1793,7 @@ static u8 get_ata_protocol(u8 cmd, int direction)
 	case ATA_CMD_NCQ_NON_DATA:
 	return SATA_PROTOCOL_FPDMA;
 
+	case ATA_CMD_DOWNLOAD_MICRO:
 	case ATA_CMD_ID_ATA:
 	case ATA_CMD_PMP_READ:
 	case ATA_CMD_READ_LOG_EXT:
@@ -1686,18 +1805,27 @@ static u8 get_ata_protocol(u8 cmd, int direction)
 	case ATA_CMD_PIO_WRITE_EXT:
 	return SATA_PROTOCOL_PIO;
 
+	case ATA_CMD_DSM:
+	case ATA_CMD_DOWNLOAD_MICRO_DMA:
+	case ATA_CMD_PMP_READ_DMA:
+	case ATA_CMD_PMP_WRITE_DMA:
 	case ATA_CMD_READ:
 	case ATA_CMD_READ_EXT:
 	case ATA_CMD_READ_LOG_DMA_EXT:
+	case ATA_CMD_READ_STREAM_DMA_EXT:
+	case ATA_CMD_TRUSTED_RCV_DMA:
+	case ATA_CMD_TRUSTED_SND_DMA:
 	case ATA_CMD_WRITE:
 	case ATA_CMD_WRITE_EXT:
+	case ATA_CMD_WRITE_FUA_EXT:
 	case ATA_CMD_WRITE_QUEUED:
 	case ATA_CMD_WRITE_LOG_DMA_EXT:
+	case ATA_CMD_WRITE_STREAM_DMA_EXT:
 	return SATA_PROTOCOL_DMA;
 
-	case ATA_CMD_DOWNLOAD_MICRO:
-	case ATA_CMD_DEV_RESET:
 	case ATA_CMD_CHK_POWER:
+	case ATA_CMD_DEV_RESET:
+	case ATA_CMD_EDD:
 	case ATA_CMD_FLUSH:
 	case ATA_CMD_FLUSH_EXT:
 	case ATA_CMD_VERIFY:
@@ -1970,9 +2098,12 @@ static void phy_bcast_v2_hw(int phy_no, struct hisi_hba *hisi_hba)
 	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 	struct asd_sas_phy *sas_phy = &phy->sas_phy;
 	struct sas_ha_struct *sas_ha = &hisi_hba->sha;
+	u32 bcast_status;
 
 	hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1);
-	sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD);
+	bcast_status = hisi_sas_phy_read32(hisi_hba, phy_no, RX_PRIMS_STATUS);
+	if (bcast_status & RX_BCAST_CHG_MSK)
+		sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD);
 	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
 			     CHL_INT0_SL_RX_BCST_ACK_MSK);
 	hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0);
@@ -2005,8 +2136,9 @@ static irqreturn_t int_chnl_int_v2_hw(int irq_no, void *p)
 			if (irq_value1) {
 				if (irq_value1 & (CHL_INT1_DMAC_RX_ECC_ERR_MSK |
 						  CHL_INT1_DMAC_TX_ECC_ERR_MSK))
-					panic("%s: DMAC RX/TX ecc bad error! (0x%x)",
-						dev_name(dev), irq_value1);
+					panic("%s: DMAC RX/TX ecc bad error!\
+					       (0x%x)",
+					      dev_name(dev), irq_value1);
 
 				hisi_sas_phy_write32(hisi_hba, phy_no,
 						     CHL_INT1, irq_value1);
@@ -2037,6 +2169,318 @@ static irqreturn_t int_chnl_int_v2_hw(int irq_no, void *p)
 	return IRQ_HANDLED;
 }
 
+static void
+one_bit_ecc_error_process_v2_hw(struct hisi_hba *hisi_hba, u32 irq_value)
+{
+	struct device *dev = &hisi_hba->pdev->dev;
+	u32 reg_val;
+
+	if (irq_value & BIT(SAS_ECC_INTR_DQE_ECC_1B_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_DQE_ECC_ADDR);
+		dev_warn(dev, "hgc_dqe_acc1b_intr found: \
+			       Ram address is 0x%08X\n",
+			 (reg_val & HGC_DQE_ECC_1B_ADDR_MSK) >>
+			 HGC_DQE_ECC_1B_ADDR_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_IOST_ECC_1B_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_IOST_ECC_ADDR);
+		dev_warn(dev, "hgc_iost_acc1b_intr found: \
+			       Ram address is 0x%08X\n",
+			 (reg_val & HGC_IOST_ECC_1B_ADDR_MSK) >>
+			 HGC_IOST_ECC_1B_ADDR_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_ITCT_ECC_1B_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_ITCT_ECC_ADDR);
+		dev_warn(dev, "hgc_itct_acc1b_intr found: \
+			       Ram address is 0x%08X\n",
+			 (reg_val & HGC_ITCT_ECC_1B_ADDR_MSK) >>
+			 HGC_ITCT_ECC_1B_ADDR_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_IOSTLIST_ECC_1B_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_LM_DFX_STATUS2);
+		dev_warn(dev, "hgc_iostl_acc1b_intr found: \
+			       memory address is 0x%08X\n",
+			 (reg_val & HGC_LM_DFX_STATUS2_IOSTLIST_MSK) >>
+			 HGC_LM_DFX_STATUS2_IOSTLIST_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_ITCTLIST_ECC_1B_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_LM_DFX_STATUS2);
+		dev_warn(dev, "hgc_itctl_acc1b_intr found: \
+			       memory address is 0x%08X\n",
+			 (reg_val & HGC_LM_DFX_STATUS2_ITCTLIST_MSK) >>
+			 HGC_LM_DFX_STATUS2_ITCTLIST_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_CQE_ECC_1B_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_CQE_ECC_ADDR);
+		dev_warn(dev, "hgc_cqe_acc1b_intr found: \
+			       Ram address is 0x%08X\n",
+			 (reg_val & HGC_CQE_ECC_1B_ADDR_MSK) >>
+			 HGC_CQE_ECC_1B_ADDR_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM0_ECC_1B_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14);
+		dev_warn(dev, "rxm_mem0_acc1b_intr found: \
+			       memory address is 0x%08X\n",
+			 (reg_val & HGC_RXM_DFX_STATUS14_MEM0_MSK) >>
+			 HGC_RXM_DFX_STATUS14_MEM0_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM1_ECC_1B_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14);
+		dev_warn(dev, "rxm_mem1_acc1b_intr found: \
+			       memory address is 0x%08X\n",
+			 (reg_val & HGC_RXM_DFX_STATUS14_MEM1_MSK) >>
+			 HGC_RXM_DFX_STATUS14_MEM1_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM2_ECC_1B_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14);
+		dev_warn(dev, "rxm_mem2_acc1b_intr found: \
+			       memory address is 0x%08X\n",
+			 (reg_val & HGC_RXM_DFX_STATUS14_MEM2_MSK) >>
+			 HGC_RXM_DFX_STATUS14_MEM2_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM3_ECC_1B_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS15);
+		dev_warn(dev, "rxm_mem3_acc1b_intr found: \
+			       memory address is 0x%08X\n",
+			 (reg_val & HGC_RXM_DFX_STATUS15_MEM3_MSK) >>
+			 HGC_RXM_DFX_STATUS15_MEM3_OFF);
+	}
+
+}
+
+static void multi_bit_ecc_error_process_v2_hw(struct hisi_hba *hisi_hba,
+		u32 irq_value)
+{
+	u32 reg_val;
+	struct device *dev = &hisi_hba->pdev->dev;
+
+	if (irq_value & BIT(SAS_ECC_INTR_DQE_ECC_MB_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_DQE_ECC_ADDR);
+		panic("%s: hgc_dqe_accbad_intr (0x%x) found: \
+		       Ram address is 0x%08X\n",
+		      dev_name(dev), irq_value,
+		      (reg_val & HGC_DQE_ECC_MB_ADDR_MSK) >>
+		      HGC_DQE_ECC_MB_ADDR_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_IOST_ECC_MB_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_IOST_ECC_ADDR);
+		panic("%s: hgc_iost_accbad_intr (0x%x) found: \
+		       Ram address is 0x%08X\n",
+		      dev_name(dev), irq_value,
+		      (reg_val & HGC_IOST_ECC_MB_ADDR_MSK) >>
+		      HGC_IOST_ECC_MB_ADDR_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_ITCT_ECC_MB_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_ITCT_ECC_ADDR);
+		panic("%s: hgc_itct_accbad_intr (0x%x) found: \
+		       Ram address is 0x%08X\n",
+		      dev_name(dev), irq_value,
+		      (reg_val & HGC_ITCT_ECC_MB_ADDR_MSK) >>
+		      HGC_ITCT_ECC_MB_ADDR_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_LM_DFX_STATUS2);
+		panic("%s: hgc_iostl_accbad_intr (0x%x) found: \
+		       memory address is 0x%08X\n",
+		      dev_name(dev), irq_value,
+		      (reg_val & HGC_LM_DFX_STATUS2_IOSTLIST_MSK) >>
+		      HGC_LM_DFX_STATUS2_IOSTLIST_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_LM_DFX_STATUS2);
+		panic("%s: hgc_itctl_accbad_intr (0x%x) found: \
+		       memory address is 0x%08X\n",
+		      dev_name(dev), irq_value,
+		      (reg_val & HGC_LM_DFX_STATUS2_ITCTLIST_MSK) >>
+		      HGC_LM_DFX_STATUS2_ITCTLIST_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_CQE_ECC_MB_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_CQE_ECC_ADDR);
+		panic("%s: hgc_cqe_accbad_intr (0x%x) found: \
+		       Ram address is 0x%08X\n",
+		      dev_name(dev), irq_value,
+		      (reg_val & HGC_CQE_ECC_MB_ADDR_MSK) >>
+		      HGC_CQE_ECC_MB_ADDR_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14);
+		panic("%s: rxm_mem0_accbad_intr (0x%x) found: \
+		       memory address is 0x%08X\n",
+		      dev_name(dev), irq_value,
+		      (reg_val & HGC_RXM_DFX_STATUS14_MEM0_MSK) >>
+		      HGC_RXM_DFX_STATUS14_MEM0_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14);
+		panic("%s: rxm_mem1_accbad_intr (0x%x) found: \
+		       memory address is 0x%08X\n",
+		      dev_name(dev), irq_value,
+		      (reg_val & HGC_RXM_DFX_STATUS14_MEM1_MSK) >>
+		      HGC_RXM_DFX_STATUS14_MEM1_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14);
+		panic("%s: rxm_mem2_accbad_intr (0x%x) found: \
+		       memory address is 0x%08X\n",
+		      dev_name(dev), irq_value,
+		      (reg_val & HGC_RXM_DFX_STATUS14_MEM2_MSK) >>
+		      HGC_RXM_DFX_STATUS14_MEM2_OFF);
+	}
+
+	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF)) {
+		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS15);
+		panic("%s: rxm_mem3_accbad_intr (0x%x) found: \
+		       memory address is 0x%08X\n",
+		      dev_name(dev), irq_value,
+		      (reg_val & HGC_RXM_DFX_STATUS15_MEM3_MSK) >>
+		      HGC_RXM_DFX_STATUS15_MEM3_OFF);
+	}
+
+}
+
+static irqreturn_t fatal_ecc_int_v2_hw(int irq_no, void *p)
+{
+	struct hisi_hba *hisi_hba = p;
+	u32 irq_value, irq_msk;
+
+	irq_msk = hisi_sas_read32(hisi_hba, SAS_ECC_INTR_MSK);
+	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, irq_msk | 0xffffffff);
+
+	irq_value = hisi_sas_read32(hisi_hba, SAS_ECC_INTR);
+	if (irq_value) {
+		one_bit_ecc_error_process_v2_hw(hisi_hba, irq_value);
+		multi_bit_ecc_error_process_v2_hw(hisi_hba, irq_value);
+	}
+
+	hisi_sas_write32(hisi_hba, SAS_ECC_INTR, irq_value);
+	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, irq_msk);
+
+	return IRQ_HANDLED;
+}
+
+#define AXI_ERR_NR	8
+static const char axi_err_info[AXI_ERR_NR][32] = {
+	"IOST_AXI_W_ERR",
+	"IOST_AXI_R_ERR",
+	"ITCT_AXI_W_ERR",
+	"ITCT_AXI_R_ERR",
+	"SATA_AXI_W_ERR",
+	"SATA_AXI_R_ERR",
+	"DQE_AXI_R_ERR",
+	"CQE_AXI_W_ERR"
+};
+
+#define FIFO_ERR_NR	5
+static const char fifo_err_info[FIFO_ERR_NR][32] = {
+	"CQE_WINFO_FIFO",
+	"CQE_MSG_FIFIO",
+	"GETDQE_FIFO",
+	"CMDP_FIFO",
+	"AWTCTRL_FIFO"
+};
+
+static irqreturn_t fatal_axi_int_v2_hw(int irq_no, void *p)
+{
+	struct hisi_hba *hisi_hba = p;
+	u32 irq_value, irq_msk, err_value;
+	struct device *dev = &hisi_hba->pdev->dev;
+
+	irq_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
+	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk | 0xfffffffe);
+
+	irq_value = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
+	if (irq_value) {
+		if (irq_value & BIT(ENT_INT_SRC3_WP_DEPTH_OFF)) {
+			hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
+					1 << ENT_INT_SRC3_WP_DEPTH_OFF);
+			panic("%s: write pointer and depth error (0x%x) \
+			       found!\n",
+			      dev_name(dev), irq_value);
+		}
+
+		if (irq_value & BIT(ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF)) {
+			hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
+					 1 <<
+					 ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF);
+			panic("%s: iptt no match slot error (0x%x) found!\n",
+			      dev_name(dev), irq_value);
+		}
+
+		if (irq_value & BIT(ENT_INT_SRC3_RP_DEPTH_OFF))
+			panic("%s: read pointer and depth error (0x%x) \
+			       found!\n",
+			      dev_name(dev), irq_value);
+
+		if (irq_value & BIT(ENT_INT_SRC3_AXI_OFF)) {
+			int i;
+
+			hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
+					1 << ENT_INT_SRC3_AXI_OFF);
+			err_value = hisi_sas_read32(hisi_hba,
+						    HGC_AXI_FIFO_ERR_INFO);
+
+			for (i = 0; i < AXI_ERR_NR; i++) {
+				if (err_value & BIT(i))
+					panic("%s: %s (0x%x) found!\n",
+					       dev_name(dev),
+					      axi_err_info[i], irq_value);
+			}
+		}
+
+		if (irq_value & BIT(ENT_INT_SRC3_FIFO_OFF)) {
+			int i;
+
+			hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
+					1 << ENT_INT_SRC3_FIFO_OFF);
+			err_value = hisi_sas_read32(hisi_hba,
+						    HGC_AXI_FIFO_ERR_INFO);
+
+			for (i = 0; i < FIFO_ERR_NR; i++) {
+				if (err_value & BIT(AXI_ERR_NR + i))
+					panic("%s: %s (0x%x) found!\n",
+					      dev_name(dev),
+					      fifo_err_info[i], irq_value);
+			}
+
+		}
+
+		if (irq_value & BIT(ENT_INT_SRC3_LM_OFF)) {
+			hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
+					1 << ENT_INT_SRC3_LM_OFF);
+			panic("%s: LM add/fetch list error (0x%x) found!\n",
+			      dev_name(dev), irq_value);
+		}
+
+		if (irq_value & BIT(ENT_INT_SRC3_ABT_OFF)) {
+			hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
+					1 << ENT_INT_SRC3_ABT_OFF);
+			panic("%s: SAS_HGC_ABT fetch LM list error (0x%x) found!\n",
+			      dev_name(dev), irq_value);
+		}
+	}
+
+	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk);
+
+	return IRQ_HANDLED;
+}
+
 static irqreturn_t cq_interrupt_v2_hw(int irq_no, void *p)
 {
 	struct hisi_sas_cq *cq = p;
@@ -2136,6 +2580,16 @@ static irqreturn_t sata_int_v2_hw(int irq_no, void *p)
 		goto end;
 	}
 
+	/* check ERR bit of Status Register */
+	if (fis->status & ATA_ERR) {
+		dev_warn(dev, "sata int: phy%d FIS status: 0x%x\n", phy_no,
+				fis->status);
+		disable_phy_v2_hw(hisi_hba, phy_no);
+		enable_phy_v2_hw(hisi_hba, phy_no);
+		res = IRQ_NONE;
+		goto end;
+	}
+
 	if (unlikely(phy_no == 8)) {
 		u32 port_state = hisi_sas_read32(hisi_hba, PORT_STATE);
 
@@ -2190,6 +2644,11 @@ static irq_handler_t phy_interrupts[HISI_SAS_PHY_INT_NR] = {
 	int_chnl_int_v2_hw,
 };
 
+static irq_handler_t fatal_interrupts[HISI_SAS_FATAL_INT_NR] = {
+	fatal_ecc_int_v2_hw,
+	fatal_axi_int_v2_hw
+};
+
 /**
  * There is a limitation in the hip06 chipset that we need
  * to map in all mbigen interrupts, even if they are not used.
@@ -2245,6 +2704,26 @@ static int interrupt_init_v2_hw(struct hisi_hba *hisi_hba)
 		}
 	}
 
+	for (i = 0; i < HISI_SAS_FATAL_INT_NR; i++) {
+		int idx = i;
+
+		irq = irq_map[idx + 81];
+		if (!irq) {
+			dev_err(dev, "irq init: fail map fatal interrupt %d\n",
+				idx);
+			return -ENOENT;
+		}
+
+		rc = devm_request_irq(dev, irq, fatal_interrupts[i], 0,
+				      DRV_NAME " fatal", hisi_hba);
+		if (rc) {
+			dev_err(dev,
+				"irq init: could not request fatal interrupt %d, rc=%d\n",
+				irq, rc);
+			return -ENOENT;
+		}
+	}
+
 	for (i = 0; i < hisi_hba->queue_count; i++) {
 		int idx = i + 96; /* First cq interrupt is irq96 */
 
@@ -2303,12 +2782,26 @@ static const struct hisi_sas_hw hisi_sas_v2_hw = {
 	.phy_enable = enable_phy_v2_hw,
 	.phy_disable = disable_phy_v2_hw,
 	.phy_hard_reset = phy_hard_reset_v2_hw,
+	.phy_set_linkrate = phy_set_linkrate_v2_hw,
+	.phy_get_max_linkrate = phy_get_max_linkrate_v2_hw,
 	.max_command_entries = HISI_SAS_COMMAND_ENTRIES_V2_HW,
 	.complete_hdr_size = sizeof(struct hisi_sas_complete_v2_hdr),
 };
 
 static int hisi_sas_v2_probe(struct platform_device *pdev)
 {
+	/*
+	 * Check if we should defer the probe before we probe the
+	 * upper layer, as it's hard to defer later on.
+	 */
+	int ret = platform_get_irq(pdev, 0);
+
+	if (ret < 0) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "cannot obtain irq\n");
+		return ret;
+	}
+
 	return hisi_sas_probe(pdev, &hisi_sas_v2_hw);
 }
 
@@ -2319,6 +2812,7 @@ static int hisi_sas_v2_remove(struct platform_device *pdev)
 
 static const struct of_device_id sas_v2_of_match[] = {
 	{ .compatible = "hisilicon,hip06-sas-v2",},
+	{ .compatible = "hisilicon,hip07-sas-v2",},
 	{},
 };
 MODULE_DEVICE_TABLE(of, sas_v2_of_match);
diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c
index a1d6ab76a514..691a09316952 100644
--- a/drivers/scsi/hpsa.c
+++ b/drivers/scsi/hpsa.c
@@ -276,6 +276,9 @@ static int hpsa_find_cfg_addrs(struct pci_dev *pdev, void __iomem *vaddr,
 static int hpsa_pci_find_memory_BAR(struct pci_dev *pdev,
 				    unsigned long *memory_bar);
 static int hpsa_lookup_board_id(struct pci_dev *pdev, u32 *board_id);
+static int wait_for_device_to_become_ready(struct ctlr_info *h,
+					   unsigned char lunaddr[],
+					   int reply_queue);
 static int hpsa_wait_for_board_state(struct pci_dev *pdev, void __iomem *vaddr,
 				     int wait_for_ready);
 static inline void finish_cmd(struct CommandList *c);
@@ -700,9 +703,7 @@ static ssize_t lunid_show(struct device *dev,
 	}
 	memcpy(lunid, hdev->scsi3addr, sizeof(lunid));
 	spin_unlock_irqrestore(&h->lock, flags);
-	return snprintf(buf, 20, "0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
-		lunid[0], lunid[1], lunid[2], lunid[3],
-		lunid[4], lunid[5], lunid[6], lunid[7]);
+	return snprintf(buf, 20, "0x%8phN\n", lunid);
 }
 
 static ssize_t unique_id_show(struct device *dev,
@@ -864,6 +865,16 @@ static ssize_t path_info_show(struct device *dev,
 	return output_len;
 }
 
+static ssize_t host_show_ctlr_num(struct device *dev,
+	struct device_attribute *attr, char *buf)
+{
+	struct ctlr_info *h;
+	struct Scsi_Host *shost = class_to_shost(dev);
+
+	h = shost_to_hba(shost);
+	return snprintf(buf, 20, "%d\n", h->ctlr);
+}
+
 static DEVICE_ATTR(raid_level, S_IRUGO, raid_level_show, NULL);
 static DEVICE_ATTR(lunid, S_IRUGO, lunid_show, NULL);
 static DEVICE_ATTR(unique_id, S_IRUGO, unique_id_show, NULL);
@@ -887,6 +898,8 @@ static DEVICE_ATTR(resettable, S_IRUGO,
 	host_show_resettable, NULL);
 static DEVICE_ATTR(lockup_detected, S_IRUGO,
 	host_show_lockup_detected, NULL);
+static DEVICE_ATTR(ctlr_num, S_IRUGO,
+	host_show_ctlr_num, NULL);
 
 static struct device_attribute *hpsa_sdev_attrs[] = {
 	&dev_attr_raid_level,
@@ -907,6 +920,7 @@ static struct device_attribute *hpsa_shost_attrs[] = {
 	&dev_attr_hp_ssd_smart_path_status,
 	&dev_attr_raid_offload_debug,
 	&dev_attr_lockup_detected,
+	&dev_attr_ctlr_num,
 	NULL,
 };
 
@@ -1001,7 +1015,7 @@ static void set_performant_mode(struct ctlr_info *h, struct CommandList *c,
 {
 	if (likely(h->transMethod & CFGTBL_Trans_Performant)) {
 		c->busaddr |= 1 | (h->blockFetchTable[c->Header.SGList] << 1);
-		if (unlikely(!h->msix_vector))
+		if (unlikely(!h->msix_vectors))
 			return;
 		if (likely(reply_queue == DEFAULT_REPLY_QUEUE))
 			c->Header.ReplyQueue =
@@ -2541,7 +2555,7 @@ static void complete_scsi_command(struct CommandList *cp)
 
 	if ((unlikely(hpsa_is_pending_event(cp)))) {
 		if (cp->reset_pending)
-			return hpsa_cmd_resolve_and_free(h, cp);
+			return hpsa_cmd_free_and_done(h, cp, cmd);
 		if (cp->abort_pending)
 			return hpsa_cmd_abort_and_free(h, cp, cmd);
 	}
@@ -2824,14 +2838,8 @@ static void hpsa_print_cmd(struct ctlr_info *h, char *txt,
 	const u8 *cdb = c->Request.CDB;
 	const u8 *lun = c->Header.LUN.LunAddrBytes;
 
-	dev_warn(&h->pdev->dev, "%s: LUN:%02x%02x%02x%02x%02x%02x%02x%02x"
-	" CDB:%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n",
-		txt, lun[0], lun[1], lun[2], lun[3],
-		lun[4], lun[5], lun[6], lun[7],
-		cdb[0], cdb[1], cdb[2], cdb[3],
-		cdb[4], cdb[5], cdb[6], cdb[7],
-		cdb[8], cdb[9], cdb[10], cdb[11],
-		cdb[12], cdb[13], cdb[14], cdb[15]);
+	dev_warn(&h->pdev->dev, "%s: LUN:%8phN CDB:%16phN\n",
+		 txt, lun, cdb);
 }
 
 static void hpsa_scsi_interpret_error(struct ctlr_info *h,
@@ -3080,6 +3088,8 @@ static int hpsa_do_reset(struct ctlr_info *h, struct hpsa_scsi_dev_t *dev,
 
 	if (unlikely(rc))
 		atomic_set(&dev->reset_cmds_out, 0);
+	else
+		wait_for_device_to_become_ready(h, scsi3addr, 0);
 
 	mutex_unlock(&h->reset_mutex);
 	return rc;
@@ -3623,8 +3633,32 @@ out:
 static inline int hpsa_scsi_do_report_phys_luns(struct ctlr_info *h,
 		struct ReportExtendedLUNdata *buf, int bufsize)
 {
-	return hpsa_scsi_do_report_luns(h, 0, buf, bufsize,
-						HPSA_REPORT_PHYS_EXTENDED);
+	int rc;
+	struct ReportLUNdata *lbuf;
+
+	rc = hpsa_scsi_do_report_luns(h, 0, buf, bufsize,
+				      HPSA_REPORT_PHYS_EXTENDED);
+	if (!rc || !hpsa_allow_any)
+		return rc;
+
+	/* REPORT PHYS EXTENDED is not supported */
+	lbuf = kzalloc(sizeof(*lbuf), GFP_KERNEL);
+	if (!lbuf)
+		return -ENOMEM;
+
+	rc = hpsa_scsi_do_report_luns(h, 0, lbuf, sizeof(*lbuf), 0);
+	if (!rc) {
+		int i;
+		u32 nphys;
+
+		/* Copy ReportLUNdata header */
+		memcpy(buf, lbuf, 8);
+		nphys = be32_to_cpu(*((__be32 *)lbuf->LUNListLength)) / 8;
+		for (i = 0; i < nphys; i++)
+			memcpy(buf->LUN[i].lunid, lbuf->LUN[i], 8);
+	}
+	kfree(lbuf);
+	return rc;
 }
 
 static inline int hpsa_scsi_do_report_log_luns(struct ctlr_info *h,
@@ -5488,7 +5522,7 @@ static int hpsa_scsi_queue_command(struct Scsi_Host *sh, struct scsi_cmnd *cmd)
 
 	dev = cmd->device->hostdata;
 	if (!dev) {
-		cmd->result = NOT_READY << 16; /* host byte */
+		cmd->result = DID_NO_CONNECT << 16;
 		cmd->scsi_done(cmd);
 		return 0;
 	}
@@ -5569,6 +5603,14 @@ static void hpsa_scan_start(struct Scsi_Host *sh)
 	if (unlikely(lockup_detected(h)))
 		return hpsa_scan_complete(h);
 
+	/*
+	 * Do the scan after a reset completion
+	 */
+	if (h->reset_in_progress) {
+		h->drv_req_rescan = 1;
+		return;
+	}
+
 	hpsa_update_scsi_devices(h);
 
 	hpsa_scan_complete(h);
@@ -5624,7 +5666,7 @@ static int hpsa_scsi_host_alloc(struct ctlr_info *h)
 	sh->sg_tablesize = h->maxsgentries;
 	sh->transportt = hpsa_sas_transport_template;
 	sh->hostdata[0] = (unsigned long) h;
-	sh->irq = h->intr[h->intr_mode];
+	sh->irq = pci_irq_vector(h->pdev, 0);
 	sh->unique_id = sh->irq;
 
 	h->scsi_host = sh;
@@ -5999,11 +6041,9 @@ static int hpsa_send_reset_as_abort_ioaccel2(struct ctlr_info *h,
 
 	if (h->raid_offload_debug > 0)
 		dev_info(&h->pdev->dev,
-			"scsi %d:%d:%d:%d %s scsi3addr 0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
+			"scsi %d:%d:%d:%d %s scsi3addr 0x%8phN\n",
 			h->scsi_host->host_no, dev->bus, dev->target, dev->lun,
-			"Reset as abort",
-			scsi3addr[0], scsi3addr[1], scsi3addr[2], scsi3addr[3],
-			scsi3addr[4], scsi3addr[5], scsi3addr[6], scsi3addr[7]);
+			"Reset as abort", scsi3addr);
 
 	if (!dev->offload_enabled) {
 		dev_warn(&h->pdev->dev,
@@ -6020,32 +6060,28 @@ static int hpsa_send_reset_as_abort_ioaccel2(struct ctlr_info *h,
 	/* send the reset */
 	if (h->raid_offload_debug > 0)
 		dev_info(&h->pdev->dev,
-			"Reset as abort: Resetting physical device at scsi3addr 0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
-			psa[0], psa[1], psa[2], psa[3],
-			psa[4], psa[5], psa[6], psa[7]);
+			"Reset as abort: Resetting physical device at scsi3addr 0x%8phN\n",
+			psa);
 	rc = hpsa_do_reset(h, dev, psa, HPSA_PHYS_TARGET_RESET, reply_queue);
 	if (rc != 0) {
 		dev_warn(&h->pdev->dev,
-			"Reset as abort: Failed on physical device at scsi3addr 0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
-			psa[0], psa[1], psa[2], psa[3],
-			psa[4], psa[5], psa[6], psa[7]);
+			"Reset as abort: Failed on physical device at scsi3addr 0x%8phN\n",
+			psa);
 		return rc; /* failed to reset */
 	}
 
 	/* wait for device to recover */
 	if (wait_for_device_to_become_ready(h, psa, reply_queue) != 0) {
 		dev_warn(&h->pdev->dev,
-			"Reset as abort: Failed: Device never recovered from reset: 0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
-			psa[0], psa[1], psa[2], psa[3],
-			psa[4], psa[5], psa[6], psa[7]);
+			"Reset as abort: Failed: Device never recovered from reset: 0x%8phN\n",
+			psa);
 		return -1;  /* failed to recover */
 	}
 
 	/* device recovered */
 	dev_info(&h->pdev->dev,
-		"Reset as abort: Device recovered from reset: scsi3addr 0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
-		psa[0], psa[1], psa[2], psa[3],
-		psa[4], psa[5], psa[6], psa[7]);
+		"Reset as abort: Device recovered from reset: scsi3addr 0x%8phN\n",
+		psa);
 
 	return rc; /* success */
 }
@@ -6663,8 +6699,7 @@ static int hpsa_big_passthru_ioctl(struct ctlr_info *h, void __user *argp)
 		return -EINVAL;
 	if (!capable(CAP_SYS_RAWIO))
 		return -EPERM;
-	ioc = (BIG_IOCTL_Command_struct *)
-	    kmalloc(sizeof(*ioc), GFP_KERNEL);
+	ioc = kmalloc(sizeof(*ioc), GFP_KERNEL);
 	if (!ioc) {
 		status = -ENOMEM;
 		goto cleanup1;
@@ -7658,67 +7693,41 @@ static int find_PCI_BAR_index(struct pci_dev *pdev, unsigned long pci_bar_addr)
 
 static void hpsa_disable_interrupt_mode(struct ctlr_info *h)
 {
-	if (h->msix_vector) {
-		if (h->pdev->msix_enabled)
-			pci_disable_msix(h->pdev);
-		h->msix_vector = 0;
-	} else if (h->msi_vector) {
-		if (h->pdev->msi_enabled)
-			pci_disable_msi(h->pdev);
-		h->msi_vector = 0;
-	}
+	pci_free_irq_vectors(h->pdev);
+	h->msix_vectors = 0;
 }
 
 /* If MSI/MSI-X is supported by the kernel we will try to enable it on
  * controllers that are capable. If not, we use legacy INTx mode.
  */
-static void hpsa_interrupt_mode(struct ctlr_info *h)
+static int hpsa_interrupt_mode(struct ctlr_info *h)
 {
-#ifdef CONFIG_PCI_MSI
-	int err, i;
-	struct msix_entry hpsa_msix_entries[MAX_REPLY_QUEUES];
-
-	for (i = 0; i < MAX_REPLY_QUEUES; i++) {
-		hpsa_msix_entries[i].vector = 0;
-		hpsa_msix_entries[i].entry = i;
-	}
+	unsigned int flags = PCI_IRQ_LEGACY;
+	int ret;
 
 	/* Some boards advertise MSI but don't really support it */
-	if ((h->board_id == 0x40700E11) || (h->board_id == 0x40800E11) ||
-	    (h->board_id == 0x40820E11) || (h->board_id == 0x40830E11))
-		goto default_int_mode;
-	if (pci_find_capability(h->pdev, PCI_CAP_ID_MSIX)) {
-		dev_info(&h->pdev->dev, "MSI-X capable controller\n");
-		h->msix_vector = MAX_REPLY_QUEUES;
-		if (h->msix_vector > num_online_cpus())
-			h->msix_vector = num_online_cpus();
-		err = pci_enable_msix_range(h->pdev, hpsa_msix_entries,
-					    1, h->msix_vector);
-		if (err < 0) {
-			dev_warn(&h->pdev->dev, "MSI-X init failed %d\n", err);
-			h->msix_vector = 0;
-			goto single_msi_mode;
-		} else if (err < h->msix_vector) {
-			dev_warn(&h->pdev->dev, "only %d MSI-X vectors "
-			       "available\n", err);
+	switch (h->board_id) {
+	case 0x40700E11:
+	case 0x40800E11:
+	case 0x40820E11:
+	case 0x40830E11:
+		break;
+	default:
+		ret = pci_alloc_irq_vectors(h->pdev, 1, MAX_REPLY_QUEUES,
+				PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
+		if (ret > 0) {
+			h->msix_vectors = ret;
+			return 0;
 		}
-		h->msix_vector = err;
-		for (i = 0; i < h->msix_vector; i++)
-			h->intr[i] = hpsa_msix_entries[i].vector;
-		return;
-	}
-single_msi_mode:
-	if (pci_find_capability(h->pdev, PCI_CAP_ID_MSI)) {
-		dev_info(&h->pdev->dev, "MSI capable controller\n");
-		if (!pci_enable_msi(h->pdev))
-			h->msi_vector = 1;
-		else
-			dev_warn(&h->pdev->dev, "MSI init failed\n");
+
+		flags |= PCI_IRQ_MSI;
+		break;
 	}
-default_int_mode:
-#endif				/* CONFIG_PCI_MSI */
-	/* if we get here we're going to use the default interrupt mode */
-	h->intr[h->intr_mode] = h->pdev->irq;
+
+	ret = pci_alloc_irq_vectors(h->pdev, 1, 1, flags);
+	if (ret < 0)
+		return ret;
+	return 0;
 }
 
 static int hpsa_lookup_board_id(struct pci_dev *pdev, u32 *board_id)
@@ -8074,7 +8083,9 @@ static int hpsa_pci_init(struct ctlr_info *h)
 
 	pci_set_master(h->pdev);
 
-	hpsa_interrupt_mode(h);
+	err = hpsa_interrupt_mode(h);
+	if (err)
+		goto clean1;
 	err = hpsa_pci_find_memory_BAR(h->pdev, &h->paddr);
 	if (err)
 		goto clean2;	/* intmode+region, pci */
@@ -8110,6 +8121,7 @@ clean3:	/* vaddr, intmode+region, pci */
 	h->vaddr = NULL;
 clean2:	/* intmode+region, pci */
 	hpsa_disable_interrupt_mode(h);
+clean1:
 	/*
 	 * call pci_disable_device before pci_release_regions per
 	 * Documentation/PCI/pci.txt
@@ -8243,34 +8255,20 @@ clean_up:
 	return -ENOMEM;
 }
 
-static void hpsa_irq_affinity_hints(struct ctlr_info *h)
-{
-	int i, cpu;
-
-	cpu = cpumask_first(cpu_online_mask);
-	for (i = 0; i < h->msix_vector; i++) {
-		irq_set_affinity_hint(h->intr[i], get_cpu_mask(cpu));
-		cpu = cpumask_next(cpu, cpu_online_mask);
-	}
-}
-
 /* clear affinity hints and free MSI-X, MSI, or legacy INTx vectors */
 static void hpsa_free_irqs(struct ctlr_info *h)
 {
 	int i;
 
-	if (!h->msix_vector || h->intr_mode != PERF_MODE_INT) {
+	if (!h->msix_vectors || h->intr_mode != PERF_MODE_INT) {
 		/* Single reply queue, only one irq to free */
-		i = h->intr_mode;
-		irq_set_affinity_hint(h->intr[i], NULL);
-		free_irq(h->intr[i], &h->q[i]);
-		h->q[i] = 0;
+		free_irq(pci_irq_vector(h->pdev, 0), &h->q[h->intr_mode]);
+		h->q[h->intr_mode] = 0;
 		return;
 	}
 
-	for (i = 0; i < h->msix_vector; i++) {
-		irq_set_affinity_hint(h->intr[i], NULL);
-		free_irq(h->intr[i], &h->q[i]);
+	for (i = 0; i < h->msix_vectors; i++) {
+		free_irq(pci_irq_vector(h->pdev, i), &h->q[i]);
 		h->q[i] = 0;
 	}
 	for (; i < MAX_REPLY_QUEUES; i++)
@@ -8291,11 +8289,11 @@ static int hpsa_request_irqs(struct ctlr_info *h,
 	for (i = 0; i < MAX_REPLY_QUEUES; i++)
 		h->q[i] = (u8) i;
 
-	if (h->intr_mode == PERF_MODE_INT && h->msix_vector > 0) {
+	if (h->intr_mode == PERF_MODE_INT && h->msix_vectors > 0) {
 		/* If performant mode and MSI-X, use multiple reply queues */
-		for (i = 0; i < h->msix_vector; i++) {
+		for (i = 0; i < h->msix_vectors; i++) {
 			sprintf(h->intrname[i], "%s-msix%d", h->devname, i);
-			rc = request_irq(h->intr[i], msixhandler,
+			rc = request_irq(pci_irq_vector(h->pdev, i), msixhandler,
 					0, h->intrname[i],
 					&h->q[i]);
 			if (rc) {
@@ -8303,9 +8301,9 @@ static int hpsa_request_irqs(struct ctlr_info *h,
 
 				dev_err(&h->pdev->dev,
 					"failed to get irq %d for %s\n",
-				       h->intr[i], h->devname);
+				       pci_irq_vector(h->pdev, i), h->devname);
 				for (j = 0; j < i; j++) {
-					free_irq(h->intr[j], &h->q[j]);
+					free_irq(pci_irq_vector(h->pdev, j), &h->q[j]);
 					h->q[j] = 0;
 				}
 				for (; j < MAX_REPLY_QUEUES; j++)
@@ -8313,33 +8311,27 @@ static int hpsa_request_irqs(struct ctlr_info *h,
 				return rc;
 			}
 		}
-		hpsa_irq_affinity_hints(h);
 	} else {
 		/* Use single reply pool */
-		if (h->msix_vector > 0 || h->msi_vector) {
-			if (h->msix_vector)
-				sprintf(h->intrname[h->intr_mode],
-					"%s-msix", h->devname);
-			else
-				sprintf(h->intrname[h->intr_mode],
-					"%s-msi", h->devname);
-			rc = request_irq(h->intr[h->intr_mode],
+		if (h->msix_vectors > 0 || h->pdev->msi_enabled) {
+			sprintf(h->intrname[0], "%s-msi%s", h->devname,
+				h->msix_vectors ? "x" : "");
+			rc = request_irq(pci_irq_vector(h->pdev, 0),
 				msixhandler, 0,
-				h->intrname[h->intr_mode],
+				h->intrname[0],
 				&h->q[h->intr_mode]);
 		} else {
 			sprintf(h->intrname[h->intr_mode],
 				"%s-intx", h->devname);
-			rc = request_irq(h->intr[h->intr_mode],
+			rc = request_irq(pci_irq_vector(h->pdev, 0),
 				intxhandler, IRQF_SHARED,
-				h->intrname[h->intr_mode],
+				h->intrname[0],
 				&h->q[h->intr_mode]);
 		}
-		irq_set_affinity_hint(h->intr[h->intr_mode], NULL);
 	}
 	if (rc) {
 		dev_err(&h->pdev->dev, "failed to get irq %d for %s\n",
-		       h->intr[h->intr_mode], h->devname);
+		       pci_irq_vector(h->pdev, 0), h->devname);
 		hpsa_free_irqs(h);
 		return -ENODEV;
 	}
@@ -8640,6 +8632,14 @@ static void hpsa_rescan_ctlr_worker(struct work_struct *work)
 	if (h->remove_in_progress)
 		return;
 
+	/*
+	 * Do the scan after the reset
+	 */
+	if (h->reset_in_progress) {
+		h->drv_req_rescan = 1;
+		return;
+	}
+
 	if (hpsa_ctlr_needs_rescan(h) || hpsa_offline_devices_ready(h)) {
 		scsi_host_get(h->scsi_host);
 		hpsa_ack_ctlr_events(h);
@@ -9525,7 +9525,7 @@ static int hpsa_put_ctlr_into_performant_mode(struct ctlr_info *h)
 			return rc;
 	}
 
-	h->nreply_queues = h->msix_vector > 0 ? h->msix_vector : 1;
+	h->nreply_queues = h->msix_vectors > 0 ? h->msix_vectors : 1;
 	hpsa_get_max_perf_mode_cmds(h);
 	/* Performant mode ring buffer and supporting data structures */
 	h->reply_queue_size = h->max_commands * sizeof(u64);
diff --git a/drivers/scsi/hpsa.h b/drivers/scsi/hpsa.h
index 9ea162de80dc..64e98295b707 100644
--- a/drivers/scsi/hpsa.h
+++ b/drivers/scsi/hpsa.h
@@ -176,9 +176,7 @@ struct ctlr_info {
 #	define DOORBELL_INT	1
 #	define SIMPLE_MODE_INT	2
 #	define MEMQ_MODE_INT	3
-	unsigned int intr[MAX_REPLY_QUEUES];
-	unsigned int msix_vector;
-	unsigned int msi_vector;
+	unsigned int msix_vectors;
 	int intr_mode; /* either PERF_MODE_INT or SIMPLE_MODE_INT */
 	struct access_method access;
 
@@ -466,7 +464,7 @@ static unsigned long SA5_performant_completed(struct ctlr_info *h, u8 q)
 	unsigned long register_value = FIFO_EMPTY;
 
 	/* msi auto clears the interrupt pending bit. */
-	if (unlikely(!(h->msi_vector || h->msix_vector))) {
+	if (unlikely(!(h->pdev->msi_enabled || h->msix_vectors))) {
 		/* flush the controller write of the reply queue by reading
 		 * outbound doorbell status register.
 		 */
diff --git a/drivers/scsi/ibmvscsi/ibmvfc.c b/drivers/scsi/ibmvscsi/ibmvfc.c
index 7e487c78279c..78b72c28a55d 100644
--- a/drivers/scsi/ibmvscsi/ibmvfc.c
+++ b/drivers/scsi/ibmvscsi/ibmvfc.c
@@ -32,6 +32,7 @@
 #include <linux/of.h>
 #include <linux/pm.h>
 #include <linux/stringify.h>
+#include <linux/bsg-lib.h>
 #include <asm/firmware.h>
 #include <asm/irq.h>
 #include <asm/vio.h>
@@ -1701,14 +1702,14 @@ static void ibmvfc_bsg_timeout_done(struct ibmvfc_event *evt)
 
 /**
  * ibmvfc_bsg_timeout - Handle a BSG timeout
- * @job:	struct fc_bsg_job that timed out
+ * @job:	struct bsg_job that timed out
  *
  * Returns:
  *	0 on success / other on failure
  **/
-static int ibmvfc_bsg_timeout(struct fc_bsg_job *job)
+static int ibmvfc_bsg_timeout(struct bsg_job *job)
 {
-	struct ibmvfc_host *vhost = shost_priv(job->shost);
+	struct ibmvfc_host *vhost = shost_priv(fc_bsg_to_shost(job));
 	unsigned long port_id = (unsigned long)job->dd_data;
 	struct ibmvfc_event *evt;
 	struct ibmvfc_tmf *tmf;
@@ -1814,41 +1815,43 @@ unlock_out:
 
 /**
  * ibmvfc_bsg_request - Handle a BSG request
- * @job:	struct fc_bsg_job to be executed
+ * @job:	struct bsg_job to be executed
  *
  * Returns:
  *	0 on success / other on failure
  **/
-static int ibmvfc_bsg_request(struct fc_bsg_job *job)
+static int ibmvfc_bsg_request(struct bsg_job *job)
 {
-	struct ibmvfc_host *vhost = shost_priv(job->shost);
-	struct fc_rport *rport = job->rport;
+	struct ibmvfc_host *vhost = shost_priv(fc_bsg_to_shost(job));
+	struct fc_rport *rport = fc_bsg_to_rport(job);
 	struct ibmvfc_passthru_mad *mad;
 	struct ibmvfc_event *evt;
 	union ibmvfc_iu rsp_iu;
 	unsigned long flags, port_id = -1;
-	unsigned int code = job->request->msgcode;
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
+	unsigned int code = bsg_request->msgcode;
 	int rc = 0, req_seg, rsp_seg, issue_login = 0;
 	u32 fc_flags, rsp_len;
 
 	ENTER;
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 	if (rport)
 		port_id = rport->port_id;
 
 	switch (code) {
 	case FC_BSG_HST_ELS_NOLOGIN:
-		port_id = (job->request->rqst_data.h_els.port_id[0] << 16) |
-			(job->request->rqst_data.h_els.port_id[1] << 8) |
-			job->request->rqst_data.h_els.port_id[2];
+		port_id = (bsg_request->rqst_data.h_els.port_id[0] << 16) |
+			(bsg_request->rqst_data.h_els.port_id[1] << 8) |
+			bsg_request->rqst_data.h_els.port_id[2];
 	case FC_BSG_RPT_ELS:
 		fc_flags = IBMVFC_FC_ELS;
 		break;
 	case FC_BSG_HST_CT:
 		issue_login = 1;
-		port_id = (job->request->rqst_data.h_ct.port_id[0] << 16) |
-			(job->request->rqst_data.h_ct.port_id[1] << 8) |
-			job->request->rqst_data.h_ct.port_id[2];
+		port_id = (bsg_request->rqst_data.h_ct.port_id[0] << 16) |
+			(bsg_request->rqst_data.h_ct.port_id[1] << 8) |
+			bsg_request->rqst_data.h_ct.port_id[2];
 	case FC_BSG_RPT_CT:
 		fc_flags = IBMVFC_FC_CT_IU;
 		break;
@@ -1937,13 +1940,14 @@ static int ibmvfc_bsg_request(struct fc_bsg_job *job)
 	if (rsp_iu.passthru.common.status)
 		rc = -EIO;
 	else
-		job->reply->reply_payload_rcv_len = rsp_len;
+		bsg_reply->reply_payload_rcv_len = rsp_len;
 
 	spin_lock_irqsave(vhost->host->host_lock, flags);
 	ibmvfc_free_event(evt);
 	spin_unlock_irqrestore(vhost->host->host_lock, flags);
-	job->reply->result = rc;
-	job->job_done(job);
+	bsg_reply->result = rc;
+	bsg_job_done(job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	rc = 0;
 out:
 	dma_unmap_sg(vhost->dev, job->request_payload.sg_list,
diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
index 642b739ad0da..c9fa3565c671 100644
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
@@ -22,7 +22,7 @@
  *
  ****************************************************************************/
 
-#define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
+#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
 
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -81,7 +81,7 @@ static void ibmvscsis_determine_resid(struct se_cmd *se_cmd,
 		}
 	} else if (se_cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
 		if (se_cmd->data_direction == DMA_TO_DEVICE) {
-			/*  residual data from an overflow write */
+			/* residual data from an overflow write */
 			rsp->flags = SRP_RSP_FLAG_DOOVER;
 			rsp->data_out_res_cnt = cpu_to_be32(residual_count);
 		} else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
@@ -101,7 +101,7 @@ static void ibmvscsis_determine_resid(struct se_cmd *se_cmd,
  * and the function returns TRUE.
  *
  * EXECUTION ENVIRONMENT:
- *      Interrupt or Process environment
+ *	Interrupt or Process environment
  */
 static bool connection_broken(struct scsi_info *vscsi)
 {
@@ -324,7 +324,7 @@ static struct viosrp_crq *ibmvscsis_cmd_q_dequeue(uint mask,
 }
 
 /**
- * ibmvscsis_send_init_message() -  send initialize message to the client
+ * ibmvscsis_send_init_message() - send initialize message to the client
  * @vscsi:	Pointer to our adapter structure
  * @format:	Which Init Message format to send
  *
@@ -382,13 +382,13 @@ static long ibmvscsis_check_init_msg(struct scsi_info *vscsi, uint *format)
 					      vscsi->cmd_q.base_addr);
 		if (crq) {
 			*format = (uint)(crq->format);
-			rc =  ERROR;
+			rc = ERROR;
 			crq->valid = INVALIDATE_CMD_RESP_EL;
 			dma_rmb();
 		}
 	} else {
 		*format = (uint)(crq->format);
-		rc =  ERROR;
+		rc = ERROR;
 		crq->valid = INVALIDATE_CMD_RESP_EL;
 		dma_rmb();
 	}
@@ -397,166 +397,6 @@ static long ibmvscsis_check_init_msg(struct scsi_info *vscsi, uint *format)
 }
 
 /**
- * ibmvscsis_establish_new_q() - Establish new CRQ queue
- * @vscsi:	Pointer to our adapter structure
- * @new_state:	New state being established after resetting the queue
- *
- * Must be called with interrupt lock held.
- */
-static long ibmvscsis_establish_new_q(struct scsi_info *vscsi,  uint new_state)
-{
-	long rc = ADAPT_SUCCESS;
-	uint format;
-
-	vscsi->flags &= PRESERVE_FLAG_FIELDS;
-	vscsi->rsp_q_timer.timer_pops = 0;
-	vscsi->debit = 0;
-	vscsi->credit = 0;
-
-	rc = vio_enable_interrupts(vscsi->dma_dev);
-	if (rc) {
-		pr_warn("reset_queue: failed to enable interrupts, rc %ld\n",
-			rc);
-		return rc;
-	}
-
-	rc = ibmvscsis_check_init_msg(vscsi, &format);
-	if (rc) {
-		dev_err(&vscsi->dev, "reset_queue: check_init_msg failed, rc %ld\n",
-			rc);
-		return rc;
-	}
-
-	if (format == UNUSED_FORMAT && new_state == WAIT_CONNECTION) {
-		rc = ibmvscsis_send_init_message(vscsi, INIT_MSG);
-		switch (rc) {
-		case H_SUCCESS:
-		case H_DROPPED:
-		case H_CLOSED:
-			rc = ADAPT_SUCCESS;
-			break;
-
-		case H_PARAMETER:
-		case H_HARDWARE:
-			break;
-
-		default:
-			vscsi->state = UNDEFINED;
-			rc = H_HARDWARE;
-			break;
-		}
-	}
-
-	return rc;
-}
-
-/**
- * ibmvscsis_reset_queue() - Reset CRQ Queue
- * @vscsi:	Pointer to our adapter structure
- * @new_state:	New state to establish after resetting the queue
- *
- * This function calls h_free_q and then calls h_reg_q and does all
- * of the bookkeeping to get us back to where we can communicate.
- *
- * Actually, we don't always call h_free_crq.  A problem was discovered
- * where one partition would close and reopen his queue, which would
- * cause his partner to get a transport event, which would cause him to
- * close and reopen his queue, which would cause the original partition
- * to get a transport event, etc., etc.  To prevent this, we don't
- * actually close our queue if the client initiated the reset, (i.e.
- * either we got a transport event or we have detected that the client's
- * queue is gone)
- *
- * EXECUTION ENVIRONMENT:
- *	Process environment, called with interrupt lock held
- */
-static void ibmvscsis_reset_queue(struct scsi_info *vscsi, uint new_state)
-{
-	int bytes;
-	long rc = ADAPT_SUCCESS;
-
-	pr_debug("reset_queue: flags 0x%x\n", vscsi->flags);
-
-	/* don't reset, the client did it for us */
-	if (vscsi->flags & (CLIENT_FAILED | TRANS_EVENT)) {
-		vscsi->flags &=  PRESERVE_FLAG_FIELDS;
-		vscsi->rsp_q_timer.timer_pops = 0;
-		vscsi->debit = 0;
-		vscsi->credit = 0;
-		vscsi->state = new_state;
-		vio_enable_interrupts(vscsi->dma_dev);
-	} else {
-		rc = ibmvscsis_free_command_q(vscsi);
-		if (rc == ADAPT_SUCCESS) {
-			vscsi->state = new_state;
-
-			bytes = vscsi->cmd_q.size * PAGE_SIZE;
-			rc = h_reg_crq(vscsi->dds.unit_id,
-				       vscsi->cmd_q.crq_token, bytes);
-			if (rc == H_CLOSED || rc == H_SUCCESS) {
-				rc = ibmvscsis_establish_new_q(vscsi,
-							       new_state);
-			}
-
-			if (rc != ADAPT_SUCCESS) {
-				pr_debug("reset_queue: reg_crq rc %ld\n", rc);
-
-				vscsi->state = ERR_DISCONNECTED;
-				vscsi->flags |=  RESPONSE_Q_DOWN;
-				ibmvscsis_free_command_q(vscsi);
-			}
-		} else {
-			vscsi->state = ERR_DISCONNECTED;
-			vscsi->flags |= RESPONSE_Q_DOWN;
-		}
-	}
-}
-
-/**
- * ibmvscsis_free_cmd_resources() - Free command resources
- * @vscsi:	Pointer to our adapter structure
- * @cmd:	Command which is not longer in use
- *
- * Must be called with interrupt lock held.
- */
-static void ibmvscsis_free_cmd_resources(struct scsi_info *vscsi,
-					 struct ibmvscsis_cmd *cmd)
-{
-	struct iu_entry *iue = cmd->iue;
-
-	switch (cmd->type) {
-	case TASK_MANAGEMENT:
-	case SCSI_CDB:
-		/*
-		 * When the queue goes down this value is cleared, so it
-		 * cannot be cleared in this general purpose function.
-		 */
-		if (vscsi->debit)
-			vscsi->debit -= 1;
-		break;
-	case ADAPTER_MAD:
-		vscsi->flags &= ~PROCESSING_MAD;
-		break;
-	case UNSET_TYPE:
-		break;
-	default:
-		dev_err(&vscsi->dev, "free_cmd_resources unknown type %d\n",
-			cmd->type);
-		break;
-	}
-
-	cmd->iue = NULL;
-	list_add_tail(&cmd->list, &vscsi->free_cmd);
-	srp_iu_put(iue);
-
-	if (list_empty(&vscsi->active_q) && list_empty(&vscsi->schedule_q) &&
-	    list_empty(&vscsi->waiting_rsp) && (vscsi->flags & WAIT_FOR_IDLE)) {
-		vscsi->flags &= ~WAIT_FOR_IDLE;
-		complete(&vscsi->wait_idle);
-	}
-}
-
-/**
  * ibmvscsis_disconnect() - Helper function to disconnect
  * @work:	Pointer to work_struct, gives access to our adapter structure
  *
@@ -575,7 +415,6 @@ static void ibmvscsis_disconnect(struct work_struct *work)
 					       proc_work);
 	u16 new_state;
 	bool wait_idle = false;
-	long rc = ADAPT_SUCCESS;
 
 	spin_lock_bh(&vscsi->intr_lock);
 	new_state = vscsi->new_state;
@@ -589,7 +428,7 @@ static void ibmvscsis_disconnect(struct work_struct *work)
 	 * should transitition to the new state
 	 */
 	switch (vscsi->state) {
-	/*  Should never be called while in this state. */
+	/* Should never be called while in this state. */
 	case NO_QUEUE:
 	/*
 	 * Can never transition from this state;
@@ -628,30 +467,24 @@ static void ibmvscsis_disconnect(struct work_struct *work)
 			vscsi->state = new_state;
 		break;
 
-	/*
-	 * If this is a transition into an error state.
-	 * a client is attempting to establish a connection
-	 * and has violated the RPA protocol.
-	 * There can be nothing pending on the adapter although
-	 * there can be requests in the command queue.
-	 */
 	case WAIT_ENABLED:
-	case PART_UP_WAIT_ENAB:
 		switch (new_state) {
-		case ERR_DISCONNECT:
-			vscsi->flags |= RESPONSE_Q_DOWN;
+		case UNCONFIGURING:
 			vscsi->state = new_state;
+			vscsi->flags |= RESPONSE_Q_DOWN;
 			vscsi->flags &= ~(SCHEDULE_DISCONNECT |
 					  DISCONNECT_SCHEDULED);
-			ibmvscsis_free_command_q(vscsi);
-			break;
-		case ERR_DISCONNECT_RECONNECT:
-			ibmvscsis_reset_queue(vscsi, WAIT_ENABLED);
+			dma_rmb();
+			if (vscsi->flags & CFG_SLEEPING) {
+				vscsi->flags &= ~CFG_SLEEPING;
+				complete(&vscsi->unconfig);
+			}
 			break;
 
 		/* should never happen */
+		case ERR_DISCONNECT:
+		case ERR_DISCONNECT_RECONNECT:
 		case WAIT_IDLE:
-			rc = ERROR;
 			dev_err(&vscsi->dev, "disconnect: invalid state %d for WAIT_IDLE\n",
 				vscsi->state);
 			break;
@@ -660,6 +493,13 @@ static void ibmvscsis_disconnect(struct work_struct *work)
 
 	case WAIT_IDLE:
 		switch (new_state) {
+		case UNCONFIGURING:
+			vscsi->flags |= RESPONSE_Q_DOWN;
+			vscsi->state = new_state;
+			vscsi->flags &= ~(SCHEDULE_DISCONNECT |
+					  DISCONNECT_SCHEDULED);
+			ibmvscsis_free_command_q(vscsi);
+			break;
 		case ERR_DISCONNECT:
 		case ERR_DISCONNECT_RECONNECT:
 			vscsi->state = new_state;
@@ -788,7 +628,6 @@ static void ibmvscsis_post_disconnect(struct scsi_info *vscsi, uint new_state,
 			break;
 
 		case WAIT_ENABLED:
-		case PART_UP_WAIT_ENAB:
 		case WAIT_IDLE:
 		case WAIT_CONNECTION:
 		case CONNECTED:
@@ -806,6 +645,310 @@ static void ibmvscsis_post_disconnect(struct scsi_info *vscsi, uint new_state,
 }
 
 /**
+ * ibmvscsis_handle_init_compl_msg() - Respond to an Init Complete Message
+ * @vscsi:	Pointer to our adapter structure
+ *
+ * Must be called with interrupt lock held.
+ */
+static long ibmvscsis_handle_init_compl_msg(struct scsi_info *vscsi)
+{
+	long rc = ADAPT_SUCCESS;
+
+	switch (vscsi->state) {
+	case NO_QUEUE:
+	case ERR_DISCONNECT:
+	case ERR_DISCONNECT_RECONNECT:
+	case ERR_DISCONNECTED:
+	case UNCONFIGURING:
+	case UNDEFINED:
+		rc = ERROR;
+		break;
+
+	case WAIT_CONNECTION:
+		vscsi->state = CONNECTED;
+		break;
+
+	case WAIT_IDLE:
+	case SRP_PROCESSING:
+	case CONNECTED:
+	case WAIT_ENABLED:
+	default:
+		rc = ERROR;
+		dev_err(&vscsi->dev, "init_msg: invalid state %d to get init compl msg\n",
+			vscsi->state);
+		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
+		break;
+	}
+
+	return rc;
+}
+
+/**
+ * ibmvscsis_handle_init_msg() - Respond to an Init Message
+ * @vscsi:	Pointer to our adapter structure
+ *
+ * Must be called with interrupt lock held.
+ */
+static long ibmvscsis_handle_init_msg(struct scsi_info *vscsi)
+{
+	long rc = ADAPT_SUCCESS;
+
+	switch (vscsi->state) {
+	case WAIT_CONNECTION:
+		rc = ibmvscsis_send_init_message(vscsi, INIT_COMPLETE_MSG);
+		switch (rc) {
+		case H_SUCCESS:
+			vscsi->state = CONNECTED;
+			break;
+
+		case H_PARAMETER:
+			dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
+				rc);
+			ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0);
+			break;
+
+		case H_DROPPED:
+			dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
+				rc);
+			rc = ERROR;
+			ibmvscsis_post_disconnect(vscsi,
+						  ERR_DISCONNECT_RECONNECT, 0);
+			break;
+
+		case H_CLOSED:
+			pr_warn("init_msg: failed to send, rc %ld\n", rc);
+			rc = 0;
+			break;
+		}
+		break;
+
+	case UNDEFINED:
+		rc = ERROR;
+		break;
+
+	case UNCONFIGURING:
+		break;
+
+	case WAIT_ENABLED:
+	case CONNECTED:
+	case SRP_PROCESSING:
+	case WAIT_IDLE:
+	case NO_QUEUE:
+	case ERR_DISCONNECT:
+	case ERR_DISCONNECT_RECONNECT:
+	case ERR_DISCONNECTED:
+	default:
+		rc = ERROR;
+		dev_err(&vscsi->dev, "init_msg: invalid state %d to get init msg\n",
+			vscsi->state);
+		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
+		break;
+	}
+
+	return rc;
+}
+
+/**
+ * ibmvscsis_init_msg() - Respond to an init message
+ * @vscsi:	Pointer to our adapter structure
+ * @crq:	Pointer to CRQ element containing the Init Message
+ *
+ * EXECUTION ENVIRONMENT:
+ *	Interrupt, interrupt lock held
+ */
+static long ibmvscsis_init_msg(struct scsi_info *vscsi, struct viosrp_crq *crq)
+{
+	long rc = ADAPT_SUCCESS;
+
+	pr_debug("init_msg: state 0x%hx\n", vscsi->state);
+
+	rc = h_vioctl(vscsi->dds.unit_id, H_GET_PARTNER_INFO,
+		      (u64)vscsi->map_ioba | ((u64)PAGE_SIZE << 32), 0, 0, 0,
+		      0);
+	if (rc == H_SUCCESS) {
+		vscsi->client_data.partition_number =
+			be64_to_cpu(*(u64 *)vscsi->map_buf);
+		pr_debug("init_msg, part num %d\n",
+			 vscsi->client_data.partition_number);
+	} else {
+		pr_debug("init_msg h_vioctl rc %ld\n", rc);
+		rc = ADAPT_SUCCESS;
+	}
+
+	if (crq->format == INIT_MSG) {
+		rc = ibmvscsis_handle_init_msg(vscsi);
+	} else if (crq->format == INIT_COMPLETE_MSG) {
+		rc = ibmvscsis_handle_init_compl_msg(vscsi);
+	} else {
+		rc = ERROR;
+		dev_err(&vscsi->dev, "init_msg: invalid format %d\n",
+			(uint)crq->format);
+		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
+	}
+
+	return rc;
+}
+
+/**
+ * ibmvscsis_establish_new_q() - Establish new CRQ queue
+ * @vscsi:	Pointer to our adapter structure
+ *
+ * Must be called with interrupt lock held.
+ */
+static long ibmvscsis_establish_new_q(struct scsi_info *vscsi)
+{
+	long rc = ADAPT_SUCCESS;
+	uint format;
+
+	vscsi->flags &= PRESERVE_FLAG_FIELDS;
+	vscsi->rsp_q_timer.timer_pops = 0;
+	vscsi->debit = 0;
+	vscsi->credit = 0;
+
+	rc = vio_enable_interrupts(vscsi->dma_dev);
+	if (rc) {
+		pr_warn("establish_new_q: failed to enable interrupts, rc %ld\n",
+			rc);
+		return rc;
+	}
+
+	rc = ibmvscsis_check_init_msg(vscsi, &format);
+	if (rc) {
+		dev_err(&vscsi->dev, "establish_new_q: check_init_msg failed, rc %ld\n",
+			rc);
+		return rc;
+	}
+
+	if (format == UNUSED_FORMAT) {
+		rc = ibmvscsis_send_init_message(vscsi, INIT_MSG);
+		switch (rc) {
+		case H_SUCCESS:
+		case H_DROPPED:
+		case H_CLOSED:
+			rc = ADAPT_SUCCESS;
+			break;
+
+		case H_PARAMETER:
+		case H_HARDWARE:
+			break;
+
+		default:
+			vscsi->state = UNDEFINED;
+			rc = H_HARDWARE;
+			break;
+		}
+	} else if (format == INIT_MSG) {
+		rc = ibmvscsis_handle_init_msg(vscsi);
+	}
+
+	return rc;
+}
+
+/**
+ * ibmvscsis_reset_queue() - Reset CRQ Queue
+ * @vscsi:	Pointer to our adapter structure
+ *
+ * This function calls h_free_q and then calls h_reg_q and does all
+ * of the bookkeeping to get us back to where we can communicate.
+ *
+ * Actually, we don't always call h_free_crq.  A problem was discovered
+ * where one partition would close and reopen his queue, which would
+ * cause his partner to get a transport event, which would cause him to
+ * close and reopen his queue, which would cause the original partition
+ * to get a transport event, etc., etc.  To prevent this, we don't
+ * actually close our queue if the client initiated the reset, (i.e.
+ * either we got a transport event or we have detected that the client's
+ * queue is gone)
+ *
+ * EXECUTION ENVIRONMENT:
+ *	Process environment, called with interrupt lock held
+ */
+static void ibmvscsis_reset_queue(struct scsi_info *vscsi)
+{
+	int bytes;
+	long rc = ADAPT_SUCCESS;
+
+	pr_debug("reset_queue: flags 0x%x\n", vscsi->flags);
+
+	/* don't reset, the client did it for us */
+	if (vscsi->flags & (CLIENT_FAILED | TRANS_EVENT)) {
+		vscsi->flags &= PRESERVE_FLAG_FIELDS;
+		vscsi->rsp_q_timer.timer_pops = 0;
+		vscsi->debit = 0;
+		vscsi->credit = 0;
+		vscsi->state = WAIT_CONNECTION;
+		vio_enable_interrupts(vscsi->dma_dev);
+	} else {
+		rc = ibmvscsis_free_command_q(vscsi);
+		if (rc == ADAPT_SUCCESS) {
+			vscsi->state = WAIT_CONNECTION;
+
+			bytes = vscsi->cmd_q.size * PAGE_SIZE;
+			rc = h_reg_crq(vscsi->dds.unit_id,
+				       vscsi->cmd_q.crq_token, bytes);
+			if (rc == H_CLOSED || rc == H_SUCCESS) {
+				rc = ibmvscsis_establish_new_q(vscsi);
+			}
+
+			if (rc != ADAPT_SUCCESS) {
+				pr_debug("reset_queue: reg_crq rc %ld\n", rc);
+
+				vscsi->state = ERR_DISCONNECTED;
+				vscsi->flags |= RESPONSE_Q_DOWN;
+				ibmvscsis_free_command_q(vscsi);
+			}
+		} else {
+			vscsi->state = ERR_DISCONNECTED;
+			vscsi->flags |= RESPONSE_Q_DOWN;
+		}
+	}
+}
+
+/**
+ * ibmvscsis_free_cmd_resources() - Free command resources
+ * @vscsi:	Pointer to our adapter structure
+ * @cmd:	Command which is not longer in use
+ *
+ * Must be called with interrupt lock held.
+ */
+static void ibmvscsis_free_cmd_resources(struct scsi_info *vscsi,
+					 struct ibmvscsis_cmd *cmd)
+{
+	struct iu_entry *iue = cmd->iue;
+
+	switch (cmd->type) {
+	case TASK_MANAGEMENT:
+	case SCSI_CDB:
+		/*
+		 * When the queue goes down this value is cleared, so it
+		 * cannot be cleared in this general purpose function.
+		 */
+		if (vscsi->debit)
+			vscsi->debit -= 1;
+		break;
+	case ADAPTER_MAD:
+		vscsi->flags &= ~PROCESSING_MAD;
+		break;
+	case UNSET_TYPE:
+		break;
+	default:
+		dev_err(&vscsi->dev, "free_cmd_resources unknown type %d\n",
+			cmd->type);
+		break;
+	}
+
+	cmd->iue = NULL;
+	list_add_tail(&cmd->list, &vscsi->free_cmd);
+	srp_iu_put(iue);
+
+	if (list_empty(&vscsi->active_q) && list_empty(&vscsi->schedule_q) &&
+	    list_empty(&vscsi->waiting_rsp) && (vscsi->flags & WAIT_FOR_IDLE)) {
+		vscsi->flags &= ~WAIT_FOR_IDLE;
+		complete(&vscsi->wait_idle);
+	}
+}
+
+/**
  * ibmvscsis_trans_event() - Handle a Transport Event
  * @vscsi:	Pointer to our adapter structure
  * @crq:	Pointer to CRQ entry containing the Transport Event
@@ -863,10 +1006,6 @@ static long ibmvscsis_trans_event(struct scsi_info *vscsi,
 						   TRANS_EVENT));
 			break;
 
-		case PART_UP_WAIT_ENAB:
-			vscsi->state = WAIT_ENABLED;
-			break;
-
 		case SRP_PROCESSING:
 			if ((vscsi->debit > 0) ||
 			    !list_empty(&vscsi->schedule_q) ||
@@ -895,7 +1034,7 @@ static long ibmvscsis_trans_event(struct scsi_info *vscsi,
 		}
 	}
 
-	rc =  vscsi->flags & SCHEDULE_DISCONNECT;
+	rc = vscsi->flags & SCHEDULE_DISCONNECT;
 
 	pr_debug("Leaving trans_event: flags 0x%x, state 0x%hx, rc %ld\n",
 		 vscsi->flags, vscsi->state, rc);
@@ -1066,16 +1205,28 @@ static void ibmvscsis_adapter_idle(struct scsi_info *vscsi)
 		free_qs = true;
 
 	switch (vscsi->state) {
+	case UNCONFIGURING:
+		ibmvscsis_free_command_q(vscsi);
+		dma_rmb();
+		isync();
+		if (vscsi->flags & CFG_SLEEPING) {
+			vscsi->flags &= ~CFG_SLEEPING;
+			complete(&vscsi->unconfig);
+		}
+		break;
 	case ERR_DISCONNECT_RECONNECT:
-		ibmvscsis_reset_queue(vscsi, WAIT_CONNECTION);
+		ibmvscsis_reset_queue(vscsi);
 		pr_debug("adapter_idle, disc_rec: flags 0x%x\n", vscsi->flags);
 		break;
 
 	case ERR_DISCONNECT:
 		ibmvscsis_free_command_q(vscsi);
-		vscsi->flags &= ~DISCONNECT_SCHEDULED;
+		vscsi->flags &= ~(SCHEDULE_DISCONNECT | DISCONNECT_SCHEDULED);
 		vscsi->flags |= RESPONSE_Q_DOWN;
-		vscsi->state = ERR_DISCONNECTED;
+		if (vscsi->tport.enabled)
+			vscsi->state = ERR_DISCONNECTED;
+		else
+			vscsi->state = WAIT_ENABLED;
 		pr_debug("adapter_idle, disc: flags 0x%x, state 0x%hx\n",
 			 vscsi->flags, vscsi->state);
 		break;
@@ -1220,7 +1371,7 @@ static long ibmvscsis_copy_crq_packet(struct scsi_info *vscsi,
  * @iue:	Information Unit containing the Adapter Info MAD request
  *
  * EXECUTION ENVIRONMENT:
- *	Interrupt adpater lock is held
+ *	Interrupt adapter lock is held
  */
 static long ibmvscsis_adapter_info(struct scsi_info *vscsi,
 				   struct iu_entry *iue)
@@ -1620,8 +1771,8 @@ static void ibmvscsis_send_messages(struct scsi_info *vscsi)
 					be64_to_cpu(msg_hi),
 					be64_to_cpu(cmd->rsp.tag));
 
-			pr_debug("send_messages: tag 0x%llx, rc %ld\n",
-				 be64_to_cpu(cmd->rsp.tag), rc);
+			pr_debug("send_messages: cmd %p, tag 0x%llx, rc %ld\n",
+				 cmd, be64_to_cpu(cmd->rsp.tag), rc);
 
 			/* if all ok free up the command element resources */
 			if (rc == H_SUCCESS) {
@@ -1691,7 +1842,7 @@ static void ibmvscsis_send_mad_resp(struct scsi_info *vscsi,
  * @crq:	Pointer to the CRQ entry containing the MAD request
  *
  * EXECUTION ENVIRONMENT:
- *	Interrupt  called with adapter lock held
+ *	Interrupt, called with adapter lock held
  */
 static long ibmvscsis_mad(struct scsi_info *vscsi, struct viosrp_crq *crq)
 {
@@ -1745,14 +1896,7 @@ static long ibmvscsis_mad(struct scsi_info *vscsi, struct viosrp_crq *crq)
 
 		pr_debug("mad: type %d\n", be32_to_cpu(mad->type));
 
-		if (be16_to_cpu(mad->length) < 0) {
-			dev_err(&vscsi->dev, "mad: length is < 0\n");
-			ibmvscsis_post_disconnect(vscsi,
-						  ERR_DISCONNECT_RECONNECT, 0);
-			rc = SRP_VIOLATION;
-		} else {
-			rc = ibmvscsis_process_mad(vscsi, iue);
-		}
+		rc = ibmvscsis_process_mad(vscsi, iue);
 
 		pr_debug("mad: status %hd, rc %ld\n", be16_to_cpu(mad->status),
 			 rc);
@@ -1864,7 +2008,7 @@ static long ibmvscsis_srp_login_rej(struct scsi_info *vscsi,
 		break;
 	case H_PERMISSION:
 		if (connection_broken(vscsi))
-			flag_bits =  RESPONSE_Q_DOWN | CLIENT_FAILED;
+			flag_bits = RESPONSE_Q_DOWN | CLIENT_FAILED;
 		dev_err(&vscsi->dev, "login_rej: error copying to client, rc %ld\n",
 			rc);
 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
@@ -2187,156 +2331,6 @@ static long ibmvscsis_ping_response(struct scsi_info *vscsi)
 }
 
 /**
- * ibmvscsis_handle_init_compl_msg() - Respond to an Init Complete Message
- * @vscsi:	Pointer to our adapter structure
- *
- * Must be called with interrupt lock held.
- */
-static long ibmvscsis_handle_init_compl_msg(struct scsi_info *vscsi)
-{
-	long rc = ADAPT_SUCCESS;
-
-	switch (vscsi->state) {
-	case NO_QUEUE:
-	case ERR_DISCONNECT:
-	case ERR_DISCONNECT_RECONNECT:
-	case ERR_DISCONNECTED:
-	case UNCONFIGURING:
-	case UNDEFINED:
-		rc = ERROR;
-		break;
-
-	case WAIT_CONNECTION:
-		vscsi->state = CONNECTED;
-		break;
-
-	case WAIT_IDLE:
-	case SRP_PROCESSING:
-	case CONNECTED:
-	case WAIT_ENABLED:
-	case PART_UP_WAIT_ENAB:
-	default:
-		rc = ERROR;
-		dev_err(&vscsi->dev, "init_msg: invalid state %d to get init compl msg\n",
-			vscsi->state);
-		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
-		break;
-	}
-
-	return rc;
-}
-
-/**
- * ibmvscsis_handle_init_msg() - Respond to an Init Message
- * @vscsi:	Pointer to our adapter structure
- *
- * Must be called with interrupt lock held.
- */
-static long ibmvscsis_handle_init_msg(struct scsi_info *vscsi)
-{
-	long rc = ADAPT_SUCCESS;
-
-	switch (vscsi->state) {
-	case WAIT_ENABLED:
-		vscsi->state = PART_UP_WAIT_ENAB;
-		break;
-
-	case WAIT_CONNECTION:
-		rc = ibmvscsis_send_init_message(vscsi, INIT_COMPLETE_MSG);
-		switch (rc) {
-		case H_SUCCESS:
-			vscsi->state = CONNECTED;
-			break;
-
-		case H_PARAMETER:
-			dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
-				rc);
-			ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0);
-			break;
-
-		case H_DROPPED:
-			dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
-				rc);
-			rc = ERROR;
-			ibmvscsis_post_disconnect(vscsi,
-						  ERR_DISCONNECT_RECONNECT, 0);
-			break;
-
-		case H_CLOSED:
-			pr_warn("init_msg: failed to send, rc %ld\n", rc);
-			rc = 0;
-			break;
-		}
-		break;
-
-	case UNDEFINED:
-		rc = ERROR;
-		break;
-
-	case UNCONFIGURING:
-		break;
-
-	case PART_UP_WAIT_ENAB:
-	case CONNECTED:
-	case SRP_PROCESSING:
-	case WAIT_IDLE:
-	case NO_QUEUE:
-	case ERR_DISCONNECT:
-	case ERR_DISCONNECT_RECONNECT:
-	case ERR_DISCONNECTED:
-	default:
-		rc = ERROR;
-		dev_err(&vscsi->dev, "init_msg: invalid state %d to get init msg\n",
-			vscsi->state);
-		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
-		break;
-	}
-
-	return rc;
-}
-
-/**
- * ibmvscsis_init_msg() - Respond to an init message
- * @vscsi:	Pointer to our adapter structure
- * @crq:	Pointer to CRQ element containing the Init Message
- *
- * EXECUTION ENVIRONMENT:
- *	Interrupt, interrupt lock held
- */
-static long ibmvscsis_init_msg(struct scsi_info *vscsi, struct viosrp_crq *crq)
-{
-	long rc = ADAPT_SUCCESS;
-
-	pr_debug("init_msg: state 0x%hx\n", vscsi->state);
-
-	rc = h_vioctl(vscsi->dds.unit_id, H_GET_PARTNER_INFO,
-		      (u64)vscsi->map_ioba | ((u64)PAGE_SIZE << 32), 0, 0, 0,
-		      0);
-	if (rc == H_SUCCESS) {
-		vscsi->client_data.partition_number =
-			be64_to_cpu(*(u64 *)vscsi->map_buf);
-		pr_debug("init_msg, part num %d\n",
-			 vscsi->client_data.partition_number);
-	} else {
-		pr_debug("init_msg h_vioctl rc %ld\n", rc);
-		rc = ADAPT_SUCCESS;
-	}
-
-	if (crq->format == INIT_MSG) {
-		rc = ibmvscsis_handle_init_msg(vscsi);
-	} else if (crq->format == INIT_COMPLETE_MSG) {
-		rc = ibmvscsis_handle_init_compl_msg(vscsi);
-	} else {
-		rc = ERROR;
-		dev_err(&vscsi->dev, "init_msg: invalid format %d\n",
-			(uint)crq->format);
-		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
-	}
-
-	return rc;
-}
-
-/**
  * ibmvscsis_parse_command() - Parse an element taken from the cmd rsp queue.
  * @vscsi:	Pointer to our adapter structure
  * @crq:	Pointer to CRQ element containing the SRP request
@@ -2391,7 +2385,7 @@ static long ibmvscsis_parse_command(struct scsi_info *vscsi,
 		break;
 
 	case VALID_TRANS_EVENT:
-		rc =  ibmvscsis_trans_event(vscsi, crq);
+		rc = ibmvscsis_trans_event(vscsi, crq);
 		break;
 
 	case VALID_INIT_MSG:
@@ -2522,7 +2516,6 @@ static void ibmvscsis_parse_cmd(struct scsi_info *vscsi,
 		dev_err(&vscsi->dev, "0x%llx: parsing SRP descriptor table failed.\n",
 			srp->tag);
 		goto fail;
-		return;
 	}
 
 	cmd->rsp.sol_not = srp->sol_not;
@@ -2559,6 +2552,10 @@ static void ibmvscsis_parse_cmd(struct scsi_info *vscsi,
 			       data_len, attr, dir, 0);
 	if (rc) {
 		dev_err(&vscsi->dev, "target_submit_cmd failed, rc %d\n", rc);
+		spin_lock_bh(&vscsi->intr_lock);
+		list_del(&cmd->list);
+		ibmvscsis_free_cmd_resources(vscsi, cmd);
+		spin_unlock_bh(&vscsi->intr_lock);
 		goto fail;
 	}
 	return;
@@ -2638,6 +2635,9 @@ static void ibmvscsis_parse_task(struct scsi_info *vscsi,
 		if (rc) {
 			dev_err(&vscsi->dev, "target_submit_tmr failed, rc %d\n",
 				rc);
+			spin_lock_bh(&vscsi->intr_lock);
+			list_del(&cmd->list);
+			spin_unlock_bh(&vscsi->intr_lock);
 			cmd->se_cmd.se_tmr_req->response =
 				TMR_FUNCTION_REJECTED;
 		}
@@ -2786,36 +2786,6 @@ static irqreturn_t ibmvscsis_interrupt(int dummy, void *data)
 }
 
 /**
- * ibmvscsis_check_q() - Helper function to Check Init Message Valid
- * @vscsi:	Pointer to our adapter structure
- *
- * Checks if a initialize message was queued by the initiatior
- * while the timing window was open.  This function is called from
- * probe after the CRQ is created and interrupts are enabled.
- * It would only be used by adapters who wait for some event before
- * completing the init handshake with the client.  For ibmvscsi, this
- * event is waiting for the port to be enabled.
- *
- * EXECUTION ENVIRONMENT:
- *	Process level only, interrupt lock held
- */
-static long ibmvscsis_check_q(struct scsi_info *vscsi)
-{
-	uint format;
-	long rc;
-
-	rc = ibmvscsis_check_init_msg(vscsi, &format);
-	if (rc)
-		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
-	else if (format == UNUSED_FORMAT)
-		vscsi->state = WAIT_ENABLED;
-	else
-		vscsi->state = PART_UP_WAIT_ENAB;
-
-	return rc;
-}
-
-/**
  * ibmvscsis_enable_change_state() - Set new state based on enabled status
  * @vscsi:	Pointer to our adapter structure
  *
@@ -2826,77 +2796,19 @@ static long ibmvscsis_check_q(struct scsi_info *vscsi)
  */
 static long ibmvscsis_enable_change_state(struct scsi_info *vscsi)
 {
+	int bytes;
 	long rc = ADAPT_SUCCESS;
 
-handle_state_change:
-	switch (vscsi->state) {
-	case WAIT_ENABLED:
-		rc = ibmvscsis_send_init_message(vscsi, INIT_MSG);
-		switch (rc) {
-		case H_SUCCESS:
-		case H_DROPPED:
-		case H_CLOSED:
-			vscsi->state =  WAIT_CONNECTION;
-			rc = ADAPT_SUCCESS;
-			break;
-
-		case H_PARAMETER:
-			break;
-
-		case H_HARDWARE:
-			break;
-
-		default:
-			vscsi->state = UNDEFINED;
-			rc = H_HARDWARE;
-			break;
-		}
-		break;
-	case PART_UP_WAIT_ENAB:
-		rc = ibmvscsis_send_init_message(vscsi, INIT_COMPLETE_MSG);
-		switch (rc) {
-		case H_SUCCESS:
-			vscsi->state = CONNECTED;
-			rc = ADAPT_SUCCESS;
-			break;
-
-		case H_DROPPED:
-		case H_CLOSED:
-			vscsi->state = WAIT_ENABLED;
-			goto handle_state_change;
-
-		case H_PARAMETER:
-			break;
-
-		case H_HARDWARE:
-			break;
-
-		default:
-			rc = H_HARDWARE;
-			break;
-		}
-		break;
-
-	case WAIT_CONNECTION:
-	case WAIT_IDLE:
-	case SRP_PROCESSING:
-	case CONNECTED:
-		rc = ADAPT_SUCCESS;
-		break;
-		/* should not be able to get here */
-	case UNCONFIGURING:
-		rc = ERROR;
-		vscsi->state = UNDEFINED;
-		break;
+	bytes = vscsi->cmd_q.size * PAGE_SIZE;
+	rc = h_reg_crq(vscsi->dds.unit_id, vscsi->cmd_q.crq_token, bytes);
+	if (rc == H_CLOSED || rc == H_SUCCESS) {
+		vscsi->state = WAIT_CONNECTION;
+		rc = ibmvscsis_establish_new_q(vscsi);
+	}
 
-		/* driver should never allow this to happen */
-	case ERR_DISCONNECT:
-	case ERR_DISCONNECT_RECONNECT:
-	default:
-		dev_err(&vscsi->dev, "in invalid state %d during enable_change_state\n",
-			vscsi->state);
-		rc = ADAPT_SUCCESS;
-		break;
+	if (rc != ADAPT_SUCCESS) {
+		vscsi->state = ERR_DISCONNECTED;
+		vscsi->flags |= RESPONSE_Q_DOWN;
 	}
 
 	return rc;
@@ -2916,7 +2828,6 @@ handle_state_change:
  */
 static long ibmvscsis_create_command_q(struct scsi_info *vscsi, int num_cmds)
 {
-	long rc = 0;
 	int pages;
 	struct vio_dev *vdev = vscsi->dma_dev;
 
@@ -2940,22 +2851,7 @@ static long ibmvscsis_create_command_q(struct scsi_info *vscsi, int num_cmds)
 		return -ENOMEM;
 	}
 
-	rc =  h_reg_crq(vscsi->dds.unit_id, vscsi->cmd_q.crq_token, PAGE_SIZE);
-	if (rc) {
-		if (rc == H_CLOSED) {
-			vscsi->state = WAIT_ENABLED;
-			rc = 0;
-		} else {
-			dma_unmap_single(&vdev->dev, vscsi->cmd_q.crq_token,
-					 PAGE_SIZE, DMA_BIDIRECTIONAL);
-			free_page((unsigned long)vscsi->cmd_q.base_addr);
-			rc = -ENODEV;
-		}
-	} else {
-		vscsi->state = WAIT_ENABLED;
-	}
-
-	return rc;
+	return 0;
 }
 
 /**
@@ -3270,7 +3166,7 @@ static void ibmvscsis_handle_crq(unsigned long data)
 	/*
 	 * if we are in a path where we are waiting for all pending commands
 	 * to complete because we received a transport event and anything in
-	 * the command queue is for a new connection,  do nothing
+	 * the command queue is for a new connection, do nothing
 	 */
 	if (TARGET_STOP(vscsi)) {
 		vio_enable_interrupts(vscsi->dma_dev);
@@ -3314,7 +3210,7 @@ cmd_work:
 				 * everything but transport events on the queue
 				 *
 				 * need to decrement the queue index so we can
-				 * look at the elment again
+				 * look at the element again
 				 */
 				if (vscsi->cmd_q.index)
 					vscsi->cmd_q.index -= 1;
@@ -3378,7 +3274,8 @@ static int ibmvscsis_probe(struct vio_dev *vdev,
 	INIT_LIST_HEAD(&vscsi->waiting_rsp);
 	INIT_LIST_HEAD(&vscsi->active_q);
 
-	snprintf(vscsi->tport.tport_name, 256, "%s", dev_name(&vdev->dev));
+	snprintf(vscsi->tport.tport_name, IBMVSCSIS_NAMELEN, "%s",
+		 dev_name(&vdev->dev));
 
 	pr_debug("probe tport_name: %s\n", vscsi->tport.tport_name);
 
@@ -3393,6 +3290,9 @@ static int ibmvscsis_probe(struct vio_dev *vdev,
 	strncat(vscsi->eye, vdev->name, MAX_EYE);
 
 	vscsi->dds.unit_id = vdev->unit_address;
+	strncpy(vscsi->dds.partition_name, partition_name,
+		sizeof(vscsi->dds.partition_name));
+	vscsi->dds.partition_num = partition_number;
 
 	spin_lock_bh(&ibmvscsis_dev_lock);
 	list_add_tail(&vscsi->list, &ibmvscsis_dev_list);
@@ -3469,6 +3369,7 @@ static int ibmvscsis_probe(struct vio_dev *vdev,
 		     (unsigned long)vscsi);
 
 	init_completion(&vscsi->wait_idle);
+	init_completion(&vscsi->unconfig);
 
 	snprintf(wq_name, 24, "ibmvscsis%s", dev_name(&vdev->dev));
 	vscsi->work_q = create_workqueue(wq_name);
@@ -3485,31 +3386,12 @@ static int ibmvscsis_probe(struct vio_dev *vdev,
 		goto destroy_WQ;
 	}
 
-	spin_lock_bh(&vscsi->intr_lock);
-	vio_enable_interrupts(vdev);
-	if (rc) {
-		dev_err(&vscsi->dev, "enabling interrupts failed, rc %d\n", rc);
-		rc = -ENODEV;
-		spin_unlock_bh(&vscsi->intr_lock);
-		goto free_irq;
-	}
-
-	if (ibmvscsis_check_q(vscsi)) {
-		rc = ERROR;
-		dev_err(&vscsi->dev, "probe: check_q failed, rc %d\n", rc);
-		spin_unlock_bh(&vscsi->intr_lock);
-		goto disable_interrupt;
-	}
-	spin_unlock_bh(&vscsi->intr_lock);
+	vscsi->state = WAIT_ENABLED;
 
 	dev_set_drvdata(&vdev->dev, vscsi);
 
 	return 0;
 
-disable_interrupt:
-	vio_disable_interrupts(vdev);
-free_irq:
-	free_irq(vdev->irq, vscsi);
 destroy_WQ:
 	destroy_workqueue(vscsi->work_q);
 unmap_buf:
@@ -3543,10 +3425,11 @@ static int ibmvscsis_remove(struct vio_dev *vdev)
 
 	pr_debug("remove (%s)\n", dev_name(&vscsi->dma_dev->dev));
 
-	/*
-	 * TBD: Need to handle if there are commands on the waiting_rsp q
-	 *      Actually, can there still be cmds outstanding to tcm?
-	 */
+	spin_lock_bh(&vscsi->intr_lock);
+	ibmvscsis_post_disconnect(vscsi, UNCONFIGURING, 0);
+	vscsi->flags |= CFG_SLEEPING;
+	spin_unlock_bh(&vscsi->intr_lock);
+	wait_for_completion(&vscsi->unconfig);
 
 	vio_disable_interrupts(vdev);
 	free_irq(vdev->irq, vscsi);
@@ -3555,7 +3438,6 @@ static int ibmvscsis_remove(struct vio_dev *vdev)
 			 DMA_BIDIRECTIONAL);
 	kfree(vscsi->map_buf);
 	tasklet_kill(&vscsi->work_task);
-	ibmvscsis_unregister_command_q(vscsi);
 	ibmvscsis_destroy_command_q(vscsi);
 	ibmvscsis_freetimer(vscsi);
 	ibmvscsis_free_cmds(vscsi);
@@ -3609,7 +3491,7 @@ static int ibmvscsis_get_system_info(void)
 
 	num = of_get_property(rootdn, "ibm,partition-no", NULL);
 	if (num)
-		partition_number = *num;
+		partition_number = of_read_number(num, 1);
 
 	of_node_put(rootdn);
 
@@ -3903,18 +3785,22 @@ static ssize_t ibmvscsis_tpg_enable_store(struct config_item *item,
 	}
 
 	if (tmp) {
-		tport->enabled = true;
 		spin_lock_bh(&vscsi->intr_lock);
+		tport->enabled = true;
 		lrc = ibmvscsis_enable_change_state(vscsi);
 		if (lrc)
 			pr_err("enable_change_state failed, rc %ld state %d\n",
 			       lrc, vscsi->state);
 		spin_unlock_bh(&vscsi->intr_lock);
 	} else {
+		spin_lock_bh(&vscsi->intr_lock);
 		tport->enabled = false;
+		/* This simulates the server going down */
+		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0);
+		spin_unlock_bh(&vscsi->intr_lock);
 	}
 
-	pr_debug("tpg_enable_store, state %d\n", vscsi->state);
+	pr_debug("tpg_enable_store, tmp %ld, state %d\n", tmp, vscsi->state);
 
 	return count;
 }
@@ -3983,10 +3869,10 @@ static struct attribute *ibmvscsis_dev_attrs[] = {
 ATTRIBUTE_GROUPS(ibmvscsis_dev);
 
 static struct class ibmvscsis_class = {
-	.name           = "ibmvscsis",
-	.dev_release    = ibmvscsis_dev_release,
-	.class_attrs    = ibmvscsis_class_attrs,
-	.dev_groups     = ibmvscsis_dev_groups,
+	.name		= "ibmvscsis",
+	.dev_release	= ibmvscsis_dev_release,
+	.class_attrs	= ibmvscsis_class_attrs,
+	.dev_groups	= ibmvscsis_dev_groups,
 };
 
 static struct vio_device_id ibmvscsis_device_table[] = {
diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
index 981a0c992b6c..98b0ca79a5c5 100644
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
@@ -204,8 +204,6 @@ struct scsi_info {
 	struct list_head waiting_rsp;
 #define NO_QUEUE                    0x00
 #define WAIT_ENABLED                0X01
-	/* driver has received an initialize command */
-#define PART_UP_WAIT_ENAB           0x02
 #define WAIT_CONNECTION             0x04
 	/* have established a connection */
 #define CONNECTED                   0x08
@@ -259,6 +257,8 @@ struct scsi_info {
 #define SCHEDULE_DISCONNECT           0x00400
 	/* disconnect handler is scheduled */
 #define DISCONNECT_SCHEDULED          0x00800
+	/* remove function is sleeping */
+#define CFG_SLEEPING                  0x01000
 	u32 flags;
 	/* adapter lock */
 	spinlock_t intr_lock;
@@ -287,6 +287,7 @@ struct scsi_info {
 
 	struct workqueue_struct *work_q;
 	struct completion wait_idle;
+	struct completion unconfig;
 	struct device dev;
 	struct vio_dev *dma_dev;
 	struct srp_target target;
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index 532474109624..835c59c777f2 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -186,16 +186,16 @@ static const struct ipr_chip_cfg_t ipr_chip_cfg[] = {
 };
 
 static const struct ipr_chip_t ipr_chip[] = {
-	{ PCI_VENDOR_ID_MYLEX, PCI_DEVICE_ID_IBM_GEMSTONE, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] },
-	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CITRINE, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] },
-	{ PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_OBSIDIAN, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] },
-	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_OBSIDIAN, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] },
-	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_OBSIDIAN_E, IPR_USE_MSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] },
-	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_SNIPE, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[1] },
-	{ PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_SCAMP, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[1] },
-	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_FPGA_E2, IPR_USE_MSI, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] },
-	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, IPR_USE_MSI, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] },
-	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_RATTLESNAKE, IPR_USE_MSI, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] }
+	{ PCI_VENDOR_ID_MYLEX, PCI_DEVICE_ID_IBM_GEMSTONE, false, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] },
+	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CITRINE, false, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] },
+	{ PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_OBSIDIAN, false, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] },
+	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_OBSIDIAN, false, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] },
+	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_OBSIDIAN_E, true, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[0] },
+	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_SNIPE, false, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[1] },
+	{ PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_SCAMP, false, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[1] },
+	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_FPGA_E2, true, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] },
+	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, true, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] },
+	{ PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_RATTLESNAKE, true, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] }
 };
 
 static int ipr_max_bus_speeds[] = {
@@ -9439,23 +9439,11 @@ static void ipr_free_mem(struct ipr_ioa_cfg *ioa_cfg)
 static void ipr_free_irqs(struct ipr_ioa_cfg *ioa_cfg)
 {
 	struct pci_dev *pdev = ioa_cfg->pdev;
+	int i;
 
-	if (ioa_cfg->intr_flag == IPR_USE_MSI ||
-	    ioa_cfg->intr_flag == IPR_USE_MSIX) {
-		int i;
-		for (i = 0; i < ioa_cfg->nvectors; i++)
-			free_irq(ioa_cfg->vectors_info[i].vec,
-				 &ioa_cfg->hrrq[i]);
-	} else
-		free_irq(pdev->irq, &ioa_cfg->hrrq[0]);
-
-	if (ioa_cfg->intr_flag == IPR_USE_MSI) {
-		pci_disable_msi(pdev);
-		ioa_cfg->intr_flag &= ~IPR_USE_MSI;
-	} else if (ioa_cfg->intr_flag == IPR_USE_MSIX) {
-		pci_disable_msix(pdev);
-		ioa_cfg->intr_flag &= ~IPR_USE_MSIX;
-	}
+	for (i = 0; i < ioa_cfg->nvectors; i++)
+		free_irq(pci_irq_vector(pdev, i), &ioa_cfg->hrrq[i]);
+	pci_free_irq_vectors(pdev);
 }
 
 /**
@@ -9883,45 +9871,6 @@ static void ipr_wait_for_pci_err_recovery(struct ipr_ioa_cfg *ioa_cfg)
 	}
 }
 
-static int ipr_enable_msix(struct ipr_ioa_cfg *ioa_cfg)
-{
-	struct msix_entry entries[IPR_MAX_MSIX_VECTORS];
-	int i, vectors;
-
-	for (i = 0; i < ARRAY_SIZE(entries); ++i)
-		entries[i].entry = i;
-
-	vectors = pci_enable_msix_range(ioa_cfg->pdev,
-					entries, 1, ipr_number_of_msix);
-	if (vectors < 0) {
-		ipr_wait_for_pci_err_recovery(ioa_cfg);
-		return vectors;
-	}
-
-	for (i = 0; i < vectors; i++)
-		ioa_cfg->vectors_info[i].vec = entries[i].vector;
-	ioa_cfg->nvectors = vectors;
-
-	return 0;
-}
-
-static int ipr_enable_msi(struct ipr_ioa_cfg *ioa_cfg)
-{
-	int i, vectors;
-
-	vectors = pci_enable_msi_range(ioa_cfg->pdev, 1, ipr_number_of_msix);
-	if (vectors < 0) {
-		ipr_wait_for_pci_err_recovery(ioa_cfg);
-		return vectors;
-	}
-
-	for (i = 0; i < vectors; i++)
-		ioa_cfg->vectors_info[i].vec = ioa_cfg->pdev->irq + i;
-	ioa_cfg->nvectors = vectors;
-
-	return 0;
-}
-
 static void name_msi_vectors(struct ipr_ioa_cfg *ioa_cfg)
 {
 	int vec_idx, n = sizeof(ioa_cfg->vectors_info[0].desc) - 1;
@@ -9934,19 +9883,20 @@ static void name_msi_vectors(struct ipr_ioa_cfg *ioa_cfg)
 	}
 }
 
-static int ipr_request_other_msi_irqs(struct ipr_ioa_cfg *ioa_cfg)
+static int ipr_request_other_msi_irqs(struct ipr_ioa_cfg *ioa_cfg,
+		struct pci_dev *pdev)
 {
 	int i, rc;
 
 	for (i = 1; i < ioa_cfg->nvectors; i++) {
-		rc = request_irq(ioa_cfg->vectors_info[i].vec,
+		rc = request_irq(pci_irq_vector(pdev, i),
 			ipr_isr_mhrrq,
 			0,
 			ioa_cfg->vectors_info[i].desc,
 			&ioa_cfg->hrrq[i]);
 		if (rc) {
 			while (--i >= 0)
-				free_irq(ioa_cfg->vectors_info[i].vec,
+				free_irq(pci_irq_vector(pdev, i),
 					&ioa_cfg->hrrq[i]);
 			return rc;
 		}
@@ -9984,8 +9934,7 @@ static irqreturn_t ipr_test_intr(int irq, void *devp)
  * ipr_test_msi - Test for Message Signaled Interrupt (MSI) support.
  * @pdev:		PCI device struct
  *
- * Description: The return value from pci_enable_msi_range() can not always be
- * trusted.  This routine sets up and initiates a test interrupt to determine
+ * Description: This routine sets up and initiates a test interrupt to determine
  * if the interrupt is received via the ipr_test_intr() service routine.
  * If the tests fails, the driver will fall back to LSI.
  *
@@ -9997,6 +9946,7 @@ static int ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, struct pci_dev *pdev)
 	int rc;
 	volatile u32 int_reg;
 	unsigned long lock_flags = 0;
+	int irq = pci_irq_vector(pdev, 0);
 
 	ENTER;
 
@@ -10008,15 +9958,12 @@ static int ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, struct pci_dev *pdev)
 	int_reg = readl(ioa_cfg->regs.sense_interrupt_mask_reg);
 	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
 
-	if (ioa_cfg->intr_flag == IPR_USE_MSIX)
-		rc = request_irq(ioa_cfg->vectors_info[0].vec, ipr_test_intr, 0, IPR_NAME, ioa_cfg);
-	else
-		rc = request_irq(pdev->irq, ipr_test_intr, 0, IPR_NAME, ioa_cfg);
+	rc = request_irq(irq, ipr_test_intr, 0, IPR_NAME, ioa_cfg);
 	if (rc) {
-		dev_err(&pdev->dev, "Can not assign irq %d\n", pdev->irq);
+		dev_err(&pdev->dev, "Can not assign irq %d\n", irq);
 		return rc;
 	} else if (ipr_debug)
-		dev_info(&pdev->dev, "IRQ assigned: %d\n", pdev->irq);
+		dev_info(&pdev->dev, "IRQ assigned: %d\n", irq);
 
 	writel(IPR_PCII_IO_DEBUG_ACKNOWLEDGE, ioa_cfg->regs.sense_interrupt_reg32);
 	int_reg = readl(ioa_cfg->regs.sense_interrupt_reg);
@@ -10033,10 +9980,7 @@ static int ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, struct pci_dev *pdev)
 
 	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
 
-	if (ioa_cfg->intr_flag == IPR_USE_MSIX)
-		free_irq(ioa_cfg->vectors_info[0].vec, ioa_cfg);
-	else
-		free_irq(pdev->irq, ioa_cfg);
+	free_irq(irq, ioa_cfg);
 
 	LEAVE;
 
@@ -10060,6 +10004,7 @@ static int ipr_probe_ioa(struct pci_dev *pdev,
 	int rc = PCIBIOS_SUCCESSFUL;
 	volatile u32 mask, uproc, interrupts;
 	unsigned long lock_flags, driver_lock_flags;
+	unsigned int irq_flag;
 
 	ENTER;
 
@@ -10175,18 +10120,18 @@ static int ipr_probe_ioa(struct pci_dev *pdev,
 		ipr_number_of_msix = IPR_MAX_MSIX_VECTORS;
 	}
 
-	if (ioa_cfg->ipr_chip->intr_type == IPR_USE_MSI &&
-			ipr_enable_msix(ioa_cfg) == 0)
-		ioa_cfg->intr_flag = IPR_USE_MSIX;
-	else if (ioa_cfg->ipr_chip->intr_type == IPR_USE_MSI &&
-			ipr_enable_msi(ioa_cfg) == 0)
-		ioa_cfg->intr_flag = IPR_USE_MSI;
-	else {
-		ioa_cfg->intr_flag = IPR_USE_LSI;
-		ioa_cfg->clear_isr = 1;
-		ioa_cfg->nvectors = 1;
-		dev_info(&pdev->dev, "Cannot enable MSI.\n");
+	irq_flag = PCI_IRQ_LEGACY;
+	if (ioa_cfg->ipr_chip->has_msi)
+		irq_flag |= PCI_IRQ_MSI | PCI_IRQ_MSIX;
+	rc = pci_alloc_irq_vectors(pdev, 1, ipr_number_of_msix, irq_flag);
+	if (rc < 0) {
+		ipr_wait_for_pci_err_recovery(ioa_cfg);
+		goto cleanup_nomem;
 	}
+	ioa_cfg->nvectors = rc;
+
+	if (!pdev->msi_enabled && !pdev->msix_enabled)
+		ioa_cfg->clear_isr = 1;
 
 	pci_set_master(pdev);
 
@@ -10199,33 +10144,23 @@ static int ipr_probe_ioa(struct pci_dev *pdev,
 		}
 	}
 
-	if (ioa_cfg->intr_flag == IPR_USE_MSI ||
-	    ioa_cfg->intr_flag == IPR_USE_MSIX) {
+	if (pdev->msi_enabled || pdev->msix_enabled) {
 		rc = ipr_test_msi(ioa_cfg, pdev);
-		if (rc == -EOPNOTSUPP) {
+		switch (rc) {
+		case 0:
+			dev_info(&pdev->dev,
+				"Request for %d MSI%ss succeeded.", ioa_cfg->nvectors,
+				pdev->msix_enabled ? "-X" : "");
+			break;
+		case -EOPNOTSUPP:
 			ipr_wait_for_pci_err_recovery(ioa_cfg);
-			if (ioa_cfg->intr_flag == IPR_USE_MSI) {
-				ioa_cfg->intr_flag &= ~IPR_USE_MSI;
-				pci_disable_msi(pdev);
-			 } else if (ioa_cfg->intr_flag == IPR_USE_MSIX) {
-				ioa_cfg->intr_flag &= ~IPR_USE_MSIX;
-				pci_disable_msix(pdev);
-			}
+			pci_free_irq_vectors(pdev);
 
-			ioa_cfg->intr_flag = IPR_USE_LSI;
 			ioa_cfg->nvectors = 1;
-		}
-		else if (rc)
+			ioa_cfg->clear_isr = 1;
+			break;
+		default:
 			goto out_msi_disable;
-		else {
-			if (ioa_cfg->intr_flag == IPR_USE_MSI)
-				dev_info(&pdev->dev,
-					"Request for %d MSIs succeeded with starting IRQ: %d\n",
-					ioa_cfg->nvectors, pdev->irq);
-			else if (ioa_cfg->intr_flag == IPR_USE_MSIX)
-				dev_info(&pdev->dev,
-					"Request for %d MSIXs succeeded.",
-					ioa_cfg->nvectors);
 		}
 	}
 
@@ -10273,15 +10208,13 @@ static int ipr_probe_ioa(struct pci_dev *pdev,
 	ipr_mask_and_clear_interrupts(ioa_cfg, ~IPR_PCII_IOA_TRANS_TO_OPER);
 	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
 
-	if (ioa_cfg->intr_flag == IPR_USE_MSI
-			|| ioa_cfg->intr_flag == IPR_USE_MSIX) {
+	if (pdev->msi_enabled || pdev->msix_enabled) {
 		name_msi_vectors(ioa_cfg);
-		rc = request_irq(ioa_cfg->vectors_info[0].vec, ipr_isr,
-			0,
+		rc = request_irq(pci_irq_vector(pdev, 0), ipr_isr, 0,
 			ioa_cfg->vectors_info[0].desc,
 			&ioa_cfg->hrrq[0]);
 		if (!rc)
-			rc = ipr_request_other_msi_irqs(ioa_cfg);
+			rc = ipr_request_other_msi_irqs(ioa_cfg, pdev);
 	} else {
 		rc = request_irq(pdev->irq, ipr_isr,
 			 IRQF_SHARED,
@@ -10323,10 +10256,7 @@ cleanup_nolog:
 	ipr_free_mem(ioa_cfg);
 out_msi_disable:
 	ipr_wait_for_pci_err_recovery(ioa_cfg);
-	if (ioa_cfg->intr_flag == IPR_USE_MSI)
-		pci_disable_msi(pdev);
-	else if (ioa_cfg->intr_flag == IPR_USE_MSIX)
-		pci_disable_msix(pdev);
+	pci_free_irq_vectors(pdev);
 cleanup_nomem:
 	iounmap(ipr_regs);
 out_disable:
diff --git a/drivers/scsi/ipr.h b/drivers/scsi/ipr.h
index 8995053d01b3..b7d2e98eb45b 100644
--- a/drivers/scsi/ipr.h
+++ b/drivers/scsi/ipr.h
@@ -1413,10 +1413,7 @@ struct ipr_chip_cfg_t {
 struct ipr_chip_t {
 	u16 vendor;
 	u16 device;
-	u16 intr_type;
-#define IPR_USE_LSI			0x00
-#define IPR_USE_MSI			0x01
-#define IPR_USE_MSIX			0x02
+	bool has_msi;
 	u16 sis_type;
 #define IPR_SIS32			0x00
 #define IPR_SIS64			0x01
@@ -1593,11 +1590,9 @@ struct ipr_ioa_cfg {
 	struct ipr_cmnd **ipr_cmnd_list;
 	dma_addr_t *ipr_cmnd_list_dma;
 
-	u16 intr_flag;
 	unsigned int nvectors;
 
 	struct {
-		unsigned short vec;
 		char desc[22];
 	} vectors_info[IPR_MAX_MSIX_VECTORS];
 
diff --git a/drivers/scsi/ips.c b/drivers/scsi/ips.c
index 02cb76fd4420..3419e1bcdff6 100644
--- a/drivers/scsi/ips.c
+++ b/drivers/scsi/ips.c
@@ -2241,9 +2241,6 @@ ips_get_bios_version(ips_ha_t * ha, int intr)
 	uint8_t minor;
 	uint8_t subminor;
 	uint8_t *buffer;
-	char hexDigits[] =
-	    { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C',
-     'D', 'E', 'F' };
 
 	METHOD_TRACE("ips_get_bios_version", 1);
 
@@ -2374,13 +2371,13 @@ ips_get_bios_version(ips_ha_t * ha, int intr)
 		}
 	}
 
-	ha->bios_version[0] = hexDigits[(major & 0xF0) >> 4];
+	ha->bios_version[0] = hex_asc_upper_hi(major);
 	ha->bios_version[1] = '.';
-	ha->bios_version[2] = hexDigits[major & 0x0F];
-	ha->bios_version[3] = hexDigits[subminor];
+	ha->bios_version[2] = hex_asc_upper_lo(major);
+	ha->bios_version[3] = hex_asc_upper_lo(subminor);
 	ha->bios_version[4] = '.';
-	ha->bios_version[5] = hexDigits[(minor & 0xF0) >> 4];
-	ha->bios_version[6] = hexDigits[minor & 0x0F];
+	ha->bios_version[5] = hex_asc_upper_hi(minor);
+	ha->bios_version[6] = hex_asc_upper_lo(minor);
 	ha->bios_version[7] = 0;
 }
 
diff --git a/drivers/scsi/isci/host.h b/drivers/scsi/isci/host.h
index 22a9bb1abae1..b3539928073c 100644
--- a/drivers/scsi/isci/host.h
+++ b/drivers/scsi/isci/host.h
@@ -295,7 +295,6 @@ enum sci_controller_states {
 #define SCI_MAX_MSIX_INT (SCI_NUM_MSI_X_INT*SCI_MAX_CONTROLLERS)
 
 struct isci_pci_info {
-	struct msix_entry msix_entries[SCI_MAX_MSIX_INT];
 	struct isci_host *hosts[SCI_MAX_CONTROLLERS];
 	struct isci_orom *orom;
 };
diff --git a/drivers/scsi/isci/init.c b/drivers/scsi/isci/init.c
index 77128d680e3b..0b5b5db0d0f8 100644
--- a/drivers/scsi/isci/init.c
+++ b/drivers/scsi/isci/init.c
@@ -350,16 +350,12 @@ static int isci_setup_interrupts(struct pci_dev *pdev)
 	 */
 	num_msix = num_controllers(pdev) * SCI_NUM_MSI_X_INT;
 
-	for (i = 0; i < num_msix; i++)
-		pci_info->msix_entries[i].entry = i;
-
-	err = pci_enable_msix_exact(pdev, pci_info->msix_entries, num_msix);
-	if (err)
+	err = pci_alloc_irq_vectors(pdev, num_msix, num_msix, PCI_IRQ_MSIX);
+	if (err < 0)
 		goto intx;
 
 	for (i = 0; i < num_msix; i++) {
 		int id = i / SCI_NUM_MSI_X_INT;
-		struct msix_entry *msix = &pci_info->msix_entries[i];
 		irq_handler_t isr;
 
 		ihost = pci_info->hosts[id];
@@ -369,8 +365,8 @@ static int isci_setup_interrupts(struct pci_dev *pdev)
 		else
 			isr = isci_msix_isr;
 
-		err = devm_request_irq(&pdev->dev, msix->vector, isr, 0,
-				       DRV_NAME"-msix", ihost);
+		err = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i),
+				isr, 0, DRV_NAME"-msix", ihost);
 		if (!err)
 			continue;
 
@@ -378,18 +374,19 @@ static int isci_setup_interrupts(struct pci_dev *pdev)
 		while (i--) {
 			id = i / SCI_NUM_MSI_X_INT;
 			ihost = pci_info->hosts[id];
-			msix = &pci_info->msix_entries[i];
-			devm_free_irq(&pdev->dev, msix->vector, ihost);
+			devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i),
+					ihost);
 		}
-		pci_disable_msix(pdev);
+		pci_free_irq_vectors(pdev);
 		goto intx;
 	}
 	return 0;
 
  intx:
 	for_each_isci_host(i, ihost, pdev) {
-		err = devm_request_irq(&pdev->dev, pdev->irq, isci_intx_isr,
-				       IRQF_SHARED, DRV_NAME"-intx", ihost);
+		err = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, 0),
+				isci_intx_isr, IRQF_SHARED, DRV_NAME"-intx",
+				ihost);
 		if (err)
 			break;
 	}
diff --git a/drivers/scsi/isci/probe_roms.c b/drivers/scsi/isci/probe_roms.c
index 8ac646e5eddc..a2bbe46f8ccb 100644
--- a/drivers/scsi/isci/probe_roms.c
+++ b/drivers/scsi/isci/probe_roms.c
@@ -54,6 +54,7 @@ struct isci_orom *isci_request_oprom(struct pci_dev *pdev)
 	len = pci_biosrom_size(pdev);
 	rom = devm_kzalloc(&pdev->dev, sizeof(*rom), GFP_KERNEL);
 	if (!rom) {
+		pci_unmap_biosrom(oprom);
 		dev_warn(&pdev->dev,
 			 "Unable to allocate memory for orom\n");
 		return NULL;
diff --git a/drivers/scsi/isci/remote_node_context.c b/drivers/scsi/isci/remote_node_context.c
index 1910100638a2..e3f2a5359d71 100644
--- a/drivers/scsi/isci/remote_node_context.c
+++ b/drivers/scsi/isci/remote_node_context.c
@@ -66,6 +66,9 @@ const char *rnc_state_name(enum scis_sds_remote_node_context_states state)
 {
 	static const char * const strings[] = RNC_STATES;
 
+	if (state >= ARRAY_SIZE(strings))
+		return "UNKNOWN";
+
 	return strings[state];
 }
 #undef C
@@ -454,7 +457,7 @@ enum sci_status sci_remote_node_context_event_handler(struct sci_remote_node_con
 				 * the device since it's being invalidated anyway */
 				dev_warn(scirdev_to_dev(rnc_to_dev(sci_rnc)),
 					"%s: SCIC Remote Node Context 0x%p was "
-					"suspeneded by hardware while being "
+					"suspended by hardware while being "
 					"invalidated.\n", __func__, sci_rnc);
 				break;
 			default:
@@ -473,7 +476,7 @@ enum sci_status sci_remote_node_context_event_handler(struct sci_remote_node_con
 				 * the device since it's being resumed anyway */
 				dev_warn(scirdev_to_dev(rnc_to_dev(sci_rnc)),
 					"%s: SCIC Remote Node Context 0x%p was "
-					"suspeneded by hardware while being resumed.\n",
+					"suspended by hardware while being resumed.\n",
 					__func__, sci_rnc);
 				break;
 			default:
diff --git a/drivers/scsi/isci/request.c b/drivers/scsi/isci/request.c
index b709d2b20880..47f66e949745 100644
--- a/drivers/scsi/isci/request.c
+++ b/drivers/scsi/isci/request.c
@@ -2473,7 +2473,7 @@ static void isci_request_process_response_iu(
 		"%s: resp_iu = %p "
 		"resp_iu->status = 0x%x,\nresp_iu->datapres = %d "
 		"resp_iu->response_data_len = %x, "
-		"resp_iu->sense_data_len = %x\nrepsonse data: ",
+		"resp_iu->sense_data_len = %x\nresponse data: ",
 		__func__,
 		resp_iu,
 		resp_iu->status,
diff --git a/drivers/scsi/libfc/fc_disc.c b/drivers/scsi/libfc/fc_disc.c
index 880a9068ca12..6103231104da 100644
--- a/drivers/scsi/libfc/fc_disc.c
+++ b/drivers/scsi/libfc/fc_disc.c
@@ -68,10 +68,14 @@ static void fc_disc_stop_rports(struct fc_disc *disc)
 
 	lport = fc_disc_lport(disc);
 
-	mutex_lock(&disc->disc_mutex);
-	list_for_each_entry_rcu(rdata, &disc->rports, peers)
-		lport->tt.rport_logoff(rdata);
-	mutex_unlock(&disc->disc_mutex);
+	rcu_read_lock();
+	list_for_each_entry_rcu(rdata, &disc->rports, peers) {
+		if (kref_get_unless_zero(&rdata->kref)) {
+			fc_rport_logoff(rdata);
+			kref_put(&rdata->kref, fc_rport_destroy);
+		}
+	}
+	rcu_read_unlock();
 }
 
 /**
@@ -150,7 +154,7 @@ static void fc_disc_recv_rscn_req(struct fc_disc *disc, struct fc_frame *fp)
 			break;
 		}
 	}
-	lport->tt.seq_els_rsp_send(fp, ELS_LS_ACC, NULL);
+	fc_seq_els_rsp_send(fp, ELS_LS_ACC, NULL);
 
 	/*
 	 * If not doing a complete rediscovery, do GPN_ID on
@@ -178,7 +182,7 @@ reject:
 	FC_DISC_DBG(disc, "Received a bad RSCN frame\n");
 	rjt_data.reason = ELS_RJT_LOGIC;
 	rjt_data.explan = ELS_EXPL_NONE;
-	lport->tt.seq_els_rsp_send(fp, ELS_LS_RJT, &rjt_data);
+	fc_seq_els_rsp_send(fp, ELS_LS_RJT, &rjt_data);
 	fc_frame_free(fp);
 }
 
@@ -289,15 +293,19 @@ static void fc_disc_done(struct fc_disc *disc, enum fc_disc_event event)
 	 * Skip ports which were never discovered.  These are the dNS port
 	 * and ports which were created by PLOGI.
 	 */
+	rcu_read_lock();
 	list_for_each_entry_rcu(rdata, &disc->rports, peers) {
-		if (!rdata->disc_id)
+		if (!kref_get_unless_zero(&rdata->kref))
 			continue;
-		if (rdata->disc_id == disc->disc_id)
-			lport->tt.rport_login(rdata);
-		else
-			lport->tt.rport_logoff(rdata);
+		if (rdata->disc_id) {
+			if (rdata->disc_id == disc->disc_id)
+				fc_rport_login(rdata);
+			else
+				fc_rport_logoff(rdata);
+		}
+		kref_put(&rdata->kref, fc_rport_destroy);
 	}
-
+	rcu_read_unlock();
 	mutex_unlock(&disc->disc_mutex);
 	disc->disc_callback(lport, event);
 	mutex_lock(&disc->disc_mutex);
@@ -446,7 +454,7 @@ static int fc_disc_gpn_ft_parse(struct fc_disc *disc, void *buf, size_t len)
 
 		if (ids.port_id != lport->port_id &&
 		    ids.port_name != lport->wwpn) {
-			rdata = lport->tt.rport_create(lport, ids.port_id);
+			rdata = fc_rport_create(lport, ids.port_id);
 			if (rdata) {
 				rdata->ids.port_name = ids.port_name;
 				rdata->disc_id = disc->disc_id;
@@ -592,7 +600,6 @@ static void fc_disc_gpn_id_resp(struct fc_seq *sp, struct fc_frame *fp,
 	lport = rdata->local_port;
 	disc = &lport->disc;
 
-	mutex_lock(&disc->disc_mutex);
 	if (PTR_ERR(fp) == -FC_EX_CLOSED)
 		goto out;
 	if (IS_ERR(fp))
@@ -607,37 +614,41 @@ static void fc_disc_gpn_id_resp(struct fc_seq *sp, struct fc_frame *fp,
 			goto redisc;
 		pn = (struct fc_ns_gid_pn *)(cp + 1);
 		port_name = get_unaligned_be64(&pn->fn_wwpn);
+		mutex_lock(&rdata->rp_mutex);
 		if (rdata->ids.port_name == -1)
 			rdata->ids.port_name = port_name;
 		else if (rdata->ids.port_name != port_name) {
 			FC_DISC_DBG(disc, "GPN_ID accepted.  WWPN changed. "
 				    "Port-id %6.6x wwpn %16.16llx\n",
 				    rdata->ids.port_id, port_name);
-			lport->tt.rport_logoff(rdata);
-
-			new_rdata = lport->tt.rport_create(lport,
-							   rdata->ids.port_id);
+			mutex_unlock(&rdata->rp_mutex);
+			fc_rport_logoff(rdata);
+			mutex_lock(&lport->disc.disc_mutex);
+			new_rdata = fc_rport_create(lport, rdata->ids.port_id);
+			mutex_unlock(&lport->disc.disc_mutex);
 			if (new_rdata) {
 				new_rdata->disc_id = disc->disc_id;
-				lport->tt.rport_login(new_rdata);
+				fc_rport_login(new_rdata);
 			}
 			goto out;
 		}
 		rdata->disc_id = disc->disc_id;
-		lport->tt.rport_login(rdata);
+		mutex_unlock(&rdata->rp_mutex);
+		fc_rport_login(rdata);
 	} else if (ntohs(cp->ct_cmd) == FC_FS_RJT) {
 		FC_DISC_DBG(disc, "GPN_ID rejected reason %x exp %x\n",
 			    cp->ct_reason, cp->ct_explan);
-		lport->tt.rport_logoff(rdata);
+		fc_rport_logoff(rdata);
 	} else {
 		FC_DISC_DBG(disc, "GPN_ID unexpected response code %x\n",
 			    ntohs(cp->ct_cmd));
 redisc:
+		mutex_lock(&disc->disc_mutex);
 		fc_disc_restart(disc);
+		mutex_unlock(&disc->disc_mutex);
 	}
 out:
-	mutex_unlock(&disc->disc_mutex);
-	kref_put(&rdata->kref, lport->tt.rport_destroy);
+	kref_put(&rdata->kref, fc_rport_destroy);
 }
 
 /**
@@ -678,7 +689,7 @@ static int fc_disc_single(struct fc_lport *lport, struct fc_disc_port *dp)
 {
 	struct fc_rport_priv *rdata;
 
-	rdata = lport->tt.rport_create(lport, dp->port_id);
+	rdata = fc_rport_create(lport, dp->port_id);
 	if (!rdata)
 		return -ENOMEM;
 	rdata->disc_id = 0;
@@ -708,7 +719,7 @@ static void fc_disc_stop(struct fc_lport *lport)
 static void fc_disc_stop_final(struct fc_lport *lport)
 {
 	fc_disc_stop(lport);
-	lport->tt.rport_flush_queue();
+	fc_rport_flush_queue();
 }
 
 /**
diff --git a/drivers/scsi/libfc/fc_elsct.c b/drivers/scsi/libfc/fc_elsct.c
index c2384d501470..6384a98048af 100644
--- a/drivers/scsi/libfc/fc_elsct.c
+++ b/drivers/scsi/libfc/fc_elsct.c
@@ -67,7 +67,7 @@ struct fc_seq *fc_elsct_send(struct fc_lport *lport, u32 did,
 	fc_fill_fc_hdr(fp, r_ctl, did, lport->port_id, fh_type,
 		       FC_FCTL_REQ, 0);
 
-	return lport->tt.exch_seq_send(lport, fp, resp, NULL, arg, timer_msec);
+	return fc_exch_seq_send(lport, fp, resp, NULL, arg, timer_msec);
 }
 EXPORT_SYMBOL(fc_elsct_send);
 
diff --git a/drivers/scsi/libfc/fc_exch.c b/drivers/scsi/libfc/fc_exch.c
index 16ca31ad5ec0..42bcf7f3a0f9 100644
--- a/drivers/scsi/libfc/fc_exch.c
+++ b/drivers/scsi/libfc/fc_exch.c
@@ -94,6 +94,7 @@ struct fc_exch_pool {
 struct fc_exch_mgr {
 	struct fc_exch_pool __percpu *pool;
 	mempool_t	*ep_pool;
+	struct fc_lport	*lport;
 	enum fc_class	class;
 	struct kref	kref;
 	u16		min_xid;
@@ -362,8 +363,10 @@ static inline void fc_exch_timer_set_locked(struct fc_exch *ep,
 
 	fc_exch_hold(ep);		/* hold for timer */
 	if (!queue_delayed_work(fc_exch_workqueue, &ep->timeout_work,
-				msecs_to_jiffies(timer_msec)))
+				msecs_to_jiffies(timer_msec))) {
+		FC_EXCH_DBG(ep, "Exchange already queued\n");
 		fc_exch_release(ep);
+	}
 }
 
 /**
@@ -406,6 +409,8 @@ static int fc_exch_done_locked(struct fc_exch *ep)
 	return rc;
 }
 
+static struct fc_exch fc_quarantine_exch;
+
 /**
  * fc_exch_ptr_get() - Return an exchange from an exchange pool
  * @pool:  Exchange Pool to get an exchange from
@@ -450,14 +455,17 @@ static void fc_exch_delete(struct fc_exch *ep)
 
 	/* update cache of free slot */
 	index = (ep->xid - ep->em->min_xid) >> fc_cpu_order;
-	if (pool->left == FC_XID_UNKNOWN)
-		pool->left = index;
-	else if (pool->right == FC_XID_UNKNOWN)
-		pool->right = index;
-	else
-		pool->next_index = index;
-
-	fc_exch_ptr_set(pool, index, NULL);
+	if (!(ep->state & FC_EX_QUARANTINE)) {
+		if (pool->left == FC_XID_UNKNOWN)
+			pool->left = index;
+		else if (pool->right == FC_XID_UNKNOWN)
+			pool->right = index;
+		else
+			pool->next_index = index;
+		fc_exch_ptr_set(pool, index, NULL);
+	} else {
+		fc_exch_ptr_set(pool, index, &fc_quarantine_exch);
+	}
 	list_del(&ep->ex_list);
 	spin_unlock_bh(&pool->lock);
 	fc_exch_release(ep);	/* drop hold for exch in mp */
@@ -525,8 +533,7 @@ out:
  * Note: The frame will be freed either by a direct call to fc_frame_free(fp)
  * or indirectly by calling libfc_function_template.frame_send().
  */
-static int fc_seq_send(struct fc_lport *lport, struct fc_seq *sp,
-		       struct fc_frame *fp)
+int fc_seq_send(struct fc_lport *lport, struct fc_seq *sp, struct fc_frame *fp)
 {
 	struct fc_exch *ep;
 	int error;
@@ -536,6 +543,7 @@ static int fc_seq_send(struct fc_lport *lport, struct fc_seq *sp,
 	spin_unlock_bh(&ep->ex_lock);
 	return error;
 }
+EXPORT_SYMBOL(fc_seq_send);
 
 /**
  * fc_seq_alloc() - Allocate a sequence for a given exchange
@@ -577,7 +585,7 @@ static struct fc_seq *fc_seq_start_next_locked(struct fc_seq *sp)
  *			 for a given sequence/exchange pair
  * @sp: The sequence/exchange to get a new exchange for
  */
-static struct fc_seq *fc_seq_start_next(struct fc_seq *sp)
+struct fc_seq *fc_seq_start_next(struct fc_seq *sp)
 {
 	struct fc_exch *ep = fc_seq_exch(sp);
 
@@ -587,16 +595,16 @@ static struct fc_seq *fc_seq_start_next(struct fc_seq *sp)
 
 	return sp;
 }
+EXPORT_SYMBOL(fc_seq_start_next);
 
 /*
  * Set the response handler for the exchange associated with a sequence.
  *
  * Note: May sleep if invoked from outside a response handler.
  */
-static void fc_seq_set_resp(struct fc_seq *sp,
-			    void (*resp)(struct fc_seq *, struct fc_frame *,
-					 void *),
-			    void *arg)
+void fc_seq_set_resp(struct fc_seq *sp,
+		     void (*resp)(struct fc_seq *, struct fc_frame *, void *),
+		     void *arg)
 {
 	struct fc_exch *ep = fc_seq_exch(sp);
 	DEFINE_WAIT(wait);
@@ -615,12 +623,20 @@ static void fc_seq_set_resp(struct fc_seq *sp,
 	ep->arg = arg;
 	spin_unlock_bh(&ep->ex_lock);
 }
+EXPORT_SYMBOL(fc_seq_set_resp);
 
 /**
  * fc_exch_abort_locked() - Abort an exchange
  * @ep:	The exchange to be aborted
  * @timer_msec: The period of time to wait before aborting
  *
+ * Abort an exchange and sequence. Generally called because of a
+ * exchange timeout or an abort from the upper layer.
+ *
+ * A timer_msec can be specified for abort timeout, if non-zero
+ * timer_msec value is specified then exchange resp handler
+ * will be called with timeout error if no response to abort.
+ *
  * Locking notes:  Called with exch lock held
  *
  * Return value: 0 on success else error code
@@ -632,9 +648,13 @@ static int fc_exch_abort_locked(struct fc_exch *ep,
 	struct fc_frame *fp;
 	int error;
 
+	FC_EXCH_DBG(ep, "exch: abort, time %d msecs\n", timer_msec);
 	if (ep->esb_stat & (ESB_ST_COMPLETE | ESB_ST_ABNORMAL) ||
-	    ep->state & (FC_EX_DONE | FC_EX_RST_CLEANUP))
+	    ep->state & (FC_EX_DONE | FC_EX_RST_CLEANUP)) {
+		FC_EXCH_DBG(ep, "exch: already completed esb %x state %x\n",
+			    ep->esb_stat, ep->state);
 		return -ENXIO;
+	}
 
 	/*
 	 * Send the abort on a new sequence if possible.
@@ -680,8 +700,7 @@ static int fc_exch_abort_locked(struct fc_exch *ep,
  *
  * Return value: 0 on success else error code
  */
-static int fc_seq_exch_abort(const struct fc_seq *req_sp,
-			     unsigned int timer_msec)
+int fc_seq_exch_abort(const struct fc_seq *req_sp, unsigned int timer_msec)
 {
 	struct fc_exch *ep;
 	int error;
@@ -758,7 +777,7 @@ static void fc_exch_timeout(struct work_struct *work)
 	u32 e_stat;
 	int rc = 1;
 
-	FC_EXCH_DBG(ep, "Exchange timed out\n");
+	FC_EXCH_DBG(ep, "Exchange timed out state %x\n", ep->state);
 
 	spin_lock_bh(&ep->ex_lock);
 	if (ep->state & (FC_EX_RST_CLEANUP | FC_EX_DONE))
@@ -821,14 +840,18 @@ static struct fc_exch *fc_exch_em_alloc(struct fc_lport *lport,
 
 	/* peek cache of free slot */
 	if (pool->left != FC_XID_UNKNOWN) {
-		index = pool->left;
-		pool->left = FC_XID_UNKNOWN;
-		goto hit;
+		if (!WARN_ON(fc_exch_ptr_get(pool, pool->left))) {
+			index = pool->left;
+			pool->left = FC_XID_UNKNOWN;
+			goto hit;
+		}
 	}
 	if (pool->right != FC_XID_UNKNOWN) {
-		index = pool->right;
-		pool->right = FC_XID_UNKNOWN;
-		goto hit;
+		if (!WARN_ON(fc_exch_ptr_get(pool, pool->right))) {
+			index = pool->right;
+			pool->right = FC_XID_UNKNOWN;
+			goto hit;
+		}
 	}
 
 	index = pool->next_index;
@@ -888,14 +911,19 @@ err:
  * EM is selected when a NULL match function pointer is encountered
  * or when a call to a match function returns true.
  */
-static inline struct fc_exch *fc_exch_alloc(struct fc_lport *lport,
-					    struct fc_frame *fp)
+static struct fc_exch *fc_exch_alloc(struct fc_lport *lport,
+				     struct fc_frame *fp)
 {
 	struct fc_exch_mgr_anchor *ema;
+	struct fc_exch *ep;
 
-	list_for_each_entry(ema, &lport->ema_list, ema_list)
-		if (!ema->match || ema->match(fp))
-			return fc_exch_em_alloc(lport, ema->mp);
+	list_for_each_entry(ema, &lport->ema_list, ema_list) {
+		if (!ema->match || ema->match(fp)) {
+			ep = fc_exch_em_alloc(lport, ema->mp);
+			if (ep)
+				return ep;
+		}
+	}
 	return NULL;
 }
 
@@ -906,14 +934,17 @@ static inline struct fc_exch *fc_exch_alloc(struct fc_lport *lport,
  */
 static struct fc_exch *fc_exch_find(struct fc_exch_mgr *mp, u16 xid)
 {
+	struct fc_lport *lport = mp->lport;
 	struct fc_exch_pool *pool;
 	struct fc_exch *ep = NULL;
 	u16 cpu = xid & fc_cpu_mask;
 
+	if (xid == FC_XID_UNKNOWN)
+		return NULL;
+
 	if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) {
-		printk_ratelimited(KERN_ERR
-			"libfc: lookup request for XID = %d, "
-			"indicates invalid CPU %d\n", xid, cpu);
+		pr_err("host%u: lport %6.6x: xid %d invalid CPU %d\n:",
+		       lport->host->host_no, lport->port_id, xid, cpu);
 		return NULL;
 	}
 
@@ -921,6 +952,10 @@ static struct fc_exch *fc_exch_find(struct fc_exch_mgr *mp, u16 xid)
 		pool = per_cpu_ptr(mp->pool, cpu);
 		spin_lock_bh(&pool->lock);
 		ep = fc_exch_ptr_get(pool, (xid - mp->min_xid) >> fc_cpu_order);
+		if (ep == &fc_quarantine_exch) {
+			FC_LPORT_DBG(lport, "xid %x quarantined\n", xid);
+			ep = NULL;
+		}
 		if (ep) {
 			WARN_ON(ep->xid != xid);
 			fc_exch_hold(ep);
@@ -938,7 +973,7 @@ static struct fc_exch *fc_exch_find(struct fc_exch_mgr *mp, u16 xid)
  *
  * Note: May sleep if invoked from outside a response handler.
  */
-static void fc_exch_done(struct fc_seq *sp)
+void fc_exch_done(struct fc_seq *sp)
 {
 	struct fc_exch *ep = fc_seq_exch(sp);
 	int rc;
@@ -951,6 +986,7 @@ static void fc_exch_done(struct fc_seq *sp)
 	if (!rc)
 		fc_exch_delete(ep);
 }
+EXPORT_SYMBOL(fc_exch_done);
 
 /**
  * fc_exch_resp() - Allocate a new exchange for a response frame
@@ -1197,8 +1233,8 @@ static void fc_exch_set_addr(struct fc_exch *ep,
  *
  * The received frame is not freed.
  */
-static void fc_seq_els_rsp_send(struct fc_frame *fp, enum fc_els_cmd els_cmd,
-				struct fc_seq_els_data *els_data)
+void fc_seq_els_rsp_send(struct fc_frame *fp, enum fc_els_cmd els_cmd,
+			 struct fc_seq_els_data *els_data)
 {
 	switch (els_cmd) {
 	case ELS_LS_RJT:
@@ -1217,6 +1253,7 @@ static void fc_seq_els_rsp_send(struct fc_frame *fp, enum fc_els_cmd els_cmd,
 		FC_LPORT_DBG(fr_dev(fp), "Invalid ELS CMD:%x\n", els_cmd);
 	}
 }
+EXPORT_SYMBOL_GPL(fc_seq_els_rsp_send);
 
 /**
  * fc_seq_send_last() - Send a sequence that is the last in the exchange
@@ -1258,8 +1295,10 @@ static void fc_seq_send_ack(struct fc_seq *sp, const struct fc_frame *rx_fp)
 	 */
 	if (fc_sof_needs_ack(fr_sof(rx_fp))) {
 		fp = fc_frame_alloc(lport, 0);
-		if (!fp)
+		if (!fp) {
+			FC_EXCH_DBG(ep, "Drop ACK request, out of memory\n");
 			return;
+		}
 
 		fh = fc_frame_header_get(fp);
 		fh->fh_r_ctl = FC_RCTL_ACK_1;
@@ -1312,13 +1351,18 @@ static void fc_exch_send_ba_rjt(struct fc_frame *rx_fp,
 	struct fc_frame_header *rx_fh;
 	struct fc_frame_header *fh;
 	struct fc_ba_rjt *rp;
+	struct fc_seq *sp;
 	struct fc_lport *lport;
 	unsigned int f_ctl;
 
 	lport = fr_dev(rx_fp);
+	sp = fr_seq(rx_fp);
 	fp = fc_frame_alloc(lport, sizeof(*rp));
-	if (!fp)
+	if (!fp) {
+		FC_EXCH_DBG(fc_seq_exch(sp),
+			     "Drop BA_RJT request, out of memory\n");
 		return;
+	}
 	fh = fc_frame_header_get(fp);
 	rx_fh = fc_frame_header_get(rx_fp);
 
@@ -1383,14 +1427,17 @@ static void fc_exch_recv_abts(struct fc_exch *ep, struct fc_frame *rx_fp)
 	if (!ep)
 		goto reject;
 
+	FC_EXCH_DBG(ep, "exch: ABTS received\n");
 	fp = fc_frame_alloc(ep->lp, sizeof(*ap));
-	if (!fp)
+	if (!fp) {
+		FC_EXCH_DBG(ep, "Drop ABTS request, out of memory\n");
 		goto free;
+	}
 
 	spin_lock_bh(&ep->ex_lock);
 	if (ep->esb_stat & ESB_ST_COMPLETE) {
 		spin_unlock_bh(&ep->ex_lock);
-
+		FC_EXCH_DBG(ep, "exch: ABTS rejected, exchange complete\n");
 		fc_frame_free(fp);
 		goto reject;
 	}
@@ -1433,7 +1480,7 @@ reject:
  * A reference will be held on the exchange/sequence for the caller, which
  * must call fc_seq_release().
  */
-static struct fc_seq *fc_seq_assign(struct fc_lport *lport, struct fc_frame *fp)
+struct fc_seq *fc_seq_assign(struct fc_lport *lport, struct fc_frame *fp)
 {
 	struct fc_exch_mgr_anchor *ema;
 
@@ -1447,15 +1494,17 @@ static struct fc_seq *fc_seq_assign(struct fc_lport *lport, struct fc_frame *fp)
 			break;
 	return fr_seq(fp);
 }
+EXPORT_SYMBOL(fc_seq_assign);
 
 /**
  * fc_seq_release() - Release the hold
  * @sp:    The sequence.
  */
-static void fc_seq_release(struct fc_seq *sp)
+void fc_seq_release(struct fc_seq *sp)
 {
 	fc_exch_release(fc_seq_exch(sp));
 }
+EXPORT_SYMBOL(fc_seq_release);
 
 /**
  * fc_exch_recv_req() - Handler for an incoming request
@@ -1491,7 +1540,7 @@ static void fc_exch_recv_req(struct fc_lport *lport, struct fc_exch_mgr *mp,
 	 * The upper-level protocol may request one later, if needed.
 	 */
 	if (fh->fh_rx_id == htons(FC_XID_UNKNOWN))
-		return lport->tt.lport_recv(lport, fp);
+		return fc_lport_recv(lport, fp);
 
 	reject = fc_seq_lookup_recip(lport, mp, fp);
 	if (reject == FC_RJT_NONE) {
@@ -1512,7 +1561,7 @@ static void fc_exch_recv_req(struct fc_lport *lport, struct fc_exch_mgr *mp,
 		 * first.
 		 */
 		if (!fc_invoke_resp(ep, sp, fp))
-			lport->tt.lport_recv(lport, fp);
+			fc_lport_recv(lport, fp);
 		fc_exch_release(ep);	/* release from lookup */
 	} else {
 		FC_LPORT_DBG(lport, "exch/seq lookup failed: reject %x\n",
@@ -1562,9 +1611,6 @@ static void fc_exch_recv_seq_resp(struct fc_exch_mgr *mp, struct fc_frame *fp)
 	if (fc_sof_is_init(sof)) {
 		sp->ssb_stat |= SSB_ST_RESP;
 		sp->id = fh->fh_seq_id;
-	} else if (sp->id != fh->fh_seq_id) {
-		atomic_inc(&mp->stats.seq_not_found);
-		goto rel;
 	}
 
 	f_ctl = ntoh24(fh->fh_f_ctl);
@@ -1761,7 +1807,10 @@ static void fc_exch_recv_bls(struct fc_exch_mgr *mp, struct fc_frame *fp)
 				fc_frame_free(fp);
 			break;
 		case FC_RCTL_BA_ABTS:
-			fc_exch_recv_abts(ep, fp);
+			if (ep)
+				fc_exch_recv_abts(ep, fp);
+			else
+				fc_frame_free(fp);
 			break;
 		default:			/* ignore junk */
 			fc_frame_free(fp);
@@ -1784,11 +1833,16 @@ static void fc_seq_ls_acc(struct fc_frame *rx_fp)
 	struct fc_lport *lport;
 	struct fc_els_ls_acc *acc;
 	struct fc_frame *fp;
+	struct fc_seq *sp;
 
 	lport = fr_dev(rx_fp);
+	sp = fr_seq(rx_fp);
 	fp = fc_frame_alloc(lport, sizeof(*acc));
-	if (!fp)
+	if (!fp) {
+		FC_EXCH_DBG(fc_seq_exch(sp),
+			    "exch: drop LS_ACC, out of memory\n");
 		return;
+	}
 	acc = fc_frame_payload_get(fp, sizeof(*acc));
 	memset(acc, 0, sizeof(*acc));
 	acc->la_cmd = ELS_LS_ACC;
@@ -1811,11 +1865,16 @@ static void fc_seq_ls_rjt(struct fc_frame *rx_fp, enum fc_els_rjt_reason reason,
 	struct fc_lport *lport;
 	struct fc_els_ls_rjt *rjt;
 	struct fc_frame *fp;
+	struct fc_seq *sp;
 
 	lport = fr_dev(rx_fp);
+	sp = fr_seq(rx_fp);
 	fp = fc_frame_alloc(lport, sizeof(*rjt));
-	if (!fp)
+	if (!fp) {
+		FC_EXCH_DBG(fc_seq_exch(sp),
+			    "exch: drop LS_ACC, out of memory\n");
 		return;
+	}
 	rjt = fc_frame_payload_get(fp, sizeof(*rjt));
 	memset(rjt, 0, sizeof(*rjt));
 	rjt->er_cmd = ELS_LS_RJT;
@@ -1960,8 +2019,7 @@ static void fc_exch_els_rec(struct fc_frame *rfp)
 	enum fc_els_rjt_reason reason = ELS_RJT_LOGIC;
 	enum fc_els_rjt_explan explan;
 	u32 sid;
-	u16 rxid;
-	u16 oxid;
+	u16 xid, rxid, oxid;
 
 	lport = fr_dev(rfp);
 	rp = fc_frame_payload_get(rfp, sizeof(*rp));
@@ -1972,18 +2030,35 @@ static void fc_exch_els_rec(struct fc_frame *rfp)
 	rxid = ntohs(rp->rec_rx_id);
 	oxid = ntohs(rp->rec_ox_id);
 
-	ep = fc_exch_lookup(lport,
-			    sid == fc_host_port_id(lport->host) ? oxid : rxid);
 	explan = ELS_EXPL_OXID_RXID;
-	if (!ep)
+	if (sid == fc_host_port_id(lport->host))
+		xid = oxid;
+	else
+		xid = rxid;
+	if (xid == FC_XID_UNKNOWN) {
+		FC_LPORT_DBG(lport,
+			     "REC request from %x: invalid rxid %x oxid %x\n",
+			     sid, rxid, oxid);
+		goto reject;
+	}
+	ep = fc_exch_lookup(lport, xid);
+	if (!ep) {
+		FC_LPORT_DBG(lport,
+			     "REC request from %x: rxid %x oxid %x not found\n",
+			     sid, rxid, oxid);
 		goto reject;
+	}
+	FC_EXCH_DBG(ep, "REC request from %x: rxid %x oxid %x\n",
+		    sid, rxid, oxid);
 	if (ep->oid != sid || oxid != ep->oxid)
 		goto rel;
 	if (rxid != FC_XID_UNKNOWN && rxid != ep->rxid)
 		goto rel;
 	fp = fc_frame_alloc(lport, sizeof(*acc));
-	if (!fp)
+	if (!fp) {
+		FC_EXCH_DBG(ep, "Drop REC request, out of memory\n");
 		goto out;
+	}
 
 	acc = fc_frame_payload_get(fp, sizeof(*acc));
 	memset(acc, 0, sizeof(*acc));
@@ -2065,6 +2140,24 @@ cleanup:
  * @arg:	The argument to be passed to the response handler
  * @timer_msec: The timeout period for the exchange
  *
+ * The exchange response handler is set in this routine to resp()
+ * function pointer. It can be called in two scenarios: if a timeout
+ * occurs or if a response frame is received for the exchange. The
+ * fc_frame pointer in response handler will also indicate timeout
+ * as error using IS_ERR related macros.
+ *
+ * The exchange destructor handler is also set in this routine.
+ * The destructor handler is invoked by EM layer when exchange
+ * is about to free, this can be used by caller to free its
+ * resources along with exchange free.
+ *
+ * The arg is passed back to resp and destructor handler.
+ *
+ * The timeout value (in msec) for an exchange is set if non zero
+ * timer_msec argument is specified. The timer is canceled when
+ * it fires or when the exchange is done. The exchange timeout handler
+ * is registered by EM layer.
+ *
  * The frame pointer with some of the header's fields must be
  * filled before calling this routine, those fields are:
  *
@@ -2075,14 +2168,13 @@ cleanup:
  * - frame control
  * - parameter or relative offset
  */
-static struct fc_seq *fc_exch_seq_send(struct fc_lport *lport,
-				       struct fc_frame *fp,
-				       void (*resp)(struct fc_seq *,
-						    struct fc_frame *fp,
-						    void *arg),
-				       void (*destructor)(struct fc_seq *,
-							  void *),
-				       void *arg, u32 timer_msec)
+struct fc_seq *fc_exch_seq_send(struct fc_lport *lport,
+				struct fc_frame *fp,
+				void (*resp)(struct fc_seq *,
+					     struct fc_frame *fp,
+					     void *arg),
+				void (*destructor)(struct fc_seq *, void *),
+				void *arg, u32 timer_msec)
 {
 	struct fc_exch *ep;
 	struct fc_seq *sp = NULL;
@@ -2101,7 +2193,7 @@ static struct fc_seq *fc_exch_seq_send(struct fc_lport *lport,
 	ep->resp = resp;
 	ep->destructor = destructor;
 	ep->arg = arg;
-	ep->r_a_tov = FC_DEF_R_A_TOV;
+	ep->r_a_tov = lport->r_a_tov;
 	ep->lp = lport;
 	sp = &ep->seq;
 
@@ -2135,6 +2227,7 @@ err:
 		fc_exch_delete(ep);
 	return NULL;
 }
+EXPORT_SYMBOL(fc_exch_seq_send);
 
 /**
  * fc_exch_rrq() - Send an ELS RRQ (Reinstate Recovery Qualifier) command
@@ -2176,6 +2269,7 @@ static void fc_exch_rrq(struct fc_exch *ep)
 		return;
 
 retry:
+	FC_EXCH_DBG(ep, "exch: RRQ send failed\n");
 	spin_lock_bh(&ep->ex_lock);
 	if (ep->state & (FC_EX_RST_CLEANUP | FC_EX_DONE)) {
 		spin_unlock_bh(&ep->ex_lock);
@@ -2218,6 +2312,8 @@ static void fc_exch_els_rrq(struct fc_frame *fp)
 	if (!ep)
 		goto reject;
 	spin_lock_bh(&ep->ex_lock);
+	FC_EXCH_DBG(ep, "RRQ request from %x: xid %x rxid %x oxid %x\n",
+		    sid, xid, ntohs(rp->rrq_rx_id), ntohs(rp->rrq_ox_id));
 	if (ep->oxid != ntohs(rp->rrq_ox_id))
 		goto unlock_reject;
 	if (ep->rxid != ntohs(rp->rrq_rx_id) &&
@@ -2385,6 +2481,7 @@ struct fc_exch_mgr *fc_exch_mgr_alloc(struct fc_lport *lport,
 		return NULL;
 
 	mp->class = class;
+	mp->lport = lport;
 	/* adjust em exch xid range for offload */
 	mp->min_xid = min_xid;
 
@@ -2558,36 +2655,9 @@ EXPORT_SYMBOL(fc_exch_recv);
  */
 int fc_exch_init(struct fc_lport *lport)
 {
-	if (!lport->tt.seq_start_next)
-		lport->tt.seq_start_next = fc_seq_start_next;
-
-	if (!lport->tt.seq_set_resp)
-		lport->tt.seq_set_resp = fc_seq_set_resp;
-
-	if (!lport->tt.exch_seq_send)
-		lport->tt.exch_seq_send = fc_exch_seq_send;
-
-	if (!lport->tt.seq_send)
-		lport->tt.seq_send = fc_seq_send;
-
-	if (!lport->tt.seq_els_rsp_send)
-		lport->tt.seq_els_rsp_send = fc_seq_els_rsp_send;
-
-	if (!lport->tt.exch_done)
-		lport->tt.exch_done = fc_exch_done;
-
 	if (!lport->tt.exch_mgr_reset)
 		lport->tt.exch_mgr_reset = fc_exch_mgr_reset;
 
-	if (!lport->tt.seq_exch_abort)
-		lport->tt.seq_exch_abort = fc_seq_exch_abort;
-
-	if (!lport->tt.seq_assign)
-		lport->tt.seq_assign = fc_seq_assign;
-
-	if (!lport->tt.seq_release)
-		lport->tt.seq_release = fc_seq_release;
-
 	return 0;
 }
 EXPORT_SYMBOL(fc_exch_init);
diff --git a/drivers/scsi/libfc/fc_fcp.c b/drivers/scsi/libfc/fc_fcp.c
index 5121272f28fd..0e67621477a8 100644
--- a/drivers/scsi/libfc/fc_fcp.c
+++ b/drivers/scsi/libfc/fc_fcp.c
@@ -122,6 +122,7 @@ static void fc_fcp_srr_error(struct fc_fcp_pkt *, struct fc_frame *);
 #define FC_HRD_ERROR		9
 #define FC_CRC_ERROR		10
 #define FC_TIMED_OUT		11
+#define FC_TRANS_RESET		12
 
 /*
  * Error recovery timeout values.
@@ -195,7 +196,7 @@ static void fc_fcp_pkt_hold(struct fc_fcp_pkt *fsp)
  * @seq: The sequence that the FCP packet is on (required by destructor API)
  * @fsp: The FCP packet to be released
  *
- * This routine is called by a destructor callback in the exch_seq_send()
+ * This routine is called by a destructor callback in the fc_exch_seq_send()
  * routine of the libfc Transport Template. The 'struct fc_seq' is a required
  * argument even though it is not used by this routine.
  *
@@ -253,8 +254,21 @@ static inline void fc_fcp_unlock_pkt(struct fc_fcp_pkt *fsp)
  */
 static void fc_fcp_timer_set(struct fc_fcp_pkt *fsp, unsigned long delay)
 {
-	if (!(fsp->state & FC_SRB_COMPL))
+	if (!(fsp->state & FC_SRB_COMPL)) {
 		mod_timer(&fsp->timer, jiffies + delay);
+		fsp->timer_delay = delay;
+	}
+}
+
+static void fc_fcp_abort_done(struct fc_fcp_pkt *fsp)
+{
+	fsp->state |= FC_SRB_ABORTED;
+	fsp->state &= ~FC_SRB_ABORT_PENDING;
+
+	if (fsp->wait_for_comp)
+		complete(&fsp->tm_done);
+	else
+		fc_fcp_complete_locked(fsp);
 }
 
 /**
@@ -264,6 +278,8 @@ static void fc_fcp_timer_set(struct fc_fcp_pkt *fsp, unsigned long delay)
  */
 static int fc_fcp_send_abort(struct fc_fcp_pkt *fsp)
 {
+	int rc;
+
 	if (!fsp->seq_ptr)
 		return -EINVAL;
 
@@ -271,7 +287,16 @@ static int fc_fcp_send_abort(struct fc_fcp_pkt *fsp)
 	put_cpu();
 
 	fsp->state |= FC_SRB_ABORT_PENDING;
-	return fsp->lp->tt.seq_exch_abort(fsp->seq_ptr, 0);
+	rc = fc_seq_exch_abort(fsp->seq_ptr, 0);
+	/*
+	 * fc_seq_exch_abort() might return -ENXIO if
+	 * the sequence is already completed
+	 */
+	if (rc == -ENXIO) {
+		fc_fcp_abort_done(fsp);
+		rc = 0;
+	}
+	return rc;
 }
 
 /**
@@ -283,16 +308,16 @@ static int fc_fcp_send_abort(struct fc_fcp_pkt *fsp)
  * fc_io_compl() will notify the SCSI-ml that the I/O is done.
  * The SCSI-ml will retry the command.
  */
-static void fc_fcp_retry_cmd(struct fc_fcp_pkt *fsp)
+static void fc_fcp_retry_cmd(struct fc_fcp_pkt *fsp, int status_code)
 {
 	if (fsp->seq_ptr) {
-		fsp->lp->tt.exch_done(fsp->seq_ptr);
+		fc_exch_done(fsp->seq_ptr);
 		fsp->seq_ptr = NULL;
 	}
 
 	fsp->state &= ~FC_SRB_ABORT_PENDING;
 	fsp->io_status = 0;
-	fsp->status_code = FC_ERROR;
+	fsp->status_code = status_code;
 	fc_fcp_complete_locked(fsp);
 }
 
@@ -402,8 +427,6 @@ static void fc_fcp_can_queue_ramp_down(struct fc_lport *lport)
 	if (!can_queue)
 		can_queue = 1;
 	lport->host->can_queue = can_queue;
-	shost_printk(KERN_ERR, lport->host, "libfc: Could not allocate frame.\n"
-		     "Reducing can_queue to %d.\n", can_queue);
 
 unlock:
 	spin_unlock_irqrestore(lport->host->host_lock, flags);
@@ -430,10 +453,29 @@ static inline struct fc_frame *fc_fcp_frame_alloc(struct fc_lport *lport,
 	put_cpu();
 	/* error case */
 	fc_fcp_can_queue_ramp_down(lport);
+	shost_printk(KERN_ERR, lport->host,
+		     "libfc: Could not allocate frame, "
+		     "reducing can_queue to %d.\n", lport->host->can_queue);
 	return NULL;
 }
 
 /**
+ * get_fsp_rec_tov() - Helper function to get REC_TOV
+ * @fsp: the FCP packet
+ *
+ * Returns rec tov in jiffies as rpriv->e_d_tov + 1 second
+ */
+static inline unsigned int get_fsp_rec_tov(struct fc_fcp_pkt *fsp)
+{
+	struct fc_rport_libfc_priv *rpriv = fsp->rport->dd_data;
+	unsigned int e_d_tov = FC_DEF_E_D_TOV;
+
+	if (rpriv && rpriv->e_d_tov > e_d_tov)
+		e_d_tov = rpriv->e_d_tov;
+	return msecs_to_jiffies(e_d_tov) + HZ;
+}
+
+/**
  * fc_fcp_recv_data() - Handler for receiving SCSI-FCP data from a target
  * @fsp: The FCP packet the data is on
  * @fp:	 The data frame
@@ -536,8 +578,10 @@ crc_err:
 	 * and completes the transfer, call the completion handler.
 	 */
 	if (unlikely(fsp->state & FC_SRB_RCV_STATUS) &&
-	    fsp->xfer_len == fsp->data_len - fsp->scsi_resid)
+	    fsp->xfer_len == fsp->data_len - fsp->scsi_resid) {
+		FC_FCP_DBG( fsp, "complete out-of-order sequence\n" );
 		fc_fcp_complete_locked(fsp);
+	}
 	return;
 err:
 	fc_fcp_recovery(fsp, host_bcode);
@@ -609,7 +653,7 @@ static int fc_fcp_send_data(struct fc_fcp_pkt *fsp, struct fc_seq *seq,
 	remaining = seq_blen;
 	fh_parm_offset = frame_offset = offset;
 	tlen = 0;
-	seq = lport->tt.seq_start_next(seq);
+	seq = fc_seq_start_next(seq);
 	f_ctl = FC_FC_REL_OFF;
 	WARN_ON(!seq);
 
@@ -687,7 +731,7 @@ static int fc_fcp_send_data(struct fc_fcp_pkt *fsp, struct fc_seq *seq,
 		/*
 		 * send fragment using for a sequence.
 		 */
-		error = lport->tt.seq_send(lport, seq, fp);
+		error = fc_seq_send(lport, seq, fp);
 		if (error) {
 			WARN_ON(1);		/* send error should be rare */
 			return error;
@@ -727,15 +771,8 @@ static void fc_fcp_abts_resp(struct fc_fcp_pkt *fsp, struct fc_frame *fp)
 		ba_done = 0;
 	}
 
-	if (ba_done) {
-		fsp->state |= FC_SRB_ABORTED;
-		fsp->state &= ~FC_SRB_ABORT_PENDING;
-
-		if (fsp->wait_for_comp)
-			complete(&fsp->tm_done);
-		else
-			fc_fcp_complete_locked(fsp);
-	}
+	if (ba_done)
+		fc_fcp_abort_done(fsp);
 }
 
 /**
@@ -764,8 +801,11 @@ static void fc_fcp_recv(struct fc_seq *seq, struct fc_frame *fp, void *arg)
 	fh = fc_frame_header_get(fp);
 	r_ctl = fh->fh_r_ctl;
 
-	if (lport->state != LPORT_ST_READY)
+	if (lport->state != LPORT_ST_READY) {
+		FC_FCP_DBG(fsp, "lport state %d, ignoring r_ctl %x\n",
+			   lport->state, r_ctl);
 		goto out;
+	}
 	if (fc_fcp_lock_pkt(fsp))
 		goto out;
 
@@ -774,8 +814,10 @@ static void fc_fcp_recv(struct fc_seq *seq, struct fc_frame *fp, void *arg)
 		goto unlock;
 	}
 
-	if (fsp->state & (FC_SRB_ABORTED | FC_SRB_ABORT_PENDING))
+	if (fsp->state & (FC_SRB_ABORTED | FC_SRB_ABORT_PENDING)) {
+		FC_FCP_DBG(fsp, "command aborted, ignoring r_ctl %x\n", r_ctl);
 		goto unlock;
+	}
 
 	if (r_ctl == FC_RCTL_DD_DATA_DESC) {
 		/*
@@ -910,7 +952,16 @@ static void fc_fcp_resp(struct fc_fcp_pkt *fsp, struct fc_frame *fp)
 			 * Wait a at least one jiffy to see if it is delivered.
 			 * If this expires without data, we may do SRR.
 			 */
-			fc_fcp_timer_set(fsp, 2);
+			if (fsp->lp->qfull) {
+				FC_FCP_DBG(fsp, "tgt %6.6x queue busy retry\n",
+					   fsp->rport->port_id);
+				return;
+			}
+			FC_FCP_DBG(fsp, "tgt %6.6x xfer len %zx data underrun "
+				   "len %x, data len %x\n",
+				   fsp->rport->port_id,
+				   fsp->xfer_len, expected_len, fsp->data_len);
+			fc_fcp_timer_set(fsp, get_fsp_rec_tov(fsp));
 			return;
 		}
 		fsp->status_code = FC_DATA_OVRRUN;
@@ -959,8 +1010,11 @@ static void fc_fcp_complete_locked(struct fc_fcp_pkt *fsp)
 		if (fsp->cdb_status == SAM_STAT_GOOD &&
 		    fsp->xfer_len < fsp->data_len && !fsp->io_status &&
 		    (!(fsp->scsi_comp_flags & FCP_RESID_UNDER) ||
-		     fsp->xfer_len < fsp->data_len - fsp->scsi_resid))
+		     fsp->xfer_len < fsp->data_len - fsp->scsi_resid)) {
+			FC_FCP_DBG(fsp, "data underrun, xfer %zx data %x\n",
+				    fsp->xfer_len, fsp->data_len);
 			fsp->status_code = FC_DATA_UNDRUN;
+		}
 	}
 
 	seq = fsp->seq_ptr;
@@ -970,7 +1024,7 @@ static void fc_fcp_complete_locked(struct fc_fcp_pkt *fsp)
 			struct fc_frame *conf_frame;
 			struct fc_seq *csp;
 
-			csp = lport->tt.seq_start_next(seq);
+			csp = fc_seq_start_next(seq);
 			conf_frame = fc_fcp_frame_alloc(fsp->lp, 0);
 			if (conf_frame) {
 				f_ctl = FC_FC_SEQ_INIT;
@@ -979,10 +1033,10 @@ static void fc_fcp_complete_locked(struct fc_fcp_pkt *fsp)
 				fc_fill_fc_hdr(conf_frame, FC_RCTL_DD_SOL_CTL,
 					       ep->did, ep->sid,
 					       FC_TYPE_FCP, f_ctl, 0);
-				lport->tt.seq_send(lport, csp, conf_frame);
+				fc_seq_send(lport, csp, conf_frame);
 			}
 		}
-		lport->tt.exch_done(seq);
+		fc_exch_done(seq);
 	}
 	/*
 	 * Some resets driven by SCSI are not I/Os and do not have
@@ -1000,10 +1054,8 @@ static void fc_fcp_complete_locked(struct fc_fcp_pkt *fsp)
  */
 static void fc_fcp_cleanup_cmd(struct fc_fcp_pkt *fsp, int error)
 {
-	struct fc_lport *lport = fsp->lp;
-
 	if (fsp->seq_ptr) {
-		lport->tt.exch_done(fsp->seq_ptr);
+		fc_exch_done(fsp->seq_ptr);
 		fsp->seq_ptr = NULL;
 	}
 	fsp->status_code = error;
@@ -1116,19 +1168,6 @@ static int fc_fcp_pkt_send(struct fc_lport *lport, struct fc_fcp_pkt *fsp)
 }
 
 /**
- * get_fsp_rec_tov() - Helper function to get REC_TOV
- * @fsp: the FCP packet
- *
- * Returns rec tov in jiffies as rpriv->e_d_tov + 1 second
- */
-static inline unsigned int get_fsp_rec_tov(struct fc_fcp_pkt *fsp)
-{
-	struct fc_rport_libfc_priv *rpriv = fsp->rport->dd_data;
-
-	return msecs_to_jiffies(rpriv->e_d_tov) + HZ;
-}
-
-/**
  * fc_fcp_cmd_send() - Send a FCP command
  * @lport: The local port to send the command on
  * @fsp:   The FCP packet the command is on
@@ -1165,8 +1204,7 @@ static int fc_fcp_cmd_send(struct fc_lport *lport, struct fc_fcp_pkt *fsp,
 		       rpriv->local_port->port_id, FC_TYPE_FCP,
 		       FC_FCTL_REQ, 0);
 
-	seq = lport->tt.exch_seq_send(lport, fp, resp, fc_fcp_pkt_destroy,
-				      fsp, 0);
+	seq = fc_exch_seq_send(lport, fp, resp, fc_fcp_pkt_destroy, fsp, 0);
 	if (!seq) {
 		rc = -1;
 		goto unlock;
@@ -1196,7 +1234,7 @@ static void fc_fcp_error(struct fc_fcp_pkt *fsp, struct fc_frame *fp)
 		return;
 
 	if (error == -FC_EX_CLOSED) {
-		fc_fcp_retry_cmd(fsp);
+		fc_fcp_retry_cmd(fsp, FC_ERROR);
 		goto unlock;
 	}
 
@@ -1222,8 +1260,16 @@ static int fc_fcp_pkt_abort(struct fc_fcp_pkt *fsp)
 	int rc = FAILED;
 	unsigned long ticks_left;
 
-	if (fc_fcp_send_abort(fsp))
+	FC_FCP_DBG(fsp, "pkt abort state %x\n", fsp->state);
+	if (fc_fcp_send_abort(fsp)) {
+		FC_FCP_DBG(fsp, "failed to send abort\n");
 		return FAILED;
+	}
+
+	if (fsp->state & FC_SRB_ABORTED) {
+		FC_FCP_DBG(fsp, "target abort cmd  completed\n");
+		return SUCCESS;
+	}
 
 	init_completion(&fsp->tm_done);
 	fsp->wait_for_comp = 1;
@@ -1301,7 +1347,7 @@ static int fc_lun_reset(struct fc_lport *lport, struct fc_fcp_pkt *fsp,
 
 	spin_lock_bh(&fsp->scsi_pkt_lock);
 	if (fsp->seq_ptr) {
-		lport->tt.exch_done(fsp->seq_ptr);
+		fc_exch_done(fsp->seq_ptr);
 		fsp->seq_ptr = NULL;
 	}
 	fsp->wait_for_comp = 0;
@@ -1355,7 +1401,7 @@ static void fc_tm_done(struct fc_seq *seq, struct fc_frame *fp, void *arg)
 	if (fh->fh_type != FC_TYPE_BLS)
 		fc_fcp_resp(fsp, fp);
 	fsp->seq_ptr = NULL;
-	fsp->lp->tt.exch_done(seq);
+	fc_exch_done(seq);
 out_unlock:
 	fc_fcp_unlock_pkt(fsp);
 out:
@@ -1394,6 +1440,15 @@ static void fc_fcp_timeout(unsigned long data)
 	if (fsp->cdb_cmd.fc_tm_flags)
 		goto unlock;
 
+	if (fsp->lp->qfull) {
+		FC_FCP_DBG(fsp, "fcp timeout, resetting timer delay %d\n",
+			   fsp->timer_delay);
+		setup_timer(&fsp->timer, fc_fcp_timeout, (unsigned long)fsp);
+		fc_fcp_timer_set(fsp, fsp->timer_delay);
+		goto unlock;
+	}
+	FC_FCP_DBG(fsp, "fcp timeout, delay %d flags %x state %x\n",
+		   fsp->timer_delay, rpriv->flags, fsp->state);
 	fsp->state |= FC_SRB_FCP_PROCESSING_TMO;
 
 	if (rpriv->flags & FC_RP_FLAGS_REC_SUPPORTED)
@@ -1486,8 +1541,8 @@ static void fc_fcp_rec_resp(struct fc_seq *seq, struct fc_frame *fp, void *arg)
 		rjt = fc_frame_payload_get(fp, sizeof(*rjt));
 		switch (rjt->er_reason) {
 		default:
-			FC_FCP_DBG(fsp, "device %x unexpected REC reject "
-				   "reason %d expl %d\n",
+			FC_FCP_DBG(fsp,
+				   "device %x invalid REC reject %d/%d\n",
 				   fsp->rport->port_id, rjt->er_reason,
 				   rjt->er_explan);
 			/* fall through */
@@ -1503,18 +1558,23 @@ static void fc_fcp_rec_resp(struct fc_seq *seq, struct fc_frame *fp, void *arg)
 			break;
 		case ELS_RJT_LOGIC:
 		case ELS_RJT_UNAB:
+			FC_FCP_DBG(fsp, "device %x REC reject %d/%d\n",
+				   fsp->rport->port_id, rjt->er_reason,
+				   rjt->er_explan);
 			/*
-			 * If no data transfer, the command frame got dropped
-			 * so we just retry.  If data was transferred, we
-			 * lost the response but the target has no record,
-			 * so we abort and retry.
+			 * If response got lost or is stuck in the
+			 * queue somewhere we have no idea if and when
+			 * the response will be received. So quarantine
+			 * the xid and retry the command.
 			 */
-			if (rjt->er_explan == ELS_EXPL_OXID_RXID &&
-			    fsp->xfer_len == 0) {
-				fc_fcp_retry_cmd(fsp);
+			if (rjt->er_explan == ELS_EXPL_OXID_RXID) {
+				struct fc_exch *ep = fc_seq_exch(fsp->seq_ptr);
+				ep->state |= FC_EX_QUARANTINE;
+				fsp->state |= FC_SRB_ABORTED;
+				fc_fcp_retry_cmd(fsp, FC_TRANS_RESET);
 				break;
 			}
-			fc_fcp_recovery(fsp, FC_ERROR);
+			fc_fcp_recovery(fsp, FC_TRANS_RESET);
 			break;
 		}
 	} else if (opcode == ELS_LS_ACC) {
@@ -1608,7 +1668,9 @@ static void fc_fcp_rec_error(struct fc_fcp_pkt *fsp, struct fc_frame *fp)
 
 	switch (error) {
 	case -FC_EX_CLOSED:
-		fc_fcp_retry_cmd(fsp);
+		FC_FCP_DBG(fsp, "REC %p fid %6.6x exchange closed\n",
+			   fsp, fsp->rport->port_id);
+		fc_fcp_retry_cmd(fsp, FC_ERROR);
 		break;
 
 	default:
@@ -1622,8 +1684,8 @@ static void fc_fcp_rec_error(struct fc_fcp_pkt *fsp, struct fc_frame *fp)
 		 * Assume REC or LS_ACC was lost.
 		 * The exchange manager will have aborted REC, so retry.
 		 */
-		FC_FCP_DBG(fsp, "REC fid %6.6x error error %d retry %d/%d\n",
-			   fsp->rport->port_id, error, fsp->recov_retry,
+		FC_FCP_DBG(fsp, "REC %p fid %6.6x exchange timeout retry %d/%d\n",
+			   fsp, fsp->rport->port_id, fsp->recov_retry,
 			   FC_MAX_RECOV_RETRY);
 		if (fsp->recov_retry++ < FC_MAX_RECOV_RETRY)
 			fc_fcp_rec(fsp);
@@ -1642,6 +1704,7 @@ out:
  */
 static void fc_fcp_recovery(struct fc_fcp_pkt *fsp, u8 code)
 {
+	FC_FCP_DBG(fsp, "start recovery code %x\n", code);
 	fsp->status_code = code;
 	fsp->cdb_status = 0;
 	fsp->io_status = 0;
@@ -1668,7 +1731,6 @@ static void fc_fcp_srr(struct fc_fcp_pkt *fsp, enum fc_rctl r_ctl, u32 offset)
 	struct fc_seq *seq;
 	struct fcp_srr *srr;
 	struct fc_frame *fp;
-	unsigned int rec_tov;
 
 	rport = fsp->rport;
 	rpriv = rport->dd_data;
@@ -1692,10 +1754,9 @@ static void fc_fcp_srr(struct fc_fcp_pkt *fsp, enum fc_rctl r_ctl, u32 offset)
 		       rpriv->local_port->port_id, FC_TYPE_FCP,
 		       FC_FCTL_REQ, 0);
 
-	rec_tov = get_fsp_rec_tov(fsp);
-	seq = lport->tt.exch_seq_send(lport, fp, fc_fcp_srr_resp,
-				      fc_fcp_pkt_destroy,
-				      fsp, jiffies_to_msecs(rec_tov));
+	seq = fc_exch_seq_send(lport, fp, fc_fcp_srr_resp,
+			       fc_fcp_pkt_destroy,
+			       fsp, get_fsp_rec_tov(fsp));
 	if (!seq)
 		goto retry;
 
@@ -1706,7 +1767,7 @@ static void fc_fcp_srr(struct fc_fcp_pkt *fsp, enum fc_rctl r_ctl, u32 offset)
 	fc_fcp_pkt_hold(fsp);		/* hold for outstanding SRR */
 	return;
 retry:
-	fc_fcp_retry_cmd(fsp);
+	fc_fcp_retry_cmd(fsp, FC_TRANS_RESET);
 }
 
 /**
@@ -1730,9 +1791,9 @@ static void fc_fcp_srr_resp(struct fc_seq *seq, struct fc_frame *fp, void *arg)
 
 	fh = fc_frame_header_get(fp);
 	/*
-	 * BUG? fc_fcp_srr_error calls exch_done which would release
+	 * BUG? fc_fcp_srr_error calls fc_exch_done which would release
 	 * the ep. But if fc_fcp_srr_error had got -FC_EX_TIMEOUT,
-	 * then fc_exch_timeout would be sending an abort. The exch_done
+	 * then fc_exch_timeout would be sending an abort. The fc_exch_done
 	 * call by fc_fcp_srr_error would prevent fc_exch.c from seeing
 	 * an abort response though.
 	 */
@@ -1753,7 +1814,7 @@ static void fc_fcp_srr_resp(struct fc_seq *seq, struct fc_frame *fp, void *arg)
 	}
 	fc_fcp_unlock_pkt(fsp);
 out:
-	fsp->lp->tt.exch_done(seq);
+	fc_exch_done(seq);
 	fc_frame_free(fp);
 }
 
@@ -1768,20 +1829,22 @@ static void fc_fcp_srr_error(struct fc_fcp_pkt *fsp, struct fc_frame *fp)
 		goto out;
 	switch (PTR_ERR(fp)) {
 	case -FC_EX_TIMEOUT:
+		FC_FCP_DBG(fsp, "SRR timeout, retries %d\n", fsp->recov_retry);
 		if (fsp->recov_retry++ < FC_MAX_RECOV_RETRY)
 			fc_fcp_rec(fsp);
 		else
 			fc_fcp_recovery(fsp, FC_TIMED_OUT);
 		break;
 	case -FC_EX_CLOSED:			/* e.g., link failure */
+		FC_FCP_DBG(fsp, "SRR error, exchange closed\n");
 		/* fall through */
 	default:
-		fc_fcp_retry_cmd(fsp);
+		fc_fcp_retry_cmd(fsp, FC_ERROR);
 		break;
 	}
 	fc_fcp_unlock_pkt(fsp);
 out:
-	fsp->lp->tt.exch_done(fsp->recov_seq);
+	fc_exch_done(fsp->recov_seq);
 }
 
 /**
@@ -1832,8 +1895,13 @@ int fc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *sc_cmd)
 	rpriv = rport->dd_data;
 
 	if (!fc_fcp_lport_queue_ready(lport)) {
-		if (lport->qfull)
+		if (lport->qfull) {
 			fc_fcp_can_queue_ramp_down(lport);
+			shost_printk(KERN_ERR, lport->host,
+				     "libfc: queue full, "
+				     "reducing can_queue to %d.\n",
+				     lport->host->can_queue);
+		}
 		rc = SCSI_MLQUEUE_HOST_BUSY;
 		goto out;
 	}
@@ -1980,15 +2048,26 @@ static void fc_io_compl(struct fc_fcp_pkt *fsp)
 		sc_cmd->result = (DID_ERROR << 16) | fsp->cdb_status;
 		break;
 	case FC_CMD_ABORTED:
-		FC_FCP_DBG(fsp, "Returning DID_ERROR to scsi-ml "
-			  "due to FC_CMD_ABORTED\n");
-		sc_cmd->result = (DID_ERROR << 16) | fsp->io_status;
+		if (host_byte(sc_cmd->result) == DID_TIME_OUT)
+			FC_FCP_DBG(fsp, "Returning DID_TIME_OUT to scsi-ml "
+				   "due to FC_CMD_ABORTED\n");
+		else {
+			FC_FCP_DBG(fsp, "Returning DID_ERROR to scsi-ml "
+				   "due to FC_CMD_ABORTED\n");
+			set_host_byte(sc_cmd, DID_ERROR);
+		}
+		sc_cmd->result |= fsp->io_status;
 		break;
 	case FC_CMD_RESET:
 		FC_FCP_DBG(fsp, "Returning DID_RESET to scsi-ml "
 			   "due to FC_CMD_RESET\n");
 		sc_cmd->result = (DID_RESET << 16);
 		break;
+	case FC_TRANS_RESET:
+		FC_FCP_DBG(fsp, "Returning DID_SOFT_ERROR to scsi-ml "
+			   "due to FC_TRANS_RESET\n");
+		sc_cmd->result = (DID_SOFT_ERROR << 16);
+		break;
 	case FC_HRD_ERROR:
 		FC_FCP_DBG(fsp, "Returning DID_NO_CONNECT to scsi-ml "
 			   "due to FC_HRD_ERROR\n");
@@ -2142,7 +2221,7 @@ int fc_eh_host_reset(struct scsi_cmnd *sc_cmd)
 
 	fc_block_scsi_eh(sc_cmd);
 
-	lport->tt.lport_reset(lport);
+	fc_lport_reset(lport);
 	wait_tmo = jiffies + FC_HOST_RESET_TIMEOUT;
 	while (!fc_fcp_lport_queue_ready(lport) && time_before(jiffies,
 							       wait_tmo))
diff --git a/drivers/scsi/libfc/fc_libfc.c b/drivers/scsi/libfc/fc_libfc.c
index c11a638f32e6..d623d084b7ec 100644
--- a/drivers/scsi/libfc/fc_libfc.c
+++ b/drivers/scsi/libfc/fc_libfc.c
@@ -226,7 +226,7 @@ void fc_fill_reply_hdr(struct fc_frame *fp, const struct fc_frame *in_fp,
 
 	sp = fr_seq(in_fp);
 	if (sp)
-		fr_seq(fp) = fr_dev(in_fp)->tt.seq_start_next(sp);
+		fr_seq(fp) = fc_seq_start_next(sp);
 	fc_fill_hdr(fp, in_fp, r_ctl, FC_FCTL_RESP, 0, parm_offset);
 }
 EXPORT_SYMBOL(fc_fill_reply_hdr);
diff --git a/drivers/scsi/libfc/fc_lport.c b/drivers/scsi/libfc/fc_lport.c
index 50c71678a156..919736a74ffa 100644
--- a/drivers/scsi/libfc/fc_lport.c
+++ b/drivers/scsi/libfc/fc_lport.c
@@ -149,7 +149,7 @@ static const char *fc_lport_state_names[] = {
  * @offset:   The offset into the response data
  */
 struct fc_bsg_info {
-	struct fc_bsg_job *job;
+	struct bsg_job *job;
 	struct fc_lport *lport;
 	u16 rsp_code;
 	struct scatterlist *sg;
@@ -200,7 +200,7 @@ static void fc_lport_rport_callback(struct fc_lport *lport,
 				     "in the DNS or FDMI state, it's in the "
 				     "%d state", rdata->ids.port_id,
 				     lport->state);
-			lport->tt.rport_logoff(rdata);
+			fc_rport_logoff(rdata);
 		}
 		break;
 	case RPORT_EV_LOGO:
@@ -237,23 +237,26 @@ static const char *fc_lport_state(struct fc_lport *lport)
  * @remote_fid:	 The FID of the ptp rport
  * @remote_wwpn: The WWPN of the ptp rport
  * @remote_wwnn: The WWNN of the ptp rport
+ *
+ * Locking Note: The lport lock is expected to be held before calling
+ * this routine.
  */
 static void fc_lport_ptp_setup(struct fc_lport *lport,
 			       u32 remote_fid, u64 remote_wwpn,
 			       u64 remote_wwnn)
 {
-	mutex_lock(&lport->disc.disc_mutex);
 	if (lport->ptp_rdata) {
-		lport->tt.rport_logoff(lport->ptp_rdata);
-		kref_put(&lport->ptp_rdata->kref, lport->tt.rport_destroy);
+		fc_rport_logoff(lport->ptp_rdata);
+		kref_put(&lport->ptp_rdata->kref, fc_rport_destroy);
 	}
-	lport->ptp_rdata = lport->tt.rport_create(lport, remote_fid);
+	mutex_lock(&lport->disc.disc_mutex);
+	lport->ptp_rdata = fc_rport_create(lport, remote_fid);
 	kref_get(&lport->ptp_rdata->kref);
 	lport->ptp_rdata->ids.port_name = remote_wwpn;
 	lport->ptp_rdata->ids.node_name = remote_wwnn;
 	mutex_unlock(&lport->disc.disc_mutex);
 
-	lport->tt.rport_login(lport->ptp_rdata);
+	fc_rport_login(lport->ptp_rdata);
 
 	fc_lport_enter_ready(lport);
 }
@@ -409,7 +412,7 @@ static void fc_lport_recv_rlir_req(struct fc_lport *lport, struct fc_frame *fp)
 	FC_LPORT_DBG(lport, "Received RLIR request while in state %s\n",
 		     fc_lport_state(lport));
 
-	lport->tt.seq_els_rsp_send(fp, ELS_LS_ACC, NULL);
+	fc_seq_els_rsp_send(fp, ELS_LS_ACC, NULL);
 	fc_frame_free(fp);
 }
 
@@ -478,7 +481,7 @@ static void fc_lport_recv_rnid_req(struct fc_lport *lport,
 	if (!req) {
 		rjt_data.reason = ELS_RJT_LOGIC;
 		rjt_data.explan = ELS_EXPL_NONE;
-		lport->tt.seq_els_rsp_send(in_fp, ELS_LS_RJT, &rjt_data);
+		fc_seq_els_rsp_send(in_fp, ELS_LS_RJT, &rjt_data);
 	} else {
 		fmt = req->rnid_fmt;
 		len = sizeof(*rp);
@@ -518,7 +521,7 @@ static void fc_lport_recv_rnid_req(struct fc_lport *lport,
  */
 static void fc_lport_recv_logo_req(struct fc_lport *lport, struct fc_frame *fp)
 {
-	lport->tt.seq_els_rsp_send(fp, ELS_LS_ACC, NULL);
+	fc_seq_els_rsp_send(fp, ELS_LS_ACC, NULL);
 	fc_lport_enter_reset(lport);
 	fc_frame_free(fp);
 }
@@ -620,9 +623,9 @@ int fc_fabric_logoff(struct fc_lport *lport)
 	lport->tt.disc_stop_final(lport);
 	mutex_lock(&lport->lp_mutex);
 	if (lport->dns_rdata)
-		lport->tt.rport_logoff(lport->dns_rdata);
+		fc_rport_logoff(lport->dns_rdata);
 	mutex_unlock(&lport->lp_mutex);
-	lport->tt.rport_flush_queue();
+	fc_rport_flush_queue();
 	mutex_lock(&lport->lp_mutex);
 	fc_lport_enter_logo(lport);
 	mutex_unlock(&lport->lp_mutex);
@@ -899,7 +902,7 @@ static void fc_lport_recv_els_req(struct fc_lport *lport,
 		/*
 		 * Check opcode.
 		 */
-		recv = lport->tt.rport_recv_req;
+		recv = fc_rport_recv_req;
 		switch (fc_frame_payload_op(fp)) {
 		case ELS_FLOGI:
 			if (!lport->point_to_multipoint)
@@ -941,15 +944,14 @@ struct fc4_prov fc_lport_els_prov = {
 };
 
 /**
- * fc_lport_recv_req() - The generic lport request handler
+ * fc_lport_recv() - The generic lport request handler
  * @lport: The lport that received the request
  * @fp: The frame the request is in
  *
  * Locking Note: This function should not be called with the lport
  *		 lock held because it may grab the lock.
  */
-static void fc_lport_recv_req(struct fc_lport *lport,
-			      struct fc_frame *fp)
+void fc_lport_recv(struct fc_lport *lport, struct fc_frame *fp)
 {
 	struct fc_frame_header *fh = fc_frame_header_get(fp);
 	struct fc_seq *sp = fr_seq(fp);
@@ -978,8 +980,9 @@ drop:
 	FC_LPORT_DBG(lport, "dropping unexpected frame type %x\n", fh->fh_type);
 	fc_frame_free(fp);
 	if (sp)
-		lport->tt.exch_done(sp);
+		fc_exch_done(sp);
 }
+EXPORT_SYMBOL(fc_lport_recv);
 
 /**
  * fc_lport_reset() - Reset a local port
@@ -1007,12 +1010,14 @@ EXPORT_SYMBOL(fc_lport_reset);
  */
 static void fc_lport_reset_locked(struct fc_lport *lport)
 {
-	if (lport->dns_rdata)
-		lport->tt.rport_logoff(lport->dns_rdata);
+	if (lport->dns_rdata) {
+		fc_rport_logoff(lport->dns_rdata);
+		lport->dns_rdata = NULL;
+	}
 
 	if (lport->ptp_rdata) {
-		lport->tt.rport_logoff(lport->ptp_rdata);
-		kref_put(&lport->ptp_rdata->kref, lport->tt.rport_destroy);
+		fc_rport_logoff(lport->ptp_rdata);
+		kref_put(&lport->ptp_rdata->kref, fc_rport_destroy);
 		lport->ptp_rdata = NULL;
 	}
 
@@ -1426,13 +1431,13 @@ static void fc_lport_enter_dns(struct fc_lport *lport)
 	fc_lport_state_enter(lport, LPORT_ST_DNS);
 
 	mutex_lock(&lport->disc.disc_mutex);
-	rdata = lport->tt.rport_create(lport, FC_FID_DIR_SERV);
+	rdata = fc_rport_create(lport, FC_FID_DIR_SERV);
 	mutex_unlock(&lport->disc.disc_mutex);
 	if (!rdata)
 		goto err;
 
 	rdata->ops = &fc_lport_rport_ops;
-	lport->tt.rport_login(rdata);
+	fc_rport_login(rdata);
 	return;
 
 err:
@@ -1543,13 +1548,13 @@ static void fc_lport_enter_fdmi(struct fc_lport *lport)
 	fc_lport_state_enter(lport, LPORT_ST_FDMI);
 
 	mutex_lock(&lport->disc.disc_mutex);
-	rdata = lport->tt.rport_create(lport, FC_FID_MGMT_SERV);
+	rdata = fc_rport_create(lport, FC_FID_MGMT_SERV);
 	mutex_unlock(&lport->disc.disc_mutex);
 	if (!rdata)
 		goto err;
 
 	rdata->ops = &fc_lport_rport_ops;
-	lport->tt.rport_login(rdata);
+	fc_rport_login(rdata);
 	return;
 
 err:
@@ -1772,7 +1777,7 @@ void fc_lport_flogi_resp(struct fc_seq *sp, struct fc_frame *fp,
 	if ((csp_flags & FC_SP_FT_FPORT) == 0) {
 		if (e_d_tov > lport->e_d_tov)
 			lport->e_d_tov = e_d_tov;
-		lport->r_a_tov = 2 * e_d_tov;
+		lport->r_a_tov = 2 * lport->e_d_tov;
 		fc_lport_set_port_id(lport, did, fp);
 		printk(KERN_INFO "host%d: libfc: "
 		       "Port (%6.6x) entered "
@@ -1784,8 +1789,10 @@ void fc_lport_flogi_resp(struct fc_seq *sp, struct fc_frame *fp,
 				   get_unaligned_be64(
 					   &flp->fl_wwnn));
 	} else {
-		lport->e_d_tov = e_d_tov;
-		lport->r_a_tov = r_a_tov;
+		if (e_d_tov > lport->e_d_tov)
+			lport->e_d_tov = e_d_tov;
+		if (r_a_tov > lport->r_a_tov)
+			lport->r_a_tov = r_a_tov;
 		fc_host_fabric_name(lport->host) =
 			get_unaligned_be64(&flp->fl_wwnn);
 		fc_lport_set_port_id(lport, did, fp);
@@ -1858,12 +1865,6 @@ EXPORT_SYMBOL(fc_lport_config);
  */
 int fc_lport_init(struct fc_lport *lport)
 {
-	if (!lport->tt.lport_recv)
-		lport->tt.lport_recv = fc_lport_recv_req;
-
-	if (!lport->tt.lport_reset)
-		lport->tt.lport_reset = fc_lport_reset;
-
 	fc_host_port_type(lport->host) = FC_PORTTYPE_NPORT;
 	fc_host_node_name(lport->host) = lport->wwnn;
 	fc_host_port_name(lport->host) = lport->wwpn;
@@ -1900,18 +1901,19 @@ static void fc_lport_bsg_resp(struct fc_seq *sp, struct fc_frame *fp,
 			      void *info_arg)
 {
 	struct fc_bsg_info *info = info_arg;
-	struct fc_bsg_job *job = info->job;
+	struct bsg_job *job = info->job;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct fc_lport *lport = info->lport;
 	struct fc_frame_header *fh;
 	size_t len;
 	void *buf;
 
 	if (IS_ERR(fp)) {
-		job->reply->result = (PTR_ERR(fp) == -FC_EX_CLOSED) ?
+		bsg_reply->result = (PTR_ERR(fp) == -FC_EX_CLOSED) ?
 			-ECONNABORTED : -ETIMEDOUT;
 		job->reply_len = sizeof(uint32_t);
-		job->state_flags |= FC_RQST_STATE_DONE;
-		job->job_done(job);
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 		kfree(info);
 		return;
 	}
@@ -1928,25 +1930,25 @@ static void fc_lport_bsg_resp(struct fc_seq *sp, struct fc_frame *fp,
 			(unsigned short)fc_frame_payload_op(fp);
 
 		/* Save the reply status of the job */
-		job->reply->reply_data.ctels_reply.status =
+		bsg_reply->reply_data.ctels_reply.status =
 			(cmd == info->rsp_code) ?
 			FC_CTELS_STATUS_OK : FC_CTELS_STATUS_REJECT;
 	}
 
-	job->reply->reply_payload_rcv_len +=
+	bsg_reply->reply_payload_rcv_len +=
 		fc_copy_buffer_to_sglist(buf, len, info->sg, &info->nents,
 					 &info->offset, NULL);
 
 	if (fr_eof(fp) == FC_EOF_T &&
 	    (ntoh24(fh->fh_f_ctl) & (FC_FC_LAST_SEQ | FC_FC_END_SEQ)) ==
 	    (FC_FC_LAST_SEQ | FC_FC_END_SEQ)) {
-		if (job->reply->reply_payload_rcv_len >
+		if (bsg_reply->reply_payload_rcv_len >
 		    job->reply_payload.payload_len)
-			job->reply->reply_payload_rcv_len =
+			bsg_reply->reply_payload_rcv_len =
 				job->reply_payload.payload_len;
-		job->reply->result = 0;
-		job->state_flags |= FC_RQST_STATE_DONE;
-		job->job_done(job);
+		bsg_reply->result = 0;
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 		kfree(info);
 	}
 	fc_frame_free(fp);
@@ -1962,7 +1964,7 @@ static void fc_lport_bsg_resp(struct fc_seq *sp, struct fc_frame *fp,
  * Locking Note: The lport lock is expected to be held before calling
  * this routine.
  */
-static int fc_lport_els_request(struct fc_bsg_job *job,
+static int fc_lport_els_request(struct bsg_job *job,
 				struct fc_lport *lport,
 				u32 did, u32 tov)
 {
@@ -2005,8 +2007,8 @@ static int fc_lport_els_request(struct fc_bsg_job *job,
 	info->nents = job->reply_payload.sg_cnt;
 	info->sg = job->reply_payload.sg_list;
 
-	if (!lport->tt.exch_seq_send(lport, fp, fc_lport_bsg_resp,
-				     NULL, info, tov)) {
+	if (!fc_exch_seq_send(lport, fp, fc_lport_bsg_resp,
+			      NULL, info, tov)) {
 		kfree(info);
 		return -ECOMM;
 	}
@@ -2023,7 +2025,7 @@ static int fc_lport_els_request(struct fc_bsg_job *job,
  * Locking Note: The lport lock is expected to be held before calling
  * this routine.
  */
-static int fc_lport_ct_request(struct fc_bsg_job *job,
+static int fc_lport_ct_request(struct bsg_job *job,
 			       struct fc_lport *lport, u32 did, u32 tov)
 {
 	struct fc_bsg_info *info;
@@ -2066,8 +2068,8 @@ static int fc_lport_ct_request(struct fc_bsg_job *job,
 	info->nents = job->reply_payload.sg_cnt;
 	info->sg = job->reply_payload.sg_list;
 
-	if (!lport->tt.exch_seq_send(lport, fp, fc_lport_bsg_resp,
-				     NULL, info, tov)) {
+	if (!fc_exch_seq_send(lport, fp, fc_lport_bsg_resp,
+			      NULL, info, tov)) {
 		kfree(info);
 		return -ECOMM;
 	}
@@ -2079,25 +2081,27 @@ static int fc_lport_ct_request(struct fc_bsg_job *job,
  *			    FC Passthrough requests
  * @job: The BSG passthrough job
  */
-int fc_lport_bsg_request(struct fc_bsg_job *job)
+int fc_lport_bsg_request(struct bsg_job *job)
 {
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct request *rsp = job->req->next_rq;
-	struct Scsi_Host *shost = job->shost;
+	struct Scsi_Host *shost = fc_bsg_to_shost(job);
 	struct fc_lport *lport = shost_priv(shost);
 	struct fc_rport *rport;
 	struct fc_rport_priv *rdata;
 	int rc = -EINVAL;
 	u32 did, tov;
 
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 	if (rsp)
 		rsp->resid_len = job->reply_payload.payload_len;
 
 	mutex_lock(&lport->lp_mutex);
 
-	switch (job->request->msgcode) {
+	switch (bsg_request->msgcode) {
 	case FC_BSG_RPT_ELS:
-		rport = job->rport;
+		rport = fc_bsg_to_rport(job);
 		if (!rport)
 			break;
 
@@ -2107,7 +2111,7 @@ int fc_lport_bsg_request(struct fc_bsg_job *job)
 		break;
 
 	case FC_BSG_RPT_CT:
-		rport = job->rport;
+		rport = fc_bsg_to_rport(job);
 		if (!rport)
 			break;
 
@@ -2117,25 +2121,25 @@ int fc_lport_bsg_request(struct fc_bsg_job *job)
 		break;
 
 	case FC_BSG_HST_CT:
-		did = ntoh24(job->request->rqst_data.h_ct.port_id);
+		did = ntoh24(bsg_request->rqst_data.h_ct.port_id);
 		if (did == FC_FID_DIR_SERV) {
 			rdata = lport->dns_rdata;
 			if (!rdata)
 				break;
 			tov = rdata->e_d_tov;
 		} else {
-			rdata = lport->tt.rport_lookup(lport, did);
+			rdata = fc_rport_lookup(lport, did);
 			if (!rdata)
 				break;
 			tov = rdata->e_d_tov;
-			kref_put(&rdata->kref, lport->tt.rport_destroy);
+			kref_put(&rdata->kref, fc_rport_destroy);
 		}
 
 		rc = fc_lport_ct_request(job, lport, did, tov);
 		break;
 
 	case FC_BSG_HST_ELS_NOLOGIN:
-		did = ntoh24(job->request->rqst_data.h_els.port_id);
+		did = ntoh24(bsg_request->rqst_data.h_els.port_id);
 		rc = fc_lport_els_request(job, lport, did, lport->e_d_tov);
 		break;
 	}
diff --git a/drivers/scsi/libfc/fc_rport.c b/drivers/scsi/libfc/fc_rport.c
index 97aeaddd600d..c991f3b822f8 100644
--- a/drivers/scsi/libfc/fc_rport.c
+++ b/drivers/scsi/libfc/fc_rport.c
@@ -44,6 +44,19 @@
  * path this potential over-use of the mutex is acceptable.
  */
 
+/*
+ * RPORT REFERENCE COUNTING
+ *
+ * A rport reference should be taken when:
+ * - an rport is allocated
+ * - a workqueue item is scheduled
+ * - an ELS request is send
+ * The reference should be dropped when:
+ * - the workqueue function has finished
+ * - the ELS response is handled
+ * - an rport is removed
+ */
+
 #include <linux/kernel.h>
 #include <linux/spinlock.h>
 #include <linux/interrupt.h>
@@ -74,8 +87,8 @@ static void fc_rport_recv_prli_req(struct fc_rport_priv *, struct fc_frame *);
 static void fc_rport_recv_prlo_req(struct fc_rport_priv *, struct fc_frame *);
 static void fc_rport_recv_logo_req(struct fc_lport *, struct fc_frame *);
 static void fc_rport_timeout(struct work_struct *);
-static void fc_rport_error(struct fc_rport_priv *, struct fc_frame *);
-static void fc_rport_error_retry(struct fc_rport_priv *, struct fc_frame *);
+static void fc_rport_error(struct fc_rport_priv *, int);
+static void fc_rport_error_retry(struct fc_rport_priv *, int);
 static void fc_rport_work(struct work_struct *);
 
 static const char *fc_rport_state_names[] = {
@@ -98,8 +111,8 @@ static const char *fc_rport_state_names[] = {
  * The reference count of the fc_rport_priv structure is
  * increased by one.
  */
-static struct fc_rport_priv *fc_rport_lookup(const struct fc_lport *lport,
-					     u32 port_id)
+struct fc_rport_priv *fc_rport_lookup(const struct fc_lport *lport,
+				      u32 port_id)
 {
 	struct fc_rport_priv *rdata = NULL, *tmp_rdata;
 
@@ -113,6 +126,7 @@ static struct fc_rport_priv *fc_rport_lookup(const struct fc_lport *lport,
 	rcu_read_unlock();
 	return rdata;
 }
+EXPORT_SYMBOL(fc_rport_lookup);
 
 /**
  * fc_rport_create() - Create a new remote port
@@ -123,12 +137,11 @@ static struct fc_rport_priv *fc_rport_lookup(const struct fc_lport *lport,
  *
  * Locking note:  must be called with the disc_mutex held.
  */
-static struct fc_rport_priv *fc_rport_create(struct fc_lport *lport,
-					     u32 port_id)
+struct fc_rport_priv *fc_rport_create(struct fc_lport *lport, u32 port_id)
 {
 	struct fc_rport_priv *rdata;
 
-	rdata = lport->tt.rport_lookup(lport, port_id);
+	rdata = fc_rport_lookup(lport, port_id);
 	if (rdata)
 		return rdata;
 
@@ -158,18 +171,20 @@ static struct fc_rport_priv *fc_rport_create(struct fc_lport *lport,
 	}
 	return rdata;
 }
+EXPORT_SYMBOL(fc_rport_create);
 
 /**
  * fc_rport_destroy() - Free a remote port after last reference is released
  * @kref: The remote port's kref
  */
-static void fc_rport_destroy(struct kref *kref)
+void fc_rport_destroy(struct kref *kref)
 {
 	struct fc_rport_priv *rdata;
 
 	rdata = container_of(kref, struct fc_rport_priv, kref);
 	kfree_rcu(rdata, rcu);
 }
+EXPORT_SYMBOL(fc_rport_destroy);
 
 /**
  * fc_rport_state() - Return a string identifying the remote port's state
@@ -242,6 +257,8 @@ static void fc_rport_state_enter(struct fc_rport_priv *rdata,
 /**
  * fc_rport_work() - Handler for remote port events in the rport_event_queue
  * @work: Handle to the remote port being dequeued
+ *
+ * Reference counting: drops kref on return
  */
 static void fc_rport_work(struct work_struct *work)
 {
@@ -272,12 +289,14 @@ static void fc_rport_work(struct work_struct *work)
 		kref_get(&rdata->kref);
 		mutex_unlock(&rdata->rp_mutex);
 
-		if (!rport)
+		if (!rport) {
+			FC_RPORT_DBG(rdata, "No rport!\n");
 			rport = fc_remote_port_add(lport->host, 0, &ids);
+		}
 		if (!rport) {
 			FC_RPORT_DBG(rdata, "Failed to add the rport\n");
-			lport->tt.rport_logoff(rdata);
-			kref_put(&rdata->kref, lport->tt.rport_destroy);
+			fc_rport_logoff(rdata);
+			kref_put(&rdata->kref, fc_rport_destroy);
 			return;
 		}
 		mutex_lock(&rdata->rp_mutex);
@@ -303,7 +322,7 @@ static void fc_rport_work(struct work_struct *work)
 			FC_RPORT_DBG(rdata, "lld callback ev %d\n", event);
 			rdata->lld_event_callback(lport, rdata, event);
 		}
-		kref_put(&rdata->kref, lport->tt.rport_destroy);
+		kref_put(&rdata->kref, fc_rport_destroy);
 		break;
 
 	case RPORT_EV_FAILED:
@@ -329,7 +348,8 @@ static void fc_rport_work(struct work_struct *work)
 			FC_RPORT_DBG(rdata, "lld callback ev %d\n", event);
 			rdata->lld_event_callback(lport, rdata, event);
 		}
-		cancel_delayed_work_sync(&rdata->retry_work);
+		if (cancel_delayed_work_sync(&rdata->retry_work))
+			kref_put(&rdata->kref, fc_rport_destroy);
 
 		/*
 		 * Reset any outstanding exchanges before freeing rport.
@@ -351,7 +371,7 @@ static void fc_rport_work(struct work_struct *work)
 			if (port_id == FC_FID_DIR_SERV) {
 				rdata->event = RPORT_EV_NONE;
 				mutex_unlock(&rdata->rp_mutex);
-				kref_put(&rdata->kref, lport->tt.rport_destroy);
+				kref_put(&rdata->kref, fc_rport_destroy);
 			} else if ((rdata->flags & FC_RP_STARTED) &&
 				   rdata->major_retries <
 				   lport->max_rport_retry_count) {
@@ -362,17 +382,21 @@ static void fc_rport_work(struct work_struct *work)
 				mutex_unlock(&rdata->rp_mutex);
 			} else {
 				FC_RPORT_DBG(rdata, "work delete\n");
+				mutex_lock(&lport->disc.disc_mutex);
 				list_del_rcu(&rdata->peers);
+				mutex_unlock(&lport->disc.disc_mutex);
 				mutex_unlock(&rdata->rp_mutex);
-				kref_put(&rdata->kref, lport->tt.rport_destroy);
+				kref_put(&rdata->kref, fc_rport_destroy);
 			}
 		} else {
 			/*
 			 * Re-open for events.  Reissue READY event if ready.
 			 */
 			rdata->event = RPORT_EV_NONE;
-			if (rdata->rp_state == RPORT_ST_READY)
+			if (rdata->rp_state == RPORT_ST_READY) {
+				FC_RPORT_DBG(rdata, "work reopen\n");
 				fc_rport_enter_ready(rdata);
+			}
 			mutex_unlock(&rdata->rp_mutex);
 		}
 		break;
@@ -381,12 +405,21 @@ static void fc_rport_work(struct work_struct *work)
 		mutex_unlock(&rdata->rp_mutex);
 		break;
 	}
+	kref_put(&rdata->kref, fc_rport_destroy);
 }
 
 /**
  * fc_rport_login() - Start the remote port login state machine
  * @rdata: The remote port to be logged in to
  *
+ * Initiates the RP state machine. It is called from the LP module.
+ * This function will issue the following commands to the N_Port
+ * identified by the FC ID provided.
+ *
+ * - PLOGI
+ * - PRLI
+ * - RTV
+ *
  * Locking Note: Called without the rport lock held. This
  * function will hold the rport lock, call an _enter_*
  * function and then unlock the rport.
@@ -395,10 +428,16 @@ static void fc_rport_work(struct work_struct *work)
  * If it appears we are already logged in, ADISC is used to verify
  * the setup.
  */
-static int fc_rport_login(struct fc_rport_priv *rdata)
+int fc_rport_login(struct fc_rport_priv *rdata)
 {
 	mutex_lock(&rdata->rp_mutex);
 
+	if (rdata->flags & FC_RP_STARTED) {
+		FC_RPORT_DBG(rdata, "port already started\n");
+		mutex_unlock(&rdata->rp_mutex);
+		return 0;
+	}
+
 	rdata->flags |= FC_RP_STARTED;
 	switch (rdata->rp_state) {
 	case RPORT_ST_READY:
@@ -408,15 +447,20 @@ static int fc_rport_login(struct fc_rport_priv *rdata)
 	case RPORT_ST_DELETE:
 		FC_RPORT_DBG(rdata, "Restart deleted port\n");
 		break;
-	default:
+	case RPORT_ST_INIT:
 		FC_RPORT_DBG(rdata, "Login to port\n");
 		fc_rport_enter_flogi(rdata);
 		break;
+	default:
+		FC_RPORT_DBG(rdata, "Login in progress, state %s\n",
+			     fc_rport_state(rdata));
+		break;
 	}
 	mutex_unlock(&rdata->rp_mutex);
 
 	return 0;
 }
+EXPORT_SYMBOL(fc_rport_login);
 
 /**
  * fc_rport_enter_delete() - Schedule a remote port to be deleted
@@ -431,6 +475,8 @@ static int fc_rport_login(struct fc_rport_priv *rdata)
  * Set the new event so that the old pending event will not occur.
  * Since we have the mutex, even if fc_rport_work() is already started,
  * it'll see the new event.
+ *
+ * Reference counting: does not modify kref
  */
 static void fc_rport_enter_delete(struct fc_rport_priv *rdata,
 				  enum fc_rport_event event)
@@ -442,8 +488,11 @@ static void fc_rport_enter_delete(struct fc_rport_priv *rdata,
 
 	fc_rport_state_enter(rdata, RPORT_ST_DELETE);
 
-	if (rdata->event == RPORT_EV_NONE)
-		queue_work(rport_event_queue, &rdata->event_work);
+	kref_get(&rdata->kref);
+	if (rdata->event == RPORT_EV_NONE &&
+	    !queue_work(rport_event_queue, &rdata->event_work))
+		kref_put(&rdata->kref, fc_rport_destroy);
+
 	rdata->event = event;
 }
 
@@ -455,7 +504,7 @@ static void fc_rport_enter_delete(struct fc_rport_priv *rdata,
  * function will hold the rport lock, call an _enter_*
  * function and then unlock the rport.
  */
-static int fc_rport_logoff(struct fc_rport_priv *rdata)
+int fc_rport_logoff(struct fc_rport_priv *rdata)
 {
 	struct fc_lport *lport = rdata->local_port;
 	u32 port_id = rdata->ids.port_id;
@@ -489,6 +538,7 @@ out:
 	mutex_unlock(&rdata->rp_mutex);
 	return 0;
 }
+EXPORT_SYMBOL(fc_rport_logoff);
 
 /**
  * fc_rport_enter_ready() - Transition to the RPORT_ST_READY state
@@ -496,6 +546,8 @@ out:
  *
  * Locking Note: The rport lock is expected to be held before calling
  * this routine.
+ *
+ * Reference counting: schedules workqueue, does not modify kref
  */
 static void fc_rport_enter_ready(struct fc_rport_priv *rdata)
 {
@@ -503,8 +555,11 @@ static void fc_rport_enter_ready(struct fc_rport_priv *rdata)
 
 	FC_RPORT_DBG(rdata, "Port is Ready\n");
 
-	if (rdata->event == RPORT_EV_NONE)
-		queue_work(rport_event_queue, &rdata->event_work);
+	kref_get(&rdata->kref);
+	if (rdata->event == RPORT_EV_NONE &&
+	    !queue_work(rport_event_queue, &rdata->event_work))
+		kref_put(&rdata->kref, fc_rport_destroy);
+
 	rdata->event = RPORT_EV_READY;
 }
 
@@ -515,6 +570,8 @@ static void fc_rport_enter_ready(struct fc_rport_priv *rdata)
  * Locking Note: Called without the rport lock held. This
  * function will hold the rport lock, call an _enter_*
  * function and then unlock the rport.
+ *
+ * Reference counting: Drops kref on return.
  */
 static void fc_rport_timeout(struct work_struct *work)
 {
@@ -522,6 +579,7 @@ static void fc_rport_timeout(struct work_struct *work)
 		container_of(work, struct fc_rport_priv, retry_work.work);
 
 	mutex_lock(&rdata->rp_mutex);
+	FC_RPORT_DBG(rdata, "Port timeout, state %s\n", fc_rport_state(rdata));
 
 	switch (rdata->rp_state) {
 	case RPORT_ST_FLOGI:
@@ -547,23 +605,25 @@ static void fc_rport_timeout(struct work_struct *work)
 	}
 
 	mutex_unlock(&rdata->rp_mutex);
+	kref_put(&rdata->kref, fc_rport_destroy);
 }
 
 /**
  * fc_rport_error() - Error handler, called once retries have been exhausted
  * @rdata: The remote port the error is happened on
- * @fp:	   The error code encapsulated in a frame pointer
+ * @err:   The error code
  *
  * Locking Note: The rport lock is expected to be held before
  * calling this routine
+ *
+ * Reference counting: does not modify kref
  */
-static void fc_rport_error(struct fc_rport_priv *rdata, struct fc_frame *fp)
+static void fc_rport_error(struct fc_rport_priv *rdata, int err)
 {
 	struct fc_lport *lport = rdata->local_port;
 
-	FC_RPORT_DBG(rdata, "Error %ld in state %s, retries %d\n",
-		     IS_ERR(fp) ? -PTR_ERR(fp) : 0,
-		     fc_rport_state(rdata), rdata->retries);
+	FC_RPORT_DBG(rdata, "Error %d in state %s, retries %d\n",
+		     -err, fc_rport_state(rdata), rdata->retries);
 
 	switch (rdata->rp_state) {
 	case RPORT_ST_FLOGI:
@@ -595,36 +655,39 @@ static void fc_rport_error(struct fc_rport_priv *rdata, struct fc_frame *fp)
 /**
  * fc_rport_error_retry() - Handler for remote port state retries
  * @rdata: The remote port whose state is to be retried
- * @fp:	   The error code encapsulated in a frame pointer
+ * @err:   The error code
  *
  * If the error was an exchange timeout retry immediately,
  * otherwise wait for E_D_TOV.
  *
  * Locking Note: The rport lock is expected to be held before
  * calling this routine
+ *
+ * Reference counting: increments kref when scheduling retry_work
  */
-static void fc_rport_error_retry(struct fc_rport_priv *rdata,
-				 struct fc_frame *fp)
+static void fc_rport_error_retry(struct fc_rport_priv *rdata, int err)
 {
-	unsigned long delay = msecs_to_jiffies(FC_DEF_E_D_TOV);
+	unsigned long delay = msecs_to_jiffies(rdata->e_d_tov);
 
 	/* make sure this isn't an FC_EX_CLOSED error, never retry those */
-	if (PTR_ERR(fp) == -FC_EX_CLOSED)
+	if (err == -FC_EX_CLOSED)
 		goto out;
 
 	if (rdata->retries < rdata->local_port->max_rport_retry_count) {
-		FC_RPORT_DBG(rdata, "Error %ld in state %s, retrying\n",
-			     PTR_ERR(fp), fc_rport_state(rdata));
+		FC_RPORT_DBG(rdata, "Error %d in state %s, retrying\n",
+			     err, fc_rport_state(rdata));
 		rdata->retries++;
 		/* no additional delay on exchange timeouts */
-		if (PTR_ERR(fp) == -FC_EX_TIMEOUT)
+		if (err == -FC_EX_TIMEOUT)
 			delay = 0;
-		schedule_delayed_work(&rdata->retry_work, delay);
+		kref_get(&rdata->kref);
+		if (!schedule_delayed_work(&rdata->retry_work, delay))
+			kref_put(&rdata->kref, fc_rport_destroy);
 		return;
 	}
 
 out:
-	fc_rport_error(rdata, fp);
+	fc_rport_error(rdata, err);
 }
 
 /**
@@ -684,8 +747,11 @@ static void fc_rport_flogi_resp(struct fc_seq *sp, struct fc_frame *fp,
 	struct fc_lport *lport = rdata->local_port;
 	struct fc_els_flogi *flogi;
 	unsigned int r_a_tov;
+	u8 opcode;
+	int err = 0;
 
-	FC_RPORT_DBG(rdata, "Received a FLOGI %s\n", fc_els_resp_type(fp));
+	FC_RPORT_DBG(rdata, "Received a FLOGI %s\n",
+		     IS_ERR(fp) ? "error" : fc_els_resp_type(fp));
 
 	if (fp == ERR_PTR(-FC_EX_CLOSED))
 		goto put;
@@ -701,18 +767,34 @@ static void fc_rport_flogi_resp(struct fc_seq *sp, struct fc_frame *fp,
 	}
 
 	if (IS_ERR(fp)) {
-		fc_rport_error(rdata, fp);
+		fc_rport_error(rdata, PTR_ERR(fp));
 		goto err;
 	}
-
-	if (fc_frame_payload_op(fp) != ELS_LS_ACC)
+	opcode = fc_frame_payload_op(fp);
+	if (opcode == ELS_LS_RJT) {
+		struct fc_els_ls_rjt *rjt;
+
+		rjt = fc_frame_payload_get(fp, sizeof(*rjt));
+		FC_RPORT_DBG(rdata, "FLOGI ELS rejected, reason %x expl %x\n",
+			     rjt->er_reason, rjt->er_explan);
+		err = -FC_EX_ELS_RJT;
 		goto bad;
-	if (fc_rport_login_complete(rdata, fp))
+	} else if (opcode != ELS_LS_ACC) {
+		FC_RPORT_DBG(rdata, "FLOGI ELS invalid opcode %x\n", opcode);
+		err = -FC_EX_ELS_RJT;
 		goto bad;
+	}
+	if (fc_rport_login_complete(rdata, fp)) {
+		FC_RPORT_DBG(rdata, "FLOGI failed, no login\n");
+		err = -FC_EX_INV_LOGIN;
+		goto bad;
+	}
 
 	flogi = fc_frame_payload_get(fp, sizeof(*flogi));
-	if (!flogi)
+	if (!flogi) {
+		err = -FC_EX_ALLOC_ERR;
 		goto bad;
+	}
 	r_a_tov = ntohl(flogi->fl_csp.sp_r_a_tov);
 	if (r_a_tov > rdata->r_a_tov)
 		rdata->r_a_tov = r_a_tov;
@@ -726,11 +808,11 @@ out:
 err:
 	mutex_unlock(&rdata->rp_mutex);
 put:
-	kref_put(&rdata->kref, lport->tt.rport_destroy);
+	kref_put(&rdata->kref, fc_rport_destroy);
 	return;
 bad:
 	FC_RPORT_DBG(rdata, "Bad FLOGI response\n");
-	fc_rport_error_retry(rdata, fp);
+	fc_rport_error_retry(rdata, err);
 	goto out;
 }
 
@@ -740,6 +822,8 @@ bad:
  *
  * Locking Note: The rport lock is expected to be held before calling
  * this routine.
+ *
+ * Reference counting: increments kref when sending ELS
  */
 static void fc_rport_enter_flogi(struct fc_rport_priv *rdata)
 {
@@ -756,20 +840,23 @@ static void fc_rport_enter_flogi(struct fc_rport_priv *rdata)
 
 	fp = fc_frame_alloc(lport, sizeof(struct fc_els_flogi));
 	if (!fp)
-		return fc_rport_error_retry(rdata, fp);
+		return fc_rport_error_retry(rdata, -FC_EX_ALLOC_ERR);
 
+	kref_get(&rdata->kref);
 	if (!lport->tt.elsct_send(lport, rdata->ids.port_id, fp, ELS_FLOGI,
 				  fc_rport_flogi_resp, rdata,
-				  2 * lport->r_a_tov))
-		fc_rport_error_retry(rdata, NULL);
-	else
-		kref_get(&rdata->kref);
+				  2 * lport->r_a_tov)) {
+		fc_rport_error_retry(rdata, -FC_EX_XMIT_ERR);
+		kref_put(&rdata->kref, fc_rport_destroy);
+	}
 }
 
 /**
  * fc_rport_recv_flogi_req() - Handle Fabric Login (FLOGI) request in p-mp mode
  * @lport: The local port that received the PLOGI request
  * @rx_fp: The PLOGI request frame
+ *
+ * Reference counting: drops kref on return
  */
 static void fc_rport_recv_flogi_req(struct fc_lport *lport,
 				    struct fc_frame *rx_fp)
@@ -799,7 +886,7 @@ static void fc_rport_recv_flogi_req(struct fc_lport *lport,
 		goto reject;
 	}
 
-	rdata = lport->tt.rport_lookup(lport, sid);
+	rdata = fc_rport_lookup(lport, sid);
 	if (!rdata) {
 		rjt_data.reason = ELS_RJT_FIP;
 		rjt_data.explan = ELS_EXPL_NOT_NEIGHBOR;
@@ -824,8 +911,7 @@ static void fc_rport_recv_flogi_req(struct fc_lport *lport,
 		 * RPORT wouldn;t have created and 'rport_lookup' would have
 		 * failed anyway in that case.
 		 */
-		if (lport->point_to_multipoint)
-			break;
+		break;
 	case RPORT_ST_DELETE:
 		mutex_unlock(&rdata->rp_mutex);
 		rjt_data.reason = ELS_RJT_FIP;
@@ -867,20 +953,27 @@ static void fc_rport_recv_flogi_req(struct fc_lport *lport,
 	fc_fill_reply_hdr(fp, rx_fp, FC_RCTL_ELS_REP, 0);
 	lport->tt.frame_send(lport, fp);
 
-	if (rdata->ids.port_name < lport->wwpn)
-		fc_rport_enter_plogi(rdata);
-	else
-		fc_rport_state_enter(rdata, RPORT_ST_PLOGI_WAIT);
+	/*
+	 * Do not proceed with the state machine if our
+	 * FLOGI has crossed with an FLOGI from the
+	 * remote port; wait for the FLOGI response instead.
+	 */
+	if (rdata->rp_state != RPORT_ST_FLOGI) {
+		if (rdata->ids.port_name < lport->wwpn)
+			fc_rport_enter_plogi(rdata);
+		else
+			fc_rport_state_enter(rdata, RPORT_ST_PLOGI_WAIT);
+	}
 out:
 	mutex_unlock(&rdata->rp_mutex);
-	kref_put(&rdata->kref, lport->tt.rport_destroy);
+	kref_put(&rdata->kref, fc_rport_destroy);
 	fc_frame_free(rx_fp);
 	return;
 
 reject_put:
-	kref_put(&rdata->kref, lport->tt.rport_destroy);
+	kref_put(&rdata->kref, fc_rport_destroy);
 reject:
-	lport->tt.seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data);
+	fc_seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data);
 	fc_frame_free(rx_fp);
 }
 
@@ -904,10 +997,13 @@ static void fc_rport_plogi_resp(struct fc_seq *sp, struct fc_frame *fp,
 	u16 cssp_seq;
 	u8 op;
 
-	mutex_lock(&rdata->rp_mutex);
-
 	FC_RPORT_DBG(rdata, "Received a PLOGI %s\n", fc_els_resp_type(fp));
 
+	if (fp == ERR_PTR(-FC_EX_CLOSED))
+		goto put;
+
+	mutex_lock(&rdata->rp_mutex);
+
 	if (rdata->rp_state != RPORT_ST_PLOGI) {
 		FC_RPORT_DBG(rdata, "Received a PLOGI response, but in state "
 			     "%s\n", fc_rport_state(rdata));
@@ -917,7 +1013,7 @@ static void fc_rport_plogi_resp(struct fc_seq *sp, struct fc_frame *fp,
 	}
 
 	if (IS_ERR(fp)) {
-		fc_rport_error_retry(rdata, fp);
+		fc_rport_error_retry(rdata, PTR_ERR(fp));
 		goto err;
 	}
 
@@ -939,14 +1035,20 @@ static void fc_rport_plogi_resp(struct fc_seq *sp, struct fc_frame *fp,
 		rdata->max_seq = csp_seq;
 		rdata->maxframe_size = fc_plogi_get_maxframe(plp, lport->mfs);
 		fc_rport_enter_prli(rdata);
-	} else
-		fc_rport_error_retry(rdata, fp);
+	} else {
+		struct fc_els_ls_rjt *rjt;
 
+		rjt = fc_frame_payload_get(fp, sizeof(*rjt));
+		FC_RPORT_DBG(rdata, "PLOGI ELS rejected, reason %x expl %x\n",
+			     rjt->er_reason, rjt->er_explan);
+		fc_rport_error_retry(rdata, -FC_EX_ELS_RJT);
+	}
 out:
 	fc_frame_free(fp);
 err:
 	mutex_unlock(&rdata->rp_mutex);
-	kref_put(&rdata->kref, lport->tt.rport_destroy);
+put:
+	kref_put(&rdata->kref, fc_rport_destroy);
 }
 
 static bool
@@ -969,6 +1071,8 @@ fc_rport_compatible_roles(struct fc_lport *lport, struct fc_rport_priv *rdata)
  *
  * Locking Note: The rport lock is expected to be held before calling
  * this routine.
+ *
+ * Reference counting: increments kref when sending ELS
  */
 static void fc_rport_enter_plogi(struct fc_rport_priv *rdata)
 {
@@ -990,17 +1094,18 @@ static void fc_rport_enter_plogi(struct fc_rport_priv *rdata)
 	fp = fc_frame_alloc(lport, sizeof(struct fc_els_flogi));
 	if (!fp) {
 		FC_RPORT_DBG(rdata, "%s frame alloc failed\n", __func__);
-		fc_rport_error_retry(rdata, fp);
+		fc_rport_error_retry(rdata, -FC_EX_ALLOC_ERR);
 		return;
 	}
 	rdata->e_d_tov = lport->e_d_tov;
 
+	kref_get(&rdata->kref);
 	if (!lport->tt.elsct_send(lport, rdata->ids.port_id, fp, ELS_PLOGI,
 				  fc_rport_plogi_resp, rdata,
-				  2 * lport->r_a_tov))
-		fc_rport_error_retry(rdata, NULL);
-	else
-		kref_get(&rdata->kref);
+				  2 * lport->r_a_tov)) {
+		fc_rport_error_retry(rdata, -FC_EX_XMIT_ERR);
+		kref_put(&rdata->kref, fc_rport_destroy);
+	}
 }
 
 /**
@@ -1022,16 +1127,20 @@ static void fc_rport_prli_resp(struct fc_seq *sp, struct fc_frame *fp,
 		struct fc_els_spp spp;
 	} *pp;
 	struct fc_els_spp temp_spp;
+	struct fc_els_ls_rjt *rjt;
 	struct fc4_prov *prov;
 	u32 roles = FC_RPORT_ROLE_UNKNOWN;
 	u32 fcp_parm = 0;
 	u8 op;
-	u8 resp_code = 0;
-
-	mutex_lock(&rdata->rp_mutex);
+	enum fc_els_spp_resp resp_code;
 
 	FC_RPORT_DBG(rdata, "Received a PRLI %s\n", fc_els_resp_type(fp));
 
+	if (fp == ERR_PTR(-FC_EX_CLOSED))
+		goto put;
+
+	mutex_lock(&rdata->rp_mutex);
+
 	if (rdata->rp_state != RPORT_ST_PRLI) {
 		FC_RPORT_DBG(rdata, "Received a PRLI response, but in state "
 			     "%s\n", fc_rport_state(rdata));
@@ -1041,7 +1150,7 @@ static void fc_rport_prli_resp(struct fc_seq *sp, struct fc_frame *fp,
 	}
 
 	if (IS_ERR(fp)) {
-		fc_rport_error_retry(rdata, fp);
+		fc_rport_error_retry(rdata, PTR_ERR(fp));
 		goto err;
 	}
 
@@ -1055,14 +1164,14 @@ static void fc_rport_prli_resp(struct fc_seq *sp, struct fc_frame *fp,
 			goto out;
 
 		resp_code = (pp->spp.spp_flags & FC_SPP_RESP_MASK);
-		FC_RPORT_DBG(rdata, "PRLI spp_flags = 0x%x\n",
-			     pp->spp.spp_flags);
+		FC_RPORT_DBG(rdata, "PRLI spp_flags = 0x%x spp_type 0x%x\n",
+			     pp->spp.spp_flags, pp->spp.spp_type);
 		rdata->spp_type = pp->spp.spp_type;
 		if (resp_code != FC_SPP_RESP_ACK) {
 			if (resp_code == FC_SPP_RESP_CONF)
-				fc_rport_error(rdata, fp);
+				fc_rport_error(rdata, -FC_EX_SEQ_ERR);
 			else
-				fc_rport_error_retry(rdata, fp);
+				fc_rport_error_retry(rdata, -FC_EX_SEQ_ERR);
 			goto out;
 		}
 		if (pp->prli.prli_spp_len < sizeof(pp->spp))
@@ -1074,13 +1183,25 @@ static void fc_rport_prli_resp(struct fc_seq *sp, struct fc_frame *fp,
 		if (fcp_parm & FCP_SPPF_CONF_COMPL)
 			rdata->flags |= FC_RP_FLAGS_CONF_REQ;
 
-		prov = fc_passive_prov[FC_TYPE_FCP];
+		/*
+		 * Call prli provider if we should act as a target
+		 */
+		prov = fc_passive_prov[rdata->spp_type];
 		if (prov) {
 			memset(&temp_spp, 0, sizeof(temp_spp));
 			prov->prli(rdata, pp->prli.prli_spp_len,
 				   &pp->spp, &temp_spp);
 		}
-
+		/*
+		 * Check if the image pair could be established
+		 */
+		if (rdata->spp_type != FC_TYPE_FCP ||
+		    !(pp->spp.spp_flags & FC_SPP_EST_IMG_PAIR)) {
+			/*
+			 * Nope; we can't use this port as a target.
+			 */
+			fcp_parm &= ~FCP_SPPF_TARG_FCN;
+		}
 		rdata->supported_classes = FC_COS_CLASS3;
 		if (fcp_parm & FCP_SPPF_INIT_FCN)
 			roles |= FC_RPORT_ROLE_FCP_INITIATOR;
@@ -1091,15 +1212,18 @@ static void fc_rport_prli_resp(struct fc_seq *sp, struct fc_frame *fp,
 		fc_rport_enter_rtv(rdata);
 
 	} else {
-		FC_RPORT_DBG(rdata, "Bad ELS response for PRLI command\n");
-		fc_rport_error_retry(rdata, fp);
+		rjt = fc_frame_payload_get(fp, sizeof(*rjt));
+		FC_RPORT_DBG(rdata, "PRLI ELS rejected, reason %x expl %x\n",
+			     rjt->er_reason, rjt->er_explan);
+		fc_rport_error_retry(rdata, FC_EX_ELS_RJT);
 	}
 
 out:
 	fc_frame_free(fp);
 err:
 	mutex_unlock(&rdata->rp_mutex);
-	kref_put(&rdata->kref, rdata->local_port->tt.rport_destroy);
+put:
+	kref_put(&rdata->kref, fc_rport_destroy);
 }
 
 /**
@@ -1108,6 +1232,8 @@ err:
  *
  * Locking Note: The rport lock is expected to be held before calling
  * this routine.
+ *
+ * Reference counting: increments kref when sending ELS
  */
 static void fc_rport_enter_prli(struct fc_rport_priv *rdata)
 {
@@ -1128,6 +1254,15 @@ static void fc_rport_enter_prli(struct fc_rport_priv *rdata)
 		return;
 	}
 
+	/*
+	 * And if the local port does not support the initiator function
+	 * there's no need to send a PRLI, either.
+	 */
+	if (!(lport->service_params & FCP_SPPF_INIT_FCN)) {
+		    fc_rport_enter_ready(rdata);
+		    return;
+	}
+
 	FC_RPORT_DBG(rdata, "Port entered PRLI state from %s state\n",
 		     fc_rport_state(rdata));
 
@@ -1135,7 +1270,7 @@ static void fc_rport_enter_prli(struct fc_rport_priv *rdata)
 
 	fp = fc_frame_alloc(lport, sizeof(*pp));
 	if (!fp) {
-		fc_rport_error_retry(rdata, fp);
+		fc_rport_error_retry(rdata, -FC_EX_ALLOC_ERR);
 		return;
 	}
 
@@ -1151,15 +1286,16 @@ static void fc_rport_enter_prli(struct fc_rport_priv *rdata)
 		       fc_host_port_id(lport->host), FC_TYPE_ELS,
 		       FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT, 0);
 
-	if (!lport->tt.exch_seq_send(lport, fp, fc_rport_prli_resp,
-				    NULL, rdata, 2 * lport->r_a_tov))
-		fc_rport_error_retry(rdata, NULL);
-	else
-		kref_get(&rdata->kref);
+	kref_get(&rdata->kref);
+	if (!fc_exch_seq_send(lport, fp, fc_rport_prli_resp,
+			      NULL, rdata, 2 * lport->r_a_tov)) {
+		fc_rport_error_retry(rdata, -FC_EX_XMIT_ERR);
+		kref_put(&rdata->kref, fc_rport_destroy);
+	}
 }
 
 /**
- * fc_rport_els_rtv_resp() - Handler for Request Timeout Value (RTV) responses
+ * fc_rport_rtv_resp() - Handler for Request Timeout Value (RTV) responses
  * @sp:	       The sequence the RTV was on
  * @fp:	       The RTV response frame
  * @rdata_arg: The remote port that sent the RTV response
@@ -1176,10 +1312,13 @@ static void fc_rport_rtv_resp(struct fc_seq *sp, struct fc_frame *fp,
 	struct fc_rport_priv *rdata = rdata_arg;
 	u8 op;
 
-	mutex_lock(&rdata->rp_mutex);
-
 	FC_RPORT_DBG(rdata, "Received a RTV %s\n", fc_els_resp_type(fp));
 
+	if (fp == ERR_PTR(-FC_EX_CLOSED))
+		goto put;
+
+	mutex_lock(&rdata->rp_mutex);
+
 	if (rdata->rp_state != RPORT_ST_RTV) {
 		FC_RPORT_DBG(rdata, "Received a RTV response, but in state "
 			     "%s\n", fc_rport_state(rdata));
@@ -1189,7 +1328,7 @@ static void fc_rport_rtv_resp(struct fc_seq *sp, struct fc_frame *fp,
 	}
 
 	if (IS_ERR(fp)) {
-		fc_rport_error(rdata, fp);
+		fc_rport_error(rdata, PTR_ERR(fp));
 		goto err;
 	}
 
@@ -1205,13 +1344,15 @@ static void fc_rport_rtv_resp(struct fc_seq *sp, struct fc_frame *fp,
 			tov = ntohl(rtv->rtv_r_a_tov);
 			if (tov == 0)
 				tov = 1;
-			rdata->r_a_tov = tov;
+			if (tov > rdata->r_a_tov)
+				rdata->r_a_tov = tov;
 			tov = ntohl(rtv->rtv_e_d_tov);
 			if (toq & FC_ELS_RTV_EDRES)
 				tov /= 1000000;
 			if (tov == 0)
 				tov = 1;
-			rdata->e_d_tov = tov;
+			if (tov > rdata->e_d_tov)
+				rdata->e_d_tov = tov;
 		}
 	}
 
@@ -1221,7 +1362,8 @@ out:
 	fc_frame_free(fp);
 err:
 	mutex_unlock(&rdata->rp_mutex);
-	kref_put(&rdata->kref, rdata->local_port->tt.rport_destroy);
+put:
+	kref_put(&rdata->kref, fc_rport_destroy);
 }
 
 /**
@@ -1230,6 +1372,8 @@ err:
  *
  * Locking Note: The rport lock is expected to be held before calling
  * this routine.
+ *
+ * Reference counting: increments kref when sending ELS
  */
 static void fc_rport_enter_rtv(struct fc_rport_priv *rdata)
 {
@@ -1243,16 +1387,52 @@ static void fc_rport_enter_rtv(struct fc_rport_priv *rdata)
 
 	fp = fc_frame_alloc(lport, sizeof(struct fc_els_rtv));
 	if (!fp) {
-		fc_rport_error_retry(rdata, fp);
+		fc_rport_error_retry(rdata, -FC_EX_ALLOC_ERR);
 		return;
 	}
 
+	kref_get(&rdata->kref);
 	if (!lport->tt.elsct_send(lport, rdata->ids.port_id, fp, ELS_RTV,
 				  fc_rport_rtv_resp, rdata,
-				  2 * lport->r_a_tov))
-		fc_rport_error_retry(rdata, NULL);
-	else
-		kref_get(&rdata->kref);
+				  2 * lport->r_a_tov)) {
+		fc_rport_error_retry(rdata, -FC_EX_XMIT_ERR);
+		kref_put(&rdata->kref, fc_rport_destroy);
+	}
+}
+
+/**
+ * fc_rport_recv_rtv_req() - Handler for Read Timeout Value (RTV) requests
+ * @rdata: The remote port that sent the RTV request
+ * @in_fp: The RTV request frame
+ *
+ * Locking Note:  Called with the lport and rport locks held.
+ */
+static void fc_rport_recv_rtv_req(struct fc_rport_priv *rdata,
+				  struct fc_frame *in_fp)
+{
+	struct fc_lport *lport = rdata->local_port;
+	struct fc_frame *fp;
+	struct fc_els_rtv_acc *rtv;
+	struct fc_seq_els_data rjt_data;
+
+	FC_RPORT_DBG(rdata, "Received RTV request\n");
+
+	fp = fc_frame_alloc(lport, sizeof(*rtv));
+	if (!fp) {
+		rjt_data.reason = ELS_RJT_UNAB;
+		rjt_data.reason = ELS_EXPL_INSUF_RES;
+		fc_seq_els_rsp_send(in_fp, ELS_LS_RJT, &rjt_data);
+		goto drop;
+	}
+	rtv = fc_frame_payload_get(fp, sizeof(*rtv));
+	rtv->rtv_cmd = ELS_LS_ACC;
+	rtv->rtv_r_a_tov = htonl(lport->r_a_tov);
+	rtv->rtv_e_d_tov = htonl(lport->e_d_tov);
+	rtv->rtv_toq = 0;
+	fc_fill_reply_hdr(fp, in_fp, FC_RCTL_ELS_REP, 0);
+	lport->tt.frame_send(lport, fp);
+drop:
+	fc_frame_free(in_fp);
 }
 
 /**
@@ -1262,15 +1442,16 @@ static void fc_rport_enter_rtv(struct fc_rport_priv *rdata)
  * @lport_arg: The local port
  */
 static void fc_rport_logo_resp(struct fc_seq *sp, struct fc_frame *fp,
-			       void *lport_arg)
+			       void *rdata_arg)
 {
-	struct fc_lport *lport = lport_arg;
+	struct fc_rport_priv *rdata = rdata_arg;
+	struct fc_lport *lport = rdata->local_port;
 
 	FC_RPORT_ID_DBG(lport, fc_seq_exch(sp)->did,
 			"Received a LOGO %s\n", fc_els_resp_type(fp));
-	if (IS_ERR(fp))
-		return;
-	fc_frame_free(fp);
+	if (!IS_ERR(fp))
+		fc_frame_free(fp);
+	kref_put(&rdata->kref, fc_rport_destroy);
 }
 
 /**
@@ -1279,6 +1460,8 @@ static void fc_rport_logo_resp(struct fc_seq *sp, struct fc_frame *fp,
  *
  * Locking Note: The rport lock is expected to be held before calling
  * this routine.
+ *
+ * Reference counting: increments kref when sending ELS
  */
 static void fc_rport_enter_logo(struct fc_rport_priv *rdata)
 {
@@ -1291,8 +1474,10 @@ static void fc_rport_enter_logo(struct fc_rport_priv *rdata)
 	fp = fc_frame_alloc(lport, sizeof(struct fc_els_logo));
 	if (!fp)
 		return;
-	(void)lport->tt.elsct_send(lport, rdata->ids.port_id, fp, ELS_LOGO,
-				   fc_rport_logo_resp, lport, 0);
+	kref_get(&rdata->kref);
+	if (!lport->tt.elsct_send(lport, rdata->ids.port_id, fp, ELS_LOGO,
+				  fc_rport_logo_resp, rdata, 0))
+		kref_put(&rdata->kref, fc_rport_destroy);
 }
 
 /**
@@ -1312,10 +1497,13 @@ static void fc_rport_adisc_resp(struct fc_seq *sp, struct fc_frame *fp,
 	struct fc_els_adisc *adisc;
 	u8 op;
 
-	mutex_lock(&rdata->rp_mutex);
-
 	FC_RPORT_DBG(rdata, "Received a ADISC response\n");
 
+	if (fp == ERR_PTR(-FC_EX_CLOSED))
+		goto put;
+
+	mutex_lock(&rdata->rp_mutex);
+
 	if (rdata->rp_state != RPORT_ST_ADISC) {
 		FC_RPORT_DBG(rdata, "Received a ADISC resp but in state %s\n",
 			     fc_rport_state(rdata));
@@ -1325,7 +1513,7 @@ static void fc_rport_adisc_resp(struct fc_seq *sp, struct fc_frame *fp,
 	}
 
 	if (IS_ERR(fp)) {
-		fc_rport_error(rdata, fp);
+		fc_rport_error(rdata, PTR_ERR(fp));
 		goto err;
 	}
 
@@ -1350,7 +1538,8 @@ out:
 	fc_frame_free(fp);
 err:
 	mutex_unlock(&rdata->rp_mutex);
-	kref_put(&rdata->kref, rdata->local_port->tt.rport_destroy);
+put:
+	kref_put(&rdata->kref, fc_rport_destroy);
 }
 
 /**
@@ -1359,6 +1548,8 @@ err:
  *
  * Locking Note: The rport lock is expected to be held before calling
  * this routine.
+ *
+ * Reference counting: increments kref when sending ELS
  */
 static void fc_rport_enter_adisc(struct fc_rport_priv *rdata)
 {
@@ -1372,15 +1563,16 @@ static void fc_rport_enter_adisc(struct fc_rport_priv *rdata)
 
 	fp = fc_frame_alloc(lport, sizeof(struct fc_els_adisc));
 	if (!fp) {
-		fc_rport_error_retry(rdata, fp);
+		fc_rport_error_retry(rdata, -FC_EX_ALLOC_ERR);
 		return;
 	}
+	kref_get(&rdata->kref);
 	if (!lport->tt.elsct_send(lport, rdata->ids.port_id, fp, ELS_ADISC,
 				  fc_rport_adisc_resp, rdata,
-				  2 * lport->r_a_tov))
-		fc_rport_error_retry(rdata, NULL);
-	else
-		kref_get(&rdata->kref);
+				  2 * lport->r_a_tov)) {
+		fc_rport_error_retry(rdata, -FC_EX_XMIT_ERR);
+		kref_put(&rdata->kref, fc_rport_destroy);
+	}
 }
 
 /**
@@ -1404,7 +1596,7 @@ static void fc_rport_recv_adisc_req(struct fc_rport_priv *rdata,
 	if (!adisc) {
 		rjt_data.reason = ELS_RJT_PROT;
 		rjt_data.explan = ELS_EXPL_INV_LEN;
-		lport->tt.seq_els_rsp_send(in_fp, ELS_LS_RJT, &rjt_data);
+		fc_seq_els_rsp_send(in_fp, ELS_LS_RJT, &rjt_data);
 		goto drop;
 	}
 
@@ -1480,7 +1672,7 @@ static void fc_rport_recv_rls_req(struct fc_rport_priv *rdata,
 	goto out;
 
 out_rjt:
-	lport->tt.seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data);
+	fc_seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data);
 out:
 	fc_frame_free(rx_fp);
 }
@@ -1494,15 +1686,21 @@ out:
  * The ELS opcode has already been validated by the caller.
  *
  * Locking Note: Called with the lport lock held.
+ *
+ * Reference counting: does not modify kref
  */
 static void fc_rport_recv_els_req(struct fc_lport *lport, struct fc_frame *fp)
 {
 	struct fc_rport_priv *rdata;
 	struct fc_seq_els_data els_data;
 
-	rdata = lport->tt.rport_lookup(lport, fc_frame_sid(fp));
-	if (!rdata)
+	rdata = fc_rport_lookup(lport, fc_frame_sid(fp));
+	if (!rdata) {
+		FC_RPORT_ID_DBG(lport, fc_frame_sid(fp),
+				"Received ELS 0x%02x from non-logged-in port\n",
+				fc_frame_payload_op(fp));
 		goto reject;
+	}
 
 	mutex_lock(&rdata->rp_mutex);
 
@@ -1512,9 +1710,21 @@ static void fc_rport_recv_els_req(struct fc_lport *lport, struct fc_frame *fp)
 	case RPORT_ST_READY:
 	case RPORT_ST_ADISC:
 		break;
+	case RPORT_ST_PLOGI:
+		if (fc_frame_payload_op(fp) == ELS_PRLI) {
+			FC_RPORT_DBG(rdata, "Reject ELS PRLI "
+				     "while in state %s\n",
+				     fc_rport_state(rdata));
+			mutex_unlock(&rdata->rp_mutex);
+			kref_put(&rdata->kref, fc_rport_destroy);
+			goto busy;
+		}
 	default:
+		FC_RPORT_DBG(rdata,
+			     "Reject ELS 0x%02x while in state %s\n",
+			     fc_frame_payload_op(fp), fc_rport_state(rdata));
 		mutex_unlock(&rdata->rp_mutex);
-		kref_put(&rdata->kref, lport->tt.rport_destroy);
+		kref_put(&rdata->kref, fc_rport_destroy);
 		goto reject;
 	}
 
@@ -1529,30 +1739,41 @@ static void fc_rport_recv_els_req(struct fc_lport *lport, struct fc_frame *fp)
 		fc_rport_recv_adisc_req(rdata, fp);
 		break;
 	case ELS_RRQ:
-		lport->tt.seq_els_rsp_send(fp, ELS_RRQ, NULL);
+		fc_seq_els_rsp_send(fp, ELS_RRQ, NULL);
 		fc_frame_free(fp);
 		break;
 	case ELS_REC:
-		lport->tt.seq_els_rsp_send(fp, ELS_REC, NULL);
+		fc_seq_els_rsp_send(fp, ELS_REC, NULL);
 		fc_frame_free(fp);
 		break;
 	case ELS_RLS:
 		fc_rport_recv_rls_req(rdata, fp);
 		break;
+	case ELS_RTV:
+		fc_rport_recv_rtv_req(rdata, fp);
+		break;
 	default:
 		fc_frame_free(fp);	/* can't happen */
 		break;
 	}
 
 	mutex_unlock(&rdata->rp_mutex);
-	kref_put(&rdata->kref, rdata->local_port->tt.rport_destroy);
+	kref_put(&rdata->kref, fc_rport_destroy);
 	return;
 
 reject:
 	els_data.reason = ELS_RJT_UNAB;
 	els_data.explan = ELS_EXPL_PLOGI_REQD;
-	lport->tt.seq_els_rsp_send(fp, ELS_LS_RJT, &els_data);
+	fc_seq_els_rsp_send(fp, ELS_LS_RJT, &els_data);
+	fc_frame_free(fp);
+	return;
+
+busy:
+	els_data.reason = ELS_RJT_BUSY;
+	els_data.explan = ELS_EXPL_NONE;
+	fc_seq_els_rsp_send(fp, ELS_LS_RJT, &els_data);
 	fc_frame_free(fp);
+	return;
 }
 
 /**
@@ -1561,8 +1782,10 @@ reject:
  * @fp:	   The request frame
  *
  * Locking Note: Called with the lport lock held.
+ *
+ * Reference counting: does not modify kref
  */
-static void fc_rport_recv_req(struct fc_lport *lport, struct fc_frame *fp)
+void fc_rport_recv_req(struct fc_lport *lport, struct fc_frame *fp)
 {
 	struct fc_seq_els_data els_data;
 
@@ -1588,16 +1811,18 @@ static void fc_rport_recv_req(struct fc_lport *lport, struct fc_frame *fp)
 	case ELS_RRQ:
 	case ELS_REC:
 	case ELS_RLS:
+	case ELS_RTV:
 		fc_rport_recv_els_req(lport, fp);
 		break;
 	default:
 		els_data.reason = ELS_RJT_UNSUP;
 		els_data.explan = ELS_EXPL_NONE;
-		lport->tt.seq_els_rsp_send(fp, ELS_LS_RJT, &els_data);
+		fc_seq_els_rsp_send(fp, ELS_LS_RJT, &els_data);
 		fc_frame_free(fp);
 		break;
 	}
 }
+EXPORT_SYMBOL(fc_rport_recv_req);
 
 /**
  * fc_rport_recv_plogi_req() - Handler for Port Login (PLOGI) requests
@@ -1605,6 +1830,8 @@ static void fc_rport_recv_req(struct fc_lport *lport, struct fc_frame *fp)
  * @rx_fp: The PLOGI request frame
  *
  * Locking Note: The rport lock is held before calling this function.
+ *
+ * Reference counting: increments kref on return
  */
 static void fc_rport_recv_plogi_req(struct fc_lport *lport,
 				    struct fc_frame *rx_fp)
@@ -1630,7 +1857,7 @@ static void fc_rport_recv_plogi_req(struct fc_lport *lport,
 
 	disc = &lport->disc;
 	mutex_lock(&disc->disc_mutex);
-	rdata = lport->tt.rport_create(lport, sid);
+	rdata = fc_rport_create(lport, sid);
 	if (!rdata) {
 		mutex_unlock(&disc->disc_mutex);
 		rjt_data.reason = ELS_RJT_UNAB;
@@ -1718,7 +1945,7 @@ out:
 	return;
 
 reject:
-	lport->tt.seq_els_rsp_send(fp, ELS_LS_RJT, &rjt_data);
+	fc_seq_els_rsp_send(fp, ELS_LS_RJT, &rjt_data);
 	fc_frame_free(fp);
 }
 
@@ -1744,7 +1971,6 @@ static void fc_rport_recv_prli_req(struct fc_rport_priv *rdata,
 	unsigned int len;
 	unsigned int plen;
 	enum fc_els_spp_resp resp;
-	enum fc_els_spp_resp passive;
 	struct fc_seq_els_data rjt_data;
 	struct fc4_prov *prov;
 
@@ -1794,15 +2020,21 @@ static void fc_rport_recv_prli_req(struct fc_rport_priv *rdata,
 		resp = 0;
 
 		if (rspp->spp_type < FC_FC4_PROV_SIZE) {
+			enum fc_els_spp_resp active = 0, passive = 0;
+
 			prov = fc_active_prov[rspp->spp_type];
 			if (prov)
-				resp = prov->prli(rdata, plen, rspp, spp);
+				active = prov->prli(rdata, plen, rspp, spp);
 			prov = fc_passive_prov[rspp->spp_type];
-			if (prov) {
+			if (prov)
 				passive = prov->prli(rdata, plen, rspp, spp);
-				if (!resp || passive == FC_SPP_RESP_ACK)
-					resp = passive;
-			}
+			if (!active || passive == FC_SPP_RESP_ACK)
+				resp = passive;
+			else
+				resp = active;
+			FC_RPORT_DBG(rdata, "PRLI rspp type %x "
+				     "active %x passive %x\n",
+				     rspp->spp_type, active, passive);
 		}
 		if (!resp) {
 			if (spp->spp_flags & FC_SPP_EST_IMG_PAIR)
@@ -1823,20 +2055,13 @@ static void fc_rport_recv_prli_req(struct fc_rport_priv *rdata,
 	fc_fill_reply_hdr(fp, rx_fp, FC_RCTL_ELS_REP, 0);
 	lport->tt.frame_send(lport, fp);
 
-	switch (rdata->rp_state) {
-	case RPORT_ST_PRLI:
-		fc_rport_enter_ready(rdata);
-		break;
-	default:
-		break;
-	}
 	goto drop;
 
 reject_len:
 	rjt_data.reason = ELS_RJT_PROT;
 	rjt_data.explan = ELS_EXPL_INV_LEN;
 reject:
-	lport->tt.seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data);
+	fc_seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data);
 drop:
 	fc_frame_free(rx_fp);
 }
@@ -1907,7 +2132,7 @@ reject_len:
 	rjt_data.reason = ELS_RJT_PROT;
 	rjt_data.explan = ELS_EXPL_INV_LEN;
 reject:
-	lport->tt.seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data);
+	fc_seq_els_rsp_send(rx_fp, ELS_LS_RJT, &rjt_data);
 drop:
 	fc_frame_free(rx_fp);
 }
@@ -1919,17 +2144,19 @@ drop:
  *
  * Locking Note: The rport lock is expected to be held before calling
  * this function.
+ *
+ * Reference counting: drops kref on return
  */
 static void fc_rport_recv_logo_req(struct fc_lport *lport, struct fc_frame *fp)
 {
 	struct fc_rport_priv *rdata;
 	u32 sid;
 
-	lport->tt.seq_els_rsp_send(fp, ELS_LS_ACC, NULL);
+	fc_seq_els_rsp_send(fp, ELS_LS_ACC, NULL);
 
 	sid = fc_frame_sid(fp);
 
-	rdata = lport->tt.rport_lookup(lport, sid);
+	rdata = fc_rport_lookup(lport, sid);
 	if (rdata) {
 		mutex_lock(&rdata->rp_mutex);
 		FC_RPORT_DBG(rdata, "Received LOGO request while in state %s\n",
@@ -1937,7 +2164,7 @@ static void fc_rport_recv_logo_req(struct fc_lport *lport, struct fc_frame *fp)
 
 		fc_rport_enter_delete(rdata, RPORT_EV_STOP);
 		mutex_unlock(&rdata->rp_mutex);
-		kref_put(&rdata->kref, rdata->local_port->tt.rport_destroy);
+		kref_put(&rdata->kref, fc_rport_destroy);
 	} else
 		FC_RPORT_ID_DBG(lport, sid,
 				"Received LOGO from non-logged-in port\n");
@@ -1947,41 +2174,11 @@ static void fc_rport_recv_logo_req(struct fc_lport *lport, struct fc_frame *fp)
 /**
  * fc_rport_flush_queue() - Flush the rport_event_queue
  */
-static void fc_rport_flush_queue(void)
+void fc_rport_flush_queue(void)
 {
 	flush_workqueue(rport_event_queue);
 }
-
-/**
- * fc_rport_init() - Initialize the remote port layer for a local port
- * @lport: The local port to initialize the remote port layer for
- */
-int fc_rport_init(struct fc_lport *lport)
-{
-	if (!lport->tt.rport_lookup)
-		lport->tt.rport_lookup = fc_rport_lookup;
-
-	if (!lport->tt.rport_create)
-		lport->tt.rport_create = fc_rport_create;
-
-	if (!lport->tt.rport_login)
-		lport->tt.rport_login = fc_rport_login;
-
-	if (!lport->tt.rport_logoff)
-		lport->tt.rport_logoff = fc_rport_logoff;
-
-	if (!lport->tt.rport_recv_req)
-		lport->tt.rport_recv_req = fc_rport_recv_req;
-
-	if (!lport->tt.rport_flush_queue)
-		lport->tt.rport_flush_queue = fc_rport_flush_queue;
-
-	if (!lport->tt.rport_destroy)
-		lport->tt.rport_destroy = fc_rport_destroy;
-
-	return 0;
-}
-EXPORT_SYMBOL(fc_rport_init);
+EXPORT_SYMBOL(fc_rport_flush_queue);
 
 /**
  * fc_rport_fcp_prli() - Handle incoming PRLI for the FCP initiator.
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index b484859464f6..8a20b4e86224 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -648,6 +648,10 @@ struct lpfc_hba {
 #define HBA_FCP_IOQ_FLUSH	0x8000 /* FCP I/O queues being flushed */
 #define HBA_FW_DUMP_OP		0x10000 /* Skips fn reset before FW dump */
 #define HBA_RECOVERABLE_UE	0x20000 /* Firmware supports recoverable UE */
+#define HBA_FORCED_LINK_SPEED	0x40000 /*
+					 * Firmware supports Forced Link Speed
+					 * capability
+					 */
 	uint32_t fcp_ring_in_use; /* When polling test if intr-hndlr active*/
 	struct lpfc_dmabuf slim2p;
 
@@ -746,6 +750,8 @@ struct lpfc_hba {
 	uint32_t cfg_oas_priority;
 	uint32_t cfg_XLanePriority;
 	uint32_t cfg_enable_bg;
+	uint32_t cfg_prot_mask;
+	uint32_t cfg_prot_guard;
 	uint32_t cfg_hostmem_hgp;
 	uint32_t cfg_log_verbose;
 	uint32_t cfg_aer_support;
diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index f1019908800e..c84775562c65 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -2759,18 +2759,14 @@ LPFC_ATTR_R(enable_npiv, 1, 0, 1,
 LPFC_ATTR_R(fcf_failover_policy, 1, 1, 2,
 	"FCF Fast failover=1 Priority failover=2");
 
-int lpfc_enable_rrq = 2;
-module_param(lpfc_enable_rrq, int, S_IRUGO);
-MODULE_PARM_DESC(lpfc_enable_rrq, "Enable RRQ functionality");
-lpfc_param_show(enable_rrq);
 /*
 # lpfc_enable_rrq: Track XRI/OXID reuse after IO failures
 #	0x0 = disabled, XRI/OXID use not tracked.
 #	0x1 = XRI/OXID reuse is timed with ratov, RRQ sent.
 #	0x2 = XRI/OXID reuse is timed with ratov, No RRQ sent.
 */
-lpfc_param_init(enable_rrq, 2, 0, 2);
-static DEVICE_ATTR(lpfc_enable_rrq, S_IRUGO, lpfc_enable_rrq_show, NULL);
+LPFC_ATTR_R(enable_rrq, 2, 0, 2,
+	"Enable RRQ functionality");
 
 /*
 # lpfc_suppress_link_up:  Bring link up at initialization
@@ -2827,14 +2823,8 @@ lpfc_txcmplq_hw_show(struct device *dev, struct device_attribute *attr,
 static DEVICE_ATTR(txcmplq_hw, S_IRUGO,
 			 lpfc_txcmplq_hw_show, NULL);
 
-int lpfc_iocb_cnt = 2;
-module_param(lpfc_iocb_cnt, int, S_IRUGO);
-MODULE_PARM_DESC(lpfc_iocb_cnt,
+LPFC_ATTR_R(iocb_cnt, 2, 1, 5,
 	"Number of IOCBs alloc for ELS, CT, and ABTS: 1k to 5k IOCBs");
-lpfc_param_show(iocb_cnt);
-lpfc_param_init(iocb_cnt, 2, 1, 5);
-static DEVICE_ATTR(lpfc_iocb_cnt, S_IRUGO,
-			 lpfc_iocb_cnt_show, NULL);
 
 /*
 # lpfc_nodev_tmo: If set, it will hold all I/O errors on devices that disappear
@@ -2887,9 +2877,9 @@ lpfc_nodev_tmo_init(struct lpfc_vport *vport, int val)
 		vport->cfg_nodev_tmo = vport->cfg_devloss_tmo;
 		if (val != LPFC_DEF_DEVLOSS_TMO)
 			lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
-					 "0407 Ignoring nodev_tmo module "
-					 "parameter because devloss_tmo is "
-					 "set.\n");
+					 "0407 Ignoring lpfc_nodev_tmo module "
+					 "parameter because lpfc_devloss_tmo "
+					 "is set.\n");
 		return 0;
 	}
 
@@ -2948,8 +2938,8 @@ lpfc_nodev_tmo_set(struct lpfc_vport *vport, int val)
 	if (vport->dev_loss_tmo_changed ||
 	    (lpfc_devloss_tmo != LPFC_DEF_DEVLOSS_TMO)) {
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
-				 "0401 Ignoring change to nodev_tmo "
-				 "because devloss_tmo is set.\n");
+				 "0401 Ignoring change to lpfc_nodev_tmo "
+				 "because lpfc_devloss_tmo is set.\n");
 		return 0;
 	}
 	if (val >= LPFC_MIN_DEVLOSS_TMO && val <= LPFC_MAX_DEVLOSS_TMO) {
@@ -2964,7 +2954,7 @@ lpfc_nodev_tmo_set(struct lpfc_vport *vport, int val)
 		return 0;
 	}
 	lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
-			 "0403 lpfc_nodev_tmo attribute cannot be set to"
+			 "0403 lpfc_nodev_tmo attribute cannot be set to "
 			 "%d, allowed range is [%d, %d]\n",
 			 val, LPFC_MIN_DEVLOSS_TMO, LPFC_MAX_DEVLOSS_TMO);
 	return -EINVAL;
@@ -3015,8 +3005,8 @@ lpfc_devloss_tmo_set(struct lpfc_vport *vport, int val)
 	}
 
 	lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
-			 "0404 lpfc_devloss_tmo attribute cannot be set to"
-			 " %d, allowed range is [%d, %d]\n",
+			 "0404 lpfc_devloss_tmo attribute cannot be set to "
+			 "%d, allowed range is [%d, %d]\n",
 			 val, LPFC_MIN_DEVLOSS_TMO, LPFC_MAX_DEVLOSS_TMO);
 	return -EINVAL;
 }
@@ -3204,6 +3194,8 @@ LPFC_VPORT_ATTR_R(scan_down, 1, 0, 1,
 # Set loop mode if you want to run as an NL_Port. Value range is [0,0x6].
 # Default value is 0.
 */
+LPFC_ATTR(topology, 0, 0, 6,
+	"Select Fibre Channel topology");
 
 /**
  * lpfc_topology_set - Set the adapters topology field
@@ -3281,11 +3273,8 @@ lpfc_topology_store(struct device *dev, struct device_attribute *attr,
 		phba->brd_no, val);
 	return -EINVAL;
 }
-static int lpfc_topology = 0;
-module_param(lpfc_topology, int, S_IRUGO);
-MODULE_PARM_DESC(lpfc_topology, "Select Fibre Channel topology");
+
 lpfc_param_show(topology)
-lpfc_param_init(topology, 0, 0, 6)
 static DEVICE_ATTR(lpfc_topology, S_IRUGO | S_IWUSR,
 		lpfc_topology_show, lpfc_topology_store);
 
@@ -3679,7 +3668,12 @@ lpfc_link_speed_store(struct device *dev, struct device_attribute *attr,
 	int nolip = 0;
 	const char *val_buf = buf;
 	int err;
-	uint32_t prev_val;
+	uint32_t prev_val, if_type;
+
+	if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf);
+	if (if_type == LPFC_SLI_INTF_IF_TYPE_2 &&
+	    phba->hba_flag & HBA_FORCED_LINK_SPEED)
+		return -EPERM;
 
 	if (!strncmp(buf, "nolip ", strlen("nolip "))) {
 		nolip = 1;
@@ -3789,6 +3783,9 @@ static DEVICE_ATTR(lpfc_link_speed, S_IRUGO | S_IWUSR,
 #       1  = aer supported and enabled (default)
 # Value range is [0,1]. Default value is 1.
 */
+LPFC_ATTR(aer_support, 1, 0, 1,
+	"Enable PCIe device AER support");
+lpfc_param_show(aer_support)
 
 /**
  * lpfc_aer_support_store - Set the adapter for aer support
@@ -3871,46 +3868,6 @@ lpfc_aer_support_store(struct device *dev, struct device_attribute *attr,
 	return rc;
 }
 
-static int lpfc_aer_support = 1;
-module_param(lpfc_aer_support, int, S_IRUGO);
-MODULE_PARM_DESC(lpfc_aer_support, "Enable PCIe device AER support");
-lpfc_param_show(aer_support)
-
-/**
- * lpfc_aer_support_init - Set the initial adapters aer support flag
- * @phba: lpfc_hba pointer.
- * @val: enable aer or disable aer flag.
- *
- * Description:
- * If val is in a valid range [0,1], then set the adapter's initial
- * cfg_aer_support field. It will be up to the driver's probe_one
- * routine to determine whether the device's AER support can be set
- * or not.
- *
- * Notes:
- * If the value is not in range log a kernel error message, and
- * choose the default value of setting AER support and return.
- *
- * Returns:
- * zero if val saved.
- * -EINVAL val out of range
- **/
-static int
-lpfc_aer_support_init(struct lpfc_hba *phba, int val)
-{
-	if (val == 0 || val == 1) {
-		phba->cfg_aer_support = val;
-		return 0;
-	}
-	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-			"2712 lpfc_aer_support attribute value %d out "
-			"of range, allowed values are 0|1, setting it "
-			"to default value of 1\n", val);
-	/* By default, try to enable AER on a device */
-	phba->cfg_aer_support = 1;
-	return -EINVAL;
-}
-
 static DEVICE_ATTR(lpfc_aer_support, S_IRUGO | S_IWUSR,
 		   lpfc_aer_support_show, lpfc_aer_support_store);
 
@@ -4055,39 +4012,10 @@ lpfc_sriov_nr_virtfn_store(struct device *dev, struct device_attribute *attr,
 	return rc;
 }
 
-static int lpfc_sriov_nr_virtfn = LPFC_DEF_VFN_PER_PFN;
-module_param(lpfc_sriov_nr_virtfn, int, S_IRUGO|S_IWUSR);
-MODULE_PARM_DESC(lpfc_sriov_nr_virtfn, "Enable PCIe device SR-IOV virtual fn");
-lpfc_param_show(sriov_nr_virtfn)
-
-/**
- * lpfc_sriov_nr_virtfn_init - Set the initial sr-iov virtual function enable
- * @phba: lpfc_hba pointer.
- * @val: link speed value.
- *
- * Description:
- * If val is in a valid range [0,255], then set the adapter's initial
- * cfg_sriov_nr_virtfn field. If it's greater than the maximum, the maximum
- * number shall be used instead. It will be up to the driver's probe_one
- * routine to determine whether the device's SR-IOV is supported or not.
- *
- * Returns:
- * zero if val saved.
- * -EINVAL val out of range
- **/
-static int
-lpfc_sriov_nr_virtfn_init(struct lpfc_hba *phba, int val)
-{
-	if (val >= 0 && val <= LPFC_MAX_VFN_PER_PFN) {
-		phba->cfg_sriov_nr_virtfn = val;
-		return 0;
-	}
+LPFC_ATTR(sriov_nr_virtfn, LPFC_DEF_VFN_PER_PFN, 0, LPFC_MAX_VFN_PER_PFN,
+	"Enable PCIe device SR-IOV virtual fn");
 
-	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-			"3017 Enabling %d virtual functions is not "
-			"allowed.\n", val);
-	return -EINVAL;
-}
+lpfc_param_show(sriov_nr_virtfn)
 static DEVICE_ATTR(lpfc_sriov_nr_virtfn, S_IRUGO | S_IWUSR,
 		   lpfc_sriov_nr_virtfn_show, lpfc_sriov_nr_virtfn_store);
 
@@ -4251,7 +4179,8 @@ lpfc_fcp_imax_init(struct lpfc_hba *phba, int val)
 	}
 
 	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-			"3016 fcp_imax: %d out of range, using default\n", val);
+			"3016 lpfc_fcp_imax: %d out of range, using default\n",
+			val);
 	phba->cfg_fcp_imax = LPFC_DEF_IMAX;
 
 	return 0;
@@ -4401,8 +4330,8 @@ lpfc_fcp_cpu_map_init(struct lpfc_hba *phba, int val)
 	}
 
 	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-			"3326 fcp_cpu_map: %d out of range, using default\n",
-			val);
+			"3326 lpfc_fcp_cpu_map: %d out of range, using "
+			"default\n", val);
 	phba->cfg_fcp_cpu_map = LPFC_DRIVER_CPU_MAP;
 
 	return 0;
@@ -4441,12 +4370,10 @@ LPFC_VPORT_ATTR_RW(first_burst_size, 0, 0, 65536,
 # to limit the I/O completion time to the parameter value.
 # The value is set in milliseconds.
 */
-static int lpfc_max_scsicmpl_time;
-module_param(lpfc_max_scsicmpl_time, int, S_IRUGO);
-MODULE_PARM_DESC(lpfc_max_scsicmpl_time,
+LPFC_VPORT_ATTR(max_scsicmpl_time, 0, 0, 60000,
 	"Use command completion time to control queue depth");
+
 lpfc_vport_param_show(max_scsicmpl_time);
-lpfc_vport_param_init(max_scsicmpl_time, 0, 0, 60000);
 static int
 lpfc_max_scsicmpl_time_set(struct lpfc_vport *vport, int val)
 {
@@ -4691,12 +4618,15 @@ unsigned int lpfc_fcp_look_ahead = LPFC_LOOK_AHEAD_OFF;
 #		HBA supports DIX Type 1: Host to HBA  Type 1 protection
 #
 */
-unsigned int lpfc_prot_mask = SHOST_DIF_TYPE1_PROTECTION |
-			      SHOST_DIX_TYPE0_PROTECTION |
-			      SHOST_DIX_TYPE1_PROTECTION;
-
-module_param(lpfc_prot_mask, uint, S_IRUGO);
-MODULE_PARM_DESC(lpfc_prot_mask, "host protection mask");
+LPFC_ATTR(prot_mask,
+	(SHOST_DIF_TYPE1_PROTECTION |
+	SHOST_DIX_TYPE0_PROTECTION |
+	SHOST_DIX_TYPE1_PROTECTION),
+	0,
+	(SHOST_DIF_TYPE1_PROTECTION |
+	SHOST_DIX_TYPE0_PROTECTION |
+	SHOST_DIX_TYPE1_PROTECTION),
+	"T10-DIF host protection capabilities mask");
 
 /*
 # lpfc_prot_guard: i
@@ -4706,9 +4636,9 @@ MODULE_PARM_DESC(lpfc_prot_mask, "host protection mask");
 #	- Default will result in registering capabilities for all guard types
 #
 */
-unsigned char lpfc_prot_guard = SHOST_DIX_GUARD_IP;
-module_param(lpfc_prot_guard, byte, S_IRUGO);
-MODULE_PARM_DESC(lpfc_prot_guard, "host protection guard type");
+LPFC_ATTR(prot_guard,
+	SHOST_DIX_GUARD_IP, SHOST_DIX_GUARD_CRC, SHOST_DIX_GUARD_IP,
+	"T10-DIF host protection guard type");
 
 /*
  * Delay initial NPort discovery when Clean Address bit is cleared in
@@ -5828,6 +5758,8 @@ lpfc_get_cfgparam(struct lpfc_hba *phba)
 	phba->cfg_oas_flags = 0;
 	phba->cfg_oas_priority = 0;
 	lpfc_enable_bg_init(phba, lpfc_enable_bg);
+	lpfc_prot_mask_init(phba, lpfc_prot_mask);
+	lpfc_prot_guard_init(phba, lpfc_prot_guard);
 	if (phba->sli_rev == LPFC_SLI_REV4)
 		phba->cfg_poll = 0;
 	else
diff --git a/drivers/scsi/lpfc/lpfc_bsg.c b/drivers/scsi/lpfc/lpfc_bsg.c
index 05dcc2abd541..7dca4d6a8883 100644
--- a/drivers/scsi/lpfc/lpfc_bsg.c
+++ b/drivers/scsi/lpfc/lpfc_bsg.c
@@ -24,6 +24,7 @@
 #include <linux/slab.h>
 #include <linux/delay.h>
 #include <linux/list.h>
+#include <linux/bsg-lib.h>
 
 #include <scsi/scsi.h>
 #include <scsi/scsi_host.h>
@@ -97,7 +98,7 @@ struct lpfc_bsg_menlo {
 #define TYPE_MENLO	4
 struct bsg_job_data {
 	uint32_t type;
-	struct fc_bsg_job *set_job; /* job waiting for this iocb to finish */
+	struct bsg_job *set_job; /* job waiting for this iocb to finish */
 	union {
 		struct lpfc_bsg_event *evt;
 		struct lpfc_bsg_iocb iocb;
@@ -211,7 +212,7 @@ lpfc_alloc_bsg_buffers(struct lpfc_hba *phba, unsigned int size,
 
 static unsigned int
 lpfc_bsg_copy_data(struct lpfc_dmabuf *dma_buffers,
-		   struct fc_bsg_buffer *bsg_buffers,
+		   struct bsg_buffer *bsg_buffers,
 		   unsigned int bytes_to_transfer, int to_buffers)
 {
 
@@ -297,7 +298,8 @@ lpfc_bsg_send_mgmt_cmd_cmp(struct lpfc_hba *phba,
 			struct lpfc_iocbq *rspiocbq)
 {
 	struct bsg_job_data *dd_data;
-	struct fc_bsg_job *job;
+	struct bsg_job *job;
+	struct fc_bsg_reply *bsg_reply;
 	IOCB_t *rsp;
 	struct lpfc_dmabuf *bmp, *cmp, *rmp;
 	struct lpfc_nodelist *ndlp;
@@ -312,6 +314,7 @@ lpfc_bsg_send_mgmt_cmd_cmp(struct lpfc_hba *phba,
 	spin_lock_irqsave(&phba->ct_ev_lock, flags);
 	job = dd_data->set_job;
 	if (job) {
+		bsg_reply = job->reply;
 		/* Prevent timeout handling from trying to abort job */
 		job->dd_data = NULL;
 	}
@@ -350,7 +353,7 @@ lpfc_bsg_send_mgmt_cmd_cmp(struct lpfc_hba *phba,
 			}
 		} else {
 			rsp_size = rsp->un.genreq64.bdl.bdeSize;
-			job->reply->reply_payload_rcv_len =
+			bsg_reply->reply_payload_rcv_len =
 				lpfc_bsg_copy_data(rmp, &job->reply_payload,
 						   rsp_size, 0);
 		}
@@ -367,8 +370,9 @@ lpfc_bsg_send_mgmt_cmd_cmp(struct lpfc_hba *phba,
 	/* Complete the job if the job is still active */
 
 	if (job) {
-		job->reply->result = rc;
-		job->job_done(job);
+		bsg_reply->result = rc;
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	}
 	return;
 }
@@ -378,12 +382,13 @@ lpfc_bsg_send_mgmt_cmd_cmp(struct lpfc_hba *phba,
  * @job: fc_bsg_job to handle
  **/
 static int
-lpfc_bsg_send_mgmt_cmd(struct fc_bsg_job *job)
+lpfc_bsg_send_mgmt_cmd(struct bsg_job *job)
 {
-	struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata;
+	struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job));
 	struct lpfc_hba *phba = vport->phba;
-	struct lpfc_rport_data *rdata = job->rport->dd_data;
+	struct lpfc_rport_data *rdata = fc_bsg_to_rport(job)->dd_data;
 	struct lpfc_nodelist *ndlp = rdata->pnode;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct ulp_bde64 *bpl = NULL;
 	uint32_t timeout;
 	struct lpfc_iocbq *cmdiocbq = NULL;
@@ -398,7 +403,7 @@ lpfc_bsg_send_mgmt_cmd(struct fc_bsg_job *job)
 	int iocb_stat;
 
 	/* in case no data is transferred */
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 
 	/* allocate our bsg tracking structure */
 	dd_data = kmalloc(sizeof(struct bsg_job_data), GFP_KERNEL);
@@ -542,7 +547,7 @@ no_ndlp:
 	kfree(dd_data);
 no_dd_data:
 	/* make error code available to userspace */
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 	job->dd_data = NULL;
 	return rc;
 }
@@ -570,7 +575,8 @@ lpfc_bsg_rport_els_cmp(struct lpfc_hba *phba,
 			struct lpfc_iocbq *rspiocbq)
 {
 	struct bsg_job_data *dd_data;
-	struct fc_bsg_job *job;
+	struct bsg_job *job;
+	struct fc_bsg_reply *bsg_reply;
 	IOCB_t *rsp;
 	struct lpfc_nodelist *ndlp;
 	struct lpfc_dmabuf *pcmd = NULL, *prsp = NULL;
@@ -588,6 +594,7 @@ lpfc_bsg_rport_els_cmp(struct lpfc_hba *phba,
 	spin_lock_irqsave(&phba->ct_ev_lock, flags);
 	job = dd_data->set_job;
 	if (job) {
+		bsg_reply = job->reply;
 		/* Prevent timeout handling from trying to abort job  */
 		job->dd_data = NULL;
 	}
@@ -609,17 +616,17 @@ lpfc_bsg_rport_els_cmp(struct lpfc_hba *phba,
 	if (job) {
 		if (rsp->ulpStatus == IOSTAT_SUCCESS) {
 			rsp_size = rsp->un.elsreq64.bdl.bdeSize;
-			job->reply->reply_payload_rcv_len =
+			bsg_reply->reply_payload_rcv_len =
 				sg_copy_from_buffer(job->reply_payload.sg_list,
 						    job->reply_payload.sg_cnt,
 						    prsp->virt,
 						    rsp_size);
 		} else if (rsp->ulpStatus == IOSTAT_LS_RJT) {
-			job->reply->reply_payload_rcv_len =
+			bsg_reply->reply_payload_rcv_len =
 				sizeof(struct fc_bsg_ctels_reply);
 			/* LS_RJT data returned in word 4 */
 			rjt_data = (uint8_t *)&rsp->un.ulpWord[4];
-			els_reply = &job->reply->reply_data.ctels_reply;
+			els_reply = &bsg_reply->reply_data.ctels_reply;
 			els_reply->status = FC_CTELS_STATUS_REJECT;
 			els_reply->rjt_data.action = rjt_data[3];
 			els_reply->rjt_data.reason_code = rjt_data[2];
@@ -637,8 +644,9 @@ lpfc_bsg_rport_els_cmp(struct lpfc_hba *phba,
 	/* Complete the job if the job is still active */
 
 	if (job) {
-		job->reply->result = rc;
-		job->job_done(job);
+		bsg_reply->result = rc;
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	}
 	return;
 }
@@ -648,12 +656,14 @@ lpfc_bsg_rport_els_cmp(struct lpfc_hba *phba,
  * @job: fc_bsg_job to handle
  **/
 static int
-lpfc_bsg_rport_els(struct fc_bsg_job *job)
+lpfc_bsg_rport_els(struct bsg_job *job)
 {
-	struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata;
+	struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job));
 	struct lpfc_hba *phba = vport->phba;
-	struct lpfc_rport_data *rdata = job->rport->dd_data;
+	struct lpfc_rport_data *rdata = fc_bsg_to_rport(job)->dd_data;
 	struct lpfc_nodelist *ndlp = rdata->pnode;
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	uint32_t elscmd;
 	uint32_t cmdsize;
 	struct lpfc_iocbq *cmdiocbq;
@@ -664,7 +674,7 @@ lpfc_bsg_rport_els(struct fc_bsg_job *job)
 	int rc = 0;
 
 	/* in case no data is transferred */
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 
 	/* verify the els command is not greater than the
 	 * maximum ELS transfer size.
@@ -684,7 +694,7 @@ lpfc_bsg_rport_els(struct fc_bsg_job *job)
 		goto no_dd_data;
 	}
 
-	elscmd = job->request->rqst_data.r_els.els_code;
+	elscmd = bsg_request->rqst_data.r_els.els_code;
 	cmdsize = job->request_payload.payload_len;
 
 	if (!lpfc_nlp_get(ndlp)) {
@@ -771,7 +781,7 @@ free_dd_data:
 
 no_dd_data:
 	/* make error code available to userspace */
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 	job->dd_data = NULL;
 	return rc;
 }
@@ -917,7 +927,8 @@ lpfc_bsg_ct_unsol_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 	struct lpfc_dmabuf *bdeBuf2 = piocbq->context3;
 	struct lpfc_hbq_entry *hbqe;
 	struct lpfc_sli_ct_request *ct_req;
-	struct fc_bsg_job *job = NULL;
+	struct bsg_job *job = NULL;
+	struct fc_bsg_reply *bsg_reply;
 	struct bsg_job_data *dd_data = NULL;
 	unsigned long flags;
 	int size = 0;
@@ -1120,13 +1131,15 @@ lpfc_bsg_ct_unsol_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 		dd_data->set_job = NULL;
 		lpfc_bsg_event_unref(evt);
 		if (job) {
-			job->reply->reply_payload_rcv_len = size;
+			bsg_reply = job->reply;
+			bsg_reply->reply_payload_rcv_len = size;
 			/* make error code available to userspace */
-			job->reply->result = 0;
+			bsg_reply->result = 0;
 			job->dd_data = NULL;
 			/* complete the job back to userspace */
 			spin_unlock_irqrestore(&phba->ct_ev_lock, flags);
-			job->job_done(job);
+			bsg_job_done(job, bsg_reply->result,
+				       bsg_reply->reply_payload_rcv_len);
 			spin_lock_irqsave(&phba->ct_ev_lock, flags);
 		}
 	}
@@ -1187,10 +1200,11 @@ lpfc_bsg_ct_unsol_abort(struct lpfc_hba *phba, struct hbq_dmabuf *dmabuf)
  * @job: SET_EVENT fc_bsg_job
  **/
 static int
-lpfc_bsg_hba_set_event(struct fc_bsg_job *job)
+lpfc_bsg_hba_set_event(struct bsg_job *job)
 {
-	struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata;
+	struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job));
 	struct lpfc_hba *phba = vport->phba;
+	struct fc_bsg_request *bsg_request = job->request;
 	struct set_ct_event *event_req;
 	struct lpfc_bsg_event *evt;
 	int rc = 0;
@@ -1208,7 +1222,7 @@ lpfc_bsg_hba_set_event(struct fc_bsg_job *job)
 	}
 
 	event_req = (struct set_ct_event *)
-		job->request->rqst_data.h_vendor.vendor_cmd;
+		bsg_request->rqst_data.h_vendor.vendor_cmd;
 	ev_mask = ((uint32_t)(unsigned long)event_req->type_mask &
 				FC_REG_EVENT_MASK);
 	spin_lock_irqsave(&phba->ct_ev_lock, flags);
@@ -1271,10 +1285,12 @@ job_error:
  * @job: GET_EVENT fc_bsg_job
  **/
 static int
-lpfc_bsg_hba_get_event(struct fc_bsg_job *job)
+lpfc_bsg_hba_get_event(struct bsg_job *job)
 {
-	struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata;
+	struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job));
 	struct lpfc_hba *phba = vport->phba;
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct get_ct_event *event_req;
 	struct get_ct_event_reply *event_reply;
 	struct lpfc_bsg_event *evt, *evt_next;
@@ -1292,10 +1308,10 @@ lpfc_bsg_hba_get_event(struct fc_bsg_job *job)
 	}
 
 	event_req = (struct get_ct_event *)
-		job->request->rqst_data.h_vendor.vendor_cmd;
+		bsg_request->rqst_data.h_vendor.vendor_cmd;
 
 	event_reply = (struct get_ct_event_reply *)
-		job->reply->reply_data.vendor_reply.vendor_rsp;
+		bsg_reply->reply_data.vendor_reply.vendor_rsp;
 	spin_lock_irqsave(&phba->ct_ev_lock, flags);
 	list_for_each_entry_safe(evt, evt_next, &phba->ct_ev_waiters, node) {
 		if (evt->reg_id == event_req->ev_reg_id) {
@@ -1315,7 +1331,7 @@ lpfc_bsg_hba_get_event(struct fc_bsg_job *job)
 	 * an error indicating that there isn't anymore
 	 */
 	if (evt_dat == NULL) {
-		job->reply->reply_payload_rcv_len = 0;
+		bsg_reply->reply_payload_rcv_len = 0;
 		rc = -ENOENT;
 		goto job_error;
 	}
@@ -1331,12 +1347,12 @@ lpfc_bsg_hba_get_event(struct fc_bsg_job *job)
 	event_reply->type = evt_dat->type;
 	event_reply->immed_data = evt_dat->immed_dat;
 	if (evt_dat->len > 0)
-		job->reply->reply_payload_rcv_len =
+		bsg_reply->reply_payload_rcv_len =
 			sg_copy_from_buffer(job->request_payload.sg_list,
 					    job->request_payload.sg_cnt,
 					    evt_dat->data, evt_dat->len);
 	else
-		job->reply->reply_payload_rcv_len = 0;
+		bsg_reply->reply_payload_rcv_len = 0;
 
 	if (evt_dat) {
 		kfree(evt_dat->data);
@@ -1347,13 +1363,14 @@ lpfc_bsg_hba_get_event(struct fc_bsg_job *job)
 	lpfc_bsg_event_unref(evt);
 	spin_unlock_irqrestore(&phba->ct_ev_lock, flags);
 	job->dd_data = NULL;
-	job->reply->result = 0;
-	job->job_done(job);
+	bsg_reply->result = 0;
+	bsg_job_done(job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	return 0;
 
 job_error:
 	job->dd_data = NULL;
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 	return rc;
 }
 
@@ -1380,7 +1397,8 @@ lpfc_issue_ct_rsp_cmp(struct lpfc_hba *phba,
 			struct lpfc_iocbq *rspiocbq)
 {
 	struct bsg_job_data *dd_data;
-	struct fc_bsg_job *job;
+	struct bsg_job *job;
+	struct fc_bsg_reply *bsg_reply;
 	IOCB_t *rsp;
 	struct lpfc_dmabuf *bmp, *cmp;
 	struct lpfc_nodelist *ndlp;
@@ -1411,6 +1429,7 @@ lpfc_issue_ct_rsp_cmp(struct lpfc_hba *phba,
 	/* Copy the completed job data or set the error status */
 
 	if (job) {
+		bsg_reply = job->reply;
 		if (rsp->ulpStatus) {
 			if (rsp->ulpStatus == IOSTAT_LOCAL_REJECT) {
 				switch (rsp->un.ulpWord[4] & IOERR_PARAM_MASK) {
@@ -1428,7 +1447,7 @@ lpfc_issue_ct_rsp_cmp(struct lpfc_hba *phba,
 				rc = -EACCES;
 			}
 		} else {
-			job->reply->reply_payload_rcv_len = 0;
+			bsg_reply->reply_payload_rcv_len = 0;
 		}
 	}
 
@@ -1442,8 +1461,9 @@ lpfc_issue_ct_rsp_cmp(struct lpfc_hba *phba,
 	/* Complete the job if the job is still active */
 
 	if (job) {
-		job->reply->result = rc;
-		job->job_done(job);
+		bsg_reply->result = rc;
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	}
 	return;
 }
@@ -1457,7 +1477,7 @@ lpfc_issue_ct_rsp_cmp(struct lpfc_hba *phba,
  * @num_entry: Number of enties in the bde.
  **/
 static int
-lpfc_issue_ct_rsp(struct lpfc_hba *phba, struct fc_bsg_job *job, uint32_t tag,
+lpfc_issue_ct_rsp(struct lpfc_hba *phba, struct bsg_job *job, uint32_t tag,
 		  struct lpfc_dmabuf *cmp, struct lpfc_dmabuf *bmp,
 		  int num_entry)
 {
@@ -1603,12 +1623,14 @@ no_dd_data:
  * @job: SEND_MGMT_RESP fc_bsg_job
  **/
 static int
-lpfc_bsg_send_mgmt_rsp(struct fc_bsg_job *job)
+lpfc_bsg_send_mgmt_rsp(struct bsg_job *job)
 {
-	struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata;
+	struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job));
 	struct lpfc_hba *phba = vport->phba;
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct send_mgmt_resp *mgmt_resp = (struct send_mgmt_resp *)
-		job->request->rqst_data.h_vendor.vendor_cmd;
+		bsg_request->rqst_data.h_vendor.vendor_cmd;
 	struct ulp_bde64 *bpl;
 	struct lpfc_dmabuf *bmp = NULL, *cmp = NULL;
 	int bpl_entries;
@@ -1618,7 +1640,7 @@ lpfc_bsg_send_mgmt_rsp(struct fc_bsg_job *job)
 	int rc = 0;
 
 	/* in case no data is transferred */
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 
 	if (!reqbfrcnt || (reqbfrcnt > (80 * BUF_SZ_4K))) {
 		rc = -ERANGE;
@@ -1664,7 +1686,7 @@ send_mgmt_rsp_free_bmp:
 	kfree(bmp);
 send_mgmt_rsp_exit:
 	/* make error code available to userspace */
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 	job->dd_data = NULL;
 	return rc;
 }
@@ -1760,8 +1782,10 @@ lpfc_bsg_diag_mode_exit(struct lpfc_hba *phba)
  * All of this is done in-line.
  */
 static int
-lpfc_sli3_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct fc_bsg_job *job)
+lpfc_sli3_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct bsg_job *job)
 {
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct diag_mode_set *loopback_mode;
 	uint32_t link_flags;
 	uint32_t timeout;
@@ -1771,7 +1795,7 @@ lpfc_sli3_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct fc_bsg_job *job)
 	int rc = 0;
 
 	/* no data to return just the return code */
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 
 	if (job->request_len < sizeof(struct fc_bsg_request) +
 	    sizeof(struct diag_mode_set)) {
@@ -1791,7 +1815,7 @@ lpfc_sli3_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct fc_bsg_job *job)
 
 	/* bring the link to diagnostic mode */
 	loopback_mode = (struct diag_mode_set *)
-		job->request->rqst_data.h_vendor.vendor_cmd;
+		bsg_request->rqst_data.h_vendor.vendor_cmd;
 	link_flags = loopback_mode->type;
 	timeout = loopback_mode->timeout * 100;
 
@@ -1864,10 +1888,11 @@ loopback_mode_exit:
 
 job_error:
 	/* make error code available to userspace */
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 	/* complete the job back to userspace if no error */
 	if (rc == 0)
-		job->job_done(job);
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	return rc;
 }
 
@@ -2015,14 +2040,16 @@ lpfc_sli4_diag_fcport_reg_setup(struct lpfc_hba *phba)
  * loopback mode in order to perform a diagnostic loopback test.
  */
 static int
-lpfc_sli4_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct fc_bsg_job *job)
+lpfc_sli4_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct bsg_job *job)
 {
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct diag_mode_set *loopback_mode;
 	uint32_t link_flags, timeout;
 	int i, rc = 0;
 
 	/* no data to return just the return code */
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 
 	if (job->request_len < sizeof(struct fc_bsg_request) +
 	    sizeof(struct diag_mode_set)) {
@@ -2054,7 +2081,7 @@ lpfc_sli4_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct fc_bsg_job *job)
 	lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
 			"3129 Bring link to diagnostic state.\n");
 	loopback_mode = (struct diag_mode_set *)
-		job->request->rqst_data.h_vendor.vendor_cmd;
+		bsg_request->rqst_data.h_vendor.vendor_cmd;
 	link_flags = loopback_mode->type;
 	timeout = loopback_mode->timeout * 100;
 
@@ -2151,10 +2178,11 @@ loopback_mode_exit:
 
 job_error:
 	/* make error code available to userspace */
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 	/* complete the job back to userspace if no error */
 	if (rc == 0)
-		job->job_done(job);
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	return rc;
 }
 
@@ -2166,17 +2194,17 @@ job_error:
  * command from the user to proper driver action routines.
  */
 static int
-lpfc_bsg_diag_loopback_mode(struct fc_bsg_job *job)
+lpfc_bsg_diag_loopback_mode(struct bsg_job *job)
 {
 	struct Scsi_Host *shost;
 	struct lpfc_vport *vport;
 	struct lpfc_hba *phba;
 	int rc;
 
-	shost = job->shost;
+	shost = fc_bsg_to_shost(job);
 	if (!shost)
 		return -ENODEV;
-	vport = (struct lpfc_vport *)job->shost->hostdata;
+	vport = shost_priv(shost);
 	if (!vport)
 		return -ENODEV;
 	phba = vport->phba;
@@ -2202,8 +2230,10 @@ lpfc_bsg_diag_loopback_mode(struct fc_bsg_job *job)
  * command from the user to proper driver action routines.
  */
 static int
-lpfc_sli4_bsg_diag_mode_end(struct fc_bsg_job *job)
+lpfc_sli4_bsg_diag_mode_end(struct bsg_job *job)
 {
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct Scsi_Host *shost;
 	struct lpfc_vport *vport;
 	struct lpfc_hba *phba;
@@ -2211,10 +2241,10 @@ lpfc_sli4_bsg_diag_mode_end(struct fc_bsg_job *job)
 	uint32_t timeout;
 	int rc, i;
 
-	shost = job->shost;
+	shost = fc_bsg_to_shost(job);
 	if (!shost)
 		return -ENODEV;
-	vport = (struct lpfc_vport *)job->shost->hostdata;
+	vport = shost_priv(shost);
 	if (!vport)
 		return -ENODEV;
 	phba = vport->phba;
@@ -2232,7 +2262,7 @@ lpfc_sli4_bsg_diag_mode_end(struct fc_bsg_job *job)
 	phba->link_flag &= ~LS_LOOPBACK_MODE;
 	spin_unlock_irq(&phba->hbalock);
 	loopback_mode_end_cmd = (struct diag_mode_set *)
-			job->request->rqst_data.h_vendor.vendor_cmd;
+			bsg_request->rqst_data.h_vendor.vendor_cmd;
 	timeout = loopback_mode_end_cmd->timeout * 100;
 
 	rc = lpfc_sli4_bsg_set_link_diag_state(phba, 0);
@@ -2263,10 +2293,11 @@ lpfc_sli4_bsg_diag_mode_end(struct fc_bsg_job *job)
 
 loopback_mode_end_exit:
 	/* make return code available to userspace */
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 	/* complete the job back to userspace if no error */
 	if (rc == 0)
-		job->job_done(job);
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	return rc;
 }
 
@@ -2278,8 +2309,10 @@ loopback_mode_end_exit:
  * applicaiton.
  */
 static int
-lpfc_sli4_bsg_link_diag_test(struct fc_bsg_job *job)
+lpfc_sli4_bsg_link_diag_test(struct bsg_job *job)
 {
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct Scsi_Host *shost;
 	struct lpfc_vport *vport;
 	struct lpfc_hba *phba;
@@ -2292,12 +2325,12 @@ lpfc_sli4_bsg_link_diag_test(struct fc_bsg_job *job)
 	struct diag_status *diag_status_reply;
 	int mbxstatus, rc = 0;
 
-	shost = job->shost;
+	shost = fc_bsg_to_shost(job);
 	if (!shost) {
 		rc = -ENODEV;
 		goto job_error;
 	}
-	vport = (struct lpfc_vport *)job->shost->hostdata;
+	vport = shost_priv(shost);
 	if (!vport) {
 		rc = -ENODEV;
 		goto job_error;
@@ -2335,7 +2368,7 @@ lpfc_sli4_bsg_link_diag_test(struct fc_bsg_job *job)
 		goto job_error;
 
 	link_diag_test_cmd = (struct sli4_link_diag *)
-			 job->request->rqst_data.h_vendor.vendor_cmd;
+			 bsg_request->rqst_data.h_vendor.vendor_cmd;
 
 	rc = lpfc_sli4_bsg_set_link_diag_state(phba, 1);
 
@@ -2385,7 +2418,7 @@ lpfc_sli4_bsg_link_diag_test(struct fc_bsg_job *job)
 	}
 
 	diag_status_reply = (struct diag_status *)
-			    job->reply->reply_data.vendor_reply.vendor_rsp;
+			    bsg_reply->reply_data.vendor_reply.vendor_rsp;
 
 	if (job->reply_len <
 	    sizeof(struct fc_bsg_request) + sizeof(struct diag_status)) {
@@ -2413,10 +2446,11 @@ link_diag_test_exit:
 
 job_error:
 	/* make error code available to userspace */
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 	/* complete the job back to userspace if no error */
 	if (rc == 0)
-		job->job_done(job);
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	return rc;
 }
 
@@ -2982,9 +3016,10 @@ err_post_rxbufs_exit:
  * of loopback mode.
  **/
 static int
-lpfc_bsg_diag_loopback_run(struct fc_bsg_job *job)
+lpfc_bsg_diag_loopback_run(struct bsg_job *job)
 {
-	struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata;
+	struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job));
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct lpfc_hba *phba = vport->phba;
 	struct lpfc_bsg_event *evt;
 	struct event_data *evdat;
@@ -3012,7 +3047,7 @@ lpfc_bsg_diag_loopback_run(struct fc_bsg_job *job)
 	uint32_t total_mem;
 
 	/* in case no data is returned return just the return code */
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 
 	if (job->request_len <
 	    sizeof(struct fc_bsg_request) + sizeof(struct diag_mode_test)) {
@@ -3237,11 +3272,11 @@ lpfc_bsg_diag_loopback_run(struct fc_bsg_job *job)
 			rc = IOCB_SUCCESS;
 			/* skip over elx loopback header */
 			rx_databuf += ELX_LOOPBACK_HEADER_SZ;
-			job->reply->reply_payload_rcv_len =
+			bsg_reply->reply_payload_rcv_len =
 				sg_copy_from_buffer(job->reply_payload.sg_list,
 						    job->reply_payload.sg_cnt,
 						    rx_databuf, size);
-			job->reply->reply_payload_rcv_len = size;
+			bsg_reply->reply_payload_rcv_len = size;
 		}
 	}
 
@@ -3271,11 +3306,12 @@ err_loopback_test_exit:
 loopback_test_exit:
 	kfree(dataout);
 	/* make error code available to userspace */
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 	job->dd_data = NULL;
 	/* complete the job back to userspace if no error */
 	if (rc == IOCB_SUCCESS)
-		job->job_done(job);
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	return rc;
 }
 
@@ -3284,9 +3320,10 @@ loopback_test_exit:
  * @job: GET_DFC_REV fc_bsg_job
  **/
 static int
-lpfc_bsg_get_dfc_rev(struct fc_bsg_job *job)
+lpfc_bsg_get_dfc_rev(struct bsg_job *job)
 {
-	struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata;
+	struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job));
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct lpfc_hba *phba = vport->phba;
 	struct get_mgmt_rev_reply *event_reply;
 	int rc = 0;
@@ -3301,7 +3338,7 @@ lpfc_bsg_get_dfc_rev(struct fc_bsg_job *job)
 	}
 
 	event_reply = (struct get_mgmt_rev_reply *)
-		job->reply->reply_data.vendor_reply.vendor_rsp;
+		bsg_reply->reply_data.vendor_reply.vendor_rsp;
 
 	if (job->reply_len <
 	    sizeof(struct fc_bsg_request) + sizeof(struct get_mgmt_rev_reply)) {
@@ -3315,9 +3352,10 @@ lpfc_bsg_get_dfc_rev(struct fc_bsg_job *job)
 	event_reply->info.a_Major = MANAGEMENT_MAJOR_REV;
 	event_reply->info.a_Minor = MANAGEMENT_MINOR_REV;
 job_error:
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 	if (rc == 0)
-		job->job_done(job);
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	return rc;
 }
 
@@ -3336,7 +3374,8 @@ static void
 lpfc_bsg_issue_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 {
 	struct bsg_job_data *dd_data;
-	struct fc_bsg_job *job;
+	struct fc_bsg_reply *bsg_reply;
+	struct bsg_job *job;
 	uint32_t size;
 	unsigned long flags;
 	uint8_t *pmb, *pmb_buf;
@@ -3364,8 +3403,9 @@ lpfc_bsg_issue_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 	/* Copy the mailbox data to the job if it is still active */
 
 	if (job) {
+		bsg_reply = job->reply;
 		size = job->reply_payload.payload_len;
-		job->reply->reply_payload_rcv_len =
+		bsg_reply->reply_payload_rcv_len =
 			sg_copy_from_buffer(job->reply_payload.sg_list,
 					    job->reply_payload.sg_cnt,
 					    pmb_buf, size);
@@ -3379,8 +3419,9 @@ lpfc_bsg_issue_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 	/* Complete the job if the job is still active */
 
 	if (job) {
-		job->reply->result = 0;
-		job->job_done(job);
+		bsg_reply->result = 0;
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	}
 	return;
 }
@@ -3510,11 +3551,12 @@ lpfc_bsg_mbox_ext_session_reset(struct lpfc_hba *phba)
  * This is routine handles BSG job for mailbox commands completions with
  * multiple external buffers.
  **/
-static struct fc_bsg_job *
+static struct bsg_job *
 lpfc_bsg_issue_mbox_ext_handle_job(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 {
 	struct bsg_job_data *dd_data;
-	struct fc_bsg_job *job;
+	struct bsg_job *job;
+	struct fc_bsg_reply *bsg_reply;
 	uint8_t *pmb, *pmb_buf;
 	unsigned long flags;
 	uint32_t size;
@@ -3529,6 +3571,7 @@ lpfc_bsg_issue_mbox_ext_handle_job(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 	spin_lock_irqsave(&phba->ct_ev_lock, flags);
 	job = dd_data->set_job;
 	if (job) {
+		bsg_reply = job->reply;
 		/* Prevent timeout handling from trying to abort job  */
 		job->dd_data = NULL;
 	}
@@ -3559,13 +3602,13 @@ lpfc_bsg_issue_mbox_ext_handle_job(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 
 	if (job) {
 		size = job->reply_payload.payload_len;
-		job->reply->reply_payload_rcv_len =
+		bsg_reply->reply_payload_rcv_len =
 			sg_copy_from_buffer(job->reply_payload.sg_list,
 					    job->reply_payload.sg_cnt,
 					    pmb_buf, size);
 
 		/* result for successful */
-		job->reply->result = 0;
+		bsg_reply->result = 0;
 
 		lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
 				"2937 SLI_CONFIG ext-buffer maibox command "
@@ -3603,7 +3646,8 @@ lpfc_bsg_issue_mbox_ext_handle_job(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 static void
 lpfc_bsg_issue_read_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 {
-	struct fc_bsg_job *job;
+	struct bsg_job *job;
+	struct fc_bsg_reply *bsg_reply;
 
 	job = lpfc_bsg_issue_mbox_ext_handle_job(phba, pmboxq);
 
@@ -3623,9 +3667,11 @@ lpfc_bsg_issue_read_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 	mempool_free(pmboxq, phba->mbox_mem_pool);
 
 	/* if the job is still active, call job done */
-	if (job)
-		job->job_done(job);
-
+	if (job) {
+		bsg_reply = job->reply;
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
+	}
 	return;
 }
 
@@ -3640,7 +3686,8 @@ lpfc_bsg_issue_read_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 static void
 lpfc_bsg_issue_write_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 {
-	struct fc_bsg_job *job;
+	struct bsg_job *job;
+	struct fc_bsg_reply *bsg_reply;
 
 	job = lpfc_bsg_issue_mbox_ext_handle_job(phba, pmboxq);
 
@@ -3658,8 +3705,11 @@ lpfc_bsg_issue_write_mbox_ext_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq)
 	lpfc_bsg_mbox_ext_session_reset(phba);
 
 	/* if the job is still active, call job done */
-	if (job)
-		job->job_done(job);
+	if (job) {
+		bsg_reply = job->reply;
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
+	}
 
 	return;
 }
@@ -3768,10 +3818,11 @@ lpfc_bsg_sli_cfg_dma_desc_setup(struct lpfc_hba *phba, enum nemb_type nemb_tp,
  * non-embedded external bufffers.
  **/
 static int
-lpfc_bsg_sli_cfg_read_cmd_ext(struct lpfc_hba *phba, struct fc_bsg_job *job,
+lpfc_bsg_sli_cfg_read_cmd_ext(struct lpfc_hba *phba, struct bsg_job *job,
 			      enum nemb_type nemb_tp,
 			      struct lpfc_dmabuf *dmabuf)
 {
+	struct fc_bsg_request *bsg_request = job->request;
 	struct lpfc_sli_config_mbox *sli_cfg_mbx;
 	struct dfc_mbox_req *mbox_req;
 	struct lpfc_dmabuf *curr_dmabuf, *next_dmabuf;
@@ -3784,7 +3835,7 @@ lpfc_bsg_sli_cfg_read_cmd_ext(struct lpfc_hba *phba, struct fc_bsg_job *job,
 	int rc, i;
 
 	mbox_req =
-	   (struct dfc_mbox_req *)job->request->rqst_data.h_vendor.vendor_cmd;
+	   (struct dfc_mbox_req *)bsg_request->rqst_data.h_vendor.vendor_cmd;
 
 	/* pointer to the start of mailbox command */
 	sli_cfg_mbx = (struct lpfc_sli_config_mbox *)dmabuf->virt;
@@ -3955,10 +4006,12 @@ job_error:
  * non-embedded external bufffers.
  **/
 static int
-lpfc_bsg_sli_cfg_write_cmd_ext(struct lpfc_hba *phba, struct fc_bsg_job *job,
+lpfc_bsg_sli_cfg_write_cmd_ext(struct lpfc_hba *phba, struct bsg_job *job,
 			       enum nemb_type nemb_tp,
 			       struct lpfc_dmabuf *dmabuf)
 {
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct dfc_mbox_req *mbox_req;
 	struct lpfc_sli_config_mbox *sli_cfg_mbx;
 	uint32_t ext_buf_cnt;
@@ -3969,7 +4022,7 @@ lpfc_bsg_sli_cfg_write_cmd_ext(struct lpfc_hba *phba, struct fc_bsg_job *job,
 	int rc = SLI_CONFIG_NOT_HANDLED, i;
 
 	mbox_req =
-	   (struct dfc_mbox_req *)job->request->rqst_data.h_vendor.vendor_cmd;
+	   (struct dfc_mbox_req *)bsg_request->rqst_data.h_vendor.vendor_cmd;
 
 	/* pointer to the start of mailbox command */
 	sli_cfg_mbx = (struct lpfc_sli_config_mbox *)dmabuf->virt;
@@ -4096,8 +4149,9 @@ lpfc_bsg_sli_cfg_write_cmd_ext(struct lpfc_hba *phba, struct fc_bsg_job *job,
 
 	/* wait for additoinal external buffers */
 
-	job->reply->result = 0;
-	job->job_done(job);
+	bsg_reply->result = 0;
+	bsg_job_done(job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	return SLI_CONFIG_HANDLED;
 
 job_error:
@@ -4119,7 +4173,7 @@ job_error:
  * with embedded sussystem 0x1 and opcodes with external HBDs.
  **/
 static int
-lpfc_bsg_handle_sli_cfg_mbox(struct lpfc_hba *phba, struct fc_bsg_job *job,
+lpfc_bsg_handle_sli_cfg_mbox(struct lpfc_hba *phba, struct bsg_job *job,
 			     struct lpfc_dmabuf *dmabuf)
 {
 	struct lpfc_sli_config_mbox *sli_cfg_mbx;
@@ -4268,8 +4322,9 @@ lpfc_bsg_mbox_ext_abort(struct lpfc_hba *phba)
  * user space through BSG.
  **/
 static int
-lpfc_bsg_read_ebuf_get(struct lpfc_hba *phba, struct fc_bsg_job *job)
+lpfc_bsg_read_ebuf_get(struct lpfc_hba *phba, struct bsg_job *job)
 {
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct lpfc_sli_config_mbox *sli_cfg_mbx;
 	struct lpfc_dmabuf *dmabuf;
 	uint8_t *pbuf;
@@ -4307,7 +4362,7 @@ lpfc_bsg_read_ebuf_get(struct lpfc_hba *phba, struct fc_bsg_job *job)
 					dmabuf, index);
 
 	pbuf = (uint8_t *)dmabuf->virt;
-	job->reply->reply_payload_rcv_len =
+	bsg_reply->reply_payload_rcv_len =
 		sg_copy_from_buffer(job->reply_payload.sg_list,
 				    job->reply_payload.sg_cnt,
 				    pbuf, size);
@@ -4321,8 +4376,9 @@ lpfc_bsg_read_ebuf_get(struct lpfc_hba *phba, struct fc_bsg_job *job)
 		lpfc_bsg_mbox_ext_session_reset(phba);
 	}
 
-	job->reply->result = 0;
-	job->job_done(job);
+	bsg_reply->result = 0;
+	bsg_job_done(job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 
 	return SLI_CONFIG_HANDLED;
 }
@@ -4336,9 +4392,10 @@ lpfc_bsg_read_ebuf_get(struct lpfc_hba *phba, struct fc_bsg_job *job)
  * from user space through BSG.
  **/
 static int
-lpfc_bsg_write_ebuf_set(struct lpfc_hba *phba, struct fc_bsg_job *job,
+lpfc_bsg_write_ebuf_set(struct lpfc_hba *phba, struct bsg_job *job,
 			struct lpfc_dmabuf *dmabuf)
 {
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct bsg_job_data *dd_data = NULL;
 	LPFC_MBOXQ_t *pmboxq = NULL;
 	MAILBOX_t *pmb;
@@ -4436,8 +4493,9 @@ lpfc_bsg_write_ebuf_set(struct lpfc_hba *phba, struct fc_bsg_job *job,
 	}
 
 	/* wait for additoinal external buffers */
-	job->reply->result = 0;
-	job->job_done(job);
+	bsg_reply->result = 0;
+	bsg_job_done(job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	return SLI_CONFIG_HANDLED;
 
 job_error:
@@ -4457,7 +4515,7 @@ job_error:
  * command with multiple non-embedded external buffers.
  **/
 static int
-lpfc_bsg_handle_sli_cfg_ebuf(struct lpfc_hba *phba, struct fc_bsg_job *job,
+lpfc_bsg_handle_sli_cfg_ebuf(struct lpfc_hba *phba, struct bsg_job *job,
 			     struct lpfc_dmabuf *dmabuf)
 {
 	int rc;
@@ -4502,14 +4560,15 @@ lpfc_bsg_handle_sli_cfg_ebuf(struct lpfc_hba *phba, struct fc_bsg_job *job,
  * (0x9B) mailbox commands and external buffers.
  **/
 static int
-lpfc_bsg_handle_sli_cfg_ext(struct lpfc_hba *phba, struct fc_bsg_job *job,
+lpfc_bsg_handle_sli_cfg_ext(struct lpfc_hba *phba, struct bsg_job *job,
 			    struct lpfc_dmabuf *dmabuf)
 {
+	struct fc_bsg_request *bsg_request = job->request;
 	struct dfc_mbox_req *mbox_req;
 	int rc = SLI_CONFIG_NOT_HANDLED;
 
 	mbox_req =
-	   (struct dfc_mbox_req *)job->request->rqst_data.h_vendor.vendor_cmd;
+	   (struct dfc_mbox_req *)bsg_request->rqst_data.h_vendor.vendor_cmd;
 
 	/* mbox command with/without single external buffer */
 	if (mbox_req->extMboxTag == 0 && mbox_req->extSeqNum == 0)
@@ -4579,9 +4638,11 @@ sli_cfg_ext_error:
  * let our completion handler finish the command.
  **/
 static int
-lpfc_bsg_issue_mbox(struct lpfc_hba *phba, struct fc_bsg_job *job,
+lpfc_bsg_issue_mbox(struct lpfc_hba *phba, struct bsg_job *job,
 	struct lpfc_vport *vport)
 {
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	LPFC_MBOXQ_t *pmboxq = NULL; /* internal mailbox queue */
 	MAILBOX_t *pmb; /* shortcut to the pmboxq mailbox */
 	/* a 4k buffer to hold the mb and extended data from/to the bsg */
@@ -4600,7 +4661,7 @@ lpfc_bsg_issue_mbox(struct lpfc_hba *phba, struct fc_bsg_job *job,
 	uint32_t size;
 
 	/* in case no data is transferred */
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 
 	/* sanity check to protect driver */
 	if (job->reply_payload.payload_len > BSG_MBOX_SIZE ||
@@ -4619,7 +4680,7 @@ lpfc_bsg_issue_mbox(struct lpfc_hba *phba, struct fc_bsg_job *job,
 	}
 
 	mbox_req =
-	    (struct dfc_mbox_req *)job->request->rqst_data.h_vendor.vendor_cmd;
+	    (struct dfc_mbox_req *)bsg_request->rqst_data.h_vendor.vendor_cmd;
 
 	/* check if requested extended data lengths are valid */
 	if ((mbox_req->inExtWLen > BSG_MBOX_SIZE/sizeof(uint32_t)) ||
@@ -4841,7 +4902,7 @@ lpfc_bsg_issue_mbox(struct lpfc_hba *phba, struct fc_bsg_job *job,
 
 		/* job finished, copy the data */
 		memcpy(pmbx, pmb, sizeof(*pmb));
-		job->reply->reply_payload_rcv_len =
+		bsg_reply->reply_payload_rcv_len =
 			sg_copy_from_buffer(job->reply_payload.sg_list,
 					    job->reply_payload.sg_cnt,
 					    pmbx, size);
@@ -4870,15 +4931,17 @@ job_cont:
  * @job: MBOX fc_bsg_job for LPFC_BSG_VENDOR_MBOX.
  **/
 static int
-lpfc_bsg_mbox_cmd(struct fc_bsg_job *job)
+lpfc_bsg_mbox_cmd(struct bsg_job *job)
 {
-	struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata;
+	struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job));
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct lpfc_hba *phba = vport->phba;
 	struct dfc_mbox_req *mbox_req;
 	int rc = 0;
 
 	/* mix-and-match backward compatibility */
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 	if (job->request_len <
 	    sizeof(struct fc_bsg_request) + sizeof(struct dfc_mbox_req)) {
 		lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
@@ -4889,7 +4952,7 @@ lpfc_bsg_mbox_cmd(struct fc_bsg_job *job)
 				      sizeof(struct fc_bsg_request)),
 				(int)sizeof(struct dfc_mbox_req));
 		mbox_req = (struct dfc_mbox_req *)
-				job->request->rqst_data.h_vendor.vendor_cmd;
+				bsg_request->rqst_data.h_vendor.vendor_cmd;
 		mbox_req->extMboxTag = 0;
 		mbox_req->extSeqNum = 0;
 	}
@@ -4898,15 +4961,16 @@ lpfc_bsg_mbox_cmd(struct fc_bsg_job *job)
 
 	if (rc == 0) {
 		/* job done */
-		job->reply->result = 0;
+		bsg_reply->result = 0;
 		job->dd_data = NULL;
-		job->job_done(job);
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	} else if (rc == 1)
 		/* job submitted, will complete later*/
 		rc = 0; /* return zero, no error */
 	else {
 		/* some error occurred */
-		job->reply->result = rc;
+		bsg_reply->result = rc;
 		job->dd_data = NULL;
 	}
 
@@ -4936,7 +5000,8 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba,
 			struct lpfc_iocbq *rspiocbq)
 {
 	struct bsg_job_data *dd_data;
-	struct fc_bsg_job *job;
+	struct bsg_job *job;
+	struct fc_bsg_reply *bsg_reply;
 	IOCB_t *rsp;
 	struct lpfc_dmabuf *bmp, *cmp, *rmp;
 	struct lpfc_bsg_menlo *menlo;
@@ -4956,6 +5021,7 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba,
 	spin_lock_irqsave(&phba->ct_ev_lock, flags);
 	job = dd_data->set_job;
 	if (job) {
+		bsg_reply = job->reply;
 		/* Prevent timeout handling from trying to abort job  */
 		job->dd_data = NULL;
 	}
@@ -4970,7 +5036,7 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba,
 		 */
 
 		menlo_resp = (struct menlo_response *)
-			job->reply->reply_data.vendor_reply.vendor_rsp;
+			bsg_reply->reply_data.vendor_reply.vendor_rsp;
 		menlo_resp->xri = rsp->ulpContext;
 		if (rsp->ulpStatus) {
 			if (rsp->ulpStatus == IOSTAT_LOCAL_REJECT) {
@@ -4990,7 +5056,7 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba,
 			}
 		} else {
 			rsp_size = rsp->un.genreq64.bdl.bdeSize;
-			job->reply->reply_payload_rcv_len =
+			bsg_reply->reply_payload_rcv_len =
 				lpfc_bsg_copy_data(rmp, &job->reply_payload,
 						   rsp_size, 0);
 		}
@@ -5007,8 +5073,9 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba,
 	/* Complete the job if active */
 
 	if (job) {
-		job->reply->result = rc;
-		job->job_done(job);
+		bsg_reply->result = rc;
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	}
 
 	return;
@@ -5024,9 +5091,11 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba,
  * supplied in the menlo request header xri field.
  **/
 static int
-lpfc_menlo_cmd(struct fc_bsg_job *job)
+lpfc_menlo_cmd(struct bsg_job *job)
 {
-	struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata;
+	struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job));
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct lpfc_hba *phba = vport->phba;
 	struct lpfc_iocbq *cmdiocbq;
 	IOCB_t *cmd;
@@ -5039,7 +5108,7 @@ lpfc_menlo_cmd(struct fc_bsg_job *job)
 	struct ulp_bde64 *bpl = NULL;
 
 	/* in case no data is returned return just the return code */
-	job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 
 	if (job->request_len <
 	    sizeof(struct fc_bsg_request) +
@@ -5069,7 +5138,7 @@ lpfc_menlo_cmd(struct fc_bsg_job *job)
 	}
 
 	menlo_cmd = (struct menlo_command *)
-		job->request->rqst_data.h_vendor.vendor_cmd;
+		bsg_request->rqst_data.h_vendor.vendor_cmd;
 
 	/* allocate our bsg tracking structure */
 	dd_data = kmalloc(sizeof(struct bsg_job_data), GFP_KERNEL);
@@ -5180,19 +5249,65 @@ free_dd:
 	kfree(dd_data);
 no_dd_data:
 	/* make error code available to userspace */
-	job->reply->result = rc;
+	bsg_reply->result = rc;
 	job->dd_data = NULL;
 	return rc;
 }
 
+static int
+lpfc_forced_link_speed(struct bsg_job *job)
+{
+	struct Scsi_Host *shost = fc_bsg_to_shost(job);
+	struct lpfc_vport *vport = shost_priv(shost);
+	struct lpfc_hba *phba = vport->phba;
+	struct fc_bsg_reply *bsg_reply = job->reply;
+	struct forced_link_speed_support_reply *forced_reply;
+	int rc = 0;
+
+	if (job->request_len <
+	    sizeof(struct fc_bsg_request) +
+	    sizeof(struct get_forced_link_speed_support)) {
+		lpfc_printf_log(phba, KERN_WARNING, LOG_LIBDFC,
+				"0048 Received FORCED_LINK_SPEED request "
+				"below minimum size\n");
+		rc = -EINVAL;
+		goto job_error;
+	}
+
+	forced_reply = (struct forced_link_speed_support_reply *)
+		bsg_reply->reply_data.vendor_reply.vendor_rsp;
+
+	if (job->reply_len <
+	    sizeof(struct fc_bsg_request) +
+	    sizeof(struct forced_link_speed_support_reply)) {
+		lpfc_printf_log(phba, KERN_WARNING, LOG_LIBDFC,
+				"0049 Received FORCED_LINK_SPEED reply below "
+				"minimum size\n");
+		rc = -EINVAL;
+		goto job_error;
+	}
+
+	forced_reply->supported = (phba->hba_flag & HBA_FORCED_LINK_SPEED)
+				   ? LPFC_FORCED_LINK_SPEED_SUPPORTED
+				   : LPFC_FORCED_LINK_SPEED_NOT_SUPPORTED;
+job_error:
+	bsg_reply->result = rc;
+	if (rc == 0)
+		bsg_job_done(job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
+	return rc;
+}
+
 /**
  * lpfc_bsg_hst_vendor - process a vendor-specific fc_bsg_job
  * @job: fc_bsg_job to handle
  **/
 static int
-lpfc_bsg_hst_vendor(struct fc_bsg_job *job)
+lpfc_bsg_hst_vendor(struct bsg_job *job)
 {
-	int command = job->request->rqst_data.h_vendor.vendor_cmd[0];
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
+	int command = bsg_request->rqst_data.h_vendor.vendor_cmd[0];
 	int rc;
 
 	switch (command) {
@@ -5227,11 +5342,14 @@ lpfc_bsg_hst_vendor(struct fc_bsg_job *job)
 	case LPFC_BSG_VENDOR_MENLO_DATA:
 		rc = lpfc_menlo_cmd(job);
 		break;
+	case LPFC_BSG_VENDOR_FORCED_LINK_SPEED:
+		rc = lpfc_forced_link_speed(job);
+		break;
 	default:
 		rc = -EINVAL;
-		job->reply->reply_payload_rcv_len = 0;
+		bsg_reply->reply_payload_rcv_len = 0;
 		/* make error code available to userspace */
-		job->reply->result = rc;
+		bsg_reply->result = rc;
 		break;
 	}
 
@@ -5243,12 +5361,14 @@ lpfc_bsg_hst_vendor(struct fc_bsg_job *job)
  * @job: fc_bsg_job to handle
  **/
 int
-lpfc_bsg_request(struct fc_bsg_job *job)
+lpfc_bsg_request(struct bsg_job *job)
 {
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	uint32_t msgcode;
 	int rc;
 
-	msgcode = job->request->msgcode;
+	msgcode = bsg_request->msgcode;
 	switch (msgcode) {
 	case FC_BSG_HST_VENDOR:
 		rc = lpfc_bsg_hst_vendor(job);
@@ -5261,9 +5381,9 @@ lpfc_bsg_request(struct fc_bsg_job *job)
 		break;
 	default:
 		rc = -EINVAL;
-		job->reply->reply_payload_rcv_len = 0;
+		bsg_reply->reply_payload_rcv_len = 0;
 		/* make error code available to userspace */
-		job->reply->result = rc;
+		bsg_reply->result = rc;
 		break;
 	}
 
@@ -5278,9 +5398,9 @@ lpfc_bsg_request(struct fc_bsg_job *job)
  * the waiting function which will handle passing the error back to userspace
  **/
 int
-lpfc_bsg_timeout(struct fc_bsg_job *job)
+lpfc_bsg_timeout(struct bsg_job *job)
 {
-	struct lpfc_vport *vport = (struct lpfc_vport *)job->shost->hostdata;
+	struct lpfc_vport *vport = shost_priv(fc_bsg_to_shost(job));
 	struct lpfc_hba *phba = vport->phba;
 	struct lpfc_iocbq *cmdiocb;
 	struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_ELS_RING];
diff --git a/drivers/scsi/lpfc/lpfc_bsg.h b/drivers/scsi/lpfc/lpfc_bsg.h
index e557bcdbcb19..f2247aa4fa17 100644
--- a/drivers/scsi/lpfc/lpfc_bsg.h
+++ b/drivers/scsi/lpfc/lpfc_bsg.h
@@ -35,6 +35,7 @@
 #define LPFC_BSG_VENDOR_MENLO_DATA		9
 #define LPFC_BSG_VENDOR_DIAG_MODE_END		10
 #define LPFC_BSG_VENDOR_LINK_DIAG_TEST		11
+#define LPFC_BSG_VENDOR_FORCED_LINK_SPEED	14
 
 struct set_ct_event {
 	uint32_t command;
@@ -284,6 +285,15 @@ struct lpfc_sli_config_mbox {
 	} un;
 };
 
+#define LPFC_FORCED_LINK_SPEED_NOT_SUPPORTED	0
+#define LPFC_FORCED_LINK_SPEED_SUPPORTED	1
+struct get_forced_link_speed_support {
+	uint32_t command;
+};
+struct forced_link_speed_support_reply {
+	uint8_t supported;
+};
+
 /* driver only */
 #define SLI_CONFIG_NOT_HANDLED		0
 #define SLI_CONFIG_HANDLED		1
diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h
index bd7576d452f2..15d2bfdf582d 100644
--- a/drivers/scsi/lpfc/lpfc_crtn.h
+++ b/drivers/scsi/lpfc/lpfc_crtn.h
@@ -397,8 +397,6 @@ extern spinlock_t _dump_buf_lock;
 extern int _dump_buf_done;
 extern spinlock_t pgcnt_lock;
 extern unsigned int pgcnt;
-extern unsigned int lpfc_prot_mask;
-extern unsigned char lpfc_prot_guard;
 extern unsigned int lpfc_fcp_look_ahead;
 
 /* Interface exported by fabric iocb scheduler */
@@ -431,8 +429,8 @@ struct lpfc_sglq *__lpfc_get_active_sglq(struct lpfc_hba *, uint16_t);
 #define HBA_EVENT_LINK_DOWN              3
 
 /* functions to support SGIOv4/bsg interface */
-int lpfc_bsg_request(struct fc_bsg_job *);
-int lpfc_bsg_timeout(struct fc_bsg_job *);
+int lpfc_bsg_request(struct bsg_job *);
+int lpfc_bsg_timeout(struct bsg_job *);
 int lpfc_bsg_ct_unsol_event(struct lpfc_hba *, struct lpfc_sli_ring *,
 			     struct lpfc_iocbq *);
 int lpfc_bsg_ct_unsol_abort(struct lpfc_hba *, struct hbq_dmabuf *);
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index b7d54bfb1df9..236e4e51d161 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -7610,7 +7610,7 @@ lpfc_els_unsol_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 	/* reject till our FLOGI completes */
 	if ((vport->port_state < LPFC_FABRIC_CFG_LINK) &&
 	    (cmd != ELS_CMD_FLOGI)) {
-		rjt_err = LSRJT_UNABLE_TPC;
+		rjt_err = LSRJT_LOGICAL_BSY;
 		rjt_exp = LSEXP_NOTHING_MORE;
 		goto lsrjt;
 	}
diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h
index ee8022737591..5646699b0516 100644
--- a/drivers/scsi/lpfc/lpfc_hw4.h
+++ b/drivers/scsi/lpfc/lpfc_hw4.h
@@ -921,6 +921,7 @@ struct mbox_header {
 #define LPFC_MBOX_OPCODE_GET_PORT_NAME			0x4D
 #define LPFC_MBOX_OPCODE_MQ_CREATE_EXT			0x5A
 #define LPFC_MBOX_OPCODE_GET_VPD_DATA			0x5B
+#define LPFC_MBOX_OPCODE_SET_HOST_DATA			0x5D
 #define LPFC_MBOX_OPCODE_SEND_ACTIVATION		0x73
 #define LPFC_MBOX_OPCODE_RESET_LICENSES			0x74
 #define LPFC_MBOX_OPCODE_GET_RSRC_EXTENT_INFO		0x9A
@@ -2289,6 +2290,9 @@ struct lpfc_mbx_read_config {
 #define lpfc_mbx_rd_conf_r_a_tov_SHIFT		0
 #define lpfc_mbx_rd_conf_r_a_tov_MASK		0x0000FFFF
 #define lpfc_mbx_rd_conf_r_a_tov_WORD		word6
+#define lpfc_mbx_rd_conf_link_speed_SHIFT	16
+#define lpfc_mbx_rd_conf_link_speed_MASK	0x0000FFFF
+#define lpfc_mbx_rd_conf_link_speed_WORD	word6
 	uint32_t rsvd_7;
 	uint32_t rsvd_8;
 	uint32_t word9;
@@ -2919,6 +2923,16 @@ struct lpfc_mbx_set_feature {
 };
 
 
+#define LPFC_SET_HOST_OS_DRIVER_VERSION    0x2
+struct lpfc_mbx_set_host_data {
+#define LPFC_HOST_OS_DRIVER_VERSION_SIZE   48
+	struct mbox_header header;
+	uint32_t param_id;
+	uint32_t param_len;
+	uint8_t  data[LPFC_HOST_OS_DRIVER_VERSION_SIZE];
+};
+
+
 struct lpfc_mbx_get_sli4_parameters {
 	struct mbox_header header;
 	struct lpfc_sli4_parameters sli4_parameters;
@@ -3313,6 +3327,7 @@ struct lpfc_mqe {
 		struct lpfc_mbx_get_port_name get_port_name;
 		struct lpfc_mbx_set_feature  set_feature;
 		struct lpfc_mbx_memory_dump_type3 mem_dump_type3;
+		struct lpfc_mbx_set_host_data set_host_data;
 		struct lpfc_mbx_nop nop;
 	} un;
 };
@@ -3981,7 +3996,8 @@ union lpfc_wqe128 {
 	struct gen_req64_wqe gen_req;
 };
 
-#define LPFC_GROUP_OJECT_MAGIC_NUM		0xfeaa0001
+#define LPFC_GROUP_OJECT_MAGIC_G5		0xfeaa0001
+#define LPFC_GROUP_OJECT_MAGIC_G6		0xfeaa0003
 #define LPFC_FILE_TYPE_GROUP			0xf7
 #define LPFC_FILE_ID_GROUP			0xa2
 struct lpfc_grp_hdr {
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index 734a0428ef0e..4776fd85514f 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -6279,34 +6279,36 @@ lpfc_setup_bg(struct lpfc_hba *phba, struct Scsi_Host *shost)
 	uint32_t old_guard;
 
 	int pagecnt = 10;
-	if (lpfc_prot_mask && lpfc_prot_guard) {
+	if (phba->cfg_prot_mask && phba->cfg_prot_guard) {
 		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
 				"1478 Registering BlockGuard with the "
 				"SCSI layer\n");
 
-		old_mask = lpfc_prot_mask;
-		old_guard = lpfc_prot_guard;
+		old_mask = phba->cfg_prot_mask;
+		old_guard = phba->cfg_prot_guard;
 
 		/* Only allow supported values */
-		lpfc_prot_mask &= (SHOST_DIF_TYPE1_PROTECTION |
+		phba->cfg_prot_mask &= (SHOST_DIF_TYPE1_PROTECTION |
 			SHOST_DIX_TYPE0_PROTECTION |
 			SHOST_DIX_TYPE1_PROTECTION);
-		lpfc_prot_guard &= (SHOST_DIX_GUARD_IP | SHOST_DIX_GUARD_CRC);
+		phba->cfg_prot_guard &= (SHOST_DIX_GUARD_IP |
+					 SHOST_DIX_GUARD_CRC);
 
 		/* DIF Type 1 protection for profiles AST1/C1 is end to end */
-		if (lpfc_prot_mask == SHOST_DIX_TYPE1_PROTECTION)
-			lpfc_prot_mask |= SHOST_DIF_TYPE1_PROTECTION;
+		if (phba->cfg_prot_mask == SHOST_DIX_TYPE1_PROTECTION)
+			phba->cfg_prot_mask |= SHOST_DIF_TYPE1_PROTECTION;
 
-		if (lpfc_prot_mask && lpfc_prot_guard) {
-			if ((old_mask != lpfc_prot_mask) ||
-				(old_guard != lpfc_prot_guard))
+		if (phba->cfg_prot_mask && phba->cfg_prot_guard) {
+			if ((old_mask != phba->cfg_prot_mask) ||
+				(old_guard != phba->cfg_prot_guard))
 				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 					"1475 Registering BlockGuard with the "
 					"SCSI layer: mask %d  guard %d\n",
-					lpfc_prot_mask, lpfc_prot_guard);
+					phba->cfg_prot_mask,
+					phba->cfg_prot_guard);
 
-			scsi_host_set_prot(shost, lpfc_prot_mask);
-			scsi_host_set_guard(shost, lpfc_prot_guard);
+			scsi_host_set_prot(shost, phba->cfg_prot_mask);
+			scsi_host_set_guard(shost, phba->cfg_prot_guard);
 		} else
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"1479 Not Registering BlockGuard with the SCSI "
@@ -6929,6 +6931,8 @@ lpfc_sli4_read_config(struct lpfc_hba *phba)
 	struct lpfc_mbx_get_func_cfg *get_func_cfg;
 	struct lpfc_rsrc_desc_fcfcoe *desc;
 	char *pdesc_0;
+	uint16_t forced_link_speed;
+	uint32_t if_type;
 	int length, i, rc = 0, rc2;
 
 	pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
@@ -7022,6 +7026,58 @@ lpfc_sli4_read_config(struct lpfc_hba *phba)
 	if (rc)
 		goto read_cfg_out;
 
+	/* Update link speed if forced link speed is supported */
+	if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf);
+	if (if_type == LPFC_SLI_INTF_IF_TYPE_2) {
+		forced_link_speed =
+			bf_get(lpfc_mbx_rd_conf_link_speed, rd_config);
+		if (forced_link_speed) {
+			phba->hba_flag |= HBA_FORCED_LINK_SPEED;
+
+			switch (forced_link_speed) {
+			case LINK_SPEED_1G:
+				phba->cfg_link_speed =
+					LPFC_USER_LINK_SPEED_1G;
+				break;
+			case LINK_SPEED_2G:
+				phba->cfg_link_speed =
+					LPFC_USER_LINK_SPEED_2G;
+				break;
+			case LINK_SPEED_4G:
+				phba->cfg_link_speed =
+					LPFC_USER_LINK_SPEED_4G;
+				break;
+			case LINK_SPEED_8G:
+				phba->cfg_link_speed =
+					LPFC_USER_LINK_SPEED_8G;
+				break;
+			case LINK_SPEED_10G:
+				phba->cfg_link_speed =
+					LPFC_USER_LINK_SPEED_10G;
+				break;
+			case LINK_SPEED_16G:
+				phba->cfg_link_speed =
+					LPFC_USER_LINK_SPEED_16G;
+				break;
+			case LINK_SPEED_32G:
+				phba->cfg_link_speed =
+					LPFC_USER_LINK_SPEED_32G;
+				break;
+			case 0xffff:
+				phba->cfg_link_speed =
+					LPFC_USER_LINK_SPEED_AUTO;
+				break;
+			default:
+				lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
+						"0047 Unrecognized link "
+						"speed : %d\n",
+						forced_link_speed);
+				phba->cfg_link_speed =
+					LPFC_USER_LINK_SPEED_AUTO;
+			}
+		}
+	}
+
 	/* Reset the DFT_HBA_Q_DEPTH to the max xri  */
 	length = phba->sli4_hba.max_cfg_param.max_xri -
 			lpfc_sli4_get_els_iocb_cnt(phba);
@@ -7256,6 +7312,7 @@ int
 lpfc_sli4_queue_create(struct lpfc_hba *phba)
 {
 	struct lpfc_queue *qdesc;
+	uint32_t wqesize;
 	int idx;
 
 	/*
@@ -7340,15 +7397,10 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 		phba->sli4_hba.fcp_cq[idx] = qdesc;
 
 		/* Create Fast Path FCP WQs */
-		if (phba->fcp_embed_io) {
-			qdesc = lpfc_sli4_queue_alloc(phba,
-						      LPFC_WQE128_SIZE,
-						      LPFC_WQE128_DEF_COUNT);
-		} else {
-			qdesc = lpfc_sli4_queue_alloc(phba,
-						      phba->sli4_hba.wq_esize,
-						      phba->sli4_hba.wq_ecount);
-		}
+		wqesize = (phba->fcp_embed_io) ?
+				LPFC_WQE128_SIZE : phba->sli4_hba.wq_esize;
+		qdesc = lpfc_sli4_queue_alloc(phba, wqesize,
+						phba->sli4_hba.wq_ecount);
 		if (!qdesc) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 					"0503 Failed allocate fast-path FCP "
@@ -10260,6 +10312,7 @@ lpfc_write_firmware(const struct firmware *fw, void *context)
 	int i, rc = 0;
 	struct lpfc_dmabuf *dmabuf, *next;
 	uint32_t offset = 0, temp_offset = 0;
+	uint32_t magic_number, ftype, fid, fsize;
 
 	/* It can be null in no-wait mode, sanity check */
 	if (!fw) {
@@ -10268,18 +10321,19 @@ lpfc_write_firmware(const struct firmware *fw, void *context)
 	}
 	image = (struct lpfc_grp_hdr *)fw->data;
 
+	magic_number = be32_to_cpu(image->magic_number);
+	ftype = bf_get_be32(lpfc_grp_hdr_file_type, image);
+	fid = bf_get_be32(lpfc_grp_hdr_id, image),
+	fsize = be32_to_cpu(image->size);
+
 	INIT_LIST_HEAD(&dma_buffer_list);
-	if ((be32_to_cpu(image->magic_number) != LPFC_GROUP_OJECT_MAGIC_NUM) ||
-	    (bf_get_be32(lpfc_grp_hdr_file_type, image) !=
-	     LPFC_FILE_TYPE_GROUP) ||
-	    (bf_get_be32(lpfc_grp_hdr_id, image) != LPFC_FILE_ID_GROUP) ||
-	    (be32_to_cpu(image->size) != fw->size)) {
+	if ((magic_number != LPFC_GROUP_OJECT_MAGIC_G5 &&
+	     magic_number != LPFC_GROUP_OJECT_MAGIC_G6) ||
+	    ftype != LPFC_FILE_TYPE_GROUP || fsize != fw->size) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"3022 Invalid FW image found. "
-				"Magic:%x Type:%x ID:%x\n",
-				be32_to_cpu(image->magic_number),
-				bf_get_be32(lpfc_grp_hdr_file_type, image),
-				bf_get_be32(lpfc_grp_hdr_id, image));
+				"Magic:%x Type:%x ID:%x Size %d %zd\n",
+				magic_number, ftype, fid, fsize, fw->size);
 		rc = -EINVAL;
 		goto release_out;
 	}
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index d197aa176dee..ad350d969bdc 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -413,15 +413,13 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc)
 		 * struct fcp_cmnd, struct fcp_rsp and the number of bde's
 		 * necessary to support the sg_tablesize.
 		 */
-		psb->data = pci_pool_alloc(phba->lpfc_scsi_dma_buf_pool,
+		psb->data = pci_pool_zalloc(phba->lpfc_scsi_dma_buf_pool,
 					GFP_KERNEL, &psb->dma_handle);
 		if (!psb->data) {
 			kfree(psb);
 			break;
 		}
 
-		/* Initialize virtual ptrs to dma_buf region. */
-		memset(psb->data, 0, phba->cfg_sg_dma_buf_size);
 
 		/* Allocate iotag for psb->cur_iocbq. */
 		iotag = lpfc_sli_next_iotag(phba, &psb->cur_iocbq);
@@ -607,7 +605,7 @@ lpfc_sli4_fcp_xri_aborted(struct lpfc_hba *phba,
 }
 
 /**
- * lpfc_sli4_post_scsi_sgl_list - Psot blocks of scsi buffer sgls from a list
+ * lpfc_sli4_post_scsi_sgl_list - Post blocks of scsi buffer sgls from a list
  * @phba: pointer to lpfc hba data structure.
  * @post_sblist: pointer to the scsi buffer list.
  *
@@ -736,7 +734,7 @@ lpfc_sli4_post_scsi_sgl_list(struct lpfc_hba *phba,
 }
 
 /**
- * lpfc_sli4_repost_scsi_sgl_list - Repsot all the allocated scsi buffer sgls
+ * lpfc_sli4_repost_scsi_sgl_list - Repost all the allocated scsi buffer sgls
  * @phba: pointer to lpfc hba data structure.
  *
  * This routine walks the list of scsi buffers that have been allocated and
@@ -821,13 +819,12 @@ lpfc_new_scsi_buf_s4(struct lpfc_vport *vport, int num_to_alloc)
 		 * for the struct fcp_cmnd, struct fcp_rsp and the number
 		 * of bde's necessary to support the sg_tablesize.
 		 */
-		psb->data = pci_pool_alloc(phba->lpfc_scsi_dma_buf_pool,
+		psb->data = pci_pool_zalloc(phba->lpfc_scsi_dma_buf_pool,
 						GFP_KERNEL, &psb->dma_handle);
 		if (!psb->data) {
 			kfree(psb);
 			break;
 		}
-		memset(psb->data, 0, phba->cfg_sg_dma_buf_size);
 
 		/*
 		 * 4K Page alignment is CRITICAL to BlockGuard, double check
@@ -857,7 +854,7 @@ lpfc_new_scsi_buf_s4(struct lpfc_vport *vport, int num_to_alloc)
 				psb->data, psb->dma_handle);
 			kfree(psb);
 			lpfc_printf_log(phba, KERN_ERR, LOG_FCP,
-					"3368 Failed to allocated IOTAG for"
+					"3368 Failed to allocate IOTAG for"
 					" XRI:0x%x\n", lxri);
 			lpfc_sli4_free_xri(phba, lxri);
 			break;
@@ -1136,7 +1133,7 @@ lpfc_release_scsi_buf(struct lpfc_hba *phba, struct lpfc_scsi_buf *psb)
  *
  * This routine does the pci dma mapping for scatter-gather list of scsi cmnd
  * field of @lpfc_cmd for device with SLI-3 interface spec. This routine scans
- * through sg elements and format the bdea. This routine also initializes all
+ * through sg elements and format the bde. This routine also initializes all
  * IOCB fields which are dependent on scsi command request buffer.
  *
  * Return codes:
@@ -1269,13 +1266,16 @@ lpfc_scsi_prep_dma_buf_s3(struct lpfc_hba *phba, struct lpfc_scsi_buf *lpfc_cmd)
 
 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 
-/* Return if if error injection is detected by Initiator */
+/* Return BG_ERR_INIT if error injection is detected by Initiator */
 #define BG_ERR_INIT	0x1
-/* Return if if error injection is detected by Target */
+/* Return BG_ERR_TGT if error injection is detected by Target */
 #define BG_ERR_TGT	0x2
-/* Return if if swapping CSUM<-->CRC is required for error injection */
+/* Return BG_ERR_SWAP if swapping CSUM<-->CRC is required for error injection */
 #define BG_ERR_SWAP	0x10
-/* Return if disabling Guard/Ref/App checking is required for error injection */
+/**
+ * Return BG_ERR_CHECK if disabling Guard/Ref/App checking is required for
+ * error injection
+ **/
 #define BG_ERR_CHECK	0x20
 
 /**
@@ -4139,13 +4139,13 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
 
 	lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
 
-	/* The sdev is not guaranteed to be valid post scsi_done upcall. */
-	cmd->scsi_done(cmd);
-
 	spin_lock_irqsave(&phba->hbalock, flags);
 	lpfc_cmd->pCmd = NULL;
 	spin_unlock_irqrestore(&phba->hbalock, flags);
 
+	/* The sdev is not guaranteed to be valid post scsi_done upcall. */
+	cmd->scsi_done(cmd);
+
 	/*
 	 * If there is a thread waiting for command completion
 	 * wake up the thread.
@@ -4822,7 +4822,7 @@ wait_for_cmpl:
 		ret = FAILED;
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
 				 "0748 abort handler timed out waiting "
-				 "for abortng I/O (xri:x%x) to complete: "
+				 "for aborting I/O (xri:x%x) to complete: "
 				 "ret %#x, ID %d, LUN %llu\n",
 				 iocb->sli4_xritag, ret,
 				 cmnd->device->id, cmnd->device->lun);
@@ -4945,26 +4945,30 @@ lpfc_check_fcp_rsp(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd)
  *   0x2002 - Success.
  **/
 static int
-lpfc_send_taskmgmt(struct lpfc_vport *vport, struct lpfc_rport_data *rdata,
-		    unsigned  tgt_id, uint64_t lun_id,
-		    uint8_t task_mgmt_cmd)
+lpfc_send_taskmgmt(struct lpfc_vport *vport, struct scsi_cmnd *cmnd,
+		   unsigned int tgt_id, uint64_t lun_id,
+		   uint8_t task_mgmt_cmd)
 {
 	struct lpfc_hba   *phba = vport->phba;
 	struct lpfc_scsi_buf *lpfc_cmd;
 	struct lpfc_iocbq *iocbq;
 	struct lpfc_iocbq *iocbqrsp;
-	struct lpfc_nodelist *pnode = rdata->pnode;
+	struct lpfc_rport_data *rdata;
+	struct lpfc_nodelist *pnode;
 	int ret;
 	int status;
 
-	if (!pnode || !NLP_CHK_NODE_ACT(pnode))
+	rdata = lpfc_rport_data_from_scsi_device(cmnd->device);
+	if (!rdata || !rdata->pnode || !NLP_CHK_NODE_ACT(rdata->pnode))
 		return FAILED;
+	pnode = rdata->pnode;
 
-	lpfc_cmd = lpfc_get_scsi_buf(phba, rdata->pnode);
+	lpfc_cmd = lpfc_get_scsi_buf(phba, pnode);
 	if (lpfc_cmd == NULL)
 		return FAILED;
 	lpfc_cmd->timeout = phba->cfg_task_mgmt_tmo;
 	lpfc_cmd->rdata = rdata;
+	lpfc_cmd->pCmd = cmnd;
 
 	status = lpfc_scsi_prep_task_mgmt_cmd(vport, lpfc_cmd, lun_id,
 					   task_mgmt_cmd);
@@ -5171,7 +5175,7 @@ lpfc_device_reset_handler(struct scsi_cmnd *cmnd)
 	fc_host_post_vendor_event(shost, fc_get_event_number(),
 		sizeof(scsi_event), (char *)&scsi_event, LPFC_NL_VENDOR_ID);
 
-	status = lpfc_send_taskmgmt(vport, rdata, tgt_id, lun_id,
+	status = lpfc_send_taskmgmt(vport, cmnd, tgt_id, lun_id,
 						FCP_LUN_RESET);
 
 	lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
@@ -5249,7 +5253,7 @@ lpfc_target_reset_handler(struct scsi_cmnd *cmnd)
 	fc_host_post_vendor_event(shost, fc_get_event_number(),
 		sizeof(scsi_event), (char *)&scsi_event, LPFC_NL_VENDOR_ID);
 
-	status = lpfc_send_taskmgmt(vport, rdata, tgt_id, lun_id,
+	status = lpfc_send_taskmgmt(vport, cmnd, tgt_id, lun_id,
 					FCP_TARGET_RESET);
 
 	lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
@@ -5328,7 +5332,7 @@ lpfc_bus_reset_handler(struct scsi_cmnd *cmnd)
 		if (!match)
 			continue;
 
-		status = lpfc_send_taskmgmt(vport, ndlp->rport->dd_data,
+		status = lpfc_send_taskmgmt(vport, cmnd,
 					i, 0, FCP_TARGET_RESET);
 
 		if (status != SUCCESS) {
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index f4f77c5b0c83..4faa7672fc1d 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -47,6 +47,7 @@
 #include "lpfc_compat.h"
 #include "lpfc_debugfs.h"
 #include "lpfc_vport.h"
+#include "lpfc_version.h"
 
 /* There are only four IOCB completion types. */
 typedef enum _lpfc_iocb_type {
@@ -2678,15 +2679,16 @@ lpfc_sli_iocbq_lookup(struct lpfc_hba *phba,
 
 	if (iotag != 0 && iotag <= phba->sli.last_iotag) {
 		cmd_iocb = phba->sli.iocbq_lookup[iotag];
-		list_del_init(&cmd_iocb->list);
 		if (cmd_iocb->iocb_flag & LPFC_IO_ON_TXCMPLQ) {
+			/* remove from txcmpl queue list */
+			list_del_init(&cmd_iocb->list);
 			cmd_iocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ;
+			return cmd_iocb;
 		}
-		return cmd_iocb;
 	}
 
 	lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
-			"0317 iotag x%x is out off "
+			"0317 iotag x%x is out of "
 			"range: max iotag x%x wd0 x%x\n",
 			iotag, phba->sli.last_iotag,
 			*(((uint32_t *) &prspiocb->iocb) + 7));
@@ -2721,8 +2723,9 @@ lpfc_sli_iocbq_lookup_by_tag(struct lpfc_hba *phba,
 			return cmd_iocb;
 		}
 	}
+
 	lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
-			"0372 iotag x%x is out off range: max iotag (x%x)\n",
+			"0372 iotag x%x is out of range: max iotag (x%x)\n",
 			iotag, phba->sli.last_iotag);
 	return NULL;
 }
@@ -6291,6 +6294,25 @@ lpfc_sli4_repost_els_sgl_list(struct lpfc_hba *phba)
 	return 0;
 }
 
+void
+lpfc_set_host_data(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox)
+{
+	uint32_t len;
+
+	len = sizeof(struct lpfc_mbx_set_host_data) -
+		sizeof(struct lpfc_sli4_cfg_mhdr);
+	lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON,
+			 LPFC_MBOX_OPCODE_SET_HOST_DATA, len,
+			 LPFC_SLI4_MBX_EMBED);
+
+	mbox->u.mqe.un.set_host_data.param_id = LPFC_SET_HOST_OS_DRIVER_VERSION;
+	mbox->u.mqe.un.set_host_data.param_len = 8;
+	snprintf(mbox->u.mqe.un.set_host_data.data,
+		 LPFC_HOST_OS_DRIVER_VERSION_SIZE,
+		 "Linux %s v"LPFC_DRIVER_VERSION,
+		 (phba->hba_flag & HBA_FCOE_MODE) ? "FCoE" : "FC");
+}
+
 /**
  * lpfc_sli4_hba_setup - SLI4 device intialization PCI function
  * @phba: Pointer to HBA context object.
@@ -6542,6 +6564,15 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)
 		goto out_free_mbox;
 	}
 
+	lpfc_set_host_data(phba, mboxq);
+
+	rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
+	if (rc) {
+		lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_SLI,
+				"2134 Failed to set host os driver version %x",
+				rc);
+	}
+
 	/* Read the port's service parameters. */
 	rc = lpfc_read_sparam(phba, mboxq, vport->vpi);
 	if (rc) {
@@ -11781,6 +11812,8 @@ lpfc_sli4_els_wcqe_to_rspiocbq(struct lpfc_hba *phba,
 	/* Look up the ELS command IOCB and create pseudo response IOCB */
 	cmdiocbq = lpfc_sli_iocbq_lookup_by_tag(phba, pring,
 				bf_get(lpfc_wcqe_c_request_tag, wcqe));
+	/* Put the iocb back on the txcmplq */
+	lpfc_sli_ringtxcmpl_put(phba, pring, cmdiocbq);
 	spin_unlock_irqrestore(&pring->ring_lock, iflags);
 
 	if (unlikely(!cmdiocbq)) {
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index c9bf20eb7223..50bfc43ebcb0 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -18,7 +18,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "11.2.0.0."
+#define LPFC_DRIVER_VERSION "11.2.0.2"
 #define LPFC_DRIVER_NAME		"lpfc"
 
 /* Used for SLI 2/3 */
diff --git a/drivers/scsi/mac_scsi.c b/drivers/scsi/mac_scsi.c
index a590089b9397..ccb68d12692c 100644
--- a/drivers/scsi/mac_scsi.c
+++ b/drivers/scsi/mac_scsi.c
@@ -28,17 +28,15 @@
 
 /* Definitions for the core NCR5380 driver. */
 
-#define NCR5380_implementation_fields   unsigned char *pdma_base; \
-                                        int pdma_residual
+#define NCR5380_implementation_fields   int pdma_residual
 
-#define NCR5380_read(reg)               macscsi_read(instance, reg)
-#define NCR5380_write(reg, value)       macscsi_write(instance, reg, value)
+#define NCR5380_read(reg)           in_8(hostdata->io + ((reg) << 4))
+#define NCR5380_write(reg, value)   out_8(hostdata->io + ((reg) << 4), value)
 
-#define NCR5380_dma_xfer_len(instance, cmd, phase) \
-        macscsi_dma_xfer_len(instance, cmd)
+#define NCR5380_dma_xfer_len            macscsi_dma_xfer_len
 #define NCR5380_dma_recv_setup          macscsi_pread
 #define NCR5380_dma_send_setup          macscsi_pwrite
-#define NCR5380_dma_residual(instance)  (hostdata->pdma_residual)
+#define NCR5380_dma_residual            macscsi_dma_residual
 
 #define NCR5380_intr                    macscsi_intr
 #define NCR5380_queue_command           macscsi_queue_command
@@ -61,20 +59,6 @@ module_param(setup_hostid, int, 0);
 static int setup_toshiba_delay = -1;
 module_param(setup_toshiba_delay, int, 0);
 
-/*
- * NCR 5380 register access functions
- */
-
-static inline char macscsi_read(struct Scsi_Host *instance, int reg)
-{
-	return in_8(instance->base + (reg << 4));
-}
-
-static inline void macscsi_write(struct Scsi_Host *instance, int reg, int value)
-{
-	out_8(instance->base + (reg << 4), value);
-}
-
 #ifndef MODULE
 static int __init mac_scsi_setup(char *str)
 {
@@ -167,16 +151,15 @@ __asm__ __volatile__					\
      : "0"(s), "1"(d), "2"(n)				\
      : "d0")
 
-static int macscsi_pread(struct Scsi_Host *instance,
-                         unsigned char *dst, int len)
+static inline int macscsi_pread(struct NCR5380_hostdata *hostdata,
+                                unsigned char *dst, int len)
 {
-	struct NCR5380_hostdata *hostdata = shost_priv(instance);
-	unsigned char *s = hostdata->pdma_base + (INPUT_DATA_REG << 4);
+	unsigned char *s = hostdata->pdma_io + (INPUT_DATA_REG << 4);
 	unsigned char *d = dst;
 	int n = len;
 	int transferred;
 
-	while (!NCR5380_poll_politely(instance, BUS_AND_STATUS_REG,
+	while (!NCR5380_poll_politely(hostdata, BUS_AND_STATUS_REG,
 	                              BASR_DRQ | BASR_PHASE_MATCH,
 	                              BASR_DRQ | BASR_PHASE_MATCH, HZ / 64)) {
 		CP_IO_TO_MEM(s, d, n);
@@ -189,23 +172,23 @@ static int macscsi_pread(struct Scsi_Host *instance,
 			return 0;
 
 		/* Target changed phase early? */
-		if (NCR5380_poll_politely2(instance, STATUS_REG, SR_REQ, SR_REQ,
+		if (NCR5380_poll_politely2(hostdata, STATUS_REG, SR_REQ, SR_REQ,
 		                           BUS_AND_STATUS_REG, BASR_ACK, BASR_ACK, HZ / 64) < 0)
 			scmd_printk(KERN_ERR, hostdata->connected,
 			            "%s: !REQ and !ACK\n", __func__);
 		if (!(NCR5380_read(BUS_AND_STATUS_REG) & BASR_PHASE_MATCH))
 			return 0;
 
-		dsprintk(NDEBUG_PSEUDO_DMA, instance,
+		dsprintk(NDEBUG_PSEUDO_DMA, hostdata->host,
 		         "%s: bus error (%d/%d)\n", __func__, transferred, len);
-		NCR5380_dprint(NDEBUG_PSEUDO_DMA, instance);
+		NCR5380_dprint(NDEBUG_PSEUDO_DMA, hostdata->host);
 		d = dst + transferred;
 		n = len - transferred;
 	}
 
 	scmd_printk(KERN_ERR, hostdata->connected,
 	            "%s: phase mismatch or !DRQ\n", __func__);
-	NCR5380_dprint(NDEBUG_PSEUDO_DMA, instance);
+	NCR5380_dprint(NDEBUG_PSEUDO_DMA, hostdata->host);
 	return -1;
 }
 
@@ -270,16 +253,15 @@ __asm__ __volatile__					\
      : "0"(s), "1"(d), "2"(n)				\
      : "d0")
 
-static int macscsi_pwrite(struct Scsi_Host *instance,
-                          unsigned char *src, int len)
+static inline int macscsi_pwrite(struct NCR5380_hostdata *hostdata,
+                                 unsigned char *src, int len)
 {
-	struct NCR5380_hostdata *hostdata = shost_priv(instance);
 	unsigned char *s = src;
-	unsigned char *d = hostdata->pdma_base + (OUTPUT_DATA_REG << 4);
+	unsigned char *d = hostdata->pdma_io + (OUTPUT_DATA_REG << 4);
 	int n = len;
 	int transferred;
 
-	while (!NCR5380_poll_politely(instance, BUS_AND_STATUS_REG,
+	while (!NCR5380_poll_politely(hostdata, BUS_AND_STATUS_REG,
 	                              BASR_DRQ | BASR_PHASE_MATCH,
 	                              BASR_DRQ | BASR_PHASE_MATCH, HZ / 64)) {
 		CP_MEM_TO_IO(s, d, n);
@@ -288,7 +270,7 @@ static int macscsi_pwrite(struct Scsi_Host *instance,
 		hostdata->pdma_residual = len - transferred;
 
 		/* Target changed phase early? */
-		if (NCR5380_poll_politely2(instance, STATUS_REG, SR_REQ, SR_REQ,
+		if (NCR5380_poll_politely2(hostdata, STATUS_REG, SR_REQ, SR_REQ,
 		                           BUS_AND_STATUS_REG, BASR_ACK, BASR_ACK, HZ / 64) < 0)
 			scmd_printk(KERN_ERR, hostdata->connected,
 			            "%s: !REQ and !ACK\n", __func__);
@@ -297,7 +279,7 @@ static int macscsi_pwrite(struct Scsi_Host *instance,
 
 		/* No bus error. */
 		if (n == 0) {
-			if (NCR5380_poll_politely(instance, TARGET_COMMAND_REG,
+			if (NCR5380_poll_politely(hostdata, TARGET_COMMAND_REG,
 			                          TCR_LAST_BYTE_SENT,
 			                          TCR_LAST_BYTE_SENT, HZ / 64) < 0)
 				scmd_printk(KERN_ERR, hostdata->connected,
@@ -305,25 +287,23 @@ static int macscsi_pwrite(struct Scsi_Host *instance,
 			return 0;
 		}
 
-		dsprintk(NDEBUG_PSEUDO_DMA, instance,
+		dsprintk(NDEBUG_PSEUDO_DMA, hostdata->host,
 		         "%s: bus error (%d/%d)\n", __func__, transferred, len);
-		NCR5380_dprint(NDEBUG_PSEUDO_DMA, instance);
+		NCR5380_dprint(NDEBUG_PSEUDO_DMA, hostdata->host);
 		s = src + transferred;
 		n = len - transferred;
 	}
 
 	scmd_printk(KERN_ERR, hostdata->connected,
 	            "%s: phase mismatch or !DRQ\n", __func__);
-	NCR5380_dprint(NDEBUG_PSEUDO_DMA, instance);
+	NCR5380_dprint(NDEBUG_PSEUDO_DMA, hostdata->host);
 
 	return -1;
 }
 
-static int macscsi_dma_xfer_len(struct Scsi_Host *instance,
+static int macscsi_dma_xfer_len(struct NCR5380_hostdata *hostdata,
                                 struct scsi_cmnd *cmd)
 {
-	struct NCR5380_hostdata *hostdata = shost_priv(instance);
-
 	if (hostdata->flags & FLAG_NO_PSEUDO_DMA ||
 	    cmd->SCp.this_residual < 16)
 		return 0;
@@ -331,6 +311,11 @@ static int macscsi_dma_xfer_len(struct Scsi_Host *instance,
 	return cmd->SCp.this_residual;
 }
 
+static int macscsi_dma_residual(struct NCR5380_hostdata *hostdata)
+{
+	return hostdata->pdma_residual;
+}
+
 #include "NCR5380.c"
 
 #define DRV_MODULE_NAME         "mac_scsi"
@@ -356,6 +341,7 @@ static struct scsi_host_template mac_scsi_template = {
 static int __init mac_scsi_probe(struct platform_device *pdev)
 {
 	struct Scsi_Host *instance;
+	struct NCR5380_hostdata *hostdata;
 	int error;
 	int host_flags = 0;
 	struct resource *irq, *pio_mem, *pdma_mem = NULL;
@@ -388,17 +374,18 @@ static int __init mac_scsi_probe(struct platform_device *pdev)
 	if (!instance)
 		return -ENOMEM;
 
-	instance->base = pio_mem->start;
 	if (irq)
 		instance->irq = irq->start;
 	else
 		instance->irq = NO_IRQ;
 
-	if (pdma_mem && setup_use_pdma) {
-		struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	hostdata = shost_priv(instance);
+	hostdata->base = pio_mem->start;
+	hostdata->io = (void *)pio_mem->start;
 
-		hostdata->pdma_base = (unsigned char *)pdma_mem->start;
-	} else
+	if (pdma_mem && setup_use_pdma)
+		hostdata->pdma_io = (void *)pdma_mem->start;
+	else
 		host_flags |= FLAG_NO_PSEUDO_DMA;
 
 	host_flags |= setup_toshiba_delay > 0 ? FLAG_TOSHIBA_DELAY : 0;
diff --git a/drivers/scsi/megaraid/megaraid_sas.h b/drivers/scsi/megaraid/megaraid_sas.h
index 3aaea713bf37..fdd519c1dd57 100644
--- a/drivers/scsi/megaraid/megaraid_sas.h
+++ b/drivers/scsi/megaraid/megaraid_sas.h
@@ -35,8 +35,8 @@
 /*
  * MegaRAID SAS Driver meta data
  */
-#define MEGASAS_VERSION				"06.811.02.00-rc1"
-#define MEGASAS_RELDATE				"April 12, 2016"
+#define MEGASAS_VERSION				"06.812.07.00-rc1"
+#define MEGASAS_RELDATE				"August 22, 2016"
 
 /*
  * Device IDs
@@ -1429,6 +1429,8 @@ enum FW_BOOT_CONTEXT {
 #define MR_MAX_REPLY_QUEUES_EXT_OFFSET_SHIFT    14
 #define MR_MAX_MSIX_REG_ARRAY                   16
 #define MR_RDPQ_MODE_OFFSET			0X00800000
+#define MR_CAN_HANDLE_SYNC_CACHE_OFFSET		0X01000000
+
 /*
 * register set for both 1068 and 1078 controllers
 * structure extended for 1078 registers
@@ -2118,7 +2120,6 @@ struct megasas_instance {
 	u32 ctrl_context_pages;
 	struct megasas_ctrl_info *ctrl_info;
 	unsigned int msix_vectors;
-	struct msix_entry msixentry[MEGASAS_MAX_MSIX_QUEUES];
 	struct megasas_irq_context irq_context[MEGASAS_MAX_MSIX_QUEUES];
 	u64 map_id;
 	u64 pd_seq_map_id;
@@ -2140,6 +2141,7 @@ struct megasas_instance {
 	u8 is_imr;
 	u8 is_rdpq;
 	bool dev_handle;
+	bool fw_sync_cache_support;
 };
 struct MR_LD_VF_MAP {
 	u32 size;
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index d8b1fbd4c8aa..6484c382f670 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -1700,11 +1700,8 @@ megasas_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd)
 		goto out_done;
 	}
 
-	/*
-	 * FW takes care of flush cache on its own for Virtual Disk.
-	 * No need to send it down for VD. For JBOD send SYNCHRONIZE_CACHE to FW.
-	 */
-	if ((scmd->cmnd[0] == SYNCHRONIZE_CACHE) && MEGASAS_IS_LOGICAL(scmd)) {
+	if ((scmd->cmnd[0] == SYNCHRONIZE_CACHE) && MEGASAS_IS_LOGICAL(scmd) &&
+		(!instance->fw_sync_cache_support)) {
 		scmd->result = DID_OK << 16;
 		goto out_done;
 	}
@@ -4840,7 +4837,7 @@ fail_alloc_cmds:
 }
 
 /*
- * megasas_setup_irqs_msix -		register legacy interrupts.
+ * megasas_setup_irqs_ioapic -		register legacy interrupts.
  * @instance:				Adapter soft state
  *
  * Do not enable interrupt, only setup ISRs.
@@ -4855,8 +4852,9 @@ megasas_setup_irqs_ioapic(struct megasas_instance *instance)
 	pdev = instance->pdev;
 	instance->irq_context[0].instance = instance;
 	instance->irq_context[0].MSIxIndex = 0;
-	if (request_irq(pdev->irq, instance->instancet->service_isr,
-		IRQF_SHARED, "megasas", &instance->irq_context[0])) {
+	if (request_irq(pci_irq_vector(pdev, 0),
+			instance->instancet->service_isr, IRQF_SHARED,
+			"megasas", &instance->irq_context[0])) {
 		dev_err(&instance->pdev->dev,
 				"Failed to register IRQ from %s %d\n",
 				__func__, __LINE__);
@@ -4877,28 +4875,23 @@ megasas_setup_irqs_ioapic(struct megasas_instance *instance)
 static int
 megasas_setup_irqs_msix(struct megasas_instance *instance, u8 is_probe)
 {
-	int i, j, cpu;
+	int i, j;
 	struct pci_dev *pdev;
 
 	pdev = instance->pdev;
 
 	/* Try MSI-x */
-	cpu = cpumask_first(cpu_online_mask);
 	for (i = 0; i < instance->msix_vectors; i++) {
 		instance->irq_context[i].instance = instance;
 		instance->irq_context[i].MSIxIndex = i;
-		if (request_irq(instance->msixentry[i].vector,
+		if (request_irq(pci_irq_vector(pdev, i),
 			instance->instancet->service_isr, 0, "megasas",
 			&instance->irq_context[i])) {
 			dev_err(&instance->pdev->dev,
 				"Failed to register IRQ for vector %d.\n", i);
-			for (j = 0; j < i; j++) {
-				if (smp_affinity_enable)
-					irq_set_affinity_hint(
-						instance->msixentry[j].vector, NULL);
-				free_irq(instance->msixentry[j].vector,
-					&instance->irq_context[j]);
-			}
+			for (j = 0; j < i; j++)
+				free_irq(pci_irq_vector(pdev, j),
+					 &instance->irq_context[j]);
 			/* Retry irq register for IO_APIC*/
 			instance->msix_vectors = 0;
 			if (is_probe)
@@ -4906,14 +4899,6 @@ megasas_setup_irqs_msix(struct megasas_instance *instance, u8 is_probe)
 			else
 				return -1;
 		}
-		if (smp_affinity_enable) {
-			if (irq_set_affinity_hint(instance->msixentry[i].vector,
-				get_cpu_mask(cpu)))
-				dev_err(&instance->pdev->dev,
-					"Failed to set affinity hint"
-					" for cpu %d\n", cpu);
-			cpu = cpumask_next(cpu, cpu_online_mask);
-		}
 	}
 	return 0;
 }
@@ -4930,14 +4915,12 @@ megasas_destroy_irqs(struct megasas_instance *instance) {
 
 	if (instance->msix_vectors)
 		for (i = 0; i < instance->msix_vectors; i++) {
-			if (smp_affinity_enable)
-				irq_set_affinity_hint(
-					instance->msixentry[i].vector, NULL);
-			free_irq(instance->msixentry[i].vector,
+			free_irq(pci_irq_vector(instance->pdev, i),
 				 &instance->irq_context[i]);
 		}
 	else
-		free_irq(instance->pdev->irq, &instance->irq_context[0]);
+		free_irq(pci_irq_vector(instance->pdev, 0),
+			 &instance->irq_context[0]);
 }
 
 /**
@@ -5095,6 +5078,8 @@ static int megasas_init_fw(struct megasas_instance *instance)
 	msix_enable = (instance->instancet->read_fw_status_reg(reg_set) &
 		       0x4000000) >> 0x1a;
 	if (msix_enable && !msix_disable) {
+		int irq_flags = PCI_IRQ_MSIX;
+
 		scratch_pad_2 = readl
 			(&instance->reg_set->outbound_scratch_pad_2);
 		/* Check max MSI-X vectors */
@@ -5131,15 +5116,18 @@ static int megasas_init_fw(struct megasas_instance *instance)
 		/* Don't bother allocating more MSI-X vectors than cpus */
 		instance->msix_vectors = min(instance->msix_vectors,
 					     (unsigned int)num_online_cpus());
-		for (i = 0; i < instance->msix_vectors; i++)
-			instance->msixentry[i].entry = i;
-		i = pci_enable_msix_range(instance->pdev, instance->msixentry,
-					  1, instance->msix_vectors);
+		if (smp_affinity_enable)
+			irq_flags |= PCI_IRQ_AFFINITY;
+		i = pci_alloc_irq_vectors(instance->pdev, 1,
+					  instance->msix_vectors, irq_flags);
 		if (i > 0)
 			instance->msix_vectors = i;
 		else
 			instance->msix_vectors = 0;
 	}
+	i = pci_alloc_irq_vectors(instance->pdev, 1, 1, PCI_IRQ_LEGACY);
+	if (i < 0)
+		goto fail_setup_irqs;
 
 	dev_info(&instance->pdev->dev,
 		"firmware supports msix\t: (%d)", fw_msix_count);
@@ -5152,11 +5140,6 @@ static int megasas_init_fw(struct megasas_instance *instance)
 	tasklet_init(&instance->isr_tasklet, instance->instancet->tasklet,
 		(unsigned long)instance);
 
-	if (instance->msix_vectors ?
-		megasas_setup_irqs_msix(instance, 1) :
-		megasas_setup_irqs_ioapic(instance))
-		goto fail_setup_irqs;
-
 	instance->ctrl_info = kzalloc(sizeof(struct megasas_ctrl_info),
 				GFP_KERNEL);
 	if (instance->ctrl_info == NULL)
@@ -5172,6 +5155,10 @@ static int megasas_init_fw(struct megasas_instance *instance)
 	if (instance->instancet->init_adapter(instance))
 		goto fail_init_adapter;
 
+	if (instance->msix_vectors ?
+		megasas_setup_irqs_msix(instance, 1) :
+		megasas_setup_irqs_ioapic(instance))
+		goto fail_init_adapter;
 
 	instance->instancet->enable_intr(instance);
 
@@ -5315,7 +5302,7 @@ fail_init_adapter:
 	megasas_destroy_irqs(instance);
 fail_setup_irqs:
 	if (instance->msix_vectors)
-		pci_disable_msix(instance->pdev);
+		pci_free_irq_vectors(instance->pdev);
 	instance->msix_vectors = 0;
 fail_ready_state:
 	kfree(instance->ctrl_info);
@@ -5584,7 +5571,6 @@ static int megasas_io_attach(struct megasas_instance *instance)
 	/*
 	 * Export parameters required by SCSI mid-layer
 	 */
-	host->irq = instance->pdev->irq;
 	host->unique_id = instance->unique_id;
 	host->can_queue = instance->max_scsi_cmds;
 	host->this_id = instance->init_id;
@@ -5947,7 +5933,7 @@ fail_io_attach:
 	else
 		megasas_release_mfi(instance);
 	if (instance->msix_vectors)
-		pci_disable_msix(instance->pdev);
+		pci_free_irq_vectors(instance->pdev);
 fail_init_mfi:
 fail_alloc_dma_buf:
 	if (instance->evt_detail)
@@ -6105,7 +6091,7 @@ megasas_suspend(struct pci_dev *pdev, pm_message_t state)
 	megasas_destroy_irqs(instance);
 
 	if (instance->msix_vectors)
-		pci_disable_msix(instance->pdev);
+		pci_free_irq_vectors(instance->pdev);
 
 	pci_save_state(pdev);
 	pci_disable_device(pdev);
@@ -6125,6 +6111,7 @@ megasas_resume(struct pci_dev *pdev)
 	int rval;
 	struct Scsi_Host *host;
 	struct megasas_instance *instance;
+	int irq_flags = PCI_IRQ_LEGACY;
 
 	instance = pci_get_drvdata(pdev);
 	host = instance->host;
@@ -6160,9 +6147,15 @@ megasas_resume(struct pci_dev *pdev)
 		goto fail_ready_state;
 
 	/* Now re-enable MSI-X */
-	if (instance->msix_vectors &&
-	    pci_enable_msix_exact(instance->pdev, instance->msixentry,
-				  instance->msix_vectors))
+	if (instance->msix_vectors) {
+		irq_flags = PCI_IRQ_MSIX;
+		if (smp_affinity_enable)
+			irq_flags |= PCI_IRQ_AFFINITY;
+	}
+	rval = pci_alloc_irq_vectors(instance->pdev, 1,
+				     instance->msix_vectors ?
+				     instance->msix_vectors : 1, irq_flags);
+	if (rval < 0)
 		goto fail_reenable_msix;
 
 	if (instance->ctrl_context) {
@@ -6245,6 +6238,34 @@ fail_reenable_msix:
 #define megasas_resume	NULL
 #endif
 
+static inline int
+megasas_wait_for_adapter_operational(struct megasas_instance *instance)
+{
+	int wait_time = MEGASAS_RESET_WAIT_TIME * 2;
+	int i;
+
+	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
+		return 1;
+
+	for (i = 0; i < wait_time; i++) {
+		if (atomic_read(&instance->adprecovery)	== MEGASAS_HBA_OPERATIONAL)
+			break;
+
+		if (!(i % MEGASAS_RESET_NOTICE_INTERVAL))
+			dev_notice(&instance->pdev->dev, "waiting for controller reset to finish\n");
+
+		msleep(1000);
+	}
+
+	if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
+		dev_info(&instance->pdev->dev, "%s timed out while waiting for HBA to recover.\n",
+			__func__);
+		return 1;
+	}
+
+	return 0;
+}
+
 /**
  * megasas_detach_one -	PCI hot"un"plug entry point
  * @pdev:		PCI device structure
@@ -6269,9 +6290,14 @@ static void megasas_detach_one(struct pci_dev *pdev)
 	if (instance->fw_crash_state != UNAVAILABLE)
 		megasas_free_host_crash_buffer(instance);
 	scsi_remove_host(instance->host);
+
+	if (megasas_wait_for_adapter_operational(instance))
+		goto skip_firing_dcmds;
+
 	megasas_flush_cache(instance);
 	megasas_shutdown_controller(instance, MR_DCMD_CTRL_SHUTDOWN);
 
+skip_firing_dcmds:
 	/* cancel the delayed work if this work still in queue*/
 	if (instance->ev != NULL) {
 		struct megasas_aen_event *ev = instance->ev;
@@ -6302,7 +6328,7 @@ static void megasas_detach_one(struct pci_dev *pdev)
 	megasas_destroy_irqs(instance);
 
 	if (instance->msix_vectors)
-		pci_disable_msix(instance->pdev);
+		pci_free_irq_vectors(instance->pdev);
 
 	if (instance->ctrl_context) {
 		megasas_release_fusion(instance);
@@ -6385,13 +6411,19 @@ static void megasas_shutdown(struct pci_dev *pdev)
 	struct megasas_instance *instance = pci_get_drvdata(pdev);
 
 	instance->unload = 1;
+
+	if (megasas_wait_for_adapter_operational(instance))
+		goto skip_firing_dcmds;
+
 	megasas_flush_cache(instance);
 	megasas_shutdown_controller(instance, MR_DCMD_CTRL_SHUTDOWN);
+
+skip_firing_dcmds:
 	instance->instancet->disable_intr(instance);
 	megasas_destroy_irqs(instance);
 
 	if (instance->msix_vectors)
-		pci_disable_msix(instance->pdev);
+		pci_free_irq_vectors(instance->pdev);
 }
 
 /**
@@ -6752,8 +6784,7 @@ static int megasas_mgmt_ioctl_fw(struct file *file, unsigned long arg)
 	if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
 		spin_unlock_irqrestore(&instance->hba_lock, flags);
 
-		dev_err(&instance->pdev->dev, "timed out while"
-			"waiting for HBA to recover\n");
+		dev_err(&instance->pdev->dev, "timed out while waiting for HBA to recover\n");
 		error = -ENODEV;
 		goto out_up;
 	}
@@ -6821,8 +6852,7 @@ static int megasas_mgmt_ioctl_aen(struct file *file, unsigned long arg)
 	spin_lock_irqsave(&instance->hba_lock, flags);
 	if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
 		spin_unlock_irqrestore(&instance->hba_lock, flags);
-		dev_err(&instance->pdev->dev, "timed out while waiting"
-				"for HBA to recover\n");
+		dev_err(&instance->pdev->dev, "timed out while waiting for HBA to recover\n");
 		return -ENODEV;
 	}
 	spin_unlock_irqrestore(&instance->hba_lock, flags);
diff --git a/drivers/scsi/megaraid/megaraid_sas_fp.c b/drivers/scsi/megaraid/megaraid_sas_fp.c
index e413113c86ac..f237d0003df3 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fp.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fp.c
@@ -782,7 +782,8 @@ static u8 mr_spanset_get_phy_params(struct megasas_instance *instance, u32 ld,
 			(raid->regTypeReqOnRead != REGION_TYPE_UNUSED))))
 			pRAID_Context->regLockFlags = REGION_TYPE_EXCLUSIVE;
 		else if (raid->level == 1) {
-			pd = MR_ArPdGet(arRef, physArm + 1, map);
+			physArm = physArm + 1;
+			pd = MR_ArPdGet(arRef, physArm, map);
 			if (pd != MR_PD_INVALID)
 				*pDevHandle = MR_PdDevHandleGet(pd, map);
 		}
@@ -879,7 +880,8 @@ u8 MR_GetPhyParams(struct megasas_instance *instance, u32 ld, u64 stripRow,
 			pRAID_Context->regLockFlags = REGION_TYPE_EXCLUSIVE;
 		else if (raid->level == 1) {
 			/* Get alternate Pd. */
-			pd = MR_ArPdGet(arRef, physArm + 1, map);
+			physArm = physArm + 1;
+			pd = MR_ArPdGet(arRef, physArm, map);
 			if (pd != MR_PD_INVALID)
 				/* Get dev handle from Pd */
 				*pDevHandle = MR_PdDevHandleGet(pd, map);
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
index 52d8bbf7feb5..24778ba4b6e8 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
@@ -748,6 +748,11 @@ megasas_ioc_init_fusion(struct megasas_instance *instance)
 		goto fail_fw_init;
 	}
 
+	instance->fw_sync_cache_support = (scratch_pad_2 &
+		MR_CAN_HANDLE_SYNC_CACHE_OFFSET) ? 1 : 0;
+	dev_info(&instance->pdev->dev, "FW supports sync cache\t: %s\n",
+		 instance->fw_sync_cache_support ? "Yes" : "No");
+
 	IOCInitMessage =
 	  dma_alloc_coherent(&instance->pdev->dev,
 			     sizeof(struct MPI2_IOC_INIT_REQUEST),
@@ -2000,6 +2005,8 @@ megasas_build_syspd_fusion(struct megasas_instance *instance,
 		io_request->DevHandle = pd_sync->seq[pd_index].devHandle;
 		pRAID_Context->regLockFlags |=
 			(MR_RL_FLAGS_SEQ_NUM_ENABLE|MR_RL_FLAGS_GRANT_DESTINATION_CUDA);
+		pRAID_Context->Type = MPI2_TYPE_CUDA;
+		pRAID_Context->nseg = 0x1;
 	} else if (fusion->fast_path_io) {
 		pRAID_Context->VirtualDiskTgtId = cpu_to_le16(device_id);
 		pRAID_Context->configSeqNum = 0;
@@ -2035,12 +2042,10 @@ megasas_build_syspd_fusion(struct megasas_instance *instance,
 		pRAID_Context->timeoutValue =
 			cpu_to_le16((os_timeout_value > timeout_limit) ?
 			timeout_limit : os_timeout_value);
-		if (fusion->adapter_type == INVADER_SERIES) {
-			pRAID_Context->Type = MPI2_TYPE_CUDA;
-			pRAID_Context->nseg = 0x1;
+		if (fusion->adapter_type == INVADER_SERIES)
 			io_request->IoFlags |=
 				cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
-		}
+
 		cmd->request_desc->SCSIIO.RequestFlags =
 			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
 				MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
@@ -2463,12 +2468,15 @@ irqreturn_t megasas_isr_fusion(int irq, void *devp)
 			/* Start collecting crash, if DMA bit is done */
 			if ((fw_state == MFI_STATE_FAULT) && dma_state)
 				schedule_work(&instance->crash_init);
-			else if (fw_state == MFI_STATE_FAULT)
-				schedule_work(&instance->work_init);
+			else if (fw_state == MFI_STATE_FAULT) {
+				if (instance->unload == 0)
+					schedule_work(&instance->work_init);
+			}
 		} else if (fw_state == MFI_STATE_FAULT) {
 			dev_warn(&instance->pdev->dev, "Iop2SysDoorbellInt"
 			       "for scsi%d\n", instance->host->host_no);
-			schedule_work(&instance->work_init);
+			if (instance->unload == 0)
+				schedule_work(&instance->work_init);
 		}
 	}
 
@@ -2823,6 +2831,7 @@ int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance,
 		dev_err(&instance->pdev->dev, "pending commands remain after waiting, "
 		       "will reset adapter scsi%d.\n",
 		       instance->host->host_no);
+		*convert = 1;
 		retval = 1;
 	}
 out:
diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h
index 95356a82ee99..fa61baf7c74d 100644
--- a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h
+++ b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h
@@ -478,6 +478,13 @@ typedef struct _MPI2_CONFIG_REPLY {
 #define MPI26_MFGPAGE_DEVID_SAS3324_3               (0x00C2)
 #define MPI26_MFGPAGE_DEVID_SAS3324_4               (0x00C3)
 
+#define MPI26_MFGPAGE_DEVID_SAS3516                 (0x00AA)
+#define MPI26_MFGPAGE_DEVID_SAS3516_1               (0x00AB)
+#define MPI26_MFGPAGE_DEVID_SAS3416                 (0x00AC)
+#define MPI26_MFGPAGE_DEVID_SAS3508                 (0x00AD)
+#define MPI26_MFGPAGE_DEVID_SAS3508_1               (0x00AE)
+#define MPI26_MFGPAGE_DEVID_SAS3408                 (0x00AF)
+
 /*Manufacturing Page 0 */
 
 typedef struct _MPI2_CONFIG_PAGE_MAN_0 {
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
index a1a5ceb42ce6..f00ef88a378a 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -849,7 +849,7 @@ _base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply)
 	ack_request->EventContext = mpi_reply->EventContext;
 	ack_request->VF_ID = 0;  /* TODO */
 	ack_request->VP_ID = 0;
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 
  out:
 
@@ -1078,7 +1078,7 @@ _base_interrupt(int irq, void *bus_id)
 	 * new reply host index value in ReplyPostIndex Field and msix_index
 	 * value in MSIxIndex field.
 	 */
-	if (ioc->msix96_vector)
+	if (ioc->combined_reply_queue)
 		writel(reply_q->reply_post_host_index | ((msix_index  & 7) <<
 			MPI2_RPHI_MSIX_INDEX_SHIFT),
 			ioc->replyPostRegisterIndex[msix_index/8]);
@@ -1959,7 +1959,7 @@ _base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
 {
 	struct msix_entry *entries, *a;
 	int r;
-	int i;
+	int i, local_max_msix_vectors;
 	u8 try_msix = 0;
 
 	if (msix_disable == -1 || msix_disable == 0)
@@ -1979,13 +1979,15 @@ _base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
 	  ioc->cpu_count, max_msix_vectors);
 
 	if (!ioc->rdpq_array_enable && max_msix_vectors == -1)
-		max_msix_vectors = 8;
+		local_max_msix_vectors = 8;
+	else
+		local_max_msix_vectors = max_msix_vectors;
 
-	if (max_msix_vectors > 0) {
-		ioc->reply_queue_count = min_t(int, max_msix_vectors,
+	if (local_max_msix_vectors > 0) {
+		ioc->reply_queue_count = min_t(int, local_max_msix_vectors,
 			ioc->reply_queue_count);
 		ioc->msix_vector_count = ioc->reply_queue_count;
-	} else if (max_msix_vectors == 0)
+	} else if (local_max_msix_vectors == 0)
 		goto try_ioapic;
 
 	if (ioc->msix_vector_count < ioc->cpu_count)
@@ -2050,7 +2052,7 @@ mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc)
 	_base_free_irq(ioc);
 	_base_disable_msix(ioc);
 
-	if (ioc->msix96_vector) {
+	if (ioc->combined_reply_queue) {
 		kfree(ioc->replyPostRegisterIndex);
 		ioc->replyPostRegisterIndex = NULL;
 	}
@@ -2160,7 +2162,7 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
 	/* Use the Combined reply queue feature only for SAS3 C0 & higher
 	 * revision HBAs and also only when reply queue count is greater than 8
 	 */
-	if (ioc->msix96_vector && ioc->reply_queue_count > 8) {
+	if (ioc->combined_reply_queue && ioc->reply_queue_count > 8) {
 		/* Determine the Supplemental Reply Post Host Index Registers
 		 * Addresse. Supplemental Reply Post Host Index Registers
 		 * starts at offset MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET and
@@ -2168,7 +2170,7 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
 		 * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET from previous one.
 		 */
 		ioc->replyPostRegisterIndex = kcalloc(
-		     MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT,
+		     ioc->combined_reply_index_count,
 		     sizeof(resource_size_t *), GFP_KERNEL);
 		if (!ioc->replyPostRegisterIndex) {
 			dfailprintk(ioc, printk(MPT3SAS_FMT
@@ -2178,14 +2180,14 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
 			goto out_fail;
 		}
 
-		for (i = 0; i < MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT; i++) {
+		for (i = 0; i < ioc->combined_reply_index_count; i++) {
 			ioc->replyPostRegisterIndex[i] = (resource_size_t *)
 			     ((u8 *)&ioc->chip->Doorbell +
 			     MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET +
 			     (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET));
 		}
 	} else
-		ioc->msix96_vector = 0;
+		ioc->combined_reply_queue = 0;
 
 	if (ioc->is_warpdrive) {
 		ioc->reply_post_host_index[0] = (resource_size_t __iomem *)
@@ -2462,15 +2464,15 @@ _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
 #endif
 
 /**
- * mpt3sas_base_put_smid_scsi_io - send SCSI_IO request to firmware
+ * _base_put_smid_scsi_io - send SCSI_IO request to firmware
  * @ioc: per adapter object
  * @smid: system request message index
  * @handle: device handle
  *
  * Return nothing.
  */
-void
-mpt3sas_base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
+static void
+_base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
 {
 	Mpi2RequestDescriptorUnion_t descriptor;
 	u64 *request = (u64 *)&descriptor;
@@ -2486,15 +2488,15 @@ mpt3sas_base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
 }
 
 /**
- * mpt3sas_base_put_smid_fast_path - send fast path request to firmware
+ * _base_put_smid_fast_path - send fast path request to firmware
  * @ioc: per adapter object
  * @smid: system request message index
  * @handle: device handle
  *
  * Return nothing.
  */
-void
-mpt3sas_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid,
+static void
+_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid,
 	u16 handle)
 {
 	Mpi2RequestDescriptorUnion_t descriptor;
@@ -2511,14 +2513,14 @@ mpt3sas_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid,
 }
 
 /**
- * mpt3sas_base_put_smid_hi_priority - send Task Managment request to firmware
+ * _base_put_smid_hi_priority - send Task Management request to firmware
  * @ioc: per adapter object
  * @smid: system request message index
  * @msix_task: msix_task will be same as msix of IO incase of task abort else 0.
  * Return nothing.
  */
-void
-mpt3sas_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid,
+static void
+_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid,
 	u16 msix_task)
 {
 	Mpi2RequestDescriptorUnion_t descriptor;
@@ -2535,14 +2537,14 @@ mpt3sas_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid,
 }
 
 /**
- * mpt3sas_base_put_smid_default - Default, primarily used for config pages
+ * _base_put_smid_default - Default, primarily used for config pages
  * @ioc: per adapter object
  * @smid: system request message index
  *
  * Return nothing.
  */
-void
-mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)
+static void
+_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)
 {
 	Mpi2RequestDescriptorUnion_t descriptor;
 	u64 *request = (u64 *)&descriptor;
@@ -2557,6 +2559,95 @@ mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)
 }
 
 /**
+* _base_put_smid_scsi_io_atomic - send SCSI_IO request to firmware using
+*   Atomic Request Descriptor
+* @ioc: per adapter object
+* @smid: system request message index
+* @handle: device handle, unused in this function, for function type match
+*
+* Return nothing.
+*/
+static void
+_base_put_smid_scsi_io_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,
+	u16 handle)
+{
+	Mpi26AtomicRequestDescriptor_t descriptor;
+	u32 *request = (u32 *)&descriptor;
+
+	descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
+	descriptor.MSIxIndex = _base_get_msix_index(ioc);
+	descriptor.SMID = cpu_to_le16(smid);
+
+	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
+}
+
+/**
+ * _base_put_smid_fast_path_atomic - send fast path request to firmware
+ * using Atomic Request Descriptor
+ * @ioc: per adapter object
+ * @smid: system request message index
+ * @handle: device handle, unused in this function, for function type match
+ * Return nothing
+ */
+static void
+_base_put_smid_fast_path_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,
+	u16 handle)
+{
+	Mpi26AtomicRequestDescriptor_t descriptor;
+	u32 *request = (u32 *)&descriptor;
+
+	descriptor.RequestFlags = MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
+	descriptor.MSIxIndex = _base_get_msix_index(ioc);
+	descriptor.SMID = cpu_to_le16(smid);
+
+	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
+}
+
+/**
+ * _base_put_smid_hi_priority_atomic - send Task Management request to
+ * firmware using Atomic Request Descriptor
+ * @ioc: per adapter object
+ * @smid: system request message index
+ * @msix_task: msix_task will be same as msix of IO incase of task abort else 0
+ *
+ * Return nothing.
+ */
+static void
+_base_put_smid_hi_priority_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,
+	u16 msix_task)
+{
+	Mpi26AtomicRequestDescriptor_t descriptor;
+	u32 *request = (u32 *)&descriptor;
+
+	descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
+	descriptor.MSIxIndex = msix_task;
+	descriptor.SMID = cpu_to_le16(smid);
+
+	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
+}
+
+/**
+ * _base_put_smid_default - Default, primarily used for config pages
+ * use Atomic Request Descriptor
+ * @ioc: per adapter object
+ * @smid: system request message index
+ *
+ * Return nothing.
+ */
+static void
+_base_put_smid_default_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid)
+{
+	Mpi26AtomicRequestDescriptor_t descriptor;
+	u32 *request = (u32 *)&descriptor;
+
+	descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
+	descriptor.MSIxIndex = _base_get_msix_index(ioc);
+	descriptor.SMID = cpu_to_le16(smid);
+
+	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
+}
+
+/**
  * _base_display_OEMs_branding - Display branding string
  * @ioc: per adapter object
  *
@@ -4070,7 +4161,7 @@ mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc,
 	    mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET)
 		ioc->ioc_link_reset_in_progress = 1;
 	init_completion(&ioc->base_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->base_cmds.done,
 	    msecs_to_jiffies(10000));
 	if ((mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
@@ -4170,7 +4261,7 @@ mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,
 	ioc->base_cmds.smid = smid;
 	memcpy(request, mpi_request, sizeof(Mpi2SepReply_t));
 	init_completion(&ioc->base_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->base_cmds.done,
 	    msecs_to_jiffies(10000));
 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
@@ -4355,6 +4446,8 @@ _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc)
 	if ((facts->IOCCapabilities &
 	      MPI2_IOCFACTS_CAPABILITY_RDPQ_ARRAY_CAPABLE))
 		ioc->rdpq_array_capable = 1;
+	if (facts->IOCCapabilities & MPI26_IOCFACTS_CAPABILITY_ATOMIC_REQ)
+		ioc->atomic_desc_capable = 1;
 	facts->FWVersion.Word = le32_to_cpu(mpi_reply.FWVersion.Word);
 	facts->IOCRequestFrameSize =
 	    le16_to_cpu(mpi_reply.IOCRequestFrameSize);
@@ -4582,7 +4675,7 @@ _base_send_port_enable(struct MPT3SAS_ADAPTER *ioc)
 	mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
 
 	init_completion(&ioc->port_enable_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ);
 	if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s: timeout\n",
@@ -4645,7 +4738,7 @@ mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc)
 	memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
 	mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
 
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	return 0;
 }
 
@@ -4764,7 +4857,7 @@ _base_event_notification(struct MPT3SAS_ADAPTER *ioc)
 		mpi_request->EventMasks[i] =
 		    cpu_to_le32(ioc->event_masks[i]);
 	init_completion(&ioc->base_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ);
 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s: timeout\n",
@@ -5138,7 +5231,7 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
 
 	/* initialize reply post host index */
 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
-		if (ioc->msix96_vector)
+		if (ioc->combined_reply_queue)
 			writel((reply_q->msix_index & 7)<<
 			   MPI2_RPHI_MSIX_INDEX_SHIFT,
 			   ioc->replyPostRegisterIndex[reply_q->msix_index/8]);
@@ -5280,9 +5373,23 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
 		ioc->build_sg = &_base_build_sg_ieee;
 		ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee;
 		ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t);
+
 		break;
 	}
 
+	if (ioc->atomic_desc_capable) {
+		ioc->put_smid_default = &_base_put_smid_default_atomic;
+		ioc->put_smid_scsi_io = &_base_put_smid_scsi_io_atomic;
+		ioc->put_smid_fast_path = &_base_put_smid_fast_path_atomic;
+		ioc->put_smid_hi_priority = &_base_put_smid_hi_priority_atomic;
+	} else {
+		ioc->put_smid_default = &_base_put_smid_default;
+		ioc->put_smid_scsi_io = &_base_put_smid_scsi_io;
+		ioc->put_smid_fast_path = &_base_put_smid_fast_path;
+		ioc->put_smid_hi_priority = &_base_put_smid_hi_priority;
+	}
+
+
 	/*
 	 * These function pointers for other requests that don't
 	 * the require IEEE scatter gather elements.
@@ -5332,6 +5439,21 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
 		goto out_free_resources;
 	}
 
+	/* allocate memory for pending OS device add list */
+	ioc->pend_os_device_add_sz = (ioc->facts.MaxDevHandle / 8);
+	if (ioc->facts.MaxDevHandle % 8)
+		ioc->pend_os_device_add_sz++;
+	ioc->pend_os_device_add = kzalloc(ioc->pend_os_device_add_sz,
+	    GFP_KERNEL);
+	if (!ioc->pend_os_device_add)
+		goto out_free_resources;
+
+	ioc->device_remove_in_progress_sz = ioc->pend_os_device_add_sz;
+	ioc->device_remove_in_progress =
+		kzalloc(ioc->device_remove_in_progress_sz, GFP_KERNEL);
+	if (!ioc->device_remove_in_progress)
+		goto out_free_resources;
+
 	ioc->fwfault_debug = mpt3sas_fwfault_debug;
 
 	/* base internal command bits */
@@ -5414,6 +5536,8 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
 		kfree(ioc->reply_post_host_index);
 	kfree(ioc->pd_handles);
 	kfree(ioc->blocking_handles);
+	kfree(ioc->device_remove_in_progress);
+	kfree(ioc->pend_os_device_add);
 	kfree(ioc->tm_cmds.reply);
 	kfree(ioc->transport_cmds.reply);
 	kfree(ioc->scsih_cmds.reply);
@@ -5455,6 +5579,8 @@ mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc)
 		kfree(ioc->reply_post_host_index);
 	kfree(ioc->pd_handles);
 	kfree(ioc->blocking_handles);
+	kfree(ioc->device_remove_in_progress);
+	kfree(ioc->pend_os_device_add);
 	kfree(ioc->pfacts);
 	kfree(ioc->ctl_cmds.reply);
 	kfree(ioc->ctl_cmds.sense);
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h
index 3e71bc1b4a80..8de0eda8cd00 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.h
@@ -73,9 +73,9 @@
 #define MPT3SAS_DRIVER_NAME		"mpt3sas"
 #define MPT3SAS_AUTHOR "Avago Technologies <MPT-FusionLinux.pdl@avagotech.com>"
 #define MPT3SAS_DESCRIPTION	"LSI MPT Fusion SAS 3.0 Device Driver"
-#define MPT3SAS_DRIVER_VERSION		"13.100.00.00"
-#define MPT3SAS_MAJOR_VERSION		13
-#define MPT3SAS_MINOR_VERSION		100
+#define MPT3SAS_DRIVER_VERSION		"14.101.00.00"
+#define MPT3SAS_MAJOR_VERSION		14
+#define MPT3SAS_MINOR_VERSION		101
 #define MPT3SAS_BUILD_VERSION		0
 #define MPT3SAS_RELEASE_VERSION	00
 
@@ -300,8 +300,9 @@
  * There are twelve Supplemental Reply Post Host Index Registers
  * and each register is at offset 0x10 bytes from the previous one.
  */
-#define MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT 12
-#define MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET (0x10)
+#define MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G3	12
+#define MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G35	16
+#define MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET	(0x10)
 
 /* OEM Identifiers */
 #define MFG10_OEM_ID_INVALID                   (0x00000000)
@@ -375,7 +376,6 @@ struct MPT3SAS_TARGET {
  * per device private data
  */
 #define MPT_DEVICE_FLAGS_INIT		0x01
-#define MPT_DEVICE_TLR_ON		0x02
 
 #define MFG_PAGE10_HIDE_SSDS_MASK	(0x00000003)
 #define MFG_PAGE10_HIDE_ALL_DISKS	(0x00)
@@ -736,7 +736,10 @@ typedef void (*MPT_BUILD_SG)(struct MPT3SAS_ADAPTER *ioc, void *psge,
 typedef void (*MPT_BUILD_ZERO_LEN_SGE)(struct MPT3SAS_ADAPTER *ioc,
 		void *paddr);
 
-
+/* To support atomic and non atomic descriptors*/
+typedef void (*PUT_SMID_IO_FP_HIP) (struct MPT3SAS_ADAPTER *ioc, u16 smid,
+	u16 funcdep);
+typedef void (*PUT_SMID_DEFAULT) (struct MPT3SAS_ADAPTER *ioc, u16 smid);
 
 /* IOC Facts and Port Facts converted from little endian to cpu */
 union mpi3_version_union {
@@ -1079,6 +1082,9 @@ struct MPT3SAS_ADAPTER {
 	void		*pd_handles;
 	u16		pd_handles_sz;
 
+	void		*pend_os_device_add;
+	u16		pend_os_device_add_sz;
+
 	/* config page */
 	u16		config_page_sz;
 	void		*config_page;
@@ -1156,7 +1162,8 @@ struct MPT3SAS_ADAPTER {
 	u8		reply_queue_count;
 	struct list_head reply_queue_list;
 
-	u8		msix96_vector;
+	u8		combined_reply_queue;
+	u8		combined_reply_index_count;
 	/* reply post register index */
 	resource_size_t	**replyPostRegisterIndex;
 
@@ -1187,6 +1194,15 @@ struct MPT3SAS_ADAPTER {
 	struct SL_WH_EVENT_TRIGGERS_T diag_trigger_event;
 	struct SL_WH_SCSI_TRIGGERS_T diag_trigger_scsi;
 	struct SL_WH_MPI_TRIGGERS_T diag_trigger_mpi;
+	void		*device_remove_in_progress;
+	u16		device_remove_in_progress_sz;
+	u8		is_gen35_ioc;
+	u8		atomic_desc_capable;
+	PUT_SMID_IO_FP_HIP put_smid_scsi_io;
+	PUT_SMID_IO_FP_HIP put_smid_fast_path;
+	PUT_SMID_IO_FP_HIP put_smid_hi_priority;
+	PUT_SMID_DEFAULT put_smid_default;
+
 };
 
 typedef u8 (*MPT_CALLBACK)(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
@@ -1232,13 +1248,6 @@ u16 mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx,
 
 u16 mpt3sas_base_get_smid(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx);
 void mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid);
-void mpt3sas_base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid,
-	u16 handle);
-void mpt3sas_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid,
-	u16 handle);
-void mpt3sas_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc,
-	u16 smid, u16 msix_task);
-void mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid);
 void mpt3sas_base_initialize_callback_handler(void);
 u8 mpt3sas_base_register_callback_handler(MPT_CALLBACK cb_func);
 void mpt3sas_base_release_callback_handler(u8 cb_idx);
diff --git a/drivers/scsi/mpt3sas/mpt3sas_config.c b/drivers/scsi/mpt3sas/mpt3sas_config.c
index cebfd734fd76..dd6270125614 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_config.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_config.c
@@ -384,7 +384,7 @@ _config_request(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigRequest_t
 	memcpy(config_request, mpi_request, sizeof(Mpi2ConfigRequest_t));
 	_config_display_some_debug(ioc, smid, "config_request", NULL);
 	init_completion(&ioc->config_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->config_cmds.done, timeout*HZ);
 	if (!(ioc->config_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s: timeout\n",
diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.c b/drivers/scsi/mpt3sas/mpt3sas_ctl.c
index 26cdc127ac89..050bd788ad02 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_ctl.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.c
@@ -654,6 +654,7 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 	size_t data_in_sz = 0;
 	long ret;
 	u16 wait_state_count;
+	u16 device_handle = MPT3SAS_INVALID_DEVICE_HANDLE;
 
 	issue_reset = 0;
 
@@ -738,10 +739,13 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 	data_in_sz = karg.data_in_size;
 
 	if (mpi_request->Function == MPI2_FUNCTION_SCSI_IO_REQUEST ||
-	    mpi_request->Function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH) {
-		if (!le16_to_cpu(mpi_request->FunctionDependent1) ||
-		    le16_to_cpu(mpi_request->FunctionDependent1) >
-		    ioc->facts.MaxDevHandle) {
+	    mpi_request->Function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH ||
+	    mpi_request->Function == MPI2_FUNCTION_SCSI_TASK_MGMT ||
+	    mpi_request->Function == MPI2_FUNCTION_SATA_PASSTHROUGH) {
+
+		device_handle = le16_to_cpu(mpi_request->FunctionDependent1);
+		if (!device_handle || (device_handle >
+		    ioc->facts.MaxDevHandle)) {
 			ret = -EINVAL;
 			mpt3sas_base_free_smid(ioc, smid);
 			goto out;
@@ -797,14 +801,20 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 		scsiio_request->SenseBufferLowAddress =
 		    mpt3sas_base_get_sense_buffer_dma(ioc, smid);
 		memset(ioc->ctl_cmds.sense, 0, SCSI_SENSE_BUFFERSIZE);
+		if (test_bit(device_handle, ioc->device_remove_in_progress)) {
+			dtmprintk(ioc, pr_info(MPT3SAS_FMT
+				"handle(0x%04x) :ioctl failed due to device removal in progress\n",
+				ioc->name, device_handle));
+			mpt3sas_base_free_smid(ioc, smid);
+			ret = -EINVAL;
+			goto out;
+		}
 		ioc->build_sg(ioc, psge, data_out_dma, data_out_sz,
 		    data_in_dma, data_in_sz);
-
 		if (mpi_request->Function == MPI2_FUNCTION_SCSI_IO_REQUEST)
-			mpt3sas_base_put_smid_scsi_io(ioc, smid,
-			    le16_to_cpu(mpi_request->FunctionDependent1));
+			ioc->put_smid_scsi_io(ioc, smid, device_handle);
 		else
-			mpt3sas_base_put_smid_default(ioc, smid);
+			ioc->put_smid_default(ioc, smid);
 		break;
 	}
 	case MPI2_FUNCTION_SCSI_TASK_MGMT:
@@ -827,11 +837,19 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 			}
 		}
 
+		if (test_bit(device_handle, ioc->device_remove_in_progress)) {
+			dtmprintk(ioc, pr_info(MPT3SAS_FMT
+				"handle(0x%04x) :ioctl failed due to device removal in progress\n",
+				ioc->name, device_handle));
+			mpt3sas_base_free_smid(ioc, smid);
+			ret = -EINVAL;
+			goto out;
+		}
 		mpt3sas_scsih_set_tm_flag(ioc, le16_to_cpu(
 		    tm_request->DevHandle));
 		ioc->build_sg_mpi(ioc, psge, data_out_dma, data_out_sz,
 		    data_in_dma, data_in_sz);
-		mpt3sas_base_put_smid_hi_priority(ioc, smid, 0);
+		ioc->put_smid_hi_priority(ioc, smid, 0);
 		break;
 	}
 	case MPI2_FUNCTION_SMP_PASSTHROUGH:
@@ -862,16 +880,30 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 		}
 		ioc->build_sg(ioc, psge, data_out_dma, data_out_sz, data_in_dma,
 		    data_in_sz);
-		mpt3sas_base_put_smid_default(ioc, smid);
+		ioc->put_smid_default(ioc, smid);
 		break;
 	}
 	case MPI2_FUNCTION_SATA_PASSTHROUGH:
+	{
+		if (test_bit(device_handle, ioc->device_remove_in_progress)) {
+			dtmprintk(ioc, pr_info(MPT3SAS_FMT
+				"handle(0x%04x) :ioctl failed due to device removal in progress\n",
+				ioc->name, device_handle));
+			mpt3sas_base_free_smid(ioc, smid);
+			ret = -EINVAL;
+			goto out;
+		}
+		ioc->build_sg(ioc, psge, data_out_dma, data_out_sz, data_in_dma,
+		    data_in_sz);
+		ioc->put_smid_default(ioc, smid);
+		break;
+	}
 	case MPI2_FUNCTION_FW_DOWNLOAD:
 	case MPI2_FUNCTION_FW_UPLOAD:
 	{
 		ioc->build_sg(ioc, psge, data_out_dma, data_out_sz, data_in_dma,
 		    data_in_sz);
-		mpt3sas_base_put_smid_default(ioc, smid);
+		ioc->put_smid_default(ioc, smid);
 		break;
 	}
 	case MPI2_FUNCTION_TOOLBOX:
@@ -886,7 +918,7 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 			ioc->build_sg_mpi(ioc, psge, data_out_dma, data_out_sz,
 				data_in_dma, data_in_sz);
 		}
-		mpt3sas_base_put_smid_default(ioc, smid);
+		ioc->put_smid_default(ioc, smid);
 		break;
 	}
 	case MPI2_FUNCTION_SAS_IO_UNIT_CONTROL:
@@ -905,7 +937,7 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg,
 	default:
 		ioc->build_sg_mpi(ioc, psge, data_out_dma, data_out_sz,
 		    data_in_dma, data_in_sz);
-		mpt3sas_base_put_smid_default(ioc, smid);
+		ioc->put_smid_default(ioc, smid);
 		break;
 	}
 
@@ -1064,7 +1096,10 @@ _ctl_getiocinfo(struct MPT3SAS_ADAPTER *ioc, void __user *arg)
 		break;
 	case MPI25_VERSION:
 	case MPI26_VERSION:
-		karg.adapter_type = MPT3_IOCTL_INTERFACE_SAS3;
+		if (ioc->is_gen35_ioc)
+			karg.adapter_type = MPT3_IOCTL_INTERFACE_SAS35;
+		else
+			karg.adapter_type = MPT3_IOCTL_INTERFACE_SAS3;
 		strcat(karg.driver_version, MPT3SAS_DRIVER_VERSION);
 		break;
 	}
@@ -1491,7 +1526,7 @@ _ctl_diag_register_2(struct MPT3SAS_ADAPTER *ioc,
 			cpu_to_le32(ioc->product_specific[buffer_type][i]);
 
 	init_completion(&ioc->ctl_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->ctl_cmds.done,
 	    MPT3_IOCTL_DEFAULT_TIMEOUT*HZ);
 
@@ -1838,7 +1873,7 @@ mpt3sas_send_diag_release(struct MPT3SAS_ADAPTER *ioc, u8 buffer_type,
 	mpi_request->VP_ID = 0;
 
 	init_completion(&ioc->ctl_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->ctl_cmds.done,
 	    MPT3_IOCTL_DEFAULT_TIMEOUT*HZ);
 
@@ -2105,7 +2140,7 @@ _ctl_diag_read_buffer(struct MPT3SAS_ADAPTER *ioc, void __user *arg)
 	mpi_request->VP_ID = 0;
 
 	init_completion(&ioc->ctl_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->ctl_cmds.done,
 	    MPT3_IOCTL_DEFAULT_TIMEOUT*HZ);
 
diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.h b/drivers/scsi/mpt3sas/mpt3sas_ctl.h
index 89408356d252..f3e17a8c1b07 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_ctl.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.h
@@ -143,6 +143,7 @@ struct mpt3_ioctl_pci_info {
 #define MPT2_IOCTL_INTERFACE_SAS2	(0x04)
 #define MPT2_IOCTL_INTERFACE_SAS2_SSS6200	(0x05)
 #define MPT3_IOCTL_INTERFACE_SAS3	(0x06)
+#define MPT3_IOCTL_INTERFACE_SAS35	(0x07)
 #define MPT2_IOCTL_VERSION_LENGTH	(32)
 
 /**
diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
index 1c4744e78173..5c8f75247d73 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
@@ -423,7 +423,7 @@ _scsih_get_sas_address(struct MPT3SAS_ADAPTER *ioc, u16 handle,
 		return 0;
 	}
 
-	/* we hit this becuase the given parent handle doesn't exist */
+	/* we hit this because the given parent handle doesn't exist */
 	if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE)
 		return -ENXIO;
 
@@ -788,6 +788,11 @@ _scsih_sas_device_add(struct MPT3SAS_ADAPTER *ioc,
 	list_add_tail(&sas_device->list, &ioc->sas_device_list);
 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 
+	if (ioc->hide_drives) {
+		clear_bit(sas_device->handle, ioc->pend_os_device_add);
+		return;
+	}
+
 	if (!mpt3sas_transport_port_add(ioc, sas_device->handle,
 	     sas_device->sas_address_parent)) {
 		_scsih_sas_device_remove(ioc, sas_device);
@@ -803,7 +808,8 @@ _scsih_sas_device_add(struct MPT3SAS_ADAPTER *ioc,
 			    sas_device->sas_address_parent);
 			_scsih_sas_device_remove(ioc, sas_device);
 		}
-	}
+	} else
+		clear_bit(sas_device->handle, ioc->pend_os_device_add);
 }
 
 /**
@@ -1517,7 +1523,7 @@ _scsih_display_sata_capabilities(struct MPT3SAS_ADAPTER *ioc,
 /*
  * raid transport support -
  * Enabled for SLES11 and newer, in older kernels the driver will panic when
- * unloading the driver followed by a load - I beleive that the subroutine
+ * unloading the driver followed by a load - I believe that the subroutine
  * raid_class_release() is not cleaning up properly.
  */
 
@@ -2279,7 +2285,7 @@ mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, uint channel,
 		msix_task = scsi_lookup->msix_io;
 	else
 		msix_task = 0;
-	mpt3sas_base_put_smid_hi_priority(ioc, smid, msix_task);
+	ioc->put_smid_hi_priority(ioc, smid, msix_task);
 	wait_for_completion_timeout(&ioc->tm_cmds.done, timeout*HZ);
 	if (!(ioc->tm_cmds.status & MPT3_CMD_COMPLETE)) {
 		pr_err(MPT3SAS_FMT "%s: timeout\n",
@@ -2837,7 +2843,7 @@ _scsih_internal_device_block(struct scsi_device *sdev,
 	if (r == -EINVAL)
 		sdev_printk(KERN_WARNING, sdev,
 		    "device_block failed with return(%d) for handle(0x%04x)\n",
-		    sas_device_priv_data->sas_target->handle, r);
+		    r, sas_device_priv_data->sas_target->handle);
 }
 
 /**
@@ -2867,20 +2873,20 @@ _scsih_internal_device_unblock(struct scsi_device *sdev,
 		sdev_printk(KERN_WARNING, sdev,
 		    "device_unblock failed with return(%d) for handle(0x%04x) "
 		    "performing a block followed by an unblock\n",
-		    sas_device_priv_data->sas_target->handle, r);
+		    r, sas_device_priv_data->sas_target->handle);
 		sas_device_priv_data->block = 1;
 		r = scsi_internal_device_block(sdev);
 		if (r)
 			sdev_printk(KERN_WARNING, sdev, "retried device_block "
 			    "failed with return(%d) for handle(0x%04x)\n",
-			    sas_device_priv_data->sas_target->handle, r);
+			    r, sas_device_priv_data->sas_target->handle);
 
 		sas_device_priv_data->block = 0;
 		r = scsi_internal_device_unblock(sdev, SDEV_RUNNING);
 		if (r)
 			sdev_printk(KERN_WARNING, sdev, "retried device_unblock"
 			    " failed with return(%d) for handle(0x%04x)\n",
-			    sas_device_priv_data->sas_target->handle, r);
+			    r, sas_device_priv_data->sas_target->handle);
 	}
 }
 
@@ -2942,7 +2948,7 @@ _scsih_ublock_io_device(struct MPT3SAS_ADAPTER *ioc, u64 sas_address)
  * @ioc: per adapter object
  * @handle: device handle
  *
- * During device pull we need to appropiately set the sdev state.
+ * During device pull we need to appropriately set the sdev state.
  */
 static void
 _scsih_block_io_all_device(struct MPT3SAS_ADAPTER *ioc)
@@ -2971,7 +2977,7 @@ _scsih_block_io_all_device(struct MPT3SAS_ADAPTER *ioc)
  * @ioc: per adapter object
  * @handle: device handle
  *
- * During device pull we need to appropiately set the sdev state.
+ * During device pull we need to appropriately set the sdev state.
  */
 static void
 _scsih_block_io_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
@@ -3138,6 +3144,8 @@ _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	if (test_bit(handle, ioc->pd_handles))
 		return;
 
+	clear_bit(handle, ioc->pend_os_device_add);
+
 	spin_lock_irqsave(&ioc->sas_device_lock, flags);
 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
 	if (sas_device && sas_device->starget &&
@@ -3192,7 +3200,8 @@ _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
 	mpi_request->DevHandle = cpu_to_le16(handle);
 	mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
-	mpt3sas_base_put_smid_hi_priority(ioc, smid, 0);
+	set_bit(handle, ioc->device_remove_in_progress);
+	ioc->put_smid_hi_priority(ioc, smid, 0);
 	mpt3sas_trigger_master(ioc, MASTER_TRIGGER_DEVICE_REMOVAL);
 
 out:
@@ -3291,7 +3300,7 @@ _scsih_tm_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
 	mpi_request->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
 	mpi_request->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
 	mpi_request->DevHandle = mpi_request_tm->DevHandle;
-	mpt3sas_base_put_smid_default(ioc, smid_sas_ctrl);
+	ioc->put_smid_default(ioc, smid_sas_ctrl);
 
 	return _scsih_check_for_pending_tm(ioc, smid);
 }
@@ -3326,6 +3335,11 @@ _scsih_sas_control_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,
 		ioc->name, le16_to_cpu(mpi_reply->DevHandle), smid,
 		le16_to_cpu(mpi_reply->IOCStatus),
 		le32_to_cpu(mpi_reply->IOCLogInfo)));
+		if (le16_to_cpu(mpi_reply->IOCStatus) ==
+		     MPI2_IOCSTATUS_SUCCESS) {
+			clear_bit(le16_to_cpu(mpi_reply->DevHandle),
+			    ioc->device_remove_in_progress);
+		}
 	} else {
 		pr_err(MPT3SAS_FMT "mpi_reply not valid at %s:%d/%s()!\n",
 		    ioc->name, __FILE__, __LINE__, __func__);
@@ -3381,7 +3395,7 @@ _scsih_tm_tr_volume_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
 	mpi_request->DevHandle = cpu_to_le16(handle);
 	mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
-	mpt3sas_base_put_smid_hi_priority(ioc, smid, 0);
+	ioc->put_smid_hi_priority(ioc, smid, 0);
 }
 
 /**
@@ -3473,7 +3487,7 @@ _scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 event,
 	ack_request->EventContext = event_context;
 	ack_request->VF_ID = 0;  /* TODO */
 	ack_request->VP_ID = 0;
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 }
 
 /**
@@ -3530,7 +3544,7 @@ _scsih_issue_delayed_sas_io_unit_ctrl(struct MPT3SAS_ADAPTER *ioc,
 	mpi_request->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
 	mpi_request->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
 	mpi_request->DevHandle = handle;
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 }
 
 /**
@@ -3930,7 +3944,7 @@ _scsih_flush_running_cmds(struct MPT3SAS_ADAPTER *ioc)
  * _scsih_setup_eedp - setup MPI request for EEDP transfer
  * @ioc: per adapter object
  * @scmd: pointer to scsi command object
- * @mpi_request: pointer to the SCSI_IO reqest message frame
+ * @mpi_request: pointer to the SCSI_IO request message frame
  *
  * Supporting protection 1 and 3.
  *
@@ -3983,6 +3997,9 @@ _scsih_setup_eedp(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,
 
 	mpi_request_3v->EEDPBlockSize =
 	    cpu_to_le16(scmd->device->sector_size);
+
+	if (ioc->is_gen35_ioc)
+		eedp_flags |= MPI25_SCSIIO_EEDPFLAGS_APPTAG_DISABLE_MODE;
 	mpi_request->EEDPFlags = cpu_to_le16(eedp_flags);
 }
 
@@ -4084,7 +4101,7 @@ scsih_qcmd(struct Scsi_Host *shost, struct scsi_cmnd *scmd)
 		scmd->result = DID_NO_CONNECT << 16;
 		scmd->scsi_done(scmd);
 		return 0;
-	/* device busy with task managment */
+	/* device busy with task management */
 	} else if (sas_target_priv_data->tm_busy ||
 	    sas_device_priv_data->block)
 		return SCSI_MLQUEUE_DEVICE_BUSY;
@@ -4154,12 +4171,12 @@ scsih_qcmd(struct Scsi_Host *shost, struct scsi_cmnd *scmd)
 		if (sas_target_priv_data->flags & MPT_TARGET_FASTPATH_IO) {
 			mpi_request->IoFlags = cpu_to_le16(scmd->cmd_len |
 			    MPI25_SCSIIO_IOFLAGS_FAST_PATH);
-			mpt3sas_base_put_smid_fast_path(ioc, smid, handle);
+			ioc->put_smid_fast_path(ioc, smid, handle);
 		} else
-			mpt3sas_base_put_smid_scsi_io(ioc, smid,
+			ioc->put_smid_scsi_io(ioc, smid,
 			    le16_to_cpu(mpi_request->DevHandle));
 	} else
-		mpt3sas_base_put_smid_default(ioc, smid);
+		ioc->put_smid_default(ioc, smid);
 	return 0;
 
  out:
@@ -4658,7 +4675,7 @@ _scsih_io_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
 		memcpy(mpi_request->CDB.CDB32, scmd->cmnd, scmd->cmd_len);
 		mpi_request->DevHandle =
 		    cpu_to_le16(sas_device_priv_data->sas_target->handle);
-		mpt3sas_base_put_smid_scsi_io(ioc, smid,
+		ioc->put_smid_scsi_io(ioc, smid,
 		    sas_device_priv_data->sas_target->handle);
 		return 0;
 	}
@@ -5383,10 +5400,10 @@ _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
 			sas_device->handle, handle);
 		sas_target_priv_data->handle = handle;
 		sas_device->handle = handle;
-		if (sas_device_pg0.Flags &
+		if (le16_to_cpu(sas_device_pg0.Flags) &
 		     MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
 			sas_device->enclosure_level =
-				le16_to_cpu(sas_device_pg0.EnclosureLevel);
+				sas_device_pg0.EnclosureLevel;
 			memcpy(sas_device->connector_name,
 				sas_device_pg0.ConnectorName, 4);
 			sas_device->connector_name[4] = '\0';
@@ -5465,6 +5482,7 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
 	device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
 	if (!(_scsih_is_end_device(device_info)))
 		return -1;
+	set_bit(handle, ioc->pend_os_device_add);
 	sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
 
 	/* check if device is present */
@@ -5483,6 +5501,7 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
 	sas_device = mpt3sas_get_sdev_by_addr(ioc,
 					sas_address);
 	if (sas_device) {
+		clear_bit(handle, ioc->pend_os_device_add);
 		sas_device_put(sas_device);
 		return -1;
 	}
@@ -5513,9 +5532,10 @@ _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
 	sas_device->fast_path = (le16_to_cpu(sas_device_pg0.Flags) &
 	    MPI25_SAS_DEVICE0_FLAGS_FAST_PATH_CAPABLE) ? 1 : 0;
 
-	if (sas_device_pg0.Flags & MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
+	if (le16_to_cpu(sas_device_pg0.Flags)
+		& MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
 		sas_device->enclosure_level =
-			le16_to_cpu(sas_device_pg0.EnclosureLevel);
+			sas_device_pg0.EnclosureLevel;
 		memcpy(sas_device->connector_name,
 			sas_device_pg0.ConnectorName, 4);
 		sas_device->connector_name[4] = '\0';
@@ -5806,6 +5826,9 @@ _scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
 			_scsih_check_device(ioc, sas_address, handle,
 			    phy_number, link_rate);
 
+			if (!test_bit(handle, ioc->pend_os_device_add))
+				break;
+
 
 		case MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED:
 
@@ -6267,7 +6290,7 @@ _scsih_ir_fastpath(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phys_disk_num)
 	    handle, phys_disk_num));
 
 	init_completion(&ioc->scsih_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->scsih_cmds.done, 10*HZ);
 
 	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
@@ -6320,7 +6343,7 @@ _scsih_reprobe_lun(struct scsi_device *sdev, void *no_uld_attach)
 {
 	sdev->no_uld_attach = no_uld_attach ? 1 : 0;
 	sdev_printk(KERN_INFO, sdev, "%s raid component\n",
-	    sdev->no_uld_attach ? "hidding" : "exposing");
+	    sdev->no_uld_attach ? "hiding" : "exposing");
 	WARN_ON(scsi_device_reprobe(sdev));
 }
 
@@ -7050,7 +7073,7 @@ Mpi2SasDevicePage0_t *sas_device_pg0)
 			if (sas_device_pg0->Flags &
 			      MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
 				sas_device->enclosure_level =
-				   le16_to_cpu(sas_device_pg0->EnclosureLevel);
+				   sas_device_pg0->EnclosureLevel;
 				memcpy(&sas_device->connector_name[0],
 					&sas_device_pg0->ConnectorName[0], 4);
 			} else {
@@ -7112,6 +7135,7 @@ _scsih_search_responding_sas_devices(struct MPT3SAS_ADAPTER *ioc)
 		sas_device_pg0.SASAddress =
 				le64_to_cpu(sas_device_pg0.SASAddress);
 		sas_device_pg0.Slot = le16_to_cpu(sas_device_pg0.Slot);
+		sas_device_pg0.Flags = le16_to_cpu(sas_device_pg0.Flags);
 		_scsih_mark_responding_sas_device(ioc, &sas_device_pg0);
 	}
 
@@ -7723,6 +7747,9 @@ mpt3sas_scsih_reset_handler(struct MPT3SAS_ADAPTER *ioc, int reset_phase)
 			complete(&ioc->tm_cmds.done);
 		}
 
+		memset(ioc->pend_os_device_add, 0, ioc->pend_os_device_add_sz);
+		memset(ioc->device_remove_in_progress, 0,
+		       ioc->device_remove_in_progress_sz);
 		_scsih_fw_event_cleanup_queue(ioc);
 		_scsih_flush_running_cmds(ioc);
 		break;
@@ -8113,7 +8140,7 @@ _scsih_ir_shutdown(struct MPT3SAS_ADAPTER *ioc)
 	if (!ioc->hide_ir_msg)
 		pr_info(MPT3SAS_FMT "IR shutdown (sending)\n", ioc->name);
 	init_completion(&ioc->scsih_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->scsih_cmds.done, 10*HZ);
 
 	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
@@ -8654,6 +8681,12 @@ _scsih_determine_hba_mpi_version(struct pci_dev *pdev)
 	case MPI26_MFGPAGE_DEVID_SAS3324_2:
 	case MPI26_MFGPAGE_DEVID_SAS3324_3:
 	case MPI26_MFGPAGE_DEVID_SAS3324_4:
+	case MPI26_MFGPAGE_DEVID_SAS3508:
+	case MPI26_MFGPAGE_DEVID_SAS3508_1:
+	case MPI26_MFGPAGE_DEVID_SAS3408:
+	case MPI26_MFGPAGE_DEVID_SAS3516:
+	case MPI26_MFGPAGE_DEVID_SAS3516_1:
+	case MPI26_MFGPAGE_DEVID_SAS3416:
 		return MPI26_VERSION;
 	}
 	return 0;
@@ -8722,10 +8755,29 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		ioc->hba_mpi_version_belonged = hba_mpi_version;
 		ioc->id = mpt3_ids++;
 		sprintf(ioc->driver_name, "%s", MPT3SAS_DRIVER_NAME);
+		switch (pdev->device) {
+		case MPI26_MFGPAGE_DEVID_SAS3508:
+		case MPI26_MFGPAGE_DEVID_SAS3508_1:
+		case MPI26_MFGPAGE_DEVID_SAS3408:
+		case MPI26_MFGPAGE_DEVID_SAS3516:
+		case MPI26_MFGPAGE_DEVID_SAS3516_1:
+		case MPI26_MFGPAGE_DEVID_SAS3416:
+			ioc->is_gen35_ioc = 1;
+			break;
+		default:
+			ioc->is_gen35_ioc = 0;
+		}
 		if ((ioc->hba_mpi_version_belonged == MPI25_VERSION &&
 			pdev->revision >= SAS3_PCI_DEVICE_C0_REVISION) ||
-			(ioc->hba_mpi_version_belonged == MPI26_VERSION))
-			ioc->msix96_vector = 1;
+			(ioc->hba_mpi_version_belonged == MPI26_VERSION)) {
+			ioc->combined_reply_queue = 1;
+			if (ioc->is_gen35_ioc)
+				ioc->combined_reply_index_count =
+				 MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G35;
+			else
+				ioc->combined_reply_index_count =
+				 MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G3;
+		}
 		break;
 	default:
 		return -ENODEV;
@@ -9128,6 +9180,19 @@ static const struct pci_device_id mpt3sas_pci_table[] = {
 		PCI_ANY_ID, PCI_ANY_ID },
 	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_4,
 		PCI_ANY_ID, PCI_ANY_ID },
+	/* Ventura, Crusader, Harpoon & Tomcat ~ 3516, 3416, 3508 & 3408*/
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3508,
+		PCI_ANY_ID, PCI_ANY_ID },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3508_1,
+		PCI_ANY_ID, PCI_ANY_ID },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3408,
+		PCI_ANY_ID, PCI_ANY_ID },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3516,
+		PCI_ANY_ID, PCI_ANY_ID },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3516_1,
+		PCI_ANY_ID, PCI_ANY_ID },
+	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3416,
+		PCI_ANY_ID, PCI_ANY_ID },
 	{0}     /* Terminating entry */
 };
 MODULE_DEVICE_TABLE(pci, mpt3sas_pci_table);
@@ -9168,7 +9233,7 @@ scsih_init(void)
 	 /* queuecommand callback hander */
 	scsi_io_cb_idx = mpt3sas_base_register_callback_handler(_scsih_io_done);
 
-	/* task managment callback handler */
+	/* task management callback handler */
 	tm_cb_idx = mpt3sas_base_register_callback_handler(_scsih_tm_done);
 
 	/* base internal commands callback handler */
diff --git a/drivers/scsi/mpt3sas/mpt3sas_transport.c b/drivers/scsi/mpt3sas/mpt3sas_transport.c
index b74faf1a69b2..7f1d5785bc30 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_transport.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_transport.c
@@ -392,7 +392,7 @@ _transport_expander_report_manufacture(struct MPT3SAS_ADAPTER *ioc,
 		"report_manufacture - send to sas_addr(0x%016llx)\n",
 		ioc->name, (unsigned long long)sas_address));
 	init_completion(&ioc->transport_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->transport_cmds.done, 10*HZ);
 
 	if (!(ioc->transport_cmds.status & MPT3_CMD_COMPLETE)) {
@@ -1198,7 +1198,7 @@ _transport_get_expander_phy_error_log(struct MPT3SAS_ADAPTER *ioc,
 		ioc->name, (unsigned long long)phy->identify.sas_address,
 		phy->number));
 	init_completion(&ioc->transport_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->transport_cmds.done, 10*HZ);
 
 	if (!(ioc->transport_cmds.status & MPT3_CMD_COMPLETE)) {
@@ -1514,7 +1514,7 @@ _transport_expander_phy_control(struct MPT3SAS_ADAPTER *ioc,
 		ioc->name, (unsigned long long)phy->identify.sas_address,
 		phy->number, phy_operation));
 	init_completion(&ioc->transport_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->transport_cmds.done, 10*HZ);
 
 	if (!(ioc->transport_cmds.status & MPT3_CMD_COMPLETE)) {
@@ -2032,7 +2032,7 @@ _transport_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 		"%s - sending smp request\n", ioc->name, __func__));
 
 	init_completion(&ioc->transport_cmds.done);
-	mpt3sas_base_put_smid_default(ioc, smid);
+	ioc->put_smid_default(ioc, smid);
 	wait_for_completion_timeout(&ioc->transport_cmds.done, 10*HZ);
 
 	if (!(ioc->transport_cmds.status & MPT3_CMD_COMPLETE)) {
diff --git a/drivers/scsi/mvsas/mv_94xx.c b/drivers/scsi/mvsas/mv_94xx.c
index 4c57d9abce7b..7de5d8d75480 100644
--- a/drivers/scsi/mvsas/mv_94xx.c
+++ b/drivers/scsi/mvsas/mv_94xx.c
@@ -668,7 +668,7 @@ static void mvs_94xx_command_active(struct mvs_info *mvi, u32 slot_idx)
 {
 	u32 tmp;
 	tmp = mvs_cr32(mvi, MVS_COMMAND_ACTIVE+(slot_idx >> 3));
-	if (tmp && 1 << (slot_idx % 32)) {
+	if (tmp & 1 << (slot_idx % 32)) {
 		mv_printk("command active %08X,  slot [%x].\n", tmp, slot_idx);
 		mvs_cw32(mvi, MVS_COMMAND_ACTIVE + (slot_idx >> 3),
 			1 << (slot_idx % 32));
diff --git a/drivers/scsi/osd/osd_initiator.c b/drivers/scsi/osd/osd_initiator.c
index 2f2a9910e30e..ef99f62831fb 100644
--- a/drivers/scsi/osd/osd_initiator.c
+++ b/drivers/scsi/osd/osd_initiator.c
@@ -1595,7 +1595,7 @@ static int _init_blk_request(struct osd_request *or,
 	}
 
 	or->request = req;
-	req->cmd_flags |= REQ_QUIET;
+	req->rq_flags |= RQF_QUIET;
 
 	req->timeout = or->timeout;
 	req->retries = or->retries;
diff --git a/drivers/scsi/osst.c b/drivers/scsi/osst.c
index 5033223f6287..a2960f5d98ec 100644
--- a/drivers/scsi/osst.c
+++ b/drivers/scsi/osst.c
@@ -368,7 +368,7 @@ static int osst_execute(struct osst_request *SRpnt, const unsigned char *cmd,
 		return DRIVER_ERROR << 24;
 
 	blk_rq_set_block_pc(req);
-	req->cmd_flags |= REQ_QUIET;
+	req->rq_flags |= RQF_QUIET;
 
 	SRpnt->bio = NULL;
 
diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c
index 845affa112f7..337982cf3d63 100644
--- a/drivers/scsi/pmcraid.c
+++ b/drivers/scsi/pmcraid.c
@@ -3787,11 +3787,11 @@ static long pmcraid_ioctl_passthrough(
 						      direction);
 		if (rc) {
 			pmcraid_err("couldn't build passthrough ioadls\n");
-			goto out_free_buffer;
+			goto out_free_cmd;
 		}
 	} else if (request_size < 0) {
 		rc = -EINVAL;
-		goto out_free_buffer;
+		goto out_free_cmd;
 	}
 
 	/* If data is being written into the device, copy the data from user
@@ -3908,6 +3908,8 @@ out_handle_response:
 
 out_free_sglist:
 	pmcraid_release_passthrough_ioadls(cmd, request_size, direction);
+
+out_free_cmd:
 	pmcraid_return_cmd(cmd);
 
 out_free_buffer:
@@ -6018,8 +6020,10 @@ static int __init pmcraid_init(void)
 
 	error = pmcraid_netlink_init();
 
-	if (error)
+	if (error) {
+		class_destroy(pmcraid_class);
 		goto out_unreg_chrdev;
+	}
 
 	error = pci_register_driver(&pmcraid_driver);
 
diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c
index 643014f82f7d..1bf8061ff803 100644
--- a/drivers/scsi/qla2xxx/qla_bsg.c
+++ b/drivers/scsi/qla2xxx/qla_bsg.c
@@ -1,4 +1,4 @@
-/*
+	/*
  * QLogic Fibre Channel HBA Driver
  * Copyright (c)  2003-2014 QLogic Corporation
  *
@@ -9,6 +9,7 @@
 #include <linux/kthread.h>
 #include <linux/vmalloc.h>
 #include <linux/delay.h>
+#include <linux/bsg-lib.h>
 
 /* BSG support for ELS/CT pass through */
 void
@@ -16,10 +17,12 @@ qla2x00_bsg_job_done(void *data, void *ptr, int res)
 {
 	srb_t *sp = (srb_t *)ptr;
 	struct scsi_qla_host *vha = (scsi_qla_host_t *)data;
-	struct fc_bsg_job *bsg_job = sp->u.bsg_job;
+	struct bsg_job *bsg_job = sp->u.bsg_job;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
 
-	bsg_job->reply->result = res;
-	bsg_job->job_done(bsg_job);
+	bsg_reply->result = res;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	sp->free(vha, sp);
 }
 
@@ -28,13 +31,15 @@ qla2x00_bsg_sp_free(void *data, void *ptr)
 {
 	srb_t *sp = (srb_t *)ptr;
 	struct scsi_qla_host *vha = sp->fcport->vha;
-	struct fc_bsg_job *bsg_job = sp->u.bsg_job;
+	struct bsg_job *bsg_job = sp->u.bsg_job;
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+
 	struct qla_hw_data *ha = vha->hw;
 	struct qla_mt_iocb_rqst_fx00 *piocb_rqst;
 
 	if (sp->type == SRB_FXIOCB_BCMD) {
 		piocb_rqst = (struct qla_mt_iocb_rqst_fx00 *)
-		    &bsg_job->request->rqst_data.h_vendor.vendor_cmd[1];
+		    &bsg_request->rqst_data.h_vendor.vendor_cmd[1];
 
 		if (piocb_rqst->flags & SRB_FXDISC_REQ_DMA_VALID)
 			dma_unmap_sg(&ha->pdev->dev,
@@ -116,9 +121,11 @@ qla24xx_fcp_prio_cfg_valid(scsi_qla_host_t *vha,
 }
 
 static int
-qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job)
+qla24xx_proc_fcp_prio_cfg_cmd(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	int ret = 0;
@@ -131,7 +138,7 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job)
 	}
 
 	/* Get the sub command */
-	oper = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1];
+	oper = bsg_request->rqst_data.h_vendor.vendor_cmd[1];
 
 	/* Only set config is allowed if config memory is not allocated */
 	if (!ha->fcp_prio_cfg && (oper != QLFC_FCP_PRIO_SET_CONFIG)) {
@@ -145,10 +152,10 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job)
 			ha->fcp_prio_cfg->attributes &=
 				~FCP_PRIO_ATTR_ENABLE;
 			qla24xx_update_all_fcp_prio(vha);
-			bsg_job->reply->result = DID_OK;
+			bsg_reply->result = DID_OK;
 		} else {
 			ret = -EINVAL;
-			bsg_job->reply->result = (DID_ERROR << 16);
+			bsg_reply->result = (DID_ERROR << 16);
 			goto exit_fcp_prio_cfg;
 		}
 		break;
@@ -160,10 +167,10 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job)
 				ha->fcp_prio_cfg->attributes |=
 				    FCP_PRIO_ATTR_ENABLE;
 				qla24xx_update_all_fcp_prio(vha);
-				bsg_job->reply->result = DID_OK;
+				bsg_reply->result = DID_OK;
 			} else {
 				ret = -EINVAL;
-				bsg_job->reply->result = (DID_ERROR << 16);
+				bsg_reply->result = (DID_ERROR << 16);
 				goto exit_fcp_prio_cfg;
 			}
 		}
@@ -173,12 +180,12 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job)
 		len = bsg_job->reply_payload.payload_len;
 		if (!len || len > FCP_PRIO_CFG_SIZE) {
 			ret = -EINVAL;
-			bsg_job->reply->result = (DID_ERROR << 16);
+			bsg_reply->result = (DID_ERROR << 16);
 			goto exit_fcp_prio_cfg;
 		}
 
-		bsg_job->reply->result = DID_OK;
-		bsg_job->reply->reply_payload_rcv_len =
+		bsg_reply->result = DID_OK;
+		bsg_reply->reply_payload_rcv_len =
 			sg_copy_from_buffer(
 			bsg_job->reply_payload.sg_list,
 			bsg_job->reply_payload.sg_cnt, ha->fcp_prio_cfg,
@@ -189,7 +196,7 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job)
 	case QLFC_FCP_PRIO_SET_CONFIG:
 		len = bsg_job->request_payload.payload_len;
 		if (!len || len > FCP_PRIO_CFG_SIZE) {
-			bsg_job->reply->result = (DID_ERROR << 16);
+			bsg_reply->result = (DID_ERROR << 16);
 			ret = -EINVAL;
 			goto exit_fcp_prio_cfg;
 		}
@@ -200,7 +207,7 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job)
 				ql_log(ql_log_warn, vha, 0x7050,
 				    "Unable to allocate memory for fcp prio "
 				    "config data (%x).\n", FCP_PRIO_CFG_SIZE);
-				bsg_job->reply->result = (DID_ERROR << 16);
+				bsg_reply->result = (DID_ERROR << 16);
 				ret = -ENOMEM;
 				goto exit_fcp_prio_cfg;
 			}
@@ -215,7 +222,7 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job)
 
 		if (!qla24xx_fcp_prio_cfg_valid(vha,
 		    (struct qla_fcp_prio_cfg *) ha->fcp_prio_cfg, 1)) {
-			bsg_job->reply->result = (DID_ERROR << 16);
+			bsg_reply->result = (DID_ERROR << 16);
 			ret = -EINVAL;
 			/* If buffer was invalidatic int
 			 * fcp_prio_cfg is of no use
@@ -229,7 +236,7 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job)
 		if (ha->fcp_prio_cfg->attributes & FCP_PRIO_ATTR_ENABLE)
 			ha->flags.fcp_prio_enabled = 1;
 		qla24xx_update_all_fcp_prio(vha);
-		bsg_job->reply->result = DID_OK;
+		bsg_reply->result = DID_OK;
 		break;
 	default:
 		ret = -EINVAL;
@@ -237,13 +244,15 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct fc_bsg_job *bsg_job)
 	}
 exit_fcp_prio_cfg:
 	if (!ret)
-		bsg_job->job_done(bsg_job);
+		bsg_job_done(bsg_job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	return ret;
 }
 
 static int
-qla2x00_process_els(struct fc_bsg_job *bsg_job)
+qla2x00_process_els(struct bsg_job *bsg_job)
 {
+	struct fc_bsg_request *bsg_request = bsg_job->request;
 	struct fc_rport *rport;
 	fc_port_t *fcport = NULL;
 	struct Scsi_Host *host;
@@ -255,15 +264,15 @@ qla2x00_process_els(struct fc_bsg_job *bsg_job)
 	int rval =  (DRIVER_ERROR << 16);
 	uint16_t nextlid = 0;
 
-	if (bsg_job->request->msgcode == FC_BSG_RPT_ELS) {
-		rport = bsg_job->rport;
+	if (bsg_request->msgcode == FC_BSG_RPT_ELS) {
+		rport = fc_bsg_to_rport(bsg_job);
 		fcport = *(fc_port_t **) rport->dd_data;
 		host = rport_to_shost(rport);
 		vha = shost_priv(host);
 		ha = vha->hw;
 		type = "FC_BSG_RPT_ELS";
 	} else {
-		host = bsg_job->shost;
+		host = fc_bsg_to_shost(bsg_job);
 		vha = shost_priv(host);
 		ha = vha->hw;
 		type = "FC_BSG_HST_ELS_NOLOGIN";
@@ -296,7 +305,7 @@ qla2x00_process_els(struct fc_bsg_job *bsg_job)
 	}
 
 	/* ELS request for rport */
-	if (bsg_job->request->msgcode == FC_BSG_RPT_ELS) {
+	if (bsg_request->msgcode == FC_BSG_RPT_ELS) {
 		/* make sure the rport is logged in,
 		 * if not perform fabric login
 		 */
@@ -322,11 +331,11 @@ qla2x00_process_els(struct fc_bsg_job *bsg_job)
 		/* Initialize all required  fields of fcport */
 		fcport->vha = vha;
 		fcport->d_id.b.al_pa =
-			bsg_job->request->rqst_data.h_els.port_id[0];
+			bsg_request->rqst_data.h_els.port_id[0];
 		fcport->d_id.b.area =
-			bsg_job->request->rqst_data.h_els.port_id[1];
+			bsg_request->rqst_data.h_els.port_id[1];
 		fcport->d_id.b.domain =
-			bsg_job->request->rqst_data.h_els.port_id[2];
+			bsg_request->rqst_data.h_els.port_id[2];
 		fcport->loop_id =
 			(fcport->d_id.b.al_pa == 0xFD) ?
 			NPH_FABRIC_CONTROLLER : NPH_F_PORT;
@@ -366,11 +375,11 @@ qla2x00_process_els(struct fc_bsg_job *bsg_job)
 	}
 
 	sp->type =
-		(bsg_job->request->msgcode == FC_BSG_RPT_ELS ?
-		SRB_ELS_CMD_RPT : SRB_ELS_CMD_HST);
+		(bsg_request->msgcode == FC_BSG_RPT_ELS ?
+		 SRB_ELS_CMD_RPT : SRB_ELS_CMD_HST);
 	sp->name =
-		(bsg_job->request->msgcode == FC_BSG_RPT_ELS ?
-		"bsg_els_rpt" : "bsg_els_hst");
+		(bsg_request->msgcode == FC_BSG_RPT_ELS ?
+		 "bsg_els_rpt" : "bsg_els_hst");
 	sp->u.bsg_job = bsg_job;
 	sp->free = qla2x00_bsg_sp_free;
 	sp->done = qla2x00_bsg_job_done;
@@ -378,7 +387,7 @@ qla2x00_process_els(struct fc_bsg_job *bsg_job)
 	ql_dbg(ql_dbg_user, vha, 0x700a,
 	    "bsg rqst type: %s els type: %x - loop-id=%x "
 	    "portid=%-2x%02x%02x.\n", type,
-	    bsg_job->request->rqst_data.h_els.command_code, fcport->loop_id,
+	    bsg_request->rqst_data.h_els.command_code, fcport->loop_id,
 	    fcport->d_id.b.domain, fcport->d_id.b.area, fcport->d_id.b.al_pa);
 
 	rval = qla2x00_start_sp(sp);
@@ -399,7 +408,7 @@ done_unmap_sg:
 	goto done_free_fcport;
 
 done_free_fcport:
-	if (bsg_job->request->msgcode == FC_BSG_RPT_ELS)
+	if (bsg_request->msgcode == FC_BSG_RPT_ELS)
 		kfree(fcport);
 done:
 	return rval;
@@ -420,10 +429,11 @@ qla24xx_calc_ct_iocbs(uint16_t dsds)
 }
 
 static int
-qla2x00_process_ct(struct fc_bsg_job *bsg_job)
+qla2x00_process_ct(struct bsg_job *bsg_job)
 {
 	srb_t *sp;
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	int rval = (DRIVER_ERROR << 16);
@@ -469,7 +479,7 @@ qla2x00_process_ct(struct fc_bsg_job *bsg_job)
 	}
 
 	loop_id =
-		(bsg_job->request->rqst_data.h_ct.preamble_word1 & 0xFF000000)
+		(bsg_request->rqst_data.h_ct.preamble_word1 & 0xFF000000)
 			>> 24;
 	switch (loop_id) {
 	case 0xFC:
@@ -500,9 +510,9 @@ qla2x00_process_ct(struct fc_bsg_job *bsg_job)
 
 	/* Initialize all required  fields of fcport */
 	fcport->vha = vha;
-	fcport->d_id.b.al_pa = bsg_job->request->rqst_data.h_ct.port_id[0];
-	fcport->d_id.b.area = bsg_job->request->rqst_data.h_ct.port_id[1];
-	fcport->d_id.b.domain = bsg_job->request->rqst_data.h_ct.port_id[2];
+	fcport->d_id.b.al_pa = bsg_request->rqst_data.h_ct.port_id[0];
+	fcport->d_id.b.area = bsg_request->rqst_data.h_ct.port_id[1];
+	fcport->d_id.b.domain = bsg_request->rqst_data.h_ct.port_id[2];
 	fcport->loop_id = loop_id;
 
 	/* Alloc SRB structure */
@@ -524,7 +534,7 @@ qla2x00_process_ct(struct fc_bsg_job *bsg_job)
 	ql_dbg(ql_dbg_user, vha, 0x7016,
 	    "bsg rqst type: %s else type: %x - "
 	    "loop-id=%x portid=%02x%02x%02x.\n", type,
-	    (bsg_job->request->rqst_data.h_ct.preamble_word2 >> 16),
+	    (bsg_request->rqst_data.h_ct.preamble_word2 >> 16),
 	    fcport->loop_id, fcport->d_id.b.domain, fcport->d_id.b.area,
 	    fcport->d_id.b.al_pa);
 
@@ -697,9 +707,11 @@ done_set_internal:
 }
 
 static int
-qla2x00_process_loopback(struct fc_bsg_job *bsg_job)
+qla2x00_process_loopback(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	int rval;
@@ -780,9 +792,9 @@ qla2x00_process_loopback(struct fc_bsg_job *bsg_job)
 	elreq.rcv_dma = rsp_data_dma;
 	elreq.transfer_size = req_data_len;
 
-	elreq.options = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1];
+	elreq.options = bsg_request->rqst_data.h_vendor.vendor_cmd[1];
 	elreq.iteration_count =
-	    bsg_job->request->rqst_data.h_vendor.vendor_cmd[2];
+	    bsg_request->rqst_data.h_vendor.vendor_cmd[2];
 
 	if (atomic_read(&vha->loop_state) == LOOP_READY &&
 	    (ha->current_topology == ISP_CFG_F ||
@@ -896,12 +908,12 @@ qla2x00_process_loopback(struct fc_bsg_job *bsg_job)
 		    "Vendor request %s failed.\n", type);
 
 		rval = 0;
-		bsg_job->reply->result = (DID_ERROR << 16);
-		bsg_job->reply->reply_payload_rcv_len = 0;
+		bsg_reply->result = (DID_ERROR << 16);
+		bsg_reply->reply_payload_rcv_len = 0;
 	} else {
 		ql_dbg(ql_dbg_user, vha, 0x702d,
 		    "Vendor request %s completed.\n", type);
-		bsg_job->reply->result = (DID_OK << 16);
+		bsg_reply->result = (DID_OK << 16);
 		sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
 			bsg_job->reply_payload.sg_cnt, rsp_data,
 			rsp_data_len);
@@ -930,14 +942,17 @@ done_unmap_req_sg:
 	    bsg_job->request_payload.sg_list,
 	    bsg_job->request_payload.sg_cnt, DMA_TO_DEVICE);
 	if (!rval)
-		bsg_job->job_done(bsg_job);
+		bsg_job_done(bsg_job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	return rval;
 }
 
 static int
-qla84xx_reset(struct fc_bsg_job *bsg_job)
+qla84xx_reset(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	int rval = 0;
@@ -948,7 +963,7 @@ qla84xx_reset(struct fc_bsg_job *bsg_job)
 		return -EINVAL;
 	}
 
-	flag = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1];
+	flag = bsg_request->rqst_data.h_vendor.vendor_cmd[1];
 
 	rval = qla84xx_reset_chip(vha, flag == A84_ISSUE_RESET_DIAG_FW);
 
@@ -960,17 +975,20 @@ qla84xx_reset(struct fc_bsg_job *bsg_job)
 	} else {
 		ql_dbg(ql_dbg_user, vha, 0x7031,
 		    "Vendor request 84xx reset completed.\n");
-		bsg_job->reply->result = DID_OK;
-		bsg_job->job_done(bsg_job);
+		bsg_reply->result = DID_OK;
+		bsg_job_done(bsg_job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	}
 
 	return rval;
 }
 
 static int
-qla84xx_updatefw(struct fc_bsg_job *bsg_job)
+qla84xx_updatefw(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	struct verify_chip_entry_84xx *mn = NULL;
@@ -1027,7 +1045,7 @@ qla84xx_updatefw(struct fc_bsg_job *bsg_job)
 		goto done_free_fw_buf;
 	}
 
-	flag = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1];
+	flag = bsg_request->rqst_data.h_vendor.vendor_cmd[1];
 	fw_ver = le32_to_cpu(*((uint32_t *)((uint32_t *)fw_buf + 2)));
 
 	memset(mn, 0, sizeof(struct access_chip_84xx));
@@ -1059,7 +1077,7 @@ qla84xx_updatefw(struct fc_bsg_job *bsg_job)
 		    "Vendor request 84xx updatefw completed.\n");
 
 		bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-		bsg_job->reply->result = DID_OK;
+		bsg_reply->result = DID_OK;
 	}
 
 	dma_pool_free(ha->s_dma_pool, mn, mn_dma);
@@ -1072,14 +1090,17 @@ done_unmap_sg:
 		bsg_job->request_payload.sg_cnt, DMA_TO_DEVICE);
 
 	if (!rval)
-		bsg_job->job_done(bsg_job);
+		bsg_job_done(bsg_job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	return rval;
 }
 
 static int
-qla84xx_mgmt_cmd(struct fc_bsg_job *bsg_job)
+qla84xx_mgmt_cmd(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	struct access_chip_84xx *mn = NULL;
@@ -1107,7 +1128,7 @@ qla84xx_mgmt_cmd(struct fc_bsg_job *bsg_job)
 	memset(mn, 0, sizeof(struct access_chip_84xx));
 	mn->entry_type = ACCESS_CHIP_IOCB_TYPE;
 	mn->entry_count = 1;
-	ql84_mgmt = (void *)bsg_job->request + sizeof(struct fc_bsg_request);
+	ql84_mgmt = (void *)bsg_request + sizeof(struct fc_bsg_request);
 	switch (ql84_mgmt->mgmt.cmd) {
 	case QLA84_MGMT_READ_MEM:
 	case QLA84_MGMT_GET_INFO:
@@ -1239,11 +1260,11 @@ qla84xx_mgmt_cmd(struct fc_bsg_job *bsg_job)
 		    "Vendor request 84xx mgmt completed.\n");
 
 		bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-		bsg_job->reply->result = DID_OK;
+		bsg_reply->result = DID_OK;
 
 		if ((ql84_mgmt->mgmt.cmd == QLA84_MGMT_READ_MEM) ||
 			(ql84_mgmt->mgmt.cmd == QLA84_MGMT_GET_INFO)) {
-			bsg_job->reply->reply_payload_rcv_len =
+			bsg_reply->reply_payload_rcv_len =
 				bsg_job->reply_payload.payload_len;
 
 			sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
@@ -1267,14 +1288,17 @@ exit_mgmt:
 	dma_pool_free(ha->s_dma_pool, mn, mn_dma);
 
 	if (!rval)
-		bsg_job->job_done(bsg_job);
+		bsg_job_done(bsg_job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	return rval;
 }
 
 static int
-qla24xx_iidma(struct fc_bsg_job *bsg_job)
+qla24xx_iidma(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	int rval = 0;
 	struct qla_port_param *port_param = NULL;
@@ -1288,7 +1312,7 @@ qla24xx_iidma(struct fc_bsg_job *bsg_job)
 		return -EINVAL;
 	}
 
-	port_param = (void *)bsg_job->request + sizeof(struct fc_bsg_request);
+	port_param = (void *)bsg_request + sizeof(struct fc_bsg_request);
 	if (port_param->fc_scsi_addr.dest_type != EXT_DEF_TYPE_WWPN) {
 		ql_log(ql_log_warn, vha, 0x7048,
 		    "Invalid destination type.\n");
@@ -1343,24 +1367,26 @@ qla24xx_iidma(struct fc_bsg_job *bsg_job)
 			bsg_job->reply_len = sizeof(struct fc_bsg_reply) +
 				sizeof(struct qla_port_param);
 
-			rsp_ptr = ((uint8_t *)bsg_job->reply) +
+			rsp_ptr = ((uint8_t *)bsg_reply) +
 				sizeof(struct fc_bsg_reply);
 
 			memcpy(rsp_ptr, port_param,
 				sizeof(struct qla_port_param));
 		}
 
-		bsg_job->reply->result = DID_OK;
-		bsg_job->job_done(bsg_job);
+		bsg_reply->result = DID_OK;
+		bsg_job_done(bsg_job, bsg_reply->result,
+			       bsg_reply->reply_payload_rcv_len);
 	}
 
 	return rval;
 }
 
 static int
-qla2x00_optrom_setup(struct fc_bsg_job *bsg_job, scsi_qla_host_t *vha,
+qla2x00_optrom_setup(struct bsg_job *bsg_job, scsi_qla_host_t *vha,
 	uint8_t is_update)
 {
+	struct fc_bsg_request *bsg_request = bsg_job->request;
 	uint32_t start = 0;
 	int valid = 0;
 	struct qla_hw_data *ha = vha->hw;
@@ -1368,7 +1394,7 @@ qla2x00_optrom_setup(struct fc_bsg_job *bsg_job, scsi_qla_host_t *vha,
 	if (unlikely(pci_channel_offline(ha->pdev)))
 		return -EINVAL;
 
-	start = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1];
+	start = bsg_request->rqst_data.h_vendor.vendor_cmd[1];
 	if (start > ha->optrom_size) {
 		ql_log(ql_log_warn, vha, 0x7055,
 		    "start %d > optrom_size %d.\n", start, ha->optrom_size);
@@ -1427,9 +1453,10 @@ qla2x00_optrom_setup(struct fc_bsg_job *bsg_job, scsi_qla_host_t *vha,
 }
 
 static int
-qla2x00_read_optrom(struct fc_bsg_job *bsg_job)
+qla2x00_read_optrom(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	int rval = 0;
@@ -1451,20 +1478,22 @@ qla2x00_read_optrom(struct fc_bsg_job *bsg_job)
 	    bsg_job->reply_payload.sg_cnt, ha->optrom_buffer,
 	    ha->optrom_region_size);
 
-	bsg_job->reply->reply_payload_rcv_len = ha->optrom_region_size;
-	bsg_job->reply->result = DID_OK;
+	bsg_reply->reply_payload_rcv_len = ha->optrom_region_size;
+	bsg_reply->result = DID_OK;
 	vfree(ha->optrom_buffer);
 	ha->optrom_buffer = NULL;
 	ha->optrom_state = QLA_SWAITING;
 	mutex_unlock(&ha->optrom_mutex);
-	bsg_job->job_done(bsg_job);
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	return rval;
 }
 
 static int
-qla2x00_update_optrom(struct fc_bsg_job *bsg_job)
+qla2x00_update_optrom(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	int rval = 0;
@@ -1486,19 +1515,21 @@ qla2x00_update_optrom(struct fc_bsg_job *bsg_job)
 	ha->isp_ops->write_optrom(vha, ha->optrom_buffer,
 	    ha->optrom_region_start, ha->optrom_region_size);
 
-	bsg_job->reply->result = DID_OK;
+	bsg_reply->result = DID_OK;
 	vfree(ha->optrom_buffer);
 	ha->optrom_buffer = NULL;
 	ha->optrom_state = QLA_SWAITING;
 	mutex_unlock(&ha->optrom_mutex);
-	bsg_job->job_done(bsg_job);
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	return rval;
 }
 
 static int
-qla2x00_update_fru_versions(struct fc_bsg_job *bsg_job)
+qla2x00_update_fru_versions(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	int rval = 0;
@@ -1509,7 +1540,7 @@ qla2x00_update_fru_versions(struct fc_bsg_job *bsg_job)
 	dma_addr_t sfp_dma;
 	void *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma);
 	if (!sfp) {
-		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_NO_MEMORY;
 		goto done;
 	}
@@ -1525,30 +1556,32 @@ qla2x00_update_fru_versions(struct fc_bsg_job *bsg_job)
 		    image->field_address.device, image->field_address.offset,
 		    sizeof(image->field_info), image->field_address.option);
 		if (rval) {
-			bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+			bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 			    EXT_STATUS_MAILBOX;
 			goto dealloc;
 		}
 		image++;
 	}
 
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = 0;
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = 0;
 
 dealloc:
 	dma_pool_free(ha->s_dma_pool, sfp, sfp_dma);
 
 done:
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-	bsg_job->reply->result = DID_OK << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 
 	return 0;
 }
 
 static int
-qla2x00_read_fru_status(struct fc_bsg_job *bsg_job)
+qla2x00_read_fru_status(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	int rval = 0;
@@ -1557,7 +1590,7 @@ qla2x00_read_fru_status(struct fc_bsg_job *bsg_job)
 	dma_addr_t sfp_dma;
 	uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma);
 	if (!sfp) {
-		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_NO_MEMORY;
 		goto done;
 	}
@@ -1571,7 +1604,7 @@ qla2x00_read_fru_status(struct fc_bsg_job *bsg_job)
 	sr->status_reg = *sfp;
 
 	if (rval) {
-		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_MAILBOX;
 		goto dealloc;
 	}
@@ -1579,24 +1612,26 @@ qla2x00_read_fru_status(struct fc_bsg_job *bsg_job)
 	sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
 	    bsg_job->reply_payload.sg_cnt, sr, sizeof(*sr));
 
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = 0;
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = 0;
 
 dealloc:
 	dma_pool_free(ha->s_dma_pool, sfp, sfp_dma);
 
 done:
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-	bsg_job->reply->reply_payload_rcv_len = sizeof(*sr);
-	bsg_job->reply->result = DID_OK << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_reply->reply_payload_rcv_len = sizeof(*sr);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 
 	return 0;
 }
 
 static int
-qla2x00_write_fru_status(struct fc_bsg_job *bsg_job)
+qla2x00_write_fru_status(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	int rval = 0;
@@ -1605,7 +1640,7 @@ qla2x00_write_fru_status(struct fc_bsg_job *bsg_job)
 	dma_addr_t sfp_dma;
 	uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma);
 	if (!sfp) {
-		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_NO_MEMORY;
 		goto done;
 	}
@@ -1619,28 +1654,30 @@ qla2x00_write_fru_status(struct fc_bsg_job *bsg_job)
 	    sizeof(sr->status_reg), sr->field_address.option);
 
 	if (rval) {
-		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_MAILBOX;
 		goto dealloc;
 	}
 
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = 0;
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = 0;
 
 dealloc:
 	dma_pool_free(ha->s_dma_pool, sfp, sfp_dma);
 
 done:
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-	bsg_job->reply->result = DID_OK << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 
 	return 0;
 }
 
 static int
-qla2x00_write_i2c(struct fc_bsg_job *bsg_job)
+qla2x00_write_i2c(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	int rval = 0;
@@ -1649,7 +1686,7 @@ qla2x00_write_i2c(struct fc_bsg_job *bsg_job)
 	dma_addr_t sfp_dma;
 	uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma);
 	if (!sfp) {
-		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_NO_MEMORY;
 		goto done;
 	}
@@ -1662,28 +1699,30 @@ qla2x00_write_i2c(struct fc_bsg_job *bsg_job)
 	    i2c->device, i2c->offset, i2c->length, i2c->option);
 
 	if (rval) {
-		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_MAILBOX;
 		goto dealloc;
 	}
 
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = 0;
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = 0;
 
 dealloc:
 	dma_pool_free(ha->s_dma_pool, sfp, sfp_dma);
 
 done:
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-	bsg_job->reply->result = DID_OK << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 
 	return 0;
 }
 
 static int
-qla2x00_read_i2c(struct fc_bsg_job *bsg_job)
+qla2x00_read_i2c(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	int rval = 0;
@@ -1692,7 +1731,7 @@ qla2x00_read_i2c(struct fc_bsg_job *bsg_job)
 	dma_addr_t sfp_dma;
 	uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma);
 	if (!sfp) {
-		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_NO_MEMORY;
 		goto done;
 	}
@@ -1704,7 +1743,7 @@ qla2x00_read_i2c(struct fc_bsg_job *bsg_job)
 		i2c->device, i2c->offset, i2c->length, i2c->option);
 
 	if (rval) {
-		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_MAILBOX;
 		goto dealloc;
 	}
@@ -1713,24 +1752,26 @@ qla2x00_read_i2c(struct fc_bsg_job *bsg_job)
 	sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
 	    bsg_job->reply_payload.sg_cnt, i2c, sizeof(*i2c));
 
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = 0;
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = 0;
 
 dealloc:
 	dma_pool_free(ha->s_dma_pool, sfp, sfp_dma);
 
 done:
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-	bsg_job->reply->reply_payload_rcv_len = sizeof(*i2c);
-	bsg_job->reply->result = DID_OK << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_reply->reply_payload_rcv_len = sizeof(*i2c);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 
 	return 0;
 }
 
 static int
-qla24xx_process_bidir_cmd(struct fc_bsg_job *bsg_job)
+qla24xx_process_bidir_cmd(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	uint32_t rval = EXT_STATUS_OK;
@@ -1895,19 +1936,21 @@ done:
 	/* Return an error vendor specific response
 	 * and complete the bsg request
 	 */
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = rval;
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = rval;
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-	bsg_job->reply->reply_payload_rcv_len = 0;
-	bsg_job->reply->result = (DID_OK) << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_reply->reply_payload_rcv_len = 0;
+	bsg_reply->result = (DID_OK) << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	/* Always return success, vendor rsp carries correct status */
 	return 0;
 }
 
 static int
-qlafx00_mgmt_cmd(struct fc_bsg_job *bsg_job)
+qlafx00_mgmt_cmd(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	int rval = (DRIVER_ERROR << 16);
@@ -1919,7 +1962,7 @@ qlafx00_mgmt_cmd(struct fc_bsg_job *bsg_job)
 
 	/* Copy the IOCB specific information */
 	piocb_rqst = (struct qla_mt_iocb_rqst_fx00 *)
-	    &bsg_job->request->rqst_data.h_vendor.vendor_cmd[1];
+	    &bsg_request->rqst_data.h_vendor.vendor_cmd[1];
 
 	/* Dump the vendor information */
 	ql_dump_buffer(ql_dbg_user + ql_dbg_verbose , vha, 0x70cf,
@@ -2027,9 +2070,10 @@ done:
 }
 
 static int
-qla26xx_serdes_op(struct fc_bsg_job *bsg_job)
+qla26xx_serdes_op(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	int rval = 0;
 	struct qla_serdes_reg sr;
@@ -2042,13 +2086,13 @@ qla26xx_serdes_op(struct fc_bsg_job *bsg_job)
 	switch (sr.cmd) {
 	case INT_SC_SERDES_WRITE_REG:
 		rval = qla2x00_write_serdes_word(vha, sr.addr, sr.val);
-		bsg_job->reply->reply_payload_rcv_len = 0;
+		bsg_reply->reply_payload_rcv_len = 0;
 		break;
 	case INT_SC_SERDES_READ_REG:
 		rval = qla2x00_read_serdes_word(vha, sr.addr, &sr.val);
 		sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
 		    bsg_job->reply_payload.sg_cnt, &sr, sizeof(sr));
-		bsg_job->reply->reply_payload_rcv_len = sizeof(sr);
+		bsg_reply->reply_payload_rcv_len = sizeof(sr);
 		break;
 	default:
 		ql_dbg(ql_dbg_user, vha, 0x708c,
@@ -2057,19 +2101,21 @@ qla26xx_serdes_op(struct fc_bsg_job *bsg_job)
 		break;
 	}
 
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 	    rval ? EXT_STATUS_MAILBOX : 0;
 
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-	bsg_job->reply->result = DID_OK << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	return 0;
 }
 
 static int
-qla8044_serdes_op(struct fc_bsg_job *bsg_job)
+qla8044_serdes_op(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	int rval = 0;
 	struct qla_serdes_reg_ex sr;
@@ -2082,13 +2128,13 @@ qla8044_serdes_op(struct fc_bsg_job *bsg_job)
 	switch (sr.cmd) {
 	case INT_SC_SERDES_WRITE_REG:
 		rval = qla8044_write_serdes_word(vha, sr.addr, sr.val);
-		bsg_job->reply->reply_payload_rcv_len = 0;
+		bsg_reply->reply_payload_rcv_len = 0;
 		break;
 	case INT_SC_SERDES_READ_REG:
 		rval = qla8044_read_serdes_word(vha, sr.addr, &sr.val);
 		sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
 		    bsg_job->reply_payload.sg_cnt, &sr, sizeof(sr));
-		bsg_job->reply->reply_payload_rcv_len = sizeof(sr);
+		bsg_reply->reply_payload_rcv_len = sizeof(sr);
 		break;
 	default:
 		ql_dbg(ql_dbg_user, vha, 0x70cf,
@@ -2097,19 +2143,21 @@ qla8044_serdes_op(struct fc_bsg_job *bsg_job)
 		break;
 	}
 
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 	    rval ? EXT_STATUS_MAILBOX : 0;
 
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-	bsg_job->reply->result = DID_OK << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	return 0;
 }
 
 static int
-qla27xx_get_flash_upd_cap(struct fc_bsg_job *bsg_job)
+qla27xx_get_flash_upd_cap(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	struct qla_flash_update_caps cap;
@@ -2125,21 +2173,23 @@ qla27xx_get_flash_upd_cap(struct fc_bsg_job *bsg_job)
 
 	sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
 	    bsg_job->reply_payload.sg_cnt, &cap, sizeof(cap));
-	bsg_job->reply->reply_payload_rcv_len = sizeof(cap);
+	bsg_reply->reply_payload_rcv_len = sizeof(cap);
 
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 	    EXT_STATUS_OK;
 
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-	bsg_job->reply->result = DID_OK << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	return 0;
 }
 
 static int
-qla27xx_set_flash_upd_cap(struct fc_bsg_job *bsg_job)
+qla27xx_set_flash_upd_cap(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	uint64_t online_fw_attr = 0;
@@ -2158,32 +2208,34 @@ qla27xx_set_flash_upd_cap(struct fc_bsg_job *bsg_job)
 			 (uint64_t)ha->fw_attributes;
 
 	if (online_fw_attr != cap.capabilities) {
-		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_INVALID_PARAM;
 		return -EINVAL;
 	}
 
 	if (cap.outage_duration < MAX_LOOP_TIMEOUT)  {
-		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_INVALID_PARAM;
 		return -EINVAL;
 	}
 
-	bsg_job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 	    EXT_STATUS_OK;
 
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-	bsg_job->reply->result = DID_OK << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	return 0;
 }
 
 static int
-qla27xx_get_bbcr_data(struct fc_bsg_job *bsg_job)
+qla27xx_get_bbcr_data(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	struct qla_bbcr_data bbcr;
@@ -2227,27 +2279,30 @@ qla27xx_get_bbcr_data(struct fc_bsg_job *bsg_job)
 done:
 	sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
 		bsg_job->reply_payload.sg_cnt, &bbcr, sizeof(bbcr));
-	bsg_job->reply->reply_payload_rcv_len = sizeof(bbcr);
+	bsg_reply->reply_payload_rcv_len = sizeof(bbcr);
 
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_OK;
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_OK;
 
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
-	bsg_job->reply->result = DID_OK << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 	return 0;
 }
 
 static int
-qla2x00_get_priv_stats(struct fc_bsg_job *bsg_job)
+qla2x00_get_priv_stats(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	struct qla_hw_data *ha = vha->hw;
 	struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev);
 	struct link_statistics *stats = NULL;
 	dma_addr_t stats_dma;
 	int rval;
-	uint32_t *cmd = bsg_job->request->rqst_data.h_vendor.vendor_cmd;
+	uint32_t *cmd = bsg_request->rqst_data.h_vendor.vendor_cmd;
 	uint options = cmd[0] == QL_VND_GET_PRIV_STATS_EX ? cmd[1] : 0;
 
 	if (test_bit(UNLOADING, &vha->dpc_flags))
@@ -2281,13 +2336,14 @@ qla2x00_get_priv_stats(struct fc_bsg_job *bsg_job)
 			bsg_job->reply_payload.sg_cnt, stats, sizeof(*stats));
 	}
 
-	bsg_job->reply->reply_payload_rcv_len = sizeof(*stats);
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+	bsg_reply->reply_payload_rcv_len = sizeof(*stats);
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 	    rval ? EXT_STATUS_MAILBOX : EXT_STATUS_OK;
 
-	bsg_job->reply_len = sizeof(*bsg_job->reply);
-	bsg_job->reply->result = DID_OK << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_job->reply_len = sizeof(*bsg_reply);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 
 	dma_free_coherent(&ha->pdev->dev, sizeof(*stats),
 		stats, stats_dma);
@@ -2296,9 +2352,10 @@ qla2x00_get_priv_stats(struct fc_bsg_job *bsg_job)
 }
 
 static int
-qla2x00_do_dport_diagnostics(struct fc_bsg_job *bsg_job)
+qla2x00_do_dport_diagnostics(struct bsg_job *bsg_job)
 {
-	struct Scsi_Host *host = bsg_job->shost;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct Scsi_Host *host = fc_bsg_to_shost(bsg_job);
 	scsi_qla_host_t *vha = shost_priv(host);
 	int rval;
 	struct qla_dport_diag *dd;
@@ -2323,13 +2380,14 @@ qla2x00_do_dport_diagnostics(struct fc_bsg_job *bsg_job)
 		    bsg_job->reply_payload.sg_cnt, dd, sizeof(*dd));
 	}
 
-	bsg_job->reply->reply_payload_rcv_len = sizeof(*dd);
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+	bsg_reply->reply_payload_rcv_len = sizeof(*dd);
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 	    rval ? EXT_STATUS_MAILBOX : EXT_STATUS_OK;
 
-	bsg_job->reply_len = sizeof(*bsg_job->reply);
-	bsg_job->reply->result = DID_OK << 16;
-	bsg_job->job_done(bsg_job);
+	bsg_job->reply_len = sizeof(*bsg_reply);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job_done(bsg_job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
 
 	kfree(dd);
 
@@ -2337,9 +2395,11 @@ qla2x00_do_dport_diagnostics(struct fc_bsg_job *bsg_job)
 }
 
 static int
-qla2x00_process_vendor_specific(struct fc_bsg_job *bsg_job)
+qla2x00_process_vendor_specific(struct bsg_job *bsg_job)
 {
-	switch (bsg_job->request->rqst_data.h_vendor.vendor_cmd[0]) {
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+
+	switch (bsg_request->rqst_data.h_vendor.vendor_cmd[0]) {
 	case QL_VND_LOOPBACK:
 		return qla2x00_process_loopback(bsg_job);
 
@@ -2413,36 +2473,38 @@ qla2x00_process_vendor_specific(struct fc_bsg_job *bsg_job)
 }
 
 int
-qla24xx_bsg_request(struct fc_bsg_job *bsg_job)
+qla24xx_bsg_request(struct bsg_job *bsg_job)
 {
+	struct fc_bsg_request *bsg_request = bsg_job->request;
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
 	int ret = -EINVAL;
 	struct fc_rport *rport;
 	struct Scsi_Host *host;
 	scsi_qla_host_t *vha;
 
 	/* In case no data transferred. */
-	bsg_job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 
-	if (bsg_job->request->msgcode == FC_BSG_RPT_ELS) {
-		rport = bsg_job->rport;
+	if (bsg_request->msgcode == FC_BSG_RPT_ELS) {
+		rport = fc_bsg_to_rport(bsg_job);
 		host = rport_to_shost(rport);
 		vha = shost_priv(host);
 	} else {
-		host = bsg_job->shost;
+		host = fc_bsg_to_shost(bsg_job);
 		vha = shost_priv(host);
 	}
 
 	if (qla2x00_reset_active(vha)) {
 		ql_dbg(ql_dbg_user, vha, 0x709f,
 		    "BSG: ISP abort active/needed -- cmd=%d.\n",
-		    bsg_job->request->msgcode);
+		    bsg_request->msgcode);
 		return -EBUSY;
 	}
 
 	ql_dbg(ql_dbg_user, vha, 0x7000,
-	    "Entered %s msgcode=0x%x.\n", __func__, bsg_job->request->msgcode);
+	    "Entered %s msgcode=0x%x.\n", __func__, bsg_request->msgcode);
 
-	switch (bsg_job->request->msgcode) {
+	switch (bsg_request->msgcode) {
 	case FC_BSG_RPT_ELS:
 	case FC_BSG_HST_ELS_NOLOGIN:
 		ret = qla2x00_process_els(bsg_job);
@@ -2464,9 +2526,10 @@ qla24xx_bsg_request(struct fc_bsg_job *bsg_job)
 }
 
 int
-qla24xx_bsg_timeout(struct fc_bsg_job *bsg_job)
+qla24xx_bsg_timeout(struct bsg_job *bsg_job)
 {
-	scsi_qla_host_t *vha = shost_priv(bsg_job->shost);
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	scsi_qla_host_t *vha = shost_priv(fc_bsg_to_shost(bsg_job));
 	struct qla_hw_data *ha = vha->hw;
 	srb_t *sp;
 	int cnt, que;
@@ -2494,13 +2557,13 @@ qla24xx_bsg_timeout(struct fc_bsg_job *bsg_job)
 						    "mbx abort_command "
 						    "failed.\n");
 						bsg_job->req->errors =
-						bsg_job->reply->result = -EIO;
+						bsg_reply->result = -EIO;
 					} else {
 						ql_dbg(ql_dbg_user, vha, 0x708a,
 						    "mbx abort_command "
 						    "success.\n");
 						bsg_job->req->errors =
-						bsg_job->reply->result = 0;
+						bsg_reply->result = 0;
 					}
 					spin_lock_irqsave(&ha->hardware_lock, flags);
 					goto done;
@@ -2510,7 +2573,7 @@ qla24xx_bsg_timeout(struct fc_bsg_job *bsg_job)
 	}
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 	ql_log(ql_log_info, vha, 0x708b, "SRB not found to abort.\n");
-	bsg_job->req->errors = bsg_job->reply->result = -ENXIO;
+	bsg_job->req->errors = bsg_reply->result = -ENXIO;
 	return 0;
 
 done:
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index 73b12e41d992..5236e3f2a06a 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -403,7 +403,7 @@ typedef struct srb {
 	int iocbs;
 	union {
 		struct srb_iocb iocb_cmd;
-		struct fc_bsg_job *bsg_job;
+		struct bsg_job *bsg_job;
 		struct srb_cmd scmd;
 	} u;
 	void (*done)(void *, void *, int);
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index 6ca00813c71f..c51d9f3359e3 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -733,8 +733,8 @@ extern int qla82xx_read_temperature(scsi_qla_host_t *);
 extern int qla8044_read_temperature(scsi_qla_host_t *);
 
 /* BSG related functions */
-extern int qla24xx_bsg_request(struct fc_bsg_job *);
-extern int qla24xx_bsg_timeout(struct fc_bsg_job *);
+extern int qla24xx_bsg_request(struct bsg_job *);
+extern int qla24xx_bsg_timeout(struct bsg_job *);
 extern int qla84xx_reset_chip(scsi_qla_host_t *, uint16_t);
 extern int qla2x00_issue_iocb_timeout(scsi_qla_host_t *, void *,
 	dma_addr_t, size_t, uint32_t);
diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c
index b41265a75ed5..221ad8907893 100644
--- a/drivers/scsi/qla2xxx/qla_iocb.c
+++ b/drivers/scsi/qla2xxx/qla_iocb.c
@@ -2197,7 +2197,8 @@ qla24xx_els_logo_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
 static void
 qla24xx_els_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
 {
-	struct fc_bsg_job *bsg_job = sp->u.bsg_job;
+	struct bsg_job *bsg_job = sp->u.bsg_job;
+	struct fc_bsg_request *bsg_request = bsg_job->request;
 
         els_iocb->entry_type = ELS_IOCB_TYPE;
         els_iocb->entry_count = 1;
@@ -2212,8 +2213,8 @@ qla24xx_els_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
 
 	els_iocb->opcode =
 	    sp->type == SRB_ELS_CMD_RPT ?
-	    bsg_job->request->rqst_data.r_els.els_code :
-	    bsg_job->request->rqst_data.h_els.command_code;
+	    bsg_request->rqst_data.r_els.els_code :
+	    bsg_request->rqst_data.h_els.command_code;
         els_iocb->port_id[0] = sp->fcport->d_id.b.al_pa;
         els_iocb->port_id[1] = sp->fcport->d_id.b.area;
         els_iocb->port_id[2] = sp->fcport->d_id.b.domain;
@@ -2250,7 +2251,7 @@ qla2x00_ct_iocb(srb_t *sp, ms_iocb_entry_t *ct_iocb)
 	uint16_t tot_dsds;
 	scsi_qla_host_t *vha = sp->fcport->vha;
 	struct qla_hw_data *ha = vha->hw;
-	struct fc_bsg_job *bsg_job = sp->u.bsg_job;
+	struct bsg_job *bsg_job = sp->u.bsg_job;
 	int loop_iterartion = 0;
 	int entry_count = 1;
 
@@ -2327,7 +2328,7 @@ qla24xx_ct_iocb(srb_t *sp, struct ct_entry_24xx *ct_iocb)
 	uint16_t tot_dsds;
         scsi_qla_host_t *vha = sp->fcport->vha;
 	struct qla_hw_data *ha = vha->hw;
-	struct fc_bsg_job *bsg_job = sp->u.bsg_job;
+	struct bsg_job *bsg_job = sp->u.bsg_job;
 	int loop_iterartion = 0;
 	int entry_count = 1;
 
@@ -2833,7 +2834,7 @@ qla25xx_build_bidir_iocb(srb_t *sp, struct scsi_qla_host *vha,
 	struct scatterlist *sg;
 	int index;
 	int entry_count = 1;
-	struct fc_bsg_job *bsg_job = sp->u.bsg_job;
+	struct bsg_job *bsg_job = sp->u.bsg_job;
 
 	/*Update entry type to indicate bidir command */
 	*((uint32_t *)(&cmd_pkt->entry_type)) =
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index 068c4e47fac9..19f18485a854 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -1356,7 +1356,8 @@ qla2x00_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 	const char func[] = "CT_IOCB";
 	const char *type;
 	srb_t *sp;
-	struct fc_bsg_job *bsg_job;
+	struct bsg_job *bsg_job;
+	struct fc_bsg_reply *bsg_reply;
 	uint16_t comp_status;
 	int res;
 
@@ -1365,6 +1366,7 @@ qla2x00_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 		return;
 
 	bsg_job = sp->u.bsg_job;
+	bsg_reply = bsg_job->reply;
 
 	type = "ct pass-through";
 
@@ -1373,32 +1375,32 @@ qla2x00_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 	/* return FC_CTELS_STATUS_OK and leave the decoding of the ELS/CT
 	 * fc payload  to the caller
 	 */
-	bsg_job->reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK;
+	bsg_reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK;
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
 
 	if (comp_status != CS_COMPLETE) {
 		if (comp_status == CS_DATA_UNDERRUN) {
 			res = DID_OK << 16;
-			bsg_job->reply->reply_payload_rcv_len =
+			bsg_reply->reply_payload_rcv_len =
 			    le16_to_cpu(((sts_entry_t *)pkt)->rsp_info_len);
 
 			ql_log(ql_log_warn, vha, 0x5048,
 			    "CT pass-through-%s error "
 			    "comp_status-status=0x%x total_byte = 0x%x.\n",
 			    type, comp_status,
-			    bsg_job->reply->reply_payload_rcv_len);
+			    bsg_reply->reply_payload_rcv_len);
 		} else {
 			ql_log(ql_log_warn, vha, 0x5049,
 			    "CT pass-through-%s error "
 			    "comp_status-status=0x%x.\n", type, comp_status);
 			res = DID_ERROR << 16;
-			bsg_job->reply->reply_payload_rcv_len = 0;
+			bsg_reply->reply_payload_rcv_len = 0;
 		}
 		ql_dump_buffer(ql_dbg_async + ql_dbg_buffer, vha, 0x5035,
 		    (uint8_t *)pkt, sizeof(*pkt));
 	} else {
 		res = DID_OK << 16;
-		bsg_job->reply->reply_payload_rcv_len =
+		bsg_reply->reply_payload_rcv_len =
 		    bsg_job->reply_payload.payload_len;
 		bsg_job->reply_len = 0;
 	}
@@ -1413,7 +1415,8 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 	const char func[] = "ELS_CT_IOCB";
 	const char *type;
 	srb_t *sp;
-	struct fc_bsg_job *bsg_job;
+	struct bsg_job *bsg_job;
+	struct fc_bsg_reply *bsg_reply;
 	uint16_t comp_status;
 	uint32_t fw_status[3];
 	uint8_t* fw_sts_ptr;
@@ -1423,6 +1426,7 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 	if (!sp)
 		return;
 	bsg_job = sp->u.bsg_job;
+	bsg_reply = bsg_job->reply;
 
 	type = NULL;
 	switch (sp->type) {
@@ -1452,13 +1456,13 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 	/* return FC_CTELS_STATUS_OK and leave the decoding of the ELS/CT
 	 * fc payload  to the caller
 	 */
-	bsg_job->reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK;
+	bsg_reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK;
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply) + sizeof(fw_status);
 
 	if (comp_status != CS_COMPLETE) {
 		if (comp_status == CS_DATA_UNDERRUN) {
 			res = DID_OK << 16;
-			bsg_job->reply->reply_payload_rcv_len =
+			bsg_reply->reply_payload_rcv_len =
 			    le16_to_cpu(((struct els_sts_entry_24xx *)pkt)->total_byte_count);
 
 			ql_dbg(ql_dbg_user, vha, 0x503f,
@@ -1480,7 +1484,7 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 			    le16_to_cpu(((struct els_sts_entry_24xx *)
 				    pkt)->error_subcode_2));
 			res = DID_ERROR << 16;
-			bsg_job->reply->reply_payload_rcv_len = 0;
+			bsg_reply->reply_payload_rcv_len = 0;
 			fw_sts_ptr = ((uint8_t*)bsg_job->req->sense) + sizeof(struct fc_bsg_reply);
 			memcpy( fw_sts_ptr, fw_status, sizeof(fw_status));
 		}
@@ -1489,7 +1493,7 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 	}
 	else {
 		res =  DID_OK << 16;
-		bsg_job->reply->reply_payload_rcv_len = bsg_job->reply_payload.payload_len;
+		bsg_reply->reply_payload_rcv_len = bsg_job->reply_payload.payload_len;
 		bsg_job->reply_len = 0;
 	}
 
@@ -1904,7 +1908,9 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt,
 	uint16_t	scsi_status;
 	uint16_t thread_id;
 	uint32_t rval = EXT_STATUS_OK;
-	struct fc_bsg_job *bsg_job = NULL;
+	struct bsg_job *bsg_job = NULL;
+	struct fc_bsg_request *bsg_request;
+	struct fc_bsg_reply *bsg_reply;
 	sts_entry_t *sts;
 	struct sts_entry_24xx *sts24;
 	sts = (sts_entry_t *) pkt;
@@ -1919,11 +1925,7 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt,
 	}
 
 	sp = req->outstanding_cmds[index];
-	if (sp) {
-		/* Free outstanding command slot. */
-		req->outstanding_cmds[index] = NULL;
-		bsg_job = sp->u.bsg_job;
-	} else {
+	if (!sp) {
 		ql_log(ql_log_warn, vha, 0x70b0,
 		    "Req:%d: Invalid ISP SCSI completion handle(0x%x)\n",
 		    req->id, index);
@@ -1932,6 +1934,12 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt,
 		return;
 	}
 
+	/* Free outstanding command slot. */
+	req->outstanding_cmds[index] = NULL;
+	bsg_job = sp->u.bsg_job;
+	bsg_request = bsg_job->request;
+	bsg_reply = bsg_job->reply;
+
 	if (IS_FWI2_CAPABLE(ha)) {
 		comp_status = le16_to_cpu(sts24->comp_status);
 		scsi_status = le16_to_cpu(sts24->scsi_status) & SS_MASK;
@@ -1940,14 +1948,14 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt,
 		scsi_status = le16_to_cpu(sts->scsi_status) & SS_MASK;
 	}
 
-	thread_id = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1];
+	thread_id = bsg_request->rqst_data.h_vendor.vendor_cmd[1];
 	switch (comp_status) {
 	case CS_COMPLETE:
 		if (scsi_status == 0) {
-			bsg_job->reply->reply_payload_rcv_len =
+			bsg_reply->reply_payload_rcv_len =
 					bsg_job->reply_payload.payload_len;
 			vha->qla_stats.input_bytes +=
-				bsg_job->reply->reply_payload_rcv_len;
+				bsg_reply->reply_payload_rcv_len;
 			vha->qla_stats.input_requests++;
 			rval = EXT_STATUS_OK;
 		}
@@ -2028,11 +2036,11 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt,
 		rval = EXT_STATUS_ERR;
 		break;
 	}
-	bsg_job->reply->reply_payload_rcv_len = 0;
+	bsg_reply->reply_payload_rcv_len = 0;
 
 done:
 	/* Return the vendor specific reply to API */
-	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = rval;
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = rval;
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
 	/* Always return DID_OK, bsg will send the vendor specific response
 	 * in this case only */
diff --git a/drivers/scsi/qla2xxx/qla_mr.c b/drivers/scsi/qla2xxx/qla_mr.c
index 15dff7099955..02f1de18bc2b 100644
--- a/drivers/scsi/qla2xxx/qla_mr.c
+++ b/drivers/scsi/qla2xxx/qla_mr.c
@@ -10,6 +10,7 @@
 #include <linux/pci.h>
 #include <linux/ratelimit.h>
 #include <linux/vmalloc.h>
+#include <linux/bsg-lib.h>
 #include <scsi/scsi_tcq.h>
 #include <linux/utsname.h>
 
@@ -2206,7 +2207,8 @@ qlafx00_ioctl_iosb_entry(scsi_qla_host_t *vha, struct req_que *req,
 {
 	const char func[] = "IOSB_IOCB";
 	srb_t *sp;
-	struct fc_bsg_job *bsg_job;
+	struct bsg_job *bsg_job;
+	struct fc_bsg_reply *bsg_reply;
 	struct srb_iocb *iocb_job;
 	int res;
 	struct qla_mt_iocb_rsp_fx00 fstatus;
@@ -2226,6 +2228,7 @@ qlafx00_ioctl_iosb_entry(scsi_qla_host_t *vha, struct req_que *req,
 			    pkt->dataword_r;
 	} else {
 		bsg_job = sp->u.bsg_job;
+		bsg_reply = bsg_job->reply;
 
 		memset(&fstatus, 0, sizeof(struct qla_mt_iocb_rsp_fx00));
 
@@ -2257,8 +2260,8 @@ qlafx00_ioctl_iosb_entry(scsi_qla_host_t *vha, struct req_que *req,
 		    sp->fcport->vha, 0x5074,
 		    (uint8_t *)fw_sts_ptr, sizeof(struct qla_mt_iocb_rsp_fx00));
 
-		res = bsg_job->reply->result = DID_OK << 16;
-		bsg_job->reply->reply_payload_rcv_len =
+		res = bsg_reply->result = DID_OK << 16;
+		bsg_reply->reply_payload_rcv_len =
 		    bsg_job->reply_payload.payload_len;
 	}
 	sp->done(vha, sp, res);
@@ -3252,7 +3255,8 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb)
 {
 	struct srb_iocb *fxio = &sp->u.iocb_cmd;
 	struct qla_mt_iocb_rqst_fx00 *piocb_rqst;
-	struct fc_bsg_job *bsg_job;
+	struct bsg_job *bsg_job;
+	struct fc_bsg_request *bsg_request;
 	struct fxdisc_entry_fx00 fx_iocb;
 	uint8_t entry_cnt = 1;
 
@@ -3301,8 +3305,9 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb)
 	} else {
 		struct scatterlist *sg;
 		bsg_job = sp->u.bsg_job;
+		bsg_request = bsg_job->request;
 		piocb_rqst = (struct qla_mt_iocb_rqst_fx00 *)
-			&bsg_job->request->rqst_data.h_vendor.vendor_cmd[1];
+			&bsg_request->rqst_data.h_vendor.vendor_cmd[1];
 
 		fx_iocb.func_num = piocb_rqst->func_type;
 		fx_iocb.adapid = piocb_rqst->adapid;
diff --git a/drivers/scsi/qla4xxx/ql4_def.h b/drivers/scsi/qla4xxx/ql4_def.h
index a7cfc270bd08..aeebefb1e9f8 100644
--- a/drivers/scsi/qla4xxx/ql4_def.h
+++ b/drivers/scsi/qla4xxx/ql4_def.h
@@ -409,18 +409,9 @@ struct qla4_8xxx_legacy_intr_set {
 
 /* MSI-X Support */
 
-#define QLA_MSIX_DEFAULT	0x00
-#define QLA_MSIX_RSP_Q		0x01
-
+#define QLA_MSIX_DEFAULT	0
+#define QLA_MSIX_RSP_Q		1
 #define QLA_MSIX_ENTRIES	2
-#define QLA_MIDX_DEFAULT	0
-#define QLA_MIDX_RSP_Q		1
-
-struct ql4_msix_entry {
-	int have_irq;
-	uint16_t msix_vector;
-	uint16_t msix_entry;
-};
 
 /*
  * ISP Operations
@@ -572,9 +563,6 @@ struct scsi_qla_host {
 #define AF_IRQ_ATTACHED			10 /* 0x00000400 */
 #define AF_DISABLE_ACB_COMPLETE		11 /* 0x00000800 */
 #define AF_HA_REMOVAL			12 /* 0x00001000 */
-#define AF_INTx_ENABLED			15 /* 0x00008000 */
-#define AF_MSI_ENABLED			16 /* 0x00010000 */
-#define AF_MSIX_ENABLED			17 /* 0x00020000 */
 #define AF_MBOX_COMMAND_NOPOLL		18 /* 0x00040000 */
 #define AF_FW_RECOVERY			19 /* 0x00080000 */
 #define AF_EEH_BUSY			20 /* 0x00100000 */
@@ -762,8 +750,6 @@ struct scsi_qla_host {
 	struct isp_operations *isp_ops;
 	struct ql82xx_hw_data hw;
 
-	struct ql4_msix_entry msix_entries[QLA_MSIX_ENTRIES];
-
 	uint32_t nx_dev_init_timeout;
 	uint32_t nx_reset_timeout;
 	void *fw_dump;
diff --git a/drivers/scsi/qla4xxx/ql4_glbl.h b/drivers/scsi/qla4xxx/ql4_glbl.h
index 2559144f5475..bce96a58f14e 100644
--- a/drivers/scsi/qla4xxx/ql4_glbl.h
+++ b/drivers/scsi/qla4xxx/ql4_glbl.h
@@ -134,7 +134,6 @@ int qla4_8xxx_get_flash_info(struct scsi_qla_host *ha);
 void qla4_82xx_enable_intrs(struct scsi_qla_host *ha);
 void qla4_82xx_disable_intrs(struct scsi_qla_host *ha);
 int qla4_8xxx_enable_msix(struct scsi_qla_host *ha);
-void qla4_8xxx_disable_msix(struct scsi_qla_host *ha);
 irqreturn_t qla4_8xxx_msi_handler(int irq, void *dev_id);
 irqreturn_t qla4_8xxx_default_intr_handler(int irq, void *dev_id);
 irqreturn_t qla4_8xxx_msix_rsp_q(int irq, void *dev_id);
diff --git a/drivers/scsi/qla4xxx/ql4_isr.c b/drivers/scsi/qla4xxx/ql4_isr.c
index 4f9c0f2be89d..d2cd33d8d67f 100644
--- a/drivers/scsi/qla4xxx/ql4_isr.c
+++ b/drivers/scsi/qla4xxx/ql4_isr.c
@@ -1107,7 +1107,7 @@ static void qla4_82xx_spurious_interrupt(struct scsi_qla_host *ha,
 	DEBUG2(ql4_printk(KERN_INFO, ha, "Spurious Interrupt\n"));
 	if (is_qla8022(ha)) {
 		writel(0, &ha->qla4_82xx_reg->host_int);
-		if (test_bit(AF_INTx_ENABLED, &ha->flags))
+		if (!ha->pdev->msi_enabled && !ha->pdev->msix_enabled)
 			qla4_82xx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg,
 			    0xfbff);
 	}
@@ -1564,19 +1564,18 @@ int qla4xxx_request_irqs(struct scsi_qla_host *ha)
 
 try_msi:
 	/* Trying MSI */
-	ret = pci_enable_msi(ha->pdev);
-	if (!ret) {
+	ret = pci_alloc_irq_vectors(ha->pdev, 1, 1, PCI_IRQ_MSI);
+	if (ret > 0) {
 		ret = request_irq(ha->pdev->irq, qla4_8xxx_msi_handler,
 			0, DRIVER_NAME, ha);
 		if (!ret) {
 			DEBUG2(ql4_printk(KERN_INFO, ha, "MSI: Enabled.\n"));
-			set_bit(AF_MSI_ENABLED, &ha->flags);
 			goto irq_attached;
 		} else {
 			ql4_printk(KERN_WARNING, ha,
 			    "MSI: Failed to reserve interrupt %d "
 			    "already in use.\n", ha->pdev->irq);
-			pci_disable_msi(ha->pdev);
+			pci_free_irq_vectors(ha->pdev);
 		}
 	}
 
@@ -1592,7 +1591,6 @@ try_intx:
 	    IRQF_SHARED, DRIVER_NAME, ha);
 	if (!ret) {
 		DEBUG2(ql4_printk(KERN_INFO, ha, "INTx: Enabled.\n"));
-		set_bit(AF_INTx_ENABLED, &ha->flags);
 		goto irq_attached;
 
 	} else {
@@ -1614,14 +1612,11 @@ irq_not_attached:
 
 void qla4xxx_free_irqs(struct scsi_qla_host *ha)
 {
-	if (test_and_clear_bit(AF_IRQ_ATTACHED, &ha->flags)) {
-		if (test_bit(AF_MSIX_ENABLED, &ha->flags)) {
-			qla4_8xxx_disable_msix(ha);
-		} else if (test_and_clear_bit(AF_MSI_ENABLED, &ha->flags)) {
-			free_irq(ha->pdev->irq, ha);
-			pci_disable_msi(ha->pdev);
-		} else if (test_and_clear_bit(AF_INTx_ENABLED, &ha->flags)) {
-			free_irq(ha->pdev->irq, ha);
-		}
-	}
+	if (!test_and_clear_bit(AF_IRQ_ATTACHED, &ha->flags))
+		return;
+
+	if (ha->pdev->msix_enabled)
+		free_irq(pci_irq_vector(ha->pdev, 1), ha);
+	free_irq(pci_irq_vector(ha->pdev, 0), ha);
+	pci_free_irq_vectors(ha->pdev);
 }
diff --git a/drivers/scsi/qla4xxx/ql4_mbx.c b/drivers/scsi/qla4xxx/ql4_mbx.c
index c291fdff1b33..1da04f323d38 100644
--- a/drivers/scsi/qla4xxx/ql4_mbx.c
+++ b/drivers/scsi/qla4xxx/ql4_mbx.c
@@ -2032,10 +2032,7 @@ int qla4xxx_set_param_ddbentry(struct scsi_qla_host *ha,
 	ptid = (uint16_t *)&fw_ddb_entry->isid[1];
 	*ptid = cpu_to_le16((uint16_t)ddb_entry->sess->target_id);
 
-	DEBUG2(ql4_printk(KERN_INFO, ha, "ISID [%02x%02x%02x%02x%02x%02x]\n",
-			  fw_ddb_entry->isid[5], fw_ddb_entry->isid[4],
-			  fw_ddb_entry->isid[3], fw_ddb_entry->isid[2],
-			  fw_ddb_entry->isid[1], fw_ddb_entry->isid[0]));
+	DEBUG2(ql4_printk(KERN_INFO, ha, "ISID [%pmR]\n", fw_ddb_entry->isid));
 
 	iscsi_opts = le16_to_cpu(fw_ddb_entry->iscsi_options);
 	memset(fw_ddb_entry->iscsi_alias, 0, sizeof(fw_ddb_entry->iscsi_alias));
diff --git a/drivers/scsi/qla4xxx/ql4_nx.c b/drivers/scsi/qla4xxx/ql4_nx.c
index 06ddd13cb7cc..e91abb327745 100644
--- a/drivers/scsi/qla4xxx/ql4_nx.c
+++ b/drivers/scsi/qla4xxx/ql4_nx.c
@@ -3945,7 +3945,7 @@ void qla4_82xx_process_mbox_intr(struct scsi_qla_host *ha, int out_count)
 		ha->isp_ops->interrupt_service_routine(ha, intr_status);
 
 		if (test_bit(AF_INTERRUPTS_ON, &ha->flags) &&
-		    test_bit(AF_INTx_ENABLED, &ha->flags))
+		    (!ha->pdev->msi_enabled && !ha->pdev->msix_enabled))
 			qla4_82xx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg,
 					0xfbff);
 	}
@@ -4094,12 +4094,8 @@ int qla4_8xxx_get_sys_info(struct scsi_qla_host *ha)
 	ha->phy_port_num = sys_info->port_num;
 	ha->iscsi_pci_func_cnt = sys_info->iscsi_pci_func_cnt;
 
-	DEBUG2(printk("scsi%ld: %s: "
-	    "mac %02x:%02x:%02x:%02x:%02x:%02x "
-	    "serial %s\n", ha->host_no, __func__,
-	    ha->my_mac[0], ha->my_mac[1], ha->my_mac[2],
-	    ha->my_mac[3], ha->my_mac[4], ha->my_mac[5],
-	    ha->serial_number));
+	DEBUG2(printk("scsi%ld: %s: mac %pM serial %s\n",
+	    ha->host_no, __func__, ha->my_mac, ha->serial_number));
 
 	status = QLA_SUCCESS;
 
@@ -4178,78 +4174,37 @@ qla4_82xx_disable_intrs(struct scsi_qla_host *ha)
 	spin_unlock_irq(&ha->hardware_lock);
 }
 
-struct ql4_init_msix_entry {
-	uint16_t entry;
-	uint16_t index;
-	const char *name;
-	irq_handler_t handler;
-};
-
-static struct ql4_init_msix_entry qla4_8xxx_msix_entries[QLA_MSIX_ENTRIES] = {
-	{ QLA_MSIX_DEFAULT, QLA_MIDX_DEFAULT,
-	    "qla4xxx (default)",
-	    (irq_handler_t)qla4_8xxx_default_intr_handler },
-	{ QLA_MSIX_RSP_Q, QLA_MIDX_RSP_Q,
-	    "qla4xxx (rsp_q)", (irq_handler_t)qla4_8xxx_msix_rsp_q },
-};
-
-void
-qla4_8xxx_disable_msix(struct scsi_qla_host *ha)
-{
-	int i;
-	struct ql4_msix_entry *qentry;
-
-	for (i = 0; i < QLA_MSIX_ENTRIES; i++) {
-		qentry = &ha->msix_entries[qla4_8xxx_msix_entries[i].index];
-		if (qentry->have_irq) {
-			free_irq(qentry->msix_vector, ha);
-			DEBUG2(ql4_printk(KERN_INFO, ha, "%s: %s\n",
-				__func__, qla4_8xxx_msix_entries[i].name));
-		}
-	}
-	pci_disable_msix(ha->pdev);
-	clear_bit(AF_MSIX_ENABLED, &ha->flags);
-}
-
 int
 qla4_8xxx_enable_msix(struct scsi_qla_host *ha)
 {
-	int i, ret;
-	struct msix_entry entries[QLA_MSIX_ENTRIES];
-	struct ql4_msix_entry *qentry;
-
-	for (i = 0; i < QLA_MSIX_ENTRIES; i++)
-		entries[i].entry = qla4_8xxx_msix_entries[i].entry;
+	int ret;
 
-	ret = pci_enable_msix_exact(ha->pdev, entries, ARRAY_SIZE(entries));
-	if (ret) {
+	ret = pci_alloc_irq_vectors(ha->pdev, QLA_MSIX_ENTRIES,
+			QLA_MSIX_ENTRIES, PCI_IRQ_MSIX);
+	if (ret < 0) {
 		ql4_printk(KERN_WARNING, ha,
 		    "MSI-X: Failed to enable support -- %d/%d\n",
 		    QLA_MSIX_ENTRIES, ret);
-		goto msix_out;
-	}
-	set_bit(AF_MSIX_ENABLED, &ha->flags);
-
-	for (i = 0; i < QLA_MSIX_ENTRIES; i++) {
-		qentry = &ha->msix_entries[qla4_8xxx_msix_entries[i].index];
-		qentry->msix_vector = entries[i].vector;
-		qentry->msix_entry = entries[i].entry;
-		qentry->have_irq = 0;
-		ret = request_irq(qentry->msix_vector,
-		    qla4_8xxx_msix_entries[i].handler, 0,
-		    qla4_8xxx_msix_entries[i].name, ha);
-		if (ret) {
-			ql4_printk(KERN_WARNING, ha,
-			    "MSI-X: Unable to register handler -- %x/%d.\n",
-			    qla4_8xxx_msix_entries[i].index, ret);
-			qla4_8xxx_disable_msix(ha);
-			goto msix_out;
-		}
-		qentry->have_irq = 1;
-		DEBUG2(ql4_printk(KERN_INFO, ha, "%s: %s\n",
-			__func__, qla4_8xxx_msix_entries[i].name));
+		return ret;
 	}
-msix_out:
+
+	ret = request_irq(pci_irq_vector(ha->pdev, 0),
+			qla4_8xxx_default_intr_handler, 0, "qla4xxx (default)",
+			ha);
+	if (ret)
+		goto out_free_vectors;
+
+	ret = request_irq(pci_irq_vector(ha->pdev, 1),
+			qla4_8xxx_msix_rsp_q, 0, "qla4xxx (rsp_q)", ha);
+	if (ret)
+		goto out_free_default_irq;
+
+	return 0;
+
+out_free_default_irq:
+	free_irq(pci_irq_vector(ha->pdev, 0), ha);
+out_free_vectors:
+	pci_free_irq_vectors(ha->pdev);
 	return ret;
 }
 
diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
index 01c3610a60cf..9fbb33fc90c7 100644
--- a/drivers/scsi/qla4xxx/ql4_os.c
+++ b/drivers/scsi/qla4xxx/ql4_os.c
@@ -6304,13 +6304,9 @@ static int qla4xxx_compare_tuple_ddb(struct scsi_qla_host *ha,
 	 * ISID would not match firmware generated ISID.
 	 */
 	if (is_isid_compare) {
-		DEBUG2(ql4_printk(KERN_INFO, ha, "%s: old ISID [%02x%02x%02x"
-			"%02x%02x%02x] New ISID [%02x%02x%02x%02x%02x%02x]\n",
-			__func__, old_tddb->isid[5], old_tddb->isid[4],
-			old_tddb->isid[3], old_tddb->isid[2], old_tddb->isid[1],
-			old_tddb->isid[0], new_tddb->isid[5], new_tddb->isid[4],
-			new_tddb->isid[3], new_tddb->isid[2], new_tddb->isid[1],
-			new_tddb->isid[0]));
+		DEBUG2(ql4_printk(KERN_INFO, ha,
+			"%s: old ISID [%pmR] New ISID [%pmR]\n",
+			__func__, old_tddb->isid, new_tddb->isid));
 
 		if (memcmp(&old_tddb->isid[0], &new_tddb->isid[0],
 			   sizeof(old_tddb->isid)))
@@ -7925,10 +7921,7 @@ qla4xxx_sysfs_ddb_get_param(struct iscsi_bus_flash_session *fnode_sess,
 		rc = sprintf(buf, "%u\n", fnode_conn->keepalive_timeout);
 		break;
 	case ISCSI_FLASHNODE_ISID:
-		rc = sprintf(buf, "%02x%02x%02x%02x%02x%02x\n",
-			     fnode_sess->isid[0], fnode_sess->isid[1],
-			     fnode_sess->isid[2], fnode_sess->isid[3],
-			     fnode_sess->isid[4], fnode_sess->isid[5]);
+		rc = sprintf(buf, "%pm\n", fnode_sess->isid);
 		break;
 	case ISCSI_FLASHNODE_TSID:
 		rc = sprintf(buf, "%u\n", fnode_sess->tsid);
diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c
index 1deb6adc411f..75455d4dab68 100644
--- a/drivers/scsi/scsi.c
+++ b/drivers/scsi/scsi.c
@@ -621,6 +621,9 @@ int scsi_change_queue_depth(struct scsi_device *sdev, int depth)
 		wmb();
 	}
 
+	if (sdev->request_queue)
+		blk_set_queue_depth(sdev->request_queue, depth);
+
 	return sdev->queue_depth;
 }
 EXPORT_SYMBOL(scsi_change_queue_depth);
diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c
index 246456925335..28fea83ae2fe 100644
--- a/drivers/scsi/scsi_devinfo.c
+++ b/drivers/scsi/scsi_devinfo.c
@@ -220,8 +220,6 @@ static struct {
 	{"NAKAMICH", "MJ-5.16S", NULL, BLIST_FORCELUN | BLIST_SINGLELUN},
 	{"NEC", "PD-1 ODX654P", NULL, BLIST_FORCELUN | BLIST_SINGLELUN},
 	{"NEC", "iStorage", NULL, BLIST_REPORTLUN2},
-	{"NETAPP", "LUN C-Mode", NULL, BLIST_SYNC_ALUA},
-	{"NETAPP", "INF-01-00", NULL, BLIST_SYNC_ALUA},
 	{"NRC", "MBR-7", NULL, BLIST_FORCELUN | BLIST_SINGLELUN},
 	{"NRC", "MBR-7.4", NULL, BLIST_FORCELUN | BLIST_SINGLELUN},
 	{"PIONEER", "CD-ROM DRM-600", NULL, BLIST_FORCELUN | BLIST_SINGLELUN},
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index 106a6adbd6f1..996e134d79fa 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -1988,7 +1988,7 @@ static void scsi_eh_lock_door(struct scsi_device *sdev)
 
 	req->cmd_len = COMMAND_SIZE(req->cmd[0]);
 
-	req->cmd_flags |= REQ_QUIET;
+	req->rq_flags |= RQF_QUIET;
 	req->timeout = 10 * HZ;
 	req->retries = 5;
 
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 2cca9cffc63f..c35b6de4ca64 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -86,10 +86,8 @@ scsi_set_blocked(struct scsi_cmnd *cmd, int reason)
 static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd)
 {
 	struct scsi_device *sdev = cmd->device;
-	struct request_queue *q = cmd->request->q;
 
-	blk_mq_requeue_request(cmd->request);
-	blk_mq_kick_requeue_list(q);
+	blk_mq_requeue_request(cmd->request, true);
 	put_device(&sdev->sdev_gendev);
 }
 
@@ -163,26 +161,11 @@ void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
 {
 	__scsi_queue_insert(cmd, reason, 1);
 }
-/**
- * scsi_execute - insert request and wait for the result
- * @sdev:	scsi device
- * @cmd:	scsi command
- * @data_direction: data direction
- * @buffer:	data buffer
- * @bufflen:	len of buffer
- * @sense:	optional sense buffer
- * @timeout:	request timeout in seconds
- * @retries:	number of times to retry request
- * @flags:	or into request flags;
- * @resid:	optional residual length
- *
- * returns the req->errors value which is the scsi_cmnd result
- * field.
- */
-int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
+
+static int __scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
 		 int data_direction, void *buffer, unsigned bufflen,
 		 unsigned char *sense, int timeout, int retries, u64 flags,
-		 int *resid)
+		 req_flags_t rq_flags, int *resid)
 {
 	struct request *req;
 	int write = (data_direction == DMA_TO_DEVICE);
@@ -203,7 +186,8 @@ int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
 	req->sense_len = 0;
 	req->retries = retries;
 	req->timeout = timeout;
-	req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT;
+	req->cmd_flags |= flags;
+	req->rq_flags |= rq_flags | RQF_QUIET | RQF_PREEMPT;
 
 	/*
 	 * head injection *required* here otherwise quiesce won't work
@@ -227,12 +211,37 @@ int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
 
 	return ret;
 }
+
+/**
+ * scsi_execute - insert request and wait for the result
+ * @sdev:	scsi device
+ * @cmd:	scsi command
+ * @data_direction: data direction
+ * @buffer:	data buffer
+ * @bufflen:	len of buffer
+ * @sense:	optional sense buffer
+ * @timeout:	request timeout in seconds
+ * @retries:	number of times to retry request
+ * @flags:	or into request flags;
+ * @resid:	optional residual length
+ *
+ * returns the req->errors value which is the scsi_cmnd result
+ * field.
+ */
+int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
+		 int data_direction, void *buffer, unsigned bufflen,
+		 unsigned char *sense, int timeout, int retries, u64 flags,
+		 int *resid)
+{
+	return __scsi_execute(sdev, cmd, data_direction, buffer, bufflen, sense,
+			timeout, retries, flags, 0, resid);
+}
 EXPORT_SYMBOL(scsi_execute);
 
 int scsi_execute_req_flags(struct scsi_device *sdev, const unsigned char *cmd,
 		     int data_direction, void *buffer, unsigned bufflen,
 		     struct scsi_sense_hdr *sshdr, int timeout, int retries,
-		     int *resid, u64 flags)
+		     int *resid, u64 flags, req_flags_t rq_flags)
 {
 	char *sense = NULL;
 	int result;
@@ -242,8 +251,8 @@ int scsi_execute_req_flags(struct scsi_device *sdev, const unsigned char *cmd,
 		if (!sense)
 			return DRIVER_ERROR << 24;
 	}
-	result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
-			      sense, timeout, retries, flags, resid);
+	result = __scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
+			      sense, timeout, retries, flags, rq_flags, resid);
 	if (sshdr)
 		scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
 
@@ -813,7 +822,7 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
 		 */
 		if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
 			;
-		else if (!(req->cmd_flags & REQ_QUIET))
+		else if (!(req->rq_flags & RQF_QUIET))
 			scsi_print_sense(cmd);
 		result = 0;
 		/* BLOCK_PC may have set error */
@@ -943,7 +952,7 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
 	switch (action) {
 	case ACTION_FAIL:
 		/* Give up and fail the remainder of the request */
-		if (!(req->cmd_flags & REQ_QUIET)) {
+		if (!(req->rq_flags & RQF_QUIET)) {
 			static DEFINE_RATELIMIT_STATE(_rs,
 					DEFAULT_RATELIMIT_INTERVAL,
 					DEFAULT_RATELIMIT_BURST);
@@ -972,7 +981,7 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
 		 * A new command will be prepared and issued.
 		 */
 		if (q->mq_ops) {
-			cmd->request->cmd_flags &= ~REQ_DONTPREP;
+			cmd->request->rq_flags &= ~RQF_DONTPREP;
 			scsi_mq_uninit_cmd(cmd);
 			scsi_mq_requeue_cmd(cmd);
 		} else {
@@ -998,8 +1007,8 @@ static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb)
 	/*
 	 * If sg table allocation fails, requeue request later.
 	 */
-	if (unlikely(sg_alloc_table_chained(&sdb->table, req->nr_phys_segments,
-					sdb->table.sgl)))
+	if (unlikely(sg_alloc_table_chained(&sdb->table,
+			blk_rq_nr_phys_segments(req), sdb->table.sgl)))
 		return BLKPREP_DEFER;
 
 	/* 
@@ -1031,7 +1040,7 @@ int scsi_init_io(struct scsi_cmnd *cmd)
 	bool is_mq = (rq->mq_ctx != NULL);
 	int error;
 
-	BUG_ON(!rq->nr_phys_segments);
+	BUG_ON(!blk_rq_nr_phys_segments(rq));
 
 	error = scsi_init_sgtable(rq, &cmd->sdb);
 	if (error)
@@ -1234,7 +1243,7 @@ scsi_prep_state_check(struct scsi_device *sdev, struct request *req)
 			/*
 			 * If the devices is blocked we defer normal commands.
 			 */
-			if (!(req->cmd_flags & REQ_PREEMPT))
+			if (!(req->rq_flags & RQF_PREEMPT))
 				ret = BLKPREP_DEFER;
 			break;
 		default:
@@ -1243,7 +1252,7 @@ scsi_prep_state_check(struct scsi_device *sdev, struct request *req)
 			 * special commands.  In particular any user initiated
 			 * command is not allowed.
 			 */
-			if (!(req->cmd_flags & REQ_PREEMPT))
+			if (!(req->rq_flags & RQF_PREEMPT))
 				ret = BLKPREP_KILL;
 			break;
 		}
@@ -1279,7 +1288,7 @@ scsi_prep_return(struct request_queue *q, struct request *req, int ret)
 			blk_delay_queue(q, SCSI_QUEUE_DELAY);
 		break;
 	default:
-		req->cmd_flags |= REQ_DONTPREP;
+		req->rq_flags |= RQF_DONTPREP;
 	}
 
 	return ret;
@@ -1736,7 +1745,7 @@ static void scsi_request_fn(struct request_queue *q)
 		 * we add the dev to the starved list so it eventually gets
 		 * a run when a tag is freed.
 		 */
-		if (blk_queue_tagged(q) && !(req->cmd_flags & REQ_QUEUED)) {
+		if (blk_queue_tagged(q) && !(req->rq_flags & RQF_QUEUED)) {
 			spin_lock_irq(shost->host_lock);
 			if (list_empty(&sdev->starved_entry))
 				list_add_tail(&sdev->starved_entry,
@@ -1801,7 +1810,7 @@ static inline int prep_to_mq(int ret)
 {
 	switch (ret) {
 	case BLKPREP_OK:
-		return 0;
+		return BLK_MQ_RQ_QUEUE_OK;
 	case BLKPREP_DEFER:
 		return BLK_MQ_RQ_QUEUE_BUSY;
 	default:
@@ -1888,7 +1897,7 @@ static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
 	int reason;
 
 	ret = prep_to_mq(scsi_prep_state_check(sdev, req));
-	if (ret)
+	if (ret != BLK_MQ_RQ_QUEUE_OK)
 		goto out;
 
 	ret = BLK_MQ_RQ_QUEUE_BUSY;
@@ -1903,11 +1912,11 @@ static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
 		goto out_dec_target_busy;
 
 
-	if (!(req->cmd_flags & REQ_DONTPREP)) {
+	if (!(req->rq_flags & RQF_DONTPREP)) {
 		ret = prep_to_mq(scsi_mq_prep_fn(req));
-		if (ret)
+		if (ret != BLK_MQ_RQ_QUEUE_OK)
 			goto out_dec_host_busy;
-		req->cmd_flags |= REQ_DONTPREP;
+		req->rq_flags |= RQF_DONTPREP;
 	} else {
 		blk_mq_start_request(req);
 	}
@@ -1941,7 +1950,6 @@ out_put_device:
 out:
 	switch (ret) {
 	case BLK_MQ_RQ_QUEUE_BUSY:
-		blk_mq_stop_hw_queue(hctx);
 		if (atomic_read(&sdev->device_busy) == 0 &&
 		    !scsi_device_blocked(sdev))
 			blk_mq_delay_queue(hctx, SCSI_QUEUE_DELAY);
@@ -1952,7 +1960,7 @@ out:
 		 * we hit an error, as we will never see this command
 		 * again.
 		 */
-		if (req->cmd_flags & REQ_DONTPREP)
+		if (req->rq_flags & RQF_DONTPREP)
 			scsi_mq_uninit_cmd(cmd);
 		break;
 	default:
@@ -1990,6 +1998,15 @@ static void scsi_exit_request(void *data, struct request *rq,
 	kfree(cmd->sense_buffer);
 }
 
+static int scsi_map_queues(struct blk_mq_tag_set *set)
+{
+	struct Scsi_Host *shost = container_of(set, struct Scsi_Host, tag_set);
+
+	if (shost->hostt->map_queues)
+		return shost->hostt->map_queues(shost);
+	return blk_mq_map_queues(set);
+}
+
 static u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
 {
 	struct device *host_dev;
@@ -2082,6 +2099,7 @@ static struct blk_mq_ops scsi_mq_ops = {
 	.timeout	= scsi_timeout,
 	.init_request	= scsi_init_request,
 	.exit_request	= scsi_exit_request,
+	.map_queues	= scsi_map_queues,
 };
 
 struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev)
@@ -2724,6 +2742,39 @@ void sdev_evt_send_simple(struct scsi_device *sdev,
 EXPORT_SYMBOL_GPL(sdev_evt_send_simple);
 
 /**
+ * scsi_request_fn_active() - number of kernel threads inside scsi_request_fn()
+ * @sdev: SCSI device to count the number of scsi_request_fn() callers for.
+ */
+static int scsi_request_fn_active(struct scsi_device *sdev)
+{
+	struct request_queue *q = sdev->request_queue;
+	int request_fn_active;
+
+	WARN_ON_ONCE(sdev->host->use_blk_mq);
+
+	spin_lock_irq(q->queue_lock);
+	request_fn_active = q->request_fn_active;
+	spin_unlock_irq(q->queue_lock);
+
+	return request_fn_active;
+}
+
+/**
+ * scsi_wait_for_queuecommand() - wait for ongoing queuecommand() calls
+ * @sdev: SCSI device pointer.
+ *
+ * Wait until the ongoing shost->hostt->queuecommand() calls that are
+ * invoked from scsi_request_fn() have finished.
+ */
+static void scsi_wait_for_queuecommand(struct scsi_device *sdev)
+{
+	WARN_ON_ONCE(sdev->host->use_blk_mq);
+
+	while (scsi_request_fn_active(sdev))
+		msleep(20);
+}
+
+/**
  *	scsi_device_quiesce - Block user issued commands.
  *	@sdev:	scsi device to quiesce.
  *
@@ -2807,8 +2858,7 @@ EXPORT_SYMBOL(scsi_target_resume);
  * @sdev:	device to block
  *
  * Block request made by scsi lld's to temporarily stop all
- * scsi commands on the specified device.  Called from interrupt
- * or normal process context.
+ * scsi commands on the specified device. May sleep.
  *
  * Returns zero if successful or error if not
  *
@@ -2817,6 +2867,10 @@ EXPORT_SYMBOL(scsi_target_resume);
  *	(which must be a legal transition).  When the device is in this
  *	state, all commands are deferred until the scsi lld reenables
  *	the device with scsi_device_unblock or device_block_tmo fires.
+ *
+ * To do: avoid that scsi_send_eh_cmnd() calls queuecommand() after
+ * scsi_internal_device_block() has blocked a SCSI device and also
+ * remove the rport mutex lock and unlock calls from srp_queuecommand().
  */
 int
 scsi_internal_device_block(struct scsi_device *sdev)
@@ -2844,6 +2898,7 @@ scsi_internal_device_block(struct scsi_device *sdev)
 		spin_lock_irqsave(q->queue_lock, flags);
 		blk_stop_queue(q);
 		spin_unlock_irqrestore(q->queue_lock, flags);
+		scsi_wait_for_queuecommand(sdev);
 	}
 
 	return 0;
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index 0f3a3869524b..03577bde6ac5 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -30,6 +30,7 @@
 #include <linux/slab.h>
 #include <linux/delay.h>
 #include <linux/kernel.h>
+#include <linux/bsg-lib.h>
 #include <scsi/scsi_device.h>
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_transport.h>
@@ -2592,7 +2593,7 @@ fc_rport_final_delete(struct work_struct *work)
 
 
 /**
- * fc_rport_create - allocates and creates a remote FC port.
+ * fc_remote_port_create - allocates and creates a remote FC port.
  * @shost:	scsi host the remote port is connected to.
  * @channel:	Channel on shost port connected to.
  * @ids:	The world wide names, fc address, and FC4 port
@@ -2605,8 +2606,8 @@ fc_rport_final_delete(struct work_struct *work)
  *	This routine assumes no locks are held on entry.
  */
 static struct fc_rport *
-fc_rport_create(struct Scsi_Host *shost, int channel,
-	struct fc_rport_identifiers  *ids)
+fc_remote_port_create(struct Scsi_Host *shost, int channel,
+		      struct fc_rport_identifiers  *ids)
 {
 	struct fc_host_attrs *fc_host = shost_to_fc_host(shost);
 	struct fc_internal *fci = to_fc_internal(shost->transportt);
@@ -2914,7 +2915,7 @@ fc_remote_port_add(struct Scsi_Host *shost, int channel,
 	spin_unlock_irqrestore(shost->host_lock, flags);
 
 	/* No consistent binding found - create new remote port entry */
-	rport = fc_rport_create(shost, channel, ids);
+	rport = fc_remote_port_create(shost, channel, ids);
 
 	return rport;
 }
@@ -3554,81 +3555,6 @@ fc_vport_sched_delete(struct work_struct *work)
  * BSG support
  */
 
-
-/**
- * fc_destroy_bsgjob - routine to teardown/delete a fc bsg job
- * @job:	fc_bsg_job that is to be torn down
- */
-static void
-fc_destroy_bsgjob(struct fc_bsg_job *job)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&job->job_lock, flags);
-	if (job->ref_cnt) {
-		spin_unlock_irqrestore(&job->job_lock, flags);
-		return;
-	}
-	spin_unlock_irqrestore(&job->job_lock, flags);
-
-	put_device(job->dev);	/* release reference for the request */
-
-	kfree(job->request_payload.sg_list);
-	kfree(job->reply_payload.sg_list);
-	kfree(job);
-}
-
-/**
- * fc_bsg_jobdone - completion routine for bsg requests that the LLD has
- *                  completed
- * @job:	fc_bsg_job that is complete
- */
-static void
-fc_bsg_jobdone(struct fc_bsg_job *job)
-{
-	struct request *req = job->req;
-	struct request *rsp = req->next_rq;
-	int err;
-
-	err = job->req->errors = job->reply->result;
-
-	if (err < 0)
-		/* we're only returning the result field in the reply */
-		job->req->sense_len = sizeof(uint32_t);
-	else
-		job->req->sense_len = job->reply_len;
-
-	/* we assume all request payload was transferred, residual == 0 */
-	req->resid_len = 0;
-
-	if (rsp) {
-		WARN_ON(job->reply->reply_payload_rcv_len > rsp->resid_len);
-
-		/* set reply (bidi) residual */
-		rsp->resid_len -= min(job->reply->reply_payload_rcv_len,
-				      rsp->resid_len);
-	}
-	blk_complete_request(req);
-}
-
-/**
- * fc_bsg_softirq_done - softirq done routine for destroying the bsg requests
- * @rq:        BSG request that holds the job to be destroyed
- */
-static void fc_bsg_softirq_done(struct request *rq)
-{
-	struct fc_bsg_job *job = rq->special;
-	unsigned long flags;
-
-	spin_lock_irqsave(&job->job_lock, flags);
-	job->state_flags |= FC_RQST_STATE_DONE;
-	job->ref_cnt--;
-	spin_unlock_irqrestore(&job->job_lock, flags);
-
-	blk_end_request_all(rq, rq->errors);
-	fc_destroy_bsgjob(job);
-}
-
 /**
  * fc_bsg_job_timeout - handler for when a bsg request timesout
  * @req:	request that timed out
@@ -3636,27 +3562,22 @@ static void fc_bsg_softirq_done(struct request *rq)
 static enum blk_eh_timer_return
 fc_bsg_job_timeout(struct request *req)
 {
-	struct fc_bsg_job *job = (void *) req->special;
-	struct Scsi_Host *shost = job->shost;
+	struct bsg_job *job = (void *) req->special;
+	struct Scsi_Host *shost = fc_bsg_to_shost(job);
+	struct fc_rport *rport = fc_bsg_to_rport(job);
 	struct fc_internal *i = to_fc_internal(shost->transportt);
-	unsigned long flags;
-	int err = 0, done = 0;
+	int err = 0, inflight = 0;
 
-	if (job->rport && job->rport->port_state == FC_PORTSTATE_BLOCKED)
+	if (rport && rport->port_state == FC_PORTSTATE_BLOCKED)
 		return BLK_EH_RESET_TIMER;
 
-	spin_lock_irqsave(&job->job_lock, flags);
-	if (job->state_flags & FC_RQST_STATE_DONE)
-		done = 1;
-	else
-		job->ref_cnt++;
-	spin_unlock_irqrestore(&job->job_lock, flags);
+	inflight = bsg_job_get(job);
 
-	if (!done && i->f->bsg_timeout) {
+	if (inflight && i->f->bsg_timeout) {
 		/* call LLDD to abort the i/o as it has timed out */
 		err = i->f->bsg_timeout(job);
 		if (err == -EAGAIN) {
-			job->ref_cnt--;
+			bsg_job_put(job);
 			return BLK_EH_RESET_TIMER;
 		} else if (err)
 			printk(KERN_ERR "ERROR: FC BSG request timeout - LLD "
@@ -3664,126 +3585,33 @@ fc_bsg_job_timeout(struct request *req)
 	}
 
 	/* the blk_end_sync_io() doesn't check the error */
-	if (done)
+	if (!inflight)
 		return BLK_EH_NOT_HANDLED;
 	else
 		return BLK_EH_HANDLED;
 }
 
-static int
-fc_bsg_map_buffer(struct fc_bsg_buffer *buf, struct request *req)
-{
-	size_t sz = (sizeof(struct scatterlist) * req->nr_phys_segments);
-
-	BUG_ON(!req->nr_phys_segments);
-
-	buf->sg_list = kzalloc(sz, GFP_KERNEL);
-	if (!buf->sg_list)
-		return -ENOMEM;
-	sg_init_table(buf->sg_list, req->nr_phys_segments);
-	buf->sg_cnt = blk_rq_map_sg(req->q, req, buf->sg_list);
-	buf->payload_len = blk_rq_bytes(req);
-	return 0;
-}
-
-
-/**
- * fc_req_to_bsgjob - Allocate/create the fc_bsg_job structure for the
- *                   bsg request
- * @shost:	SCSI Host corresponding to the bsg object
- * @rport:	(optional) FC Remote Port corresponding to the bsg object
- * @req:	BSG request that needs a job structure
- */
-static int
-fc_req_to_bsgjob(struct Scsi_Host *shost, struct fc_rport *rport,
-	struct request *req)
-{
-	struct fc_internal *i = to_fc_internal(shost->transportt);
-	struct request *rsp = req->next_rq;
-	struct fc_bsg_job *job;
-	int ret;
-
-	BUG_ON(req->special);
-
-	job = kzalloc(sizeof(struct fc_bsg_job) + i->f->dd_bsg_size,
-			GFP_KERNEL);
-	if (!job)
-		return -ENOMEM;
-
-	/*
-	 * Note: this is a bit silly.
-	 * The request gets formatted as a SGIO v4 ioctl request, which
-	 * then gets reformatted as a blk request, which then gets
-	 * reformatted as a fc bsg request. And on completion, we have
-	 * to wrap return results such that SGIO v4 thinks it was a scsi
-	 * status.  I hope this was all worth it.
-	 */
-
-	req->special = job;
-	job->shost = shost;
-	job->rport = rport;
-	job->req = req;
-	if (i->f->dd_bsg_size)
-		job->dd_data = (void *)&job[1];
-	spin_lock_init(&job->job_lock);
-	job->request = (struct fc_bsg_request *)req->cmd;
-	job->request_len = req->cmd_len;
-	job->reply = req->sense;
-	job->reply_len = SCSI_SENSE_BUFFERSIZE;	/* Size of sense buffer
-						 * allocated */
-	if (req->bio) {
-		ret = fc_bsg_map_buffer(&job->request_payload, req);
-		if (ret)
-			goto failjob_rls_job;
-	}
-	if (rsp && rsp->bio) {
-		ret = fc_bsg_map_buffer(&job->reply_payload, rsp);
-		if (ret)
-			goto failjob_rls_rqst_payload;
-	}
-	job->job_done = fc_bsg_jobdone;
-	if (rport)
-		job->dev = &rport->dev;
-	else
-		job->dev = &shost->shost_gendev;
-	get_device(job->dev);		/* take a reference for the request */
-
-	job->ref_cnt = 1;
-
-	return 0;
-
-
-failjob_rls_rqst_payload:
-	kfree(job->request_payload.sg_list);
-failjob_rls_job:
-	kfree(job);
-	return -ENOMEM;
-}
-
-
-enum fc_dispatch_result {
-	FC_DISPATCH_BREAK,	/* on return, q is locked, break from q loop */
-	FC_DISPATCH_LOCKED,	/* on return, q is locked, continue on */
-	FC_DISPATCH_UNLOCKED,	/* on return, q is unlocked, continue on */
-};
-
-
 /**
  * fc_bsg_host_dispatch - process fc host bsg requests and dispatch to LLDD
- * @q:		fc host request queue
  * @shost:	scsi host rport attached to
  * @job:	bsg job to be processed
  */
-static enum fc_dispatch_result
-fc_bsg_host_dispatch(struct request_queue *q, struct Scsi_Host *shost,
-			 struct fc_bsg_job *job)
+static int fc_bsg_host_dispatch(struct Scsi_Host *shost, struct bsg_job *job)
 {
 	struct fc_internal *i = to_fc_internal(shost->transportt);
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	int cmdlen = sizeof(uint32_t);	/* start with length of msgcode */
 	int ret;
 
+	/* check if we really have all the request data needed */
+	if (job->request_len < cmdlen) {
+		ret = -ENOMSG;
+		goto fail_host_msg;
+	}
+
 	/* Validate the host command */
-	switch (job->request->msgcode) {
+	switch (bsg_request->msgcode) {
 	case FC_BSG_HST_ADD_RPORT:
 		cmdlen += sizeof(struct fc_bsg_host_add_rport);
 		break;
@@ -3815,7 +3643,7 @@ fc_bsg_host_dispatch(struct request_queue *q, struct Scsi_Host *shost,
 	case FC_BSG_HST_VENDOR:
 		cmdlen += sizeof(struct fc_bsg_host_vendor);
 		if ((shost->hostt->vendor_id == 0L) ||
-		    (job->request->rqst_data.h_vendor.vendor_id !=
+		    (bsg_request->rqst_data.h_vendor.vendor_id !=
 			shost->hostt->vendor_id)) {
 			ret = -ESRCH;
 			goto fail_host_msg;
@@ -3827,24 +3655,19 @@ fc_bsg_host_dispatch(struct request_queue *q, struct Scsi_Host *shost,
 		goto fail_host_msg;
 	}
 
-	/* check if we really have all the request data needed */
-	if (job->request_len < cmdlen) {
-		ret = -ENOMSG;
-		goto fail_host_msg;
-	}
-
 	ret = i->f->bsg_request(job);
 	if (!ret)
-		return FC_DISPATCH_UNLOCKED;
+		return 0;
 
 fail_host_msg:
 	/* return the errno failure code as the only status */
 	BUG_ON(job->reply_len < sizeof(uint32_t));
-	job->reply->reply_payload_rcv_len = 0;
-	job->reply->result = ret;
+	bsg_reply->reply_payload_rcv_len = 0;
+	bsg_reply->result = ret;
 	job->reply_len = sizeof(uint32_t);
-	fc_bsg_jobdone(job);
-	return FC_DISPATCH_UNLOCKED;
+	bsg_job_done(job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
+	return 0;
 }
 
 
@@ -3855,34 +3678,38 @@ fail_host_msg:
 static void
 fc_bsg_goose_queue(struct fc_rport *rport)
 {
-	if (!rport->rqst_q)
+	struct request_queue *q = rport->rqst_q;
+	unsigned long flags;
+
+	if (!q)
 		return;
 
-	/*
-	 * This get/put dance makes no sense
-	 */
-	get_device(&rport->dev);
-	blk_run_queue_async(rport->rqst_q);
-	put_device(&rport->dev);
+	spin_lock_irqsave(q->queue_lock, flags);
+	blk_run_queue_async(q);
+	spin_unlock_irqrestore(q->queue_lock, flags);
 }
 
 /**
  * fc_bsg_rport_dispatch - process rport bsg requests and dispatch to LLDD
- * @q:		rport request queue
  * @shost:	scsi host rport attached to
- * @rport:	rport request destined to
  * @job:	bsg job to be processed
  */
-static enum fc_dispatch_result
-fc_bsg_rport_dispatch(struct request_queue *q, struct Scsi_Host *shost,
-			 struct fc_rport *rport, struct fc_bsg_job *job)
+static int fc_bsg_rport_dispatch(struct Scsi_Host *shost, struct bsg_job *job)
 {
 	struct fc_internal *i = to_fc_internal(shost->transportt);
+	struct fc_bsg_request *bsg_request = job->request;
+	struct fc_bsg_reply *bsg_reply = job->reply;
 	int cmdlen = sizeof(uint32_t);	/* start with length of msgcode */
 	int ret;
 
+	/* check if we really have all the request data needed */
+	if (job->request_len < cmdlen) {
+		ret = -ENOMSG;
+		goto fail_rport_msg;
+	}
+
 	/* Validate the rport command */
-	switch (job->request->msgcode) {
+	switch (bsg_request->msgcode) {
 	case FC_BSG_RPT_ELS:
 		cmdlen += sizeof(struct fc_bsg_rport_els);
 		goto check_bidi;
@@ -3902,133 +3729,31 @@ check_bidi:
 		goto fail_rport_msg;
 	}
 
-	/* check if we really have all the request data needed */
-	if (job->request_len < cmdlen) {
-		ret = -ENOMSG;
-		goto fail_rport_msg;
-	}
-
 	ret = i->f->bsg_request(job);
 	if (!ret)
-		return FC_DISPATCH_UNLOCKED;
+		return 0;
 
 fail_rport_msg:
 	/* return the errno failure code as the only status */
 	BUG_ON(job->reply_len < sizeof(uint32_t));
-	job->reply->reply_payload_rcv_len = 0;
-	job->reply->result = ret;
+	bsg_reply->reply_payload_rcv_len = 0;
+	bsg_reply->result = ret;
 	job->reply_len = sizeof(uint32_t);
-	fc_bsg_jobdone(job);
-	return FC_DISPATCH_UNLOCKED;
-}
-
-
-/**
- * fc_bsg_request_handler - generic handler for bsg requests
- * @q:		request queue to manage
- * @shost:	Scsi_Host related to the bsg object
- * @rport:	FC remote port related to the bsg object (optional)
- * @dev:	device structure for bsg object
- */
-static void
-fc_bsg_request_handler(struct request_queue *q, struct Scsi_Host *shost,
-		       struct fc_rport *rport, struct device *dev)
-{
-	struct request *req;
-	struct fc_bsg_job *job;
-	enum fc_dispatch_result ret;
-
-	if (!get_device(dev))
-		return;
-
-	while (1) {
-		if (rport && (rport->port_state == FC_PORTSTATE_BLOCKED) &&
-		    !(rport->flags & FC_RPORT_FAST_FAIL_TIMEDOUT))
-			break;
-
-		req = blk_fetch_request(q);
-		if (!req)
-			break;
-
-		if (rport && (rport->port_state != FC_PORTSTATE_ONLINE)) {
-			req->errors = -ENXIO;
-			spin_unlock_irq(q->queue_lock);
-			blk_end_request_all(req, -ENXIO);
-			spin_lock_irq(q->queue_lock);
-			continue;
-		}
-
-		spin_unlock_irq(q->queue_lock);
-
-		ret = fc_req_to_bsgjob(shost, rport, req);
-		if (ret) {
-			req->errors = ret;
-			blk_end_request_all(req, ret);
-			spin_lock_irq(q->queue_lock);
-			continue;
-		}
-
-		job = req->special;
-
-		/* check if we have the msgcode value at least */
-		if (job->request_len < sizeof(uint32_t)) {
-			BUG_ON(job->reply_len < sizeof(uint32_t));
-			job->reply->reply_payload_rcv_len = 0;
-			job->reply->result = -ENOMSG;
-			job->reply_len = sizeof(uint32_t);
-			fc_bsg_jobdone(job);
-			spin_lock_irq(q->queue_lock);
-			continue;
-		}
-
-		/* the dispatch routines will unlock the queue_lock */
-		if (rport)
-			ret = fc_bsg_rport_dispatch(q, shost, rport, job);
-		else
-			ret = fc_bsg_host_dispatch(q, shost, job);
-
-		/* did dispatcher hit state that can't process any more */
-		if (ret == FC_DISPATCH_BREAK)
-			break;
-
-		/* did dispatcher had released the lock */
-		if (ret == FC_DISPATCH_UNLOCKED)
-			spin_lock_irq(q->queue_lock);
-	}
-
-	spin_unlock_irq(q->queue_lock);
-	put_device(dev);
-	spin_lock_irq(q->queue_lock);
-}
-
-
-/**
- * fc_bsg_host_handler - handler for bsg requests for a fc host
- * @q:		fc host request queue
- */
-static void
-fc_bsg_host_handler(struct request_queue *q)
-{
-	struct Scsi_Host *shost = q->queuedata;
-
-	fc_bsg_request_handler(q, shost, NULL, &shost->shost_gendev);
+	bsg_job_done(job, bsg_reply->result,
+		       bsg_reply->reply_payload_rcv_len);
+	return 0;
 }
 
-
-/**
- * fc_bsg_rport_handler - handler for bsg requests for a fc rport
- * @q:		rport request queue
- */
-static void
-fc_bsg_rport_handler(struct request_queue *q)
+static int fc_bsg_dispatch(struct bsg_job *job)
 {
-	struct fc_rport *rport = q->queuedata;
-	struct Scsi_Host *shost = rport_to_shost(rport);
+	struct Scsi_Host *shost = fc_bsg_to_shost(job);
 
-	fc_bsg_request_handler(q, shost, rport, &rport->dev);
+	if (scsi_is_fc_rport(job->dev))
+		return fc_bsg_rport_dispatch(shost, job);
+	else
+		return fc_bsg_host_dispatch(shost, job);
 }
 
-
 /**
  * fc_bsg_hostadd - Create and add the bsg hooks so we can receive requests
  * @shost:	shost for fc_host
@@ -4051,33 +3776,42 @@ fc_bsg_hostadd(struct Scsi_Host *shost, struct fc_host_attrs *fc_host)
 	snprintf(bsg_name, sizeof(bsg_name),
 		 "fc_host%d", shost->host_no);
 
-	q = __scsi_alloc_queue(shost, fc_bsg_host_handler);
+	q = __scsi_alloc_queue(shost, bsg_request_fn);
 	if (!q) {
-		printk(KERN_ERR "fc_host%d: bsg interface failed to "
-				"initialize - no request queue\n",
-				 shost->host_no);
+		dev_err(dev,
+			"fc_host%d: bsg interface failed to initialize - no request queue\n",
+			shost->host_no);
 		return -ENOMEM;
 	}
 
-	q->queuedata = shost;
-	queue_flag_set_unlocked(QUEUE_FLAG_BIDI, q);
-	blk_queue_softirq_done(q, fc_bsg_softirq_done);
-	blk_queue_rq_timed_out(q, fc_bsg_job_timeout);
-	blk_queue_rq_timeout(q, FC_DEFAULT_BSG_TIMEOUT);
-
-	err = bsg_register_queue(q, dev, bsg_name, NULL);
+	err = bsg_setup_queue(dev, q, bsg_name, fc_bsg_dispatch,
+				 i->f->dd_bsg_size);
 	if (err) {
-		printk(KERN_ERR "fc_host%d: bsg interface failed to "
-				"initialize - register queue\n",
-				shost->host_no);
+		dev_err(dev,
+			"fc_host%d: bsg interface failed to initialize - setup queue\n",
+			shost->host_no);
 		blk_cleanup_queue(q);
 		return err;
 	}
-
+	blk_queue_rq_timed_out(q, fc_bsg_job_timeout);
+	blk_queue_rq_timeout(q, FC_DEFAULT_BSG_TIMEOUT);
 	fc_host->rqst_q = q;
 	return 0;
 }
 
+static int fc_bsg_rport_prep(struct request_queue *q, struct request *req)
+{
+	struct fc_rport *rport = dev_to_rport(q->queuedata);
+
+	if (rport->port_state == FC_PORTSTATE_BLOCKED &&
+	    !(rport->flags & FC_RPORT_FAST_FAIL_TIMEDOUT))
+		return BLKPREP_DEFER;
+
+	if (rport->port_state != FC_PORTSTATE_ONLINE)
+		return BLKPREP_KILL;
+
+	return BLKPREP_OK;
+}
 
 /**
  * fc_bsg_rportadd - Create and add the bsg hooks so we can receive requests
@@ -4097,29 +3831,22 @@ fc_bsg_rportadd(struct Scsi_Host *shost, struct fc_rport *rport)
 	if (!i->f->bsg_request)
 		return -ENOTSUPP;
 
-	q = __scsi_alloc_queue(shost, fc_bsg_rport_handler);
+	q = __scsi_alloc_queue(shost, bsg_request_fn);
 	if (!q) {
-		printk(KERN_ERR "%s: bsg interface failed to "
-				"initialize - no request queue\n",
-				 dev->kobj.name);
+		dev_err(dev, "bsg interface failed to initialize - no request queue\n");
 		return -ENOMEM;
 	}
 
-	q->queuedata = rport;
-	queue_flag_set_unlocked(QUEUE_FLAG_BIDI, q);
-	blk_queue_softirq_done(q, fc_bsg_softirq_done);
-	blk_queue_rq_timed_out(q, fc_bsg_job_timeout);
-	blk_queue_rq_timeout(q, BLK_DEFAULT_SG_TIMEOUT);
-
-	err = bsg_register_queue(q, dev, NULL, NULL);
+	err = bsg_setup_queue(dev, q, NULL, fc_bsg_dispatch, i->f->dd_bsg_size);
 	if (err) {
-		printk(KERN_ERR "%s: bsg interface failed to "
-				"initialize - register queue\n",
-				 dev->kobj.name);
+		dev_err(dev, "failed to setup bsg queue\n");
 		blk_cleanup_queue(q);
 		return err;
 	}
 
+	blk_queue_prep_rq(q, fc_bsg_rport_prep);
+	blk_queue_rq_timed_out(q, fc_bsg_job_timeout);
+	blk_queue_rq_timeout(q, BLK_DEFAULT_SG_TIMEOUT);
 	rport->rqst_q = q;
 	return 0;
 }
diff --git a/drivers/scsi/scsi_transport_srp.c b/drivers/scsi/scsi_transport_srp.c
index e3cd3ece4412..b87a78673f65 100644
--- a/drivers/scsi/scsi_transport_srp.c
+++ b/drivers/scsi/scsi_transport_srp.c
@@ -24,7 +24,6 @@
 #include <linux/err.h>
 #include <linux/slab.h>
 #include <linux/string.h>
-#include <linux/delay.h>
 
 #include <scsi/scsi.h>
 #include <scsi/scsi_cmnd.h>
@@ -115,21 +114,12 @@ static DECLARE_TRANSPORT_CLASS(srp_host_class, "srp_host", srp_host_setup,
 static DECLARE_TRANSPORT_CLASS(srp_rport_class, "srp_remote_ports",
 			       NULL, NULL, NULL);
 
-#define SRP_PID(p) \
-	(p)->port_id[0], (p)->port_id[1], (p)->port_id[2], (p)->port_id[3], \
-	(p)->port_id[4], (p)->port_id[5], (p)->port_id[6], (p)->port_id[7], \
-	(p)->port_id[8], (p)->port_id[9], (p)->port_id[10], (p)->port_id[11], \
-	(p)->port_id[12], (p)->port_id[13], (p)->port_id[14], (p)->port_id[15]
-
-#define SRP_PID_FMT "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:" \
-	"%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x"
-
 static ssize_t
 show_srp_rport_id(struct device *dev, struct device_attribute *attr,
 		  char *buf)
 {
 	struct srp_rport *rport = transport_class_to_srp_rport(dev);
-	return sprintf(buf, SRP_PID_FMT "\n", SRP_PID(rport));
+	return sprintf(buf, "%16phC\n", rport->port_id);
 }
 
 static DEVICE_ATTR(port_id, S_IRUGO, show_srp_rport_id, NULL);
@@ -402,36 +392,6 @@ static void srp_reconnect_work(struct work_struct *work)
 	}
 }
 
-/**
- * scsi_request_fn_active() - number of kernel threads inside scsi_request_fn()
- * @shost: SCSI host for which to count the number of scsi_request_fn() callers.
- *
- * To do: add support for scsi-mq in this function.
- */
-static int scsi_request_fn_active(struct Scsi_Host *shost)
-{
-	struct scsi_device *sdev;
-	struct request_queue *q;
-	int request_fn_active = 0;
-
-	shost_for_each_device(sdev, shost) {
-		q = sdev->request_queue;
-
-		spin_lock_irq(q->queue_lock);
-		request_fn_active += q->request_fn_active;
-		spin_unlock_irq(q->queue_lock);
-	}
-
-	return request_fn_active;
-}
-
-/* Wait until ongoing shost->hostt->queuecommand() calls have finished. */
-static void srp_wait_for_queuecommand(struct Scsi_Host *shost)
-{
-	while (scsi_request_fn_active(shost))
-		msleep(20);
-}
-
 static void __rport_fail_io_fast(struct srp_rport *rport)
 {
 	struct Scsi_Host *shost = rport_to_shost(rport);
@@ -441,14 +401,17 @@ static void __rport_fail_io_fast(struct srp_rport *rport)
 
 	if (srp_rport_set_state(rport, SRP_RPORT_FAIL_FAST))
 		return;
+	/*
+	 * Call scsi_target_block() to wait for ongoing shost->queuecommand()
+	 * calls before invoking i->f->terminate_rport_io().
+	 */
+	scsi_target_block(rport->dev.parent);
 	scsi_target_unblock(rport->dev.parent, SDEV_TRANSPORT_OFFLINE);
 
 	/* Involve the LLD if possible to terminate all I/O on the rport. */
 	i = to_srp_internal(shost->transportt);
-	if (i->f->terminate_rport_io) {
-		srp_wait_for_queuecommand(shost);
+	if (i->f->terminate_rport_io)
 		i->f->terminate_rport_io(rport);
-	}
 }
 
 /**
@@ -576,7 +539,6 @@ int srp_reconnect_rport(struct srp_rport *rport)
 	if (res)
 		goto out;
 	scsi_target_block(&shost->shost_gendev);
-	srp_wait_for_queuecommand(shost);
 	res = rport->state != SRP_RPORT_LOST ? i->f->reconnect(rport) : -ENODEV;
 	pr_debug("%s (state %d): transport.reconnect() returned %d\n",
 		 dev_name(&shost->shost_gendev), rport->state, res);
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index 51e56296f465..1622e23138e0 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -93,6 +93,7 @@ MODULE_ALIAS_BLOCKDEV_MAJOR(SCSI_DISK15_MAJOR);
 MODULE_ALIAS_SCSI_DEVICE(TYPE_DISK);
 MODULE_ALIAS_SCSI_DEVICE(TYPE_MOD);
 MODULE_ALIAS_SCSI_DEVICE(TYPE_RBC);
+MODULE_ALIAS_SCSI_DEVICE(TYPE_ZBC);
 
 #if !defined(CONFIG_DEBUG_BLOCK_EXT_DEVT)
 #define SD_MINORS	16
@@ -163,7 +164,7 @@ cache_type_store(struct device *dev, struct device_attribute *attr,
 	static const char temp[] = "temporary ";
 	int len;
 
-	if (sdp->type != TYPE_DISK)
+	if (sdp->type != TYPE_DISK && sdp->type != TYPE_ZBC)
 		/* no cache control on RBC devices; theoretically they
 		 * can do it, but there's probably so many exceptions
 		 * it's not worth the risk */
@@ -262,7 +263,7 @@ allow_restart_store(struct device *dev, struct device_attribute *attr,
 	if (!capable(CAP_SYS_ADMIN))
 		return -EACCES;
 
-	if (sdp->type != TYPE_DISK)
+	if (sdp->type != TYPE_DISK && sdp->type != TYPE_ZBC)
 		return -EINVAL;
 
 	sdp->allow_restart = simple_strtoul(buf, NULL, 10);
@@ -392,6 +393,11 @@ provisioning_mode_store(struct device *dev, struct device_attribute *attr,
 	if (!capable(CAP_SYS_ADMIN))
 		return -EACCES;
 
+	if (sd_is_zoned(sdkp)) {
+		sd_config_discard(sdkp, SD_LBP_DISABLE);
+		return count;
+	}
+
 	if (sdp->type != TYPE_DISK)
 		return -EINVAL;
 
@@ -459,7 +465,7 @@ max_write_same_blocks_store(struct device *dev, struct device_attribute *attr,
 	if (!capable(CAP_SYS_ADMIN))
 		return -EACCES;
 
-	if (sdp->type != TYPE_DISK)
+	if (sdp->type != TYPE_DISK && sdp->type != TYPE_ZBC)
 		return -EINVAL;
 
 	err = kstrtoul(buf, 10, &max);
@@ -710,7 +716,6 @@ static int sd_setup_discard_cmnd(struct scsi_cmnd *cmd)
 	struct scsi_disk *sdkp = scsi_disk(rq->rq_disk);
 	sector_t sector = blk_rq_pos(rq);
 	unsigned int nr_sectors = blk_rq_sectors(rq);
-	unsigned int nr_bytes = blk_rq_bytes(rq);
 	unsigned int len;
 	int ret;
 	char *buf;
@@ -766,24 +771,19 @@ static int sd_setup_discard_cmnd(struct scsi_cmnd *cmd)
 		goto out;
 	}
 
-	rq->completion_data = page;
 	rq->timeout = SD_TIMEOUT;
 
 	cmd->transfersize = len;
 	cmd->allowed = SD_MAX_RETRIES;
 
-	/*
-	 * Initially __data_len is set to the amount of data that needs to be
-	 * transferred to the target. This amount depends on whether WRITE SAME
-	 * or UNMAP is being used. After the scatterlist has been mapped by
-	 * scsi_init_io() we set __data_len to the size of the area to be
-	 * discarded on disk. This allows us to report completion on the full
-	 * amount of blocks described by the request.
-	 */
-	blk_add_request_payload(rq, page, 0, len);
-	ret = scsi_init_io(cmd);
-	rq->__data_len = nr_bytes;
+	rq->special_vec.bv_page = page;
+	rq->special_vec.bv_offset = 0;
+	rq->special_vec.bv_len = len;
+
+	rq->rq_flags |= RQF_SPECIAL_PAYLOAD;
+	rq->resid_len = len;
 
+	ret = scsi_init_io(cmd);
 out:
 	if (ret != BLKPREP_OK)
 		__free_page(page);
@@ -844,6 +844,12 @@ static int sd_setup_write_same_cmnd(struct scsi_cmnd *cmd)
 
 	BUG_ON(bio_offset(bio) || bio_iovec(bio).bv_len != sdp->sector_size);
 
+	if (sd_is_zoned(sdkp)) {
+		ret = sd_zbc_setup_write_cmnd(cmd);
+		if (ret != BLKPREP_OK)
+			return ret;
+	}
+
 	sector >>= ilog2(sdp->sector_size) - 9;
 	nr_sectors >>= ilog2(sdp->sector_size) - 9;
 
@@ -901,19 +907,25 @@ static int sd_setup_read_write_cmnd(struct scsi_cmnd *SCpnt)
 	struct request *rq = SCpnt->request;
 	struct scsi_device *sdp = SCpnt->device;
 	struct gendisk *disk = rq->rq_disk;
-	struct scsi_disk *sdkp;
+	struct scsi_disk *sdkp = scsi_disk(disk);
 	sector_t block = blk_rq_pos(rq);
 	sector_t threshold;
 	unsigned int this_count = blk_rq_sectors(rq);
 	unsigned int dif, dix;
+	bool zoned_write = sd_is_zoned(sdkp) && rq_data_dir(rq) == WRITE;
 	int ret;
 	unsigned char protect;
 
+	if (zoned_write) {
+		ret = sd_zbc_setup_write_cmnd(SCpnt);
+		if (ret != BLKPREP_OK)
+			return ret;
+	}
+
 	ret = scsi_init_io(SCpnt);
 	if (ret != BLKPREP_OK)
 		goto out;
 	SCpnt = rq->special;
-	sdkp = scsi_disk(disk);
 
 	/* from here on until we're complete, any goto out
 	 * is used for a killable error condition */
@@ -1013,8 +1025,7 @@ static int sd_setup_read_write_cmnd(struct scsi_cmnd *SCpnt)
 	} else if (rq_data_dir(rq) == READ) {
 		SCpnt->cmnd[0] = READ_6;
 	} else {
-		scmd_printk(KERN_ERR, SCpnt, "Unknown command %llu,%llx\n",
-			    req_op(rq), (unsigned long long) rq->cmd_flags);
+		scmd_printk(KERN_ERR, SCpnt, "Unknown command %d\n", req_op(rq));
 		goto out;
 	}
 
@@ -1132,6 +1143,9 @@ static int sd_setup_read_write_cmnd(struct scsi_cmnd *SCpnt)
 	 */
 	ret = BLKPREP_OK;
  out:
+	if (zoned_write && ret != BLKPREP_OK)
+		sd_zbc_cancel_write_cmnd(SCpnt);
+
 	return ret;
 }
 
@@ -1149,6 +1163,10 @@ static int sd_init_command(struct scsi_cmnd *cmd)
 	case REQ_OP_READ:
 	case REQ_OP_WRITE:
 		return sd_setup_read_write_cmnd(cmd);
+	case REQ_OP_ZONE_REPORT:
+		return sd_zbc_setup_report_cmnd(cmd);
+	case REQ_OP_ZONE_RESET:
+		return sd_zbc_setup_reset_cmnd(cmd);
 	default:
 		BUG();
 	}
@@ -1158,8 +1176,8 @@ static void sd_uninit_command(struct scsi_cmnd *SCpnt)
 {
 	struct request *rq = SCpnt->request;
 
-	if (req_op(rq) == REQ_OP_DISCARD)
-		__free_page(rq->completion_data);
+	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
+		__free_page(rq->special_vec.bv_page);
 
 	if (SCpnt->cmnd != rq->cmd) {
 		mempool_free(SCpnt->cmnd, sd_cdb_pool);
@@ -1495,7 +1513,7 @@ static int sd_sync_cache(struct scsi_disk *sdkp)
 		 */
 		res = scsi_execute_req_flags(sdp, cmd, DMA_NONE, NULL, 0,
 					     &sshdr, timeout, SD_MAX_RETRIES,
-					     NULL, REQ_PM);
+					     NULL, 0, RQF_PM);
 		if (res == 0)
 			break;
 	}
@@ -1780,7 +1798,10 @@ static int sd_done(struct scsi_cmnd *SCpnt)
 	unsigned char op = SCpnt->cmnd[0];
 	unsigned char unmap = SCpnt->cmnd[1] & 8;
 
-	if (req_op(req) == REQ_OP_DISCARD || req_op(req) == REQ_OP_WRITE_SAME) {
+	switch (req_op(req)) {
+	case REQ_OP_DISCARD:
+	case REQ_OP_WRITE_SAME:
+	case REQ_OP_ZONE_RESET:
 		if (!result) {
 			good_bytes = blk_rq_bytes(req);
 			scsi_set_resid(SCpnt, 0);
@@ -1788,6 +1809,17 @@ static int sd_done(struct scsi_cmnd *SCpnt)
 			good_bytes = 0;
 			scsi_set_resid(SCpnt, blk_rq_bytes(req));
 		}
+		break;
+	case REQ_OP_ZONE_REPORT:
+		if (!result) {
+			good_bytes = scsi_bufflen(SCpnt)
+				- scsi_get_resid(SCpnt);
+			scsi_set_resid(SCpnt, 0);
+		} else {
+			good_bytes = 0;
+			scsi_set_resid(SCpnt, blk_rq_bytes(req));
+		}
+		break;
 	}
 
 	if (result) {
@@ -1840,7 +1872,7 @@ static int sd_done(struct scsi_cmnd *SCpnt)
 
 					good_bytes = 0;
 					req->__data_len = blk_rq_bytes(req);
-					req->cmd_flags |= REQ_QUIET;
+					req->rq_flags |= RQF_QUIET;
 				}
 			}
 		}
@@ -1848,7 +1880,11 @@ static int sd_done(struct scsi_cmnd *SCpnt)
 	default:
 		break;
 	}
+
  out:
+	if (sd_is_zoned(sdkp))
+		sd_zbc_complete(SCpnt, good_bytes, &sshdr);
+
 	SCSI_LOG_HLCOMPLETE(1, scmd_printk(KERN_INFO, SCpnt,
 					   "sd_done: completed %d of %d bytes\n",
 					   good_bytes, scsi_bufflen(SCpnt)));
@@ -1983,7 +2019,6 @@ sd_spinup_disk(struct scsi_disk *sdkp)
 	}
 }
 
-
 /*
  * Determine whether disk supports Data Integrity Field.
  */
@@ -2133,6 +2168,9 @@ static int read_capacity_16(struct scsi_disk *sdkp, struct scsi_device *sdp,
 	/* Logical blocks per physical block exponent */
 	sdkp->physical_block_size = (1 << (buffer[13] & 0xf)) * sector_size;
 
+	/* RC basis */
+	sdkp->rc_basis = (buffer[12] >> 4) & 0x3;
+
 	/* Lowest aligned logical block */
 	alignment = ((buffer[14] & 0x3f) << 8 | buffer[15]) * sector_size;
 	blk_queue_alignment_offset(sdp->request_queue, alignment);
@@ -2242,7 +2280,6 @@ sd_read_capacity(struct scsi_disk *sdkp, unsigned char *buffer)
 {
 	int sector_size;
 	struct scsi_device *sdp = sdkp->device;
-	sector_t old_capacity = sdkp->capacity;
 
 	if (sd_try_rc16_first(sdp)) {
 		sector_size = read_capacity_16(sdkp, sdp, buffer);
@@ -2323,35 +2360,44 @@ got_data:
 		sector_size = 512;
 	}
 	blk_queue_logical_block_size(sdp->request_queue, sector_size);
+	blk_queue_physical_block_size(sdp->request_queue,
+				      sdkp->physical_block_size);
+	sdkp->device->sector_size = sector_size;
 
-	{
-		char cap_str_2[10], cap_str_10[10];
+	if (sdkp->capacity > 0xffffffff)
+		sdp->use_16_for_rw = 1;
 
-		string_get_size(sdkp->capacity, sector_size,
-				STRING_UNITS_2, cap_str_2, sizeof(cap_str_2));
-		string_get_size(sdkp->capacity, sector_size,
-				STRING_UNITS_10, cap_str_10,
-				sizeof(cap_str_10));
+}
 
-		if (sdkp->first_scan || old_capacity != sdkp->capacity) {
-			sd_printk(KERN_NOTICE, sdkp,
-				  "%llu %d-byte logical blocks: (%s/%s)\n",
-				  (unsigned long long)sdkp->capacity,
-				  sector_size, cap_str_10, cap_str_2);
+/*
+ * Print disk capacity
+ */
+static void
+sd_print_capacity(struct scsi_disk *sdkp,
+		  sector_t old_capacity)
+{
+	int sector_size = sdkp->device->sector_size;
+	char cap_str_2[10], cap_str_10[10];
 
-			if (sdkp->physical_block_size != sector_size)
-				sd_printk(KERN_NOTICE, sdkp,
-					  "%u-byte physical blocks\n",
-					  sdkp->physical_block_size);
-		}
-	}
+	string_get_size(sdkp->capacity, sector_size,
+			STRING_UNITS_2, cap_str_2, sizeof(cap_str_2));
+	string_get_size(sdkp->capacity, sector_size,
+			STRING_UNITS_10, cap_str_10,
+			sizeof(cap_str_10));
 
-	if (sdkp->capacity > 0xffffffff)
-		sdp->use_16_for_rw = 1;
+	if (sdkp->first_scan || old_capacity != sdkp->capacity) {
+		sd_printk(KERN_NOTICE, sdkp,
+			  "%llu %d-byte logical blocks: (%s/%s)\n",
+			  (unsigned long long)sdkp->capacity,
+			  sector_size, cap_str_10, cap_str_2);
 
-	blk_queue_physical_block_size(sdp->request_queue,
-				      sdkp->physical_block_size);
-	sdkp->device->sector_size = sector_size;
+		if (sdkp->physical_block_size != sector_size)
+			sd_printk(KERN_NOTICE, sdkp,
+				  "%u-byte physical blocks\n",
+				  sdkp->physical_block_size);
+
+		sd_zbc_print_zones(sdkp);
+	}
 }
 
 /* called with buffer of length 512 */
@@ -2419,9 +2465,7 @@ sd_read_write_protect_flag(struct scsi_disk *sdkp, unsigned char *buffer)
 		if (sdkp->first_scan || old_wp != sdkp->write_prot) {
 			sd_printk(KERN_NOTICE, sdkp, "Write Protect is %s\n",
 				  sdkp->write_prot ? "on" : "off");
-			sd_printk(KERN_DEBUG, sdkp,
-				  "Mode Sense: %02x %02x %02x %02x\n",
-				  buffer[0], buffer[1], buffer[2], buffer[3]);
+			sd_printk(KERN_DEBUG, sdkp, "Mode Sense: %4ph\n", buffer);
 		}
 	}
 }
@@ -2613,7 +2657,7 @@ static void sd_read_app_tag_own(struct scsi_disk *sdkp, unsigned char *buffer)
 	struct scsi_mode_data data;
 	struct scsi_sense_hdr sshdr;
 
-	if (sdp->type != TYPE_DISK)
+	if (sdp->type != TYPE_DISK && sdp->type != TYPE_ZBC)
 		return;
 
 	if (sdkp->protection_type == 0)
@@ -2720,6 +2764,7 @@ static void sd_read_block_limits(struct scsi_disk *sdkp)
  */
 static void sd_read_block_characteristics(struct scsi_disk *sdkp)
 {
+	struct request_queue *q = sdkp->disk->queue;
 	unsigned char *buffer;
 	u16 rot;
 	const int vpd_len = 64;
@@ -2734,10 +2779,21 @@ static void sd_read_block_characteristics(struct scsi_disk *sdkp)
 	rot = get_unaligned_be16(&buffer[4]);
 
 	if (rot == 1) {
-		queue_flag_set_unlocked(QUEUE_FLAG_NONROT, sdkp->disk->queue);
-		queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, sdkp->disk->queue);
+		queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
+		queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, q);
 	}
 
+	sdkp->zoned = (buffer[8] >> 4) & 3;
+	if (sdkp->zoned == 1)
+		q->limits.zoned = BLK_ZONED_HA;
+	else if (sdkp->device->type == TYPE_ZBC)
+		q->limits.zoned = BLK_ZONED_HM;
+	else
+		q->limits.zoned = BLK_ZONED_NONE;
+	if (blk_queue_is_zoned(q) && sdkp->first_scan)
+		sd_printk(KERN_NOTICE, sdkp, "Host-%s zoned block device\n",
+		      q->limits.zoned == BLK_ZONED_HM ? "managed" : "aware");
+
  out:
 	kfree(buffer);
 }
@@ -2809,6 +2865,7 @@ static int sd_revalidate_disk(struct gendisk *disk)
 	struct scsi_disk *sdkp = scsi_disk(disk);
 	struct scsi_device *sdp = sdkp->device;
 	struct request_queue *q = sdkp->disk->queue;
+	sector_t old_capacity = sdkp->capacity;
 	unsigned char *buffer;
 	unsigned int dev_max, rw_max;
 
@@ -2842,8 +2899,11 @@ static int sd_revalidate_disk(struct gendisk *disk)
 			sd_read_block_provisioning(sdkp);
 			sd_read_block_limits(sdkp);
 			sd_read_block_characteristics(sdkp);
+			sd_zbc_read_zones(sdkp, buffer);
 		}
 
+		sd_print_capacity(sdkp, old_capacity);
+
 		sd_read_write_protect_flag(sdkp, buffer);
 		sd_read_cache_type(sdkp, buffer);
 		sd_read_app_tag_own(sdkp, buffer);
@@ -3041,9 +3101,16 @@ static int sd_probe(struct device *dev)
 
 	scsi_autopm_get_device(sdp);
 	error = -ENODEV;
-	if (sdp->type != TYPE_DISK && sdp->type != TYPE_MOD && sdp->type != TYPE_RBC)
+	if (sdp->type != TYPE_DISK &&
+	    sdp->type != TYPE_ZBC &&
+	    sdp->type != TYPE_MOD &&
+	    sdp->type != TYPE_RBC)
 		goto out;
 
+#ifndef CONFIG_BLK_DEV_ZONED
+	if (sdp->type == TYPE_ZBC)
+		goto out;
+#endif
 	SCSI_LOG_HLQUEUE(3, sdev_printk(KERN_INFO, sdp,
 					"sd_probe\n"));
 
@@ -3147,6 +3214,8 @@ static int sd_remove(struct device *dev)
 	del_gendisk(sdkp->disk);
 	sd_shutdown(dev);
 
+	sd_zbc_remove(sdkp);
+
 	blk_register_region(devt, SD_MINORS, NULL,
 			    sd_default_probe, NULL, NULL);
 
@@ -3200,7 +3269,7 @@ static int sd_start_stop_device(struct scsi_disk *sdkp, int start)
 		return -ENODEV;
 
 	res = scsi_execute_req_flags(sdp, cmd, DMA_NONE, NULL, 0, &sshdr,
-			       SD_TIMEOUT, SD_MAX_RETRIES, NULL, REQ_PM);
+			       SD_TIMEOUT, SD_MAX_RETRIES, NULL, 0, RQF_PM);
 	if (res) {
 		sd_print_result(sdkp, "Start/Stop Unit failed", res);
 		if (driver_byte(res) & DRIVER_SENSE)
diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h
index c8d986368da9..4dac35e96a75 100644
--- a/drivers/scsi/sd.h
+++ b/drivers/scsi/sd.h
@@ -64,6 +64,15 @@ struct scsi_disk {
 	struct scsi_device *device;
 	struct device	dev;
 	struct gendisk	*disk;
+#ifdef CONFIG_BLK_DEV_ZONED
+	unsigned int	nr_zones;
+	unsigned int	zone_blocks;
+	unsigned int	zone_shift;
+	unsigned long	*zones_wlock;
+	unsigned int	zones_optimal_open;
+	unsigned int	zones_optimal_nonseq;
+	unsigned int	zones_max_open;
+#endif
 	atomic_t	openers;
 	sector_t	capacity;	/* size in logical blocks */
 	u32		max_xfer_blocks;
@@ -94,6 +103,9 @@ struct scsi_disk {
 	unsigned	lbpvpd : 1;
 	unsigned	ws10 : 1;
 	unsigned	ws16 : 1;
+	unsigned	rc_basis: 2;
+	unsigned	zoned: 2;
+	unsigned	urswrz : 1;
 };
 #define to_scsi_disk(obj) container_of(obj,struct scsi_disk,dev)
 
@@ -156,6 +168,11 @@ static inline unsigned int logical_to_bytes(struct scsi_device *sdev, sector_t b
 	return blocks * sdev->sector_size;
 }
 
+static inline sector_t sectors_to_logical(struct scsi_device *sdev, sector_t sector)
+{
+	return sector >> (ilog2(sdev->sector_size) - 9);
+}
+
 /*
  * Look up the DIX operation based on whether the command is read or
  * write and whether dix and dif are enabled.
@@ -239,4 +256,57 @@ static inline void sd_dif_complete(struct scsi_cmnd *cmd, unsigned int a)
 
 #endif /* CONFIG_BLK_DEV_INTEGRITY */
 
+static inline int sd_is_zoned(struct scsi_disk *sdkp)
+{
+	return sdkp->zoned == 1 || sdkp->device->type == TYPE_ZBC;
+}
+
+#ifdef CONFIG_BLK_DEV_ZONED
+
+extern int sd_zbc_read_zones(struct scsi_disk *sdkp, unsigned char *buffer);
+extern void sd_zbc_remove(struct scsi_disk *sdkp);
+extern void sd_zbc_print_zones(struct scsi_disk *sdkp);
+extern int sd_zbc_setup_write_cmnd(struct scsi_cmnd *cmd);
+extern void sd_zbc_cancel_write_cmnd(struct scsi_cmnd *cmd);
+extern int sd_zbc_setup_report_cmnd(struct scsi_cmnd *cmd);
+extern int sd_zbc_setup_reset_cmnd(struct scsi_cmnd *cmd);
+extern void sd_zbc_complete(struct scsi_cmnd *cmd, unsigned int good_bytes,
+			    struct scsi_sense_hdr *sshdr);
+
+#else /* CONFIG_BLK_DEV_ZONED */
+
+static inline int sd_zbc_read_zones(struct scsi_disk *sdkp,
+				    unsigned char *buf)
+{
+	return 0;
+}
+
+static inline void sd_zbc_remove(struct scsi_disk *sdkp) {}
+
+static inline void sd_zbc_print_zones(struct scsi_disk *sdkp) {}
+
+static inline int sd_zbc_setup_write_cmnd(struct scsi_cmnd *cmd)
+{
+	/* Let the drive fail requests */
+	return BLKPREP_OK;
+}
+
+static inline void sd_zbc_cancel_write_cmnd(struct scsi_cmnd *cmd) {}
+
+static inline int sd_zbc_setup_report_cmnd(struct scsi_cmnd *cmd)
+{
+	return BLKPREP_INVALID;
+}
+
+static inline int sd_zbc_setup_reset_cmnd(struct scsi_cmnd *cmd)
+{
+	return BLKPREP_INVALID;
+}
+
+static inline void sd_zbc_complete(struct scsi_cmnd *cmd,
+				   unsigned int good_bytes,
+				   struct scsi_sense_hdr *sshdr) {}
+
+#endif /* CONFIG_BLK_DEV_ZONED */
+
 #endif /* _SCSI_DISK_H */
diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c
new file mode 100644
index 000000000000..92620c8ea8ad
--- /dev/null
+++ b/drivers/scsi/sd_zbc.c
@@ -0,0 +1,648 @@
+/*
+ * SCSI Zoned Block commands
+ *
+ * Copyright (C) 2014-2015 SUSE Linux GmbH
+ * Written by: Hannes Reinecke <hare@suse.de>
+ * Modified by: Damien Le Moal <damien.lemoal@hgst.com>
+ * Modified by: Shaun Tancheff <shaun.tancheff@seagate.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.  If not, write to
+ * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
+ * USA.
+ *
+ */
+
+#include <linux/blkdev.h>
+
+#include <asm/unaligned.h>
+
+#include <scsi/scsi.h>
+#include <scsi/scsi_cmnd.h>
+#include <scsi/scsi_dbg.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_driver.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_eh.h>
+
+#include "sd.h"
+#include "scsi_priv.h"
+
+enum zbc_zone_type {
+	ZBC_ZONE_TYPE_CONV = 0x1,
+	ZBC_ZONE_TYPE_SEQWRITE_REQ,
+	ZBC_ZONE_TYPE_SEQWRITE_PREF,
+	ZBC_ZONE_TYPE_RESERVED,
+};
+
+enum zbc_zone_cond {
+	ZBC_ZONE_COND_NO_WP,
+	ZBC_ZONE_COND_EMPTY,
+	ZBC_ZONE_COND_IMP_OPEN,
+	ZBC_ZONE_COND_EXP_OPEN,
+	ZBC_ZONE_COND_CLOSED,
+	ZBC_ZONE_COND_READONLY = 0xd,
+	ZBC_ZONE_COND_FULL,
+	ZBC_ZONE_COND_OFFLINE,
+};
+
+/**
+ * Convert a zone descriptor to a zone struct.
+ */
+static void sd_zbc_parse_report(struct scsi_disk *sdkp,
+				u8 *buf,
+				struct blk_zone *zone)
+{
+	struct scsi_device *sdp = sdkp->device;
+
+	memset(zone, 0, sizeof(struct blk_zone));
+
+	zone->type = buf[0] & 0x0f;
+	zone->cond = (buf[1] >> 4) & 0xf;
+	if (buf[1] & 0x01)
+		zone->reset = 1;
+	if (buf[1] & 0x02)
+		zone->non_seq = 1;
+
+	zone->len = logical_to_sectors(sdp, get_unaligned_be64(&buf[8]));
+	zone->start = logical_to_sectors(sdp, get_unaligned_be64(&buf[16]));
+	zone->wp = logical_to_sectors(sdp, get_unaligned_be64(&buf[24]));
+	if (zone->type != ZBC_ZONE_TYPE_CONV &&
+	    zone->cond == ZBC_ZONE_COND_FULL)
+		zone->wp = zone->start + zone->len;
+}
+
+/**
+ * Issue a REPORT ZONES scsi command.
+ */
+static int sd_zbc_report_zones(struct scsi_disk *sdkp, unsigned char *buf,
+			       unsigned int buflen, sector_t lba)
+{
+	struct scsi_device *sdp = sdkp->device;
+	const int timeout = sdp->request_queue->rq_timeout;
+	struct scsi_sense_hdr sshdr;
+	unsigned char cmd[16];
+	unsigned int rep_len;
+	int result;
+
+	memset(cmd, 0, 16);
+	cmd[0] = ZBC_IN;
+	cmd[1] = ZI_REPORT_ZONES;
+	put_unaligned_be64(lba, &cmd[2]);
+	put_unaligned_be32(buflen, &cmd[10]);
+	memset(buf, 0, buflen);
+
+	result = scsi_execute_req(sdp, cmd, DMA_FROM_DEVICE,
+				  buf, buflen, &sshdr,
+				  timeout, SD_MAX_RETRIES, NULL);
+	if (result) {
+		sd_printk(KERN_ERR, sdkp,
+			  "REPORT ZONES lba %llu failed with %d/%d\n",
+			  (unsigned long long)lba,
+			  host_byte(result), driver_byte(result));
+		return -EIO;
+	}
+
+	rep_len = get_unaligned_be32(&buf[0]);
+	if (rep_len < 64) {
+		sd_printk(KERN_ERR, sdkp,
+			  "REPORT ZONES report invalid length %u\n",
+			  rep_len);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+int sd_zbc_setup_report_cmnd(struct scsi_cmnd *cmd)
+{
+	struct request *rq = cmd->request;
+	struct scsi_disk *sdkp = scsi_disk(rq->rq_disk);
+	sector_t lba, sector = blk_rq_pos(rq);
+	unsigned int nr_bytes = blk_rq_bytes(rq);
+	int ret;
+
+	WARN_ON(nr_bytes == 0);
+
+	if (!sd_is_zoned(sdkp))
+		/* Not a zoned device */
+		return BLKPREP_KILL;
+
+	ret = scsi_init_io(cmd);
+	if (ret != BLKPREP_OK)
+		return ret;
+
+	cmd->cmd_len = 16;
+	memset(cmd->cmnd, 0, cmd->cmd_len);
+	cmd->cmnd[0] = ZBC_IN;
+	cmd->cmnd[1] = ZI_REPORT_ZONES;
+	lba = sectors_to_logical(sdkp->device, sector);
+	put_unaligned_be64(lba, &cmd->cmnd[2]);
+	put_unaligned_be32(nr_bytes, &cmd->cmnd[10]);
+	/* Do partial report for speeding things up */
+	cmd->cmnd[14] = ZBC_REPORT_ZONE_PARTIAL;
+
+	cmd->sc_data_direction = DMA_FROM_DEVICE;
+	cmd->sdb.length = nr_bytes;
+	cmd->transfersize = sdkp->device->sector_size;
+	cmd->allowed = 0;
+
+	/*
+	 * Report may return less bytes than requested. Make sure
+	 * to report completion on the entire initial request.
+	 */
+	rq->__data_len = nr_bytes;
+
+	return BLKPREP_OK;
+}
+
+static void sd_zbc_report_zones_complete(struct scsi_cmnd *scmd,
+					 unsigned int good_bytes)
+{
+	struct request *rq = scmd->request;
+	struct scsi_disk *sdkp = scsi_disk(rq->rq_disk);
+	struct sg_mapping_iter miter;
+	struct blk_zone_report_hdr hdr;
+	struct blk_zone zone;
+	unsigned int offset, bytes = 0;
+	unsigned long flags;
+	u8 *buf;
+
+	if (good_bytes < 64)
+		return;
+
+	memset(&hdr, 0, sizeof(struct blk_zone_report_hdr));
+
+	sg_miter_start(&miter, scsi_sglist(scmd), scsi_sg_count(scmd),
+		       SG_MITER_TO_SG | SG_MITER_ATOMIC);
+
+	local_irq_save(flags);
+	while (sg_miter_next(&miter) && bytes < good_bytes) {
+
+		buf = miter.addr;
+		offset = 0;
+
+		if (bytes == 0) {
+			/* Set the report header */
+			hdr.nr_zones = min_t(unsigned int,
+					 (good_bytes - 64) / 64,
+					 get_unaligned_be32(&buf[0]) / 64);
+			memcpy(buf, &hdr, sizeof(struct blk_zone_report_hdr));
+			offset += 64;
+			bytes += 64;
+		}
+
+		/* Parse zone descriptors */
+		while (offset < miter.length && hdr.nr_zones) {
+			WARN_ON(offset > miter.length);
+			buf = miter.addr + offset;
+			sd_zbc_parse_report(sdkp, buf, &zone);
+			memcpy(buf, &zone, sizeof(struct blk_zone));
+			offset += 64;
+			bytes += 64;
+			hdr.nr_zones--;
+		}
+
+		if (!hdr.nr_zones)
+			break;
+
+	}
+	sg_miter_stop(&miter);
+	local_irq_restore(flags);
+}
+
+static inline sector_t sd_zbc_zone_sectors(struct scsi_disk *sdkp)
+{
+	return logical_to_sectors(sdkp->device, sdkp->zone_blocks);
+}
+
+static inline unsigned int sd_zbc_zone_no(struct scsi_disk *sdkp,
+					  sector_t sector)
+{
+	return sectors_to_logical(sdkp->device, sector) >> sdkp->zone_shift;
+}
+
+int sd_zbc_setup_reset_cmnd(struct scsi_cmnd *cmd)
+{
+	struct request *rq = cmd->request;
+	struct scsi_disk *sdkp = scsi_disk(rq->rq_disk);
+	sector_t sector = blk_rq_pos(rq);
+	sector_t block = sectors_to_logical(sdkp->device, sector);
+	unsigned int zno = block >> sdkp->zone_shift;
+
+	if (!sd_is_zoned(sdkp))
+		/* Not a zoned device */
+		return BLKPREP_KILL;
+
+	if (sdkp->device->changed)
+		return BLKPREP_KILL;
+
+	if (sector & (sd_zbc_zone_sectors(sdkp) - 1))
+		/* Unaligned request */
+		return BLKPREP_KILL;
+
+	/* Do not allow concurrent reset and writes */
+	if (sdkp->zones_wlock &&
+	    test_and_set_bit(zno, sdkp->zones_wlock))
+		return BLKPREP_DEFER;
+
+	cmd->cmd_len = 16;
+	memset(cmd->cmnd, 0, cmd->cmd_len);
+	cmd->cmnd[0] = ZBC_OUT;
+	cmd->cmnd[1] = ZO_RESET_WRITE_POINTER;
+	put_unaligned_be64(block, &cmd->cmnd[2]);
+
+	rq->timeout = SD_TIMEOUT;
+	cmd->sc_data_direction = DMA_NONE;
+	cmd->transfersize = 0;
+	cmd->allowed = 0;
+
+	return BLKPREP_OK;
+}
+
+int sd_zbc_setup_write_cmnd(struct scsi_cmnd *cmd)
+{
+	struct request *rq = cmd->request;
+	struct scsi_disk *sdkp = scsi_disk(rq->rq_disk);
+	sector_t sector = blk_rq_pos(rq);
+	sector_t zone_sectors = sd_zbc_zone_sectors(sdkp);
+	unsigned int zno = sd_zbc_zone_no(sdkp, sector);
+
+	/*
+	 * Note: Checks of the alignment of the write command on
+	 * logical blocks is done in sd.c
+	 */
+
+	/* Do not allow zone boundaries crossing on host-managed drives */
+	if (blk_queue_zoned_model(sdkp->disk->queue) == BLK_ZONED_HM &&
+	    (sector & (zone_sectors - 1)) + blk_rq_sectors(rq) > zone_sectors)
+		return BLKPREP_KILL;
+
+	/*
+	 * Do not issue more than one write at a time per
+	 * zone. This solves write ordering problems due to
+	 * the unlocking of the request queue in the dispatch
+	 * path in the non scsi-mq case. For scsi-mq, this
+	 * also avoids potential write reordering when multiple
+	 * threads running on different CPUs write to the same
+	 * zone (with a synchronized sequential pattern).
+	 */
+	if (sdkp->zones_wlock &&
+	    test_and_set_bit(zno, sdkp->zones_wlock))
+		return BLKPREP_DEFER;
+
+	return BLKPREP_OK;
+}
+
+static void sd_zbc_unlock_zone(struct request *rq)
+{
+	struct scsi_disk *sdkp = scsi_disk(rq->rq_disk);
+
+	if (sdkp->zones_wlock) {
+		unsigned int zno = sd_zbc_zone_no(sdkp, blk_rq_pos(rq));
+		WARN_ON_ONCE(!test_bit(zno, sdkp->zones_wlock));
+		clear_bit_unlock(zno, sdkp->zones_wlock);
+		smp_mb__after_atomic();
+	}
+}
+
+void sd_zbc_cancel_write_cmnd(struct scsi_cmnd *cmd)
+{
+	sd_zbc_unlock_zone(cmd->request);
+}
+
+void sd_zbc_complete(struct scsi_cmnd *cmd,
+		     unsigned int good_bytes,
+		     struct scsi_sense_hdr *sshdr)
+{
+	int result = cmd->result;
+	struct request *rq = cmd->request;
+
+	switch (req_op(rq)) {
+	case REQ_OP_WRITE:
+	case REQ_OP_WRITE_SAME:
+	case REQ_OP_ZONE_RESET:
+
+		/* Unlock the zone */
+		sd_zbc_unlock_zone(rq);
+
+		if (!result ||
+		    sshdr->sense_key != ILLEGAL_REQUEST)
+			break;
+
+		switch (sshdr->asc) {
+		case 0x24:
+			/*
+			 * INVALID FIELD IN CDB error: For a zone reset,
+			 * this means that a reset of a conventional
+			 * zone was attempted. Nothing to worry about in
+			 * this case, so be quiet about the error.
+			 */
+			if (req_op(rq) == REQ_OP_ZONE_RESET)
+				rq->rq_flags |= RQF_QUIET;
+			break;
+		case 0x21:
+			/*
+			 * INVALID ADDRESS FOR WRITE error: It is unlikely that
+			 * retrying write requests failed with any kind of
+			 * alignement error will result in success. So don't.
+			 */
+			cmd->allowed = 0;
+			break;
+		}
+
+		break;
+
+	case REQ_OP_ZONE_REPORT:
+
+		if (!result)
+			sd_zbc_report_zones_complete(cmd, good_bytes);
+		break;
+
+	}
+}
+
+/**
+ * Read zoned block device characteristics (VPD page B6).
+ */
+static int sd_zbc_read_zoned_characteristics(struct scsi_disk *sdkp,
+					     unsigned char *buf)
+{
+
+	if (scsi_get_vpd_page(sdkp->device, 0xb6, buf, 64)) {
+		sd_printk(KERN_NOTICE, sdkp,
+			  "Unconstrained-read check failed\n");
+		return -ENODEV;
+	}
+
+	if (sdkp->device->type != TYPE_ZBC) {
+		/* Host-aware */
+		sdkp->urswrz = 1;
+		sdkp->zones_optimal_open = get_unaligned_be64(&buf[8]);
+		sdkp->zones_optimal_nonseq = get_unaligned_be64(&buf[12]);
+		sdkp->zones_max_open = 0;
+	} else {
+		/* Host-managed */
+		sdkp->urswrz = buf[4] & 1;
+		sdkp->zones_optimal_open = 0;
+		sdkp->zones_optimal_nonseq = 0;
+		sdkp->zones_max_open = get_unaligned_be64(&buf[16]);
+	}
+
+	return 0;
+}
+
+/**
+ * Check reported capacity.
+ */
+static int sd_zbc_check_capacity(struct scsi_disk *sdkp,
+				 unsigned char *buf)
+{
+	sector_t lba;
+	int ret;
+
+	if (sdkp->rc_basis != 0)
+		return 0;
+
+	/* Do a report zone to get the maximum LBA to check capacity */
+	ret = sd_zbc_report_zones(sdkp, buf, SD_BUF_SIZE, 0);
+	if (ret)
+		return ret;
+
+	/* The max_lba field is the capacity of this device */
+	lba = get_unaligned_be64(&buf[8]);
+	if (lba + 1 == sdkp->capacity)
+		return 0;
+
+	if (sdkp->first_scan)
+		sd_printk(KERN_WARNING, sdkp,
+			  "Changing capacity from %llu to max LBA+1 %llu\n",
+			  (unsigned long long)sdkp->capacity,
+			  (unsigned long long)lba + 1);
+	sdkp->capacity = lba + 1;
+
+	return 0;
+}
+
+#define SD_ZBC_BUF_SIZE 131072
+
+static int sd_zbc_check_zone_size(struct scsi_disk *sdkp)
+{
+	u64 zone_blocks;
+	sector_t block = 0;
+	unsigned char *buf;
+	unsigned char *rec;
+	unsigned int buf_len;
+	unsigned int list_length;
+	int ret;
+	u8 same;
+
+	sdkp->zone_blocks = 0;
+
+	/* Get a buffer */
+	buf = kmalloc(SD_ZBC_BUF_SIZE, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	/* Do a report zone to get the same field */
+	ret = sd_zbc_report_zones(sdkp, buf, SD_ZBC_BUF_SIZE, 0);
+	if (ret) {
+		zone_blocks = 0;
+		goto out;
+	}
+
+	same = buf[4] & 0x0f;
+	if (same > 0) {
+		rec = &buf[64];
+		zone_blocks = get_unaligned_be64(&rec[8]);
+		goto out;
+	}
+
+	/*
+	 * Check the size of all zones: all zones must be of
+	 * equal size, except the last zone which can be smaller
+	 * than other zones.
+	 */
+	do {
+
+		/* Parse REPORT ZONES header */
+		list_length = get_unaligned_be32(&buf[0]) + 64;
+		rec = buf + 64;
+		if (list_length < SD_ZBC_BUF_SIZE)
+			buf_len = list_length;
+		else
+			buf_len = SD_ZBC_BUF_SIZE;
+
+		/* Parse zone descriptors */
+		while (rec < buf + buf_len) {
+			zone_blocks = get_unaligned_be64(&rec[8]);
+			if (sdkp->zone_blocks == 0) {
+				sdkp->zone_blocks = zone_blocks;
+			} else if (zone_blocks != sdkp->zone_blocks &&
+				   (block + zone_blocks < sdkp->capacity
+				    || zone_blocks > sdkp->zone_blocks)) {
+				zone_blocks = 0;
+				goto out;
+			}
+			block += zone_blocks;
+			rec += 64;
+		}
+
+		if (block < sdkp->capacity) {
+			ret = sd_zbc_report_zones(sdkp, buf,
+						  SD_ZBC_BUF_SIZE, block);
+			if (ret)
+				return ret;
+		}
+
+	} while (block < sdkp->capacity);
+
+	zone_blocks = sdkp->zone_blocks;
+
+out:
+	kfree(buf);
+
+	if (!zone_blocks) {
+		if (sdkp->first_scan)
+			sd_printk(KERN_NOTICE, sdkp,
+				  "Devices with non constant zone "
+				  "size are not supported\n");
+		return -ENODEV;
+	}
+
+	if (!is_power_of_2(zone_blocks)) {
+		if (sdkp->first_scan)
+			sd_printk(KERN_NOTICE, sdkp,
+				  "Devices with non power of 2 zone "
+				  "size are not supported\n");
+		return -ENODEV;
+	}
+
+	if (logical_to_sectors(sdkp->device, zone_blocks) > UINT_MAX) {
+		if (sdkp->first_scan)
+			sd_printk(KERN_NOTICE, sdkp,
+				  "Zone size too large\n");
+		return -ENODEV;
+	}
+
+	sdkp->zone_blocks = zone_blocks;
+
+	return 0;
+}
+
+static int sd_zbc_setup(struct scsi_disk *sdkp)
+{
+
+	/* chunk_sectors indicates the zone size */
+	blk_queue_chunk_sectors(sdkp->disk->queue,
+			logical_to_sectors(sdkp->device, sdkp->zone_blocks));
+	sdkp->zone_shift = ilog2(sdkp->zone_blocks);
+	sdkp->nr_zones = sdkp->capacity >> sdkp->zone_shift;
+	if (sdkp->capacity & (sdkp->zone_blocks - 1))
+		sdkp->nr_zones++;
+
+	if (!sdkp->zones_wlock) {
+		sdkp->zones_wlock = kcalloc(BITS_TO_LONGS(sdkp->nr_zones),
+					    sizeof(unsigned long),
+					    GFP_KERNEL);
+		if (!sdkp->zones_wlock)
+			return -ENOMEM;
+	}
+
+	return 0;
+}
+
+int sd_zbc_read_zones(struct scsi_disk *sdkp,
+		      unsigned char *buf)
+{
+	sector_t capacity;
+	int ret = 0;
+
+	if (!sd_is_zoned(sdkp))
+		/*
+		 * Device managed or normal SCSI disk,
+		 * no special handling required
+		 */
+		return 0;
+
+
+	/* Get zoned block device characteristics */
+	ret = sd_zbc_read_zoned_characteristics(sdkp, buf);
+	if (ret)
+		goto err;
+
+	/*
+	 * Check for unconstrained reads: host-managed devices with
+	 * constrained reads (drives failing read after write pointer)
+	 * are not supported.
+	 */
+	if (!sdkp->urswrz) {
+		if (sdkp->first_scan)
+			sd_printk(KERN_NOTICE, sdkp,
+			  "constrained reads devices are not supported\n");
+		ret = -ENODEV;
+		goto err;
+	}
+
+	/* Check capacity */
+	ret = sd_zbc_check_capacity(sdkp, buf);
+	if (ret)
+		goto err;
+	capacity = logical_to_sectors(sdkp->device, sdkp->capacity);
+
+	/*
+	 * Check zone size: only devices with a constant zone size (except
+	 * an eventual last runt zone) that is a power of 2 are supported.
+	 */
+	ret = sd_zbc_check_zone_size(sdkp);
+	if (ret)
+		goto err;
+
+	/* The drive satisfies the kernel restrictions: set it up */
+	ret = sd_zbc_setup(sdkp);
+	if (ret)
+		goto err;
+
+	/* READ16/WRITE16 is mandatory for ZBC disks */
+	sdkp->device->use_16_for_rw = 1;
+	sdkp->device->use_10_for_rw = 0;
+
+	return 0;
+
+err:
+	sdkp->capacity = 0;
+
+	return ret;
+}
+
+void sd_zbc_remove(struct scsi_disk *sdkp)
+{
+	kfree(sdkp->zones_wlock);
+	sdkp->zones_wlock = NULL;
+}
+
+void sd_zbc_print_zones(struct scsi_disk *sdkp)
+{
+	if (!sd_is_zoned(sdkp) || !sdkp->capacity)
+		return;
+
+	if (sdkp->capacity & (sdkp->zone_blocks - 1))
+		sd_printk(KERN_NOTICE, sdkp,
+			  "%u zones of %u logical blocks + 1 runt zone\n",
+			  sdkp->nr_zones - 1,
+			  sdkp->zone_blocks);
+	else
+		sd_printk(KERN_NOTICE, sdkp,
+			  "%u zones of %u logical blocks\n",
+			  sdkp->nr_zones,
+			  sdkp->zone_blocks);
+}
diff --git a/drivers/scsi/smartpqi/smartpqi.h b/drivers/scsi/smartpqi/smartpqi.h
index 07b6444d3e0a..b673825f46b5 100644
--- a/drivers/scsi/smartpqi/smartpqi.h
+++ b/drivers/scsi/smartpqi/smartpqi.h
@@ -929,8 +929,6 @@ struct pqi_ctrl_info {
 	int		max_msix_vectors;
 	int		num_msix_vectors_enabled;
 	int		num_msix_vectors_initialized;
-	u32		msix_vectors[PQI_MAX_MSIX_VECTORS];
-	void		*intr_data[PQI_MAX_MSIX_VECTORS];
 	int		event_irq;
 	struct Scsi_Host *scsi_host;
 
diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c
index a535b2661f38..8702d9cf8040 100644
--- a/drivers/scsi/smartpqi/smartpqi_init.c
+++ b/drivers/scsi/smartpqi/smartpqi_init.c
@@ -25,6 +25,7 @@
 #include <linux/rtc.h>
 #include <linux/bcd.h>
 #include <linux/cciss_ioctl.h>
+#include <linux/blk-mq-pci.h>
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_cmnd.h>
 #include <scsi/scsi_device.h>
@@ -2887,19 +2888,19 @@ static irqreturn_t pqi_irq_handler(int irq, void *data)
 
 static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info)
 {
+	struct pci_dev *pdev = ctrl_info->pci_dev;
 	int i;
 	int rc;
 
-	ctrl_info->event_irq = ctrl_info->msix_vectors[0];
+	ctrl_info->event_irq = pci_irq_vector(pdev, 0);
 
 	for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) {
-		rc = request_irq(ctrl_info->msix_vectors[i],
-			pqi_irq_handler, 0,
-			DRIVER_NAME_SHORT, ctrl_info->intr_data[i]);
+		rc = request_irq(pci_irq_vector(pdev, i), pqi_irq_handler, 0,
+			DRIVER_NAME_SHORT, &ctrl_info->queue_groups[i]);
 		if (rc) {
-			dev_err(&ctrl_info->pci_dev->dev,
+			dev_err(&pdev->dev,
 				"irq %u init failed with error %d\n",
-				ctrl_info->msix_vectors[i], rc);
+				pci_irq_vector(pdev, i), rc);
 			return rc;
 		}
 		ctrl_info->num_msix_vectors_initialized++;
@@ -2908,72 +2909,23 @@ static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info)
 	return 0;
 }
 
-static void pqi_free_irqs(struct pqi_ctrl_info *ctrl_info)
-{
-	int i;
-
-	for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)
-		free_irq(ctrl_info->msix_vectors[i],
-			ctrl_info->intr_data[i]);
-}
-
 static int pqi_enable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)
 {
-	unsigned int i;
-	int max_vectors;
-	int num_vectors_enabled;
-	struct msix_entry msix_entries[PQI_MAX_MSIX_VECTORS];
-
-	max_vectors = ctrl_info->num_queue_groups;
-
-	for (i = 0; i < max_vectors; i++)
-		msix_entries[i].entry = i;
-
-	num_vectors_enabled = pci_enable_msix_range(ctrl_info->pci_dev,
-		msix_entries, PQI_MIN_MSIX_VECTORS, max_vectors);
+	int ret;
 
-	if (num_vectors_enabled < 0) {
+	ret = pci_alloc_irq_vectors(ctrl_info->pci_dev,
+			PQI_MIN_MSIX_VECTORS, ctrl_info->num_queue_groups,
+			PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
+	if (ret < 0) {
 		dev_err(&ctrl_info->pci_dev->dev,
-			"MSI-X init failed with error %d\n",
-			num_vectors_enabled);
-		return num_vectors_enabled;
-	}
-
-	ctrl_info->num_msix_vectors_enabled = num_vectors_enabled;
-	for (i = 0; i < num_vectors_enabled; i++) {
-		ctrl_info->msix_vectors[i] = msix_entries[i].vector;
-		ctrl_info->intr_data[i] = &ctrl_info->queue_groups[i];
+			"MSI-X init failed with error %d\n", ret);
+		return ret;
 	}
 
+	ctrl_info->num_msix_vectors_enabled = ret;
 	return 0;
 }
 
-static void pqi_irq_set_affinity_hint(struct pqi_ctrl_info *ctrl_info)
-{
-	int i;
-	int rc;
-	int cpu;
-
-	cpu = cpumask_first(cpu_online_mask);
-	for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++) {
-		rc = irq_set_affinity_hint(ctrl_info->msix_vectors[i],
-			get_cpu_mask(cpu));
-		if (rc)
-			dev_err(&ctrl_info->pci_dev->dev,
-				"error %d setting affinity hint for irq vector %u\n",
-				rc, ctrl_info->msix_vectors[i]);
-		cpu = cpumask_next(cpu, cpu_online_mask);
-	}
-}
-
-static void pqi_irq_unset_affinity_hint(struct pqi_ctrl_info *ctrl_info)
-{
-	int i;
-
-	for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)
-		irq_set_affinity_hint(ctrl_info->msix_vectors[i], NULL);
-}
-
 static int pqi_alloc_operational_queues(struct pqi_ctrl_info *ctrl_info)
 {
 	unsigned int i;
@@ -4743,6 +4695,13 @@ static int pqi_slave_configure(struct scsi_device *sdev)
 	return 0;
 }
 
+static int pqi_map_queues(struct Scsi_Host *shost)
+{
+	struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
+
+	return blk_mq_pci_map_queues(&shost->tag_set, ctrl_info->pci_dev);
+}
+
 static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info,
 	void __user *arg)
 {
@@ -5130,6 +5089,7 @@ static struct scsi_host_template pqi_driver_template = {
 	.ioctl = pqi_ioctl,
 	.slave_alloc = pqi_slave_alloc,
 	.slave_configure = pqi_slave_configure,
+	.map_queues = pqi_map_queues,
 	.sdev_attrs = pqi_sdev_attrs,
 	.shost_attrs = pqi_shost_attrs,
 };
@@ -5159,7 +5119,7 @@ static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info)
 	shost->cmd_per_lun = shost->can_queue;
 	shost->sg_tablesize = ctrl_info->sg_tablesize;
 	shost->transportt = pqi_sas_transport_template;
-	shost->irq = ctrl_info->msix_vectors[0];
+	shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0);
 	shost->unique_id = shost->irq;
 	shost->nr_hw_queues = ctrl_info->num_queue_groups;
 	shost->hostdata[0] = (unsigned long)ctrl_info;
@@ -5409,8 +5369,6 @@ static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
 	if (rc)
 		return rc;
 
-	pqi_irq_set_affinity_hint(ctrl_info);
-
 	rc = pqi_create_queues(ctrl_info);
 	if (rc)
 		return rc;
@@ -5557,10 +5515,14 @@ static inline void pqi_free_ctrl_info(struct pqi_ctrl_info *ctrl_info)
 
 static void pqi_free_interrupts(struct pqi_ctrl_info *ctrl_info)
 {
-	pqi_irq_unset_affinity_hint(ctrl_info);
-	pqi_free_irqs(ctrl_info);
-	if (ctrl_info->num_msix_vectors_enabled)
-		pci_disable_msix(ctrl_info->pci_dev);
+	int i;
+
+	for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++) {
+		free_irq(pci_irq_vector(ctrl_info->pci_dev, i),
+				&ctrl_info->queue_groups[i]);
+	}
+
+	pci_free_irq_vectors(ctrl_info->pci_dev);
 }
 
 static void pqi_free_ctrl_resources(struct pqi_ctrl_info *ctrl_info)
diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c
index 618422ea3a41..605887d5ee57 100644
--- a/drivers/scsi/st.c
+++ b/drivers/scsi/st.c
@@ -546,7 +546,7 @@ static int st_scsi_execute(struct st_request *SRpnt, const unsigned char *cmd,
 		return DRIVER_ERROR << 24;
 
 	blk_rq_set_block_pc(req);
-	req->cmd_flags |= REQ_QUIET;
+	req->rq_flags |= RQF_QUIET;
 
 	mdata->null_mapped = 1;
 
diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c
index 8ccfc9ea874b..05526b71541b 100644
--- a/drivers/scsi/storvsc_drv.c
+++ b/drivers/scsi/storvsc_drv.c
@@ -1495,9 +1495,9 @@ static int storvsc_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scmnd)
 	if (sg_count) {
 		if (sg_count > MAX_PAGE_BUFFER_COUNT) {
 
-			payload_sz = (sg_count * sizeof(void *) +
+			payload_sz = (sg_count * sizeof(u64) +
 				      sizeof(struct vmbus_packet_mpb_array));
-			payload = kmalloc(payload_sz, GFP_ATOMIC);
+			payload = kzalloc(payload_sz, GFP_ATOMIC);
 			if (!payload)
 				return SCSI_MLQUEUE_DEVICE_BUSY;
 		}
diff --git a/drivers/scsi/sun3_scsi.c b/drivers/scsi/sun3_scsi.c
index 3c4c07038948..88db6992420e 100644
--- a/drivers/scsi/sun3_scsi.c
+++ b/drivers/scsi/sun3_scsi.c
@@ -43,20 +43,18 @@
 
 #define NCR5380_implementation_fields   /* none */
 
-#define NCR5380_read(reg)               sun3scsi_read(reg)
-#define NCR5380_write(reg, value)       sun3scsi_write(reg, value)
+#define NCR5380_read(reg)               in_8(hostdata->io + (reg))
+#define NCR5380_write(reg, value)       out_8(hostdata->io + (reg), value)
 
 #define NCR5380_queue_command           sun3scsi_queue_command
 #define NCR5380_bus_reset               sun3scsi_bus_reset
 #define NCR5380_abort                   sun3scsi_abort
 #define NCR5380_info                    sun3scsi_info
 
-#define NCR5380_dma_recv_setup(instance, data, count) (count)
-#define NCR5380_dma_send_setup(instance, data, count) (count)
-#define NCR5380_dma_residual(instance) \
-        sun3scsi_dma_residual(instance)
-#define NCR5380_dma_xfer_len(instance, cmd, phase) \
-        sun3scsi_dma_xfer_len(cmd->SCp.this_residual, cmd)
+#define NCR5380_dma_xfer_len            sun3scsi_dma_xfer_len
+#define NCR5380_dma_recv_setup          sun3scsi_dma_count
+#define NCR5380_dma_send_setup          sun3scsi_dma_count
+#define NCR5380_dma_residual            sun3scsi_dma_residual
 
 #define NCR5380_acquire_dma_irq(instance)    (1)
 #define NCR5380_release_dma_irq(instance)
@@ -82,7 +80,6 @@ module_param(setup_hostid, int, 0);
 #define SUN3_DVMA_BUFSIZE 0xe000
 
 static struct scsi_cmnd *sun3_dma_setup_done;
-static unsigned char *sun3_scsi_regp;
 static volatile struct sun3_dma_regs *dregs;
 static struct sun3_udc_regs *udc_regs;
 static unsigned char *sun3_dma_orig_addr;
@@ -90,20 +87,6 @@ static unsigned long sun3_dma_orig_count;
 static int sun3_dma_active;
 static unsigned long last_residual;
 
-/*
- * NCR 5380 register access functions
- */
-
-static inline unsigned char sun3scsi_read(int reg)
-{
-	return in_8(sun3_scsi_regp + reg);
-}
-
-static inline void sun3scsi_write(int reg, int value)
-{
-	out_8(sun3_scsi_regp + reg, value);
-}
-
 #ifndef SUN3_SCSI_VME
 /* dma controller register access functions */
 
@@ -158,8 +141,8 @@ static irqreturn_t scsi_sun3_intr(int irq, void *dev)
 }
 
 /* sun3scsi_dma_setup() -- initialize the dma controller for a read/write */
-static unsigned long sun3scsi_dma_setup(struct Scsi_Host *instance,
-                                void *data, unsigned long count, int write_flag)
+static int sun3scsi_dma_setup(struct NCR5380_hostdata *hostdata,
+                              unsigned char *data, int count, int write_flag)
 {
 	void *addr;
 
@@ -211,9 +194,10 @@ static unsigned long sun3scsi_dma_setup(struct Scsi_Host *instance,
 	dregs->csr |= CSR_FIFO;
 	
 	if(dregs->fifo_count != count) { 
-		shost_printk(KERN_ERR, instance, "FIFO mismatch %04x not %04x\n",
+		shost_printk(KERN_ERR, hostdata->host,
+		             "FIFO mismatch %04x not %04x\n",
 		             dregs->fifo_count, (unsigned int) count);
-		NCR5380_dprint(NDEBUG_DMA, instance);
+		NCR5380_dprint(NDEBUG_DMA, hostdata->host);
 	}
 
 	/* setup udc */
@@ -248,14 +232,34 @@ static unsigned long sun3scsi_dma_setup(struct Scsi_Host *instance,
 
 }
 
-static inline unsigned long sun3scsi_dma_residual(struct Scsi_Host *instance)
+static int sun3scsi_dma_count(struct NCR5380_hostdata *hostdata,
+                              unsigned char *data, int count)
+{
+	return count;
+}
+
+static inline int sun3scsi_dma_recv_setup(struct NCR5380_hostdata *hostdata,
+                                          unsigned char *data, int count)
+{
+	return sun3scsi_dma_setup(hostdata, data, count, 0);
+}
+
+static inline int sun3scsi_dma_send_setup(struct NCR5380_hostdata *hostdata,
+                                          unsigned char *data, int count)
+{
+	return sun3scsi_dma_setup(hostdata, data, count, 1);
+}
+
+static int sun3scsi_dma_residual(struct NCR5380_hostdata *hostdata)
 {
 	return last_residual;
 }
 
-static inline unsigned long sun3scsi_dma_xfer_len(unsigned long wanted_len,
-                                                  struct scsi_cmnd *cmd)
+static int sun3scsi_dma_xfer_len(struct NCR5380_hostdata *hostdata,
+                                 struct scsi_cmnd *cmd)
 {
+	int wanted_len = cmd->SCp.this_residual;
+
 	if (wanted_len < DMA_MIN_SIZE || cmd->request->cmd_type != REQ_TYPE_FS)
 		return 0;
 
@@ -428,9 +432,10 @@ static struct scsi_host_template sun3_scsi_template = {
 static int __init sun3_scsi_probe(struct platform_device *pdev)
 {
 	struct Scsi_Host *instance;
+	struct NCR5380_hostdata *hostdata;
 	int error;
 	struct resource *irq, *mem;
-	unsigned char *ioaddr;
+	void __iomem *ioaddr;
 	int host_flags = 0;
 #ifdef SUN3_SCSI_VME
 	int i;
@@ -493,8 +498,6 @@ static int __init sun3_scsi_probe(struct platform_device *pdev)
 	}
 #endif
 
-	sun3_scsi_regp = ioaddr;
-
 	instance = scsi_host_alloc(&sun3_scsi_template,
 	                           sizeof(struct NCR5380_hostdata));
 	if (!instance) {
@@ -502,9 +505,12 @@ static int __init sun3_scsi_probe(struct platform_device *pdev)
 		goto fail_alloc;
 	}
 
-	instance->io_port = (unsigned long)ioaddr;
 	instance->irq = irq->start;
 
+	hostdata = shost_priv(instance);
+	hostdata->base = mem->start;
+	hostdata->io = ioaddr;
+
 	error = NCR5380_init(instance, host_flags);
 	if (error)
 		goto fail_init;
@@ -552,13 +558,15 @@ fail_init:
 fail_alloc:
 	if (udc_regs)
 		dvma_free(udc_regs);
-	iounmap(sun3_scsi_regp);
+	iounmap(ioaddr);
 	return error;
 }
 
 static int __exit sun3_scsi_remove(struct platform_device *pdev)
 {
 	struct Scsi_Host *instance = platform_get_drvdata(pdev);
+	struct NCR5380_hostdata *hostdata = shost_priv(instance);
+	void __iomem *ioaddr = hostdata->io;
 
 	scsi_remove_host(instance);
 	free_irq(instance->irq, instance);
@@ -566,7 +574,7 @@ static int __exit sun3_scsi_remove(struct platform_device *pdev)
 	scsi_host_put(instance);
 	if (udc_regs)
 		dvma_free(udc_regs);
-	iounmap(sun3_scsi_regp);
+	iounmap(ioaddr);
 	return 0;
 }
 
diff --git a/drivers/scsi/ufs/ufs-qcom.c b/drivers/scsi/ufs/ufs-qcom.c
index 3aedf73f1131..aa43bfea0d00 100644
--- a/drivers/scsi/ufs/ufs-qcom.c
+++ b/drivers/scsi/ufs/ufs-qcom.c
@@ -1094,10 +1094,12 @@ static void ufs_qcom_set_caps(struct ufs_hba *hba)
  * ufs_qcom_setup_clocks - enables/disable clocks
  * @hba: host controller instance
  * @on: If true, enable clocks else disable them.
+ * @status: PRE_CHANGE or POST_CHANGE notify
  *
  * Returns 0 on success, non-zero on failure.
  */
-static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on)
+static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on,
+				 enum ufs_notify_change_status status)
 {
 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);
 	int err;
@@ -1111,18 +1113,9 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on)
 	if (!host)
 		return 0;
 
-	if (on) {
-		err = ufs_qcom_phy_enable_iface_clk(host->generic_phy);
-		if (err)
-			goto out;
+	if (on && (status == POST_CHANGE)) {
+		phy_power_on(host->generic_phy);
 
-		err = ufs_qcom_phy_enable_ref_clk(host->generic_phy);
-		if (err) {
-			dev_err(hba->dev, "%s enable phy ref clock failed, err=%d\n",
-				__func__, err);
-			ufs_qcom_phy_disable_iface_clk(host->generic_phy);
-			goto out;
-		}
 		/* enable the device ref clock for HS mode*/
 		if (ufshcd_is_hs_mode(&hba->pwr_info))
 			ufs_qcom_dev_ref_clk_ctrl(host, true);
@@ -1130,14 +1123,15 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on)
 		if (vote == host->bus_vote.min_bw_vote)
 			ufs_qcom_update_bus_bw_vote(host);
 
-	} else {
-
-		/* M-PHY RMMI interface clocks can be turned off */
-		ufs_qcom_phy_disable_iface_clk(host->generic_phy);
-		if (!ufs_qcom_is_link_active(hba))
+	} else if (!on && (status == PRE_CHANGE)) {
+		if (!ufs_qcom_is_link_active(hba)) {
 			/* disable device ref_clk */
 			ufs_qcom_dev_ref_clk_ctrl(host, false);
 
+			/* powering off PHY during aggressive clk gating */
+			phy_power_off(host->generic_phy);
+		}
+
 		vote = host->bus_vote.min_bw_vote;
 	}
 
@@ -1146,7 +1140,6 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on)
 		dev_err(hba->dev, "%s: set bus vote failed %d\n",
 				__func__, err);
 
-out:
 	return err;
 }
 
@@ -1204,12 +1197,12 @@ static int ufs_qcom_init(struct ufs_hba *hba)
 	if (IS_ERR(host->generic_phy)) {
 		err = PTR_ERR(host->generic_phy);
 		dev_err(dev, "%s: PHY get failed %d\n", __func__, err);
-		goto out;
+		goto out_variant_clear;
 	}
 
 	err = ufs_qcom_bus_register(host);
 	if (err)
-		goto out_host_free;
+		goto out_variant_clear;
 
 	ufs_qcom_get_controller_revision(hba, &host->hw_ver.major,
 		&host->hw_ver.minor, &host->hw_ver.step);
@@ -1254,7 +1247,7 @@ static int ufs_qcom_init(struct ufs_hba *hba)
 	ufs_qcom_set_caps(hba);
 	ufs_qcom_advertise_quirks(hba);
 
-	ufs_qcom_setup_clocks(hba, true);
+	ufs_qcom_setup_clocks(hba, true, POST_CHANGE);
 
 	if (hba->dev->id < MAX_UFS_QCOM_HOSTS)
 		ufs_qcom_hosts[hba->dev->id] = host;
@@ -1274,8 +1267,7 @@ out_disable_phy:
 	phy_power_off(host->generic_phy);
 out_unregister_bus:
 	phy_exit(host->generic_phy);
-out_host_free:
-	devm_kfree(dev, host);
+out_variant_clear:
 	ufshcd_set_variant(hba, NULL);
 out:
 	return err;
@@ -1287,6 +1279,7 @@ static void ufs_qcom_exit(struct ufs_hba *hba)
 
 	ufs_qcom_disable_lane_clks(host);
 	phy_power_off(host->generic_phy);
+	phy_exit(host->generic_phy);
 }
 
 static int ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(struct ufs_hba *hba,
diff --git a/drivers/scsi/ufs/ufs.h b/drivers/scsi/ufs/ufs.h
index 845b874e2977..8e6709a3fb6b 100644
--- a/drivers/scsi/ufs/ufs.h
+++ b/drivers/scsi/ufs/ufs.h
@@ -46,6 +46,7 @@
 #define QUERY_DESC_HDR_SIZE       2
 #define QUERY_OSF_SIZE            (GENERAL_UPIU_REQUEST_SIZE - \
 					(sizeof(struct utp_upiu_header)))
+#define RESPONSE_UPIU_SENSE_DATA_LENGTH	18
 
 #define UPIU_HEADER_DWORD(byte3, byte2, byte1, byte0)\
 			cpu_to_be32((byte3 << 24) | (byte2 << 16) |\
@@ -162,7 +163,7 @@ enum desc_header_offset {
 };
 
 enum ufs_desc_max_size {
-	QUERY_DESC_DEVICE_MAX_SIZE		= 0x1F,
+	QUERY_DESC_DEVICE_MAX_SIZE		= 0x40,
 	QUERY_DESC_CONFIGURAION_MAX_SIZE	= 0x90,
 	QUERY_DESC_UNIT_MAX_SIZE		= 0x23,
 	QUERY_DESC_INTERCONNECT_MAX_SIZE	= 0x06,
@@ -416,7 +417,7 @@ struct utp_cmd_rsp {
 	__be32 residual_transfer_count;
 	__be32 reserved[4];
 	__be16 sense_data_len;
-	u8 sense_data[18];
+	u8 sense_data[RESPONSE_UPIU_SENSE_DATA_LENGTH];
 };
 
 /**
diff --git a/drivers/scsi/ufs/ufs_quirks.h b/drivers/scsi/ufs/ufs_quirks.h
index 22f881e9253a..f7983058f3f7 100644
--- a/drivers/scsi/ufs/ufs_quirks.h
+++ b/drivers/scsi/ufs/ufs_quirks.h
@@ -128,6 +128,13 @@ struct ufs_dev_fix {
  */
 #define UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM	(1 << 6)
 
+/*
+ * Some UFS devices require host PA_TACTIVATE to be lower than device
+ * PA_TACTIVATE, enabling this quirk ensure this.
+ */
+#define UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE	(1 << 7)
+
+
 struct ufs_hba;
 void ufs_advertise_fixup_device(struct ufs_hba *hba);
 
@@ -140,6 +147,8 @@ static struct ufs_dev_fix ufs_fixups[] = {
 		UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
 	UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
 		UFS_DEVICE_NO_FASTAUTO),
+	UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
+		UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
 	UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
 		UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
 	UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
diff --git a/drivers/scsi/ufs/ufshcd-pci.c b/drivers/scsi/ufs/ufshcd-pci.c
index d15eaa466c59..52b546fb509b 100644
--- a/drivers/scsi/ufs/ufshcd-pci.c
+++ b/drivers/scsi/ufs/ufshcd-pci.c
@@ -104,6 +104,7 @@ static void ufshcd_pci_remove(struct pci_dev *pdev)
 	pm_runtime_forbid(&pdev->dev);
 	pm_runtime_get_noresume(&pdev->dev);
 	ufshcd_remove(hba);
+	ufshcd_dealloc_host(hba);
 }
 
 /**
@@ -147,6 +148,7 @@ ufshcd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	err = ufshcd_init(hba, mmio_base, pdev->irq);
 	if (err) {
 		dev_err(&pdev->dev, "Initialization failed\n");
+		ufshcd_dealloc_host(hba);
 		return err;
 	}
 
diff --git a/drivers/scsi/ufs/ufshcd-pltfrm.c b/drivers/scsi/ufs/ufshcd-pltfrm.c
index db53f38da864..a72a4ba78125 100644
--- a/drivers/scsi/ufs/ufshcd-pltfrm.c
+++ b/drivers/scsi/ufs/ufshcd-pltfrm.c
@@ -163,7 +163,7 @@ static int ufshcd_populate_vreg(struct device *dev, const char *name,
 	if (ret) {
 		dev_err(dev, "%s: unable to find %s err %d\n",
 				__func__, prop_name, ret);
-		goto out_free;
+		goto out;
 	}
 
 	vreg->min_uA = 0;
@@ -185,9 +185,6 @@ static int ufshcd_populate_vreg(struct device *dev, const char *name,
 
 	goto out;
 
-out_free:
-	devm_kfree(dev, vreg);
-	vreg = NULL;
 out:
 	if (!ret)
 		*out_vreg = vreg;
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
index 05c745663c10..ef8548c3a423 100644
--- a/drivers/scsi/ufs/ufshcd.c
+++ b/drivers/scsi/ufs/ufshcd.c
@@ -45,6 +45,8 @@
 #include "ufs_quirks.h"
 #include "unipro.h"
 
+#define UFSHCD_REQ_SENSE_SIZE	18
+
 #define UFSHCD_ENABLE_INTRS	(UTP_TRANSFER_REQ_COMPL |\
 				 UTP_TASK_REQ_COMPL |\
 				 UFSHCD_ERROR_MASK)
@@ -57,15 +59,9 @@
 #define NOP_OUT_TIMEOUT    30 /* msecs */
 
 /* Query request retries */
-#define QUERY_REQ_RETRIES 10
+#define QUERY_REQ_RETRIES 3
 /* Query request timeout */
-#define QUERY_REQ_TIMEOUT 30 /* msec */
-/*
- * Query request timeout for fDeviceInit flag
- * fDeviceInit query response time for some devices is too large that default
- * QUERY_REQ_TIMEOUT may not be enough for such devices.
- */
-#define QUERY_FDEVICEINIT_REQ_TIMEOUT 600 /* msec */
+#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
 
 /* Task management command timeout */
 #define TM_CMD_TIMEOUT	100 /* msecs */
@@ -123,6 +119,7 @@ enum {
 	UFSHCD_STATE_RESET,
 	UFSHCD_STATE_ERROR,
 	UFSHCD_STATE_OPERATIONAL,
+	UFSHCD_STATE_EH_SCHEDULED,
 };
 
 /* UFSHCD error handling flags */
@@ -598,6 +595,20 @@ static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
 		return false;
 }
 
+static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
+{
+	if (ufshcd_is_clkscaling_enabled(hba)) {
+		devfreq_suspend_device(hba->devfreq);
+		hba->clk_scaling.window_start_t = 0;
+	}
+}
+
+static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
+{
+	if (ufshcd_is_clkscaling_enabled(hba))
+		devfreq_resume_device(hba->devfreq);
+}
+
 static void ufshcd_ungate_work(struct work_struct *work)
 {
 	int ret;
@@ -631,8 +642,7 @@ static void ufshcd_ungate_work(struct work_struct *work)
 		hba->clk_gating.is_suspended = false;
 	}
 unblock_reqs:
-	if (ufshcd_is_clkscaling_enabled(hba))
-		devfreq_resume_device(hba->devfreq);
+	ufshcd_resume_clkscaling(hba);
 	scsi_unblock_requests(hba->host);
 }
 
@@ -660,6 +670,21 @@ int ufshcd_hold(struct ufs_hba *hba, bool async)
 start:
 	switch (hba->clk_gating.state) {
 	case CLKS_ON:
+		/*
+		 * Wait for the ungate work to complete if in progress.
+		 * Though the clocks may be in ON state, the link could
+		 * still be in hibner8 state if hibern8 is allowed
+		 * during clock gating.
+		 * Make sure we exit hibern8 state also in addition to
+		 * clocks being ON.
+		 */
+		if (ufshcd_can_hibern8_during_gating(hba) &&
+		    ufshcd_is_link_hibern8(hba)) {
+			spin_unlock_irqrestore(hba->host->host_lock, flags);
+			flush_work(&hba->clk_gating.ungate_work);
+			spin_lock_irqsave(hba->host->host_lock, flags);
+			goto start;
+		}
 		break;
 	case REQ_CLKS_OFF:
 		if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
@@ -709,7 +734,14 @@ static void ufshcd_gate_work(struct work_struct *work)
 	unsigned long flags;
 
 	spin_lock_irqsave(hba->host->host_lock, flags);
-	if (hba->clk_gating.is_suspended) {
+	/*
+	 * In case you are here to cancel this work the gating state
+	 * would be marked as REQ_CLKS_ON. In this case save time by
+	 * skipping the gating work and exit after changing the clock
+	 * state to CLKS_ON.
+	 */
+	if (hba->clk_gating.is_suspended ||
+		(hba->clk_gating.state == REQ_CLKS_ON)) {
 		hba->clk_gating.state = CLKS_ON;
 		goto rel_lock;
 	}
@@ -731,10 +763,7 @@ static void ufshcd_gate_work(struct work_struct *work)
 		ufshcd_set_link_hibern8(hba);
 	}
 
-	if (ufshcd_is_clkscaling_enabled(hba)) {
-		devfreq_suspend_device(hba->devfreq);
-		hba->clk_scaling.window_start_t = 0;
-	}
+	ufshcd_suspend_clkscaling(hba);
 
 	if (!ufshcd_is_link_active(hba))
 		ufshcd_setup_clocks(hba, false);
@@ -878,6 +907,8 @@ void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
 	ufshcd_clk_scaling_start_busy(hba);
 	__set_bit(task_tag, &hba->outstanding_reqs);
 	ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
+	/* Make sure that doorbell is committed immediately */
+	wmb();
 }
 
 /**
@@ -889,10 +920,14 @@ static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
 	int len;
 	if (lrbp->sense_buffer &&
 	    ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
+		int len_to_copy;
+
 		len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
+		len_to_copy = min_t(int, RESPONSE_UPIU_SENSE_DATA_LENGTH, len);
+
 		memcpy(lrbp->sense_buffer,
 			lrbp->ucd_rsp_ptr->sr.sense_data,
-			min_t(int, len, SCSI_SENSE_BUFFERSIZE));
+			min_t(int, len_to_copy, UFSHCD_REQ_SENSE_SIZE));
 	}
 }
 
@@ -1088,7 +1123,7 @@ ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
  *
  * Returns 0 in case of success, non-zero value in case of failure
  */
-static int ufshcd_map_sg(struct ufshcd_lrb *lrbp)
+static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
 {
 	struct ufshcd_sg_entry *prd_table;
 	struct scatterlist *sg;
@@ -1102,8 +1137,13 @@ static int ufshcd_map_sg(struct ufshcd_lrb *lrbp)
 		return sg_segments;
 
 	if (sg_segments) {
-		lrbp->utr_descriptor_ptr->prd_table_length =
-					cpu_to_le16((u16) (sg_segments));
+		if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
+			lrbp->utr_descriptor_ptr->prd_table_length =
+				cpu_to_le16((u16)(sg_segments *
+					sizeof(struct ufshcd_sg_entry)));
+		else
+			lrbp->utr_descriptor_ptr->prd_table_length =
+				cpu_to_le16((u16) (sg_segments));
 
 		prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
 
@@ -1410,6 +1450,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
 	switch (hba->ufshcd_state) {
 	case UFSHCD_STATE_OPERATIONAL:
 		break;
+	case UFSHCD_STATE_EH_SCHEDULED:
 	case UFSHCD_STATE_RESET:
 		err = SCSI_MLQUEUE_HOST_BUSY;
 		goto out_unlock;
@@ -1457,7 +1498,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
 
 	WARN_ON(lrbp->cmd);
 	lrbp->cmd = cmd;
-	lrbp->sense_bufflen = SCSI_SENSE_BUFFERSIZE;
+	lrbp->sense_bufflen = UFSHCD_REQ_SENSE_SIZE;
 	lrbp->sense_buffer = cmd->sense_buffer;
 	lrbp->task_tag = tag;
 	lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
@@ -1465,15 +1506,18 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
 
 	ufshcd_comp_scsi_upiu(hba, lrbp);
 
-	err = ufshcd_map_sg(lrbp);
+	err = ufshcd_map_sg(hba, lrbp);
 	if (err) {
 		lrbp->cmd = NULL;
 		clear_bit_unlock(tag, &hba->lrb_in_use);
 		goto out;
 	}
+	/* Make sure descriptors are ready before ringing the doorbell */
+	wmb();
 
 	/* issue command to the controller */
 	spin_lock_irqsave(hba->host->host_lock, flags);
+	ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
 	ufshcd_send_command(hba, tag);
 out_unlock:
 	spin_unlock_irqrestore(hba->host->host_lock, flags);
@@ -1581,6 +1625,8 @@ static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
 	time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
 			msecs_to_jiffies(max_timeout));
 
+	/* Make sure descriptors are ready before ringing the doorbell */
+	wmb();
 	spin_lock_irqsave(hba->host->host_lock, flags);
 	hba->dev_cmd.complete = NULL;
 	if (likely(time_left)) {
@@ -1683,6 +1729,7 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
 	/* Make sure descriptors are ready before ringing the doorbell */
 	wmb();
 	spin_lock_irqsave(hba->host->host_lock, flags);
+	ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
 	ufshcd_send_command(hba, tag);
 	spin_unlock_irqrestore(hba->host->host_lock, flags);
 
@@ -1789,9 +1836,6 @@ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
 		goto out_unlock;
 	}
 
-	if (idn == QUERY_FLAG_IDN_FDEVICEINIT)
-		timeout = QUERY_FDEVICEINIT_REQ_TIMEOUT;
-
 	err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
 
 	if (err) {
@@ -1861,8 +1905,8 @@ static int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
 	err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
 
 	if (err) {
-		dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n",
-				__func__, opcode, idn, err);
+		dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
+				__func__, opcode, idn, index, err);
 		goto out_unlock;
 	}
 
@@ -1961,8 +2005,8 @@ static int __ufshcd_query_descriptor(struct ufs_hba *hba,
 	err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
 
 	if (err) {
-		dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, err = %d\n",
-				__func__, opcode, idn, err);
+		dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
+				__func__, opcode, idn, index, err);
 		goto out_unlock;
 	}
 
@@ -2055,18 +2099,41 @@ static int ufshcd_read_desc_param(struct ufs_hba *hba,
 					desc_id, desc_index, 0, desc_buf,
 					&buff_len);
 
-	if (ret || (buff_len < ufs_query_desc_max_size[desc_id]) ||
-	    (desc_buf[QUERY_DESC_LENGTH_OFFSET] !=
-	     ufs_query_desc_max_size[desc_id])
-	    || (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id)) {
-		dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d param_offset %d buff_len %d ret %d",
-			__func__, desc_id, param_offset, buff_len, ret);
-		if (!ret)
-			ret = -EINVAL;
+	if (ret) {
+		dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
+			__func__, desc_id, desc_index, param_offset, ret);
 
 		goto out;
 	}
 
+	/* Sanity check */
+	if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
+		dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
+			__func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/*
+	 * While reading variable size descriptors (like string descriptor),
+	 * some UFS devices may report the "LENGTH" (field in "Transaction
+	 * Specific fields" of Query Response UPIU) same as what was requested
+	 * in Query Request UPIU instead of reporting the actual size of the
+	 * variable size descriptor.
+	 * Although it's safe to ignore the "LENGTH" field for variable size
+	 * descriptors as we can always derive the length of the descriptor from
+	 * the descriptor header fields. Hence this change impose the length
+	 * match check only for fixed size descriptors (for which we always
+	 * request the correct size as part of Query Request UPIU).
+	 */
+	if ((desc_id != QUERY_DESC_IDN_STRING) &&
+	    (buff_len != desc_buf[QUERY_DESC_LENGTH_OFFSET])) {
+		dev_err(hba->dev, "%s: desc_buf length mismatch: buff_len %d, buff_len(desc_header) %d",
+			__func__, buff_len, desc_buf[QUERY_DESC_LENGTH_OFFSET]);
+		ret = -EINVAL;
+		goto out;
+	}
+
 	if (is_kmalloc)
 		memcpy(param_read_buf, &desc_buf[param_offset], param_size);
 out:
@@ -2088,7 +2155,18 @@ static inline int ufshcd_read_power_desc(struct ufs_hba *hba,
 					 u8 *buf,
 					 u32 size)
 {
-	return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size);
+	int err = 0;
+	int retries;
+
+	for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
+		/* Read descriptor*/
+		err = ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size);
+		if (!err)
+			break;
+		dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
+	}
+
+	return err;
 }
 
 int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
@@ -2320,12 +2398,21 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba)
 				cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
 
 		/* Response upiu and prdt offset should be in double words */
-		utrdlp[i].response_upiu_offset =
+		if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
+			utrdlp[i].response_upiu_offset =
+				cpu_to_le16(response_offset);
+			utrdlp[i].prd_table_offset =
+				cpu_to_le16(prdt_offset);
+			utrdlp[i].response_upiu_length =
+				cpu_to_le16(ALIGNED_UPIU_SIZE);
+		} else {
+			utrdlp[i].response_upiu_offset =
 				cpu_to_le16((response_offset >> 2));
-		utrdlp[i].prd_table_offset =
+			utrdlp[i].prd_table_offset =
 				cpu_to_le16((prdt_offset >> 2));
-		utrdlp[i].response_upiu_length =
+			utrdlp[i].response_upiu_length =
 				cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
+		}
 
 		hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
 		hba->lrb[i].ucd_req_ptr =
@@ -2429,10 +2516,10 @@ int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
 				set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
 	} while (ret && peer && --retries);
 
-	if (!retries)
+	if (ret)
 		dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
-				set, UIC_GET_ATTR_ID(attr_sel), mib_val,
-				retries);
+			set, UIC_GET_ATTR_ID(attr_sel), mib_val,
+			UFS_UIC_COMMAND_RETRIES - retries);
 
 	return ret;
 }
@@ -2496,9 +2583,10 @@ int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
 				get, UIC_GET_ATTR_ID(attr_sel), ret);
 	} while (ret && peer && --retries);
 
-	if (!retries)
+	if (ret)
 		dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
-				get, UIC_GET_ATTR_ID(attr_sel), retries);
+			get, UIC_GET_ATTR_ID(attr_sel),
+			UFS_UIC_COMMAND_RETRIES - retries);
 
 	if (mib_val && !ret)
 		*mib_val = uic_cmd.argument3;
@@ -2651,6 +2739,8 @@ static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
 	int ret;
 	struct uic_command uic_cmd = {0};
 
+	ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
+
 	uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
 	ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
 
@@ -2664,7 +2754,9 @@ static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
 		 */
 		if (ufshcd_link_recovery(hba))
 			ret = -ENOLINK;
-	}
+	} else
+		ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
+								POST_CHANGE);
 
 	return ret;
 }
@@ -2687,13 +2779,17 @@ static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
 	struct uic_command uic_cmd = {0};
 	int ret;
 
+	ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
+
 	uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
 	ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
 	if (ret) {
 		dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
 			__func__, ret);
 		ret = ufshcd_link_recovery(hba);
-	}
+	} else
+		ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
+								POST_CHANGE);
 
 	return ret;
 }
@@ -2725,8 +2821,8 @@ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
 	if (hba->max_pwr_info.is_valid)
 		return 0;
 
-	pwr_info->pwr_tx = FASTAUTO_MODE;
-	pwr_info->pwr_rx = FASTAUTO_MODE;
+	pwr_info->pwr_tx = FAST_MODE;
+	pwr_info->pwr_rx = FAST_MODE;
 	pwr_info->hs_rate = PA_HS_MODE_B;
 
 	/* Get the connected lane count */
@@ -2757,7 +2853,7 @@ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
 				__func__, pwr_info->gear_rx);
 			return -EINVAL;
 		}
-		pwr_info->pwr_rx = SLOWAUTO_MODE;
+		pwr_info->pwr_rx = SLOW_MODE;
 	}
 
 	ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
@@ -2770,7 +2866,7 @@ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
 				__func__, pwr_info->gear_tx);
 			return -EINVAL;
 		}
-		pwr_info->pwr_tx = SLOWAUTO_MODE;
+		pwr_info->pwr_tx = SLOW_MODE;
 	}
 
 	hba->max_pwr_info.is_valid = true;
@@ -3090,7 +3186,16 @@ static int ufshcd_link_startup(struct ufs_hba *hba)
 {
 	int ret;
 	int retries = DME_LINKSTARTUP_RETRIES;
+	bool link_startup_again = false;
+
+	/*
+	 * If UFS device isn't active then we will have to issue link startup
+	 * 2 times to make sure the device state move to active.
+	 */
+	if (!ufshcd_is_ufs_dev_active(hba))
+		link_startup_again = true;
 
+link_startup:
 	do {
 		ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
 
@@ -3116,6 +3221,12 @@ static int ufshcd_link_startup(struct ufs_hba *hba)
 		/* failed to get the link up... retire */
 		goto out;
 
+	if (link_startup_again) {
+		link_startup_again = false;
+		retries = DME_LINKSTARTUP_RETRIES;
+		goto link_startup;
+	}
+
 	if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
 		ret = ufshcd_disable_device_tx_lcc(hba);
 		if (ret)
@@ -3181,16 +3292,24 @@ static void ufshcd_set_queue_depth(struct scsi_device *sdev)
 {
 	int ret = 0;
 	u8 lun_qdepth;
+	int retries;
 	struct ufs_hba *hba;
 
 	hba = shost_priv(sdev->host);
 
 	lun_qdepth = hba->nutrs;
-	ret = ufshcd_read_unit_desc_param(hba,
-					  ufshcd_scsi_to_upiu_lun(sdev->lun),
-					  UNIT_DESC_PARAM_LU_Q_DEPTH,
-					  &lun_qdepth,
-					  sizeof(lun_qdepth));
+	for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
+		/* Read descriptor*/
+		ret = ufshcd_read_unit_desc_param(hba,
+				  ufshcd_scsi_to_upiu_lun(sdev->lun),
+				  UNIT_DESC_PARAM_LU_Q_DEPTH,
+				  &lun_qdepth,
+				  sizeof(lun_qdepth));
+		if (!ret || ret == -ENOTSUPP)
+			break;
+
+		dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, ret);
+	}
 
 	/* Some WLUN doesn't support unit descriptor */
 	if (ret == -EOPNOTSUPP)
@@ -4097,6 +4216,17 @@ static void ufshcd_update_uic_error(struct ufs_hba *hba)
 {
 	u32 reg;
 
+	/* PHY layer lane error */
+	reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
+	/* Ignore LINERESET indication, as this is not an error */
+	if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
+			(reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK))
+		/*
+		 * To know whether this error is fatal or not, DB timeout
+		 * must be checked but this error is handled separately.
+		 */
+		dev_dbg(hba->dev, "%s: UIC Lane error reported\n", __func__);
+
 	/* PA_INIT_ERROR is fatal and needs UIC reset */
 	reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
 	if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
@@ -4158,7 +4288,7 @@ static void ufshcd_check_errors(struct ufs_hba *hba)
 			/* block commands from scsi mid-layer */
 			scsi_block_requests(hba->host);
 
-			hba->ufshcd_state = UFSHCD_STATE_ERROR;
+			hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED;
 			schedule_work(&hba->eh_work);
 		}
 	}
@@ -4311,6 +4441,8 @@ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
 	task_req_upiup->input_param1 = cpu_to_be32(lun_id);
 	task_req_upiup->input_param2 = cpu_to_be32(task_id);
 
+	ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function);
+
 	/* send command to the controller */
 	__set_bit(free_slot, &hba->outstanding_tasks);
 
@@ -4318,6 +4450,8 @@ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
 	wmb();
 
 	ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
+	/* Make sure that doorbell is committed immediately */
+	wmb();
 
 	spin_unlock_irqrestore(host->host_lock, flags);
 
@@ -4722,6 +4856,24 @@ out:
 	return icc_level;
 }
 
+static int ufshcd_set_icc_levels_attr(struct ufs_hba *hba, u32 icc_level)
+{
+	int ret = 0;
+	int retries;
+
+	for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
+		/* write attribute */
+		ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
+			QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
+		if (!ret)
+			break;
+
+		dev_dbg(hba->dev, "%s: failed with error %d\n", __func__, ret);
+	}
+
+	return ret;
+}
+
 static void ufshcd_init_icc_levels(struct ufs_hba *hba)
 {
 	int ret;
@@ -4742,9 +4894,8 @@ static void ufshcd_init_icc_levels(struct ufs_hba *hba)
 	dev_dbg(hba->dev, "%s: setting icc_level 0x%x",
 			__func__, hba->init_prefetch_data.icc_level);
 
-	ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
-		QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0,
-		&hba->init_prefetch_data.icc_level);
+	ret = ufshcd_set_icc_levels_attr(hba,
+				 hba->init_prefetch_data.icc_level);
 
 	if (ret)
 		dev_err(hba->dev,
@@ -4965,6 +5116,76 @@ out:
 	return ret;
 }
 
+/**
+ * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
+ * less than device PA_TACTIVATE time.
+ * @hba: per-adapter instance
+ *
+ * Some UFS devices require host PA_TACTIVATE to be lower than device
+ * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
+ * for such devices.
+ *
+ * Returns zero on success, non-zero error value on failure.
+ */
+static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
+{
+	int ret = 0;
+	u32 granularity, peer_granularity;
+	u32 pa_tactivate, peer_pa_tactivate;
+	u32 pa_tactivate_us, peer_pa_tactivate_us;
+	u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
+
+	ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
+				  &granularity);
+	if (ret)
+		goto out;
+
+	ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
+				  &peer_granularity);
+	if (ret)
+		goto out;
+
+	if ((granularity < PA_GRANULARITY_MIN_VAL) ||
+	    (granularity > PA_GRANULARITY_MAX_VAL)) {
+		dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
+			__func__, granularity);
+		return -EINVAL;
+	}
+
+	if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
+	    (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
+		dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
+			__func__, peer_granularity);
+		return -EINVAL;
+	}
+
+	ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
+	if (ret)
+		goto out;
+
+	ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
+				  &peer_pa_tactivate);
+	if (ret)
+		goto out;
+
+	pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
+	peer_pa_tactivate_us = peer_pa_tactivate *
+			     gran_to_us_table[peer_granularity - 1];
+
+	if (pa_tactivate_us > peer_pa_tactivate_us) {
+		u32 new_peer_pa_tactivate;
+
+		new_peer_pa_tactivate = pa_tactivate_us /
+				      gran_to_us_table[peer_granularity - 1];
+		new_peer_pa_tactivate++;
+		ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
+					  new_peer_pa_tactivate);
+	}
+
+out:
+	return ret;
+}
+
 static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
 {
 	if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
@@ -4975,6 +5196,9 @@ static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
 	if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
 		/* set 1ms timeout for PA_TACTIVATE */
 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
+
+	if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
+		ufshcd_quirk_tune_host_pa_tactivate(hba);
 }
 
 /**
@@ -5027,9 +5251,11 @@ static int ufshcd_probe_hba(struct ufs_hba *hba)
 			__func__);
 	} else {
 		ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
-		if (ret)
+		if (ret) {
 			dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
 					__func__, ret);
+			goto out;
+		}
 	}
 
 	/* set the state as operational after switching to desired gear */
@@ -5062,8 +5288,7 @@ static int ufshcd_probe_hba(struct ufs_hba *hba)
 		hba->is_init_prefetch = true;
 
 	/* Resume devfreq after UFS device is detected */
-	if (ufshcd_is_clkscaling_enabled(hba))
-		devfreq_resume_device(hba->devfreq);
+	ufshcd_resume_clkscaling(hba);
 
 out:
 	/*
@@ -5389,6 +5614,10 @@ static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
 	if (!head || list_empty(head))
 		goto out;
 
+	ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
+	if (ret)
+		return ret;
+
 	list_for_each_entry(clki, head, list) {
 		if (!IS_ERR_OR_NULL(clki->clk)) {
 			if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
@@ -5410,7 +5639,10 @@ static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
 		}
 	}
 
-	ret = ufshcd_vops_setup_clocks(hba, on);
+	ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
+	if (ret)
+		return ret;
+
 out:
 	if (ret) {
 		list_for_each_entry(clki, head, list) {
@@ -5500,8 +5732,6 @@ static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
 	if (!hba->vops)
 		return;
 
-	ufshcd_vops_setup_clocks(hba, false);
-
 	ufshcd_vops_setup_regulators(hba, false);
 
 	ufshcd_vops_exit(hba);
@@ -5564,6 +5794,7 @@ static void ufshcd_hba_exit(struct ufs_hba *hba)
 	if (hba->is_powered) {
 		ufshcd_variant_hba_exit(hba);
 		ufshcd_setup_vreg(hba, false);
+		ufshcd_suspend_clkscaling(hba);
 		ufshcd_setup_clocks(hba, false);
 		ufshcd_setup_hba_vreg(hba, false);
 		hba->is_powered = false;
@@ -5577,20 +5808,20 @@ ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
 				0,
 				0,
 				0,
-				SCSI_SENSE_BUFFERSIZE,
+				UFSHCD_REQ_SENSE_SIZE,
 				0};
 	char *buffer;
 	int ret;
 
-	buffer = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);
+	buffer = kzalloc(UFSHCD_REQ_SENSE_SIZE, GFP_KERNEL);
 	if (!buffer) {
 		ret = -ENOMEM;
 		goto out;
 	}
 
 	ret = scsi_execute_req_flags(sdp, cmd, DMA_FROM_DEVICE, buffer,
-				SCSI_SENSE_BUFFERSIZE, NULL,
-				msecs_to_jiffies(1000), 3, NULL, REQ_PM);
+				UFSHCD_REQ_SENSE_SIZE, NULL,
+				msecs_to_jiffies(1000), 3, NULL, 0, RQF_PM);
 	if (ret)
 		pr_err("%s: failed with err %d\n", __func__, ret);
 
@@ -5652,11 +5883,11 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
 
 	/*
 	 * Current function would be generally called from the power management
-	 * callbacks hence set the REQ_PM flag so that it doesn't resume the
+	 * callbacks hence set the RQF_PM flag so that it doesn't resume the
 	 * already suspended childs.
 	 */
 	ret = scsi_execute_req_flags(sdp, cmd, DMA_NONE, NULL, 0, &sshdr,
-				     START_STOP_TIMEOUT, 0, NULL, REQ_PM);
+				     START_STOP_TIMEOUT, 0, NULL, 0, RQF_PM);
 	if (ret) {
 		sdev_printk(KERN_WARNING, sdp,
 			    "START_STOP failed for power mode: %d, result %x\n",
@@ -5766,7 +5997,6 @@ static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
 	    !hba->dev_info.is_lu_power_on_wp) {
 		ret = ufshcd_setup_vreg(hba, true);
 	} else if (!ufshcd_is_ufs_dev_active(hba)) {
-		ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
 		if (!ret && !ufshcd_is_link_active(hba)) {
 			ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
 			if (ret)
@@ -5775,6 +6005,7 @@ static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
 			if (ret)
 				goto vccq_lpm;
 		}
+		ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
 	}
 	goto out;
 
@@ -5839,6 +6070,8 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 	ufshcd_hold(hba, false);
 	hba->clk_gating.is_suspended = true;
 
+	ufshcd_suspend_clkscaling(hba);
+
 	if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
 			req_link_state == UIC_LINK_ACTIVE_STATE) {
 		goto disable_clks;
@@ -5846,12 +6079,12 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 
 	if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
 	    (req_link_state == hba->uic_link_state))
-		goto out;
+		goto enable_gating;
 
 	/* UFS device & link must be active before we enter in this function */
 	if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
 		ret = -EINVAL;
-		goto out;
+		goto enable_gating;
 	}
 
 	if (ufshcd_is_runtime_pm(pm_op)) {
@@ -5888,15 +6121,6 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 
 disable_clks:
 	/*
-	 * The clock scaling needs access to controller registers. Hence, Wait
-	 * for pending clock scaling work to be done before clocks are
-	 * turned off.
-	 */
-	if (ufshcd_is_clkscaling_enabled(hba)) {
-		devfreq_suspend_device(hba->devfreq);
-		hba->clk_scaling.window_start_t = 0;
-	}
-	/*
 	 * Call vendor specific suspend callback. As these callbacks may access
 	 * vendor specific host controller register space call them before the
 	 * host clocks are ON.
@@ -5905,10 +6129,6 @@ disable_clks:
 	if (ret)
 		goto set_link_active;
 
-	ret = ufshcd_vops_setup_clocks(hba, false);
-	if (ret)
-		goto vops_resume;
-
 	if (!ufshcd_is_link_active(hba))
 		ufshcd_setup_clocks(hba, false);
 	else
@@ -5925,9 +6145,8 @@ disable_clks:
 	ufshcd_hba_vreg_set_lpm(hba);
 	goto out;
 
-vops_resume:
-	ufshcd_vops_resume(hba, pm_op);
 set_link_active:
+	ufshcd_resume_clkscaling(hba);
 	ufshcd_vreg_set_hpm(hba);
 	if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
 		ufshcd_set_link_active(hba);
@@ -5937,6 +6156,7 @@ set_dev_active:
 	if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
 		ufshcd_disable_auto_bkops(hba);
 enable_gating:
+	ufshcd_resume_clkscaling(hba);
 	hba->clk_gating.is_suspended = false;
 	ufshcd_release(hba);
 out:
@@ -6015,8 +6235,7 @@ static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 	ufshcd_urgent_bkops(hba);
 	hba->clk_gating.is_suspended = false;
 
-	if (ufshcd_is_clkscaling_enabled(hba))
-		devfreq_resume_device(hba->devfreq);
+	ufshcd_resume_clkscaling(hba);
 
 	/* Schedule clock gating in case of no access to UFS device yet */
 	ufshcd_release(hba);
@@ -6030,6 +6249,7 @@ disable_vreg:
 	ufshcd_vreg_set_lpm(hba);
 disable_irq_and_vops_clks:
 	ufshcd_disable_irq(hba);
+	ufshcd_suspend_clkscaling(hba);
 	ufshcd_setup_clocks(hba, false);
 out:
 	hba->pm_op_in_progress = 0;
@@ -6052,16 +6272,13 @@ int ufshcd_system_suspend(struct ufs_hba *hba)
 	if (!hba || !hba->is_powered)
 		return 0;
 
-	if (pm_runtime_suspended(hba->dev)) {
-		if (hba->rpm_lvl == hba->spm_lvl)
-			/*
-			 * There is possibility that device may still be in
-			 * active state during the runtime suspend.
-			 */
-			if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
-			    hba->curr_dev_pwr_mode) && !hba->auto_bkops_enabled)
-				goto out;
+	if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
+	     hba->curr_dev_pwr_mode) &&
+	    (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) ==
+	     hba->uic_link_state))
+		goto out;
 
+	if (pm_runtime_suspended(hba->dev)) {
 		/*
 		 * UFS device and/or UFS link low power states during runtime
 		 * suspend seems to be different than what is expected during
@@ -6092,7 +6309,10 @@ EXPORT_SYMBOL(ufshcd_system_suspend);
 
 int ufshcd_system_resume(struct ufs_hba *hba)
 {
-	if (!hba || !hba->is_powered || pm_runtime_suspended(hba->dev))
+	if (!hba)
+		return -EINVAL;
+
+	if (!hba->is_powered || pm_runtime_suspended(hba->dev))
 		/*
 		 * Let the runtime resume take care of resuming
 		 * if runtime suspended.
@@ -6113,7 +6333,10 @@ EXPORT_SYMBOL(ufshcd_system_resume);
  */
 int ufshcd_runtime_suspend(struct ufs_hba *hba)
 {
-	if (!hba || !hba->is_powered)
+	if (!hba)
+		return -EINVAL;
+
+	if (!hba->is_powered)
 		return 0;
 
 	return ufshcd_suspend(hba, UFS_RUNTIME_PM);
@@ -6143,10 +6366,13 @@ EXPORT_SYMBOL(ufshcd_runtime_suspend);
  */
 int ufshcd_runtime_resume(struct ufs_hba *hba)
 {
-	if (!hba || !hba->is_powered)
+	if (!hba)
+		return -EINVAL;
+
+	if (!hba->is_powered)
 		return 0;
-	else
-		return ufshcd_resume(hba, UFS_RUNTIME_PM);
+
+	return ufshcd_resume(hba, UFS_RUNTIME_PM);
 }
 EXPORT_SYMBOL(ufshcd_runtime_resume);
 
@@ -6198,11 +6424,7 @@ void ufshcd_remove(struct ufs_hba *hba)
 	ufshcd_disable_intr(hba, hba->intr_mask);
 	ufshcd_hba_stop(hba, true);
 
-	scsi_host_put(hba->host);
-
 	ufshcd_exit_clk_gating(hba);
-	if (ufshcd_is_clkscaling_enabled(hba))
-		devfreq_remove_device(hba->devfreq);
 	ufshcd_hba_exit(hba);
 }
 EXPORT_SYMBOL_GPL(ufshcd_remove);
@@ -6324,15 +6546,47 @@ static int ufshcd_devfreq_target(struct device *dev,
 {
 	int err = 0;
 	struct ufs_hba *hba = dev_get_drvdata(dev);
+	bool release_clk_hold = false;
+	unsigned long irq_flags;
 
 	if (!ufshcd_is_clkscaling_enabled(hba))
 		return -EINVAL;
 
+	spin_lock_irqsave(hba->host->host_lock, irq_flags);
+	if (ufshcd_eh_in_progress(hba)) {
+		spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
+		return 0;
+	}
+
+	if (ufshcd_is_clkgating_allowed(hba) &&
+	    (hba->clk_gating.state != CLKS_ON)) {
+		if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
+			/* hold the vote until the scaling work is completed */
+			hba->clk_gating.active_reqs++;
+			release_clk_hold = true;
+			hba->clk_gating.state = CLKS_ON;
+		} else {
+			/*
+			 * Clock gating work seems to be running in parallel
+			 * hence skip scaling work to avoid deadlock between
+			 * current scaling work and gating work.
+			 */
+			spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
+			return 0;
+		}
+	}
+	spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
+
 	if (*freq == UINT_MAX)
 		err = ufshcd_scale_clks(hba, true);
 	else if (*freq == 0)
 		err = ufshcd_scale_clks(hba, false);
 
+	spin_lock_irqsave(hba->host->host_lock, irq_flags);
+	if (release_clk_hold)
+		__ufshcd_release(hba);
+	spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
+
 	return err;
 }
 
@@ -6498,7 +6752,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
 	}
 
 	if (ufshcd_is_clkscaling_enabled(hba)) {
-		hba->devfreq = devfreq_add_device(dev, &ufs_devfreq_profile,
+		hba->devfreq = devm_devfreq_add_device(dev, &ufs_devfreq_profile,
 						   "simple_ondemand", NULL);
 		if (IS_ERR(hba->devfreq)) {
 			dev_err(hba->dev, "Unable to register with devfreq %ld\n",
@@ -6507,18 +6761,19 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
 			goto out_remove_scsi_host;
 		}
 		/* Suspend devfreq until the UFS device is detected */
-		devfreq_suspend_device(hba->devfreq);
-		hba->clk_scaling.window_start_t = 0;
+		ufshcd_suspend_clkscaling(hba);
 	}
 
 	/* Hold auto suspend until async scan completes */
 	pm_runtime_get_sync(dev);
 
 	/*
-	 * The device-initialize-sequence hasn't been invoked yet.
-	 * Set the device to power-off state
+	 * We are assuming that device wasn't put in sleep/power-down
+	 * state exclusively during the boot stage before kernel.
+	 * This assumption helps avoid doing link startup twice during
+	 * ufshcd_probe_hba().
 	 */
-	ufshcd_set_ufs_dev_poweroff(hba);
+	ufshcd_set_ufs_dev_active(hba);
 
 	async_schedule(ufshcd_async_scan, hba);
 
@@ -6530,7 +6785,6 @@ exit_gating:
 	ufshcd_exit_clk_gating(hba);
 out_disable:
 	hba->is_irq_enabled = false;
-	scsi_host_put(host);
 	ufshcd_hba_exit(hba);
 out_error:
 	return err;
diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h
index 430bef111293..7d9ff22acfea 100644
--- a/drivers/scsi/ufs/ufshcd.h
+++ b/drivers/scsi/ufs/ufshcd.h
@@ -261,6 +261,12 @@ struct ufs_pwr_mode_info {
  * @pwr_change_notify: called before and after a power mode change
  *			is carried out to allow vendor spesific capabilities
  *			to be set.
+ * @setup_xfer_req: called before any transfer request is issued
+ *                  to set some things
+ * @setup_task_mgmt: called before any task management request is issued
+ *                  to set some things
+ * @hibern8_notify: called around hibern8 enter/exit
+ *		    to configure some things
  * @suspend: called during host controller PM callback
  * @resume: called during host controller PM callback
  * @dbg_register_dump: used to dump controller debug information
@@ -273,7 +279,8 @@ struct ufs_hba_variant_ops {
 	u32	(*get_ufs_hci_version)(struct ufs_hba *);
 	int	(*clk_scale_notify)(struct ufs_hba *, bool,
 				    enum ufs_notify_change_status);
-	int	(*setup_clocks)(struct ufs_hba *, bool);
+	int	(*setup_clocks)(struct ufs_hba *, bool,
+				enum ufs_notify_change_status);
 	int     (*setup_regulators)(struct ufs_hba *, bool);
 	int	(*hce_enable_notify)(struct ufs_hba *,
 				     enum ufs_notify_change_status);
@@ -283,6 +290,10 @@ struct ufs_hba_variant_ops {
 					enum ufs_notify_change_status status,
 					struct ufs_pa_layer_attr *,
 					struct ufs_pa_layer_attr *);
+	void	(*setup_xfer_req)(struct ufs_hba *, int, bool);
+	void	(*setup_task_mgmt)(struct ufs_hba *, int, u8);
+	void    (*hibern8_notify)(struct ufs_hba *, enum uic_cmd_dme,
+				       enum ufs_notify_change_status);
 	int     (*suspend)(struct ufs_hba *, enum ufs_pm_op);
 	int     (*resume)(struct ufs_hba *, enum ufs_pm_op);
 	void	(*dbg_register_dump)(struct ufs_hba *hba);
@@ -474,6 +485,12 @@ struct ufs_hba {
 	 */
 	#define UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION		UFS_BIT(5)
 
+	/*
+	 * This quirk needs to be enabled if the host contoller regards
+	 * resolution of the values of PRDTO and PRDTL in UTRD as byte.
+	 */
+	#define UFSHCD_QUIRK_PRDT_BYTE_GRAN			UFS_BIT(7)
+
 	unsigned int quirks;	/* Deviations from standard UFSHCI spec. */
 
 	/* Device deviations from standard UFS device spec. */
@@ -755,10 +772,11 @@ static inline int ufshcd_vops_clk_scale_notify(struct ufs_hba *hba,
 	return 0;
 }
 
-static inline int ufshcd_vops_setup_clocks(struct ufs_hba *hba, bool on)
+static inline int ufshcd_vops_setup_clocks(struct ufs_hba *hba, bool on,
+					enum ufs_notify_change_status status)
 {
 	if (hba->vops && hba->vops->setup_clocks)
-		return hba->vops->setup_clocks(hba, on);
+		return hba->vops->setup_clocks(hba, on, status);
 	return 0;
 }
 
@@ -799,6 +817,28 @@ static inline int ufshcd_vops_pwr_change_notify(struct ufs_hba *hba,
 	return -ENOTSUPP;
 }
 
+static inline void ufshcd_vops_setup_xfer_req(struct ufs_hba *hba, int tag,
+					bool is_scsi_cmd)
+{
+	if (hba->vops && hba->vops->setup_xfer_req)
+		return hba->vops->setup_xfer_req(hba, tag, is_scsi_cmd);
+}
+
+static inline void ufshcd_vops_setup_task_mgmt(struct ufs_hba *hba,
+					int tag, u8 tm_function)
+{
+	if (hba->vops && hba->vops->setup_task_mgmt)
+		return hba->vops->setup_task_mgmt(hba, tag, tm_function);
+}
+
+static inline void ufshcd_vops_hibern8_notify(struct ufs_hba *hba,
+					enum uic_cmd_dme cmd,
+					enum ufs_notify_change_status status)
+{
+	if (hba->vops && hba->vops->hibern8_notify)
+		return hba->vops->hibern8_notify(hba, cmd, status);
+}
+
 static inline int ufshcd_vops_suspend(struct ufs_hba *hba, enum ufs_pm_op op)
 {
 	if (hba->vops && hba->vops->suspend)
diff --git a/drivers/scsi/ufs/ufshci.h b/drivers/scsi/ufs/ufshci.h
index 9599741ff606..5d978867be57 100644
--- a/drivers/scsi/ufs/ufshci.h
+++ b/drivers/scsi/ufs/ufshci.h
@@ -83,6 +83,8 @@ enum {
 	MASK_UIC_DME_TEST_MODE_SUPPORT		= 0x04000000,
 };
 
+#define UFS_MASK(mask, offset)		((mask) << (offset))
+
 /* UFS Version 08h */
 #define MINOR_VERSION_NUM_MASK		UFS_MASK(0xFFFF, 0)
 #define MAJOR_VERSION_NUM_MASK		UFS_MASK(0xFFFF, 16)
@@ -166,6 +168,7 @@ enum {
 /* UECPA - Host UIC Error Code PHY Adapter Layer 38h */
 #define UIC_PHY_ADAPTER_LAYER_ERROR			UFS_BIT(31)
 #define UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK		0x1F
+#define UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK		0xF
 
 /* UECDL - Host UIC Error Code Data Link Layer 3Ch */
 #define UIC_DATA_LINK_LAYER_ERROR		UFS_BIT(31)
diff --git a/drivers/scsi/ufs/unipro.h b/drivers/scsi/ufs/unipro.h
index eff8b5675575..23129d7b2678 100644
--- a/drivers/scsi/ufs/unipro.h
+++ b/drivers/scsi/ufs/unipro.h
@@ -123,6 +123,7 @@
 #define PA_MAXRXHSGEAR		0x1587
 #define PA_RXHSUNTERMCAP	0x15A5
 #define PA_RXLSTERMCAP		0x15A6
+#define PA_GRANULARITY		0x15AA
 #define PA_PACPREQTIMEOUT	0x1590
 #define PA_PACPREQEOBTIMEOUT	0x1591
 #define PA_HIBERN8TIME		0x15A7
@@ -158,6 +159,9 @@
 #define VS_DEBUGOMC		0xD09E
 #define VS_POWERSTATE		0xD083
 
+#define PA_GRANULARITY_MIN_VAL	1
+#define PA_GRANULARITY_MAX_VAL	6
+
 /* PHY Adapter Protocol Constants */
 #define PA_MAXDATALANES	4
 
diff --git a/drivers/scsi/xen-scsifront.c b/drivers/scsi/xen-scsifront.c
index 9dc8687bf048..9aa1fe1fc939 100644
--- a/drivers/scsi/xen-scsifront.c
+++ b/drivers/scsi/xen-scsifront.c
@@ -79,10 +79,13 @@
 struct vscsifrnt_shadow {
 	/* command between backend and frontend */
 	unsigned char act;
+	uint8_t nr_segments;
 	uint16_t rqid;
+	uint16_t ref_rqid;
 
 	unsigned int nr_grants;		/* number of grants in gref[] */
 	struct scsiif_request_segment *sg;	/* scatter/gather elements */
+	struct scsiif_request_segment seg[VSCSIIF_SG_TABLESIZE];
 
 	/* Do reset or abort function. */
 	wait_queue_head_t wq_reset;	/* reset work queue           */
@@ -172,68 +175,90 @@ static void scsifront_put_rqid(struct vscsifrnt_info *info, uint32_t id)
 		scsifront_wake_up(info);
 }
 
-static struct vscsiif_request *scsifront_pre_req(struct vscsifrnt_info *info)
+static int scsifront_do_request(struct vscsifrnt_info *info,
+				struct vscsifrnt_shadow *shadow)
 {
 	struct vscsiif_front_ring *ring = &(info->ring);
 	struct vscsiif_request *ring_req;
+	struct scsi_cmnd *sc = shadow->sc;
 	uint32_t id;
+	int i, notify;
+
+	if (RING_FULL(&info->ring))
+		return -EBUSY;
 
 	id = scsifront_get_rqid(info);	/* use id in response */
 	if (id >= VSCSIIF_MAX_REQS)
-		return NULL;
+		return -EBUSY;
 
-	ring_req = RING_GET_REQUEST(&(info->ring), ring->req_prod_pvt);
+	info->shadow[id] = shadow;
+	shadow->rqid = id;
 
+	ring_req = RING_GET_REQUEST(&(info->ring), ring->req_prod_pvt);
 	ring->req_prod_pvt++;
 
-	ring_req->rqid = (uint16_t)id;
+	ring_req->rqid        = id;
+	ring_req->act         = shadow->act;
+	ring_req->ref_rqid    = shadow->ref_rqid;
+	ring_req->nr_segments = shadow->nr_segments;
 
-	return ring_req;
-}
+	ring_req->id      = sc->device->id;
+	ring_req->lun     = sc->device->lun;
+	ring_req->channel = sc->device->channel;
+	ring_req->cmd_len = sc->cmd_len;
 
-static void scsifront_do_request(struct vscsifrnt_info *info)
-{
-	struct vscsiif_front_ring *ring = &(info->ring);
-	int notify;
+	BUG_ON(sc->cmd_len > VSCSIIF_MAX_COMMAND_SIZE);
+
+	memcpy(ring_req->cmnd, sc->cmnd, sc->cmd_len);
+
+	ring_req->sc_data_direction   = (uint8_t)sc->sc_data_direction;
+	ring_req->timeout_per_command = sc->request->timeout / HZ;
+
+	for (i = 0; i < (shadow->nr_segments & ~VSCSIIF_SG_GRANT); i++)
+		ring_req->seg[i] = shadow->seg[i];
 
 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(ring, notify);
 	if (notify)
 		notify_remote_via_irq(info->irq);
+
+	return 0;
 }
 
-static void scsifront_gnttab_done(struct vscsifrnt_info *info, uint32_t id)
+static void scsifront_gnttab_done(struct vscsifrnt_info *info,
+				  struct vscsifrnt_shadow *shadow)
 {
-	struct vscsifrnt_shadow *s = info->shadow[id];
 	int i;
 
-	if (s->sc->sc_data_direction == DMA_NONE)
+	if (shadow->sc->sc_data_direction == DMA_NONE)
 		return;
 
-	for (i = 0; i < s->nr_grants; i++) {
-		if (unlikely(gnttab_query_foreign_access(s->gref[i]) != 0)) {
+	for (i = 0; i < shadow->nr_grants; i++) {
+		if (unlikely(gnttab_query_foreign_access(shadow->gref[i]))) {
 			shost_printk(KERN_ALERT, info->host, KBUILD_MODNAME
 				     "grant still in use by backend\n");
 			BUG();
 		}
-		gnttab_end_foreign_access(s->gref[i], 0, 0UL);
+		gnttab_end_foreign_access(shadow->gref[i], 0, 0UL);
 	}
 
-	kfree(s->sg);
+	kfree(shadow->sg);
 }
 
 static void scsifront_cdb_cmd_done(struct vscsifrnt_info *info,
 				   struct vscsiif_response *ring_rsp)
 {
+	struct vscsifrnt_shadow *shadow;
 	struct scsi_cmnd *sc;
 	uint32_t id;
 	uint8_t sense_len;
 
 	id = ring_rsp->rqid;
-	sc = info->shadow[id]->sc;
+	shadow = info->shadow[id];
+	sc = shadow->sc;
 
 	BUG_ON(sc == NULL);
 
-	scsifront_gnttab_done(info, id);
+	scsifront_gnttab_done(info, shadow);
 	scsifront_put_rqid(info, id);
 
 	sc->result = ring_rsp->rslt;
@@ -366,7 +391,6 @@ static void scsifront_finish_all(struct vscsifrnt_info *info)
 
 static int map_data_for_request(struct vscsifrnt_info *info,
 				struct scsi_cmnd *sc,
-				struct vscsiif_request *ring_req,
 				struct vscsifrnt_shadow *shadow)
 {
 	grant_ref_t gref_head;
@@ -379,7 +403,6 @@ static int map_data_for_request(struct vscsifrnt_info *info,
 	struct scatterlist *sg;
 	struct scsiif_request_segment *seg;
 
-	ring_req->nr_segments = 0;
 	if (sc->sc_data_direction == DMA_NONE || !data_len)
 		return 0;
 
@@ -398,7 +421,7 @@ static int map_data_for_request(struct vscsifrnt_info *info,
 		if (!shadow->sg)
 			return -ENOMEM;
 	}
-	seg = shadow->sg ? : ring_req->seg;
+	seg = shadow->sg ? : shadow->seg;
 
 	err = gnttab_alloc_grant_references(seg_grants + data_grants,
 					    &gref_head);
@@ -423,9 +446,9 @@ static int map_data_for_request(struct vscsifrnt_info *info,
 				info->dev->otherend_id,
 				xen_page_to_gfn(page), 1);
 			shadow->gref[ref_cnt] = ref;
-			ring_req->seg[ref_cnt].gref   = ref;
-			ring_req->seg[ref_cnt].offset = (uint16_t)off;
-			ring_req->seg[ref_cnt].length = (uint16_t)bytes;
+			shadow->seg[ref_cnt].gref   = ref;
+			shadow->seg[ref_cnt].offset = (uint16_t)off;
+			shadow->seg[ref_cnt].length = (uint16_t)bytes;
 
 			page++;
 			len -= bytes;
@@ -473,44 +496,14 @@ static int map_data_for_request(struct vscsifrnt_info *info,
 	}
 
 	if (seg_grants)
-		ring_req->nr_segments = VSCSIIF_SG_GRANT | seg_grants;
+		shadow->nr_segments = VSCSIIF_SG_GRANT | seg_grants;
 	else
-		ring_req->nr_segments = (uint8_t)ref_cnt;
+		shadow->nr_segments = (uint8_t)ref_cnt;
 	shadow->nr_grants = ref_cnt;
 
 	return 0;
 }
 
-static struct vscsiif_request *scsifront_command2ring(
-		struct vscsifrnt_info *info, struct scsi_cmnd *sc,
-		struct vscsifrnt_shadow *shadow)
-{
-	struct vscsiif_request *ring_req;
-
-	memset(shadow, 0, sizeof(*shadow));
-
-	ring_req = scsifront_pre_req(info);
-	if (!ring_req)
-		return NULL;
-
-	info->shadow[ring_req->rqid] = shadow;
-	shadow->rqid = ring_req->rqid;
-
-	ring_req->id      = sc->device->id;
-	ring_req->lun     = sc->device->lun;
-	ring_req->channel = sc->device->channel;
-	ring_req->cmd_len = sc->cmd_len;
-
-	BUG_ON(sc->cmd_len > VSCSIIF_MAX_COMMAND_SIZE);
-
-	memcpy(ring_req->cmnd, sc->cmnd, sc->cmd_len);
-
-	ring_req->sc_data_direction   = (uint8_t)sc->sc_data_direction;
-	ring_req->timeout_per_command = sc->request->timeout / HZ;
-
-	return ring_req;
-}
-
 static int scsifront_enter(struct vscsifrnt_info *info)
 {
 	if (info->pause)
@@ -536,36 +529,25 @@ static int scsifront_queuecommand(struct Scsi_Host *shost,
 				  struct scsi_cmnd *sc)
 {
 	struct vscsifrnt_info *info = shost_priv(shost);
-	struct vscsiif_request *ring_req;
 	struct vscsifrnt_shadow *shadow = scsi_cmd_priv(sc);
 	unsigned long flags;
 	int err;
-	uint16_t rqid;
+
+	sc->result = 0;
+	memset(shadow, 0, sizeof(*shadow));
+
+	shadow->sc  = sc;
+	shadow->act = VSCSIIF_ACT_SCSI_CDB;
 
 	spin_lock_irqsave(shost->host_lock, flags);
 	if (scsifront_enter(info)) {
 		spin_unlock_irqrestore(shost->host_lock, flags);
 		return SCSI_MLQUEUE_HOST_BUSY;
 	}
-	if (RING_FULL(&info->ring))
-		goto busy;
-
-	ring_req = scsifront_command2ring(info, sc, shadow);
-	if (!ring_req)
-		goto busy;
-
-	sc->result = 0;
-
-	rqid = ring_req->rqid;
-	ring_req->act = VSCSIIF_ACT_SCSI_CDB;
 
-	shadow->sc  = sc;
-	shadow->act = VSCSIIF_ACT_SCSI_CDB;
-
-	err = map_data_for_request(info, sc, ring_req, shadow);
+	err = map_data_for_request(info, sc, shadow);
 	if (err < 0) {
 		pr_debug("%s: err %d\n", __func__, err);
-		scsifront_put_rqid(info, rqid);
 		scsifront_return(info);
 		spin_unlock_irqrestore(shost->host_lock, flags);
 		if (err == -ENOMEM)
@@ -575,7 +557,11 @@ static int scsifront_queuecommand(struct Scsi_Host *shost,
 		return 0;
 	}
 
-	scsifront_do_request(info);
+	if (scsifront_do_request(info, shadow)) {
+		scsifront_gnttab_done(info, shadow);
+		goto busy;
+	}
+
 	scsifront_return(info);
 	spin_unlock_irqrestore(shost->host_lock, flags);
 
@@ -598,26 +584,30 @@ static int scsifront_action_handler(struct scsi_cmnd *sc, uint8_t act)
 	struct Scsi_Host *host = sc->device->host;
 	struct vscsifrnt_info *info = shost_priv(host);
 	struct vscsifrnt_shadow *shadow, *s = scsi_cmd_priv(sc);
-	struct vscsiif_request *ring_req;
 	int err = 0;
 
-	shadow = kmalloc(sizeof(*shadow), GFP_NOIO);
+	shadow = kzalloc(sizeof(*shadow), GFP_NOIO);
 	if (!shadow)
 		return FAILED;
 
+	shadow->act = act;
+	shadow->rslt_reset = RSLT_RESET_WAITING;
+	shadow->sc = sc;
+	shadow->ref_rqid = s->rqid;
+	init_waitqueue_head(&shadow->wq_reset);
+
 	spin_lock_irq(host->host_lock);
 
 	for (;;) {
-		if (!RING_FULL(&info->ring)) {
-			ring_req = scsifront_command2ring(info, sc, shadow);
-			if (ring_req)
-				break;
-		}
-		if (err || info->pause) {
-			spin_unlock_irq(host->host_lock);
-			kfree(shadow);
-			return FAILED;
-		}
+		if (scsifront_enter(info))
+			goto fail;
+
+		if (!scsifront_do_request(info, shadow))
+			break;
+
+		scsifront_return(info);
+		if (err)
+			goto fail;
 		info->wait_ring_available = 1;
 		spin_unlock_irq(host->host_lock);
 		err = wait_event_interruptible(info->wq_sync,
@@ -625,22 +615,6 @@ static int scsifront_action_handler(struct scsi_cmnd *sc, uint8_t act)
 		spin_lock_irq(host->host_lock);
 	}
 
-	if (scsifront_enter(info)) {
-		spin_unlock_irq(host->host_lock);
-		return FAILED;
-	}
-
-	ring_req->act = act;
-	ring_req->ref_rqid = s->rqid;
-
-	shadow->act = act;
-	shadow->rslt_reset = RSLT_RESET_WAITING;
-	init_waitqueue_head(&shadow->wq_reset);
-
-	ring_req->nr_segments = 0;
-
-	scsifront_do_request(info);
-
 	spin_unlock_irq(host->host_lock);
 	err = wait_event_interruptible(shadow->wq_reset, shadow->wait_reset);
 	spin_lock_irq(host->host_lock);
@@ -659,6 +633,11 @@ static int scsifront_action_handler(struct scsi_cmnd *sc, uint8_t act)
 	scsifront_return(info);
 	spin_unlock_irq(host->host_lock);
 	return err;
+
+fail:
+	spin_unlock_irq(host->host_lock);
+	kfree(shadow);
+	return FAILED;
 }
 
 static int scsifront_eh_abort_handler(struct scsi_cmnd *sc)
@@ -1060,13 +1039,9 @@ static void scsifront_read_backend_params(struct xenbus_device *dev,
 					  struct vscsifrnt_info *info)
 {
 	unsigned int sg_grant, nr_segs;
-	int ret;
 	struct Scsi_Host *host = info->host;
 
-	ret = xenbus_scanf(XBT_NIL, dev->otherend, "feature-sg-grant", "%u",
-			   &sg_grant);
-	if (ret != 1)
-		sg_grant = 0;
+	sg_grant = xenbus_read_unsigned(dev->otherend, "feature-sg-grant", 0);
 	nr_segs = min_t(unsigned int, sg_grant, SG_ALL);
 	nr_segs = max_t(unsigned int, nr_segs, VSCSIIF_SG_TABLESIZE);
 	nr_segs = min_t(unsigned int, nr_segs,
diff --git a/drivers/soc/Kconfig b/drivers/soc/Kconfig
index e6e90e80519a..f31bceb69c0d 100644
--- a/drivers/soc/Kconfig
+++ b/drivers/soc/Kconfig
@@ -1,8 +1,7 @@
 menu "SOC (System On Chip) specific Drivers"
 
 source "drivers/soc/bcm/Kconfig"
-source "drivers/soc/fsl/qbman/Kconfig"
-source "drivers/soc/fsl/qe/Kconfig"
+source "drivers/soc/fsl/Kconfig"
 source "drivers/soc/mediatek/Kconfig"
 source "drivers/soc/qcom/Kconfig"
 source "drivers/soc/rockchip/Kconfig"
diff --git a/drivers/soc/fsl/Kconfig b/drivers/soc/fsl/Kconfig
new file mode 100644
index 000000000000..7a9fb9baa66d
--- /dev/null
+++ b/drivers/soc/fsl/Kconfig
@@ -0,0 +1,18 @@
+#
+# Freescale SOC drivers
+#
+
+source "drivers/soc/fsl/qbman/Kconfig"
+source "drivers/soc/fsl/qe/Kconfig"
+
+config FSL_GUTS
+	bool
+	select SOC_BUS
+	help
+	  The global utilities block controls power management, I/O device
+	  enabling, power-onreset(POR) configuration monitoring, alternate
+	  function selection for multiplexed signals,and clock control.
+	  This driver is to manage and access global utilities block.
+	  Initially only reading SVR and registering soc device are supported.
+	  Other guts accesses, such as reading RCW, should eventually be moved
+	  into this driver as well.
diff --git a/drivers/soc/fsl/Makefile b/drivers/soc/fsl/Makefile
index 75e1f5334821..44b3bebef24a 100644
--- a/drivers/soc/fsl/Makefile
+++ b/drivers/soc/fsl/Makefile
@@ -5,3 +5,4 @@
 obj-$(CONFIG_FSL_DPAA)                 += qbman/
 obj-$(CONFIG_QUICC_ENGINE)		+= qe/
 obj-$(CONFIG_CPM)			+= qe/
+obj-$(CONFIG_FSL_GUTS)			+= guts.o
diff --git a/drivers/soc/fsl/guts.c b/drivers/soc/fsl/guts.c
new file mode 100644
index 000000000000..6af7a11f09a5
--- /dev/null
+++ b/drivers/soc/fsl/guts.c
@@ -0,0 +1,239 @@
+/*
+ * Freescale QorIQ Platforms GUTS Driver
+ *
+ * Copyright (C) 2016 Freescale Semiconductor, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/of_fdt.h>
+#include <linux/sys_soc.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/fsl/guts.h>
+
+struct guts {
+	struct ccsr_guts __iomem *regs;
+	bool little_endian;
+};
+
+struct fsl_soc_die_attr {
+	char	*die;
+	u32	svr;
+	u32	mask;
+};
+
+static struct guts *guts;
+static struct soc_device_attribute soc_dev_attr;
+static struct soc_device *soc_dev;
+
+
+/* SoC die attribute definition for QorIQ platform */
+static const struct fsl_soc_die_attr fsl_soc_die[] = {
+	/*
+	 * Power Architecture-based SoCs T Series
+	 */
+
+	/* Die: T4240, SoC: T4240/T4160/T4080 */
+	{ .die		= "T4240",
+	  .svr		= 0x82400000,
+	  .mask		= 0xfff00000,
+	},
+	/* Die: T1040, SoC: T1040/T1020/T1042/T1022 */
+	{ .die		= "T1040",
+	  .svr		= 0x85200000,
+	  .mask		= 0xfff00000,
+	},
+	/* Die: T2080, SoC: T2080/T2081 */
+	{ .die		= "T2080",
+	  .svr		= 0x85300000,
+	  .mask		= 0xfff00000,
+	},
+	/* Die: T1024, SoC: T1024/T1014/T1023/T1013 */
+	{ .die		= "T1024",
+	  .svr		= 0x85400000,
+	  .mask		= 0xfff00000,
+	},
+
+	/*
+	 * ARM-based SoCs LS Series
+	 */
+
+	/* Die: LS1043A, SoC: LS1043A/LS1023A */
+	{ .die		= "LS1043A",
+	  .svr		= 0x87920000,
+	  .mask		= 0xffff0000,
+	},
+	/* Die: LS2080A, SoC: LS2080A/LS2040A/LS2085A */
+	{ .die		= "LS2080A",
+	  .svr		= 0x87010000,
+	  .mask		= 0xff3f0000,
+	},
+	/* Die: LS1088A, SoC: LS1088A/LS1048A/LS1084A/LS1044A */
+	{ .die		= "LS1088A",
+	  .svr		= 0x87030000,
+	  .mask		= 0xff3f0000,
+	},
+	/* Die: LS1012A, SoC: LS1012A */
+	{ .die		= "LS1012A",
+	  .svr		= 0x87040000,
+	  .mask		= 0xffff0000,
+	},
+	/* Die: LS1046A, SoC: LS1046A/LS1026A */
+	{ .die		= "LS1046A",
+	  .svr		= 0x87070000,
+	  .mask		= 0xffff0000,
+	},
+	/* Die: LS2088A, SoC: LS2088A/LS2048A/LS2084A/LS2044A */
+	{ .die		= "LS2088A",
+	  .svr		= 0x87090000,
+	  .mask		= 0xff3f0000,
+	},
+	/* Die: LS1021A, SoC: LS1021A/LS1020A/LS1022A */
+	{ .die		= "LS1021A",
+	  .svr		= 0x87000000,
+	  .mask		= 0xfff70000,
+	},
+	{ },
+};
+
+static const struct fsl_soc_die_attr *fsl_soc_die_match(
+	u32 svr, const struct fsl_soc_die_attr *matches)
+{
+	while (matches->svr) {
+		if (matches->svr == (svr & matches->mask))
+			return matches;
+		matches++;
+	};
+	return NULL;
+}
+
+u32 fsl_guts_get_svr(void)
+{
+	u32 svr = 0;
+
+	if (!guts || !guts->regs)
+		return svr;
+
+	if (guts->little_endian)
+		svr = ioread32(&guts->regs->svr);
+	else
+		svr = ioread32be(&guts->regs->svr);
+
+	return svr;
+}
+EXPORT_SYMBOL(fsl_guts_get_svr);
+
+static int fsl_guts_probe(struct platform_device *pdev)
+{
+	struct device_node *root, *np = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	const struct fsl_soc_die_attr *soc_die;
+	const char *machine;
+	u32 svr;
+
+	/* Initialize guts */
+	guts = devm_kzalloc(dev, sizeof(*guts), GFP_KERNEL);
+	if (!guts)
+		return -ENOMEM;
+
+	guts->little_endian = of_property_read_bool(np, "little-endian");
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	guts->regs = devm_ioremap_resource(dev, res);
+	if (IS_ERR(guts->regs))
+		return PTR_ERR(guts->regs);
+
+	/* Register soc device */
+	root = of_find_node_by_path("/");
+	if (of_property_read_string(root, "model", &machine))
+		of_property_read_string_index(root, "compatible", 0, &machine);
+	of_node_put(root);
+	if (machine)
+		soc_dev_attr.machine = devm_kstrdup(dev, machine, GFP_KERNEL);
+
+	svr = fsl_guts_get_svr();
+	soc_die = fsl_soc_die_match(svr, fsl_soc_die);
+	if (soc_die) {
+		soc_dev_attr.family = devm_kasprintf(dev, GFP_KERNEL,
+						     "QorIQ %s", soc_die->die);
+	} else {
+		soc_dev_attr.family = devm_kasprintf(dev, GFP_KERNEL, "QorIQ");
+	}
+	soc_dev_attr.soc_id = devm_kasprintf(dev, GFP_KERNEL,
+					     "svr:0x%08x", svr);
+	soc_dev_attr.revision = devm_kasprintf(dev, GFP_KERNEL, "%d.%d",
+					       (svr >>  4) & 0xf, svr & 0xf);
+
+	soc_dev = soc_device_register(&soc_dev_attr);
+	if (IS_ERR(soc_dev))
+		return PTR_ERR(soc_dev);
+
+	pr_info("Machine: %s\n", soc_dev_attr.machine);
+	pr_info("SoC family: %s\n", soc_dev_attr.family);
+	pr_info("SoC ID: %s, Revision: %s\n",
+		soc_dev_attr.soc_id, soc_dev_attr.revision);
+	return 0;
+}
+
+static int fsl_guts_remove(struct platform_device *dev)
+{
+	soc_device_unregister(soc_dev);
+	return 0;
+}
+
+/*
+ * Table for matching compatible strings, for device tree
+ * guts node, for Freescale QorIQ SOCs.
+ */
+static const struct of_device_id fsl_guts_of_match[] = {
+	{ .compatible = "fsl,qoriq-device-config-1.0", },
+	{ .compatible = "fsl,qoriq-device-config-2.0", },
+	{ .compatible = "fsl,p1010-guts", },
+	{ .compatible = "fsl,p1020-guts", },
+	{ .compatible = "fsl,p1021-guts", },
+	{ .compatible = "fsl,p1022-guts", },
+	{ .compatible = "fsl,p1023-guts", },
+	{ .compatible = "fsl,p2020-guts", },
+	{ .compatible = "fsl,bsc9131-guts", },
+	{ .compatible = "fsl,bsc9132-guts", },
+	{ .compatible = "fsl,mpc8536-guts", },
+	{ .compatible = "fsl,mpc8544-guts", },
+	{ .compatible = "fsl,mpc8548-guts", },
+	{ .compatible = "fsl,mpc8568-guts", },
+	{ .compatible = "fsl,mpc8569-guts", },
+	{ .compatible = "fsl,mpc8572-guts", },
+	{ .compatible = "fsl,ls1021a-dcfg", },
+	{ .compatible = "fsl,ls1043a-dcfg", },
+	{ .compatible = "fsl,ls2080a-dcfg", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, fsl_guts_of_match);
+
+static struct platform_driver fsl_guts_driver = {
+	.driver = {
+		.name = "fsl-guts",
+		.of_match_table = fsl_guts_of_match,
+	},
+	.probe = fsl_guts_probe,
+	.remove = fsl_guts_remove,
+};
+
+static int __init fsl_guts_init(void)
+{
+	return platform_driver_register(&fsl_guts_driver);
+}
+core_initcall(fsl_guts_init);
+
+static void __exit fsl_guts_exit(void)
+{
+	platform_driver_unregister(&fsl_guts_driver);
+}
+module_exit(fsl_guts_exit);
diff --git a/drivers/soc/fsl/qbman/bman.c b/drivers/soc/fsl/qbman/bman.c
index ffa48fdbb1a9..a3d6d7cfa929 100644
--- a/drivers/soc/fsl/qbman/bman.c
+++ b/drivers/soc/fsl/qbman/bman.c
@@ -167,12 +167,12 @@ struct bm_portal {
 /* Cache-inhibited register access. */
 static inline u32 bm_in(struct bm_portal *p, u32 offset)
 {
-	return __raw_readl(p->addr.ci + offset);
+	return be32_to_cpu(__raw_readl(p->addr.ci + offset));
 }
 
 static inline void bm_out(struct bm_portal *p, u32 offset, u32 val)
 {
-	__raw_writel(val, p->addr.ci + offset);
+	__raw_writel(cpu_to_be32(val), p->addr.ci + offset);
 }
 
 /* Cache Enabled Portal Access */
@@ -188,7 +188,7 @@ static inline void bm_cl_touch_ro(struct bm_portal *p, u32 offset)
 
 static inline u32 bm_ce_in(struct bm_portal *p, u32 offset)
 {
-	return __raw_readl(p->addr.ce + offset);
+	return be32_to_cpu(__raw_readl(p->addr.ce + offset));
 }
 
 struct bman_portal {
@@ -391,7 +391,7 @@ static void bm_rcr_finish(struct bm_portal *portal)
 
 	i = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);
 	if (i != rcr_ptr2idx(rcr->cursor))
-		pr_crit("losing uncommited RCR entries\n");
+		pr_crit("losing uncommitted RCR entries\n");
 
 	i = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);
 	if (i != rcr->ci)
diff --git a/drivers/soc/fsl/qbman/bman_ccsr.c b/drivers/soc/fsl/qbman/bman_ccsr.c
index 9deb0524543f..a8e8389a6894 100644
--- a/drivers/soc/fsl/qbman/bman_ccsr.c
+++ b/drivers/soc/fsl/qbman/bman_ccsr.c
@@ -181,8 +181,7 @@ static int fsl_bman_probe(struct platform_device *pdev)
 			node->full_name);
 		return -ENXIO;
 	}
-	bm_ccsr_start = devm_ioremap(dev, res->start,
-				     res->end - res->start + 1);
+	bm_ccsr_start = devm_ioremap(dev, res->start, resource_size(res));
 	if (!bm_ccsr_start)
 		return -ENXIO;
 
diff --git a/drivers/soc/fsl/qbman/bman_portal.c b/drivers/soc/fsl/qbman/bman_portal.c
index 986f64690e6e..8354d4dabdad 100644
--- a/drivers/soc/fsl/qbman/bman_portal.c
+++ b/drivers/soc/fsl/qbman/bman_portal.c
@@ -126,15 +126,19 @@ static int bman_portal_probe(struct platform_device *pdev)
 	pcfg->irq = irq;
 
 	va = ioremap_prot(addr_phys[0]->start, resource_size(addr_phys[0]), 0);
-	if (!va)
+	if (!va) {
+		dev_err(dev, "ioremap::CE failed\n");
 		goto err_ioremap1;
+	}
 
 	pcfg->addr_virt[DPAA_PORTAL_CE] = va;
 
 	va = ioremap_prot(addr_phys[1]->start, resource_size(addr_phys[1]),
 			  _PAGE_GUARDED | _PAGE_NO_CACHE);
-	if (!va)
+	if (!va) {
+		dev_err(dev, "ioremap::CI failed\n");
 		goto err_ioremap2;
+	}
 
 	pcfg->addr_virt[DPAA_PORTAL_CI] = va;
 
@@ -150,8 +154,10 @@ static int bman_portal_probe(struct platform_device *pdev)
 	spin_unlock(&bman_lock);
 	pcfg->cpu = cpu;
 
-	if (!init_pcfg(pcfg))
-		goto err_ioremap2;
+	if (!init_pcfg(pcfg)) {
+		dev_err(dev, "portal init failed\n");
+		goto err_portal_init;
+	}
 
 	/* clear irq affinity if assigned cpu is offline */
 	if (!cpu_online(cpu))
@@ -159,10 +165,11 @@ static int bman_portal_probe(struct platform_device *pdev)
 
 	return 0;
 
+err_portal_init:
+	iounmap(pcfg->addr_virt[DPAA_PORTAL_CI]);
 err_ioremap2:
 	iounmap(pcfg->addr_virt[DPAA_PORTAL_CE]);
 err_ioremap1:
-	dev_err(dev, "ioremap failed\n");
 	return -ENXIO;
 }
 
diff --git a/drivers/soc/fsl/qbman/dpaa_sys.h b/drivers/soc/fsl/qbman/dpaa_sys.h
index b63fd72295c6..2eaf3184f61d 100644
--- a/drivers/soc/fsl/qbman/dpaa_sys.h
+++ b/drivers/soc/fsl/qbman/dpaa_sys.h
@@ -38,6 +38,7 @@
 #include <linux/kthread.h>
 #include <linux/vmalloc.h>
 #include <linux/platform_device.h>
+#include <linux/of.h>
 #include <linux/of_reserved_mem.h>
 #include <linux/prefetch.h>
 #include <linux/genalloc.h>
diff --git a/drivers/soc/fsl/qbman/qman.c b/drivers/soc/fsl/qbman/qman.c
index 119054bc922b..6f509f68085e 100644
--- a/drivers/soc/fsl/qbman/qman.c
+++ b/drivers/soc/fsl/qbman/qman.c
@@ -140,10 +140,10 @@ enum qm_mr_cmode {		/* matches QCSP_CFG::MM */
 struct qm_eqcr_entry {
 	u8 _ncw_verb; /* writes to this are non-coherent */
 	u8 dca;
-	u16 seqnum;
-	u32 orp;	/* 24-bit */
-	u32 fqid;	/* 24-bit */
-	u32 tag;
+	__be16 seqnum;
+	u8 __reserved[4];
+	__be32 fqid;	/* 24-bit */
+	__be32 tag;
 	struct qm_fd fd;
 	u8 __reserved3[32];
 } __packed;
@@ -183,41 +183,22 @@ struct qm_mr {
 };
 
 /* MC (Management Command) command */
-/* "Query FQ" */
-struct qm_mcc_queryfq {
+/* "FQ" command layout */
+struct qm_mcc_fq {
 	u8 _ncw_verb;
 	u8 __reserved1[3];
-	u32 fqid;	/* 24-bit */
+	__be32 fqid;	/* 24-bit */
 	u8 __reserved2[56];
 } __packed;
-/* "Alter FQ State Commands " */
-struct qm_mcc_alterfq {
-	u8 _ncw_verb;
-	u8 __reserved1[3];
-	u32 fqid;	/* 24-bit */
-	u8 __reserved2;
-	u8 count;	/* number of consecutive FQID */
-	u8 __reserved3[10];
-	u32 context_b;	/* frame queue context b */
-	u8 __reserved4[40];
-} __packed;
 
-/* "Query CGR" */
-struct qm_mcc_querycgr {
+/* "CGR" command layout */
+struct qm_mcc_cgr {
 	u8 _ncw_verb;
 	u8 __reserved1[30];
 	u8 cgid;
 	u8 __reserved2[32];
 };
 
-struct qm_mcc_querywq {
-	u8 _ncw_verb;
-	u8 __reserved;
-	/* select channel if verb != QUERYWQ_DEDICATED */
-	u16 channel_wq; /* ignores wq (3 lsbits): _res[0-2] */
-	u8 __reserved2[60];
-} __packed;
-
 #define QM_MCC_VERB_VBIT		0x80
 #define QM_MCC_VERB_MASK		0x7f	/* where the verb contains; */
 #define QM_MCC_VERB_INITFQ_PARKED	0x40
@@ -243,12 +224,9 @@ union qm_mc_command {
 		u8 __reserved[63];
 	};
 	struct qm_mcc_initfq initfq;
-	struct qm_mcc_queryfq queryfq;
-	struct qm_mcc_alterfq alterfq;
 	struct qm_mcc_initcgr initcgr;
-	struct qm_mcc_querycgr querycgr;
-	struct qm_mcc_querywq querywq;
-	struct qm_mcc_queryfq_np queryfq_np;
+	struct qm_mcc_fq fq;
+	struct qm_mcc_cgr cgr;
 };
 
 /* MC (Management Command) result */
@@ -343,12 +321,12 @@ struct qm_portal {
 /* Cache-inhibited register access. */
 static inline u32 qm_in(struct qm_portal *p, u32 offset)
 {
-	return __raw_readl(p->addr.ci + offset);
+	return be32_to_cpu(__raw_readl(p->addr.ci + offset));
 }
 
 static inline void qm_out(struct qm_portal *p, u32 offset, u32 val)
 {
-	__raw_writel(val, p->addr.ci + offset);
+	__raw_writel(cpu_to_be32(val), p->addr.ci + offset);
 }
 
 /* Cache Enabled Portal Access */
@@ -364,7 +342,7 @@ static inline void qm_cl_touch_ro(struct qm_portal *p, u32 offset)
 
 static inline u32 qm_ce_in(struct qm_portal *p, u32 offset)
 {
-	return __raw_readl(p->addr.ce + offset);
+	return be32_to_cpu(__raw_readl(p->addr.ce + offset));
 }
 
 /* --- EQCR API --- */
@@ -443,7 +421,7 @@ static inline void qm_eqcr_finish(struct qm_portal *portal)
 
 	DPAA_ASSERT(!eqcr->busy);
 	if (pi != eqcr_ptr2idx(eqcr->cursor))
-		pr_crit("losing uncommited EQCR entries\n");
+		pr_crit("losing uncommitted EQCR entries\n");
 	if (ci != eqcr->ci)
 		pr_crit("missing existing EQCR completions\n");
 	if (eqcr->ci != eqcr_ptr2idx(eqcr->cursor))
@@ -492,8 +470,7 @@ static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal
 static inline void eqcr_commit_checks(struct qm_eqcr *eqcr)
 {
 	DPAA_ASSERT(eqcr->busy);
-	DPAA_ASSERT(eqcr->cursor->orp == (eqcr->cursor->orp & 0x00ffffff));
-	DPAA_ASSERT(eqcr->cursor->fqid == (eqcr->cursor->fqid & 0x00ffffff));
+	DPAA_ASSERT(!(be32_to_cpu(eqcr->cursor->fqid) & ~QM_FQID_MASK));
 	DPAA_ASSERT(eqcr->available >= 1);
 }
 
@@ -962,8 +939,6 @@ struct qman_portal {
 	u32 sdqcr;
 	/* probing time config params for cpu-affine portals */
 	const struct qm_portal_config *config;
-	/* needed for providing a non-NULL device to dma_map_***() */
-	struct platform_device *pdev;
 	/* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
 	struct qman_cgrs *cgrs;
 	/* linked-list of CSCN handlers. */
@@ -1133,7 +1108,6 @@ static int qman_create_portal(struct qman_portal *portal,
 			      const struct qman_cgrs *cgrs)
 {
 	struct qm_portal *p;
-	char buf[16];
 	int ret;
 	u32 isdr;
 
@@ -1196,15 +1170,6 @@ static int qman_create_portal(struct qman_portal *portal,
 	portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
 			QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS |
 			QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;
-	sprintf(buf, "qportal-%d", c->channel);
-	portal->pdev = platform_device_alloc(buf, -1);
-	if (!portal->pdev)
-		goto fail_devalloc;
-	if (dma_set_mask(&portal->pdev->dev, DMA_BIT_MASK(40)))
-		goto fail_devadd;
-	ret = platform_device_add(portal->pdev);
-	if (ret)
-		goto fail_devadd;
 	isdr = 0xffffffff;
 	qm_out(p, QM_REG_ISDR, isdr);
 	portal->irq_sources = 0;
@@ -1239,8 +1204,8 @@ static int qman_create_portal(struct qman_portal *portal,
 		/* special handling, drain just in case it's a few FQRNIs */
 		const union qm_mr_entry *e = qm_mr_current(p);
 
-		dev_err(c->dev, "MR dirty, VB 0x%x, rc 0x%x\n, addr 0x%x",
-			e->verb, e->ern.rc, e->ern.fd.addr_lo);
+		dev_err(c->dev, "MR dirty, VB 0x%x, rc 0x%x, addr 0x%llx\n",
+			e->verb, e->ern.rc, qm_fd_addr_get64(&e->ern.fd));
 		goto fail_dqrr_mr_empty;
 	}
 	/* Success */
@@ -1256,10 +1221,6 @@ fail_eqcr_empty:
 fail_affinity:
 	free_irq(c->irq, portal);
 fail_irq:
-	platform_device_del(portal->pdev);
-fail_devadd:
-	platform_device_put(portal->pdev);
-fail_devalloc:
 	kfree(portal->cgrs);
 fail_cgrs:
 	qm_mc_finish(p);
@@ -1321,9 +1282,6 @@ static void qman_destroy_portal(struct qman_portal *qm)
 	qm_dqrr_finish(&qm->p);
 	qm_eqcr_finish(&qm->p);
 
-	platform_device_del(qm->pdev);
-	platform_device_put(qm->pdev);
-
 	qm->config = NULL;
 }
 
@@ -1428,7 +1386,7 @@ static void qm_mr_process_task(struct work_struct *work)
 			case QM_MR_VERB_FQRN:
 			case QM_MR_VERB_FQRL:
 				/* Lookup in the retirement table */
-				fq = fqid_to_fq(msg->fq.fqid);
+				fq = fqid_to_fq(qm_fqid_get(&msg->fq));
 				if (WARN_ON(!fq))
 					break;
 				fq_state_change(p, fq, msg, verb);
@@ -1437,7 +1395,7 @@ static void qm_mr_process_task(struct work_struct *work)
 				break;
 			case QM_MR_VERB_FQPN:
 				/* Parked */
-				fq = tag_to_fq(msg->fq.contextB);
+				fq = tag_to_fq(be32_to_cpu(msg->fq.context_b));
 				fq_state_change(p, fq, msg, verb);
 				if (fq->cb.fqs)
 					fq->cb.fqs(p, fq, msg);
@@ -1451,7 +1409,7 @@ static void qm_mr_process_task(struct work_struct *work)
 			}
 		} else {
 			/* Its a software ERN */
-			fq = tag_to_fq(msg->ern.tag);
+			fq = tag_to_fq(be32_to_cpu(msg->ern.tag));
 			fq->cb.ern(p, fq, msg);
 		}
 		num++;
@@ -1536,7 +1494,7 @@ static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 
 		if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
 			/*
-			 * VDQCR: don't trust contextB as the FQ may have
+			 * VDQCR: don't trust context_b as the FQ may have
 			 * been configured for h/w consumption and we're
 			 * draining it post-retirement.
 			 */
@@ -1562,8 +1520,8 @@ static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 			if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
 				clear_vdqcr(p, fq);
 		} else {
-			/* SDQCR: contextB points to the FQ */
-			fq = tag_to_fq(dq->contextB);
+			/* SDQCR: context_b points to the FQ */
+			fq = tag_to_fq(be32_to_cpu(dq->context_b));
 			/* Now let the callback do its stuff */
 			res = fq->cb.dqrr(p, fq, dq);
 			/*
@@ -1780,9 +1738,9 @@ int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
 	if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
 		return -EINVAL;
 #endif
-	if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) {
+	if (opts && (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_OAC)) {
 		/* And can't be set at the same time as TDTHRESH */
-		if (opts->we_mask & QM_INITFQ_WE_TDTHRESH)
+		if (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_TDTHRESH)
 			return -EINVAL;
 	}
 	/* Issue an INITFQ_[PARKED|SCHED] management command */
@@ -1796,37 +1754,49 @@ int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
 	mcc = qm_mc_start(&p->p);
 	if (opts)
 		mcc->initfq = *opts;
-	mcc->initfq.fqid = fq->fqid;
+	qm_fqid_set(&mcc->fq, fq->fqid);
 	mcc->initfq.count = 0;
 	/*
-	 * If the FQ does *not* have the TO_DCPORTAL flag, contextB is set as a
+	 * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a
 	 * demux pointer. Otherwise, the caller-provided value is allowed to
 	 * stand, don't overwrite it.
 	 */
 	if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
 		dma_addr_t phys_fq;
 
-		mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB;
-		mcc->initfq.fqd.context_b = fq_to_tag(fq);
+		mcc->initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CONTEXTB);
+		mcc->initfq.fqd.context_b = cpu_to_be32(fq_to_tag(fq));
 		/*
 		 *  and the physical address - NB, if the user wasn't trying to
 		 * set CONTEXTA, clear the stashing settings.
 		 */
-		if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) {
-			mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA;
+		if (!(be16_to_cpu(mcc->initfq.we_mask) &
+				  QM_INITFQ_WE_CONTEXTA)) {
+			mcc->initfq.we_mask |=
+				cpu_to_be16(QM_INITFQ_WE_CONTEXTA);
 			memset(&mcc->initfq.fqd.context_a, 0,
 				sizeof(mcc->initfq.fqd.context_a));
 		} else {
-			phys_fq = dma_map_single(&p->pdev->dev, fq, sizeof(*fq),
-						 DMA_TO_DEVICE);
+			struct qman_portal *p = qman_dma_portal;
+
+			phys_fq = dma_map_single(p->config->dev, fq,
+						 sizeof(*fq), DMA_TO_DEVICE);
+			if (dma_mapping_error(p->config->dev, phys_fq)) {
+				dev_err(p->config->dev, "dma_mapping failed\n");
+				ret = -EIO;
+				goto out;
+			}
+
 			qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
 		}
 	}
 	if (flags & QMAN_INITFQ_FLAG_LOCAL) {
 		int wq = 0;
 
-		if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) {
-			mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ;
+		if (!(be16_to_cpu(mcc->initfq.we_mask) &
+				  QM_INITFQ_WE_DESTWQ)) {
+			mcc->initfq.we_mask |=
+				cpu_to_be16(QM_INITFQ_WE_DESTWQ);
 			wq = 4;
 		}
 		qm_fqd_set_destwq(&mcc->initfq.fqd, p->config->channel, wq);
@@ -1845,13 +1815,13 @@ int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
 		goto out;
 	}
 	if (opts) {
-		if (opts->we_mask & QM_INITFQ_WE_FQCTRL) {
-			if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE)
+		if (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_FQCTRL) {
+			if (be16_to_cpu(opts->fqd.fq_ctrl) & QM_FQCTRL_CGE)
 				fq_set(fq, QMAN_FQ_STATE_CGR_EN);
 			else
 				fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
 		}
-		if (opts->we_mask & QM_INITFQ_WE_CGID)
+		if (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_CGID)
 			fq->cgr_groupid = opts->fqd.cgid;
 	}
 	fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
@@ -1884,7 +1854,7 @@ int qman_schedule_fq(struct qman_fq *fq)
 		goto out;
 	}
 	mcc = qm_mc_start(&p->p);
-	mcc->alterfq.fqid = fq->fqid;
+	qm_fqid_set(&mcc->fq, fq->fqid);
 	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
 	if (!qm_mc_result_timeout(&p->p, &mcr)) {
 		dev_err(p->config->dev, "ALTER_SCHED timeout\n");
@@ -1927,7 +1897,7 @@ int qman_retire_fq(struct qman_fq *fq, u32 *flags)
 		goto out;
 	}
 	mcc = qm_mc_start(&p->p);
-	mcc->alterfq.fqid = fq->fqid;
+	qm_fqid_set(&mcc->fq, fq->fqid);
 	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
 	if (!qm_mc_result_timeout(&p->p, &mcr)) {
 		dev_crit(p->config->dev, "ALTER_RETIRE timeout\n");
@@ -1970,8 +1940,8 @@ int qman_retire_fq(struct qman_fq *fq, u32 *flags)
 
 			msg.verb = QM_MR_VERB_FQRNI;
 			msg.fq.fqs = mcr->alterfq.fqs;
-			msg.fq.fqid = fq->fqid;
-			msg.fq.contextB = fq_to_tag(fq);
+			qm_fqid_set(&msg.fq, fq->fqid);
+			msg.fq.context_b = cpu_to_be32(fq_to_tag(fq));
 			fq->cb.fqs(p, fq, &msg);
 		}
 	} else if (res == QM_MCR_RESULT_PENDING) {
@@ -2006,7 +1976,7 @@ int qman_oos_fq(struct qman_fq *fq)
 		goto out;
 	}
 	mcc = qm_mc_start(&p->p);
-	mcc->alterfq.fqid = fq->fqid;
+	qm_fqid_set(&mcc->fq, fq->fqid);
 	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
 	if (!qm_mc_result_timeout(&p->p, &mcr)) {
 		ret = -ETIMEDOUT;
@@ -2032,7 +2002,7 @@ int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
 	int ret = 0;
 
 	mcc = qm_mc_start(&p->p);
-	mcc->queryfq.fqid = fq->fqid;
+	qm_fqid_set(&mcc->fq, fq->fqid);
 	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
 	if (!qm_mc_result_timeout(&p->p, &mcr)) {
 		ret = -ETIMEDOUT;
@@ -2058,7 +2028,7 @@ static int qman_query_fq_np(struct qman_fq *fq,
 	int ret = 0;
 
 	mcc = qm_mc_start(&p->p);
-	mcc->queryfq.fqid = fq->fqid;
+	qm_fqid_set(&mcc->fq, fq->fqid);
 	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
 	if (!qm_mc_result_timeout(&p->p, &mcr)) {
 		ret = -ETIMEDOUT;
@@ -2086,7 +2056,7 @@ static int qman_query_cgr(struct qman_cgr *cgr,
 	int ret = 0;
 
 	mcc = qm_mc_start(&p->p);
-	mcc->querycgr.cgid = cgr->cgrid;
+	mcc->cgr.cgid = cgr->cgrid;
 	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
 	if (!qm_mc_result_timeout(&p->p, &mcr)) {
 		ret = -ETIMEDOUT;
@@ -2239,8 +2209,8 @@ int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd)
 	if (unlikely(!eq))
 		goto out;
 
-	eq->fqid = fq->fqid;
-	eq->tag = fq_to_tag(fq);
+	qm_fqid_set(eq, fq->fqid);
+	eq->tag = cpu_to_be32(fq_to_tag(fq));
 	eq->fd = *fd;
 
 	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE);
@@ -2282,7 +2252,24 @@ out:
 }
 
 #define PORTAL_IDX(n)	(n->config->channel - QM_CHANNEL_SWPORTAL0)
-#define TARG_MASK(n)	(BIT(31) >> PORTAL_IDX(n))
+
+/* congestion state change notification target update control */
+static void qm_cgr_cscn_targ_set(struct __qm_mc_cgr *cgr, int pi, u32 val)
+{
+	if (qman_ip_rev >= QMAN_REV30)
+		cgr->cscn_targ_upd_ctrl = cpu_to_be16(pi |
+					QM_CGR_TARG_UDP_CTRL_WRITE_BIT);
+	else
+		cgr->cscn_targ = cpu_to_be32(val | QM_CGR_TARG_PORTAL(pi));
+}
+
+static void qm_cgr_cscn_targ_clear(struct __qm_mc_cgr *cgr, int pi, u32 val)
+{
+	if (qman_ip_rev >= QMAN_REV30)
+		cgr->cscn_targ_upd_ctrl = cpu_to_be16(pi);
+	else
+		cgr->cscn_targ = cpu_to_be32(val & ~QM_CGR_TARG_PORTAL(pi));
+}
 
 static u8 qman_cgr_cpus[CGR_NUM];
 
@@ -2305,7 +2292,6 @@ int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
 		    struct qm_mcc_initcgr *opts)
 {
 	struct qm_mcr_querycgr cgr_state;
-	struct qm_mcc_initcgr local_opts = {};
 	int ret;
 	struct qman_portal *p;
 
@@ -2327,22 +2313,18 @@ int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
 	spin_lock(&p->cgr_lock);
 
 	if (opts) {
+		struct qm_mcc_initcgr local_opts = *opts;
+
 		ret = qman_query_cgr(cgr, &cgr_state);
 		if (ret)
 			goto out;
-		if (opts)
-			local_opts = *opts;
-		if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
-			local_opts.cgr.cscn_targ_upd_ctrl =
-				QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p);
-		else
-			/* Overwrite TARG */
-			local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
-						   TARG_MASK(p);
-		local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
+
+		qm_cgr_cscn_targ_set(&local_opts.cgr, PORTAL_IDX(p),
+				     be32_to_cpu(cgr_state.cgr.cscn_targ));
+		local_opts.we_mask |= cpu_to_be16(QM_CGR_WE_CSCN_TARG);
 
 		/* send init if flags indicate so */
-		if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
+		if (flags & QMAN_CGR_FLAG_USE_INIT)
 			ret = qm_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
 					    &local_opts);
 		else
@@ -2405,13 +2387,11 @@ int qman_delete_cgr(struct qman_cgr *cgr)
 		list_add(&cgr->node, &p->cgr_cbs);
 		goto release_lock;
 	}
-	/* Overwrite TARG */
-	local_opts.we_mask = QM_CGR_WE_CSCN_TARG;
-	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
-		local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p);
-	else
-		local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ &
-							 ~(TARG_MASK(p));
+
+	local_opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_TARG);
+	qm_cgr_cscn_targ_clear(&local_opts.cgr, PORTAL_IDX(p),
+			       be32_to_cpu(cgr_state.cgr.cscn_targ));
+
 	ret = qm_modify_cgr(cgr, 0, &local_opts);
 	if (ret)
 		/* add back to the list */
@@ -2501,7 +2481,7 @@ static int _qm_dqrr_consume_and_match(struct qm_portal *p, u32 fqid, int s,
 	} while (wait && !dqrr);
 
 	while (dqrr) {
-		if (dqrr->fqid == fqid && (dqrr->stat & s))
+		if (qm_fqid_get(dqrr) == fqid && (dqrr->stat & s))
 			found = 1;
 		qm_dqrr_cdc_consume_1ptr(p, dqrr, 0);
 		qm_dqrr_pvb_update(p);
@@ -2537,7 +2517,7 @@ static int qman_shutdown_fq(u32 fqid)
 	dev = p->config->dev;
 	/* Determine the state of the FQID */
 	mcc = qm_mc_start(&p->p);
-	mcc->queryfq_np.fqid = fqid;
+	qm_fqid_set(&mcc->fq, fqid);
 	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
 	if (!qm_mc_result_timeout(&p->p, &mcr)) {
 		dev_err(dev, "QUERYFQ_NP timeout\n");
@@ -2552,7 +2532,7 @@ static int qman_shutdown_fq(u32 fqid)
 
 	/* Query which channel the FQ is using */
 	mcc = qm_mc_start(&p->p);
-	mcc->queryfq.fqid = fqid;
+	qm_fqid_set(&mcc->fq, fqid);
 	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
 	if (!qm_mc_result_timeout(&p->p, &mcr)) {
 		dev_err(dev, "QUERYFQ timeout\n");
@@ -2572,7 +2552,7 @@ static int qman_shutdown_fq(u32 fqid)
 	case QM_MCR_NP_STATE_PARKED:
 		orl_empty = 0;
 		mcc = qm_mc_start(&p->p);
-		mcc->alterfq.fqid = fqid;
+		qm_fqid_set(&mcc->fq, fqid);
 		qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
 		if (!qm_mc_result_timeout(&p->p, &mcr)) {
 			dev_err(dev, "QUERYFQ_NP timeout\n");
@@ -2667,7 +2647,7 @@ static int qman_shutdown_fq(u32 fqid)
 			cpu_relax();
 		}
 		mcc = qm_mc_start(&p->p);
-		mcc->alterfq.fqid = fqid;
+		qm_fqid_set(&mcc->fq, fqid);
 		qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
 		if (!qm_mc_result_timeout(&p->p, &mcr)) {
 			ret = -ETIMEDOUT;
@@ -2687,7 +2667,7 @@ static int qman_shutdown_fq(u32 fqid)
 	case QM_MCR_NP_STATE_RETIRED:
 		/* Send OOS Command */
 		mcc = qm_mc_start(&p->p);
-		mcc->alterfq.fqid = fqid;
+		qm_fqid_set(&mcc->fq, fqid);
 		qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
 		if (!qm_mc_result_timeout(&p->p, &mcr)) {
 			ret = -ETIMEDOUT;
@@ -2722,6 +2702,7 @@ const struct qm_portal_config *qman_get_qm_portal_config(
 {
 	return portal->config;
 }
+EXPORT_SYMBOL(qman_get_qm_portal_config);
 
 struct gen_pool *qm_fqalloc; /* FQID allocator */
 struct gen_pool *qm_qpalloc; /* pool-channel allocator */
@@ -2789,15 +2770,18 @@ static int qpool_cleanup(u32 qp)
 		struct qm_mcr_queryfq_np np;
 
 		err = qman_query_fq_np(&fq, &np);
-		if (err)
+		if (err == -ERANGE)
 			/* FQID range exceeded, found no problems */
 			return 0;
+		else if (WARN_ON(err))
+			return err;
+
 		if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) {
 			struct qm_fqd fqd;
 
 			err = qman_query_fq(&fq, &fqd);
 			if (WARN_ON(err))
-				return 0;
+				return err;
 			if (qm_fqd_get_chan(&fqd) == qp) {
 				/* The channel is the FQ's target, clean it */
 				err = qman_shutdown_fq(fq.fqid);
@@ -2836,7 +2820,7 @@ static int cgr_cleanup(u32 cgrid)
 	 * error, looking for non-OOS FQDs whose CGR is the CGR being released
 	 */
 	struct qman_fq fq = {
-		.fqid = 1
+		.fqid = QM_FQID_RANGE_START
 	};
 	int err;
 
@@ -2844,16 +2828,19 @@ static int cgr_cleanup(u32 cgrid)
 		struct qm_mcr_queryfq_np np;
 
 		err = qman_query_fq_np(&fq, &np);
-		if (err)
+		if (err == -ERANGE)
 			/* FQID range exceeded, found no problems */
 			return 0;
+		else if (WARN_ON(err))
+			return err;
+
 		if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) {
 			struct qm_fqd fqd;
 
 			err = qman_query_fq(&fq, &fqd);
 			if (WARN_ON(err))
-				return 0;
-			if ((fqd.fq_ctrl & QM_FQCTRL_CGE) &&
+				return err;
+			if (be16_to_cpu(fqd.fq_ctrl) & QM_FQCTRL_CGE &&
 			    fqd.cgid == cgrid) {
 				pr_err("CRGID 0x%x is being used by FQID 0x%x, CGR will be leaked\n",
 				       cgrid, fq.fqid);
diff --git a/drivers/soc/fsl/qbman/qman_ccsr.c b/drivers/soc/fsl/qbman/qman_ccsr.c
index 0cace9e0077e..f4e6e70de259 100644
--- a/drivers/soc/fsl/qbman/qman_ccsr.c
+++ b/drivers/soc/fsl/qbman/qman_ccsr.c
@@ -444,6 +444,9 @@ static int zero_priv_mem(struct device *dev, struct device_node *node,
 	/* map as cacheable, non-guarded */
 	void __iomem *tmpp = ioremap_prot(addr, sz, 0);
 
+	if (!tmpp)
+		return -ENOMEM;
+
 	memset_io(tmpp, 0, sz);
 	flush_dcache_range((unsigned long)tmpp,
 			   (unsigned long)tmpp + sz);
diff --git a/drivers/soc/fsl/qbman/qman_portal.c b/drivers/soc/fsl/qbman/qman_portal.c
index d068e4820f49..adbaa30d3c5a 100644
--- a/drivers/soc/fsl/qbman/qman_portal.c
+++ b/drivers/soc/fsl/qbman/qman_portal.c
@@ -30,6 +30,9 @@
 
 #include "qman_priv.h"
 
+struct qman_portal *qman_dma_portal;
+EXPORT_SYMBOL(qman_dma_portal);
+
 /* Enable portal interupts (as opposed to polling mode) */
 #define CONFIG_FSL_DPA_PIRQ_SLOW  1
 #define CONFIG_FSL_DPA_PIRQ_FAST  1
@@ -150,6 +153,10 @@ static struct qman_portal *init_pcfg(struct qm_portal_config *pcfg)
 		/* all assigned portals are initialized now */
 		qman_init_cgr_all();
 	}
+
+	if (!qman_dma_portal)
+		qman_dma_portal = p;
+
 	spin_unlock(&qman_lock);
 
 	dev_info(pcfg->dev, "Portal initialised, cpu %d\n", pcfg->cpu);
@@ -217,9 +224,9 @@ static int qman_portal_probe(struct platform_device *pdev)
 	struct device_node *node = dev->of_node;
 	struct qm_portal_config *pcfg;
 	struct resource *addr_phys[2];
-	const u32 *channel;
 	void __iomem *va;
-	int irq, len, cpu;
+	int irq, cpu, err;
+	u32 val;
 
 	pcfg = devm_kmalloc(dev, sizeof(*pcfg), GFP_KERNEL);
 	if (!pcfg)
@@ -243,13 +250,13 @@ static int qman_portal_probe(struct platform_device *pdev)
 		return -ENXIO;
 	}
 
-	channel = of_get_property(node, "cell-index", &len);
-	if (!channel || (len != 4)) {
+	err = of_property_read_u32(node, "cell-index", &val);
+	if (err) {
 		dev_err(dev, "Can't get %s property 'cell-index'\n",
 			node->full_name);
-		return -ENXIO;
+		return err;
 	}
-	pcfg->channel = *channel;
+	pcfg->channel = val;
 	pcfg->cpu = -1;
 	irq = platform_get_irq(pdev, 0);
 	if (irq <= 0) {
@@ -259,15 +266,19 @@ static int qman_portal_probe(struct platform_device *pdev)
 	pcfg->irq = irq;
 
 	va = ioremap_prot(addr_phys[0]->start, resource_size(addr_phys[0]), 0);
-	if (!va)
+	if (!va) {
+		dev_err(dev, "ioremap::CE failed\n");
 		goto err_ioremap1;
+	}
 
 	pcfg->addr_virt[DPAA_PORTAL_CE] = va;
 
 	va = ioremap_prot(addr_phys[1]->start, resource_size(addr_phys[1]),
 			  _PAGE_GUARDED | _PAGE_NO_CACHE);
-	if (!va)
+	if (!va) {
+		dev_err(dev, "ioremap::CI failed\n");
 		goto err_ioremap2;
+	}
 
 	pcfg->addr_virt[DPAA_PORTAL_CI] = va;
 
@@ -285,8 +296,15 @@ static int qman_portal_probe(struct platform_device *pdev)
 	spin_unlock(&qman_lock);
 	pcfg->cpu = cpu;
 
-	if (!init_pcfg(pcfg))
-		goto err_ioremap2;
+	if (dma_set_mask(dev, DMA_BIT_MASK(40))) {
+		dev_err(dev, "dma_set_mask() failed\n");
+		goto err_portal_init;
+	}
+
+	if (!init_pcfg(pcfg)) {
+		dev_err(dev, "portal init failed\n");
+		goto err_portal_init;
+	}
 
 	/* clear irq affinity if assigned cpu is offline */
 	if (!cpu_online(cpu))
@@ -294,10 +312,11 @@ static int qman_portal_probe(struct platform_device *pdev)
 
 	return 0;
 
+err_portal_init:
+	iounmap(pcfg->addr_virt[DPAA_PORTAL_CI]);
 err_ioremap2:
 	iounmap(pcfg->addr_virt[DPAA_PORTAL_CE]);
 err_ioremap1:
-	dev_err(dev, "ioremap failed\n");
 	return -ENXIO;
 }
 
diff --git a/drivers/soc/fsl/qbman/qman_priv.h b/drivers/soc/fsl/qbman/qman_priv.h
index 5cf821e623a9..53685b59718e 100644
--- a/drivers/soc/fsl/qbman/qman_priv.h
+++ b/drivers/soc/fsl/qbman/qman_priv.h
@@ -73,29 +73,23 @@ struct qm_mcr_querycgr {
 	struct __qm_mc_cgr cgr; /* CGR fields */
 	u8 __reserved2[6];
 	u8 i_bcnt_hi;	/* high 8-bits of 40-bit "Instant" */
-	u32 i_bcnt_lo;	/* low 32-bits of 40-bit */
+	__be32 i_bcnt_lo;	/* low 32-bits of 40-bit */
 	u8 __reserved3[3];
 	u8 a_bcnt_hi;	/* high 8-bits of 40-bit "Average" */
-	u32 a_bcnt_lo;	/* low 32-bits of 40-bit */
-	u32 cscn_targ_swp[4];
+	__be32 a_bcnt_lo;	/* low 32-bits of 40-bit */
+	__be32 cscn_targ_swp[4];
 } __packed;
 
 static inline u64 qm_mcr_querycgr_i_get64(const struct qm_mcr_querycgr *q)
 {
-	return ((u64)q->i_bcnt_hi << 32) | (u64)q->i_bcnt_lo;
+	return ((u64)q->i_bcnt_hi << 32) | be32_to_cpu(q->i_bcnt_lo);
 }
 static inline u64 qm_mcr_querycgr_a_get64(const struct qm_mcr_querycgr *q)
 {
-	return ((u64)q->a_bcnt_hi << 32) | (u64)q->a_bcnt_lo;
+	return ((u64)q->a_bcnt_hi << 32) | be32_to_cpu(q->a_bcnt_lo);
 }
 
 /* "Query FQ Non-Programmable Fields" */
-struct qm_mcc_queryfq_np {
-	u8 _ncw_verb;
-	u8 __reserved1[3];
-	u32 fqid;	/* 24-bit */
-	u8 __reserved2[56];
-} __packed;
 
 struct qm_mcr_queryfq_np {
 	u8 verb;
@@ -367,5 +361,6 @@ int qman_alloc_fq_table(u32 num_fqids);
 #define QM_PIRQ_VISIBLE	(QM_PIRQ_SLOW | QM_PIRQ_DQRI)
 
 extern struct qman_portal *affine_portals[NR_CPUS];
+extern struct qman_portal *qman_dma_portal;
 const struct qm_portal_config *qman_get_qm_portal_config(
 						struct qman_portal *portal);
diff --git a/drivers/soc/fsl/qbman/qman_test_api.c b/drivers/soc/fsl/qbman/qman_test_api.c
index 6880ff17f45e..2895d062cf51 100644
--- a/drivers/soc/fsl/qbman/qman_test_api.c
+++ b/drivers/soc/fsl/qbman/qman_test_api.c
@@ -65,7 +65,7 @@ static void fd_init(struct qm_fd *fd)
 {
 	qm_fd_addr_set64(fd, 0xabdeadbeefLLU);
 	qm_fd_set_contig_big(fd, 0x0000ffff);
-	fd->cmd = 0xfeedf00d;
+	fd->cmd = cpu_to_be32(0xfeedf00d);
 }
 
 static void fd_inc(struct qm_fd *fd)
@@ -86,26 +86,19 @@ static void fd_inc(struct qm_fd *fd)
 	len--;
 	qm_fd_set_param(fd, fmt, off, len);
 
-	fd->cmd++;
+	fd->cmd = cpu_to_be32(be32_to_cpu(fd->cmd) + 1);
 }
 
 /* The only part of the 'fd' we can't memcmp() is the ppid */
-static int fd_cmp(const struct qm_fd *a, const struct qm_fd *b)
+static bool fd_neq(const struct qm_fd *a, const struct qm_fd *b)
 {
-	int r = (qm_fd_addr_get64(a) == qm_fd_addr_get64(b)) ? 0 : -1;
+	bool neq = qm_fd_addr_get64(a) != qm_fd_addr_get64(b);
 
-	if (!r) {
-		enum qm_fd_format fmt_a, fmt_b;
+	neq |= qm_fd_get_format(a) != qm_fd_get_format(b);
+	neq |= a->cfg != b->cfg;
+	neq |= a->cmd != b->cmd;
 
-		fmt_a = qm_fd_get_format(a);
-		fmt_b = qm_fd_get_format(b);
-		r = fmt_a - fmt_b;
-	}
-	if (!r)
-		r = a->cfg - b->cfg;
-	if (!r)
-		r = a->cmd - b->cmd;
-	return r;
+	return neq;
 }
 
 /* test */
@@ -217,12 +210,12 @@ static enum qman_cb_dqrr_result cb_dqrr(struct qman_portal *p,
 					struct qman_fq *fq,
 					const struct qm_dqrr_entry *dq)
 {
-	if (WARN_ON(fd_cmp(&fd_dq, &dq->fd))) {
+	if (WARN_ON(fd_neq(&fd_dq, &dq->fd))) {
 		pr_err("BADNESS: dequeued frame doesn't match;\n");
 		return qman_cb_dqrr_consume;
 	}
 	fd_inc(&fd_dq);
-	if (!(dq->stat & QM_DQRR_STAT_UNSCHEDULED) && !fd_cmp(&fd_dq, &fd)) {
+	if (!(dq->stat & QM_DQRR_STAT_UNSCHEDULED) && !fd_neq(&fd_dq, &fd)) {
 		sdqcr_complete = 1;
 		wake_up(&waitqueue);
 	}
diff --git a/drivers/soc/fsl/qbman/qman_test_stash.c b/drivers/soc/fsl/qbman/qman_test_stash.c
index 43cf66ba42f5..e87b65403b67 100644
--- a/drivers/soc/fsl/qbman/qman_test_stash.c
+++ b/drivers/soc/fsl/qbman/qman_test_stash.c
@@ -175,7 +175,7 @@ static DEFINE_PER_CPU(struct hp_cpu, hp_cpus);
 
 /* links together the hp_cpu structs, in first-come first-serve order. */
 static LIST_HEAD(hp_cpu_list);
-static spinlock_t hp_lock = __SPIN_LOCK_UNLOCKED(hp_lock);
+static DEFINE_SPINLOCK(hp_lock);
 
 static unsigned int hp_cpu_list_length;
 
@@ -191,6 +191,9 @@ static void *__frame_ptr;
 static u32 *frame_ptr;
 static dma_addr_t frame_dma;
 
+/* needed for dma_map*() */
+static const struct qm_portal_config *pcfg;
+
 /* the main function waits on this */
 static DECLARE_WAIT_QUEUE_HEAD(queue);
 
@@ -210,16 +213,14 @@ static int allocate_frame_data(void)
 {
 	u32 lfsr = HP_FIRST_WORD;
 	int loop;
-	struct platform_device *pdev = platform_device_alloc("foobar", -1);
 
-	if (!pdev) {
-		pr_crit("platform_device_alloc() failed");
-		return -EIO;
-	}
-	if (platform_device_add(pdev)) {
-		pr_crit("platform_device_add() failed");
+	if (!qman_dma_portal) {
+		pr_crit("portal not available\n");
 		return -EIO;
 	}
+
+	pcfg = qman_get_qm_portal_config(qman_dma_portal);
+
 	__frame_ptr = kmalloc(4 * HP_NUM_WORDS, GFP_KERNEL);
 	if (!__frame_ptr)
 		return -ENOMEM;
@@ -229,15 +230,22 @@ static int allocate_frame_data(void)
 		frame_ptr[loop] = lfsr;
 		lfsr = do_lfsr(lfsr);
 	}
-	frame_dma = dma_map_single(&pdev->dev, frame_ptr, 4 * HP_NUM_WORDS,
+
+	frame_dma = dma_map_single(pcfg->dev, frame_ptr, 4 * HP_NUM_WORDS,
 				   DMA_BIDIRECTIONAL);
-	platform_device_del(pdev);
-	platform_device_put(pdev);
+	if (dma_mapping_error(pcfg->dev, frame_dma)) {
+		pr_crit("dma mapping failure\n");
+		kfree(__frame_ptr);
+		return -EIO;
+	}
+
 	return 0;
 }
 
 static void deallocate_frame_data(void)
 {
+	dma_unmap_single(pcfg->dev, frame_dma, 4 * HP_NUM_WORDS,
+			 DMA_BIDIRECTIONAL);
 	kfree(__frame_ptr);
 }
 
@@ -249,7 +257,8 @@ static inline int process_frame_data(struct hp_handler *handler,
 	int loop;
 
 	if (qm_fd_addr_get64(fd) != handler->addr) {
-		pr_crit("bad frame address");
+		pr_crit("bad frame address, [%llX != %llX]\n",
+			qm_fd_addr_get64(fd), handler->addr);
 		return -EIO;
 	}
 	for (loop = 0; loop < HP_NUM_WORDS; loop++, p++) {
@@ -397,8 +406,9 @@ static int init_handler(void *h)
 		goto failed;
 	}
 	memset(&opts, 0, sizeof(opts));
-	opts.we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_CONTEXTA;
-	opts.fqd.fq_ctrl = QM_FQCTRL_CTXASTASHING;
+	opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL |
+				   QM_INITFQ_WE_CONTEXTA);
+	opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CTXASTASHING);
 	qm_fqd_set_stashing(&opts.fqd, 0, STASH_DATA_CL, STASH_CTX_CL);
 	err = qman_init_fq(&handler->rx, QMAN_INITFQ_FLAG_SCHED |
 			   QMAN_INITFQ_FLAG_LOCAL, &opts);
diff --git a/drivers/soc/fsl/qe/qe.c b/drivers/soc/fsl/qe/qe.c
index 2707a827261b..ade168f5328e 100644
--- a/drivers/soc/fsl/qe/qe.c
+++ b/drivers/soc/fsl/qe/qe.c
@@ -717,9 +717,5 @@ static struct platform_driver qe_driver = {
 	.resume = qe_resume,
 };
 
-static int __init qe_drv_init(void)
-{
-	return platform_driver_register(&qe_driver);
-}
-device_initcall(qe_drv_init);
+builtin_platform_driver(qe_driver);
 #endif /* defined(CONFIG_SUSPEND) && defined(CONFIG_PPC_85xx) */
diff --git a/drivers/soc/mediatek/Kconfig b/drivers/soc/mediatek/Kconfig
index 0a4ea809a61b..609bb3424c14 100644
--- a/drivers/soc/mediatek/Kconfig
+++ b/drivers/soc/mediatek/Kconfig
@@ -23,7 +23,7 @@ config MTK_PMIC_WRAP
 config MTK_SCPSYS
 	bool "MediaTek SCPSYS Support"
 	depends on ARCH_MEDIATEK || COMPILE_TEST
-	default ARM64 && ARCH_MEDIATEK
+	default ARCH_MEDIATEK
 	select REGMAP
 	select MTK_INFRACFG
 	select PM_GENERIC_DOMAINS if PM
diff --git a/drivers/soc/mediatek/mtk-scpsys.c b/drivers/soc/mediatek/mtk-scpsys.c
index 837effe19907..beb79162369a 100644
--- a/drivers/soc/mediatek/mtk-scpsys.c
+++ b/drivers/soc/mediatek/mtk-scpsys.c
@@ -11,17 +11,16 @@
  * GNU General Public License for more details.
  */
 #include <linux/clk.h>
-#include <linux/delay.h>
+#include <linux/init.h>
 #include <linux/io.h>
-#include <linux/kernel.h>
 #include <linux/mfd/syscon.h>
-#include <linux/init.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
-#include <linux/regmap.h>
-#include <linux/soc/mediatek/infracfg.h>
 #include <linux/regulator/consumer.h>
+#include <linux/soc/mediatek/infracfg.h>
+
+#include <dt-bindings/power/mt2701-power.h>
 #include <dt-bindings/power/mt8173-power.h>
 
 #define SPM_VDE_PWR_CON			0x0210
@@ -29,11 +28,17 @@
 #define SPM_VEN_PWR_CON			0x0230
 #define SPM_ISP_PWR_CON			0x0238
 #define SPM_DIS_PWR_CON			0x023c
+#define SPM_CONN_PWR_CON		0x0280
 #define SPM_VEN2_PWR_CON		0x0298
-#define SPM_AUDIO_PWR_CON		0x029c
+#define SPM_AUDIO_PWR_CON		0x029c	/* MT8173 */
+#define SPM_BDP_PWR_CON			0x029c	/* MT2701 */
+#define SPM_ETH_PWR_CON			0x02a0
+#define SPM_HIF_PWR_CON			0x02a4
+#define SPM_IFR_MSC_PWR_CON		0x02a8
 #define SPM_MFG_2D_PWR_CON		0x02c0
 #define SPM_MFG_ASYNC_PWR_CON		0x02c4
 #define SPM_USB_PWR_CON			0x02cc
+
 #define SPM_PWR_STATUS			0x060c
 #define SPM_PWR_STATUS_2ND		0x0610
 
@@ -43,10 +48,15 @@
 #define PWR_ON_2ND_BIT			BIT(3)
 #define PWR_CLK_DIS_BIT			BIT(4)
 
+#define PWR_STATUS_CONN			BIT(1)
 #define PWR_STATUS_DISP			BIT(3)
 #define PWR_STATUS_MFG			BIT(4)
 #define PWR_STATUS_ISP			BIT(5)
 #define PWR_STATUS_VDEC			BIT(7)
+#define PWR_STATUS_BDP			BIT(14)
+#define PWR_STATUS_ETH			BIT(15)
+#define PWR_STATUS_HIF			BIT(16)
+#define PWR_STATUS_IFR_MSC		BIT(17)
 #define PWR_STATUS_VENC_LT		BIT(20)
 #define PWR_STATUS_VENC			BIT(21)
 #define PWR_STATUS_MFG_2D		BIT(22)
@@ -55,12 +65,23 @@
 #define PWR_STATUS_USB			BIT(25)
 
 enum clk_id {
-	MT8173_CLK_NONE,
-	MT8173_CLK_MM,
-	MT8173_CLK_MFG,
-	MT8173_CLK_VENC,
-	MT8173_CLK_VENC_LT,
-	MT8173_CLK_MAX,
+	CLK_NONE,
+	CLK_MM,
+	CLK_MFG,
+	CLK_VENC,
+	CLK_VENC_LT,
+	CLK_ETHIF,
+	CLK_MAX,
+};
+
+static const char * const clk_names[] = {
+	NULL,
+	"mm",
+	"mfg",
+	"venc",
+	"venc_lt",
+	"ethif",
+	NULL,
 };
 
 #define MAX_CLKS	2
@@ -76,98 +97,6 @@ struct scp_domain_data {
 	bool active_wakeup;
 };
 
-static const struct scp_domain_data scp_domain_data[] = {
-	[MT8173_POWER_DOMAIN_VDEC] = {
-		.name = "vdec",
-		.sta_mask = PWR_STATUS_VDEC,
-		.ctl_offs = SPM_VDE_PWR_CON,
-		.sram_pdn_bits = GENMASK(11, 8),
-		.sram_pdn_ack_bits = GENMASK(12, 12),
-		.clk_id = {MT8173_CLK_MM},
-	},
-	[MT8173_POWER_DOMAIN_VENC] = {
-		.name = "venc",
-		.sta_mask = PWR_STATUS_VENC,
-		.ctl_offs = SPM_VEN_PWR_CON,
-		.sram_pdn_bits = GENMASK(11, 8),
-		.sram_pdn_ack_bits = GENMASK(15, 12),
-		.clk_id = {MT8173_CLK_MM, MT8173_CLK_VENC},
-	},
-	[MT8173_POWER_DOMAIN_ISP] = {
-		.name = "isp",
-		.sta_mask = PWR_STATUS_ISP,
-		.ctl_offs = SPM_ISP_PWR_CON,
-		.sram_pdn_bits = GENMASK(11, 8),
-		.sram_pdn_ack_bits = GENMASK(13, 12),
-		.clk_id = {MT8173_CLK_MM},
-	},
-	[MT8173_POWER_DOMAIN_MM] = {
-		.name = "mm",
-		.sta_mask = PWR_STATUS_DISP,
-		.ctl_offs = SPM_DIS_PWR_CON,
-		.sram_pdn_bits = GENMASK(11, 8),
-		.sram_pdn_ack_bits = GENMASK(12, 12),
-		.clk_id = {MT8173_CLK_MM},
-		.bus_prot_mask = MT8173_TOP_AXI_PROT_EN_MM_M0 |
-			MT8173_TOP_AXI_PROT_EN_MM_M1,
-	},
-	[MT8173_POWER_DOMAIN_VENC_LT] = {
-		.name = "venc_lt",
-		.sta_mask = PWR_STATUS_VENC_LT,
-		.ctl_offs = SPM_VEN2_PWR_CON,
-		.sram_pdn_bits = GENMASK(11, 8),
-		.sram_pdn_ack_bits = GENMASK(15, 12),
-		.clk_id = {MT8173_CLK_MM, MT8173_CLK_VENC_LT},
-	},
-	[MT8173_POWER_DOMAIN_AUDIO] = {
-		.name = "audio",
-		.sta_mask = PWR_STATUS_AUDIO,
-		.ctl_offs = SPM_AUDIO_PWR_CON,
-		.sram_pdn_bits = GENMASK(11, 8),
-		.sram_pdn_ack_bits = GENMASK(15, 12),
-		.clk_id = {MT8173_CLK_NONE},
-	},
-	[MT8173_POWER_DOMAIN_USB] = {
-		.name = "usb",
-		.sta_mask = PWR_STATUS_USB,
-		.ctl_offs = SPM_USB_PWR_CON,
-		.sram_pdn_bits = GENMASK(11, 8),
-		.sram_pdn_ack_bits = GENMASK(15, 12),
-		.clk_id = {MT8173_CLK_NONE},
-		.active_wakeup = true,
-	},
-	[MT8173_POWER_DOMAIN_MFG_ASYNC] = {
-		.name = "mfg_async",
-		.sta_mask = PWR_STATUS_MFG_ASYNC,
-		.ctl_offs = SPM_MFG_ASYNC_PWR_CON,
-		.sram_pdn_bits = GENMASK(11, 8),
-		.sram_pdn_ack_bits = 0,
-		.clk_id = {MT8173_CLK_MFG},
-	},
-	[MT8173_POWER_DOMAIN_MFG_2D] = {
-		.name = "mfg_2d",
-		.sta_mask = PWR_STATUS_MFG_2D,
-		.ctl_offs = SPM_MFG_2D_PWR_CON,
-		.sram_pdn_bits = GENMASK(11, 8),
-		.sram_pdn_ack_bits = GENMASK(13, 12),
-		.clk_id = {MT8173_CLK_NONE},
-	},
-	[MT8173_POWER_DOMAIN_MFG] = {
-		.name = "mfg",
-		.sta_mask = PWR_STATUS_MFG,
-		.ctl_offs = SPM_MFG_PWR_CON,
-		.sram_pdn_bits = GENMASK(13, 8),
-		.sram_pdn_ack_bits = GENMASK(21, 16),
-		.clk_id = {MT8173_CLK_NONE},
-		.bus_prot_mask = MT8173_TOP_AXI_PROT_EN_MFG_S |
-			MT8173_TOP_AXI_PROT_EN_MFG_M0 |
-			MT8173_TOP_AXI_PROT_EN_MFG_M1 |
-			MT8173_TOP_AXI_PROT_EN_MFG_SNOOP_OUT,
-	},
-};
-
-#define NUM_DOMAINS	ARRAY_SIZE(scp_domain_data)
-
 struct scp;
 
 struct scp_domain {
@@ -179,7 +108,7 @@ struct scp_domain {
 };
 
 struct scp {
-	struct scp_domain domains[NUM_DOMAINS];
+	struct scp_domain *domains;
 	struct genpd_onecell_data pd_data;
 	struct device *dev;
 	void __iomem *base;
@@ -408,57 +337,55 @@ static bool scpsys_active_wakeup(struct device *dev)
 	return scpd->data->active_wakeup;
 }
 
-static int scpsys_probe(struct platform_device *pdev)
+static void init_clks(struct platform_device *pdev, struct clk **clk)
+{
+	int i;
+
+	for (i = CLK_NONE + 1; i < CLK_MAX; i++)
+		clk[i] = devm_clk_get(&pdev->dev, clk_names[i]);
+}
+
+static struct scp *init_scp(struct platform_device *pdev,
+			const struct scp_domain_data *scp_domain_data, int num)
 {
 	struct genpd_onecell_data *pd_data;
 	struct resource *res;
-	int i, j, ret;
+	int i, j;
 	struct scp *scp;
-	struct clk *clk[MT8173_CLK_MAX];
+	struct clk *clk[CLK_MAX];
 
 	scp = devm_kzalloc(&pdev->dev, sizeof(*scp), GFP_KERNEL);
 	if (!scp)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	scp->dev = &pdev->dev;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	scp->base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(scp->base))
-		return PTR_ERR(scp->base);
+		return ERR_CAST(scp->base);
+
+	scp->domains = devm_kzalloc(&pdev->dev,
+				sizeof(*scp->domains) * num, GFP_KERNEL);
+	if (!scp->domains)
+		return ERR_PTR(-ENOMEM);
 
 	pd_data = &scp->pd_data;
 
 	pd_data->domains = devm_kzalloc(&pdev->dev,
-			sizeof(*pd_data->domains) * NUM_DOMAINS, GFP_KERNEL);
+			sizeof(*pd_data->domains) * num, GFP_KERNEL);
 	if (!pd_data->domains)
-		return -ENOMEM;
-
-	clk[MT8173_CLK_MM] = devm_clk_get(&pdev->dev, "mm");
-	if (IS_ERR(clk[MT8173_CLK_MM]))
-		return PTR_ERR(clk[MT8173_CLK_MM]);
-
-	clk[MT8173_CLK_MFG] = devm_clk_get(&pdev->dev, "mfg");
-	if (IS_ERR(clk[MT8173_CLK_MFG]))
-		return PTR_ERR(clk[MT8173_CLK_MFG]);
-
-	clk[MT8173_CLK_VENC] = devm_clk_get(&pdev->dev, "venc");
-	if (IS_ERR(clk[MT8173_CLK_VENC]))
-		return PTR_ERR(clk[MT8173_CLK_VENC]);
-
-	clk[MT8173_CLK_VENC_LT] = devm_clk_get(&pdev->dev, "venc_lt");
-	if (IS_ERR(clk[MT8173_CLK_VENC_LT]))
-		return PTR_ERR(clk[MT8173_CLK_VENC_LT]);
+		return ERR_PTR(-ENOMEM);
 
 	scp->infracfg = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
 			"infracfg");
 	if (IS_ERR(scp->infracfg)) {
 		dev_err(&pdev->dev, "Cannot find infracfg controller: %ld\n",
 				PTR_ERR(scp->infracfg));
-		return PTR_ERR(scp->infracfg);
+		return ERR_CAST(scp->infracfg);
 	}
 
-	for (i = 0; i < NUM_DOMAINS; i++) {
+	for (i = 0; i < num; i++) {
 		struct scp_domain *scpd = &scp->domains[i];
 		const struct scp_domain_data *data = &scp_domain_data[i];
 
@@ -467,13 +394,15 @@ static int scpsys_probe(struct platform_device *pdev)
 			if (PTR_ERR(scpd->supply) == -ENODEV)
 				scpd->supply = NULL;
 			else
-				return PTR_ERR(scpd->supply);
+				return ERR_CAST(scpd->supply);
 		}
 	}
 
-	pd_data->num_domains = NUM_DOMAINS;
+	pd_data->num_domains = num;
+
+	init_clks(pdev, clk);
 
-	for (i = 0; i < NUM_DOMAINS; i++) {
+	for (i = 0; i < num; i++) {
 		struct scp_domain *scpd = &scp->domains[i];
 		struct generic_pm_domain *genpd = &scpd->genpd;
 		const struct scp_domain_data *data = &scp_domain_data[i];
@@ -482,13 +411,37 @@ static int scpsys_probe(struct platform_device *pdev)
 		scpd->scp = scp;
 
 		scpd->data = data;
-		for (j = 0; j < MAX_CLKS && data->clk_id[j]; j++)
-			scpd->clk[j] = clk[data->clk_id[j]];
+
+		for (j = 0; j < MAX_CLKS && data->clk_id[j]; j++) {
+			struct clk *c = clk[data->clk_id[j]];
+
+			if (IS_ERR(c)) {
+				dev_err(&pdev->dev, "%s: clk unavailable\n",
+					data->name);
+				return ERR_CAST(c);
+			}
+
+			scpd->clk[j] = c;
+		}
 
 		genpd->name = data->name;
 		genpd->power_off = scpsys_power_off;
 		genpd->power_on = scpsys_power_on;
 		genpd->dev_ops.active_wakeup = scpsys_active_wakeup;
+	}
+
+	return scp;
+}
+
+static void mtk_register_power_domains(struct platform_device *pdev,
+				struct scp *scp, int num)
+{
+	struct genpd_onecell_data *pd_data;
+	int i, ret;
+
+	for (i = 0; i < num; i++) {
+		struct scp_domain *scpd = &scp->domains[i];
+		struct generic_pm_domain *genpd = &scpd->genpd;
 
 		/*
 		 * Initially turn on all domains to make the domains usable
@@ -507,6 +460,222 @@ static int scpsys_probe(struct platform_device *pdev)
 	 * valid.
 	 */
 
+	pd_data = &scp->pd_data;
+
+	ret = of_genpd_add_provider_onecell(pdev->dev.of_node, pd_data);
+	if (ret)
+		dev_err(&pdev->dev, "Failed to add OF provider: %d\n", ret);
+}
+
+/*
+ * MT2701 power domain support
+ */
+
+static const struct scp_domain_data scp_domain_data_mt2701[] = {
+	[MT2701_POWER_DOMAIN_CONN] = {
+		.name = "conn",
+		.sta_mask = PWR_STATUS_CONN,
+		.ctl_offs = SPM_CONN_PWR_CON,
+		.bus_prot_mask = 0x0104,
+		.clk_id = {CLK_NONE},
+		.active_wakeup = true,
+	},
+	[MT2701_POWER_DOMAIN_DISP] = {
+		.name = "disp",
+		.sta_mask = PWR_STATUS_DISP,
+		.ctl_offs = SPM_DIS_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.clk_id = {CLK_MM},
+		.bus_prot_mask = 0x0002,
+		.active_wakeup = true,
+	},
+	[MT2701_POWER_DOMAIN_MFG] = {
+		.name = "mfg",
+		.sta_mask = PWR_STATUS_MFG,
+		.ctl_offs = SPM_MFG_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(12, 12),
+		.clk_id = {CLK_MFG},
+		.active_wakeup = true,
+	},
+	[MT2701_POWER_DOMAIN_VDEC] = {
+		.name = "vdec",
+		.sta_mask = PWR_STATUS_VDEC,
+		.ctl_offs = SPM_VDE_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(12, 12),
+		.clk_id = {CLK_MM},
+		.active_wakeup = true,
+	},
+	[MT2701_POWER_DOMAIN_ISP] = {
+		.name = "isp",
+		.sta_mask = PWR_STATUS_ISP,
+		.ctl_offs = SPM_ISP_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(13, 12),
+		.clk_id = {CLK_MM},
+		.active_wakeup = true,
+	},
+	[MT2701_POWER_DOMAIN_BDP] = {
+		.name = "bdp",
+		.sta_mask = PWR_STATUS_BDP,
+		.ctl_offs = SPM_BDP_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.clk_id = {CLK_NONE},
+		.active_wakeup = true,
+	},
+	[MT2701_POWER_DOMAIN_ETH] = {
+		.name = "eth",
+		.sta_mask = PWR_STATUS_ETH,
+		.ctl_offs = SPM_ETH_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(15, 12),
+		.clk_id = {CLK_ETHIF},
+		.active_wakeup = true,
+	},
+	[MT2701_POWER_DOMAIN_HIF] = {
+		.name = "hif",
+		.sta_mask = PWR_STATUS_HIF,
+		.ctl_offs = SPM_HIF_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(15, 12),
+		.clk_id = {CLK_ETHIF},
+		.active_wakeup = true,
+	},
+	[MT2701_POWER_DOMAIN_IFR_MSC] = {
+		.name = "ifr_msc",
+		.sta_mask = PWR_STATUS_IFR_MSC,
+		.ctl_offs = SPM_IFR_MSC_PWR_CON,
+		.clk_id = {CLK_NONE},
+		.active_wakeup = true,
+	},
+};
+
+#define NUM_DOMAINS_MT2701	ARRAY_SIZE(scp_domain_data_mt2701)
+
+static int __init scpsys_probe_mt2701(struct platform_device *pdev)
+{
+	struct scp *scp;
+
+	scp = init_scp(pdev, scp_domain_data_mt2701, NUM_DOMAINS_MT2701);
+	if (IS_ERR(scp))
+		return PTR_ERR(scp);
+
+	mtk_register_power_domains(pdev, scp, NUM_DOMAINS_MT2701);
+
+	return 0;
+}
+
+/*
+ * MT8173 power domain support
+ */
+
+static const struct scp_domain_data scp_domain_data_mt8173[] = {
+	[MT8173_POWER_DOMAIN_VDEC] = {
+		.name = "vdec",
+		.sta_mask = PWR_STATUS_VDEC,
+		.ctl_offs = SPM_VDE_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(12, 12),
+		.clk_id = {CLK_MM},
+	},
+	[MT8173_POWER_DOMAIN_VENC] = {
+		.name = "venc",
+		.sta_mask = PWR_STATUS_VENC,
+		.ctl_offs = SPM_VEN_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(15, 12),
+		.clk_id = {CLK_MM, CLK_VENC},
+	},
+	[MT8173_POWER_DOMAIN_ISP] = {
+		.name = "isp",
+		.sta_mask = PWR_STATUS_ISP,
+		.ctl_offs = SPM_ISP_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(13, 12),
+		.clk_id = {CLK_MM},
+	},
+	[MT8173_POWER_DOMAIN_MM] = {
+		.name = "mm",
+		.sta_mask = PWR_STATUS_DISP,
+		.ctl_offs = SPM_DIS_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(12, 12),
+		.clk_id = {CLK_MM},
+		.bus_prot_mask = MT8173_TOP_AXI_PROT_EN_MM_M0 |
+			MT8173_TOP_AXI_PROT_EN_MM_M1,
+	},
+	[MT8173_POWER_DOMAIN_VENC_LT] = {
+		.name = "venc_lt",
+		.sta_mask = PWR_STATUS_VENC_LT,
+		.ctl_offs = SPM_VEN2_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(15, 12),
+		.clk_id = {CLK_MM, CLK_VENC_LT},
+	},
+	[MT8173_POWER_DOMAIN_AUDIO] = {
+		.name = "audio",
+		.sta_mask = PWR_STATUS_AUDIO,
+		.ctl_offs = SPM_AUDIO_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(15, 12),
+		.clk_id = {CLK_NONE},
+	},
+	[MT8173_POWER_DOMAIN_USB] = {
+		.name = "usb",
+		.sta_mask = PWR_STATUS_USB,
+		.ctl_offs = SPM_USB_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(15, 12),
+		.clk_id = {CLK_NONE},
+		.active_wakeup = true,
+	},
+	[MT8173_POWER_DOMAIN_MFG_ASYNC] = {
+		.name = "mfg_async",
+		.sta_mask = PWR_STATUS_MFG_ASYNC,
+		.ctl_offs = SPM_MFG_ASYNC_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = 0,
+		.clk_id = {CLK_MFG},
+	},
+	[MT8173_POWER_DOMAIN_MFG_2D] = {
+		.name = "mfg_2d",
+		.sta_mask = PWR_STATUS_MFG_2D,
+		.ctl_offs = SPM_MFG_2D_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(13, 12),
+		.clk_id = {CLK_NONE},
+	},
+	[MT8173_POWER_DOMAIN_MFG] = {
+		.name = "mfg",
+		.sta_mask = PWR_STATUS_MFG,
+		.ctl_offs = SPM_MFG_PWR_CON,
+		.sram_pdn_bits = GENMASK(13, 8),
+		.sram_pdn_ack_bits = GENMASK(21, 16),
+		.clk_id = {CLK_NONE},
+		.bus_prot_mask = MT8173_TOP_AXI_PROT_EN_MFG_S |
+			MT8173_TOP_AXI_PROT_EN_MFG_M0 |
+			MT8173_TOP_AXI_PROT_EN_MFG_M1 |
+			MT8173_TOP_AXI_PROT_EN_MFG_SNOOP_OUT,
+	},
+};
+
+#define NUM_DOMAINS_MT8173	ARRAY_SIZE(scp_domain_data_mt8173)
+
+static int __init scpsys_probe_mt8173(struct platform_device *pdev)
+{
+	struct scp *scp;
+	struct genpd_onecell_data *pd_data;
+	int ret;
+
+	scp = init_scp(pdev, scp_domain_data_mt8173, NUM_DOMAINS_MT8173);
+	if (IS_ERR(scp))
+		return PTR_ERR(scp);
+
+	mtk_register_power_domains(pdev, scp, NUM_DOMAINS_MT8173);
+
+	pd_data = &scp->pd_data;
+
 	ret = pm_genpd_add_subdomain(pd_data->domains[MT8173_POWER_DOMAIN_MFG_ASYNC],
 		pd_data->domains[MT8173_POWER_DOMAIN_MFG_2D]);
 	if (ret && IS_ENABLED(CONFIG_PM))
@@ -517,21 +686,39 @@ static int scpsys_probe(struct platform_device *pdev)
 	if (ret && IS_ENABLED(CONFIG_PM))
 		dev_err(&pdev->dev, "Failed to add subdomain: %d\n", ret);
 
-	ret = of_genpd_add_provider_onecell(pdev->dev.of_node, pd_data);
-	if (ret)
-		dev_err(&pdev->dev, "Failed to add OF provider: %d\n", ret);
-
 	return 0;
 }
 
+/*
+ * scpsys driver init
+ */
+
 static const struct of_device_id of_scpsys_match_tbl[] = {
 	{
+		.compatible = "mediatek,mt2701-scpsys",
+		.data = scpsys_probe_mt2701,
+	}, {
 		.compatible = "mediatek,mt8173-scpsys",
+		.data = scpsys_probe_mt8173,
 	}, {
 		/* sentinel */
 	}
 };
 
+static int scpsys_probe(struct platform_device *pdev)
+{
+	int (*probe)(struct platform_device *);
+	const struct of_device_id *of_id;
+
+	of_id = of_match_node(of_scpsys_match_tbl, pdev->dev.of_node);
+	if (!of_id || !of_id->data)
+		return -EINVAL;
+
+	probe = of_id->data;
+
+	return probe(pdev);
+}
+
 static struct platform_driver scpsys_drv = {
 	.probe = scpsys_probe,
 	.driver = {
diff --git a/drivers/soc/renesas/Makefile b/drivers/soc/renesas/Makefile
index 623039c3514c..d9115cb5ed9d 100644
--- a/drivers/soc/renesas/Makefile
+++ b/drivers/soc/renesas/Makefile
@@ -1,3 +1,12 @@
+obj-$(CONFIG_SOC_BUS)		+= renesas-soc.o
+
+obj-$(CONFIG_ARCH_RCAR_GEN1)	+= rcar-rst.o
+obj-$(CONFIG_ARCH_RCAR_GEN2)	+= rcar-rst.o
+obj-$(CONFIG_ARCH_R8A7795)	+= rcar-rst.o
+obj-$(CONFIG_ARCH_R8A7796)	+= rcar-rst.o
+
+obj-$(CONFIG_ARCH_R8A7743)	+= rcar-sysc.o r8a7743-sysc.o
+obj-$(CONFIG_ARCH_R8A7745)	+= rcar-sysc.o r8a7745-sysc.o
 obj-$(CONFIG_ARCH_R8A7779)	+= rcar-sysc.o r8a7779-sysc.o
 obj-$(CONFIG_ARCH_R8A7790)	+= rcar-sysc.o r8a7790-sysc.o
 obj-$(CONFIG_ARCH_R8A7791)	+= rcar-sysc.o r8a7791-sysc.o
diff --git a/drivers/soc/renesas/r8a7743-sysc.c b/drivers/soc/renesas/r8a7743-sysc.c
new file mode 100644
index 000000000000..9583a327d90c
--- /dev/null
+++ b/drivers/soc/renesas/r8a7743-sysc.c
@@ -0,0 +1,32 @@
+/*
+ * Renesas RZ/G1M System Controller
+ *
+ * Copyright (C) 2016 Cogent Embedded Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation; of the License.
+ */
+
+#include <linux/bug.h>
+#include <linux/kernel.h>
+
+#include <dt-bindings/power/r8a7743-sysc.h>
+
+#include "rcar-sysc.h"
+
+static const struct rcar_sysc_area r8a7743_areas[] __initconst = {
+	{ "always-on",	    0, 0, R8A7743_PD_ALWAYS_ON,	-1, PD_ALWAYS_ON },
+	{ "ca15-scu",	0x180, 0, R8A7743_PD_CA15_SCU,	R8A7743_PD_ALWAYS_ON,
+	  PD_SCU },
+	{ "ca15-cpu0",	 0x40, 0, R8A7743_PD_CA15_CPU0,	R8A7743_PD_CA15_SCU,
+	  PD_CPU_NOCR },
+	{ "ca15-cpu1",	 0x40, 1, R8A7743_PD_CA15_CPU1,	R8A7743_PD_CA15_SCU,
+	  PD_CPU_NOCR },
+	{ "sgx",	 0xc0, 0, R8A7743_PD_SGX,	R8A7743_PD_ALWAYS_ON },
+};
+
+const struct rcar_sysc_info r8a7743_sysc_info __initconst = {
+	.areas = r8a7743_areas,
+	.num_areas = ARRAY_SIZE(r8a7743_areas),
+};
diff --git a/drivers/soc/renesas/r8a7745-sysc.c b/drivers/soc/renesas/r8a7745-sysc.c
new file mode 100644
index 000000000000..d17887c08aa1
--- /dev/null
+++ b/drivers/soc/renesas/r8a7745-sysc.c
@@ -0,0 +1,32 @@
+/*
+ * Renesas RZ/G1E System Controller
+ *
+ * Copyright (C) 2016 Cogent Embedded Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation; of the License.
+ */
+
+#include <linux/bug.h>
+#include <linux/kernel.h>
+
+#include <dt-bindings/power/r8a7745-sysc.h>
+
+#include "rcar-sysc.h"
+
+static const struct rcar_sysc_area r8a7745_areas[] __initconst = {
+	{ "always-on",	    0, 0, R8A7745_PD_ALWAYS_ON,	-1, PD_ALWAYS_ON },
+	{ "ca7-scu",	0x100, 0, R8A7745_PD_CA7_SCU,	R8A7745_PD_ALWAYS_ON,
+	  PD_SCU },
+	{ "ca7-cpu0",	0x1c0, 0, R8A7745_PD_CA7_CPU0,	R8A7745_PD_CA7_SCU,
+	  PD_CPU_NOCR },
+	{ "ca7-cpu1",	0x1c0, 1, R8A7745_PD_CA7_CPU1,	R8A7745_PD_CA7_SCU,
+	  PD_CPU_NOCR },
+	{ "sgx",	 0xc0, 0, R8A7745_PD_SGX,	R8A7745_PD_ALWAYS_ON },
+};
+
+const struct rcar_sysc_info r8a7745_sysc_info __initconst = {
+	.areas = r8a7745_areas,
+	.num_areas = ARRAY_SIZE(r8a7745_areas),
+};
diff --git a/drivers/soc/renesas/rcar-rst.c b/drivers/soc/renesas/rcar-rst.c
new file mode 100644
index 000000000000..a6d1c26d3167
--- /dev/null
+++ b/drivers/soc/renesas/rcar-rst.c
@@ -0,0 +1,92 @@
+/*
+ * R-Car Gen1 RESET/WDT, R-Car Gen2, Gen3, and RZ/G RST Driver
+ *
+ * Copyright (C) 2016 Glider bvba
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/of_address.h>
+#include <linux/soc/renesas/rcar-rst.h>
+
+struct rst_config {
+	unsigned int modemr;	/* Mode Monitoring Register Offset */
+};
+
+static const struct rst_config rcar_rst_gen1 __initconst = {
+	.modemr = 0x20,
+};
+
+static const struct rst_config rcar_rst_gen2 __initconst = {
+	.modemr = 0x60,
+};
+
+static const struct of_device_id rcar_rst_matches[] __initconst = {
+	/* RZ/G is handled like R-Car Gen2 */
+	{ .compatible = "renesas,r8a7743-rst", .data = &rcar_rst_gen2 },
+	{ .compatible = "renesas,r8a7745-rst", .data = &rcar_rst_gen2 },
+	/* R-Car Gen1 */
+	{ .compatible = "renesas,r8a7778-reset-wdt", .data = &rcar_rst_gen1 },
+	{ .compatible = "renesas,r8a7779-reset-wdt", .data = &rcar_rst_gen1 },
+	/* R-Car Gen2 */
+	{ .compatible = "renesas,r8a7790-rst", .data = &rcar_rst_gen2 },
+	{ .compatible = "renesas,r8a7791-rst", .data = &rcar_rst_gen2 },
+	{ .compatible = "renesas,r8a7792-rst", .data = &rcar_rst_gen2 },
+	{ .compatible = "renesas,r8a7793-rst", .data = &rcar_rst_gen2 },
+	{ .compatible = "renesas,r8a7794-rst", .data = &rcar_rst_gen2 },
+	/* R-Car Gen3 is handled like R-Car Gen2 */
+	{ .compatible = "renesas,r8a7795-rst", .data = &rcar_rst_gen2 },
+	{ .compatible = "renesas,r8a7796-rst", .data = &rcar_rst_gen2 },
+	{ /* sentinel */ }
+};
+
+static void __iomem *rcar_rst_base __initdata;
+static u32 saved_mode __initdata;
+
+static int __init rcar_rst_init(void)
+{
+	const struct of_device_id *match;
+	const struct rst_config *cfg;
+	struct device_node *np;
+	void __iomem *base;
+	int error = 0;
+
+	np = of_find_matching_node_and_match(NULL, rcar_rst_matches, &match);
+	if (!np)
+		return -ENODEV;
+
+	base = of_iomap(np, 0);
+	if (!base) {
+		pr_warn("%s: Cannot map regs\n", np->full_name);
+		error = -ENOMEM;
+		goto out_put;
+	}
+
+	rcar_rst_base = base;
+	cfg = match->data;
+	saved_mode = ioread32(base + cfg->modemr);
+
+	pr_debug("%s: MODE = 0x%08x\n", np->full_name, saved_mode);
+
+out_put:
+	of_node_put(np);
+	return error;
+}
+
+int __init rcar_rst_read_mode_pins(u32 *mode)
+{
+	int error;
+
+	if (!rcar_rst_base) {
+		error = rcar_rst_init();
+		if (error)
+			return error;
+	}
+
+	*mode = saved_mode;
+	return 0;
+}
diff --git a/drivers/soc/renesas/rcar-sysc.c b/drivers/soc/renesas/rcar-sysc.c
index 65c8e1eb90c0..225c35c79d9a 100644
--- a/drivers/soc/renesas/rcar-sysc.c
+++ b/drivers/soc/renesas/rcar-sysc.c
@@ -275,6 +275,12 @@ finalize:
 }
 
 static const struct of_device_id rcar_sysc_matches[] = {
+#ifdef CONFIG_ARCH_R8A7743
+	{ .compatible = "renesas,r8a7743-sysc", .data = &r8a7743_sysc_info },
+#endif
+#ifdef CONFIG_ARCH_R8A7745
+	{ .compatible = "renesas,r8a7745-sysc", .data = &r8a7745_sysc_info },
+#endif
 #ifdef CONFIG_ARCH_R8A7779
 	{ .compatible = "renesas,r8a7779-sysc", .data = &r8a7779_sysc_info },
 #endif
diff --git a/drivers/soc/renesas/rcar-sysc.h b/drivers/soc/renesas/rcar-sysc.h
index 77dbe861473f..f6e842e2976e 100644
--- a/drivers/soc/renesas/rcar-sysc.h
+++ b/drivers/soc/renesas/rcar-sysc.h
@@ -50,6 +50,8 @@ struct rcar_sysc_info {
 	unsigned int num_areas;
 };
 
+extern const struct rcar_sysc_info r8a7743_sysc_info;
+extern const struct rcar_sysc_info r8a7745_sysc_info;
 extern const struct rcar_sysc_info r8a7779_sysc_info;
 extern const struct rcar_sysc_info r8a7790_sysc_info;
 extern const struct rcar_sysc_info r8a7791_sysc_info;
diff --git a/drivers/soc/renesas/renesas-soc.c b/drivers/soc/renesas/renesas-soc.c
new file mode 100644
index 000000000000..330960312296
--- /dev/null
+++ b/drivers/soc/renesas/renesas-soc.c
@@ -0,0 +1,257 @@
+/*
+ * Renesas SoC Identification
+ *
+ * Copyright (C) 2014-2016 Glider bvba
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/sys_soc.h>
+
+
+struct renesas_family {
+	const char name[16];
+	u32 reg;			/* CCCR or PRR, if not in DT */
+};
+
+static const struct renesas_family fam_rcar_gen1 __initconst __maybe_unused = {
+	.name	= "R-Car Gen1",
+	.reg	= 0xff000044,		/* PRR (Product Register) */
+};
+
+static const struct renesas_family fam_rcar_gen2 __initconst __maybe_unused = {
+	.name	= "R-Car Gen2",
+	.reg	= 0xff000044,		/* PRR (Product Register) */
+};
+
+static const struct renesas_family fam_rcar_gen3 __initconst __maybe_unused = {
+	.name	= "R-Car Gen3",
+	.reg	= 0xfff00044,		/* PRR (Product Register) */
+};
+
+static const struct renesas_family fam_rmobile __initconst __maybe_unused = {
+	.name	= "R-Mobile",
+	.reg	= 0xe600101c,		/* CCCR (Common Chip Code Register) */
+};
+
+static const struct renesas_family fam_rza __initconst __maybe_unused = {
+	.name	= "RZ/A",
+};
+
+static const struct renesas_family fam_rzg __initconst __maybe_unused = {
+	.name	= "RZ/G",
+	.reg	= 0xff000044,		/* PRR (Product Register) */
+};
+
+static const struct renesas_family fam_shmobile __initconst __maybe_unused = {
+	.name	= "SH-Mobile",
+	.reg	= 0xe600101c,		/* CCCR (Common Chip Code Register) */
+};
+
+
+struct renesas_soc {
+	const struct renesas_family *family;
+	u8 id;
+};
+
+static const struct renesas_soc soc_rz_a1h __initconst __maybe_unused = {
+	.family	= &fam_rza,
+};
+
+static const struct renesas_soc soc_rmobile_ape6 __initconst __maybe_unused = {
+	.family	= &fam_rmobile,
+	.id	= 0x3f,
+};
+
+static const struct renesas_soc soc_rmobile_a1 __initconst __maybe_unused = {
+	.family	= &fam_rmobile,
+	.id	= 0x40,
+};
+
+static const struct renesas_soc soc_rz_g1m __initconst __maybe_unused = {
+	.family	= &fam_rzg,
+	.id	= 0x47,
+};
+
+static const struct renesas_soc soc_rz_g1e __initconst __maybe_unused = {
+	.family	= &fam_rzg,
+	.id	= 0x4c,
+};
+
+static const struct renesas_soc soc_rcar_m1a __initconst __maybe_unused = {
+	.family	= &fam_rcar_gen1,
+};
+
+static const struct renesas_soc soc_rcar_h1 __initconst __maybe_unused = {
+	.family	= &fam_rcar_gen1,
+	.id	= 0x3b,
+};
+
+static const struct renesas_soc soc_rcar_h2 __initconst __maybe_unused = {
+	.family	= &fam_rcar_gen2,
+	.id	= 0x45,
+};
+
+static const struct renesas_soc soc_rcar_m2_w __initconst __maybe_unused = {
+	.family	= &fam_rcar_gen2,
+	.id	= 0x47,
+};
+
+static const struct renesas_soc soc_rcar_v2h __initconst __maybe_unused = {
+	.family	= &fam_rcar_gen2,
+	.id	= 0x4a,
+};
+
+static const struct renesas_soc soc_rcar_m2_n __initconst __maybe_unused = {
+	.family	= &fam_rcar_gen2,
+	.id	= 0x4b,
+};
+
+static const struct renesas_soc soc_rcar_e2 __initconst __maybe_unused = {
+	.family	= &fam_rcar_gen2,
+	.id	= 0x4c,
+};
+
+static const struct renesas_soc soc_rcar_h3 __initconst __maybe_unused = {
+	.family	= &fam_rcar_gen3,
+	.id	= 0x4f,
+};
+
+static const struct renesas_soc soc_rcar_m3_w __initconst __maybe_unused = {
+	.family	= &fam_rcar_gen3,
+	.id	= 0x52,
+};
+
+static const struct renesas_soc soc_shmobile_ag5 __initconst __maybe_unused = {
+	.family	= &fam_shmobile,
+	.id	= 0x37,
+};
+
+
+static const struct of_device_id renesas_socs[] __initconst = {
+#ifdef CONFIG_ARCH_R7S72100
+	{ .compatible = "renesas,r7s72100",	.data = &soc_rz_a1h },
+#endif
+#ifdef CONFIG_ARCH_R8A73A4
+	{ .compatible = "renesas,r8a73a4",	.data = &soc_rmobile_ape6 },
+#endif
+#ifdef CONFIG_ARCH_R8A7740
+	{ .compatible = "renesas,r8a7740",	.data = &soc_rmobile_a1 },
+#endif
+#ifdef CONFIG_ARCH_R8A7743
+	{ .compatible = "renesas,r8a7743",	.data = &soc_rz_g1m },
+#endif
+#ifdef CONFIG_ARCH_R8A7745
+	{ .compatible = "renesas,r8a7745",	.data = &soc_rz_g1e },
+#endif
+#ifdef CONFIG_ARCH_R8A7778
+	{ .compatible = "renesas,r8a7778",	.data = &soc_rcar_m1a },
+#endif
+#ifdef CONFIG_ARCH_R8A7779
+	{ .compatible = "renesas,r8a7779",	.data = &soc_rcar_h1 },
+#endif
+#ifdef CONFIG_ARCH_R8A7790
+	{ .compatible = "renesas,r8a7790",	.data = &soc_rcar_h2 },
+#endif
+#ifdef CONFIG_ARCH_R8A7791
+	{ .compatible = "renesas,r8a7791",	.data = &soc_rcar_m2_w },
+#endif
+#ifdef CONFIG_ARCH_R8A7792
+	{ .compatible = "renesas,r8a7792",	.data = &soc_rcar_v2h },
+#endif
+#ifdef CONFIG_ARCH_R8A7793
+	{ .compatible = "renesas,r8a7793",	.data = &soc_rcar_m2_n },
+#endif
+#ifdef CONFIG_ARCH_R8A7794
+	{ .compatible = "renesas,r8a7794",	.data = &soc_rcar_e2 },
+#endif
+#ifdef CONFIG_ARCH_R8A7795
+	{ .compatible = "renesas,r8a7795",	.data = &soc_rcar_h3 },
+#endif
+#ifdef CONFIG_ARCH_R8A7796
+	{ .compatible = "renesas,r8a7796",	.data = &soc_rcar_m3_w },
+#endif
+#ifdef CONFIG_ARCH_SH73A0
+	{ .compatible = "renesas,sh73a0",	.data = &soc_shmobile_ag5 },
+#endif
+	{ /* sentinel */ }
+};
+
+static int __init renesas_soc_init(void)
+{
+	struct soc_device_attribute *soc_dev_attr;
+	const struct renesas_family *family;
+	const struct of_device_id *match;
+	const struct renesas_soc *soc;
+	void __iomem *chipid = NULL;
+	struct soc_device *soc_dev;
+	struct device_node *np;
+	unsigned int product;
+
+	match = of_match_node(renesas_socs, of_root);
+	if (!match)
+		return -ENODEV;
+
+	soc = match->data;
+	family = soc->family;
+
+	/* Try PRR first, then hardcoded fallback */
+	np = of_find_compatible_node(NULL, NULL, "renesas,prr");
+	if (np) {
+		chipid = of_iomap(np, 0);
+		of_node_put(np);
+	} else if (soc->id) {
+		chipid = ioremap(family->reg, 4);
+	}
+	if (chipid) {
+		product = readl(chipid);
+		iounmap(chipid);
+		if (soc->id && ((product >> 8) & 0xff) != soc->id) {
+			pr_warn("SoC mismatch (product = 0x%x)\n", product);
+			return -ENODEV;
+		}
+	}
+
+	soc_dev_attr = kzalloc(sizeof(*soc_dev_attr), GFP_KERNEL);
+	if (!soc_dev_attr)
+		return -ENOMEM;
+
+	np = of_find_node_by_path("/");
+	of_property_read_string(np, "model", &soc_dev_attr->machine);
+	of_node_put(np);
+
+	soc_dev_attr->family = kstrdup_const(family->name, GFP_KERNEL);
+	soc_dev_attr->soc_id = kstrdup_const(strchr(match->compatible, ',') + 1,
+					     GFP_KERNEL);
+	if (chipid)
+		soc_dev_attr->revision = kasprintf(GFP_KERNEL, "ES%u.%u",
+						   ((product >> 4) & 0x0f) + 1,
+						   product & 0xf);
+
+	pr_info("Detected Renesas %s %s %s\n", soc_dev_attr->family,
+		soc_dev_attr->soc_id, soc_dev_attr->revision ?: "");
+
+	soc_dev = soc_device_register(soc_dev_attr);
+	if (IS_ERR(soc_dev)) {
+		kfree(soc_dev_attr->revision);
+		kfree_const(soc_dev_attr->soc_id);
+		kfree_const(soc_dev_attr->family);
+		kfree(soc_dev_attr);
+		return PTR_ERR(soc_dev);
+	}
+
+	return 0;
+}
+core_initcall(renesas_soc_init);
diff --git a/drivers/soc/rockchip/pm_domains.c b/drivers/soc/rockchip/pm_domains.c
index 7acd1517dd37..1c78c42416c6 100644
--- a/drivers/soc/rockchip/pm_domains.c
+++ b/drivers/soc/rockchip/pm_domains.c
@@ -9,6 +9,7 @@
  */
 
 #include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/err.h>
 #include <linux/pm_clock.h>
 #include <linux/pm_domain.h>
@@ -105,12 +106,24 @@ static bool rockchip_pmu_domain_is_idle(struct rockchip_pm_domain *pd)
 	return (val & pd_info->idle_mask) == pd_info->idle_mask;
 }
 
+static unsigned int rockchip_pmu_read_ack(struct rockchip_pmu *pmu)
+{
+	unsigned int val;
+
+	regmap_read(pmu->regmap, pmu->info->ack_offset, &val);
+	return val;
+}
+
 static int rockchip_pmu_set_idle_request(struct rockchip_pm_domain *pd,
 					 bool idle)
 {
 	const struct rockchip_domain_info *pd_info = pd->info;
+	struct generic_pm_domain *genpd = &pd->genpd;
 	struct rockchip_pmu *pmu = pd->pmu;
+	unsigned int target_ack;
 	unsigned int val;
+	bool is_idle;
+	int ret;
 
 	if (pd_info->req_mask == 0)
 		return 0;
@@ -120,12 +133,26 @@ static int rockchip_pmu_set_idle_request(struct rockchip_pm_domain *pd,
 
 	dsb(sy);
 
-	do {
-		regmap_read(pmu->regmap, pmu->info->ack_offset, &val);
-	} while ((val & pd_info->ack_mask) != (idle ? pd_info->ack_mask : 0));
+	/* Wait util idle_ack = 1 */
+	target_ack = idle ? pd_info->ack_mask : 0;
+	ret = readx_poll_timeout_atomic(rockchip_pmu_read_ack, pmu, val,
+					(val & pd_info->ack_mask) == target_ack,
+					0, 10000);
+	if (ret) {
+		dev_err(pmu->dev,
+			"failed to get ack on domain '%s', val=0x%x\n",
+			genpd->name, val);
+		return ret;
+	}
 
-	while (rockchip_pmu_domain_is_idle(pd) != idle)
-		cpu_relax();
+	ret = readx_poll_timeout_atomic(rockchip_pmu_domain_is_idle, pd,
+					is_idle, is_idle == idle, 0, 10000);
+	if (ret) {
+		dev_err(pmu->dev,
+			"failed to set idle on domain '%s', val=%d\n",
+			genpd->name, is_idle);
+		return ret;
+	}
 
 	return 0;
 }
@@ -198,6 +225,8 @@ static void rockchip_do_pmu_set_power_domain(struct rockchip_pm_domain *pd,
 					     bool on)
 {
 	struct rockchip_pmu *pmu = pd->pmu;
+	struct generic_pm_domain *genpd = &pd->genpd;
+	bool is_on;
 
 	if (pd->info->pwr_mask == 0)
 		return;
@@ -207,8 +236,13 @@ static void rockchip_do_pmu_set_power_domain(struct rockchip_pm_domain *pd,
 
 	dsb(sy);
 
-	while (rockchip_pmu_domain_is_on(pd) != on)
-		cpu_relax();
+	if (readx_poll_timeout_atomic(rockchip_pmu_domain_is_on, pd, is_on,
+				      is_on == on, 0, 10000)) {
+		dev_err(pmu->dev,
+			"failed to set domain '%s', val=%d\n",
+			genpd->name, is_on);
+		return;
+	}
 }
 
 static int rockchip_pd_power(struct rockchip_pm_domain *pd, bool power_on)
@@ -445,7 +479,16 @@ err_out:
 
 static void rockchip_pm_remove_one_domain(struct rockchip_pm_domain *pd)
 {
-	int i;
+	int i, ret;
+
+	/*
+	 * We're in the error cleanup already, so we only complain,
+	 * but won't emit another error on top of the original one.
+	 */
+	ret = pm_genpd_remove(&pd->genpd);
+	if (ret < 0)
+		dev_err(pd->pmu->dev, "failed to remove domain '%s' : %d - state may be inconsistent\n",
+			pd->genpd.name, ret);
 
 	for (i = 0; i < pd->num_clks; i++) {
 		clk_unprepare(pd->clks[i]);
@@ -597,10 +640,12 @@ static int rockchip_pm_domain_probe(struct platform_device *pdev)
 	 * Configure power up and down transition delays for CORE
 	 * and GPU domains.
 	 */
-	rockchip_configure_pd_cnt(pmu, pmu_info->core_pwrcnt_offset,
-				  pmu_info->core_power_transition_time);
-	rockchip_configure_pd_cnt(pmu, pmu_info->gpu_pwrcnt_offset,
-				  pmu_info->gpu_power_transition_time);
+	if (pmu_info->core_power_transition_time)
+		rockchip_configure_pd_cnt(pmu, pmu_info->core_pwrcnt_offset,
+					pmu_info->core_power_transition_time);
+	if (pmu_info->gpu_pwrcnt_offset)
+		rockchip_configure_pd_cnt(pmu, pmu_info->gpu_pwrcnt_offset,
+					pmu_info->gpu_power_transition_time);
 
 	error = -ENODEV;
 
@@ -627,7 +672,11 @@ static int rockchip_pm_domain_probe(struct platform_device *pdev)
 		goto err_out;
 	}
 
-	of_genpd_add_provider_onecell(np, &pmu->genpd_data);
+	error = of_genpd_add_provider_onecell(np, &pmu->genpd_data);
+	if (error) {
+		dev_err(dev, "failed to add provider: %d\n", error);
+		goto err_out;
+	}
 
 	return 0;
 
@@ -722,11 +771,7 @@ static const struct rockchip_pmu_info rk3399_pmu = {
 	.idle_offset = 0x64,
 	.ack_offset = 0x68,
 
-	.core_pwrcnt_offset = 0x9c,
-	.gpu_pwrcnt_offset = 0xa4,
-
-	.core_power_transition_time = 24,
-	.gpu_power_transition_time = 24,
+	/* ARM Trusted Firmware manages power transition times */
 
 	.num_domains = ARRAY_SIZE(rk3399_pm_domains),
 	.domain_info = rk3399_pm_domains,
diff --git a/drivers/soc/tegra/Kconfig b/drivers/soc/tegra/Kconfig
index 03089ad2fc65..e5e124c07066 100644
--- a/drivers/soc/tegra/Kconfig
+++ b/drivers/soc/tegra/Kconfig
@@ -77,5 +77,19 @@ config ARCH_TEGRA_210_SOC
 	  controllers, such as GPIO, I2C, SPI, SDHCI, PCIe, SATA and XHCI, to
 	  name only a few.
 
+config ARCH_TEGRA_186_SOC
+	bool "NVIDIA Tegra186 SoC"
+	select MAILBOX
+	select TEGRA_BPMP
+	select TEGRA_HSP_MBOX
+	select TEGRA_IVC
+	help
+	  Enable support for the NVIDIA Tegar186 SoC. The Tegra186 features a
+	  combination of Denver and Cortex-A57 CPU cores and a GPU based on
+	  the Pascal architecture. It contains an ADSP with a Cortex-A9 CPU
+	  used for audio processing, hardware video encoders/decoders with
+	  multi-format support, ISP for image capture processing and BPMP for
+	  power management.
+
 endif
 endif
diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c
index 7792ed88d80b..e233dd5dcab3 100644
--- a/drivers/soc/tegra/pmc.c
+++ b/drivers/soc/tegra/pmc.c
@@ -45,29 +45,31 @@
 #include <soc/tegra/pmc.h>
 
 #define PMC_CNTRL			0x0
-#define  PMC_CNTRL_SYSCLK_POLARITY	(1 << 10)  /* sys clk polarity */
-#define  PMC_CNTRL_SYSCLK_OE		(1 << 11)  /* system clock enable */
-#define  PMC_CNTRL_SIDE_EFFECT_LP0	(1 << 14)  /* LP0 when CPU pwr gated */
-#define  PMC_CNTRL_CPU_PWRREQ_POLARITY	(1 << 15)  /* CPU pwr req polarity */
-#define  PMC_CNTRL_CPU_PWRREQ_OE	(1 << 16)  /* CPU pwr req enable */
-#define  PMC_CNTRL_INTR_POLARITY	(1 << 17)  /* inverts INTR polarity */
-#define  PMC_CNTRL_MAIN_RST		(1 <<  4)
+#define  PMC_CNTRL_INTR_POLARITY	BIT(17) /* inverts INTR polarity */
+#define  PMC_CNTRL_CPU_PWRREQ_OE	BIT(16) /* CPU pwr req enable */
+#define  PMC_CNTRL_CPU_PWRREQ_POLARITY	BIT(15) /* CPU pwr req polarity */
+#define  PMC_CNTRL_SIDE_EFFECT_LP0	BIT(14) /* LP0 when CPU pwr gated */
+#define  PMC_CNTRL_SYSCLK_OE		BIT(11) /* system clock enable */
+#define  PMC_CNTRL_SYSCLK_POLARITY	BIT(10) /* sys clk polarity */
+#define  PMC_CNTRL_MAIN_RST		BIT(4)
 
 #define DPD_SAMPLE			0x020
-#define  DPD_SAMPLE_ENABLE		(1 << 0)
+#define  DPD_SAMPLE_ENABLE		BIT(0)
 #define  DPD_SAMPLE_DISABLE		(0 << 0)
 
 #define PWRGATE_TOGGLE			0x30
-#define  PWRGATE_TOGGLE_START		(1 << 8)
+#define  PWRGATE_TOGGLE_START		BIT(8)
 
 #define REMOVE_CLAMPING			0x34
 
 #define PWRGATE_STATUS			0x38
 
+#define PMC_PWR_DET			0x48
+
 #define PMC_SCRATCH0			0x50
-#define  PMC_SCRATCH0_MODE_RECOVERY	(1 << 31)
-#define  PMC_SCRATCH0_MODE_BOOTLOADER	(1 << 30)
-#define  PMC_SCRATCH0_MODE_RCM		(1 << 1)
+#define  PMC_SCRATCH0_MODE_RECOVERY	BIT(31)
+#define  PMC_SCRATCH0_MODE_BOOTLOADER	BIT(30)
+#define  PMC_SCRATCH0_MODE_RCM		BIT(1)
 #define  PMC_SCRATCH0_MODE_MASK		(PMC_SCRATCH0_MODE_RECOVERY | \
 					 PMC_SCRATCH0_MODE_BOOTLOADER | \
 					 PMC_SCRATCH0_MODE_RCM)
@@ -75,11 +77,13 @@
 #define PMC_CPUPWRGOOD_TIMER		0xc8
 #define PMC_CPUPWROFF_TIMER		0xcc
 
+#define PMC_PWR_DET_VALUE		0xe4
+
 #define PMC_SCRATCH41			0x140
 
 #define PMC_SENSOR_CTRL			0x1b0
-#define PMC_SENSOR_CTRL_SCRATCH_WRITE	(1 << 2)
-#define PMC_SENSOR_CTRL_ENABLE_RST	(1 << 1)
+#define  PMC_SENSOR_CTRL_SCRATCH_WRITE	BIT(2)
+#define  PMC_SENSOR_CTRL_ENABLE_RST	BIT(1)
 
 #define PMC_RST_STATUS			0x1b4
 #define  PMC_RST_STATUS_POR		0
@@ -90,10 +94,10 @@
 #define  PMC_RST_STATUS_AOTAG		5
 
 #define IO_DPD_REQ			0x1b8
-#define  IO_DPD_REQ_CODE_IDLE		(0 << 30)
-#define  IO_DPD_REQ_CODE_OFF		(1 << 30)
-#define  IO_DPD_REQ_CODE_ON		(2 << 30)
-#define  IO_DPD_REQ_CODE_MASK		(3 << 30)
+#define  IO_DPD_REQ_CODE_IDLE		(0U << 30)
+#define  IO_DPD_REQ_CODE_OFF		(1U << 30)
+#define  IO_DPD_REQ_CODE_ON		(2U << 30)
+#define  IO_DPD_REQ_CODE_MASK		(3U << 30)
 
 #define IO_DPD_STATUS			0x1bc
 #define IO_DPD2_REQ			0x1c0
@@ -101,16 +105,16 @@
 #define SEL_DPD_TIM			0x1c8
 
 #define PMC_SCRATCH54			0x258
-#define PMC_SCRATCH54_DATA_SHIFT	8
-#define PMC_SCRATCH54_ADDR_SHIFT	0
+#define  PMC_SCRATCH54_DATA_SHIFT	8
+#define  PMC_SCRATCH54_ADDR_SHIFT	0
 
 #define PMC_SCRATCH55			0x25c
-#define PMC_SCRATCH55_RESET_TEGRA	(1 << 31)
-#define PMC_SCRATCH55_CNTRL_ID_SHIFT	27
-#define PMC_SCRATCH55_PINMUX_SHIFT	24
-#define PMC_SCRATCH55_16BITOP		(1 << 15)
-#define PMC_SCRATCH55_CHECKSUM_SHIFT	16
-#define PMC_SCRATCH55_I2CSLV1_SHIFT	0
+#define  PMC_SCRATCH55_RESET_TEGRA	BIT(31)
+#define  PMC_SCRATCH55_CNTRL_ID_SHIFT	27
+#define  PMC_SCRATCH55_PINMUX_SHIFT	24
+#define  PMC_SCRATCH55_16BITOP		BIT(15)
+#define  PMC_SCRATCH55_CHECKSUM_SHIFT	16
+#define  PMC_SCRATCH55_I2CSLV1_SHIFT	0
 
 #define GPU_RG_CNTRL			0x2d4
 
@@ -124,6 +128,12 @@ struct tegra_powergate {
 	unsigned int num_resets;
 };
 
+struct tegra_io_pad_soc {
+	enum tegra_io_pad id;
+	unsigned int dpd;
+	unsigned int voltage;
+};
+
 struct tegra_pmc_soc {
 	unsigned int num_powergates;
 	const char *const *powergates;
@@ -132,6 +142,9 @@ struct tegra_pmc_soc {
 
 	bool has_tsense_reset;
 	bool has_gpu_clamps;
+
+	const struct tegra_io_pad_soc *io_pads;
+	unsigned int num_io_pads;
 };
 
 /**
@@ -238,8 +251,6 @@ static int tegra_powergate_lookup(struct tegra_pmc *pmc, const char *name)
 			return i;
 	}
 
-	dev_err(pmc->dev, "powergate %s not found\n", name);
-
 	return -ENODEV;
 }
 
@@ -456,13 +467,12 @@ disable_clks:
 static int tegra_genpd_power_on(struct generic_pm_domain *domain)
 {
 	struct tegra_powergate *pg = to_powergate(domain);
-	struct tegra_pmc *pmc = pg->pmc;
 	int err;
 
 	err = tegra_powergate_power_up(pg, true);
 	if (err)
-		dev_err(pmc->dev, "failed to turn on PM domain %s: %d\n",
-			pg->genpd.name, err);
+		pr_err("failed to turn on PM domain %s: %d\n", pg->genpd.name,
+		       err);
 
 	return err;
 }
@@ -470,13 +480,12 @@ static int tegra_genpd_power_on(struct generic_pm_domain *domain)
 static int tegra_genpd_power_off(struct generic_pm_domain *domain)
 {
 	struct tegra_powergate *pg = to_powergate(domain);
-	struct tegra_pmc *pmc = pg->pmc;
 	int err;
 
 	err = tegra_powergate_power_down(pg);
 	if (err)
-		dev_err(pmc->dev, "failed to turn off PM domain %s: %d\n",
-			pg->genpd.name, err);
+		pr_err("failed to turn off PM domain %s: %d\n",
+		       pg->genpd.name, err);
 
 	return err;
 }
@@ -801,8 +810,7 @@ static void tegra_powergate_add(struct tegra_pmc *pmc, struct device_node *np)
 
 	id = tegra_powergate_lookup(pmc, np->name);
 	if (id < 0) {
-		dev_err(pmc->dev, "powergate lookup failed for %s: %d\n",
-			np->name, id);
+		pr_err("powergate lookup failed for %s: %d\n", np->name, id);
 		goto free_mem;
 	}
 
@@ -822,20 +830,22 @@ static void tegra_powergate_add(struct tegra_pmc *pmc, struct device_node *np)
 
 	err = tegra_powergate_of_get_clks(pg, np);
 	if (err < 0) {
-		dev_err(pmc->dev, "failed to get clocks for %s: %d\n",
-			np->name, err);
+		pr_err("failed to get clocks for %s: %d\n", np->name, err);
 		goto set_available;
 	}
 
 	err = tegra_powergate_of_get_resets(pg, np, off);
 	if (err < 0) {
-		dev_err(pmc->dev, "failed to get resets for %s: %d\n",
-			np->name, err);
+		pr_err("failed to get resets for %s: %d\n", np->name, err);
 		goto remove_clks;
 	}
 
-	if (!IS_ENABLED(CONFIG_PM_GENERIC_DOMAINS))
-		goto power_on_cleanup;
+	if (!IS_ENABLED(CONFIG_PM_GENERIC_DOMAINS)) {
+		if (off)
+			WARN_ON(tegra_powergate_power_up(pg, true));
+
+		goto remove_resets;
+	}
 
 	/*
 	 * FIXME: If XHCI is enabled for Tegra, then power-up the XUSB
@@ -846,25 +856,33 @@ static void tegra_powergate_add(struct tegra_pmc *pmc, struct device_node *np)
 	 * to be unused.
 	 */
 	if (IS_ENABLED(CONFIG_USB_XHCI_TEGRA) &&
-	    (id == TEGRA_POWERGATE_XUSBA || id == TEGRA_POWERGATE_XUSBC))
-		goto power_on_cleanup;
+	    (id == TEGRA_POWERGATE_XUSBA || id == TEGRA_POWERGATE_XUSBC)) {
+		if (off)
+			WARN_ON(tegra_powergate_power_up(pg, true));
+
+		goto remove_resets;
+	}
 
-	pm_genpd_init(&pg->genpd, NULL, off);
+	err = pm_genpd_init(&pg->genpd, NULL, off);
+	if (err < 0) {
+		pr_err("failed to initialise PM domain %s: %d\n", np->name,
+		       err);
+		goto remove_resets;
+	}
 
 	err = of_genpd_add_provider_simple(np, &pg->genpd);
 	if (err < 0) {
-		dev_err(pmc->dev, "failed to add genpd provider for %s: %d\n",
-			np->name, err);
-		goto remove_resets;
+		pr_err("failed to add PM domain provider for %s: %d\n",
+		       np->name, err);
+		goto remove_genpd;
 	}
 
-	dev_dbg(pmc->dev, "added power domain %s\n", pg->genpd.name);
+	pr_debug("added PM domain %s\n", pg->genpd.name);
 
 	return;
 
-power_on_cleanup:
-	if (off)
-		WARN_ON(tegra_powergate_power_up(pg, true));
+remove_genpd:
+	pm_genpd_remove(&pg->genpd);
 
 remove_resets:
 	while (pg->num_resets--)
@@ -908,21 +926,36 @@ static void tegra_powergate_init(struct tegra_pmc *pmc,
 	of_node_put(np);
 }
 
-static int tegra_io_rail_prepare(unsigned int id, unsigned long *request,
-				 unsigned long *status, unsigned int *bit)
+static const struct tegra_io_pad_soc *
+tegra_io_pad_find(struct tegra_pmc *pmc, enum tegra_io_pad id)
 {
+	unsigned int i;
+
+	for (i = 0; i < pmc->soc->num_io_pads; i++)
+		if (pmc->soc->io_pads[i].id == id)
+			return &pmc->soc->io_pads[i];
+
+	return NULL;
+}
+
+static int tegra_io_pad_prepare(enum tegra_io_pad id, unsigned long *request,
+				unsigned long *status, u32 *mask)
+{
+	const struct tegra_io_pad_soc *pad;
 	unsigned long rate, value;
 
-	*bit = id % 32;
+	pad = tegra_io_pad_find(pmc, id);
+	if (!pad) {
+		pr_err("invalid I/O pad ID %u\n", id);
+		return -ENOENT;
+	}
 
-	/*
-	 * There are two sets of 30 bits to select IO rails, but bits 30 and
-	 * 31 are control bits rather than IO rail selection bits.
-	 */
-	if (id > 63 || *bit == 30 || *bit == 31)
-		return -EINVAL;
+	if (pad->dpd == UINT_MAX)
+		return -ENOTSUPP;
 
-	if (id < 32) {
+	*mask = BIT(pad->dpd % 32);
+
+	if (pad->dpd < 32) {
 		*status = IO_DPD_STATUS;
 		*request = IO_DPD_REQ;
 	} else {
@@ -931,6 +964,10 @@ static int tegra_io_rail_prepare(unsigned int id, unsigned long *request,
 	}
 
 	rate = clk_get_rate(pmc->clk);
+	if (!rate) {
+		pr_err("failed to get clock rate\n");
+		return -ENODEV;
+	}
 
 	tegra_pmc_writel(DPD_SAMPLE_ENABLE, DPD_SAMPLE);
 
@@ -942,10 +979,10 @@ static int tegra_io_rail_prepare(unsigned int id, unsigned long *request,
 	return 0;
 }
 
-static int tegra_io_rail_poll(unsigned long offset, unsigned long mask,
-			      unsigned long val, unsigned long timeout)
+static int tegra_io_pad_poll(unsigned long offset, u32 mask,
+			     u32 val, unsigned long timeout)
 {
-	unsigned long value;
+	u32 value;
 
 	timeout = jiffies + msecs_to_jiffies(timeout);
 
@@ -960,67 +997,164 @@ static int tegra_io_rail_poll(unsigned long offset, unsigned long mask,
 	return -ETIMEDOUT;
 }
 
-static void tegra_io_rail_unprepare(void)
+static void tegra_io_pad_unprepare(void)
 {
 	tegra_pmc_writel(DPD_SAMPLE_DISABLE, DPD_SAMPLE);
 }
 
-int tegra_io_rail_power_on(unsigned int id)
+/**
+ * tegra_io_pad_power_enable() - enable power to I/O pad
+ * @id: Tegra I/O pad ID for which to enable power
+ *
+ * Returns: 0 on success or a negative error code on failure.
+ */
+int tegra_io_pad_power_enable(enum tegra_io_pad id)
 {
 	unsigned long request, status;
-	unsigned int bit;
+	u32 mask;
 	int err;
 
 	mutex_lock(&pmc->powergates_lock);
 
-	err = tegra_io_rail_prepare(id, &request, &status, &bit);
-	if (err)
-		goto error;
+	err = tegra_io_pad_prepare(id, &request, &status, &mask);
+	if (err < 0) {
+		pr_err("failed to prepare I/O pad: %d\n", err);
+		goto unlock;
+	}
 
-	tegra_pmc_writel(IO_DPD_REQ_CODE_OFF | BIT(bit), request);
+	tegra_pmc_writel(IO_DPD_REQ_CODE_OFF | mask, request);
 
-	err = tegra_io_rail_poll(status, BIT(bit), 0, 250);
-	if (err) {
-		pr_info("tegra_io_rail_poll() failed: %d\n", err);
-		goto error;
+	err = tegra_io_pad_poll(status, mask, 0, 250);
+	if (err < 0) {
+		pr_err("failed to enable I/O pad: %d\n", err);
+		goto unlock;
 	}
 
-	tegra_io_rail_unprepare();
+	tegra_io_pad_unprepare();
 
-error:
+unlock:
 	mutex_unlock(&pmc->powergates_lock);
-
 	return err;
 }
-EXPORT_SYMBOL(tegra_io_rail_power_on);
+EXPORT_SYMBOL(tegra_io_pad_power_enable);
 
-int tegra_io_rail_power_off(unsigned int id)
+/**
+ * tegra_io_pad_power_disable() - disable power to I/O pad
+ * @id: Tegra I/O pad ID for which to disable power
+ *
+ * Returns: 0 on success or a negative error code on failure.
+ */
+int tegra_io_pad_power_disable(enum tegra_io_pad id)
 {
 	unsigned long request, status;
-	unsigned int bit;
+	u32 mask;
 	int err;
 
 	mutex_lock(&pmc->powergates_lock);
 
-	err = tegra_io_rail_prepare(id, &request, &status, &bit);
-	if (err) {
-		pr_info("tegra_io_rail_prepare() failed: %d\n", err);
-		goto error;
+	err = tegra_io_pad_prepare(id, &request, &status, &mask);
+	if (err < 0) {
+		pr_err("failed to prepare I/O pad: %d\n", err);
+		goto unlock;
 	}
 
-	tegra_pmc_writel(IO_DPD_REQ_CODE_ON | BIT(bit), request);
+	tegra_pmc_writel(IO_DPD_REQ_CODE_ON | mask, request);
 
-	err = tegra_io_rail_poll(status, BIT(bit), BIT(bit), 250);
-	if (err)
-		goto error;
+	err = tegra_io_pad_poll(status, mask, mask, 250);
+	if (err < 0) {
+		pr_err("failed to disable I/O pad: %d\n", err);
+		goto unlock;
+	}
 
-	tegra_io_rail_unprepare();
+	tegra_io_pad_unprepare();
 
-error:
+unlock:
 	mutex_unlock(&pmc->powergates_lock);
-
 	return err;
 }
+EXPORT_SYMBOL(tegra_io_pad_power_disable);
+
+int tegra_io_pad_set_voltage(enum tegra_io_pad id,
+			     enum tegra_io_pad_voltage voltage)
+{
+	const struct tegra_io_pad_soc *pad;
+	u32 value;
+
+	pad = tegra_io_pad_find(pmc, id);
+	if (!pad)
+		return -ENOENT;
+
+	if (pad->voltage == UINT_MAX)
+		return -ENOTSUPP;
+
+	mutex_lock(&pmc->powergates_lock);
+
+	/* write-enable PMC_PWR_DET_VALUE[pad->voltage] */
+	value = tegra_pmc_readl(PMC_PWR_DET);
+	value |= BIT(pad->voltage);
+	tegra_pmc_writel(value, PMC_PWR_DET);
+
+	/* update I/O voltage */
+	value = tegra_pmc_readl(PMC_PWR_DET_VALUE);
+
+	if (voltage == TEGRA_IO_PAD_1800000UV)
+		value &= ~BIT(pad->voltage);
+	else
+		value |= BIT(pad->voltage);
+
+	tegra_pmc_writel(value, PMC_PWR_DET_VALUE);
+
+	mutex_unlock(&pmc->powergates_lock);
+
+	usleep_range(100, 250);
+
+	return 0;
+}
+EXPORT_SYMBOL(tegra_io_pad_set_voltage);
+
+int tegra_io_pad_get_voltage(enum tegra_io_pad id)
+{
+	const struct tegra_io_pad_soc *pad;
+	u32 value;
+
+	pad = tegra_io_pad_find(pmc, id);
+	if (!pad)
+		return -ENOENT;
+
+	if (pad->voltage == UINT_MAX)
+		return -ENOTSUPP;
+
+	value = tegra_pmc_readl(PMC_PWR_DET_VALUE);
+
+	if ((value & BIT(pad->voltage)) == 0)
+		return TEGRA_IO_PAD_1800000UV;
+
+	return TEGRA_IO_PAD_3300000UV;
+}
+EXPORT_SYMBOL(tegra_io_pad_get_voltage);
+
+/**
+ * tegra_io_rail_power_on() - enable power to I/O rail
+ * @id: Tegra I/O pad ID for which to enable power
+ *
+ * See also: tegra_io_pad_power_enable()
+ */
+int tegra_io_rail_power_on(unsigned int id)
+{
+	return tegra_io_pad_power_enable(id);
+}
+EXPORT_SYMBOL(tegra_io_rail_power_on);
+
+/**
+ * tegra_io_rail_power_off() - disable power to I/O rail
+ * @id: Tegra I/O pad ID for which to disable power
+ *
+ * See also: tegra_io_pad_power_disable()
+ */
+int tegra_io_rail_power_off(unsigned int id)
+{
+	return tegra_io_pad_power_disable(id);
+}
 EXPORT_SYMBOL(tegra_io_rail_power_off);
 
 #ifdef CONFIG_PM_SLEEP
@@ -1454,6 +1588,39 @@ static const u8 tegra124_cpu_powergates[] = {
 	TEGRA_POWERGATE_CPU3,
 };
 
+static const struct tegra_io_pad_soc tegra124_io_pads[] = {
+	{ .id = TEGRA_IO_PAD_AUDIO, .dpd = 17, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_BB, .dpd = 15, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_CAM, .dpd = 36, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_COMP, .dpd = 22, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_CSIA, .dpd = 0, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_CSIB, .dpd = 1, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_CSIE, .dpd = 44, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_DSI, .dpd = 2, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_DSIB, .dpd = 39, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_DSIC, .dpd = 40, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_DSID, .dpd = 41, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_HDMI, .dpd = 28, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_HSIC, .dpd = 19, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_HV, .dpd = 38, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_LVDS, .dpd = 57, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_MIPI_BIAS, .dpd = 3, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_NAND, .dpd = 13, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_PEX_BIAS, .dpd = 4, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_PEX_CLK1, .dpd = 5, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_PEX_CLK2, .dpd = 6, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_PEX_CNTRL, .dpd = 32, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_SDMMC1, .dpd = 33, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_SDMMC3, .dpd = 34, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_SDMMC4, .dpd = 35, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_SYS_DDC, .dpd = 58, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_UART, .dpd = 14, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_USB0, .dpd = 9, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_USB1, .dpd = 10, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_USB2, .dpd = 11, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_USB_BIAS, .dpd = 12, .voltage = UINT_MAX },
+};
+
 static const struct tegra_pmc_soc tegra124_pmc_soc = {
 	.num_powergates = ARRAY_SIZE(tegra124_powergates),
 	.powergates = tegra124_powergates,
@@ -1461,6 +1628,8 @@ static const struct tegra_pmc_soc tegra124_pmc_soc = {
 	.cpu_powergates = tegra124_cpu_powergates,
 	.has_tsense_reset = true,
 	.has_gpu_clamps = true,
+	.num_io_pads = ARRAY_SIZE(tegra124_io_pads),
+	.io_pads = tegra124_io_pads,
 };
 
 static const char * const tegra210_powergates[] = {
@@ -1497,6 +1666,47 @@ static const u8 tegra210_cpu_powergates[] = {
 	TEGRA_POWERGATE_CPU3,
 };
 
+static const struct tegra_io_pad_soc tegra210_io_pads[] = {
+	{ .id = TEGRA_IO_PAD_AUDIO, .dpd = 17, .voltage = 5 },
+	{ .id = TEGRA_IO_PAD_AUDIO_HV, .dpd = 61, .voltage = 18 },
+	{ .id = TEGRA_IO_PAD_CAM, .dpd = 36, .voltage = 10 },
+	{ .id = TEGRA_IO_PAD_CSIA, .dpd = 0, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_CSIB, .dpd = 1, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_CSIC, .dpd = 42, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_CSID, .dpd = 43, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_CSIE, .dpd = 44, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_CSIF, .dpd = 45, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_DBG, .dpd = 25, .voltage = 19 },
+	{ .id = TEGRA_IO_PAD_DEBUG_NONAO, .dpd = 26, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_DMIC, .dpd = 50, .voltage = 20 },
+	{ .id = TEGRA_IO_PAD_DP, .dpd = 51, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_DSI, .dpd = 2, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_DSIB, .dpd = 39, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_DSIC, .dpd = 40, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_DSID, .dpd = 41, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_EMMC, .dpd = 35, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_EMMC2, .dpd = 37, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_GPIO, .dpd = 27, .voltage = 21 },
+	{ .id = TEGRA_IO_PAD_HDMI, .dpd = 28, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_HSIC, .dpd = 19, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_LVDS, .dpd = 57, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_MIPI_BIAS, .dpd = 3, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_PEX_BIAS, .dpd = 4, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_PEX_CLK1, .dpd = 5, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_PEX_CLK2, .dpd = 6, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_PEX_CNTRL, .dpd = UINT_MAX, .voltage = 11 },
+	{ .id = TEGRA_IO_PAD_SDMMC1, .dpd = 33, .voltage = 12 },
+	{ .id = TEGRA_IO_PAD_SDMMC3, .dpd = 34, .voltage = 13 },
+	{ .id = TEGRA_IO_PAD_SPI, .dpd = 46, .voltage = 22 },
+	{ .id = TEGRA_IO_PAD_SPI_HV, .dpd = 47, .voltage = 23 },
+	{ .id = TEGRA_IO_PAD_UART, .dpd = 14, .voltage = 2 },
+	{ .id = TEGRA_IO_PAD_USB0, .dpd = 9, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_USB1, .dpd = 10, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_USB2, .dpd = 11, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_USB3, .dpd = 18, .voltage = UINT_MAX },
+	{ .id = TEGRA_IO_PAD_USB_BIAS, .dpd = 12, .voltage = UINT_MAX },
+};
+
 static const struct tegra_pmc_soc tegra210_pmc_soc = {
 	.num_powergates = ARRAY_SIZE(tegra210_powergates),
 	.powergates = tegra210_powergates,
@@ -1504,6 +1714,8 @@ static const struct tegra_pmc_soc tegra210_pmc_soc = {
 	.cpu_powergates = tegra210_cpu_powergates,
 	.has_tsense_reset = true,
 	.has_gpu_clamps = true,
+	.num_io_pads = ARRAY_SIZE(tegra210_io_pads),
+	.io_pads = tegra210_io_pads,
 };
 
 static const struct of_device_id tegra_pmc_match[] = {
diff --git a/drivers/soc/ti/knav_qmss_queue.c b/drivers/soc/ti/knav_qmss_queue.c
index b73e3534f67b..eacad57f2977 100644
--- a/drivers/soc/ti/knav_qmss_queue.c
+++ b/drivers/soc/ti/knav_qmss_queue.c
@@ -1228,7 +1228,7 @@ static int knav_setup_queue_range(struct knav_device *kdev,
 
 		range->num_irqs++;
 
-		if (oirq.args_count == 3)
+		if (IS_ENABLED(CONFIG_SMP) && oirq.args_count == 3)
 			range->irqs[i].cpu_map =
 				(oirq.args[2] & 0x0000ff00) >> 8;
 	}
diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index b7995474148c..ec4aa252d6e8 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -67,6 +67,13 @@ config SPI_ATH79
 	  This enables support for the SPI controller present on the
 	  Atheros AR71XX/AR724X/AR913X SoCs.
 
+config SPI_ARMADA_3700
+	tristate "Marvell Armada 3700 SPI Controller"
+	depends on (ARCH_MVEBU && OF) || COMPILE_TEST
+	help
+	  This enables support for the SPI controller present on the
+	  Marvell Armada 3700 SoCs.
+
 config SPI_ATMEL
 	tristate "Atmel SPI Controller"
 	depends on HAS_DMA
@@ -264,6 +271,12 @@ config SPI_FALCON
 	  has only been tested with m25p80 type chips. The hardware has no
 	  support for other types of SPI peripherals.
 
+config SPI_FSL_LPSPI
+	tristate "Freescale i.MX LPSPI controller"
+	depends on ARCH_MXC || COMPILE_TEST
+	help
+	  This enables Freescale i.MX LPSPI controllers in master mode.
+
 config SPI_GPIO
 	tristate "GPIO-based bitbanging SPI Master"
 	depends on GPIOLIB || COMPILE_TEST
@@ -373,7 +386,6 @@ config SPI_FSL_DSPI
 config SPI_FSL_ESPI
 	tristate "Freescale eSPI controller"
 	depends on FSL_SOC
-	select SPI_FSL_LIB
 	help
 	  This enables using the Freescale eSPI controllers in master mode.
 	  From MPC8536, 85xx platform uses the controller, and all P10xx,
@@ -451,7 +463,8 @@ config SPI_ORION
 	tristate "Orion SPI master"
 	depends on PLAT_ORION || ARCH_MVEBU || COMPILE_TEST
 	help
-	  This enables using the SPI master controller on the Orion chips.
+	  This enables using the SPI master controller on the Orion
+	  and MVEBU chips.
 
 config SPI_PIC32
 	tristate "Microchip PIC32 series SPI"
@@ -553,7 +566,7 @@ config SPI_S3C24XX_FIQ
 
 config SPI_S3C64XX
 	tristate "Samsung S3C64XX series type SPI"
-	depends on (PLAT_SAMSUNG || ARCH_EXYNOS)
+	depends on (PLAT_SAMSUNG || ARCH_EXYNOS || COMPILE_TEST)
 	help
 	  SPI driver for Samsung S3C64XX and newer SoCs.
 
diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile
index aa939d955521..7a6b64662c82 100644
--- a/drivers/spi/Makefile
+++ b/drivers/spi/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_SPI_LOOPBACK_TEST)		+= spi-loopback-test.o
 
 # SPI master controller drivers (bus)
 obj-$(CONFIG_SPI_ALTERA)		+= spi-altera.o
+obj-$(CONFIG_SPI_ARMADA_3700)		+= spi-armada-3700.o
 obj-$(CONFIG_SPI_ATMEL)			+= spi-atmel.o
 obj-$(CONFIG_SPI_ATH79)			+= spi-ath79.o
 obj-$(CONFIG_SPI_AU1550)		+= spi-au1550.o
@@ -43,6 +44,7 @@ obj-$(CONFIG_SPI_FSL_CPM)		+= spi-fsl-cpm.o
 obj-$(CONFIG_SPI_FSL_DSPI)		+= spi-fsl-dspi.o
 obj-$(CONFIG_SPI_FSL_LIB)		+= spi-fsl-lib.o
 obj-$(CONFIG_SPI_FSL_ESPI)		+= spi-fsl-espi.o
+obj-$(CONFIG_SPI_FSL_LPSPI)		+= spi-fsl-lpspi.o
 obj-$(CONFIG_SPI_FSL_SPI)		+= spi-fsl-spi.o
 obj-$(CONFIG_SPI_GPIO)			+= spi-gpio.o
 obj-$(CONFIG_SPI_IMG_SPFI)		+= spi-img-spfi.o
diff --git a/drivers/spi/spi-armada-3700.c b/drivers/spi/spi-armada-3700.c
new file mode 100644
index 000000000000..e89da0af45d2
--- /dev/null
+++ b/drivers/spi/spi-armada-3700.c
@@ -0,0 +1,923 @@
+/*
+ * Marvell Armada-3700 SPI controller driver
+ *
+ * Copyright (C) 2016 Marvell Ltd.
+ *
+ * Author: Wilson Ding <dingwei@marvell.com>
+ * Author: Romain Perier <romain.perier@free-electrons.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/of_device.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/spi/spi.h>
+
+#define DRIVER_NAME			"armada_3700_spi"
+
+#define A3700_SPI_TIMEOUT		10
+
+/* SPI Register Offest */
+#define A3700_SPI_IF_CTRL_REG		0x00
+#define A3700_SPI_IF_CFG_REG		0x04
+#define A3700_SPI_DATA_OUT_REG		0x08
+#define A3700_SPI_DATA_IN_REG		0x0C
+#define A3700_SPI_IF_INST_REG		0x10
+#define A3700_SPI_IF_ADDR_REG		0x14
+#define A3700_SPI_IF_RMODE_REG		0x18
+#define A3700_SPI_IF_HDR_CNT_REG	0x1C
+#define A3700_SPI_IF_DIN_CNT_REG	0x20
+#define A3700_SPI_IF_TIME_REG		0x24
+#define A3700_SPI_INT_STAT_REG		0x28
+#define A3700_SPI_INT_MASK_REG		0x2C
+
+/* A3700_SPI_IF_CTRL_REG */
+#define A3700_SPI_EN			BIT(16)
+#define A3700_SPI_ADDR_NOT_CONFIG	BIT(12)
+#define A3700_SPI_WFIFO_OVERFLOW	BIT(11)
+#define A3700_SPI_WFIFO_UNDERFLOW	BIT(10)
+#define A3700_SPI_RFIFO_OVERFLOW	BIT(9)
+#define A3700_SPI_RFIFO_UNDERFLOW	BIT(8)
+#define A3700_SPI_WFIFO_FULL		BIT(7)
+#define A3700_SPI_WFIFO_EMPTY		BIT(6)
+#define A3700_SPI_RFIFO_FULL		BIT(5)
+#define A3700_SPI_RFIFO_EMPTY		BIT(4)
+#define A3700_SPI_WFIFO_RDY		BIT(3)
+#define A3700_SPI_RFIFO_RDY		BIT(2)
+#define A3700_SPI_XFER_RDY		BIT(1)
+#define A3700_SPI_XFER_DONE		BIT(0)
+
+/* A3700_SPI_IF_CFG_REG */
+#define A3700_SPI_WFIFO_THRS		BIT(28)
+#define A3700_SPI_RFIFO_THRS		BIT(24)
+#define A3700_SPI_AUTO_CS		BIT(20)
+#define A3700_SPI_DMA_RD_EN		BIT(18)
+#define A3700_SPI_FIFO_MODE		BIT(17)
+#define A3700_SPI_SRST			BIT(16)
+#define A3700_SPI_XFER_START		BIT(15)
+#define A3700_SPI_XFER_STOP		BIT(14)
+#define A3700_SPI_INST_PIN		BIT(13)
+#define A3700_SPI_ADDR_PIN		BIT(12)
+#define A3700_SPI_DATA_PIN1		BIT(11)
+#define A3700_SPI_DATA_PIN0		BIT(10)
+#define A3700_SPI_FIFO_FLUSH		BIT(9)
+#define A3700_SPI_RW_EN			BIT(8)
+#define A3700_SPI_CLK_POL		BIT(7)
+#define A3700_SPI_CLK_PHA		BIT(6)
+#define A3700_SPI_BYTE_LEN		BIT(5)
+#define A3700_SPI_CLK_PRESCALE		BIT(0)
+#define A3700_SPI_CLK_PRESCALE_MASK	(0x1f)
+
+#define A3700_SPI_WFIFO_THRS_BIT	28
+#define A3700_SPI_RFIFO_THRS_BIT	24
+#define A3700_SPI_FIFO_THRS_MASK	0x7
+
+#define A3700_SPI_DATA_PIN_MASK		0x3
+
+/* A3700_SPI_IF_HDR_CNT_REG */
+#define A3700_SPI_DUMMY_CNT_BIT		12
+#define A3700_SPI_DUMMY_CNT_MASK	0x7
+#define A3700_SPI_RMODE_CNT_BIT		8
+#define A3700_SPI_RMODE_CNT_MASK	0x3
+#define A3700_SPI_ADDR_CNT_BIT		4
+#define A3700_SPI_ADDR_CNT_MASK		0x7
+#define A3700_SPI_INSTR_CNT_BIT		0
+#define A3700_SPI_INSTR_CNT_MASK	0x3
+
+/* A3700_SPI_IF_TIME_REG */
+#define A3700_SPI_CLK_CAPT_EDGE		BIT(7)
+
+/* Flags and macros for struct a3700_spi */
+#define A3700_INSTR_CNT			1
+#define A3700_ADDR_CNT			3
+#define A3700_DUMMY_CNT			1
+
+struct a3700_spi {
+	struct spi_master *master;
+	void __iomem *base;
+	struct clk *clk;
+	unsigned int irq;
+	unsigned int flags;
+	bool xmit_data;
+	const u8 *tx_buf;
+	u8 *rx_buf;
+	size_t buf_len;
+	u8 byte_len;
+	u32 wait_mask;
+	struct completion done;
+	u32 addr_cnt;
+	u32 instr_cnt;
+	size_t hdr_cnt;
+};
+
+static u32 spireg_read(struct a3700_spi *a3700_spi, u32 offset)
+{
+	return readl(a3700_spi->base + offset);
+}
+
+static void spireg_write(struct a3700_spi *a3700_spi, u32 offset, u32 data)
+{
+	writel(data, a3700_spi->base + offset);
+}
+
+static void a3700_spi_auto_cs_unset(struct a3700_spi *a3700_spi)
+{
+	u32 val;
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+	val &= ~A3700_SPI_AUTO_CS;
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+}
+
+static void a3700_spi_activate_cs(struct a3700_spi *a3700_spi, unsigned int cs)
+{
+	u32 val;
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CTRL_REG);
+	val |= (A3700_SPI_EN << cs);
+	spireg_write(a3700_spi, A3700_SPI_IF_CTRL_REG, val);
+}
+
+static void a3700_spi_deactivate_cs(struct a3700_spi *a3700_spi,
+				    unsigned int cs)
+{
+	u32 val;
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CTRL_REG);
+	val &= ~(A3700_SPI_EN << cs);
+	spireg_write(a3700_spi, A3700_SPI_IF_CTRL_REG, val);
+}
+
+static int a3700_spi_pin_mode_set(struct a3700_spi *a3700_spi,
+				  unsigned int pin_mode)
+{
+	u32 val;
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+	val &= ~(A3700_SPI_INST_PIN | A3700_SPI_ADDR_PIN);
+	val &= ~(A3700_SPI_DATA_PIN0 | A3700_SPI_DATA_PIN1);
+
+	switch (pin_mode) {
+	case 1:
+		break;
+	case 2:
+		val |= A3700_SPI_DATA_PIN0;
+		break;
+	case 4:
+		val |= A3700_SPI_DATA_PIN1;
+		break;
+	default:
+		dev_err(&a3700_spi->master->dev, "wrong pin mode %u", pin_mode);
+		return -EINVAL;
+	}
+
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+
+	return 0;
+}
+
+static void a3700_spi_fifo_mode_set(struct a3700_spi *a3700_spi)
+{
+	u32 val;
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+	val |= A3700_SPI_FIFO_MODE;
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+}
+
+static void a3700_spi_mode_set(struct a3700_spi *a3700_spi,
+			       unsigned int mode_bits)
+{
+	u32 val;
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+
+	if (mode_bits & SPI_CPOL)
+		val |= A3700_SPI_CLK_POL;
+	else
+		val &= ~A3700_SPI_CLK_POL;
+
+	if (mode_bits & SPI_CPHA)
+		val |= A3700_SPI_CLK_PHA;
+	else
+		val &= ~A3700_SPI_CLK_PHA;
+
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+}
+
+static void a3700_spi_clock_set(struct a3700_spi *a3700_spi,
+				unsigned int speed_hz, u16 mode)
+{
+	u32 val;
+	u32 prescale;
+
+	prescale = DIV_ROUND_UP(clk_get_rate(a3700_spi->clk), speed_hz);
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+	val = val & ~A3700_SPI_CLK_PRESCALE_MASK;
+
+	val = val | (prescale & A3700_SPI_CLK_PRESCALE_MASK);
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+
+	if (prescale <= 2) {
+		val = spireg_read(a3700_spi, A3700_SPI_IF_TIME_REG);
+		val |= A3700_SPI_CLK_CAPT_EDGE;
+		spireg_write(a3700_spi, A3700_SPI_IF_TIME_REG, val);
+	}
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+	val &= ~(A3700_SPI_CLK_POL | A3700_SPI_CLK_PHA);
+
+	if (mode & SPI_CPOL)
+		val |= A3700_SPI_CLK_POL;
+
+	if (mode & SPI_CPHA)
+		val |= A3700_SPI_CLK_PHA;
+
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+}
+
+static void a3700_spi_bytelen_set(struct a3700_spi *a3700_spi, unsigned int len)
+{
+	u32 val;
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+	if (len == 4)
+		val |= A3700_SPI_BYTE_LEN;
+	else
+		val &= ~A3700_SPI_BYTE_LEN;
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+
+	a3700_spi->byte_len = len;
+}
+
+static int a3700_spi_fifo_flush(struct a3700_spi *a3700_spi)
+{
+	int timeout = A3700_SPI_TIMEOUT;
+	u32 val;
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+	val |= A3700_SPI_FIFO_FLUSH;
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+
+	while (--timeout) {
+		val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+		if (!(val & A3700_SPI_FIFO_FLUSH))
+			return 0;
+		udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int a3700_spi_init(struct a3700_spi *a3700_spi)
+{
+	struct spi_master *master = a3700_spi->master;
+	u32 val;
+	int i, ret = 0;
+
+	/* Reset SPI unit */
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+	val |= A3700_SPI_SRST;
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+
+	udelay(A3700_SPI_TIMEOUT);
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+	val &= ~A3700_SPI_SRST;
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+
+	/* Disable AUTO_CS and deactivate all chip-selects */
+	a3700_spi_auto_cs_unset(a3700_spi);
+	for (i = 0; i < master->num_chipselect; i++)
+		a3700_spi_deactivate_cs(a3700_spi, i);
+
+	/* Enable FIFO mode */
+	a3700_spi_fifo_mode_set(a3700_spi);
+
+	/* Set SPI mode */
+	a3700_spi_mode_set(a3700_spi, master->mode_bits);
+
+	/* Reset counters */
+	spireg_write(a3700_spi, A3700_SPI_IF_HDR_CNT_REG, 0);
+	spireg_write(a3700_spi, A3700_SPI_IF_DIN_CNT_REG, 0);
+
+	/* Mask the interrupts and clear cause bits */
+	spireg_write(a3700_spi, A3700_SPI_INT_MASK_REG, 0);
+	spireg_write(a3700_spi, A3700_SPI_INT_STAT_REG, ~0U);
+
+	return ret;
+}
+
+static irqreturn_t a3700_spi_interrupt(int irq, void *dev_id)
+{
+	struct spi_master *master = dev_id;
+	struct a3700_spi *a3700_spi;
+	u32 cause;
+
+	a3700_spi = spi_master_get_devdata(master);
+
+	/* Get interrupt causes */
+	cause = spireg_read(a3700_spi, A3700_SPI_INT_STAT_REG);
+
+	if (!cause || !(a3700_spi->wait_mask & cause))
+		return IRQ_NONE;
+
+	/* mask and acknowledge the SPI interrupts */
+	spireg_write(a3700_spi, A3700_SPI_INT_MASK_REG, 0);
+	spireg_write(a3700_spi, A3700_SPI_INT_STAT_REG, cause);
+
+	/* Wake up the transfer */
+	if (a3700_spi->wait_mask & cause)
+		complete(&a3700_spi->done);
+
+	return IRQ_HANDLED;
+}
+
+static bool a3700_spi_wait_completion(struct spi_device *spi)
+{
+	struct a3700_spi *a3700_spi;
+	unsigned int timeout;
+	unsigned int ctrl_reg;
+	unsigned long timeout_jiffies;
+
+	a3700_spi = spi_master_get_devdata(spi->master);
+
+	/* SPI interrupt is edge-triggered, which means an interrupt will
+	 * be generated only when detecting a specific status bit changed
+	 * from '0' to '1'. So when we start waiting for a interrupt, we
+	 * need to check status bit in control reg first, if it is already 1,
+	 * then we do not need to wait for interrupt
+	 */
+	ctrl_reg = spireg_read(a3700_spi, A3700_SPI_IF_CTRL_REG);
+	if (a3700_spi->wait_mask & ctrl_reg)
+		return true;
+
+	reinit_completion(&a3700_spi->done);
+
+	spireg_write(a3700_spi, A3700_SPI_INT_MASK_REG,
+		     a3700_spi->wait_mask);
+
+	timeout_jiffies = msecs_to_jiffies(A3700_SPI_TIMEOUT);
+	timeout = wait_for_completion_timeout(&a3700_spi->done,
+					      timeout_jiffies);
+
+	a3700_spi->wait_mask = 0;
+
+	if (timeout)
+		return true;
+
+	/* there might be the case that right after we checked the
+	 * status bits in this routine and before start to wait for
+	 * interrupt by wait_for_completion_timeout, the interrupt
+	 * happens, to avoid missing it we need to double check
+	 * status bits in control reg, if it is already 1, then
+	 * consider that we have the interrupt successfully and
+	 * return true.
+	 */
+	ctrl_reg = spireg_read(a3700_spi, A3700_SPI_IF_CTRL_REG);
+	if (a3700_spi->wait_mask & ctrl_reg)
+		return true;
+
+	spireg_write(a3700_spi, A3700_SPI_INT_MASK_REG, 0);
+
+	return true;
+}
+
+static bool a3700_spi_transfer_wait(struct spi_device *spi,
+				    unsigned int bit_mask)
+{
+	struct a3700_spi *a3700_spi;
+
+	a3700_spi = spi_master_get_devdata(spi->master);
+	a3700_spi->wait_mask = bit_mask;
+
+	return a3700_spi_wait_completion(spi);
+}
+
+static void a3700_spi_fifo_thres_set(struct a3700_spi *a3700_spi,
+				     unsigned int bytes)
+{
+	u32 val;
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+	val &= ~(A3700_SPI_FIFO_THRS_MASK << A3700_SPI_RFIFO_THRS_BIT);
+	val |= (bytes - 1) << A3700_SPI_RFIFO_THRS_BIT;
+	val &= ~(A3700_SPI_FIFO_THRS_MASK << A3700_SPI_WFIFO_THRS_BIT);
+	val |= (7 - bytes) << A3700_SPI_WFIFO_THRS_BIT;
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+}
+
+static void a3700_spi_transfer_setup(struct spi_device *spi,
+				    struct spi_transfer *xfer)
+{
+	struct a3700_spi *a3700_spi;
+	unsigned int byte_len;
+
+	a3700_spi = spi_master_get_devdata(spi->master);
+
+	a3700_spi_clock_set(a3700_spi, xfer->speed_hz, spi->mode);
+
+	byte_len = xfer->bits_per_word >> 3;
+
+	a3700_spi_fifo_thres_set(a3700_spi, byte_len);
+}
+
+static void a3700_spi_set_cs(struct spi_device *spi, bool enable)
+{
+	struct a3700_spi *a3700_spi = spi_master_get_devdata(spi->master);
+
+	if (!enable)
+		a3700_spi_activate_cs(a3700_spi, spi->chip_select);
+	else
+		a3700_spi_deactivate_cs(a3700_spi, spi->chip_select);
+}
+
+static void a3700_spi_header_set(struct a3700_spi *a3700_spi)
+{
+	u32 instr_cnt = 0, addr_cnt = 0, dummy_cnt = 0;
+	u32 val = 0;
+
+	/* Clear the header registers */
+	spireg_write(a3700_spi, A3700_SPI_IF_INST_REG, 0);
+	spireg_write(a3700_spi, A3700_SPI_IF_ADDR_REG, 0);
+	spireg_write(a3700_spi, A3700_SPI_IF_RMODE_REG, 0);
+
+	/* Set header counters */
+	if (a3700_spi->tx_buf) {
+		if (a3700_spi->buf_len <= a3700_spi->instr_cnt) {
+			instr_cnt = a3700_spi->buf_len;
+		} else if (a3700_spi->buf_len <= (a3700_spi->instr_cnt +
+						  a3700_spi->addr_cnt)) {
+			instr_cnt = a3700_spi->instr_cnt;
+			addr_cnt = a3700_spi->buf_len - instr_cnt;
+		} else if (a3700_spi->buf_len <= a3700_spi->hdr_cnt) {
+			instr_cnt = a3700_spi->instr_cnt;
+			addr_cnt = a3700_spi->addr_cnt;
+			/* Need to handle the normal write case with 1 byte
+			 * data
+			 */
+			if (!a3700_spi->tx_buf[instr_cnt + addr_cnt])
+				dummy_cnt = a3700_spi->buf_len - instr_cnt -
+					    addr_cnt;
+		}
+		val |= ((instr_cnt & A3700_SPI_INSTR_CNT_MASK)
+			<< A3700_SPI_INSTR_CNT_BIT);
+		val |= ((addr_cnt & A3700_SPI_ADDR_CNT_MASK)
+			<< A3700_SPI_ADDR_CNT_BIT);
+		val |= ((dummy_cnt & A3700_SPI_DUMMY_CNT_MASK)
+			<< A3700_SPI_DUMMY_CNT_BIT);
+	}
+	spireg_write(a3700_spi, A3700_SPI_IF_HDR_CNT_REG, val);
+
+	/* Update the buffer length to be transferred */
+	a3700_spi->buf_len -= (instr_cnt + addr_cnt + dummy_cnt);
+
+	/* Set Instruction */
+	val = 0;
+	while (instr_cnt--) {
+		val = (val << 8) | a3700_spi->tx_buf[0];
+		a3700_spi->tx_buf++;
+	}
+	spireg_write(a3700_spi, A3700_SPI_IF_INST_REG, val);
+
+	/* Set Address */
+	val = 0;
+	while (addr_cnt--) {
+		val = (val << 8) | a3700_spi->tx_buf[0];
+		a3700_spi->tx_buf++;
+	}
+	spireg_write(a3700_spi, A3700_SPI_IF_ADDR_REG, val);
+}
+
+static int a3700_is_wfifo_full(struct a3700_spi *a3700_spi)
+{
+	u32 val;
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CTRL_REG);
+	return (val & A3700_SPI_WFIFO_FULL);
+}
+
+static int a3700_spi_fifo_write(struct a3700_spi *a3700_spi)
+{
+	u32 val;
+	int i = 0;
+
+	while (!a3700_is_wfifo_full(a3700_spi) && a3700_spi->buf_len) {
+		val = 0;
+		if (a3700_spi->buf_len >= 4) {
+			val = cpu_to_le32(*(u32 *)a3700_spi->tx_buf);
+			spireg_write(a3700_spi, A3700_SPI_DATA_OUT_REG, val);
+
+			a3700_spi->buf_len -= 4;
+			a3700_spi->tx_buf += 4;
+		} else {
+			/*
+			 * If the remained buffer length is less than 4-bytes,
+			 * we should pad the write buffer with all ones. So that
+			 * it avoids overwrite the unexpected bytes following
+			 * the last one.
+			 */
+			val = GENMASK(31, 0);
+			while (a3700_spi->buf_len) {
+				val &= ~(0xff << (8 * i));
+				val |= *a3700_spi->tx_buf++ << (8 * i);
+				i++;
+				a3700_spi->buf_len--;
+
+				spireg_write(a3700_spi, A3700_SPI_DATA_OUT_REG,
+					     val);
+			}
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int a3700_is_rfifo_empty(struct a3700_spi *a3700_spi)
+{
+	u32 val = spireg_read(a3700_spi, A3700_SPI_IF_CTRL_REG);
+
+	return (val & A3700_SPI_RFIFO_EMPTY);
+}
+
+static int a3700_spi_fifo_read(struct a3700_spi *a3700_spi)
+{
+	u32 val;
+
+	while (!a3700_is_rfifo_empty(a3700_spi) && a3700_spi->buf_len) {
+		val = spireg_read(a3700_spi, A3700_SPI_DATA_IN_REG);
+		if (a3700_spi->buf_len >= 4) {
+			u32 data = le32_to_cpu(val);
+			memcpy(a3700_spi->rx_buf, &data, 4);
+
+			a3700_spi->buf_len -= 4;
+			a3700_spi->rx_buf += 4;
+		} else {
+			/*
+			 * When remain bytes is not larger than 4, we should
+			 * avoid memory overwriting and just write the left rx
+			 * buffer bytes.
+			 */
+			while (a3700_spi->buf_len) {
+				*a3700_spi->rx_buf = val & 0xff;
+				val >>= 8;
+
+				a3700_spi->buf_len--;
+				a3700_spi->rx_buf++;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static void a3700_spi_transfer_abort_fifo(struct a3700_spi *a3700_spi)
+{
+	int timeout = A3700_SPI_TIMEOUT;
+	u32 val;
+
+	val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+	val |= A3700_SPI_XFER_STOP;
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+
+	while (--timeout) {
+		val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+		if (!(val & A3700_SPI_XFER_START))
+			break;
+		udelay(1);
+	}
+
+	a3700_spi_fifo_flush(a3700_spi);
+
+	val &= ~A3700_SPI_XFER_STOP;
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+}
+
+static int a3700_spi_prepare_message(struct spi_master *master,
+				     struct spi_message *message)
+{
+	struct a3700_spi *a3700_spi = spi_master_get_devdata(master);
+	struct spi_device *spi = message->spi;
+	int ret;
+
+	ret = clk_enable(a3700_spi->clk);
+	if (ret) {
+		dev_err(&spi->dev, "failed to enable clk with error %d\n", ret);
+		return ret;
+	}
+
+	/* Flush the FIFOs */
+	ret = a3700_spi_fifo_flush(a3700_spi);
+	if (ret)
+		return ret;
+
+	a3700_spi_bytelen_set(a3700_spi, 4);
+
+	return 0;
+}
+
+static int a3700_spi_transfer_one(struct spi_master *master,
+				  struct spi_device *spi,
+				  struct spi_transfer *xfer)
+{
+	struct a3700_spi *a3700_spi = spi_master_get_devdata(master);
+	int ret = 0, timeout = A3700_SPI_TIMEOUT;
+	unsigned int nbits = 0;
+	u32 val;
+
+	a3700_spi_transfer_setup(spi, xfer);
+
+	a3700_spi->tx_buf  = xfer->tx_buf;
+	a3700_spi->rx_buf  = xfer->rx_buf;
+	a3700_spi->buf_len = xfer->len;
+
+	/* SPI transfer headers */
+	a3700_spi_header_set(a3700_spi);
+
+	if (xfer->tx_buf)
+		nbits = xfer->tx_nbits;
+	else if (xfer->rx_buf)
+		nbits = xfer->rx_nbits;
+
+	a3700_spi_pin_mode_set(a3700_spi, nbits);
+
+	if (xfer->rx_buf) {
+		/* Set read data length */
+		spireg_write(a3700_spi, A3700_SPI_IF_DIN_CNT_REG,
+			     a3700_spi->buf_len);
+		/* Start READ transfer */
+		val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+		val &= ~A3700_SPI_RW_EN;
+		val |= A3700_SPI_XFER_START;
+		spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+	} else if (xfer->tx_buf) {
+		/* Start Write transfer */
+		val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+		val |= (A3700_SPI_XFER_START | A3700_SPI_RW_EN);
+		spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+
+		/*
+		 * If there are data to be written to the SPI device, xmit_data
+		 * flag is set true; otherwise the instruction in SPI_INSTR does
+		 * not require data to be written to the SPI device, then
+		 * xmit_data flag is set false.
+		 */
+		a3700_spi->xmit_data = (a3700_spi->buf_len != 0);
+	}
+
+	while (a3700_spi->buf_len) {
+		if (a3700_spi->tx_buf) {
+			/* Wait wfifo ready */
+			if (!a3700_spi_transfer_wait(spi,
+						     A3700_SPI_WFIFO_RDY)) {
+				dev_err(&spi->dev,
+					"wait wfifo ready timed out\n");
+				ret = -ETIMEDOUT;
+				goto error;
+			}
+			/* Fill up the wfifo */
+			ret = a3700_spi_fifo_write(a3700_spi);
+			if (ret)
+				goto error;
+		} else if (a3700_spi->rx_buf) {
+			/* Wait rfifo ready */
+			if (!a3700_spi_transfer_wait(spi,
+						     A3700_SPI_RFIFO_RDY)) {
+				dev_err(&spi->dev,
+					"wait rfifo ready timed out\n");
+				ret = -ETIMEDOUT;
+				goto error;
+			}
+			/* Drain out the rfifo */
+			ret = a3700_spi_fifo_read(a3700_spi);
+			if (ret)
+				goto error;
+		}
+	}
+
+	/*
+	 * Stop a write transfer in fifo mode:
+	 *	- wait all the bytes in wfifo to be shifted out
+	 *	 - set XFER_STOP bit
+	 *	- wait XFER_START bit clear
+	 *	- clear XFER_STOP bit
+	 * Stop a read transfer in fifo mode:
+	 *	- the hardware is to reset the XFER_START bit
+	 *	   after the number of bytes indicated in DIN_CNT
+	 *	   register
+	 *	- just wait XFER_START bit clear
+	 */
+	if (a3700_spi->tx_buf) {
+		if (a3700_spi->xmit_data) {
+			/*
+			 * If there are data written to the SPI device, wait
+			 * until SPI_WFIFO_EMPTY is 1 to wait for all data to
+			 * transfer out of write FIFO.
+			 */
+			if (!a3700_spi_transfer_wait(spi,
+						     A3700_SPI_WFIFO_EMPTY)) {
+				dev_err(&spi->dev, "wait wfifo empty timed out\n");
+				return -ETIMEDOUT;
+			}
+		} else {
+			/*
+			 * If the instruction in SPI_INSTR does not require data
+			 * to be written to the SPI device, wait until SPI_RDY
+			 * is 1 for the SPI interface to be in idle.
+			 */
+			if (!a3700_spi_transfer_wait(spi, A3700_SPI_XFER_RDY)) {
+				dev_err(&spi->dev, "wait xfer ready timed out\n");
+				return -ETIMEDOUT;
+			}
+		}
+
+		val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+		val |= A3700_SPI_XFER_STOP;
+		spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+	}
+
+	while (--timeout) {
+		val = spireg_read(a3700_spi, A3700_SPI_IF_CFG_REG);
+		if (!(val & A3700_SPI_XFER_START))
+			break;
+		udelay(1);
+	}
+
+	if (timeout == 0) {
+		dev_err(&spi->dev, "wait transfer start clear timed out\n");
+		ret = -ETIMEDOUT;
+		goto error;
+	}
+
+	val &= ~A3700_SPI_XFER_STOP;
+	spireg_write(a3700_spi, A3700_SPI_IF_CFG_REG, val);
+	goto out;
+
+error:
+	a3700_spi_transfer_abort_fifo(a3700_spi);
+out:
+	spi_finalize_current_transfer(master);
+
+	return ret;
+}
+
+static int a3700_spi_unprepare_message(struct spi_master *master,
+				       struct spi_message *message)
+{
+	struct a3700_spi *a3700_spi = spi_master_get_devdata(master);
+
+	clk_disable(a3700_spi->clk);
+
+	return 0;
+}
+
+static const struct of_device_id a3700_spi_dt_ids[] = {
+	{ .compatible = "marvell,armada-3700-spi", .data = NULL },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, a3700_spi_dt_ids);
+
+static int a3700_spi_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *of_node = dev->of_node;
+	struct resource *res;
+	struct spi_master *master;
+	struct a3700_spi *spi;
+	u32 num_cs = 0;
+	int ret = 0;
+
+	master = spi_alloc_master(dev, sizeof(*spi));
+	if (!master) {
+		dev_err(dev, "master allocation failed\n");
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	if (of_property_read_u32(of_node, "num-cs", &num_cs)) {
+		dev_err(dev, "could not find num-cs\n");
+		ret = -ENXIO;
+		goto error;
+	}
+
+	master->bus_num = pdev->id;
+	master->dev.of_node = of_node;
+	master->mode_bits = SPI_MODE_3;
+	master->num_chipselect = num_cs;
+	master->bits_per_word_mask = SPI_BPW_MASK(8) | SPI_BPW_MASK(32);
+	master->prepare_message =  a3700_spi_prepare_message;
+	master->transfer_one = a3700_spi_transfer_one;
+	master->unprepare_message = a3700_spi_unprepare_message;
+	master->set_cs = a3700_spi_set_cs;
+	master->flags = SPI_MASTER_HALF_DUPLEX;
+	master->mode_bits |= (SPI_RX_DUAL | SPI_RX_DUAL |
+			      SPI_RX_QUAD | SPI_TX_QUAD);
+
+	platform_set_drvdata(pdev, master);
+
+	spi = spi_master_get_devdata(master);
+	memset(spi, 0, sizeof(struct a3700_spi));
+
+	spi->master = master;
+	spi->instr_cnt = A3700_INSTR_CNT;
+	spi->addr_cnt = A3700_ADDR_CNT;
+	spi->hdr_cnt = A3700_INSTR_CNT + A3700_ADDR_CNT +
+		       A3700_DUMMY_CNT;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	spi->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(spi->base)) {
+		ret = PTR_ERR(spi->base);
+		goto error;
+	}
+
+	spi->irq = platform_get_irq(pdev, 0);
+	if (spi->irq < 0) {
+		dev_err(dev, "could not get irq: %d\n", spi->irq);
+		ret = -ENXIO;
+		goto error;
+	}
+
+	init_completion(&spi->done);
+
+	spi->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(spi->clk)) {
+		dev_err(dev, "could not find clk: %ld\n", PTR_ERR(spi->clk));
+		goto error;
+	}
+
+	ret = clk_prepare(spi->clk);
+	if (ret) {
+		dev_err(dev, "could not prepare clk: %d\n", ret);
+		goto error;
+	}
+
+	ret = a3700_spi_init(spi);
+	if (ret)
+		goto error_clk;
+
+	ret = devm_request_irq(dev, spi->irq, a3700_spi_interrupt, 0,
+			       dev_name(dev), master);
+	if (ret) {
+		dev_err(dev, "could not request IRQ: %d\n", ret);
+		goto error_clk;
+	}
+
+	ret = devm_spi_register_master(dev, master);
+	if (ret) {
+		dev_err(dev, "Failed to register master\n");
+		goto error_clk;
+	}
+
+	return 0;
+
+error_clk:
+	clk_disable_unprepare(spi->clk);
+error:
+	spi_master_put(master);
+out:
+	return ret;
+}
+
+static int a3700_spi_remove(struct platform_device *pdev)
+{
+	struct spi_master *master = platform_get_drvdata(pdev);
+	struct a3700_spi *spi = spi_master_get_devdata(master);
+
+	clk_unprepare(spi->clk);
+	spi_master_put(master);
+
+	return 0;
+}
+
+static struct platform_driver a3700_spi_driver = {
+	.driver = {
+		.name	= DRIVER_NAME,
+		.owner	= THIS_MODULE,
+		.of_match_table = of_match_ptr(a3700_spi_dt_ids),
+	},
+	.probe		= a3700_spi_probe,
+	.remove		= a3700_spi_remove,
+};
+
+module_platform_driver(a3700_spi_driver);
+
+MODULE_DESCRIPTION("Armada-3700 SPI driver");
+MODULE_AUTHOR("Wilson Ding <dingwei@marvell.com>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:" DRIVER_NAME);
diff --git a/drivers/spi/spi-ath79.c b/drivers/spi/spi-ath79.c
index 6165bf21d427..f369174fbd88 100644
--- a/drivers/spi/spi-ath79.c
+++ b/drivers/spi/spi-ath79.c
@@ -304,6 +304,7 @@ static const struct of_device_id ath79_spi_of_match[] = {
 	{ .compatible = "qca,ar7100-spi", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, ath79_spi_of_match);
 
 static struct platform_driver ath79_spi_driver = {
 	.probe		= ath79_spi_probe,
diff --git a/drivers/spi/spi-atmel.c b/drivers/spi/spi-atmel.c
index 8feac599e9ab..0e7712bac3b6 100644
--- a/drivers/spi/spi-atmel.c
+++ b/drivers/spi/spi-atmel.c
@@ -24,6 +24,7 @@
 
 #include <linux/io.h>
 #include <linux/gpio.h>
+#include <linux/of_gpio.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/pm_runtime.h>
 
@@ -264,17 +265,6 @@
 
 #define AUTOSUSPEND_TIMEOUT	2000
 
-struct atmel_spi_dma {
-	struct dma_chan			*chan_rx;
-	struct dma_chan			*chan_tx;
-	struct scatterlist		sgrx;
-	struct scatterlist		sgtx;
-	struct dma_async_tx_descriptor	*data_desc_rx;
-	struct dma_async_tx_descriptor	*data_desc_tx;
-
-	struct at_dma_slave	dma_slave;
-};
-
 struct atmel_spi_caps {
 	bool	is_spi2;
 	bool	has_wdrbt;
@@ -295,6 +285,7 @@ struct atmel_spi {
 	int			irq;
 	struct clk		*clk;
 	struct platform_device	*pdev;
+	unsigned long		spi_clk;
 
 	struct spi_transfer	*current_transfer;
 	int			current_remaining_bytes;
@@ -302,17 +293,11 @@ struct atmel_spi {
 
 	struct completion	xfer_completion;
 
-	/* scratch buffer */
-	void			*buffer;
-	dma_addr_t		buffer_dma;
-
 	struct atmel_spi_caps	caps;
 
 	bool			use_dma;
 	bool			use_pdc;
 	bool			use_cs_gpios;
-	/* dmaengine data */
-	struct atmel_spi_dma	dma;
 
 	bool			keep_cs;
 	bool			cs_active;
@@ -326,7 +311,7 @@ struct atmel_spi_device {
 	u32			csr;
 };
 
-#define BUFFER_SIZE		PAGE_SIZE
+#define SPI_MAX_DMA_XFER	65535 /* true for both PDC and DMA */
 #define INVALID_DMA_ADDRESS	0xffffffff
 
 /*
@@ -456,10 +441,20 @@ static inline bool atmel_spi_use_dma(struct atmel_spi *as,
 	return as->use_dma && xfer->len >= DMA_MIN_BYTES;
 }
 
+static bool atmel_spi_can_dma(struct spi_master *master,
+			      struct spi_device *spi,
+			      struct spi_transfer *xfer)
+{
+	struct atmel_spi *as = spi_master_get_devdata(master);
+
+	return atmel_spi_use_dma(as, xfer);
+}
+
 static int atmel_spi_dma_slave_config(struct atmel_spi *as,
 				struct dma_slave_config *slave_config,
 				u8 bits_per_word)
 {
+	struct spi_master *master = platform_get_drvdata(as->pdev);
 	int err = 0;
 
 	if (bits_per_word > 8) {
@@ -491,7 +486,7 @@ static int atmel_spi_dma_slave_config(struct atmel_spi *as,
 	 * path works the same whether FIFOs are available (and enabled) or not.
 	 */
 	slave_config->direction = DMA_MEM_TO_DEV;
-	if (dmaengine_slave_config(as->dma.chan_tx, slave_config)) {
+	if (dmaengine_slave_config(master->dma_tx, slave_config)) {
 		dev_err(&as->pdev->dev,
 			"failed to configure tx dma channel\n");
 		err = -EINVAL;
@@ -506,7 +501,7 @@ static int atmel_spi_dma_slave_config(struct atmel_spi *as,
 	 * enabled) or not.
 	 */
 	slave_config->direction = DMA_DEV_TO_MEM;
-	if (dmaengine_slave_config(as->dma.chan_rx, slave_config)) {
+	if (dmaengine_slave_config(master->dma_rx, slave_config)) {
 		dev_err(&as->pdev->dev,
 			"failed to configure rx dma channel\n");
 		err = -EINVAL;
@@ -515,7 +510,8 @@ static int atmel_spi_dma_slave_config(struct atmel_spi *as,
 	return err;
 }
 
-static int atmel_spi_configure_dma(struct atmel_spi *as)
+static int atmel_spi_configure_dma(struct spi_master *master,
+				   struct atmel_spi *as)
 {
 	struct dma_slave_config	slave_config;
 	struct device *dev = &as->pdev->dev;
@@ -525,26 +521,26 @@ static int atmel_spi_configure_dma(struct atmel_spi *as)
 	dma_cap_zero(mask);
 	dma_cap_set(DMA_SLAVE, mask);
 
-	as->dma.chan_tx = dma_request_slave_channel_reason(dev, "tx");
-	if (IS_ERR(as->dma.chan_tx)) {
-		err = PTR_ERR(as->dma.chan_tx);
+	master->dma_tx = dma_request_slave_channel_reason(dev, "tx");
+	if (IS_ERR(master->dma_tx)) {
+		err = PTR_ERR(master->dma_tx);
 		if (err == -EPROBE_DEFER) {
 			dev_warn(dev, "no DMA channel available at the moment\n");
-			return err;
+			goto error_clear;
 		}
 		dev_err(dev,
 			"DMA TX channel not available, SPI unable to use DMA\n");
 		err = -EBUSY;
-		goto error;
+		goto error_clear;
 	}
 
 	/*
 	 * No reason to check EPROBE_DEFER here since we have already requested
 	 * tx channel. If it fails here, it's for another reason.
 	 */
-	as->dma.chan_rx = dma_request_slave_channel(dev, "rx");
+	master->dma_rx = dma_request_slave_channel(dev, "rx");
 
-	if (!as->dma.chan_rx) {
+	if (!master->dma_rx) {
 		dev_err(dev,
 			"DMA RX channel not available, SPI unable to use DMA\n");
 		err = -EBUSY;
@@ -557,31 +553,38 @@ static int atmel_spi_configure_dma(struct atmel_spi *as)
 
 	dev_info(&as->pdev->dev,
 			"Using %s (tx) and %s (rx) for DMA transfers\n",
-			dma_chan_name(as->dma.chan_tx),
-			dma_chan_name(as->dma.chan_rx));
+			dma_chan_name(master->dma_tx),
+			dma_chan_name(master->dma_rx));
+
 	return 0;
 error:
-	if (as->dma.chan_rx)
-		dma_release_channel(as->dma.chan_rx);
-	if (!IS_ERR(as->dma.chan_tx))
-		dma_release_channel(as->dma.chan_tx);
+	if (master->dma_rx)
+		dma_release_channel(master->dma_rx);
+	if (!IS_ERR(master->dma_tx))
+		dma_release_channel(master->dma_tx);
+error_clear:
+	master->dma_tx = master->dma_rx = NULL;
 	return err;
 }
 
-static void atmel_spi_stop_dma(struct atmel_spi *as)
+static void atmel_spi_stop_dma(struct spi_master *master)
 {
-	if (as->dma.chan_rx)
-		dmaengine_terminate_all(as->dma.chan_rx);
-	if (as->dma.chan_tx)
-		dmaengine_terminate_all(as->dma.chan_tx);
+	if (master->dma_rx)
+		dmaengine_terminate_all(master->dma_rx);
+	if (master->dma_tx)
+		dmaengine_terminate_all(master->dma_tx);
 }
 
-static void atmel_spi_release_dma(struct atmel_spi *as)
+static void atmel_spi_release_dma(struct spi_master *master)
 {
-	if (as->dma.chan_rx)
-		dma_release_channel(as->dma.chan_rx);
-	if (as->dma.chan_tx)
-		dma_release_channel(as->dma.chan_tx);
+	if (master->dma_rx) {
+		dma_release_channel(master->dma_rx);
+		master->dma_rx = NULL;
+	}
+	if (master->dma_tx) {
+		dma_release_channel(master->dma_tx);
+		master->dma_tx = NULL;
+	}
 }
 
 /* This function is called by the DMA driver from tasklet context */
@@ -611,14 +614,10 @@ static void atmel_spi_next_xfer_single(struct spi_master *master,
 		cpu_relax();
 	}
 
-	if (xfer->tx_buf) {
-		if (xfer->bits_per_word > 8)
-			spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos));
-		else
-			spi_writel(as, TDR, *(u8 *)(xfer->tx_buf + xfer_pos));
-	} else {
-		spi_writel(as, TDR, 0);
-	}
+	if (xfer->bits_per_word > 8)
+		spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos));
+	else
+		spi_writel(as, TDR, *(u8 *)(xfer->tx_buf + xfer_pos));
 
 	dev_dbg(master->dev.parent,
 		"  start pio xfer %p: len %u tx %p rx %p bitpw %d\n",
@@ -665,17 +664,12 @@ static void atmel_spi_next_xfer_fifo(struct spi_master *master,
 
 	/* Fill TX FIFO */
 	while (num_data >= 2) {
-		if (xfer->tx_buf) {
-			if (xfer->bits_per_word > 8) {
-				td0 = *words++;
-				td1 = *words++;
-			} else {
-				td0 = *bytes++;
-				td1 = *bytes++;
-			}
+		if (xfer->bits_per_word > 8) {
+			td0 = *words++;
+			td1 = *words++;
 		} else {
-			td0 = 0;
-			td1 = 0;
+			td0 = *bytes++;
+			td1 = *bytes++;
 		}
 
 		spi_writel(as, TDR, (td1 << 16) | td0);
@@ -683,14 +677,10 @@ static void atmel_spi_next_xfer_fifo(struct spi_master *master,
 	}
 
 	if (num_data) {
-		if (xfer->tx_buf) {
-			if (xfer->bits_per_word > 8)
-				td0 = *words++;
-			else
-				td0 = *bytes++;
-		} else {
-			td0 = 0;
-		}
+		if (xfer->bits_per_word > 8)
+			td0 = *words++;
+		else
+			td0 = *bytes++;
 
 		spi_writew(as, TDR, td0);
 		num_data--;
@@ -730,13 +720,12 @@ static int atmel_spi_next_xfer_dma_submit(struct spi_master *master,
 				u32 *plen)
 {
 	struct atmel_spi	*as = spi_master_get_devdata(master);
-	struct dma_chan		*rxchan = as->dma.chan_rx;
-	struct dma_chan		*txchan = as->dma.chan_tx;
+	struct dma_chan		*rxchan = master->dma_rx;
+	struct dma_chan		*txchan = master->dma_tx;
 	struct dma_async_tx_descriptor *rxdesc;
 	struct dma_async_tx_descriptor *txdesc;
 	struct dma_slave_config	slave_config;
 	dma_cookie_t		cookie;
-	u32	len = *plen;
 
 	dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_dma_submit\n");
 
@@ -747,44 +736,22 @@ static int atmel_spi_next_xfer_dma_submit(struct spi_master *master,
 	/* release lock for DMA operations */
 	atmel_spi_unlock(as);
 
-	/* prepare the RX dma transfer */
-	sg_init_table(&as->dma.sgrx, 1);
-	if (xfer->rx_buf) {
-		as->dma.sgrx.dma_address = xfer->rx_dma + xfer->len - *plen;
-	} else {
-		as->dma.sgrx.dma_address = as->buffer_dma;
-		if (len > BUFFER_SIZE)
-			len = BUFFER_SIZE;
-	}
-
-	/* prepare the TX dma transfer */
-	sg_init_table(&as->dma.sgtx, 1);
-	if (xfer->tx_buf) {
-		as->dma.sgtx.dma_address = xfer->tx_dma + xfer->len - *plen;
-	} else {
-		as->dma.sgtx.dma_address = as->buffer_dma;
-		if (len > BUFFER_SIZE)
-			len = BUFFER_SIZE;
-		memset(as->buffer, 0, len);
-	}
-
-	sg_dma_len(&as->dma.sgtx) = len;
-	sg_dma_len(&as->dma.sgrx) = len;
-
-	*plen = len;
+	*plen = xfer->len;
 
 	if (atmel_spi_dma_slave_config(as, &slave_config,
 				       xfer->bits_per_word))
 		goto err_exit;
 
 	/* Send both scatterlists */
-	rxdesc = dmaengine_prep_slave_sg(rxchan, &as->dma.sgrx, 1,
+	rxdesc = dmaengine_prep_slave_sg(rxchan,
+					 xfer->rx_sg.sgl, xfer->rx_sg.nents,
 					 DMA_FROM_DEVICE,
 					 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 	if (!rxdesc)
 		goto err_dma;
 
-	txdesc = dmaengine_prep_slave_sg(txchan, &as->dma.sgtx, 1,
+	txdesc = dmaengine_prep_slave_sg(txchan,
+					 xfer->tx_sg.sgl, xfer->tx_sg.nents,
 					 DMA_TO_DEVICE,
 					 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 	if (!txdesc)
@@ -818,7 +785,7 @@ static int atmel_spi_next_xfer_dma_submit(struct spi_master *master,
 
 err_dma:
 	spi_writel(as, IDR, SPI_BIT(OVRES));
-	atmel_spi_stop_dma(as);
+	atmel_spi_stop_dma(master);
 err_exit:
 	atmel_spi_lock(as);
 	return -ENOMEM;
@@ -830,30 +797,10 @@ static void atmel_spi_next_xfer_data(struct spi_master *master,
 				dma_addr_t *rx_dma,
 				u32 *plen)
 {
-	struct atmel_spi	*as = spi_master_get_devdata(master);
-	u32			len = *plen;
-
-	/* use scratch buffer only when rx or tx data is unspecified */
-	if (xfer->rx_buf)
-		*rx_dma = xfer->rx_dma + xfer->len - *plen;
-	else {
-		*rx_dma = as->buffer_dma;
-		if (len > BUFFER_SIZE)
-			len = BUFFER_SIZE;
-	}
-
-	if (xfer->tx_buf)
-		*tx_dma = xfer->tx_dma + xfer->len - *plen;
-	else {
-		*tx_dma = as->buffer_dma;
-		if (len > BUFFER_SIZE)
-			len = BUFFER_SIZE;
-		memset(as->buffer, 0, len);
-		dma_sync_single_for_device(&as->pdev->dev,
-				as->buffer_dma, len, DMA_TO_DEVICE);
-	}
-
-	*plen = len;
+	*rx_dma = xfer->rx_dma + xfer->len - *plen;
+	*tx_dma = xfer->tx_dma + xfer->len - *plen;
+	if (*plen > master->max_dma_len)
+		*plen = master->max_dma_len;
 }
 
 static int atmel_spi_set_xfer_speed(struct atmel_spi *as,
@@ -864,7 +811,7 @@ static int atmel_spi_set_xfer_speed(struct atmel_spi *as,
 	unsigned long		bus_hz;
 
 	/* v1 chips start out at half the peripheral bus speed. */
-	bus_hz = clk_get_rate(as->clk);
+	bus_hz = as->spi_clk;
 	if (!atmel_spi_is_v2(as))
 		bus_hz /= 2;
 
@@ -1025,16 +972,12 @@ atmel_spi_pump_single_data(struct atmel_spi *as, struct spi_transfer *xfer)
 	u16		*rxp16;
 	unsigned long	xfer_pos = xfer->len - as->current_remaining_bytes;
 
-	if (xfer->rx_buf) {
-		if (xfer->bits_per_word > 8) {
-			rxp16 = (u16 *)(((u8 *)xfer->rx_buf) + xfer_pos);
-			*rxp16 = spi_readl(as, RDR);
-		} else {
-			rxp = ((u8 *)xfer->rx_buf) + xfer_pos;
-			*rxp = spi_readl(as, RDR);
-		}
+	if (xfer->bits_per_word > 8) {
+		rxp16 = (u16 *)(((u8 *)xfer->rx_buf) + xfer_pos);
+		*rxp16 = spi_readl(as, RDR);
 	} else {
-		spi_readl(as, RDR);
+		rxp = ((u8 *)xfer->rx_buf) + xfer_pos;
+		*rxp = spi_readl(as, RDR);
 	}
 	if (xfer->bits_per_word > 8) {
 		if (as->current_remaining_bytes > 2)
@@ -1073,12 +1016,10 @@ atmel_spi_pump_fifo_data(struct atmel_spi *as, struct spi_transfer *xfer)
 	/* Read data */
 	while (num_data) {
 		rd = spi_readl(as, RDR);
-		if (xfer->rx_buf) {
-			if (xfer->bits_per_word > 8)
-				*words++ = rd;
-			else
-				*bytes++ = rd;
-		}
+		if (xfer->bits_per_word > 8)
+			*words++ = rd;
+		else
+			*bytes++ = rd;
 		num_data--;
 	}
 }
@@ -1204,7 +1145,6 @@ static int atmel_spi_setup(struct spi_device *spi)
 	u32			csr;
 	unsigned int		bits = spi->bits_per_word;
 	unsigned int		npcs_pin;
-	int			ret;
 
 	as = spi_master_get_devdata(spi->master);
 
@@ -1247,16 +1187,9 @@ static int atmel_spi_setup(struct spi_device *spi)
 		if (!asd)
 			return -ENOMEM;
 
-		if (as->use_cs_gpios) {
-			ret = gpio_request(npcs_pin, dev_name(&spi->dev));
-			if (ret) {
-				kfree(asd);
-				return ret;
-			}
-
+		if (as->use_cs_gpios)
 			gpio_direction_output(npcs_pin,
 					      !(spi->mode & SPI_CS_HIGH));
-		}
 
 		asd->npcs_pin = npcs_pin;
 		spi->controller_state = asd;
@@ -1307,7 +1240,7 @@ static int atmel_spi_one_transfer(struct spi_master *master,
 	 * better fault reporting.
 	 */
 	if ((!msg->is_dma_mapped)
-		&& (atmel_spi_use_dma(as, xfer)	|| as->use_pdc)) {
+		&& as->use_pdc) {
 		if (atmel_spi_dma_map_xfer(as, xfer) < 0)
 			return -ENOMEM;
 	}
@@ -1380,11 +1313,11 @@ static int atmel_spi_one_transfer(struct spi_master *master,
 			spi_readl(as, SR);
 
 		} else if (atmel_spi_use_dma(as, xfer)) {
-			atmel_spi_stop_dma(as);
+			atmel_spi_stop_dma(master);
 		}
 
 		if (!msg->is_dma_mapped
-			&& (atmel_spi_use_dma(as, xfer) || as->use_pdc))
+			&& as->use_pdc)
 			atmel_spi_dma_unmap_xfer(master, xfer);
 
 		return 0;
@@ -1395,7 +1328,7 @@ static int atmel_spi_one_transfer(struct spi_master *master,
 	}
 
 	if (!msg->is_dma_mapped
-		&& (atmel_spi_use_dma(as, xfer) || as->use_pdc))
+		&& as->use_pdc)
 		atmel_spi_dma_unmap_xfer(master, xfer);
 
 	if (xfer->delay_usecs)
@@ -1471,13 +1404,11 @@ msg_done:
 static void atmel_spi_cleanup(struct spi_device *spi)
 {
 	struct atmel_spi_device	*asd = spi->controller_state;
-	unsigned		gpio = (unsigned long) spi->controller_data;
 
 	if (!asd)
 		return;
 
 	spi->controller_state = NULL;
-	gpio_free(gpio);
 	kfree(asd);
 }
 
@@ -1499,6 +1430,39 @@ static void atmel_get_caps(struct atmel_spi *as)
 }
 
 /*-------------------------------------------------------------------------*/
+static int atmel_spi_gpio_cs(struct platform_device *pdev)
+{
+	struct spi_master	*master = platform_get_drvdata(pdev);
+	struct atmel_spi	*as = spi_master_get_devdata(master);
+	struct device_node	*np = master->dev.of_node;
+	int			i;
+	int			ret = 0;
+	int			nb = 0;
+
+	if (!as->use_cs_gpios)
+		return 0;
+
+	if (!np)
+		return 0;
+
+	nb = of_gpio_named_count(np, "cs-gpios");
+	for (i = 0; i < nb; i++) {
+		int cs_gpio = of_get_named_gpio(pdev->dev.of_node,
+						"cs-gpios", i);
+
+		if (cs_gpio == -EPROBE_DEFER)
+			return cs_gpio;
+
+		if (gpio_is_valid(cs_gpio)) {
+			ret = devm_gpio_request(&pdev->dev, cs_gpio,
+						dev_name(&pdev->dev));
+			if (ret)
+				return ret;
+		}
+	}
+
+	return 0;
+}
 
 static int atmel_spi_probe(struct platform_device *pdev)
 {
@@ -1537,29 +1501,23 @@ static int atmel_spi_probe(struct platform_device *pdev)
 	master->bus_num = pdev->id;
 	master->num_chipselect = master->dev.of_node ? 0 : 4;
 	master->setup = atmel_spi_setup;
+	master->flags = (SPI_MASTER_MUST_RX | SPI_MASTER_MUST_TX);
 	master->transfer_one_message = atmel_spi_transfer_one_message;
 	master->cleanup = atmel_spi_cleanup;
 	master->auto_runtime_pm = true;
+	master->max_dma_len = SPI_MAX_DMA_XFER;
+	master->can_dma = atmel_spi_can_dma;
 	platform_set_drvdata(pdev, master);
 
 	as = spi_master_get_devdata(master);
 
-	/*
-	 * Scratch buffer is used for throwaway rx and tx data.
-	 * It's coherent to minimize dcache pollution.
-	 */
-	as->buffer = dma_alloc_coherent(&pdev->dev, BUFFER_SIZE,
-					&as->buffer_dma, GFP_KERNEL);
-	if (!as->buffer)
-		goto out_free;
-
 	spin_lock_init(&as->lock);
 
 	as->pdev = pdev;
 	as->regs = devm_ioremap_resource(&pdev->dev, regs);
 	if (IS_ERR(as->regs)) {
 		ret = PTR_ERR(as->regs);
-		goto out_free_buffer;
+		goto out_unmap_regs;
 	}
 	as->phybase = regs->start;
 	as->irq = irq;
@@ -1577,14 +1535,19 @@ static int atmel_spi_probe(struct platform_device *pdev)
 		master->num_chipselect = 4;
 	}
 
+	ret = atmel_spi_gpio_cs(pdev);
+	if (ret)
+		goto out_unmap_regs;
+
 	as->use_dma = false;
 	as->use_pdc = false;
 	if (as->caps.has_dma_support) {
-		ret = atmel_spi_configure_dma(as);
-		if (ret == 0)
+		ret = atmel_spi_configure_dma(master, as);
+		if (ret == 0) {
 			as->use_dma = true;
-		else if (ret == -EPROBE_DEFER)
+		} else if (ret == -EPROBE_DEFER) {
 			return ret;
+		}
 	} else {
 		as->use_pdc = true;
 	}
@@ -1606,6 +1569,9 @@ static int atmel_spi_probe(struct platform_device *pdev)
 	ret = clk_prepare_enable(clk);
 	if (ret)
 		goto out_free_irq;
+
+	as->spi_clk = clk_get_rate(clk);
+
 	spi_writel(as, CR, SPI_BIT(SWRST));
 	spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */
 	if (as->caps.has_wdrbt) {
@@ -1626,10 +1592,6 @@ static int atmel_spi_probe(struct platform_device *pdev)
 		spi_writel(as, CR, SPI_BIT(FIFOEN));
 	}
 
-	/* go! */
-	dev_info(&pdev->dev, "Atmel SPI Controller at 0x%08lx (irq %d)\n",
-			(unsigned long)regs->start, irq);
-
 	pm_runtime_set_autosuspend_delay(&pdev->dev, AUTOSUSPEND_TIMEOUT);
 	pm_runtime_use_autosuspend(&pdev->dev);
 	pm_runtime_set_active(&pdev->dev);
@@ -1639,6 +1601,10 @@ static int atmel_spi_probe(struct platform_device *pdev)
 	if (ret)
 		goto out_free_dma;
 
+	/* go! */
+	dev_info(&pdev->dev, "Atmel SPI Controller at 0x%08lx (irq %d)\n",
+			(unsigned long)regs->start, irq);
+
 	return 0;
 
 out_free_dma:
@@ -1646,16 +1612,13 @@ out_free_dma:
 	pm_runtime_set_suspended(&pdev->dev);
 
 	if (as->use_dma)
-		atmel_spi_release_dma(as);
+		atmel_spi_release_dma(master);
 
 	spi_writel(as, CR, SPI_BIT(SWRST));
 	spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */
 	clk_disable_unprepare(clk);
 out_free_irq:
 out_unmap_regs:
-out_free_buffer:
-	dma_free_coherent(&pdev->dev, BUFFER_SIZE, as->buffer,
-			as->buffer_dma);
 out_free:
 	spi_master_put(master);
 	return ret;
@@ -1671,8 +1634,8 @@ static int atmel_spi_remove(struct platform_device *pdev)
 	/* reset the hardware and block queue progress */
 	spin_lock_irq(&as->lock);
 	if (as->use_dma) {
-		atmel_spi_stop_dma(as);
-		atmel_spi_release_dma(as);
+		atmel_spi_stop_dma(master);
+		atmel_spi_release_dma(master);
 	}
 
 	spi_writel(as, CR, SPI_BIT(SWRST));
@@ -1680,9 +1643,6 @@ static int atmel_spi_remove(struct platform_device *pdev)
 	spi_readl(as, SR);
 	spin_unlock_irq(&as->lock);
 
-	dma_free_coherent(&pdev->dev, BUFFER_SIZE, as->buffer,
-			as->buffer_dma);
-
 	clk_disable_unprepare(as->clk);
 
 	pm_runtime_put_noidle(&pdev->dev);
diff --git a/drivers/spi/spi-axi-spi-engine.c b/drivers/spi/spi-axi-spi-engine.c
index 2b1456e5e221..319225d7e761 100644
--- a/drivers/spi/spi-axi-spi-engine.c
+++ b/drivers/spi/spi-axi-spi-engine.c
@@ -574,6 +574,7 @@ static const struct of_device_id spi_engine_match_table[] = {
 	{ .compatible = "adi,axi-spi-engine-1.00.a" },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, spi_engine_match_table);
 
 static struct platform_driver spi_engine_driver = {
 	.probe = spi_engine_probe,
diff --git a/drivers/spi/spi-dw.c b/drivers/spi/spi-dw.c
index 27960e46135d..b715a26a9148 100644
--- a/drivers/spi/spi-dw.c
+++ b/drivers/spi/spi-dw.c
@@ -502,6 +502,7 @@ int dw_spi_add_host(struct device *dev, struct dw_spi *dws)
 	master->handle_err = dw_spi_handle_err;
 	master->max_speed_hz = dws->max_freq;
 	master->dev.of_node = dev->of_node;
+	master->flags = SPI_MASTER_GPIO_SS;
 
 	/* Basic HW init */
 	spi_hw_init(dev, dws);
diff --git a/drivers/spi/spi-fsl-dspi.c b/drivers/spi/spi-fsl-dspi.c
index a67b0ff6a362..14c8e7ce1913 100644
--- a/drivers/spi/spi-fsl-dspi.c
+++ b/drivers/spi/spi-fsl-dspi.c
@@ -15,6 +15,8 @@
 
 #include <linux/clk.h>
 #include <linux/delay.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/errno.h>
 #include <linux/interrupt.h>
@@ -40,6 +42,7 @@
 #define TRAN_STATE_WORD_ODD_NUM	0x04
 
 #define DSPI_FIFO_SIZE			4
+#define DSPI_DMA_BUFSIZE		(DSPI_FIFO_SIZE * 1024)
 
 #define SPI_MCR		0x00
 #define SPI_MCR_MASTER		(1 << 31)
@@ -72,6 +75,11 @@
 #define SPI_SR_TCFQF		0x80000000
 #define SPI_SR_CLEAR		0xdaad0000
 
+#define SPI_RSER_TFFFE		BIT(25)
+#define SPI_RSER_TFFFD		BIT(24)
+#define SPI_RSER_RFDFE		BIT(17)
+#define SPI_RSER_RFDFD		BIT(16)
+
 #define SPI_RSER		0x30
 #define SPI_RSER_EOQFE		0x10000000
 #define SPI_RSER_TCFQE		0x80000000
@@ -109,6 +117,8 @@
 
 #define SPI_TCR_TCNT_MAX	0x10000
 
+#define DMA_COMPLETION_TIMEOUT	msecs_to_jiffies(3000)
+
 struct chip_data {
 	u32 mcr_val;
 	u32 ctar_val;
@@ -118,6 +128,7 @@ struct chip_data {
 enum dspi_trans_mode {
 	DSPI_EOQ_MODE = 0,
 	DSPI_TCFQ_MODE,
+	DSPI_DMA_MODE,
 };
 
 struct fsl_dspi_devtype_data {
@@ -126,7 +137,7 @@ struct fsl_dspi_devtype_data {
 };
 
 static const struct fsl_dspi_devtype_data vf610_data = {
-	.trans_mode = DSPI_EOQ_MODE,
+	.trans_mode = DSPI_DMA_MODE,
 	.max_clock_factor = 2,
 };
 
@@ -140,6 +151,23 @@ static const struct fsl_dspi_devtype_data ls2085a_data = {
 	.max_clock_factor = 8,
 };
 
+struct fsl_dspi_dma {
+	/* Length of transfer in words of DSPI_FIFO_SIZE */
+	u32 curr_xfer_len;
+
+	u32 *tx_dma_buf;
+	struct dma_chan *chan_tx;
+	dma_addr_t tx_dma_phys;
+	struct completion cmd_tx_complete;
+	struct dma_async_tx_descriptor *tx_desc;
+
+	u32 *rx_dma_buf;
+	struct dma_chan *chan_rx;
+	dma_addr_t rx_dma_phys;
+	struct completion cmd_rx_complete;
+	struct dma_async_tx_descriptor *rx_desc;
+};
+
 struct fsl_dspi {
 	struct spi_master	*master;
 	struct platform_device	*pdev;
@@ -166,8 +194,11 @@ struct fsl_dspi {
 	u32			waitflags;
 
 	u32			spi_tcnt;
+	struct fsl_dspi_dma	*dma;
 };
 
+static u32 dspi_data_to_pushr(struct fsl_dspi *dspi, int tx_word);
+
 static inline int is_double_byte_mode(struct fsl_dspi *dspi)
 {
 	unsigned int val;
@@ -177,6 +208,255 @@ static inline int is_double_byte_mode(struct fsl_dspi *dspi)
 	return ((val & SPI_FRAME_BITS_MASK) == SPI_FRAME_BITS(8)) ? 0 : 1;
 }
 
+static void dspi_tx_dma_callback(void *arg)
+{
+	struct fsl_dspi *dspi = arg;
+	struct fsl_dspi_dma *dma = dspi->dma;
+
+	complete(&dma->cmd_tx_complete);
+}
+
+static void dspi_rx_dma_callback(void *arg)
+{
+	struct fsl_dspi *dspi = arg;
+	struct fsl_dspi_dma *dma = dspi->dma;
+	int rx_word;
+	int i;
+	u16 d;
+
+	rx_word = is_double_byte_mode(dspi);
+
+	if (!(dspi->dataflags & TRAN_STATE_RX_VOID)) {
+		for (i = 0; i < dma->curr_xfer_len; i++) {
+			d = dspi->dma->rx_dma_buf[i];
+			rx_word ? (*(u16 *)dspi->rx = d) :
+						(*(u8 *)dspi->rx = d);
+			dspi->rx += rx_word + 1;
+		}
+	}
+
+	complete(&dma->cmd_rx_complete);
+}
+
+static int dspi_next_xfer_dma_submit(struct fsl_dspi *dspi)
+{
+	struct fsl_dspi_dma *dma = dspi->dma;
+	struct device *dev = &dspi->pdev->dev;
+	int time_left;
+	int tx_word;
+	int i;
+
+	tx_word = is_double_byte_mode(dspi);
+
+	for (i = 0; i < dma->curr_xfer_len; i++) {
+		dspi->dma->tx_dma_buf[i] = dspi_data_to_pushr(dspi, tx_word);
+		if ((dspi->cs_change) && (!dspi->len))
+			dspi->dma->tx_dma_buf[i] &= ~SPI_PUSHR_CONT;
+	}
+
+	dma->tx_desc = dmaengine_prep_slave_single(dma->chan_tx,
+					dma->tx_dma_phys,
+					dma->curr_xfer_len *
+					DMA_SLAVE_BUSWIDTH_4_BYTES,
+					DMA_MEM_TO_DEV,
+					DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	if (!dma->tx_desc) {
+		dev_err(dev, "Not able to get desc for DMA xfer\n");
+		return -EIO;
+	}
+
+	dma->tx_desc->callback = dspi_tx_dma_callback;
+	dma->tx_desc->callback_param = dspi;
+	if (dma_submit_error(dmaengine_submit(dma->tx_desc))) {
+		dev_err(dev, "DMA submit failed\n");
+		return -EINVAL;
+	}
+
+	dma->rx_desc = dmaengine_prep_slave_single(dma->chan_rx,
+					dma->rx_dma_phys,
+					dma->curr_xfer_len *
+					DMA_SLAVE_BUSWIDTH_4_BYTES,
+					DMA_DEV_TO_MEM,
+					DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	if (!dma->rx_desc) {
+		dev_err(dev, "Not able to get desc for DMA xfer\n");
+		return -EIO;
+	}
+
+	dma->rx_desc->callback = dspi_rx_dma_callback;
+	dma->rx_desc->callback_param = dspi;
+	if (dma_submit_error(dmaengine_submit(dma->rx_desc))) {
+		dev_err(dev, "DMA submit failed\n");
+		return -EINVAL;
+	}
+
+	reinit_completion(&dspi->dma->cmd_rx_complete);
+	reinit_completion(&dspi->dma->cmd_tx_complete);
+
+	dma_async_issue_pending(dma->chan_rx);
+	dma_async_issue_pending(dma->chan_tx);
+
+	time_left = wait_for_completion_timeout(&dspi->dma->cmd_tx_complete,
+					DMA_COMPLETION_TIMEOUT);
+	if (time_left == 0) {
+		dev_err(dev, "DMA tx timeout\n");
+		dmaengine_terminate_all(dma->chan_tx);
+		dmaengine_terminate_all(dma->chan_rx);
+		return -ETIMEDOUT;
+	}
+
+	time_left = wait_for_completion_timeout(&dspi->dma->cmd_rx_complete,
+					DMA_COMPLETION_TIMEOUT);
+	if (time_left == 0) {
+		dev_err(dev, "DMA rx timeout\n");
+		dmaengine_terminate_all(dma->chan_tx);
+		dmaengine_terminate_all(dma->chan_rx);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int dspi_dma_xfer(struct fsl_dspi *dspi)
+{
+	struct fsl_dspi_dma *dma = dspi->dma;
+	struct device *dev = &dspi->pdev->dev;
+	int curr_remaining_bytes;
+	int bytes_per_buffer;
+	int word = 1;
+	int ret = 0;
+
+	if (is_double_byte_mode(dspi))
+		word = 2;
+	curr_remaining_bytes = dspi->len;
+	bytes_per_buffer = DSPI_DMA_BUFSIZE / DSPI_FIFO_SIZE;
+	while (curr_remaining_bytes) {
+		/* Check if current transfer fits the DMA buffer */
+		dma->curr_xfer_len = curr_remaining_bytes / word;
+		if (dma->curr_xfer_len > bytes_per_buffer)
+			dma->curr_xfer_len = bytes_per_buffer;
+
+		ret = dspi_next_xfer_dma_submit(dspi);
+		if (ret) {
+			dev_err(dev, "DMA transfer failed\n");
+			goto exit;
+
+		} else {
+			curr_remaining_bytes -= dma->curr_xfer_len * word;
+			if (curr_remaining_bytes < 0)
+				curr_remaining_bytes = 0;
+		}
+	}
+
+exit:
+	return ret;
+}
+
+static int dspi_request_dma(struct fsl_dspi *dspi, phys_addr_t phy_addr)
+{
+	struct fsl_dspi_dma *dma;
+	struct dma_slave_config cfg;
+	struct device *dev = &dspi->pdev->dev;
+	int ret;
+
+	dma = devm_kzalloc(dev, sizeof(*dma), GFP_KERNEL);
+	if (!dma)
+		return -ENOMEM;
+
+	dma->chan_rx = dma_request_slave_channel(dev, "rx");
+	if (!dma->chan_rx) {
+		dev_err(dev, "rx dma channel not available\n");
+		ret = -ENODEV;
+		return ret;
+	}
+
+	dma->chan_tx = dma_request_slave_channel(dev, "tx");
+	if (!dma->chan_tx) {
+		dev_err(dev, "tx dma channel not available\n");
+		ret = -ENODEV;
+		goto err_tx_channel;
+	}
+
+	dma->tx_dma_buf = dma_alloc_coherent(dev, DSPI_DMA_BUFSIZE,
+					&dma->tx_dma_phys, GFP_KERNEL);
+	if (!dma->tx_dma_buf) {
+		ret = -ENOMEM;
+		goto err_tx_dma_buf;
+	}
+
+	dma->rx_dma_buf = dma_alloc_coherent(dev, DSPI_DMA_BUFSIZE,
+					&dma->rx_dma_phys, GFP_KERNEL);
+	if (!dma->rx_dma_buf) {
+		ret = -ENOMEM;
+		goto err_rx_dma_buf;
+	}
+
+	cfg.src_addr = phy_addr + SPI_POPR;
+	cfg.dst_addr = phy_addr + SPI_PUSHR;
+	cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	cfg.src_maxburst = 1;
+	cfg.dst_maxburst = 1;
+
+	cfg.direction = DMA_DEV_TO_MEM;
+	ret = dmaengine_slave_config(dma->chan_rx, &cfg);
+	if (ret) {
+		dev_err(dev, "can't configure rx dma channel\n");
+		ret = -EINVAL;
+		goto err_slave_config;
+	}
+
+	cfg.direction = DMA_MEM_TO_DEV;
+	ret = dmaengine_slave_config(dma->chan_tx, &cfg);
+	if (ret) {
+		dev_err(dev, "can't configure tx dma channel\n");
+		ret = -EINVAL;
+		goto err_slave_config;
+	}
+
+	dspi->dma = dma;
+	init_completion(&dma->cmd_tx_complete);
+	init_completion(&dma->cmd_rx_complete);
+
+	return 0;
+
+err_slave_config:
+	dma_free_coherent(dev, DSPI_DMA_BUFSIZE,
+			dma->rx_dma_buf, dma->rx_dma_phys);
+err_rx_dma_buf:
+	dma_free_coherent(dev, DSPI_DMA_BUFSIZE,
+			dma->tx_dma_buf, dma->tx_dma_phys);
+err_tx_dma_buf:
+	dma_release_channel(dma->chan_tx);
+err_tx_channel:
+	dma_release_channel(dma->chan_rx);
+
+	devm_kfree(dev, dma);
+	dspi->dma = NULL;
+
+	return ret;
+}
+
+static void dspi_release_dma(struct fsl_dspi *dspi)
+{
+	struct fsl_dspi_dma *dma = dspi->dma;
+	struct device *dev = &dspi->pdev->dev;
+
+	if (dma) {
+		if (dma->chan_tx) {
+			dma_unmap_single(dev, dma->tx_dma_phys,
+					DSPI_DMA_BUFSIZE, DMA_TO_DEVICE);
+			dma_release_channel(dma->chan_tx);
+		}
+
+		if (dma->chan_rx) {
+			dma_unmap_single(dev, dma->rx_dma_phys,
+					DSPI_DMA_BUFSIZE, DMA_FROM_DEVICE);
+			dma_release_channel(dma->chan_rx);
+		}
+	}
+}
+
 static void hz_to_spi_baud(char *pbr, char *br, int speed_hz,
 		unsigned long clkrate)
 {
@@ -425,6 +705,12 @@ static int dspi_transfer_one_message(struct spi_master *master,
 			regmap_write(dspi->regmap, SPI_RSER, SPI_RSER_TCFQE);
 			dspi_tcfq_write(dspi);
 			break;
+		case DSPI_DMA_MODE:
+			regmap_write(dspi->regmap, SPI_RSER,
+				SPI_RSER_TFFFE | SPI_RSER_TFFFD |
+				SPI_RSER_RFDFE | SPI_RSER_RFDFD);
+			status = dspi_dma_xfer(dspi);
+			break;
 		default:
 			dev_err(&dspi->pdev->dev, "unsupported trans_mode %u\n",
 				trans_mode);
@@ -432,9 +718,13 @@ static int dspi_transfer_one_message(struct spi_master *master,
 			goto out;
 		}
 
-		if (wait_event_interruptible(dspi->waitq, dspi->waitflags))
-			dev_err(&dspi->pdev->dev, "wait transfer complete fail!\n");
-		dspi->waitflags = 0;
+		if (trans_mode != DSPI_DMA_MODE) {
+			if (wait_event_interruptible(dspi->waitq,
+						dspi->waitflags))
+				dev_err(&dspi->pdev->dev,
+					"wait transfer complete fail!\n");
+			dspi->waitflags = 0;
+		}
 
 		if (transfer->delay_usecs)
 			udelay(transfer->delay_usecs);
@@ -740,6 +1030,13 @@ static int dspi_probe(struct platform_device *pdev)
 	if (ret)
 		goto out_master_put;
 
+	if (dspi->devtype_data->trans_mode == DSPI_DMA_MODE) {
+		if (dspi_request_dma(dspi, res->start)) {
+			dev_err(&pdev->dev, "can't get dma channels\n");
+			goto out_clk_put;
+		}
+	}
+
 	master->max_speed_hz =
 		clk_get_rate(dspi->clk) / dspi->devtype_data->max_clock_factor;
 
@@ -768,6 +1065,7 @@ static int dspi_remove(struct platform_device *pdev)
 	struct fsl_dspi *dspi = spi_master_get_devdata(master);
 
 	/* Disconnect from the SPI framework */
+	dspi_release_dma(dspi);
 	clk_disable_unprepare(dspi->clk);
 	spi_unregister_master(dspi->master);
 
diff --git a/drivers/spi/spi-fsl-espi.c b/drivers/spi/spi-fsl-espi.c
index 2c175b9495f7..1d332e23f6ed 100644
--- a/drivers/spi/spi-fsl-espi.c
+++ b/drivers/spi/spi-fsl-espi.c
@@ -23,8 +23,6 @@
 #include <linux/pm_runtime.h>
 #include <sysdev/fsl_soc.h>
 
-#include "spi-fsl-lib.h"
-
 /* eSPI Controller registers */
 #define ESPI_SPMODE	0x00	/* eSPI mode register */
 #define ESPI_SPIE	0x04	/* eSPI event register */
@@ -54,8 +52,11 @@
 #define CSMODE_AFT(x)		((x) << 8)
 #define CSMODE_CG(x)		((x) << 3)
 
+#define FSL_ESPI_FIFO_SIZE	32
+#define FSL_ESPI_RXTHR		15
+
 /* Default mode/csmode for eSPI controller */
-#define SPMODE_INIT_VAL (SPMODE_TXTHR(4) | SPMODE_RXTHR(3))
+#define SPMODE_INIT_VAL (SPMODE_TXTHR(4) | SPMODE_RXTHR(FSL_ESPI_RXTHR))
 #define CSMODE_INIT_VAL (CSMODE_POL_1 | CSMODE_BEF(0) \
 		| CSMODE_AFT(0) | CSMODE_CG(1))
 
@@ -90,219 +91,342 @@
 
 #define AUTOSUSPEND_TIMEOUT 2000
 
-static inline u32 fsl_espi_read_reg(struct mpc8xxx_spi *mspi, int offset)
+struct fsl_espi {
+	struct device *dev;
+	void __iomem *reg_base;
+
+	struct list_head *m_transfers;
+	struct spi_transfer *tx_t;
+	unsigned int tx_pos;
+	bool tx_done;
+	struct spi_transfer *rx_t;
+	unsigned int rx_pos;
+	bool rx_done;
+
+	bool swab;
+	unsigned int rxskip;
+
+	spinlock_t lock;
+
+	u32 spibrg;             /* SPIBRG input clock */
+
+	struct completion done;
+};
+
+struct fsl_espi_cs {
+	u32 hw_mode;
+};
+
+static inline u32 fsl_espi_read_reg(struct fsl_espi *espi, int offset)
 {
-	return ioread32be(mspi->reg_base + offset);
+	return ioread32be(espi->reg_base + offset);
 }
 
-static inline u8 fsl_espi_read_reg8(struct mpc8xxx_spi *mspi, int offset)
+static inline u16 fsl_espi_read_reg16(struct fsl_espi *espi, int offset)
 {
-	return ioread8(mspi->reg_base + offset);
+	return ioread16be(espi->reg_base + offset);
 }
 
-static inline void fsl_espi_write_reg(struct mpc8xxx_spi *mspi, int offset,
-				      u32 val)
+static inline u8 fsl_espi_read_reg8(struct fsl_espi *espi, int offset)
 {
-	iowrite32be(val, mspi->reg_base + offset);
+	return ioread8(espi->reg_base + offset);
 }
 
-static inline void fsl_espi_write_reg8(struct mpc8xxx_spi *mspi, int offset,
-				       u8 val)
+static inline void fsl_espi_write_reg(struct fsl_espi *espi, int offset,
+				      u32 val)
 {
-	iowrite8(val, mspi->reg_base + offset);
+	iowrite32be(val, espi->reg_base + offset);
 }
 
-static void fsl_espi_copy_to_buf(struct spi_message *m,
-				 struct mpc8xxx_spi *mspi)
+static inline void fsl_espi_write_reg16(struct fsl_espi *espi, int offset,
+					u16 val)
 {
-	struct spi_transfer *t;
-	u8 *buf = mspi->local_buf;
-
-	list_for_each_entry(t, &m->transfers, transfer_list) {
-		if (t->tx_buf)
-			memcpy(buf, t->tx_buf, t->len);
-		else
-			memset(buf, 0, t->len);
-		buf += t->len;
-	}
+	iowrite16be(val, espi->reg_base + offset);
 }
 
-static void fsl_espi_copy_from_buf(struct spi_message *m,
-				   struct mpc8xxx_spi *mspi)
+static inline void fsl_espi_write_reg8(struct fsl_espi *espi, int offset,
+				       u8 val)
 {
-	struct spi_transfer *t;
-	u8 *buf = mspi->local_buf;
-
-	list_for_each_entry(t, &m->transfers, transfer_list) {
-		if (t->rx_buf)
-			memcpy(t->rx_buf, buf, t->len);
-		buf += t->len;
-	}
+	iowrite8(val, espi->reg_base + offset);
 }
 
 static int fsl_espi_check_message(struct spi_message *m)
 {
-	struct mpc8xxx_spi *mspi = spi_master_get_devdata(m->spi->master);
+	struct fsl_espi *espi = spi_master_get_devdata(m->spi->master);
 	struct spi_transfer *t, *first;
 
 	if (m->frame_length > SPCOM_TRANLEN_MAX) {
-		dev_err(mspi->dev, "message too long, size is %u bytes\n",
+		dev_err(espi->dev, "message too long, size is %u bytes\n",
 			m->frame_length);
 		return -EMSGSIZE;
 	}
 
 	first = list_first_entry(&m->transfers, struct spi_transfer,
 				 transfer_list);
+
 	list_for_each_entry(t, &m->transfers, transfer_list) {
 		if (first->bits_per_word != t->bits_per_word ||
 		    first->speed_hz != t->speed_hz) {
-			dev_err(mspi->dev, "bits_per_word/speed_hz should be the same for all transfers\n");
+			dev_err(espi->dev, "bits_per_word/speed_hz should be the same for all transfers\n");
 			return -EINVAL;
 		}
 	}
 
+	/* ESPI supports MSB-first transfers for word size 8 / 16 only */
+	if (!(m->spi->mode & SPI_LSB_FIRST) && first->bits_per_word != 8 &&
+	    first->bits_per_word != 16) {
+		dev_err(espi->dev,
+			"MSB-first transfer not supported for wordsize %u\n",
+			first->bits_per_word);
+		return -EINVAL;
+	}
+
 	return 0;
 }
 
-static void fsl_espi_change_mode(struct spi_device *spi)
+static unsigned int fsl_espi_check_rxskip_mode(struct spi_message *m)
 {
-	struct mpc8xxx_spi *mspi = spi_master_get_devdata(spi->master);
-	struct spi_mpc8xxx_cs *cs = spi->controller_state;
-	u32 tmp;
-	unsigned long flags;
-
-	/* Turn off IRQs locally to minimize time that SPI is disabled. */
-	local_irq_save(flags);
-
-	/* Turn off SPI unit prior changing mode */
-	tmp = fsl_espi_read_reg(mspi, ESPI_SPMODE);
-	fsl_espi_write_reg(mspi, ESPI_SPMODE, tmp & ~SPMODE_ENABLE);
-	fsl_espi_write_reg(mspi, ESPI_SPMODEx(spi->chip_select),
-			      cs->hw_mode);
-	fsl_espi_write_reg(mspi, ESPI_SPMODE, tmp);
-
-	local_irq_restore(flags);
+	struct spi_transfer *t;
+	unsigned int i = 0, rxskip = 0;
+
+	/*
+	 * prerequisites for ESPI rxskip mode:
+	 * - message has two transfers
+	 * - first transfer is a write and second is a read
+	 *
+	 * In addition the current low-level transfer mechanism requires
+	 * that the rxskip bytes fit into the TX FIFO. Else the transfer
+	 * would hang because after the first FSL_ESPI_FIFO_SIZE bytes
+	 * the TX FIFO isn't re-filled.
+	 */
+	list_for_each_entry(t, &m->transfers, transfer_list) {
+		if (i == 0) {
+			if (!t->tx_buf || t->rx_buf ||
+			    t->len > FSL_ESPI_FIFO_SIZE)
+				return 0;
+			rxskip = t->len;
+		} else if (i == 1) {
+			if (t->tx_buf || !t->rx_buf)
+				return 0;
+		}
+		i++;
+	}
+
+	return i == 2 ? rxskip : 0;
 }
 
-static u32 fsl_espi_tx_buf_lsb(struct mpc8xxx_spi *mpc8xxx_spi)
+static void fsl_espi_fill_tx_fifo(struct fsl_espi *espi, u32 events)
 {
-	u32 data;
-	u16 data_h;
-	u16 data_l;
-	const u32 *tx = mpc8xxx_spi->tx;
+	u32 tx_fifo_avail;
+	unsigned int tx_left;
+	const void *tx_buf;
+
+	/* if events is zero transfer has not started and tx fifo is empty */
+	tx_fifo_avail = events ? SPIE_TXCNT(events) :  FSL_ESPI_FIFO_SIZE;
+start:
+	tx_left = espi->tx_t->len - espi->tx_pos;
+	tx_buf = espi->tx_t->tx_buf;
+	while (tx_fifo_avail >= min(4U, tx_left) && tx_left) {
+		if (tx_left >= 4) {
+			if (!tx_buf)
+				fsl_espi_write_reg(espi, ESPI_SPITF, 0);
+			else if (espi->swab)
+				fsl_espi_write_reg(espi, ESPI_SPITF,
+					swahb32p(tx_buf + espi->tx_pos));
+			else
+				fsl_espi_write_reg(espi, ESPI_SPITF,
+					*(u32 *)(tx_buf + espi->tx_pos));
+			espi->tx_pos += 4;
+			tx_left -= 4;
+			tx_fifo_avail -= 4;
+		} else if (tx_left >= 2 && tx_buf && espi->swab) {
+			fsl_espi_write_reg16(espi, ESPI_SPITF,
+					swab16p(tx_buf + espi->tx_pos));
+			espi->tx_pos += 2;
+			tx_left -= 2;
+			tx_fifo_avail -= 2;
+		} else {
+			if (!tx_buf)
+				fsl_espi_write_reg8(espi, ESPI_SPITF, 0);
+			else
+				fsl_espi_write_reg8(espi, ESPI_SPITF,
+					*(u8 *)(tx_buf + espi->tx_pos));
+			espi->tx_pos += 1;
+			tx_left -= 1;
+			tx_fifo_avail -= 1;
+		}
+	}
 
-	if (!tx)
-		return 0;
+	if (!tx_left) {
+		/* Last transfer finished, in rxskip mode only one is needed */
+		if (list_is_last(&espi->tx_t->transfer_list,
+		    espi->m_transfers) || espi->rxskip) {
+			espi->tx_done = true;
+			return;
+		}
+		espi->tx_t = list_next_entry(espi->tx_t, transfer_list);
+		espi->tx_pos = 0;
+		/* continue with next transfer if tx fifo is not full */
+		if (tx_fifo_avail)
+			goto start;
+	}
+}
 
-	data = *tx++ << mpc8xxx_spi->tx_shift;
-	data_l = data & 0xffff;
-	data_h = (data >> 16) & 0xffff;
-	swab16s(&data_l);
-	swab16s(&data_h);
-	data = data_h | data_l;
+static void fsl_espi_read_rx_fifo(struct fsl_espi *espi, u32 events)
+{
+	u32 rx_fifo_avail = SPIE_RXCNT(events);
+	unsigned int rx_left;
+	void *rx_buf;
+
+start:
+	rx_left = espi->rx_t->len - espi->rx_pos;
+	rx_buf = espi->rx_t->rx_buf;
+	while (rx_fifo_avail >= min(4U, rx_left) && rx_left) {
+		if (rx_left >= 4) {
+			u32 val = fsl_espi_read_reg(espi, ESPI_SPIRF);
+
+			if (rx_buf && espi->swab)
+				*(u32 *)(rx_buf + espi->rx_pos) = swahb32(val);
+			else if (rx_buf)
+				*(u32 *)(rx_buf + espi->rx_pos) = val;
+			espi->rx_pos += 4;
+			rx_left -= 4;
+			rx_fifo_avail -= 4;
+		} else if (rx_left >= 2 && rx_buf && espi->swab) {
+			u16 val = fsl_espi_read_reg16(espi, ESPI_SPIRF);
+
+			*(u16 *)(rx_buf + espi->rx_pos) = swab16(val);
+			espi->rx_pos += 2;
+			rx_left -= 2;
+			rx_fifo_avail -= 2;
+		} else {
+			u8 val = fsl_espi_read_reg8(espi, ESPI_SPIRF);
+
+			if (rx_buf)
+				*(u8 *)(rx_buf + espi->rx_pos) = val;
+			espi->rx_pos += 1;
+			rx_left -= 1;
+			rx_fifo_avail -= 1;
+		}
+	}
 
-	mpc8xxx_spi->tx = tx;
-	return data;
+	if (!rx_left) {
+		if (list_is_last(&espi->rx_t->transfer_list,
+		    espi->m_transfers)) {
+			espi->rx_done = true;
+			return;
+		}
+		espi->rx_t = list_next_entry(espi->rx_t, transfer_list);
+		espi->rx_pos = 0;
+		/* continue with next transfer if rx fifo is not empty */
+		if (rx_fifo_avail)
+			goto start;
+	}
 }
 
 static void fsl_espi_setup_transfer(struct spi_device *spi,
 					struct spi_transfer *t)
 {
-	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(spi->master);
+	struct fsl_espi *espi = spi_master_get_devdata(spi->master);
 	int bits_per_word = t ? t->bits_per_word : spi->bits_per_word;
-	u32 hz = t ? t->speed_hz : spi->max_speed_hz;
-	u8 pm;
-	struct spi_mpc8xxx_cs *cs = spi->controller_state;
-
-	cs->rx_shift = 0;
-	cs->tx_shift = 0;
-	cs->get_rx = mpc8xxx_spi_rx_buf_u32;
-	cs->get_tx = mpc8xxx_spi_tx_buf_u32;
-	if (bits_per_word <= 8) {
-		cs->rx_shift = 8 - bits_per_word;
-	} else {
-		cs->rx_shift = 16 - bits_per_word;
-		if (spi->mode & SPI_LSB_FIRST)
-			cs->get_tx = fsl_espi_tx_buf_lsb;
-	}
-
-	mpc8xxx_spi->rx_shift = cs->rx_shift;
-	mpc8xxx_spi->tx_shift = cs->tx_shift;
-	mpc8xxx_spi->get_rx = cs->get_rx;
-	mpc8xxx_spi->get_tx = cs->get_tx;
+	u32 pm, hz = t ? t->speed_hz : spi->max_speed_hz;
+	struct fsl_espi_cs *cs = spi_get_ctldata(spi);
+	u32 hw_mode_old = cs->hw_mode;
 
 	/* mask out bits we are going to set */
 	cs->hw_mode &= ~(CSMODE_LEN(0xF) | CSMODE_DIV16 | CSMODE_PM(0xF));
 
 	cs->hw_mode |= CSMODE_LEN(bits_per_word - 1);
 
-	if ((mpc8xxx_spi->spibrg / hz) > 64) {
+	pm = DIV_ROUND_UP(espi->spibrg, hz * 4) - 1;
+
+	if (pm > 15) {
 		cs->hw_mode |= CSMODE_DIV16;
-		pm = DIV_ROUND_UP(mpc8xxx_spi->spibrg, hz * 16 * 4);
-
-		WARN_ONCE(pm > 33, "%s: Requested speed is too low: %d Hz. "
-			  "Will use %d Hz instead.\n", dev_name(&spi->dev),
-				hz, mpc8xxx_spi->spibrg / (4 * 16 * (32 + 1)));
-		if (pm > 33)
-			pm = 33;
-	} else {
-		pm = DIV_ROUND_UP(mpc8xxx_spi->spibrg, hz * 4);
+		pm = DIV_ROUND_UP(espi->spibrg, hz * 16 * 4) - 1;
 	}
-	if (pm)
-		pm--;
-	if (pm < 2)
-		pm = 2;
 
 	cs->hw_mode |= CSMODE_PM(pm);
 
-	fsl_espi_change_mode(spi);
+	/* don't write the mode register if the mode doesn't change */
+	if (cs->hw_mode != hw_mode_old)
+		fsl_espi_write_reg(espi, ESPI_SPMODEx(spi->chip_select),
+				   cs->hw_mode);
 }
 
 static int fsl_espi_bufs(struct spi_device *spi, struct spi_transfer *t)
 {
-	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(spi->master);
-	u32 word;
+	struct fsl_espi *espi = spi_master_get_devdata(spi->master);
+	unsigned int rx_len = t->len;
+	u32 mask, spcom;
 	int ret;
 
-	mpc8xxx_spi->len = t->len;
-	mpc8xxx_spi->count = roundup(t->len, 4) / 4;
-
-	mpc8xxx_spi->tx = t->tx_buf;
-	mpc8xxx_spi->rx = t->rx_buf;
-
-	reinit_completion(&mpc8xxx_spi->done);
+	reinit_completion(&espi->done);
 
 	/* Set SPCOM[CS] and SPCOM[TRANLEN] field */
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPCOM,
-		(SPCOM_CS(spi->chip_select) | SPCOM_TRANLEN(t->len - 1)));
+	spcom = SPCOM_CS(spi->chip_select);
+	spcom |= SPCOM_TRANLEN(t->len - 1);
+
+	/* configure RXSKIP mode */
+	if (espi->rxskip) {
+		spcom |= SPCOM_RXSKIP(espi->rxskip);
+		rx_len = t->len - espi->rxskip;
+		if (t->rx_nbits == SPI_NBITS_DUAL)
+			spcom |= SPCOM_DO;
+	}
+
+	fsl_espi_write_reg(espi, ESPI_SPCOM, spcom);
 
-	/* enable rx ints */
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIM, SPIM_RNE);
+	/* enable interrupts */
+	mask = SPIM_DON;
+	if (rx_len > FSL_ESPI_FIFO_SIZE)
+		mask |= SPIM_RXT;
+	fsl_espi_write_reg(espi, ESPI_SPIM, mask);
 
-	/* transmit word */
-	word = mpc8xxx_spi->get_tx(mpc8xxx_spi);
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPITF, word);
+	/* Prevent filling the fifo from getting interrupted */
+	spin_lock_irq(&espi->lock);
+	fsl_espi_fill_tx_fifo(espi, 0);
+	spin_unlock_irq(&espi->lock);
 
 	/* Won't hang up forever, SPI bus sometimes got lost interrupts... */
-	ret = wait_for_completion_timeout(&mpc8xxx_spi->done, 2 * HZ);
+	ret = wait_for_completion_timeout(&espi->done, 2 * HZ);
 	if (ret == 0)
-		dev_err(mpc8xxx_spi->dev,
-			"Transaction hanging up (left %d bytes)\n",
-			mpc8xxx_spi->count);
+		dev_err(espi->dev, "Transfer timed out!\n");
 
 	/* disable rx ints */
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIM, 0);
+	fsl_espi_write_reg(espi, ESPI_SPIM, 0);
 
-	return mpc8xxx_spi->count > 0 ? -EMSGSIZE : 0;
+	return ret == 0 ? -ETIMEDOUT : 0;
 }
 
 static int fsl_espi_trans(struct spi_message *m, struct spi_transfer *trans)
 {
-	struct mpc8xxx_spi *mspi = spi_master_get_devdata(m->spi->master);
+	struct fsl_espi *espi = spi_master_get_devdata(m->spi->master);
 	struct spi_device *spi = m->spi;
 	int ret;
 
-	fsl_espi_copy_to_buf(m, mspi);
+	/* In case of LSB-first and bits_per_word > 8 byte-swap all words */
+	espi->swab = spi->mode & SPI_LSB_FIRST && trans->bits_per_word > 8;
+
+	espi->m_transfers = &m->transfers;
+	espi->tx_t = list_first_entry(&m->transfers, struct spi_transfer,
+				      transfer_list);
+	espi->tx_pos = 0;
+	espi->tx_done = false;
+	espi->rx_t = list_first_entry(&m->transfers, struct spi_transfer,
+				      transfer_list);
+	espi->rx_pos = 0;
+	espi->rx_done = false;
+
+	espi->rxskip = fsl_espi_check_rxskip_mode(m);
+	if (trans->rx_nbits == SPI_NBITS_DUAL && !espi->rxskip) {
+		dev_err(espi->dev, "Dual output mode requires RXSKIP mode!\n");
+		return -EINVAL;
+	}
+
+	/* In RXSKIP mode skip first transfer for reads */
+	if (espi->rxskip)
+		espi->rx_t = list_next_entry(espi->rx_t, transfer_list);
+
 	fsl_espi_setup_transfer(spi, trans);
 
 	ret = fsl_espi_bufs(spi, trans);
@@ -310,19 +434,13 @@ static int fsl_espi_trans(struct spi_message *m, struct spi_transfer *trans)
 	if (trans->delay_usecs)
 		udelay(trans->delay_usecs);
 
-	fsl_espi_setup_transfer(spi, NULL);
-
-	if (!ret)
-		fsl_espi_copy_from_buf(m, mspi);
-
 	return ret;
 }
 
 static int fsl_espi_do_one_msg(struct spi_master *master,
 			       struct spi_message *m)
 {
-	struct mpc8xxx_spi *mspi = spi_master_get_devdata(m->spi->master);
-	unsigned int delay_usecs = 0;
+	unsigned int delay_usecs = 0, rx_nbits = 0;
 	struct spi_transfer *t, trans = {};
 	int ret;
 
@@ -333,6 +451,8 @@ static int fsl_espi_do_one_msg(struct spi_master *master,
 	list_for_each_entry(t, &m->transfers, transfer_list) {
 		if (t->delay_usecs > delay_usecs)
 			delay_usecs = t->delay_usecs;
+		if (t->rx_nbits > rx_nbits)
+			rx_nbits = t->rx_nbits;
 	}
 
 	t = list_first_entry(&m->transfers, struct spi_transfer,
@@ -342,8 +462,7 @@ static int fsl_espi_do_one_msg(struct spi_master *master,
 	trans.speed_hz = t->speed_hz;
 	trans.bits_per_word = t->bits_per_word;
 	trans.delay_usecs = delay_usecs;
-	trans.tx_buf = mspi->local_buf;
-	trans.rx_buf = mspi->local_buf;
+	trans.rx_nbits = rx_nbits;
 
 	if (trans.len)
 		ret = fsl_espi_trans(m, &trans);
@@ -360,12 +479,9 @@ out:
 
 static int fsl_espi_setup(struct spi_device *spi)
 {
-	struct mpc8xxx_spi *mpc8xxx_spi;
+	struct fsl_espi *espi;
 	u32 loop_mode;
-	struct spi_mpc8xxx_cs *cs = spi_get_ctldata(spi);
-
-	if (!spi->max_speed_hz)
-		return -EINVAL;
+	struct fsl_espi_cs *cs = spi_get_ctldata(spi);
 
 	if (!cs) {
 		cs = kzalloc(sizeof(*cs), GFP_KERNEL);
@@ -374,12 +490,11 @@ static int fsl_espi_setup(struct spi_device *spi)
 		spi_set_ctldata(spi, cs);
 	}
 
-	mpc8xxx_spi = spi_master_get_devdata(spi->master);
+	espi = spi_master_get_devdata(spi->master);
 
-	pm_runtime_get_sync(mpc8xxx_spi->dev);
+	pm_runtime_get_sync(espi->dev);
 
-	cs->hw_mode = fsl_espi_read_reg(mpc8xxx_spi,
-					   ESPI_SPMODEx(spi->chip_select));
+	cs->hw_mode = fsl_espi_read_reg(espi, ESPI_SPMODEx(spi->chip_select));
 	/* mask out bits we are going to set */
 	cs->hw_mode &= ~(CSMODE_CP_BEGIN_EDGECLK | CSMODE_CI_INACTIVEHIGH
 			 | CSMODE_REV);
@@ -392,115 +507,74 @@ static int fsl_espi_setup(struct spi_device *spi)
 		cs->hw_mode |= CSMODE_REV;
 
 	/* Handle the loop mode */
-	loop_mode = fsl_espi_read_reg(mpc8xxx_spi, ESPI_SPMODE);
+	loop_mode = fsl_espi_read_reg(espi, ESPI_SPMODE);
 	loop_mode &= ~SPMODE_LOOP;
 	if (spi->mode & SPI_LOOP)
 		loop_mode |= SPMODE_LOOP;
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, loop_mode);
+	fsl_espi_write_reg(espi, ESPI_SPMODE, loop_mode);
 
 	fsl_espi_setup_transfer(spi, NULL);
 
-	pm_runtime_mark_last_busy(mpc8xxx_spi->dev);
-	pm_runtime_put_autosuspend(mpc8xxx_spi->dev);
+	pm_runtime_mark_last_busy(espi->dev);
+	pm_runtime_put_autosuspend(espi->dev);
 
 	return 0;
 }
 
 static void fsl_espi_cleanup(struct spi_device *spi)
 {
-	struct spi_mpc8xxx_cs *cs = spi_get_ctldata(spi);
+	struct fsl_espi_cs *cs = spi_get_ctldata(spi);
 
 	kfree(cs);
 	spi_set_ctldata(spi, NULL);
 }
 
-static void fsl_espi_cpu_irq(struct mpc8xxx_spi *mspi, u32 events)
+static void fsl_espi_cpu_irq(struct fsl_espi *espi, u32 events)
 {
-	/* We need handle RX first */
-	if (events & SPIE_RNE) {
-		u32 rx_data, tmp;
-		u8 rx_data_8;
-		int rx_nr_bytes = 4;
-		int ret;
-
-		/* Spin until RX is done */
-		if (SPIE_RXCNT(events) < min(4, mspi->len)) {
-			ret = spin_event_timeout(
-				!(SPIE_RXCNT(events =
-				fsl_espi_read_reg(mspi, ESPI_SPIE)) <
-						min(4, mspi->len)),
-						10000, 0); /* 10 msec */
-			if (!ret)
-				dev_err(mspi->dev,
-					 "tired waiting for SPIE_RXCNT\n");
-		}
+	if (!espi->rx_done)
+		fsl_espi_read_rx_fifo(espi, events);
 
-		if (mspi->len >= 4) {
-			rx_data = fsl_espi_read_reg(mspi, ESPI_SPIRF);
-		} else if (mspi->len <= 0) {
-			dev_err(mspi->dev,
-				"unexpected RX(SPIE_RNE) interrupt occurred,\n"
-				"(local rxlen %d bytes, reg rxlen %d bytes)\n",
-				min(4, mspi->len), SPIE_RXCNT(events));
-			rx_nr_bytes = 0;
-		} else {
-			rx_nr_bytes = mspi->len;
-			tmp = mspi->len;
-			rx_data = 0;
-			while (tmp--) {
-				rx_data_8 = fsl_espi_read_reg8(mspi,
-							       ESPI_SPIRF);
-				rx_data |= (rx_data_8 << (tmp * 8));
-			}
-
-			rx_data <<= (4 - mspi->len) * 8;
-		}
+	if (!espi->tx_done)
+		fsl_espi_fill_tx_fifo(espi, events);
 
-		mspi->len -= rx_nr_bytes;
+	if (!espi->tx_done || !espi->rx_done)
+		return;
 
-		if (rx_nr_bytes && mspi->rx)
-			mspi->get_rx(rx_data, mspi);
-	}
+	/* we're done, but check for errors before returning */
+	events = fsl_espi_read_reg(espi, ESPI_SPIE);
 
-	if (!(events & SPIE_TNF)) {
-		int ret;
-
-		/* spin until TX is done */
-		ret = spin_event_timeout(((events = fsl_espi_read_reg(
-				mspi, ESPI_SPIE)) & SPIE_TNF), 1000, 0);
-		if (!ret) {
-			dev_err(mspi->dev, "tired waiting for SPIE_TNF\n");
-			complete(&mspi->done);
-			return;
-		}
-	}
+	if (!(events & SPIE_DON))
+		dev_err(espi->dev,
+			"Transfer done but SPIE_DON isn't set!\n");
 
-	mspi->count -= 1;
-	if (mspi->count) {
-		u32 word = mspi->get_tx(mspi);
+	if (SPIE_RXCNT(events) || SPIE_TXCNT(events) != FSL_ESPI_FIFO_SIZE)
+		dev_err(espi->dev, "Transfer done but rx/tx fifo's aren't empty!\n");
 
-		fsl_espi_write_reg(mspi, ESPI_SPITF, word);
-	} else {
-		complete(&mspi->done);
-	}
+	complete(&espi->done);
 }
 
 static irqreturn_t fsl_espi_irq(s32 irq, void *context_data)
 {
-	struct mpc8xxx_spi *mspi = context_data;
+	struct fsl_espi *espi = context_data;
 	u32 events;
 
+	spin_lock(&espi->lock);
+
 	/* Get interrupt events(tx/rx) */
-	events = fsl_espi_read_reg(mspi, ESPI_SPIE);
-	if (!events)
+	events = fsl_espi_read_reg(espi, ESPI_SPIE);
+	if (!events) {
+		spin_unlock(&espi->lock);
 		return IRQ_NONE;
+	}
 
-	dev_vdbg(mspi->dev, "%s: events %x\n", __func__, events);
+	dev_vdbg(espi->dev, "%s: events %x\n", __func__, events);
 
-	fsl_espi_cpu_irq(mspi, events);
+	fsl_espi_cpu_irq(espi, events);
 
 	/* Clear the events */
-	fsl_espi_write_reg(mspi, ESPI_SPIE, events);
+	fsl_espi_write_reg(espi, ESPI_SPIE, events);
+
+	spin_unlock(&espi->lock);
 
 	return IRQ_HANDLED;
 }
@@ -509,12 +583,12 @@ static irqreturn_t fsl_espi_irq(s32 irq, void *context_data)
 static int fsl_espi_runtime_suspend(struct device *dev)
 {
 	struct spi_master *master = dev_get_drvdata(dev);
-	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(master);
+	struct fsl_espi *espi = spi_master_get_devdata(master);
 	u32 regval;
 
-	regval = fsl_espi_read_reg(mpc8xxx_spi, ESPI_SPMODE);
+	regval = fsl_espi_read_reg(espi, ESPI_SPMODE);
 	regval &= ~SPMODE_ENABLE;
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, regval);
+	fsl_espi_write_reg(espi, ESPI_SPMODE, regval);
 
 	return 0;
 }
@@ -522,12 +596,12 @@ static int fsl_espi_runtime_suspend(struct device *dev)
 static int fsl_espi_runtime_resume(struct device *dev)
 {
 	struct spi_master *master = dev_get_drvdata(dev);
-	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(master);
+	struct fsl_espi *espi = spi_master_get_devdata(master);
 	u32 regval;
 
-	regval = fsl_espi_read_reg(mpc8xxx_spi, ESPI_SPMODE);
+	regval = fsl_espi_read_reg(espi, ESPI_SPMODE);
 	regval |= SPMODE_ENABLE;
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, regval);
+	fsl_espi_write_reg(espi, ESPI_SPMODE, regval);
 
 	return 0;
 }
@@ -538,96 +612,105 @@ static size_t fsl_espi_max_message_size(struct spi_device *spi)
 	return SPCOM_TRANLEN_MAX;
 }
 
+static void fsl_espi_init_regs(struct device *dev, bool initial)
+{
+	struct spi_master *master = dev_get_drvdata(dev);
+	struct fsl_espi *espi = spi_master_get_devdata(master);
+	struct device_node *nc;
+	u32 csmode, cs, prop;
+	int ret;
+
+	/* SPI controller initializations */
+	fsl_espi_write_reg(espi, ESPI_SPMODE, 0);
+	fsl_espi_write_reg(espi, ESPI_SPIM, 0);
+	fsl_espi_write_reg(espi, ESPI_SPCOM, 0);
+	fsl_espi_write_reg(espi, ESPI_SPIE, 0xffffffff);
+
+	/* Init eSPI CS mode register */
+	for_each_available_child_of_node(master->dev.of_node, nc) {
+		/* get chip select */
+		ret = of_property_read_u32(nc, "reg", &cs);
+		if (ret || cs >= master->num_chipselect)
+			continue;
+
+		csmode = CSMODE_INIT_VAL;
+
+		/* check if CSBEF is set in device tree */
+		ret = of_property_read_u32(nc, "fsl,csbef", &prop);
+		if (!ret) {
+			csmode &= ~(CSMODE_BEF(0xf));
+			csmode |= CSMODE_BEF(prop);
+		}
+
+		/* check if CSAFT is set in device tree */
+		ret = of_property_read_u32(nc, "fsl,csaft", &prop);
+		if (!ret) {
+			csmode &= ~(CSMODE_AFT(0xf));
+			csmode |= CSMODE_AFT(prop);
+		}
+
+		fsl_espi_write_reg(espi, ESPI_SPMODEx(cs), csmode);
+
+		if (initial)
+			dev_info(dev, "cs=%u, init_csmode=0x%x\n", cs, csmode);
+	}
+
+	/* Enable SPI interface */
+	fsl_espi_write_reg(espi, ESPI_SPMODE, SPMODE_INIT_VAL | SPMODE_ENABLE);
+}
+
 static int fsl_espi_probe(struct device *dev, struct resource *mem,
-			  unsigned int irq)
+			  unsigned int irq, unsigned int num_cs)
 {
-	struct fsl_spi_platform_data *pdata = dev_get_platdata(dev);
 	struct spi_master *master;
-	struct mpc8xxx_spi *mpc8xxx_spi;
-	struct device_node *nc;
-	const __be32 *prop;
-	u32 regval, csmode;
-	int i, len, ret;
+	struct fsl_espi *espi;
+	int ret;
 
-	master = spi_alloc_master(dev, sizeof(struct mpc8xxx_spi));
+	master = spi_alloc_master(dev, sizeof(struct fsl_espi));
 	if (!master)
 		return -ENOMEM;
 
 	dev_set_drvdata(dev, master);
 
-	mpc8xxx_spi_probe(dev, mem, irq);
-
+	master->mode_bits = SPI_RX_DUAL | SPI_CPOL | SPI_CPHA | SPI_CS_HIGH |
+			    SPI_LSB_FIRST | SPI_LOOP;
+	master->dev.of_node = dev->of_node;
 	master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 16);
 	master->setup = fsl_espi_setup;
 	master->cleanup = fsl_espi_cleanup;
 	master->transfer_one_message = fsl_espi_do_one_msg;
 	master->auto_runtime_pm = true;
 	master->max_message_size = fsl_espi_max_message_size;
+	master->num_chipselect = num_cs;
 
-	mpc8xxx_spi = spi_master_get_devdata(master);
+	espi = spi_master_get_devdata(master);
+	spin_lock_init(&espi->lock);
 
-	mpc8xxx_spi->local_buf =
-		devm_kmalloc(dev, SPCOM_TRANLEN_MAX, GFP_KERNEL);
-	if (!mpc8xxx_spi->local_buf) {
-		ret = -ENOMEM;
+	espi->dev = dev;
+	espi->spibrg = fsl_get_sys_freq();
+	if (espi->spibrg == -1) {
+		dev_err(dev, "Can't get sys frequency!\n");
+		ret = -EINVAL;
 		goto err_probe;
 	}
+	/* determined by clock divider fields DIV16/PM in register SPMODEx */
+	master->min_speed_hz = DIV_ROUND_UP(espi->spibrg, 4 * 16 * 16);
+	master->max_speed_hz = DIV_ROUND_UP(espi->spibrg, 4);
 
-	mpc8xxx_spi->reg_base = devm_ioremap_resource(dev, mem);
-	if (IS_ERR(mpc8xxx_spi->reg_base)) {
-		ret = PTR_ERR(mpc8xxx_spi->reg_base);
+	init_completion(&espi->done);
+
+	espi->reg_base = devm_ioremap_resource(dev, mem);
+	if (IS_ERR(espi->reg_base)) {
+		ret = PTR_ERR(espi->reg_base);
 		goto err_probe;
 	}
 
 	/* Register for SPI Interrupt */
-	ret = devm_request_irq(dev, mpc8xxx_spi->irq, fsl_espi_irq,
-			  0, "fsl_espi", mpc8xxx_spi);
+	ret = devm_request_irq(dev, irq, fsl_espi_irq, 0, "fsl_espi", espi);
 	if (ret)
 		goto err_probe;
 
-	if (mpc8xxx_spi->flags & SPI_QE_CPU_MODE) {
-		mpc8xxx_spi->rx_shift = 16;
-		mpc8xxx_spi->tx_shift = 24;
-	}
-
-	/* SPI controller initializations */
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, 0);
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIM, 0);
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPCOM, 0);
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIE, 0xffffffff);
-
-	/* Init eSPI CS mode register */
-	for_each_available_child_of_node(master->dev.of_node, nc) {
-		/* get chip select */
-		prop = of_get_property(nc, "reg", &len);
-		if (!prop || len < sizeof(*prop))
-			continue;
-		i = be32_to_cpup(prop);
-		if (i < 0 || i >= pdata->max_chipselect)
-			continue;
-
-		csmode = CSMODE_INIT_VAL;
-		/* check if CSBEF is set in device tree */
-		prop = of_get_property(nc, "fsl,csbef", &len);
-		if (prop && len >= sizeof(*prop)) {
-			csmode &= ~(CSMODE_BEF(0xf));
-			csmode |= CSMODE_BEF(be32_to_cpup(prop));
-		}
-		/* check if CSAFT is set in device tree */
-		prop = of_get_property(nc, "fsl,csaft", &len);
-		if (prop && len >= sizeof(*prop)) {
-			csmode &= ~(CSMODE_AFT(0xf));
-			csmode |= CSMODE_AFT(be32_to_cpup(prop));
-		}
-		fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODEx(i), csmode);
-
-		dev_info(dev, "cs=%d, init_csmode=0x%x\n", i, csmode);
-	}
-
-	/* Enable SPI interface */
-	regval = pdata->initial_spmode | SPMODE_INIT_VAL | SPMODE_ENABLE;
-
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, regval);
+	fsl_espi_init_regs(dev, true);
 
 	pm_runtime_set_autosuspend_delay(dev, AUTOSUSPEND_TIMEOUT);
 	pm_runtime_use_autosuspend(dev);
@@ -639,8 +722,7 @@ static int fsl_espi_probe(struct device *dev, struct resource *mem,
 	if (ret < 0)
 		goto err_pm;
 
-	dev_info(dev, "at 0x%p (irq = %d)\n", mpc8xxx_spi->reg_base,
-		 mpc8xxx_spi->irq);
+	dev_info(dev, "at 0x%p (irq = %u)\n", espi->reg_base, irq);
 
 	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_autosuspend(dev);
@@ -659,20 +741,16 @@ err_probe:
 static int of_fsl_espi_get_chipselects(struct device *dev)
 {
 	struct device_node *np = dev->of_node;
-	struct fsl_spi_platform_data *pdata = dev_get_platdata(dev);
-	const u32 *prop;
-	int len;
+	u32 num_cs;
+	int ret;
 
-	prop = of_get_property(np, "fsl,espi-num-chipselects", &len);
-	if (!prop || len < sizeof(*prop)) {
+	ret = of_property_read_u32(np, "fsl,espi-num-chipselects", &num_cs);
+	if (ret) {
 		dev_err(dev, "No 'fsl,espi-num-chipselects' property\n");
-		return -EINVAL;
+		return 0;
 	}
 
-	pdata->max_chipselect = *prop;
-	pdata->cs_control = NULL;
-
-	return 0;
+	return num_cs;
 }
 
 static int of_fsl_espi_probe(struct platform_device *ofdev)
@@ -680,16 +758,17 @@ static int of_fsl_espi_probe(struct platform_device *ofdev)
 	struct device *dev = &ofdev->dev;
 	struct device_node *np = ofdev->dev.of_node;
 	struct resource mem;
-	unsigned int irq;
+	unsigned int irq, num_cs;
 	int ret;
 
-	ret = of_mpc8xxx_spi_probe(ofdev);
-	if (ret)
-		return ret;
+	if (of_property_read_bool(np, "mode")) {
+		dev_err(dev, "mode property is not supported on ESPI!\n");
+		return -EINVAL;
+	}
 
-	ret = of_fsl_espi_get_chipselects(dev);
-	if (ret)
-		return ret;
+	num_cs = of_fsl_espi_get_chipselects(dev);
+	if (!num_cs)
+		return -EINVAL;
 
 	ret = of_address_to_resource(np, 0, &mem);
 	if (ret)
@@ -699,7 +778,7 @@ static int of_fsl_espi_probe(struct platform_device *ofdev)
 	if (!irq)
 		return -EINVAL;
 
-	return fsl_espi_probe(dev, &mem, irq);
+	return fsl_espi_probe(dev, &mem, irq, num_cs);
 }
 
 static int of_fsl_espi_remove(struct platform_device *dev)
@@ -721,38 +800,15 @@ static int of_fsl_espi_suspend(struct device *dev)
 		return ret;
 	}
 
-	ret = pm_runtime_force_suspend(dev);
-	if (ret < 0)
-		return ret;
-
-	return 0;
+	return pm_runtime_force_suspend(dev);
 }
 
 static int of_fsl_espi_resume(struct device *dev)
 {
-	struct fsl_spi_platform_data *pdata = dev_get_platdata(dev);
 	struct spi_master *master = dev_get_drvdata(dev);
-	struct mpc8xxx_spi *mpc8xxx_spi;
-	u32 regval;
-	int i, ret;
-
-	mpc8xxx_spi = spi_master_get_devdata(master);
-
-	/* SPI controller initializations */
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, 0);
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIM, 0);
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPCOM, 0);
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPIE, 0xffffffff);
-
-	/* Init eSPI CS mode register */
-	for (i = 0; i < pdata->max_chipselect; i++)
-		fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODEx(i),
-				      CSMODE_INIT_VAL);
-
-	/* Enable SPI interface */
-	regval = pdata->initial_spmode | SPMODE_INIT_VAL | SPMODE_ENABLE;
+	int ret;
 
-	fsl_espi_write_reg(mpc8xxx_spi, ESPI_SPMODE, regval);
+	fsl_espi_init_regs(dev, false);
 
 	ret = pm_runtime_force_resume(dev);
 	if (ret < 0)
diff --git a/drivers/spi/spi-fsl-lib.h b/drivers/spi/spi-fsl-lib.h
index 2925c8089fd9..f303f306b38e 100644
--- a/drivers/spi/spi-fsl-lib.h
+++ b/drivers/spi/spi-fsl-lib.h
@@ -28,10 +28,6 @@ struct mpc8xxx_spi {
 	/* rx & tx bufs from the spi_transfer */
 	const void *tx;
 	void *rx;
-#if IS_ENABLED(CONFIG_SPI_FSL_ESPI)
-	int len;
-	u8 *local_buf;
-#endif
 
 	int subblock;
 	struct spi_pram __iomem *pram;
diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c
new file mode 100644
index 000000000000..52551f6d0c7d
--- /dev/null
+++ b/drivers/spi/spi-fsl-lpspi.c
@@ -0,0 +1,525 @@
+/*
+ * Freescale i.MX7ULP LPSPI driver
+ *
+ * Copyright 2016 Freescale Semiconductor, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/spi/spi.h>
+#include <linux/spi/spi_bitbang.h>
+#include <linux/types.h>
+
+#define DRIVER_NAME "fsl_lpspi"
+
+/* i.MX7ULP LPSPI registers */
+#define IMX7ULP_VERID	0x0
+#define IMX7ULP_PARAM	0x4
+#define IMX7ULP_CR	0x10
+#define IMX7ULP_SR	0x14
+#define IMX7ULP_IER	0x18
+#define IMX7ULP_DER	0x1c
+#define IMX7ULP_CFGR0	0x20
+#define IMX7ULP_CFGR1	0x24
+#define IMX7ULP_DMR0	0x30
+#define IMX7ULP_DMR1	0x34
+#define IMX7ULP_CCR	0x40
+#define IMX7ULP_FCR	0x58
+#define IMX7ULP_FSR	0x5c
+#define IMX7ULP_TCR	0x60
+#define IMX7ULP_TDR	0x64
+#define IMX7ULP_RSR	0x70
+#define IMX7ULP_RDR	0x74
+
+/* General control register field define */
+#define CR_RRF		BIT(9)
+#define CR_RTF		BIT(8)
+#define CR_RST		BIT(1)
+#define CR_MEN		BIT(0)
+#define SR_TCF		BIT(10)
+#define SR_RDF		BIT(1)
+#define SR_TDF		BIT(0)
+#define IER_TCIE	BIT(10)
+#define IER_RDIE	BIT(1)
+#define IER_TDIE	BIT(0)
+#define CFGR1_PCSCFG	BIT(27)
+#define CFGR1_PCSPOL	BIT(8)
+#define CFGR1_NOSTALL	BIT(3)
+#define CFGR1_MASTER	BIT(0)
+#define RSR_RXEMPTY	BIT(1)
+#define TCR_CPOL	BIT(31)
+#define TCR_CPHA	BIT(30)
+#define TCR_CONT	BIT(21)
+#define TCR_CONTC	BIT(20)
+#define TCR_RXMSK	BIT(19)
+#define TCR_TXMSK	BIT(18)
+
+static int clkdivs[] = {1, 2, 4, 8, 16, 32, 64, 128};
+
+struct lpspi_config {
+	u8 bpw;
+	u8 chip_select;
+	u8 prescale;
+	u16 mode;
+	u32 speed_hz;
+};
+
+struct fsl_lpspi_data {
+	struct device *dev;
+	void __iomem *base;
+	struct clk *clk;
+
+	void *rx_buf;
+	const void *tx_buf;
+	void (*tx)(struct fsl_lpspi_data *);
+	void (*rx)(struct fsl_lpspi_data *);
+
+	u32 remain;
+	u8 txfifosize;
+	u8 rxfifosize;
+
+	struct lpspi_config config;
+	struct completion xfer_done;
+};
+
+static const struct of_device_id fsl_lpspi_dt_ids[] = {
+	{ .compatible = "fsl,imx7ulp-spi", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, fsl_lpspi_dt_ids);
+
+#define LPSPI_BUF_RX(type)						\
+static void fsl_lpspi_buf_rx_##type(struct fsl_lpspi_data *fsl_lpspi)	\
+{									\
+	unsigned int val = readl(fsl_lpspi->base + IMX7ULP_RDR);	\
+									\
+	if (fsl_lpspi->rx_buf) {					\
+		*(type *)fsl_lpspi->rx_buf = val;			\
+		fsl_lpspi->rx_buf += sizeof(type);                      \
+	}								\
+}
+
+#define LPSPI_BUF_TX(type)						\
+static void fsl_lpspi_buf_tx_##type(struct fsl_lpspi_data *fsl_lpspi)	\
+{									\
+	type val = 0;							\
+									\
+	if (fsl_lpspi->tx_buf) {					\
+		val = *(type *)fsl_lpspi->tx_buf;			\
+		fsl_lpspi->tx_buf += sizeof(type);			\
+	}								\
+									\
+	fsl_lpspi->remain -= sizeof(type);				\
+	writel(val, fsl_lpspi->base + IMX7ULP_TDR);			\
+}
+
+LPSPI_BUF_RX(u8)
+LPSPI_BUF_TX(u8)
+LPSPI_BUF_RX(u16)
+LPSPI_BUF_TX(u16)
+LPSPI_BUF_RX(u32)
+LPSPI_BUF_TX(u32)
+
+static void fsl_lpspi_intctrl(struct fsl_lpspi_data *fsl_lpspi,
+			      unsigned int enable)
+{
+	writel(enable, fsl_lpspi->base + IMX7ULP_IER);
+}
+
+static int lpspi_prepare_xfer_hardware(struct spi_master *master)
+{
+	struct fsl_lpspi_data *fsl_lpspi = spi_master_get_devdata(master);
+
+	return clk_prepare_enable(fsl_lpspi->clk);
+}
+
+static int lpspi_unprepare_xfer_hardware(struct spi_master *master)
+{
+	struct fsl_lpspi_data *fsl_lpspi = spi_master_get_devdata(master);
+
+	clk_disable_unprepare(fsl_lpspi->clk);
+
+	return 0;
+}
+
+static int fsl_lpspi_txfifo_empty(struct fsl_lpspi_data *fsl_lpspi)
+{
+	u32 txcnt;
+	unsigned long orig_jiffies = jiffies;
+
+	do {
+		txcnt = readl(fsl_lpspi->base + IMX7ULP_FSR) & 0xff;
+
+		if (time_after(jiffies, orig_jiffies + msecs_to_jiffies(500))) {
+			dev_dbg(fsl_lpspi->dev, "txfifo empty timeout\n");
+			return -ETIMEDOUT;
+		}
+		cond_resched();
+
+	} while (txcnt);
+
+	return 0;
+}
+
+static void fsl_lpspi_write_tx_fifo(struct fsl_lpspi_data *fsl_lpspi)
+{
+	u8 txfifo_cnt;
+
+	txfifo_cnt = readl(fsl_lpspi->base + IMX7ULP_FSR) & 0xff;
+
+	while (txfifo_cnt < fsl_lpspi->txfifosize) {
+		if (!fsl_lpspi->remain)
+			break;
+		fsl_lpspi->tx(fsl_lpspi);
+		txfifo_cnt++;
+	}
+
+	if (!fsl_lpspi->remain && (txfifo_cnt < fsl_lpspi->txfifosize))
+		writel(0, fsl_lpspi->base + IMX7ULP_TDR);
+	else
+		fsl_lpspi_intctrl(fsl_lpspi, IER_TDIE);
+}
+
+static void fsl_lpspi_read_rx_fifo(struct fsl_lpspi_data *fsl_lpspi)
+{
+	while (!(readl(fsl_lpspi->base + IMX7ULP_RSR) & RSR_RXEMPTY))
+		fsl_lpspi->rx(fsl_lpspi);
+}
+
+static void fsl_lpspi_set_cmd(struct fsl_lpspi_data *fsl_lpspi,
+			      bool is_first_xfer)
+{
+	u32 temp = 0;
+
+	temp |= fsl_lpspi->config.bpw - 1;
+	temp |= fsl_lpspi->config.prescale << 27;
+	temp |= (fsl_lpspi->config.mode & 0x3) << 30;
+	temp |= (fsl_lpspi->config.chip_select & 0x3) << 24;
+
+	/*
+	 * Set TCR_CONT will keep SS asserted after current transfer.
+	 * For the first transfer, clear TCR_CONTC to assert SS.
+	 * For subsequent transfer, set TCR_CONTC to keep SS asserted.
+	 */
+	temp |= TCR_CONT;
+	if (is_first_xfer)
+		temp &= ~TCR_CONTC;
+	else
+		temp |= TCR_CONTC;
+
+	writel(temp, fsl_lpspi->base + IMX7ULP_TCR);
+
+	dev_dbg(fsl_lpspi->dev, "TCR=0x%x\n", temp);
+}
+
+static void fsl_lpspi_set_watermark(struct fsl_lpspi_data *fsl_lpspi)
+{
+	u32 temp;
+
+	temp = fsl_lpspi->txfifosize >> 1 | (fsl_lpspi->rxfifosize >> 1) << 16;
+
+	writel(temp, fsl_lpspi->base + IMX7ULP_FCR);
+
+	dev_dbg(fsl_lpspi->dev, "FCR=0x%x\n", temp);
+}
+
+static int fsl_lpspi_set_bitrate(struct fsl_lpspi_data *fsl_lpspi)
+{
+	struct lpspi_config config = fsl_lpspi->config;
+	unsigned int perclk_rate, scldiv;
+	u8 prescale;
+
+	perclk_rate = clk_get_rate(fsl_lpspi->clk);
+	for (prescale = 0; prescale < 8; prescale++) {
+		scldiv = perclk_rate /
+			 (clkdivs[prescale] * config.speed_hz) - 2;
+		if (scldiv < 256) {
+			fsl_lpspi->config.prescale = prescale;
+			break;
+		}
+	}
+
+	if (prescale == 8 && scldiv >= 256)
+		return -EINVAL;
+
+	writel(scldiv, fsl_lpspi->base + IMX7ULP_CCR);
+
+	dev_dbg(fsl_lpspi->dev, "perclk=%d, speed=%d, prescale =%d, scldiv=%d\n",
+		perclk_rate, config.speed_hz, prescale, scldiv);
+
+	return 0;
+}
+
+static int fsl_lpspi_config(struct fsl_lpspi_data *fsl_lpspi)
+{
+	u32 temp;
+	int ret;
+
+	temp = CR_RST;
+	writel(temp, fsl_lpspi->base + IMX7ULP_CR);
+	writel(0, fsl_lpspi->base + IMX7ULP_CR);
+
+	ret = fsl_lpspi_set_bitrate(fsl_lpspi);
+	if (ret)
+		return ret;
+
+	fsl_lpspi_set_watermark(fsl_lpspi);
+
+	temp = CFGR1_PCSCFG | CFGR1_MASTER | CFGR1_NOSTALL;
+	if (fsl_lpspi->config.mode & SPI_CS_HIGH)
+		temp |= CFGR1_PCSPOL;
+	writel(temp, fsl_lpspi->base + IMX7ULP_CFGR1);
+
+	temp = readl(fsl_lpspi->base + IMX7ULP_CR);
+	temp |= CR_RRF | CR_RTF | CR_MEN;
+	writel(temp, fsl_lpspi->base + IMX7ULP_CR);
+
+	return 0;
+}
+
+static void fsl_lpspi_setup_transfer(struct spi_device *spi,
+				     struct spi_transfer *t)
+{
+	struct fsl_lpspi_data *fsl_lpspi = spi_master_get_devdata(spi->master);
+
+	fsl_lpspi->config.mode = spi->mode;
+	fsl_lpspi->config.bpw = t ? t->bits_per_word : spi->bits_per_word;
+	fsl_lpspi->config.speed_hz = t ? t->speed_hz : spi->max_speed_hz;
+	fsl_lpspi->config.chip_select = spi->chip_select;
+
+	if (!fsl_lpspi->config.speed_hz)
+		fsl_lpspi->config.speed_hz = spi->max_speed_hz;
+	if (!fsl_lpspi->config.bpw)
+		fsl_lpspi->config.bpw = spi->bits_per_word;
+
+	/* Initialize the functions for transfer */
+	if (fsl_lpspi->config.bpw <= 8) {
+		fsl_lpspi->rx = fsl_lpspi_buf_rx_u8;
+		fsl_lpspi->tx = fsl_lpspi_buf_tx_u8;
+	} else if (fsl_lpspi->config.bpw <= 16) {
+		fsl_lpspi->rx = fsl_lpspi_buf_rx_u16;
+		fsl_lpspi->tx = fsl_lpspi_buf_tx_u16;
+	} else {
+		fsl_lpspi->rx = fsl_lpspi_buf_rx_u32;
+		fsl_lpspi->tx = fsl_lpspi_buf_tx_u32;
+	}
+
+	fsl_lpspi_config(fsl_lpspi);
+}
+
+static int fsl_lpspi_transfer_one(struct spi_master *master,
+				  struct spi_device *spi,
+				  struct spi_transfer *t)
+{
+	struct fsl_lpspi_data *fsl_lpspi = spi_master_get_devdata(master);
+	int ret;
+
+	fsl_lpspi->tx_buf = t->tx_buf;
+	fsl_lpspi->rx_buf = t->rx_buf;
+	fsl_lpspi->remain = t->len;
+
+	reinit_completion(&fsl_lpspi->xfer_done);
+	fsl_lpspi_write_tx_fifo(fsl_lpspi);
+
+	ret = wait_for_completion_timeout(&fsl_lpspi->xfer_done, HZ);
+	if (!ret) {
+		dev_dbg(fsl_lpspi->dev, "wait for completion timeout\n");
+		return -ETIMEDOUT;
+	}
+
+	ret = fsl_lpspi_txfifo_empty(fsl_lpspi);
+	if (ret)
+		return ret;
+
+	fsl_lpspi_read_rx_fifo(fsl_lpspi);
+
+	return 0;
+}
+
+static int fsl_lpspi_transfer_one_msg(struct spi_master *master,
+				      struct spi_message *msg)
+{
+	struct fsl_lpspi_data *fsl_lpspi = spi_master_get_devdata(master);
+	struct spi_device *spi = msg->spi;
+	struct spi_transfer *xfer;
+	bool is_first_xfer = true;
+	u32 temp;
+	int ret;
+
+	msg->status = 0;
+	msg->actual_length = 0;
+
+	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
+		fsl_lpspi_setup_transfer(spi, xfer);
+		fsl_lpspi_set_cmd(fsl_lpspi, is_first_xfer);
+
+		is_first_xfer = false;
+
+		ret = fsl_lpspi_transfer_one(master, spi, xfer);
+		if (ret < 0)
+			goto complete;
+
+		msg->actual_length += xfer->len;
+	}
+
+complete:
+	/* de-assert SS, then finalize current message */
+	temp = readl(fsl_lpspi->base + IMX7ULP_TCR);
+	temp &= ~TCR_CONTC;
+	writel(temp, fsl_lpspi->base + IMX7ULP_TCR);
+
+	msg->status = ret;
+	spi_finalize_current_message(master);
+
+	return ret;
+}
+
+static irqreturn_t fsl_lpspi_isr(int irq, void *dev_id)
+{
+	struct fsl_lpspi_data *fsl_lpspi = dev_id;
+	u32 temp;
+
+	fsl_lpspi_intctrl(fsl_lpspi, 0);
+	temp = readl(fsl_lpspi->base + IMX7ULP_SR);
+
+	fsl_lpspi_read_rx_fifo(fsl_lpspi);
+
+	if (temp & SR_TDF) {
+		fsl_lpspi_write_tx_fifo(fsl_lpspi);
+
+		if (!fsl_lpspi->remain)
+			complete(&fsl_lpspi->xfer_done);
+
+		return IRQ_HANDLED;
+	}
+
+	return IRQ_NONE;
+}
+
+static int fsl_lpspi_probe(struct platform_device *pdev)
+{
+	struct fsl_lpspi_data *fsl_lpspi;
+	struct spi_master *master;
+	struct resource *res;
+	int ret, irq;
+	u32 temp;
+
+	master = spi_alloc_master(&pdev->dev, sizeof(struct fsl_lpspi_data));
+	if (!master)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, master);
+
+	master->bits_per_word_mask = SPI_BPW_RANGE_MASK(8, 32);
+	master->bus_num = pdev->id;
+
+	fsl_lpspi = spi_master_get_devdata(master);
+	fsl_lpspi->dev = &pdev->dev;
+
+	master->transfer_one_message = fsl_lpspi_transfer_one_msg;
+	master->prepare_transfer_hardware = lpspi_prepare_xfer_hardware;
+	master->unprepare_transfer_hardware = lpspi_unprepare_xfer_hardware;
+	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
+	master->flags = SPI_MASTER_MUST_RX | SPI_MASTER_MUST_TX;
+	master->dev.of_node = pdev->dev.of_node;
+	master->bus_num = pdev->id;
+
+	init_completion(&fsl_lpspi->xfer_done);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	fsl_lpspi->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(fsl_lpspi->base)) {
+		ret = PTR_ERR(fsl_lpspi->base);
+		goto out_master_put;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		ret = irq;
+		goto out_master_put;
+	}
+
+	ret = devm_request_irq(&pdev->dev, irq, fsl_lpspi_isr, 0,
+			       dev_name(&pdev->dev), fsl_lpspi);
+	if (ret) {
+		dev_err(&pdev->dev, "can't get irq%d: %d\n", irq, ret);
+		goto out_master_put;
+	}
+
+	fsl_lpspi->clk = devm_clk_get(&pdev->dev, "ipg");
+	if (IS_ERR(fsl_lpspi->clk)) {
+		ret = PTR_ERR(fsl_lpspi->clk);
+		goto out_master_put;
+	}
+
+	ret = clk_prepare_enable(fsl_lpspi->clk);
+	if (ret) {
+		dev_err(&pdev->dev, "can't enable lpspi clock, ret=%d\n", ret);
+		goto out_master_put;
+	}
+
+	temp = readl(fsl_lpspi->base + IMX7ULP_PARAM);
+	fsl_lpspi->txfifosize = 1 << (temp & 0x0f);
+	fsl_lpspi->rxfifosize = 1 << ((temp >> 8) & 0x0f);
+
+	clk_disable_unprepare(fsl_lpspi->clk);
+
+	ret = devm_spi_register_master(&pdev->dev, master);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "spi_register_master error.\n");
+		goto out_master_put;
+	}
+
+	return 0;
+
+out_master_put:
+	spi_master_put(master);
+
+	return ret;
+}
+
+static int fsl_lpspi_remove(struct platform_device *pdev)
+{
+	struct spi_master *master = platform_get_drvdata(pdev);
+	struct fsl_lpspi_data *fsl_lpspi = spi_master_get_devdata(master);
+
+	clk_disable_unprepare(fsl_lpspi->clk);
+
+	return 0;
+}
+
+static struct platform_driver fsl_lpspi_driver = {
+	.driver = {
+		   .name = DRIVER_NAME,
+		   .of_match_table = fsl_lpspi_dt_ids,
+		   },
+	.probe = fsl_lpspi_probe,
+	.remove = fsl_lpspi_remove,
+};
+module_platform_driver(fsl_lpspi_driver);
+
+MODULE_DESCRIPTION("LPSPI Master Controller driver");
+MODULE_AUTHOR("Gao Pan <pandy.gao@nxp.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/spi/spi-imx.c b/drivers/spi/spi-imx.c
index deb782f6556c..32ced64a5bb9 100644
--- a/drivers/spi/spi-imx.c
+++ b/drivers/spi/spi-imx.c
@@ -173,15 +173,16 @@ static int mxc_clkdivs[] = {0, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192,
 
 /* MX21, MX27 */
 static unsigned int spi_imx_clkdiv_1(unsigned int fin,
-		unsigned int fspi, unsigned int max)
+		unsigned int fspi, unsigned int max, unsigned int *fres)
 {
 	int i;
 
 	for (i = 2; i < max; i++)
 		if (fspi * mxc_clkdivs[i] >= fin)
-			return i;
+			break;
 
-	return max;
+	*fres = fin / mxc_clkdivs[i];
+	return i;
 }
 
 /* MX1, MX31, MX35, MX51 CSPI */
@@ -442,6 +443,7 @@ static void mx51_ecspi_reset(struct spi_imx_data *spi_imx)
 #define MX31_CSPICTRL_ENABLE	(1 << 0)
 #define MX31_CSPICTRL_MASTER	(1 << 1)
 #define MX31_CSPICTRL_XCH	(1 << 2)
+#define MX31_CSPICTRL_SMC	(1 << 3)
 #define MX31_CSPICTRL_POL	(1 << 4)
 #define MX31_CSPICTRL_PHA	(1 << 5)
 #define MX31_CSPICTRL_SSCTL	(1 << 6)
@@ -452,6 +454,10 @@ static void mx51_ecspi_reset(struct spi_imx_data *spi_imx)
 #define MX35_CSPICTRL_CS_SHIFT	12
 #define MX31_CSPICTRL_DR_SHIFT	16
 
+#define MX31_CSPI_DMAREG	0x10
+#define MX31_DMAREG_RH_DEN	(1<<4)
+#define MX31_DMAREG_TH_DEN	(1<<1)
+
 #define MX31_CSPISTATUS		0x14
 #define MX31_STATUS_RR		(1 << 3)
 
@@ -511,6 +517,9 @@ static int mx31_config(struct spi_device *spi, struct spi_imx_config *config)
 			(is_imx35_cspi(spi_imx) ? MX35_CSPICTRL_CS_SHIFT :
 						  MX31_CSPICTRL_CS_SHIFT);
 
+	if (spi_imx->usedma)
+		reg |= MX31_CSPICTRL_SMC;
+
 	writel(reg, spi_imx->base + MXC_CSPICTRL);
 
 	reg = readl(spi_imx->base + MX31_CSPI_TESTREG);
@@ -520,6 +529,13 @@ static int mx31_config(struct spi_device *spi, struct spi_imx_config *config)
 		reg &= ~MX31_TEST_LBC;
 	writel(reg, spi_imx->base + MX31_CSPI_TESTREG);
 
+	if (spi_imx->usedma) {
+		/* configure DMA requests when RXFIFO is half full and
+		   when TXFIFO is half empty */
+		writel(MX31_DMAREG_RH_DEN | MX31_DMAREG_TH_DEN,
+			spi_imx->base + MX31_CSPI_DMAREG);
+	}
+
 	return 0;
 }
 
@@ -574,9 +590,12 @@ static int mx21_config(struct spi_device *spi, struct spi_imx_config *config)
 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);
 	unsigned int reg = MX21_CSPICTRL_ENABLE | MX21_CSPICTRL_MASTER;
 	unsigned int max = is_imx27_cspi(spi_imx) ? 16 : 18;
+	unsigned int clk;
+
+	reg |= spi_imx_clkdiv_1(spi_imx->spi_clk, config->speed_hz, max, &clk)
+		<< MX21_CSPICTRL_DR_SHIFT;
+	spi_imx->spi_bus_clk = clk;
 
-	reg |= spi_imx_clkdiv_1(spi_imx->spi_clk, config->speed_hz, max) <<
-		MX21_CSPICTRL_DR_SHIFT;
 	reg |= config->bpw - 1;
 
 	if (spi->mode & SPI_CPHA)
@@ -1244,10 +1263,10 @@ static int spi_imx_probe(struct platform_device *pdev)
 
 	spi_imx->spi_clk = clk_get_rate(spi_imx->clk_per);
 	/*
-	 * Only validated on i.mx6 now, can remove the constrain if validated on
-	 * other chips.
+	 * Only validated on i.mx35 and i.mx6 now, can remove the constraint
+	 * if validated on other chips.
 	 */
-	if (is_imx51_ecspi(spi_imx)) {
+	if (is_imx35_cspi(spi_imx) || is_imx51_ecspi(spi_imx)) {
 		ret = spi_imx_sdma_init(&pdev->dev, spi_imx, master);
 		if (ret == -EPROBE_DEFER)
 			goto out_clk_put;
diff --git a/drivers/spi/spi-jcore.c b/drivers/spi/spi-jcore.c
index f8117b80fa22..cebfea5faa4b 100644
--- a/drivers/spi/spi-jcore.c
+++ b/drivers/spi/spi-jcore.c
@@ -214,6 +214,7 @@ static const struct of_device_id jcore_spi_of_match[] = {
 	{ .compatible = "jcore,spi2" },
 	{},
 };
+MODULE_DEVICE_TABLE(of, jcore_spi_of_match);
 
 static struct platform_driver jcore_spi_driver = {
 	.probe = jcore_spi_probe,
diff --git a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c
index d5157b2222ce..79800e991ccd 100644
--- a/drivers/spi/spi-omap2-mcspi.c
+++ b/drivers/spi/spi-omap2-mcspi.c
@@ -1386,20 +1386,13 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 	regs_offset = pdata->regs_offset;
 
 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (r == NULL) {
-		status = -ENODEV;
-		goto free_master;
-	}
-
-	r->start += regs_offset;
-	r->end += regs_offset;
-	mcspi->phys = r->start;
-
 	mcspi->base = devm_ioremap_resource(&pdev->dev, r);
 	if (IS_ERR(mcspi->base)) {
 		status = PTR_ERR(mcspi->base);
 		goto free_master;
 	}
+	mcspi->phys = r->start + regs_offset;
+	mcspi->base += regs_offset;
 
 	mcspi->dev = &pdev->dev;
 
diff --git a/drivers/spi/spi-orion.c b/drivers/spi/spi-orion.c
index ded37025b445..6b001c4a5640 100644
--- a/drivers/spi/spi-orion.c
+++ b/drivers/spi/spi-orion.c
@@ -138,37 +138,62 @@ static int orion_spi_baudrate_set(struct spi_device *spi, unsigned int speed)
 	tclk_hz = clk_get_rate(orion_spi->clk);
 
 	if (devdata->typ == ARMADA_SPI) {
-		unsigned int clk, spr, sppr, sppr2, err;
-		unsigned int best_spr, best_sppr, best_err;
-
-		best_err = speed;
-		best_spr = 0;
-		best_sppr = 0;
-
-		/* Iterate over the valid range looking for best fit */
-		for (sppr = 0; sppr < 8; sppr++) {
-			sppr2 = 0x1 << sppr;
-
-			spr = tclk_hz / sppr2;
-			spr = DIV_ROUND_UP(spr, speed);
-			if ((spr == 0) || (spr > 15))
-				continue;
-
-			clk = tclk_hz / (spr * sppr2);
-			err = speed - clk;
-
-			if (err < best_err) {
-				best_spr = spr;
-				best_sppr = sppr;
-				best_err = err;
-			}
-		}
+		/*
+		 * Given the core_clk (tclk_hz) and the target rate (speed) we
+		 * determine the best values for SPR (in [0 .. 15]) and SPPR (in
+		 * [0..7]) such that
+		 *
+		 * 	core_clk / (SPR * 2 ** SPPR)
+		 *
+		 * is as big as possible but not bigger than speed.
+		 */
 
-		if ((best_sppr == 0) && (best_spr == 0))
-			return -EINVAL;
+		/* best integer divider: */
+		unsigned divider = DIV_ROUND_UP(tclk_hz, speed);
+		unsigned spr, sppr;
+
+		if (divider < 16) {
+			/* This is the easy case, divider is less than 16 */
+			spr = divider;
+			sppr = 0;
+
+		} else {
+			unsigned two_pow_sppr;
+			/*
+			 * Find the highest bit set in divider. This and the
+			 * three next bits define SPR (apart from rounding).
+			 * SPPR is then the number of zero bits that must be
+			 * appended:
+			 */
+			sppr = fls(divider) - 4;
+
+			/*
+			 * As SPR only has 4 bits, we have to round divider up
+			 * to the next multiple of 2 ** sppr.
+			 */
+			two_pow_sppr = 1 << sppr;
+			divider = (divider + two_pow_sppr - 1) & -two_pow_sppr;
+
+			/*
+			 * recalculate sppr as rounding up divider might have
+			 * increased it enough to change the position of the
+			 * highest set bit. In this case the bit that now
+			 * doesn't make it into SPR is 0, so there is no need to
+			 * round again.
+			 */
+			sppr = fls(divider) - 4;
+			spr = divider >> sppr;
+
+			/*
+			 * Now do range checking. SPR is constructed to have a
+			 * width of 4 bits, so this is fine for sure. So we
+			 * still need to check for sppr to fit into 3 bits:
+			 */
+			if (sppr > 7)
+				return -EINVAL;
+		}
 
-		prescale = ((best_sppr & 0x6) << 5) |
-			((best_sppr & 0x1) << 4) | best_spr;
+		prescale = ((sppr & 0x6) << 5) | ((sppr & 0x1) << 4) | spr;
 	} else {
 		/*
 		 * the supported rates are: 4,6,8...30
diff --git a/drivers/spi/spi-pxa2xx.h b/drivers/spi/spi-pxa2xx.h
index ce31b8199bb3..2823a00a9405 100644
--- a/drivers/spi/spi-pxa2xx.h
+++ b/drivers/spi/spi-pxa2xx.h
@@ -109,7 +109,6 @@ static  inline void pxa2xx_spi_write(const struct driver_data *drv_data,
 #define DONE_STATE ((void *)2)
 #define ERROR_STATE ((void *)-1)
 
-#define IS_DMA_ALIGNED(x)	IS_ALIGNED((unsigned long)(x), DMA_ALIGNMENT)
 #define DMA_ALIGNMENT		8
 
 static inline int pxa25x_ssp_comp(struct driver_data *drv_data)
diff --git a/drivers/spi/spi-rspi.c b/drivers/spi/spi-rspi.c
index a816f07e168e..9daf50031737 100644
--- a/drivers/spi/spi-rspi.c
+++ b/drivers/spi/spi-rspi.c
@@ -413,7 +413,7 @@ static unsigned int qspi_set_send_trigger(struct rspi_data *rspi,
 	return n;
 }
 
-static void qspi_set_receive_trigger(struct rspi_data *rspi, unsigned int len)
+static int qspi_set_receive_trigger(struct rspi_data *rspi, unsigned int len)
 {
 	unsigned int n;
 
@@ -428,6 +428,7 @@ static void qspi_set_receive_trigger(struct rspi_data *rspi, unsigned int len)
 		qspi_update(rspi, SPBFCR_RXTRG_MASK,
 			     SPBFCR_RXTRG_1B, QSPI_SPBFCR);
 	}
+	return n;
 }
 
 #define set_config_register(spi, n) spi->ops->set_config_register(spi, n)
@@ -785,6 +786,9 @@ static int qspi_transfer_out_in(struct rspi_data *rspi,
 
 static int qspi_transfer_out(struct rspi_data *rspi, struct spi_transfer *xfer)
 {
+	const u8 *tx = xfer->tx_buf;
+	unsigned int n = xfer->len;
+	unsigned int i, len;
 	int ret;
 
 	if (rspi->master->can_dma && __rspi_can_dma(rspi, xfer)) {
@@ -793,9 +797,23 @@ static int qspi_transfer_out(struct rspi_data *rspi, struct spi_transfer *xfer)
 			return ret;
 	}
 
-	ret = rspi_pio_transfer(rspi, xfer->tx_buf, NULL, xfer->len);
-	if (ret < 0)
-		return ret;
+	while (n > 0) {
+		len = qspi_set_send_trigger(rspi, n);
+		if (len == QSPI_BUFFER_SIZE) {
+			ret = rspi_wait_for_tx_empty(rspi);
+			if (ret < 0) {
+				dev_err(&rspi->master->dev, "transmit timeout\n");
+				return ret;
+			}
+			for (i = 0; i < len; i++)
+				rspi_write_data(rspi, *tx++);
+		} else {
+			ret = rspi_pio_transfer(rspi, tx, NULL, n);
+			if (ret < 0)
+				return ret;
+		}
+		n -= len;
+	}
 
 	/* Wait for the last transmission */
 	rspi_wait_for_tx_empty(rspi);
@@ -805,13 +823,37 @@ static int qspi_transfer_out(struct rspi_data *rspi, struct spi_transfer *xfer)
 
 static int qspi_transfer_in(struct rspi_data *rspi, struct spi_transfer *xfer)
 {
+	u8 *rx = xfer->rx_buf;
+	unsigned int n = xfer->len;
+	unsigned int i, len;
+	int ret;
+
 	if (rspi->master->can_dma && __rspi_can_dma(rspi, xfer)) {
 		int ret = rspi_dma_transfer(rspi, NULL, &xfer->rx_sg);
 		if (ret != -EAGAIN)
 			return ret;
 	}
 
-	return rspi_pio_transfer(rspi, NULL, xfer->rx_buf, xfer->len);
+	while (n > 0) {
+		len = qspi_set_receive_trigger(rspi, n);
+		if (len == QSPI_BUFFER_SIZE) {
+			ret = rspi_wait_for_rx_full(rspi);
+			if (ret < 0) {
+				dev_err(&rspi->master->dev, "receive timeout\n");
+				return ret;
+			}
+			for (i = 0; i < len; i++)
+				*rx++ = rspi_read_data(rspi);
+		} else {
+			ret = rspi_pio_transfer(rspi, NULL, rx, n);
+			if (ret < 0)
+				return ret;
+			*rx++ = ret;
+		}
+		n -= len;
+	}
+
+	return 0;
 }
 
 static int qspi_transfer_one(struct spi_master *master, struct spi_device *spi,
diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c
index 3c09e94cf827..28dfdce4beae 100644
--- a/drivers/spi/spi-s3c64xx.c
+++ b/drivers/spi/spi-s3c64xx.c
@@ -341,27 +341,20 @@ static void s3c64xx_spi_set_cs(struct spi_device *spi, bool enable)
 static int s3c64xx_spi_prepare_transfer(struct spi_master *spi)
 {
 	struct s3c64xx_spi_driver_data *sdd = spi_master_get_devdata(spi);
-	dma_filter_fn filter = sdd->cntrlr_info->filter;
 	struct device *dev = &sdd->pdev->dev;
-	dma_cap_mask_t mask;
 
 	if (is_polling(sdd))
 		return 0;
 
-	dma_cap_zero(mask);
-	dma_cap_set(DMA_SLAVE, mask);
-
 	/* Acquire DMA channels */
-	sdd->rx_dma.ch = dma_request_slave_channel_compat(mask, filter,
-			   sdd->cntrlr_info->dma_rx, dev, "rx");
+	sdd->rx_dma.ch = dma_request_slave_channel(dev, "rx");
 	if (!sdd->rx_dma.ch) {
 		dev_err(dev, "Failed to get RX DMA channel\n");
 		return -EBUSY;
 	}
 	spi->dma_rx = sdd->rx_dma.ch;
 
-	sdd->tx_dma.ch = dma_request_slave_channel_compat(mask, filter,
-			   sdd->cntrlr_info->dma_tx, dev, "tx");
+	sdd->tx_dma.ch = dma_request_slave_channel(dev, "tx");
 	if (!sdd->tx_dma.ch) {
 		dev_err(dev, "Failed to get TX DMA channel\n");
 		dma_release_channel(sdd->rx_dma.ch);
@@ -1091,11 +1084,6 @@ static int s3c64xx_spi_probe(struct platform_device *pdev)
 
 	sdd->cur_bpw = 8;
 
-	if (!sdd->pdev->dev.of_node && (!sci->dma_tx || !sci->dma_rx)) {
-		dev_warn(&pdev->dev, "Unable to get SPI tx/rx DMA data. Switching to poll mode\n");
-		sdd->port_conf->quirks = S3C64XX_SPI_QUIRK_POLL;
-	}
-
 	sdd->tx_dma.direction = DMA_MEM_TO_DEV;
 	sdd->rx_dma.direction = DMA_DEV_TO_MEM;
 
@@ -1205,9 +1193,8 @@ static int s3c64xx_spi_probe(struct platform_device *pdev)
 
 	dev_dbg(&pdev->dev, "Samsung SoC SPI Driver loaded for Bus SPI-%d with %d Slaves attached\n",
 					sdd->port_id, master->num_chipselect);
-	dev_dbg(&pdev->dev, "\tIOmem=[%pR]\tFIFO %dbytes\tDMA=[Rx-%p, Tx-%p]\n",
-					mem_res, (FIFO_LVL_MASK(sdd) >> 1) + 1,
-					sci->dma_rx, sci->dma_tx);
+	dev_dbg(&pdev->dev, "\tIOmem=[%pR]\tFIFO %dbytes\n",
+					mem_res, (FIFO_LVL_MASK(sdd) >> 1) + 1);
 
 	pm_runtime_mark_last_busy(&pdev->dev);
 	pm_runtime_put_autosuspend(&pdev->dev);
diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c
index 1de3a772eb7d..0012ad02e569 100644
--- a/drivers/spi/spi-sh-msiof.c
+++ b/drivers/spi/spi-sh-msiof.c
@@ -980,6 +980,7 @@ static const struct of_device_id sh_msiof_match[] = {
 	{ .compatible = "renesas,msiof-r8a7792",   .data = &r8a779x_data },
 	{ .compatible = "renesas,msiof-r8a7793",   .data = &r8a779x_data },
 	{ .compatible = "renesas,msiof-r8a7794",   .data = &r8a779x_data },
+	{ .compatible = "renesas,msiof-r8a7796",   .data = &r8a779x_data },
 	{},
 };
 MODULE_DEVICE_TABLE(of, sh_msiof_match);
diff --git a/drivers/spi/spi-sun4i.c b/drivers/spi/spi-sun4i.c
index 4969dc10684a..c5cd635c28f3 100644
--- a/drivers/spi/spi-sun4i.c
+++ b/drivers/spi/spi-sun4i.c
@@ -46,6 +46,8 @@
 #define SUN4I_CTL_TP				BIT(18)
 
 #define SUN4I_INT_CTL_REG		0x0c
+#define SUN4I_INT_CTL_RF_F34			BIT(4)
+#define SUN4I_INT_CTL_TF_E34			BIT(12)
 #define SUN4I_INT_CTL_TC			BIT(16)
 
 #define SUN4I_INT_STA_REG		0x10
@@ -61,11 +63,14 @@
 #define SUN4I_CLK_CTL_CDR1(div)			(((div) & SUN4I_CLK_CTL_CDR1_MASK) << 8)
 #define SUN4I_CLK_CTL_DRS			BIT(12)
 
+#define SUN4I_MAX_XFER_SIZE			0xffffff
+
 #define SUN4I_BURST_CNT_REG		0x20
-#define SUN4I_BURST_CNT(cnt)			((cnt) & 0xffffff)
+#define SUN4I_BURST_CNT(cnt)			((cnt) & SUN4I_MAX_XFER_SIZE)
 
 #define SUN4I_XMIT_CNT_REG		0x24
-#define SUN4I_XMIT_CNT(cnt)			((cnt) & 0xffffff)
+#define SUN4I_XMIT_CNT(cnt)			((cnt) & SUN4I_MAX_XFER_SIZE)
+
 
 #define SUN4I_FIFO_STA_REG		0x28
 #define SUN4I_FIFO_STA_RF_CNT_MASK		0x7f
@@ -96,6 +101,31 @@ static inline void sun4i_spi_write(struct sun4i_spi *sspi, u32 reg, u32 value)
 	writel(value, sspi->base_addr + reg);
 }
 
+static inline u32 sun4i_spi_get_tx_fifo_count(struct sun4i_spi *sspi)
+{
+	u32 reg = sun4i_spi_read(sspi, SUN4I_FIFO_STA_REG);
+
+	reg >>= SUN4I_FIFO_STA_TF_CNT_BITS;
+
+	return reg & SUN4I_FIFO_STA_TF_CNT_MASK;
+}
+
+static inline void sun4i_spi_enable_interrupt(struct sun4i_spi *sspi, u32 mask)
+{
+	u32 reg = sun4i_spi_read(sspi, SUN4I_INT_CTL_REG);
+
+	reg |= mask;
+	sun4i_spi_write(sspi, SUN4I_INT_CTL_REG, reg);
+}
+
+static inline void sun4i_spi_disable_interrupt(struct sun4i_spi *sspi, u32 mask)
+{
+	u32 reg = sun4i_spi_read(sspi, SUN4I_INT_CTL_REG);
+
+	reg &= ~mask;
+	sun4i_spi_write(sspi, SUN4I_INT_CTL_REG, reg);
+}
+
 static inline void sun4i_spi_drain_fifo(struct sun4i_spi *sspi, int len)
 {
 	u32 reg, cnt;
@@ -118,10 +148,13 @@ static inline void sun4i_spi_drain_fifo(struct sun4i_spi *sspi, int len)
 
 static inline void sun4i_spi_fill_fifo(struct sun4i_spi *sspi, int len)
 {
+	u32 cnt;
 	u8 byte;
 
-	if (len > sspi->len)
-		len = sspi->len;
+	/* See how much data we can fit */
+	cnt = SUN4I_FIFO_DEPTH - sun4i_spi_get_tx_fifo_count(sspi);
+
+	len = min3(len, (int)cnt, sspi->len);
 
 	while (len--) {
 		byte = sspi->tx_buf ? *sspi->tx_buf++ : 0;
@@ -184,10 +217,10 @@ static int sun4i_spi_transfer_one(struct spi_master *master,
 	u32 reg;
 
 	/* We don't support transfer larger than the FIFO */
-	if (tfr->len > SUN4I_FIFO_DEPTH)
+	if (tfr->len > SUN4I_MAX_XFER_SIZE)
 		return -EMSGSIZE;
 
-	if (tfr->tx_buf && tfr->len >= SUN4I_FIFO_DEPTH)
+	if (tfr->tx_buf && tfr->len >= SUN4I_MAX_XFER_SIZE)
 		return -EMSGSIZE;
 
 	reinit_completion(&sspi->done);
@@ -286,7 +319,11 @@ static int sun4i_spi_transfer_one(struct spi_master *master,
 	sun4i_spi_fill_fifo(sspi, SUN4I_FIFO_DEPTH - 1);
 
 	/* Enable the interrupts */
-	sun4i_spi_write(sspi, SUN4I_INT_CTL_REG, SUN4I_INT_CTL_TC);
+	sun4i_spi_enable_interrupt(sspi, SUN4I_INT_CTL_TC |
+					 SUN4I_INT_CTL_RF_F34);
+	/* Only enable Tx FIFO interrupt if we really need it */
+	if (tx_len > SUN4I_FIFO_DEPTH)
+		sun4i_spi_enable_interrupt(sspi, SUN4I_INT_CTL_TF_E34);
 
 	/* Start the transfer */
 	reg = sun4i_spi_read(sspi, SUN4I_CTL_REG);
@@ -306,7 +343,6 @@ static int sun4i_spi_transfer_one(struct spi_master *master,
 		goto out;
 	}
 
-	sun4i_spi_drain_fifo(sspi, SUN4I_FIFO_DEPTH);
 
 out:
 	sun4i_spi_write(sspi, SUN4I_INT_CTL_REG, 0);
@@ -322,10 +358,33 @@ static irqreturn_t sun4i_spi_handler(int irq, void *dev_id)
 	/* Transfer complete */
 	if (status & SUN4I_INT_CTL_TC) {
 		sun4i_spi_write(sspi, SUN4I_INT_STA_REG, SUN4I_INT_CTL_TC);
+		sun4i_spi_drain_fifo(sspi, SUN4I_FIFO_DEPTH);
 		complete(&sspi->done);
 		return IRQ_HANDLED;
 	}
 
+	/* Receive FIFO 3/4 full */
+	if (status & SUN4I_INT_CTL_RF_F34) {
+		sun4i_spi_drain_fifo(sspi, SUN4I_FIFO_DEPTH);
+		/* Only clear the interrupt _after_ draining the FIFO */
+		sun4i_spi_write(sspi, SUN4I_INT_STA_REG, SUN4I_INT_CTL_RF_F34);
+		return IRQ_HANDLED;
+	}
+
+	/* Transmit FIFO 3/4 empty */
+	if (status & SUN4I_INT_CTL_TF_E34) {
+		sun4i_spi_fill_fifo(sspi, SUN4I_FIFO_DEPTH);
+
+		if (!sspi->len)
+			/* nothing left to transmit */
+			sun4i_spi_disable_interrupt(sspi, SUN4I_INT_CTL_TF_E34);
+
+		/* Only clear the interrupt _after_ re-seeding the FIFO */
+		sun4i_spi_write(sspi, SUN4I_INT_STA_REG, SUN4I_INT_CTL_TF_E34);
+
+		return IRQ_HANDLED;
+	}
+
 	return IRQ_NONE;
 }
 
diff --git a/drivers/spi/spi-sun6i.c b/drivers/spi/spi-sun6i.c
index 9918a57a6a6e..e3114832c485 100644
--- a/drivers/spi/spi-sun6i.c
+++ b/drivers/spi/spi-sun6i.c
@@ -17,6 +17,7 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/reset.h>
@@ -24,6 +25,7 @@
 #include <linux/spi/spi.h>
 
 #define SUN6I_FIFO_DEPTH		128
+#define SUN8I_FIFO_DEPTH		64
 
 #define SUN6I_GBL_CTL_REG		0x04
 #define SUN6I_GBL_CTL_BUS_ENABLE		BIT(0)
@@ -90,6 +92,7 @@ struct sun6i_spi {
 	const u8		*tx_buf;
 	u8			*rx_buf;
 	int			len;
+	unsigned long		fifo_depth;
 };
 
 static inline u32 sun6i_spi_read(struct sun6i_spi *sspi, u32 reg)
@@ -155,7 +158,9 @@ static void sun6i_spi_set_cs(struct spi_device *spi, bool enable)
 
 static size_t sun6i_spi_max_transfer_size(struct spi_device *spi)
 {
-	return SUN6I_FIFO_DEPTH - 1;
+	struct sun6i_spi *sspi = spi_master_get_devdata(spi->master);
+
+	return sspi->fifo_depth - 1;
 }
 
 static int sun6i_spi_transfer_one(struct spi_master *master,
@@ -170,7 +175,7 @@ static int sun6i_spi_transfer_one(struct spi_master *master,
 	u32 reg;
 
 	/* We don't support transfer larger than the FIFO */
-	if (tfr->len > SUN6I_FIFO_DEPTH)
+	if (tfr->len > sspi->fifo_depth)
 		return -EINVAL;
 
 	reinit_completion(&sspi->done);
@@ -265,7 +270,7 @@ static int sun6i_spi_transfer_one(struct spi_master *master,
 			SUN6I_BURST_CTL_CNT_STC(tx_len));
 
 	/* Fill the TX FIFO */
-	sun6i_spi_fill_fifo(sspi, SUN6I_FIFO_DEPTH);
+	sun6i_spi_fill_fifo(sspi, sspi->fifo_depth);
 
 	/* Enable the interrupts */
 	sun6i_spi_write(sspi, SUN6I_INT_CTL_REG, SUN6I_INT_CTL_TC);
@@ -288,7 +293,7 @@ static int sun6i_spi_transfer_one(struct spi_master *master,
 		goto out;
 	}
 
-	sun6i_spi_drain_fifo(sspi, SUN6I_FIFO_DEPTH);
+	sun6i_spi_drain_fifo(sspi, sspi->fifo_depth);
 
 out:
 	sun6i_spi_write(sspi, SUN6I_INT_CTL_REG, 0);
@@ -398,6 +403,8 @@ static int sun6i_spi_probe(struct platform_device *pdev)
 	}
 
 	sspi->master = master;
+	sspi->fifo_depth = (unsigned long)of_device_get_match_data(&pdev->dev);
+
 	master->max_speed_hz = 100 * 1000 * 1000;
 	master->min_speed_hz = 3 * 1000;
 	master->set_cs = sun6i_spi_set_cs;
@@ -470,7 +477,8 @@ static int sun6i_spi_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id sun6i_spi_match[] = {
-	{ .compatible = "allwinner,sun6i-a31-spi", },
+	{ .compatible = "allwinner,sun6i-a31-spi", .data = (void *)SUN6I_FIFO_DEPTH },
+	{ .compatible = "allwinner,sun8i-h3-spi",  .data = (void *)SUN8I_FIFO_DEPTH },
 	{}
 };
 MODULE_DEVICE_TABLE(of, sun6i_spi_match);
diff --git a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c
index caeac66a3977..ec6fb09e2e17 100644
--- a/drivers/spi/spi-ti-qspi.c
+++ b/drivers/spi/spi-ti-qspi.c
@@ -411,6 +411,7 @@ static int ti_qspi_dma_xfer(struct ti_qspi *qspi, dma_addr_t dma_dst,
 	tx->callback = ti_qspi_dma_callback;
 	tx->callback_param = qspi;
 	cookie = tx->tx_submit(tx);
+	reinit_completion(&qspi->transfer_complete);
 
 	ret = dma_submit_error(cookie);
 	if (ret) {
diff --git a/drivers/spi/spi-topcliff-pch.c b/drivers/spi/spi-topcliff-pch.c
index c54ee6674471..fcb991034c3d 100644
--- a/drivers/spi/spi-topcliff-pch.c
+++ b/drivers/spi/spi-topcliff-pch.c
@@ -1268,11 +1268,8 @@ static void pch_spi_free_resources(struct pch_spi_board_data *board_dat,
 static int pch_spi_get_resources(struct pch_spi_board_data *board_dat,
 				 struct pch_spi_data *data)
 {
-	int retval = 0;
-
 	dev_dbg(&board_dat->pdev->dev, "%s ENTRY\n", __func__);
 
-
 	/* reset PCH SPI h/w */
 	pch_spi_reset(data->master);
 	dev_dbg(&board_dat->pdev->dev,
@@ -1280,15 +1277,7 @@ static int pch_spi_get_resources(struct pch_spi_board_data *board_dat,
 
 	dev_dbg(&board_dat->pdev->dev, "%s data->irq_reg_sts=true\n", __func__);
 
-	if (retval != 0) {
-		dev_err(&board_dat->pdev->dev,
-			"%s FAIL:invoking pch_spi_free_resources\n", __func__);
-		pch_spi_free_resources(board_dat, data);
-	}
-
-	dev_dbg(&board_dat->pdev->dev, "%s Return=%d\n", __func__, retval);
-
-	return retval;
+	return 0;
 }
 
 static void pch_free_dma_buf(struct pch_spi_board_data *board_dat,
diff --git a/drivers/spi/spi-xlp.c b/drivers/spi/spi-xlp.c
index 4071a729eb2f..bea7a93a6046 100644
--- a/drivers/spi/spi-xlp.c
+++ b/drivers/spi/spi-xlp.c
@@ -451,6 +451,7 @@ static const struct of_device_id xlp_spi_dt_id[] = {
 	{ .compatible = "netlogic,xlp832-spi" },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, xlp_spi_dt_id);
 
 static struct platform_driver xlp_spi_driver = {
 	.probe	= xlp_spi_probe,
diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c
index 838783c3fed0..656dd3e3220c 100644
--- a/drivers/spi/spi.c
+++ b/drivers/spi/spi.c
@@ -697,10 +697,15 @@ static void spi_set_cs(struct spi_device *spi, bool enable)
 	if (spi->mode & SPI_CS_HIGH)
 		enable = !enable;
 
-	if (gpio_is_valid(spi->cs_gpio))
+	if (gpio_is_valid(spi->cs_gpio)) {
 		gpio_set_value(spi->cs_gpio, !enable);
-	else if (spi->master->set_cs)
+		/* Some SPI masters need both GPIO CS & slave_select */
+		if ((spi->master->flags & SPI_MASTER_GPIO_SS) &&
+		    spi->master->set_cs)
+			spi->master->set_cs(spi, !enable);
+	} else if (spi->master->set_cs) {
 		spi->master->set_cs(spi, !enable);
+	}
 }
 
 #ifdef CONFIG_HAS_DMA
@@ -720,6 +725,7 @@ static int spi_map_buf(struct spi_master *master, struct device *dev,
 	int desc_len;
 	int sgs;
 	struct page *vm_page;
+	struct scatterlist *sg;
 	void *sg_buf;
 	size_t min;
 	int i, ret;
@@ -738,6 +744,7 @@ static int spi_map_buf(struct spi_master *master, struct device *dev,
 	if (ret != 0)
 		return ret;
 
+	sg = &sgt->sgl[0];
 	for (i = 0; i < sgs; i++) {
 
 		if (vmalloced_buf || kmap_buf) {
@@ -751,16 +758,17 @@ static int spi_map_buf(struct spi_master *master, struct device *dev,
 				sg_free_table(sgt);
 				return -ENOMEM;
 			}
-			sg_set_page(&sgt->sgl[i], vm_page,
+			sg_set_page(sg, vm_page,
 				    min, offset_in_page(buf));
 		} else {
 			min = min_t(size_t, len, desc_len);
 			sg_buf = buf;
-			sg_set_buf(&sgt->sgl[i], sg_buf, min);
+			sg_set_buf(sg, sg_buf, min);
 		}
 
 		buf += min;
 		len -= min;
+		sg = sg_next(sg);
 	}
 
 	ret = dma_map_sg(dev, sgt->sgl, sgt->nents, dir);
@@ -1034,8 +1042,14 @@ static int spi_transfer_one_message(struct spi_master *master,
 		if (msg->status != -EINPROGRESS)
 			goto out;
 
-		if (xfer->delay_usecs)
-			udelay(xfer->delay_usecs);
+		if (xfer->delay_usecs) {
+			u16 us = xfer->delay_usecs;
+
+			if (us <= 10)
+				udelay(us);
+			else
+				usleep_range(us, us + DIV_ROUND_UP(us, 10));
+		}
 
 		if (xfer->cs_change) {
 			if (list_is_last(&xfer->transfer_list,
diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c
index 2e05046f866b..9e2e099baf8c 100644
--- a/drivers/spi/spidev.c
+++ b/drivers/spi/spidev.c
@@ -696,6 +696,7 @@ static struct class *spidev_class;
 static const struct of_device_id spidev_dt_ids[] = {
 	{ .compatible = "rohm,dh2228fv" },
 	{ .compatible = "lineartechnology,ltc2488" },
+	{ .compatible = "ge,achc" },
 	{},
 };
 MODULE_DEVICE_TABLE(of, spidev_dt_ids);
diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig
index 58a7b3504b82..cd005cd41413 100644
--- a/drivers/staging/Kconfig
+++ b/drivers/staging/Kconfig
@@ -24,8 +24,6 @@ menuconfig STAGING
 
 if STAGING
 
-source "drivers/staging/slicoss/Kconfig"
-
 source "drivers/staging/wlan-ng/Kconfig"
 
 source "drivers/staging/comedi/Kconfig"
diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile
index 2fa9745db614..831e2e891989 100644
--- a/drivers/staging/Makefile
+++ b/drivers/staging/Makefile
@@ -1,7 +1,6 @@
 # Makefile for staging directory
 
 obj-y				+= media/
-obj-$(CONFIG_SLICOSS)		+= slicoss/
 obj-$(CONFIG_PRISM2_USB)	+= wlan-ng/
 obj-$(CONFIG_COMEDI)		+= comedi/
 obj-$(CONFIG_FB_OLPC_DCON)	+= olpc_dcon/
@@ -41,4 +40,4 @@ obj-$(CONFIG_MOST)		+= most/
 obj-$(CONFIG_ISDN_I4L)		+= i4l/
 obj-$(CONFIG_KS7010)		+= ks7010/
 obj-$(CONFIG_GREYBUS)		+= greybus/
-obj-$(CONFIG_BCM2708_VCHIQ)	+= vc04_services/
+obj-$(CONFIG_BCM2835_VCHIQ)	+= vc04_services/
diff --git a/drivers/staging/android/TODO b/drivers/staging/android/TODO
index 64d8c8720960..8f3ac37bfe12 100644
--- a/drivers/staging/android/TODO
+++ b/drivers/staging/android/TODO
@@ -25,13 +25,5 @@ ion/
    exposes existing cma regions and doesn't reserve unecessarily memory when
    booting a system which doesn't use ion.
 
-sync framework:
- - remove CONFIG_SW_SYNC_USER, it is used only for testing/debugging and
- should not be upstreamed.
- - port CONFIG_SW_SYNC_USER tests interfaces to use debugfs somehow
- - port libsync tests to kselftest
- - clean up and ABI check for security issues
- - move it to drivers/base/dma-buf
-
 Please send patches to Greg Kroah-Hartman <greg@kroah.com> and Cc:
 Arve Hjønnevåg <arve@android.com> and Riley Andrews <riandrews@android.com>
diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c
index ca9a53c03f0f..7cbad0d45b9c 100644
--- a/drivers/staging/android/ashmem.c
+++ b/drivers/staging/android/ashmem.c
@@ -100,39 +100,43 @@ static DEFINE_MUTEX(ashmem_mutex);
 static struct kmem_cache *ashmem_area_cachep __read_mostly;
 static struct kmem_cache *ashmem_range_cachep __read_mostly;
 
-#define range_size(range) \
-	((range)->pgend - (range)->pgstart + 1)
+static inline unsigned long range_size(struct ashmem_range *range)
+{
+	return range->pgend - range->pgstart + 1;
+}
 
-#define range_on_lru(range) \
-	((range)->purged == ASHMEM_NOT_PURGED)
+static inline bool range_on_lru(struct ashmem_range *range)
+{
+	return range->purged == ASHMEM_NOT_PURGED;
+}
 
-static inline int page_range_subsumes_range(struct ashmem_range *range,
-					    size_t start, size_t end)
+static inline bool page_range_subsumes_range(struct ashmem_range *range,
+					     size_t start, size_t end)
 {
-	return (((range)->pgstart >= (start)) && ((range)->pgend <= (end)));
+	return (range->pgstart >= start) && (range->pgend <= end);
 }
 
-static inline int page_range_subsumed_by_range(struct ashmem_range *range,
-					       size_t start, size_t end)
+static inline bool page_range_subsumed_by_range(struct ashmem_range *range,
+						size_t start, size_t end)
 {
-	return (((range)->pgstart <= (start)) && ((range)->pgend >= (end)));
+	return (range->pgstart <= start) && (range->pgend >= end);
 }
 
-static inline int page_in_range(struct ashmem_range *range, size_t page)
+static inline bool page_in_range(struct ashmem_range *range, size_t page)
 {
-	return (((range)->pgstart <= (page)) && ((range)->pgend >= (page)));
+	return (range->pgstart <= page) && (range->pgend >= page);
 }
 
-static inline int page_range_in_range(struct ashmem_range *range,
-				      size_t start, size_t end)
+static inline bool page_range_in_range(struct ashmem_range *range,
+				       size_t start, size_t end)
 {
-	return (page_in_range(range, start) || page_in_range(range, end) ||
-		page_range_subsumes_range(range, start, end));
+	return page_in_range(range, start) || page_in_range(range, end) ||
+		page_range_subsumes_range(range, start, end);
 }
 
-static inline int range_before_page(struct ashmem_range *range, size_t page)
+static inline bool range_before_page(struct ashmem_range *range, size_t page)
 {
-	return ((range)->pgend < (page));
+	return range->pgend < page;
 }
 
 #define PROT_MASK		(PROT_EXEC | PROT_READ | PROT_WRITE)
diff --git a/drivers/staging/android/ion/ion.c b/drivers/staging/android/ion/ion.c
index 209a8f7ef02b..b653451843c8 100644
--- a/drivers/staging/android/ion/ion.c
+++ b/drivers/staging/android/ion/ion.c
@@ -882,7 +882,7 @@ static int ion_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	BUG_ON(!buffer->pages || !buffer->pages[vmf->pgoff]);
 
 	pfn = page_to_pfn(ion_buffer_page(buffer->pages[vmf->pgoff]));
-	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
+	ret = vm_insert_pfn(vma, vmf->address, pfn);
 	mutex_unlock(&buffer->lock);
 	if (ret)
 		return VM_FAULT_ERROR;
@@ -1013,7 +1013,7 @@ static int ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
 	return 0;
 }
 
-static struct dma_buf_ops dma_buf_ops = {
+static const struct dma_buf_ops dma_buf_ops = {
 	.map_dma_buf = ion_map_dma_buf,
 	.unmap_dma_buf = ion_unmap_dma_buf,
 	.mmap = ion_mmap,
diff --git a/drivers/staging/android/ion/ion_dummy_driver.c b/drivers/staging/android/ion/ion_dummy_driver.c
index b23f2c76c753..cf5c010d32bc 100644
--- a/drivers/staging/android/ion/ion_dummy_driver.c
+++ b/drivers/staging/android/ion/ion_dummy_driver.c
@@ -58,7 +58,7 @@ static struct ion_platform_heap dummy_heaps[] = {
 		},
 };
 
-static struct ion_platform_data dummy_ion_pdata = {
+static const struct ion_platform_data dummy_ion_pdata = {
 	.nr = ARRAY_SIZE(dummy_heaps),
 	.heaps = dummy_heaps,
 };
diff --git a/drivers/staging/android/ion/ion_system_heap.c b/drivers/staging/android/ion/ion_system_heap.c
index 7e023d505af8..3ebbb75746e8 100644
--- a/drivers/staging/android/ion/ion_system_heap.c
+++ b/drivers/staging/android/ion/ion_system_heap.c
@@ -30,7 +30,7 @@
 
 static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN |
 				     __GFP_NORETRY) & ~__GFP_RECLAIM;
-static gfp_t low_order_gfp_flags  = (GFP_HIGHUSER | __GFP_ZERO);
+static gfp_t low_order_gfp_flags  = GFP_HIGHUSER | __GFP_ZERO;
 static const unsigned int orders[] = {8, 4, 0};
 
 static int order_to_index(unsigned int order)
diff --git a/drivers/staging/android/uapi/ion_test.h b/drivers/staging/android/uapi/ion_test.h
index ffef06f63133..480242e02f8d 100644
--- a/drivers/staging/android/uapi/ion_test.h
+++ b/drivers/staging/android/uapi/ion_test.h
@@ -66,5 +66,4 @@ struct ion_test_rw_data {
 #define ION_IOC_TEST_KERNEL_MAPPING \
 			_IOW(ION_IOC_MAGIC, 0xf2, struct ion_test_rw_data)
 
-
 #endif /* _UAPI_LINUX_ION_H */
diff --git a/drivers/staging/clocking-wizard/clk-xlnx-clock-wizard.c b/drivers/staging/clocking-wizard/clk-xlnx-clock-wizard.c
index 7b8be5293883..bf3fe7c61be5 100644
--- a/drivers/staging/clocking-wizard/clk-xlnx-clock-wizard.c
+++ b/drivers/staging/clocking-wizard/clk-xlnx-clock-wizard.c
@@ -68,7 +68,7 @@ struct clk_wzrd {
 	struct clk *axi_clk;
 	struct clk *clks_internal[wzrd_clk_int_max];
 	struct clk *clkout[WZRD_NUM_OUTPUTS];
-	int speed_grade;
+	unsigned int speed_grade;
 	bool suspended;
 };
 
diff --git a/drivers/staging/comedi/comedi.h b/drivers/staging/comedi/comedi.h
index 08fb26b51a5f..a1c1081906c5 100644
--- a/drivers/staging/comedi/comedi.h
+++ b/drivers/staging/comedi/comedi.h
@@ -245,6 +245,22 @@ enum comedi_subdevice_type {
 /* configuration instructions */
 
 /**
+ * enum comedi_io_direction - COMEDI I/O directions
+ * @COMEDI_INPUT:	Input.
+ * @COMEDI_OUTPUT:	Output.
+ * @COMEDI_OPENDRAIN:	Open-drain (or open-collector) output.
+ *
+ * These are used by the %INSN_CONFIG_DIO_QUERY configuration instruction to
+ * report a direction.  They may also be used in other places where a direction
+ * needs to be specified.
+ */
+enum comedi_io_direction {
+	COMEDI_INPUT = 0,
+	COMEDI_OUTPUT = 1,
+	COMEDI_OPENDRAIN = 2
+};
+
+/**
  * enum configuration_ids - COMEDI configuration instruction codes
  * @INSN_CONFIG_DIO_INPUT:	Configure digital I/O as input.
  * @INSN_CONFIG_DIO_OUTPUT:	Configure digital I/O as output.
@@ -296,9 +312,9 @@ enum comedi_subdevice_type {
  * @INSN_CONFIG_PWM_GET_H_BRIDGE: Get PWM H bridge duty cycle and polarity.
  */
 enum configuration_ids {
-	INSN_CONFIG_DIO_INPUT = 0,
-	INSN_CONFIG_DIO_OUTPUT = 1,
-	INSN_CONFIG_DIO_OPENDRAIN = 2,
+	INSN_CONFIG_DIO_INPUT = COMEDI_INPUT,
+	INSN_CONFIG_DIO_OUTPUT = COMEDI_OUTPUT,
+	INSN_CONFIG_DIO_OPENDRAIN = COMEDI_OPENDRAIN,
 	INSN_CONFIG_ANALOG_TRIG = 16,
 /*	INSN_CONFIG_WAVEFORM = 17, */
 /*	INSN_CONFIG_TRIG = 18, */
@@ -397,22 +413,6 @@ enum comedi_digital_trig_op {
 };
 
 /**
- * enum comedi_io_direction - COMEDI I/O directions
- * @COMEDI_INPUT:	Input.
- * @COMEDI_OUTPUT:	Output.
- * @COMEDI_OPENDRAIN:	Open-drain (or open-collector) output.
- *
- * These are used by the %INSN_CONFIG_DIO_QUERY configuration instruction to
- * report a direction.  They may also be used in other places where a direction
- * needs to be specified.
- */
-enum comedi_io_direction {
-	COMEDI_INPUT = 0,
-	COMEDI_OUTPUT = 1,
-	COMEDI_OPENDRAIN = 2
-};
-
-/**
  * enum comedi_support_level - support level for a COMEDI feature
  * @COMEDI_UNKNOWN_SUPPORT:	Unspecified support for feature.
  * @COMEDI_SUPPORTED:		Feature is supported.
@@ -1104,18 +1104,19 @@ enum ni_gpct_other_select {
 enum ni_gpct_arm_source {
 	NI_GPCT_ARM_IMMEDIATE = 0x0,
 	/*
-	 * Start both the counter and the adjacent pared
-	 * counter simultaneously
+	 * Start both the counter and the adjacent paired counter simultaneously
 	 */
 	NI_GPCT_ARM_PAIRED_IMMEDIATE = 0x1,
 	/*
-	 * NI doesn't document bits for selecting hardware arm triggers.
-	 * If the NI_GPCT_ARM_UNKNOWN bit is set, we will pass the least
-	 * significant bits (3 bits for 660x or 5 bits for m-series)
-	 * through to the hardware.  This will at least allow someone to
-	 * figure out what the bits do later.
+	 * If the NI_GPCT_HW_ARM bit is set, we will pass the least significant
+	 * bits (3 bits for 660x or 5 bits for m-series) through to the
+	 * hardware. To select a hardware trigger, pass the appropriate select
+	 * bit, e.g.,
+	 * NI_GPCT_HW_ARM | NI_GPCT_AI_START1_GATE_SELECT or
+	 * NI_GPCT_HW_ARM | NI_GPCT_PFI_GATE_SELECT(pfi_number)
 	 */
-	NI_GPCT_ARM_UNKNOWN = 0x1000,
+	NI_GPCT_HW_ARM = 0x1000,
+	NI_GPCT_ARM_UNKNOWN = NI_GPCT_HW_ARM,	/* for backward compatibility */
 };
 
 /* digital filtering options for ni 660x for use with INSN_CONFIG_FILTER. */
diff --git a/drivers/staging/comedi/comedidev.h b/drivers/staging/comedi/comedidev.h
index dcb637665eb7..0c7c37a8ff33 100644
--- a/drivers/staging/comedi/comedidev.h
+++ b/drivers/staging/comedi/comedidev.h
@@ -426,6 +426,18 @@ enum comedi_cb {
  * handler will be called with the COMEDI device structure's board_ptr member
  * pointing to the matched pointer to a board name within the driver's private
  * array of static, read-only board type information.
+ *
+ * The @detach handler has two roles.  If a COMEDI device was successfully
+ * configured by the @attach or @auto_attach handler, it is called when the
+ * device is being deconfigured (by the %COMEDI_DEVCONFIG ioctl, or due to
+ * unloading of the driver, or due to device removal).  It is also called when
+ * the @attach or @auto_attach handler returns an error.  Therefore, the
+ * @attach or @auto_attach handlers can defer clean-up on error until the
+ * @detach handler is called.  If the @attach or @auto_attach handlers free
+ * any resources themselves, they must prevent the @detach handler from
+ * freeing the same resources.  The @detach handler must not assume that all
+ * resources requested by the @attach or @auto_attach handler were
+ * successfully allocated.
  */
 struct comedi_driver {
 	/* private: */
diff --git a/drivers/staging/comedi/drivers/cb_pcidda.c b/drivers/staging/comedi/drivers/cb_pcidda.c
index ccb37d1f0f8e..987414741605 100644
--- a/drivers/staging/comedi/drivers/cb_pcidda.c
+++ b/drivers/staging/comedi/drivers/cb_pcidda.c
@@ -248,8 +248,8 @@ static void cb_pcidda_write_caldac(struct comedi_device *dev,
 	cb_pcidda_serial_out(dev, value, num_caldac_bits);
 
 /*
-* latch stream into appropriate caldac deselect reference dac
-*/
+ * latch stream into appropriate caldac deselect reference dac
+ */
 	cal2_bits = DESELECT_REF_DAC_BIT | DUMMY_BIT;
 	/*  deactivate caldacs (one caldac for every two channels) */
 	for (i = 0; i < max_num_caldacs; i++)
diff --git a/drivers/staging/comedi/drivers/mite.c b/drivers/staging/comedi/drivers/mite.c
index b1c0860135d0..05126ba4ba51 100644
--- a/drivers/staging/comedi/drivers/mite.c
+++ b/drivers/staging/comedi/drivers/mite.c
@@ -837,7 +837,7 @@ static int mite_setup(struct comedi_device *dev, struct mite *mite,
 	 * of 0x61f and bursts worked. 6281 powered up with register value of
 	 * 0x1f and bursts didn't work. The NI windows driver reads the
 	 * register, then does a bitwise-or of 0x600 with it and writes it back.
-	*
+	 *
 	 * The bits 0x90180700 in MITE_UNKNOWN_DMA_BURST_REG can be
 	 * written and read back.  The bits 0x1f always read as 1.
 	 * The rest always read as zero.
diff --git a/drivers/staging/comedi/drivers/ni_mio_common.c b/drivers/staging/comedi/drivers/ni_mio_common.c
index 0f97d7b611d7..b2e382888981 100644
--- a/drivers/staging/comedi/drivers/ni_mio_common.c
+++ b/drivers/staging/comedi/drivers/ni_mio_common.c
@@ -1832,11 +1832,10 @@ static int ni_ai_insn_read(struct comedi_device *dev,
 			   unsigned int *data)
 {
 	struct ni_private *devpriv = dev->private;
-	unsigned int mask = (s->maxdata + 1) >> 1;
+	unsigned int mask = s->maxdata;
 	int i, n;
 	unsigned int signbits;
 	unsigned int d;
-	unsigned long dl;
 
 	ni_load_channelgain_list(dev, s, 1, &insn->chanspec);
 
@@ -1875,7 +1874,7 @@ static int ni_ai_insn_read(struct comedi_device *dev,
 				return -ETIME;
 			}
 			d += signbits;
-			data[n] = d;
+			data[n] = d & 0xffff;
 		}
 	} else if (devpriv->is_6143) {
 		for (n = 0; n < insn->n; n++) {
@@ -1887,15 +1886,15 @@ static int ni_ai_insn_read(struct comedi_device *dev,
 			 * bit to move a single 16bit stranded sample into
 			 * the FIFO.
 			 */
-			dl = 0;
+			d = 0;
 			for (i = 0; i < NI_TIMEOUT; i++) {
 				if (ni_readl(dev, NI6143_AI_FIFO_STATUS_REG) &
 				    0x01) {
 					/* Get stranded sample into FIFO */
 					ni_writel(dev, 0x01,
 						  NI6143_AI_FIFO_CTRL_REG);
-					dl = ni_readl(dev,
-						      NI6143_AI_FIFO_DATA_REG);
+					d = ni_readl(dev,
+						     NI6143_AI_FIFO_DATA_REG);
 					break;
 				}
 			}
@@ -1903,7 +1902,7 @@ static int ni_ai_insn_read(struct comedi_device *dev,
 				dev_err(dev->class_dev, "timeout\n");
 				return -ETIME;
 			}
-			data[n] = (((dl >> 16) & 0xFFFF) + signbits) & 0xFFFF;
+			data[n] = (((d >> 16) & 0xFFFF) + signbits) & 0xFFFF;
 		}
 	} else {
 		for (n = 0; n < insn->n; n++) {
@@ -1919,14 +1918,13 @@ static int ni_ai_insn_read(struct comedi_device *dev,
 				return -ETIME;
 			}
 			if (devpriv->is_m_series) {
-				dl = ni_readl(dev, NI_M_AI_FIFO_DATA_REG);
-				dl &= mask;
-				data[n] = dl;
+				d = ni_readl(dev, NI_M_AI_FIFO_DATA_REG);
+				d &= mask;
+				data[n] = d;
 			} else {
 				d = ni_readw(dev, NI_E_AI_FIFO_DATA_REG);
-				/* subtle: needs to be short addition */
 				d += signbits;
-				data[n] = d;
+				data[n] = d & 0xffff;
 			}
 		}
 	}
@@ -2729,66 +2727,36 @@ static int ni_ao_insn_write(struct comedi_device *dev,
 	return insn->n;
 }
 
-static int ni_ao_insn_config(struct comedi_device *dev,
-			     struct comedi_subdevice *s,
-			     struct comedi_insn *insn, unsigned int *data)
-{
-	const struct ni_board_struct *board = dev->board_ptr;
-	struct ni_private *devpriv = dev->private;
-	unsigned int nbytes;
-
-	switch (data[0]) {
-	case INSN_CONFIG_GET_HARDWARE_BUFFER_SIZE:
-		switch (data[1]) {
-		case COMEDI_OUTPUT:
-			nbytes = comedi_samples_to_bytes(s,
-							 board->ao_fifo_depth);
-			data[2] = 1 + nbytes;
-			if (devpriv->mite)
-				data[2] += devpriv->mite->fifo_size;
-			break;
-		case COMEDI_INPUT:
-			data[2] = 0;
-			break;
-		default:
-			return -EINVAL;
-		}
-		return 0;
-	default:
-		break;
-	}
-
-	return -EINVAL;
-}
-
-static int ni_ao_inttrig(struct comedi_device *dev,
-			 struct comedi_subdevice *s,
-			 unsigned int trig_num)
+/*
+ * Arms the AO device in preparation for a trigger event.
+ * This function also allocates and prepares a DMA channel (or FIFO if DMA is
+ * not used).  As a part of this preparation, this function preloads the DAC
+ * registers with the first values of the output stream.  This ensures that the
+ * first clock cycle after the trigger can be used for output.
+ *
+ * Note that this function _must_ happen after a user has written data to the
+ * output buffers via either mmap or write(fileno,...).
+ */
+static int ni_ao_arm(struct comedi_device *dev,
+		     struct comedi_subdevice *s)
 {
 	struct ni_private *devpriv = dev->private;
-	struct comedi_cmd *cmd = &s->async->cmd;
 	int ret;
 	int interrupt_b_bits;
 	int i;
 	static const int timeout = 1000;
 
 	/*
-	 * Require trig_num == cmd->start_arg when cmd->start_src == TRIG_INT.
-	 * For backwards compatibility, also allow trig_num == 0 when
-	 * cmd->start_src != TRIG_INT (i.e. when cmd->start_src == TRIG_EXT);
-	 * in that case, the internal trigger is being used as a pre-trigger
-	 * before the external trigger.
+	 * Prevent ao from doing things like trying to allocate the ao dma
+	 * channel multiple times.
 	 */
-	if (!(trig_num == cmd->start_arg ||
-	      (trig_num == 0 && cmd->start_src != TRIG_INT)))
+	if (!devpriv->ao_needs_arming) {
+		dev_dbg(dev->class_dev, "%s: device does not need arming!\n",
+			__func__);
 		return -EINVAL;
+	}
 
-	/*
-	 * Null trig at beginning prevent ao start trigger from executing more
-	 * than once per command (and doing things like trying to allocate the
-	 * ao dma channel multiple times).
-	 */
-	s->async->inttrig = NULL;
+	devpriv->ao_needs_arming = 0;
 
 	ni_set_bits(dev, NISTC_INTB_ENA_REG,
 		    NISTC_INTB_ENA_AO_FIFO | NISTC_INTB_ENA_AO_ERR, 0);
@@ -2840,6 +2808,75 @@ static int ni_ao_inttrig(struct comedi_device *dev,
 			   devpriv->ao_cmd1,
 		      NISTC_AO_CMD1_REG);
 
+	return 0;
+}
+
+static int ni_ao_insn_config(struct comedi_device *dev,
+			     struct comedi_subdevice *s,
+			     struct comedi_insn *insn, unsigned int *data)
+{
+	const struct ni_board_struct *board = dev->board_ptr;
+	struct ni_private *devpriv = dev->private;
+	unsigned int nbytes;
+
+	switch (data[0]) {
+	case INSN_CONFIG_GET_HARDWARE_BUFFER_SIZE:
+		switch (data[1]) {
+		case COMEDI_OUTPUT:
+			nbytes = comedi_samples_to_bytes(s,
+							 board->ao_fifo_depth);
+			data[2] = 1 + nbytes;
+			if (devpriv->mite)
+				data[2] += devpriv->mite->fifo_size;
+			break;
+		case COMEDI_INPUT:
+			data[2] = 0;
+			break;
+		default:
+			return -EINVAL;
+		}
+		return 0;
+	case INSN_CONFIG_ARM:
+		return ni_ao_arm(dev, s);
+	default:
+		break;
+	}
+
+	return -EINVAL;
+}
+
+static int ni_ao_inttrig(struct comedi_device *dev,
+			 struct comedi_subdevice *s,
+			 unsigned int trig_num)
+{
+	struct ni_private *devpriv = dev->private;
+	struct comedi_cmd *cmd = &s->async->cmd;
+	int ret;
+
+	/*
+	 * Require trig_num == cmd->start_arg when cmd->start_src == TRIG_INT.
+	 * For backwards compatibility, also allow trig_num == 0 when
+	 * cmd->start_src != TRIG_INT (i.e. when cmd->start_src == TRIG_EXT);
+	 * in that case, the internal trigger is being used as a pre-trigger
+	 * before the external trigger.
+	 */
+	if (!(trig_num == cmd->start_arg ||
+	      (trig_num == 0 && cmd->start_src != TRIG_INT)))
+		return -EINVAL;
+
+	/*
+	 * Null trig at beginning prevent ao start trigger from executing more
+	 * than once per command.
+	 */
+	s->async->inttrig = NULL;
+
+	if (devpriv->ao_needs_arming) {
+		/* only arm this device if it still needs arming */
+		ret = ni_ao_arm(dev, s);
+		if (ret)
+			return ret;
+	}
+
 	ni_stc_writew(dev, NISTC_AO_CMD2_START1_PULSE | devpriv->ao_cmd2,
 		      NISTC_AO_CMD2_REG);
 
@@ -3227,10 +3264,17 @@ static int ni_ao_cmd(struct comedi_device *dev, struct comedi_subdevice *s)
 	ni_ao_cmd_set_interrupts(dev, s);
 
 	/*
-	 * arm(ing) and star(ting) happen in ni_ao_inttrig, which _must_ be
-	 * called for ao commands since 1) TRIG_NOW is not supported and 2) DMA
-	 * must be setup and initially written to before arm/start happen.
+	 * arm(ing) must happen later so that DMA can be setup and DACs
+	 * preloaded with the actual output buffer before starting.
+	 *
+	 * start(ing) must happen _after_ arming is completed.  Starting can be
+	 * done either via ni_ao_inttrig, or via an external trigger.
+	 *
+	 * **Currently, ni_ao_inttrig will automatically attempt a call to
+	 * ni_ao_arm if the device still needs arming at that point.  This
+	 * allows backwards compatibility.
 	 */
+	devpriv->ao_needs_arming = 1;
 	return 0;
 }
 
diff --git a/drivers/staging/comedi/drivers/ni_stc.h b/drivers/staging/comedi/drivers/ni_stc.h
index 1966519cb6e5..f27b545f83eb 100644
--- a/drivers/staging/comedi/drivers/ni_stc.h
+++ b/drivers/staging/comedi/drivers/ni_stc.h
@@ -1053,6 +1053,20 @@ struct ni_private {
 	unsigned int is_67xx:1;
 	unsigned int is_6711:1;
 	unsigned int is_6713:1;
+
+	/*
+	 * Boolean value of whether device needs to be armed.
+	 *
+	 * Currently, only NI AO devices are known to be needing arming, since
+	 * the DAC registers must be preloaded before triggering.
+	 * This variable should only be set true during a command operation
+	 * (e.g ni_ao_cmd) and should then be set false by the arming
+	 * function (e.g. ni_ao_arm).
+	 *
+	 * This variable helps to ensure that multiple DMA allocations are not
+	 * possible.
+	 */
+	unsigned int ao_needs_arming:1;
 };
 
 static const struct comedi_lrange range_ni_E_ao_ext;
diff --git a/drivers/staging/comedi/drivers/ni_tio.c b/drivers/staging/comedi/drivers/ni_tio.c
index 5ab49a798164..15cb4088467b 100644
--- a/drivers/staging/comedi/drivers/ni_tio.c
+++ b/drivers/staging/comedi/drivers/ni_tio.c
@@ -452,8 +452,9 @@ static void ni_tio_set_sync_mode(struct ni_gpct *counter)
 	unsigned int bits = 0;
 	unsigned int reg;
 	unsigned int mode;
-	unsigned int clk_src;
-	u64 ps;
+	unsigned int clk_src = 0;
+	u64 ps = 0;
+	int ret;
 	bool force_alt_sync;
 
 	/* only m series and 660x variants have counting mode registers */
@@ -483,9 +484,12 @@ static void ni_tio_set_sync_mode(struct ni_gpct *counter)
 		break;
 	}
 
-	ni_tio_generic_clock_src_select(counter, &clk_src);
-	ni_tio_clock_period_ps(counter, clk_src, &ps);
-
+	ret = ni_tio_generic_clock_src_select(counter, &clk_src);
+	if (ret)
+		return;
+	ret = ni_tio_clock_period_ps(counter, clk_src, &ps);
+	if (ret)
+		return;
 	/*
 	 * It's not clear what we should do if clock_period is unknown, so we
 	 * are not using the alt sync bit in that case.
@@ -809,7 +813,7 @@ static int ni_tio_get_clock_src(struct ni_gpct *counter,
 				unsigned int *clock_source,
 				unsigned int *period_ns)
 {
-	u64 temp64;
+	u64 temp64 = 0;
 	int ret;
 
 	ret = ni_tio_generic_clock_src_select(counter, clock_source);
diff --git a/drivers/staging/comedi/drivers/pcl818.c b/drivers/staging/comedi/drivers/pcl818.c
index 5aeed44dff70..5b5df0596ad9 100644
--- a/drivers/staging/comedi/drivers/pcl818.c
+++ b/drivers/staging/comedi/drivers/pcl818.c
@@ -771,9 +771,9 @@ static int pcl818_ai_cancel(struct comedi_device *dev,
 		     s->async->scans_done < cmd->stop_arg)) {
 			if (!devpriv->ai_cmd_canceled) {
 				/*
-				* Wait for running dma transfer to end,
-				* do cleanup in interrupt.
-				*/
+				 * Wait for running dma transfer to end,
+				 * do cleanup in interrupt.
+				 */
 				devpriv->ai_cmd_canceled = 1;
 				return 0;
 			}
diff --git a/drivers/staging/comedi/drivers/s626.c b/drivers/staging/comedi/drivers/s626.c
index c14a02564432..0dd5fe286855 100644
--- a/drivers/staging/comedi/drivers/s626.c
+++ b/drivers/staging/comedi/drivers/s626.c
@@ -75,24 +75,24 @@ struct s626_buffer_dma {
 };
 
 struct s626_private {
-	uint8_t ai_cmd_running;		/* ai_cmd is running */
+	u8 ai_cmd_running;		/* ai_cmd is running */
 	unsigned int ai_sample_timer;	/* time between samples in
 					 * units of the timer */
 	int ai_convert_count;		/* conversion counter */
 	unsigned int ai_convert_timer;	/* time between conversion in
 					 * units of the timer */
-	uint16_t counter_int_enabs;	/* counter interrupt enable mask
+	u16 counter_int_enabs;	        /* counter interrupt enable mask
 					 * for MISC2 register */
-	uint8_t adc_items;		/* number of items in ADC poll list */
+	u8 adc_items;		        /* number of items in ADC poll list */
 	struct s626_buffer_dma rps_buf;	/* DMA buffer used to hold ADC (RPS1)
 					 * program */
 	struct s626_buffer_dma ana_buf;	/* DMA buffer used to receive ADC data
 					 * and hold DAC data */
-	uint32_t *dac_wbuf;		/* pointer to logical adrs of DMA buffer
+	u32 *dac_wbuf;		        /* pointer to logical adrs of DMA buffer
 					 * used to hold DAC data */
-	uint16_t dacpol;		/* image of DAC polarity register */
-	uint8_t trim_setpoint[12];	/* images of TrimDAC setpoints */
-	uint32_t i2c_adrs;		/* I2C device address for onboard EEPROM
+	u16 dacpol;		        /* image of DAC polarity register */
+	u8 trim_setpoint[12];	        /* images of TrimDAC setpoints */
+	u32 i2c_adrs;		        /* I2C device address for onboard EEPROM
 					 * (board rev dependent) */
 };
 
@@ -179,7 +179,7 @@ static void s626_debi_transfer(struct comedi_device *dev)
 /*
  * Read a value from a gate array register.
  */
-static uint16_t s626_debi_read(struct comedi_device *dev, uint16_t addr)
+static u16 s626_debi_read(struct comedi_device *dev, u16 addr)
 {
 	/* Set up DEBI control register value in shadow RAM */
 	writel(S626_DEBI_CMD_RDWORD | addr, dev->mmio + S626_P_DEBICMD);
@@ -193,8 +193,8 @@ static uint16_t s626_debi_read(struct comedi_device *dev, uint16_t addr)
 /*
  * Write a value to a gate array register.
  */
-static void s626_debi_write(struct comedi_device *dev, uint16_t addr,
-			    uint16_t wdata)
+static void s626_debi_write(struct comedi_device *dev, u16 addr,
+			    u16 wdata)
 {
 	/* Set up DEBI control register value in shadow RAM */
 	writel(S626_DEBI_CMD_WRWORD | addr, dev->mmio + S626_P_DEBICMD);
@@ -241,7 +241,7 @@ static int s626_i2c_handshake_eoc(struct comedi_device *dev,
 	return -EBUSY;
 }
 
-static int s626_i2c_handshake(struct comedi_device *dev, uint32_t val)
+static int s626_i2c_handshake(struct comedi_device *dev, u32 val)
 {
 	unsigned int ctrl;
 	int ret;
@@ -267,8 +267,8 @@ static int s626_i2c_handshake(struct comedi_device *dev, uint32_t val)
 	return ctrl & S626_I2C_ERR;
 }
 
-/* Read uint8_t from EEPROM. */
-static uint8_t s626_i2c_read(struct comedi_device *dev, uint8_t addr)
+/* Read u8 from EEPROM. */
+static u8 s626_i2c_read(struct comedi_device *dev, u8 addr)
 {
 	struct s626_private *devpriv = dev->private;
 
@@ -304,10 +304,10 @@ static uint8_t s626_i2c_read(struct comedi_device *dev, uint8_t addr)
 /* ***********  DAC FUNCTIONS *********** */
 
 /* TrimDac LogicalChan-to-PhysicalChan mapping table. */
-static const uint8_t s626_trimchan[] = { 10, 9, 8, 3, 2, 7, 6, 1, 0, 5, 4 };
+static const u8 s626_trimchan[] = { 10, 9, 8, 3, 2, 7, 6, 1, 0, 5, 4 };
 
 /* TrimDac LogicalChan-to-EepromAdrs mapping table. */
-static const uint8_t s626_trimadrs[] = {
+static const u8 s626_trimadrs[] = {
 	0x40, 0x41, 0x42, 0x50, 0x51, 0x52, 0x53, 0x60, 0x61, 0x62, 0x63
 };
 
@@ -357,7 +357,7 @@ static int s626_send_dac_eoc(struct comedi_device *dev,
  * channel 2.  Assumes: (1) TSL2 slot records initialized, and (2)
  * dacpol contains valid target image.
  */
-static int s626_send_dac(struct comedi_device *dev, uint32_t val)
+static int s626_send_dac(struct comedi_device *dev, u32 val)
 {
 	struct s626_private *devpriv = dev->private;
 	int ret;
@@ -516,12 +516,12 @@ static int s626_send_dac(struct comedi_device *dev, uint32_t val)
  * Private helper function: Write setpoint to an application DAC channel.
  */
 static int s626_set_dac(struct comedi_device *dev,
-			uint16_t chan, int16_t dacdata)
+			u16 chan, int16_t dacdata)
 {
 	struct s626_private *devpriv = dev->private;
-	uint16_t signmask;
-	uint32_t ws_image;
-	uint32_t val;
+	u16 signmask;
+	u32 ws_image;
+	u32 val;
 
 	/*
 	 * Adjust DAC data polarity and set up Polarity Control Register image.
@@ -535,7 +535,7 @@ static int s626_set_dac(struct comedi_device *dev,
 	}
 
 	/* Limit DAC setpoint value to valid range. */
-	if ((uint16_t)dacdata > 0x1FFF)
+	if ((u16)dacdata > 0x1FFF)
 		dacdata = 0x1FFF;
 
 	/*
@@ -575,23 +575,23 @@ static int s626_set_dac(struct comedi_device *dev,
 				 * (write to non-existent trimdac). */
 	val |= 0x00004000;	/* Address the two main dual-DAC devices
 				 * (TSL's chip select enables target device). */
-	val |= ((uint32_t)(chan & 1) << 15);	/* Address the DAC channel
+	val |= ((u32)(chan & 1) << 15);	/* Address the DAC channel
 						 * within the device. */
-	val |= (uint32_t)dacdata;	/* Include DAC setpoint data. */
+	val |= (u32)dacdata;	/* Include DAC setpoint data. */
 	return s626_send_dac(dev, val);
 }
 
 static int s626_write_trim_dac(struct comedi_device *dev,
-			       uint8_t logical_chan, uint8_t dac_data)
+			       u8 logical_chan, u8 dac_data)
 {
 	struct s626_private *devpriv = dev->private;
-	uint32_t chan;
+	u32 chan;
 
 	/*
 	 * Save the new setpoint in case the application needs to read it back
 	 * later.
 	 */
-	devpriv->trim_setpoint[logical_chan] = (uint8_t)dac_data;
+	devpriv->trim_setpoint[logical_chan] = (u8)dac_data;
 
 	/* Map logical channel number to physical channel number. */
 	chan = s626_trimchan[logical_chan];
@@ -633,7 +633,7 @@ static int s626_write_trim_dac(struct comedi_device *dev,
 
 static int s626_load_trim_dacs(struct comedi_device *dev)
 {
-	uint8_t i;
+	u8 i;
 	int ret;
 
 	/* Copy TrimDac setpoint values from EEPROM to TrimDacs. */
@@ -661,7 +661,7 @@ static int s626_load_trim_dacs(struct comedi_device *dev)
  * latches B.
  */
 static void s626_set_latch_source(struct comedi_device *dev,
-				  unsigned int chan, uint16_t value)
+				  unsigned int chan, u16 value)
 {
 	s626_debi_replace(dev, S626_LP_CRB(chan),
 			  ~(S626_CRBMSK_INTCTRL | S626_CRBMSK_LATCHSRC),
@@ -672,7 +672,7 @@ static void s626_set_latch_source(struct comedi_device *dev,
  * Write value into counter preload register.
  */
 static void s626_preload(struct comedi_device *dev,
-			 unsigned int chan, uint32_t value)
+			 unsigned int chan, u32 value)
 {
 	s626_debi_write(dev, S626_LP_CNTR(chan), value);
 	s626_debi_write(dev, S626_LP_CNTR(chan) + 2, value >> 16);
@@ -686,7 +686,7 @@ static void s626_preload(struct comedi_device *dev,
 static void s626_reset_cap_flags(struct comedi_device *dev,
 				 unsigned int chan)
 {
-	uint16_t set;
+	u16 set;
 
 	set = S626_SET_CRB_INTRESETCMD(1);
 	if (chan < 3)
@@ -704,12 +704,12 @@ static void s626_reset_cap_flags(struct comedi_device *dev,
  * ClkPol, ClkEnab, IndexSrc, IndexPol, LoadSrc.
  */
 static void s626_set_mode_a(struct comedi_device *dev,
-			    unsigned int chan, uint16_t setup,
-			    uint16_t disable_int_src)
+			    unsigned int chan, u16 setup,
+			    u16 disable_int_src)
 {
 	struct s626_private *devpriv = dev->private;
-	uint16_t cra;
-	uint16_t crb;
+	u16 cra;
+	u16 crb;
 	unsigned int cntsrc, clkmult, clkpol;
 
 	/* Initialize CRA and CRB images. */
@@ -782,12 +782,12 @@ static void s626_set_mode_a(struct comedi_device *dev,
 }
 
 static void s626_set_mode_b(struct comedi_device *dev,
-			    unsigned int chan, uint16_t setup,
-			    uint16_t disable_int_src)
+			    unsigned int chan, u16 setup,
+			    u16 disable_int_src)
 {
 	struct s626_private *devpriv = dev->private;
-	uint16_t cra;
-	uint16_t crb;
+	u16 cra;
+	u16 crb;
 	unsigned int cntsrc, clkmult, clkpol;
 
 	/* Initialize CRA and CRB images. */
@@ -868,7 +868,7 @@ static void s626_set_mode_b(struct comedi_device *dev,
 
 static void s626_set_mode(struct comedi_device *dev,
 			  unsigned int chan,
-			  uint16_t setup, uint16_t disable_int_src)
+			  u16 setup, u16 disable_int_src)
 {
 	if (chan < 3)
 		s626_set_mode_a(dev, chan, setup, disable_int_src);
@@ -880,7 +880,7 @@ static void s626_set_mode(struct comedi_device *dev,
  * Return/set a counter's enable.  enab: 0=always enabled, 1=enabled by index.
  */
 static void s626_set_enable(struct comedi_device *dev,
-			    unsigned int chan, uint16_t enab)
+			    unsigned int chan, u16 enab)
 {
 	unsigned int mask = S626_CRBMSK_INTCTRL;
 	unsigned int set;
@@ -901,11 +901,11 @@ static void s626_set_enable(struct comedi_device *dev,
  * 2=OverflowA (B counters only), 3=disabled.
  */
 static void s626_set_load_trig(struct comedi_device *dev,
-			       unsigned int chan, uint16_t trig)
+			       unsigned int chan, u16 trig)
 {
-	uint16_t reg;
-	uint16_t mask;
-	uint16_t set;
+	u16 reg;
+	u16 mask;
+	u16 set;
 
 	if (chan < 3) {
 		reg = S626_LP_CRA(chan);
@@ -925,11 +925,11 @@ static void s626_set_load_trig(struct comedi_device *dev,
  * 2=IndexOnly, 3=IndexAndOverflow.
  */
 static void s626_set_int_src(struct comedi_device *dev,
-			     unsigned int chan, uint16_t int_source)
+			     unsigned int chan, u16 int_source)
 {
 	struct s626_private *devpriv = dev->private;
-	uint16_t cra_reg = S626_LP_CRA(chan);
-	uint16_t crb_reg = S626_LP_CRB(chan);
+	u16 cra_reg = S626_LP_CRA(chan);
+	u16 crb_reg = S626_LP_CRB(chan);
 
 	if (chan < 3) {
 		/* Reset any pending counter overflow or index captures */
@@ -941,7 +941,7 @@ static void s626_set_int_src(struct comedi_device *dev,
 		s626_debi_replace(dev, cra_reg, ~S626_CRAMSK_INTSRC_A,
 				  S626_SET_CRA_INTSRC_A(int_source));
 	} else {
-		uint16_t crb;
+		u16 crb;
 
 		/* Cache writeable CRB register image */
 		crb = s626_debi_read(dev, crb_reg);
@@ -985,7 +985,7 @@ static void s626_pulse_index(struct comedi_device *dev,
 			     unsigned int chan)
 {
 	if (chan < 3) {
-		uint16_t cra;
+		u16 cra;
 
 		cra = s626_debi_read(dev, S626_LP_CRA(chan));
 
@@ -994,7 +994,7 @@ static void s626_pulse_index(struct comedi_device *dev,
 				(cra ^ S626_CRAMSK_INDXPOL_A));
 		s626_debi_write(dev, S626_LP_CRA(chan), cra);
 	} else {
-		uint16_t crb;
+		u16 crb;
 
 		crb = s626_debi_read(dev, S626_LP_CRB(chan));
 		crb &= ~S626_CRBMSK_INTCTRL;
@@ -1062,7 +1062,7 @@ static int s626_dio_clear_irq(struct comedi_device *dev)
 }
 
 static void s626_handle_dio_interrupt(struct comedi_device *dev,
-				      uint16_t irqbit, uint8_t group)
+				      u16 irqbit, u8 group)
 {
 	struct s626_private *devpriv = dev->private;
 	struct comedi_subdevice *s = dev->read_subdev;
@@ -1110,8 +1110,8 @@ static void s626_handle_dio_interrupt(struct comedi_device *dev,
 
 static void s626_check_dio_interrupts(struct comedi_device *dev)
 {
-	uint16_t irqbit;
-	uint8_t group;
+	u16 irqbit;
+	u8 group;
 
 	for (group = 0; group < S626_DIO_BANKS; group++) {
 		/* read interrupt type */
@@ -1131,7 +1131,7 @@ static void s626_check_counter_interrupts(struct comedi_device *dev)
 	struct comedi_subdevice *s = dev->read_subdev;
 	struct comedi_async *async = s->async;
 	struct comedi_cmd *cmd = &async->cmd;
-	uint16_t irqbit;
+	u16 irqbit;
 
 	/* read interrupt type */
 	irqbit = s626_debi_read(dev, S626_LP_RDMISC2);
@@ -1196,7 +1196,7 @@ static bool s626_handle_eos_interrupt(struct comedi_device *dev)
 	 * first uint16_t in the buffer because it contains junk data
 	 * from the final ADC of the previous poll list scan.
 	 */
-	uint32_t *readaddr = (uint32_t *)devpriv->ana_buf.logical_base + 1;
+	u32 *readaddr = (u32 *)devpriv->ana_buf.logical_base + 1;
 	int i;
 
 	/* get the data and hand it over to comedi */
@@ -1231,7 +1231,7 @@ static irqreturn_t s626_irq_handler(int irq, void *d)
 {
 	struct comedi_device *dev = d;
 	unsigned long flags;
-	uint32_t irqtype, irqstatus;
+	u32 irqtype, irqstatus;
 
 	if (!dev->attached)
 		return IRQ_NONE;
@@ -1272,25 +1272,25 @@ static irqreturn_t s626_irq_handler(int irq, void *d)
 /*
  * This function builds the RPS program for hardware driven acquisition.
  */
-static void s626_reset_adc(struct comedi_device *dev, uint8_t *ppl)
+static void s626_reset_adc(struct comedi_device *dev, u8 *ppl)
 {
 	struct s626_private *devpriv = dev->private;
 	struct comedi_subdevice *s = dev->read_subdev;
 	struct comedi_cmd *cmd = &s->async->cmd;
-	uint32_t *rps;
-	uint32_t jmp_adrs;
-	uint16_t i;
-	uint16_t n;
-	uint32_t local_ppl;
+	u32 *rps;
+	u32 jmp_adrs;
+	u16 i;
+	u16 n;
+	u32 local_ppl;
 
 	/* Stop RPS program in case it is currently running */
 	s626_mc_disable(dev, S626_MC1_ERPS1, S626_P_MC1);
 
 	/* Set starting logical address to write RPS commands. */
-	rps = (uint32_t *)devpriv->rps_buf.logical_base;
+	rps = (u32 *)devpriv->rps_buf.logical_base;
 
 	/* Initialize RPS instruction pointer */
-	writel((uint32_t)devpriv->rps_buf.physical_base,
+	writel((u32)devpriv->rps_buf.physical_base,
 	       dev->mmio + S626_P_RPSADDR1);
 
 	/* Construct RPS program in rps_buf DMA buffer */
@@ -1372,8 +1372,8 @@ static void s626_reset_adc(struct comedi_device *dev, uint8_t *ppl)
 		 * flushes the RPS' instruction prefetch pipeline.
 		 */
 		jmp_adrs =
-			(uint32_t)devpriv->rps_buf.physical_base +
-			(uint32_t)((unsigned long)rps -
+			(u32)devpriv->rps_buf.physical_base +
+			(u32)((unsigned long)rps -
 				   (unsigned long)devpriv->
 						  rps_buf.logical_base);
 		for (i = 0; i < (10 * S626_RPSCLK_PER_US / 2); i++) {
@@ -1408,7 +1408,7 @@ static void s626_reset_adc(struct comedi_device *dev, uint8_t *ppl)
 		/* Transfer ADC data from FB BUFFER 1 register to DMA buffer. */
 		*rps++ = S626_RPS_STREG |
 			 (S626_BUGFIX_STREG(S626_P_FB_BUFFER1) >> 2);
-		*rps++ = (uint32_t)devpriv->ana_buf.physical_base +
+		*rps++ = (u32)devpriv->ana_buf.physical_base +
 			 (devpriv->adc_items << 2);
 
 		/*
@@ -1452,7 +1452,7 @@ static void s626_reset_adc(struct comedi_device *dev, uint8_t *ppl)
 
 	/* Transfer final ADC data from FB BUFFER 1 register to DMA buffer. */
 	*rps++ = S626_RPS_STREG | (S626_BUGFIX_STREG(S626_P_FB_BUFFER1) >> 2);
-	*rps++ = (uint32_t)devpriv->ana_buf.physical_base +
+	*rps++ = (u32)devpriv->ana_buf.physical_base +
 		 (devpriv->adc_items << 2);
 
 	/* Indicate ADC scan loop is finished. */
@@ -1465,7 +1465,7 @@ static void s626_reset_adc(struct comedi_device *dev, uint8_t *ppl)
 
 	/* Restart RPS program at its beginning. */
 	*rps++ = S626_RPS_JUMP;	/* Branch to start of RPS program. */
-	*rps++ = (uint32_t)devpriv->rps_buf.physical_base;
+	*rps++ = (u32)devpriv->rps_buf.physical_base;
 
 	/* End of RPS program build */
 }
@@ -1488,11 +1488,11 @@ static int s626_ai_insn_read(struct comedi_device *dev,
 			     struct comedi_insn *insn,
 			     unsigned int *data)
 {
-	uint16_t chan = CR_CHAN(insn->chanspec);
-	uint16_t range = CR_RANGE(insn->chanspec);
-	uint16_t adc_spec = 0;
-	uint32_t gpio_image;
-	uint32_t tmp;
+	u16 chan = CR_CHAN(insn->chanspec);
+	u16 range = CR_RANGE(insn->chanspec);
+	u16 adc_spec = 0;
+	u32 gpio_image;
+	u32 tmp;
 	int ret;
 	int n;
 
@@ -1585,7 +1585,7 @@ static int s626_ai_insn_read(struct comedi_device *dev,
 	return n;
 }
 
-static int s626_ai_load_polllist(uint8_t *ppl, struct comedi_cmd *cmd)
+static int s626_ai_load_polllist(u8 *ppl, struct comedi_cmd *cmd)
 {
 	int n;
 
@@ -1651,7 +1651,7 @@ static int s626_ns_to_timer(unsigned int *nanosec, unsigned int flags)
 static void s626_timer_load(struct comedi_device *dev,
 			    unsigned int chan, int tick)
 {
-	uint16_t setup =
+	u16 setup =
 		/* Preload upon index. */
 		S626_SET_STD_LOADSRC(S626_LOADSRC_INDX) |
 		/* Disable hardware index. */
@@ -1664,7 +1664,7 @@ static void s626_timer_load(struct comedi_device *dev,
 		S626_SET_STD_CLKMULT(S626_CLKMULT_1X) |
 		/* Enabled by index */
 		S626_SET_STD_CLKENAB(S626_CLKENAB_INDEX);
-	uint16_t value_latchsrc = S626_LATCHSRC_A_INDXA;
+	u16 value_latchsrc = S626_LATCHSRC_A_INDXA;
 	/* uint16_t enab = S626_CLKENAB_ALWAYS; */
 
 	s626_set_mode(dev, chan, setup, false);
@@ -1693,7 +1693,7 @@ static void s626_timer_load(struct comedi_device *dev,
 static int s626_ai_cmd(struct comedi_device *dev, struct comedi_subdevice *s)
 {
 	struct s626_private *devpriv = dev->private;
-	uint8_t ppl[16];
+	u8 ppl[16];
 	struct comedi_cmd *cmd = &s->async->cmd;
 	int tick;
 
@@ -1953,7 +1953,7 @@ static int s626_ao_insn_write(struct comedi_device *dev,
 
 static void s626_dio_init(struct comedi_device *dev)
 {
-	uint16_t group;
+	u16 group;
 
 	/* Prepare to treat writes to WRCapSel as capture disables. */
 	s626_debi_write(dev, S626_LP_MISC1, S626_MISC1_NOEDCAP);
@@ -2017,7 +2017,7 @@ static int s626_enc_insn_config(struct comedi_device *dev,
 				struct comedi_insn *insn, unsigned int *data)
 {
 	unsigned int chan = CR_CHAN(insn->chanspec);
-	uint16_t setup =
+	u16 setup =
 		/* Preload upon index. */
 		S626_SET_STD_LOADSRC(S626_LOADSRC_INDX) |
 		/* Disable hardware index. */
@@ -2032,8 +2032,8 @@ static int s626_enc_insn_config(struct comedi_device *dev,
 		S626_SET_STD_CLKENAB(S626_CLKENAB_INDEX);
 	/* uint16_t disable_int_src = true; */
 	/* uint32_t Preloadvalue;              //Counter initial value */
-	uint16_t value_latchsrc = S626_LATCHSRC_AB_READ;
-	uint16_t enab = S626_CLKENAB_ALWAYS;
+	u16 value_latchsrc = S626_LATCHSRC_AB_READ;
+	u16 enab = S626_CLKENAB_ALWAYS;
 
 	/* (data==NULL) ? (Preloadvalue=0) : (Preloadvalue=data[0]); */
 
@@ -2052,7 +2052,7 @@ static int s626_enc_insn_read(struct comedi_device *dev,
 			      unsigned int *data)
 {
 	unsigned int chan = CR_CHAN(insn->chanspec);
-	uint16_t cntr_latch_reg = S626_LP_CNTR(chan);
+	u16 cntr_latch_reg = S626_LP_CNTR(chan);
 	int i;
 
 	for (i = 0; i < insn->n; i++) {
@@ -2090,7 +2090,7 @@ static int s626_enc_insn_write(struct comedi_device *dev,
 	return 1;
 }
 
-static void s626_write_misc2(struct comedi_device *dev, uint16_t new_image)
+static void s626_write_misc2(struct comedi_device *dev, u16 new_image)
 {
 	s626_debi_write(dev, S626_LP_MISC1, S626_MISC1_WENABLE);
 	s626_debi_write(dev, S626_LP_WRMISC2, new_image);
@@ -2100,7 +2100,7 @@ static void s626_write_misc2(struct comedi_device *dev, uint16_t new_image)
 static void s626_counters_init(struct comedi_device *dev)
 {
 	int chan;
-	uint16_t setup =
+	u16 setup =
 		/* Preload upon index. */
 		S626_SET_STD_LOADSRC(S626_LOADSRC_INDX) |
 		/* Disable hardware index. */
@@ -2169,7 +2169,7 @@ static int s626_initialize(struct comedi_device *dev)
 {
 	struct s626_private *devpriv = dev->private;
 	dma_addr_t phys_buf;
-	uint16_t chan;
+	u16 chan;
 	int i;
 	int ret;
 
@@ -2248,7 +2248,7 @@ static int s626_initialize(struct comedi_device *dev)
 	 */
 
 	/* Physical start of RPS program */
-	writel((uint32_t)devpriv->rps_buf.physical_base,
+	writel((u32)devpriv->rps_buf.physical_base,
 	       dev->mmio + S626_P_RPSADDR1);
 	/* RPS program performs no explicit mem writes */
 	writel(0, dev->mmio + S626_P_RPSPAGE1);
@@ -2318,16 +2318,16 @@ static int s626_initialize(struct comedi_device *dev)
 	 * enabled.
 	 */
 	phys_buf = devpriv->ana_buf.physical_base +
-		   (S626_DAC_WDMABUF_OS * sizeof(uint32_t));
-	writel((uint32_t)phys_buf, dev->mmio + S626_P_BASEA2_OUT);
-	writel((uint32_t)(phys_buf + sizeof(uint32_t)),
+		   (S626_DAC_WDMABUF_OS * sizeof(u32));
+	writel((u32)phys_buf, dev->mmio + S626_P_BASEA2_OUT);
+	writel((u32)(phys_buf + sizeof(u32)),
 	       dev->mmio + S626_P_PROTA2_OUT);
 
 	/*
 	 * Cache Audio2's output DMA buffer logical address.  This is
 	 * where DAC data is buffered for A2 output DMA transfers.
 	 */
-	devpriv->dac_wbuf = (uint32_t *)devpriv->ana_buf.logical_base +
+	devpriv->dac_wbuf = (u32 *)devpriv->ana_buf.logical_base +
 			    S626_DAC_WDMABUF_OS;
 
 	/*
diff --git a/drivers/staging/comedi/kcomedilib/kcomedilib_main.c b/drivers/staging/comedi/kcomedilib/kcomedilib_main.c
index d0a8a28edd36..55d43c076b1c 100644
--- a/drivers/staging/comedi/kcomedilib/kcomedilib_main.c
+++ b/drivers/staging/comedi/kcomedilib/kcomedilib_main.c
@@ -250,3 +250,15 @@ int comedi_get_n_channels(struct comedi_device *dev, unsigned int subdevice)
 	return n;
 }
 EXPORT_SYMBOL_GPL(comedi_get_n_channels);
+
+static int __init kcomedilib_module_init(void)
+{
+	return 0;
+}
+
+static void __exit kcomedilib_module_exit(void)
+{
+}
+
+module_init(kcomedilib_module_init);
+module_exit(kcomedilib_module_exit);
diff --git a/drivers/staging/dgnc/Makefile b/drivers/staging/dgnc/Makefile
index 995c874f40eb..40ff0d007695 100644
--- a/drivers/staging/dgnc/Makefile
+++ b/drivers/staging/dgnc/Makefile
@@ -2,5 +2,4 @@ obj-$(CONFIG_DGNC) += dgnc.o
 
 dgnc-objs :=   dgnc_cls.o dgnc_driver.o\
                dgnc_mgmt.o dgnc_neo.o\
-               dgnc_tty.o dgnc_sysfs.o\
-	       dgnc_utils.o
+               dgnc_tty.o dgnc_utils.o
diff --git a/drivers/staging/dgnc/dgnc_cls.c b/drivers/staging/dgnc/dgnc_cls.c
index aedca66cbe41..c20ffdd254d8 100644
--- a/drivers/staging/dgnc/dgnc_cls.c
+++ b/drivers/staging/dgnc/dgnc_cls.c
@@ -385,9 +385,8 @@ static void cls_copy_data_from_uart_to_queue(struct channel_t *ch)
 		ch->ch_rxcount++;
 	}
 
-	/*
-	 * Write new final heads to channel structure.
-	 */
+	/* Write new final heads to channel structure. */
+
 	ch->ch_r_head = head & RQUEUEMASK;
 	ch->ch_e_head = head & EQUEUEMASK;
 
@@ -666,9 +665,8 @@ static void cls_param(struct tty_struct *tty)
 	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
 		return;
 
-	/*
-	 * If baud rate is zero, flush queues, and set mval to drop DTR.
-	 */
+	/* If baud rate is zero, flush queues, and set mval to drop DTR. */
+
 	if ((ch->ch_c_cflag & (CBAUD)) == 0) {
 		ch->ch_r_head = 0;
 		ch->ch_r_tail = 0;
@@ -887,9 +885,8 @@ static void cls_param(struct tty_struct *tty)
 	cls_parse_modem(ch, readb(&ch->ch_cls_uart->msr));
 }
 
-/*
- * Our board poller function.
- */
+/* Our board poller function. */
+
 static void cls_tasklet(unsigned long data)
 {
 	struct dgnc_board *bd = (struct dgnc_board *)data;
@@ -914,9 +911,8 @@ static void cls_tasklet(unsigned long data)
 	 */
 	spin_lock_irqsave(&bd->bd_intr_lock, flags);
 
-	/*
-	 * If board is ready, parse deeper to see if there is anything to do.
-	 */
+	/* If board is ready, parse deeper to see if there is anything to do. */
+
 	if ((state == BOARD_READY) && (ports > 0)) {
 		/* Loop on each port */
 		for (i = 0; i < ports; i++) {
@@ -938,9 +934,8 @@ static void cls_tasklet(unsigned long data)
 			cls_copy_data_from_queue_to_uart(ch);
 			dgnc_wakeup_writes(ch);
 
-			/*
-			 * Check carrier function.
-			 */
+			/* Check carrier function. */
+
 			dgnc_carrier(ch);
 
 			/*
@@ -992,9 +987,8 @@ static irqreturn_t cls_intr(int irq, void *voidbrd)
 	for (i = 0; i < brd->nasync; i++)
 		cls_parse_isr(brd, i);
 
-	/*
-	 * Schedule tasklet to more in-depth servicing at a better time.
-	 */
+	/* Schedule tasklet to more in-depth servicing at a better time. */
+
 	tasklet_schedule(&brd->helper_tasklet);
 
 	spin_unlock_irqrestore(&brd->bd_intr_lock, flags);
@@ -1043,9 +1037,7 @@ static int cls_drain(struct tty_struct *tty, uint seconds)
 	un->un_flags |= UN_EMPTY;
 	spin_unlock_irqrestore(&ch->ch_lock, flags);
 
-	/*
-	 * NOTE: Do something with time passed in.
-	 */
+	/* NOTE: Do something with time passed in. */
 
 	/* If ret is non-zero, user ctrl-c'ed us */
 
@@ -1112,9 +1104,8 @@ static void cls_uart_init(struct channel_t *ch)
 	readb(&ch->ch_cls_uart->msr);
 }
 
-/*
- * Turns off UART.
- */
+/* Turns off UART.  */
+
 static void cls_uart_off(struct channel_t *ch)
 {
 	writeb(0, &ch->ch_cls_uart->ier);
@@ -1160,9 +1151,8 @@ static void cls_send_break(struct channel_t *ch, int msecs)
 	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
 		return;
 
-	/*
-	 * If we receive a time of 0, this means turn off the break.
-	 */
+	/* If we receive a time of 0, this means turn off the break. */
+
 	if (msecs == 0) {
 		/* Turn break off, and unset some variables */
 		if (ch->ch_flags & CH_BREAK_SENDING) {
diff --git a/drivers/staging/dgnc/dgnc_cls.h b/drivers/staging/dgnc/dgnc_cls.h
index 2597e36d38c4..463ad30efb3b 100644
--- a/drivers/staging/dgnc/dgnc_cls.h
+++ b/drivers/staging/dgnc/dgnc_cls.h
@@ -69,7 +69,7 @@ struct cls_uart_struct {
 #define UART_EXAR654_EFR_IXON     0x2     /* Receiver compares Xon1/Xoff1 */
 #define UART_EXAR654_EFR_IXOFF    0x8     /* Transmit Xon1/Xoff1 */
 #define UART_EXAR654_EFR_RTSDTR   0x40    /* Auto RTS/DTR Flow Control Enable */
-#define UART_EXAR654_EFR_CTSDSR   0x80    /* Auto CTS/DSR Flow COntrol Enable */
+#define UART_EXAR654_EFR_CTSDSR   0x80    /* Auto CTS/DSR Flow Control Enable */
 #define UART_EXAR654_IER_XOFF     0x20    /* Xoff Interrupt Enable */
 #define UART_EXAR654_IER_RTSDTR   0x40    /* Output Interrupt Enable */
 #define UART_EXAR654_IER_CTSDSR   0x80    /* Input Interrupt Enable */
diff --git a/drivers/staging/dgnc/dgnc_driver.c b/drivers/staging/dgnc/dgnc_driver.c
index fd372d3afa46..5381dbddd8bb 100644
--- a/drivers/staging/dgnc/dgnc_driver.c
+++ b/drivers/staging/dgnc/dgnc_driver.c
@@ -24,31 +24,14 @@
 #include "dgnc_tty.h"
 #include "dgnc_cls.h"
 #include "dgnc_neo.h"
-#include "dgnc_sysfs.h"
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Digi International, http://www.digi.com");
 MODULE_DESCRIPTION("Driver for the Digi International Neo and Classic PCI based product line");
 MODULE_SUPPORTED_DEVICE("dgnc");
 
-/**************************************************************************
- *
- * protos for this file
- *
- */
-static int		dgnc_start(void);
-static int dgnc_request_irq(struct dgnc_board *brd);
-static void dgnc_free_irq(struct dgnc_board *brd);
-static struct dgnc_board *dgnc_found_board(struct pci_dev *pdev, int id);
-static void		dgnc_cleanup_board(struct dgnc_board *brd);
-static void		dgnc_poll_handler(ulong dummy);
-static int		dgnc_init_one(struct pci_dev *pdev,
-				      const struct pci_device_id *ent);
-static int		dgnc_do_remap(struct dgnc_board *brd);
+/* File operations permitted on Control/Management major. */
 
-/*
- * File operations permitted on Control/Management major.
- */
 static const struct file_operations dgnc_board_fops = {
 	.owner		=	THIS_MODULE,
 	.unlocked_ioctl =	dgnc_mgmt_ioctl,
@@ -56,9 +39,8 @@ static const struct file_operations dgnc_board_fops = {
 	.release	=	dgnc_mgmt_close
 };
 
-/*
- * Globals
- */
+/* Globals */
+
 uint			dgnc_num_boards;
 struct dgnc_board		*dgnc_board[MAXBOARDS];
 DEFINE_SPINLOCK(dgnc_global_lock);
@@ -66,14 +48,12 @@ DEFINE_SPINLOCK(dgnc_poll_lock); /* Poll scheduling lock */
 uint			dgnc_major;
 int			dgnc_poll_tick = 20;	/* Poll interval - 20 ms */
 
-/*
- * Static vars.
- */
+/* Static vars. */
+
 static struct class *dgnc_class;
 
-/*
- * Poller stuff
- */
+/* Poller stuff */
+
 static ulong		dgnc_poll_time; /* Time of next poll */
 static uint		dgnc_poll_stop; /* Used to tell poller to stop */
 static struct timer_list dgnc_poll_timer;
@@ -93,7 +73,7 @@ struct board_id {
 	unsigned int is_pci_express;
 };
 
-static struct board_id dgnc_ids[] = {
+static const struct board_id dgnc_ids[] = {
 	{	PCI_DEVICE_CLASSIC_4_PCI_NAME,		4,	0	},
 	{	PCI_DEVICE_CLASSIC_4_422_PCI_NAME,	4,	0	},
 	{	PCI_DEVICE_CLASSIC_8_PCI_NAME,		8,	0	},
@@ -114,274 +94,20 @@ static struct board_id dgnc_ids[] = {
 	{	NULL,					0,	0	}
 };
 
-static struct pci_driver dgnc_driver = {
-	.name		= "dgnc",
-	.probe		= dgnc_init_one,
-	.id_table       = dgnc_pci_tbl,
-};
-
-/************************************************************************
- *
- * Driver load/unload functions
- *
- ************************************************************************/
-
-static void cleanup(bool sysfiles)
-{
-	int i;
-	unsigned long flags;
-
-	spin_lock_irqsave(&dgnc_poll_lock, flags);
-	dgnc_poll_stop = 1;
-	spin_unlock_irqrestore(&dgnc_poll_lock, flags);
-
-	/* Turn off poller right away. */
-	del_timer_sync(&dgnc_poll_timer);
-
-	if (sysfiles)
-		dgnc_remove_driver_sysfiles(&dgnc_driver);
-
-	device_destroy(dgnc_class, MKDEV(dgnc_major, 0));
-	class_destroy(dgnc_class);
-	unregister_chrdev(dgnc_major, "dgnc");
-
-	for (i = 0; i < dgnc_num_boards; ++i) {
-		dgnc_remove_ports_sysfiles(dgnc_board[i]);
-		dgnc_cleanup_tty(dgnc_board[i]);
-		dgnc_cleanup_board(dgnc_board[i]);
-	}
-
-	dgnc_tty_post_uninit();
-}
-
-/*
- * dgnc_cleanup_module()
- *
- * Module unload.  This is where it all ends.
- */
-static void __exit dgnc_cleanup_module(void)
-{
-	cleanup(true);
-	pci_unregister_driver(&dgnc_driver);
-}
-
-/*
- * init_module()
- *
- * Module load.  This is where it all starts.
- */
-static int __init dgnc_init_module(void)
-{
-	int rc;
-
-	/*
-	 * Initialize global stuff
-	 */
-	rc = dgnc_start();
-
-	if (rc < 0)
-		return rc;
-
-	/*
-	 * Find and configure all the cards
-	 */
-	rc = pci_register_driver(&dgnc_driver);
-	if (rc) {
-		pr_warn("WARNING: dgnc driver load failed.  No Digi Neo or Classic boards found.\n");
-		cleanup(false);
-		return rc;
-	}
-	dgnc_create_driver_sysfiles(&dgnc_driver);
-
-	return 0;
-}
-
-module_init(dgnc_init_module);
-module_exit(dgnc_cleanup_module);
+/* Remap PCI memory. */
 
-/*
- * Start of driver.
- */
-static int dgnc_start(void)
+static int dgnc_do_remap(struct dgnc_board *brd)
 {
 	int rc = 0;
-	unsigned long flags;
-	struct device *dev;
-
-	/* make sure timer is initialized before we do anything else */
-	init_timer(&dgnc_poll_timer);
-
-	/*
-	 * Register our base character device into the kernel.
-	 * This allows the download daemon to connect to the downld device
-	 * before any of the boards are init'ed.
-	 *
-	 * Register management/dpa devices
-	 */
-	rc = register_chrdev(0, "dgnc", &dgnc_board_fops);
-	if (rc < 0) {
-		pr_err(DRVSTR ": Can't register dgnc driver device (%d)\n", rc);
-		return rc;
-	}
-	dgnc_major = rc;
-
-	dgnc_class = class_create(THIS_MODULE, "dgnc_mgmt");
-	if (IS_ERR(dgnc_class)) {
-		rc = PTR_ERR(dgnc_class);
-		pr_err(DRVSTR ": Can't create dgnc_mgmt class (%d)\n", rc);
-		goto failed_class;
-	}
-
-	dev = device_create(dgnc_class, NULL,
-			    MKDEV(dgnc_major, 0),
-			NULL, "dgnc_mgmt");
-	if (IS_ERR(dev)) {
-		rc = PTR_ERR(dev);
-		pr_err(DRVSTR ": Can't create device (%d)\n", rc);
-		goto failed_device;
-	}
-
-	/*
-	 * Init any global tty stuff.
-	 */
-	rc = dgnc_tty_preinit();
-
-	if (rc < 0) {
-		pr_err(DRVSTR ": tty preinit - not enough memory (%d)\n", rc);
-		goto failed_tty;
-	}
-
-	/* Start the poller */
-	spin_lock_irqsave(&dgnc_poll_lock, flags);
-	setup_timer(&dgnc_poll_timer, dgnc_poll_handler, 0);
-	dgnc_poll_time = jiffies + dgnc_jiffies_from_ms(dgnc_poll_tick);
-	dgnc_poll_timer.expires = dgnc_poll_time;
-	spin_unlock_irqrestore(&dgnc_poll_lock, flags);
-
-	add_timer(&dgnc_poll_timer);
-
-	return 0;
-
-failed_tty:
-	device_destroy(dgnc_class, MKDEV(dgnc_major, 0));
-failed_device:
-	class_destroy(dgnc_class);
-failed_class:
-	unregister_chrdev(dgnc_major, "dgnc");
-	return rc;
-}
-
-/* returns count (>= 0), or negative on error */
-static int dgnc_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
-{
-	int rc;
-	struct dgnc_board *brd;
-
-	/* wake up and enable device */
-	rc = pci_enable_device(pdev);
-
-	if (rc)
-		return -EIO;
-
-	brd = dgnc_found_board(pdev, ent->driver_data);
-	if (IS_ERR(brd))
-		return PTR_ERR(brd);
-
-	/*
-	 * Do tty device initialization.
-	 */
-
-	rc = dgnc_tty_register(brd);
-	if (rc < 0) {
-		pr_err(DRVSTR ": Can't register tty devices (%d)\n", rc);
-		goto failed;
-	}
-
-	rc = dgnc_request_irq(brd);
-	if (rc < 0) {
-		pr_err(DRVSTR ": Can't finalize board init (%d)\n", rc);
-		goto unregister_tty;
-	}
-
-	rc = dgnc_tty_init(brd);
-	if (rc < 0) {
-		pr_err(DRVSTR ": Can't init tty devices (%d)\n", rc);
-		goto free_irq;
-	}
-
-	brd->state = BOARD_READY;
-	brd->dpastatus = BD_RUNNING;
 
-	dgnc_create_ports_sysfiles(brd);
-
-	dgnc_board[dgnc_num_boards++] = brd;
-
-	return 0;
-
-free_irq:
-	dgnc_free_irq(brd);
-unregister_tty:
-	dgnc_tty_unregister(brd);
-
-failed:
-	kfree(brd);
+	brd->re_map_membase = ioremap(brd->membase, 0x1000);
+	if (!brd->re_map_membase)
+		rc = -ENOMEM;
 
 	return rc;
 }
 
 /*
- * dgnc_cleanup_board()
- *
- * Free all the memory associated with a board
- */
-static void dgnc_cleanup_board(struct dgnc_board *brd)
-{
-	int i = 0;
-
-	if (!brd || brd->magic != DGNC_BOARD_MAGIC)
-		return;
-
-	switch (brd->device) {
-	case PCI_DEVICE_CLASSIC_4_DID:
-	case PCI_DEVICE_CLASSIC_8_DID:
-	case PCI_DEVICE_CLASSIC_4_422_DID:
-	case PCI_DEVICE_CLASSIC_8_422_DID:
-
-		/* Tell card not to interrupt anymore. */
-		outb(0, brd->iobase + 0x4c);
-		break;
-
-	default:
-		break;
-	}
-
-	if (brd->irq)
-		free_irq(brd->irq, brd);
-
-	tasklet_kill(&brd->helper_tasklet);
-
-	if (brd->re_map_membase) {
-		iounmap(brd->re_map_membase);
-		brd->re_map_membase = NULL;
-	}
-
-	/* Free all allocated channels structs */
-	for (i = 0; i < MAXPORTS ; i++) {
-		if (brd->channels[i]) {
-			kfree(brd->channels[i]->ch_rqueue);
-			kfree(brd->channels[i]->ch_equeue);
-			kfree(brd->channels[i]->ch_wqueue);
-			kfree(brd->channels[i]);
-			brd->channels[i] = NULL;
-		}
-	}
-
-	dgnc_board[brd->boardnum] = NULL;
-
-	kfree(brd);
-}
-
-/*
  * dgnc_found_board()
  *
  * A board has been found, init it.
@@ -587,21 +313,6 @@ static void dgnc_free_irq(struct dgnc_board *brd)
 }
 
 /*
- * Remap PCI memory.
- */
-static int dgnc_do_remap(struct dgnc_board *brd)
-{
-	int rc = 0;
-
-	brd->re_map_membase = ioremap(brd->membase, 0x1000);
-	if (!brd->re_map_membase)
-		rc = -ENOMEM;
-
-	return rc;
-}
-
-/*
- *
  * Function:
  *
  *    dgnc_poll_handler
@@ -623,7 +334,6 @@ static int dgnc_do_remap(struct dgnc_board *brd)
  *    As each timer expires, it determines (a) whether the "transmit"
  *    waiter needs to be woken up, and (b) whether the poller needs to
  *    be rescheduled.
- *
  */
 
 static void dgnc_poll_handler(ulong dummy)
@@ -651,9 +361,8 @@ static void dgnc_poll_handler(ulong dummy)
 		spin_unlock_irqrestore(&brd->bd_lock, flags);
 	}
 
-	/*
-	 * Schedule ourself back at the nominal wakeup interval.
-	 */
+	/* Schedule ourself back at the nominal wakeup interval. */
+
 	spin_lock_irqsave(&dgnc_poll_lock, flags);
 	dgnc_poll_time += dgnc_jiffies_from_ms(dgnc_poll_tick);
 
@@ -669,3 +378,240 @@ static void dgnc_poll_handler(ulong dummy)
 	if (!dgnc_poll_stop)
 		add_timer(&dgnc_poll_timer);
 }
+
+/* returns count (>= 0), or negative on error */
+static int dgnc_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	int rc;
+	struct dgnc_board *brd;
+
+	/* wake up and enable device */
+	rc = pci_enable_device(pdev);
+
+	if (rc)
+		return -EIO;
+
+	brd = dgnc_found_board(pdev, ent->driver_data);
+	if (IS_ERR(brd))
+		return PTR_ERR(brd);
+
+	/* Do tty device initialization. */
+
+	rc = dgnc_tty_register(brd);
+	if (rc < 0) {
+		pr_err(DRVSTR ": Can't register tty devices (%d)\n", rc);
+		goto failed;
+	}
+
+	rc = dgnc_request_irq(brd);
+	if (rc < 0) {
+		pr_err(DRVSTR ": Can't finalize board init (%d)\n", rc);
+		goto unregister_tty;
+	}
+
+	rc = dgnc_tty_init(brd);
+	if (rc < 0) {
+		pr_err(DRVSTR ": Can't init tty devices (%d)\n", rc);
+		goto free_irq;
+	}
+
+	brd->state = BOARD_READY;
+	brd->dpastatus = BD_RUNNING;
+
+	dgnc_board[dgnc_num_boards++] = brd;
+
+	return 0;
+
+free_irq:
+	dgnc_free_irq(brd);
+unregister_tty:
+	dgnc_tty_unregister(brd);
+
+failed:
+	kfree(brd);
+
+	return rc;
+}
+
+static struct pci_driver dgnc_driver = {
+	.name		= "dgnc",
+	.probe		= dgnc_init_one,
+	.id_table       = dgnc_pci_tbl,
+};
+
+/* Start of driver. */
+
+static int dgnc_start(void)
+{
+	int rc = 0;
+	unsigned long flags;
+	struct device *dev;
+
+	/* make sure timer is initialized before we do anything else */
+	init_timer(&dgnc_poll_timer);
+
+	/*
+	 * Register our base character device into the kernel.
+	 * This allows the download daemon to connect to the downld device
+	 * before any of the boards are init'ed.
+	 *
+	 * Register management/dpa devices
+	 */
+	rc = register_chrdev(0, "dgnc", &dgnc_board_fops);
+	if (rc < 0) {
+		pr_err(DRVSTR ": Can't register dgnc driver device (%d)\n", rc);
+		return rc;
+	}
+	dgnc_major = rc;
+
+	dgnc_class = class_create(THIS_MODULE, "dgnc_mgmt");
+	if (IS_ERR(dgnc_class)) {
+		rc = PTR_ERR(dgnc_class);
+		pr_err(DRVSTR ": Can't create dgnc_mgmt class (%d)\n", rc);
+		goto failed_class;
+	}
+
+	dev = device_create(dgnc_class, NULL,
+			    MKDEV(dgnc_major, 0),
+			NULL, "dgnc_mgmt");
+	if (IS_ERR(dev)) {
+		rc = PTR_ERR(dev);
+		pr_err(DRVSTR ": Can't create device (%d)\n", rc);
+		goto failed_device;
+	}
+
+	/* Start the poller */
+	spin_lock_irqsave(&dgnc_poll_lock, flags);
+	setup_timer(&dgnc_poll_timer, dgnc_poll_handler, 0);
+	dgnc_poll_time = jiffies + dgnc_jiffies_from_ms(dgnc_poll_tick);
+	dgnc_poll_timer.expires = dgnc_poll_time;
+	spin_unlock_irqrestore(&dgnc_poll_lock, flags);
+
+	add_timer(&dgnc_poll_timer);
+
+	return 0;
+
+failed_device:
+	class_destroy(dgnc_class);
+failed_class:
+	unregister_chrdev(dgnc_major, "dgnc");
+	return rc;
+}
+
+/*
+ * dgnc_cleanup_board()
+ *
+ * Free all the memory associated with a board
+ */
+static void dgnc_cleanup_board(struct dgnc_board *brd)
+{
+	int i = 0;
+
+	if (!brd || brd->magic != DGNC_BOARD_MAGIC)
+		return;
+
+	switch (brd->device) {
+	case PCI_DEVICE_CLASSIC_4_DID:
+	case PCI_DEVICE_CLASSIC_8_DID:
+	case PCI_DEVICE_CLASSIC_4_422_DID:
+	case PCI_DEVICE_CLASSIC_8_422_DID:
+
+		/* Tell card not to interrupt anymore. */
+		outb(0, brd->iobase + 0x4c);
+		break;
+
+	default:
+		break;
+	}
+
+	if (brd->irq)
+		free_irq(brd->irq, brd);
+
+	tasklet_kill(&brd->helper_tasklet);
+
+	if (brd->re_map_membase) {
+		iounmap(brd->re_map_membase);
+		brd->re_map_membase = NULL;
+	}
+
+	/* Free all allocated channels structs */
+	for (i = 0; i < MAXPORTS ; i++) {
+		if (brd->channels[i]) {
+			kfree(brd->channels[i]->ch_rqueue);
+			kfree(brd->channels[i]->ch_equeue);
+			kfree(brd->channels[i]->ch_wqueue);
+			kfree(brd->channels[i]);
+			brd->channels[i] = NULL;
+		}
+	}
+
+	dgnc_board[brd->boardnum] = NULL;
+
+	kfree(brd);
+}
+
+/* Driver load/unload functions */
+
+static void cleanup(void)
+{
+	int i;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dgnc_poll_lock, flags);
+	dgnc_poll_stop = 1;
+	spin_unlock_irqrestore(&dgnc_poll_lock, flags);
+
+	/* Turn off poller right away. */
+	del_timer_sync(&dgnc_poll_timer);
+
+	device_destroy(dgnc_class, MKDEV(dgnc_major, 0));
+	class_destroy(dgnc_class);
+	unregister_chrdev(dgnc_major, "dgnc");
+
+	for (i = 0; i < dgnc_num_boards; ++i) {
+		dgnc_cleanup_tty(dgnc_board[i]);
+		dgnc_cleanup_board(dgnc_board[i]);
+	}
+}
+
+/*
+ * dgnc_cleanup_module()
+ *
+ * Module unload.  This is where it all ends.
+ */
+static void __exit dgnc_cleanup_module(void)
+{
+	cleanup();
+	pci_unregister_driver(&dgnc_driver);
+}
+
+/*
+ * init_module()
+ *
+ * Module load.  This is where it all starts.
+ */
+static int __init dgnc_init_module(void)
+{
+	int rc;
+
+	/* Initialize global stuff */
+
+	rc = dgnc_start();
+
+	if (rc < 0)
+		return rc;
+
+	/* Find and configure all the cards */
+
+	rc = pci_register_driver(&dgnc_driver);
+	if (rc) {
+		pr_warn("WARNING: dgnc driver load failed.  No Digi Neo or Classic boards found.\n");
+		cleanup();
+		return rc;
+	}
+
+	return 0;
+}
+
+module_init(dgnc_init_module);
+module_exit(dgnc_cleanup_module);
diff --git a/drivers/staging/dgnc/dgnc_driver.h b/drivers/staging/dgnc/dgnc_driver.h
index 879202663a98..c8119f2fe881 100644
--- a/drivers/staging/dgnc/dgnc_driver.h
+++ b/drivers/staging/dgnc/dgnc_driver.h
@@ -12,11 +12,8 @@
  * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
  * PURPOSE.  See the GNU General Public License for more details.
  *
- *************************************************************************
- *
  * Driver includes
- *
- *************************************************************************/
+ */
 
 #ifndef __DGNC_DRIVER_H
 #define __DGNC_DRIVER_H
@@ -26,19 +23,14 @@
 #include <linux/interrupt.h>
 
 #include "digi.h"		/* Digi specific ioctl header */
-#include "dgnc_sysfs.h"		/* Support for SYSFS */
 
-/*************************************************************************
- *
- * Driver defines
- *
- *************************************************************************/
+/* Driver defines */
 
-/* Driver identification and error statments */
-#define	PROCSTR		"dgnc"			/* /proc entries	 */
-#define	DEVSTR		"/dev/dg/dgnc"		/* /dev entries		 */
-#define	DRVSTR		"dgnc"			/* Driver name string	 */
-#define	DG_PART		"40002369_F"		/* RPM part number	 */
+/* Driver identification and error statements */
+#define	PROCSTR		"dgnc"			/* /proc entries */
+#define	DEVSTR		"/dev/dg/dgnc"		/* /dev entries */
+#define	DRVSTR		"dgnc"			/* Driver name string */
+#define	DG_PART		"40002369_F"		/* RPM part number */
 
 #define TRC_TO_CONSOLE 1
 
@@ -61,7 +53,8 @@
 #define PORT_NUM(dev)	((dev) & 0x7f)
 #define IS_PRINT(dev)	(((dev) & 0xff) >= 0x80)
 
-/* MAX number of stop characters we will send
+/*
+ *MAX number of stop characters we will send
  * when our read queue is getting full
  */
 #define MAX_STOPS_SENT 5
@@ -88,35 +81,28 @@
 #define   _POSIX_VDISABLE '\0'
 #endif
 
-/*
- * All the possible states the driver can be while being loaded.
- */
+/* All the possible states the driver can be while being loaded. */
+
 enum {
 	DRIVER_INITIALIZED = 0,
 	DRIVER_READY
 };
 
-/*
- * All the possible states the board can be while booting up.
- */
+/* All the possible states the board can be while booting up. */
+
 enum {
 	BOARD_FAILED = 0,
 	BOARD_FOUND,
 	BOARD_READY
 };
 
-/*************************************************************************
- *
- * Structures and closely related defines.
- *
- *************************************************************************/
+/* Structures and closely related defines. */
 
 struct dgnc_board;
 struct channel_t;
 
-/************************************************************************
- * Per board operations structure				       *
- ************************************************************************/
+/* Per board operations structure */
+
 struct board_ops {
 	void (*tasklet)(unsigned long data);
 	irqreturn_t (*intr)(int irq, void *voidbrd);
@@ -138,16 +124,14 @@ struct board_ops {
 	void (*send_immediate_char)(struct channel_t *ch, unsigned char);
 };
 
-/************************************************************************
- * Device flag definitions for bd_flags.
- ************************************************************************/
+/* Device flag definitions for bd_flags. */
+
 #define BD_IS_PCI_EXPRESS     0x0001	  /* Is a PCI Express board */
 
-/*
- *	Per-board information
- */
+/*	Per-board information */
+
 struct dgnc_board {
-	int		magic;		/* Board Magic number.  */
+	int		magic;		/* Board Magic number. */
 	int		boardnum;	/* Board number: 0-32 */
 
 	int		type;		/* Type of board */
@@ -220,62 +204,56 @@ struct dgnc_board {
 
 };
 
-/************************************************************************
- * Unit flag definitions for un_flags.
- ************************************************************************/
-#define UN_ISOPEN	0x0001		/* Device is open		*/
-#define UN_CLOSING	0x0002		/* Line is being closed		*/
-#define UN_IMM		0x0004		/* Service immediately		*/
-#define UN_BUSY		0x0008		/* Some work this channel	*/
-#define UN_BREAKI	0x0010		/* Input break received		*/
+/* Unit flag definitions for un_flags. */
+#define UN_ISOPEN	0x0001		/* Device is open */
+#define UN_CLOSING	0x0002		/* Line is being closed	*/
+#define UN_IMM		0x0004		/* Service immediately */
+#define UN_BUSY		0x0008		/* Some work this channel */
+#define UN_BREAKI	0x0010		/* Input break received	*/
 #define UN_PWAIT	0x0020		/* Printer waiting for terminal	*/
-#define UN_TIME		0x0040		/* Waiting on time		*/
-#define UN_EMPTY	0x0080		/* Waiting output queue empty	*/
+#define UN_TIME		0x0040		/* Waiting on time */
+#define UN_EMPTY	0x0080		/* Waiting output queue empty */
 #define UN_LOW		0x0100		/* Waiting output low water mark*/
-#define UN_EXCL_OPEN	0x0200		/* Open for exclusive use	*/
-#define UN_WOPEN	0x0400		/* Device waiting for open	*/
-#define UN_WIOCTL	0x0800		/* Device waiting for open	*/
-#define UN_HANGUP	0x8000		/* Carrier lost			*/
+#define UN_EXCL_OPEN	0x0200		/* Open for exclusive use */
+#define UN_WOPEN	0x0400		/* Device waiting for open */
+#define UN_WIOCTL	0x0800		/* Device waiting for open */
+#define UN_HANGUP	0x8000		/* Carrier lost	*/
 
 struct device;
 
-/************************************************************************
- * Structure for terminal or printer unit.
- ************************************************************************/
+/* Structure for terminal or printer unit. */
 struct un_t {
-	int	magic;		/* Unit Magic Number.			*/
+	int	magic;		/* Unit Magic Number. */
 	struct	channel_t *un_ch;
 	ulong	un_time;
 	uint	un_type;
-	uint	un_open_count;	/* Counter of opens to port		*/
-	struct tty_struct *un_tty;/* Pointer to unit tty structure	*/
-	uint	un_flags;	/* Unit flags				*/
+	uint	un_open_count;		/* Counter of opens to port */
+	struct tty_struct *un_tty;	/* Pointer to unit tty structure */
+	uint	un_flags;		/* Unit flags */
 	wait_queue_head_t un_flags_wait; /* Place to sleep to wait on unit */
-	uint	un_dev;		/* Minor device number			*/
+	uint	un_dev;			/* Minor device number */
 	struct device *un_sysfs;
 };
 
-/************************************************************************
- * Device flag definitions for ch_flags.
- ************************************************************************/
-#define CH_PRON		0x0001		/* Printer on string		*/
-#define CH_STOP		0x0002		/* Output is stopped		*/
-#define CH_STOPI	0x0004		/* Input is stopped		*/
-#define CH_CD		0x0008		/* Carrier is present		*/
-#define CH_FCAR		0x0010		/* Carrier forced on		*/
-#define CH_HANGUP       0x0020		/* Hangup received		*/
-
-#define CH_RECEIVER_OFF	0x0040		/* Receiver is off		*/
-#define CH_OPENING	0x0080		/* Port in fragile open state	*/
-#define CH_CLOSING	0x0100		/* Port in fragile close state	*/
-#define CH_FIFO_ENABLED 0x0200		/* Port has FIFOs enabled	*/
-#define CH_TX_FIFO_EMPTY 0x0400		/* TX Fifo is completely empty	*/
-#define CH_TX_FIFO_LWM  0x0800		/* TX Fifo is below Low Water	*/
-#define CH_BREAK_SENDING 0x1000		/* Break is being sent		*/
-#define CH_LOOPBACK 0x2000		/* Channel is in lookback mode	*/
+/* Device flag definitions for ch_flags. */
+#define CH_PRON		0x0001		/* Printer on string */
+#define CH_STOP		0x0002		/* Output is stopped */
+#define CH_STOPI	0x0004		/* Input is stopped */
+#define CH_CD		0x0008		/* Carrier is present */
+#define CH_FCAR		0x0010		/* Carrier forced on */
+#define CH_HANGUP       0x0020		/* Hangup received */
+
+#define CH_RECEIVER_OFF	0x0040		/* Receiver is off */
+#define CH_OPENING	0x0080		/* Port in fragile open state */
+#define CH_CLOSING	0x0100		/* Port in fragile close state */
+#define CH_FIFO_ENABLED 0x0200		/* Port has FIFOs enabled */
+#define CH_TX_FIFO_EMPTY 0x0400		/* TX Fifo is completely empty */
+#define CH_TX_FIFO_LWM  0x0800		/* TX Fifo is below Low Water */
+#define CH_BREAK_SENDING 0x1000		/* Break is being sent */
+#define CH_LOOPBACK	0x2000		/* Channel is in lookback mode */
 #define CH_BAUD0	0x08000		/* Used for checking B0 transitions */
-#define CH_FORCED_STOP  0x20000		/* Output is forcibly stopped	*/
-#define CH_FORCED_STOPI 0x40000		/* Input is forcibly stopped	*/
+#define CH_FORCED_STOP  0x20000		/* Output is forcibly stopped */
+#define CH_FORCED_STOPI 0x40000		/* Input is forcibly stopped */
 
 /* Our Read/Error/Write queue sizes */
 #define RQUEUEMASK	0x1FFF		/* 8 K - 1 */
@@ -285,43 +263,41 @@ struct un_t {
 #define EQUEUESIZE	RQUEUESIZE
 #define WQUEUESIZE	(WQUEUEMASK + 1)
 
-/************************************************************************
- * Channel information structure.
- ************************************************************************/
+/* Channel information structure. */
 struct channel_t {
-	int magic;			/* Channel Magic Number		*/
-	struct dgnc_board	*ch_bd;		/* Board structure pointer */
+	int magic;			/* Channel Magic Number	*/
+	struct dgnc_board *ch_bd;	/* Board structure pointer */
 	struct digi_t	ch_digi;	/* Transparent Print structure  */
-	struct un_t	ch_tun;		/* Terminal unit info	   */
-	struct un_t	ch_pun;		/* Printer unit info	    */
+	struct un_t	ch_tun;		/* Terminal unit info */
+	struct un_t	ch_pun;		/* Printer unit info */
 
 	spinlock_t	ch_lock;	/* provide for serialization */
 	wait_queue_head_t ch_flags_wait;
 
-	uint		ch_portnum;	/* Port number, 0 offset.	*/
-	uint		ch_open_count;	/* open count			*/
-	uint		ch_flags;	/* Channel flags		*/
+	uint		ch_portnum;	/* Port number, 0 offset. */
+	uint		ch_open_count;	/* open count */
+	uint		ch_flags;	/* Channel flags */
 
 	ulong		ch_close_delay;	/* How long we should
 					 * drop RTS/DTR for
 					 */
 
-	ulong		ch_cpstime;	/* Time for CPS calculations    */
+	ulong		ch_cpstime;	/* Time for CPS calculations */
 
-	tcflag_t	ch_c_iflag;	/* channel iflags	       */
-	tcflag_t	ch_c_cflag;	/* channel cflags	       */
-	tcflag_t	ch_c_oflag;	/* channel oflags	       */
-	tcflag_t	ch_c_lflag;	/* channel lflags	       */
-	unsigned char	ch_stopc;	/* Stop character	       */
-	unsigned char	ch_startc;	/* Start character	      */
+	tcflag_t	ch_c_iflag;	/* channel iflags */
+	tcflag_t	ch_c_cflag;	/* channel cflags */
+	tcflag_t	ch_c_oflag;	/* channel oflags */
+	tcflag_t	ch_c_lflag;	/* channel lflags */
+	unsigned char	ch_stopc;	/* Stop character */
+	unsigned char	ch_startc;	/* Start character */
 
 	uint		ch_old_baud;	/* Cache of the current baud */
 	uint		ch_custom_speed;/* Custom baud, if set */
 
 	uint		ch_wopen;	/* Waiting for open process cnt */
 
-	unsigned char		ch_mostat;	/* FEP output modem status */
-	unsigned char		ch_mistat;	/* FEP input modem status */
+	unsigned char	ch_mostat;	/* FEP output modem status */
+	unsigned char	ch_mistat;	/* FEP input modem status */
 
 	struct neo_uart_struct __iomem *ch_neo_uart;	/* Pointer to the
 							 * "mapped" UART struct
@@ -347,10 +323,10 @@ struct channel_t {
 	ulong		ch_rxcount;	/* total of data received so far */
 	ulong		ch_txcount;	/* total of data transmitted so far */
 
-	unsigned char		ch_r_tlevel;	/* Receive Trigger level */
-	unsigned char		ch_t_tlevel;	/* Transmit Trigger level */
+	unsigned char	ch_r_tlevel;	/* Receive Trigger level */
+	unsigned char	ch_t_tlevel;	/* Transmit Trigger level */
 
-	unsigned char		ch_r_watermark;	/* Receive Watermark */
+	unsigned char	ch_r_watermark;	/* Receive Watermark */
 
 	ulong		ch_stop_sending_break;	/* Time we should STOP
 						 * sending a break
@@ -374,16 +350,15 @@ struct channel_t {
 
 };
 
-/*
- * Our Global Variables.
- */
+/* Our Global Variables. */
+
 extern uint		dgnc_major;		/* Our driver/mgmt major */
 extern int		dgnc_poll_tick;		/* Poll interval - 20 ms */
 extern spinlock_t	dgnc_global_lock;	/* Driver global spinlock */
 extern spinlock_t	dgnc_poll_lock;		/* Poll scheduling lock */
 extern uint		dgnc_num_boards;	/* Total number of boards */
-extern struct dgnc_board	*dgnc_board[MAXBOARDS];	/* Array of board
-							 * structs
-							 */
+extern struct dgnc_board *dgnc_board[MAXBOARDS];/* Array of board
+						 * structs
+						 */
 
 #endif
diff --git a/drivers/staging/dgnc/dgnc_mgmt.c b/drivers/staging/dgnc/dgnc_mgmt.c
index 683c098391d9..9d9b15d6358a 100644
--- a/drivers/staging/dgnc/dgnc_mgmt.c
+++ b/drivers/staging/dgnc/dgnc_mgmt.c
@@ -13,13 +13,11 @@
  * PURPOSE.  See the GNU General Public License for more details.
  */
 
-/************************************************************************
- *
+/*
  * This file implements the mgmt functionality for the
  * Neo and ClassicBoard based product lines.
- *
- ************************************************************************
  */
+
 #include <linux/kernel.h>
 #include <linux/ctype.h>
 #include <linux/sched.h>	/* For jiffies, task states */
diff --git a/drivers/staging/dgnc/dgnc_neo.c b/drivers/staging/dgnc/dgnc_neo.c
index 5becb3741b67..3eefefe53174 100644
--- a/drivers/staging/dgnc/dgnc_neo.c
+++ b/drivers/staging/dgnc/dgnc_neo.c
@@ -107,7 +107,8 @@ static inline void neo_set_cts_flow_control(struct channel_t *ch)
 	/* Turn off auto Xon flow control */
 	efr &= ~UART_17158_EFR_IXON;
 
-	/* Why? Because Exar's spec says we have to zero it
+	/*
+	 * Why? Because Exar's spec says we have to zero it
 	 * out before setting it
 	 */
 	writeb(0, &ch->ch_neo_uart->efr);
@@ -145,7 +146,8 @@ static inline void neo_set_rts_flow_control(struct channel_t *ch)
 	ier &= ~UART_17158_IER_XOFF;
 	efr &= ~UART_17158_EFR_IXOFF;
 
-	/* Why? Because Exar's spec says we have to zero it
+	/*
+	 * Why? Because Exar's spec says we have to zero it
 	 * out before setting it
 	 */
 	writeb(0, &ch->ch_neo_uart->efr);
@@ -185,7 +187,8 @@ static inline void neo_set_ixon_flow_control(struct channel_t *ch)
 	/* Turn on auto Xon flow control */
 	efr |= (UART_17158_EFR_ECB | UART_17158_EFR_IXON);
 
-	/* Why? Because Exar's spec says we have to zero it
+	/*
+	 * Why? Because Exar's spec says we have to zero it
 	 * out before setting it
 	 */
 	writeb(0, &ch->ch_neo_uart->efr);
@@ -225,7 +228,8 @@ static inline void neo_set_ixoff_flow_control(struct channel_t *ch)
 	ier |= UART_17158_IER_XOFF;
 	efr |= (UART_17158_EFR_ECB | UART_17158_EFR_IXOFF);
 
-	/* Why? Because Exar's spec says we have to zero it
+	/*
+	 * Why? Because Exar's spec says we have to zero it
 	 * out before setting it
 	 */
 	writeb(0, &ch->ch_neo_uart->efr);
@@ -268,7 +272,8 @@ static inline void neo_set_no_input_flow_control(struct channel_t *ch)
 	else
 		efr &= ~(UART_17158_EFR_ECB | UART_17158_EFR_IXOFF);
 
-	/* Why? Because Exar's spec says we have to zero
+	/*
+	 * Why? Because Exar's spec says we have to zero
 	 * it out before setting it
 	 */
 	writeb(0, &ch->ch_neo_uart->efr);
@@ -308,7 +313,8 @@ static inline void neo_set_no_output_flow_control(struct channel_t *ch)
 	else
 		efr &= ~(UART_17158_EFR_ECB | UART_17158_EFR_IXON);
 
-	/* Why? Because Exar's spec says we have to zero it
+	/*
+	 * Why? Because Exar's spec says we have to zero it
 	 * out before setting it
 	 */
 	writeb(0, &ch->ch_neo_uart->efr);
@@ -351,9 +357,8 @@ static inline void neo_set_new_start_stop_chars(struct channel_t *ch)
 	neo_pci_posting_flush(ch->ch_bd);
 }
 
-/*
- * No locks are assumed to be held when calling this function.
- */
+/* No locks are assumed to be held when calling this function. */
+
 static inline void neo_clear_break(struct channel_t *ch, int force)
 {
 	unsigned long flags;
@@ -381,9 +386,8 @@ static inline void neo_clear_break(struct channel_t *ch, int force)
 	spin_unlock_irqrestore(&ch->ch_lock, flags);
 }
 
-/*
- * Parse the ISR register.
- */
+/* Parse the ISR register. */
+
 static inline void neo_parse_isr(struct dgnc_board *brd, uint port)
 {
 	struct channel_t *ch;
@@ -412,8 +416,8 @@ static inline void neo_parse_isr(struct dgnc_board *brd, uint port)
 		if (isr & (UART_17158_IIR_RDI_TIMEOUT | UART_IIR_RDI)) {
 			/* Read data from uart -> queue */
 			neo_copy_data_from_uart_to_queue(ch);
-
-			/* Call our tty layer to enforce queue
+			/*
+			 * Call our tty layer to enforce queue
 			 * flow control if needed.
 			 */
 			spin_lock_irqsave(&ch->ch_lock, flags);
@@ -438,7 +442,8 @@ static inline void neo_parse_isr(struct dgnc_board *brd, uint port)
 			 * one it was, so we can suspend or resume data flow.
 			 */
 			if (cause == UART_17158_XON_DETECT) {
-				/* Is output stopped right now, if so,
+				/*
+				 * Is output stopped right now, if so,
 				 * resume it
 				 */
 				if (brd->channels[port]->ch_flags & CH_STOP) {
@@ -609,9 +614,8 @@ static void neo_param(struct tty_struct *tty)
 	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
 		return;
 
-	/*
-	 * If baud rate is zero, flush queues, and set mval to drop DTR.
-	 */
+	/* If baud rate is zero, flush queues, and set mval to drop DTR. */
+
 	if ((ch->ch_c_cflag & (CBAUD)) == 0) {
 		ch->ch_r_head = 0;
 		ch->ch_r_tail = 0;
@@ -672,7 +676,8 @@ static void neo_param(struct tty_struct *tty)
 				4800,   9600,   19200,  38400 }
 		};
 
-		/* Only use the TXPrint baud rate if the terminal unit
+		/*
+		 * Only use the TXPrint baud rate if the terminal unit
 		 * is NOT open
 		 */
 		if (!(ch->ch_tun.un_flags & UN_ISOPEN) &&
@@ -797,7 +802,8 @@ static void neo_param(struct tty_struct *tty)
 	if (ch->ch_digi.digi_flags & CTSPACE || ch->ch_c_cflag & CRTSCTS) {
 		neo_set_cts_flow_control(ch);
 	} else if (ch->ch_c_iflag & IXON) {
-		/* If start/stop is set to disable, then we should
+		/*
+		 * If start/stop is set to disable, then we should
 		 * disable flow control
 		 */
 		if ((ch->ch_startc == _POSIX_VDISABLE) ||
@@ -812,7 +818,8 @@ static void neo_param(struct tty_struct *tty)
 	if (ch->ch_digi.digi_flags & RTSPACE || ch->ch_c_cflag & CRTSCTS) {
 		neo_set_rts_flow_control(ch);
 	} else if (ch->ch_c_iflag & IXOFF) {
-		/* If start/stop is set to disable, then we should
+		/*
+		 * If start/stop is set to disable, then we should
 		 * disable flow control
 		 */
 		if ((ch->ch_startc == _POSIX_VDISABLE) ||
@@ -840,9 +847,8 @@ static void neo_param(struct tty_struct *tty)
 	neo_parse_modem(ch, readb(&ch->ch_neo_uart->msr));
 }
 
-/*
- * Our board poller function.
- */
+/* Our board poller function. */
+
 static void neo_tasklet(unsigned long data)
 {
 	struct dgnc_board *bd = (struct dgnc_board *)data;
@@ -867,9 +873,8 @@ static void neo_tasklet(unsigned long data)
 	 */
 	spin_lock_irqsave(&bd->bd_intr_lock, flags);
 
-	/*
-	 * If board is ready, parse deeper to see if there is anything to do.
-	 */
+	/* If board is ready, parse deeper to see if there is anything to do. */
+
 	if ((state == BOARD_READY) && (ports > 0)) {
 		/* Loop on each port */
 		for (i = 0; i < ports; i++) {
@@ -997,9 +1002,9 @@ static irqreturn_t neo_intr(int irq, void *voidbrd)
 			break;
 
 		case UART_17158_RX_LINE_STATUS:
-			/*
-			 * RXRDY and RX LINE Status (logic OR of LSR[4:1])
-			 */
+
+			/* RXRDY and RX LINE Status (logic OR of LSR[4:1]) */
+
 			neo_parse_lsr(brd, port);
 			break;
 
@@ -1022,9 +1027,9 @@ static irqreturn_t neo_intr(int irq, void *voidbrd)
 			break;
 
 		case UART_17158_MSR:
-			/*
-			 * MSR or flow control was seen.
-			 */
+
+			/* MSR or flow control was seen. */
+
 			neo_parse_isr(brd, port);
 			break;
 
@@ -1041,9 +1046,8 @@ static irqreturn_t neo_intr(int irq, void *voidbrd)
 		port++;
 	}
 
-	/*
-	 * Schedule tasklet to more in-depth servicing at a better time.
-	 */
+	/* Schedule tasklet to more in-depth servicing at a better time. */
+
 	tasklet_schedule(&brd->helper_tasklet);
 
 	spin_unlock_irqrestore(&brd->bd_intr_lock, flags);
@@ -1238,9 +1242,8 @@ static void neo_copy_data_from_uart_to_queue(struct channel_t *ch)
 			ch->ch_flags |= (CH_TX_FIFO_EMPTY | CH_TX_FIFO_LWM);
 		}
 
-		/*
-		 * Discard character if we are ignoring the error mask.
-		 */
+		/* Discard character if we are ignoring the error mask. */
+
 		if (linestatus & error_mask)  {
 			unsigned char discard;
 
@@ -1279,9 +1282,8 @@ static void neo_copy_data_from_uart_to_queue(struct channel_t *ch)
 		ch->ch_rxcount++;
 	}
 
-	/*
-	 * Write new final heads to channel structure.
-	 */
+	/* Write new final heads to channel structure. */
+
 	ch->ch_r_head = head & RQUEUEMASK;
 	ch->ch_e_head = head & EQUEUEMASK;
 
@@ -1412,9 +1414,8 @@ static void neo_copy_data_from_queue_to_uart(struct channel_t *ch)
 	    (ch->ch_flags & CH_BREAK_SENDING))
 		goto exit_unlock;
 
-	/*
-	 * If FIFOs are disabled. Send data directly to txrx register
-	 */
+	/* If FIFOs are disabled. Send data directly to txrx register */
+
 	if (!(ch->ch_flags & CH_FIFO_ENABLED)) {
 		unsigned char lsrbits = readb(&ch->ch_neo_uart->lsr);
 
@@ -1458,9 +1459,8 @@ static void neo_copy_data_from_queue_to_uart(struct channel_t *ch)
 		goto exit_unlock;
 	}
 
-	/*
-	 * We have to do it this way, because of the EXAR TXFIFO count bug.
-	 */
+	/* We have to do it this way, because of the EXAR TXFIFO count bug. */
+
 	if ((ch->ch_bd->dvid & 0xf0) < UART_XR17E158_DVID) {
 		if (!(ch->ch_flags & (CH_TX_FIFO_EMPTY | CH_TX_FIFO_LWM)))
 			goto exit_unlock;
@@ -1645,9 +1645,8 @@ static void neo_send_stop_character(struct channel_t *ch)
 	}
 }
 
-/*
- * neo_uart_init
- */
+/* neo_uart_init */
+
 static void neo_uart_init(struct channel_t *ch)
 {
 	writeb(0, &ch->ch_neo_uart->ier);
@@ -1668,9 +1667,8 @@ static void neo_uart_init(struct channel_t *ch)
 	neo_pci_posting_flush(ch->ch_bd);
 }
 
-/*
- * Make the UART completely turn off.
- */
+/* Make the UART completely turn off. */
+
 static void neo_uart_off(struct channel_t *ch)
 {
 	/* Turn off UART enhanced bits */
@@ -1705,9 +1703,8 @@ static uint neo_get_uart_bytes_left(struct channel_t *ch)
 /* Channel lock MUST be held by the calling function! */
 static void neo_send_break(struct channel_t *ch, int msecs)
 {
-	/*
-	 * If we receive a time of 0, this means turn off the break.
-	 */
+	/* If we receive a time of 0, this means turn off the break. */
+
 	if (msecs == 0) {
 		if (ch->ch_flags & CH_BREAK_SENDING) {
 			unsigned char temp = readb(&ch->ch_neo_uart->lcr);
diff --git a/drivers/staging/dgnc/dgnc_neo.h b/drivers/staging/dgnc/dgnc_neo.h
index abddd48353d0..77ecd9baae45 100644
--- a/drivers/staging/dgnc/dgnc_neo.h
+++ b/drivers/staging/dgnc/dgnc_neo.h
@@ -18,37 +18,38 @@
 
 #include "dgnc_driver.h"
 
-/************************************************************************
- * Per channel/port NEO UART structure					*
- ************************************************************************
- *		Base Structure Entries Usage Meanings to Host		*
- *									*
- *	W = read write		R = read only				*
- *			U = Unused.					*
- ************************************************************************/
+/*
+ *	Per channel/port NEO UART structure
+ *	Base Structure Entries Usage Meanings to Host
+ *
+ *	W = read write		R = read only
+ *			U = Unused.
+ */
 
 struct neo_uart_struct {
-	u8 txrx;		/* WR  RHR/THR - Holding Reg */
+	u8 txrx;	/* WR  RHR/THR - Holding Reg */
 	u8 ier;		/* WR  IER - Interrupt Enable Reg */
-	u8 isr_fcr;		/* WR  ISR/FCR - Interrupt Status Reg/Fifo Control Reg */
+	u8 isr_fcr;	/* WR  ISR/FCR - Interrupt Status Reg/Fifo
+			 * Control Reg
+			 */
 	u8 lcr;		/* WR  LCR - Line Control Reg */
 	u8 mcr;		/* WR  MCR - Modem Control Reg */
 	u8 lsr;		/* WR  LSR - Line Status Reg */
 	u8 msr;		/* WR  MSR - Modem Status Reg */
 	u8 spr;		/* WR  SPR - Scratch Pad Reg */
-	u8 fctr;		/* WR  FCTR - Feature Control Reg */
+	u8 fctr;	/* WR  FCTR - Feature Control Reg */
 	u8 efr;		/* WR  EFR - Enhanced Function Reg */
-	u8 tfifo;		/* WR  TXCNT/TXTRG - Transmit FIFO Reg */
-	u8 rfifo;		/* WR  RXCNT/RXTRG - Receive  FIFO Reg */
+	u8 tfifo;	/* WR  TXCNT/TXTRG - Transmit FIFO Reg */
+	u8 rfifo;	/* WR  RXCNT/RXTRG - Receive  FIFO Reg */
 	u8 xoffchar1;	/* WR  XOFF 1 - XOff Character 1 Reg */
 	u8 xoffchar2;	/* WR  XOFF 2 - XOff Character 2 Reg */
 	u8 xonchar1;	/* WR  XON 1 - Xon Character 1 Reg */
 	u8 xonchar2;	/* WR  XON 2 - XOn Character 2 Reg */
 
 	u8 reserved1[0x2ff - 0x200]; /* U   Reserved by Exar */
-	u8 txrxburst[64];	/* RW  64 bytes of RX/TX FIFO Data */
+	u8 txrxburst[64];	     /* RW  64 bytes of RX/TX FIFO Data */
 	u8 reserved2[0x37f - 0x340]; /* U   Reserved by Exar */
-	u8 rxburst_with_errors[64];	/* R  64 bytes of RX FIFO Data + LSR */
+	u8 rxburst_with_errors[64];  /* R  64 bytes of RX FIFO Data + LSR */
 };
 
 /* Where to read the extended interrupt register (32bits instead of 8bits) */
@@ -108,7 +109,9 @@ struct neo_uart_struct {
 /* 17158 Extended IIR's */
 #define UART_17158_IIR_RDI_TIMEOUT	0x0C	/* Receiver data TIMEOUT */
 #define UART_17158_IIR_XONXOFF		0x10	/* Received an XON/XOFF char */
-#define UART_17158_IIR_HWFLOW_STATE_CHANGE 0x20	/* CTS/DSR or RTS/DTR state change */
+#define UART_17158_IIR_HWFLOW_STATE_CHANGE 0x20	/* CTS/DSR or RTS/DTR
+						 * state change
+						 */
 #define UART_17158_IIR_FIFO_ENABLED	0xC0	/* 16550 FIFOs are Enabled */
 
 /*
@@ -119,8 +122,12 @@ struct neo_uart_struct {
 #define UART_17158_RXRDY_TIMEOUT	0x2	/* RX Ready Timeout */
 #define UART_17158_TXRDY		0x3	/* TX Ready */
 #define UART_17158_MSR			0x4	/* Modem State Change */
-#define UART_17158_TX_AND_FIFO_CLR	0x40	/* Transmitter Holding Reg Empty */
-#define UART_17158_RX_FIFO_DATA_ERROR	0x80	/* UART detected an RX FIFO Data error */
+#define UART_17158_TX_AND_FIFO_CLR	0x40	/* Transmitter Holding
+						 * Reg Empty
+						 */
+#define UART_17158_RX_FIFO_DATA_ERROR	0x80	/* UART detected an RX FIFO
+						 * Data error
+						 */
 
 /*
  * These are the EXTENDED definitions for the 17C158's Interrupt
@@ -130,19 +137,22 @@ struct neo_uart_struct {
 #define UART_17158_EFR_IXON	0x2	/* Receiver compares Xon1/Xoff1 */
 #define UART_17158_EFR_IXOFF	0x8	/* Transmit Xon1/Xoff1 */
 #define UART_17158_EFR_RTSDTR	0x40	/* Auto RTS/DTR Flow Control Enable */
-#define UART_17158_EFR_CTSDSR	0x80	/* Auto CTS/DSR Flow COntrol Enable */
+#define UART_17158_EFR_CTSDSR	0x80	/* Auto CTS/DSR Flow Control Enable */
 
-#define UART_17158_XOFF_DETECT	0x1	/* Indicates whether chip saw an incoming XOFF char  */
-#define UART_17158_XON_DETECT	0x2	/* Indicates whether chip saw an incoming XON char */
+#define UART_17158_XOFF_DETECT	0x1	/* Indicates whether chip saw an
+					 * incoming XOFF char
+					 */
+#define UART_17158_XON_DETECT	0x2	/* Indicates whether chip saw an
+					 * incoming XON char
+					 */
 
 #define UART_17158_IER_RSVD1	0x10	/* Reserved by Exar */
 #define UART_17158_IER_XOFF	0x20	/* Xoff Interrupt Enable */
 #define UART_17158_IER_RTSDTR	0x40	/* Output Interrupt Enable */
 #define UART_17158_IER_CTSDSR	0x80	/* Input Interrupt Enable */
 
-/*
- * Our Global Variables
- */
+/* Our Global Variables */
+
 extern struct board_ops dgnc_neo_ops;
 
 #endif
diff --git a/drivers/staging/dgnc/dgnc_sysfs.c b/drivers/staging/dgnc/dgnc_sysfs.c
deleted file mode 100644
index 290bf6e226ac..000000000000
--- a/drivers/staging/dgnc/dgnc_sysfs.c
+++ /dev/null
@@ -1,703 +0,0 @@
-/*
- * Copyright 2004 Digi International (www.digi.com)
- *      Scott H Kilau <Scott_Kilau at digi dot com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED; without even the
- * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
- * PURPOSE.  See the GNU General Public License for more details.
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/ctype.h>
-#include <linux/string.h>
-#include <linux/serial_reg.h>
-#include <linux/device.h>
-#include <linux/pci.h>
-#include <linux/kdev_t.h>
-
-#include "dgnc_driver.h"
-#include "dgnc_mgmt.h"
-
-static ssize_t version_show(struct device_driver *ddp, char *buf)
-{
-	return snprintf(buf, PAGE_SIZE, "%s\n", DG_PART);
-}
-static DRIVER_ATTR_RO(version);
-
-static ssize_t boards_show(struct device_driver *ddp, char *buf)
-{
-	return snprintf(buf, PAGE_SIZE, "%d\n", dgnc_num_boards);
-}
-static DRIVER_ATTR_RO(boards);
-
-static ssize_t maxboards_show(struct device_driver *ddp, char *buf)
-{
-	return snprintf(buf, PAGE_SIZE, "%d\n", MAXBOARDS);
-}
-static DRIVER_ATTR_RO(maxboards);
-
-static ssize_t pollrate_show(struct device_driver *ddp, char *buf)
-{
-	return snprintf(buf, PAGE_SIZE, "%dms\n", dgnc_poll_tick);
-}
-
-static ssize_t pollrate_store(struct device_driver *ddp,
-			      const char *buf, size_t count)
-{
-	unsigned long flags;
-	int tick;
-	int ret;
-
-	ret = sscanf(buf, "%d\n", &tick);
-	if (ret != 1)
-		return -EINVAL;
-
-	spin_lock_irqsave(&dgnc_poll_lock, flags);
-	dgnc_poll_tick = tick;
-	spin_unlock_irqrestore(&dgnc_poll_lock, flags);
-
-	return count;
-}
-static DRIVER_ATTR_RW(pollrate);
-
-void dgnc_create_driver_sysfiles(struct pci_driver *dgnc_driver)
-{
-	int rc = 0;
-	struct device_driver *driverfs = &dgnc_driver->driver;
-
-	rc |= driver_create_file(driverfs, &driver_attr_version);
-	rc |= driver_create_file(driverfs, &driver_attr_boards);
-	rc |= driver_create_file(driverfs, &driver_attr_maxboards);
-	rc |= driver_create_file(driverfs, &driver_attr_pollrate);
-	if (rc)
-		pr_err("DGNC: sysfs driver_create_file failed!\n");
-}
-
-void dgnc_remove_driver_sysfiles(struct pci_driver *dgnc_driver)
-{
-	struct device_driver *driverfs = &dgnc_driver->driver;
-
-	driver_remove_file(driverfs, &driver_attr_version);
-	driver_remove_file(driverfs, &driver_attr_boards);
-	driver_remove_file(driverfs, &driver_attr_maxboards);
-	driver_remove_file(driverfs, &driver_attr_pollrate);
-}
-
-#define DGNC_VERIFY_BOARD(p, bd)				\
-	do {							\
-		if (!p)						\
-			return 0;				\
-								\
-		bd = dev_get_drvdata(p);			\
-		if (!bd || bd->magic != DGNC_BOARD_MAGIC)	\
-			return 0;				\
-		if (bd->state != BOARD_READY)			\
-			return 0;				\
-	} while (0)
-
-static ssize_t vpd_show(struct device *p, struct device_attribute *attr,
-			char *buf)
-{
-	struct dgnc_board *bd;
-	int count = 0;
-	int i = 0;
-
-	DGNC_VERIFY_BOARD(p, bd);
-
-	count += sprintf(buf + count,
-		"\n      0  1  2  3  4  5  6  7  8  9  A  B  C  D  E  F");
-	for (i = 0; i < 0x40 * 2; i++) {
-		if (!(i % 16))
-			count += sprintf(buf + count, "\n%04X ", i * 2);
-		count += sprintf(buf + count, "%02X ", bd->vpd[i]);
-	}
-	count += sprintf(buf + count, "\n");
-
-	return count;
-}
-static DEVICE_ATTR_RO(vpd);
-
-static ssize_t serial_number_show(struct device *p,
-				  struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	int count = 0;
-
-	DGNC_VERIFY_BOARD(p, bd);
-
-	if (bd->serial_num[0] == '\0')
-		count += sprintf(buf + count, "<UNKNOWN>\n");
-	else
-		count += sprintf(buf + count, "%s\n", bd->serial_num);
-
-	return count;
-}
-static DEVICE_ATTR_RO(serial_number);
-
-static ssize_t ports_state_show(struct device *p,
-				struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	int count = 0;
-	int i = 0;
-
-	DGNC_VERIFY_BOARD(p, bd);
-
-	for (i = 0; i < bd->nasync; i++) {
-		count += snprintf(buf + count, PAGE_SIZE - count,
-			"%d %s\n", bd->channels[i]->ch_portnum,
-			bd->channels[i]->ch_open_count ? "Open" : "Closed");
-	}
-	return count;
-}
-static DEVICE_ATTR_RO(ports_state);
-
-static ssize_t ports_baud_show(struct device *p,
-			       struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	int count = 0;
-	int i = 0;
-
-	DGNC_VERIFY_BOARD(p, bd);
-
-	for (i = 0; i < bd->nasync; i++) {
-		count +=  snprintf(buf + count, PAGE_SIZE - count,
-			"%d %d\n", bd->channels[i]->ch_portnum,
-			bd->channels[i]->ch_old_baud);
-	}
-	return count;
-}
-static DEVICE_ATTR_RO(ports_baud);
-
-static ssize_t ports_msignals_show(struct device *p,
-				   struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	int count = 0;
-	int i = 0;
-
-	DGNC_VERIFY_BOARD(p, bd);
-
-	for (i = 0; i < bd->nasync; i++) {
-		struct channel_t *ch = bd->channels[i];
-
-		if (ch->ch_open_count) {
-			count += snprintf(buf + count, PAGE_SIZE - count,
-				"%d %s %s %s %s %s %s\n",
-				ch->ch_portnum,
-				(ch->ch_mostat & UART_MCR_RTS) ? "RTS" : "",
-				(ch->ch_mistat & UART_MSR_CTS) ? "CTS" : "",
-				(ch->ch_mostat & UART_MCR_DTR) ? "DTR" : "",
-				(ch->ch_mistat & UART_MSR_DSR) ? "DSR" : "",
-				(ch->ch_mistat & UART_MSR_DCD) ? "DCD" : "",
-				(ch->ch_mistat & UART_MSR_RI)  ? "RI"  : "");
-		} else {
-			count += snprintf(buf + count, PAGE_SIZE - count,
-				"%d\n", ch->ch_portnum);
-		}
-	}
-	return count;
-}
-static DEVICE_ATTR_RO(ports_msignals);
-
-static ssize_t ports_iflag_show(struct device *p,
-				struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	int count = 0;
-	int i = 0;
-
-	DGNC_VERIFY_BOARD(p, bd);
-
-	for (i = 0; i < bd->nasync; i++) {
-		count += snprintf(buf + count, PAGE_SIZE - count, "%d %x\n",
-			bd->channels[i]->ch_portnum,
-			bd->channels[i]->ch_c_iflag);
-	}
-	return count;
-}
-static DEVICE_ATTR_RO(ports_iflag);
-
-static ssize_t ports_cflag_show(struct device *p,
-				struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	int count = 0;
-	int i = 0;
-
-	DGNC_VERIFY_BOARD(p, bd);
-
-	for (i = 0; i < bd->nasync; i++) {
-		count += snprintf(buf + count, PAGE_SIZE - count, "%d %x\n",
-			bd->channels[i]->ch_portnum,
-			bd->channels[i]->ch_c_cflag);
-	}
-	return count;
-}
-static DEVICE_ATTR_RO(ports_cflag);
-
-static ssize_t ports_oflag_show(struct device *p,
-				struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	int count = 0;
-	int i = 0;
-
-	DGNC_VERIFY_BOARD(p, bd);
-
-	for (i = 0; i < bd->nasync; i++) {
-		count += snprintf(buf + count, PAGE_SIZE - count, "%d %x\n",
-			bd->channels[i]->ch_portnum,
-			bd->channels[i]->ch_c_oflag);
-	}
-	return count;
-}
-static DEVICE_ATTR_RO(ports_oflag);
-
-static ssize_t ports_lflag_show(struct device *p,
-				struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	int count = 0;
-	int i = 0;
-
-	DGNC_VERIFY_BOARD(p, bd);
-
-	for (i = 0; i < bd->nasync; i++) {
-		count += snprintf(buf + count, PAGE_SIZE - count, "%d %x\n",
-			bd->channels[i]->ch_portnum,
-			bd->channels[i]->ch_c_lflag);
-	}
-	return count;
-}
-static DEVICE_ATTR_RO(ports_lflag);
-
-static ssize_t ports_digi_flag_show(struct device *p,
-				    struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	int count = 0;
-	int i = 0;
-
-	DGNC_VERIFY_BOARD(p, bd);
-
-	for (i = 0; i < bd->nasync; i++) {
-		count += snprintf(buf + count, PAGE_SIZE - count, "%d %x\n",
-			bd->channels[i]->ch_portnum,
-			bd->channels[i]->ch_digi.digi_flags);
-	}
-	return count;
-}
-static DEVICE_ATTR_RO(ports_digi_flag);
-
-static ssize_t ports_rxcount_show(struct device *p,
-				  struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	int count = 0;
-	int i = 0;
-
-	DGNC_VERIFY_BOARD(p, bd);
-
-	for (i = 0; i < bd->nasync; i++) {
-		count += snprintf(buf + count, PAGE_SIZE - count, "%d %ld\n",
-			bd->channels[i]->ch_portnum,
-			bd->channels[i]->ch_rxcount);
-	}
-	return count;
-}
-static DEVICE_ATTR_RO(ports_rxcount);
-
-static ssize_t ports_txcount_show(struct device *p,
-				  struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	int count = 0;
-	int i = 0;
-
-	DGNC_VERIFY_BOARD(p, bd);
-
-	for (i = 0; i < bd->nasync; i++) {
-		count += snprintf(buf + count, PAGE_SIZE - count, "%d %ld\n",
-			bd->channels[i]->ch_portnum,
-			bd->channels[i]->ch_txcount);
-	}
-	return count;
-}
-static DEVICE_ATTR_RO(ports_txcount);
-
-/* this function creates the sys files that will export each signal status
- * to sysfs each value will be put in a separate filename
- */
-void dgnc_create_ports_sysfiles(struct dgnc_board *bd)
-{
-	int rc = 0;
-
-	dev_set_drvdata(&bd->pdev->dev, bd);
-	rc |= device_create_file(&bd->pdev->dev, &dev_attr_ports_state);
-	rc |= device_create_file(&bd->pdev->dev, &dev_attr_ports_baud);
-	rc |= device_create_file(&bd->pdev->dev, &dev_attr_ports_msignals);
-	rc |= device_create_file(&bd->pdev->dev, &dev_attr_ports_iflag);
-	rc |= device_create_file(&bd->pdev->dev, &dev_attr_ports_cflag);
-	rc |= device_create_file(&bd->pdev->dev, &dev_attr_ports_oflag);
-	rc |= device_create_file(&bd->pdev->dev, &dev_attr_ports_lflag);
-	rc |= device_create_file(&bd->pdev->dev, &dev_attr_ports_digi_flag);
-	rc |= device_create_file(&bd->pdev->dev, &dev_attr_ports_rxcount);
-	rc |= device_create_file(&bd->pdev->dev, &dev_attr_ports_txcount);
-	rc |= device_create_file(&bd->pdev->dev, &dev_attr_vpd);
-	rc |= device_create_file(&bd->pdev->dev, &dev_attr_serial_number);
-	if (rc)
-		dev_err(&bd->pdev->dev, "dgnc: sysfs device_create_file failed!\n");
-}
-
-/* removes all the sys files created for that port */
-void dgnc_remove_ports_sysfiles(struct dgnc_board *bd)
-{
-	device_remove_file(&bd->pdev->dev, &dev_attr_ports_state);
-	device_remove_file(&bd->pdev->dev, &dev_attr_ports_baud);
-	device_remove_file(&bd->pdev->dev, &dev_attr_ports_msignals);
-	device_remove_file(&bd->pdev->dev, &dev_attr_ports_iflag);
-	device_remove_file(&bd->pdev->dev, &dev_attr_ports_cflag);
-	device_remove_file(&bd->pdev->dev, &dev_attr_ports_oflag);
-	device_remove_file(&bd->pdev->dev, &dev_attr_ports_lflag);
-	device_remove_file(&bd->pdev->dev, &dev_attr_ports_digi_flag);
-	device_remove_file(&bd->pdev->dev, &dev_attr_ports_rxcount);
-	device_remove_file(&bd->pdev->dev, &dev_attr_ports_txcount);
-	device_remove_file(&bd->pdev->dev, &dev_attr_vpd);
-	device_remove_file(&bd->pdev->dev, &dev_attr_serial_number);
-}
-
-static ssize_t tty_state_show(struct device *d,
-			      struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	struct channel_t *ch;
-	struct un_t *un;
-
-	if (!d)
-		return 0;
-	un = dev_get_drvdata(d);
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return 0;
-	ch = un->un_ch;
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return 0;
-	bd = ch->ch_bd;
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return 0;
-	if (bd->state != BOARD_READY)
-		return 0;
-
-	return snprintf(buf, PAGE_SIZE, "%s",
-			un->un_open_count ? "Open" : "Closed");
-}
-static DEVICE_ATTR_RO(tty_state);
-
-static ssize_t tty_baud_show(struct device *d,
-			     struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	struct channel_t *ch;
-	struct un_t *un;
-
-	if (!d)
-		return 0;
-	un = dev_get_drvdata(d);
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return 0;
-	ch = un->un_ch;
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return 0;
-	bd = ch->ch_bd;
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return 0;
-	if (bd->state != BOARD_READY)
-		return 0;
-
-	return snprintf(buf, PAGE_SIZE, "%d\n", ch->ch_old_baud);
-}
-static DEVICE_ATTR_RO(tty_baud);
-
-static ssize_t tty_msignals_show(struct device *d,
-				 struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	struct channel_t *ch;
-	struct un_t *un;
-
-	if (!d)
-		return 0;
-	un = dev_get_drvdata(d);
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return 0;
-	ch = un->un_ch;
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return 0;
-	bd = ch->ch_bd;
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return 0;
-	if (bd->state != BOARD_READY)
-		return 0;
-
-	if (ch->ch_open_count) {
-		return snprintf(buf, PAGE_SIZE, "%s %s %s %s %s %s\n",
-			(ch->ch_mostat & UART_MCR_RTS) ? "RTS" : "",
-			(ch->ch_mistat & UART_MSR_CTS) ? "CTS" : "",
-			(ch->ch_mostat & UART_MCR_DTR) ? "DTR" : "",
-			(ch->ch_mistat & UART_MSR_DSR) ? "DSR" : "",
-			(ch->ch_mistat & UART_MSR_DCD) ? "DCD" : "",
-			(ch->ch_mistat & UART_MSR_RI)  ? "RI"  : "");
-	}
-	return 0;
-}
-static DEVICE_ATTR_RO(tty_msignals);
-
-static ssize_t tty_iflag_show(struct device *d,
-			      struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	struct channel_t *ch;
-	struct un_t *un;
-
-	if (!d)
-		return 0;
-	un = dev_get_drvdata(d);
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return 0;
-	ch = un->un_ch;
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return 0;
-	bd = ch->ch_bd;
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return 0;
-	if (bd->state != BOARD_READY)
-		return 0;
-
-	return snprintf(buf, PAGE_SIZE, "%x\n", ch->ch_c_iflag);
-}
-static DEVICE_ATTR_RO(tty_iflag);
-
-static ssize_t tty_cflag_show(struct device *d,
-			      struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	struct channel_t *ch;
-	struct un_t *un;
-
-	if (!d)
-		return 0;
-	un = dev_get_drvdata(d);
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return 0;
-	ch = un->un_ch;
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return 0;
-	bd = ch->ch_bd;
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return 0;
-	if (bd->state != BOARD_READY)
-		return 0;
-
-	return snprintf(buf, PAGE_SIZE, "%x\n", ch->ch_c_cflag);
-}
-static DEVICE_ATTR_RO(tty_cflag);
-
-static ssize_t tty_oflag_show(struct device *d,
-			      struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	struct channel_t *ch;
-	struct un_t *un;
-
-	if (!d)
-		return 0;
-	un = dev_get_drvdata(d);
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return 0;
-	ch = un->un_ch;
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return 0;
-	bd = ch->ch_bd;
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return 0;
-	if (bd->state != BOARD_READY)
-		return 0;
-
-	return snprintf(buf, PAGE_SIZE, "%x\n", ch->ch_c_oflag);
-}
-static DEVICE_ATTR_RO(tty_oflag);
-
-static ssize_t tty_lflag_show(struct device *d,
-			      struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	struct channel_t *ch;
-	struct un_t *un;
-
-	if (!d)
-		return 0;
-	un = dev_get_drvdata(d);
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return 0;
-	ch = un->un_ch;
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return 0;
-	bd = ch->ch_bd;
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return 0;
-	if (bd->state != BOARD_READY)
-		return 0;
-
-	return snprintf(buf, PAGE_SIZE, "%x\n", ch->ch_c_lflag);
-}
-static DEVICE_ATTR_RO(tty_lflag);
-
-static ssize_t tty_digi_flag_show(struct device *d,
-				  struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	struct channel_t *ch;
-	struct un_t *un;
-
-	if (!d)
-		return 0;
-	un = dev_get_drvdata(d);
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return 0;
-	ch = un->un_ch;
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return 0;
-	bd = ch->ch_bd;
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return 0;
-	if (bd->state != BOARD_READY)
-		return 0;
-
-	return snprintf(buf, PAGE_SIZE, "%x\n", ch->ch_digi.digi_flags);
-}
-static DEVICE_ATTR_RO(tty_digi_flag);
-
-static ssize_t tty_rxcount_show(struct device *d,
-				struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	struct channel_t *ch;
-	struct un_t *un;
-
-	if (!d)
-		return 0;
-	un = dev_get_drvdata(d);
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return 0;
-	ch = un->un_ch;
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return 0;
-	bd = ch->ch_bd;
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return 0;
-	if (bd->state != BOARD_READY)
-		return 0;
-
-	return snprintf(buf, PAGE_SIZE, "%ld\n", ch->ch_rxcount);
-}
-static DEVICE_ATTR_RO(tty_rxcount);
-
-static ssize_t tty_txcount_show(struct device *d,
-				struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	struct channel_t *ch;
-	struct un_t *un;
-
-	if (!d)
-		return 0;
-	un = dev_get_drvdata(d);
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return 0;
-	ch = un->un_ch;
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return 0;
-	bd = ch->ch_bd;
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return 0;
-	if (bd->state != BOARD_READY)
-		return 0;
-
-	return snprintf(buf, PAGE_SIZE, "%ld\n", ch->ch_txcount);
-}
-static DEVICE_ATTR_RO(tty_txcount);
-
-static ssize_t tty_custom_name_show(struct device *d,
-				    struct device_attribute *attr, char *buf)
-{
-	struct dgnc_board *bd;
-	struct channel_t *ch;
-	struct un_t *un;
-
-	if (!d)
-		return 0;
-	un = dev_get_drvdata(d);
-	if (!un || un->magic != DGNC_UNIT_MAGIC)
-		return 0;
-	ch = un->un_ch;
-	if (!ch || ch->magic != DGNC_CHANNEL_MAGIC)
-		return 0;
-	bd = ch->ch_bd;
-	if (!bd || bd->magic != DGNC_BOARD_MAGIC)
-		return 0;
-	if (bd->state != BOARD_READY)
-		return 0;
-
-	return snprintf(buf, PAGE_SIZE, "%sn%d%c\n",
-		(un->un_type == DGNC_PRINT) ? "pr" : "tty",
-		bd->boardnum + 1, 'a' + ch->ch_portnum);
-}
-static DEVICE_ATTR_RO(tty_custom_name);
-
-static struct attribute *dgnc_sysfs_tty_entries[] = {
-	&dev_attr_tty_state.attr,
-	&dev_attr_tty_baud.attr,
-	&dev_attr_tty_msignals.attr,
-	&dev_attr_tty_iflag.attr,
-	&dev_attr_tty_cflag.attr,
-	&dev_attr_tty_oflag.attr,
-	&dev_attr_tty_lflag.attr,
-	&dev_attr_tty_digi_flag.attr,
-	&dev_attr_tty_rxcount.attr,
-	&dev_attr_tty_txcount.attr,
-	&dev_attr_tty_custom_name.attr,
-	NULL
-};
-
-static const struct attribute_group dgnc_tty_attribute_group = {
-	.name = NULL,
-	.attrs = dgnc_sysfs_tty_entries,
-};
-
-void dgnc_create_tty_sysfs(struct un_t *un, struct device *c)
-{
-	int ret;
-
-	ret = sysfs_create_group(&c->kobj, &dgnc_tty_attribute_group);
-	if (ret) {
-		dev_err(c, "dgnc: failed to create sysfs tty device attributes.\n");
-		sysfs_remove_group(&c->kobj, &dgnc_tty_attribute_group);
-		return;
-	}
-
-	dev_set_drvdata(c, un);
-}
-
-void dgnc_remove_tty_sysfs(struct device *c)
-{
-	sysfs_remove_group(&c->kobj, &dgnc_tty_attribute_group);
-}
-
diff --git a/drivers/staging/dgnc/dgnc_sysfs.h b/drivers/staging/dgnc/dgnc_sysfs.h
deleted file mode 100644
index 7be7d55bc49e..000000000000
--- a/drivers/staging/dgnc/dgnc_sysfs.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2003 Digi International (www.digi.com)
- *	Scott H Kilau <Scott_Kilau at digi dot com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED; without even the
- * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
- * PURPOSE.  See the GNU General Public License for more details.
- */
-
-#ifndef __DGNC_SYSFS_H
-#define __DGNC_SYSFS_H
-
-#include <linux/device.h>
-#include "dgnc_driver.h"
-
-struct dgnc_board;
-struct channel_t;
-struct un_t;
-struct pci_driver;
-struct class_device;
-
-void dgnc_create_ports_sysfiles(struct dgnc_board *bd);
-void dgnc_remove_ports_sysfiles(struct dgnc_board *bd);
-
-void dgnc_create_driver_sysfiles(struct pci_driver *);
-void dgnc_remove_driver_sysfiles(struct pci_driver *);
-
-int dgnc_tty_class_init(void);
-int dgnc_tty_class_destroy(void);
-
-void dgnc_create_tty_sysfs(struct un_t *un, struct device *c);
-void dgnc_remove_tty_sysfs(struct device *c);
-
-#endif
diff --git a/drivers/staging/dgnc/dgnc_tty.c b/drivers/staging/dgnc/dgnc_tty.c
index 953d9310fa74..1e10c0fe4745 100644
--- a/drivers/staging/dgnc/dgnc_tty.c
+++ b/drivers/staging/dgnc/dgnc_tty.c
@@ -13,13 +13,9 @@
  * PURPOSE.  See the GNU General Public License for more details.
  */
 
-/************************************************************************
- *
+/*
  * This file implements the tty driver functionality for the
  * Neo and ClassicBoard PCI based product lines.
- *
- ************************************************************************
- *
  */
 
 #include <linux/kernel.h>
@@ -39,27 +35,20 @@
 #include "dgnc_tty.h"
 #include "dgnc_neo.h"
 #include "dgnc_cls.h"
-#include "dgnc_sysfs.h"
 #include "dgnc_utils.h"
 
-/*
- * internal variables
- */
-static unsigned char		*dgnc_TmpWriteBuf;
-
-/*
- * Default transparent print information.
- */
-static struct digi_t dgnc_digi_init = {
-	.digi_flags =	DIGI_COOK,	/* Flags			*/
-	.digi_maxcps =	100,		/* Max CPS			*/
-	.digi_maxchar =	50,		/* Max chars in print queue	*/
-	.digi_bufsize =	100,		/* Printer buffer size		*/
-	.digi_onlen =	4,		/* size of printer on string	*/
-	.digi_offlen =	4,		/* size of printer off string	*/
-	.digi_onstr =	"\033[5i",	/* ANSI printer on string ]	*/
-	.digi_offstr =	"\033[4i",	/* ANSI printer off string ]	*/
-	.digi_term =	"ansi"		/* default terminal type	*/
+/* Default transparent print information. */
+
+static const struct digi_t dgnc_digi_init = {
+	.digi_flags =	DIGI_COOK,	/* Flags */
+	.digi_maxcps =	100,		/* Max CPS */
+	.digi_maxchar =	50,		/* Max chars in print queue */
+	.digi_bufsize =	100,		/* Printer buffer size */
+	.digi_onlen =	4,		/* size of printer on string */
+	.digi_offlen =	4,		/* size of printer off string */
+	.digi_onstr =	"\033[5i",	/* ANSI printer on string ] */
+	.digi_offstr =	"\033[4i",	/* ANSI printer off string ] */
+	.digi_term =	"ansi"		/* default terminal type */
 };
 
 /*
@@ -69,7 +58,7 @@ static struct digi_t dgnc_digi_init = {
  * This defines a raw port at 9600 baud, 8 data bits, no parity,
  * 1 stop bit.
  */
-static struct ktermios DgncDefaultTermios = {
+static struct ktermios default_termios = {
 	.c_iflag =	(DEFAULT_IFLAGS),	/* iflags */
 	.c_oflag =	(DEFAULT_OFLAGS),	/* oflags */
 	.c_cflag =	(DEFAULT_CFLAGS),	/* cflags */
@@ -113,6 +102,8 @@ static int dgnc_tty_write(struct tty_struct *tty, const unsigned char *buf,
 static void dgnc_tty_set_termios(struct tty_struct *tty,
 				 struct ktermios *old_termios);
 static void dgnc_tty_send_xchar(struct tty_struct *tty, char ch);
+static void dgnc_set_signal_low(struct channel_t *ch, const unsigned char line);
+static void dgnc_wake_up_unit(struct un_t *unit);
 
 static const struct tty_operations dgnc_tty_ops = {
 	.open = dgnc_tty_open,
@@ -137,36 +128,7 @@ static const struct tty_operations dgnc_tty_ops = {
 	.send_xchar = dgnc_tty_send_xchar
 };
 
-/************************************************************************
- *
- * TTY Initialization/Cleanup Functions
- *
- ************************************************************************/
-
-/*
- * dgnc_tty_preinit()
- *
- * Initialize any global tty related data before we download any boards.
- */
-int dgnc_tty_preinit(void)
-{
-	/*
-	 * Allocate a buffer for doing the copy from user space to
-	 * kernel space in dgnc_write().  We only use one buffer and
-	 * control access to it with a semaphore.  If we are paging, we
-	 * are already in trouble so one buffer won't hurt much anyway.
-	 *
-	 * We are okay to sleep in the malloc, as this routine
-	 * is only called during module load, (not in interrupt context),
-	 * and with no locks held.
-	 */
-	dgnc_TmpWriteBuf = kmalloc(WRITEBUFLEN, GFP_KERNEL);
-
-	if (!dgnc_TmpWriteBuf)
-		return -ENOMEM;
-
-	return 0;
-}
+/* TTY Initialization/Cleanup Functions */
 
 /*
  * dgnc_tty_register()
@@ -194,7 +156,7 @@ int dgnc_tty_register(struct dgnc_board *brd)
 	brd->serial_driver->minor_start = 0;
 	brd->serial_driver->type = TTY_DRIVER_TYPE_SERIAL;
 	brd->serial_driver->subtype = SERIAL_TYPE_NORMAL;
-	brd->serial_driver->init_termios = DgncDefaultTermios;
+	brd->serial_driver->init_termios = default_termios;
 	brd->serial_driver->driver_name = DRVSTR;
 
 	/*
@@ -233,7 +195,7 @@ int dgnc_tty_register(struct dgnc_board *brd)
 	brd->print_driver->minor_start = 0x80;
 	brd->print_driver->type = TTY_DRIVER_TYPE_SERIAL;
 	brd->print_driver->subtype = SERIAL_TYPE_NORMAL;
-	brd->print_driver->init_termios = DgncDefaultTermios;
+	brd->print_driver->init_termios = default_termios;
 	brd->print_driver->driver_name = DRVSTR;
 
 	/*
@@ -285,9 +247,7 @@ int dgnc_tty_init(struct dgnc_board *brd)
 	if (!brd)
 		return -ENXIO;
 
-	/*
-	 * Initialize board structure elements.
-	 */
+	/* Initialize board structure elements. */
 
 	vaddr = brd->re_map_membase;
 
@@ -345,12 +305,10 @@ int dgnc_tty_init(struct dgnc_board *brd)
 			classp = tty_register_device(brd->serial_driver, i,
 						     &ch->ch_bd->pdev->dev);
 			ch->ch_tun.un_sysfs = classp;
-			dgnc_create_tty_sysfs(&ch->ch_tun, classp);
 
 			classp = tty_register_device(brd->print_driver, i,
 						     &ch->ch_bd->pdev->dev);
 			ch->ch_pun.un_sysfs = classp;
-			dgnc_create_tty_sysfs(&ch->ch_pun, classp);
 		}
 	}
 
@@ -365,17 +323,6 @@ err_free_channels:
 }
 
 /*
- * dgnc_tty_post_uninit()
- *
- * UnInitialize any global tty related data.
- */
-void dgnc_tty_post_uninit(void)
-{
-	kfree(dgnc_TmpWriteBuf);
-	dgnc_TmpWriteBuf = NULL;
-}
-
-/*
  * dgnc_cleanup_tty()
  *
  * Uninitialize the TTY portion of this driver.  Free all memory and
@@ -385,20 +332,14 @@ void dgnc_cleanup_tty(struct dgnc_board *brd)
 {
 	int i = 0;
 
-	for (i = 0; i < brd->nasync; i++) {
-		if (brd->channels[i])
-			dgnc_remove_tty_sysfs(brd->channels[i]->
-					      ch_tun.un_sysfs);
+	for (i = 0; i < brd->nasync; i++)
 		tty_unregister_device(brd->serial_driver, i);
-	}
+
 	tty_unregister_driver(brd->serial_driver);
 
-	for (i = 0; i < brd->nasync; i++) {
-		if (brd->channels[i])
-			dgnc_remove_tty_sysfs(brd->channels[i]->
-					      ch_pun.un_sysfs);
+	for (i = 0; i < brd->nasync; i++)
 		tty_unregister_device(brd->print_driver, i);
-	}
+
 	tty_unregister_driver(brd->print_driver);
 
 	put_tty_driver(brd->serial_driver);
@@ -437,9 +378,7 @@ static void dgnc_wmove(struct channel_t *ch, char *buf, uint n)
 	}
 
 	if (n > 0) {
-		/*
-		 * Move rest of data.
-		 */
+		/* Move rest of data. */
 		remain = n;
 		memcpy(ch->ch_wqueue + head, buf, remain);
 		head += remain;
@@ -509,9 +448,8 @@ void dgnc_input(struct channel_t *ch)
 		goto exit_unlock;
 	}
 
-	/*
-	 * If we are throttled, simply don't read any data.
-	 */
+	/* If we are throttled, simply don't read any data. */
+
 	if (ch->ch_flags & CH_FORCED_STOPI)
 		goto exit_unlock;
 
@@ -624,10 +562,10 @@ exit_unlock:
 		tty_ldisc_deref(ld);
 }
 
-/************************************************************************
+/*
  * Determines when CARRIER changes state and takes appropriate
  * action.
- ************************************************************************/
+ */
 void dgnc_carrier(struct channel_t *ch)
 {
 	int virt_carrier = 0;
@@ -645,28 +583,24 @@ void dgnc_carrier(struct channel_t *ch)
 	if (ch->ch_c_cflag & CLOCAL)
 		virt_carrier = 1;
 
-	/*
-	 * Test for a VIRTUAL carrier transition to HIGH.
-	 */
+	/* Test for a VIRTUAL carrier transition to HIGH. */
+
 	if (((ch->ch_flags & CH_FCAR) == 0) && (virt_carrier == 1)) {
 		/*
 		 * When carrier rises, wake any threads waiting
 		 * for carrier in the open routine.
 		 */
-
 		if (waitqueue_active(&ch->ch_flags_wait))
 			wake_up_interruptible(&ch->ch_flags_wait);
 	}
 
-	/*
-	 * Test for a PHYSICAL carrier transition to HIGH.
-	 */
+	/* Test for a PHYSICAL carrier transition to HIGH. */
+
 	if (((ch->ch_flags & CH_CD) == 0) && (phys_carrier == 1)) {
 		/*
 		 * When carrier rises, wake any threads waiting
 		 * for carrier in the open routine.
 		 */
-
 		if (waitqueue_active(&ch->ch_flags_wait))
 			wake_up_interruptible(&ch->ch_flags_wait);
 	}
@@ -704,9 +638,8 @@ void dgnc_carrier(struct channel_t *ch)
 			tty_hangup(ch->ch_pun.un_tty);
 	}
 
-	/*
-	 *  Make sure that our cached values reflect the current reality.
-	 */
+	/*  Make sure that our cached values reflect the current reality. */
+
 	if (virt_carrier == 1)
 		ch->ch_flags |= CH_FCAR;
 	else
@@ -718,9 +651,8 @@ void dgnc_carrier(struct channel_t *ch)
 		ch->ch_flags &= ~CH_CD;
 }
 
-/*
- *  Assign the custom baud rate to the channel structure
- */
+/*  Assign the custom baud rate to the channel structure */
+
 static void dgnc_set_custom_speed(struct channel_t *ch, uint newrate)
 {
 	int testdiv;
@@ -854,6 +786,12 @@ void dgnc_check_queue_flow_control(struct channel_t *ch)
 	}
 }
 
+static void dgnc_set_signal_low(struct channel_t *ch, const unsigned char sig)
+{
+	ch->ch_mostat &= ~(sig);
+	ch->ch_bd->bd_ops->assert_modem_signals(ch);
+}
+
 void dgnc_wakeup_writes(struct channel_t *ch)
 {
 	int qlen = 0;
@@ -864,9 +802,8 @@ void dgnc_wakeup_writes(struct channel_t *ch)
 
 	spin_lock_irqsave(&ch->ch_lock, flags);
 
-	/*
-	 * If channel now has space, wake up anyone waiting on the condition.
-	 */
+	/* If channel now has space, wake up anyone waiting on the condition. */
+
 	qlen = ch->ch_w_head - ch->ch_w_tail;
 	if (qlen < 0)
 		qlen += WQUEUESIZE;
@@ -892,19 +829,15 @@ void dgnc_wakeup_writes(struct channel_t *ch)
 				 * If RTS Toggle mode is on, whenever
 				 * the queue and UART is empty, keep RTS low.
 				 */
-				if (ch->ch_digi.digi_flags & DIGI_RTS_TOGGLE) {
-					ch->ch_mostat &= ~(UART_MCR_RTS);
-					ch->ch_bd->bd_ops->assert_modem_signals(ch);
-				}
+				if (ch->ch_digi.digi_flags & DIGI_RTS_TOGGLE)
+					dgnc_set_signal_low(ch, UART_MCR_RTS);
 
 				/*
 				 * If DTR Toggle mode is on, whenever
 				 * the queue and UART is empty, keep DTR low.
 				 */
-				if (ch->ch_digi.digi_flags & DIGI_DTR_TOGGLE) {
-					ch->ch_mostat &= ~(UART_MCR_DTR);
-					ch->ch_bd->bd_ops->assert_modem_signals(ch);
-				}
+				if (ch->ch_digi.digi_flags & DIGI_DTR_TOGGLE)
+					dgnc_set_signal_low(ch, UART_MCR_DTR);
 			}
 		}
 
@@ -930,7 +863,7 @@ void dgnc_wakeup_writes(struct channel_t *ch)
 	spin_unlock_irqrestore(&ch->ch_lock, flags);
 }
 
-struct dgnc_board *find_board_by_major(unsigned int major)
+static struct dgnc_board *find_board_by_major(unsigned int major)
 {
 	int i;
 
@@ -948,16 +881,10 @@ struct dgnc_board *find_board_by_major(unsigned int major)
 	return NULL;
 }
 
-/************************************************************************
- *
- * TTY Entry points and helper functions
- *
- ************************************************************************/
+/* TTY Entry points and helper functions */
+
+/* dgnc_tty_open() */
 
-/*
- * dgnc_tty_open()
- *
- */
 static int dgnc_tty_open(struct tty_struct *tty, struct file *file)
 {
 	struct dgnc_board	*brd;
@@ -1045,8 +972,8 @@ static int dgnc_tty_open(struct tty_struct *tty, struct file *file)
 	 * ch_flags_wait to wake us back up.
 	 */
 	rc = wait_event_interruptible(ch->ch_flags_wait,
-		(((ch->ch_tun.un_flags | ch->ch_pun.un_flags) &
-		  UN_CLOSING) == 0));
+			(((ch->ch_tun.un_flags |
+			   ch->ch_pun.un_flags) & UN_CLOSING) == 0));
 
 	/* If ret is non-zero, user ctrl-c'ed us */
 	if (rc)
@@ -1057,9 +984,8 @@ static int dgnc_tty_open(struct tty_struct *tty, struct file *file)
 	/* Store our unit into driver_data, so we always have it available. */
 	tty->driver_data = un;
 
-	/*
-	 * Initialize tty's
-	 */
+	/* Initialize tty's */
+
 	if (!(un->un_flags & UN_ISOPEN)) {
 		/* Store important variables. */
 		un->un_tty     = tty;
@@ -1096,13 +1022,10 @@ static int dgnc_tty_open(struct tty_struct *tty, struct file *file)
 	ch->ch_flags &= ~(CH_OPENING);
 	wake_up_interruptible(&ch->ch_flags_wait);
 
-	/*
-	 * Initialize if neither terminal or printer is open.
-	 */
+	/* Initialize if neither terminal or printer is open. */
+
 	if (!((ch->ch_tun.un_flags | ch->ch_pun.un_flags) & UN_ISOPEN)) {
-		/*
-		 * Flush input queues.
-		 */
+		/* Flush input queues. */
 		ch->ch_r_head = 0;
 		ch->ch_r_tail = 0;
 		ch->ch_e_head = 0;
@@ -1138,16 +1061,13 @@ static int dgnc_tty_open(struct tty_struct *tty, struct file *file)
 		brd->bd_ops->uart_init(ch);
 	}
 
-	/*
-	 * Run param in case we changed anything
-	 */
+	/* Run param in case we changed anything */
+
 	brd->bd_ops->param(tty);
 
 	dgnc_carrier(ch);
 
-	/*
-	 * follow protocol for opening port
-	 */
+	/* follow protocol for opening port */
 
 	spin_unlock_irqrestore(&ch->ch_lock, flags);
 
@@ -1248,9 +1168,8 @@ static int dgnc_block_til_ready(struct tty_struct *tty,
 			break;
 		}
 
-		/*
-		 * Store the flags before we let go of channel lock
-		 */
+		/* Store the flags before we let go of channel lock */
+
 		if (sleep_on_un_flags)
 			old_flags = ch->ch_tun.un_flags | ch->ch_pun.un_flags;
 		else
@@ -1269,12 +1188,13 @@ static int dgnc_block_til_ready(struct tty_struct *tty,
 		 * from the current value.
 		 */
 		if (sleep_on_un_flags)
-			retval = wait_event_interruptible(un->un_flags_wait,
-				(old_flags != (ch->ch_tun.un_flags |
-					       ch->ch_pun.un_flags)));
+			retval = wait_event_interruptible
+				(un->un_flags_wait,
+				 (old_flags != (ch->ch_tun.un_flags |
+						ch->ch_pun.un_flags)));
 		else
 			retval = wait_event_interruptible(ch->ch_flags_wait,
-				(old_flags != ch->ch_flags));
+					(old_flags != ch->ch_flags));
 
 		/*
 		 * We got woken up for some reason.
@@ -1304,10 +1224,8 @@ static void dgnc_tty_hangup(struct tty_struct *tty)
 	dgnc_tty_flush_buffer(tty);
 }
 
-/*
- * dgnc_tty_close()
- *
- */
+/* dgnc_tty_close() */
+
 static void dgnc_tty_close(struct tty_struct *tty, struct file *file)
 {
 	struct dgnc_board *bd;
@@ -1377,9 +1295,8 @@ static void dgnc_tty_close(struct tty_struct *tty, struct file *file)
 	    !(ch->ch_digi.digi_flags & DIGI_PRINTER)) {
 		ch->ch_flags &= ~(CH_STOPI | CH_FORCED_STOPI);
 
-		/*
-		 * turn off print device when closing print device.
-		 */
+		/* turn off print device when closing print device. */
+
 		if ((un->un_type == DGNC_PRINT) && (ch->ch_flags & CH_PRON)) {
 			dgnc_wmove(ch, ch->ch_digi.digi_offstr,
 				   (int)ch->ch_digi.digi_offlen);
@@ -1399,9 +1316,8 @@ static void dgnc_tty_close(struct tty_struct *tty, struct file *file)
 
 		tty->closing = 0;
 
-		/*
-		 * If we have HUPCL set, lower DTR and RTS
-		 */
+		/* If we have HUPCL set, lower DTR and RTS */
+
 		if (ch->ch_c_cflag & HUPCL) {
 			/* Drop RTS/DTR */
 			ch->ch_mostat &= ~(UART_MCR_DTR | UART_MCR_RTS);
@@ -1424,9 +1340,8 @@ static void dgnc_tty_close(struct tty_struct *tty, struct file *file)
 		/* Turn off UART interrupts for this port */
 		ch->ch_bd->bd_ops->uart_off(ch);
 	} else {
-		/*
-		 * turn off print device when closing print device.
-		 */
+		/* turn off print device when closing print device. */
+
 		if ((un->un_type == DGNC_PRINT) && (ch->ch_flags & CH_PRON)) {
 			dgnc_wmove(ch, ch->ch_digi.digi_offstr,
 				   (int)ch->ch_digi.digi_offlen);
@@ -1543,7 +1458,7 @@ static int dgnc_tty_write_room(struct tty_struct *tty)
 	int ret = 0;
 	unsigned long flags;
 
-	if (!tty || !dgnc_TmpWriteBuf)
+	if (!tty)
 		return 0;
 
 	un = tty->driver_data;
@@ -1598,9 +1513,8 @@ static int dgnc_tty_write_room(struct tty_struct *tty)
  */
 static int dgnc_tty_put_char(struct tty_struct *tty, unsigned char c)
 {
-	/*
-	 * Simply call tty_write.
-	 */
+	/* Simply call tty_write. */
+
 	dgnc_tty_write(tty, &c, 1);
 	return 1;
 }
@@ -1623,7 +1537,7 @@ static int dgnc_tty_write(struct tty_struct *tty,
 	ushort tmask;
 	uint remain;
 
-	if (!tty || !dgnc_TmpWriteBuf)
+	if (!tty)
 		return 0;
 
 	un = tty->driver_data;
@@ -1667,9 +1581,8 @@ static int dgnc_tty_write(struct tty_struct *tty,
 	 */
 	count = min(count, bufcount);
 
-	/*
-	 * Bail if no space left.
-	 */
+	/* Bail if no space left. */
+
 	if (count <= 0)
 		goto exit_retry;
 
@@ -1712,9 +1625,7 @@ static int dgnc_tty_write(struct tty_struct *tty,
 	}
 
 	if (n > 0) {
-		/*
-		 * Move rest of data.
-		 */
+		/* Move rest of data. */
 		remain = n;
 		memcpy(ch->ch_wqueue + head, buf, remain);
 		head += remain;
@@ -1749,9 +1660,7 @@ exit_retry:
 	return 0;
 }
 
-/*
- * Return modem signals to ld.
- */
+/* Return modem signals to ld. */
 
 static int dgnc_tty_tiocmget(struct tty_struct *tty)
 {
@@ -1960,9 +1869,8 @@ static void dgnc_tty_send_xchar(struct tty_struct *tty, char c)
 	dev_dbg(tty->dev, "dgnc_tty_send_xchar finish\n");
 }
 
-/*
- * Return modem signals to ld.
- */
+/* Return modem signals to ld. */
+
 static inline int dgnc_get_mstat(struct channel_t *ch)
 {
 	unsigned char mstat;
@@ -1994,9 +1902,8 @@ static inline int dgnc_get_mstat(struct channel_t *ch)
 	return result;
 }
 
-/*
- * Return modem signals to ld.
- */
+/* Return modem signals to ld. */
+
 static int dgnc_get_modem_info(struct channel_t *ch,
 			       unsigned int  __user *value)
 {
@@ -2070,9 +1977,6 @@ static int dgnc_set_modem_info(struct channel_t *ch,
  * dgnc_tty_digigeta()
  *
  * Ioctl to get the information for ditty.
- *
- *
- *
  */
 static int dgnc_tty_digigeta(struct tty_struct *tty,
 			     struct digi_t __user *retinfo)
@@ -2112,9 +2016,6 @@ static int dgnc_tty_digigeta(struct tty_struct *tty,
  * dgnc_tty_digiseta()
  *
  * Ioctl to set the information for ditty.
- *
- *
- *
  */
 static int dgnc_tty_digiseta(struct tty_struct *tty,
 			     struct digi_t __user *new_info)
@@ -2145,9 +2046,8 @@ static int dgnc_tty_digiseta(struct tty_struct *tty,
 
 	spin_lock_irqsave(&ch->ch_lock, flags);
 
-	/*
-	 * Handle transistions to and from RTS Toggle.
-	 */
+	/* Handle transitions to and from RTS Toggle. */
+
 	if (!(ch->ch_digi.digi_flags & DIGI_RTS_TOGGLE) &&
 	    (new_digi.digi_flags & DIGI_RTS_TOGGLE))
 		ch->ch_mostat &= ~(UART_MCR_RTS);
@@ -2155,9 +2055,8 @@ static int dgnc_tty_digiseta(struct tty_struct *tty,
 	    !(new_digi.digi_flags & DIGI_RTS_TOGGLE))
 		ch->ch_mostat |= (UART_MCR_RTS);
 
-	/*
-	 * Handle transistions to and from DTR Toggle.
-	 */
+	/* Handle transitions to and from DTR Toggle. */
+
 	if (!(ch->ch_digi.digi_flags & DIGI_DTR_TOGGLE) &&
 	    (new_digi.digi_flags & DIGI_DTR_TOGGLE))
 		ch->ch_mostat &= ~(UART_MCR_DTR);
@@ -2195,9 +2094,8 @@ static int dgnc_tty_digiseta(struct tty_struct *tty,
 	return 0;
 }
 
-/*
- * dgnc_set_termios()
- */
+/* dgnc_set_termios() */
+
 static void dgnc_tty_set_termios(struct tty_struct *tty,
 				 struct ktermios *old_termios)
 {
@@ -2428,11 +2326,18 @@ static void dgnc_tty_flush_buffer(struct tty_struct *tty)
 	spin_unlock_irqrestore(&ch->ch_lock, flags);
 }
 
-/*****************************************************************************
- *
- * The IOCTL function and all of its helpers
+/*
+ * dgnc_wake_up_unit()
  *
- *****************************************************************************/
+ * Wakes up processes waiting in the unit's (teminal/printer) wait queue
+ */
+static void dgnc_wake_up_unit(struct un_t *unit)
+{
+	unit->un_flags &= ~(UN_LOW | UN_EMPTY);
+	wake_up_interruptible(&unit->un_flags_wait);
+}
+
+/* The IOCTL function and all of its helpers */
 
 /*
  * dgnc_tty_ioctl()
@@ -2506,7 +2411,8 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 		return 0;
 
 	case TCSBRKP:
-		/* support for POSIX tcsendbreak()
+		/*
+		 * support for POSIX tcsendbreak()
 		 * According to POSIX.1 spec (7.2.2.1.2) breaks should be
 		 * between 0.25 and 0.5 seconds so we'll ask for something
 		 * in the middle: 0.375 seconds.
@@ -2583,9 +2489,7 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 		spin_unlock_irqrestore(&ch->ch_lock, flags);
 		return dgnc_set_modem_info(ch, cmd, uarg);
 
-		/*
-		 * Here are any additional ioctl's that we want to implement
-		 */
+		/* Here are any additional ioctl's that we want to implement */
 
 	case TCFLSH:
 		/*
@@ -2615,17 +2519,11 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 				ch->ch_w_head = ch->ch_w_tail;
 				ch_bd_ops->flush_uart_write(ch);
 
-				if (ch->ch_tun.un_flags & (UN_LOW | UN_EMPTY)) {
-					ch->ch_tun.un_flags &=
-						~(UN_LOW | UN_EMPTY);
-					wake_up_interruptible(&ch->ch_tun.un_flags_wait);
-				}
-
-				if (ch->ch_pun.un_flags & (UN_LOW|UN_EMPTY)) {
-					ch->ch_pun.un_flags &=
-						~(UN_LOW | UN_EMPTY);
-					wake_up_interruptible(&ch->ch_pun.un_flags_wait);
-				}
+				if (ch->ch_tun.un_flags & (UN_LOW | UN_EMPTY))
+					dgnc_wake_up_unit(&ch->ch_tun);
+
+				if (ch->ch_pun.un_flags & (UN_LOW | UN_EMPTY))
+					dgnc_wake_up_unit(&ch->ch_pun);
 			}
 		}
 
@@ -2705,9 +2603,10 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 	case DIGI_LOOPBACK:
 		{
 			uint loopback = 0;
-			/* Let go of locks when accessing user space,
+			/*
+			 * Let go of locks when accessing user space,
 			 * could sleep
-			*/
+			 */
 			spin_unlock_irqrestore(&ch->ch_lock, flags);
 			rc = get_user(loopback, (unsigned int __user *)arg);
 			if (rc)
@@ -2749,7 +2648,7 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 	 * This ioctl allows insertion of a character into the front
 	 * of any pending data to be transmitted.
 	 *
-	 * This ioctl is to satify the "Send Character Immediate"
+	 * This ioctl is to satisfy the "Send Character Immediate"
 	 * call that the RealPort protocol spec requires.
 	 */
 	case DIGI_REALPORT_SENDIMMEDIATE:
@@ -2769,7 +2668,7 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 	/*
 	 * This ioctl returns all the current counts for the port.
 	 *
-	 * This ioctl is to satify the "Line Error Counters"
+	 * This ioctl is to satisfy the "Line Error Counters"
 	 * call that the RealPort protocol spec requires.
 	 */
 	case DIGI_REALPORT_GETCOUNTERS:
@@ -2795,7 +2694,7 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 	/*
 	 * This ioctl returns all current events.
 	 *
-	 * This ioctl is to satify the "Event Reporting"
+	 * This ioctl is to satisfy the "Event Reporting"
 	 * call that the RealPort protocol spec requires.
 	 */
 	case DIGI_REALPORT_GETEVENTS:
@@ -2831,23 +2730,23 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 
 		spin_unlock_irqrestore(&ch->ch_lock, flags);
 
-		/*
-		 * Get data from user first.
-		 */
+		/* Get data from user first. */
+
 		if (copy_from_user(&buf, uarg, sizeof(buf)))
 			return -EFAULT;
 
 		spin_lock_irqsave(&ch->ch_lock, flags);
 
-		/*
-		 * Figure out how much data is in our RX and TX queues.
-		 */
+		/* Figure out how much data is in our RX and TX queues. */
+
 		buf.rxbuf = (ch->ch_r_head - ch->ch_r_tail) & RQUEUEMASK;
 		buf.txbuf = (ch->ch_w_head - ch->ch_w_tail) & WQUEUEMASK;
 
 		/*
-		 * Is the UART empty? Add that value to whats in our TX queue.
+		 * Is the UART empty?
+		 * Add that value to whats in our TX queue.
 		 */
+
 		count = buf.txbuf + ch_bd_ops->get_uart_bytes_left(ch);
 
 		/*
@@ -2867,9 +2766,8 @@ static int dgnc_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 		if (buf.txbuf > tdist)
 			buf.txbuf = tdist;
 
-		/*
-		 * Report whether our queue and UART TX are completely empty.
-		 */
+		/* Report whether our queue and UART TX are completely empty. */
+
 		if (count)
 			buf.txdone = 0;
 		else
diff --git a/drivers/staging/dgnc/dgnc_tty.h b/drivers/staging/dgnc/dgnc_tty.h
index 24c9a412211e..1ee0eeeb4730 100644
--- a/drivers/staging/dgnc/dgnc_tty.h
+++ b/drivers/staging/dgnc/dgnc_tty.h
@@ -21,11 +21,9 @@
 int	dgnc_tty_register(struct dgnc_board *brd);
 void dgnc_tty_unregister(struct dgnc_board *brd);
 
-int	dgnc_tty_preinit(void);
-int     dgnc_tty_init(struct dgnc_board *);
+int     dgnc_tty_init(struct dgnc_board *brd);
 
-void	dgnc_tty_post_uninit(void);
-void	dgnc_cleanup_tty(struct dgnc_board *);
+void	dgnc_cleanup_tty(struct dgnc_board *brd);
 
 void	dgnc_input(struct channel_t *ch);
 void	dgnc_carrier(struct channel_t *ch);
diff --git a/drivers/staging/dgnc/digi.h b/drivers/staging/dgnc/digi.h
index 5b983e6f5ee2..ec2e3dda6119 100644
--- a/drivers/staging/dgnc/digi.h
+++ b/drivers/staging/dgnc/digi.h
@@ -17,16 +17,16 @@
 #define __DIGI_H
 
 #ifndef TIOCM_LE
-#define		TIOCM_LE	0x01		/* line enable		*/
-#define		TIOCM_DTR	0x02		/* data terminal ready	*/
-#define		TIOCM_RTS	0x04		/* request to send	*/
-#define		TIOCM_ST	0x08		/* secondary transmit	*/
-#define		TIOCM_SR	0x10		/* secondary receive	*/
-#define		TIOCM_CTS	0x20		/* clear to send	*/
-#define		TIOCM_CAR	0x40		/* carrier detect	*/
-#define		TIOCM_RNG	0x80		/* ring	indicator	*/
-#define		TIOCM_DSR	0x100		/* data set ready	*/
-#define		TIOCM_RI	TIOCM_RNG	/* ring (alternate)	*/
+#define		TIOCM_LE	0x01		/* line enable */
+#define		TIOCM_DTR	0x02		/* data terminal ready */
+#define		TIOCM_RTS	0x04		/* request to send */
+#define		TIOCM_ST	0x08		/* secondary transmit */
+#define		TIOCM_SR	0x10		/* secondary receive */
+#define		TIOCM_CTS	0x20		/* clear to send */
+#define		TIOCM_CAR	0x40		/* carrier detect */
+#define		TIOCM_RNG	0x80		/* ring	indicator */
+#define		TIOCM_DSR	0x100		/* data set ready */
+#define		TIOCM_RI	TIOCM_RNG	/* ring (alternate) */
 #define		TIOCM_CD	TIOCM_CAR	/* carrier detect (alt)	*/
 #endif
 
@@ -40,72 +40,71 @@
 #define	TIOCMBIS	(('d' << 8) | 255)	/* set modem ctrl state */
 #endif
 
-#define DIGI_GETA	(('e' << 8) | 94)	/* Read params		*/
-#define DIGI_SETA	(('e' << 8) | 95)	/* Set params		*/
-#define DIGI_SETAW	(('e' << 8) | 96)	/* Drain & set params	*/
+#define DIGI_GETA	(('e' << 8) | 94)	/* Read params */
+#define DIGI_SETA	(('e' << 8) | 95)	/* Set params */
+#define DIGI_SETAW	(('e' << 8) | 96)	/* Drain & set params */
 #define DIGI_SETAF	(('e' << 8) | 97)	/* Drain, flush & set params */
-#define DIGI_GET_NI_INFO (('d' << 8) | 250) /* Non-intelligent state info */
-#define DIGI_LOOPBACK (('d' << 8) | 252) /*
-					* Enable/disable UART
-					* internal loopback
-					*/
-#define DIGI_FAST	0x0002		/* Fast baud rates		*/
-#define RTSPACE		0x0004		/* RTS input flow control	*/
-#define CTSPACE		0x0008		/* CTS output flow control	*/
+#define DIGI_GET_NI_INFO (('d' << 8) | 250)	/* Non-intelligent state info */
+#define DIGI_LOOPBACK (('d' << 8) | 252)	/*
+						 * Enable/disable UART
+						 * internal loopback
+						 */
+#define DIGI_FAST	0x0002		/* Fast baud rates */
+#define RTSPACE		0x0004		/* RTS input flow control */
+#define CTSPACE		0x0008		/* CTS output flow control */
 #define DIGI_COOK	0x0080		/* Cooked processing done in FEP */
-#define DIGI_FORCEDCD	0x0100		/* Force carrier		*/
-#define	DIGI_ALTPIN	0x0200		/* Alternate RJ-45 pin config	*/
+#define DIGI_FORCEDCD	0x0100		/* Force carrier */
+#define	DIGI_ALTPIN	0x0200		/* Alternate RJ-45 pin config */
 #define	DIGI_PRINTER	0x0800		/* Hold port open for flow cntrl*/
-#define DIGI_DTR_TOGGLE	0x2000		/* Support DTR Toggle           */
-#define DIGI_RTS_TOGGLE	0x8000		/* Support RTS Toggle		*/
-#define DIGI_PLEN	28		/* String length		*/
-#define	DIGI_TSIZ	10		/* Terminal string len		*/
+#define DIGI_DTR_TOGGLE	0x2000		/* Support DTR Toggle */
+#define DIGI_RTS_TOGGLE	0x8000		/* Support RTS Toggle */
+#define DIGI_PLEN	28		/* String length */
+#define	DIGI_TSIZ	10		/* Terminal string len */
 
-/************************************************************************
+/*
  * Structure used with ioctl commands for DIGI parameters.
- ************************************************************************/
+ */
 struct digi_t {
-	unsigned short	digi_flags;		/* Flags (see above)	*/
-	unsigned short	digi_maxcps;		/* Max printer CPS	*/
+	unsigned short	digi_flags;		/* Flags (see above) */
+	unsigned short	digi_maxcps;		/* Max printer CPS */
 	unsigned short	digi_maxchar;		/* Max chars in print queue */
-	unsigned short	digi_bufsize;		/* Buffer size		*/
-	unsigned char	digi_onlen;		/* Length of ON string	*/
+	unsigned short	digi_bufsize;		/* Buffer size */
+	unsigned char	digi_onlen;		/* Length of ON string */
 	unsigned char	digi_offlen;		/* Length of OFF string	*/
-	char		digi_onstr[DIGI_PLEN];	/* Printer on string	*/
-	char		digi_offstr[DIGI_PLEN];	/* Printer off string	*/
-	char		digi_term[DIGI_TSIZ];	/* terminal string	*/
+	char		digi_onstr[DIGI_PLEN];	/* Printer on string */
+	char		digi_offstr[DIGI_PLEN];	/* Printer off string */
+	char		digi_term[DIGI_TSIZ];	/* terminal string */
 };
 
-/************************************************************************
- * Structure to get driver status information
- ************************************************************************/
+/* Structure to get driver status information */
+
 struct digi_dinfo {
-	unsigned int	dinfo_nboards;		/* # boards configured	*/
+	unsigned int	dinfo_nboards;		/* # boards configured */
 	char		dinfo_reserved[12];	/* for future expansion */
-	char		dinfo_version[16];	/* driver version       */
+	char		dinfo_version[16];	/* driver version */
 };
 
-#define	DIGI_GETDD	(('d' << 8) | 248)	/* get driver info      */
+#define	DIGI_GETDD	(('d' << 8) | 248)	/* get driver info */
 
-/************************************************************************
+/*
  * Structure used with ioctl commands for per-board information
  *
  * physsize and memsize differ when board has "windowed" memory
- ************************************************************************/
+ */
 struct digi_info {
-	unsigned int	info_bdnum;		/* Board number (0 based)  */
-	unsigned int	info_ioport;		/* io port address         */
-	unsigned int	info_physaddr;		/* memory address          */
+	unsigned int	info_bdnum;		/* Board number (0 based) */
+	unsigned int	info_ioport;		/* io port address */
+	unsigned int	info_physaddr;		/* memory address */
 	unsigned int	info_physsize;		/* Size of host mem window */
 	unsigned int	info_memsize;		/* Amount of dual-port mem */
-						/* on board                */
-	unsigned short	info_bdtype;		/* Board type              */
-	unsigned short	info_nports;		/* number of ports         */
-	char		info_bdstate;		/* board state             */
-	char		info_reserved[7];	/* for future expansion    */
+						/* on board */
+	unsigned short	info_bdtype;		/* Board type */
+	unsigned short	info_nports;		/* number of ports */
+	char		info_bdstate;		/* board state */
+	char		info_reserved[7];	/* for future expansion */
 };
 
-#define	DIGI_GETBD	(('d' << 8) | 249)	/* get board info          */
+#define	DIGI_GETBD	(('d' << 8) | 249)	/* get board info */
 
 struct digi_getbuffer /* Struct for holding buffer use counts */
 {
@@ -139,7 +138,7 @@ struct digi_getcounter {
 #define DIGI_REALPORT_GETEVENTS (('e' << 8) | 111)
 
 #define EV_OPU 0x0001 /* !<Output paused by client */
-#define EV_OPS 0x0002 /* !<Output paused by reqular sw flowctrl */
+#define EV_OPS 0x0002 /* !<Output paused by regular sw flowctrl */
 #define EV_IPU 0x0010 /* !<Input paused unconditionally by user */
 #define EV_IPS 0x0020 /* !<Input paused by high/low water marks */
 #define EV_TXB 0x0040 /* !<Transmit break pending */
diff --git a/drivers/staging/emxx_udc/emxx_udc.c b/drivers/staging/emxx_udc/emxx_udc.c
index c3e298843b43..3f42fa8b0bf3 100644
--- a/drivers/staging/emxx_udc/emxx_udc.c
+++ b/drivers/staging/emxx_udc/emxx_udc.c
@@ -153,7 +153,6 @@ static void _nbu2ss_ep0_complete(struct usb_ep *_ep, struct usb_request *_req)
 	udc = (struct nbu2ss_udc *)_req->context;
 	p_ctrl = &udc->ctrl;
 	if ((p_ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
-
 		if (p_ctrl->bRequest == USB_REQ_SET_FEATURE) {
 			/*-------------------------------------------------*/
 			/* SET_FEATURE */
@@ -263,7 +262,7 @@ static int _nbu2ss_ep_init(struct nbu2ss_udc *udc, struct nbu2ss_ep *ep)
 	}
 
 	_nbu2ss_bitset(&udc->p_regs->EP_REGS[num].EP_CONTROL, data);
-	_nbu2ss_endpoint_toggle_reset(udc, (ep->epnum|ep->direct));
+	_nbu2ss_endpoint_toggle_reset(udc, (ep->epnum | ep->direct));
 
 	if (ep->direct == USB_DIR_OUT) {
 		/*---------------------------------------------------------*/
@@ -460,7 +459,7 @@ static void _nbu2ss_ep_in_end(
 		if (length)
 			_nbu2ss_writel(&preg->EP_REGS[num].EP_WRITE, data32);
 
-		data = ((((u32)length) << 5) & EPn_DW) | EPn_DEND;
+		data = (((length) << 5) & EPn_DW) | EPn_DEND;
 		_nbu2ss_bitset(&preg->EP_REGS[num].EP_CONTROL, data);
 
 		_nbu2ss_bitset(&preg->EP_REGS[num].EP_CONTROL, EPn_AUTO);
@@ -753,7 +752,6 @@ static int _nbu2ss_ep0_out_transfer(
 	/* Receive data confirmation */
 	iRecvLength = _nbu2ss_readl(&udc->p_regs->EP0_LENGTH) & EP0_LDATA;
 	if (iRecvLength != 0) {
-
 		fRcvZero = 0;
 
 		iRemainSize = req->req.length - req->req.actual;
@@ -928,9 +926,8 @@ static int _nbu2ss_epn_out_pio(
 
 	req->req.actual += result;
 
-	if ((req->req.actual == req->req.length)
-			|| ((req->req.actual % ep->ep.maxpacket) != 0)) {
-
+	if ((req->req.actual == req->req.length) ||
+	    ((req->req.actual % ep->ep.maxpacket) != 0)) {
 		result = 0;
 	}
 
@@ -956,9 +953,8 @@ static int _nbu2ss_epn_out_data(
 
 	iBufSize = min((req->req.length - req->req.actual), data_size);
 
-	if ((ep->ep_type != USB_ENDPOINT_XFER_INT)
-		&& (req->req.dma != 0)
-		&& (iBufSize  >= sizeof(u32))) {
+	if ((ep->ep_type != USB_ENDPOINT_XFER_INT) && (req->req.dma != 0) &&
+	    (iBufSize  >= sizeof(u32))) {
 		nret = _nbu2ss_out_dma(udc, req, num, iBufSize);
 	} else {
 		iBufSize = min_t(u32, iBufSize, ep->ep.maxpacket);
@@ -999,9 +995,8 @@ static int _nbu2ss_epn_out_transfer(
 			}
 		}
 	} else {
-		if ((req->req.actual == req->req.length)
-			|| ((req->req.actual % ep->ep.maxpacket) != 0)) {
-
+		if ((req->req.actual == req->req.length) ||
+		    ((req->req.actual % ep->ep.maxpacket) != 0)) {
 			result = 0;
 		}
 	}
@@ -1170,9 +1165,8 @@ static int _nbu2ss_epn_in_data(
 
 	num = ep->epnum - 1;
 
-	if ((ep->ep_type != USB_ENDPOINT_XFER_INT)
-		&& (req->req.dma != 0)
-		&& (data_size >= sizeof(u32))) {
+	if ((ep->ep_type != USB_ENDPOINT_XFER_INT) && (req->req.dma != 0) &&
+	    (data_size >= sizeof(u32))) {
 		nret = _nbu2ss_in_dma(udc, ep, req, num, data_size);
 	} else {
 		data_size = min_t(u32, data_size, ep->ep.maxpacket);
@@ -1557,7 +1551,6 @@ static void _nbu2ss_epn_set_stall(
 		for (limit_cnt = 0
 			; limit_cnt < IN_DATA_EMPTY_COUNT
 			; limit_cnt++) {
-
 			regdata = _nbu2ss_readl(
 				&preg->EP_REGS[ep->epnum - 1].EP_STATUS);
 
@@ -1582,11 +1575,8 @@ static int std_req_get_status(struct nbu2ss_udc *udc)
 	u8	ep_adrs;
 	int	result = -EINVAL;
 
-	if ((udc->ctrl.wValue != 0x0000)
-		|| (direction != USB_DIR_IN)) {
-
+	if ((udc->ctrl.wValue != 0x0000) || (direction != USB_DIR_IN))
 		return result;
-	}
 
 	length = min_t(u16, udc->ctrl.wLength, sizeof(status_data));
 
@@ -1852,7 +1842,7 @@ static inline void _nbu2ss_ep0_int(struct nbu2ss_udc *udc)
 
 	status = _nbu2ss_readl(&udc->p_regs->EP0_STATUS);
 	intr = status & EP0_STATUS_RW_BIT;
-	_nbu2ss_writel(&udc->p_regs->EP0_STATUS, ~(u32)intr);
+	_nbu2ss_writel(&udc->p_regs->EP0_STATUS, ~intr);
 
 	status &= (SETUP_INT | EP0_IN_INT | EP0_OUT_INT
 			| STG_END_INT | EP0_OUT_NULL_INT);
@@ -1897,9 +1887,8 @@ static inline void _nbu2ss_ep0_int(struct nbu2ss_udc *udc)
 			break;
 
 		case EP0_OUT_STATUS_PAHSE:
-			if ((status & STG_END_INT)
-			|| (status & SETUP_INT)
-			|| (status & EP0_OUT_NULL_INT)) {
+			if ((status & STG_END_INT) || (status & SETUP_INT) ||
+			    (status & EP0_OUT_NULL_INT)) {
 				status &= ~(STG_END_INT
 						| EP0_OUT_INT
 						| EP0_OUT_NULL_INT);
@@ -1982,7 +1971,6 @@ static inline void _nbu2ss_epn_in_int(
 
 	} else {
 		if (req->zero && ((req->req.actual % ep->ep.maxpacket) == 0)) {
-
 			status =
 			_nbu2ss_readl(&preg->EP_REGS[ep->epnum - 1].EP_STATUS);
 
@@ -2127,7 +2115,7 @@ static inline void _nbu2ss_epn_int(struct nbu2ss_udc *udc, u32 epnum)
 	status = _nbu2ss_readl(&udc->p_regs->EP_REGS[num].EP_STATUS);
 
 	/* Interrupt Clear */
-	_nbu2ss_writel(&udc->p_regs->EP_REGS[num].EP_STATUS, ~(u32)status);
+	_nbu2ss_writel(&udc->p_regs->EP_REGS[num].EP_STATUS, ~status);
 
 	req = list_first_entry_or_null(&ep->queue, struct nbu2ss_req, queue);
 	if (!req) {
@@ -2330,7 +2318,6 @@ static inline void _nbu2ss_check_vbus(struct nbu2ss_udc *udc)
 	/* VBUS ON Check*/
 	reg_dt = gpio_get_value(VBUS_VALUE);
 	if (reg_dt == 0) {
-
 		udc->linux_suspended = 0;
 
 		_nbu2ss_reset_controller(udc);
@@ -2502,7 +2489,6 @@ static irqreturn_t _nbu2ss_udc_irq(int irq, void *_udc)
 			int_bit = status >> 8;
 
 			for (epnum = 0; epnum < NUM_ENDPOINTS; epnum++) {
-
 				if (0x01 & int_bit)
 					_nbu2ss_ep_int(udc, epnum);
 
@@ -2546,9 +2532,8 @@ static int nbu2ss_ep_enable(
 	}
 
 	ep_type = usb_endpoint_type(desc);
-	if ((ep_type == USB_ENDPOINT_XFER_CONTROL)
-		|| (ep_type == USB_ENDPOINT_XFER_ISOC)) {
-
+	if ((ep_type == USB_ENDPOINT_XFER_CONTROL) ||
+	    (ep_type == USB_ENDPOINT_XFER_ISOC)) {
 		pr_err(" *** %s, bat bmAttributes\n", __func__);
 		return -EINVAL;
 	}
@@ -2557,9 +2542,7 @@ static int nbu2ss_ep_enable(
 	if (udc->vbus_active == 0)
 		return -ESHUTDOWN;
 
-	if ((!udc->driver)
-		|| (udc->gadget.speed == USB_SPEED_UNKNOWN)) {
-
+	if ((!udc->driver) || (udc->gadget.speed == USB_SPEED_UNKNOWN)) {
 		dev_err(ep->udc->dev, " *** %s, udc !!\n", __func__);
 		return -ESHUTDOWN;
 	}
@@ -2674,10 +2657,7 @@ static int nbu2ss_ep_queue(
 	}
 
 	req = container_of(_req, struct nbu2ss_req, req);
-	if (unlikely
-	    (!_req->complete || !_req->buf
-	     || !list_empty(&req->queue))) {
-
+	if (unlikely(!_req->complete || !_req->buf || !list_empty(&req->queue))) {
 		if (!_req->complete)
 			pr_err("udc: %s --- !_req->complete\n", __func__);
 
@@ -2736,7 +2716,6 @@ static int nbu2ss_ep_queue(
 	list_add_tail(&req->queue, &ep->queue);
 
 	if (bflag && !ep->stalled) {
-
 		result = _nbu2ss_start_transfer(udc, ep, req, FALSE);
 		if (result < 0) {
 			dev_err(udc->dev, " *** %s, result = %d\n", __func__,
@@ -2938,7 +2917,7 @@ static void  nbu2ss_ep_fifo_flush(struct usb_ep *_ep)
 }
 
 /*-------------------------------------------------------------------------*/
-static struct usb_ep_ops nbu2ss_ep_ops = {
+static const struct usb_ep_ops nbu2ss_ep_ops = {
 	.enable		= nbu2ss_ep_enable,
 	.disable	= nbu2ss_ep_disable,
 
@@ -2979,9 +2958,7 @@ static int nbu2ss_gad_get_frame(struct usb_gadget *pgadget)
 	if (data == 0)
 		return -EINVAL;
 
-	data = _nbu2ss_readl(&udc->p_regs->USB_ADDRESS) & FRAME;
-
-	return data;
+	return _nbu2ss_readl(&udc->p_regs->USB_ADDRESS) & FRAME;
 }
 
 /*-------------------------------------------------------------------------*/
@@ -3307,8 +3284,8 @@ static int nbu2ss_drv_remove(struct platform_device *pdev)
 	for (i = 0; i < NUM_ENDPOINTS; i++) {
 		ep = &udc->ep[i];
 		if (ep->virt_buf)
-			dma_free_coherent(NULL, PAGE_SIZE,
-				(void *)ep->virt_buf, ep->phys_buf);
+			dma_free_coherent(NULL, PAGE_SIZE, (void *)ep->virt_buf,
+					  ep->phys_buf);
 	}
 
 	/* Interrupt Handler - Release */
diff --git a/drivers/staging/fbtft/fb_agm1264k-fl.c b/drivers/staging/fbtft/fb_agm1264k-fl.c
index 7561385761e9..a6e3af74a904 100644
--- a/drivers/staging/fbtft/fb_agm1264k-fl.c
+++ b/drivers/staging/fbtft/fb_agm1264k-fl.c
@@ -264,6 +264,39 @@ construct_line_bitmap(struct fbtft_par *par, u8 *dest, signed short *src,
 	}
 }
 
+static void iterate_diffusion_matrix(u32 xres, u32 yres, int x,
+				     int y, signed short *convert_buf,
+				     signed short pixel, signed short error)
+{
+	u16 i, j;
+
+	/* diffusion matrix row */
+	for (i = 0; i < DIFFUSING_MATRIX_WIDTH; ++i)
+		/* diffusion matrix column */
+		for (j = 0; j < DIFFUSING_MATRIX_HEIGHT; ++j) {
+			signed short *write_pos;
+			signed char coeff;
+
+			/* skip pixels out of zone */
+			if (x + i < 0 || x + i >= xres || y + j >= yres)
+				continue;
+			write_pos = &convert_buf[(y + j) * xres + x + i];
+			coeff = diffusing_matrix[i][j];
+			if (-1 == coeff)
+				/* pixel itself */
+				*write_pos = pixel;
+			else {
+				signed short p = *write_pos + error * coeff;
+
+				if (p > WHITE)
+					p = WHITE;
+				if (p < BLACK)
+					p = BLACK;
+				*write_pos = p;
+			}
+		}
+}
+
 static int write_vmem(struct fbtft_par *par, size_t offset, size_t len)
 {
 	u16 *vmem16 = (u16 *)par->info->screen_buffer;
@@ -303,7 +336,6 @@ static int write_vmem(struct fbtft_par *par, size_t offset, size_t len)
 			signed short error_b = pixel - BLACK;
 			signed short error_w = pixel - WHITE;
 			signed short error;
-			u16 i, j;
 
 			/* what color close? */
 			if (abs(error_b) >= abs(error_w)) {
@@ -318,36 +350,10 @@ static int write_vmem(struct fbtft_par *par, size_t offset, size_t len)
 
 			error /= 8;
 
-			/* diffusion matrix row */
-			for (i = 0; i < DIFFUSING_MATRIX_WIDTH; ++i)
-				/* diffusion matrix column */
-				for (j = 0; j < DIFFUSING_MATRIX_HEIGHT; ++j) {
-					signed short *write_pos;
-					signed char coeff;
-
-					/* skip pixels out of zone */
-					if (x + i < 0 ||
-						x + i >= par->info->var.xres
-						|| y + j >= par->info->var.yres)
-						continue;
-					write_pos = &convert_buf[
-						(y + j) * par->info->var.xres +
-						x + i];
-					coeff = diffusing_matrix[i][j];
-					if (coeff == -1)
-						/* pixel itself */
-						*write_pos = pixel;
-					else {
-						signed short p = *write_pos +
-							error * coeff;
-
-						if (p > WHITE)
-							p = WHITE;
-						if (p < BLACK)
-							p = BLACK;
-						*write_pos = p;
-					}
-				}
+			iterate_diffusion_matrix(par->info->var.xres,
+						 par->info->var.yres,
+						 x, y, convert_buf,
+						 pixel, error);
 		}
 
 	/* 1 string = 2 pages */
diff --git a/drivers/staging/fbtft/fb_ili9325.c b/drivers/staging/fbtft/fb_ili9325.c
index c31e2e051d4a..19e33bab9cac 100644
--- a/drivers/staging/fbtft/fb_ili9325.c
+++ b/drivers/staging/fbtft/fb_ili9325.c
@@ -33,26 +33,23 @@
 			"04 16 2 7 6 3 2 1 7 7"
 
 static unsigned int bt = 6; /* VGL=Vci*4 , VGH=Vci*4 */
-module_param(bt, uint, 0);
+module_param(bt, uint, 0000);
 MODULE_PARM_DESC(bt, "Sets the factor used in the step-up circuits");
 
 static unsigned int vc = 0x03; /* Vci1=Vci*0.80 */
-module_param(vc, uint, 0);
-MODULE_PARM_DESC(vc,
-"Sets the ratio factor of Vci to generate the reference voltages Vci1");
+module_param(vc, uint, 0000);
+MODULE_PARM_DESC(vc, "Sets the ratio factor of Vci to generate the reference voltages Vci1");
 
 static unsigned int vrh = 0x0d; /* VREG1OUT=Vci*1.85 */
-module_param(vrh, uint, 0);
-MODULE_PARM_DESC(vrh,
-"Set the amplifying rate (1.6 ~ 1.9) of Vci applied to output the VREG1OUT");
+module_param(vrh, uint, 0000);
+MODULE_PARM_DESC(vrh, "Set the amplifying rate (1.6 ~ 1.9) of Vci applied to output the VREG1OUT");
 
 static unsigned int vdv = 0x12; /* VCOMH amplitude=VREG1OUT*0.98 */
-module_param(vdv, uint, 0);
-MODULE_PARM_DESC(vdv,
-"Select the factor of VREG1OUT to set the amplitude of Vcom");
+module_param(vdv, uint, 0000);
+MODULE_PARM_DESC(vdv, "Select the factor of VREG1OUT to set the amplitude of Vcom");
 
 static unsigned int vcm = 0x0a; /* VCOMH=VREG1OUT*0.735 */
-module_param(vcm, uint, 0);
+module_param(vcm, uint, 0000);
 MODULE_PARM_DESC(vcm, "Set the internal VcomH voltage");
 
 /*
diff --git a/drivers/staging/fbtft/fb_ili9481.c b/drivers/staging/fbtft/fb_ili9481.c
index 242adb3859bd..4e75f5abe2f9 100644
--- a/drivers/staging/fbtft/fb_ili9481.c
+++ b/drivers/staging/fbtft/fb_ili9481.c
@@ -27,7 +27,7 @@
 #define WIDTH		320
 #define HEIGHT		480
 
-static int default_init_sequence[] = {
+static s16 default_init_sequence[] = {
 	/* SLP_OUT - Sleep out */
 	-1, MIPI_DCS_EXIT_SLEEP_MODE,
 	-2, 50,
diff --git a/drivers/staging/fbtft/fb_ili9486.c b/drivers/staging/fbtft/fb_ili9486.c
index fa38d8885f0b..f4b314265f9e 100644
--- a/drivers/staging/fbtft/fb_ili9486.c
+++ b/drivers/staging/fbtft/fb_ili9486.c
@@ -26,7 +26,7 @@
 #define HEIGHT		480
 
 /* this init sequence matches PiScreen */
-static int default_init_sequence[] = {
+static s16 default_init_sequence[] = {
 	/* Interface Mode Control */
 	-1, 0xb0, 0x0,
 	-1, MIPI_DCS_EXIT_SLEEP_MODE,
diff --git a/drivers/staging/fbtft/fb_s6d02a1.c b/drivers/staging/fbtft/fb_s6d02a1.c
index 774b0ff69e6d..eb712aa0d692 100644
--- a/drivers/staging/fbtft/fb_s6d02a1.c
+++ b/drivers/staging/fbtft/fb_s6d02a1.c
@@ -24,7 +24,7 @@
 
 #define DRVNAME "fb_s6d02a1"
 
-static int default_init_sequence[] = {
+static s16 default_init_sequence[] = {
 
 	-1, 0xf0, 0x5a, 0x5a,
 
diff --git a/drivers/staging/fbtft/fb_st7735r.c b/drivers/staging/fbtft/fb_st7735r.c
index 6670f2bb62ec..710b74bbba97 100644
--- a/drivers/staging/fbtft/fb_st7735r.c
+++ b/drivers/staging/fbtft/fb_st7735r.c
@@ -25,7 +25,7 @@
 #define DEFAULT_GAMMA   "0F 1A 0F 18 2F 28 20 22 1F 1B 23 37 00 07 02 10\n" \
 			"0F 1B 0F 17 33 2C 29 2E 30 30 39 3F 00 07 03 10"
 
-static int default_init_sequence[] = {
+static s16 default_init_sequence[] = {
 	-1, MIPI_DCS_SOFT_RESET,
 	-2, 150,                               /* delay */
 
diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c
index 587f68aa466c..bbe89c9c4fb9 100644
--- a/drivers/staging/fbtft/fbtft-core.c
+++ b/drivers/staging/fbtft/fbtft-core.c
@@ -253,7 +253,8 @@ static int fbtft_backlight_update_status(struct backlight_device *bd)
 		"%s: polarity=%d, power=%d, fb_blank=%d\n",
 		__func__, polarity, bd->props.power, bd->props.fb_blank);
 
-	if ((bd->props.power == FB_BLANK_UNBLANK) && (bd->props.fb_blank == FB_BLANK_UNBLANK))
+	if ((bd->props.power == FB_BLANK_UNBLANK) &&
+	    (bd->props.fb_blank == FB_BLANK_UNBLANK))
 		gpio_set_value(par->gpio.led[0], polarity);
 	else
 		gpio_set_value(par->gpio.led[0], !polarity);
@@ -299,7 +300,8 @@ void fbtft_register_backlight(struct fbtft_par *par)
 		bl_props.state |= BL_CORE_DRIVER1;
 
 	bd = backlight_device_register(dev_driver_string(par->info->device),
-				par->info->device, par, &fbtft_bl_ops, &bl_props);
+				       par->info->device, par,
+				       &fbtft_bl_ops, &bl_props);
 	if (IS_ERR(bd)) {
 		dev_err(par->info->device,
 			"cannot register backlight device (%ld)\n",
@@ -350,9 +352,11 @@ static void fbtft_update_display(struct fbtft_par *par, unsigned int start_line,
 	bool timeit = false;
 	int ret = 0;
 
-	if (unlikely(par->debug & (DEBUG_TIME_FIRST_UPDATE | DEBUG_TIME_EACH_UPDATE))) {
+	if (unlikely(par->debug & (DEBUG_TIME_FIRST_UPDATE |
+			DEBUG_TIME_EACH_UPDATE))) {
 		if ((par->debug & DEBUG_TIME_EACH_UPDATE) ||
-				((par->debug & DEBUG_TIME_FIRST_UPDATE) && !par->first_update_done)) {
+				((par->debug & DEBUG_TIME_FIRST_UPDATE) &&
+				!par->first_update_done)) {
 			ts_start = ktime_get();
 			timeit = true;
 		}
@@ -361,15 +365,17 @@ static void fbtft_update_display(struct fbtft_par *par, unsigned int start_line,
 	/* Sanity checks */
 	if (start_line > end_line) {
 		dev_warn(par->info->device,
-			"%s: start_line=%u is larger than end_line=%u. Shouldn't happen, will do full display update\n",
-			__func__, start_line, end_line);
+			 "%s: start_line=%u is larger than end_line=%u. Shouldn't happen, will do full display update\n",
+			 __func__, start_line, end_line);
 		start_line = 0;
 		end_line = par->info->var.yres - 1;
 	}
-	if (start_line > par->info->var.yres - 1 || end_line > par->info->var.yres - 1) {
+	if (start_line > par->info->var.yres - 1 ||
+	    end_line > par->info->var.yres - 1) {
 		dev_warn(par->info->device,
 			"%s: start_line=%u or end_line=%u is larger than max=%d. Shouldn't happen, will do full display update\n",
-			__func__, start_line, end_line, par->info->var.yres - 1);
+			 __func__, start_line,
+			 end_line, par->info->var.yres - 1);
 		start_line = 0;
 		end_line = par->info->var.yres - 1;
 	}
@@ -660,12 +666,13 @@ struct fb_info *fbtft_framebuffer_alloc(struct fbtft_display *display,
 	unsigned int bpp = display->bpp;
 	unsigned int fps = display->fps;
 	int vmem_size, i;
-	int *init_sequence = display->init_sequence;
+	s16 *init_sequence = display->init_sequence;
 	char *gamma = display->gamma;
 	unsigned long *gamma_curves = NULL;
 
 	/* sanity check */
-	if (display->gamma_num * display->gamma_len > FBTFT_GAMMA_MAX_VALUES_TOTAL) {
+	if (display->gamma_num * display->gamma_len >
+			FBTFT_GAMMA_MAX_VALUES_TOTAL) {
 		dev_err(dev, "FBTFT_GAMMA_MAX_VALUES_TOTAL=%d is exceeded\n",
 			FBTFT_GAMMA_MAX_VALUES_TOTAL);
 		return NULL;
@@ -832,11 +839,13 @@ struct fb_info *fbtft_framebuffer_alloc(struct fbtft_display *display,
 #ifdef CONFIG_HAS_DMA
 		if (dma) {
 			dev->coherent_dma_mask = ~0;
-			txbuf = dmam_alloc_coherent(dev, txbuflen, &par->txbuf.dma, GFP_DMA);
+			txbuf = dmam_alloc_coherent(dev, txbuflen,
+						    &par->txbuf.dma, GFP_DMA);
 		} else
 #endif
 		{
-			txbuf = devm_kzalloc(par->info->device, txbuflen, GFP_KERNEL);
+			txbuf = devm_kzalloc(par->info->device,
+					     txbuflen, GFP_KERNEL);
 		}
 		if (!txbuf)
 			goto alloc_fail;
diff --git a/drivers/staging/fbtft/fbtft.h b/drivers/staging/fbtft/fbtft.h
index 89c4b5b76ce6..aacdde92cc2e 100644
--- a/drivers/staging/fbtft/fbtft.h
+++ b/drivers/staging/fbtft/fbtft.h
@@ -124,7 +124,7 @@ struct fbtft_display {
 	unsigned int bpp;
 	unsigned int fps;
 	int txbuflen;
-	int *init_sequence;
+	s16 *init_sequence;
 	char *gamma;
 	int gamma_num;
 	int gamma_len;
@@ -229,7 +229,7 @@ struct fbtft_par {
 		int led[16];
 		int aux[16];
 	} gpio;
-	int *init_sequence;
+	s16 *init_sequence;
 	struct {
 		struct mutex lock;
 		unsigned long *curves;
diff --git a/drivers/staging/fbtft/fbtft_device.c b/drivers/staging/fbtft/fbtft_device.c
index e9211831b6a1..de46f8d988d2 100644
--- a/drivers/staging/fbtft/fbtft_device.c
+++ b/drivers/staging/fbtft/fbtft_device.c
@@ -96,9 +96,9 @@ static unsigned int buswidth = 8;
 module_param(buswidth, uint, 0);
 MODULE_PARM_DESC(buswidth, "Display bus width, used with the custom argument");
 
-static int init[FBTFT_MAX_INIT_SEQUENCE];
+static s16 init[FBTFT_MAX_INIT_SEQUENCE];
 static int init_num;
-module_param_array(init, int, &init_num, 0);
+module_param_array(init, short, &init_num, 0);
 MODULE_PARM_DESC(init, "Init sequence, used with the custom argument");
 
 static unsigned long debug;
@@ -131,7 +131,7 @@ static void adafruit18_green_tab_set_addr_win(struct fbtft_par *par,
 		"D0 00 14 15 13 2C 42 43 4E 09 16 14 18 21\n" \
 		"D0 00 14 15 13 0B 43 55 53 0C 17 14 23 20"
 
-static int cberry28_init_sequence[] = {
+static s16 cberry28_init_sequence[] = {
 	/* turn off sleep mode */
 	-1, MIPI_DCS_EXIT_SLEEP_MODE,
 	-2, 120,
@@ -180,7 +180,7 @@ static int cberry28_init_sequence[] = {
 	-3,
 };
 
-static int hy28b_init_sequence[] = {
+static s16 hy28b_init_sequence[] = {
 	-1, 0x00e7, 0x0010, -1, 0x0000, 0x0001,
 	-1, 0x0001, 0x0100, -1, 0x0002, 0x0700,
 	-1, 0x0003, 0x1030, -1, 0x0004, 0x0000,
@@ -211,7 +211,7 @@ static int hy28b_init_sequence[] = {
 	"04 1F 4 7 7 0 7 7 6 0\n" \
 	"0F 00 1 7 4 0 0 0 6 7"
 
-static int pitft_init_sequence[] = {
+static s16 pitft_init_sequence[] = {
 	-1, MIPI_DCS_SOFT_RESET,
 	-2, 5,
 	-1, MIPI_DCS_SET_DISPLAY_OFF,
@@ -242,7 +242,7 @@ static int pitft_init_sequence[] = {
 	-3
 };
 
-static int waveshare32b_init_sequence[] = {
+static s16 waveshare32b_init_sequence[] = {
 	-1, 0xCB, 0x39, 0x2C, 0x00, 0x34, 0x02,
 	-1, 0xCF, 0x00, 0xC1, 0x30,
 	-1, 0xE8, 0x85, 0x00, 0x78,
diff --git a/drivers/staging/fbtft/flexfb.c b/drivers/staging/fbtft/flexfb.c
index ce0d254148e4..ded10718712b 100644
--- a/drivers/staging/fbtft/flexfb.c
+++ b/drivers/staging/fbtft/flexfb.c
@@ -38,9 +38,9 @@ static unsigned int height;
 module_param(height, uint, 0);
 MODULE_PARM_DESC(height, "Display height");
 
-static int init[512];
+static s16 init[512];
 static int init_num;
-module_param_array(init, int, &init_num, 0);
+module_param_array(init, short, &init_num, 0);
 MODULE_PARM_DESC(init, "Init sequence");
 
 static unsigned int setaddrwin;
@@ -63,68 +63,316 @@ static bool latched;
 module_param(latched, bool, 0);
 MODULE_PARM_DESC(latched, "Use with latched 16-bit databus");
 
-static int *initp;
+static s16 *initp;
 static int initp_num;
 
 /* default init sequences */
-static int st7735r_init[] = {
--1, 0x01, -2, 150, -1, 0x11, -2, 500, -1, 0xB1, 0x01, 0x2C, 0x2D, -1, 0xB2, 0x01, 0x2C, 0x2D, -1, 0xB3, 0x01, 0x2C, 0x2D, 0x01, 0x2C, 0x2D,
--1, 0xB4, 0x07, -1, 0xC0, 0xA2, 0x02, 0x84, -1, 0xC1, 0xC5, -1, 0xC2, 0x0A, 0x00, -1, 0xC3, 0x8A, 0x2A, -1, 0xC4, 0x8A, 0xEE, -1, 0xC5, 0x0E,
--1, 0x20, -1, 0x36, 0xC0, -1, 0x3A, 0x05, -1, 0xE0, 0x0f, 0x1a, 0x0f, 0x18, 0x2f, 0x28, 0x20, 0x22, 0x1f, 0x1b, 0x23, 0x37, 0x00, 0x07, 0x02, 0x10,
--1, 0xE1, 0x0f, 0x1b, 0x0f, 0x17, 0x33, 0x2c, 0x29, 0x2e, 0x30, 0x30, 0x39, 0x3f, 0x00, 0x07, 0x03, 0x10, -1, 0x29, -2, 100, -1, 0x13, -2, 10, -3 };
-
-static int ssd1289_init[] = {
--1, 0x00, 0x0001, -1, 0x03, 0xA8A4, -1, 0x0C, 0x0000, -1, 0x0D, 0x080C, -1, 0x0E, 0x2B00, -1, 0x1E, 0x00B7, -1, 0x01, 0x2B3F, -1, 0x02, 0x0600,
--1, 0x10, 0x0000, -1, 0x11, 0x6070, -1, 0x05, 0x0000, -1, 0x06, 0x0000, -1, 0x16, 0xEF1C, -1, 0x17, 0x0003, -1, 0x07, 0x0233, -1, 0x0B, 0x0000,
--1, 0x0F, 0x0000, -1, 0x41, 0x0000, -1, 0x42, 0x0000, -1, 0x48, 0x0000, -1, 0x49, 0x013F, -1, 0x4A, 0x0000, -1, 0x4B, 0x0000, -1, 0x44, 0xEF00,
--1, 0x45, 0x0000, -1, 0x46, 0x013F, -1, 0x30, 0x0707, -1, 0x31, 0x0204, -1, 0x32, 0x0204, -1, 0x33, 0x0502, -1, 0x34, 0x0507, -1, 0x35, 0x0204,
--1, 0x36, 0x0204, -1, 0x37, 0x0502, -1, 0x3A, 0x0302, -1, 0x3B, 0x0302, -1, 0x23, 0x0000, -1, 0x24, 0x0000, -1, 0x25, 0x8000, -1, 0x4f, 0x0000,
--1, 0x4e, 0x0000, -1, 0x22, -3 };
-
-static int hx8340bn_init[] = {
--1, 0xC1, 0xFF, 0x83, 0x40, -1, 0x11, -2, 150, -1, 0xCA, 0x70, 0x00, 0xD9, -1, 0xB0, 0x01, 0x11,
--1, 0xC9, 0x90, 0x49, 0x10, 0x28, 0x28, 0x10, 0x00, 0x06, -2, 20, -1, 0xC2, 0x60, 0x71, 0x01, 0x0E, 0x05, 0x02, 0x09, 0x31, 0x0A,
--1, 0xC3, 0x67, 0x30, 0x61, 0x17, 0x48, 0x07, 0x05, 0x33, -2, 10, -1, 0xB5, 0x35, 0x20, 0x45, -1, 0xB4, 0x33, 0x25, 0x4C, -2, 10,
--1, 0x3A, 0x05, -1, 0x29, -2, 10, -3 };
-
-static int ili9225_init[] = {
--1, 0x0001, 0x011C, -1, 0x0002, 0x0100, -1, 0x0003, 0x1030, -1, 0x0008, 0x0808, -1, 0x000C, 0x0000, -1, 0x000F, 0x0A01, -1, 0x0020, 0x0000,
--1, 0x0021, 0x0000, -2, 50, -1, 0x0010, 0x0A00, -1, 0x0011, 0x1038, -2, 50, -1, 0x0012, 0x1121, -1, 0x0013, 0x004E, -1, 0x0014, 0x676F,
--1, 0x0030, 0x0000, -1, 0x0031, 0x00DB, -1, 0x0032, 0x0000, -1, 0x0033, 0x0000, -1, 0x0034, 0x00DB, -1, 0x0035, 0x0000, -1, 0x0036, 0x00AF,
--1, 0x0037, 0x0000, -1, 0x0038, 0x00DB, -1, 0x0039, 0x0000, -1, 0x0050, 0x0000, -1, 0x0051, 0x060A, -1, 0x0052, 0x0D0A, -1, 0x0053, 0x0303,
--1, 0x0054, 0x0A0D, -1, 0x0055, 0x0A06, -1, 0x0056, 0x0000, -1, 0x0057, 0x0303, -1, 0x0058, 0x0000, -1, 0x0059, 0x0000, -2, 50,
--1, 0x0007, 0x1017, -2, 50, -3 };
-
-static int ili9320_init[] = {
--1, 0x00E5, 0x8000, -1, 0x0000, 0x0001, -1, 0x0001, 0x0100, -1, 0x0002, 0x0700, -1, 0x0003, 0x1030, -1, 0x0004, 0x0000, -1, 0x0008, 0x0202,
--1, 0x0009, 0x0000, -1, 0x000A, 0x0000, -1, 0x000C, 0x0000, -1, 0x000D, 0x0000, -1, 0x000F, 0x0000, -1, 0x0010, 0x0000, -1, 0x0011, 0x0007,
--1, 0x0012, 0x0000, -1, 0x0013, 0x0000, -2, 200, -1, 0x0010, 0x17B0, -1, 0x0011, 0x0031, -2, 50, -1, 0x0012, 0x0138, -2, 50, -1, 0x0013, 0x1800,
--1, 0x0029, 0x0008, -2, 50, -1, 0x0020, 0x0000, -1, 0x0021, 0x0000, -1, 0x0030, 0x0000, -1, 0x0031, 0x0505, -1, 0x0032, 0x0004,
--1, 0x0035, 0x0006, -1, 0x0036, 0x0707, -1, 0x0037, 0x0105, -1, 0x0038, 0x0002, -1, 0x0039, 0x0707, -1, 0x003C, 0x0704, -1, 0x003D, 0x0807,
--1, 0x0050, 0x0000, -1, 0x0051, 0x00EF, -1, 0x0052, 0x0000, -1, 0x0053, 0x013F, -1, 0x0060, 0x2700, -1, 0x0061, 0x0001, -1, 0x006A, 0x0000,
--1, 0x0080, 0x0000, -1, 0x0081, 0x0000, -1, 0x0082, 0x0000, -1, 0x0083, 0x0000, -1, 0x0084, 0x0000, -1, 0x0085, 0x0000, -1, 0x0090, 0x0010,
--1, 0x0092, 0x0000, -1, 0x0093, 0x0003, -1, 0x0095, 0x0110, -1, 0x0097, 0x0000, -1, 0x0098, 0x0000, -1, 0x0007, 0x0173, -3 };
-
-static int ili9325_init[] = {
--1, 0x00E3, 0x3008, -1, 0x00E7, 0x0012, -1, 0x00EF, 0x1231, -1, 0x0001, 0x0100, -1, 0x0002, 0x0700, -1, 0x0003, 0x1030, -1, 0x0004, 0x0000,
--1, 0x0008, 0x0207, -1, 0x0009, 0x0000, -1, 0x000A, 0x0000, -1, 0x000C, 0x0000, -1, 0x000D, 0x0000, -1, 0x000F, 0x0000, -1, 0x0010, 0x0000,
--1, 0x0011, 0x0007, -1, 0x0012, 0x0000, -1, 0x0013, 0x0000, -2, 200, -1, 0x0010, 0x1690, -1, 0x0011, 0x0223, -2, 50, -1, 0x0012, 0x000D, -2, 50,
--1, 0x0013, 0x1200, -1, 0x0029, 0x000A, -1, 0x002B, 0x000C, -2, 50, -1, 0x0020, 0x0000, -1, 0x0021, 0x0000, -1, 0x0030, 0x0000,
--1, 0x0031, 0x0506, -1, 0x0032, 0x0104, -1, 0x0035, 0x0207, -1, 0x0036, 0x000F, -1, 0x0037, 0x0306, -1, 0x0038, 0x0102, -1, 0x0039, 0x0707,
--1, 0x003C, 0x0702, -1, 0x003D, 0x1604, -1, 0x0050, 0x0000, -1, 0x0051, 0x00EF, -1, 0x0052, 0x0000, -1, 0x0053, 0x013F, -1, 0x0060, 0xA700,
--1, 0x0061, 0x0001, -1, 0x006A, 0x0000, -1, 0x0080, 0x0000, -1, 0x0081, 0x0000, -1, 0x0082, 0x0000, -1, 0x0083, 0x0000, -1, 0x0084, 0x0000,
--1, 0x0085, 0x0000, -1, 0x0090, 0x0010, -1, 0x0092, 0x0600, -1, 0x0007, 0x0133, -3 };
-
-static int ili9341_init[] = {
--1, 0x28, -2, 20, -1, 0xCF, 0x00, 0x83, 0x30, -1, 0xED, 0x64, 0x03, 0x12, 0x81, -1, 0xE8, 0x85, 0x01, 0x79,
--1, 0xCB, 0x39, 0x2c, 0x00, 0x34, 0x02, -1, 0xF7, 0x20, -1, 0xEA, 0x00, 0x00, -1, 0xC0, 0x26, -1, 0xC1, 0x11,
--1, 0xC5, 0x35, 0x3E, -1, 0xC7, 0xBE, -1, 0xB1, 0x00, 0x1B, -1, 0xB6, 0x0a, 0x82, 0x27, 0x00, -1, 0xB7, 0x07,
--1, 0x3A, 0x55, -1, 0x36, 0x48, -1, 0x11, -2, 120, -1, 0x29, -2, 20, -3 };
-
-static int ssd1351_init[] = { -1, 0xfd, 0x12, -1, 0xfd, 0xb1, -1, 0xae, -1, 0xb3, 0xf1, -1, 0xca, 0x7f, -1, 0xa0, 0x74,
-			      -1, 0x15, 0x00, 0x7f, -1, 0x75, 0x00, 0x7f, -1, 0xa1, 0x00, -1, 0xa2, 0x00, -1, 0xb5, 0x00,
-			      -1, 0xab, 0x01, -1, 0xb1, 0x32, -1, 0xb4, 0xa0, 0xb5, 0x55, -1, 0xbb, 0x17, -1, 0xbe, 0x05,
-			      -1, 0xc1, 0xc8, 0x80, 0xc8, -1, 0xc7, 0x0f, -1, 0xb6, 0x01, -1, 0xa6, -1, 0xaf, -3 };
+static s16 st7735r_init[] = {
+	-1, 0x01,
+	-2, 150,
+	-1, 0x11,
+	-2, 500,
+	-1, 0xB1, 0x01, 0x2C, 0x2D,
+	-1, 0xB2, 0x01, 0x2C, 0x2D,
+	-1, 0xB3, 0x01, 0x2C, 0x2D, 0x01, 0x2C, 0x2D,
+	-1, 0xB4, 0x07,
+	-1, 0xC0, 0xA2, 0x02, 0x84,
+	-1, 0xC1, 0xC5,
+	-1, 0xC2, 0x0A, 0x00,
+	-1, 0xC3, 0x8A, 0x2A,
+	-1, 0xC4, 0x8A, 0xEE,
+	-1, 0xC5, 0x0E,
+	-1, 0x20,
+	-1, 0x36, 0xC0,
+	-1, 0x3A, 0x05,
+	-1, 0xE0, 0x0f, 0x1a, 0x0f, 0x18, 0x2f, 0x28, 0x20, 0x22,
+	    0x1f, 0x1b, 0x23, 0x37, 0x00, 0x07, 0x02, 0x10,
+	-1, 0xE1, 0x0f, 0x1b, 0x0f, 0x17, 0x33, 0x2c, 0x29, 0x2e,
+	    0x30, 0x30, 0x39, 0x3f, 0x00, 0x07, 0x03, 0x10,
+	-1, 0x29,
+	-2, 100,
+	-1, 0x13,
+	-2, 10,
+	-3
+};
+
+static s16 ssd1289_init[] = {
+	-1, 0x00, 0x0001,
+	-1, 0x03, 0xA8A4,
+	-1, 0x0C, 0x0000,
+	-1, 0x0D, 0x080C,
+	-1, 0x0E, 0x2B00,
+	-1, 0x1E, 0x00B7,
+	-1, 0x01, 0x2B3F,
+	-1, 0x02, 0x0600,
+	-1, 0x10, 0x0000,
+	-1, 0x11, 0x6070,
+	-1, 0x05, 0x0000,
+	-1, 0x06, 0x0000,
+	-1, 0x16, 0xEF1C,
+	-1, 0x17, 0x0003,
+	-1, 0x07, 0x0233,
+	-1, 0x0B, 0x0000,
+	-1, 0x0F, 0x0000,
+	-1, 0x41, 0x0000,
+	-1, 0x42, 0x0000,
+	-1, 0x48, 0x0000,
+	-1, 0x49, 0x013F,
+	-1, 0x4A, 0x0000,
+	-1, 0x4B, 0x0000,
+	-1, 0x44, 0xEF00,
+	-1, 0x45, 0x0000,
+	-1, 0x46, 0x013F,
+	-1, 0x30, 0x0707,
+	-1, 0x31, 0x0204,
+	-1, 0x32, 0x0204,
+	-1, 0x33, 0x0502,
+	-1, 0x34, 0x0507,
+	-1, 0x35, 0x0204,
+	-1, 0x36, 0x0204,
+	-1, 0x37, 0x0502,
+	-1, 0x3A, 0x0302,
+	-1, 0x3B, 0x0302,
+	-1, 0x23, 0x0000,
+	-1, 0x24, 0x0000,
+	-1, 0x25, 0x8000,
+	-1, 0x4f, 0x0000,
+	-1, 0x4e, 0x0000,
+	-1, 0x22,
+	-3
+};
+
+static s16 hx8340bn_init[] = {
+	-1, 0xC1, 0xFF, 0x83, 0x40,
+	-1, 0x11,
+	-2, 150,
+	-1, 0xCA, 0x70, 0x00, 0xD9,
+	-1, 0xB0, 0x01, 0x11,
+	-1, 0xC9, 0x90, 0x49, 0x10, 0x28, 0x28, 0x10, 0x00, 0x06,
+	-2, 20,
+	-1, 0xC2, 0x60, 0x71, 0x01, 0x0E, 0x05, 0x02, 0x09, 0x31, 0x0A,
+	-1, 0xC3, 0x67, 0x30, 0x61, 0x17, 0x48, 0x07, 0x05, 0x33,
+	-2, 10,
+	-1, 0xB5, 0x35, 0x20, 0x45,
+	-1, 0xB4, 0x33, 0x25, 0x4C,
+	-2, 10,
+	-1, 0x3A, 0x05,
+	-1, 0x29,
+	-2, 10,
+	-3
+};
+
+static s16 ili9225_init[] = {
+	-1, 0x0001, 0x011C,
+	-1, 0x0002, 0x0100,
+	-1, 0x0003, 0x1030,
+	-1, 0x0008, 0x0808,
+	-1, 0x000C, 0x0000,
+	-1, 0x000F, 0x0A01,
+	-1, 0x0020, 0x0000,
+	-1, 0x0021, 0x0000,
+	-2, 50,
+	-1, 0x0010, 0x0A00,
+	-1, 0x0011, 0x1038,
+	-2, 50,
+	-1, 0x0012, 0x1121,
+	-1, 0x0013, 0x004E,
+	-1, 0x0014, 0x676F,
+	-1, 0x0030, 0x0000,
+	-1, 0x0031, 0x00DB,
+	-1, 0x0032, 0x0000,
+	-1, 0x0033, 0x0000,
+	-1, 0x0034, 0x00DB,
+	-1, 0x0035, 0x0000,
+	-1, 0x0036, 0x00AF,
+	-1, 0x0037, 0x0000,
+	-1, 0x0038, 0x00DB,
+	-1, 0x0039, 0x0000,
+	-1, 0x0050, 0x0000,
+	-1, 0x0051, 0x060A,
+	-1, 0x0052, 0x0D0A,
+	-1, 0x0053, 0x0303,
+	-1, 0x0054, 0x0A0D,
+	-1, 0x0055, 0x0A06,
+	-1, 0x0056, 0x0000,
+	-1, 0x0057, 0x0303,
+	-1, 0x0058, 0x0000,
+	-1, 0x0059, 0x0000,
+	-2, 50,
+	-1, 0x0007, 0x1017,
+	-2, 50,
+	-3
+};
+
+static s16 ili9320_init[] = {
+	-1, 0x00E5, 0x8000,
+	-1, 0x0000, 0x0001,
+	-1, 0x0001, 0x0100,
+	-1, 0x0002, 0x0700,
+	-1, 0x0003, 0x1030,
+	-1, 0x0004, 0x0000,
+	-1, 0x0008, 0x0202,
+	-1, 0x0009, 0x0000,
+	-1, 0x000A, 0x0000,
+	-1, 0x000C, 0x0000,
+	-1, 0x000D, 0x0000,
+	-1, 0x000F, 0x0000,
+	-1, 0x0010, 0x0000,
+	-1, 0x0011, 0x0007,
+	-1, 0x0012, 0x0000,
+	-1, 0x0013, 0x0000,
+	-2, 200,
+	-1, 0x0010, 0x17B0,
+	-1, 0x0011, 0x0031,
+	-2, 50,
+	-1, 0x0012, 0x0138,
+	-2, 50,
+	-1, 0x0013, 0x1800,
+	-1, 0x0029, 0x0008,
+	-2, 50,
+	-1, 0x0020, 0x0000,
+	-1, 0x0021, 0x0000,
+	-1, 0x0030, 0x0000,
+	-1, 0x0031, 0x0505,
+	-1, 0x0032, 0x0004,
+	-1, 0x0035, 0x0006,
+	-1, 0x0036, 0x0707,
+	-1, 0x0037, 0x0105,
+	-1, 0x0038, 0x0002,
+	-1, 0x0039, 0x0707,
+	-1, 0x003C, 0x0704,
+	-1, 0x003D, 0x0807,
+	-1, 0x0050, 0x0000,
+	-1, 0x0051, 0x00EF,
+	-1, 0x0052, 0x0000,
+	-1, 0x0053, 0x013F,
+	-1, 0x0060, 0x2700,
+	-1, 0x0061, 0x0001,
+	-1, 0x006A, 0x0000,
+	-1, 0x0080, 0x0000,
+	-1, 0x0081, 0x0000,
+	-1, 0x0082, 0x0000,
+	-1, 0x0083, 0x0000,
+	-1, 0x0084, 0x0000,
+	-1, 0x0085, 0x0000,
+	-1, 0x0090, 0x0010,
+	-1, 0x0092, 0x0000,
+	-1, 0x0093, 0x0003,
+	-1, 0x0095, 0x0110,
+	-1, 0x0097, 0x0000,
+	-1, 0x0098, 0x0000,
+	-1, 0x0007, 0x0173,
+	-3
+};
+
+static s16 ili9325_init[] = {
+	-1, 0x00E3, 0x3008,
+	-1, 0x00E7, 0x0012,
+	-1, 0x00EF, 0x1231,
+	-1, 0x0001, 0x0100,
+	-1, 0x0002, 0x0700,
+	-1, 0x0003, 0x1030,
+	-1, 0x0004, 0x0000,
+	-1, 0x0008, 0x0207,
+	-1, 0x0009, 0x0000,
+	-1, 0x000A, 0x0000,
+	-1, 0x000C, 0x0000,
+	-1, 0x000D, 0x0000,
+	-1, 0x000F, 0x0000,
+	-1, 0x0010, 0x0000,
+	-1, 0x0011, 0x0007,
+	-1, 0x0012, 0x0000,
+	-1, 0x0013, 0x0000,
+	-2, 200,
+	-1, 0x0010, 0x1690,
+	-1, 0x0011, 0x0223,
+	-2, 50,
+	-1, 0x0012, 0x000D,
+	-2, 50,
+	-1, 0x0013, 0x1200,
+	-1, 0x0029, 0x000A,
+	-1, 0x002B, 0x000C,
+	-2, 50,
+	-1, 0x0020, 0x0000,
+	-1, 0x0021, 0x0000,
+	-1, 0x0030, 0x0000,
+	-1, 0x0031, 0x0506,
+	-1, 0x0032, 0x0104,
+	-1, 0x0035, 0x0207,
+	-1, 0x0036, 0x000F,
+	-1, 0x0037, 0x0306,
+	-1, 0x0038, 0x0102,
+	-1, 0x0039, 0x0707,
+	-1, 0x003C, 0x0702,
+	-1, 0x003D, 0x1604,
+	-1, 0x0050, 0x0000,
+	-1, 0x0051, 0x00EF,
+	-1, 0x0052, 0x0000,
+	-1, 0x0053, 0x013F,
+	-1, 0x0060, 0xA700,
+	-1, 0x0061, 0x0001,
+	-1, 0x006A, 0x0000,
+	-1, 0x0080, 0x0000,
+	-1, 0x0081, 0x0000,
+	-1, 0x0082, 0x0000,
+	-1, 0x0083, 0x0000,
+	-1, 0x0084, 0x0000,
+	-1, 0x0085, 0x0000,
+	-1, 0x0090, 0x0010,
+	-1, 0x0092, 0x0600,
+	-1, 0x0007, 0x0133,
+	-3
+};
+
+static s16 ili9341_init[] = {
+	-1, 0x28,
+	-2, 20,
+	-1, 0xCF, 0x00, 0x83, 0x30,
+	-1, 0xED, 0x64, 0x03, 0x12, 0x81,
+	-1, 0xE8, 0x85, 0x01, 0x79,
+	-1, 0xCB, 0x39, 0x2c, 0x00, 0x34, 0x02,
+	-1, 0xF7, 0x20,
+	-1, 0xEA, 0x00, 0x00,
+	-1, 0xC0, 0x26,
+	-1, 0xC1, 0x11,
+	-1, 0xC5, 0x35, 0x3E,
+	-1, 0xC7, 0xBE,
+	-1, 0xB1, 0x00, 0x1B,
+	-1, 0xB6, 0x0a, 0x82, 0x27, 0x00,
+	-1, 0xB7, 0x07,
+	-1, 0x3A, 0x55,
+	-1, 0x36, 0x48,
+	-1, 0x11,
+	-2, 120,
+	-1, 0x29,
+	-2, 20,
+	-3
+};
+
+static s16 ssd1351_init[] = {
+	-1, 0xfd, 0x12,
+	-1, 0xfd, 0xb1,
+	-1, 0xae,
+	-1, 0xb3, 0xf1,
+	-1, 0xca, 0x7f,
+	-1, 0xa0, 0x74,
+	-1, 0x15, 0x00, 0x7f,
+	-1, 0x75, 0x00, 0x7f,
+	-1, 0xa1, 0x00,
+	-1, 0xa2, 0x00,
+	-1, 0xb5, 0x00,
+	-1, 0xab, 0x01,
+	-1, 0xb1, 0x32,
+	-1, 0xb4, 0xa0, 0xb5, 0x55,
+	-1, 0xbb, 0x17,
+	-1, 0xbe, 0x05,
+	-1, 0xc1, 0xc8, 0x80, 0xc8,
+	-1, 0xc7, 0x0f,
+	-1, 0xb6, 0x01,
+	-1, 0xa6,
+	-1, 0xaf,
+	-3
+};
 
 /**
  * struct flexfb_lcd_controller - Describes the LCD controller properties
@@ -142,7 +390,7 @@ struct flexfb_lcd_controller {
 	unsigned int height;
 	unsigned int setaddrwin;
 	unsigned int regwidth;
-	int *init_seq;
+	s16 *init_seq;
 	int init_seq_sz;
 };
 
@@ -582,6 +830,7 @@ static const struct platform_device_id flexfb_platform_ids[] = {
 	{ "flexpfb", 0 },
 	{ },
 };
+MODULE_DEVICE_TABLE(platform, flexfb_platform_ids);
 
 static struct platform_driver flexfb_platform_driver = {
 	.driver = {
diff --git a/drivers/staging/fsl-mc/bus/Kconfig b/drivers/staging/fsl-mc/bus/Kconfig
index 1f959339c671..5c009ab48f00 100644
--- a/drivers/staging/fsl-mc/bus/Kconfig
+++ b/drivers/staging/fsl-mc/bus/Kconfig
@@ -1,25 +1,17 @@
 #
-# Freescale Management Complex (MC) bus drivers
+# DPAA2 fsl-mc bus
 #
-# Copyright (C) 2014 Freescale Semiconductor, Inc.
+# Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
 #
 # This file is released under the GPLv2
 #
 
 config FSL_MC_BUS
-	bool "Freescale Management Complex (MC) bus driver"
-	depends on OF && ARM64
+	bool "QorIQ DPAA2 fsl-mc bus driver"
+	depends on OF && ARCH_LAYERSCAPE
 	select GENERIC_MSI_IRQ_DOMAIN
 	help
-	  Driver to enable the bus infrastructure for the Freescale
-          QorIQ Management Complex (fsl-mc). The fsl-mc is a hardware
-	  module of the QorIQ LS2 SoCs, that does resource management
-	  for hardware building-blocks in the SoC that can be used
-	  to dynamically create networking hardware objects such as
-	  network interfaces (NICs), crypto accelerator instances,
-	  or L2 switches.
-
-	  Only enable this option when building the kernel for
-	  Freescale QorQIQ LS2xxxx SoCs.
-
-
+	  Driver to enable the bus infrastructure for the QorIQ DPAA2
+	  architecture.  The fsl-mc bus driver handles discovery of
+	  DPAA2 objects (which are represented as Linux devices) and
+	  binding objects to drivers.
diff --git a/drivers/staging/fsl-mc/include/dpbp-cmd.h b/drivers/staging/fsl-mc/bus/dpbp-cmd.h
index 2860411ddb51..7d86539b5414 100644
--- a/drivers/staging/fsl-mc/include/dpbp-cmd.h
+++ b/drivers/staging/fsl-mc/bus/dpbp-cmd.h
@@ -1,4 +1,5 @@
-/* Copyright 2013-2016 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -11,7 +12,6 @@
  * names of any contributors may be used to endorse or promote products
  * derived from this software without specific prior written permission.
  *
- *
  * ALTERNATIVELY, this software may be distributed under the terms of the
  * GNU General Public License ("GPL") as published by the Free Software
  * Foundation, either version 2 of that License or (at your option) any
@@ -33,37 +33,48 @@
 #define _FSL_DPBP_CMD_H
 
 /* DPBP Version */
-#define DPBP_VER_MAJOR				2
+#define DPBP_VER_MAJOR				3
 #define DPBP_VER_MINOR				2
 
+/* Command versioning */
+#define DPBP_CMD_BASE_VERSION			1
+#define DPBP_CMD_ID_OFFSET			4
+
+#define DPBP_CMD(id)	((id << DPBP_CMD_ID_OFFSET) | DPBP_CMD_BASE_VERSION)
+
 /* Command IDs */
-#define DPBP_CMDID_CLOSE				0x800
-#define DPBP_CMDID_OPEN					0x804
-#define DPBP_CMDID_CREATE				0x904
-#define DPBP_CMDID_DESTROY				0x900
-
-#define DPBP_CMDID_ENABLE				0x002
-#define DPBP_CMDID_DISABLE				0x003
-#define DPBP_CMDID_GET_ATTR				0x004
-#define DPBP_CMDID_RESET				0x005
-#define DPBP_CMDID_IS_ENABLED				0x006
-
-#define DPBP_CMDID_SET_IRQ				0x010
-#define DPBP_CMDID_GET_IRQ				0x011
-#define DPBP_CMDID_SET_IRQ_ENABLE			0x012
-#define DPBP_CMDID_GET_IRQ_ENABLE			0x013
-#define DPBP_CMDID_SET_IRQ_MASK				0x014
-#define DPBP_CMDID_GET_IRQ_MASK				0x015
-#define DPBP_CMDID_GET_IRQ_STATUS			0x016
-#define DPBP_CMDID_CLEAR_IRQ_STATUS			0x017
-
-#define DPBP_CMDID_SET_NOTIFICATIONS		0x01b0
-#define DPBP_CMDID_GET_NOTIFICATIONS		0x01b1
+#define DPBP_CMDID_CLOSE		DPBP_CMD(0x800)
+#define DPBP_CMDID_OPEN			DPBP_CMD(0x804)
+#define DPBP_CMDID_CREATE		DPBP_CMD(0x904)
+#define DPBP_CMDID_DESTROY		DPBP_CMD(0x984)
+#define DPBP_CMDID_GET_API_VERSION	DPBP_CMD(0xa04)
+
+#define DPBP_CMDID_ENABLE		DPBP_CMD(0x002)
+#define DPBP_CMDID_DISABLE		DPBP_CMD(0x003)
+#define DPBP_CMDID_GET_ATTR		DPBP_CMD(0x004)
+#define DPBP_CMDID_RESET		DPBP_CMD(0x005)
+#define DPBP_CMDID_IS_ENABLED		DPBP_CMD(0x006)
+
+#define DPBP_CMDID_SET_IRQ		DPBP_CMD(0x010)
+#define DPBP_CMDID_GET_IRQ		DPBP_CMD(0x011)
+#define DPBP_CMDID_SET_IRQ_ENABLE	DPBP_CMD(0x012)
+#define DPBP_CMDID_GET_IRQ_ENABLE	DPBP_CMD(0x013)
+#define DPBP_CMDID_SET_IRQ_MASK		DPBP_CMD(0x014)
+#define DPBP_CMDID_GET_IRQ_MASK		DPBP_CMD(0x015)
+#define DPBP_CMDID_GET_IRQ_STATUS	DPBP_CMD(0x016)
+#define DPBP_CMDID_CLEAR_IRQ_STATUS	DPBP_CMD(0x017)
+
+#define DPBP_CMDID_SET_NOTIFICATIONS	DPBP_CMD(0x01b0)
+#define DPBP_CMDID_GET_NOTIFICATIONS	DPBP_CMD(0x01b1)
 
 struct dpbp_cmd_open {
 	__le32 dpbp_id;
 };
 
+struct dpbp_cmd_destroy {
+	__le32 object_id;
+};
+
 #define DPBP_ENABLE			0x1
 
 struct dpbp_rsp_is_enabled {
diff --git a/drivers/staging/fsl-mc/bus/dpbp.c b/drivers/staging/fsl-mc/bus/dpbp.c
index 5d4cd812a400..cf4782f6a049 100644
--- a/drivers/staging/fsl-mc/bus/dpbp.c
+++ b/drivers/staging/fsl-mc/bus/dpbp.c
@@ -1,4 +1,5 @@
-/* Copyright 2013-2016 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -11,7 +12,6 @@
  * names of any contributors may be used to endorse or promote products
  * derived from this software without specific prior written permission.
  *
- *
  * ALTERNATIVELY, this software may be distributed under the terms of the
  * GNU General Public License ("GPL") as published by the Free Software
  * Foundation, either version 2 of that License or (at your option) any
@@ -32,7 +32,8 @@
 #include "../include/mc-sys.h"
 #include "../include/mc-cmd.h"
 #include "../include/dpbp.h"
-#include "../include/dpbp-cmd.h"
+
+#include "dpbp-cmd.h"
 
 /**
  * dpbp_open() - Open a control session for the specified object.
@@ -107,28 +108,26 @@ EXPORT_SYMBOL(dpbp_close);
 /**
  * dpbp_create() - Create the DPBP object.
  * @mc_io:	Pointer to MC portal's I/O object
+ * @dprc_token:	Parent container token; '0' for default container
  * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
  * @cfg:	Configuration structure
- * @token:	Returned token; use in subsequent API calls
+ * @obj_id:	Returned object id; use in subsequent API calls
  *
  * Create the DPBP object, allocate required resources and
  * perform required initialization.
  *
- * The object can be created either by declaring it in the
- * DPL file, or by calling this function.
- * This function returns a unique authentication token,
- * associated with the specific object ID and the specific MC
- * portal; this token must be used in all subsequent calls to
- * this specific object. For objects that are created using the
- * DPL file, call dpbp_open function to get an authentication
- * token first.
+ * This function accepts an authentication token of a parent
+ * container that this object should be assigned to and returns
+ * an object id. This object_id will be used in all subsequent calls to
+ * this specific object.
  *
  * Return:	'0' on Success; Error code otherwise.
  */
 int dpbp_create(struct fsl_mc_io *mc_io,
+		u16 dprc_token,
 		u32 cmd_flags,
 		const struct dpbp_cfg *cfg,
-		u16 *token)
+		u32 *obj_id)
 {
 	struct mc_command cmd = { 0 };
 	int err;
@@ -137,7 +136,7 @@ int dpbp_create(struct fsl_mc_io *mc_io,
 
 	/* prepare command */
 	cmd.header = mc_encode_cmd_header(DPBP_CMDID_CREATE,
-					  cmd_flags, 0);
+					  cmd_flags, dprc_token);
 
 	/* send command to mc*/
 	err = mc_send_command(mc_io, &cmd);
@@ -145,7 +144,7 @@ int dpbp_create(struct fsl_mc_io *mc_io,
 		return err;
 
 	/* retrieve response parameters */
-	*token = mc_cmd_hdr_read_token(&cmd);
+	*obj_id = mc_cmd_read_object_id(&cmd);
 
 	return 0;
 }
@@ -153,20 +152,25 @@ int dpbp_create(struct fsl_mc_io *mc_io,
 /**
  * dpbp_destroy() - Destroy the DPBP object and release all its resources.
  * @mc_io:	Pointer to MC portal's I/O object
+ * @dprc_token:	Parent container token; '0' for default container
  * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
- * @token:	Token of DPBP object
+ * @obj_id:	ID of DPBP object
  *
  * Return:	'0' on Success; error code otherwise.
  */
 int dpbp_destroy(struct fsl_mc_io *mc_io,
+		 u16 dprc_token,
 		 u32 cmd_flags,
-		 u16 token)
+		 u32 obj_id)
 {
+	struct dpbp_cmd_destroy *cmd_params;
 	struct mc_command cmd = { 0 };
 
 	/* prepare command */
 	cmd.header = mc_encode_cmd_header(DPBP_CMDID_DESTROY,
-					  cmd_flags, token);
+					  cmd_flags, dprc_token);
+	cmd_params = (struct dpbp_cmd_destroy *)cmd.params;
+	cmd_params->object_id = cpu_to_le32(obj_id);
 
 	/* send command to mc*/
 	return mc_send_command(mc_io, &cmd);
@@ -609,8 +613,6 @@ int dpbp_get_attributes(struct fsl_mc_io *mc_io,
 	rsp_params = (struct dpbp_rsp_get_attributes *)cmd.params;
 	attr->bpid = le16_to_cpu(rsp_params->bpid);
 	attr->id = le32_to_cpu(rsp_params->id);
-	attr->version.major = le16_to_cpu(rsp_params->version_major);
-	attr->version.minor = le16_to_cpu(rsp_params->version_minor);
 
 	return 0;
 }
@@ -689,3 +691,35 @@ int dpbp_get_notifications(struct fsl_mc_io *mc_io,
 
 	return 0;
 }
+
+/**
+ * dpbp_get_api_version - Get Data Path Buffer Pool API version
+ * @mc_io:	Pointer to Mc portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @major_ver:	Major version of Buffer Pool API
+ * @minor_ver:	Minor version of Buffer Pool API
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dpbp_get_api_version(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 *major_ver,
+			 u16 *minor_ver)
+{
+	struct mc_command cmd = { 0 };
+	int err;
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPBP_CMDID_GET_API_VERSION,
+					  cmd_flags, 0);
+
+	/* send command to mc */
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	/* retrieve response parameters */
+	mc_cmd_read_api_version(&cmd, major_ver, minor_ver);
+
+	return 0;
+}
diff --git a/drivers/staging/fsl-mc/include/dpcon-cmd.h b/drivers/staging/fsl-mc/bus/dpcon-cmd.h
index 536b2ef13507..d0a5e194c5e1 100644
--- a/drivers/staging/fsl-mc/include/dpcon-cmd.h
+++ b/drivers/staging/fsl-mc/bus/dpcon-cmd.h
@@ -1,4 +1,5 @@
-/* Copyright 2013-2015 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -11,7 +12,6 @@
  * names of any contributors may be used to endorse or promote products
  * derived from this software without specific prior written permission.
  *
- *
  * ALTERNATIVELY, this software may be distributed under the terms of the
  * GNU General Public License ("GPL") as published by the Free Software
  * Foundation, either version 2 of that License or (at your option) any
diff --git a/drivers/staging/fsl-mc/bus/dpmcp-cmd.h b/drivers/staging/fsl-mc/bus/dpmcp-cmd.h
index d098a6d8f6bc..7cb514963c26 100644
--- a/drivers/staging/fsl-mc/bus/dpmcp-cmd.h
+++ b/drivers/staging/fsl-mc/bus/dpmcp-cmd.h
@@ -1,4 +1,5 @@
-/* Copyright 2013-2016 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -11,7 +12,6 @@
  * names of any contributors may be used to endorse or promote products
  * derived from this software without specific prior written permission.
  *
- *
  * ALTERNATIVELY, this software may be distributed under the terms of the
  * GNU General Public License ("GPL") as published by the Free Software
  * Foundation, either version 2 of that License or (at your option) any
@@ -33,25 +33,32 @@
 #define _FSL_DPMCP_CMD_H
 
 /* Minimal supported DPMCP Version */
-#define DPMCP_MIN_VER_MAJOR				3
-#define DPMCP_MIN_VER_MINOR				0
+#define DPMCP_MIN_VER_MAJOR		3
+#define DPMCP_MIN_VER_MINOR		0
+
+/* Command versioning */
+#define DPMCP_CMD_BASE_VERSION		1
+#define DPMCP_CMD_ID_OFFSET		4
+
+#define DPMCP_CMD(id)	((id << DPMCP_CMD_ID_OFFSET) | DPMCP_CMD_BASE_VERSION)
 
 /* Command IDs */
-#define DPMCP_CMDID_CLOSE				0x800
-#define DPMCP_CMDID_OPEN				0x80b
-#define DPMCP_CMDID_CREATE				0x90b
-#define DPMCP_CMDID_DESTROY				0x900
-
-#define DPMCP_CMDID_GET_ATTR				0x004
-#define DPMCP_CMDID_RESET				0x005
-
-#define DPMCP_CMDID_SET_IRQ				0x010
-#define DPMCP_CMDID_GET_IRQ				0x011
-#define DPMCP_CMDID_SET_IRQ_ENABLE			0x012
-#define DPMCP_CMDID_GET_IRQ_ENABLE			0x013
-#define DPMCP_CMDID_SET_IRQ_MASK			0x014
-#define DPMCP_CMDID_GET_IRQ_MASK			0x015
-#define DPMCP_CMDID_GET_IRQ_STATUS			0x016
+#define DPMCP_CMDID_CLOSE		DPMCP_CMD(0x800)
+#define DPMCP_CMDID_OPEN		DPMCP_CMD(0x80b)
+#define DPMCP_CMDID_CREATE		DPMCP_CMD(0x90b)
+#define DPMCP_CMDID_DESTROY		DPMCP_CMD(0x98b)
+#define DPMCP_CMDID_GET_API_VERSION	DPMCP_CMD(0xa0b)
+
+#define DPMCP_CMDID_GET_ATTR		DPMCP_CMD(0x004)
+#define DPMCP_CMDID_RESET		DPMCP_CMD(0x005)
+
+#define DPMCP_CMDID_SET_IRQ		DPMCP_CMD(0x010)
+#define DPMCP_CMDID_GET_IRQ		DPMCP_CMD(0x011)
+#define DPMCP_CMDID_SET_IRQ_ENABLE	DPMCP_CMD(0x012)
+#define DPMCP_CMDID_GET_IRQ_ENABLE	DPMCP_CMD(0x013)
+#define DPMCP_CMDID_SET_IRQ_MASK	DPMCP_CMD(0x014)
+#define DPMCP_CMDID_GET_IRQ_MASK	DPMCP_CMD(0x015)
+#define DPMCP_CMDID_GET_IRQ_STATUS	DPMCP_CMD(0x016)
 
 struct dpmcp_cmd_open {
 	__le32 dpmcp_id;
@@ -61,6 +68,10 @@ struct dpmcp_cmd_create {
 	__le32 portal_id;
 };
 
+struct dpmcp_cmd_destroy {
+	__le32 object_id;
+};
+
 struct dpmcp_cmd_set_irq {
 	/* cmd word 0 */
 	u8 irq_index;
diff --git a/drivers/staging/fsl-mc/bus/dpmcp.c b/drivers/staging/fsl-mc/bus/dpmcp.c
index 55766f78a528..e4d16519bcb4 100644
--- a/drivers/staging/fsl-mc/bus/dpmcp.c
+++ b/drivers/staging/fsl-mc/bus/dpmcp.c
@@ -1,4 +1,5 @@
-/* Copyright 2013-2016 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -11,7 +12,6 @@
  * names of any contributors may be used to endorse or promote products
  * derived from this software without specific prior written permission.
  *
- *
  * ALTERNATIVELY, this software may be distributed under the terms of the
  * GNU General Public License ("GPL") as published by the Free Software
  * Foundation, either version 2 of that License or (at your option) any
@@ -106,28 +106,29 @@ int dpmcp_close(struct fsl_mc_io *mc_io,
 /**
  * dpmcp_create() - Create the DPMCP object.
  * @mc_io:	Pointer to MC portal's I/O object
+ * @dprc_token:	Parent container token; '0' for default container
  * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
  * @cfg:	Configuration structure
- * @token:	Returned token; use in subsequent API calls
+ * @obj_id:	Returned object id; use in subsequent API calls
  *
  * Create the DPMCP object, allocate required resources and
  * perform required initialization.
  *
  * The object can be created either by declaring it in the
  * DPL file, or by calling this function.
- * This function returns a unique authentication token,
- * associated with the specific object ID and the specific MC
- * portal; this token must be used in all subsequent calls to
- * this specific object. For objects that are created using the
- * DPL file, call dpmcp_open function to get an authentication
- * token first.
+
+ * This function accepts an authentication token of a parent
+ * container that this object should be assigned to and returns
+ * an object id. This object_id will be used in all subsequent calls to
+ * this specific object.
  *
  * Return:	'0' on Success; Error code otherwise.
  */
 int dpmcp_create(struct fsl_mc_io *mc_io,
+		 u16 dprc_token,
 		 u32 cmd_flags,
 		 const struct dpmcp_cfg *cfg,
-		 u16 *token)
+		 u32 *obj_id)
 {
 	struct mc_command cmd = { 0 };
 	struct dpmcp_cmd_create *cmd_params;
@@ -136,7 +137,7 @@ int dpmcp_create(struct fsl_mc_io *mc_io,
 
 	/* prepare command */
 	cmd.header = mc_encode_cmd_header(DPMCP_CMDID_CREATE,
-					  cmd_flags, 0);
+					  cmd_flags, dprc_token);
 	cmd_params = (struct dpmcp_cmd_create *)cmd.params;
 	cmd_params->portal_id = cpu_to_le32(cfg->portal_id);
 
@@ -146,7 +147,7 @@ int dpmcp_create(struct fsl_mc_io *mc_io,
 		return err;
 
 	/* retrieve response parameters */
-	*token = mc_cmd_hdr_read_token(&cmd);
+	*obj_id = mc_cmd_read_object_id(&cmd);
 
 	return 0;
 }
@@ -154,20 +155,25 @@ int dpmcp_create(struct fsl_mc_io *mc_io,
 /**
  * dpmcp_destroy() - Destroy the DPMCP object and release all its resources.
  * @mc_io:	Pointer to MC portal's I/O object
+ * @dprc_token:	Parent container token; '0' for default container
  * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
- * @token:	Token of DPMCP object
+ * @obj_id:	ID of DPMCP object
  *
  * Return:	'0' on Success; error code otherwise.
  */
 int dpmcp_destroy(struct fsl_mc_io *mc_io,
+		  u16 dprc_token,
 		  u32 cmd_flags,
-		  u16 token)
+		  u32 obj_id)
 {
 	struct mc_command cmd = { 0 };
+	struct dpmcp_cmd_destroy *cmd_params;
 
 	/* prepare command */
 	cmd.header = mc_encode_cmd_header(DPMCP_CMDID_DESTROY,
-					  cmd_flags, token);
+					  cmd_flags, dprc_token);
+	cmd_params = (struct dpmcp_cmd_destroy *)cmd.params;
+	cmd_params->object_id = cpu_to_le32(obj_id);
 
 	/* send command to mc*/
 	return mc_send_command(mc_io, &cmd);
@@ -497,8 +503,38 @@ int dpmcp_get_attributes(struct fsl_mc_io *mc_io,
 	/* retrieve response parameters */
 	rsp_params = (struct dpmcp_rsp_get_attributes *)cmd.params;
 	attr->id = le32_to_cpu(rsp_params->id);
-	attr->version.major = le16_to_cpu(rsp_params->version_major);
-	attr->version.minor = le16_to_cpu(rsp_params->version_minor);
+
+	return 0;
+}
+
+/**
+ * dpmcp_get_api_version - Get Data Path Management Command Portal API version
+ * @mc_io:	Pointer to Mc portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @major_ver:	Major version of Data Path Management Command Portal API
+ * @minor_ver:	Minor version of Data Path Management Command Portal API
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dpmcp_get_api_version(struct fsl_mc_io *mc_io,
+			  u32 cmd_flags,
+			  u16 *major_ver,
+			  u16 *minor_ver)
+{
+	struct mc_command cmd = { 0 };
+	int err;
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPMCP_CMDID_GET_API_VERSION,
+					  cmd_flags, 0);
+
+	/* send command to mc */
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	/* retrieve response parameters */
+	mc_cmd_read_api_version(&cmd, major_ver, minor_ver);
 
 	return 0;
 }
diff --git a/drivers/staging/fsl-mc/bus/dpmcp.h b/drivers/staging/fsl-mc/bus/dpmcp.h
index fe79d4d9293d..98a100d543f6 100644
--- a/drivers/staging/fsl-mc/bus/dpmcp.h
+++ b/drivers/staging/fsl-mc/bus/dpmcp.h
@@ -1,4 +1,5 @@
-/* Copyright 2013-2015 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -11,7 +12,6 @@
  * names of any contributors may be used to endorse or promote products
  * derived from this software without specific prior written permission.
  *
- *
  * ALTERNATIVELY, this software may be distributed under the terms of the
  * GNU General Public License ("GPL") as published by the Free Software
  * Foundation, either version 2 of that License or (at your option) any
@@ -32,23 +32,24 @@
 #ifndef __FSL_DPMCP_H
 #define __FSL_DPMCP_H
 
-/* Data Path Management Command Portal API
+/*
+ * Data Path Management Command Portal API
  * Contains initialization APIs and runtime control APIs for DPMCP
  */
 
 struct fsl_mc_io;
 
 int dpmcp_open(struct fsl_mc_io *mc_io,
-	       uint32_t cmd_flags,
+	       u32 cmd_flags,
 	       int dpmcp_id,
-	       uint16_t *token);
+	       u16 *token);
 
 /* Get portal ID from pool */
 #define DPMCP_GET_PORTAL_ID_FROM_POOL (-1)
 
 int dpmcp_close(struct fsl_mc_io *mc_io,
-		uint32_t cmd_flags,
-		uint16_t token);
+		u32 cmd_flags,
+		u16 token);
 
 /**
  * struct dpmcp_cfg - Structure representing DPMCP configuration
@@ -59,18 +60,20 @@ struct dpmcp_cfg {
 	int portal_id;
 };
 
-int dpmcp_create(struct fsl_mc_io	*mc_io,
-		 uint32_t		cmd_flags,
-		 const struct dpmcp_cfg	*cfg,
-		uint16_t		*token);
+int dpmcp_create(struct fsl_mc_io *mc_io,
+		 u16 dprc_token,
+		 u32 cmd_flags,
+		 const struct dpmcp_cfg *cfg,
+		 u32 *obj_id);
 
 int dpmcp_destroy(struct fsl_mc_io *mc_io,
-		  uint32_t cmd_flags,
-		  uint16_t token);
+		  u16 dprc_token,
+		  u32 cmd_flags,
+		  u32 obj_id);
 
 int dpmcp_reset(struct fsl_mc_io *mc_io,
-		uint32_t cmd_flags,
-		uint16_t token);
+		u32 cmd_flags,
+		u16 token);
 
 /* IRQ */
 /* IRQ Index */
@@ -85,75 +88,65 @@ int dpmcp_reset(struct fsl_mc_io *mc_io,
  * @irq_num: A user defined number associated with this IRQ
  */
 struct dpmcp_irq_cfg {
-	     uint64_t		paddr;
-	     uint32_t		val;
-	     int		irq_num;
+	     u64 paddr;
+	     u32 val;
+	     int irq_num;
 };
 
-int dpmcp_set_irq(struct fsl_mc_io	*mc_io,
-		  uint32_t		cmd_flags,
-		  uint16_t		token,
-		 uint8_t		irq_index,
-		  struct dpmcp_irq_cfg	*irq_cfg);
-
-int dpmcp_get_irq(struct fsl_mc_io	*mc_io,
-		  uint32_t		cmd_flags,
-		  uint16_t		token,
-		 uint8_t		irq_index,
-		 int			*type,
-		 struct dpmcp_irq_cfg	*irq_cfg);
-
-int dpmcp_set_irq_enable(struct fsl_mc_io	*mc_io,
-			 uint32_t		cmd_flags,
-			 uint16_t		token,
-			uint8_t			irq_index,
-			uint8_t			en);
-
-int dpmcp_get_irq_enable(struct fsl_mc_io	*mc_io,
-			 uint32_t		cmd_flags,
-			 uint16_t		token,
-			uint8_t			irq_index,
-			uint8_t			*en);
-
-int dpmcp_set_irq_mask(struct fsl_mc_io	*mc_io,
-		       uint32_t	cmd_flags,
-		       uint16_t		token,
-		      uint8_t		irq_index,
-		      uint32_t		mask);
-
-int dpmcp_get_irq_mask(struct fsl_mc_io	*mc_io,
-		       uint32_t	cmd_flags,
-		       uint16_t		token,
-		      uint8_t		irq_index,
-		      uint32_t		*mask);
-
-int dpmcp_get_irq_status(struct fsl_mc_io	*mc_io,
-			 uint32_t		cmd_flags,
-			 uint16_t		token,
-			uint8_t			irq_index,
-			uint32_t		*status);
+int dpmcp_set_irq(struct fsl_mc_io *mc_io,
+		  u32 cmd_flags,
+		  u16 token,
+		  u8 irq_index,
+		  struct dpmcp_irq_cfg *irq_cfg);
+
+int dpmcp_get_irq(struct fsl_mc_io *mc_io,
+		  u32 cmd_flags,
+		  u16 token,
+		  u8 irq_index,
+		  int *type,
+		  struct dpmcp_irq_cfg *irq_cfg);
+
+int dpmcp_set_irq_enable(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 token,
+			 u8 irq_index,
+			 u8 en);
+
+int dpmcp_get_irq_enable(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 token,
+			 u8 irq_index,
+			 u8 *en);
+
+int dpmcp_set_irq_mask(struct fsl_mc_io *mc_io,
+		       u32 cmd_flags,
+		       u16 token,
+		       u8 irq_index,
+		       u32 mask);
+
+int dpmcp_get_irq_mask(struct fsl_mc_io *mc_io,
+		       u32 cmd_flags,
+		       u16 token,
+		       u8 irq_index,
+		       u32 *mask);
+
+int dpmcp_get_irq_status(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 token,
+			 u8 irq_index,
+			 u32 *status);
 
 /**
  * struct dpmcp_attr - Structure representing DPMCP attributes
  * @id:		DPMCP object ID
- * @version:	DPMCP version
  */
 struct dpmcp_attr {
 	int id;
-	/**
-	 * struct version - Structure representing DPMCP version
-	 * @major:	DPMCP major version
-	 * @minor:	DPMCP minor version
-	 */
-	struct {
-		uint16_t major;
-		uint16_t minor;
-	} version;
 };
 
-int dpmcp_get_attributes(struct fsl_mc_io	*mc_io,
-			 uint32_t		cmd_flags,
-			 uint16_t		token,
-			struct dpmcp_attr	*attr);
+int dpmcp_get_attributes(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 token,
+			 struct dpmcp_attr *attr);
 
 #endif /* __FSL_DPMCP_H */
diff --git a/drivers/staging/fsl-mc/bus/dpmng-cmd.h b/drivers/staging/fsl-mc/bus/dpmng-cmd.h
index a7b77d58c8cd..cdddfb80eecc 100644
--- a/drivers/staging/fsl-mc/bus/dpmng-cmd.h
+++ b/drivers/staging/fsl-mc/bus/dpmng-cmd.h
@@ -12,7 +12,6 @@
  *       names of any contributors may be used to endorse or promote products
  *       derived from this software without specific prior written permission.
  *
- *
  * ALTERNATIVELY, this software may be distributed under the terms of the
  * GNU General Public License ("GPL") as published by the Free Software
  * Foundation, either version 2 of that License or (at your option) any
@@ -41,13 +40,14 @@
 #ifndef __FSL_DPMNG_CMD_H
 #define __FSL_DPMNG_CMD_H
 
-/* Command IDs */
-#define DPMNG_CMDID_GET_CONT_ID			0x830
-#define DPMNG_CMDID_GET_VERSION			0x831
+/* Command versioning */
+#define DPMNG_CMD_BASE_VERSION		1
+#define DPMNG_CMD_ID_OFFSET		4
 
-struct dpmng_rsp_get_container_id {
-	__le32 container_id;
-};
+#define DPMNG_CMD(id)	((id << DPMNG_CMD_ID_OFFSET) | DPMNG_CMD_BASE_VERSION)
+
+/* Command IDs */
+#define DPMNG_CMDID_GET_VERSION		DPMNG_CMD(0x831)
 
 struct dpmng_rsp_get_version {
 	__le32 revision;
diff --git a/drivers/staging/fsl-mc/bus/dpmng.c b/drivers/staging/fsl-mc/bus/dpmng.c
index 96b1d67756fa..ad5d5bbec529 100644
--- a/drivers/staging/fsl-mc/bus/dpmng.c
+++ b/drivers/staging/fsl-mc/bus/dpmng.c
@@ -1,4 +1,5 @@
-/* Copyright 2013-2016 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -11,7 +12,6 @@
  * names of any contributors may be used to endorse or promote products
  * derived from this software without specific prior written permission.
  *
- *
  * ALTERNATIVELY, this software may be distributed under the terms of the
  * GNU General Public License ("GPL") as published by the Free Software
  * Foundation, either version 2 of that License or (at your option) any
@@ -72,36 +72,3 @@ int mc_get_version(struct fsl_mc_io *mc_io,
 }
 EXPORT_SYMBOL(mc_get_version);
 
-/**
- * dpmng_get_container_id() - Get container ID associated with a given portal.
- * @mc_io:		Pointer to MC portal's I/O object
- * @cmd_flags:		Command flags; one or more of 'MC_CMD_FLAG_'
- * @container_id:	Requested container ID
- *
- * Return:	'0' on Success; Error code otherwise.
- */
-int dpmng_get_container_id(struct fsl_mc_io *mc_io,
-			   u32 cmd_flags,
-			   int *container_id)
-{
-	struct mc_command cmd = { 0 };
-	struct dpmng_rsp_get_container_id *rsp_params;
-	int err;
-
-	/* prepare command */
-	cmd.header = mc_encode_cmd_header(DPMNG_CMDID_GET_CONT_ID,
-					  cmd_flags,
-					  0);
-
-	/* send command to mc*/
-	err = mc_send_command(mc_io, &cmd);
-	if (err)
-		return err;
-
-	/* retrieve response parameters */
-	rsp_params = (struct dpmng_rsp_get_container_id *)cmd.params;
-	*container_id = le32_to_cpu(rsp_params->container_id);
-
-	return 0;
-}
-
diff --git a/drivers/staging/fsl-mc/bus/dprc-cmd.h b/drivers/staging/fsl-mc/bus/dprc-cmd.h
index 009d65673155..588b8cafdbc7 100644
--- a/drivers/staging/fsl-mc/bus/dprc-cmd.h
+++ b/drivers/staging/fsl-mc/bus/dprc-cmd.h
@@ -12,7 +12,6 @@
  *       names of any contributors may be used to endorse or promote products
  *       derived from this software without specific prior written permission.
  *
- *
  * ALTERNATIVELY, this software may be distributed under the terms of the
  * GNU General Public License ("GPL") as published by the Free Software
  * Foundation, either version 2 of that License or (at your option) any
@@ -42,48 +41,56 @@
 #define _FSL_DPRC_CMD_H
 
 /* Minimal supported DPRC Version */
-#define DPRC_MIN_VER_MAJOR			5
+#define DPRC_MIN_VER_MAJOR			6
 #define DPRC_MIN_VER_MINOR			0
 
+/* Command versioning */
+#define DPRC_CMD_BASE_VERSION			1
+#define DPRC_CMD_ID_OFFSET			4
+
+#define DPRC_CMD(id)	((id << DPRC_CMD_ID_OFFSET) | DPRC_CMD_BASE_VERSION)
+
 /* Command IDs */
-#define DPRC_CMDID_CLOSE			0x800
-#define DPRC_CMDID_OPEN				0x805
-#define DPRC_CMDID_CREATE			0x905
-
-#define DPRC_CMDID_GET_ATTR			0x004
-#define DPRC_CMDID_RESET_CONT			0x005
-
-#define DPRC_CMDID_SET_IRQ			0x010
-#define DPRC_CMDID_GET_IRQ			0x011
-#define DPRC_CMDID_SET_IRQ_ENABLE		0x012
-#define DPRC_CMDID_GET_IRQ_ENABLE		0x013
-#define DPRC_CMDID_SET_IRQ_MASK			0x014
-#define DPRC_CMDID_GET_IRQ_MASK			0x015
-#define DPRC_CMDID_GET_IRQ_STATUS		0x016
-#define DPRC_CMDID_CLEAR_IRQ_STATUS		0x017
-
-#define DPRC_CMDID_CREATE_CONT			0x151
-#define DPRC_CMDID_DESTROY_CONT			0x152
-#define DPRC_CMDID_SET_RES_QUOTA		0x155
-#define DPRC_CMDID_GET_RES_QUOTA		0x156
-#define DPRC_CMDID_ASSIGN			0x157
-#define DPRC_CMDID_UNASSIGN			0x158
-#define DPRC_CMDID_GET_OBJ_COUNT		0x159
-#define DPRC_CMDID_GET_OBJ			0x15A
-#define DPRC_CMDID_GET_RES_COUNT		0x15B
-#define DPRC_CMDID_GET_RES_IDS			0x15C
-#define DPRC_CMDID_GET_OBJ_REG			0x15E
-#define DPRC_CMDID_SET_OBJ_IRQ			0x15F
-#define DPRC_CMDID_GET_OBJ_IRQ			0x160
-#define DPRC_CMDID_SET_OBJ_LABEL		0x161
-#define DPRC_CMDID_GET_OBJ_DESC			0x162
-
-#define DPRC_CMDID_CONNECT			0x167
-#define DPRC_CMDID_DISCONNECT			0x168
-#define DPRC_CMDID_GET_POOL			0x169
-#define DPRC_CMDID_GET_POOL_COUNT		0x16A
-
-#define DPRC_CMDID_GET_CONNECTION		0x16C
+#define DPRC_CMDID_CLOSE                        DPRC_CMD(0x800)
+#define DPRC_CMDID_OPEN                         DPRC_CMD(0x805)
+#define DPRC_CMDID_CREATE                       DPRC_CMD(0x905)
+#define DPRC_CMDID_GET_API_VERSION              DPRC_CMD(0xa05)
+
+#define DPRC_CMDID_GET_ATTR                     DPRC_CMD(0x004)
+#define DPRC_CMDID_RESET_CONT                   DPRC_CMD(0x005)
+
+#define DPRC_CMDID_SET_IRQ                      DPRC_CMD(0x010)
+#define DPRC_CMDID_GET_IRQ                      DPRC_CMD(0x011)
+#define DPRC_CMDID_SET_IRQ_ENABLE               DPRC_CMD(0x012)
+#define DPRC_CMDID_GET_IRQ_ENABLE               DPRC_CMD(0x013)
+#define DPRC_CMDID_SET_IRQ_MASK                 DPRC_CMD(0x014)
+#define DPRC_CMDID_GET_IRQ_MASK                 DPRC_CMD(0x015)
+#define DPRC_CMDID_GET_IRQ_STATUS               DPRC_CMD(0x016)
+#define DPRC_CMDID_CLEAR_IRQ_STATUS             DPRC_CMD(0x017)
+
+#define DPRC_CMDID_CREATE_CONT                  DPRC_CMD(0x151)
+#define DPRC_CMDID_DESTROY_CONT                 DPRC_CMD(0x152)
+#define DPRC_CMDID_GET_CONT_ID                  DPRC_CMD(0x830)
+#define DPRC_CMDID_SET_RES_QUOTA                DPRC_CMD(0x155)
+#define DPRC_CMDID_GET_RES_QUOTA                DPRC_CMD(0x156)
+#define DPRC_CMDID_ASSIGN                       DPRC_CMD(0x157)
+#define DPRC_CMDID_UNASSIGN                     DPRC_CMD(0x158)
+#define DPRC_CMDID_GET_OBJ_COUNT                DPRC_CMD(0x159)
+#define DPRC_CMDID_GET_OBJ                      DPRC_CMD(0x15A)
+#define DPRC_CMDID_GET_RES_COUNT                DPRC_CMD(0x15B)
+#define DPRC_CMDID_GET_RES_IDS                  DPRC_CMD(0x15C)
+#define DPRC_CMDID_GET_OBJ_REG                  DPRC_CMD(0x15E)
+#define DPRC_CMDID_SET_OBJ_IRQ                  DPRC_CMD(0x15F)
+#define DPRC_CMDID_GET_OBJ_IRQ                  DPRC_CMD(0x160)
+#define DPRC_CMDID_SET_OBJ_LABEL                DPRC_CMD(0x161)
+#define DPRC_CMDID_GET_OBJ_DESC                 DPRC_CMD(0x162)
+
+#define DPRC_CMDID_CONNECT                      DPRC_CMD(0x167)
+#define DPRC_CMDID_DISCONNECT                   DPRC_CMD(0x168)
+#define DPRC_CMDID_GET_POOL                     DPRC_CMD(0x169)
+#define DPRC_CMDID_GET_POOL_COUNT               DPRC_CMD(0x16A)
+
+#define DPRC_CMDID_GET_CONNECTION               DPRC_CMD(0x16C)
 
 struct dprc_cmd_open {
 	__le32 container_id;
@@ -199,9 +206,6 @@ struct dprc_rsp_get_attributes {
 	/* response word 1 */
 	__le32 options;
 	__le32 portal_id;
-	/* response word 2 */
-	__le16 version_major;
-	__le16 version_minor;
 };
 
 struct dprc_cmd_set_res_quota {
diff --git a/drivers/staging/fsl-mc/bus/dprc-driver.c b/drivers/staging/fsl-mc/bus/dprc-driver.c
index c5ee4639682b..4e416d89b736 100644
--- a/drivers/staging/fsl-mc/bus/dprc-driver.c
+++ b/drivers/staging/fsl-mc/bus/dprc-driver.c
@@ -1,7 +1,7 @@
 /*
  * Freescale data path resource container (DPRC) driver
  *
- * Copyright (C) 2014 Freescale Semiconductor, Inc.
+ * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
  * Author: German Rivera <German.Rivera@freescale.com>
  *
  * This file is licensed under the terms of the GNU General Public
@@ -505,7 +505,7 @@ static int register_dprc_irq_handler(struct fsl_mc_device *mc_dev)
 					  dprc_irq0_handler,
 					  dprc_irq0_handler_thread,
 					  IRQF_NO_SUSPEND | IRQF_ONESHOT,
-					  "FSL MC DPRC irq0",
+					  dev_name(&mc_dev->dev),
 					  &mc_dev->dev);
 	if (error < 0) {
 		dev_err(&mc_dev->dev,
@@ -597,6 +597,7 @@ static int dprc_probe(struct fsl_mc_device *mc_dev)
 	struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev);
 	bool mc_io_created = false;
 	bool msi_domain_set = false;
+	u16 major_ver, minor_ver;
 
 	if (WARN_ON(strcmp(mc_dev->obj_desc.type, "dprc") != 0))
 		return -EINVAL;
@@ -669,13 +670,21 @@ static int dprc_probe(struct fsl_mc_device *mc_dev)
 		goto error_cleanup_open;
 	}
 
-	if (mc_bus->dprc_attr.version.major < DPRC_MIN_VER_MAJOR ||
-	   (mc_bus->dprc_attr.version.major == DPRC_MIN_VER_MAJOR &&
-	    mc_bus->dprc_attr.version.minor < DPRC_MIN_VER_MINOR)) {
+	error = dprc_get_api_version(mc_dev->mc_io, 0,
+				     &major_ver,
+				     &minor_ver);
+	if (error < 0) {
+		dev_err(&mc_dev->dev, "dprc_get_api_version() failed: %d\n",
+			error);
+		goto error_cleanup_open;
+	}
+
+	if (major_ver < DPRC_MIN_VER_MAJOR ||
+	   (major_ver == DPRC_MIN_VER_MAJOR &&
+	    minor_ver < DPRC_MIN_VER_MINOR)) {
 		dev_err(&mc_dev->dev,
 			"ERROR: DPRC version %d.%d not supported\n",
-			mc_bus->dprc_attr.version.major,
-			mc_bus->dprc_attr.version.minor);
+			major_ver, minor_ver);
 		error = -ENOTSUPP;
 		goto error_cleanup_open;
 	}
diff --git a/drivers/staging/fsl-mc/bus/dprc.c b/drivers/staging/fsl-mc/bus/dprc.c
index 9fea3def6041..572edd4c066e 100644
--- a/drivers/staging/fsl-mc/bus/dprc.c
+++ b/drivers/staging/fsl-mc/bus/dprc.c
@@ -1,4 +1,5 @@
-/* Copyright 2013-2016 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -11,7 +12,6 @@
  * names of any contributors may be used to endorse or promote products
  * derived from this software without specific prior written permission.
  *
- *
  * ALTERNATIVELY, this software may be distributed under the terms of the
  * GNU General Public License ("GPL") as published by the Free Software
  * Foundation, either version 2 of that License or (at your option) any
@@ -565,8 +565,6 @@ int dprc_get_attributes(struct fsl_mc_io *mc_io,
 	attr->icid = le16_to_cpu(rsp_params->icid);
 	attr->options = le32_to_cpu(rsp_params->options);
 	attr->portal_id = le32_to_cpu(rsp_params->portal_id);
-	attr->version.major = le16_to_cpu(rsp_params->version_major);
-	attr->version.minor = le16_to_cpu(rsp_params->version_minor);
 
 	return 0;
 }
@@ -1386,3 +1384,66 @@ int dprc_get_connection(struct fsl_mc_io *mc_io,
 
 	return 0;
 }
+
+/**
+ * dprc_get_api_version - Get Data Path Resource Container API version
+ * @mc_io:	Pointer to Mc portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @major_ver:	Major version of Data Path Resource Container API
+ * @minor_ver:	Minor version of Data Path Resource Container API
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprc_get_api_version(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 *major_ver,
+			 u16 *minor_ver)
+{
+	struct mc_command cmd = { 0 };
+	int err;
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPRC_CMDID_GET_API_VERSION,
+					  cmd_flags, 0);
+
+	/* send command to mc */
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	/* retrieve response parameters */
+	mc_cmd_read_api_version(&cmd, major_ver, minor_ver);
+
+	return 0;
+}
+
+/**
+ * dprc_get_container_id - Get container ID associated with a given portal.
+ * @mc_io:		Pointer to Mc portal's I/O object
+ * @cmd_flags:		Command flags; one or more of 'MC_CMD_FLAG_'
+ * @container_id:	Requested container id
+ *
+ * Return:	'0' on Success; Error code otherwise.
+ */
+int dprc_get_container_id(struct fsl_mc_io *mc_io,
+			  u32 cmd_flags,
+			  int *container_id)
+{
+	struct mc_command cmd = { 0 };
+	int err;
+
+	/* prepare command */
+	cmd.header = mc_encode_cmd_header(DPRC_CMDID_GET_CONT_ID,
+					  cmd_flags,
+					  0);
+
+	/* send command to mc*/
+	err = mc_send_command(mc_io, &cmd);
+	if (err)
+		return err;
+
+	/* retrieve response parameters */
+	*container_id = (int)mc_cmd_read_object_id(&cmd);
+
+	return 0;
+}
diff --git a/drivers/staging/fsl-mc/bus/fsl-mc-allocator.c b/drivers/staging/fsl-mc/bus/fsl-mc-allocator.c
index e93ab53bae67..ce07096c3b1f 100644
--- a/drivers/staging/fsl-mc/bus/fsl-mc-allocator.c
+++ b/drivers/staging/fsl-mc/bus/fsl-mc-allocator.c
@@ -1,7 +1,7 @@
 /*
- * Freescale MC object device allocator driver
+ * fsl-mc object allocator driver
  *
- * Copyright (C) 2013 Freescale Semiconductor, Inc.
+ * Copyright (C) 2013-2016 Freescale Semiconductor, Inc.
  *
  * This file is licensed under the terms of the GNU General Public
  * License version 2. This program is licensed "as is" without any
@@ -12,9 +12,9 @@
 #include <linux/msi.h>
 #include "../include/mc-bus.h"
 #include "../include/mc-sys.h"
-#include "../include/dpbp-cmd.h"
-#include "../include/dpcon-cmd.h"
 
+#include "dpbp-cmd.h"
+#include "dpcon-cmd.h"
 #include "fsl-mc-private.h"
 
 #define FSL_MC_IS_ALLOCATABLE(_obj_type) \
@@ -23,15 +23,12 @@
 	 strcmp(_obj_type, "dpcon") == 0)
 
 /**
- * fsl_mc_resource_pool_add_device - add allocatable device to a resource
- * pool of a given MC bus
+ * fsl_mc_resource_pool_add_device - add allocatable object to a resource
+ * pool of a given fsl-mc bus
  *
- * @mc_bus: pointer to the MC bus
- * @pool_type: MC bus pool type
- * @mc_dev: Pointer to allocatable MC object device
- *
- * It adds an allocatable MC object device to a container's resource pool of
- * the given resource type
+ * @mc_bus: pointer to the fsl-mc bus
+ * @pool_type: pool type
+ * @mc_dev: pointer to allocatable fsl-mc device
  */
 static int __must_check fsl_mc_resource_pool_add_device(struct fsl_mc_bus
 								*mc_bus,
@@ -95,10 +92,10 @@ out:
  * fsl_mc_resource_pool_remove_device - remove an allocatable device from a
  * resource pool
  *
- * @mc_dev: Pointer to allocatable MC object device
+ * @mc_dev: pointer to allocatable fsl-mc device
  *
- * It permanently removes an allocatable MC object device from the resource
- * pool, the device is currently in, as long as it is in the pool's free list.
+ * It permanently removes an allocatable fsl-mc device from the resource
+ * pool. It's an error if the device is in use.
  */
 static int __must_check fsl_mc_resource_pool_remove_device(struct fsl_mc_device
 								   *mc_dev)
@@ -255,17 +252,18 @@ out_unlock:
 EXPORT_SYMBOL_GPL(fsl_mc_resource_free);
 
 /**
- * fsl_mc_object_allocate - Allocates a MC object device of the given
- * pool type from a given MC bus
+ * fsl_mc_object_allocate - Allocates an fsl-mc object of the given
+ * pool type from a given fsl-mc bus instance
  *
- * @mc_dev: MC device for which the MC object device is to be allocated
- * @pool_type: MC bus resource pool type
- * @new_mc_dev: Pointer to area where the pointer to the allocated
- * MC object device is to be returned
+ * @mc_dev: fsl-mc device which is used in conjunction with the
+ * allocated object
+ * @pool_type: pool type
+ * @new_mc_dev: pointer to area where the pointer to the allocated device
+ * is to be returned
  *
- * This function allocates a MC object device from the device's parent DPRC,
- * from the corresponding MC bus' pool of allocatable MC object devices of
- * the given resource type. mc_dev cannot be a DPRC itself.
+ * Allocatable objects are always used in conjunction with some functional
+ * device.  This function allocates an object of the specified type from
+ * the DPRC containing the functional device.
  *
  * NOTE: pool_type must be different from FSL_MC_POOL_MCP, since MC
  * portals are allocated using fsl_mc_portal_allocate(), instead of
@@ -312,10 +310,9 @@ error:
 EXPORT_SYMBOL_GPL(fsl_mc_object_allocate);
 
 /**
- * fsl_mc_object_free - Returns an allocatable MC object device to the
- * corresponding resource pool of a given MC bus.
- *
- * @mc_adev: Pointer to the MC object device
+ * fsl_mc_object_free - Returns an fsl-mc object to the resource
+ * pool where it came from.
+ * @mc_adev: Pointer to the fsl-mc device
  */
 void fsl_mc_object_free(struct fsl_mc_device *mc_adev)
 {
@@ -332,8 +329,14 @@ void fsl_mc_object_free(struct fsl_mc_device *mc_adev)
 EXPORT_SYMBOL_GPL(fsl_mc_object_free);
 
 /*
- * Initialize the interrupt pool associated with a MC bus.
- * It allocates a block of IRQs from the GIC-ITS
+ * A DPRC and the devices in the DPRC all share the same GIC-ITS device
+ * ID.  A block of IRQs is pre-allocated and maintained in a pool
+ * from which devices can allocate them when needed.
+ */
+
+/*
+ * Initialize the interrupt pool associated with an fsl-mc bus.
+ * It allocates a block of IRQs from the GIC-ITS.
  */
 int fsl_mc_populate_irq_pool(struct fsl_mc_bus *mc_bus,
 			     unsigned int irq_count)
@@ -395,7 +398,7 @@ cleanup_msi_irqs:
 EXPORT_SYMBOL_GPL(fsl_mc_populate_irq_pool);
 
 /**
- * Teardown the interrupt pool associated with an MC bus.
+ * Teardown the interrupt pool associated with an fsl-mc bus.
  * It frees the IRQs that were allocated to the pool, back to the GIC-ITS.
  */
 void fsl_mc_cleanup_irq_pool(struct fsl_mc_bus *mc_bus)
@@ -422,11 +425,7 @@ void fsl_mc_cleanup_irq_pool(struct fsl_mc_bus *mc_bus)
 EXPORT_SYMBOL_GPL(fsl_mc_cleanup_irq_pool);
 
 /**
- * It allocates the IRQs required by a given MC object device. The
- * IRQs are allocated from the interrupt pool associated with the
- * MC bus that contains the device, if the device is not a DPRC device.
- * Otherwise, the IRQs are allocated from the interrupt pool associated
- * with the MC bus that represents the DPRC device itself.
+ * Allocate the IRQs required by a given fsl-mc device.
  */
 int __must_check fsl_mc_allocate_irqs(struct fsl_mc_device *mc_dev)
 {
@@ -495,8 +494,7 @@ error_resource_alloc:
 EXPORT_SYMBOL_GPL(fsl_mc_allocate_irqs);
 
 /*
- * It frees the IRQs that were allocated for a MC object device, by
- * returning them to the corresponding interrupt pool.
+ * Frees the IRQs that were allocated for an fsl-mc device.
  */
 void fsl_mc_free_irqs(struct fsl_mc_device *mc_dev)
 {
@@ -605,7 +603,7 @@ static int fsl_mc_allocator_probe(struct fsl_mc_device *mc_dev)
 		return error;
 
 	dev_dbg(&mc_dev->dev,
-		"Allocatable MC object device bound to fsl_mc_allocator driver");
+		"Allocatable fsl-mc device bound to fsl_mc_allocator driver");
 	return 0;
 }
 
@@ -627,7 +625,7 @@ static int fsl_mc_allocator_remove(struct fsl_mc_device *mc_dev)
 	}
 
 	dev_dbg(&mc_dev->dev,
-		"Allocatable MC object device unbound from fsl_mc_allocator driver");
+		"Allocatable fsl-mc device unbound from fsl_mc_allocator driver");
 	return 0;
 }
 
diff --git a/drivers/staging/fsl-mc/bus/fsl-mc-bus.c b/drivers/staging/fsl-mc/bus/fsl-mc-bus.c
index 44f64b6f0fc9..5ac373c0c716 100644
--- a/drivers/staging/fsl-mc/bus/fsl-mc-bus.c
+++ b/drivers/staging/fsl-mc/bus/fsl-mc-bus.c
@@ -1,7 +1,7 @@
 /*
  * Freescale Management Complex (MC) bus driver
  *
- * Copyright (C) 2014 Freescale Semiconductor, Inc.
+ * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
  * Author: German Rivera <German.Rivera@freescale.com>
  *
  * This file is licensed under the terms of the GNU General Public
@@ -9,6 +9,8 @@
  * warranty of any kind, whether express or implied.
  */
 
+#define pr_fmt(fmt) "fsl-mc: " fmt
+
 #include <linux/module.h>
 #include <linux/of_device.h>
 #include <linux/of_address.h>
@@ -34,7 +36,7 @@ static struct kmem_cache *mc_dev_cache;
 
 /**
  * struct fsl_mc - Private data of a "fsl,qoriq-mc" platform device
- * @root_mc_bus_dev: MC object device representing the root DPRC
+ * @root_mc_bus_dev: fsl-mc device representing the root DPRC
  * @num_translation_ranges: number of entries in addr_translation_ranges
  * @translation_ranges: array of bus to system address translation ranges
  */
@@ -62,8 +64,8 @@ struct fsl_mc_addr_translation_range {
 
 /**
  * fsl_mc_bus_match - device to driver matching callback
- * @dev: the MC object device structure to match against
- * @drv: the device driver to search for matching MC object device id
+ * @dev: the fsl-mc device to match against
+ * @drv: the device driver to search for matching fsl-mc object type
  * structures
  *
  * Returns 1 on success, 0 otherwise.
@@ -91,7 +93,7 @@ static int fsl_mc_bus_match(struct device *dev, struct device_driver *drv)
 
 	/*
 	 * Traverse the match_id table of the given driver, trying to find
-	 * a matching for the given MC object device.
+	 * a matching for the given device.
 	 */
 	for (id = mc_drv->match_id_table; id->vendor != 0x0; id++) {
 		if (id->vendor == mc_dev->obj_desc.vendor &&
@@ -164,8 +166,7 @@ static int fsl_mc_driver_probe(struct device *dev)
 
 	error = mc_drv->probe(mc_dev);
 	if (error < 0) {
-		dev_err(dev, "MC object device probe callback failed: %d\n",
-			error);
+		dev_err(dev, "%s failed: %d\n", __func__, error);
 		return error;
 	}
 
@@ -183,9 +184,7 @@ static int fsl_mc_driver_remove(struct device *dev)
 
 	error = mc_drv->remove(mc_dev);
 	if (error < 0) {
-		dev_err(dev,
-			"MC object device remove callback failed: %d\n",
-			error);
+		dev_err(dev, "%s failed: %d\n", __func__, error);
 		return error;
 	}
 
@@ -232,8 +231,6 @@ int __fsl_mc_driver_register(struct fsl_mc_driver *mc_driver,
 		return error;
 	}
 
-	pr_info("MC object device driver %s registered\n",
-		mc_driver->driver.name);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(__fsl_mc_driver_register);
@@ -315,21 +312,6 @@ static int get_dprc_icid(struct fsl_mc_io *mc_io,
 	return error;
 }
 
-static int get_dprc_version(struct fsl_mc_io *mc_io,
-			    int container_id, u16 *major, u16 *minor)
-{
-	struct dprc_attributes attr;
-	int error;
-
-	error = get_dprc_attr(mc_io, container_id, &attr);
-	if (error == 0) {
-		*major = attr.version.major;
-		*minor = attr.version.minor;
-	}
-
-	return error;
-}
-
 static int translate_mc_addr(struct fsl_mc_device *mc_dev,
 			     enum dprc_region_type mc_region_type,
 			     u64 mc_offset, phys_addr_t *phys_addr)
@@ -452,7 +434,7 @@ bool fsl_mc_is_root_dprc(struct device *dev)
 }
 
 /**
- * Add a newly discovered MC object device to be visible in Linux
+ * Add a newly discovered fsl-mc device to be visible in Linux
  */
 int fsl_mc_device_add(struct dprc_obj_desc *obj_desc,
 		      struct fsl_mc_io *mc_io,
@@ -533,8 +515,8 @@ int fsl_mc_device_add(struct dprc_obj_desc *obj_desc,
 			goto error_cleanup_dev;
 	} else {
 		/*
-		 * A non-DPRC MC object device has to be a child of another
-		 * MC object (specifically a DPRC object)
+		 * A non-DPRC object has to be a child of a DPRC, use the
+		 * parent's ICID and interrupt domain.
 		 */
 		mc_dev->icid = parent_mc_dev->icid;
 		mc_dev->dma_mask = FSL_MC_DEFAULT_DMA_MASK;
@@ -572,8 +554,7 @@ int fsl_mc_device_add(struct dprc_obj_desc *obj_desc,
 	}
 
 	(void)get_device(&mc_dev->dev);
-	dev_dbg(parent_dev, "Added MC object device %s\n",
-		dev_name(&mc_dev->dev));
+	dev_dbg(parent_dev, "added %s\n", dev_name(&mc_dev->dev));
 
 	*new_mc_dev = mc_dev;
 	return 0;
@@ -590,10 +571,10 @@ error_cleanup_dev:
 EXPORT_SYMBOL_GPL(fsl_mc_device_add);
 
 /**
- * fsl_mc_device_remove - Remove a MC object device from being visible to
+ * fsl_mc_device_remove - Remove an fsl-mc device from being visible to
  * Linux
  *
- * @mc_dev: Pointer to a MC object device object
+ * @mc_dev: Pointer to an fsl-mc device
  */
 void fsl_mc_device_remove(struct fsl_mc_device *mc_dev)
 {
@@ -749,8 +730,6 @@ static int fsl_mc_bus_probe(struct platform_device *pdev)
 	struct mc_version mc_version;
 	struct resource res;
 
-	dev_info(&pdev->dev, "Root MC bus device probed");
-
 	mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
 	if (!mc)
 		return -ENOMEM;
@@ -783,8 +762,7 @@ static int fsl_mc_bus_probe(struct platform_device *pdev)
 		goto error_cleanup_mc_io;
 	}
 
-	dev_info(&pdev->dev,
-		 "Freescale Management Complex Firmware version: %u.%u.%u\n",
+	dev_info(&pdev->dev, "MC firmware version: %u.%u.%u\n",
 		 mc_version.major, mc_version.minor, mc_version.revision);
 
 	error = get_mc_addr_translation_ranges(&pdev->dev,
@@ -793,7 +771,7 @@ static int fsl_mc_bus_probe(struct platform_device *pdev)
 	if (error < 0)
 		goto error_cleanup_mc_io;
 
-	error = dpmng_get_container_id(mc_io, 0, &container_id);
+	error = dprc_get_container_id(mc_io, 0, &container_id);
 	if (error < 0) {
 		dev_err(&pdev->dev,
 			"dpmng_get_container_id() failed: %d\n", error);
@@ -801,8 +779,9 @@ static int fsl_mc_bus_probe(struct platform_device *pdev)
 	}
 
 	memset(&obj_desc, 0, sizeof(struct dprc_obj_desc));
-	error = get_dprc_version(mc_io, container_id,
-				 &obj_desc.ver_major, &obj_desc.ver_minor);
+	error = dprc_get_api_version(mc_io, 0,
+				     &obj_desc.ver_major,
+				     &obj_desc.ver_minor);
 	if (error < 0)
 		goto error_cleanup_mc_io;
 
@@ -840,7 +819,6 @@ static int fsl_mc_bus_remove(struct platform_device *pdev)
 	fsl_destroy_mc_io(mc->root_mc_bus_dev->mc_io);
 	mc->root_mc_bus_dev->mc_io = NULL;
 
-	dev_info(&pdev->dev, "Root MC bus device removed");
 	return 0;
 }
 
@@ -875,12 +853,10 @@ static int __init fsl_mc_bus_driver_init(void)
 
 	error = bus_register(&fsl_mc_bus_type);
 	if (error < 0) {
-		pr_err("fsl-mc bus type registration failed: %d\n", error);
+		pr_err("bus type registration failed: %d\n", error);
 		goto error_cleanup_cache;
 	}
 
-	pr_info("fsl-mc bus type registered\n");
-
 	error = platform_driver_register(&fsl_mc_bus_driver);
 	if (error < 0) {
 		pr_err("platform_driver_register() failed: %d\n", error);
diff --git a/drivers/staging/fsl-mc/bus/fsl-mc-msi.c b/drivers/staging/fsl-mc/bus/fsl-mc-msi.c
index 3d46b1b1fa18..7975c6e6fee3 100644
--- a/drivers/staging/fsl-mc/bus/fsl-mc-msi.c
+++ b/drivers/staging/fsl-mc/bus/fsl-mc-msi.c
@@ -1,7 +1,7 @@
 /*
  * Freescale Management Complex (MC) bus driver MSI support
  *
- * Copyright (C) 2015 Freescale Semiconductor, Inc.
+ * Copyright (C) 2015-2016 Freescale Semiconductor, Inc.
  * Author: German Rivera <German.Rivera@freescale.com>
  *
  * This file is licensed under the terms of the GNU General Public
diff --git a/drivers/staging/fsl-mc/bus/fsl-mc-private.h b/drivers/staging/fsl-mc/bus/fsl-mc-private.h
index d459c2673f39..5c49c9d2df6a 100644
--- a/drivers/staging/fsl-mc/bus/fsl-mc-private.h
+++ b/drivers/staging/fsl-mc/bus/fsl-mc-private.h
@@ -10,6 +10,9 @@
 #ifndef _FSL_MC_PRIVATE_H_
 #define _FSL_MC_PRIVATE_H_
 
+#include "../include/mc.h"
+#include "../include/mc-bus.h"
+
 int __must_check fsl_mc_device_add(struct dprc_obj_desc *obj_desc,
 				   struct fsl_mc_io *mc_io,
 				   struct device *parent_dev,
diff --git a/drivers/staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c b/drivers/staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c
index 7a6ac640752f..6b1cd574644f 100644
--- a/drivers/staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c
+++ b/drivers/staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c
@@ -1,7 +1,7 @@
 /*
  * Freescale Management Complex (MC) bus driver MSI support
  *
- * Copyright (C) 2015 Freescale Semiconductor, Inc.
+ * Copyright (C) 2015-2016 Freescale Semiconductor, Inc.
  * Author: German Rivera <German.Rivera@freescale.com>
  *
  * This file is licensed under the terms of the GNU General Public
@@ -19,7 +19,7 @@
 #include "../include/mc-bus.h"
 
 static struct irq_chip its_msi_irq_chip = {
-	.name = "fsl-mc-bus-msi",
+	.name = "ITS-fMSI",
 	.irq_mask = irq_chip_mask_parent,
 	.irq_unmask = irq_chip_unmask_parent,
 	.irq_eoi = irq_chip_eoi_parent,
diff --git a/drivers/staging/fsl-mc/bus/mc-io.c b/drivers/staging/fsl-mc/bus/mc-io.c
index 798c965fe203..d66b87f0903b 100644
--- a/drivers/staging/fsl-mc/bus/mc-io.c
+++ b/drivers/staging/fsl-mc/bus/mc-io.c
@@ -1,4 +1,5 @@
-/* Copyright 2013-2016 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -11,7 +12,6 @@
  *       names of any contributors may be used to endorse or promote products
  *       derived from this software without specific prior written permission.
  *
- *
  * ALTERNATIVELY, this software may be distributed under the terms of the
  * GNU General Public License ("GPL") as published by the Free Software
  * Foundation, either version 2 of that License or (at your option) any
diff --git a/drivers/staging/fsl-mc/bus/mc-sys.c b/drivers/staging/fsl-mc/bus/mc-sys.c
index 285917c7c8e4..4d82802b384d 100644
--- a/drivers/staging/fsl-mc/bus/mc-sys.c
+++ b/drivers/staging/fsl-mc/bus/mc-sys.c
@@ -1,4 +1,5 @@
-/* Copyright 2013-2014 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * I/O services to send MC commands to the MC hardware
  *
@@ -13,7 +14,6 @@
  *       names of any contributors may be used to endorse or promote products
  *       derived from this software without specific prior written permission.
  *
- *
  * ALTERNATIVELY, this software may be distributed under the terms of the
  * GNU General Public License ("GPL") as published by the Free Software
  * Foundation, either version 2 of that License or (at your option) any
@@ -67,7 +67,7 @@ static u16 mc_cmd_hdr_read_cmdid(struct mc_command *cmd)
 	struct mc_cmd_header *hdr = (struct mc_cmd_header *)&cmd->header;
 	u16 cmd_id = le16_to_cpu(hdr->cmd_id);
 
-	return (cmd_id & MC_CMD_HDR_CMDID_MASK) >> MC_CMD_HDR_CMDID_SHIFT;
+	return cmd_id;
 }
 
 static int mc_status_to_error(enum mc_cmd_status status)
@@ -200,7 +200,7 @@ static int mc_polling_wait_preemptible(struct fsl_mc_io *mc_io,
 
 		if (time_after_eq(jiffies, jiffies_until_timeout)) {
 			dev_dbg(mc_io->dev,
-				"MC command timed out (portal: %#llx, obj handle: %#x, command: %#x)\n",
+				"MC command timed out (portal: %#llx, dprc handle: %#x, command: %#x)\n",
 				 mc_io->portal_phys_addr,
 				 (unsigned int)mc_cmd_hdr_read_token(cmd),
 				 (unsigned int)mc_cmd_hdr_read_cmdid(cmd));
@@ -240,7 +240,7 @@ static int mc_polling_wait_atomic(struct fsl_mc_io *mc_io,
 		timeout_usecs -= MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS;
 		if (timeout_usecs == 0) {
 			dev_dbg(mc_io->dev,
-				"MC command timed out (portal: %#llx, obj handle: %#x, command: %#x)\n",
+				"MC command timed out (portal: %#llx, dprc handle: %#x, command: %#x)\n",
 				 mc_io->portal_phys_addr,
 				 (unsigned int)mc_cmd_hdr_read_token(cmd),
 				 (unsigned int)mc_cmd_hdr_read_cmdid(cmd));
@@ -294,7 +294,7 @@ int mc_send_command(struct fsl_mc_io *mc_io, struct mc_command *cmd)
 
 	if (status != MC_CMD_STATUS_OK) {
 		dev_dbg(mc_io->dev,
-			"MC command failed: portal: %#llx, obj handle: %#x, command: %#x, status: %s (%#x)\n",
+			"MC command failed: portal: %#llx, dprc handle: %#x, command: %#x, status: %s (%#x)\n",
 			 mc_io->portal_phys_addr,
 			 (unsigned int)mc_cmd_hdr_read_token(cmd),
 			 (unsigned int)mc_cmd_hdr_read_cmdid(cmd),
diff --git a/drivers/staging/fsl-mc/include/dpbp.h b/drivers/staging/fsl-mc/include/dpbp.h
index e14e85a5d6df..bf34b1e0e730 100644
--- a/drivers/staging/fsl-mc/include/dpbp.h
+++ b/drivers/staging/fsl-mc/include/dpbp.h
@@ -1,4 +1,5 @@
-/* Copyright 2013-2015 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -32,7 +33,8 @@
 #ifndef __FSL_DPBP_H
 #define __FSL_DPBP_H
 
-/* Data Path Buffer Pool API
+/*
+ * Data Path Buffer Pool API
  * Contains initialization APIs and runtime control APIs for DPBP
  */
 
@@ -44,8 +46,8 @@ int dpbp_open(struct fsl_mc_io *mc_io,
 	      u16 *token);
 
 int dpbp_close(struct fsl_mc_io *mc_io,
-	       u32		cmd_flags,
-	       u16	token);
+	       u32 cmd_flags,
+	       u16 token);
 
 /**
  * struct dpbp_cfg - Structure representing DPBP configuration
@@ -55,14 +57,16 @@ struct dpbp_cfg {
 	u32 options;
 };
 
-int dpbp_create(struct fsl_mc_io	*mc_io,
-		u32		cmd_flags,
-		const struct dpbp_cfg	*cfg,
-		u16		*token);
+int dpbp_create(struct fsl_mc_io *mc_io,
+		u16 dprc_token,
+		u32 cmd_flags,
+		const struct dpbp_cfg *cfg,
+		u32 *obj_id);
 
 int dpbp_destroy(struct fsl_mc_io *mc_io,
+		 u16 dprc_token,
 		 u32 cmd_flags,
-		 u16 token);
+		 u32 obj_id);
 
 int dpbp_enable(struct fsl_mc_io *mc_io,
 		u32 cmd_flags,
@@ -88,85 +92,75 @@ int dpbp_reset(struct fsl_mc_io *mc_io,
  * @irq_num: A user defined number associated with this IRQ
  */
 struct dpbp_irq_cfg {
-	     u64		addr;
-	     u32		val;
-	     int		irq_num;
+	     u64 addr;
+	     u32 val;
+	     int irq_num;
 };
 
-int dpbp_set_irq(struct fsl_mc_io	*mc_io,
-		 u32		cmd_flags,
-		 u16		token,
-		 u8		irq_index,
-		 struct dpbp_irq_cfg	*irq_cfg);
-
-int dpbp_get_irq(struct fsl_mc_io	*mc_io,
-		 u32		cmd_flags,
-		 u16		token,
-		 u8		irq_index,
-		 int			*type,
-		 struct dpbp_irq_cfg	*irq_cfg);
-
-int dpbp_set_irq_enable(struct fsl_mc_io	*mc_io,
-			u32		cmd_flags,
-			u16		token,
-			u8			irq_index,
-			u8			en);
-
-int dpbp_get_irq_enable(struct fsl_mc_io	*mc_io,
-			u32		cmd_flags,
-			u16		token,
-			u8			irq_index,
-			u8			*en);
+int dpbp_set_irq(struct fsl_mc_io *mc_io,
+		 u32 cmd_flags,
+		 u16 token,
+		 u8 irq_index,
+		 struct dpbp_irq_cfg *irq_cfg);
+
+int dpbp_get_irq(struct fsl_mc_io *mc_io,
+		 u32 cmd_flags,
+		 u16 token,
+		 u8 irq_index,
+		 int *type,
+		 struct dpbp_irq_cfg *irq_cfg);
+
+int dpbp_set_irq_enable(struct fsl_mc_io *mc_io,
+			u32 cmd_flags,
+			u16 token,
+			u8 irq_index,
+			u8 en);
+
+int dpbp_get_irq_enable(struct fsl_mc_io *mc_io,
+			u32 cmd_flags,
+			u16 token,
+			u8 irq_index,
+			u8 *en);
 
 int dpbp_set_irq_mask(struct fsl_mc_io	*mc_io,
-		      u32		cmd_flags,
-		      u16		token,
-		      u8		irq_index,
-		      u32		mask);
+		      u32 cmd_flags,
+		      u16 token,
+		      u8 irq_index,
+		      u32 mask);
 
 int dpbp_get_irq_mask(struct fsl_mc_io	*mc_io,
-		      u32		cmd_flags,
-		      u16		token,
-		      u8		irq_index,
-		      u32		*mask);
-
-int dpbp_get_irq_status(struct fsl_mc_io	*mc_io,
-			u32		cmd_flags,
-			u16		token,
-			u8			irq_index,
-			u32		*status);
-
-int dpbp_clear_irq_status(struct fsl_mc_io	*mc_io,
-			  u32		cmd_flags,
-			  u16		token,
-			  u8		irq_index,
-			  u32		status);
+		      u32 cmd_flags,
+		      u16 token,
+		      u8 irq_index,
+		      u32 *mask);
+
+int dpbp_get_irq_status(struct fsl_mc_io *mc_io,
+			u32 cmd_flags,
+			u16 token,
+			u8 irq_index,
+			u32 *status);
+
+int dpbp_clear_irq_status(struct fsl_mc_io *mc_io,
+			  u32 cmd_flags,
+			  u16 token,
+			  u8 irq_index,
+			  u32 status);
 
 /**
  * struct dpbp_attr - Structure representing DPBP attributes
  * @id:		DPBP object ID
- * @version:	DPBP version
  * @bpid:	Hardware buffer pool ID; should be used as an argument in
  *		acquire/release operations on buffers
  */
 struct dpbp_attr {
 	int id;
-	/**
-	 * struct version - Structure representing DPBP version
-	 * @major:	DPBP major version
-	 * @minor:	DPBP minor version
-	 */
-	struct {
-		u16 major;
-		u16 minor;
-	} version;
 	u16 bpid;
 };
 
-int dpbp_get_attributes(struct fsl_mc_io	*mc_io,
-			u32	cmd_flags,
-			u16		token,
-			struct dpbp_attr	*attr);
+int dpbp_get_attributes(struct fsl_mc_io *mc_io,
+			u32 cmd_flags,
+			u16 token,
+			struct dpbp_attr *attr);
 
 /**
  *  DPBP notifications options
@@ -196,24 +190,29 @@ int dpbp_get_attributes(struct fsl_mc_io	*mc_io,
  * @options: Mask of available options; use 'DPBP_NOTIF_OPT_<X>' values
  */
 struct dpbp_notification_cfg {
-	u32	depletion_entry;
-	u32	depletion_exit;
-	u32	surplus_entry;
-	u32	surplus_exit;
-	u64	message_iova;
-	u64	message_ctx;
-	u16	options;
+	u32 depletion_entry;
+	u32 depletion_exit;
+	u32 surplus_entry;
+	u32 surplus_exit;
+	u64 message_iova;
+	u64 message_ctx;
+	u16 options;
 };
 
-int dpbp_set_notifications(struct fsl_mc_io	*mc_io,
-			   u32		cmd_flags,
-			   u16		token,
-			   struct dpbp_notification_cfg	*cfg);
+int dpbp_set_notifications(struct fsl_mc_io *mc_io,
+			   u32 cmd_flags,
+			   u16 token,
+			   struct dpbp_notification_cfg *cfg);
+
+int dpbp_get_notifications(struct fsl_mc_io *mc_io,
+			   u32 cmd_flags,
+			   u16 token,
+			   struct dpbp_notification_cfg *cfg);
 
-int dpbp_get_notifications(struct fsl_mc_io	*mc_io,
-			   u32		cmd_flags,
-			   u16		token,
-			   struct dpbp_notification_cfg	*cfg);
+int dpbp_get_api_version(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 *major_ver,
+			 u16 *minor_ver);
 
 /** @} */
 
diff --git a/drivers/staging/fsl-mc/include/dpmng.h b/drivers/staging/fsl-mc/include/dpmng.h
index e5cfd017f9a5..7d8e255da578 100644
--- a/drivers/staging/fsl-mc/include/dpmng.h
+++ b/drivers/staging/fsl-mc/include/dpmng.h
@@ -1,4 +1,5 @@
-/* Copyright 2013-2015 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -32,7 +33,8 @@
 #ifndef __FSL_DPMNG_H
 #define __FSL_DPMNG_H
 
-/* Management Complex General API
+/*
+ * Management Complex General API
  * Contains general API for the Management Complex firmware
  */
 
@@ -58,12 +60,12 @@ struct mc_version {
 	u32 revision;
 };
 
-int mc_get_version(struct fsl_mc_io	*mc_io,
-		   u32		cmd_flags,
-		   struct mc_version	*mc_ver_info);
+int mc_get_version(struct fsl_mc_io *mc_io,
+		   u32 cmd_flags,
+		   struct mc_version *mc_ver_info);
 
-int dpmng_get_container_id(struct fsl_mc_io	*mc_io,
-			   u32		cmd_flags,
-			   int			*container_id);
+int dpmng_get_container_id(struct fsl_mc_io *mc_io,
+			   u32 cmd_flags,
+			   int *container_id);
 
 #endif /* __FSL_DPMNG_H */
diff --git a/drivers/staging/fsl-mc/include/dprc.h b/drivers/staging/fsl-mc/include/dprc.h
index 593b2bbe7f71..f9ea769ccfab 100644
--- a/drivers/staging/fsl-mc/include/dprc.h
+++ b/drivers/staging/fsl-mc/include/dprc.h
@@ -1,4 +1,5 @@
-/* Copyright 2013-2015 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -34,7 +35,8 @@
 
 #include "mc-cmd.h"
 
-/* Data Path Resource Container API
+/*
+ * Data Path Resource Container API
  * Contains DPRC API for managing and querying DPAA resources
  */
 
@@ -70,12 +72,14 @@ int dprc_close(struct fsl_mc_io *mc_io,
  * and can be retrieved using dprc_get_attributes()
  */
 
-/* Spawn Policy Option allowed - Indicates that the new container is allowed
+/*
+ * Spawn Policy Option allowed - Indicates that the new container is allowed
  * to spawn and have its own child containers.
  */
 #define DPRC_CFG_OPT_SPAWN_ALLOWED		0x00000001
 
-/* General Container allocation policy - Indicates that the new container is
+/*
+ * General Container allocation policy - Indicates that the new container is
  * allowed to allocate requested resources from its parent container; if not
  * set, the container is only allowed to use resources in its own pools; Note
  * that this is a container's global policy, but the parent container may
@@ -83,12 +87,14 @@ int dprc_close(struct fsl_mc_io *mc_io,
  */
 #define DPRC_CFG_OPT_ALLOC_ALLOWED		0x00000002
 
-/* Object initialization allowed - software context associated with this
+/*
+ * Object initialization allowed - software context associated with this
  * container is allowed to invoke object initialization operations.
  */
 #define DPRC_CFG_OPT_OBJ_CREATE_ALLOWED	0x00000004
 
-/* Topology change allowed - software context associated with this
+/*
+ * Topology change allowed - software context associated with this
  * container is allowed to invoke topology operations, such as attach/detach
  * of network objects.
  */
@@ -116,17 +122,17 @@ struct dprc_cfg {
 	char label[16];
 };
 
-int dprc_create_container(struct fsl_mc_io	*mc_io,
-			  u32		cmd_flags,
-			  u16		token,
-			  struct dprc_cfg	*cfg,
-			  int			*child_container_id,
-			  u64		*child_portal_offset);
+int dprc_create_container(struct fsl_mc_io *mc_io,
+			  u32 cmd_flags,
+			  u16 token,
+			  struct dprc_cfg *cfg,
+			  int *child_container_id,
+			  u64 *child_portal_offset);
 
-int dprc_destroy_container(struct fsl_mc_io	*mc_io,
-			   u32		cmd_flags,
-			   u16		token,
-			   int			child_container_id);
+int dprc_destroy_container(struct fsl_mc_io *mc_io,
+			   u32 cmd_flags,
+			   u16 token,
+			   int child_container_id);
 
 int dprc_reset_container(struct fsl_mc_io *mc_io,
 			 u32 cmd_flags,
@@ -139,7 +145,7 @@ int dprc_reset_container(struct fsl_mc_io *mc_io,
 #define DPRC_IRQ_INDEX          0
 
 /* Number of dprc's IRQs */
-#define DPRC_NUM_OF_IRQS		1
+#define DPRC_NUM_OF_IRQS	1
 
 /* DPRC IRQ events */
 
@@ -151,12 +157,14 @@ int dprc_reset_container(struct fsl_mc_io *mc_io,
 #define DPRC_IRQ_EVENT_RES_ADDED		0x00000004
 /* IRQ event - Indicates that resources removed from the container */
 #define DPRC_IRQ_EVENT_RES_REMOVED		0x00000008
-/* IRQ event - Indicates that one of the descendant containers that opened by
+/*
+ * IRQ event - Indicates that one of the descendant containers that opened by
  * this container is destroyed
  */
 #define DPRC_IRQ_EVENT_CONTAINER_DESTROYED	0x00000010
 
-/* IRQ event - Indicates that on one of the container's opened object is
+/*
+ * IRQ event - Indicates that on one of the container's opened object is
  * destroyed
  */
 #define DPRC_IRQ_EVENT_OBJ_DESTROYED		0x00000020
@@ -171,59 +179,59 @@ int dprc_reset_container(struct fsl_mc_io *mc_io,
  * @irq_num:	A user defined number associated with this IRQ
  */
 struct dprc_irq_cfg {
-	     phys_addr_t	paddr;
-	     u32		val;
-	     int		irq_num;
+	     phys_addr_t paddr;
+	     u32 val;
+	     int irq_num;
 };
 
-int dprc_set_irq(struct fsl_mc_io	*mc_io,
-		 u32		cmd_flags,
-		 u16		token,
-		 u8		irq_index,
-		 struct dprc_irq_cfg	*irq_cfg);
-
-int dprc_get_irq(struct fsl_mc_io	*mc_io,
-		 u32		cmd_flags,
-		 u16		token,
-		 u8		irq_index,
-		 int			*type,
-		 struct dprc_irq_cfg	*irq_cfg);
-
-int dprc_set_irq_enable(struct fsl_mc_io	*mc_io,
-			u32		cmd_flags,
-			u16		token,
-			u8			irq_index,
-			u8			en);
-
-int dprc_get_irq_enable(struct fsl_mc_io	*mc_io,
-			u32		cmd_flags,
-			u16		token,
-			u8			irq_index,
-			u8			*en);
-
-int dprc_set_irq_mask(struct fsl_mc_io	*mc_io,
-		      u32		cmd_flags,
-		      u16		token,
-		      u8		irq_index,
-		      u32		mask);
-
-int dprc_get_irq_mask(struct fsl_mc_io	*mc_io,
-		      u32		cmd_flags,
-		      u16		token,
-		      u8		irq_index,
-		      u32		*mask);
-
-int dprc_get_irq_status(struct fsl_mc_io	*mc_io,
-			u32		cmd_flags,
-			u16		token,
-			u8			irq_index,
-			u32		*status);
-
-int dprc_clear_irq_status(struct fsl_mc_io	*mc_io,
-			  u32		cmd_flags,
-			  u16		token,
-			  u8		irq_index,
-			  u32		status);
+int dprc_set_irq(struct fsl_mc_io *mc_io,
+		 u32 cmd_flags,
+		 u16 token,
+		 u8 irq_index,
+		 struct dprc_irq_cfg *irq_cfg);
+
+int dprc_get_irq(struct fsl_mc_io *mc_io,
+		 u32 cmd_flags,
+		 u16 token,
+		 u8 irq_index,
+		 int *type,
+		 struct dprc_irq_cfg *irq_cfg);
+
+int dprc_set_irq_enable(struct fsl_mc_io *mc_io,
+			u32 cmd_flags,
+			u16 token,
+			u8 irq_index,
+			u8 en);
+
+int dprc_get_irq_enable(struct fsl_mc_io *mc_io,
+			u32 cmd_flags,
+			u16 token,
+			u8 irq_index,
+			u8 *en);
+
+int dprc_set_irq_mask(struct fsl_mc_io *mc_io,
+		      u32 cmd_flags,
+		      u16 token,
+		      u8 irq_index,
+		      u32 mask);
+
+int dprc_get_irq_mask(struct fsl_mc_io *mc_io,
+		      u32 cmd_flags,
+		      u16 token,
+		      u8 irq_index,
+		      u32 *mask);
+
+int dprc_get_irq_status(struct fsl_mc_io *mc_io,
+			u32 cmd_flags,
+			u16 token,
+			u8 irq_index,
+			u32 *status);
+
+int dprc_clear_irq_status(struct fsl_mc_io *mc_io,
+			  u32 cmd_flags,
+			  u16 token,
+			  u8 irq_index,
+			  u32 status);
 
 /**
  * struct dprc_attributes - Container attributes
@@ -231,63 +239,56 @@ int dprc_clear_irq_status(struct fsl_mc_io	*mc_io,
  * @icid: Container's ICID
  * @portal_id: Container's portal ID
  * @options: Container's options as set at container's creation
- * @version: DPRC version
  */
 struct dprc_attributes {
 	int container_id;
 	u16 icid;
 	int portal_id;
 	u64 options;
-	/**
-	 * struct version - DPRC version
-	 * @major: DPRC major version
-	 * @minor: DPRC minor version
-	 */
-	struct {
-		u16 major;
-		u16 minor;
-	} version;
 };
 
-int dprc_get_attributes(struct fsl_mc_io	*mc_io,
-			u32		cmd_flags,
-			u16		token,
-			struct dprc_attributes	*attributes);
+int dprc_get_attributes(struct fsl_mc_io *mc_io,
+			u32 cmd_flags,
+			u16 token,
+			struct dprc_attributes *attributes);
 
 int dprc_set_res_quota(struct fsl_mc_io	*mc_io,
-		       u32		cmd_flags,
-		       u16		token,
-		       int		child_container_id,
-		       char		*type,
-		       u16		quota);
+		       u32 cmd_flags,
+		       u16 token,
+		       int child_container_id,
+		       char *type,
+		       u16 quota);
 
 int dprc_get_res_quota(struct fsl_mc_io	*mc_io,
-		       u32		cmd_flags,
-		       u16		token,
-		       int		child_container_id,
-		       char		*type,
-		       u16		*quota);
+		       u32 cmd_flags,
+		       u16 token,
+		       int child_container_id,
+		       char *type,
+		       u16 *quota);
 
 /* Resource request options */
 
-/* Explicit resource ID request - The requested objects/resources
+/*
+ * Explicit resource ID request - The requested objects/resources
  * are explicit and sequential (in case of resources).
  * The base ID is given at res_req at base_align field
  */
-#define DPRC_RES_REQ_OPT_EXPLICIT		0x00000001
+#define DPRC_RES_REQ_OPT_EXPLICIT	0x00000001
 
-/* Aligned resources request - Relevant only for resources
+/*
+ * Aligned resources request - Relevant only for resources
  * request (and not objects). Indicates that resources base ID should be
  * sequential and aligned to the value given at dprc_res_req base_align field
  */
-#define DPRC_RES_REQ_OPT_ALIGNED		0x00000002
+#define DPRC_RES_REQ_OPT_ALIGNED	0x00000002
 
-/* Plugged Flag - Relevant only for object assignment request.
+/*
+ * Plugged Flag - Relevant only for object assignment request.
  * Indicates that after all objects assigned. An interrupt will be invoked at
  * the relevant GPP. The assigned object will be marked as plugged.
  * plugged objects can't be assigned from their container
  */
-#define DPRC_RES_REQ_OPT_PLUGGED		0x00000004
+#define DPRC_RES_REQ_OPT_PLUGGED	0x00000004
 
 /**
  * struct dprc_res_req - Resource request descriptor, to be used in assignment
@@ -312,33 +313,33 @@ struct dprc_res_req {
 	int id_base_align;
 };
 
-int dprc_assign(struct fsl_mc_io	*mc_io,
-		u32		cmd_flags,
-		u16		token,
-		int			container_id,
-		struct dprc_res_req	*res_req);
-
-int dprc_unassign(struct fsl_mc_io	*mc_io,
-		  u32		cmd_flags,
-		  u16		token,
-		  int			child_container_id,
-		  struct dprc_res_req	*res_req);
-
-int dprc_get_pool_count(struct fsl_mc_io	*mc_io,
-			u32		cmd_flags,
-			u16		token,
-			int			*pool_count);
-
-int dprc_get_pool(struct fsl_mc_io	*mc_io,
-		  u32		cmd_flags,
-		  u16		token,
-		  int			pool_index,
-		  char			*type);
+int dprc_assign(struct fsl_mc_io *mc_io,
+		u32 cmd_flags,
+		u16 token,
+		int container_id,
+		struct dprc_res_req *res_req);
+
+int dprc_unassign(struct fsl_mc_io *mc_io,
+		  u32 cmd_flags,
+		  u16 token,
+		  int child_container_id,
+		  struct dprc_res_req *res_req);
+
+int dprc_get_pool_count(struct fsl_mc_io *mc_io,
+			u32 cmd_flags,
+			u16 token,
+			int *pool_count);
+
+int dprc_get_pool(struct fsl_mc_io *mc_io,
+		  u32 cmd_flags,
+		  u16 token,
+		  int pool_index,
+		  char *type);
 
 int dprc_get_obj_count(struct fsl_mc_io *mc_io,
-		       u32		cmd_flags,
-		       u16		token,
-		       int		*obj_count);
+		       u32 cmd_flags,
+		       u16 token,
+		       int *obj_count);
 
 /* Objects Attributes Flags */
 
@@ -353,7 +354,7 @@ int dprc_get_obj_count(struct fsl_mc_io *mc_io,
  * masters;
  * user is responsible for proper memory handling through IOMMU configuration.
  */
-#define DPRC_OBJ_FLAG_NO_MEM_SHAREABILITY		0x0001
+#define DPRC_OBJ_FLAG_NO_MEM_SHAREABILITY	0x0001
 
 /**
  * struct dprc_obj_desc - Object descriptor, returned from dprc_get_obj()
@@ -381,41 +382,41 @@ struct dprc_obj_desc {
 	u16 flags;
 };
 
-int dprc_get_obj(struct fsl_mc_io	*mc_io,
-		 u32		cmd_flags,
-		 u16		token,
-		 int			obj_index,
-		 struct dprc_obj_desc	*obj_desc);
-
-int dprc_get_obj_desc(struct fsl_mc_io		*mc_io,
-		      u32		cmd_flags,
-			u16		token,
-			char			*obj_type,
-			int			obj_id,
-			struct dprc_obj_desc	*obj_desc);
-
-int dprc_set_obj_irq(struct fsl_mc_io		*mc_io,
-		     u32			cmd_flags,
-		     u16			token,
-		     char			*obj_type,
-		     int			obj_id,
-		     u8			irq_index,
-		     struct dprc_irq_cfg	*irq_cfg);
-
-int dprc_get_obj_irq(struct fsl_mc_io		*mc_io,
-		     u32			cmd_flags,
-		     u16			token,
-		     char			*obj_type,
-		     int			obj_id,
-		     u8			irq_index,
-		     int			*type,
-		     struct dprc_irq_cfg	*irq_cfg);
-
-int dprc_get_res_count(struct fsl_mc_io	*mc_io,
-		       u32		cmd_flags,
-		       u16		token,
-		       char		*type,
-		       int		*res_count);
+int dprc_get_obj(struct fsl_mc_io *mc_io,
+		 u32 cmd_flags,
+		 u16 token,
+		 int obj_index,
+		 struct dprc_obj_desc *obj_desc);
+
+int dprc_get_obj_desc(struct fsl_mc_io *mc_io,
+		      u32 cmd_flags,
+		      u16 token,
+		      char *obj_type,
+		      int obj_id,
+		      struct dprc_obj_desc *obj_desc);
+
+int dprc_set_obj_irq(struct fsl_mc_io *mc_io,
+		     u32 cmd_flags,
+		     u16 token,
+		     char *obj_type,
+		     int obj_id,
+		     u8 irq_index,
+		     struct dprc_irq_cfg *irq_cfg);
+
+int dprc_get_obj_irq(struct fsl_mc_io *mc_io,
+		     u32 cmd_flags,
+		     u16 token,
+		     char *obj_type,
+		     int obj_id,
+		     u8 irq_index,
+		     int *type,
+		     struct dprc_irq_cfg *irq_cfg);
+
+int dprc_get_res_count(struct fsl_mc_io *mc_io,
+		       u32 cmd_flags,
+		       u16 token,
+		       char *type,
+		       int *res_count);
 
 /**
  * enum dprc_iter_status - Iteration status
@@ -444,11 +445,11 @@ struct dprc_res_ids_range_desc {
 	enum dprc_iter_status iter_status;
 };
 
-int dprc_get_res_ids(struct fsl_mc_io			*mc_io,
-		     u32				cmd_flags,
-		     u16				token,
-		     char				*type,
-		     struct dprc_res_ids_range_desc	*range_desc);
+int dprc_get_res_ids(struct fsl_mc_io *mc_io,
+		     u32 cmd_flags,
+		     u16 token,
+		     char *type,
+		     struct dprc_res_ids_range_desc *range_desc);
 
 /* Region flags */
 /* Cacheable - Indicates that region should be mapped as cacheable */
@@ -481,20 +482,20 @@ struct dprc_region_desc {
 	enum dprc_region_type type;
 };
 
-int dprc_get_obj_region(struct fsl_mc_io	*mc_io,
-			u32		cmd_flags,
-			u16		token,
-			char			*obj_type,
-			int			obj_id,
-			u8			region_index,
-			struct dprc_region_desc	*region_desc);
-
-int dprc_set_obj_label(struct fsl_mc_io	*mc_io,
-		       u32		cmd_flags,
-		       u16		token,
-		       char		*obj_type,
-		       int		obj_id,
-		       char		*label);
+int dprc_get_obj_region(struct fsl_mc_io *mc_io,
+			u32 cmd_flags,
+			u16 token,
+			char *obj_type,
+			int obj_id,
+			u8 region_index,
+			struct dprc_region_desc *region_desc);
+
+int dprc_set_obj_label(struct fsl_mc_io *mc_io,
+		       u32 cmd_flags,
+		       u16 token,
+		       char *obj_type,
+		       int obj_id,
+		       char *label);
 
 /**
  * struct dprc_endpoint - Endpoint description for link connect/disconnect
@@ -521,24 +522,33 @@ struct dprc_connection_cfg {
 	u32 max_rate;
 };
 
-int dprc_connect(struct fsl_mc_io		*mc_io,
-		 u32			cmd_flags,
-		 u16			token,
-		 const struct dprc_endpoint	*endpoint1,
-		 const struct dprc_endpoint	*endpoint2,
+int dprc_connect(struct fsl_mc_io *mc_io,
+		 u32 cmd_flags,
+		 u16 token,
+		 const struct dprc_endpoint *endpoint1,
+		 const struct dprc_endpoint *endpoint2,
 		 const struct dprc_connection_cfg *cfg);
 
-int dprc_disconnect(struct fsl_mc_io		*mc_io,
-		    u32			cmd_flags,
-		    u16			token,
-		    const struct dprc_endpoint	*endpoint);
-
-int dprc_get_connection(struct fsl_mc_io		*mc_io,
-			u32			cmd_flags,
-			u16			token,
-			const struct dprc_endpoint	*endpoint1,
-			struct dprc_endpoint		*endpoint2,
-			int				*state);
+int dprc_disconnect(struct fsl_mc_io *mc_io,
+		    u32 cmd_flags,
+		    u16 token,
+		    const struct dprc_endpoint *endpoint);
+
+int dprc_get_connection(struct fsl_mc_io *mc_io,
+			u32 cmd_flags,
+			u16 token,
+			const struct dprc_endpoint *endpoint1,
+			struct dprc_endpoint *endpoint2,
+			int *state);
+
+int dprc_get_api_version(struct fsl_mc_io *mc_io,
+			 u32 cmd_flags,
+			 u16 *major_ver,
+			 u16 *minor_ver);
+
+int dprc_get_container_id(struct fsl_mc_io *mc_io,
+			  u32 cmd_flags,
+			  int *container_id);
 
 #endif /* _FSL_DPRC_H */
 
diff --git a/drivers/staging/fsl-mc/include/mc-bus.h b/drivers/staging/fsl-mc/include/mc-bus.h
index 170684a57ca2..42700de94d59 100644
--- a/drivers/staging/fsl-mc/include/mc-bus.h
+++ b/drivers/staging/fsl-mc/include/mc-bus.h
@@ -1,7 +1,7 @@
 /*
  * Freescale Management Complex (MC) bus declarations
  *
- * Copyright (C) 2014 Freescale Semiconductor, Inc.
+ * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
  * Author: German Rivera <German.Rivera@freescale.com>
  *
  * This file is licensed under the terms of the GNU General Public
@@ -42,8 +42,8 @@ struct msi_domain_info;
  */
 struct fsl_mc_resource_pool {
 	enum fsl_mc_pool_type type;
-	int16_t max_count;
-	int16_t free_count;
+	int max_count;
+	int free_count;
 	struct mutex mutex;	/* serializes access to free_list */
 	struct list_head free_list;
 	struct fsl_mc_bus *mc_bus;
diff --git a/drivers/staging/fsl-mc/include/mc-cmd.h b/drivers/staging/fsl-mc/include/mc-cmd.h
index 5decb9890c31..2e08aa31b084 100644
--- a/drivers/staging/fsl-mc/include/mc-cmd.h
+++ b/drivers/staging/fsl-mc/include/mc-cmd.h
@@ -1,4 +1,5 @@
-/* Copyright 2013-2015 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -48,6 +49,15 @@ struct mc_command {
 	u64 params[MC_CMD_NUM_OF_PARAMS];
 };
 
+struct mc_rsp_create {
+	__le32 object_id;
+};
+
+struct mc_rsp_api_ver {
+	__le16 major_ver;
+	__le16 minor_ver;
+};
+
 enum mc_cmd_status {
 	MC_CMD_STATUS_OK = 0x0, /* Completed successfully */
 	MC_CMD_STATUS_READY = 0x1, /* Ready to be processed */
@@ -72,11 +82,6 @@ enum mc_cmd_status {
 /* Command completion flag */
 #define MC_CMD_FLAG_INTR_DIS	0x01
 
-#define MC_CMD_HDR_CMDID_MASK		0xFFF0
-#define MC_CMD_HDR_CMDID_SHIFT		4
-#define MC_CMD_HDR_TOKEN_MASK		0xFFC0
-#define MC_CMD_HDR_TOKEN_SHIFT		6
-
 static inline u64 mc_encode_cmd_header(u16 cmd_id,
 				       u32 cmd_flags,
 				       u16 token)
@@ -84,10 +89,8 @@ static inline u64 mc_encode_cmd_header(u16 cmd_id,
 	u64 header = 0;
 	struct mc_cmd_header *hdr = (struct mc_cmd_header *)&header;
 
-	hdr->cmd_id = cpu_to_le16((cmd_id << MC_CMD_HDR_CMDID_SHIFT) &
-				  MC_CMD_HDR_CMDID_MASK);
-	hdr->token = cpu_to_le16((token << MC_CMD_HDR_TOKEN_SHIFT) &
-				 MC_CMD_HDR_TOKEN_MASK);
+	hdr->cmd_id = cpu_to_le16(cmd_id);
+	hdr->token  = cpu_to_le16(token);
 	hdr->status = MC_CMD_STATUS_READY;
 	if (cmd_flags & MC_CMD_FLAG_PRI)
 		hdr->flags_hw = MC_CMD_FLAG_PRI;
@@ -102,7 +105,26 @@ static inline u16 mc_cmd_hdr_read_token(struct mc_command *cmd)
 	struct mc_cmd_header *hdr = (struct mc_cmd_header *)&cmd->header;
 	u16 token = le16_to_cpu(hdr->token);
 
-	return (token & MC_CMD_HDR_TOKEN_MASK) >> MC_CMD_HDR_TOKEN_SHIFT;
+	return token;
+}
+
+static inline u32 mc_cmd_read_object_id(struct mc_command *cmd)
+{
+	struct mc_rsp_create *rsp_params;
+
+	rsp_params = (struct mc_rsp_create *)cmd->params;
+	return le32_to_cpu(rsp_params->object_id);
+}
+
+static inline void mc_cmd_read_api_version(struct mc_command *cmd,
+					   u16 *major_ver,
+					   u16 *minor_ver)
+{
+	struct mc_rsp_api_ver *rsp_params;
+
+	rsp_params = (struct mc_rsp_api_ver *)cmd->params;
+	*major_ver = le16_to_cpu(rsp_params->major_ver);
+	*minor_ver = le16_to_cpu(rsp_params->minor_ver);
 }
 
 #endif /* __FSL_MC_CMD_H */
diff --git a/drivers/staging/fsl-mc/include/mc-sys.h b/drivers/staging/fsl-mc/include/mc-sys.h
index 89ad0cf54702..dca7f9084e05 100644
--- a/drivers/staging/fsl-mc/include/mc-sys.h
+++ b/drivers/staging/fsl-mc/include/mc-sys.h
@@ -1,4 +1,5 @@
-/* Copyright 2013-2014 Freescale Semiconductor Inc.
+/*
+ * Copyright 2013-2016 Freescale Semiconductor Inc.
  *
  * Interface of the I/O services to send MC commands to the MC hardware
  *
diff --git a/drivers/staging/fsl-mc/include/mc.h b/drivers/staging/fsl-mc/include/mc.h
index f6e720e84460..1c46c0c2a895 100644
--- a/drivers/staging/fsl-mc/include/mc.h
+++ b/drivers/staging/fsl-mc/include/mc.h
@@ -1,7 +1,7 @@
 /*
  * Freescale Management Complex (MC) bus public interface
  *
- * Copyright (C) 2014 Freescale Semiconductor, Inc.
+ * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
  * Author: German Rivera <German.Rivera@freescale.com>
  *
  * This file is licensed under the terms of the GNU General Public
@@ -81,7 +81,7 @@ enum fsl_mc_pool_type {
  */
 struct fsl_mc_resource {
 	enum fsl_mc_pool_type type;
-	int32_t id;
+	s32 id;
 	void *data;
 	struct fsl_mc_resource_pool *parent_pool;
 	struct list_head node;
diff --git a/drivers/staging/fwserial/fwserial.c b/drivers/staging/fwserial/fwserial.c
index 49c718b91e55..41a49c8194e5 100644
--- a/drivers/staging/fwserial/fwserial.c
+++ b/drivers/staging/fwserial/fwserial.c
@@ -1667,12 +1667,6 @@ static inline void fill_plug_rsp_nack(struct fwserial_mgmt_pkt *pkt)
 	pkt->hdr.len = cpu_to_be16(mgmt_pkt_expected_len(pkt->hdr.code));
 }
 
-static inline void fill_unplug_req(struct fwserial_mgmt_pkt *pkt)
-{
-	pkt->hdr.code = cpu_to_be16(FWSC_VIRT_CABLE_UNPLUG);
-	pkt->hdr.len = cpu_to_be16(mgmt_pkt_expected_len(pkt->hdr.code));
-}
-
 static inline void fill_unplug_rsp_nack(struct fwserial_mgmt_pkt *pkt)
 {
 	pkt->hdr.code = cpu_to_be16(FWSC_VIRT_CABLE_UNPLUG_RSP | FWSC_RSP_NACK);
diff --git a/drivers/staging/gdm724x/gdm_lte.h b/drivers/staging/gdm724x/gdm_lte.h
index 88414e5a70cc..7ddeabc0e50a 100644
--- a/drivers/staging/gdm724x/gdm_lte.h
+++ b/drivers/staging/gdm724x/gdm_lte.h
@@ -47,15 +47,15 @@ struct phy_dev {
 	void	*priv_dev;
 	struct net_device *dev[MAX_NIC_TYPE];
 	int	(*send_hci_func)(void *priv_dev, void *data, int len,
-			void (*cb)(void *cb_data), void *cb_data);
+				 void (*cb)(void *cb_data), void *cb_data);
 	int	(*send_sdu_func)(void *priv_dev, void *data, int len,
-			unsigned int dftEpsId, unsigned int epsId,
-			void (*cb)(void *cb_data), void *cb_data,
-			int dev_idx, int nic_type);
+				 unsigned int dftEpsId, unsigned int epsId,
+				 void (*cb)(void *cb_data), void *cb_data,
+				 int dev_idx, int nic_type);
 	int	(*rcv_func)(void *priv_dev,
-			int (*cb)(void *cb_data, void *data, int len,
-				  int context),
-			void *cb_data, int context);
+			    int (*cb)(void *cb_data, void *data, int len,
+				      int context),
+			    void *cb_data, int context);
 	struct gdm_endian * (*get_endian)(void *priv_dev);
 };
 
diff --git a/drivers/staging/gdm724x/gdm_tty.h b/drivers/staging/gdm724x/gdm_tty.h
index 297438b4ddcb..195c5902989f 100644
--- a/drivers/staging/gdm724x/gdm_tty.h
+++ b/drivers/staging/gdm724x/gdm_tty.h
@@ -17,7 +17,6 @@
 #include <linux/types.h>
 #include <linux/tty.h>
 
-
 #define TTY_MAX_COUNT		2
 
 #define MAX_ISSUE_NUM 3
diff --git a/drivers/staging/gdm724x/netlink_k.h b/drivers/staging/gdm724x/netlink_k.h
index 7cf979b3f826..5ebd73157f5a 100644
--- a/drivers/staging/gdm724x/netlink_k.h
+++ b/drivers/staging/gdm724x/netlink_k.h
@@ -18,7 +18,8 @@
 #include <net/sock.h>
 
 struct sock *netlink_init(int unit,
-	void (*cb)(struct net_device *dev, u16 type, void *msg, int len));
+			  void (*cb)(struct net_device *dev,
+				     u16 type, void *msg, int len));
 int netlink_send(struct sock *sock, int group, u16 type, void *msg, int len);
 
 #endif /* _NETLINK_K_H_ */
diff --git a/drivers/staging/greybus/arche-apb-ctrl.c b/drivers/staging/greybus/arche-apb-ctrl.c
index 70323aa11f24..3fda0cd6bb42 100644
--- a/drivers/staging/greybus/arche-apb-ctrl.c
+++ b/drivers/staging/greybus/arche-apb-ctrl.c
@@ -183,7 +183,7 @@ static int standby_boot_seq(struct platform_device *pdev)
 	 * Pasted from WDM spec,
 	 *  - A falling edge on POWEROFF_L is detected (a)
 	 *  - WDM enters standby mode, but no output signals are changed
-	 * */
+	 */
 
 	/* TODO: POWEROFF_L is input to WDM module  */
 	apb->state = ARCHE_PLATFORM_STATE_STANDBY;
@@ -285,8 +285,10 @@ static ssize_t state_store(struct device *dev,
 		if (apb->state == ARCHE_PLATFORM_STATE_FW_FLASHING)
 			return count;
 
-		/* First we want to make sure we power off everything
-		 * and then enter FW flashing state */
+		/*
+		 * First we want to make sure we power off everything
+		 * and then enter FW flashing state
+		 */
 		poweroff_seq(pdev);
 		ret = fw_flashing_seq(pdev);
 	} else {
diff --git a/drivers/staging/greybus/arche-platform.c b/drivers/staging/greybus/arche-platform.c
index d33d6fe078ad..338c2d3ee842 100644
--- a/drivers/staging/greybus/arche-platform.c
+++ b/drivers/staging/greybus/arche-platform.c
@@ -457,7 +457,8 @@ retry:
 			goto exit;
 
 		/* First we want to make sure we power off everything
-		 * and then activate back again */
+		 * and then activate back again
+		 */
 		device_for_each_child(arche_pdata->dev, NULL, apb_poweroff);
 		arche_platform_poweroff_seq(arche_pdata);
 
diff --git a/drivers/staging/greybus/audio_codec.c b/drivers/staging/greybus/audio_codec.c
index 8a0744b58a32..f8862c6d7102 100644
--- a/drivers/staging/greybus/audio_codec.c
+++ b/drivers/staging/greybus/audio_codec.c
@@ -405,7 +405,6 @@ static void gbcodec_shutdown(struct snd_pcm_substream *substream,
 	params->state = GBAUDIO_CODEC_SHUTDOWN;
 	mutex_unlock(&codec->lock);
 	pm_relax(dai->dev);
-	return;
 }
 
 static int gbcodec_hw_params(struct snd_pcm_substream *substream,
@@ -655,8 +654,10 @@ static int gbcodec_mute_stream(struct snd_soc_dai *dai, int mute, int stream)
 			ret = gb_audio_apbridgea_shutdown_rx(data->connection,
 							     0);
 		params->state = GBAUDIO_CODEC_STOP;
-	} else
+	} else {
 		ret = -EINVAL;
+	}
+
 	if (ret)
 		dev_err_ratelimited(dai->dev,
 				    "%s:Error during %s %s stream:%d\n",
diff --git a/drivers/staging/greybus/audio_codec.h b/drivers/staging/greybus/audio_codec.h
index ca027bd99ad7..62fd93939a1f 100644
--- a/drivers/staging/greybus/audio_codec.h
+++ b/drivers/staging/greybus/audio_codec.h
@@ -158,7 +158,6 @@ struct gbaudio_module_info {
 	int dev_id;	/* check if it should be bundle_id/hd_cport_id */
 	int vid;
 	int pid;
-	int slot;
 	int type;
 	int set_uevent;
 	char vstr[NAME_SIZE];
diff --git a/drivers/staging/greybus/audio_manager.h b/drivers/staging/greybus/audio_manager.h
index c4ca09754a6a..5ab8f5e0ed3f 100644
--- a/drivers/staging/greybus/audio_manager.h
+++ b/drivers/staging/greybus/audio_manager.h
@@ -18,10 +18,9 @@
 
 struct gb_audio_manager_module_descriptor {
 	char name[GB_AUDIO_MANAGER_MODULE_NAME_LEN];
-	int slot;
 	int vid;
 	int pid;
-	int cport;
+	int intf_id;
 	unsigned int ip_devices;
 	unsigned int op_devices;
 };
diff --git a/drivers/staging/greybus/audio_manager_module.c b/drivers/staging/greybus/audio_manager_module.c
index a10e96ad79c1..adc16977452d 100644
--- a/drivers/staging/greybus/audio_manager_module.c
+++ b/drivers/staging/greybus/audio_manager_module.c
@@ -81,16 +81,6 @@ static ssize_t gb_audio_module_name_show(
 static struct gb_audio_manager_module_attribute gb_audio_module_name_attribute =
 	__ATTR(name, 0664, gb_audio_module_name_show, NULL);
 
-static ssize_t gb_audio_module_slot_show(
-	struct gb_audio_manager_module *module,
-	struct gb_audio_manager_module_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d", module->desc.slot);
-}
-
-static struct gb_audio_manager_module_attribute gb_audio_module_slot_attribute =
-	__ATTR(slot, 0664, gb_audio_module_slot_show, NULL);
-
 static ssize_t gb_audio_module_vid_show(
 	struct gb_audio_manager_module *module,
 	struct gb_audio_manager_module_attribute *attr, char *buf)
@@ -111,16 +101,16 @@ static ssize_t gb_audio_module_pid_show(
 static struct gb_audio_manager_module_attribute gb_audio_module_pid_attribute =
 	__ATTR(pid, 0664, gb_audio_module_pid_show, NULL);
 
-static ssize_t gb_audio_module_cport_show(
+static ssize_t gb_audio_module_intf_id_show(
 	struct gb_audio_manager_module *module,
 	struct gb_audio_manager_module_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d", module->desc.cport);
+	return sprintf(buf, "%d", module->desc.intf_id);
 }
 
 static struct gb_audio_manager_module_attribute
-					gb_audio_module_cport_attribute =
-	__ATTR(cport, 0664, gb_audio_module_cport_show, NULL);
+					gb_audio_module_intf_id_attribute =
+	__ATTR(intf_id, 0664, gb_audio_module_intf_id_show, NULL);
 
 static ssize_t gb_audio_module_ip_devices_show(
 	struct gb_audio_manager_module *module,
@@ -146,10 +136,9 @@ static struct gb_audio_manager_module_attribute
 
 static struct attribute *gb_audio_module_default_attrs[] = {
 	&gb_audio_module_name_attribute.attr,
-	&gb_audio_module_slot_attribute.attr,
 	&gb_audio_module_vid_attribute.attr,
 	&gb_audio_module_pid_attribute.attr,
-	&gb_audio_module_cport_attribute.attr,
+	&gb_audio_module_intf_id_attribute.attr,
 	&gb_audio_module_ip_devices_attribute.attr,
 	&gb_audio_module_op_devices_attribute.attr,
 	NULL,   /* need to NULL terminate the list of attributes */
@@ -164,29 +153,26 @@ static struct kobj_type gb_audio_module_type = {
 static void send_add_uevent(struct gb_audio_manager_module *module)
 {
 	char name_string[128];
-	char slot_string[64];
 	char vid_string[64];
 	char pid_string[64];
-	char cport_string[64];
+	char intf_id_string[64];
 	char ip_devices_string[64];
 	char op_devices_string[64];
 
 	char *envp[] = {
 		name_string,
-		slot_string,
 		vid_string,
 		pid_string,
-		cport_string,
+		intf_id_string,
 		ip_devices_string,
 		op_devices_string,
 		NULL
 	};
 
 	snprintf(name_string, 128, "NAME=%s", module->desc.name);
-	snprintf(slot_string, 64, "SLOT=%d", module->desc.slot);
 	snprintf(vid_string, 64, "VID=%d", module->desc.vid);
 	snprintf(pid_string, 64, "PID=%d", module->desc.pid);
-	snprintf(cport_string, 64, "CPORT=%d", module->desc.cport);
+	snprintf(intf_id_string, 64, "INTF_ID=%d", module->desc.intf_id);
 	snprintf(ip_devices_string, 64, "I/P DEVICES=0x%X",
 		 module->desc.ip_devices);
 	snprintf(op_devices_string, 64, "O/P DEVICES=0x%X",
@@ -246,13 +232,12 @@ int gb_audio_manager_module_create(
 
 void gb_audio_manager_module_dump(struct gb_audio_manager_module *module)
 {
-	pr_info("audio module #%d name=%s slot=%d vid=%d pid=%d cport=%d i/p devices=0x%X o/p devices=0x%X\n",
+	pr_info("audio module #%d name=%s vid=%d pid=%d intf_id=%d i/p devices=0x%X o/p devices=0x%X\n",
 		module->id,
 		module->desc.name,
-		module->desc.slot,
 		module->desc.vid,
 		module->desc.pid,
-		module->desc.cport,
+		module->desc.intf_id,
 		module->desc.ip_devices,
 		module->desc.op_devices);
 }
diff --git a/drivers/staging/greybus/audio_manager_sysfs.c b/drivers/staging/greybus/audio_manager_sysfs.c
index d8bf8591ff9e..34ebd147052f 100644
--- a/drivers/staging/greybus/audio_manager_sysfs.c
+++ b/drivers/staging/greybus/audio_manager_sysfs.c
@@ -20,10 +20,9 @@ static ssize_t manager_sysfs_add_store(
 
 	int num = sscanf(buf,
 			"name=%" GB_AUDIO_MANAGER_MODULE_NAME_LEN_SSCANF "s "
-			"slot=%d vid=%d pid=%d cport=%d i/p devices=0x%X"
-			"o/p devices=0x%X",
-			desc.name, &desc.slot, &desc.vid, &desc.pid,
-			&desc.cport, &desc.ip_devices, &desc.op_devices);
+			"vid=%d pid=%d intf_id=%d i/p devices=0x%X o/p devices=0x%X",
+			desc.name, &desc.vid, &desc.pid, &desc.intf_id,
+			&desc.ip_devices, &desc.op_devices);
 
 	if (num != 7)
 		return -EINVAL;
@@ -44,7 +43,7 @@ static ssize_t manager_sysfs_remove_store(
 {
 	int id;
 
-	int num = sscanf(buf, "%d", &id);
+	int num = kstrtoint(buf, 10, &id);
 
 	if (num != 1)
 		return -EINVAL;
@@ -65,16 +64,17 @@ static ssize_t manager_sysfs_dump_store(
 {
 	int id;
 
-	int num = sscanf(buf, "%d", &id);
+	int num = kstrtoint(buf, 10, &id);
 
 	if (num == 1) {
 		num = gb_audio_manager_dump_module(id);
 		if (num)
 			return num;
-	} else if (!strncmp("all", buf, 3))
+	} else if (!strncmp("all", buf, 3)) {
 		gb_audio_manager_dump_all();
-	else
+	} else {
 		return -EINVAL;
+	}
 
 	return count;
 }
diff --git a/drivers/staging/greybus/audio_module.c b/drivers/staging/greybus/audio_module.c
index ae1c0fa85752..17a9948b1ba1 100644
--- a/drivers/staging/greybus/audio_module.c
+++ b/drivers/staging/greybus/audio_module.c
@@ -207,10 +207,8 @@ static int gb_audio_add_data_connection(struct gbaudio_module_info *gbmodule,
 	struct gbaudio_data_connection *dai;
 
 	dai = devm_kzalloc(gbmodule->dev, sizeof(*dai), GFP_KERNEL);
-	if (!dai) {
-		dev_err(gbmodule->dev, "DAI Malloc failure\n");
+	if (!dai)
 		return -ENOMEM;
-	}
 
 	connection = gb_connection_create_offloaded(bundle,
 					le16_to_cpu(cport_desc->id),
@@ -345,10 +343,9 @@ static int gb_audio_probe(struct gb_bundle *bundle,
 	dev_dbg(dev, "Inform set_event:%d to above layer\n", 1);
 	/* prepare for the audio manager */
 	strlcpy(desc.name, gbmodule->name, GB_AUDIO_MANAGER_MODULE_NAME_LEN);
-	desc.slot = 1; /* todo */
 	desc.vid = 2; /* todo */
 	desc.pid = 3; /* todo */
-	desc.cport = gbmodule->dev_id;
+	desc.intf_id = gbmodule->dev_id;
 	desc.op_devices = gbmodule->op_devices;
 	desc.ip_devices = gbmodule->ip_devices;
 	gbmodule->manager_id = gb_audio_manager_add(&desc);
diff --git a/drivers/staging/greybus/audio_topology.c b/drivers/staging/greybus/audio_topology.c
index b6251691a33d..8b216ca99cf9 100644
--- a/drivers/staging/greybus/audio_topology.c
+++ b/drivers/staging/greybus/audio_topology.c
@@ -114,6 +114,7 @@ static int gbaudio_map_widgetname(struct gbaudio_module_info *module,
 				  const char *name)
 {
 	struct gbaudio_widget *widget;
+
 	list_for_each_entry(widget, &module->widget_list, list) {
 		if (!strncmp(widget->name, name, NAME_SIZE))
 			return widget->id;
@@ -1044,8 +1045,10 @@ static int gbaudio_tplg_create_widget(struct gbaudio_module_info *module,
 			control->texts = (const char * const *)
 				gb_generate_enum_strings(module, gbenum);
 			control->items = gbenum->items;
-		} else
+		} else {
 			csize = sizeof(struct gb_audio_control);
+		}
+
 		*w_size += csize;
 		curr = (void *)curr + csize;
 		list_add(&control->list, &module->widget_ctl_list);
@@ -1190,8 +1193,9 @@ static int gbaudio_tplg_process_kcontrols(struct gbaudio_module_info *module,
 			control->texts = (const char * const *)
 				gb_generate_enum_strings(module, gbenum);
 			control->items = gbenum->items;
-		} else
+		} else {
 			csize = sizeof(struct gb_audio_control);
+		}
 
 		list_add(&control->list, &module->ctl_list);
 		dev_dbg(module->dev, "%d:%s created of type %d\n", curr->id,
diff --git a/drivers/staging/greybus/camera.c b/drivers/staging/greybus/camera.c
index 491bdd720c0c..1c5b41ae6774 100644
--- a/drivers/staging/greybus/camera.c
+++ b/drivers/staging/greybus/camera.c
@@ -289,6 +289,7 @@ static const int gb_camera_configure_streams_validate_response(
 
 	for (i = 0; i < resp->num_streams; i++) {
 		struct gb_camera_stream_config_response *cfg = &resp->config[i];
+
 		if (cfg->padding) {
 			gcam_err(gcam, "stream #%u padding != 0\n", i);
 			return -EIO;
@@ -796,7 +797,7 @@ static int gb_camera_op_configure_streams(void *priv, unsigned int *nstreams,
 	if (gb_nstreams > GB_CAMERA_MAX_STREAMS)
 		return -EINVAL;
 
-	gb_streams = kzalloc(gb_nstreams * sizeof(*gb_streams), GFP_KERNEL);
+	gb_streams = kcalloc(gb_nstreams, sizeof(*gb_streams), GFP_KERNEL);
 	if (!gb_streams)
 		return -ENOMEM;
 
@@ -937,7 +938,7 @@ static ssize_t gb_camera_debugfs_configure_streams(struct gb_camera *gcam,
 		return ret;
 
 	/* For each stream to configure parse width, height and format */
-	streams = kzalloc(nstreams * sizeof(*streams), GFP_KERNEL);
+	streams = kcalloc(nstreams, sizeof(*streams), GFP_KERNEL);
 	if (!streams)
 		return -ENOMEM;
 
@@ -1118,7 +1119,7 @@ static ssize_t gb_camera_debugfs_write(struct file *file,
 	char *kbuf;
 
 	if (len > 1024)
-	       return -EINVAL;
+		return -EINVAL;
 
 	kbuf = kmalloc(len + 1, GFP_KERNEL);
 	if (!kbuf)
diff --git a/drivers/staging/greybus/es2.c b/drivers/staging/greybus/es2.c
index baab460eeaa3..f1d256df06d5 100644
--- a/drivers/staging/greybus/es2.c
+++ b/drivers/staging/greybus/es2.c
@@ -175,10 +175,9 @@ static int output_sync(struct es2_ap_dev *es2, void *req, u16 size, u8 cmd)
 	u8 *data;
 	int retval;
 
-	data = kmalloc(size, GFP_KERNEL);
+	data = kmemdup(req, size, GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
-	memcpy(data, req, size);
 
 	retval = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
 				 cmd,
@@ -1034,7 +1033,7 @@ static struct arpc *arpc_alloc(void *payload, u16 size, u8 type)
 		goto err_free_req;
 
 	rpc->req->type = type;
-	rpc->req->size = cpu_to_le16(sizeof(rpc->req) + size);
+	rpc->req->size = cpu_to_le16(sizeof(*rpc->req) + size);
 	memcpy(rpc->req->data, payload, size);
 
 	init_completion(&rpc->response_received);
diff --git a/drivers/staging/greybus/log.c b/drivers/staging/greybus/log.c
index 70dd9e5a1cf2..1a18ab1ff8aa 100644
--- a/drivers/staging/greybus/log.c
+++ b/drivers/staging/greybus/log.c
@@ -55,8 +55,10 @@ static int gb_log_request_handler(struct gb_operation *op)
 	/* Ensure the buffer is 0 terminated */
 	receive->msg[len - 1] = '\0';
 
-	/* Print with dev_dbg() so that it can be easily turned off using
-	 * dynamic debugging (and prevent any DoS) */
+	/*
+	 * Print with dev_dbg() so that it can be easily turned off using
+	 * dynamic debugging (and prevent any DoS)
+	 */
 	dev_dbg(dev, "%s", receive->msg);
 
 	return 0;
diff --git a/drivers/staging/greybus/sdio.c b/drivers/staging/greybus/sdio.c
index 5649ef1e379d..66b37ea29ef0 100644
--- a/drivers/staging/greybus/sdio.c
+++ b/drivers/staging/greybus/sdio.c
@@ -191,9 +191,8 @@ static int _gb_sdio_process_events(struct gb_sdio_host *host, u8 event)
 		state_changed = 1;
 	}
 
-	if (event & GB_SDIO_WP) {
+	if (event & GB_SDIO_WP)
 		host->read_only = true;
-	}
 
 	if (state_changed) {
 		dev_info(mmc_dev(host->mmc), "card %s now event\n",
diff --git a/drivers/staging/greybus/timesync.c b/drivers/staging/greybus/timesync.c
index 2e68af7dea6d..e586627f4bbc 100644
--- a/drivers/staging/greybus/timesync.c
+++ b/drivers/staging/greybus/timesync.c
@@ -807,11 +807,11 @@ static int gb_timesync_schedule(struct gb_timesync_svc *timesync_svc, int state)
 		return -EINVAL;
 
 	mutex_lock(&timesync_svc->mutex);
-	if (timesync_svc->state !=  GB_TIMESYNC_STATE_INVALID) {
+	if (timesync_svc->state !=  GB_TIMESYNC_STATE_INVALID)
 		gb_timesync_set_state_atomic(timesync_svc, state);
-	} else {
+	else
 		ret = -ENODEV;
-	}
+
 	mutex_unlock(&timesync_svc->mutex);
 	return ret;
 }
diff --git a/drivers/staging/greybus/uart.c b/drivers/staging/greybus/uart.c
index 2633d2bfb1b4..6d39f4a04754 100644
--- a/drivers/staging/greybus/uart.c
+++ b/drivers/staging/greybus/uart.c
@@ -623,9 +623,6 @@ static int get_serial_info(struct gb_tty *gb_tty,
 {
 	struct serial_struct tmp;
 
-	if (!info)
-		return -EINVAL;
-
 	memset(&tmp, 0, sizeof(tmp));
 	tmp.flags = ASYNC_LOW_LATENCY | ASYNC_SKIP_TEST;
 	tmp.type = PORT_16550A;
@@ -711,25 +708,20 @@ static int wait_serial_change(struct gb_tty *gb_tty, unsigned long arg)
 	return retval;
 }
 
-static int get_serial_usage(struct gb_tty *gb_tty,
-			    struct serial_icounter_struct __user *count)
+static int gb_tty_get_icount(struct tty_struct *tty,
+			     struct serial_icounter_struct *icount)
 {
-	struct serial_icounter_struct icount;
-	int retval = 0;
-
-	memset(&icount, 0, sizeof(icount));
-	icount.dsr = gb_tty->iocount.dsr;
-	icount.rng = gb_tty->iocount.rng;
-	icount.dcd = gb_tty->iocount.dcd;
-	icount.frame = gb_tty->iocount.frame;
-	icount.overrun = gb_tty->iocount.overrun;
-	icount.parity = gb_tty->iocount.parity;
-	icount.brk = gb_tty->iocount.brk;
+	struct gb_tty *gb_tty = tty->driver_data;
 
-	if (copy_to_user(count, &icount, sizeof(icount)) > 0)
-		retval = -EFAULT;
+	icount->dsr = gb_tty->iocount.dsr;
+	icount->rng = gb_tty->iocount.rng;
+	icount->dcd = gb_tty->iocount.dcd;
+	icount->frame = gb_tty->iocount.frame;
+	icount->overrun = gb_tty->iocount.overrun;
+	icount->parity = gb_tty->iocount.parity;
+	icount->brk = gb_tty->iocount.brk;
 
-	return retval;
+	return 0;
 }
 
 static int gb_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
@@ -746,9 +738,6 @@ static int gb_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
 				       (struct serial_struct __user *)arg);
 	case TIOCMIWAIT:
 		return wait_serial_change(gb_tty, arg);
-	case TIOCGICOUNT:
-		return get_serial_usage(gb_tty,
-					(struct serial_icounter_struct __user *)arg);
 	}
 
 	return -ENOIOCTLCMD;
@@ -830,9 +819,10 @@ static const struct tty_operations gb_ops = {
 	.set_termios =		gb_tty_set_termios,
 	.tiocmget =		gb_tty_tiocmget,
 	.tiocmset =		gb_tty_tiocmset,
+	.get_icount =		gb_tty_get_icount,
 };
 
-static struct tty_port_operations gb_port_ops = {
+static const struct tty_port_operations gb_port_ops = {
 	.dtr_rts =		gb_tty_dtr_rts,
 	.activate =		gb_tty_port_activate,
 	.shutdown =		gb_tty_port_shutdown,
diff --git a/drivers/staging/gs_fpgaboot/gs_fpgaboot.c b/drivers/staging/gs_fpgaboot/gs_fpgaboot.c
index 8ed4d395be58..19b550fff04b 100644
--- a/drivers/staging/gs_fpgaboot/gs_fpgaboot.c
+++ b/drivers/staging/gs_fpgaboot/gs_fpgaboot.c
@@ -38,7 +38,7 @@ static u8 bits_magic[] = {
 static struct platform_device	*firmware_pdev;
 
 static char	*file = "xlinx_fpga_firmware.bit";
-module_param(file, charp, S_IRUGO);
+module_param(file, charp, 0444);
 MODULE_PARM_DESC(file, "Xilinx FPGA firmware file.");
 
 static void read_bitstream(char *bitdata, char *buf, int *offset, int rdsize)
diff --git a/drivers/staging/i4l/act2000/act2000_isa.c b/drivers/staging/i4l/act2000/act2000_isa.c
index ad7a0391369f..76ff5de65781 100644
--- a/drivers/staging/i4l/act2000/act2000_isa.c
+++ b/drivers/staging/i4l/act2000/act2000_isa.c
@@ -259,6 +259,7 @@ act2000_isa_receive(act2000_card *card)
 					       "act2000_isa_receive: Invalid CAPI msg\n");
 					{
 						int i; __u8 *p; __u8 *t; __u8 tmp[30];
+
 						for (i = 0, p = (__u8 *)&card->idat.isa.rcvhdr, t = tmp; i < 8; i++)
 							t += sprintf(t, "%02x ", *(p++));
 						printk(KERN_WARNING "act2000_isa_receive: %s\n", tmp);
diff --git a/drivers/staging/i4l/act2000/capi.c b/drivers/staging/i4l/act2000/capi.c
index 62f56294853c..61386a78fb91 100644
--- a/drivers/staging/i4l/act2000/capi.c
+++ b/drivers/staging/i4l/act2000/capi.c
@@ -99,7 +99,7 @@ actcapi_chkhdr(act2000_card *card, actcapi_msghdr *hdr)
 	for (i = 0; i < num_valid_imsg; i++)
 		if ((hdr->cmd.cmd == valid_msg[i].cmd.cmd) &&
 		    (hdr->cmd.subcmd == valid_msg[i].cmd.subcmd)) {
-			return (i ? 1 : 2);
+			return i ? 1 : 2;
 		}
 	return 0;
 }
@@ -506,6 +506,7 @@ static int
 new_plci(act2000_card *card, __u16 plci)
 {
 	int i;
+
 	for (i = 0; i < ACT2000_BCH; i++)
 		if (card->bch[i].plci == 0x8000) {
 			card->bch[i].plci = plci;
@@ -518,6 +519,7 @@ static int
 find_plci(act2000_card *card, __u16 plci)
 {
 	int i;
+
 	for (i = 0; i < ACT2000_BCH; i++)
 		if (card->bch[i].plci == plci)
 			return i;
@@ -528,6 +530,7 @@ static int
 find_ncci(act2000_card *card, __u16 ncci)
 {
 	int i;
+
 	for (i = 0; i < ACT2000_BCH; i++)
 		if (card->bch[i].ncci == ncci)
 			return i;
@@ -538,6 +541,7 @@ static int
 find_dialing(act2000_card *card, __u16 callref)
 {
 	int i;
+
 	for (i = 0; i < ACT2000_BCH; i++)
 		if ((card->bch[i].callref == callref) &&
 		    (card->bch[i].fsm_state == ACT2000_STATE_OCALL))
@@ -1088,6 +1092,7 @@ actcapi_debug_msg(struct sk_buff *skb, int direction)
 			int l = msg->hdr.len - 12;
 			int j;
 			char *p = tmp;
+
 			for (j = 0; j < l; j++)
 				p += sprintf(p, "%02x ", msg->msg.info_ind.el.display[j]);
 			printk(KERN_DEBUG " D = '%s'\n", tmp);
diff --git a/drivers/staging/i4l/act2000/module.c b/drivers/staging/i4l/act2000/module.c
index 99c9c0a1c63e..6aa120319e52 100644
--- a/drivers/staging/i4l/act2000/module.c
+++ b/drivers/staging/i4l/act2000/module.c
@@ -19,8 +19,7 @@
 #include <linux/slab.h>
 #include <linux/init.h>
 
-static unsigned short act2000_isa_ports[] =
-{
+static unsigned short act2000_isa_ports[] = {
 	0x0200, 0x0240, 0x0280, 0x02c0, 0x0300, 0x0340, 0x0380,
 	0xcfe0, 0xcfa0, 0xcf60, 0xcf20, 0xcee0, 0xcea0, 0xce60,
 };
@@ -95,7 +94,7 @@ act2000_find_msn(act2000_card *card, char *msn, int ia5)
 		p = p->next;
 	}
 	if (!ia5)
-		return (1 << (eaz - '0'));
+		return 1 << (eaz - '0');
 	else
 		return eaz;
 }
@@ -111,10 +110,10 @@ act2000_find_eaz(act2000_card *card, char eaz)
 
 	while (p) {
 		if (p->eaz == eaz)
-			return (p->msn);
+			return p->msn;
 		p = p->next;
 	}
-	return ("\0");
+	return "\0";
 }
 
 /*
@@ -293,7 +292,7 @@ act2000_command(act2000_card *card, isdn_ctrl *c)
 			if (ret)
 				return ret;
 			if (card->flags & ACT2000_FLAGS_RUNNING)
-				return (actcapi_manufacturer_req_msn(card));
+				return actcapi_manufacturer_req_msn(card);
 			return 0;
 		case ACT2000_IOCTL_ADDCARD:
 			if (copy_from_user(&cdef, arg,
@@ -377,6 +376,7 @@ act2000_command(act2000_card *card, isdn_ctrl *c)
 			}
 			if (card->ptype == ISDN_PTYPE_1TR6) {
 				int i;
+
 				chan->eazmask = 0;
 				for (i = 0; i < strlen(c->parm.num); i++)
 					if (isdigit(c->parm.num[i]))
@@ -512,7 +512,7 @@ if_command(isdn_ctrl *c)
 	act2000_card *card = act2000_findcard(c->driver);
 
 	if (card)
-		return (act2000_command(card, c));
+		return act2000_command(card, c);
 	printk(KERN_ERR
 	       "act2000: if_command %d called with invalid driverId %d!\n",
 	       c->command, c->driver);
@@ -527,7 +527,7 @@ if_writecmd(const u_char __user *buf, int len, int id, int channel)
 	if (card) {
 		if (!(card->flags & ACT2000_FLAGS_RUNNING))
 			return -ENODEV;
-		return (len);
+		return len;
 	}
 	printk(KERN_ERR
 	       "act2000: if_writecmd called with invalid driverId!\n");
@@ -542,7 +542,7 @@ if_readstatus(u_char __user *buf, int len, int id, int channel)
 	if (card) {
 		if (!(card->flags & ACT2000_FLAGS_RUNNING))
 			return -ENODEV;
-		return (act2000_readstatus(buf, len, card));
+		return act2000_readstatus(buf, len, card);
 	}
 	printk(KERN_ERR
 	       "act2000: if_readstatus called with invalid driverId!\n");
@@ -557,7 +557,7 @@ if_sendbuf(int id, int channel, int ack, struct sk_buff *skb)
 	if (card) {
 		if (!(card->flags & ACT2000_FLAGS_RUNNING))
 			return -ENODEV;
-		return (act2000_sendbuf(card, channel, ack, skb));
+		return act2000_sendbuf(card, channel, ack, skb);
 	}
 	printk(KERN_ERR
 	       "act2000: if_sendbuf called with invalid driverId!\n");
@@ -574,6 +574,7 @@ act2000_alloccard(int bus, int port, int irq, char *id)
 {
 	int i;
 	act2000_card *card;
+
 	if (!(card = kzalloc(sizeof(act2000_card), GFP_KERNEL))) {
 		printk(KERN_WARNING
 		       "act2000: (%s) Could not allocate card-struct.\n", id);
@@ -776,7 +777,7 @@ act2000_addcard(int bus, int port, int irq, char *id)
 			failed++;
 		}
 	}
-	return (added - failed);
+	return added - failed;
 }
 
 #define DRIVERNAME "IBM Active 2000 ISDN driver"
@@ -795,6 +796,7 @@ static void __exit act2000_exit(void)
 {
 	act2000_card *card = cards;
 	act2000_card *last;
+
 	while (card) {
 		unregister_card(card);
 		del_timer_sync(&card->ptimer);
diff --git a/drivers/staging/i4l/icn/icn.c b/drivers/staging/i4l/icn/icn.c
index 514bfc2c5b53..3750ba38adc5 100644
--- a/drivers/staging/i4l/icn/icn.c
+++ b/drivers/staging/i4l/icn/icn.c
@@ -411,8 +411,7 @@ typedef struct icn_stat {
 	int action;
 } icn_stat;
 /* *INDENT-OFF* */
-static icn_stat icn_stat_table[] =
-{
+static icn_stat icn_stat_table[] = {
 	{"BCON_",          ISDN_STAT_BCONN, 1},	/* B-Channel connected        */
 	{"BDIS_",          ISDN_STAT_BHUP,  2},	/* B-Channel disconnected     */
 	/*
diff --git a/drivers/staging/i4l/icn/icn.h b/drivers/staging/i4l/icn/icn.h
index f8f2e76d34bf..07e2e0196527 100644
--- a/drivers/staging/i4l/icn/icn.h
+++ b/drivers/staging/i4l/icn/icn.h
@@ -54,7 +54,7 @@ typedef struct icn_cdef {
 
 /* some useful macros for debugging */
 #ifdef ICN_DEBUG_PORT
-#define OUTB_P(v, p) {printk(KERN_DEBUG "icn: outb_p(0x%02x,0x%03x)\n", v, p); outb_p(v, p);}
+#define OUTB_P(v, p) {pr_debug("icn: outb_p(0x%02x,0x%03x)\n", v, p); outb_p(v, p);}
 #else
 #define OUTB_P outb
 #endif
@@ -186,8 +186,7 @@ typedef icn_dev *icn_devptr;
 #ifdef __KERNEL__
 
 static icn_card *cards = (icn_card *) 0;
-static u_char chan2bank[] =
-{0, 4, 8, 12};                  /* for icn_map_channel() */
+static u_char chan2bank[] = {0, 4, 8, 12};                  /* for icn_map_channel() */
 
 static icn_dev dev;
 
diff --git a/drivers/staging/i4l/pcbit/callbacks.c b/drivers/staging/i4l/pcbit/callbacks.c
index efb6d6a3639a..212ab0b229d4 100644
--- a/drivers/staging/i4l/pcbit/callbacks.c
+++ b/drivers/staging/i4l/pcbit/callbacks.c
@@ -22,7 +22,7 @@
 #include <linux/mm.h>
 #include <linux/skbuff.h>
 
-#include <asm/io.h>
+#include <linux/io.h>
 
 #include <linux/isdnif.h>
 
diff --git a/drivers/staging/i4l/pcbit/capi.c b/drivers/staging/i4l/pcbit/capi.c
index 373f90feda5a..a6c4e00dc726 100644
--- a/drivers/staging/i4l/pcbit/capi.c
+++ b/drivers/staging/i4l/pcbit/capi.c
@@ -27,7 +27,6 @@
  *              encode our number in CallerPN and ConnectedPN
  */
 
-#include <linux/string.h>
 #include <linux/kernel.h>
 
 #include <linux/types.h>
@@ -36,8 +35,8 @@
 
 #include <linux/skbuff.h>
 
-#include <asm/io.h>
-#include <asm/string.h>
+#include <linux/io.h>
+#include <linux/string.h>
 
 #include <linux/isdnif.h>
 
diff --git a/drivers/staging/i4l/pcbit/drv.c b/drivers/staging/i4l/pcbit/drv.c
index d417df5efb5f..89b0b5b94ce5 100644
--- a/drivers/staging/i4l/pcbit/drv.c
+++ b/drivers/staging/i4l/pcbit/drv.c
@@ -27,12 +27,11 @@
 #include <linux/slab.h>
 #include <linux/mm.h>
 #include <linux/interrupt.h>
-#include <linux/string.h>
 #include <linux/skbuff.h>
 
 #include <linux/isdnif.h>
-#include <asm/string.h>
-#include <asm/io.h>
+#include <linux/string.h>
+#include <linux/io.h>
 #include <linux/ioport.h>
 
 #include "pcbit.h"
diff --git a/drivers/staging/i4l/pcbit/edss1.c b/drivers/staging/i4l/pcbit/edss1.c
index 6d291d548423..5980d1b5da95 100644
--- a/drivers/staging/i4l/pcbit/edss1.c
+++ b/drivers/staging/i4l/pcbit/edss1.c
@@ -23,7 +23,7 @@
 #include <linux/skbuff.h>
 
 #include <linux/timer.h>
-#include <asm/io.h>
+#include <linux/io.h>
 
 #include <linux/isdnif.h>
 
diff --git a/drivers/staging/i4l/pcbit/layer2.c b/drivers/staging/i4l/pcbit/layer2.c
index a136c72547e5..0592bf6ee9c9 100644
--- a/drivers/staging/i4l/pcbit/layer2.c
+++ b/drivers/staging/i4l/pcbit/layer2.c
@@ -36,7 +36,7 @@
 
 #include <linux/isdnif.h>
 
-#include <asm/io.h>
+#include <linux/io.h>
 
 
 #include "pcbit.h"
diff --git a/drivers/staging/iio/Documentation/light/sysfs-bus-iio-light-tsl2583 b/drivers/staging/iio/Documentation/light/sysfs-bus-iio-light-tsl2583
deleted file mode 100644
index 470f7ad9c073..000000000000
--- a/drivers/staging/iio/Documentation/light/sysfs-bus-iio-light-tsl2583
+++ /dev/null
@@ -1,6 +0,0 @@
-What:		/sys/bus/iio/devices/device[n]/in_illuminance0_calibrate
-KernelVersion:	2.6.37
-Contact:	linux-iio@vger.kernel.org
-Description:
-		This property causes an internal calibration of the als gain trim
-		value which is later used in calculating illuminance in lux.
diff --git a/drivers/staging/iio/TODO b/drivers/staging/iio/TODO
index 93a896883e37..4922402e2e98 100644
--- a/drivers/staging/iio/TODO
+++ b/drivers/staging/iio/TODO
@@ -1,76 +1,8 @@
-2009 8/18
-
-Core:
-1) Get reviews
-2) Additional testing
-3) Ensure all desirable features present by adding more devices.
-   Major changes not expected except in response to comments
-
-Max1363 core:
-1) Possibly add sysfs exports of constant useful to userspace.
-Would be nice
-2) Support hardware generated interrupts
-3) Expand device set. Lots of other maxim adc's have very
-   similar interfaces.
-
-MXS LRADC driver:
-This is a classic MFD device as it combines the following subdevices
- - touchscreen controller (input subsystem related device)
- - general purpose ADC channels
- - battery voltage monitor (power subsystem related device)
- - die temperature monitor (thermal management)
-
-At least the battery voltage and die temperature feature is required in-kernel
-by a driver of the SoC's battery charging unit to avoid any damage to the
-silicon and the battery.
-
-TSL2561
-Would be nice
-1) Open question of userspace vs kernel space balance when
-converting to useful light measurements from device ones.
-2) Add sysfs elements necessary to allow device agnostic
-unit conversion.
-
-LIS3L02DQ core
-
-LIS3L02DQ ring
-
-KXSD9
-Currently minimal driver, would be nice to add:
-1) Support for all chip generated interrupts (events),
-basically get support up to level of lis3l02dq driver.
-
-Ring buffer core
-
-SCA3000
-Would be nice
-1) Testing on devices other than sca3000-e05
-
-Trigger core support
-1) Discussion of approach. Is it general enough?
-
-Ring Buffer:
-1) Discussion of approach.
-There are probably better ways of doing this. The
-intention is to allow for more than one software ring
-buffer implementation as different users will have
-different requirements.  This one suits mid range
-frequencies (100Hz - 4kHz).
-2) Lots of testing
-
-GPIO trigger
-1) Add control over the type of interrupt etc.  This will
-necessitate a header that is also visible from arch board
-files. (avoided at the moment to keep the driver set
-contained in staging).
+2016 10/09
 
 ADI Drivers:
 CC the device-drivers-devel@blackfin.uclinux.org mailing list when
 e-mailing the normal IIO list (see below).
 
-Documentation
-1) Lots of cleanup and expansion.
-2) Some device require individual docs.
-
 Contact: Jonathan Cameron <jic23@kernel.org>.
 Mailing list: linux-iio@vger.kernel.org
diff --git a/drivers/staging/iio/accel/Kconfig b/drivers/staging/iio/accel/Kconfig
index 1c994b57c7d2..c6b0f5eae7ab 100644
--- a/drivers/staging/iio/accel/Kconfig
+++ b/drivers/staging/iio/accel/Kconfig
@@ -51,14 +51,4 @@ config ADIS16240
 	  To compile this driver as a module, say M here: the module will be
 	  called adis16240.
 
-config SCA3000
-	depends on IIO_BUFFER
-	depends on SPI
-	tristate "VTI SCA3000 series accelerometers"
-	help
-	  Say Y here to build support for the VTI SCA3000 series of SPI
-	  accelerometers. These devices use a hardware ring buffer.
-
-	  To compile this driver as a module, say M here: the module will be
-	  called sca3000.
 endmenu
diff --git a/drivers/staging/iio/accel/Makefile b/drivers/staging/iio/accel/Makefile
index 1810a434a755..febb137b60c4 100644
--- a/drivers/staging/iio/accel/Makefile
+++ b/drivers/staging/iio/accel/Makefile
@@ -13,6 +13,3 @@ obj-$(CONFIG_ADIS16209) += adis16209.o
 
 adis16240-y             := adis16240_core.o
 obj-$(CONFIG_ADIS16240) += adis16240.o
-
-sca3000-y		:= sca3000_core.o sca3000_ring.o
-obj-$(CONFIG_SCA3000)	+= sca3000.o
diff --git a/drivers/staging/iio/accel/sca3000.h b/drivers/staging/iio/accel/sca3000.h
deleted file mode 100644
index 4dcc8575cbe3..000000000000
--- a/drivers/staging/iio/accel/sca3000.h
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * sca3000.c -- support VTI sca3000 series accelerometers
- *              via SPI
- *
- * Copyright (c) 2007 Jonathan Cameron <jic23@kernel.org>
- *
- * Partly based upon tle62x0.c
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * Initial mode is direct measurement.
- *
- * Untested things
- *
- * Temperature reading (the e05 I'm testing with doesn't have a sensor)
- *
- * Free fall detection mode - supported but untested as I'm not droping my
- * dubious wire rig far enough to test it.
- *
- * Unsupported as yet
- *
- * Time stamping of data from ring. Various ideas on how to do this but none
- * are remotely simple. Suggestions welcome.
- *
- * Individual enabling disabling of channels going into ring buffer
- *
- * Overflow handling (this is signaled for all but 8 bit ring buffer mode.)
- *
- * Motion detector using AND combinations of signals.
- *
- * Note: Be very careful about not touching an register bytes marked
- * as reserved on the data sheet. They really mean it as changing convents of
- * some will cause the device to lock up.
- *
- * Known issues - on rare occasions the interrupts lock up. Not sure why as yet.
- * Can probably alleviate this by reading the interrupt register on start, but
- * that is really just brushing the problem under the carpet.
- */
-#ifndef _SCA3000
-#define _SCA3000
-
-#define SCA3000_WRITE_REG(a) (((a) << 2) | 0x02)
-#define SCA3000_READ_REG(a) ((a) << 2)
-
-#define SCA3000_REG_ADDR_REVID			0x00
-#define SCA3000_REVID_MAJOR_MASK		0xf0
-#define SCA3000_REVID_MINOR_MASK		0x0f
-
-#define SCA3000_REG_ADDR_STATUS			0x02
-#define SCA3000_LOCKED				0x20
-#define SCA3000_EEPROM_CS_ERROR			0x02
-#define SCA3000_SPI_FRAME_ERROR			0x01
-
-/* All reads done using register decrement so no need to directly access LSBs */
-#define SCA3000_REG_ADDR_X_MSB			0x05
-#define SCA3000_REG_ADDR_Y_MSB			0x07
-#define SCA3000_REG_ADDR_Z_MSB			0x09
-
-#define SCA3000_REG_ADDR_RING_OUT		0x0f
-
-/* Temp read untested - the e05 doesn't have the sensor */
-#define SCA3000_REG_ADDR_TEMP_MSB		0x13
-
-#define SCA3000_REG_ADDR_MODE			0x14
-#define SCA3000_MODE_PROT_MASK			0x28
-
-#define SCA3000_RING_BUF_ENABLE			0x80
-#define SCA3000_RING_BUF_8BIT			0x40
-/*
- * Free fall detection triggers an interrupt if the acceleration
- * is below a threshold for equivalent of 25cm drop
- */
-#define SCA3000_FREE_FALL_DETECT		0x10
-#define SCA3000_MEAS_MODE_NORMAL		0x00
-#define SCA3000_MEAS_MODE_OP_1			0x01
-#define SCA3000_MEAS_MODE_OP_2			0x02
-
-/*
- * In motion detection mode the accelerations are band pass filtered
- * (approx 1 - 25Hz) and then a programmable threshold used to trigger
- * and interrupt.
- */
-#define SCA3000_MEAS_MODE_MOT_DET		0x03
-
-#define SCA3000_REG_ADDR_BUF_COUNT		0x15
-
-#define SCA3000_REG_ADDR_INT_STATUS		0x16
-
-#define SCA3000_INT_STATUS_THREE_QUARTERS	0x80
-#define SCA3000_INT_STATUS_HALF			0x40
-
-#define SCA3000_INT_STATUS_FREE_FALL		0x08
-#define SCA3000_INT_STATUS_Y_TRIGGER		0x04
-#define SCA3000_INT_STATUS_X_TRIGGER		0x02
-#define SCA3000_INT_STATUS_Z_TRIGGER		0x01
-
-/* Used to allow access to multiplexed registers */
-#define SCA3000_REG_ADDR_CTRL_SEL		0x18
-/* Only available for SCA3000-D03 and SCA3000-D01 */
-#define SCA3000_REG_CTRL_SEL_I2C_DISABLE	0x01
-#define SCA3000_REG_CTRL_SEL_MD_CTRL		0x02
-#define SCA3000_REG_CTRL_SEL_MD_Y_TH		0x03
-#define SCA3000_REG_CTRL_SEL_MD_X_TH		0x04
-#define SCA3000_REG_CTRL_SEL_MD_Z_TH		0x05
-/*
- * BE VERY CAREFUL WITH THIS, IF 3 BITS ARE NOT SET the device
- * will not function
- */
-#define SCA3000_REG_CTRL_SEL_OUT_CTRL		0x0B
-#define SCA3000_OUT_CTRL_PROT_MASK		0xE0
-#define SCA3000_OUT_CTRL_BUF_X_EN		0x10
-#define SCA3000_OUT_CTRL_BUF_Y_EN		0x08
-#define SCA3000_OUT_CTRL_BUF_Z_EN		0x04
-#define SCA3000_OUT_CTRL_BUF_DIV_MASK		0x03
-#define SCA3000_OUT_CTRL_BUF_DIV_4		0x02
-#define SCA3000_OUT_CTRL_BUF_DIV_2		0x01
-
-/*
- * Control which motion detector interrupts are on.
- * For now only OR combinations are supported.
- */
-#define SCA3000_MD_CTRL_PROT_MASK		0xC0
-#define SCA3000_MD_CTRL_OR_Y			0x01
-#define SCA3000_MD_CTRL_OR_X			0x02
-#define SCA3000_MD_CTRL_OR_Z			0x04
-/* Currently unsupported */
-#define SCA3000_MD_CTRL_AND_Y			0x08
-#define SCA3000_MD_CTRL_AND_X			0x10
-#define SAC3000_MD_CTRL_AND_Z			0x20
-
-/*
- * Some control registers of complex access methods requiring this register to
- * be used to remove a lock.
- */
-#define SCA3000_REG_ADDR_UNLOCK			0x1e
-
-#define SCA3000_REG_ADDR_INT_MASK		0x21
-#define SCA3000_INT_MASK_PROT_MASK		0x1C
-
-#define SCA3000_INT_MASK_RING_THREE_QUARTER	0x80
-#define SCA3000_INT_MASK_RING_HALF		0x40
-
-#define SCA3000_INT_MASK_ALL_INTS		0x02
-#define SCA3000_INT_MASK_ACTIVE_HIGH		0x01
-#define SCA3000_INT_MASK_ACTIVE_LOW		0x00
-
-/* Values of multiplexed registers (write to ctrl_data after select) */
-#define SCA3000_REG_ADDR_CTRL_DATA		0x22
-
-/*
- * Measurement modes available on some sca3000 series chips. Code assumes others
- * may become available in the future.
- *
- * Bypass - Bypass the low-pass filter in the signal channel so as to increase
- *          signal bandwidth.
- *
- * Narrow - Narrow low-pass filtering of the signal channel and half output
- *          data rate by decimation.
- *
- * Wide - Widen low-pass filtering of signal channel to increase bandwidth
- */
-#define SCA3000_OP_MODE_BYPASS			0x01
-#define SCA3000_OP_MODE_NARROW			0x02
-#define SCA3000_OP_MODE_WIDE			0x04
-#define SCA3000_MAX_TX 6
-#define SCA3000_MAX_RX 2
-
-/**
- * struct sca3000_state - device instance state information
- * @us:			the associated spi device
- * @info:			chip variant information
- * @interrupt_handler_ws:	event interrupt handler for all events
- * @last_timestamp:		the timestamp of the last event
- * @mo_det_use_count:		reference counter for the motion detection unit
- * @lock:			lock used to protect elements of sca3000_state
- *				and the underlying device state.
- * @bpse:			number of bits per scan element
- * @tx:			dma-able transmit buffer
- * @rx:			dma-able receive buffer
- **/
-struct sca3000_state {
-	struct spi_device		*us;
-	const struct sca3000_chip_info	*info;
-	struct work_struct		interrupt_handler_ws;
-	s64				last_timestamp;
-	int				mo_det_use_count;
-	struct mutex			lock;
-	int				bpse;
-	/* Can these share a cacheline ? */
-	u8				rx[2] ____cacheline_aligned;
-	u8				tx[6] ____cacheline_aligned;
-};
-
-/**
- * struct sca3000_chip_info - model dependent parameters
- * @scale:			scale * 10^-6
- * @temp_output:		some devices have temperature sensors.
- * @measurement_mode_freq:	normal mode sampling frequency
- * @option_mode_1:		first optional mode. Not all models have one
- * @option_mode_1_freq:		option mode 1 sampling frequency
- * @option_mode_2:		second optional mode. Not all chips have one
- * @option_mode_2_freq:		option mode 2 sampling frequency
- *
- * This structure is used to hold information about the functionality of a given
- * sca3000 variant.
- **/
-struct sca3000_chip_info {
-	unsigned int		scale;
-	bool			temp_output;
-	int			measurement_mode_freq;
-	int			option_mode_1;
-	int			option_mode_1_freq;
-	int			option_mode_2;
-	int			option_mode_2_freq;
-	int			mot_det_mult_xz[6];
-	int			mot_det_mult_y[7];
-};
-
-int sca3000_read_data_short(struct sca3000_state *st,
-			    u8 reg_address_high,
-			    int len);
-
-/**
- * sca3000_write_reg() write a single register
- * @address:	address of register on chip
- * @val:	value to be written to register
- *
- * The main lock must be held.
- **/
-int sca3000_write_reg(struct sca3000_state *st, u8 address, u8 val);
-
-#ifdef CONFIG_IIO_BUFFER
-/**
- * sca3000_register_ring_funcs() setup the ring state change functions
- **/
-void sca3000_register_ring_funcs(struct iio_dev *indio_dev);
-
-/**
- * sca3000_configure_ring() - allocate and configure ring buffer
- * @indio_dev: iio-core device whose ring is to be configured
- *
- * The hardware ring buffer needs far fewer ring buffer functions than
- * a software one as a lot of things are handled automatically.
- * This function also tells the iio core that our device supports a
- * hardware ring buffer mode.
- **/
-int sca3000_configure_ring(struct iio_dev *indio_dev);
-
-/**
- * sca3000_unconfigure_ring() - deallocate the ring buffer
- * @indio_dev: iio-core device whose ring we are freeing
- **/
-void sca3000_unconfigure_ring(struct iio_dev *indio_dev);
-
-/**
- * sca3000_ring_int_process() handles ring related event pushing and escalation
- * @val:	the event code
- **/
-void sca3000_ring_int_process(u8 val, struct iio_buffer *ring);
-
-#else
-static inline void sca3000_register_ring_funcs(struct iio_dev *indio_dev)
-{
-}
-
-static inline
-int sca3000_register_ring_access_and_init(struct iio_dev *indio_dev)
-{
-	return 0;
-}
-
-static inline void sca3000_ring_int_process(u8 val, void *ring)
-{
-}
-
-#endif
-#endif /* _SCA3000 */
diff --git a/drivers/staging/iio/accel/sca3000_core.c b/drivers/staging/iio/accel/sca3000_core.c
deleted file mode 100644
index 564b36d4f648..000000000000
--- a/drivers/staging/iio/accel/sca3000_core.c
+++ /dev/null
@@ -1,1210 +0,0 @@
-/*
- * sca3000_core.c -- support VTI sca3000 series accelerometers via SPI
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * Copyright (c) 2009 Jonathan Cameron <jic23@kernel.org>
- *
- * See industrialio/accels/sca3000.h for comments.
- */
-
-#include <linux/interrupt.h>
-#include <linux/fs.h>
-#include <linux/device.h>
-#include <linux/slab.h>
-#include <linux/kernel.h>
-#include <linux/spi/spi.h>
-#include <linux/sysfs.h>
-#include <linux/module.h>
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-#include <linux/iio/events.h>
-#include <linux/iio/buffer.h>
-
-#include "sca3000.h"
-
-enum sca3000_variant {
-	d01,
-	e02,
-	e04,
-	e05,
-};
-
-/*
- * Note where option modes are not defined, the chip simply does not
- * support any.
- * Other chips in the sca3000 series use i2c and are not included here.
- *
- * Some of these devices are only listed in the family data sheet and
- * do not actually appear to be available.
- */
-static const struct sca3000_chip_info sca3000_spi_chip_info_tbl[] = {
-	[d01] = {
-		.scale = 7357,
-		.temp_output = true,
-		.measurement_mode_freq = 250,
-		.option_mode_1 = SCA3000_OP_MODE_BYPASS,
-		.option_mode_1_freq = 250,
-		.mot_det_mult_xz = {50, 100, 200, 350, 650, 1300},
-		.mot_det_mult_y = {50, 100, 150, 250, 450, 850, 1750},
-	},
-	[e02] = {
-		.scale = 9810,
-		.measurement_mode_freq = 125,
-		.option_mode_1 = SCA3000_OP_MODE_NARROW,
-		.option_mode_1_freq = 63,
-		.mot_det_mult_xz = {100, 150, 300, 550, 1050, 2050},
-		.mot_det_mult_y = {50, 100, 200, 350, 700, 1350, 2700},
-	},
-	[e04] = {
-		.scale = 19620,
-		.measurement_mode_freq = 100,
-		.option_mode_1 = SCA3000_OP_MODE_NARROW,
-		.option_mode_1_freq = 50,
-		.option_mode_2 = SCA3000_OP_MODE_WIDE,
-		.option_mode_2_freq = 400,
-		.mot_det_mult_xz = {200, 300, 600, 1100, 2100, 4100},
-		.mot_det_mult_y = {100, 200, 400, 7000, 1400, 2700, 54000},
-	},
-	[e05] = {
-		.scale = 61313,
-		.measurement_mode_freq = 200,
-		.option_mode_1 = SCA3000_OP_MODE_NARROW,
-		.option_mode_1_freq = 50,
-		.option_mode_2 = SCA3000_OP_MODE_WIDE,
-		.option_mode_2_freq = 400,
-		.mot_det_mult_xz = {600, 900, 1700, 3200, 6100, 11900},
-		.mot_det_mult_y = {300, 600, 1200, 2000, 4100, 7800, 15600},
-	},
-};
-
-int sca3000_write_reg(struct sca3000_state *st, u8 address, u8 val)
-{
-	st->tx[0] = SCA3000_WRITE_REG(address);
-	st->tx[1] = val;
-	return spi_write(st->us, st->tx, 2);
-}
-
-int sca3000_read_data_short(struct sca3000_state *st,
-			    u8 reg_address_high,
-			    int len)
-{
-	struct spi_transfer xfer[2] = {
-		{
-			.len = 1,
-			.tx_buf = st->tx,
-		}, {
-			.len = len,
-			.rx_buf = st->rx,
-		}
-	};
-	st->tx[0] = SCA3000_READ_REG(reg_address_high);
-
-	return spi_sync_transfer(st->us, xfer, ARRAY_SIZE(xfer));
-}
-
-/**
- * sca3000_reg_lock_on() test if the ctrl register lock is on
- *
- * Lock must be held.
- **/
-static int sca3000_reg_lock_on(struct sca3000_state *st)
-{
-	int ret;
-
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_STATUS, 1);
-	if (ret < 0)
-		return ret;
-
-	return !(st->rx[0] & SCA3000_LOCKED);
-}
-
-/**
- * __sca3000_unlock_reg_lock() unlock the control registers
- *
- * Note the device does not appear to support doing this in a single transfer.
- * This should only ever be used as part of ctrl reg read.
- * Lock must be held before calling this
- **/
-static int __sca3000_unlock_reg_lock(struct sca3000_state *st)
-{
-	struct spi_transfer xfer[3] = {
-		{
-			.len = 2,
-			.cs_change = 1,
-			.tx_buf = st->tx,
-		}, {
-			.len = 2,
-			.cs_change = 1,
-			.tx_buf = st->tx + 2,
-		}, {
-			.len = 2,
-			.tx_buf = st->tx + 4,
-		},
-	};
-	st->tx[0] = SCA3000_WRITE_REG(SCA3000_REG_ADDR_UNLOCK);
-	st->tx[1] = 0x00;
-	st->tx[2] = SCA3000_WRITE_REG(SCA3000_REG_ADDR_UNLOCK);
-	st->tx[3] = 0x50;
-	st->tx[4] = SCA3000_WRITE_REG(SCA3000_REG_ADDR_UNLOCK);
-	st->tx[5] = 0xA0;
-
-	return spi_sync_transfer(st->us, xfer, ARRAY_SIZE(xfer));
-}
-
-/**
- * sca3000_write_ctrl_reg() write to a lock protect ctrl register
- * @sel: selects which registers we wish to write to
- * @val: the value to be written
- *
- * Certain control registers are protected against overwriting by the lock
- * register and use a shared write address. This function allows writing of
- * these registers.
- * Lock must be held.
- **/
-static int sca3000_write_ctrl_reg(struct sca3000_state *st,
-				  u8 sel,
-				  uint8_t val)
-{
-	int ret;
-
-	ret = sca3000_reg_lock_on(st);
-	if (ret < 0)
-		goto error_ret;
-	if (ret) {
-		ret = __sca3000_unlock_reg_lock(st);
-		if (ret)
-			goto error_ret;
-	}
-
-	/* Set the control select register */
-	ret = sca3000_write_reg(st, SCA3000_REG_ADDR_CTRL_SEL, sel);
-	if (ret)
-		goto error_ret;
-
-	/* Write the actual value into the register */
-	ret = sca3000_write_reg(st, SCA3000_REG_ADDR_CTRL_DATA, val);
-
-error_ret:
-	return ret;
-}
-
-/**
- * sca3000_read_ctrl_reg() read from lock protected control register.
- *
- * Lock must be held.
- **/
-static int sca3000_read_ctrl_reg(struct sca3000_state *st,
-				 u8 ctrl_reg)
-{
-	int ret;
-
-	ret = sca3000_reg_lock_on(st);
-	if (ret < 0)
-		goto error_ret;
-	if (ret) {
-		ret = __sca3000_unlock_reg_lock(st);
-		if (ret)
-			goto error_ret;
-	}
-	/* Set the control select register */
-	ret = sca3000_write_reg(st, SCA3000_REG_ADDR_CTRL_SEL, ctrl_reg);
-	if (ret)
-		goto error_ret;
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_CTRL_DATA, 1);
-	if (ret)
-		goto error_ret;
-	return st->rx[0];
-error_ret:
-	return ret;
-}
-
-/**
- * sca3000_show_rev() - sysfs interface to read the chip revision number
- **/
-static ssize_t sca3000_show_rev(struct device *dev,
-				struct device_attribute *attr,
-				char *buf)
-{
-	int len = 0, ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct sca3000_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->lock);
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_REVID, 1);
-	if (ret < 0)
-		goto error_ret;
-	len += sprintf(buf + len,
-		       "major=%d, minor=%d\n",
-		       st->rx[0] & SCA3000_REVID_MAJOR_MASK,
-		       st->rx[0] & SCA3000_REVID_MINOR_MASK);
-error_ret:
-	mutex_unlock(&st->lock);
-
-	return ret ? ret : len;
-}
-
-/**
- * sca3000_show_available_measurement_modes() display available modes
- *
- * This is all read from chip specific data in the driver. Not all
- * of the sca3000 series support modes other than normal.
- **/
-static ssize_t
-sca3000_show_available_measurement_modes(struct device *dev,
-					 struct device_attribute *attr,
-					 char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int len = 0;
-
-	len += sprintf(buf + len, "0 - normal mode");
-	switch (st->info->option_mode_1) {
-	case SCA3000_OP_MODE_NARROW:
-		len += sprintf(buf + len, ", 1 - narrow mode");
-		break;
-	case SCA3000_OP_MODE_BYPASS:
-		len += sprintf(buf + len, ", 1 - bypass mode");
-		break;
-	}
-	switch (st->info->option_mode_2) {
-	case SCA3000_OP_MODE_WIDE:
-		len += sprintf(buf + len, ", 2 - wide mode");
-		break;
-	}
-	/* always supported */
-	len += sprintf(buf + len, " 3 - motion detection\n");
-
-	return len;
-}
-
-/**
- * sca3000_show_measurement_mode() sysfs read of current mode
- **/
-static ssize_t
-sca3000_show_measurement_mode(struct device *dev,
-			      struct device_attribute *attr,
-			      char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int len = 0, ret;
-
-	mutex_lock(&st->lock);
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_MODE, 1);
-	if (ret)
-		goto error_ret;
-	/* mask bottom 2 bits - only ones that are relevant */
-	st->rx[0] &= 0x03;
-	switch (st->rx[0]) {
-	case SCA3000_MEAS_MODE_NORMAL:
-		len += sprintf(buf + len, "0 - normal mode\n");
-		break;
-	case SCA3000_MEAS_MODE_MOT_DET:
-		len += sprintf(buf + len, "3 - motion detection\n");
-		break;
-	case SCA3000_MEAS_MODE_OP_1:
-		switch (st->info->option_mode_1) {
-		case SCA3000_OP_MODE_NARROW:
-			len += sprintf(buf + len, "1 - narrow mode\n");
-			break;
-		case SCA3000_OP_MODE_BYPASS:
-			len += sprintf(buf + len, "1 - bypass mode\n");
-			break;
-		}
-		break;
-	case SCA3000_MEAS_MODE_OP_2:
-		switch (st->info->option_mode_2) {
-		case SCA3000_OP_MODE_WIDE:
-			len += sprintf(buf + len, "2 - wide mode\n");
-			break;
-		}
-		break;
-	}
-
-error_ret:
-	mutex_unlock(&st->lock);
-
-	return ret ? ret : len;
-}
-
-/**
- * sca3000_store_measurement_mode() set the current mode
- **/
-static ssize_t
-sca3000_store_measurement_mode(struct device *dev,
-			       struct device_attribute *attr,
-			       const char *buf,
-			       size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int ret;
-	u8 mask = 0x03;
-	u8 val;
-
-	mutex_lock(&st->lock);
-	ret = kstrtou8(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	if (val > 3) {
-		ret = -EINVAL;
-		goto error_ret;
-	}
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_MODE, 1);
-	if (ret)
-		goto error_ret;
-	st->rx[0] &= ~mask;
-	st->rx[0] |= (val & mask);
-	ret = sca3000_write_reg(st, SCA3000_REG_ADDR_MODE, st->rx[0]);
-	if (ret)
-		goto error_ret;
-	mutex_unlock(&st->lock);
-
-	return len;
-
-error_ret:
-	mutex_unlock(&st->lock);
-
-	return ret;
-}
-
-/*
- * Not even vaguely standard attributes so defined here rather than
- * in the relevant IIO core headers
- */
-static IIO_DEVICE_ATTR(measurement_mode_available, S_IRUGO,
-		       sca3000_show_available_measurement_modes,
-		       NULL, 0);
-
-static IIO_DEVICE_ATTR(measurement_mode, S_IRUGO | S_IWUSR,
-		       sca3000_show_measurement_mode,
-		       sca3000_store_measurement_mode,
-		       0);
-
-/* More standard attributes */
-
-static IIO_DEVICE_ATTR(revision, S_IRUGO, sca3000_show_rev, NULL, 0);
-
-static const struct iio_event_spec sca3000_event = {
-	.type = IIO_EV_TYPE_MAG,
-	.dir = IIO_EV_DIR_RISING,
-	.mask_separate = BIT(IIO_EV_INFO_VALUE) | BIT(IIO_EV_INFO_ENABLE),
-};
-
-#define SCA3000_CHAN(index, mod)				\
-	{							\
-		.type = IIO_ACCEL,				\
-		.modified = 1,					\
-		.channel2 = mod,				\
-		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
-		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),\
-		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),\
-		.address = index,				\
-		.scan_index = index,				\
-		.scan_type = {					\
-			.sign = 's',				\
-			.realbits = 11,				\
-			.storagebits = 16,			\
-			.shift = 5,				\
-		},						\
-		.event_spec = &sca3000_event,			\
-		.num_event_specs = 1,				\
-	}
-
-static const struct iio_chan_spec sca3000_channels[] = {
-	SCA3000_CHAN(0, IIO_MOD_X),
-	SCA3000_CHAN(1, IIO_MOD_Y),
-	SCA3000_CHAN(2, IIO_MOD_Z),
-};
-
-static const struct iio_chan_spec sca3000_channels_with_temp[] = {
-	SCA3000_CHAN(0, IIO_MOD_X),
-	SCA3000_CHAN(1, IIO_MOD_Y),
-	SCA3000_CHAN(2, IIO_MOD_Z),
-	{
-		.type = IIO_TEMP,
-		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
-		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) |
-			BIT(IIO_CHAN_INFO_OFFSET),
-		/* No buffer support */
-		.scan_index = -1,
-	},
-};
-
-static u8 sca3000_addresses[3][3] = {
-	[0] = {SCA3000_REG_ADDR_X_MSB, SCA3000_REG_CTRL_SEL_MD_X_TH,
-	       SCA3000_MD_CTRL_OR_X},
-	[1] = {SCA3000_REG_ADDR_Y_MSB, SCA3000_REG_CTRL_SEL_MD_Y_TH,
-	       SCA3000_MD_CTRL_OR_Y},
-	[2] = {SCA3000_REG_ADDR_Z_MSB, SCA3000_REG_CTRL_SEL_MD_Z_TH,
-	       SCA3000_MD_CTRL_OR_Z},
-};
-
-/**
- * __sca3000_get_base_freq() obtain mode specific base frequency
- *
- * lock must be held
- **/
-static inline int __sca3000_get_base_freq(struct sca3000_state *st,
-					  const struct sca3000_chip_info *info,
-					  int *base_freq)
-{
-	int ret;
-
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_MODE, 1);
-	if (ret)
-		goto error_ret;
-	switch (0x03 & st->rx[0]) {
-	case SCA3000_MEAS_MODE_NORMAL:
-		*base_freq = info->measurement_mode_freq;
-		break;
-	case SCA3000_MEAS_MODE_OP_1:
-		*base_freq = info->option_mode_1_freq;
-		break;
-	case SCA3000_MEAS_MODE_OP_2:
-		*base_freq = info->option_mode_2_freq;
-		break;
-	default:
-		ret = -EINVAL;
-	}
-error_ret:
-	return ret;
-}
-
-/**
- * read_raw handler for IIO_CHAN_INFO_SAMP_FREQ
- *
- * lock must be held
- **/
-static int read_raw_samp_freq(struct sca3000_state *st, int *val)
-{
-	int ret;
-
-	ret = __sca3000_get_base_freq(st, st->info, val);
-	if (ret)
-		return ret;
-
-	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL);
-	if (ret < 0)
-		return ret;
-
-	if (*val > 0) {
-		ret &= SCA3000_OUT_CTRL_BUF_DIV_MASK;
-		switch (ret) {
-		case SCA3000_OUT_CTRL_BUF_DIV_2:
-			*val /= 2;
-			break;
-		case SCA3000_OUT_CTRL_BUF_DIV_4:
-			*val /= 4;
-			break;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * write_raw handler for IIO_CHAN_INFO_SAMP_FREQ
- *
- * lock must be held
- **/
-static int write_raw_samp_freq(struct sca3000_state *st, int val)
-{
-	int ret, base_freq, ctrlval;
-
-	ret = __sca3000_get_base_freq(st, st->info, &base_freq);
-	if (ret)
-		return ret;
-
-	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL);
-	if (ret < 0)
-		return ret;
-
-	ctrlval = ret & ~SCA3000_OUT_CTRL_BUF_DIV_MASK;
-
-	if (val == base_freq / 2)
-		ctrlval |= SCA3000_OUT_CTRL_BUF_DIV_2;
-	if (val == base_freq / 4)
-		ctrlval |= SCA3000_OUT_CTRL_BUF_DIV_4;
-	else if (val != base_freq)
-		return -EINVAL;
-
-	return sca3000_write_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL,
-				     ctrlval);
-}
-
-static int sca3000_read_raw(struct iio_dev *indio_dev,
-			    struct iio_chan_spec const *chan,
-			    int *val,
-			    int *val2,
-			    long mask)
-{
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int ret;
-	u8 address;
-
-	switch (mask) {
-	case IIO_CHAN_INFO_RAW:
-		mutex_lock(&st->lock);
-		if (chan->type == IIO_ACCEL) {
-			if (st->mo_det_use_count) {
-				mutex_unlock(&st->lock);
-				return -EBUSY;
-			}
-			address = sca3000_addresses[chan->address][0];
-			ret = sca3000_read_data_short(st, address, 2);
-			if (ret < 0) {
-				mutex_unlock(&st->lock);
-				return ret;
-			}
-			*val = (be16_to_cpup((__be16 *)st->rx) >> 3) & 0x1FFF;
-			*val = ((*val) << (sizeof(*val) * 8 - 13)) >>
-				(sizeof(*val) * 8 - 13);
-		} else {
-			/* get the temperature when available */
-			ret = sca3000_read_data_short(st,
-						      SCA3000_REG_ADDR_TEMP_MSB,
-						      2);
-			if (ret < 0) {
-				mutex_unlock(&st->lock);
-				return ret;
-			}
-			*val = ((st->rx[0] & 0x3F) << 3) |
-			       ((st->rx[1] & 0xE0) >> 5);
-		}
-		mutex_unlock(&st->lock);
-		return IIO_VAL_INT;
-	case IIO_CHAN_INFO_SCALE:
-		*val = 0;
-		if (chan->type == IIO_ACCEL)
-			*val2 = st->info->scale;
-		else /* temperature */
-			*val2 = 555556;
-		return IIO_VAL_INT_PLUS_MICRO;
-	case IIO_CHAN_INFO_OFFSET:
-		*val = -214;
-		*val2 = 600000;
-		return IIO_VAL_INT_PLUS_MICRO;
-	case IIO_CHAN_INFO_SAMP_FREQ:
-		mutex_lock(&st->lock);
-		ret = read_raw_samp_freq(st, val);
-		mutex_unlock(&st->lock);
-		return ret ? ret : IIO_VAL_INT;
-	default:
-		return -EINVAL;
-	}
-}
-
-static int sca3000_write_raw(struct iio_dev *indio_dev,
-			     struct iio_chan_spec const *chan,
-			     int val, int val2, long mask)
-{
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int ret;
-
-	switch (mask) {
-	case IIO_CHAN_INFO_SAMP_FREQ:
-		if (val2)
-			return -EINVAL;
-		mutex_lock(&st->lock);
-		ret = write_raw_samp_freq(st, val);
-		mutex_unlock(&st->lock);
-		return ret;
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-/**
- * sca3000_read_av_freq() sysfs function to get available frequencies
- *
- * The later modes are only relevant to the ring buffer - and depend on current
- * mode. Note that data sheet gives rather wide tolerances for these so integer
- * division will give good enough answer and not all chips have them specified
- * at all.
- **/
-static ssize_t sca3000_read_av_freq(struct device *dev,
-				    struct device_attribute *attr,
-				    char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int len = 0, ret, val;
-
-	mutex_lock(&st->lock);
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_MODE, 1);
-	val = st->rx[0];
-	mutex_unlock(&st->lock);
-	if (ret)
-		goto error_ret;
-
-	switch (val & 0x03) {
-	case SCA3000_MEAS_MODE_NORMAL:
-		len += sprintf(buf + len, "%d %d %d\n",
-			       st->info->measurement_mode_freq,
-			       st->info->measurement_mode_freq / 2,
-			       st->info->measurement_mode_freq / 4);
-		break;
-	case SCA3000_MEAS_MODE_OP_1:
-		len += sprintf(buf + len, "%d %d %d\n",
-			       st->info->option_mode_1_freq,
-			       st->info->option_mode_1_freq / 2,
-			       st->info->option_mode_1_freq / 4);
-		break;
-	case SCA3000_MEAS_MODE_OP_2:
-		len += sprintf(buf + len, "%d %d %d\n",
-			       st->info->option_mode_2_freq,
-			       st->info->option_mode_2_freq / 2,
-			       st->info->option_mode_2_freq / 4);
-		break;
-	}
-	return len;
-error_ret:
-	return ret;
-}
-
-/*
- * Should only really be registered if ring buffer support is compiled in.
- * Does no harm however and doing it right would add a fair bit of complexity
- */
-static IIO_DEV_ATTR_SAMP_FREQ_AVAIL(sca3000_read_av_freq);
-
-/**
- * sca3000_read_thresh() - query of a threshold
- **/
-static int sca3000_read_thresh(struct iio_dev *indio_dev,
-			       const struct iio_chan_spec *chan,
-			       enum iio_event_type type,
-			       enum iio_event_direction dir,
-			       enum iio_event_info info,
-			       int *val, int *val2)
-{
-	int ret, i;
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int num = chan->channel2;
-
-	mutex_lock(&st->lock);
-	ret = sca3000_read_ctrl_reg(st, sca3000_addresses[num][1]);
-	mutex_unlock(&st->lock);
-	if (ret < 0)
-		return ret;
-	*val = 0;
-	if (num == 1)
-		for_each_set_bit(i, (unsigned long *)&ret,
-				 ARRAY_SIZE(st->info->mot_det_mult_y))
-			*val += st->info->mot_det_mult_y[i];
-	else
-		for_each_set_bit(i, (unsigned long *)&ret,
-				 ARRAY_SIZE(st->info->mot_det_mult_xz))
-			*val += st->info->mot_det_mult_xz[i];
-
-	return IIO_VAL_INT;
-}
-
-/**
- * sca3000_write_thresh() control of threshold
- **/
-static int sca3000_write_thresh(struct iio_dev *indio_dev,
-				const struct iio_chan_spec *chan,
-				enum iio_event_type type,
-				enum iio_event_direction dir,
-				enum iio_event_info info,
-				int val, int val2)
-{
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int num = chan->channel2;
-	int ret;
-	int i;
-	u8 nonlinear = 0;
-
-	if (num == 1) {
-		i = ARRAY_SIZE(st->info->mot_det_mult_y);
-		while (i > 0)
-			if (val >= st->info->mot_det_mult_y[--i]) {
-				nonlinear |= (1 << i);
-				val -= st->info->mot_det_mult_y[i];
-			}
-	} else {
-		i = ARRAY_SIZE(st->info->mot_det_mult_xz);
-		while (i > 0)
-			if (val >= st->info->mot_det_mult_xz[--i]) {
-				nonlinear |= (1 << i);
-				val -= st->info->mot_det_mult_xz[i];
-			}
-	}
-
-	mutex_lock(&st->lock);
-	ret = sca3000_write_ctrl_reg(st, sca3000_addresses[num][1], nonlinear);
-	mutex_unlock(&st->lock);
-
-	return ret;
-}
-
-static struct attribute *sca3000_attributes[] = {
-	&iio_dev_attr_revision.dev_attr.attr,
-	&iio_dev_attr_measurement_mode_available.dev_attr.attr,
-	&iio_dev_attr_measurement_mode.dev_attr.attr,
-	&iio_dev_attr_sampling_frequency_available.dev_attr.attr,
-	NULL,
-};
-
-static const struct attribute_group sca3000_attribute_group = {
-	.attrs = sca3000_attributes,
-};
-
-/**
- * sca3000_event_handler() - handling ring and non ring events
- *
- * Ring related interrupt handler. Depending on event, push to
- * the ring buffer event chrdev or the event one.
- *
- * This function is complicated by the fact that the devices can signify ring
- * and non ring events via the same interrupt line and they can only
- * be distinguished via a read of the relevant status register.
- **/
-static irqreturn_t sca3000_event_handler(int irq, void *private)
-{
-	struct iio_dev *indio_dev = private;
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int ret, val;
-	s64 last_timestamp = iio_get_time_ns(indio_dev);
-
-	/*
-	 * Could lead if badly timed to an extra read of status reg,
-	 * but ensures no interrupt is missed.
-	 */
-	mutex_lock(&st->lock);
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_INT_STATUS, 1);
-	val = st->rx[0];
-	mutex_unlock(&st->lock);
-	if (ret)
-		goto done;
-
-	sca3000_ring_int_process(val, indio_dev->buffer);
-
-	if (val & SCA3000_INT_STATUS_FREE_FALL)
-		iio_push_event(indio_dev,
-			       IIO_MOD_EVENT_CODE(IIO_ACCEL,
-						  0,
-						  IIO_MOD_X_AND_Y_AND_Z,
-						  IIO_EV_TYPE_MAG,
-						  IIO_EV_DIR_FALLING),
-			       last_timestamp);
-
-	if (val & SCA3000_INT_STATUS_Y_TRIGGER)
-		iio_push_event(indio_dev,
-			       IIO_MOD_EVENT_CODE(IIO_ACCEL,
-						  0,
-						  IIO_MOD_Y,
-						  IIO_EV_TYPE_MAG,
-						  IIO_EV_DIR_RISING),
-			       last_timestamp);
-
-	if (val & SCA3000_INT_STATUS_X_TRIGGER)
-		iio_push_event(indio_dev,
-			       IIO_MOD_EVENT_CODE(IIO_ACCEL,
-						  0,
-						  IIO_MOD_X,
-						  IIO_EV_TYPE_MAG,
-						  IIO_EV_DIR_RISING),
-			       last_timestamp);
-
-	if (val & SCA3000_INT_STATUS_Z_TRIGGER)
-		iio_push_event(indio_dev,
-			       IIO_MOD_EVENT_CODE(IIO_ACCEL,
-						  0,
-						  IIO_MOD_Z,
-						  IIO_EV_TYPE_MAG,
-						  IIO_EV_DIR_RISING),
-			       last_timestamp);
-
-done:
-	return IRQ_HANDLED;
-}
-
-/**
- * sca3000_read_event_config() what events are enabled
- **/
-static int sca3000_read_event_config(struct iio_dev *indio_dev,
-				     const struct iio_chan_spec *chan,
-				     enum iio_event_type type,
-				     enum iio_event_direction dir)
-{
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int ret;
-	u8 protect_mask = 0x03;
-	int num = chan->channel2;
-
-	/* read current value of mode register */
-	mutex_lock(&st->lock);
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_MODE, 1);
-	if (ret)
-		goto error_ret;
-
-	if ((st->rx[0] & protect_mask) != SCA3000_MEAS_MODE_MOT_DET) {
-		ret = 0;
-	} else {
-		ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_MD_CTRL);
-		if (ret < 0)
-			goto error_ret;
-		/* only supporting logical or's for now */
-		ret = !!(ret & sca3000_addresses[num][2]);
-	}
-error_ret:
-	mutex_unlock(&st->lock);
-
-	return ret;
-}
-
-/**
- * sca3000_query_free_fall_mode() is free fall mode enabled
- **/
-static ssize_t sca3000_query_free_fall_mode(struct device *dev,
-					    struct device_attribute *attr,
-					    char *buf)
-{
-	int ret;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int val;
-
-	mutex_lock(&st->lock);
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_MODE, 1);
-	val = st->rx[0];
-	mutex_unlock(&st->lock);
-	if (ret < 0)
-		return ret;
-	return sprintf(buf, "%d\n", !!(val & SCA3000_FREE_FALL_DETECT));
-}
-
-/**
- * sca3000_set_free_fall_mode() simple on off control for free fall int
- *
- * In these chips the free fall detector should send an interrupt if
- * the device falls more than 25cm.  This has not been tested due
- * to fragile wiring.
- **/
-static ssize_t sca3000_set_free_fall_mode(struct device *dev,
-					  struct device_attribute *attr,
-					  const char *buf,
-					  size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct sca3000_state *st = iio_priv(indio_dev);
-	u8 val;
-	int ret;
-	u8 protect_mask = SCA3000_FREE_FALL_DETECT;
-
-	mutex_lock(&st->lock);
-	ret = kstrtou8(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-
-	/* read current value of mode register */
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_MODE, 1);
-	if (ret)
-		goto error_ret;
-
-	/* if off and should be on */
-	if (val && !(st->rx[0] & protect_mask))
-		ret = sca3000_write_reg(st, SCA3000_REG_ADDR_MODE,
-					(st->rx[0] | SCA3000_FREE_FALL_DETECT));
-	/* if on and should be off */
-	else if (!val && (st->rx[0] & protect_mask))
-		ret = sca3000_write_reg(st, SCA3000_REG_ADDR_MODE,
-					(st->rx[0] & ~protect_mask));
-error_ret:
-	mutex_unlock(&st->lock);
-
-	return ret ? ret : len;
-}
-
-/**
- * sca3000_write_event_config() simple on off control for motion detector
- *
- * This is a per axis control, but enabling any will result in the
- * motion detector unit being enabled.
- * N.B. enabling motion detector stops normal data acquisition.
- * There is a complexity in knowing which mode to return to when
- * this mode is disabled.  Currently normal mode is assumed.
- **/
-static int sca3000_write_event_config(struct iio_dev *indio_dev,
-				      const struct iio_chan_spec *chan,
-				      enum iio_event_type type,
-				      enum iio_event_direction dir,
-				      int state)
-{
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int ret, ctrlval;
-	u8 protect_mask = 0x03;
-	int num = chan->channel2;
-
-	mutex_lock(&st->lock);
-	/*
-	 * First read the motion detector config to find out if
-	 * this axis is on
-	 */
-	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_MD_CTRL);
-	if (ret < 0)
-		goto exit_point;
-	ctrlval = ret;
-	/* if off and should be on */
-	if (state && !(ctrlval & sca3000_addresses[num][2])) {
-		ret = sca3000_write_ctrl_reg(st,
-					     SCA3000_REG_CTRL_SEL_MD_CTRL,
-					     ctrlval |
-					     sca3000_addresses[num][2]);
-		if (ret)
-			goto exit_point;
-		st->mo_det_use_count++;
-	} else if (!state && (ctrlval & sca3000_addresses[num][2])) {
-		ret = sca3000_write_ctrl_reg(st,
-					     SCA3000_REG_CTRL_SEL_MD_CTRL,
-					     ctrlval &
-					     ~(sca3000_addresses[num][2]));
-		if (ret)
-			goto exit_point;
-		st->mo_det_use_count--;
-	}
-
-	/* read current value of mode register */
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_MODE, 1);
-	if (ret)
-		goto exit_point;
-	/* if off and should be on */
-	if ((st->mo_det_use_count) &&
-	    ((st->rx[0] & protect_mask) != SCA3000_MEAS_MODE_MOT_DET))
-		ret = sca3000_write_reg(st, SCA3000_REG_ADDR_MODE,
-					(st->rx[0] & ~protect_mask)
-					| SCA3000_MEAS_MODE_MOT_DET);
-	/* if on and should be off */
-	else if (!(st->mo_det_use_count) &&
-		 ((st->rx[0] & protect_mask) == SCA3000_MEAS_MODE_MOT_DET))
-		ret = sca3000_write_reg(st, SCA3000_REG_ADDR_MODE,
-					(st->rx[0] & ~protect_mask));
-exit_point:
-	mutex_unlock(&st->lock);
-
-	return ret;
-}
-
-/* Free fall detector related event attribute */
-static IIO_DEVICE_ATTR_NAMED(accel_xayaz_mag_falling_en,
-			     in_accel_x & y & z_mag_falling_en,
-			     S_IRUGO | S_IWUSR,
-			     sca3000_query_free_fall_mode,
-			     sca3000_set_free_fall_mode,
-			     0);
-
-static IIO_CONST_ATTR_NAMED(accel_xayaz_mag_falling_period,
-			    in_accel_x & y & z_mag_falling_period,
-			    "0.226");
-
-static struct attribute *sca3000_event_attributes[] = {
-	&iio_dev_attr_accel_xayaz_mag_falling_en.dev_attr.attr,
-	&iio_const_attr_accel_xayaz_mag_falling_period.dev_attr.attr,
-	NULL,
-};
-
-static struct attribute_group sca3000_event_attribute_group = {
-	.attrs = sca3000_event_attributes,
-	.name = "events",
-};
-
-/**
- * sca3000_clean_setup() get the device into a predictable state
- *
- * Devices use flash memory to store many of the register values
- * and hence can come up in somewhat unpredictable states.
- * Hence reset everything on driver load.
- **/
-static int sca3000_clean_setup(struct sca3000_state *st)
-{
-	int ret;
-
-	mutex_lock(&st->lock);
-	/* Ensure all interrupts have been acknowledged */
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_INT_STATUS, 1);
-	if (ret)
-		goto error_ret;
-
-	/* Turn off all motion detection channels */
-	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_MD_CTRL);
-	if (ret < 0)
-		goto error_ret;
-	ret = sca3000_write_ctrl_reg(st, SCA3000_REG_CTRL_SEL_MD_CTRL,
-				     ret & SCA3000_MD_CTRL_PROT_MASK);
-	if (ret)
-		goto error_ret;
-
-	/* Disable ring buffer */
-	ret = sca3000_read_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL);
-	if (ret < 0)
-		goto error_ret;
-	ret = sca3000_write_ctrl_reg(st, SCA3000_REG_CTRL_SEL_OUT_CTRL,
-				     (ret & SCA3000_OUT_CTRL_PROT_MASK)
-				     | SCA3000_OUT_CTRL_BUF_X_EN
-				     | SCA3000_OUT_CTRL_BUF_Y_EN
-				     | SCA3000_OUT_CTRL_BUF_Z_EN
-				     | SCA3000_OUT_CTRL_BUF_DIV_4);
-	if (ret)
-		goto error_ret;
-	/* Enable interrupts, relevant to mode and set up as active low */
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_INT_MASK, 1);
-	if (ret)
-		goto error_ret;
-	ret = sca3000_write_reg(st,
-				SCA3000_REG_ADDR_INT_MASK,
-				(ret & SCA3000_INT_MASK_PROT_MASK)
-				| SCA3000_INT_MASK_ACTIVE_LOW);
-	if (ret)
-		goto error_ret;
-	/*
-	 * Select normal measurement mode, free fall off, ring off
-	 * Ring in 12 bit mode - it is fine to overwrite reserved bits 3,5
-	 * as that occurs in one of the example on the datasheet
-	 */
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_MODE, 1);
-	if (ret)
-		goto error_ret;
-	ret = sca3000_write_reg(st, SCA3000_REG_ADDR_MODE,
-				(st->rx[0] & SCA3000_MODE_PROT_MASK));
-	st->bpse = 11;
-
-error_ret:
-	mutex_unlock(&st->lock);
-	return ret;
-}
-
-static const struct iio_info sca3000_info = {
-	.attrs = &sca3000_attribute_group,
-	.read_raw = &sca3000_read_raw,
-	.write_raw = &sca3000_write_raw,
-	.event_attrs = &sca3000_event_attribute_group,
-	.read_event_value = &sca3000_read_thresh,
-	.write_event_value = &sca3000_write_thresh,
-	.read_event_config = &sca3000_read_event_config,
-	.write_event_config = &sca3000_write_event_config,
-	.driver_module = THIS_MODULE,
-};
-
-static int sca3000_probe(struct spi_device *spi)
-{
-	int ret;
-	struct sca3000_state *st;
-	struct iio_dev *indio_dev;
-
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
-	if (!indio_dev)
-		return -ENOMEM;
-
-	st = iio_priv(indio_dev);
-	spi_set_drvdata(spi, indio_dev);
-	st->us = spi;
-	mutex_init(&st->lock);
-	st->info = &sca3000_spi_chip_info_tbl[spi_get_device_id(spi)
-					      ->driver_data];
-
-	indio_dev->dev.parent = &spi->dev;
-	indio_dev->name = spi_get_device_id(spi)->name;
-	indio_dev->info = &sca3000_info;
-	if (st->info->temp_output) {
-		indio_dev->channels = sca3000_channels_with_temp;
-		indio_dev->num_channels =
-			ARRAY_SIZE(sca3000_channels_with_temp);
-	} else {
-		indio_dev->channels = sca3000_channels;
-		indio_dev->num_channels = ARRAY_SIZE(sca3000_channels);
-	}
-	indio_dev->modes = INDIO_DIRECT_MODE;
-
-	sca3000_configure_ring(indio_dev);
-	ret = iio_device_register(indio_dev);
-	if (ret < 0)
-		return ret;
-
-	if (spi->irq) {
-		ret = request_threaded_irq(spi->irq,
-					   NULL,
-					   &sca3000_event_handler,
-					   IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
-					   "sca3000",
-					   indio_dev);
-		if (ret)
-			goto error_unregister_dev;
-	}
-	sca3000_register_ring_funcs(indio_dev);
-	ret = sca3000_clean_setup(st);
-	if (ret)
-		goto error_free_irq;
-	return 0;
-
-error_free_irq:
-	if (spi->irq)
-		free_irq(spi->irq, indio_dev);
-error_unregister_dev:
-	iio_device_unregister(indio_dev);
-	return ret;
-}
-
-static int sca3000_stop_all_interrupts(struct sca3000_state *st)
-{
-	int ret;
-
-	mutex_lock(&st->lock);
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_INT_MASK, 1);
-	if (ret)
-		goto error_ret;
-	ret = sca3000_write_reg(st, SCA3000_REG_ADDR_INT_MASK,
-				(st->rx[0] &
-				 ~(SCA3000_INT_MASK_RING_THREE_QUARTER |
-				   SCA3000_INT_MASK_RING_HALF |
-				   SCA3000_INT_MASK_ALL_INTS)));
-error_ret:
-	mutex_unlock(&st->lock);
-	return ret;
-}
-
-static int sca3000_remove(struct spi_device *spi)
-{
-	struct iio_dev *indio_dev = spi_get_drvdata(spi);
-	struct sca3000_state *st = iio_priv(indio_dev);
-
-	/* Must ensure no interrupts can be generated after this! */
-	sca3000_stop_all_interrupts(st);
-	if (spi->irq)
-		free_irq(spi->irq, indio_dev);
-	iio_device_unregister(indio_dev);
-	sca3000_unconfigure_ring(indio_dev);
-
-	return 0;
-}
-
-static const struct spi_device_id sca3000_id[] = {
-	{"sca3000_d01", d01},
-	{"sca3000_e02", e02},
-	{"sca3000_e04", e04},
-	{"sca3000_e05", e05},
-	{}
-};
-MODULE_DEVICE_TABLE(spi, sca3000_id);
-
-static struct spi_driver sca3000_driver = {
-	.driver = {
-		.name = "sca3000",
-	},
-	.probe = sca3000_probe,
-	.remove = sca3000_remove,
-	.id_table = sca3000_id,
-};
-module_spi_driver(sca3000_driver);
-
-MODULE_AUTHOR("Jonathan Cameron <jic23@kernel.org>");
-MODULE_DESCRIPTION("VTI SCA3000 Series Accelerometers SPI driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/iio/accel/sca3000_ring.c b/drivers/staging/iio/accel/sca3000_ring.c
deleted file mode 100644
index d1cb9b9cf22b..000000000000
--- a/drivers/staging/iio/accel/sca3000_ring.c
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * sca3000_ring.c -- support VTI sca3000 series accelerometers via SPI
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * Copyright (c) 2009 Jonathan Cameron <jic23@kernel.org>
- *
- */
-
-#include <linux/interrupt.h>
-#include <linux/fs.h>
-#include <linux/slab.h>
-#include <linux/kernel.h>
-#include <linux/spi/spi.h>
-#include <linux/sysfs.h>
-#include <linux/sched.h>
-#include <linux/poll.h>
-
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-#include <linux/iio/buffer.h>
-#include "../ring_hw.h"
-#include "sca3000.h"
-
-/* RFC / future work
- *
- * The internal ring buffer doesn't actually change what it holds depending
- * on which signals are enabled etc, merely whether you can read them.
- * As such the scan mode selection is somewhat different than for a software
- * ring buffer and changing it actually covers any data already in the buffer.
- * Currently scan elements aren't configured so it doesn't matter.
- */
-
-static int sca3000_read_data(struct sca3000_state *st,
-			     u8 reg_address_high,
-			     u8 **rx_p,
-			     int len)
-{
-	int ret;
-	struct spi_transfer xfer[2] = {
-		{
-			.len = 1,
-			.tx_buf = st->tx,
-		}, {
-			.len = len,
-		}
-	};
-	*rx_p = kmalloc(len, GFP_KERNEL);
-	if (!*rx_p) {
-		ret = -ENOMEM;
-		goto error_ret;
-	}
-	xfer[1].rx_buf = *rx_p;
-	st->tx[0] = SCA3000_READ_REG(reg_address_high);
-	ret = spi_sync_transfer(st->us, xfer, ARRAY_SIZE(xfer));
-	if (ret) {
-		dev_err(get_device(&st->us->dev), "problem reading register");
-		goto error_free_rx;
-	}
-
-	return 0;
-error_free_rx:
-	kfree(*rx_p);
-error_ret:
-	return ret;
-}
-
-/**
- * sca3000_read_first_n_hw_rb() - main ring access, pulls data from ring
- * @r:			the ring
- * @count:		number of samples to try and pull
- * @data:		output the actual samples pulled from the hw ring
- *
- * Currently does not provide timestamps.  As the hardware doesn't add them they
- * can only be inferred approximately from ring buffer events such as 50% full
- * and knowledge of when buffer was last emptied.  This is left to userspace.
- **/
-static int sca3000_read_first_n_hw_rb(struct iio_buffer *r,
-				      size_t count, char __user *buf)
-{
-	struct iio_hw_buffer *hw_ring = iio_to_hw_buf(r);
-	struct iio_dev *indio_dev = hw_ring->private;
-	struct sca3000_state *st = iio_priv(indio_dev);
-	u8 *rx;
-	int ret, i, num_available, num_read = 0;
-	int bytes_per_sample = 1;
-
-	if (st->bpse == 11)
-		bytes_per_sample = 2;
-
-	mutex_lock(&st->lock);
-	if (count % bytes_per_sample) {
-		ret = -EINVAL;
-		goto error_ret;
-	}
-
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_BUF_COUNT, 1);
-	if (ret)
-		goto error_ret;
-	num_available = st->rx[0];
-	/*
-	 * num_available is the total number of samples available
-	 * i.e. number of time points * number of channels.
-	 */
-	if (count > num_available * bytes_per_sample)
-		num_read = num_available * bytes_per_sample;
-	else
-		num_read = count;
-
-	ret = sca3000_read_data(st,
-				SCA3000_REG_ADDR_RING_OUT,
-				&rx, num_read);
-	if (ret)
-		goto error_ret;
-
-	for (i = 0; i < num_read / sizeof(u16); i++)
-		*(((u16 *)rx) + i) = be16_to_cpup((__be16 *)rx + i);
-
-	if (copy_to_user(buf, rx, num_read))
-		ret = -EFAULT;
-	kfree(rx);
-	r->stufftoread = 0;
-error_ret:
-	mutex_unlock(&st->lock);
-
-	return ret ? ret : num_read;
-}
-
-static size_t sca3000_ring_buf_data_available(struct iio_buffer *r)
-{
-	return r->stufftoread ? r->watermark : 0;
-}
-
-/**
- * sca3000_query_ring_int() is the hardware ring status interrupt enabled
- **/
-static ssize_t sca3000_query_ring_int(struct device *dev,
-				      struct device_attribute *attr,
-				      char *buf)
-{
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	int ret, val;
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct sca3000_state *st = iio_priv(indio_dev);
-
-	mutex_lock(&st->lock);
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_INT_MASK, 1);
-	val = st->rx[0];
-	mutex_unlock(&st->lock);
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%d\n", !!(val & this_attr->address));
-}
-
-/**
- * sca3000_set_ring_int() set state of ring status interrupt
- **/
-static ssize_t sca3000_set_ring_int(struct device *dev,
-				    struct device_attribute *attr,
-				    const char *buf,
-				    size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct sca3000_state *st = iio_priv(indio_dev);
-	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	u8 val;
-	int ret;
-
-	mutex_lock(&st->lock);
-	ret = kstrtou8(buf, 10, &val);
-	if (ret)
-		goto error_ret;
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_INT_MASK, 1);
-	if (ret)
-		goto error_ret;
-	if (val)
-		ret = sca3000_write_reg(st,
-					SCA3000_REG_ADDR_INT_MASK,
-					st->rx[0] | this_attr->address);
-	else
-		ret = sca3000_write_reg(st,
-					SCA3000_REG_ADDR_INT_MASK,
-					st->rx[0] & ~this_attr->address);
-error_ret:
-	mutex_unlock(&st->lock);
-
-	return ret ? ret : len;
-}
-
-static IIO_DEVICE_ATTR(50_percent, S_IRUGO | S_IWUSR,
-		       sca3000_query_ring_int,
-		       sca3000_set_ring_int,
-		       SCA3000_INT_MASK_RING_HALF);
-
-static IIO_DEVICE_ATTR(75_percent, S_IRUGO | S_IWUSR,
-		       sca3000_query_ring_int,
-		       sca3000_set_ring_int,
-		       SCA3000_INT_MASK_RING_THREE_QUARTER);
-
-static ssize_t sca3000_show_buffer_scale(struct device *dev,
-					 struct device_attribute *attr,
-					 char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct sca3000_state *st = iio_priv(indio_dev);
-
-	return sprintf(buf, "0.%06d\n", 4 * st->info->scale);
-}
-
-static IIO_DEVICE_ATTR(in_accel_scale,
-		       S_IRUGO,
-		       sca3000_show_buffer_scale,
-		       NULL,
-		       0);
-
-/*
- * Ring buffer attributes
- * This device is a bit unusual in that the sampling frequency and bpse
- * only apply to the ring buffer.  At all times full rate and accuracy
- * is available via direct reading from registers.
- */
-static const struct attribute *sca3000_ring_attributes[] = {
-	&iio_dev_attr_50_percent.dev_attr.attr,
-	&iio_dev_attr_75_percent.dev_attr.attr,
-	&iio_dev_attr_in_accel_scale.dev_attr.attr,
-	NULL,
-};
-
-static struct iio_buffer *sca3000_rb_allocate(struct iio_dev *indio_dev)
-{
-	struct iio_buffer *buf;
-	struct iio_hw_buffer *ring;
-
-	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
-	if (!ring)
-		return NULL;
-
-	ring->private = indio_dev;
-	buf = &ring->buf;
-	buf->stufftoread = 0;
-	buf->length = 64;
-	buf->attrs = sca3000_ring_attributes;
-	iio_buffer_init(buf);
-
-	return buf;
-}
-
-static void sca3000_ring_release(struct iio_buffer *r)
-{
-	kfree(iio_to_hw_buf(r));
-}
-
-static const struct iio_buffer_access_funcs sca3000_ring_access_funcs = {
-	.read_first_n = &sca3000_read_first_n_hw_rb,
-	.data_available = sca3000_ring_buf_data_available,
-	.release = sca3000_ring_release,
-
-	.modes = INDIO_BUFFER_HARDWARE,
-};
-
-int sca3000_configure_ring(struct iio_dev *indio_dev)
-{
-	struct iio_buffer *buffer;
-
-	buffer = sca3000_rb_allocate(indio_dev);
-	if (!buffer)
-		return -ENOMEM;
-	indio_dev->modes |= INDIO_BUFFER_HARDWARE;
-
-	indio_dev->buffer->access = &sca3000_ring_access_funcs;
-
-	iio_device_attach_buffer(indio_dev, buffer);
-
-	return 0;
-}
-
-void sca3000_unconfigure_ring(struct iio_dev *indio_dev)
-{
-	iio_buffer_put(indio_dev->buffer);
-}
-
-static inline
-int __sca3000_hw_ring_state_set(struct iio_dev *indio_dev, bool state)
-{
-	struct sca3000_state *st = iio_priv(indio_dev);
-	int ret;
-
-	mutex_lock(&st->lock);
-	ret = sca3000_read_data_short(st, SCA3000_REG_ADDR_MODE, 1);
-	if (ret)
-		goto error_ret;
-	if (state) {
-		dev_info(&indio_dev->dev, "supposedly enabling ring buffer\n");
-		ret = sca3000_write_reg(st,
-					SCA3000_REG_ADDR_MODE,
-					(st->rx[0] | SCA3000_RING_BUF_ENABLE));
-	} else
-		ret = sca3000_write_reg(st,
-					SCA3000_REG_ADDR_MODE,
-					(st->rx[0] & ~SCA3000_RING_BUF_ENABLE));
-error_ret:
-	mutex_unlock(&st->lock);
-
-	return ret;
-}
-
-/**
- * sca3000_hw_ring_preenable() hw ring buffer preenable function
- *
- * Very simple enable function as the chip will allows normal reads
- * during ring buffer operation so as long as it is indeed running
- * before we notify the core, the precise ordering does not matter.
- **/
-static int sca3000_hw_ring_preenable(struct iio_dev *indio_dev)
-{
-	return __sca3000_hw_ring_state_set(indio_dev, 1);
-}
-
-static int sca3000_hw_ring_postdisable(struct iio_dev *indio_dev)
-{
-	return __sca3000_hw_ring_state_set(indio_dev, 0);
-}
-
-static const struct iio_buffer_setup_ops sca3000_ring_setup_ops = {
-	.preenable = &sca3000_hw_ring_preenable,
-	.postdisable = &sca3000_hw_ring_postdisable,
-};
-
-void sca3000_register_ring_funcs(struct iio_dev *indio_dev)
-{
-	indio_dev->setup_ops = &sca3000_ring_setup_ops;
-}
-
-/**
- * sca3000_ring_int_process() ring specific interrupt handling.
- *
- * This is only split from the main interrupt handler so as to
- * reduce the amount of code if the ring buffer is not enabled.
- **/
-void sca3000_ring_int_process(u8 val, struct iio_buffer *ring)
-{
-	if (val & (SCA3000_INT_STATUS_THREE_QUARTERS |
-		   SCA3000_INT_STATUS_HALF)) {
-		ring->stufftoread = true;
-		wake_up_interruptible(&ring->pollq);
-	}
-}
diff --git a/drivers/staging/iio/adc/Makefile b/drivers/staging/iio/adc/Makefile
index 3cdd83ccec8e..ac09485923b6 100644
--- a/drivers/staging/iio/adc/Makefile
+++ b/drivers/staging/iio/adc/Makefile
@@ -2,7 +2,6 @@
 # Makefile for industrial I/O ADC drivers
 #
 
-ad7606-y := ad7606_core.o ad7606_ring.o
 obj-$(CONFIG_AD7606_IFACE_PARALLEL) += ad7606_par.o
 obj-$(CONFIG_AD7606_IFACE_SPI) += ad7606_spi.o
 obj-$(CONFIG_AD7606) += ad7606.o
diff --git a/drivers/staging/iio/adc/ad7192.c b/drivers/staging/iio/adc/ad7192.c
index 1cf6b79801a9..1fb68c01abd5 100644
--- a/drivers/staging/iio/adc/ad7192.c
+++ b/drivers/staging/iio/adc/ad7192.c
@@ -152,7 +152,8 @@
  */
 
 struct ad7192_state {
-	struct regulator		*reg;
+	struct regulator		*avdd;
+	struct regulator		*dvdd;
 	u16				int_vref_mv;
 	u32				mclk;
 	u32				f_order;
@@ -322,57 +323,6 @@ out:
 	return ret;
 }
 
-static ssize_t ad7192_read_frequency(struct device *dev,
-				     struct device_attribute *attr,
-				     char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ad7192_state *st = iio_priv(indio_dev);
-
-	return sprintf(buf, "%d\n", st->mclk /
-			(st->f_order * 1024 * AD7192_MODE_RATE(st->mode)));
-}
-
-static ssize_t ad7192_write_frequency(struct device *dev,
-				      struct device_attribute *attr,
-				      const char *buf,
-				      size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ad7192_state *st = iio_priv(indio_dev);
-	unsigned long lval;
-	int div, ret;
-
-	ret = kstrtoul(buf, 10, &lval);
-	if (ret)
-		return ret;
-	if (lval == 0)
-		return -EINVAL;
-
-	ret = iio_device_claim_direct_mode(indio_dev);
-	if (ret)
-		return ret;
-
-	div = st->mclk / (lval * st->f_order * 1024);
-	if (div < 1 || div > 1023) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	st->mode &= ~AD7192_MODE_RATE(-1);
-	st->mode |= AD7192_MODE_RATE(div);
-	ad_sd_write_reg(&st->sd, AD7192_REG_MODE, 3, st->mode);
-
-out:
-	iio_device_release_direct_mode(indio_dev);
-
-	return ret ? ret : len;
-}
-
-static IIO_DEV_ATTR_SAMP_FREQ(S_IWUSR | S_IRUGO,
-		ad7192_read_frequency,
-		ad7192_write_frequency);
-
 static ssize_t
 ad7192_show_scale_available(struct device *dev,
 			    struct device_attribute *attr, char *buf)
@@ -471,7 +421,6 @@ static IIO_DEVICE_ATTR(ac_excitation_en, S_IRUGO | S_IWUSR,
 		       AD7192_REG_MODE);
 
 static struct attribute *ad7192_attributes[] = {
-	&iio_dev_attr_sampling_frequency.dev_attr.attr,
 	&iio_dev_attr_in_v_m_v_scale_available.dev_attr.attr,
 	&iio_dev_attr_in_voltage_scale_available.dev_attr.attr,
 	&iio_dev_attr_bridge_switch_en.dev_attr.attr,
@@ -484,7 +433,6 @@ static const struct attribute_group ad7192_attribute_group = {
 };
 
 static struct attribute *ad7195_attributes[] = {
-	&iio_dev_attr_sampling_frequency.dev_attr.attr,
 	&iio_dev_attr_in_v_m_v_scale_available.dev_attr.attr,
 	&iio_dev_attr_in_voltage_scale_available.dev_attr.attr,
 	&iio_dev_attr_bridge_switch_en.dev_attr.attr,
@@ -536,6 +484,10 @@ static int ad7192_read_raw(struct iio_dev *indio_dev,
 		if (chan->type == IIO_TEMP)
 			*val -= 273 * ad7192_get_temp_scale(unipolar);
 		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*val = st->mclk /
+			(st->f_order * 1024 * AD7192_MODE_RATE(st->mode));
+		return IIO_VAL_INT;
 	}
 
 	return -EINVAL;
@@ -548,7 +500,7 @@ static int ad7192_write_raw(struct iio_dev *indio_dev,
 			    long mask)
 {
 	struct ad7192_state *st = iio_priv(indio_dev);
-	int ret, i;
+	int ret, i, div;
 	unsigned int tmp;
 
 	ret = iio_device_claim_direct_mode(indio_dev);
@@ -572,6 +524,22 @@ static int ad7192_write_raw(struct iio_dev *indio_dev,
 				break;
 			}
 		break;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (!val) {
+			ret = -EINVAL;
+			break;
+		}
+
+		div = st->mclk / (val * st->f_order * 1024);
+		if (div < 1 || div > 1023) {
+			ret = -EINVAL;
+			break;
+		}
+
+		st->mode &= ~AD7192_MODE_RATE(-1);
+		st->mode |= AD7192_MODE_RATE(div);
+		ad_sd_write_reg(&st->sd, AD7192_REG_MODE, 3, st->mode);
+		break;
 	default:
 		ret = -EINVAL;
 	}
@@ -585,7 +553,14 @@ static int ad7192_write_raw_get_fmt(struct iio_dev *indio_dev,
 				    struct iio_chan_spec const *chan,
 				    long mask)
 {
-	return IIO_VAL_INT_PLUS_NANO;
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		return IIO_VAL_INT_PLUS_NANO;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
 }
 
 static const struct iio_info ad7192_info = {
@@ -659,15 +634,30 @@ static int ad7192_probe(struct spi_device *spi)
 
 	st = iio_priv(indio_dev);
 
-	st->reg = devm_regulator_get(&spi->dev, "vcc");
-	if (!IS_ERR(st->reg)) {
-		ret = regulator_enable(st->reg);
-		if (ret)
-			return ret;
+	st->avdd = devm_regulator_get(&spi->dev, "avdd");
+	if (IS_ERR(st->avdd))
+		return PTR_ERR(st->avdd);
 
-		voltage_uv = regulator_get_voltage(st->reg);
+	ret = regulator_enable(st->avdd);
+	if (ret) {
+		dev_err(&spi->dev, "Failed to enable specified AVdd supply\n");
+		return ret;
+	}
+
+	st->dvdd = devm_regulator_get(&spi->dev, "dvdd");
+	if (IS_ERR(st->dvdd)) {
+		ret = PTR_ERR(st->dvdd);
+		goto error_disable_avdd;
 	}
 
+	ret = regulator_enable(st->dvdd);
+	if (ret) {
+		dev_err(&spi->dev, "Failed to enable specified DVdd supply\n");
+		goto error_disable_avdd;
+	}
+
+	voltage_uv = regulator_get_voltage(st->avdd);
+
 	if (pdata->vref_mv)
 		st->int_vref_mv = pdata->vref_mv;
 	else if (voltage_uv)
@@ -701,7 +691,7 @@ static int ad7192_probe(struct spi_device *spi)
 
 	ret = ad_sd_setup_buffer_and_trigger(indio_dev);
 	if (ret)
-		goto error_disable_reg;
+		goto error_disable_dvdd;
 
 	ret = ad7192_setup(st, pdata);
 	if (ret)
@@ -714,9 +704,10 @@ static int ad7192_probe(struct spi_device *spi)
 
 error_remove_trigger:
 	ad_sd_cleanup_buffer_and_trigger(indio_dev);
-error_disable_reg:
-	if (!IS_ERR(st->reg))
-		regulator_disable(st->reg);
+error_disable_dvdd:
+	regulator_disable(st->dvdd);
+error_disable_avdd:
+	regulator_disable(st->avdd);
 
 	return ret;
 }
@@ -729,8 +720,8 @@ static int ad7192_remove(struct spi_device *spi)
 	iio_device_unregister(indio_dev);
 	ad_sd_cleanup_buffer_and_trigger(indio_dev);
 
-	if (!IS_ERR(st->reg))
-		regulator_disable(st->reg);
+	regulator_disable(st->dvdd);
+	regulator_disable(st->avdd);
 
 	return 0;
 }
diff --git a/drivers/staging/iio/adc/ad7280a.c b/drivers/staging/iio/adc/ad7280a.c
index b460dda7eb65..ee679ac0368f 100644
--- a/drivers/staging/iio/adc/ad7280a.c
+++ b/drivers/staging/iio/adc/ad7280a.c
@@ -777,7 +777,7 @@ static struct attribute *ad7280_event_attributes[] = {
 	NULL,
 };
 
-static struct attribute_group ad7280_event_attrs_group = {
+static const struct attribute_group ad7280_event_attrs_group = {
 	.attrs = ad7280_event_attributes,
 };
 
diff --git a/drivers/staging/iio/adc/ad7606_core.c b/drivers/staging/iio/adc/ad7606.c
index f79ee61851f6..453190864b2f 100644
--- a/drivers/staging/iio/adc/ad7606_core.c
+++ b/drivers/staging/iio/adc/ad7606.c
@@ -13,7 +13,7 @@
 #include <linux/sysfs.h>
 #include <linux/regulator/consumer.h>
 #include <linux/err.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/delay.h>
 #include <linux/sched.h>
 #include <linux/module.h>
@@ -21,58 +21,109 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/iio/buffer.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
 
 #include "ad7606.h"
 
-int ad7606_reset(struct ad7606_state *st)
+static int ad7606_reset(struct ad7606_state *st)
 {
-	if (gpio_is_valid(st->pdata->gpio_reset)) {
-		gpio_set_value(st->pdata->gpio_reset, 1);
+	if (st->gpio_reset) {
+		gpiod_set_value(st->gpio_reset, 1);
 		ndelay(100); /* t_reset >= 100ns */
-		gpio_set_value(st->pdata->gpio_reset, 0);
+		gpiod_set_value(st->gpio_reset, 0);
 		return 0;
 	}
 
 	return -ENODEV;
 }
 
+static int ad7606_read_samples(struct ad7606_state *st)
+{
+	unsigned int num = st->chip_info->num_channels;
+	u16 *data = st->data;
+	int ret;
+
+	/*
+	 * The frstdata signal is set to high while and after reading the sample
+	 * of the first channel and low for all other channels. This can be used
+	 * to check that the incoming data is correctly aligned. During normal
+	 * operation the data should never become unaligned, but some glitch or
+	 * electrostatic discharge might cause an extra read or clock cycle.
+	 * Monitoring the frstdata signal allows to recover from such failure
+	 * situations.
+	 */
+
+	if (st->gpio_frstdata) {
+		ret = st->bops->read_block(st->dev, 1, data);
+		if (ret)
+			return ret;
+
+		if (!gpiod_get_value(st->gpio_frstdata)) {
+			ad7606_reset(st);
+			return -EIO;
+		}
+
+		data++;
+		num--;
+	}
+
+	return st->bops->read_block(st->dev, num, data);
+}
+
+static irqreturn_t ad7606_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct ad7606_state *st = iio_priv(pf->indio_dev);
+
+	gpiod_set_value(st->gpio_convst, 1);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * ad7606_poll_bh_to_ring() bh of trigger launched polling to ring buffer
+ * @work_s:	the work struct through which this was scheduled
+ *
+ * Currently there is no option in this driver to disable the saving of
+ * timestamps within the ring.
+ * I think the one copy of this at a time was to avoid problems if the
+ * trigger was set far too high and the reads then locked up the computer.
+ **/
+static void ad7606_poll_bh_to_ring(struct work_struct *work_s)
+{
+	struct ad7606_state *st = container_of(work_s, struct ad7606_state,
+						poll_work);
+	struct iio_dev *indio_dev = iio_priv_to_dev(st);
+	int ret;
+
+	ret = ad7606_read_samples(st);
+	if (ret == 0)
+		iio_push_to_buffers_with_timestamp(indio_dev, st->data,
+						   iio_get_time_ns(indio_dev));
+
+	gpiod_set_value(st->gpio_convst, 0);
+	iio_trigger_notify_done(indio_dev->trig);
+}
+
 static int ad7606_scan_direct(struct iio_dev *indio_dev, unsigned int ch)
 {
 	struct ad7606_state *st = iio_priv(indio_dev);
 	int ret;
 
 	st->done = false;
-	gpio_set_value(st->pdata->gpio_convst, 1);
+	gpiod_set_value(st->gpio_convst, 1);
 
 	ret = wait_event_interruptible(st->wq_data_avail, st->done);
 	if (ret)
 		goto error_ret;
 
-	if (gpio_is_valid(st->pdata->gpio_frstdata)) {
-		ret = st->bops->read_block(st->dev, 1, st->data);
-		if (ret)
-			goto error_ret;
-		if (!gpio_get_value(st->pdata->gpio_frstdata)) {
-			/* This should never happen */
-			ad7606_reset(st);
-			ret = -EIO;
-			goto error_ret;
-		}
-		ret = st->bops->read_block(st->dev,
-			st->chip_info->num_channels - 1, &st->data[1]);
-		if (ret)
-			goto error_ret;
-	} else {
-		ret = st->bops->read_block(st->dev,
-			st->chip_info->num_channels, st->data);
-		if (ret)
-			goto error_ret;
-	}
-
-	ret = st->data[ch];
+	ret = ad7606_read_samples(st);
+	if (ret == 0)
+		ret = st->data[ch];
 
 error_ret:
-	gpio_set_value(st->pdata->gpio_convst, 0);
+	gpiod_set_value(st->gpio_convst, 0);
 
 	return ret;
 }
@@ -103,6 +154,9 @@ static int ad7606_read_raw(struct iio_dev *indio_dev,
 		*val = st->range * 2;
 		*val2 = st->chip_info->channels[0].scan_type.realbits;
 		return IIO_VAL_FRACTIONAL_LOG2;
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		*val = st->oversampling;
+		return IIO_VAL_INT;
 	}
 	return -EINVAL;
 }
@@ -129,12 +183,11 @@ static ssize_t ad7606_store_range(struct device *dev,
 	if (ret)
 		return ret;
 
-	if (!(lval == 5000 || lval == 10000)) {
-		dev_err(dev, "range is not supported\n");
+	if (!(lval == 5000 || lval == 10000))
 		return -EINVAL;
-	}
+
 	mutex_lock(&indio_dev->mlock);
-	gpio_set_value(st->pdata->gpio_range, lval == 10000);
+	gpiod_set_value(st->gpio_range, lval == 10000);
 	st->range = lval;
 	mutex_unlock(&indio_dev->mlock);
 
@@ -145,19 +198,9 @@ static IIO_DEVICE_ATTR(in_voltage_range, S_IRUGO | S_IWUSR,
 		       ad7606_show_range, ad7606_store_range, 0);
 static IIO_CONST_ATTR(in_voltage_range_available, "5000 10000");
 
-static ssize_t ad7606_show_oversampling_ratio(struct device *dev,
-					      struct device_attribute *attr,
-					      char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ad7606_state *st = iio_priv(indio_dev);
-
-	return sprintf(buf, "%u\n", st->oversampling);
-}
-
 static int ad7606_oversampling_get_index(unsigned int val)
 {
-	unsigned char supported[] = {0, 2, 4, 8, 16, 32, 64};
+	unsigned char supported[] = {1, 2, 4, 8, 16, 32, 64};
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(supported); i++)
@@ -167,44 +210,45 @@ static int ad7606_oversampling_get_index(unsigned int val)
 	return -EINVAL;
 }
 
-static ssize_t ad7606_store_oversampling_ratio(struct device *dev,
-					       struct device_attribute *attr,
-					       const char *buf, size_t count)
+static int ad7606_write_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int val,
+			    int val2,
+			    long mask)
 {
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ad7606_state *st = iio_priv(indio_dev);
-	unsigned long lval;
+	int values[3];
 	int ret;
 
-	ret = kstrtoul(buf, 10, &lval);
-	if (ret)
-		return ret;
+	switch (mask) {
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		if (val2)
+			return -EINVAL;
+		ret = ad7606_oversampling_get_index(val);
+		if (ret < 0)
+			return ret;
 
-	ret = ad7606_oversampling_get_index(lval);
-	if (ret < 0) {
-		dev_err(dev, "oversampling %lu is not supported\n", lval);
-		return ret;
-	}
+		values[0] = (ret >> 0) & 1;
+		values[1] = (ret >> 1) & 1;
+		values[2] = (ret >> 2) & 1;
 
-	mutex_lock(&indio_dev->mlock);
-	gpio_set_value(st->pdata->gpio_os0, (ret >> 0) & 1);
-	gpio_set_value(st->pdata->gpio_os1, (ret >> 1) & 1);
-	gpio_set_value(st->pdata->gpio_os1, (ret >> 2) & 1);
-	st->oversampling = lval;
-	mutex_unlock(&indio_dev->mlock);
+		mutex_lock(&indio_dev->mlock);
+		gpiod_set_array_value(ARRAY_SIZE(values), st->gpio_os->desc,
+				      values);
+		st->oversampling = val;
+		mutex_unlock(&indio_dev->mlock);
 
-	return count;
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
-static IIO_DEVICE_ATTR(oversampling_ratio, S_IRUGO | S_IWUSR,
-		       ad7606_show_oversampling_ratio,
-		       ad7606_store_oversampling_ratio, 0);
-static IIO_CONST_ATTR(oversampling_ratio_available, "0 2 4 8 16 32 64");
+static IIO_CONST_ATTR(oversampling_ratio_available, "1 2 4 8 16 32 64");
 
 static struct attribute *ad7606_attributes_os_and_range[] = {
 	&iio_dev_attr_in_voltage_range.dev_attr.attr,
 	&iio_const_attr_in_voltage_range_available.dev_attr.attr,
-	&iio_dev_attr_oversampling_ratio.dev_attr.attr,
 	&iio_const_attr_oversampling_ratio_available.dev_attr.attr,
 	NULL,
 };
@@ -214,7 +258,6 @@ static const struct attribute_group ad7606_attribute_group_os_and_range = {
 };
 
 static struct attribute *ad7606_attributes_os[] = {
-	&iio_dev_attr_oversampling_ratio.dev_attr.attr,
 	&iio_const_attr_oversampling_ratio_available.dev_attr.attr,
 	NULL,
 };
@@ -241,6 +284,8 @@ static const struct attribute_group ad7606_attribute_group_range = {
 		.address = num,					\
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),\
+		.info_mask_shared_by_all =			\
+			BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),	\
 		.scan_index = num,				\
 		.scan_type = {					\
 			.sign = 's',				\
@@ -267,20 +312,14 @@ static const struct ad7606_chip_info ad7606_chip_info_tbl[] = {
 	 * More devices added in future
 	 */
 	[ID_AD7606_8] = {
-		.name = "ad7606",
-		.int_vref_mv = 2500,
 		.channels = ad7606_channels,
 		.num_channels = 9,
 	},
 	[ID_AD7606_6] = {
-		.name = "ad7606-6",
-		.int_vref_mv = 2500,
 		.channels = ad7606_channels,
 		.num_channels = 7,
 	},
 	[ID_AD7606_4] = {
-		.name = "ad7606-4",
-		.int_vref_mv = 2500,
 		.channels = ad7606_channels,
 		.num_channels = 5,
 	},
@@ -288,119 +327,34 @@ static const struct ad7606_chip_info ad7606_chip_info_tbl[] = {
 
 static int ad7606_request_gpios(struct ad7606_state *st)
 {
-	struct gpio gpio_array[3] = {
-		[0] = {
-			.gpio =  st->pdata->gpio_os0,
-			.flags = GPIOF_DIR_OUT | ((st->oversampling & 1) ?
-				 GPIOF_INIT_HIGH : GPIOF_INIT_LOW),
-			.label = "AD7606_OS0",
-		},
-		[1] = {
-			.gpio =  st->pdata->gpio_os1,
-			.flags = GPIOF_DIR_OUT | ((st->oversampling & 2) ?
-				 GPIOF_INIT_HIGH : GPIOF_INIT_LOW),
-			.label = "AD7606_OS1",
-		},
-		[2] = {
-			.gpio =  st->pdata->gpio_os2,
-			.flags = GPIOF_DIR_OUT | ((st->oversampling & 4) ?
-				 GPIOF_INIT_HIGH : GPIOF_INIT_LOW),
-			.label = "AD7606_OS2",
-		},
-	};
-	int ret;
-
-	if (gpio_is_valid(st->pdata->gpio_convst)) {
-		ret = gpio_request_one(st->pdata->gpio_convst,
-				       GPIOF_OUT_INIT_LOW,
-				       "AD7606_CONVST");
-		if (ret) {
-			dev_err(st->dev, "failed to request GPIO CONVST\n");
-			goto error_ret;
-		}
-	} else {
-		ret = -EIO;
-		goto error_ret;
-	}
-
-	if (gpio_is_valid(st->pdata->gpio_os0) &&
-	    gpio_is_valid(st->pdata->gpio_os1) &&
-	    gpio_is_valid(st->pdata->gpio_os2)) {
-		ret = gpio_request_array(gpio_array, ARRAY_SIZE(gpio_array));
-		if (ret < 0)
-			goto error_free_convst;
-	}
-
-	if (gpio_is_valid(st->pdata->gpio_reset)) {
-		ret = gpio_request_one(st->pdata->gpio_reset,
-				       GPIOF_OUT_INIT_LOW,
-				       "AD7606_RESET");
-		if (ret < 0)
-			goto error_free_os;
-	}
-
-	if (gpio_is_valid(st->pdata->gpio_range)) {
-		ret = gpio_request_one(st->pdata->gpio_range, GPIOF_DIR_OUT |
-				       ((st->range == 10000) ? GPIOF_INIT_HIGH :
-					GPIOF_INIT_LOW), "AD7606_RANGE");
-		if (ret < 0)
-			goto error_free_reset;
-	}
-	if (gpio_is_valid(st->pdata->gpio_stby)) {
-		ret = gpio_request_one(st->pdata->gpio_stby,
-				       GPIOF_OUT_INIT_HIGH,
-				       "AD7606_STBY");
-		if (ret < 0)
-			goto error_free_range;
-	}
-
-	if (gpio_is_valid(st->pdata->gpio_frstdata)) {
-		ret = gpio_request_one(st->pdata->gpio_frstdata, GPIOF_IN,
-				       "AD7606_FRSTDATA");
-		if (ret < 0)
-			goto error_free_stby;
-	}
-
-	return 0;
-
-error_free_stby:
-	if (gpio_is_valid(st->pdata->gpio_stby))
-		gpio_free(st->pdata->gpio_stby);
-error_free_range:
-	if (gpio_is_valid(st->pdata->gpio_range))
-		gpio_free(st->pdata->gpio_range);
-error_free_reset:
-	if (gpio_is_valid(st->pdata->gpio_reset))
-		gpio_free(st->pdata->gpio_reset);
-error_free_os:
-	if (gpio_is_valid(st->pdata->gpio_os0) &&
-	    gpio_is_valid(st->pdata->gpio_os1) &&
-	    gpio_is_valid(st->pdata->gpio_os2))
-		gpio_free_array(gpio_array, ARRAY_SIZE(gpio_array));
-error_free_convst:
-	gpio_free(st->pdata->gpio_convst);
-error_ret:
-	return ret;
-}
-
-static void ad7606_free_gpios(struct ad7606_state *st)
-{
-	if (gpio_is_valid(st->pdata->gpio_frstdata))
-		gpio_free(st->pdata->gpio_frstdata);
-	if (gpio_is_valid(st->pdata->gpio_stby))
-		gpio_free(st->pdata->gpio_stby);
-	if (gpio_is_valid(st->pdata->gpio_range))
-		gpio_free(st->pdata->gpio_range);
-	if (gpio_is_valid(st->pdata->gpio_reset))
-		gpio_free(st->pdata->gpio_reset);
-	if (gpio_is_valid(st->pdata->gpio_os0) &&
-	    gpio_is_valid(st->pdata->gpio_os1) &&
-	    gpio_is_valid(st->pdata->gpio_os2)) {
-		gpio_free(st->pdata->gpio_os2);
-		gpio_free(st->pdata->gpio_os1);
-		gpio_free(st->pdata->gpio_os0);
-	}
-	gpio_free(st->pdata->gpio_convst);
+	struct device *dev = st->dev;
+
+	st->gpio_convst = devm_gpiod_get(dev, "conversion-start",
+					 GPIOD_OUT_LOW);
+	if (IS_ERR(st->gpio_convst))
+		return PTR_ERR(st->gpio_convst);
+
+	st->gpio_reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(st->gpio_reset))
+		return PTR_ERR(st->gpio_reset);
+
+	st->gpio_range = devm_gpiod_get_optional(dev, "range", GPIOD_OUT_LOW);
+	if (IS_ERR(st->gpio_range))
+		return PTR_ERR(st->gpio_range);
+
+	st->gpio_standby = devm_gpiod_get_optional(dev, "standby",
+						   GPIOD_OUT_HIGH);
+	if (IS_ERR(st->gpio_standby))
+		return PTR_ERR(st->gpio_standby);
+
+	st->gpio_frstdata = devm_gpiod_get_optional(dev, "first-data",
+						    GPIOD_IN);
+	if (IS_ERR(st->gpio_frstdata))
+		return PTR_ERR(st->gpio_frstdata);
+
+	st->gpio_os = devm_gpiod_get_array_optional(dev, "oversampling-ratio",
+			GPIOD_OUT_LOW);
+	return PTR_ERR_OR_ZERO(st->gpio_os);
 }
 
 /**
@@ -429,12 +383,14 @@ static const struct iio_info ad7606_info_no_os_or_range = {
 static const struct iio_info ad7606_info_os_and_range = {
 	.driver_module = THIS_MODULE,
 	.read_raw = &ad7606_read_raw,
+	.write_raw = &ad7606_write_raw,
 	.attrs = &ad7606_attribute_group_os_and_range,
 };
 
 static const struct iio_info ad7606_info_os = {
 	.driver_module = THIS_MODULE,
 	.read_raw = &ad7606_read_raw,
+	.write_raw = &ad7606_write_raw,
 	.attrs = &ad7606_attribute_group_os,
 };
 
@@ -444,81 +400,73 @@ static const struct iio_info ad7606_info_range = {
 	.attrs = &ad7606_attribute_group_range,
 };
 
-struct iio_dev *ad7606_probe(struct device *dev, int irq,
-			     void __iomem *base_address,
-			     unsigned int id,
-			     const struct ad7606_bus_ops *bops)
+int ad7606_probe(struct device *dev, int irq, void __iomem *base_address,
+		 const char *name, unsigned int id,
+		 const struct ad7606_bus_ops *bops)
 {
-	struct ad7606_platform_data *pdata = dev->platform_data;
 	struct ad7606_state *st;
 	int ret;
 	struct iio_dev *indio_dev;
 
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
-		return ERR_PTR(-ENOMEM);
+		return -ENOMEM;
 
 	st = iio_priv(indio_dev);
 
 	st->dev = dev;
 	st->bops = bops;
 	st->base_address = base_address;
-	st->range = pdata->default_range == 10000 ? 10000 : 5000;
+	st->range = 5000;
+	st->oversampling = 1;
+	INIT_WORK(&st->poll_work, &ad7606_poll_bh_to_ring);
 
-	ret = ad7606_oversampling_get_index(pdata->default_os);
-	if (ret < 0) {
-		dev_warn(dev, "oversampling %d is not supported\n",
-			 pdata->default_os);
-		st->oversampling = 0;
-	} else {
-		st->oversampling = pdata->default_os;
-	}
+	st->reg = devm_regulator_get(dev, "avcc");
+	if (IS_ERR(st->reg))
+		return PTR_ERR(st->reg);
 
-	st->reg = devm_regulator_get(dev, "vcc");
-	if (!IS_ERR(st->reg)) {
-		ret = regulator_enable(st->reg);
-		if (ret)
-			return ERR_PTR(ret);
+	ret = regulator_enable(st->reg);
+	if (ret) {
+		dev_err(dev, "Failed to enable specified AVcc supply\n");
+		return ret;
 	}
 
-	st->pdata = pdata;
+	ret = ad7606_request_gpios(st);
+	if (ret)
+		goto error_disable_reg;
+
 	st->chip_info = &ad7606_chip_info_tbl[id];
 
 	indio_dev->dev.parent = dev;
-	if (gpio_is_valid(st->pdata->gpio_os0) &&
-	    gpio_is_valid(st->pdata->gpio_os1) &&
-	    gpio_is_valid(st->pdata->gpio_os2)) {
-		if (gpio_is_valid(st->pdata->gpio_range))
+	if (st->gpio_os) {
+		if (st->gpio_range)
 			indio_dev->info = &ad7606_info_os_and_range;
 		else
 			indio_dev->info = &ad7606_info_os;
 	} else {
-		if (gpio_is_valid(st->pdata->gpio_range))
+		if (st->gpio_range)
 			indio_dev->info = &ad7606_info_range;
 		else
 			indio_dev->info = &ad7606_info_no_os_or_range;
 	}
 	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->name = st->chip_info->name;
+	indio_dev->name = name;
 	indio_dev->channels = st->chip_info->channels;
 	indio_dev->num_channels = st->chip_info->num_channels;
 
 	init_waitqueue_head(&st->wq_data_avail);
 
-	ret = ad7606_request_gpios(st);
-	if (ret)
-		goto error_disable_reg;
-
 	ret = ad7606_reset(st);
 	if (ret)
 		dev_warn(st->dev, "failed to RESET: no RESET GPIO specified\n");
 
-	ret = request_irq(irq, ad7606_interrupt,
-			  IRQF_TRIGGER_FALLING, st->chip_info->name, indio_dev);
+	ret = request_irq(irq, ad7606_interrupt, IRQF_TRIGGER_FALLING, name,
+			  indio_dev);
 	if (ret)
-		goto error_free_gpios;
+		goto error_disable_reg;
 
-	ret = ad7606_register_ring_funcs_and_init(indio_dev);
+	ret = iio_triggered_buffer_setup(indio_dev, &ad7606_trigger_handler,
+					 NULL, NULL);
 	if (ret)
 		goto error_free_irq;
 
@@ -526,35 +474,31 @@ struct iio_dev *ad7606_probe(struct device *dev, int irq,
 	if (ret)
 		goto error_unregister_ring;
 
-	return indio_dev;
+	dev_set_drvdata(dev, indio_dev);
+
+	return 0;
 error_unregister_ring:
-	ad7606_ring_cleanup(indio_dev);
+	iio_triggered_buffer_cleanup(indio_dev);
 
 error_free_irq:
 	free_irq(irq, indio_dev);
 
-error_free_gpios:
-	ad7606_free_gpios(st);
-
 error_disable_reg:
-	if (!IS_ERR(st->reg))
-		regulator_disable(st->reg);
-	return ERR_PTR(ret);
+	regulator_disable(st->reg);
+	return ret;
 }
 EXPORT_SYMBOL_GPL(ad7606_probe);
 
-int ad7606_remove(struct iio_dev *indio_dev, int irq)
+int ad7606_remove(struct device *dev, int irq)
 {
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct ad7606_state *st = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
-	ad7606_ring_cleanup(indio_dev);
+	iio_triggered_buffer_cleanup(indio_dev);
 
 	free_irq(irq, indio_dev);
-	if (!IS_ERR(st->reg))
-		regulator_disable(st->reg);
-
-	ad7606_free_gpios(st);
+	regulator_disable(st->reg);
 
 	return 0;
 }
@@ -567,10 +511,9 @@ static int ad7606_suspend(struct device *dev)
 	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct ad7606_state *st = iio_priv(indio_dev);
 
-	if (gpio_is_valid(st->pdata->gpio_stby)) {
-		if (gpio_is_valid(st->pdata->gpio_range))
-			gpio_set_value(st->pdata->gpio_range, 1);
-		gpio_set_value(st->pdata->gpio_stby, 0);
+	if (st->gpio_standby) {
+		gpiod_set_value(st->gpio_range, 1);
+		gpiod_set_value(st->gpio_standby, 0);
 	}
 
 	return 0;
@@ -581,12 +524,9 @@ static int ad7606_resume(struct device *dev)
 	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct ad7606_state *st = iio_priv(indio_dev);
 
-	if (gpio_is_valid(st->pdata->gpio_stby)) {
-		if (gpio_is_valid(st->pdata->gpio_range))
-			gpio_set_value(st->pdata->gpio_range,
-				       st->range == 10000);
-
-		gpio_set_value(st->pdata->gpio_stby, 1);
+	if (st->gpio_standby) {
+		gpiod_set_value(st->gpio_range, st->range == 10000);
+		gpiod_set_value(st->gpio_standby, 1);
 		ad7606_reset(st);
 	}
 
diff --git a/drivers/staging/iio/adc/ad7606.h b/drivers/staging/iio/adc/ad7606.h
index 39f50440d915..746f9553d2ba 100644
--- a/drivers/staging/iio/adc/ad7606.h
+++ b/drivers/staging/iio/adc/ad7606.h
@@ -9,48 +9,14 @@
 #ifndef IIO_ADC_AD7606_H_
 #define IIO_ADC_AD7606_H_
 
-/*
- * TODO: struct ad7606_platform_data needs to go into include/linux/iio
- */
-
-/**
- * struct ad7606_platform_data - platform/board specific information
- * @default_os:		default oversampling value {0, 2, 4, 8, 16, 32, 64}
- * @default_range:	default range +/-{5000, 10000} mVolt
- * @gpio_convst:	number of gpio connected to the CONVST pin
- * @gpio_reset:		gpio connected to the RESET pin, if not used set to -1
- * @gpio_range:		gpio connected to the RANGE pin, if not used set to -1
- * @gpio_os0:		gpio connected to the OS0 pin, if not used set to -1
- * @gpio_os1:		gpio connected to the OS1 pin, if not used set to -1
- * @gpio_os2:		gpio connected to the OS2 pin, if not used set to -1
- * @gpio_frstdata:	gpio connected to the FRSTDAT pin, if not used set to -1
- * @gpio_stby:		gpio connected to the STBY pin, if not used set to -1
- */
-
-struct ad7606_platform_data {
-	unsigned int			default_os;
-	unsigned int			default_range;
-	unsigned int			gpio_convst;
-	unsigned int			gpio_reset;
-	unsigned int			gpio_range;
-	unsigned int			gpio_os0;
-	unsigned int			gpio_os1;
-	unsigned int			gpio_os2;
-	unsigned int			gpio_frstdata;
-	unsigned int			gpio_stby;
-};
-
 /**
  * struct ad7606_chip_info - chip specific information
  * @name:		identification string for chip
- * @int_vref_mv:	the internal reference voltage
  * @channels:		channel specification
  * @num_channels:	number of channels
  */
 
 struct ad7606_chip_info {
-	const char			*name;
-	u16				int_vref_mv;
 	const struct iio_chan_spec	*channels;
 	unsigned int			num_channels;
 };
@@ -62,7 +28,6 @@ struct ad7606_chip_info {
 struct ad7606_state {
 	struct device			*dev;
 	const struct ad7606_chip_info	*chip_info;
-	struct ad7606_platform_data	*pdata;
 	struct regulator		*reg;
 	struct work_struct		poll_work;
 	wait_queue_head_t		wq_data_avail;
@@ -72,12 +37,19 @@ struct ad7606_state {
 	bool				done;
 	void __iomem			*base_address;
 
+	struct gpio_desc		*gpio_convst;
+	struct gpio_desc		*gpio_reset;
+	struct gpio_desc		*gpio_range;
+	struct gpio_desc		*gpio_standby;
+	struct gpio_desc		*gpio_frstdata;
+	struct gpio_descs		*gpio_os;
+
 	/*
 	 * DMA (thus cache coherency maintenance) requires the
 	 * transfer buffers to live in their own cache lines.
+	 * 8 * 16-bit samples + 64-bit timestamp
 	 */
-
-	unsigned short			data[8] ____cacheline_aligned;
+	unsigned short			data[12] ____cacheline_aligned;
 };
 
 struct ad7606_bus_ops {
@@ -85,11 +57,10 @@ struct ad7606_bus_ops {
 	int (*read_block)(struct device *, int, void *);
 };
 
-struct iio_dev *ad7606_probe(struct device *dev, int irq,
-			      void __iomem *base_address, unsigned int id,
-			      const struct ad7606_bus_ops *bops);
-int ad7606_remove(struct iio_dev *indio_dev, int irq);
-int ad7606_reset(struct ad7606_state *st);
+int ad7606_probe(struct device *dev, int irq, void __iomem *base_address,
+		 const char *name, unsigned int id,
+		 const struct ad7606_bus_ops *bops);
+int ad7606_remove(struct device *dev, int irq);
 
 enum ad7606_supported_device_ids {
 	ID_AD7606_8,
@@ -97,9 +68,6 @@ enum ad7606_supported_device_ids {
 	ID_AD7606_4
 };
 
-int ad7606_register_ring_funcs_and_init(struct iio_dev *indio_dev);
-void ad7606_ring_cleanup(struct iio_dev *indio_dev);
-
 #ifdef CONFIG_PM_SLEEP
 extern const struct dev_pm_ops ad7606_pm_ops;
 #define AD7606_PM_OPS (&ad7606_pm_ops)
diff --git a/drivers/staging/iio/adc/ad7606_par.c b/drivers/staging/iio/adc/ad7606_par.c
index 84d23930fdde..cd6c410c0484 100644
--- a/drivers/staging/iio/adc/ad7606_par.c
+++ b/drivers/staging/iio/adc/ad7606_par.c
@@ -49,8 +49,8 @@ static const struct ad7606_bus_ops ad7606_par8_bops = {
 
 static int ad7606_par_probe(struct platform_device *pdev)
 {
+	const struct platform_device_id *id = platform_get_device_id(pdev);
 	struct resource *res;
-	struct iio_dev *indio_dev;
 	void __iomem *addr;
 	resource_size_t remap_size;
 	int irq;
@@ -68,26 +68,15 @@ static int ad7606_par_probe(struct platform_device *pdev)
 
 	remap_size = resource_size(res);
 
-	indio_dev = ad7606_probe(&pdev->dev, irq, addr,
-				 platform_get_device_id(pdev)->driver_data,
-				 remap_size > 1 ? &ad7606_par16_bops :
-				 &ad7606_par8_bops);
-
-	if (IS_ERR(indio_dev))
-		return PTR_ERR(indio_dev);
-
-	platform_set_drvdata(pdev, indio_dev);
-
-	return 0;
+	return ad7606_probe(&pdev->dev, irq, addr,
+			    id->name, id->driver_data,
+			    remap_size > 1 ? &ad7606_par16_bops :
+			    &ad7606_par8_bops);
 }
 
 static int ad7606_par_remove(struct platform_device *pdev)
 {
-	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
-
-	ad7606_remove(indio_dev, platform_get_irq(pdev, 0));
-
-	return 0;
+	return ad7606_remove(&pdev->dev, platform_get_irq(pdev, 0));
 }
 
 static const struct platform_device_id ad7606_driver_ids[] = {
diff --git a/drivers/staging/iio/adc/ad7606_ring.c b/drivers/staging/iio/adc/ad7606_ring.c
deleted file mode 100644
index 0572df9aad85..000000000000
--- a/drivers/staging/iio/adc/ad7606_ring.c
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright 2011-2012 Analog Devices Inc.
- *
- * Licensed under the GPL-2.
- *
- */
-
-#include <linux/interrupt.h>
-#include <linux/gpio.h>
-#include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/slab.h>
-
-#include <linux/iio/iio.h>
-#include <linux/iio/buffer.h>
-#include <linux/iio/trigger_consumer.h>
-#include <linux/iio/triggered_buffer.h>
-
-#include "ad7606.h"
-
-/**
- * ad7606_trigger_handler_th() th/bh of trigger launched polling to ring buffer
- *
- **/
-static irqreturn_t ad7606_trigger_handler_th_bh(int irq, void *p)
-{
-	struct iio_poll_func *pf = p;
-	struct ad7606_state *st = iio_priv(pf->indio_dev);
-
-	gpio_set_value(st->pdata->gpio_convst, 1);
-
-	return IRQ_HANDLED;
-}
-
-/**
- * ad7606_poll_bh_to_ring() bh of trigger launched polling to ring buffer
- * @work_s:	the work struct through which this was scheduled
- *
- * Currently there is no option in this driver to disable the saving of
- * timestamps within the ring.
- * I think the one copy of this at a time was to avoid problems if the
- * trigger was set far too high and the reads then locked up the computer.
- **/
-static void ad7606_poll_bh_to_ring(struct work_struct *work_s)
-{
-	struct ad7606_state *st = container_of(work_s, struct ad7606_state,
-						poll_work);
-	struct iio_dev *indio_dev = iio_priv_to_dev(st);
-	__u8 *buf;
-	int ret;
-
-	buf = kzalloc(indio_dev->scan_bytes, GFP_KERNEL);
-	if (!buf)
-		return;
-
-	if (gpio_is_valid(st->pdata->gpio_frstdata)) {
-		ret = st->bops->read_block(st->dev, 1, buf);
-		if (ret)
-			goto done;
-		if (!gpio_get_value(st->pdata->gpio_frstdata)) {
-			/* This should never happen. However
-			 * some signal glitch caused by bad PCB desgin or
-			 * electrostatic discharge, could cause an extra read
-			 * or clock. This allows recovery.
-			 */
-			ad7606_reset(st);
-			goto done;
-		}
-		ret = st->bops->read_block(st->dev,
-			st->chip_info->num_channels - 1, buf + 2);
-		if (ret)
-			goto done;
-	} else {
-		ret = st->bops->read_block(st->dev,
-			st->chip_info->num_channels, buf);
-		if (ret)
-			goto done;
-	}
-
-	iio_push_to_buffers_with_timestamp(indio_dev, buf,
-					   iio_get_time_ns(indio_dev));
-done:
-	gpio_set_value(st->pdata->gpio_convst, 0);
-	iio_trigger_notify_done(indio_dev->trig);
-	kfree(buf);
-}
-
-int ad7606_register_ring_funcs_and_init(struct iio_dev *indio_dev)
-{
-	struct ad7606_state *st = iio_priv(indio_dev);
-
-	INIT_WORK(&st->poll_work, &ad7606_poll_bh_to_ring);
-
-	return iio_triggered_buffer_setup(indio_dev,
-		&ad7606_trigger_handler_th_bh, &ad7606_trigger_handler_th_bh,
-		NULL);
-}
-
-void ad7606_ring_cleanup(struct iio_dev *indio_dev)
-{
-	iio_triggered_buffer_cleanup(indio_dev);
-}
diff --git a/drivers/staging/iio/adc/ad7606_spi.c b/drivers/staging/iio/adc/ad7606_spi.c
index 9587fa86dc69..c9b1f26685f4 100644
--- a/drivers/staging/iio/adc/ad7606_spi.c
+++ b/drivers/staging/iio/adc/ad7606_spi.c
@@ -42,25 +42,16 @@ static const struct ad7606_bus_ops ad7606_spi_bops = {
 
 static int ad7606_spi_probe(struct spi_device *spi)
 {
-	struct iio_dev *indio_dev;
+	const struct spi_device_id *id = spi_get_device_id(spi);
 
-	indio_dev = ad7606_probe(&spi->dev, spi->irq, NULL,
-				 spi_get_device_id(spi)->driver_data,
-				 &ad7606_spi_bops);
-
-	if (IS_ERR(indio_dev))
-		return PTR_ERR(indio_dev);
-
-	spi_set_drvdata(spi, indio_dev);
-
-	return 0;
+	return ad7606_probe(&spi->dev, spi->irq, NULL,
+			    id->name, id->driver_data,
+			    &ad7606_spi_bops);
 }
 
 static int ad7606_spi_remove(struct spi_device *spi)
 {
-	struct iio_dev *indio_dev = dev_get_drvdata(&spi->dev);
-
-	return ad7606_remove(indio_dev, spi->irq);
+	return ad7606_remove(&spi->dev, spi->irq);
 }
 
 static const struct spi_device_id ad7606_id[] = {
diff --git a/drivers/staging/iio/adc/ad7780.c b/drivers/staging/iio/adc/ad7780.c
index c9a0c2aa602f..e14960038d3e 100644
--- a/drivers/staging/iio/adc/ad7780.c
+++ b/drivers/staging/iio/adc/ad7780.c
@@ -173,14 +173,16 @@ static int ad7780_probe(struct spi_device *spi)
 
 	ad_sd_init(&st->sd, indio_dev, spi, &ad7780_sigma_delta_info);
 
-	st->reg = devm_regulator_get(&spi->dev, "vcc");
-	if (!IS_ERR(st->reg)) {
-		ret = regulator_enable(st->reg);
-		if (ret)
-			return ret;
-
-		voltage_uv = regulator_get_voltage(st->reg);
+	st->reg = devm_regulator_get(&spi->dev, "avdd");
+	if (IS_ERR(st->reg))
+		return PTR_ERR(st->reg);
+
+	ret = regulator_enable(st->reg);
+	if (ret) {
+		dev_err(&spi->dev, "Failed to enable specified AVdd supply\n");
+		return ret;
 	}
+	voltage_uv = regulator_get_voltage(st->reg);
 
 	st->chip_info =
 		&ad7780_chip_info_tbl[spi_get_device_id(spi)->driver_data];
@@ -222,8 +224,7 @@ static int ad7780_probe(struct spi_device *spi)
 error_cleanup_buffer_and_trigger:
 	ad_sd_cleanup_buffer_and_trigger(indio_dev);
 error_disable_reg:
-	if (!IS_ERR(st->reg))
-		regulator_disable(st->reg);
+	regulator_disable(st->reg);
 
 	return ret;
 }
@@ -236,8 +237,7 @@ static int ad7780_remove(struct spi_device *spi)
 	iio_device_unregister(indio_dev);
 	ad_sd_cleanup_buffer_and_trigger(indio_dev);
 
-	if (!IS_ERR(st->reg))
-		regulator_disable(st->reg);
+	regulator_disable(st->reg);
 
 	return 0;
 }
diff --git a/drivers/staging/iio/adc/ad7816.c b/drivers/staging/iio/adc/ad7816.c
index 5e8115b01011..72551f827382 100644
--- a/drivers/staging/iio/adc/ad7816.c
+++ b/drivers/staging/iio/adc/ad7816.c
@@ -327,7 +327,7 @@ static struct attribute *ad7816_event_attributes[] = {
 	NULL,
 };
 
-static struct attribute_group ad7816_event_attribute_group = {
+static const struct attribute_group ad7816_event_attribute_group = {
 	.attrs = ad7816_event_attributes,
 	.name = "events",
 };
diff --git a/drivers/staging/iio/addac/adt7316.c b/drivers/staging/iio/addac/adt7316.c
index 3faffe59c933..a7d90c8bac5e 100644
--- a/drivers/staging/iio/addac/adt7316.c
+++ b/drivers/staging/iio/addac/adt7316.c
@@ -2039,7 +2039,7 @@ static struct attribute *adt7316_event_attributes[] = {
 	NULL,
 };
 
-static struct attribute_group adt7316_event_attribute_group = {
+static const struct attribute_group adt7316_event_attribute_group = {
 	.attrs = adt7316_event_attributes,
 	.name = "events",
 };
@@ -2060,7 +2060,7 @@ static struct attribute *adt7516_event_attributes[] = {
 	NULL,
 };
 
-static struct attribute_group adt7516_event_attribute_group = {
+static const struct attribute_group adt7516_event_attribute_group = {
 	.attrs = adt7516_event_attributes,
 	.name = "events",
 };
diff --git a/drivers/staging/iio/cdc/ad7150.c b/drivers/staging/iio/cdc/ad7150.c
index 5578a077fcfb..6998c3ddfb6a 100644
--- a/drivers/staging/iio/cdc/ad7150.c
+++ b/drivers/staging/iio/cdc/ad7150.c
@@ -562,7 +562,7 @@ static struct attribute *ad7150_event_attributes[] = {
 	NULL,
 };
 
-static struct attribute_group ad7150_event_attribute_group = {
+static const struct attribute_group ad7150_event_attribute_group = {
 	.attrs = ad7150_event_attributes,
 	.name = "events",
 };
diff --git a/drivers/staging/iio/cdc/ad7152.c b/drivers/staging/iio/cdc/ad7152.c
index 485d0a5af53c..b91b50f345bd 100644
--- a/drivers/staging/iio/cdc/ad7152.c
+++ b/drivers/staging/iio/cdc/ad7152.c
@@ -89,6 +89,7 @@ struct ad7152_chip_info {
 	 */
 	u8	filter_rate_setup;
 	u8	setup[2];
+	struct mutex state_lock;	/* protect hardware state */
 };
 
 static inline ssize_t ad7152_start_calib(struct device *dev,
@@ -115,10 +116,10 @@ static inline ssize_t ad7152_start_calib(struct device *dev,
 	else
 		regval |= AD7152_CONF_CH2EN;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&chip->state_lock);
 	ret = i2c_smbus_write_byte_data(chip->client, AD7152_REG_CFG, regval);
 	if (ret < 0) {
-		mutex_unlock(&indio_dev->mlock);
+		mutex_unlock(&chip->state_lock);
 		return ret;
 	}
 
@@ -126,14 +127,15 @@ static inline ssize_t ad7152_start_calib(struct device *dev,
 		mdelay(20);
 		ret = i2c_smbus_read_byte_data(chip->client, AD7152_REG_CFG);
 		if (ret < 0) {
-			mutex_unlock(&indio_dev->mlock);
+			mutex_unlock(&chip->state_lock);
 			return ret;
 		}
 	} while ((ret == regval) && timeout--);
 
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&chip->state_lock);
 	return len;
 }
+
 static ssize_t ad7152_start_offset_calib(struct device *dev,
 					 struct device_attribute *attr,
 					 const char *buf,
@@ -142,6 +144,7 @@ static ssize_t ad7152_start_offset_calib(struct device *dev,
 	return ad7152_start_calib(dev, attr, buf, len,
 				  AD7152_CONF_MODE_OFFS_CAL);
 }
+
 static ssize_t ad7152_start_gain_calib(struct device *dev,
 				       struct device_attribute *attr,
 				       const char *buf,
@@ -165,63 +168,12 @@ static const unsigned char ad7152_filter_rate_table[][2] = {
 	{200, 5 + 1}, {50, 20 + 1}, {20, 50 + 1}, {17, 60 + 1},
 };
 
-static ssize_t ad7152_show_filter_rate_setup(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ad7152_chip_info *chip = iio_priv(indio_dev);
-
-	return sprintf(buf, "%d\n",
-		       ad7152_filter_rate_table[chip->filter_rate_setup][0]);
-}
-
-static ssize_t ad7152_store_filter_rate_setup(struct device *dev,
-		struct device_attribute *attr,
-		const char *buf,
-		size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ad7152_chip_info *chip = iio_priv(indio_dev);
-	u8 data;
-	int ret, i;
-
-	ret = kstrtou8(buf, 10, &data);
-	if (ret < 0)
-		return ret;
-
-	for (i = 0; i < ARRAY_SIZE(ad7152_filter_rate_table); i++)
-		if (data >= ad7152_filter_rate_table[i][0])
-			break;
-
-	if (i >= ARRAY_SIZE(ad7152_filter_rate_table))
-		i = ARRAY_SIZE(ad7152_filter_rate_table) - 1;
-
-	mutex_lock(&indio_dev->mlock);
-	ret = i2c_smbus_write_byte_data(chip->client,
-			AD7152_REG_CFG2, AD7152_CFG2_OSR(i));
-	if (ret < 0) {
-		mutex_unlock(&indio_dev->mlock);
-		return ret;
-	}
-
-	chip->filter_rate_setup = i;
-	mutex_unlock(&indio_dev->mlock);
-
-	return len;
-}
-
-static IIO_DEV_ATTR_SAMP_FREQ(S_IRUGO | S_IWUSR,
-		ad7152_show_filter_rate_setup,
-		ad7152_store_filter_rate_setup);
-
 static IIO_CONST_ATTR_SAMP_FREQ_AVAIL("200 50 20 17");
 
 static IIO_CONST_ATTR(in_capacitance_scale_available,
 		      "0.000061050 0.000030525 0.000015263 0.000007631");
 
 static struct attribute *ad7152_attributes[] = {
-	&iio_dev_attr_sampling_frequency.dev_attr.attr,
 	&iio_dev_attr_in_capacitance0_calibbias_calibration.dev_attr.attr,
 	&iio_dev_attr_in_capacitance1_calibbias_calibration.dev_attr.attr,
 	&iio_dev_attr_in_capacitance0_calibscale_calibration.dev_attr.attr,
@@ -247,6 +199,51 @@ static const int ad7152_scale_table[] = {
 	30525, 7631, 15263, 61050
 };
 
+/**
+ * read_raw handler for IIO_CHAN_INFO_SAMP_FREQ
+ *
+ * lock must be held
+ **/
+static int ad7152_read_raw_samp_freq(struct device *dev, int *val)
+{
+	struct ad7152_chip_info *chip = iio_priv(dev_to_iio_dev(dev));
+
+	*val = ad7152_filter_rate_table[chip->filter_rate_setup][0];
+
+	return 0;
+}
+
+/**
+ * write_raw handler for IIO_CHAN_INFO_SAMP_FREQ
+ *
+ * lock must be held
+ **/
+static int ad7152_write_raw_samp_freq(struct device *dev, int val)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct ad7152_chip_info *chip = iio_priv(indio_dev);
+	int ret, i;
+
+	for (i = 0; i < ARRAY_SIZE(ad7152_filter_rate_table); i++)
+		if (val >= ad7152_filter_rate_table[i][0])
+			break;
+
+	if (i >= ARRAY_SIZE(ad7152_filter_rate_table))
+		i = ARRAY_SIZE(ad7152_filter_rate_table) - 1;
+
+	mutex_lock(&chip->state_lock);
+	ret = i2c_smbus_write_byte_data(chip->client,
+					AD7152_REG_CFG2, AD7152_CFG2_OSR(i));
+	if (ret < 0) {
+		mutex_unlock(&chip->state_lock);
+		return ret;
+	}
+
+	chip->filter_rate_setup = i;
+	mutex_unlock(&chip->state_lock);
+
+	return ret;
+}
 static int ad7152_write_raw(struct iio_dev *indio_dev,
 			    struct iio_chan_spec const *chan,
 			    int val,
@@ -256,7 +253,7 @@ static int ad7152_write_raw(struct iio_dev *indio_dev,
 	struct ad7152_chip_info *chip = iio_priv(indio_dev);
 	int ret, i;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&chip->state_lock);
 
 	switch (mask) {
 	case IIO_CHAN_INFO_CALIBSCALE:
@@ -309,14 +306,26 @@ static int ad7152_write_raw(struct iio_dev *indio_dev,
 
 		ret = 0;
 		break;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (val2) {
+			ret = -EINVAL;
+			goto out;
+		}
+		ret = ad7152_write_raw_samp_freq(&indio_dev->dev, val);
+		if (ret < 0)
+			goto out;
+
+		ret = 0;
+		break;
 	default:
 		ret = -EINVAL;
 	}
 
 out:
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&chip->state_lock);
 	return ret;
 }
+
 static int ad7152_read_raw(struct iio_dev *indio_dev,
 			   struct iio_chan_spec const *chan,
 			   int *val, int *val2,
@@ -326,7 +335,7 @@ static int ad7152_read_raw(struct iio_dev *indio_dev,
 	int ret;
 	u8 regval = 0;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&chip->state_lock);
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
@@ -403,11 +412,18 @@ static int ad7152_read_raw(struct iio_dev *indio_dev,
 
 		ret = IIO_VAL_INT_PLUS_NANO;
 		break;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		ret = ad7152_read_raw_samp_freq(&indio_dev->dev, val);
+		if (ret < 0)
+			goto out;
+
+		ret = IIO_VAL_INT;
+		break;
 	default:
 		ret = -EINVAL;
 	}
 out:
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&chip->state_lock);
 	return ret;
 }
 
@@ -440,6 +456,7 @@ static const struct iio_chan_spec ad7152_channels[] = {
 		BIT(IIO_CHAN_INFO_CALIBSCALE) |
 		BIT(IIO_CHAN_INFO_CALIBBIAS) |
 		BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 	}, {
 		.type = IIO_CAPACITANCE,
 		.differential = 1,
@@ -450,6 +467,7 @@ static const struct iio_chan_spec ad7152_channels[] = {
 		BIT(IIO_CHAN_INFO_CALIBSCALE) |
 		BIT(IIO_CHAN_INFO_CALIBBIAS) |
 		BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 	}, {
 		.type = IIO_CAPACITANCE,
 		.indexed = 1,
@@ -458,6 +476,7 @@ static const struct iio_chan_spec ad7152_channels[] = {
 		BIT(IIO_CHAN_INFO_CALIBSCALE) |
 		BIT(IIO_CHAN_INFO_CALIBBIAS) |
 		BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 	}, {
 		.type = IIO_CAPACITANCE,
 		.differential = 1,
@@ -468,8 +487,10 @@ static const struct iio_chan_spec ad7152_channels[] = {
 		BIT(IIO_CHAN_INFO_CALIBSCALE) |
 		BIT(IIO_CHAN_INFO_CALIBBIAS) |
 		BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 	}
 };
+
 /*
  * device probe and remove
  */
@@ -489,6 +510,7 @@ static int ad7152_probe(struct i2c_client *client,
 	i2c_set_clientdata(client, indio_dev);
 
 	chip->client = client;
+	mutex_init(&chip->state_lock);
 
 	/* Establish that the iio_dev is a child of the i2c device */
 	indio_dev->name = id->name;
diff --git a/drivers/staging/iio/cdc/ad7746.c b/drivers/staging/iio/cdc/ad7746.c
index 5771d4ee8ef1..81f8b9ee1120 100644
--- a/drivers/staging/iio/cdc/ad7746.c
+++ b/drivers/staging/iio/cdc/ad7746.c
@@ -70,8 +70,10 @@
 #define AD7746_EXCSETUP_EXCLVL(x)	(((x) & 0x3) << 0)
 
 /* Config Register Bit Designations (AD7746_REG_CFG) */
-#define AD7746_CONF_VTFS(x)		((x) << 6)
-#define AD7746_CONF_CAPFS(x)		((x) << 3)
+#define AD7746_CONF_VTFS_SHIFT		6
+#define AD7746_CONF_CAPFS_SHIFT		3
+#define AD7746_CONF_VTFS_MASK		GENMASK(7, 6)
+#define AD7746_CONF_CAPFS_MASK		GENMASK(5, 3)
 #define AD7746_CONF_MODE_IDLE		(0 << 0)
 #define AD7746_CONF_MODE_CONT_CONV	(1 << 0)
 #define AD7746_CONF_MODE_SINGLE_CONV	(2 << 0)
@@ -122,7 +124,8 @@ static const struct iio_chan_spec ad7746_channels[] = {
 		.indexed = 1,
 		.channel = 0,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
-		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) |
+			BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7746_REG_VT_DATA_HIGH << 8 |
 			AD7746_VTSETUP_VTMD_EXT_VIN,
 	},
@@ -132,7 +135,8 @@ static const struct iio_chan_spec ad7746_channels[] = {
 		.channel = 1,
 		.extend_name = "supply",
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
-		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) |
+			BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7746_REG_VT_DATA_HIGH << 8 |
 			AD7746_VTSETUP_VTMD_VDD_MON,
 	},
@@ -159,7 +163,7 @@ static const struct iio_chan_spec ad7746_channels[] = {
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
 		BIT(IIO_CHAN_INFO_CALIBSCALE) | BIT(IIO_CHAN_INFO_OFFSET),
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_CALIBBIAS) |
-		BIT(IIO_CHAN_INFO_SCALE),
+		BIT(IIO_CHAN_INFO_SCALE) | BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7746_REG_CAP_DATA_HIGH << 8,
 	},
 	[CIN1_DIFF] = {
@@ -171,7 +175,7 @@ static const struct iio_chan_spec ad7746_channels[] = {
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
 		BIT(IIO_CHAN_INFO_CALIBSCALE) | BIT(IIO_CHAN_INFO_OFFSET),
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_CALIBBIAS) |
-		BIT(IIO_CHAN_INFO_SCALE),
+		BIT(IIO_CHAN_INFO_SCALE) | BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7746_REG_CAP_DATA_HIGH << 8 |
 			AD7746_CAPSETUP_CAPDIFF
 	},
@@ -182,7 +186,7 @@ static const struct iio_chan_spec ad7746_channels[] = {
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
 		BIT(IIO_CHAN_INFO_CALIBSCALE) | BIT(IIO_CHAN_INFO_OFFSET),
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_CALIBBIAS) |
-		BIT(IIO_CHAN_INFO_SCALE),
+		BIT(IIO_CHAN_INFO_SCALE) | BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7746_REG_CAP_DATA_HIGH << 8 |
 			AD7746_CAPSETUP_CIN2,
 	},
@@ -195,7 +199,7 @@ static const struct iio_chan_spec ad7746_channels[] = {
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
 		BIT(IIO_CHAN_INFO_CALIBSCALE) | BIT(IIO_CHAN_INFO_OFFSET),
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_CALIBBIAS) |
-		BIT(IIO_CHAN_INFO_SCALE),
+		BIT(IIO_CHAN_INFO_SCALE) | BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7746_REG_CAP_DATA_HIGH << 8 |
 			AD7746_CAPSETUP_CAPDIFF | AD7746_CAPSETUP_CIN2,
 	}
@@ -215,15 +219,16 @@ static int ad7746_select_channel(struct iio_dev *indio_dev,
 			    struct iio_chan_spec const *chan)
 {
 	struct ad7746_chip_info *chip = iio_priv(indio_dev);
-	int ret, delay;
+	int ret, delay, idx;
 	u8 vt_setup, cap_setup;
 
 	switch (chan->type) {
 	case IIO_CAPACITANCE:
 		cap_setup = (chan->address & 0xFF) | AD7746_CAPSETUP_CAPEN;
 		vt_setup = chip->vt_setup & ~AD7746_VTSETUP_VTEN;
-		delay = ad7746_cap_filter_rate_table[(chip->config >> 3) &
-			0x7][1];
+		idx = (chip->config & AD7746_CONF_CAPFS_MASK) >>
+			AD7746_CONF_CAPFS_SHIFT;
+		delay = ad7746_cap_filter_rate_table[idx][1];
 
 		if (chip->capdac_set != chan->channel) {
 			ret = i2c_smbus_write_byte_data(chip->client,
@@ -244,8 +249,9 @@ static int ad7746_select_channel(struct iio_dev *indio_dev,
 	case IIO_TEMP:
 		vt_setup = (chan->address & 0xFF) | AD7746_VTSETUP_VTEN;
 		cap_setup = chip->cap_setup & ~AD7746_CAPSETUP_CAPEN;
-		delay = ad7746_cap_filter_rate_table[(chip->config >> 6) &
-			0x3][1];
+		idx = (chip->config & AD7746_CONF_VTFS_MASK) >>
+			AD7746_CONF_VTFS_SHIFT;
+		delay = ad7746_cap_filter_rate_table[idx][1];
 		break;
 	default:
 		return -EINVAL;
@@ -355,101 +361,47 @@ static IIO_DEVICE_ATTR(in_capacitance1_calibscale_calibration,
 static IIO_DEVICE_ATTR(in_voltage0_calibscale_calibration,
 		       S_IWUSR, NULL, ad7746_start_gain_calib, VIN);
 
-static ssize_t ad7746_show_cap_filter_rate_setup(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+static int ad7746_store_cap_filter_rate_setup(struct ad7746_chip_info *chip,
+					      int val)
 {
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ad7746_chip_info *chip = iio_priv(indio_dev);
-
-	return sprintf(buf, "%d\n", ad7746_cap_filter_rate_table[
-			(chip->config >> 3) & 0x7][0]);
-}
-
-static ssize_t ad7746_store_cap_filter_rate_setup(struct device *dev,
-		struct device_attribute *attr,
-		const char *buf,
-		size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ad7746_chip_info *chip = iio_priv(indio_dev);
-	u8 data;
-	int ret, i;
-
-	ret = kstrtou8(buf, 10, &data);
-	if (ret < 0)
-		return ret;
+	int i;
 
 	for (i = 0; i < ARRAY_SIZE(ad7746_cap_filter_rate_table); i++)
-		if (data >= ad7746_cap_filter_rate_table[i][0])
+		if (val >= ad7746_cap_filter_rate_table[i][0])
 			break;
 
 	if (i >= ARRAY_SIZE(ad7746_cap_filter_rate_table))
 		i = ARRAY_SIZE(ad7746_cap_filter_rate_table) - 1;
 
-	mutex_lock(&indio_dev->mlock);
-	chip->config &= ~AD7746_CONF_CAPFS(0x7);
-	chip->config |= AD7746_CONF_CAPFS(i);
-	mutex_unlock(&indio_dev->mlock);
+	chip->config &= ~AD7746_CONF_CAPFS_MASK;
+	chip->config |= i << AD7746_CONF_CAPFS_SHIFT;
 
-	return len;
+	return 0;
 }
 
-static ssize_t ad7746_show_vt_filter_rate_setup(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+static int ad7746_store_vt_filter_rate_setup(struct ad7746_chip_info *chip,
+					     int val)
 {
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ad7746_chip_info *chip = iio_priv(indio_dev);
-
-	return sprintf(buf, "%d\n", ad7746_vt_filter_rate_table[
-			(chip->config >> 6) & 0x3][0]);
-}
-
-static ssize_t ad7746_store_vt_filter_rate_setup(struct device *dev,
-		struct device_attribute *attr,
-		const char *buf,
-		size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct ad7746_chip_info *chip = iio_priv(indio_dev);
-	u8 data;
-	int ret, i;
-
-	ret = kstrtou8(buf, 10, &data);
-	if (ret < 0)
-		return ret;
+	int i;
 
 	for (i = 0; i < ARRAY_SIZE(ad7746_vt_filter_rate_table); i++)
-		if (data >= ad7746_vt_filter_rate_table[i][0])
+		if (val >= ad7746_vt_filter_rate_table[i][0])
 			break;
 
 	if (i >= ARRAY_SIZE(ad7746_vt_filter_rate_table))
 		i = ARRAY_SIZE(ad7746_vt_filter_rate_table) - 1;
 
-	mutex_lock(&indio_dev->mlock);
-	chip->config &= ~AD7746_CONF_VTFS(0x3);
-	chip->config |= AD7746_CONF_VTFS(i);
-	mutex_unlock(&indio_dev->mlock);
+	chip->config &= ~AD7746_CONF_VTFS_MASK;
+	chip->config |= i << AD7746_CONF_VTFS_SHIFT;
 
-	return len;
+	return 0;
 }
 
-static IIO_DEVICE_ATTR(in_capacitance_sampling_frequency,
-		       S_IRUGO | S_IWUSR, ad7746_show_cap_filter_rate_setup,
-			ad7746_store_cap_filter_rate_setup, 0);
-
-static IIO_DEVICE_ATTR(in_voltage_sampling_frequency,
-		       S_IRUGO | S_IWUSR, ad7746_show_vt_filter_rate_setup,
-		       ad7746_store_vt_filter_rate_setup, 0);
-
 static IIO_CONST_ATTR(in_voltage_sampling_frequency_available, "50 31 16 8");
 static IIO_CONST_ATTR(in_capacitance_sampling_frequency_available,
 		       "91 84 50 26 16 13 11 9");
 
 static struct attribute *ad7746_attributes[] = {
-	&iio_dev_attr_in_capacitance_sampling_frequency.dev_attr.attr,
-	&iio_dev_attr_in_voltage_sampling_frequency.dev_attr.attr,
 	&iio_dev_attr_in_capacitance0_calibbias_calibration.dev_attr.attr,
 	&iio_dev_attr_in_capacitance0_calibscale_calibration.dev_attr.attr,
 	&iio_dev_attr_in_capacitance1_calibscale_calibration.dev_attr.attr,
@@ -547,6 +499,23 @@ static int ad7746_write_raw(struct iio_dev *indio_dev,
 
 		ret = 0;
 		break;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (val2) {
+			ret = -EINVAL;
+			goto out;
+		}
+
+		switch (chan->type) {
+		case IIO_CAPACITANCE:
+			ret = ad7746_store_cap_filter_rate_setup(chip, val);
+			break;
+		case IIO_VOLTAGE:
+			ret = ad7746_store_vt_filter_rate_setup(chip, val);
+			break;
+		default:
+			ret = -EINVAL;
+		}
+		break;
 	default:
 		ret = -EINVAL;
 	}
@@ -562,7 +531,7 @@ static int ad7746_read_raw(struct iio_dev *indio_dev,
 			   long mask)
 {
 	struct ad7746_chip_info *chip = iio_priv(indio_dev);
-	int ret, delay;
+	int ret, delay, idx;
 	u8 regval, reg;
 
 	mutex_lock(&indio_dev->mlock);
@@ -667,6 +636,24 @@ static int ad7746_read_raw(struct iio_dev *indio_dev,
 		}
 
 		break;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		switch (chan->type) {
+		case IIO_CAPACITANCE:
+			idx = (chip->config & AD7746_CONF_CAPFS_MASK) >>
+				AD7746_CONF_CAPFS_SHIFT;
+			*val = ad7746_cap_filter_rate_table[idx][0];
+			ret = IIO_VAL_INT;
+			break;
+		case IIO_VOLTAGE:
+			idx = (chip->config & AD7746_CONF_VTFS_MASK) >>
+				AD7746_CONF_VTFS_SHIFT;
+			*val = ad7746_vt_filter_rate_table[idx][0];
+			ret = IIO_VAL_INT;
+			break;
+		default:
+			ret = -EINVAL;
+		}
+		break;
 	default:
 		ret = -EINVAL;
 	}
diff --git a/drivers/staging/iio/frequency/ad9832.c b/drivers/staging/iio/frequency/ad9832.c
index 358400b22d33..a5b2f068168d 100644
--- a/drivers/staging/iio/frequency/ad9832.c
+++ b/drivers/staging/iio/frequency/ad9832.c
@@ -204,7 +204,6 @@ static int ad9832_probe(struct spi_device *spi)
 	struct ad9832_platform_data *pdata = dev_get_platdata(&spi->dev);
 	struct iio_dev *indio_dev;
 	struct ad9832_state *st;
-	struct regulator *reg;
 	int ret;
 
 	if (!pdata) {
@@ -212,21 +211,35 @@ static int ad9832_probe(struct spi_device *spi)
 		return -ENODEV;
 	}
 
-	reg = devm_regulator_get(&spi->dev, "vcc");
-	if (!IS_ERR(reg)) {
-		ret = regulator_enable(reg);
-		if (ret)
-			return ret;
-	}
-
 	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
-	if (!indio_dev) {
-		ret = -ENOMEM;
-		goto error_disable_reg;
-	}
+	if (!indio_dev)
+		return -ENOMEM;
+
 	spi_set_drvdata(spi, indio_dev);
 	st = iio_priv(indio_dev);
-	st->reg = reg;
+
+	st->avdd = devm_regulator_get(&spi->dev, "avdd");
+	if (IS_ERR(st->avdd))
+		return PTR_ERR(st->avdd);
+
+	ret = regulator_enable(st->avdd);
+	if (ret) {
+		dev_err(&spi->dev, "Failed to enable specified AVDD supply\n");
+		return ret;
+	}
+
+	st->dvdd = devm_regulator_get(&spi->dev, "dvdd");
+	if (IS_ERR(st->dvdd)) {
+		ret = PTR_ERR(st->dvdd);
+		goto error_disable_avdd;
+	}
+
+	ret = regulator_enable(st->dvdd);
+	if (ret) {
+		dev_err(&spi->dev, "Failed to enable specified DVDD supply\n");
+		goto error_disable_avdd;
+	}
+
 	st->mclk = pdata->mclk;
 	st->spi = spi;
 
@@ -277,42 +290,43 @@ static int ad9832_probe(struct spi_device *spi)
 	ret = spi_sync(st->spi, &st->msg);
 	if (ret) {
 		dev_err(&spi->dev, "device init failed\n");
-		goto error_disable_reg;
+		goto error_disable_dvdd;
 	}
 
 	ret = ad9832_write_frequency(st, AD9832_FREQ0HM, pdata->freq0);
 	if (ret)
-		goto error_disable_reg;
+		goto error_disable_dvdd;
 
 	ret = ad9832_write_frequency(st, AD9832_FREQ1HM, pdata->freq1);
 	if (ret)
-		goto error_disable_reg;
+		goto error_disable_dvdd;
 
 	ret = ad9832_write_phase(st, AD9832_PHASE0H, pdata->phase0);
 	if (ret)
-		goto error_disable_reg;
+		goto error_disable_dvdd;
 
 	ret = ad9832_write_phase(st, AD9832_PHASE1H, pdata->phase1);
 	if (ret)
-		goto error_disable_reg;
+		goto error_disable_dvdd;
 
 	ret = ad9832_write_phase(st, AD9832_PHASE2H, pdata->phase2);
 	if (ret)
-		goto error_disable_reg;
+		goto error_disable_dvdd;
 
 	ret = ad9832_write_phase(st, AD9832_PHASE3H, pdata->phase3);
 	if (ret)
-		goto error_disable_reg;
+		goto error_disable_dvdd;
 
 	ret = iio_device_register(indio_dev);
 	if (ret)
-		goto error_disable_reg;
+		goto error_disable_dvdd;
 
 	return 0;
 
-error_disable_reg:
-	if (!IS_ERR(reg))
-		regulator_disable(reg);
+error_disable_dvdd:
+	regulator_disable(st->dvdd);
+error_disable_avdd:
+	regulator_disable(st->avdd);
 
 	return ret;
 }
@@ -323,8 +337,8 @@ static int ad9832_remove(struct spi_device *spi)
 	struct ad9832_state *st = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
-	if (!IS_ERR(st->reg))
-		regulator_disable(st->reg);
+	regulator_disable(st->dvdd);
+	regulator_disable(st->avdd);
 
 	return 0;
 }
diff --git a/drivers/staging/iio/frequency/ad9832.h b/drivers/staging/iio/frequency/ad9832.h
index d32323b46be6..1b08b04482a4 100644
--- a/drivers/staging/iio/frequency/ad9832.h
+++ b/drivers/staging/iio/frequency/ad9832.h
@@ -58,7 +58,8 @@
 /**
  * struct ad9832_state - driver instance specific data
  * @spi:		spi_device
- * @reg:		supply regulator
+ * @avdd:		supply regulator for the analog section
+ * @dvdd:		supply regulator for the digital section
  * @mclk:		external master clock
  * @ctrl_fp:		cached frequency/phase control word
  * @ctrl_ss:		cached sync/selsrc control word
@@ -76,7 +77,8 @@
 
 struct ad9832_state {
 	struct spi_device		*spi;
-	struct regulator		*reg;
+	struct regulator		*avdd;
+	struct regulator		*dvdd;
 	unsigned long			mclk;
 	unsigned short			ctrl_fp;
 	unsigned short			ctrl_ss;
diff --git a/drivers/staging/iio/frequency/ad9834.c b/drivers/staging/iio/frequency/ad9834.c
index 6366216e4f37..19216af1dfc9 100644
--- a/drivers/staging/iio/frequency/ad9834.c
+++ b/drivers/staging/iio/frequency/ad9834.c
@@ -329,11 +329,14 @@ static int ad9834_probe(struct spi_device *spi)
 		return -ENODEV;
 	}
 
-	reg = devm_regulator_get(&spi->dev, "vcc");
-	if (!IS_ERR(reg)) {
-		ret = regulator_enable(reg);
-		if (ret)
-			return ret;
+	reg = devm_regulator_get(&spi->dev, "avdd");
+	if (IS_ERR(reg))
+		return PTR_ERR(reg);
+
+	ret = regulator_enable(reg);
+	if (ret) {
+		dev_err(&spi->dev, "Failed to enable specified AVDD supply\n");
+		return ret;
 	}
 
 	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
@@ -416,8 +419,7 @@ static int ad9834_probe(struct spi_device *spi)
 	return 0;
 
 error_disable_reg:
-	if (!IS_ERR(reg))
-		regulator_disable(reg);
+	regulator_disable(reg);
 
 	return ret;
 }
@@ -428,8 +430,7 @@ static int ad9834_remove(struct spi_device *spi)
 	struct ad9834_state *st = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
-	if (!IS_ERR(st->reg))
-		regulator_disable(st->reg);
+	regulator_disable(st->reg);
 
 	return 0;
 }
diff --git a/drivers/staging/iio/impedance-analyzer/ad5933.c b/drivers/staging/iio/impedance-analyzer/ad5933.c
index 3892a7470410..944789843938 100644
--- a/drivers/staging/iio/impedance-analyzer/ad5933.c
+++ b/drivers/staging/iio/impedance-analyzer/ad5933.c
@@ -726,13 +726,16 @@ static int ad5933_probe(struct i2c_client *client,
 	if (!pdata)
 		pdata = &ad5933_default_pdata;
 
-	st->reg = devm_regulator_get(&client->dev, "vcc");
-	if (!IS_ERR(st->reg)) {
-		ret = regulator_enable(st->reg);
-		if (ret)
-			return ret;
-		voltage_uv = regulator_get_voltage(st->reg);
+	st->reg = devm_regulator_get(&client->dev, "vdd");
+	if (IS_ERR(st->reg))
+		return PTR_ERR(st->reg);
+
+	ret = regulator_enable(st->reg);
+	if (ret) {
+		dev_err(&client->dev, "Failed to enable specified VDD supply\n");
+		return ret;
 	}
+	voltage_uv = regulator_get_voltage(st->reg);
 
 	if (voltage_uv)
 		st->vref_mv = voltage_uv / 1000;
@@ -775,8 +778,7 @@ static int ad5933_probe(struct i2c_client *client,
 error_unreg_ring:
 	iio_kfifo_free(indio_dev->buffer);
 error_disable_reg:
-	if (!IS_ERR(st->reg))
-		regulator_disable(st->reg);
+	regulator_disable(st->reg);
 
 	return ret;
 }
@@ -788,8 +790,7 @@ static int ad5933_remove(struct i2c_client *client)
 
 	iio_device_unregister(indio_dev);
 	iio_kfifo_free(indio_dev->buffer);
-	if (!IS_ERR(st->reg))
-		regulator_disable(st->reg);
+	regulator_disable(st->reg);
 
 	return 0;
 }
diff --git a/drivers/staging/iio/light/Kconfig b/drivers/staging/iio/light/Kconfig
index ca8d6e66c899..4fbf6298c0f3 100644
--- a/drivers/staging/iio/light/Kconfig
+++ b/drivers/staging/iio/light/Kconfig
@@ -3,18 +3,6 @@
 #
 menu "Light sensors"
 
-config SENSORS_ISL29018
-	tristate "ISL 29018 light and proximity sensor"
-	depends on I2C
-	select REGMAP_I2C
-	default n
-	help
-	 If you say yes here you get support for ambient light sensing and
-	 proximity infrared sensing from Intersil ISL29018.
-	 This driver will provide the measurements of ambient light intensity
-	 in lux, proximity infrared sensing and normal infrared sensing.
-	 Data from sensor is accessible via sysfs.
-
 config SENSORS_ISL29028
 	tristate "Intersil ISL29028 Concurrent Light and Proximity Sensor"
 	depends on I2C
@@ -25,13 +13,6 @@ config SENSORS_ISL29028
 	 Proximity value via iio. The ISL29028 provides the concurrent sensing
 	 of ambient light and proximity.
 
-config TSL2583
-	tristate "TAOS TSL2580, TSL2581 and TSL2583 light-to-digital converters"
-	depends on I2C
-	help
-	 Provides support for the TAOS tsl2580, tsl2581 and tsl2583 devices.
-	 Access ALS data via iio, sysfs.
-
 config TSL2x7x
 	tristate "TAOS TSL/TMD2x71 and TSL/TMD2x72 Family of light and proximity sensors"
 	depends on I2C
diff --git a/drivers/staging/iio/light/Makefile b/drivers/staging/iio/light/Makefile
index 9960fdf7c15b..f8693e9fdc94 100644
--- a/drivers/staging/iio/light/Makefile
+++ b/drivers/staging/iio/light/Makefile
@@ -2,7 +2,5 @@
 # Makefile for industrial I/O Light sensors
 #
 
-obj-$(CONFIG_SENSORS_ISL29018)	+= isl29018.o
 obj-$(CONFIG_SENSORS_ISL29028)	+= isl29028.o
-obj-$(CONFIG_TSL2583)	+= tsl2583.o
 obj-$(CONFIG_TSL2x7x)	+= tsl2x7x_core.o
diff --git a/drivers/staging/iio/light/tsl2583.c b/drivers/staging/iio/light/tsl2583.c
deleted file mode 100644
index 08f1583ee34e..000000000000
--- a/drivers/staging/iio/light/tsl2583.c
+++ /dev/null
@@ -1,963 +0,0 @@
-/*
- * Device driver for monitoring ambient light intensity (lux)
- * within the TAOS tsl258x family of devices (tsl2580, tsl2581).
- *
- * Copyright (c) 2011, TAOS Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA	02110-1301, USA.
- */
-
-#include <linux/kernel.h>
-#include <linux/i2c.h>
-#include <linux/errno.h>
-#include <linux/delay.h>
-#include <linux/string.h>
-#include <linux/mutex.h>
-#include <linux/unistd.h>
-#include <linux/slab.h>
-#include <linux/module.h>
-#include <linux/iio/iio.h>
-
-#define TSL258X_MAX_DEVICE_REGS		32
-
-/* Triton register offsets */
-#define	TSL258X_REG_MAX		8
-
-/* Device Registers and Masks */
-#define TSL258X_CNTRL			0x00
-#define TSL258X_ALS_TIME		0X01
-#define TSL258X_INTERRUPT		0x02
-#define TSL258X_GAIN			0x07
-#define TSL258X_REVID			0x11
-#define TSL258X_CHIPID			0x12
-#define TSL258X_ALS_CHAN0LO		0x14
-#define TSL258X_ALS_CHAN0HI		0x15
-#define TSL258X_ALS_CHAN1LO		0x16
-#define TSL258X_ALS_CHAN1HI		0x17
-#define TSL258X_TMR_LO			0x18
-#define TSL258X_TMR_HI			0x19
-
-/* tsl2583 cmd reg masks */
-#define TSL258X_CMD_REG			0x80
-#define TSL258X_CMD_SPL_FN		0x60
-#define TSL258X_CMD_ALS_INT_CLR	0X01
-
-/* tsl2583 cntrl reg masks */
-#define TSL258X_CNTL_ADC_ENBL	0x02
-#define TSL258X_CNTL_PWR_ON		0x01
-
-/* tsl2583 status reg masks */
-#define TSL258X_STA_ADC_VALID	0x01
-#define TSL258X_STA_ADC_INTR	0x10
-
-/* Lux calculation constants */
-#define	TSL258X_LUX_CALC_OVER_FLOW		65535
-
-enum {
-	TSL258X_CHIP_UNKNOWN = 0,
-	TSL258X_CHIP_WORKING = 1,
-	TSL258X_CHIP_SUSPENDED = 2
-};
-
-/* Per-device data */
-struct taos_als_info {
-	u16 als_ch0;
-	u16 als_ch1;
-	u16 lux;
-};
-
-struct taos_settings {
-	int als_time;
-	int als_gain;
-	int als_gain_trim;
-	int als_cal_target;
-};
-
-struct tsl2583_chip {
-	struct mutex als_mutex;
-	struct i2c_client *client;
-	struct taos_als_info als_cur_info;
-	struct taos_settings taos_settings;
-	int als_time_scale;
-	int als_saturation;
-	int taos_chip_status;
-	u8 taos_config[8];
-};
-
-/*
- * Initial values for device - this values can/will be changed by driver.
- * and applications as needed.
- * These values are dynamic.
- */
-static const u8 taos_config[8] = {
-		0x00, 0xee, 0x00, 0x03, 0x00, 0xFF, 0xFF, 0x00
-}; /*	cntrl atime intC  Athl0 Athl1 Athh0 Athh1 gain */
-
-struct taos_lux {
-	unsigned int ratio;
-	unsigned int ch0;
-	unsigned int ch1;
-};
-
-/* This structure is intentionally large to accommodate updates via sysfs. */
-/* Sized to 11 = max 10 segments + 1 termination segment */
-/* Assumption is one and only one type of glass used  */
-static struct taos_lux taos_device_lux[11] = {
-	{  9830,  8520, 15729 },
-	{ 12452, 10807, 23344 },
-	{ 14746,  6383, 11705 },
-	{ 17695,  4063,  6554 },
-};
-
-struct gainadj {
-	s16 ch0;
-	s16 ch1;
-};
-
-/* Index = (0 - 3) Used to validate the gain selection index */
-static const struct gainadj gainadj[] = {
-	{ 1, 1 },
-	{ 8, 8 },
-	{ 16, 16 },
-	{ 107, 115 }
-};
-
-/*
- * Provides initial operational parameter defaults.
- * These defaults may be changed through the device's sysfs files.
- */
-static void taos_defaults(struct tsl2583_chip *chip)
-{
-	/* Operational parameters */
-	chip->taos_settings.als_time = 100;
-	/* must be a multiple of 50mS */
-	chip->taos_settings.als_gain = 0;
-	/* this is actually an index into the gain table */
-	/* assume clear glass as default */
-	chip->taos_settings.als_gain_trim = 1000;
-	/* default gain trim to account for aperture effects */
-	chip->taos_settings.als_cal_target = 130;
-	/* Known external ALS reading used for calibration */
-}
-
-/*
- * Read a number of bytes starting at register (reg) location.
- * Return 0, or i2c_smbus_write_byte ERROR code.
- */
-static int
-taos_i2c_read(struct i2c_client *client, u8 reg, u8 *val, unsigned int len)
-{
-	int i, ret;
-
-	for (i = 0; i < len; i++) {
-		/* select register to write */
-		ret = i2c_smbus_write_byte(client, (TSL258X_CMD_REG | reg));
-		if (ret < 0) {
-			dev_err(&client->dev,
-				"taos_i2c_read failed to write register %x\n",
-				reg);
-			return ret;
-		}
-		/* read the data */
-		*val = i2c_smbus_read_byte(client);
-		val++;
-		reg++;
-	}
-	return 0;
-}
-
-/*
- * Reads and calculates current lux value.
- * The raw ch0 and ch1 values of the ambient light sensed in the last
- * integration cycle are read from the device.
- * Time scale factor array values are adjusted based on the integration time.
- * The raw values are multiplied by a scale factor, and device gain is obtained
- * using gain index. Limit checks are done next, then the ratio of a multiple
- * of ch1 value, to the ch0 value, is calculated. The array taos_device_lux[]
- * declared above is then scanned to find the first ratio value that is just
- * above the ratio we just calculated. The ch0 and ch1 multiplier constants in
- * the array are then used along with the time scale factor array values, to
- * calculate the lux.
- */
-static int taos_get_lux(struct iio_dev *indio_dev)
-{
-	u16 ch0, ch1; /* separated ch0/ch1 data from device */
-	u32 lux; /* raw lux calculated from device data */
-	u64 lux64;
-	u32 ratio;
-	u8 buf[5];
-	struct taos_lux *p;
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-	int i, ret;
-	u32 ch0lux = 0;
-	u32 ch1lux = 0;
-
-	if (mutex_trylock(&chip->als_mutex) == 0) {
-		dev_info(&chip->client->dev, "taos_get_lux device is busy\n");
-		return chip->als_cur_info.lux; /* busy, so return LAST VALUE */
-	}
-
-	if (chip->taos_chip_status != TSL258X_CHIP_WORKING) {
-		/* device is not enabled */
-		dev_err(&chip->client->dev, "taos_get_lux device is not enabled\n");
-		ret = -EBUSY;
-		goto out_unlock;
-	}
-
-	ret = taos_i2c_read(chip->client, (TSL258X_CMD_REG), &buf[0], 1);
-	if (ret < 0) {
-		dev_err(&chip->client->dev, "taos_get_lux failed to read CMD_REG\n");
-		goto out_unlock;
-	}
-	/* is data new & valid */
-	if (!(buf[0] & TSL258X_STA_ADC_INTR)) {
-		dev_err(&chip->client->dev, "taos_get_lux data not valid\n");
-		ret = chip->als_cur_info.lux; /* return LAST VALUE */
-		goto out_unlock;
-	}
-
-	for (i = 0; i < 4; i++) {
-		int reg = TSL258X_CMD_REG | (TSL258X_ALS_CHAN0LO + i);
-
-		ret = taos_i2c_read(chip->client, reg, &buf[i], 1);
-		if (ret < 0) {
-			dev_err(&chip->client->dev,
-				"taos_get_lux failed to read register %x\n",
-				reg);
-			goto out_unlock;
-		}
-	}
-
-	/*
-	 * clear status, really interrupt status (interrupts are off), but
-	 * we use the bit anyway - don't forget 0x80 - this is a command
-	 */
-	ret = i2c_smbus_write_byte(chip->client,
-				   (TSL258X_CMD_REG | TSL258X_CMD_SPL_FN |
-				    TSL258X_CMD_ALS_INT_CLR));
-
-	if (ret < 0) {
-		dev_err(&chip->client->dev,
-			"taos_i2c_write_command failed in taos_get_lux, err = %d\n",
-			ret);
-		goto out_unlock; /* have no data, so return failure */
-	}
-
-	/* extract ALS/lux data */
-	ch0 = le16_to_cpup((const __le16 *)&buf[0]);
-	ch1 = le16_to_cpup((const __le16 *)&buf[2]);
-
-	chip->als_cur_info.als_ch0 = ch0;
-	chip->als_cur_info.als_ch1 = ch1;
-
-	if ((ch0 >= chip->als_saturation) || (ch1 >= chip->als_saturation))
-		goto return_max;
-
-	if (!ch0) {
-		/* have no data, so return LAST VALUE */
-		ret = 0;
-		chip->als_cur_info.lux = 0;
-		goto out_unlock;
-	}
-	/* calculate ratio */
-	ratio = (ch1 << 15) / ch0;
-	/* convert to unscaled lux using the pointer to the table */
-	for (p = (struct taos_lux *)taos_device_lux;
-	     p->ratio != 0 && p->ratio < ratio; p++)
-		;
-
-	if (p->ratio == 0) {
-		lux = 0;
-	} else {
-		ch0lux = ((ch0 * p->ch0) +
-			  (gainadj[chip->taos_settings.als_gain].ch0 >> 1))
-			 / gainadj[chip->taos_settings.als_gain].ch0;
-		ch1lux = ((ch1 * p->ch1) +
-			  (gainadj[chip->taos_settings.als_gain].ch1 >> 1))
-			 / gainadj[chip->taos_settings.als_gain].ch1;
-		lux = ch0lux - ch1lux;
-	}
-
-	/* note: lux is 31 bit max at this point */
-	if (ch1lux > ch0lux) {
-		dev_dbg(&chip->client->dev, "No Data - Return last value\n");
-		ret = 0;
-		chip->als_cur_info.lux = 0;
-		goto out_unlock;
-	}
-
-	/* adjust for active time scale */
-	if (chip->als_time_scale == 0)
-		lux = 0;
-	else
-		lux = (lux + (chip->als_time_scale >> 1)) /
-			chip->als_time_scale;
-
-	/* Adjust for active gain scale.
-	 * The taos_device_lux tables above have a factor of 8192 built in,
-	 * so we need to shift right.
-	 * User-specified gain provides a multiplier.
-	 * Apply user-specified gain before shifting right to retain precision.
-	 * Use 64 bits to avoid overflow on multiplication.
-	 * Then go back to 32 bits before division to avoid using div_u64().
-	 */
-	lux64 = lux;
-	lux64 = lux64 * chip->taos_settings.als_gain_trim;
-	lux64 >>= 13;
-	lux = lux64;
-	lux = (lux + 500) / 1000;
-	if (lux > TSL258X_LUX_CALC_OVER_FLOW) { /* check for overflow */
-return_max:
-		lux = TSL258X_LUX_CALC_OVER_FLOW;
-	}
-
-	/* Update the structure with the latest VALID lux. */
-	chip->als_cur_info.lux = lux;
-	ret = lux;
-
-out_unlock:
-	mutex_unlock(&chip->als_mutex);
-	return ret;
-}
-
-/*
- * Obtain single reading and calculate the als_gain_trim (later used
- * to derive actual lux).
- * Return updated gain_trim value.
- */
-static int taos_als_calibrate(struct iio_dev *indio_dev)
-{
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-	u8 reg_val;
-	unsigned int gain_trim_val;
-	int ret;
-	int lux_val;
-
-	ret = i2c_smbus_write_byte(chip->client,
-				   (TSL258X_CMD_REG | TSL258X_CNTRL));
-	if (ret < 0) {
-		dev_err(&chip->client->dev,
-			"taos_als_calibrate failed to reach the CNTRL register, ret=%d\n",
-			ret);
-		return ret;
-	}
-
-	reg_val = i2c_smbus_read_byte(chip->client);
-	if ((reg_val & (TSL258X_CNTL_ADC_ENBL | TSL258X_CNTL_PWR_ON))
-			!= (TSL258X_CNTL_ADC_ENBL | TSL258X_CNTL_PWR_ON)) {
-		dev_err(&chip->client->dev,
-			"taos_als_calibrate failed: device not powered on with ADC enabled\n");
-		return -1;
-	}
-
-	ret = i2c_smbus_write_byte(chip->client,
-				   (TSL258X_CMD_REG | TSL258X_CNTRL));
-	if (ret < 0) {
-		dev_err(&chip->client->dev,
-			"taos_als_calibrate failed to reach the STATUS register, ret=%d\n",
-			ret);
-		return ret;
-	}
-	reg_val = i2c_smbus_read_byte(chip->client);
-
-	if ((reg_val & TSL258X_STA_ADC_VALID) != TSL258X_STA_ADC_VALID) {
-		dev_err(&chip->client->dev,
-			"taos_als_calibrate failed: STATUS - ADC not valid.\n");
-		return -ENODATA;
-	}
-	lux_val = taos_get_lux(indio_dev);
-	if (lux_val < 0) {
-		dev_err(&chip->client->dev, "taos_als_calibrate failed to get lux\n");
-		return lux_val;
-	}
-	gain_trim_val = (unsigned int)(((chip->taos_settings.als_cal_target)
-			* chip->taos_settings.als_gain_trim) / lux_val);
-
-	if ((gain_trim_val < 250) || (gain_trim_val > 4000)) {
-		dev_err(&chip->client->dev,
-			"taos_als_calibrate failed: trim_val of %d is out of range\n",
-			gain_trim_val);
-		return -ENODATA;
-	}
-	chip->taos_settings.als_gain_trim = (int)gain_trim_val;
-
-	return (int)gain_trim_val;
-}
-
-/*
- * Turn the device on.
- * Configuration must be set before calling this function.
- */
-static int taos_chip_on(struct iio_dev *indio_dev)
-{
-	int i;
-	int ret;
-	u8 *uP;
-	u8 utmp;
-	int als_count;
-	int als_time;
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-
-	/* and make sure we're not already on */
-	if (chip->taos_chip_status == TSL258X_CHIP_WORKING) {
-		/* if forcing a register update - turn off, then on */
-		dev_info(&chip->client->dev, "device is already enabled\n");
-		return -EINVAL;
-	}
-
-	/* determine als integration register */
-	als_count = (chip->taos_settings.als_time * 100 + 135) / 270;
-	if (!als_count)
-		als_count = 1; /* ensure at least one cycle */
-
-	/* convert back to time (encompasses overrides) */
-	als_time = (als_count * 27 + 5) / 10;
-	chip->taos_config[TSL258X_ALS_TIME] = 256 - als_count;
-
-	/* Set the gain based on taos_settings struct */
-	chip->taos_config[TSL258X_GAIN] = chip->taos_settings.als_gain;
-
-	/* set chip struct re scaling and saturation */
-	chip->als_saturation = als_count * 922; /* 90% of full scale */
-	chip->als_time_scale = (als_time + 25) / 50;
-
-	/*
-	 * TSL258x Specific power-on / adc enable sequence
-	 * Power on the device 1st.
-	 */
-	utmp = TSL258X_CNTL_PWR_ON;
-	ret = i2c_smbus_write_byte_data(chip->client,
-					TSL258X_CMD_REG | TSL258X_CNTRL, utmp);
-	if (ret < 0) {
-		dev_err(&chip->client->dev, "taos_chip_on failed on CNTRL reg.\n");
-		return ret;
-	}
-
-	/*
-	 * Use the following shadow copy for our delay before enabling ADC.
-	 * Write all the registers.
-	 */
-	for (i = 0, uP = chip->taos_config; i < TSL258X_REG_MAX; i++) {
-		ret = i2c_smbus_write_byte_data(chip->client,
-						TSL258X_CMD_REG + i,
-						*uP++);
-		if (ret < 0) {
-			dev_err(&chip->client->dev,
-				"taos_chip_on failed on reg %d.\n", i);
-			return ret;
-		}
-	}
-
-	usleep_range(3000, 3500);
-	/*
-	 * NOW enable the ADC
-	 * initialize the desired mode of operation
-	 */
-	utmp = TSL258X_CNTL_PWR_ON | TSL258X_CNTL_ADC_ENBL;
-	ret = i2c_smbus_write_byte_data(chip->client,
-					TSL258X_CMD_REG | TSL258X_CNTRL,
-					utmp);
-	if (ret < 0) {
-		dev_err(&chip->client->dev, "taos_chip_on failed on 2nd CTRL reg.\n");
-		return ret;
-	}
-	chip->taos_chip_status = TSL258X_CHIP_WORKING;
-
-	return ret;
-}
-
-static int taos_chip_off(struct iio_dev *indio_dev)
-{
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-
-	/* turn device off */
-	chip->taos_chip_status = TSL258X_CHIP_SUSPENDED;
-	return i2c_smbus_write_byte_data(chip->client,
-					TSL258X_CMD_REG | TSL258X_CNTRL,
-					0x00);
-}
-
-/* Sysfs Interface Functions */
-
-static ssize_t taos_power_state_show(struct device *dev,
-				     struct device_attribute *attr, char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-
-	return sprintf(buf, "%d\n", chip->taos_chip_status);
-}
-
-static ssize_t taos_power_state_store(struct device *dev,
-				      struct device_attribute *attr,
-				      const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	int value;
-
-	if (kstrtoint(buf, 0, &value))
-		return -EINVAL;
-
-	if (!value)
-		taos_chip_off(indio_dev);
-	else
-		taos_chip_on(indio_dev);
-
-	return len;
-}
-
-static ssize_t taos_gain_show(struct device *dev,
-			      struct device_attribute *attr, char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-	char gain[4] = {0};
-
-	switch (chip->taos_settings.als_gain) {
-	case 0:
-		strcpy(gain, "001");
-		break;
-	case 1:
-		strcpy(gain, "008");
-		break;
-	case 2:
-		strcpy(gain, "016");
-		break;
-	case 3:
-		strcpy(gain, "111");
-		break;
-	}
-
-	return sprintf(buf, "%s\n", gain);
-}
-
-static ssize_t taos_gain_store(struct device *dev,
-			       struct device_attribute *attr,
-			       const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-	int value;
-
-	if (kstrtoint(buf, 0, &value))
-		return -EINVAL;
-
-	switch (value) {
-	case 1:
-		chip->taos_settings.als_gain = 0;
-		break;
-	case 8:
-		chip->taos_settings.als_gain = 1;
-		break;
-	case 16:
-		chip->taos_settings.als_gain = 2;
-		break;
-	case 111:
-		chip->taos_settings.als_gain = 3;
-		break;
-	default:
-		dev_err(dev, "Invalid Gain Index (must be 1,8,16,111)\n");
-		return -1;
-	}
-
-	return len;
-}
-
-static ssize_t taos_gain_available_show(struct device *dev,
-					struct device_attribute *attr,
-					char *buf)
-{
-	return sprintf(buf, "%s\n", "1 8 16 111");
-}
-
-static ssize_t taos_als_time_show(struct device *dev,
-				  struct device_attribute *attr, char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-
-	return sprintf(buf, "%d\n", chip->taos_settings.als_time);
-}
-
-static ssize_t taos_als_time_store(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-	int value;
-
-	if (kstrtoint(buf, 0, &value))
-		return -EINVAL;
-
-	if ((value < 50) || (value > 650))
-		return -EINVAL;
-
-	if (value % 50)
-		return -EINVAL;
-
-	chip->taos_settings.als_time = value;
-
-	return len;
-}
-
-static ssize_t taos_als_time_available_show(struct device *dev,
-					    struct device_attribute *attr,
-					    char *buf)
-{
-	return sprintf(buf, "%s\n",
-		"50 100 150 200 250 300 350 400 450 500 550 600 650");
-}
-
-static ssize_t taos_als_trim_show(struct device *dev,
-				  struct device_attribute *attr, char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-
-	return sprintf(buf, "%d\n", chip->taos_settings.als_gain_trim);
-}
-
-static ssize_t taos_als_trim_store(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-	int value;
-
-	if (kstrtoint(buf, 0, &value))
-		return -EINVAL;
-
-	if (value)
-		chip->taos_settings.als_gain_trim = value;
-
-	return len;
-}
-
-static ssize_t taos_als_cal_target_show(struct device *dev,
-					struct device_attribute *attr,
-					char *buf)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-
-	return sprintf(buf, "%d\n", chip->taos_settings.als_cal_target);
-}
-
-static ssize_t taos_als_cal_target_store(struct device *dev,
-					 struct device_attribute *attr,
-					 const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-	int value;
-
-	if (kstrtoint(buf, 0, &value))
-		return -EINVAL;
-
-	if (value)
-		chip->taos_settings.als_cal_target = value;
-
-	return len;
-}
-
-static ssize_t taos_lux_show(struct device *dev, struct device_attribute *attr,
-			     char *buf)
-{
-	int ret;
-
-	ret = taos_get_lux(dev_to_iio_dev(dev));
-	if (ret < 0)
-		return ret;
-
-	return sprintf(buf, "%d\n", ret);
-}
-
-static ssize_t taos_do_calibrate(struct device *dev,
-				 struct device_attribute *attr,
-				 const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	int value;
-
-	if (kstrtoint(buf, 0, &value))
-		return -EINVAL;
-
-	if (value == 1)
-		taos_als_calibrate(indio_dev);
-
-	return len;
-}
-
-static ssize_t taos_luxtable_show(struct device *dev,
-				  struct device_attribute *attr, char *buf)
-{
-	int i;
-	int offset = 0;
-
-	for (i = 0; i < ARRAY_SIZE(taos_device_lux); i++) {
-		offset += sprintf(buf + offset, "%u,%u,%u,",
-				  taos_device_lux[i].ratio,
-				  taos_device_lux[i].ch0,
-				  taos_device_lux[i].ch1);
-		if (taos_device_lux[i].ratio == 0) {
-			/*
-			 * We just printed the first "0" entry.
-			 * Now get rid of the extra "," and break.
-			 */
-			offset--;
-			break;
-		}
-	}
-
-	offset += sprintf(buf + offset, "\n");
-	return offset;
-}
-
-static ssize_t taos_luxtable_store(struct device *dev,
-				   struct device_attribute *attr,
-				   const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-	int value[ARRAY_SIZE(taos_device_lux) * 3 + 1];
-	int n;
-
-	get_options(buf, ARRAY_SIZE(value), value);
-
-	/* We now have an array of ints starting at value[1], and
-	 * enumerated by value[0].
-	 * We expect each group of three ints is one table entry,
-	 * and the last table entry is all 0.
-	 */
-	n = value[0];
-	if ((n % 3) || n < 6 || n > ((ARRAY_SIZE(taos_device_lux) - 1) * 3)) {
-		dev_info(dev, "LUX TABLE INPUT ERROR 1 Value[0]=%d\n", n);
-		return -EINVAL;
-	}
-	if ((value[(n - 2)] | value[(n - 1)] | value[n]) != 0) {
-		dev_info(dev, "LUX TABLE INPUT ERROR 2 Value[0]=%d\n", n);
-		return -EINVAL;
-	}
-
-	if (chip->taos_chip_status == TSL258X_CHIP_WORKING)
-		taos_chip_off(indio_dev);
-
-	/* Zero out the table */
-	memset(taos_device_lux, 0, sizeof(taos_device_lux));
-	memcpy(taos_device_lux, &value[1], (value[0] * 4));
-
-	taos_chip_on(indio_dev);
-
-	return len;
-}
-
-static DEVICE_ATTR(power_state, S_IRUGO | S_IWUSR,
-		taos_power_state_show, taos_power_state_store);
-
-static DEVICE_ATTR(illuminance0_calibscale, S_IRUGO | S_IWUSR,
-		taos_gain_show, taos_gain_store);
-static DEVICE_ATTR(illuminance0_calibscale_available, S_IRUGO,
-		taos_gain_available_show, NULL);
-
-static DEVICE_ATTR(illuminance0_integration_time, S_IRUGO | S_IWUSR,
-		taos_als_time_show, taos_als_time_store);
-static DEVICE_ATTR(illuminance0_integration_time_available, S_IRUGO,
-		taos_als_time_available_show, NULL);
-
-static DEVICE_ATTR(illuminance0_calibbias, S_IRUGO | S_IWUSR,
-		taos_als_trim_show, taos_als_trim_store);
-
-static DEVICE_ATTR(illuminance0_input_target, S_IRUGO | S_IWUSR,
-		taos_als_cal_target_show, taos_als_cal_target_store);
-
-static DEVICE_ATTR(illuminance0_input, S_IRUGO, taos_lux_show, NULL);
-static DEVICE_ATTR(illuminance0_calibrate, S_IWUSR, NULL, taos_do_calibrate);
-static DEVICE_ATTR(illuminance0_lux_table, S_IRUGO | S_IWUSR,
-		taos_luxtable_show, taos_luxtable_store);
-
-static struct attribute *sysfs_attrs_ctrl[] = {
-	&dev_attr_power_state.attr,
-	&dev_attr_illuminance0_calibscale.attr,			/* Gain  */
-	&dev_attr_illuminance0_calibscale_available.attr,
-	&dev_attr_illuminance0_integration_time.attr,	/* I time*/
-	&dev_attr_illuminance0_integration_time_available.attr,
-	&dev_attr_illuminance0_calibbias.attr,			/* trim  */
-	&dev_attr_illuminance0_input_target.attr,
-	&dev_attr_illuminance0_input.attr,
-	&dev_attr_illuminance0_calibrate.attr,
-	&dev_attr_illuminance0_lux_table.attr,
-	NULL
-};
-
-static const struct attribute_group tsl2583_attribute_group = {
-	.attrs = sysfs_attrs_ctrl,
-};
-
-/* Use the default register values to identify the Taos device */
-static int taos_tsl258x_device(unsigned char *bufp)
-{
-	return ((bufp[TSL258X_CHIPID] & 0xf0) == 0x90);
-}
-
-static const struct iio_info tsl2583_info = {
-	.attrs = &tsl2583_attribute_group,
-	.driver_module = THIS_MODULE,
-};
-
-/*
- * Client probe function - When a valid device is found, the driver's device
- * data structure is updated, and initialization completes successfully.
- */
-static int taos_probe(struct i2c_client *clientp,
-		      const struct i2c_device_id *idp)
-{
-	int i, ret;
-	unsigned char buf[TSL258X_MAX_DEVICE_REGS];
-	struct tsl2583_chip *chip;
-	struct iio_dev *indio_dev;
-
-	if (!i2c_check_functionality(clientp->adapter,
-				     I2C_FUNC_SMBUS_BYTE_DATA)) {
-		dev_err(&clientp->dev, "taos_probe() - i2c smbus byte data func unsupported\n");
-		return -EOPNOTSUPP;
-	}
-
-	indio_dev = devm_iio_device_alloc(&clientp->dev, sizeof(*chip));
-	if (!indio_dev)
-		return -ENOMEM;
-	chip = iio_priv(indio_dev);
-	chip->client = clientp;
-	i2c_set_clientdata(clientp, indio_dev);
-
-	mutex_init(&chip->als_mutex);
-	chip->taos_chip_status = TSL258X_CHIP_UNKNOWN;
-	memcpy(chip->taos_config, taos_config, sizeof(chip->taos_config));
-
-	for (i = 0; i < TSL258X_MAX_DEVICE_REGS; i++) {
-		ret = i2c_smbus_write_byte(clientp,
-				(TSL258X_CMD_REG | (TSL258X_CNTRL + i)));
-		if (ret < 0) {
-			dev_err(&clientp->dev,
-				"i2c_smbus_write_byte to cmd reg failed in taos_probe(), err = %d\n",
-				ret);
-			return ret;
-		}
-		ret = i2c_smbus_read_byte(clientp);
-		if (ret < 0) {
-			dev_err(&clientp->dev,
-				"i2c_smbus_read_byte from reg failed in taos_probe(), err = %d\n",
-				ret);
-			return ret;
-		}
-		buf[i] = ret;
-	}
-
-	if (!taos_tsl258x_device(buf)) {
-		dev_info(&clientp->dev,
-			 "i2c device found but does not match expected id in taos_probe()\n");
-		return -EINVAL;
-	}
-
-	ret = i2c_smbus_write_byte(clientp, (TSL258X_CMD_REG | TSL258X_CNTRL));
-	if (ret < 0) {
-		dev_err(&clientp->dev,
-			"i2c_smbus_write_byte() to cmd reg failed in taos_probe(), err = %d\n",
-			ret);
-		return ret;
-	}
-
-	indio_dev->info = &tsl2583_info;
-	indio_dev->dev.parent = &clientp->dev;
-	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->name = chip->client->name;
-	ret = devm_iio_device_register(indio_dev->dev.parent, indio_dev);
-	if (ret) {
-		dev_err(&clientp->dev, "iio registration failed\n");
-		return ret;
-	}
-
-	/* Load up the V2 defaults (these are hard coded defaults for now) */
-	taos_defaults(chip);
-
-	/* Make sure the chip is on */
-	taos_chip_on(indio_dev);
-
-	dev_info(&clientp->dev, "Light sensor found.\n");
-	return 0;
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int taos_suspend(struct device *dev)
-{
-	struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev));
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-	int ret = 0;
-
-	mutex_lock(&chip->als_mutex);
-
-	if (chip->taos_chip_status == TSL258X_CHIP_WORKING) {
-		ret = taos_chip_off(indio_dev);
-		chip->taos_chip_status = TSL258X_CHIP_SUSPENDED;
-	}
-
-	mutex_unlock(&chip->als_mutex);
-	return ret;
-}
-
-static int taos_resume(struct device *dev)
-{
-	struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev));
-	struct tsl2583_chip *chip = iio_priv(indio_dev);
-	int ret = 0;
-
-	mutex_lock(&chip->als_mutex);
-
-	if (chip->taos_chip_status == TSL258X_CHIP_SUSPENDED)
-		ret = taos_chip_on(indio_dev);
-
-	mutex_unlock(&chip->als_mutex);
-	return ret;
-}
-
-static SIMPLE_DEV_PM_OPS(taos_pm_ops, taos_suspend, taos_resume);
-#define TAOS_PM_OPS (&taos_pm_ops)
-#else
-#define TAOS_PM_OPS NULL
-#endif
-
-static struct i2c_device_id taos_idtable[] = {
-	{ "tsl2580", 0 },
-	{ "tsl2581", 1 },
-	{ "tsl2583", 2 },
-	{}
-};
-MODULE_DEVICE_TABLE(i2c, taos_idtable);
-
-/* Driver definition */
-static struct i2c_driver taos_driver = {
-	.driver = {
-		.name = "tsl2583",
-		.pm = TAOS_PM_OPS,
-	},
-	.id_table = taos_idtable,
-	.probe = taos_probe,
-};
-module_i2c_driver(taos_driver);
-
-MODULE_AUTHOR("J. August Brenner<jbrenner@taosinc.com>");
-MODULE_DESCRIPTION("TAOS tsl2583 ambient light sensor driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/iio/meter/ade7758_core.c b/drivers/staging/iio/meter/ade7758_core.c
index ebb8a1993303..3af8f77b8e41 100644
--- a/drivers/staging/iio/meter/ade7758_core.c
+++ b/drivers/staging/iio/meter/ade7758_core.c
@@ -465,38 +465,26 @@ err_ret:
 	return ret;
 }
 
-static ssize_t ade7758_read_frequency(struct device *dev,
-				      struct device_attribute *attr, char *buf)
+static int ade7758_read_samp_freq(struct device *dev, int *val)
 {
 	int ret;
 	u8 t;
-	int sps;
 
 	ret = ade7758_spi_read_reg_8(dev, ADE7758_WAVMODE, &t);
 	if (ret)
 		return ret;
 
 	t = (t >> 5) & 0x3;
-	sps = 26040 / (1 << t);
+	*val = 26040 / (1 << t);
 
-	return sprintf(buf, "%d SPS\n", sps);
+	return 0;
 }
 
-static ssize_t ade7758_write_frequency(struct device *dev,
-				       struct device_attribute *attr,
-				       const char *buf, size_t len)
+static int ade7758_write_samp_freq(struct device *dev, int val)
 {
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	u16 val;
 	int ret;
 	u8 reg, t;
 
-	ret = kstrtou16(buf, 10, &val);
-	if (ret)
-		return ret;
-
-	mutex_lock(&indio_dev->mlock);
-
 	switch (val) {
 	case 26040:
 		t = 0;
@@ -525,9 +513,49 @@ static ssize_t ade7758_write_frequency(struct device *dev,
 	ret = ade7758_spi_write_reg_8(dev, ADE7758_WAVMODE, reg);
 
 out:
-	mutex_unlock(&indio_dev->mlock);
+	return ret;
+}
 
-	return ret ? ret : len;
+static int ade7758_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val,
+			    int *val2,
+			    long mask)
+{
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		mutex_lock(&indio_dev->mlock);
+		ret = ade7758_read_samp_freq(&indio_dev->dev, val);
+		mutex_unlock(&indio_dev->mlock);
+		return ret;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static int ade7758_write_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
+{
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (val2)
+			return -EINVAL;
+		mutex_lock(&indio_dev->mlock);
+		ret = ade7758_write_samp_freq(&indio_dev->dev, val);
+		mutex_unlock(&indio_dev->mlock);
+		return ret;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
 }
 
 static IIO_DEV_ATTR_TEMP_RAW(ade7758_read_8bit);
@@ -553,17 +581,12 @@ static IIO_DEV_ATTR_BVAHR(ade7758_read_16bit,
 static IIO_DEV_ATTR_CVAHR(ade7758_read_16bit,
 		ADE7758_CVAHR);
 
-static IIO_DEV_ATTR_SAMP_FREQ(S_IWUSR | S_IRUGO,
-		ade7758_read_frequency,
-		ade7758_write_frequency);
-
 static IIO_CONST_ATTR_SAMP_FREQ_AVAIL("26040 13020 6510 3255");
 
 static struct attribute *ade7758_attributes[] = {
 	&iio_dev_attr_in_temp_raw.dev_attr.attr,
 	&iio_const_attr_in_temp_offset.dev_attr.attr,
 	&iio_const_attr_in_temp_scale.dev_attr.attr,
-	&iio_dev_attr_sampling_frequency.dev_attr.attr,
 	&iio_const_attr_sampling_frequency_available.dev_attr.attr,
 	&iio_dev_attr_awatthr.dev_attr.attr,
 	&iio_dev_attr_bwatthr.dev_attr.attr,
@@ -611,6 +634,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.type = IIO_VOLTAGE,
 		.indexed = 1,
 		.channel = 0,
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_A, AD7758_VOLTAGE),
 		.scan_index = 0,
 		.scan_type = {
@@ -622,6 +646,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.type = IIO_CURRENT,
 		.indexed = 1,
 		.channel = 0,
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_A, AD7758_CURRENT),
 		.scan_index = 1,
 		.scan_type = {
@@ -634,6 +659,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.indexed = 1,
 		.channel = 0,
 		.extend_name = "apparent",
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_A, AD7758_APP_PWR),
 		.scan_index = 2,
 		.scan_type = {
@@ -646,6 +672,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.indexed = 1,
 		.channel = 0,
 		.extend_name = "active",
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_A, AD7758_ACT_PWR),
 		.scan_index = 3,
 		.scan_type = {
@@ -658,6 +685,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.indexed = 1,
 		.channel = 0,
 		.extend_name = "reactive",
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_A, AD7758_REACT_PWR),
 		.scan_index = 4,
 		.scan_type = {
@@ -669,6 +697,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.type = IIO_VOLTAGE,
 		.indexed = 1,
 		.channel = 1,
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_B, AD7758_VOLTAGE),
 		.scan_index = 5,
 		.scan_type = {
@@ -680,6 +709,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.type = IIO_CURRENT,
 		.indexed = 1,
 		.channel = 1,
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_B, AD7758_CURRENT),
 		.scan_index = 6,
 		.scan_type = {
@@ -692,6 +722,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.indexed = 1,
 		.channel = 1,
 		.extend_name = "apparent",
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_B, AD7758_APP_PWR),
 		.scan_index = 7,
 		.scan_type = {
@@ -704,6 +735,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.indexed = 1,
 		.channel = 1,
 		.extend_name = "active",
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_B, AD7758_ACT_PWR),
 		.scan_index = 8,
 		.scan_type = {
@@ -716,6 +748,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.indexed = 1,
 		.channel = 1,
 		.extend_name = "reactive",
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_B, AD7758_REACT_PWR),
 		.scan_index = 9,
 		.scan_type = {
@@ -727,6 +760,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.type = IIO_VOLTAGE,
 		.indexed = 1,
 		.channel = 2,
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_C, AD7758_VOLTAGE),
 		.scan_index = 10,
 		.scan_type = {
@@ -738,6 +772,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.type = IIO_CURRENT,
 		.indexed = 1,
 		.channel = 2,
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_C, AD7758_CURRENT),
 		.scan_index = 11,
 		.scan_type = {
@@ -750,6 +785,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.indexed = 1,
 		.channel = 2,
 		.extend_name = "apparent",
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_C, AD7758_APP_PWR),
 		.scan_index = 12,
 		.scan_type = {
@@ -762,6 +798,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.indexed = 1,
 		.channel = 2,
 		.extend_name = "active",
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_C, AD7758_ACT_PWR),
 		.scan_index = 13,
 		.scan_type = {
@@ -774,6 +811,7 @@ static const struct iio_chan_spec ade7758_channels[] = {
 		.indexed = 1,
 		.channel = 2,
 		.extend_name = "reactive",
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.address = AD7758_WT(AD7758_PHASE_C, AD7758_REACT_PWR),
 		.scan_index = 14,
 		.scan_type = {
@@ -787,6 +825,8 @@ static const struct iio_chan_spec ade7758_channels[] = {
 
 static const struct iio_info ade7758_info = {
 	.attrs = &ade7758_attribute_group,
+	.read_raw = &ade7758_read_raw,
+	.write_raw = &ade7758_write_raw,
 	.driver_module = THIS_MODULE,
 };
 
diff --git a/drivers/staging/iio/ring_hw.h b/drivers/staging/iio/ring_hw.h
deleted file mode 100644
index 75bf47bfee78..000000000000
--- a/drivers/staging/iio/ring_hw.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * ring_hw.h - common functionality for iio hardware ring buffers
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * Copyright (c) 2009 Jonathan Cameron <jic23@kernel.org>
- *
- */
-
-#ifndef _RING_HW_H_
-#define _RING_HW_H_
-
-/**
- * struct iio_hw_ring_buffer- hardware ring buffer
- * @buf:	generic ring buffer elements
- * @private:	device specific data
- */
-struct iio_hw_buffer {
-	struct iio_buffer buf;
-	void *private;
-};
-
-#define iio_to_hw_buf(r) container_of(r, struct iio_hw_buffer, buf)
-
-#endif /* _RING_HW_H_ */
diff --git a/drivers/staging/ks7010/ks7010_sdio.c b/drivers/staging/ks7010/ks7010_sdio.c
index 81c46f4d0935..a604c83c957e 100644
--- a/drivers/staging/ks7010/ks7010_sdio.c
+++ b/drivers/staging/ks7010/ks7010_sdio.c
@@ -35,18 +35,18 @@ MODULE_DEVICE_TABLE(sdio, ks7010_sdio_ids);
 /* macro */
 
 #define inc_txqhead(priv) \
-        (priv->tx_dev.qhead = (priv->tx_dev.qhead + 1) % TX_DEVICE_BUFF_SIZE)
+	(priv->tx_dev.qhead = (priv->tx_dev.qhead + 1) % TX_DEVICE_BUFF_SIZE)
 #define inc_txqtail(priv) \
-        (priv->tx_dev.qtail = (priv->tx_dev.qtail + 1) % TX_DEVICE_BUFF_SIZE)
+	(priv->tx_dev.qtail = (priv->tx_dev.qtail + 1) % TX_DEVICE_BUFF_SIZE)
 #define cnt_txqbody(priv) \
-        (((priv->tx_dev.qtail + TX_DEVICE_BUFF_SIZE) - (priv->tx_dev.qhead)) % TX_DEVICE_BUFF_SIZE)
+	(((priv->tx_dev.qtail + TX_DEVICE_BUFF_SIZE) - (priv->tx_dev.qhead)) % TX_DEVICE_BUFF_SIZE)
 
 #define inc_rxqhead(priv) \
-        (priv->rx_dev.qhead = (priv->rx_dev.qhead + 1) % RX_DEVICE_BUFF_SIZE)
+	(priv->rx_dev.qhead = (priv->rx_dev.qhead + 1) % RX_DEVICE_BUFF_SIZE)
 #define inc_rxqtail(priv) \
-        (priv->rx_dev.qtail = (priv->rx_dev.qtail + 1) % RX_DEVICE_BUFF_SIZE)
+	(priv->rx_dev.qtail = (priv->rx_dev.qtail + 1) % RX_DEVICE_BUFF_SIZE)
 #define cnt_rxqbody(priv) \
-        (((priv->rx_dev.qtail + RX_DEVICE_BUFF_SIZE) - (priv->rx_dev.qhead)) % RX_DEVICE_BUFF_SIZE)
+	(((priv->rx_dev.qtail + RX_DEVICE_BUFF_SIZE) - (priv->rx_dev.qhead)) % RX_DEVICE_BUFF_SIZE)
 
 static int ks7010_sdio_read(struct ks_wlan_private *priv, unsigned int address,
 			    unsigned char *buffer, int length)
@@ -76,10 +76,9 @@ static int ks7010_sdio_write(struct ks_wlan_private *priv, unsigned int address,
 	card = priv->ks_wlan_hw.sdio_card;
 
 	if (length == 1)	/* CMD52 */
-		sdio_writeb(card->func, *buffer, (unsigned int)address, &rc);
+		sdio_writeb(card->func, *buffer, address, &rc);
 	else	/* CMD53 */
-		rc = sdio_memcpy_toio(card->func, (unsigned int)address, buffer,
-				      length);
+		rc = sdio_memcpy_toio(card->func, address, buffer, length);
 
 	if (rc != 0)
 		DPRINTK(1, "sdio error=%d size=%d\n", rc, length);
@@ -255,7 +254,7 @@ int ks_wlan_hw_power_save(struct ks_wlan_private *priv)
 
 static int enqueue_txdev(struct ks_wlan_private *priv, unsigned char *p,
 			 unsigned long size,
-			 void (*complete_handler) (void *arg1, void *arg2),
+			 void (*complete_handler)(void *arg1, void *arg2),
 			 void *arg1, void *arg2)
 {
 	struct tx_device_buffer *sp;
@@ -294,6 +293,7 @@ static int write_to_device(struct ks_wlan_private *priv, unsigned char *buffer,
 	int retval;
 	unsigned char rw_data;
 	struct hostif_hdr *hdr;
+
 	hdr = (struct hostif_hdr *)buffer;
 
 	DPRINTK(4, "size=%d\n", hdr->size);
@@ -353,11 +353,12 @@ static void tx_device_task(void *dev)
 }
 
 int ks_wlan_hw_tx(struct ks_wlan_private *priv, void *p, unsigned long size,
-		  void (*complete_handler) (void *arg1, void *arg2),
+		  void (*complete_handler)(void *arg1, void *arg2),
 		  void *arg1, void *arg2)
 {
 	int result = 0;
 	struct hostif_hdr *hdr;
+
 	hdr = (struct hostif_hdr *)p;
 
 	if (hdr->event < HIF_DATA_REQ || HIF_REQ_MAX < hdr->event) {
@@ -412,7 +413,7 @@ static void ks_wlan_hw_rx(void *dev, uint16_t size)
 	/* receive data */
 	if (cnt_rxqbody(priv) >= (RX_DEVICE_BUFF_SIZE - 1)) {
 		/* in case of buffer overflow */
-		DPRINTK(1, "rx buffer overflow \n");
+		DPRINTK(1, "rx buffer overflow\n");
 		goto error_out;
 	}
 	rx_buffer = &priv->rx_dev.rx_dev_buff[priv->rx_dev.qtail];
@@ -658,10 +659,12 @@ static void ks_sdio_interrupt(struct sdio_func *func)
 static int trx_device_init(struct ks_wlan_private *priv)
 {
 	/* initialize values (tx) */
-	priv->tx_dev.qtail = priv->tx_dev.qhead = 0;
+	priv->tx_dev.qhead = 0;
+	priv->tx_dev.qtail = 0;
 
 	/* initialize values (rx) */
-	priv->rx_dev.qtail = priv->rx_dev.qhead = 0;
+	priv->rx_dev.qhead = 0;
+	priv->rx_dev.qtail = 0;
 
 	/* initialize spinLock (tx,rx) */
 	spin_lock_init(&priv->tx_dev.tx_dev_lock);
@@ -718,7 +721,7 @@ static int ks7010_sdio_update_index(struct ks_wlan_private *priv, u32 index)
 	return rc;
 }
 
-#define ROM_BUFF_SIZE (64*1024)
+#define ROM_BUFF_SIZE (64 * 1024)
 static int ks7010_sdio_data_compare(struct ks_wlan_private *priv, u32 address,
 				    unsigned char *data, unsigned int size)
 {
@@ -955,7 +958,7 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 	priv = NULL;
 	netdev = NULL;
 
-	/* initilize ks_sdio_card */
+	/* initialize ks_sdio_card */
 	card = kzalloc(sizeof(*card), GFP_KERNEL);
 	if (!card)
 		return -ENOMEM;
@@ -1117,6 +1120,7 @@ static void ks7010_sdio_remove(struct sdio_func *func)
 	int ret;
 	struct ks_sdio_card *card;
 	struct ks_wlan_private *priv;
+
 	DPRINTK(1, "ks7010_sdio_remove()\n");
 
 	card = sdio_get_drvdata(func);
@@ -1142,6 +1146,7 @@ static void ks7010_sdio_remove(struct sdio_func *func)
 		/* send stop request to MAC */
 		{
 			struct hostif_stop_request_t *pp;
+
 			pp = kzalloc(hif_align_size(sizeof(*pp)), GFP_KERNEL);
 			if (!pp) {
 				DPRINTK(3, "allocate memory failed..\n");
diff --git a/drivers/staging/ks7010/ks7010_sdio.h b/drivers/staging/ks7010/ks7010_sdio.h
index c72064b48bd8..0f5fd848e23d 100644
--- a/drivers/staging/ks7010/ks7010_sdio.h
+++ b/drivers/staging/ks7010/ks7010_sdio.h
@@ -1,5 +1,5 @@
 /*
- *   Driver for KeyStream, KS7010 based SDIO cards. 
+ *   Driver for KeyStream, KS7010 based SDIO cards.
  *
  *   Copyright (C) 2006-2008 KeyStream Corp.
  *   Copyright (C) 2009 Renesas Technology Corp.
@@ -41,7 +41,7 @@
 /* Write Index Register */
 #define WRITE_INDEX		0x000010
 
-/* Write Status/Read Data Size Register 
+/* Write Status/Read Data Size Register
  * for network packet (less than 2048 bytes data)
  */
 #define WSTATUS_RSIZE		0x000014
@@ -53,14 +53,14 @@
 /* ARM to SD interrupt Pending */
 #define INT_PENDING		0x000024
 
-#define INT_GCR_B		(1<<7)
-#define INT_GCR_A		(1<<6)
-#define INT_WRITE_STATUS	(1<<5)
-#define INT_WRITE_INDEX		(1<<4)
-#define INT_WRITE_SIZE		(1<<3)
-#define INT_READ_STATUS		(1<<2)
-#define INT_READ_INDEX		(1<<1)
-#define INT_READ_SIZE		(1<<0)
+#define INT_GCR_B              BIT(7)
+#define INT_GCR_A              BIT(6)
+#define INT_WRITE_STATUS       BIT(5)
+#define INT_WRITE_INDEX        BIT(4)
+#define INT_WRITE_SIZE         BIT(3)
+#define INT_READ_STATUS        BIT(2)
+#define INT_READ_INDEX         BIT(1)
+#define INT_READ_SIZE          BIT(0)
 
 /* General Communication Register A */
 #define GCR_A			0x000028
@@ -100,7 +100,7 @@ struct hw_info_t {
 struct ks_sdio_packet {
 	struct ks_sdio_packet *next;
 	u16 nb;
-	u8 buffer[0] __attribute__ ((aligned(4)));
+	u8 buffer[0] __aligned(4);
 };
 
 struct ks_sdio_card {
diff --git a/drivers/staging/ks7010/ks_hostif.c b/drivers/staging/ks7010/ks_hostif.c
index c57ca581550a..1fbd495e5e63 100644
--- a/drivers/staging/ks7010/ks_hostif.c
+++ b/drivers/staging/ks7010/ks_hostif.c
@@ -23,11 +23,11 @@
 
 /* macro */
 #define inc_smeqhead(priv) \
-        ( priv->sme_i.qhead = (priv->sme_i.qhead + 1) % SME_EVENT_BUFF_SIZE )
+	(priv->sme_i.qhead = (priv->sme_i.qhead + 1) % SME_EVENT_BUFF_SIZE)
 #define inc_smeqtail(priv) \
-        ( priv->sme_i.qtail = (priv->sme_i.qtail + 1) % SME_EVENT_BUFF_SIZE )
+	(priv->sme_i.qtail = (priv->sme_i.qtail + 1) % SME_EVENT_BUFF_SIZE)
 #define cnt_smeqbody(priv) \
-        (((priv->sme_i.qtail + SME_EVENT_BUFF_SIZE) - (priv->sme_i.qhead)) % SME_EVENT_BUFF_SIZE )
+	(((priv->sme_i.qtail + SME_EVENT_BUFF_SIZE) - (priv->sme_i.qhead)) % SME_EVENT_BUFF_SIZE)
 
 #define KS_WLAN_MEM_FLAG (GFP_ATOMIC)
 
@@ -97,11 +97,10 @@ int ks_wlan_do_power_save(struct ks_wlan_private *priv)
 {
 	DPRINTK(4, "psstatus.status=%d\n", atomic_read(&priv->psstatus.status));
 
-	if ((priv->connect_status & CONNECT_STATUS_MASK) == CONNECT_STATUS) {
+	if ((priv->connect_status & CONNECT_STATUS_MASK) == CONNECT_STATUS)
 		hostif_sme_enqueue(priv, SME_POW_MNGMT_REQUEST);
-	} else {
+	else
 		priv->dev_state = DEVICE_STATE_READY;
-	}
 	return 0;
 }
 
@@ -187,13 +186,7 @@ int get_current_ap(struct ks_wlan_private *priv, struct link_ap_info_t *ap_info)
 		memcpy(wrqu.ap_addr.sa_data,
 		       &(priv->current_ap.bssid[0]), ETH_ALEN);
 		DPRINTK(3,
-			"IWEVENT: connect bssid=%02x:%02x:%02x:%02x:%02x:%02x\n",
-			(unsigned char)wrqu.ap_addr.sa_data[0],
-			(unsigned char)wrqu.ap_addr.sa_data[1],
-			(unsigned char)wrqu.ap_addr.sa_data[2],
-			(unsigned char)wrqu.ap_addr.sa_data[3],
-			(unsigned char)wrqu.ap_addr.sa_data[4],
-			(unsigned char)wrqu.ap_addr.sa_data[5]);
+			"IWEVENT: connect bssid=%pM\n", wrqu.ap_addr.sa_data);
 		wireless_send_event(netdev, SIOCGIWAP, &wrqu, NULL);
 	}
 	DPRINTK(4, "\n    Link AP\n");
@@ -420,16 +413,11 @@ void hostif_data_indication(struct ks_wlan_private *priv)
 					/*  needed parameters: count, keyid, key type, TSC */
 					sprintf(buf,
 						"MLME-MICHAELMICFAILURE.indication(keyid=%d %scast addr="
-						"%02x:%02x:%02x:%02x:%02x:%02x)",
+						"%pM)",
 						auth_type - 1,
 						eth_hdr->
 						h_dest[0] & 0x01 ? "broad" :
-						"uni", eth_hdr->h_source[0],
-						eth_hdr->h_source[1],
-						eth_hdr->h_source[2],
-						eth_hdr->h_source[3],
-						eth_hdr->h_source[4],
-						eth_hdr->h_source[5]);
+						"uni", eth_hdr->h_source);
 					memset(&wrqu, 0, sizeof(wrqu));
 					wrqu.data.length = strlen(buf);
 					DPRINTK(4,
@@ -476,8 +464,6 @@ void hostif_data_indication(struct ks_wlan_private *priv)
 			skb->dev->last_rx = jiffies;
 			netif_rx(skb);
 		} else {
-			printk(KERN_WARNING
-			       "ks_wlan: Memory squeeze, dropping packet.\n");
 			priv->nstats.rx_dropped++;
 		}
 		break;
@@ -511,8 +497,6 @@ void hostif_data_indication(struct ks_wlan_private *priv)
 			skb->dev->last_rx = jiffies;
 			netif_rx(skb);
 		} else {
-			printk(KERN_WARNING
-			       "ks_wlan: Memory squeeze, dropping packet.\n");
 			priv->nstats.rx_dropped++;
 		}
 		break;
@@ -560,10 +544,7 @@ void hostif_mib_get_confirm(struct ks_wlan_private *priv)
 		dev->dev_addr[5] = priv->eth_addr[5];
 		dev->dev_addr[6] = 0x00;
 		dev->dev_addr[7] = 0x00;
-		printk(KERN_INFO
-		       "ks_wlan: MAC ADDRESS = %02x:%02x:%02x:%02x:%02x:%02x\n",
-		       priv->eth_addr[0], priv->eth_addr[1], priv->eth_addr[2],
-		       priv->eth_addr[3], priv->eth_addr[4], priv->eth_addr[5]);
+		netdev_info(dev, "MAC ADDRESS = %pM\n", priv->eth_addr);
 		break;
 	case DOT11_PRODUCT_VERSION:
 		/* firmware version */
@@ -571,8 +552,8 @@ void hostif_mib_get_confirm(struct ks_wlan_private *priv)
 		priv->version_size = priv->rx_size;
 		memcpy(priv->firmware_version, priv->rxp, priv->rx_size);
 		priv->firmware_version[priv->rx_size] = '\0';
-		printk(KERN_INFO "ks_wlan: firmware ver. = %s\n",
-		       priv->firmware_version);
+		netdev_info(dev, "firmware ver. = %s\n",
+			    priv->firmware_version);
 		hostif_sme_enqueue(priv, SME_GET_PRODUCT_VERSION);
 		/* wake_up_interruptible_all(&priv->confirm_wait); */
 		complete(&priv->confirm_wait);
@@ -592,12 +573,12 @@ void hostif_mib_get_confirm(struct ks_wlan_private *priv)
 		} else if (priv->eeprom_sum.type == 1) {
 			if (priv->eeprom_sum.result == 0) {
 				priv->eeprom_checksum = EEPROM_NG;
-				printk("LOCAL_EEPROM_SUM NG\n");
+				netdev_info(dev, "LOCAL_EEPROM_SUM NG\n");
 			} else if (priv->eeprom_sum.result == 1) {
 				priv->eeprom_checksum = EEPROM_OK;
 			}
 		} else {
-			printk("LOCAL_EEPROM_SUM error!\n");
+			netdev_err(dev, "LOCAL_EEPROM_SUM error!\n");
 		}
 		break;
 	default:
@@ -705,15 +686,13 @@ void hostif_mib_set_confirm(struct ks_wlan_private *priv)
 		break;
 	case DOT11_GMK1_TSC:
 		DPRINTK(2, "DOT11_GMK1_TSC:mib_status=%d\n", (int)mib_status);
-		if (atomic_read(&priv->psstatus.snooze_guard)) {
+		if (atomic_read(&priv->psstatus.snooze_guard))
 			atomic_set(&priv->psstatus.snooze_guard, 0);
-		}
 		break;
 	case DOT11_GMK2_TSC:
 		DPRINTK(2, "DOT11_GMK2_TSC:mib_status=%d\n", (int)mib_status);
-		if (atomic_read(&priv->psstatus.snooze_guard)) {
+		if (atomic_read(&priv->psstatus.snooze_guard))
 			atomic_set(&priv->psstatus.snooze_guard, 0);
-		}
 		break;
 	case LOCAL_PMK:
 		DPRINTK(2, "LOCAL_PMK:mib_status=%d\n", (int)mib_status);
@@ -766,8 +745,9 @@ void hostif_sleep_confirm(struct ks_wlan_private *priv)
 static
 void hostif_start_confirm(struct ks_wlan_private *priv)
 {
-#ifdef  WPS
+#ifdef WPS
 	union iwreq_data wrqu;
+
 	wrqu.data.length = 0;
 	wrqu.data.flags = 0;
 	wrqu.ap_addr.sa_family = ARPHRD_ETHER;
@@ -789,6 +769,7 @@ void hostif_connect_indication(struct ks_wlan_private *priv)
 	unsigned int old_status = priv->connect_status;
 	struct net_device *netdev = priv->net_dev;
 	union iwreq_data wrqu0;
+
 	connect_code = get_WORD(priv);
 
 	switch (connect_code) {
@@ -894,7 +875,7 @@ void hostif_stop_confirm(struct ks_wlan_private *priv)
 		netif_carrier_off(netdev);
 		tmp = FORCE_DISCONNECT & priv->connect_status;
 		priv->connect_status = tmp | DISCONNECT_STATUS;
-		printk("IWEVENT: disconnect\n");
+		netdev_info(netdev, "IWEVENT: disconnect\n");
 
 		wrqu0.data.length = 0;
 		wrqu0.data.flags = 0;
@@ -904,7 +885,7 @@ void hostif_stop_confirm(struct ks_wlan_private *priv)
 		    && (old_status & CONNECT_STATUS_MASK) == CONNECT_STATUS) {
 			eth_zero_addr(wrqu0.ap_addr.sa_data);
 			DPRINTK(3, "IWEVENT: disconnect\n");
-			printk("IWEVENT: disconnect\n");
+			netdev_info(netdev, "IWEVENT: disconnect\n");
 			DPRINTK(3, "disconnect :: scan_ind_count=%d\n",
 				priv->scan_ind_count);
 			wireless_send_event(netdev, SIOCGIWAP, &wrqu0, NULL);
@@ -928,6 +909,7 @@ static
 void hostif_infrastructure_set_confirm(struct ks_wlan_private *priv)
 {
 	uint16_t result_code;
+
 	DPRINTK(3, "\n");
 	result_code = get_WORD(priv);
 	DPRINTK(3, "result code = %d\n", result_code);
@@ -993,6 +975,7 @@ void hostif_bss_scan_confirm(struct ks_wlan_private *priv)
 	unsigned int result_code;
 	struct net_device *dev = priv->net_dev;
 	union iwreq_data wrqu;
+
 	result_code = get_DWORD(priv);
 	DPRINTK(2, "result=%d :: scan_ind_count=%d\n", result_code,
 		priv->scan_ind_count);
@@ -1110,7 +1093,7 @@ void hostif_event_check(struct ks_wlan_private *priv)
 	case HIF_AP_SET_CONF:
 	default:
 		//DPRINTK(1, "undefined event[%04X]\n", event);
-		printk("undefined event[%04X]\n", event);
+		netdev_err(priv->net_dev, "undefined event[%04X]\n", event);
 		/* wake_up_all(&priv->confirm_wait); */
 		complete(&priv->confirm_wait);
 		break;
@@ -1184,9 +1167,7 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *packet)
 	eth = (struct ethhdr *)packet->data;
 	if (memcmp(&priv->eth_addr[0], eth->h_source, ETH_ALEN)) {
 		DPRINTK(1, "invalid mac address !!\n");
-		DPRINTK(1, "ethernet->h_source=%02X:%02X:%02X:%02X:%02X:%02X\n",
-			eth->h_source[0], eth->h_source[1], eth->h_source[2],
-			eth->h_source[3], eth->h_source[4], eth->h_source[5]);
+		DPRINTK(1, "ethernet->h_source=%pM\n", eth->h_source);
 		dev_kfree_skb(packet);
 		kfree(pp);
 		return -3;
@@ -1244,7 +1225,7 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *packet)
 			pp->auth_type = cpu_to_le16((uint16_t) TYPE_AUTH);	/* no encryption */
 		} else {
 			if (priv->wpa.pairwise_suite == IW_AUTH_CIPHER_TKIP) {
-				MichaelMICFunction(&michel_mic, (uint8_t *) priv->wpa.key[0].tx_mic_key, (uint8_t *) & pp->data[0], (int)packet_len, (uint8_t) 0,	/* priority */
+				MichaelMICFunction(&michel_mic, (uint8_t *) priv->wpa.key[0].tx_mic_key, (uint8_t *) &pp->data[0], (int)packet_len, (uint8_t) 0,	/* priority */
 						   (uint8_t *) michel_mic.
 						   Result);
 				memcpy(p, michel_mic.Result, 8);
@@ -1294,10 +1275,11 @@ int hostif_data_request(struct ks_wlan_private *priv, struct sk_buff *packet)
 	return result;
 }
 
-#define ps_confirm_wait_inc(priv)  do{if(atomic_read(&priv->psstatus.status) > PS_ACTIVE_SET){ \
-                                                  atomic_inc(&priv->psstatus.confirm_wait); \
-                                                  /* atomic_set(&priv->psstatus.status, PS_CONF_WAIT);*/ \
-                                      } }while(0)
+#define ps_confirm_wait_inc(priv) do { \
+	if (atomic_read(&priv->psstatus.status) > PS_ACTIVE_SET) { \
+		atomic_inc(&priv->psstatus.confirm_wait); \
+		/* atomic_set(&priv->psstatus.status, PS_CONF_WAIT);*/ \
+	} } while (0)
 
 static
 void hostif_mib_get_request(struct ks_wlan_private *priv,
@@ -1891,6 +1873,7 @@ static
 void hostif_sme_set_wep(struct ks_wlan_private *priv, int type)
 {
 	uint32_t val;
+
 	switch (type) {
 	case SME_WEP_INDEX_REQUEST:
 		val = cpu_to_le32((uint32_t) (priv->reg.wep_index));
@@ -1936,18 +1919,17 @@ void hostif_sme_set_wep(struct ks_wlan_private *priv, int type)
 		break;
 	}
 
-	return;
 }
 
 struct wpa_suite_t {
 	unsigned short size;
 	unsigned char suite[4][CIPHER_ID_LEN];
-} __attribute__ ((packed));
+} __packed;
 
 struct rsn_mode_t {
 	uint32_t rsn_mode;
 	uint16_t rsn_capability;
-} __attribute__ ((packed));
+} __packed;
 
 static
 void hostif_sme_set_rsn(struct ks_wlan_private *priv, int type)
@@ -2125,7 +2107,6 @@ void hostif_sme_set_rsn(struct ks_wlan_private *priv, int type)
 		break;
 
 	}
-	return;
 }
 
 static
@@ -2216,10 +2197,7 @@ void hostif_sme_mode_setup(struct ks_wlan_private *priv)
 		} else {
 			hostif_infrastructure_set2_request(priv);
 			DPRINTK(2,
-				"Infra bssid = %02x:%02x:%02x:%02x:%02x:%02x\n",
-				priv->reg.bssid[0], priv->reg.bssid[1],
-				priv->reg.bssid[2], priv->reg.bssid[3],
-				priv->reg.bssid[4], priv->reg.bssid[5]);
+				"Infra bssid = %pM\n", priv->reg.bssid);
 		}
 		break;
 	case MODE_ADHOC:
@@ -2229,17 +2207,13 @@ void hostif_sme_mode_setup(struct ks_wlan_private *priv)
 		} else {
 			hostif_adhoc_set2_request(priv);
 			DPRINTK(2,
-				"Adhoc bssid = %02x:%02x:%02x:%02x:%02x:%02x\n",
-				priv->reg.bssid[0], priv->reg.bssid[1],
-				priv->reg.bssid[2], priv->reg.bssid[3],
-				priv->reg.bssid[4], priv->reg.bssid[5]);
+				"Adhoc bssid = %pM\n", priv->reg.bssid);
 		}
 		break;
 	default:
 		break;
 	}
 
-	return;
 }
 
 static
@@ -2340,7 +2314,6 @@ void hostif_sme_powermgt_set(struct ks_wlan_private *priv)
 	}
 	hostif_power_mngmt_request(priv, mode, wake_up, receiveDTIMs);
 
-	return;
 }
 
 static
@@ -2358,13 +2331,13 @@ void hostif_sme_sleep_set(struct ks_wlan_private *priv)
 		break;
 	}
 
-	return;
 }
 
 static
 void hostif_sme_set_key(struct ks_wlan_private *priv, int type)
 {
 	uint32_t val;
+
 	switch (type) {
 	case SME_SET_FLAG:
 		val = cpu_to_le32((uint32_t) (priv->reg.privacy_invoked));
@@ -2416,7 +2389,6 @@ void hostif_sme_set_key(struct ks_wlan_private *priv, int type)
 				       &priv->wpa.key[2].rx_seq[0]);
 		break;
 	}
-	return;
 }
 
 static
@@ -2427,16 +2399,14 @@ void hostif_sme_set_pmksa(struct ks_wlan_private *priv)
 		struct {
 			uint8_t bssid[ETH_ALEN];
 			uint8_t pmkid[IW_PMKID_LEN];
-		} __attribute__ ((packed)) list[PMK_LIST_MAX];
-	} __attribute__ ((packed)) pmkcache;
+		} __packed list[PMK_LIST_MAX];
+	} __packed pmkcache;
 	struct pmk_t *pmk;
-	struct list_head *ptr;
 	int i;
 
 	DPRINTK(4, "pmklist.size=%d\n", priv->pmklist.size);
 	i = 0;
-	list_for_each(ptr, &priv->pmklist.head) {
-		pmk = list_entry(ptr, struct pmk_t, list);
+	list_for_each_entry(pmk, &priv->pmklist.head, list) {
 		if (i < PMK_LIST_MAX) {
 			memcpy(pmkcache.list[i].bssid, pmk->bssid, ETH_ALEN);
 			memcpy(pmkcache.list[i].pmkid, pmk->pmkid,
@@ -2461,9 +2431,8 @@ void hostif_sme_execute(struct ks_wlan_private *priv, int event)
 	DPRINTK(3, "event=%d\n", event);
 	switch (event) {
 	case SME_START:
-		if (priv->dev_state == DEVICE_STATE_BOOT) {
+		if (priv->dev_state == DEVICE_STATE_BOOT)
 			hostif_mib_get_request(priv, DOT11_MAC_ADDRESS);
-		}
 		break;
 	case SME_MULTICAST_REQUEST:
 		hostif_sme_multicast_set(priv);
@@ -2508,14 +2477,12 @@ void hostif_sme_execute(struct ks_wlan_private *priv, int event)
 		}
 		break;
 	case SME_GET_MAC_ADDRESS:
-		if (priv->dev_state == DEVICE_STATE_BOOT) {
+		if (priv->dev_state == DEVICE_STATE_BOOT)
 			hostif_mib_get_request(priv, DOT11_PRODUCT_VERSION);
-		}
 		break;
 	case SME_GET_PRODUCT_VERSION:
-		if (priv->dev_state == DEVICE_STATE_BOOT) {
+		if (priv->dev_state == DEVICE_STATE_BOOT)
 			priv->dev_state = DEVICE_STATE_PREINIT;
-		}
 		break;
 	case SME_STOP_REQUEST:
 		hostif_stop_request(priv);
@@ -2594,9 +2561,8 @@ void hostif_sme_execute(struct ks_wlan_private *priv, int event)
 		/* for power save */
 		atomic_set(&priv->psstatus.snooze_guard, 0);
 		atomic_set(&priv->psstatus.confirm_wait, 0);
-		if (priv->dev_state == DEVICE_STATE_PREINIT) {
+		if (priv->dev_state == DEVICE_STATE_PREINIT)
 			priv->dev_state = DEVICE_STATE_INIT;
-		}
 		/* wake_up_interruptible_all(&priv->confirm_wait); */
 		complete(&priv->confirm_wait);
 		break;
@@ -2652,7 +2618,6 @@ void hostif_sme_task(unsigned long dev)
 				tasklet_schedule(&priv->sme_task);
 		}
 	}
-	return;
 }
 
 /* send to Station Management Entity module */
@@ -2672,7 +2637,7 @@ void hostif_sme_enqueue(struct ks_wlan_private *priv, unsigned short event)
 	} else {
 		/* in case of buffer overflow */
 		//DPRINTK(2,"sme queue buffer overflow\n");
-		printk("sme queue buffer overflow\n");
+		netdev_err(priv->net_dev, "sme queue buffer overflow\n");
 	}
 
 	tasklet_schedule(&priv->sme_task);
@@ -2736,5 +2701,4 @@ int hostif_init(struct ks_wlan_private *priv)
 void hostif_exit(struct ks_wlan_private *priv)
 {
 	tasklet_kill(&priv->sme_task);
-	return;
 }
diff --git a/drivers/staging/ks7010/ks_wlan.h b/drivers/staging/ks7010/ks_wlan.h
index c2cc288ae899..279e9b06fc4b 100644
--- a/drivers/staging/ks7010/ks_wlan.h
+++ b/drivers/staging/ks7010/ks_wlan.h
@@ -14,7 +14,6 @@
 
 #define WPS
 
-#include <linux/version.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -25,13 +24,13 @@
 #include <linux/netdevice.h>	/* struct net_device_stats,  struct sk_buff     */
 #include <linux/etherdevice.h>
 #include <linux/wireless.h>
-#include <asm/atomic.h>	/* struct atmic_t                               */
+#include <linux/atomic.h>	/* struct atomic_t */
 #include <linux/timer.h>	/* struct timer_list */
 #include <linux/string.h>
 #include <linux/completion.h>	/* struct completion */
 #include <linux/workqueue.h>
 
-#include <asm/io.h>
+#include <linux/io.h>
 
 #include "ks7010_sdio.h"
 
@@ -43,36 +42,36 @@
 #endif
 
 struct ks_wlan_parameter {
-	uint8_t operation_mode;	/* Operation Mode */
-	uint8_t channel;	/*  Channel */
-	uint8_t tx_rate;	/*  Transmit Rate */
+	u8 operation_mode;	/* Operation Mode */
+	u8 channel;	/*  Channel */
+	u8 tx_rate;	/*  Transmit Rate */
 	struct {
-		uint8_t size;
-		uint8_t body[16];
+		u8 size;
+		u8 body[16];
 	} rate_set;
-	uint8_t bssid[ETH_ALEN];	/* BSSID */
+	u8 bssid[ETH_ALEN];	/* BSSID */
 	struct {
-		uint8_t size;
-		uint8_t body[32 + 1];
+		u8 size;
+		u8 body[32 + 1];
 	} ssid;	/*  SSID */
-	uint8_t preamble;	/*  Preamble */
-	uint8_t powermgt;	/*  PowerManagementMode */
-	uint32_t scan_type;	/*  AP List Scan Type */
+	u8 preamble;	/*  Preamble */
+	u8 powermgt;	/*  PowerManagementMode */
+	u32 scan_type;	/*  AP List Scan Type */
 #define BEACON_LOST_COUNT_MIN 0
 #define BEACON_LOST_COUNT_MAX 65535
-	uint32_t beacon_lost_count;	/*  Beacon Lost Count */
-	uint32_t rts;	/*  RTS Threashold */
-	uint32_t fragment;	/*  Fragmentation Threashold */
-	uint32_t privacy_invoked;
-	uint32_t wep_index;
+	u32 beacon_lost_count;	/*  Beacon Lost Count */
+	u32 rts;	/*  RTS Threashold */
+	u32 fragment;	/*  Fragmentation Threashold */
+	u32 privacy_invoked;
+	u32 wep_index;
 	struct {
-		uint8_t size;
-		uint8_t val[13 * 2 + 1];
+		u8 size;
+		u8 val[13 * 2 + 1];
 	} wep_key[4];
-	uint16_t authenticate_type;
-	uint16_t phy_type;	/* 11b/11g/11bg mode type */
-	uint16_t cts_mode;	/* for 11g/11bg mode cts mode */
-	uint16_t phy_info_timer;	/* phy information timer */
+	u16 authenticate_type;
+	u16 phy_type;	/* 11b/11g/11bg mode type */
+	u16 cts_mode;	/* for 11g/11bg mode cts mode */
+	u16 phy_info_timer;	/* phy information timer */
 };
 
 enum {
@@ -216,37 +215,37 @@ struct hostt_t {
 
 #define RSN_IE_BODY_MAX 64
 struct rsn_ie_t {
-	uint8_t id;	/* 0xdd = WPA or 0x30 = RSN */
-	uint8_t size;	/* max ? 255 ? */
-	uint8_t body[RSN_IE_BODY_MAX];
+	u8 id;	/* 0xdd = WPA or 0x30 = RSN */
+	u8 size;	/* max ? 255 ? */
+	u8 body[RSN_IE_BODY_MAX];
 } __packed;
 
 #ifdef WPS
 #define WPS_IE_BODY_MAX 255
 struct wps_ie_t {
-	uint8_t id;	/* 221 'dd <len> 00 50 F2 04' */
-	uint8_t size;	/* max ? 255 ? */
-	uint8_t body[WPS_IE_BODY_MAX];
+	u8 id;	/* 221 'dd <len> 00 50 F2 04' */
+	u8 size;	/* max ? 255 ? */
+	u8 body[WPS_IE_BODY_MAX];
 } __packed;
 #endif /* WPS */
 
 struct local_ap_t {
-	uint8_t bssid[6];
-	uint8_t rssi;
-	uint8_t sq;
+	u8 bssid[6];
+	u8 rssi;
+	u8 sq;
 	struct {
-		uint8_t size;
-		uint8_t body[32];
-		uint8_t ssid_pad;
+		u8 size;
+		u8 body[32];
+		u8 ssid_pad;
 	} ssid;
 	struct {
-		uint8_t size;
-		uint8_t body[16];
-		uint8_t rate_pad;
+		u8 size;
+		u8 body[16];
+		u8 rate_pad;
 	} rate_set;
-	uint16_t capability;
-	uint8_t channel;
-	uint8_t noise;
+	u16 capability;
+	u8 channel;
+	u8 noise;
 	struct rsn_ie_t wpa_ie;
 	struct rsn_ie_t rsn_ie;
 #ifdef WPS
@@ -262,15 +261,15 @@ struct local_aplist_t {
 };
 
 struct local_gain_t {
-	uint8_t TxMode;
-	uint8_t RxMode;
-	uint8_t TxGain;
-	uint8_t RxGain;
+	u8 TxMode;
+	u8 RxMode;
+	u8 TxGain;
+	u8 RxGain;
 };
 
 struct local_eeprom_sum_t {
-	uint8_t type;
-	uint8_t result;
+	u8 type;
+	u8 result;
 };
 
 enum {
@@ -352,25 +351,25 @@ enum {
 #define MIC_KEY_SIZE 8
 
 struct wpa_key_t {
-	uint32_t ext_flags;	/* IW_ENCODE_EXT_xxx */
-	uint8_t tx_seq[IW_ENCODE_SEQ_MAX_SIZE];	/* LSB first */
-	uint8_t rx_seq[IW_ENCODE_SEQ_MAX_SIZE];	/* LSB first */
+	u32 ext_flags;	/* IW_ENCODE_EXT_xxx */
+	u8 tx_seq[IW_ENCODE_SEQ_MAX_SIZE];	/* LSB first */
+	u8 rx_seq[IW_ENCODE_SEQ_MAX_SIZE];	/* LSB first */
 	struct sockaddr addr;	/* ff:ff:ff:ff:ff:ff for broadcast/multicast
 				 * (group) keys or unicast address for
 				 * individual keys */
-	uint16_t alg;
-	uint16_t key_len;	/* WEP: 5 or 13, TKIP: 32, CCMP: 16 */
-	uint8_t key_val[IW_ENCODING_TOKEN_MAX];
-	uint8_t tx_mic_key[MIC_KEY_SIZE];
-	uint8_t rx_mic_key[MIC_KEY_SIZE];
+	u16 alg;
+	u16 key_len;	/* WEP: 5 or 13, TKIP: 32, CCMP: 16 */
+	u8 key_val[IW_ENCODING_TOKEN_MAX];
+	u8 tx_mic_key[MIC_KEY_SIZE];
+	u8 rx_mic_key[MIC_KEY_SIZE];
 };
 #define WPA_KEY_INDEX_MAX 4
 #define WPA_RX_SEQ_LEN 6
 
 struct mic_failure_t {
-	uint16_t failure;	/* MIC Failure counter 0 or 1 or 2 */
-	uint16_t counter;	/* 1sec counter 0-60 */
-	uint32_t last_failure_time;
+	u16 failure;	/* MIC Failure counter 0 or 1 or 2 */
+	u16 counter;	/* 1sec counter 0-60 */
+	u32 last_failure_time;
 	int stop;	/* stop flag */
 };
 
@@ -391,12 +390,12 @@ struct wpa_status_t {
 #include <linux/list.h>
 #define PMK_LIST_MAX 8
 struct pmk_list_t {
-	uint16_t size;
+	u16 size;
 	struct list_head head;
 	struct pmk_t {
 		struct list_head list;
-		uint8_t bssid[ETH_ALEN];
-		uint8_t pmkid[IW_PMKID_LEN];
+		u8 bssid[ETH_ALEN];
+		u8 pmkid[IW_PMKID_LEN];
 	} pmk[PMK_LIST_MAX];
 };
 
@@ -404,7 +403,7 @@ struct pmk_list_t {
 struct wps_status_t {
 	int wps_enabled;
 	int ielen;
-	uint8_t ie[255];
+	u8 ie[255];
 };
 #endif /* WPS */
 
@@ -439,7 +438,7 @@ struct ks_wlan_private {
 	struct pmk_list_t pmklist;
 	/* wireless parameter */
 	struct ks_wlan_parameter reg;
-	uint8_t current_rate;
+	u8 current_rate;
 
 	char nick[IW_ESSID_MAX_SIZE + 1];
 
@@ -472,24 +471,24 @@ struct ks_wlan_private {
 	/* spinlock_t lock; */
 #define FORCE_DISCONNECT    0x80000000
 #define CONNECT_STATUS_MASK 0x7FFFFFFF
-	uint32_t connect_status;	/* connect status */
+	u32 connect_status;	/* connect status */
 	int infra_status;	/* Infractructure status */
 
-	uint8_t data_buff[0x1000];
+	u8 data_buff[0x1000];
 
-	uint8_t scan_ssid_len;
-	uint8_t scan_ssid[IW_ESSID_MAX_SIZE + 1];
+	u8 scan_ssid_len;
+	u8 scan_ssid[IW_ESSID_MAX_SIZE + 1];
 	struct local_gain_t gain;
 #ifdef WPS
 	struct net_device *l2_dev;
 	int l2_fd;
 	struct wps_status_t wps;
 #endif /* WPS */
-	uint8_t sleep_mode;
+	u8 sleep_mode;
 
-	uint8_t region;
+	u8 region;
 	struct local_eeprom_sum_t eeprom_sum;
-	uint8_t eeprom_checksum;
+	u8 eeprom_checksum;
 
 	struct hostt_t hostt;
 
diff --git a/drivers/staging/ks7010/ks_wlan_net.c b/drivers/staging/ks7010/ks_wlan_net.c
index b2b4fa4c3834..e5d04adaeb1a 100644
--- a/drivers/staging/ks7010/ks_wlan_net.c
+++ b/drivers/staging/ks7010/ks_wlan_net.c
@@ -24,9 +24,9 @@
 #include <linux/pci.h>
 #include <linux/ctype.h>
 #include <linux/timer.h>
-#include <asm/atomic.h>
+#include <linux/atomic.h>
 #include <linux/io.h>
-#include <asm/uaccess.h>
+#include <linux/uaccess.h>
 
 static int wep_on_off;
 #define	WEP_OFF		0
@@ -50,10 +50,10 @@ static const long frequency_list[] = { 2412, 2417, 2422, 2427, 2432, 2437, 2442,
 /* A few details needed for WEP (Wireless Equivalent Privacy) */
 #define MAX_KEY_SIZE 13	/* 128 (?) bits */
 #define MIN_KEY_SIZE  5	/* 40 bits RC4 - WEP */
-typedef struct wep_key_t {
+struct wep_key {
 	u16 len;
 	u8 key[16];	/* 40-bit and 104-bit keys */
-} wep_key_t;
+};
 
 /* Backward compatibility */
 #ifndef IW_ENCODE_NOKEY
@@ -88,9 +88,9 @@ int ks_wlan_update_phy_information(struct ks_wlan_private *priv)
 
 	DPRINTK(4, "in_interrupt = %ld\n", in_interrupt());
 
-	if (priv->dev_state < DEVICE_STATE_READY) {
+	if (priv->dev_state < DEVICE_STATE_READY)
 		return -1;	/* not finished initialize */
-	}
+
 	if (atomic_read(&update_phyinfo))
 		return 1;
 
@@ -182,19 +182,18 @@ static int ks_wlan_get_name(struct net_device *dev,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
-	if (priv->dev_state < DEVICE_STATE_READY) {
+	if (priv->dev_state < DEVICE_STATE_READY)
 		strcpy(cwrq, "NOT READY!");
-	} else if (priv->reg.phy_type == D_11B_ONLY_MODE) {
+	else if (priv->reg.phy_type == D_11B_ONLY_MODE)
 		strcpy(cwrq, "IEEE 802.11b");
-	} else if (priv->reg.phy_type == D_11G_ONLY_MODE) {
+	else if (priv->reg.phy_type == D_11G_ONLY_MODE)
 		strcpy(cwrq, "IEEE 802.11g");
-	} else {
+	else
 		strcpy(cwrq, "IEEE 802.11b/g");
-	}
 
 	return 0;
 }
@@ -209,9 +208,8 @@ static int ks_wlan_set_freq(struct net_device *dev,
 	    (struct ks_wlan_private *)netdev_priv(dev);
 	int rc = -EINPROGRESS;	/* Call commit handler */
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 
 	/* for SLEEP MODE */
 	/* If setting by frequency, convert to a channel */
@@ -219,6 +217,7 @@ static int ks_wlan_set_freq(struct net_device *dev,
 	    (fwrq->m >= (int)2.412e8) && (fwrq->m <= (int)2.487e8)) {
 		int f = fwrq->m / 100000;
 		int c = 0;
+
 		while ((c < 14) && (f != frequency_list[c]))
 			c++;
 		/* Hack to fall through... */
@@ -257,13 +256,13 @@ static int ks_wlan_get_freq(struct net_device *dev,
 	    (struct ks_wlan_private *)netdev_priv(dev);
 	int f;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
-	if ((priv->connect_status & CONNECT_STATUS_MASK) == CONNECT_STATUS) {
+	if ((priv->connect_status & CONNECT_STATUS_MASK) == CONNECT_STATUS)
 		f = (int)priv->current_ap.channel;
-	} else
+		else
 		f = (int)priv->reg.channel;
 	fwrq->m = frequency_list[f - 1] * 100000;
 	fwrq->e = 1;
@@ -283,9 +282,9 @@ static int ks_wlan_set_essid(struct net_device *dev,
 
 	DPRINTK(2, " %d\n", dwrq->flags);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 
 	/* for SLEEP MODE */
 	/* Check if we asked for `any' */
@@ -301,14 +300,14 @@ static int ks_wlan_set_essid(struct net_device *dev,
 			len--;
 
 		/* Check the size of the string */
-		if (len > IW_ESSID_MAX_SIZE) {
+		if (len > IW_ESSID_MAX_SIZE)
 			return -EINVAL;
-		}
+
 #else
 		/* Check the size of the string */
-		if (dwrq->length > IW_ESSID_MAX_SIZE + 1) {
+		if (dwrq->length > IW_ESSID_MAX_SIZE + 1)
 			return -E2BIG;
-		}
+
 #endif
 
 		/* Set the SSID */
@@ -340,9 +339,9 @@ static int ks_wlan_get_essid(struct net_device *dev,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 
 	/* for SLEEP MODE */
 	/* Note : if dwrq->flags != 0, we should
@@ -385,25 +384,23 @@ static int ks_wlan_set_wap(struct net_device *dev, struct iw_request_info *info,
 
 	DPRINTK(2, "\n");
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	if (priv->reg.operation_mode == MODE_ADHOC ||
 	    priv->reg.operation_mode == MODE_INFRASTRUCTURE) {
-		memcpy(priv->reg.bssid, (u8 *) & ap_addr->sa_data, ETH_ALEN);
+		memcpy(priv->reg.bssid, &ap_addr->sa_data, ETH_ALEN);
 
-		if (is_valid_ether_addr((u8 *) priv->reg.bssid)) {
+		if (is_valid_ether_addr((u8 *)priv->reg.bssid))
 			priv->need_commit |= SME_MODE_SET;
-		}
+
 	} else {
 		eth_zero_addr(priv->reg.bssid);
 		return -EOPNOTSUPP;
 	}
 
-	DPRINTK(2, "bssid = %02x:%02x:%02x:%02x:%02x:%02x\n",
-		priv->reg.bssid[0], priv->reg.bssid[1], priv->reg.bssid[2],
-		priv->reg.bssid[3], priv->reg.bssid[4], priv->reg.bssid[5]);
+	DPRINTK(2, "bssid = %pM\n", priv->reg.bssid);
 
 	/* Write it to the card */
 	if (priv->need_commit) {
@@ -421,15 +418,14 @@ static int ks_wlan_get_wap(struct net_device *dev, struct iw_request_info *info,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
-	if ((priv->connect_status & CONNECT_STATUS_MASK) == CONNECT_STATUS) {
+	if ((priv->connect_status & CONNECT_STATUS_MASK) == CONNECT_STATUS)
 		memcpy(awrq->sa_data, &(priv->current_ap.bssid[0]), ETH_ALEN);
-	} else {
+	else
 		eth_zero_addr(awrq->sa_data);
-	}
 
 	awrq->sa_family = ARPHRD_ETHER;
 
@@ -445,15 +441,14 @@ static int ks_wlan_set_nick(struct net_device *dev,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 
 	/* for SLEEP MODE */
 	/* Check the size of the string */
-	if (dwrq->length > 16 + 1) {
+	if (dwrq->length > 16 + 1)
 		return -E2BIG;
-	}
+
 	memset(priv->nick, 0, sizeof(priv->nick));
 	memcpy(priv->nick, extra, dwrq->length);
 
@@ -469,9 +464,9 @@ static int ks_wlan_get_nick(struct net_device *dev,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	strncpy(extra, priv->nick, 16);
 	extra[16] = '\0';
@@ -490,9 +485,9 @@ static int ks_wlan_set_rate(struct net_device *dev,
 	    (struct ks_wlan_private *)netdev_priv(dev);
 	int i = 0;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	if (priv->reg.phy_type == D_11B_ONLY_MODE) {
 		if (vwrq->fixed == 1) {
@@ -727,13 +722,13 @@ static int ks_wlan_get_rate(struct net_device *dev,
 	DPRINTK(2, "in_interrupt = %ld update_phyinfo = %d\n",
 		in_interrupt(), atomic_read(&update_phyinfo));
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
-	if (!atomic_read(&update_phyinfo)) {
+	if (!atomic_read(&update_phyinfo))
 		ks_wlan_update_phy_information(priv);
-	}
+
 	vwrq->value = ((priv->current_rate) & RATE_MASK) * 500000;
 	if (priv->reg.tx_rate == TX_RATE_FIXED)
 		vwrq->fixed = 1;
@@ -752,15 +747,15 @@ static int ks_wlan_set_rts(struct net_device *dev, struct iw_request_info *info,
 	    (struct ks_wlan_private *)netdev_priv(dev);
 	int rthr = vwrq->value;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	if (vwrq->disabled)
 		rthr = 2347;
-	if ((rthr < 0) || (rthr > 2347)) {
+	if ((rthr < 0) || (rthr > 2347))
 		return -EINVAL;
-	}
+
 	priv->reg.rts = rthr;
 	priv->need_commit |= SME_RTS;
 
@@ -775,9 +770,9 @@ static int ks_wlan_get_rts(struct net_device *dev, struct iw_request_info *info,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	vwrq->value = priv->reg.rts;
 	vwrq->disabled = (vwrq->value >= 2347);
@@ -796,15 +791,15 @@ static int ks_wlan_set_frag(struct net_device *dev,
 	    (struct ks_wlan_private *)netdev_priv(dev);
 	int fthr = vwrq->value;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	if (vwrq->disabled)
 		fthr = 2346;
-	if ((fthr < 256) || (fthr > 2346)) {
+	if ((fthr < 256) || (fthr > 2346))
 		return -EINVAL;
-	}
+
 	fthr &= ~0x1;	/* Get an even value - is it really needed ??? */
 	priv->reg.fragment = fthr;
 	priv->need_commit |= SME_FRAG;
@@ -821,9 +816,9 @@ static int ks_wlan_get_frag(struct net_device *dev,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	vwrq->value = priv->reg.fragment;
 	vwrq->disabled = (vwrq->value >= 2346);
@@ -835,7 +830,7 @@ static int ks_wlan_get_frag(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Wireless Handler : set Mode of Operation */
 static int ks_wlan_set_mode(struct net_device *dev,
-			    struct iw_request_info *info, __u32 * uwrq,
+			    struct iw_request_info *info, __u32 *uwrq,
 			    char *extra)
 {
 	struct ks_wlan_private *priv =
@@ -843,9 +838,9 @@ static int ks_wlan_set_mode(struct net_device *dev,
 
 	DPRINTK(2, "mode=%d\n", *uwrq);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	switch (*uwrq) {
 	case IW_MODE_ADHOC:
@@ -871,15 +866,14 @@ static int ks_wlan_set_mode(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Wireless Handler : get Mode of Operation */
 static int ks_wlan_get_mode(struct net_device *dev,
-			    struct iw_request_info *info, __u32 * uwrq,
+			    struct iw_request_info *info, __u32 *uwrq,
 			    char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 
 	/* for SLEEP MODE */
 	/* If not managed, assume it's ad-hoc */
@@ -906,16 +900,15 @@ static int ks_wlan_set_encode(struct net_device *dev,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	wep_key_t key;
+	struct wep_key key;
 	int index = (dwrq->flags & IW_ENCODE_INDEX);
 	int current_index = priv->reg.wep_index;
 	int i;
 
 	DPRINTK(2, "flags=%04X\n", dwrq->flags);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 
 	/* for SLEEP MODE */
 	/* index check */
@@ -959,9 +952,9 @@ static int ks_wlan_set_encode(struct net_device *dev,
 			}
 			/* Send the key to the card */
 			priv->reg.wep_key[index].size = key.len;
-			for (i = 0; i < (priv->reg.wep_key[index].size); i++) {
+			for (i = 0; i < (priv->reg.wep_key[index].size); i++)
 				priv->reg.wep_key[index].val[i] = key.key[i];
-			}
+
 			priv->need_commit |= (SME_WEP_VAL1 << index);
 			priv->reg.wep_index = index;
 			priv->need_commit |= SME_WEP_INDEX;
@@ -973,9 +966,9 @@ static int ks_wlan_set_encode(struct net_device *dev,
 			priv->reg.wep_key[2].size = 0;
 			priv->reg.wep_key[3].size = 0;
 			priv->reg.privacy_invoked = 0x00;
-			if (priv->reg.authenticate_type == AUTH_TYPE_SHARED_KEY) {
+			if (priv->reg.authenticate_type == AUTH_TYPE_SHARED_KEY)
 				priv->need_commit |= SME_MODE_SET;
-			}
+
 			priv->reg.authenticate_type = AUTH_TYPE_OPEN_SYSTEM;
 			wep_on_off = WEP_OFF;
 			priv->need_commit |= SME_WEP_FLAG;
@@ -997,14 +990,14 @@ static int ks_wlan_set_encode(struct net_device *dev,
 		priv->need_commit |= SME_WEP_FLAG;
 
 	if (dwrq->flags & IW_ENCODE_OPEN) {
-		if (priv->reg.authenticate_type == AUTH_TYPE_SHARED_KEY) {
+		if (priv->reg.authenticate_type == AUTH_TYPE_SHARED_KEY)
 			priv->need_commit |= SME_MODE_SET;
-		}
+
 		priv->reg.authenticate_type = AUTH_TYPE_OPEN_SYSTEM;
 	} else if (dwrq->flags & IW_ENCODE_RESTRICTED) {
-		if (priv->reg.authenticate_type == AUTH_TYPE_OPEN_SYSTEM) {
+		if (priv->reg.authenticate_type == AUTH_TYPE_OPEN_SYSTEM)
 			priv->need_commit |= SME_MODE_SET;
-		}
+
 		priv->reg.authenticate_type = AUTH_TYPE_SHARED_KEY;
 	}
 //      return -EINPROGRESS;            /* Call commit handler */
@@ -1026,9 +1019,9 @@ static int ks_wlan_get_encode(struct net_device *dev,
 	char zeros[16];
 	int index = (dwrq->flags & IW_ENCODE_INDEX) - 1;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	dwrq->flags = IW_ENCODE_DISABLED;
 
@@ -1056,9 +1049,8 @@ static int ks_wlan_get_encode(struct net_device *dev,
 	/* Copy the key to the user buffer */
 	if ((index >= 0) && (index < 4))
 		dwrq->length = priv->reg.wep_key[index].size;
-	if (dwrq->length > 16) {
+	if (dwrq->length > 16)
 		dwrq->length = 0;
-	}
 #if 1	/* IW_ENCODE_NOKEY; */
 	if (dwrq->length) {
 		if ((index >= 0) && (index < 4))
@@ -1086,9 +1078,8 @@ static int ks_wlan_get_txpow(struct net_device *dev,
 			     struct iw_request_info *info,
 			     struct iw_param *vwrq, char *extra)
 {
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 
 	/* for SLEEP MODE */
 	/* Not Support */
@@ -1113,9 +1104,8 @@ static int ks_wlan_get_retry(struct net_device *dev,
 			     struct iw_request_info *info,
 			     struct iw_param *vwrq, char *extra)
 {
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 
 	/* for SLEEP MODE */
 	/* Not Support */
@@ -1139,9 +1129,9 @@ static int ks_wlan_get_range(struct net_device *dev,
 
 	DPRINTK(2, "\n");
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	dwrq->length = sizeof(struct iw_range);
 	memset(range, 0, sizeof(*range));
@@ -1267,9 +1257,9 @@ static int ks_wlan_set_power(struct net_device *dev,
 	    (struct ks_wlan_private *)netdev_priv(dev);
 	short enabled;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	enabled = vwrq->disabled ? 0 : 1;
 	if (enabled == 0) {	/* 0 */
@@ -1301,9 +1291,8 @@ static int ks_wlan_get_power(struct net_device *dev,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	if (priv->reg.powermgt > 0)
 		vwrq->disabled = 0;
@@ -1322,9 +1311,8 @@ static int ks_wlan_get_iwstats(struct net_device *dev,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	vwrq->qual = 0;	/* not supported */
 	vwrq->level = priv->wstats.qual.level;
@@ -1372,9 +1360,8 @@ static int ks_wlan_get_aplist(struct net_device *dev,
 
 	int i;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	for (i = 0; i < priv->aplist.size; i++) {
 		memcpy(address[i].sa_data, &(priv->aplist.ap[i].bssid[0]),
@@ -1404,11 +1391,11 @@ static int ks_wlan_set_scan(struct net_device *dev,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 	struct iw_scan_req *req = NULL;
+
 	DPRINTK(2, "\n");
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 
 	/* for SLEEP MODE */
 	/* specified SSID SCAN */
@@ -1598,11 +1585,11 @@ static int ks_wlan_get_scan(struct net_device *dev,
 	    (struct ks_wlan_private *)netdev_priv(dev);
 	int i;
 	char *current_ev = extra;
+
 	DPRINTK(2, "\n");
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	if (priv->sme_i.sme_flag & SME_AP_SCAN) {
 		DPRINTK(2, "flag AP_SCAN\n");
@@ -1675,9 +1662,8 @@ static int ks_wlan_set_genie(struct net_device *dev,
 
 	DPRINTK(2, "\n");
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	return 0;
 //      return -EOPNOTSUPP;
@@ -1696,26 +1682,23 @@ static int ks_wlan_set_auth_mode(struct net_device *dev,
 
 	DPRINTK(2, "index=%d:value=%08X\n", index, value);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	switch (index) {
 	case IW_AUTH_WPA_VERSION:	/* 0 */
 		switch (value) {
 		case IW_AUTH_WPA_VERSION_DISABLED:
 			priv->wpa.version = value;
-			if (priv->wpa.rsn_enabled) {
+			if (priv->wpa.rsn_enabled)
 				priv->wpa.rsn_enabled = 0;
-			}
 			priv->need_commit |= SME_RSN;
 			break;
 		case IW_AUTH_WPA_VERSION_WPA:
 		case IW_AUTH_WPA_VERSION_WPA2:
 			priv->wpa.version = value;
-			if (!(priv->wpa.rsn_enabled)) {
+			if (!(priv->wpa.rsn_enabled))
 				priv->wpa.rsn_enabled = 1;
-			}
 			priv->need_commit |= SME_RSN;
 			break;
 		default:
@@ -1832,11 +1815,11 @@ static int ks_wlan_get_auth_mode(struct net_device *dev,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 	int index = (vwrq->flags & IW_AUTH_INDEX);
+
 	DPRINTK(2, "index=%d\n", index);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 
 	/* for SLEEP MODE */
 	/*  WPA (not used ?? wpa_supplicant) */
@@ -1886,18 +1869,17 @@ static int ks_wlan_set_encode_ext(struct net_device *dev,
 	DPRINTK(2, "flags=%04X:: ext_flags=%08X\n", dwrq->flags,
 		enc->ext_flags);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	if (index < 1 || index > 4)
 		return -EINVAL;
 	else
 		index--;
 
-	if (dwrq->flags & IW_ENCODE_DISABLED) {
+	if (dwrq->flags & IW_ENCODE_DISABLED)
 		priv->wpa.key[index].key_len = 0;
-	}
 
 	if (enc) {
 		priv->wpa.key[index].ext_flags = enc->ext_flags;
@@ -1986,9 +1968,8 @@ static int ks_wlan_get_encode_ext(struct net_device *dev,
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 
 	/* for SLEEP MODE */
 	/*  WPA (not used ?? wpa_supplicant)
@@ -2015,13 +1996,13 @@ static int ks_wlan_set_pmksa(struct net_device *dev,
 
 	DPRINTK(2, "\n");
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
-	if (!extra) {
+	if (!extra)
 		return -EINVAL;
-	}
+
 	pmksa = (struct iw_pmksa *)extra;
 	DPRINTK(2, "cmd=%d\n", pmksa->cmd);
 
@@ -2141,16 +2122,16 @@ static struct iw_statistics *ks_get_wireless_stats(struct net_device *dev)
 /*------------------------------------------------------------------*/
 /* Private handler : set stop request */
 static int ks_wlan_set_stop_request(struct net_device *dev,
-				    struct iw_request_info *info, __u32 * uwrq,
+				    struct iw_request_info *info, __u32 *uwrq,
 				    char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 	DPRINTK(2, "\n");
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	if (!(*uwrq))
 		return -EINVAL;
@@ -2173,15 +2154,14 @@ static int ks_wlan_set_mlme(struct net_device *dev,
 
 	DPRINTK(2, ":%d :%d\n", mlme->cmd, mlme->reason_code);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	switch (mlme->cmd) {
 	case IW_MLME_DEAUTH:
-		if (mlme->reason_code == WLAN_REASON_MIC_FAILURE) {
+		if (mlme->reason_code == WLAN_REASON_MIC_FAILURE)
 			return 0;
-		}
 	case IW_MLME_DISASSOC:
 		mode = 1;
 		return ks_wlan_set_stop_request(dev, NULL, &mode, NULL);
@@ -2207,14 +2187,14 @@ static int ks_wlan_get_firmware_version(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : set force disconnect status */
 static int ks_wlan_set_detach(struct net_device *dev,
-			      struct iw_request_info *info, __u32 * uwrq,
+			      struct iw_request_info *info, __u32 *uwrq,
 			      char *extra)
 {
 	struct ks_wlan_private *priv = (struct ks_wlan_private *)dev->priv;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	if (*uwrq == CONNECT_STATUS) {	/* 0 */
 		priv->connect_status &= ~FORCE_DISCONNECT;
@@ -2232,14 +2212,14 @@ static int ks_wlan_set_detach(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : get force disconnect status */
 static int ks_wlan_get_detach(struct net_device *dev,
-			      struct iw_request_info *info, __u32 * uwrq,
+			      struct iw_request_info *info, __u32 *uwrq,
 			      char *extra)
 {
 	struct ks_wlan_private *priv = (struct ks_wlan_private *)dev->priv;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	*uwrq = ((priv->connect_status & FORCE_DISCONNECT) ? 1 : 0);
 	return 0;
@@ -2248,14 +2228,14 @@ static int ks_wlan_get_detach(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : get connect status */
 static int ks_wlan_get_connect(struct net_device *dev,
-			       struct iw_request_info *info, __u32 * uwrq,
+			       struct iw_request_info *info, __u32 *uwrq,
 			       char *extra)
 {
 	struct ks_wlan_private *priv = (struct ks_wlan_private *)dev->priv;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	*uwrq = (priv->connect_status & CONNECT_STATUS_MASK);
 	return 0;
@@ -2265,15 +2245,15 @@ static int ks_wlan_get_connect(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : set preamble */
 static int ks_wlan_set_preamble(struct net_device *dev,
-				struct iw_request_info *info, __u32 * uwrq,
+				struct iw_request_info *info, __u32 *uwrq,
 				char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	if (*uwrq == LONG_PREAMBLE) {	/* 0 */
 		priv->reg.preamble = LONG_PREAMBLE;
@@ -2290,15 +2270,15 @@ static int ks_wlan_set_preamble(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : get preamble */
 static int ks_wlan_get_preamble(struct net_device *dev,
-				struct iw_request_info *info, __u32 * uwrq,
+				struct iw_request_info *info, __u32 *uwrq,
 				char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	*uwrq = priv->reg.preamble;
 	return 0;
@@ -2307,15 +2287,15 @@ static int ks_wlan_get_preamble(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : set power save mode */
 static int ks_wlan_set_powermgt(struct net_device *dev,
-				struct iw_request_info *info, __u32 * uwrq,
+				struct iw_request_info *info, __u32 *uwrq,
 				char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	if (*uwrq == POWMGT_ACTIVE_MODE) {	/* 0 */
 		priv->reg.powermgt = POWMGT_ACTIVE_MODE;
@@ -2340,15 +2320,15 @@ static int ks_wlan_set_powermgt(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : get power save made */
 static int ks_wlan_get_powermgt(struct net_device *dev,
-				struct iw_request_info *info, __u32 * uwrq,
+				struct iw_request_info *info, __u32 *uwrq,
 				char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
+
 	/* for SLEEP MODE */
 	*uwrq = priv->reg.powermgt;
 	return 0;
@@ -2357,15 +2337,14 @@ static int ks_wlan_get_powermgt(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : set scan type */
 static int ks_wlan_set_scan_type(struct net_device *dev,
-				 struct iw_request_info *info, __u32 * uwrq,
+				 struct iw_request_info *info, __u32 *uwrq,
 				 char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	if (*uwrq == ACTIVE_SCAN) {	/* 0 */
 		priv->reg.scan_type = ACTIVE_SCAN;
@@ -2380,15 +2359,14 @@ static int ks_wlan_set_scan_type(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : get scan type */
 static int ks_wlan_get_scan_type(struct net_device *dev,
-				 struct iw_request_info *info, __u32 * uwrq,
+				 struct iw_request_info *info, __u32 *uwrq,
 				 char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	*uwrq = priv->reg.scan_type;
 	return 0;
@@ -2404,9 +2382,8 @@ static int ks_wlan_data_write(struct net_device *dev,
 	struct ks_wlan_private *priv = (struct ks_wlan_private *)dev->priv;
 	unsigned char *wbuff = NULL;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	wbuff = (unsigned char *)kmalloc(dwrq->length, GFP_ATOMIC);
 	if (!wbuff)
@@ -2428,9 +2405,8 @@ static int ks_wlan_data_read(struct net_device *dev,
 	struct ks_wlan_private *priv = (struct ks_wlan_private *)dev->priv;
 	unsigned short read_length;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	if (!atomic_read(&priv->event_count)) {
 		if (priv->dev_state < DEVICE_STATE_BOOT) {	/* Remove device */
@@ -2488,9 +2464,8 @@ static int ks_wlan_get_wep_ascii(struct net_device *dev,
 	int i, j, len = 0;
 	char tmp[WEP_ASCII_BUFF_SIZE];
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	strcpy(tmp, " WEP keys ASCII \n");
 	len += strlen(" WEP keys ASCII \n");
@@ -2531,19 +2506,18 @@ static int ks_wlan_get_wep_ascii(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : set beacon lost count */
 static int ks_wlan_set_beacon_lost(struct net_device *dev,
-				   struct iw_request_info *info, __u32 * uwrq,
+				   struct iw_request_info *info, __u32 *uwrq,
 				   char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
-	if (*uwrq >= BEACON_LOST_COUNT_MIN && *uwrq <= BEACON_LOST_COUNT_MAX) {
+	if (*uwrq >= BEACON_LOST_COUNT_MIN && *uwrq <= BEACON_LOST_COUNT_MAX)
 		priv->reg.beacon_lost_count = *uwrq;
-	} else
+	else
 		return -EINVAL;
 
 	if (priv->reg.operation_mode == MODE_INFRASTRUCTURE) {
@@ -2556,15 +2530,14 @@ static int ks_wlan_set_beacon_lost(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : get beacon lost count */
 static int ks_wlan_get_beacon_lost(struct net_device *dev,
-				   struct iw_request_info *info, __u32 * uwrq,
+				   struct iw_request_info *info, __u32 *uwrq,
 				   char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	*uwrq = priv->reg.beacon_lost_count;
 	return 0;
@@ -2573,15 +2546,14 @@ static int ks_wlan_get_beacon_lost(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : set phy type */
 static int ks_wlan_set_phy_type(struct net_device *dev,
-				struct iw_request_info *info, __u32 * uwrq,
+				struct iw_request_info *info, __u32 *uwrq,
 				char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	if (*uwrq == D_11B_ONLY_MODE) {	/* 0 */
 		priv->reg.phy_type = D_11B_ONLY_MODE;
@@ -2599,15 +2571,14 @@ static int ks_wlan_set_phy_type(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : get phy type */
 static int ks_wlan_get_phy_type(struct net_device *dev,
-				struct iw_request_info *info, __u32 * uwrq,
+				struct iw_request_info *info, __u32 *uwrq,
 				char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	*uwrq = priv->reg.phy_type;
 	return 0;
@@ -2616,15 +2587,14 @@ static int ks_wlan_get_phy_type(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : set cts mode */
 static int ks_wlan_set_cts_mode(struct net_device *dev,
-				struct iw_request_info *info, __u32 * uwrq,
+				struct iw_request_info *info, __u32 *uwrq,
 				char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	if (*uwrq == CTS_MODE_FALSE) {	/* 0 */
 		priv->reg.cts_mode = CTS_MODE_FALSE;
@@ -2644,15 +2614,14 @@ static int ks_wlan_set_cts_mode(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : get cts mode */
 static int ks_wlan_get_cts_mode(struct net_device *dev,
-				struct iw_request_info *info, __u32 * uwrq,
+				struct iw_request_info *info, __u32 *uwrq,
 				char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	*uwrq = priv->reg.cts_mode;
 	return 0;
@@ -2662,7 +2631,7 @@ static int ks_wlan_get_cts_mode(struct net_device *dev,
 /* Private handler : set sleep mode */
 static int ks_wlan_set_sleep_mode(struct net_device *dev,
 				  struct iw_request_info *info,
-				  __u32 * uwrq, char *extra)
+				  __u32 *uwrq, char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
@@ -2692,7 +2661,7 @@ static int ks_wlan_set_sleep_mode(struct net_device *dev,
 /* Private handler : get sleep mode */
 static int ks_wlan_get_sleep_mode(struct net_device *dev,
 				  struct iw_request_info *info,
-				  __u32 * uwrq, char *extra)
+				  __u32 *uwrq, char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
@@ -2708,16 +2677,15 @@ static int ks_wlan_get_sleep_mode(struct net_device *dev,
 /* Private handler : set phy information timer */
 static int ks_wlan_set_phy_information_timer(struct net_device *dev,
 					     struct iw_request_info *info,
-					     __u32 * uwrq, char *extra)
+					     __u32 *uwrq, char *extra)
 {
 	struct ks_wlan_private *priv = (struct ks_wlan_private *)dev->priv;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	if (*uwrq >= 0 && *uwrq <= 0xFFFF)	/* 0-65535 */
-		priv->reg.phy_info_timer = (uint16_t) * uwrq;
+		priv->reg.phy_info_timer = (uint16_t)*uwrq;
 	else
 		return -EINVAL;
 
@@ -2730,13 +2698,12 @@ static int ks_wlan_set_phy_information_timer(struct net_device *dev,
 /* Private handler : get phy information timer */
 static int ks_wlan_get_phy_information_timer(struct net_device *dev,
 					     struct iw_request_info *info,
-					     __u32 * uwrq, char *extra)
+					     __u32 *uwrq, char *extra)
 {
 	struct ks_wlan_private *priv = (struct ks_wlan_private *)dev->priv;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	*uwrq = priv->reg.phy_info_timer;
 	return 0;
@@ -2747,16 +2714,15 @@ static int ks_wlan_get_phy_information_timer(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : set WPS enable */
 static int ks_wlan_set_wps_enable(struct net_device *dev,
-				  struct iw_request_info *info, __u32 * uwrq,
+				  struct iw_request_info *info, __u32 *uwrq,
 				  char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 	DPRINTK(2, "\n");
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	if (*uwrq == 0 || *uwrq == 1)
 		priv->wps.wps_enabled = *uwrq;
@@ -2771,16 +2737,15 @@ static int ks_wlan_set_wps_enable(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : get WPS enable */
 static int ks_wlan_get_wps_enable(struct net_device *dev,
-				  struct iw_request_info *info, __u32 * uwrq,
+				  struct iw_request_info *info, __u32 *uwrq,
 				  char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 	DPRINTK(2, "\n");
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	*uwrq = priv->wps.wps_enabled;
 	netdev_info(dev, "return=%d\n", *uwrq);
@@ -2801,16 +2766,14 @@ static int ks_wlan_set_wps_probe_req(struct net_device *dev,
 
 	DPRINTK(2, "\n");
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	DPRINTK(2, "dwrq->length=%d\n", dwrq->length);
 
 	/* length check */
-	if (p[1] + 2 != dwrq->length || dwrq->length > 256) {
+	if (p[1] + 2 != dwrq->length || dwrq->length > 256)
 		return -EINVAL;
-	}
 
 	priv->wps.ielen = p[1] + 2 + 1;	/* IE header + IE + sizeof(len) */
 	len = p[1] + 2;	/* IE header + IE */
@@ -2833,14 +2796,14 @@ static int ks_wlan_set_wps_probe_req(struct net_device *dev,
 /* Private handler : get WPS probe req */
 static int ks_wlan_get_wps_probe_req(struct net_device *dev,
 				     struct iw_request_info *info,
-				     __u32 * uwrq, char *extra)
+				     __u32 *uwrq, char *extra)
 {
 	struct ks_wlan_private *priv = (struct ks_wlan_private *)dev->priv;
+
 	DPRINTK(2, "\n");
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	return 0;
 }
@@ -2850,18 +2813,17 @@ static int ks_wlan_get_wps_probe_req(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : set tx gain control value */
 static int ks_wlan_set_tx_gain(struct net_device *dev,
-			       struct iw_request_info *info, __u32 * uwrq,
+			       struct iw_request_info *info, __u32 *uwrq,
 			       char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	if (*uwrq >= 0 && *uwrq <= 0xFF)	/* 0-255 */
-		priv->gain.TxGain = (uint8_t) * uwrq;
+		priv->gain.TxGain = (uint8_t)*uwrq;
 	else
 		return -EINVAL;
 
@@ -2877,15 +2839,14 @@ static int ks_wlan_set_tx_gain(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : get tx gain control value */
 static int ks_wlan_get_tx_gain(struct net_device *dev,
-			       struct iw_request_info *info, __u32 * uwrq,
+			       struct iw_request_info *info, __u32 *uwrq,
 			       char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	*uwrq = priv->gain.TxGain;
 	hostif_sme_enqueue(priv, SME_GET_GAIN);
@@ -2895,18 +2856,17 @@ static int ks_wlan_get_tx_gain(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : set rx gain control value */
 static int ks_wlan_set_rx_gain(struct net_device *dev,
-			       struct iw_request_info *info, __u32 * uwrq,
+			       struct iw_request_info *info, __u32 *uwrq,
 			       char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	if (*uwrq >= 0 && *uwrq <= 0xFF)	/* 0-255 */
-		priv->gain.RxGain = (uint8_t) * uwrq;
+		priv->gain.RxGain = (uint8_t)*uwrq;
 	else
 		return -EINVAL;
 
@@ -2922,15 +2882,14 @@ static int ks_wlan_set_rx_gain(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : get rx gain control value */
 static int ks_wlan_get_rx_gain(struct net_device *dev,
-			       struct iw_request_info *info, __u32 * uwrq,
+			       struct iw_request_info *info, __u32 *uwrq,
 			       char *extra)
 {
 	struct ks_wlan_private *priv =
 	    (struct ks_wlan_private *)netdev_priv(dev);
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	*uwrq = priv->gain.RxGain;
 	hostif_sme_enqueue(priv, SME_GET_GAIN);
@@ -2941,17 +2900,16 @@ static int ks_wlan_get_rx_gain(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : set region value */
 static int ks_wlan_set_region(struct net_device *dev,
-			      struct iw_request_info *info, __u32 * uwrq,
+			      struct iw_request_info *info, __u32 *uwrq,
 			      char *extra)
 {
 	struct ks_wlan_private *priv = (struct ks_wlan_private *)dev->priv;
 
-	if (priv->sleep_mode == SLP_SLEEP) {
+	if (priv->sleep_mode == SLP_SLEEP)
 		return -EPERM;
-	}
 	/* for SLEEP MODE */
 	if (*uwrq >= 0x9 && *uwrq <= 0xF)	/* 0x9-0xf */
-		priv->region = (uint8_t) * uwrq;
+		priv->region = (uint8_t)*uwrq;
 	else
 		return -EINVAL;
 
@@ -2963,7 +2921,7 @@ static int ks_wlan_set_region(struct net_device *dev,
 /*------------------------------------------------------------------*/
 /* Private handler : get eeprom checksum result */
 static int ks_wlan_get_eeprom_cksum(struct net_device *dev,
-				    struct iw_request_info *info, __u32 * uwrq,
+				    struct iw_request_info *info, __u32 *uwrq,
 				    char *extra)
 {
 	struct ks_wlan_private *priv =
@@ -3090,7 +3048,7 @@ static void print_hif_event(struct net_device *dev, int event)
 /*------------------------------------------------------------------*/
 /* Private handler : get host command history */
 static int ks_wlan_hostt(struct net_device *dev, struct iw_request_info *info,
-			 __u32 * uwrq, char *extra)
+			 __u32 *uwrq, char *extra)
 {
 	int i, event;
 	struct ks_wlan_private *priv =
@@ -3293,6 +3251,7 @@ static int ks_wlan_netdev_ioctl(struct net_device *dev, struct ifreq *rq,
 {
 	int rc = 0;
 	struct iwreq *wrq = (struct iwreq *)rq;
+
 	switch (cmd) {
 	case SIOCIWFIRSTPRIV + 20:	/* KS_WLAN_SET_STOP_REQ */
 		rc = ks_wlan_set_stop_request(dev, NULL, &(wrq->u.mode), NULL);
@@ -3311,9 +3270,8 @@ struct net_device_stats *ks_wlan_get_stats(struct net_device *dev)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
-	if (priv->dev_state < DEVICE_STATE_READY) {
+	if (priv->dev_state < DEVICE_STATE_READY)
 		return NULL;	/* not finished initialize */
-	}
 
 	return &priv->nstats;
 }
@@ -3323,6 +3281,7 @@ int ks_wlan_set_mac_address(struct net_device *dev, void *addr)
 {
 	struct ks_wlan_private *priv = netdev_priv(dev);
 	struct sockaddr *mac_addr = (struct sockaddr *)addr;
+
 	if (netif_running(dev))
 		return -EBUSY;
 	memcpy(dev->dev_addr, mac_addr->sa_data, dev->addr_len);
@@ -3330,10 +3289,7 @@ int ks_wlan_set_mac_address(struct net_device *dev, void *addr)
 
 	priv->mac_address_valid = 0;
 	hostif_sme_enqueue(priv, SME_MACADDRESS_SET_REQUEST);
-	netdev_info(dev,
-	       "ks_wlan: MAC ADDRESS = %02x:%02x:%02x:%02x:%02x:%02x\n",
-	       priv->eth_addr[0], priv->eth_addr[1], priv->eth_addr[2],
-	       priv->eth_addr[3], priv->eth_addr[4], priv->eth_addr[5]);
+	netdev_info(dev, "ks_wlan:  MAC ADDRESS = %pM\n", priv->eth_addr);
 	return 0;
 }
 
@@ -3344,9 +3300,8 @@ void ks_wlan_tx_timeout(struct net_device *dev)
 
 	DPRINTK(1, "head(%d) tail(%d)!!\n", priv->tx_dev.qhead,
 		priv->tx_dev.qtail);
-	if (!netif_queue_stopped(dev)) {
+	if (!netif_queue_stopped(dev))
 		netif_stop_queue(dev);
-	}
 	priv->nstats.tx_errors++;
 	netif_wake_queue(dev);
 }
@@ -3375,9 +3330,8 @@ int ks_wlan_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	netif_trans_update(dev);
 
 	DPRINTK(4, "rc=%d\n", rc);
-	if (rc) {
+	if (rc)
 		rc = 0;
-	}
 
 	return rc;
 }
@@ -3410,9 +3364,8 @@ void ks_wlan_set_multicast_list(struct net_device *dev)
 	struct ks_wlan_private *priv = netdev_priv(dev);
 
 	DPRINTK(4, "\n");
-	if (priv->dev_state < DEVICE_STATE_READY) {
+	if (priv->dev_state < DEVICE_STATE_READY)
 		return;	/* not finished initialize */
-	}
 	hostif_sme_enqueue(priv, SME_MULTICAST_REQUEST);
 }
 
@@ -3426,8 +3379,8 @@ int ks_wlan_open(struct net_device *dev)
 	if (!priv->mac_address_valid) {
 		netdev_err(dev, "ks_wlan : %s Not READY !!\n", dev->name);
 		return -EBUSY;
-	} else
-		netif_start_queue(dev);
+	}
+	netif_start_queue(dev);
 
 	return 0;
 }
@@ -3474,9 +3427,8 @@ int ks_wlan_net_start(struct net_device *dev)
 
 	/* phy information update timer */
 	atomic_set(&update_phyinfo, 0);
-	init_timer(&update_phyinfo_timer);
-	update_phyinfo_timer.function = ks_wlan_update_phyinfo_timeout;
-	update_phyinfo_timer.data = (unsigned long)priv;
+	setup_timer(&update_phyinfo_timer, ks_wlan_update_phyinfo_timeout,
+		    (unsigned long)priv);
 
 	/* dummy address set */
 	memcpy(priv->eth_addr, dummy_addr, ETH_ALEN);
diff --git a/drivers/staging/ks7010/michael_mic.c b/drivers/staging/ks7010/michael_mic.c
index 78ae2b8fb7f3..2f535c08e172 100644
--- a/drivers/staging/ks7010/michael_mic.c
+++ b/drivers/staging/ks7010/michael_mic.c
@@ -14,10 +14,11 @@
 #include "michael_mic.h"
 
 // Rotation functions on 32 bit values
-#define ROL32( A, n ) 	( ((A) << (n)) | ( ((A)>>(32-(n))) & ( (1UL << (n)) - 1 ) ) )
-#define ROR32( A, n ) 	ROL32( (A), 32-(n) )
+#define ROL32(A, n)	(((A) << (n)) | (((A)>>(32-(n))) & ((1UL << (n)) - 1)))
+#define ROR32(A, n)	ROL32((A), 32-(n))
 // Convert from Byte[] to UInt32 in a portable way
-#define getUInt32( A, B ) 	(uint32_t)(A[B+0] << 0) + (A[B+1] << 8) + (A[B+2] << 16) + (A[B+3] << 24)
+#define getUInt32(A, B)	((uint32_t)(A[B+0] << 0) \
+		+ (A[B+1] << 8) + (A[B+2] << 16) + (A[B+3] << 24))
 
 // Convert from UInt32 to Byte[] in a portable way
 #define putUInt32(A, B, C)					\
@@ -48,21 +49,22 @@ void MichaelInitializeFunction(struct michel_mic_t *Mic, uint8_t *key)
 }
 
 #define MichaelBlockFunction(L, R)				\
-do{								\
-	R ^= ROL32( L, 17 );					\
+do {								\
+	R ^= ROL32(L, 17);					\
 	L += R;							\
 	R ^= ((L & 0xff00ff00) >> 8) | ((L & 0x00ff00ff) << 8);	\
 	L += R;							\
-	R ^= ROL32( L, 3 );					\
+	R ^= ROL32(L, 3);					\
 	L += R;							\
-	R ^= ROR32( L, 2 );					\
+	R ^= ROR32(L, 2);					\
 	L += R;							\
-}while(0)
+} while (0)
 
 static
 void MichaelAppend(struct michel_mic_t *Mic, uint8_t *src, int nBytes)
 {
 	int addlen;
+
 	if (Mic->nBytesInM) {
 		addlen = 4 - Mic->nBytesInM;
 		if (addlen > nBytes)
@@ -96,7 +98,8 @@ void MichaelAppend(struct michel_mic_t *Mic, uint8_t *src, int nBytes)
 static
 void MichaelGetMIC(struct michel_mic_t *Mic, uint8_t *dst)
 {
-	uint8_t *data = Mic->M;
+	u8 *data = Mic->M;
+
 	switch (Mic->nBytesInM) {
 	case 0:
 		Mic->L ^= 0x5a;
@@ -122,11 +125,11 @@ void MichaelGetMIC(struct michel_mic_t *Mic, uint8_t *dst)
 	MichaelClear(Mic);
 }
 
-void MichaelMICFunction(struct michel_mic_t *Mic, uint8_t *Key,
-			uint8_t *Data, int Len, uint8_t priority,
-			uint8_t *Result)
+void MichaelMICFunction(struct michel_mic_t *Mic, u8 *Key,
+			u8 *Data, int Len, u8 priority,
+			u8 *Result)
 {
-	uint8_t pad_data[4] = { priority, 0, 0, 0 };
+	u8 pad_data[4] = { priority, 0, 0, 0 };
 	// Compute the MIC value
 	/*
 	 * IEEE802.11i  page 47
diff --git a/drivers/staging/ks7010/michael_mic.h b/drivers/staging/ks7010/michael_mic.h
index efaa21788fc7..248f849fc4a5 100644
--- a/drivers/staging/ks7010/michael_mic.h
+++ b/drivers/staging/ks7010/michael_mic.h
@@ -11,15 +11,15 @@
 
 /* MichelMIC routine define */
 struct michel_mic_t {
-	uint32_t K0;	// Key 
-	uint32_t K1;	// Key 
-	uint32_t L;	// Current state 
-	uint32_t R;	// Current state 
-	uint8_t M[4];	// Message accumulator (single word) 
-	int nBytesInM;	// # bytes in M 
-	uint8_t Result[8];
+	u32 K0;	// Key
+	u32 K1;	// Key
+	u32 L;	// Current state
+	u32 R;	// Current state
+	u8 M[4];	// Message accumulator (single word)
+	int nBytesInM;	// # bytes in M
+	u8 Result[8];
 };
 
-void MichaelMICFunction(struct michel_mic_t *Mic, uint8_t *Key,
-			uint8_t *Data, int Len, uint8_t priority,
-			uint8_t *Result);
+void MichaelMICFunction(struct michel_mic_t *Mic, u8 *Key,
+			u8 *Data, int Len, u8 priority,
+			u8 *Result);
diff --git a/drivers/staging/lustre/include/linux/libcfs/curproc.h b/drivers/staging/lustre/include/linux/libcfs/curproc.h
index be0675d8ff5e..1ea27c9e3708 100644
--- a/drivers/staging/lustre/include/linux/libcfs/curproc.h
+++ b/drivers/staging/lustre/include/linux/libcfs/curproc.h
@@ -53,7 +53,7 @@
 #define current_pid()		(current->pid)
 #define current_comm()		(current->comm)
 
-typedef __u32 cfs_cap_t;
+typedef u32 cfs_cap_t;
 
 #define CFS_CAP_CHOWN		   0
 #define CFS_CAP_DAC_OVERRIDE	    1
@@ -65,15 +65,15 @@ typedef __u32 cfs_cap_t;
 #define CFS_CAP_SYS_BOOT	       23
 #define CFS_CAP_SYS_RESOURCE	   24
 
-#define CFS_CAP_FS_MASK ((1 << CFS_CAP_CHOWN) |		 \
-			 (1 << CFS_CAP_DAC_OVERRIDE) |	  \
-			 (1 << CFS_CAP_DAC_READ_SEARCH) |       \
-			 (1 << CFS_CAP_FOWNER) |		\
-			 (1 << CFS_CAP_FSETID) |	       \
-			 (1 << CFS_CAP_LINUX_IMMUTABLE) |       \
-			 (1 << CFS_CAP_SYS_ADMIN) |	     \
-			 (1 << CFS_CAP_SYS_BOOT) |	      \
-			 (1 << CFS_CAP_SYS_RESOURCE))
+#define CFS_CAP_FS_MASK (BIT(CFS_CAP_CHOWN) |		\
+			 BIT(CFS_CAP_DAC_OVERRIDE) |	\
+			 BIT(CFS_CAP_DAC_READ_SEARCH) |	\
+			 BIT(CFS_CAP_FOWNER) |		\
+			 BIT(CFS_CAP_FSETID) |		\
+			 BIT(CFS_CAP_LINUX_IMMUTABLE) | \
+			 BIT(CFS_CAP_SYS_ADMIN) |	\
+			 BIT(CFS_CAP_SYS_BOOT) |	\
+			 BIT(CFS_CAP_SYS_RESOURCE))
 
 void cfs_cap_raise(cfs_cap_t cap);
 void cfs_cap_lower(cfs_cap_t cap);
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs.h b/drivers/staging/lustre/include/linux/libcfs/libcfs.h
index 3b92d38d37e2..cc2c0e97bb7e 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs.h
@@ -61,7 +61,7 @@
 sigset_t cfs_block_allsigs(void);
 sigset_t cfs_block_sigs(unsigned long sigs);
 sigset_t cfs_block_sigsinv(unsigned long sigs);
-void cfs_restore_sigs(sigset_t);
+void cfs_restore_sigs(sigset_t sigset);
 void cfs_clear_sigpending(void);
 
 /*
@@ -71,7 +71,7 @@ void cfs_clear_sigpending(void);
 /* returns a random 32-bit integer */
 unsigned int cfs_rand(void);
 /* seed the generator */
-void cfs_srand(unsigned int, unsigned int);
+void cfs_srand(unsigned int seed1, unsigned int seed2);
 void cfs_get_random_bytes(void *buf, int size);
 
 #include "libcfs_debug.h"
@@ -125,7 +125,6 @@ extern struct miscdevice libcfs_dev;
 /**
  * The path of debug log dump upcall script.
  */
-extern char lnet_upcall[1024];
 extern char lnet_debug_log_upcall[1024];
 
 extern struct cfs_wi_sched *cfs_sched_rehash;
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_cpu.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_cpu.h
index 81d8079e3b5e..6d8752a368fa 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_cpu.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_cpu.h
@@ -92,7 +92,7 @@ struct cfs_cpt_table {
 	/* node mask */
 	nodemask_t		ctb_nodemask;
 	/* version */
-	__u64			ctb_version;
+	u64			ctb_version;
 };
 
 static inline cpumask_t *
@@ -211,7 +211,7 @@ int cfs_cpu_ht_nsiblings(int cpu);
  */
 void *cfs_percpt_alloc(struct cfs_cpt_table *cptab, unsigned int size);
 /*
- * destory per-cpu-partition variable
+ * destroy per-cpu-partition variable
  */
 void cfs_percpt_free(void *vars);
 int cfs_percpt_number(void *vars);
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_crypto.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_crypto.h
index 02be7d7608a5..8f34c5ddc63e 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_crypto.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_crypto.h
@@ -29,10 +29,12 @@
 #define _LIBCFS_CRYPTO_H
 
 struct cfs_crypto_hash_type {
-	char		*cht_name;      /**< hash algorithm name, equal to
-					 * format name for crypto api */
-	unsigned int    cht_key;	/**< init key by default (valid for
-					 * 4 bytes context like crc32, adler */
+	char		*cht_name;      /*< hash algorithm name, equal to
+					 * format name for crypto api
+					 */
+	unsigned int    cht_key;	/*< init key by default (valid for
+					 * 4 bytes context like crc32, adler
+					 */
 	unsigned int    cht_size;       /**< hash digest size */
 };
 
@@ -135,7 +137,7 @@ static inline unsigned char cfs_crypto_hash_alg(const char *algname)
 	enum cfs_crypto_hash_alg hash_alg;
 
 	for (hash_alg = 0; hash_alg < CFS_HASH_ALG_MAX; hash_alg++)
-		if (strcmp(hash_types[hash_alg].cht_name, algname) == 0)
+		if (!strcmp(hash_types[hash_alg].cht_name, algname))
 			return hash_alg;
 
 	return CFS_HASH_ALG_UNKNOWN;
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_fail.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_fail.h
index bdbbe934584c..fedb46dff696 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_fail.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_fail.h
@@ -39,8 +39,8 @@ extern int cfs_fail_err;
 extern wait_queue_head_t cfs_race_waitq;
 extern int cfs_race_state;
 
-int __cfs_fail_check_set(__u32 id, __u32 value, int set);
-int __cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set);
+int __cfs_fail_check_set(u32 id, u32 value, int set);
+int __cfs_fail_timeout_set(u32 id, u32 value, int ms, int set);
 
 enum {
 	CFS_FAIL_LOC_NOSET      = 0,
@@ -55,11 +55,11 @@ enum {
 
 #define CFS_FAILED_BIT       30
 /* CFS_FAILED is 0x40000000 */
-#define CFS_FAILED	  (1 << CFS_FAILED_BIT)
+#define CFS_FAILED		BIT(CFS_FAILED_BIT)
 
 #define CFS_FAIL_ONCE_BIT    31
 /* CFS_FAIL_ONCE is 0x80000000 */
-#define CFS_FAIL_ONCE       (1 << CFS_FAIL_ONCE_BIT)
+#define CFS_FAIL_ONCE		BIT(CFS_FAIL_ONCE_BIT)
 
 /* The following flags aren't made to be combined */
 #define CFS_FAIL_SKIP	0x20000000 /* skip N times then fail */
@@ -69,14 +69,14 @@ enum {
 
 #define CFS_FAULT	0x02000000 /* match any CFS_FAULT_CHECK */
 
-static inline bool CFS_FAIL_PRECHECK(__u32 id)
+static inline bool CFS_FAIL_PRECHECK(u32 id)
 {
-	return cfs_fail_loc != 0 &&
+	return cfs_fail_loc &&
 	       ((cfs_fail_loc & CFS_FAIL_MASK_LOC) == (id & CFS_FAIL_MASK_LOC) ||
-	        (cfs_fail_loc & id & CFS_FAULT));
+		(cfs_fail_loc & id & CFS_FAULT));
 }
 
-static inline int cfs_fail_check_set(__u32 id, __u32 value,
+static inline int cfs_fail_check_set(u32 id, u32 value,
 				     int set, int quiet)
 {
 	int ret = 0;
@@ -103,28 +103,34 @@ static inline int cfs_fail_check_set(__u32 id, __u32 value,
 #define CFS_FAIL_CHECK_QUIET(id) \
 	cfs_fail_check_set(id, 0, CFS_FAIL_LOC_NOSET, 1)
 
-/* If id hit cfs_fail_loc and cfs_fail_val == (-1 or value) return 1,
- * otherwise return 0 */
+/*
+ * If id hit cfs_fail_loc and cfs_fail_val == (-1 or value) return 1,
+ * otherwise return 0
+ */
 #define CFS_FAIL_CHECK_VALUE(id, value) \
 	cfs_fail_check_set(id, value, CFS_FAIL_LOC_VALUE, 0)
 #define CFS_FAIL_CHECK_VALUE_QUIET(id, value) \
 	cfs_fail_check_set(id, value, CFS_FAIL_LOC_VALUE, 1)
 
-/* If id hit cfs_fail_loc, cfs_fail_loc |= value and return 1,
- * otherwise return 0 */
+/*
+ * If id hit cfs_fail_loc, cfs_fail_loc |= value and return 1,
+ * otherwise return 0
+ */
 #define CFS_FAIL_CHECK_ORSET(id, value) \
 	cfs_fail_check_set(id, value, CFS_FAIL_LOC_ORSET, 0)
 #define CFS_FAIL_CHECK_ORSET_QUIET(id, value) \
 	cfs_fail_check_set(id, value, CFS_FAIL_LOC_ORSET, 1)
 
-/* If id hit cfs_fail_loc, cfs_fail_loc = value and return 1,
- * otherwise return 0 */
+/*
+ * If id hit cfs_fail_loc, cfs_fail_loc = value and return 1,
+ * otherwise return 0
+ */
 #define CFS_FAIL_CHECK_RESET(id, value) \
 	cfs_fail_check_set(id, value, CFS_FAIL_LOC_RESET, 0)
 #define CFS_FAIL_CHECK_RESET_QUIET(id, value) \
 	cfs_fail_check_set(id, value, CFS_FAIL_LOC_RESET, 1)
 
-static inline int cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
+static inline int cfs_fail_timeout_set(u32 id, u32 value, int ms, int set)
 {
 	if (unlikely(CFS_FAIL_PRECHECK(id)))
 		return __cfs_fail_timeout_set(id, value, ms, set);
@@ -138,8 +144,10 @@ static inline int cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
 #define CFS_FAIL_TIMEOUT_MS(id, ms) \
 	cfs_fail_timeout_set(id, 0, ms, CFS_FAIL_LOC_NOSET)
 
-/* If id hit cfs_fail_loc, cfs_fail_loc |= value and
- * sleep seconds or milliseconds */
+/*
+ * If id hit cfs_fail_loc, cfs_fail_loc |= value and
+ * sleep seconds or milliseconds
+ */
 #define CFS_FAIL_TIMEOUT_ORSET(id, value, secs) \
 	cfs_fail_timeout_set(id, value, secs * 1000, CFS_FAIL_LOC_ORSET)
 
@@ -152,13 +160,14 @@ static inline int cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
 #define CFS_FAULT_CHECK(id)			\
 	CFS_FAIL_CHECK(CFS_FAULT | (id))
 
-/* The idea here is to synchronise two threads to force a race. The
+/*
+ * The idea here is to synchronise two threads to force a race. The
  * first thread that calls this with a matching fail_loc is put to
  * sleep. The next thread that calls with the same fail_loc wakes up
- * the first and continues. */
-static inline void cfs_race(__u32 id)
+ * the first and continues.
+ */
+static inline void cfs_race(u32 id)
 {
-
 	if (CFS_FAIL_PRECHECK(id)) {
 		if (unlikely(__cfs_fail_check_set(id, 0, CFS_FAIL_LOC_NOSET))) {
 			int rc;
@@ -166,7 +175,7 @@ static inline void cfs_race(__u32 id)
 			cfs_race_state = 0;
 			CERROR("cfs_race id %x sleeping\n", id);
 			rc = wait_event_interruptible(cfs_race_waitq,
-						      cfs_race_state != 0);
+						      !!cfs_race_state);
 			CERROR("cfs_fail_race id %x awake, rc=%d\n", id, rc);
 		} else {
 			CERROR("cfs_fail_race id %x waking\n", id);
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_hash.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_hash.h
index 6949a1846635..0cc2fc465c1a 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_hash.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_hash.h
@@ -57,8 +57,10 @@
 
 /** disable debug */
 #define CFS_HASH_DEBUG_NONE	0
-/** record hash depth and output to console when it's too deep,
- *  computing overhead is low but consume more memory */
+/*
+ * record hash depth and output to console when it's too deep,
+ * computing overhead is low but consume more memory
+ */
 #define CFS_HASH_DEBUG_1	1
 /** expensive, check key validation */
 #define CFS_HASH_DEBUG_2	2
@@ -87,8 +89,8 @@ union cfs_hash_lock {
  */
 struct cfs_hash_bucket {
 	union cfs_hash_lock	hsb_lock;	/**< bucket lock */
-	__u32			hsb_count;	/**< current entries */
-	__u32			hsb_version;	/**< change version */
+	u32			hsb_count;	/**< current entries */
+	u32			hsb_version;	/**< change version */
 	unsigned int		hsb_index;	/**< index of bucket */
 	int			hsb_depmax;	/**< max depth on bucket */
 	long			hsb_head[0];	/**< hash-head array */
@@ -123,38 +125,40 @@ enum cfs_hash_tag {
 	 *  . Some functions will be disabled with this flag, i.e:
 	 *    cfs_hash_for_each_empty, cfs_hash_rehash
 	 */
-	CFS_HASH_NO_LOCK	= 1 << 0,
+	CFS_HASH_NO_LOCK	= BIT(0),
 	/** no bucket lock, use one spinlock to protect the whole hash */
-	CFS_HASH_NO_BKTLOCK	= 1 << 1,
+	CFS_HASH_NO_BKTLOCK	= BIT(1),
 	/** rwlock to protect bucket */
-	CFS_HASH_RW_BKTLOCK	= 1 << 2,
+	CFS_HASH_RW_BKTLOCK	= BIT(2),
 	/** spinlock to protect bucket */
-	CFS_HASH_SPIN_BKTLOCK	= 1 << 3,
+	CFS_HASH_SPIN_BKTLOCK	= BIT(3),
 	/** always add new item to tail */
-	CFS_HASH_ADD_TAIL	= 1 << 4,
+	CFS_HASH_ADD_TAIL	= BIT(4),
 	/** hash-table doesn't have refcount on item */
-	CFS_HASH_NO_ITEMREF	= 1 << 5,
+	CFS_HASH_NO_ITEMREF	= BIT(5),
 	/** big name for param-tree */
-	CFS_HASH_BIGNAME	= 1 << 6,
+	CFS_HASH_BIGNAME	= BIT(6),
 	/** track global count */
-	CFS_HASH_COUNTER	= 1 << 7,
+	CFS_HASH_COUNTER	= BIT(7),
 	/** rehash item by new key */
-	CFS_HASH_REHASH_KEY	= 1 << 8,
+	CFS_HASH_REHASH_KEY	= BIT(8),
 	/** Enable dynamic hash resizing */
-	CFS_HASH_REHASH		= 1 << 9,
+	CFS_HASH_REHASH		= BIT(9),
 	/** can shrink hash-size */
-	CFS_HASH_SHRINK		= 1 << 10,
+	CFS_HASH_SHRINK		= BIT(10),
 	/** assert hash is empty on exit */
-	CFS_HASH_ASSERT_EMPTY	= 1 << 11,
+	CFS_HASH_ASSERT_EMPTY	= BIT(11),
 	/** record hlist depth */
-	CFS_HASH_DEPTH		= 1 << 12,
+	CFS_HASH_DEPTH		= BIT(12),
 	/**
 	 * rehash is always scheduled in a different thread, so current
 	 * change on hash table is non-blocking
 	 */
-	CFS_HASH_NBLK_CHANGE	= 1 << 13,
-	/** NB, we typed hs_flags as  __u16, please change it
-	 * if you need to extend >=16 flags */
+	CFS_HASH_NBLK_CHANGE	= BIT(13),
+	/**
+	 * NB, we typed hs_flags as  u16, please change it
+	 * if you need to extend >=16 flags
+	 */
 };
 
 /** most used attributes */
@@ -201,8 +205,10 @@ enum cfs_hash_tag {
  */
 
 struct cfs_hash {
-	/** serialize with rehash, or serialize all operations if
-	 * the hash-table has CFS_HASH_NO_BKTLOCK */
+	/**
+	 * serialize with rehash, or serialize all operations if
+	 * the hash-table has CFS_HASH_NO_BKTLOCK
+	 */
 	union cfs_hash_lock		hs_lock;
 	/** hash operations */
 	struct cfs_hash_ops		*hs_ops;
@@ -215,31 +221,31 @@ struct cfs_hash {
 	/** total number of items on this hash-table */
 	atomic_t			hs_count;
 	/** hash flags, see cfs_hash_tag for detail */
-	__u16				hs_flags;
+	u16				hs_flags;
 	/** # of extra-bytes for bucket, for user saving extended attributes */
-	__u16				hs_extra_bytes;
+	u16				hs_extra_bytes;
 	/** wants to iterate */
-	__u8				hs_iterating;
+	u8				hs_iterating;
 	/** hash-table is dying */
-	__u8				hs_exiting;
+	u8				hs_exiting;
 	/** current hash bits */
-	__u8				hs_cur_bits;
+	u8				hs_cur_bits;
 	/** min hash bits */
-	__u8				hs_min_bits;
+	u8				hs_min_bits;
 	/** max hash bits */
-	__u8				hs_max_bits;
+	u8				hs_max_bits;
 	/** bits for rehash */
-	__u8				hs_rehash_bits;
+	u8				hs_rehash_bits;
 	/** bits for each bucket */
-	__u8				hs_bkt_bits;
+	u8				hs_bkt_bits;
 	/** resize min threshold */
-	__u16				hs_min_theta;
+	u16				hs_min_theta;
 	/** resize max threshold */
-	__u16				hs_max_theta;
+	u16				hs_max_theta;
 	/** resize count */
-	__u32				hs_rehash_count;
+	u32				hs_rehash_count;
 	/** # of iterators (caller of cfs_hash_for_each_*) */
-	__u32				hs_iterators;
+	u32				hs_iterators;
 	/** rehash workitem */
 	struct cfs_workitem		hs_rehash_wi;
 	/** refcount on this hash table */
@@ -291,8 +297,8 @@ struct cfs_hash_hlist_ops {
 
 struct cfs_hash_ops {
 	/** return hashed value from @key */
-	unsigned (*hs_hash)(struct cfs_hash *hs, const void *key,
-			    unsigned mask);
+	unsigned int (*hs_hash)(struct cfs_hash *hs, const void *key,
+				unsigned int mask);
 	/** return key address of @hnode */
 	void *   (*hs_key)(struct hlist_node *hnode);
 	/** copy key from @hnode to @key */
@@ -317,110 +323,112 @@ struct cfs_hash_ops {
 
 /** total number of buckets in @hs */
 #define CFS_HASH_NBKT(hs)	\
-	(1U << ((hs)->hs_cur_bits - (hs)->hs_bkt_bits))
+	BIT((hs)->hs_cur_bits - (hs)->hs_bkt_bits)
 
 /** total number of buckets in @hs while rehashing */
 #define CFS_HASH_RH_NBKT(hs)	\
-	(1U << ((hs)->hs_rehash_bits - (hs)->hs_bkt_bits))
+	BIT((hs)->hs_rehash_bits - (hs)->hs_bkt_bits)
 
 /** number of hlist for in bucket */
-#define CFS_HASH_BKT_NHLIST(hs) (1U << (hs)->hs_bkt_bits)
+#define CFS_HASH_BKT_NHLIST(hs)	BIT((hs)->hs_bkt_bits)
 
 /** total number of hlist in @hs */
-#define CFS_HASH_NHLIST(hs)     (1U << (hs)->hs_cur_bits)
+#define CFS_HASH_NHLIST(hs)	BIT((hs)->hs_cur_bits)
 
 /** total number of hlist in @hs while rehashing */
-#define CFS_HASH_RH_NHLIST(hs)  (1U << (hs)->hs_rehash_bits)
+#define CFS_HASH_RH_NHLIST(hs)	BIT((hs)->hs_rehash_bits)
 
 static inline int
 cfs_hash_with_no_lock(struct cfs_hash *hs)
 {
 	/* caller will serialize all operations for this hash-table */
-	return (hs->hs_flags & CFS_HASH_NO_LOCK) != 0;
+	return hs->hs_flags & CFS_HASH_NO_LOCK;
 }
 
 static inline int
 cfs_hash_with_no_bktlock(struct cfs_hash *hs)
 {
 	/* no bucket lock, one single lock to protect the hash-table */
-	return (hs->hs_flags & CFS_HASH_NO_BKTLOCK) != 0;
+	return hs->hs_flags & CFS_HASH_NO_BKTLOCK;
 }
 
 static inline int
 cfs_hash_with_rw_bktlock(struct cfs_hash *hs)
 {
 	/* rwlock to protect hash bucket */
-	return (hs->hs_flags & CFS_HASH_RW_BKTLOCK) != 0;
+	return hs->hs_flags & CFS_HASH_RW_BKTLOCK;
 }
 
 static inline int
 cfs_hash_with_spin_bktlock(struct cfs_hash *hs)
 {
 	/* spinlock to protect hash bucket */
-	return (hs->hs_flags & CFS_HASH_SPIN_BKTLOCK) != 0;
+	return hs->hs_flags & CFS_HASH_SPIN_BKTLOCK;
 }
 
 static inline int
 cfs_hash_with_add_tail(struct cfs_hash *hs)
 {
-	return (hs->hs_flags & CFS_HASH_ADD_TAIL) != 0;
+	return hs->hs_flags & CFS_HASH_ADD_TAIL;
 }
 
 static inline int
 cfs_hash_with_no_itemref(struct cfs_hash *hs)
 {
-	/* hash-table doesn't keep refcount on item,
+	/*
+	 * hash-table doesn't keep refcount on item,
 	 * item can't be removed from hash unless it's
-	 * ZERO refcount */
-	return (hs->hs_flags & CFS_HASH_NO_ITEMREF) != 0;
+	 * ZERO refcount
+	 */
+	return hs->hs_flags & CFS_HASH_NO_ITEMREF;
 }
 
 static inline int
 cfs_hash_with_bigname(struct cfs_hash *hs)
 {
-	return (hs->hs_flags & CFS_HASH_BIGNAME) != 0;
+	return hs->hs_flags & CFS_HASH_BIGNAME;
 }
 
 static inline int
 cfs_hash_with_counter(struct cfs_hash *hs)
 {
-	return (hs->hs_flags & CFS_HASH_COUNTER) != 0;
+	return hs->hs_flags & CFS_HASH_COUNTER;
 }
 
 static inline int
 cfs_hash_with_rehash(struct cfs_hash *hs)
 {
-	return (hs->hs_flags & CFS_HASH_REHASH) != 0;
+	return hs->hs_flags & CFS_HASH_REHASH;
 }
 
 static inline int
 cfs_hash_with_rehash_key(struct cfs_hash *hs)
 {
-	return (hs->hs_flags & CFS_HASH_REHASH_KEY) != 0;
+	return hs->hs_flags & CFS_HASH_REHASH_KEY;
 }
 
 static inline int
 cfs_hash_with_shrink(struct cfs_hash *hs)
 {
-	return (hs->hs_flags & CFS_HASH_SHRINK) != 0;
+	return hs->hs_flags & CFS_HASH_SHRINK;
 }
 
 static inline int
 cfs_hash_with_assert_empty(struct cfs_hash *hs)
 {
-	return (hs->hs_flags & CFS_HASH_ASSERT_EMPTY) != 0;
+	return hs->hs_flags & CFS_HASH_ASSERT_EMPTY;
 }
 
 static inline int
 cfs_hash_with_depth(struct cfs_hash *hs)
 {
-	return (hs->hs_flags & CFS_HASH_DEPTH) != 0;
+	return hs->hs_flags & CFS_HASH_DEPTH;
 }
 
 static inline int
 cfs_hash_with_nblk_change(struct cfs_hash *hs)
 {
-	return (hs->hs_flags & CFS_HASH_NBLK_CHANGE) != 0;
+	return hs->hs_flags & CFS_HASH_NBLK_CHANGE;
 }
 
 static inline int
@@ -434,14 +442,14 @@ static inline int
 cfs_hash_is_rehashing(struct cfs_hash *hs)
 {
 	/* rehash is launched */
-	return hs->hs_rehash_bits != 0;
+	return !!hs->hs_rehash_bits;
 }
 
 static inline int
 cfs_hash_is_iterating(struct cfs_hash *hs)
 {
 	/* someone is calling cfs_hash_for_each_* */
-	return hs->hs_iterating || hs->hs_iterators != 0;
+	return hs->hs_iterating || hs->hs_iterators;
 }
 
 static inline int
@@ -453,7 +461,7 @@ cfs_hash_bkt_size(struct cfs_hash *hs)
 }
 
 static inline unsigned
-cfs_hash_id(struct cfs_hash *hs, const void *key, unsigned mask)
+cfs_hash_id(struct cfs_hash *hs, const void *key, unsigned int mask)
 {
 	return hs->hs_ops->hs_hash(hs, key, mask);
 }
@@ -562,7 +570,7 @@ cfs_hash_bd_index_get(struct cfs_hash *hs, struct cfs_hash_bd *bd)
 }
 
 static inline void
-cfs_hash_bd_index_set(struct cfs_hash *hs, unsigned index,
+cfs_hash_bd_index_set(struct cfs_hash *hs, unsigned int index,
 		      struct cfs_hash_bd *bd)
 {
 	bd->bd_bucket = hs->hs_buckets[index >> hs->hs_bkt_bits];
@@ -576,14 +584,14 @@ cfs_hash_bd_extra_get(struct cfs_hash *hs, struct cfs_hash_bd *bd)
 	       cfs_hash_bkt_size(hs) - hs->hs_extra_bytes;
 }
 
-static inline __u32
+static inline u32
 cfs_hash_bd_version_get(struct cfs_hash_bd *bd)
 {
 	/* need hold cfs_hash_bd_lock */
 	return bd->bd_bucket->hsb_version;
 }
 
-static inline __u32
+static inline u32
 cfs_hash_bd_count_get(struct cfs_hash_bd *bd)
 {
 	/* need hold cfs_hash_bd_lock */
@@ -669,10 +677,10 @@ cfs_hash_dual_bd_finddel_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds,
 
 /* Hash init/cleanup functions */
 struct cfs_hash *
-cfs_hash_create(char *name, unsigned cur_bits, unsigned max_bits,
-		unsigned bkt_bits, unsigned extra_bytes,
-		unsigned min_theta, unsigned max_theta,
-		struct cfs_hash_ops *ops, unsigned flags);
+cfs_hash_create(char *name, unsigned int cur_bits, unsigned int max_bits,
+		unsigned int bkt_bits, unsigned int extra_bytes,
+		unsigned int min_theta, unsigned int max_theta,
+		struct cfs_hash_ops *ops, unsigned int flags);
 
 struct cfs_hash *cfs_hash_getref(struct cfs_hash *hs);
 void cfs_hash_putref(struct cfs_hash *hs);
@@ -700,27 +708,28 @@ typedef int (*cfs_hash_for_each_cb_t)(struct cfs_hash *hs,
 void *
 cfs_hash_lookup(struct cfs_hash *hs, const void *key);
 void
-cfs_hash_for_each(struct cfs_hash *hs, cfs_hash_for_each_cb_t, void *data);
+cfs_hash_for_each(struct cfs_hash *hs, cfs_hash_for_each_cb_t cb, void *data);
 void
-cfs_hash_for_each_safe(struct cfs_hash *hs, cfs_hash_for_each_cb_t, void *data);
+cfs_hash_for_each_safe(struct cfs_hash *hs, cfs_hash_for_each_cb_t cb,
+		       void *data);
 int
-cfs_hash_for_each_nolock(struct cfs_hash *hs, cfs_hash_for_each_cb_t,
-			 void *data);
+cfs_hash_for_each_nolock(struct cfs_hash *hs, cfs_hash_for_each_cb_t cb,
+			 void *data, int start);
 int
-cfs_hash_for_each_empty(struct cfs_hash *hs, cfs_hash_for_each_cb_t,
+cfs_hash_for_each_empty(struct cfs_hash *hs, cfs_hash_for_each_cb_t cb,
 			void *data);
 void
 cfs_hash_for_each_key(struct cfs_hash *hs, const void *key,
-		      cfs_hash_for_each_cb_t, void *data);
+		      cfs_hash_for_each_cb_t cb, void *data);
 typedef int (*cfs_hash_cond_opt_cb_t)(void *obj, void *data);
 void
-cfs_hash_cond_del(struct cfs_hash *hs, cfs_hash_cond_opt_cb_t, void *data);
+cfs_hash_cond_del(struct cfs_hash *hs, cfs_hash_cond_opt_cb_t cb, void *data);
 
 void
-cfs_hash_hlist_for_each(struct cfs_hash *hs, unsigned hindex,
-			cfs_hash_for_each_cb_t, void *data);
+cfs_hash_hlist_for_each(struct cfs_hash *hs, unsigned int hindex,
+			cfs_hash_for_each_cb_t cb, void *data);
 int  cfs_hash_is_empty(struct cfs_hash *hs);
-__u64 cfs_hash_size_get(struct cfs_hash *hs);
+u64 cfs_hash_size_get(struct cfs_hash *hs);
 
 /*
  * Rehash - Theta is calculated to be the average chained
@@ -766,8 +775,8 @@ cfs_hash_bucket_validate(struct cfs_hash *hs, struct cfs_hash_bd *bd,
 #endif /* CFS_HASH_DEBUG_LEVEL */
 
 #define CFS_HASH_THETA_BITS	10
-#define CFS_HASH_MIN_THETA	(1U << (CFS_HASH_THETA_BITS - 1))
-#define CFS_HASH_MAX_THETA	(1U << (CFS_HASH_THETA_BITS + 1))
+#define CFS_HASH_MIN_THETA	BIT(CFS_HASH_THETA_BITS - 1)
+#define CFS_HASH_MAX_THETA	BIT(CFS_HASH_THETA_BITS + 1)
 
 /* Return integer component of theta */
 static inline int __cfs_hash_theta_int(int theta)
@@ -792,8 +801,8 @@ static inline void
 __cfs_hash_set_theta(struct cfs_hash *hs, int min, int max)
 {
 	LASSERT(min < max);
-	hs->hs_min_theta = (__u16)min;
-	hs->hs_max_theta = (__u16)max;
+	hs->hs_min_theta = (u16)min;
+	hs->hs_max_theta = (u16)max;
 }
 
 /* Generic debug formatting routines mainly for proc handler */
@@ -805,11 +814,11 @@ void cfs_hash_debug_str(struct cfs_hash *hs, struct seq_file *m);
  * Generic djb2 hash algorithm for character arrays.
  */
 static inline unsigned
-cfs_hash_djb2_hash(const void *key, size_t size, unsigned mask)
+cfs_hash_djb2_hash(const void *key, size_t size, unsigned int mask)
 {
-	unsigned i, hash = 5381;
+	unsigned int i, hash = 5381;
 
-	LASSERT(key != NULL);
+	LASSERT(key);
 
 	for (i = 0; i < size; i++)
 		hash = hash * 33 + ((char *)key)[i];
@@ -821,7 +830,7 @@ cfs_hash_djb2_hash(const void *key, size_t size, unsigned mask)
  * Generic u32 hash algorithm.
  */
 static inline unsigned
-cfs_hash_u32_hash(const __u32 key, unsigned mask)
+cfs_hash_u32_hash(const u32 key, unsigned int mask)
 {
 	return ((key * CFS_GOLDEN_RATIO_PRIME_32) & mask);
 }
@@ -830,9 +839,9 @@ cfs_hash_u32_hash(const __u32 key, unsigned mask)
  * Generic u64 hash algorithm.
  */
 static inline unsigned
-cfs_hash_u64_hash(const __u64 key, unsigned mask)
+cfs_hash_u64_hash(const u64 key, unsigned int mask)
 {
-	return ((unsigned)(key * CFS_GOLDEN_RATIO_PRIME_64) & mask);
+	return ((unsigned int)(key * CFS_GOLDEN_RATIO_PRIME_64) & mask);
 }
 
 /** iterate over all buckets in @bds (array of struct cfs_hash_bd) */
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h
index e0e1a5d0949d..aab15d8112a4 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h
@@ -75,7 +75,7 @@ do {									\
 
 #define KLASSERT(e) LASSERT(e)
 
-void __noreturn lbug_with_loc(struct libcfs_debug_msg_data *);
+void __noreturn lbug_with_loc(struct libcfs_debug_msg_data *msg);
 
 #define LBUG()							  \
 do {								    \
@@ -96,7 +96,7 @@ do {									    \
 
 #define LIBCFS_ALLOC_POST(ptr, size)					    \
 do {									    \
-	if (unlikely((ptr) == NULL)) {					    \
+	if (unlikely(!(ptr))) {						    \
 		CERROR("LNET: out of memory at %s:%d (tried to alloc '"	    \
 		       #ptr "' = %d)\n", __FILE__, __LINE__, (int)(size));  \
 	} else {							    \
@@ -147,7 +147,7 @@ do {									    \
 
 #define LIBCFS_FREE(ptr, size)					  \
 do {								    \
-	if (unlikely((ptr) == NULL)) {				  \
+	if (unlikely(!(ptr))) {						\
 		CERROR("LIBCFS: free NULL '" #ptr "' (%d bytes) at "    \
 		       "%s:%d\n", (int)(size), __FILE__, __LINE__);	\
 		break;						  \
@@ -169,8 +169,6 @@ do {								    \
 #define ntohs(x) ___ntohs(x)
 #endif
 
-void libcfs_run_upcall(char **argv);
-void libcfs_run_lbug_upcall(struct libcfs_debug_msg_data *);
 void libcfs_debug_dumplog(void);
 int libcfs_debug_init(unsigned long bufsize);
 int libcfs_debug_cleanup(void);
@@ -280,7 +278,7 @@ do {							    \
 #define CFS_FREE_PTR(ptr)       LIBCFS_FREE(ptr, sizeof(*(ptr)))
 
 /** Compile-time assertion.
-
+ *
  * Check an invariant described by a constant expression at compile time by
  * forcing a compiler error if it does not hold.  \a cond must be a constant
  * expression as defined by the ISO C Standard:
@@ -306,7 +304,8 @@ do {							    \
 /* --------------------------------------------------------------------
  * Light-weight trace
  * Support for temporary event tracing with minimal Heisenberg effect.
- * -------------------------------------------------------------------- */
+ * --------------------------------------------------------------------
+ */
 
 #define MKSTR(ptr) ((ptr)) ? (ptr) : ""
 
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_string.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_string.h
index 0ee60ff336f2..41795d9b3b9b 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_string.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_string.h
@@ -62,9 +62,9 @@ struct cfs_range_expr {
 	 * Link to cfs_expr_list::el_exprs.
 	 */
 	struct list_head	re_link;
-	__u32		re_lo;
-	__u32		re_hi;
-	__u32		re_stride;
+	u32		re_lo;
+	u32		re_hi;
+	u32		re_stride;
 };
 
 struct cfs_expr_list {
@@ -74,24 +74,26 @@ struct cfs_expr_list {
 
 char *cfs_trimwhite(char *str);
 int cfs_gettok(struct cfs_lstr *next, char delim, struct cfs_lstr *res);
-int cfs_str2num_check(char *str, int nob, unsigned *num,
-		      unsigned min, unsigned max);
-int cfs_expr_list_match(__u32 value, struct cfs_expr_list *expr_list);
+int cfs_str2num_check(char *str, int nob, unsigned int *num,
+		      unsigned int min, unsigned int max);
+int cfs_expr_list_match(u32 value, struct cfs_expr_list *expr_list);
 int cfs_expr_list_print(char *buffer, int count,
 			struct cfs_expr_list *expr_list);
 int cfs_expr_list_values(struct cfs_expr_list *expr_list,
-			 int max, __u32 **values);
+			 int max, u32 **values);
 static inline void
-cfs_expr_list_values_free(__u32 *values, int num)
+cfs_expr_list_values_free(u32 *values, int num)
 {
-	/* This array is allocated by LIBCFS_ALLOC(), so it shouldn't be freed
+	/*
+	 * This array is allocated by LIBCFS_ALLOC(), so it shouldn't be freed
 	 * by OBD_FREE() if it's called by module other than libcfs & LNet,
-	 * otherwise we will see fake memory leak */
+	 * otherwise we will see fake memory leak
+	 */
 	LIBCFS_FREE(values, num * sizeof(values[0]));
 }
 
 void cfs_expr_list_free(struct cfs_expr_list *expr_list);
-int cfs_expr_list_parse(char *str, int len, unsigned min, unsigned max,
+int cfs_expr_list_parse(char *str, int len, unsigned int min, unsigned int max,
 			struct cfs_expr_list **elpp);
 void cfs_expr_list_free_list(struct list_head *list);
 
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_workitem.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_workitem.h
index a7e1340e69a1..2accd9a85472 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_workitem.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_workitem.h
@@ -62,9 +62,9 @@
 
 struct cfs_wi_sched;
 
-void cfs_wi_sched_destroy(struct cfs_wi_sched *);
+void cfs_wi_sched_destroy(struct cfs_wi_sched *sched);
 int cfs_wi_sched_create(char *name, struct cfs_cpt_table *cptab, int cpt,
-			int nthrs, struct cfs_wi_sched **);
+			int nthrs, struct cfs_wi_sched **sched_pp);
 
 struct cfs_workitem;
 
diff --git a/drivers/staging/lustre/include/linux/libcfs/linux/linux-cpu.h b/drivers/staging/lustre/include/linux/libcfs/linux/linux-cpu.h
index f63cb47bc309..dd0cd0442b86 100644
--- a/drivers/staging/lustre/include/linux/libcfs/linux/linux-cpu.h
+++ b/drivers/staging/lustre/include/linux/libcfs/linux/linux-cpu.h
@@ -52,17 +52,17 @@ struct cfs_cpu_partition {
 	/* nodes mask for this partition */
 	nodemask_t			*cpt_nodemask;
 	/* spread rotor for NUMA allocator */
-	unsigned			cpt_spread_rotor;
+	unsigned int			cpt_spread_rotor;
 };
 
 /** descriptor for CPU partitions */
 struct cfs_cpt_table {
 	/* version, reserved for hotplug */
-	unsigned			ctb_version;
+	unsigned int			ctb_version;
 	/* spread rotor for NUMA allocator */
-	unsigned			ctb_spread_rotor;
+	unsigned int			ctb_spread_rotor;
 	/* # of CPU partitions */
-	unsigned			ctb_nparts;
+	unsigned int			ctb_nparts;
 	/* partitions tables */
 	struct cfs_cpu_partition	*ctb_parts;
 	/* shadow HW CPU to CPU partition ID */
diff --git a/drivers/staging/lustre/include/linux/libcfs/linux/linux-time.h b/drivers/staging/lustre/include/linux/libcfs/linux/linux-time.h
index b646acd1f7e7..709e1ce98d8d 100644
--- a/drivers/staging/lustre/include/linux/libcfs/linux/linux-time.h
+++ b/drivers/staging/lustre/include/linux/libcfs/linux/linux-time.h
@@ -76,23 +76,23 @@ static inline long cfs_duration_sec(long d)
 
 #define cfs_time_current_64 get_jiffies_64
 
-static inline __u64 cfs_time_add_64(__u64 t, __u64 d)
+static inline u64 cfs_time_add_64(u64 t, u64 d)
 {
 	return t + d;
 }
 
-static inline __u64 cfs_time_shift_64(int seconds)
+static inline u64 cfs_time_shift_64(int seconds)
 {
 	return cfs_time_add_64(cfs_time_current_64(),
 			       cfs_time_seconds(seconds));
 }
 
-static inline int cfs_time_before_64(__u64 t1, __u64 t2)
+static inline int cfs_time_before_64(u64 t1, u64 t2)
 {
 	return (__s64)t2 - (__s64)t1 > 0;
 }
 
-static inline int cfs_time_beforeq_64(__u64 t1, __u64 t2)
+static inline int cfs_time_beforeq_64(u64 t1, u64 t2)
 {
 	return (__s64)t2 - (__s64)t1 >= 0;
 }
diff --git a/drivers/staging/lustre/include/linux/lnet/lnetst.h b/drivers/staging/lustre/include/linux/lnet/lnetst.h
index 417044552d3f..8a84888635ff 100644
--- a/drivers/staging/lustre/include/linux/lnet/lnetst.h
+++ b/drivers/staging/lustre/include/linux/lnet/lnetst.h
@@ -244,7 +244,7 @@ typedef struct {
 	int		 lstio_ses_timeout;	/* IN: session timeout */
 	int		 lstio_ses_force;	/* IN: force create ? */
 	/** IN: session features */
-	unsigned	 lstio_ses_feats;
+	unsigned int	 lstio_ses_feats;
 	lst_sid_t __user *lstio_ses_idp;	/* OUT: session id */
 	int		 lstio_ses_nmlen;	/* IN: name length */
 	char __user	 *lstio_ses_namep;	/* IN: session name */
@@ -255,7 +255,7 @@ typedef struct {
 	lst_sid_t __user	*lstio_ses_idp;		/* OUT: session id */
 	int __user		*lstio_ses_keyp;	/* OUT: local key */
 	/** OUT: session features */
-	unsigned __user		*lstio_ses_featp;
+	unsigned int __user	*lstio_ses_featp;
 	lstcon_ndlist_ent_t __user *lstio_ses_ndinfo;	/* OUT: */
 	int			 lstio_ses_nmlen;	/* IN: name length */
 	char __user		*lstio_ses_namep;	/* OUT: session name */
@@ -328,7 +328,7 @@ typedef struct {
 	char __user		*lstio_grp_namep;	/* IN: group name */
 	int			 lstio_grp_count;	/* IN: # of nodes */
 	/** OUT: session features */
-	unsigned __user		*lstio_grp_featp;
+	unsigned int __user	*lstio_grp_featp;
 	lnet_process_id_t __user *lstio_grp_idsp;	/* IN: nodes */
 	struct list_head __user	*lstio_grp_resultp;	/* OUT: list head of
 								result buffer */
@@ -490,6 +490,8 @@ typedef struct {
 	int	blk_size;       /* size (bytes) */
 	int	blk_time;       /* time of running the test*/
 	int	blk_flags;      /* reserved flags */
+	int	blk_cli_off;	/* bulk offset on client */
+	int	blk_srv_off;	/* reserved: bulk offset on server */
 } lst_test_bulk_param_t;
 
 typedef struct {
diff --git a/drivers/staging/lustre/include/linux/lnet/types.h b/drivers/staging/lustre/include/linux/lnet/types.h
index f8be0e2f7bf7..8ca1e9d0cfe2 100644
--- a/drivers/staging/lustre/include/linux/lnet/types.h
+++ b/drivers/staging/lustre/include/linux/lnet/types.h
@@ -34,6 +34,7 @@
 #define __LNET_TYPES_H__
 
 #include <linux/types.h>
+#include <linux/bvec.h>
 
 /** \addtogroup lnet
  * @{
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
index 9e8802181452..7f761b327166 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1489,7 +1489,7 @@ out_fpo:
 static void kiblnd_fail_fmr_poolset(struct kib_fmr_poolset *fps,
 				    struct list_head *zombies)
 {
-	if (!fps->fps_net) /* intialized? */
+	if (!fps->fps_net) /* initialized? */
 		return;
 
 	spin_lock(&fps->fps_lock);
@@ -1637,7 +1637,7 @@ int kiblnd_fmr_pool_map(struct kib_fmr_poolset *fps, struct kib_tx *tx,
 {
 	__u64 *pages = tx->tx_pages;
 	bool is_rx = (rd != tx->tx_rd);
-        bool tx_pages_mapped = 0;
+	bool tx_pages_mapped = false;
 	struct kib_fmr_pool *fpo;
 	int npages = 0;
 	__u64 version;
@@ -1812,7 +1812,7 @@ static void kiblnd_destroy_pool_list(struct list_head *head)
 
 static void kiblnd_fail_poolset(struct kib_poolset *ps, struct list_head *zombies)
 {
-	if (!ps->ps_net) /* intialized? */
+	if (!ps->ps_net) /* initialized? */
 		return;
 
 	spin_lock(&ps->ps_lock);
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
index b27de8888149..c7917abf9944 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -1912,12 +1912,12 @@ kiblnd_close_conn_locked(struct kib_conn *conn, int error)
 		       libcfs_nid2str(peer->ibp_nid));
 	} else {
 		CNETERR("Closing conn to %s: error %d%s%s%s%s%s\n",
-		        libcfs_nid2str(peer->ibp_nid), error,
-		        list_empty(&conn->ibc_tx_queue) ? "" : "(sending)",
-		        list_empty(&conn->ibc_tx_noops) ? "" : "(sending_noops)",
-		        list_empty(&conn->ibc_tx_queue_rsrvd) ? "" : "(sending_rsrvd)",
-		        list_empty(&conn->ibc_tx_queue_nocred) ? "" : "(sending_nocred)",
-		        list_empty(&conn->ibc_active_txs) ? "" : "(waiting)");
+			libcfs_nid2str(peer->ibp_nid), error,
+			list_empty(&conn->ibc_tx_queue) ? "" : "(sending)",
+			list_empty(&conn->ibc_tx_noops) ? "" : "(sending_noops)",
+			list_empty(&conn->ibc_tx_queue_rsrvd) ? "" : "(sending_rsrvd)",
+			list_empty(&conn->ibc_tx_queue_nocred) ? "" : "(sending_nocred)",
+			list_empty(&conn->ibc_active_txs) ? "" : "(waiting)");
 	}
 
 	dev = ((struct kib_net *)peer->ibp_ni->ni_data)->ibn_dev;
@@ -2643,7 +2643,7 @@ kiblnd_check_reconnect(struct kib_conn *conn, int version,
 	if (incarnation)
 		peer->ibp_incarnation = incarnation;
 out:
-        write_unlock_irqrestore(glock, flags);
+	write_unlock_irqrestore(glock, flags);
 
 	CNETERR("%s: %s (%s), %x, %x, msg_size: %d, queue_depth: %d/%d, max_frags: %d/%d\n",
 		libcfs_nid2str(peer->ibp_nid),
@@ -2651,7 +2651,7 @@ out:
 		reason, IBLND_MSG_VERSION, version, msg_size,
 		conn->ibc_queue_depth, queue_dep,
 		conn->ibc_max_frags, frag_num);
-        /**
+	/**
 	 * if conn::ibc_reconnect is TRUE, connd will reconnect to the peer
 	 * while destroying the zombie
 	 */
@@ -2976,7 +2976,7 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
 	case RDMA_CM_EVENT_ADDR_ERROR:
 		peer = (struct kib_peer *)cmid->context;
 		CNETERR("%s: ADDR ERROR %d\n",
-		        libcfs_nid2str(peer->ibp_nid), event->status);
+			libcfs_nid2str(peer->ibp_nid), event->status);
 		kiblnd_peer_connect_failed(peer, 1, -EHOSTUNREACH);
 		kiblnd_peer_decref(peer);
 		return -EHOSTUNREACH;      /* rc destroys cmid */
@@ -3021,7 +3021,7 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
 			return kiblnd_active_connect(cmid);
 
 		CNETERR("Can't resolve route for %s: %d\n",
-		        libcfs_nid2str(peer->ibp_nid), event->status);
+			libcfs_nid2str(peer->ibp_nid), event->status);
 		kiblnd_peer_connect_failed(peer, 1, event->status);
 		kiblnd_peer_decref(peer);
 		return event->status;	   /* rc destroys cmid */
@@ -3031,7 +3031,7 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event)
 		LASSERT(conn->ibc_state == IBLND_CONN_ACTIVE_CONNECT ||
 			conn->ibc_state == IBLND_CONN_PASSIVE_WAIT);
 		CNETERR("%s: UNREACHABLE %d\n",
-		        libcfs_nid2str(conn->ibc_peer->ibp_nid), event->status);
+			libcfs_nid2str(conn->ibc_peer->ibp_nid), event->status);
 		kiblnd_connreq_done(conn, -ENETDOWN);
 		kiblnd_conn_decref(conn);
 		return 0;
@@ -3269,14 +3269,14 @@ kiblnd_disconnect_conn(struct kib_conn *conn)
 #define KIB_RECONN_HIGH_RACE	10
 /**
  * Allow connd to take a break and handle other things after consecutive
- * reconnection attemps.
+ * reconnection attempts.
  */
 #define KIB_RECONN_BREAK	100
 
 int
 kiblnd_connd(void *arg)
 {
-	spinlock_t *lock= &kiblnd_data.kib_connd_lock;
+	spinlock_t *lock = &kiblnd_data.kib_connd_lock;
 	wait_queue_t wait;
 	unsigned long flags;
 	struct kib_conn *conn;
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.c b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.c
index cbc9a9c5385f..b74cf635faee 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.c
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.c
@@ -96,7 +96,8 @@ ksocknal_destroy_route(struct ksock_route *route)
 }
 
 static int
-ksocknal_create_peer(struct ksock_peer **peerp, lnet_ni_t *ni, lnet_process_id_t id)
+ksocknal_create_peer(struct ksock_peer **peerp, lnet_ni_t *ni,
+		     lnet_process_id_t id)
 {
 	int cpt = lnet_cpt_of_nid(id.nid);
 	struct ksock_net *net = ni->ni_data;
@@ -319,7 +320,8 @@ ksocknal_get_peer_info(lnet_ni_t *ni, int index,
 }
 
 static void
-ksocknal_associate_route_conn_locked(struct ksock_route *route, struct ksock_conn *conn)
+ksocknal_associate_route_conn_locked(struct ksock_route *route,
+				     struct ksock_conn *conn)
 {
 	struct ksock_peer *peer = route->ksnr_peer;
 	int type = conn->ksnc_type;
@@ -821,7 +823,8 @@ ksocknal_select_ips(struct ksock_peer *peer, __u32 *peerips, int n_peerips)
 				if (k < peer->ksnp_n_passive_ips) /* using it already */
 					continue;
 
-				k = ksocknal_match_peerip(iface, peerips, n_peerips);
+				k = ksocknal_match_peerip(iface, peerips,
+							  n_peerips);
 				xor = ip ^ peerips[k];
 				this_netmatch = !(xor & iface->ksni_netmask) ? 1 : 0;
 
@@ -1302,8 +1305,11 @@ ksocknal_create_conn(lnet_ni_t *ni, struct ksock_route *route,
 
 	/* Take packets blocking for this connection. */
 	list_for_each_entry_safe(tx, txtmp, &peer->ksnp_tx_queue, tx_list) {
-		if (conn->ksnc_proto->pro_match_tx(conn, tx, tx->tx_nonblk) == SOCKNAL_MATCH_NO)
-				continue;
+		int match = conn->ksnc_proto->pro_match_tx(conn, tx,
+							   tx->tx_nonblk);
+
+		if (match == SOCKNAL_MATCH_NO)
+			continue;
 
 		list_del(&tx->tx_list);
 		ksocknal_queue_tx_locked(tx, conn);
@@ -1493,8 +1499,8 @@ ksocknal_close_conn_locked(struct ksock_conn *conn, int error)
 			spin_unlock_bh(&conn->ksnc_scheduler->kss_lock);
 		}
 
-		peer->ksnp_proto = NULL;	/* renegotiate protocol version */
-		peer->ksnp_error = error;       /* stash last conn close reason */
+		peer->ksnp_proto = NULL;  /* renegotiate protocol version */
+		peer->ksnp_error = error; /* stash last conn close reason */
 
 		if (list_empty(&peer->ksnp_routes)) {
 			/*
@@ -1786,7 +1792,8 @@ ksocknal_close_matching_conns(lnet_process_id_t id, __u32 ipaddr)
 			      (id.pid == LNET_PID_ANY || id.pid == peer->ksnp_id.pid)))
 				continue;
 
-			count += ksocknal_close_peer_conns_locked(peer, ipaddr, 0);
+			count += ksocknal_close_peer_conns_locked(peer, ipaddr,
+								  0);
 		}
 	}
 
@@ -2026,7 +2033,10 @@ ksocknal_add_interface(lnet_ni_t *ni, __u32 ipaddress, __u32 netmask)
 		}
 
 		rc = 0;
-		/* NB only new connections will pay attention to the new interface! */
+		/*
+		 * NB only new connections will pay attention to the
+		 * new interface!
+		 */
 	}
 
 	write_unlock_bh(&ksocknal_data.ksnd_global_lock);
@@ -2200,8 +2210,9 @@ ksocknal_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg)
 		int txmem;
 		int rxmem;
 		int nagle;
-		struct ksock_conn *conn = ksocknal_get_conn_by_idx(ni, data->ioc_count);
+		struct ksock_conn *conn;
 
+		conn = ksocknal_get_conn_by_idx(ni, data->ioc_count);
 		if (!conn)
 			return -ENOENT;
 
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h
index e6ca0cf52691..842c45393b38 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h
@@ -84,7 +84,8 @@ struct ksock_sched {				/* per scheduler state */
 	struct list_head        kss_zombie_noop_txs; /* zombie noop tx list */
 	wait_queue_head_t       kss_waitq;	/* where scheduler sleeps */
 	int                     kss_nconns;     /* # connections assigned to
-						 * this scheduler */
+						 * this scheduler
+						 */
 	struct ksock_sched_info *kss_info;	/* owner of it */
 };
 
@@ -110,15 +111,19 @@ struct ksock_interface {			/* in-use interface */
 
 struct ksock_tunables {
 	int          *ksnd_timeout;            /* "stuck" socket timeout
-						* (seconds) */
+						* (seconds)
+						*/
 	int          *ksnd_nscheds;            /* # scheduler threads in each
-						* pool while starting */
+						* pool while starting
+						*/
 	int          *ksnd_nconnds;            /* # connection daemons */
 	int          *ksnd_nconnds_max;        /* max # connection daemons */
 	int          *ksnd_min_reconnectms;    /* first connection retry after
-						* (ms)... */
+						* (ms)...
+						*/
 	int          *ksnd_max_reconnectms;    /* ...exponentially increasing to
-						* this */
+						* this
+						*/
 	int          *ksnd_eager_ack;          /* make TCP ack eagerly? */
 	int          *ksnd_typed_conns;        /* drive sockets by type? */
 	int          *ksnd_min_bulk;           /* smallest "large" message */
@@ -126,9 +131,11 @@ struct ksock_tunables {
 	int          *ksnd_rx_buffer_size;     /* socket rx buffer size */
 	int          *ksnd_nagle;              /* enable NAGLE? */
 	int          *ksnd_round_robin;        /* round robin for multiple
-						* interfaces */
+						* interfaces
+						*/
 	int          *ksnd_keepalive;          /* # secs for sending keepalive
-						* NOOP */
+						* NOOP
+						*/
 	int          *ksnd_keepalive_idle;     /* # idle secs before 1st probe
 						*/
 	int          *ksnd_keepalive_count;    /* # probes */
@@ -137,20 +144,26 @@ struct ksock_tunables {
 	int          *ksnd_peertxcredits;      /* # concurrent sends to 1 peer
 						*/
 	int          *ksnd_peerrtrcredits;     /* # per-peer router buffer
-						* credits */
+						* credits
+						*/
 	int          *ksnd_peertimeout;        /* seconds to consider peer dead
 						*/
 	int          *ksnd_enable_csum;        /* enable check sum */
 	int          *ksnd_inject_csum_error;  /* set non-zero to inject
-						* checksum error */
+						* checksum error
+						*/
 	int          *ksnd_nonblk_zcack;       /* always send zc-ack on
-						* non-blocking connection */
+						* non-blocking connection
+						*/
 	unsigned int *ksnd_zc_min_payload;     /* minimum zero copy payload
-						* size */
+						* size
+						*/
 	int          *ksnd_zc_recv;            /* enable ZC receive (for
-						* Chelsio TOE) */
+						* Chelsio TOE)
+						*/
 	int          *ksnd_zc_recv_min_nfrags; /* minimum # of fragments to
-						* enable ZC receive */
+						* enable ZC receive
+						*/
 };
 
 struct ksock_net {
@@ -174,9 +187,11 @@ struct ksock_nal_data {
 	int                     ksnd_nnets;             /* # networks set up */
 	struct list_head        ksnd_nets;              /* list of nets */
 	rwlock_t                ksnd_global_lock;       /* stabilize peer/conn
-							 * ops */
+							 * ops
+							 */
 	struct list_head        *ksnd_peers;            /* hash table of all my
-							 * known peers */
+							 * known peers
+							 */
 	int                     ksnd_peer_hash_size;    /* size of ksnd_peers */
 
 	int                     ksnd_nthreads;          /* # live threads */
@@ -187,11 +202,14 @@ struct ksock_nal_data {
 	atomic_t                ksnd_nactive_txs;       /* #active txs */
 
 	struct list_head        ksnd_deathrow_conns;    /* conns to close:
-							 * reaper_lock*/
+							 * reaper_lock
+							 */
 	struct list_head        ksnd_zombie_conns;      /* conns to free:
-							 * reaper_lock */
+							 * reaper_lock
+							 */
 	struct list_head        ksnd_enomem_conns;      /* conns to retry:
-							 * reaper_lock*/
+							 * reaper_lock
+							 */
 	wait_queue_head_t       ksnd_reaper_waitq;      /* reaper sleeps here */
 	unsigned long	        ksnd_reaper_waketime;   /* when reaper will wake
 							 */
@@ -201,30 +219,34 @@ struct ksock_nal_data {
 	int                     ksnd_stall_tx;          /* test sluggish sender
 							 */
 	int                     ksnd_stall_rx;          /* test sluggish
-							 * receiver */
-
+							 * receiver
+							 */
 	struct list_head        ksnd_connd_connreqs;    /* incoming connection
-							 * requests */
+							 * requests
+							 */
 	struct list_head        ksnd_connd_routes;      /* routes waiting to be
-							 * connected */
+							 * connected
+							 */
 	wait_queue_head_t       ksnd_connd_waitq;       /* connds sleep here */
 	int                     ksnd_connd_connecting;  /* # connds connecting
 							 */
 	time64_t                ksnd_connd_failed_stamp;/* time stamp of the
 							 * last failed
-							 * connecting attempt */
+							 * connecting attempt
+							 */
 	time64_t                ksnd_connd_starting_stamp;/* time stamp of the
 							   * last starting connd
 							   */
-	unsigned                ksnd_connd_starting;    /* # starting connd */
-	unsigned                ksnd_connd_running;     /* # running connd */
+	unsigned int		ksnd_connd_starting;	/* # starting connd */
+	unsigned int		ksnd_connd_running;	/* # running connd */
 	spinlock_t              ksnd_connd_lock;        /* serialise */
 
 	struct list_head        ksnd_idle_noop_txs;     /* list head for freed
-							 * noop tx */
+							 * noop tx
+							 */
 	spinlock_t              ksnd_tx_lock;           /* serialise, g_lock
-							 * unsafe */
-
+							 * unsafe
+							 */
 };
 
 #define SOCKNAL_INIT_NOTHING 0
@@ -304,18 +326,21 @@ struct ksock_conn {
 	struct list_head   ksnc_list;         /* stash on peer's conn list */
 	struct socket      *ksnc_sock;        /* actual socket */
 	void               *ksnc_saved_data_ready;  /* socket's original
-						     * data_ready() callback */
+						     * data_ready() callback
+						     */
 	void               *ksnc_saved_write_space; /* socket's original
-						     * write_space() callback */
+						     * write_space() callback
+						     */
 	atomic_t           ksnc_conn_refcount;/* conn refcount */
 	atomic_t           ksnc_sock_refcount;/* sock refcount */
 	struct ksock_sched *ksnc_scheduler;	/* who schedules this connection
-					         */
+						 */
 	__u32              ksnc_myipaddr;     /* my IP */
 	__u32              ksnc_ipaddr;       /* peer's IP */
 	int                ksnc_port;         /* peer's port */
 	signed int         ksnc_type:3;       /* type of connection, should be
-					       * signed value */
+					       * signed value
+					       */
 	unsigned int       ksnc_closing:1;    /* being shut down */
 	unsigned int       ksnc_flip:1;       /* flip or not, only for V2.x */
 	unsigned int       ksnc_zc_capable:1; /* enable to ZC */
@@ -323,9 +348,11 @@ struct ksock_conn {
 
 	/* reader */
 	struct list_head   ksnc_rx_list;      /* where I enq waiting input or a
-					       * forwarding descriptor */
+					       * forwarding descriptor
+					       */
 	unsigned long      ksnc_rx_deadline;  /* when (in jiffies) receive times
-					       * out */
+					       * out
+					       */
 	__u8               ksnc_rx_started;   /* started receiving a message */
 	__u8               ksnc_rx_ready;     /* data ready to read */
 	__u8               ksnc_rx_scheduled; /* being progressed */
@@ -338,7 +365,8 @@ struct ksock_conn {
 	lnet_kiov_t        *ksnc_rx_kiov;     /* the page frags */
 	union ksock_rxiovspace ksnc_rx_iov_space; /* space for frag descriptors */
 	__u32              ksnc_rx_csum;      /* partial checksum for incoming
-					       * data */
+					       * data
+					       */
 	void               *ksnc_cookie;      /* rx lnet_finalize passthru arg
 					       */
 	ksock_msg_t        ksnc_msg;          /* incoming message buffer:
@@ -346,14 +374,16 @@ struct ksock_conn {
 					       * whole struct
 					       * V1.x message is a bare
 					       * lnet_hdr_t, it's stored in
-					       * ksnc_msg.ksm_u.lnetmsg */
-
+					       * ksnc_msg.ksm_u.lnetmsg
+					       */
 	/* WRITER */
 	struct list_head   ksnc_tx_list;      /* where I enq waiting for output
-					       * space */
+					       * space
+					       */
 	struct list_head   ksnc_tx_queue;     /* packets waiting to be sent */
-	struct ksock_tx         *ksnc_tx_carrier;  /* next TX that can carry a LNet
-					       * message or ZC-ACK */
+	struct ksock_tx	  *ksnc_tx_carrier;   /* next TX that can carry a LNet
+					       * message or ZC-ACK
+					       */
 	unsigned long      ksnc_tx_deadline;  /* when (in jiffies) tx times out
 					       */
 	int                ksnc_tx_bufnob;    /* send buffer marker */
@@ -361,7 +391,8 @@ struct ksock_conn {
 	int		   ksnc_tx_ready;     /* write space */
 	int		   ksnc_tx_scheduled; /* being progressed */
 	unsigned long      ksnc_tx_last_post; /* time stamp of the last posted
-					       * TX */
+					       * TX
+					       */
 };
 
 struct ksock_route {
@@ -370,20 +401,24 @@ struct ksock_route {
 	struct ksock_peer *ksnr_peer;          /* owning peer */
 	atomic_t          ksnr_refcount;       /* # users */
 	unsigned long     ksnr_timeout;        /* when (in jiffies) reconnection
-						* can happen next */
+						* can happen next
+						*/
 	long              ksnr_retry_interval; /* how long between retries */
 	__u32             ksnr_myipaddr;       /* my IP */
 	__u32             ksnr_ipaddr;         /* IP address to connect to */
 	int               ksnr_port;           /* port to connect to */
 	unsigned int      ksnr_scheduled:1;    /* scheduled for attention */
 	unsigned int      ksnr_connecting:1;   /* connection establishment in
-						* progress */
+						* progress
+						*/
 	unsigned int      ksnr_connected:4;    /* connections established by
-						* type */
+						* type
+						*/
 	unsigned int      ksnr_deleted:1;      /* been removed from peer? */
 	unsigned int      ksnr_share_count;    /* created explicitly? */
 	int               ksnr_conn_count;     /* # conns established by this
-						* route */
+						* route
+						*/
 };
 
 #define SOCKNAL_KEEPALIVE_PING 1 /* cookie for keepalive ping */
@@ -391,7 +426,8 @@ struct ksock_route {
 struct ksock_peer {
 	struct list_head   ksnp_list;           /* stash on global peer list */
 	unsigned long      ksnp_last_alive;     /* when (in jiffies) I was last
-						 * alive */
+						 * alive
+						 */
 	lnet_process_id_t  ksnp_id;             /* who's on the other end(s) */
 	atomic_t           ksnp_refcount;       /* # users */
 	int                ksnp_sharecount;     /* lconf usage counter */
@@ -408,7 +444,8 @@ struct ksock_peer {
 	struct list_head   ksnp_tx_queue;       /* waiting packets */
 	spinlock_t         ksnp_lock;           /* serialize, g_lock unsafe */
 	struct list_head   ksnp_zc_req_list;    /* zero copy requests wait for
-						 * ACK  */
+						 * ACK
+						 */
 	unsigned long      ksnp_send_keepalive; /* time to send keepalive */
 	lnet_ni_t          *ksnp_ni;            /* which network */
 	int                ksnp_n_passive_ips;  /* # of... */
@@ -429,7 +466,8 @@ extern struct ksock_tunables ksocknal_tunables;
 #define SOCKNAL_MATCH_NO  0 /* TX can't match type of connection */
 #define SOCKNAL_MATCH_YES 1 /* TX matches type of connection */
 #define SOCKNAL_MATCH_MAY 2 /* TX can be sent on the connection, but not
-			     * preferred */
+			     * preferred
+			     */
 
 struct ksock_proto {
 	/* version number of protocol */
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_cb.c b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_cb.c
index c1c6f604e6ad..972f6094be75 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_cb.c
@@ -620,7 +620,8 @@ ksocknal_launch_all_connections_locked(struct ksock_peer *peer)
 }
 
 struct ksock_conn *
-ksocknal_find_conn_locked(struct ksock_peer *peer, struct ksock_tx *tx, int nonblk)
+ksocknal_find_conn_locked(struct ksock_peer *peer, struct ksock_tx *tx,
+			  int nonblk)
 {
 	struct list_head *tmp;
 	struct ksock_conn *conn;
@@ -630,10 +631,12 @@ ksocknal_find_conn_locked(struct ksock_peer *peer, struct ksock_tx *tx, int nonb
 	int fnob = 0;
 
 	list_for_each(tmp, &peer->ksnp_conns) {
-		struct ksock_conn *c  = list_entry(tmp, struct ksock_conn, ksnc_list);
-		int nob = atomic_read(&c->ksnc_tx_nob) +
-			c->ksnc_sock->sk->sk_wmem_queued;
-		int rc;
+		struct ksock_conn *c;
+		int nob, rc;
+
+		c = list_entry(tmp, struct ksock_conn, ksnc_list);
+		nob = atomic_read(&c->ksnc_tx_nob) +
+		      c->ksnc_sock->sk->sk_wmem_queued;
 
 		LASSERT(!c->ksnc_closing);
 		LASSERT(c->ksnc_proto &&
@@ -752,9 +755,9 @@ ksocknal_queue_tx_locked(struct ksock_tx *tx, struct ksock_conn *conn)
 		LASSERT(msg->ksm_zc_cookies[1]);
 		LASSERT(conn->ksnc_proto->pro_queue_tx_zcack);
 
+		/* ZC ACK piggybacked on ztx release tx later */
 		if (conn->ksnc_proto->pro_queue_tx_zcack(conn, tx, 0))
-			ztx = tx; /* ZC ACK piggybacked on ztx release tx later */
-
+			ztx = tx;
 	} else {
 		/*
 		 * It's a normal packet - can it piggback a noop zc-ack that
@@ -796,7 +799,8 @@ ksocknal_find_connectable_route_locked(struct ksock_peer *peer)
 
 		LASSERT(!route->ksnr_connecting || route->ksnr_scheduled);
 
-		if (route->ksnr_scheduled)      /* connections being established */
+		/* connections being established */
+		if (route->ksnr_scheduled)
 			continue;
 
 		/* all route types connected ? */
@@ -1514,7 +1518,10 @@ int ksocknal_scheduler(void *arg)
 			rc = ksocknal_process_transmit(conn, tx);
 
 			if (rc == -ENOMEM || rc == -EAGAIN) {
-				/* Incomplete send: replace tx on HEAD of tx_queue */
+				/*
+				 * Incomplete send: replace tx on HEAD of
+				 * tx_queue
+				 */
 				spin_lock_bh(&sched->kss_lock);
 				list_add(&tx->tx_list, &conn->ksnc_tx_queue);
 			} else {
@@ -1724,7 +1731,8 @@ ksocknal_recv_hello(lnet_ni_t *ni, struct ksock_conn *conn,
 	timeout = active ? *ksocknal_tunables.ksnd_timeout :
 			    lnet_acceptor_timeout();
 
-	rc = lnet_sock_read(sock, &hello->kshm_magic, sizeof(hello->kshm_magic), timeout);
+	rc = lnet_sock_read(sock, &hello->kshm_magic,
+			    sizeof(hello->kshm_magic), timeout);
 	if (rc) {
 		CERROR("Error %d reading HELLO from %pI4h\n",
 		       rc, &conn->ksnc_ipaddr);
@@ -1798,7 +1806,8 @@ ksocknal_recv_hello(lnet_ni_t *ni, struct ksock_conn *conn,
 	    conn->ksnc_port > LNET_ACCEPTOR_MAX_RESERVED_PORT) {
 		/* Userspace NAL assigns peer process ID from socket */
 		recv_id.pid = conn->ksnc_port | LNET_PID_USERFLAG;
-		recv_id.nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), conn->ksnc_ipaddr);
+		recv_id.nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid),
+					 conn->ksnc_ipaddr);
 	} else {
 		recv_id.nid = hello->kshm_src_nid;
 		recv_id.pid = hello->kshm_src_pid;
@@ -1882,7 +1891,8 @@ ksocknal_connect(struct ksock_route *route)
 		if (peer->ksnp_accepting > 0) {
 			CDEBUG(D_NET,
 			       "peer %s(%d) already connecting to me, retry later.\n",
-			       libcfs_nid2str(peer->ksnp_id.nid), peer->ksnp_accepting);
+			       libcfs_nid2str(peer->ksnp_id.nid),
+			       peer->ksnp_accepting);
 			retry_later = 1;
 		}
 
@@ -2241,7 +2251,8 @@ ksocknal_connd(void *arg)
 
 		/* Nothing to do for 'timeout'  */
 		set_current_state(TASK_INTERRUPTIBLE);
-		add_wait_queue_exclusive(&ksocknal_data.ksnd_connd_waitq, &wait);
+		add_wait_queue_exclusive(&ksocknal_data.ksnd_connd_waitq,
+					 &wait);
 		spin_unlock_bh(connd_lock);
 
 		nloops = 0;
@@ -2371,7 +2382,8 @@ ksocknal_send_keepalive_locked(struct ksock_peer *peer)
 	struct ksock_conn *conn;
 	struct ksock_tx *tx;
 
-	if (list_empty(&peer->ksnp_conns)) /* last_alive will be updated by create_conn */
+	/* last_alive will be updated by create_conn */
+	if (list_empty(&peer->ksnp_conns))
 		return 0;
 
 	if (peer->ksnp_proto != &ksocknal_protocol_v3x)
@@ -2473,8 +2485,8 @@ ksocknal_check_peer_timeouts(int idx)
 		 * holding only shared lock
 		 */
 		if (!list_empty(&peer->ksnp_tx_queue)) {
-			struct ksock_tx *tx = list_entry(peer->ksnp_tx_queue.next,
-						    struct ksock_tx, tx_list);
+			tx = list_entry(peer->ksnp_tx_queue.next,
+					struct ksock_tx, tx_list);
 
 			if (cfs_time_aftereq(cfs_time_current(),
 					     tx->tx_deadline)) {
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c
index 6c95e989ca12..4bcab4bcc2de 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c
@@ -202,7 +202,8 @@ ksocknal_lib_recv_iov(struct ksock_conn *conn)
 				fragnob = sum;
 
 			conn->ksnc_rx_csum = ksocknal_csum(conn->ksnc_rx_csum,
-							   iov[i].iov_base, fragnob);
+							   iov[i].iov_base,
+							   fragnob);
 		}
 		conn->ksnc_msg.ksm_csum = saved_csum;
 	}
@@ -291,7 +292,8 @@ ksocknal_lib_csum_tx(struct ksock_tx *tx)
 }
 
 int
-ksocknal_lib_get_conn_tunables(struct ksock_conn *conn, int *txmem, int *rxmem, int *nagle)
+ksocknal_lib_get_conn_tunables(struct ksock_conn *conn, int *txmem,
+			       int *rxmem, int *nagle)
 {
 	struct socket *sock = conn->ksnc_sock;
 	int len;
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_proto.c b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_proto.c
index 82e174f6d9fe..8f0ff6ca1f39 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_proto.c
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_proto.c
@@ -194,7 +194,10 @@ ksocknal_queue_tx_zcack_v3(struct ksock_conn *conn,
 	}
 
 	if (!tx->tx_msg.ksm_zc_cookies[0]) {
-		/* NOOP tx has only one ZC-ACK cookie, can carry at least one more */
+		/*
+		 * NOOP tx has only one ZC-ACK cookie,
+		 * can carry at least one more
+		 */
 		if (tx->tx_msg.ksm_zc_cookies[1] > cookie) {
 			tx->tx_msg.ksm_zc_cookies[0] = tx->tx_msg.ksm_zc_cookies[1];
 			tx->tx_msg.ksm_zc_cookies[1] = cookie;
@@ -203,7 +206,10 @@ ksocknal_queue_tx_zcack_v3(struct ksock_conn *conn,
 		}
 
 		if (tx->tx_msg.ksm_zc_cookies[0] - tx->tx_msg.ksm_zc_cookies[1] > 2) {
-			/* not likely to carry more ACKs, skip it to simplify logic */
+			/*
+			 * not likely to carry more ACKs, skip it
+			 * to simplify logic
+			 */
 			ksocknal_next_tx_carrier(conn);
 		}
 
@@ -237,7 +243,10 @@ ksocknal_queue_tx_zcack_v3(struct ksock_conn *conn,
 		}
 
 	} else {
-		/* ksm_zc_cookies[0] < ksm_zc_cookies[1], it is range of cookies */
+		/*
+		 * ksm_zc_cookies[0] < ksm_zc_cookies[1],
+		 * it is range of cookies
+		 */
 		if (cookie >= tx->tx_msg.ksm_zc_cookies[0] &&
 		    cookie <= tx->tx_msg.ksm_zc_cookies[1]) {
 			CWARN("%s: duplicated ZC cookie: %llu\n",
@@ -425,7 +434,8 @@ ksocknal_handle_zcack(struct ksock_conn *conn, __u64 cookie1, __u64 cookie2)
 				 tx_zc_list) {
 		__u64 c = tx->tx_msg.ksm_zc_cookies[0];
 
-		if (c == cookie1 || c == cookie2 || (cookie1 < c && c < cookie2)) {
+		if (c == cookie1 || c == cookie2 ||
+		    (cookie1 < c && c < cookie2)) {
 			tx->tx_msg.ksm_zc_cookies[0] = 0;
 			list_del(&tx->tx_zc_list);
 			list_add(&tx->tx_zc_list, &zlist);
@@ -639,7 +649,8 @@ out:
 }
 
 static int
-ksocknal_recv_hello_v2(struct ksock_conn *conn, ksock_hello_msg_t *hello, int timeout)
+ksocknal_recv_hello_v2(struct ksock_conn *conn, ksock_hello_msg_t *hello,
+		       int timeout)
 {
 	struct socket *sock = conn->ksnc_sock;
 	int rc;
@@ -737,7 +748,10 @@ ksocknal_pack_msg_v2(struct ksock_tx *tx)
 		tx->tx_nob = offsetof(ksock_msg_t,  ksm_u.lnetmsg.ksnm_hdr);
 		tx->tx_resid = offsetof(ksock_msg_t,  ksm_u.lnetmsg.ksnm_hdr);
 	}
-	/* Don't checksum before start sending, because packet can be piggybacked with ACK */
+	/*
+	 * Don't checksum before start sending, because packet can be
+	 * piggybacked with ACK
+	 */
 }
 
 static void
diff --git a/drivers/staging/lustre/lnet/libcfs/debug.c b/drivers/staging/lustre/lnet/libcfs/debug.c
index 23b36b890964..a38db2322225 100644
--- a/drivers/staging/lustre/lnet/libcfs/debug.c
+++ b/drivers/staging/lustre/lnet/libcfs/debug.c
@@ -57,7 +57,7 @@ static int libcfs_param_debug_mb_set(const char *val,
 				     const struct kernel_param *kp)
 {
 	int rc;
-	unsigned num;
+	unsigned int num;
 
 	rc = kstrtouint(val, 0, &num);
 	if (rc < 0)
@@ -228,7 +228,8 @@ int libcfs_panic_in_progress;
 static const char *
 libcfs_debug_subsys2str(int subsys)
 {
-	static const char *libcfs_debug_subsystems[] = LIBCFS_DEBUG_SUBSYS_NAMES;
+	static const char * const libcfs_debug_subsystems[] =
+		LIBCFS_DEBUG_SUBSYS_NAMES;
 
 	if (subsys >= ARRAY_SIZE(libcfs_debug_subsystems))
 		return NULL;
@@ -240,7 +241,8 @@ libcfs_debug_subsys2str(int subsys)
 static const char *
 libcfs_debug_dbg2str(int debug)
 {
-	static const char *libcfs_debug_masks[] = LIBCFS_DEBUG_MASKS_NAMES;
+	static const char * const libcfs_debug_masks[] =
+		LIBCFS_DEBUG_MASKS_NAMES;
 
 	if (debug >= ARRAY_SIZE(libcfs_debug_masks))
 		return NULL;
@@ -253,17 +255,17 @@ libcfs_debug_mask2str(char *str, int size, int mask, int is_subsys)
 {
 	const char *(*fn)(int bit) = is_subsys ? libcfs_debug_subsys2str :
 						 libcfs_debug_dbg2str;
-	int	   len = 0;
-	const char   *token;
-	int	   i;
+	int len = 0;
+	const char *token;
+	int i;
 
-	if (mask == 0) {			/* "0" */
+	if (!mask) {			/* "0" */
 		if (size > 0)
 			str[0] = '0';
 		len = 1;
 	} else {				/* space-separated tokens */
 		for (i = 0; i < 32; i++) {
-			if ((mask & (1 << i)) == 0)
+			if (!(mask & (1 << i)))
 				continue;
 
 			token = fn(i);
@@ -276,7 +278,7 @@ libcfs_debug_mask2str(char *str, int size, int mask, int is_subsys)
 				len++;
 			}
 
-			while (*token != 0) {
+			while (*token) {
 				if (len < size)
 					str[len] = *token;
 				token++;
@@ -299,10 +301,10 @@ libcfs_debug_str2mask(int *mask, const char *str, int is_subsys)
 {
 	const char *(*fn)(int bit) = is_subsys ? libcfs_debug_subsys2str :
 						 libcfs_debug_dbg2str;
-	int	 m = 0;
-	int	 matched;
-	int	 n;
-	int	 t;
+	int m = 0;
+	int matched;
+	int n;
+	int t;
 
 	/* Allow a number for backwards compatibility */
 
@@ -313,7 +315,7 @@ libcfs_debug_str2mask(int *mask, const char *str, int is_subsys)
 	t = sscanf(str, "%i%n", &m, &matched);
 	if (t >= 1 && matched == n) {
 		/* don't print warning for lctl set_param debug=0 or -1 */
-		if (m != 0 && m != -1)
+		if (m && m != -1)
 			CWARN("You are trying to use a numerical value for the mask - this will be deprecated in a future release.\n");
 		*mask = m;
 		return 0;
@@ -387,8 +389,8 @@ EXPORT_SYMBOL(libcfs_debug_dumplog);
 
 int libcfs_debug_init(unsigned long bufsize)
 {
-	int    rc = 0;
 	unsigned int max = libcfs_debug_mb;
+	int rc = 0;
 
 	init_waitqueue_head(&debug_ctlwq);
 
@@ -414,9 +416,9 @@ int libcfs_debug_init(unsigned long bufsize)
 		max = max / num_possible_cpus();
 		max <<= (20 - PAGE_SHIFT);
 	}
-	rc = cfs_tracefile_init(max);
 
-	if (rc == 0) {
+	rc = cfs_tracefile_init(max);
+	if (!rc) {
 		libcfs_register_panic_notifier();
 		libcfs_debug_mb = cfs_trace_get_debug_mb();
 	}
diff --git a/drivers/staging/lustre/lnet/libcfs/fail.c b/drivers/staging/lustre/lnet/libcfs/fail.c
index e4b1a0a86eae..12dd50ad4efb 100644
--- a/drivers/staging/lustre/lnet/libcfs/fail.c
+++ b/drivers/staging/lustre/lnet/libcfs/fail.c
@@ -46,7 +46,7 @@ EXPORT_SYMBOL(cfs_race_waitq);
 int cfs_race_state;
 EXPORT_SYMBOL(cfs_race_state);
 
-int __cfs_fail_check_set(__u32 id, __u32 value, int set)
+int __cfs_fail_check_set(u32 id, u32 value, int set)
 {
 	static atomic_t cfs_fail_count = ATOMIC_INIT(0);
 
@@ -113,6 +113,7 @@ int __cfs_fail_check_set(__u32 id, __u32 value, int set)
 		break;
 	case CFS_FAIL_LOC_RESET:
 		cfs_fail_loc = value;
+		atomic_set(&cfs_fail_count, 0);
 		break;
 	default:
 		LASSERTF(0, "called with bad set %u\n", set);
@@ -123,7 +124,7 @@ int __cfs_fail_check_set(__u32 id, __u32 value, int set)
 }
 EXPORT_SYMBOL(__cfs_fail_check_set);
 
-int __cfs_fail_timeout_set(__u32 id, __u32 value, int ms, int set)
+int __cfs_fail_timeout_set(u32 id, u32 value, int ms, int set)
 {
 	int ret;
 
diff --git a/drivers/staging/lustre/lnet/libcfs/hash.c b/drivers/staging/lustre/lnet/libcfs/hash.c
index 23283b6e09ab..c93c59d8fe6c 100644
--- a/drivers/staging/lustre/lnet/libcfs/hash.c
+++ b/drivers/staging/lustre/lnet/libcfs/hash.c
@@ -289,7 +289,7 @@ cfs_hash_hd_hhead_size(struct cfs_hash *hs)
 static struct hlist_head *
 cfs_hash_hd_hhead(struct cfs_hash *hs, struct cfs_hash_bd *bd)
 {
-	struct cfs_hash_head_dep   *head;
+	struct cfs_hash_head_dep *head;
 
 	head = (struct cfs_hash_head_dep *)&bd->bd_bucket->hsb_head[0];
 	return &head[bd->bd_offset].hd_head;
@@ -492,7 +492,7 @@ cfs_hash_bd_get(struct cfs_hash *hs, const void *key, struct cfs_hash_bd *bd)
 		cfs_hash_bd_from_key(hs, hs->hs_buckets,
 				     hs->hs_cur_bits, key, bd);
 	} else {
-		LASSERT(hs->hs_rehash_bits != 0);
+		LASSERT(hs->hs_rehash_bits);
 		cfs_hash_bd_from_key(hs, hs->hs_rehash_buckets,
 				     hs->hs_rehash_bits, key, bd);
 	}
@@ -507,14 +507,14 @@ cfs_hash_bd_dep_record(struct cfs_hash *hs, struct cfs_hash_bd *bd, int dep_cur)
 
 	bd->bd_bucket->hsb_depmax = dep_cur;
 # if CFS_HASH_DEBUG_LEVEL >= CFS_HASH_DEBUG_1
-	if (likely(warn_on_depth == 0 ||
+	if (likely(!warn_on_depth ||
 		   max(warn_on_depth, hs->hs_dep_max) >= dep_cur))
 		return;
 
 	spin_lock(&hs->hs_dep_lock);
-	hs->hs_dep_max  = dep_cur;
-	hs->hs_dep_bkt  = bd->bd_bucket->hsb_index;
-	hs->hs_dep_off  = bd->bd_offset;
+	hs->hs_dep_max = dep_cur;
+	hs->hs_dep_bkt = bd->bd_bucket->hsb_index;
+	hs->hs_dep_off = bd->bd_offset;
 	hs->hs_dep_bits = hs->hs_cur_bits;
 	spin_unlock(&hs->hs_dep_lock);
 
@@ -531,7 +531,7 @@ cfs_hash_bd_add_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd,
 	rc = hs->hs_hops->hop_hnode_add(hs, bd, hnode);
 	cfs_hash_bd_dep_record(hs, bd, rc);
 	bd->bd_bucket->hsb_version++;
-	if (unlikely(bd->bd_bucket->hsb_version == 0))
+	if (unlikely(!bd->bd_bucket->hsb_version))
 		bd->bd_bucket->hsb_version++;
 	bd->bd_bucket->hsb_count++;
 
@@ -551,7 +551,7 @@ cfs_hash_bd_del_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd,
 	LASSERT(bd->bd_bucket->hsb_count > 0);
 	bd->bd_bucket->hsb_count--;
 	bd->bd_bucket->hsb_version++;
-	if (unlikely(bd->bd_bucket->hsb_version == 0))
+	if (unlikely(!bd->bd_bucket->hsb_version))
 		bd->bd_bucket->hsb_version++;
 
 	if (cfs_hash_with_counter(hs)) {
@@ -571,7 +571,7 @@ cfs_hash_bd_move_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd_old,
 	struct cfs_hash_bucket *nbkt = bd_new->bd_bucket;
 	int rc;
 
-	if (cfs_hash_bd_compare(bd_old, bd_new) == 0)
+	if (!cfs_hash_bd_compare(bd_old, bd_new))
 		return;
 
 	/* use cfs_hash_bd_hnode_add/del, to avoid atomic & refcount ops
@@ -584,11 +584,11 @@ cfs_hash_bd_move_locked(struct cfs_hash *hs, struct cfs_hash_bd *bd_old,
 	LASSERT(obkt->hsb_count > 0);
 	obkt->hsb_count--;
 	obkt->hsb_version++;
-	if (unlikely(obkt->hsb_version == 0))
+	if (unlikely(!obkt->hsb_version))
 		obkt->hsb_version++;
 	nbkt->hsb_count++;
 	nbkt->hsb_version++;
-	if (unlikely(nbkt->hsb_version == 0))
+	if (unlikely(!nbkt->hsb_version))
 		nbkt->hsb_version++;
 }
 
@@ -629,7 +629,7 @@ cfs_hash_bd_lookup_intent(struct cfs_hash *hs, struct cfs_hash_bd *bd,
 	struct hlist_head *hhead = cfs_hash_bd_hhead(hs, bd);
 	struct hlist_node *ehnode;
 	struct hlist_node *match;
-	int intent_add = (intent & CFS_HS_LOOKUP_MASK_ADD) != 0;
+	int intent_add = intent & CFS_HS_LOOKUP_MASK_ADD;
 
 	/* with this function, we can avoid a lot of useless refcount ops,
 	 * which are expensive atomic operations most time.
@@ -643,13 +643,13 @@ cfs_hash_bd_lookup_intent(struct cfs_hash *hs, struct cfs_hash_bd *bd,
 			continue;
 
 		/* match and ... */
-		if ((intent & CFS_HS_LOOKUP_MASK_DEL) != 0) {
+		if (intent & CFS_HS_LOOKUP_MASK_DEL) {
 			cfs_hash_bd_del_locked(hs, bd, ehnode);
 			return ehnode;
 		}
 
 		/* caller wants refcount? */
-		if ((intent & CFS_HS_LOOKUP_MASK_REF) != 0)
+		if (intent & CFS_HS_LOOKUP_MASK_REF)
 			cfs_hash_get(hs, ehnode);
 		return ehnode;
 	}
@@ -682,7 +682,7 @@ EXPORT_SYMBOL(cfs_hash_bd_peek_locked);
 
 static void
 cfs_hash_multi_bd_lock(struct cfs_hash *hs, struct cfs_hash_bd *bds,
-		       unsigned n, int excl)
+		       unsigned int n, int excl)
 {
 	struct cfs_hash_bucket *prev = NULL;
 	int i;
@@ -704,7 +704,7 @@ cfs_hash_multi_bd_lock(struct cfs_hash *hs, struct cfs_hash_bd *bds,
 
 static void
 cfs_hash_multi_bd_unlock(struct cfs_hash *hs, struct cfs_hash_bd *bds,
-			 unsigned n, int excl)
+			 unsigned int n, int excl)
 {
 	struct cfs_hash_bucket *prev = NULL;
 	int i;
@@ -719,10 +719,10 @@ cfs_hash_multi_bd_unlock(struct cfs_hash *hs, struct cfs_hash_bd *bds,
 
 static struct hlist_node *
 cfs_hash_multi_bd_lookup_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds,
-				unsigned n, const void *key)
+				unsigned int n, const void *key)
 {
 	struct hlist_node *ehnode;
-	unsigned i;
+	unsigned int i;
 
 	cfs_hash_for_each_bd(bds, n, i) {
 		ehnode = cfs_hash_bd_lookup_intent(hs, &bds[i], key, NULL,
@@ -735,12 +735,12 @@ cfs_hash_multi_bd_lookup_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds,
 
 static struct hlist_node *
 cfs_hash_multi_bd_findadd_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds,
-				 unsigned n, const void *key,
+				 unsigned int n, const void *key,
 				 struct hlist_node *hnode, int noref)
 {
 	struct hlist_node *ehnode;
 	int intent;
-	unsigned i;
+	unsigned int i;
 
 	LASSERT(hnode);
 	intent = (!noref * CFS_HS_LOOKUP_MASK_REF) | CFS_HS_LOOKUP_IT_PEEK;
@@ -766,7 +766,7 @@ cfs_hash_multi_bd_findadd_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds,
 
 static struct hlist_node *
 cfs_hash_multi_bd_finddel_locked(struct cfs_hash *hs, struct cfs_hash_bd *bds,
-				 unsigned n, const void *key,
+				 unsigned int n, const void *key,
 				 struct hlist_node *hnode)
 {
 	struct hlist_node *ehnode;
@@ -815,7 +815,7 @@ cfs_hash_dual_bd_get(struct cfs_hash *hs, const void *key,
 		return;
 	}
 
-	LASSERT(hs->hs_rehash_bits != 0);
+	LASSERT(hs->hs_rehash_bits);
 	cfs_hash_bd_from_key(hs, hs->hs_rehash_buckets,
 			     hs->hs_rehash_bits, key, &bds[1]);
 
@@ -883,7 +883,7 @@ cfs_hash_buckets_realloc(struct cfs_hash *hs, struct cfs_hash_bucket **old_bkts,
 	struct cfs_hash_bucket **new_bkts;
 	int i;
 
-	LASSERT(old_size == 0 || old_bkts);
+	LASSERT(!old_size || old_bkts);
 
 	if (old_bkts && old_size == new_size)
 		return old_bkts;
@@ -908,9 +908,9 @@ cfs_hash_buckets_realloc(struct cfs_hash *hs, struct cfs_hash_bucket **old_bkts,
 			return NULL;
 		}
 
-		new_bkts[i]->hsb_index   = i;
-		new_bkts[i]->hsb_version = 1;  /* shouldn't be zero */
-		new_bkts[i]->hsb_depmax  = -1; /* unknown */
+		new_bkts[i]->hsb_index = i;
+		new_bkts[i]->hsb_version = 1;	/* shouldn't be zero */
+		new_bkts[i]->hsb_depmax = -1;	/* unknown */
 		bd.bd_bucket = new_bkts[i];
 		cfs_hash_bd_for_each_hlist(hs, &bd, hhead)
 			INIT_HLIST_HEAD(hhead);
@@ -950,9 +950,9 @@ static int cfs_hash_dep_print(struct cfs_workitem *wi)
 	int bits;
 
 	spin_lock(&hs->hs_dep_lock);
-	dep  = hs->hs_dep_max;
-	bkt  = hs->hs_dep_bkt;
-	off  = hs->hs_dep_off;
+	dep = hs->hs_dep_max;
+	bkt = hs->hs_dep_bkt;
+	off = hs->hs_dep_off;
 	bits = hs->hs_dep_bits;
 	spin_unlock(&hs->hs_dep_lock);
 
@@ -976,7 +976,7 @@ static void cfs_hash_depth_wi_cancel(struct cfs_hash *hs)
 		return;
 
 	spin_lock(&hs->hs_dep_lock);
-	while (hs->hs_dep_bits != 0) {
+	while (hs->hs_dep_bits) {
 		spin_unlock(&hs->hs_dep_lock);
 		cond_resched();
 		spin_lock(&hs->hs_dep_lock);
@@ -992,10 +992,10 @@ static inline void cfs_hash_depth_wi_cancel(struct cfs_hash *hs) {}
 #endif /* CFS_HASH_DEBUG_LEVEL >= CFS_HASH_DEBUG_1 */
 
 struct cfs_hash *
-cfs_hash_create(char *name, unsigned cur_bits, unsigned max_bits,
-		unsigned bkt_bits, unsigned extra_bytes,
-		unsigned min_theta, unsigned max_theta,
-		struct cfs_hash_ops *ops, unsigned flags)
+cfs_hash_create(char *name, unsigned int cur_bits, unsigned int max_bits,
+		unsigned int bkt_bits, unsigned int extra_bytes,
+		unsigned int min_theta, unsigned int max_theta,
+		struct cfs_hash_ops *ops, unsigned int flags)
 {
 	struct cfs_hash *hs;
 	int len;
@@ -1010,18 +1010,17 @@ cfs_hash_create(char *name, unsigned cur_bits, unsigned max_bits,
 	LASSERT(ops->hs_get);
 	LASSERT(ops->hs_put_locked);
 
-	if ((flags & CFS_HASH_REHASH) != 0)
+	if (flags & CFS_HASH_REHASH)
 		flags |= CFS_HASH_COUNTER; /* must have counter */
 
 	LASSERT(cur_bits > 0);
 	LASSERT(cur_bits >= bkt_bits);
 	LASSERT(max_bits >= cur_bits && max_bits < 31);
-	LASSERT(ergo((flags & CFS_HASH_REHASH) == 0, cur_bits == max_bits));
-	LASSERT(ergo((flags & CFS_HASH_REHASH) != 0,
-		     (flags & CFS_HASH_NO_LOCK) == 0));
-	LASSERT(ergo((flags & CFS_HASH_REHASH_KEY) != 0, ops->hs_keycpy));
+	LASSERT(ergo(!(flags & CFS_HASH_REHASH), cur_bits == max_bits));
+	LASSERT(ergo(flags & CFS_HASH_REHASH, !(flags & CFS_HASH_NO_LOCK)));
+	LASSERT(ergo(flags & CFS_HASH_REHASH_KEY, ops->hs_keycpy));
 
-	len = (flags & CFS_HASH_BIGNAME) == 0 ?
+	len = !(flags & CFS_HASH_BIGNAME) ?
 	      CFS_HASH_NAME_LEN : CFS_HASH_BIGNAME_LEN;
 	LIBCFS_ALLOC(hs, offsetof(struct cfs_hash, hs_name[len]));
 	if (!hs)
@@ -1036,12 +1035,12 @@ cfs_hash_create(char *name, unsigned cur_bits, unsigned max_bits,
 	cfs_hash_lock_setup(hs);
 	cfs_hash_hlist_setup(hs);
 
-	hs->hs_cur_bits = (__u8)cur_bits;
-	hs->hs_min_bits = (__u8)cur_bits;
-	hs->hs_max_bits = (__u8)max_bits;
-	hs->hs_bkt_bits = (__u8)bkt_bits;
+	hs->hs_cur_bits = (u8)cur_bits;
+	hs->hs_min_bits = (u8)cur_bits;
+	hs->hs_max_bits = (u8)max_bits;
+	hs->hs_bkt_bits = (u8)bkt_bits;
 
-	hs->hs_ops	   = ops;
+	hs->hs_ops = ops;
 	hs->hs_extra_bytes = extra_bytes;
 	hs->hs_rehash_bits = 0;
 	cfs_wi_init(&hs->hs_rehash_wi, hs, cfs_hash_rehash_worker);
@@ -1107,12 +1106,12 @@ cfs_hash_destroy(struct cfs_hash *hs)
 				cfs_hash_exit(hs, hnode);
 			}
 		}
-		LASSERT(bd.bd_bucket->hsb_count == 0);
+		LASSERT(!bd.bd_bucket->hsb_count);
 		cfs_hash_bd_unlock(hs, &bd, 1);
 		cond_resched();
 	}
 
-	LASSERT(atomic_read(&hs->hs_count) == 0);
+	LASSERT(!atomic_read(&hs->hs_count));
 
 	cfs_hash_buckets_free(hs->hs_buckets, cfs_hash_bkt_size(hs),
 			      0, CFS_HASH_NBKT(hs));
@@ -1216,7 +1215,7 @@ cfs_hash_find_or_add(struct cfs_hash *hs, const void *key,
 	struct cfs_hash_bd bds[2];
 	int bits = 0;
 
-	LASSERT(hlist_unhashed(hnode));
+	LASSERTF(hlist_unhashed(hnode), "hnode = %p\n", hnode);
 
 	cfs_hash_lock(hs, 0);
 	cfs_hash_dual_bd_get_and_lock(hs, key, bds, 1);
@@ -1293,7 +1292,7 @@ cfs_hash_del(struct cfs_hash *hs, const void *key, struct hlist_node *hnode)
 	}
 
 	if (hnode) {
-		obj  = cfs_hash_object(hs, hnode);
+		obj = cfs_hash_object(hs, hnode);
 		bits = cfs_hash_rehash_bits(hs);
 	}
 
@@ -1388,7 +1387,7 @@ cfs_hash_for_each_exit(struct cfs_hash *hs)
 	bits = cfs_hash_rehash_bits(hs);
 	cfs_hash_unlock(hs, 1);
 	/* NB: it's race on cfs_has_t::hs_iterating, see above */
-	if (remained == 0)
+	if (!remained)
 		hs->hs_iterating = 0;
 	if (bits > 0) {
 		cfs_hash_rehash(hs, atomic_read(&hs->hs_count) <
@@ -1406,14 +1405,14 @@ cfs_hash_for_each_exit(struct cfs_hash *hs)
  *    . if @removal_safe is true, use can remove current item by
  *      cfs_hash_bd_del_locked
  */
-static __u64
+static u64
 cfs_hash_for_each_tight(struct cfs_hash *hs, cfs_hash_for_each_cb_t func,
 			void *data, int remove_safe)
 {
 	struct hlist_node *hnode;
 	struct hlist_node *pos;
 	struct cfs_hash_bd bd;
-	__u64 count = 0;
+	u64 count = 0;
 	int excl = !!remove_safe;
 	int loop = 0;
 	int i;
@@ -1526,7 +1525,7 @@ cfs_hash_is_empty(struct cfs_hash *hs)
 }
 EXPORT_SYMBOL(cfs_hash_is_empty);
 
-__u64
+u64
 cfs_hash_size_get(struct cfs_hash *hs)
 {
 	return cfs_hash_with_counter(hs) ?
@@ -1552,26 +1551,33 @@ EXPORT_SYMBOL(cfs_hash_size_get);
  */
 static int
 cfs_hash_for_each_relax(struct cfs_hash *hs, cfs_hash_for_each_cb_t func,
-			void *data)
+			void *data, int start)
 {
 	struct hlist_node *hnode;
 	struct hlist_node *tmp;
 	struct cfs_hash_bd bd;
-	__u32 version;
+	u32 version;
 	int count = 0;
 	int stop_on_change;
-	int rc;
+	int end = -1;
+	int rc = 0;
 	int i;
 
 	stop_on_change = cfs_hash_with_rehash_key(hs) ||
 			 !cfs_hash_with_no_itemref(hs) ||
 			 !hs->hs_ops->hs_put_locked;
 	cfs_hash_lock(hs, 0);
+again:
 	LASSERT(!cfs_hash_is_rehashing(hs));
 
 	cfs_hash_for_each_bucket(hs, &bd, i) {
 		struct hlist_head *hhead;
 
+		if (i < start)
+			continue;
+		else if (end > 0 && i >= end)
+			break;
+
 		cfs_hash_bd_lock(hs, &bd, 0);
 		version = cfs_hash_bd_version_get(&bd);
 
@@ -1611,14 +1617,19 @@ cfs_hash_for_each_relax(struct cfs_hash *hs, cfs_hash_for_each_cb_t func,
 		if (rc) /* callback wants to break iteration */
 			break;
 	}
-	cfs_hash_unlock(hs, 0);
+	if (start > 0 && !rc) {
+		end = start;
+		start = 0;
+		goto again;
+	}
 
+	cfs_hash_unlock(hs, 0);
 	return count;
 }
 
 int
 cfs_hash_for_each_nolock(struct cfs_hash *hs, cfs_hash_for_each_cb_t func,
-			 void *data)
+			 void *data, int start)
 {
 	if (cfs_hash_with_no_lock(hs) ||
 	    cfs_hash_with_rehash_key(hs) ||
@@ -1630,7 +1641,7 @@ cfs_hash_for_each_nolock(struct cfs_hash *hs, cfs_hash_for_each_cb_t func,
 		return -EOPNOTSUPP;
 
 	cfs_hash_for_each_enter(hs);
-	cfs_hash_for_each_relax(hs, func, data);
+	cfs_hash_for_each_relax(hs, func, data, start);
 	cfs_hash_for_each_exit(hs);
 
 	return 0;
@@ -1652,7 +1663,7 @@ int
 cfs_hash_for_each_empty(struct cfs_hash *hs, cfs_hash_for_each_cb_t func,
 			void *data)
 {
-	unsigned i = 0;
+	unsigned int i = 0;
 
 	if (cfs_hash_with_no_lock(hs))
 		return -EOPNOTSUPP;
@@ -1662,7 +1673,7 @@ cfs_hash_for_each_empty(struct cfs_hash *hs, cfs_hash_for_each_cb_t func,
 		return -EOPNOTSUPP;
 
 	cfs_hash_for_each_enter(hs);
-	while (cfs_hash_for_each_relax(hs, func, data)) {
+	while (cfs_hash_for_each_relax(hs, func, data, 0)) {
 		CDEBUG(D_INFO, "Try to empty hash: %s, loop: %u\n",
 		       hs->hs_name, i++);
 	}
@@ -1672,7 +1683,7 @@ cfs_hash_for_each_empty(struct cfs_hash *hs, cfs_hash_for_each_cb_t func,
 EXPORT_SYMBOL(cfs_hash_for_each_empty);
 
 void
-cfs_hash_hlist_for_each(struct cfs_hash *hs, unsigned hindex,
+cfs_hash_hlist_for_each(struct cfs_hash *hs, unsigned int hindex,
 			cfs_hash_for_each_cb_t func, void *data)
 {
 	struct hlist_head *hhead;
@@ -1704,7 +1715,7 @@ EXPORT_SYMBOL(cfs_hash_hlist_for_each);
  * the passed callback @func and pass to it as an argument each hash
  * item and the private @data. During the callback the bucket lock
  * is held so the callback must never sleep.
-   */
+ */
 void
 cfs_hash_for_each_key(struct cfs_hash *hs, const void *key,
 		      cfs_hash_for_each_cb_t func, void *data)
@@ -1936,7 +1947,7 @@ out:
 	/* can't refer to @hs anymore because it could be destroyed */
 	if (bkts)
 		cfs_hash_buckets_free(bkts, bsize, new_size, old_size);
-	if (rc != 0)
+	if (rc)
 		CDEBUG(D_INFO, "early quit of rehashing: %d\n", rc);
 	/* return 1 only if cfs_wi_exit is called */
 	return rc == -ESRCH;
@@ -2005,7 +2016,7 @@ cfs_hash_full_bkts(struct cfs_hash *hs)
 	if (!hs->hs_rehash_buckets)
 		return hs->hs_buckets;
 
-	LASSERT(hs->hs_rehash_bits != 0);
+	LASSERT(hs->hs_rehash_bits);
 	return hs->hs_rehash_bits > hs->hs_cur_bits ?
 	       hs->hs_rehash_buckets : hs->hs_buckets;
 }
@@ -2017,7 +2028,7 @@ cfs_hash_full_nbkt(struct cfs_hash *hs)
 	if (!hs->hs_rehash_buckets)
 		return CFS_HASH_NBKT(hs);
 
-	LASSERT(hs->hs_rehash_bits != 0);
+	LASSERT(hs->hs_rehash_bits);
 	return hs->hs_rehash_bits > hs->hs_cur_bits ?
 	       CFS_HASH_RH_NBKT(hs) : CFS_HASH_NBKT(hs);
 }
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c b/drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c
index 33352af6c27f..55caa19def51 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c
@@ -74,7 +74,7 @@ EXPORT_SYMBOL(cfs_cpt_table_free);
 int
 cfs_cpt_table_print(struct cfs_cpt_table *cptab, char *buf, int len)
 {
-	int	rc;
+	int rc;
 
 	rc = snprintf(buf, len, "%d\t: %d\n", 0, 0);
 	len -= rc;
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_lock.c b/drivers/staging/lustre/lnet/libcfs/libcfs_lock.c
index 83543f928279..1967b97c4afc 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_lock.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_lock.c
@@ -52,9 +52,9 @@ struct cfs_percpt_lock *
 cfs_percpt_lock_create(struct cfs_cpt_table *cptab,
 		       struct lock_class_key *keys)
 {
-	struct cfs_percpt_lock	*pcl;
-	spinlock_t		*lock;
-	int			i;
+	struct cfs_percpt_lock *pcl;
+	spinlock_t *lock;
+	int i;
 
 	/* NB: cptab can be NULL, pcl will be for HW CPUs on that case */
 	LIBCFS_ALLOC(pcl, sizeof(*pcl));
@@ -73,7 +73,7 @@ cfs_percpt_lock_create(struct cfs_cpt_table *cptab,
 
 	cfs_percpt_for_each(lock, i, pcl->pcl_locks) {
 		spin_lock_init(lock);
-		if (keys != NULL)
+		if (keys)
 			lockdep_set_class(lock, &keys[i]);
 	}
 
@@ -94,8 +94,8 @@ void
 cfs_percpt_lock(struct cfs_percpt_lock *pcl, int index)
 	__acquires(pcl->pcl_locks)
 {
-	int	ncpt = cfs_cpt_number(pcl->pcl_cptab);
-	int	i;
+	int ncpt = cfs_cpt_number(pcl->pcl_cptab);
+	int i;
 
 	LASSERT(index >= CFS_PERCPT_LOCK_EX && index < ncpt);
 
@@ -114,7 +114,7 @@ cfs_percpt_lock(struct cfs_percpt_lock *pcl, int index)
 	/* exclusive lock request */
 	for (i = 0; i < ncpt; i++) {
 		spin_lock(pcl->pcl_locks[i]);
-		if (i == 0) {
+		if (!i) {
 			LASSERT(!pcl->pcl_locked);
 			/* nobody should take private lock after this
 			 * so I wouldn't starve for too long time
@@ -130,8 +130,8 @@ void
 cfs_percpt_unlock(struct cfs_percpt_lock *pcl, int index)
 	__releases(pcl->pcl_locks)
 {
-	int	ncpt = cfs_cpt_number(pcl->pcl_cptab);
-	int	i;
+	int ncpt = cfs_cpt_number(pcl->pcl_cptab);
+	int i;
 
 	index = ncpt == 1 ? 0 : index;
 
@@ -141,7 +141,7 @@ cfs_percpt_unlock(struct cfs_percpt_lock *pcl, int index)
 	}
 
 	for (i = ncpt - 1; i >= 0; i--) {
-		if (i == 0) {
+		if (!i) {
 			LASSERT(pcl->pcl_locked);
 			pcl->pcl_locked = 0;
 		}
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_mem.c b/drivers/staging/lustre/lnet/libcfs/libcfs_mem.c
index d0e81bb41cdc..ef085ba23194 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_mem.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_mem.c
@@ -43,8 +43,8 @@ struct cfs_var_array {
 void
 cfs_percpt_free(void *vars)
 {
-	struct	cfs_var_array *arr;
-	int	i;
+	struct cfs_var_array *arr;
+	int i;
 
 	arr = container_of(vars, struct cfs_var_array, va_ptrs[0]);
 
@@ -72,9 +72,9 @@ EXPORT_SYMBOL(cfs_percpt_free);
 void *
 cfs_percpt_alloc(struct cfs_cpt_table *cptab, unsigned int size)
 {
-	struct cfs_var_array	*arr;
-	int			count;
-	int			i;
+	struct cfs_var_array *arr;
+	int count;
+	int i;
 
 	count = cfs_cpt_number(cptab);
 
@@ -120,8 +120,8 @@ EXPORT_SYMBOL(cfs_percpt_number);
 void
 cfs_array_free(void *vars)
 {
-	struct cfs_var_array	*arr;
-	int			i;
+	struct cfs_var_array *arr;
+	int i;
 
 	arr = container_of(vars, struct cfs_var_array, va_ptrs[0]);
 
@@ -144,15 +144,15 @@ EXPORT_SYMBOL(cfs_array_free);
 void *
 cfs_array_alloc(int count, unsigned int size)
 {
-	struct cfs_var_array	*arr;
-	int			i;
+	struct cfs_var_array *arr;
+	int i;
 
 	LIBCFS_ALLOC(arr, offsetof(struct cfs_var_array, va_ptrs[count]));
 	if (!arr)
 		return NULL;
 
-	arr->va_count	= count;
-	arr->va_size	= size;
+	arr->va_count = count;
+	arr->va_size = size;
 
 	for (i = 0; i < count; i++) {
 		LIBCFS_ALLOC(arr->va_ptrs[i], size);
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_string.c b/drivers/staging/lustre/lnet/libcfs/libcfs_string.c
index 56a614d7713b..02de1ee720fd 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_string.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_string.c
@@ -79,7 +79,7 @@ int cfs_str2mask(const char *str, const char *(*bit2str)(int bit),
 		for (i = 0; i < 32; i++) {
 			debugstr = bit2str(i);
 			if (debugstr && strlen(debugstr) == len &&
-			    strncasecmp(str, debugstr, len) == 0) {
+			    !strncasecmp(str, debugstr, len)) {
 				if (op == '-')
 					newmask &= ~(1 << i);
 				else
@@ -89,7 +89,7 @@ int cfs_str2mask(const char *str, const char *(*bit2str)(int bit),
 			}
 		}
 		if (!found && len == 3 &&
-		    (strncasecmp(str, "ALL", len) == 0)) {
+		    !strncasecmp(str, "ALL", len)) {
 			if (op == '-')
 				newmask = minmask;
 			else
@@ -112,7 +112,7 @@ int cfs_str2mask(const char *str, const char *(*bit2str)(int bit),
 char *cfs_firststr(char *str, size_t size)
 {
 	size_t i = 0;
-	char  *end;
+	char *end;
 
 	/* trim leading spaces */
 	while (i < size && *str && isspace(*str)) {
@@ -182,7 +182,7 @@ cfs_gettok(struct cfs_lstr *next, char delim, struct cfs_lstr *res)
 		next->ls_len--;
 	}
 
-	if (next->ls_len == 0) /* whitespaces only */
+	if (!next->ls_len) /* whitespaces only */
 		return 0;
 
 	if (*next->ls_str == delim) {
@@ -222,8 +222,8 @@ EXPORT_SYMBOL(cfs_gettok);
  * \retval 0 otherwise
  */
 int
-cfs_str2num_check(char *str, int nob, unsigned *num,
-		  unsigned min, unsigned max)
+cfs_str2num_check(char *str, int nob, unsigned int *num,
+		  unsigned int min, unsigned int max)
 {
 	bool all_numbers = true;
 	char *endp, cache;
@@ -273,11 +273,11 @@ EXPORT_SYMBOL(cfs_str2num_check);
  * -ENOMEM will be returned.
  */
 static int
-cfs_range_expr_parse(struct cfs_lstr *src, unsigned min, unsigned max,
+cfs_range_expr_parse(struct cfs_lstr *src, unsigned int min, unsigned int max,
 		     int bracketed, struct cfs_range_expr **expr)
 {
-	struct cfs_range_expr	*re;
-	struct cfs_lstr		tok;
+	struct cfs_range_expr *re;
+	struct cfs_lstr tok;
 
 	LIBCFS_ALLOC(re, sizeof(*re));
 	if (!re)
@@ -391,7 +391,7 @@ cfs_expr_list_print(char *buffer, int count, struct cfs_expr_list *expr_list)
 		i += scnprintf(buffer + i, count - i, "[");
 
 	list_for_each_entry(expr, &expr_list->el_exprs, re_link) {
-		if (j++ != 0)
+		if (j++)
 			i += scnprintf(buffer + i, count - i, ",");
 		i += cfs_range_expr_print(buffer + i, count - i, expr,
 					  numexprs > 1);
@@ -411,13 +411,13 @@ EXPORT_SYMBOL(cfs_expr_list_print);
  * \retval 0 otherwise
  */
 int
-cfs_expr_list_match(__u32 value, struct cfs_expr_list *expr_list)
+cfs_expr_list_match(u32 value, struct cfs_expr_list *expr_list)
 {
-	struct cfs_range_expr	*expr;
+	struct cfs_range_expr *expr;
 
 	list_for_each_entry(expr, &expr_list->el_exprs, re_link) {
 		if (value >= expr->re_lo && value <= expr->re_hi &&
-		    ((value - expr->re_lo) % expr->re_stride) == 0)
+		    !((value - expr->re_lo) % expr->re_stride))
 			return 1;
 	}
 
@@ -433,21 +433,21 @@ EXPORT_SYMBOL(cfs_expr_list_match);
  * \retval < 0 for failure
  */
 int
-cfs_expr_list_values(struct cfs_expr_list *expr_list, int max, __u32 **valpp)
+cfs_expr_list_values(struct cfs_expr_list *expr_list, int max, u32 **valpp)
 {
-	struct cfs_range_expr	*expr;
-	__u32			*val;
-	int			count = 0;
-	int			i;
+	struct cfs_range_expr *expr;
+	u32 *val;
+	int count = 0;
+	int i;
 
 	list_for_each_entry(expr, &expr_list->el_exprs, re_link) {
 		for (i = expr->re_lo; i <= expr->re_hi; i++) {
-			if (((i - expr->re_lo) % expr->re_stride) == 0)
+			if (!((i - expr->re_lo) % expr->re_stride))
 				count++;
 		}
 	}
 
-	if (count == 0) /* empty expression list */
+	if (!count) /* empty expression list */
 		return 0;
 
 	if (count > max) {
@@ -463,7 +463,7 @@ cfs_expr_list_values(struct cfs_expr_list *expr_list, int max, __u32 **valpp)
 	count = 0;
 	list_for_each_entry(expr, &expr_list->el_exprs, re_link) {
 		for (i = expr->re_lo; i <= expr->re_hi; i++) {
-			if (((i - expr->re_lo) % expr->re_stride) == 0)
+			if (!((i - expr->re_lo) % expr->re_stride))
 				val[count++] = i;
 		}
 	}
@@ -501,13 +501,13 @@ EXPORT_SYMBOL(cfs_expr_list_free);
  * \retval -errno otherwise
  */
 int
-cfs_expr_list_parse(char *str, int len, unsigned min, unsigned max,
+cfs_expr_list_parse(char *str, int len, unsigned int min, unsigned int max,
 		    struct cfs_expr_list **elpp)
 {
-	struct cfs_expr_list	*expr_list;
-	struct cfs_range_expr	*expr;
-	struct cfs_lstr		src;
-	int			rc;
+	struct cfs_expr_list *expr_list;
+	struct cfs_range_expr *expr;
+	struct cfs_lstr	src;
+	int rc;
 
 	LIBCFS_ALLOC(expr_list, sizeof(*expr_list));
 	if (!expr_list)
@@ -533,18 +533,18 @@ cfs_expr_list_parse(char *str, int len, unsigned min, unsigned max,
 			}
 
 			rc = cfs_range_expr_parse(&tok, min, max, 1, &expr);
-			if (rc != 0)
+			if (rc)
 				break;
 
 			list_add_tail(&expr->re_link, &expr_list->el_exprs);
 		}
 	} else {
 		rc = cfs_range_expr_parse(&src, min, max, 0, &expr);
-		if (rc == 0)
+		if (!rc)
 			list_add_tail(&expr->re_link, &expr_list->el_exprs);
 	}
 
-	if (rc != 0)
+	if (rc)
 		cfs_expr_list_free(expr_list);
 	else
 		*elpp = expr_list;
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
index e8b1a61420de..6b9cf06e8df2 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
@@ -55,6 +55,8 @@ MODULE_PARM_DESC(cpu_npartitions, "# of CPU partitions");
  * i.e: "N 0[0,1] 1[2,3]" the first character 'N' means numbers in bracket
  *       are NUMA node ID, number before bracket is CPU partition ID.
  *
+ * i.e: "N", shortcut expression to create CPT from NUMA & CPU topology
+ *
  * NB: If user specified cpu_pattern, cpu_npartitions will be ignored
  */
 static char	*cpu_pattern = "";
@@ -88,7 +90,7 @@ cfs_node_to_cpumask(int node, cpumask_t *mask)
 void
 cfs_cpt_table_free(struct cfs_cpt_table *cptab)
 {
-	int	i;
+	int i;
 
 	if (cptab->ctb_cpu2cpt) {
 		LIBCFS_FREE(cptab->ctb_cpu2cpt,
@@ -126,7 +128,7 @@ struct cfs_cpt_table *
 cfs_cpt_table_alloc(unsigned int ncpt)
 {
 	struct cfs_cpt_table *cptab;
-	int	i;
+	int i;
 
 	LIBCFS_ALLOC(cptab, sizeof(*cptab));
 	if (!cptab)
@@ -177,10 +179,10 @@ EXPORT_SYMBOL(cfs_cpt_table_alloc);
 int
 cfs_cpt_table_print(struct cfs_cpt_table *cptab, char *buf, int len)
 {
-	char	*tmp = buf;
-	int	rc = 0;
-	int	i;
-	int	j;
+	char *tmp = buf;
+	int rc = 0;
+	int i;
+	int j;
 
 	for (i = 0; i < cptab->ctb_nparts; i++) {
 		if (len > 0) {
@@ -271,7 +273,7 @@ EXPORT_SYMBOL(cfs_cpt_nodemask);
 int
 cfs_cpt_set_cpu(struct cfs_cpt_table *cptab, int cpt, int cpu)
 {
-	int	node;
+	int node;
 
 	LASSERT(cpt >= 0 && cpt < cptab->ctb_nparts);
 
@@ -311,8 +313,8 @@ EXPORT_SYMBOL(cfs_cpt_set_cpu);
 void
 cfs_cpt_unset_cpu(struct cfs_cpt_table *cptab, int cpt, int cpu)
 {
-	int	node;
-	int	i;
+	int node;
+	int i;
 
 	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
 
@@ -371,9 +373,9 @@ EXPORT_SYMBOL(cfs_cpt_unset_cpu);
 int
 cfs_cpt_set_cpumask(struct cfs_cpt_table *cptab, int cpt, cpumask_t *mask)
 {
-	int	i;
+	int i;
 
-	if (cpumask_weight(mask) == 0 ||
+	if (!cpumask_weight(mask) ||
 	    cpumask_any_and(mask, cpu_online_mask) >= nr_cpu_ids) {
 		CDEBUG(D_INFO, "No online CPU is found in the CPU mask for CPU partition %d\n",
 		       cpt);
@@ -392,7 +394,7 @@ EXPORT_SYMBOL(cfs_cpt_set_cpumask);
 void
 cfs_cpt_unset_cpumask(struct cfs_cpt_table *cptab, int cpt, cpumask_t *mask)
 {
-	int	i;
+	int i;
 
 	for_each_cpu(i, mask)
 		cfs_cpt_unset_cpu(cptab, cpt, i);
@@ -402,8 +404,8 @@ EXPORT_SYMBOL(cfs_cpt_unset_cpumask);
 int
 cfs_cpt_set_node(struct cfs_cpt_table *cptab, int cpt, int node)
 {
-	cpumask_t	*mask;
-	int		rc;
+	cpumask_t *mask;
+	int rc;
 
 	if (node < 0 || node >= MAX_NUMNODES) {
 		CDEBUG(D_INFO,
@@ -449,7 +451,7 @@ EXPORT_SYMBOL(cfs_cpt_unset_node);
 int
 cfs_cpt_set_nodemask(struct cfs_cpt_table *cptab, int cpt, nodemask_t *mask)
 {
-	int	i;
+	int i;
 
 	for_each_node_mask(i, *mask) {
 		if (!cfs_cpt_set_node(cptab, cpt, i))
@@ -463,7 +465,7 @@ EXPORT_SYMBOL(cfs_cpt_set_nodemask);
 void
 cfs_cpt_unset_nodemask(struct cfs_cpt_table *cptab, int cpt, nodemask_t *mask)
 {
-	int	i;
+	int i;
 
 	for_each_node_mask(i, *mask)
 		cfs_cpt_unset_node(cptab, cpt, i);
@@ -473,8 +475,8 @@ EXPORT_SYMBOL(cfs_cpt_unset_nodemask);
 void
 cfs_cpt_clear(struct cfs_cpt_table *cptab, int cpt)
 {
-	int	last;
-	int	i;
+	int last;
+	int i;
 
 	if (cpt == CFS_CPT_ANY) {
 		last = cptab->ctb_nparts - 1;
@@ -493,10 +495,10 @@ EXPORT_SYMBOL(cfs_cpt_clear);
 int
 cfs_cpt_spread_node(struct cfs_cpt_table *cptab, int cpt)
 {
-	nodemask_t	*mask;
-	int		weight;
-	int		rotor;
-	int		node;
+	nodemask_t *mask;
+	int weight;
+	int rotor;
+	int node;
 
 	/* convert CPU partition ID to HW node id */
 
@@ -514,7 +516,7 @@ cfs_cpt_spread_node(struct cfs_cpt_table *cptab, int cpt)
 	rotor %= weight;
 
 	for_each_node_mask(node, *mask) {
-		if (rotor-- == 0)
+		if (!rotor--)
 			return node;
 	}
 
@@ -526,8 +528,8 @@ EXPORT_SYMBOL(cfs_cpt_spread_node);
 int
 cfs_cpt_current(struct cfs_cpt_table *cptab, int remap)
 {
-	int	cpu = smp_processor_id();
-	int	cpt = cptab->ctb_cpu2cpt[cpu];
+	int cpu = smp_processor_id();
+	int cpt = cptab->ctb_cpu2cpt[cpu];
 
 	if (cpt < 0) {
 		if (!remap)
@@ -555,10 +557,10 @@ EXPORT_SYMBOL(cfs_cpt_of_cpu);
 int
 cfs_cpt_bind(struct cfs_cpt_table *cptab, int cpt)
 {
-	cpumask_t	*cpumask;
-	nodemask_t	*nodemask;
-	int		rc;
-	int		i;
+	cpumask_t *cpumask;
+	nodemask_t *nodemask;
+	int rc;
+	int i;
 
 	LASSERT(cpt == CFS_CPT_ANY || (cpt >= 0 && cpt < cptab->ctb_nparts));
 
@@ -582,7 +584,7 @@ cfs_cpt_bind(struct cfs_cpt_table *cptab, int cpt)
 
 		rc = set_cpus_allowed_ptr(current, cpumask);
 		set_mems_allowed(*nodemask);
-		if (rc == 0)
+		if (!rc)
 			schedule(); /* switch to allowed CPU */
 
 		return rc;
@@ -601,10 +603,10 @@ static int
 cfs_cpt_choose_ncpus(struct cfs_cpt_table *cptab, int cpt,
 		     cpumask_t *node, int number)
 {
-	cpumask_t	*socket = NULL;
-	cpumask_t	*core = NULL;
-	int		rc = 0;
-	int		cpu;
+	cpumask_t *socket = NULL;
+	cpumask_t *core = NULL;
+	int rc = 0;
+	int cpu;
 
 	LASSERT(number > 0);
 
@@ -638,7 +640,7 @@ cfs_cpt_choose_ncpus(struct cfs_cpt_table *cptab, int cpt,
 		LASSERT(!cpumask_empty(socket));
 
 		while (!cpumask_empty(socket)) {
-			int     i;
+			int i;
 
 			/* get cpumask for hts in the same core */
 			cpumask_copy(core, topology_sibling_cpumask(cpu));
@@ -656,14 +658,14 @@ cfs_cpt_choose_ncpus(struct cfs_cpt_table *cptab, int cpt,
 					goto out;
 				}
 
-				if (--number == 0)
+				if (!--number)
 					goto out;
 			}
 			cpu = cpumask_first(socket);
 		}
 	}
 
- out:
+out:
 	if (socket)
 		LIBCFS_FREE(socket, cpumask_size());
 	if (core)
@@ -676,9 +678,9 @@ cfs_cpt_choose_ncpus(struct cfs_cpt_table *cptab, int cpt,
 static unsigned int
 cfs_cpt_num_estimate(void)
 {
-	unsigned nnode = num_online_nodes();
-	unsigned ncpu  = num_online_cpus();
-	unsigned ncpt;
+	unsigned int nnode = num_online_nodes();
+	unsigned int ncpu = num_online_cpus();
+	unsigned int ncpt;
 
 	if (ncpu <= CPT_WEIGHT_MIN) {
 		ncpt = 1;
@@ -703,14 +705,14 @@ cfs_cpt_num_estimate(void)
 
 	ncpt = nnode;
 
- out:
+out:
 #if (BITS_PER_LONG == 32)
 	/* config many CPU partitions on 32-bit system could consume
 	 * too much memory
 	 */
 	ncpt = min(2U, ncpt);
 #endif
-	while (ncpu % ncpt != 0)
+	while (ncpu % ncpt)
 		ncpt--; /* worst case is 1 */
 
 	return ncpt;
@@ -720,11 +722,11 @@ static struct cfs_cpt_table *
 cfs_cpt_table_create(int ncpt)
 {
 	struct cfs_cpt_table *cptab = NULL;
-	cpumask_t	*mask = NULL;
-	int		cpt = 0;
-	int		num;
-	int		rc;
-	int		i;
+	cpumask_t *mask = NULL;
+	int cpt = 0;
+	int num;
+	int rc;
+	int i;
 
 	rc = cfs_cpt_num_estimate();
 	if (ncpt <= 0)
@@ -735,7 +737,7 @@ cfs_cpt_table_create(int ncpt)
 		      ncpt, rc);
 	}
 
-	if (num_online_cpus() % ncpt != 0) {
+	if (num_online_cpus() % ncpt) {
 		CERROR("CPU number %d is not multiple of cpu_npartition %d, please try different cpu_npartitions value or set pattern string by cpu_pattern=STRING\n",
 		       (int)num_online_cpus(), ncpt);
 		goto failed;
@@ -748,7 +750,7 @@ cfs_cpt_table_create(int ncpt)
 	}
 
 	num = num_online_cpus() / ncpt;
-	if (num == 0) {
+	if (!num) {
 		CERROR("CPU changed while setting CPU partition\n");
 		goto failed;
 	}
@@ -764,7 +766,7 @@ cfs_cpt_table_create(int ncpt)
 
 		while (!cpumask_empty(mask)) {
 			struct cfs_cpu_partition *part;
-			int    n;
+			int n;
 
 			/*
 			 * Each emulated NUMA node has all allowed CPUs in
@@ -817,27 +819,36 @@ cfs_cpt_table_create(int ncpt)
 static struct cfs_cpt_table *
 cfs_cpt_table_create_pattern(char *pattern)
 {
-	struct cfs_cpt_table	*cptab;
-	char			*str	= pattern;
-	int			node	= 0;
-	int			high;
-	int			ncpt;
-	int			c;
-
-	for (ncpt = 0;; ncpt++) { /* quick scan bracket */
-		str = strchr(str, '[');
-		if (!str)
-			break;
-		str++;
-	}
+	struct cfs_cpt_table *cptab;
+	char *str;
+	int node = 0;
+	int high;
+	int ncpt = 0;
+	int cpt;
+	int rc;
+	int c;
+	int i;
 
 	str = cfs_trimwhite(pattern);
 	if (*str == 'n' || *str == 'N') {
 		pattern = str + 1;
-		node = 1;
+		if (*pattern != '\0') {
+			node = 1;
+		} else { /* shortcut to create CPT from NUMA & CPU topology */
+			node = -1;
+			ncpt = num_online_nodes();
+		}
+	}
+
+	if (!ncpt) { /* scanning bracket which is mark of partition */
+		for (str = pattern;; str++, ncpt++) {
+			str = strchr(str, '[');
+			if (!str)
+				break;
+		}
 	}
 
-	if (ncpt == 0 ||
+	if (!ncpt ||
 	    (node && ncpt > num_online_nodes()) ||
 	    (!node && ncpt > num_online_cpus())) {
 		CERROR("Invalid pattern %s, or too many partitions %d\n",
@@ -845,25 +856,39 @@ cfs_cpt_table_create_pattern(char *pattern)
 		return NULL;
 	}
 
-	high = node ? MAX_NUMNODES - 1 : nr_cpu_ids - 1;
-
 	cptab = cfs_cpt_table_alloc(ncpt);
 	if (!cptab) {
 		CERROR("Failed to allocate cpu partition table\n");
 		return NULL;
 	}
 
+	if (node < 0) { /* shortcut to create CPT from NUMA & CPU topology */
+		cpt = 0;
+
+		for_each_online_node(i) {
+			if (cpt >= ncpt) {
+				CERROR("CPU changed while setting CPU partition table, %d/%d\n",
+				       cpt, ncpt);
+				goto failed;
+			}
+
+			rc = cfs_cpt_set_node(cptab, cpt++, i);
+			if (!rc)
+				goto failed;
+		}
+		return cptab;
+	}
+
+	high = node ? MAX_NUMNODES - 1 : nr_cpu_ids - 1;
+
 	for (str = cfs_trimwhite(pattern), c = 0;; c++) {
-		struct cfs_range_expr	*range;
-		struct cfs_expr_list	*el;
-		char			*bracket = strchr(str, '[');
-		int			cpt;
-		int			rc;
-		int			i;
-		int			n;
+		struct cfs_range_expr *range;
+		struct cfs_expr_list *el;
+		char *bracket = strchr(str, '[');
+		int n;
 
 		if (!bracket) {
-			if (*str != 0) {
+			if (*str) {
 				CERROR("Invalid pattern %s\n", str);
 				goto failed;
 			}
@@ -886,7 +911,7 @@ cfs_cpt_table_create_pattern(char *pattern)
 			goto failed;
 		}
 
-		if (cfs_cpt_weight(cptab, cpt) != 0) {
+		if (cfs_cpt_weight(cptab, cpt)) {
 			CERROR("Partition %d has already been set.\n", cpt);
 			goto failed;
 		}
@@ -905,14 +930,14 @@ cfs_cpt_table_create_pattern(char *pattern)
 		}
 
 		if (cfs_expr_list_parse(str, (bracket - str) + 1,
-					0, high, &el) != 0) {
+					0, high, &el)) {
 			CERROR("Can't parse number range: %s\n", str);
 			goto failed;
 		}
 
 		list_for_each_entry(range, &el->el_exprs, re_link) {
 			for (i = range->re_lo; i <= range->re_hi; i++) {
-				if ((i - range->re_lo) % range->re_stride != 0)
+				if ((i - range->re_lo) % range->re_stride)
 					continue;
 
 				rc = node ? cfs_cpt_set_node(cptab, cpt, i) :
@@ -945,8 +970,8 @@ cfs_cpt_table_create_pattern(char *pattern)
 static int
 cfs_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
 {
-	unsigned int  cpu = (unsigned long)hcpu;
-	bool	     warn;
+	unsigned int cpu = (unsigned long)hcpu;
+	bool warn;
 
 	switch (action) {
 	case CPU_DEAD:
@@ -1019,7 +1044,7 @@ cfs_cpu_init(void)
 	register_hotcpu_notifier(&cfs_cpu_notifier);
 #endif
 
-	if (*cpu_pattern != 0) {
+	if (*cpu_pattern) {
 		cfs_cpt_table = cfs_cpt_table_create_pattern(cpu_pattern);
 		if (!cfs_cpt_table) {
 			CERROR("Failed to create cptab from pattern %s\n",
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c
index 7f56d2c9dd00..68e34b4a76c9 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c
@@ -64,7 +64,7 @@ static int cfs_crypto_hash_alloc(enum cfs_crypto_hash_alg hash_alg,
 				 unsigned int key_len)
 {
 	struct crypto_ahash *tfm;
-	int     err = 0;
+	int err = 0;
 
 	*type = cfs_crypto_hash_type(hash_alg);
 
@@ -93,12 +93,12 @@ static int cfs_crypto_hash_alloc(enum cfs_crypto_hash_alg hash_alg,
 
 	if (key)
 		err = crypto_ahash_setkey(tfm, key, key_len);
-	else if ((*type)->cht_key != 0)
+	else if ((*type)->cht_key)
 		err = crypto_ahash_setkey(tfm,
 					  (unsigned char *)&((*type)->cht_key),
 					  (*type)->cht_size);
 
-	if (err != 0) {
+	if (err) {
 		ahash_request_free(*req);
 		crypto_free_ahash(tfm);
 		return err;
@@ -147,16 +147,16 @@ int cfs_crypto_hash_digest(enum cfs_crypto_hash_alg hash_alg,
 			   unsigned char *key, unsigned int key_len,
 			   unsigned char *hash, unsigned int *hash_len)
 {
-	struct scatterlist	sl;
+	struct scatterlist sl;
 	struct ahash_request *req;
-	int			err;
-	const struct cfs_crypto_hash_type	*type;
+	int err;
+	const struct cfs_crypto_hash_type *type;
 
-	if (!buf || buf_len == 0 || !hash_len)
+	if (!buf || !buf_len || !hash_len)
 		return -EINVAL;
 
 	err = cfs_crypto_hash_alloc(hash_alg, &type, &req, key, key_len);
-	if (err != 0)
+	if (err)
 		return err;
 
 	if (!hash || *hash_len < type->cht_size) {
@@ -177,7 +177,7 @@ int cfs_crypto_hash_digest(enum cfs_crypto_hash_alg hash_alg,
 EXPORT_SYMBOL(cfs_crypto_hash_digest);
 
 /**
- * Allocate and initialize desriptor for hash algorithm.
+ * Allocate and initialize descriptor for hash algorithm.
  *
  * This should be used to initialize a hash descriptor for multiple calls
  * to a single hash function when computing the hash across multiple
@@ -198,8 +198,8 @@ cfs_crypto_hash_init(enum cfs_crypto_hash_alg hash_alg,
 		     unsigned char *key, unsigned int key_len)
 {
 	struct ahash_request *req;
-	int		     err;
-	const struct cfs_crypto_hash_type       *type;
+	int err;
+	const struct cfs_crypto_hash_type *type;
 
 	err = cfs_crypto_hash_alloc(hash_alg, &type, &req, key, key_len);
 
@@ -273,7 +273,7 @@ EXPORT_SYMBOL(cfs_crypto_hash_update);
 int cfs_crypto_hash_final(struct cfs_crypto_hash_desc *hdesc,
 			  unsigned char *hash, unsigned int *hash_len)
 {
-	int     err;
+	int err;
 	struct ahash_request *req = (void *)hdesc;
 	int size = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
 
@@ -312,8 +312,8 @@ static void cfs_crypto_performance_test(enum cfs_crypto_hash_alg hash_alg)
 {
 	int buf_len = max(PAGE_SIZE, 1048576UL);
 	void *buf;
-	unsigned long		   start, end;
-	int			     bcount, err = 0;
+	unsigned long start, end;
+	int bcount, err = 0;
 	struct page *page;
 	unsigned char hash[CFS_CRYPTO_HASH_DIGESTSIZE_MAX];
 	unsigned int hash_len = sizeof(hash);
@@ -358,7 +358,7 @@ out_err:
 		CDEBUG(D_INFO, "Crypto hash algorithm %s test error: rc = %d\n",
 		       cfs_crypto_hash_name(hash_alg), err);
 	} else {
-		unsigned long   tmp;
+		unsigned long tmp;
 
 		tmp = ((bcount * buf_len / jiffies_to_msecs(end - start)) *
 		       1000) / (1024 * 1024);
@@ -440,6 +440,6 @@ int cfs_crypto_register(void)
  */
 void cfs_crypto_unregister(void)
 {
-	if (adler32 == 0)
+	if (!adler32)
 		cfs_crypto_adler32_unregister();
 }
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.h b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.h
index 18e8cd4d8758..d0b3aa80cfa6 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.h
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.h
@@ -1,4 +1,4 @@
- /*
+/*
  * GPL HEADER START
  *
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-debug.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-debug.c
index 435b784c52f8..39a72e3f0c18 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-debug.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-debug.c
@@ -57,7 +57,6 @@
 
 #include <linux/kallsyms.h>
 
-char lnet_upcall[1024] = "/usr/lib/lustre/lnet_upcall";
 char lnet_debug_log_upcall[1024] = "/usr/lib/lustre/lnet_debug_log_upcall";
 
 /**
@@ -68,11 +67,12 @@ char lnet_debug_log_upcall[1024] = "/usr/lib/lustre/lnet_debug_log_upcall";
 void libcfs_run_debug_log_upcall(char *file)
 {
 	char *argv[3];
-	int   rc;
-	char *envp[] = {
+	int rc;
+	static const char * const envp[] = {
 		"HOME=/",
 		"PATH=/sbin:/bin:/usr/sbin:/usr/bin",
-		NULL};
+		NULL
+	};
 
 	argv[0] = lnet_debug_log_upcall;
 
@@ -81,7 +81,7 @@ void libcfs_run_debug_log_upcall(char *file)
 
 	argv[2] = NULL;
 
-	rc = call_usermodehelper(argv[0], argv, envp, 1);
+	rc = call_usermodehelper(argv[0], argv, (char **)envp, 1);
 	if (rc < 0 && rc != -ENOENT) {
 		CERROR("Error %d invoking LNET debug log upcall %s %s; check /sys/kernel/debug/lnet/debug_log_upcall\n",
 		       rc, argv[0], argv[1]);
@@ -91,57 +91,6 @@ void libcfs_run_debug_log_upcall(char *file)
 	}
 }
 
-void libcfs_run_upcall(char **argv)
-{
-	int   rc;
-	int   argc;
-	char *envp[] = {
-		"HOME=/",
-		"PATH=/sbin:/bin:/usr/sbin:/usr/bin",
-		NULL};
-
-	argv[0] = lnet_upcall;
-	argc = 1;
-	while (argv[argc])
-		argc++;
-
-	LASSERT(argc >= 2);
-
-	rc = call_usermodehelper(argv[0], argv, envp, 1);
-	if (rc < 0 && rc != -ENOENT) {
-		CERROR("Error %d invoking LNET upcall %s %s%s%s%s%s%s%s%s; check /sys/kernel/debug/lnet/upcall\n",
-		       rc, argv[0], argv[1],
-		       argc < 3 ? "" : ",", argc < 3 ? "" : argv[2],
-		       argc < 4 ? "" : ",", argc < 4 ? "" : argv[3],
-		       argc < 5 ? "" : ",", argc < 5 ? "" : argv[4],
-		       argc < 6 ? "" : ",...");
-	} else {
-		CDEBUG(D_HA, "Invoked LNET upcall %s %s%s%s%s%s%s%s%s\n",
-		       argv[0], argv[1],
-		       argc < 3 ? "" : ",", argc < 3 ? "" : argv[2],
-		       argc < 4 ? "" : ",", argc < 4 ? "" : argv[3],
-		       argc < 5 ? "" : ",", argc < 5 ? "" : argv[4],
-		       argc < 6 ? "" : ",...");
-	}
-}
-
-void libcfs_run_lbug_upcall(struct libcfs_debug_msg_data *msgdata)
-{
-	char *argv[6];
-	char buf[32];
-
-	snprintf(buf, sizeof(buf), "%d", msgdata->msg_line);
-
-	argv[1] = "LBUG";
-	argv[2] = (char *)msgdata->msg_file;
-	argv[3] = (char *)msgdata->msg_fn;
-	argv[4] = buf;
-	argv[5] = NULL;
-
-	libcfs_run_upcall(argv);
-}
-EXPORT_SYMBOL(libcfs_run_lbug_upcall);
-
 /* coverity[+kill] */
 void __noreturn lbug_with_loc(struct libcfs_debug_msg_data *msgdata)
 {
@@ -156,7 +105,6 @@ void __noreturn lbug_with_loc(struct libcfs_debug_msg_data *msgdata)
 	dump_stack();
 	if (!libcfs_panic_on_lbug)
 		libcfs_debug_dumplog();
-	libcfs_run_lbug_upcall(msgdata);
 	if (libcfs_panic_on_lbug)
 		panic("LBUG");
 	set_task_state(current, TASK_UNINTERRUPTIBLE);
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-module.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-module.c
index 38308f8b6aae..3f5d58babc2f 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-module.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-module.c
@@ -83,7 +83,7 @@ static inline bool libcfs_ioctl_is_invalid(struct libcfs_ioctl_data *data)
 		CERROR("LIBCFS ioctl: plen2 nonzero but no pbuf2 pointer\n");
 		return true;
 	}
-	if ((__u32)libcfs_ioctl_packlen(data) != data->ioc_hdr.ioc_len) {
+	if ((u32)libcfs_ioctl_packlen(data) != data->ioc_hdr.ioc_len) {
 		CERROR("LIBCFS ioctl: packlen != ioc_len\n");
 		return true;
 	}
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-prim.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-prim.c
index 291d286eab48..cf902154f0aa 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-prim.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-prim.c
@@ -45,8 +45,8 @@
 sigset_t
 cfs_block_allsigs(void)
 {
-	unsigned long	  flags;
-	sigset_t	old;
+	unsigned long flags;
+	sigset_t old;
 
 	spin_lock_irqsave(&current->sighand->siglock, flags);
 	old = current->blocked;
@@ -60,8 +60,8 @@ EXPORT_SYMBOL(cfs_block_allsigs);
 
 sigset_t cfs_block_sigs(unsigned long sigs)
 {
-	unsigned long  flags;
-	sigset_t	old;
+	unsigned long flags;
+	sigset_t old;
 
 	spin_lock_irqsave(&current->sighand->siglock, flags);
 	old = current->blocked;
@@ -91,7 +91,7 @@ EXPORT_SYMBOL(cfs_block_sigsinv);
 void
 cfs_restore_sigs(sigset_t old)
 {
-	unsigned long  flags;
+	unsigned long flags;
 
 	spin_lock_irqsave(&current->sighand->siglock, flags);
 	current->blocked = old;
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-tracefile.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-tracefile.c
index 8b551d2708ba..75eb84e7f0f8 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-tracefile.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-tracefile.c
@@ -49,8 +49,8 @@ static DECLARE_RWSEM(cfs_tracefile_sem);
 
 int cfs_tracefile_init_arch(void)
 {
-	int    i;
-	int    j;
+	int i;
+	int j;
 	struct cfs_trace_cpu_data *tcd;
 
 	/* initialize trace_data */
@@ -85,14 +85,14 @@ int cfs_tracefile_init_arch(void)
 
 out:
 	cfs_tracefile_fini_arch();
-	printk(KERN_ERR "lnet: Not enough memory\n");
+	pr_err("lnet: Not enough memory\n");
 	return -ENOMEM;
 }
 
 void cfs_tracefile_fini_arch(void)
 {
-	int    i;
-	int    j;
+	int i;
+	int j;
 
 	for (i = 0; i < num_possible_cpus(); i++)
 		for (j = 0; j < 3; j++) {
@@ -224,26 +224,26 @@ void cfs_print_to_console(struct ptldebug_header *hdr, int mask,
 {
 	char *prefix = "Lustre", *ptype = NULL;
 
-	if ((mask & D_EMERG) != 0) {
+	if (mask & D_EMERG) {
 		prefix = dbghdr_to_err_string(hdr);
 		ptype = KERN_EMERG;
-	} else if ((mask & D_ERROR) != 0) {
+	} else if (mask & D_ERROR) {
 		prefix = dbghdr_to_err_string(hdr);
 		ptype = KERN_ERR;
-	} else if ((mask & D_WARNING) != 0) {
+	} else if (mask & D_WARNING) {
 		prefix = dbghdr_to_info_string(hdr);
 		ptype = KERN_WARNING;
-	} else if ((mask & (D_CONSOLE | libcfs_printk)) != 0) {
+	} else if (mask & (D_CONSOLE | libcfs_printk)) {
 		prefix = dbghdr_to_info_string(hdr);
 		ptype = KERN_INFO;
 	}
 
-	if ((mask & D_CONSOLE) != 0) {
-		printk("%s%s: %.*s", ptype, prefix, len, buf);
+	if (mask & D_CONSOLE) {
+		pr_info("%s%s: %.*s", ptype, prefix, len, buf);
 	} else {
-		printk("%s%s: %d:%d:(%s:%d:%s()) %.*s", ptype, prefix,
-		       hdr->ph_pid, hdr->ph_extern_pid, file, hdr->ph_line_num,
-		       fn, len, buf);
+		pr_info("%s%s: %d:%d:(%s:%d:%s()) %.*s", ptype, prefix,
+			hdr->ph_pid, hdr->ph_extern_pid, file,
+			hdr->ph_line_num, fn, len, buf);
 	}
 }
 
diff --git a/drivers/staging/lustre/lnet/libcfs/module.c b/drivers/staging/lustre/lnet/libcfs/module.c
index 86b4d25cad46..161e04226521 100644
--- a/drivers/staging/lustre/lnet/libcfs/module.c
+++ b/drivers/staging/lustre/lnet/libcfs/module.c
@@ -183,12 +183,12 @@ EXPORT_SYMBOL(lprocfs_call_handler);
 static int __proc_dobitmasks(void *data, int write,
 			     loff_t pos, void __user *buffer, int nob)
 {
-	const int     tmpstrlen = 512;
-	char	 *tmpstr;
-	int	   rc;
+	const int tmpstrlen = 512;
+	char *tmpstr;
+	int rc;
 	unsigned int *mask = data;
-	int	   is_subsys = (mask == &libcfs_subsystem_debug) ? 1 : 0;
-	int	   is_printk = (mask == &libcfs_printk) ? 1 : 0;
+	int is_subsys = (mask == &libcfs_subsystem_debug) ? 1 : 0;
+	int is_printk = (mask == &libcfs_printk) ? 1 : 0;
 
 	rc = cfs_trace_allocate_string_buffer(&tmpstr, tmpstrlen);
 	if (rc < 0)
@@ -293,8 +293,8 @@ static int __proc_cpt_table(void *data, int write,
 			    loff_t pos, void __user *buffer, int nob)
 {
 	char *buf = NULL;
-	int   len = 4096;
-	int   rc  = 0;
+	int len = 4096;
+	int rc  = 0;
 
 	if (write)
 		return -EPERM;
@@ -365,14 +365,6 @@ static struct ctl_table lnet_table[] = {
 		.mode     = 0444,
 		.proc_handler = &proc_cpt_table,
 	},
-
-	{
-		.procname = "upcall",
-		.data     = lnet_upcall,
-		.maxlen   = sizeof(lnet_upcall),
-		.mode     = 0644,
-		.proc_handler = &proc_dostring,
-	},
 	{
 		.procname = "debug_log_upcall",
 		.data     = lnet_debug_log_upcall,
@@ -547,7 +539,7 @@ static int libcfs_init(void)
 	}
 
 	rc = cfs_cpu_init();
-	if (rc != 0)
+	if (rc)
 		goto cleanup_debug;
 
 	rc = misc_register(&libcfs_dev);
@@ -566,7 +558,7 @@ static int libcfs_init(void)
 	rc = min(cfs_cpt_weight(cfs_cpt_table, CFS_CPT_ANY), 4);
 	rc = cfs_wi_sched_create("cfs_rh", cfs_cpt_table, CFS_CPT_ANY,
 				 rc, &cfs_sched_rehash);
-	if (rc != 0) {
+	if (rc) {
 		CERROR("Startup workitem scheduler: error: %d\n", rc);
 		goto cleanup_deregister;
 	}
diff --git a/drivers/staging/lustre/lnet/libcfs/prng.c b/drivers/staging/lustre/lnet/libcfs/prng.c
index a9bdb284fd15..21d5a3912c5f 100644
--- a/drivers/staging/lustre/lnet/libcfs/prng.c
+++ b/drivers/staging/lustre/lnet/libcfs/prng.c
@@ -33,7 +33,7 @@
  * x(n)=a*x(n-1)+carry mod 2^16 and y(n)=b*y(n-1)+carry mod 2^16,
  * number and carry packed within the same 32 bit integer.
  * algorithm recommended by Marsaglia
-*/
+ */
 
 #include "../../include/linux/libcfs/libcfs.h"
 
diff --git a/drivers/staging/lustre/lnet/libcfs/tracefile.c b/drivers/staging/lustre/lnet/libcfs/tracefile.c
index 1c7efdfaffcf..d7b29f8997c0 100644
--- a/drivers/staging/lustre/lnet/libcfs/tracefile.c
+++ b/drivers/staging/lustre/lnet/libcfs/tracefile.c
@@ -59,13 +59,13 @@ struct page_collection {
 	 * ->tcd_daemon_pages and ->tcd_pages to the ->pc_pages. Otherwise,
 	 * only ->tcd_pages are spilled.
 	 */
-	int		pc_want_daemon_pages;
+	int			pc_want_daemon_pages;
 };
 
 struct tracefiled_ctl {
 	struct completion	tctl_start;
 	struct completion	tctl_stop;
-	wait_queue_head_t		tctl_waitq;
+	wait_queue_head_t	tctl_waitq;
 	pid_t			tctl_pid;
 	atomic_t		tctl_shutdown;
 };
@@ -77,24 +77,24 @@ struct cfs_trace_page {
 	/*
 	 * page itself
 	 */
-	struct page	  *page;
+	struct page		*page;
 	/*
 	 * linkage into one of the lists in trace_data_union or
 	 * page_collection
 	 */
-	struct list_head	   linkage;
+	struct list_head	linkage;
 	/*
 	 * number of bytes used within this page
 	 */
-	unsigned int	 used;
+	unsigned int		used;
 	/*
 	 * cpu that owns this page
 	 */
-	unsigned short       cpu;
+	unsigned short		cpu;
 	/*
 	 * type(context) of this page
 	 */
-	unsigned short       type;
+	unsigned short		type;
 };
 
 static void put_pages_on_tcd_daemon_list(struct page_collection *pc,
@@ -108,7 +108,7 @@ cfs_tage_from_list(struct list_head *list)
 
 static struct cfs_trace_page *cfs_tage_alloc(gfp_t gfp)
 {
-	struct page	    *page;
+	struct page *page;
 	struct cfs_trace_page *tage;
 
 	/* My caller is trying to free memory */
@@ -236,7 +236,7 @@ static void cfs_tcd_shrink(struct cfs_trace_cpu_data *tcd)
 	INIT_LIST_HEAD(&pc.pc_pages);
 
 	list_for_each_entry_safe(tage, tmp, &tcd->tcd_pages, linkage) {
-		if (pgcount-- == 0)
+		if (!pgcount--)
 			break;
 
 		list_move_tail(&tage->linkage, &pc.pc_pages);
@@ -278,7 +278,7 @@ int libcfs_debug_msg(struct libcfs_debug_msg_data *msgdata,
 		     const char *format, ...)
 {
 	va_list args;
-	int     rc;
+	int rc;
 
 	va_start(args, format);
 	rc = libcfs_debug_vmsg2(msgdata, format, args, NULL);
@@ -293,21 +293,21 @@ int libcfs_debug_vmsg2(struct libcfs_debug_msg_data *msgdata,
 		       const char *format2, ...)
 {
 	struct cfs_trace_cpu_data *tcd = NULL;
-	struct ptldebug_header     header = {0};
-	struct cfs_trace_page     *tage;
+	struct ptldebug_header header = { 0 };
+	struct cfs_trace_page *tage;
 	/* string_buf is used only if tcd != NULL, and is always set then */
-	char		      *string_buf = NULL;
-	char		      *debug_buf;
-	int			known_size;
-	int			needed = 85; /* average message length */
-	int			max_nob;
-	va_list		    ap;
-	int			depth;
-	int			i;
-	int			remain;
-	int			mask = msgdata->msg_mask;
-	const char		*file = kbasename(msgdata->msg_file);
-	struct cfs_debug_limit_state   *cdls = msgdata->msg_cdls;
+	char *string_buf = NULL;
+	char *debug_buf;
+	int known_size;
+	int needed = 85; /* average message length */
+	int max_nob;
+	va_list ap;
+	int depth;
+	int i;
+	int remain;
+	int mask = msgdata->msg_mask;
+	const char *file = kbasename(msgdata->msg_file);
+	struct cfs_debug_limit_state *cdls = msgdata->msg_cdls;
 
 	tcd = cfs_trace_get_tcd();
 
@@ -320,7 +320,7 @@ int libcfs_debug_vmsg2(struct libcfs_debug_msg_data *msgdata,
 	if (!tcd)		/* arch may not log in IRQ context */
 		goto console;
 
-	if (tcd->tcd_cur_pages == 0)
+	if (!tcd->tcd_cur_pages)
 		header.ph_flags |= PH_FLAG_FIRST_RECORD;
 
 	if (tcd->tcd_shutting_down) {
@@ -423,7 +423,7 @@ int libcfs_debug_vmsg2(struct libcfs_debug_msg_data *msgdata,
 	__LASSERT(tage->used <= PAGE_SIZE);
 
 console:
-	if ((mask & libcfs_printk) == 0) {
+	if (!(mask & libcfs_printk)) {
 		/* no console output requested */
 		if (tcd)
 			cfs_trace_put_tcd(tcd);
@@ -432,7 +432,7 @@ console:
 
 	if (cdls) {
 		if (libcfs_console_ratelimit &&
-		    cdls->cdls_next != 0 &&     /* not first time ever */
+		    cdls->cdls_next &&		/* not first time ever */
 		    !cfs_time_after(cfs_time_current(), cdls->cdls_next)) {
 			/* skipping a console message */
 			cdls->cdls_count++;
@@ -489,7 +489,7 @@ console:
 		put_cpu();
 	}
 
-	if (cdls && cdls->cdls_count != 0) {
+	if (cdls && cdls->cdls_count) {
 		string_buf = cfs_trace_get_console_buffer();
 
 		needed = snprintf(string_buf, CFS_TRACE_CONSOLE_BUFFER_SIZE,
@@ -535,9 +535,9 @@ panic_collect_pages(struct page_collection *pc)
 	 * CPUs have been stopped during a panic.  If this isn't true for some
 	 * arch, this will have to be implemented separately in each arch.
 	 */
-	int			i;
-	int			j;
 	struct cfs_trace_cpu_data *tcd;
+	int i;
+	int j;
 
 	INIT_LIST_HEAD(&pc->pc_pages);
 
@@ -698,11 +698,11 @@ void cfs_trace_debug_print(void)
 
 int cfs_tracefile_dump_all_pages(char *filename)
 {
-	struct page_collection	pc;
-	struct file		*filp;
-	struct cfs_trace_page	*tage;
-	struct cfs_trace_page	*tmp;
-	char			*buf;
+	struct page_collection pc;
+	struct file *filp;
+	struct cfs_trace_page *tage;
+	struct cfs_trace_page *tmp;
+	char *buf;
 	mm_segment_t __oldfs;
 	int rc;
 
@@ -778,7 +778,7 @@ void cfs_trace_flush_pages(void)
 int cfs_trace_copyin_string(char *knl_buffer, int knl_buffer_nob,
 			    const char __user *usr_buffer, int usr_buffer_nob)
 {
-	int    nob;
+	int nob;
 
 	if (usr_buffer_nob > knl_buffer_nob)
 		return -EOVERFLOW;
@@ -810,7 +810,7 @@ int cfs_trace_copyout_string(char __user *usr_buffer, int usr_buffer_nob,
 	 * NB if 'append' != NULL, it's a single character to append to the
 	 * copied out string - usually "\n" or "" (i.e. a terminating zero byte)
 	 */
-	int   nob = strlen(knl_buffer);
+	int nob = strlen(knl_buffer);
 
 	if (nob > usr_buffer_nob)
 		nob = usr_buffer_nob;
@@ -843,16 +843,16 @@ int cfs_trace_allocate_string_buffer(char **str, int nob)
 
 int cfs_trace_dump_debug_buffer_usrstr(void __user *usr_str, int usr_str_nob)
 {
-	char	 *str;
-	int	   rc;
+	char *str;
+	int rc;
 
 	rc = cfs_trace_allocate_string_buffer(&str, usr_str_nob + 1);
-	if (rc != 0)
+	if (rc)
 		return rc;
 
 	rc = cfs_trace_copyin_string(str, usr_str_nob + 1,
 				     usr_str, usr_str_nob);
-	if (rc != 0)
+	if (rc)
 		goto out;
 
 	if (str[0] != '/') {
@@ -867,17 +867,17 @@ out:
 
 int cfs_trace_daemon_command(char *str)
 {
-	int       rc = 0;
+	int rc = 0;
 
 	cfs_tracefile_write_lock();
 
-	if (strcmp(str, "stop") == 0) {
+	if (!strcmp(str, "stop")) {
 		cfs_tracefile_write_unlock();
 		cfs_trace_stop_thread();
 		cfs_tracefile_write_lock();
 		memset(cfs_tracefile, 0, sizeof(cfs_tracefile));
 
-	} else if (strncmp(str, "size=", 5) == 0) {
+	} else if (!strncmp(str, "size=", 5)) {
 		unsigned long tmp;
 
 		rc = kstrtoul(str + 5, 10, &tmp);
@@ -909,15 +909,15 @@ int cfs_trace_daemon_command(char *str)
 int cfs_trace_daemon_command_usrstr(void __user *usr_str, int usr_str_nob)
 {
 	char *str;
-	int   rc;
+	int rc;
 
 	rc = cfs_trace_allocate_string_buffer(&str, usr_str_nob + 1);
-	if (rc != 0)
+	if (rc)
 		return rc;
 
 	rc = cfs_trace_copyin_string(str, usr_str_nob + 1,
 				     usr_str, usr_str_nob);
-	if (rc == 0)
+	if (!rc)
 		rc = cfs_trace_daemon_command(str);
 
 	kfree(str);
@@ -1003,7 +1003,7 @@ static int tracefiled(void *arg)
 
 		filp = NULL;
 		cfs_tracefile_read_lock();
-		if (cfs_tracefile[0] != 0) {
+		if (cfs_tracefile[0]) {
 			filp = filp_open(cfs_tracefile,
 					 O_CREAT | O_RDWR | O_LARGEFILE,
 					 0600);
@@ -1072,7 +1072,7 @@ static int tracefiled(void *arg)
 		__LASSERT(list_empty(&pc.pc_pages));
 end_loop:
 		if (atomic_read(&tctl->tctl_shutdown)) {
-			if (last_loop == 0) {
+			if (!last_loop) {
 				last_loop = 1;
 				continue;
 			} else {
@@ -1135,13 +1135,13 @@ void cfs_trace_stop_thread(void)
 int cfs_tracefile_init(int max_pages)
 {
 	struct cfs_trace_cpu_data *tcd;
-	int		    i;
-	int		    j;
-	int		    rc;
-	int		    factor;
+	int i;
+	int j;
+	int rc;
+	int factor;
 
 	rc = cfs_tracefile_init_arch();
-	if (rc != 0)
+	if (rc)
 		return rc;
 
 	cfs_tcd_for_each(tcd, i, j) {
diff --git a/drivers/staging/lustre/lnet/libcfs/tracefile.h b/drivers/staging/lustre/lnet/libcfs/tracefile.h
index d878676bc375..f644cbc5a277 100644
--- a/drivers/staging/lustre/lnet/libcfs/tracefile.h
+++ b/drivers/staging/lustre/lnet/libcfs/tracefile.h
@@ -45,7 +45,7 @@ enum cfs_trace_buf_type {
 /* trace file lock routines */
 
 #define TRACEFILE_NAME_SIZE 1024
-extern char      cfs_tracefile[TRACEFILE_NAME_SIZE];
+extern char cfs_tracefile[TRACEFILE_NAME_SIZE];
 extern long long cfs_tracefile_size;
 
 void libcfs_run_debug_log_upcall(char *file);
@@ -80,7 +80,7 @@ int cfs_trace_get_debug_mb(void);
 void libcfs_debug_dumplog_internal(void *arg);
 void libcfs_register_panic_notifier(void);
 void libcfs_unregister_panic_notifier(void);
-extern int  libcfs_panic_in_progress;
+extern int libcfs_panic_in_progress;
 int cfs_trace_max_debug_mb(void);
 
 #define TCD_MAX_PAGES (5 << (20 - PAGE_SHIFT))
@@ -113,14 +113,14 @@ union cfs_trace_data_union {
 		 * tcd_for_each_type_lock
 		 */
 		spinlock_t		tcd_lock;
-		unsigned long	   tcd_lock_flags;
+		unsigned long		tcd_lock_flags;
 
 		/*
 		 * pages with trace records not yet processed by tracefiled.
 		 */
-		struct list_head	      tcd_pages;
+		struct list_head	tcd_pages;
 		/* number of pages on ->tcd_pages */
-		unsigned long	   tcd_cur_pages;
+		unsigned long		tcd_cur_pages;
 
 		/*
 		 * pages with trace records already processed by
@@ -132,9 +132,9 @@ union cfs_trace_data_union {
 		 * (put_pages_on_daemon_list()). LRU pages from this list are
 		 * discarded when list grows too large.
 		 */
-		struct list_head	      tcd_daemon_pages;
+		struct list_head	tcd_daemon_pages;
 		/* number of pages on ->tcd_daemon_pages */
-		unsigned long	   tcd_cur_daemon_pages;
+		unsigned long		tcd_cur_daemon_pages;
 
 		/*
 		 * Maximal number of pages allowed on ->tcd_pages and
@@ -142,7 +142,7 @@ union cfs_trace_data_union {
 		 * Always TCD_MAX_PAGES * tcd_pages_factor / 100 in current
 		 * implementation.
 		 */
-		unsigned long	   tcd_max_pages;
+		unsigned long		tcd_max_pages;
 
 		/*
 		 * preallocated pages to write trace records into. Pages from
@@ -166,15 +166,15 @@ union cfs_trace_data_union {
 		 * TCD_STOCK_PAGES pagesful are consumed by trace records all
 		 * emitted in non-blocking contexts. Which is quite unlikely.
 		 */
-		struct list_head	      tcd_stock_pages;
+		struct list_head	tcd_stock_pages;
 		/* number of pages on ->tcd_stock_pages */
-		unsigned long	   tcd_cur_stock_pages;
+		unsigned long		tcd_cur_stock_pages;
 
-		unsigned short	  tcd_shutting_down;
-		unsigned short	  tcd_cpu;
-		unsigned short	  tcd_type;
+		unsigned short		tcd_shutting_down;
+		unsigned short		tcd_cpu;
+		unsigned short		tcd_type;
 		/* The factors to share debug memory. */
-		unsigned short	  tcd_pages_factor;
+		unsigned short		tcd_pages_factor;
 	} tcd;
 	char __pad[L1_CACHE_ALIGN(sizeof(struct cfs_trace_cpu_data))];
 };
diff --git a/drivers/staging/lustre/lnet/libcfs/workitem.c b/drivers/staging/lustre/lnet/libcfs/workitem.c
index e98c818a14fb..d0512da6bcde 100644
--- a/drivers/staging/lustre/lnet/libcfs/workitem.c
+++ b/drivers/staging/lustre/lnet/libcfs/workitem.c
@@ -45,7 +45,7 @@ struct cfs_wi_sched {
 	/* chain on global list */
 	struct list_head		ws_list;
 	/** serialised workitems */
-	spinlock_t		ws_lock;
+	spinlock_t			ws_lock;
 	/** where schedulers sleep */
 	wait_queue_head_t		ws_waitq;
 	/** concurrent workitems */
@@ -59,26 +59,26 @@ struct cfs_wi_sched {
 	 */
 	struct list_head		ws_rerunq;
 	/** CPT-table for this scheduler */
-	struct cfs_cpt_table	*ws_cptab;
+	struct cfs_cpt_table		*ws_cptab;
 	/** CPT id for affinity */
-	int			ws_cpt;
+	int				ws_cpt;
 	/** number of scheduled workitems */
-	int			ws_nscheduled;
+	int				ws_nscheduled;
 	/** started scheduler thread, protected by cfs_wi_data::wi_glock */
-	unsigned int		ws_nthreads:30;
+	unsigned int			ws_nthreads:30;
 	/** shutting down, protected by cfs_wi_data::wi_glock */
-	unsigned int		ws_stopping:1;
+	unsigned int			ws_stopping:1;
 	/** serialize starting thread, protected by cfs_wi_data::wi_glock */
-	unsigned int		ws_starting:1;
+	unsigned int			ws_starting:1;
 	/** scheduler name */
-	char			ws_name[CFS_WS_NAME_LEN];
+	char				ws_name[CFS_WS_NAME_LEN];
 };
 
 static struct cfs_workitem_data {
 	/** serialize */
 	spinlock_t		wi_glock;
 	/** list of all schedulers */
-	struct list_head		wi_scheds;
+	struct list_head	wi_scheds;
 	/** WI module is initialized */
 	int			wi_init;
 	/** shutting down the whole WI module */
@@ -136,7 +136,7 @@ EXPORT_SYMBOL(cfs_wi_exit);
 int
 cfs_wi_deschedule(struct cfs_wi_sched *sched, struct cfs_workitem *wi)
 {
-	int	rc;
+	int rc;
 
 	LASSERT(!in_interrupt()); /* because we use plain spinlock */
 	LASSERT(!sched->ws_stopping);
@@ -202,13 +202,13 @@ EXPORT_SYMBOL(cfs_wi_schedule);
 
 static int cfs_wi_scheduler(void *arg)
 {
-	struct cfs_wi_sched	*sched = (struct cfs_wi_sched *)arg;
+	struct cfs_wi_sched *sched = (struct cfs_wi_sched *)arg;
 
 	cfs_block_allsigs();
 
 	/* CPT affinity scheduler? */
 	if (sched->ws_cptab)
-		if (cfs_cpt_bind(sched->ws_cptab, sched->ws_cpt) != 0)
+		if (cfs_cpt_bind(sched->ws_cptab, sched->ws_cpt))
 			CWARN("Failed to bind %s on CPT %d\n",
 			      sched->ws_name, sched->ws_cpt);
 
@@ -223,8 +223,8 @@ static int cfs_wi_scheduler(void *arg)
 	spin_lock(&sched->ws_lock);
 
 	while (!sched->ws_stopping) {
-		int	     nloops = 0;
-		int	     rc;
+		int nloops = 0;
+		int rc;
 		struct cfs_workitem *wi;
 
 		while (!list_empty(&sched->ws_runq) &&
@@ -238,16 +238,16 @@ static int cfs_wi_scheduler(void *arg)
 			LASSERT(sched->ws_nscheduled > 0);
 			sched->ws_nscheduled--;
 
-			wi->wi_running   = 1;
+			wi->wi_running = 1;
 			wi->wi_scheduled = 0;
 
 			spin_unlock(&sched->ws_lock);
 			nloops++;
 
-			rc = (*wi->wi_action) (wi);
+			rc = (*wi->wi_action)(wi);
 
 			spin_lock(&sched->ws_lock);
-			if (rc != 0) /* WI should be dead, even be freed! */
+			if (rc) /* WI should be dead, even be freed! */
 				continue;
 
 			wi->wi_running = 0;
@@ -273,7 +273,7 @@ static int cfs_wi_scheduler(void *arg)
 
 		spin_unlock(&sched->ws_lock);
 		rc = wait_event_interruptible_exclusive(sched->ws_waitq,
-						!cfs_wi_sched_cansleep(sched));
+							!cfs_wi_sched_cansleep(sched));
 		spin_lock(&sched->ws_lock);
 	}
 
@@ -289,7 +289,7 @@ static int cfs_wi_scheduler(void *arg)
 void
 cfs_wi_sched_destroy(struct cfs_wi_sched *sched)
 {
-	int	i;
+	int i;
 
 	LASSERT(cfs_wi_data.wi_init);
 	LASSERT(!cfs_wi_data.wi_stopping);
@@ -325,7 +325,7 @@ cfs_wi_sched_destroy(struct cfs_wi_sched *sched)
 	list_del(&sched->ws_list);
 
 	spin_unlock(&cfs_wi_data.wi_glock);
-	LASSERT(sched->ws_nscheduled == 0);
+	LASSERT(!sched->ws_nscheduled);
 
 	LIBCFS_FREE(sched, sizeof(*sched));
 }
@@ -335,8 +335,8 @@ int
 cfs_wi_sched_create(char *name, struct cfs_cpt_table *cptab,
 		    int cpt, int nthrs, struct cfs_wi_sched **sched_pp)
 {
-	struct cfs_wi_sched	*sched;
-	int			rc;
+	struct cfs_wi_sched *sched;
+	int rc;
 
 	LASSERT(cfs_wi_data.wi_init);
 	LASSERT(!cfs_wi_data.wi_stopping);
@@ -364,7 +364,7 @@ cfs_wi_sched_create(char *name, struct cfs_cpt_table *cptab,
 
 	rc = 0;
 	while (nthrs > 0)  {
-		char	name[16];
+		char name[16];
 		struct task_struct *task;
 
 		spin_lock(&cfs_wi_data.wi_glock);
@@ -431,7 +431,7 @@ cfs_wi_startup(void)
 void
 cfs_wi_shutdown(void)
 {
-	struct cfs_wi_sched	*sched;
+	struct cfs_wi_sched *sched;
 	struct cfs_wi_sched *temp;
 
 	spin_lock(&cfs_wi_data.wi_glock);
@@ -447,7 +447,7 @@ cfs_wi_shutdown(void)
 	list_for_each_entry(sched, &cfs_wi_data.wi_scheds, ws_list) {
 		spin_lock(&cfs_wi_data.wi_glock);
 
-		while (sched->ws_nthreads != 0) {
+		while (sched->ws_nthreads) {
 			spin_unlock(&cfs_wi_data.wi_glock);
 			set_current_state(TASK_UNINTERRUPTIBLE);
 			schedule_timeout(cfs_time_seconds(1) / 20);
diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
index 4daf828198c3..b2ba10d59f84 100644
--- a/drivers/staging/lustre/lnet/lnet/api-ni.c
+++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
@@ -1551,16 +1551,16 @@ LNetNIInit(lnet_pid_t requested_pid)
 
 		rc = lnet_check_routes();
 		if (rc)
-			goto err_destory_routes;
+			goto err_destroy_routes;
 
 		rc = lnet_rtrpools_alloc(im_a_router);
 		if (rc)
-			goto err_destory_routes;
+			goto err_destroy_routes;
 	}
 
 	rc = lnet_acceptor_start();
 	if (rc)
-		goto err_destory_routes;
+		goto err_destroy_routes;
 
 	the_lnet.ln_refcount = 1;
 	/* Now I may use my own API functions... */
@@ -1587,7 +1587,7 @@ err_stop_ping:
 err_acceptor_stop:
 	the_lnet.ln_refcount = 0;
 	lnet_acceptor_stop();
-err_destory_routes:
+err_destroy_routes:
 	if (!the_lnet.ln_nis_from_mod_params)
 		lnet_destroy_routes();
 err_shutdown_lndnis:
diff --git a/drivers/staging/lustre/lnet/lnet/lib-me.c b/drivers/staging/lustre/lnet/lnet/lib-me.c
index b430046dc294..eb796a86e6ab 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-me.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-me.c
@@ -271,21 +271,3 @@ lnet_me_unlink(lnet_me_t *me)
 	lnet_res_lh_invalidate(&me->me_lh);
 	lnet_me_free(me);
 }
-
-#if 0
-static void
-lib_me_dump(lnet_me_t *me)
-{
-	CWARN("Match Entry %p (%#llx)\n", me,
-	      me->me_lh.lh_cookie);
-
-	CWARN("\tMatch/Ignore\t= %016lx / %016lx\n",
-	      me->me_match_bits, me->me_ignore_bits);
-
-	CWARN("\tMD\t= %p\n", me->md);
-	CWARN("\tprev\t= %p\n",
-	      list_entry(me->me_list.prev, lnet_me_t, me_list));
-	CWARN("\tnext\t= %p\n",
-	      list_entry(me->me_list.next, lnet_me_t, me_list));
-}
-#endif
diff --git a/drivers/staging/lustre/lnet/lnet/lib-move.c b/drivers/staging/lustre/lnet/lnet/lib-move.c
index 48e6f8f2392f..f3dd6e42f4d4 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-move.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-move.c
@@ -192,6 +192,7 @@ lnet_copy_iov2iter(struct iov_iter *to,
 	left = siov->iov_len - soffset;
 	do {
 		size_t n, copy = left;
+
 		LASSERT(nsiov > 0);
 
 		if (copy > nob)
diff --git a/drivers/staging/lustre/lnet/lnet/nidstrings.c b/drivers/staging/lustre/lnet/lnet/nidstrings.c
index a6d7a6159b8f..a9fe3e69daae 100644
--- a/drivers/staging/lustre/lnet/lnet/nidstrings.c
+++ b/drivers/staging/lustre/lnet/lnet/nidstrings.c
@@ -193,7 +193,7 @@ add_nidrange(const struct cfs_lstr *src,
 	struct netstrfns *nf;
 	struct nidrange *nr;
 	int endlen;
-	unsigned netnum;
+	unsigned int netnum;
 
 	if (src->ls_len >= LNET_NIDSTR_SIZE)
 		return NULL;
@@ -247,10 +247,8 @@ parse_nidrange(struct cfs_lstr *src, struct list_head *nidlist)
 {
 	struct cfs_lstr addrrange;
 	struct cfs_lstr net;
-	struct cfs_lstr tmp;
 	struct nidrange *nr;
 
-	tmp = *src;
 	if (!cfs_gettok(src, '@', &addrrange))
 		goto failed;
 
@@ -1156,7 +1154,7 @@ EXPORT_SYMBOL(libcfs_nid2str_r);
 static struct netstrfns *
 libcfs_str2net_internal(const char *str, __u32 *net)
 {
-	struct netstrfns *uninitialized_var(nf);
+	struct netstrfns *nf = NULL;
 	int nob;
 	unsigned int netnum;
 	int i;
diff --git a/drivers/staging/lustre/lnet/lnet/router.c b/drivers/staging/lustre/lnet/lnet/router.c
index 063ad55ec950..8afa0abf15cd 100644
--- a/drivers/staging/lustre/lnet/lnet/router.c
+++ b/drivers/staging/lustre/lnet/lnet/router.c
@@ -903,6 +903,7 @@ lnet_create_rc_data_locked(lnet_peer_t *gateway)
 {
 	lnet_rc_data_t *rcd = NULL;
 	lnet_ping_info_t *pi;
+	lnet_md_t md;
 	int rc;
 	int i;
 
@@ -925,15 +926,15 @@ lnet_create_rc_data_locked(lnet_peer_t *gateway)
 	}
 	rcd->rcd_pinginfo = pi;
 
+	md.start = pi;
+	md.user_ptr = rcd;
+	md.length = LNET_PINGINFO_SIZE;
+	md.threshold = LNET_MD_THRESH_INF;
+	md.options = LNET_MD_TRUNCATE;
+	md.eq_handle = the_lnet.ln_rc_eqh;
+
 	LASSERT(!LNetHandleIsInvalid(the_lnet.ln_rc_eqh));
-	rc = LNetMDBind((lnet_md_t){.start     = pi,
-				    .user_ptr  = rcd,
-				    .length    = LNET_PINGINFO_SIZE,
-				    .threshold = LNET_MD_THRESH_INF,
-				    .options   = LNET_MD_TRUNCATE,
-				    .eq_handle = the_lnet.ln_rc_eqh},
-			LNET_UNLINK,
-			&rcd->rcd_mdh);
+	rc = LNetMDBind(md, LNET_UNLINK, &rcd->rcd_mdh);
 	if (rc < 0) {
 		CERROR("Can't bind MD: %d\n", rc);
 		goto out;
diff --git a/drivers/staging/lustre/lnet/selftest/brw_test.c b/drivers/staging/lustre/lnet/selftest/brw_test.c
index b20c5d394e3b..67b460f41d6e 100644
--- a/drivers/staging/lustre/lnet/selftest/brw_test.c
+++ b/drivers/staging/lustre/lnet/selftest/brw_test.c
@@ -44,6 +44,10 @@ static int brw_inject_errors;
 module_param(brw_inject_errors, int, 0644);
 MODULE_PARM_DESC(brw_inject_errors, "# data errors to inject randomly, zero by default");
 
+#define BRW_POISON	0xbeefbeefbeefbeefULL
+#define BRW_MAGIC	0xeeb0eeb1eeb2eeb3ULL
+#define BRW_MSIZE	sizeof(u64)
+
 static void
 brw_client_fini(struct sfw_test_instance *tsi)
 {
@@ -67,6 +71,7 @@ brw_client_init(struct sfw_test_instance *tsi)
 {
 	struct sfw_session *sn = tsi->tsi_batch->bat_session;
 	int flags;
+	int off;
 	int npg;
 	int len;
 	int opc;
@@ -87,6 +92,7 @@ brw_client_init(struct sfw_test_instance *tsi)
 		 * but we have to keep it for compatibility
 		 */
 		len = npg * PAGE_SIZE;
+		off = 0;
 	} else {
 		struct test_bulk_req_v1 *breq = &tsi->tsi_u.bulk_v1;
 
@@ -99,9 +105,13 @@ brw_client_init(struct sfw_test_instance *tsi)
 		opc = breq->blk_opc;
 		flags = breq->blk_flags;
 		len = breq->blk_len;
-		npg = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+		off = breq->blk_offset & ~PAGE_MASK;
+		npg = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
 	}
 
+	if (off % BRW_MSIZE)
+		return -EINVAL;
+
 	if (npg > LNET_MAX_IOV || npg <= 0)
 		return -EINVAL;
 
@@ -114,7 +124,7 @@ brw_client_init(struct sfw_test_instance *tsi)
 
 	list_for_each_entry(tsu, &tsi->tsi_units, tsu_list) {
 		bulk = srpc_alloc_bulk(lnet_cpt_of_nid(tsu->tsu_dest.nid),
-				       npg, len, opc == LST_BRW_READ);
+				       off, npg, len, opc == LST_BRW_READ);
 		if (!bulk) {
 			brw_client_fini(tsi);
 			return -ENOMEM;
@@ -126,12 +136,7 @@ brw_client_init(struct sfw_test_instance *tsi)
 	return 0;
 }
 
-#define BRW_POISON	0xbeefbeefbeefbeefULL
-#define BRW_MAGIC	0xeeb0eeb1eeb2eeb3ULL
-#define BRW_MSIZE	sizeof(__u64)
-
-static int
-brw_inject_one_error(void)
+int brw_inject_one_error(void)
 {
 	struct timespec64 ts;
 
@@ -147,12 +152,13 @@ brw_inject_one_error(void)
 }
 
 static void
-brw_fill_page(struct page *pg, int pattern, __u64 magic)
+brw_fill_page(struct page *pg, int off, int len, int pattern, __u64 magic)
 {
-	char *addr = page_address(pg);
+	char *addr = page_address(pg) + off;
 	int i;
 
 	LASSERT(addr);
+	LASSERT(!(off % BRW_MSIZE) && !(len % BRW_MSIZE));
 
 	if (pattern == LST_BRW_CHECK_NONE)
 		return;
@@ -162,14 +168,16 @@ brw_fill_page(struct page *pg, int pattern, __u64 magic)
 
 	if (pattern == LST_BRW_CHECK_SIMPLE) {
 		memcpy(addr, &magic, BRW_MSIZE);
-		addr += PAGE_SIZE - BRW_MSIZE;
-		memcpy(addr, &magic, BRW_MSIZE);
+		if (len > BRW_MSIZE) {
+			addr += PAGE_SIZE - BRW_MSIZE;
+			memcpy(addr, &magic, BRW_MSIZE);
+		}
 		return;
 	}
 
 	if (pattern == LST_BRW_CHECK_FULL) {
-		for (i = 0; i < PAGE_SIZE / BRW_MSIZE; i++)
-			memcpy(addr + i * BRW_MSIZE, &magic, BRW_MSIZE);
+		for (i = 0; i < len; i += BRW_MSIZE)
+			memcpy(addr + i, &magic, BRW_MSIZE);
 		return;
 	}
 
@@ -177,13 +185,14 @@ brw_fill_page(struct page *pg, int pattern, __u64 magic)
 }
 
 static int
-brw_check_page(struct page *pg, int pattern, __u64 magic)
+brw_check_page(struct page *pg, int off, int len, int pattern, __u64 magic)
 {
-	char *addr = page_address(pg);
+	char *addr = page_address(pg) + off;
 	__u64 data = 0; /* make compiler happy */
 	int i;
 
 	LASSERT(addr);
+	LASSERT(!(off % BRW_MSIZE) && !(len % BRW_MSIZE));
 
 	if (pattern == LST_BRW_CHECK_NONE)
 		return 0;
@@ -193,21 +202,21 @@ brw_check_page(struct page *pg, int pattern, __u64 magic)
 		if (data != magic)
 			goto bad_data;
 
-		addr += PAGE_SIZE - BRW_MSIZE;
-		data = *((__u64 *)addr);
-		if (data != magic)
-			goto bad_data;
-
+		if (len > BRW_MSIZE) {
+			addr += PAGE_SIZE - BRW_MSIZE;
+			data = *((__u64 *)addr);
+			if (data != magic)
+				goto bad_data;
+		}
 		return 0;
 	}
 
 	if (pattern == LST_BRW_CHECK_FULL) {
-		for (i = 0; i < PAGE_SIZE / BRW_MSIZE; i++) {
-			data = *(((__u64 *)addr) + i);
+		for (i = 0; i < len; i += BRW_MSIZE) {
+			data = *(u64 *)(addr + i);
 			if (data != magic)
 				goto bad_data;
 		}
-
 		return 0;
 	}
 
@@ -226,8 +235,12 @@ brw_fill_bulk(struct srpc_bulk *bk, int pattern, __u64 magic)
 	struct page *pg;
 
 	for (i = 0; i < bk->bk_niov; i++) {
+		int off, len;
+
 		pg = bk->bk_iovs[i].bv_page;
-		brw_fill_page(pg, pattern, magic);
+		off = bk->bk_iovs[i].bv_offset;
+		len = bk->bk_iovs[i].bv_len;
+		brw_fill_page(pg, off, len, pattern, magic);
 	}
 }
 
@@ -238,8 +251,12 @@ brw_check_bulk(struct srpc_bulk *bk, int pattern, __u64 magic)
 	struct page *pg;
 
 	for (i = 0; i < bk->bk_niov; i++) {
+		int off, len;
+
 		pg = bk->bk_iovs[i].bv_page;
-		if (brw_check_page(pg, pattern, magic)) {
+		off = bk->bk_iovs[i].bv_offset;
+		len = bk->bk_iovs[i].bv_len;
+		if (brw_check_page(pg, off, len, pattern, magic)) {
 			CERROR("Bulk page %p (%d/%d) is corrupted!\n",
 			       pg, i, bk->bk_niov);
 			return 1;
@@ -276,6 +293,7 @@ brw_client_prep_rpc(struct sfw_test_unit *tsu,
 		len = npg * PAGE_SIZE;
 	} else {
 		struct test_bulk_req_v1 *breq = &tsi->tsi_u.bulk_v1;
+		int off;
 
 		/*
 		 * I should never get this step if it's unknown feature
@@ -286,7 +304,8 @@ brw_client_prep_rpc(struct sfw_test_unit *tsu,
 		opc = breq->blk_opc;
 		flags = breq->blk_flags;
 		len = breq->blk_len;
-		npg = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+		off = breq->blk_offset;
+		npg = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
 	}
 
 	rc = sfw_create_test_rpc(tsu, dest, sn->sn_features, npg, len, &rpc);
diff --git a/drivers/staging/lustre/lnet/selftest/conctl.c b/drivers/staging/lustre/lnet/selftest/conctl.c
index b786f8b4a73d..94383023c1be 100644
--- a/drivers/staging/lustre/lnet/selftest/conctl.c
+++ b/drivers/staging/lustre/lnet/selftest/conctl.c
@@ -315,7 +315,7 @@ lst_group_update_ioctl(lstio_group_update_args_t *args)
 static int
 lst_nodes_add_ioctl(lstio_group_nodes_args_t *args)
 {
-	unsigned feats;
+	unsigned int feats;
 	int rc;
 	char *name;
 
@@ -742,6 +742,10 @@ static int lst_test_add_ioctl(lstio_test_args_t *args)
 	     PAGE_SIZE - sizeof(struct lstcon_test)))
 		return -EINVAL;
 
+	/* Enforce zero parameter length if there's no parameter */
+	if (!args->lstio_tes_param && args->lstio_tes_param_len)
+		return -EINVAL;
+
 	LIBCFS_ALLOC(batch_name, args->lstio_tes_bat_nmlen + 1);
 	if (!batch_name)
 		return rc;
diff --git a/drivers/staging/lustre/lnet/selftest/conrpc.c b/drivers/staging/lustre/lnet/selftest/conrpc.c
index 55afb53b0743..994422c62487 100644
--- a/drivers/staging/lustre/lnet/selftest/conrpc.c
+++ b/drivers/staging/lustre/lnet/selftest/conrpc.c
@@ -86,8 +86,9 @@ lstcon_rpc_done(struct srpc_client_rpc *rpc)
 }
 
 static int
-lstcon_rpc_init(struct lstcon_node *nd, int service, unsigned feats,
-		int bulk_npg, int bulk_len, int embedded, struct lstcon_rpc *crpc)
+lstcon_rpc_init(struct lstcon_node *nd, int service, unsigned int feats,
+		int bulk_npg, int bulk_len, int embedded,
+		struct lstcon_rpc *crpc)
 {
 	crpc->crp_rpc = sfw_create_rpc(nd->nd_id, service,
 				       feats, bulk_npg, bulk_len,
@@ -111,7 +112,7 @@ lstcon_rpc_init(struct lstcon_node *nd, int service, unsigned feats,
 }
 
 static int
-lstcon_rpc_prep(struct lstcon_node *nd, int service, unsigned feats,
+lstcon_rpc_prep(struct lstcon_node *nd, int service, unsigned int feats,
 		int bulk_npg, int bulk_len, struct lstcon_rpc **crpcpp)
 {
 	struct lstcon_rpc *crpc = NULL;
@@ -292,8 +293,8 @@ lstcon_rpc_trans_abort(struct lstcon_rpc_trans *trans, int error)
 
 		spin_lock(&rpc->crpc_lock);
 
-		if (!crpc->crp_posted ||	/* not posted */
-		    crpc->crp_stamp) {		/* rpc done or aborted already */
+		if (!crpc->crp_posted || /* not posted */
+		    crpc->crp_stamp) {	 /* rpc done or aborted already */
 			if (!crpc->crp_stamp) {
 				crpc->crp_stamp = cfs_time_current();
 				crpc->crp_status = -EINTR;
@@ -589,7 +590,7 @@ lstcon_rpc_trans_destroy(struct lstcon_rpc_trans *trans)
 
 int
 lstcon_sesrpc_prep(struct lstcon_node *nd, int transop,
-		   unsigned feats, struct lstcon_rpc **crpc)
+		   unsigned int feats, struct lstcon_rpc **crpc)
 {
 	struct srpc_mksn_reqst *msrq;
 	struct srpc_rmsn_reqst *rsrq;
@@ -627,7 +628,8 @@ lstcon_sesrpc_prep(struct lstcon_node *nd, int transop,
 }
 
 int
-lstcon_dbgrpc_prep(struct lstcon_node *nd, unsigned feats, struct lstcon_rpc **crpc)
+lstcon_dbgrpc_prep(struct lstcon_node *nd, unsigned int feats,
+		   struct lstcon_rpc **crpc)
 {
 	struct srpc_debug_reqst *drq;
 	int rc;
@@ -645,7 +647,7 @@ lstcon_dbgrpc_prep(struct lstcon_node *nd, unsigned feats, struct lstcon_rpc **c
 }
 
 int
-lstcon_batrpc_prep(struct lstcon_node *nd, int transop, unsigned feats,
+lstcon_batrpc_prep(struct lstcon_node *nd, int transop, unsigned int feats,
 		   struct lstcon_tsb_hdr *tsb, struct lstcon_rpc **crpc)
 {
 	struct lstcon_batch *batch;
@@ -678,7 +680,8 @@ lstcon_batrpc_prep(struct lstcon_node *nd, int transop, unsigned feats,
 }
 
 int
-lstcon_statrpc_prep(struct lstcon_node *nd, unsigned feats, struct lstcon_rpc **crpc)
+lstcon_statrpc_prep(struct lstcon_node *nd, unsigned int feats,
+		    struct lstcon_rpc **crpc)
 {
 	struct srpc_stat_reqst *srq;
 	int rc;
@@ -776,7 +779,8 @@ lstcon_pingrpc_prep(lst_test_ping_param_t *param, struct srpc_test_reqst *req)
 }
 
 static int
-lstcon_bulkrpc_v0_prep(lst_test_bulk_param_t *param, struct srpc_test_reqst *req)
+lstcon_bulkrpc_v0_prep(lst_test_bulk_param_t *param,
+		       struct srpc_test_reqst *req)
 {
 	struct test_bulk_req *brq = &req->tsr_u.bulk_v0;
 
@@ -789,20 +793,21 @@ lstcon_bulkrpc_v0_prep(lst_test_bulk_param_t *param, struct srpc_test_reqst *req
 }
 
 static int
-lstcon_bulkrpc_v1_prep(lst_test_bulk_param_t *param, struct srpc_test_reqst *req)
+lstcon_bulkrpc_v1_prep(lst_test_bulk_param_t *param, bool is_client,
+		       struct srpc_test_reqst *req)
 {
 	struct test_bulk_req_v1 *brq = &req->tsr_u.bulk_v1;
 
 	brq->blk_opc = param->blk_opc;
 	brq->blk_flags = param->blk_flags;
 	brq->blk_len = param->blk_size;
-	brq->blk_offset	= 0; /* reserved */
+	brq->blk_offset	= is_client ? param->blk_cli_off : param->blk_srv_off;
 
 	return 0;
 }
 
 int
-lstcon_testrpc_prep(struct lstcon_node *nd, int transop, unsigned feats,
+lstcon_testrpc_prep(struct lstcon_node *nd, int transop, unsigned int feats,
 		    struct lstcon_test *test, struct lstcon_rpc **crpc)
 {
 	struct lstcon_group *sgrp = test->tes_src_grp;
@@ -897,7 +902,8 @@ lstcon_testrpc_prep(struct lstcon_node *nd, int transop, unsigned feats,
 						    &test->tes_param[0], trq);
 		} else {
 			rc = lstcon_bulkrpc_v1_prep((lst_test_bulk_param_t *)
-						    &test->tes_param[0], trq);
+						    &test->tes_param[0],
+						    trq->tsr_is_client, trq);
 		}
 
 		break;
@@ -1084,7 +1090,7 @@ lstcon_rpc_trans_ndlist(struct list_head *ndlist,
 	struct lstcon_ndlink *ndl;
 	struct lstcon_node *nd;
 	struct lstcon_rpc *rpc;
-	unsigned feats;
+	unsigned int feats;
 	int rc;
 
 	/* Creating session RPG for list of nodes */
diff --git a/drivers/staging/lustre/lnet/selftest/conrpc.h b/drivers/staging/lustre/lnet/selftest/conrpc.h
index 7ec6fc96959e..e629e87c461c 100644
--- a/drivers/staging/lustre/lnet/selftest/conrpc.h
+++ b/drivers/staging/lustre/lnet/selftest/conrpc.h
@@ -78,8 +78,8 @@ struct lstcon_rpc_trans {
 	struct list_head  tas_olink;	     /* link chain on owner list */
 	struct list_head  tas_link;	     /* link chain on global list */
 	int		  tas_opc;	     /* operation code of transaction */
-	unsigned	  tas_feats_updated; /* features mask is uptodate */
-	unsigned	  tas_features;      /* test features mask */
+	unsigned int	  tas_feats_updated; /* features mask is uptodate */
+	unsigned int	  tas_features;      /* test features mask */
 	wait_queue_head_t tas_waitq;	     /* wait queue head */
 	atomic_t	  tas_remaining;     /* # of un-scheduled rpcs */
 	struct list_head  tas_rpcs_list;     /* queued requests */
@@ -106,14 +106,16 @@ typedef int (*lstcon_rpc_readent_func_t)(int, struct srpc_msg *,
 					 lstcon_rpc_ent_t __user *);
 
 int  lstcon_sesrpc_prep(struct lstcon_node *nd, int transop,
-			unsigned version, struct lstcon_rpc **crpc);
+			unsigned int version, struct lstcon_rpc **crpc);
 int  lstcon_dbgrpc_prep(struct lstcon_node *nd,
-			unsigned version, struct lstcon_rpc **crpc);
-int  lstcon_batrpc_prep(struct lstcon_node *nd, int transop, unsigned version,
-			struct lstcon_tsb_hdr *tsb, struct lstcon_rpc **crpc);
-int  lstcon_testrpc_prep(struct lstcon_node *nd, int transop, unsigned version,
-			 struct lstcon_test *test, struct lstcon_rpc **crpc);
-int  lstcon_statrpc_prep(struct lstcon_node *nd, unsigned version,
+			unsigned int version, struct lstcon_rpc **crpc);
+int  lstcon_batrpc_prep(struct lstcon_node *nd, int transop,
+			unsigned int version, struct lstcon_tsb_hdr *tsb,
+			struct lstcon_rpc **crpc);
+int  lstcon_testrpc_prep(struct lstcon_node *nd, int transop,
+			 unsigned int version, struct lstcon_test *test,
+			 struct lstcon_rpc **crpc);
+int  lstcon_statrpc_prep(struct lstcon_node *nd, unsigned int version,
 			 struct lstcon_rpc **crpc);
 void lstcon_rpc_put(struct lstcon_rpc *crpc);
 int  lstcon_rpc_trans_prep(struct list_head *translist,
@@ -129,7 +131,8 @@ int  lstcon_rpc_trans_interpreter(struct lstcon_rpc_trans *trans,
 				  lstcon_rpc_readent_func_t readent);
 void lstcon_rpc_trans_abort(struct lstcon_rpc_trans *trans, int error);
 void lstcon_rpc_trans_destroy(struct lstcon_rpc_trans *trans);
-void lstcon_rpc_trans_addreq(struct lstcon_rpc_trans *trans, struct lstcon_rpc *req);
+void lstcon_rpc_trans_addreq(struct lstcon_rpc_trans *trans,
+			     struct lstcon_rpc *req);
 int  lstcon_rpc_trans_postwait(struct lstcon_rpc_trans *trans, int timeout);
 int  lstcon_rpc_pinger_start(void);
 void lstcon_rpc_pinger_stop(void);
diff --git a/drivers/staging/lustre/lnet/selftest/console.c b/drivers/staging/lustre/lnet/selftest/console.c
index a0fcbf3bcc95..1456d2395cc9 100644
--- a/drivers/staging/lustre/lnet/selftest/console.c
+++ b/drivers/staging/lustre/lnet/selftest/console.c
@@ -86,7 +86,7 @@ lstcon_node_find(lnet_process_id_t id, struct lstcon_node **ndpp, int create)
 	if (!create)
 		return -ENOENT;
 
-	LIBCFS_ALLOC(*ndpp, sizeof(struct lstcon_node) + sizeof(struct lstcon_ndlink));
+	LIBCFS_ALLOC(*ndpp, sizeof(**ndpp) + sizeof(*ndl));
 	if (!*ndpp)
 		return -ENOMEM;
 
@@ -131,12 +131,12 @@ lstcon_node_put(struct lstcon_node *nd)
 	list_del(&ndl->ndl_link);
 	list_del(&ndl->ndl_hlink);
 
-	LIBCFS_FREE(nd, sizeof(struct lstcon_node) + sizeof(struct lstcon_ndlink));
+	LIBCFS_FREE(nd, sizeof(*nd) + sizeof(*ndl));
 }
 
 static int
-lstcon_ndlink_find(struct list_head *hash,
-		   lnet_process_id_t id, struct lstcon_ndlink **ndlpp, int create)
+lstcon_ndlink_find(struct list_head *hash, lnet_process_id_t id,
+		   struct lstcon_ndlink **ndlpp, int create)
 {
 	unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 	struct lstcon_ndlink *ndl;
@@ -230,7 +230,8 @@ lstcon_group_addref(struct lstcon_group *grp)
 	grp->grp_ref++;
 }
 
-static void lstcon_group_ndlink_release(struct lstcon_group *, struct lstcon_ndlink *);
+static void lstcon_group_ndlink_release(struct lstcon_group *,
+					struct lstcon_ndlink *);
 
 static void
 lstcon_group_drain(struct lstcon_group *grp, int keep)
@@ -397,7 +398,8 @@ lstcon_sesrpc_readent(int transop, struct srpc_msg *msg,
 static int
 lstcon_group_nodes_add(struct lstcon_group *grp,
 		       int count, lnet_process_id_t __user *ids_up,
-		       unsigned *featp, struct list_head __user *result_up)
+		       unsigned int *featp,
+		       struct list_head __user *result_up)
 {
 	struct lstcon_rpc_trans *trans;
 	struct lstcon_ndlink	*ndl;
@@ -542,7 +544,8 @@ lstcon_group_add(char *name)
 
 int
 lstcon_nodes_add(char *name, int count, lnet_process_id_t __user *ids_up,
-		 unsigned *featp, struct list_head __user *result_up)
+		 unsigned int *featp,
+		 struct list_head __user *result_up)
 {
 	struct lstcon_group *grp;
 	int rc;
@@ -820,7 +823,7 @@ lstcon_group_info(char *name, lstcon_ndlist_ent_t __user *gents_p,
 
 	lstcon_group_decref(grp);
 
-	return 0;
+	return rc;
 }
 
 static int
@@ -1181,7 +1184,8 @@ lstcon_testrpc_condition(int transop, struct lstcon_node *nd, void *arg)
 }
 
 static int
-lstcon_test_nodes_add(struct lstcon_test *test, struct list_head __user *result_up)
+lstcon_test_nodes_add(struct lstcon_test *test,
+		      struct list_head __user *result_up)
 {
 	struct lstcon_rpc_trans *trans;
 	struct lstcon_group *grp;
@@ -1364,7 +1368,8 @@ out:
 }
 
 static int
-lstcon_test_find(struct lstcon_batch *batch, int idx, struct lstcon_test **testpp)
+lstcon_test_find(struct lstcon_batch *batch, int idx,
+		 struct lstcon_test **testpp)
 {
 	struct lstcon_test *test;
 
@@ -1702,7 +1707,7 @@ lstcon_new_session_id(lst_sid_t *sid)
 }
 
 int
-lstcon_session_new(char *name, int key, unsigned feats,
+lstcon_session_new(char *name, int key, unsigned int feats,
 		   int timeout, int force, lst_sid_t __user *sid_up)
 {
 	int rc = 0;
@@ -1868,7 +1873,7 @@ lstcon_session_end(void)
 }
 
 int
-lstcon_session_feats_check(unsigned feats)
+lstcon_session_feats_check(unsigned int feats)
 {
 	int rc = 0;
 
diff --git a/drivers/staging/lustre/lnet/selftest/console.h b/drivers/staging/lustre/lnet/selftest/console.h
index 78388a611c22..5dc1de48a10e 100644
--- a/drivers/staging/lustre/lnet/selftest/console.h
+++ b/drivers/staging/lustre/lnet/selftest/console.h
@@ -92,14 +92,16 @@ struct lstcon_batch {
 	int		 bat_ntest;	  /* # of test */
 	int		 bat_state;	  /* state of the batch */
 	int		 bat_arg;	  /* parameter for run|stop, timeout
-					   * for run, force for stop */
+					   * for run, force for stop
+					   */
 	char		 bat_name[LST_NAME_SIZE];/* name of batch */
 
 	struct list_head bat_test_list;   /* list head of tests (struct lstcon_test)
 					   */
 	struct list_head bat_trans_list;  /* list head of transaction */
 	struct list_head bat_cli_list;	  /* list head of client nodes
-					   * (struct lstcon_node) */
+					   * (struct lstcon_node)
+					   */
 	struct list_head *bat_cli_hash;   /* hash table of client nodes */
 	struct list_head bat_srv_list;	  /* list head of server nodes */
 	struct list_head *bat_srv_hash;   /* hash table of server nodes */
@@ -144,13 +146,14 @@ struct lstcon_session {
 	int		    ses_timeout;      /* timeout in seconds */
 	time64_t	    ses_laststamp;    /* last operation stamp (seconds)
 					       */
-	unsigned	    ses_features;     /* tests features of the session
+	unsigned int	    ses_features;     /* tests features of the session
 					       */
-	unsigned	    ses_feats_updated:1; /* features are synced with
-						  * remote test nodes */
-	unsigned	    ses_force:1;      /* force creating */
-	unsigned	    ses_shutdown:1;   /* session is shutting down */
-	unsigned	    ses_expired:1;    /* console is timedout */
+	unsigned int	    ses_feats_updated:1; /* features are synced with
+						  * remote test nodes
+						  */
+	unsigned int	    ses_force:1;      /* force creating */
+	unsigned int	    ses_shutdown:1;   /* session is shutting down */
+	unsigned int	    ses_expired:1;    /* console is timedout */
 	__u64		    ses_id_cookie;    /* batch id cookie */
 	char		    ses_name[LST_NAME_SIZE];/* session name */
 	struct lstcon_rpc_trans	*ses_ping;		/* session pinger */
@@ -188,14 +191,14 @@ int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_hdr *hdr);
 int lstcon_console_init(void);
 int lstcon_console_fini(void);
 int lstcon_session_match(lst_sid_t sid);
-int lstcon_session_new(char *name, int key, unsigned version,
+int lstcon_session_new(char *name, int key, unsigned int version,
 		       int timeout, int flags, lst_sid_t __user *sid_up);
 int lstcon_session_info(lst_sid_t __user *sid_up, int __user *key,
 			unsigned __user *verp, lstcon_ndlist_ent_t __user *entp,
 			char __user *name_up, int len);
 int lstcon_session_end(void);
 int lstcon_session_debug(int timeout, struct list_head __user *result_up);
-int lstcon_session_feats_check(unsigned feats);
+int lstcon_session_feats_check(unsigned int feats);
 int lstcon_batch_debug(int timeout, char *name,
 		       int client, struct list_head __user *result_up);
 int lstcon_group_debug(int timeout, char *name,
@@ -207,7 +210,7 @@ int lstcon_group_del(char *name);
 int lstcon_group_clean(char *name, int args);
 int lstcon_group_refresh(char *name, struct list_head __user *result_up);
 int lstcon_nodes_add(char *name, int nnd, lnet_process_id_t __user *nds_up,
-		     unsigned *featp, struct list_head __user *result_up);
+		     unsigned int *featp, struct list_head __user *result_up);
 int lstcon_nodes_remove(char *name, int nnd, lnet_process_id_t __user *nds_up,
 			struct list_head __user *result_up);
 int lstcon_group_info(char *name, lstcon_ndlist_ent_t __user *gent_up,
diff --git a/drivers/staging/lustre/lnet/selftest/framework.c b/drivers/staging/lustre/lnet/selftest/framework.c
index abbd6287b4bd..48dcc330dc9b 100644
--- a/drivers/staging/lustre/lnet/selftest/framework.c
+++ b/drivers/staging/lustre/lnet/selftest/framework.c
@@ -131,7 +131,8 @@ sfw_find_test_case(int id)
 }
 
 static int
-sfw_register_test(struct srpc_service *service, struct sfw_test_client_ops *cliops)
+sfw_register_test(struct srpc_service *service,
+		  struct sfw_test_client_ops *cliops)
 {
 	struct sfw_test_case *tsc;
 
@@ -254,7 +255,7 @@ sfw_session_expired(void *data)
 
 static inline void
 sfw_init_session(struct sfw_session *sn, lst_sid_t sid,
-		 unsigned features, const char *name)
+		 unsigned int features, const char *name)
 {
 	struct stt_timer *timer = &sn->sn_timer;
 
@@ -469,7 +470,8 @@ sfw_make_session(struct srpc_mksn_reqst *request, struct srpc_mksn_reply *reply)
 }
 
 static int
-sfw_remove_session(struct srpc_rmsn_reqst *request, struct srpc_rmsn_reply *reply)
+sfw_remove_session(struct srpc_rmsn_reqst *request,
+		   struct srpc_rmsn_reply *reply)
 {
 	struct sfw_session *sn = sfw_data.fw_session;
 
@@ -501,7 +503,8 @@ sfw_remove_session(struct srpc_rmsn_reqst *request, struct srpc_rmsn_reply *repl
 }
 
 static int
-sfw_debug_session(struct srpc_debug_reqst *request, struct srpc_debug_reply *reply)
+sfw_debug_session(struct srpc_debug_reqst *request,
+		  struct srpc_debug_reply *reply)
 {
 	struct sfw_session *sn = sfw_data.fw_session;
 
@@ -897,7 +900,7 @@ sfw_test_rpc_done(struct srpc_client_rpc *rpc)
 
 int
 sfw_create_test_rpc(struct sfw_test_unit *tsu, lnet_process_id_t peer,
-		    unsigned features, int nblk, int blklen,
+		    unsigned int features, int nblk, int blklen,
 		    struct srpc_client_rpc **rpcpp)
 {
 	struct srpc_client_rpc *rpc = NULL;
@@ -1064,7 +1067,8 @@ sfw_stop_batch(struct sfw_batch *tsb, int force)
 }
 
 static int
-sfw_query_batch(struct sfw_batch *tsb, int testidx, struct srpc_batch_reply *reply)
+sfw_query_batch(struct sfw_batch *tsb, int testidx,
+		struct srpc_batch_reply *reply)
 {
 	struct sfw_test_instance *tsi;
 
@@ -1101,7 +1105,7 @@ sfw_alloc_pages(struct srpc_server_rpc *rpc, int cpt, int npages, int len,
 	LASSERT(!rpc->srpc_bulk);
 	LASSERT(npages > 0 && npages <= LNET_MAX_IOV);
 
-	rpc->srpc_bulk = srpc_alloc_bulk(cpt, npages, len, sink);
+	rpc->srpc_bulk = srpc_alloc_bulk(cpt, 0, npages, len, sink);
 	if (!rpc->srpc_bulk)
 		return -ENOMEM;
 
@@ -1179,7 +1183,8 @@ sfw_add_test(struct srpc_server_rpc *rpc)
 }
 
 static int
-sfw_control_batch(struct srpc_batch_reqst *request, struct srpc_batch_reply *reply)
+sfw_control_batch(struct srpc_batch_reqst *request,
+		  struct srpc_batch_reply *reply)
 {
 	struct sfw_session *sn = sfw_data.fw_session;
 	int rc = 0;
@@ -1225,7 +1230,7 @@ sfw_handle_server_rpc(struct srpc_server_rpc *rpc)
 	struct srpc_service *sv = rpc->srpc_scd->scd_svc;
 	struct srpc_msg *reply = &rpc->srpc_replymsg;
 	struct srpc_msg *request = &rpc->srpc_reqstbuf->buf_msg;
-	unsigned features = LST_FEATS_MASK;
+	unsigned int features = LST_FEATS_MASK;
 	int rc = 0;
 
 	LASSERT(!sfw_data.fw_active_srpc);
@@ -1375,7 +1380,7 @@ sfw_bulk_ready(struct srpc_server_rpc *rpc, int status)
 
 struct srpc_client_rpc *
 sfw_create_rpc(lnet_process_id_t peer, int service,
-	       unsigned features, int nbulkiov, int bulklen,
+	       unsigned int features, int nbulkiov, int bulklen,
 	       void (*done)(struct srpc_client_rpc *), void *priv)
 {
 	struct srpc_client_rpc *rpc = NULL;
diff --git a/drivers/staging/lustre/lnet/selftest/ping_test.c b/drivers/staging/lustre/lnet/selftest/ping_test.c
index 9331ca4e3606..b9601b00a273 100644
--- a/drivers/staging/lustre/lnet/selftest/ping_test.c
+++ b/drivers/staging/lustre/lnet/selftest/ping_test.c
@@ -159,8 +159,8 @@ ping_client_done_rpc(struct sfw_test_unit *tsu, struct srpc_client_rpc *rpc)
 
 	ktime_get_real_ts64(&ts);
 	CDEBUG(D_NET, "%d reply in %u usec\n", reply->pnr_seq,
-	       (unsigned)((ts.tv_sec - reqst->pnr_time_sec) * 1000000 +
-			  (ts.tv_nsec / NSEC_PER_USEC - reqst->pnr_time_usec)));
+	       (unsigned int)((ts.tv_sec - reqst->pnr_time_sec) * 1000000 +
+			      (ts.tv_nsec / NSEC_PER_USEC - reqst->pnr_time_usec)));
 }
 
 static int
diff --git a/drivers/staging/lustre/lnet/selftest/rpc.c b/drivers/staging/lustre/lnet/selftest/rpc.c
index f5619d8744ef..ce9de8c9be57 100644
--- a/drivers/staging/lustre/lnet/selftest/rpc.c
+++ b/drivers/staging/lustre/lnet/selftest/rpc.c
@@ -84,14 +84,13 @@ void srpc_set_counters(const srpc_counters_t *cnt)
 }
 
 static int
-srpc_add_bulk_page(struct srpc_bulk *bk, struct page *pg, int i, int nob)
+srpc_add_bulk_page(struct srpc_bulk *bk, struct page *pg, int i, int off,
+		   int nob)
 {
-	nob = min_t(int, nob, PAGE_SIZE);
+	LASSERT(off < PAGE_SIZE);
+	LASSERT(nob > 0 && nob <= PAGE_SIZE);
 
-	LASSERT(nob > 0);
-	LASSERT(i >= 0 && i < bk->bk_niov);
-
-	bk->bk_iovs[i].bv_offset = 0;
+	bk->bk_iovs[i].bv_offset = off;
 	bk->bk_iovs[i].bv_page = pg;
 	bk->bk_iovs[i].bv_len = nob;
 	return nob;
@@ -117,7 +116,8 @@ srpc_free_bulk(struct srpc_bulk *bk)
 }
 
 struct srpc_bulk *
-srpc_alloc_bulk(int cpt, unsigned bulk_npg, unsigned bulk_len, int sink)
+srpc_alloc_bulk(int cpt, unsigned int bulk_off, unsigned int bulk_npg,
+		unsigned int bulk_len, int sink)
 {
 	struct srpc_bulk *bk;
 	int i;
@@ -148,8 +148,11 @@ srpc_alloc_bulk(int cpt, unsigned bulk_npg, unsigned bulk_len, int sink)
 			return NULL;
 		}
 
-		nob = srpc_add_bulk_page(bk, pg, i, bulk_len);
+		nob = min_t(unsigned int, bulk_off + bulk_len, PAGE_SIZE) -
+		      bulk_off;
+		srpc_add_bulk_page(bk, pg, i, bulk_off, nob);
 		bulk_len -= nob;
+		bulk_off = 0;
 	}
 
 	return bk;
@@ -693,7 +696,8 @@ srpc_finish_service(struct srpc_service *sv)
 
 /* called with sv->sv_lock held */
 static void
-srpc_service_recycle_buffer(struct srpc_service_cd *scd, struct srpc_buffer *buf)
+srpc_service_recycle_buffer(struct srpc_service_cd *scd,
+			    struct srpc_buffer *buf)
 __must_hold(&scd->scd_lock)
 {
 	if (!scd->scd_svc->sv_shuttingdown && scd->scd_buf_adjust >= 0) {
diff --git a/drivers/staging/lustre/lnet/selftest/rpc.h b/drivers/staging/lustre/lnet/selftest/rpc.h
index 4ab2ee264004..f353a634cc8e 100644
--- a/drivers/staging/lustre/lnet/selftest/rpc.h
+++ b/drivers/staging/lustre/lnet/selftest/rpc.h
@@ -113,7 +113,8 @@ struct srpc_join_reply {
 	__u32			join_status;	/* returned status */
 	lst_sid_t		join_sid;	/* session id */
 	__u32			join_timeout;	/* # seconds' inactivity to
-						 * expire */
+						 * expire
+						 */
 	char			join_session[LST_NAME_SIZE]; /* session name */
 } WIRE_ATTR;
 
@@ -175,7 +176,7 @@ struct test_bulk_req_v1 {
 	__u16		   blk_opc;	   /* bulk operation code */
 	__u16		   blk_flags;	   /* data check flags */
 	__u32		   blk_len;	   /* data length */
-	__u32		   blk_offset;	   /* reserved: offset */
+	__u32		   blk_offset;	   /* offset */
 } WIRE_ATTR;
 
 struct test_ping_req {
@@ -190,7 +191,8 @@ struct srpc_test_reqst {
 	lst_bid_t		tsr_bid;	/* batch id */
 	__u32			tsr_service;	/* test type: bulk|ping|... */
 	__u32			tsr_loop;	/* test client loop count or
-						 * # server buffers needed */
+						 * # server buffers needed
+						 */
 	__u32			tsr_concur;	/* concurrency of test */
 	__u8			tsr_is_client;	/* is test client or not */
 	__u8			tsr_stop_onerr; /* stop on error */
diff --git a/drivers/staging/lustre/lnet/selftest/selftest.h b/drivers/staging/lustre/lnet/selftest/selftest.h
index d033ac03d953..c8833a016b6d 100644
--- a/drivers/staging/lustre/lnet/selftest/selftest.h
+++ b/drivers/staging/lustre/lnet/selftest/selftest.h
@@ -131,7 +131,8 @@ srpc_service2reply(int service)
 
 enum srpc_event_type {
 	SRPC_BULK_REQ_RCVD   = 1, /* passive bulk request(PUT sink/GET source)
-				   * received */
+				   * received
+				   */
 	SRPC_BULK_PUT_SENT   = 2, /* active bulk PUT sent (source) */
 	SRPC_BULK_GET_RPLD   = 3, /* active bulk GET replied (sink) */
 	SRPC_REPLY_RCVD      = 4, /* incoming reply received */
@@ -295,7 +296,8 @@ struct srpc_service_cd {
 #define SFW_TEST_WI_MIN		256
 #define SFW_TEST_WI_MAX		2048
 /* extra buffers for tolerating buggy peers, or unbalanced number
- * of peers between partitions	*/
+ * of peers between partitions
+ */
 #define SFW_TEST_WI_EXTRA	64
 
 /* number of server workitems (mini-thread) for framework service */
@@ -347,9 +349,11 @@ struct sfw_batch {
 
 struct sfw_test_client_ops {
 	int  (*tso_init)(struct sfw_test_instance *tsi); /* initialize test
-							  * client */
+							  * client
+							  */
 	void (*tso_fini)(struct sfw_test_instance *tsi); /* finalize test
-							  * client */
+							  * client
+							  */
 	int  (*tso_prep_rpc)(struct sfw_test_unit *tsu,
 			     lnet_process_id_t dest,
 			     struct srpc_client_rpc **rpc);	/* prep a tests rpc */
@@ -374,7 +378,8 @@ struct sfw_test_instance {
 	spinlock_t		   tsi_lock;		/* serialize */
 	unsigned int		   tsi_stopping:1;	/* test is stopping */
 	atomic_t		   tsi_nactive;		/* # of active test
-							 * unit */
+							 * unit
+							 */
 	struct list_head	   tsi_units;		/* test units */
 	struct list_head	   tsi_free_rpcs;	/* free rpcs */
 	struct list_head	   tsi_active_rpcs;	/* active rpcs */
@@ -386,8 +391,10 @@ struct sfw_test_instance {
 	} tsi_u;
 };
 
-/* XXX: trailing (PAGE_SIZE % sizeof(lnet_process_id_t)) bytes at the end of
- * pages are not used */
+/*
+ * XXX: trailing (PAGE_SIZE % sizeof(lnet_process_id_t)) bytes at the end of
+ * pages are not used
+ */
 #define SFW_MAX_CONCUR	   LST_MAX_CONCUR
 #define SFW_ID_PER_PAGE    (PAGE_SIZE / sizeof(lnet_process_id_packed_t))
 #define SFW_MAX_NDESTS	   (LNET_MAX_IOV * SFW_ID_PER_PAGE)
@@ -410,10 +417,10 @@ struct sfw_test_case {
 
 struct srpc_client_rpc *
 sfw_create_rpc(lnet_process_id_t peer, int service,
-	       unsigned features, int nbulkiov, int bulklen,
+	       unsigned int features, int nbulkiov, int bulklen,
 	       void (*done)(struct srpc_client_rpc *), void *priv);
 int sfw_create_test_rpc(struct sfw_test_unit *tsu,
-			lnet_process_id_t peer, unsigned features,
+			lnet_process_id_t peer, unsigned int features,
 			int nblk, int blklen, struct srpc_client_rpc **rpc);
 void sfw_abort_rpc(struct srpc_client_rpc *rpc);
 void sfw_post_rpc(struct srpc_client_rpc *rpc);
@@ -434,8 +441,9 @@ srpc_create_client_rpc(lnet_process_id_t peer, int service,
 void srpc_post_rpc(struct srpc_client_rpc *rpc);
 void srpc_abort_rpc(struct srpc_client_rpc *rpc, int why);
 void srpc_free_bulk(struct srpc_bulk *bk);
-struct srpc_bulk *srpc_alloc_bulk(int cpt, unsigned bulk_npg,
-				  unsigned bulk_len, int sink);
+struct srpc_bulk *srpc_alloc_bulk(int cpt, unsigned int off,
+				  unsigned int bulk_npg, unsigned int bulk_len,
+				  int sink);
 int srpc_send_rpc(struct swi_workitem *wi);
 int srpc_send_reply(struct srpc_server_rpc *rpc);
 int srpc_add_service(struct srpc_service *sv);
diff --git a/drivers/staging/lustre/lnet/selftest/timer.c b/drivers/staging/lustre/lnet/selftest/timer.c
index dcd22580b1f0..2fe692df19d0 100644
--- a/drivers/staging/lustre/lnet/selftest/timer.c
+++ b/drivers/staging/lustre/lnet/selftest/timer.c
@@ -46,16 +46,17 @@
  * to cover a time period of 1024 seconds into the future before wrapping.
  */
 #define STTIMER_MINPOLL        3	/* log2 min poll interval (8 s) */
-#define STTIMER_SLOTTIME       (1 << STTIMER_MINPOLL)
+#define STTIMER_SLOTTIME	BIT(STTIMER_MINPOLL)
 #define STTIMER_SLOTTIMEMASK   (~(STTIMER_SLOTTIME - 1))
-#define STTIMER_NSLOTS	       (1 << 7)
+#define STTIMER_NSLOTS		BIT(7)
 #define STTIMER_SLOT(t)	       (&stt_data.stt_hash[(((t) >> STTIMER_MINPOLL) & \
 						    (STTIMER_NSLOTS - 1))])
 
 static struct st_timer_data {
 	spinlock_t	  stt_lock;
 	unsigned long	  stt_prev_slot; /* start time of the slot processed
-					  * previously */
+					  * previously
+					  */
 	struct list_head  stt_hash[STTIMER_NSLOTS];
 	int		  stt_shuttingdown;
 	wait_queue_head_t stt_waitq;
diff --git a/drivers/staging/lustre/lustre/fid/fid_request.c b/drivers/staging/lustre/lustre/fid/fid_request.c
index edd72b926f81..999f250ceed0 100644
--- a/drivers/staging/lustre/lustre/fid/fid_request.c
+++ b/drivers/staging/lustre/lustre/fid/fid_request.c
@@ -74,7 +74,7 @@ static int seq_client_rpc(struct lu_client_seq *seq,
 
 	/* Zero out input range, this is not recovery yet. */
 	in = req_capsule_client_get(&req->rq_pill, &RMF_SEQ_RANGE);
-	range_init(in);
+	lu_seq_range_init(in);
 
 	ptlrpc_request_set_replen(req);
 
@@ -112,25 +112,21 @@ static int seq_client_rpc(struct lu_client_seq *seq,
 
 	ptlrpc_at_set_req_timeout(req);
 
-	if (opc != SEQ_ALLOC_SUPER && seq->lcs_type == LUSTRE_SEQ_METADATA)
-		mdc_get_rpc_lock(exp->exp_obd->u.cli.cl_rpc_lock, NULL);
 	rc = ptlrpc_queue_wait(req);
-	if (opc != SEQ_ALLOC_SUPER && seq->lcs_type == LUSTRE_SEQ_METADATA)
-		mdc_put_rpc_lock(exp->exp_obd->u.cli.cl_rpc_lock, NULL);
 	if (rc)
 		goto out_req;
 
 	out = req_capsule_server_get(&req->rq_pill, &RMF_SEQ_RANGE);
 	*output = *out;
 
-	if (!range_is_sane(output)) {
+	if (!lu_seq_range_is_sane(output)) {
 		CERROR("%s: Invalid range received from server: "
 		       DRANGE "\n", seq->lcs_name, PRANGE(output));
 		rc = -EINVAL;
 		goto out_req;
 	}
 
-	if (range_is_exhausted(output)) {
+	if (lu_seq_range_is_exhausted(output)) {
 		CERROR("%s: Range received from server is exhausted: "
 		       DRANGE "]\n", seq->lcs_name, PRANGE(output));
 		rc = -EINVAL;
@@ -170,9 +166,9 @@ static int seq_client_alloc_seq(const struct lu_env *env,
 {
 	int rc;
 
-	LASSERT(range_is_sane(&seq->lcs_space));
+	LASSERT(lu_seq_range_is_sane(&seq->lcs_space));
 
-	if (range_is_exhausted(&seq->lcs_space)) {
+	if (lu_seq_range_is_exhausted(&seq->lcs_space)) {
 		rc = seq_client_alloc_meta(env, seq);
 		if (rc) {
 			CERROR("%s: Can't allocate new meta-sequence, rc %d\n",
@@ -185,7 +181,7 @@ static int seq_client_alloc_seq(const struct lu_env *env,
 		rc = 0;
 	}
 
-	LASSERT(!range_is_exhausted(&seq->lcs_space));
+	LASSERT(!lu_seq_range_is_exhausted(&seq->lcs_space));
 	*seqnr = seq->lcs_space.lsr_start;
 	seq->lcs_space.lsr_start += 1;
 
@@ -320,7 +316,7 @@ void seq_client_flush(struct lu_client_seq *seq)
 
 	seq->lcs_space.lsr_index = -1;
 
-	range_init(&seq->lcs_space);
+	lu_seq_range_init(&seq->lcs_space);
 	mutex_unlock(&seq->lcs_mutex);
 }
 EXPORT_SYMBOL(seq_client_flush);
diff --git a/drivers/staging/lustre/lustre/fid/lproc_fid.c b/drivers/staging/lustre/lustre/fid/lproc_fid.c
index 3ed32d77f38b..97d4849c7199 100644
--- a/drivers/staging/lustre/lustre/fid/lproc_fid.c
+++ b/drivers/staging/lustre/lustre/fid/lproc_fid.c
@@ -83,7 +83,7 @@ ldebugfs_fid_write_common(const char __user *buffer, size_t count,
 		    (unsigned long long *)&tmp.lsr_end);
 	if (rc != 2)
 		return -EINVAL;
-	if (!range_is_sane(&tmp) || range_is_zero(&tmp) ||
+	if (!lu_seq_range_is_sane(&tmp) || lu_seq_range_is_zero(&tmp) ||
 	    tmp.lsr_start < range->lsr_start || tmp.lsr_end > range->lsr_end)
 		return -EINVAL;
 	*range = tmp;
diff --git a/drivers/staging/lustre/lustre/fld/fld_cache.c b/drivers/staging/lustre/lustre/fld/fld_cache.c
index 0100a935f4ff..11f697496180 100644
--- a/drivers/staging/lustre/lustre/fld/fld_cache.c
+++ b/drivers/staging/lustre/lustre/fld/fld_cache.c
@@ -143,7 +143,7 @@ restart_fixup:
 		c_range = &f_curr->fce_range;
 		n_range = &f_next->fce_range;
 
-		LASSERT(range_is_sane(c_range));
+		LASSERT(lu_seq_range_is_sane(c_range));
 		if (&f_next->fce_list == head)
 			break;
 
@@ -358,7 +358,7 @@ struct fld_cache_entry
 {
 	struct fld_cache_entry *f_new;
 
-	LASSERT(range_is_sane(range));
+	LASSERT(lu_seq_range_is_sane(range));
 
 	f_new = kzalloc(sizeof(*f_new), GFP_NOFS);
 	if (!f_new)
@@ -503,7 +503,7 @@ int fld_cache_lookup(struct fld_cache *cache,
 		}
 
 		prev = flde;
-		if (range_within(&flde->fce_range, seq)) {
+		if (lu_seq_range_within(&flde->fce_range, seq)) {
 			*range = flde->fce_range;
 
 			cache->fci_stat.fst_cache++;
diff --git a/drivers/staging/lustre/lustre/fld/fld_internal.h b/drivers/staging/lustre/lustre/fld/fld_internal.h
index 08eaec735d6f..4a7f0b71c48d 100644
--- a/drivers/staging/lustre/lustre/fld/fld_internal.h
+++ b/drivers/staging/lustre/lustre/fld/fld_internal.h
@@ -62,11 +62,6 @@
 #include "../include/lustre_req_layout.h"
 #include "../include/lustre_fld.h"
 
-enum {
-	LUSTRE_FLD_INIT = 1 << 0,
-	LUSTRE_FLD_RUN  = 1 << 1
-};
-
 struct fld_stats {
 	__u64   fst_count;
 	__u64   fst_cache;
diff --git a/drivers/staging/lustre/lustre/fld/fld_request.c b/drivers/staging/lustre/lustre/fld/fld_request.c
index 0de72b717ce5..4cade7a16800 100644
--- a/drivers/staging/lustre/lustre/fld/fld_request.c
+++ b/drivers/staging/lustre/lustre/fld/fld_request.c
@@ -159,11 +159,6 @@ int fld_client_add_target(struct lu_client_fld *fld,
 	LASSERT(name);
 	LASSERT(tar->ft_srv || tar->ft_exp);
 
-	if (fld->lcf_flags != LUSTRE_FLD_INIT) {
-		CERROR("%s: Attempt to add target %s (idx %llu) on fly - skip it\n",
-		       fld->lcf_name, name, tar->ft_idx);
-		return 0;
-	}
 	CDEBUG(D_INFO, "%s: Adding target %s (idx %llu)\n",
 	       fld->lcf_name, name, tar->ft_idx);
 
@@ -282,7 +277,6 @@ int fld_client_init(struct lu_client_fld *fld,
 	fld->lcf_count = 0;
 	spin_lock_init(&fld->lcf_lock);
 	fld->lcf_hash = &fld_hash[hash];
-	fld->lcf_flags = LUSTRE_FLD_INIT;
 	INIT_LIST_HEAD(&fld->lcf_targets);
 
 	cache_size = FLD_CLIENT_CACHE_SIZE /
@@ -421,8 +415,6 @@ int fld_client_lookup(struct lu_client_fld *fld, u64 seq, u32 *mds,
 	struct lu_fld_target *target;
 	int rc;
 
-	fld->lcf_flags |= LUSTRE_FLD_RUN;
-
 	rc = fld_cache_lookup(fld->lcf_cache, seq, &res);
 	if (rc == 0) {
 		*mds = res.lsr_index;
diff --git a/drivers/staging/lustre/lustre/include/cl_object.h b/drivers/staging/lustre/lustre/include/cl_object.h
index 89292c93dcd5..dc685610c4c4 100644
--- a/drivers/staging/lustre/lustre/include/cl_object.h
+++ b/drivers/staging/lustre/lustre/include/cl_object.h
@@ -59,10 +59,6 @@
  *		 read/write system call it is associated with the single user
  *		 thread, that issued the system call).
  *
- *   - cl_req      represents a collection of pages for a transfer. cl_req is
- *		 constructed by req-forming engine that tries to saturate
- *		 transport with large and continuous transfers.
- *
  * Terminology
  *
  *     - to avoid confusion high-level I/O operation like read or write system
@@ -103,11 +99,8 @@
 struct inode;
 
 struct cl_device;
-struct cl_device_operations;
 
 struct cl_object;
-struct cl_object_page_operations;
-struct cl_object_lock_operations;
 
 struct cl_page;
 struct cl_page_slice;
@@ -120,27 +113,7 @@ struct cl_page_operations;
 struct cl_io;
 struct cl_io_slice;
 
-struct cl_req;
-struct cl_req_slice;
-
-/**
- * Operations for each data device in the client stack.
- *
- * \see vvp_cl_ops, lov_cl_ops, lovsub_cl_ops, osc_cl_ops
- */
-struct cl_device_operations {
-	/**
-	 * Initialize cl_req. This method is called top-to-bottom on all
-	 * devices in the stack to get them a chance to allocate layer-private
-	 * data, and to attach them to the cl_req by calling
-	 * cl_req_slice_add().
-	 *
-	 * \see osc_req_init(), lov_req_init(), lovsub_req_init()
-	 * \see vvp_req_init()
-	 */
-	int (*cdo_req_init)(const struct lu_env *env, struct cl_device *dev,
-			    struct cl_req *req);
-};
+struct cl_req_attr;
 
 /**
  * Device in the client stack.
@@ -150,8 +123,6 @@ struct cl_device_operations {
 struct cl_device {
 	/** Super-class. */
 	struct lu_device		   cd_lu_dev;
-	/** Per-layer operation vector. */
-	const struct cl_device_operations *cd_ops;
 };
 
 /** \addtogroup cl_object cl_object
@@ -267,7 +238,7 @@ struct cl_object_conf {
 		/**
 		 * Object layout. This is consumed by lov.
 		 */
-		struct lustre_md *coc_md;
+		struct lu_buf	  coc_layout;
 		/**
 		 * Description of particular stripe location in the
 		 * cluster. This is consumed by osc.
@@ -301,6 +272,26 @@ enum {
 	OBJECT_CONF_WAIT = 2
 };
 
+enum {
+	CL_LAYOUT_GEN_NONE	= (u32)-2,	/* layout lock was cancelled */
+	CL_LAYOUT_GEN_EMPTY	= (u32)-1,	/* for empty layout */
+};
+
+struct cl_layout {
+	/** the buffer to return the layout in lov_mds_md format. */
+	struct lu_buf	cl_buf;
+	/** size of layout in lov_mds_md format. */
+	size_t		cl_size;
+	/** Layout generation. */
+	u32		cl_layout_gen;
+	/**
+	 * True if this is a released file.
+	 * Temporarily added for released file truncate in ll_setattr_raw().
+	 * It will be removed later. -Jinshan
+	 */
+	bool		cl_is_released;
+};
+
 /**
  * Operations implemented for each cl object layer.
  *
@@ -400,6 +391,27 @@ struct cl_object_operations {
 	 */
 	int (*coo_getstripe)(const struct lu_env *env, struct cl_object *obj,
 			     struct lov_user_md __user *lum);
+	/**
+	 * Get FIEMAP mapping from the object.
+	 */
+	int (*coo_fiemap)(const struct lu_env *env, struct cl_object *obj,
+			  struct ll_fiemap_info_key *fmkey,
+			  struct fiemap *fiemap, size_t *buflen);
+	/**
+	 * Get layout and generation of the object.
+	 */
+	int (*coo_layout_get)(const struct lu_env *env, struct cl_object *obj,
+			      struct cl_layout *layout);
+	/**
+	 * Get maximum size of the object.
+	 */
+	loff_t (*coo_maxbytes)(struct cl_object *obj);
+	/**
+	 * Set request attributes.
+	 */
+	void (*coo_req_attr_set)(const struct lu_env *env,
+				 struct cl_object *obj,
+				 struct cl_req_attr *attr);
 };
 
 /**
@@ -591,7 +603,7 @@ enum cl_page_state {
 	 *
 	 *     - [cl_page_state::CPS_PAGEOUT] page is dirty, the
 	 *     req-formation engine decides that it wants to include this page
-	 *     into an cl_req being constructed, and yanks it from the cache;
+	 *     into an RPC being constructed, and yanks it from the cache;
 	 *
 	 *     - [cl_page_state::CPS_FREEING] VM callback is executed to
 	 *     evict the page form the memory;
@@ -660,7 +672,7 @@ enum cl_page_state {
 	 * Page is being read in, as a part of a transfer. This is quite
 	 * similar to the cl_page_state::CPS_PAGEOUT state, except that
 	 * read-in is always "immediate"---there is no such thing a sudden
-	 * construction of read cl_req from cached, presumably not up to date,
+	 * construction of read request from cached, presumably not up to date,
 	 * pages.
 	 *
 	 * Underlying VM page is locked for the duration of transfer.
@@ -714,8 +726,6 @@ struct cl_page {
 	struct list_head	 cp_batch;
 	/** List of slices. Immutable after creation. */
 	struct list_head	 cp_layers;
-	/** Linkage of pages within cl_req. */
-	struct list_head         cp_flight;
 	/**
 	 * Page state. This field is const to avoid accidental update, it is
 	 * modified only internally within cl_page.c. Protected by a VM lock.
@@ -732,12 +742,6 @@ struct cl_page {
 	 * by sub-io. Protected by a VM lock.
 	 */
 	struct cl_io	    *cp_owner;
-	/**
-	 * Owning IO request in cl_page_state::CPS_PAGEOUT and
-	 * cl_page_state::CPS_PAGEIN states. This field is maintained only in
-	 * the top-level pages. Protected by a VM lock.
-	 */
-	struct cl_req	   *cp_req;
 	/** List of references to this page, for debugging. */
 	struct lu_ref	    cp_reference;
 	/** Link to an object, for debugging. */
@@ -779,7 +783,6 @@ enum cl_lock_mode {
 
 /**
  * Requested transfer type.
- * \ingroup cl_req
  */
 enum cl_req_type {
 	CRT_READ,
@@ -884,26 +887,6 @@ struct cl_page_operations {
 	/** Destructor. Frees resources and slice itself. */
 	void (*cpo_fini)(const struct lu_env *env,
 			 struct cl_page_slice *slice);
-
-	/**
-	 * Checks whether the page is protected by a cl_lock. This is a
-	 * per-layer method, because certain layers have ways to check for the
-	 * lock much more efficiently than through the generic locks scan, or
-	 * implement locking mechanisms separate from cl_lock, e.g.,
-	 * LL_FILE_GROUP_LOCKED in vvp. If \a pending is true, check for locks
-	 * being canceled, or scheduled for cancellation as soon as the last
-	 * user goes away, too.
-	 *
-	 * \retval    -EBUSY: page is protected by a lock of a given mode;
-	 * \retval  -ENODATA: page is not protected by a lock;
-	 * \retval	 0: this layer cannot decide.
-	 *
-	 * \see cl_page_is_under_lock()
-	 */
-	int (*cpo_is_under_lock)(const struct lu_env *env,
-				 const struct cl_page_slice *slice,
-				 struct cl_io *io, pgoff_t *max);
-
 	/**
 	 * Optional debugging helper. Prints given page slice.
 	 *
@@ -915,8 +898,7 @@ struct cl_page_operations {
 	/**
 	 * \name transfer
 	 *
-	 * Transfer methods. See comment on cl_req for a description of
-	 * transfer formation and life-cycle.
+	 * Transfer methods.
 	 *
 	 * @{
 	 */
@@ -962,7 +944,7 @@ struct cl_page_operations {
 				       int ioret);
 		/**
 		 * Called when cached page is about to be added to the
-		 * cl_req as a part of req formation.
+		 * ptlrpc request as a part of req formation.
 		 *
 		 * \return    0       : proceed with this page;
 		 * \return    -EAGAIN : skip this page;
@@ -1365,7 +1347,6 @@ struct cl_2queue {
  *     (3) sort all locks to avoid dead-locks, and acquire them
  *
  *     (4) process the chunk: call per-page methods
- *	 (cl_io_operations::cio_read_page() for read,
  *	 cl_io_operations::cio_prepare_write(),
  *	 cl_io_operations::cio_commit_write() for write)
  *
@@ -1388,6 +1369,8 @@ enum cl_io_type {
 	CIT_WRITE,
 	/** truncate, utime system calls */
 	CIT_SETATTR,
+	/** get data version */
+	CIT_DATA_VERSION,
 	/**
 	 * page fault handling
 	 */
@@ -1467,6 +1450,31 @@ struct cl_io_slice {
 
 typedef void (*cl_commit_cbt)(const struct lu_env *, struct cl_io *,
 			      struct cl_page *);
+
+struct cl_read_ahead {
+	/*
+	 * Maximum page index the readahead window will end.
+	 * This is determined DLM lock coverage, RPC and stripe boundary.
+	 * cra_end is included.
+	 */
+	pgoff_t cra_end;
+	/*
+	 * Release routine. If readahead holds resources underneath, this
+	 * function should be called to release it.
+	 */
+	void (*cra_release)(const struct lu_env *env, void *cbdata);
+	/* Callback data for cra_release routine */
+	void *cra_cbdata;
+};
+
+static inline void cl_read_ahead_release(const struct lu_env *env,
+					 struct cl_read_ahead *ra)
+{
+	if (ra->cra_release)
+		ra->cra_release(env, ra->cra_cbdata);
+	memset(ra, 0, sizeof(*ra));
+}
+
 /**
  * Per-layer io operations.
  * \see vvp_io_ops, lov_io_ops, lovsub_io_ops, osc_io_ops
@@ -1573,16 +1581,13 @@ struct cl_io_operations {
 				 struct cl_page_list *queue, int from, int to,
 				 cl_commit_cbt cb);
 	/**
-	 * Read missing page.
-	 *
-	 * Called by a top-level cl_io_operations::op[CIT_READ]::cio_start()
-	 * method, when it hits not-up-to-date page in the range. Optional.
+	 * Decide maximum read ahead extent
 	 *
 	 * \pre io->ci_type == CIT_READ
 	 */
-	int (*cio_read_page)(const struct lu_env *env,
-			     const struct cl_io_slice *slice,
-			     const struct cl_page_slice *page);
+	int (*cio_read_ahead)(const struct lu_env *env,
+			      const struct cl_io_slice *slice,
+			      pgoff_t start, struct cl_read_ahead *ra);
 	/**
 	 * Optional debugging helper. Print given io slice.
 	 */
@@ -1765,10 +1770,15 @@ struct cl_io {
 		struct cl_io_rw_common ci_rw;
 		struct cl_setattr_io {
 			struct ost_lvb   sa_attr;
+			unsigned int		 sa_attr_flags;
 			unsigned int     sa_valid;
 			int		sa_stripe_index;
-			struct lu_fid  *sa_parent_fid;
+			const struct lu_fid	*sa_parent_fid;
 		} ci_setattr;
+		struct cl_data_version_io {
+			u64 dv_data_version;
+			int dv_flags;
+		} ci_data_version;
 		struct cl_fault_io {
 			/** page index within file. */
 			pgoff_t	 ft_index;
@@ -1836,179 +1846,20 @@ struct cl_io {
 
 /** @} cl_io */
 
-/** \addtogroup cl_req cl_req
- * @{
- */
-/** \struct cl_req
- * Transfer.
- *
- * There are two possible modes of transfer initiation on the client:
- *
- *     - immediate transfer: this is started when a high level io wants a page
- *       or a collection of pages to be transferred right away. Examples:
- *       read-ahead, synchronous read in the case of non-page aligned write,
- *       page write-out as a part of extent lock cancellation, page write-out
- *       as a part of memory cleansing. Immediate transfer can be both
- *       cl_req_type::CRT_READ and cl_req_type::CRT_WRITE;
- *
- *     - opportunistic transfer (cl_req_type::CRT_WRITE only), that happens
- *       when io wants to transfer a page to the server some time later, when
- *       it can be done efficiently. Example: pages dirtied by the write(2)
- *       path.
- *
- * In any case, transfer takes place in the form of a cl_req, which is a
- * representation for a network RPC.
- *
- * Pages queued for an opportunistic transfer are cached until it is decided
- * that efficient RPC can be composed of them. This decision is made by "a
- * req-formation engine", currently implemented as a part of osc
- * layer. Req-formation depends on many factors: the size of the resulting
- * RPC, whether or not multi-object RPCs are supported by the server,
- * max-rpc-in-flight limitations, size of the dirty cache, etc.
- *
- * For the immediate transfer io submits a cl_page_list, that req-formation
- * engine slices into cl_req's, possibly adding cached pages to some of
- * the resulting req's.
- *
- * Whenever a page from cl_page_list is added to a newly constructed req, its
- * cl_page_operations::cpo_prep() layer methods are called. At that moment,
- * page state is atomically changed from cl_page_state::CPS_OWNED to
- * cl_page_state::CPS_PAGEOUT or cl_page_state::CPS_PAGEIN, cl_page::cp_owner
- * is zeroed, and cl_page::cp_req is set to the
- * req. cl_page_operations::cpo_prep() method at the particular layer might
- * return -EALREADY to indicate that it does not need to submit this page
- * at all. This is possible, for example, if page, submitted for read,
- * became up-to-date in the meantime; and for write, the page don't have
- * dirty bit marked. \see cl_io_submit_rw()
- *
- * Whenever a cached page is added to a newly constructed req, its
- * cl_page_operations::cpo_make_ready() layer methods are called. At that
- * moment, page state is atomically changed from cl_page_state::CPS_CACHED to
- * cl_page_state::CPS_PAGEOUT, and cl_page::cp_req is set to
- * req. cl_page_operations::cpo_make_ready() method at the particular layer
- * might return -EAGAIN to indicate that this page is not eligible for the
- * transfer right now.
- *
- * FUTURE
- *
- * Plan is to divide transfers into "priority bands" (indicated when
- * submitting cl_page_list, and queuing a page for the opportunistic transfer)
- * and allow glueing of cached pages to immediate transfers only within single
- * band. This would make high priority transfers (like lock cancellation or
- * memory pressure induced write-out) really high priority.
- *
- */
-
 /**
  * Per-transfer attributes.
  */
 struct cl_req_attr {
+	enum cl_req_type cra_type;
+	u64		 cra_flags;
+	struct cl_page	*cra_page;
+
 	/** Generic attributes for the server consumption. */
 	struct obdo	*cra_oa;
 	/** Jobid */
 	char		 cra_jobid[LUSTRE_JOBID_SIZE];
 };
 
-/**
- * Transfer request operations definable at every layer.
- *
- * Concurrency: transfer formation engine synchronizes calls to all transfer
- * methods.
- */
-struct cl_req_operations {
-	/**
-	 * Invoked top-to-bottom by cl_req_prep() when transfer formation is
-	 * complete (all pages are added).
-	 *
-	 * \see osc_req_prep()
-	 */
-	int  (*cro_prep)(const struct lu_env *env,
-			 const struct cl_req_slice *slice);
-	/**
-	 * Called top-to-bottom to fill in \a oa fields. This is called twice
-	 * with different flags, see bug 10150 and osc_build_req().
-	 *
-	 * \param obj an object from cl_req which attributes are to be set in
-	 *	    \a oa.
-	 *
-	 * \param oa struct obdo where attributes are placed
-	 *
-	 * \param flags \a oa fields to be filled.
-	 */
-	void (*cro_attr_set)(const struct lu_env *env,
-			     const struct cl_req_slice *slice,
-			     const struct cl_object *obj,
-			     struct cl_req_attr *attr, u64 flags);
-	/**
-	 * Called top-to-bottom from cl_req_completion() to notify layers that
-	 * transfer completed. Has to free all state allocated by
-	 * cl_device_operations::cdo_req_init().
-	 */
-	void (*cro_completion)(const struct lu_env *env,
-			       const struct cl_req_slice *slice, int ioret);
-};
-
-/**
- * A per-object state that (potentially multi-object) transfer request keeps.
- */
-struct cl_req_obj {
-	/** object itself */
-	struct cl_object   *ro_obj;
-	/** reference to cl_req_obj::ro_obj. For debugging. */
-	struct lu_ref_link  ro_obj_ref;
-	/* something else? Number of pages for a given object? */
-};
-
-/**
- * Transfer request.
- *
- * Transfer requests are not reference counted, because IO sub-system owns
- * them exclusively and knows when to free them.
- *
- * Life cycle.
- *
- * cl_req is created by cl_req_alloc() that calls
- * cl_device_operations::cdo_req_init() device methods to allocate per-req
- * state in every layer.
- *
- * Then pages are added (cl_req_page_add()), req keeps track of all objects it
- * contains pages for.
- *
- * Once all pages were collected, cl_page_operations::cpo_prep() method is
- * called top-to-bottom. At that point layers can modify req, let it pass, or
- * deny it completely. This is to support things like SNS that have transfer
- * ordering requirements invisible to the individual req-formation engine.
- *
- * On transfer completion (or transfer timeout, or failure to initiate the
- * transfer of an allocated req), cl_req_operations::cro_completion() method
- * is called, after execution of cl_page_operations::cpo_completion() of all
- * req's pages.
- */
-struct cl_req {
-	enum cl_req_type      crq_type;
-	/** A list of pages being transferred */
-	struct list_head	    crq_pages;
-	/** Number of pages in cl_req::crq_pages */
-	unsigned	      crq_nrpages;
-	/** An array of objects which pages are in ->crq_pages */
-	struct cl_req_obj    *crq_o;
-	/** Number of elements in cl_req::crq_objs[] */
-	unsigned	      crq_nrobjs;
-	struct list_head	    crq_layers;
-};
-
-/**
- * Per-layer state for request.
- */
-struct cl_req_slice {
-	struct cl_req    *crs_req;
-	struct cl_device *crs_dev;
-	struct list_head	crs_linkage;
-	const struct cl_req_operations *crs_ops;
-};
-
-/* @} cl_req */
-
 enum cache_stats_item {
 	/** how many cache lookups were performed */
 	CS_lookup = 0,
@@ -2153,9 +2004,6 @@ void cl_lock_slice_add(struct cl_lock *lock, struct cl_lock_slice *slice,
 		       const struct cl_lock_operations *ops);
 void cl_io_slice_add(struct cl_io *io, struct cl_io_slice *slice,
 		     struct cl_object *obj, const struct cl_io_operations *ops);
-void cl_req_slice_add(struct cl_req *req, struct cl_req_slice *slice,
-		      struct cl_device *dev,
-		      const struct cl_req_operations *ops);
 /** @} helpers */
 
 /** \defgroup cl_object cl_object
@@ -2183,6 +2031,12 @@ int cl_object_prune(const struct lu_env *env, struct cl_object *obj);
 void cl_object_kill(const struct lu_env *env, struct cl_object *obj);
 int  cl_object_getstripe(const struct lu_env *env, struct cl_object *obj,
 			 struct lov_user_md __user *lum);
+int cl_object_fiemap(const struct lu_env *env, struct cl_object *obj,
+		     struct ll_fiemap_info_key *fmkey, struct fiemap *fiemap,
+		     size_t *buflen);
+int cl_object_layout_get(const struct lu_env *env, struct cl_object *obj,
+			 struct cl_layout *cl);
+loff_t cl_object_maxbytes(struct cl_object *obj);
 
 /**
  * Returns true, iff \a o0 and \a o1 are slices of the same object.
@@ -2302,8 +2156,6 @@ void cl_page_discard(const struct lu_env *env, struct cl_io *io,
 void cl_page_delete(const struct lu_env *env, struct cl_page *pg);
 int cl_page_is_vmlocked(const struct lu_env *env, const struct cl_page *pg);
 void cl_page_export(const struct lu_env *env, struct cl_page *pg, int uptodate);
-int cl_page_is_under_lock(const struct lu_env *env, struct cl_io *io,
-			  struct cl_page *page, pgoff_t *max_index);
 loff_t cl_offset(const struct cl_object *obj, pgoff_t idx);
 pgoff_t cl_index(const struct cl_object *obj, loff_t offset);
 size_t cl_page_size(const struct cl_object *obj);
@@ -2414,8 +2266,6 @@ int cl_io_lock_add(const struct lu_env *env, struct cl_io *io,
 		   struct cl_io_lock_link *link);
 int cl_io_lock_alloc_add(const struct lu_env *env, struct cl_io *io,
 			 struct cl_lock_descr *descr);
-int cl_io_read_page(const struct lu_env *env, struct cl_io *io,
-		    struct cl_page *page);
 int cl_io_submit_rw(const struct lu_env *env, struct cl_io *io,
 		    enum cl_req_type iot, struct cl_2queue *queue);
 int cl_io_submit_sync(const struct lu_env *env, struct cl_io *io,
@@ -2424,6 +2274,8 @@ int cl_io_submit_sync(const struct lu_env *env, struct cl_io *io,
 int cl_io_commit_async(const struct lu_env *env, struct cl_io *io,
 		       struct cl_page_list *queue, int from, int to,
 		       cl_commit_cbt cb);
+int cl_io_read_ahead(const struct lu_env *env, struct cl_io *io,
+		     pgoff_t start, struct cl_read_ahead *ra);
 int cl_io_is_going(const struct lu_env *env);
 
 /**
@@ -2520,19 +2372,8 @@ void cl_2queue_init_page(struct cl_2queue *queue, struct cl_page *page);
 
 /** @} cl_page_list */
 
-/** \defgroup cl_req cl_req
- * @{
- */
-struct cl_req *cl_req_alloc(const struct lu_env *env, struct cl_page *page,
-			    enum cl_req_type crt, int nr_objects);
-
-void cl_req_page_add(const struct lu_env *env, struct cl_req *req,
-		     struct cl_page *page);
-void cl_req_page_done(const struct lu_env *env, struct cl_page *page);
-int  cl_req_prep(const struct lu_env *env, struct cl_req *req);
-void cl_req_attr_set(const struct lu_env *env, struct cl_req *req,
-		     struct cl_req_attr *attr, u64 flags);
-void cl_req_completion(const struct lu_env *env, struct cl_req *req, int ioret);
+void cl_req_attr_set(const struct lu_env *env, struct cl_object *obj,
+		     struct cl_req_attr *attr);
 
 /** \defgroup cl_sync_io cl_sync_io
  * @{
@@ -2568,8 +2409,6 @@ void cl_sync_io_end(const struct lu_env *env, struct cl_sync_io *anchor);
 
 /** @} cl_sync_io */
 
-/** @} cl_req */
-
 /** \defgroup cl_env cl_env
  *
  * lu_env handling for a client.
@@ -2593,35 +2432,13 @@ void cl_sync_io_end(const struct lu_env *env, struct cl_sync_io *anchor);
  *     - allocation and destruction of environment is amortized by caching no
  *     longer used environments instead of destroying them;
  *
- *     - there is a notion of "current" environment, attached to the kernel
- *     data structure representing current thread Top-level lustre code
- *     allocates an environment and makes it current, then calls into
- *     non-lustre code, that in turn calls lustre back. Low-level lustre
- *     code thus called can fetch environment created by the top-level code
- *     and reuse it, avoiding additional environment allocation.
- *       Right now, three interfaces can attach the cl_env to running thread:
- *       - cl_env_get
- *       - cl_env_implant
- *       - cl_env_reexit(cl_env_reenter had to be called priorly)
- *
  * \see lu_env, lu_context, lu_context_key
  * @{
  */
 
-struct cl_env_nest {
-	int   cen_refcheck;
-	void *cen_cookie;
-};
-
 struct lu_env *cl_env_get(int *refcheck);
 struct lu_env *cl_env_alloc(int *refcheck, __u32 tags);
-struct lu_env *cl_env_nested_get(struct cl_env_nest *nest);
 void cl_env_put(struct lu_env *env, int *refcheck);
-void cl_env_nested_put(struct cl_env_nest *nest, struct lu_env *env);
-void *cl_env_reenter(void);
-void cl_env_reexit(void *cookie);
-void cl_env_implant(struct lu_env *env, int *refcheck);
-void cl_env_unplant(struct lu_env *env, int *refcheck);
 unsigned int cl_env_cache_purge(unsigned int nr);
 struct lu_env *cl_env_percpu_get(void);
 void cl_env_percpu_put(struct lu_env *env);
diff --git a/drivers/staging/lustre/lustre/include/llog_swab.h b/drivers/staging/lustre/lustre/include/llog_swab.h
new file mode 100644
index 000000000000..fd7ffb154ad1
--- /dev/null
+++ b/drivers/staging/lustre/lustre/include/llog_swab.h
@@ -0,0 +1,65 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ *
+ * Copyright (c) 2011, 2014, Intel Corporation.
+ *
+ * Copyright 2015 Cray Inc, all rights reserved.
+ * Author: Ben Evans.
+ *
+ * We assume all nodes are either little-endian or big-endian, and we
+ * always send messages in the sender's native format.  The receiver
+ * detects the message format by checking the 'magic' field of the message
+ * (see lustre_msg_swabbed() below).
+ *
+ * Each type has corresponding 'lustre_swab_xxxtypexxx()' routines
+ * are implemented in ptlrpc/pack_generic.c.  These 'swabbers' convert the
+ * type from "other" endian, in-place in the message buffer.
+ *
+ * A swabber takes a single pointer argument.  The caller must already have
+ * verified that the length of the message buffer >= sizeof (type).
+ *
+ * For variable length types, a second 'lustre_swab_v_xxxtypexxx()' routine
+ * may be defined that swabs just the variable part, after the caller has
+ * verified that the message buffer is large enough.
+ */
+
+#ifndef _LLOG_SWAB_H_
+#define _LLOG_SWAB_H_
+
+#include "lustre/lustre_idl.h"
+struct lustre_cfg;
+
+void lustre_swab_lu_fid(struct lu_fid *fid);
+void lustre_swab_ost_id(struct ost_id *oid);
+void lustre_swab_llogd_body(struct llogd_body *d);
+void lustre_swab_llog_hdr(struct llog_log_hdr *h);
+void lustre_swab_llogd_conn_body(struct llogd_conn_body *d);
+void lustre_swab_llog_rec(struct llog_rec_hdr *rec);
+void lustre_swab_lu_seq_range(struct lu_seq_range *range);
+void lustre_swab_lustre_cfg(struct lustre_cfg *lcfg);
+void lustre_swab_cfg_marker(struct cfg_marker *marker,
+			    int swab, int size);
+
+#endif
diff --git a/drivers/staging/lustre/lustre/include/lprocfs_status.h b/drivers/staging/lustre/lustre/include/lprocfs_status.h
index cc0713ef8ae5..62753dae0bfa 100644
--- a/drivers/staging/lustre/lustre/include/lprocfs_status.h
+++ b/drivers/staging/lustre/lustre/include/lprocfs_status.h
@@ -43,6 +43,8 @@
 #include <linux/spinlock.h>
 #include <linux/types.h>
 
+#include "../../include/linux/libcfs/libcfs.h"
+#include "lustre_cfg.h"
 #include "lustre/lustre_idl.h"
 
 struct lprocfs_vars {
@@ -540,7 +542,8 @@ lprocfs_alloc_stats(unsigned int num, enum lprocfs_stats_flags flags);
 void lprocfs_clear_stats(struct lprocfs_stats *stats);
 void lprocfs_free_stats(struct lprocfs_stats **stats);
 void lprocfs_counter_init(struct lprocfs_stats *stats, int index,
-			  unsigned conf, const char *name, const char *units);
+			  unsigned int conf, const char *name,
+			  const char *units);
 struct obd_export;
 int lprocfs_exp_cleanup(struct obd_export *exp);
 struct dentry *ldebugfs_add_simple(struct dentry *root,
@@ -701,9 +704,9 @@ static struct lustre_attr lustre_attr_##name = __ATTR(name, mode, show, store)
 extern const struct sysfs_ops lustre_sysfs_ops;
 
 struct root_squash_info;
-int lprocfs_wr_root_squash(const char *buffer, unsigned long count,
+int lprocfs_wr_root_squash(const char __user *buffer, unsigned long count,
 			   struct root_squash_info *squash, char *name);
-int lprocfs_wr_nosquash_nids(const char *buffer, unsigned long count,
+int lprocfs_wr_nosquash_nids(const char __user *buffer, unsigned long count,
 			     struct root_squash_info *squash, char *name);
 
 /* all quota proc functions */
diff --git a/drivers/staging/lustre/lustre/include/lustre/ll_fiemap.h b/drivers/staging/lustre/lustre/include/lustre/ll_fiemap.h
index c2340d643e84..b8ad5559a3b9 100644
--- a/drivers/staging/lustre/lustre/include/lustre/ll_fiemap.h
+++ b/drivers/staging/lustre/lustre/include/lustre/ll_fiemap.h
@@ -41,79 +41,24 @@
 #ifndef _LUSTRE_FIEMAP_H
 #define _LUSTRE_FIEMAP_H
 
-struct ll_fiemap_extent {
-	__u64 fe_logical;  /* logical offset in bytes for the start of
-			    * the extent from the beginning of the file
-			    */
-	__u64 fe_physical; /* physical offset in bytes for the start
-			    * of the extent from the beginning of the disk
-			    */
-	__u64 fe_length;   /* length in bytes for this extent */
-	__u64 fe_reserved64[2];
-	__u32 fe_flags;    /* FIEMAP_EXTENT_* flags for this extent */
-	__u32 fe_device;   /* device number for this extent */
-	__u32 fe_reserved[2];
-};
-
-struct ll_user_fiemap {
-	__u64 fm_start;  /* logical offset (inclusive) at
-			  * which to start mapping (in)
-			  */
-	__u64 fm_length; /* logical length of mapping which
-			  * userspace wants (in)
-			  */
-	__u32 fm_flags;  /* FIEMAP_FLAG_* flags for request (in/out) */
-	__u32 fm_mapped_extents;/* number of extents that were mapped (out) */
-	__u32 fm_extent_count;  /* size of fm_extents array (in) */
-	__u32 fm_reserved;
-	struct ll_fiemap_extent fm_extents[0]; /* array of mapped extents (out) */
-};
-
-#define FIEMAP_MAX_OFFSET      (~0ULL)
+#ifndef __KERNEL__
+#include <stddef.h>
+#include <fiemap.h>
+#endif
 
-#define FIEMAP_FLAG_SYNC		0x00000001 /* sync file data before
-						    * map
-						    */
-#define FIEMAP_FLAG_XATTR		0x00000002 /* map extended attribute
-						    * tree
-						    */
-#define FIEMAP_EXTENT_LAST		0x00000001 /* Last extent in file. */
-#define FIEMAP_EXTENT_UNKNOWN		0x00000002 /* Data location unknown. */
-#define FIEMAP_EXTENT_DELALLOC		0x00000004 /* Location still pending.
-						    * Sets EXTENT_UNKNOWN.
-						    */
-#define FIEMAP_EXTENT_ENCODED		0x00000008 /* Data can not be read
-						    * while fs is unmounted
-						    */
-#define FIEMAP_EXTENT_DATA_ENCRYPTED	0x00000080 /* Data is encrypted by fs.
-						    * Sets EXTENT_NO_DIRECT.
-						    */
-#define FIEMAP_EXTENT_NOT_ALIGNED       0x00000100 /* Extent offsets may not be
-						    * block aligned.
-						    */
-#define FIEMAP_EXTENT_DATA_INLINE       0x00000200 /* Data mixed with metadata.
-						    * Sets EXTENT_NOT_ALIGNED.*/
-#define FIEMAP_EXTENT_DATA_TAIL		0x00000400 /* Multiple files in block.
-						    * Sets EXTENT_NOT_ALIGNED.
-						    */
-#define FIEMAP_EXTENT_UNWRITTEN		0x00000800 /* Space allocated, but
-						    * no data (i.e. zero).
-						    */
-#define FIEMAP_EXTENT_MERGED		0x00001000 /* File does not natively
-						    * support extents. Result
-						    * merged for efficiency.
-						    */
+/* XXX: We use fiemap_extent::fe_reserved[0] */
+#define fe_device	fe_reserved[0]
 
 static inline size_t fiemap_count_to_size(size_t extent_count)
 {
-	return (sizeof(struct ll_user_fiemap) + extent_count *
-					       sizeof(struct ll_fiemap_extent));
+	return sizeof(struct fiemap) + extent_count *
+				       sizeof(struct fiemap_extent);
 }
 
 static inline unsigned fiemap_size_to_count(size_t array_size)
 {
-	return ((array_size - sizeof(struct ll_user_fiemap)) /
-					       sizeof(struct ll_fiemap_extent));
+	return (array_size - sizeof(struct fiemap)) /
+		sizeof(struct fiemap_extent);
 }
 
 #define FIEMAP_FLAG_DEVICE_ORDER 0x40000000 /* return device ordered mapping */
diff --git a/drivers/staging/lustre/lustre/include/lustre/lustre_idl.h b/drivers/staging/lustre/lustre/include/lustre/lustre_idl.h
index 72eaee95c6b8..65ce503ad595 100644
--- a/drivers/staging/lustre/lustre/include/lustre/lustre_idl.h
+++ b/drivers/staging/lustre/lustre/include/lustre/lustre_idl.h
@@ -48,8 +48,7 @@
  * that the Lustre wire protocol is not influenced by external dependencies.
  *
  * The only other acceptable items in this file are VERY SIMPLE accessor
- * functions to avoid callers grubbing inside the structures, and the
- * prototypes of the swabber functions for each struct.  Nothing that
+ * functions to avoid callers grubbing inside the structures. Nothing that
  * depends on external functions or definitions should be in here.
  *
  * Structs must be properly aligned to put 64-bit values on an 8-byte
@@ -64,23 +63,6 @@
  * in the code to ensure that new/old clients that see this larger struct
  * do not fail, otherwise you need to implement protocol compatibility).
  *
- * We assume all nodes are either little-endian or big-endian, and we
- * always send messages in the sender's native format.  The receiver
- * detects the message format by checking the 'magic' field of the message
- * (see lustre_msg_swabbed() below).
- *
- * Each wire type has corresponding 'lustre_swab_xxxtypexxx()' routines,
- * implemented either here, inline (trivial implementations) or in
- * ptlrpc/pack_generic.c.  These 'swabbers' convert the type from "other"
- * endian, in-place in the message buffer.
- *
- * A swabber takes a single pointer argument.  The caller must already have
- * verified that the length of the message buffer >= sizeof (type).
- *
- * For variable length types, a second 'lustre_swab_v_xxxtypexxx()' routine
- * may be defined that swabs just the variable part, after the caller has
- * verified that the message buffer is large enough.
- *
  * @{
  */
 
@@ -192,113 +174,6 @@ struct lu_seq_range_array {
 
 #define LU_SEQ_RANGE_MASK	0x3
 
-static inline unsigned fld_range_type(const struct lu_seq_range *range)
-{
-	return range->lsr_flags & LU_SEQ_RANGE_MASK;
-}
-
-static inline bool fld_range_is_ost(const struct lu_seq_range *range)
-{
-	return fld_range_type(range) == LU_SEQ_RANGE_OST;
-}
-
-static inline bool fld_range_is_mdt(const struct lu_seq_range *range)
-{
-	return fld_range_type(range) == LU_SEQ_RANGE_MDT;
-}
-
-/**
- * This all range is only being used when fld client sends fld query request,
- * but it does not know whether the seq is MDT or OST, so it will send req
- * with ALL type, which means either seq type gotten from lookup can be
- * expected.
- */
-static inline unsigned fld_range_is_any(const struct lu_seq_range *range)
-{
-	return fld_range_type(range) == LU_SEQ_RANGE_ANY;
-}
-
-static inline void fld_range_set_type(struct lu_seq_range *range,
-				      unsigned flags)
-{
-	range->lsr_flags |= flags;
-}
-
-static inline void fld_range_set_mdt(struct lu_seq_range *range)
-{
-	fld_range_set_type(range, LU_SEQ_RANGE_MDT);
-}
-
-static inline void fld_range_set_ost(struct lu_seq_range *range)
-{
-	fld_range_set_type(range, LU_SEQ_RANGE_OST);
-}
-
-static inline void fld_range_set_any(struct lu_seq_range *range)
-{
-	fld_range_set_type(range, LU_SEQ_RANGE_ANY);
-}
-
-/**
- * returns  width of given range \a r
- */
-
-static inline __u64 range_space(const struct lu_seq_range *range)
-{
-	return range->lsr_end - range->lsr_start;
-}
-
-/**
- * initialize range to zero
- */
-
-static inline void range_init(struct lu_seq_range *range)
-{
-	memset(range, 0, sizeof(*range));
-}
-
-/**
- * check if given seq id \a s is within given range \a r
- */
-
-static inline bool range_within(const struct lu_seq_range *range,
-				__u64 s)
-{
-	return s >= range->lsr_start && s < range->lsr_end;
-}
-
-static inline bool range_is_sane(const struct lu_seq_range *range)
-{
-	return (range->lsr_end >= range->lsr_start);
-}
-
-static inline bool range_is_zero(const struct lu_seq_range *range)
-{
-	return (range->lsr_start == 0 && range->lsr_end == 0);
-}
-
-static inline bool range_is_exhausted(const struct lu_seq_range *range)
-
-{
-	return range_space(range) == 0;
-}
-
-/* return 0 if two range have the same location */
-static inline int range_compare_loc(const struct lu_seq_range *r1,
-				    const struct lu_seq_range *r2)
-{
-	return r1->lsr_index != r2->lsr_index ||
-	       r1->lsr_flags != r2->lsr_flags;
-}
-
-#define DRANGE "[%#16.16Lx-%#16.16Lx):%x:%s"
-
-#define PRANGE(range)		\
-	(range)->lsr_start,	\
-	(range)->lsr_end,	\
-	(range)->lsr_index,	\
-	fld_range_is_mdt(range) ? "mdt" : "ost"
-
 /** \defgroup lu_fid lu_fid
  * @{
  */
@@ -310,7 +185,7 @@ static inline int range_compare_loc(const struct lu_seq_range *r1,
  */
 enum lma_compat {
 	LMAC_HSM	= 0x00000001,
-	LMAC_SOM	= 0x00000002,
+/*	LMAC_SOM	= 0x00000002, obsolete since 2.8.0 */
 	LMAC_NOT_IN_OI	= 0x00000004, /* the object does NOT need OI mapping */
 	LMAC_FID_ON_OST = 0x00000008, /* For OST-object, its OI mapping is
 				       * under /O/<seq>/d<x>.
@@ -644,13 +519,14 @@ static inline void ostid_set_id(struct ost_id *oi, __u64 oid)
 {
 	if (fid_seq_is_mdt0(oi->oi.oi_seq)) {
 		if (oid >= IDIF_MAX_OID) {
-			CERROR("Bad %llu to set " DOSTID "\n", oid, POSTID(oi));
+			CERROR("Too large OID %#llx to set MDT0 " DOSTID "\n",
+			       oid, POSTID(oi));
 			return;
 		}
 		oi->oi.oi_id = oid;
 	} else if (fid_is_idif(&oi->oi_fid)) {
 		if (oid >= IDIF_MAX_OID) {
-			CERROR("Bad %llu to set "DOSTID"\n",
+			CERROR("Too large OID %#llx to set IDIF " DOSTID "\n",
 			       oid, POSTID(oi));
 			return;
 		}
@@ -676,7 +552,7 @@ static inline int fid_set_id(struct lu_fid *fid, __u64 oid)
 
 	if (fid_is_idif(fid)) {
 		if (oid >= IDIF_MAX_OID) {
-			CERROR("Too large OID %#llx to set IDIF "DFID"\n",
+			CERROR("Too large OID %#llx to set IDIF " DFID "\n",
 			       (unsigned long long)oid, PFID(fid));
 			return -EBADF;
 		}
@@ -685,7 +561,7 @@ static inline int fid_set_id(struct lu_fid *fid, __u64 oid)
 		fid->f_ver = oid >> 48;
 	} else {
 		if (oid >= OBIF_MAX_OID) {
-			CERROR("Too large OID %#llx to set REG "DFID"\n",
+			CERROR("Too large OID %#llx to set REG " DFID "\n",
 			       (unsigned long long)oid, PFID(fid));
 			return -EBADF;
 		}
@@ -785,8 +661,6 @@ static inline ino_t lu_igif_ino(const struct lu_fid *fid)
 	return fid_seq(fid);
 }
 
-void lustre_swab_ost_id(struct ost_id *oid);
-
 /**
  * Get inode generation from a igif.
  * \param fid a igif to get inode generation from.
@@ -847,9 +721,6 @@ static inline bool fid_is_sane(const struct lu_fid *fid)
 		fid_seq_is_rsvd(fid_seq(fid)));
 }
 
-void lustre_swab_lu_fid(struct lu_fid *fid);
-void lustre_swab_lu_seq_range(struct lu_seq_range *range);
-
 static inline bool lu_fid_eq(const struct lu_fid *f0, const struct lu_fid *f1)
 {
 	return memcmp(f0, f1, sizeof(*f0)) == 0;
@@ -1099,8 +970,10 @@ struct ptlrpc_body_v3 {
 	__u32 pb_version;
 	__u32 pb_opc;
 	__u32 pb_status;
-	__u64 pb_last_xid;
-	__u64 pb_last_seen;
+	__u64 pb_last_xid; /* highest replied XID without lower unreplied XID */
+	__u16 pb_tag;      /* virtual slot idx for multiple modifying RPCs */
+	__u16 pb_padding0;
+	__u32 pb_padding1;
 	__u64 pb_last_committed;
 	__u64 pb_transno;
 	__u32 pb_flags;
@@ -1112,8 +985,11 @@ struct ptlrpc_body_v3 {
 	__u64 pb_slv;
 	/* VBR: pre-versions */
 	__u64 pb_pre_versions[PTLRPC_NUM_VERSIONS];
+	__u64 pb_mbits; /**< match bits for bulk request */
 	/* padding for future needs */
-	__u64 pb_padding[4];
+	__u64 pb_padding64_0;
+	__u64 pb_padding64_1;
+	__u64 pb_padding64_2;
 	char  pb_jobid[LUSTRE_JOBID_SIZE];
 };
 
@@ -1125,8 +1001,10 @@ struct ptlrpc_body_v2 {
 	__u32 pb_version;
 	__u32 pb_opc;
 	__u32 pb_status;
-	__u64 pb_last_xid;
-	__u64 pb_last_seen;
+	__u64 pb_last_xid; /* highest replied XID without lower unreplied XID */
+	__u16 pb_tag;      /* virtual slot idx for multiple modifying RPCs */
+	__u16 pb_padding0;
+	__u32 pb_padding1;
 	__u64 pb_last_committed;
 	__u64 pb_transno;
 	__u32 pb_flags;
@@ -1140,12 +1018,13 @@ struct ptlrpc_body_v2 {
 	__u64 pb_slv;
 	/* VBR: pre-versions */
 	__u64 pb_pre_versions[PTLRPC_NUM_VERSIONS];
+	__u64 pb_mbits; /**< unused in V2 */
 	/* padding for future needs */
-	__u64 pb_padding[4];
+	__u64 pb_padding64_0;
+	__u64 pb_padding64_1;
+	__u64 pb_padding64_2;
 };
 
-void lustre_swab_ptlrpc_body(struct ptlrpc_body *pb);
-
 /* message body offset for lustre_msg_v2 */
 /* ptlrpc body offset in all request/reply messages */
 #define MSG_PTLRPC_BODY_OFF	     0
@@ -1282,7 +1161,16 @@ void lustre_swab_ptlrpc_body(struct ptlrpc_body *pb);
 							 */
 #define OBD_CONNECT_LFSCK	0x40000000000000ULL/* support online LFSCK */
 #define OBD_CONNECT_UNLINK_CLOSE 0x100000000000000ULL/* close file in unlink */
+#define OBD_CONNECT_MULTIMODRPCS 0x200000000000000ULL /* support multiple modify
+						       *  RPCs in parallel
+						       */
 #define OBD_CONNECT_DIR_STRIPE	 0x400000000000000ULL/* striped DNE dir */
+#define OBD_CONNECT_SUBTREE	 0x800000000000000ULL /* fileset mount */
+#define OBD_CONNECT_LOCK_AHEAD	 0x1000000000000000ULL /* lock ahead */
+/** bulk matchbits is sent within ptlrpc_body */
+#define OBD_CONNECT_BULK_MBITS	 0x2000000000000000ULL
+#define OBD_CONNECT_OBDOPACK	 0x4000000000000000ULL /* compact OUT obdo */
+#define OBD_CONNECT_FLAGS2	 0x8000000000000000ULL /* second flags word */
 
 /* XXX README XXX:
  * Please DO NOT add flag values here before first ensuring that this same
@@ -1313,25 +1201,6 @@ void lustre_swab_ptlrpc_body(struct ptlrpc_body *pb);
  * If we eventually have separate connect data for different types, which we
  * almost certainly will, then perhaps we stick a union in here.
  */
-struct obd_connect_data_v1 {
-	__u64 ocd_connect_flags; /* OBD_CONNECT_* per above */
-	__u32 ocd_version;	 /* lustre release version number */
-	__u32 ocd_grant;	 /* initial cache grant amount (bytes) */
-	__u32 ocd_index;	 /* LOV index to connect to */
-	__u32 ocd_brw_size;	 /* Maximum BRW size in bytes, must be 2^n */
-	__u64 ocd_ibits_known;   /* inode bits this client understands */
-	__u8  ocd_blocksize;     /* log2 of the backend filesystem blocksize */
-	__u8  ocd_inodespace;    /* log2 of the per-inode space consumption */
-	__u16 ocd_grant_extent;  /* per-extent grant overhead, in 1K blocks */
-	__u32 ocd_unused;	/* also fix lustre_swab_connect */
-	__u64 ocd_transno;       /* first transno from client to be replayed */
-	__u32 ocd_group;	 /* MDS group on OST */
-	__u32 ocd_cksum_types;   /* supported checksum algorithms */
-	__u32 ocd_max_easize;    /* How big LOV EA can be on MDS */
-	__u32 ocd_instance;      /* also fix lustre_swab_connect */
-	__u64 ocd_maxbytes;      /* Maximum stripe size in bytes */
-};
-
 struct obd_connect_data {
 	__u64 ocd_connect_flags; /* OBD_CONNECT_* per above */
 	__u32 ocd_version;	 /* lustre release version number */
@@ -1354,8 +1223,10 @@ struct obd_connect_data {
 	 * any field after ocd_maxbytes on the receiver without a valid flag
 	 * may result in out-of-bound memory access and kernel oops.
 	 */
-	__u64 padding1;	  /* added 2.1.0. also fix lustre_swab_connect */
-	__u64 padding2;	  /* added 2.1.0. also fix lustre_swab_connect */
+	__u16 ocd_maxmodrpcs;	/* Maximum modify RPCs in parallel */
+	__u16 padding0;		/* added 2.1.0. also fix lustre_swab_connect */
+	__u32 padding1;		/* added 2.1.0. also fix lustre_swab_connect */
+	__u64 ocd_connect_flags2;
 	__u64 padding3;	  /* added 2.1.0. also fix lustre_swab_connect */
 	__u64 padding4;	  /* added 2.1.0. also fix lustre_swab_connect */
 	__u64 padding5;	  /* added 2.1.0. also fix lustre_swab_connect */
@@ -1380,8 +1251,6 @@ struct obd_connect_data {
  * reserve the flag for future use.
  */
 
-void lustre_swab_connect(struct obd_connect_data *ocd);
-
 /*
  * Supported checksum algorithms. Up to 32 checksum types are supported.
  * (32-bit mask stored in obd_connect_data::ocd_cksum_types)
@@ -1416,7 +1285,7 @@ enum ost_cmd {
 	OST_STATFS     = 13,
 	OST_SYNC       = 16,
 	OST_SET_INFO   = 17,
-	OST_QUOTACHECK = 18,
+	OST_QUOTACHECK = 18, /* not used since 2.4 */
 	OST_QUOTACTL   = 19,
 	OST_QUOTA_ADJUST_QUNIT = 20, /* not used since 2.4 */
 	OST_LAST_OPC
@@ -1580,8 +1449,6 @@ static inline void lmm_oi_cpu_to_le(struct ost_id *dst_oi,
 	dst_oi->oi.oi_seq = cpu_to_le64(src_oi->oi.oi_seq);
 }
 
-/* extern void lustre_swab_lov_mds_md(struct lov_mds_md *llm); */
-
 #define MAX_MD_SIZE							\
 	(sizeof(struct lov_mds_md) + 4 * sizeof(struct lov_ost_data))
 #define MIN_MD_SIZE							\
@@ -1674,7 +1541,7 @@ lov_mds_md_max_stripe_count(size_t buf_size, __u32 lmm_magic)
 #define OBD_MD_FLCKSUM     (0x00100000ULL) /* bulk data checksum */
 #define OBD_MD_FLQOS       (0x00200000ULL) /* quality of service stats */
 /*#define OBD_MD_FLOSCOPQ    (0x00400000ULL) osc opaque data, never used */
-#define OBD_MD_FLCOOKIE    (0x00800000ULL) /* log cancellation cookie */
+/*	OBD_MD_FLCOOKIE    (0x00800000ULL) obsolete in 2.8 */
 #define OBD_MD_FLGROUP     (0x01000000ULL) /* group */
 #define OBD_MD_FLFID       (0x02000000ULL) /* ->ost write inline fid */
 #define OBD_MD_FLEPOCH     (0x04000000ULL) /* ->ost write with ioepoch */
@@ -1713,7 +1580,9 @@ lov_mds_md_max_stripe_count(size_t buf_size, __u32 lmm_magic)
 /*	OBD_MD_FLRMTRGETFACL (0x0008000000000000ULL) lfs rgetfacl, obsolete */
 
 #define OBD_MD_FLDATAVERSION (0x0010000000000000ULL) /* iversion sum */
-#define OBD_MD_FLRELEASED    (0x0020000000000000ULL) /* file released */
+#define OBD_MD_CLOSE_INTENT_EXECED (0x0020000000000000ULL) /* close intent
+							    * executed
+							    */
 
 #define OBD_MD_DEFAULT_MEA   (0x0040000000000000ULL) /* default MEA */
 
@@ -1742,11 +1611,6 @@ struct hsm_state_set {
 	__u64	hss_clearmask;
 };
 
-void lustre_swab_hsm_user_state(struct hsm_user_state *hus);
-void lustre_swab_hsm_state_set(struct hsm_state_set *hss);
-
-void lustre_swab_obd_statfs(struct obd_statfs *os);
-
 /* ost_body.data values for OST_BRW */
 
 #define OBD_BRW_READ		0x01
@@ -1786,14 +1650,16 @@ struct obd_ioobj {
 	__u32		ioo_bufcnt;	/* number of niobufs for this object */
 };
 
+/*
+ * NOTE: IOOBJ_MAX_BRW_BITS defines the _offset_ of the max_brw field in
+ * ioo_max_brw, NOT the maximum number of bits in PTLRPC_BULK_OPS_BITS.
+ * That said, ioo_max_brw is a 32-bit field so the limit is also 16 bits.
+ */
 #define IOOBJ_MAX_BRW_BITS	16
-#define IOOBJ_TYPE_MASK		((1U << IOOBJ_MAX_BRW_BITS) - 1)
 #define ioobj_max_brw_get(ioo)	(((ioo)->ioo_max_brw >> IOOBJ_MAX_BRW_BITS) + 1)
 #define ioobj_max_brw_set(ioo, num)					\
 do { (ioo)->ioo_max_brw = ((num) - 1) << IOOBJ_MAX_BRW_BITS; } while (0)
 
-void lustre_swab_obd_ioobj(struct obd_ioobj *ioo);
-
 /* multiple of 8 bytes => can array */
 struct niobuf_remote {
 	__u64	rnb_offset;
@@ -1801,8 +1667,6 @@ struct niobuf_remote {
 	__u32	rnb_flags;
 };
 
-void lustre_swab_niobuf_remote(struct niobuf_remote *nbr);
-
 /* lock value block communicated between the filter and llite */
 
 /* OST_LVB_ERR_INIT is needed because the return code in rc is
@@ -1824,8 +1688,6 @@ struct ost_lvb_v1 {
 	__u64		lvb_blocks;
 };
 
-void lustre_swab_ost_lvb_v1(struct ost_lvb_v1 *lvb);
-
 struct ost_lvb {
 	__u64		lvb_size;
 	__s64		lvb_mtime;
@@ -1838,8 +1700,6 @@ struct ost_lvb {
 	__u32		lvb_padding;
 };
 
-void lustre_swab_ost_lvb(struct ost_lvb *lvb);
-
 /*
  *   lquota data structures
  */
@@ -1866,8 +1726,6 @@ struct obd_quotactl {
 	struct obd_dqblk	qc_dqblk;
 };
 
-void lustre_swab_obd_quotactl(struct obd_quotactl *q);
-
 #define Q_COPY(out, in, member) (out)->member = (in)->member
 
 #define QCTL_COPY(out, in)		\
@@ -1905,8 +1763,6 @@ struct lquota_lvb {
 	__u64	lvb_pad1;
 };
 
-void lustre_swab_lquota_lvb(struct lquota_lvb *lvb);
-
 /* op codes */
 enum quota_cmd {
 	QUOTA_DQACQ	= 601,
@@ -1933,9 +1789,9 @@ enum mds_cmd {
 	MDS_PIN			= 42, /* obsolete, never used in a release */
 	MDS_UNPIN		= 43, /* obsolete, never used in a release */
 	MDS_SYNC		= 44,
-	MDS_DONE_WRITING	= 45,
+	MDS_DONE_WRITING	= 45, /* obsolete since 2.8.0 */
 	MDS_SET_INFO		= 46,
-	MDS_QUOTACHECK		= 47,
+	MDS_QUOTACHECK		= 47, /* not used since 2.4 */
 	MDS_QUOTACTL		= 48,
 	MDS_GETXATTR		= 49,
 	MDS_SETXATTR		= 50, /* obsolete, now it's MDS_REINT op */
@@ -1972,8 +1828,6 @@ enum mdt_reint_cmd {
 	REINT_MAX
 };
 
-void lustre_swab_generic_32s(__u32 *val);
-
 /* the disposition of the intent outlines what was executed */
 #define DISP_IT_EXECD	0x00000001
 #define DISP_LOOKUP_EXECD    0x00000002
@@ -2031,36 +1885,19 @@ enum {
 #define MDS_STATUS_CONN 1
 #define MDS_STATUS_LOV 2
 
-/* mdt_thread_info.mti_flags. */
-enum md_op_flags {
-	/* The flag indicates Size-on-MDS attributes are changed. */
-	MF_SOM_CHANGE	   = (1 << 0),
-	/* Flags indicates an epoch opens or closes. */
-	MF_EPOCH_OPEN	   = (1 << 1),
-	MF_EPOCH_CLOSE	  = (1 << 2),
-	MF_MDC_CANCEL_FID1      = (1 << 3),
-	MF_MDC_CANCEL_FID2      = (1 << 4),
-	MF_MDC_CANCEL_FID3      = (1 << 5),
-	MF_MDC_CANCEL_FID4      = (1 << 6),
-	/* There is a pending attribute update. */
-	MF_SOM_AU	       = (1 << 7),
-	/* Cancel OST locks while getattr OST attributes. */
-	MF_GETATTR_LOCK	 = (1 << 8),
-	MF_GET_MDT_IDX	  = (1 << 9),
-};
-
-#define MF_SOM_LOCAL_FLAGS (MF_SOM_CHANGE | MF_EPOCH_OPEN | MF_EPOCH_CLOSE)
-
-#define LUSTRE_BFLAG_UNCOMMITTED_WRITES   0x1
-
 /* these should be identical to their EXT4_*_FL counterparts, they are
  * redefined here only to avoid dragging in fs/ext4/ext4.h
  */
 #define LUSTRE_SYNC_FL	 0x00000008 /* Synchronous updates */
 #define LUSTRE_IMMUTABLE_FL    0x00000010 /* Immutable file */
 #define LUSTRE_APPEND_FL       0x00000020 /* writes to file may only append */
+#define LUSTRE_NODUMP_FL	0x00000040 /* do not dump file */
 #define LUSTRE_NOATIME_FL      0x00000080 /* do not update atime */
+#define LUSTRE_INDEX_FL		0x00001000 /* hash-indexed directory */
 #define LUSTRE_DIRSYNC_FL      0x00010000 /* dirsync behaviour (dir only) */
+#define LUSTRE_TOPDIR_FL	0x00020000 /* Top of directory hierarchies*/
+#define LUSTRE_DIRECTIO_FL	0x00100000 /* Use direct i/o */
+#define LUSTRE_INLINE_DATA_FL	0x10000000 /* Inode has inline data. */
 
 /* Convert wire LUSTRE_*_FL to corresponding client local VFS S_* values
  * for the client inode i_flags.  The LUSTRE_*_FL are the Lustre wire
@@ -2113,7 +1950,7 @@ struct mdt_body {
 	__u32	mbo_mode;
 	__u32	mbo_uid;
 	__u32	mbo_gid;
-	__u32	mbo_flags;
+	__u32	mbo_flags;	/* LUSTRE_*_FL file attributes */
 	__u32	mbo_rdev;
 	__u32	mbo_nlink;	/* #bytes to read in the case of MDS_READPAGE */
 	__u32	mbo_unused2;	/* was "generation" until 2.4.0 */
@@ -2121,7 +1958,7 @@ struct mdt_body {
 	__u32	mbo_eadatasize;
 	__u32	mbo_aclsize;
 	__u32	mbo_max_mdsize;
-	__u32	mbo_max_cookiesize;
+	__u32	mbo_unused3;	/* was max_cookiesize until 2.8 */
 	__u32	mbo_uid_h;	/* high 32-bits of uid, for FUID */
 	__u32	mbo_gid_h;	/* high 32-bits of gid, for FUID */
 	__u32	mbo_padding_5;	/* also fix lustre_swab_mdt_body */
@@ -2132,17 +1969,13 @@ struct mdt_body {
 	__u64	mbo_padding_10;
 }; /* 216 */
 
-void lustre_swab_mdt_body(struct mdt_body *b);
-
 struct mdt_ioepoch {
-	struct lustre_handle handle;
-	__u64  ioepoch;
-	__u32  flags;
-	__u32  padding;
+	struct lustre_handle mio_handle;
+	__u64 mio_unused1; /* was ioepoch */
+	__u32 mio_unused2; /* was flags */
+	__u32 mio_padding;
 };
 
-void lustre_swab_mdt_ioepoch(struct mdt_ioepoch *b);
-
 /* permissions for md_perm.mp_perm */
 enum {
 	CFS_SETUID_PERM = 0x01,
@@ -2178,8 +2011,6 @@ struct mdt_rec_setattr {
 	__u32	   sa_padding_5;
 };
 
-void lustre_swab_mdt_rec_setattr(struct mdt_rec_setattr *sa);
-
 /*
  * Attribute flags used in mdt_rec_setattr::sa_valid.
  * The kernel's #defines for ATTR_* should not be used over the network
@@ -2207,12 +2038,9 @@ void lustre_swab_mdt_rec_setattr(struct mdt_rec_setattr *sa);
 
 #define MDS_FMODE_CLOSED	 00000000
 #define MDS_FMODE_EXEC	   00000004
-/* IO Epoch is opened on a closed file. */
-#define MDS_FMODE_EPOCH	  01000000
-/* IO Epoch is opened on a file truncate. */
-#define MDS_FMODE_TRUNC	  02000000
-/* Size-on-MDS Attribute Update is pending. */
-#define MDS_FMODE_SOM	    04000000
+/*	MDS_FMODE_EPOCH		01000000 obsolete since 2.8.0 */
+/*	MDS_FMODE_TRUNC		02000000 obsolete since 2.8.0 */
+/*	MDS_FMODE_SOM		04000000 obsolete since 2.8.0 */
 
 #define MDS_OPEN_CREATED	 00000010
 #define MDS_OPEN_CROSS	   00000020
@@ -2258,7 +2086,7 @@ enum mds_op_bias {
 	MDS_CROSS_REF		= 1 << 1,
 	MDS_VTX_BYPASS		= 1 << 2,
 	MDS_PERM_BYPASS		= 1 << 3,
-	MDS_SOM			= 1 << 4,
+/*	MDS_SOM			= 1 << 4, obsolete since 2.8.0 */
 	MDS_QUOTA_IGNORE	= 1 << 5,
 	MDS_CLOSE_CLEANUP	= 1 << 6,
 	MDS_KEEP_ORPHAN		= 1 << 7,
@@ -2268,6 +2096,7 @@ enum mds_op_bias {
 	MDS_OWNEROVERRIDE	= 1 << 11,
 	MDS_HSM_RELEASE		= 1 << 12,
 	MDS_RENAME_MIGRATE	= BIT(13),
+	MDS_CLOSE_LAYOUT_SWAP   = BIT(14),
 };
 
 /* instance of mdt_reint_rec */
@@ -2456,8 +2285,6 @@ struct mdt_rec_reint {
 	__u32	   rr_padding_4; /* also fix lustre_swab_mdt_rec_reint */
 };
 
-void lustre_swab_mdt_rec_reint(struct mdt_rec_reint *rr);
-
 /* lmv structures */
 struct lmv_desc {
 	__u32 ld_tgt_count;		/* how many MDS's */
@@ -2547,8 +2374,6 @@ union lmv_mds_md {
 	struct lmv_user_md	lmv_user_md;
 };
 
-void lustre_swab_lmv_mds_md(union lmv_mds_md *lmm);
-
 static inline ssize_t lmv_mds_md_size(int stripe_count, unsigned int lmm_magic)
 {
 	ssize_t len = -EINVAL;
@@ -2652,8 +2477,6 @@ struct lov_desc {
 
 #define ld_magic ld_active_tgt_count       /* for swabbing from llogs */
 
-void lustre_swab_lov_desc(struct lov_desc *ld);
-
 /*
  *   LDLM requests:
  */
@@ -2749,24 +2572,38 @@ struct ldlm_flock_wire {
  * on the resource type.
  */
 
-typedef union {
+union ldlm_wire_policy_data {
 	struct ldlm_extent l_extent;
 	struct ldlm_flock_wire l_flock;
 	struct ldlm_inodebits l_inodebits;
-} ldlm_wire_policy_data_t;
+};
 
 union ldlm_gl_desc {
 	struct ldlm_gl_lquota_desc	lquota_desc;
 };
 
-void lustre_swab_gl_desc(union ldlm_gl_desc *);
+enum ldlm_intent_flags {
+	IT_OPEN		= BIT(0),
+	IT_CREAT	= BIT(1),
+	IT_OPEN_CREAT	= BIT(1) | BIT(0),
+	IT_READDIR	= BIT(2),
+	IT_GETATTR	= BIT(3),
+	IT_LOOKUP	= BIT(4),
+	IT_UNLINK	= BIT(5),
+	IT_TRUNC	= BIT(6),
+	IT_GETXATTR	= BIT(7),
+	IT_EXEC		= BIT(8),
+	IT_PIN		= BIT(9),
+	IT_LAYOUT	= BIT(10),
+	IT_QUOTA_DQACQ	= BIT(11),
+	IT_QUOTA_CONN	= BIT(12),
+	IT_SETXATTR	= BIT(13),
+};
 
 struct ldlm_intent {
 	__u64 opc;
 };
 
-void lustre_swab_ldlm_intent(struct ldlm_intent *i);
-
 struct ldlm_resource_desc {
 	enum ldlm_type lr_type;
 	__u32 lr_padding;       /* also fix lustre_swab_ldlm_resource_desc */
@@ -2777,7 +2614,7 @@ struct ldlm_lock_desc {
 	struct ldlm_resource_desc l_resource;
 	enum ldlm_mode l_req_mode;
 	enum ldlm_mode l_granted_mode;
-	ldlm_wire_policy_data_t l_policy_data;
+	union ldlm_wire_policy_data l_policy_data;
 };
 
 #define LDLM_LOCKREQ_HANDLES 2
@@ -2790,8 +2627,6 @@ struct ldlm_request {
 	struct lustre_handle lock_handle[LDLM_LOCKREQ_HANDLES];
 };
 
-void lustre_swab_ldlm_request(struct ldlm_request *rq);
-
 /* If LDLM_ENQUEUE, 1 slot is already occupied, 1 is available.
  * Otherwise, 2 are available.
  */
@@ -2813,8 +2648,6 @@ struct ldlm_reply {
 	__u64  lock_policy_res2;
 };
 
-void lustre_swab_ldlm_reply(struct ldlm_reply *r);
-
 #define ldlm_flags_to_wire(flags)    ((__u32)(flags))
 #define ldlm_flags_from_wire(flags)  ((__u64)(flags))
 
@@ -2858,8 +2691,6 @@ struct mgs_target_info {
 	char	     mti_params[MTI_PARAM_MAXLEN];
 };
 
-void lustre_swab_mgs_target_info(struct mgs_target_info *oinfo);
-
 struct mgs_nidtbl_entry {
 	__u64	   mne_version;    /* table version of this entry */
 	__u32	   mne_instance;   /* target instance # */
@@ -2874,8 +2705,6 @@ struct mgs_nidtbl_entry {
 	} u;
 };
 
-void lustre_swab_mgs_nidtbl_entry(struct mgs_nidtbl_entry *oinfo);
-
 struct mgs_config_body {
 	char     mcb_name[MTI_NAME_MAXLEN]; /* logname */
 	__u64    mcb_offset;    /* next index of config log to request */
@@ -2885,15 +2714,11 @@ struct mgs_config_body {
 	__u32    mcb_units;     /* # of units for bulk transfer */
 };
 
-void lustre_swab_mgs_config_body(struct mgs_config_body *body);
-
 struct mgs_config_res {
 	__u64    mcr_offset;    /* index of last config log */
 	__u64    mcr_size;      /* size of the log */
 };
 
-void lustre_swab_mgs_config_res(struct mgs_config_res *body);
-
 /* Config marker flags (in config log) */
 #define CM_START       0x01
 #define CM_END	 0x02
@@ -2913,8 +2738,6 @@ struct cfg_marker {
 	char	      cm_comment[MTI_NAME_MAXLEN];
 };
 
-void lustre_swab_cfg_marker(struct cfg_marker *marker, int swab, int size);
-
 /*
  * Opcodes for multiple servers.
  */
@@ -2922,7 +2745,7 @@ void lustre_swab_cfg_marker(struct cfg_marker *marker, int swab, int size);
 enum obd_cmd {
 	OBD_PING = 400,
 	OBD_LOG_CANCEL,
-	OBD_QC_CALLBACK,
+	OBD_QC_CALLBACK, /* not used since 2.4 */
 	OBD_IDX_READ,
 	OBD_LAST_OPC
 };
@@ -3155,23 +2978,32 @@ struct llog_gen_rec {
 	struct llog_rec_tail	lgr_tail;
 };
 
-/* On-disk header structure of each log object, stored in little endian order */
-#define LLOG_CHUNK_SIZE	 8192
-#define LLOG_HEADER_SIZE	(96)
-#define LLOG_BITMAP_BYTES       (LLOG_CHUNK_SIZE - LLOG_HEADER_SIZE)
-
-#define LLOG_MIN_REC_SIZE       (24) /* round(llog_rec_hdr + llog_rec_tail) */
-
 /* flags for the logs */
 enum llog_flag {
 	LLOG_F_ZAP_WHEN_EMPTY	= 0x1,
 	LLOG_F_IS_CAT		= 0x2,
 	LLOG_F_IS_PLAIN		= 0x4,
 	LLOG_F_EXT_JOBID        = BIT(3),
+	LLOG_F_IS_FIXSIZE	= BIT(4),
 
+	/*
+	 * Note: Flags covered by LLOG_F_EXT_MASK will be inherited from
+	 * catlog to plain log, so do not add LLOG_F_IS_FIXSIZE here,
+	 * because the catlog record is usually fixed size, but its plain
+	 * log record can be variable
+	 */
 	LLOG_F_EXT_MASK = LLOG_F_EXT_JOBID,
 };
 
+/* On-disk header structure of each log object, stored in little endian order */
+#define LLOG_MIN_CHUNK_SIZE	8192
+#define LLOG_HEADER_SIZE	(96)	/* sizeof (llog_log_hdr) +
+					 * sizeof(llh_tail) - sizeof(llh_bitmap)
+					 */
+#define LLOG_BITMAP_BYTES	(LLOG_MIN_CHUNK_SIZE - LLOG_HEADER_SIZE)
+#define LLOG_MIN_REC_SIZE	(24)	/* round(llog_rec_hdr + llog_rec_tail) */
+
+/* flags for the logs */
 struct llog_log_hdr {
 	struct llog_rec_hdr     llh_hdr;
 	__s64		   llh_timestamp;
@@ -3183,13 +3015,30 @@ struct llog_log_hdr {
 	/* for a catalog the first plain slot is next to it */
 	struct obd_uuid	 llh_tgtuuid;
 	__u32		   llh_reserved[LLOG_HEADER_SIZE / sizeof(__u32) - 23];
+	/* These fields must always be at the end of the llog_log_hdr.
+	 * Note: llh_bitmap size is variable because llog chunk size could be
+	 * bigger than LLOG_MIN_CHUNK_SIZE, i.e. sizeof(llog_log_hdr) > 8192
+	 * bytes, and the real size is stored in llh_hdr.lrh_len, which means
+	 * llh_tail should only be referred by LLOG_HDR_TAIL().
+	 * But this structure is also used by client/server llog interface
+	 * (see llog_client.c), it will be kept in its original way to avoid
+	 * compatibility issue.
+	 */
 	__u32		   llh_bitmap[LLOG_BITMAP_BYTES / sizeof(__u32)];
 	struct llog_rec_tail    llh_tail;
 } __packed;
 
-#define LLOG_BITMAP_SIZE(llh)  (__u32)((llh->llh_hdr.lrh_len -		\
-					llh->llh_bitmap_offset -	\
-					sizeof(llh->llh_tail)) * 8)
+#undef LLOG_HEADER_SIZE
+#undef LLOG_BITMAP_BYTES
+
+#define LLOG_HDR_BITMAP_SIZE(llh) (__u32)((llh->llh_hdr.lrh_len -	\
+					   llh->llh_bitmap_offset -	\
+					   sizeof(llh->llh_tail)) * 8)
+#define LLOG_HDR_BITMAP(llh)	(__u32 *)((char *)(llh) +		\
+					  (llh)->llh_bitmap_offset)
+#define LLOG_HDR_TAIL(llh)	((struct llog_rec_tail *)((char *)llh + \
+							 llh->llh_hdr.lrh_len - \
+							 sizeof(llh->llh_tail)))
 
 /** log cookies are used to reference a specific log file and a record
  * therein
@@ -3259,7 +3108,8 @@ struct obdo {
 	__u32		   o_parent_ver;
 	struct lustre_handle    o_handle;  /* brw: lock handle to prolong locks
 					    */
-	struct llog_cookie      o_lcookie; /* destroy: unlink cookie from MDS
+	struct llog_cookie      o_lcookie; /* destroy: unlink cookie from MDS,
+					    * obsolete in 2.8, reused in OSP
 					    */
 	__u32			o_uid_h;
 	__u32			o_gid_h;
@@ -3333,30 +3183,11 @@ struct ost_body {
 
 /* Key for FIEMAP to be used in get_info calls */
 struct ll_fiemap_info_key {
-	char    name[8];
-	struct  obdo oa;
-	struct  ll_user_fiemap fiemap;
+	char		lfik_name[8];
+	struct obdo	lfik_oa;
+	struct fiemap	lfik_fiemap;
 };
 
-void lustre_swab_ost_body(struct ost_body *b);
-void lustre_swab_ost_last_id(__u64 *id);
-void lustre_swab_fiemap(struct ll_user_fiemap *fiemap);
-
-void lustre_swab_lov_user_md_v1(struct lov_user_md_v1 *lum);
-void lustre_swab_lov_user_md_v3(struct lov_user_md_v3 *lum);
-void lustre_swab_lov_user_md_objects(struct lov_user_ost_data *lod,
-				     int stripe_count);
-void lustre_swab_lov_mds_md(struct lov_mds_md *lmm);
-
-/* llog_swab.c */
-void lustre_swab_llogd_body(struct llogd_body *d);
-void lustre_swab_llog_hdr(struct llog_log_hdr *h);
-void lustre_swab_llogd_conn_body(struct llogd_conn_body *d);
-void lustre_swab_llog_rec(struct llog_rec_hdr *rec);
-
-struct lustre_cfg;
-void lustre_swab_lustre_cfg(struct lustre_cfg *lcfg);
-
 /* Functions for dumping PTLRPC fields */
 void dump_rniobuf(struct niobuf_remote *rnb);
 void dump_ioo(struct obd_ioobj *nb);
@@ -3394,8 +3225,6 @@ struct lustre_capa {
 	__u8	    lc_hmac[CAPA_HMAC_MAX_LEN];   /** HMAC */
 } __packed;
 
-void lustre_swab_lustre_capa(struct lustre_capa *c);
-
 /** lustre_capa::lc_opc */
 enum {
 	CAPA_OPC_BODY_WRITE   = 1 << 0,  /**< write object data */
@@ -3458,8 +3287,6 @@ struct getinfo_fid2path {
 	char	    gf_path[0];
 } __packed;
 
-void lustre_swab_fid2path(struct getinfo_fid2path *gf);
-
 /** path2parent request/reply structures */
 struct getparent {
 	struct lu_fid	gp_fid;		/**< parent FID */
@@ -3486,8 +3313,6 @@ struct layout_intent {
 	__u64 li_end;
 };
 
-void lustre_swab_layout_intent(struct layout_intent *li);
-
 /**
  * On the wire version of hsm_progress structure.
  *
@@ -3506,13 +3331,6 @@ struct hsm_progress_kernel {
 	__u64			hpk_padding2;
 } __packed;
 
-void lustre_swab_hsm_user_state(struct hsm_user_state *hus);
-void lustre_swab_hsm_current_action(struct hsm_current_action *action);
-void lustre_swab_hsm_progress_kernel(struct hsm_progress_kernel *hpk);
-void lustre_swab_hsm_user_state(struct hsm_user_state *hus);
-void lustre_swab_hsm_user_item(struct hsm_user_item *hui);
-void lustre_swab_hsm_request(struct hsm_request *hr);
-
 /** layout swap request structure
  * fid1 and fid2 are in mdt_body
  */
@@ -3520,8 +3338,6 @@ struct mdc_swap_layouts {
 	__u64	   msl_flags;
 } __packed;
 
-void lustre_swab_swap_layouts(struct mdc_swap_layouts *msl);
-
 struct close_data {
 	struct lustre_handle	cd_handle;
 	struct lu_fid		cd_fid;
@@ -3529,7 +3345,5 @@ struct close_data {
 	__u64			cd_reserved[8];
 };
 
-void lustre_swab_close_data(struct close_data *data);
-
 #endif
 /** @} lustreidl */
diff --git a/drivers/staging/lustre/lustre/include/lustre/lustre_ioctl.h b/drivers/staging/lustre/lustre/include/lustre/lustre_ioctl.h
index f3d7c94c3b50..eb08df33b2db 100644
--- a/drivers/staging/lustre/lustre/include/lustre/lustre_ioctl.h
+++ b/drivers/staging/lustre/lustre/include/lustre/lustre_ioctl.h
@@ -363,8 +363,8 @@ obd_ioctl_unpack(struct obd_ioctl_data *data, char *pbuf, int max_len)
 /*	OBD_IOC_LOV_GETSTRIPE	155 LL_IOC_LOV_GETSTRIPE */
 /*	OBD_IOC_LOV_SETEA	156 LL_IOC_LOV_SETEA */
 /*	lustre/lustre_user.h	157-159 */
-#define	OBD_IOC_QUOTACHECK	_IOW('f', 160, int)
-#define	OBD_IOC_POLL_QUOTACHECK	_IOR('f', 161, struct if_quotacheck *)
+/*	OBD_IOC_QUOTACHECK	_IOW('f', 160, int) */
+/*	OBD_IOC_POLL_QUOTACHECK	_IOR('f', 161, struct if_quotacheck *) */
 #define OBD_IOC_QUOTACTL	_IOWR('f', 162, struct if_quotactl)
 /*	lustre/lustre_user.h	163-176 */
 #define OBD_IOC_CHANGELOG_REG	_IOW('f', 177, struct obd_ioctl_data)
diff --git a/drivers/staging/lustre/lustre/include/lustre/lustre_user.h b/drivers/staging/lustre/lustre/include/lustre/lustre_user.h
index 6fc985571cba..3301ad652db1 100644
--- a/drivers/staging/lustre/lustre/include/lustre/lustre_user.h
+++ b/drivers/staging/lustre/lustre/include/lustre/lustre_user.h
@@ -63,9 +63,13 @@
 #if __BITS_PER_LONG != 64 || defined(__ARCH_WANT_STAT64)
 typedef struct stat64   lstat_t;
 #define lstat_f  lstat64
+#define fstat_f		fstat64
+#define fstatat_f	fstatat64
 #else
 typedef struct stat     lstat_t;
 #define lstat_f  lstat
+#define fstat_f		fstat
+#define fstatat_f	fstatat
 #endif
 
 #define HAVE_LOV_USER_MDS_DATA
@@ -82,7 +86,6 @@ typedef struct stat     lstat_t;
 #define FSFILT_IOC_SETVERSION	     _IOW('f', 4, long)
 #define FSFILT_IOC_GETVERSION_OLD	 _IOR('v', 1, long)
 #define FSFILT_IOC_SETVERSION_OLD	 _IOW('v', 2, long)
-#define FSFILT_IOC_FIEMAP		 _IOWR('f', 11, struct ll_user_fiemap)
 #endif
 
 /* FIEMAP flags supported by Lustre */
@@ -235,7 +238,7 @@ struct ost_id {
 /* #define LL_IOC_POLL_QUOTACHECK	161 OBD_IOC_POLL_QUOTACHECK */
 /* #define LL_IOC_QUOTACTL		162 OBD_IOC_QUOTACTL */
 #define IOC_OBD_STATFS		  _IOWR('f', 164, struct obd_statfs *)
-#define IOC_LOV_GETINFO		 _IOWR('f', 165, struct lov_user_mds_data *)
+/*	IOC_LOV_GETINFO			165 obsolete */
 #define LL_IOC_FLUSHCTX		 _IOW('f', 166, long)
 /* LL_IOC_RMTACL			167 obsolete */
 #define LL_IOC_GETOBDCOUNT	      _IOR('f', 168, long)
@@ -343,6 +346,9 @@ enum ll_lease_type {
 #define LOV_ALL_STRIPES       0xffff /* only valid for directories */
 #define LOV_V1_INSANE_STRIPE_COUNT 65532 /* maximum stripe count bz13933 */
 
+#define XATTR_LUSTRE_PREFIX	"lustre."
+#define XATTR_LUSTRE_LOV	"lustre.lov"
+
 #define lov_user_ost_data lov_user_ost_data_v1
 struct lov_user_ost_data_v1 {     /* per-stripe data structure */
 	struct ost_id l_ost_oi;	  /* OST object ID */
@@ -451,8 +457,6 @@ static inline int lmv_user_md_size(int stripes, int lmm_magic)
 		      stripes * sizeof(struct lmv_user_mds_data);
 }
 
-void lustre_swab_lmv_user_md(struct lmv_user_md *lum);
-
 struct ll_recreate_obj {
 	__u64 lrc_id;
 	__u32 lrc_ost_idx;
@@ -522,25 +526,20 @@ static inline void obd_uuid2fsname(char *buf, char *uuid, int buflen)
 }
 
 /* printf display format
- * e.g. printf("file FID is "DFID"\n", PFID(fid));
+ * * usage: printf("file FID is "DFID"\n", PFID(fid));
  */
 #define FID_NOBRACE_LEN 40
 #define FID_LEN (FID_NOBRACE_LEN + 2)
 #define DFID_NOBRACE "%#llx:0x%x:0x%x"
 #define DFID "["DFID_NOBRACE"]"
-#define PFID(fid)     \
-	(fid)->f_seq, \
-	(fid)->f_oid, \
-	(fid)->f_ver
+#define PFID(fid) (unsigned long long)(fid)->f_seq, (fid)->f_oid, (fid)->f_ver
 
-/* scanf input parse format -- strip '[' first.
- * e.g. sscanf(fidstr, SFID, RFID(&fid));
+/* scanf input parse format for fids in DFID_NOBRACE format
+ * Need to strip '[' from DFID format first or use "["SFID"]" at caller.
+ * usage: sscanf(fidstr, SFID, RFID(&fid));
  */
 #define SFID "0x%llx:0x%x:0x%x"
-#define RFID(fid)     \
-	&((fid)->f_seq), \
-	&((fid)->f_oid), \
-	&((fid)->f_ver)
+#define RFID(fid) &((fid)->f_seq), &((fid)->f_oid), &((fid)->f_ver)
 
 /********* Quotas **********/
 
@@ -551,23 +550,18 @@ static inline void obd_uuid2fsname(char *buf, char *uuid, int buflen)
 #define Q_FINVALIDATE  0x800104 /* deprecated as of 2.4 */
 
 /* these must be explicitly translated into linux Q_* in ll_dir_ioctl */
-#define LUSTRE_Q_QUOTAON    0x800002     /* turn quotas on */
-#define LUSTRE_Q_QUOTAOFF   0x800003     /* turn quotas off */
+#define LUSTRE_Q_QUOTAON    0x800002	/* deprecated as of 2.4 */
+#define LUSTRE_Q_QUOTAOFF   0x800003	/* deprecated as of 2.4 */
 #define LUSTRE_Q_GETINFO    0x800005     /* get information about quota files */
 #define LUSTRE_Q_SETINFO    0x800006     /* set information about quota files */
 #define LUSTRE_Q_GETQUOTA   0x800007     /* get user quota structure */
 #define LUSTRE_Q_SETQUOTA   0x800008     /* set user quota structure */
 /* lustre-specific control commands */
-#define LUSTRE_Q_INVALIDATE  0x80000b     /* invalidate quota data */
-#define LUSTRE_Q_FINVALIDATE 0x80000c     /* invalidate filter quota data */
+#define LUSTRE_Q_INVALIDATE  0x80000b	/* deprecated as of 2.4 */
+#define LUSTRE_Q_FINVALIDATE 0x80000c	/* deprecated as of 2.4 */
 
 #define UGQUOTA 2       /* set both USRQUOTA and GRPQUOTA */
 
-struct if_quotacheck {
-	char		    obd_type[16];
-	struct obd_uuid	 obd_uuid;
-};
-
 #define IDENTITY_DOWNCALL_MAGIC 0x6d6dd629
 
 /* permission */
@@ -649,6 +643,7 @@ struct if_quotactl {
 #define SWAP_LAYOUTS_CHECK_DV2		(1 << 1)
 #define SWAP_LAYOUTS_KEEP_MTIME		(1 << 2)
 #define SWAP_LAYOUTS_KEEP_ATIME		(1 << 3)
+#define SWAP_LAYOUTS_CLOSE		BIT(4)
 
 /* Swap XATTR_NAME_HSM as well, only on the MDT so far */
 #define SWAP_LAYOUTS_MDS_HSM		(1 << 31)
@@ -999,6 +994,7 @@ struct ioc_data_version {
  * See HSM_FLAGS below.
  */
 enum hsm_states {
+	HS_NONE		= 0x00000000,
 	HS_EXISTS	= 0x00000001,
 	HS_DIRTY	= 0x00000002,
 	HS_RELEASED	= 0x00000004,
diff --git a/drivers/staging/lustre/lustre/include/lustre_compat.h b/drivers/staging/lustre/lustre/include/lustre_compat.h
index 567c438e93cb..300e96fb032a 100644
--- a/drivers/staging/lustre/lustre/include/lustre_compat.h
+++ b/drivers/staging/lustre/lustre/include/lustre_compat.h
@@ -74,4 +74,6 @@
 # define ext2_find_next_zero_bit  find_next_zero_bit_le
 #endif
 
+#define TIMES_SET_FLAGS (ATTR_MTIME_SET | ATTR_ATIME_SET | ATTR_TIMES_SET)
+
 #endif /* _LUSTRE_COMPAT_H */
diff --git a/drivers/staging/lustre/lustre/include/lustre_dlm.h b/drivers/staging/lustre/lustre/include/lustre_dlm.h
index d03534432624..b7e61d082e55 100644
--- a/drivers/staging/lustre/lustre/include/lustre_dlm.h
+++ b/drivers/staging/lustre/lustre/include/lustre_dlm.h
@@ -59,7 +59,7 @@ struct obd_device;
 #define OBD_LDLM_DEVICENAME  "ldlm"
 
 #define LDLM_DEFAULT_LRU_SIZE (100 * num_online_cpus())
-#define LDLM_DEFAULT_MAX_ALIVE (cfs_time_seconds(36000))
+#define LDLM_DEFAULT_MAX_ALIVE (cfs_time_seconds(3900)) /* 65 min */
 #define LDLM_DEFAULT_PARALLEL_AST_LIMIT 1024
 
 /**
@@ -86,10 +86,10 @@ enum ldlm_error {
  * decisions about lack of conflicts or do any autonomous lock granting without
  * first speaking to a server.
  */
-typedef enum {
+enum ldlm_side {
 	LDLM_NAMESPACE_SERVER = 1 << 0,
 	LDLM_NAMESPACE_CLIENT = 1 << 1
-} ldlm_side_t;
+};
 
 /**
  * The blocking callback is overloaded to perform two functions.  These flags
@@ -359,7 +359,7 @@ struct ldlm_namespace {
 	struct obd_device	*ns_obd;
 
 	/** Flag indicating if namespace is on client instead of server */
-	ldlm_side_t		ns_client;
+	enum ldlm_side		ns_client;
 
 	/** Resource hash table for namespace. */
 	struct cfs_hash		*ns_rs_hash;
@@ -550,20 +550,18 @@ struct ldlm_flock {
 	__u64 owner;
 	__u64 blocking_owner;
 	struct obd_export *blocking_export;
-	/* Protected by the hash lock */
-	__u32 blocking_refs;
 	__u32 pid;
 };
 
-typedef union {
+union ldlm_policy_data {
 	struct ldlm_extent l_extent;
 	struct ldlm_flock l_flock;
 	struct ldlm_inodebits l_inodebits;
-} ldlm_policy_data_t;
+};
 
 void ldlm_convert_policy_to_local(struct obd_export *exp, enum ldlm_type type,
-				  const ldlm_wire_policy_data_t *wpolicy,
-				  ldlm_policy_data_t *lpolicy);
+				  const union ldlm_wire_policy_data *wpolicy,
+				  union ldlm_policy_data *lpolicy);
 
 enum lvb_type {
 	LVB_T_NONE	= 0,
@@ -692,7 +690,7 @@ struct ldlm_lock {
 	 * Representation of private data specific for a lock type.
 	 * Examples are: extent range for extent lock or bitmask for ibits locks
 	 */
-	ldlm_policy_data_t	l_policy_data;
+	union ldlm_policy_data	l_policy_data;
 
 	/**
 	 * Lock state flags. Protected by lr_lock.
@@ -967,8 +965,8 @@ struct ldlm_ast_work {
  * Common ldlm_enqueue parameters
  */
 struct ldlm_enqueue_info {
-	__u32 ei_type;   /** Type of the lock being enqueued. */
-	__u32 ei_mode;   /** Mode of the lock being enqueued. */
+	enum ldlm_type	ei_type;  /** Type of the lock being enqueued. */
+	enum ldlm_mode	ei_mode;  /** Mode of the lock being enqueued. */
 	void *ei_cb_bl;  /** blocking lock callback */
 	void *ei_cb_cp;  /** lock completion callback */
 	void *ei_cb_gl;  /** lock glimpse callback */
@@ -979,7 +977,7 @@ struct ldlm_enqueue_info {
 extern struct obd_ops ldlm_obd_ops;
 
 extern char *ldlm_lockname[];
-char *ldlm_it2str(int it);
+const char *ldlm_it2str(enum ldlm_intent_flags it);
 
 /**
  * Just a fancy CDEBUG call with log level preset to LDLM_DEBUG.
@@ -1168,16 +1166,18 @@ do {					    \
 struct ldlm_lock *ldlm_lock_get(struct ldlm_lock *lock);
 void ldlm_lock_put(struct ldlm_lock *lock);
 void ldlm_lock2desc(struct ldlm_lock *lock, struct ldlm_lock_desc *desc);
-void ldlm_lock_addref(const struct lustre_handle *lockh, __u32 mode);
-int  ldlm_lock_addref_try(const struct lustre_handle *lockh, __u32 mode);
-void ldlm_lock_decref(const struct lustre_handle *lockh, __u32 mode);
-void ldlm_lock_decref_and_cancel(const struct lustre_handle *lockh, __u32 mode);
+void ldlm_lock_addref(const struct lustre_handle *lockh, enum ldlm_mode mode);
+int  ldlm_lock_addref_try(const struct lustre_handle *lockh,
+			  enum ldlm_mode mode);
+void ldlm_lock_decref(const struct lustre_handle *lockh, enum ldlm_mode mode);
+void ldlm_lock_decref_and_cancel(const struct lustre_handle *lockh,
+				 enum ldlm_mode mode);
 void ldlm_lock_fail_match_locked(struct ldlm_lock *lock);
 void ldlm_lock_allow_match(struct ldlm_lock *lock);
 void ldlm_lock_allow_match_locked(struct ldlm_lock *lock);
 enum ldlm_mode ldlm_lock_match(struct ldlm_namespace *ns, __u64 flags,
 			       const struct ldlm_res_id *,
-			       enum ldlm_type type, ldlm_policy_data_t *,
+			       enum ldlm_type type, union ldlm_policy_data *,
 			       enum ldlm_mode mode, struct lustre_handle *,
 			       int unref);
 enum ldlm_mode ldlm_revalidate_lock_handle(const struct lustre_handle *lockh,
@@ -1189,7 +1189,7 @@ void ldlm_unlink_lock_skiplist(struct ldlm_lock *req);
 /* resource.c */
 struct ldlm_namespace *
 ldlm_namespace_new(struct obd_device *obd, char *name,
-		   ldlm_side_t client, enum ldlm_appetite apt,
+		   enum ldlm_side client, enum ldlm_appetite apt,
 		   enum ldlm_ns_type ns_type);
 int ldlm_namespace_cleanup(struct ldlm_namespace *ns, __u64 flags);
 void ldlm_namespace_get(struct ldlm_namespace *ns);
@@ -1208,7 +1208,7 @@ void ldlm_resource_add_lock(struct ldlm_resource *res,
 			    struct ldlm_lock *lock);
 void ldlm_resource_unlink_lock(struct ldlm_lock *lock);
 void ldlm_res2desc(struct ldlm_resource *res, struct ldlm_resource_desc *desc);
-void ldlm_dump_all_namespaces(ldlm_side_t client, int level);
+void ldlm_dump_all_namespaces(enum ldlm_side client, int level);
 void ldlm_namespace_dump(int level, struct ldlm_namespace *);
 void ldlm_resource_dump(int level, struct ldlm_resource *);
 int ldlm_lock_change_resource(struct ldlm_namespace *, struct ldlm_lock *,
@@ -1241,7 +1241,7 @@ int ldlm_completion_ast(struct ldlm_lock *lock, __u64 flags, void *data);
 int ldlm_cli_enqueue(struct obd_export *exp, struct ptlrpc_request **reqp,
 		     struct ldlm_enqueue_info *einfo,
 		     const struct ldlm_res_id *res_id,
-		     ldlm_policy_data_t const *policy, __u64 *flags,
+		     union ldlm_policy_data const *policy, __u64 *flags,
 		     void *lvb, __u32 lvb_len, enum lvb_type lvb_type,
 		     struct lustre_handle *lockh, int async);
 int ldlm_prep_enqueue_req(struct obd_export *exp,
@@ -1265,13 +1265,13 @@ int ldlm_cli_cancel_unused(struct ldlm_namespace *, const struct ldlm_res_id *,
 			   enum ldlm_cancel_flags flags, void *opaque);
 int ldlm_cli_cancel_unused_resource(struct ldlm_namespace *ns,
 				    const struct ldlm_res_id *res_id,
-				    ldlm_policy_data_t *policy,
+				    union ldlm_policy_data *policy,
 				    enum ldlm_mode mode,
 				    enum ldlm_cancel_flags flags,
 				    void *opaque);
 int ldlm_cancel_resource_local(struct ldlm_resource *res,
 			       struct list_head *cancels,
-			       ldlm_policy_data_t *policy,
+			       union ldlm_policy_data *policy,
 			       enum ldlm_mode mode, __u64 lock_flags,
 			       enum ldlm_cancel_flags cancel_flags,
 			       void *opaque);
@@ -1333,7 +1333,7 @@ int ldlm_pools_init(void);
 void ldlm_pools_fini(void);
 
 int ldlm_pool_init(struct ldlm_pool *pl, struct ldlm_namespace *ns,
-		   int idx, ldlm_side_t client);
+		   int idx, enum ldlm_side client);
 void ldlm_pool_fini(struct ldlm_pool *pl);
 void ldlm_pool_add(struct ldlm_pool *pl, struct ldlm_lock *lock);
 void ldlm_pool_del(struct ldlm_pool *pl, struct ldlm_lock *lock);
diff --git a/drivers/staging/lustre/lustre/include/lustre_fid.h b/drivers/staging/lustre/lustre/include/lustre_fid.h
index 316780693193..b5a1aadbcb93 100644
--- a/drivers/staging/lustre/lustre/include/lustre_fid.h
+++ b/drivers/staging/lustre/lustre/include/lustre_fid.h
@@ -150,6 +150,7 @@
 
 #include "../../include/linux/libcfs/libcfs.h"
 #include "lustre/lustre_idl.h"
+#include "seq_range.h"
 
 struct lu_env;
 struct lu_site;
diff --git a/drivers/staging/lustre/lustre/include/lustre_fld.h b/drivers/staging/lustre/lustre/include/lustre_fld.h
index 932410d3e3cc..6ef1b03cb986 100644
--- a/drivers/staging/lustre/lustre/include/lustre_fld.h
+++ b/drivers/staging/lustre/lustre/include/lustre_fld.h
@@ -103,8 +103,6 @@ struct lu_client_fld {
 
 	/** Client fld debugfs entry name. */
 	char			 lcf_name[LUSTRE_MDT_MAXNAMELEN];
-
-	int			 lcf_flags;
 };
 
 /* Client methods */
diff --git a/drivers/staging/lustre/lustre/include/lustre_ha.h b/drivers/staging/lustre/lustre/include/lustre_ha.h
index cde7ed702c86..dec1e99d594d 100644
--- a/drivers/staging/lustre/lustre/include/lustre_ha.h
+++ b/drivers/staging/lustre/lustre/include/lustre_ha.h
@@ -53,6 +53,7 @@ void ptlrpc_activate_import(struct obd_import *imp);
 void ptlrpc_deactivate_import(struct obd_import *imp);
 void ptlrpc_invalidate_import(struct obd_import *imp);
 void ptlrpc_fail_import(struct obd_import *imp, __u32 conn_cnt);
+void ptlrpc_pinger_force(struct obd_import *imp);
 
 /** @} ha */
 
diff --git a/drivers/staging/lustre/lustre/include/lustre_import.h b/drivers/staging/lustre/lustre/include/lustre_import.h
index 5461ba33d90c..f0c931ce1a67 100644
--- a/drivers/staging/lustre/lustre/include/lustre_import.h
+++ b/drivers/staging/lustre/lustre/include/lustre_import.h
@@ -185,6 +185,11 @@ struct obd_import {
 	struct list_head	       *imp_replay_cursor;
 	/** @} */
 
+	/** List of not replied requests */
+	struct list_head	imp_unreplied_list;
+	/** Known maximal replied XID */
+	__u64			imp_known_replied_xid;
+
 	/** obd device for this import */
 	struct obd_device	*imp_obd;
 
@@ -294,7 +299,9 @@ struct obd_import {
 				   */
 				  imp_force_reconnect:1,
 				  /* import has tried to connect with server */
-				  imp_connect_tried:1;
+				  imp_connect_tried:1,
+				 /* connected but not FULL yet */
+				 imp_connected:1;
 	__u32		     imp_connect_op;
 	struct obd_connect_data   imp_connect_data;
 	__u64		     imp_connect_flags_orig;
diff --git a/drivers/staging/lustre/lustre/include/lustre_lib.h b/drivers/staging/lustre/lustre/include/lustre_lib.h
index 6b231913ba2e..27f3148c4344 100644
--- a/drivers/staging/lustre/lustre/include/lustre_lib.h
+++ b/drivers/staging/lustre/lustre/include/lustre_lib.h
@@ -350,8 +350,6 @@ do {									   \
 	l_wait_event_exclusive_head(wq, condition, &lwi);       \
 })
 
-#define LIBLUSTRE_CLIENT (0)
-
 /** @} lib */
 
 #endif /* _LUSTRE_LIB_H */
diff --git a/drivers/staging/lustre/lustre/include/lustre_lmv.h b/drivers/staging/lustre/lustre/include/lustre_lmv.h
index d7f7afa8dfa7..5aa3645e64dc 100644
--- a/drivers/staging/lustre/lustre/include/lustre_lmv.h
+++ b/drivers/staging/lustre/lustre/include/lustre_lmv.h
@@ -76,18 +76,7 @@ lsm_md_eq(const struct lmv_stripe_md *lsm1, const struct lmv_stripe_md *lsm2)
 
 union lmv_mds_md;
 
-int lmv_unpack_md(struct obd_export *exp, struct lmv_stripe_md **lsmp,
-		  const union lmv_mds_md *lmm, int stripe_count);
-
-static inline int lmv_alloc_memmd(struct lmv_stripe_md **lsmp, int stripe_count)
-{
-	return lmv_unpack_md(NULL, lsmp, NULL, stripe_count);
-}
-
-static inline void lmv_free_memmd(struct lmv_stripe_md *lsm)
-{
-	lmv_unpack_md(NULL, &lsm, NULL, 0);
-}
+void lmv_free_memmd(struct lmv_stripe_md *lsm);
 
 static inline void lmv1_le_to_cpu(struct lmv_mds_md_v1 *lmv_dst,
 				  const struct lmv_mds_md_v1 *lmv_src)
diff --git a/drivers/staging/lustre/lustre/include/lustre_log.h b/drivers/staging/lustre/lustre/include/lustre_log.h
index 995b266932e3..35e37eb1bc2c 100644
--- a/drivers/staging/lustre/lustre/include/lustre_log.h
+++ b/drivers/staging/lustre/lustre/include/lustre_log.h
@@ -214,6 +214,7 @@ struct llog_handle {
 	spinlock_t		 lgh_hdr_lock; /* protect lgh_hdr data */
 	struct llog_logid	 lgh_id; /* id of this log */
 	struct llog_log_hdr	*lgh_hdr;
+	size_t			 lgh_hdr_size;
 	int			 lgh_last_idx;
 	int			 lgh_cur_idx; /* used during llog_process */
 	__u64			 lgh_cur_offset; /* used during llog_process */
@@ -244,6 +245,11 @@ struct llog_ctxt {
 	struct mutex		 loc_mutex; /* protect loc_imp */
 	atomic_t	     loc_refcount;
 	long		     loc_flags; /* flags, see above defines */
+	/*
+	 * llog chunk size, and llog record size can not be bigger than
+	 * loc_chunk_size
+	 */
+	__u32			loc_chunk_size;
 };
 
 #define LLOG_PROC_BREAK 0x0001
diff --git a/drivers/staging/lustre/lustre/include/lustre_mdc.h b/drivers/staging/lustre/lustre/include/lustre_mdc.h
index 8fc2d3f2dfd6..198ceb0c66f9 100644
--- a/drivers/staging/lustre/lustre/include/lustre_mdc.h
+++ b/drivers/staging/lustre/lustre/include/lustre_mdc.h
@@ -156,16 +156,39 @@ static inline void mdc_put_rpc_lock(struct mdc_rpc_lock *lck,
 	mutex_unlock(&lck->rpcl_mutex);
 }
 
+static inline void mdc_get_mod_rpc_slot(struct ptlrpc_request *req,
+					struct lookup_intent *it)
+{
+	struct client_obd *cli = &req->rq_import->imp_obd->u.cli;
+	u32 opc;
+	u16 tag;
+
+	opc = lustre_msg_get_opc(req->rq_reqmsg);
+	tag = obd_get_mod_rpc_slot(cli, opc, it);
+	lustre_msg_set_tag(req->rq_reqmsg, tag);
+}
+
+static inline void mdc_put_mod_rpc_slot(struct ptlrpc_request *req,
+					struct lookup_intent *it)
+{
+	struct client_obd *cli = &req->rq_import->imp_obd->u.cli;
+	u32 opc;
+	u16 tag;
+
+	opc = lustre_msg_get_opc(req->rq_reqmsg);
+	tag = lustre_msg_get_tag(req->rq_reqmsg);
+	obd_put_mod_rpc_slot(cli, opc, it, tag);
+}
+
 /**
- * Update the maximum possible easize and cookiesize.
+ * Update the maximum possible easize.
  *
- * The values are learned from ptlrpc replies sent by the MDT.  The
- * default easize and cookiesize is initialized to the minimum value but
- * allowed to grow up to a single page in size if required to handle the
+ * This value is learned from ptlrpc replies sent by the MDT. The
+ * default easize is initialized to the minimum value but allowed
+ * to grow up to a single page in size if required to handle the
  * common case.
  *
- * \see client_obd::cl_default_mds_easize and
- * client_obd::cl_default_mds_cookiesize
+ * \see client_obd::cl_default_mds_easize
  *
  * \param[in] exp	export for MDC device
  * \param[in] body	body of ptlrpc reply from MDT
@@ -176,7 +199,7 @@ static inline void mdc_update_max_ea_from_body(struct obd_export *exp,
 {
 	if (body->mbo_valid & OBD_MD_FLMODEASIZE) {
 		struct client_obd *cli = &exp->exp_obd->u.cli;
-		u32 def_cookiesize, def_easize;
+		u32 def_easize;
 
 		if (cli->cl_max_mds_easize < body->mbo_max_mdsize)
 			cli->cl_max_mds_easize = body->mbo_max_mdsize;
@@ -184,13 +207,6 @@ static inline void mdc_update_max_ea_from_body(struct obd_export *exp,
 		def_easize = min_t(__u32, body->mbo_max_mdsize,
 				   OBD_MAX_DEFAULT_EA_SIZE);
 		cli->cl_default_mds_easize = def_easize;
-
-		if (cli->cl_max_mds_cookiesize < body->mbo_max_cookiesize)
-			cli->cl_max_mds_cookiesize = body->mbo_max_cookiesize;
-
-		def_cookiesize = min_t(__u32, body->mbo_max_cookiesize,
-				       OBD_MAX_DEFAULT_COOKIE_SIZE);
-		cli->cl_default_mds_cookiesize = def_cookiesize;
 	}
 }
 
diff --git a/drivers/staging/lustre/lustre/include/lustre_net.h b/drivers/staging/lustre/lustre/include/lustre_net.h
index e9aba99ee52a..411eb0dc7f38 100644
--- a/drivers/staging/lustre/lustre/include/lustre_net.h
+++ b/drivers/staging/lustre/lustre/include/lustre_net.h
@@ -50,6 +50,7 @@
  * @{
  */
 
+#include <linux/uio.h>
 #include "../../include/linux/libcfs/libcfs.h"
 #include "../../include/linux/lnet/nidstr.h"
 #include "../../include/linux/lnet/api.h"
@@ -68,13 +69,17 @@
 #define PTLRPC_MD_OPTIONS  0
 
 /**
- * Max # of bulk operations in one request.
+ * log2 max # of bulk operations in one request: 2=4MB/RPC, 5=32MB/RPC, ...
  * In order for the client and server to properly negotiate the maximum
  * possible transfer size, PTLRPC_BULK_OPS_COUNT must be a power-of-two
  * value.  The client is free to limit the actual RPC size for any bulk
  * transfer via cl_max_pages_per_rpc to some non-power-of-two value.
+ * NOTE: This is limited to 16 (=64GB RPCs) by IOOBJ_MAX_BRW_BITS.
  */
-#define PTLRPC_BULK_OPS_BITS	2
+#define PTLRPC_BULK_OPS_BITS	4
+#if PTLRPC_BULK_OPS_BITS > 16
+#error "More than 65536 BRW RPCs not allowed by IOOBJ_MAX_BRW_BITS."
+#endif
 #define PTLRPC_BULK_OPS_COUNT	(1U << PTLRPC_BULK_OPS_BITS)
 /**
  * PTLRPC_BULK_OPS_MASK is for the convenience of the client only, and
@@ -437,6 +442,10 @@ struct ptlrpc_reply_state {
 	unsigned long	  rs_committed:1;/* the transaction was committed
 					  * and the rs was dispatched
 					  */
+	atomic_t		rs_refcount;	/* number of users */
+	/** Number of locks awaiting client ACK */
+	int			rs_nlocks;
+
 	/** Size of the state */
 	int		    rs_size;
 	/** opcode */
@@ -449,7 +458,6 @@ struct ptlrpc_reply_state {
 	struct ptlrpc_service_part *rs_svcpt;
 	/** Lnet metadata handle for the reply */
 	lnet_handle_md_t       rs_md_h;
-	atomic_t	   rs_refcount;
 
 	/** Context for the service thread */
 	struct ptlrpc_svc_ctx *rs_svc_ctx;
@@ -466,8 +474,6 @@ struct ptlrpc_reply_state {
 	 */
 	struct lustre_msg     *rs_msg;	  /* reply message */
 
-	/** Number of locks awaiting client ACK */
-	int		    rs_nlocks;
 	/** Handles of locks awaiting client reply ACK */
 	struct lustre_handle   rs_locks[RS_MAX_LOCKS];
 	/** Lock modes of locks in \a rs_locks */
@@ -515,717 +521,7 @@ struct lu_env;
 
 struct ldlm_lock;
 
-/**
- * \defgroup nrs Network Request Scheduler
- * @{
- */
-struct ptlrpc_nrs_policy;
-struct ptlrpc_nrs_resource;
-struct ptlrpc_nrs_request;
-
-/**
- * NRS control operations.
- *
- * These are common for all policies.
- */
-enum ptlrpc_nrs_ctl {
-	/**
-	 * Not a valid opcode.
-	 */
-	PTLRPC_NRS_CTL_INVALID,
-	/**
-	 * Activate the policy.
-	 */
-	PTLRPC_NRS_CTL_START,
-	/**
-	 * Reserved for multiple primary policies, which may be a possibility
-	 * in the future.
-	 */
-	PTLRPC_NRS_CTL_STOP,
-	/**
-	 * Policies can start using opcodes from this value and onwards for
-	 * their own purposes; the assigned value itself is arbitrary.
-	 */
-	PTLRPC_NRS_CTL_1ST_POL_SPEC = 0x20,
-};
-
-/**
- * ORR policy operations
- */
-enum nrs_ctl_orr {
-	NRS_CTL_ORR_RD_QUANTUM = PTLRPC_NRS_CTL_1ST_POL_SPEC,
-	NRS_CTL_ORR_WR_QUANTUM,
-	NRS_CTL_ORR_RD_OFF_TYPE,
-	NRS_CTL_ORR_WR_OFF_TYPE,
-	NRS_CTL_ORR_RD_SUPP_REQ,
-	NRS_CTL_ORR_WR_SUPP_REQ,
-};
-
-/**
- * NRS policy operations.
- *
- * These determine the behaviour of a policy, and are called in response to
- * NRS core events.
- */
-struct ptlrpc_nrs_pol_ops {
-	/**
-	 * Called during policy registration; this operation is optional.
-	 *
-	 * \param[in,out] policy The policy being initialized
-	 */
-	int	(*op_policy_init)(struct ptlrpc_nrs_policy *policy);
-	/**
-	 * Called during policy unregistration; this operation is optional.
-	 *
-	 * \param[in,out] policy The policy being unregistered/finalized
-	 */
-	void	(*op_policy_fini)(struct ptlrpc_nrs_policy *policy);
-	/**
-	 * Called when activating a policy via lprocfs; policies allocate and
-	 * initialize their resources here; this operation is optional.
-	 *
-	 * \param[in,out] policy The policy being started
-	 *
-	 * \see nrs_policy_start_locked()
-	 */
-	int	(*op_policy_start)(struct ptlrpc_nrs_policy *policy);
-	/**
-	 * Called when deactivating a policy via lprocfs; policies deallocate
-	 * their resources here; this operation is optional
-	 *
-	 * \param[in,out] policy The policy being stopped
-	 *
-	 * \see nrs_policy_stop0()
-	 */
-	void	(*op_policy_stop)(struct ptlrpc_nrs_policy *policy);
-	/**
-	 * Used for policy-specific operations; i.e. not generic ones like
-	 * \e PTLRPC_NRS_CTL_START and \e PTLRPC_NRS_CTL_GET_INFO; analogous
-	 * to an ioctl; this operation is optional.
-	 *
-	 * \param[in,out]	 policy The policy carrying out operation \a opc
-	 * \param[in]	  opc	 The command operation being carried out
-	 * \param[in,out] arg	 An generic buffer for communication between the
-	 *			 user and the control operation
-	 *
-	 * \retval -ve error
-	 * \retval   0 success
-	 *
-	 * \see ptlrpc_nrs_policy_control()
-	 */
-	int	(*op_policy_ctl)(struct ptlrpc_nrs_policy *policy,
-				 enum ptlrpc_nrs_ctl opc, void *arg);
-
-	/**
-	 * Called when obtaining references to the resources of the resource
-	 * hierarchy for a request that has arrived for handling at the PTLRPC
-	 * service. Policies should return -ve for requests they do not wish
-	 * to handle. This operation is mandatory.
-	 *
-	 * \param[in,out] policy  The policy we're getting resources for.
-	 * \param[in,out] nrq	  The request we are getting resources for.
-	 * \param[in]	  parent  The parent resource of the resource being
-	 *			  requested; set to NULL if none.
-	 * \param[out]	  resp	  The resource is to be returned here; the
-	 *			  fallback policy in an NRS head should
-	 *			  \e always return a non-NULL pointer value.
-	 * \param[in]  moving_req When set, signifies that this is an attempt
-	 *			  to obtain resources for a request being moved
-	 *			  to the high-priority NRS head by
-	 *			  ldlm_lock_reorder_req().
-	 *			  This implies two things:
-	 *			  1. We are under obd_export::exp_rpc_lock and
-	 *			  so should not sleep.
-	 *			  2. We should not perform non-idempotent or can
-	 *			  skip performing idempotent operations that
-	 *			  were carried out when resources were first
-	 *			  taken for the request when it was initialized
-	 *			  in ptlrpc_nrs_req_initialize().
-	 *
-	 * \retval 0, +ve The level of the returned resource in the resource
-	 *		  hierarchy; currently only 0 (for a non-leaf resource)
-	 *		  and 1 (for a leaf resource) are supported by the
-	 *		  framework.
-	 * \retval -ve	  error
-	 *
-	 * \see ptlrpc_nrs_req_initialize()
-	 * \see ptlrpc_nrs_hpreq_add_nolock()
-	 */
-	int	(*op_res_get)(struct ptlrpc_nrs_policy *policy,
-			      struct ptlrpc_nrs_request *nrq,
-			      const struct ptlrpc_nrs_resource *parent,
-			      struct ptlrpc_nrs_resource **resp,
-			      bool moving_req);
-	/**
-	 * Called when releasing references taken for resources in the resource
-	 * hierarchy for the request; this operation is optional.
-	 *
-	 * \param[in,out] policy The policy the resource belongs to
-	 * \param[in] res	 The resource to be freed
-	 *
-	 * \see ptlrpc_nrs_req_finalize()
-	 * \see ptlrpc_nrs_hpreq_add_nolock()
-	 */
-	void	(*op_res_put)(struct ptlrpc_nrs_policy *policy,
-			      const struct ptlrpc_nrs_resource *res);
-
-	/**
-	 * Obtains a request for handling from the policy, and optionally
-	 * removes the request from the policy; this operation is mandatory.
-	 *
-	 * \param[in,out] policy The policy to poll
-	 * \param[in]	  peek	 When set, signifies that we just want to
-	 *			 examine the request, and not handle it, so the
-	 *			 request is not removed from the policy.
-	 * \param[in]	  force	 When set, it will force a policy to return a
-	 *			 request if it has one queued.
-	 *
-	 * \retval NULL No request available for handling
-	 * \retval valid-pointer The request polled for handling
-	 *
-	 * \see ptlrpc_nrs_req_get_nolock()
-	 */
-	struct ptlrpc_nrs_request *
-		(*op_req_get)(struct ptlrpc_nrs_policy *policy, bool peek,
-			      bool force);
-	/**
-	 * Called when attempting to add a request to a policy for later
-	 * handling; this operation is mandatory.
-	 *
-	 * \param[in,out] policy  The policy on which to enqueue \a nrq
-	 * \param[in,out] nrq The request to enqueue
-	 *
-	 * \retval 0	success
-	 * \retval != 0	error
-	 *
-	 * \see ptlrpc_nrs_req_add_nolock()
-	 */
-	int	(*op_req_enqueue)(struct ptlrpc_nrs_policy *policy,
-				  struct ptlrpc_nrs_request *nrq);
-	/**
-	 * Removes a request from the policy's set of pending requests. Normally
-	 * called after a request has been polled successfully from the policy
-	 * for handling; this operation is mandatory.
-	 *
-	 * \param[in,out] policy The policy the request \a nrq belongs to
-	 * \param[in,out] nrq    The request to dequeue
-	 */
-	void	(*op_req_dequeue)(struct ptlrpc_nrs_policy *policy,
-				  struct ptlrpc_nrs_request *nrq);
-	/**
-	 * Called after the request being carried out. Could be used for
-	 * job/resource control; this operation is optional.
-	 *
-	 * \param[in,out] policy The policy which is stopping to handle request
-	 *			 \a nrq
-	 * \param[in,out] nrq	 The request
-	 *
-	 * \pre assert_spin_locked(&svcpt->scp_req_lock)
-	 *
-	 * \see ptlrpc_nrs_req_stop_nolock()
-	 */
-	void	(*op_req_stop)(struct ptlrpc_nrs_policy *policy,
-			       struct ptlrpc_nrs_request *nrq);
-	/**
-	 * Registers the policy's lprocfs interface with a PTLRPC service.
-	 *
-	 * \param[in] svc The service
-	 *
-	 * \retval 0	success
-	 * \retval != 0	error
-	 */
-	int	(*op_lprocfs_init)(struct ptlrpc_service *svc);
-	/**
-	 * Unegisters the policy's lprocfs interface with a PTLRPC service.
-	 *
-	 * In cases of failed policy registration in
-	 * \e ptlrpc_nrs_policy_register(), this function may be called for a
-	 * service which has not registered the policy successfully, so
-	 * implementations of this method should make sure their operations are
-	 * safe in such cases.
-	 *
-	 * \param[in] svc The service
-	 */
-	void	(*op_lprocfs_fini)(struct ptlrpc_service *svc);
-};
-
-/**
- * Policy flags
- */
-enum nrs_policy_flags {
-	/**
-	 * Fallback policy, use this flag only on a single supported policy per
-	 * service. The flag cannot be used on policies that use
-	 * \e PTLRPC_NRS_FL_REG_EXTERN
-	 */
-	PTLRPC_NRS_FL_FALLBACK		= (1 << 0),
-	/**
-	 * Start policy immediately after registering.
-	 */
-	PTLRPC_NRS_FL_REG_START		= (1 << 1),
-	/**
-	 * This is a policy registering from a module different to the one NRS
-	 * core ships in (currently ptlrpc).
-	 */
-	PTLRPC_NRS_FL_REG_EXTERN	= (1 << 2),
-};
-
-/**
- * NRS queue type.
- *
- * Denotes whether an NRS instance is for handling normal or high-priority
- * RPCs, or whether an operation pertains to one or both of the NRS instances
- * in a service.
- */
-enum ptlrpc_nrs_queue_type {
-	PTLRPC_NRS_QUEUE_REG	= (1 << 0),
-	PTLRPC_NRS_QUEUE_HP	= (1 << 1),
-	PTLRPC_NRS_QUEUE_BOTH	= (PTLRPC_NRS_QUEUE_REG | PTLRPC_NRS_QUEUE_HP)
-};
-
-/**
- * NRS head
- *
- * A PTLRPC service has at least one NRS head instance for handling normal
- * priority RPCs, and may optionally have a second NRS head instance for
- * handling high-priority RPCs. Each NRS head maintains a list of available
- * policies, of which one and only one policy is acting as the fallback policy,
- * and optionally a different policy may be acting as the primary policy. For
- * all RPCs handled by this NRS head instance, NRS core will first attempt to
- * enqueue the RPC using the primary policy (if any). The fallback policy is
- * used in the following cases:
- * - when there was no primary policy in the
- *   ptlrpc_nrs_pol_state::NRS_POL_STATE_STARTED state at the time the request
- *   was initialized.
- * - when the primary policy that was at the
- *   ptlrpc_nrs_pol_state::PTLRPC_NRS_POL_STATE_STARTED state at the time the
- *   RPC was initialized, denoted it did not wish, or for some other reason was
- *   not able to handle the request, by returning a non-valid NRS resource
- *   reference.
- * - when the primary policy that was at the
- *   ptlrpc_nrs_pol_state::PTLRPC_NRS_POL_STATE_STARTED state at the time the
- *   RPC was initialized, fails later during the request enqueueing stage.
- *
- * \see nrs_resource_get_safe()
- * \see nrs_request_enqueue()
- */
-struct ptlrpc_nrs {
-	spinlock_t			nrs_lock;
-	/** XXX Possibly replace svcpt->scp_req_lock with another lock here. */
-	/**
-	 * List of registered policies
-	 */
-	struct list_head			nrs_policy_list;
-	/**
-	 * List of policies with queued requests. Policies that have any
-	 * outstanding requests are queued here, and this list is queried
-	 * in a round-robin manner from NRS core when obtaining a request
-	 * for handling. This ensures that requests from policies that at some
-	 * point transition away from the
-	 * ptlrpc_nrs_pol_state::NRS_POL_STATE_STARTED state are drained.
-	 */
-	struct list_head			nrs_policy_queued;
-	/**
-	 * Service partition for this NRS head
-	 */
-	struct ptlrpc_service_part     *nrs_svcpt;
-	/**
-	 * Primary policy, which is the preferred policy for handling RPCs
-	 */
-	struct ptlrpc_nrs_policy       *nrs_policy_primary;
-	/**
-	 * Fallback policy, which is the backup policy for handling RPCs
-	 */
-	struct ptlrpc_nrs_policy       *nrs_policy_fallback;
-	/**
-	 * This NRS head handles either HP or regular requests
-	 */
-	enum ptlrpc_nrs_queue_type	nrs_queue_type;
-	/**
-	 * # queued requests from all policies in this NRS head
-	 */
-	unsigned long			nrs_req_queued;
-	/**
-	 * # scheduled requests from all policies in this NRS head
-	 */
-	unsigned long			nrs_req_started;
-	/**
-	 * # policies on this NRS
-	 */
-	unsigned			nrs_num_pols;
-	/**
-	 * This NRS head is in progress of starting a policy
-	 */
-	unsigned			nrs_policy_starting:1;
-	/**
-	 * In progress of shutting down the whole NRS head; used during
-	 * unregistration
-	 */
-	unsigned			nrs_stopping:1;
-};
-
-#define NRS_POL_NAME_MAX		16
-
-struct ptlrpc_nrs_pol_desc;
-
-/**
- * Service compatibility predicate; this determines whether a policy is adequate
- * for handling RPCs of a particular PTLRPC service.
- *
- * XXX:This should give the same result during policy registration and
- * unregistration, and for all partitions of a service; so the result should not
- * depend on temporal service or other properties, that may influence the
- * result.
- */
-typedef bool (*nrs_pol_desc_compat_t) (const struct ptlrpc_service *svc,
-				       const struct ptlrpc_nrs_pol_desc *desc);
-
-struct ptlrpc_nrs_pol_conf {
-	/**
-	 * Human-readable policy name
-	 */
-	char				   nc_name[NRS_POL_NAME_MAX];
-	/**
-	 * NRS operations for this policy
-	 */
-	const struct ptlrpc_nrs_pol_ops	  *nc_ops;
-	/**
-	 * Service compatibility predicate
-	 */
-	nrs_pol_desc_compat_t		   nc_compat;
-	/**
-	 * Set for policies that support a single ptlrpc service, i.e. ones that
-	 * have \a pd_compat set to nrs_policy_compat_one(). The variable value
-	 * depicts the name of the single service that such policies are
-	 * compatible with.
-	 */
-	const char			  *nc_compat_svc_name;
-	/**
-	 * Owner module for this policy descriptor; policies registering from a
-	 * different module to the one the NRS framework is held within
-	 * (currently ptlrpc), should set this field to THIS_MODULE.
-	 */
-	struct module			  *nc_owner;
-	/**
-	 * Policy registration flags; a bitmask of \e nrs_policy_flags
-	 */
-	unsigned			   nc_flags;
-};
-
-/**
- * NRS policy registering descriptor
- *
- * Is used to hold a description of a policy that can be passed to NRS core in
- * order to register the policy with NRS heads in different PTLRPC services.
- */
-struct ptlrpc_nrs_pol_desc {
-	/**
-	 * Human-readable policy name
-	 */
-	char					pd_name[NRS_POL_NAME_MAX];
-	/**
-	 * Link into nrs_core::nrs_policies
-	 */
-	struct list_head				pd_list;
-	/**
-	 * NRS operations for this policy
-	 */
-	const struct ptlrpc_nrs_pol_ops	       *pd_ops;
-	/**
-	 * Service compatibility predicate
-	 */
-	nrs_pol_desc_compat_t			pd_compat;
-	/**
-	 * Set for policies that are compatible with only one PTLRPC service.
-	 *
-	 * \see ptlrpc_nrs_pol_conf::nc_compat_svc_name
-	 */
-	const char			       *pd_compat_svc_name;
-	/**
-	 * Owner module for this policy descriptor.
-	 *
-	 * We need to hold a reference to the module whenever we might make use
-	 * of any of the module's contents, i.e.
-	 * - If one or more instances of the policy are at a state where they
-	 *   might be handling a request, i.e.
-	 *   ptlrpc_nrs_pol_state::NRS_POL_STATE_STARTED or
-	 *   ptlrpc_nrs_pol_state::NRS_POL_STATE_STOPPING as we will have to
-	 *   call into the policy's ptlrpc_nrs_pol_ops() handlers. A reference
-	 *   is taken on the module when
-	 *   \e ptlrpc_nrs_pol_desc::pd_refs becomes 1, and released when it
-	 *   becomes 0, so that we hold only one reference to the module maximum
-	 *   at any time.
-	 *
-	 *   We do not need to hold a reference to the module, even though we
-	 *   might use code and data from the module, in the following cases:
-	 * - During external policy registration, because this should happen in
-	 *   the module's init() function, in which case the module is safe from
-	 *   removal because a reference is being held on the module by the
-	 *   kernel, and iirc kmod (and I guess module-init-tools also) will
-	 *   serialize any racing processes properly anyway.
-	 * - During external policy unregistration, because this should happen
-	 *   in a module's exit() function, and any attempts to start a policy
-	 *   instance would need to take a reference on the module, and this is
-	 *   not possible once we have reached the point where the exit()
-	 *   handler is called.
-	 * - During service registration and unregistration, as service setup
-	 *   and cleanup, and policy registration, unregistration and policy
-	 *   instance starting, are serialized by \e nrs_core::nrs_mutex, so
-	 *   as long as users adhere to the convention of registering policies
-	 *   in init() and unregistering them in module exit() functions, there
-	 *   should not be a race between these operations.
-	 * - During any policy-specific lprocfs operations, because a reference
-	 *   is held by the kernel on a proc entry that has been entered by a
-	 *   syscall, so as long as proc entries are removed during unregistration time,
-	 *   then unregistration and lprocfs operations will be properly
-	 *   serialized.
-	 */
-	struct module			       *pd_owner;
-	/**
-	 * Bitmask of \e nrs_policy_flags
-	 */
-	unsigned				pd_flags;
-	/**
-	 * # of references on this descriptor
-	 */
-	atomic_t				pd_refs;
-};
-
-/**
- * NRS policy state
- *
- * Policies transition from one state to the other during their lifetime
- */
-enum ptlrpc_nrs_pol_state {
-	/**
-	 * Not a valid policy state.
-	 */
-	NRS_POL_STATE_INVALID,
-	/**
-	 * Policies are at this state either at the start of their life, or
-	 * transition here when the user selects a different policy to act
-	 * as the primary one.
-	 */
-	NRS_POL_STATE_STOPPED,
-	/**
-	 * Policy is progress of stopping
-	 */
-	NRS_POL_STATE_STOPPING,
-	/**
-	 * Policy is in progress of starting
-	 */
-	NRS_POL_STATE_STARTING,
-	/**
-	 * A policy is in this state in two cases:
-	 * - it is the fallback policy, which is always in this state.
-	 * - it has been activated by the user; i.e. it is the primary policy,
-	 */
-	NRS_POL_STATE_STARTED,
-};
-
-/**
- * NRS policy information
- *
- * Used for obtaining information for the status of a policy via lprocfs
- */
-struct ptlrpc_nrs_pol_info {
-	/**
-	 * Policy name
-	 */
-	char				pi_name[NRS_POL_NAME_MAX];
-	/**
-	 * Current policy state
-	 */
-	enum ptlrpc_nrs_pol_state	pi_state;
-	/**
-	 * # RPCs enqueued for later dispatching by the policy
-	 */
-	long				pi_req_queued;
-	/**
-	 * # RPCs started for dispatch by the policy
-	 */
-	long				pi_req_started;
-	/**
-	 * Is this a fallback policy?
-	 */
-	unsigned			pi_fallback:1;
-};
-
-/**
- * NRS policy
- *
- * There is one instance of this for each policy in each NRS head of each
- * PTLRPC service partition.
- */
-struct ptlrpc_nrs_policy {
-	/**
-	 * Linkage into the NRS head's list of policies,
-	 * ptlrpc_nrs:nrs_policy_list
-	 */
-	struct list_head			pol_list;
-	/**
-	 * Linkage into the NRS head's list of policies with enqueued
-	 * requests ptlrpc_nrs:nrs_policy_queued
-	 */
-	struct list_head			pol_list_queued;
-	/**
-	 * Current state of this policy
-	 */
-	enum ptlrpc_nrs_pol_state	pol_state;
-	/**
-	 * Bitmask of nrs_policy_flags
-	 */
-	unsigned			pol_flags;
-	/**
-	 * # RPCs enqueued for later dispatching by the policy
-	 */
-	long				pol_req_queued;
-	/**
-	 * # RPCs started for dispatch by the policy
-	 */
-	long				pol_req_started;
-	/**
-	 * Usage Reference count taken on the policy instance
-	 */
-	long				pol_ref;
-	/**
-	 * The NRS head this policy has been created at
-	 */
-	struct ptlrpc_nrs	       *pol_nrs;
-	/**
-	 * Private policy data; varies by policy type
-	 */
-	void			       *pol_private;
-	/**
-	 * Policy descriptor for this policy instance.
-	 */
-	struct ptlrpc_nrs_pol_desc     *pol_desc;
-};
-
-/**
- * NRS resource
- *
- * Resources are embedded into two types of NRS entities:
- * - Inside NRS policies, in the policy's private data in
- *   ptlrpc_nrs_policy::pol_private
- * - In objects that act as prime-level scheduling entities in different NRS
- *   policies; e.g. on a policy that performs round robin or similar order
- *   scheduling across client NIDs, there would be one NRS resource per unique
- *   client NID. On a policy which performs round robin scheduling across
- *   backend filesystem objects, there would be one resource associated with
- *   each of the backend filesystem objects partaking in the scheduling
- *   performed by the policy.
- *
- * NRS resources share a parent-child relationship, in which resources embedded
- * in policy instances are the parent entities, with all scheduling entities
- * a policy schedules across being the children, thus forming a simple resource
- * hierarchy. This hierarchy may be extended with one or more levels in the
- * future if the ability to have more than one primary policy is added.
- *
- * Upon request initialization, references to the then active NRS policies are
- * taken and used to later handle the dispatching of the request with one of
- * these policies.
- *
- * \see nrs_resource_get_safe()
- * \see ptlrpc_nrs_req_add()
- */
-struct ptlrpc_nrs_resource {
-	/**
-	 * This NRS resource's parent; is NULL for resources embedded in NRS
-	 * policy instances; i.e. those are top-level ones.
-	 */
-	struct ptlrpc_nrs_resource     *res_parent;
-	/**
-	 * The policy associated with this resource.
-	 */
-	struct ptlrpc_nrs_policy       *res_policy;
-};
-
-enum {
-	NRS_RES_FALLBACK,
-	NRS_RES_PRIMARY,
-	NRS_RES_MAX
-};
-
-/* \name fifo
- *
- * FIFO policy
- *
- * This policy is a logical wrapper around previous, non-NRS functionality.
- * It dispatches RPCs in the same order as they arrive from the network. This
- * policy is currently used as the fallback policy, and the only enabled policy
- * on all NRS heads of all PTLRPC service partitions.
- * @{
- */
-
-/**
- * Private data structure for the FIFO policy
- */
-struct nrs_fifo_head {
-	/**
-	 * Resource object for policy instance.
-	 */
-	struct ptlrpc_nrs_resource	fh_res;
-	/**
-	 * List of queued requests.
-	 */
-	struct list_head			fh_list;
-	/**
-	 * For debugging purposes.
-	 */
-	__u64				fh_sequence;
-};
-
-struct nrs_fifo_req {
-	struct list_head		fr_list;
-	__u64			fr_sequence;
-};
-
-/** @} fifo */
-
-/**
- * NRS request
- *
- * Instances of this object exist embedded within ptlrpc_request; the main
- * purpose of this object is to hold references to the request's resources
- * for the lifetime of the request, and to hold properties that policies use
- * use for determining the request's scheduling priority.
- */
-struct ptlrpc_nrs_request {
-	/**
-	 * The request's resource hierarchy.
-	 */
-	struct ptlrpc_nrs_resource     *nr_res_ptrs[NRS_RES_MAX];
-	/**
-	 * Index into ptlrpc_nrs_request::nr_res_ptrs of the resource of the
-	 * policy that was used to enqueue the request.
-	 *
-	 * \see nrs_request_enqueue()
-	 */
-	unsigned			nr_res_idx;
-	unsigned			nr_initialized:1;
-	unsigned			nr_enqueued:1;
-	unsigned			nr_started:1;
-	unsigned			nr_finalized:1;
-
-	/**
-	 * Policy-specific fields, used for determining a request's scheduling
-	 * priority, and other supporting functionality.
-	 */
-	union {
-		/**
-		 * Fields for the FIFO policy
-		 */
-		struct nrs_fifo_req	fifo;
-	} nr_u;
-	/**
-	 * Externally-registering policies may want to use this to allocate
-	 * their own request properties.
-	 */
-	void			       *ext;
-};
-
-/** @} nrs */
+#include "lustre_nrs.h"
 
 /**
  * Basic request prioritization operations structure.
@@ -1304,6 +600,8 @@ struct ptlrpc_cli_req {
 	union ptlrpc_async_args		 cr_async_args;
 	/** Opaq data for replay and commit callbacks. */
 	void				*cr_cb_data;
+	/** Link to the imp->imp_unreplied_list */
+	struct list_head		 cr_unreplied_list;
 	/**
 	 * Commit callback, called when request is committed and about to be
 	 * freed.
@@ -1343,6 +641,7 @@ struct ptlrpc_cli_req {
 #define rq_interpret_reply	rq_cli.cr_reply_interp
 #define rq_async_args		rq_cli.cr_async_args
 #define rq_cb_data		rq_cli.cr_cb_data
+#define rq_unreplied_list	rq_cli.cr_unreplied_list
 #define rq_commit_cb		rq_cli.cr_commit_cb
 #define rq_replay_cb		rq_cli.cr_replay_cb
 
@@ -1505,6 +804,8 @@ struct ptlrpc_request {
 	__u64 rq_transno;
 	/** xid */
 	__u64 rq_xid;
+	/** bulk match bits */
+	u64				rq_mbits;
 	/**
 	 * List item to for replay list. Not yet committed requests get linked
 	 * there.
@@ -1793,10 +1094,93 @@ struct ptlrpc_bulk_page {
 	struct page     *bp_page;
 };
 
-#define BULK_GET_SOURCE   0
-#define BULK_PUT_SINK     1
-#define BULK_GET_SINK     2
-#define BULK_PUT_SOURCE   3
+enum ptlrpc_bulk_op_type {
+	PTLRPC_BULK_OP_ACTIVE	= 0x00000001,
+	PTLRPC_BULK_OP_PASSIVE	= 0x00000002,
+	PTLRPC_BULK_OP_PUT	= 0x00000004,
+	PTLRPC_BULK_OP_GET	= 0x00000008,
+	PTLRPC_BULK_BUF_KVEC	= 0x00000010,
+	PTLRPC_BULK_BUF_KIOV	= 0x00000020,
+	PTLRPC_BULK_GET_SOURCE	= PTLRPC_BULK_OP_PASSIVE | PTLRPC_BULK_OP_GET,
+	PTLRPC_BULK_PUT_SINK	= PTLRPC_BULK_OP_PASSIVE | PTLRPC_BULK_OP_PUT,
+	PTLRPC_BULK_GET_SINK	= PTLRPC_BULK_OP_ACTIVE | PTLRPC_BULK_OP_GET,
+	PTLRPC_BULK_PUT_SOURCE	= PTLRPC_BULK_OP_ACTIVE | PTLRPC_BULK_OP_PUT,
+};
+
+static inline bool ptlrpc_is_bulk_op_get(enum ptlrpc_bulk_op_type type)
+{
+	return (type & PTLRPC_BULK_OP_GET) == PTLRPC_BULK_OP_GET;
+}
+
+static inline bool ptlrpc_is_bulk_get_source(enum ptlrpc_bulk_op_type type)
+{
+	return (type & PTLRPC_BULK_GET_SOURCE) == PTLRPC_BULK_GET_SOURCE;
+}
+
+static inline bool ptlrpc_is_bulk_put_sink(enum ptlrpc_bulk_op_type type)
+{
+	return (type & PTLRPC_BULK_PUT_SINK) == PTLRPC_BULK_PUT_SINK;
+}
+
+static inline bool ptlrpc_is_bulk_get_sink(enum ptlrpc_bulk_op_type type)
+{
+	return (type & PTLRPC_BULK_GET_SINK) == PTLRPC_BULK_GET_SINK;
+}
+
+static inline bool ptlrpc_is_bulk_put_source(enum ptlrpc_bulk_op_type type)
+{
+	return (type & PTLRPC_BULK_PUT_SOURCE) == PTLRPC_BULK_PUT_SOURCE;
+}
+
+static inline bool ptlrpc_is_bulk_desc_kvec(enum ptlrpc_bulk_op_type type)
+{
+	return ((type & PTLRPC_BULK_BUF_KVEC) | (type & PTLRPC_BULK_BUF_KIOV))
+		== PTLRPC_BULK_BUF_KVEC;
+}
+
+static inline bool ptlrpc_is_bulk_desc_kiov(enum ptlrpc_bulk_op_type type)
+{
+	return ((type & PTLRPC_BULK_BUF_KVEC) | (type & PTLRPC_BULK_BUF_KIOV))
+		== PTLRPC_BULK_BUF_KIOV;
+}
+
+static inline bool ptlrpc_is_bulk_op_active(enum ptlrpc_bulk_op_type type)
+{
+	return ((type & PTLRPC_BULK_OP_ACTIVE) |
+		(type & PTLRPC_BULK_OP_PASSIVE)) == PTLRPC_BULK_OP_ACTIVE;
+}
+
+static inline bool ptlrpc_is_bulk_op_passive(enum ptlrpc_bulk_op_type type)
+{
+	return ((type & PTLRPC_BULK_OP_ACTIVE) |
+		(type & PTLRPC_BULK_OP_PASSIVE)) == PTLRPC_BULK_OP_PASSIVE;
+}
+
+struct ptlrpc_bulk_frag_ops {
+	/**
+	 * Add a page \a page to the bulk descriptor \a desc
+	 * Data to transfer in the page starts at offset \a pageoffset and
+	 * amount of data to transfer from the page is \a len
+	 */
+	void (*add_kiov_frag)(struct ptlrpc_bulk_desc *desc,
+			      struct page *page, int pageoffset, int len);
+
+	/*
+	 * Add a \a fragment to the bulk descriptor \a desc.
+	 * Data to transfer in the fragment is pointed to by \a frag
+	 * The size of the fragment is \a len
+	 */
+	int (*add_iov_frag)(struct ptlrpc_bulk_desc *desc, void *frag, int len);
+
+	/**
+	 * Uninitialize and free bulk descriptor \a desc.
+	 * Works on bulk descriptors both from server and client side.
+	 */
+	void (*release_frags)(struct ptlrpc_bulk_desc *desc);
+};
+
+extern const struct ptlrpc_bulk_frag_ops ptlrpc_bulk_kiov_pin_ops;
+extern const struct ptlrpc_bulk_frag_ops ptlrpc_bulk_kiov_nopin_ops;
 
 /**
  * Definition of bulk descriptor.
@@ -1811,14 +1195,14 @@ struct ptlrpc_bulk_page {
 struct ptlrpc_bulk_desc {
 	/** completed with failure */
 	unsigned long bd_failure:1;
-	/** {put,get}{source,sink} */
-	unsigned long bd_type:2;
 	/** client side */
 	unsigned long bd_registered:1;
 	/** For serialization with callback */
 	spinlock_t bd_lock;
 	/** Import generation when request for this bulk was sent */
 	int bd_import_generation;
+	/** {put,get}{source,sink}{kvec,kiov} */
+	enum ptlrpc_bulk_op_type bd_type;
 	/** LNet portal for this bulk */
 	__u32 bd_portal;
 	/** Server side - export this bulk created for */
@@ -1827,13 +1211,14 @@ struct ptlrpc_bulk_desc {
 	struct obd_import *bd_import;
 	/** Back pointer to the request */
 	struct ptlrpc_request *bd_req;
+	struct ptlrpc_bulk_frag_ops *bd_frag_ops;
 	wait_queue_head_t	    bd_waitq;	/* server side only WQ */
 	int		    bd_iov_count;    /* # entries in bd_iov */
 	int		    bd_max_iov;      /* allocated size of bd_iov */
 	int		    bd_nob;	  /* # bytes covered */
 	int		    bd_nob_transferred; /* # bytes GOT/PUT */
 
-	__u64		  bd_last_xid;
+	u64			bd_last_mbits;
 
 	struct ptlrpc_cb_id    bd_cbid;	 /* network callback info */
 	lnet_nid_t	     bd_sender;       /* stash event::sender */
@@ -1842,14 +1227,31 @@ struct ptlrpc_bulk_desc {
 	/** array of associated MDs */
 	lnet_handle_md_t	bd_mds[PTLRPC_BULK_OPS_COUNT];
 
-	/*
-	 * encrypt iov, size is either 0 or bd_iov_count.
-	 */
-	lnet_kiov_t	   *bd_enc_iov;
-
-	lnet_kiov_t	    bd_iov[0];
+	union {
+		struct {
+			/*
+			 * encrypt iov, size is either 0 or bd_iov_count.
+			 */
+			struct bio_vec *bd_enc_vec;
+			struct bio_vec *bd_vec;	/* Array of bio_vecs */
+		} bd_kiov;
+
+		struct {
+			struct kvec *bd_enc_kvec;
+			struct kvec *bd_kvec;	/* Array of kvecs */
+		} bd_kvec;
+	} bd_u;
 };
 
+#define GET_KIOV(desc)			((desc)->bd_u.bd_kiov.bd_vec)
+#define BD_GET_KIOV(desc, i)		((desc)->bd_u.bd_kiov.bd_vec[i])
+#define GET_ENC_KIOV(desc)		((desc)->bd_u.bd_kiov.bd_enc_vec)
+#define BD_GET_ENC_KIOV(desc, i)	((desc)->bd_u.bd_kiov.bd_enc_vec[i])
+#define GET_KVEC(desc)			((desc)->bd_u.bd_kvec.bd_kvec)
+#define BD_GET_KVEC(desc, i)		((desc)->bd_u.bd_kvec.bd_kvec[i])
+#define GET_ENC_KVEC(desc)		((desc)->bd_u.bd_kvec.bd_enc_kvec)
+#define BD_GET_ENC_KVEC(desc, i)	((desc)->bd_u.bd_kvec.bd_enc_kvec[i])
+
 enum {
 	SVC_STOPPED     = 1 << 0,
 	SVC_STOPPING    = 1 << 1,
@@ -2464,21 +1866,17 @@ int ptlrpc_request_bufs_pack(struct ptlrpc_request *request,
 void ptlrpc_req_finished(struct ptlrpc_request *request);
 struct ptlrpc_request *ptlrpc_request_addref(struct ptlrpc_request *req);
 struct ptlrpc_bulk_desc *ptlrpc_prep_bulk_imp(struct ptlrpc_request *req,
-					      unsigned npages, unsigned max_brw,
-					      unsigned type, unsigned portal);
-void __ptlrpc_free_bulk(struct ptlrpc_bulk_desc *bulk, int pin);
-static inline void ptlrpc_free_bulk_pin(struct ptlrpc_bulk_desc *bulk)
-{
-	__ptlrpc_free_bulk(bulk, 1);
-}
-
-static inline void ptlrpc_free_bulk_nopin(struct ptlrpc_bulk_desc *bulk)
-{
-	__ptlrpc_free_bulk(bulk, 0);
-}
-
+					      unsigned int nfrags,
+					      unsigned int max_brw,
+					      unsigned int type,
+					      unsigned int portal,
+					      const struct ptlrpc_bulk_frag_ops *ops);
+
+int ptlrpc_prep_bulk_frag(struct ptlrpc_bulk_desc *desc,
+			  void *frag, int len);
 void __ptlrpc_prep_bulk_page(struct ptlrpc_bulk_desc *desc,
-			     struct page *page, int pageoffset, int len, int);
+			     struct page *page, int pageoffset, int len,
+			     int pin);
 static inline void ptlrpc_prep_bulk_page_pin(struct ptlrpc_bulk_desc *desc,
 					     struct page *page, int pageoffset,
 					     int len)
@@ -2493,6 +1891,16 @@ static inline void ptlrpc_prep_bulk_page_nopin(struct ptlrpc_bulk_desc *desc,
 	__ptlrpc_prep_bulk_page(desc, page, pageoffset, len, 0);
 }
 
+void ptlrpc_free_bulk(struct ptlrpc_bulk_desc *bulk);
+
+static inline void ptlrpc_release_bulk_page_pin(struct ptlrpc_bulk_desc *desc)
+{
+	int i;
+
+	for (i = 0; i < desc->bd_iov_count ; i++)
+		put_page(BD_GET_KIOV(desc, i).bv_page);
+}
+
 void ptlrpc_retain_replayable_request(struct ptlrpc_request *req,
 				      struct obd_import *imp);
 __u64 ptlrpc_next_xid(void);
@@ -2652,6 +2060,7 @@ struct lustre_handle *lustre_msg_get_handle(struct lustre_msg *msg);
 __u32 lustre_msg_get_type(struct lustre_msg *msg);
 void lustre_msg_add_version(struct lustre_msg *msg, u32 version);
 __u32 lustre_msg_get_opc(struct lustre_msg *msg);
+__u16 lustre_msg_get_tag(struct lustre_msg *msg);
 __u64 lustre_msg_get_last_committed(struct lustre_msg *msg);
 __u64 *lustre_msg_get_versions(struct lustre_msg *msg);
 __u64 lustre_msg_get_transno(struct lustre_msg *msg);
@@ -2670,6 +2079,8 @@ void lustre_msg_set_handle(struct lustre_msg *msg,
 			   struct lustre_handle *handle);
 void lustre_msg_set_type(struct lustre_msg *msg, __u32 type);
 void lustre_msg_set_opc(struct lustre_msg *msg, __u32 opc);
+void lustre_msg_set_last_xid(struct lustre_msg *msg, u64 last_xid);
+void lustre_msg_set_tag(struct lustre_msg *msg, __u16 tag);
 void lustre_msg_set_versions(struct lustre_msg *msg, __u64 *versions);
 void lustre_msg_set_transno(struct lustre_msg *msg, __u64 transno);
 void lustre_msg_set_status(struct lustre_msg *msg, __u32 status);
@@ -2679,6 +2090,7 @@ void lustre_msg_set_timeout(struct lustre_msg *msg, __u32 timeout);
 void lustre_msg_set_service_time(struct lustre_msg *msg, __u32 service_time);
 void lustre_msg_set_jobid(struct lustre_msg *msg, char *jobid);
 void lustre_msg_set_cksum(struct lustre_msg *msg, __u32 cksum);
+void lustre_msg_set_mbits(struct lustre_msg *msg, u64 mbits);
 
 static inline void
 lustre_shrink_reply(struct ptlrpc_request *req, int segment,
diff --git a/drivers/staging/lustre/lustre/include/lustre_nrs.h b/drivers/staging/lustre/lustre/include/lustre_nrs.h
new file mode 100644
index 000000000000..a5028aaa19cd
--- /dev/null
+++ b/drivers/staging/lustre/lustre/include/lustre_nrs.h
@@ -0,0 +1,717 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License version 2 for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2014, Intel Corporation.
+ *
+ * Copyright 2012 Xyratex Technology Limited
+ */
+/*
+ *
+ * Network Request Scheduler (NRS)
+ *
+ */
+
+#ifndef _LUSTRE_NRS_H
+#define _LUSTRE_NRS_H
+
+/**
+ * \defgroup nrs Network Request Scheduler
+ * @{
+ */
+struct ptlrpc_nrs_policy;
+struct ptlrpc_nrs_resource;
+struct ptlrpc_nrs_request;
+
+/**
+ * NRS control operations.
+ *
+ * These are common for all policies.
+ */
+enum ptlrpc_nrs_ctl {
+	/**
+	 * Not a valid opcode.
+	 */
+	PTLRPC_NRS_CTL_INVALID,
+	/**
+	 * Activate the policy.
+	 */
+	PTLRPC_NRS_CTL_START,
+	/**
+	 * Reserved for multiple primary policies, which may be a possibility
+	 * in the future.
+	 */
+	PTLRPC_NRS_CTL_STOP,
+	/**
+	 * Policies can start using opcodes from this value and onwards for
+	 * their own purposes; the assigned value itself is arbitrary.
+	 */
+	PTLRPC_NRS_CTL_1ST_POL_SPEC = 0x20,
+};
+
+/**
+ * NRS policy operations.
+ *
+ * These determine the behaviour of a policy, and are called in response to
+ * NRS core events.
+ */
+struct ptlrpc_nrs_pol_ops {
+	/**
+	 * Called during policy registration; this operation is optional.
+	 *
+	 * \param[in,out] policy The policy being initialized
+	 */
+	int	(*op_policy_init)(struct ptlrpc_nrs_policy *policy);
+	/**
+	 * Called during policy unregistration; this operation is optional.
+	 *
+	 * \param[in,out] policy The policy being unregistered/finalized
+	 */
+	void	(*op_policy_fini)(struct ptlrpc_nrs_policy *policy);
+	/**
+	 * Called when activating a policy via lprocfs; policies allocate and
+	 * initialize their resources here; this operation is optional.
+	 *
+	 * \param[in,out] policy The policy being started
+	 *
+	 * \see nrs_policy_start_locked()
+	 */
+	int	(*op_policy_start)(struct ptlrpc_nrs_policy *policy);
+	/**
+	 * Called when deactivating a policy via lprocfs; policies deallocate
+	 * their resources here; this operation is optional
+	 *
+	 * \param[in,out] policy The policy being stopped
+	 *
+	 * \see nrs_policy_stop0()
+	 */
+	void	(*op_policy_stop)(struct ptlrpc_nrs_policy *policy);
+	/**
+	 * Used for policy-specific operations; i.e. not generic ones like
+	 * \e PTLRPC_NRS_CTL_START and \e PTLRPC_NRS_CTL_GET_INFO; analogous
+	 * to an ioctl; this operation is optional.
+	 *
+	 * \param[in,out]	 policy The policy carrying out operation \a opc
+	 * \param[in]	  opc	 The command operation being carried out
+	 * \param[in,out] arg	 An generic buffer for communication between the
+	 *			 user and the control operation
+	 *
+	 * \retval -ve error
+	 * \retval   0 success
+	 *
+	 * \see ptlrpc_nrs_policy_control()
+	 */
+	int	(*op_policy_ctl)(struct ptlrpc_nrs_policy *policy,
+				 enum ptlrpc_nrs_ctl opc, void *arg);
+
+	/**
+	 * Called when obtaining references to the resources of the resource
+	 * hierarchy for a request that has arrived for handling at the PTLRPC
+	 * service. Policies should return -ve for requests they do not wish
+	 * to handle. This operation is mandatory.
+	 *
+	 * \param[in,out] policy  The policy we're getting resources for.
+	 * \param[in,out] nrq	  The request we are getting resources for.
+	 * \param[in]	  parent  The parent resource of the resource being
+	 *			  requested; set to NULL if none.
+	 * \param[out]	  resp	  The resource is to be returned here; the
+	 *			  fallback policy in an NRS head should
+	 *			  \e always return a non-NULL pointer value.
+	 * \param[in]  moving_req When set, signifies that this is an attempt
+	 *			  to obtain resources for a request being moved
+	 *			  to the high-priority NRS head by
+	 *			  ldlm_lock_reorder_req().
+	 *			  This implies two things:
+	 *			  1. We are under obd_export::exp_rpc_lock and
+	 *			  so should not sleep.
+	 *			  2. We should not perform non-idempotent or can
+	 *			  skip performing idempotent operations that
+	 *			  were carried out when resources were first
+	 *			  taken for the request when it was initialized
+	 *			  in ptlrpc_nrs_req_initialize().
+	 *
+	 * \retval 0, +ve The level of the returned resource in the resource
+	 *		  hierarchy; currently only 0 (for a non-leaf resource)
+	 *		  and 1 (for a leaf resource) are supported by the
+	 *		  framework.
+	 * \retval -ve	  error
+	 *
+	 * \see ptlrpc_nrs_req_initialize()
+	 * \see ptlrpc_nrs_hpreq_add_nolock()
+	 * \see ptlrpc_nrs_req_hp_move()
+	 */
+	int	(*op_res_get)(struct ptlrpc_nrs_policy *policy,
+			      struct ptlrpc_nrs_request *nrq,
+			      const struct ptlrpc_nrs_resource *parent,
+			      struct ptlrpc_nrs_resource **resp,
+			      bool moving_req);
+	/**
+	 * Called when releasing references taken for resources in the resource
+	 * hierarchy for the request; this operation is optional.
+	 *
+	 * \param[in,out] policy The policy the resource belongs to
+	 * \param[in] res	 The resource to be freed
+	 *
+	 * \see ptlrpc_nrs_req_finalize()
+	 * \see ptlrpc_nrs_hpreq_add_nolock()
+	 * \see ptlrpc_nrs_req_hp_move()
+	 */
+	void	(*op_res_put)(struct ptlrpc_nrs_policy *policy,
+			      const struct ptlrpc_nrs_resource *res);
+
+	/**
+	 * Obtains a request for handling from the policy, and optionally
+	 * removes the request from the policy; this operation is mandatory.
+	 *
+	 * \param[in,out] policy The policy to poll
+	 * \param[in]	  peek	 When set, signifies that we just want to
+	 *			 examine the request, and not handle it, so the
+	 *			 request is not removed from the policy.
+	 * \param[in]	  force  When set, it will force a policy to return a
+	 *			 request if it has one queued.
+	 *
+	 * \retval NULL No request available for handling
+	 * \retval valid-pointer The request polled for handling
+	 *
+	 * \see ptlrpc_nrs_req_get_nolock()
+	 */
+	struct ptlrpc_nrs_request *
+		(*op_req_get)(struct ptlrpc_nrs_policy *policy, bool peek,
+			      bool force);
+	/**
+	 * Called when attempting to add a request to a policy for later
+	 * handling; this operation is mandatory.
+	 *
+	 * \param[in,out] policy  The policy on which to enqueue \a nrq
+	 * \param[in,out] nrq The request to enqueue
+	 *
+	 * \retval 0	success
+	 * \retval != 0 error
+	 *
+	 * \see ptlrpc_nrs_req_add_nolock()
+	 */
+	int	(*op_req_enqueue)(struct ptlrpc_nrs_policy *policy,
+				  struct ptlrpc_nrs_request *nrq);
+	/**
+	 * Removes a request from the policy's set of pending requests. Normally
+	 * called after a request has been polled successfully from the policy
+	 * for handling; this operation is mandatory.
+	 *
+	 * \param[in,out] policy The policy the request \a nrq belongs to
+	 * \param[in,out] nrq	 The request to dequeue
+	 *
+	 * \see ptlrpc_nrs_req_del_nolock()
+	 */
+	void	(*op_req_dequeue)(struct ptlrpc_nrs_policy *policy,
+				  struct ptlrpc_nrs_request *nrq);
+	/**
+	 * Called after the request being carried out. Could be used for
+	 * job/resource control; this operation is optional.
+	 *
+	 * \param[in,out] policy The policy which is stopping to handle request
+	 *			 \a nrq
+	 * \param[in,out] nrq	 The request
+	 *
+	 * \pre assert_spin_locked(&svcpt->scp_req_lock)
+	 *
+	 * \see ptlrpc_nrs_req_stop_nolock()
+	 */
+	void	(*op_req_stop)(struct ptlrpc_nrs_policy *policy,
+			       struct ptlrpc_nrs_request *nrq);
+	/**
+	 * Registers the policy's lprocfs interface with a PTLRPC service.
+	 *
+	 * \param[in] svc The service
+	 *
+	 * \retval 0	success
+	 * \retval != 0 error
+	 */
+	int	(*op_lprocfs_init)(struct ptlrpc_service *svc);
+	/**
+	 * Unegisters the policy's lprocfs interface with a PTLRPC service.
+	 *
+	 * In cases of failed policy registration in
+	 * \e ptlrpc_nrs_policy_register(), this function may be called for a
+	 * service which has not registered the policy successfully, so
+	 * implementations of this method should make sure their operations are
+	 * safe in such cases.
+	 *
+	 * \param[in] svc The service
+	 */
+	void	(*op_lprocfs_fini)(struct ptlrpc_service *svc);
+};
+
+/**
+ * Policy flags
+ */
+enum nrs_policy_flags {
+	/**
+	 * Fallback policy, use this flag only on a single supported policy per
+	 * service. The flag cannot be used on policies that use
+	 * \e PTLRPC_NRS_FL_REG_EXTERN
+	 */
+	PTLRPC_NRS_FL_FALLBACK		= BIT(0),
+	/**
+	 * Start policy immediately after registering.
+	 */
+	PTLRPC_NRS_FL_REG_START		= BIT(1),
+	/**
+	 * This is a policy registering from a module different to the one NRS
+	 * core ships in (currently ptlrpc).
+	 */
+	PTLRPC_NRS_FL_REG_EXTERN	= BIT(2),
+};
+
+/**
+ * NRS queue type.
+ *
+ * Denotes whether an NRS instance is for handling normal or high-priority
+ * RPCs, or whether an operation pertains to one or both of the NRS instances
+ * in a service.
+ */
+enum ptlrpc_nrs_queue_type {
+	PTLRPC_NRS_QUEUE_REG	= BIT(0),
+	PTLRPC_NRS_QUEUE_HP	= BIT(1),
+	PTLRPC_NRS_QUEUE_BOTH	= (PTLRPC_NRS_QUEUE_REG | PTLRPC_NRS_QUEUE_HP)
+};
+
+/**
+ * NRS head
+ *
+ * A PTLRPC service has at least one NRS head instance for handling normal
+ * priority RPCs, and may optionally have a second NRS head instance for
+ * handling high-priority RPCs. Each NRS head maintains a list of available
+ * policies, of which one and only one policy is acting as the fallback policy,
+ * and optionally a different policy may be acting as the primary policy. For
+ * all RPCs handled by this NRS head instance, NRS core will first attempt to
+ * enqueue the RPC using the primary policy (if any). The fallback policy is
+ * used in the following cases:
+ * - when there was no primary policy in the
+ *   ptlrpc_nrs_pol_state::NRS_POL_STATE_STARTED state at the time the request
+ *   was initialized.
+ * - when the primary policy that was at the
+ *   ptlrpc_nrs_pol_state::PTLRPC_NRS_POL_STATE_STARTED state at the time the
+ *   RPC was initialized, denoted it did not wish, or for some other reason was
+ *   not able to handle the request, by returning a non-valid NRS resource
+ *   reference.
+ * - when the primary policy that was at the
+ *   ptlrpc_nrs_pol_state::PTLRPC_NRS_POL_STATE_STARTED state at the time the
+ *   RPC was initialized, fails later during the request enqueueing stage.
+ *
+ * \see nrs_resource_get_safe()
+ * \see nrs_request_enqueue()
+ */
+struct ptlrpc_nrs {
+	spinlock_t			nrs_lock;
+	/** XXX Possibly replace svcpt->scp_req_lock with another lock here. */
+	/**
+	 * List of registered policies
+	 */
+	struct list_head		nrs_policy_list;
+	/**
+	 * List of policies with queued requests. Policies that have any
+	 * outstanding requests are queued here, and this list is queried
+	 * in a round-robin manner from NRS core when obtaining a request
+	 * for handling. This ensures that requests from policies that at some
+	 * point transition away from the
+	 * ptlrpc_nrs_pol_state::NRS_POL_STATE_STARTED state are drained.
+	 */
+	struct list_head		nrs_policy_queued;
+	/**
+	 * Service partition for this NRS head
+	 */
+	struct ptlrpc_service_part     *nrs_svcpt;
+	/**
+	 * Primary policy, which is the preferred policy for handling RPCs
+	 */
+	struct ptlrpc_nrs_policy       *nrs_policy_primary;
+	/**
+	 * Fallback policy, which is the backup policy for handling RPCs
+	 */
+	struct ptlrpc_nrs_policy       *nrs_policy_fallback;
+	/**
+	 * This NRS head handles either HP or regular requests
+	 */
+	enum ptlrpc_nrs_queue_type	nrs_queue_type;
+	/**
+	 * # queued requests from all policies in this NRS head
+	 */
+	unsigned long			nrs_req_queued;
+	/**
+	 * # scheduled requests from all policies in this NRS head
+	 */
+	unsigned long			nrs_req_started;
+	/**
+	 * # policies on this NRS
+	 */
+	unsigned int			nrs_num_pols;
+	/**
+	 * This NRS head is in progress of starting a policy
+	 */
+	unsigned int			nrs_policy_starting:1;
+	/**
+	 * In progress of shutting down the whole NRS head; used during
+	 * unregistration
+	 */
+	unsigned int			nrs_stopping:1;
+	/**
+	 * NRS policy is throttling request
+	 */
+	unsigned int			nrs_throttling:1;
+};
+
+#define NRS_POL_NAME_MAX		16
+#define NRS_POL_ARG_MAX			16
+
+struct ptlrpc_nrs_pol_desc;
+
+/**
+ * Service compatibility predicate; this determines whether a policy is adequate
+ * for handling RPCs of a particular PTLRPC service.
+ *
+ * XXX:This should give the same result during policy registration and
+ * unregistration, and for all partitions of a service; so the result should not
+ * depend on temporal service or other properties, that may influence the
+ * result.
+ */
+typedef bool (*nrs_pol_desc_compat_t)(const struct ptlrpc_service *svc,
+				      const struct ptlrpc_nrs_pol_desc *desc);
+
+struct ptlrpc_nrs_pol_conf {
+	/**
+	 * Human-readable policy name
+	 */
+	char				   nc_name[NRS_POL_NAME_MAX];
+	/**
+	 * NRS operations for this policy
+	 */
+	const struct ptlrpc_nrs_pol_ops   *nc_ops;
+	/**
+	 * Service compatibility predicate
+	 */
+	nrs_pol_desc_compat_t		   nc_compat;
+	/**
+	 * Set for policies that support a single ptlrpc service, i.e. ones that
+	 * have \a pd_compat set to nrs_policy_compat_one(). The variable value
+	 * depicts the name of the single service that such policies are
+	 * compatible with.
+	 */
+	const char			  *nc_compat_svc_name;
+	/**
+	 * Owner module for this policy descriptor; policies registering from a
+	 * different module to the one the NRS framework is held within
+	 * (currently ptlrpc), should set this field to THIS_MODULE.
+	 */
+	struct module			  *nc_owner;
+	/**
+	 * Policy registration flags; a bitmask of \e nrs_policy_flags
+	 */
+	unsigned int			   nc_flags;
+};
+
+/**
+ * NRS policy registering descriptor
+ *
+ * Is used to hold a description of a policy that can be passed to NRS core in
+ * order to register the policy with NRS heads in different PTLRPC services.
+ */
+struct ptlrpc_nrs_pol_desc {
+	/**
+	 * Human-readable policy name
+	 */
+	char					pd_name[NRS_POL_NAME_MAX];
+	/**
+	 * Link into nrs_core::nrs_policies
+	 */
+	struct list_head			pd_list;
+	/**
+	 * NRS operations for this policy
+	 */
+	const struct ptlrpc_nrs_pol_ops        *pd_ops;
+	/**
+	 * Service compatibility predicate
+	 */
+	nrs_pol_desc_compat_t			pd_compat;
+	/**
+	 * Set for policies that are compatible with only one PTLRPC service.
+	 *
+	 * \see ptlrpc_nrs_pol_conf::nc_compat_svc_name
+	 */
+	const char			       *pd_compat_svc_name;
+	/**
+	 * Owner module for this policy descriptor.
+	 *
+	 * We need to hold a reference to the module whenever we might make use
+	 * of any of the module's contents, i.e.
+	 * - If one or more instances of the policy are at a state where they
+	 *   might be handling a request, i.e.
+	 *   ptlrpc_nrs_pol_state::NRS_POL_STATE_STARTED or
+	 *   ptlrpc_nrs_pol_state::NRS_POL_STATE_STOPPING as we will have to
+	 *   call into the policy's ptlrpc_nrs_pol_ops() handlers. A reference
+	 *   is taken on the module when
+	 *   \e ptlrpc_nrs_pol_desc::pd_refs becomes 1, and released when it
+	 *   becomes 0, so that we hold only one reference to the module maximum
+	 *   at any time.
+	 *
+	 *   We do not need to hold a reference to the module, even though we
+	 *   might use code and data from the module, in the following cases:
+	 * - During external policy registration, because this should happen in
+	 *   the module's init() function, in which case the module is safe from
+	 *   removal because a reference is being held on the module by the
+	 *   kernel, and iirc kmod (and I guess module-init-tools also) will
+	 *   serialize any racing processes properly anyway.
+	 * - During external policy unregistration, because this should happen
+	 *   in a module's exit() function, and any attempts to start a policy
+	 *   instance would need to take a reference on the module, and this is
+	 *   not possible once we have reached the point where the exit()
+	 *   handler is called.
+	 * - During service registration and unregistration, as service setup
+	 *   and cleanup, and policy registration, unregistration and policy
+	 *   instance starting, are serialized by \e nrs_core::nrs_mutex, so
+	 *   as long as users adhere to the convention of registering policies
+	 *   in init() and unregistering them in module exit() functions, there
+	 *   should not be a race between these operations.
+	 * - During any policy-specific lprocfs operations, because a reference
+	 *   is held by the kernel on a proc entry that has been entered by a
+	 *   syscall, so as long as proc entries are removed during
+	 *   unregistration time, then unregistration and lprocfs operations
+	 *   will be properly serialized.
+	 */
+	struct module			       *pd_owner;
+	/**
+	 * Bitmask of \e nrs_policy_flags
+	 */
+	unsigned int				pd_flags;
+	/**
+	 * # of references on this descriptor
+	 */
+	atomic_t				pd_refs;
+};
+
+/**
+ * NRS policy state
+ *
+ * Policies transition from one state to the other during their lifetime
+ */
+enum ptlrpc_nrs_pol_state {
+	/**
+	 * Not a valid policy state.
+	 */
+	NRS_POL_STATE_INVALID,
+	/**
+	 * Policies are at this state either at the start of their life, or
+	 * transition here when the user selects a different policy to act
+	 * as the primary one.
+	 */
+	NRS_POL_STATE_STOPPED,
+	/**
+	 * Policy is progress of stopping
+	 */
+	NRS_POL_STATE_STOPPING,
+	/**
+	 * Policy is in progress of starting
+	 */
+	NRS_POL_STATE_STARTING,
+	/**
+	 * A policy is in this state in two cases:
+	 * - it is the fallback policy, which is always in this state.
+	 * - it has been activated by the user; i.e. it is the primary policy,
+	 */
+	NRS_POL_STATE_STARTED,
+};
+
+/**
+ * NRS policy information
+ *
+ * Used for obtaining information for the status of a policy via lprocfs
+ */
+struct ptlrpc_nrs_pol_info {
+	/**
+	 * Policy name
+	 */
+	char				pi_name[NRS_POL_NAME_MAX];
+	/**
+	 * Policy argument
+	 */
+	char				pi_arg[NRS_POL_ARG_MAX];
+	/**
+	 * Current policy state
+	 */
+	enum ptlrpc_nrs_pol_state	pi_state;
+	/**
+	 * # RPCs enqueued for later dispatching by the policy
+	 */
+	long				pi_req_queued;
+	/**
+	 * # RPCs started for dispatch by the policy
+	 */
+	long				pi_req_started;
+	/**
+	 * Is this a fallback policy?
+	 */
+	unsigned			pi_fallback:1;
+};
+
+/**
+ * NRS policy
+ *
+ * There is one instance of this for each policy in each NRS head of each
+ * PTLRPC service partition.
+ */
+struct ptlrpc_nrs_policy {
+	/**
+	 * Linkage into the NRS head's list of policies,
+	 * ptlrpc_nrs:nrs_policy_list
+	 */
+	struct list_head		pol_list;
+	/**
+	 * Linkage into the NRS head's list of policies with enqueued
+	 * requests ptlrpc_nrs:nrs_policy_queued
+	 */
+	struct list_head		pol_list_queued;
+	/**
+	 * Current state of this policy
+	 */
+	enum ptlrpc_nrs_pol_state	pol_state;
+	/**
+	 * Bitmask of nrs_policy_flags
+	 */
+	unsigned int			pol_flags;
+	/**
+	 * # RPCs enqueued for later dispatching by the policy
+	 */
+	long				pol_req_queued;
+	/**
+	 * # RPCs started for dispatch by the policy
+	 */
+	long				pol_req_started;
+	/**
+	 * Usage Reference count taken on the policy instance
+	 */
+	long				pol_ref;
+	/**
+	 * Human-readable policy argument
+	 */
+	char				pol_arg[NRS_POL_ARG_MAX];
+	/**
+	 * The NRS head this policy has been created at
+	 */
+	struct ptlrpc_nrs	       *pol_nrs;
+	/**
+	 * Private policy data; varies by policy type
+	 */
+	void			       *pol_private;
+	/**
+	 * Policy descriptor for this policy instance.
+	 */
+	struct ptlrpc_nrs_pol_desc     *pol_desc;
+};
+
+/**
+ * NRS resource
+ *
+ * Resources are embedded into two types of NRS entities:
+ * - Inside NRS policies, in the policy's private data in
+ *   ptlrpc_nrs_policy::pol_private
+ * - In objects that act as prime-level scheduling entities in different NRS
+ *   policies; e.g. on a policy that performs round robin or similar order
+ *   scheduling across client NIDs, there would be one NRS resource per unique
+ *   client NID. On a policy which performs round robin scheduling across
+ *   backend filesystem objects, there would be one resource associated with
+ *   each of the backend filesystem objects partaking in the scheduling
+ *   performed by the policy.
+ *
+ * NRS resources share a parent-child relationship, in which resources embedded
+ * in policy instances are the parent entities, with all scheduling entities
+ * a policy schedules across being the children, thus forming a simple resource
+ * hierarchy. This hierarchy may be extended with one or more levels in the
+ * future if the ability to have more than one primary policy is added.
+ *
+ * Upon request initialization, references to the then active NRS policies are
+ * taken and used to later handle the dispatching of the request with one of
+ * these policies.
+ *
+ * \see nrs_resource_get_safe()
+ * \see ptlrpc_nrs_req_add()
+ */
+struct ptlrpc_nrs_resource {
+	/**
+	 * This NRS resource's parent; is NULL for resources embedded in NRS
+	 * policy instances; i.e. those are top-level ones.
+	 */
+	struct ptlrpc_nrs_resource     *res_parent;
+	/**
+	 * The policy associated with this resource.
+	 */
+	struct ptlrpc_nrs_policy       *res_policy;
+};
+
+enum {
+	NRS_RES_FALLBACK,
+	NRS_RES_PRIMARY,
+	NRS_RES_MAX
+};
+
+#include "lustre_nrs_fifo.h"
+
+/**
+ * NRS request
+ *
+ * Instances of this object exist embedded within ptlrpc_request; the main
+ * purpose of this object is to hold references to the request's resources
+ * for the lifetime of the request, and to hold properties that policies use
+ * use for determining the request's scheduling priority.
+ **/
+struct ptlrpc_nrs_request {
+	/**
+	 * The request's resource hierarchy.
+	 */
+	struct ptlrpc_nrs_resource     *nr_res_ptrs[NRS_RES_MAX];
+	/**
+	 * Index into ptlrpc_nrs_request::nr_res_ptrs of the resource of the
+	 * policy that was used to enqueue the request.
+	 *
+	 * \see nrs_request_enqueue()
+	 */
+	unsigned int			nr_res_idx;
+	unsigned int			nr_initialized:1;
+	unsigned int			nr_enqueued:1;
+	unsigned int			nr_started:1;
+	unsigned int			nr_finalized:1;
+
+	/**
+	 * Policy-specific fields, used for determining a request's scheduling
+	 * priority, and other supporting functionality.
+	 */
+	union {
+		/**
+		 * Fields for the FIFO policy
+		 */
+		struct nrs_fifo_req	fifo;
+	} nr_u;
+	/**
+	 * Externally-registering policies may want to use this to allocate
+	 * their own request properties.
+	 */
+	void			       *ext;
+};
+
+/** @} nrs */
+#endif
diff --git a/drivers/staging/lustre/lustre/include/lustre_nrs_fifo.h b/drivers/staging/lustre/lustre/include/lustre_nrs_fifo.h
new file mode 100644
index 000000000000..3b5418eac6c4
--- /dev/null
+++ b/drivers/staging/lustre/lustre/include/lustre_nrs_fifo.h
@@ -0,0 +1,70 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License version 2 for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2014, Intel Corporation.
+ *
+ * Copyright 2012 Xyratex Technology Limited
+ */
+/*
+ *
+ * Network Request Scheduler (NRS) First-in First-out (FIFO) policy
+ *
+ */
+
+#ifndef _LUSTRE_NRS_FIFO_H
+#define _LUSTRE_NRS_FIFO_H
+
+/* \name fifo
+ *
+ * FIFO policy
+ *
+ * This policy is a logical wrapper around previous, non-NRS functionality.
+ * It dispatches RPCs in the same order as they arrive from the network. This
+ * policy is currently used as the fallback policy, and the only enabled policy
+ * on all NRS heads of all PTLRPC service partitions.
+ * @{
+ */
+
+/**
+ * Private data structure for the FIFO policy
+ */
+struct nrs_fifo_head {
+	/**
+	 * Resource object for policy instance.
+	 */
+	struct ptlrpc_nrs_resource	fh_res;
+	/**
+	 * List of queued requests.
+	 */
+	struct list_head		fh_list;
+	/**
+	 * For debugging purposes.
+	 */
+	__u64				fh_sequence;
+};
+
+struct nrs_fifo_req {
+	struct list_head	fr_list;
+	__u64			fr_sequence;
+};
+
+/** @} fifo */
+#endif
diff --git a/drivers/staging/lustre/lustre/include/lustre_req_layout.h b/drivers/staging/lustre/lustre/include/lustre_req_layout.h
index a13558e53274..fbcd39572cd0 100644
--- a/drivers/staging/lustre/lustre/include/lustre_req_layout.h
+++ b/drivers/staging/lustre/lustre/include/lustre_req_layout.h
@@ -148,13 +148,12 @@ extern struct req_format RQF_MDS_GETATTR;
  */
 extern struct req_format RQF_MDS_GETATTR_NAME;
 extern struct req_format RQF_MDS_CLOSE;
-extern struct req_format RQF_MDS_RELEASE_CLOSE;
+extern struct req_format RQF_MDS_INTENT_CLOSE;
 extern struct req_format RQF_MDS_CONNECT;
 extern struct req_format RQF_MDS_DISCONNECT;
 extern struct req_format RQF_MDS_GET_INFO;
 extern struct req_format RQF_MDS_READPAGE;
 extern struct req_format RQF_MDS_WRITEPAGE;
-extern struct req_format RQF_MDS_DONE_WRITING;
 extern struct req_format RQF_MDS_REINT;
 extern struct req_format RQF_MDS_REINT_CREATE;
 extern struct req_format RQF_MDS_REINT_CREATE_ACL;
@@ -166,10 +165,9 @@ extern struct req_format RQF_MDS_REINT_LINK;
 extern struct req_format RQF_MDS_REINT_RENAME;
 extern struct req_format RQF_MDS_REINT_SETATTR;
 extern struct req_format RQF_MDS_REINT_SETXATTR;
-extern struct req_format RQF_MDS_QUOTACHECK;
 extern struct req_format RQF_MDS_QUOTACTL;
-extern struct req_format RQF_QC_CALLBACK;
 extern struct req_format RQF_MDS_SWAP_LAYOUTS;
+extern struct req_format RQF_MDS_REINT_MIGRATE;
 /* MDS hsm formats */
 extern struct req_format RQF_MDS_HSM_STATE_GET;
 extern struct req_format RQF_MDS_HSM_STATE_SET;
@@ -181,7 +179,6 @@ extern struct req_format RQF_MDS_HSM_REQUEST;
 /* OST req_format */
 extern struct req_format RQF_OST_CONNECT;
 extern struct req_format RQF_OST_DISCONNECT;
-extern struct req_format RQF_OST_QUOTACHECK;
 extern struct req_format RQF_OST_QUOTACTL;
 extern struct req_format RQF_OST_GETATTR;
 extern struct req_format RQF_OST_SETATTR;
diff --git a/drivers/staging/lustre/lustre/include/lustre_sec.h b/drivers/staging/lustre/lustre/include/lustre_sec.h
index 90c183424802..03a970bcac55 100644
--- a/drivers/staging/lustre/lustre/include/lustre_sec.h
+++ b/drivers/staging/lustre/lustre/include/lustre_sec.h
@@ -50,6 +50,7 @@ struct brw_page;
 /* Linux specific */
 struct key;
 struct seq_file;
+struct lustre_cfg;
 
 /*
  * forward declaration
@@ -1029,6 +1030,8 @@ int  sptlrpc_target_export_check(struct obd_export *exp,
 
 /* bulk security api */
 void sptlrpc_enc_pool_put_pages(struct ptlrpc_bulk_desc *desc);
+int get_free_pages_in_pool(void);
+int pool_is_at_full_capacity(void);
 
 int sptlrpc_cli_wrap_bulk(struct ptlrpc_request *req,
 			  struct ptlrpc_bulk_desc *desc);
diff --git a/drivers/staging/lustre/lustre/include/lustre_swab.h b/drivers/staging/lustre/lustre/include/lustre_swab.h
new file mode 100644
index 000000000000..26d01c2d6633
--- /dev/null
+++ b/drivers/staging/lustre/lustre/include/lustre_swab.h
@@ -0,0 +1,102 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ *
+ * Copyright (c) 2011, 2014, Intel Corporation.
+ *
+ * Copyright 2015 Cray Inc, all rights reserved.
+ * Author: Ben Evans.
+ *
+ * We assume all nodes are either little-endian or big-endian, and we
+ * always send messages in the sender's native format.  The receiver
+ * detects the message format by checking the 'magic' field of the message
+ * (see lustre_msg_swabbed() below).
+ *
+ * Each wire type has corresponding 'lustre_swab_xxxtypexxx()' routines
+ * are implemented in ptlrpc/lustre_swab.c.  These 'swabbers' convert the
+ * type from "other" endian, in-place in the message buffer.
+ *
+ * A swabber takes a single pointer argument.  The caller must already have
+ * verified that the length of the message buffer >= sizeof (type).
+ *
+ * For variable length types, a second 'lustre_swab_v_xxxtypexxx()' routine
+ * may be defined that swabs just the variable part, after the caller has
+ * verified that the message buffer is large enough.
+ */
+
+#ifndef _LUSTRE_SWAB_H_
+#define _LUSTRE_SWAB_H_
+
+#include "lustre/lustre_idl.h"
+
+void lustre_swab_ptlrpc_body(struct ptlrpc_body *pb);
+void lustre_swab_connect(struct obd_connect_data *ocd);
+void lustre_swab_hsm_user_state(struct hsm_user_state *hus);
+void lustre_swab_hsm_state_set(struct hsm_state_set *hss);
+void lustre_swab_obd_statfs(struct obd_statfs *os);
+void lustre_swab_obd_ioobj(struct obd_ioobj *ioo);
+void lustre_swab_niobuf_remote(struct niobuf_remote *nbr);
+void lustre_swab_ost_lvb_v1(struct ost_lvb_v1 *lvb);
+void lustre_swab_ost_lvb(struct ost_lvb *lvb);
+void lustre_swab_obd_quotactl(struct obd_quotactl *q);
+void lustre_swab_lquota_lvb(struct lquota_lvb *lvb);
+void lustre_swab_generic_32s(__u32 *val);
+void lustre_swab_mdt_body(struct mdt_body *b);
+void lustre_swab_mdt_ioepoch(struct mdt_ioepoch *b);
+void lustre_swab_mdt_rec_setattr(struct mdt_rec_setattr *sa);
+void lustre_swab_mdt_rec_reint(struct mdt_rec_reint *rr);
+void lustre_swab_lmv_desc(struct lmv_desc *ld);
+void lustre_swab_lmv_mds_md(union lmv_mds_md *lmm);
+void lustre_swab_lov_desc(struct lov_desc *ld);
+void lustre_swab_gl_desc(union ldlm_gl_desc *desc);
+void lustre_swab_ldlm_intent(struct ldlm_intent *i);
+void lustre_swab_ldlm_request(struct ldlm_request *rq);
+void lustre_swab_ldlm_reply(struct ldlm_reply *r);
+void lustre_swab_mgs_target_info(struct mgs_target_info *oinfo);
+void lustre_swab_mgs_nidtbl_entry(struct mgs_nidtbl_entry *oinfo);
+void lustre_swab_mgs_config_body(struct mgs_config_body *body);
+void lustre_swab_mgs_config_res(struct mgs_config_res *body);
+void lustre_swab_ost_body(struct ost_body *b);
+void lustre_swab_ost_last_id(__u64 *id);
+void lustre_swab_fiemap(struct fiemap *fiemap);
+void lustre_swab_lov_user_md_v1(struct lov_user_md_v1 *lum);
+void lustre_swab_lov_user_md_v3(struct lov_user_md_v3 *lum);
+void lustre_swab_lov_user_md_objects(struct lov_user_ost_data *lod,
+				     int stripe_count);
+void lustre_swab_lov_mds_md(struct lov_mds_md *lmm);
+void lustre_swab_lustre_capa(struct lustre_capa *c);
+void lustre_swab_lustre_capa_key(struct lustre_capa_key *k);
+void lustre_swab_fid2path(struct getinfo_fid2path *gf);
+void lustre_swab_layout_intent(struct layout_intent *li);
+void lustre_swab_hsm_user_state(struct hsm_user_state *hus);
+void lustre_swab_hsm_current_action(struct hsm_current_action *action);
+void lustre_swab_hsm_progress_kernel(struct hsm_progress_kernel *hpk);
+void lustre_swab_hsm_user_state(struct hsm_user_state *hus);
+void lustre_swab_hsm_user_item(struct hsm_user_item *hui);
+void lustre_swab_hsm_request(struct hsm_request *hr);
+void lustre_swab_swap_layouts(struct mdc_swap_layouts *msl);
+void lustre_swab_close_data(struct close_data *data);
+void lustre_swab_lmv_user_md(struct lmv_user_md *lum);
+
+#endif
diff --git a/drivers/staging/lustre/lustre/include/obd.h b/drivers/staging/lustre/lustre/include/obd.h
index f6fc4dd05bd6..0f48e9c3d9e3 100644
--- a/drivers/staging/lustre/lustre/include/obd.h
+++ b/drivers/staging/lustre/lustre/include/obd.h
@@ -73,70 +73,17 @@ static inline void loi_init(struct lov_oinfo *loi)
 {
 }
 
-/*
- * If we are unable to get the maximum object size from the OST in
- * ocd_maxbytes using OBD_CONNECT_MAXBYTES, then we fall back to using
- * the old maximum object size from ext3.
- */
-#define LUSTRE_EXT3_STRIPE_MAXBYTES 0x1fffffff000ULL
-
-struct lov_stripe_md {
-	atomic_t     lsm_refc;
-	spinlock_t	lsm_lock;
-	pid_t	    lsm_lock_owner; /* debugging */
-
-	/* maximum possible file size, might change as OSTs status changes,
-	 * e.g. disconnected, deactivated
-	 */
-	__u64		lsm_maxbytes;
-	struct ost_id	lsm_oi;
-	__u32		lsm_magic;
-	__u32		lsm_stripe_size;
-	__u32		lsm_pattern;	/* striping pattern (RAID0, RAID1) */
-	__u16		lsm_stripe_count;
-	__u16		lsm_layout_gen;
-	char		lsm_pool_name[LOV_MAXPOOLNAME + 1];
-	struct lov_oinfo *lsm_oinfo[0];
-};
-
-static inline bool lsm_is_released(struct lov_stripe_md *lsm)
-{
-	return !!(lsm->lsm_pattern & LOV_PATTERN_F_RELEASED);
-}
-
-static inline bool lsm_has_objects(struct lov_stripe_md *lsm)
-{
-	if (!lsm)
-		return false;
-	if (lsm_is_released(lsm))
-		return false;
-	return true;
-}
-
-static inline int lov_stripe_md_size(unsigned int stripe_count)
-{
-	struct lov_stripe_md lsm;
-
-	return sizeof(lsm) + stripe_count * sizeof(lsm.lsm_oinfo[0]);
-}
-
+struct lov_stripe_md;
 struct obd_info;
 
 typedef int (*obd_enqueue_update_f)(void *cookie, int rc);
 
 /* obd info for a particular level (lov, osc). */
 struct obd_info {
-	/* Flags used for set request specific flags:
-	   - while lock handling, the flags obtained on the enqueue
-	   request are set here.
-	   - while stats, the flags used for control delay/resend.
-	   - while setattr, the flags used for distinguish punch operation
-	 */
+	/* OBD_STATFS_* flags */
 	__u64		   oi_flags;
 	/* lsm data specific for every OSC. */
 	struct lov_stripe_md   *oi_md;
-	/* obdo data specific for every OSC, if needed at all. */
-	struct obdo	    *oi_oa;
 	/* statfs data specific for every OSC, if needed at all. */
 	struct obd_statfs      *oi_osfs;
 	/* An update callback which is called to update some data on upper
@@ -204,7 +151,6 @@ enum obd_cl_sem_lock_class {
  * on the MDS.
  */
 #define OBD_MAX_DEFAULT_EA_SIZE		4096
-#define OBD_MAX_DEFAULT_COOKIE_SIZE	4096
 
 struct mdc_rpc_lock;
 struct obd_import;
@@ -214,7 +160,7 @@ struct client_obd {
 	struct obd_import       *cl_import; /* ptlrpc connection state */
 	size_t			 cl_conn_count;
 	/*
-	 * Cache maximum and default values for easize and cookiesize. This is
+	 * Cache maximum and default values for easize. This is
 	 * strictly a performance optimization to minimize calls to
 	 * obd_size_diskmd(). The default values are used to calculate the
 	 * initial size of a request buffer. The ptlrpc layer will resize the
@@ -235,18 +181,6 @@ struct client_obd {
 	 * run-time if a larger observed size is advertised by the MDT.
 	 */
 	u32			 cl_max_mds_easize;
-	/* Default cookie size for llog cookies (see struct llog_cookie). It is
-	 * initialized to zero at mount-time, then it tracks the largest
-	 * observed cookie size advertised by the MDT, up to a maximum value of
-	 * OBD_MAX_DEFAULT_COOKIE_SIZE. Note that llog_cookies are not
-	 * used by clients communicating with MDS versions 2.4.0 and later.
-	 */
-	u32			 cl_default_mds_cookiesize;
-	/* Maximum possible cookie size computed at mount-time based on
-	 * the number of OSTs in the filesystem. May be increased at
-	 * run-time if a larger observed size is advertised by the MDT.
-	 */
-	u32			 cl_max_mds_cookiesize;
 
 	enum lustre_sec_part     cl_sp_me;
 	enum lustre_sec_part     cl_sp_to;
@@ -313,15 +247,42 @@ struct client_obd {
 	struct obd_histogram     cl_read_offset_hist;
 	struct obd_histogram     cl_write_offset_hist;
 
-	/* lru for osc caching pages */
+	/* LRU for osc caching pages */
 	struct cl_client_cache	*cl_cache;
-	struct list_head	 cl_lru_osc; /* member of cl_cache->ccc_lru */
+	/** member of cl_cache->ccc_lru */
+	struct list_head	 cl_lru_osc;
+	/** # of available LRU slots left in the per-OSC cache.
+	 * Available LRU slots are shared by all OSCs of the same file system,
+	 * therefore this is a pointer to cl_client_cache::ccc_lru_left.
+	 */
 	atomic_long_t		*cl_lru_left;
+	/** # of busy LRU pages. A page is considered busy if it's in writeback
+	 * queue, or in transfer. Busy pages can't be discarded so they are not
+	 * in LRU cache.
+	 */
 	atomic_long_t		 cl_lru_busy;
+	/** # of LRU pages in the cache for this client_obd */
 	atomic_long_t		 cl_lru_in_list;
+	/** # of threads are shrinking LRU cache. To avoid contention, it's not
+	 * allowed to have multiple threads shrinking LRU cache.
+	 */
 	atomic_t		 cl_lru_shrinkers;
-	struct list_head	 cl_lru_list; /* lru page list */
-	spinlock_t		 cl_lru_list_lock; /* page list protector */
+	/** The time when this LRU cache was last used. */
+	time64_t		 cl_lru_last_used;
+	/** stats: how many reclaims have happened for this client_obd.
+	 * reclaim and shrink - shrink is async, voluntarily rebalancing;
+	 * reclaim is sync, initiated by IO thread when the LRU slots are
+	 * in shortage.
+	 */
+	u64			 cl_lru_reclaim;
+	/** List of LRU pages for this client_obd */
+	struct list_head	 cl_lru_list;
+	/** Lock for LRU page list */
+	spinlock_t		 cl_lru_list_lock;
+	/** # of unstable pages in this client_obd.
+	 * An unstable page is a page state that WRITE RPC has finished but
+	 * the transaction has NOT yet committed.
+	 */
 	atomic_long_t		 cl_unstable_count;
 
 	/* number of in flight destroy rpcs is limited to max_rpcs_in_flight */
@@ -329,7 +290,17 @@ struct client_obd {
 	wait_queue_head_t	      cl_destroy_waitq;
 
 	struct mdc_rpc_lock     *cl_rpc_lock;
-	struct mdc_rpc_lock     *cl_close_lock;
+
+	/* modify rpcs in flight
+	 * currently used for metadata only
+	 */
+	spinlock_t		 cl_mod_rpcs_lock;
+	u16			 cl_max_mod_rpcs_in_flight;
+	u16			 cl_mod_rpcs_in_flight;
+	u16			 cl_close_rpcs_in_flight;
+	wait_queue_head_t	 cl_mod_rpcs_waitq;
+	unsigned long		*cl_mod_tag_bitmap;
+	struct obd_histogram	 cl_mod_rpcs_hist;
 
 	/* mgc datastruct */
 	atomic_t	     cl_mgc_refcount;
@@ -345,13 +316,6 @@ struct client_obd {
 	/* also protected by the poorly named _loi_list_lock lock above */
 	struct osc_async_rc      cl_ar;
 
-	/* used by quotacheck when the servers are older than 2.4 */
-	int		      cl_qchk_stat; /* quotacheck stat of the peer */
-#define CL_NOT_QUOTACHECKED 1   /* client->cl_qchk_stat init value */
-#if OBD_OCD_VERSION(2, 7, 53, 0) < LUSTRE_VERSION_CODE
-#warning "please consider removing quotacheck compatibility code"
-#endif
-
 	/* sequence manager */
 	struct lu_client_seq    *cl_seq;
 
@@ -454,8 +418,6 @@ struct lmv_obd {
 	int			connected;
 	int			max_easize;
 	int			max_def_easize;
-	int			max_cookiesize;
-	int			max_def_cookiesize;
 
 	u32			tgts_size; /* size of tgts array */
 	struct lmv_tgt_desc	**tgts;
@@ -469,9 +431,9 @@ struct niobuf_local {
 	__u32		lnb_page_offset;
 	__u32		lnb_len;
 	__u32		lnb_flags;
+	int		lnb_rc;
 	struct page	*lnb_page;
 	void		*lnb_data;
-	int		lnb_rc;
 };
 
 #define LUSTRE_FLD_NAME	 "fld"
@@ -512,21 +474,6 @@ struct niobuf_local {
 /* Don't conflict with on-wire flags OBD_BRW_WRITE, etc */
 #define N_LOCAL_TEMP_PAGE 0x10000000
 
-struct obd_trans_info {
-	__u64		    oti_xid;
-	/* Only used on the server side for tracking acks. */
-	struct oti_req_ack_lock {
-		struct lustre_handle lock;
-		__u32		mode;
-	}			oti_ack_locks[4];
-	void		    *oti_handle;
-	struct llog_cookie       oti_onecookie;
-	struct llog_cookie      *oti_logcookies;
-
-	/** VBR: versions */
-	__u64		    oti_pre_version;
-};
-
 /*
  * Events signalled through obd_notify() upcall-chain.
  */
@@ -587,15 +534,14 @@ struct lvfs_run_ctxt {
 
 struct obd_device {
 	struct obd_type	*obd_type;
-	__u32		   obd_magic;
+	u32			 obd_magic; /* OBD_DEVICE_MAGIC */
+	int			 obd_minor; /* device number: lctl dl */
+	struct lu_device	*obd_lu_dev;
 
 	/* common and UUID name of this device */
-	char		    obd_name[MAX_OBD_NAME];
-	struct obd_uuid	 obd_uuid;
-
-	struct lu_device       *obd_lu_dev;
+	struct obd_uuid		 obd_uuid;
+	char			 obd_name[MAX_OBD_NAME];
 
-	int		     obd_minor;
 	/* bitfield modification is protected by obd_dev_lock */
 	unsigned long obd_attached:1,      /* finished attach */
 		      obd_set_up:1,	/* finished setup */
@@ -619,22 +565,22 @@ struct obd_device {
 	unsigned long obd_recovery_expired:1;
 	/* uuid-export hash body */
 	struct cfs_hash	     *obd_uuid_hash;
-	atomic_t	    obd_refcount;
 	wait_queue_head_t	     obd_refcount_waitq;
 	struct list_head	      obd_exports;
 	struct list_head	      obd_unlinked_exports;
 	struct list_head	      obd_delayed_exports;
+	atomic_t			obd_refcount;
 	int		     obd_num_exports;
 	spinlock_t		obd_nid_lock;
 	struct ldlm_namespace  *obd_namespace;
 	struct ptlrpc_client	obd_ldlm_client; /* XXX OST/MDS only */
 	/* a spinlock is OK for what we do now, may need a semaphore later */
 	spinlock_t		obd_dev_lock; /* protect OBD bitfield above */
-	struct mutex		obd_dev_mutex;
-	__u64			obd_last_committed;
 	spinlock_t		obd_osfs_lock;
 	struct obd_statfs	obd_osfs;       /* locked by obd_osfs_lock */
 	__u64			obd_osfs_age;
+	u64			obd_last_committed;
+	struct mutex		obd_dev_mutex;
 	struct lvfs_run_ctxt	obd_lvfs_ctxt;
 	struct obd_llog_group	obd_olg;	/* default llog group */
 	struct obd_device	*obd_observer;
@@ -648,12 +594,13 @@ struct obd_device {
 		struct lov_obd lov;
 		struct lmv_obd lmv;
 	} u;
+
 	/* Fields used by LProcFS */
-	unsigned int	   obd_cntr_base;
-	struct lprocfs_stats  *obd_stats;
+	struct lprocfs_stats	*obd_stats;
+	unsigned int		 obd_cntr_base;
 
-	unsigned int	   md_cntr_base;
-	struct lprocfs_stats  *md_stats;
+	struct lprocfs_stats	*md_stats;
+	unsigned int		 md_cntr_base;
 
 	struct dentry		*obd_debugfs_entry;
 	struct dentry		*obd_svc_debugfs_entry;
@@ -665,9 +612,11 @@ struct obd_device {
 	/**
 	 * Ldlm pool part. Save last calculated SLV and Limit.
 	 */
-	rwlock_t		obd_pool_lock;
-	int		    obd_pool_limit;
-	__u64		  obd_pool_slv;
+	rwlock_t		 obd_pool_lock;
+	u64			 obd_pool_slv;
+	int			 obd_pool_limit;
+
+	int			 obd_conn_inprogress;
 
 	/**
 	 * A list of outstanding class_incref()'s against this obd. For
@@ -675,19 +624,10 @@ struct obd_device {
 	 */
 	struct lu_ref	  obd_reference;
 
-	int		       obd_conn_inprogress;
-
 	struct kobject		obd_kobj; /* sysfs object */
 	struct completion	obd_kobj_unregister;
 };
 
-enum obd_cleanup_stage {
-/* Special case hack for MDS LOVs */
-	OBD_CLEANUP_EARLY,
-/* can be directly mapped to .ldto_device_fini() */
-	OBD_CLEANUP_EXPORTS,
-};
-
 /* get/set_info keys */
 #define KEY_ASYNC	       "async"
 #define KEY_CHANGELOG_CLEAR     "changelog_clear"
@@ -704,7 +644,6 @@ enum obd_cleanup_stage {
 #define KEY_INTERMDS	    "inter_mds"
 #define KEY_LAST_ID	     "last_id"
 #define KEY_LAST_FID		"last_fid"
-#define KEY_LOVDESC	     "lovdesc"
 #define KEY_MAX_EASIZE		"max_easize"
 #define KEY_DEFAULT_EASIZE	"default_easize"
 #define KEY_MGSSEC	      "mgssec"
@@ -720,22 +659,6 @@ enum obd_cleanup_stage {
 
 struct lu_context;
 
-/* /!\ must be coherent with include/linux/namei.h on patched kernel */
-#define IT_OPEN     (1 << 0)
-#define IT_CREAT    (1 << 1)
-#define IT_READDIR  (1 << 2)
-#define IT_GETATTR  (1 << 3)
-#define IT_LOOKUP   (1 << 4)
-#define IT_UNLINK   (1 << 5)
-#define IT_TRUNC    (1 << 6)
-#define IT_GETXATTR (1 << 7)
-#define IT_EXEC     (1 << 8)
-#define IT_PIN      (1 << 9)
-#define IT_LAYOUT   (1 << 10)
-#define IT_QUOTA_DQACQ (1 << 11)
-#define IT_QUOTA_CONN  (1 << 12)
-#define IT_SETXATTR (1 << 13)
-
 static inline int it_to_lock_mode(struct lookup_intent *it)
 {
 	/* CREAT needs to be tested before open (both could be set) */
@@ -755,6 +678,14 @@ static inline int it_to_lock_mode(struct lookup_intent *it)
 	return -EINVAL;
 }
 
+enum md_op_flags {
+	MF_MDC_CANCEL_FID1	= BIT(0),
+	MF_MDC_CANCEL_FID2      = BIT(1),
+	MF_MDC_CANCEL_FID3      = BIT(2),
+	MF_MDC_CANCEL_FID4      = BIT(3),
+	MF_GET_MDT_IDX          = BIT(4),
+};
+
 enum md_cli_flags {
 	CLI_SET_MEA	= BIT(0),
 	CLI_RM_ENTRY	= BIT(1),
@@ -789,8 +720,6 @@ struct md_op_data {
 	__u64		   op_valid;
 	loff_t		  op_attr_blocks;
 
-	/* Size-on-MDS epoch and flags. */
-	__u64		   op_ioepoch;
 	__u32		   op_flags;
 
 	/* Various operation flags. */
@@ -839,15 +768,13 @@ struct obd_ops {
 	int (*iocontrol)(unsigned int cmd, struct obd_export *exp, int len,
 			 void *karg, void __user *uarg);
 	int (*get_info)(const struct lu_env *env, struct obd_export *,
-			__u32 keylen, void *key, __u32 *vallen, void *val,
-			struct lov_stripe_md *lsm);
+			__u32 keylen, void *key, __u32 *vallen, void *val);
 	int (*set_info_async)(const struct lu_env *, struct obd_export *,
 			      __u32 keylen, void *key,
 			      __u32 vallen, void *val,
 			      struct ptlrpc_request_set *set);
 	int (*setup)(struct obd_device *dev, struct lustre_cfg *cfg);
-	int (*precleanup)(struct obd_device *dev,
-			  enum obd_cleanup_stage cleanup_stage);
+	int (*precleanup)(struct obd_device *dev);
 	int (*cleanup)(struct obd_device *dev);
 	int (*process_config)(struct obd_device *dev, u32 len, void *data);
 	int (*postrecov)(struct obd_device *dev);
@@ -887,35 +814,23 @@ struct obd_ops {
 		      struct obd_statfs *osfs, __u64 max_age, __u32 flags);
 	int (*statfs_async)(struct obd_export *exp, struct obd_info *oinfo,
 			    __u64 max_age, struct ptlrpc_request_set *set);
-	int (*packmd)(struct obd_export *exp, struct lov_mds_md **disk_tgt,
-		      struct lov_stripe_md *mem_src);
-	int (*unpackmd)(struct obd_export *exp,
-			struct lov_stripe_md **mem_tgt,
-			struct lov_mds_md *disk_src, int disk_len);
 	int (*create)(const struct lu_env *env, struct obd_export *exp,
-		      struct obdo *oa, struct obd_trans_info *oti);
+		      struct obdo *oa);
 	int (*destroy)(const struct lu_env *env, struct obd_export *exp,
-		       struct obdo *oa, struct obd_trans_info *oti);
+		       struct obdo *oa);
 	int (*setattr)(const struct lu_env *, struct obd_export *exp,
-		       struct obd_info *oinfo, struct obd_trans_info *oti);
-	int (*setattr_async)(struct obd_export *exp, struct obd_info *oinfo,
-			     struct obd_trans_info *oti,
-			     struct ptlrpc_request_set *rqset);
+		       struct obdo *oa);
 	int (*getattr)(const struct lu_env *env, struct obd_export *exp,
-		       struct obd_info *oinfo);
-	int (*getattr_async)(struct obd_export *exp, struct obd_info *oinfo,
-			     struct ptlrpc_request_set *set);
+		       struct obdo *oa);
 	int (*preprw)(const struct lu_env *env, int cmd,
 		      struct obd_export *exp, struct obdo *oa, int objcount,
 		      struct obd_ioobj *obj, struct niobuf_remote *remote,
-		      int *nr_pages, struct niobuf_local *local,
-		      struct obd_trans_info *oti);
+		      int *nr_pages, struct niobuf_local *local);
 	int (*commitrw)(const struct lu_env *env, int cmd,
 			struct obd_export *exp, struct obdo *oa,
 			int objcount, struct obd_ioobj *obj,
 			struct niobuf_remote *remote, int pages,
-			struct niobuf_local *local,
-			struct obd_trans_info *oti, int rc);
+			struct niobuf_local *local, int rc);
 	int (*init_export)(struct obd_export *exp);
 	int (*destroy_export)(struct obd_export *exp);
 
@@ -930,8 +845,6 @@ struct obd_ops {
 	struct obd_uuid *(*get_uuid)(struct obd_export *exp);
 
 	/* quota methods */
-	int (*quotacheck)(struct obd_device *, struct obd_export *,
-			  struct obd_quotactl *);
 	int (*quotactl)(struct obd_device *, struct obd_export *,
 			struct obd_quotactl *);
 
@@ -954,7 +867,7 @@ struct obd_ops {
 /* lmv structures */
 struct lustre_md {
 	struct mdt_body	 *body;
-	struct lov_stripe_md    *lsm;
+	struct lu_buf		 layout;
 	struct lmv_stripe_md    *lmv;
 #ifdef CONFIG_FS_POSIX_ACL
 	struct posix_acl	*posix_acl;
@@ -992,10 +905,8 @@ struct md_ops {
 	int (*create)(struct obd_export *, struct md_op_data *,
 		      const void *, size_t, umode_t, uid_t, gid_t,
 		      cfs_cap_t, __u64, struct ptlrpc_request **);
-	int (*done_writing)(struct obd_export *, struct md_op_data  *,
-			    struct md_open_data *);
 	int (*enqueue)(struct obd_export *, struct ldlm_enqueue_info *,
-		       const ldlm_policy_data_t *,
+		       const union ldlm_policy_data *,
 		       struct lookup_intent *, struct md_op_data *,
 		       struct lustre_handle *, __u64);
 	int (*getattr)(struct obd_export *, struct md_op_data *,
@@ -1012,8 +923,7 @@ struct md_ops {
 		      const char *, size_t, const char *, size_t,
 		      struct ptlrpc_request **);
 	int (*setattr)(struct obd_export *, struct md_op_data *, void *,
-		       size_t, void *, size_t, struct ptlrpc_request **,
-			 struct md_open_data **mod);
+		       size_t, struct ptlrpc_request **);
 	int (*sync)(struct obd_export *, const struct lu_fid *,
 		    struct ptlrpc_request **);
 	int (*read_page)(struct obd_export *, struct md_op_data *,
@@ -1030,7 +940,7 @@ struct md_ops {
 			u64, const char *, const char *, int, int, int,
 			struct ptlrpc_request **);
 
-	int (*init_ea_size)(struct obd_export *, u32, u32, u32, u32);
+	int (*init_ea_size)(struct obd_export *, u32, u32);
 
 	int (*get_lustre_md)(struct obd_export *, struct ptlrpc_request *,
 			     struct obd_export *, struct obd_export *,
@@ -1052,11 +962,11 @@ struct md_ops {
 
 	enum ldlm_mode (*lock_match)(struct obd_export *, __u64,
 				     const struct lu_fid *, enum ldlm_type,
-				     ldlm_policy_data_t *, enum ldlm_mode,
+				     union ldlm_policy_data *, enum ldlm_mode,
 				     struct lustre_handle *);
 
 	int (*cancel_unused)(struct obd_export *, const struct lu_fid *,
-			     ldlm_policy_data_t *, enum ldlm_mode,
+			     union ldlm_policy_data *, enum ldlm_mode,
 			     enum ldlm_cancel_flags flags, void *opaque);
 
 	int (*get_fid_from_lsm)(struct obd_export *,
@@ -1071,6 +981,8 @@ struct md_ops {
 	int (*revalidate_lock)(struct obd_export *, struct lookup_intent *,
 			       struct lu_fid *, __u64 *bits);
 
+	int (*unpackmd)(struct obd_export *exp, struct lmv_stripe_md **plsm,
+			const union lmv_mds_md *lmv, size_t lmv_size);
 	/*
 	 * NOTE: If adding ops, add another LPROCFS_MD_OP_INIT() line to
 	 * lprocfs_alloc_md_stats() in obdclass/lprocfs_status.c. Also, add a
@@ -1078,33 +990,6 @@ struct md_ops {
 	 */
 };
 
-struct lsm_operations {
-	void (*lsm_free)(struct lov_stripe_md *);
-	void (*lsm_stripe_by_index)(struct lov_stripe_md *, int *, u64 *,
-				    u64 *);
-	void (*lsm_stripe_by_offset)(struct lov_stripe_md *, int *, u64 *,
-				     u64 *);
-	int (*lsm_lmm_verify)(struct lov_mds_md *lmm, int lmm_bytes,
-			      __u16 *stripe_count);
-	int (*lsm_unpackmd)(struct lov_obd *lov, struct lov_stripe_md *lsm,
-			    struct lov_mds_md *lmm);
-};
-
-extern const struct lsm_operations lsm_v1_ops;
-extern const struct lsm_operations lsm_v3_ops;
-static inline const struct lsm_operations *lsm_op_find(int magic)
-{
-	switch (magic) {
-	case LOV_MAGIC_V1:
-	       return &lsm_v1_ops;
-	case LOV_MAGIC_V3:
-	       return &lsm_v3_ops;
-	default:
-	       CERROR("Cannot recognize lsm_magic %08x\n", magic);
-	       return NULL;
-	}
-}
-
 static inline struct md_open_data *obd_mod_alloc(void)
 {
 	struct md_open_data *mod;
diff --git a/drivers/staging/lustre/lustre/include/obd_class.h b/drivers/staging/lustre/lustre/include/obd_class.h
index 16094dbec08b..7ec25202cd22 100644
--- a/drivers/staging/lustre/lustre/include/obd_class.h
+++ b/drivers/staging/lustre/lustre/include/obd_class.h
@@ -100,6 +100,13 @@ int obd_get_request_slot(struct client_obd *cli);
 void obd_put_request_slot(struct client_obd *cli);
 __u32 obd_get_max_rpcs_in_flight(struct client_obd *cli);
 int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max);
+int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, u16 max);
+int obd_mod_rpc_stats_seq_show(struct client_obd *cli, struct seq_file *seq);
+
+u16 obd_get_mod_rpc_slot(struct client_obd *cli, u32 opc,
+			 struct lookup_intent *it);
+void obd_put_mod_rpc_slot(struct client_obd *cli, u32 opc,
+			  struct lookup_intent *it, u16 tag);
 
 struct llog_handle;
 struct llog_rec_hdr;
@@ -175,10 +182,13 @@ struct lustre_profile {
 	char	    *lp_profile;
 	char	    *lp_dt;
 	char	    *lp_md;
+	int			lp_refs;
+	bool			lp_list_deleted;
 };
 
 struct lustre_profile *class_get_profile(const char *prof);
 void class_del_profile(const char *prof);
+void class_put_profile(struct lustre_profile *lprof);
 void class_del_profiles(void);
 
 #if LUSTRE_TRACKS_LOCK_EXP_REFS
@@ -269,10 +279,8 @@ static inline int lprocfs_climp_check(struct obd_device *obd)
 struct inode;
 struct lu_attr;
 struct obdo;
-void obdo_refresh_inode(struct inode *dst, const struct obdo *src, u32 valid);
 
 void obdo_to_ioobj(const struct obdo *oa, struct obd_ioobj *ioobj);
-void md_from_obdo(struct md_op_data *op_data, const struct obdo *oa, u32 valid);
 
 #define OBT(dev)	(dev)->obd_type
 #define OBP(dev, op)    (dev)->obd_type->typ_dt_ops->op
@@ -417,16 +425,14 @@ static inline int class_devno_max(void)
 
 static inline int obd_get_info(const struct lu_env *env,
 			       struct obd_export *exp, __u32 keylen,
-			       void *key, __u32 *vallen, void *val,
-			       struct lov_stripe_md *lsm)
+			       void *key, __u32 *vallen, void *val)
 {
 	int rc;
 
 	EXP_CHECK_DT_OP(exp, get_info);
 	EXP_COUNTER_INCREMENT(exp, get_info);
 
-	rc = OBP(exp->exp_obd, get_info)(env, exp, keylen, key, vallen, val,
-					 lsm);
+	rc = OBP(exp->exp_obd, get_info)(env, exp, keylen, key, vallen, val);
 	return rc;
 }
 
@@ -505,8 +511,7 @@ static inline int obd_setup(struct obd_device *obd, struct lustre_cfg *cfg)
 	return rc;
 }
 
-static inline int obd_precleanup(struct obd_device *obd,
-				 enum obd_cleanup_stage cleanup_stage)
+static inline int obd_precleanup(struct obd_device *obd)
 {
 	int rc;
 	DECLARE_LU_VARS(ldt, d);
@@ -517,20 +522,18 @@ static inline int obd_precleanup(struct obd_device *obd,
 	ldt = obd->obd_type->typ_lu;
 	d = obd->obd_lu_dev;
 	if (ldt && d) {
-		if (cleanup_stage == OBD_CLEANUP_EXPORTS) {
-			struct lu_env env;
+		struct lu_env env;
 
-			rc = lu_env_init(&env, ldt->ldt_ctx_tags);
-			if (rc == 0) {
-				ldt->ldt_ops->ldto_device_fini(&env, d);
-				lu_env_fini(&env);
-			}
+		rc = lu_env_init(&env, ldt->ldt_ctx_tags);
+		if (!rc) {
+			ldt->ldt_ops->ldto_device_fini(&env, d);
+			lu_env_fini(&env);
 		}
 	}
 	OBD_CHECK_DT_OP(obd, precleanup, 0);
 	OBD_COUNTER_INCREMENT(obd, precleanup);
 
-	rc = OBP(obd, precleanup)(obd, cleanup_stage);
+	rc = OBP(obd, precleanup)(obd);
 	return rc;
 }
 
@@ -612,181 +615,51 @@ obd_process_config(struct obd_device *obd, int datalen, void *data)
 	return rc;
 }
 
-/* Pack an in-memory MD struct for storage on disk.
- * Returns +ve size of packed MD (0 for free), or -ve error.
- *
- * If @disk_tgt == NULL, MD size is returned (max size if @mem_src == NULL).
- * If @*disk_tgt != NULL and @mem_src == NULL, @*disk_tgt will be freed.
- * If @*disk_tgt == NULL, it will be allocated
- */
-static inline int obd_packmd(struct obd_export *exp,
-			     struct lov_mds_md **disk_tgt,
-			     struct lov_stripe_md *mem_src)
-{
-	int rc;
-
-	EXP_CHECK_DT_OP(exp, packmd);
-	EXP_COUNTER_INCREMENT(exp, packmd);
-
-	rc = OBP(exp->exp_obd, packmd)(exp, disk_tgt, mem_src);
-	return rc;
-}
-
-static inline int obd_size_diskmd(struct obd_export *exp,
-				  struct lov_stripe_md *mem_src)
-{
-	return obd_packmd(exp, NULL, mem_src);
-}
-
-static inline int obd_free_diskmd(struct obd_export *exp,
-				  struct lov_mds_md **disk_tgt)
-{
-	LASSERT(disk_tgt);
-	LASSERT(*disk_tgt);
-	/*
-	 * LU-2590, for caller's convenience, *disk_tgt could be host
-	 * endianness, it needs swab to LE if necessary, while just
-	 * lov_mds_md header needs it for figuring out how much memory
-	 * needs to be freed.
-	 */
-	if ((cpu_to_le32(LOV_MAGIC) != LOV_MAGIC) &&
-	    (((*disk_tgt)->lmm_magic == LOV_MAGIC_V1) ||
-	     ((*disk_tgt)->lmm_magic == LOV_MAGIC_V3)))
-		lustre_swab_lov_mds_md(*disk_tgt);
-	return obd_packmd(exp, disk_tgt, NULL);
-}
-
-/* Unpack an MD struct from disk to in-memory format.
- * Returns +ve size of unpacked MD (0 for free), or -ve error.
- *
- * If @mem_tgt == NULL, MD size is returned (max size if @disk_src == NULL).
- * If @*mem_tgt != NULL and @disk_src == NULL, @*mem_tgt will be freed.
- * If @*mem_tgt == NULL, it will be allocated
- */
-static inline int obd_unpackmd(struct obd_export *exp,
-			       struct lov_stripe_md **mem_tgt,
-			       struct lov_mds_md *disk_src,
-			       int disk_len)
-{
-	int rc;
-
-	EXP_CHECK_DT_OP(exp, unpackmd);
-	EXP_COUNTER_INCREMENT(exp, unpackmd);
-
-	rc = OBP(exp->exp_obd, unpackmd)(exp, mem_tgt, disk_src, disk_len);
-	return rc;
-}
-
-static inline int obd_free_memmd(struct obd_export *exp,
-				 struct lov_stripe_md **mem_tgt)
-{
-	int rc;
-
-	LASSERT(mem_tgt);
-	LASSERT(*mem_tgt);
-	rc = obd_unpackmd(exp, mem_tgt, NULL, 0);
-	*mem_tgt = NULL;
-	return rc;
-}
-
 static inline int obd_create(const struct lu_env *env, struct obd_export *exp,
-			     struct obdo *obdo, struct obd_trans_info *oti)
+			     struct obdo *obdo)
 {
 	int rc;
 
 	EXP_CHECK_DT_OP(exp, create);
 	EXP_COUNTER_INCREMENT(exp, create);
 
-	rc = OBP(exp->exp_obd, create)(env, exp, obdo, oti);
+	rc = OBP(exp->exp_obd, create)(env, exp, obdo);
 	return rc;
 }
 
 static inline int obd_destroy(const struct lu_env *env, struct obd_export *exp,
-			      struct obdo *obdo, struct obd_trans_info *oti)
+			      struct obdo *obdo)
 {
 	int rc;
 
 	EXP_CHECK_DT_OP(exp, destroy);
 	EXP_COUNTER_INCREMENT(exp, destroy);
 
-	rc = OBP(exp->exp_obd, destroy)(env, exp, obdo, oti);
+	rc = OBP(exp->exp_obd, destroy)(env, exp, obdo);
 	return rc;
 }
 
 static inline int obd_getattr(const struct lu_env *env, struct obd_export *exp,
-			      struct obd_info *oinfo)
+			      struct obdo *oa)
 {
 	int rc;
 
 	EXP_CHECK_DT_OP(exp, getattr);
 	EXP_COUNTER_INCREMENT(exp, getattr);
 
-	rc = OBP(exp->exp_obd, getattr)(env, exp, oinfo);
-	return rc;
-}
-
-static inline int obd_getattr_async(struct obd_export *exp,
-				    struct obd_info *oinfo,
-				    struct ptlrpc_request_set *set)
-{
-	int rc;
-
-	EXP_CHECK_DT_OP(exp, getattr_async);
-	EXP_COUNTER_INCREMENT(exp, getattr_async);
-
-	rc = OBP(exp->exp_obd, getattr_async)(exp, oinfo, set);
+	rc = OBP(exp->exp_obd, getattr)(env, exp, oa);
 	return rc;
 }
 
 static inline int obd_setattr(const struct lu_env *env, struct obd_export *exp,
-			      struct obd_info *oinfo,
-			      struct obd_trans_info *oti)
+			      struct obdo *oa)
 {
 	int rc;
 
 	EXP_CHECK_DT_OP(exp, setattr);
 	EXP_COUNTER_INCREMENT(exp, setattr);
 
-	rc = OBP(exp->exp_obd, setattr)(env, exp, oinfo, oti);
-	return rc;
-}
-
-/* This performs all the requests set init/wait/destroy actions. */
-static inline int obd_setattr_rqset(struct obd_export *exp,
-				    struct obd_info *oinfo,
-				    struct obd_trans_info *oti)
-{
-	struct ptlrpc_request_set *set = NULL;
-	int rc;
-
-	EXP_CHECK_DT_OP(exp, setattr_async);
-	EXP_COUNTER_INCREMENT(exp, setattr_async);
-
-	set =  ptlrpc_prep_set();
-	if (!set)
-		return -ENOMEM;
-
-	rc = OBP(exp->exp_obd, setattr_async)(exp, oinfo, oti, set);
-	if (rc == 0)
-		rc = ptlrpc_set_wait(set);
-	ptlrpc_set_destroy(set);
-	return rc;
-}
-
-/* This adds all the requests into @set if @set != NULL, otherwise
- * all requests are sent asynchronously without waiting for response.
- */
-static inline int obd_setattr_async(struct obd_export *exp,
-				    struct obd_info *oinfo,
-				    struct obd_trans_info *oti,
-				    struct ptlrpc_request_set *set)
-{
-	int rc;
-
-	EXP_CHECK_DT_OP(exp, setattr_async);
-	EXP_COUNTER_INCREMENT(exp, setattr_async);
-
-	rc = OBP(exp->exp_obd, setattr_async)(exp, oinfo, oti, set);
+	rc = OBP(exp->exp_obd, setattr)(env, exp, oa);
 	return rc;
 }
 
@@ -1053,15 +926,16 @@ static inline int obd_statfs_rqset(struct obd_export *exp,
 				   __u32 flags)
 {
 	struct ptlrpc_request_set *set = NULL;
-	struct obd_info oinfo = { };
+	struct obd_info oinfo = {
+		.oi_osfs = osfs,
+		.oi_flags = flags,
+	};
 	int rc = 0;
 
-	set =  ptlrpc_prep_set();
+	set = ptlrpc_prep_set();
 	if (!set)
 		return -ENOMEM;
 
-	oinfo.oi_osfs = osfs;
-	oinfo.oi_flags = flags;
 	rc = obd_statfs_async(exp, &oinfo, max_age, set);
 	if (rc == 0)
 		rc = ptlrpc_set_wait(set);
@@ -1112,8 +986,7 @@ static inline int obd_preprw(const struct lu_env *env, int cmd,
 			     struct obd_export *exp, struct obdo *oa,
 			     int objcount, struct obd_ioobj *obj,
 			     struct niobuf_remote *remote, int *pages,
-			     struct niobuf_local *local,
-			     struct obd_trans_info *oti)
+			     struct niobuf_local *local)
 {
 	int rc;
 
@@ -1121,7 +994,7 @@ static inline int obd_preprw(const struct lu_env *env, int cmd,
 	EXP_COUNTER_INCREMENT(exp, preprw);
 
 	rc = OBP(exp->exp_obd, preprw)(env, cmd, exp, oa, objcount, obj, remote,
-				       pages, local, oti);
+				       pages, local);
 	return rc;
 }
 
@@ -1129,14 +1002,13 @@ static inline int obd_commitrw(const struct lu_env *env, int cmd,
 			       struct obd_export *exp, struct obdo *oa,
 			       int objcount, struct obd_ioobj *obj,
 			       struct niobuf_remote *rnb, int pages,
-			       struct niobuf_local *local,
-			       struct obd_trans_info *oti, int rc)
+			       struct niobuf_local *local, int rc)
 {
 	EXP_CHECK_DT_OP(exp, commitrw);
 	EXP_COUNTER_INCREMENT(exp, commitrw);
 
 	rc = OBP(exp->exp_obd, commitrw)(env, cmd, exp, oa, objcount, obj,
-					 rnb, pages, local, oti, rc);
+					 rnb, pages, local, rc);
 	return rc;
 }
 
@@ -1219,18 +1091,6 @@ static inline int obd_notify_observer(struct obd_device *observer,
 	return rc1 ? rc1 : rc2;
 }
 
-static inline int obd_quotacheck(struct obd_export *exp,
-				 struct obd_quotactl *oqctl)
-{
-	int rc;
-
-	EXP_CHECK_DT_OP(exp, quotacheck);
-	EXP_COUNTER_INCREMENT(exp, quotacheck);
-
-	rc = OBP(exp->exp_obd, quotacheck)(exp->exp_obd, exp, oqctl);
-	return rc;
-}
-
 static inline int obd_quotactl(struct obd_export *exp,
 			       struct obd_quotactl *oqctl)
 {
@@ -1346,21 +1206,9 @@ static inline int md_create(struct obd_export *exp, struct md_op_data *op_data,
 	return rc;
 }
 
-static inline int md_done_writing(struct obd_export *exp,
-				  struct md_op_data *op_data,
-				  struct md_open_data *mod)
-{
-	int rc;
-
-	EXP_CHECK_MD_OP(exp, done_writing);
-	EXP_MD_COUNTER_INCREMENT(exp, done_writing);
-	rc = MDP(exp->exp_obd, done_writing)(exp, op_data, mod);
-	return rc;
-}
-
 static inline int md_enqueue(struct obd_export *exp,
 			     struct ldlm_enqueue_info *einfo,
-			     const ldlm_policy_data_t *policy,
+			     const union ldlm_policy_data *policy,
 			     struct lookup_intent *it,
 			     struct md_op_data *op_data,
 			     struct lustre_handle *lockh,
@@ -1428,16 +1276,14 @@ static inline int md_rename(struct obd_export *exp, struct md_op_data *op_data,
 }
 
 static inline int md_setattr(struct obd_export *exp, struct md_op_data *op_data,
-			     void *ea, size_t ealen, void *ea2, size_t ea2len,
-			     struct ptlrpc_request **request,
-			     struct md_open_data **mod)
+			     void *ea, size_t ealen,
+			     struct ptlrpc_request **request)
 {
 	int rc;
 
 	EXP_CHECK_MD_OP(exp, setattr);
 	EXP_MD_COUNTER_INCREMENT(exp, setattr);
-	rc = MDP(exp->exp_obd, setattr)(exp, op_data, ea, ealen,
-					ea2, ea2len, request, mod);
+	rc = MDP(exp->exp_obd, setattr)(exp, op_data, ea, ealen, request);
 	return rc;
 }
 
@@ -1561,7 +1407,7 @@ static inline int md_set_lock_data(struct obd_export *exp,
 
 static inline int md_cancel_unused(struct obd_export *exp,
 				   const struct lu_fid *fid,
-				   ldlm_policy_data_t *policy,
+				   union ldlm_policy_data *policy,
 				   enum ldlm_mode mode,
 				   enum ldlm_cancel_flags flags,
 				   void *opaque)
@@ -1579,7 +1425,7 @@ static inline int md_cancel_unused(struct obd_export *exp,
 static inline enum ldlm_mode md_lock_match(struct obd_export *exp, __u64 flags,
 					   const struct lu_fid *fid,
 					   enum ldlm_type type,
-					   ldlm_policy_data_t *policy,
+					   union ldlm_policy_data *policy,
 					   enum ldlm_mode mode,
 					   struct lustre_handle *lockh)
 {
@@ -1589,14 +1435,12 @@ static inline enum ldlm_mode md_lock_match(struct obd_export *exp, __u64 flags,
 					     policy, mode, lockh);
 }
 
-static inline int md_init_ea_size(struct obd_export *exp, int easize,
-				  int def_asize, int cookiesize,
-				  int def_cookiesize)
+static inline int md_init_ea_size(struct obd_export *exp, u32 easize,
+				  u32 def_asize)
 {
 	EXP_CHECK_MD_OP(exp, init_ea_size);
 	EXP_MD_COUNTER_INCREMENT(exp, init_ea_size);
-	return MDP(exp->exp_obd, init_ea_size)(exp, easize, def_asize,
-					       cookiesize, def_cookiesize);
+	return MDP(exp->exp_obd, init_ea_size)(exp, easize, def_asize);
 }
 
 static inline int md_intent_getattr_async(struct obd_export *exp,
@@ -1636,6 +1480,24 @@ static inline int md_get_fid_from_lsm(struct obd_export *exp,
 	return rc;
 }
 
+/* Unpack an MD struct from disk to in-memory format.
+ * Returns +ve size of unpacked MD (0 for free), or -ve error.
+ *
+ * If *plsm != NULL and lmm == NULL then *lsm will be freed.
+ * If *plsm == NULL then it will be allocated.
+ */
+static inline int md_unpackmd(struct obd_export *exp,
+			      struct lmv_stripe_md **plsm,
+			      const union lmv_mds_md *lmm, size_t lmm_size)
+{
+	int rc;
+
+	EXP_CHECK_MD_OP(exp, unpackmd);
+	EXP_MD_COUNTER_INCREMENT(exp, unpackmd);
+	rc = MDP(exp->exp_obd, unpackmd)(exp, plsm, lmm, lmm_size);
+	return rc;
+}
+
 /* OBD Metadata Support */
 
 int obd_init_caches(void);
diff --git a/drivers/staging/lustre/lustre/include/obd_support.h b/drivers/staging/lustre/lustre/include/obd_support.h
index b346a7f10aa4..aaedec7d793c 100644
--- a/drivers/staging/lustre/lustre/include/obd_support.h
+++ b/drivers/staging/lustre/lustre/include/obd_support.h
@@ -172,14 +172,14 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_MDS_ALL_REQUEST_NET     0x123
 #define OBD_FAIL_MDS_SYNC_NET	    0x124
 #define OBD_FAIL_MDS_SYNC_PACK	   0x125
-#define OBD_FAIL_MDS_DONE_WRITING_NET    0x126
-#define OBD_FAIL_MDS_DONE_WRITING_PACK   0x127
+/*	OBD_FAIL_MDS_DONE_WRITING_NET	0x126 obsolete since 2.8.0 */
+/*	OBD_FAIL_MDS_DONE_WRITING_PACK	0x127 obsolete since 2.8.0 */
 #define OBD_FAIL_MDS_ALLOC_OBDO	  0x128
 #define OBD_FAIL_MDS_PAUSE_OPEN	  0x129
 #define OBD_FAIL_MDS_STATFS_LCW_SLEEP    0x12a
 #define OBD_FAIL_MDS_OPEN_CREATE	 0x12b
 #define OBD_FAIL_MDS_OST_SETATTR	 0x12c
-#define OBD_FAIL_MDS_QUOTACHECK_NET      0x12d
+/*	OBD_FAIL_MDS_QUOTACHECK_NET      0x12d obsolete since 2.4 */
 #define OBD_FAIL_MDS_QUOTACTL_NET	0x12e
 #define OBD_FAIL_MDS_CLIENT_ADD	  0x12f
 #define OBD_FAIL_MDS_GETXATTR_NET	0x130
@@ -264,7 +264,7 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_OST_ENOSPC	      0x215
 #define OBD_FAIL_OST_EROFS	       0x216
 #define OBD_FAIL_OST_ENOENT	      0x217
-#define OBD_FAIL_OST_QUOTACHECK_NET      0x218
+/*	OBD_FAIL_OST_QUOTACHECK_NET      0x218 obsolete since 2.4 */
 #define OBD_FAIL_OST_QUOTACTL_NET	0x219
 #define OBD_FAIL_OST_CHECKSUM_RECEIVE    0x21a
 #define OBD_FAIL_OST_CHECKSUM_SEND       0x21b
@@ -321,6 +321,8 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_LDLM_CP_CB_WAIT4	 0x322
 #define OBD_FAIL_LDLM_CP_CB_WAIT5	 0x323
 
+#define OBD_FAIL_LDLM_GRANT_CHECK        0x32a
+
 /* LOCKLESS IO */
 #define OBD_FAIL_LDLM_SET_CONTENTION     0x385
 
@@ -343,6 +345,7 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_OSC_CP_ENQ_RACE	 0x410
 #define OBD_FAIL_OSC_NO_GRANT	    0x411
 #define OBD_FAIL_OSC_DELAY_SETTIME	 0x412
+#define OBD_FAIL_OSC_DELAY_IO		 0x414
 
 #define OBD_FAIL_PTLRPC		  0x500
 #define OBD_FAIL_PTLRPC_ACK	      0x501
@@ -373,7 +376,7 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_OBD_PING_NET	    0x600
 #define OBD_FAIL_OBD_LOG_CANCEL_NET      0x601
 #define OBD_FAIL_OBD_LOGD_NET	    0x602
-#define OBD_FAIL_OBD_QC_CALLBACK_NET     0x603
+/*	OBD_FAIL_OBD_QC_CALLBACK_NET     0x603 obsolete since 2.4 */
 #define OBD_FAIL_OBD_DQACQ	       0x604
 #define OBD_FAIL_OBD_LLOG_SETUP	  0x605
 #define OBD_FAIL_OBD_LOG_CANCEL_REP      0x606
@@ -458,6 +461,8 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_LOV_INIT			    0x1403
 #define OBD_FAIL_GLIMPSE_DELAY			    0x1404
 #define OBD_FAIL_LLITE_XATTR_ENOMEM		    0x1405
+#define OBD_FAIL_MAKE_LOVEA_HOLE		    0x1406
+#define OBD_FAIL_LLITE_LOST_LAYOUT		    0x1407
 #define OBD_FAIL_GETATTR_DELAY			    0x1409
 
 #define OBD_FAIL_FID_INDIR	0x1501
diff --git a/drivers/staging/lustre/lustre/include/seq_range.h b/drivers/staging/lustre/lustre/include/seq_range.h
new file mode 100644
index 000000000000..30c4dd66d5c4
--- /dev/null
+++ b/drivers/staging/lustre/lustre/include/seq_range.h
@@ -0,0 +1,199 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ *
+ * Copyright (c) 2011, 2014, Intel Corporation.
+ *
+ * Copyright 2015 Cray Inc, all rights reserved.
+ * Author: Ben Evans.
+ *
+ * Define lu_seq_range  associated functions
+ */
+
+#ifndef _SEQ_RANGE_H_
+#define _SEQ_RANGE_H_
+
+#include "lustre/lustre_idl.h"
+
+/**
+ * computes the sequence range type \a range
+ */
+
+static inline unsigned int fld_range_type(const struct lu_seq_range *range)
+{
+	return range->lsr_flags & LU_SEQ_RANGE_MASK;
+}
+
+/**
+ *  Is this sequence range an OST? \a range
+ */
+
+static inline bool fld_range_is_ost(const struct lu_seq_range *range)
+{
+	return fld_range_type(range) == LU_SEQ_RANGE_OST;
+}
+
+/**
+ *  Is this sequence range an MDT? \a range
+ */
+
+static inline bool fld_range_is_mdt(const struct lu_seq_range *range)
+{
+	return fld_range_type(range) == LU_SEQ_RANGE_MDT;
+}
+
+/**
+ * ANY range is only used when the fld client sends a fld query request,
+ * but it does not know whether the seq is an MDT or OST, so it will send the
+ * request with ANY type, which means any seq type from the lookup can be
+ * expected. /a range
+ */
+static inline unsigned int fld_range_is_any(const struct lu_seq_range *range)
+{
+	return fld_range_type(range) == LU_SEQ_RANGE_ANY;
+}
+
+/**
+ * Apply flags to range \a range \a flags
+ */
+
+static inline void fld_range_set_type(struct lu_seq_range *range,
+				      unsigned int flags)
+{
+	range->lsr_flags |= flags;
+}
+
+/**
+ * Add MDT to range type \a range
+ */
+
+static inline void fld_range_set_mdt(struct lu_seq_range *range)
+{
+	fld_range_set_type(range, LU_SEQ_RANGE_MDT);
+}
+
+/**
+ * Add OST to range type \a range
+ */
+
+static inline void fld_range_set_ost(struct lu_seq_range *range)
+{
+	fld_range_set_type(range, LU_SEQ_RANGE_OST);
+}
+
+/**
+ * Add ANY to range type \a range
+ */
+
+static inline void fld_range_set_any(struct lu_seq_range *range)
+{
+	fld_range_set_type(range, LU_SEQ_RANGE_ANY);
+}
+
+/**
+ * computes width of given sequence range \a range
+ */
+
+static inline u64 lu_seq_range_space(const struct lu_seq_range *range)
+{
+	return range->lsr_end - range->lsr_start;
+}
+
+/**
+ * initialize range to zero \a range
+ */
+
+static inline void lu_seq_range_init(struct lu_seq_range *range)
+{
+	memset(range, 0, sizeof(*range));
+}
+
+/**
+ * check if given seq id \a s is within given range \a range
+ */
+
+static inline bool lu_seq_range_within(const struct lu_seq_range *range,
+				       u64 seq)
+{
+	return seq >= range->lsr_start && seq < range->lsr_end;
+}
+
+/**
+ * Is the range sane?  Is the end after the beginning? \a range
+ */
+
+static inline bool lu_seq_range_is_sane(const struct lu_seq_range *range)
+{
+	return range->lsr_end >= range->lsr_start;
+}
+
+/**
+ * Is the range 0? \a range
+ */
+
+static inline bool lu_seq_range_is_zero(const struct lu_seq_range *range)
+{
+	return range->lsr_start == 0 && range->lsr_end == 0;
+}
+
+/**
+ * Is the range out of space? \a range
+ */
+
+static inline bool lu_seq_range_is_exhausted(const struct lu_seq_range *range)
+{
+	return lu_seq_range_space(range) == 0;
+}
+
+/**
+ * return 0 if two ranges have the same location, nonzero if they are
+ * different \a r1 \a r2
+ */
+
+static inline int lu_seq_range_compare_loc(const struct lu_seq_range *r1,
+					   const struct lu_seq_range *r2)
+{
+	return r1->lsr_index != r2->lsr_index ||
+		r1->lsr_flags != r2->lsr_flags;
+}
+
+#if !defined(__REQ_LAYOUT_USER__)
+/**
+ * byte swap range structure \a range
+ */
+
+void lustre_swab_lu_seq_range(struct lu_seq_range *range);
+#endif
+/**
+ * printf string and argument list for sequence range
+ */
+#define DRANGE "[%#16.16llx-%#16.16llx]:%x:%s"
+
+#define PRANGE(range)		\
+	(range)->lsr_start,	\
+	(range)->lsr_end,	\
+	(range)->lsr_index,	\
+	fld_range_is_mdt(range) ? "mdt" : "ost"
+
+#endif
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c b/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c
index ecf472e4813d..32b73ee62639 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c
@@ -193,6 +193,26 @@ void ldlm_extent_add_lock(struct ldlm_resource *res,
 	 * add the locks into grant list, for debug purpose, ..
 	 */
 	ldlm_resource_add_lock(res, &res->lr_granted, lock);
+
+	if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_GRANT_CHECK)) {
+		struct ldlm_lock *lck;
+
+		list_for_each_entry_reverse(lck, &res->lr_granted,
+					    l_res_link) {
+			if (lck == lock)
+				continue;
+			if (lockmode_compat(lck->l_granted_mode,
+					    lock->l_granted_mode))
+				continue;
+			if (ldlm_extent_overlap(&lck->l_req_extent,
+						&lock->l_req_extent)) {
+				CDEBUG(D_ERROR, "granting conflicting lock %p %p\n",
+				       lck, lock);
+				ldlm_resource_dump(D_ERROR, res);
+				LBUG();
+			}
+		}
+	}
 }
 
 /** Remove cancelled lock from resource interval tree. */
@@ -220,8 +240,8 @@ void ldlm_extent_unlink_lock(struct ldlm_lock *lock)
 	}
 }
 
-void ldlm_extent_policy_wire_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				      ldlm_policy_data_t *lpolicy)
+void ldlm_extent_policy_wire_to_local(const union ldlm_wire_policy_data *wpolicy,
+				      union ldlm_policy_data *lpolicy)
 {
 	memset(lpolicy, 0, sizeof(*lpolicy));
 	lpolicy->l_extent.start = wpolicy->l_extent.start;
@@ -229,8 +249,8 @@ void ldlm_extent_policy_wire_to_local(const ldlm_wire_policy_data_t *wpolicy,
 	lpolicy->l_extent.gid = wpolicy->l_extent.gid;
 }
 
-void ldlm_extent_policy_local_to_wire(const ldlm_policy_data_t *lpolicy,
-				      ldlm_wire_policy_data_t *wpolicy)
+void ldlm_extent_policy_local_to_wire(const union ldlm_policy_data *lpolicy,
+				      union ldlm_wire_policy_data *wpolicy)
 {
 	memset(wpolicy, 0, sizeof(*wpolicy));
 	wpolicy->l_extent.start = lpolicy->l_extent.start;
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c b/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c
index 861f36f039b5..722160784f83 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c
@@ -612,22 +612,8 @@ granted:
 }
 EXPORT_SYMBOL(ldlm_flock_completion_ast);
 
-void ldlm_flock_policy_wire18_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				       ldlm_policy_data_t *lpolicy)
-{
-	memset(lpolicy, 0, sizeof(*lpolicy));
-	lpolicy->l_flock.start = wpolicy->l_flock.lfw_start;
-	lpolicy->l_flock.end = wpolicy->l_flock.lfw_end;
-	lpolicy->l_flock.pid = wpolicy->l_flock.lfw_pid;
-	/* Compat code, old clients had no idea about owner field and
-	 * relied solely on pid for ownership. Introduced in LU-104, 2.1,
-	 * April 2011
-	 */
-	lpolicy->l_flock.owner = wpolicy->l_flock.lfw_pid;
-}
-
-void ldlm_flock_policy_wire21_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				       ldlm_policy_data_t *lpolicy)
+void ldlm_flock_policy_wire_to_local(const union ldlm_wire_policy_data *wpolicy,
+				     union ldlm_policy_data *lpolicy)
 {
 	memset(lpolicy, 0, sizeof(*lpolicy));
 	lpolicy->l_flock.start = wpolicy->l_flock.lfw_start;
@@ -636,8 +622,8 @@ void ldlm_flock_policy_wire21_to_local(const ldlm_wire_policy_data_t *wpolicy,
 	lpolicy->l_flock.owner = wpolicy->l_flock.lfw_owner;
 }
 
-void ldlm_flock_policy_local_to_wire(const ldlm_policy_data_t *lpolicy,
-				     ldlm_wire_policy_data_t *wpolicy)
+void ldlm_flock_policy_local_to_wire(const union ldlm_policy_data *lpolicy,
+				     union ldlm_wire_policy_data *wpolicy)
 {
 	memset(wpolicy, 0, sizeof(*wpolicy));
 	wpolicy->l_flock.lfw_start = lpolicy->l_flock.start;
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_inodebits.c b/drivers/staging/lustre/lustre/ldlm/ldlm_inodebits.c
index 79f4e6fa193e..8e1709dc073c 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_inodebits.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_inodebits.c
@@ -54,15 +54,15 @@
 #include "../include/lustre_lib.h"
 #include "ldlm_internal.h"
 
-void ldlm_ibits_policy_wire_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				     ldlm_policy_data_t *lpolicy)
+void ldlm_ibits_policy_wire_to_local(const union ldlm_wire_policy_data *wpolicy,
+				     union ldlm_policy_data *lpolicy)
 {
 	memset(lpolicy, 0, sizeof(*lpolicy));
 	lpolicy->l_inodebits.bits = wpolicy->l_inodebits.bits;
 }
 
-void ldlm_ibits_policy_local_to_wire(const ldlm_policy_data_t *lpolicy,
-				     ldlm_wire_policy_data_t *wpolicy)
+void ldlm_ibits_policy_local_to_wire(const union ldlm_policy_data *lpolicy,
+				     union ldlm_wire_policy_data *wpolicy)
 {
 	memset(wpolicy, 0, sizeof(*wpolicy));
 	wpolicy->l_inodebits.bits = lpolicy->l_inodebits.bits;
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_internal.h b/drivers/staging/lustre/lustre/ldlm/ldlm_internal.h
index 5e82cfc245b2..5c02501d0560 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_internal.h
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_internal.h
@@ -39,13 +39,13 @@ extern struct list_head ldlm_srv_namespace_list;
 extern struct mutex ldlm_cli_namespace_lock;
 extern struct list_head ldlm_cli_active_namespace_list;
 
-static inline int ldlm_namespace_nr_read(ldlm_side_t client)
+static inline int ldlm_namespace_nr_read(enum ldlm_side client)
 {
 	return client == LDLM_NAMESPACE_SERVER ?
 		ldlm_srv_namespace_nr : ldlm_cli_namespace_nr;
 }
 
-static inline void ldlm_namespace_nr_inc(ldlm_side_t client)
+static inline void ldlm_namespace_nr_inc(enum ldlm_side client)
 {
 	if (client == LDLM_NAMESPACE_SERVER)
 		ldlm_srv_namespace_nr++;
@@ -53,7 +53,7 @@ static inline void ldlm_namespace_nr_inc(ldlm_side_t client)
 		ldlm_cli_namespace_nr++;
 }
 
-static inline void ldlm_namespace_nr_dec(ldlm_side_t client)
+static inline void ldlm_namespace_nr_dec(enum ldlm_side client)
 {
 	if (client == LDLM_NAMESPACE_SERVER)
 		ldlm_srv_namespace_nr--;
@@ -61,13 +61,13 @@ static inline void ldlm_namespace_nr_dec(ldlm_side_t client)
 		ldlm_cli_namespace_nr--;
 }
 
-static inline struct list_head *ldlm_namespace_list(ldlm_side_t client)
+static inline struct list_head *ldlm_namespace_list(enum ldlm_side client)
 {
 	return client == LDLM_NAMESPACE_SERVER ?
 		&ldlm_srv_namespace_list : &ldlm_cli_active_namespace_list;
 }
 
-static inline struct mutex *ldlm_namespace_lock(ldlm_side_t client)
+static inline struct mutex *ldlm_namespace_lock(enum ldlm_side client)
 {
 	return client == LDLM_NAMESPACE_SERVER ?
 		&ldlm_srv_namespace_lock : &ldlm_cli_namespace_lock;
@@ -79,22 +79,23 @@ static inline int ldlm_ns_empty(struct ldlm_namespace *ns)
 	return atomic_read(&ns->ns_bref) == 0;
 }
 
-void ldlm_namespace_move_to_active_locked(struct ldlm_namespace *, ldlm_side_t);
+void ldlm_namespace_move_to_active_locked(struct ldlm_namespace *,
+					  enum ldlm_side);
 void ldlm_namespace_move_to_inactive_locked(struct ldlm_namespace *,
-					    ldlm_side_t);
-struct ldlm_namespace *ldlm_namespace_first_locked(ldlm_side_t);
+					    enum ldlm_side);
+struct ldlm_namespace *ldlm_namespace_first_locked(enum ldlm_side);
 
 /* ldlm_request.c */
 /* Cancel lru flag, it indicates we cancel aged locks. */
 enum {
-	LDLM_CANCEL_AGED   = 1 << 0, /* Cancel aged locks (non lru resize). */
-	LDLM_CANCEL_PASSED = 1 << 1, /* Cancel passed number of locks. */
-	LDLM_CANCEL_SHRINK = 1 << 2, /* Cancel locks from shrinker. */
-	LDLM_CANCEL_LRUR   = 1 << 3, /* Cancel locks from lru resize. */
-	LDLM_CANCEL_NO_WAIT = 1 << 4, /* Cancel locks w/o blocking (neither
-				       * sending nor waiting for any rpcs)
-				       */
-	LDLM_CANCEL_LRUR_NO_WAIT = 1 << 5, /* LRUR + NO_WAIT */
+	LDLM_LRU_FLAG_AGED	= BIT(0), /* Cancel aged locks (non lru resize). */
+	LDLM_LRU_FLAG_PASSED	= BIT(1), /* Cancel passed number of locks. */
+	LDLM_LRU_FLAG_SHRINK	= BIT(2), /* Cancel locks from shrinker. */
+	LDLM_LRU_FLAG_LRUR	= BIT(3), /* Cancel locks from lru resize. */
+	LDLM_LRU_FLAG_NO_WAIT	= BIT(4), /* Cancel locks w/o blocking (neither
+					   * sending nor waiting for any rpcs)
+					   */
+	LDLM_LRU_FLAG_LRUR_NO_WAIT = BIT(5), /* LRUR + NO_WAIT */
 };
 
 int ldlm_cancel_lru(struct ldlm_namespace *ns, int nr,
@@ -137,10 +138,10 @@ ldlm_lock_create(struct ldlm_namespace *ns, const struct ldlm_res_id *,
 		 void *data, __u32 lvb_len, enum lvb_type lvb_type);
 enum ldlm_error ldlm_lock_enqueue(struct ldlm_namespace *, struct ldlm_lock **,
 				  void *cookie, __u64 *flags);
-void ldlm_lock_addref_internal(struct ldlm_lock *, __u32 mode);
-void ldlm_lock_addref_internal_nolock(struct ldlm_lock *, __u32 mode);
-void ldlm_lock_decref_internal(struct ldlm_lock *, __u32 mode);
-void ldlm_lock_decref_internal_nolock(struct ldlm_lock *, __u32 mode);
+void ldlm_lock_addref_internal(struct ldlm_lock *, enum ldlm_mode mode);
+void ldlm_lock_addref_internal_nolock(struct ldlm_lock *, enum ldlm_mode mode);
+void ldlm_lock_decref_internal(struct ldlm_lock *, enum ldlm_mode mode);
+void ldlm_lock_decref_internal_nolock(struct ldlm_lock *, enum ldlm_mode mode);
 int ldlm_run_ast_work(struct ldlm_namespace *ns, struct list_head *rpc_list,
 		      enum ldlm_desc_ast_t ast_type);
 int ldlm_lock_remove_from_lru_check(struct ldlm_lock *lock, time_t last_use);
@@ -311,28 +312,25 @@ static inline int is_granted_or_cancelled(struct ldlm_lock *lock)
 	return ret;
 }
 
-typedef void (*ldlm_policy_wire_to_local_t)(const ldlm_wire_policy_data_t *,
-					    ldlm_policy_data_t *);
-
-typedef void (*ldlm_policy_local_to_wire_t)(const ldlm_policy_data_t *,
-					    ldlm_wire_policy_data_t *);
-
-void ldlm_plain_policy_wire_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				     ldlm_policy_data_t *lpolicy);
-void ldlm_plain_policy_local_to_wire(const ldlm_policy_data_t *lpolicy,
-				     ldlm_wire_policy_data_t *wpolicy);
-void ldlm_ibits_policy_wire_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				     ldlm_policy_data_t *lpolicy);
-void ldlm_ibits_policy_local_to_wire(const ldlm_policy_data_t *lpolicy,
-				     ldlm_wire_policy_data_t *wpolicy);
-void ldlm_extent_policy_wire_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				      ldlm_policy_data_t *lpolicy);
-void ldlm_extent_policy_local_to_wire(const ldlm_policy_data_t *lpolicy,
-				      ldlm_wire_policy_data_t *wpolicy);
-void ldlm_flock_policy_wire18_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				       ldlm_policy_data_t *lpolicy);
-void ldlm_flock_policy_wire21_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				       ldlm_policy_data_t *lpolicy);
-
-void ldlm_flock_policy_local_to_wire(const ldlm_policy_data_t *lpolicy,
-				     ldlm_wire_policy_data_t *wpolicy);
+typedef void (*ldlm_policy_wire_to_local_t)(const union ldlm_wire_policy_data *,
+					    union ldlm_policy_data *);
+
+typedef void (*ldlm_policy_local_to_wire_t)(const union ldlm_policy_data *,
+					    union ldlm_wire_policy_data *);
+
+void ldlm_plain_policy_wire_to_local(const union ldlm_wire_policy_data *wpolicy,
+				     union ldlm_policy_data *lpolicy);
+void ldlm_plain_policy_local_to_wire(const union ldlm_policy_data *lpolicy,
+				     union ldlm_wire_policy_data *wpolicy);
+void ldlm_ibits_policy_wire_to_local(const union ldlm_wire_policy_data *wpolicy,
+				     union ldlm_policy_data *lpolicy);
+void ldlm_ibits_policy_local_to_wire(const union ldlm_policy_data *lpolicy,
+				     union ldlm_wire_policy_data *wpolicy);
+void ldlm_extent_policy_wire_to_local(const union ldlm_wire_policy_data *wpolicy,
+				      union ldlm_policy_data *lpolicy);
+void ldlm_extent_policy_local_to_wire(const union ldlm_policy_data *lpolicy,
+				      union ldlm_wire_policy_data *wpolicy);
+void ldlm_flock_policy_wire_to_local(const union ldlm_wire_policy_data *wpolicy,
+				     union ldlm_policy_data *lpolicy);
+void ldlm_flock_policy_local_to_wire(const union ldlm_policy_data *lpolicy,
+				     union ldlm_wire_policy_data *wpolicy);
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c b/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c
index 153e990c494e..9be01426c955 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c
@@ -170,6 +170,9 @@ int client_import_del_conn(struct obd_import *imp, struct obd_uuid *uuid)
 				ptlrpc_connection_put(dlmexp->exp_connection);
 				dlmexp->exp_connection = NULL;
 			}
+
+			if (dlmexp)
+				class_export_put(dlmexp);
 		}
 
 		list_del(&imp_conn->oic_item);
@@ -372,6 +375,25 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
 	} else {
 		cli->cl_max_rpcs_in_flight = OBD_MAX_RIF_DEFAULT;
 	}
+
+	spin_lock_init(&cli->cl_mod_rpcs_lock);
+	spin_lock_init(&cli->cl_mod_rpcs_hist.oh_lock);
+	cli->cl_max_mod_rpcs_in_flight = 0;
+	cli->cl_mod_rpcs_in_flight = 0;
+	cli->cl_close_rpcs_in_flight = 0;
+	init_waitqueue_head(&cli->cl_mod_rpcs_waitq);
+	cli->cl_mod_tag_bitmap = NULL;
+
+	if (connect_op == MDS_CONNECT) {
+		cli->cl_max_mod_rpcs_in_flight = cli->cl_max_rpcs_in_flight - 1;
+		cli->cl_mod_tag_bitmap = kcalloc(BITS_TO_LONGS(OBD_MAX_RIF_MAX),
+						 sizeof(long), GFP_NOFS);
+		if (!cli->cl_mod_tag_bitmap) {
+			rc = -ENOMEM;
+			goto err;
+		}
+	}
+
 	rc = ldlm_get_ref();
 	if (rc) {
 		CERROR("ldlm_get_ref failed: %d\n", rc);
@@ -399,9 +421,8 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
 	}
 
 	cli->cl_import = imp;
-	/* cli->cl_max_mds_{easize,cookiesize} updated by mdc_init_ea_size() */
+	/* cli->cl_max_mds_easize updated by mdc_init_ea_size() */
 	cli->cl_max_mds_easize = sizeof(struct lov_mds_md_v3);
-	cli->cl_max_mds_cookiesize = sizeof(struct llog_cookie);
 
 	if (LUSTRE_CFG_BUFLEN(lcfg, 3) > 0) {
 		if (!strcmp(lustre_cfg_string(lcfg, 3), "inactive")) {
@@ -425,8 +446,6 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
 		goto err_import;
 	}
 
-	cli->cl_qchk_stat = CL_NOT_QUOTACHECKED;
-
 	return rc;
 
 err_import:
@@ -434,12 +453,16 @@ err_import:
 err_ldlm:
 	ldlm_put_ref();
 err:
+	kfree(cli->cl_mod_tag_bitmap);
+	cli->cl_mod_tag_bitmap = NULL;
 	return rc;
 }
 EXPORT_SYMBOL(client_obd_setup);
 
 int client_obd_cleanup(struct obd_device *obddev)
 {
+	struct client_obd *cli = &obddev->u.cli;
+
 	ldlm_namespace_free_post(obddev->obd_namespace);
 	obddev->obd_namespace = NULL;
 
@@ -447,6 +470,10 @@ int client_obd_cleanup(struct obd_device *obddev)
 	LASSERT(!obddev->u.cli.cl_import);
 
 	ldlm_put_ref();
+
+	kfree(cli->cl_mod_tag_bitmap);
+	cli->cl_mod_tag_bitmap = NULL;
+
 	return 0;
 }
 EXPORT_SYMBOL(client_obd_cleanup);
@@ -461,6 +488,7 @@ int client_connect_import(const struct lu_env *env,
 	struct obd_import       *imp    = cli->cl_import;
 	struct obd_connect_data *ocd;
 	struct lustre_handle    conn    = { 0 };
+	bool is_mdc = false;
 	int		     rc;
 
 	*exp = NULL;
@@ -487,6 +515,10 @@ int client_connect_import(const struct lu_env *env,
 	ocd = &imp->imp_connect_data;
 	if (data) {
 		*ocd = *data;
+		is_mdc = !strncmp(imp->imp_obd->obd_type->typ_name,
+				  LUSTRE_MDC_NAME, 3);
+		if (is_mdc)
+			data->ocd_connect_flags |= OBD_CONNECT_MULTIMODRPCS;
 		imp->imp_connect_flags_orig = data->ocd_connect_flags;
 	}
 
@@ -502,6 +534,11 @@ int client_connect_import(const struct lu_env *env,
 			 ocd->ocd_connect_flags, "old %#llx, new %#llx\n",
 			 data->ocd_connect_flags, ocd->ocd_connect_flags);
 		data->ocd_connect_flags = ocd->ocd_connect_flags;
+		/* clear the flag as it was not set and is not known
+		 * by upper layers
+		 */
+		if (is_mdc)
+			data->ocd_connect_flags &= ~OBD_CONNECT_MULTIMODRPCS;
 	}
 
 	ptlrpc_pinger_add_import(imp);
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c b/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c
index 3c48b4fb96f1..a4a291acb659 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c
@@ -39,6 +39,7 @@
 
 #include "../../include/linux/libcfs/libcfs.h"
 #include "../include/lustre_intent.h"
+#include "../include/lustre_swab.h"
 #include "../include/obd_class.h"
 #include "ldlm_internal.h"
 
@@ -63,17 +64,10 @@ static char *ldlm_typename[] = {
 	[LDLM_IBITS]	= "IBT",
 };
 
-static ldlm_policy_wire_to_local_t ldlm_policy_wire18_to_local[] = {
+static ldlm_policy_wire_to_local_t ldlm_policy_wire_to_local[] = {
 	[LDLM_PLAIN - LDLM_MIN_TYPE]	= ldlm_plain_policy_wire_to_local,
 	[LDLM_EXTENT - LDLM_MIN_TYPE]	= ldlm_extent_policy_wire_to_local,
-	[LDLM_FLOCK - LDLM_MIN_TYPE]	= ldlm_flock_policy_wire18_to_local,
-	[LDLM_IBITS - LDLM_MIN_TYPE]	= ldlm_ibits_policy_wire_to_local,
-};
-
-static ldlm_policy_wire_to_local_t ldlm_policy_wire21_to_local[] = {
-	[LDLM_PLAIN - LDLM_MIN_TYPE]	= ldlm_plain_policy_wire_to_local,
-	[LDLM_EXTENT - LDLM_MIN_TYPE]	= ldlm_extent_policy_wire_to_local,
-	[LDLM_FLOCK - LDLM_MIN_TYPE]	= ldlm_flock_policy_wire21_to_local,
+	[LDLM_FLOCK - LDLM_MIN_TYPE]	= ldlm_flock_policy_wire_to_local,
 	[LDLM_IBITS - LDLM_MIN_TYPE]	= ldlm_ibits_policy_wire_to_local,
 };
 
@@ -88,8 +82,8 @@ static ldlm_policy_local_to_wire_t ldlm_policy_local_to_wire[] = {
  * Converts lock policy from local format to on the wire lock_desc format
  */
 static void ldlm_convert_policy_to_wire(enum ldlm_type type,
-					const ldlm_policy_data_t *lpolicy,
-					ldlm_wire_policy_data_t *wpolicy)
+					const union ldlm_policy_data *lpolicy,
+					union ldlm_wire_policy_data *wpolicy)
 {
 	ldlm_policy_local_to_wire_t convert;
 
@@ -102,23 +96,17 @@ static void ldlm_convert_policy_to_wire(enum ldlm_type type,
  * Converts lock policy from on the wire lock_desc format to local format
  */
 void ldlm_convert_policy_to_local(struct obd_export *exp, enum ldlm_type type,
-				  const ldlm_wire_policy_data_t *wpolicy,
-				  ldlm_policy_data_t *lpolicy)
+				  const union ldlm_wire_policy_data *wpolicy,
+				  union ldlm_policy_data *lpolicy)
 {
 	ldlm_policy_wire_to_local_t convert;
-	int new_client;
 
-	/** some badness for 2.0.0 clients, but 2.0.0 isn't supported */
-	new_client = (exp_connect_flags(exp) & OBD_CONNECT_FULL20) != 0;
-	if (new_client)
-		convert = ldlm_policy_wire21_to_local[type - LDLM_MIN_TYPE];
-	else
-		convert = ldlm_policy_wire18_to_local[type - LDLM_MIN_TYPE];
+	convert = ldlm_policy_wire_to_local[type - LDLM_MIN_TYPE];
 
 	convert(wpolicy, lpolicy);
 }
 
-char *ldlm_it2str(int it)
+const char *ldlm_it2str(enum ldlm_intent_flags it)
 {
 	switch (it) {
 	case IT_OPEN:
@@ -140,7 +128,7 @@ char *ldlm_it2str(int it)
 	case IT_LAYOUT:
 		return "layout";
 	default:
-		CERROR("Unknown intent %d\n", it);
+		CERROR("Unknown intent 0x%08x\n", it);
 		return "UNKNOWN";
 	}
 }
@@ -512,7 +500,6 @@ int ldlm_lock_change_resource(struct ldlm_namespace *ns, struct ldlm_lock *lock,
 
 	return 0;
 }
-EXPORT_SYMBOL(ldlm_lock_change_resource);
 
 /** \defgroup ldlm_handles LDLM HANDLES
  * Ways to get hold of locks without any addresses.
@@ -595,7 +582,6 @@ void ldlm_lock2desc(struct ldlm_lock *lock, struct ldlm_lock_desc *desc)
 				    &lock->l_policy_data,
 				    &desc->l_policy_data);
 }
-EXPORT_SYMBOL(ldlm_lock2desc);
 
 /**
  * Add a lock to list of conflicting locks to send AST to.
@@ -658,7 +644,7 @@ static void ldlm_add_ast_work_item(struct ldlm_lock *lock,
  * r/w reference type is determined by \a mode
  * Calls ldlm_lock_addref_internal.
  */
-void ldlm_lock_addref(const struct lustre_handle *lockh, __u32 mode)
+void ldlm_lock_addref(const struct lustre_handle *lockh, enum ldlm_mode mode)
 {
 	struct ldlm_lock *lock;
 
@@ -676,7 +662,8 @@ EXPORT_SYMBOL(ldlm_lock_addref);
  * Removes lock from LRU if it is there.
  * Assumes the LDLM lock is already locked.
  */
-void ldlm_lock_addref_internal_nolock(struct ldlm_lock *lock, __u32 mode)
+void ldlm_lock_addref_internal_nolock(struct ldlm_lock *lock,
+				      enum ldlm_mode mode)
 {
 	ldlm_lock_remove_from_lru(lock);
 	if (mode & (LCK_NL | LCK_CR | LCK_PR)) {
@@ -700,7 +687,7 @@ void ldlm_lock_addref_internal_nolock(struct ldlm_lock *lock, __u32 mode)
  *
  * \retval -EAGAIN lock is being canceled.
  */
-int ldlm_lock_addref_try(const struct lustre_handle *lockh, __u32 mode)
+int ldlm_lock_addref_try(const struct lustre_handle *lockh, enum ldlm_mode mode)
 {
 	struct ldlm_lock *lock;
 	int	       result;
@@ -726,7 +713,7 @@ EXPORT_SYMBOL(ldlm_lock_addref_try);
  * Locks LDLM lock and calls ldlm_lock_addref_internal_nolock to do the work.
  * Only called for local locks.
  */
-void ldlm_lock_addref_internal(struct ldlm_lock *lock, __u32 mode)
+void ldlm_lock_addref_internal(struct ldlm_lock *lock, enum ldlm_mode mode)
 {
 	lock_res_and_lock(lock);
 	ldlm_lock_addref_internal_nolock(lock, mode);
@@ -740,7 +727,8 @@ void ldlm_lock_addref_internal(struct ldlm_lock *lock, __u32 mode)
  * Does NOT add lock to LRU if no r/w references left to accommodate flock locks
  * that cannot be placed in LRU.
  */
-void ldlm_lock_decref_internal_nolock(struct ldlm_lock *lock, __u32 mode)
+void ldlm_lock_decref_internal_nolock(struct ldlm_lock *lock,
+				      enum ldlm_mode mode)
 {
 	LDLM_DEBUG(lock, "ldlm_lock_decref(%s)", ldlm_lockname[mode]);
 	if (mode & (LCK_NL | LCK_CR | LCK_PR)) {
@@ -766,7 +754,7 @@ void ldlm_lock_decref_internal_nolock(struct ldlm_lock *lock, __u32 mode)
  * on the namespace.
  * For blocked LDLM locks if r/w count drops to zero, blocking_ast is called.
  */
-void ldlm_lock_decref_internal(struct ldlm_lock *lock, __u32 mode)
+void ldlm_lock_decref_internal(struct ldlm_lock *lock, enum ldlm_mode mode)
 {
 	struct ldlm_namespace *ns;
 
@@ -786,11 +774,16 @@ void ldlm_lock_decref_internal(struct ldlm_lock *lock, __u32 mode)
 	}
 
 	if (!lock->l_readers && !lock->l_writers &&
-	    ldlm_is_cbpending(lock)) {
+	    (ldlm_is_cbpending(lock) || lock->l_req_mode == LCK_GROUP)) {
 		/* If we received a blocked AST and this was the last reference,
 		 * run the callback.
+		 * Group locks are special:
+		 * They must not go in LRU, but they are not called back
+		 * like non-group locks, instead they are manually released.
+		 * They have an l_writers reference which they keep until
+		 * they are manually released, so we remove them when they have
+		 * no more reader or writer references. - LU-6368
 		 */
-
 		LDLM_DEBUG(lock, "final decref done on cbpending lock");
 
 		LDLM_LOCK_GET(lock); /* dropped by bl thread */
@@ -832,7 +825,7 @@ void ldlm_lock_decref_internal(struct ldlm_lock *lock, __u32 mode)
 /**
  * Decrease reader/writer refcount for LDLM lock with handle \a lockh
  */
-void ldlm_lock_decref(const struct lustre_handle *lockh, __u32 mode)
+void ldlm_lock_decref(const struct lustre_handle *lockh, enum ldlm_mode mode)
 {
 	struct ldlm_lock *lock = __ldlm_handle2lock(lockh, 0);
 
@@ -846,10 +839,9 @@ EXPORT_SYMBOL(ldlm_lock_decref);
  * Decrease reader/writer refcount for LDLM lock with handle
  * \a lockh and mark it for subsequent cancellation once r/w refcount
  * drops to zero instead of putting into LRU.
- *
- * Typical usage is for GROUP locks which we cannot allow to be cached.
  */
-void ldlm_lock_decref_and_cancel(const struct lustre_handle *lockh, __u32 mode)
+void ldlm_lock_decref_and_cancel(const struct lustre_handle *lockh,
+				 enum ldlm_mode mode)
 {
 	struct ldlm_lock *lock = __ldlm_handle2lock(lockh, 0);
 
@@ -1055,88 +1047,173 @@ void ldlm_grant_lock(struct ldlm_lock *lock, struct list_head *work_list)
 }
 
 /**
- * Search for a lock with given properties in a queue.
+ * Describe the overlap between two locks.  itree_overlap_cb data.
+ */
+struct lock_match_data {
+	struct ldlm_lock	*lmd_old;
+	struct ldlm_lock	*lmd_lock;
+	enum ldlm_mode		*lmd_mode;
+	union ldlm_policy_data	*lmd_policy;
+	__u64			 lmd_flags;
+	int			 lmd_unref;
+};
+
+/**
+ * Check if the given @lock meets the criteria for a match.
+ * A reference on the lock is taken if matched.
  *
- * \retval a referenced lock or NULL.  See the flag descriptions below, in the
- * comment above ldlm_lock_match
+ * \param lock	test-against this lock
+ * \param data	parameters
  */
-static struct ldlm_lock *search_queue(struct list_head *queue,
-				      enum ldlm_mode *mode,
-				      ldlm_policy_data_t *policy,
-				      struct ldlm_lock *old_lock,
-				      __u64 flags, int unref)
+static int lock_matches(struct ldlm_lock *lock, struct lock_match_data *data)
 {
-	struct ldlm_lock *lock;
-	struct list_head       *tmp;
+	union ldlm_policy_data *lpol = &lock->l_policy_data;
+	enum ldlm_mode match;
 
-	list_for_each(tmp, queue) {
-		enum ldlm_mode match;
+	if (lock == data->lmd_old)
+		return INTERVAL_ITER_STOP;
 
-		lock = list_entry(tmp, struct ldlm_lock, l_res_link);
-
-		if (lock == old_lock)
-			break;
+	/*
+	 * Check if this lock can be matched.
+	 * Used by LU-2919(exclusive open) for open lease lock
+	 */
+	if (ldlm_is_excl(lock))
+		return INTERVAL_ITER_CONT;
 
-		/* Check if this lock can be matched.
-		 * Used by LU-2919(exclusive open) for open lease lock
-		 */
-		if (ldlm_is_excl(lock))
-			continue;
+	/*
+	 * llite sometimes wants to match locks that will be
+	 * canceled when their users drop, but we allow it to match
+	 * if it passes in CBPENDING and the lock still has users.
+	 * this is generally only going to be used by children
+	 * whose parents already hold a lock so forward progress
+	 * can still happen.
+	 */
+	if (ldlm_is_cbpending(lock) &&
+	    !(data->lmd_flags & LDLM_FL_CBPENDING))
+		return INTERVAL_ITER_CONT;
 
-		/* llite sometimes wants to match locks that will be
-		 * canceled when their users drop, but we allow it to match
-		 * if it passes in CBPENDING and the lock still has users.
-		 * this is generally only going to be used by children
-		 * whose parents already hold a lock so forward progress
-		 * can still happen.
-		 */
-		if (ldlm_is_cbpending(lock) && !(flags & LDLM_FL_CBPENDING))
-			continue;
-		if (!unref && ldlm_is_cbpending(lock) &&
-		    lock->l_readers == 0 && lock->l_writers == 0)
-			continue;
+	if (!data->lmd_unref && ldlm_is_cbpending(lock) &&
+	    !lock->l_readers && !lock->l_writers)
+		return INTERVAL_ITER_CONT;
 
-		if (!(lock->l_req_mode & *mode))
-			continue;
-		match = lock->l_req_mode;
+	if (!(lock->l_req_mode & *data->lmd_mode))
+		return INTERVAL_ITER_CONT;
+	match = lock->l_req_mode;
 
-		if (lock->l_resource->lr_type == LDLM_EXTENT &&
-		    (lock->l_policy_data.l_extent.start >
-		     policy->l_extent.start ||
-		     lock->l_policy_data.l_extent.end < policy->l_extent.end))
-			continue;
+	switch (lock->l_resource->lr_type) {
+	case LDLM_EXTENT:
+		if (lpol->l_extent.start > data->lmd_policy->l_extent.start ||
+		    lpol->l_extent.end < data->lmd_policy->l_extent.end)
+			return INTERVAL_ITER_CONT;
 
 		if (unlikely(match == LCK_GROUP) &&
-		    lock->l_resource->lr_type == LDLM_EXTENT &&
-		    policy->l_extent.gid != LDLM_GID_ANY &&
-		    lock->l_policy_data.l_extent.gid != policy->l_extent.gid)
-			continue;
-
-		/* We match if we have existing lock with same or wider set
+		    data->lmd_policy->l_extent.gid != LDLM_GID_ANY &&
+		    lpol->l_extent.gid != data->lmd_policy->l_extent.gid)
+			return INTERVAL_ITER_CONT;
+		break;
+	case LDLM_IBITS:
+		/*
+		 * We match if we have existing lock with same or wider set
 		 * of bits.
 		 */
-		if (lock->l_resource->lr_type == LDLM_IBITS &&
-		    ((lock->l_policy_data.l_inodebits.bits &
-		      policy->l_inodebits.bits) !=
-		      policy->l_inodebits.bits))
-			continue;
+		if ((lpol->l_inodebits.bits &
+		     data->lmd_policy->l_inodebits.bits) !=
+		    data->lmd_policy->l_inodebits.bits)
+			return INTERVAL_ITER_CONT;
+		break;
+	default:
+		break;
+	}
+	/*
+	 * We match if we have existing lock with same or wider set
+	 * of bits.
+	 */
+	if (!data->lmd_unref && LDLM_HAVE_MASK(lock, GONE))
+		return INTERVAL_ITER_CONT;
+
+	if ((data->lmd_flags & LDLM_FL_LOCAL_ONLY) &&
+	    !ldlm_is_local(lock))
+		return INTERVAL_ITER_CONT;
+
+	if (data->lmd_flags & LDLM_FL_TEST_LOCK) {
+		LDLM_LOCK_GET(lock);
+		ldlm_lock_touch_in_lru(lock);
+	} else {
+		ldlm_lock_addref_internal_nolock(lock, match);
+	}
+
+	*data->lmd_mode = match;
+	data->lmd_lock = lock;
+
+	return INTERVAL_ITER_STOP;
+}
+
+static unsigned int itree_overlap_cb(struct interval_node *in, void *args)
+{
+	struct ldlm_interval *node = to_ldlm_interval(in);
+	struct lock_match_data *data = args;
+	struct ldlm_lock *lock;
+	int rc;
+
+	list_for_each_entry(lock, &node->li_group, l_sl_policy) {
+		rc = lock_matches(lock, data);
+		if (rc == INTERVAL_ITER_STOP)
+			return INTERVAL_ITER_STOP;
+	}
+	return INTERVAL_ITER_CONT;
+}
+
+/**
+ * Search for a lock with given parameters in interval trees.
+ *
+ * \param res	search for a lock in this resource
+ * \param data	parameters
+ *
+ * \retval	a referenced lock or NULL.
+ */
+static struct ldlm_lock *search_itree(struct ldlm_resource *res,
+				      struct lock_match_data *data)
+{
+	struct interval_node_extent ext = {
+		.start	= data->lmd_policy->l_extent.start,
+		.end	= data->lmd_policy->l_extent.end
+	};
+	int idx;
+
+	for (idx = 0; idx < LCK_MODE_NUM; idx++) {
+		struct ldlm_interval_tree *tree = &res->lr_itree[idx];
 
-		if (!unref && LDLM_HAVE_MASK(lock, GONE))
+		if (!tree->lit_root)
 			continue;
 
-		if ((flags & LDLM_FL_LOCAL_ONLY) && !ldlm_is_local(lock))
+		if (!(tree->lit_mode & *data->lmd_mode))
 			continue;
 
-		if (flags & LDLM_FL_TEST_LOCK) {
-			LDLM_LOCK_GET(lock);
-			ldlm_lock_touch_in_lru(lock);
-		} else {
-			ldlm_lock_addref_internal_nolock(lock, match);
-		}
-		*mode = match;
-		return lock;
+		interval_search(tree->lit_root, &ext,
+				itree_overlap_cb, data);
 	}
+	return data->lmd_lock;
+}
 
+/**
+ * Search for a lock with given properties in a queue.
+ *
+ * \param queue	search for a lock in this queue
+ * \param data	parameters
+ *
+ * \retval	a referenced lock or NULL.
+ */
+static struct ldlm_lock *search_queue(struct list_head *queue,
+				      struct lock_match_data *data)
+{
+	struct ldlm_lock *lock;
+	int rc;
+
+	list_for_each_entry(lock, queue, l_res_link) {
+		rc = lock_matches(lock, data);
+		if (rc == INTERVAL_ITER_STOP)
+			return data->lmd_lock;
+	}
 	return NULL;
 }
 
@@ -1147,7 +1224,6 @@ void ldlm_lock_fail_match_locked(struct ldlm_lock *lock)
 		wake_up_all(&lock->l_waitq);
 	}
 }
-EXPORT_SYMBOL(ldlm_lock_fail_match_locked);
 
 /**
  * Mark lock as "matchable" by OST.
@@ -1208,35 +1284,45 @@ EXPORT_SYMBOL(ldlm_lock_allow_match);
 enum ldlm_mode ldlm_lock_match(struct ldlm_namespace *ns, __u64 flags,
 			       const struct ldlm_res_id *res_id,
 			       enum ldlm_type type,
-			       ldlm_policy_data_t *policy,
+			       union ldlm_policy_data *policy,
 			       enum ldlm_mode mode,
 			       struct lustre_handle *lockh, int unref)
 {
+	struct lock_match_data data = {
+		.lmd_old	= NULL,
+		.lmd_lock	= NULL,
+		.lmd_mode	= &mode,
+		.lmd_policy	= policy,
+		.lmd_flags	= flags,
+		.lmd_unref	= unref,
+	};
 	struct ldlm_resource *res;
-	struct ldlm_lock *lock, *old_lock = NULL;
+	struct ldlm_lock *lock;
 	int rc = 0;
 
 	if (!ns) {
-		old_lock = ldlm_handle2lock(lockh);
-		LASSERT(old_lock);
+		data.lmd_old = ldlm_handle2lock(lockh);
+		LASSERT(data.lmd_old);
 
-		ns = ldlm_lock_to_ns(old_lock);
-		res_id = &old_lock->l_resource->lr_name;
-		type = old_lock->l_resource->lr_type;
-		mode = old_lock->l_req_mode;
+		ns = ldlm_lock_to_ns(data.lmd_old);
+		res_id = &data.lmd_old->l_resource->lr_name;
+		type = data.lmd_old->l_resource->lr_type;
+		*data.lmd_mode = data.lmd_old->l_req_mode;
 	}
 
 	res = ldlm_resource_get(ns, NULL, res_id, type, 0);
 	if (IS_ERR(res)) {
-		LASSERT(!old_lock);
+		LASSERT(!data.lmd_old);
 		return 0;
 	}
 
 	LDLM_RESOURCE_ADDREF(res);
 	lock_res(res);
 
-	lock = search_queue(&res->lr_granted, &mode, policy, old_lock,
-			    flags, unref);
+	if (res->lr_type == LDLM_EXTENT)
+		lock = search_itree(res, &data);
+	else
+		lock = search_queue(&res->lr_granted, &data);
 	if (lock) {
 		rc = 1;
 		goto out;
@@ -1245,14 +1331,12 @@ enum ldlm_mode ldlm_lock_match(struct ldlm_namespace *ns, __u64 flags,
 		rc = 0;
 		goto out;
 	}
-	lock = search_queue(&res->lr_waiting, &mode, policy, old_lock,
-			    flags, unref);
+	lock = search_queue(&res->lr_waiting, &data);
 	if (lock) {
 		rc = 1;
 		goto out;
 	}
-
- out:
+out:
 	unlock_res(res);
 	LDLM_RESOURCE_DELREF(res);
 	ldlm_resource_putref(res);
@@ -1324,8 +1408,8 @@ enum ldlm_mode ldlm_lock_match(struct ldlm_namespace *ns, __u64 flags,
 				  (type == LDLM_PLAIN || type == LDLM_IBITS) ?
 					res_id->name[3] : policy->l_extent.end);
 	}
-	if (old_lock)
-		LDLM_LOCK_PUT(old_lock);
+	if (data.lmd_old)
+		LDLM_LOCK_PUT(data.lmd_old);
 
 	return rc ? mode : 0;
 }
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c b/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c
index fde697ebaadc..12647af5a336 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c
@@ -511,23 +511,6 @@ static inline void ldlm_callback_errmsg(struct ptlrpc_request *req,
 		CWARN("Send reply failed, maybe cause bug 21636.\n");
 }
 
-static int ldlm_handle_qc_callback(struct ptlrpc_request *req)
-{
-	struct obd_quotactl *oqctl;
-	struct client_obd *cli = &req->rq_export->exp_obd->u.cli;
-
-	oqctl = req_capsule_client_get(&req->rq_pill, &RMF_OBD_QUOTACTL);
-	if (!oqctl) {
-		CERROR("Can't unpack obd_quotactl\n");
-		return -EPROTO;
-	}
-
-	oqctl->qc_stat = ptlrpc_status_ntoh(oqctl->qc_stat);
-
-	cli->cl_qchk_stat = oqctl->qc_stat;
-	return 0;
-}
-
 /* TODO: handle requests in a similar way as MDT: see mdt_handle_common() */
 static int ldlm_callback_handler(struct ptlrpc_request *req)
 {
@@ -577,13 +560,6 @@ static int ldlm_callback_handler(struct ptlrpc_request *req)
 		rc = ldlm_handle_setinfo(req);
 		ldlm_callback_reply(req, rc);
 		return 0;
-	case OBD_QC_CALLBACK:
-		req_capsule_set(&req->rq_pill, &RQF_QC_CALLBACK);
-		if (OBD_FAIL_CHECK(OBD_FAIL_OBD_QC_CALLBACK_NET))
-			return 0;
-		rc = ldlm_handle_qc_callback(req);
-		ldlm_callback_reply(req, rc);
-		return 0;
 	default:
 		CERROR("unknown opcode %u\n",
 		       lustre_msg_get_opc(req->rq_reqmsg));
@@ -858,7 +834,6 @@ int ldlm_get_ref(void)
 
 	return rc;
 }
-EXPORT_SYMBOL(ldlm_get_ref);
 
 void ldlm_put_ref(void)
 {
@@ -875,7 +850,6 @@ void ldlm_put_ref(void)
 	}
 	mutex_unlock(&ldlm_ref_mutex);
 }
-EXPORT_SYMBOL(ldlm_put_ref);
 
 static ssize_t cancel_unused_locks_before_replay_show(struct kobject *kobj,
 						      struct attribute *attr,
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_plain.c b/drivers/staging/lustre/lustre/ldlm/ldlm_plain.c
index 0aed39c46154..862ea0a1dc97 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_plain.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_plain.c
@@ -54,14 +54,14 @@
 
 #include "ldlm_internal.h"
 
-void ldlm_plain_policy_wire_to_local(const ldlm_wire_policy_data_t *wpolicy,
-				     ldlm_policy_data_t *lpolicy)
+void ldlm_plain_policy_wire_to_local(const union ldlm_wire_policy_data *wpolicy,
+				     union ldlm_policy_data *lpolicy)
 {
 	/* No policy for plain locks */
 }
 
-void ldlm_plain_policy_local_to_wire(const ldlm_policy_data_t *lpolicy,
-				     ldlm_wire_policy_data_t *wpolicy)
+void ldlm_plain_policy_local_to_wire(const union ldlm_policy_data *lpolicy,
+				     union ldlm_wire_policy_data *wpolicy)
 {
 	/* No policy for plain locks */
 }
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c b/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c
index 9a1136e32dfc..8dfb3c8e6b7a 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c
@@ -293,7 +293,7 @@ static int ldlm_cli_pool_recalc(struct ldlm_pool *pl)
 	 * take into account pl->pl_recalc_time here.
 	 */
 	ret = ldlm_cancel_lru(container_of(pl, struct ldlm_namespace, ns_pool),
-			      0, LCF_ASYNC, LDLM_CANCEL_LRUR);
+			      0, LCF_ASYNC, LDLM_LRU_FLAG_LRUR);
 
 out:
 	spin_lock(&pl->pl_lock);
@@ -339,7 +339,7 @@ static int ldlm_cli_pool_shrink(struct ldlm_pool *pl,
 	if (nr == 0)
 		return (unused / 100) * sysctl_vfs_cache_pressure;
 	else
-		return ldlm_cancel_lru(ns, nr, LCF_ASYNC, LDLM_CANCEL_SHRINK);
+		return ldlm_cancel_lru(ns, nr, LCF_ASYNC, LDLM_LRU_FLAG_SHRINK);
 }
 
 static const struct ldlm_pool_ops ldlm_cli_pool_ops = {
@@ -356,10 +356,10 @@ static int ldlm_pool_recalc(struct ldlm_pool *pl)
 	u32 recalc_interval_sec;
 	int count;
 
-	recalc_interval_sec = ktime_get_seconds() - pl->pl_recalc_time;
+	recalc_interval_sec = ktime_get_real_seconds() - pl->pl_recalc_time;
 	if (recalc_interval_sec > 0) {
 		spin_lock(&pl->pl_lock);
-		recalc_interval_sec = ktime_get_seconds() - pl->pl_recalc_time;
+		recalc_interval_sec = ktime_get_real_seconds() - pl->pl_recalc_time;
 
 		if (recalc_interval_sec > 0) {
 			/*
@@ -382,7 +382,7 @@ static int ldlm_pool_recalc(struct ldlm_pool *pl)
 				    count);
 	}
 
-	recalc_interval_sec = pl->pl_recalc_time - ktime_get_seconds() +
+	recalc_interval_sec = pl->pl_recalc_time - ktime_get_real_seconds() +
 			      pl->pl_recalc_period;
 	if (recalc_interval_sec <= 0) {
 		/* DEBUG: should be re-removed after LU-4536 is fixed */
@@ -651,13 +651,13 @@ static void ldlm_pool_debugfs_fini(struct ldlm_pool *pl)
 }
 
 int ldlm_pool_init(struct ldlm_pool *pl, struct ldlm_namespace *ns,
-		   int idx, ldlm_side_t client)
+		   int idx, enum ldlm_side client)
 {
 	int rc;
 
 	spin_lock_init(&pl->pl_lock);
 	atomic_set(&pl->pl_granted, 0);
-	pl->pl_recalc_time = ktime_get_seconds();
+	pl->pl_recalc_time = ktime_get_real_seconds();
 	atomic_set(&pl->pl_lock_volume_factor, 1);
 
 	atomic_set(&pl->pl_grant_rate, 0);
@@ -684,7 +684,6 @@ int ldlm_pool_init(struct ldlm_pool *pl, struct ldlm_namespace *ns,
 
 	return rc;
 }
-EXPORT_SYMBOL(ldlm_pool_init);
 
 void ldlm_pool_fini(struct ldlm_pool *pl)
 {
@@ -698,7 +697,6 @@ void ldlm_pool_fini(struct ldlm_pool *pl)
 	 */
 	POISON(pl, 0x5a, sizeof(*pl));
 }
-EXPORT_SYMBOL(ldlm_pool_fini);
 
 /**
  * Add new taken ldlm lock \a lock into pool \a pl accounting.
@@ -724,7 +722,6 @@ void ldlm_pool_add(struct ldlm_pool *pl, struct ldlm_lock *lock)
 	 * with too long call paths.
 	 */
 }
-EXPORT_SYMBOL(ldlm_pool_add);
 
 /**
  * Remove ldlm lock \a lock from pool \a pl accounting.
@@ -743,7 +740,6 @@ void ldlm_pool_del(struct ldlm_pool *pl, struct ldlm_lock *lock)
 
 	lprocfs_counter_incr(pl->pl_stats, LDLM_POOL_CANCEL_STAT);
 }
-EXPORT_SYMBOL(ldlm_pool_del);
 
 /**
  * Returns current \a pl SLV.
@@ -792,13 +788,12 @@ static struct completion ldlm_pools_comp;
  * count locks from all namespaces (if possible). Returns number of
  * cached locks.
  */
-static unsigned long ldlm_pools_count(ldlm_side_t client, gfp_t gfp_mask)
+static unsigned long ldlm_pools_count(enum ldlm_side client, gfp_t gfp_mask)
 {
 	unsigned long total = 0;
 	int nr_ns;
 	struct ldlm_namespace *ns;
 	struct ldlm_namespace *ns_old = NULL; /* loop detection */
-	void *cookie;
 
 	if (client == LDLM_NAMESPACE_CLIENT && !(gfp_mask & __GFP_FS))
 		return 0;
@@ -806,8 +801,6 @@ static unsigned long ldlm_pools_count(ldlm_side_t client, gfp_t gfp_mask)
 	CDEBUG(D_DLMTRACE, "Request to count %s locks from all pools\n",
 	       client == LDLM_NAMESPACE_CLIENT ? "client" : "server");
 
-	cookie = cl_env_reenter();
-
 	/*
 	 * Find out how many resources we may release.
 	 */
@@ -816,7 +809,6 @@ static unsigned long ldlm_pools_count(ldlm_side_t client, gfp_t gfp_mask)
 		mutex_lock(ldlm_namespace_lock(client));
 		if (list_empty(ldlm_namespace_list(client))) {
 			mutex_unlock(ldlm_namespace_lock(client));
-			cl_env_reexit(cookie);
 			return 0;
 		}
 		ns = ldlm_namespace_first_locked(client);
@@ -842,22 +834,19 @@ static unsigned long ldlm_pools_count(ldlm_side_t client, gfp_t gfp_mask)
 		ldlm_namespace_put(ns);
 	}
 
-	cl_env_reexit(cookie);
 	return total;
 }
 
-static unsigned long ldlm_pools_scan(ldlm_side_t client, int nr, gfp_t gfp_mask)
+static unsigned long ldlm_pools_scan(enum ldlm_side client, int nr,
+				     gfp_t gfp_mask)
 {
 	unsigned long freed = 0;
 	int tmp, nr_ns;
 	struct ldlm_namespace *ns;
-	void *cookie;
 
 	if (client == LDLM_NAMESPACE_CLIENT && !(gfp_mask & __GFP_FS))
 		return -1;
 
-	cookie = cl_env_reenter();
-
 	/*
 	 * Shrink at least ldlm_namespace_nr_read(client) namespaces.
 	 */
@@ -887,7 +876,6 @@ static unsigned long ldlm_pools_scan(ldlm_side_t client, int nr, gfp_t gfp_mask)
 		freed += ldlm_pool_shrink(&ns->ns_pool, cancel, gfp_mask);
 		ldlm_namespace_put(ns);
 	}
-	cl_env_reexit(cookie);
 	/*
 	 * we only decrease the SLV in server pools shrinker, return
 	 * SHRINK_STOP to kernel to avoid needless loop. LU-1128
@@ -908,7 +896,7 @@ static unsigned long ldlm_pools_cli_scan(struct shrinker *s,
 			       sc->gfp_mask);
 }
 
-static int ldlm_pools_recalc(ldlm_side_t client)
+static int ldlm_pools_recalc(enum ldlm_side client)
 {
 	struct ldlm_namespace *ns;
 	struct ldlm_namespace *ns_old = NULL;
@@ -1095,7 +1083,6 @@ int ldlm_pools_init(void)
 
 	return rc;
 }
-EXPORT_SYMBOL(ldlm_pools_init);
 
 void ldlm_pools_fini(void)
 {
@@ -1104,4 +1091,3 @@ void ldlm_pools_fini(void)
 
 	ldlm_pools_thread_stop();
 }
-EXPORT_SYMBOL(ldlm_pools_fini);
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_request.c b/drivers/staging/lustre/lustre/ldlm/ldlm_request.c
index 35ba6f14d95f..c1f8693f94a5 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_request.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_request.c
@@ -93,11 +93,7 @@ static int ldlm_expired_completion_wait(void *data)
 	if (!lock->l_conn_export) {
 		static unsigned long next_dump, last_dump;
 
-		LCONSOLE_WARN("lock timed out (enqueued at %lld, %llds ago)\n",
-			      (s64)lock->l_last_activity,
-			      (s64)(ktime_get_real_seconds() -
-				    lock->l_last_activity));
-		LDLM_DEBUG(lock, "lock timed out (enqueued at %lld, %llds ago); not entering recovery in server code, just going back to sleep",
+		LDLM_ERROR(lock, "lock timed out (enqueued at %lld, %llds ago); not entering recovery in server code, just going back to sleep",
 			   (s64)lock->l_last_activity,
 			   (s64)(ktime_get_real_seconds() -
 				 lock->l_last_activity));
@@ -475,12 +471,7 @@ int ldlm_cli_enqueue_fini(struct obd_export *exp, struct ptlrpc_request *req,
 				   "client-side enqueue, new policy data");
 	}
 
-	if ((*flags) & LDLM_FL_AST_SENT ||
-	    /* Cancel extent locks as soon as possible on a liblustre client,
-	     * because it cannot handle asynchronous ASTs robustly (see
-	     * bug 7311).
-	     */
-	    (LIBLUSTRE_CLIENT && type == LDLM_EXTENT)) {
+	if ((*flags) & LDLM_FL_AST_SENT) {
 		lock_res_and_lock(lock);
 		lock->l_flags |= LDLM_FL_CBPENDING |  LDLM_FL_BL_AST;
 		unlock_res_and_lock(lock);
@@ -602,7 +593,7 @@ int ldlm_prep_elc_req(struct obd_export *exp, struct ptlrpc_request *req,
 		avail = ldlm_capsule_handles_avail(pill, RCL_CLIENT, canceloff);
 
 		flags = ns_connect_lru_resize(ns) ?
-			LDLM_CANCEL_LRUR_NO_WAIT : LDLM_CANCEL_AGED;
+			LDLM_LRU_FLAG_LRUR_NO_WAIT : LDLM_LRU_FLAG_AGED;
 		to_free = !ns_connect_lru_resize(ns) &&
 			  opc == LDLM_ENQUEUE ? 1 : 0;
 
@@ -657,6 +648,27 @@ int ldlm_prep_enqueue_req(struct obd_export *exp, struct ptlrpc_request *req,
 }
 EXPORT_SYMBOL(ldlm_prep_enqueue_req);
 
+static struct ptlrpc_request *ldlm_enqueue_pack(struct obd_export *exp,
+						int lvb_len)
+{
+	struct ptlrpc_request *req;
+	int rc;
+
+	req = ptlrpc_request_alloc(class_exp2cliimp(exp), &RQF_LDLM_ENQUEUE);
+	if (!req)
+		return ERR_PTR(-ENOMEM);
+
+	rc = ldlm_prep_enqueue_req(exp, req, NULL, 0);
+	if (rc) {
+		ptlrpc_request_free(req);
+		return ERR_PTR(rc);
+	}
+
+	req_capsule_set_size(&req->rq_pill, &RMF_DLM_LVB, RCL_SERVER, lvb_len);
+	ptlrpc_request_set_replen(req);
+	return req;
+}
+
 /**
  * Client-side lock enqueue.
  *
@@ -670,7 +682,7 @@ EXPORT_SYMBOL(ldlm_prep_enqueue_req);
 int ldlm_cli_enqueue(struct obd_export *exp, struct ptlrpc_request **reqp,
 		     struct ldlm_enqueue_info *einfo,
 		     const struct ldlm_res_id *res_id,
-		     ldlm_policy_data_t const *policy, __u64 *flags,
+		     union ldlm_policy_data const *policy, __u64 *flags,
 		     void *lvb, __u32 lvb_len, enum lvb_type lvb_type,
 		     struct lustre_handle *lockh, int async)
 {
@@ -727,17 +739,14 @@ int ldlm_cli_enqueue(struct obd_export *exp, struct ptlrpc_request **reqp,
 	lock->l_last_activity = ktime_get_real_seconds();
 
 	/* lock not sent to server yet */
-
 	if (!reqp || !*reqp) {
-		req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp),
-						&RQF_LDLM_ENQUEUE,
-						LUSTRE_DLM_VERSION,
-						LDLM_ENQUEUE);
-		if (!req) {
+		req = ldlm_enqueue_pack(exp, lvb_len);
+		if (IS_ERR(req)) {
 			failed_lock_cleanup(ns, lock, einfo->ei_mode);
 			LDLM_LOCK_RELEASE(lock);
-			return -ENOMEM;
+			return PTR_ERR(req);
 		}
+
 		req_passed_in = 0;
 		if (reqp)
 			*reqp = req;
@@ -757,24 +766,6 @@ int ldlm_cli_enqueue(struct obd_export *exp, struct ptlrpc_request **reqp,
 	body->lock_flags = ldlm_flags_to_wire(*flags);
 	body->lock_handle[0] = *lockh;
 
-	/* Continue as normal. */
-	if (!req_passed_in) {
-		if (lvb_len > 0)
-			req_capsule_extend(&req->rq_pill,
-					   &RQF_LDLM_ENQUEUE_LVB);
-		req_capsule_set_size(&req->rq_pill, &RMF_DLM_LVB, RCL_SERVER,
-				     lvb_len);
-		ptlrpc_request_set_replen(req);
-	}
-
-	/*
-	 * Liblustre client doesn't get extent locks, except for O_APPEND case
-	 * where [0, OBD_OBJECT_EOF] lock is taken, or truncate, where
-	 * [i_size, OBD_OBJECT_EOF] lock is taken.
-	 */
-	LASSERT(ergo(LIBLUSTRE_CLIENT, einfo->ei_type != LDLM_EXTENT ||
-		     policy->l_extent.end == OBD_OBJECT_EOF));
-
 	if (async) {
 		LASSERT(reqp);
 		return 0;
@@ -1022,7 +1013,6 @@ int ldlm_cli_update_pool(struct ptlrpc_request *req)
 
 	return 0;
 }
-EXPORT_SYMBOL(ldlm_cli_update_pool);
 
 /**
  * Client side lock cancel.
@@ -1067,7 +1057,7 @@ int ldlm_cli_cancel(const struct lustre_handle *lockh,
 
 		ns = ldlm_lock_to_ns(lock);
 		flags = ns_connect_lru_resize(ns) ?
-			LDLM_CANCEL_LRUR : LDLM_CANCEL_AGED;
+			LDLM_LRU_FLAG_LRUR : LDLM_LRU_FLAG_AGED;
 		count += ldlm_cancel_lru_local(ns, &cancels, 0, avail - 1,
 					       LCF_BL_AST, flags);
 	}
@@ -1125,7 +1115,6 @@ int ldlm_cli_cancel_list_local(struct list_head *cancels, int count,
 
 	return count;
 }
-EXPORT_SYMBOL(ldlm_cli_cancel_list_local);
 
 /**
  * Cancel as many locks as possible w/o sending any RPCs (e.g. to write back
@@ -1184,6 +1173,14 @@ static enum ldlm_policy_res ldlm_cancel_lrur_policy(struct ldlm_namespace *ns,
 	if (count && added >= count)
 		return LDLM_POLICY_KEEP_LOCK;
 
+	/*
+	 * Despite of the LV, It doesn't make sense to keep the lock which
+	 * is unused for ns_max_age time.
+	 */
+	if (cfs_time_after(cfs_time_current(),
+			   cfs_time_add(lock->l_last_used, ns->ns_max_age)))
+		return LDLM_POLICY_CANCEL_LOCK;
+
 	slv = ldlm_pool_get_slv(pl);
 	lvf = ldlm_pool_get_lvf(pl);
 	la = cfs_duration_sec(cfs_time_sub(cur, lock->l_last_used));
@@ -1287,21 +1284,21 @@ typedef enum ldlm_policy_res (*ldlm_cancel_lru_policy_t)(
 static ldlm_cancel_lru_policy_t
 ldlm_cancel_lru_policy(struct ldlm_namespace *ns, int flags)
 {
-	if (flags & LDLM_CANCEL_NO_WAIT)
+	if (flags & LDLM_LRU_FLAG_NO_WAIT)
 		return ldlm_cancel_no_wait_policy;
 
 	if (ns_connect_lru_resize(ns)) {
-		if (flags & LDLM_CANCEL_SHRINK)
+		if (flags & LDLM_LRU_FLAG_SHRINK)
 			/* We kill passed number of old locks. */
 			return ldlm_cancel_passed_policy;
-		else if (flags & LDLM_CANCEL_LRUR)
+		else if (flags & LDLM_LRU_FLAG_LRUR)
 			return ldlm_cancel_lrur_policy;
-		else if (flags & LDLM_CANCEL_PASSED)
+		else if (flags & LDLM_LRU_FLAG_PASSED)
 			return ldlm_cancel_passed_policy;
-		else if (flags & LDLM_CANCEL_LRUR_NO_WAIT)
+		else if (flags & LDLM_LRU_FLAG_LRUR_NO_WAIT)
 			return ldlm_cancel_lrur_no_wait_policy;
 	} else {
-		if (flags & LDLM_CANCEL_AGED)
+		if (flags & LDLM_LRU_FLAG_AGED)
 			return ldlm_cancel_aged_policy;
 	}
 
@@ -1325,21 +1322,21 @@ ldlm_cancel_lru_policy(struct ldlm_namespace *ns, int flags)
  *
  * Calling policies for enabled LRU resize:
  * ----------------------------------------
- * flags & LDLM_CANCEL_LRUR - use LRU resize policy (SLV from server) to
- *			    cancel not more than \a count locks;
+ * flags & LDLM_LRU_FLAG_LRUR	- use LRU resize policy (SLV from server) to
+ *				  cancel not more than \a count locks;
  *
- * flags & LDLM_CANCEL_PASSED - cancel \a count number of old locks (located at
- *			      the beginning of LRU list);
+ * flags & LDLM_LRU_FLAG_PASSED - cancel \a count number of old locks (located at
+ *				  the beginning of LRU list);
  *
- * flags & LDLM_CANCEL_SHRINK - cancel not more than \a count locks according to
- *			      memory pressure policy function;
+ * flags & LDLM_LRU_FLAG_SHRINK - cancel not more than \a count locks according to
+ *				  memory pressure policy function;
  *
- * flags & LDLM_CANCEL_AGED - cancel \a count locks according to "aged policy".
+ * flags & LDLM_LRU_FLAG_AGED   - cancel \a count locks according to "aged policy".
  *
- * flags & LDLM_CANCEL_NO_WAIT - cancel as many unused locks as possible
- *			       (typically before replaying locks) w/o
- *			       sending any RPCs or waiting for any
- *			       outstanding RPC to complete.
+ * flags & LDLM_LRU_FLAG_NO_WAIT - cancel as many unused locks as possible
+ *				   (typically before replaying locks) w/o
+ *				   sending any RPCs or waiting for any
+ *				   outstanding RPC to complete.
  */
 static int ldlm_prepare_lru_list(struct ldlm_namespace *ns,
 				 struct list_head *cancels, int count, int max,
@@ -1348,7 +1345,7 @@ static int ldlm_prepare_lru_list(struct ldlm_namespace *ns,
 	ldlm_cancel_lru_policy_t pf;
 	struct ldlm_lock *lock, *next;
 	int added = 0, unused, remained;
-	int no_wait = flags & (LDLM_CANCEL_NO_WAIT | LDLM_CANCEL_LRUR_NO_WAIT);
+	int no_wait = flags & (LDLM_LRU_FLAG_NO_WAIT | LDLM_LRU_FLAG_LRUR_NO_WAIT);
 
 	spin_lock(&ns->ns_lock);
 	unused = ns->ns_nr_unused;
@@ -1531,7 +1528,7 @@ int ldlm_cancel_lru(struct ldlm_namespace *ns, int nr,
  */
 int ldlm_cancel_resource_local(struct ldlm_resource *res,
 			       struct list_head *cancels,
-			       ldlm_policy_data_t *policy,
+			       union ldlm_policy_data *policy,
 			       enum ldlm_mode mode, __u64 lock_flags,
 			       enum ldlm_cancel_flags cancel_flags,
 			       void *opaque)
@@ -1648,7 +1645,7 @@ EXPORT_SYMBOL(ldlm_cli_cancel_list);
  */
 int ldlm_cli_cancel_unused_resource(struct ldlm_namespace *ns,
 				    const struct ldlm_res_id *res_id,
-				    ldlm_policy_data_t *policy,
+				    union ldlm_policy_data *policy,
 				    enum ldlm_mode mode,
 				    enum ldlm_cancel_flags flags,
 				    void *opaque)
@@ -1723,7 +1720,7 @@ int ldlm_cli_cancel_unused(struct ldlm_namespace *ns,
 						       opaque);
 	} else {
 		cfs_hash_for_each_nolock(ns->ns_rs_hash,
-					 ldlm_cli_hash_cancel_unused, &arg);
+					 ldlm_cli_hash_cancel_unused, &arg, 0);
 		return ELDLM_OK;
 	}
 }
@@ -1796,7 +1793,7 @@ static void ldlm_namespace_foreach(struct ldlm_namespace *ns,
 	};
 
 	cfs_hash_for_each_nolock(ns->ns_rs_hash,
-				 ldlm_res_iter_helper, &helper);
+				 ldlm_res_iter_helper, &helper, 0);
 }
 
 /* non-blocking function to manipulate a lock whose cb_data is being put away.
@@ -1840,7 +1837,7 @@ static int ldlm_chain_lock_for_replay(struct ldlm_lock *lock, void *closure)
 	 * bug 17614: locks being actively cancelled. Get a reference
 	 * on a lock so that it does not disappear under us (e.g. due to cancel)
 	 */
-	if (!(lock->l_flags & (LDLM_FL_FAILED | LDLM_FL_CANCELING))) {
+	if (!(lock->l_flags & (LDLM_FL_FAILED | LDLM_FL_BL_DONE))) {
 		list_add(&lock->l_pending_chain, list);
 		LDLM_LOCK_GET(lock);
 	}
@@ -1909,7 +1906,7 @@ static int replay_one_lock(struct obd_import *imp, struct ldlm_lock *lock)
 	int flags;
 
 	/* Bug 11974: Do not replay a lock which is actively being canceled */
-	if (ldlm_is_canceling(lock)) {
+	if (ldlm_is_bl_done(lock)) {
 		LDLM_DEBUG(lock, "Not replaying canceled lock:");
 		return 0;
 	}
@@ -2003,11 +2000,11 @@ static void ldlm_cancel_unused_locks_for_replay(struct ldlm_namespace *ns)
 	       ldlm_ns_name(ns), ns->ns_nr_unused);
 
 	/* We don't need to care whether or not LRU resize is enabled
-	 * because the LDLM_CANCEL_NO_WAIT policy doesn't use the
+	 * because the LDLM_LRU_FLAG_NO_WAIT policy doesn't use the
 	 * count parameter
 	 */
 	canceled = ldlm_cancel_lru_local(ns, &cancels, ns->ns_nr_unused, 0,
-					 LCF_LOCAL, LDLM_CANCEL_NO_WAIT);
+					 LCF_LOCAL, LDLM_LRU_FLAG_NO_WAIT);
 
 	CDEBUG(D_DLMTRACE, "Canceled %d unused locks from namespace %s\n",
 	       canceled, ldlm_ns_name(ns));
@@ -2048,4 +2045,3 @@ int ldlm_replay_locks(struct obd_import *imp)
 
 	return rc;
 }
-EXPORT_SYMBOL(ldlm_replay_locks);
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c b/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c
index a09c25aea698..b22f5bae7201 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c
@@ -226,7 +226,7 @@ static ssize_t lru_size_store(struct kobject *kobj, struct attribute *attr,
 
 			/* Try to cancel all @ns_nr_unused locks. */
 			canceled = ldlm_cancel_lru(ns, unused, 0,
-						   LDLM_CANCEL_PASSED);
+						   LDLM_LRU_FLAG_PASSED);
 			if (canceled < unused) {
 				CDEBUG(D_DLMTRACE,
 				       "not all requested locks are canceled, requested: %d, canceled: %d\n",
@@ -237,7 +237,7 @@ static ssize_t lru_size_store(struct kobject *kobj, struct attribute *attr,
 		} else {
 			tmp = ns->ns_max_unused;
 			ns->ns_max_unused = 0;
-			ldlm_cancel_lru(ns, 0, 0, LDLM_CANCEL_PASSED);
+			ldlm_cancel_lru(ns, 0, 0, LDLM_LRU_FLAG_PASSED);
 			ns->ns_max_unused = tmp;
 		}
 		return count;
@@ -262,7 +262,7 @@ static ssize_t lru_size_store(struct kobject *kobj, struct attribute *attr,
 		       "changing namespace %s unused locks from %u to %u\n",
 		       ldlm_ns_name(ns), ns->ns_nr_unused,
 		       (unsigned int)tmp);
-		ldlm_cancel_lru(ns, tmp, LCF_ASYNC, LDLM_CANCEL_PASSED);
+		ldlm_cancel_lru(ns, tmp, LCF_ASYNC, LDLM_LRU_FLAG_PASSED);
 
 		if (!lru_resize) {
 			CDEBUG(D_DLMTRACE,
@@ -276,7 +276,7 @@ static ssize_t lru_size_store(struct kobject *kobj, struct attribute *attr,
 		       ldlm_ns_name(ns), ns->ns_max_unused,
 		       (unsigned int)tmp);
 		ns->ns_max_unused = (unsigned int)tmp;
-		ldlm_cancel_lru(ns, 0, LCF_ASYNC, LDLM_CANCEL_PASSED);
+		ldlm_cancel_lru(ns, 0, LCF_ASYNC, LDLM_LRU_FLAG_PASSED);
 
 		/* Make sure that LRU resize was originally supported before
 		 * turning it on here.
@@ -445,8 +445,8 @@ static struct ldlm_resource *ldlm_resource_getref(struct ldlm_resource *res)
 	return res;
 }
 
-static unsigned ldlm_res_hop_hash(struct cfs_hash *hs,
-				  const void *key, unsigned mask)
+static unsigned int ldlm_res_hop_hash(struct cfs_hash *hs,
+				      const void *key, unsigned int mask)
 {
 	const struct ldlm_res_id     *id  = key;
 	unsigned int		val = 0;
@@ -457,8 +457,8 @@ static unsigned ldlm_res_hop_hash(struct cfs_hash *hs,
 	return val & mask;
 }
 
-static unsigned ldlm_res_hop_fid_hash(struct cfs_hash *hs,
-				      const void *key, unsigned mask)
+static unsigned int ldlm_res_hop_fid_hash(struct cfs_hash *hs,
+					  const void *key, unsigned int mask)
 {
 	const struct ldlm_res_id *id = key;
 	struct lu_fid       fid;
@@ -612,7 +612,7 @@ static struct ldlm_ns_hash_def ldlm_ns_hash_defs[] = {
 
 /** Register \a ns in the list of namespaces */
 static void ldlm_namespace_register(struct ldlm_namespace *ns,
-				    ldlm_side_t client)
+				    enum ldlm_side client)
 {
 	mutex_lock(ldlm_namespace_lock(client));
 	LASSERT(list_empty(&ns->ns_list_chain));
@@ -625,7 +625,7 @@ static void ldlm_namespace_register(struct ldlm_namespace *ns,
  * Create and initialize new empty namespace.
  */
 struct ldlm_namespace *ldlm_namespace_new(struct obd_device *obd, char *name,
-					  ldlm_side_t client,
+					  enum ldlm_side client,
 					  enum ldlm_appetite apt,
 					  enum ldlm_ns_type ns_type)
 {
@@ -855,8 +855,10 @@ int ldlm_namespace_cleanup(struct ldlm_namespace *ns, __u64 flags)
 		return ELDLM_OK;
 	}
 
-	cfs_hash_for_each_nolock(ns->ns_rs_hash, ldlm_resource_clean, &flags);
-	cfs_hash_for_each_nolock(ns->ns_rs_hash, ldlm_resource_complain, NULL);
+	cfs_hash_for_each_nolock(ns->ns_rs_hash, ldlm_resource_clean,
+				 &flags, 0);
+	cfs_hash_for_each_nolock(ns->ns_rs_hash, ldlm_resource_complain,
+				 NULL, 0);
 	return ELDLM_OK;
 }
 EXPORT_SYMBOL(ldlm_namespace_cleanup);
@@ -952,7 +954,7 @@ void ldlm_namespace_free_prior(struct ldlm_namespace *ns,
 
 /** Unregister \a ns from the list of namespaces. */
 static void ldlm_namespace_unregister(struct ldlm_namespace *ns,
-				      ldlm_side_t client)
+				      enum ldlm_side client)
 {
 	mutex_lock(ldlm_namespace_lock(client));
 	LASSERT(!list_empty(&ns->ns_list_chain));
@@ -999,7 +1001,6 @@ void ldlm_namespace_get(struct ldlm_namespace *ns)
 {
 	atomic_inc(&ns->ns_bref);
 }
-EXPORT_SYMBOL(ldlm_namespace_get);
 
 /* This is only for callers that care about refcount */
 static int ldlm_namespace_get_return(struct ldlm_namespace *ns)
@@ -1014,11 +1015,10 @@ void ldlm_namespace_put(struct ldlm_namespace *ns)
 		spin_unlock(&ns->ns_lock);
 	}
 }
-EXPORT_SYMBOL(ldlm_namespace_put);
 
 /** Should be called with ldlm_namespace_lock(client) taken. */
 void ldlm_namespace_move_to_active_locked(struct ldlm_namespace *ns,
-					  ldlm_side_t client)
+					  enum ldlm_side client)
 {
 	LASSERT(!list_empty(&ns->ns_list_chain));
 	LASSERT(mutex_is_locked(ldlm_namespace_lock(client)));
@@ -1027,7 +1027,7 @@ void ldlm_namespace_move_to_active_locked(struct ldlm_namespace *ns,
 
 /** Should be called with ldlm_namespace_lock(client) taken. */
 void ldlm_namespace_move_to_inactive_locked(struct ldlm_namespace *ns,
-					    ldlm_side_t client)
+					    enum ldlm_side client)
 {
 	LASSERT(!list_empty(&ns->ns_list_chain));
 	LASSERT(mutex_is_locked(ldlm_namespace_lock(client)));
@@ -1035,7 +1035,7 @@ void ldlm_namespace_move_to_inactive_locked(struct ldlm_namespace *ns,
 }
 
 /** Should be called with ldlm_namespace_lock(client) taken. */
-struct ldlm_namespace *ldlm_namespace_first_locked(ldlm_side_t client)
+struct ldlm_namespace *ldlm_namespace_first_locked(enum ldlm_side client)
 {
 	LASSERT(mutex_is_locked(ldlm_namespace_lock(client)));
 	LASSERT(!list_empty(ldlm_namespace_list(client)));
@@ -1305,7 +1305,7 @@ void ldlm_res2desc(struct ldlm_resource *res, struct ldlm_resource_desc *desc)
  * Print information about all locks in all namespaces on this node to debug
  * log.
  */
-void ldlm_dump_all_namespaces(ldlm_side_t client, int level)
+void ldlm_dump_all_namespaces(enum ldlm_side client, int level)
 {
 	struct list_head *tmp;
 
@@ -1323,7 +1323,6 @@ void ldlm_dump_all_namespaces(ldlm_side_t client, int level)
 
 	mutex_unlock(ldlm_namespace_lock(client));
 }
-EXPORT_SYMBOL(ldlm_dump_all_namespaces);
 
 static int ldlm_res_hash_dump(struct cfs_hash *hs, struct cfs_hash_bd *bd,
 			      struct hlist_node *hnode, void *arg)
@@ -1355,12 +1354,11 @@ void ldlm_namespace_dump(int level, struct ldlm_namespace *ns)
 
 	cfs_hash_for_each_nolock(ns->ns_rs_hash,
 				 ldlm_res_hash_dump,
-				 (void *)(unsigned long)level);
+				 (void *)(unsigned long)level, 0);
 	spin_lock(&ns->ns_lock);
 	ns->ns_next_dump = cfs_time_shift(10);
 	spin_unlock(&ns->ns_lock);
 }
-EXPORT_SYMBOL(ldlm_namespace_dump);
 
 /**
  * Print information about all locks in this resource to debug log.
diff --git a/drivers/staging/lustre/lustre/llite/Makefile b/drivers/staging/lustre/lustre/llite/Makefile
index 1ac0940bd8df..322d4fa63f5d 100644
--- a/drivers/staging/lustre/lustre/llite/Makefile
+++ b/drivers/staging/lustre/lustre/llite/Makefile
@@ -1,7 +1,7 @@
 obj-$(CONFIG_LUSTRE_FS) += lustre.o
-lustre-y := dcache.o dir.o file.o llite_close.o llite_lib.o llite_nfs.o \
-	    rw.o namei.o symlink.o llite_mmap.o range_lock.o \
-	    xattr.o xattr_cache.o rw26.o super25.o statahead.o \
-	    glimpse.o lcommon_cl.o lcommon_misc.o \
-	    vvp_dev.o vvp_page.o vvp_lock.o vvp_io.o vvp_object.o vvp_req.o \
+lustre-y := dcache.o dir.o file.o llite_lib.o llite_nfs.o \
+	    rw.o rw26.o namei.o symlink.o llite_mmap.o range_lock.o \
+	    xattr.o xattr_cache.o xattr_security.o \
+	    super25.o statahead.o glimpse.o lcommon_cl.o lcommon_misc.o \
+	    vvp_dev.o vvp_page.o vvp_lock.o vvp_io.o vvp_object.o \
 	    lproc_llite.o
diff --git a/drivers/staging/lustre/lustre/llite/dir.c b/drivers/staging/lustre/lustre/llite/dir.c
index 7f32a539d260..ea5d247a3f70 100644
--- a/drivers/staging/lustre/lustre/llite/dir.c
+++ b/drivers/staging/lustre/lustre/llite/dir.c
@@ -51,6 +51,8 @@
 #include "../include/lustre_dlm.h"
 #include "../include/lustre_fid.h"
 #include "../include/lustre_kernelcomm.h"
+#include "../include/lustre_swab.h"
+
 #include "llite_internal.h"
 
 /*
@@ -410,6 +412,8 @@ static int ll_dir_setdirstripe(struct inode *parent, struct lmv_user_md *lump,
 	struct ptlrpc_request *request = NULL;
 	struct md_op_data *op_data;
 	struct ll_sb_info *sbi = ll_i2sbi(parent);
+	struct inode *inode = NULL;
+	struct dentry dentry;
 	int err;
 
 	if (unlikely(lump->lum_magic != LMV_USER_MAGIC))
@@ -419,6 +423,10 @@ static int ll_dir_setdirstripe(struct inode *parent, struct lmv_user_md *lump,
 	       PFID(ll_inode2fid(parent)), parent, dirname,
 	       (int)lump->lum_stripe_offset, lump->lum_stripe_count);
 
+	if (lump->lum_stripe_count > 1 &&
+	    !(exp_connect_flags(sbi->ll_md_exp) & OBD_CONNECT_DIR_STRIPE))
+		return -EINVAL;
+
 	if (lump->lum_magic != cpu_to_le32(LMV_USER_MAGIC))
 		lustre_swab_lmv_user_md(lump);
 
@@ -439,8 +447,17 @@ static int ll_dir_setdirstripe(struct inode *parent, struct lmv_user_md *lump,
 			from_kgid(&init_user_ns, current_fsgid()),
 			cfs_curproc_cap_pack(), 0, &request);
 	ll_finish_md_op_data(op_data);
+
+	err = ll_prep_inode(&inode, request, parent->i_sb, NULL);
 	if (err)
 		goto err_exit;
+
+	memset(&dentry, 0, sizeof(dentry));
+	dentry.d_inode = inode;
+
+	err = ll_init_security(&dentry, inode, parent);
+	iput(inode);
+
 err_exit:
 	ptlrpc_req_finished(request);
 	return err;
@@ -501,8 +518,7 @@ int ll_dir_setstripe(struct inode *inode, struct lov_user_md *lump,
 		return PTR_ERR(op_data);
 
 	/* swabbing is done in lov_setstripe() on server side */
-	rc = md_setattr(sbi->ll_md_exp, op_data, lump, lum_size,
-			NULL, 0, &req, NULL);
+	rc = md_setattr(sbi->ll_md_exp, op_data, lump, lum_size, &req);
 	ll_finish_md_op_data(op_data);
 	ptlrpc_req_finished(req);
 	if (rc) {
@@ -682,7 +698,7 @@ static int ll_ioc_copy_start(struct super_block *sb, struct hsm_copy *copy)
 {
 	struct ll_sb_info		*sbi = ll_s2sbi(sb);
 	struct hsm_progress_kernel	 hpk;
-	int				 rc;
+	int rc2, rc = 0;
 
 	/* Forge a hsm_progress based on data from copy. */
 	hpk.hpk_fid = copy->hc_hai.hai_fid;
@@ -732,10 +748,10 @@ progress:
 	/* On error, the request should be considered as completed */
 	if (hpk.hpk_errval > 0)
 		hpk.hpk_flags |= HP_FLAG_COMPLETED;
-	rc = obd_iocontrol(LL_IOC_HSM_PROGRESS, sbi->ll_md_exp, sizeof(hpk),
-			   &hpk, NULL);
+	rc2 = obd_iocontrol(LL_IOC_HSM_PROGRESS, sbi->ll_md_exp, sizeof(hpk),
+			    &hpk, NULL);
 
-	return rc;
+	return rc ? rc : rc2;
 }
 
 /**
@@ -757,7 +773,7 @@ static int ll_ioc_copy_end(struct super_block *sb, struct hsm_copy *copy)
 {
 	struct ll_sb_info		*sbi = ll_s2sbi(sb);
 	struct hsm_progress_kernel	 hpk;
-	int				 rc;
+	int rc2, rc = 0;
 
 	/* If you modify the logic here, also check llapi_hsm_copy_end(). */
 	/* Take care: copy->hc_hai.hai_action, len, gid and data are not
@@ -823,18 +839,18 @@ static int ll_ioc_copy_end(struct super_block *sb, struct hsm_copy *copy)
 			 * when the file will not be modified for some tunable
 			 * time
 			 */
-			/* we do not notify caller */
 			hpk.hpk_flags &= ~HP_FLAG_RETRY;
+			rc = -EBUSY;
 			/* hpk_errval must be >= 0 */
-			hpk.hpk_errval = EBUSY;
+			hpk.hpk_errval = -rc;
 		}
 	}
 
 progress:
-	rc = obd_iocontrol(LL_IOC_HSM_PROGRESS, sbi->ll_md_exp, sizeof(hpk),
-			   &hpk, NULL);
+	rc2 = obd_iocontrol(LL_IOC_HSM_PROGRESS, sbi->ll_md_exp, sizeof(hpk),
+			    &hpk, NULL);
 
-	return rc;
+	return rc ? rc : rc2;
 }
 
 static int copy_and_ioctl(int cmd, struct obd_export *exp,
@@ -862,10 +878,6 @@ static int quotactl_ioctl(struct ll_sb_info *sbi, struct if_quotactl *qctl)
 	int rc = 0;
 
 	switch (cmd) {
-	case LUSTRE_Q_INVALIDATE:
-	case LUSTRE_Q_FINVALIDATE:
-	case Q_QUOTAON:
-	case Q_QUOTAOFF:
 	case Q_SETQUOTA:
 	case Q_SETINFO:
 		if (!capable(CFS_CAP_SYS_ADMIN))
@@ -930,10 +942,6 @@ static int quotactl_ioctl(struct ll_sb_info *sbi, struct if_quotactl *qctl)
 		QCTL_COPY(oqctl, qctl);
 		rc = obd_quotactl(sbi->ll_md_exp, oqctl);
 		if (rc) {
-			if (rc != -EALREADY && cmd == Q_QUOTAON) {
-				oqctl->qc_cmd = Q_QUOTAOFF;
-				obd_quotactl(sbi->ll_md_exp, oqctl);
-			}
 			kfree(oqctl);
 			return rc;
 		}
@@ -1077,7 +1085,7 @@ static long ll_dir_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 			goto out_free;
 		}
 
-		rc = ll_get_fid_by_name(inode, filename, namelen, NULL);
+		rc = ll_get_fid_by_name(inode, filename, namelen, NULL, NULL);
 		if (rc < 0) {
 			CERROR("%s: lookup %.*s failed: rc = %d\n",
 			       ll_get_fsname(inode->i_sb, NULL, 0), namelen,
@@ -1189,6 +1197,7 @@ lmv_out_free:
 		struct lmv_user_md *tmp = NULL;
 		union lmv_mds_md *lmm = NULL;
 		u64 valid = 0;
+		int max_stripe_count;
 		int stripe_count;
 		int mdt_index;
 		int lum_size;
@@ -1200,6 +1209,7 @@ lmv_out_free:
 		if (copy_from_user(&lum, ulmv, sizeof(*ulmv)))
 			return -EFAULT;
 
+		max_stripe_count = lum.lum_stripe_count;
 		/*
 		 * lum_magic will indicate which stripe the ioctl will like
 		 * to get, LMV_MAGIC_V1 is for normal LMV stripe, LMV_USER_MAGIC
@@ -1219,9 +1229,6 @@ lmv_out_free:
 
 		/* Get default LMV EA */
 		if (lum.lum_magic == LMV_USER_MAGIC) {
-			if (rc)
-				goto finish_req;
-
 			if (lmmsize > sizeof(*ulmv)) {
 				rc = -EINVAL;
 				goto finish_req;
@@ -1234,6 +1241,16 @@ lmv_out_free:
 		}
 
 		stripe_count = lmv_mds_md_stripe_count_get(lmm);
+		if (max_stripe_count < stripe_count) {
+			lum.lum_stripe_count = stripe_count;
+			if (copy_to_user(ulmv, &lum, sizeof(lum))) {
+				rc = -EFAULT;
+				goto finish_req;
+			}
+			rc = -E2BIG;
+			goto finish_req;
+		}
+
 		lum_size = lmv_user_md_size(stripe_count, LMV_MAGIC_V1);
 		tmp = kzalloc(lum_size, GFP_NOFS);
 		if (!tmp) {
@@ -1370,134 +1387,6 @@ out_req:
 			ll_putname(filename);
 		return rc;
 	}
-	case IOC_LOV_GETINFO: {
-		struct lov_user_mds_data __user *lumd;
-		struct lov_stripe_md *lsm;
-		struct lov_user_md __user *lum;
-		struct lov_mds_md *lmm;
-		int lmmsize;
-		lstat_t st;
-
-		lumd = (struct lov_user_mds_data __user *)arg;
-		lum = &lumd->lmd_lmm;
-
-		rc = ll_get_max_mdsize(sbi, &lmmsize);
-		if (rc)
-			return rc;
-
-		lmm = libcfs_kvzalloc(lmmsize, GFP_NOFS);
-		if (!lmm)
-			return -ENOMEM;
-		if (copy_from_user(lmm, lum, lmmsize)) {
-			rc = -EFAULT;
-			goto free_lmm;
-		}
-
-		switch (lmm->lmm_magic) {
-		case LOV_USER_MAGIC_V1:
-			if (cpu_to_le32(LOV_USER_MAGIC_V1) == LOV_USER_MAGIC_V1)
-				break;
-			/* swab objects first so that stripes num will be sane */
-			lustre_swab_lov_user_md_objects(
-				((struct lov_user_md_v1 *)lmm)->lmm_objects,
-				((struct lov_user_md_v1 *)lmm)->lmm_stripe_count);
-			lustre_swab_lov_user_md_v1((struct lov_user_md_v1 *)lmm);
-			break;
-		case LOV_USER_MAGIC_V3:
-			if (cpu_to_le32(LOV_USER_MAGIC_V3) == LOV_USER_MAGIC_V3)
-				break;
-			/* swab objects first so that stripes num will be sane */
-			lustre_swab_lov_user_md_objects(
-				((struct lov_user_md_v3 *)lmm)->lmm_objects,
-				((struct lov_user_md_v3 *)lmm)->lmm_stripe_count);
-			lustre_swab_lov_user_md_v3((struct lov_user_md_v3 *)lmm);
-			break;
-		default:
-			rc = -EINVAL;
-			goto free_lmm;
-		}
-
-		rc = obd_unpackmd(sbi->ll_dt_exp, &lsm, lmm, lmmsize);
-		if (rc < 0) {
-			rc = -ENOMEM;
-			goto free_lmm;
-		}
-
-		/* Perform glimpse_size operation. */
-		memset(&st, 0, sizeof(st));
-
-		rc = ll_glimpse_ioctl(sbi, lsm, &st);
-		if (rc)
-			goto free_lsm;
-
-		if (copy_to_user(&lumd->lmd_st, &st, sizeof(st))) {
-			rc = -EFAULT;
-			goto free_lsm;
-		}
-
-free_lsm:
-		obd_free_memmd(sbi->ll_dt_exp, &lsm);
-free_lmm:
-		kvfree(lmm);
-		return rc;
-	}
-	case OBD_IOC_QUOTACHECK: {
-		struct obd_quotactl *oqctl;
-		int error = 0;
-
-		if (!capable(CFS_CAP_SYS_ADMIN))
-			return -EPERM;
-
-		oqctl = kzalloc(sizeof(*oqctl), GFP_NOFS);
-		if (!oqctl)
-			return -ENOMEM;
-		oqctl->qc_type = arg;
-		rc = obd_quotacheck(sbi->ll_md_exp, oqctl);
-		if (rc < 0) {
-			CDEBUG(D_INFO, "md_quotacheck failed: rc %d\n", rc);
-			error = rc;
-		}
-
-		rc = obd_quotacheck(sbi->ll_dt_exp, oqctl);
-		if (rc < 0)
-			CDEBUG(D_INFO, "obd_quotacheck failed: rc %d\n", rc);
-
-		kfree(oqctl);
-		return error ?: rc;
-	}
-	case OBD_IOC_POLL_QUOTACHECK: {
-		struct if_quotacheck *check;
-
-		if (!capable(CFS_CAP_SYS_ADMIN))
-			return -EPERM;
-
-		check = kzalloc(sizeof(*check), GFP_NOFS);
-		if (!check)
-			return -ENOMEM;
-
-		rc = obd_iocontrol(cmd, sbi->ll_md_exp, 0, (void *)check,
-				   NULL);
-		if (rc) {
-			CDEBUG(D_QUOTA, "mdc ioctl %d failed: %d\n", cmd, rc);
-			if (copy_to_user((void __user *)arg, check,
-					 sizeof(*check)))
-				CDEBUG(D_QUOTA, "copy_to_user failed\n");
-			goto out_poll;
-		}
-
-		rc = obd_iocontrol(cmd, sbi->ll_dt_exp, 0, (void *)check,
-				   NULL);
-		if (rc) {
-			CDEBUG(D_QUOTA, "osc ioctl %d failed: %d\n", cmd, rc);
-			if (copy_to_user((void __user *)arg, check,
-					 sizeof(*check)))
-				CDEBUG(D_QUOTA, "copy_to_user failed\n");
-			goto out_poll;
-		}
-out_poll:
-		kfree(check);
-		return rc;
-	}
 	case OBD_IOC_QUOTACTL: {
 		struct if_quotactl *qctl;
 
@@ -1536,7 +1425,7 @@ out_quotactl:
 		exp = count ? sbi->ll_md_exp : sbi->ll_dt_exp;
 		vallen = sizeof(count);
 		rc = obd_get_info(NULL, exp, sizeof(KEY_TGT_COUNT),
-				  KEY_TGT_COUNT, &vallen, &count, NULL);
+				  KEY_TGT_COUNT, &vallen, &count);
 		if (rc) {
 			CERROR("get target count failed: %d\n", rc);
 			return rc;
diff --git a/drivers/staging/lustre/lustre/llite/file.c b/drivers/staging/lustre/lustre/llite/file.c
index e1d784bae064..f634c11216e6 100644
--- a/drivers/staging/lustre/lustre/llite/file.c
+++ b/drivers/staging/lustre/lustre/llite/file.c
@@ -44,6 +44,7 @@
 #include <linux/mount.h>
 #include "../include/lustre/ll_fiemap.h"
 #include "../include/lustre/lustre_ioctl.h"
+#include "../include/lustre_swab.h"
 
 #include "../include/cl_object.h"
 #include "llite_internal.h"
@@ -75,60 +76,56 @@ static void ll_file_data_put(struct ll_file_data *fd)
 		kmem_cache_free(ll_file_data_slab, fd);
 }
 
-void ll_pack_inode2opdata(struct inode *inode, struct md_op_data *op_data,
-			  struct lustre_handle *fh)
+/**
+ * Packs all the attributes into @op_data for the CLOSE rpc.
+ */
+static void ll_prepare_close(struct inode *inode, struct md_op_data *op_data,
+			     struct obd_client_handle *och)
 {
-	op_data->op_fid1 = ll_i2info(inode)->lli_fid;
+	struct ll_inode_info *lli = ll_i2info(inode);
+
+	ll_prep_md_op_data(op_data, inode, NULL, NULL,
+			   0, 0, LUSTRE_OPC_ANY, NULL);
+
 	op_data->op_attr.ia_mode = inode->i_mode;
 	op_data->op_attr.ia_atime = inode->i_atime;
 	op_data->op_attr.ia_mtime = inode->i_mtime;
 	op_data->op_attr.ia_ctime = inode->i_ctime;
 	op_data->op_attr.ia_size = i_size_read(inode);
+	op_data->op_attr.ia_valid |= ATTR_MODE | ATTR_ATIME | ATTR_ATIME_SET |
+				     ATTR_MTIME | ATTR_MTIME_SET |
+				     ATTR_CTIME | ATTR_CTIME_SET;
 	op_data->op_attr_blocks = inode->i_blocks;
 	op_data->op_attr_flags = ll_inode_to_ext_flags(inode->i_flags);
-	op_data->op_ioepoch = ll_i2info(inode)->lli_ioepoch;
-	if (fh)
-		op_data->op_handle = *fh;
+	op_data->op_handle = och->och_fh;
 
-	if (ll_i2info(inode)->lli_flags & LLIF_DATA_MODIFIED)
+	/*
+	 * For HSM: if inode data has been modified, pack it so that
+	 * MDT can set data dirty flag in the archive.
+	 */
+	if (och->och_flags & FMODE_WRITE &&
+	    test_and_clear_bit(LLIF_DATA_MODIFIED, &lli->lli_flags))
 		op_data->op_bias |= MDS_DATA_MODIFIED;
 }
 
 /**
- * Closes the IO epoch and packs all the attributes into @op_data for
- * the CLOSE rpc.
+ * Perform a close, possibly with a bias.
+ * The meaning of "data" depends on the value of "bias".
+ *
+ * If \a bias is MDS_HSM_RELEASE then \a data is a pointer to the data version.
+ * If \a bias is MDS_CLOSE_LAYOUT_SWAP then \a data is a pointer to the inode to
+ * swap layouts with.
  */
-static void ll_prepare_close(struct inode *inode, struct md_op_data *op_data,
-			     struct obd_client_handle *och)
-{
-	op_data->op_attr.ia_valid = ATTR_MODE | ATTR_ATIME | ATTR_ATIME_SET |
-					ATTR_MTIME | ATTR_MTIME_SET |
-					ATTR_CTIME | ATTR_CTIME_SET;
-
-	if (!(och->och_flags & FMODE_WRITE))
-		goto out;
-
-	if (!exp_connect_som(ll_i2mdexp(inode)) || !S_ISREG(inode->i_mode))
-		op_data->op_attr.ia_valid |= ATTR_SIZE | ATTR_BLOCKS;
-	else
-		ll_ioepoch_close(inode, op_data, &och, 0);
-
-out:
-	ll_pack_inode2opdata(inode, op_data, &och->och_fh);
-	ll_prep_md_op_data(op_data, inode, NULL, NULL,
-			   0, 0, LUSTRE_OPC_ANY, NULL);
-}
-
 static int ll_close_inode_openhandle(struct obd_export *md_exp,
-				     struct inode *inode,
 				     struct obd_client_handle *och,
-				     const __u64 *data_version)
+				     struct inode *inode,
+				     enum mds_op_bias bias,
+				     void *data)
 {
 	struct obd_export *exp = ll_i2mdexp(inode);
 	struct md_op_data *op_data;
 	struct ptlrpc_request *req = NULL;
 	struct obd_device *obd = class_exp2obd(exp);
-	int epoch_close = 1;
 	int rc;
 
 	if (!obd) {
@@ -150,65 +147,51 @@ static int ll_close_inode_openhandle(struct obd_export *md_exp,
 	}
 
 	ll_prepare_close(inode, op_data, och);
-	if (data_version) {
-		/* Pass in data_version implies release. */
+	switch (bias) {
+	case MDS_CLOSE_LAYOUT_SWAP:
+		LASSERT(data);
+		op_data->op_bias |= MDS_CLOSE_LAYOUT_SWAP;
+		op_data->op_data_version = 0;
+		op_data->op_lease_handle = och->och_lease_handle;
+		op_data->op_fid2 = *ll_inode2fid(data);
+		break;
+
+	case MDS_HSM_RELEASE:
+		LASSERT(data);
 		op_data->op_bias |= MDS_HSM_RELEASE;
-		op_data->op_data_version = *data_version;
+		op_data->op_data_version = *(__u64 *)data;
 		op_data->op_lease_handle = och->och_lease_handle;
 		op_data->op_attr.ia_valid |= ATTR_SIZE | ATTR_BLOCKS;
+		break;
+
+	default:
+		LASSERT(!data);
+		break;
 	}
-	epoch_close = op_data->op_flags & MF_EPOCH_CLOSE;
+
 	rc = md_close(md_exp, op_data, och->och_mod, &req);
-	if (rc == -EAGAIN) {
-		/* This close must have the epoch closed. */
-		LASSERT(epoch_close);
-		/* MDS has instructed us to obtain Size-on-MDS attribute from
-		 * OSTs and send setattr to back to MDS.
-		 */
-		rc = ll_som_update(inode, op_data);
-		if (rc) {
-			CERROR("%s: inode "DFID" mdc Size-on-MDS update failed: rc = %d\n",
-			       ll_i2mdexp(inode)->exp_obd->obd_name,
-			       PFID(ll_inode2fid(inode)), rc);
-			rc = 0;
-		}
-	} else if (rc) {
+	if (rc) {
 		CERROR("%s: inode "DFID" mdc close failed: rc = %d\n",
 		       ll_i2mdexp(inode)->exp_obd->obd_name,
 		       PFID(ll_inode2fid(inode)), rc);
 	}
 
-	/* DATA_MODIFIED flag was successfully sent on close, cancel data
-	 * modification flag.
-	 */
-	if (rc == 0 && (op_data->op_bias & MDS_DATA_MODIFIED)) {
-		struct ll_inode_info *lli = ll_i2info(inode);
-
-		spin_lock(&lli->lli_lock);
-		lli->lli_flags &= ~LLIF_DATA_MODIFIED;
-		spin_unlock(&lli->lli_lock);
-	}
-
-	if (rc == 0 && op_data->op_bias & MDS_HSM_RELEASE) {
+	if (op_data->op_bias & (MDS_HSM_RELEASE | MDS_CLOSE_LAYOUT_SWAP) &&
+	    !rc) {
 		struct mdt_body *body;
 
 		body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
-		if (!(body->mbo_valid & OBD_MD_FLRELEASED))
+		if (!(body->mbo_valid & OBD_MD_CLOSE_INTENT_EXECED))
 			rc = -EBUSY;
 	}
 
 	ll_finish_md_op_data(op_data);
 
 out:
-	if (exp_connect_som(exp) && !epoch_close &&
-	    S_ISREG(inode->i_mode) && (och->och_flags & FMODE_WRITE)) {
-		ll_queue_done_writing(inode, LLIF_DONE_WRITING);
-	} else {
-		md_clear_open_replay_data(md_exp, och);
-		/* Free @och if it is not waiting for DONE_WRITING. */
-		och->och_fh.cookie = DEAD_HANDLE_MAGIC;
-		kfree(och);
-	}
+	md_clear_open_replay_data(md_exp, och);
+	och->och_fh.cookie = DEAD_HANDLE_MAGIC;
+	kfree(och);
+
 	if (req) /* This is close request */
 		ptlrpc_req_finished(req);
 	return rc;
@@ -252,7 +235,7 @@ int ll_md_real_close(struct inode *inode, fmode_t fmode)
 		 * be closed.
 		 */
 		rc = ll_close_inode_openhandle(ll_i2sbi(inode)->ll_md_exp,
-					       inode, och, NULL);
+					       och, inode, 0, NULL);
 	}
 
 	return rc;
@@ -266,7 +249,9 @@ static int ll_md_close(struct obd_export *md_exp, struct inode *inode,
 	int lockmode;
 	__u64 flags = LDLM_FL_BLOCK_GRANTED | LDLM_FL_TEST_LOCK;
 	struct lustre_handle lockh;
-	ldlm_policy_data_t policy = {.l_inodebits = {MDS_INODELOCK_OPEN} };
+	union ldlm_policy_data policy = {
+		.l_inodebits = { MDS_INODELOCK_OPEN }
+	};
 	int rc = 0;
 
 	/* clear group lock, if present */
@@ -288,7 +273,8 @@ static int ll_md_close(struct obd_export *md_exp, struct inode *inode,
 	}
 
 	if (fd->fd_och) {
-		rc = ll_close_inode_openhandle(md_exp, inode, fd->fd_och, NULL);
+		rc = ll_close_inode_openhandle(md_exp, fd->fd_och, inode, 0,
+					       NULL);
 		fd->fd_och = NULL;
 		goto out;
 	}
@@ -437,20 +423,6 @@ out:
 	return rc;
 }
 
-/**
- * Assign an obtained @ioepoch to client's inode. No lock is needed, MDS does
- * not believe attributes if a few ioepoch holders exist. Attributes for
- * previous ioepoch if new one is opened are also skipped by MDS.
- */
-void ll_ioepoch_open(struct ll_inode_info *lli, __u64 ioepoch)
-{
-	if (ioepoch && lli->lli_ioepoch != ioepoch) {
-		lli->lli_ioepoch = ioepoch;
-		CDEBUG(D_INODE, "Epoch %llu opened on "DFID"\n",
-		       ioepoch, PFID(&lli->lli_fid));
-	}
-}
-
 static int ll_och_fill(struct obd_export *md_exp, struct lookup_intent *it,
 		       struct obd_client_handle *och)
 {
@@ -470,23 +442,17 @@ static int ll_local_open(struct file *file, struct lookup_intent *it,
 			 struct ll_file_data *fd, struct obd_client_handle *och)
 {
 	struct inode *inode = file_inode(file);
-	struct ll_inode_info *lli = ll_i2info(inode);
 
 	LASSERT(!LUSTRE_FPRIVATE(file));
 
 	LASSERT(fd);
 
 	if (och) {
-		struct mdt_body *body;
 		int rc;
 
 		rc = ll_och_fill(ll_i2sbi(inode)->ll_md_exp, it, och);
 		if (rc != 0)
 			return rc;
-
-		body = req_capsule_server_get(&it->it_request->rq_pill,
-					      &RMF_MDT_BODY);
-		ll_ioepoch_open(lli, body->mbo_ioepoch);
 	}
 
 	LUSTRE_FPRIVATE(file) = fd;
@@ -677,12 +643,6 @@ restart:
 	if (!S_ISREG(inode->i_mode))
 		goto out_och_free;
 
-	if (!lli->lli_has_smd &&
-	    (cl_is_lov_delay_create(file->f_flags) ||
-	     (file->f_mode & FMODE_WRITE) == 0)) {
-		CDEBUG(D_INODE, "object creation was delayed\n");
-		goto out_och_free;
-	}
 	cl_lov_delay_create_clear(&file->f_flags);
 	goto out_och_free;
 
@@ -867,7 +827,7 @@ out_close:
 		it.it_lock_mode = 0;
 		och->och_lease_handle.cookie = 0ULL;
 	}
-	rc2 = ll_close_inode_openhandle(sbi->ll_md_exp, inode, och, NULL);
+	rc2 = ll_close_inode_openhandle(sbi->ll_md_exp, och, inode, 0, NULL);
 	if (rc2 < 0)
 		CERROR("%s: error closing file "DFID": %d\n",
 		       ll_get_fsname(inode->i_sb, NULL, 0),
@@ -881,6 +841,69 @@ out:
 }
 
 /**
+ * Check whether a layout swap can be done between two inodes.
+ *
+ * \param[in] inode1  First inode to check
+ * \param[in] inode2  Second inode to check
+ *
+ * \retval 0 on success, layout swap can be performed between both inodes
+ * \retval negative error code if requirements are not met
+ */
+static int ll_check_swap_layouts_validity(struct inode *inode1,
+					  struct inode *inode2)
+{
+	if (!S_ISREG(inode1->i_mode) || !S_ISREG(inode2->i_mode))
+		return -EINVAL;
+
+	if (inode_permission(inode1, MAY_WRITE) ||
+	    inode_permission(inode2, MAY_WRITE))
+		return -EPERM;
+
+	if (inode1->i_sb != inode2->i_sb)
+		return -EXDEV;
+
+	return 0;
+}
+
+static int ll_swap_layouts_close(struct obd_client_handle *och,
+				 struct inode *inode, struct inode *inode2)
+{
+	const struct lu_fid *fid1 = ll_inode2fid(inode);
+	const struct lu_fid *fid2;
+	int rc;
+
+	CDEBUG(D_INODE, "%s: biased close of file " DFID "\n",
+	       ll_get_fsname(inode->i_sb, NULL, 0), PFID(fid1));
+
+	rc = ll_check_swap_layouts_validity(inode, inode2);
+	if (rc < 0)
+		goto out_free_och;
+
+	/* We now know that inode2 is a lustre inode */
+	fid2 = ll_inode2fid(inode2);
+
+	rc = lu_fid_cmp(fid1, fid2);
+	if (!rc) {
+		rc = -EINVAL;
+		goto out_free_och;
+	}
+
+	/*
+	 * Close the file and swap layouts between inode & inode2.
+	 * NB: lease lock handle is released in mdc_close_layout_swap_pack()
+	 * because we still need it to pack l_remote_handle to MDT.
+	 */
+	rc = ll_close_inode_openhandle(ll_i2sbi(inode)->ll_md_exp, och, inode,
+				       MDS_CLOSE_LAYOUT_SWAP, inode2);
+
+	och = NULL; /* freed in ll_close_inode_openhandle() */
+
+out_free_och:
+	kfree(och);
+	return rc;
+}
+
+/**
  * Release lease and close the file.
  * It will check if the lease has ever broken.
  */
@@ -907,84 +930,7 @@ static int ll_lease_close(struct obd_client_handle *och, struct inode *inode,
 		*lease_broken = cancelled;
 
 	return ll_close_inode_openhandle(ll_i2sbi(inode)->ll_md_exp,
-					 inode, och, NULL);
-}
-
-/* Fills the obdo with the attributes for the lsm */
-static int ll_lsm_getattr(struct lov_stripe_md *lsm, struct obd_export *exp,
-			  struct obdo *obdo, __u64 ioepoch, int dv_flags)
-{
-	struct ptlrpc_request_set *set;
-	struct obd_info	    oinfo = { };
-	int			rc;
-
-	LASSERT(lsm);
-
-	oinfo.oi_md = lsm;
-	oinfo.oi_oa = obdo;
-	oinfo.oi_oa->o_oi = lsm->lsm_oi;
-	oinfo.oi_oa->o_mode = S_IFREG;
-	oinfo.oi_oa->o_ioepoch = ioepoch;
-	oinfo.oi_oa->o_valid = OBD_MD_FLID | OBD_MD_FLTYPE |
-			       OBD_MD_FLSIZE | OBD_MD_FLBLOCKS |
-			       OBD_MD_FLBLKSZ | OBD_MD_FLATIME |
-			       OBD_MD_FLMTIME | OBD_MD_FLCTIME |
-			       OBD_MD_FLGROUP | OBD_MD_FLEPOCH |
-			       OBD_MD_FLDATAVERSION;
-	if (dv_flags & (LL_DV_WR_FLUSH | LL_DV_RD_FLUSH)) {
-		oinfo.oi_oa->o_valid |= OBD_MD_FLFLAGS;
-		oinfo.oi_oa->o_flags |= OBD_FL_SRVLOCK;
-		if (dv_flags & LL_DV_WR_FLUSH)
-			oinfo.oi_oa->o_flags |= OBD_FL_FLUSH;
-	}
-
-	set = ptlrpc_prep_set();
-	if (!set) {
-		CERROR("cannot allocate ptlrpc set: rc = %d\n", -ENOMEM);
-		rc = -ENOMEM;
-	} else {
-		rc = obd_getattr_async(exp, &oinfo, set);
-		if (rc == 0)
-			rc = ptlrpc_set_wait(set);
-		ptlrpc_set_destroy(set);
-	}
-	if (rc == 0) {
-		oinfo.oi_oa->o_valid &= (OBD_MD_FLBLOCKS | OBD_MD_FLBLKSZ |
-					 OBD_MD_FLATIME | OBD_MD_FLMTIME |
-					 OBD_MD_FLCTIME | OBD_MD_FLSIZE |
-					 OBD_MD_FLDATAVERSION | OBD_MD_FLFLAGS);
-		if (dv_flags & LL_DV_WR_FLUSH &&
-		    !(oinfo.oi_oa->o_valid & OBD_MD_FLFLAGS &&
-		      oinfo.oi_oa->o_flags & OBD_FL_FLUSH))
-			return -ENOTSUPP;
-	}
-	return rc;
-}
-
-/**
-  * Performs the getattr on the inode and updates its fields.
-  * If @sync != 0, perform the getattr under the server-side lock.
-  */
-int ll_inode_getattr(struct inode *inode, struct obdo *obdo,
-		     __u64 ioepoch, int sync)
-{
-	struct lov_stripe_md *lsm;
-	int rc;
-
-	lsm = ccc_inode_lsm_get(inode);
-	rc = ll_lsm_getattr(lsm, ll_i2dtexp(inode),
-			    obdo, ioepoch, sync ? LL_DV_RD_FLUSH : 0);
-	if (rc == 0) {
-		struct ost_id *oi = lsm ? &lsm->lsm_oi : &obdo->o_oi;
-
-		obdo_refresh_inode(inode, obdo, obdo->o_valid);
-		CDEBUG(D_INODE, "objid " DOSTID " size %llu, blocks %llu, blksize %lu\n",
-		       POSTID(oi), i_size_read(inode),
-		       (unsigned long long)inode->i_blocks,
-		       1UL << inode->i_blkbits);
-	}
-	ccc_inode_lsm_put(inode, lsm);
-	return rc;
+					 och, inode, 0, NULL);
 }
 
 int ll_merge_attr(const struct lu_env *env, struct inode *inode)
@@ -1043,23 +989,6 @@ out_size_unlock:
 	return rc;
 }
 
-int ll_glimpse_ioctl(struct ll_sb_info *sbi, struct lov_stripe_md *lsm,
-		     lstat_t *st)
-{
-	struct obdo obdo = { 0 };
-	int rc;
-
-	rc = ll_lsm_getattr(lsm, sbi->ll_dt_exp, &obdo, 0, 0);
-	if (rc == 0) {
-		st->st_size   = obdo.o_size;
-		st->st_blocks = obdo.o_blocks;
-		st->st_mtime  = obdo.o_mtime;
-		st->st_atime  = obdo.o_atime;
-		st->st_ctime  = obdo.o_ctime;
-	}
-	return rc;
-}
-
 static bool file_is_noatime(const struct file *file)
 {
 	const struct vfsmount *mnt = file->f_path.mnt;
@@ -1117,9 +1046,11 @@ ll_file_io_generic(const struct lu_env *env, struct vvp_io_args *args,
 {
 	struct ll_inode_info *lli = ll_i2info(file_inode(file));
 	struct ll_file_data  *fd  = LUSTRE_FPRIVATE(file);
+	struct vvp_io *vio = vvp_env_io(env);
 	struct range_lock range;
 	struct cl_io	 *io;
-	ssize_t	       result;
+	ssize_t result = 0;
+	int rc = 0;
 
 	CDEBUG(D_VFSTRACE, "file: %pD, type: %d ppos: %llu, count: %zu\n",
 	       file, iot, *ppos, count);
@@ -1151,18 +1082,15 @@ restart:
 			CDEBUG(D_VFSTRACE, "Range lock [%llu, %llu]\n",
 			       range.rl_node.in_extent.start,
 			       range.rl_node.in_extent.end);
-			result = range_lock(&lli->lli_write_tree,
-					    &range);
-			if (result < 0)
+			rc = range_lock(&lli->lli_write_tree, &range);
+			if (rc < 0)
 				goto out;
 
 			range_locked = true;
 		}
-		down_read(&lli->lli_trunc_sem);
 		ll_cl_add(file, env, io);
-		result = cl_io_loop(env, io);
+		rc = cl_io_loop(env, io);
 		ll_cl_remove(file, env);
-		up_read(&lli->lli_trunc_sem);
 		if (range_locked) {
 			CDEBUG(D_VFSTRACE, "Range unlock [%llu, %llu]\n",
 			       range.rl_node.in_extent.start,
@@ -1171,24 +1099,26 @@ restart:
 		}
 	} else {
 		/* cl_io_rw_init() handled IO */
-		result = io->ci_result;
+		rc = io->ci_result;
 	}
 
 	if (io->ci_nob > 0) {
 		result = io->ci_nob;
+		count -= io->ci_nob;
 		*ppos = io->u.ci_wr.wr.crw_pos;
+
+		/* prepare IO restart */
+		if (count > 0)
+			args->u.normal.via_iter = vio->vui_iter;
 	}
-	goto out;
 out:
 	cl_io_fini(env, io);
-	/* If any bit been read/written (result != 0), we just return
-	 * short read/write instead of restart io.
-	 */
-	if ((result == 0 || result == -ENODATA) && io->ci_need_restart) {
-		CDEBUG(D_VFSTRACE, "Restart %s on %pD from %lld, count:%zu\n",
+
+	if ((!rc || rc == -ENODATA) && count > 0 && io->ci_need_restart) {
+		CDEBUG(D_VFSTRACE, "%s: restart %s from %lld, count:%zu, result: %zd\n",
+		       file_dentry(file)->d_name.name,
 		       iot == CIT_READ ? "read" : "write",
-		       file, *ppos, count);
-		LASSERTF(io->ci_nob == 0, "%zd\n", io->ci_nob);
+		       *ppos, count, result);
 		goto restart;
 	}
 
@@ -1201,13 +1131,19 @@ out:
 			ll_stats_ops_tally(ll_i2sbi(file_inode(file)),
 					   LPROC_LL_WRITE_BYTES, result);
 			fd->fd_write_failed = false;
-		} else if (result != -ERESTARTSYS) {
+		} else if (!result && !rc) {
+			rc = io->ci_result;
+			if (rc < 0)
+				fd->fd_write_failed = true;
+			else
+				fd->fd_write_failed = false;
+		} else if (rc != -ERESTARTSYS) {
 			fd->fd_write_failed = true;
 		}
 	}
 	CDEBUG(D_VFSTRACE, "iot: %d, result: %zd\n", iot, result);
 
-	return result;
+	return result > 0 ? result : rc;
 }
 
 static ssize_t ll_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
@@ -1259,37 +1195,22 @@ int ll_lov_setstripe_ea_info(struct inode *inode, struct dentry *dentry,
 			     __u64 flags, struct lov_user_md *lum,
 			     int lum_size)
 {
-	struct lov_stripe_md *lsm = NULL;
 	struct lookup_intent oit = {
 		.it_op = IT_OPEN,
 		.it_flags = flags | MDS_OPEN_BY_FID,
 	};
 	int rc = 0;
 
-	lsm = ccc_inode_lsm_get(inode);
-	if (lsm) {
-		ccc_inode_lsm_put(inode, lsm);
-		CDEBUG(D_IOCTL, "stripe already exists for inode "DFID"\n",
-		       PFID(ll_inode2fid(inode)));
-		rc = -EEXIST;
-		goto out;
-	}
-
 	ll_inode_size_lock(inode);
 	rc = ll_intent_file_open(dentry, lum, lum_size, &oit);
 	if (rc < 0)
 		goto out_unlock;
-	rc = oit.it_status;
-	if (rc < 0)
-		goto out_unlock;
 
 	ll_release_openhandle(inode, &oit);
 
 out_unlock:
 	ll_inode_size_unlock(inode);
 	ll_intent_release(&oit);
-	ccc_inode_lsm_put(inode, lsm);
-out:
 	return rc;
 }
 
@@ -1566,7 +1487,7 @@ int ll_release_openhandle(struct inode *inode, struct lookup_intent *it)
 	ll_och_fill(ll_i2sbi(inode)->ll_md_exp, it, och);
 
 	rc = ll_close_inode_openhandle(ll_i2sbi(inode)->ll_md_exp,
-				       inode, och, NULL);
+				       och, inode, 0, NULL);
 out:
 	/* this one is in place of ll_file_open */
 	if (it_disposition(it, DISP_ENQ_OPEN_REF)) {
@@ -1579,15 +1500,17 @@ out:
 /**
  * Get size for inode for which FIEMAP mapping is requested.
  * Make the FIEMAP get_info call and returns the result.
+ *
+ * \param fiemap	kernel buffer to hold extens
+ * \param num_bytes	kernel buffer size
  */
-static int ll_do_fiemap(struct inode *inode, struct ll_user_fiemap *fiemap,
+static int ll_do_fiemap(struct inode *inode, struct fiemap *fiemap,
 			size_t num_bytes)
 {
-	struct obd_export *exp = ll_i2dtexp(inode);
-	struct lov_stripe_md *lsm = NULL;
-	struct ll_fiemap_info_key fm_key = { .name = KEY_FIEMAP, };
-	__u32 vallen = num_bytes;
-	int rc;
+	struct ll_fiemap_info_key fmkey = { .lfik_name = KEY_FIEMAP, };
+	struct lu_env *env;
+	int refcheck;
+	int rc = 0;
 
 	/* Checks for fiemap flags */
 	if (fiemap->fm_flags & ~LUSTRE_FIEMAP_FLAGS_COMPAT) {
@@ -1602,21 +1525,9 @@ static int ll_do_fiemap(struct inode *inode, struct ll_user_fiemap *fiemap,
 			return rc;
 	}
 
-	lsm = ccc_inode_lsm_get(inode);
-	if (!lsm)
-		return -ENOENT;
-
-	/* If the stripe_count > 1 and the application does not understand
-	 * DEVICE_ORDER flag, then it cannot interpret the extents correctly.
-	 */
-	if (lsm->lsm_stripe_count > 1 &&
-	    !(fiemap->fm_flags & FIEMAP_FLAG_DEVICE_ORDER)) {
-		rc = -EOPNOTSUPP;
-		goto out;
-	}
-
-	fm_key.oa.o_oi = lsm->lsm_oi;
-	fm_key.oa.o_valid = OBD_MD_FLID | OBD_MD_FLGROUP;
+	env = cl_env_get(&refcheck);
+	if (IS_ERR(env))
+		return PTR_ERR(env);
 
 	if (i_size_read(inode) == 0) {
 		rc = ll_glimpse_size(inode);
@@ -1624,24 +1535,23 @@ static int ll_do_fiemap(struct inode *inode, struct ll_user_fiemap *fiemap,
 			goto out;
 	}
 
-	obdo_from_inode(&fm_key.oa, inode, OBD_MD_FLSIZE);
-	obdo_set_parent_fid(&fm_key.oa, &ll_i2info(inode)->lli_fid);
+	fmkey.lfik_oa.o_valid = OBD_MD_FLID | OBD_MD_FLGROUP;
+	obdo_from_inode(&fmkey.lfik_oa, inode, OBD_MD_FLSIZE);
+	obdo_set_parent_fid(&fmkey.lfik_oa, &ll_i2info(inode)->lli_fid);
+
 	/* If filesize is 0, then there would be no objects for mapping */
-	if (fm_key.oa.o_size == 0) {
+	if (fmkey.lfik_oa.o_size == 0) {
 		fiemap->fm_mapped_extents = 0;
 		rc = 0;
 		goto out;
 	}
 
-	memcpy(&fm_key.fiemap, fiemap, sizeof(*fiemap));
-
-	rc = obd_get_info(NULL, exp, sizeof(fm_key), &fm_key, &vallen,
-			  fiemap, lsm);
-	if (rc)
-		CERROR("obd_get_info failed: rc = %d\n", rc);
+	memcpy(&fmkey.lfik_fiemap, fiemap, sizeof(*fiemap));
 
+	rc = cl_object_fiemap(env, ll_i2info(inode)->lli_clob,
+			      &fmkey, fiemap, &num_bytes);
 out:
-	ccc_inode_lsm_put(inode, lsm);
+	cl_env_put(env, &refcheck);
 	return rc;
 }
 
@@ -1689,113 +1599,56 @@ gf_free:
 	return rc;
 }
 
-static int ll_ioctl_fiemap(struct inode *inode, unsigned long arg)
-{
-	struct ll_user_fiemap *fiemap_s;
-	size_t num_bytes, ret_bytes;
-	unsigned int extent_count;
-	int rc = 0;
-
-	/* Get the extent count so we can calculate the size of
-	 * required fiemap buffer
-	 */
-	if (get_user(extent_count,
-		     &((struct ll_user_fiemap __user *)arg)->fm_extent_count))
-		return -EFAULT;
-
-	if (extent_count >=
-	    (SIZE_MAX - sizeof(*fiemap_s)) / sizeof(struct ll_fiemap_extent))
-		return -EINVAL;
-	num_bytes = sizeof(*fiemap_s) + (extent_count *
-					 sizeof(struct ll_fiemap_extent));
-
-	fiemap_s = libcfs_kvzalloc(num_bytes, GFP_NOFS);
-	if (!fiemap_s)
-		return -ENOMEM;
-
-	/* get the fiemap value */
-	if (copy_from_user(fiemap_s, (struct ll_user_fiemap __user *)arg,
-			   sizeof(*fiemap_s))) {
-		rc = -EFAULT;
-		goto error;
-	}
-
-	/* If fm_extent_count is non-zero, read the first extent since
-	 * it is used to calculate end_offset and device from previous
-	 * fiemap call.
-	 */
-	if (extent_count) {
-		if (copy_from_user(&fiemap_s->fm_extents[0],
-				   (char __user *)arg + sizeof(*fiemap_s),
-				   sizeof(struct ll_fiemap_extent))) {
-			rc = -EFAULT;
-			goto error;
-		}
-	}
-
-	rc = ll_do_fiemap(inode, fiemap_s, num_bytes);
-	if (rc)
-		goto error;
-
-	ret_bytes = sizeof(struct ll_user_fiemap);
-
-	if (extent_count != 0)
-		ret_bytes += (fiemap_s->fm_mapped_extents *
-				 sizeof(struct ll_fiemap_extent));
-
-	if (copy_to_user((void __user *)arg, fiemap_s, ret_bytes))
-		rc = -EFAULT;
-
-error:
-	kvfree(fiemap_s);
-	return rc;
-}
-
 /*
  * Read the data_version for inode.
  *
  * This value is computed using stripe object version on OST.
  * Version is computed using server side locking.
  *
- * @param sync  if do sync on the OST side;
+ * @param flags if do sync on the OST side;
  *		0: no sync
  *		LL_DV_RD_FLUSH: flush dirty pages, LCK_PR on OSTs
  *		LL_DV_WR_FLUSH: drop all caching pages, LCK_PW on OSTs
  */
 int ll_data_version(struct inode *inode, __u64 *data_version, int flags)
 {
-	struct lov_stripe_md	*lsm = NULL;
-	struct ll_sb_info	*sbi = ll_i2sbi(inode);
-	struct obdo		*obdo = NULL;
-	int			 rc;
+	struct cl_object *obj = ll_i2info(inode)->lli_clob;
+	struct lu_env *env;
+	struct cl_io *io;
+	int refcheck;
+	int result;
 
-	/* If no stripe, we consider version is 0. */
-	lsm = ccc_inode_lsm_get(inode);
-	if (!lsm_has_objects(lsm)) {
+	/* If no file object initialized, we consider its version is 0. */
+	if (!obj) {
 		*data_version = 0;
-		CDEBUG(D_INODE, "No object for inode\n");
-		rc = 0;
-		goto out;
+		return 0;
 	}
 
-	obdo = kzalloc(sizeof(*obdo), GFP_NOFS);
-	if (!obdo) {
-		rc = -ENOMEM;
-		goto out;
-	}
+	env = cl_env_get(&refcheck);
+	if (IS_ERR(env))
+		return PTR_ERR(env);
 
-	rc = ll_lsm_getattr(lsm, sbi->ll_dt_exp, obdo, 0, flags);
-	if (rc == 0) {
-		if (!(obdo->o_valid & OBD_MD_FLDATAVERSION))
-			rc = -EOPNOTSUPP;
-		else
-			*data_version = obdo->o_data_version;
-	}
+	io = vvp_env_thread_io(env);
+	io->ci_obj = obj;
+	io->u.ci_data_version.dv_data_version = 0;
+	io->u.ci_data_version.dv_flags = flags;
 
-	kfree(obdo);
-out:
-	ccc_inode_lsm_put(inode, lsm);
-	return rc;
+restart:
+	if (!cl_io_init(env, io, CIT_DATA_VERSION, io->ci_obj))
+		result = cl_io_loop(env, io);
+	else
+		result = io->ci_result;
+
+	*data_version = io->u.ci_data_version.dv_data_version;
+
+	cl_io_fini(env, io);
+
+	if (unlikely(io->ci_need_restart))
+		goto restart;
+
+	cl_env_put(env, &refcheck);
+
+	return result;
 }
 
 /*
@@ -1803,11 +1656,11 @@ out:
  */
 int ll_hsm_release(struct inode *inode)
 {
-	struct cl_env_nest nest;
 	struct lu_env *env;
 	struct obd_client_handle *och = NULL;
 	__u64 data_version = 0;
 	int rc;
+	int refcheck;
 
 	CDEBUG(D_INODE, "%s: Releasing file "DFID".\n",
 	       ll_get_fsname(inode->i_sb, NULL, 0),
@@ -1824,21 +1677,21 @@ int ll_hsm_release(struct inode *inode)
 	if (rc != 0)
 		goto out;
 
-	env = cl_env_nested_get(&nest);
+	env = cl_env_get(&refcheck);
 	if (IS_ERR(env)) {
 		rc = PTR_ERR(env);
 		goto out;
 	}
 
 	ll_merge_attr(env, inode);
-	cl_env_nested_put(&nest, env);
+	cl_env_put(env, &refcheck);
 
 	/* Release the file.
 	 * NB: lease lock handle is released in mdc_hsm_release_pack() because
 	 * we still need it to pack l_remote_handle to MDT.
 	 */
-	rc = ll_close_inode_openhandle(ll_i2sbi(inode)->ll_md_exp, inode, och,
-				       &data_version);
+	rc = ll_close_inode_openhandle(ll_i2sbi(inode)->ll_md_exp, och, inode,
+				       MDS_HSM_RELEASE, &data_version);
 	och = NULL;
 
 out:
@@ -1849,10 +1702,12 @@ out:
 }
 
 struct ll_swap_stack {
-	struct iattr		 ia1, ia2;
-	__u64			 dv1, dv2;
-	struct inode		*inode1, *inode2;
-	bool			 check_dv1, check_dv2;
+	u64		dv1;
+	u64		dv2;
+	struct inode   *inode1;
+	struct inode   *inode2;
+	bool		check_dv1;
+	bool		check_dv2;
 };
 
 static int ll_swap_layouts(struct file *file1, struct file *file2,
@@ -1872,21 +1727,9 @@ static int ll_swap_layouts(struct file *file1, struct file *file2,
 	llss->inode1 = file_inode(file1);
 	llss->inode2 = file_inode(file2);
 
-	if (!S_ISREG(llss->inode2->i_mode)) {
-		rc = -EINVAL;
-		goto free;
-	}
-
-	if (inode_permission(llss->inode1, MAY_WRITE) ||
-	    inode_permission(llss->inode2, MAY_WRITE)) {
-		rc = -EPERM;
+	rc = ll_check_swap_layouts_validity(llss->inode1, llss->inode2);
+	if (rc < 0)
 		goto free;
-	}
-
-	if (llss->inode2->i_sb != llss->inode1->i_sb) {
-		rc = -EXDEV;
-		goto free;
-	}
 
 	/* we use 2 bool because it is easier to swap than 2 bits */
 	if (lsl->sl_flags & SWAP_LAYOUTS_CHECK_DV1)
@@ -1900,10 +1743,8 @@ static int ll_swap_layouts(struct file *file1, struct file *file2,
 	llss->dv2 = lsl->sl_dv2;
 
 	rc = lu_fid_cmp(ll_inode2fid(llss->inode1), ll_inode2fid(llss->inode2));
-	if (rc == 0) /* same file, done! */ {
-		rc = 0;
+	if (!rc) /* same file, done! */
 		goto free;
-	}
 
 	if (rc < 0) { /* sequentialize it */
 		swap(llss->inode1, llss->inode2);
@@ -1925,19 +1766,6 @@ static int ll_swap_layouts(struct file *file1, struct file *file2,
 		}
 	}
 
-	/* to be able to restore mtime and atime after swap
-	 * we need to first save them
-	 */
-	if (lsl->sl_flags &
-	    (SWAP_LAYOUTS_KEEP_MTIME | SWAP_LAYOUTS_KEEP_ATIME)) {
-		llss->ia1.ia_mtime = llss->inode1->i_mtime;
-		llss->ia1.ia_atime = llss->inode1->i_atime;
-		llss->ia1.ia_valid = ATTR_MTIME | ATTR_ATIME;
-		llss->ia2.ia_mtime = llss->inode2->i_mtime;
-		llss->ia2.ia_atime = llss->inode2->i_atime;
-		llss->ia2.ia_valid = ATTR_MTIME | ATTR_ATIME;
-	}
-
 	/* ultimate check, before swapping the layouts we check if
 	 * dataversion has changed (if requested)
 	 */
@@ -1987,39 +1815,6 @@ putgl:
 		ll_put_grouplock(llss->inode1, file1, gid);
 	}
 
-	/* rc can be set from obd_iocontrol() or from a GOTO(putgl, ...) */
-	if (rc != 0)
-		goto free;
-
-	/* clear useless flags */
-	if (!(lsl->sl_flags & SWAP_LAYOUTS_KEEP_MTIME)) {
-		llss->ia1.ia_valid &= ~ATTR_MTIME;
-		llss->ia2.ia_valid &= ~ATTR_MTIME;
-	}
-
-	if (!(lsl->sl_flags & SWAP_LAYOUTS_KEEP_ATIME)) {
-		llss->ia1.ia_valid &= ~ATTR_ATIME;
-		llss->ia2.ia_valid &= ~ATTR_ATIME;
-	}
-
-	/* update time if requested */
-	rc = 0;
-	if (llss->ia2.ia_valid != 0) {
-		inode_lock(llss->inode1);
-		rc = ll_setattr(file1->f_path.dentry, &llss->ia2);
-		inode_unlock(llss->inode1);
-	}
-
-	if (llss->ia1.ia_valid != 0) {
-		int rc1;
-
-		inode_lock(llss->inode2);
-		rc1 = ll_setattr(file2->f_path.dentry, &llss->ia1);
-		inode_unlock(llss->inode2);
-		if (rc == 0)
-			rc = rc1;
-	}
-
 free:
 	kfree(llss);
 
@@ -2176,24 +1971,52 @@ ll_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 				   sizeof(struct lustre_swap_layouts)))
 			return -EFAULT;
 
-		if ((file->f_flags & O_ACCMODE) == 0) /* O_RDONLY */
+		if ((file->f_flags & O_ACCMODE) == O_RDONLY)
 			return -EPERM;
 
 		file2 = fget(lsl.sl_fd);
 		if (!file2)
 			return -EBADF;
 
-		rc = -EPERM;
-		if ((file2->f_flags & O_ACCMODE) != 0) /* O_WRONLY or O_RDWR */
+		/* O_WRONLY or O_RDWR */
+		if ((file2->f_flags & O_ACCMODE) == O_RDONLY) {
+			rc = -EPERM;
+			goto out;
+		}
+
+		if (lsl.sl_flags & SWAP_LAYOUTS_CLOSE) {
+			struct obd_client_handle *och = NULL;
+			struct ll_inode_info *lli;
+			struct inode *inode2;
+
+			if (lsl.sl_flags != SWAP_LAYOUTS_CLOSE) {
+				rc = -EINVAL;
+				goto out;
+			}
+
+			lli = ll_i2info(inode);
+			mutex_lock(&lli->lli_och_mutex);
+			if (fd->fd_lease_och) {
+				och = fd->fd_lease_och;
+				fd->fd_lease_och = NULL;
+			}
+			mutex_unlock(&lli->lli_och_mutex);
+			if (!och) {
+				rc = -ENOLCK;
+				goto out;
+			}
+			inode2 = file_inode(file2);
+			rc = ll_swap_layouts_close(och, inode, inode2);
+		} else {
 			rc = ll_swap_layouts(file, file2, &lsl);
+		}
+out:
 		fput(file2);
 		return rc;
 	}
 	case LL_IOC_LOV_GETSTRIPE:
 		return ll_file_getstripe(inode,
 					 (struct lov_user_md __user *)arg);
-	case FSFILT_IOC_FIEMAP:
-		return ll_ioctl_fiemap(inode, arg);
 	case FSFILT_IOC_GETFLAGS:
 	case FSFILT_IOC_SETFLAGS:
 		return ll_iocontrol(inode, file, cmd, arg);
@@ -2489,17 +2312,17 @@ static int ll_flush(struct file *file, fl_owner_t id)
 int cl_sync_file_range(struct inode *inode, loff_t start, loff_t end,
 		       enum cl_fsync_mode mode, int ignore_layout)
 {
-	struct cl_env_nest nest;
 	struct lu_env *env;
 	struct cl_io *io;
 	struct cl_fsync_io *fio;
 	int result;
+	int refcheck;
 
 	if (mode != CL_FSYNC_NONE && mode != CL_FSYNC_LOCAL &&
 	    mode != CL_FSYNC_DISCARD && mode != CL_FSYNC_ALL)
 		return -EINVAL;
 
-	env = cl_env_nested_get(&nest);
+	env = cl_env_get(&refcheck);
 	if (IS_ERR(env))
 		return PTR_ERR(env);
 
@@ -2522,7 +2345,7 @@ int cl_sync_file_range(struct inode *inode, loff_t start, loff_t end,
 	if (result == 0)
 		result = fio->fi_nr_written;
 	cl_io_fini(env, io);
-	cl_env_nested_put(&nest, env);
+	cl_env_put(env, &refcheck);
 
 	return result;
 }
@@ -2549,9 +2372,11 @@ int ll_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 		lli->lli_async_rc = 0;
 		if (rc == 0)
 			rc = err;
-		err = lov_read_and_clear_async_rc(lli->lli_clob);
-		if (rc == 0)
-			rc = err;
+		if (lli->lli_clob) {
+			err = lov_read_and_clear_async_rc(lli->lli_clob);
+			if (rc == 0)
+				rc = err;
+		}
 	}
 
 	err = md_sync(ll_i2sbi(inode)->ll_md_exp, ll_inode2fid(inode), &req);
@@ -2588,7 +2413,7 @@ ll_file_flock(struct file *file, int cmd, struct file_lock *file_lock)
 	};
 	struct md_op_data *op_data;
 	struct lustre_handle lockh = {0};
-	ldlm_policy_data_t flock = { {0} };
+	union ldlm_policy_data flock = { { 0 } };
 	int fl_type = file_lock->fl_type;
 	__u64 flags = 0;
 	int rc;
@@ -2707,7 +2532,8 @@ ll_file_flock(struct file *file, int cmd, struct file_lock *file_lock)
 }
 
 int ll_get_fid_by_name(struct inode *parent, const char *name,
-		       int namelen, struct lu_fid *fid)
+		       int namelen, struct lu_fid *fid,
+		       struct inode **inode)
 {
 	struct md_op_data *op_data = NULL;
 	struct ptlrpc_request *req;
@@ -2719,7 +2545,7 @@ int ll_get_fid_by_name(struct inode *parent, const char *name,
 	if (IS_ERR(op_data))
 		return PTR_ERR(op_data);
 
-	op_data->op_valid = OBD_MD_FLID;
+	op_data->op_valid = OBD_MD_FLID | OBD_MD_FLTYPE;
 	rc = md_getattr_name(ll_i2sbi(parent)->ll_md_exp, op_data, &req);
 	ll_finish_md_op_data(op_data);
 	if (rc < 0)
@@ -2732,6 +2558,9 @@ int ll_get_fid_by_name(struct inode *parent, const char *name,
 	}
 	if (fid)
 		*fid = body->mbo_fid1;
+
+	if (inode)
+		rc = ll_prep_inode(inode, req, parent->i_sb, NULL);
 out_req:
 	ptlrpc_req_finished(req);
 	return rc;
@@ -2741,9 +2570,12 @@ int ll_migrate(struct inode *parent, struct file *file, int mdtidx,
 	       const char *name, int namelen)
 {
 	struct ptlrpc_request *request = NULL;
+	struct obd_client_handle *och = NULL;
 	struct inode *child_inode = NULL;
 	struct dentry *dchild = NULL;
 	struct md_op_data *op_data;
+	struct mdt_body *body;
+	u64 data_version = 0;
 	struct qstr qstr;
 	int rc;
 
@@ -2762,22 +2594,25 @@ int ll_migrate(struct inode *parent, struct file *file, int mdtidx,
 	dchild = d_lookup(file_dentry(file), &qstr);
 	if (dchild) {
 		op_data->op_fid3 = *ll_inode2fid(dchild->d_inode);
-		if (dchild->d_inode) {
+		if (dchild->d_inode)
 			child_inode = igrab(dchild->d_inode);
-			if (child_inode) {
-				inode_lock(child_inode);
-				op_data->op_fid3 = *ll_inode2fid(child_inode);
-				ll_invalidate_aliases(child_inode);
-			}
-		}
 		dput(dchild);
-	} else {
+	}
+
+	if (!child_inode) {
 		rc = ll_get_fid_by_name(parent, name, namelen,
-					&op_data->op_fid3);
+					&op_data->op_fid3, &child_inode);
 		if (rc)
 			goto out_free;
 	}
 
+	if (!child_inode) {
+		rc = -EINVAL;
+		goto out_free;
+	}
+
+	inode_lock(child_inode);
+	op_data->op_fid3 = *ll_inode2fid(child_inode);
 	if (!fid_is_sane(&op_data->op_fid3)) {
 		CERROR("%s: migrate %s, but fid "DFID" is insane\n",
 		       ll_get_fsname(parent->i_sb, NULL, 0), name,
@@ -2796,6 +2631,26 @@ int ll_migrate(struct inode *parent, struct file *file, int mdtidx,
 		rc = 0;
 		goto out_free;
 	}
+again:
+	if (S_ISREG(child_inode->i_mode)) {
+		och = ll_lease_open(child_inode, NULL, FMODE_WRITE, 0);
+		if (IS_ERR(och)) {
+			rc = PTR_ERR(och);
+			och = NULL;
+			goto out_free;
+		}
+
+		rc = ll_data_version(child_inode, &data_version,
+				     LL_DV_WR_FLUSH);
+		if (rc)
+			goto out_free;
+
+		op_data->op_handle = och->och_fh;
+		op_data->op_data = och->och_mod;
+		op_data->op_data_version = data_version;
+		op_data->op_lease_handle = och->och_lease_handle;
+		op_data->op_bias |= MDS_RENAME_MIGRATE;
+	}
 
 	op_data->op_mds = mdtidx;
 	op_data->op_cli_flags = CLI_MIGRATE;
@@ -2804,10 +2659,32 @@ int ll_migrate(struct inode *parent, struct file *file, int mdtidx,
 	if (!rc)
 		ll_update_times(request, parent);
 
-	ptlrpc_req_finished(request);
+	body = req_capsule_server_get(&request->rq_pill, &RMF_MDT_BODY);
+	if (!body) {
+		rc = -EPROTO;
+		goto out_free;
+	}
+
+	/*
+	 * If the server does release layout lock, then we cleanup
+	 * the client och here, otherwise release it in out_free:
+	 */
+	if (och && body->mbo_valid & OBD_MD_CLOSE_INTENT_EXECED) {
+		obd_mod_put(och->och_mod);
+		md_clear_open_replay_data(ll_i2sbi(parent)->ll_md_exp, och);
+		och->och_fh.cookie = DEAD_HANDLE_MAGIC;
+		kfree(och);
+		och = NULL;
+	}
 
+	ptlrpc_req_finished(request);
+	/* Try again if the file layout has changed. */
+	if (rc == -EAGAIN && S_ISREG(child_inode->i_mode))
+		goto again;
 out_free:
 	if (child_inode) {
+		if (och) /* close the file */
+			ll_lease_close(och, child_inode, NULL);
 		clear_nlink(child_inode);
 		inode_unlock(child_inode);
 		iput(child_inode);
@@ -2837,7 +2714,7 @@ int ll_have_md_lock(struct inode *inode, __u64 *bits,
 		    enum ldlm_mode l_req_mode)
 {
 	struct lustre_handle lockh;
-	ldlm_policy_data_t policy;
+	union ldlm_policy_data policy;
 	enum ldlm_mode mode = (l_req_mode == LCK_MINMODE) ?
 			      (LCK_CR | LCK_CW | LCK_PR | LCK_PW) : l_req_mode;
 	struct lu_fid *fid;
@@ -2878,7 +2755,7 @@ enum ldlm_mode ll_take_md_lock(struct inode *inode, __u64 bits,
 			       struct lustre_handle *lockh, __u64 flags,
 			       enum ldlm_mode mode)
 {
-	ldlm_policy_data_t policy = { .l_inodebits = {bits} };
+	union ldlm_policy_data policy = { .l_inodebits = { bits } };
 	struct lu_fid *fid;
 
 	fid = &ll_i2info(inode)->lli_fid;
@@ -2893,6 +2770,13 @@ static int ll_inode_revalidate_fini(struct inode *inode, int rc)
 	/* Already unlinked. Just update nlink and return success */
 	if (rc == -ENOENT) {
 		clear_nlink(inode);
+		/* If it is striped directory, and there is bad stripe
+		 * Let's revalidate the dentry again, instead of returning
+		 * error
+		 */
+		if (S_ISDIR(inode->i_mode) && ll_i2info(inode)->lli_lsm_md)
+			return 0;
+
 		/* This path cannot be hit for regular files unless in
 		 * case of obscure races, so no need to validate size.
 		 */
@@ -3040,6 +2924,8 @@ static int ll_inode_revalidate(struct dentry *dentry, __u64 ibits)
 		LTIME_S(inode->i_mtime) = ll_i2info(inode)->lli_mtime;
 		LTIME_S(inode->i_ctime) = ll_i2info(inode)->lli_ctime;
 	} else {
+		struct ll_inode_info *lli = ll_i2info(inode);
+
 		/* In case of restore, the MDT has the right size and has
 		 * already send it back without granting the layout lock,
 		 * inode is up-to-date so glimpse is useless.
@@ -3047,7 +2933,7 @@ static int ll_inode_revalidate(struct dentry *dentry, __u64 ibits)
 		 * restore the MDT holds the layout lock so the glimpse will
 		 * block up to the end of restore (getattr will block)
 		 */
-		if (!(ll_i2info(inode)->lli_flags & LLIF_FILE_RESTORING))
+		if (!test_bit(LLIF_FILE_RESTORING, &lli->lli_flags))
 			rc = ll_glimpse_size(inode);
 	}
 	return rc;
@@ -3095,13 +2981,12 @@ static int ll_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 {
 	int rc;
 	size_t num_bytes;
-	struct ll_user_fiemap *fiemap;
+	struct fiemap *fiemap;
 	unsigned int extent_count = fieinfo->fi_extents_max;
 
 	num_bytes = sizeof(*fiemap) + (extent_count *
-				       sizeof(struct ll_fiemap_extent));
+				       sizeof(struct fiemap_extent));
 	fiemap = libcfs_kvzalloc(num_bytes, GFP_NOFS);
-
 	if (!fiemap)
 		return -ENOMEM;
 
@@ -3109,9 +2994,10 @@ static int ll_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 	fiemap->fm_extent_count = fieinfo->fi_extents_max;
 	fiemap->fm_start = start;
 	fiemap->fm_length = len;
+
 	if (extent_count > 0 &&
 	    copy_from_user(&fiemap->fm_extents[0], fieinfo->fi_extents_start,
-			   sizeof(struct ll_fiemap_extent)) != 0) {
+			   sizeof(struct fiemap_extent))) {
 		rc = -EFAULT;
 		goto out;
 	}
@@ -3123,11 +3009,10 @@ static int ll_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 	if (extent_count > 0 &&
 	    copy_to_user(fieinfo->fi_extents_start, &fiemap->fm_extents[0],
 			 fiemap->fm_mapped_extents *
-			 sizeof(struct ll_fiemap_extent)) != 0) {
+			 sizeof(struct fiemap_extent))) {
 		rc = -EFAULT;
 		goto out;
 	}
-
 out:
 	kvfree(fiemap);
 	return rc;
@@ -3370,35 +3255,50 @@ ll_iocontrol_call(struct inode *inode, struct file *file,
 int ll_layout_conf(struct inode *inode, const struct cl_object_conf *conf)
 {
 	struct ll_inode_info *lli = ll_i2info(inode);
-	struct cl_env_nest nest;
+	struct cl_object *obj = lli->lli_clob;
 	struct lu_env *env;
-	int result;
+	int rc;
+	int refcheck;
 
-	if (!lli->lli_clob)
+	if (!obj)
 		return 0;
 
-	env = cl_env_nested_get(&nest);
+	env = cl_env_get(&refcheck);
 	if (IS_ERR(env))
 		return PTR_ERR(env);
 
-	result = cl_conf_set(env, lli->lli_clob, conf);
-	cl_env_nested_put(&nest, env);
+	rc = cl_conf_set(env, obj, conf);
+	if (rc < 0)
+		goto out;
 
 	if (conf->coc_opc == OBJECT_CONF_SET) {
 		struct ldlm_lock *lock = conf->coc_lock;
+		struct cl_layout cl = {
+			.cl_layout_gen = 0,
+		};
 
 		LASSERT(lock);
 		LASSERT(ldlm_has_layout(lock));
-		if (result == 0) {
-			/* it can only be allowed to match after layout is
-			 * applied to inode otherwise false layout would be
-			 * seen. Applying layout should happen before dropping
-			 * the intent lock.
-			 */
-			ldlm_lock_allow_match(lock);
-		}
+
+		/* it can only be allowed to match after layout is
+		 * applied to inode otherwise false layout would be
+		 * seen. Applying layout should happen before dropping
+		 * the intent lock.
+		 */
+		ldlm_lock_allow_match(lock);
+
+		rc = cl_object_layout_get(env, obj, &cl);
+		if (rc < 0)
+			goto out;
+
+		CDEBUG(D_VFSTRACE, DFID ": layout version change: %u -> %u\n",
+		       PFID(&lli->lli_fid), ll_layout_version_get(lli),
+		       cl.cl_layout_gen);
+		ll_layout_version_set(lli, cl.cl_layout_gen);
 	}
-	return result;
+out:
+	cl_env_put(env, &refcheck);
+	return rc;
 }
 
 /* Fetch layout from MDT with getxattr request, if it's not ready yet */
@@ -3477,12 +3377,11 @@ out:
  * in this function.
  */
 static int ll_layout_lock_set(struct lustre_handle *lockh, enum ldlm_mode mode,
-			      struct inode *inode, __u32 *gen, bool reconf)
+			      struct inode *inode)
 {
 	struct ll_inode_info *lli = ll_i2info(inode);
 	struct ll_sb_info    *sbi = ll_i2sbi(inode);
 	struct ldlm_lock *lock;
-	struct lustre_md md = { NULL };
 	struct cl_object_conf conf;
 	int rc = 0;
 	bool lvb_ready;
@@ -3494,8 +3393,8 @@ static int ll_layout_lock_set(struct lustre_handle *lockh, enum ldlm_mode mode,
 	LASSERT(lock);
 	LASSERT(ldlm_has_layout(lock));
 
-	LDLM_DEBUG(lock, "File "DFID"(%p) being reconfigured: %d",
-		   PFID(&lli->lli_fid), inode, reconf);
+	LDLM_DEBUG(lock, "File " DFID "(%p) being reconfigured",
+		   PFID(&lli->lli_fid), inode);
 
 	/* in case this is a caching lock and reinstate with new inode */
 	md_set_lock_data(sbi->ll_md_exp, lockh, inode, NULL);
@@ -3506,15 +3405,8 @@ static int ll_layout_lock_set(struct lustre_handle *lockh, enum ldlm_mode mode,
 	/* checking lvb_ready is racy but this is okay. The worst case is
 	 * that multi processes may configure the file on the same time.
 	 */
-	if (lvb_ready || !reconf) {
-		rc = -ENODATA;
-		if (lvb_ready) {
-			/* layout_gen must be valid if layout lock is not
-			 * cancelled and stripe has already set
-			 */
-			*gen = ll_layout_version_get(lli);
-			rc = 0;
-		}
+	if (lvb_ready) {
+		rc = 0;
 		goto out;
 	}
 
@@ -3524,39 +3416,19 @@ static int ll_layout_lock_set(struct lustre_handle *lockh, enum ldlm_mode mode,
 
 	/* for layout lock, lmm is returned in lock's lvb.
 	 * lvb_data is immutable if the lock is held so it's safe to access it
-	 * without res lock. See the description in ldlm_lock_decref_internal()
-	 * for the condition to free lvb_data of layout lock
-	 */
-	if (lock->l_lvb_data) {
-		rc = obd_unpackmd(sbi->ll_dt_exp, &md.lsm,
-				  lock->l_lvb_data, lock->l_lvb_len);
-		if (rc >= 0) {
-			*gen = LL_LAYOUT_GEN_EMPTY;
-			if (md.lsm)
-				*gen = md.lsm->lsm_layout_gen;
-			rc = 0;
-		} else {
-			CERROR("%s: file " DFID " unpackmd error: %d\n",
-			       ll_get_fsname(inode->i_sb, NULL, 0),
-			       PFID(&lli->lli_fid), rc);
-		}
-	}
-	if (rc < 0)
-		goto out;
-
-	/* set layout to file. Unlikely this will fail as old layout was
+	 * without res lock.
+	 *
+	 * set layout to file. Unlikely this will fail as old layout was
 	 * surely eliminated
 	 */
 	memset(&conf, 0, sizeof(conf));
 	conf.coc_opc = OBJECT_CONF_SET;
 	conf.coc_inode = inode;
 	conf.coc_lock = lock;
-	conf.u.coc_md = &md;
+	conf.u.coc_layout.lb_buf = lock->l_lvb_data;
+	conf.u.coc_layout.lb_len = lock->l_lvb_len;
 	rc = ll_layout_conf(inode, &conf);
 
-	if (md.lsm)
-		obd_free_memmd(sbi->ll_dt_exp, &md.lsm);
-
 	/* refresh layout failed, need to wait */
 	wait_layout = rc == -EBUSY;
 
@@ -3584,20 +3456,7 @@ out:
 	return rc;
 }
 
-/**
- * This function checks if there exists a LAYOUT lock on the client side,
- * or enqueues it if it doesn't have one in cache.
- *
- * This function will not hold layout lock so it may be revoked any time after
- * this function returns. Any operations depend on layout should be redone
- * in that case.
- *
- * This function should be called before lov_io_init() to get an uptodate
- * layout version, the caller should save the version number and after IO
- * is finished, this function should be called again to verify that layout
- * is not changed during IO time.
- */
-int ll_layout_refresh(struct inode *inode, __u32 *gen)
+static int ll_layout_refresh_locked(struct inode *inode)
 {
 	struct ll_inode_info  *lli = ll_i2info(inode);
 	struct ll_sb_info     *sbi = ll_i2sbi(inode);
@@ -3613,17 +3472,6 @@ int ll_layout_refresh(struct inode *inode, __u32 *gen)
 	};
 	int rc;
 
-	*gen = ll_layout_version_get(lli);
-	if (!(sbi->ll_flags & LL_SBI_LAYOUT_LOCK) || *gen != LL_LAYOUT_GEN_NONE)
-		return 0;
-
-	/* sanity checks */
-	LASSERT(fid_is_sane(ll_inode2fid(inode)));
-	LASSERT(S_ISREG(inode->i_mode));
-
-	/* take layout lock mutex to enqueue layout lock exclusively. */
-	mutex_lock(&lli->lli_layout_mutex);
-
 again:
 	/* mostly layout lock is caching on the local side, so try to match
 	 * it before grabbing layout lock mutex.
@@ -3631,20 +3479,16 @@ again:
 	mode = ll_take_md_lock(inode, MDS_INODELOCK_LAYOUT, &lockh, 0,
 			       LCK_CR | LCK_CW | LCK_PR | LCK_PW);
 	if (mode != 0) { /* hit cached lock */
-		rc = ll_layout_lock_set(&lockh, mode, inode, gen, true);
+		rc = ll_layout_lock_set(&lockh, mode, inode);
 		if (rc == -EAGAIN)
 			goto again;
-
-		mutex_unlock(&lli->lli_layout_mutex);
 		return rc;
 	}
 
 	op_data = ll_prep_md_op_data(NULL, inode, inode, NULL,
 				     0, 0, LUSTRE_OPC_ANY, NULL);
-	if (IS_ERR(op_data)) {
-		mutex_unlock(&lli->lli_layout_mutex);
+	if (IS_ERR(op_data))
 		return PTR_ERR(op_data);
-	}
 
 	/* have to enqueue one */
 	memset(&it, 0, sizeof(it));
@@ -3668,10 +3512,50 @@ again:
 	if (rc == 0) {
 		/* set lock data in case this is a new lock */
 		ll_set_lock_data(sbi->ll_md_exp, inode, &it, NULL);
-		rc = ll_layout_lock_set(&lockh, mode, inode, gen, true);
+		rc = ll_layout_lock_set(&lockh, mode, inode);
 		if (rc == -EAGAIN)
 			goto again;
 	}
+
+	return rc;
+}
+
+/**
+ * This function checks if there exists a LAYOUT lock on the client side,
+ * or enqueues it if it doesn't have one in cache.
+ *
+ * This function will not hold layout lock so it may be revoked any time after
+ * this function returns. Any operations depend on layout should be redone
+ * in that case.
+ *
+ * This function should be called before lov_io_init() to get an uptodate
+ * layout version, the caller should save the version number and after IO
+ * is finished, this function should be called again to verify that layout
+ * is not changed during IO time.
+ */
+int ll_layout_refresh(struct inode *inode, __u32 *gen)
+{
+	struct ll_inode_info *lli = ll_i2info(inode);
+	struct ll_sb_info *sbi = ll_i2sbi(inode);
+	int rc;
+
+	*gen = ll_layout_version_get(lli);
+	if (!(sbi->ll_flags & LL_SBI_LAYOUT_LOCK) || *gen != CL_LAYOUT_GEN_NONE)
+		return 0;
+
+	/* sanity checks */
+	LASSERT(fid_is_sane(ll_inode2fid(inode)));
+	LASSERT(S_ISREG(inode->i_mode));
+
+	/* take layout lock mutex to enqueue layout lock exclusively. */
+	mutex_lock(&lli->lli_layout_mutex);
+
+	rc = ll_layout_refresh_locked(inode);
+	if (rc < 0)
+		goto out;
+
+	*gen = ll_layout_version_get(lli);
+out:
 	mutex_unlock(&lli->lli_layout_mutex);
 
 	return rc;
diff --git a/drivers/staging/lustre/lustre/llite/glimpse.c b/drivers/staging/lustre/lustre/llite/glimpse.c
index 22507b9c6d69..504498de536e 100644
--- a/drivers/staging/lustre/lustre/llite/glimpse.c
+++ b/drivers/staging/lustre/lustre/llite/glimpse.c
@@ -80,69 +80,60 @@ blkcnt_t dirty_cnt(struct inode *inode)
 int cl_glimpse_lock(const struct lu_env *env, struct cl_io *io,
 		    struct inode *inode, struct cl_object *clob, int agl)
 {
-	struct ll_inode_info *lli   = ll_i2info(inode);
 	const struct lu_fid  *fid   = lu_object_fid(&clob->co_lu);
-	int result;
+	struct cl_lock *lock = vvp_env_lock(env);
+	struct cl_lock_descr *descr = &lock->cll_descr;
+	int result = 0;
+
+	CDEBUG(D_DLMTRACE, "Glimpsing inode " DFID "\n", PFID(fid));
+
+	/* NOTE: this looks like DLM lock request, but it may
+	 *       not be one. Due to CEF_ASYNC flag (translated
+	 *       to LDLM_FL_HAS_INTENT by osc), this is
+	 *       glimpse request, that won't revoke any
+	 *       conflicting DLM locks held. Instead,
+	 *       ll_glimpse_callback() will be called on each
+	 *       client holding a DLM lock against this file,
+	 *       and resulting size will be returned for each
+	 *       stripe. DLM lock on [0, EOF] is acquired only
+	 *       if there were no conflicting locks. If there
+	 *       were conflicting locks, enqueuing or waiting
+	 *       fails with -ENAVAIL, but valid inode
+	 *       attributes are returned anyway.
+	 */
+	*descr = whole_file;
+	descr->cld_obj = clob;
+	descr->cld_mode = CLM_READ;
+	descr->cld_enq_flags = CEF_ASYNC | CEF_MUST;
+	if (agl)
+		descr->cld_enq_flags |= CEF_AGL;
+	/*
+	 * CEF_ASYNC is used because glimpse sub-locks cannot
+	 * deadlock (because they never conflict with other
+	 * locks) and, hence, can be enqueued out-of-order.
+	 *
+	 * CEF_MUST protects glimpse lock from conversion into
+	 * a lockless mode.
+	 */
+	result = cl_lock_request(env, io, lock);
+	if (result < 0)
+		return result;
 
-	result = 0;
-	if (!(lli->lli_flags & LLIF_MDS_SIZE_LOCK)) {
-		CDEBUG(D_DLMTRACE, "Glimpsing inode " DFID "\n", PFID(fid));
-		if (lli->lli_has_smd) {
-			struct cl_lock *lock = vvp_env_lock(env);
-			struct cl_lock_descr *descr = &lock->cll_descr;
-
-			/* NOTE: this looks like DLM lock request, but it may
-			 *       not be one. Due to CEF_ASYNC flag (translated
-			 *       to LDLM_FL_HAS_INTENT by osc), this is
-			 *       glimpse request, that won't revoke any
-			 *       conflicting DLM locks held. Instead,
-			 *       ll_glimpse_callback() will be called on each
-			 *       client holding a DLM lock against this file,
-			 *       and resulting size will be returned for each
-			 *       stripe. DLM lock on [0, EOF] is acquired only
-			 *       if there were no conflicting locks. If there
-			 *       were conflicting locks, enqueuing or waiting
-			 *       fails with -ENAVAIL, but valid inode
-			 *       attributes are returned anyway.
-			 */
-			*descr = whole_file;
-			descr->cld_obj   = clob;
-			descr->cld_mode  = CLM_READ;
-			descr->cld_enq_flags = CEF_ASYNC | CEF_MUST;
-			if (agl)
-				descr->cld_enq_flags |= CEF_AGL;
+	if (!agl) {
+		ll_merge_attr(env, inode);
+		if (i_size_read(inode) > 0 && !inode->i_blocks) {
 			/*
-			 * CEF_ASYNC is used because glimpse sub-locks cannot
-			 * deadlock (because they never conflict with other
-			 * locks) and, hence, can be enqueued out-of-order.
-			 *
-			 * CEF_MUST protects glimpse lock from conversion into
-			 * a lockless mode.
+			 * LU-417: Add dirty pages block count
+			 * lest i_blocks reports 0, some "cp" or
+			 * "tar" may think it's a completely
+			 * sparse file and skip it.
 			 */
-			result = cl_lock_request(env, io, lock);
-			if (result < 0)
-				return result;
-
-			if (!agl) {
-				ll_merge_attr(env, inode);
-				if (i_size_read(inode) > 0 &&
-				    inode->i_blocks == 0) {
-					/*
-					 * LU-417: Add dirty pages block count
-					 * lest i_blocks reports 0, some "cp" or
-					 * "tar" may think it's a completely
-					 * sparse file and skip it.
-					 */
-					inode->i_blocks = dirty_cnt(inode);
-				}
-			}
-			cl_lock_release(env, lock);
-		} else {
-			CDEBUG(D_DLMTRACE, "No objects for inode\n");
-			ll_merge_attr(env, inode);
+			inode->i_blocks = dirty_cnt(inode);
 		}
 	}
 
+	cl_lock_release(env, lock);
+
 	return result;
 }
 
@@ -212,39 +203,3 @@ again:
 	}
 	return result;
 }
-
-int cl_local_size(struct inode *inode)
-{
-	struct lu_env	   *env = NULL;
-	struct cl_io	    *io  = NULL;
-	struct cl_object	*clob;
-	int		      result;
-	int		      refcheck;
-
-	if (!ll_i2info(inode)->lli_has_smd)
-		return 0;
-
-	result = cl_io_get(inode, &env, &io, &refcheck);
-	if (result <= 0)
-		return result;
-
-	clob = io->ci_obj;
-	result = cl_io_init(env, io, CIT_MISC, clob);
-	if (result > 0) {
-		result = io->ci_result;
-	} else if (result == 0) {
-		struct cl_lock *lock = vvp_env_lock(env);
-
-		lock->cll_descr = whole_file;
-		lock->cll_descr.cld_enq_flags = CEF_PEEK;
-		lock->cll_descr.cld_obj = clob;
-		result = cl_lock_request(env, io, lock);
-		if (result == 0) {
-			ll_merge_attr(env, inode);
-			cl_lock_release(env, lock);
-		}
-	}
-	cl_io_fini(env, io);
-	cl_env_put(env, &refcheck);
-	return result;
-}
diff --git a/drivers/staging/lustre/lustre/llite/lcommon_cl.c b/drivers/staging/lustre/lustre/llite/lcommon_cl.c
index 084330d08f7a..dd1cfd8f5213 100644
--- a/drivers/staging/lustre/lustre/llite/lcommon_cl.c
+++ b/drivers/staging/lustre/lustre/llite/lcommon_cl.c
@@ -80,7 +80,8 @@ int cl_inode_fini_refcheck;
  */
 static DEFINE_MUTEX(cl_inode_fini_guard);
 
-int cl_setattr_ost(struct inode *inode, const struct iattr *attr)
+int cl_setattr_ost(struct cl_object *obj, const struct iattr *attr,
+		   unsigned int attr_flags)
 {
 	struct lu_env *env;
 	struct cl_io  *io;
@@ -92,14 +93,15 @@ int cl_setattr_ost(struct inode *inode, const struct iattr *attr)
 		return PTR_ERR(env);
 
 	io = vvp_env_thread_io(env);
-	io->ci_obj = ll_i2info(inode)->lli_clob;
+	io->ci_obj = obj;
 
 	io->u.ci_setattr.sa_attr.lvb_atime = LTIME_S(attr->ia_atime);
 	io->u.ci_setattr.sa_attr.lvb_mtime = LTIME_S(attr->ia_mtime);
 	io->u.ci_setattr.sa_attr.lvb_ctime = LTIME_S(attr->ia_ctime);
 	io->u.ci_setattr.sa_attr.lvb_size = attr->ia_size;
+	io->u.ci_setattr.sa_attr_flags = attr_flags;
 	io->u.ci_setattr.sa_valid = attr->ia_valid;
-	io->u.ci_setattr.sa_parent_fid = ll_inode2fid(inode);
+	io->u.ci_setattr.sa_parent_fid = lu_object_fid(&obj->co_lu);
 
 again:
 	if (cl_io_init(env, io, CIT_SETATTR, io->ci_obj) == 0) {
@@ -148,7 +150,7 @@ int cl_file_inode_init(struct inode *inode, struct lustre_md *md)
 	struct cl_object_conf conf = {
 		.coc_inode = inode,
 		.u = {
-			.coc_md    = md
+			.coc_layout = md->layout,
 		}
 	};
 	int result = 0;
@@ -182,7 +184,6 @@ int cl_file_inode_init(struct inode *inode, struct lustre_md *md)
 			 * locked by I_NEW bit.
 			 */
 			lli->lli_clob = clob;
-			lli->lli_has_smd = lsm_has_objects(md->lsm);
 			lu_object_ref_add(&clob->co_lu, "inode", inode);
 		} else {
 			result = PTR_ERR(clob);
@@ -245,15 +246,11 @@ void cl_inode_fini(struct inode *inode)
 	int emergency;
 
 	if (clob) {
-		void		    *cookie;
-
-		cookie = cl_env_reenter();
 		env = cl_env_get(&refcheck);
 		emergency = IS_ERR(env);
 		if (emergency) {
 			mutex_lock(&cl_inode_fini_guard);
 			LASSERT(cl_inode_fini_env);
-			cl_env_implant(cl_inode_fini_env, &refcheck);
 			env = cl_inode_fini_env;
 		}
 		/*
@@ -265,13 +262,10 @@ void cl_inode_fini(struct inode *inode)
 		lu_object_ref_del(&clob->co_lu, "inode", inode);
 		cl_object_put_last(env, clob);
 		lli->lli_clob = NULL;
-		if (emergency) {
-			cl_env_unplant(cl_inode_fini_env, &refcheck);
+		if (emergency)
 			mutex_unlock(&cl_inode_fini_guard);
-		} else {
+		else
 			cl_env_put(env, &refcheck);
-		}
-		cl_env_reexit(cookie);
 	}
 }
 
@@ -302,22 +296,3 @@ __u32 cl_fid_build_gen(const struct lu_fid *fid)
 	gen = fid_flatten(fid) >> 32;
 	return gen;
 }
-
-/* lsm is unreliable after hsm implementation as layout can be changed at
- * any time. This is only to support old, non-clio-ized interfaces. It will
- * cause deadlock if clio operations are called with this extra layout refcount
- * because in case the layout changed during the IO, ll_layout_refresh() will
- * have to wait for the refcount to become zero to destroy the older layout.
- *
- * Notice that the lsm returned by this function may not be valid unless called
- * inside layout lock - MDS_INODELOCK_LAYOUT.
- */
-struct lov_stripe_md *ccc_inode_lsm_get(struct inode *inode)
-{
-	return lov_lsm_get(ll_i2info(inode)->lli_clob);
-}
-
-inline void ccc_inode_lsm_put(struct inode *inode, struct lov_stripe_md *lsm)
-{
-	lov_lsm_put(ll_i2info(inode)->lli_clob, lsm);
-}
diff --git a/drivers/staging/lustre/lustre/llite/lcommon_misc.c b/drivers/staging/lustre/lustre/llite/lcommon_misc.c
index fb346c12dad2..f48660ed350f 100644
--- a/drivers/staging/lustre/lustre/llite/lcommon_misc.c
+++ b/drivers/staging/lustre/lustre/llite/lcommon_misc.c
@@ -47,36 +47,29 @@
  */
 int cl_init_ea_size(struct obd_export *md_exp, struct obd_export *dt_exp)
 {
-	struct lov_stripe_md lsm = { .lsm_magic = LOV_MAGIC_V3 };
-	__u32 valsize = sizeof(struct lov_desc);
-	int rc, easize, def_easize, cookiesize;
-	struct lov_desc desc;
-	__u16 stripes, def_stripes;
-
-	rc = obd_get_info(NULL, dt_exp, sizeof(KEY_LOVDESC), KEY_LOVDESC,
-			  &valsize, &desc, NULL);
+	u32 val_size, max_easize, def_easize;
+	int rc;
+
+	val_size = sizeof(max_easize);
+	rc = obd_get_info(NULL, dt_exp, sizeof(KEY_MAX_EASIZE), KEY_MAX_EASIZE,
+			  &val_size, &max_easize);
 	if (rc)
 		return rc;
 
-	stripes = min_t(__u32, desc.ld_tgt_count, LOV_MAX_STRIPE_COUNT);
-	lsm.lsm_stripe_count = stripes;
-	easize = obd_size_diskmd(dt_exp, &lsm);
-
-	def_stripes = min_t(__u32, desc.ld_default_stripe_count,
-			    LOV_MAX_STRIPE_COUNT);
-	lsm.lsm_stripe_count = def_stripes;
-	def_easize = obd_size_diskmd(dt_exp, &lsm);
-
-	cookiesize = stripes * sizeof(struct llog_cookie);
+	val_size = sizeof(def_easize);
+	rc = obd_get_info(NULL, dt_exp, sizeof(KEY_DEFAULT_EASIZE),
+			  KEY_DEFAULT_EASIZE, &val_size, &def_easize);
+	if (rc)
+		return rc;
 
-	/* default cookiesize is 0 because from 2.4 server doesn't send
+	/*
+	 * default cookiesize is 0 because from 2.4 server doesn't send
 	 * llog cookies to client.
 	 */
-	CDEBUG(D_HA,
-	       "updating def/max_easize: %d/%d def/max_cookiesize: 0/%d\n",
-	       def_easize, easize, cookiesize);
+	CDEBUG(D_HA, "updating def/max_easize: %d/%d\n",
+	       def_easize, max_easize);
 
-	rc = md_init_ea_size(md_exp, easize, def_easize, cookiesize, 0);
+	rc = md_init_ea_size(md_exp, max_easize, def_easize);
 	return rc;
 }
 
@@ -169,13 +162,11 @@ int cl_get_grouplock(struct cl_object *obj, unsigned long gid, int nonblock,
 		return rc;
 	}
 
-	cg->lg_env  = cl_env_get(&refcheck);
+	cg->lg_env  = env;
 	cg->lg_io   = io;
 	cg->lg_lock = lock;
 	cg->lg_gid  = gid;
-	LASSERT(cg->lg_env == env);
 
-	cl_env_unplant(env, &refcheck);
 	return 0;
 }
 
@@ -184,14 +175,10 @@ void cl_put_grouplock(struct ll_grouplock *cg)
 	struct lu_env  *env  = cg->lg_env;
 	struct cl_io   *io   = cg->lg_io;
 	struct cl_lock *lock = cg->lg_lock;
-	int	     refcheck;
 
 	LASSERT(cg->lg_env);
 	LASSERT(cg->lg_gid);
 
-	cl_env_implant(env, &refcheck);
-	cl_env_put(env, &refcheck);
-
 	cl_lock_release(env, lock);
 	cl_io_fini(env, io);
 	cl_env_put(env, NULL);
diff --git a/drivers/staging/lustre/lustre/llite/llite_close.c b/drivers/staging/lustre/lustre/llite/llite_close.c
deleted file mode 100644
index 8644631bf2ba..000000000000
--- a/drivers/staging/lustre/lustre/llite/llite_close.c
+++ /dev/null
@@ -1,395 +0,0 @@
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- *
- * Copyright (c) 2011, 2012, Intel Corporation.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- *
- * lustre/llite/llite_close.c
- *
- * Lustre Lite routines to issue a secondary close after writeback
- */
-
-#include <linux/module.h>
-
-#define DEBUG_SUBSYSTEM S_LLITE
-
-#include "llite_internal.h"
-
-/** records that a write is in flight */
-void vvp_write_pending(struct vvp_object *club, struct vvp_page *page)
-{
-	struct ll_inode_info *lli = ll_i2info(club->vob_inode);
-
-	spin_lock(&lli->lli_lock);
-	lli->lli_flags |= LLIF_SOM_DIRTY;
-	if (page && list_empty(&page->vpg_pending_linkage))
-		list_add(&page->vpg_pending_linkage, &club->vob_pending_list);
-	spin_unlock(&lli->lli_lock);
-}
-
-/** records that a write has completed */
-void vvp_write_complete(struct vvp_object *club, struct vvp_page *page)
-{
-	struct ll_inode_info *lli = ll_i2info(club->vob_inode);
-	int rc = 0;
-
-	spin_lock(&lli->lli_lock);
-	if (page && !list_empty(&page->vpg_pending_linkage)) {
-		list_del_init(&page->vpg_pending_linkage);
-		rc = 1;
-	}
-	spin_unlock(&lli->lli_lock);
-	if (rc)
-		ll_queue_done_writing(club->vob_inode, 0);
-}
-
-/** Queues DONE_WRITING if
- * - done writing is allowed;
- * - inode has no no dirty pages;
- */
-void ll_queue_done_writing(struct inode *inode, unsigned long flags)
-{
-	struct ll_inode_info *lli = ll_i2info(inode);
-	struct vvp_object *club = cl2vvp(ll_i2info(inode)->lli_clob);
-
-	spin_lock(&lli->lli_lock);
-	lli->lli_flags |= flags;
-
-	if ((lli->lli_flags & LLIF_DONE_WRITING) &&
-	    list_empty(&club->vob_pending_list)) {
-		struct ll_close_queue *lcq = ll_i2sbi(inode)->ll_lcq;
-
-		if (lli->lli_flags & LLIF_MDS_SIZE_LOCK)
-			CWARN("%s: file "DFID"(flags %u) Size-on-MDS valid, done writing allowed and no diry pages\n",
-			      ll_get_fsname(inode->i_sb, NULL, 0),
-			      PFID(ll_inode2fid(inode)), lli->lli_flags);
-		/* DONE_WRITING is allowed and inode has no dirty page. */
-		spin_lock(&lcq->lcq_lock);
-
-		LASSERT(list_empty(&lli->lli_close_list));
-		CDEBUG(D_INODE, "adding inode "DFID" to close list\n",
-		       PFID(ll_inode2fid(inode)));
-		list_add_tail(&lli->lli_close_list, &lcq->lcq_head);
-
-		/* Avoid a concurrent insertion into the close thread queue:
-		 * an inode is already in the close thread, open(), write(),
-		 * close() happen, epoch is closed as the inode is marked as
-		 * LLIF_EPOCH_PENDING. When pages are written inode should not
-		 * be inserted into the queue again, clear this flag to avoid
-		 * it.
-		 */
-		lli->lli_flags &= ~LLIF_DONE_WRITING;
-
-		wake_up(&lcq->lcq_waitq);
-		spin_unlock(&lcq->lcq_lock);
-	}
-	spin_unlock(&lli->lli_lock);
-}
-
-/** Pack SOM attributes info @opdata for CLOSE, DONE_WRITING rpc. */
-void ll_done_writing_attr(struct inode *inode, struct md_op_data *op_data)
-{
-	struct ll_inode_info *lli = ll_i2info(inode);
-
-	op_data->op_flags |= MF_SOM_CHANGE;
-	/* Check if Size-on-MDS attributes are valid. */
-	if (lli->lli_flags & LLIF_MDS_SIZE_LOCK)
-		CERROR("%s: inode "DFID"(flags %u) MDS holds lock on Size-on-MDS attributes\n",
-		       ll_get_fsname(inode->i_sb, NULL, 0),
-		       PFID(ll_inode2fid(inode)), lli->lli_flags);
-
-	if (!cl_local_size(inode)) {
-		/* Send Size-on-MDS Attributes if valid. */
-		op_data->op_attr.ia_valid |= ATTR_MTIME_SET | ATTR_CTIME_SET |
-				ATTR_ATIME_SET | ATTR_SIZE | ATTR_BLOCKS;
-	}
-}
-
-/** Closes ioepoch and packs Size-on-MDS attribute if needed into @op_data. */
-void ll_ioepoch_close(struct inode *inode, struct md_op_data *op_data,
-		      struct obd_client_handle **och, unsigned long flags)
-{
-	struct ll_inode_info *lli = ll_i2info(inode);
-	struct vvp_object *club = cl2vvp(ll_i2info(inode)->lli_clob);
-
-	spin_lock(&lli->lli_lock);
-	if (!(list_empty(&club->vob_pending_list))) {
-		if (!(lli->lli_flags & LLIF_EPOCH_PENDING)) {
-			LASSERT(*och);
-			LASSERT(!lli->lli_pending_och);
-			/* Inode is dirty and there is no pending write done
-			 * request yet, DONE_WRITE is to be sent later.
-			 */
-			lli->lli_flags |= LLIF_EPOCH_PENDING;
-			lli->lli_pending_och = *och;
-			spin_unlock(&lli->lli_lock);
-
-			inode = igrab(inode);
-			LASSERT(inode);
-			goto out;
-		}
-		if (flags & LLIF_DONE_WRITING) {
-			/* Some pages are still dirty, it is early to send
-			 * DONE_WRITE. Wait until all pages will be flushed
-			 * and try DONE_WRITE again later.
-			 */
-			LASSERT(!(lli->lli_flags & LLIF_DONE_WRITING));
-			lli->lli_flags |= LLIF_DONE_WRITING;
-			spin_unlock(&lli->lli_lock);
-
-			inode = igrab(inode);
-			LASSERT(inode);
-			goto out;
-		}
-	}
-	CDEBUG(D_INODE, "Epoch %llu closed on "DFID"\n",
-	       ll_i2info(inode)->lli_ioepoch, PFID(&lli->lli_fid));
-	op_data->op_flags |= MF_EPOCH_CLOSE;
-
-	if (flags & LLIF_DONE_WRITING) {
-		LASSERT(lli->lli_flags & LLIF_SOM_DIRTY);
-		LASSERT(!(lli->lli_flags & LLIF_DONE_WRITING));
-		*och = lli->lli_pending_och;
-		lli->lli_pending_och = NULL;
-		lli->lli_flags &= ~LLIF_EPOCH_PENDING;
-	} else {
-		/* Pack Size-on-MDS inode attributes only if they has changed */
-		if (!(lli->lli_flags & LLIF_SOM_DIRTY)) {
-			spin_unlock(&lli->lli_lock);
-			goto out;
-		}
-
-		/* There is a pending DONE_WRITE -- close epoch with no
-		 * attribute change.
-		 */
-		if (lli->lli_flags & LLIF_EPOCH_PENDING) {
-			spin_unlock(&lli->lli_lock);
-			goto out;
-		}
-	}
-
-	LASSERT(list_empty(&club->vob_pending_list));
-	lli->lli_flags &= ~LLIF_SOM_DIRTY;
-	spin_unlock(&lli->lli_lock);
-	ll_done_writing_attr(inode, op_data);
-
-out:
-	return;
-}
-
-/**
- * Cliens updates SOM attributes on MDS (including llog cookies):
- * obd_getattr with no lock and md_setattr.
- */
-int ll_som_update(struct inode *inode, struct md_op_data *op_data)
-{
-	struct ll_inode_info *lli = ll_i2info(inode);
-	struct ptlrpc_request *request = NULL;
-	__u32 old_flags;
-	struct obdo *oa;
-	int rc;
-
-	LASSERT(op_data);
-	if (lli->lli_flags & LLIF_MDS_SIZE_LOCK)
-		CERROR("%s: inode "DFID"(flags %u) MDS holds lock on Size-on-MDS attributes\n",
-		       ll_get_fsname(inode->i_sb, NULL, 0),
-		       PFID(ll_inode2fid(inode)), lli->lli_flags);
-
-	oa = kmem_cache_zalloc(obdo_cachep, GFP_NOFS);
-	if (!oa) {
-		CERROR("can't allocate memory for Size-on-MDS update.\n");
-		return -ENOMEM;
-	}
-
-	old_flags = op_data->op_flags;
-	op_data->op_flags = MF_SOM_CHANGE;
-
-	/* If inode is already in another epoch, skip getattr from OSTs. */
-	if (lli->lli_ioepoch == op_data->op_ioepoch) {
-		rc = ll_inode_getattr(inode, oa, op_data->op_ioepoch,
-				      old_flags & MF_GETATTR_LOCK);
-		if (rc) {
-			oa->o_valid = 0;
-			if (rc != -ENOENT)
-				CERROR("%s: inode_getattr failed - unable to send a Size-on-MDS attribute update for inode "DFID": rc = %d\n",
-				       ll_get_fsname(inode->i_sb, NULL, 0),
-				       PFID(ll_inode2fid(inode)), rc);
-		} else {
-			CDEBUG(D_INODE, "Size-on-MDS update on "DFID"\n",
-			       PFID(&lli->lli_fid));
-		}
-		/* Install attributes into op_data. */
-		md_from_obdo(op_data, oa, oa->o_valid);
-	}
-
-	rc = md_setattr(ll_i2sbi(inode)->ll_md_exp, op_data,
-			NULL, 0, NULL, 0, &request, NULL);
-	ptlrpc_req_finished(request);
-
-	kmem_cache_free(obdo_cachep, oa);
-	return rc;
-}
-
-/**
- * Closes the ioepoch and packs all the attributes into @op_data for
- * DONE_WRITING rpc.
- */
-static void ll_prepare_done_writing(struct inode *inode,
-				    struct md_op_data *op_data,
-				    struct obd_client_handle **och)
-{
-	ll_ioepoch_close(inode, op_data, och, LLIF_DONE_WRITING);
-	/* If there is no @och, we do not do D_W yet. */
-	if (!*och)
-		return;
-
-	ll_pack_inode2opdata(inode, op_data, &(*och)->och_fh);
-	ll_prep_md_op_data(op_data, inode, NULL, NULL,
-			   0, 0, LUSTRE_OPC_ANY, NULL);
-}
-
-/** Send a DONE_WRITING rpc. */
-static void ll_done_writing(struct inode *inode)
-{
-	struct obd_client_handle *och = NULL;
-	struct md_op_data *op_data;
-	int rc;
-
-	LASSERT(exp_connect_som(ll_i2mdexp(inode)));
-
-	op_data = kzalloc(sizeof(*op_data), GFP_NOFS);
-	if (!op_data)
-		return;
-
-	ll_prepare_done_writing(inode, op_data, &och);
-	/* If there is no @och, we do not do D_W yet. */
-	if (!och)
-		goto out;
-
-	rc = md_done_writing(ll_i2sbi(inode)->ll_md_exp, op_data, NULL);
-	if (rc == -EAGAIN)
-		/* MDS has instructed us to obtain Size-on-MDS attribute from
-		 * OSTs and send setattr to back to MDS.
-		 */
-		rc = ll_som_update(inode, op_data);
-	else if (rc) {
-		CERROR("%s: inode "DFID" mdc done_writing failed: rc = %d\n",
-		       ll_get_fsname(inode->i_sb, NULL, 0),
-		       PFID(ll_inode2fid(inode)), rc);
-	}
-out:
-	ll_finish_md_op_data(op_data);
-	if (och) {
-		md_clear_open_replay_data(ll_i2sbi(inode)->ll_md_exp, och);
-		kfree(och);
-	}
-}
-
-static struct ll_inode_info *ll_close_next_lli(struct ll_close_queue *lcq)
-{
-	struct ll_inode_info *lli = NULL;
-
-	spin_lock(&lcq->lcq_lock);
-
-	if (!list_empty(&lcq->lcq_head)) {
-		lli = list_entry(lcq->lcq_head.next, struct ll_inode_info,
-				 lli_close_list);
-		list_del_init(&lli->lli_close_list);
-	} else if (atomic_read(&lcq->lcq_stop)) {
-		lli = ERR_PTR(-EALREADY);
-	}
-
-	spin_unlock(&lcq->lcq_lock);
-	return lli;
-}
-
-static int ll_close_thread(void *arg)
-{
-	struct ll_close_queue *lcq = arg;
-
-	complete(&lcq->lcq_comp);
-
-	while (1) {
-		struct l_wait_info lwi = { 0 };
-		struct ll_inode_info *lli;
-		struct inode *inode;
-
-		l_wait_event_exclusive(lcq->lcq_waitq,
-				       (lli = ll_close_next_lli(lcq)) != NULL,
-				       &lwi);
-		if (IS_ERR(lli))
-			break;
-
-		inode = ll_info2i(lli);
-		CDEBUG(D_INFO, "done_writing for inode "DFID"\n",
-		       PFID(ll_inode2fid(inode)));
-		ll_done_writing(inode);
-		iput(inode);
-	}
-
-	CDEBUG(D_INFO, "ll_close exiting\n");
-	complete(&lcq->lcq_comp);
-	return 0;
-}
-
-int ll_close_thread_start(struct ll_close_queue **lcq_ret)
-{
-	struct ll_close_queue *lcq;
-	struct task_struct *task;
-
-	if (OBD_FAIL_CHECK(OBD_FAIL_LDLM_CLOSE_THREAD))
-		return -EINTR;
-
-	lcq = kzalloc(sizeof(*lcq), GFP_NOFS);
-	if (!lcq)
-		return -ENOMEM;
-
-	spin_lock_init(&lcq->lcq_lock);
-	INIT_LIST_HEAD(&lcq->lcq_head);
-	init_waitqueue_head(&lcq->lcq_waitq);
-	init_completion(&lcq->lcq_comp);
-
-	task = kthread_run(ll_close_thread, lcq, "ll_close");
-	if (IS_ERR(task)) {
-		kfree(lcq);
-		return PTR_ERR(task);
-	}
-
-	wait_for_completion(&lcq->lcq_comp);
-	*lcq_ret = lcq;
-	return 0;
-}
-
-void ll_close_thread_shutdown(struct ll_close_queue *lcq)
-{
-	init_completion(&lcq->lcq_comp);
-	atomic_inc(&lcq->lcq_stop);
-	wake_up(&lcq->lcq_waitq);
-	wait_for_completion(&lcq->lcq_comp);
-	kfree(lcq);
-}
diff --git a/drivers/staging/lustre/lustre/llite/llite_internal.h b/drivers/staging/lustre/lustre/llite/llite_internal.h
index 4bc551279aa4..2f46d475cd7d 100644
--- a/drivers/staging/lustre/lustre/llite/llite_internal.h
+++ b/drivers/staging/lustre/lustre/llite/llite_internal.h
@@ -97,31 +97,20 @@ struct ll_grouplock {
 	unsigned long	 lg_gid;
 };
 
-enum lli_flags {
-	/* MDS has an authority for the Size-on-MDS attributes. */
-	LLIF_MDS_SIZE_LOCK      = (1 << 0),
-	/* Epoch close is postponed. */
-	LLIF_EPOCH_PENDING      = (1 << 1),
-	/* DONE WRITING is allowed. */
-	LLIF_DONE_WRITING       = (1 << 2),
-	/* Sizeon-on-MDS attributes are changed. An attribute update needs to
-	 * be sent to MDS.
-	 */
-	LLIF_SOM_DIRTY	  = (1 << 3),
+enum ll_file_flags {
 	/* File data is modified. */
-	LLIF_DATA_MODIFIED      = (1 << 4),
+	LLIF_DATA_MODIFIED	= 0,
 	/* File is being restored */
-	LLIF_FILE_RESTORING	= (1 << 5),
+	LLIF_FILE_RESTORING	= 1,
 	/* Xattr cache is attached to the file */
-	LLIF_XATTR_CACHE	= (1 << 6),
+	LLIF_XATTR_CACHE	= 2,
 };
 
 struct ll_inode_info {
 	__u32				lli_inode_magic;
-	__u32				lli_flags;
-	__u64				lli_ioepoch;
 
 	spinlock_t			lli_lock;
+	unsigned long			lli_flags;
 	struct posix_acl		*lli_posix_acl;
 
 	/* identifying fields for both metadata and data stacks. */
@@ -129,14 +118,6 @@ struct ll_inode_info {
 	/* master inode fid for stripe directory */
 	struct lu_fid		   lli_pfid;
 
-	struct list_head	      lli_close_list;
-
-	/* handle is to be sent to MDS later on done_writing and setattr.
-	 * Open handle data are needed for the recovery to reconstruct
-	 * the inode state on the MDS. XXX: recovery is not ready yet.
-	 */
-	struct obd_client_handle       *lli_pending_och;
-
 	/* We need all three because every inode may be opened in different
 	 * modes
 	 */
@@ -204,7 +185,6 @@ struct ll_inode_info {
 		struct {
 			struct mutex			lli_size_mutex;
 			char			       *lli_symlink_name;
-			__u64				lli_maxbytes;
 			/*
 			 * struct rw_semaphore {
 			 *    signed long	count;     // align d.d_def_acl
@@ -245,7 +225,6 @@ struct ll_inode_info {
 	 *      In the future, if more members are added only for directory,
 	 *      some of the following members can be moved into u.f.
 	 */
-	bool			    lli_has_smd;
 	struct cl_object	       *lli_clob;
 
 	/* mutex to request for layout lock exclusively. */
@@ -282,6 +261,9 @@ int ll_xattr_cache_destroy(struct inode *inode);
 int ll_xattr_cache_get(struct inode *inode, const char *name,
 		       char *buffer, size_t size, __u64 valid);
 
+int ll_init_security(struct dentry *dentry, struct inode *inode,
+		     struct inode *dir);
+
 /*
  * Locking to guarantee consistency of non-atomic updates to long long i_size,
  * consistency between file size and KMS.
@@ -400,7 +382,7 @@ enum stats_track_type {
 #define LL_SBI_LOCALFLOCK       0x200 /* Local flocks support by kernel */
 #define LL_SBI_LRU_RESIZE       0x400 /* lru resize support */
 #define LL_SBI_LAZYSTATFS       0x800 /* lazystatfs mount option */
-#define LL_SBI_SOM_PREVIEW     0x1000 /* SOM preview mount option */
+/*	LL_SBI_SOM_PREVIEW     0x1000    SOM preview mount option, obsolete */
 #define LL_SBI_32BIT_API       0x2000 /* generate 32 bit inodes. */
 #define LL_SBI_64BIT_HASH      0x4000 /* support 64-bits dir hash/offset */
 #define LL_SBI_AGL_ENABLED     0x8000 /* enable agl */
@@ -409,6 +391,8 @@ enum stats_track_type {
 #define LL_SBI_USER_FID2PATH  0x40000 /* allow fid2path by unprivileged users */
 #define LL_SBI_XATTR_CACHE    0x80000 /* support for xattr cache */
 #define LL_SBI_NOROOTSQUASH	0x100000 /* do not apply root squash */
+#define LL_SBI_ALWAYS_PING	0x200000 /* always ping even if server
+					  * suppress_pings */
 
 #define LL_SBI_FLAGS {	\
 	"nolck",	\
@@ -432,6 +416,7 @@ enum stats_track_type {
 	"user_fid2path",\
 	"xattr_cache",	\
 	"norootsquash",	\
+	"always_ping",	\
 }
 
 /*
@@ -466,10 +451,10 @@ struct ll_sb_info {
 
 	int		       ll_flags;
 	unsigned int		  ll_umounting:1,
-				  ll_xattr_cache_enabled:1;
-	struct lustre_client_ocd  ll_lco;
+				  ll_xattr_cache_enabled:1,
+				  ll_client_common_fill_super_succeeded:1;
 
-	struct ll_close_queue    *ll_lcq;
+	struct lustre_client_ocd  ll_lco;
 
 	struct lprocfs_stats     *ll_stats; /* lprocfs stats counter */
 
@@ -630,8 +615,6 @@ struct ll_file_data {
 	struct list_head fd_lccs; /* list of ll_cl_context */
 };
 
-struct lov_stripe_md;
-
 extern struct dentry *llite_root;
 extern struct kset *llite_kset;
 
@@ -682,8 +665,6 @@ enum {
 	LPROC_LL_WRITE_BYTES,
 	LPROC_LL_BRW_READ,
 	LPROC_LL_BRW_WRITE,
-	LPROC_LL_OSC_READ,
-	LPROC_LL_OSC_WRITE,
 	LPROC_LL_IOCTL,
 	LPROC_LL_OPEN,
 	LPROC_LL_RELEASE,
@@ -741,9 +722,7 @@ int ll_writepage(struct page *page, struct writeback_control *wbc);
 int ll_writepages(struct address_space *, struct writeback_control *wbc);
 int ll_readpage(struct file *file, struct page *page);
 void ll_readahead_init(struct inode *inode, struct ll_readahead_state *ras);
-int ll_readahead(const struct lu_env *env, struct cl_io *io,
-		 struct cl_page_list *queue, struct ll_readahead_state *ras,
-		 bool hit);
+int vvp_io_write_commit(const struct lu_env *env, struct cl_io *io);
 struct ll_cl_context *ll_cl_find(struct file *file);
 void ll_cl_add(struct file *file, const struct lu_env *env, struct cl_io *io);
 void ll_cl_remove(struct file *file, const struct lu_env *env);
@@ -762,25 +741,14 @@ enum ldlm_mode ll_take_md_lock(struct inode *inode, __u64 bits,
 			       enum ldlm_mode mode);
 int ll_file_open(struct inode *inode, struct file *file);
 int ll_file_release(struct inode *inode, struct file *file);
-int ll_glimpse_ioctl(struct ll_sb_info *sbi,
-		     struct lov_stripe_md *lsm, lstat_t *st);
-void ll_ioepoch_open(struct ll_inode_info *lli, __u64 ioepoch);
 int ll_release_openhandle(struct inode *, struct lookup_intent *);
 int ll_md_real_close(struct inode *inode, fmode_t fmode);
-void ll_ioepoch_close(struct inode *inode, struct md_op_data *op_data,
-		      struct obd_client_handle **och, unsigned long flags);
-void ll_done_writing_attr(struct inode *inode, struct md_op_data *op_data);
-int ll_som_update(struct inode *inode, struct md_op_data *op_data);
-int ll_inode_getattr(struct inode *inode, struct obdo *obdo,
-		     __u64 ioepoch, int sync);
-void ll_pack_inode2opdata(struct inode *inode, struct md_op_data *op_data,
-			  struct lustre_handle *fh);
 int ll_getattr(struct vfsmount *mnt, struct dentry *de, struct kstat *stat);
 struct posix_acl *ll_get_acl(struct inode *inode, int type);
 int ll_migrate(struct inode *parent, struct file *file, int mdtidx,
 	       const char *name, int namelen);
 int ll_get_fid_by_name(struct inode *parent, const char *name,
-		       int namelen, struct lu_fid *fid);
+		       int namelen, struct lu_fid *fid, struct inode **inode);
 int ll_inode_permission(struct inode *inode, int mask);
 
 int ll_lov_setstripe_ea_info(struct inode *inode, struct dentry *dentry,
@@ -818,6 +786,7 @@ int ll_fill_super(struct super_block *sb, struct vfsmount *mnt);
 void ll_put_super(struct super_block *sb);
 void ll_kill_super(struct super_block *sb);
 struct inode *ll_inode_from_resource_lock(struct ldlm_lock *lock);
+void ll_dir_clear_lsm_md(struct inode *inode);
 void ll_clear_inode(struct inode *inode);
 int ll_setattr_raw(struct dentry *dentry, struct iattr *attr, bool hsm_import);
 int ll_setattr(struct dentry *de, struct iattr *attr);
@@ -891,18 +860,6 @@ int ll_dir_get_parent_fid(struct inode *dir, struct lu_fid *parent_fid);
 /* llite/symlink.c */
 extern const struct inode_operations ll_fast_symlink_inode_operations;
 
-/* llite/llite_close.c */
-struct ll_close_queue {
-	spinlock_t		lcq_lock;
-	struct list_head		lcq_head;
-	wait_queue_head_t		lcq_waitq;
-	struct completion	lcq_comp;
-	atomic_t		lcq_stop;
-};
-
-void vvp_write_pending(struct vvp_object *club, struct vvp_page *page);
-void vvp_write_complete(struct vvp_object *club, struct vvp_page *page);
-
 /**
  * IO arguments for various VFS I/O interfaces.
  */
@@ -945,15 +902,11 @@ static inline struct vvp_io_args *ll_env_args(const struct lu_env *env)
 	return &ll_env_info(env)->lti_args;
 }
 
-void ll_queue_done_writing(struct inode *inode, unsigned long flags);
-void ll_close_thread_shutdown(struct ll_close_queue *lcq);
-int ll_close_thread_start(struct ll_close_queue **lcq_ret);
-
 /* llite/llite_mmap.c */
 
 int ll_teardown_mmaps(struct address_space *mapping, __u64 first, __u64 last);
 int ll_file_mmap(struct file *file, struct vm_area_struct *vma);
-void policy_from_vma(ldlm_policy_data_t *policy, struct vm_area_struct *vma,
+void policy_from_vma(union ldlm_policy_data *policy, struct vm_area_struct *vma,
 		     unsigned long addr, size_t count);
 struct vm_area_struct *our_vma(struct mm_struct *mm, unsigned long addr,
 			       size_t count);
@@ -1024,9 +977,14 @@ static inline struct lu_fid *ll_inode2fid(struct inode *inode)
 	return fid;
 }
 
-static inline __u64 ll_file_maxbytes(struct inode *inode)
+static inline loff_t ll_file_maxbytes(struct inode *inode)
 {
-	return ll_i2info(inode)->lli_maxbytes;
+	struct cl_object *obj = ll_i2info(inode)->lli_clob;
+
+	if (!obj)
+		return MAX_LFS_FILESIZE;
+
+	return min_t(loff_t, cl_object_maxbytes(obj), MAX_LFS_FILESIZE);
 }
 
 /* llite/xattr.c */
@@ -1043,17 +1001,18 @@ extern const struct xattr_handler *ll_xattr_handlers[];
 ssize_t ll_listxattr(struct dentry *dentry, char *buffer, size_t size);
 int ll_xattr_list(struct inode *inode, const char *name, int type,
 		  void *buffer, size_t size, __u64 valid);
+const struct xattr_handler *get_xattr_type(const char *name);
 
 /**
  * Common IO arguments for various VFS I/O interfaces.
  */
 int cl_sb_init(struct super_block *sb);
 int cl_sb_fini(struct super_block *sb);
-void ll_io_init(struct cl_io *io, const struct file *file, int write);
 
-void ras_update(struct ll_sb_info *sbi, struct inode *inode,
-		struct ll_readahead_state *ras, unsigned long index,
-		unsigned hit);
+enum ras_update_flags {
+	LL_RAS_HIT  = 0x1,
+	LL_RAS_MMAP = 0x2
+};
 void ll_ra_count_put(struct ll_sb_info *sbi, unsigned long len);
 void ll_ra_stats_inc(struct inode *inode, enum ra_stat which);
 
@@ -1258,15 +1217,6 @@ struct ll_dio_pages {
 	int	   ldp_nr;
 };
 
-static inline void cl_stats_tally(struct cl_device *dev, enum cl_req_type crt,
-				  int rc)
-{
-	int opc = (crt == CRT_READ) ? LPROC_LL_OSC_READ :
-				      LPROC_LL_OSC_WRITE;
-
-	ll_stats_ops_tally(ll_s2sbi(cl2vvp_dev(dev)->vdv_sb), opc, rc);
-}
-
 ssize_t ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io,
 			   int rw, struct inode *inode,
 			   struct ll_dio_pages *pv);
@@ -1365,11 +1315,6 @@ static inline void d_lustre_revalidate(struct dentry *dentry)
 	spin_unlock(&dentry->d_lock);
 }
 
-enum {
-	LL_LAYOUT_GEN_NONE  = ((__u32)-2),	/* layout lock was cancelled */
-	LL_LAYOUT_GEN_EMPTY = ((__u32)-1)	/* for empty layout */
-};
-
 int ll_layout_conf(struct inode *inode, const struct cl_object_conf *conf);
 int ll_layout_refresh(struct inode *inode, __u32 *gen);
 int ll_layout_restore(struct inode *inode, loff_t start, __u64 length);
@@ -1383,14 +1328,14 @@ int ll_page_sync_io(const struct lu_env *env, struct cl_io *io,
 int ll_getparent(struct file *file, struct getparent __user *arg);
 
 /* lcommon_cl.c */
-int cl_setattr_ost(struct inode *inode, const struct iattr *attr);
+int cl_setattr_ost(struct cl_object *obj, const struct iattr *attr,
+		   unsigned int attr_flags);
 
 extern struct lu_env *cl_inode_fini_env;
 extern int cl_inode_fini_refcheck;
 
 int cl_file_inode_init(struct inode *inode, struct lustre_md *md);
 void cl_inode_fini(struct inode *inode);
-int cl_local_size(struct inode *inode);
 
 __u64 cl_fid_build_ino(const struct lu_fid *fid, int api32);
 __u32 cl_fid_build_gen(const struct lu_fid *fid);
diff --git a/drivers/staging/lustre/lustre/llite/llite_lib.c b/drivers/staging/lustre/lustre/llite/llite_lib.c
index e5c62f4ce3d8..25f5aed97f63 100644
--- a/drivers/staging/lustre/lustre/llite/llite_lib.c
+++ b/drivers/staging/lustre/lustre/llite/llite_lib.c
@@ -191,10 +191,8 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 				  OBD_CONNECT_FLOCK_DEAD |
 				  OBD_CONNECT_DISP_STRIPE | OBD_CONNECT_LFSCK |
 				  OBD_CONNECT_OPEN_BY_FID |
-				  OBD_CONNECT_DIR_STRIPE;
-
-	if (sbi->ll_flags & LL_SBI_SOM_PREVIEW)
-		data->ocd_connect_flags |= OBD_CONNECT_SOM;
+				  OBD_CONNECT_DIR_STRIPE |
+				  OBD_CONNECT_BULK_MBITS;
 
 	if (sbi->ll_flags & LL_SBI_LRU_RESIZE)
 		data->ocd_connect_flags |= OBD_CONNECT_LRU_RESIZE;
@@ -226,6 +224,10 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 	/* real client */
 	data->ocd_connect_flags |= OBD_CONNECT_REAL;
 
+	/* always ping even if server suppress_pings */
+	if (sbi->ll_flags & LL_SBI_ALWAYS_PING)
+		data->ocd_connect_flags &= ~OBD_CONNECT_PINGLESS;
+
 	data->ocd_brw_size = MD_MAX_BRW_SIZE;
 
 	err = obd_connect(NULL, &sbi->ll_md_exp, obd, &sbi->ll_sb_uuid,
@@ -288,7 +290,7 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 
 	size = sizeof(*data);
 	err = obd_get_info(NULL, sbi->ll_md_exp, sizeof(KEY_CONN_DATA),
-			   KEY_CONN_DATA,  &size, data, NULL);
+			   KEY_CONN_DATA,  &size, data);
 	if (err) {
 		CERROR("%s: Get connect data failed: rc = %d\n",
 		       sbi->ll_md_exp->exp_obd->obd_name, err);
@@ -355,10 +357,9 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 				  OBD_CONNECT_64BITHASH | OBD_CONNECT_MAXBYTES |
 				  OBD_CONNECT_EINPROGRESS |
 				  OBD_CONNECT_JOBSTATS | OBD_CONNECT_LVB_TYPE |
-				  OBD_CONNECT_LAYOUTLOCK | OBD_CONNECT_PINGLESS;
-
-	if (sbi->ll_flags & LL_SBI_SOM_PREVIEW)
-		data->ocd_connect_flags |= OBD_CONNECT_SOM;
+				  OBD_CONNECT_LAYOUTLOCK |
+				  OBD_CONNECT_PINGLESS | OBD_CONNECT_LFSCK |
+				  OBD_CONNECT_BULK_MBITS;
 
 	if (!OBD_FAIL_CHECK(OBD_FAIL_OSC_CONNECT_CKSUM)) {
 		/* OBD_CONNECT_CKSUM should always be set, even if checksums are
@@ -376,6 +377,10 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 
 	data->ocd_connect_flags |= OBD_CONNECT_LRU_RESIZE;
 
+	/* always ping even if server suppress_pings */
+	if (sbi->ll_flags & LL_SBI_ALWAYS_PING)
+		data->ocd_connect_flags &= ~OBD_CONNECT_PINGLESS;
+
 	CDEBUG(D_RPCTRACE, "ocd_connect_flags: %#llx ocd_version: %d ocd_grant: %d\n",
 	       data->ocd_connect_flags,
 	       data->ocd_version, data->ocd_grant);
@@ -475,8 +480,6 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 	ptlrpc_req_finished(request);
 
 	if (IS_ERR(root)) {
-		if (lmd.lsm)
-			obd_free_memmd(sbi->ll_dt_exp, &lmd.lsm);
 #ifdef CONFIG_FS_POSIX_ACL
 		if (lmd.posix_acl) {
 			posix_acl_release(lmd.posix_acl);
@@ -488,12 +491,6 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 		goto out_root;
 	}
 
-	err = ll_close_thread_start(&sbi->ll_lcq);
-	if (err) {
-		CERROR("cannot start close thread: rc %d\n", err);
-		goto out_root;
-	}
-
 	checksum = sbi->ll_flags & LL_SBI_CHECKSUM;
 	err = obd_set_info_async(NULL, sbi->ll_dt_exp, sizeof(KEY_CHECKSUM),
 				 KEY_CHECKSUM, sizeof(checksum), &checksum,
@@ -572,10 +569,18 @@ int ll_get_max_mdsize(struct ll_sb_info *sbi, int *lmmsize)
 {
 	int size, rc;
 
-	*lmmsize = obd_size_diskmd(sbi->ll_dt_exp, NULL);
+	size = sizeof(*lmmsize);
+	rc = obd_get_info(NULL, sbi->ll_dt_exp, sizeof(KEY_MAX_EASIZE),
+			  KEY_MAX_EASIZE, &size, lmmsize);
+	if (rc) {
+		CERROR("%s: cannot get max LOV EA size: rc = %d\n",
+		       sbi->ll_dt_exp->exp_obd->obd_name, rc);
+		return rc;
+	}
+
 	size = sizeof(int);
 	rc = obd_get_info(NULL, sbi->ll_md_exp, sizeof(KEY_MAX_EASIZE),
-			  KEY_MAX_EASIZE, &size, lmmsize, NULL);
+			  KEY_MAX_EASIZE, &size, lmmsize);
 	if (rc)
 		CERROR("Get max mdsize error rc %d\n", rc);
 
@@ -599,7 +604,7 @@ int ll_get_default_mdsize(struct ll_sb_info *sbi, int *lmmsize)
 
 	size = sizeof(int);
 	rc = obd_get_info(NULL, sbi->ll_md_exp, sizeof(KEY_DEFAULT_EASIZE),
-			  KEY_DEFAULT_EASIZE, &size, lmmsize, NULL);
+			  KEY_DEFAULT_EASIZE, &size, lmmsize);
 	if (rc)
 		CERROR("Get default mdsize error rc %d\n", rc);
 
@@ -633,8 +638,6 @@ static void client_common_put_super(struct super_block *sb)
 {
 	struct ll_sb_info *sbi = ll_s2sbi(sb);
 
-	ll_close_thread_shutdown(sbi->ll_lcq);
-
 	cl_sb_fini(sb);
 
 	obd_fid_fini(sbi->ll_dt_exp->exp_obd);
@@ -725,6 +728,18 @@ static int ll_options(char *options, int *flags)
 			*flags &= ~tmp;
 			goto next;
 		}
+		tmp = ll_set_opt("context", s1, 1);
+		if (tmp)
+			goto next;
+		tmp = ll_set_opt("fscontext", s1, 1);
+		if (tmp)
+			goto next;
+		tmp = ll_set_opt("defcontext", s1, 1);
+		if (tmp)
+			goto next;
+		tmp = ll_set_opt("rootcontext", s1, 1);
+		if (tmp)
+			goto next;
 		tmp = ll_set_opt("user_fid2path", s1, LL_SBI_USER_FID2PATH);
 		if (tmp) {
 			*flags |= tmp;
@@ -766,11 +781,6 @@ static int ll_options(char *options, int *flags)
 			*flags &= ~tmp;
 			goto next;
 		}
-		tmp = ll_set_opt("som_preview", s1, LL_SBI_SOM_PREVIEW);
-		if (tmp) {
-			*flags |= tmp;
-			goto next;
-		}
 		tmp = ll_set_opt("32bitapi", s1, LL_SBI_32BIT_API);
 		if (tmp) {
 			*flags |= tmp;
@@ -786,6 +796,11 @@ static int ll_options(char *options, int *flags)
 			*flags &= ~tmp;
 			goto next;
 		}
+		tmp = ll_set_opt("always_ping", s1, LL_SBI_ALWAYS_PING);
+		if (tmp) {
+			*flags |= tmp;
+			goto next;
+		}
 		LCONSOLE_ERROR_MSG(0x152, "Unknown option '%s', won't mount.\n",
 				   s1);
 		return -EINVAL;
@@ -804,14 +819,10 @@ void ll_lli_init(struct ll_inode_info *lli)
 {
 	lli->lli_inode_magic = LLI_INODE_MAGIC;
 	lli->lli_flags = 0;
-	lli->lli_ioepoch = 0;
-	lli->lli_maxbytes = MAX_LFS_FILESIZE;
 	spin_lock_init(&lli->lli_lock);
 	lli->lli_posix_acl = NULL;
 	/* Do not set lli_fid, it has been initialized already. */
 	fid_zero(&lli->lli_pfid);
-	INIT_LIST_HEAD(&lli->lli_close_list);
-	lli->lli_pending_och = NULL;
 	lli->lli_mds_read_och = NULL;
 	lli->lli_mds_write_och = NULL;
 	lli->lli_mds_exec_och = NULL;
@@ -820,9 +831,8 @@ void ll_lli_init(struct ll_inode_info *lli)
 	lli->lli_open_fd_exec_count = 0;
 	mutex_init(&lli->lli_och_mutex);
 	spin_lock_init(&lli->lli_agl_lock);
-	lli->lli_has_smd = false;
 	spin_lock_init(&lli->lli_layout_lock);
-	ll_layout_version_set(lli, LL_LAYOUT_GEN_NONE);
+	ll_layout_version_set(lli, CL_LAYOUT_GEN_NONE);
 	lli->lli_clob = NULL;
 
 	init_rwsem(&lli->lli_xattrs_list_rwsem);
@@ -941,10 +951,14 @@ int ll_fill_super(struct super_block *sb, struct vfsmount *mnt)
 
 	/* connections, registrations, sb setup */
 	err = client_common_fill_super(sb, md, dt, mnt);
+	if (!err)
+		sbi->ll_client_common_fill_super_succeeded = 1;
 
 out_free:
 	kfree(md);
 	kfree(dt);
+	if (lprof)
+		class_put_profile(lprof);
 	if (err)
 		ll_put_super(sb);
 	else if (sbi->ll_flags & LL_SBI_VERBOSE)
@@ -1002,7 +1016,7 @@ void ll_put_super(struct super_block *sb)
 		}
 	}
 
-	if (sbi->ll_lcq) {
+	if (sbi->ll_client_common_fill_super_succeeded) {
 		/* Only if client_common_fill_super succeeded */
 		client_common_put_super(sb);
 	}
@@ -1057,7 +1071,7 @@ struct inode *ll_inode_from_resource_lock(struct ldlm_lock *lock)
 	return inode;
 }
 
-static void ll_dir_clear_lsm_md(struct inode *inode)
+void ll_dir_clear_lsm_md(struct inode *inode)
 {
 	struct ll_inode_info *lli = ll_i2info(inode);
 
@@ -1205,16 +1219,44 @@ static int ll_update_lsm_md(struct inode *inode, struct lustre_md *md)
 
 	/* set the directory layout */
 	if (!lli->lli_lsm_md) {
+		struct cl_attr *attr;
+
 		rc = ll_init_lsm_md(inode, md);
 		if (rc)
 			return rc;
 
-		lli->lli_lsm_md = lsm;
 		/*
 		 * set lsm_md to NULL, so the following free lustre_md
 		 * will not free this lsm
 		 */
 		md->lmv = NULL;
+		lli->lli_lsm_md = lsm;
+
+		attr = kzalloc(sizeof(*attr), GFP_NOFS);
+		if (!attr)
+			return -ENOMEM;
+
+		/* validate the lsm */
+		rc = md_merge_attr(ll_i2mdexp(inode), lsm, attr,
+				   ll_md_blocking_ast);
+		if (rc) {
+			kfree(attr);
+			return rc;
+		}
+
+		if (md->body->mbo_valid & OBD_MD_FLNLINK)
+			md->body->mbo_nlink = attr->cat_nlink;
+		if (md->body->mbo_valid & OBD_MD_FLSIZE)
+			md->body->mbo_size = attr->cat_size;
+		if (md->body->mbo_valid & OBD_MD_FLATIME)
+			md->body->mbo_atime = attr->cat_atime;
+		if (md->body->mbo_valid & OBD_MD_FLCTIME)
+			md->body->mbo_ctime = attr->cat_ctime;
+		if (md->body->mbo_valid & OBD_MD_FLMTIME)
+			md->body->mbo_mtime = attr->cat_mtime;
+
+		kfree(attr);
+
 		CDEBUG(D_INODE, "Set lsm %p magic %x to "DFID"\n", lsm,
 		       lsm->lsm_md_magic, PFID(ll_inode2fid(inode)));
 		return 0;
@@ -1272,9 +1314,6 @@ void ll_clear_inode(struct inode *inode)
 		LASSERT(lli->lli_opendir_pid == 0);
 	}
 
-	spin_lock(&lli->lli_lock);
-	ll_i2info(inode)->lli_flags &= ~LLIF_MDS_SIZE_LOCK;
-	spin_unlock(&lli->lli_lock);
 	md_null_inode(sbi->ll_md_exp, ll_inode2fid(inode));
 
 	LASSERT(!lli->lli_open_fd_write_count);
@@ -1313,13 +1352,11 @@ void ll_clear_inode(struct inode *inode)
 	 * cl_object still uses inode lsm.
 	 */
 	cl_inode_fini(inode);
-	lli->lli_has_smd = false;
 }
 
 #define TIMES_SET_FLAGS (ATTR_MTIME_SET | ATTR_ATIME_SET | ATTR_TIMES_SET)
 
-static int ll_md_setattr(struct dentry *dentry, struct md_op_data *op_data,
-			 struct md_open_data **mod)
+static int ll_md_setattr(struct dentry *dentry, struct md_op_data *op_data)
 {
 	struct lustre_md md;
 	struct inode *inode = d_inode(dentry);
@@ -1332,8 +1369,7 @@ static int ll_md_setattr(struct dentry *dentry, struct md_op_data *op_data,
 	if (IS_ERR(op_data))
 		return PTR_ERR(op_data);
 
-	rc = md_setattr(sbi->ll_md_exp, op_data, NULL, 0, NULL, 0,
-			&request, mod);
+	rc = md_setattr(sbi->ll_md_exp, op_data, NULL, 0, &request);
 	if (rc) {
 		ptlrpc_req_finished(request);
 		if (rc == -ENOENT) {
@@ -1369,48 +1405,12 @@ static int ll_md_setattr(struct dentry *dentry, struct md_op_data *op_data,
 	rc = simple_setattr(dentry, &op_data->op_attr);
 	op_data->op_attr.ia_valid = ia_valid;
 
-	/* Extract epoch data if obtained. */
-	op_data->op_handle = md.body->mbo_handle;
-	op_data->op_ioepoch = md.body->mbo_ioepoch;
-
 	rc = ll_update_inode(inode, &md);
 	ptlrpc_req_finished(request);
 
 	return rc;
 }
 
-/* Close IO epoch and send Size-on-MDS attribute update. */
-static int ll_setattr_done_writing(struct inode *inode,
-				   struct md_op_data *op_data,
-				   struct md_open_data *mod)
-{
-	struct ll_inode_info *lli = ll_i2info(inode);
-	int rc = 0;
-
-	if (!S_ISREG(inode->i_mode))
-		return 0;
-
-	CDEBUG(D_INODE, "Epoch %llu closed on "DFID" for truncate\n",
-	       op_data->op_ioepoch, PFID(&lli->lli_fid));
-
-	op_data->op_flags = MF_EPOCH_CLOSE;
-	ll_done_writing_attr(inode, op_data);
-	ll_pack_inode2opdata(inode, op_data, NULL);
-
-	rc = md_done_writing(ll_i2sbi(inode)->ll_md_exp, op_data, mod);
-	if (rc == -EAGAIN)
-		/* MDS has instructed us to obtain Size-on-MDS attribute
-		 * from OSTs and send setattr to back to MDS.
-		 */
-		rc = ll_som_update(inode, op_data);
-	else if (rc) {
-		CERROR("%s: inode "DFID" mdc truncate failed: rc = %d\n",
-		       ll_i2sbi(inode)->ll_md_exp->exp_obd->obd_name,
-		       PFID(ll_inode2fid(inode)), rc);
-	}
-	return rc;
-}
-
 /* If this inode has objects allocated to it (lsm != NULL), then the OST
  * object(s) determine the file size and mtime.  Otherwise, the MDS will
  * keep these values until such a time that objects are allocated for it.
@@ -1431,9 +1431,8 @@ int ll_setattr_raw(struct dentry *dentry, struct iattr *attr, bool hsm_import)
 	struct inode *inode = d_inode(dentry);
 	struct ll_inode_info *lli = ll_i2info(inode);
 	struct md_op_data *op_data = NULL;
-	struct md_open_data *mod = NULL;
 	bool file_is_released = false;
-	int rc = 0, rc1 = 0;
+	int rc = 0;
 
 	CDEBUG(D_VFSTRACE, "%s: setattr inode "DFID"(%p) from %llu to %llu, valid %x, hsm_import %d\n",
 	       ll_get_fsname(inode->i_sb, NULL, 0), PFID(&lli->lli_fid), inode,
@@ -1503,14 +1502,33 @@ int ll_setattr_raw(struct dentry *dentry, struct iattr *attr, bool hsm_import)
 	 * but other attributes must be set
 	 */
 	if (S_ISREG(inode->i_mode)) {
-		struct lov_stripe_md *lsm;
+		struct cl_layout cl = {
+			.cl_is_released = false,
+		};
+		struct lu_env *env;
+		int refcheck;
 		__u32 gen;
 
-		ll_layout_refresh(inode, &gen);
-		lsm = ccc_inode_lsm_get(inode);
-		if (lsm && lsm->lsm_pattern & LOV_PATTERN_F_RELEASED)
-			file_is_released = true;
-		ccc_inode_lsm_put(inode, lsm);
+		rc = ll_layout_refresh(inode, &gen);
+		if (rc < 0)
+			goto out;
+
+		/*
+		 * XXX: the only place we need to know the layout type,
+		 * this will be removed by a later patch. -Jinshan
+		 */
+		env = cl_env_get(&refcheck);
+		if (IS_ERR(env)) {
+			rc = PTR_ERR(env);
+			goto out;
+		}
+
+		rc = cl_object_layout_get(env, lli->lli_clob, &cl);
+		cl_env_put(env, &refcheck);
+		if (rc < 0)
+			goto out;
+
+		file_is_released = cl.cl_is_released;
 
 		if (!hsm_import && attr->ia_valid & ATTR_SIZE) {
 			if (file_is_released) {
@@ -1527,32 +1545,16 @@ int ll_setattr_raw(struct dentry *dentry, struct iattr *attr, bool hsm_import)
 			 * modified, flag it.
 			 */
 			attr->ia_valid |= MDS_OPEN_OWNEROVERRIDE;
-			spin_lock(&lli->lli_lock);
-			lli->lli_flags |= LLIF_DATA_MODIFIED;
-			spin_unlock(&lli->lli_lock);
 			op_data->op_bias |= MDS_DATA_MODIFIED;
 		}
 	}
 
 	memcpy(&op_data->op_attr, attr, sizeof(*attr));
 
-	/* Open epoch for truncate. */
-	if (exp_connect_som(ll_i2mdexp(inode)) && !hsm_import &&
-	    (attr->ia_valid & (ATTR_SIZE | ATTR_MTIME | ATTR_MTIME_SET)))
-		op_data->op_flags = MF_EPOCH_OPEN;
-
-	rc = ll_md_setattr(dentry, op_data, &mod);
+	rc = ll_md_setattr(dentry, op_data);
 	if (rc)
 		goto out;
 
-	/* RPC to MDT is sent, cancel data modification flag */
-	if (op_data->op_bias & MDS_DATA_MODIFIED) {
-		spin_lock(&lli->lli_lock);
-		lli->lli_flags &= ~LLIF_DATA_MODIFIED;
-		spin_unlock(&lli->lli_lock);
-	}
-
-	ll_ioepoch_open(lli, op_data->op_ioepoch);
 	if (!S_ISREG(inode->i_mode) || file_is_released) {
 		rc = 0;
 		goto out;
@@ -1568,19 +1570,11 @@ int ll_setattr_raw(struct dentry *dentry, struct iattr *attr, bool hsm_import)
 		 * setting times to past, but it is necessary due to possible
 		 * time de-synchronization between MDT inode and OST objects
 		 */
-		if (attr->ia_valid & ATTR_SIZE)
-			down_write(&lli->lli_trunc_sem);
-		rc = cl_setattr_ost(inode, attr);
-		if (attr->ia_valid & ATTR_SIZE)
-			up_write(&lli->lli_trunc_sem);
+		rc = cl_setattr_ost(ll_i2info(inode)->lli_clob, attr, 0);
 	}
 out:
-	if (op_data->op_ioepoch) {
-		rc1 = ll_setattr_done_writing(inode, op_data, mod);
-		if (!rc)
-			rc = rc1;
-	}
-	ll_finish_md_op_data(op_data);
+	if (op_data)
+		ll_finish_md_op_data(op_data);
 
 	if (!S_ISDIR(inode->i_mode)) {
 		inode_lock(inode);
@@ -1736,19 +1730,10 @@ int ll_update_inode(struct inode *inode, struct lustre_md *md)
 {
 	struct ll_inode_info *lli = ll_i2info(inode);
 	struct mdt_body *body = md->body;
-	struct lov_stripe_md *lsm = md->lsm;
 	struct ll_sb_info *sbi = ll_i2sbi(inode);
 
-	LASSERT((lsm != NULL) == ((body->mbo_valid & OBD_MD_FLEASIZE) != 0));
-	if (lsm) {
-		if (!lli->lli_has_smd &&
-		    !(sbi->ll_flags & LL_SBI_LAYOUT_LOCK))
-			cl_file_inode_init(inode, md);
-
-		lli->lli_maxbytes = lsm->lsm_maxbytes;
-		if (lli->lli_maxbytes > MAX_LFS_FILESIZE)
-			lli->lli_maxbytes = MAX_LFS_FILESIZE;
-	}
+	if (body->mbo_valid & OBD_MD_FLEASIZE)
+		cl_file_inode_init(inode, md);
 
 	if (S_ISDIR(inode->i_mode)) {
 		int rc;
@@ -1828,48 +1813,11 @@ int ll_update_inode(struct inode *inode, struct lustre_md *md)
 	LASSERT(fid_seq(&lli->lli_fid) != 0);
 
 	if (body->mbo_valid & OBD_MD_FLSIZE) {
-		if (exp_connect_som(ll_i2mdexp(inode)) &&
-		    S_ISREG(inode->i_mode)) {
-			struct lustre_handle lockh;
-			enum ldlm_mode mode;
-
-			/* As it is possible a blocking ast has been processed
-			 * by this time, we need to check there is an UPDATE
-			 * lock on the client and set LLIF_MDS_SIZE_LOCK holding
-			 * it.
-			 */
-			mode = ll_take_md_lock(inode, MDS_INODELOCK_UPDATE,
-					       &lockh, LDLM_FL_CBPENDING,
-					       LCK_CR | LCK_CW |
-					       LCK_PR | LCK_PW);
-			if (mode) {
-				if (lli->lli_flags & (LLIF_DONE_WRITING |
-						      LLIF_EPOCH_PENDING |
-						      LLIF_SOM_DIRTY)) {
-					CERROR("%s: inode "DFID" flags %u still has size authority! do not trust the size got from MDS\n",
-					       sbi->ll_md_exp->exp_obd->obd_name,
-					       PFID(ll_inode2fid(inode)),
-					       lli->lli_flags);
-				} else {
-					/* Use old size assignment to avoid
-					 * deadlock bz14138 & bz14326
-					 */
-					i_size_write(inode, body->mbo_size);
-					spin_lock(&lli->lli_lock);
-					lli->lli_flags |= LLIF_MDS_SIZE_LOCK;
-					spin_unlock(&lli->lli_lock);
-				}
-				ldlm_lock_decref(&lockh, mode);
-			}
-		} else {
-			/* Use old size assignment to avoid
-			 * deadlock bz14138 & bz14326
-			 */
-			i_size_write(inode, body->mbo_size);
+		i_size_write(inode, body->mbo_size);
 
-			CDEBUG(D_VFSTRACE, "inode=%lu, updating i_size %llu\n",
-			       inode->i_ino, (unsigned long long)body->mbo_size);
-		}
+		CDEBUG(D_VFSTRACE, "inode=" DFID ", updating i_size %llu\n",
+		       PFID(ll_inode2fid(inode)),
+		       (unsigned long long)body->mbo_size);
 
 		if (body->mbo_valid & OBD_MD_FLBLOCKS)
 			inode->i_blocks = body->mbo_blocks;
@@ -1877,7 +1825,7 @@ int ll_update_inode(struct inode *inode, struct lustre_md *md)
 
 	if (body->mbo_valid & OBD_MD_TSTATE) {
 		if (body->mbo_t_state & MS_RESTORE)
-			lli->lli_flags |= LLIF_FILE_RESTORING;
+			set_bit(LLIF_FILE_RESTORING, &lli->lli_flags);
 	}
 
 	return 0;
@@ -1892,8 +1840,6 @@ int ll_read_inode2(struct inode *inode, void *opaque)
 	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p)\n",
 	       PFID(&lli->lli_fid), inode);
 
-	LASSERT(!lli->lli_has_smd);
-
 	/* Core attributes from the MDS first.  This is a new inode, and
 	 * the VFS doesn't zero times in the core inode so we have to do
 	 * it ourselves.  They will be overwritten by either MDS or OST
@@ -1988,9 +1934,9 @@ int ll_iocontrol(struct inode *inode, struct file *file,
 		return put_user(flags, (int __user *)arg);
 	}
 	case FSFILT_IOC_SETFLAGS: {
-		struct lov_stripe_md *lsm;
-		struct obd_info oinfo = { };
 		struct md_op_data *op_data;
+		struct cl_object *obj;
+		struct iattr *attr;
 
 		if (get_user(flags, (int __user *)arg))
 			return -EFAULT;
@@ -2002,8 +1948,7 @@ int ll_iocontrol(struct inode *inode, struct file *file,
 
 		op_data->op_attr_flags = flags;
 		op_data->op_attr.ia_valid |= ATTR_ATTR_FLAG;
-		rc = md_setattr(sbi->ll_md_exp, op_data,
-				NULL, 0, NULL, 0, &req, NULL);
+		rc = md_setattr(sbi->ll_md_exp, op_data, NULL, 0, &req);
 		ll_finish_md_op_data(op_data);
 		ptlrpc_req_finished(req);
 		if (rc)
@@ -2011,30 +1956,17 @@ int ll_iocontrol(struct inode *inode, struct file *file,
 
 		inode->i_flags = ll_ext_to_inode_flags(flags);
 
-		lsm = ccc_inode_lsm_get(inode);
-		if (!lsm_has_objects(lsm)) {
-			ccc_inode_lsm_put(inode, lsm);
+		obj = ll_i2info(inode)->lli_clob;
+		if (!obj)
 			return 0;
-		}
 
-		oinfo.oi_oa = kmem_cache_zalloc(obdo_cachep, GFP_NOFS);
-		if (!oinfo.oi_oa) {
-			ccc_inode_lsm_put(inode, lsm);
+		attr = kzalloc(sizeof(*attr), GFP_NOFS);
+		if (!attr)
 			return -ENOMEM;
-		}
-		oinfo.oi_md = lsm;
-		oinfo.oi_oa->o_oi = lsm->lsm_oi;
-		oinfo.oi_oa->o_flags = flags;
-		oinfo.oi_oa->o_valid = OBD_MD_FLID | OBD_MD_FLFLAGS |
-				       OBD_MD_FLGROUP;
-		obdo_set_parent_fid(oinfo.oi_oa, &ll_i2info(inode)->lli_fid);
-		rc = obd_setattr_rqset(sbi->ll_dt_exp, &oinfo, NULL);
-		kmem_cache_free(obdo_cachep, oinfo.oi_oa);
-		ccc_inode_lsm_put(inode, lsm);
-
-		if (rc && rc != -EPERM && rc != -EACCES)
-			CERROR("osc_setattr_async fails: rc = %d\n", rc);
 
+		attr->ia_valid = ATTR_ATTR_FLAG;
+		rc = cl_setattr_ost(obj, attr, flags);
+		kfree(attr);
 		return rc;
 	}
 	default:
@@ -2164,7 +2096,6 @@ void ll_open_cleanup(struct super_block *sb, struct ptlrpc_request *open_req)
 		return;
 
 	op_data->op_fid1 = body->mbo_fid1;
-	op_data->op_ioepoch = body->mbo_ioepoch;
 	op_data->op_handle = body->mbo_handle;
 	op_data->op_mod_time = get_seconds();
 	md_close(exp, op_data, NULL, &close_req);
@@ -2244,17 +2175,14 @@ int ll_prep_inode(struct inode **inode, struct ptlrpc_request *req,
 			conf.coc_opc = OBJECT_CONF_SET;
 			conf.coc_inode = *inode;
 			conf.coc_lock = lock;
-			conf.u.coc_md = &md;
+			conf.u.coc_layout = md.layout;
 			(void)ll_layout_conf(*inode, &conf);
 		}
 		LDLM_LOCK_PUT(lock);
 	}
 
 out:
-	if (md.lsm)
-		obd_free_memmd(sbi->ll_dt_exp, &md.lsm);
 	md_free_lustre_md(sbi->ll_md_exp, &md);
-
 cleanup:
 	if (rc != 0 && it && it->it_op & IT_OPEN)
 		ll_open_cleanup(sb ? sb : (*inode)->i_sb, req);
@@ -2380,8 +2308,9 @@ struct md_op_data *ll_prep_md_op_data(struct md_op_data *op_data,
 	op_data->op_default_stripe_offset = -1;
 	if (S_ISDIR(i1->i_mode)) {
 		op_data->op_mea1 = ll_i2info(i1)->lli_lsm_md;
-		op_data->op_default_stripe_offset =
-			ll_i2info(i1)->lli_def_stripe_offset;
+		if (opc == LUSTRE_OPC_MKDIR)
+			op_data->op_default_stripe_offset =
+				ll_i2info(i1)->lli_def_stripe_offset;
 	}
 
 	if (i2) {
@@ -2405,8 +2334,6 @@ struct md_op_data *ll_prep_md_op_data(struct md_op_data *op_data,
 	op_data->op_fsuid = from_kuid(&init_user_ns, current_fsuid());
 	op_data->op_fsgid = from_kgid(&init_user_ns, current_fsgid());
 	op_data->op_cap = cfs_curproc_cap_pack();
-	op_data->op_bias = 0;
-	op_data->op_cli_flags = 0;
 	if ((opc == LUSTRE_OPC_CREATE) && name &&
 	    filename_is_volatile(name, namelen, &op_data->op_mds))
 		op_data->op_bias |= MDS_CREATE_VOLATILE;
@@ -2414,10 +2341,6 @@ struct md_op_data *ll_prep_md_op_data(struct md_op_data *op_data,
 		op_data->op_mds = 0;
 	op_data->op_data = data;
 
-	/* When called by ll_setattr_raw, file is i1. */
-	if (ll_i2info(i1)->lli_flags & LLIF_DATA_MODIFIED)
-		op_data->op_bias |= MDS_DATA_MODIFIED;
-
 	return op_data;
 }
 
@@ -2451,6 +2374,9 @@ int ll_show_options(struct seq_file *seq, struct dentry *dentry)
 	if (sbi->ll_flags & LL_SBI_USER_FID2PATH)
 		seq_puts(seq, ",user_fid2path");
 
+	if (sbi->ll_flags & LL_SBI_ALWAYS_PING)
+		seq_puts(seq, ",always_ping");
+
 	return 0;
 }
 
diff --git a/drivers/staging/lustre/lustre/llite/llite_mmap.c b/drivers/staging/lustre/lustre/llite/llite_mmap.c
index 436691814a5e..ee01f20d8b11 100644
--- a/drivers/staging/lustre/lustre/llite/llite_mmap.c
+++ b/drivers/staging/lustre/lustre/llite/llite_mmap.c
@@ -47,7 +47,7 @@
 
 static const struct vm_operations_struct ll_file_vm_ops;
 
-void policy_from_vma(ldlm_policy_data_t *policy,
+void policy_from_vma(union ldlm_policy_data *policy,
 		     struct vm_area_struct *vma, unsigned long addr,
 		     size_t count)
 {
@@ -80,43 +80,24 @@ struct vm_area_struct *our_vma(struct mm_struct *mm, unsigned long addr,
  * API independent part for page fault initialization.
  * \param vma - virtual memory area addressed to page fault
  * \param env - corespondent lu_env to processing
- * \param nest - nested level
  * \param index - page index corespondent to fault.
  * \parm ra_flags - vma readahead flags.
  *
- * \return allocated and initialized env for fault operation.
- * \retval EINVAL if env can't allocated
- * \return other error codes from cl_io_init.
+ * \return error codes from cl_io_init.
  */
 static struct cl_io *
-ll_fault_io_init(struct vm_area_struct *vma, struct lu_env **env_ret,
-		 struct cl_env_nest *nest, pgoff_t index,
-		 unsigned long *ra_flags)
+ll_fault_io_init(struct lu_env *env, struct vm_area_struct *vma,
+		 pgoff_t index, unsigned long *ra_flags)
 {
 	struct file	       *file = vma->vm_file;
 	struct inode	       *inode = file_inode(file);
 	struct cl_io	       *io;
 	struct cl_fault_io     *fio;
-	struct lu_env	       *env;
 	int			rc;
 
-	*env_ret = NULL;
 	if (ll_file_nolock(file))
 		return ERR_PTR(-EOPNOTSUPP);
 
-	/*
-	 * page fault can be called when lustre IO is
-	 * already active for the current thread, e.g., when doing read/write
-	 * against user level buffer mapped from Lustre buffer. To avoid
-	 * stomping on existing context, optionally force an allocation of a new
-	 * one.
-	 */
-	env = cl_env_nested_get(nest);
-	if (IS_ERR(env))
-		return ERR_PTR(-EINVAL);
-
-	*env_ret = env;
-
 restart:
 	io = vvp_env_thread_io(env);
 	io->ci_obj = ll_i2info(inode)->lli_clob;
@@ -155,7 +136,6 @@ restart:
 		if (io->ci_need_restart)
 			goto restart;
 
-		cl_env_nested_put(nest, env);
 		io = ERR_PTR(rc);
 	}
 
@@ -169,13 +149,17 @@ static int ll_page_mkwrite0(struct vm_area_struct *vma, struct page *vmpage,
 	struct lu_env	   *env;
 	struct cl_io	    *io;
 	struct vvp_io	   *vio;
-	struct cl_env_nest       nest;
 	int		      result;
+	int refcheck;
 	sigset_t	     set;
 	struct inode	     *inode;
 	struct ll_inode_info     *lli;
 
-	io = ll_fault_io_init(vma, &env,  &nest, vmpage->index, NULL);
+	env = cl_env_get(&refcheck);
+	if (IS_ERR(env))
+		return PTR_ERR(env);
+
+	io = ll_fault_io_init(env, vma, vmpage->index, NULL);
 	if (IS_ERR(io)) {
 		result = PTR_ERR(io);
 		goto out;
@@ -231,17 +215,14 @@ static int ll_page_mkwrite0(struct vm_area_struct *vma, struct page *vmpage,
 			result = -EAGAIN;
 		}
 
-		if (result == 0) {
-			spin_lock(&lli->lli_lock);
-			lli->lli_flags |= LLIF_DATA_MODIFIED;
-			spin_unlock(&lli->lli_lock);
-		}
+		if (!result)
+			set_bit(LLIF_DATA_MODIFIED, &lli->lli_flags);
 	}
 
 out_io:
 	cl_io_fini(env, io);
-	cl_env_nested_put(&nest, env);
 out:
+	cl_env_put(env, &refcheck);
 	CDEBUG(D_MMAP, "%s mkwrite with %d\n", current->comm, result);
 	LASSERT(ergo(result == 0, PageLocked(vmpage)));
 
@@ -285,13 +266,19 @@ static int ll_fault0(struct vm_area_struct *vma, struct vm_fault *vmf)
 	struct vvp_io	   *vio = NULL;
 	struct page	     *vmpage;
 	unsigned long	    ra_flags;
-	struct cl_env_nest       nest;
-	int		      result;
+	int		      result = 0;
 	int		      fault_ret = 0;
+	int refcheck;
+
+	env = cl_env_get(&refcheck);
+	if (IS_ERR(env))
+		return PTR_ERR(env);
 
-	io = ll_fault_io_init(vma, &env,  &nest, vmf->pgoff, &ra_flags);
-	if (IS_ERR(io))
-		return to_fault_error(PTR_ERR(io));
+	io = ll_fault_io_init(env, vma, vmf->pgoff, &ra_flags);
+	if (IS_ERR(io)) {
+		result = to_fault_error(PTR_ERR(io));
+		goto out;
+	}
 
 	result = io->ci_result;
 	if (result == 0) {
@@ -322,14 +309,15 @@ static int ll_fault0(struct vm_area_struct *vma, struct vm_fault *vmf)
 		}
 	}
 	cl_io_fini(env, io);
-	cl_env_nested_put(&nest, env);
 
 	vma->vm_flags |= ra_flags;
+
+out:
+	cl_env_put(env, &refcheck);
 	if (result != 0 && !(fault_ret & VM_FAULT_RETRY))
 		fault_ret |= to_fault_error(result);
 
-	CDEBUG(D_MMAP, "%s fault %d/%d\n",
-	       current->comm, fault_ret, result);
+	CDEBUG(D_MMAP, "%s fault %d/%d\n", current->comm, fault_ret, result);
 	return fault_ret;
 }
 
@@ -381,6 +369,7 @@ static int ll_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
 	bool retry;
 	int result;
 
+	file_update_time(vma->vm_file);
 	do {
 		retry = false;
 		result = ll_page_mkwrite0(vma, vmf->page, &retry);
diff --git a/drivers/staging/lustre/lustre/llite/llite_nfs.c b/drivers/staging/lustre/lustre/llite/llite_nfs.c
index 709230571b4b..c63236580b0f 100644
--- a/drivers/staging/lustre/lustre/llite/llite_nfs.c
+++ b/drivers/staging/lustre/lustre/llite/llite_nfs.c
@@ -226,7 +226,7 @@ static int ll_encode_fh(struct inode *inode, __u32 *fh, int *plen,
 
 static int ll_nfs_get_name_filldir(struct dir_context *ctx, const char *name,
 				   int namelen, loff_t hash, u64 ino,
-				   unsigned type)
+				   unsigned int type)
 {
 	/* It is hack to access lde_fid for comparison with lgd_fid.
 	 * So the input 'name' must be part of the 'lu_dirent'.
diff --git a/drivers/staging/lustre/lustre/llite/lproc_llite.c b/drivers/staging/lustre/lustre/llite/lproc_llite.c
index 23fda9d98bff..03682c10fc9e 100644
--- a/drivers/staging/lustre/lustre/llite/lproc_llite.c
+++ b/drivers/staging/lustre/lustre/llite/lproc_llite.c
@@ -1060,10 +1060,6 @@ static const struct llite_file_opcode {
 				   "brw_read" },
 	{ LPROC_LL_BRW_WRITE,      LPROCFS_CNTR_AVGMINMAX | LPROCFS_TYPE_PAGES,
 				   "brw_write" },
-	{ LPROC_LL_OSC_READ,       LPROCFS_CNTR_AVGMINMAX | LPROCFS_TYPE_BYTES,
-				   "osc_read" },
-	{ LPROC_LL_OSC_WRITE,      LPROCFS_CNTR_AVGMINMAX | LPROCFS_TYPE_BYTES,
-				   "osc_write" },
 	{ LPROC_LL_IOCTL,	  LPROCFS_TYPE_REGS, "ioctl" },
 	{ LPROC_LL_OPEN,	   LPROCFS_TYPE_REGS, "open" },
 	{ LPROC_LL_RELEASE,	LPROCFS_TYPE_REGS, "close" },
diff --git a/drivers/staging/lustre/lustre/llite/namei.c b/drivers/staging/lustre/lustre/llite/namei.c
index 180f35e3afd9..9426759aedc9 100644
--- a/drivers/staging/lustre/lustre/llite/namei.c
+++ b/drivers/staging/lustre/lustre/llite/namei.c
@@ -113,13 +113,18 @@ struct inode *ll_iget(struct super_block *sb, ino_t hash,
 	if (inode->i_state & I_NEW) {
 		rc = ll_read_inode2(inode, md);
 		if (!rc && S_ISREG(inode->i_mode) &&
-		    !ll_i2info(inode)->lli_clob) {
-			CDEBUG(D_INODE, "%s: apply lsm %p to inode "DFID"\n",
-			       ll_get_fsname(sb, NULL, 0), md->lsm,
-			       PFID(ll_inode2fid(inode)));
+		    !ll_i2info(inode)->lli_clob)
 			rc = cl_file_inode_init(inode, md);
-		}
+
 		if (rc) {
+			/*
+			 * Let's clear directory lsm here, otherwise
+			 * make_bad_inode() will reset the inode mode
+			 * to regular, then ll_clear_inode will not
+			 * be able to clear lsm_md
+			 */
+			if (S_ISDIR(inode->i_mode))
+				ll_dir_clear_lsm_md(inode);
 			make_bad_inode(inode);
 			unlock_new_inode(inode);
 			iput(inode);
@@ -132,6 +137,8 @@ struct inode *ll_iget(struct super_block *sb, ino_t hash,
 		CDEBUG(D_VFSTRACE, "got inode: "DFID"(%p): rc = %d\n",
 		       PFID(&md->body->mbo_fid1), inode, rc);
 		if (rc) {
+			if (S_ISDIR(inode->i_mode))
+				ll_dir_clear_lsm_md(inode);
 			iput(inode);
 			inode = ERR_PTR(rc);
 		}
@@ -258,7 +265,9 @@ int ll_md_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
 			struct ll_inode_info *lli = ll_i2info(inode);
 
 			spin_lock(&lli->lli_lock);
-			lli->lli_flags &= ~LLIF_MDS_SIZE_LOCK;
+			LTIME_S(inode->i_mtime) = 0;
+			LTIME_S(inode->i_atime) = 0;
+			LTIME_S(inode->i_ctime) = 0;
 			spin_unlock(&lli->lli_lock);
 		}
 
@@ -287,11 +296,39 @@ int ll_md_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
 
 				hash = cl_fid_build_ino(&lli->lli_pfid,
 							ll_need_32bit_api(ll_i2sbi(inode)));
-
-				master_inode = ilookup5(inode->i_sb, hash,
-							ll_test_inode_by_fid,
-							(void *)&lli->lli_pfid);
-				if (master_inode && !IS_ERR(master_inode)) {
+				/*
+				 * Do not lookup the inode with ilookup5,
+				 * otherwise it will cause dead lock,
+				 *
+				 * 1. Client1 send chmod req to the MDT0, then
+				 * on MDT0, it enqueues master and all of its
+				 * slaves lock, (mdt_attr_set() ->
+				 * mdt_lock_slaves()), after gets master and
+				 * stripe0 lock, it will send the enqueue req
+				 * (for stripe1) to MDT1, then MDT1 finds the
+				 * lock has been granted to client2. Then MDT1
+				 * sends blocking ast to client2.
+				 *
+				 * 2. At the same time, client2 tries to unlink
+				 * the striped dir (rm -rf striped_dir), and
+				 * during lookup, it will hold the master inode
+				 * of the striped directory, whose inode state
+				 * is NEW, then tries to revalidate all of its
+				 * slaves, (ll_prep_inode()->ll_iget()->
+				 * ll_read_inode2()-> ll_update_inode().). And
+				 * it will be blocked on the server side because
+				 * of 1.
+				 *
+				 * 3. Then the client get the blocking_ast req,
+				 * cancel the lock, but being blocked if using
+				 * ->ilookup5()), because master inode state is
+				 *  NEW.
+				 */
+				master_inode = ilookup5_nowait(inode->i_sb,
+							       hash,
+							       ll_test_inode_by_fid,
+							       (void *)&lli->lli_pfid);
+				if (master_inode) {
 					ll_invalidate_negative_children(master_inode);
 					iput(master_inode);
 				}
@@ -535,6 +572,10 @@ static struct dentry *ll_lookup_it(struct inode *parent, struct dentry *dentry,
 		}
 	}
 
+	if (it->it_op & IT_OPEN && it->it_flags & FMODE_WRITE &&
+	    dentry->d_sb->s_flags & MS_RDONLY)
+		return ERR_PTR(-EROFS);
+
 	if (it->it_op & IT_CREAT)
 		opc = LUSTRE_OPC_CREATE;
 	else
@@ -801,7 +842,8 @@ static int ll_create_it(struct inode *dir, struct dentry *dentry,
 		return PTR_ERR(inode);
 
 	d_instantiate(dentry, inode);
-	return 0;
+
+	return ll_init_security(dentry, inode, dir);
 }
 
 void ll_update_times(struct ptlrpc_request *request, struct inode *inode)
@@ -896,6 +938,8 @@ again:
 		goto err_exit;
 
 	d_instantiate(dentry, inode);
+
+	err = ll_init_security(dentry, inode, dir);
 err_exit:
 	if (request)
 		ptlrpc_req_finished(request);
diff --git a/drivers/staging/lustre/lustre/llite/rw.c b/drivers/staging/lustre/lustre/llite/rw.c
index 50c0152ba022..f10e092979fe 100644
--- a/drivers/staging/lustre/lustre/llite/rw.c
+++ b/drivers/staging/lustre/lustre/llite/rw.c
@@ -47,6 +47,7 @@
 #include <linux/pagemap.h>
 /* current_is_kswapd() */
 #include <linux/swap.h>
+#include <linux/bvec.h>
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
@@ -180,90 +181,73 @@ void ll_ras_enter(struct file *f)
 	spin_unlock(&ras->ras_lock);
 }
 
-static int cl_read_ahead_page(const struct lu_env *env, struct cl_io *io,
-			      struct cl_page_list *queue, struct cl_page *page,
-			      struct cl_object *clob, pgoff_t *max_index)
+/**
+ * Initiates read-ahead of a page with given index.
+ *
+ * \retval +ve:	page was already uptodate so it will be skipped
+ *		from being added;
+ * \retval -ve:	page wasn't added to \a queue for error;
+ * \retval   0:	page was added into \a queue for read ahead.
+ */
+static int ll_read_ahead_page(const struct lu_env *env, struct cl_io *io,
+			      struct cl_page_list *queue, pgoff_t index)
 {
-	struct page *vmpage = page->cp_vmpage;
+	enum ra_stat which = _NR_RA_STAT; /* keep gcc happy */
+	struct cl_object *clob = io->ci_obj;
+	struct inode *inode = vvp_object_inode(clob);
+	const char *msg = NULL;
+	struct cl_page *page;
 	struct vvp_page *vpg;
-	int	      rc;
+	struct page *vmpage;
+	int rc = 0;
+
+	vmpage = grab_cache_page_nowait(inode->i_mapping, index);
+	if (!vmpage) {
+		which = RA_STAT_FAILED_GRAB_PAGE;
+		msg = "g_c_p_n failed";
+		rc = -EBUSY;
+		goto out;
+	}
+
+	/* Check if vmpage was truncated or reclaimed */
+	if (vmpage->mapping != inode->i_mapping) {
+		which = RA_STAT_WRONG_GRAB_PAGE;
+		msg = "g_c_p_n returned invalid page";
+		rc = -EBUSY;
+		goto out;
+	}
+
+	page = cl_page_find(env, clob, vmpage->index, vmpage, CPT_CACHEABLE);
+	if (IS_ERR(page)) {
+		which = RA_STAT_FAILED_GRAB_PAGE;
+		msg = "cl_page_find failed";
+		rc = PTR_ERR(page);
+		goto out;
+	}
 
-	rc = 0;
-	cl_page_assume(env, io, page);
 	lu_ref_add(&page->cp_reference, "ra", current);
+	cl_page_assume(env, io, page);
 	vpg = cl2vvp_page(cl_object_page_slice(clob, page));
 	if (!vpg->vpg_defer_uptodate && !PageUptodate(vmpage)) {
-		CDEBUG(D_READA, "page index %lu, max_index: %lu\n",
-		       vvp_index(vpg), *max_index);
-		if (*max_index == 0 || vvp_index(vpg) > *max_index)
-			rc = cl_page_is_under_lock(env, io, page, max_index);
-		if (rc == 0) {
-			vpg->vpg_defer_uptodate = 1;
-			vpg->vpg_ra_used = 0;
-			cl_page_list_add(queue, page);
-			rc = 1;
-		} else {
-			cl_page_discard(env, io, page);
-			rc = -ENOLCK;
-		}
+		vpg->vpg_defer_uptodate = 1;
+		vpg->vpg_ra_used = 0;
+		cl_page_list_add(queue, page);
 	} else {
 		/* skip completed pages */
 		cl_page_unassume(env, io, page);
+		/* This page is already uptodate, returning a positive number
+		 * to tell the callers about this
+		 */
+		rc = 1;
 	}
+
 	lu_ref_del(&page->cp_reference, "ra", current);
 	cl_page_put(env, page);
-	return rc;
-}
-
-/**
- * Initiates read-ahead of a page with given index.
- *
- * \retval     +ve: page was added to \a queue.
- *
- * \retval -ENOLCK: there is no extent lock for this part of a file, stop
- *		  read-ahead.
- *
- * \retval  -ve, 0: page wasn't added to \a queue for other reason.
- */
-static int ll_read_ahead_page(const struct lu_env *env, struct cl_io *io,
-			      struct cl_page_list *queue,
-			      pgoff_t index, pgoff_t *max_index)
-{
-	struct cl_object *clob  = io->ci_obj;
-	struct inode     *inode = vvp_object_inode(clob);
-	struct page      *vmpage;
-	struct cl_page   *page;
-	enum ra_stat      which = _NR_RA_STAT; /* keep gcc happy */
-	int	       rc    = 0;
-	const char       *msg   = NULL;
-
-	vmpage = grab_cache_page_nowait(inode->i_mapping, index);
+out:
 	if (vmpage) {
-		/* Check if vmpage was truncated or reclaimed */
-		if (vmpage->mapping == inode->i_mapping) {
-			page = cl_page_find(env, clob, vmpage->index,
-					    vmpage, CPT_CACHEABLE);
-			if (!IS_ERR(page)) {
-				rc = cl_read_ahead_page(env, io, queue,
-							page, clob, max_index);
-				if (rc == -ENOLCK) {
-					which = RA_STAT_FAILED_MATCH;
-					msg   = "lock match failed";
-				}
-			} else {
-				which = RA_STAT_FAILED_GRAB_PAGE;
-				msg   = "cl_page_find failed";
-			}
-		} else {
-			which = RA_STAT_WRONG_GRAB_PAGE;
-			msg   = "g_c_p_n returned invalid page";
-		}
-		if (rc != 1)
+		if (rc)
 			unlock_page(vmpage);
 		put_page(vmpage);
-	} else {
-		which = RA_STAT_FAILED_GRAB_PAGE;
-		msg   = "g_c_p_n failed";
 	}
 	if (msg) {
 		ll_ra_stats_inc(inode, which);
@@ -378,12 +362,12 @@ static int ll_read_ahead_pages(const struct lu_env *env,
 			       struct cl_io *io, struct cl_page_list *queue,
 			       struct ra_io_arg *ria,
 			       unsigned long *reserved_pages,
-			       unsigned long *ra_end)
+			       pgoff_t *ra_end)
 {
+	struct cl_read_ahead ra = { 0 };
 	int rc, count = 0;
 	bool stride_ria;
 	pgoff_t page_idx;
-	pgoff_t max_index = 0;
 
 	LASSERT(ria);
 	RIA_DEBUG(ria);
@@ -392,14 +376,23 @@ static int ll_read_ahead_pages(const struct lu_env *env,
 	for (page_idx = ria->ria_start;
 	     page_idx <= ria->ria_end && *reserved_pages > 0; page_idx++) {
 		if (ras_inside_ra_window(page_idx, ria)) {
+			if (!ra.cra_end || ra.cra_end < page_idx) {
+				cl_read_ahead_release(env, &ra);
+
+				rc = cl_io_read_ahead(env, io, page_idx, &ra);
+				if (rc < 0)
+					break;
+
+				LASSERTF(ra.cra_end >= page_idx,
+					 "object: %p, indcies %lu / %lu\n",
+					 io->ci_obj, ra.cra_end, page_idx);
+			}
+
 			/* If the page is inside the read-ahead window*/
-			rc = ll_read_ahead_page(env, io, queue,
-						page_idx, &max_index);
-			if (rc == 1) {
+			rc = ll_read_ahead_page(env, io, queue, page_idx);
+			if (!rc) {
 				(*reserved_pages)--;
 				count++;
-			} else if (rc == -ENOLCK) {
-				break;
 			}
 		} else if (stride_ria) {
 			/* If it is not in the read-ahead window, and it is
@@ -425,19 +418,21 @@ static int ll_read_ahead_pages(const struct lu_env *env,
 			}
 		}
 	}
+	cl_read_ahead_release(env, &ra);
+
 	*ra_end = page_idx;
 	return count;
 }
 
-int ll_readahead(const struct lu_env *env, struct cl_io *io,
-		 struct cl_page_list *queue, struct ll_readahead_state *ras,
-		 bool hit)
+static int ll_readahead(const struct lu_env *env, struct cl_io *io,
+			struct cl_page_list *queue,
+			struct ll_readahead_state *ras, bool hit)
 {
 	struct vvp_io *vio = vvp_env_io(env);
 	struct ll_thread_info *lti = ll_env_info(env);
 	struct cl_attr *attr = vvp_env_thread_attr(env);
-	unsigned long start = 0, end = 0, reserved;
-	unsigned long ra_end, len, mlen = 0;
+	unsigned long len, mlen = 0, reserved;
+	pgoff_t ra_end, start = 0, end = 0;
 	struct inode *inode;
 	struct ra_io_arg *ria = &lti->lti_ria;
 	struct cl_object *clob;
@@ -463,30 +458,25 @@ int ll_readahead(const struct lu_env *env, struct cl_io *io,
 
 	spin_lock(&ras->ras_lock);
 
-	/* Enlarge the RA window to encompass the full read */
-	if (vio->vui_ra_valid &&
-	    ras->ras_window_start + ras->ras_window_len <
-	    vio->vui_ra_start + vio->vui_ra_count) {
-		ras->ras_window_len = vio->vui_ra_start + vio->vui_ra_count -
-				      ras->ras_window_start;
-	}
+	/**
+	 * Note: other thread might rollback the ras_next_readahead,
+	 * if it can not get the full size of prepared pages, see the
+	 * end of this function. For stride read ahead, it needs to
+	 * make sure the offset is no less than ras_stride_offset,
+	 * so that stride read ahead can work correctly.
+	 */
+	if (stride_io_mode(ras))
+		start = max(ras->ras_next_readahead, ras->ras_stride_offset);
+	else
+		start = ras->ras_next_readahead;
 
-	/* Reserve a part of the read-ahead window that we'll be issuing */
-	if (ras->ras_window_len > 0) {
-		/*
-		 * Note: other thread might rollback the ras_next_readahead,
-		 * if it can not get the full size of prepared pages, see the
-		 * end of this function. For stride read ahead, it needs to
-		 * make sure the offset is no less than ras_stride_offset,
-		 * so that stride read ahead can work correctly.
-		 */
-		if (stride_io_mode(ras))
-			start = max(ras->ras_next_readahead,
-				    ras->ras_stride_offset);
-		else
-			start = ras->ras_next_readahead;
+	if (ras->ras_window_len > 0)
 		end = ras->ras_window_start + ras->ras_window_len - 1;
-	}
+
+	/* Enlarge the RA window to encompass the full read */
+	if (vio->vui_ra_valid &&
+	    end < vio->vui_ra_start + vio->vui_ra_count - 1)
+		end = vio->vui_ra_start + vio->vui_ra_count - 1;
 
 	if (end != 0) {
 		unsigned long rpc_boundary;
@@ -575,8 +565,8 @@ int ll_readahead(const struct lu_env *env, struct cl_io *io,
 	 * if the region we failed to issue read-ahead on is still ahead
 	 * of the app and behind the next index to start read-ahead from
 	 */
-	CDEBUG(D_READA, "ra_end %lu end %lu stride end %lu\n",
-	       ra_end, end, ria->ria_end);
+	CDEBUG(D_READA, "ra_end = %lu end = %lu stride end = %lu pages = %d\n",
+	       ra_end, end, ria->ria_end, ret);
 
 	if (ra_end != end + 1) {
 		ll_ra_stats_inc(inode, RA_STAT_FAILED_REACH_END);
@@ -608,7 +598,7 @@ static void ras_reset(struct inode *inode, struct ll_readahead_state *ras,
 	ras->ras_consecutive_pages = 0;
 	ras->ras_window_len = 0;
 	ras_set_start(inode, ras, index);
-	ras->ras_next_readahead = max(ras->ras_window_start, index);
+	ras->ras_next_readahead = max(ras->ras_window_start, index + 1);
 
 	RAS_CDEBUG(ras);
 }
@@ -737,12 +727,13 @@ static void ras_increase_window(struct inode *inode,
 					  ra->ra_max_pages_per_file);
 }
 
-void ras_update(struct ll_sb_info *sbi, struct inode *inode,
-		struct ll_readahead_state *ras, unsigned long index,
-		unsigned hit)
+static void ras_update(struct ll_sb_info *sbi, struct inode *inode,
+		       struct ll_readahead_state *ras, unsigned long index,
+		       enum ras_update_flags flags)
 {
 	struct ll_ra_info *ra = &sbi->ll_ra_info;
 	int zero = 0, stride_detect = 0, ra_miss = 0;
+	bool hit = flags & LL_RAS_HIT;
 
 	spin_lock(&ras->ras_lock);
 
@@ -772,7 +763,7 @@ void ras_update(struct ll_sb_info *sbi, struct inode *inode,
 	 * to for subsequent IO.  The mmap case does not increment
 	 * ras_requests and thus can never trigger this behavior.
 	 */
-	if (ras->ras_requests == 2 && !ras->ras_request_index) {
+	if (ras->ras_requests >= 2 && !ras->ras_request_index) {
 		__u64 kms_pages;
 
 		kms_pages = (i_size_read(inode) + PAGE_SIZE - 1) >>
@@ -784,8 +775,7 @@ void ras_update(struct ll_sb_info *sbi, struct inode *inode,
 		if (kms_pages &&
 		    kms_pages <= ra->ra_max_read_ahead_whole_pages) {
 			ras->ras_window_start = 0;
-			ras->ras_last_readpage = 0;
-			ras->ras_next_readahead = 0;
+			ras->ras_next_readahead = index + 1;
 			ras->ras_window_len = min(ra->ra_max_pages_per_file,
 				ra->ra_max_read_ahead_whole_pages);
 			goto out_unlock;
@@ -815,13 +805,20 @@ void ras_update(struct ll_sb_info *sbi, struct inode *inode,
 		if (ra_miss) {
 			if (index_in_stride_window(ras, index) &&
 			    stride_io_mode(ras)) {
-				/*If stride-RA hit cache miss, the stride dector
-				 *will not be reset to avoid the overhead of
-				 *redetecting read-ahead mode
-				 */
 				if (index != ras->ras_last_readpage + 1)
 					ras->ras_consecutive_pages = 0;
 				ras_reset(inode, ras, index);
+
+				/* If stride-RA hit cache miss, the stride
+				 * detector will not be reset to avoid the
+				 * overhead of redetecting read-ahead mode,
+				 * but on the condition that the stride window
+				 * is still intersect with normal sequential
+				 * read-ahead window.
+				 */
+				if (ras->ras_window_start <
+				    ras->ras_stride_offset)
+					ras_stride_reset(ras);
 				RAS_CDEBUG(ras);
 			} else {
 				/* Reset both stride window and normal RA
@@ -866,8 +863,13 @@ void ras_update(struct ll_sb_info *sbi, struct inode *inode,
 	/* Trigger RA in the mmap case where ras_consecutive_requests
 	 * is not incremented and thus can't be used to trigger RA
 	 */
-	if (!ras->ras_window_len && ras->ras_consecutive_pages == 4) {
-		ras->ras_window_len = RAS_INCREASE_STEP(inode);
+	if (ras->ras_consecutive_pages >= 4 && flags & LL_RAS_MMAP) {
+		ras_increase_window(inode, ras, ra);
+		/*
+		 * reset consecutive pages so that the readahead window can
+		 * grow gradually.
+		 */
+		ras->ras_consecutive_pages = 0;
 		goto out_unlock;
 	}
 
@@ -902,17 +904,17 @@ int ll_writepage(struct page *vmpage, struct writeback_control *wbc)
 	struct cl_io	   *io;
 	struct cl_page	 *page;
 	struct cl_object       *clob;
-	struct cl_env_nest      nest;
 	bool redirtied = false;
 	bool unlocked = false;
 	int result;
+	int refcheck;
 
 	LASSERT(PageLocked(vmpage));
 	LASSERT(!PageWriteback(vmpage));
 
 	LASSERT(ll_i2dtexp(inode));
 
-	env = cl_env_nested_get(&nest);
+	env = cl_env_get(&refcheck);
 	if (IS_ERR(env)) {
 		result = PTR_ERR(env);
 		goto out;
@@ -977,7 +979,7 @@ int ll_writepage(struct page *vmpage, struct writeback_control *wbc)
 		}
 	}
 
-	cl_env_nested_put(&nest, env);
+	cl_env_put(env, &refcheck);
 	goto out;
 
 out:
@@ -1087,6 +1089,63 @@ void ll_cl_remove(struct file *file, const struct lu_env *env)
 	write_unlock(&fd->fd_lock);
 }
 
+static int ll_io_read_page(const struct lu_env *env, struct cl_io *io,
+			   struct cl_page *page)
+{
+	struct inode *inode = vvp_object_inode(page->cp_obj);
+	struct ll_file_data *fd = vvp_env_io(env)->vui_fd;
+	struct ll_readahead_state *ras = &fd->fd_ras;
+	struct cl_2queue *queue  = &io->ci_queue;
+	struct ll_sb_info *sbi = ll_i2sbi(inode);
+	struct vvp_page *vpg;
+	int rc = 0;
+
+	vpg = cl2vvp_page(cl_object_page_slice(page->cp_obj, page));
+	if (sbi->ll_ra_info.ra_max_pages_per_file > 0 &&
+	    sbi->ll_ra_info.ra_max_pages > 0) {
+		struct vvp_io *vio = vvp_env_io(env);
+		enum ras_update_flags flags = 0;
+
+		if (vpg->vpg_defer_uptodate)
+			flags |= LL_RAS_HIT;
+		if (!vio->vui_ra_valid)
+			flags |= LL_RAS_MMAP;
+		ras_update(sbi, inode, ras, vvp_index(vpg), flags);
+	}
+
+	if (vpg->vpg_defer_uptodate) {
+		vpg->vpg_ra_used = 1;
+		cl_page_export(env, page, 1);
+	}
+
+	cl_2queue_init(queue);
+	/*
+	 * Add page into the queue even when it is marked uptodate above.
+	 * this will unlock it automatically as part of cl_page_list_disown().
+	 */
+	cl_page_list_add(&queue->c2_qin, page);
+	if (sbi->ll_ra_info.ra_max_pages_per_file > 0 &&
+	    sbi->ll_ra_info.ra_max_pages > 0) {
+		int rc2;
+
+		rc2 = ll_readahead(env, io, &queue->c2_qin, ras,
+				   vpg->vpg_defer_uptodate);
+		CDEBUG(D_READA, DFID "%d pages read ahead at %lu\n",
+		       PFID(ll_inode2fid(inode)), rc2, vvp_index(vpg));
+	}
+
+	if (queue->c2_qin.pl_nr > 0)
+		rc = cl_io_submit_rw(env, io, CRT_READ, queue);
+
+	/*
+	 * Unlock unsent pages in case of error.
+	 */
+	cl_page_list_disown(env, io, &queue->c2_qin);
+	cl_2queue_fini(env, queue);
+
+	return rc;
+}
+
 int ll_readpage(struct file *file, struct page *vmpage)
 {
 	struct cl_object *clob = ll_i2info(file_inode(file))->lli_clob;
@@ -1110,7 +1169,7 @@ int ll_readpage(struct file *file, struct page *vmpage)
 		LASSERT(page->cp_type == CPT_CACHEABLE);
 		if (likely(!PageUptodate(vmpage))) {
 			cl_page_assume(env, io, page);
-			result = cl_io_read_page(env, io, page);
+			result = ll_io_read_page(env, io, page);
 		} else {
 			/* Page from a non-object file. */
 			unlock_page(vmpage);
diff --git a/drivers/staging/lustre/lustre/llite/rw26.c b/drivers/staging/lustre/lustre/llite/rw26.c
index 26f3a37873a7..21e06e5b514e 100644
--- a/drivers/staging/lustre/lustre/llite/rw26.c
+++ b/drivers/staging/lustre/lustre/llite/rw26.c
@@ -71,8 +71,6 @@ static void ll_invalidatepage(struct page *vmpage, unsigned int offset,
 	struct cl_page   *page;
 	struct cl_object *obj;
 
-	int refcheck;
-
 	LASSERT(PageLocked(vmpage));
 	LASSERT(!PageWriteback(vmpage));
 
@@ -82,28 +80,27 @@ static void ll_invalidatepage(struct page *vmpage, unsigned int offset,
 	 * happening with locked page too
 	 */
 	if (offset == 0 && length == PAGE_SIZE) {
-		env = cl_env_get(&refcheck);
-		if (!IS_ERR(env)) {
-			inode = vmpage->mapping->host;
-			obj = ll_i2info(inode)->lli_clob;
-			if (obj) {
-				page = cl_vmpage_page(vmpage, obj);
-				if (page) {
-					cl_page_delete(env, page);
-					cl_page_put(env, page);
-				}
-			} else {
-				LASSERT(vmpage->private == 0);
+		/* See the comment in ll_releasepage() */
+		env = cl_env_percpu_get();
+		LASSERT(!IS_ERR(env));
+		inode = vmpage->mapping->host;
+		obj = ll_i2info(inode)->lli_clob;
+		if (obj) {
+			page = cl_vmpage_page(vmpage, obj);
+			if (page) {
+				cl_page_delete(env, page);
+				cl_page_put(env, page);
 			}
-			cl_env_put(env, &refcheck);
+		} else {
+			LASSERT(vmpage->private == 0);
 		}
+		cl_env_percpu_put(env);
 	}
 }
 
 static int ll_releasepage(struct page *vmpage, gfp_t gfp_mask)
 {
 	struct lu_env     *env;
-	void			*cookie;
 	struct cl_object  *obj;
 	struct cl_page    *page;
 	struct address_space *mapping;
@@ -129,7 +126,6 @@ static int ll_releasepage(struct page *vmpage, gfp_t gfp_mask)
 	if (!page)
 		return 1;
 
-	cookie = cl_env_reenter();
 	env = cl_env_percpu_get();
 	LASSERT(!IS_ERR(env));
 
@@ -155,7 +151,6 @@ static int ll_releasepage(struct page *vmpage, gfp_t gfp_mask)
 	cl_page_put(env, page);
 
 	cl_env_percpu_put(env);
-	cl_env_reexit(cookie);
 	return result;
 }
 
@@ -340,19 +335,15 @@ static ssize_t ll_direct_IO_26_seg(const struct lu_env *env, struct cl_io *io,
 		       PAGE_SIZE) & ~(DT_MAX_BRW_SIZE - 1))
 static ssize_t ll_direct_IO_26(struct kiocb *iocb, struct iov_iter *iter)
 {
-	struct lu_env *env;
+	struct ll_cl_context *lcc;
+	const struct lu_env *env;
 	struct cl_io *io;
 	struct file *file = iocb->ki_filp;
 	struct inode *inode = file->f_mapping->host;
 	loff_t file_offset = iocb->ki_pos;
 	ssize_t count = iov_iter_count(iter);
 	ssize_t tot_bytes = 0, result = 0;
-	struct ll_inode_info *lli = ll_i2info(inode);
 	long size = MAX_DIO_SIZE;
-	int refcheck;
-
-	if (!lli->lli_has_smd)
-		return -EBADF;
 
 	/* FIXME: io smaller than PAGE_SIZE is broken on ia64 ??? */
 	if ((file_offset & ~PAGE_MASK) || (count & ~PAGE_MASK))
@@ -367,9 +358,13 @@ static ssize_t ll_direct_IO_26(struct kiocb *iocb, struct iov_iter *iter)
 	if (iov_iter_alignment(iter) & ~PAGE_MASK)
 		return -EINVAL;
 
-	env = cl_env_get(&refcheck);
+	lcc = ll_cl_find(file);
+	if (!lcc)
+		return -EIO;
+
+	env = lcc->lcc_env;
 	LASSERT(!IS_ERR(env));
-	io = vvp_env_io(env)->vui_cl.cis_io;
+	io = lcc->lcc_io;
 	LASSERT(io);
 
 	while (iov_iter_count(iter)) {
@@ -426,7 +421,6 @@ out:
 		vio->u.write.vui_written += tot_bytes;
 	}
 
-	cl_env_put(env, &refcheck);
 	return tot_bytes ? tot_bytes : result;
 }
 
@@ -466,13 +460,13 @@ static int ll_prepare_partial_page(const struct lu_env *env, struct cl_io *io,
 }
 
 static int ll_write_begin(struct file *file, struct address_space *mapping,
-			  loff_t pos, unsigned len, unsigned flags,
+			  loff_t pos, unsigned int len, unsigned int flags,
 			  struct page **pagep, void **fsdata)
 {
 	struct ll_cl_context *lcc;
-	const struct lu_env  *env;
+	const struct lu_env *env = NULL;
 	struct cl_io   *io;
-	struct cl_page *page;
+	struct cl_page *page = NULL;
 	struct cl_object *clob = ll_i2info(mapping->host)->lli_clob;
 	pgoff_t index = pos >> PAGE_SHIFT;
 	struct page *vmpage = NULL;
@@ -484,6 +478,7 @@ static int ll_write_begin(struct file *file, struct address_space *mapping,
 
 	lcc = ll_cl_find(file);
 	if (!lcc) {
+		io = NULL;
 		result = -EIO;
 		goto out;
 	}
@@ -560,6 +555,12 @@ out:
 			unlock_page(vmpage);
 			put_page(vmpage);
 		}
+		if (!IS_ERR_OR_NULL(page)) {
+			lu_ref_del(&page->cp_reference, "cl_io", io);
+			cl_page_put(env, page);
+		}
+		if (io)
+			io->ci_result = result;
 	} else {
 		*pagep = vmpage;
 		*fsdata = lcc;
@@ -576,7 +577,7 @@ static int ll_write_end(struct file *file, struct address_space *mapping,
 	struct cl_io *io;
 	struct vvp_io *vio;
 	struct cl_page *page;
-	unsigned from = pos & (PAGE_SIZE - 1);
+	unsigned int from = pos & (PAGE_SIZE - 1);
 	bool unplug = false;
 	int result = 0;
 
@@ -629,6 +630,8 @@ static int ll_write_end(struct file *file, struct address_space *mapping,
 	    file->f_flags & O_SYNC || IS_SYNC(file_inode(file)))
 		result = vvp_io_write_commit(env, io);
 
+	if (result < 0)
+		io->ci_result = result;
 	return result >= 0 ? copied : result;
 }
 
diff --git a/drivers/staging/lustre/lustre/llite/statahead.c b/drivers/staging/lustre/lustre/llite/statahead.c
index 0677513476ec..4769a2230ae1 100644
--- a/drivers/staging/lustre/lustre/llite/statahead.c
+++ b/drivers/staging/lustre/lustre/llite/statahead.c
@@ -659,8 +659,8 @@ static int ll_statahead_interpret(struct ptlrpc_request *req,
 	struct ll_inode_info     *lli = ll_i2info(dir);
 	struct ll_statahead_info *sai = lli->lli_sai;
 	struct sa_entry *entry = (struct sa_entry *)minfo->mi_cbdata;
+	wait_queue_head_t *waitq = NULL;
 	__u64 handle = 0;
-	bool wakeup;
 
 	if (it_disposition(it, DISP_LOOKUP_NEG))
 		rc = -ENOENT;
@@ -693,7 +693,8 @@ static int ll_statahead_interpret(struct ptlrpc_request *req,
 
 	spin_lock(&lli->lli_sa_lock);
 	if (rc) {
-		wakeup = __sa_make_ready(sai, entry, rc);
+		if (__sa_make_ready(sai, entry, rc))
+			waitq = &sai->sai_waitq;
 	} else {
 		entry->se_minfo = minfo;
 		entry->se_req = ptlrpc_request_addref(req);
@@ -704,13 +705,15 @@ static int ll_statahead_interpret(struct ptlrpc_request *req,
 		 * with parent's lock held, for example: unlink.
 		 */
 		entry->se_handle = handle;
-		wakeup = !sa_has_callback(sai);
+		if (!sa_has_callback(sai))
+			waitq = &sai->sai_thread.t_ctl_waitq;
+
 		list_add_tail(&entry->se_list, &sai->sai_interim_entries);
 	}
 	sai->sai_replied++;
 
-	if (wakeup)
-		wake_up(&sai->sai_thread.t_ctl_waitq);
+	if (waitq)
+		wake_up(waitq);
 	spin_unlock(&lli->lli_sa_lock);
 
 	return rc;
@@ -1397,10 +1400,10 @@ static int revalidate_statahead_dentry(struct inode *dir,
 				       struct dentry **dentryp,
 				       bool unplug)
 {
+	struct ll_inode_info *lli = ll_i2info(dir);
 	struct sa_entry *entry = NULL;
 	struct l_wait_info lwi = { 0 };
 	struct ll_dentry_data *ldd;
-	struct ll_inode_info *lli;
 	int rc = 0;
 
 	if ((*dentryp)->d_name.name[0] == '.') {
@@ -1446,7 +1449,9 @@ static int revalidate_statahead_dentry(struct inode *dir,
 		sa_handle_callback(sai);
 
 	if (!sa_ready(entry)) {
+		spin_lock(&lli->lli_sa_lock);
 		sai->sai_index_wait = entry->se_index;
+		spin_unlock(&lli->lli_sa_lock);
 		lwi = LWI_TIMEOUT_INTR(cfs_time_seconds(30), NULL,
 				       LWI_ON_SIGNAL_NOOP, NULL);
 		rc = l_wait_event(sai->sai_waitq, sa_ready(entry), &lwi);
@@ -1475,6 +1480,7 @@ static int revalidate_statahead_dentry(struct inode *dir,
 
 				alias = ll_splice_alias(inode, *dentryp);
 				if (IS_ERR(alias)) {
+					ll_intent_release(&it);
 					rc = PTR_ERR(alias);
 					goto out_unplug;
 				}
@@ -1493,6 +1499,7 @@ static int revalidate_statahead_dentry(struct inode *dir,
 				       *dentryp,
 				       PFID(ll_inode2fid((*dentryp)->d_inode)),
 				       PFID(ll_inode2fid(inode)));
+				ll_intent_release(&it);
 				rc = -ESTALE;
 				goto out_unplug;
 			}
@@ -1512,7 +1519,6 @@ out_unplug:
 	 * dentry_may_statahead().
 	 */
 	ldd = ll_d2d(*dentryp);
-	lli = ll_i2info(dir);
 	/* ldd can be NULL if llite lookup failed. */
 	if (ldd)
 		ldd->lld_sa_generation = lli->lli_sa_generation;
diff --git a/drivers/staging/lustre/lustre/llite/vvp_dev.c b/drivers/staging/lustre/lustre/llite/vvp_dev.c
index 8aa8ecc09a48..12c129f7e4ad 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_dev.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_dev.c
@@ -55,7 +55,6 @@
 static struct kmem_cache *ll_thread_kmem;
 struct kmem_cache *vvp_lock_kmem;
 struct kmem_cache *vvp_object_kmem;
-struct kmem_cache *vvp_req_kmem;
 static struct kmem_cache *vvp_session_kmem;
 static struct kmem_cache *vvp_thread_kmem;
 
@@ -76,11 +75,6 @@ static struct lu_kmem_descr vvp_caches[] = {
 		.ckd_size  = sizeof(struct vvp_object),
 	},
 	{
-		.ckd_cache = &vvp_req_kmem,
-		.ckd_name  = "vvp_req_kmem",
-		.ckd_size  = sizeof(struct vvp_req),
-	},
-	{
 		.ckd_cache = &vvp_session_kmem,
 		.ckd_name  = "vvp_session_kmem",
 		.ckd_size  = sizeof(struct vvp_session)
@@ -177,10 +171,6 @@ static const struct lu_device_operations vvp_lu_ops = {
 	.ldo_object_alloc      = vvp_object_alloc
 };
 
-static const struct cl_device_operations vvp_cl_ops = {
-	.cdo_req_init = vvp_req_init
-};
-
 static struct lu_device *vvp_device_free(const struct lu_env *env,
 					 struct lu_device *d)
 {
@@ -213,7 +203,6 @@ static struct lu_device *vvp_device_alloc(const struct lu_env *env,
 	lud = &vdv->vdv_cl.cd_lu_dev;
 	cl_device_init(&vdv->vdv_cl, t);
 	vvp2lu_dev(vdv)->ld_ops = &vvp_lu_ops;
-	vdv->vdv_cl.cd_ops = &vvp_cl_ops;
 
 	site = kzalloc(sizeof(*site), GFP_NOFS);
 	if (site) {
@@ -332,7 +321,6 @@ int cl_sb_init(struct super_block *sb)
 		cl = cl_type_setup(env, NULL, &vvp_device_type,
 				   sbi->ll_dt_exp->exp_obd->obd_lu_dev);
 		if (!IS_ERR(cl)) {
-			cl2vvp_dev(cl)->vdv_sb = sb;
 			sbi->ll_cl = cl;
 			sbi->ll_site = cl2lu_dev(cl)->ld_site;
 		}
@@ -521,11 +509,10 @@ static void vvp_pgcache_page_show(const struct lu_env *env,
 
 	vpg = cl2vvp_page(cl_page_at(page, &vvp_device_type));
 	vmpage = vpg->vpg_page;
-	seq_printf(seq, " %5i | %p %p %s %s %s %s | %p "DFID"(%p) %lu %u [",
+	seq_printf(seq, " %5i | %p %p %s %s %s | %p " DFID "(%p) %lu %u [",
 		   0 /* gen */,
 		   vpg, page,
 		   "none",
-		   vpg->vpg_write_queued ? "wq" : "- ",
 		   vpg->vpg_defer_uptodate ? "du" : "- ",
 		   PageWriteback(vmpage) ? "wb" : "-",
 		   vmpage, PFID(ll_inode2fid(vmpage->mapping->host)),
diff --git a/drivers/staging/lustre/lustre/llite/vvp_internal.h b/drivers/staging/lustre/lustre/llite/vvp_internal.h
index 4464ad258387..c60d0414ac25 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_internal.h
+++ b/drivers/staging/lustre/lustre/llite/vvp_internal.h
@@ -42,9 +42,7 @@
 
 enum obd_notify_event;
 struct inode;
-struct lov_stripe_md;
 struct lustre_md;
-struct obd_capa;
 struct obd_device;
 struct obd_export;
 struct page;
@@ -122,7 +120,6 @@ extern struct lu_context_key vvp_thread_key;
 
 extern struct kmem_cache *vvp_lock_kmem;
 extern struct kmem_cache *vvp_object_kmem;
-extern struct kmem_cache *vvp_req_kmem;
 
 struct vvp_thread_info {
 	struct cl_lock		vti_lock;
@@ -195,14 +192,6 @@ struct vvp_object {
 	struct inode           *vob_inode;
 
 	/**
-	 * A list of dirty pages pending IO in the cache. Used by
-	 * SOM. Protected by ll_inode_info::lli_lock.
-	 *
-	 * \see vvp_page::vpg_pending_linkage
-	 */
-	struct list_head	vob_pending_list;
-
-	/**
 	 * Number of transient pages.  This is no longer protected by i_sem,
 	 * and needs to be atomic.  This is not actually used for anything,
 	 * and can probably be removed.
@@ -235,15 +224,7 @@ struct vvp_object {
 struct vvp_page {
 	struct cl_page_slice vpg_cl;
 	unsigned int	vpg_defer_uptodate:1,
-			vpg_ra_used:1,
-			vpg_write_queued:1;
-	/**
-	 * Non-empty iff this page is already counted in
-	 * vvp_object::vob_pending_list. This list is only used as a flag,
-	 * that is, never iterated through, only checked for list_empty(), but
-	 * having a list is useful for debugging.
-	 */
-	struct list_head	   vpg_pending_linkage;
+			vpg_ra_used:1;
 	/** VM page */
 	struct page	  *vpg_page;
 };
@@ -260,7 +241,6 @@ static inline pgoff_t vvp_index(struct vvp_page *vvp)
 
 struct vvp_device {
 	struct cl_device    vdv_cl;
-	struct super_block *vdv_sb;
 	struct cl_device   *vdv_next;
 };
 
@@ -268,10 +248,6 @@ struct vvp_lock {
 	struct cl_lock_slice vlk_cl;
 };
 
-struct vvp_req {
-	struct cl_req_slice  vrq_cl;
-};
-
 void *ccc_key_init(const struct lu_context *ctx,
 		   struct lu_context_key *key);
 void ccc_key_fini(const struct lu_context *ctx,
@@ -325,21 +301,8 @@ static inline struct vvp_lock *cl2vvp_lock(const struct cl_lock_slice *slice)
 # define CLOBINVRNT(env, clob, expr)					\
 	((void)sizeof(env), (void)sizeof(clob), (void)sizeof(!!(expr)))
 
-/**
- * New interfaces to get and put lov_stripe_md from lov layer. This violates
- * layering because lov_stripe_md is supposed to be a private data in lov.
- *
- * NB: If you find you have to use these interfaces for your new code, please
- * think about it again. These interfaces may be removed in the future for
- * better layering.
- */
-struct lov_stripe_md *lov_lsm_get(struct cl_object *clobj);
-void lov_lsm_put(struct cl_object *clobj, struct lov_stripe_md *lsm);
 int lov_read_and_clear_async_rc(struct cl_object *clob);
 
-struct lov_stripe_md *ccc_inode_lsm_get(struct inode *inode);
-void ccc_inode_lsm_put(struct inode *inode, struct lov_stripe_md *lsm);
-
 int vvp_io_init(const struct lu_env *env, struct cl_object *obj,
 		struct cl_io *io);
 int vvp_io_write_commit(const struct lu_env *env, struct cl_io *io);
@@ -347,8 +310,6 @@ int vvp_lock_init(const struct lu_env *env, struct cl_object *obj,
 		  struct cl_lock *lock, const struct cl_io *io);
 int vvp_page_init(const struct lu_env *env, struct cl_object *obj,
 		  struct cl_page *page, pgoff_t index);
-int vvp_req_init(const struct lu_env *env, struct cl_device *dev,
-		 struct cl_req *req);
 struct lu_object *vvp_object_alloc(const struct lu_env *env,
 				   const struct lu_object_header *hdr,
 				   struct lu_device *dev);
diff --git a/drivers/staging/lustre/lustre/llite/vvp_io.c b/drivers/staging/lustre/lustre/llite/vvp_io.c
index 2b7f182a15e2..697cbfbe9374 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_io.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_io.c
@@ -72,9 +72,10 @@ static bool can_populate_pages(const struct lu_env *env, struct cl_io *io,
 		/* don't need lock here to check lli_layout_gen as we have held
 		 * extent lock and GROUP lock has to hold to swap layout
 		 */
-		if (ll_layout_version_get(lli) != vio->vui_layout_gen) {
+		if (ll_layout_version_get(lli) != vio->vui_layout_gen ||
+		    OBD_FAIL_CHECK_RESET(OBD_FAIL_LLITE_LOST_LAYOUT, 0)) {
 			io->ci_need_restart = 1;
-			/* this will return application a short read/write */
+			/* this will cause a short read/write */
 			io->ci_continue = 0;
 			rc = false;
 		}
@@ -328,8 +329,8 @@ static void vvp_io_fini(const struct lu_env *env, const struct cl_io_slice *ios)
 			       vio->vui_layout_gen, gen);
 			/* today successful restore is the only possible case */
 			/* restore was done, clear restoring state */
-			ll_i2info(vvp_object_inode(obj))->lli_flags &=
-				~LLIF_FILE_RESTORING;
+			clear_bit(LLIF_FILE_RESTORING,
+				  &ll_i2info(inode)->lli_flags);
 		}
 	}
 }
@@ -369,7 +370,7 @@ static int vvp_mmap_locks(const struct lu_env *env,
 	struct mm_struct       *mm = current->mm;
 	struct vm_area_struct  *vma;
 	struct cl_lock_descr   *descr = &cti->vti_descr;
-	ldlm_policy_data_t      policy;
+	union ldlm_policy_data policy;
 	unsigned long	   addr;
 	ssize_t		 count;
 	int		 result = 0;
@@ -450,7 +451,8 @@ static void vvp_io_advance(const struct lu_env *env,
 	struct vvp_io	 *vio = cl2vvp_io(env, ios);
 	CLOBINVRNT(env, obj, vvp_object_invariant(obj));
 
-	iov_iter_reexpand(vio->vui_iter, vio->vui_tot_count  -= nob);
+	vio->vui_tot_count -= nob;
+	iov_iter_reexpand(vio->vui_iter, vio->vui_tot_count);
 }
 
 static void vvp_io_update_iov(const struct lu_env *env,
@@ -551,9 +553,16 @@ static int vvp_io_setattr_lock(const struct lu_env *env,
 		if (new_size == 0)
 			enqflags = CEF_DISCARD_DATA;
 	} else {
-		if ((io->u.ci_setattr.sa_attr.lvb_mtime >=
-		     io->u.ci_setattr.sa_attr.lvb_ctime) ||
-		    (io->u.ci_setattr.sa_attr.lvb_atime >=
+		unsigned int valid = io->u.ci_setattr.sa_valid;
+
+		if (!(valid & TIMES_SET_FLAGS))
+			return 0;
+
+		if ((!(valid & ATTR_MTIME) ||
+		     io->u.ci_setattr.sa_attr.lvb_mtime >=
+		     io->u.ci_setattr.sa_attr.lvb_ctime) &&
+		    (!(valid & ATTR_ATIME) ||
+		     io->u.ci_setattr.sa_attr.lvb_atime >=
 		     io->u.ci_setattr.sa_attr.lvb_ctime))
 			return 0;
 		new_size = 0;
@@ -580,14 +589,6 @@ static int vvp_do_vmtruncate(struct inode *inode, size_t size)
 	return result;
 }
 
-static int vvp_io_setattr_trunc(const struct lu_env *env,
-				const struct cl_io_slice *ios,
-				struct inode *inode, loff_t size)
-{
-	inode_dio_wait(inode);
-	return 0;
-}
-
 static int vvp_io_setattr_time(const struct lu_env *env,
 			       const struct cl_io_slice *ios)
 {
@@ -618,15 +619,20 @@ static int vvp_io_setattr_start(const struct lu_env *env,
 {
 	struct cl_io	*io    = ios->cis_io;
 	struct inode	*inode = vvp_object_inode(io->ci_obj);
-	int result = 0;
+	struct ll_inode_info *lli = ll_i2info(inode);
 
-	inode_lock(inode);
-	if (cl_io_is_trunc(io))
-		result = vvp_io_setattr_trunc(env, ios, inode,
-					io->u.ci_setattr.sa_attr.lvb_size);
-	if (result == 0)
-		result = vvp_io_setattr_time(env, ios);
-	return result;
+	if (cl_io_is_trunc(io)) {
+		down_write(&lli->lli_trunc_sem);
+		inode_lock(inode);
+		inode_dio_wait(inode);
+	} else {
+		inode_lock(inode);
+	}
+
+	if (io->u.ci_setattr.sa_valid & TIMES_SET_FLAGS)
+		return vvp_io_setattr_time(env, ios);
+
+	return 0;
 }
 
 static void vvp_io_setattr_end(const struct lu_env *env,
@@ -634,14 +640,18 @@ static void vvp_io_setattr_end(const struct lu_env *env,
 {
 	struct cl_io *io    = ios->cis_io;
 	struct inode *inode = vvp_object_inode(io->ci_obj);
+	struct ll_inode_info *lli = ll_i2info(inode);
 
-	if (cl_io_is_trunc(io))
+	if (cl_io_is_trunc(io)) {
 		/* Truncate in memory pages - they must be clean pages
 		 * because osc has already notified to destroy osc_extents.
 		 */
 		vvp_do_vmtruncate(inode, io->u.ci_setattr.sa_attr.lvb_size);
-
-	inode_unlock(inode);
+		inode_unlock(inode);
+		up_write(&lli->lli_trunc_sem);
+	} else {
+		inode_unlock(inode);
+	}
 }
 
 static void vvp_io_setattr_fini(const struct lu_env *env,
@@ -657,6 +667,7 @@ static int vvp_io_read_start(const struct lu_env *env,
 	struct cl_io      *io    = ios->cis_io;
 	struct cl_object  *obj   = io->ci_obj;
 	struct inode      *inode = vvp_object_inode(obj);
+	struct ll_inode_info *lli = ll_i2info(inode);
 	struct file       *file  = vio->vui_fd->fd_file;
 
 	int     result;
@@ -669,6 +680,8 @@ static int vvp_io_read_start(const struct lu_env *env,
 
 	CDEBUG(D_VFSTRACE, "read: -> [%lli, %lli)\n", pos, pos + cnt);
 
+	down_read(&lli->lli_trunc_sem);
+
 	if (!can_populate_pages(env, io, inode))
 		return 0;
 
@@ -770,16 +783,11 @@ static int vvp_io_commit_sync(const struct lu_env *env, struct cl_io *io,
 static void write_commit_callback(const struct lu_env *env, struct cl_io *io,
 				  struct cl_page *page)
 {
-	struct vvp_page *vpg;
 	struct page *vmpage = page->cp_vmpage;
-	struct cl_object *clob = cl_io_top(io)->ci_obj;
 
 	SetPageUptodate(vmpage);
 	set_page_dirty(vmpage);
 
-	vpg = cl2vvp_page(cl_object_page_slice(clob, page));
-	vvp_write_pending(cl2vvp(clob), vpg);
-
 	cl_page_disown(env, io, page);
 
 	/* held in ll_cl_init() */
@@ -899,10 +907,13 @@ static int vvp_io_write_start(const struct lu_env *env,
 	struct cl_io       *io    = ios->cis_io;
 	struct cl_object   *obj   = io->ci_obj;
 	struct inode       *inode = vvp_object_inode(obj);
+	struct ll_inode_info *lli = ll_i2info(inode);
 	ssize_t result = 0;
 	loff_t pos = io->u.ci_wr.wr.crw_pos;
 	size_t cnt = io->u.ci_wr.wr.crw_count;
 
+	down_read(&lli->lli_trunc_sem);
+
 	if (!can_populate_pages(env, io, inode))
 		return 0;
 
@@ -921,6 +932,20 @@ static int vvp_io_write_start(const struct lu_env *env,
 
 	CDEBUG(D_VFSTRACE, "write: [%lli, %lli)\n", pos, pos + (long long)cnt);
 
+	/*
+	 * The maximum Lustre file size is variable, based on the OST maximum
+	 * object size and number of stripes.  This needs another check in
+	 * addition to the VFS checks earlier.
+	 */
+	if (pos + cnt > ll_file_maxbytes(inode)) {
+		CDEBUG(D_INODE,
+		       "%s: file " DFID " offset %llu > maxbytes %llu\n",
+		       ll_get_fsname(inode->i_sb, NULL, 0),
+		       PFID(ll_inode2fid(inode)), pos + cnt,
+		       ll_file_maxbytes(inode));
+		return -EFBIG;
+	}
+
 	if (!vio->vui_iter) {
 		/* from a temp io in ll_cl_init(). */
 		result = 0;
@@ -957,11 +982,7 @@ static int vvp_io_write_start(const struct lu_env *env,
 		}
 	}
 	if (result > 0) {
-		struct ll_inode_info *lli = ll_i2info(inode);
-
-		spin_lock(&lli->lli_lock);
-		lli->lli_flags |= LLIF_DATA_MODIFIED;
-		spin_unlock(&lli->lli_lock);
+		set_bit(LLIF_DATA_MODIFIED, &(ll_i2info(inode))->lli_flags);
 
 		if (result < cnt)
 			io->ci_continue = 0;
@@ -972,6 +993,15 @@ static int vvp_io_write_start(const struct lu_env *env,
 	return result;
 }
 
+static void vvp_io_rw_end(const struct lu_env *env,
+			  const struct cl_io_slice *ios)
+{
+	struct inode *inode = vvp_object_inode(ios->cis_obj);
+	struct ll_inode_info *lli = ll_i2info(inode);
+
+	up_read(&lli->lli_trunc_sem);
+}
+
 static int vvp_io_kernel_fault(struct vvp_fault_io *cfio)
 {
 	struct vm_fault *vmf = cfio->ft_vmf;
@@ -984,7 +1014,7 @@ static int vvp_io_kernel_fault(struct vvp_fault_io *cfio)
 		       "page %p map %p index %lu flags %lx count %u priv %0lx: got addr %p type NOPAGE\n",
 		       vmf->page, vmf->page->mapping, vmf->page->index,
 		       (long)vmf->page->flags, page_count(vmf->page),
-		       page_private(vmf->page), vmf->virtual_address);
+		       page_private(vmf->page), (void *)vmf->address);
 		if (unlikely(!(cfio->ft_flags & VM_FAULT_LOCKED))) {
 			lock_page(vmf->page);
 			cfio->ft_flags |= VM_FAULT_LOCKED;
@@ -995,12 +1025,12 @@ static int vvp_io_kernel_fault(struct vvp_fault_io *cfio)
 	}
 
 	if (cfio->ft_flags & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) {
-		CDEBUG(D_PAGE, "got addr %p - SIGBUS\n", vmf->virtual_address);
+		CDEBUG(D_PAGE, "got addr %p - SIGBUS\n", (void *)vmf->address);
 		return -EFAULT;
 	}
 
 	if (cfio->ft_flags & VM_FAULT_OOM) {
-		CDEBUG(D_PAGE, "got addr %p - OOM\n", vmf->virtual_address);
+		CDEBUG(D_PAGE, "got addr %p - OOM\n", (void *)vmf->address);
 		return -ENOMEM;
 	}
 
@@ -1014,13 +1044,7 @@ static int vvp_io_kernel_fault(struct vvp_fault_io *cfio)
 static void mkwrite_commit_callback(const struct lu_env *env, struct cl_io *io,
 				    struct cl_page *page)
 {
-	struct vvp_page *vpg;
-	struct cl_object *clob = cl_io_top(io)->ci_obj;
-
 	set_page_dirty(page->cp_vmpage);
-
-	vpg = cl2vvp_page(cl_object_page_slice(clob, page));
-	vvp_write_pending(cl2vvp(clob), vpg);
 }
 
 static int vvp_io_fault_start(const struct lu_env *env,
@@ -1030,6 +1054,7 @@ static int vvp_io_fault_start(const struct lu_env *env,
 	struct cl_io	*io      = ios->cis_io;
 	struct cl_object    *obj     = io->ci_obj;
 	struct inode        *inode   = vvp_object_inode(obj);
+	struct ll_inode_info *lli = ll_i2info(inode);
 	struct cl_fault_io  *fio     = &io->u.ci_fault;
 	struct vvp_fault_io *cfio    = &vio->u.fault;
 	loff_t	       offset;
@@ -1039,11 +1064,7 @@ static int vvp_io_fault_start(const struct lu_env *env,
 	loff_t	       size;
 	pgoff_t		     last_index;
 
-	if (fio->ft_executable &&
-	    inode->i_mtime.tv_sec != vio->u.fault.ft_mtime)
-		CWARN("binary "DFID
-		      " changed while waiting for the page fault lock\n",
-		      PFID(lu_object_fid(&obj->co_lu)));
+	down_read(&lli->lli_trunc_sem);
 
 	/* offset of the last byte on the page */
 	offset = cl_offset(obj, fio->ft_index + 1) - 1;
@@ -1192,6 +1213,17 @@ out:
 	return result;
 }
 
+static void vvp_io_fault_end(const struct lu_env *env,
+			     const struct cl_io_slice *ios)
+{
+	struct inode *inode = vvp_object_inode(ios->cis_obj);
+	struct ll_inode_info *lli = ll_i2info(inode);
+
+	CLOBINVRNT(env, ios->cis_io->ci_obj,
+		   vvp_object_invariant(ios->cis_io->ci_obj));
+	up_read(&lli->lli_trunc_sem);
+}
+
 static int vvp_io_fsync_start(const struct lu_env *env,
 			      const struct cl_io_slice *ios)
 {
@@ -1202,46 +1234,23 @@ static int vvp_io_fsync_start(const struct lu_env *env,
 	return 0;
 }
 
-static int vvp_io_read_page(const struct lu_env *env,
-			    const struct cl_io_slice *ios,
-			    const struct cl_page_slice *slice)
+static int vvp_io_read_ahead(const struct lu_env *env,
+			     const struct cl_io_slice *ios,
+			     pgoff_t start, struct cl_read_ahead *ra)
 {
-	struct cl_io	      *io     = ios->cis_io;
-	struct vvp_page           *vpg    = cl2vvp_page(slice);
-	struct cl_page	    *page   = slice->cpl_page;
-	struct inode              *inode  = vvp_object_inode(slice->cpl_obj);
-	struct ll_sb_info	 *sbi    = ll_i2sbi(inode);
-	struct ll_file_data       *fd     = cl2vvp_io(env, ios)->vui_fd;
-	struct ll_readahead_state *ras    = &fd->fd_ras;
-	struct cl_2queue	  *queue  = &io->ci_queue;
-
-	if (sbi->ll_ra_info.ra_max_pages_per_file &&
-	    sbi->ll_ra_info.ra_max_pages)
-		ras_update(sbi, inode, ras, vvp_index(vpg),
-			   vpg->vpg_defer_uptodate);
-
-	if (vpg->vpg_defer_uptodate) {
-		vpg->vpg_ra_used = 1;
-		cl_page_export(env, page, 1);
-	}
-	/*
-	 * Add page into the queue even when it is marked uptodate above.
-	 * this will unlock it automatically as part of cl_page_list_disown().
-	 */
+	int result = 0;
 
-	cl_page_list_add(&queue->c2_qin, page);
-	if (sbi->ll_ra_info.ra_max_pages_per_file &&
-	    sbi->ll_ra_info.ra_max_pages)
-		ll_readahead(env, io, &queue->c2_qin, ras,
-			     vpg->vpg_defer_uptodate);
+	if (ios->cis_io->ci_type == CIT_READ ||
+	    ios->cis_io->ci_type == CIT_FAULT) {
+		struct vvp_io *vio = cl2vvp_io(env, ios);
 
-	return 0;
-}
+		if (unlikely(vio->vui_fd->fd_flags & LL_FILE_GROUP_LOCKED)) {
+			ra->cra_end = CL_PAGE_EOF;
+			result = 1; /* no need to call down */
+		}
+	}
 
-static void vvp_io_end(const struct lu_env *env, const struct cl_io_slice *ios)
-{
-	CLOBINVRNT(env, ios->cis_io->ci_obj,
-		   vvp_object_invariant(ios->cis_io->ci_obj));
+	return result;
 }
 
 static const struct cl_io_operations vvp_io_ops = {
@@ -1250,6 +1259,7 @@ static const struct cl_io_operations vvp_io_ops = {
 			.cio_fini	= vvp_io_fini,
 			.cio_lock      = vvp_io_read_lock,
 			.cio_start     = vvp_io_read_start,
+			.cio_end	= vvp_io_rw_end,
 			.cio_advance	= vvp_io_advance,
 		},
 		[CIT_WRITE] = {
@@ -1258,6 +1268,7 @@ static const struct cl_io_operations vvp_io_ops = {
 			.cio_iter_fini = vvp_io_write_iter_fini,
 			.cio_lock      = vvp_io_write_lock,
 			.cio_start     = vvp_io_write_start,
+			.cio_end	= vvp_io_rw_end,
 			.cio_advance   = vvp_io_advance,
 		},
 		[CIT_SETATTR] = {
@@ -1272,7 +1283,7 @@ static const struct cl_io_operations vvp_io_ops = {
 			.cio_iter_init = vvp_io_fault_iter_init,
 			.cio_lock      = vvp_io_fault_lock,
 			.cio_start     = vvp_io_fault_start,
-			.cio_end       = vvp_io_end,
+			.cio_end       = vvp_io_fault_end,
 		},
 		[CIT_FSYNC] = {
 			.cio_start  = vvp_io_fsync_start,
@@ -1282,7 +1293,7 @@ static const struct cl_io_operations vvp_io_ops = {
 			.cio_fini   = vvp_io_fini
 		}
 	},
-	.cio_read_page     = vvp_io_read_page,
+	.cio_read_ahead	= vvp_io_read_ahead,
 };
 
 int vvp_io_init(const struct lu_env *env, struct cl_object *obj,
diff --git a/drivers/staging/lustre/lustre/llite/vvp_object.c b/drivers/staging/lustre/lustre/llite/vvp_object.c
index b57195d15674..8e18cf86cefc 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_object.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_object.c
@@ -65,8 +65,7 @@ static int vvp_object_print(const struct lu_env *env, void *cookie,
 	struct inode	 *inode = obj->vob_inode;
 	struct ll_inode_info *lli;
 
-	(*p)(env, cookie, "(%s %d %d) inode: %p ",
-	     list_empty(&obj->vob_pending_list) ? "-" : "+",
+	(*p)(env, cookie, "(%d %d) inode: %p ",
 	     atomic_read(&obj->vob_transient_pages),
 	     atomic_read(&obj->vob_mmap_cnt), inode);
 	if (inode) {
@@ -133,7 +132,7 @@ static int vvp_conf_set(const struct lu_env *env, struct cl_object *obj,
 		CDEBUG(D_VFSTRACE, DFID ": losing layout lock\n",
 		       PFID(&lli->lli_fid));
 
-		ll_layout_version_set(lli, LL_LAYOUT_GEN_NONE);
+		ll_layout_version_set(lli, CL_LAYOUT_GEN_NONE);
 
 		/* Clean up page mmap for this inode.
 		 * The reason for us to do this is that if the page has
@@ -146,27 +145,8 @@ static int vvp_conf_set(const struct lu_env *env, struct cl_object *obj,
 		 */
 		unmap_mapping_range(conf->coc_inode->i_mapping,
 				    0, OBD_OBJECT_EOF, 0);
-
-		return 0;
 	}
 
-	if (conf->coc_opc != OBJECT_CONF_SET)
-		return 0;
-
-	if (conf->u.coc_md && conf->u.coc_md->lsm) {
-		CDEBUG(D_VFSTRACE, DFID ": layout version change: %u -> %u\n",
-		       PFID(&lli->lli_fid), lli->lli_layout_gen,
-		       conf->u.coc_md->lsm->lsm_layout_gen);
-
-		lli->lli_has_smd = lsm_has_objects(conf->u.coc_md->lsm);
-		ll_layout_version_set(lli, conf->u.coc_md->lsm->lsm_layout_gen);
-	} else {
-		CDEBUG(D_VFSTRACE, DFID ": layout nuked: %u.\n",
-		       PFID(&lli->lli_fid), lli->lli_layout_gen);
-
-		lli->lli_has_smd = false;
-		ll_layout_version_set(lli, LL_LAYOUT_GEN_EMPTY);
-	}
 	return 0;
 }
 
@@ -204,6 +184,26 @@ static int vvp_object_glimpse(const struct lu_env *env,
 	return 0;
 }
 
+static void vvp_req_attr_set(const struct lu_env *env, struct cl_object *obj,
+			     struct cl_req_attr *attr)
+{
+	u64 valid_flags = OBD_MD_FLTYPE;
+	struct inode *inode;
+	struct obdo *oa;
+
+	oa = attr->cra_oa;
+	inode = vvp_object_inode(obj);
+
+	if (attr->cra_type == CRT_WRITE)
+		valid_flags |= OBD_MD_FLMTIME | OBD_MD_FLCTIME |
+			       OBD_MD_FLUID | OBD_MD_FLGID;
+	obdo_from_inode(oa, inode, valid_flags & attr->cra_flags);
+	obdo_set_parent_fid(oa, &ll_i2info(inode)->lli_fid);
+	if (OBD_FAIL_CHECK(OBD_FAIL_LFSCK_INVALID_PFID))
+		oa->o_parent_oid++;
+	memcpy(attr->cra_jobid, ll_i2info(inode)->lli_jobid, LUSTRE_JOBID_SIZE);
+}
+
 static const struct cl_object_operations vvp_ops = {
 	.coo_page_init = vvp_page_init,
 	.coo_lock_init = vvp_lock_init,
@@ -212,7 +212,8 @@ static const struct cl_object_operations vvp_ops = {
 	.coo_attr_update = vvp_attr_update,
 	.coo_conf_set  = vvp_conf_set,
 	.coo_prune     = vvp_prune,
-	.coo_glimpse   = vvp_object_glimpse
+	.coo_glimpse		= vvp_object_glimpse,
+	.coo_req_attr_set	= vvp_req_attr_set
 };
 
 static int vvp_object_init0(const struct lu_env *env,
@@ -240,7 +241,6 @@ static int vvp_object_init(const struct lu_env *env, struct lu_object *obj,
 		const struct cl_object_conf *cconf;
 
 		cconf = lu2cl_conf(conf);
-		INIT_LIST_HEAD(&vob->vob_pending_list);
 		lu_object_add(obj, below);
 		result = vvp_object_init0(env, vob, cconf);
 	} else {
diff --git a/drivers/staging/lustre/lustre/llite/vvp_page.c b/drivers/staging/lustre/lustre/llite/vvp_page.c
index 046e84d7a158..23d66308ff20 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_page.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_page.c
@@ -162,13 +162,10 @@ static void vvp_page_delete(const struct lu_env *env,
 	LASSERT((struct cl_page *)vmpage->private == page);
 	LASSERT(inode == vvp_object_inode(obj));
 
-	vvp_write_complete(cl2vvp(obj), cl2vvp_page(slice));
-
 	/* Drop the reference count held in vvp_page_init */
 	refc = atomic_dec_return(&page->cp_ref);
 	LASSERTF(refc >= 1, "page = %p, refc = %d\n", page, refc);
 
-	ClearPageUptodate(vmpage);
 	ClearPagePrivate(vmpage);
 	vmpage->private = 0;
 	/*
@@ -221,8 +218,6 @@ static int vvp_page_prep_write(const struct lu_env *env,
 	if (!pg->cp_sync_io)
 		set_page_writeback(vmpage);
 
-	vvp_write_pending(cl2vvp(slice->cpl_obj), cl2vvp_page(slice));
-
 	return 0;
 }
 
@@ -287,19 +282,6 @@ static void vvp_page_completion_write(const struct lu_env *env,
 
 	CL_PAGE_HEADER(D_PAGE, env, pg, "completing WRITE with %d\n", ioret);
 
-	/*
-	 * TODO: Actually it makes sense to add the page into oap pending
-	 * list again and so that we don't need to take the page out from
-	 * SoM write pending list, if we just meet a recoverable error,
-	 * -ENOMEM, etc.
-	 * To implement this, we just need to return a non zero value in
-	 * ->cpo_completion method. The underlying transfer should be notified
-	 * and then re-add the page into pending transfer queue.  -jay
-	 */
-
-	vpg->vpg_write_queued = 0;
-	vvp_write_complete(cl2vvp(slice->cpl_obj), vpg);
-
 	if (pg->cp_sync_io) {
 		LASSERT(PageLocked(vmpage));
 		LASSERT(!PageWriteback(vmpage));
@@ -341,7 +323,6 @@ static int vvp_page_make_ready(const struct lu_env *env,
 		LASSERT(pg->cp_state == CPS_CACHED);
 		/* This actually clears the dirty bit in the radix tree. */
 		set_page_writeback(vmpage);
-		vvp_write_pending(cl2vvp(slice->cpl_obj), cl2vvp_page(slice));
 		CL_PAGE_HEADER(D_PAGE, env, pg, "readied\n");
 	} else if (pg->cp_state == CPS_PAGEOUT) {
 		/* is it possible for osc_flush_async_page() to already
@@ -357,20 +338,6 @@ static int vvp_page_make_ready(const struct lu_env *env,
 	return result;
 }
 
-static int vvp_page_is_under_lock(const struct lu_env *env,
-				  const struct cl_page_slice *slice,
-				  struct cl_io *io, pgoff_t *max_index)
-{
-	if (io->ci_type == CIT_READ || io->ci_type == CIT_WRITE ||
-	    io->ci_type == CIT_FAULT) {
-		struct vvp_io *vio = vvp_env_io(env);
-
-		if (unlikely(vio->vui_fd->fd_flags & LL_FILE_GROUP_LOCKED))
-			*max_index = CL_PAGE_EOF;
-	}
-	return 0;
-}
-
 static int vvp_page_print(const struct lu_env *env,
 			  const struct cl_page_slice *slice,
 			  void *cookie, lu_printer_t printer)
@@ -378,9 +345,8 @@ static int vvp_page_print(const struct lu_env *env,
 	struct vvp_page *vpg = cl2vvp_page(slice);
 	struct page     *vmpage = vpg->vpg_page;
 
-	(*printer)(env, cookie, LUSTRE_VVP_NAME "-page@%p(%d:%d:%d) vm@%p ",
-		   vpg, vpg->vpg_defer_uptodate, vpg->vpg_ra_used,
-		   vpg->vpg_write_queued, vmpage);
+	(*printer)(env, cookie, LUSTRE_VVP_NAME "-page@%p(%d:%d) vm@%p ",
+		   vpg, vpg->vpg_defer_uptodate, vpg->vpg_ra_used, vmpage);
 	if (vmpage) {
 		(*printer)(env, cookie, "%lx %d:%d %lx %lu %slru",
 			   (long)vmpage->flags, page_count(vmpage),
@@ -416,7 +382,6 @@ static const struct cl_page_operations vvp_page_ops = {
 	.cpo_is_vmlocked   = vvp_page_is_vmlocked,
 	.cpo_fini	  = vvp_page_fini,
 	.cpo_print	 = vvp_page_print,
-	.cpo_is_under_lock = vvp_page_is_under_lock,
 	.io = {
 		[CRT_READ] = {
 			.cpo_prep	= vvp_page_prep_read,
@@ -515,7 +480,6 @@ static const struct cl_page_operations vvp_transient_page_ops = {
 	.cpo_fini	  = vvp_transient_page_fini,
 	.cpo_is_vmlocked   = vvp_transient_page_is_vmlocked,
 	.cpo_print	 = vvp_page_print,
-	.cpo_is_under_lock	= vvp_page_is_under_lock,
 	.io = {
 		[CRT_READ] = {
 			.cpo_prep	= vvp_transient_page_prep,
@@ -539,7 +503,6 @@ int vvp_page_init(const struct lu_env *env, struct cl_object *obj,
 	vpg->vpg_page = vmpage;
 	get_page(vmpage);
 
-	INIT_LIST_HEAD(&vpg->vpg_pending_linkage);
 	if (page->cp_type == CPT_CACHEABLE) {
 		/* in cache, decref in vvp_page_delete */
 		atomic_inc(&page->cp_ref);
diff --git a/drivers/staging/lustre/lustre/llite/vvp_req.c b/drivers/staging/lustre/lustre/llite/vvp_req.c
deleted file mode 100644
index e3f4c790d646..000000000000
--- a/drivers/staging/lustre/lustre/llite/vvp_req.c
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- *
- * Copyright (c) 2011, 2014, Intel Corporation.
- */
-
-#define DEBUG_SUBSYSTEM S_LLITE
-
-#include "../include/lustre/lustre_idl.h"
-#include "../include/cl_object.h"
-#include "../include/obd.h"
-#include "../include/obd_support.h"
-#include "llite_internal.h"
-#include "vvp_internal.h"
-
-static inline struct vvp_req *cl2vvp_req(const struct cl_req_slice *slice)
-{
-	return container_of0(slice, struct vvp_req, vrq_cl);
-}
-
-/**
- * Implementation of struct cl_req_operations::cro_attr_set() for VVP
- * layer. VVP is responsible for
- *
- *    - o_[mac]time
- *
- *    - o_mode
- *
- *    - o_parent_seq
- *
- *    - o_[ug]id
- *
- *    - o_parent_oid
- *
- *    - o_parent_ver
- *
- *    - o_ioepoch,
- *
- */
-static void vvp_req_attr_set(const struct lu_env *env,
-			     const struct cl_req_slice *slice,
-			     const struct cl_object *obj,
-			     struct cl_req_attr *attr, u64 flags)
-{
-	struct inode *inode;
-	struct obdo  *oa;
-	u32	      valid_flags;
-
-	oa = attr->cra_oa;
-	inode = vvp_object_inode(obj);
-	valid_flags = OBD_MD_FLTYPE;
-
-	if (slice->crs_req->crq_type == CRT_WRITE) {
-		if (flags & OBD_MD_FLEPOCH) {
-			oa->o_valid |= OBD_MD_FLEPOCH;
-			oa->o_ioepoch = ll_i2info(inode)->lli_ioepoch;
-			valid_flags |= OBD_MD_FLMTIME | OBD_MD_FLCTIME |
-				       OBD_MD_FLUID | OBD_MD_FLGID;
-		}
-	}
-	obdo_from_inode(oa, inode, valid_flags & flags);
-	obdo_set_parent_fid(oa, &ll_i2info(inode)->lli_fid);
-	if (OBD_FAIL_CHECK(OBD_FAIL_LFSCK_INVALID_PFID))
-		oa->o_parent_oid++;
-	memcpy(attr->cra_jobid, ll_i2info(inode)->lli_jobid,
-	       LUSTRE_JOBID_SIZE);
-}
-
-static void vvp_req_completion(const struct lu_env *env,
-			       const struct cl_req_slice *slice, int ioret)
-{
-	struct vvp_req *vrq;
-
-	if (ioret > 0)
-		cl_stats_tally(slice->crs_dev, slice->crs_req->crq_type, ioret);
-
-	vrq = cl2vvp_req(slice);
-	kmem_cache_free(vvp_req_kmem, vrq);
-}
-
-static const struct cl_req_operations vvp_req_ops = {
-	.cro_attr_set   = vvp_req_attr_set,
-	.cro_completion = vvp_req_completion
-};
-
-int vvp_req_init(const struct lu_env *env, struct cl_device *dev,
-		 struct cl_req *req)
-{
-	struct vvp_req *vrq;
-	int result;
-
-	vrq = kmem_cache_zalloc(vvp_req_kmem, GFP_NOFS);
-	if (vrq) {
-		cl_req_slice_add(req, &vrq->vrq_cl, dev, &vvp_req_ops);
-		result = 0;
-	} else {
-		result = -ENOMEM;
-	}
-	return result;
-}
diff --git a/drivers/staging/lustre/lustre/llite/xattr.c b/drivers/staging/lustre/lustre/llite/xattr.c
index e070adb7a3cc..7a848ebc57c1 100644
--- a/drivers/staging/lustre/lustre/llite/xattr.c
+++ b/drivers/staging/lustre/lustre/llite/xattr.c
@@ -44,48 +44,39 @@
 
 #include "llite_internal.h"
 
-static
-int get_xattr_type(const char *name)
+const struct xattr_handler *get_xattr_type(const char *name)
 {
-	if (!strcmp(name, XATTR_NAME_POSIX_ACL_ACCESS))
-		return XATTR_ACL_ACCESS_T;
+	int i = 0;
 
-	if (!strcmp(name, XATTR_NAME_POSIX_ACL_DEFAULT))
-		return XATTR_ACL_DEFAULT_T;
+	while (ll_xattr_handlers[i]) {
+		size_t len = strlen(ll_xattr_handlers[i]->prefix);
 
-	if (!strncmp(name, XATTR_USER_PREFIX,
-		     sizeof(XATTR_USER_PREFIX) - 1))
-		return XATTR_USER_T;
-
-	if (!strncmp(name, XATTR_TRUSTED_PREFIX,
-		     sizeof(XATTR_TRUSTED_PREFIX) - 1))
-		return XATTR_TRUSTED_T;
-
-	if (!strncmp(name, XATTR_SECURITY_PREFIX,
-		     sizeof(XATTR_SECURITY_PREFIX) - 1))
-		return XATTR_SECURITY_T;
-
-	if (!strncmp(name, XATTR_LUSTRE_PREFIX,
-		     sizeof(XATTR_LUSTRE_PREFIX) - 1))
-		return XATTR_LUSTRE_T;
-
-	return XATTR_OTHER_T;
+		if (!strncmp(ll_xattr_handlers[i]->prefix, name, len))
+			return ll_xattr_handlers[i];
+		i++;
+	}
+	return NULL;
 }
 
-static
-int xattr_type_filter(struct ll_sb_info *sbi, int xattr_type)
+static int xattr_type_filter(struct ll_sb_info *sbi,
+			     const struct xattr_handler *handler)
 {
-	if ((xattr_type == XATTR_ACL_ACCESS_T ||
-	     xattr_type == XATTR_ACL_DEFAULT_T) &&
+	/* No handler means XATTR_OTHER_T */
+	if (!handler)
+		return -EOPNOTSUPP;
+
+	if ((handler->flags == XATTR_ACL_ACCESS_T ||
+	     handler->flags == XATTR_ACL_DEFAULT_T) &&
 	   !(sbi->ll_flags & LL_SBI_ACL))
 		return -EOPNOTSUPP;
 
-	if (xattr_type == XATTR_USER_T && !(sbi->ll_flags & LL_SBI_USER_XATTR))
+	if (handler->flags == XATTR_USER_T &&
+	    !(sbi->ll_flags & LL_SBI_USER_XATTR))
 		return -EOPNOTSUPP;
-	if (xattr_type == XATTR_TRUSTED_T && !capable(CFS_CAP_SYS_ADMIN))
+
+	if (handler->flags == XATTR_TRUSTED_T &&
+	    !capable(CFS_CAP_SYS_ADMIN))
 		return -EPERM;
-	if (xattr_type == XATTR_OTHER_T)
-		return -EOPNOTSUPP;
 
 	return 0;
 }
@@ -111,7 +102,7 @@ ll_xattr_set_common(const struct xattr_handler *handler,
 		valid = OBD_MD_FLXATTR;
 	}
 
-	rc = xattr_type_filter(sbi, handler->flags);
+	rc = xattr_type_filter(sbi, handler);
 	if (rc)
 		return rc;
 
@@ -121,8 +112,9 @@ ll_xattr_set_common(const struct xattr_handler *handler,
 		return -EPERM;
 
 	/* b10667: ignore lustre special xattr for now */
-	if ((handler->flags == XATTR_TRUSTED_T && !strcmp(name, "lov")) ||
-	    (handler->flags == XATTR_LUSTRE_T && !strcmp(name, "lov")))
+	if (!strcmp(name, "hsm") ||
+	    ((handler->flags == XATTR_TRUSTED_T && !strcmp(name, "lov")) ||
+	     (handler->flags == XATTR_LUSTRE_T && !strcmp(name, "lov"))))
 		return 0;
 
 	/* b15587: ignore security.capability xattr for now */
@@ -135,6 +127,11 @@ ll_xattr_set_common(const struct xattr_handler *handler,
 	    strcmp(name, "selinux") == 0)
 		return -EOPNOTSUPP;
 
+	/*FIXME: enable IMA when the conditions are ready */
+	if (handler->flags == XATTR_SECURITY_T &&
+	    (!strcmp(name, "ima") || !strcmp(name, "evm")))
+		return -EOPNOTSUPP;
+
 	sprintf(fullname, "%s%s\n", handler->prefix, name);
 	rc = md_setxattr(sbi->ll_md_exp, ll_inode2fid(inode),
 			 valid, fullname, pv, size, 0, flags,
@@ -151,6 +148,37 @@ ll_xattr_set_common(const struct xattr_handler *handler,
 	return 0;
 }
 
+static int get_hsm_state(struct inode *inode, u32 *hus_states)
+{
+	struct md_op_data *op_data;
+	struct hsm_user_state *hus;
+	int rc;
+
+	hus = kzalloc(sizeof(*hus), GFP_NOFS);
+	if (!hus)
+		return -ENOMEM;
+
+	op_data = ll_prep_md_op_data(NULL, inode, NULL, NULL, 0, 0,
+				     LUSTRE_OPC_ANY, hus);
+	if (!IS_ERR(op_data)) {
+		rc = obd_iocontrol(LL_IOC_HSM_STATE_GET, ll_i2mdexp(inode),
+				   sizeof(*op_data), op_data, NULL);
+		if (!rc)
+			*hus_states = hus->hus_states;
+		else
+			CDEBUG(D_VFSTRACE, "obd_iocontrol failed. rc = %d\n",
+			       rc);
+
+		ll_finish_md_op_data(op_data);
+	} else {
+		rc = PTR_ERR(op_data);
+		CDEBUG(D_VFSTRACE, "Could not prepare the opdata. rc = %d\n",
+		       rc);
+	}
+	kfree(hus);
+	return rc;
+}
+
 static int ll_xattr_set(const struct xattr_handler *handler,
 			struct dentry *dentry, struct inode *inode,
 			const char *name, const void *value, size_t size,
@@ -187,6 +215,31 @@ static int ll_xattr_set(const struct xattr_handler *handler,
 		if (lump && lump->lmm_stripe_offset == 0)
 			lump->lmm_stripe_offset = -1;
 
+		/* Avoid anyone directly setting the RELEASED flag. */
+		if (lump && (lump->lmm_pattern & LOV_PATTERN_F_RELEASED)) {
+			/* Only if we have a released flag check if the file
+			 * was indeed archived.
+			 */
+			u32 state = HS_NONE;
+
+			rc = get_hsm_state(inode, &state);
+			if (rc)
+				return rc;
+
+			if (!(state & HS_ARCHIVED)) {
+				CDEBUG(D_VFSTRACE,
+				       "hus_states state = %x, pattern = %x\n",
+				state, lump->lmm_pattern);
+				/*
+				 * Here the state is: real file is not
+				 * archived but user is requesting to set
+				 * the RELEASED flag so we mask off the
+				 * released flag from the request
+				 */
+				lump->lmm_pattern ^= LOV_PATTERN_F_RELEASED;
+			}
+		}
+
 		if (lump && S_ISREG(inode->i_mode)) {
 			__u64 it_flags = FMODE_WRITE;
 			int lum_size;
@@ -225,7 +278,8 @@ ll_xattr_list(struct inode *inode, const char *name, int type, void *buffer,
 	void *xdata;
 	int rc;
 
-	if (sbi->ll_xattr_cache_enabled && type != XATTR_ACL_ACCESS_T) {
+	if (sbi->ll_xattr_cache_enabled && type != XATTR_ACL_ACCESS_T &&
+	    (type != XATTR_SECURITY_T || strcmp(name, "security.selinux"))) {
 		rc = ll_xattr_cache_get(inode, name, buffer, size, valid);
 		if (rc == -EAGAIN)
 			goto getxattr_nocache;
@@ -313,7 +367,7 @@ static int ll_xattr_get_common(const struct xattr_handler *handler,
 
 	ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_GETXATTR, 1);
 
-	rc = xattr_type_filter(sbi, handler->flags);
+	rc = xattr_type_filter(sbi, handler);
 	if (rc)
 		return rc;
 
@@ -353,80 +407,99 @@ static int ll_xattr_get_common(const struct xattr_handler *handler,
 			     OBD_MD_FLXATTR);
 }
 
-static int ll_xattr_get(const struct xattr_handler *handler,
-			struct dentry *dentry, struct inode *inode,
-			const char *name, void *buffer, size_t size)
+static ssize_t ll_getxattr_lov(struct inode *inode, void *buf, size_t buf_size)
 {
-	LASSERT(inode);
-	LASSERT(name);
+	ssize_t rc;
 
-	CDEBUG(D_VFSTRACE, "VFS Op:inode="DFID"(%p), xattr %s\n",
-	       PFID(ll_inode2fid(inode)), inode, name);
+	if (S_ISREG(inode->i_mode)) {
+		struct cl_object *obj = ll_i2info(inode)->lli_clob;
+		struct cl_layout cl = {
+			.cl_buf.lb_buf = buf,
+			.cl_buf.lb_len = buf_size,
+		};
+		struct lu_env *env;
+		int refcheck;
+
+		if (!obj)
+			return -ENODATA;
 
-	if (!strcmp(name, "lov")) {
-		struct lov_stripe_md *lsm;
-		struct lov_user_md *lump;
-		struct lov_mds_md *lmm = NULL;
-		struct ptlrpc_request *request = NULL;
-		int rc = 0, lmmsize = 0;
+		env = cl_env_get(&refcheck);
+		if (IS_ERR(env))
+			return PTR_ERR(env);
 
-		ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_GETXATTR, 1);
-
-		if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode))
-			return -ENODATA;
+		rc = cl_object_layout_get(env, obj, &cl);
+		if (rc < 0)
+			goto out_env;
 
-		lsm = ccc_inode_lsm_get(inode);
-		if (!lsm) {
-			if (S_ISDIR(inode->i_mode)) {
-				rc = ll_dir_getstripe(inode, (void **)&lmm,
-						      &lmmsize, &request, 0);
-			} else {
-				rc = -ENODATA;
-			}
-		} else {
-			/* LSM is present already after lookup/getattr call.
-			 * we need to grab layout lock once it is implemented
-			 */
-			rc = obd_packmd(ll_i2dtexp(inode), &lmm, lsm);
-			lmmsize = rc;
+		if (!cl.cl_size) {
+			rc = -ENODATA;
+			goto out_env;
 		}
-		ccc_inode_lsm_put(inode, lsm);
 
+		rc = cl.cl_size;
+
+		if (!buf_size)
+			goto out_env;
+
+		LASSERT(buf && rc <= buf_size);
+
+		/*
+		 * Do not return layout gen for getxattr() since
+		 * otherwise it would confuse tar --xattr by
+		 * recognizing layout gen as stripe offset when the
+		 * file is restored. See LU-2809.
+		 */
+		((struct lov_mds_md *)buf)->lmm_layout_gen = 0;
+out_env:
+		cl_env_put(env, &refcheck);
+
+		return rc;
+	} else if (S_ISDIR(inode->i_mode)) {
+		struct ptlrpc_request *req = NULL;
+		struct lov_mds_md *lmm = NULL;
+		int lmm_size = 0;
+
+		rc = ll_dir_getstripe(inode, (void **)&lmm, &lmm_size,
+				      &req, 0);
 		if (rc < 0)
-			goto out;
+			goto out_req;
 
-		if (size == 0) {
-			/* used to call ll_get_max_mdsize() forward to get
-			 * the maximum buffer size, while some apps (such as
-			 * rsync 3.0.x) care much about the exact xattr value
-			 * size
-			 */
-			rc = lmmsize;
-			goto out;
+		if (!buf_size) {
+			rc = lmm_size;
+			goto out_req;
 		}
 
-		if (size < lmmsize) {
-			CERROR("server bug: replied size %d > %d for %pd (%s)\n",
-			       lmmsize, (int)size, dentry, name);
+		if (buf_size < lmm_size) {
 			rc = -ERANGE;
-			goto out;
+			goto out_req;
 		}
 
-		lump = buffer;
-		memcpy(lump, lmm, lmmsize);
-		/* do not return layout gen for getxattr otherwise it would
-		 * confuse tar --xattr by recognizing layout gen as stripe
-		 * offset when the file is restored. See LU-2809.
-		 */
-		lump->lmm_layout_gen = 0;
+		memcpy(buf, lmm, lmm_size);
+		rc = lmm_size;
+out_req:
+		if (req)
+			ptlrpc_req_finished(req);
 
-		rc = lmmsize;
-out:
-		if (request)
-			ptlrpc_req_finished(request);
-		else if (lmm)
-			obd_free_diskmd(ll_i2dtexp(inode), &lmm);
 		return rc;
+	} else {
+		return -ENODATA;
+	}
+}
+
+static int ll_xattr_get(const struct xattr_handler *handler,
+			struct dentry *dentry, struct inode *inode,
+			const char *name, void *buffer, size_t size)
+{
+	LASSERT(inode);
+	LASSERT(name);
+
+	CDEBUG(D_VFSTRACE, "VFS Op:inode=" DFID "(%p), xattr %s\n",
+	       PFID(ll_inode2fid(inode)), inode, name);
+
+	if (!strcmp(name, "lov")) {
+		ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_GETXATTR, 1);
+
+		return ll_getxattr_lov(inode, buffer, size);
 	}
 
 	return ll_xattr_get_common(handler, dentry, inode, name, buffer, size);
@@ -435,10 +508,10 @@ out:
 ssize_t ll_listxattr(struct dentry *dentry, char *buffer, size_t size)
 {
 	struct inode *inode = d_inode(dentry);
-	int rc = 0, rc2 = 0;
-	struct lov_mds_md *lmm = NULL;
-	struct ptlrpc_request *request = NULL;
-	int lmmsize;
+	struct ll_sb_info *sbi = ll_i2sbi(inode);
+	char *xattr_name;
+	ssize_t rc, rc2;
+	size_t len, rem;
 
 	LASSERT(inode);
 
@@ -450,65 +523,48 @@ ssize_t ll_listxattr(struct dentry *dentry, char *buffer, size_t size)
 	rc = ll_xattr_list(inode, NULL, XATTR_OTHER_T, buffer, size,
 			   OBD_MD_FLXATTRLS);
 	if (rc < 0)
-		goto out;
-
-	if (buffer) {
-		struct ll_sb_info *sbi = ll_i2sbi(inode);
-		char *xattr_name = buffer;
-		int xlen, rem = rc;
-
-		while (rem > 0) {
-			xlen = strnlen(xattr_name, rem - 1) + 1;
-			rem -= xlen;
-			if (xattr_type_filter(sbi,
-					get_xattr_type(xattr_name)) == 0) {
-				/* skip OK xattr type
-				 * leave it in buffer
-				 */
-				xattr_name += xlen;
-				continue;
-			}
-			/* move up remaining xattrs in buffer
-			 * removing the xattr that is not OK
-			 */
-			memmove(xattr_name, xattr_name + xlen, rem);
-			rc -= xlen;
+		return rc;
+	/*
+	 * If we're being called to get the size of the xattr list
+	 * (buf_size == 0) then just assume that a lustre.lov xattr
+	 * exists.
+	 */
+	if (!size)
+		return rc + sizeof(XATTR_LUSTRE_LOV);
+
+	xattr_name = buffer;
+	rem = rc;
+
+	while (rem > 0) {
+		len = strnlen(xattr_name, rem - 1) + 1;
+		rem -= len;
+		if (!xattr_type_filter(sbi, get_xattr_type(xattr_name))) {
+			/* Skip OK xattr type leave it in buffer */
+			xattr_name += len;
+			continue;
 		}
-	}
-	if (S_ISREG(inode->i_mode)) {
-		if (!ll_i2info(inode)->lli_has_smd)
-			rc2 = -1;
-	} else if (S_ISDIR(inode->i_mode)) {
-		rc2 = ll_dir_getstripe(inode, (void **)&lmm, &lmmsize,
-				       &request, 0);
+
+		/*
+		 * Move up remaining xattrs in buffer
+		 * removing the xattr that is not OK
+		 */
+		memmove(xattr_name, xattr_name + len, rem);
+		rc -= len;
 	}
 
-	if (rc2 < 0) {
-		rc2 = 0;
-		goto out;
-	} else if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode)) {
-		const int prefix_len = sizeof(XATTR_LUSTRE_PREFIX) - 1;
-		const size_t name_len   = sizeof("lov") - 1;
-		const size_t total_len  = prefix_len + name_len + 1;
-
-		if (((rc + total_len) > size) && buffer) {
-			ptlrpc_req_finished(request);
-			return -ERANGE;
-		}
+	rc2 = ll_getxattr_lov(inode, NULL, 0);
+	if (rc2 == -ENODATA)
+		return rc;
 
-		if (buffer) {
-			buffer += rc;
-			memcpy(buffer, XATTR_LUSTRE_PREFIX, prefix_len);
-			memcpy(buffer + prefix_len, "lov", name_len);
-			buffer[prefix_len + name_len] = '\0';
-		}
-		rc2 = total_len;
-	}
-out:
-	ptlrpc_req_finished(request);
-	rc = rc + rc2;
+	if (rc2 < 0)
+		return rc2;
 
-	return rc;
+	if (size < rc + sizeof(XATTR_LUSTRE_LOV))
+		return -ERANGE;
+
+	memcpy(buffer + rc, XATTR_LUSTRE_LOV, sizeof(XATTR_LUSTRE_LOV));
+
+	return rc + sizeof(XATTR_LUSTRE_LOV);
 }
 
 static const struct xattr_handler ll_user_xattr_handler = {
diff --git a/drivers/staging/lustre/lustre/llite/xattr_cache.c b/drivers/staging/lustre/lustre/llite/xattr_cache.c
index 50a19a40bd4e..38f75f6aa887 100644
--- a/drivers/staging/lustre/lustre/llite/xattr_cache.c
+++ b/drivers/staging/lustre/lustre/llite/xattr_cache.c
@@ -26,8 +26,8 @@ struct ll_xattr_entry {
 					     */
 	char			*xe_name;   /* xattr name, \0-terminated */
 	char			*xe_value;  /* xattr value */
-	unsigned		xe_namelen; /* strlen(xe_name) + 1 */
-	unsigned		xe_vallen;  /* xattr value length */
+	unsigned int		xe_namelen; /* strlen(xe_name) + 1 */
+	unsigned int		xe_vallen;  /* xattr value length */
 };
 
 static struct kmem_cache *xattr_kmem;
@@ -60,7 +60,7 @@ void ll_xattr_fini(void)
 static void ll_xattr_cache_init(struct ll_inode_info *lli)
 {
 	INIT_LIST_HEAD(&lli->lli_xattrs);
-	lli->lli_flags |= LLIF_XATTR_CACHE;
+	set_bit(LLIF_XATTR_CACHE, &lli->lli_flags);
 }
 
 /**
@@ -104,7 +104,7 @@ static int ll_xattr_cache_find(struct list_head *cache,
 static int ll_xattr_cache_add(struct list_head *cache,
 			      const char *xattr_name,
 			      const char *xattr_val,
-			      unsigned xattr_val_len)
+			      unsigned int xattr_val_len)
 {
 	struct ll_xattr_entry *xattr;
 
@@ -216,7 +216,7 @@ static int ll_xattr_cache_list(struct list_head *cache,
  */
 static int ll_xattr_cache_valid(struct ll_inode_info *lli)
 {
-	return !!(lli->lli_flags & LLIF_XATTR_CACHE);
+	return test_bit(LLIF_XATTR_CACHE, &lli->lli_flags);
 }
 
 /**
@@ -233,7 +233,8 @@ static int ll_xattr_cache_destroy_locked(struct ll_inode_info *lli)
 
 	while (ll_xattr_cache_del(&lli->lli_xattrs, NULL) == 0)
 		; /* empty loop */
-	lli->lli_flags &= ~LLIF_XATTR_CACHE;
+
+	clear_bit(LLIF_XATTR_CACHE, &lli->lli_flags);
 
 	return 0;
 }
@@ -415,6 +416,10 @@ static int ll_xattr_cache_refill(struct inode *inode, struct lookup_intent *oit)
 			CDEBUG(D_CACHE, "not caching %s\n",
 			       XATTR_NAME_ACL_ACCESS);
 			rc = 0;
+		} else if (!strcmp(xdata, "security.selinux")) {
+			/* Filter out security.selinux, it is cached in slab */
+			CDEBUG(D_CACHE, "not caching security.selinux\n");
+			rc = 0;
 		} else {
 			rc = ll_xattr_cache_add(&lli->lli_xattrs, xdata, xval,
 						*xsizes);
diff --git a/drivers/staging/lustre/lustre/llite/xattr_security.c b/drivers/staging/lustre/lustre/llite/xattr_security.c
new file mode 100644
index 000000000000..d61d8018001a
--- /dev/null
+++ b/drivers/staging/lustre/lustre/llite/xattr_security.c
@@ -0,0 +1,88 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see http://www.gnu.org/licenses
+ *
+ * GPL HEADER END
+ */
+
+/*
+ * Copyright (c) 2014 Bull SAS
+ * Author: Sebastien Buisson sebastien.buisson@bull.net
+ */
+
+/*
+ * lustre/llite/xattr_security.c
+ * Handler for storing security labels as extended attributes.
+ */
+#include <linux/security.h>
+#include <linux/xattr.h>
+#include "llite_internal.h"
+
+/**
+ * A helper function for ll_security_inode_init_security()
+ * that takes care of setting xattrs
+ *
+ * Get security context of @inode from @xattr_array,
+ * and put it in 'security.xxx' xattr of dentry
+ * stored in @fs_info.
+ *
+ * \retval 0        success
+ * \retval -ENOMEM  if no memory could be allocated for xattr name
+ * \retval < 0      failure to set xattr
+ */
+static int
+ll_initxattrs(struct inode *inode, const struct xattr *xattr_array,
+	      void *fs_info)
+{
+	const struct xattr_handler *handler;
+	struct dentry *dentry = fs_info;
+	const struct xattr *xattr;
+	int err = 0;
+
+	handler = get_xattr_type(XATTR_SECURITY_PREFIX);
+	if (!handler)
+		return -ENXIO;
+
+	for (xattr = xattr_array; xattr->name; xattr++) {
+		err = handler->set(handler, dentry, inode, xattr->name,
+				   xattr->value, xattr->value_len,
+				   XATTR_CREATE);
+		if (err < 0)
+			break;
+	}
+	return err;
+}
+
+/**
+ * Initializes security context
+ *
+ * Get security context of @inode in @dir,
+ * and put it in 'security.xxx' xattr of @dentry.
+ *
+ * \retval 0        success, or SELinux is disabled
+ * \retval -ENOMEM  if no memory could be allocated for xattr name
+ * \retval < 0      failure to get security context or set xattr
+ */
+int
+ll_init_security(struct dentry *dentry, struct inode *inode, struct inode *dir)
+{
+	if (!selinux_is_enabled())
+		return 0;
+
+	return security_inode_init_security(inode, dir, NULL,
+					    &ll_initxattrs, dentry);
+}
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_intent.c b/drivers/staging/lustre/lustre/lmv/lmv_intent.c
index 9f4e826bb0af..b1071cf5a70c 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_intent.c
+++ b/drivers/staging/lustre/lustre/lmv/lmv_intent.c
@@ -223,7 +223,14 @@ int lmv_revalidate_slaves(struct obd_export *exp,
 			LASSERT(body);
 
 			if (unlikely(body->mbo_nlink < 2)) {
-				CERROR("%s: nlink %d < 2 corrupt stripe %d "DFID":" DFID"\n",
+				/*
+				 * If this is bad stripe, most likely due
+				 * to the race between close(unlink) and
+				 * getattr, let's return -EONENT, so llite
+				 * will revalidate the dentry see
+				 * ll_inode_revalidate_fini()
+				 */
+				CDEBUG(D_INODE, "%s: nlink %d < 2 corrupt stripe %d "DFID":" DFID"\n",
 				       obd->obd_name, body->mbo_nlink, i,
 				       PFID(&lsm->lsm_md_oinfo[i].lmo_fid),
 				       PFID(&lsm->lsm_md_oinfo[0].lmo_fid));
@@ -233,7 +240,7 @@ int lmv_revalidate_slaves(struct obd_export *exp,
 					it.it_lock_mode = 0;
 				}
 
-				rc = -EIO;
+				rc = -ENOENT;
 				goto cleanup;
 			}
 
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_internal.h b/drivers/staging/lustre/lustre/lmv/lmv_internal.h
index 52b03745ac19..12731a17e263 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_internal.h
+++ b/drivers/staging/lustre/lustre/lmv/lmv_internal.h
@@ -54,9 +54,6 @@ int __lmv_fid_alloc(struct lmv_obd *lmv, struct lu_fid *fid, u32 mds);
 int lmv_fid_alloc(const struct lu_env *env, struct obd_export *exp,
 		  struct lu_fid *fid, struct md_op_data *op_data);
 
-int lmv_unpack_md(struct obd_export *exp, struct lmv_stripe_md **lsmp,
-		  const union lmv_mds_md *lmm, int stripe_count);
-
 int lmv_revalidate_slaves(struct obd_export *exp,
 			  const struct lmv_stripe_md *lsm,
 			  ldlm_blocking_callback cb_blocking,
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_obd.c b/drivers/staging/lustre/lustre/lmv/lmv_obd.c
index 7dbb2b946acf..f124f6c05ea4 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_obd.c
+++ b/drivers/staging/lustre/lustre/lmv/lmv_obd.c
@@ -62,6 +62,7 @@ static void lmv_activate_target(struct lmv_obd *lmv,
 
 	tgt->ltd_active = activate;
 	lmv->desc.ld_active_tgt_count += (activate ? 1 : -1);
+	tgt->ltd_exp->exp_obd->obd_inactive = !activate;
 }
 
 /**
@@ -245,8 +246,7 @@ static int lmv_connect(const struct lu_env *env,
 	return rc;
 }
 
-static int lmv_init_ea_size(struct obd_export *exp, u32 easize, u32 def_easize,
-			    u32 cookiesize, u32 def_cookiesize)
+static int lmv_init_ea_size(struct obd_export *exp, u32 easize, u32 def_easize)
 {
 	struct obd_device   *obd = exp->exp_obd;
 	struct lmv_obd      *lmv = &obd->u.lmv;
@@ -262,14 +262,7 @@ static int lmv_init_ea_size(struct obd_export *exp, u32 easize, u32 def_easize,
 		lmv->max_def_easize = def_easize;
 		change = 1;
 	}
-	if (lmv->max_cookiesize < cookiesize) {
-		lmv->max_cookiesize = cookiesize;
-		change = 1;
-	}
-	if (lmv->max_def_cookiesize < def_cookiesize) {
-		lmv->max_def_cookiesize = def_cookiesize;
-		change = 1;
-	}
+
 	if (change == 0)
 		return 0;
 
@@ -284,8 +277,7 @@ static int lmv_init_ea_size(struct obd_export *exp, u32 easize, u32 def_easize,
 			continue;
 		}
 
-		rc = md_init_ea_size(tgt->ltd_exp, easize, def_easize,
-				     cookiesize, def_cookiesize);
+		rc = md_init_ea_size(tgt->ltd_exp, easize, def_easize);
 		if (rc) {
 			CERROR("%s: obd_init_ea_size() failed on MDT target %d: rc = %d\n",
 			       obd->obd_name, i, rc);
@@ -368,8 +360,7 @@ static int lmv_connect_mdc(struct obd_device *obd, struct lmv_tgt_desc *tgt)
 	tgt->ltd_exp = mdc_exp;
 	lmv->desc.ld_active_tgt_count++;
 
-	md_init_ea_size(tgt->ltd_exp, lmv->max_easize, lmv->max_def_easize,
-			lmv->max_cookiesize, lmv->max_def_cookiesize);
+	md_init_ea_size(tgt->ltd_exp, lmv->max_easize, lmv->max_def_easize);
 
 	CDEBUG(D_CONFIG, "Connected to %s(%s) successfully (%d)\n",
 	       mdc_obd->obd_name, mdc_obd->obd_uuid.uuid,
@@ -396,27 +387,23 @@ static int lmv_add_target(struct obd_device *obd, struct obd_uuid *uuidp,
 			  __u32 index, int gen)
 {
 	struct lmv_obd      *lmv = &obd->u.lmv;
+	struct obd_device *mdc_obd;
 	struct lmv_tgt_desc *tgt;
 	int orig_tgt_count = 0;
 	int		  rc = 0;
 
 	CDEBUG(D_CONFIG, "Target uuid: %s. index %d\n", uuidp->uuid, index);
 
-	mutex_lock(&lmv->lmv_init_mutex);
-
-	if (lmv->desc.ld_tgt_count == 0) {
-		struct obd_device *mdc_obd;
-
-		mdc_obd = class_find_client_obd(uuidp, LUSTRE_MDC_NAME,
-						&obd->obd_uuid);
-		if (!mdc_obd) {
-			mutex_unlock(&lmv->lmv_init_mutex);
-			CERROR("%s: Target %s not attached: rc = %d\n",
-			       obd->obd_name, uuidp->uuid, -EINVAL);
-			return -EINVAL;
-		}
+	mdc_obd = class_find_client_obd(uuidp, LUSTRE_MDC_NAME,
+					&obd->obd_uuid);
+	if (!mdc_obd) {
+		CERROR("%s: Target %s not attached: rc = %d\n",
+		       obd->obd_name, uuidp->uuid, -EINVAL);
+		return -EINVAL;
 	}
 
+	mutex_lock(&lmv->lmv_init_mutex);
+
 	if ((index < lmv->tgts_size) && lmv->tgts[index]) {
 		tgt = lmv->tgts[index];
 		CERROR("%s: UUID %s already assigned at LOV target index %d: rc = %d\n",
@@ -472,22 +459,27 @@ static int lmv_add_target(struct obd_device *obd, struct obd_uuid *uuidp,
 		lmv->desc.ld_tgt_count = index + 1;
 	}
 
-	if (lmv->connected) {
-		rc = lmv_connect_mdc(obd, tgt);
-		if (rc) {
-			spin_lock(&lmv->lmv_lock);
-			if (lmv->desc.ld_tgt_count == index + 1)
-				lmv->desc.ld_tgt_count = orig_tgt_count;
-			memset(tgt, 0, sizeof(*tgt));
-			spin_unlock(&lmv->lmv_lock);
-		} else {
-			int easize = sizeof(struct lmv_stripe_md) +
-				lmv->desc.ld_tgt_count * sizeof(struct lu_fid);
-			lmv_init_ea_size(obd->obd_self_export, easize, 0, 0, 0);
-		}
+	if (!lmv->connected) {
+		/* lmv_check_connect() will connect this target. */
+		mutex_unlock(&lmv->lmv_init_mutex);
+		return rc;
 	}
 
+	/* Otherwise let's connect it ourselves */
 	mutex_unlock(&lmv->lmv_init_mutex);
+	rc = lmv_connect_mdc(obd, tgt);
+	if (rc) {
+		spin_lock(&lmv->lmv_lock);
+		if (lmv->desc.ld_tgt_count == index + 1)
+			lmv->desc.ld_tgt_count = orig_tgt_count;
+		memset(tgt, 0, sizeof(*tgt));
+		spin_unlock(&lmv->lmv_lock);
+	} else {
+		int easize = sizeof(struct lmv_stripe_md) +
+			     lmv->desc.ld_tgt_count * sizeof(struct lu_fid);
+		lmv_init_ea_size(obd->obd_self_export, easize, 0);
+	}
+
 	return rc;
 }
 
@@ -538,7 +530,7 @@ int lmv_check_connect(struct obd_device *obd)
 	class_export_put(lmv->exp);
 	lmv->connected = 1;
 	easize = lmv_mds_md_size(lmv->desc.ld_tgt_count, LMV_MAGIC);
-	lmv_init_ea_size(obd->obd_self_export, easize, 0, 0, 0);
+	lmv_init_ea_size(obd->obd_self_export, easize, 0);
 	mutex_unlock(&lmv->lmv_init_mutex);
 	return 0;
 
@@ -1128,9 +1120,7 @@ static int lmv_iocontrol(unsigned int cmd, struct obd_export *exp,
 			mdc_obd = class_exp2obd(tgt->ltd_exp);
 			mdc_obd->obd_force = obddev->obd_force;
 			err = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg);
-			if (err == -ENODATA && cmd == OBD_IOC_POLL_QUOTACHECK) {
-				return err;
-			} else if (err) {
+			if (err) {
 				if (tgt->ltd_active) {
 					CERROR("error: iocontrol MDC %s on MDTidx %d cmd %x: err = %d\n",
 					       tgt->ltd_uuid.uuid, i, cmd, err);
@@ -1284,7 +1274,6 @@ static int lmv_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
 	obd_str2uuid(&lmv->desc.ld_uuid, desc->ld_uuid.uuid);
 	lmv->desc.ld_tgt_count = 0;
 	lmv->desc.ld_active_tgt_count = 0;
-	lmv->max_cookiesize = 0;
 	lmv->max_def_easize = 0;
 	lmv->max_easize = 0;
 	lmv->lmv_placement = PLACEMENT_CHAR_POLICY;
@@ -1630,27 +1619,28 @@ lmv_locate_mds(struct lmv_obd *lmv, struct md_op_data *op_data,
 	 * ct_restore().
 	 */
 	if (op_data->op_bias & MDS_CREATE_VOLATILE &&
-	    (int)op_data->op_mds != -1 && lsm) {
+	    (int)op_data->op_mds != -1) {
 		int i;
 
 		tgt = lmv_get_target(lmv, op_data->op_mds, NULL);
 		if (IS_ERR(tgt))
 			return tgt;
 
-		/* refill the right parent fid */
-		for (i = 0; i < lsm->lsm_md_stripe_count; i++) {
-			struct lmv_oinfo *oinfo;
+		if (lsm) {
+			/* refill the right parent fid */
+			for (i = 0; i < lsm->lsm_md_stripe_count; i++) {
+				struct lmv_oinfo *oinfo;
 
-			oinfo = &lsm->lsm_md_oinfo[i];
-			if (oinfo->lmo_mds == op_data->op_mds) {
-				*fid = oinfo->lmo_fid;
-				break;
+				oinfo = &lsm->lsm_md_oinfo[i];
+				if (oinfo->lmo_mds == op_data->op_mds) {
+					*fid = oinfo->lmo_fid;
+					break;
+				}
 			}
-		}
 
-		/* Hmm, can not find the stripe by mdt_index(op_mds) */
-		if (i == lsm->lsm_md_stripe_count)
-			tgt = ERR_PTR(-EINVAL);
+			if (i == lsm->lsm_md_stripe_count)
+				*fid = lsm->lsm_md_oinfo[0].lmo_fid;
+		}
 
 		return tgt;
 	}
@@ -1728,30 +1718,9 @@ static int lmv_create(struct obd_export *exp, struct md_op_data *op_data,
 	return rc;
 }
 
-static int lmv_done_writing(struct obd_export *exp,
-			    struct md_op_data *op_data,
-			    struct md_open_data *mod)
-{
-	struct obd_device     *obd = exp->exp_obd;
-	struct lmv_obd	*lmv = &obd->u.lmv;
-	struct lmv_tgt_desc   *tgt;
-	int		    rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
-
-	tgt = lmv_find_target(lmv, &op_data->op_fid1);
-	if (IS_ERR(tgt))
-		return PTR_ERR(tgt);
-
-	rc = md_done_writing(tgt->ltd_exp, op_data, mod);
-	return rc;
-}
-
 static int
 lmv_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
-	    const ldlm_policy_data_t *policy,
+	    const union ldlm_policy_data *policy,
 	    struct lookup_intent *it, struct md_op_data *op_data,
 	    struct lustre_handle *lockh, __u64 extra_lock_flags)
 {
@@ -1847,7 +1816,7 @@ static int lmv_early_cancel(struct obd_export *exp, struct lmv_tgt_desc *tgt,
 	struct lu_fid	  *fid = md_op_data_fid(op_data, flag);
 	struct obd_device      *obd = exp->exp_obd;
 	struct lmv_obd	 *lmv = &obd->u.lmv;
-	ldlm_policy_data_t      policy = { {0} };
+	union ldlm_policy_data policy = { { 0 } };
 	int		     rc = 0;
 
 	if (!fid_is_sane(fid))
@@ -1937,7 +1906,10 @@ static int lmv_rename(struct obd_export *exp, struct md_op_data *op_data,
 {
 	struct obd_device       *obd = exp->exp_obd;
 	struct lmv_obd	  *lmv = &obd->u.lmv;
+	struct obd_export *target_exp;
 	struct lmv_tgt_desc     *src_tgt;
+	struct lmv_tgt_desc *tgt_tgt;
+	struct mdt_body *body;
 	int			rc;
 
 	LASSERT(oldlen != 0);
@@ -1977,6 +1949,10 @@ static int lmv_rename(struct obd_export *exp, struct md_op_data *op_data,
 		if (rc)
 			return rc;
 		src_tgt = lmv_find_target(lmv, &op_data->op_fid3);
+		if (IS_ERR(src_tgt))
+			return PTR_ERR(src_tgt);
+
+		target_exp = src_tgt->ltd_exp;
 	} else {
 		if (op_data->op_mea1) {
 			struct lmv_stripe_md *lsm = op_data->op_mea1;
@@ -1985,29 +1961,27 @@ static int lmv_rename(struct obd_export *exp, struct md_op_data *op_data,
 							     oldlen,
 							     &op_data->op_fid1,
 							     &op_data->op_mds);
-			if (IS_ERR(src_tgt))
-				return PTR_ERR(src_tgt);
 		} else {
 			src_tgt = lmv_find_target(lmv, &op_data->op_fid1);
-			if (IS_ERR(src_tgt))
-				return PTR_ERR(src_tgt);
-
-			op_data->op_mds = src_tgt->ltd_idx;
 		}
+		if (IS_ERR(src_tgt))
+			return PTR_ERR(src_tgt);
 
 		if (op_data->op_mea2) {
 			struct lmv_stripe_md *lsm = op_data->op_mea2;
-			const struct lmv_oinfo *oinfo;
 
-			oinfo = lsm_name_to_stripe_info(lsm, new, newlen);
-			if (IS_ERR(oinfo))
-				return PTR_ERR(oinfo);
-
-			op_data->op_fid2 = oinfo->lmo_fid;
+			tgt_tgt = lmv_locate_target_for_name(lmv, lsm, new,
+							     newlen,
+							     &op_data->op_fid2,
+							     &op_data->op_mds);
+		} else {
+			tgt_tgt = lmv_find_target(lmv, &op_data->op_fid2);
 		}
+		if (IS_ERR(tgt_tgt))
+			return PTR_ERR(tgt_tgt);
+
+		target_exp = tgt_tgt->ltd_exp;
 	}
-	if (IS_ERR(src_tgt))
-		return PTR_ERR(src_tgt);
 
 	/*
 	 * LOOKUP lock on src child (fid3) should also be cancelled for
@@ -2048,26 +2022,56 @@ static int lmv_rename(struct obd_export *exp, struct md_op_data *op_data,
 			return rc;
 	}
 
+retry_rename:
 	/*
 	 * Cancel all the locks on tgt child (fid4).
 	 */
-	if (fid_is_sane(&op_data->op_fid4))
+	if (fid_is_sane(&op_data->op_fid4)) {
+		struct lmv_tgt_desc *tgt;
+
 		rc = lmv_early_cancel(exp, NULL, op_data, src_tgt->ltd_idx,
 				      LCK_EX, MDS_INODELOCK_FULL,
 				      MF_MDC_CANCEL_FID4);
+		if (rc)
+			return rc;
+
+		tgt = lmv_find_target(lmv, &op_data->op_fid4);
+		if (IS_ERR(tgt))
+			return PTR_ERR(tgt);
 
-	CDEBUG(D_INODE, DFID":m%d to "DFID"\n", PFID(&op_data->op_fid1),
-	       op_data->op_mds, PFID(&op_data->op_fid2));
+		/*
+		 * Since the target child might be destroyed, and it might
+		 * become orphan, and we can only check orphan on the local
+		 * MDT right now, so we send rename request to the MDT where
+		 * target child is located. If target child does not exist,
+		 * then it will send the request to the target parent
+		 */
+		target_exp = tgt->ltd_exp;
+	}
 
-	rc = md_rename(src_tgt->ltd_exp, op_data, old, oldlen,
-		       new, newlen, request);
-	return rc;
+	rc = md_rename(target_exp, op_data, old, oldlen, new, newlen, request);
+	if (rc && rc != -EREMOTE)
+		return rc;
+
+	body = req_capsule_server_get(&(*request)->rq_pill, &RMF_MDT_BODY);
+	if (!body)
+		return -EPROTO;
+
+	/* Not cross-ref case, just get out of here. */
+	if (likely(!(body->mbo_valid & OBD_MD_MDS)))
+		return rc;
+
+	CDEBUG(D_INODE, "%s: try rename to another MDT for " DFID "\n",
+	       exp->exp_obd->obd_name, PFID(&body->mbo_fid1));
+
+	op_data->op_fid4 = body->mbo_fid1;
+	ptlrpc_req_finished(*request);
+	*request = NULL;
+	goto retry_rename;
 }
 
 static int lmv_setattr(struct obd_export *exp, struct md_op_data *op_data,
-		       void *ea, size_t ealen, void *ea2, size_t ea2len,
-		       struct ptlrpc_request **request,
-		       struct md_open_data **mod)
+		       void *ea, size_t ealen, struct ptlrpc_request **request)
 {
 	struct obd_device       *obd = exp->exp_obd;
 	struct lmv_obd	  *lmv = &obd->u.lmv;
@@ -2086,10 +2090,7 @@ static int lmv_setattr(struct obd_export *exp, struct md_op_data *op_data,
 	if (IS_ERR(tgt))
 		return PTR_ERR(tgt);
 
-	rc = md_setattr(tgt->ltd_exp, op_data, ea, ealen, ea2,
-			ea2len, request, mod);
-
-	return rc;
+	return md_setattr(tgt->ltd_exp, op_data, ea, ealen, request);
 }
 
 static int lmv_sync(struct obd_export *exp, const struct lu_fid *fid,
@@ -2623,23 +2624,10 @@ try_next_stripe:
 	goto retry_unlink;
 }
 
-static int lmv_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
+static int lmv_precleanup(struct obd_device *obd)
 {
-	struct lmv_obd *lmv = &obd->u.lmv;
-
-	switch (stage) {
-	case OBD_CLEANUP_EARLY:
-		/* XXX: here should be calling obd_precleanup() down to
-		 * stack.
-		 */
-		break;
-	case OBD_CLEANUP_EXPORTS:
-		fld_client_debugfs_fini(&lmv->lmv_fld);
-		lprocfs_obd_cleanup(obd);
-		break;
-	default:
-		break;
-	}
+	fld_client_debugfs_fini(&obd->u.lmv.lmv_fld);
+	lprocfs_obd_cleanup(obd);
 	return 0;
 }
 
@@ -2654,14 +2642,12 @@ static int lmv_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
  * \param[in]  key	identifier of key to get value for
  * \param[in]  vallen	size of \a val
  * \param[out] val	pointer to storage location for value
- * \param[in]  lsm	optional striping metadata of object
  *
  * \retval 0		on success
  * \retval negative	negated errno on failure
  */
 static int lmv_get_info(const struct lu_env *env, struct obd_export *exp,
-			__u32 keylen, void *key, __u32 *vallen, void *val,
-			struct lov_stripe_md *lsm)
+			__u32 keylen, void *key, __u32 *vallen, void *val)
 {
 	struct obd_device       *obd;
 	struct lmv_obd	  *lmv;
@@ -2693,7 +2679,7 @@ static int lmv_get_info(const struct lu_env *env, struct obd_export *exp,
 				continue;
 
 			if (!obd_get_info(env, tgt->ltd_exp, keylen, key,
-					  vallen, val, NULL))
+					  vallen, val))
 				return 0;
 		}
 		return -EINVAL;
@@ -2709,7 +2695,7 @@ static int lmv_get_info(const struct lu_env *env, struct obd_export *exp,
 		 * desc.
 		 */
 		rc = obd_get_info(env, lmv->tgts[0]->ltd_exp, keylen, key,
-				  vallen, val, NULL);
+				  vallen, val);
 		if (!rc && KEY_IS(KEY_CONN_DATA))
 			exp->exp_connect_data = *(struct obd_connect_data *)val;
 		return rc;
@@ -2777,90 +2763,6 @@ static int lmv_set_info_async(const struct lu_env *env, struct obd_export *exp,
 	return -EINVAL;
 }
 
-static int lmv_pack_md_v1(const struct lmv_stripe_md *lsm,
-			  struct lmv_mds_md_v1 *lmm1)
-{
-	int cplen;
-	int i;
-
-	lmm1->lmv_magic = cpu_to_le32(lsm->lsm_md_magic);
-	lmm1->lmv_stripe_count = cpu_to_le32(lsm->lsm_md_stripe_count);
-	lmm1->lmv_master_mdt_index = cpu_to_le32(lsm->lsm_md_master_mdt_index);
-	lmm1->lmv_hash_type = cpu_to_le32(lsm->lsm_md_hash_type);
-	cplen = strlcpy(lmm1->lmv_pool_name, lsm->lsm_md_pool_name,
-			sizeof(lmm1->lmv_pool_name));
-	if (cplen >= sizeof(lmm1->lmv_pool_name))
-		return -E2BIG;
-
-	for (i = 0; i < lsm->lsm_md_stripe_count; i++)
-		fid_cpu_to_le(&lmm1->lmv_stripe_fids[i],
-			      &lsm->lsm_md_oinfo[i].lmo_fid);
-	return 0;
-}
-
-static int
-lmv_pack_md(union lmv_mds_md **lmmp, const struct lmv_stripe_md *lsm,
-	    int stripe_count)
-{
-	int lmm_size = 0, rc = 0;
-	bool allocated = false;
-
-	LASSERT(lmmp);
-
-	/* Free lmm */
-	if (*lmmp && !lsm) {
-		int stripe_cnt;
-
-		stripe_cnt = lmv_mds_md_stripe_count_get(*lmmp);
-		lmm_size = lmv_mds_md_size(stripe_cnt,
-					   le32_to_cpu((*lmmp)->lmv_magic));
-		if (!lmm_size)
-			return -EINVAL;
-		kvfree(*lmmp);
-		*lmmp = NULL;
-		return 0;
-	}
-
-	/* Alloc lmm */
-	if (!*lmmp && !lsm) {
-		lmm_size = lmv_mds_md_size(stripe_count, LMV_MAGIC);
-		LASSERT(lmm_size > 0);
-		*lmmp = libcfs_kvzalloc(lmm_size, GFP_NOFS);
-		if (!*lmmp)
-			return -ENOMEM;
-		lmv_mds_md_stripe_count_set(*lmmp, stripe_count);
-		(*lmmp)->lmv_magic = cpu_to_le32(LMV_MAGIC);
-		return lmm_size;
-	}
-
-	/* pack lmm */
-	LASSERT(lsm);
-	lmm_size = lmv_mds_md_size(lsm->lsm_md_stripe_count,
-				   lsm->lsm_md_magic);
-	if (!*lmmp) {
-		*lmmp = libcfs_kvzalloc(lmm_size, GFP_NOFS);
-		if (!*lmmp)
-			return -ENOMEM;
-		allocated = true;
-	}
-
-	switch (lsm->lsm_md_magic) {
-	case LMV_MAGIC_V1:
-		rc = lmv_pack_md_v1(lsm, &(*lmmp)->lmv_md_v1);
-		break;
-	default:
-		rc = -EINVAL;
-		break;
-	}
-
-	if (rc && allocated) {
-		kvfree(*lmmp);
-		*lmmp = NULL;
-	}
-
-	return lmm_size;
-}
-
 static int lmv_unpack_md_v1(struct obd_export *exp, struct lmv_stripe_md *lsm,
 			    const struct lmv_mds_md_v1 *lmm1)
 {
@@ -2903,8 +2805,8 @@ static int lmv_unpack_md_v1(struct obd_export *exp, struct lmv_stripe_md *lsm,
 	return rc;
 }
 
-int lmv_unpack_md(struct obd_export *exp, struct lmv_stripe_md **lsmp,
-		  const union lmv_mds_md *lmm, int stripe_count)
+static int lmv_unpackmd(struct obd_export *exp, struct lmv_stripe_md **lsmp,
+			const union lmv_mds_md *lmm, size_t lmm_size)
 {
 	struct lmv_stripe_md *lsm;
 	bool allocated = false;
@@ -2933,17 +2835,6 @@ int lmv_unpack_md(struct obd_export *exp, struct lmv_stripe_md **lsmp,
 		return 0;
 	}
 
-	/* Alloc memmd */
-	if (!lsm && !lmm) {
-		lsm_size = lmv_stripe_md_size(stripe_count);
-		lsm = libcfs_kvzalloc(lsm_size, GFP_NOFS);
-		if (!lsm)
-			return -ENOMEM;
-		lsm->lsm_md_stripe_count = stripe_count;
-		*lsmp = lsm;
-		return 0;
-	}
-
 	if (le32_to_cpu(lmm->lmv_magic) == LMV_MAGIC_STRIPE)
 		return -EPERM;
 
@@ -2991,38 +2882,17 @@ int lmv_unpack_md(struct obd_export *exp, struct lmv_stripe_md **lsmp,
 	}
 	return lsm_size;
 }
-EXPORT_SYMBOL(lmv_unpack_md);
 
-static int lmv_unpackmd(struct obd_export *exp, struct lov_stripe_md **lsmp,
-			struct lov_mds_md *lmm, int disk_len)
+void lmv_free_memmd(struct lmv_stripe_md *lsm)
 {
-	return lmv_unpack_md(exp, (struct lmv_stripe_md **)lsmp,
-			     (union lmv_mds_md *)lmm, disk_len);
-}
-
-static int lmv_packmd(struct obd_export *exp, struct lov_mds_md **lmmp,
-		      struct lov_stripe_md *lsm)
-{
-	const struct lmv_stripe_md *lmv = (struct lmv_stripe_md *)lsm;
-	struct obd_device *obd = exp->exp_obd;
-	struct lmv_obd *lmv_obd = &obd->u.lmv;
-	int stripe_count;
-
-	if (!lmmp) {
-		if (lsm)
-			stripe_count = lmv->lsm_md_stripe_count;
-		else
-			stripe_count = lmv_obd->desc.ld_tgt_count;
-
-		return lmv_mds_md_size(stripe_count, LMV_MAGIC_V1);
-	}
-
-	return lmv_pack_md((union lmv_mds_md **)lmmp, lmv, 0);
+	lmv_unpackmd(NULL, &lsm, NULL, 0);
 }
+EXPORT_SYMBOL(lmv_free_memmd);
 
 static int lmv_cancel_unused(struct obd_export *exp, const struct lu_fid *fid,
-			     ldlm_policy_data_t *policy, enum ldlm_mode mode,
-			     enum ldlm_cancel_flags flags, void *opaque)
+			     union ldlm_policy_data *policy,
+			     enum ldlm_mode mode, enum ldlm_cancel_flags flags,
+			     void *opaque)
 {
 	struct obd_device       *obd = exp->exp_obd;
 	struct lmv_obd	  *lmv = &obd->u.lmv;
@@ -3064,7 +2934,7 @@ static int lmv_set_lock_data(struct obd_export *exp,
 static enum ldlm_mode lmv_lock_match(struct obd_export *exp, __u64 flags,
 				     const struct lu_fid *fid,
 				     enum ldlm_type type,
-				     ldlm_policy_data_t *policy,
+				     union ldlm_policy_data *policy,
 				     enum ldlm_mode mode,
 				     struct lustre_handle *lockh)
 {
@@ -3271,32 +3141,6 @@ static int lmv_quotactl(struct obd_device *unused, struct obd_export *exp,
 	return rc;
 }
 
-static int lmv_quotacheck(struct obd_device *unused, struct obd_export *exp,
-			  struct obd_quotactl *oqctl)
-{
-	struct obd_device   *obd = class_exp2obd(exp);
-	struct lmv_obd      *lmv = &obd->u.lmv;
-	struct lmv_tgt_desc *tgt;
-	int rc = 0;
-	u32 i;
-
-	for (i = 0; i < lmv->desc.ld_tgt_count; i++) {
-		int err;
-
-		tgt = lmv->tgts[i];
-		if (!tgt || !tgt->ltd_exp || !tgt->ltd_active) {
-			CERROR("lmv idx %d inactive\n", i);
-			return -EIO;
-		}
-
-		err = obd_quotacheck(tgt->ltd_exp, oqctl);
-		if (err && !rc)
-			rc = err;
-	}
-
-	return rc;
-}
-
 static int lmv_merge_attr(struct obd_export *exp,
 			  const struct lmv_stripe_md *lsm,
 			  struct cl_attr *attr,
@@ -3349,12 +3193,9 @@ static struct obd_ops lmv_obd_ops = {
 	.statfs		= lmv_statfs,
 	.get_info	= lmv_get_info,
 	.set_info_async	= lmv_set_info_async,
-	.packmd		= lmv_packmd,
-	.unpackmd	= lmv_unpackmd,
 	.notify		= lmv_notify,
 	.get_uuid	= lmv_get_uuid,
 	.iocontrol	= lmv_iocontrol,
-	.quotacheck	= lmv_quotacheck,
 	.quotactl	= lmv_quotactl
 };
 
@@ -3363,7 +3204,6 @@ static struct md_ops lmv_md_ops = {
 	.null_inode		= lmv_null_inode,
 	.close			= lmv_close,
 	.create			= lmv_create,
-	.done_writing		= lmv_done_writing,
 	.enqueue		= lmv_enqueue,
 	.getattr		= lmv_getattr,
 	.getxattr		= lmv_getxattr,
@@ -3388,6 +3228,7 @@ static struct md_ops lmv_md_ops = {
 	.intent_getattr_async	= lmv_intent_getattr_async,
 	.revalidate_lock	= lmv_revalidate_lock,
 	.get_fid_from_lsm	= lmv_get_fid_from_lsm,
+	.unpackmd		= lmv_unpackmd,
 };
 
 static int __init lmv_init(void)
diff --git a/drivers/staging/lustre/lustre/lov/lov_cl_internal.h b/drivers/staging/lustre/lustre/lov/lov_cl_internal.h
index 4d2b7d303fea..c49a34bf10e5 100644
--- a/drivers/staging/lustre/lustre/lov/lov_cl_internal.h
+++ b/drivers/staging/lustre/lustre/lov/lov_cl_internal.h
@@ -217,7 +217,7 @@ struct lov_object {
 
 	union lov_layout_state {
 		struct lov_layout_raid0 {
-			unsigned	       lo_nr;
+			unsigned int	       lo_nr;
 			/**
 			 * When this is true, lov_object::lo_attr contains
 			 * valid up to date attributes for a top-level
@@ -412,7 +412,6 @@ struct lov_io_sub {
 	int		  sub_refcheck;
 	int		  sub_refcheck2;
 	int		  sub_reenter;
-	void		*sub_cookie;
 };
 
 /**
@@ -473,20 +472,6 @@ struct lov_session {
 	struct lov_sublock_env ls_subenv;
 };
 
-/**
- * State of transfer for lov.
- */
-struct lov_req {
-	struct cl_req_slice lr_cl;
-};
-
-/**
- * State of transfer for lovsub.
- */
-struct lovsub_req {
-	struct cl_req_slice lsrq_cl;
-};
-
 extern struct lu_device_type lov_device_type;
 extern struct lu_device_type lovsub_device_type;
 
@@ -497,11 +482,9 @@ extern struct kmem_cache *lov_lock_kmem;
 extern struct kmem_cache *lov_object_kmem;
 extern struct kmem_cache *lov_thread_kmem;
 extern struct kmem_cache *lov_session_kmem;
-extern struct kmem_cache *lov_req_kmem;
 
 extern struct kmem_cache *lovsub_lock_kmem;
 extern struct kmem_cache *lovsub_object_kmem;
-extern struct kmem_cache *lovsub_req_kmem;
 
 extern struct kmem_cache *lov_lock_link_kmem;
 
@@ -700,11 +683,6 @@ static inline struct lov_page *cl2lov_page(const struct cl_page_slice *slice)
 	return container_of0(slice, struct lov_page, lps_cl);
 }
 
-static inline struct lov_req *cl2lov_req(const struct cl_req_slice *slice)
-{
-	return container_of0(slice, struct lov_req, lr_cl);
-}
-
 static inline struct lovsub_page *
 cl2lovsub_page(const struct cl_page_slice *slice)
 {
@@ -712,11 +690,6 @@ cl2lovsub_page(const struct cl_page_slice *slice)
 	return container_of0(slice, struct lovsub_page, lsb_cl);
 }
 
-static inline struct lovsub_req *cl2lovsub_req(const struct cl_req_slice *slice)
-{
-	return container_of0(slice, struct lovsub_req, lsrq_cl);
-}
-
 static inline struct lov_io *cl2lov_io(const struct lu_env *env,
 				       const struct cl_io_slice *ios)
 {
diff --git a/drivers/staging/lustre/lustre/lov/lov_dev.c b/drivers/staging/lustre/lustre/lov/lov_dev.c
index 056ae2ed88e8..7301f6e579a1 100644
--- a/drivers/staging/lustre/lustre/lov/lov_dev.c
+++ b/drivers/staging/lustre/lustre/lov/lov_dev.c
@@ -46,11 +46,9 @@ struct kmem_cache *lov_lock_kmem;
 struct kmem_cache *lov_object_kmem;
 struct kmem_cache *lov_thread_kmem;
 struct kmem_cache *lov_session_kmem;
-struct kmem_cache *lov_req_kmem;
 
 struct kmem_cache *lovsub_lock_kmem;
 struct kmem_cache *lovsub_object_kmem;
-struct kmem_cache *lovsub_req_kmem;
 
 struct kmem_cache *lov_lock_link_kmem;
 
@@ -79,11 +77,6 @@ struct lu_kmem_descr lov_caches[] = {
 		.ckd_size  = sizeof(struct lov_session)
 	},
 	{
-		.ckd_cache = &lov_req_kmem,
-		.ckd_name  = "lov_req_kmem",
-		.ckd_size  = sizeof(struct lov_req)
-	},
-	{
 		.ckd_cache = &lovsub_lock_kmem,
 		.ckd_name  = "lovsub_lock_kmem",
 		.ckd_size  = sizeof(struct lovsub_lock)
@@ -94,11 +87,6 @@ struct lu_kmem_descr lov_caches[] = {
 		.ckd_size  = sizeof(struct lovsub_object)
 	},
 	{
-		.ckd_cache = &lovsub_req_kmem,
-		.ckd_name  = "lovsub_req_kmem",
-		.ckd_size  = sizeof(struct lovsub_req)
-	},
-	{
 		.ckd_cache = &lov_lock_link_kmem,
 		.ckd_name  = "lov_lock_link_kmem",
 		.ckd_size  = sizeof(struct lov_lock_link)
@@ -110,25 +98,6 @@ struct lu_kmem_descr lov_caches[] = {
 
 /*****************************************************************************
  *
- * Lov transfer operations.
- *
- */
-
-static void lov_req_completion(const struct lu_env *env,
-			       const struct cl_req_slice *slice, int ioret)
-{
-	struct lov_req *lr;
-
-	lr = cl2lov_req(slice);
-	kmem_cache_free(lov_req_kmem, lr);
-}
-
-static const struct cl_req_operations lov_req_ops = {
-	.cro_completion = lov_req_completion
-};
-
-/*****************************************************************************
- *
  * Lov device and device type functions.
  *
  */
@@ -248,26 +217,6 @@ static int lov_device_init(const struct lu_env *env, struct lu_device *d,
 	return rc;
 }
 
-static int lov_req_init(const struct lu_env *env, struct cl_device *dev,
-			struct cl_req *req)
-{
-	struct lov_req *lr;
-	int result;
-
-	lr = kmem_cache_zalloc(lov_req_kmem, GFP_NOFS);
-	if (lr) {
-		cl_req_slice_add(req, &lr->lr_cl, dev, &lov_req_ops);
-		result = 0;
-	} else {
-		result = -ENOMEM;
-	}
-	return result;
-}
-
-static const struct cl_device_operations lov_cl_ops = {
-	.cdo_req_init = lov_req_init
-};
-
 static void lov_emerg_free(struct lov_device_emerg **emrg, int nr)
 {
 	int i;
@@ -478,7 +427,6 @@ static struct lu_device *lov_device_alloc(const struct lu_env *env,
 	cl_device_init(&ld->ld_cl, t);
 	d = lov2lu_dev(ld);
 	d->ld_ops	= &lov_lu_ops;
-	ld->ld_cl.cd_ops = &lov_cl_ops;
 
 	mutex_init(&ld->ld_mutex);
 	lockdep_set_class(&ld->ld_mutex, &cl_lov_device_mutex_class);
diff --git a/drivers/staging/lustre/lustre/lov/lov_ea.c b/drivers/staging/lustre/lustre/lov/lov_ea.c
index 214c561767e0..ac0bf64c08c1 100644
--- a/drivers/staging/lustre/lustre/lov/lov_ea.c
+++ b/drivers/staging/lustre/lustre/lov/lov_ea.c
@@ -76,18 +76,19 @@ static int lsm_lmm_verify_common(struct lov_mds_md *lmm, int lmm_bytes,
 	return 0;
 }
 
-struct lov_stripe_md *lsm_alloc_plain(__u16 stripe_count, int *size)
+struct lov_stripe_md *lsm_alloc_plain(u16 stripe_count)
 {
+	size_t oinfo_ptrs_size, lsm_size;
 	struct lov_stripe_md *lsm;
 	struct lov_oinfo     *loi;
-	int		   i, oinfo_ptrs_size;
+	int i;
 
 	LASSERT(stripe_count <= LOV_MAX_STRIPE_COUNT);
 
 	oinfo_ptrs_size = sizeof(struct lov_oinfo *) * stripe_count;
-	*size = sizeof(struct lov_stripe_md) + oinfo_ptrs_size;
+	lsm_size = sizeof(*lsm) + oinfo_ptrs_size;
 
-	lsm = libcfs_kvzalloc(*size, GFP_NOFS);
+	lsm = libcfs_kvzalloc(lsm_size, GFP_NOFS);
 	if (!lsm)
 		return NULL;
 
@@ -117,9 +118,43 @@ void lsm_free_plain(struct lov_stripe_md *lsm)
 	kvfree(lsm);
 }
 
-static void lsm_unpackmd_common(struct lov_stripe_md *lsm,
-				struct lov_mds_md *lmm)
+/*
+ * Find minimum stripe maxbytes value.  For inactive or
+ * reconnecting targets use LUSTRE_EXT3_STRIPE_MAXBYTES.
+ */
+static loff_t lov_tgt_maxbytes(struct lov_tgt_desc *tgt)
+{
+	loff_t maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES;
+	struct obd_import *imp;
+
+	if (!tgt->ltd_active)
+		return maxbytes;
+
+	imp = tgt->ltd_obd->u.cli.cl_import;
+	if (!imp)
+		return maxbytes;
+
+	spin_lock(&imp->imp_lock);
+	if (imp->imp_state == LUSTRE_IMP_FULL &&
+	    (imp->imp_connect_data.ocd_connect_flags & OBD_CONNECT_MAXBYTES) &&
+	     imp->imp_connect_data.ocd_maxbytes > 0)
+		maxbytes = imp->imp_connect_data.ocd_maxbytes;
+
+	spin_unlock(&imp->imp_lock);
+
+	return maxbytes;
+}
+
+static int lsm_unpackmd_common(struct lov_obd *lov,
+			       struct lov_stripe_md *lsm,
+			       struct lov_mds_md *lmm,
+			       struct lov_ost_data_v1 *objects)
 {
+	loff_t stripe_maxbytes = LLONG_MAX;
+	unsigned int stripe_count;
+	struct lov_oinfo *loi;
+	unsigned int i;
+
 	/*
 	 * This supposes lov_mds_md_v1/v3 first fields are
 	 * are the same
@@ -129,11 +164,54 @@ static void lsm_unpackmd_common(struct lov_stripe_md *lsm,
 	lsm->lsm_pattern = le32_to_cpu(lmm->lmm_pattern);
 	lsm->lsm_layout_gen = le16_to_cpu(lmm->lmm_layout_gen);
 	lsm->lsm_pool_name[0] = '\0';
+
+	stripe_count = lsm_is_released(lsm) ? 0 : lsm->lsm_stripe_count;
+
+	for (i = 0; i < stripe_count; i++) {
+		loff_t tgt_bytes;
+
+		loi = lsm->lsm_oinfo[i];
+		ostid_le_to_cpu(&objects[i].l_ost_oi, &loi->loi_oi);
+		loi->loi_ost_idx = le32_to_cpu(objects[i].l_ost_idx);
+		loi->loi_ost_gen = le32_to_cpu(objects[i].l_ost_gen);
+		if (lov_oinfo_is_dummy(loi))
+			continue;
+
+		if (loi->loi_ost_idx >= lov->desc.ld_tgt_count &&
+		    !lov2obd(lov)->obd_process_conf) {
+			CERROR("%s: OST index %d more than OST count %d\n",
+			       (char *)lov->desc.ld_uuid.uuid,
+			       loi->loi_ost_idx, lov->desc.ld_tgt_count);
+			lov_dump_lmm_v1(D_WARNING, lmm);
+			return -EINVAL;
+		}
+
+		if (!lov->lov_tgts[loi->loi_ost_idx]) {
+			CERROR("%s: OST index %d missing\n",
+			       (char *)lov->desc.ld_uuid.uuid,
+			       loi->loi_ost_idx);
+			lov_dump_lmm_v1(D_WARNING, lmm);
+			continue;
+		}
+
+		tgt_bytes = lov_tgt_maxbytes(lov->lov_tgts[loi->loi_ost_idx]);
+		stripe_maxbytes = min_t(loff_t, stripe_maxbytes, tgt_bytes);
+	}
+
+	if (stripe_maxbytes == LLONG_MAX)
+		stripe_maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES;
+
+	if (!lsm->lsm_stripe_count)
+		lsm->lsm_maxbytes = stripe_maxbytes * lov->desc.ld_tgt_count;
+	else
+		lsm->lsm_maxbytes = stripe_maxbytes * lsm->lsm_stripe_count;
+
+	return 0;
 }
 
 static void
 lsm_stripe_by_index_plain(struct lov_stripe_md *lsm, int *stripeno,
-			  u64 *lov_off, u64 *swidth)
+			  loff_t *lov_off, loff_t *swidth)
 {
 	if (swidth)
 		*swidth = (u64)lsm->lsm_stripe_size * lsm->lsm_stripe_count;
@@ -141,36 +219,12 @@ lsm_stripe_by_index_plain(struct lov_stripe_md *lsm, int *stripeno,
 
 static void
 lsm_stripe_by_offset_plain(struct lov_stripe_md *lsm, int *stripeno,
-			   u64 *lov_off, u64 *swidth)
+			   loff_t *lov_off, loff_t *swidth)
 {
 	if (swidth)
 		*swidth = (u64)lsm->lsm_stripe_size * lsm->lsm_stripe_count;
 }
 
-/* Find minimum stripe maxbytes value.  For inactive or
- * reconnecting targets use LUSTRE_EXT3_STRIPE_MAXBYTES.
- */
-static void lov_tgt_maxbytes(struct lov_tgt_desc *tgt, __u64 *stripe_maxbytes)
-{
-	struct obd_import *imp = tgt->ltd_obd->u.cli.cl_import;
-
-	if (!imp || !tgt->ltd_active) {
-		*stripe_maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES;
-		return;
-	}
-
-	spin_lock(&imp->imp_lock);
-	if (imp->imp_state == LUSTRE_IMP_FULL &&
-	    (imp->imp_connect_data.ocd_connect_flags & OBD_CONNECT_MAXBYTES) &&
-	    imp->imp_connect_data.ocd_maxbytes > 0) {
-		if (*stripe_maxbytes > imp->imp_connect_data.ocd_maxbytes)
-			*stripe_maxbytes = imp->imp_connect_data.ocd_maxbytes;
-	} else {
-		*stripe_maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES;
-	}
-	spin_unlock(&imp->imp_lock);
-}
-
 static int lsm_lmm_verify_v1(struct lov_mds_md_v1 *lmm, int lmm_bytes,
 			     __u16 *stripe_count)
 {
@@ -197,45 +251,7 @@ static int lsm_lmm_verify_v1(struct lov_mds_md_v1 *lmm, int lmm_bytes,
 static int lsm_unpackmd_v1(struct lov_obd *lov, struct lov_stripe_md *lsm,
 			   struct lov_mds_md_v1 *lmm)
 {
-	struct lov_oinfo *loi;
-	int i;
-	int stripe_count;
-	__u64 stripe_maxbytes = OBD_OBJECT_EOF;
-
-	lsm_unpackmd_common(lsm, lmm);
-
-	stripe_count = lsm_is_released(lsm) ? 0 : lsm->lsm_stripe_count;
-
-	for (i = 0; i < stripe_count; i++) {
-		/* XXX LOV STACKING call down to osc_unpackmd() */
-		loi = lsm->lsm_oinfo[i];
-		ostid_le_to_cpu(&lmm->lmm_objects[i].l_ost_oi, &loi->loi_oi);
-		loi->loi_ost_idx = le32_to_cpu(lmm->lmm_objects[i].l_ost_idx);
-		loi->loi_ost_gen = le32_to_cpu(lmm->lmm_objects[i].l_ost_gen);
-		if (lov_oinfo_is_dummy(loi))
-			continue;
-
-		if (loi->loi_ost_idx >= lov->desc.ld_tgt_count) {
-			CERROR("OST index %d more than OST count %d\n",
-			       loi->loi_ost_idx, lov->desc.ld_tgt_count);
-			lov_dump_lmm_v1(D_WARNING, lmm);
-			return -EINVAL;
-		}
-		if (!lov->lov_tgts[loi->loi_ost_idx]) {
-			CERROR("OST index %d missing\n", loi->loi_ost_idx);
-			lov_dump_lmm_v1(D_WARNING, lmm);
-			return -EINVAL;
-		}
-		/* calculate the minimum stripe max bytes */
-		lov_tgt_maxbytes(lov->lov_tgts[loi->loi_ost_idx],
-				 &stripe_maxbytes);
-	}
-
-	lsm->lsm_maxbytes = stripe_maxbytes * lsm->lsm_stripe_count;
-	if (lsm->lsm_stripe_count == 0)
-		lsm->lsm_maxbytes = stripe_maxbytes * lov->desc.ld_tgt_count;
-
-	return 0;
+	return lsm_unpackmd_common(lov, lsm, lmm, lmm->lmm_objects);
 }
 
 const struct lsm_operations lsm_v1_ops = {
@@ -275,55 +291,21 @@ static int lsm_lmm_verify_v3(struct lov_mds_md *lmmv1, int lmm_bytes,
 }
 
 static int lsm_unpackmd_v3(struct lov_obd *lov, struct lov_stripe_md *lsm,
-			   struct lov_mds_md *lmmv1)
+			   struct lov_mds_md *lmm)
 {
-	struct lov_mds_md_v3 *lmm;
-	struct lov_oinfo *loi;
-	int i;
-	int stripe_count;
-	__u64 stripe_maxbytes = OBD_OBJECT_EOF;
-	int cplen = 0;
+	struct lov_mds_md_v3 *lmm_v3 = (struct lov_mds_md_v3 *)lmm;
+	size_t cplen = 0;
+	int rc;
 
-	lmm = (struct lov_mds_md_v3 *)lmmv1;
+	rc = lsm_unpackmd_common(lov, lsm, lmm, lmm_v3->lmm_objects);
+	if (rc)
+		return rc;
 
-	lsm_unpackmd_common(lsm, (struct lov_mds_md_v1 *)lmm);
-
-	stripe_count = lsm_is_released(lsm) ? 0 : lsm->lsm_stripe_count;
-
-	cplen = strlcpy(lsm->lsm_pool_name, lmm->lmm_pool_name,
+	cplen = strlcpy(lsm->lsm_pool_name, lmm_v3->lmm_pool_name,
 			sizeof(lsm->lsm_pool_name));
 	if (cplen >= sizeof(lsm->lsm_pool_name))
 		return -E2BIG;
 
-	for (i = 0; i < stripe_count; i++) {
-		/* XXX LOV STACKING call down to osc_unpackmd() */
-		loi = lsm->lsm_oinfo[i];
-		ostid_le_to_cpu(&lmm->lmm_objects[i].l_ost_oi, &loi->loi_oi);
-		loi->loi_ost_idx = le32_to_cpu(lmm->lmm_objects[i].l_ost_idx);
-		loi->loi_ost_gen = le32_to_cpu(lmm->lmm_objects[i].l_ost_gen);
-		if (lov_oinfo_is_dummy(loi))
-			continue;
-
-		if (loi->loi_ost_idx >= lov->desc.ld_tgt_count) {
-			CERROR("OST index %d more than OST count %d\n",
-			       loi->loi_ost_idx, lov->desc.ld_tgt_count);
-			lov_dump_lmm_v3(D_WARNING, lmm);
-			return -EINVAL;
-		}
-		if (!lov->lov_tgts[loi->loi_ost_idx]) {
-			CERROR("OST index %d missing\n", loi->loi_ost_idx);
-			lov_dump_lmm_v3(D_WARNING, lmm);
-			return -EINVAL;
-		}
-		/* calculate the minimum stripe max bytes */
-		lov_tgt_maxbytes(lov->lov_tgts[loi->loi_ost_idx],
-				 &stripe_maxbytes);
-	}
-
-	lsm->lsm_maxbytes = stripe_maxbytes * lsm->lsm_stripe_count;
-	if (lsm->lsm_stripe_count == 0)
-		lsm->lsm_maxbytes = stripe_maxbytes * lov->desc.ld_tgt_count;
-
 	return 0;
 }
 
diff --git a/drivers/staging/lustre/lustre/lov/lov_internal.h b/drivers/staging/lustre/lustre/lov/lov_internal.h
index 07e5ede3e952..774499c74daa 100644
--- a/drivers/staging/lustre/lustre/lov/lov_internal.h
+++ b/drivers/staging/lustre/lustre/lov/lov_internal.h
@@ -36,6 +36,77 @@
 #include "../include/obd_class.h"
 #include "../include/lustre/lustre_user.h"
 
+/*
+ * If we are unable to get the maximum object size from the OST in
+ * ocd_maxbytes using OBD_CONNECT_MAXBYTES, then we fall back to using
+ * the old maximum object size from ext3.
+ */
+#define LUSTRE_EXT3_STRIPE_MAXBYTES 0x1fffffff000ULL
+
+struct lov_stripe_md {
+	atomic_t	lsm_refc;
+	spinlock_t	lsm_lock;
+	pid_t		lsm_lock_owner; /* debugging */
+
+	/*
+	 * maximum possible file size, might change as OSTs status changes,
+	 * e.g. disconnected, deactivated
+	 */
+	loff_t		lsm_maxbytes;
+	struct ost_id	lsm_oi;
+	u32		lsm_magic;
+	u32		lsm_stripe_size;
+	u32		lsm_pattern; /* RAID0, RAID1, released, ... */
+	u16		lsm_stripe_count;
+	u16		lsm_layout_gen;
+	char		lsm_pool_name[LOV_MAXPOOLNAME + 1];
+	struct lov_oinfo	*lsm_oinfo[0];
+};
+
+static inline bool lsm_is_released(struct lov_stripe_md *lsm)
+{
+	return !!(lsm->lsm_pattern & LOV_PATTERN_F_RELEASED);
+}
+
+static inline bool lsm_has_objects(struct lov_stripe_md *lsm)
+{
+	if (!lsm)
+		return false;
+
+	if (lsm_is_released(lsm))
+		return false;
+
+	return true;
+}
+
+struct lsm_operations {
+	void (*lsm_free)(struct lov_stripe_md *);
+	void (*lsm_stripe_by_index)(struct lov_stripe_md *, int *, loff_t *,
+				    loff_t *);
+	void (*lsm_stripe_by_offset)(struct lov_stripe_md *, int *, loff_t *,
+				     loff_t *);
+	int (*lsm_lmm_verify)(struct lov_mds_md *lmm, int lmm_bytes,
+			      u16 *stripe_count);
+	int (*lsm_unpackmd)(struct lov_obd *lov, struct lov_stripe_md *lsm,
+			    struct lov_mds_md *lmm);
+};
+
+extern const struct lsm_operations lsm_v1_ops;
+extern const struct lsm_operations lsm_v3_ops;
+
+static inline const struct lsm_operations *lsm_op_find(int magic)
+{
+	switch (magic) {
+	case LOV_MAGIC_V1:
+		return &lsm_v1_ops;
+	case LOV_MAGIC_V3:
+		return &lsm_v3_ops;
+	default:
+		CERROR("unrecognized lsm_magic %08x\n", magic);
+		return NULL;
+	}
+}
+
 /* lov_do_div64(a, b) returns a % b, and a = a / b.
  * The 32-bit code is LOV-specific due to knowing about stripe limits in
  * order to reduce the divisor to a 32-bit number.  If the divisor is
@@ -110,8 +181,6 @@ struct lov_request_set {
 	atomic_t			set_completes;
 	atomic_t			set_success;
 	atomic_t			set_finish_checked;
-	struct llog_cookie		*set_cookies;
-	int				set_cookie_sent;
 	struct list_head			set_list;
 	wait_queue_head_t			set_waitq;
 };
@@ -132,8 +201,6 @@ static inline void lov_put_reqset(struct lov_request_set *set)
 	(char *)((lv)->lov_tgts[index]->ltd_uuid.uuid)
 
 /* lov_merge.c */
-void lov_merge_attrs(struct obdo *tgt, struct obdo *src, u64 valid,
-		     struct lov_stripe_md *lsm, int stripeno, int *set);
 int lov_merge_lvb_kms(struct lov_stripe_md *lsm,
 		      struct ost_lvb *lvb, __u64 *kms_place);
 
@@ -150,17 +217,9 @@ pgoff_t lov_stripe_pgoff(struct lov_stripe_md *lsm, pgoff_t stripe_index,
 			 int stripe);
 
 /* lov_request.c */
-int lov_update_common_set(struct lov_request_set *set,
-			  struct lov_request *req, int rc);
 int lov_prep_getattr_set(struct obd_export *exp, struct obd_info *oinfo,
 			 struct lov_request_set **reqset);
 int lov_fini_getattr_set(struct lov_request_set *set);
-int lov_prep_setattr_set(struct obd_export *exp, struct obd_info *oinfo,
-			 struct obd_trans_info *oti,
-			 struct lov_request_set **reqset);
-int lov_update_setattr_set(struct lov_request_set *set,
-			   struct lov_request *req, int rc);
-int lov_fini_setattr_set(struct lov_request_set *set);
 int lov_prep_statfs_set(struct obd_device *obd, struct obd_info *oinfo,
 			struct lov_request_set **reqset);
 int lov_fini_statfs(struct obd_device *obd, struct obd_statfs *osfs,
@@ -186,12 +245,10 @@ int lov_del_target(struct obd_device *obd, __u32 index,
 		   struct obd_uuid *uuidp, int gen);
 
 /* lov_pack.c */
-int lov_packmd(struct obd_export *exp, struct lov_mds_md **lmm,
-	       struct lov_stripe_md *lsm);
-int lov_unpackmd(struct obd_export *exp, struct lov_stripe_md **lsmp,
-		 struct lov_mds_md *lmm, int lmm_bytes);
-int lov_alloc_memmd(struct lov_stripe_md **lsmp, __u16 stripe_count,
-		    int pattern, int magic);
+ssize_t lov_lsm_pack(const struct lov_stripe_md *lsm, void *buf,
+		     size_t buf_size);
+struct lov_stripe_md *lov_unpackmd(struct lov_obd *lov, struct lov_mds_md *lmm,
+				   size_t lmm_size);
 int lov_free_memmd(struct lov_stripe_md **lsmp);
 
 void lov_dump_lmm_v1(int level, struct lov_mds_md_v1 *lmm);
@@ -199,7 +256,7 @@ void lov_dump_lmm_v3(int level, struct lov_mds_md_v3 *lmm);
 void lov_dump_lmm_common(int level, void *lmmp);
 
 /* lov_ea.c */
-struct lov_stripe_md *lsm_alloc_plain(__u16 stripe_count, int *size);
+struct lov_stripe_md *lsm_alloc_plain(u16 stripe_count);
 void lsm_free_plain(struct lov_stripe_md *lsm);
 void dump_lsm(unsigned int level, const struct lov_stripe_md *lsm);
 
@@ -244,4 +301,9 @@ static inline bool lov_oinfo_is_dummy(const struct lov_oinfo *loi)
 	return false;
 }
 
+static inline struct obd_device *lov2obd(const struct lov_obd *lov)
+{
+	return container_of0(lov, struct obd_device, u.lov);
+}
+
 #endif
diff --git a/drivers/staging/lustre/lustre/lov/lov_io.c b/drivers/staging/lustre/lustre/lov/lov_io.c
index d10157985ed9..002326c282a7 100644
--- a/drivers/staging/lustre/lustre/lov/lov_io.c
+++ b/drivers/staging/lustre/lustre/lov/lov_io.c
@@ -86,6 +86,8 @@ static void lov_io_sub_inherit(struct cl_io *io, struct lov_io *lio,
 	switch (io->ci_type) {
 	case CIT_SETATTR: {
 		io->u.ci_setattr.sa_attr = parent->u.ci_setattr.sa_attr;
+		io->u.ci_setattr.sa_attr_flags =
+					parent->u.ci_setattr.sa_attr_flags;
 		io->u.ci_setattr.sa_valid = parent->u.ci_setattr.sa_valid;
 		io->u.ci_setattr.sa_stripe_index = stripe;
 		io->u.ci_setattr.sa_parent_fid =
@@ -98,6 +100,12 @@ static void lov_io_sub_inherit(struct cl_io *io, struct lov_io *lio,
 		}
 		break;
 	}
+	case CIT_DATA_VERSION: {
+		io->u.ci_data_version.dv_data_version = 0;
+		io->u.ci_data_version.dv_flags =
+			parent->u.ci_data_version.dv_flags;
+		break;
+	}
 	case CIT_FAULT: {
 		struct cl_object *obj = parent->ci_obj;
 		loff_t off = cl_offset(obj, parent->u.ci_fault.ft_index);
@@ -159,12 +167,7 @@ static int lov_io_sub_init(const struct lu_env *env, struct lov_io *lio,
 		sub->sub_env = ld->ld_emrg[stripe]->emrg_env;
 		sub->sub_borrowed = 1;
 	} else {
-		void *cookie;
-
-		/* obtain new environment */
-		cookie = cl_env_reenter();
 		sub->sub_env = cl_env_get(&sub->sub_refcheck);
-		cl_env_reexit(cookie);
 		if (IS_ERR(sub->sub_env))
 			result = PTR_ERR(sub->sub_env);
 
@@ -337,6 +340,11 @@ static int lov_io_slice_init(struct lov_io *lio, struct lov_object *obj,
 		lio->lis_endpos = OBD_OBJECT_EOF;
 		break;
 
+	case CIT_DATA_VERSION:
+		lio->lis_pos = 0;
+		lio->lis_endpos = OBD_OBJECT_EOF;
+		break;
+
 	case CIT_FAULT: {
 		pgoff_t index = io->u.ci_fault.ft_index;
 
@@ -514,6 +522,24 @@ static int lov_io_end_wrapper(const struct lu_env *env, struct cl_io *io)
 	return 0;
 }
 
+static void
+lov_io_data_version_end(const struct lu_env *env, const struct cl_io_slice *ios)
+{
+	struct lov_io *lio = cl2lov_io(env, ios);
+	struct cl_io *parent = lio->lis_cl.cis_io;
+	struct lov_io_sub *sub;
+
+	list_for_each_entry(sub, &lio->lis_active, sub_linkage) {
+		lov_io_end_wrapper(env, sub->sub_io);
+
+		parent->u.ci_data_version.dv_data_version +=
+			sub->sub_io->u.ci_data_version.dv_data_version;
+
+		if (!parent->ci_result)
+			parent->ci_result = sub->sub_io->ci_result;
+	}
+}
+
 static int lov_io_iter_fini_wrapper(const struct lu_env *env, struct cl_io *io)
 {
 	cl_io_iter_fini(env, io);
@@ -555,6 +581,65 @@ static void lov_io_unlock(const struct lu_env *env,
 	LASSERT(rc == 0);
 }
 
+static int lov_io_read_ahead(const struct lu_env *env,
+			     const struct cl_io_slice *ios,
+			     pgoff_t start, struct cl_read_ahead *ra)
+{
+	struct lov_io *lio = cl2lov_io(env, ios);
+	struct lov_object *loo = lio->lis_object;
+	struct cl_object *obj = lov2cl(loo);
+	struct lov_layout_raid0 *r0 = lov_r0(loo);
+	unsigned int pps; /* pages per stripe */
+	struct lov_io_sub *sub;
+	pgoff_t ra_end;
+	loff_t suboff;
+	int stripe;
+	int rc;
+
+	stripe = lov_stripe_number(loo->lo_lsm, cl_offset(obj, start));
+	if (unlikely(!r0->lo_sub[stripe]))
+		return -EIO;
+
+	sub = lov_sub_get(env, lio, stripe);
+	if (IS_ERR(sub))
+		return PTR_ERR(sub);
+
+	lov_stripe_offset(loo->lo_lsm, cl_offset(obj, start), stripe, &suboff);
+	rc = cl_io_read_ahead(sub->sub_env, sub->sub_io,
+			      cl_index(lovsub2cl(r0->lo_sub[stripe]), suboff),
+			      ra);
+	lov_sub_put(sub);
+
+	CDEBUG(D_READA, DFID " cra_end = %lu, stripes = %d, rc = %d\n",
+	       PFID(lu_object_fid(lov2lu(loo))), ra->cra_end, r0->lo_nr, rc);
+	if (rc)
+		return rc;
+
+	/**
+	 * Adjust the stripe index by layout of raid0. ra->cra_end is
+	 * the maximum page index covered by an underlying DLM lock.
+	 * This function converts cra_end from stripe level to file
+	 * level, and make sure it's not beyond stripe boundary.
+	 */
+	if (r0->lo_nr == 1)	/* single stripe file */
+		return 0;
+
+	/* cra_end is stripe level, convert it into file level */
+	ra_end = ra->cra_end;
+	if (ra_end != CL_PAGE_EOF)
+		ra_end = lov_stripe_pgoff(loo->lo_lsm, ra_end, stripe);
+
+	pps = loo->lo_lsm->lsm_stripe_size >> PAGE_SHIFT;
+
+	CDEBUG(D_READA, DFID " max_index = %lu, pps = %u, stripe_size = %u, stripe no = %u, start index = %lu\n",
+	       PFID(lu_object_fid(lov2lu(loo))), ra_end, pps,
+	       loo->lo_lsm->lsm_stripe_size, stripe, start);
+
+	/* never exceed the end of the stripe */
+	ra->cra_end = min_t(pgoff_t, ra_end, start + pps - start % pps - 1);
+	return 0;
+}
+
 /**
  * lov implementation of cl_operations::cio_submit() method. It takes a list
  * of pages in \a queue, splits it into per-stripe sub-lists, invokes
@@ -779,6 +864,15 @@ static const struct cl_io_operations lov_io_ops = {
 			.cio_start     = lov_io_start,
 			.cio_end       = lov_io_end
 		},
+		[CIT_DATA_VERSION] = {
+			.cio_fini	= lov_io_fini,
+			.cio_iter_init	= lov_io_iter_init,
+			.cio_iter_fini	= lov_io_iter_fini,
+			.cio_lock	= lov_io_lock,
+			.cio_unlock	= lov_io_unlock,
+			.cio_start	= lov_io_start,
+			.cio_end	= lov_io_data_version_end,
+		},
 		[CIT_FAULT] = {
 			.cio_fini      = lov_io_fini,
 			.cio_iter_init = lov_io_iter_init,
@@ -801,6 +895,7 @@ static const struct cl_io_operations lov_io_ops = {
 			.cio_fini   = lov_io_fini
 		}
 	},
+	.cio_read_ahead			= lov_io_read_ahead,
 	.cio_submit                    = lov_io_submit,
 	.cio_commit_async              = lov_io_commit_async,
 };
@@ -820,6 +915,13 @@ static void lov_empty_io_fini(const struct lu_env *env,
 		wake_up_all(&lov->lo_waitq);
 }
 
+static int lov_empty_io_submit(const struct lu_env *env,
+			       const struct cl_io_slice *ios,
+			       enum cl_req_type crt, struct cl_2queue *queue)
+{
+	return -EBADF;
+}
+
 static void lov_empty_impossible(const struct lu_env *env,
 				 struct cl_io_slice *ios)
 {
@@ -870,7 +972,7 @@ static const struct cl_io_operations lov_empty_io_ops = {
 			.cio_fini   = lov_empty_io_fini
 		}
 	},
-	.cio_submit                    = LOV_EMPTY_IMPOSSIBLE,
+	.cio_submit			= lov_empty_io_submit,
 	.cio_commit_async              = LOV_EMPTY_IMPOSSIBLE
 };
 
@@ -909,6 +1011,7 @@ int lov_io_init_empty(const struct lu_env *env, struct cl_object *obj,
 		break;
 	case CIT_FSYNC:
 	case CIT_SETATTR:
+	case CIT_DATA_VERSION:
 		result = 1;
 		break;
 	case CIT_WRITE:
@@ -944,6 +1047,7 @@ int lov_io_init_released(const struct lu_env *env, struct cl_object *obj,
 		LASSERTF(0, "invalid type %d\n", io->ci_type);
 	case CIT_MISC:
 	case CIT_FSYNC:
+	case CIT_DATA_VERSION:
 		result = 1;
 		break;
 	case CIT_SETATTR:
diff --git a/drivers/staging/lustre/lustre/lov/lov_merge.c b/drivers/staging/lustre/lustre/lov/lov_merge.c
index 674af106b50b..391dfd207177 100644
--- a/drivers/staging/lustre/lustre/lov/lov_merge.c
+++ b/drivers/staging/lustre/lustre/lov/lov_merge.c
@@ -104,53 +104,3 @@ int lov_merge_lvb_kms(struct lov_stripe_md *lsm,
 	lvb->lvb_ctime = current_ctime;
 	return rc;
 }
-
-void lov_merge_attrs(struct obdo *tgt, struct obdo *src, u64 valid,
-		     struct lov_stripe_md *lsm, int stripeno, int *set)
-{
-	valid &= src->o_valid;
-
-	if (*set) {
-		tgt->o_valid &= valid;
-		if (valid & OBD_MD_FLSIZE) {
-			/* this handles sparse files properly */
-			u64 lov_size;
-
-			lov_size = lov_stripe_size(lsm, src->o_size, stripeno);
-			if (lov_size > tgt->o_size)
-				tgt->o_size = lov_size;
-		}
-		if (valid & OBD_MD_FLBLOCKS)
-			tgt->o_blocks += src->o_blocks;
-		if (valid & OBD_MD_FLBLKSZ)
-			tgt->o_blksize += src->o_blksize;
-		if (valid & OBD_MD_FLCTIME && tgt->o_ctime < src->o_ctime)
-			tgt->o_ctime = src->o_ctime;
-		if (valid & OBD_MD_FLMTIME && tgt->o_mtime < src->o_mtime)
-			tgt->o_mtime = src->o_mtime;
-		if (valid & OBD_MD_FLDATAVERSION)
-			tgt->o_data_version += src->o_data_version;
-
-		/* handle flags */
-		if (valid & OBD_MD_FLFLAGS)
-			tgt->o_flags &= src->o_flags;
-		else
-			tgt->o_flags = 0;
-	} else {
-		memcpy(tgt, src, sizeof(*tgt));
-		tgt->o_oi = lsm->lsm_oi;
-		tgt->o_valid = valid;
-		if (valid & OBD_MD_FLSIZE)
-			tgt->o_size = lov_stripe_size(lsm, src->o_size,
-						      stripeno);
-		tgt->o_flags = 0;
-		if (valid & OBD_MD_FLFLAGS)
-			tgt->o_flags = src->o_flags;
-	}
-
-	/* data_version needs to be valid on all stripes to be correct! */
-	if (!(valid & OBD_MD_FLDATAVERSION))
-		tgt->o_valid &= ~OBD_MD_FLDATAVERSION;
-
-	*set += 1;
-}
diff --git a/drivers/staging/lustre/lustre/lov/lov_obd.c b/drivers/staging/lustre/lustre/lov/lov_obd.c
index b23016f7ec26..63b064523c6a 100644
--- a/drivers/staging/lustre/lustre/lov/lov_obd.c
+++ b/drivers/staging/lustre/lustre/lov/lov_obd.c
@@ -40,19 +40,20 @@
 #define DEBUG_SUBSYSTEM S_LOV
 #include "../../include/linux/libcfs/libcfs.h"
 
-#include "../include/obd_support.h"
-#include "../include/lustre/lustre_ioctl.h"
-#include "../include/lustre_lib.h"
-#include "../include/lustre_net.h"
 #include "../include/lustre/lustre_idl.h"
+#include "../include/lustre/lustre_ioctl.h"
+
+#include "../include/cl_object.h"
 #include "../include/lustre_dlm.h"
+#include "../include/lustre_fid.h"
+#include "../include/lustre_lib.h"
 #include "../include/lustre_mds.h"
-#include "../include/obd_class.h"
-#include "../include/lprocfs_status.h"
+#include "../include/lustre_net.h"
 #include "../include/lustre_param.h"
-#include "../include/cl_object.h"
-#include "../include/lustre/ll_fiemap.h"
-#include "../include/lustre_fid.h"
+#include "../include/lustre_swab.h"
+#include "../include/lprocfs_status.h"
+#include "../include/obd_class.h"
+#include "../include/obd_support.h"
 
 #include "lov_internal.h"
 
@@ -826,29 +827,6 @@ out:
 	return rc;
 }
 
-static int lov_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
-{
-	struct lov_obd *lov = &obd->u.lov;
-
-	switch (stage) {
-	case OBD_CLEANUP_EARLY: {
-		int i;
-
-		for (i = 0; i < lov->desc.ld_tgt_count; i++) {
-			if (!lov->lov_tgts[i] || !lov->lov_tgts[i]->ltd_active)
-				continue;
-			obd_precleanup(class_exp2obd(lov->lov_tgts[i]->ltd_exp),
-				       OBD_CLEANUP_EARLY);
-		}
-		break;
-	}
-	default:
-		break;
-	}
-
-	return 0;
-}
-
 static int lov_cleanup(struct obd_device *obd)
 {
 	struct lov_obd *lov = &obd->u.lov;
@@ -972,163 +950,6 @@ out:
 	return rc;
 }
 
-#define ASSERT_LSM_MAGIC(lsmp)						  \
-do {									    \
-	LASSERT((lsmp));						\
-	LASSERTF(((lsmp)->lsm_magic == LOV_MAGIC_V1 ||			  \
-		 (lsmp)->lsm_magic == LOV_MAGIC_V3),			    \
-		 "%p->lsm_magic=%x\n", (lsmp), (lsmp)->lsm_magic);	      \
-} while (0)
-
-static int lov_getattr_interpret(struct ptlrpc_request_set *rqset,
-				 void *data, int rc)
-{
-	struct lov_request_set *lovset = (struct lov_request_set *)data;
-	int err;
-
-	/* don't do attribute merge if this async op failed */
-	if (rc)
-		atomic_set(&lovset->set_completes, 0);
-	err = lov_fini_getattr_set(lovset);
-	return rc ? rc : err;
-}
-
-static int lov_getattr_async(struct obd_export *exp, struct obd_info *oinfo,
-			     struct ptlrpc_request_set *rqset)
-{
-	struct lov_request_set *lovset;
-	struct lov_obd *lov;
-	struct lov_request *req;
-	int rc = 0, err;
-
-	LASSERT(oinfo);
-	ASSERT_LSM_MAGIC(oinfo->oi_md);
-
-	if (!exp || !exp->exp_obd)
-		return -ENODEV;
-
-	lov = &exp->exp_obd->u.lov;
-
-	rc = lov_prep_getattr_set(exp, oinfo, &lovset);
-	if (rc)
-		return rc;
-
-	CDEBUG(D_INFO, "objid "DOSTID": %ux%u byte stripes\n",
-	       POSTID(&oinfo->oi_md->lsm_oi), oinfo->oi_md->lsm_stripe_count,
-	       oinfo->oi_md->lsm_stripe_size);
-
-	list_for_each_entry(req, &lovset->set_list, rq_link) {
-		CDEBUG(D_INFO, "objid " DOSTID "[%d] has subobj " DOSTID " at idx%u\n",
-		       POSTID(&oinfo->oi_oa->o_oi), req->rq_stripe,
-		       POSTID(&req->rq_oi.oi_oa->o_oi), req->rq_idx);
-		rc = obd_getattr_async(lov->lov_tgts[req->rq_idx]->ltd_exp,
-				       &req->rq_oi, rqset);
-		if (rc) {
-			CERROR("%s: getattr objid "DOSTID" subobj"
-			       DOSTID" on OST idx %d: rc = %d\n",
-			       exp->exp_obd->obd_name,
-			       POSTID(&oinfo->oi_oa->o_oi),
-			       POSTID(&req->rq_oi.oi_oa->o_oi),
-			       req->rq_idx, rc);
-			goto out;
-		}
-	}
-
-	if (!list_empty(&rqset->set_requests)) {
-		LASSERT(rc == 0);
-		LASSERT(!rqset->set_interpret);
-		rqset->set_interpret = lov_getattr_interpret;
-		rqset->set_arg = (void *)lovset;
-		return rc;
-	}
-out:
-	if (rc)
-		atomic_set(&lovset->set_completes, 0);
-	err = lov_fini_getattr_set(lovset);
-	return rc ? rc : err;
-}
-
-static int lov_setattr_interpret(struct ptlrpc_request_set *rqset,
-				 void *data, int rc)
-{
-	struct lov_request_set *lovset = (struct lov_request_set *)data;
-	int err;
-
-	if (rc)
-		atomic_set(&lovset->set_completes, 0);
-	err = lov_fini_setattr_set(lovset);
-	return rc ? rc : err;
-}
-
-/* If @oti is given, the request goes from MDS and responses from OSTs are not
- * needed. Otherwise, a client is waiting for responses.
- */
-static int lov_setattr_async(struct obd_export *exp, struct obd_info *oinfo,
-			     struct obd_trans_info *oti,
-			     struct ptlrpc_request_set *rqset)
-{
-	struct lov_request_set *set;
-	struct lov_request *req;
-	struct lov_obd *lov;
-	int rc = 0;
-
-	LASSERT(oinfo);
-	ASSERT_LSM_MAGIC(oinfo->oi_md);
-	if (oinfo->oi_oa->o_valid & OBD_MD_FLCOOKIE) {
-		LASSERT(oti);
-		LASSERT(oti->oti_logcookies);
-	}
-
-	if (!exp || !exp->exp_obd)
-		return -ENODEV;
-
-	lov = &exp->exp_obd->u.lov;
-	rc = lov_prep_setattr_set(exp, oinfo, oti, &set);
-	if (rc)
-		return rc;
-
-	CDEBUG(D_INFO, "objid "DOSTID": %ux%u byte stripes\n",
-	       POSTID(&oinfo->oi_md->lsm_oi),
-	       oinfo->oi_md->lsm_stripe_count,
-	       oinfo->oi_md->lsm_stripe_size);
-
-	list_for_each_entry(req, &set->set_list, rq_link) {
-		if (oinfo->oi_oa->o_valid & OBD_MD_FLCOOKIE)
-			oti->oti_logcookies = set->set_cookies + req->rq_stripe;
-
-		CDEBUG(D_INFO, "objid " DOSTID "[%d] has subobj " DOSTID " at idx%u\n",
-		       POSTID(&oinfo->oi_oa->o_oi), req->rq_stripe,
-		       POSTID(&req->rq_oi.oi_oa->o_oi), req->rq_idx);
-
-		rc = obd_setattr_async(lov->lov_tgts[req->rq_idx]->ltd_exp,
-				       &req->rq_oi, oti, rqset);
-		if (rc) {
-			CERROR("error: setattr objid "DOSTID" subobj"
-			       DOSTID" on OST idx %d: rc = %d\n",
-			       POSTID(&set->set_oi->oi_oa->o_oi),
-			       POSTID(&req->rq_oi.oi_oa->o_oi),
-			       req->rq_idx, rc);
-			break;
-		}
-	}
-
-	/* If we are not waiting for responses on async requests, return. */
-	if (rc || !rqset || list_empty(&rqset->set_requests)) {
-		int err;
-
-		if (rc)
-			atomic_set(&set->set_completes, 0);
-		err = lov_fini_setattr_set(set);
-		return rc ? rc : err;
-	}
-
-	LASSERT(!rqset->set_interpret);
-	rqset->set_interpret = lov_setattr_interpret;
-	rqset->set_arg = (void *)set;
-
-	return 0;
-}
-
 int lov_statfs_interpret(struct ptlrpc_request_set *rqset, void *data, int rc)
 {
 	struct lov_request_set *lovset = (struct lov_request_set *)data;
@@ -1183,7 +1004,10 @@ static int lov_statfs(const struct lu_env *env, struct obd_export *exp,
 		      struct obd_statfs *osfs, __u64 max_age, __u32 flags)
 {
 	struct ptlrpc_request_set *set = NULL;
-	struct obd_info oinfo = { };
+	struct obd_info oinfo = {
+		.oi_osfs = osfs,
+		.oi_flags = flags,
+	};
 	int rc = 0;
 
 	/* for obdclass we forbid using obd_statfs_rqset, but prefer using async
@@ -1193,8 +1017,6 @@ static int lov_statfs(const struct lu_env *env, struct obd_export *exp,
 	if (!set)
 		return -ENOMEM;
 
-	oinfo.oi_osfs = osfs;
-	oinfo.oi_flags = flags;
 	rc = lov_statfs_async(exp, &oinfo, max_age, set);
 	if (rc == 0)
 		rc = ptlrpc_set_wait(set);
@@ -1235,8 +1057,8 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 
 		/* copy UUID */
 		if (copy_to_user(data->ioc_pbuf2, obd2cli_tgt(osc_obd),
-				 min((int)data->ioc_plen2,
-				     (int)sizeof(struct obd_uuid))))
+				 min_t(unsigned long, data->ioc_plen2,
+				       sizeof(struct obd_uuid))))
 			return -EFAULT;
 
 		memcpy(&flags, data->ioc_inlbuf1, sizeof(__u32));
@@ -1249,8 +1071,8 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 		if (rc)
 			return rc;
 		if (copy_to_user(data->ioc_pbuf1, &stat_buf,
-				 min((int)data->ioc_plen1,
-				     (int)sizeof(stat_buf))))
+				 min_t(unsigned long, data->ioc_plen1,
+				       sizeof(stat_buf))))
 			return -EFAULT;
 		break;
 	}
@@ -1367,8 +1189,6 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 			osc_obd->obd_force = obddev->obd_force;
 			err = obd_iocontrol(cmd, lov->lov_tgts[i]->ltd_exp,
 					    len, karg, uarg);
-			if (err == -ENODATA && cmd == OBD_IOC_POLL_QUOTACHECK)
-				return err;
 			if (err) {
 				if (lov->lov_tgts[i]->ltd_active) {
 					CDEBUG(err == -ENOTTY ?
@@ -1391,454 +1211,35 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 	return rc;
 }
 
-#define FIEMAP_BUFFER_SIZE 4096
-
-/**
- * Non-zero fe_logical indicates that this is a continuation FIEMAP
- * call. The local end offset and the device are sent in the first
- * fm_extent. This function calculates the stripe number from the index.
- * This function returns a stripe_no on which mapping is to be restarted.
- *
- * This function returns fm_end_offset which is the in-OST offset at which
- * mapping should be restarted. If fm_end_offset=0 is returned then caller
- * will re-calculate proper offset in next stripe.
- * Note that the first extent is passed to lov_get_info via the value field.
- *
- * \param fiemap fiemap request header
- * \param lsm striping information for the file
- * \param fm_start logical start of mapping
- * \param fm_end logical end of mapping
- * \param start_stripe starting stripe will be returned in this
- */
-static u64 fiemap_calc_fm_end_offset(struct ll_user_fiemap *fiemap,
-				     struct lov_stripe_md *lsm, u64 fm_start,
-				     u64 fm_end, int *start_stripe)
-{
-	u64 local_end = fiemap->fm_extents[0].fe_logical;
-	u64 lun_start, lun_end;
-	u64 fm_end_offset;
-	int stripe_no = -1, i;
-
-	if (fiemap->fm_extent_count == 0 ||
-	    fiemap->fm_extents[0].fe_logical == 0)
-		return 0;
-
-	/* Find out stripe_no from ost_index saved in the fe_device */
-	for (i = 0; i < lsm->lsm_stripe_count; i++) {
-		struct lov_oinfo *oinfo = lsm->lsm_oinfo[i];
-
-		if (lov_oinfo_is_dummy(oinfo))
-			continue;
-
-		if (oinfo->loi_ost_idx == fiemap->fm_extents[0].fe_device) {
-			stripe_no = i;
-			break;
-		}
-	}
-	if (stripe_no == -1)
-		return -EINVAL;
-
-	/* If we have finished mapping on previous device, shift logical
-	 * offset to start of next device
-	 */
-	if ((lov_stripe_intersects(lsm, stripe_no, fm_start, fm_end,
-				   &lun_start, &lun_end)) != 0 &&
-				   local_end < lun_end) {
-		fm_end_offset = local_end;
-		*start_stripe = stripe_no;
-	} else {
-		/* This is a special value to indicate that caller should
-		 * calculate offset in next stripe.
-		 */
-		fm_end_offset = 0;
-		*start_stripe = (stripe_no + 1) % lsm->lsm_stripe_count;
-	}
-
-	return fm_end_offset;
-}
-
-/**
- * We calculate on which OST the mapping will end. If the length of mapping
- * is greater than (stripe_size * stripe_count) then the last_stripe will
- * will be one just before start_stripe. Else we check if the mapping
- * intersects each OST and find last_stripe.
- * This function returns the last_stripe and also sets the stripe_count
- * over which the mapping is spread
- *
- * \param lsm striping information for the file
- * \param fm_start logical start of mapping
- * \param fm_end logical end of mapping
- * \param start_stripe starting stripe of the mapping
- * \param stripe_count the number of stripes across which to map is returned
- *
- * \retval last_stripe return the last stripe of the mapping
- */
-static int fiemap_calc_last_stripe(struct lov_stripe_md *lsm, u64 fm_start,
-				   u64 fm_end, int start_stripe,
-				   int *stripe_count)
-{
-	int last_stripe;
-	u64 obd_start, obd_end;
-	int i, j;
-
-	if (fm_end - fm_start > lsm->lsm_stripe_size * lsm->lsm_stripe_count) {
-		last_stripe = start_stripe < 1 ? lsm->lsm_stripe_count - 1 :
-							      start_stripe - 1;
-		*stripe_count = lsm->lsm_stripe_count;
-	} else {
-		for (j = 0, i = start_stripe; j < lsm->lsm_stripe_count;
-		     i = (i + 1) % lsm->lsm_stripe_count, j++) {
-			if ((lov_stripe_intersects(lsm, i, fm_start, fm_end,
-						   &obd_start, &obd_end)) == 0)
-				break;
-		}
-		*stripe_count = j;
-		last_stripe = (start_stripe + j - 1) % lsm->lsm_stripe_count;
-	}
-
-	return last_stripe;
-}
-
-/**
- * Set fe_device and copy extents from local buffer into main return buffer.
- *
- * \param fiemap fiemap request header
- * \param lcl_fm_ext array of local fiemap extents to be copied
- * \param ost_index OST index to be written into the fm_device field for each
-		    extent
- * \param ext_count number of extents to be copied
- * \param current_extent where to start copying in main extent array
- */
-static void fiemap_prepare_and_copy_exts(struct ll_user_fiemap *fiemap,
-					 struct ll_fiemap_extent *lcl_fm_ext,
-					 int ost_index, unsigned int ext_count,
-					 int current_extent)
-{
-	char *to;
-	int ext;
-
-	for (ext = 0; ext < ext_count; ext++) {
-		lcl_fm_ext[ext].fe_device = ost_index;
-		lcl_fm_ext[ext].fe_flags |= FIEMAP_EXTENT_NET;
-	}
-
-	/* Copy fm_extent's from fm_local to return buffer */
-	to = (char *)fiemap + fiemap_count_to_size(current_extent);
-	memcpy(to, lcl_fm_ext, ext_count * sizeof(struct ll_fiemap_extent));
-}
-
-/**
- * Break down the FIEMAP request and send appropriate calls to individual OSTs.
- * This also handles the restarting of FIEMAP calls in case mapping overflows
- * the available number of extents in single call.
- */
-static int lov_fiemap(struct lov_obd *lov, __u32 keylen, void *key,
-		      __u32 *vallen, void *val, struct lov_stripe_md *lsm)
-{
-	struct ll_fiemap_info_key *fm_key = key;
-	struct ll_user_fiemap *fiemap = val;
-	struct ll_user_fiemap *fm_local = NULL;
-	struct ll_fiemap_extent *lcl_fm_ext;
-	int count_local;
-	unsigned int get_num_extents = 0;
-	int ost_index = 0, actual_start_stripe, start_stripe;
-	u64 fm_start, fm_end, fm_length, fm_end_offset;
-	u64 curr_loc;
-	int current_extent = 0, rc = 0, i;
-	/* Whether have we collected enough extents */
-	bool enough = false;
-	int ost_eof = 0; /* EOF for object */
-	int ost_done = 0; /* done with required mapping for this OST? */
-	int last_stripe;
-	int cur_stripe = 0, cur_stripe_wrap = 0, stripe_count;
-	unsigned int buffer_size = FIEMAP_BUFFER_SIZE;
-
-	if (!lsm_has_objects(lsm)) {
-		if (lsm && lsm_is_released(lsm) && (fm_key->fiemap.fm_start <
-		    fm_key->oa.o_size)) {
-			/*
-			 * released file, return a minimal FIEMAP if
-			 * request fits in file-size.
-			 */
-			fiemap->fm_mapped_extents = 1;
-			fiemap->fm_extents[0].fe_logical =
-					fm_key->fiemap.fm_start;
-			if (fm_key->fiemap.fm_start + fm_key->fiemap.fm_length <
-			    fm_key->oa.o_size) {
-				fiemap->fm_extents[0].fe_length =
-					fm_key->fiemap.fm_length;
-			} else {
-				fiemap->fm_extents[0].fe_length =
-					fm_key->oa.o_size - fm_key->fiemap.fm_start;
-				fiemap->fm_extents[0].fe_flags |=
-						(FIEMAP_EXTENT_UNKNOWN |
-						 FIEMAP_EXTENT_LAST);
-			}
-		}
-		rc = 0;
-		goto out;
-	}
-
-	if (fiemap_count_to_size(fm_key->fiemap.fm_extent_count) < buffer_size)
-		buffer_size = fiemap_count_to_size(fm_key->fiemap.fm_extent_count);
-
-	fm_local = libcfs_kvzalloc(buffer_size, GFP_NOFS);
-	if (!fm_local) {
-		rc = -ENOMEM;
-		goto out;
-	}
-	lcl_fm_ext = &fm_local->fm_extents[0];
-
-	count_local = fiemap_size_to_count(buffer_size);
-
-	memcpy(fiemap, &fm_key->fiemap, sizeof(*fiemap));
-	fm_start = fiemap->fm_start;
-	fm_length = fiemap->fm_length;
-	/* Calculate start stripe, last stripe and length of mapping */
-	start_stripe = lov_stripe_number(lsm, fm_start);
-	actual_start_stripe = start_stripe;
-	fm_end = (fm_length == ~0ULL ? fm_key->oa.o_size :
-						fm_start + fm_length - 1);
-	/* If fm_length != ~0ULL but fm_start+fm_length-1 exceeds file size */
-	if (fm_end > fm_key->oa.o_size)
-		fm_end = fm_key->oa.o_size;
-
-	last_stripe = fiemap_calc_last_stripe(lsm, fm_start, fm_end,
-					      actual_start_stripe,
-					      &stripe_count);
-
-	fm_end_offset = fiemap_calc_fm_end_offset(fiemap, lsm, fm_start,
-						  fm_end, &start_stripe);
-	if (fm_end_offset == -EINVAL) {
-		rc = -EINVAL;
-		goto out;
-	}
-
-	if (fiemap_count_to_size(fiemap->fm_extent_count) > *vallen)
-		fiemap->fm_extent_count = fiemap_size_to_count(*vallen);
-	if (fiemap->fm_extent_count == 0) {
-		get_num_extents = 1;
-		count_local = 0;
-	}
-	/* Check each stripe */
-	for (cur_stripe = start_stripe, i = 0; i < stripe_count;
-	     i++, cur_stripe = (cur_stripe + 1) % lsm->lsm_stripe_count) {
-		u64 req_fm_len; /* Stores length of required mapping */
-		u64 len_mapped_single_call;
-		u64 lun_start, lun_end, obd_object_end;
-		unsigned int ext_count;
-
-		cur_stripe_wrap = cur_stripe;
-
-		/* Find out range of mapping on this stripe */
-		if ((lov_stripe_intersects(lsm, cur_stripe, fm_start, fm_end,
-					   &lun_start, &obd_object_end)) == 0)
-			continue;
-
-		if (lov_oinfo_is_dummy(lsm->lsm_oinfo[cur_stripe])) {
-			rc = -EIO;
-			goto out;
-		}
-
-		/* If this is a continuation FIEMAP call and we are on
-		 * starting stripe then lun_start needs to be set to
-		 * fm_end_offset
-		 */
-		if (fm_end_offset != 0 && cur_stripe == start_stripe)
-			lun_start = fm_end_offset;
-
-		if (fm_length != ~0ULL) {
-			/* Handle fm_start + fm_length overflow */
-			if (fm_start + fm_length < fm_start)
-				fm_length = ~0ULL - fm_start;
-			lun_end = lov_size_to_stripe(lsm, fm_start + fm_length,
-						     cur_stripe);
-		} else {
-			lun_end = ~0ULL;
-		}
-
-		if (lun_start == lun_end)
-			continue;
-
-		req_fm_len = obd_object_end - lun_start;
-		fm_local->fm_length = 0;
-		len_mapped_single_call = 0;
-
-		/* If the output buffer is very large and the objects have many
-		 * extents we may need to loop on a single OST repeatedly
-		 */
-		ost_eof = 0;
-		ost_done = 0;
-		do {
-			if (get_num_extents == 0) {
-				/* Don't get too many extents. */
-				if (current_extent + count_local >
-				    fiemap->fm_extent_count)
-					count_local = fiemap->fm_extent_count -
-								 current_extent;
-			}
-
-			lun_start += len_mapped_single_call;
-			fm_local->fm_length = req_fm_len - len_mapped_single_call;
-			req_fm_len = fm_local->fm_length;
-			fm_local->fm_extent_count = enough ? 1 : count_local;
-			fm_local->fm_mapped_extents = 0;
-			fm_local->fm_flags = fiemap->fm_flags;
-
-			fm_key->oa.o_oi = lsm->lsm_oinfo[cur_stripe]->loi_oi;
-			ost_index = lsm->lsm_oinfo[cur_stripe]->loi_ost_idx;
-
-			if (ost_index < 0 ||
-			    ost_index >= lov->desc.ld_tgt_count) {
-				rc = -EINVAL;
-				goto out;
-			}
-
-			/* If OST is inactive, return extent with UNKNOWN flag */
-			if (!lov->lov_tgts[ost_index]->ltd_active) {
-				fm_local->fm_flags |= FIEMAP_EXTENT_LAST;
-				fm_local->fm_mapped_extents = 1;
-
-				lcl_fm_ext[0].fe_logical = lun_start;
-				lcl_fm_ext[0].fe_length = obd_object_end -
-								      lun_start;
-				lcl_fm_ext[0].fe_flags |= FIEMAP_EXTENT_UNKNOWN;
-
-				goto inactive_tgt;
-			}
-
-			fm_local->fm_start = lun_start;
-			fm_local->fm_flags &= ~FIEMAP_FLAG_DEVICE_ORDER;
-			memcpy(&fm_key->fiemap, fm_local, sizeof(*fm_local));
-			*vallen = fiemap_count_to_size(fm_local->fm_extent_count);
-			rc = obd_get_info(NULL,
-					  lov->lov_tgts[ost_index]->ltd_exp,
-					  keylen, key, vallen, fm_local, lsm);
-			if (rc != 0)
-				goto out;
-
-inactive_tgt:
-			ext_count = fm_local->fm_mapped_extents;
-			if (ext_count == 0) {
-				ost_done = 1;
-				/* If last stripe has hole at the end,
-				 * then we need to return
-				 */
-				if (cur_stripe_wrap == last_stripe) {
-					fiemap->fm_mapped_extents = 0;
-					goto finish;
-				}
-				break;
-			} else if (enough) {
-				/*
-				 * We've collected enough extents and there are
-				 * more extents after it.
-				 */
-				goto finish;
-			}
-
-			/* If we just need num of extents then go to next device */
-			if (get_num_extents) {
-				current_extent += ext_count;
-				break;
-			}
-
-			len_mapped_single_call =
-				lcl_fm_ext[ext_count - 1].fe_logical -
-				lun_start + lcl_fm_ext[ext_count - 1].fe_length;
-
-			/* Have we finished mapping on this device? */
-			if (req_fm_len <= len_mapped_single_call)
-				ost_done = 1;
-
-			/* Clear the EXTENT_LAST flag which can be present on
-			 * last extent
-			 */
-			if (lcl_fm_ext[ext_count - 1].fe_flags &
-			    FIEMAP_EXTENT_LAST)
-				lcl_fm_ext[ext_count - 1].fe_flags &=
-							    ~FIEMAP_EXTENT_LAST;
-
-			curr_loc = lov_stripe_size(lsm,
-					lcl_fm_ext[ext_count - 1].fe_logical +
-					lcl_fm_ext[ext_count - 1].fe_length,
-					cur_stripe);
-			if (curr_loc >= fm_key->oa.o_size)
-				ost_eof = 1;
-
-			fiemap_prepare_and_copy_exts(fiemap, lcl_fm_ext,
-						     ost_index, ext_count,
-						     current_extent);
-
-			current_extent += ext_count;
-
-			/* Ran out of available extents? */
-			if (current_extent >= fiemap->fm_extent_count)
-				enough = true;
-		} while (ost_done == 0 && ost_eof == 0);
-
-		if (cur_stripe_wrap == last_stripe)
-			goto finish;
-	}
-
-finish:
-	/* Indicate that we are returning device offsets unless file just has
-	 * single stripe
-	 */
-	if (lsm->lsm_stripe_count > 1)
-		fiemap->fm_flags |= FIEMAP_FLAG_DEVICE_ORDER;
-
-	if (get_num_extents)
-		goto skip_last_device_calc;
-
-	/* Check if we have reached the last stripe and whether mapping for that
-	 * stripe is done.
-	 */
-	if (cur_stripe_wrap == last_stripe) {
-		if (ost_done || ost_eof)
-			fiemap->fm_extents[current_extent - 1].fe_flags |=
-							     FIEMAP_EXTENT_LAST;
-	}
-
-skip_last_device_calc:
-	fiemap->fm_mapped_extents = current_extent;
-
-out:
-	kvfree(fm_local);
-	return rc;
-}
-
 static int lov_get_info(const struct lu_env *env, struct obd_export *exp,
-			__u32 keylen, void *key, __u32 *vallen, void *val,
-			struct lov_stripe_md *lsm)
+			__u32 keylen, void *key, __u32 *vallen, void *val)
 {
 	struct obd_device *obddev = class_exp2obd(exp);
 	struct lov_obd *lov = &obddev->u.lov;
-	int rc;
+	struct lov_desc *ld = &lov->desc;
+	int rc = 0;
 
 	if (!vallen || !val)
 		return -EFAULT;
 
 	obd_getref(obddev);
 
-	if (KEY_IS(KEY_LOVDESC)) {
-		struct lov_desc *desc_ret = val;
-		*desc_ret = lov->desc;
+	if (KEY_IS(KEY_MAX_EASIZE)) {
+		u32 max_stripe_count = min_t(u32, ld->ld_active_tgt_count,
+					     LOV_MAX_STRIPE_COUNT);
 
-		rc = 0;
-		goto out;
-	} else if (KEY_IS(KEY_FIEMAP)) {
-		rc = lov_fiemap(lov, keylen, key, vallen, val, lsm);
-		goto out;
+		*((u32 *)val) = lov_mds_md_size(max_stripe_count, LOV_MAGIC_V3);
+	} else if (KEY_IS(KEY_DEFAULT_EASIZE)) {
+		u32 def_stripe_count = min_t(u32, ld->ld_default_stripe_count,
+					     LOV_MAX_STRIPE_COUNT);
+
+		*((u32 *)val) = lov_mds_md_size(def_stripe_count, LOV_MAGIC_V3);
 	} else if (KEY_IS(KEY_TGT_COUNT)) {
 		*((int *)val) = lov->desc.ld_tgt_count;
-		rc = 0;
-		goto out;
+	} else {
+		rc = -EINVAL;
 	}
 
-	rc = -EINVAL;
-
-out:
 	obd_putref(obddev);
 	return rc;
 }
@@ -1926,12 +1327,8 @@ static int lov_quotactl(struct obd_device *obd, struct obd_export *exp,
 	__u64		bhardlimit = 0;
 	int		  i, rc = 0;
 
-	if (oqctl->qc_cmd != LUSTRE_Q_QUOTAON &&
-	    oqctl->qc_cmd != LUSTRE_Q_QUOTAOFF &&
-	    oqctl->qc_cmd != Q_GETOQUOTA &&
-	    oqctl->qc_cmd != Q_INITQUOTA &&
-	    oqctl->qc_cmd != LUSTRE_Q_SETQUOTA &&
-	    oqctl->qc_cmd != Q_FINVALIDATE) {
+	if (oqctl->qc_cmd != Q_GETOQUOTA &&
+	    oqctl->qc_cmd != LUSTRE_Q_SETQUOTA) {
 		CERROR("bad quota opc %x for lov obd\n", oqctl->qc_cmd);
 		return -EFAULT;
 	}
@@ -1978,63 +1375,15 @@ static int lov_quotactl(struct obd_device *obd, struct obd_export *exp,
 	return rc;
 }
 
-static int lov_quotacheck(struct obd_device *obd, struct obd_export *exp,
-			  struct obd_quotactl *oqctl)
-{
-	struct lov_obd *lov = &obd->u.lov;
-	int	     i, rc = 0;
-
-	obd_getref(obd);
-
-	for (i = 0; i < lov->desc.ld_tgt_count; i++) {
-		if (!lov->lov_tgts[i])
-			continue;
-
-		/* Skip quota check on the administratively disabled OSTs. */
-		if (!lov->lov_tgts[i]->ltd_activate) {
-			CWARN("lov idx %d was administratively disabled, skip quotacheck on it.\n",
-			      i);
-			continue;
-		}
-
-		if (!lov->lov_tgts[i]->ltd_active) {
-			CERROR("lov idx %d inactive\n", i);
-			rc = -EIO;
-			goto out;
-		}
-	}
-
-	for (i = 0; i < lov->desc.ld_tgt_count; i++) {
-		int err;
-
-		if (!lov->lov_tgts[i] || !lov->lov_tgts[i]->ltd_activate)
-			continue;
-
-		err = obd_quotacheck(lov->lov_tgts[i]->ltd_exp, oqctl);
-		if (err && !rc)
-			rc = err;
-	}
-
-out:
-	obd_putref(obd);
-
-	return rc;
-}
-
 static struct obd_ops lov_obd_ops = {
 	.owner          = THIS_MODULE,
 	.setup          = lov_setup,
-	.precleanup     = lov_precleanup,
 	.cleanup        = lov_cleanup,
 	/*.process_config       = lov_process_config,*/
 	.connect        = lov_connect,
 	.disconnect     = lov_disconnect,
 	.statfs         = lov_statfs,
 	.statfs_async   = lov_statfs_async,
-	.packmd         = lov_packmd,
-	.unpackmd       = lov_unpackmd,
-	.getattr_async  = lov_getattr_async,
-	.setattr_async  = lov_setattr_async,
 	.iocontrol      = lov_iocontrol,
 	.get_info       = lov_get_info,
 	.set_info_async = lov_set_info_async,
@@ -2046,7 +1395,6 @@ static struct obd_ops lov_obd_ops = {
 	.getref         = lov_getref,
 	.putref         = lov_putref,
 	.quotactl       = lov_quotactl,
-	.quotacheck     = lov_quotacheck,
 };
 
 struct kmem_cache *lov_oinfo_slab;
diff --git a/drivers/staging/lustre/lustre/lov/lov_object.c b/drivers/staging/lustre/lustre/lov/lov_object.c
index 52f736338887..76d4256fa828 100644
--- a/drivers/staging/lustre/lustre/lov/lov_object.c
+++ b/drivers/staging/lustre/lustre/lov/lov_object.c
@@ -39,6 +39,11 @@
 
 #include "lov_cl_internal.h"
 
+static inline struct lov_device *lov_object_dev(struct lov_object *obj)
+{
+	return lu2lov_dev(obj->lo_cl.co_lu.lo_dev);
+}
+
 /** \addtogroup lov
  *  @{
  */
@@ -51,7 +56,7 @@
 
 struct lov_layout_operations {
 	int (*llo_init)(const struct lu_env *env, struct lov_device *dev,
-			struct lov_object *lov,
+			struct lov_object *lov, struct lov_stripe_md *lsm,
 			const struct cl_object_conf *conf,
 			union lov_layout_state *state);
 	int (*llo_delete)(const struct lu_env *env, struct lov_object *lov,
@@ -75,12 +80,11 @@ struct lov_layout_operations {
 
 static int lov_layout_wait(const struct lu_env *env, struct lov_object *lov);
 
-void lov_lsm_put(struct cl_object *unused, struct lov_stripe_md *lsm)
+static void lov_lsm_put(struct lov_stripe_md *lsm)
 {
 	if (lsm)
 		lov_free_memmd(&lsm);
 }
-EXPORT_SYMBOL(lov_lsm_put);
 
 /*****************************************************************************
  *
@@ -97,17 +101,17 @@ static void lov_install_empty(const struct lu_env *env,
 	 */
 }
 
-static int lov_init_empty(const struct lu_env *env,
-			  struct lov_device *dev, struct lov_object *lov,
+static int lov_init_empty(const struct lu_env *env, struct lov_device *dev,
+			  struct lov_object *lov, struct lov_stripe_md *lsm,
 			  const struct cl_object_conf *conf,
-			  union  lov_layout_state *state)
+			  union lov_layout_state *state)
 {
 	return 0;
 }
 
 static void lov_install_raid0(const struct lu_env *env,
 			      struct lov_object *lov,
-			      union  lov_layout_state *state)
+			      union lov_layout_state *state)
 {
 }
 
@@ -212,8 +216,8 @@ static int lov_page_slice_fixup(struct lov_object *lov,
 	return cl_object_header(stripe)->coh_page_bufsize;
 }
 
-static int lov_init_raid0(const struct lu_env *env,
-			  struct lov_device *dev, struct lov_object *lov,
+static int lov_init_raid0(const struct lu_env *env, struct lov_device *dev,
+			  struct lov_object *lov, struct lov_stripe_md *lsm,
 			  const struct cl_object_conf *conf,
 			  union  lov_layout_state *state)
 {
@@ -223,7 +227,6 @@ static int lov_init_raid0(const struct lu_env *env,
 	struct cl_object	*stripe;
 	struct lov_thread_info  *lti     = lov_env_info(env);
 	struct cl_object_conf   *subconf = &lti->lti_stripe_conf;
-	struct lov_stripe_md    *lsm     = conf->u.coc_md->lsm;
 	struct lu_fid	   *ofid    = &lti->lti_fid;
 	struct lov_layout_raid0 *r0      = &state->raid0;
 
@@ -298,13 +301,11 @@ out:
 	return result;
 }
 
-static int lov_init_released(const struct lu_env *env,
-			     struct lov_device *dev, struct lov_object *lov,
+static int lov_init_released(const struct lu_env *env, struct lov_device *dev,
+			     struct lov_object *lov, struct lov_stripe_md *lsm,
 			     const struct cl_object_conf *conf,
 			     union  lov_layout_state *state)
 {
-	struct lov_stripe_md *lsm = conf->u.coc_md->lsm;
-
 	LASSERT(lsm);
 	LASSERT(lsm_is_released(lsm));
 	LASSERT(!lov->lo_lsm);
@@ -313,6 +314,40 @@ static int lov_init_released(const struct lu_env *env,
 	return 0;
 }
 
+static struct cl_object *lov_find_subobj(const struct lu_env *env,
+					 struct lov_object *lov,
+					 struct lov_stripe_md *lsm,
+					 int stripe_idx)
+{
+	struct lov_device *dev = lu2lov_dev(lov2lu(lov)->lo_dev);
+	struct lov_oinfo *oinfo = lsm->lsm_oinfo[stripe_idx];
+	struct lov_thread_info *lti = lov_env_info(env);
+	struct lu_fid *ofid = &lti->lti_fid;
+	struct cl_device *subdev;
+	struct cl_object *result;
+	int ost_idx;
+	int rc;
+
+	if (lov->lo_type != LLT_RAID0) {
+		result = NULL;
+		goto out;
+	}
+
+	ost_idx = oinfo->loi_ost_idx;
+	rc = ostid_to_fid(ofid, &oinfo->loi_oi, ost_idx);
+	if (rc) {
+		result = NULL;
+		goto out;
+	}
+
+	subdev = lovsub2cl_dev(dev->ld_target[ost_idx]);
+	result = lov_sub_find(env, subdev, ofid, NULL);
+out:
+	if (!result)
+		result = ERR_PTR(-EINVAL);
+	return result;
+}
+
 static int lov_delete_empty(const struct lu_env *env, struct lov_object *lov,
 			    union lov_layout_state *state)
 {
@@ -687,31 +722,24 @@ static int lov_layout_wait(const struct lu_env *env, struct lov_object *lov)
 }
 
 static int lov_layout_change(const struct lu_env *unused,
-			     struct lov_object *lov,
+			     struct lov_object *lov, struct lov_stripe_md *lsm,
 			     const struct cl_object_conf *conf)
 {
-	int result;
-	enum lov_layout_type llt = LLT_EMPTY;
+	enum lov_layout_type llt = lov_type(lsm);
 	union lov_layout_state *state = &lov->u;
 	const struct lov_layout_operations *old_ops;
 	const struct lov_layout_operations *new_ops;
-
-	void *cookie;
 	struct lu_env *env;
 	int refcheck;
+	int rc;
 
 	LASSERT(0 <= lov->lo_type && lov->lo_type < ARRAY_SIZE(lov_dispatch));
 
-	if (conf->u.coc_md)
-		llt = lov_type(conf->u.coc_md->lsm);
-	LASSERT(0 <= llt && llt < ARRAY_SIZE(lov_dispatch));
-
-	cookie = cl_env_reenter();
 	env = cl_env_get(&refcheck);
-	if (IS_ERR(env)) {
-		cl_env_reexit(cookie);
+	if (IS_ERR(env))
 		return PTR_ERR(env);
-	}
+
+	LASSERT(0 <= llt && llt < ARRAY_SIZE(lov_dispatch));
 
 	CDEBUG(D_INODE, DFID" from %s to %s\n",
 	       PFID(lu_object_fid(lov2lu(lov))),
@@ -720,38 +748,37 @@ static int lov_layout_change(const struct lu_env *unused,
 	old_ops = &lov_dispatch[lov->lo_type];
 	new_ops = &lov_dispatch[llt];
 
-	result = cl_object_prune(env, &lov->lo_cl);
-	if (result != 0)
+	rc = cl_object_prune(env, &lov->lo_cl);
+	if (rc)
+		goto out;
+
+	rc = old_ops->llo_delete(env, lov, &lov->u);
+	if (rc)
 		goto out;
 
-	result = old_ops->llo_delete(env, lov, &lov->u);
-	if (result == 0) {
-		old_ops->llo_fini(env, lov, &lov->u);
+	old_ops->llo_fini(env, lov, &lov->u);
 
-		LASSERT(atomic_read(&lov->lo_active_ios) == 0);
+	LASSERT(!atomic_read(&lov->lo_active_ios));
 
-		lov->lo_type = LLT_EMPTY;
-		/* page bufsize fixup */
-		cl_object_header(&lov->lo_cl)->coh_page_bufsize -=
+	lov->lo_type = LLT_EMPTY;
+
+	/* page bufsize fixup */
+	cl_object_header(&lov->lo_cl)->coh_page_bufsize -=
 			lov_page_slice_fixup(lov, NULL);
 
-		result = new_ops->llo_init(env,
-					lu2lov_dev(lov->lo_cl.co_lu.lo_dev),
-					lov, conf, state);
-		if (result == 0) {
-			new_ops->llo_install(env, lov, state);
-			lov->lo_type = llt;
-		} else {
-			new_ops->llo_delete(env, lov, state);
-			new_ops->llo_fini(env, lov, state);
-			/* this file becomes an EMPTY file. */
-		}
+	rc = new_ops->llo_init(env, lov_object_dev(lov), lov, lsm, conf, state);
+	if (rc) {
+		new_ops->llo_delete(env, lov, state);
+		new_ops->llo_fini(env, lov, state);
+		/* this file becomes an EMPTY file. */
+		goto out;
 	}
 
+	new_ops->llo_install(env, lov, state);
+	lov->lo_type = llt;
 out:
 	cl_env_put(env, &refcheck);
-	cl_env_reexit(cookie);
-	return result;
+	return rc;
 }
 
 /*****************************************************************************
@@ -762,26 +789,38 @@ out:
 int lov_object_init(const struct lu_env *env, struct lu_object *obj,
 		    const struct lu_object_conf *conf)
 {
-	struct lov_device	    *dev   = lu2lov_dev(obj->lo_dev);
 	struct lov_object	    *lov   = lu2lov(obj);
+	struct lov_device *dev = lov_object_dev(lov);
 	const struct cl_object_conf  *cconf = lu2cl_conf(conf);
 	union  lov_layout_state      *set   = &lov->u;
 	const struct lov_layout_operations *ops;
-	int result;
+	struct lov_stripe_md *lsm = NULL;
+	int rc;
 
 	init_rwsem(&lov->lo_type_guard);
 	atomic_set(&lov->lo_active_ios, 0);
 	init_waitqueue_head(&lov->lo_waitq);
-
 	cl_object_page_init(lu2cl(obj), sizeof(struct lov_page));
 
+	lov->lo_type = LLT_EMPTY;
+	if (cconf->u.coc_layout.lb_buf) {
+		lsm = lov_unpackmd(dev->ld_lov,
+				   cconf->u.coc_layout.lb_buf,
+				   cconf->u.coc_layout.lb_len);
+		if (IS_ERR(lsm))
+			return PTR_ERR(lsm);
+	}
+
 	/* no locking is necessary, as object is being created */
-	lov->lo_type = lov_type(cconf->u.coc_md->lsm);
+	lov->lo_type = lov_type(lsm);
 	ops = &lov_dispatch[lov->lo_type];
-	result = ops->llo_init(env, dev, lov, cconf, set);
-	if (result == 0)
+	rc = ops->llo_init(env, dev, lov, lsm, cconf, set);
+	if (!rc)
 		ops->llo_install(env, lov, set);
-	return result;
+
+	lov_lsm_put(lsm);
+
+	return rc;
 }
 
 static int lov_conf_set(const struct lu_env *env, struct cl_object *obj,
@@ -791,6 +830,15 @@ static int lov_conf_set(const struct lu_env *env, struct cl_object *obj,
 	struct lov_object	*lov = cl2lov(obj);
 	int			 result = 0;
 
+	if (conf->coc_opc == OBJECT_CONF_SET &&
+	    conf->u.coc_layout.lb_buf) {
+		lsm = lov_unpackmd(lov_object_dev(lov)->ld_lov,
+				   conf->u.coc_layout.lb_buf,
+				   conf->u.coc_layout.lb_len);
+		if (IS_ERR(lsm))
+			return PTR_ERR(lsm);
+	}
+
 	lov_conf_lock(lov);
 	if (conf->coc_opc == OBJECT_CONF_INVALIDATE) {
 		lov->lo_layout_invalid = true;
@@ -810,8 +858,6 @@ static int lov_conf_set(const struct lu_env *env, struct cl_object *obj,
 
 	LASSERT(conf->coc_opc == OBJECT_CONF_SET);
 
-	if (conf->u.coc_md)
-		lsm = conf->u.coc_md->lsm;
 	if ((!lsm && !lov->lo_lsm) ||
 	    ((lsm && lov->lo_lsm) &&
 	     (lov->lo_lsm->lsm_layout_gen == lsm->lsm_layout_gen) &&
@@ -829,11 +875,12 @@ static int lov_conf_set(const struct lu_env *env, struct cl_object *obj,
 		goto out;
 	}
 
-	result = lov_layout_change(env, lov, conf);
+	result = lov_layout_change(env, lov, lsm, conf);
 	lov->lo_layout_invalid = result != 0;
 
 out:
 	lov_conf_unlock(lov);
+	lov_lsm_put(lsm);
 	CDEBUG(D_INODE, DFID" lo_layout_invalid=%d\n",
 	       PFID(lu_object_fid(lov2lu(lov))), lov->lo_layout_invalid);
 	return result;
@@ -911,6 +958,473 @@ int lov_lock_init(const struct lu_env *env, struct cl_object *obj,
 				    io);
 }
 
+/**
+ * We calculate on which OST the mapping will end. If the length of mapping
+ * is greater than (stripe_size * stripe_count) then the last_stripe will
+ * will be one just before start_stripe. Else we check if the mapping
+ * intersects each OST and find last_stripe.
+ * This function returns the last_stripe and also sets the stripe_count
+ * over which the mapping is spread
+ *
+ * \param lsm [in]		striping information for the file
+ * \param fm_start [in]		logical start of mapping
+ * \param fm_end [in]		logical end of mapping
+ * \param start_stripe [in]	starting stripe of the mapping
+ * \param stripe_count [out]	the number of stripes across which to map is
+ *				returned
+ *
+ * \retval last_stripe		return the last stripe of the mapping
+ */
+static int fiemap_calc_last_stripe(struct lov_stripe_md *lsm,
+				   loff_t fm_start, loff_t fm_end,
+				   int start_stripe, int *stripe_count)
+{
+	int last_stripe;
+	loff_t obd_start;
+	loff_t obd_end;
+	int i, j;
+
+	if (fm_end - fm_start > lsm->lsm_stripe_size * lsm->lsm_stripe_count) {
+		last_stripe = (start_stripe < 1 ? lsm->lsm_stripe_count - 1 :
+			       start_stripe - 1);
+		*stripe_count = lsm->lsm_stripe_count;
+	} else {
+		for (j = 0, i = start_stripe; j < lsm->lsm_stripe_count;
+		     i = (i + 1) % lsm->lsm_stripe_count, j++) {
+			if (!(lov_stripe_intersects(lsm, i, fm_start, fm_end,
+						    &obd_start, &obd_end)))
+				break;
+		}
+		*stripe_count = j;
+		last_stripe = (start_stripe + j - 1) % lsm->lsm_stripe_count;
+	}
+
+	return last_stripe;
+}
+
+/**
+ * Set fe_device and copy extents from local buffer into main return buffer.
+ *
+ * \param fiemap [out]		fiemap to hold all extents
+ * \param lcl_fm_ext [in]	array of fiemap extents get from OSC layer
+ * \param ost_index [in]	OST index to be written into the fm_device
+ *				field for each extent
+ * \param ext_count [in]	number of extents to be copied
+ * \param current_extent [in]	where to start copying in the extent array
+ */
+static void fiemap_prepare_and_copy_exts(struct fiemap *fiemap,
+					 struct fiemap_extent *lcl_fm_ext,
+					 int ost_index, unsigned int ext_count,
+					 int current_extent)
+{
+	unsigned int ext;
+	char *to;
+
+	for (ext = 0; ext < ext_count; ext++) {
+		lcl_fm_ext[ext].fe_device = ost_index;
+		lcl_fm_ext[ext].fe_flags |= FIEMAP_EXTENT_NET;
+	}
+
+	/* Copy fm_extent's from fm_local to return buffer */
+	to = (char *)fiemap + fiemap_count_to_size(current_extent);
+	memcpy(to, lcl_fm_ext, ext_count * sizeof(struct fiemap_extent));
+}
+
+#define FIEMAP_BUFFER_SIZE 4096
+
+/**
+ * Non-zero fe_logical indicates that this is a continuation FIEMAP
+ * call. The local end offset and the device are sent in the first
+ * fm_extent. This function calculates the stripe number from the index.
+ * This function returns a stripe_no on which mapping is to be restarted.
+ *
+ * This function returns fm_end_offset which is the in-OST offset at which
+ * mapping should be restarted. If fm_end_offset=0 is returned then caller
+ * will re-calculate proper offset in next stripe.
+ * Note that the first extent is passed to lov_get_info via the value field.
+ *
+ * \param fiemap [in]		fiemap request header
+ * \param lsm [in]		striping information for the file
+ * \param fm_start [in]		logical start of mapping
+ * \param fm_end [in]		logical end of mapping
+ * \param start_stripe [out]	starting stripe will be returned in this
+ */
+static loff_t fiemap_calc_fm_end_offset(struct fiemap *fiemap,
+					struct lov_stripe_md *lsm,
+					loff_t fm_start, loff_t fm_end,
+					int *start_stripe)
+{
+	loff_t local_end = fiemap->fm_extents[0].fe_logical;
+	loff_t lun_start, lun_end;
+	loff_t fm_end_offset;
+	int stripe_no = -1;
+	int i;
+
+	if (!fiemap->fm_extent_count || !fiemap->fm_extents[0].fe_logical)
+		return 0;
+
+	/* Find out stripe_no from ost_index saved in the fe_device */
+	for (i = 0; i < lsm->lsm_stripe_count; i++) {
+		struct lov_oinfo *oinfo = lsm->lsm_oinfo[i];
+
+		if (lov_oinfo_is_dummy(oinfo))
+			continue;
+
+		if (oinfo->loi_ost_idx == fiemap->fm_extents[0].fe_device) {
+			stripe_no = i;
+			break;
+		}
+	}
+
+	if (stripe_no == -1)
+		return -EINVAL;
+
+	/*
+	 * If we have finished mapping on previous device, shift logical
+	 * offset to start of next device
+	 */
+	if (lov_stripe_intersects(lsm, stripe_no, fm_start, fm_end,
+				  &lun_start, &lun_end) &&
+	    local_end < lun_end) {
+		fm_end_offset = local_end;
+		*start_stripe = stripe_no;
+	} else {
+		/* This is a special value to indicate that caller should
+		 * calculate offset in next stripe.
+		 */
+		fm_end_offset = 0;
+		*start_stripe = (stripe_no + 1) % lsm->lsm_stripe_count;
+	}
+
+	return fm_end_offset;
+}
+
+/**
+ * Break down the FIEMAP request and send appropriate calls to individual OSTs.
+ * This also handles the restarting of FIEMAP calls in case mapping overflows
+ * the available number of extents in single call.
+ *
+ * \param env [in]		lustre environment
+ * \param obj [in]		file object
+ * \param fmkey [in]		fiemap request header and other info
+ * \param fiemap [out]		fiemap buffer holding retrived map extents
+ * \param buflen [in/out]	max buffer length of @fiemap, when iterate
+ *				each OST, it is used to limit max map needed
+ * \retval 0	success
+ * \retval < 0	error
+ */
+static int lov_object_fiemap(const struct lu_env *env, struct cl_object *obj,
+			     struct ll_fiemap_info_key *fmkey,
+			     struct fiemap *fiemap, size_t *buflen)
+{
+	struct lov_obd *lov = lu2lov_dev(obj->co_lu.lo_dev)->ld_lov;
+	unsigned int buffer_size = FIEMAP_BUFFER_SIZE;
+	struct fiemap_extent *lcl_fm_ext;
+	struct cl_object *subobj = NULL;
+	struct fiemap *fm_local = NULL;
+	struct lov_stripe_md *lsm;
+	loff_t fm_start;
+	loff_t fm_end;
+	loff_t fm_length;
+	loff_t fm_end_offset;
+	int count_local;
+	int ost_index = 0;
+	int start_stripe;
+	int current_extent = 0;
+	int rc = 0;
+	int last_stripe;
+	int cur_stripe = 0;
+	int cur_stripe_wrap = 0;
+	int stripe_count;
+	/* Whether have we collected enough extents */
+	bool enough = false;
+	/* EOF for object */
+	bool ost_eof = false;
+	/* done with required mapping for this OST? */
+	bool ost_done = false;
+
+	lsm = lov_lsm_addref(cl2lov(obj));
+	if (!lsm)
+		return -ENODATA;
+
+	/**
+	 * If the stripe_count > 1 and the application does not understand
+	 * DEVICE_ORDER flag, it cannot interpret the extents correctly.
+	 */
+	if (lsm->lsm_stripe_count > 1 &&
+	    !(fiemap->fm_flags & FIEMAP_FLAG_DEVICE_ORDER)) {
+		rc = -ENOTSUPP;
+		goto out;
+	}
+
+	if (lsm_is_released(lsm)) {
+		if (fiemap->fm_start < fmkey->lfik_oa.o_size) {
+			/**
+			 * released file, return a minimal FIEMAP if
+			 * request fits in file-size.
+			 */
+			fiemap->fm_mapped_extents = 1;
+			fiemap->fm_extents[0].fe_logical = fiemap->fm_start;
+			if (fiemap->fm_start + fiemap->fm_length <
+			    fmkey->lfik_oa.o_size)
+				fiemap->fm_extents[0].fe_length =
+					 fiemap->fm_length;
+			else
+				fiemap->fm_extents[0].fe_length =
+					fmkey->lfik_oa.o_size -
+					fiemap->fm_start;
+			fiemap->fm_extents[0].fe_flags |=
+				FIEMAP_EXTENT_UNKNOWN | FIEMAP_EXTENT_LAST;
+		}
+		rc = 0;
+		goto out;
+	}
+
+	if (fiemap_count_to_size(fiemap->fm_extent_count) < buffer_size)
+		buffer_size = fiemap_count_to_size(fiemap->fm_extent_count);
+
+	fm_local = libcfs_kvzalloc(buffer_size, GFP_NOFS);
+	if (!fm_local) {
+		rc = -ENOMEM;
+		goto out;
+	}
+	lcl_fm_ext = &fm_local->fm_extents[0];
+	count_local = fiemap_size_to_count(buffer_size);
+
+	fm_start = fiemap->fm_start;
+	fm_length = fiemap->fm_length;
+	/* Calculate start stripe, last stripe and length of mapping */
+	start_stripe = lov_stripe_number(lsm, fm_start);
+	fm_end = (fm_length == ~0ULL) ? fmkey->lfik_oa.o_size :
+					fm_start + fm_length - 1;
+	/* If fm_length != ~0ULL but fm_start_fm_length-1 exceeds file size */
+	if (fm_end > fmkey->lfik_oa.o_size)
+		fm_end = fmkey->lfik_oa.o_size;
+
+	last_stripe = fiemap_calc_last_stripe(lsm, fm_start, fm_end,
+					      start_stripe, &stripe_count);
+	fm_end_offset = fiemap_calc_fm_end_offset(fiemap, lsm, fm_start, fm_end,
+						  &start_stripe);
+	if (fm_end_offset == -EINVAL) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	/**
+	 * Requested extent count exceeds the fiemap buffer size, shrink our
+	 * ambition.
+	 */
+	if (fiemap_count_to_size(fiemap->fm_extent_count) > *buflen)
+		fiemap->fm_extent_count = fiemap_size_to_count(*buflen);
+	if (!fiemap->fm_extent_count)
+		count_local = 0;
+
+	/* Check each stripe */
+	for (cur_stripe = start_stripe; stripe_count > 0;
+	     --stripe_count,
+	     cur_stripe = (cur_stripe + 1) % lsm->lsm_stripe_count) {
+		loff_t req_fm_len; /* Stores length of required mapping */
+		loff_t len_mapped_single_call;
+		loff_t lun_start;
+		loff_t lun_end;
+		loff_t obd_object_end;
+		unsigned int ext_count;
+
+		cur_stripe_wrap = cur_stripe;
+
+		/* Find out range of mapping on this stripe */
+		if (!(lov_stripe_intersects(lsm, cur_stripe, fm_start, fm_end,
+					    &lun_start, &obd_object_end)))
+			continue;
+
+		if (lov_oinfo_is_dummy(lsm->lsm_oinfo[cur_stripe])) {
+			rc = -EIO;
+			goto out;
+		}
+
+		/*
+		 * If this is a continuation FIEMAP call and we are on
+		 * starting stripe then lun_start needs to be set to
+		 * fm_end_offset
+		 */
+		if (fm_end_offset && cur_stripe == start_stripe)
+			lun_start = fm_end_offset;
+
+		if (fm_length != ~0ULL) {
+			/* Handle fm_start + fm_length overflow */
+			if (fm_start + fm_length < fm_start)
+				fm_length = ~0ULL - fm_start;
+			lun_end = lov_size_to_stripe(lsm, fm_start + fm_length,
+						     cur_stripe);
+		} else {
+			lun_end = ~0ULL;
+		}
+
+		if (lun_start == lun_end)
+			continue;
+
+		req_fm_len = obd_object_end - lun_start;
+		fm_local->fm_length = 0;
+		len_mapped_single_call = 0;
+
+		/* find lobsub object */
+		subobj = lov_find_subobj(env, cl2lov(obj), lsm,
+					 cur_stripe);
+		if (IS_ERR(subobj)) {
+			rc = PTR_ERR(subobj);
+			goto out;
+		}
+		/*
+		 * If the output buffer is very large and the objects have many
+		 * extents we may need to loop on a single OST repeatedly
+		 */
+		ost_eof = false;
+		ost_done = false;
+		do {
+			if (fiemap->fm_extent_count > 0) {
+				/* Don't get too many extents. */
+				if (current_extent + count_local >
+				    fiemap->fm_extent_count)
+					count_local = fiemap->fm_extent_count -
+						      current_extent;
+			}
+
+			lun_start += len_mapped_single_call;
+			fm_local->fm_length = req_fm_len -
+					      len_mapped_single_call;
+			req_fm_len = fm_local->fm_length;
+			fm_local->fm_extent_count = enough ? 1 : count_local;
+			fm_local->fm_mapped_extents = 0;
+			fm_local->fm_flags = fiemap->fm_flags;
+
+			ost_index = lsm->lsm_oinfo[cur_stripe]->loi_ost_idx;
+
+			if (ost_index < 0 ||
+			    ost_index >= lov->desc.ld_tgt_count) {
+				rc = -EINVAL;
+				goto obj_put;
+			}
+			/*
+			 * If OST is inactive, return extent with UNKNOWN
+			 * flag.
+			 */
+			if (!lov->lov_tgts[ost_index]->ltd_active) {
+				fm_local->fm_flags |= FIEMAP_EXTENT_LAST;
+				fm_local->fm_mapped_extents = 1;
+
+				lcl_fm_ext[0].fe_logical = lun_start;
+				lcl_fm_ext[0].fe_length = obd_object_end -
+							  lun_start;
+				lcl_fm_ext[0].fe_flags |= FIEMAP_EXTENT_UNKNOWN;
+
+				goto inactive_tgt;
+			}
+
+			fm_local->fm_start = lun_start;
+			fm_local->fm_flags &= ~FIEMAP_FLAG_DEVICE_ORDER;
+			memcpy(&fmkey->lfik_fiemap, fm_local, sizeof(*fm_local));
+			*buflen = fiemap_count_to_size(fm_local->fm_extent_count);
+
+			rc = cl_object_fiemap(env, subobj, fmkey, fm_local,
+					      buflen);
+			if (rc)
+				goto obj_put;
+inactive_tgt:
+			ext_count = fm_local->fm_mapped_extents;
+			if (!ext_count) {
+				ost_done = true;
+				/*
+				 * If last stripe has hold at the end,
+				 * we need to return
+				 */
+				if (cur_stripe_wrap == last_stripe) {
+					fiemap->fm_mapped_extents = 0;
+					goto finish;
+				}
+				break;
+			} else if (enough) {
+				/*
+				 * We've collected enough extents and there are
+				 * more extents after it.
+				 */
+				goto finish;
+			}
+
+			/* If we just need num of extents, got to next device */
+			if (!fiemap->fm_extent_count) {
+				current_extent += ext_count;
+				break;
+			}
+
+			/* prepare to copy retrived map extents */
+			len_mapped_single_call =
+				lcl_fm_ext[ext_count - 1].fe_logical -
+				lun_start + lcl_fm_ext[ext_count - 1].fe_length;
+
+			/* Have we finished mapping on this device? */
+			if (req_fm_len <= len_mapped_single_call)
+				ost_done = true;
+
+			/*
+			 * Clear the EXTENT_LAST flag which can be present on
+			 * the last extent
+			 */
+			if (lcl_fm_ext[ext_count - 1].fe_flags &
+			    FIEMAP_EXTENT_LAST)
+				lcl_fm_ext[ext_count - 1].fe_flags &=
+					~FIEMAP_EXTENT_LAST;
+
+			if (lov_stripe_size(lsm,
+					    lcl_fm_ext[ext_count - 1].fe_logical +
+					    lcl_fm_ext[ext_count - 1].fe_length,
+					    cur_stripe) >= fmkey->lfik_oa.o_size)
+				ost_eof = true;
+
+			fiemap_prepare_and_copy_exts(fiemap, lcl_fm_ext,
+						     ost_index, ext_count,
+						     current_extent);
+			current_extent += ext_count;
+
+			/* Ran out of available extents? */
+			if (current_extent >= fiemap->fm_extent_count)
+				enough = true;
+		} while (!ost_done && !ost_eof);
+
+		cl_object_put(env, subobj);
+		subobj = NULL;
+
+		if (cur_stripe_wrap == last_stripe)
+			goto finish;
+	} /* for each stripe */
+finish:
+	/*
+	 * Indicate that we are returning device offsets unless file just has
+	 * single stripe
+	 */
+	if (lsm->lsm_stripe_count > 1)
+		fiemap->fm_flags |= FIEMAP_FLAG_DEVICE_ORDER;
+
+	if (!fiemap->fm_extent_count)
+		goto skip_last_device_calc;
+
+	/*
+	 * Check if we have reached the last stripe and whether mapping for that
+	 * stripe is done.
+	 */
+	if ((cur_stripe_wrap == last_stripe) && (ost_done || ost_eof))
+		fiemap->fm_extents[current_extent - 1].fe_flags |=
+							FIEMAP_EXTENT_LAST;
+skip_last_device_calc:
+	fiemap->fm_mapped_extents = current_extent;
+obj_put:
+	if (subobj)
+		cl_object_put(env, subobj);
+out:
+	kvfree(fm_local);
+	lov_lsm_put(lsm);
+	return rc;
+}
+
 static int lov_object_getstripe(const struct lu_env *env, struct cl_object *obj,
 				struct lov_user_md __user *lum)
 {
@@ -923,10 +1437,53 @@ static int lov_object_getstripe(const struct lu_env *env, struct cl_object *obj,
 		return -ENODATA;
 
 	rc = lov_getstripe(cl2lov(obj), lsm, lum);
-	lov_lsm_put(obj, lsm);
+	lov_lsm_put(lsm);
 	return rc;
 }
 
+static int lov_object_layout_get(const struct lu_env *env,
+				 struct cl_object *obj,
+				 struct cl_layout *cl)
+{
+	struct lov_object *lov = cl2lov(obj);
+	struct lov_stripe_md *lsm = lov_lsm_addref(lov);
+	struct lu_buf *buf = &cl->cl_buf;
+	ssize_t rc;
+
+	if (!lsm) {
+		cl->cl_size = 0;
+		cl->cl_layout_gen = CL_LAYOUT_GEN_EMPTY;
+		cl->cl_is_released = false;
+
+		return 0;
+	}
+
+	cl->cl_size = lov_mds_md_size(lsm->lsm_stripe_count, lsm->lsm_magic);
+	cl->cl_layout_gen = lsm->lsm_layout_gen;
+	cl->cl_is_released = lsm_is_released(lsm);
+
+	rc = lov_lsm_pack(lsm, buf->lb_buf, buf->lb_len);
+	lov_lsm_put(lsm);
+
+	return rc < 0 ? rc : 0;
+}
+
+static loff_t lov_object_maxbytes(struct cl_object *obj)
+{
+	struct lov_object *lov = cl2lov(obj);
+	struct lov_stripe_md *lsm = lov_lsm_addref(lov);
+	loff_t maxbytes;
+
+	if (!lsm)
+		return LLONG_MAX;
+
+	maxbytes = lsm->lsm_maxbytes;
+
+	lov_lsm_put(lsm);
+
+	return maxbytes;
+}
+
 static const struct cl_object_operations lov_ops = {
 	.coo_page_init = lov_page_init,
 	.coo_lock_init = lov_lock_init,
@@ -934,7 +1491,10 @@ static const struct cl_object_operations lov_ops = {
 	.coo_attr_get  = lov_attr_get,
 	.coo_attr_update = lov_attr_update,
 	.coo_conf_set  = lov_conf_set,
-	.coo_getstripe = lov_object_getstripe
+	.coo_getstripe = lov_object_getstripe,
+	.coo_layout_get	 = lov_object_layout_get,
+	.coo_maxbytes	 = lov_object_maxbytes,
+	.coo_fiemap	 = lov_object_fiemap,
 };
 
 static const struct lu_object_operations lov_lu_obj_ops = {
@@ -986,22 +1546,6 @@ struct lov_stripe_md *lov_lsm_addref(struct lov_object *lov)
 	return lsm;
 }
 
-struct lov_stripe_md *lov_lsm_get(struct cl_object *clobj)
-{
-	struct lu_object *luobj;
-	struct lov_stripe_md *lsm = NULL;
-
-	if (!clobj)
-		return NULL;
-
-	luobj = lu_object_locate(&cl_object_header(clobj)->coh_lu,
-				 &lov_device_type);
-	if (luobj)
-		lsm = lov_lsm_addref(lu2lov(luobj));
-	return lsm;
-}
-EXPORT_SYMBOL(lov_lsm_get);
-
 int lov_read_and_clear_async_rc(struct cl_object *clob)
 {
 	struct lu_object *luobj;
diff --git a/drivers/staging/lustre/lustre/lov/lov_pack.c b/drivers/staging/lustre/lustre/lov/lov_pack.c
index be6e9857ce2a..6c93d180aef7 100644
--- a/drivers/staging/lustre/lustre/lov/lov_pack.c
+++ b/drivers/staging/lustre/lustre/lov/lov_pack.c
@@ -38,14 +38,17 @@
 
 #define DEBUG_SUBSYSTEM S_LOV
 
+#include "../include/lustre/lustre_idl.h"
+#include "../include/lustre/lustre_user.h"
+
 #include "../include/lustre_net.h"
+#include "../include/lustre_swab.h"
 #include "../include/obd.h"
 #include "../include/obd_class.h"
 #include "../include/obd_support.h"
-#include "../include/lustre/lustre_user.h"
 
-#include "lov_internal.h"
 #include "lov_cl_internal.h"
+#include "lov_internal.h"
 
 void lov_dump_lmm_common(int level, void *lmmp)
 {
@@ -97,120 +100,54 @@ void lov_dump_lmm_v3(int level, struct lov_mds_md_v3 *lmm)
 			     le16_to_cpu(lmm->lmm_stripe_count));
 }
 
-/* Pack LOV object metadata for disk storage.  It is packed in LE byte
- * order and is opaque to the networking layer.
+/**
+ * Pack LOV striping metadata for disk storage format (in little
+ * endian byte order).
  *
- * XXX In the future, this will be enhanced to get the EA size from the
- *     underlying OSC device(s) to get their EA sizes so we can stack
- *     LOVs properly.  For now lov_mds_md_size() just assumes one u64
- *     per stripe.
+ * This follows the getxattr() conventions. If \a buf_size is zero
+ * then return the size needed. If \a buf_size is too small then
+ * return -ERANGE. Otherwise return the size of the result.
  */
-int lov_obd_packmd(struct lov_obd *lov, struct lov_mds_md **lmmp,
-		   struct lov_stripe_md *lsm)
+ssize_t lov_lsm_pack(const struct lov_stripe_md *lsm, void *buf,
+		     size_t buf_size)
 {
-	struct lov_mds_md_v1 *lmmv1;
-	struct lov_mds_md_v3 *lmmv3;
-	__u16 stripe_count;
 	struct lov_ost_data_v1 *lmm_objects;
-	int lmm_size, lmm_magic;
-	int i;
-	int cplen = 0;
-
-	if (lsm) {
-		lmm_magic = lsm->lsm_magic;
-	} else {
-		if (lmmp && *lmmp)
-			lmm_magic = le32_to_cpu((*lmmp)->lmm_magic);
-		else
-			/* lsm == NULL and lmmp == NULL */
-			lmm_magic = LOV_MAGIC;
-	}
-
-	if ((lmm_magic != LOV_MAGIC_V1) &&
-	    (lmm_magic != LOV_MAGIC_V3)) {
-		CERROR("bad mem LOV MAGIC: 0x%08X != 0x%08X nor 0x%08X\n",
-		       lmm_magic, LOV_MAGIC_V1, LOV_MAGIC_V3);
-		return -EINVAL;
-	}
-
-	if (lsm) {
-		/* If we are just sizing the EA, limit the stripe count
-		 * to the actual number of OSTs in this filesystem.
-		 */
-		if (!lmmp) {
-			stripe_count = lov_get_stripecnt(lov, lmm_magic,
-							 lsm->lsm_stripe_count);
-			lsm->lsm_stripe_count = stripe_count;
-		} else if (!lsm_is_released(lsm)) {
-			stripe_count = lsm->lsm_stripe_count;
-		} else {
-			stripe_count = 0;
-		}
-	} else {
-		/*
-		 * To calculate maximum easize by active targets at present,
-		 * which is exactly the maximum easize to be seen by LOV
-		 */
-		stripe_count = lov->desc.ld_active_tgt_count;
-	}
+	struct lov_mds_md_v1 *lmmv1 = buf;
+	struct lov_mds_md_v3 *lmmv3 = buf;
+	size_t lmm_size;
+	unsigned int i;
 
-	/* XXX LOV STACKING call into osc for sizes */
-	lmm_size = lov_mds_md_size(stripe_count, lmm_magic);
-
-	if (!lmmp)
+	lmm_size = lov_mds_md_size(lsm->lsm_stripe_count, lsm->lsm_magic);
+	if (!buf_size)
 		return lmm_size;
 
-	if (*lmmp && !lsm) {
-		stripe_count = le16_to_cpu((*lmmp)->lmm_stripe_count);
-		lmm_size = lov_mds_md_size(stripe_count, lmm_magic);
-		kvfree(*lmmp);
-		*lmmp = NULL;
-		return 0;
-	}
-
-	if (!*lmmp) {
-		*lmmp = libcfs_kvzalloc(lmm_size, GFP_NOFS);
-		if (!*lmmp)
-			return -ENOMEM;
-	}
-
-	CDEBUG(D_INFO, "lov_packmd: LOV_MAGIC 0x%08X, lmm_size = %d\n",
-	       lmm_magic, lmm_size);
-
-	lmmv1 = *lmmp;
-	lmmv3 = (struct lov_mds_md_v3 *)*lmmp;
-	if (lmm_magic == LOV_MAGIC_V3)
-		lmmv3->lmm_magic = cpu_to_le32(LOV_MAGIC_V3);
-	else
-		lmmv1->lmm_magic = cpu_to_le32(LOV_MAGIC_V1);
-
-	if (!lsm)
-		return lmm_size;
+	if (buf_size < lmm_size)
+		return -ERANGE;
 
-	/* lmmv1 and lmmv3 point to the same struct and have the
+	/*
+	 * lmmv1 and lmmv3 point to the same struct and have the
 	 * same first fields
 	 */
+	lmmv1->lmm_magic = cpu_to_le32(lsm->lsm_magic);
 	lmm_oi_cpu_to_le(&lmmv1->lmm_oi, &lsm->lsm_oi);
 	lmmv1->lmm_stripe_size = cpu_to_le32(lsm->lsm_stripe_size);
-	lmmv1->lmm_stripe_count = cpu_to_le16(stripe_count);
+	lmmv1->lmm_stripe_count = cpu_to_le16(lsm->lsm_stripe_count);
 	lmmv1->lmm_pattern = cpu_to_le32(lsm->lsm_pattern);
 	lmmv1->lmm_layout_gen = cpu_to_le16(lsm->lsm_layout_gen);
+
 	if (lsm->lsm_magic == LOV_MAGIC_V3) {
-		cplen = strlcpy(lmmv3->lmm_pool_name, lsm->lsm_pool_name,
-				sizeof(lmmv3->lmm_pool_name));
-		if (cplen >= sizeof(lmmv3->lmm_pool_name))
-			return -E2BIG;
+		CLASSERT(sizeof(lsm->lsm_pool_name) ==
+			 sizeof(lmmv3->lmm_pool_name));
+		strlcpy(lmmv3->lmm_pool_name, lsm->lsm_pool_name,
+			sizeof(lmmv3->lmm_pool_name));
 		lmm_objects = lmmv3->lmm_objects;
 	} else {
 		lmm_objects = lmmv1->lmm_objects;
 	}
 
-	for (i = 0; i < stripe_count; i++) {
+	for (i = 0; i < lsm->lsm_stripe_count; i++) {
 		struct lov_oinfo *loi = lsm->lsm_oinfo[i];
-		/* XXX LOV STACKING call down to osc_packmd() to do packing */
-		LASSERTF(ostid_id(&loi->loi_oi) != 0, "lmm_oi "DOSTID
-			 " stripe %u/%u idx %u\n", POSTID(&lmmv1->lmm_oi),
-			 i, stripe_count, loi->loi_ost_idx);
+
 		ostid_cpu_to_le(&loi->loi_oi, &lmm_objects[i].l_ost_oi);
 		lmm_objects[i].l_ost_gen = cpu_to_le32(loi->loi_ost_gen);
 		lmm_objects[i].l_ost_idx = cpu_to_le32(loi->loi_ost_idx);
@@ -219,15 +156,6 @@ int lov_obd_packmd(struct lov_obd *lov, struct lov_mds_md **lmmp,
 	return lmm_size;
 }
 
-int lov_packmd(struct obd_export *exp, struct lov_mds_md **lmmp,
-	       struct lov_stripe_md *lsm)
-{
-	struct obd_device *obd = class_exp2obd(exp);
-	struct lov_obd *lov = &obd->u.lov;
-
-	return lov_obd_packmd(lov, lmmp, lsm);
-}
-
 /* Find the max stripecount we should use */
 __u16 lov_get_stripecnt(struct lov_obd *lov, __u32 magic, __u16 stripe_count)
 {
@@ -270,34 +198,34 @@ static int lov_verify_lmm(void *lmm, int lmm_bytes, __u16 *stripe_count)
 	return rc;
 }
 
-int lov_alloc_memmd(struct lov_stripe_md **lsmp, __u16 stripe_count,
-		    int pattern, int magic)
+struct lov_stripe_md *lov_lsm_alloc(u16 stripe_count, u32 pattern, u32 magic)
 {
-	int i, lsm_size;
+	struct lov_stripe_md *lsm;
+	unsigned int i;
 
-	CDEBUG(D_INFO, "alloc lsm, stripe_count %d\n", stripe_count);
+	CDEBUG(D_INFO, "alloc lsm, stripe_count %u\n", stripe_count);
 
-	*lsmp = lsm_alloc_plain(stripe_count, &lsm_size);
-	if (!*lsmp) {
-		CERROR("can't allocate lsmp stripe_count %d\n", stripe_count);
-		return -ENOMEM;
+	lsm = lsm_alloc_plain(stripe_count);
+	if (!lsm) {
+		CERROR("cannot allocate LSM stripe_count %u\n", stripe_count);
+		return ERR_PTR(-ENOMEM);
 	}
 
-	atomic_set(&(*lsmp)->lsm_refc, 1);
-	spin_lock_init(&(*lsmp)->lsm_lock);
-	(*lsmp)->lsm_magic = magic;
-	(*lsmp)->lsm_stripe_count = stripe_count;
-	(*lsmp)->lsm_maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES * stripe_count;
-	(*lsmp)->lsm_pattern = pattern;
-	(*lsmp)->lsm_pool_name[0] = '\0';
-	(*lsmp)->lsm_layout_gen = 0;
+	atomic_set(&lsm->lsm_refc, 1);
+	spin_lock_init(&lsm->lsm_lock);
+	lsm->lsm_magic = magic;
+	lsm->lsm_stripe_count = stripe_count;
+	lsm->lsm_maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES * stripe_count;
+	lsm->lsm_pattern = pattern;
+	lsm->lsm_pool_name[0] = '\0';
+	lsm->lsm_layout_gen = 0;
 	if (stripe_count > 0)
-		(*lsmp)->lsm_oinfo[0]->loi_ost_idx = ~0;
+		lsm->lsm_oinfo[0]->loi_ost_idx = ~0;
 
 	for (i = 0; i < stripe_count; i++)
-		loi_init((*lsmp)->lsm_oinfo[i]);
+		loi_init(lsm->lsm_oinfo[i]);
 
-	return lsm_size;
+	return lsm;
 }
 
 int lov_free_memmd(struct lov_stripe_md **lsmp)
@@ -317,56 +245,34 @@ int lov_free_memmd(struct lov_stripe_md **lsmp)
 /* Unpack LOV object metadata from disk storage.  It is packed in LE byte
  * order and is opaque to the networking layer.
  */
-int lov_unpackmd(struct obd_export *exp,  struct lov_stripe_md **lsmp,
-		 struct lov_mds_md *lmm, int lmm_bytes)
+struct lov_stripe_md *lov_unpackmd(struct lov_obd *lov, struct lov_mds_md *lmm,
+				   size_t lmm_size)
 {
-	struct obd_device *obd = class_exp2obd(exp);
-	struct lov_obd *lov = &obd->u.lov;
-	int rc = 0, lsm_size;
-	__u16 stripe_count;
-	__u32 magic;
-	__u32 pattern;
-
-	/* If passed an MDS struct use values from there, otherwise defaults */
-	if (lmm) {
-		rc = lov_verify_lmm(lmm, lmm_bytes, &stripe_count);
-		if (rc)
-			return rc;
-		magic = le32_to_cpu(lmm->lmm_magic);
-		pattern = le32_to_cpu(lmm->lmm_pattern);
-	} else {
-		magic = LOV_MAGIC;
-		stripe_count = lov_get_stripecnt(lov, magic, 0);
-		pattern = LOV_PATTERN_RAID0;
-	}
+	struct lov_stripe_md *lsm;
+	u16 stripe_count;
+	u32 pattern;
+	u32 magic;
+	int rc;
 
-	/* If we aren't passed an lsmp struct, we just want the size */
-	if (!lsmp) {
-		/* XXX LOV STACKING call into osc for sizes */
-		LBUG();
-		return lov_stripe_md_size(stripe_count);
-	}
-	/* If we are passed an allocated struct but nothing to unpack, free */
-	if (*lsmp && !lmm) {
-		lov_free_memmd(lsmp);
-		return 0;
-	}
+	rc = lov_verify_lmm(lmm, lmm_size, &stripe_count);
+	if (rc)
+		return ERR_PTR(rc);
 
-	lsm_size = lov_alloc_memmd(lsmp, stripe_count, pattern, magic);
-	if (lsm_size < 0)
-		return lsm_size;
+	magic = le32_to_cpu(lmm->lmm_magic);
+	pattern = le32_to_cpu(lmm->lmm_pattern);
 
-	/* If we are passed a pointer but nothing to unpack, we only alloc */
-	if (!lmm)
-		return lsm_size;
+	lsm = lov_lsm_alloc(stripe_count, pattern, magic);
+	if (IS_ERR(lsm))
+		return lsm;
 
-	rc = lsm_op_find(magic)->lsm_unpackmd(lov, *lsmp, lmm);
+	LASSERT(lsm_op_find(magic));
+	rc = lsm_op_find(magic)->lsm_unpackmd(lov, lsm, lmm);
 	if (rc) {
-		lov_free_memmd(lsmp);
-		return rc;
+		lov_free_memmd(&lsm);
+		return ERR_PTR(rc);
 	}
 
-	return lsm_size;
+	return lsm;
 }
 
 /* Retrieve object striping information.
@@ -378,15 +284,14 @@ int lov_unpackmd(struct obd_export *exp,  struct lov_stripe_md **lsmp,
 int lov_getstripe(struct lov_object *obj, struct lov_stripe_md *lsm,
 		  struct lov_user_md __user *lump)
 {
-	/*
-	 * XXX huge struct allocated on stack.
-	 */
 	/* we use lov_user_md_v3 because it is larger than lov_user_md_v1 */
-	struct lov_obd *lov;
 	struct lov_user_md_v3 lum;
-	struct lov_mds_md *lmmk = NULL;
-	int rc, lmmk_size, lmm_size;
-	int lum_size;
+	struct lov_mds_md *lmmk;
+	u32 stripe_count;
+	ssize_t lmm_size;
+	size_t lmmk_size;
+	size_t lum_size;
+	int rc;
 	mm_segment_t seg;
 
 	if (!lsm)
@@ -399,6 +304,18 @@ int lov_getstripe(struct lov_object *obj, struct lov_stripe_md *lsm,
 	seg = get_fs();
 	set_fs(KERNEL_DS);
 
+	if (lsm->lsm_magic != LOV_MAGIC_V1 && lsm->lsm_magic != LOV_MAGIC_V3) {
+		CERROR("bad LSM MAGIC: 0x%08X != 0x%08X nor 0x%08X\n",
+		       lsm->lsm_magic, LOV_MAGIC_V1, LOV_MAGIC_V3);
+		rc = -EIO;
+		goto out;
+	}
+
+	if (!lsm_is_released(lsm))
+		stripe_count = lsm->lsm_stripe_count;
+	else
+		stripe_count = 0;
+
 	/* we only need the header part from user space to get lmm_magic and
 	 * lmm_stripe_count, (the header part is common to v1 and v3)
 	 */
@@ -417,32 +334,40 @@ int lov_getstripe(struct lov_object *obj, struct lov_stripe_md *lsm,
 	if (lum.lmm_stripe_count &&
 	    (lum.lmm_stripe_count < lsm->lsm_stripe_count)) {
 		/* Return right size of stripe to user */
-		lum.lmm_stripe_count = lsm->lsm_stripe_count;
+		lum.lmm_stripe_count = stripe_count;
 		rc = copy_to_user(lump, &lum, lum_size);
 		rc = -EOVERFLOW;
 		goto out;
 	}
-	lov = lu2lov_dev(obj->lo_cl.co_lu.lo_dev)->ld_lov;
-	rc = lov_obd_packmd(lov, &lmmk, lsm);
-	if (rc < 0)
+	lmmk_size = lov_mds_md_size(stripe_count, lsm->lsm_magic);
+
+
+	lmmk = libcfs_kvzalloc(lmmk_size, GFP_NOFS);
+	if (!lmmk) {
+		rc = -ENOMEM;
 		goto out;
-	lmmk_size = rc;
-	lmm_size = rc;
-	rc = 0;
+	}
+
+	lmm_size = lov_lsm_pack(lsm, lmmk, lmmk_size);
+	if (lmm_size < 0) {
+		rc = lmm_size;
+		goto out_free;
+	}
 
 	/* FIXME: Bug 1185 - copy fields properly when structs change */
 	/* struct lov_user_md_v3 and struct lov_mds_md_v3 must be the same */
 	CLASSERT(sizeof(lum) == sizeof(struct lov_mds_md_v3));
 	CLASSERT(sizeof(lum.lmm_objects[0]) == sizeof(lmmk->lmm_objects[0]));
 
-	if ((cpu_to_le32(LOV_MAGIC) != LOV_MAGIC) &&
-	    ((lmmk->lmm_magic == cpu_to_le32(LOV_MAGIC_V1)) ||
-	    (lmmk->lmm_magic == cpu_to_le32(LOV_MAGIC_V3)))) {
+	if (cpu_to_le32(LOV_MAGIC) != LOV_MAGIC &&
+	    (lmmk->lmm_magic == cpu_to_le32(LOV_MAGIC_V1) ||
+	     lmmk->lmm_magic == cpu_to_le32(LOV_MAGIC_V3))) {
 		lustre_swab_lov_mds_md(lmmk);
 		lustre_swab_lov_user_md_objects(
 				(struct lov_user_ost_data *)lmmk->lmm_objects,
 				lmmk->lmm_stripe_count);
 	}
+
 	if (lum.lmm_magic == LOV_USER_MAGIC) {
 		/* User request for v1, we need skip lmm_pool_name */
 		if (lmmk->lmm_magic == LOV_MAGIC_V3) {
@@ -474,9 +399,11 @@ int lov_getstripe(struct lov_object *obj, struct lov_stripe_md *lsm,
 	((struct lov_user_md *)lmmk)->lmm_stripe_count = lum.lmm_stripe_count;
 	if (copy_to_user(lump, lmmk, lmm_size))
 		rc = -EFAULT;
+	else
+		rc = 0;
 
 out_free:
-	kfree(lmmk);
+	kvfree(lmmk);
 out:
 	set_fs(seg);
 	return rc;
diff --git a/drivers/staging/lustre/lustre/lov/lov_page.c b/drivers/staging/lustre/lustre/lov/lov_page.c
index 00bfabad78eb..62ceb6dfdfdf 100644
--- a/drivers/staging/lustre/lustre/lov/lov_page.c
+++ b/drivers/staging/lustre/lustre/lov/lov_page.c
@@ -49,51 +49,6 @@
  *
  */
 
-/**
- * Adjust the stripe index by layout of raid0. @max_index is the maximum
- * page index covered by an underlying DLM lock.
- * This function converts max_index from stripe level to file level, and make
- * sure it's not beyond one stripe.
- */
-static int lov_raid0_page_is_under_lock(const struct lu_env *env,
-					const struct cl_page_slice *slice,
-					struct cl_io *unused,
-					pgoff_t *max_index)
-{
-	struct lov_object *loo = cl2lov(slice->cpl_obj);
-	struct lov_layout_raid0 *r0 = lov_r0(loo);
-	pgoff_t index = *max_index;
-	unsigned int pps; /* pages per stripe */
-
-	CDEBUG(D_READA, DFID "*max_index = %lu, nr = %d\n",
-	       PFID(lu_object_fid(lov2lu(loo))), index, r0->lo_nr);
-
-	if (index == 0) /* the page is not covered by any lock */
-		return 0;
-
-	if (r0->lo_nr == 1) /* single stripe file */
-		return 0;
-
-	/* max_index is stripe level, convert it into file level */
-	if (index != CL_PAGE_EOF) {
-		int stripeno = lov_page_stripe(slice->cpl_page);
-		*max_index = lov_stripe_pgoff(loo->lo_lsm, index, stripeno);
-	}
-
-	/* calculate the end of current stripe */
-	pps = loo->lo_lsm->lsm_stripe_size >> PAGE_SHIFT;
-	index = slice->cpl_index + pps - slice->cpl_index % pps - 1;
-
-	CDEBUG(D_READA, DFID "*max_index = %lu, index = %lu, pps = %u, stripe_size = %u, stripe no = %u, page index = %lu\n",
-	       PFID(lu_object_fid(lov2lu(loo))), *max_index, index, pps,
-	       loo->lo_lsm->lsm_stripe_size, lov_page_stripe(slice->cpl_page),
-	       slice->cpl_index);
-
-	/* never exceed the end of the stripe */
-	*max_index = min_t(pgoff_t, *max_index, index);
-	return 0;
-}
-
 static int lov_raid0_page_print(const struct lu_env *env,
 				const struct cl_page_slice *slice,
 				void *cookie, lu_printer_t printer)
@@ -104,7 +59,6 @@ static int lov_raid0_page_print(const struct lu_env *env,
 }
 
 static const struct cl_page_operations lov_raid0_page_ops = {
-	.cpo_is_under_lock = lov_raid0_page_is_under_lock,
 	.cpo_print  = lov_raid0_page_print
 };
 
diff --git a/drivers/staging/lustre/lustre/lov/lov_pool.c b/drivers/staging/lustre/lustre/lov/lov_pool.c
index f8c8a361ef79..7daa8671fdc3 100644
--- a/drivers/staging/lustre/lustre/lov/lov_pool.c
+++ b/drivers/staging/lustre/lustre/lov/lov_pool.c
@@ -81,7 +81,8 @@ static void lov_pool_putref_locked(struct pool_desc *pool)
  * Chapter 6.4.
  * Addison Wesley, 1973
  */
-static __u32 pool_hashfn(struct cfs_hash *hash_body, const void *key, unsigned mask)
+static __u32 pool_hashfn(struct cfs_hash *hash_body, const void *key,
+			 unsigned int mask)
 {
 	int i;
 	__u32 result;
diff --git a/drivers/staging/lustre/lustre/lov/lov_request.c b/drivers/staging/lustre/lustre/lov/lov_request.c
index 09dcaf484c89..d43cc88ae641 100644
--- a/drivers/staging/lustre/lustre/lov/lov_request.c
+++ b/drivers/staging/lustre/lustre/lov/lov_request.c
@@ -44,7 +44,6 @@ static void lov_init_set(struct lov_request_set *set)
 	atomic_set(&set->set_completes, 0);
 	atomic_set(&set->set_success, 0);
 	atomic_set(&set->set_finish_checked, 0);
-	set->set_cookies = NULL;
 	INIT_LIST_HEAD(&set->set_list);
 	atomic_set(&set->set_refcount, 1);
 	init_waitqueue_head(&set->set_waitq);
@@ -61,8 +60,6 @@ void lov_finish_set(struct lov_request_set *set)
 							 rq_link);
 		list_del_init(&req->rq_link);
 
-		if (req->rq_oi.oi_oa)
-			kmem_cache_free(obdo_cachep, req->rq_oi.oi_oa);
 		kfree(req->rq_oi.oi_osfs);
 		kfree(req);
 	}
@@ -97,22 +94,6 @@ static void lov_update_set(struct lov_request_set *set,
 	wake_up(&set->set_waitq);
 }
 
-int lov_update_common_set(struct lov_request_set *set,
-			  struct lov_request *req, int rc)
-{
-	struct lov_obd *lov = &set->set_exp->exp_obd->u.lov;
-
-	lov_update_set(set, req, rc);
-
-	/* grace error on inactive ost */
-	if (rc && !(lov->lov_tgts[req->rq_idx] &&
-		    lov->lov_tgts[req->rq_idx]->ltd_active))
-		rc = 0;
-
-	/* FIXME in raid1 regime, should return 0 */
-	return rc;
-}
-
 static void lov_set_add_req(struct lov_request *req,
 			    struct lov_request_set *set)
 {
@@ -183,279 +164,6 @@ out:
 	return rc;
 }
 
-static int common_attr_done(struct lov_request_set *set)
-{
-	struct lov_request *req;
-	struct obdo *tmp_oa;
-	int rc = 0, attrset = 0;
-
-	if (!set->set_oi->oi_oa)
-		return 0;
-
-	if (!atomic_read(&set->set_success))
-		return -EIO;
-
-	tmp_oa = kmem_cache_zalloc(obdo_cachep, GFP_NOFS);
-	if (!tmp_oa) {
-		rc = -ENOMEM;
-		goto out;
-	}
-
-	list_for_each_entry(req, &set->set_list, rq_link) {
-		if (!req->rq_complete || req->rq_rc)
-			continue;
-		if (req->rq_oi.oi_oa->o_valid == 0)   /* inactive stripe */
-			continue;
-		lov_merge_attrs(tmp_oa, req->rq_oi.oi_oa,
-				req->rq_oi.oi_oa->o_valid,
-				set->set_oi->oi_md, req->rq_stripe, &attrset);
-	}
-	if (!attrset) {
-		CERROR("No stripes had valid attrs\n");
-		rc = -EIO;
-	}
-	if ((set->set_oi->oi_oa->o_valid & OBD_MD_FLEPOCH) &&
-	    (set->set_oi->oi_md->lsm_stripe_count != attrset)) {
-		/* When we take attributes of some epoch, we require all the
-		 * ost to be active.
-		 */
-		CERROR("Not all the stripes had valid attrs\n");
-		rc = -EIO;
-		goto out;
-	}
-
-	tmp_oa->o_oi = set->set_oi->oi_oa->o_oi;
-	memcpy(set->set_oi->oi_oa, tmp_oa, sizeof(*set->set_oi->oi_oa));
-out:
-	if (tmp_oa)
-		kmem_cache_free(obdo_cachep, tmp_oa);
-	return rc;
-}
-
-int lov_fini_getattr_set(struct lov_request_set *set)
-{
-	int rc = 0;
-
-	if (!set)
-		return 0;
-	LASSERT(set->set_exp);
-	if (atomic_read(&set->set_completes))
-		rc = common_attr_done(set);
-
-	lov_put_reqset(set);
-
-	return rc;
-}
-
-/* The callback for osc_getattr_async that finalizes a request info when a
- * response is received.
- */
-static int cb_getattr_update(void *cookie, int rc)
-{
-	struct obd_info *oinfo = cookie;
-	struct lov_request *lovreq;
-
-	lovreq = container_of(oinfo, struct lov_request, rq_oi);
-	return lov_update_common_set(lovreq->rq_rqset, lovreq, rc);
-}
-
-int lov_prep_getattr_set(struct obd_export *exp, struct obd_info *oinfo,
-			 struct lov_request_set **reqset)
-{
-	struct lov_request_set *set;
-	struct lov_obd *lov = &exp->exp_obd->u.lov;
-	int rc = 0, i;
-
-	set = kzalloc(sizeof(*set), GFP_NOFS);
-	if (!set)
-		return -ENOMEM;
-	lov_init_set(set);
-
-	set->set_exp = exp;
-	set->set_oi = oinfo;
-
-	for (i = 0; i < oinfo->oi_md->lsm_stripe_count; i++) {
-		struct lov_oinfo *loi;
-		struct lov_request *req;
-
-		loi = oinfo->oi_md->lsm_oinfo[i];
-		if (lov_oinfo_is_dummy(loi))
-			continue;
-
-		if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) {
-			CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
-			if (oinfo->oi_oa->o_valid & OBD_MD_FLEPOCH) {
-				/* SOM requires all the OSTs to be active. */
-				rc = -EIO;
-				goto out_set;
-			}
-			continue;
-		}
-
-		req = kzalloc(sizeof(*req), GFP_NOFS);
-		if (!req) {
-			rc = -ENOMEM;
-			goto out_set;
-		}
-
-		req->rq_stripe = i;
-		req->rq_idx = loi->loi_ost_idx;
-
-		req->rq_oi.oi_oa = kmem_cache_zalloc(obdo_cachep, GFP_NOFS);
-		if (!req->rq_oi.oi_oa) {
-			kfree(req);
-			rc = -ENOMEM;
-			goto out_set;
-		}
-		memcpy(req->rq_oi.oi_oa, oinfo->oi_oa,
-		       sizeof(*req->rq_oi.oi_oa));
-		req->rq_oi.oi_oa->o_oi = loi->loi_oi;
-		req->rq_oi.oi_cb_up = cb_getattr_update;
-
-		lov_set_add_req(req, set);
-	}
-	if (!set->set_count) {
-		rc = -EIO;
-		goto out_set;
-	}
-	*reqset = set;
-	return rc;
-out_set:
-	lov_fini_getattr_set(set);
-	return rc;
-}
-
-int lov_fini_setattr_set(struct lov_request_set *set)
-{
-	int rc = 0;
-
-	if (!set)
-		return 0;
-	LASSERT(set->set_exp);
-	if (atomic_read(&set->set_completes)) {
-		rc = common_attr_done(set);
-		/* FIXME update qos data here */
-	}
-
-	lov_put_reqset(set);
-	return rc;
-}
-
-int lov_update_setattr_set(struct lov_request_set *set,
-			   struct lov_request *req, int rc)
-{
-	struct lov_obd *lov = &req->rq_rqset->set_exp->exp_obd->u.lov;
-	struct lov_stripe_md *lsm = req->rq_rqset->set_oi->oi_md;
-
-	lov_update_set(set, req, rc);
-
-	/* grace error on inactive ost */
-	if (rc && !(lov->lov_tgts[req->rq_idx] &&
-		    lov->lov_tgts[req->rq_idx]->ltd_active))
-		rc = 0;
-
-	if (rc == 0) {
-		if (req->rq_oi.oi_oa->o_valid & OBD_MD_FLCTIME)
-			lsm->lsm_oinfo[req->rq_stripe]->loi_lvb.lvb_ctime =
-				req->rq_oi.oi_oa->o_ctime;
-		if (req->rq_oi.oi_oa->o_valid & OBD_MD_FLMTIME)
-			lsm->lsm_oinfo[req->rq_stripe]->loi_lvb.lvb_mtime =
-				req->rq_oi.oi_oa->o_mtime;
-		if (req->rq_oi.oi_oa->o_valid & OBD_MD_FLATIME)
-			lsm->lsm_oinfo[req->rq_stripe]->loi_lvb.lvb_atime =
-				req->rq_oi.oi_oa->o_atime;
-	}
-
-	return rc;
-}
-
-/* The callback for osc_setattr_async that finalizes a request info when a
- * response is received.
- */
-static int cb_setattr_update(void *cookie, int rc)
-{
-	struct obd_info *oinfo = cookie;
-	struct lov_request *lovreq;
-
-	lovreq = container_of(oinfo, struct lov_request, rq_oi);
-	return lov_update_setattr_set(lovreq->rq_rqset, lovreq, rc);
-}
-
-int lov_prep_setattr_set(struct obd_export *exp, struct obd_info *oinfo,
-			 struct obd_trans_info *oti,
-			 struct lov_request_set **reqset)
-{
-	struct lov_request_set *set;
-	struct lov_obd *lov = &exp->exp_obd->u.lov;
-	int rc = 0, i;
-
-	set = kzalloc(sizeof(*set), GFP_NOFS);
-	if (!set)
-		return -ENOMEM;
-	lov_init_set(set);
-
-	set->set_exp = exp;
-	set->set_oi = oinfo;
-	if (oti && oinfo->oi_oa->o_valid & OBD_MD_FLCOOKIE)
-		set->set_cookies = oti->oti_logcookies;
-
-	for (i = 0; i < oinfo->oi_md->lsm_stripe_count; i++) {
-		struct lov_oinfo *loi = oinfo->oi_md->lsm_oinfo[i];
-		struct lov_request *req;
-
-		if (lov_oinfo_is_dummy(loi))
-			continue;
-
-		if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) {
-			CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx);
-			continue;
-		}
-
-		req = kzalloc(sizeof(*req), GFP_NOFS);
-		if (!req) {
-			rc = -ENOMEM;
-			goto out_set;
-		}
-		req->rq_stripe = i;
-		req->rq_idx = loi->loi_ost_idx;
-
-		req->rq_oi.oi_oa = kmem_cache_zalloc(obdo_cachep, GFP_NOFS);
-		if (!req->rq_oi.oi_oa) {
-			kfree(req);
-			rc = -ENOMEM;
-			goto out_set;
-		}
-		memcpy(req->rq_oi.oi_oa, oinfo->oi_oa,
-		       sizeof(*req->rq_oi.oi_oa));
-		req->rq_oi.oi_oa->o_oi = loi->loi_oi;
-		req->rq_oi.oi_oa->o_stripe_idx = i;
-		req->rq_oi.oi_cb_up = cb_setattr_update;
-
-		if (oinfo->oi_oa->o_valid & OBD_MD_FLSIZE) {
-			int off = lov_stripe_offset(oinfo->oi_md,
-						    oinfo->oi_oa->o_size, i,
-						    &req->rq_oi.oi_oa->o_size);
-
-			if (off < 0 && req->rq_oi.oi_oa->o_size)
-				req->rq_oi.oi_oa->o_size--;
-
-			CDEBUG(D_INODE, "stripe %d has size %llu/%llu\n",
-			       i, req->rq_oi.oi_oa->o_size,
-			       oinfo->oi_oa->o_size);
-		}
-		lov_set_add_req(req, set);
-	}
-	if (!set->set_count) {
-		rc = -EIO;
-		goto out_set;
-	}
-	*reqset = set;
-	return rc;
-out_set:
-	lov_fini_setattr_set(set);
-	return rc;
-}
-
 #define LOV_U64_MAX ((__u64)~0ULL)
 #define LOV_SUM_MAX(tot, add)					   \
 	do {							    \
diff --git a/drivers/staging/lustre/lustre/lov/lovsub_dev.c b/drivers/staging/lustre/lustre/lov/lovsub_dev.c
index b519a1940e1e..5d6536f8a4f7 100644
--- a/drivers/staging/lustre/lustre/lov/lovsub_dev.c
+++ b/drivers/staging/lustre/lustre/lov/lovsub_dev.c
@@ -44,46 +44,6 @@
 
 /*****************************************************************************
  *
- * Lovsub transfer operations.
- *
- */
-
-static void lovsub_req_completion(const struct lu_env *env,
-				  const struct cl_req_slice *slice, int ioret)
-{
-	struct lovsub_req *lsr;
-
-	lsr = cl2lovsub_req(slice);
-	kmem_cache_free(lovsub_req_kmem, lsr);
-}
-
-/**
- * Implementation of struct cl_req_operations::cro_attr_set() for lovsub
- * layer. Lov and lovsub are responsible only for struct obdo::o_stripe_idx
- * field, which is filled there.
- */
-static void lovsub_req_attr_set(const struct lu_env *env,
-				const struct cl_req_slice *slice,
-				const struct cl_object *obj,
-				struct cl_req_attr *attr, u64 flags)
-{
-	struct lovsub_object *subobj;
-
-	subobj = cl2lovsub(obj);
-	/*
-	 * There is no OBD_MD_* flag for obdo::o_stripe_idx, so set it
-	 * unconditionally. It never changes anyway.
-	 */
-	attr->cra_oa->o_stripe_idx = subobj->lso_index;
-}
-
-static const struct cl_req_operations lovsub_req_ops = {
-	.cro_attr_set   = lovsub_req_attr_set,
-	.cro_completion = lovsub_req_completion
-};
-
-/*****************************************************************************
- *
  * Lov-sub device and device type functions.
  *
  */
@@ -137,32 +97,12 @@ static struct lu_device *lovsub_device_free(const struct lu_env *env,
 	return next;
 }
 
-static int lovsub_req_init(const struct lu_env *env, struct cl_device *dev,
-			   struct cl_req *req)
-{
-	struct lovsub_req *lsr;
-	int result;
-
-	lsr = kmem_cache_zalloc(lovsub_req_kmem, GFP_NOFS);
-	if (lsr) {
-		cl_req_slice_add(req, &lsr->lsrq_cl, dev, &lovsub_req_ops);
-		result = 0;
-	} else {
-		result = -ENOMEM;
-	}
-	return result;
-}
-
 static const struct lu_device_operations lovsub_lu_ops = {
 	.ldo_object_alloc      = lovsub_object_alloc,
 	.ldo_process_config    = NULL,
 	.ldo_recovery_complete = NULL
 };
 
-static const struct cl_device_operations lovsub_cl_ops = {
-	.cdo_req_init = lovsub_req_init
-};
-
 static struct lu_device *lovsub_device_alloc(const struct lu_env *env,
 					     struct lu_device_type *t,
 					     struct lustre_cfg *cfg)
@@ -178,7 +118,6 @@ static struct lu_device *lovsub_device_alloc(const struct lu_env *env,
 		if (result == 0) {
 			d = lovsub2lu_dev(lsd);
 			d->ld_ops	 = &lovsub_lu_ops;
-			lsd->acid_cl.cd_ops = &lovsub_cl_ops;
 		} else {
 			d = ERR_PTR(result);
 		}
diff --git a/drivers/staging/lustre/lustre/lov/lovsub_object.c b/drivers/staging/lustre/lustre/lov/lovsub_object.c
index a2bac7a3b71b..011296ee16e6 100644
--- a/drivers/staging/lustre/lustre/lov/lovsub_object.c
+++ b/drivers/staging/lustre/lustre/lov/lovsub_object.c
@@ -116,11 +116,31 @@ static int lovsub_object_glimpse(const struct lu_env *env,
 	return cl_object_glimpse(env, &los->lso_super->lo_cl, lvb);
 }
 
+/**
+ * Implementation of struct cl_object_operations::coo_req_attr_set() for lovsub
+ * layer. Lov and lovsub are responsible only for struct obdo::o_stripe_idx
+ * field, which is filled there.
+ */
+static void lovsub_req_attr_set(const struct lu_env *env, struct cl_object *obj,
+				struct cl_req_attr *attr)
+{
+	struct lovsub_object *subobj = cl2lovsub(obj);
+
+	cl_req_attr_set(env, &subobj->lso_super->lo_cl, attr);
+
+	/*
+	 * There is no OBD_MD_* flag for obdo::o_stripe_idx, so set it
+	 * unconditionally. It never changes anyway.
+	 */
+	attr->cra_oa->o_stripe_idx = subobj->lso_index;
+}
+
 static const struct cl_object_operations lovsub_ops = {
 	.coo_page_init = lovsub_page_init,
 	.coo_lock_init = lovsub_lock_init,
 	.coo_attr_update = lovsub_attr_update,
-	.coo_glimpse   = lovsub_object_glimpse
+	.coo_glimpse		= lovsub_object_glimpse,
+	.coo_req_attr_set	= lovsub_req_attr_set
 };
 
 static const struct lu_object_operations lovsub_lu_obj_ops = {
diff --git a/drivers/staging/lustre/lustre/mdc/lproc_mdc.c b/drivers/staging/lustre/lustre/mdc/lproc_mdc.c
index fca9450de57c..9021c465c044 100644
--- a/drivers/staging/lustre/lustre/mdc/lproc_mdc.c
+++ b/drivers/staging/lustre/lustre/mdc/lproc_mdc.c
@@ -36,6 +36,42 @@
 #include "../include/lprocfs_status.h"
 #include "mdc_internal.h"
 
+static ssize_t active_show(struct kobject *kobj, struct attribute *attr,
+			   char *buf)
+{
+	struct obd_device *dev = container_of(kobj, struct obd_device,
+					      obd_kobj);
+
+	return sprintf(buf, "%u\n", !dev->u.cli.cl_import->imp_deactive);
+}
+
+static ssize_t active_store(struct kobject *kobj, struct attribute *attr,
+			    const char *buffer, size_t count)
+{
+	struct obd_device *dev = container_of(kobj, struct obd_device,
+					      obd_kobj);
+	unsigned long val;
+	int rc;
+
+	rc = kstrtoul(buffer, 10, &val);
+	if (rc)
+		return rc;
+
+	if (val < 0 || val > 1)
+		return -ERANGE;
+
+	/* opposite senses */
+	if (dev->u.cli.cl_import->imp_deactive == val) {
+		rc = ptlrpc_set_import_active(dev->u.cli.cl_import, val);
+		if (rc)
+			count = rc;
+	} else {
+		CDEBUG(D_CONFIG, "activate %lu: ignoring repeat request\n", val);
+	}
+	return count;
+}
+LUSTRE_RW_ATTR(active);
+
 static ssize_t max_rpcs_in_flight_show(struct kobject *kobj,
 				       struct attribute *attr,
 				       char *buf)
@@ -73,6 +109,64 @@ static ssize_t max_rpcs_in_flight_store(struct kobject *kobj,
 }
 LUSTRE_RW_ATTR(max_rpcs_in_flight);
 
+static ssize_t max_mod_rpcs_in_flight_show(struct kobject *kobj,
+					   struct attribute *attr,
+					   char *buf)
+{
+	struct obd_device *dev = container_of(kobj, struct obd_device,
+					      obd_kobj);
+	u16 max;
+	int len;
+
+	max = dev->u.cli.cl_max_mod_rpcs_in_flight;
+	len = sprintf(buf, "%hu\n", max);
+
+	return len;
+}
+
+static ssize_t max_mod_rpcs_in_flight_store(struct kobject *kobj,
+					    struct attribute *attr,
+					    const char *buffer,
+					    size_t count)
+{
+	struct obd_device *dev = container_of(kobj, struct obd_device,
+					      obd_kobj);
+	u16 val;
+	int rc;
+
+	rc = kstrtou16(buffer, 10, &val);
+	if (rc)
+		return rc;
+
+	rc = obd_set_max_mod_rpcs_in_flight(&dev->u.cli, val);
+	if (rc)
+		count = rc;
+
+	return count;
+}
+LUSTRE_RW_ATTR(max_mod_rpcs_in_flight);
+
+static int mdc_rpc_stats_seq_show(struct seq_file *seq, void *v)
+{
+	struct obd_device *dev = seq->private;
+
+	return obd_mod_rpc_stats_seq_show(&dev->u.cli, seq);
+}
+
+static ssize_t mdc_rpc_stats_seq_write(struct file *file,
+				       const char __user *buf,
+				       size_t len, loff_t *off)
+{
+	struct seq_file *seq = file->private_data;
+	struct obd_device *dev = seq->private;
+	struct client_obd *cli = &dev->u.cli;
+
+	lprocfs_oh_clear(&cli->cl_mod_rpcs_hist);
+
+	return len;
+}
+LPROC_SEQ_FOPS(mdc_rpc_stats);
+
 LPROC_SEQ_FOPS_WR_ONLY(mdc, ping);
 
 LPROC_SEQ_FOPS_RO_TYPE(mdc, connect_flags);
@@ -112,11 +206,15 @@ static struct lprocfs_vars lprocfs_mdc_obd_vars[] = {
 	{ "import",		&mdc_import_fops,		NULL, 0 },
 	{ "state",		&mdc_state_fops,		NULL, 0 },
 	{ "pinger_recov",	&mdc_pinger_recov_fops,		NULL, 0 },
+	{ .name =	"rpc_stats",
+	  .fops =	&mdc_rpc_stats_fops		},
 	{ NULL }
 };
 
 static struct attribute *mdc_attrs[] = {
+	&lustre_attr_active.attr,
 	&lustre_attr_max_rpcs_in_flight.attr,
+	&lustre_attr_max_mod_rpcs_in_flight.attr,
 	&lustre_attr_max_pages_per_rpc.attr,
 	NULL,
 };
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_internal.h b/drivers/staging/lustre/lustre/mdc/mdc_internal.h
index f446c1c2584b..881c6a0676a6 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_internal.h
+++ b/drivers/staging/lustre/lustre/mdc/mdc_internal.h
@@ -46,7 +46,7 @@ void mdc_readdir_pack(struct ptlrpc_request *req, __u64 pgoff, size_t size,
 void mdc_getattr_pack(struct ptlrpc_request *req, __u64 valid, u32 flags,
 		      struct md_op_data *data, size_t ea_size);
 void mdc_setattr_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
-		      void *ea, size_t ealen, void *ea2, size_t ea2len);
+		      void *ea, size_t ealen);
 void mdc_create_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 		     const void *data, size_t datalen, umode_t mode, uid_t uid,
 		     gid_t gid, cfs_cap_t capability, __u64 rdev);
@@ -75,7 +75,7 @@ int mdc_intent_lock(struct obd_export *exp,
 		    __u64 extra_lock_flags);
 
 int mdc_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
-		const ldlm_policy_data_t *policy,
+		const union ldlm_policy_data *policy,
 		struct lookup_intent *it, struct md_op_data *op_data,
 		struct lustre_handle *lockh, __u64 extra_lock_flags);
 
@@ -105,12 +105,11 @@ int mdc_rename(struct obd_export *exp, struct md_op_data *op_data,
 	       const char *new, size_t newlen,
 	       struct ptlrpc_request **request);
 int mdc_setattr(struct obd_export *exp, struct md_op_data *op_data,
-		void *ea, size_t ealen, void *ea2, size_t ea2len,
-		struct ptlrpc_request **request, struct md_open_data **mod);
+		void *ea, size_t ealen, struct ptlrpc_request **request);
 int mdc_unlink(struct obd_export *exp, struct md_op_data *op_data,
 	       struct ptlrpc_request **request);
 int mdc_cancel_unused(struct obd_export *exp, const struct lu_fid *fid,
-		      ldlm_policy_data_t *policy, enum ldlm_mode mode,
+		      union ldlm_policy_data *policy, enum ldlm_mode mode,
 		      enum ldlm_cancel_flags flags, void *opaque);
 
 int mdc_revalidate_lock(struct obd_export *exp, struct lookup_intent *it,
@@ -122,7 +121,8 @@ int mdc_intent_getattr_async(struct obd_export *exp,
 
 enum ldlm_mode mdc_lock_match(struct obd_export *exp, __u64 flags,
 			      const struct lu_fid *fid, enum ldlm_type type,
-			      ldlm_policy_data_t *policy, enum ldlm_mode mode,
+			      union ldlm_policy_data *policy,
+			      enum ldlm_mode mode,
 			      struct lustre_handle *lockh);
 
 static inline int mdc_prep_elc_req(struct obd_export *exp,
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_lib.c b/drivers/staging/lustre/lustre/mdc/mdc_lib.c
index aac7e04873e2..f35e1f9afdef 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_lib.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_lib.c
@@ -139,7 +139,7 @@ void mdc_create_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 	rec->cr_time     = op_data->op_mod_time;
 	rec->cr_suppgid1 = op_data->op_suppgids[0];
 	rec->cr_suppgid2 = op_data->op_suppgids[1];
-	flags = op_data->op_flags & MF_SOM_LOCAL_FLAGS;
+	flags = 0;
 	if (op_data->op_bias & MDS_CREATE_VOLATILE)
 		flags |= MDS_OPEN_VOLATILE;
 	set_mrc_cr_flags(rec, flags);
@@ -301,16 +301,16 @@ static void mdc_setattr_pack_rec(struct mdt_rec_setattr *rec,
 static void mdc_ioepoch_pack(struct mdt_ioepoch *epoch,
 			     struct md_op_data *op_data)
 {
-	memcpy(&epoch->handle, &op_data->op_handle, sizeof(epoch->handle));
-	epoch->ioepoch = op_data->op_ioepoch;
-	epoch->flags = op_data->op_flags & MF_SOM_LOCAL_FLAGS;
+	epoch->mio_handle = op_data->op_handle;
+	epoch->mio_unused1 = 0;
+	epoch->mio_unused2 = 0;
+	epoch->mio_padding = 0;
 }
 
 void mdc_setattr_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
-		      void *ea, size_t ealen, void *ea2, size_t ea2len)
+		      void *ea, size_t ealen)
 {
 	struct mdt_rec_setattr *rec;
-	struct mdt_ioepoch *epoch;
 	struct lov_user_md *lum = NULL;
 
 	CLASSERT(sizeof(struct mdt_rec_reint) ==
@@ -318,11 +318,6 @@ void mdc_setattr_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 	rec = req_capsule_client_get(&req->rq_pill, &RMF_REC_REINT);
 	mdc_setattr_pack_rec(rec, op_data);
 
-	if (op_data->op_flags & (MF_SOM_CHANGE | MF_EPOCH_OPEN)) {
-		epoch = req_capsule_client_get(&req->rq_pill, &RMF_MDT_EPOCH);
-		mdc_ioepoch_pack(epoch, op_data);
-	}
-
 	if (ealen == 0)
 		return;
 
@@ -335,12 +330,6 @@ void mdc_setattr_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 	} else {
 		memcpy(lum, ea, ealen);
 	}
-
-	if (ea2len == 0)
-		return;
-
-	memcpy(req_capsule_client_get(&req->rq_pill, &RMF_LOGCOOKIES), ea2,
-	       ea2len);
 }
 
 void mdc_unlink_pack(struct ptlrpc_request *req, struct md_op_data *op_data)
@@ -387,6 +376,31 @@ void mdc_link_pack(struct ptlrpc_request *req, struct md_op_data *op_data)
 	mdc_pack_name(req, &RMF_NAME, op_data->op_name, op_data->op_namelen);
 }
 
+static void mdc_intent_close_pack(struct ptlrpc_request *req,
+				  struct md_op_data *op_data)
+{
+	enum mds_op_bias bias = op_data->op_bias;
+	struct close_data *data;
+	struct ldlm_lock *lock;
+
+	if (!(bias & (MDS_HSM_RELEASE | MDS_CLOSE_LAYOUT_SWAP |
+		      MDS_RENAME_MIGRATE)))
+		return;
+
+	data = req_capsule_client_get(&req->rq_pill, &RMF_CLOSE_DATA);
+	LASSERT(data);
+
+	lock = ldlm_handle2lock(&op_data->op_lease_handle);
+	if (lock) {
+		data->cd_handle = lock->l_remote_handle;
+		LDLM_LOCK_PUT(lock);
+	}
+	ldlm_cli_cancel(&op_data->op_lease_handle, LCF_LOCAL);
+
+	data->cd_data_version = op_data->op_data_version;
+	data->cd_fid = op_data->op_fid2;
+}
+
 void mdc_rename_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 		     const char *old, size_t oldlen,
 		     const char *new, size_t newlen)
@@ -415,6 +429,15 @@ void mdc_rename_pack(struct ptlrpc_request *req, struct md_op_data *op_data,
 
 	if (new)
 		mdc_pack_name(req, &RMF_SYMTGT, new, newlen);
+
+	if (op_data->op_cli_flags & CLI_MIGRATE &&
+	    op_data->op_bias & MDS_RENAME_MIGRATE) {
+		struct mdt_ioepoch *epoch;
+
+		mdc_intent_close_pack(req, op_data);
+		epoch = req_capsule_client_get(&req->rq_pill, &RMF_MDT_EPOCH);
+		mdc_ioepoch_pack(epoch, op_data);
+	}
 }
 
 void mdc_getattr_pack(struct ptlrpc_request *req, __u64 valid, u32 flags,
@@ -441,27 +464,6 @@ void mdc_getattr_pack(struct ptlrpc_request *req, __u64 valid, u32 flags,
 			      op_data->op_namelen);
 }
 
-static void mdc_hsm_release_pack(struct ptlrpc_request *req,
-				 struct md_op_data *op_data)
-{
-	if (op_data->op_bias & MDS_HSM_RELEASE) {
-		struct close_data *data;
-		struct ldlm_lock *lock;
-
-		data = req_capsule_client_get(&req->rq_pill, &RMF_CLOSE_DATA);
-
-		lock = ldlm_handle2lock(&op_data->op_lease_handle);
-		if (lock) {
-			data->cd_handle = lock->l_remote_handle;
-			LDLM_LOCK_PUT(lock);
-		}
-		ldlm_cli_cancel(&op_data->op_lease_handle, LCF_LOCAL);
-
-		data->cd_data_version = op_data->op_data_version;
-		data->cd_fid = op_data->op_fid2;
-	}
-}
-
 void mdc_close_pack(struct ptlrpc_request *req, struct md_op_data *op_data)
 {
 	struct mdt_ioepoch *epoch;
@@ -484,5 +486,5 @@ void mdc_close_pack(struct ptlrpc_request *req, struct md_op_data *op_data)
 		rec->sa_valid &= ~MDS_ATTR_ATIME;
 
 	mdc_ioepoch_pack(epoch, op_data);
-	mdc_hsm_release_pack(req, op_data);
+	mdc_intent_close_pack(req, op_data);
 }
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_locks.c b/drivers/staging/lustre/lustre/mdc/mdc_locks.c
index f1f6c082fa42..54ebb9952d66 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_locks.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_locks.c
@@ -38,10 +38,12 @@
 #include "../include/obd.h"
 #include "../include/obd_class.h"
 #include "../include/lustre_dlm.h"
-#include "../include/lustre_fid.h"	/* fid_res_name_eq() */
+#include "../include/lustre_fid.h"
 #include "../include/lustre_mdc.h"
 #include "../include/lustre_net.h"
 #include "../include/lustre_req_layout.h"
+#include "../include/lustre_swab.h"
+
 #include "mdc_internal.h"
 
 struct mdc_getattr_args {
@@ -131,7 +133,8 @@ int mdc_set_lock_data(struct obd_export *exp, const struct lustre_handle *lockh,
 
 enum ldlm_mode mdc_lock_match(struct obd_export *exp, __u64 flags,
 			      const struct lu_fid *fid, enum ldlm_type type,
-			      ldlm_policy_data_t *policy, enum ldlm_mode mode,
+			      union ldlm_policy_data *policy,
+			      enum ldlm_mode mode,
 			      struct lustre_handle *lockh)
 {
 	struct ldlm_res_id res_id;
@@ -147,7 +150,7 @@ enum ldlm_mode mdc_lock_match(struct obd_export *exp, __u64 flags,
 
 int mdc_cancel_unused(struct obd_export *exp,
 		      const struct lu_fid *fid,
-		      ldlm_policy_data_t *policy,
+		      union ldlm_policy_data *policy,
 		      enum ldlm_mode mode,
 		      enum ldlm_cancel_flags flags,
 		      void *opaque)
@@ -386,8 +389,6 @@ static struct ptlrpc_request *mdc_intent_unlink_pack(struct obd_export *exp,
 
 	req_capsule_set_size(&req->rq_pill, &RMF_MDT_MD, RCL_SERVER,
 			     obddev->u.cli.cl_default_mds_easize);
-	req_capsule_set_size(&req->rq_pill, &RMF_ACL, RCL_SERVER,
-			     obddev->u.cli.cl_default_mds_cookiesize);
 	ptlrpc_request_set_replen(req);
 	return req;
 }
@@ -688,20 +689,20 @@ static int mdc_finish_enqueue(struct obd_export *exp,
  * we don't know in advance the file type.
  */
 int mdc_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
-		const ldlm_policy_data_t *policy,
+		const union ldlm_policy_data *policy,
 		struct lookup_intent *it, struct md_op_data *op_data,
 		struct lustre_handle *lockh, u64 extra_lock_flags)
 {
-	static const ldlm_policy_data_t lookup_policy = {
+	static const union ldlm_policy_data lookup_policy = {
 		.l_inodebits = { MDS_INODELOCK_LOOKUP }
 	};
-	static const ldlm_policy_data_t update_policy = {
+	static const union ldlm_policy_data update_policy = {
 		.l_inodebits = { MDS_INODELOCK_UPDATE }
 	};
-	static const ldlm_policy_data_t layout_policy = {
+	static const union ldlm_policy_data layout_policy = {
 		.l_inodebits = { MDS_INODELOCK_LAYOUT }
 	};
-	static const ldlm_policy_data_t getxattr_policy = {
+	static const union ldlm_policy_data getxattr_policy = {
 		.l_inodebits = { MDS_INODELOCK_XATTR }
 	};
 	struct obd_device *obddev = class_exp2obd(exp);
@@ -762,27 +763,22 @@ resend:
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 
-	if (req && it && it->it_op & IT_CREAT)
-		/* ask ptlrpc not to resend on EINPROGRESS since we have our own
-		 * retry logic
-		 */
-		req->rq_no_retry_einprogress = 1;
-
 	if (resends) {
 		req->rq_generation_set = 1;
 		req->rq_import_generation = generation;
 		req->rq_sent = ktime_get_real_seconds() + resends;
 	}
 
-	/* It is important to obtain rpc_lock first (if applicable), so that
-	 * threads that are serialised with rpc_lock are not polluting our
-	 * rpcs in flight counter. We do not do flock request limiting, though
+	/* It is important to obtain modify RPC slot first (if applicable), so
+	 * that threads that are waiting for a modify RPC slot are not polluting
+	 * our rpcs in flight counter.
+	 * We do not do flock request limiting, though
 	 */
 	if (it) {
-		mdc_get_rpc_lock(obddev->u.cli.cl_rpc_lock, it);
+		mdc_get_mod_rpc_slot(req, it);
 		rc = obd_get_request_slot(&obddev->u.cli);
 		if (rc != 0) {
-			mdc_put_rpc_lock(obddev->u.cli.cl_rpc_lock, it);
+			mdc_put_mod_rpc_slot(req, it);
 			mdc_clear_replay_flag(req, 0);
 			ptlrpc_req_finished(req);
 			return rc;
@@ -809,7 +805,7 @@ resend:
 	}
 
 	obd_put_request_slot(&obddev->u.cli);
-	mdc_put_rpc_lock(obddev->u.cli.cl_rpc_lock, it);
+	mdc_put_mod_rpc_slot(req, it);
 
 	if (rc < 0) {
 		CDEBUG(D_INFO, "%s: ldlm_cli_enqueue failed: rc = %d\n",
@@ -825,11 +821,12 @@ resend:
 	lockrep->lock_policy_res2 =
 		ptlrpc_status_ntoh(lockrep->lock_policy_res2);
 
-	/* Retry the create infinitely when we get -EINPROGRESS from
-	 * server. This is required by the new quota design.
+	/*
+	 * Retry infinitely when the server returns -EINPROGRESS for the
+	 * intent operation, when server returns -EINPROGRESS for acquiring
+	 * intent lock, we'll retry in after_reply().
 	 */
-	if (it->it_op & IT_CREAT &&
-	    (int)lockrep->lock_policy_res2 == -EINPROGRESS) {
+	if (it->it_op && (int)lockrep->lock_policy_res2 == -EINPROGRESS) {
 		mdc_clear_replay_flag(req, rc);
 		ptlrpc_req_finished(req);
 		resends++;
@@ -931,7 +928,7 @@ static int mdc_finish_intent_lock(struct obd_export *exp,
 	 */
 	lock = ldlm_handle2lock(lockh);
 	if (lock) {
-		ldlm_policy_data_t policy = lock->l_policy_data;
+		union ldlm_policy_data policy = lock->l_policy_data;
 
 		LDLM_DEBUG(lock, "matching against this");
 
@@ -967,7 +964,7 @@ int mdc_revalidate_lock(struct obd_export *exp, struct lookup_intent *it,
 	 */
 	struct ldlm_res_id res_id;
 	struct lustre_handle lockh;
-	ldlm_policy_data_t policy;
+	union ldlm_policy_data policy;
 	enum ldlm_mode mode;
 
 	if (it->it_lock_handle) {
@@ -1169,10 +1166,9 @@ int mdc_intent_getattr_async(struct obd_export *exp,
 	 *     for statahead currently. Consider CMD in future, such two bits
 	 *     maybe managed by different MDS, should be adjusted then.
 	 */
-	ldlm_policy_data_t       policy = {
-					.l_inodebits = { MDS_INODELOCK_LOOKUP |
-							 MDS_INODELOCK_UPDATE }
-				 };
+	union ldlm_policy_data policy = {
+		.l_inodebits = { MDS_INODELOCK_LOOKUP | MDS_INODELOCK_UPDATE }
+	};
 	int		      rc = 0;
 	__u64		    flags = LDLM_FL_HAS_INTENT;
 
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_reint.c b/drivers/staging/lustre/lustre/mdc/mdc_reint.c
index c921e471fa27..07b168490f09 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_reint.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_reint.c
@@ -40,17 +40,15 @@
 #include "../include/lustre_fid.h"
 
 /* mdc_setattr does its own semaphore handling */
-static int mdc_reint(struct ptlrpc_request *request,
-		     struct mdc_rpc_lock *rpc_lock,
-		     int level)
+static int mdc_reint(struct ptlrpc_request *request, int level)
 {
 	int rc;
 
 	request->rq_send_state = level;
 
-	mdc_get_rpc_lock(rpc_lock, NULL);
+	mdc_get_mod_rpc_slot(request, NULL);
 	rc = ptlrpc_queue_wait(request);
-	mdc_put_rpc_lock(rpc_lock, NULL);
+	mdc_put_mod_rpc_slot(request, NULL);
 	if (rc)
 		CDEBUG(D_INFO, "error in handling %d\n", rc);
 	else if (!req_capsule_server_get(&request->rq_pill, &RMF_MDT_BODY))
@@ -68,7 +66,7 @@ int mdc_resource_get_unused(struct obd_export *exp, const struct lu_fid *fid,
 			    __u64 bits)
 {
 	struct ldlm_namespace *ns = exp->exp_obd->obd_namespace;
-	ldlm_policy_data_t policy = {};
+	union ldlm_policy_data policy = {};
 	struct ldlm_res_id res_id;
 	struct ldlm_resource *res;
 	int count;
@@ -99,13 +97,10 @@ int mdc_resource_get_unused(struct obd_export *exp, const struct lu_fid *fid,
 }
 
 int mdc_setattr(struct obd_export *exp, struct md_op_data *op_data,
-		void *ea, size_t ealen, void *ea2, size_t ea2len,
-		struct ptlrpc_request **request, struct md_open_data **mod)
+		void *ea, size_t ealen, struct ptlrpc_request **request)
 {
 	LIST_HEAD(cancels);
 	struct ptlrpc_request *req;
-	struct mdc_rpc_lock *rpc_lock;
-	struct obd_device *obd = exp->exp_obd;
 	int count = 0, rc;
 	__u64 bits;
 
@@ -122,12 +117,9 @@ int mdc_setattr(struct obd_export *exp, struct md_op_data *op_data,
 		ldlm_lock_list_put(&cancels, l_bl_ast, count);
 		return -ENOMEM;
 	}
-	if ((op_data->op_flags & (MF_SOM_CHANGE | MF_EPOCH_OPEN)) == 0)
-		req_capsule_set_size(&req->rq_pill, &RMF_MDT_EPOCH, RCL_CLIENT,
-				     0);
+	req_capsule_set_size(&req->rq_pill, &RMF_MDT_EPOCH, RCL_CLIENT, 0);
 	req_capsule_set_size(&req->rq_pill, &RMF_EADATA, RCL_CLIENT, ealen);
-	req_capsule_set_size(&req->rq_pill, &RMF_LOGCOOKIES, RCL_CLIENT,
-			     ea2len);
+	req_capsule_set_size(&req->rq_pill, &RMF_LOGCOOKIES, RCL_CLIENT, 0);
 
 	rc = mdc_prep_elc_req(exp, req, MDS_REINT, &cancels, count);
 	if (rc) {
@@ -135,63 +127,21 @@ int mdc_setattr(struct obd_export *exp, struct md_op_data *op_data,
 		return rc;
 	}
 
-	rpc_lock = obd->u.cli.cl_rpc_lock;
-
 	if (op_data->op_attr.ia_valid & (ATTR_MTIME | ATTR_CTIME))
 		CDEBUG(D_INODE, "setting mtime %ld, ctime %ld\n",
 		       LTIME_S(op_data->op_attr.ia_mtime),
 		       LTIME_S(op_data->op_attr.ia_ctime));
-	mdc_setattr_pack(req, op_data, ea, ealen, ea2, ea2len);
+	mdc_setattr_pack(req, op_data, ea, ealen);
 
 	ptlrpc_request_set_replen(req);
-	if (mod && (op_data->op_flags & MF_EPOCH_OPEN) &&
-	    req->rq_import->imp_replayable) {
-		LASSERT(!*mod);
-
-		*mod = obd_mod_alloc();
-		if (!*mod) {
-			DEBUG_REQ(D_ERROR, req, "Can't allocate md_open_data");
-		} else {
-			req->rq_replay = 1;
-			req->rq_cb_data = *mod;
-			(*mod)->mod_open_req = req;
-			req->rq_commit_cb = mdc_commit_open;
-			(*mod)->mod_is_create = true;
-			/**
-			 * Take an extra reference on \var mod, it protects \var
-			 * mod from being freed on eviction (commit callback is
-			 * called despite rq_replay flag).
-			 * Will be put on mdc_done_writing().
-			 */
-			obd_mod_get(*mod);
-		}
-	}
-
-	rc = mdc_reint(req, rpc_lock, LUSTRE_IMP_FULL);
 
-	/* Save the obtained info in the original RPC for the replay case. */
-	if (rc == 0 && (op_data->op_flags & MF_EPOCH_OPEN)) {
-		struct mdt_ioepoch *epoch;
-		struct mdt_body  *body;
+	rc = mdc_reint(req, LUSTRE_IMP_FULL);
 
-		epoch = req_capsule_client_get(&req->rq_pill, &RMF_MDT_EPOCH);
-		body = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
-		epoch->handle = body->mbo_handle;
-		epoch->ioepoch = body->mbo_ioepoch;
-		req->rq_replay_cb = mdc_replay_open;
-	/** bug 3633, open may be committed and estale answer is not error */
-	} else if (rc == -ESTALE && (op_data->op_flags & MF_SOM_CHANGE)) {
-		rc = 0;
-	} else if (rc == -ERESTARTSYS) {
+	if (rc == -ERESTARTSYS)
 		rc = 0;
-	}
+
 	*request = req;
-	if (rc && req->rq_commit_cb) {
-		/* Put an extra reference on \var mod on error case. */
-		if (mod && *mod)
-			obd_mod_put(*mod);
-		req->rq_commit_cb(req);
-	}
+
 	return rc;
 }
 
@@ -264,7 +214,7 @@ rebuild:
 	}
 	level = LUSTRE_IMP_FULL;
  resend:
-	rc = mdc_reint(req, exp->exp_obd->u.cli.cl_rpc_lock, level);
+	rc = mdc_reint(req, level);
 
 	/* Resend if we were told to. */
 	if (rc == -ERESTARTSYS) {
@@ -332,13 +282,11 @@ int mdc_unlink(struct obd_export *exp, struct md_op_data *op_data,
 
 	req_capsule_set_size(&req->rq_pill, &RMF_MDT_MD, RCL_SERVER,
 			     obd->u.cli.cl_default_mds_easize);
-	req_capsule_set_size(&req->rq_pill, &RMF_LOGCOOKIES, RCL_SERVER,
-			     obd->u.cli.cl_default_mds_cookiesize);
 	ptlrpc_request_set_replen(req);
 
 	*request = req;
 
-	rc = mdc_reint(req, obd->u.cli.cl_rpc_lock, LUSTRE_IMP_FULL);
+	rc = mdc_reint(req, LUSTRE_IMP_FULL);
 	if (rc == -ERESTARTSYS)
 		rc = 0;
 	return rc;
@@ -348,7 +296,6 @@ int mdc_link(struct obd_export *exp, struct md_op_data *op_data,
 	     struct ptlrpc_request **request)
 {
 	LIST_HEAD(cancels);
-	struct obd_device *obd = exp->exp_obd;
 	struct ptlrpc_request *req;
 	int count = 0, rc;
 
@@ -380,7 +327,7 @@ int mdc_link(struct obd_export *exp, struct md_op_data *op_data,
 	mdc_link_pack(req, op_data);
 	ptlrpc_request_set_replen(req);
 
-	rc = mdc_reint(req, obd->u.cli.cl_rpc_lock, LUSTRE_IMP_FULL);
+	rc = mdc_reint(req, LUSTRE_IMP_FULL);
 	*request = req;
 	if (rc == -ERESTARTSYS)
 		rc = 0;
@@ -419,7 +366,8 @@ int mdc_rename(struct obd_export *exp, struct md_op_data *op_data,
 						 MDS_INODELOCK_FULL);
 
 	req = ptlrpc_request_alloc(class_exp2cliimp(exp),
-				   &RQF_MDS_REINT_RENAME);
+				   op_data->op_cli_flags & CLI_MIGRATE ?
+				   &RQF_MDS_REINT_MIGRATE : &RQF_MDS_REINT_RENAME);
 	if (!req) {
 		ldlm_lock_list_put(&cancels, l_bl_ast, count);
 		return -ENOMEM;
@@ -435,6 +383,23 @@ int mdc_rename(struct obd_export *exp, struct md_op_data *op_data,
 		return rc;
 	}
 
+	if (op_data->op_cli_flags & CLI_MIGRATE && op_data->op_data) {
+		struct md_open_data *mod = op_data->op_data;
+
+		LASSERTF(mod->mod_open_req &&
+			 mod->mod_open_req->rq_type != LI_POISON,
+			 "POISONED open %p!\n", mod->mod_open_req);
+
+		DEBUG_REQ(D_HA, mod->mod_open_req, "matched open");
+		/*
+		 * We no longer want to preserve this open for replay even
+		 * though the open was committed. b=3632, b=3633
+		 */
+		spin_lock(&mod->mod_open_req->rq_lock);
+		mod->mod_open_req->rq_replay = 0;
+		spin_unlock(&mod->mod_open_req->rq_lock);
+	}
+
 	if (exp_connect_cancelset(exp) && req)
 		ldlm_cli_cancel_list(&cancels, count, req, 0);
 
@@ -442,11 +407,9 @@ int mdc_rename(struct obd_export *exp, struct md_op_data *op_data,
 
 	req_capsule_set_size(&req->rq_pill, &RMF_MDT_MD, RCL_SERVER,
 			     obd->u.cli.cl_default_mds_easize);
-	req_capsule_set_size(&req->rq_pill, &RMF_LOGCOOKIES, RCL_SERVER,
-			     obd->u.cli.cl_default_mds_cookiesize);
 	ptlrpc_request_set_replen(req);
 
-	rc = mdc_reint(req, obd->u.cli.cl_rpc_lock, LUSTRE_IMP_FULL);
+	rc = mdc_reint(req, LUSTRE_IMP_FULL);
 	*request = req;
 	if (rc == -ERESTARTSYS)
 		rc = 0;
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_request.c b/drivers/staging/lustre/lustre/mdc/mdc_request.c
index f56ea643f9bf..2cfd913f9bc5 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_request.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_request.c
@@ -38,15 +38,18 @@
 # include <linux/init.h>
 # include <linux/utsname.h>
 
+#include "../include/cl_object.h"
+#include "../include/llog_swab.h"
+#include "../include/lprocfs_status.h"
 #include "../include/lustre_acl.h"
+#include "../include/lustre_fid.h"
 #include "../include/lustre/lustre_ioctl.h"
-#include "../include/obd_class.h"
+#include "../include/lustre_kernelcomm.h"
 #include "../include/lustre_lmv.h"
-#include "../include/lustre_fid.h"
-#include "../include/lprocfs_status.h"
-#include "../include/lustre_param.h"
 #include "../include/lustre_log.h"
-#include "../include/lustre_kernelcomm.h"
+#include "../include/lustre_param.h"
+#include "../include/lustre_swab.h"
+#include "../include/obd_class.h"
 
 #include "mdc_internal.h"
 
@@ -327,12 +330,12 @@ static int mdc_xattr_common(struct obd_export *exp,
 
 	/* make rpc */
 	if (opcode == MDS_REINT)
-		mdc_get_rpc_lock(exp->exp_obd->u.cli.cl_rpc_lock, NULL);
+		mdc_get_mod_rpc_slot(req, NULL);
 
 	rc = ptlrpc_queue_wait(req);
 
 	if (opcode == MDS_REINT)
-		mdc_put_rpc_lock(exp->exp_obd->u.cli.cl_rpc_lock, NULL);
+		mdc_put_mod_rpc_slot(req, NULL);
 
 	if (rc)
 		ptlrpc_req_finished(req);
@@ -420,9 +423,6 @@ static int mdc_get_lustre_md(struct obd_export *exp,
 	md->body = req_capsule_server_get(pill, &RMF_MDT_BODY);
 
 	if (md->body->mbo_valid & OBD_MD_FLEASIZE) {
-		int lmmsize;
-		struct lov_mds_md *lmm;
-
 		if (!S_ISREG(md->body->mbo_mode)) {
 			CDEBUG(D_INFO,
 			       "OBD_MD_FLEASIZE set, should be a regular file, but is not\n");
@@ -436,28 +436,18 @@ static int mdc_get_lustre_md(struct obd_export *exp,
 			rc = -EPROTO;
 			goto out;
 		}
-		lmmsize = md->body->mbo_eadatasize;
-		lmm = req_capsule_server_sized_get(pill, &RMF_MDT_MD, lmmsize);
-		if (!lmm) {
-			rc = -EPROTO;
-			goto out;
-		}
-
-		rc = obd_unpackmd(dt_exp, &md->lsm, lmm, lmmsize);
-		if (rc < 0)
-			goto out;
 
-		if (rc < (typeof(rc))sizeof(*md->lsm)) {
-			CDEBUG(D_INFO,
-			       "lsm size too small: rc < sizeof (*md->lsm) (%d < %d)\n",
-			       rc, (int)sizeof(*md->lsm));
+		md->layout.lb_len = md->body->mbo_eadatasize;
+		md->layout.lb_buf = req_capsule_server_sized_get(pill,
+								 &RMF_MDT_MD,
+								 md->layout.lb_len);
+		if (!md->layout.lb_buf) {
 			rc = -EPROTO;
 			goto out;
 		}
-
 	} else if (md->body->mbo_valid & OBD_MD_FLDIREA) {
-		int lmvsize;
-		struct lov_mds_md *lmv;
+		const union lmv_mds_md *lmv;
+		size_t lmv_size;
 
 		if (!S_ISDIR(md->body->mbo_mode)) {
 			CDEBUG(D_INFO,
@@ -466,22 +456,21 @@ static int mdc_get_lustre_md(struct obd_export *exp,
 			goto out;
 		}
 
-		if (md->body->mbo_eadatasize == 0) {
+		lmv_size = md->body->mbo_eadatasize;
+		if (!lmv_size) {
 			CDEBUG(D_INFO,
 			       "OBD_MD_FLDIREA is set, but eadatasize 0\n");
 			return -EPROTO;
 		}
 		if (md->body->mbo_valid & OBD_MD_MEA) {
-			lmvsize = md->body->mbo_eadatasize;
 			lmv = req_capsule_server_sized_get(pill, &RMF_MDT_MD,
-							   lmvsize);
+							   lmv_size);
 			if (!lmv) {
 				rc = -EPROTO;
 				goto out;
 			}
 
-			rc = obd_unpackmd(md_exp, (void *)&md->lmv, lmv,
-					  lmvsize);
+			rc = md_unpackmd(md_exp, &md->lmv, lmv, lmv_size);
 			if (rc < 0)
 				goto out;
 
@@ -517,8 +506,6 @@ out:
 #ifdef CONFIG_FS_POSIX_ACL
 		posix_acl_release(md->posix_acl);
 #endif
-		if (md->lsm)
-			obd_free_memmd(dt_exp, &md->lsm);
 	}
 	return rc;
 }
@@ -528,10 +515,6 @@ static int mdc_free_lustre_md(struct obd_export *exp, struct lustre_md *md)
 	return 0;
 }
 
-/**
- * Handles both OPEN and SETATTR RPCs for OPEN-CLOSE and SETATTR-DONE_WRITING
- * RPC chains.
- */
 void mdc_replay_open(struct ptlrpc_request *req)
 {
 	struct md_open_data *mod = req->rq_cb_data;
@@ -565,15 +548,15 @@ void mdc_replay_open(struct ptlrpc_request *req)
 		__u32 opc = lustre_msg_get_opc(close_req->rq_reqmsg);
 		struct mdt_ioepoch *epoch;
 
-		LASSERT(opc == MDS_CLOSE || opc == MDS_DONE_WRITING);
+		LASSERT(opc == MDS_CLOSE);
 		epoch = req_capsule_client_get(&close_req->rq_pill,
 					       &RMF_MDT_EPOCH);
 		LASSERT(epoch);
 
 		if (och)
-			LASSERT(!memcmp(&old, &epoch->handle, sizeof(old)));
+			LASSERT(!memcmp(&old, &epoch->mio_handle, sizeof(old)));
 		DEBUG_REQ(D_HA, close_req, "updating close body with new fh");
-		epoch->handle = body->mbo_handle;
+		epoch->mio_handle = body->mbo_handle;
 	}
 }
 
@@ -715,22 +698,6 @@ static int mdc_clear_open_replay_data(struct obd_export *exp,
 	return 0;
 }
 
-/* Prepares the request for the replay by the given reply */
-static void mdc_close_handle_reply(struct ptlrpc_request *req,
-				   struct md_op_data *op_data, int rc) {
-	struct mdt_body  *repbody;
-	struct mdt_ioepoch *epoch;
-
-	if (req && rc == -EAGAIN) {
-		repbody = req_capsule_server_get(&req->rq_pill, &RMF_MDT_BODY);
-		epoch = req_capsule_client_get(&req->rq_pill, &RMF_MDT_EPOCH);
-
-		epoch->flags |= MF_SOM_AU;
-		if (repbody->mbo_valid & OBD_MD_FLGETATTRLOCK)
-			op_data->op_flags |= MF_GETATTR_LOCK;
-	}
-}
-
 static int mdc_close(struct obd_export *exp, struct md_op_data *op_data,
 		     struct md_open_data *mod, struct ptlrpc_request **request)
 {
@@ -740,9 +707,8 @@ static int mdc_close(struct obd_export *exp, struct md_op_data *op_data,
 	int                    rc;
 	int		       saved_rc = 0;
 
-	req_fmt = &RQF_MDS_CLOSE;
 	if (op_data->op_bias & MDS_HSM_RELEASE) {
-		req_fmt = &RQF_MDS_RELEASE_CLOSE;
+		req_fmt = &RQF_MDS_INTENT_CLOSE;
 
 		/* allocate a FID for volatile file */
 		rc = mdc_fid_alloc(NULL, exp, &op_data->op_fid2, op_data);
@@ -752,6 +718,10 @@ static int mdc_close(struct obd_export *exp, struct md_op_data *op_data,
 			/* save the errcode and proceed to close */
 			saved_rc = rc;
 		}
+	} else if (op_data->op_bias & MDS_CLOSE_LAYOUT_SWAP) {
+		req_fmt = &RQF_MDS_INTENT_CLOSE;
+	} else {
+		req_fmt = &RQF_MDS_CLOSE;
 	}
 
 	*request = NULL;
@@ -807,14 +777,12 @@ static int mdc_close(struct obd_export *exp, struct md_op_data *op_data,
 
 	req_capsule_set_size(&req->rq_pill, &RMF_MDT_MD, RCL_SERVER,
 			     obd->u.cli.cl_default_mds_easize);
-	req_capsule_set_size(&req->rq_pill, &RMF_LOGCOOKIES, RCL_SERVER,
-			     obd->u.cli.cl_default_mds_cookiesize);
 
 	ptlrpc_request_set_replen(req);
 
-	mdc_get_rpc_lock(obd->u.cli.cl_close_lock, NULL);
+	mdc_get_mod_rpc_slot(req, NULL);
 	rc = ptlrpc_queue_wait(req);
-	mdc_put_rpc_lock(obd->u.cli.cl_close_lock, NULL);
+	mdc_put_mod_rpc_slot(req, NULL);
 
 	if (!req->rq_repmsg) {
 		CDEBUG(D_RPCTRACE, "request failed to send: %p, %d\n", req,
@@ -857,79 +825,9 @@ out:
 		obd_mod_put(mod);
 	}
 	*request = req;
-	mdc_close_handle_reply(req, op_data, rc);
 	return rc < 0 ? rc : saved_rc;
 }
 
-static int mdc_done_writing(struct obd_export *exp, struct md_op_data *op_data,
-			    struct md_open_data *mod)
-{
-	struct obd_device     *obd = class_exp2obd(exp);
-	struct ptlrpc_request *req;
-	int		    rc;
-
-	req = ptlrpc_request_alloc(class_exp2cliimp(exp),
-				   &RQF_MDS_DONE_WRITING);
-	if (!req)
-		return -ENOMEM;
-
-	rc = ptlrpc_request_pack(req, LUSTRE_MDS_VERSION, MDS_DONE_WRITING);
-	if (rc) {
-		ptlrpc_request_free(req);
-		return rc;
-	}
-
-	if (mod) {
-		LASSERTF(mod->mod_open_req &&
-			 mod->mod_open_req->rq_type != LI_POISON,
-			 "POISONED setattr %p!\n", mod->mod_open_req);
-
-		mod->mod_close_req = req;
-		DEBUG_REQ(D_HA, mod->mod_open_req, "matched setattr");
-		/* We no longer want to preserve this setattr for replay even
-		 * though the open was committed. b=3632, b=3633
-		 */
-		spin_lock(&mod->mod_open_req->rq_lock);
-		mod->mod_open_req->rq_replay = 0;
-		spin_unlock(&mod->mod_open_req->rq_lock);
-	}
-
-	mdc_close_pack(req, op_data);
-	ptlrpc_request_set_replen(req);
-
-	mdc_get_rpc_lock(obd->u.cli.cl_close_lock, NULL);
-	rc = ptlrpc_queue_wait(req);
-	mdc_put_rpc_lock(obd->u.cli.cl_close_lock, NULL);
-
-	if (rc == -ESTALE) {
-		/**
-		 * it can be allowed error after 3633 if open or setattr were
-		 * committed and server failed before close was sent.
-		 * Let's check if mod exists and return no error in that case
-		 */
-		if (mod) {
-			if (mod->mod_open_req->rq_committed)
-				rc = 0;
-		}
-	}
-
-	if (mod) {
-		if (rc != 0)
-			mod->mod_close_req = NULL;
-		LASSERT(mod->mod_open_req);
-		mdc_free_open(mod);
-
-		/* Since now, mod is accessed through setattr req only,
-		 * thus DW req does not keep a reference on mod anymore.
-		 */
-		obd_mod_put(mod);
-	}
-
-	mdc_close_handle_reply(req, op_data, rc);
-	ptlrpc_req_finished(req);
-	return rc;
-}
-
 static int mdc_getpage(struct obd_export *exp, const struct lu_fid *fid,
 		       u64 offset, struct page **pages, int npages,
 		       struct ptlrpc_request **request)
@@ -959,8 +857,10 @@ restart_bulk:
 	req->rq_request_portal = MDS_READPAGE_PORTAL;
 	ptlrpc_at_set_req_timeout(req);
 
-	desc = ptlrpc_prep_bulk_imp(req, npages, 1, BULK_PUT_SINK,
-				    MDS_BULK_PORTAL);
+	desc = ptlrpc_prep_bulk_imp(req, npages, 1,
+				    PTLRPC_BULK_PUT_SINK | PTLRPC_BULK_BUF_KIOV,
+				    MDS_BULK_PORTAL,
+				    &ptlrpc_bulk_kiov_pin_ops);
 	if (!desc) {
 		ptlrpc_request_free(req);
 		return -ENOMEM;
@@ -968,7 +868,7 @@ restart_bulk:
 
 	/* NB req now owns desc and will free it when it gets freed */
 	for (i = 0; i < npages; i++)
-		ptlrpc_prep_bulk_page_pin(desc, pages[i], 0, PAGE_SIZE);
+		desc->bd_frag_ops->add_kiov_frag(desc, pages[i], 0, PAGE_SIZE);
 
 	mdc_readdir_pack(req, offset, PAGE_SIZE * npages, fid);
 
@@ -1546,7 +1446,7 @@ static int mdc_ioc_fid2path(struct obd_export *exp, struct getinfo_fid2path *gf)
 	/* Val is struct getinfo_fid2path result plus path */
 	vallen = sizeof(*gf) + gf->gf_pathlen;
 
-	rc = obd_get_info(NULL, exp, keylen, key, &vallen, gf, NULL);
+	rc = obd_get_info(NULL, exp, keylen, key, &vallen, gf);
 	if (rc != 0 && rc != -EREMOTE)
 		goto out;
 
@@ -1558,8 +1458,11 @@ static int mdc_ioc_fid2path(struct obd_export *exp, struct getinfo_fid2path *gf)
 		goto out;
 	}
 
-	CDEBUG(D_IOCTL, "path get "DFID" from %llu #%d\n%s\n",
-	       PFID(&gf->gf_fid), gf->gf_recno, gf->gf_linkno, gf->gf_path);
+	CDEBUG(D_IOCTL, "path got " DFID " from %llu #%d: %s\n",
+	       PFID(&gf->gf_fid), gf->gf_recno, gf->gf_linkno,
+	       gf->gf_pathlen < 512 ? gf->gf_path :
+	       /* only log the last 512 characters of the path */
+	       gf->gf_path + gf->gf_pathlen - 512);
 
 out:
 	kfree(key);
@@ -1595,7 +1498,9 @@ static int mdc_ioc_hsm_progress(struct obd_export *exp,
 
 	ptlrpc_request_set_replen(req);
 
-	rc = mdc_queue_wait(req);
+	mdc_get_mod_rpc_slot(req, NULL);
+	rc = ptlrpc_queue_wait(req);
+	mdc_put_mod_rpc_slot(req, NULL);
 out:
 	ptlrpc_req_finished(req);
 	return rc;
@@ -1773,7 +1678,9 @@ static int mdc_ioc_hsm_state_set(struct obd_export *exp,
 
 	ptlrpc_request_set_replen(req);
 
-	rc = mdc_queue_wait(req);
+	mdc_get_mod_rpc_slot(req, NULL);
+	rc = ptlrpc_queue_wait(req);
+	mdc_put_mod_rpc_slot(req, NULL);
 out:
 	ptlrpc_req_finished(req);
 	return rc;
@@ -1836,7 +1743,9 @@ static int mdc_ioc_hsm_request(struct obd_export *exp,
 
 	ptlrpc_request_set_replen(req);
 
-	rc = mdc_queue_wait(req);
+	mdc_get_mod_rpc_slot(req, NULL);
+	rc = ptlrpc_queue_wait(req);
+	mdc_put_mod_rpc_slot(req, NULL);
 out:
 	ptlrpc_req_finished(req);
 	return rc;
@@ -1957,10 +1866,8 @@ static int mdc_changelog_send_thread(void *csdata)
 
 	/* Send EOF no matter what our result */
 	kuch = changelog_kuc_hdr(cs->cs_buf, sizeof(*kuch), cs->cs_flags);
-	if (kuch) {
-		kuch->kuc_msgtype = CL_EOF;
-		libcfs_kkuc_msg_put(cs->cs_fp, kuch);
-	}
+	kuch->kuc_msgtype = CL_EOF;
+	libcfs_kkuc_msg_put(cs->cs_fp, kuch);
 
 out:
 	fput(cs->cs_fp);
@@ -2015,52 +1922,6 @@ static int mdc_ioc_changelog_send(struct obd_device *obd,
 static int mdc_ioc_hsm_ct_start(struct obd_export *exp,
 				struct lustre_kernelcomm *lk);
 
-static int mdc_quotacheck(struct obd_device *unused, struct obd_export *exp,
-			  struct obd_quotactl *oqctl)
-{
-	struct client_obd       *cli = &exp->exp_obd->u.cli;
-	struct ptlrpc_request   *req;
-	struct obd_quotactl     *body;
-	int		      rc;
-
-	req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp),
-					&RQF_MDS_QUOTACHECK, LUSTRE_MDS_VERSION,
-					MDS_QUOTACHECK);
-	if (!req)
-		return -ENOMEM;
-
-	body = req_capsule_client_get(&req->rq_pill, &RMF_OBD_QUOTACTL);
-	*body = *oqctl;
-
-	ptlrpc_request_set_replen(req);
-
-	/* the next poll will find -ENODATA, that means quotacheck is
-	 * going on
-	 */
-	cli->cl_qchk_stat = -ENODATA;
-	rc = ptlrpc_queue_wait(req);
-	if (rc)
-		cli->cl_qchk_stat = rc;
-	ptlrpc_req_finished(req);
-	return rc;
-}
-
-static int mdc_quota_poll_check(struct obd_export *exp,
-				struct if_quotacheck *qchk)
-{
-	struct client_obd *cli = &exp->exp_obd->u.cli;
-	int rc;
-
-	qchk->obd_uuid = cli->cl_target_uuid;
-	memcpy(qchk->obd_type, LUSTRE_MDS_NAME, strlen(LUSTRE_MDS_NAME));
-
-	rc = cli->cl_qchk_stat;
-	/* the client is not the previous one */
-	if (rc == CL_NOT_QUOTACHECKED)
-		rc = -EINTR;
-	return rc;
-}
-
 static int mdc_quotactl(struct obd_device *unused, struct obd_export *exp,
 			struct obd_quotactl *oqctl)
 {
@@ -2215,9 +2076,6 @@ static int mdc_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 	case IOC_OSC_SET_ACTIVE:
 		rc = ptlrpc_set_import_active(imp, data->ioc_offset);
 		goto out;
-	case OBD_IOC_POLL_QUOTACHECK:
-		rc = mdc_quota_poll_check(exp, (struct if_quotacheck *)karg);
-		goto out;
 	case OBD_IOC_PING_TARGET:
 		rc = ptlrpc_obd_ping(obd);
 		goto out;
@@ -2528,8 +2386,7 @@ static int mdc_set_info_async(const struct lu_env *env,
 }
 
 static int mdc_get_info(const struct lu_env *env, struct obd_export *exp,
-			__u32 keylen, void *key, __u32 *vallen, void *val,
-			struct lov_stripe_md *lsm)
+			__u32 keylen, void *key, __u32 *vallen, void *val)
 {
 	int rc = -EINVAL;
 
@@ -2733,29 +2590,17 @@ static void mdc_llog_finish(struct obd_device *obd)
 
 static int mdc_setup(struct obd_device *obd, struct lustre_cfg *cfg)
 {
-	struct client_obd *cli = &obd->u.cli;
 	struct lprocfs_static_vars lvars = { NULL };
 	int rc;
 
-	cli->cl_rpc_lock = kzalloc(sizeof(*cli->cl_rpc_lock), GFP_NOFS);
-	if (!cli->cl_rpc_lock)
-		return -ENOMEM;
-	mdc_init_rpc_lock(cli->cl_rpc_lock);
-
 	rc = ptlrpcd_addref();
 	if (rc < 0)
-		goto err_rpc_lock;
-
-	cli->cl_close_lock = kzalloc(sizeof(*cli->cl_close_lock), GFP_NOFS);
-	if (!cli->cl_close_lock) {
-		rc = -ENOMEM;
-		goto err_ptlrpcd_decref;
-	}
-	mdc_init_rpc_lock(cli->cl_close_lock);
+		return rc;
 
 	rc = client_obd_setup(obd, cfg);
 	if (rc)
-		goto err_close_lock;
+		goto err_ptlrpcd_decref;
+
 	lprocfs_mdc_init_vars(&lvars);
 	lprocfs_obd_setup(obd, lvars.obd_vars, lvars.sysfs_vars);
 	sptlrpc_lprocfs_cliobd_attach(obd);
@@ -2769,29 +2614,25 @@ static int mdc_setup(struct obd_device *obd, struct lustre_cfg *cfg)
 	if (rc) {
 		mdc_cleanup(obd);
 		CERROR("failed to setup llogging subsystems\n");
+		return rc;
 	}
 
 	return rc;
 
-err_close_lock:
-	kfree(cli->cl_close_lock);
 err_ptlrpcd_decref:
 	ptlrpcd_decref();
-err_rpc_lock:
-	kfree(cli->cl_rpc_lock);
 	return rc;
 }
 
-/* Initialize the default and maximum LOV EA and cookie sizes.  This allows
+/* Initialize the default and maximum LOV EA sizes. This allows
  * us to make MDS RPCs with large enough reply buffers to hold a default
- * sized EA and cookie without having to calculate this (via a call into the
+ * sized EA without having to calculate this (via a call into the
  * LOV + OSCs) each time we make an RPC.  The maximum size is also tracked
  * but not used to avoid wastefully vmalloc()'ing large reply buffers when
  * a large number of stripes is possible.  If a larger reply buffer is
  * required it will be reallocated in the ptlrpc layer due to overflow.
  */
-static int mdc_init_ea_size(struct obd_export *exp, u32 easize, u32 def_easize,
-			    u32 cookiesize, u32 def_cookiesize)
+static int mdc_init_ea_size(struct obd_export *exp, u32 easize, u32 def_easize)
 {
 	struct obd_device *obd = exp->exp_obd;
 	struct client_obd *cli = &obd->u.cli;
@@ -2802,42 +2643,24 @@ static int mdc_init_ea_size(struct obd_export *exp, u32 easize, u32 def_easize,
 	if (cli->cl_default_mds_easize < def_easize)
 		cli->cl_default_mds_easize = def_easize;
 
-	if (cli->cl_max_mds_cookiesize < cookiesize)
-		cli->cl_max_mds_cookiesize = cookiesize;
-
-	if (cli->cl_default_mds_cookiesize < def_cookiesize)
-		cli->cl_default_mds_cookiesize = def_cookiesize;
-
 	return 0;
 }
 
-static int mdc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
+static int mdc_precleanup(struct obd_device *obd)
 {
-	switch (stage) {
-	case OBD_CLEANUP_EARLY:
-		break;
-	case OBD_CLEANUP_EXPORTS:
-		/* Failsafe, ok if racy */
-		if (obd->obd_type->typ_refcnt <= 1)
-			libcfs_kkuc_group_rem(0, KUC_GRP_HSM);
+	/* Failsafe, ok if racy */
+	if (obd->obd_type->typ_refcnt <= 1)
+		libcfs_kkuc_group_rem(0, KUC_GRP_HSM);
 
-		obd_cleanup_client_import(obd);
-		ptlrpc_lprocfs_unregister_obd(obd);
-		lprocfs_obd_cleanup(obd);
-
-		mdc_llog_finish(obd);
-		break;
-	}
+	obd_cleanup_client_import(obd);
+	ptlrpc_lprocfs_unregister_obd(obd);
+	lprocfs_obd_cleanup(obd);
+	mdc_llog_finish(obd);
 	return 0;
 }
 
 static int mdc_cleanup(struct obd_device *obd)
 {
-	struct client_obd *cli = &obd->u.cli;
-
-	kfree(cli->cl_rpc_lock);
-	kfree(cli->cl_close_lock);
-
 	ptlrpcd_decref();
 
 	return client_obd_cleanup(obd);
@@ -2881,7 +2704,6 @@ static struct obd_ops mdc_obd_ops = {
 	.process_config = mdc_process_config,
 	.get_uuid       = mdc_get_uuid,
 	.quotactl       = mdc_quotactl,
-	.quotacheck     = mdc_quotacheck
 };
 
 static struct md_ops mdc_md_ops = {
@@ -2889,7 +2711,6 @@ static struct md_ops mdc_md_ops = {
 	.null_inode		= mdc_null_inode,
 	.close			= mdc_close,
 	.create			= mdc_create,
-	.done_writing		= mdc_done_writing,
 	.enqueue		= mdc_enqueue,
 	.getattr		= mdc_getattr,
 	.getattr_name		= mdc_getattr_name,
diff --git a/drivers/staging/lustre/lustre/mgc/mgc_request.c b/drivers/staging/lustre/lustre/mgc/mgc_request.c
index 23374cae5133..b9c522a3c7a4 100644
--- a/drivers/staging/lustre/lustre/mgc/mgc_request.c
+++ b/drivers/staging/lustre/lustre/mgc/mgc_request.c
@@ -38,11 +38,13 @@
 #define D_MGC D_CONFIG /*|D_WARNING*/
 
 #include <linux/module.h>
-#include "../include/obd_class.h"
-#include "../include/lustre_dlm.h"
+
 #include "../include/lprocfs_status.h"
-#include "../include/lustre_log.h"
+#include "../include/lustre_dlm.h"
 #include "../include/lustre_disk.h"
+#include "../include/lustre_log.h"
+#include "../include/lustre_swab.h"
+#include "../include/obd_class.h"
 
 #include "mgc_internal.h"
 
@@ -373,7 +375,7 @@ out_err:
 	return rc;
 }
 
-DEFINE_MUTEX(llog_process_lock);
+static DEFINE_MUTEX(llog_process_lock);
 
 /** Stop watching for updates on this log.
  */
@@ -684,35 +686,33 @@ static int mgc_llog_fini(const struct lu_env *env, struct obd_device *obd)
 }
 
 static atomic_t mgc_count = ATOMIC_INIT(0);
-static int mgc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
+static int mgc_precleanup(struct obd_device *obd)
 {
 	int rc = 0;
 	int temp;
 
-	switch (stage) {
-	case OBD_CLEANUP_EARLY:
-		break;
-	case OBD_CLEANUP_EXPORTS:
-		if (atomic_dec_and_test(&mgc_count)) {
-			LASSERT(rq_state & RQ_RUNNING);
-			/* stop requeue thread */
-			temp = RQ_STOP;
-		} else {
-			/* wakeup requeue thread to clean our cld */
-			temp = RQ_NOW | RQ_PRECLEANUP;
-		}
-		spin_lock(&config_list_lock);
-		rq_state |= temp;
-		spin_unlock(&config_list_lock);
-		wake_up(&rq_waitq);
-		if (temp & RQ_STOP)
-			wait_for_completion(&rq_exit);
-		obd_cleanup_client_import(obd);
-		rc = mgc_llog_fini(NULL, obd);
-		if (rc != 0)
-			CERROR("failed to cleanup llogging subsystems\n");
-		break;
+	if (atomic_dec_and_test(&mgc_count)) {
+		LASSERT(rq_state & RQ_RUNNING);
+		/* stop requeue thread */
+		temp = RQ_STOP;
+	} else {
+		/* wakeup requeue thread to clean our cld */
+		temp = RQ_NOW | RQ_PRECLEANUP;
 	}
+
+	spin_lock(&config_list_lock);
+	rq_state |= temp;
+	spin_unlock(&config_list_lock);
+	wake_up(&rq_waitq);
+
+	if (temp & RQ_STOP)
+		wait_for_completion(&rq_exit);
+	obd_cleanup_client_import(obd);
+
+	rc = mgc_llog_fini(NULL, obd);
+	if (rc)
+		CERROR("failed to cleanup llogging subsystems\n");
+
 	return rc;
 }
 
@@ -887,8 +887,8 @@ static int mgc_set_mgs_param(struct obd_export *exp,
 }
 
 /* Take a config lock so we can get cancel notifications */
-static int mgc_enqueue(struct obd_export *exp, struct lov_stripe_md *lsm,
-		       __u32 type, ldlm_policy_data_t *policy, __u32 mode,
+static int mgc_enqueue(struct obd_export *exp, __u32 type,
+		       union ldlm_policy_data *policy, __u32 mode,
 		       __u64 *flags, void *bl_cb, void *cp_cb, void *gl_cb,
 		       void *data, __u32 lvb_len, void *lvb_swabber,
 		       struct lustre_handle *lockh)
@@ -1059,8 +1059,7 @@ static int mgc_set_info_async(const struct lu_env *env, struct obd_export *exp,
 }
 
 static int mgc_get_info(const struct lu_env *env, struct obd_export *exp,
-			__u32 keylen, void *key, __u32 *vallen, void *val,
-			struct lov_stripe_md *unused)
+			__u32 keylen, void *key, __u32 *vallen, void *val)
 {
 	int rc = -EINVAL;
 
@@ -1387,15 +1386,17 @@ again:
 	body->mcb_units  = nrpages;
 
 	/* allocate bulk transfer descriptor */
-	desc = ptlrpc_prep_bulk_imp(req, nrpages, 1, BULK_PUT_SINK,
-				    MGS_BULK_PORTAL);
+	desc = ptlrpc_prep_bulk_imp(req, nrpages, 1,
+				    PTLRPC_BULK_PUT_SINK | PTLRPC_BULK_BUF_KIOV,
+				    MGS_BULK_PORTAL,
+				    &ptlrpc_bulk_kiov_pin_ops);
 	if (!desc) {
 		rc = -ENOMEM;
 		goto out;
 	}
 
 	for (i = 0; i < nrpages; i++)
-		ptlrpc_prep_bulk_page_pin(desc, pages[i], 0, PAGE_SIZE);
+		desc->bd_frag_ops->add_kiov_frag(desc, pages[i], 0, PAGE_SIZE);
 
 	ptlrpc_request_set_replen(req);
 	rc = ptlrpc_queue_wait(req);
@@ -1553,14 +1554,52 @@ out_free:
 	return rc;
 }
 
-/** Get a config log from the MGS and process it.
- * This func is called for both clients and servers.
- * Copy the log locally before parsing it if appropriate (non-MGS server)
+static bool mgc_import_in_recovery(struct obd_import *imp)
+{
+	bool in_recovery = true;
+
+	spin_lock(&imp->imp_lock);
+	if (imp->imp_state == LUSTRE_IMP_FULL ||
+	    imp->imp_state == LUSTRE_IMP_CLOSED)
+		in_recovery = false;
+	spin_unlock(&imp->imp_lock);
+
+	return in_recovery;
+}
+
+/**
+ * Get a configuration log from the MGS and process it.
+ *
+ * This function is called for both clients and servers to process the
+ * configuration log from the MGS.  The MGC enqueues a DLM lock on the
+ * log from the MGS, and if the lock gets revoked the MGC will be notified
+ * by the lock cancellation callback that the config log has changed,
+ * and will enqueue another MGS lock on it, and then continue processing
+ * the new additions to the end of the log.
+ *
+ * Since the MGC import is not replayable, if the import is being evicted
+ * (rcl == -ESHUTDOWN, \see ptlrpc_import_delay_req()), retry to process
+ * the log until recovery is finished or the import is closed.
+ *
+ * Make a local copy of the log before parsing it if appropriate (non-MGS
+ * server) so that the server can start even when the MGS is down.
+ *
+ * There shouldn't be multiple processes running process_log at once --
+ * sounds like badness.  It actually might be fine, as long as they're not
+ * trying to update from the same log simultaneously, in which case we
+ * should use a per-log semaphore instead of cld_lock.
+ *
+ * \param[in] mgc	MGC device by which to fetch the configuration log
+ * \param[in] cld	log processing state (stored in lock callback data)
+ *
+ * \retval		0 on success
+ * \retval		negative errno on failure
  */
 int mgc_process_log(struct obd_device *mgc, struct config_llog_data *cld)
 {
 	struct lustre_handle lockh = { 0 };
 	__u64 flags = LDLM_FL_NO_LRU;
+	bool retry = false;
 	int rc = 0, rcl;
 
 	LASSERT(cld);
@@ -1570,6 +1609,7 @@ int mgc_process_log(struct obd_device *mgc, struct config_llog_data *cld)
 	 * we're not trying to update from the same log
 	 * simultaneously (in which case we should use a per-log sem.)
 	 */
+restart:
 	mutex_lock(&cld->cld_lock);
 	if (cld->cld_stopping) {
 		mutex_unlock(&cld->cld_lock);
@@ -1582,7 +1622,7 @@ int mgc_process_log(struct obd_device *mgc, struct config_llog_data *cld)
 	       cld->cld_cfg.cfg_instance, cld->cld_cfg.cfg_last_idx + 1);
 
 	/* Get the cfg lock on the llog */
-	rcl = mgc_enqueue(mgc->u.cli.cl_mgc_mgsexp, NULL, LDLM_PLAIN, NULL,
+	rcl = mgc_enqueue(mgc->u.cli.cl_mgc_mgsexp, LDLM_PLAIN, NULL,
 			  LCK_CR, &flags, NULL, NULL, NULL,
 			  cld, 0, NULL, &lockh);
 	if (rcl == 0) {
@@ -1593,18 +1633,57 @@ int mgc_process_log(struct obd_device *mgc, struct config_llog_data *cld)
 	} else {
 		CDEBUG(D_MGC, "Can't get cfg lock: %d\n", rcl);
 
-		/* mark cld_lostlock so that it will requeue
-		 * after MGC becomes available.
-		 */
-		cld->cld_lostlock = 1;
+		if (rcl == -ESHUTDOWN &&
+		    atomic_read(&mgc->u.cli.cl_mgc_refcount) > 0 && !retry) {
+			int secs = cfs_time_seconds(obd_timeout);
+			struct obd_import *imp;
+			struct l_wait_info lwi;
+
+			mutex_unlock(&cld->cld_lock);
+			imp = class_exp2cliimp(mgc->u.cli.cl_mgc_mgsexp);
+
+			/*
+			 * Let's force the pinger, and wait the import to be
+			 * connected, note: since mgc import is non-replayable,
+			 * and even the import state is disconnected, it does
+			 * not mean the "recovery" is stopped, so we will keep
+			 * waitting until timeout or the import state is
+			 * FULL or closed
+			 */
+			ptlrpc_pinger_force(imp);
+
+			lwi = LWI_TIMEOUT(secs, NULL, NULL);
+			l_wait_event(imp->imp_recovery_waitq,
+				     !mgc_import_in_recovery(imp), &lwi);
+
+			if (imp->imp_state == LUSTRE_IMP_FULL) {
+				retry = true;
+				goto restart;
+			} else {
+				mutex_lock(&cld->cld_lock);
+				cld->cld_lostlock = 1;
+			}
+		} else {
+			/* mark cld_lostlock so that it will requeue
+			 * after MGC becomes available.
+			 */
+			cld->cld_lostlock = 1;
+		}
 		/* Get extra reference, it will be put in requeue thread */
 		config_log_get(cld);
 	}
 
 	if (cld_is_recover(cld)) {
 		rc = 0; /* this is not a fatal error for recover log */
-		if (rcl == 0)
+		if (!rcl) {
 			rc = mgc_process_recover_log(mgc, cld);
+			if (rc) {
+				CERROR("%s: recover log %s failed: rc = %d not fatal.\n",
+				       mgc->obd_name, cld->cld_logname, rc);
+				rc = 0;
+				cld->cld_lostlock = 1;
+			}
+		}
 	} else {
 		rc = mgc_process_cfg_log(mgc, cld, rcl != 0);
 	}
diff --git a/drivers/staging/lustre/lustre/obdclass/Makefile b/drivers/staging/lustre/lustre/obdclass/Makefile
index b42e109b30e0..af570c0db15b 100644
--- a/drivers/staging/lustre/lustre/obdclass/Makefile
+++ b/drivers/staging/lustre/lustre/obdclass/Makefile
@@ -1,6 +1,6 @@
 obj-$(CONFIG_LUSTRE_FS) += obdclass.o
 
-obdclass-y := linux/linux-module.o linux/linux-obdo.o linux/linux-sysctl.o \
+obdclass-y := linux/linux-module.o linux/linux-sysctl.o \
 	      llog.o llog_cat.o llog_obd.o llog_swab.o class_obd.o debug.o \
 	      genops.o uuid.o lprocfs_status.o lprocfs_counters.o \
 	      lustre_handles.o lustre_peer.o statfs_pack.o linkea.o \
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_internal.h b/drivers/staging/lustre/lustre/obdclass/cl_internal.h
index e866754a42d5..7b403fbd5f94 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_internal.h
+++ b/drivers/staging/lustre/lustre/obdclass/cl_internal.h
@@ -50,25 +50,6 @@ enum clt_nesting_level {
 };
 
 /**
- * Counters used to check correctness of cl_lock interface usage.
- */
-struct cl_thread_counters {
-	/**
-	 * Number of outstanding calls to cl_lock_mutex_get() made by the
-	 * current thread. For debugging.
-	 */
-	int	   ctc_nr_locks_locked;
-	/** List of locked locks. */
-	struct lu_ref ctc_locks_locked;
-	/** Number of outstanding holds on locks. */
-	int	   ctc_nr_held;
-	/** Number of outstanding uses on locks. */
-	int	   ctc_nr_used;
-	/** Number of held extent locks. */
-	int	   ctc_nr_locks_acquired;
-};
-
-/**
  * Thread local state internal for generic cl-code.
  */
 struct cl_thread_info {
@@ -83,10 +64,6 @@ struct cl_thread_info {
 	 */
 	struct cl_lock_descr clt_descr;
 	struct cl_page_list  clt_list;
-	/**
-	 * Counters for every level of lock nesting.
-	 */
-	struct cl_thread_counters clt_counters[CNL_NR];
 	/** @} debugging */
 
 	/*
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_io.c b/drivers/staging/lustre/lustre/obdclass/cl_io.c
index bc4b7b6b9a20..3f42457b0d7d 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_io.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_io.c
@@ -126,6 +126,7 @@ void cl_io_fini(const struct lu_env *env, struct cl_io *io)
 	switch (io->ci_type) {
 	case CIT_READ:
 	case CIT_WRITE:
+	case CIT_DATA_VERSION:
 		break;
 	case CIT_FAULT:
 		break;
@@ -411,7 +412,6 @@ void cl_io_unlock(const struct lu_env *env, struct cl_io *io)
 			scan->cis_iop->op[io->ci_type].cio_unlock(env, scan);
 	}
 	io->ci_state = CIS_UNLOCKED;
-	LASSERT(!cl_env_info(env)->clt_counters[CNL_TOP].ctc_nr_locks_acquired);
 }
 EXPORT_SYMBOL(cl_io_unlock);
 
@@ -586,67 +586,32 @@ void cl_io_end(const struct lu_env *env, struct cl_io *io)
 }
 EXPORT_SYMBOL(cl_io_end);
 
-static const struct cl_page_slice *
-cl_io_slice_page(const struct cl_io_slice *ios, struct cl_page *page)
-{
-	const struct cl_page_slice *slice;
-
-	slice = cl_page_at(page, ios->cis_obj->co_lu.lo_dev->ld_type);
-	LINVRNT(slice);
-	return slice;
-}
-
 /**
- * Called by read io, when page has to be read from the server.
+ * Called by read io, to decide the readahead extent
  *
- * \see cl_io_operations::cio_read_page()
+ * \see cl_io_operations::cio_read_ahead()
  */
-int cl_io_read_page(const struct lu_env *env, struct cl_io *io,
-		    struct cl_page *page)
+int cl_io_read_ahead(const struct lu_env *env, struct cl_io *io,
+		     pgoff_t start, struct cl_read_ahead *ra)
 {
 	const struct cl_io_slice *scan;
-	struct cl_2queue	 *queue;
 	int		       result = 0;
 
 	LINVRNT(io->ci_type == CIT_READ || io->ci_type == CIT_FAULT);
-	LINVRNT(cl_page_is_owned(page, io));
 	LINVRNT(io->ci_state == CIS_IO_GOING || io->ci_state == CIS_LOCKED);
 	LINVRNT(cl_io_invariant(io));
 
-	queue = &io->ci_queue;
-
-	cl_2queue_init(queue);
-	/*
-	 * ->cio_read_page() methods called in the loop below are supposed to
-	 * never block waiting for network (the only subtle point is the
-	 * creation of new pages for read-ahead that might result in cache
-	 * shrinking, but currently only clean pages are shrunk and this
-	 * requires no network io).
-	 *
-	 * Should this ever starts blocking, retry loop would be needed for
-	 * "parallel io" (see CLO_REPEAT loops in cl_lock.c).
-	 */
 	cl_io_for_each(scan, io) {
-		if (scan->cis_iop->cio_read_page) {
-			const struct cl_page_slice *slice;
+		if (!scan->cis_iop->cio_read_ahead)
+			continue;
 
-			slice = cl_io_slice_page(scan, page);
-			LINVRNT(slice);
-			result = scan->cis_iop->cio_read_page(env, scan, slice);
-			if (result != 0)
-				break;
-		}
+		result = scan->cis_iop->cio_read_ahead(env, scan, start, ra);
+		if (result)
+			break;
 	}
-	if (result == 0 && queue->c2_qin.pl_nr > 0)
-		result = cl_io_submit_rw(env, io, CRT_READ, queue);
-	/*
-	 * Unlock unsent pages in case of error.
-	 */
-	cl_page_list_disown(env, io, &queue->c2_qin);
-	cl_2queue_fini(env, queue);
-	return result;
+	return result > 0 ? 0 : result;
 }
-EXPORT_SYMBOL(cl_io_read_page);
+EXPORT_SYMBOL(cl_io_read_ahead);
 
 /**
  * Commit a list of contiguous pages into writeback cache.
@@ -1080,235 +1045,18 @@ struct cl_io *cl_io_top(struct cl_io *io)
 EXPORT_SYMBOL(cl_io_top);
 
 /**
- * Adds request slice to the compound request.
- *
- * This is called by cl_device_operations::cdo_req_init() methods to add a
- * per-layer state to the request. New state is added at the end of
- * cl_req::crq_layers list, that is, it is at the bottom of the stack.
- *
- * \see cl_lock_slice_add(), cl_page_slice_add(), cl_io_slice_add()
- */
-void cl_req_slice_add(struct cl_req *req, struct cl_req_slice *slice,
-		      struct cl_device *dev,
-		      const struct cl_req_operations *ops)
-{
-	list_add_tail(&slice->crs_linkage, &req->crq_layers);
-	slice->crs_dev = dev;
-	slice->crs_ops = ops;
-	slice->crs_req = req;
-}
-EXPORT_SYMBOL(cl_req_slice_add);
-
-static void cl_req_free(const struct lu_env *env, struct cl_req *req)
-{
-	unsigned i;
-
-	LASSERT(list_empty(&req->crq_pages));
-	LASSERT(req->crq_nrpages == 0);
-	LINVRNT(list_empty(&req->crq_layers));
-	LINVRNT(equi(req->crq_nrobjs > 0, req->crq_o));
-
-	if (req->crq_o) {
-		for (i = 0; i < req->crq_nrobjs; ++i) {
-			struct cl_object *obj = req->crq_o[i].ro_obj;
-
-			if (obj) {
-				lu_object_ref_del_at(&obj->co_lu,
-						     &req->crq_o[i].ro_obj_ref,
-						     "cl_req", req);
-				cl_object_put(env, obj);
-			}
-		}
-		kfree(req->crq_o);
-	}
-	kfree(req);
-}
-
-static int cl_req_init(const struct lu_env *env, struct cl_req *req,
-		       struct cl_page *page)
-{
-	struct cl_device     *dev;
-	struct cl_page_slice *slice;
-	int result;
-
-	result = 0;
-	list_for_each_entry(slice, &page->cp_layers, cpl_linkage) {
-		dev = lu2cl_dev(slice->cpl_obj->co_lu.lo_dev);
-		if (dev->cd_ops->cdo_req_init) {
-			result = dev->cd_ops->cdo_req_init(env, dev, req);
-			if (result != 0)
-				break;
-		}
-	}
-	return result;
-}
-
-/**
- * Invokes per-request transfer completion call-backs
- * (cl_req_operations::cro_completion()) bottom-to-top.
- */
-void cl_req_completion(const struct lu_env *env, struct cl_req *req, int rc)
-{
-	struct cl_req_slice *slice;
-
-	/*
-	 * for the lack of list_for_each_entry_reverse_safe()...
-	 */
-	while (!list_empty(&req->crq_layers)) {
-		slice = list_entry(req->crq_layers.prev,
-				   struct cl_req_slice, crs_linkage);
-		list_del_init(&slice->crs_linkage);
-		if (slice->crs_ops->cro_completion)
-			slice->crs_ops->cro_completion(env, slice, rc);
-	}
-	cl_req_free(env, req);
-}
-EXPORT_SYMBOL(cl_req_completion);
-
-/**
- * Allocates new transfer request.
- */
-struct cl_req *cl_req_alloc(const struct lu_env *env, struct cl_page *page,
-			    enum cl_req_type crt, int nr_objects)
-{
-	struct cl_req *req;
-
-	LINVRNT(nr_objects > 0);
-
-	req = kzalloc(sizeof(*req), GFP_NOFS);
-	if (req) {
-		int result;
-
-		req->crq_type = crt;
-		INIT_LIST_HEAD(&req->crq_pages);
-		INIT_LIST_HEAD(&req->crq_layers);
-
-		req->crq_o = kcalloc(nr_objects, sizeof(req->crq_o[0]),
-				     GFP_NOFS);
-		if (req->crq_o) {
-			req->crq_nrobjs = nr_objects;
-			result = cl_req_init(env, req, page);
-		} else {
-			result = -ENOMEM;
-		}
-		if (result != 0) {
-			cl_req_completion(env, req, result);
-			req = ERR_PTR(result);
-		}
-	} else {
-		req = ERR_PTR(-ENOMEM);
-	}
-	return req;
-}
-EXPORT_SYMBOL(cl_req_alloc);
-
-/**
- * Adds a page to a request.
- */
-void cl_req_page_add(const struct lu_env *env,
-		     struct cl_req *req, struct cl_page *page)
-{
-	struct cl_object  *obj;
-	struct cl_req_obj *rqo;
-	unsigned int i;
-
-	LASSERT(list_empty(&page->cp_flight));
-	LASSERT(!page->cp_req);
-
-	CL_PAGE_DEBUG(D_PAGE, env, page, "req %p, %d, %u\n",
-		      req, req->crq_type, req->crq_nrpages);
-
-	list_add_tail(&page->cp_flight, &req->crq_pages);
-	++req->crq_nrpages;
-	page->cp_req = req;
-	obj = cl_object_top(page->cp_obj);
-	for (i = 0, rqo = req->crq_o; obj != rqo->ro_obj; ++i, ++rqo) {
-		if (!rqo->ro_obj) {
-			rqo->ro_obj = obj;
-			cl_object_get(obj);
-			lu_object_ref_add_at(&obj->co_lu, &rqo->ro_obj_ref,
-					     "cl_req", req);
-			break;
-		}
-	}
-	LASSERT(i < req->crq_nrobjs);
-}
-EXPORT_SYMBOL(cl_req_page_add);
-
-/**
- * Removes a page from a request.
- */
-void cl_req_page_done(const struct lu_env *env, struct cl_page *page)
-{
-	struct cl_req *req = page->cp_req;
-
-	LASSERT(!list_empty(&page->cp_flight));
-	LASSERT(req->crq_nrpages > 0);
-
-	list_del_init(&page->cp_flight);
-	--req->crq_nrpages;
-	page->cp_req = NULL;
-}
-EXPORT_SYMBOL(cl_req_page_done);
-
-/**
- * Notifies layers that request is about to depart by calling
- * cl_req_operations::cro_prep() top-to-bottom.
- */
-int cl_req_prep(const struct lu_env *env, struct cl_req *req)
-{
-	unsigned int i;
-	int result;
-	const struct cl_req_slice *slice;
-
-	/*
-	 * Check that the caller of cl_req_alloc() didn't lie about the number
-	 * of objects.
-	 */
-	for (i = 0; i < req->crq_nrobjs; ++i)
-		LASSERT(req->crq_o[i].ro_obj);
-
-	result = 0;
-	list_for_each_entry(slice, &req->crq_layers, crs_linkage) {
-		if (slice->crs_ops->cro_prep) {
-			result = slice->crs_ops->cro_prep(env, slice);
-			if (result != 0)
-				break;
-		}
-	}
-	return result;
-}
-EXPORT_SYMBOL(cl_req_prep);
-
-/**
  * Fills in attributes that are passed to server together with transfer. Only
  * attributes from \a flags may be touched. This can be called multiple times
  * for the same request.
  */
-void cl_req_attr_set(const struct lu_env *env, struct cl_req *req,
-		     struct cl_req_attr *attr, u64 flags)
+void cl_req_attr_set(const struct lu_env *env, struct cl_object *obj,
+		     struct cl_req_attr *attr)
 {
-	const struct cl_req_slice *slice;
-	struct cl_page	    *page;
-	unsigned int i;
-
-	LASSERT(!list_empty(&req->crq_pages));
-
-	/* Take any page to use as a model. */
-	page = list_entry(req->crq_pages.next, struct cl_page, cp_flight);
-
-	for (i = 0; i < req->crq_nrobjs; ++i) {
-		list_for_each_entry(slice, &req->crq_layers, crs_linkage) {
-			const struct cl_page_slice *scan;
-			const struct cl_object     *obj;
-
-			scan = cl_page_at(page,
-					  slice->crs_dev->cd_lu_dev.ld_type);
-			obj = scan->cpl_obj;
-			if (slice->crs_ops->cro_attr_set)
-				slice->crs_ops->cro_attr_set(env, slice, obj,
-							     attr + i, flags);
-		}
+	struct cl_object *scan;
+
+	cl_object_for_each(scan, obj) {
+		if (scan->co_ops->coo_req_attr_set)
+			scan->co_ops->coo_req_attr_set(env, scan, attr);
 	}
 }
 EXPORT_SYMBOL(cl_req_attr_set);
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_object.c b/drivers/staging/lustre/lustre/obdclass/cl_object.c
index 3199dd4a3b72..f5d4e23c64b7 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_object.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_object.c
@@ -335,7 +335,7 @@ int cl_object_getstripe(const struct lu_env *env, struct cl_object *obj,
 		if (obj->co_ops->coo_getstripe) {
 			result = obj->co_ops->coo_getstripe(env, obj, uarg);
 			if (result)
-			break;
+				break;
 		}
 	}
 	return result;
@@ -343,6 +343,67 @@ int cl_object_getstripe(const struct lu_env *env, struct cl_object *obj,
 EXPORT_SYMBOL(cl_object_getstripe);
 
 /**
+ * Get fiemap extents from file object.
+ *
+ * \param env [in]	lustre environment
+ * \param obj [in]	file object
+ * \param key [in]	fiemap request argument
+ * \param fiemap [out]	fiemap extents mapping retrived
+ * \param buflen [in]	max buffer length of @fiemap
+ *
+ * \retval 0	success
+ * \retval < 0	error
+ */
+int cl_object_fiemap(const struct lu_env *env, struct cl_object *obj,
+		     struct ll_fiemap_info_key *key,
+		     struct fiemap *fiemap, size_t *buflen)
+{
+	struct lu_object_header *top;
+	int result = 0;
+
+	top = obj->co_lu.lo_header;
+	list_for_each_entry(obj, &top->loh_layers, co_lu.lo_linkage) {
+		if (obj->co_ops->coo_fiemap) {
+			result = obj->co_ops->coo_fiemap(env, obj, key, fiemap,
+							 buflen);
+			if (result)
+				break;
+		}
+	}
+	return result;
+}
+EXPORT_SYMBOL(cl_object_fiemap);
+
+int cl_object_layout_get(const struct lu_env *env, struct cl_object *obj,
+			 struct cl_layout *cl)
+{
+	struct lu_object_header *top = obj->co_lu.lo_header;
+
+	list_for_each_entry(obj, &top->loh_layers, co_lu.lo_linkage) {
+		if (obj->co_ops->coo_layout_get)
+			return obj->co_ops->coo_layout_get(env, obj, cl);
+	}
+
+	return -EOPNOTSUPP;
+}
+EXPORT_SYMBOL(cl_object_layout_get);
+
+loff_t cl_object_maxbytes(struct cl_object *obj)
+{
+	struct lu_object_header *top = obj->co_lu.lo_header;
+	loff_t maxbytes = LLONG_MAX;
+
+	list_for_each_entry(obj, &top->loh_layers, co_lu.lo_linkage) {
+		if (obj->co_ops->coo_maxbytes)
+			maxbytes = min_t(loff_t, obj->co_ops->coo_maxbytes(obj),
+					 maxbytes);
+	}
+
+	return maxbytes;
+}
+EXPORT_SYMBOL(cl_object_maxbytes);
+
+/**
  * Helper function removing all object locks, and marking object for
  * deletion. All object pages must have been deleted at this point.
  *
@@ -483,36 +544,20 @@ EXPORT_SYMBOL(cl_site_stats_print);
  * bz20044, bz22683.
  */
 
-static LIST_HEAD(cl_envs);
-static unsigned int cl_envs_cached_nr;
-static unsigned int cl_envs_cached_max = 128; /* XXX: prototype: arbitrary limit
-					       * for now.
-					       */
-static DEFINE_SPINLOCK(cl_envs_guard);
+static unsigned int cl_envs_cached_max = 32; /* XXX: prototype: arbitrary limit
+					      * for now.
+					      */
+static struct cl_env_cache {
+	rwlock_t		cec_guard;
+	unsigned int		cec_count;
+	struct list_head	cec_envs;
+} *cl_envs = NULL;
 
 struct cl_env {
 	void	     *ce_magic;
 	struct lu_env     ce_lu;
 	struct lu_context ce_ses;
 
-	/**
-	 * This allows cl_env to be entered into cl_env_hash which implements
-	 * the current thread -> client environment lookup.
-	 */
-	struct hlist_node  ce_node;
-	/**
-	 * Owner for the current cl_env.
-	 *
-	 * If LL_TASK_CL_ENV is defined, this point to the owning current,
-	 * only for debugging purpose ;
-	 * Otherwise hash is used, and this is the key for cfs_hash.
-	 * Now current thread pid is stored. Note using thread pointer would
-	 * lead to unbalanced hash because of its specific allocation locality
-	 * and could be varied for different platforms and OSes, even different
-	 * OS versions.
-	 */
-	void	     *ce_owner;
-
 	/*
 	 * Linkage into global list of all client environments. Used for
 	 * garbage collection.
@@ -536,122 +581,13 @@ static void cl_env_init0(struct cl_env *cle, void *debug)
 {
 	LASSERT(cle->ce_ref == 0);
 	LASSERT(cle->ce_magic == &cl_env_init0);
-	LASSERT(!cle->ce_debug && !cle->ce_owner);
+	LASSERT(!cle->ce_debug);
 
 	cle->ce_ref = 1;
 	cle->ce_debug = debug;
 	CL_ENV_INC(busy);
 }
 
-/*
- * The implementation of using hash table to connect cl_env and thread
- */
-
-static struct cfs_hash *cl_env_hash;
-
-static unsigned cl_env_hops_hash(struct cfs_hash *lh,
-				 const void *key, unsigned mask)
-{
-#if BITS_PER_LONG == 64
-	return cfs_hash_u64_hash((__u64)key, mask);
-#else
-	return cfs_hash_u32_hash((__u32)key, mask);
-#endif
-}
-
-static void *cl_env_hops_obj(struct hlist_node *hn)
-{
-	struct cl_env *cle = hlist_entry(hn, struct cl_env, ce_node);
-
-	LASSERT(cle->ce_magic == &cl_env_init0);
-	return (void *)cle;
-}
-
-static int cl_env_hops_keycmp(const void *key, struct hlist_node *hn)
-{
-	struct cl_env *cle = cl_env_hops_obj(hn);
-
-	LASSERT(cle->ce_owner);
-	return (key == cle->ce_owner);
-}
-
-static void cl_env_hops_noop(struct cfs_hash *hs, struct hlist_node *hn)
-{
-	struct cl_env *cle = hlist_entry(hn, struct cl_env, ce_node);
-
-	LASSERT(cle->ce_magic == &cl_env_init0);
-}
-
-static struct cfs_hash_ops cl_env_hops = {
-	.hs_hash	= cl_env_hops_hash,
-	.hs_key		= cl_env_hops_obj,
-	.hs_keycmp      = cl_env_hops_keycmp,
-	.hs_object      = cl_env_hops_obj,
-	.hs_get		= cl_env_hops_noop,
-	.hs_put_locked  = cl_env_hops_noop,
-};
-
-static inline struct cl_env *cl_env_fetch(void)
-{
-	struct cl_env *cle;
-
-	cle = cfs_hash_lookup(cl_env_hash, (void *)(long)current->pid);
-	LASSERT(ergo(cle, cle->ce_magic == &cl_env_init0));
-	return cle;
-}
-
-static inline void cl_env_attach(struct cl_env *cle)
-{
-	if (cle) {
-		int rc;
-
-		LASSERT(!cle->ce_owner);
-		cle->ce_owner = (void *)(long)current->pid;
-		rc = cfs_hash_add_unique(cl_env_hash, cle->ce_owner,
-					 &cle->ce_node);
-		LASSERT(rc == 0);
-	}
-}
-
-static inline void cl_env_do_detach(struct cl_env *cle)
-{
-	void *cookie;
-
-	LASSERT(cle->ce_owner == (void *)(long)current->pid);
-	cookie = cfs_hash_del(cl_env_hash, cle->ce_owner,
-			      &cle->ce_node);
-	LASSERT(cookie == cle);
-	cle->ce_owner = NULL;
-}
-
-static int cl_env_store_init(void)
-{
-	cl_env_hash = cfs_hash_create("cl_env",
-				      HASH_CL_ENV_BITS, HASH_CL_ENV_BITS,
-				      HASH_CL_ENV_BKT_BITS, 0,
-				      CFS_HASH_MIN_THETA,
-				      CFS_HASH_MAX_THETA,
-				      &cl_env_hops,
-				      CFS_HASH_RW_BKTLOCK);
-	return cl_env_hash ? 0 : -ENOMEM;
-}
-
-static void cl_env_store_fini(void)
-{
-	cfs_hash_putref(cl_env_hash);
-}
-
-static inline struct cl_env *cl_env_detach(struct cl_env *cle)
-{
-	if (!cle)
-		cle = cl_env_fetch();
-
-	if (cle && cle->ce_owner)
-		cl_env_do_detach(cle);
-
-	return cle;
-}
-
 static struct lu_env *cl_env_new(__u32 ctx_tags, __u32 ses_tags, void *debug)
 {
 	struct lu_env *env;
@@ -701,16 +637,20 @@ static struct lu_env *cl_env_obtain(void *debug)
 {
 	struct cl_env *cle;
 	struct lu_env *env;
+	int cpu = get_cpu();
 
-	spin_lock(&cl_envs_guard);
-	LASSERT(equi(cl_envs_cached_nr == 0, list_empty(&cl_envs)));
-	if (cl_envs_cached_nr > 0) {
+	read_lock(&cl_envs[cpu].cec_guard);
+	LASSERT(equi(cl_envs[cpu].cec_count == 0,
+		     list_empty(&cl_envs[cpu].cec_envs)));
+	if (cl_envs[cpu].cec_count > 0) {
 		int rc;
 
-		cle = container_of(cl_envs.next, struct cl_env, ce_linkage);
+		cle = container_of(cl_envs[cpu].cec_envs.next, struct cl_env,
+				   ce_linkage);
 		list_del_init(&cle->ce_linkage);
-		cl_envs_cached_nr--;
-		spin_unlock(&cl_envs_guard);
+		cl_envs[cpu].cec_count--;
+		read_unlock(&cl_envs[cpu].cec_guard);
+		put_cpu();
 
 		env = &cle->ce_lu;
 		rc = lu_env_refill(env);
@@ -723,7 +663,8 @@ static struct lu_env *cl_env_obtain(void *debug)
 			env = ERR_PTR(rc);
 		}
 	} else {
-		spin_unlock(&cl_envs_guard);
+		read_unlock(&cl_envs[cpu].cec_guard);
+		put_cpu();
 		env = cl_env_new(lu_context_tags_default,
 				 lu_session_tags_default, debug);
 	}
@@ -735,27 +676,6 @@ static inline struct cl_env *cl_env_container(struct lu_env *env)
 	return container_of(env, struct cl_env, ce_lu);
 }
 
-static struct lu_env *cl_env_peek(int *refcheck)
-{
-	struct lu_env *env;
-	struct cl_env *cle;
-
-	CL_ENV_INC(lookup);
-
-	/* check that we don't go far from untrusted pointer */
-	CLASSERT(offsetof(struct cl_env, ce_magic) == 0);
-
-	env = NULL;
-	cle = cl_env_fetch();
-	if (cle) {
-		CL_ENV_INC(hit);
-		env = &cle->ce_lu;
-		*refcheck = ++cle->ce_ref;
-	}
-	CDEBUG(D_OTHER, "%d@%p\n", cle ? cle->ce_ref : 0, cle);
-	return env;
-}
-
 /**
  * Returns lu_env: if there already is an environment associated with the
  * current thread, it is returned, otherwise, new environment is allocated.
@@ -773,17 +693,13 @@ struct lu_env *cl_env_get(int *refcheck)
 {
 	struct lu_env *env;
 
-	env = cl_env_peek(refcheck);
-	if (!env) {
-		env = cl_env_obtain(__builtin_return_address(0));
-		if (!IS_ERR(env)) {
-			struct cl_env *cle;
+	env = cl_env_obtain(__builtin_return_address(0));
+	if (!IS_ERR(env)) {
+		struct cl_env *cle;
 
-			cle = cl_env_container(env);
-			cl_env_attach(cle);
-			*refcheck = cle->ce_ref;
-			CDEBUG(D_OTHER, "%d@%p\n", cle->ce_ref, cle);
-		}
+		cle = cl_env_container(env);
+		*refcheck = cle->ce_ref;
+		CDEBUG(D_OTHER, "%d@%p\n", cle->ce_ref, cle);
 	}
 	return env;
 }
@@ -798,7 +714,6 @@ struct lu_env *cl_env_alloc(int *refcheck, __u32 tags)
 {
 	struct lu_env *env;
 
-	LASSERT(!cl_env_peek(refcheck));
 	env = cl_env_new(tags, tags, __builtin_return_address(0));
 	if (!IS_ERR(env)) {
 		struct cl_env *cle;
@@ -813,7 +728,6 @@ EXPORT_SYMBOL(cl_env_alloc);
 
 static void cl_env_exit(struct cl_env *cle)
 {
-	LASSERT(!cle->ce_owner);
 	lu_context_exit(&cle->ce_lu.le_ctx);
 	lu_context_exit(&cle->ce_ses);
 }
@@ -826,20 +740,25 @@ static void cl_env_exit(struct cl_env *cle)
 unsigned int cl_env_cache_purge(unsigned int nr)
 {
 	struct cl_env *cle;
+	unsigned int i;
 
-	spin_lock(&cl_envs_guard);
-	for (; !list_empty(&cl_envs) && nr > 0; --nr) {
-		cle = container_of(cl_envs.next, struct cl_env, ce_linkage);
-		list_del_init(&cle->ce_linkage);
-		LASSERT(cl_envs_cached_nr > 0);
-		cl_envs_cached_nr--;
-		spin_unlock(&cl_envs_guard);
+	for_each_possible_cpu(i) {
+		write_lock(&cl_envs[i].cec_guard);
+		for (; !list_empty(&cl_envs[i].cec_envs) && nr > 0; --nr) {
+			cle = container_of(cl_envs[i].cec_envs.next,
+					   struct cl_env, ce_linkage);
+			list_del_init(&cle->ce_linkage);
+			LASSERT(cl_envs[i].cec_count > 0);
+			cl_envs[i].cec_count--;
+			write_unlock(&cl_envs[i].cec_guard);
 
-		cl_env_fini(cle);
-		spin_lock(&cl_envs_guard);
+			cl_env_fini(cle);
+			write_lock(&cl_envs[i].cec_guard);
+		}
+		LASSERT(equi(cl_envs[i].cec_count == 0,
+			     list_empty(&cl_envs[i].cec_envs)));
+		write_unlock(&cl_envs[i].cec_guard);
 	}
-	LASSERT(equi(cl_envs_cached_nr == 0, list_empty(&cl_envs)));
-	spin_unlock(&cl_envs_guard);
 	return nr;
 }
 EXPORT_SYMBOL(cl_env_cache_purge);
@@ -862,8 +781,9 @@ void cl_env_put(struct lu_env *env, int *refcheck)
 
 	CDEBUG(D_OTHER, "%d@%p\n", cle->ce_ref, cle);
 	if (--cle->ce_ref == 0) {
+		int cpu = get_cpu();
+
 		CL_ENV_DEC(busy);
-		cl_env_detach(cle);
 		cle->ce_debug = NULL;
 		cl_env_exit(cle);
 		/*
@@ -872,107 +792,22 @@ void cl_env_put(struct lu_env *env, int *refcheck)
 		 * Return environment to the cache only when it was allocated
 		 * with the standard tags.
 		 */
-		if (cl_envs_cached_nr < cl_envs_cached_max &&
+		if (cl_envs[cpu].cec_count < cl_envs_cached_max &&
 		    (env->le_ctx.lc_tags & ~LCT_HAS_EXIT) == LCT_CL_THREAD &&
 		    (env->le_ses->lc_tags & ~LCT_HAS_EXIT) == LCT_SESSION) {
-			spin_lock(&cl_envs_guard);
-			list_add(&cle->ce_linkage, &cl_envs);
-			cl_envs_cached_nr++;
-			spin_unlock(&cl_envs_guard);
+			read_lock(&cl_envs[cpu].cec_guard);
+			list_add(&cle->ce_linkage, &cl_envs[cpu].cec_envs);
+			cl_envs[cpu].cec_count++;
+			read_unlock(&cl_envs[cpu].cec_guard);
 		} else {
 			cl_env_fini(cle);
 		}
+		put_cpu();
 	}
 }
 EXPORT_SYMBOL(cl_env_put);
 
 /**
- * Declares a point of re-entrancy.
- *
- * \see cl_env_reexit()
- */
-void *cl_env_reenter(void)
-{
-	return cl_env_detach(NULL);
-}
-EXPORT_SYMBOL(cl_env_reenter);
-
-/**
- * Exits re-entrancy.
- */
-void cl_env_reexit(void *cookie)
-{
-	cl_env_detach(NULL);
-	cl_env_attach(cookie);
-}
-EXPORT_SYMBOL(cl_env_reexit);
-
-/**
- * Setup user-supplied \a env as a current environment. This is to be used to
- * guaranteed that environment exists even when cl_env_get() fails. It is up
- * to user to ensure proper concurrency control.
- *
- * \see cl_env_unplant()
- */
-void cl_env_implant(struct lu_env *env, int *refcheck)
-{
-	struct cl_env *cle = cl_env_container(env);
-
-	LASSERT(cle->ce_ref > 0);
-
-	cl_env_attach(cle);
-	cl_env_get(refcheck);
-	CDEBUG(D_OTHER, "%d@%p\n", cle->ce_ref, cle);
-}
-EXPORT_SYMBOL(cl_env_implant);
-
-/**
- * Detach environment installed earlier by cl_env_implant().
- */
-void cl_env_unplant(struct lu_env *env, int *refcheck)
-{
-	struct cl_env *cle = cl_env_container(env);
-
-	LASSERT(cle->ce_ref > 1);
-
-	CDEBUG(D_OTHER, "%d@%p\n", cle->ce_ref, cle);
-
-	cl_env_detach(cle);
-	cl_env_put(env, refcheck);
-}
-EXPORT_SYMBOL(cl_env_unplant);
-
-struct lu_env *cl_env_nested_get(struct cl_env_nest *nest)
-{
-	struct lu_env *env;
-
-	nest->cen_cookie = NULL;
-	env = cl_env_peek(&nest->cen_refcheck);
-	if (env) {
-		if (!cl_io_is_going(env))
-			return env;
-		cl_env_put(env, &nest->cen_refcheck);
-		nest->cen_cookie = cl_env_reenter();
-	}
-	env = cl_env_get(&nest->cen_refcheck);
-	if (IS_ERR(env)) {
-		cl_env_reexit(nest->cen_cookie);
-		return env;
-	}
-
-	LASSERT(!cl_io_is_going(env));
-	return env;
-}
-EXPORT_SYMBOL(cl_env_nested_get);
-
-void cl_env_nested_put(struct cl_env_nest *nest, struct lu_env *env)
-{
-	cl_env_put(env, &nest->cen_refcheck);
-	cl_env_reexit(nest->cen_cookie);
-}
-EXPORT_SYMBOL(cl_env_nested_put);
-
-/**
  * Converts struct ost_lvb to struct cl_attr.
  *
  * \see cl_attr2lvb
@@ -999,6 +834,10 @@ static int cl_env_percpu_init(void)
 	for_each_possible_cpu(i) {
 		struct lu_env *env;
 
+		rwlock_init(&cl_envs[i].cec_guard);
+		INIT_LIST_HEAD(&cl_envs[i].cec_envs);
+		cl_envs[i].cec_count = 0;
+
 		cle = &cl_env_percpu[i];
 		env = &cle->ce_lu;
 
@@ -1066,7 +905,6 @@ void cl_env_percpu_put(struct lu_env *env)
 	LASSERT(cle->ce_ref == 0);
 
 	CL_ENV_DEC(busy);
-	cl_env_detach(cle);
 	cle->ce_debug = NULL;
 
 	put_cpu();
@@ -1080,7 +918,6 @@ struct lu_env *cl_env_percpu_get(void)
 	cle = &cl_env_percpu[get_cpu()];
 	cl_env_init0(cle, __builtin_return_address(0));
 
-	cl_env_attach(cle);
 	return &cle->ce_lu;
 }
 EXPORT_SYMBOL(cl_env_percpu_get);
@@ -1144,51 +981,19 @@ LU_KEY_INIT_FINI(cl0, struct cl_thread_info);
 static void *cl_key_init(const struct lu_context *ctx,
 			 struct lu_context_key *key)
 {
-	struct cl_thread_info *info;
-
-	info = cl0_key_init(ctx, key);
-	if (!IS_ERR(info)) {
-		size_t i;
-
-		for (i = 0; i < ARRAY_SIZE(info->clt_counters); ++i)
-			lu_ref_init(&info->clt_counters[i].ctc_locks_locked);
-	}
-	return info;
+	return cl0_key_init(ctx, key);
 }
 
 static void cl_key_fini(const struct lu_context *ctx,
 			struct lu_context_key *key, void *data)
 {
-	struct cl_thread_info *info;
-	size_t i;
-
-	info = data;
-	for (i = 0; i < ARRAY_SIZE(info->clt_counters); ++i)
-		lu_ref_fini(&info->clt_counters[i].ctc_locks_locked);
 	cl0_key_fini(ctx, key, data);
 }
 
-static void cl_key_exit(const struct lu_context *ctx,
-			struct lu_context_key *key, void *data)
-{
-	struct cl_thread_info *info = data;
-	size_t i;
-
-	for (i = 0; i < ARRAY_SIZE(info->clt_counters); ++i) {
-		LASSERT(info->clt_counters[i].ctc_nr_held == 0);
-		LASSERT(info->clt_counters[i].ctc_nr_used == 0);
-		LASSERT(info->clt_counters[i].ctc_nr_locks_acquired == 0);
-		LASSERT(info->clt_counters[i].ctc_nr_locks_locked == 0);
-		lu_ref_fini(&info->clt_counters[i].ctc_locks_locked);
-		lu_ref_init(&info->clt_counters[i].ctc_locks_locked);
-	}
-}
-
 static struct lu_context_key cl_key = {
 	.lct_tags = LCT_CL_THREAD,
 	.lct_init = cl_key_init,
 	.lct_fini = cl_key_fini,
-	.lct_exit = cl_key_exit
 };
 
 static struct lu_kmem_descr cl_object_caches[] = {
@@ -1212,13 +1017,15 @@ int cl_global_init(void)
 {
 	int result;
 
-	result = cl_env_store_init();
-	if (result)
-		return result;
+	cl_envs = kzalloc(sizeof(*cl_envs) * num_possible_cpus(), GFP_KERNEL);
+	if (!cl_envs) {
+		result = -ENOMEM;
+		goto out;
+	}
 
 	result = lu_kmem_init(cl_object_caches);
 	if (result)
-		goto out_store;
+		goto out_envs;
 
 	LU_CONTEXT_KEY_INIT(&cl_key);
 	result = lu_context_key_register(&cl_key);
@@ -1228,16 +1035,17 @@ int cl_global_init(void)
 	result = cl_env_percpu_init();
 	if (result)
 		/* no cl_env_percpu_fini on error */
-		goto out_context;
+		goto out_keys;
 
 	return 0;
 
-out_context:
+out_keys:
 	lu_context_key_degister(&cl_key);
 out_kmem:
 	lu_kmem_fini(cl_object_caches);
-out_store:
-	cl_env_store_fini();
+out_envs:
+	kfree(cl_envs);
+out:
 	return result;
 }
 
@@ -1249,5 +1057,5 @@ void cl_global_fini(void)
 	cl_env_percpu_fini();
 	lu_context_key_degister(&cl_key);
 	lu_kmem_fini(cl_object_caches);
-	cl_env_store_fini();
+	kfree(cl_envs);
 }
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_page.c b/drivers/staging/lustre/lustre/obdclass/cl_page.c
index 63973ba096da..cd9a40ca4448 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_page.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_page.c
@@ -99,7 +99,6 @@ static void cl_page_free(const struct lu_env *env, struct cl_page *page)
 
 	PASSERT(env, page, list_empty(&page->cp_batch));
 	PASSERT(env, page, !page->cp_owner);
-	PASSERT(env, page, !page->cp_req);
 	PASSERT(env, page, page->cp_state == CPS_FREEING);
 
 	while (!list_empty(&page->cp_layers)) {
@@ -150,7 +149,6 @@ struct cl_page *cl_page_alloc(const struct lu_env *env,
 		page->cp_type = type;
 		INIT_LIST_HEAD(&page->cp_layers);
 		INIT_LIST_HEAD(&page->cp_batch);
-		INIT_LIST_HEAD(&page->cp_flight);
 		lu_ref_init(&page->cp_reference);
 		head = o->co_lu.lo_header;
 		list_for_each_entry(o, &head->loh_layers, co_lu.lo_linkage) {
@@ -390,30 +388,6 @@ EXPORT_SYMBOL(cl_page_at);
 	__result;						       \
 })
 
-#define CL_PAGE_INVOKE_REVERSE(_env, _page, _op, _proto, ...)		\
-({									\
-	const struct lu_env        *__env  = (_env);			\
-	struct cl_page             *__page = (_page);			\
-	const struct cl_page_slice *__scan;				\
-	int                         __result;				\
-	ptrdiff_t                   __op   = (_op);			\
-	int                       (*__method)_proto;			\
-									\
-	__result = 0;							\
-	list_for_each_entry_reverse(__scan, &__page->cp_layers,		\
-					cpl_linkage) {			\
-		__method = *(void **)((char *)__scan->cpl_ops +  __op);	\
-		if (__method) {						\
-			__result = (*__method)(__env, __scan, ## __VA_ARGS__); \
-			if (__result != 0)				\
-				break;					\
-		}							\
-	}								\
-	if (__result > 0)						\
-		__result = 0;						\
-	__result;							\
-})
-
 #define CL_PAGE_INVOID(_env, _page, _op, _proto, ...)		   \
 do {								    \
 	const struct lu_env	*__env  = (_env);		    \
@@ -552,7 +526,6 @@ static int cl_page_own0(const struct lu_env *env, struct cl_io *io,
 					io, nonblock);
 		if (result == 0) {
 			PASSERT(env, pg, !pg->cp_owner);
-			PASSERT(env, pg, !pg->cp_req);
 			pg->cp_owner = cl_io_top(io);
 			cl_page_owner_set(pg);
 			if (pg->cp_state != CPS_FREEING) {
@@ -694,7 +667,7 @@ static void cl_page_delete0(const struct lu_env *env, struct cl_page *pg)
 	PASSERT(env, pg, pg->cp_state != CPS_FREEING);
 
 	/*
-	 * Severe all ways to obtain new pointers to @pg.
+	 * Sever all ways to obtain new pointers to @pg.
 	 */
 	cl_page_owner_clear(pg);
 
@@ -845,8 +818,6 @@ void cl_page_completion(const struct lu_env *env,
 	struct cl_sync_io *anchor = pg->cp_sync_io;
 
 	PASSERT(env, pg, crt < CRT_NR);
-	/* cl_page::cp_req already cleared by the caller (osc_completion()) */
-	PASSERT(env, pg, !pg->cp_req);
 	PASSERT(env, pg, pg->cp_state == cl_req_type_state(crt));
 
 	CL_PAGE_HEADER(D_TRACE, env, pg, "%d %d\n", crt, ioret);
@@ -860,16 +831,8 @@ void cl_page_completion(const struct lu_env *env,
 	if (anchor) {
 		LASSERT(pg->cp_sync_io == anchor);
 		pg->cp_sync_io = NULL;
-	}
-	/*
-	 * As page->cp_obj is pinned by a reference from page->cp_req, it is
-	 * safe to call cl_page_put() without risking object destruction in a
-	 * non-blocking context.
-	 */
-	cl_page_put(env, pg);
-
-	if (anchor)
 		cl_sync_io_note(env, anchor, ioret);
+	}
 }
 EXPORT_SYMBOL(cl_page_completion);
 
@@ -927,29 +890,6 @@ int cl_page_flush(const struct lu_env *env, struct cl_io *io,
 EXPORT_SYMBOL(cl_page_flush);
 
 /**
- * Checks whether page is protected by any extent lock is at least required
- * mode.
- *
- * \return the same as in cl_page_operations::cpo_is_under_lock() method.
- * \see cl_page_operations::cpo_is_under_lock()
- */
-int cl_page_is_under_lock(const struct lu_env *env, struct cl_io *io,
-			  struct cl_page *page, pgoff_t *max_index)
-{
-	int rc;
-
-	PINVRNT(env, page, cl_page_invariant(page));
-
-	rc = CL_PAGE_INVOKE_REVERSE(env, page, CL_PAGE_OP(cpo_is_under_lock),
-				    (const struct lu_env *,
-				     const struct cl_page_slice *,
-				      struct cl_io *, pgoff_t *),
-				    io, max_index);
-	return rc;
-}
-EXPORT_SYMBOL(cl_page_is_under_lock);
-
-/**
  * Tells transfer engine that only part of a page is to be transmitted.
  *
  * \see cl_page_operations::cpo_clip()
@@ -974,10 +914,10 @@ void cl_page_header_print(const struct lu_env *env, void *cookie,
 			  lu_printer_t printer, const struct cl_page *pg)
 {
 	(*printer)(env, cookie,
-		   "page@%p[%d %p %d %d %p %p]\n",
+		   "page@%p[%d %p %d %d %p]\n",
 		   pg, atomic_read(&pg->cp_ref), pg->cp_obj,
 		   pg->cp_state, pg->cp_type,
-		   pg->cp_owner, pg->cp_req);
+		   pg->cp_owner);
 }
 EXPORT_SYMBOL(cl_page_header_print);
 
diff --git a/drivers/staging/lustre/lustre/obdclass/genops.c b/drivers/staging/lustre/lustre/obdclass/genops.c
index cf8bb2a2f40b..fa0d38ddccb2 100644
--- a/drivers/staging/lustre/lustre/obdclass/genops.c
+++ b/drivers/staging/lustre/lustre/obdclass/genops.c
@@ -907,6 +907,8 @@ struct obd_import *class_new_import(struct obd_device *obd)
 	INIT_LIST_HEAD(&imp->imp_sending_list);
 	INIT_LIST_HEAD(&imp->imp_delayed_list);
 	INIT_LIST_HEAD(&imp->imp_committed_list);
+	INIT_LIST_HEAD(&imp->imp_unreplied_list);
+	imp->imp_known_replied_xid = 0;
 	imp->imp_replay_cursor = &imp->imp_committed_list;
 	spin_lock_init(&imp->imp_lock);
 	imp->imp_last_success_conn = 0;
@@ -1408,13 +1410,33 @@ EXPORT_SYMBOL(obd_get_max_rpcs_in_flight);
 int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max)
 {
 	struct obd_request_slot_waiter *orsw;
+	const char *typ_name;
 	__u32 old;
 	int diff;
+	int rc;
 	int i;
 
 	if (max > OBD_MAX_RIF_MAX || max < 1)
 		return -ERANGE;
 
+	typ_name = cli->cl_import->imp_obd->obd_type->typ_name;
+	if (!strcmp(typ_name, LUSTRE_MDC_NAME)) {
+		/*
+		 * adjust max_mod_rpcs_in_flight to ensure it is always
+		 * strictly lower that max_rpcs_in_flight
+		 */
+		if (max < 2) {
+			CERROR("%s: cannot set max_rpcs_in_flight to 1 because it must be higher than max_mod_rpcs_in_flight value\n",
+			       cli->cl_import->imp_obd->obd_name);
+			return -ERANGE;
+		}
+		if (max <= cli->cl_max_mod_rpcs_in_flight) {
+			rc = obd_set_max_mod_rpcs_in_flight(cli, max - 1);
+			if (rc)
+				return rc;
+		}
+	}
+
 	spin_lock(&cli->cl_loi_list_lock);
 	old = cli->cl_max_rpcs_in_flight;
 	cli->cl_max_rpcs_in_flight = max;
@@ -1436,3 +1458,209 @@ int obd_set_max_rpcs_in_flight(struct client_obd *cli, __u32 max)
 	return 0;
 }
 EXPORT_SYMBOL(obd_set_max_rpcs_in_flight);
+
+int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, __u16 max)
+{
+	struct obd_connect_data *ocd;
+	u16 maxmodrpcs;
+	u16 prev;
+
+	if (max > OBD_MAX_RIF_MAX || max < 1)
+		return -ERANGE;
+
+	/* cannot exceed or equal max_rpcs_in_flight */
+	if (max >= cli->cl_max_rpcs_in_flight) {
+		CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) higher or equal to max_rpcs_in_flight value (%u)\n",
+		       cli->cl_import->imp_obd->obd_name,
+		       max, cli->cl_max_rpcs_in_flight);
+		return -ERANGE;
+	}
+
+	/* cannot exceed max modify RPCs in flight supported by the server */
+	ocd = &cli->cl_import->imp_connect_data;
+	if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS)
+		maxmodrpcs = ocd->ocd_maxmodrpcs;
+	else
+		maxmodrpcs = 1;
+	if (max > maxmodrpcs) {
+		CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) higher than max_mod_rpcs_per_client value (%hu) returned by the server at connection\n",
+		       cli->cl_import->imp_obd->obd_name,
+		       max, maxmodrpcs);
+		return -ERANGE;
+	}
+
+	spin_lock(&cli->cl_mod_rpcs_lock);
+
+	prev = cli->cl_max_mod_rpcs_in_flight;
+	cli->cl_max_mod_rpcs_in_flight = max;
+
+	/* wakeup waiters if limit has been increased */
+	if (cli->cl_max_mod_rpcs_in_flight > prev)
+		wake_up(&cli->cl_mod_rpcs_waitq);
+
+	spin_unlock(&cli->cl_mod_rpcs_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL(obd_set_max_mod_rpcs_in_flight);
+
+#define pct(a, b) (b ? (a * 100) / b : 0)
+
+int obd_mod_rpc_stats_seq_show(struct client_obd *cli, struct seq_file *seq)
+{
+	unsigned long mod_tot = 0, mod_cum;
+	struct timespec64 now;
+	int i;
+
+	ktime_get_real_ts64(&now);
+
+	spin_lock(&cli->cl_mod_rpcs_lock);
+
+	seq_printf(seq, "snapshot_time:		%llu.%9lu (secs.nsecs)\n",
+		   (s64)now.tv_sec, (unsigned long)now.tv_nsec);
+	seq_printf(seq, "modify_RPCs_in_flight:  %hu\n",
+		   cli->cl_mod_rpcs_in_flight);
+
+	seq_puts(seq, "\n\t\t\tmodify\n");
+	seq_puts(seq, "rpcs in flight        rpcs   %% cum %%\n");
+
+	mod_tot = lprocfs_oh_sum(&cli->cl_mod_rpcs_hist);
+
+	mod_cum = 0;
+	for (i = 0; i < OBD_HIST_MAX; i++) {
+		unsigned long mod = cli->cl_mod_rpcs_hist.oh_buckets[i];
+
+		mod_cum += mod;
+		seq_printf(seq, "%d:\t\t%10lu %3lu %3lu\n",
+			   i, mod, pct(mod, mod_tot),
+			   pct(mod_cum, mod_tot));
+		if (mod_cum == mod_tot)
+			break;
+	}
+
+	spin_unlock(&cli->cl_mod_rpcs_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL(obd_mod_rpc_stats_seq_show);
+#undef pct
+
+/*
+ * The number of modify RPCs sent in parallel is limited
+ * because the server has a finite number of slots per client to
+ * store request result and ensure reply reconstruction when needed.
+ * On the client, this limit is stored in cl_max_mod_rpcs_in_flight
+ * that takes into account server limit and cl_max_rpcs_in_flight
+ * value.
+ * On the MDC client, to avoid a potential deadlock (see Bugzilla 3462),
+ * one close request is allowed above the maximum.
+ */
+static inline bool obd_mod_rpc_slot_avail_locked(struct client_obd *cli,
+						 bool close_req)
+{
+	bool avail;
+
+	/* A slot is available if
+	 * - number of modify RPCs in flight is less than the max
+	 * - it's a close RPC and no other close request is in flight
+	 */
+	avail = cli->cl_mod_rpcs_in_flight < cli->cl_max_mod_rpcs_in_flight ||
+		(close_req && !cli->cl_close_rpcs_in_flight);
+
+	return avail;
+}
+
+static inline bool obd_mod_rpc_slot_avail(struct client_obd *cli,
+					  bool close_req)
+{
+	bool avail;
+
+	spin_lock(&cli->cl_mod_rpcs_lock);
+	avail = obd_mod_rpc_slot_avail_locked(cli, close_req);
+	spin_unlock(&cli->cl_mod_rpcs_lock);
+	return avail;
+}
+
+/* Get a modify RPC slot from the obd client @cli according
+ * to the kind of operation @opc that is going to be sent
+ * and the intent @it of the operation if it applies.
+ * If the maximum number of modify RPCs in flight is reached
+ * the thread is put to sleep.
+ * Returns the tag to be set in the request message. Tag 0
+ * is reserved for non-modifying requests.
+ */
+u16 obd_get_mod_rpc_slot(struct client_obd *cli, __u32 opc,
+			 struct lookup_intent *it)
+{
+	struct l_wait_info lwi = LWI_INTR(NULL, NULL);
+	bool close_req = false;
+	u16 i, max;
+
+	/* read-only metadata RPCs don't consume a slot on MDT
+	 * for reply reconstruction
+	 */
+	if (it && (it->it_op == IT_GETATTR || it->it_op == IT_LOOKUP ||
+		   it->it_op == IT_LAYOUT || it->it_op == IT_READDIR))
+		return 0;
+
+	if (opc == MDS_CLOSE)
+		close_req = true;
+
+	do {
+		spin_lock(&cli->cl_mod_rpcs_lock);
+		max = cli->cl_max_mod_rpcs_in_flight;
+		if (obd_mod_rpc_slot_avail_locked(cli, close_req)) {
+			/* there is a slot available */
+			cli->cl_mod_rpcs_in_flight++;
+			if (close_req)
+				cli->cl_close_rpcs_in_flight++;
+			lprocfs_oh_tally(&cli->cl_mod_rpcs_hist,
+					 cli->cl_mod_rpcs_in_flight);
+			/* find a free tag */
+			i = find_first_zero_bit(cli->cl_mod_tag_bitmap,
+						max + 1);
+			LASSERT(i < OBD_MAX_RIF_MAX);
+			LASSERT(!test_and_set_bit(i, cli->cl_mod_tag_bitmap));
+			spin_unlock(&cli->cl_mod_rpcs_lock);
+			/* tag 0 is reserved for non-modify RPCs */
+			return i + 1;
+		}
+		spin_unlock(&cli->cl_mod_rpcs_lock);
+
+		CDEBUG(D_RPCTRACE, "%s: sleeping for a modify RPC slot opc %u, max %hu\n",
+		       cli->cl_import->imp_obd->obd_name, opc, max);
+
+		l_wait_event(cli->cl_mod_rpcs_waitq,
+			     obd_mod_rpc_slot_avail(cli, close_req), &lwi);
+	} while (true);
+}
+EXPORT_SYMBOL(obd_get_mod_rpc_slot);
+
+/*
+ * Put a modify RPC slot from the obd client @cli according
+ * to the kind of operation @opc that has been sent and the
+ * intent @it of the operation if it applies.
+ */
+void obd_put_mod_rpc_slot(struct client_obd *cli, u32 opc,
+			  struct lookup_intent *it, u16 tag)
+{
+	bool close_req = false;
+
+	if (it && (it->it_op == IT_GETATTR || it->it_op == IT_LOOKUP ||
+		   it->it_op == IT_LAYOUT || it->it_op == IT_READDIR))
+		return;
+
+	if (opc == MDS_CLOSE)
+		close_req = true;
+
+	spin_lock(&cli->cl_mod_rpcs_lock);
+	cli->cl_mod_rpcs_in_flight--;
+	if (close_req)
+		cli->cl_close_rpcs_in_flight--;
+	/* release the tag in the bitmap */
+	LASSERT(tag - 1 < OBD_MAX_RIF_MAX);
+	LASSERT(test_and_clear_bit(tag - 1, cli->cl_mod_tag_bitmap) != 0);
+	spin_unlock(&cli->cl_mod_rpcs_lock);
+	wake_up(&cli->cl_mod_rpcs_waitq);
+}
+EXPORT_SYMBOL(obd_put_mod_rpc_slot);
diff --git a/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c b/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c
index be09e04b042f..9f5e8299d7e4 100644
--- a/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c
+++ b/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c
@@ -217,8 +217,8 @@ static ssize_t pinger_show(struct kobject *kobj, struct attribute *attr,
 	return sprintf(buf, "%s\n", "on");
 }
 
-static ssize_t health_show(struct kobject *kobj, struct attribute *attr,
-			   char *buf)
+static ssize_t
+health_check_show(struct kobject *kobj, struct attribute *attr, char *buf)
 {
 	bool healthy = true;
 	int i;
@@ -311,14 +311,14 @@ EXPORT_SYMBOL_GPL(debugfs_lustre_root);
 
 LUSTRE_RO_ATTR(version);
 LUSTRE_RO_ATTR(pinger);
-LUSTRE_RO_ATTR(health);
+LUSTRE_RO_ATTR(health_check);
 LUSTRE_RW_ATTR(jobid_var);
 LUSTRE_RW_ATTR(jobid_name);
 
 static struct attribute *lustre_attrs[] = {
 	&lustre_attr_version.attr,
 	&lustre_attr_pinger.attr,
-	&lustre_attr_health.attr,
+	&lustre_attr_health_check.attr,
 	&lustre_attr_jobid_name.attr,
 	&lustre_attr_jobid_var.attr,
 	NULL,
diff --git a/drivers/staging/lustre/lustre/obdclass/linux/linux-obdo.c b/drivers/staging/lustre/lustre/obdclass/linux/linux-obdo.c
deleted file mode 100644
index 41b77a30feb3..000000000000
--- a/drivers/staging/lustre/lustre/obdclass/linux/linux-obdo.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- *
- * Copyright (c) 2011, 2012, Intel Corporation.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- *
- * lustre/obdclass/linux/linux-obdo.c
- *
- * Object Devices Class Driver
- * These are the only exported functions, they provide some generic
- * infrastructure for managing object devices
- */
-
-#define DEBUG_SUBSYSTEM S_CLASS
-
-#include <linux/module.h>
-#include "../../include/obd_class.h"
-#include "../../include/lustre/lustre_idl.h"
-
-#include <linux/fs.h>
-
-void obdo_refresh_inode(struct inode *dst, const struct obdo *src, u32 valid)
-{
-	valid &= src->o_valid;
-
-	if (valid & (OBD_MD_FLCTIME | OBD_MD_FLMTIME))
-		CDEBUG(D_INODE,
-		       "valid %#llx, cur time %lu/%lu, new %llu/%llu\n",
-		       src->o_valid, LTIME_S(dst->i_mtime),
-		       LTIME_S(dst->i_ctime), src->o_mtime, src->o_ctime);
-
-	if (valid & OBD_MD_FLATIME && src->o_atime > LTIME_S(dst->i_atime))
-		LTIME_S(dst->i_atime) = src->o_atime;
-	if (valid & OBD_MD_FLMTIME && src->o_mtime > LTIME_S(dst->i_mtime))
-		LTIME_S(dst->i_mtime) = src->o_mtime;
-	if (valid & OBD_MD_FLCTIME && src->o_ctime > LTIME_S(dst->i_ctime))
-		LTIME_S(dst->i_ctime) = src->o_ctime;
-	if (valid & OBD_MD_FLSIZE)
-		i_size_write(dst, src->o_size);
-	/* optimum IO size */
-	if (valid & OBD_MD_FLBLKSZ && src->o_blksize > (1 << dst->i_blkbits))
-		dst->i_blkbits = ffs(src->o_blksize) - 1;
-
-	if (dst->i_blkbits < PAGE_SHIFT)
-		dst->i_blkbits = PAGE_SHIFT;
-
-	/* allocation of space */
-	if (valid & OBD_MD_FLBLOCKS && src->o_blocks > dst->i_blocks)
-		/*
-		 * XXX shouldn't overflow be checked here like in
-		 * obdo_to_inode().
-		 */
-		dst->i_blocks = src->o_blocks;
-}
-EXPORT_SYMBOL(obdo_refresh_inode);
diff --git a/drivers/staging/lustre/lustre/obdclass/llog.c b/drivers/staging/lustre/lustre/obdclass/llog.c
index 43797f106745..736ea1067c93 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog.c
+++ b/drivers/staging/lustre/lustre/obdclass/llog.c
@@ -43,8 +43,9 @@
 
 #define DEBUG_SUBSYSTEM S_LOG
 
-#include "../include/obd_class.h"
+#include "../include/llog_swab.h"
 #include "../include/lustre_log.h"
+#include "../include/obd_class.h"
 #include "llog_internal.h"
 
 /*
@@ -80,8 +81,7 @@ static void llog_free_handle(struct llog_handle *loghandle)
 		LASSERT(list_empty(&loghandle->u.phd.phd_entry));
 	else if (loghandle->lgh_hdr->llh_flags & LLOG_F_IS_CAT)
 		LASSERT(list_empty(&loghandle->u.chd.chd_head));
-	LASSERT(sizeof(*loghandle->lgh_hdr) == LLOG_CHUNK_SIZE);
-	kfree(loghandle->lgh_hdr);
+	kvfree(loghandle->lgh_hdr);
 out:
 	kfree(loghandle);
 }
@@ -115,20 +115,29 @@ static int llog_read_header(const struct lu_env *env,
 	rc = lop->lop_read_header(env, handle);
 	if (rc == LLOG_EEMPTY) {
 		struct llog_log_hdr *llh = handle->lgh_hdr;
+		size_t len;
 
+		/* lrh_len should be initialized in llog_init_handle */
 		handle->lgh_last_idx = 0; /* header is record with index 0 */
 		llh->llh_count = 1;	 /* for the header record */
 		llh->llh_hdr.lrh_type = LLOG_HDR_MAGIC;
-		llh->llh_hdr.lrh_len = LLOG_CHUNK_SIZE;
-		llh->llh_tail.lrt_len = LLOG_CHUNK_SIZE;
+		LASSERT(handle->lgh_ctxt->loc_chunk_size >= LLOG_MIN_CHUNK_SIZE);
+		llh->llh_hdr.lrh_len = handle->lgh_ctxt->loc_chunk_size;
 		llh->llh_hdr.lrh_index = 0;
-		llh->llh_tail.lrt_index = 0;
 		llh->llh_timestamp = ktime_get_real_seconds();
 		if (uuid)
 			memcpy(&llh->llh_tgtuuid, uuid,
 			       sizeof(llh->llh_tgtuuid));
 		llh->llh_bitmap_offset = offsetof(typeof(*llh), llh_bitmap);
-		ext2_set_bit(0, llh->llh_bitmap);
+		/*
+		 * Since update llog header might also call this function,
+		 * let's reset the bitmap to 0 here
+		 */
+		len = llh->llh_hdr.lrh_len - llh->llh_bitmap_offset;
+		memset(LLOG_HDR_BITMAP(llh), 0, len - sizeof(llh->llh_tail));
+		ext2_set_bit(0, LLOG_HDR_BITMAP(llh));
+		LLOG_HDR_TAIL(llh)->lrt_len = llh->llh_hdr.lrh_len;
+		LLOG_HDR_TAIL(llh)->lrt_index = llh->llh_hdr.lrh_index;
 		rc = 0;
 	}
 	return rc;
@@ -137,16 +146,19 @@ static int llog_read_header(const struct lu_env *env,
 int llog_init_handle(const struct lu_env *env, struct llog_handle *handle,
 		     int flags, struct obd_uuid *uuid)
 {
+	int chunk_size = handle->lgh_ctxt->loc_chunk_size;
 	enum llog_flag fmt = flags & LLOG_F_EXT_MASK;
 	struct llog_log_hdr	*llh;
 	int			 rc;
 
 	LASSERT(!handle->lgh_hdr);
 
-	llh = kzalloc(sizeof(*llh), GFP_NOFS);
+	LASSERT(chunk_size >= LLOG_MIN_CHUNK_SIZE);
+	llh = libcfs_kvzalloc(sizeof(*llh), GFP_NOFS);
 	if (!llh)
 		return -ENOMEM;
 	handle->lgh_hdr = llh;
+	handle->lgh_hdr_size = chunk_size;
 	/* first assign flags to use llog_client_ops */
 	llh->llh_flags = flags;
 	rc = llog_read_header(env, handle, uuid);
@@ -189,6 +201,7 @@ int llog_init_handle(const struct lu_env *env, struct llog_handle *handle,
 		LASSERT(list_empty(&handle->u.chd.chd_head));
 		INIT_LIST_HEAD(&handle->u.chd.chd_head);
 		llh->llh_size = sizeof(struct llog_logid_rec);
+		llh->llh_flags |= LLOG_F_IS_FIXSIZE;
 	} else if (!(flags & LLOG_F_IS_PLAIN)) {
 		CERROR("%s: unknown flags: %#x (expected %#x or %#x)\n",
 		       handle->lgh_ctxt->loc_obd->obd_name,
@@ -198,7 +211,7 @@ int llog_init_handle(const struct lu_env *env, struct llog_handle *handle,
 	llh->llh_flags |= fmt;
 out:
 	if (rc) {
-		kfree(llh);
+		kvfree(llh);
 		handle->lgh_hdr = NULL;
 	}
 	return rc;
@@ -212,15 +225,21 @@ static int llog_process_thread(void *arg)
 	struct llog_log_hdr		*llh = loghandle->lgh_hdr;
 	struct llog_process_cat_data	*cd  = lpi->lpi_catdata;
 	char				*buf;
-	__u64				 cur_offset = LLOG_CHUNK_SIZE;
-	__u64				 last_offset;
+	u64 cur_offset, tmp_offset;
+	int chunk_size;
 	int				 rc = 0, index = 1, last_index;
 	int				 saved_index = 0;
 	int				 last_called_index = 0;
 
-	LASSERT(llh);
+	if (!llh)
+		return -EINVAL;
+
+	cur_offset = llh->llh_hdr.lrh_len;
+	chunk_size = llh->llh_hdr.lrh_len;
+	/* expect chunk_size to be power of two */
+	LASSERT(is_power_of_2(chunk_size));
 
-	buf = kzalloc(LLOG_CHUNK_SIZE, GFP_NOFS);
+	buf = libcfs_kvzalloc(chunk_size, GFP_NOFS);
 	if (!buf) {
 		lpi->lpi_rc = -ENOMEM;
 		return 0;
@@ -233,41 +252,53 @@ static int llog_process_thread(void *arg)
 	if (cd && cd->lpcd_last_idx)
 		last_index = cd->lpcd_last_idx;
 	else
-		last_index = LLOG_BITMAP_BYTES * 8 - 1;
-
-	/* Record is not in this buffer. */
-	if (index > last_index)
-		goto out;
+		last_index = LLOG_HDR_BITMAP_SIZE(llh) - 1;
 
 	while (rc == 0) {
+		unsigned int buf_offset = 0;
 		struct llog_rec_hdr *rec;
+		bool partial_chunk;
+		off_t chunk_offset;
 
 		/* skip records not set in bitmap */
 		while (index <= last_index &&
-		       !ext2_test_bit(index, llh->llh_bitmap))
+		       !ext2_test_bit(index, LLOG_HDR_BITMAP(llh)))
 			++index;
 
-		LASSERT(index <= last_index + 1);
-		if (index == last_index + 1)
+		if (index > last_index)
 			break;
-repeat:
+
 		CDEBUG(D_OTHER, "index: %d last_index %d\n",
 		       index, last_index);
-
+repeat:
 		/* get the buf with our target record; avoid old garbage */
-		memset(buf, 0, LLOG_CHUNK_SIZE);
-		last_offset = cur_offset;
+		memset(buf, 0, chunk_size);
 		rc = llog_next_block(lpi->lpi_env, loghandle, &saved_index,
-				     index, &cur_offset, buf, LLOG_CHUNK_SIZE);
+				     index, &cur_offset, buf, chunk_size);
 		if (rc)
 			goto out;
 
+		/*
+		 * NB: after llog_next_block() call the cur_offset is the
+		 * offset of the next block after read one.
+		 * The absolute offset of the current chunk is calculated
+		 * from cur_offset value and stored in chunk_offset variable.
+		 */
+		tmp_offset = cur_offset;
+		if (do_div(tmp_offset, chunk_size)) {
+			partial_chunk = true;
+			chunk_offset = cur_offset & ~(chunk_size - 1);
+		} else {
+			partial_chunk = false;
+			chunk_offset = cur_offset - chunk_size;
+		}
+
 		/* NB: when rec->lrh_len is accessed it is already swabbed
 		 * since it is used at the "end" of the loop and the rec
 		 * swabbing is done at the beginning of the loop.
 		 */
-		for (rec = (struct llog_rec_hdr *)buf;
-		     (char *)rec < buf + LLOG_CHUNK_SIZE;
+		for (rec = (struct llog_rec_hdr *)(buf + buf_offset);
+		     (char *)rec < buf + chunk_size;
 		     rec = llog_rec_hdr_next(rec)) {
 			CDEBUG(D_OTHER, "processing rec 0x%p type %#x\n",
 			       rec, rec->lrh_type);
@@ -278,15 +309,29 @@ repeat:
 			CDEBUG(D_OTHER, "after swabbing, type=%#x idx=%d\n",
 			       rec->lrh_type, rec->lrh_index);
 
-			if (rec->lrh_index == 0) {
-				/* probably another rec just got added? */
-				rc = 0;
-				if (index <= loghandle->lgh_last_idx)
-					goto repeat;
-				goto out; /* no more records */
+			/*
+			 * for partial chunk the end of it is zeroed, check
+			 * for index 0 to distinguish it.
+			 */
+			if (partial_chunk && !rec->lrh_index) {
+				/* concurrent llog_add() might add new records
+				 * while llog_processing, check this is not
+				 * the case and re-read the current chunk
+				 * otherwise.
+				 */
+				if (index > loghandle->lgh_last_idx) {
+					rc = 0;
+					goto out;
+				}
+				CDEBUG(D_OTHER, "Re-read last llog buffer for new records, index %u, last %u\n",
+				       index, loghandle->lgh_last_idx);
+				/* save offset inside buffer for the re-read */
+				buf_offset = (char *)rec - (char *)buf;
+				cur_offset = chunk_offset;
+				goto repeat;
 			}
-			if (rec->lrh_len == 0 ||
-			    rec->lrh_len > LLOG_CHUNK_SIZE) {
+
+			if (!rec->lrh_len || rec->lrh_len > chunk_size) {
 				CWARN("invalid length %d in llog record for index %d/%d\n",
 				      rec->lrh_len,
 				      rec->lrh_index, index);
@@ -300,32 +345,38 @@ repeat:
 				continue;
 			}
 
+			if (rec->lrh_index != index) {
+				CERROR("%s: Invalid record: index %u but expected %u\n",
+				       loghandle->lgh_ctxt->loc_obd->obd_name,
+				       rec->lrh_index, index);
+				rc = -ERANGE;
+				goto out;
+			}
+
 			CDEBUG(D_OTHER,
 			       "lrh_index: %d lrh_len: %d (%d remains)\n",
 			       rec->lrh_index, rec->lrh_len,
-			       (int)(buf + LLOG_CHUNK_SIZE - (char *)rec));
+			       (int)(buf + chunk_size - (char *)rec));
 
 			loghandle->lgh_cur_idx = rec->lrh_index;
 			loghandle->lgh_cur_offset = (char *)rec - (char *)buf +
-						    last_offset;
+						    chunk_offset;
 
 			/* if set, process the callback on this record */
-			if (ext2_test_bit(index, llh->llh_bitmap)) {
+			if (ext2_test_bit(index, LLOG_HDR_BITMAP(llh))) {
 				rc = lpi->lpi_cb(lpi->lpi_env, loghandle, rec,
 						 lpi->lpi_cbdata);
 				last_called_index = index;
 				if (rc)
 					goto out;
-			} else {
-				CDEBUG(D_OTHER, "Skipped index %d\n", index);
 			}
 
-			/* next record, still in buffer? */
-			++index;
-			if (index > last_index) {
+			/* exit if the last index is reached */
+			if (index >= last_index) {
 				rc = 0;
 				goto out;
 			}
+			index++;
 		}
 	}
 
diff --git a/drivers/staging/lustre/lustre/obdclass/llog_obd.c b/drivers/staging/lustre/lustre/obdclass/llog_obd.c
index a4277d684614..8574ad401f66 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog_obd.c
+++ b/drivers/staging/lustre/lustre/obdclass/llog_obd.c
@@ -158,6 +158,7 @@ int llog_setup(const struct lu_env *env, struct obd_device *obd,
 	mutex_init(&ctxt->loc_mutex);
 	ctxt->loc_exp = class_export_get(disk_obd->obd_self_export);
 	ctxt->loc_flags = LLOG_CTXT_FLAG_UNINITIALIZED;
+	ctxt->loc_chunk_size = LLOG_MIN_CHUNK_SIZE;
 
 	rc = llog_group_set_ctxt(olg, ctxt, index);
 	if (rc) {
diff --git a/drivers/staging/lustre/lustre/obdclass/llog_swab.c b/drivers/staging/lustre/lustre/obdclass/llog_swab.c
index 8c4c1b3f1b45..723c212c6747 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog_swab.c
+++ b/drivers/staging/lustre/lustre/obdclass/llog_swab.c
@@ -38,6 +38,7 @@
 
 #define DEBUG_SUBSYSTEM S_LOG
 
+#include "../include/llog_swab.h"
 #include "../include/lustre_log.h"
 
 static void print_llogd_body(struct llogd_body *d)
@@ -244,7 +245,7 @@ void lustre_swab_llog_rec(struct llog_rec_hdr *rec)
 		__swab32s(&llh->llh_flags);
 		__swab32s(&llh->llh_size);
 		__swab32s(&llh->llh_cat_idx);
-		tail = &llh->llh_tail;
+		tail = LLOG_HDR_TAIL(llh);
 		break;
 	}
 	case LLOG_LOGID_MAGIC:
@@ -290,8 +291,10 @@ static void print_llog_hdr(struct llog_log_hdr *h)
 	CDEBUG(D_OTHER, "\tllh_flags: %#x\n", h->llh_flags);
 	CDEBUG(D_OTHER, "\tllh_size: %#x\n", h->llh_size);
 	CDEBUG(D_OTHER, "\tllh_cat_idx: %#x\n", h->llh_cat_idx);
-	CDEBUG(D_OTHER, "\tllh_tail.lrt_index: %#x\n", h->llh_tail.lrt_index);
-	CDEBUG(D_OTHER, "\tllh_tail.lrt_len: %#x\n", h->llh_tail.lrt_len);
+	CDEBUG(D_OTHER, "\tllh_tail.lrt_index: %#x\n",
+	       LLOG_HDR_TAIL(h)->lrt_index);
+	CDEBUG(D_OTHER, "\tllh_tail.lrt_len: %#x\n",
+	       LLOG_HDR_TAIL(h)->lrt_len);
 }
 
 void lustre_swab_llog_hdr(struct llog_log_hdr *h)
diff --git a/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c b/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c
index 852a5acfefab..2c99717b0aba 100644
--- a/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c
+++ b/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c
@@ -100,9 +100,13 @@ static const char * const obd_connect_names[] = {
 	"lfsck",
 	"unknown",
 	"unlink_close",
-	"unknown",
+	"multi_mod_rpcs",
 	"dir_stripe",
-	"unknown",
+	"subtree",
+	"lock_ahead",
+	"bulk_mbits",
+	"compact_obdo",
+	"second_flags",
 	NULL
 };
 
@@ -127,7 +131,7 @@ EXPORT_SYMBOL(obd_connect_flags2str);
 static void obd_connect_data_seqprint(struct seq_file *m,
 				      struct obd_connect_data *ocd)
 {
-	int flags;
+	u64 flags;
 
 	LASSERT(ocd);
 	flags = ocd->ocd_connect_flags;
@@ -172,6 +176,9 @@ static void obd_connect_data_seqprint(struct seq_file *m,
 	if (flags & OBD_CONNECT_MAXBYTES)
 		seq_printf(m, "       max_object_bytes: %llx\n",
 			   ocd->ocd_maxbytes);
+	if (flags & OBD_CONNECT_MULTIMODRPCS)
+		seq_printf(m, "       max_mod_rpcs: %hu\n",
+			   ocd->ocd_maxmodrpcs);
 }
 
 int lprocfs_read_frac_helper(char *buffer, unsigned long count, long val,
@@ -396,10 +403,17 @@ int lprocfs_wr_uint(struct file *file, const char __user *buffer,
 	char dummy[MAX_STRING_SIZE + 1], *end;
 	unsigned long tmp;
 
-	dummy[MAX_STRING_SIZE] = '\0';
-	if (copy_from_user(dummy, buffer, MAX_STRING_SIZE))
+	if (count >= sizeof(dummy))
+		return -EINVAL;
+
+	if (count == 0)
+		return 0;
+
+	if (copy_from_user(dummy, buffer, count))
 		return -EFAULT;
 
+	dummy[count] = '\0';
+
 	tmp = simple_strtoul(dummy, &end, 0);
 	if (dummy == end)
 		return -EINVAL;
@@ -1275,7 +1289,8 @@ int ldebugfs_register_stats(struct dentry *parent, const char *name,
 EXPORT_SYMBOL_GPL(ldebugfs_register_stats);
 
 void lprocfs_counter_init(struct lprocfs_stats *stats, int index,
-			  unsigned conf, const char *name, const char *units)
+			  unsigned int conf, const char *name,
+			  const char *units)
 {
 	struct lprocfs_counter_header	*header;
 	struct lprocfs_counter		*percpu_cntr;
diff --git a/drivers/staging/lustre/lustre/obdclass/lu_object.c b/drivers/staging/lustre/lustre/obdclass/lu_object.c
index 054e567e6c8d..7971562a3efd 100644
--- a/drivers/staging/lustre/lustre/obdclass/lu_object.c
+++ b/drivers/staging/lustre/lustre/obdclass/lu_object.c
@@ -68,6 +68,7 @@ enum {
 
 #define LU_SITE_BITS_MIN	12
 #define LU_SITE_BITS_MAX	24
+#define LU_SITE_BITS_MAX_CL	19
 /**
  * total 256 buckets, we don't want too many buckets because:
  * - consume too much memory
@@ -338,7 +339,7 @@ int lu_site_purge(const struct lu_env *env, struct lu_site *s, int nr)
 	struct cfs_hash_bd	    bd2;
 	struct list_head	       dispose;
 	int		      did_sth;
-	unsigned int start;
+	unsigned int start = 0;
 	int		      count;
 	int		      bnr;
 	unsigned int i;
@@ -351,7 +352,8 @@ int lu_site_purge(const struct lu_env *env, struct lu_site *s, int nr)
 	 * Under LRU list lock, scan LRU list and move unreferenced objects to
 	 * the dispose list, removing them from LRU and hash table.
 	 */
-	start = s->ls_purge_start;
+	if (nr != ~0)
+		start = s->ls_purge_start;
 	bnr = (nr == ~0) ? -1 : nr / (int)CFS_HASH_NBKT(s->ls_obj_hash) + 1;
  again:
 	/*
@@ -877,6 +879,9 @@ static unsigned long lu_htable_order(struct lu_device *top)
 	unsigned long cache_size;
 	unsigned long bits;
 
+	if (!strcmp(top->ld_type->ldt_name, LUSTRE_VVP_NAME))
+		bits_max = LU_SITE_BITS_MAX_CL;
+
 	/*
 	 * Calculate hash table size, assuming that we want reasonable
 	 * performance when 20% of total memory is occupied by cache of
@@ -909,8 +914,8 @@ static unsigned long lu_htable_order(struct lu_device *top)
 	return clamp_t(typeof(bits), bits, LU_SITE_BITS_MIN, bits_max);
 }
 
-static unsigned lu_obj_hop_hash(struct cfs_hash *hs,
-				const void *key, unsigned mask)
+static unsigned int lu_obj_hop_hash(struct cfs_hash *hs,
+				    const void *key, unsigned int mask)
 {
 	struct lu_fid  *fid = (struct lu_fid *)key;
 	__u32	   hash;
@@ -1311,6 +1316,7 @@ enum {
 static struct lu_context_key *lu_keys[LU_CONTEXT_KEY_NR] = { NULL, };
 
 static DEFINE_SPINLOCK(lu_keys_guard);
+static atomic_t lu_key_initing_cnt = ATOMIC_INIT(0);
 
 /**
  * Global counter incremented whenever key is registered, unregistered,
@@ -1318,7 +1324,7 @@ static DEFINE_SPINLOCK(lu_keys_guard);
  * lu_context_refill(). No locking is provided, as initialization and shutdown
  * are supposed to be externally serialized.
  */
-static unsigned key_set_version;
+static unsigned int key_set_version;
 
 /**
  * Register new key.
@@ -1385,6 +1391,19 @@ void lu_context_key_degister(struct lu_context_key *key)
 	++key_set_version;
 	spin_lock(&lu_keys_guard);
 	key_fini(&lu_shrink_env.le_ctx, key->lct_index);
+
+	/**
+	 * Wait until all transient contexts referencing this key have
+	 * run lu_context_key::lct_fini() method.
+	 */
+	while (atomic_read(&key->lct_used) > 1) {
+		spin_unlock(&lu_keys_guard);
+		CDEBUG(D_INFO, "lu_context_key_degister: \"%s\" %p, %d\n",
+		       key->lct_owner ? key->lct_owner->name : "", key,
+		       atomic_read(&key->lct_used));
+		schedule();
+		spin_lock(&lu_keys_guard);
+	}
 	if (lu_keys[key->lct_index]) {
 		lu_keys[key->lct_index] = NULL;
 		lu_ref_fini(&key->lct_reference);
@@ -1507,14 +1526,25 @@ void lu_context_key_quiesce(struct lu_context_key *key)
 
 	if (!(key->lct_tags & LCT_QUIESCENT)) {
 		/*
-		 * XXX layering violation.
-		 */
-		cl_env_cache_purge(~0);
-		key->lct_tags |= LCT_QUIESCENT;
-		/*
 		 * XXX memory barrier has to go here.
 		 */
 		spin_lock(&lu_keys_guard);
+		key->lct_tags |= LCT_QUIESCENT;
+
+		/**
+		 * Wait until all lu_context_key::lct_init() methods
+		 * have completed.
+		 */
+		while (atomic_read(&lu_key_initing_cnt) > 0) {
+			spin_unlock(&lu_keys_guard);
+			CDEBUG(D_INFO, "lu_context_key_quiesce: \"%s\" %p, %d (%d)\n",
+			       key->lct_owner ? key->lct_owner->name : "",
+			       key, atomic_read(&key->lct_used),
+			atomic_read(&lu_key_initing_cnt));
+			schedule();
+			spin_lock(&lu_keys_guard);
+		}
+
 		list_for_each_entry(ctx, &lu_context_remembered, lc_remember)
 			key_fini(ctx, key->lct_index);
 		spin_unlock(&lu_keys_guard);
@@ -1546,6 +1576,19 @@ static int keys_fill(struct lu_context *ctx)
 {
 	unsigned int i;
 
+	/*
+	 * A serialisation with lu_context_key_quiesce() is needed, but some
+	 * "key->lct_init()" are calling kernel memory allocation routine and
+	 * can't be called while holding a spin_lock.
+	 * "lu_keys_guard" is held while incrementing "lu_key_initing_cnt"
+	 * to ensure the start of the serialisation.
+	 * An atomic_t variable is still used, in order not to reacquire the
+	 * lock when decrementing the counter.
+	 */
+	spin_lock(&lu_keys_guard);
+	atomic_inc(&lu_key_initing_cnt);
+	spin_unlock(&lu_keys_guard);
+
 	LINVRNT(ctx->lc_value);
 	for (i = 0; i < ARRAY_SIZE(lu_keys); ++i) {
 		struct lu_context_key *key;
@@ -1563,12 +1606,19 @@ static int keys_fill(struct lu_context *ctx)
 			LINVRNT(key->lct_init);
 			LINVRNT(key->lct_index == i);
 
+			LASSERT(key->lct_owner);
+			if (!(ctx->lc_tags & LCT_NOREF) &&
+			    !try_module_get(key->lct_owner)) {
+				/* module is unloading, skip this key */
+				continue;
+			}
+
 			value = key->lct_init(ctx, key);
-			if (IS_ERR(value))
+			if (unlikely(IS_ERR(value))) {
+				atomic_dec(&lu_key_initing_cnt);
 				return PTR_ERR(value);
+			}
 
-			if (!(ctx->lc_tags & LCT_NOREF))
-				try_module_get(key->lct_owner);
 			lu_ref_add_atomic(&key->lct_reference, "ctx", ctx);
 			atomic_inc(&key->lct_used);
 			/*
@@ -1582,6 +1632,7 @@ static int keys_fill(struct lu_context *ctx)
 		}
 		ctx->lc_version = key_set_version;
 	}
+	atomic_dec(&lu_key_initing_cnt);
 	return 0;
 }
 
@@ -1663,6 +1714,9 @@ void lu_context_exit(struct lu_context *ctx)
 	ctx->lc_state = LCS_LEFT;
 	if (ctx->lc_tags & LCT_HAS_EXIT && ctx->lc_value) {
 		for (i = 0; i < ARRAY_SIZE(lu_keys); ++i) {
+			/* could race with key quiescency */
+			if (ctx->lc_tags & LCT_REMEMBER)
+				spin_lock(&lu_keys_guard);
 			if (ctx->lc_value[i]) {
 				struct lu_context_key *key;
 
@@ -1671,6 +1725,8 @@ void lu_context_exit(struct lu_context *ctx)
 					key->lct_exit(ctx,
 						      key, ctx->lc_value[i]);
 			}
+			if (ctx->lc_tags & LCT_REMEMBER)
+				spin_unlock(&lu_keys_guard);
 		}
 	}
 }
@@ -1930,7 +1986,7 @@ int lu_site_stats_print(const struct lu_site *s, struct seq_file *m)
 	memset(&stats, 0, sizeof(stats));
 	lu_site_stats_get(s->ls_obj_hash, &stats, 1);
 
-	seq_printf(m, "%d/%d %d/%d %d %d %d %d %d %d %d %d\n",
+	seq_printf(m, "%d/%d %d/%ld %d %d %d %d %d %d %d %d\n",
 		   stats.lss_busy,
 		   stats.lss_total,
 		   stats.lss_populated,
diff --git a/drivers/staging/lustre/lustre/obdclass/obd_config.c b/drivers/staging/lustre/lustre/obdclass/obd_config.c
index bbed1b72d52e..9ca84c7d49de 100644
--- a/drivers/staging/lustre/lustre/obdclass/obd_config.c
+++ b/drivers/staging/lustre/lustre/obdclass/obd_config.c
@@ -35,12 +35,15 @@
  */
 
 #define DEBUG_SUBSYSTEM S_CLASS
-#include "../include/obd_class.h"
+
 #include <linux/string.h>
+
 #include "../include/lustre/lustre_ioctl.h"
-#include "../include/lustre_log.h"
+#include "../include/llog_swab.h"
 #include "../include/lprocfs_status.h"
+#include "../include/lustre_log.h"
 #include "../include/lustre_param.h"
+#include "../include/obd_class.h"
 
 #include "llog_internal.h"
 
@@ -446,7 +449,7 @@ static int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg)
 	LASSERT(obd->obd_self_export);
 
 	/* Precleanup, we must make sure all exports get destroyed. */
-	err = obd_precleanup(obd, OBD_CLEANUP_EXPORTS);
+	err = obd_precleanup(obd);
 	if (err)
 		CERROR("Precleanup %s returned %d\n",
 		       obd->obd_name, err);
@@ -585,16 +588,21 @@ static int class_del_conn(struct obd_device *obd, struct lustre_cfg *lcfg)
 }
 
 static LIST_HEAD(lustre_profile_list);
+static DEFINE_SPINLOCK(lustre_profile_list_lock);
 
 struct lustre_profile *class_get_profile(const char *prof)
 {
 	struct lustre_profile *lprof;
 
+	spin_lock(&lustre_profile_list_lock);
 	list_for_each_entry(lprof, &lustre_profile_list, lp_list) {
 		if (!strcmp(lprof->lp_profile, prof)) {
+			lprof->lp_refs++;
+			spin_unlock(&lustre_profile_list_lock);
 			return lprof;
 		}
 	}
+	spin_unlock(&lustre_profile_list_lock);
 	return NULL;
 }
 EXPORT_SYMBOL(class_get_profile);
@@ -639,7 +647,11 @@ static int class_add_profile(int proflen, char *prof, int osclen, char *osc,
 		}
 	}
 
+	spin_lock(&lustre_profile_list_lock);
+	lprof->lp_refs = 1;
+	lprof->lp_list_deleted = false;
 	list_add(&lprof->lp_list, &lustre_profile_list);
+	spin_unlock(&lustre_profile_list_lock);
 	return err;
 
 free_lp_dt:
@@ -659,27 +671,59 @@ void class_del_profile(const char *prof)
 
 	lprof = class_get_profile(prof);
 	if (lprof) {
+		spin_lock(&lustre_profile_list_lock);
+		/* because get profile increments the ref counter */
+		lprof->lp_refs--;
 		list_del(&lprof->lp_list);
-		kfree(lprof->lp_profile);
-		kfree(lprof->lp_dt);
-		kfree(lprof->lp_md);
-		kfree(lprof);
+		lprof->lp_list_deleted = true;
+		spin_unlock(&lustre_profile_list_lock);
+
+		class_put_profile(lprof);
 	}
 }
 EXPORT_SYMBOL(class_del_profile);
 
+void class_put_profile(struct lustre_profile *lprof)
+{
+	spin_lock(&lustre_profile_list_lock);
+	if (--lprof->lp_refs > 0) {
+		LASSERT(lprof->lp_refs > 0);
+		spin_unlock(&lustre_profile_list_lock);
+		return;
+	}
+	spin_unlock(&lustre_profile_list_lock);
+
+	/* confirm not a negative number */
+	LASSERT(!lprof->lp_refs);
+
+	/*
+	 * At least one class_del_profile/profiles must be called
+	 * on the target profile or lustre_profile_list will corrupt
+	 */
+	LASSERT(lprof->lp_list_deleted);
+	kfree(lprof->lp_profile);
+	kfree(lprof->lp_dt);
+	kfree(lprof->lp_md);
+	kfree(lprof);
+}
+EXPORT_SYMBOL(class_put_profile);
+
 /* COMPAT_146 */
 void class_del_profiles(void)
 {
 	struct lustre_profile *lprof, *n;
 
+	spin_lock(&lustre_profile_list_lock);
 	list_for_each_entry_safe(lprof, n, &lustre_profile_list, lp_list) {
 		list_del(&lprof->lp_list);
-		kfree(lprof->lp_profile);
-		kfree(lprof->lp_dt);
-		kfree(lprof->lp_md);
-		kfree(lprof);
+		lprof->lp_list_deleted = true;
+		spin_unlock(&lustre_profile_list_lock);
+
+		class_put_profile(lprof);
+
+		spin_lock(&lustre_profile_list_lock);
 	}
+	spin_unlock(&lustre_profile_list_lock);
 }
 EXPORT_SYMBOL(class_del_profiles);
 
@@ -1406,8 +1450,8 @@ EXPORT_SYMBOL(class_manual_cleanup);
  * uuid<->export lustre hash operations
  */
 
-static unsigned
-uuid_hash(struct cfs_hash *hs, const void *key, unsigned mask)
+static unsigned int
+uuid_hash(struct cfs_hash *hs, const void *key, unsigned int mask)
 {
 	return cfs_hash_djb2_hash(((struct obd_uuid *)key)->uuid,
 				  sizeof(((struct obd_uuid *)key)->uuid), mask);
diff --git a/drivers/staging/lustre/lustre/obdclass/obd_mount.c b/drivers/staging/lustre/lustre/obdclass/obd_mount.c
index 0d3a3b05a637..2283e920d839 100644
--- a/drivers/staging/lustre/lustre/obdclass/obd_mount.c
+++ b/drivers/staging/lustre/lustre/obdclass/obd_mount.c
@@ -261,7 +261,7 @@ int lustre_start_mgc(struct super_block *sb)
 
 			rc = obd_get_info(NULL, obd->obd_self_export,
 					  strlen(KEY_CONN_DATA), KEY_CONN_DATA,
-					  &vallen, data, NULL);
+					  &vallen, data);
 			LASSERT(rc == 0);
 			has_ir = OCD_HAS_FLAG(data, IMP_RECOV);
 			if (has_ir ^ !(*flags & LMD_FLG_NOIR)) {
@@ -382,7 +382,7 @@ int lustre_start_mgc(struct super_block *sb)
 	/* We connect to the MGS at setup, and don't disconnect until cleanup */
 	data->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_AT |
 				  OBD_CONNECT_FULL20 | OBD_CONNECT_IMP_RECOV |
-				  OBD_CONNECT_LVB_TYPE;
+				  OBD_CONNECT_LVB_TYPE | OBD_CONNECT_BULK_MBITS;
 
 #if OBD_OCD_VERSION(3, 0, 53, 0) > LUSTRE_VERSION_CODE
 	data->ocd_connect_flags |= OBD_CONNECT_MNE_SWAB;
@@ -1216,8 +1216,7 @@ static struct file_system_type lustre_fs_type = {
 	.name	 = "lustre",
 	.mount	= lustre_mount,
 	.kill_sb      = lustre_kill_super,
-	.fs_flags     = FS_BINARY_MOUNTDATA | FS_REQUIRES_DEV |
-			FS_RENAME_DOES_D_MOVE,
+	.fs_flags	= FS_REQUIRES_DEV | FS_RENAME_DOES_D_MOVE,
 };
 MODULE_ALIAS_FS("lustre");
 
diff --git a/drivers/staging/lustre/lustre/obdclass/obdo.c b/drivers/staging/lustre/lustre/obdclass/obdo.c
index 79104a66da96..c52b9e07d7dd 100644
--- a/drivers/staging/lustre/lustre/obdclass/obdo.c
+++ b/drivers/staging/lustre/lustre/obdclass/obdo.c
@@ -124,68 +124,3 @@ void obdo_to_ioobj(const struct obdo *oa, struct obd_ioobj *ioobj)
 	ioobj->ioo_max_brw = 0;
 }
 EXPORT_SYMBOL(obdo_to_ioobj);
-
-static void iattr_from_obdo(struct iattr *attr, const struct obdo *oa,
-			    u32 valid)
-{
-	valid &= oa->o_valid;
-
-	if (valid & (OBD_MD_FLCTIME | OBD_MD_FLMTIME))
-		CDEBUG(D_INODE, "valid %#llx, new time %llu/%llu\n",
-		       oa->o_valid, oa->o_mtime, oa->o_ctime);
-
-	attr->ia_valid = 0;
-	if (valid & OBD_MD_FLATIME) {
-		LTIME_S(attr->ia_atime) = oa->o_atime;
-		attr->ia_valid |= ATTR_ATIME;
-	}
-	if (valid & OBD_MD_FLMTIME) {
-		LTIME_S(attr->ia_mtime) = oa->o_mtime;
-		attr->ia_valid |= ATTR_MTIME;
-	}
-	if (valid & OBD_MD_FLCTIME) {
-		LTIME_S(attr->ia_ctime) = oa->o_ctime;
-		attr->ia_valid |= ATTR_CTIME;
-	}
-	if (valid & OBD_MD_FLSIZE) {
-		attr->ia_size = oa->o_size;
-		attr->ia_valid |= ATTR_SIZE;
-	}
-#if 0   /* you shouldn't be able to change a file's type with setattr */
-	if (valid & OBD_MD_FLTYPE) {
-		attr->ia_mode = (attr->ia_mode & ~S_IFMT) |
-				(oa->o_mode & S_IFMT);
-		attr->ia_valid |= ATTR_MODE;
-	}
-#endif
-	if (valid & OBD_MD_FLMODE) {
-		attr->ia_mode = (attr->ia_mode & S_IFMT) |
-				(oa->o_mode & ~S_IFMT);
-		attr->ia_valid |= ATTR_MODE;
-		if (!in_group_p(make_kgid(&init_user_ns, oa->o_gid)) &&
-		    !capable(CFS_CAP_FSETID))
-			attr->ia_mode &= ~S_ISGID;
-	}
-	if (valid & OBD_MD_FLUID) {
-		attr->ia_uid = make_kuid(&init_user_ns, oa->o_uid);
-		attr->ia_valid |= ATTR_UID;
-	}
-	if (valid & OBD_MD_FLGID) {
-		attr->ia_gid = make_kgid(&init_user_ns, oa->o_gid);
-		attr->ia_valid |= ATTR_GID;
-	}
-}
-
-void md_from_obdo(struct md_op_data *op_data, const struct obdo *oa, u32 valid)
-{
-	iattr_from_obdo(&op_data->op_attr, oa, valid);
-	if (valid & OBD_MD_FLBLOCKS) {
-		op_data->op_attr_blocks = oa->o_blocks;
-		op_data->op_attr.ia_valid |= ATTR_BLOCKS;
-	}
-	if (valid & OBD_MD_FLFLAGS) {
-		op_data->op_attr_flags = oa->o_flags;
-		op_data->op_attr.ia_valid |= ATTR_ATTR_FLAG;
-	}
-}
-EXPORT_SYMBOL(md_from_obdo);
diff --git a/drivers/staging/lustre/lustre/obdecho/echo_client.c b/drivers/staging/lustre/lustre/obdecho/echo_client.c
index 505582ff4d1e..549076193bde 100644
--- a/drivers/staging/lustre/lustre/obdecho/echo_client.c
+++ b/drivers/staging/lustre/lustre/obdecho/echo_client.c
@@ -55,7 +55,7 @@ struct echo_device {
 	struct echo_client_obd *ed_ec;
 
 	struct cl_site	  ed_site_myself;
-	struct cl_site	 *ed_site;
+	struct lu_site		*ed_site;
 	struct lu_device       *ed_next;
 };
 
@@ -505,9 +505,6 @@ static const struct lu_device_operations echo_device_lu_ops = {
 
 /** @} echo_lu_dev_ops */
 
-static const struct cl_device_operations echo_device_cl_ops = {
-};
-
 /** \defgroup echo_init Setup and teardown
  *
  * Init and fini functions for echo client.
@@ -527,17 +524,19 @@ static int echo_site_init(const struct lu_env *env, struct echo_device *ed)
 	}
 
 	rc = lu_site_init_finish(&site->cs_lu);
-	if (rc)
+	if (rc) {
+		cl_site_fini(site);
 		return rc;
+	}
 
-	ed->ed_site = site;
+	ed->ed_site = &site->cs_lu;
 	return 0;
 }
 
 static void echo_site_fini(const struct lu_env *env, struct echo_device *ed)
 {
 	if (ed->ed_site) {
-		cl_site_fini(ed->ed_site);
+		lu_site_fini(ed->ed_site);
 		ed->ed_site = NULL;
 	}
 }
@@ -561,16 +560,10 @@ static void echo_thread_key_fini(const struct lu_context *ctx,
 	kmem_cache_free(echo_thread_kmem, info);
 }
 
-static void echo_thread_key_exit(const struct lu_context *ctx,
-				 struct lu_context_key *key, void *data)
-{
-}
-
 static struct lu_context_key echo_thread_key = {
 	.lct_tags = LCT_CL_THREAD,
 	.lct_init = echo_thread_key_init,
 	.lct_fini = echo_thread_key_fini,
-	.lct_exit = echo_thread_key_exit
 };
 
 static void *echo_session_key_init(const struct lu_context *ctx,
@@ -592,16 +585,10 @@ static void echo_session_key_fini(const struct lu_context *ctx,
 	kmem_cache_free(echo_session_kmem, session);
 }
 
-static void echo_session_key_exit(const struct lu_context *ctx,
-				  struct lu_context_key *key, void *data)
-{
-}
-
 static struct lu_context_key echo_session_key = {
 	.lct_tags = LCT_SESSION,
 	.lct_init = echo_session_key_init,
 	.lct_fini = echo_session_key_fini,
-	.lct_exit = echo_session_key_exit
 };
 
 LU_TYPE_INIT_FINI(echo, &echo_thread_key, &echo_session_key);
@@ -630,7 +617,6 @@ static struct lu_device *echo_device_alloc(const struct lu_env *env,
 		goto out_free;
 
 	cd->cd_lu_dev.ld_ops = &echo_device_lu_ops;
-	cd->cd_ops = &echo_device_cl_ops;
 
 	obd = class_name2obd(lustre_cfg_string(cfg, 0));
 	LASSERT(obd);
@@ -674,7 +660,7 @@ static struct lu_device *echo_device_alloc(const struct lu_env *env,
 			goto out_cleanup;
 		}
 
-		next->ld_site = &ed->ed_site->cs_lu;
+		next->ld_site = ed->ed_site;
 		rc = next->ld_type->ldt_ops->ldto_device_init(env, next,
 						next->ld_type->ldt_name,
 							      NULL);
@@ -741,7 +727,7 @@ static struct lu_device *echo_device_free(const struct lu_env *env,
 	CDEBUG(D_INFO, "echo device:%p is going to be freed, next = %p\n",
 	       ed, next);
 
-	lu_site_purge(env, &ed->ed_site->cs_lu, -1);
+	lu_site_purge(env, ed->ed_site, -1);
 
 	/* check if there are objects still alive.
 	 * It shouldn't have any object because lu_site_purge would cleanup
@@ -754,7 +740,7 @@ static struct lu_device *echo_device_free(const struct lu_env *env,
 	spin_unlock(&ec->ec_lock);
 
 	/* purge again */
-	lu_site_purge(env, &ed->ed_site->cs_lu, -1);
+	lu_site_purge(env, ed->ed_site, -1);
 
 	CDEBUG(D_INFO,
 	       "Waiting for the reference of echo object to be dropped\n");
@@ -766,7 +752,7 @@ static struct lu_device *echo_device_free(const struct lu_env *env,
 		CERROR("echo_client still has objects at cleanup time, wait for 1 second\n");
 		set_current_state(TASK_UNINTERRUPTIBLE);
 		schedule_timeout(cfs_time_seconds(1));
-		lu_site_purge(env, &ed->ed_site->cs_lu, -1);
+		lu_site_purge(env, ed->ed_site, -1);
 		spin_lock(&ec->ec_lock);
 	}
 	spin_unlock(&ec->ec_lock);
@@ -780,11 +766,13 @@ static struct lu_device *echo_device_free(const struct lu_env *env,
 	while (next)
 		next = next->ld_type->ldt_ops->ldto_device_free(env, next);
 
-	LASSERT(ed->ed_site == lu2cl_site(d->ld_site));
+	LASSERT(ed->ed_site == d->ld_site);
 	echo_site_fini(env, ed);
 	cl_device_fini(&ed->ed_cl);
 	kfree(ed);
 
+	cl_env_cache_purge(~0);
+
 	return NULL;
 }
 
@@ -1100,7 +1088,7 @@ out:
 static u64 last_object_id;
 
 static int echo_create_object(const struct lu_env *env, struct echo_device *ed,
-			      struct obdo *oa, struct obd_trans_info *oti)
+			      struct obdo *oa)
 {
 	struct echo_object     *eco;
 	struct echo_client_obd *ec = ed->ed_ec;
@@ -1117,7 +1105,7 @@ static int echo_create_object(const struct lu_env *env, struct echo_device *ed,
 	if (!ostid_id(&oa->o_oi))
 		ostid_set_id(&oa->o_oi, ++last_object_id);
 
-	rc = obd_create(env, ec->ec_exp, oa, oti);
+	rc = obd_create(env, ec->ec_exp, oa);
 	if (rc != 0) {
 		CERROR("Cannot create objects: rc = %d\n", rc);
 		goto failed;
@@ -1137,7 +1125,7 @@ static int echo_create_object(const struct lu_env *env, struct echo_device *ed,
 
  failed:
 	if (created && rc)
-		obd_destroy(env, ec->ec_exp, oa, oti);
+		obd_destroy(env, ec->ec_exp, oa);
 	if (rc)
 		CERROR("create object failed with: rc = %d\n", rc);
 	return rc;
@@ -1237,8 +1225,7 @@ static int echo_client_page_debug_check(struct page *page, u64 id,
 
 static int echo_client_kbrw(struct echo_device *ed, int rw, struct obdo *oa,
 			    struct echo_object *eco, u64 offset,
-			    u64 count, int async,
-			    struct obd_trans_info *oti)
+			    u64 count, int async)
 {
 	u32	       npages;
 	struct brw_page	*pga;
@@ -1332,12 +1319,11 @@ static int echo_client_prep_commit(const struct lu_env *env,
 				   struct obd_export *exp, int rw,
 				   struct obdo *oa, struct echo_object *eco,
 				   u64 offset, u64 count,
-				   u64 batch, struct obd_trans_info *oti,
-				   int async)
+				   u64 batch, int async)
 {
 	struct obd_ioobj ioo;
 	struct niobuf_local *lnb;
-	struct niobuf_remote *rnb;
+	struct niobuf_remote rnb;
 	u64 off;
 	u64 npages, tot_pages;
 	int i, ret = 0, brw_flags = 0;
@@ -1349,9 +1335,7 @@ static int echo_client_prep_commit(const struct lu_env *env,
 	tot_pages = count >> PAGE_SHIFT;
 
 	lnb = kcalloc(npages, sizeof(struct niobuf_local), GFP_NOFS);
-	rnb = kcalloc(npages, sizeof(struct niobuf_remote), GFP_NOFS);
-
-	if (!lnb || !rnb) {
+	if (!lnb) {
 		ret = -ENOMEM;
 		goto out;
 	}
@@ -1363,26 +1347,22 @@ static int echo_client_prep_commit(const struct lu_env *env,
 
 	off = offset;
 
-	for (; tot_pages; tot_pages -= npages) {
+	for (; tot_pages > 0; tot_pages -= npages) {
 		int lpages;
 
 		if (tot_pages < npages)
 			npages = tot_pages;
 
-		for (i = 0; i < npages; i++, off += PAGE_SIZE) {
-			rnb[i].rnb_offset = off;
-			rnb[i].rnb_len = PAGE_SIZE;
-			rnb[i].rnb_flags = brw_flags;
-		}
-
-		ioo.ioo_bufcnt = npages;
+		rnb.rnb_offset = off;
+		rnb.rnb_len = npages * PAGE_SIZE;
+		rnb.rnb_flags = brw_flags;
+		ioo.ioo_bufcnt = 1;
+		off += npages * PAGE_SIZE;
 
 		lpages = npages;
-		ret = obd_preprw(env, rw, exp, oa, 1, &ioo, rnb, &lpages,
-				 lnb, oti);
+		ret = obd_preprw(env, rw, exp, oa, 1, &ioo, &rnb, &lpages, lnb);
 		if (ret != 0)
 			goto out;
-		LASSERT(lpages == npages);
 
 		for (i = 0; i < lpages; i++) {
 			struct page *page = lnb[i].lnb_page;
@@ -1401,24 +1381,21 @@ static int echo_client_prep_commit(const struct lu_env *env,
 
 			if (rw == OBD_BRW_WRITE)
 				echo_client_page_debug_setup(page, rw,
-							    ostid_id(&oa->o_oi),
-							     rnb[i].rnb_offset,
-							     rnb[i].rnb_len);
+							     ostid_id(&oa->o_oi),
+							     lnb[i].lnb_file_offset,
+							     lnb[i].lnb_len);
 			else
 				echo_client_page_debug_check(page,
-							    ostid_id(&oa->o_oi),
-							     rnb[i].rnb_offset,
-							     rnb[i].rnb_len);
+							     ostid_id(&oa->o_oi),
+							     lnb[i].lnb_file_offset,
+							     lnb[i].lnb_len);
 		}
 
-		ret = obd_commitrw(env, rw, exp, oa, 1, &ioo,
-				   rnb, npages, lnb, oti, ret);
+		ret = obd_commitrw(env, rw, exp, oa, 1, &ioo, &rnb, npages, lnb,
+				   ret);
 		if (ret != 0)
 			goto out;
 
-		/* Reset oti otherwise it would confuse ldiskfs. */
-		memset(oti, 0, sizeof(*oti));
-
 		/* Reuse env context. */
 		lu_context_exit((struct lu_context *)&env->le_ctx);
 		lu_context_enter((struct lu_context *)&env->le_ctx);
@@ -1426,14 +1403,12 @@ static int echo_client_prep_commit(const struct lu_env *env,
 
 out:
 	kfree(lnb);
-	kfree(rnb);
 	return ret;
 }
 
 static int echo_client_brw_ioctl(const struct lu_env *env, int rw,
 				 struct obd_export *exp,
-				 struct obd_ioctl_data *data,
-				 struct obd_trans_info *dummy_oti)
+				 struct obd_ioctl_data *data)
 {
 	struct obd_device *obd = class_exp2obd(exp);
 	struct echo_device *ed = obd2echo_dev(obd);
@@ -1470,15 +1445,13 @@ static int echo_client_brw_ioctl(const struct lu_env *env, int rw,
 	case 1:
 		/* fall through */
 	case 2:
-		rc = echo_client_kbrw(ed, rw, oa,
-				      eco, data->ioc_offset,
-				      data->ioc_count, async, dummy_oti);
+		rc = echo_client_kbrw(ed, rw, oa, eco, data->ioc_offset,
+				      data->ioc_count, async);
 		break;
 	case 3:
-		rc = echo_client_prep_commit(env, ec->ec_exp, rw, oa,
-					     eco, data->ioc_offset,
-					     data->ioc_count, data->ioc_plen1,
-					     dummy_oti, async);
+		rc = echo_client_prep_commit(env, ec->ec_exp, rw, oa, eco,
+					     data->ioc_offset, data->ioc_count,
+					     data->ioc_plen1, async);
 		break;
 	default:
 		rc = -EINVAL;
@@ -1496,16 +1469,11 @@ echo_client_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 	struct echo_client_obd *ec = ed->ed_ec;
 	struct echo_object     *eco;
 	struct obd_ioctl_data  *data = karg;
-	struct obd_trans_info   dummy_oti;
 	struct lu_env	  *env;
-	struct oti_req_ack_lock *ack_lock;
 	struct obdo	    *oa;
 	struct lu_fid	   fid;
 	int		     rw = OBD_BRW_READ;
 	int		     rc = 0;
-	int		     i;
-
-	memset(&dummy_oti, 0, sizeof(dummy_oti));
 
 	oa = &data->ioc_obdo1;
 	if (!(oa->o_valid & OBD_MD_FLGROUP)) {
@@ -1535,7 +1503,7 @@ echo_client_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 			goto out;
 		}
 
-		rc = echo_create_object(env, ed, oa, &dummy_oti);
+		rc = echo_create_object(env, ed, oa);
 		goto out;
 
 	case OBD_IOC_DESTROY:
@@ -1546,7 +1514,7 @@ echo_client_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 
 		rc = echo_get_object(&eco, ed, oa);
 		if (rc == 0) {
-			rc = obd_destroy(env, ec->ec_exp, oa, &dummy_oti);
+			rc = obd_destroy(env, ec->ec_exp, oa);
 			if (rc == 0)
 				eco->eo_deleted = 1;
 			echo_put_object(eco);
@@ -1556,11 +1524,7 @@ echo_client_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 	case OBD_IOC_GETATTR:
 		rc = echo_get_object(&eco, ed, oa);
 		if (rc == 0) {
-			struct obd_info oinfo = {
-				.oi_oa = oa,
-			};
-
-			rc = obd_getattr(env, ec->ec_exp, &oinfo);
+			rc = obd_getattr(env, ec->ec_exp, oa);
 			echo_put_object(eco);
 		}
 		goto out;
@@ -1573,11 +1537,7 @@ echo_client_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 
 		rc = echo_get_object(&eco, ed, oa);
 		if (rc == 0) {
-			struct obd_info oinfo = {
-				.oi_oa = oa,
-			};
-
-			rc = obd_setattr(env, ec->ec_exp, &oinfo, NULL);
+			rc = obd_setattr(env, ec->ec_exp, oa);
 			echo_put_object(eco);
 		}
 		goto out;
@@ -1591,7 +1551,7 @@ echo_client_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 		rw = OBD_BRW_WRITE;
 		/* fall through */
 	case OBD_IOC_BRW_READ:
-		rc = echo_client_brw_ioctl(env, rw, exp, data, &dummy_oti);
+		rc = echo_client_brw_ioctl(env, rw, exp, data);
 		goto out;
 
 	default:
@@ -1604,14 +1564,6 @@ out:
 	lu_env_fini(env);
 	kfree(env);
 
-	/* XXX this should be in a helper also called by target_send_reply */
-	for (ack_lock = dummy_oti.oti_ack_locks, i = 0; i < 4;
-	     i++, ack_lock++) {
-		if (!ack_lock->mode)
-			break;
-		ldlm_lock_decref(&ack_lock->lock, ack_lock->mode);
-	}
-
 	return rc;
 }
 
diff --git a/drivers/staging/lustre/lustre/osc/lproc_osc.c b/drivers/staging/lustre/lustre/osc/lproc_osc.c
index f0062d44ee03..575b2969ad83 100644
--- a/drivers/staging/lustre/lustre/osc/lproc_osc.c
+++ b/drivers/staging/lustre/lustre/osc/lproc_osc.c
@@ -162,7 +162,7 @@ static ssize_t max_dirty_mb_store(struct kobject *kobj,
 	pages_number *= 1 << (20 - PAGE_SHIFT); /* MB -> pages */
 
 	if (pages_number <= 0 ||
-	    pages_number > OSC_MAX_DIRTY_MB_MAX << (20 - PAGE_SHIFT) ||
+	    pages_number >= OSC_MAX_DIRTY_MB_MAX << (20 - PAGE_SHIFT) ||
 	    pages_number > totalram_pages / 4) /* 1/4 of RAM */
 		return -ERANGE;
 
@@ -183,10 +183,12 @@ static int osc_cached_mb_seq_show(struct seq_file *m, void *v)
 
 	seq_printf(m,
 		   "used_mb: %ld\n"
-		   "busy_cnt: %ld\n",
+		   "busy_cnt: %ld\n"
+		   "reclaim: %llu\n",
 		   (atomic_long_read(&cli->cl_lru_in_list) +
 		    atomic_long_read(&cli->cl_lru_busy)) >> shift,
-		   atomic_long_read(&cli->cl_lru_busy));
+		   atomic_long_read(&cli->cl_lru_busy),
+		   cli->cl_lru_reclaim);
 
 	return 0;
 }
@@ -585,7 +587,8 @@ static ssize_t max_pages_per_rpc_store(struct kobject *kobj,
 	chunk_mask = ~((1 << (cli->cl_chunkbits - PAGE_SHIFT)) - 1);
 	/* max_pages_per_rpc must be chunk aligned */
 	val = (val + ~chunk_mask) & chunk_mask;
-	if (val == 0 || val > ocd->ocd_brw_size >> PAGE_SHIFT) {
+	if (!val || (ocd->ocd_brw_size &&
+		     val > ocd->ocd_brw_size >> PAGE_SHIFT)) {
 		return -ERANGE;
 	}
 	spin_lock(&cli->cl_loi_list_lock);
diff --git a/drivers/staging/lustre/lustre/osc/osc_cache.c b/drivers/staging/lustre/lustre/osc/osc_cache.c
index 4bbe219add98..b0f030c6c9c9 100644
--- a/drivers/staging/lustre/lustre/osc/osc_cache.c
+++ b/drivers/staging/lustre/lustre/osc/osc_cache.c
@@ -360,6 +360,7 @@ static struct osc_extent *osc_extent_alloc(struct osc_object *obj)
 
 	RB_CLEAR_NODE(&ext->oe_node);
 	ext->oe_obj = obj;
+	cl_object_get(osc2cl(obj));
 	atomic_set(&ext->oe_refc, 1);
 	atomic_set(&ext->oe_users, 0);
 	INIT_LIST_HEAD(&ext->oe_link);
@@ -398,6 +399,7 @@ static void osc_extent_put(const struct lu_env *env, struct osc_extent *ext)
 			LDLM_LOCK_PUT(ext->oe_dlmlock);
 			ext->oe_dlmlock = NULL;
 		}
+		cl_object_put(env, osc2cl(ext->oe_obj));
 		osc_extent_free(ext);
 	}
 }
@@ -959,7 +961,7 @@ static int osc_extent_wait(const struct lu_env *env, struct osc_extent *ext,
 	if (rc == -ETIMEDOUT) {
 		OSC_EXTENT_DUMP(D_ERROR, ext,
 				"%s: wait ext to %u timedout, recovery in progress?\n",
-				osc_export(obj)->exp_obd->obd_name, state);
+				cli_name(osc_cli(obj)), state);
 
 		lwi = LWI_INTR(NULL, NULL);
 		rc = l_wait_event(ext->oe_waitq, extent_wait_cb(ext, state),
@@ -977,7 +979,6 @@ static int osc_extent_wait(const struct lu_env *env, struct osc_extent *ext,
 static int osc_extent_truncate(struct osc_extent *ext, pgoff_t trunc_index,
 			       bool partial)
 {
-	struct cl_env_nest nest;
 	struct lu_env *env;
 	struct cl_io *io;
 	struct osc_object *obj = ext->oe_obj;
@@ -990,6 +991,7 @@ static int osc_extent_truncate(struct osc_extent *ext, pgoff_t trunc_index,
 	int grants = 0;
 	int nr_pages = 0;
 	int rc = 0;
+	int refcheck;
 
 	LASSERT(sanity_check(ext) == 0);
 	EASSERT(ext->oe_state == OES_TRUNC, ext);
@@ -999,7 +1001,7 @@ static int osc_extent_truncate(struct osc_extent *ext, pgoff_t trunc_index,
 	 * We can't use that env from osc_cache_truncate_start() because
 	 * it's from lov_io_sub and not fully initialized.
 	 */
-	env = cl_env_nested_get(&nest);
+	env = cl_env_get(&refcheck);
 	io  = &osc_env_info(env)->oti_io;
 	io->ci_obj = cl_object_top(osc2cl(obj));
 	rc = cl_io_init(env, io, CIT_MISC, io->ci_obj);
@@ -1085,7 +1087,7 @@ static int osc_extent_truncate(struct osc_extent *ext, pgoff_t trunc_index,
 
 out:
 	cl_io_fini(env, io);
-	cl_env_nested_put(&nest, env);
+	cl_env_put(env, &refcheck);
 	return rc;
 }
 
@@ -1327,7 +1329,6 @@ static int osc_completion(const struct lu_env *env, struct osc_async_page *oap,
 {
 	struct osc_page *opg = oap2osc_page(oap);
 	struct cl_page    *page = oap2cl_page(oap);
-	struct osc_object *obj = cl2osc(opg->ops_cl.cpl_obj);
 	enum cl_req_type crt;
 	int srvlock;
 
@@ -1338,25 +1339,10 @@ static int osc_completion(const struct lu_env *env, struct osc_async_page *oap,
 		 "cp_state:%u, cmd:%d\n", page->cp_state, cmd);
 	LASSERT(opg->ops_transfer_pinned);
 
-	/*
-	 * page->cp_req can be NULL if io submission failed before
-	 * cl_req was allocated.
-	 */
-	if (page->cp_req)
-		cl_req_page_done(env, page);
-	LASSERT(!page->cp_req);
-
 	crt = cmd == OBD_BRW_READ ? CRT_READ : CRT_WRITE;
 	/* Clear opg->ops_transfer_pinned before VM lock is released. */
 	opg->ops_transfer_pinned = 0;
 
-	spin_lock(&obj->oo_seatbelt);
-	LASSERT(opg->ops_submitter);
-	LASSERT(!list_empty(&opg->ops_inflight));
-	list_del_init(&opg->ops_inflight);
-	opg->ops_submitter = NULL;
-	spin_unlock(&obj->oo_seatbelt);
-
 	opg->ops_submit_time = 0;
 	srvlock = oap->oap_brw_flags & OBD_BRW_SRVLOCK;
 
@@ -1380,16 +1366,17 @@ static int osc_completion(const struct lu_env *env, struct osc_async_page *oap,
 	lu_ref_del(&page->cp_reference, "transfer", page);
 
 	cl_page_completion(env, page, crt, rc);
+	cl_page_put(env, page);
 
 	return 0;
 }
 
 #define OSC_DUMP_GRANT(lvl, cli, fmt, args...) do {			      \
 	struct client_obd *__tmp = (cli);				      \
-	CDEBUG(lvl, "%s: grant { dirty: %ld/%ld dirty_pages: %ld/%lu "	      \
+	CDEBUG(lvl, "%s: grant { dirty: %lu/%lu dirty_pages: %ld/%lu "	      \
 	       "dropped: %ld avail: %ld, reserved: %ld, flight: %d }"	      \
 	       "lru {in list: %ld, left: %ld, waiters: %d }" fmt "\n",	      \
-	       __tmp->cl_import->imp_obd->obd_name,			      \
+	       cli_name(__tmp),						      \
 	       __tmp->cl_dirty_pages, __tmp->cl_dirty_max_pages,	      \
 	       atomic_long_read(&obd_dirty_pages), obd_max_dirty_pages,	      \
 	       __tmp->cl_lost_grant, __tmp->cl_avail_grant,		      \
@@ -1627,7 +1614,7 @@ static int osc_enter_cache(const struct lu_env *env, struct client_obd *cli,
 		osc_io_unplug_async(env, cli, NULL);
 
 		CDEBUG(D_CACHE, "%s: sleeping for cache space @ %p for %p\n",
-		       cli->cl_import->imp_obd->obd_name, &ocw, oap);
+		       cli_name(cli), &ocw, oap);
 
 		rc = l_wait_event(ocw.ocw_waitq, ocw_granted(cli, &ocw), &lwi);
 
@@ -1671,7 +1658,7 @@ static int osc_enter_cache(const struct lu_env *env, struct client_obd *cli,
 		break;
 	default:
 		CDEBUG(D_CACHE, "%s: event for cache space @ %p never arrived due to %d, fall back to sync i/o\n",
-		       cli->cl_import->imp_obd->obd_name, &ocw, rc);
+		       cli_name(cli), &ocw, rc);
 		break;
 	}
 out:
@@ -1931,7 +1918,8 @@ static int try_to_add_extent_for_io(struct client_obd *cli,
 		}
 
 		if (tmp->oe_srvlock != ext->oe_srvlock ||
-		    !tmp->oe_grants != !ext->oe_grants)
+		    !tmp->oe_grants != !ext->oe_grants ||
+		    tmp->oe_no_merge || ext->oe_no_merge)
 			return 0;
 
 		/* remove break for strict check */
@@ -2250,14 +2238,9 @@ static int osc_io_unplug0(const struct lu_env *env, struct client_obd *cli,
 		return 0;
 
 	if (!async) {
-		/* disable osc_lru_shrink() temporarily to avoid
-		 * potential stack overrun problem. LU-2859
-		 */
-		atomic_inc(&cli->cl_lru_shrinkers);
 		spin_lock(&cli->cl_loi_list_lock);
 		osc_check_rpcs(env, cli);
 		spin_unlock(&cli->cl_loi_list_lock);
-		atomic_dec(&cli->cl_lru_shrinkers);
 	} else {
 		CDEBUG(D_CACHE, "Queue writeback work for client %p.\n", cli);
 		LASSERT(cli->cl_writeback_work);
@@ -2479,7 +2462,6 @@ int osc_teardown_async_page(const struct lu_env *env,
 			    struct osc_object *obj, struct osc_page *ops)
 {
 	struct osc_async_page *oap = &ops->ops_oap;
-	struct osc_extent *ext = NULL;
 	int rc = 0;
 
 	LASSERT(oap->oap_magic == OAP_MAGIC);
@@ -2487,12 +2469,15 @@ int osc_teardown_async_page(const struct lu_env *env,
 	CDEBUG(D_INFO, "teardown oap %p page %p at index %lu.\n",
 	       oap, ops, osc_index(oap2osc(oap)));
 
-	osc_object_lock(obj);
 	if (!list_empty(&oap->oap_rpc_item)) {
 		CDEBUG(D_CACHE, "oap %p is not in cache.\n", oap);
 		rc = -EBUSY;
 	} else if (!list_empty(&oap->oap_pending_item)) {
+		struct osc_extent *ext = NULL;
+
+		osc_object_lock(obj);
 		ext = osc_extent_lookup(obj, osc_index(oap2osc(oap)));
+		osc_object_unlock(obj);
 		/* only truncated pages are allowed to be taken out.
 		 * See osc_extent_truncate() and osc_cache_truncate_start()
 		 * for details.
@@ -2502,10 +2487,9 @@ int osc_teardown_async_page(const struct lu_env *env,
 					osc_index(oap2osc(oap)));
 			rc = -EBUSY;
 		}
+		if (ext)
+			osc_extent_put(env, ext);
 	}
-	osc_object_unlock(obj);
-	if (ext)
-		osc_extent_put(env, ext);
 	return rc;
 }
 
@@ -2666,11 +2650,13 @@ int osc_queue_sync_pages(const struct lu_env *env, struct osc_object *obj,
 	struct osc_async_page *oap, *tmp;
 	int page_count = 0;
 	int mppr = cli->cl_max_pages_per_rpc;
+	bool can_merge = true;
 	pgoff_t start = CL_PAGE_EOF;
 	pgoff_t end = 0;
 
 	list_for_each_entry(oap, list, oap_pending_item) {
-		pgoff_t index = osc_index(oap2osc(oap));
+		struct osc_page *opg = oap2osc_page(oap);
+		pgoff_t index = osc_index(opg);
 
 		if (index > end)
 			end = index;
@@ -2678,6 +2664,9 @@ int osc_queue_sync_pages(const struct lu_env *env, struct osc_object *obj,
 			start = index;
 		++page_count;
 		mppr <<= (page_count > mppr);
+
+		if (unlikely(opg->ops_from > 0 || opg->ops_to < PAGE_SIZE))
+			can_merge = false;
 	}
 
 	ext = osc_extent_alloc(obj);
@@ -2691,6 +2680,7 @@ int osc_queue_sync_pages(const struct lu_env *env, struct osc_object *obj,
 
 	ext->oe_rw = !!(cmd & OBD_BRW_READ);
 	ext->oe_sync = 1;
+	ext->oe_no_merge = !can_merge;
 	ext->oe_urgent = 1;
 	ext->oe_start = start;
 	ext->oe_end = end;
@@ -3158,7 +3148,8 @@ static int check_and_discard_cb(const struct lu_env *env, struct cl_io *io,
 		struct cl_page *page = ops->ops_cl.cpl_page;
 
 		/* refresh non-overlapped index */
-		tmp = osc_dlmlock_at_pgoff(env, osc, index, 0, 0);
+		tmp = osc_dlmlock_at_pgoff(env, osc, index,
+					   OSC_DAP_FL_TEST_LOCK);
 		if (tmp) {
 			__u64 end = tmp->l_policy_data.l_extent.end;
 			/* Cache the first-non-overlapped index so as to skip
diff --git a/drivers/staging/lustre/lustre/osc/osc_cl_internal.h b/drivers/staging/lustre/lustre/osc/osc_cl_internal.h
index 9c8de15c309c..cce55a9689f0 100644
--- a/drivers/staging/lustre/lustre/osc/osc_cl_internal.h
+++ b/drivers/staging/lustre/lustre/osc/osc_cl_internal.h
@@ -77,7 +77,6 @@ struct osc_io {
 
 	/** write osc_lock for this IO, used by osc_extent_find(). */
 	struct osc_lock   *oi_write_osclock;
-	struct obd_info    oi_info;
 	struct obdo	oi_oa;
 	struct osc_async_cbargs {
 		bool		  opc_rpc_sent;
@@ -87,13 +86,6 @@ struct osc_io {
 };
 
 /**
- * State of transfer for osc.
- */
-struct osc_req {
-	struct cl_req_slice    or_cl;
-};
-
-/**
  * State maintained by osc layer for the duration of a system call.
  */
 struct osc_session {
@@ -103,7 +95,7 @@ struct osc_session {
 #define OTI_PVEC_SIZE 256
 struct osc_thread_info {
 	struct ldlm_res_id      oti_resname;
-	ldlm_policy_data_t      oti_policy;
+	union ldlm_policy_data	oti_policy;
 	struct cl_lock_descr    oti_descr;
 	struct cl_attr	  oti_attr;
 	struct lustre_handle    oti_handle;
@@ -116,6 +108,7 @@ struct osc_thread_info {
 	pgoff_t			oti_next_index;
 	pgoff_t			oti_fn_index; /* first non-overlapped index */
 	struct cl_sync_io	oti_anchor;
+	struct cl_req_attr	oti_req_attr;
 };
 
 struct osc_object {
@@ -127,16 +120,6 @@ struct osc_object {
 	int		oo_contended;
 	unsigned long	 oo_contention_time;
 	/**
-	 * List of pages in transfer.
-	 */
-	struct list_head	 oo_inflight[CRT_NR];
-	/**
-	 * Lock, protecting osc_page::ops_inflight, because a seat-belt is
-	 * locked during take-off and landing.
-	 */
-	spinlock_t	   oo_seatbelt;
-
-	/**
 	 * used by the osc to keep track of what objects to build into rpcs.
 	 * Protected by client_obd->cli_loi_list_lock.
 	 */
@@ -364,15 +347,6 @@ struct osc_page {
 	 */
 	struct list_head	      ops_lru;
 	/**
-	 * Linkage into a per-osc_object list of pages in flight. For
-	 * debugging.
-	 */
-	struct list_head	    ops_inflight;
-	/**
-	 * Thread that submitted this page for transfer. For debugging.
-	 */
-	struct task_struct	*ops_submitter;
-	/**
 	 * Submit time - the time when the page is starting RPC. For debugging.
 	 */
 	unsigned long	    ops_submit_time;
@@ -382,7 +356,6 @@ extern struct kmem_cache *osc_lock_kmem;
 extern struct kmem_cache *osc_object_kmem;
 extern struct kmem_cache *osc_thread_kmem;
 extern struct kmem_cache *osc_session_kmem;
-extern struct kmem_cache *osc_req_kmem;
 extern struct kmem_cache *osc_extent_kmem;
 
 extern struct lu_device_type osc_device_type;
@@ -396,15 +369,14 @@ int osc_lock_init(const struct lu_env *env,
 		  const struct cl_io *io);
 int osc_io_init(const struct lu_env *env,
 		struct cl_object *obj, struct cl_io *io);
-int osc_req_init(const struct lu_env *env, struct cl_device *dev,
-		 struct cl_req *req);
 struct lu_object *osc_object_alloc(const struct lu_env *env,
 				   const struct lu_object_header *hdr,
 				   struct lu_device *dev);
 int osc_page_init(const struct lu_env *env, struct cl_object *obj,
 		  struct cl_page *page, pgoff_t ind);
 
-void osc_index2policy(ldlm_policy_data_t *policy, const struct cl_object *obj,
+void osc_index2policy(union ldlm_policy_data *policy,
+		      const struct cl_object *obj,
 		      pgoff_t start, pgoff_t end);
 int osc_lvb_print(const struct lu_env *env, void *cookie,
 		  lu_printer_t p, const struct ost_lvb *lvb);
@@ -554,6 +526,16 @@ static inline struct osc_page *oap2osc_page(struct osc_async_page *oap)
 	return (struct osc_page *)container_of(oap, struct osc_page, ops_oap);
 }
 
+static inline struct osc_page *
+osc_cl_page_osc(struct cl_page *page, struct osc_object *osc)
+{
+	const struct cl_page_slice *slice;
+
+	LASSERT(osc);
+	slice = cl_object_page_slice(&osc->oo_cl, page);
+	return cl2osc_page(slice);
+}
+
 static inline struct osc_lock *cl2osc_lock(const struct cl_lock_slice *slice)
 {
 	LINVRNT(osc_is_object(&slice->cls_obj->co_lu));
@@ -615,6 +597,10 @@ struct osc_extent {
 			   oe_rw:1,
 	/** sync extent, queued by osc_queue_sync_pages() */
 				oe_sync:1,
+	/** set if this extent has partial, sync pages.
+	 * Extents with partial page(s) can't merge with others in RPC
+	 */
+				oe_no_merge:1,
 			   oe_srvlock:1,
 			   oe_memalloc:1,
 	/** an ACTIVE extent is going to be truncated, so when this extent
diff --git a/drivers/staging/lustre/lustre/osc/osc_dev.c b/drivers/staging/lustre/lustre/osc/osc_dev.c
index 83d30c135ba4..c5d62aeaeab5 100644
--- a/drivers/staging/lustre/lustre/osc/osc_dev.c
+++ b/drivers/staging/lustre/lustre/osc/osc_dev.c
@@ -29,7 +29,7 @@
  * This file is part of Lustre, http://www.lustre.org/
  * Lustre is a trademark of Sun Microsystems, Inc.
  *
- * Implementation of cl_device, cl_req for OSC layer.
+ * Implementation of cl_device, for OSC layer.
  *
  *   Author: Nikita Danilov <nikita.danilov@sun.com>
  */
@@ -49,7 +49,6 @@ struct kmem_cache *osc_lock_kmem;
 struct kmem_cache *osc_object_kmem;
 struct kmem_cache *osc_thread_kmem;
 struct kmem_cache *osc_session_kmem;
-struct kmem_cache *osc_req_kmem;
 struct kmem_cache *osc_extent_kmem;
 struct kmem_cache *osc_quota_kmem;
 
@@ -75,11 +74,6 @@ struct lu_kmem_descr osc_caches[] = {
 		.ckd_size  = sizeof(struct osc_session)
 	},
 	{
-		.ckd_cache = &osc_req_kmem,
-		.ckd_name  = "osc_req_kmem",
-		.ckd_size  = sizeof(struct osc_req)
-	},
-	{
 		.ckd_cache = &osc_extent_kmem,
 		.ckd_name  = "osc_extent_kmem",
 		.ckd_size  = sizeof(struct osc_extent)
@@ -94,8 +88,6 @@ struct lu_kmem_descr osc_caches[] = {
 	}
 };
 
-struct lock_class_key osc_ast_guard_class;
-
 /*****************************************************************************
  *
  * Type conversions.
@@ -178,10 +170,6 @@ static const struct lu_device_operations osc_lu_ops = {
 	.ldo_recovery_complete = NULL
 };
 
-static const struct cl_device_operations osc_cl_ops = {
-	.cdo_req_init = osc_req_init
-};
-
 static int osc_device_init(const struct lu_env *env, struct lu_device *d,
 			   const char *name, struct lu_device *next)
 {
@@ -220,7 +208,6 @@ static struct lu_device *osc_device_alloc(const struct lu_env *env,
 	cl_device_init(&od->od_cl, t);
 	d = osc2lu_dev(od);
 	d->ld_ops = &osc_lu_ops;
-	od->od_cl.cd_ops = &osc_cl_ops;
 
 	/* Setup OSC OBD */
 	obd = class_name2obd(lustre_cfg_string(cfg, 0));
diff --git a/drivers/staging/lustre/lustre/osc/osc_internal.h b/drivers/staging/lustre/lustre/osc/osc_internal.h
index 67fe0a254991..688783dcc1e4 100644
--- a/drivers/staging/lustre/lustre/osc/osc_internal.h
+++ b/drivers/staging/lustre/lustre/osc/osc_internal.h
@@ -107,26 +107,24 @@ typedef int (*osc_enqueue_upcall_f)(void *cookie, struct lustre_handle *lockh,
 				    int rc);
 
 int osc_enqueue_base(struct obd_export *exp, struct ldlm_res_id *res_id,
-		     __u64 *flags, ldlm_policy_data_t *policy,
+		     __u64 *flags, union ldlm_policy_data *policy,
 		     struct ost_lvb *lvb, int kms_valid,
 		     osc_enqueue_upcall_f upcall,
 		     void *cookie, struct ldlm_enqueue_info *einfo,
 		     struct ptlrpc_request_set *rqset, int async, int agl);
-int osc_cancel_base(struct lustre_handle *lockh, __u32 mode);
 
 int osc_match_base(struct obd_export *exp, struct ldlm_res_id *res_id,
-		   __u32 type, ldlm_policy_data_t *policy, __u32 mode,
+		   __u32 type, union ldlm_policy_data *policy, __u32 mode,
 		   __u64 *flags, void *data, struct lustre_handle *lockh,
 		   int unref);
 
-int osc_setattr_async_base(struct obd_export *exp, struct obd_info *oinfo,
-			   struct obd_trans_info *oti,
-			   obd_enqueue_update_f upcall, void *cookie,
-			   struct ptlrpc_request_set *rqset);
-int osc_punch_base(struct obd_export *exp, struct obd_info *oinfo,
+int osc_setattr_async(struct obd_export *exp, struct obdo *oa,
+		      obd_enqueue_update_f upcall, void *cookie,
+		      struct ptlrpc_request_set *rqset);
+int osc_punch_base(struct obd_export *exp, struct obdo *oa,
 		   obd_enqueue_update_f upcall, void *cookie,
 		   struct ptlrpc_request_set *rqset);
-int osc_sync_base(struct obd_export *exp, struct obd_info *oinfo,
+int osc_sync_base(struct osc_object *exp, struct obdo *oa,
 		  obd_enqueue_update_f upcall, void *cookie,
 		  struct ptlrpc_request_set *rqset);
 
@@ -135,7 +133,7 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 		  struct list_head *ext_list, int cmd);
 long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
 		    long target, bool force);
-long osc_lru_reclaim(struct client_obd *cli);
+long osc_lru_reclaim(struct client_obd *cli, unsigned long npages);
 
 unsigned long osc_ldlm_weigh_ast(struct ldlm_lock *dlmlock);
 
@@ -157,6 +155,11 @@ static inline unsigned long rpcs_in_flight(struct client_obd *cli)
 	return cli->cl_r_in_flight + cli->cl_w_in_flight;
 }
 
+static inline char *cli_name(struct client_obd *cli)
+{
+	return cli->cl_import->imp_obd->obd_name;
+}
+
 struct osc_device {
 	struct cl_device    od_cl;
 	struct obd_export  *od_exp;
@@ -192,15 +195,27 @@ int osc_quota_setdq(struct client_obd *cli, const unsigned int qid[],
 int osc_quota_chkdq(struct client_obd *cli, const unsigned int qid[]);
 int osc_quotactl(struct obd_device *unused, struct obd_export *exp,
 		 struct obd_quotactl *oqctl);
-int osc_quotacheck(struct obd_device *unused, struct obd_export *exp,
-		   struct obd_quotactl *oqctl);
-int osc_quota_poll_check(struct obd_export *exp, struct if_quotacheck *qchk);
 void osc_inc_unstable_pages(struct ptlrpc_request *req);
 void osc_dec_unstable_pages(struct ptlrpc_request *req);
 bool osc_over_unstable_soft_limit(struct client_obd *cli);
 
+/**
+ * Bit flags for osc_dlm_lock_at_pageoff().
+ */
+enum osc_dap_flags {
+	/**
+	 * Just check if the desired lock exists, it won't hold reference
+	 * count on lock.
+	 */
+	OSC_DAP_FL_TEST_LOCK	= BIT(0),
+	/**
+	 * Return the lock even if it is being canceled.
+	 */
+	OSC_DAP_FL_CANCELING	= BIT(1),
+};
+
 struct ldlm_lock *osc_dlmlock_at_pgoff(const struct lu_env *env,
 				       struct osc_object *obj, pgoff_t index,
-				       int pending, int canceling);
+				       enum osc_dap_flags flags);
 
 #endif /* OSC_INTERNAL_H */
diff --git a/drivers/staging/lustre/lustre/osc/osc_io.c b/drivers/staging/lustre/lustre/osc/osc_io.c
index 8a559cbcdd0c..228a97c098fe 100644
--- a/drivers/staging/lustre/lustre/osc/osc_io.c
+++ b/drivers/staging/lustre/lustre/osc/osc_io.c
@@ -49,12 +49,6 @@
  *
  */
 
-static struct osc_req *cl2osc_req(const struct cl_req_slice *slice)
-{
-	LINVRNT(slice->crs_dev->cd_lu_dev.ld_type == &osc_device_type);
-	return container_of0(slice, struct osc_req, or_cl);
-}
-
 static struct osc_io *cl2osc_io(const struct lu_env *env,
 				const struct cl_io_slice *slice)
 {
@@ -64,20 +58,6 @@ static struct osc_io *cl2osc_io(const struct lu_env *env,
 	return oio;
 }
 
-static struct osc_page *osc_cl_page_osc(struct cl_page *page,
-					struct osc_object *osc)
-{
-	const struct cl_page_slice *slice;
-
-	if (osc)
-		slice = cl_object_page_slice(&osc->oo_cl, page);
-	else
-		slice = cl_page_at(page, &osc_device_type);
-	LASSERT(slice);
-
-	return cl2osc_page(slice);
-}
-
 /*****************************************************************************
  *
  * io operations.
@@ -88,6 +68,45 @@ static void osc_io_fini(const struct lu_env *env, const struct cl_io_slice *io)
 {
 }
 
+static void osc_read_ahead_release(const struct lu_env *env, void *cbdata)
+{
+	struct ldlm_lock *dlmlock = cbdata;
+	struct lustre_handle lockh;
+
+	ldlm_lock2handle(dlmlock, &lockh);
+	ldlm_lock_decref(&lockh, LCK_PR);
+	LDLM_LOCK_PUT(dlmlock);
+}
+
+static int osc_io_read_ahead(const struct lu_env *env,
+			     const struct cl_io_slice *ios,
+			     pgoff_t start, struct cl_read_ahead *ra)
+{
+	struct osc_object *osc = cl2osc(ios->cis_obj);
+	struct ldlm_lock *dlmlock;
+	int result = -ENODATA;
+
+	dlmlock = osc_dlmlock_at_pgoff(env, osc, start, 0);
+	if (dlmlock) {
+		LASSERT(dlmlock->l_ast_data == osc);
+		if (dlmlock->l_req_mode != LCK_PR) {
+			struct lustre_handle lockh;
+
+			ldlm_lock2handle(dlmlock, &lockh);
+			ldlm_lock_addref(&lockh, LCK_PR);
+			ldlm_lock_decref(&lockh, dlmlock->l_req_mode);
+		}
+
+		ra->cra_end = cl_index(osc2cl(osc),
+				       dlmlock->l_policy_data.l_extent.end);
+		ra->cra_release = osc_read_ahead_release;
+		ra->cra_cbdata = dlmlock;
+		result = 0;
+	}
+
+	return result;
+}
+
 /**
  * An implementation of cl_io_operations::cio_io_submit() method for osc
  * layer. Iterates over pages in the in-queue, prepares each for io by calling
@@ -334,7 +353,7 @@ static int osc_io_rw_iter_init(const struct lu_env *env,
 		npages = max_pages;
 
 	c = atomic_long_read(cli->cl_lru_left);
-	if (c < npages && osc_lru_reclaim(cli) > 0)
+	if (c < npages && osc_lru_reclaim(cli, npages) > 0)
 		c = atomic_long_read(cli->cl_lru_left);
 	while (c >= npages) {
 		if (c == atomic_long_cmpxchg(cli->cl_lru_left, c, c - npages)) {
@@ -343,6 +362,17 @@ static int osc_io_rw_iter_init(const struct lu_env *env,
 		}
 		c = atomic_long_read(cli->cl_lru_left);
 	}
+	if (atomic_long_read(cli->cl_lru_left) < max_pages) {
+		/*
+		 * If there aren't enough pages in the per-OSC LRU then
+		 * wake up the LRU thread to try and clear out space, so
+		 * we don't block if pages are being dirtied quickly.
+		 */
+		CDEBUG(D_CACHE, "%s: queue LRU, left: %lu/%ld.\n",
+		       cli_name(cli), atomic_long_read(cli->cl_lru_left),
+		       max_pages);
+		(void)ptlrpcd_queue_work(cli->cl_lru_work);
+	}
 
 	return 0;
 }
@@ -446,7 +476,6 @@ static int osc_io_setattr_start(const struct lu_env *env,
 	__u64 size = io->u.ci_setattr.sa_attr.lvb_size;
 	unsigned int ia_valid = io->u.ci_setattr.sa_valid;
 	int result = 0;
-	struct obd_info oinfo = { };
 
 	/* truncate cache dirty pages first */
 	if (cl_io_is_trunc(io))
@@ -486,11 +515,19 @@ static int osc_io_setattr_start(const struct lu_env *env,
 		oa->o_oi = loi->loi_oi;
 		obdo_set_parent_fid(oa, io->u.ci_setattr.sa_parent_fid);
 		oa->o_stripe_idx = io->u.ci_setattr.sa_stripe_index;
-		oa->o_mtime = attr->cat_mtime;
-		oa->o_atime = attr->cat_atime;
-		oa->o_ctime = attr->cat_ctime;
-		oa->o_valid |= OBD_MD_FLID | OBD_MD_FLGROUP | OBD_MD_FLATIME |
-			       OBD_MD_FLCTIME | OBD_MD_FLMTIME;
+		oa->o_valid |= OBD_MD_FLID | OBD_MD_FLGROUP;
+		if (ia_valid & ATTR_CTIME) {
+			oa->o_valid |= OBD_MD_FLCTIME;
+			oa->o_ctime = attr->cat_ctime;
+		}
+		if (ia_valid & ATTR_ATIME) {
+			oa->o_valid |= OBD_MD_FLATIME;
+			oa->o_atime = attr->cat_atime;
+		}
+		if (ia_valid & ATTR_MTIME) {
+			oa->o_valid |= OBD_MD_FLMTIME;
+			oa->o_mtime = attr->cat_mtime;
+		}
 		if (ia_valid & ATTR_SIZE) {
 			oa->o_size = size;
 			oa->o_blocks = OBD_OBJECT_EOF;
@@ -503,19 +540,21 @@ static int osc_io_setattr_start(const struct lu_env *env,
 		} else {
 			LASSERT(oio->oi_lockless == 0);
 		}
+		if (ia_valid & ATTR_ATTR_FLAG) {
+			oa->o_flags = io->u.ci_setattr.sa_attr_flags;
+			oa->o_valid |= OBD_MD_FLFLAGS;
+		}
 
-		oinfo.oi_oa = oa;
 		init_completion(&cbargs->opc_sync);
 
 		if (ia_valid & ATTR_SIZE)
 			result = osc_punch_base(osc_export(cl2osc(obj)),
-						&oinfo, osc_async_upcall,
+						oa, osc_async_upcall,
 						cbargs, PTLRPCD_SET);
 		else
-			result = osc_setattr_async_base(osc_export(cl2osc(obj)),
-							&oinfo, NULL,
-							osc_async_upcall,
-							cbargs, PTLRPCD_SET);
+			result = osc_setattr_async(osc_export(cl2osc(obj)),
+						   oa, osc_async_upcall,
+						   cbargs, PTLRPCD_SET);
 		cbargs->opc_rpc_sent = result == 0;
 	}
 	return result;
@@ -557,6 +596,107 @@ static void osc_io_setattr_end(const struct lu_env *env,
 	}
 }
 
+struct osc_data_version_args {
+	struct osc_io *dva_oio;
+};
+
+static int
+osc_data_version_interpret(const struct lu_env *env, struct ptlrpc_request *req,
+			   void *arg, int rc)
+{
+	struct osc_data_version_args *dva = arg;
+	struct osc_io *oio = dva->dva_oio;
+	const struct ost_body *body;
+
+	if (rc < 0)
+		goto out;
+
+	body = req_capsule_server_get(&req->rq_pill, &RMF_OST_BODY);
+	if (!body) {
+		rc = -EPROTO;
+		goto out;
+	}
+
+	lustre_get_wire_obdo(&req->rq_import->imp_connect_data, &oio->oi_oa,
+			     &body->oa);
+out:
+	oio->oi_cbarg.opc_rc = rc;
+	complete(&oio->oi_cbarg.opc_sync);
+
+	return 0;
+}
+
+static int osc_io_data_version_start(const struct lu_env *env,
+				     const struct cl_io_slice *slice)
+{
+	struct cl_data_version_io *dv = &slice->cis_io->u.ci_data_version;
+	struct osc_io *oio = cl2osc_io(env, slice);
+	struct osc_async_cbargs *cbargs = &oio->oi_cbarg;
+	struct osc_object *obj = cl2osc(slice->cis_obj);
+	struct obd_export *exp = osc_export(obj);
+	struct lov_oinfo *loi = obj->oo_oinfo;
+	struct osc_data_version_args *dva;
+	struct obdo *oa = &oio->oi_oa;
+	struct ptlrpc_request *req;
+	struct ost_body *body;
+	int rc;
+
+	memset(oa, 0, sizeof(*oa));
+	oa->o_oi = loi->loi_oi;
+	oa->o_valid = OBD_MD_FLID | OBD_MD_FLGROUP;
+
+	if (dv->dv_flags & (LL_DV_RD_FLUSH | LL_DV_WR_FLUSH)) {
+		oa->o_valid |= OBD_MD_FLFLAGS;
+		oa->o_flags |= OBD_FL_SRVLOCK;
+		if (dv->dv_flags & LL_DV_WR_FLUSH)
+			oa->o_flags |= OBD_FL_FLUSH;
+	}
+
+	init_completion(&cbargs->opc_sync);
+
+	req = ptlrpc_request_alloc(class_exp2cliimp(exp), &RQF_OST_GETATTR);
+	if (!req)
+		return -ENOMEM;
+
+	rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_GETATTR);
+	if (rc < 0) {
+		ptlrpc_request_free(req);
+		return rc;
+	}
+
+	body = req_capsule_client_get(&req->rq_pill, &RMF_OST_BODY);
+	lustre_set_wire_obdo(&req->rq_import->imp_connect_data, &body->oa, oa);
+
+	ptlrpc_request_set_replen(req);
+	req->rq_interpret_reply = osc_data_version_interpret;
+	CLASSERT(sizeof(*dva) <= sizeof(req->rq_async_args));
+	dva = ptlrpc_req_async_args(req);
+	dva->dva_oio = oio;
+
+	ptlrpcd_add_req(req);
+
+	return 0;
+}
+
+static void osc_io_data_version_end(const struct lu_env *env,
+				    const struct cl_io_slice *slice)
+{
+	struct cl_data_version_io *dv = &slice->cis_io->u.ci_data_version;
+	struct osc_io *oio = cl2osc_io(env, slice);
+	struct osc_async_cbargs *cbargs = &oio->oi_cbarg;
+
+	wait_for_completion(&cbargs->opc_sync);
+
+	if (cbargs->opc_rc) {
+		slice->cis_io->ci_result = cbargs->opc_rc;
+	} else if (!(oio->oi_oa.o_valid & OBD_MD_FLDATAVERSION)) {
+		slice->cis_io->ci_result = -EOPNOTSUPP;
+	} else {
+		dv->dv_data_version = oio->oi_oa.o_data_version;
+		slice->cis_io->ci_result = 0;
+	}
+}
+
 static int osc_io_read_start(const struct lu_env *env,
 			     const struct cl_io_slice *slice)
 {
@@ -595,7 +735,6 @@ static int osc_fsync_ost(const struct lu_env *env, struct osc_object *obj,
 {
 	struct osc_io *oio = osc_env_io(env);
 	struct obdo *oa = &oio->oi_oa;
-	struct obd_info *oinfo = &oio->oi_info;
 	struct lov_oinfo *loi = obj->oo_oinfo;
 	struct osc_async_cbargs *cbargs = &oio->oi_cbarg;
 	int rc = 0;
@@ -611,12 +750,9 @@ static int osc_fsync_ost(const struct lu_env *env, struct osc_object *obj,
 
 	obdo_set_parent_fid(oa, fio->fi_fid);
 
-	memset(oinfo, 0, sizeof(*oinfo));
-	oinfo->oi_oa = oa;
 	init_completion(&cbargs->opc_sync);
 
-	rc = osc_sync_base(osc_export(obj), oinfo, osc_async_upcall, cbargs,
-			   PTLRPCD_SET);
+	rc = osc_sync_base(obj, oa, osc_async_upcall, cbargs, PTLRPCD_SET);
 	return rc;
 }
 
@@ -710,6 +846,10 @@ static const struct cl_io_operations osc_io_ops = {
 			.cio_start  = osc_io_setattr_start,
 			.cio_end    = osc_io_setattr_end
 		},
+		[CIT_DATA_VERSION] = {
+			.cio_start	= osc_io_data_version_start,
+			.cio_end	= osc_io_data_version_end,
+		},
 		[CIT_FAULT] = {
 			.cio_start  = osc_io_fault_start,
 			.cio_end    = osc_io_end,
@@ -724,6 +864,7 @@ static const struct cl_io_operations osc_io_ops = {
 			.cio_fini   = osc_io_fini
 		}
 	},
+	.cio_read_ahead			= osc_io_read_ahead,
 	.cio_submit                 = osc_io_submit,
 	.cio_commit_async           = osc_io_commit_async
 };
@@ -734,103 +875,6 @@ static const struct cl_io_operations osc_io_ops = {
  *
  */
 
-static int osc_req_prep(const struct lu_env *env,
-			const struct cl_req_slice *slice)
-{
-	return 0;
-}
-
-static void osc_req_completion(const struct lu_env *env,
-			       const struct cl_req_slice *slice, int ioret)
-{
-	struct osc_req *or;
-
-	or = cl2osc_req(slice);
-	kmem_cache_free(osc_req_kmem, or);
-}
-
-/**
- * Implementation of struct cl_req_operations::cro_attr_set() for osc
- * layer. osc is responsible for struct obdo::o_id and struct obdo::o_seq
- * fields.
- */
-static void osc_req_attr_set(const struct lu_env *env,
-			     const struct cl_req_slice *slice,
-			     const struct cl_object *obj,
-			     struct cl_req_attr *attr, u64 flags)
-{
-	struct lov_oinfo *oinfo;
-	struct cl_req *clerq;
-	struct cl_page *apage; /* _some_ page in @clerq */
-	struct ldlm_lock *lock;  /* _some_ lock protecting @apage */
-	struct osc_page *opg;
-	struct obdo *oa;
-	struct ost_lvb *lvb;
-
-	oinfo = cl2osc(obj)->oo_oinfo;
-	lvb = &oinfo->loi_lvb;
-	oa = attr->cra_oa;
-
-	if ((flags & OBD_MD_FLMTIME) != 0) {
-		oa->o_mtime = lvb->lvb_mtime;
-		oa->o_valid |= OBD_MD_FLMTIME;
-	}
-	if ((flags & OBD_MD_FLATIME) != 0) {
-		oa->o_atime = lvb->lvb_atime;
-		oa->o_valid |= OBD_MD_FLATIME;
-	}
-	if ((flags & OBD_MD_FLCTIME) != 0) {
-		oa->o_ctime = lvb->lvb_ctime;
-		oa->o_valid |= OBD_MD_FLCTIME;
-	}
-	if (flags & OBD_MD_FLGROUP) {
-		ostid_set_seq(&oa->o_oi, ostid_seq(&oinfo->loi_oi));
-		oa->o_valid |= OBD_MD_FLGROUP;
-	}
-	if (flags & OBD_MD_FLID) {
-		ostid_set_id(&oa->o_oi, ostid_id(&oinfo->loi_oi));
-		oa->o_valid |= OBD_MD_FLID;
-	}
-	if (flags & OBD_MD_FLHANDLE) {
-		clerq = slice->crs_req;
-		LASSERT(!list_empty(&clerq->crq_pages));
-		apage = container_of(clerq->crq_pages.next,
-				     struct cl_page, cp_flight);
-		opg = osc_cl_page_osc(apage, NULL);
-		lock = osc_dlmlock_at_pgoff(env, cl2osc(obj), osc_index(opg),
-					    1, 1);
-		if (!lock && !opg->ops_srvlock) {
-			struct ldlm_resource *res;
-			struct ldlm_res_id *resname;
-
-			CL_PAGE_DEBUG(D_ERROR, env, apage, "uncovered page!\n");
-
-			resname = &osc_env_info(env)->oti_resname;
-			ostid_build_res_name(&oinfo->loi_oi, resname);
-			res = ldlm_resource_get(
-				osc_export(cl2osc(obj))->exp_obd->obd_namespace,
-				NULL, resname, LDLM_EXTENT, 0);
-			ldlm_resource_dump(D_ERROR, res);
-
-			dump_stack();
-			LBUG();
-		}
-
-		/* check for lockless io. */
-		if (lock) {
-			oa->o_handle = lock->l_remote_handle;
-			oa->o_valid |= OBD_MD_FLHANDLE;
-			LDLM_LOCK_PUT(lock);
-		}
-	}
-}
-
-static const struct cl_req_operations osc_req_ops = {
-	.cro_prep       = osc_req_prep,
-	.cro_attr_set   = osc_req_attr_set,
-	.cro_completion = osc_req_completion
-};
-
 int osc_io_init(const struct lu_env *env,
 		struct cl_object *obj, struct cl_io *io)
 {
@@ -841,20 +885,4 @@ int osc_io_init(const struct lu_env *env,
 	return 0;
 }
 
-int osc_req_init(const struct lu_env *env, struct cl_device *dev,
-		 struct cl_req *req)
-{
-	struct osc_req *or;
-	int result;
-
-	or = kmem_cache_zalloc(osc_req_kmem, GFP_NOFS);
-	if (or) {
-		cl_req_slice_add(req, &or->or_cl, dev, &osc_req_ops);
-		result = 0;
-	} else {
-		result = -ENOMEM;
-	}
-	return result;
-}
-
 /** @} osc */
diff --git a/drivers/staging/lustre/lustre/osc/osc_lock.c b/drivers/staging/lustre/lustre/osc/osc_lock.c
index 39a8a5851603..5f799a4c78f9 100644
--- a/drivers/staging/lustre/lustre/osc/osc_lock.c
+++ b/drivers/staging/lustre/lustre/osc/osc_lock.c
@@ -145,7 +145,7 @@ static void osc_lock_fini(const struct lu_env *env,
 
 static void osc_lock_build_policy(const struct lu_env *env,
 				  const struct cl_lock *lock,
-				  ldlm_policy_data_t *policy)
+				  union ldlm_policy_data *policy)
 {
 	const struct cl_lock_descr *d = &lock->cll_descr;
 
@@ -188,7 +188,7 @@ static void osc_lock_lvb_update(const struct lu_env *env,
 	struct cl_object *obj = osc2cl(osc);
 	struct lov_oinfo *oinfo = osc->oo_oinfo;
 	struct cl_attr *attr = &osc_env_info(env)->oti_attr;
-	unsigned valid;
+	unsigned int valid;
 
 	valid = CAT_BLOCKS | CAT_ATIME | CAT_CTIME | CAT_MTIME | CAT_SIZE;
 	if (!lvb)
@@ -294,10 +294,10 @@ static int osc_lock_upcall(void *cookie, struct lustre_handle *lockh,
 	struct osc_lock *oscl = cookie;
 	struct cl_lock_slice *slice = &oscl->ols_cl;
 	struct lu_env *env;
-	struct cl_env_nest nest;
 	int rc;
+	int refcheck;
 
-	env = cl_env_nested_get(&nest);
+	env = cl_env_get(&refcheck);
 	/* should never happen, similar to osc_ldlm_blocking_ast(). */
 	LASSERT(!IS_ERR(env));
 
@@ -336,7 +336,7 @@ static int osc_lock_upcall(void *cookie, struct lustre_handle *lockh,
 
 	if (oscl->ols_owner)
 		cl_sync_io_note(env, oscl->ols_owner, rc);
-	cl_env_nested_put(&nest, env);
+	cl_env_put(env, &refcheck);
 
 	return rc;
 }
@@ -347,9 +347,9 @@ static int osc_lock_upcall_agl(void *cookie, struct lustre_handle *lockh,
 	struct osc_object *osc = cookie;
 	struct ldlm_lock *dlmlock;
 	struct lu_env *env;
-	struct cl_env_nest nest;
+	int refcheck;
 
-	env = cl_env_nested_get(&nest);
+	env = cl_env_get(&refcheck);
 	LASSERT(!IS_ERR(env));
 
 	if (errcode == ELDLM_LOCK_MATCHED) {
@@ -374,7 +374,7 @@ static int osc_lock_upcall_agl(void *cookie, struct lustre_handle *lockh,
 
 out:
 	cl_object_put(env, osc2cl(osc));
-	cl_env_nested_put(&nest, env);
+	cl_env_put(env, &refcheck);
 	return ldlm_error2errno(errcode);
 }
 
@@ -382,11 +382,11 @@ static int osc_lock_flush(struct osc_object *obj, pgoff_t start, pgoff_t end,
 			  enum cl_lock_mode mode, int discard)
 {
 	struct lu_env *env;
-	struct cl_env_nest nest;
+	int refcheck;
 	int rc = 0;
 	int rc2 = 0;
 
-	env = cl_env_nested_get(&nest);
+	env = cl_env_get(&refcheck);
 	if (IS_ERR(env))
 		return PTR_ERR(env);
 
@@ -404,7 +404,7 @@ static int osc_lock_flush(struct osc_object *obj, pgoff_t start, pgoff_t end,
 	if (rc == 0 && rc2 < 0)
 		rc = rc2;
 
-	cl_env_nested_put(&nest, env);
+	cl_env_put(env, &refcheck);
 	return rc;
 }
 
@@ -536,7 +536,7 @@ static int osc_ldlm_blocking_ast(struct ldlm_lock *dlmlock,
 	}
 	case LDLM_CB_CANCELING: {
 		struct lu_env *env;
-		struct cl_env_nest nest;
+		int refcheck;
 
 		/*
 		 * This can be called in the context of outer IO, e.g.,
@@ -549,14 +549,14 @@ static int osc_ldlm_blocking_ast(struct ldlm_lock *dlmlock,
 		 * new environment has to be created to not corrupt outer
 		 * context.
 		 */
-		env = cl_env_nested_get(&nest);
+		env = cl_env_get(&refcheck);
 		if (IS_ERR(env)) {
 			result = PTR_ERR(env);
 			break;
 		}
 
 		result = osc_dlm_blocking_ast0(env, dlmlock, data, flag);
-		cl_env_nested_put(&nest, env);
+		cl_env_put(env, &refcheck);
 		break;
 		}
 	default:
@@ -568,61 +568,63 @@ static int osc_ldlm_blocking_ast(struct ldlm_lock *dlmlock,
 static int osc_ldlm_glimpse_ast(struct ldlm_lock *dlmlock, void *data)
 {
 	struct ptlrpc_request *req = data;
-	struct cl_env_nest nest;
 	struct lu_env *env;
 	struct ost_lvb *lvb;
 	struct req_capsule *cap;
+	struct cl_object *obj = NULL;
 	int result;
+	int refcheck;
 
 	LASSERT(lustre_msg_get_opc(req->rq_reqmsg) == LDLM_GL_CALLBACK);
 
-	env = cl_env_nested_get(&nest);
-	if (!IS_ERR(env)) {
-		struct cl_object *obj = NULL;
+	env = cl_env_get(&refcheck);
+	if (IS_ERR(env)) {
+		result = PTR_ERR(env);
+		goto out;
+	}
 
-		lock_res_and_lock(dlmlock);
-		if (dlmlock->l_ast_data) {
-			obj = osc2cl(dlmlock->l_ast_data);
-			cl_object_get(obj);
-		}
-		unlock_res_and_lock(dlmlock);
+	lock_res_and_lock(dlmlock);
+	if (dlmlock->l_ast_data) {
+		obj = osc2cl(dlmlock->l_ast_data);
+		cl_object_get(obj);
+	}
+	unlock_res_and_lock(dlmlock);
 
-		if (obj) {
-			/* Do not grab the mutex of cl_lock for glimpse.
-			 * See LU-1274 for details.
-			 * BTW, it's okay for cl_lock to be cancelled during
-			 * this period because server can handle this race.
-			 * See ldlm_server_glimpse_ast() for details.
-			 * cl_lock_mutex_get(env, lock);
-			 */
-			cap = &req->rq_pill;
-			req_capsule_extend(cap, &RQF_LDLM_GL_CALLBACK);
-			req_capsule_set_size(cap, &RMF_DLM_LVB, RCL_SERVER,
-					     sizeof(*lvb));
-			result = req_capsule_server_pack(cap);
-			if (result == 0) {
-				lvb = req_capsule_server_get(cap, &RMF_DLM_LVB);
-				result = cl_object_glimpse(env, obj, lvb);
-			}
-			if (!exp_connect_lvb_type(req->rq_export))
-				req_capsule_shrink(&req->rq_pill,
-						   &RMF_DLM_LVB,
-						   sizeof(struct ost_lvb_v1),
-						   RCL_SERVER);
-			cl_object_put(env, obj);
-		} else {
-			/*
-			 * These errors are normal races, so we don't want to
-			 * fill the console with messages by calling
-			 * ptlrpc_error()
-			 */
-			lustre_pack_reply(req, 1, NULL, NULL);
-			result = -ELDLM_NO_LOCK_DATA;
+	if (obj) {
+		/* Do not grab the mutex of cl_lock for glimpse.
+		 * See LU-1274 for details.
+		 * BTW, it's okay for cl_lock to be cancelled during
+		 * this period because server can handle this race.
+		 * See ldlm_server_glimpse_ast() for details.
+		 * cl_lock_mutex_get(env, lock);
+		 */
+		cap = &req->rq_pill;
+		req_capsule_extend(cap, &RQF_LDLM_GL_CALLBACK);
+		req_capsule_set_size(cap, &RMF_DLM_LVB, RCL_SERVER,
+				     sizeof(*lvb));
+		result = req_capsule_server_pack(cap);
+		if (result == 0) {
+			lvb = req_capsule_server_get(cap, &RMF_DLM_LVB);
+			result = cl_object_glimpse(env, obj, lvb);
+		}
+		if (!exp_connect_lvb_type(req->rq_export)) {
+			req_capsule_shrink(&req->rq_pill, &RMF_DLM_LVB,
+					   sizeof(struct ost_lvb_v1),
+					   RCL_SERVER);
 		}
-		cl_env_nested_put(&nest, env);
+		cl_object_put(env, obj);
 	} else {
-		result = PTR_ERR(env);
+		/*
+		 * These errors are normal races, so we don't want to
+		 * fill the console with messages by calling
+		 * ptlrpc_error()
+		 */
+		lustre_pack_reply(req, 1, NULL, NULL);
+		result = -ELDLM_NO_LOCK_DATA;
 	}
+	cl_env_put(env, &refcheck);
+
+out:
 	req->rq_status = result;
 	return result;
 }
@@ -677,12 +679,12 @@ static unsigned long osc_lock_weight(const struct lu_env *env,
  */
 unsigned long osc_ldlm_weigh_ast(struct ldlm_lock *dlmlock)
 {
-	struct cl_env_nest       nest;
 	struct lu_env           *env;
 	struct osc_object	*obj;
 	struct osc_lock		*oscl;
 	unsigned long            weight;
 	bool			 found = false;
+	int refcheck;
 
 	might_sleep();
 	/*
@@ -692,7 +694,7 @@ unsigned long osc_ldlm_weigh_ast(struct ldlm_lock *dlmlock)
 	 * the upper context because cl_lock_put don't modify environment
 	 * variables. But just in case ..
 	 */
-	env = cl_env_nested_get(&nest);
+	env = cl_env_get(&refcheck);
 	if (IS_ERR(env))
 		/* Mostly because lack of memory, do not eliminate this lock */
 		return 1;
@@ -722,7 +724,7 @@ unsigned long osc_ldlm_weigh_ast(struct ldlm_lock *dlmlock)
 	weight = osc_lock_weight(env, obj, &dlmlock->l_policy_data.l_extent);
 
 out:
-	cl_env_nested_put(&nest, env);
+	cl_env_put(env, &refcheck);
 	return weight;
 }
 
@@ -912,7 +914,7 @@ static int osc_lock_enqueue(const struct lu_env *env,
 	struct osc_lock *oscl = cl2osc_lock(slice);
 	struct cl_lock *lock = slice->cls_lock;
 	struct ldlm_res_id *resname = &info->oti_resname;
-	ldlm_policy_data_t *policy = &info->oti_policy;
+	union ldlm_policy_data *policy = &info->oti_policy;
 	osc_enqueue_upcall_f upcall = osc_lock_upcall;
 	void *cookie = oscl;
 	bool async = false;
@@ -1009,7 +1011,7 @@ static void osc_lock_detach(const struct lu_env *env, struct osc_lock *olck)
 
 	if (olck->ols_hold) {
 		olck->ols_hold = 0;
-		osc_cancel_base(&olck->ols_handle, olck->ols_einfo.ei_mode);
+		ldlm_lock_decref(&olck->ols_handle, olck->ols_einfo.ei_mode);
 		olck->ols_handle.cookie = 0ULL;
 	}
 
@@ -1180,11 +1182,11 @@ int osc_lock_init(const struct lu_env *env,
  */
 struct ldlm_lock *osc_dlmlock_at_pgoff(const struct lu_env *env,
 				       struct osc_object *obj, pgoff_t index,
-				       int pending, int canceling)
+				       enum osc_dap_flags dap_flags)
 {
 	struct osc_thread_info *info = osc_env_info(env);
 	struct ldlm_res_id *resname = &info->oti_resname;
-	ldlm_policy_data_t *policy  = &info->oti_policy;
+	union ldlm_policy_data *policy = &info->oti_policy;
 	struct lustre_handle lockh;
 	struct ldlm_lock *lock = NULL;
 	enum ldlm_mode mode;
@@ -1194,17 +1196,18 @@ struct ldlm_lock *osc_dlmlock_at_pgoff(const struct lu_env *env,
 	osc_index2policy(policy, osc2cl(obj), index, index);
 	policy->l_extent.gid = LDLM_GID_ANY;
 
-	flags = LDLM_FL_BLOCK_GRANTED | LDLM_FL_TEST_LOCK;
-	if (pending)
-		flags |= LDLM_FL_CBPENDING;
+	flags = LDLM_FL_BLOCK_GRANTED | LDLM_FL_CBPENDING;
+	if (dap_flags & OSC_DAP_FL_TEST_LOCK)
+		flags |= LDLM_FL_TEST_LOCK;
+
 	/*
 	 * It is fine to match any group lock since there could be only one
 	 * with a uniq gid and it conflicts with all other lock modes too
 	 */
 again:
-	mode = ldlm_lock_match(osc_export(obj)->exp_obd->obd_namespace,
-			       flags, resname, LDLM_EXTENT, policy,
-			       LCK_PR | LCK_PW | LCK_GROUP, &lockh, canceling);
+	mode = osc_match_base(osc_export(obj), resname, LDLM_EXTENT, policy,
+			      LCK_PR | LCK_PW | LCK_GROUP, &flags, obj, &lockh,
+			      dap_flags & OSC_DAP_FL_CANCELING);
 	if (mode != 0) {
 		lock = ldlm_handle2lock(&lockh);
 		/* RACE: the lock is cancelled so let's try again */
diff --git a/drivers/staging/lustre/lustre/osc/osc_object.c b/drivers/staging/lustre/lustre/osc/osc_object.c
index aae3a2d4243f..e0c3324857dd 100644
--- a/drivers/staging/lustre/lustre/osc/osc_object.c
+++ b/drivers/staging/lustre/lustre/osc/osc_object.c
@@ -71,13 +71,8 @@ static int osc_object_init(const struct lu_env *env, struct lu_object *obj,
 {
 	struct osc_object *osc = lu2osc(obj);
 	const struct cl_object_conf *cconf = lu2cl_conf(conf);
-	int i;
 
 	osc->oo_oinfo = cconf->u.coc_oinfo;
-	spin_lock_init(&osc->oo_seatbelt);
-	for (i = 0; i < CRT_NR; ++i)
-		INIT_LIST_HEAD(&osc->oo_inflight[i]);
-
 	INIT_LIST_HEAD(&osc->oo_ready_item);
 	INIT_LIST_HEAD(&osc->oo_hp_ready_item);
 	INIT_LIST_HEAD(&osc->oo_write_item);
@@ -103,10 +98,6 @@ static int osc_object_init(const struct lu_env *env, struct lu_object *obj,
 static void osc_object_free(const struct lu_env *env, struct lu_object *obj)
 {
 	struct osc_object *osc = lu2osc(obj);
-	int i;
-
-	for (i = 0; i < CRT_NR; ++i)
-		LASSERT(list_empty(&osc->oo_inflight[i]));
 
 	LASSERT(list_empty(&osc->oo_ready_item));
 	LASSERT(list_empty(&osc->oo_hp_ready_item));
@@ -218,6 +209,94 @@ static int osc_object_prune(const struct lu_env *env, struct cl_object *obj)
 	return 0;
 }
 
+static int osc_object_fiemap(const struct lu_env *env, struct cl_object *obj,
+			     struct ll_fiemap_info_key *fmkey,
+			     struct fiemap *fiemap, size_t *buflen)
+{
+	struct obd_export *exp = osc_export(cl2osc(obj));
+	union ldlm_policy_data policy;
+	struct ptlrpc_request *req;
+	struct lustre_handle lockh;
+	struct ldlm_res_id resid;
+	enum ldlm_mode mode = 0;
+	struct fiemap *reply;
+	char *tmp;
+	int rc;
+
+	fmkey->lfik_oa.o_oi = cl2osc(obj)->oo_oinfo->loi_oi;
+	if (!(fmkey->lfik_fiemap.fm_flags & FIEMAP_FLAG_SYNC))
+		goto skip_locking;
+
+	policy.l_extent.start = fmkey->lfik_fiemap.fm_start & PAGE_MASK;
+
+	if (OBD_OBJECT_EOF - fmkey->lfik_fiemap.fm_length <=
+	    fmkey->lfik_fiemap.fm_start + PAGE_SIZE - 1)
+		policy.l_extent.end = OBD_OBJECT_EOF;
+	else
+		policy.l_extent.end = (fmkey->lfik_fiemap.fm_start +
+				       fmkey->lfik_fiemap.fm_length +
+				       PAGE_SIZE - 1) & PAGE_MASK;
+
+	ostid_build_res_name(&fmkey->lfik_oa.o_oi, &resid);
+	mode = ldlm_lock_match(exp->exp_obd->obd_namespace,
+			       LDLM_FL_BLOCK_GRANTED | LDLM_FL_LVB_READY,
+			       &resid, LDLM_EXTENT, &policy,
+			       LCK_PR | LCK_PW, &lockh, 0);
+	if (mode) { /* lock is cached on client */
+		if (mode != LCK_PR) {
+			ldlm_lock_addref(&lockh, LCK_PR);
+			ldlm_lock_decref(&lockh, LCK_PW);
+		}
+	} else { /* no cached lock, needs acquire lock on server side */
+		fmkey->lfik_oa.o_valid |= OBD_MD_FLFLAGS;
+		fmkey->lfik_oa.o_flags |= OBD_FL_SRVLOCK;
+	}
+
+skip_locking:
+	req = ptlrpc_request_alloc(class_exp2cliimp(exp),
+				   &RQF_OST_GET_INFO_FIEMAP);
+	if (!req) {
+		rc = -ENOMEM;
+		goto drop_lock;
+	}
+
+	req_capsule_set_size(&req->rq_pill, &RMF_FIEMAP_KEY, RCL_CLIENT,
+			     sizeof(*fmkey));
+	req_capsule_set_size(&req->rq_pill, &RMF_FIEMAP_VAL, RCL_CLIENT,
+			     *buflen);
+	req_capsule_set_size(&req->rq_pill, &RMF_FIEMAP_VAL, RCL_SERVER,
+			     *buflen);
+
+	rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_GET_INFO);
+	if (rc) {
+		ptlrpc_request_free(req);
+		goto drop_lock;
+	}
+	tmp = req_capsule_client_get(&req->rq_pill, &RMF_FIEMAP_KEY);
+	memcpy(tmp, fmkey, sizeof(*fmkey));
+	tmp = req_capsule_client_get(&req->rq_pill, &RMF_FIEMAP_VAL);
+	memcpy(tmp, fiemap, *buflen);
+	ptlrpc_request_set_replen(req);
+
+	rc = ptlrpc_queue_wait(req);
+	if (rc)
+		goto fini_req;
+
+	reply = req_capsule_server_get(&req->rq_pill, &RMF_FIEMAP_VAL);
+	if (!reply) {
+		rc = -EPROTO;
+		goto fini_req;
+	}
+
+	memcpy(fiemap, reply, *buflen);
+fini_req:
+	ptlrpc_req_finished(req);
+drop_lock:
+	if (mode)
+		ldlm_lock_decref(&lockh, LCK_PR);
+	return rc;
+}
+
 void osc_object_set_contended(struct osc_object *obj)
 {
 	obj->oo_contention_time = cfs_time_current();
@@ -256,6 +335,76 @@ int osc_object_is_contended(struct osc_object *obj)
 	return 1;
 }
 
+/**
+ * Implementation of struct cl_object_operations::coo_req_attr_set() for osc
+ * layer. osc is responsible for struct obdo::o_id and struct obdo::o_seq
+ * fields.
+ */
+static void osc_req_attr_set(const struct lu_env *env, struct cl_object *obj,
+			     struct cl_req_attr *attr)
+{
+	u64 flags = attr->cra_flags;
+	struct lov_oinfo *oinfo;
+	struct ost_lvb *lvb;
+	struct obdo *oa;
+
+	oinfo = cl2osc(obj)->oo_oinfo;
+	lvb = &oinfo->loi_lvb;
+	oa = attr->cra_oa;
+
+	if (flags & OBD_MD_FLMTIME) {
+		oa->o_mtime = lvb->lvb_mtime;
+		oa->o_valid |= OBD_MD_FLMTIME;
+	}
+	if (flags & OBD_MD_FLATIME) {
+		oa->o_atime = lvb->lvb_atime;
+		oa->o_valid |= OBD_MD_FLATIME;
+	}
+	if (flags & OBD_MD_FLCTIME) {
+		oa->o_ctime = lvb->lvb_ctime;
+		oa->o_valid |= OBD_MD_FLCTIME;
+	}
+	if (flags & OBD_MD_FLGROUP) {
+		ostid_set_seq(&oa->o_oi, ostid_seq(&oinfo->loi_oi));
+		oa->o_valid |= OBD_MD_FLGROUP;
+	}
+	if (flags & OBD_MD_FLID) {
+		ostid_set_id(&oa->o_oi, ostid_id(&oinfo->loi_oi));
+		oa->o_valid |= OBD_MD_FLID;
+	}
+	if (flags & OBD_MD_FLHANDLE) {
+		struct ldlm_lock *lock;
+		struct osc_page *opg;
+
+		opg = osc_cl_page_osc(attr->cra_page, cl2osc(obj));
+		lock = osc_dlmlock_at_pgoff(env, cl2osc(obj), osc_index(opg),
+					    OSC_DAP_FL_TEST_LOCK | OSC_DAP_FL_CANCELING);
+		if (!lock && !opg->ops_srvlock) {
+			struct ldlm_resource *res;
+			struct ldlm_res_id *resname;
+
+			CL_PAGE_DEBUG(D_ERROR, env, attr->cra_page,
+				      "uncovered page!\n");
+
+			resname = &osc_env_info(env)->oti_resname;
+			ostid_build_res_name(&oinfo->loi_oi, resname);
+			res = ldlm_resource_get(
+				osc_export(cl2osc(obj))->exp_obd->obd_namespace,
+				NULL, resname, LDLM_EXTENT, 0);
+			ldlm_resource_dump(D_ERROR, res);
+
+			LBUG();
+		}
+
+		/* check for lockless io. */
+		if (lock) {
+			oa->o_handle = lock->l_remote_handle;
+			oa->o_valid |= OBD_MD_FLHANDLE;
+			LDLM_LOCK_PUT(lock);
+		}
+	}
+}
+
 static const struct cl_object_operations osc_ops = {
 	.coo_page_init = osc_page_init,
 	.coo_lock_init = osc_lock_init,
@@ -263,7 +412,9 @@ static const struct cl_object_operations osc_ops = {
 	.coo_attr_get  = osc_attr_get,
 	.coo_attr_update = osc_attr_update,
 	.coo_glimpse   = osc_object_glimpse,
-	.coo_prune     = osc_object_prune
+	.coo_prune	 = osc_object_prune,
+	.coo_fiemap		= osc_object_fiemap,
+	.coo_req_attr_set	= osc_req_attr_set
 };
 
 static const struct lu_object_operations osc_lu_obj_ops = {
diff --git a/drivers/staging/lustre/lustre/osc/osc_page.c b/drivers/staging/lustre/lustre/osc/osc_page.c
index 2a7a70aa9e80..e356e4af08e1 100644
--- a/drivers/staging/lustre/lustre/osc/osc_page.c
+++ b/drivers/staging/lustre/lustre/osc/osc_page.c
@@ -37,6 +37,7 @@
 
 #define DEBUG_SUBSYSTEM S_OSC
 
+#include <linux/math64.h>
 #include "osc_cl_internal.h"
 
 static void osc_lru_del(struct client_obd *cli, struct osc_page *opg);
@@ -86,11 +87,6 @@ static void osc_page_transfer_add(const struct lu_env *env,
 	struct osc_object *obj = cl2osc(opg->ops_cl.cpl_obj);
 
 	osc_lru_use(osc_cli(obj), opg);
-
-	spin_lock(&obj->oo_seatbelt);
-	list_add(&opg->ops_inflight, &obj->oo_inflight[crt]);
-	opg->ops_submitter = current;
-	spin_unlock(&obj->oo_seatbelt);
 }
 
 int osc_page_cache_add(const struct lu_env *env,
@@ -109,7 +105,8 @@ int osc_page_cache_add(const struct lu_env *env,
 	return result;
 }
 
-void osc_index2policy(ldlm_policy_data_t *policy, const struct cl_object *obj,
+void osc_index2policy(union ldlm_policy_data *policy,
+		      const struct cl_object *obj,
 		      pgoff_t start, pgoff_t end)
 {
 	memset(policy, 0, sizeof(*policy));
@@ -117,25 +114,6 @@ void osc_index2policy(ldlm_policy_data_t *policy, const struct cl_object *obj,
 	policy->l_extent.end = cl_offset(obj, end + 1) - 1;
 }
 
-static int osc_page_is_under_lock(const struct lu_env *env,
-				  const struct cl_page_slice *slice,
-				  struct cl_io *unused, pgoff_t *max_index)
-{
-	struct osc_page *opg = cl2osc_page(slice);
-	struct ldlm_lock *dlmlock;
-	int result = -ENODATA;
-
-	dlmlock = osc_dlmlock_at_pgoff(env, cl2osc(slice->cpl_obj),
-				       osc_index(opg), 1, 0);
-	if (dlmlock) {
-		*max_index = cl_index(slice->cpl_obj,
-				      dlmlock->l_policy_data.l_extent.end);
-		LDLM_LOCK_PUT(dlmlock);
-		result = 0;
-	}
-	return result;
-}
-
 static const char *osc_list(struct list_head *head)
 {
 	return list_empty(head) ? "-" : "+";
@@ -158,7 +136,7 @@ static int osc_page_print(const struct lu_env *env,
 	struct osc_object *obj = cl2osc(slice->cpl_obj);
 	struct client_obd *cli = &osc_export(obj)->exp_obd->u.cli;
 
-	return (*printer)(env, cookie, LUSTRE_OSC_NAME "-page@%p %lu: 1< %#x %d %u %s %s > 2< %llu %u %u %#x %#x | %p %p %p > 3< %s %p %d %lu %d > 4< %d %d %d %lu %s | %s %s %s %s > 5< %s %s %s %s | %d %s | %d %s %s>\n",
+	return (*printer)(env, cookie, LUSTRE_OSC_NAME "-page@%p %lu: 1< %#x %d %u %s %s > 2< %llu %u %u %#x %#x | %p %p %p > 3< %d %lu %d > 4< %d %d %d %lu %s | %s %s %s %s > 5< %s %s %s %s | %d %s | %d %s %s>\n",
 			  opg, osc_index(opg),
 			  /* 1 */
 			  oap->oap_magic, oap->oap_cmd,
@@ -170,8 +148,7 @@ static int osc_page_print(const struct lu_env *env,
 			  oap->oap_async_flags, oap->oap_brw_flags,
 			  oap->oap_request, oap->oap_cli, obj,
 			  /* 3 */
-			  osc_list(&opg->ops_inflight),
-			  opg->ops_submitter, opg->ops_transfer_pinned,
+			  opg->ops_transfer_pinned,
 			  osc_submit_duration(opg), opg->ops_srvlock,
 			  /* 4 */
 			  cli->cl_r_in_flight, cli->cl_w_in_flight,
@@ -210,14 +187,6 @@ static void osc_page_delete(const struct lu_env *env,
 		LASSERT(0);
 	}
 
-	spin_lock(&obj->oo_seatbelt);
-	if (opg->ops_submitter) {
-		LASSERT(!list_empty(&opg->ops_inflight));
-		list_del_init(&opg->ops_inflight);
-		opg->ops_submitter = NULL;
-	}
-	spin_unlock(&obj->oo_seatbelt);
-
 	osc_lru_del(osc_cli(obj), opg);
 
 	if (slice->cpl_page->cp_type == CPT_CACHEABLE) {
@@ -276,7 +245,6 @@ static int osc_page_flush(const struct lu_env *env,
 static const struct cl_page_operations osc_page_ops = {
 	.cpo_print	 = osc_page_print,
 	.cpo_delete	= osc_page_delete,
-	.cpo_is_under_lock = osc_page_is_under_lock,
 	.cpo_clip	   = osc_page_clip,
 	.cpo_cancel	 = osc_page_cancel,
 	.cpo_flush	  = osc_page_flush
@@ -301,10 +269,6 @@ int osc_page_init(const struct lu_env *env, struct cl_object *obj,
 		cl_page_slice_add(page, &opg->ops_cl, obj, index,
 				  &osc_page_ops);
 	}
-	/* ops_inflight and ops_lru are the same field, but it doesn't
-	 * hurt to initialize it twice :-)
-	 */
-	INIT_LIST_HEAD(&opg->ops_inflight);
 	INIT_LIST_HEAD(&opg->ops_lru);
 
 	/* reserve an LRU space for this page */
@@ -362,16 +326,27 @@ void osc_page_submit(const struct lu_env *env, struct osc_page *opg,
  * OSC to free slots voluntarily to maintain a reasonable number of free slots
  * at any time.
  */
-
 static DECLARE_WAIT_QUEUE_HEAD(osc_lru_waitq);
-/* LRU pages are freed in batch mode. OSC should at least free this
- * number of pages to avoid running out of LRU budget, and..
+
+/**
+ * LRU pages are freed in batch mode. OSC should at least free this
+ * number of pages to avoid running out of LRU slots.
+ */
+static inline int lru_shrink_min(struct client_obd *cli)
+{
+	return cli->cl_max_pages_per_rpc * 2;
+}
+
+/**
+ * free this number at most otherwise it will take too long time to finish.
  */
-static const int lru_shrink_min = 2 << (20 - PAGE_SHIFT);  /* 2M */
-/* free this number at most otherwise it will take too long time to finish. */
-static const int lru_shrink_max = 8 << (20 - PAGE_SHIFT); /* 8M */
+static inline int lru_shrink_max(struct client_obd *cli)
+{
+	return cli->cl_max_pages_per_rpc * cli->cl_max_rpcs_in_flight;
+}
 
-/* Check if we can free LRU slots from this OSC. If there exists LRU waiters,
+/**
+ * Check if we can free LRU slots from this OSC. If there exists LRU waiters,
  * we should free slots aggressively. In this way, slots are freed in a steady
  * step to maintain fairness among OSCs.
  *
@@ -388,13 +363,20 @@ static int osc_cache_too_much(struct client_obd *cli)
 	/* if it's going to run out LRU slots, we should free some, but not
 	 * too much to maintain fairness among OSCs.
 	 */
-	if (atomic_long_read(cli->cl_lru_left) < cache->ccc_lru_max >> 4) {
+	if (atomic_long_read(cli->cl_lru_left) < cache->ccc_lru_max >> 2) {
 		if (pages >= budget)
-			return lru_shrink_max;
+			return lru_shrink_max(cli);
 		else if (pages >= budget / 2)
-			return lru_shrink_min;
-	} else if (pages >= budget * 2) {
-		return lru_shrink_min;
+			return lru_shrink_min(cli);
+	} else {
+		time64_t duration = ktime_get_real_seconds();
+
+		/* knock out pages by duration of no IO activity */
+		duration -= cli->cl_lru_last_used;
+		duration >>= 6; /* approximately 1 minute */
+		if (duration > 0 &&
+		    pages >= div64_s64((s64)budget, duration))
+			return lru_shrink_min(cli);
 	}
 	return 0;
 }
@@ -402,11 +384,21 @@ static int osc_cache_too_much(struct client_obd *cli)
 int lru_queue_work(const struct lu_env *env, void *data)
 {
 	struct client_obd *cli = data;
+	int count;
 
-	CDEBUG(D_CACHE, "Run LRU work for client obd %p.\n", cli);
+	CDEBUG(D_CACHE, "%s: run LRU work for client obd\n", cli_name(cli));
 
-	if (osc_cache_too_much(cli))
-		osc_lru_shrink(env, cli, lru_shrink_max, true);
+	count = osc_cache_too_much(cli);
+	if (count > 0) {
+		int rc = osc_lru_shrink(env, cli, count, false);
+
+		CDEBUG(D_CACHE, "%s: shrank %d/%d pages from client obd\n",
+		       cli_name(cli), rc, count);
+		if (rc >= count) {
+			CDEBUG(D_CACHE, "%s: queue again\n", cli_name(cli));
+			ptlrpcd_queue_work(cli->cl_lru_work);
+		}
+	}
 
 	return 0;
 }
@@ -433,10 +425,10 @@ void osc_lru_add_batch(struct client_obd *cli, struct list_head *plist)
 		list_splice_tail(&lru, &cli->cl_lru_list);
 		atomic_long_sub(npages, &cli->cl_lru_busy);
 		atomic_long_add(npages, &cli->cl_lru_in_list);
+		cli->cl_lru_last_used = ktime_get_real_seconds();
 		spin_unlock(&cli->cl_lru_list_lock);
 
-		/* XXX: May set force to be true for better performance */
-		if (osc_cache_too_much(cli))
+		if (waitqueue_active(&osc_lru_waitq))
 			(void)ptlrpcd_queue_work(cli->cl_lru_work);
 	}
 }
@@ -469,8 +461,10 @@ static void osc_lru_del(struct client_obd *cli, struct osc_page *opg)
 		 * this osc occupies too many LRU pages and kernel is
 		 * stealing one of them.
 		 */
-		if (!memory_pressure_get())
+		if (osc_cache_too_much(cli)) {
+			CDEBUG(D_CACHE, "%s: queue LRU work\n", cli_name(cli));
 			(void)ptlrpcd_queue_work(cli->cl_lru_work);
+		}
 		wake_up(&osc_lru_waitq);
 	} else {
 		LASSERT(list_empty(&opg->ops_lru));
@@ -502,6 +496,7 @@ static void discard_pagevec(const struct lu_env *env, struct cl_io *io,
 		struct cl_page *page = pvec[i];
 
 		LASSERT(cl_page_is_owned(page, io));
+		cl_page_delete(env, page);
 		cl_page_discard(env, io, page);
 		cl_page_disown(env, io, page);
 		cl_page_put(env, page);
@@ -542,7 +537,6 @@ long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
 	struct cl_object *clobj = NULL;
 	struct cl_page **pvec;
 	struct osc_page *opg;
-	struct osc_page *temp;
 	int maxscan = 0;
 	long count = 0;
 	int index = 0;
@@ -552,6 +546,8 @@ long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
 	if (atomic_long_read(&cli->cl_lru_in_list) == 0 || target <= 0)
 		return 0;
 
+	CDEBUG(D_CACHE, "%s: shrinkers: %d, force: %d\n",
+	       cli_name(cli), atomic_read(&cli->cl_lru_shrinkers), force);
 	if (!force) {
 		if (atomic_read(&cli->cl_lru_shrinkers) > 0)
 			return -EBUSY;
@@ -568,14 +564,21 @@ long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
 	io = &osc_env_info(env)->oti_io;
 
 	spin_lock(&cli->cl_lru_list_lock);
+	if (force)
+		cli->cl_lru_reclaim++;
 	maxscan = min(target << 1, atomic_long_read(&cli->cl_lru_in_list));
-	list_for_each_entry_safe(opg, temp, &cli->cl_lru_list, ops_lru) {
+	while (!list_empty(&cli->cl_lru_list)) {
 		struct cl_page *page;
 		bool will_free = false;
 
+		if (!force && atomic_read(&cli->cl_lru_shrinkers) > 1)
+			break;
+
 		if (--maxscan < 0)
 			break;
 
+		opg = list_entry(cli->cl_lru_list.next, struct osc_page,
+				 ops_lru);
 		page = opg->ops_cl.cpl_page;
 		if (lru_page_busy(cli, page)) {
 			list_move_tail(&opg->ops_lru, &cli->cl_lru_list);
@@ -662,34 +665,43 @@ long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli,
 	return count > 0 ? count : rc;
 }
 
-long osc_lru_reclaim(struct client_obd *cli)
+/**
+ * Reclaim LRU pages by an IO thread. The caller wants to reclaim at least
+ * \@npages of LRU slots. For performance consideration, it's better to drop
+ * LRU pages in batch. Therefore, the actual number is adjusted at least
+ * max_pages_per_rpc.
+ */
+long osc_lru_reclaim(struct client_obd *cli, unsigned long npages)
 {
-	struct cl_env_nest nest;
 	struct lu_env *env;
 	struct cl_client_cache *cache = cli->cl_cache;
 	int max_scans;
+	int refcheck;
 	long rc = 0;
 
 	LASSERT(cache);
 
-	env = cl_env_nested_get(&nest);
+	env = cl_env_get(&refcheck);
 	if (IS_ERR(env))
 		return 0;
 
-	rc = osc_lru_shrink(env, cli, osc_cache_too_much(cli), false);
-	if (rc != 0) {
-		if (rc == -EBUSY)
-			rc = 0;
-
-		CDEBUG(D_CACHE, "%s: Free %ld pages from own LRU: %p.\n",
-		       cli->cl_import->imp_obd->obd_name, rc, cli);
+	npages = max_t(int, npages, cli->cl_max_pages_per_rpc);
+	CDEBUG(D_CACHE, "%s: start to reclaim %ld pages from LRU\n",
+	       cli_name(cli), npages);
+	rc = osc_lru_shrink(env, cli, npages, true);
+	if (rc >= npages) {
+		CDEBUG(D_CACHE, "%s: reclaimed %ld/%ld pages from LRU\n",
+		       cli_name(cli), rc, npages);
+		if (osc_cache_too_much(cli) > 0)
+			ptlrpcd_queue_work(cli->cl_lru_work);
 		goto out;
+	} else if (rc > 0) {
+		npages -= rc;
 	}
 
-	CDEBUG(D_CACHE, "%s: cli %p no free slots, pages: %ld, busy: %ld.\n",
-	       cli->cl_import->imp_obd->obd_name, cli,
-	       atomic_long_read(&cli->cl_lru_in_list),
-	       atomic_long_read(&cli->cl_lru_busy));
+	CDEBUG(D_CACHE, "%s: cli %p no free slots, pages: %ld/%ld, want: %ld\n",
+	       cli_name(cli), cli, atomic_long_read(&cli->cl_lru_in_list),
+	       atomic_long_read(&cli->cl_lru_busy), npages);
 
 	/* Reclaim LRU slots from other client_obd as it can't free enough
 	 * from its own. This should rarely happen.
@@ -706,7 +718,7 @@ long osc_lru_reclaim(struct client_obd *cli)
 				 cl_lru_osc);
 
 		CDEBUG(D_CACHE, "%s: cli %p LRU pages: %ld, busy: %ld.\n",
-		       cli->cl_import->imp_obd->obd_name, cli,
+		       cli_name(cli), cli,
 		       atomic_long_read(&cli->cl_lru_in_list),
 		       atomic_long_read(&cli->cl_lru_busy));
 
@@ -714,19 +726,20 @@ long osc_lru_reclaim(struct client_obd *cli)
 		if (osc_cache_too_much(cli) > 0) {
 			spin_unlock(&cache->ccc_lru_lock);
 
-			rc = osc_lru_shrink(env, cli, osc_cache_too_much(cli),
-					    true);
+			rc = osc_lru_shrink(env, cli, npages, true);
 			spin_lock(&cache->ccc_lru_lock);
-			if (rc != 0)
+			if (rc >= npages)
 				break;
+			if (rc > 0)
+				npages -= rc;
 		}
 	}
 	spin_unlock(&cache->ccc_lru_lock);
 
 out:
-	cl_env_nested_put(&nest, env);
+	cl_env_put(env, &refcheck);
 	CDEBUG(D_CACHE, "%s: cli %p freed %ld pages.\n",
-	       cli->cl_import->imp_obd->obd_name, cli, rc);
+	       cli_name(cli), cli, rc);
 	return rc;
 }
 
@@ -756,7 +769,7 @@ static int osc_lru_reserve(const struct lu_env *env, struct osc_object *obj,
 	LASSERT(atomic_long_read(cli->cl_lru_left) >= 0);
 	while (!atomic_long_add_unless(cli->cl_lru_left, -1, 0)) {
 		/* run out of LRU spaces, try to drop some by itself */
-		rc = osc_lru_reclaim(cli);
+		rc = osc_lru_reclaim(cli, 1);
 		if (rc < 0)
 			break;
 		if (rc > 0)
@@ -796,8 +809,10 @@ static inline void unstable_page_accounting(struct ptlrpc_bulk_desc *desc,
 	int count = 0;
 	int i;
 
+	LASSERT(ptlrpc_is_bulk_desc_kiov(desc->bd_type));
+
 	for (i = 0; i < page_count; i++) {
-		pg_data_t *pgdat = page_pgdat(desc->bd_iov[i].bv_page);
+		pg_data_t *pgdat = page_pgdat(BD_GET_KIOV(desc, i).bv_page);
 
 		if (likely(pgdat == last)) {
 			++count;
@@ -857,7 +872,7 @@ void osc_dec_unstable_pages(struct ptlrpc_request *req)
 	if (!unstable_count)
 		wake_up_all(&cli->cl_cache->ccc_unstable_waitq);
 
-	if (osc_cache_too_much(cli))
+	if (waitqueue_active(&osc_lru_waitq))
 		(void)ptlrpcd_queue_work(cli->cl_lru_work);
 }
 
@@ -913,8 +928,7 @@ bool osc_over_unstable_soft_limit(struct client_obd *cli)
 
 	CDEBUG(D_CACHE,
 	       "%s: cli: %p unstable pages: %lu, osc unstable pages: %lu\n",
-	       cli->cl_import->imp_obd->obd_name, cli,
-	       unstable_nr, osc_unstable_count);
+	       cli_name(cli), cli, unstable_nr, osc_unstable_count);
 
 	/*
 	 * If the LRU slots are in shortage - 25% remaining AND this OSC
diff --git a/drivers/staging/lustre/lustre/osc/osc_quota.c b/drivers/staging/lustre/lustre/osc/osc_quota.c
index 194d8ede40a2..fed4da63ee45 100644
--- a/drivers/staging/lustre/lustre/osc/osc_quota.c
+++ b/drivers/staging/lustre/lustre/osc/osc_quota.c
@@ -106,7 +106,7 @@ int osc_quota_setdq(struct client_obd *cli, const unsigned int qid[],
 			}
 
 			CDEBUG(D_QUOTA, "%s: setdq to insert for %s %d (%d)\n",
-			       cli->cl_import->imp_obd->obd_name,
+			       cli_name(cli),
 			       type == USRQUOTA ? "user" : "group",
 			       qid[type], rc);
 		} else {
@@ -122,7 +122,7 @@ int osc_quota_setdq(struct client_obd *cli, const unsigned int qid[],
 				kmem_cache_free(osc_quota_kmem, oqi);
 
 			CDEBUG(D_QUOTA, "%s: setdq to remove for %s %d (%p)\n",
-			       cli->cl_import->imp_obd->obd_name,
+			       cli_name(cli),
 			       type == USRQUOTA ? "user" : "group",
 			       qid[type], oqi);
 		}
@@ -134,8 +134,8 @@ int osc_quota_setdq(struct client_obd *cli, const unsigned int qid[],
 /*
  * Hash operations for uid/gid <-> osc_quota_info
  */
-static unsigned
-oqi_hashfn(struct cfs_hash *hs, const void *key, unsigned mask)
+static unsigned int
+oqi_hashfn(struct cfs_hash *hs, const void *key, unsigned int mask)
 {
 	return cfs_hash_u32_hash(*((__u32 *)key), mask);
 }
@@ -281,47 +281,3 @@ int osc_quotactl(struct obd_device *unused, struct obd_export *exp,
 
 	return rc;
 }
-
-int osc_quotacheck(struct obd_device *unused, struct obd_export *exp,
-		   struct obd_quotactl *oqctl)
-{
-	struct client_obd *cli = &exp->exp_obd->u.cli;
-	struct ptlrpc_request *req;
-	struct obd_quotactl *body;
-	int rc;
-
-	req = ptlrpc_request_alloc_pack(class_exp2cliimp(exp),
-					&RQF_OST_QUOTACHECK, LUSTRE_OST_VERSION,
-					OST_QUOTACHECK);
-	if (!req)
-		return -ENOMEM;
-
-	body = req_capsule_client_get(&req->rq_pill, &RMF_OBD_QUOTACTL);
-	*body = *oqctl;
-
-	ptlrpc_request_set_replen(req);
-
-	/* the next poll will find -ENODATA, that means quotacheck is going on
-	 */
-	cli->cl_qchk_stat = -ENODATA;
-	rc = ptlrpc_queue_wait(req);
-	if (rc)
-		cli->cl_qchk_stat = rc;
-	ptlrpc_req_finished(req);
-	return rc;
-}
-
-int osc_quota_poll_check(struct obd_export *exp, struct if_quotacheck *qchk)
-{
-	struct client_obd *cli = &exp->exp_obd->u.cli;
-	int rc;
-
-	qchk->obd_uuid = cli->cl_target_uuid;
-	memcpy(qchk->obd_type, LUSTRE_OST_NAME, strlen(LUSTRE_OST_NAME));
-
-	rc = cli->cl_qchk_stat;
-	/* the client is not the previous one */
-	if (rc == CL_NOT_QUOTACHECKED)
-		rc = -EINTR;
-	return rc;
-}
diff --git a/drivers/staging/lustre/lustre/osc/osc_request.c b/drivers/staging/lustre/lustre/osc/osc_request.c
index 749781f022e2..7143564ae7e7 100644
--- a/drivers/staging/lustre/lustre/osc/osc_request.c
+++ b/drivers/staging/lustre/lustre/osc/osc_request.c
@@ -68,7 +68,6 @@ struct osc_brw_async_args {
 	struct client_obd *aa_cli;
 	struct list_head	 aa_oaps;
 	struct list_head	 aa_exts;
-	struct cl_req     *aa_clerq;
 };
 
 struct osc_async_args {
@@ -82,7 +81,8 @@ struct osc_setattr_args {
 };
 
 struct osc_fsync_args {
-	struct obd_info     *fa_oi;
+	struct osc_object	*fa_obj;
+	struct obdo		*fa_oa;
 	obd_enqueue_update_f fa_upcall;
 	void		*fa_cookie;
 };
@@ -103,140 +103,19 @@ static void osc_release_ppga(struct brw_page **ppga, u32 count);
 static int brw_interpret(const struct lu_env *env,
 			 struct ptlrpc_request *req, void *data, int rc);
 
-/* Unpack OSC object metadata from disk storage (LE byte order). */
-static int osc_unpackmd(struct obd_export *exp, struct lov_stripe_md **lsmp,
-			struct lov_mds_md *lmm, int lmm_bytes)
-{
-	int lsm_size;
-	struct obd_import *imp = class_exp2cliimp(exp);
-
-	if (lmm) {
-		if (lmm_bytes < sizeof(*lmm)) {
-			CERROR("%s: lov_mds_md too small: %d, need %d\n",
-			       exp->exp_obd->obd_name, lmm_bytes,
-			       (int)sizeof(*lmm));
-			return -EINVAL;
-		}
-		/* XXX LOV_MAGIC etc check? */
-
-		if (unlikely(ostid_id(&lmm->lmm_oi) == 0)) {
-			CERROR("%s: zero lmm_object_id: rc = %d\n",
-			       exp->exp_obd->obd_name, -EINVAL);
-			return -EINVAL;
-		}
-	}
-
-	lsm_size = lov_stripe_md_size(1);
-	if (!lsmp)
-		return lsm_size;
-
-	if (*lsmp && !lmm) {
-		kfree((*lsmp)->lsm_oinfo[0]);
-		kfree(*lsmp);
-		*lsmp = NULL;
-		return 0;
-	}
-
-	if (!*lsmp) {
-		*lsmp = kzalloc(lsm_size, GFP_NOFS);
-		if (unlikely(!*lsmp))
-			return -ENOMEM;
-		(*lsmp)->lsm_oinfo[0] = kzalloc(sizeof(struct lov_oinfo),
-						GFP_NOFS);
-		if (unlikely(!(*lsmp)->lsm_oinfo[0])) {
-			kfree(*lsmp);
-			return -ENOMEM;
-		}
-		loi_init((*lsmp)->lsm_oinfo[0]);
-	} else if (unlikely(ostid_id(&(*lsmp)->lsm_oi) == 0)) {
-		return -EBADF;
-	}
-
-	if (lmm)
-		/* XXX zero *lsmp? */
-		ostid_le_to_cpu(&lmm->lmm_oi, &(*lsmp)->lsm_oi);
-
-	if (imp &&
-	    (imp->imp_connect_data.ocd_connect_flags & OBD_CONNECT_MAXBYTES))
-		(*lsmp)->lsm_maxbytes = imp->imp_connect_data.ocd_maxbytes;
-	else
-		(*lsmp)->lsm_maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES;
-
-	return lsm_size;
-}
-
 static inline void osc_pack_req_body(struct ptlrpc_request *req,
-				     struct obd_info *oinfo)
+				     struct obdo *oa)
 {
 	struct ost_body *body;
 
 	body = req_capsule_client_get(&req->rq_pill, &RMF_OST_BODY);
 	LASSERT(body);
 
-	lustre_set_wire_obdo(&req->rq_import->imp_connect_data, &body->oa,
-			     oinfo->oi_oa);
-}
-
-static int osc_getattr_interpret(const struct lu_env *env,
-				 struct ptlrpc_request *req,
-				 struct osc_async_args *aa, int rc)
-{
-	struct ost_body *body;
-
-	if (rc != 0)
-		goto out;
-
-	body = req_capsule_server_get(&req->rq_pill, &RMF_OST_BODY);
-	if (body) {
-		CDEBUG(D_INODE, "mode: %o\n", body->oa.o_mode);
-		lustre_get_wire_obdo(&req->rq_import->imp_connect_data,
-				     aa->aa_oi->oi_oa, &body->oa);
-
-		/* This should really be sent by the OST */
-		aa->aa_oi->oi_oa->o_blksize = DT_MAX_BRW_SIZE;
-		aa->aa_oi->oi_oa->o_valid |= OBD_MD_FLBLKSZ;
-	} else {
-		CDEBUG(D_INFO, "can't unpack ost_body\n");
-		rc = -EPROTO;
-		aa->aa_oi->oi_oa->o_valid = 0;
-	}
-out:
-	rc = aa->aa_oi->oi_cb_up(aa->aa_oi, rc);
-	return rc;
-}
-
-static int osc_getattr_async(struct obd_export *exp, struct obd_info *oinfo,
-			     struct ptlrpc_request_set *set)
-{
-	struct ptlrpc_request *req;
-	struct osc_async_args *aa;
-	int rc;
-
-	req = ptlrpc_request_alloc(class_exp2cliimp(exp), &RQF_OST_GETATTR);
-	if (!req)
-		return -ENOMEM;
-
-	rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_GETATTR);
-	if (rc) {
-		ptlrpc_request_free(req);
-		return rc;
-	}
-
-	osc_pack_req_body(req, oinfo);
-
-	ptlrpc_request_set_replen(req);
-	req->rq_interpret_reply = (ptlrpc_interpterer_t)osc_getattr_interpret;
-
-	CLASSERT(sizeof(*aa) <= sizeof(req->rq_async_args));
-	aa = ptlrpc_req_async_args(req);
-	aa->aa_oi = oinfo;
-
-	ptlrpc_set_add_req(set, req);
-	return 0;
+	lustre_set_wire_obdo(&req->rq_import->imp_connect_data, &body->oa, oa);
 }
 
 static int osc_getattr(const struct lu_env *env, struct obd_export *exp,
-		       struct obd_info *oinfo)
+		       struct obdo *oa)
 {
 	struct ptlrpc_request *req;
 	struct ost_body *body;
@@ -252,7 +131,7 @@ static int osc_getattr(const struct lu_env *env, struct obd_export *exp,
 		return rc;
 	}
 
-	osc_pack_req_body(req, oinfo);
+	osc_pack_req_body(req, oa);
 
 	ptlrpc_request_set_replen(req);
 
@@ -267,11 +146,11 @@ static int osc_getattr(const struct lu_env *env, struct obd_export *exp,
 	}
 
 	CDEBUG(D_INODE, "mode: %o\n", body->oa.o_mode);
-	lustre_get_wire_obdo(&req->rq_import->imp_connect_data, oinfo->oi_oa,
+	lustre_get_wire_obdo(&req->rq_import->imp_connect_data, oa,
 			     &body->oa);
 
-	oinfo->oi_oa->o_blksize = cli_brw_size(exp->exp_obd);
-	oinfo->oi_oa->o_valid |= OBD_MD_FLBLKSZ;
+	oa->o_blksize = cli_brw_size(exp->exp_obd);
+	oa->o_valid |= OBD_MD_FLBLKSZ;
 
  out:
 	ptlrpc_req_finished(req);
@@ -279,13 +158,13 @@ static int osc_getattr(const struct lu_env *env, struct obd_export *exp,
 }
 
 static int osc_setattr(const struct lu_env *env, struct obd_export *exp,
-		       struct obd_info *oinfo, struct obd_trans_info *oti)
+		       struct obdo *oa)
 {
 	struct ptlrpc_request *req;
 	struct ost_body *body;
 	int rc;
 
-	LASSERT(oinfo->oi_oa->o_valid & OBD_MD_FLGROUP);
+	LASSERT(oa->o_valid & OBD_MD_FLGROUP);
 
 	req = ptlrpc_request_alloc(class_exp2cliimp(exp), &RQF_OST_SETATTR);
 	if (!req)
@@ -297,7 +176,7 @@ static int osc_setattr(const struct lu_env *env, struct obd_export *exp,
 		return rc;
 	}
 
-	osc_pack_req_body(req, oinfo);
+	osc_pack_req_body(req, oa);
 
 	ptlrpc_request_set_replen(req);
 
@@ -311,7 +190,7 @@ static int osc_setattr(const struct lu_env *env, struct obd_export *exp,
 		goto out;
 	}
 
-	lustre_get_wire_obdo(&req->rq_import->imp_connect_data, oinfo->oi_oa,
+	lustre_get_wire_obdo(&req->rq_import->imp_connect_data, oa,
 			     &body->oa);
 
 out:
@@ -341,10 +220,9 @@ out:
 	return rc;
 }
 
-int osc_setattr_async_base(struct obd_export *exp, struct obd_info *oinfo,
-			   struct obd_trans_info *oti,
-			   obd_enqueue_update_f upcall, void *cookie,
-			   struct ptlrpc_request_set *rqset)
+int osc_setattr_async(struct obd_export *exp, struct obdo *oa,
+		      obd_enqueue_update_f upcall, void *cookie,
+		      struct ptlrpc_request_set *rqset)
 {
 	struct ptlrpc_request *req;
 	struct osc_setattr_args *sa;
@@ -360,10 +238,7 @@ int osc_setattr_async_base(struct obd_export *exp, struct obd_info *oinfo,
 		return rc;
 	}
 
-	if (oti && oinfo->oi_oa->o_valid & OBD_MD_FLCOOKIE)
-		oinfo->oi_oa->o_lcookie = *oti->oti_logcookies;
-
-	osc_pack_req_body(req, oinfo);
+	osc_pack_req_body(req, oa);
 
 	ptlrpc_request_set_replen(req);
 
@@ -377,7 +252,7 @@ int osc_setattr_async_base(struct obd_export *exp, struct obd_info *oinfo,
 
 		CLASSERT(sizeof(*sa) <= sizeof(req->rq_async_args));
 		sa = ptlrpc_req_async_args(req);
-		sa->sa_oa = oinfo->oi_oa;
+		sa->sa_oa = oa;
 		sa->sa_upcall = upcall;
 		sa->sa_cookie = cookie;
 
@@ -390,16 +265,8 @@ int osc_setattr_async_base(struct obd_export *exp, struct obd_info *oinfo,
 	return 0;
 }
 
-static int osc_setattr_async(struct obd_export *exp, struct obd_info *oinfo,
-			     struct obd_trans_info *oti,
-			     struct ptlrpc_request_set *rqset)
-{
-	return osc_setattr_async_base(exp, oinfo, oti,
-				      oinfo->oi_cb_up, oinfo, rqset);
-}
-
 static int osc_create(const struct lu_env *env, struct obd_export *exp,
-		      struct obdo *oa, struct obd_trans_info *oti)
+		      struct obdo *oa)
 {
 	struct ptlrpc_request *req;
 	struct ost_body *body;
@@ -428,15 +295,6 @@ static int osc_create(const struct lu_env *env, struct obd_export *exp,
 
 	ptlrpc_request_set_replen(req);
 
-	if ((oa->o_valid & OBD_MD_FLFLAGS) &&
-	    oa->o_flags == OBD_FL_DELORPHAN) {
-		DEBUG_REQ(D_HA, req,
-			  "delorphan from OST integration");
-		/* Don't resend the delorphan req */
-		req->rq_no_resend = 1;
-		req->rq_no_delay = 1;
-	}
-
 	rc = ptlrpc_queue_wait(req);
 	if (rc)
 		goto out_req;
@@ -453,12 +311,6 @@ static int osc_create(const struct lu_env *env, struct obd_export *exp,
 	oa->o_blksize = cli_brw_size(exp->exp_obd);
 	oa->o_valid |= OBD_MD_FLBLKSZ;
 
-	if (oti && oa->o_valid & OBD_MD_FLCOOKIE) {
-		if (!oti->oti_logcookies)
-			oti->oti_logcookies = &oti->oti_onecookie;
-		*oti->oti_logcookies = oa->o_lcookie;
-	}
-
 	CDEBUG(D_HA, "transno: %lld\n",
 	       lustre_msg_get_transno(req->rq_repmsg));
 out_req:
@@ -467,7 +319,7 @@ out:
 	return rc;
 }
 
-int osc_punch_base(struct obd_export *exp, struct obd_info *oinfo,
+int osc_punch_base(struct obd_export *exp, struct obdo *oa,
 		   obd_enqueue_update_f upcall, void *cookie,
 		   struct ptlrpc_request_set *rqset)
 {
@@ -491,14 +343,14 @@ int osc_punch_base(struct obd_export *exp, struct obd_info *oinfo,
 	body = req_capsule_client_get(&req->rq_pill, &RMF_OST_BODY);
 	LASSERT(body);
 	lustre_set_wire_obdo(&req->rq_import->imp_connect_data, &body->oa,
-			     oinfo->oi_oa);
+			     oa);
 
 	ptlrpc_request_set_replen(req);
 
 	req->rq_interpret_reply = (ptlrpc_interpterer_t)osc_setattr_interpret;
 	CLASSERT(sizeof(*sa) <= sizeof(req->rq_async_args));
 	sa = ptlrpc_req_async_args(req);
-	sa->sa_oa = oinfo->oi_oa;
+	sa->sa_oa = oa;
 	sa->sa_upcall = upcall;
 	sa->sa_cookie = cookie;
 	if (rqset == PTLRPCD_SET)
@@ -513,8 +365,11 @@ static int osc_sync_interpret(const struct lu_env *env,
 			      struct ptlrpc_request *req,
 			      void *arg, int rc)
 {
+	struct cl_attr *attr = &osc_env_info(env)->oti_attr;
 	struct osc_fsync_args *fa = arg;
+	unsigned long valid = 0;
 	struct ost_body *body;
+	struct cl_object *obj;
 
 	if (rc)
 		goto out;
@@ -526,16 +381,30 @@ static int osc_sync_interpret(const struct lu_env *env,
 		goto out;
 	}
 
-	*fa->fa_oi->oi_oa = body->oa;
+	*fa->fa_oa = body->oa;
+	obj = osc2cl(fa->fa_obj);
+
+	/* Update osc object's blocks attribute */
+	cl_object_attr_lock(obj);
+	if (body->oa.o_valid & OBD_MD_FLBLOCKS) {
+		attr->cat_blocks = body->oa.o_blocks;
+		valid |= CAT_BLOCKS;
+	}
+
+	if (valid)
+		cl_object_attr_update(env, obj, attr, valid);
+	cl_object_attr_unlock(obj);
+
 out:
 	rc = fa->fa_upcall(fa->fa_cookie, rc);
 	return rc;
 }
 
-int osc_sync_base(struct obd_export *exp, struct obd_info *oinfo,
+int osc_sync_base(struct osc_object *obj, struct obdo *oa,
 		  obd_enqueue_update_f upcall, void *cookie,
 		  struct ptlrpc_request_set *rqset)
 {
+	struct obd_export *exp = osc_export(obj);
 	struct ptlrpc_request *req;
 	struct ost_body *body;
 	struct osc_fsync_args *fa;
@@ -555,14 +424,15 @@ int osc_sync_base(struct obd_export *exp, struct obd_info *oinfo,
 	body = req_capsule_client_get(&req->rq_pill, &RMF_OST_BODY);
 	LASSERT(body);
 	lustre_set_wire_obdo(&req->rq_import->imp_connect_data, &body->oa,
-			     oinfo->oi_oa);
+			     oa);
 
 	ptlrpc_request_set_replen(req);
 	req->rq_interpret_reply = osc_sync_interpret;
 
 	CLASSERT(sizeof(*fa) <= sizeof(req->rq_async_args));
 	fa = ptlrpc_req_async_args(req);
-	fa->fa_oi = oinfo;
+	fa->fa_obj = obj;
+	fa->fa_oa = oa;
 	fa->fa_upcall = upcall;
 	fa->fa_cookie = cookie;
 
@@ -639,19 +509,8 @@ static int osc_can_send_destroy(struct client_obd *cli)
 	return 0;
 }
 
-/* Destroy requests can be async always on the client, and we don't even really
- * care about the return code since the client cannot do anything at all about
- * a destroy failure.
- * When the MDS is unlinking a filename, it saves the file objects into a
- * recovery llog, and these object records are cancelled when the OST reports
- * they were destroyed and sync'd to disk (i.e. transaction committed).
- * If the client dies, or the OST is down when the object should be destroyed,
- * the records are not cancelled, and when the OST reconnects to the MDS next,
- * it will retrieve the llog unlink logs and then sends the log cancellation
- * cookies to the MDS after committing destroy transactions.
- */
 static int osc_destroy(const struct lu_env *env, struct obd_export *exp,
-		       struct obdo *oa, struct obd_trans_info *oti)
+		       struct obdo *oa)
 {
 	struct client_obd *cli = &exp->exp_obd->u.cli;
 	struct ptlrpc_request *req;
@@ -683,32 +542,22 @@ static int osc_destroy(const struct lu_env *env, struct obd_export *exp,
 	req->rq_request_portal = OST_IO_PORTAL; /* bug 7198 */
 	ptlrpc_at_set_req_timeout(req);
 
-	if (oti && oa->o_valid & OBD_MD_FLCOOKIE)
-		oa->o_lcookie = *oti->oti_logcookies;
 	body = req_capsule_client_get(&req->rq_pill, &RMF_OST_BODY);
 	LASSERT(body);
 	lustre_set_wire_obdo(&req->rq_import->imp_connect_data, &body->oa, oa);
 
 	ptlrpc_request_set_replen(req);
 
-	/* If osc_destroy is for destroying the unlink orphan,
-	 * sent from MDT to OST, which should not be blocked here,
-	 * because the process might be triggered by ptlrpcd, and
-	 * it is not good to block ptlrpcd thread (b=16006
-	 **/
-	if (!(oa->o_flags & OBD_FL_DELORPHAN)) {
-		req->rq_interpret_reply = osc_destroy_interpret;
-		if (!osc_can_send_destroy(cli)) {
-			struct l_wait_info lwi = LWI_INTR(LWI_ON_SIGNAL_NOOP,
-							  NULL);
-
-			/*
-			 * Wait until the number of on-going destroy RPCs drops
-			 * under max_rpc_in_flight
-			 */
-			l_wait_event_exclusive(cli->cl_destroy_waitq,
-					       osc_can_send_destroy(cli), &lwi);
-		}
+	req->rq_interpret_reply = osc_destroy_interpret;
+	if (!osc_can_send_destroy(cli)) {
+		struct l_wait_info lwi = LWI_INTR(LWI_ON_SIGNAL_NOOP, NULL);
+
+		/*
+		 * Wait until the number of on-going destroy RPCs drops
+		 * under max_rpc_in_flight
+		 */
+		l_wait_event_exclusive(cli->cl_destroy_waitq,
+				       osc_can_send_destroy(cli), &lwi);
 	}
 
 	/* Do not wait for response */
@@ -734,14 +583,13 @@ static void osc_announce_cached(struct client_obd *cli, struct obdo *oa,
 		oa->o_undirty = 0;
 	} else if (unlikely(atomic_long_read(&obd_dirty_pages) -
 			    atomic_long_read(&obd_dirty_transit_pages) >
-			    (obd_max_dirty_pages + 1))) {
+			    (long)(obd_max_dirty_pages + 1))) {
 		/* The atomic_read() allowing the atomic_inc() are
 		 * not covered by a lock thus they may safely race and trip
 		 * this CERROR() unless we add in a small fudge factor (+1).
 		 */
-		CERROR("%s: dirty %ld + %ld > system dirty_max %lu\n",
-		       cli->cl_import->imp_obd->obd_name,
-		       atomic_long_read(&obd_dirty_pages),
+		CERROR("%s: dirty %ld + %ld > system dirty_max %ld\n",
+		       cli_name(cli), atomic_long_read(&obd_dirty_pages),
 		       atomic_long_read(&obd_dirty_transit_pages),
 		       obd_max_dirty_pages);
 		oa->o_undirty = 0;
@@ -936,12 +784,10 @@ static int osc_add_shrink_grant(struct client_obd *client)
 				       osc_grant_shrink_grant_cb, NULL,
 				       &client->cl_grant_shrink_list);
 	if (rc) {
-		CERROR("add grant client %s error %d\n",
-		       client->cl_import->imp_obd->obd_name, rc);
+		CERROR("add grant client %s error %d\n", cli_name(client), rc);
 		return rc;
 	}
-	CDEBUG(D_CACHE, "add grant client %s\n",
-	       client->cl_import->imp_obd->obd_name);
+	CDEBUG(D_CACHE, "add grant client %s\n", cli_name(client));
 	osc_update_next_shrink(client);
 	return 0;
 }
@@ -970,23 +816,13 @@ static void osc_init_grant(struct client_obd *cli, struct obd_connect_data *ocd)
 		cli->cl_avail_grant = ocd->ocd_grant -
 				      (cli->cl_dirty_pages << PAGE_SHIFT);
 
-	if (cli->cl_avail_grant < 0) {
-		CWARN("%s: available grant < 0: avail/ocd/dirty %ld/%u/%ld\n",
-		      cli->cl_import->imp_obd->obd_name, cli->cl_avail_grant,
-		      ocd->ocd_grant, cli->cl_dirty_pages << PAGE_SHIFT);
-		/* workaround for servers which do not have the patch from
-		 * LU-2679
-		 */
-		cli->cl_avail_grant = ocd->ocd_grant;
-	}
-
 	/* determine the appropriate chunk size used by osc_extent. */
 	cli->cl_chunkbits = max_t(int, PAGE_SHIFT, ocd->ocd_blocksize);
 	spin_unlock(&cli->cl_loi_list_lock);
 
 	CDEBUG(D_CACHE, "%s, setting cl_avail_grant: %ld cl_lost_grant: %ld chunk bits: %d\n",
-	       cli->cl_import->imp_obd->obd_name,
-	       cli->cl_avail_grant, cli->cl_lost_grant, cli->cl_chunkbits);
+	       cli_name(cli), cli->cl_avail_grant, cli->cl_lost_grant,
+	       cli->cl_chunkbits);
 
 	if (ocd->ocd_connect_flags & OBD_CONNECT_GRANT_SHRINK &&
 	    list_empty(&cli->cl_grant_shrink_list))
@@ -1072,9 +908,9 @@ static int check_write_rcs(struct ptlrpc_request *req,
 static inline int can_merge_pages(struct brw_page *p1, struct brw_page *p2)
 {
 	if (p1->flag != p2->flag) {
-		unsigned mask = ~(OBD_BRW_FROM_GRANT | OBD_BRW_NOCACHE |
-				  OBD_BRW_SYNC | OBD_BRW_ASYNC |
-				  OBD_BRW_NOQUOTA | OBD_BRW_SOFT_SYNC);
+		unsigned int mask = ~(OBD_BRW_FROM_GRANT | OBD_BRW_NOCACHE |
+				      OBD_BRW_SYNC | OBD_BRW_ASYNC |
+				      OBD_BRW_NOQUOTA | OBD_BRW_SOFT_SYNC);
 
 		/* warn if we try to combine flags that we don't know to be
 		 * safe to combine
@@ -1097,7 +933,6 @@ static u32 osc_checksum_bulk(int nob, u32 pg_count,
 	int i = 0;
 	struct cfs_crypto_hash_desc *hdesc;
 	unsigned int bufsize;
-	int err;
 	unsigned char cfs_alg = cksum_obd2cfs(cksum_type);
 
 	LASSERT(pg_count > 0);
@@ -1139,7 +974,7 @@ static u32 osc_checksum_bulk(int nob, u32 pg_count,
 	}
 
 	bufsize = sizeof(cksum);
-	err = cfs_crypto_hash_final(hdesc, (unsigned char *)&cksum, &bufsize);
+	cfs_crypto_hash_final(hdesc, (unsigned char *)&cksum, &bufsize);
 
 	/* For sending we only compute the wrong checksum instead
 	 * of corrupting the data so it is still correct on a redo
@@ -1151,8 +986,7 @@ static u32 osc_checksum_bulk(int nob, u32 pg_count,
 }
 
 static int osc_brw_prep_request(int cmd, struct client_obd *cli,
-				struct obdo *oa,
-				struct lov_stripe_md *lsm, u32 page_count,
+				struct obdo *oa, u32 page_count,
 				struct brw_page **pga,
 				struct ptlrpc_request **reqp,
 				int reserve,
@@ -1210,8 +1044,9 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,
 
 	desc = ptlrpc_prep_bulk_imp(req, page_count,
 		cli->cl_import->imp_connect_data.ocd_brw_size >> LNET_MTU_BITS,
-		opc == OST_WRITE ? BULK_GET_SOURCE : BULK_PUT_SINK,
-		OST_BULK_PORTAL);
+		(opc == OST_WRITE ? PTLRPC_BULK_GET_SOURCE :
+		 PTLRPC_BULK_PUT_SINK) | PTLRPC_BULK_BUF_KIOV, OST_BULK_PORTAL,
+		 &ptlrpc_bulk_kiov_pin_ops);
 
 	if (!desc) {
 		rc = -ENOMEM;
@@ -1259,7 +1094,7 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,
 		LASSERT((pga[0]->flag & OBD_BRW_SRVLOCK) ==
 			(pg->flag & OBD_BRW_SRVLOCK));
 
-		ptlrpc_prep_bulk_page_pin(desc, pg->pg, poff, pg->count);
+		desc->bd_frag_ops->add_kiov_frag(desc, pg->pg, poff, pg->count);
 		requested_nob += pg->count;
 
 		if (i > 0 && can_merge_pages(pg_prev, pg)) {
@@ -1569,7 +1404,6 @@ static int osc_brw_redo_request(struct ptlrpc_request *request,
 	rc = osc_brw_prep_request(lustre_msg_get_opc(request->rq_reqmsg) ==
 					OST_WRITE ? OBD_BRW_WRITE : OBD_BRW_READ,
 				  aa->aa_cli, aa->aa_oa,
-				  NULL /* lsm unused by osc currently */,
 				  aa->aa_page_count, aa->aa_ppga,
 				  &new_req, 0, 1);
 	if (rc)
@@ -1764,8 +1598,6 @@ static int brw_interpret(const struct lu_env *env,
 	LASSERT(list_empty(&aa->aa_exts));
 	LASSERT(list_empty(&aa->aa_oaps));
 
-	cl_req_completion(env, aa->aa_clerq, rc < 0 ? rc :
-			  req->rq_bulk->bd_nob_transferred);
 	osc_release_ppga(aa->aa_ppga, aa->aa_page_count);
 	ptlrpc_lprocfs_brw(req, req->rq_bulk->bd_nob_transferred);
 
@@ -1818,9 +1650,7 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	struct osc_brw_async_args *aa = NULL;
 	struct obdo *oa = NULL;
 	struct osc_async_page *oap;
-	struct osc_async_page *tmp;
-	struct cl_req *clerq = NULL;
-	enum cl_req_type crt = (cmd & OBD_BRW_WRITE) ? CRT_WRITE : CRT_READ;
+	struct osc_object *obj = NULL;
 	struct cl_req_attr *crattr = NULL;
 	u64 starting_offset = OBD_OBJECT_EOF;
 	u64 ending_offset = 0;
@@ -1828,6 +1658,7 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	int mem_tight = 0;
 	int page_count = 0;
 	bool soft_sync = false;
+	bool interrupted = false;
 	int i;
 	int rc;
 	struct ost_body *body;
@@ -1839,32 +1670,15 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	list_for_each_entry(ext, ext_list, oe_link) {
 		LASSERT(ext->oe_state == OES_RPC);
 		mem_tight |= ext->oe_memalloc;
-		list_for_each_entry(oap, &ext->oe_pages, oap_pending_item) {
-			++page_count;
-			list_add_tail(&oap->oap_rpc_item, &rpc_list);
-			if (starting_offset > oap->oap_obj_off)
-				starting_offset = oap->oap_obj_off;
-			else
-				LASSERT(oap->oap_page_off == 0);
-			if (ending_offset < oap->oap_obj_off + oap->oap_count)
-				ending_offset = oap->oap_obj_off +
-						oap->oap_count;
-			else
-				LASSERT(oap->oap_page_off + oap->oap_count ==
-					PAGE_SIZE);
-		}
+		page_count += ext->oe_nr_pages;
+		if (!obj)
+			obj = ext->oe_obj;
 	}
 
 	soft_sync = osc_over_unstable_soft_limit(cli);
 	if (mem_tight)
 		mpflag = cfs_memory_pressure_get_and_set();
 
-	crattr = kzalloc(sizeof(*crattr), GFP_NOFS);
-	if (!crattr) {
-		rc = -ENOMEM;
-		goto out;
-	}
-
 	pga = kcalloc(page_count, sizeof(*pga), GFP_NOFS);
 	if (!pga) {
 		rc = -ENOMEM;
@@ -1878,44 +1692,46 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	}
 
 	i = 0;
-	list_for_each_entry(oap, &rpc_list, oap_rpc_item) {
-		struct cl_page *page = oap2cl_page(oap);
-
-		if (!clerq) {
-			clerq = cl_req_alloc(env, page, crt,
-					     1 /* only 1-object rpcs for now */);
-			if (IS_ERR(clerq)) {
-				rc = PTR_ERR(clerq);
-				goto out;
-			}
+	list_for_each_entry(ext, ext_list, oe_link) {
+		list_for_each_entry(oap, &ext->oe_pages, oap_pending_item) {
+			if (mem_tight)
+				oap->oap_brw_flags |= OBD_BRW_MEMALLOC;
+			if (soft_sync)
+				oap->oap_brw_flags |= OBD_BRW_SOFT_SYNC;
+			pga[i] = &oap->oap_brw_page;
+			pga[i]->off = oap->oap_obj_off + oap->oap_page_off;
+			i++;
+
+			list_add_tail(&oap->oap_rpc_item, &rpc_list);
+			if (starting_offset == OBD_OBJECT_EOF ||
+			    starting_offset > oap->oap_obj_off)
+				starting_offset = oap->oap_obj_off;
+			else
+				LASSERT(!oap->oap_page_off);
+			if (ending_offset < oap->oap_obj_off + oap->oap_count)
+				ending_offset = oap->oap_obj_off +
+						oap->oap_count;
+			else
+				LASSERT(oap->oap_page_off + oap->oap_count ==
+					PAGE_SIZE);
+			if (oap->oap_interrupted)
+				interrupted = true;
 		}
-		if (mem_tight)
-			oap->oap_brw_flags |= OBD_BRW_MEMALLOC;
-		if (soft_sync)
-			oap->oap_brw_flags |= OBD_BRW_SOFT_SYNC;
-		pga[i] = &oap->oap_brw_page;
-		pga[i]->off = oap->oap_obj_off + oap->oap_page_off;
-		CDEBUG(0, "put page %p index %lu oap %p flg %x to pga\n",
-		       pga[i]->pg, oap->oap_page->index, oap,
-		       pga[i]->flag);
-		i++;
-		cl_req_page_add(env, clerq, page);
 	}
 
-	/* always get the data for the obdo for the rpc */
-	LASSERT(clerq);
-	crattr->cra_oa = oa;
-	cl_req_attr_set(env, clerq, crattr, ~0ULL);
+	/* first page in the list */
+	oap = list_entry(rpc_list.next, typeof(*oap), oap_rpc_item);
 
-	rc = cl_req_prep(env, clerq);
-	if (rc != 0) {
-		CERROR("cl_req_prep failed: %d\n", rc);
-		goto out;
-	}
+	crattr = &osc_env_info(env)->oti_req_attr;
+	memset(crattr, 0, sizeof(*crattr));
+	crattr->cra_type = (cmd & OBD_BRW_WRITE) ? CRT_WRITE : CRT_READ;
+	crattr->cra_flags = ~0ULL;
+	crattr->cra_page = oap2cl_page(oap);
+	crattr->cra_oa = oa;
+	cl_req_attr_set(env, osc2cl(obj), crattr);
 
 	sort_brw_pages(pga, page_count);
-	rc = osc_brw_prep_request(cmd, cli, oa, NULL, page_count,
-				  pga, &req, 1, 0);
+	rc = osc_brw_prep_request(cmd, cli, oa, page_count, pga, &req, 1, 0);
 	if (rc != 0) {
 		CERROR("prep_req failed: %d\n", rc);
 		goto out;
@@ -1924,8 +1740,10 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	req->rq_commit_cb = brw_commit;
 	req->rq_interpret_reply = brw_interpret;
 
-	if (mem_tight != 0)
-		req->rq_memalloc = 1;
+	req->rq_memalloc = mem_tight != 0;
+	oap->oap_request = ptlrpc_request_addref(req);
+	if (interrupted && !req->rq_intr)
+		ptlrpc_mark_interrupted(req);
 
 	/* Need to update the timestamps after the request is built in case
 	 * we race with setattr (locally or in queue at OST).  If OST gets
@@ -1935,9 +1753,8 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	 */
 	body = req_capsule_client_get(&req->rq_pill, &RMF_OST_BODY);
 	crattr->cra_oa = &body->oa;
-	cl_req_attr_set(env, clerq, crattr,
-			OBD_MD_FLMTIME | OBD_MD_FLCTIME | OBD_MD_FLATIME);
-
+	crattr->cra_flags = OBD_MD_FLMTIME | OBD_MD_FLCTIME | OBD_MD_FLATIME;
+	cl_req_attr_set(env, osc2cl(obj), crattr);
 	lustre_msg_set_jobid(req->rq_reqmsg, crattr->cra_jobid);
 
 	CLASSERT(sizeof(*aa) <= sizeof(req->rq_async_args));
@@ -1946,24 +1763,6 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	list_splice_init(&rpc_list, &aa->aa_oaps);
 	INIT_LIST_HEAD(&aa->aa_exts);
 	list_splice_init(ext_list, &aa->aa_exts);
-	aa->aa_clerq = clerq;
-
-	/* queued sync pages can be torn down while the pages
-	 * were between the pending list and the rpc
-	 */
-	tmp = NULL;
-	list_for_each_entry(oap, &aa->aa_oaps, oap_rpc_item) {
-		/* only one oap gets a request reference */
-		if (!tmp)
-			tmp = oap;
-		if (oap->oap_interrupted && !req->rq_intr) {
-			CDEBUG(D_INODE, "oap %p in req %p interrupted\n",
-			       oap, req);
-			ptlrpc_mark_interrupted(req);
-		}
-	}
-	if (tmp)
-		tmp->oap_request = ptlrpc_request_addref(req);
 
 	spin_lock(&cli->cl_loi_list_lock);
 	starting_offset >>= PAGE_SHIFT;
@@ -1985,6 +1784,7 @@ int osc_build_rpc(const struct lu_env *env, struct client_obd *cli,
 	DEBUG_REQ(D_INODE, req, "%d pages, aa %p. now %ur/%dw in flight",
 		  page_count, aa, cli->cl_r_in_flight,
 		  cli->cl_w_in_flight);
+	OBD_FAIL_TIMEOUT(OBD_FAIL_OSC_DELAY_IO, cfs_fail_val);
 
 	ptlrpcd_add_req(req);
 	rc = 0;
@@ -1993,8 +1793,6 @@ out:
 	if (mem_tight != 0)
 		cfs_memory_pressure_restore(mpflag);
 
-	kfree(crattr);
-
 	if (rc != 0) {
 		LASSERT(!req);
 
@@ -2010,22 +1808,15 @@ out:
 			list_del_init(&ext->oe_link);
 			osc_extent_finish(env, ext, 0, rc);
 		}
-		if (clerq && !IS_ERR(clerq))
-			cl_req_completion(env, clerq, rc);
 	}
 	return rc;
 }
 
-static int osc_set_lock_data_with_check(struct ldlm_lock *lock,
-					struct ldlm_enqueue_info *einfo)
+static int osc_set_lock_data(struct ldlm_lock *lock, void *data)
 {
-	void *data = einfo->ei_cbdata;
 	int set = 0;
 
-	LASSERT(lock->l_blocking_ast == einfo->ei_cb_bl);
-	LASSERT(lock->l_resource->lr_type == einfo->ei_type);
-	LASSERT(lock->l_completion_ast == einfo->ei_cb_cp);
-	LASSERT(lock->l_glimpse_ast == einfo->ei_cb_gl);
+	LASSERT(lock);
 
 	lock_res_and_lock(lock);
 
@@ -2039,21 +1830,6 @@ static int osc_set_lock_data_with_check(struct ldlm_lock *lock,
 	return set;
 }
 
-static int osc_set_data_with_check(struct lustre_handle *lockh,
-				   struct ldlm_enqueue_info *einfo)
-{
-	struct ldlm_lock *lock = ldlm_handle2lock(lockh);
-	int set = 0;
-
-	if (lock) {
-		set = osc_set_lock_data_with_check(lock, einfo);
-		LDLM_LOCK_PUT(lock);
-	} else
-		CERROR("lockh %p, data %p - client evicted?\n",
-		       lockh, einfo->ei_cbdata);
-	return set;
-}
-
 static int osc_enqueue_fini(struct ptlrpc_request *req,
 			    osc_enqueue_upcall_f upcall, void *cookie,
 			    struct lustre_handle *lockh, enum ldlm_mode mode,
@@ -2153,7 +1929,7 @@ struct ptlrpc_request_set *PTLRPCD_SET = (void *)1;
  * release locks just after they are obtained.
  */
 int osc_enqueue_base(struct obd_export *exp, struct ldlm_res_id *res_id,
-		     __u64 *flags, ldlm_policy_data_t *policy,
+		     __u64 *flags, union ldlm_policy_data *policy,
 		     struct ost_lvb *lvb, int kms_valid,
 		     osc_enqueue_upcall_f upcall, void *cookie,
 		     struct ldlm_enqueue_info *einfo,
@@ -2219,7 +1995,7 @@ int osc_enqueue_base(struct obd_export *exp, struct ldlm_res_id *res_id,
 			ldlm_lock_decref(&lockh, mode);
 			LDLM_LOCK_PUT(matched);
 			return -ECANCELED;
-		} else if (osc_set_lock_data_with_check(matched, einfo)) {
+		} else if (osc_set_lock_data(matched, einfo->ei_cbdata)) {
 			*flags |= LDLM_FL_LVB_READY;
 			/* We already have a lock, and it's referenced. */
 			(*upcall)(cookie, &lockh, ELDLM_LOCK_MATCHED);
@@ -2304,7 +2080,7 @@ no_match:
 }
 
 int osc_match_base(struct obd_export *exp, struct ldlm_res_id *res_id,
-		   __u32 type, ldlm_policy_data_t *policy, __u32 mode,
+		   __u32 type, union ldlm_policy_data *policy, __u32 mode,
 		   __u64 *flags, void *data, struct lustre_handle *lockh,
 		   int unref)
 {
@@ -2331,31 +2107,20 @@ int osc_match_base(struct obd_export *exp, struct ldlm_res_id *res_id,
 		rc |= LCK_PW;
 	rc = ldlm_lock_match(obd->obd_namespace, lflags,
 			     res_id, type, policy, rc, lockh, unref);
-	if (rc) {
-		if (data) {
-			if (!osc_set_data_with_check(lockh, data)) {
-				if (!(lflags & LDLM_FL_TEST_LOCK))
-					ldlm_lock_decref(lockh, rc);
-				return 0;
-			}
-		}
-		if (!(lflags & LDLM_FL_TEST_LOCK) && mode != rc) {
-			ldlm_lock_addref(lockh, LCK_PR);
-			ldlm_lock_decref(lockh, LCK_PW);
-		}
+	if (!rc || lflags & LDLM_FL_TEST_LOCK)
 		return rc;
-	}
-	return rc;
-}
 
-int osc_cancel_base(struct lustre_handle *lockh, __u32 mode)
-{
-	if (unlikely(mode == LCK_GROUP))
-		ldlm_lock_decref_and_cancel(lockh, mode);
-	else
-		ldlm_lock_decref(lockh, mode);
+	if (data) {
+		struct ldlm_lock *lock = ldlm_handle2lock(lockh);
 
-	return 0;
+		LASSERT(lock);
+		if (!osc_set_lock_data(lock, data)) {
+			ldlm_lock_decref(lockh, rc);
+			rc = 0;
+		}
+		LDLM_LOCK_PUT(lock);
+	}
+	return rc;
 }
 
 static int osc_statfs_interpret(const struct lu_env *env,
@@ -2526,9 +2291,6 @@ static int osc_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 		err = ptlrpc_set_import_active(obd->u.cli.cl_import,
 					       data->ioc_offset);
 		goto out;
-	case OBD_IOC_POLL_QUOTACHECK:
-		err = osc_quota_poll_check(exp, karg);
-		goto out;
 	case OBD_IOC_PING_TARGET:
 		err = ptlrpc_obd_ping(obd);
 		goto out;
@@ -2543,103 +2305,6 @@ out:
 	return err;
 }
 
-static int osc_get_info(const struct lu_env *env, struct obd_export *exp,
-			u32 keylen, void *key, __u32 *vallen, void *val,
-			struct lov_stripe_md *lsm)
-{
-	if (!vallen || !val)
-		return -EFAULT;
-
-	if (KEY_IS(KEY_FIEMAP)) {
-		struct ll_fiemap_info_key *fm_key = key;
-		struct ldlm_res_id res_id;
-		ldlm_policy_data_t policy;
-		struct lustre_handle lockh;
-		enum ldlm_mode mode = 0;
-		struct ptlrpc_request *req;
-		struct ll_user_fiemap *reply;
-		char *tmp;
-		int rc;
-
-		if (!(fm_key->fiemap.fm_flags & FIEMAP_FLAG_SYNC))
-			goto skip_locking;
-
-		policy.l_extent.start = fm_key->fiemap.fm_start &
-						PAGE_MASK;
-
-		if (OBD_OBJECT_EOF - fm_key->fiemap.fm_length <=
-		    fm_key->fiemap.fm_start + PAGE_SIZE - 1)
-			policy.l_extent.end = OBD_OBJECT_EOF;
-		else
-			policy.l_extent.end = (fm_key->fiemap.fm_start +
-				fm_key->fiemap.fm_length +
-				PAGE_SIZE - 1) & PAGE_MASK;
-
-		ostid_build_res_name(&fm_key->oa.o_oi, &res_id);
-		mode = ldlm_lock_match(exp->exp_obd->obd_namespace,
-				       LDLM_FL_BLOCK_GRANTED |
-				       LDLM_FL_LVB_READY,
-				       &res_id, LDLM_EXTENT, &policy,
-				       LCK_PR | LCK_PW, &lockh, 0);
-		if (mode) { /* lock is cached on client */
-			if (mode != LCK_PR) {
-				ldlm_lock_addref(&lockh, LCK_PR);
-				ldlm_lock_decref(&lockh, LCK_PW);
-			}
-		} else { /* no cached lock, needs acquire lock on server side */
-			fm_key->oa.o_valid |= OBD_MD_FLFLAGS;
-			fm_key->oa.o_flags |= OBD_FL_SRVLOCK;
-		}
-
-skip_locking:
-		req = ptlrpc_request_alloc(class_exp2cliimp(exp),
-					   &RQF_OST_GET_INFO_FIEMAP);
-		if (!req) {
-			rc = -ENOMEM;
-			goto drop_lock;
-		}
-
-		req_capsule_set_size(&req->rq_pill, &RMF_FIEMAP_KEY,
-				     RCL_CLIENT, keylen);
-		req_capsule_set_size(&req->rq_pill, &RMF_FIEMAP_VAL,
-				     RCL_CLIENT, *vallen);
-		req_capsule_set_size(&req->rq_pill, &RMF_FIEMAP_VAL,
-				     RCL_SERVER, *vallen);
-
-		rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_GET_INFO);
-		if (rc) {
-			ptlrpc_request_free(req);
-			goto drop_lock;
-		}
-
-		tmp = req_capsule_client_get(&req->rq_pill, &RMF_FIEMAP_KEY);
-		memcpy(tmp, key, keylen);
-		tmp = req_capsule_client_get(&req->rq_pill, &RMF_FIEMAP_VAL);
-		memcpy(tmp, val, *vallen);
-
-		ptlrpc_request_set_replen(req);
-		rc = ptlrpc_queue_wait(req);
-		if (rc)
-			goto fini_req;
-
-		reply = req_capsule_server_get(&req->rq_pill, &RMF_FIEMAP_VAL);
-		if (!reply) {
-			rc = -EPROTO;
-			goto fini_req;
-		}
-
-		memcpy(val, reply, *vallen);
-fini_req:
-		ptlrpc_req_finished(req);
-drop_lock:
-		if (mode)
-			ldlm_lock_decref(&lockh, LCK_PR);
-		return rc;
-	}
-
-	return -EINVAL;
-}
-
 static int osc_set_info_async(const struct lu_env *env, struct obd_export *exp,
 			      u32 keylen, void *key, u32 vallen,
 			      void *val, struct ptlrpc_request_set *set)
@@ -2999,47 +2664,33 @@ out_ptlrpcd:
 	return rc;
 }
 
-static int osc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
+static int osc_precleanup(struct obd_device *obd)
 {
-	switch (stage) {
-	case OBD_CLEANUP_EARLY: {
-		struct obd_import *imp;
-
-		imp = obd->u.cli.cl_import;
-		CDEBUG(D_HA, "Deactivating import %s\n", obd->obd_name);
-		/* ptlrpc_abort_inflight to stop an mds_lov_synchronize */
-		ptlrpc_deactivate_import(imp);
-		spin_lock(&imp->imp_lock);
-		imp->imp_pingable = 0;
-		spin_unlock(&imp->imp_lock);
-		break;
+	struct client_obd *cli = &obd->u.cli;
+
+	/* LU-464
+	 * for echo client, export may be on zombie list, wait for
+	 * zombie thread to cull it, because cli.cl_import will be
+	 * cleared in client_disconnect_export():
+	 *   class_export_destroy() -> obd_cleanup() ->
+	 *   echo_device_free() -> echo_client_cleanup() ->
+	 *   obd_disconnect() -> osc_disconnect() ->
+	 *   client_disconnect_export()
+	 */
+	obd_zombie_barrier();
+	if (cli->cl_writeback_work) {
+		ptlrpcd_destroy_work(cli->cl_writeback_work);
+		cli->cl_writeback_work = NULL;
 	}
-	case OBD_CLEANUP_EXPORTS: {
-		struct client_obd *cli = &obd->u.cli;
-		/* LU-464
-		 * for echo client, export may be on zombie list, wait for
-		 * zombie thread to cull it, because cli.cl_import will be
-		 * cleared in client_disconnect_export():
-		 *   class_export_destroy() -> obd_cleanup() ->
-		 *   echo_device_free() -> echo_client_cleanup() ->
-		 *   obd_disconnect() -> osc_disconnect() ->
-		 *   client_disconnect_export()
-		 */
-		obd_zombie_barrier();
-		if (cli->cl_writeback_work) {
-			ptlrpcd_destroy_work(cli->cl_writeback_work);
-			cli->cl_writeback_work = NULL;
-		}
-		if (cli->cl_lru_work) {
-			ptlrpcd_destroy_work(cli->cl_lru_work);
-			cli->cl_lru_work = NULL;
-		}
-		obd_cleanup_client_import(obd);
-		ptlrpc_lprocfs_unregister_obd(obd);
-		lprocfs_obd_cleanup(obd);
-		break;
-		}
+
+	if (cli->cl_lru_work) {
+		ptlrpcd_destroy_work(cli->cl_lru_work);
+		cli->cl_lru_work = NULL;
 	}
+
+	obd_cleanup_client_import(obd);
+	ptlrpc_lprocfs_unregister_obd(obd);
+	lprocfs_obd_cleanup(obd);
 	return 0;
 }
 
@@ -3104,24 +2755,18 @@ static struct obd_ops osc_obd_ops = {
 	.disconnect     = osc_disconnect,
 	.statfs         = osc_statfs,
 	.statfs_async   = osc_statfs_async,
-	.unpackmd       = osc_unpackmd,
 	.create         = osc_create,
 	.destroy        = osc_destroy,
 	.getattr        = osc_getattr,
-	.getattr_async  = osc_getattr_async,
 	.setattr        = osc_setattr,
-	.setattr_async  = osc_setattr_async,
 	.iocontrol      = osc_iocontrol,
-	.get_info       = osc_get_info,
 	.set_info_async = osc_set_info_async,
 	.import_event   = osc_import_event,
 	.process_config = osc_process_config,
 	.quotactl       = osc_quotactl,
-	.quotacheck     = osc_quotacheck,
 };
 
 extern struct lu_kmem_descr osc_caches[];
-extern struct lock_class_key osc_ast_guard_class;
 
 static int __init osc_init(void)
 {
diff --git a/drivers/staging/lustre/lustre/ptlrpc/client.c b/drivers/staging/lustre/lustre/ptlrpc/client.c
index 8c51d51a678b..804741362bc0 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/client.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/client.c
@@ -43,6 +43,18 @@
 
 #include "ptlrpc_internal.h"
 
+const struct ptlrpc_bulk_frag_ops ptlrpc_bulk_kiov_pin_ops = {
+	.add_kiov_frag	= ptlrpc_prep_bulk_page_pin,
+	.release_frags	= ptlrpc_release_bulk_page_pin,
+};
+EXPORT_SYMBOL(ptlrpc_bulk_kiov_pin_ops);
+
+const struct ptlrpc_bulk_frag_ops ptlrpc_bulk_kiov_nopin_ops = {
+	.add_kiov_frag	= ptlrpc_prep_bulk_page_nopin,
+	.release_frags	= NULL,
+};
+EXPORT_SYMBOL(ptlrpc_bulk_kiov_nopin_ops);
+
 static int ptlrpc_send_new_req(struct ptlrpc_request *req);
 static int ptlrpcd_check_work(struct ptlrpc_request *req);
 static int ptlrpc_unregister_reply(struct ptlrpc_request *request, int async);
@@ -95,24 +107,43 @@ struct ptlrpc_connection *ptlrpc_uuid_to_connection(struct obd_uuid *uuid)
  * Allocate and initialize new bulk descriptor on the sender.
  * Returns pointer to the descriptor or NULL on error.
  */
-struct ptlrpc_bulk_desc *ptlrpc_new_bulk(unsigned npages, unsigned max_brw,
-					 unsigned type, unsigned portal)
+struct ptlrpc_bulk_desc *ptlrpc_new_bulk(unsigned int nfrags,
+					 unsigned int max_brw,
+					 enum ptlrpc_bulk_op_type type,
+					 unsigned int portal,
+					 const struct ptlrpc_bulk_frag_ops *ops)
 {
 	struct ptlrpc_bulk_desc *desc;
 	int i;
 
-	desc = kzalloc(offsetof(struct ptlrpc_bulk_desc, bd_iov[npages]),
-		       GFP_NOFS);
+	/* ensure that only one of KIOV or IOVEC is set but not both */
+	LASSERT((ptlrpc_is_bulk_desc_kiov(type) && ops->add_kiov_frag) ||
+		(ptlrpc_is_bulk_desc_kvec(type) && ops->add_iov_frag));
+
+	desc = kzalloc(sizeof(*desc), GFP_NOFS);
 	if (!desc)
 		return NULL;
 
+	if (type & PTLRPC_BULK_BUF_KIOV) {
+		GET_KIOV(desc) = kcalloc(nfrags, sizeof(*GET_KIOV(desc)),
+					 GFP_NOFS);
+		if (!GET_KIOV(desc))
+			goto free_desc;
+	} else {
+		GET_KVEC(desc) = kcalloc(nfrags, sizeof(*GET_KVEC(desc)),
+					 GFP_NOFS);
+		if (!GET_KVEC(desc))
+			goto free_desc;
+	}
+
 	spin_lock_init(&desc->bd_lock);
 	init_waitqueue_head(&desc->bd_waitq);
-	desc->bd_max_iov = npages;
+	desc->bd_max_iov = nfrags;
 	desc->bd_iov_count = 0;
 	desc->bd_portal = portal;
 	desc->bd_type = type;
 	desc->bd_md_count = 0;
+	desc->bd_frag_ops = (struct ptlrpc_bulk_frag_ops *)ops;
 	LASSERT(max_brw > 0);
 	desc->bd_md_max_brw = min(max_brw, PTLRPC_BULK_OPS_COUNT);
 	/*
@@ -123,24 +154,31 @@ struct ptlrpc_bulk_desc *ptlrpc_new_bulk(unsigned npages, unsigned max_brw,
 		LNetInvalidateHandle(&desc->bd_mds[i]);
 
 	return desc;
+free_desc:
+	kfree(desc);
+	return NULL;
 }
 
 /**
  * Prepare bulk descriptor for specified outgoing request \a req that
- * can fit \a npages * pages. \a type is bulk type. \a portal is where
+ * can fit \a nfrags * pages. \a type is bulk type. \a portal is where
  * the bulk to be sent. Used on client-side.
  * Returns pointer to newly allocated initialized bulk descriptor or NULL on
  * error.
  */
 struct ptlrpc_bulk_desc *ptlrpc_prep_bulk_imp(struct ptlrpc_request *req,
-					      unsigned npages, unsigned max_brw,
-					      unsigned type, unsigned portal)
+					      unsigned int nfrags,
+					      unsigned int max_brw,
+					      unsigned int type,
+					      unsigned int portal,
+					      const struct ptlrpc_bulk_frag_ops *ops)
 {
 	struct obd_import *imp = req->rq_import;
 	struct ptlrpc_bulk_desc *desc;
 
-	LASSERT(type == BULK_PUT_SINK || type == BULK_GET_SOURCE);
-	desc = ptlrpc_new_bulk(npages, max_brw, type, portal);
+	LASSERT(ptlrpc_is_bulk_op_passive(type));
+
+	desc = ptlrpc_new_bulk(nfrags, max_brw, type, portal, ops);
 	if (!desc)
 		return NULL;
 
@@ -158,56 +196,82 @@ struct ptlrpc_bulk_desc *ptlrpc_prep_bulk_imp(struct ptlrpc_request *req,
 }
 EXPORT_SYMBOL(ptlrpc_prep_bulk_imp);
 
-/**
- * Add a page \a page to the bulk descriptor \a desc.
- * Data to transfer in the page starts at offset \a pageoffset and
- * amount of data to transfer from the page is \a len
- */
 void __ptlrpc_prep_bulk_page(struct ptlrpc_bulk_desc *desc,
 			     struct page *page, int pageoffset, int len, int pin)
 {
+	struct bio_vec *kiov;
+
 	LASSERT(desc->bd_iov_count < desc->bd_max_iov);
 	LASSERT(page);
 	LASSERT(pageoffset >= 0);
 	LASSERT(len > 0);
 	LASSERT(pageoffset + len <= PAGE_SIZE);
+	LASSERT(ptlrpc_is_bulk_desc_kiov(desc->bd_type));
+
+	kiov = &BD_GET_KIOV(desc, desc->bd_iov_count);
 
 	desc->bd_nob += len;
 
 	if (pin)
 		get_page(page);
 
-	ptlrpc_add_bulk_page(desc, page, pageoffset, len);
+	kiov->bv_page = page;
+	kiov->bv_offset = pageoffset;
+	kiov->bv_len = len;
+
+	desc->bd_iov_count++;
 }
 EXPORT_SYMBOL(__ptlrpc_prep_bulk_page);
 
-/**
- * Uninitialize and free bulk descriptor \a desc.
- * Works on bulk descriptors both from server and client side.
- */
-void __ptlrpc_free_bulk(struct ptlrpc_bulk_desc *desc, int unpin)
+int ptlrpc_prep_bulk_frag(struct ptlrpc_bulk_desc *desc,
+			  void *frag, int len)
 {
-	int i;
+	struct kvec *iovec;
+
+	LASSERT(desc->bd_iov_count < desc->bd_max_iov);
+	LASSERT(frag);
+	LASSERT(len > 0);
+	LASSERT(ptlrpc_is_bulk_desc_kvec(desc->bd_type));
 
+	iovec = &BD_GET_KVEC(desc, desc->bd_iov_count);
+
+	desc->bd_nob += len;
+
+	iovec->iov_base = frag;
+	iovec->iov_len = len;
+
+	desc->bd_iov_count++;
+
+	return desc->bd_nob;
+}
+EXPORT_SYMBOL(ptlrpc_prep_bulk_frag);
+
+void ptlrpc_free_bulk(struct ptlrpc_bulk_desc *desc)
+{
 	LASSERT(desc->bd_iov_count != LI_POISON); /* not freed already */
 	LASSERT(desc->bd_md_count == 0);	 /* network hands off */
 	LASSERT((desc->bd_export != NULL) ^ (desc->bd_import != NULL));
+	LASSERT(desc->bd_frag_ops);
 
-	sptlrpc_enc_pool_put_pages(desc);
+	if (ptlrpc_is_bulk_desc_kiov(desc->bd_type))
+		sptlrpc_enc_pool_put_pages(desc);
 
 	if (desc->bd_export)
 		class_export_put(desc->bd_export);
 	else
 		class_import_put(desc->bd_import);
 
-	if (unpin) {
-		for (i = 0; i < desc->bd_iov_count; i++)
-			put_page(desc->bd_iov[i].bv_page);
-	}
+	if (desc->bd_frag_ops->release_frags)
+		desc->bd_frag_ops->release_frags(desc);
+
+	if (ptlrpc_is_bulk_desc_kiov(desc->bd_type))
+		kfree(GET_KIOV(desc));
+	else
+		kfree(GET_KVEC(desc));
 
 	kfree(desc);
 }
-EXPORT_SYMBOL(__ptlrpc_free_bulk);
+EXPORT_SYMBOL(ptlrpc_free_bulk);
 
 /**
  * Set server timelimit for this req, i.e. how long are we willing to wait
@@ -589,6 +653,42 @@ static void __ptlrpc_free_req_to_pool(struct ptlrpc_request *request)
 	spin_unlock(&pool->prp_lock);
 }
 
+void ptlrpc_add_unreplied(struct ptlrpc_request *req)
+{
+	struct obd_import	*imp = req->rq_import;
+	struct list_head	*tmp;
+	struct ptlrpc_request	*iter;
+
+	assert_spin_locked(&imp->imp_lock);
+	LASSERT(list_empty(&req->rq_unreplied_list));
+
+	/* unreplied list is sorted by xid in ascending order */
+	list_for_each_prev(tmp, &imp->imp_unreplied_list) {
+		iter = list_entry(tmp, struct ptlrpc_request,
+				  rq_unreplied_list);
+
+		LASSERT(req->rq_xid != iter->rq_xid);
+		if (req->rq_xid < iter->rq_xid)
+			continue;
+		list_add(&req->rq_unreplied_list, &iter->rq_unreplied_list);
+		return;
+	}
+	list_add(&req->rq_unreplied_list, &imp->imp_unreplied_list);
+}
+
+void ptlrpc_assign_next_xid_nolock(struct ptlrpc_request *req)
+{
+	req->rq_xid = ptlrpc_next_xid();
+	ptlrpc_add_unreplied(req);
+}
+
+static inline void ptlrpc_assign_next_xid(struct ptlrpc_request *req)
+{
+	spin_lock(&req->rq_import->imp_lock);
+	ptlrpc_assign_next_xid_nolock(req);
+	spin_unlock(&req->rq_import->imp_lock);
+}
+
 int ptlrpc_request_bufs_pack(struct ptlrpc_request *request,
 			     __u32 version, int opcode, char **bufs,
 			     struct ptlrpc_cli_ctx *ctx)
@@ -637,8 +737,8 @@ int ptlrpc_request_bufs_pack(struct ptlrpc_request *request,
 
 	ptlrpc_at_set_req_timeout(request);
 
-	request->rq_xid = ptlrpc_next_xid();
 	lustre_msg_set_opc(request->rq_reqmsg, opcode);
+	ptlrpc_assign_next_xid(request);
 
 	/* Let's setup deadline for req/reply/bulk unlink for opcode. */
 	if (cfs_fail_val == opcode) {
@@ -1129,7 +1229,9 @@ static int ptlrpc_check_status(struct ptlrpc_request *req)
 		lnet_nid_t nid = imp->imp_connection->c_peer.nid;
 		__u32 opc = lustre_msg_get_opc(req->rq_reqmsg);
 
-		if (ptlrpc_console_allow(req))
+		/* -EAGAIN is normal when using POSIX flocks */
+		if (ptlrpc_console_allow(req) &&
+		    !(opc == LDLM_ENQUEUE && err == -EAGAIN))
 			LCONSOLE_ERROR_MSG(0x011, "%s: operation %s to node %s failed: rc = %d\n",
 					   imp->imp_obd->obd_name,
 					   ll_opcode2str(opc),
@@ -1166,6 +1268,24 @@ static void ptlrpc_save_versions(struct ptlrpc_request *req)
 	       versions[0], versions[1]);
 }
 
+__u64 ptlrpc_known_replied_xid(struct obd_import *imp)
+{
+	struct ptlrpc_request *req;
+
+	assert_spin_locked(&imp->imp_lock);
+	if (list_empty(&imp->imp_unreplied_list))
+		return 0;
+
+	req = list_entry(imp->imp_unreplied_list.next, struct ptlrpc_request,
+			 rq_unreplied_list);
+	LASSERTF(req->rq_xid >= 1, "XID:%llu\n", req->rq_xid);
+
+	if (imp->imp_known_replied_xid < req->rq_xid - 1)
+		imp->imp_known_replied_xid = req->rq_xid - 1;
+
+	return req->rq_xid - 1;
+}
+
 /**
  * Callback function called when client receives RPC reply for \a req.
  * Returns 0 on success or error code.
@@ -1180,6 +1300,7 @@ static int after_reply(struct ptlrpc_request *req)
 	int rc;
 	struct timespec64 work_start;
 	long timediff;
+	u64 committed;
 
 	LASSERT(obd);
 	/* repbuf must be unlinked */
@@ -1206,6 +1327,10 @@ static int after_reply(struct ptlrpc_request *req)
 		return 0;
 	}
 
+	ktime_get_real_ts64(&work_start);
+	timediff = (work_start.tv_sec - req->rq_sent_tv.tv_sec) * USEC_PER_SEC +
+		   (work_start.tv_nsec - req->rq_sent_tv.tv_nsec) /
+								 NSEC_PER_USEC;
 	/*
 	 * NB Until this point, the whole of the incoming message,
 	 * including buflens, status etc is in the sender's byte order.
@@ -1235,13 +1360,6 @@ static int after_reply(struct ptlrpc_request *req)
 		spin_unlock(&req->rq_lock);
 		req->rq_nr_resend++;
 
-		/* allocate new xid to avoid reply reconstruction */
-		if (!req->rq_bulk) {
-			/* new xid is already allocated for bulk in ptlrpc_check_set() */
-			req->rq_xid = ptlrpc_next_xid();
-			DEBUG_REQ(D_RPCTRACE, req, "Allocating new xid for resend on EINPROGRESS");
-		}
-
 		/* Readjust the timeout for current conditions */
 		ptlrpc_at_set_req_timeout(req);
 		/*
@@ -1255,13 +1373,14 @@ static int after_reply(struct ptlrpc_request *req)
 		else
 			req->rq_sent = now + req->rq_nr_resend;
 
+		/* Resend for EINPROGRESS will use a new XID */
+		spin_lock(&imp->imp_lock);
+		list_del_init(&req->rq_unreplied_list);
+		spin_unlock(&imp->imp_lock);
+
 		return 0;
 	}
 
-	ktime_get_real_ts64(&work_start);
-	timediff = (work_start.tv_sec - req->rq_sent_tv.tv_sec) * USEC_PER_SEC +
-		   (work_start.tv_nsec - req->rq_sent_tv.tv_nsec) /
-								 NSEC_PER_USEC;
 	if (obd->obd_svc_stats) {
 		lprocfs_counter_add(obd->obd_svc_stats, PTLRPC_REQWAIT_CNTR,
 				    timediff);
@@ -1338,10 +1457,9 @@ static int after_reply(struct ptlrpc_request *req)
 		}
 
 		/* Replay-enabled imports return commit-status information. */
-		if (lustre_msg_get_last_committed(req->rq_repmsg)) {
-			imp->imp_peer_committed_transno =
-				lustre_msg_get_last_committed(req->rq_repmsg);
-		}
+		committed = lustre_msg_get_last_committed(req->rq_repmsg);
+		if (likely(committed > imp->imp_peer_committed_transno))
+			imp->imp_peer_committed_transno = committed;
 
 		ptlrpc_free_committed(imp);
 
@@ -1373,9 +1491,17 @@ static int after_reply(struct ptlrpc_request *req)
 static int ptlrpc_send_new_req(struct ptlrpc_request *req)
 {
 	struct obd_import *imp = req->rq_import;
+	u64 min_xid = 0;
 	int rc;
 
 	LASSERT(req->rq_phase == RQ_PHASE_NEW);
+
+	/* do not try to go further if there is not enough memory in enc_pool */
+	if (req->rq_sent && req->rq_bulk)
+		if (req->rq_bulk->bd_iov_count > get_free_pages_in_pool() &&
+		    pool_is_at_full_capacity())
+			return -ENOMEM;
+
 	if (req->rq_sent && (req->rq_sent > ktime_get_real_seconds()) &&
 	    (!req->rq_generation_set ||
 	     req->rq_import_generation == imp->imp_generation))
@@ -1385,6 +1511,9 @@ static int ptlrpc_send_new_req(struct ptlrpc_request *req)
 
 	spin_lock(&imp->imp_lock);
 
+	LASSERT(req->rq_xid);
+	LASSERT(!list_empty(&req->rq_unreplied_list));
+
 	if (!req->rq_generation_set)
 		req->rq_import_generation = imp->imp_generation;
 
@@ -1414,8 +1543,25 @@ static int ptlrpc_send_new_req(struct ptlrpc_request *req)
 	LASSERT(list_empty(&req->rq_list));
 	list_add_tail(&req->rq_list, &imp->imp_sending_list);
 	atomic_inc(&req->rq_import->imp_inflight);
+
+	/* find the known replied XID from the unreplied list, CONNECT
+	 * and DISCONNECT requests are skipped to make the sanity check
+	 * on server side happy. see process_req_last_xid().
+	 *
+	 * For CONNECT: Because replay requests have lower XID, it'll
+	 * break the sanity check if CONNECT bump the exp_last_xid on
+	 * server.
+	 *
+	 * For DISCONNECT: Since client will abort inflight RPC before
+	 * sending DISCONNECT, DISCONNECT may carry an XID which higher
+	 * than the inflight RPC.
+	 */
+	if (!ptlrpc_req_is_connect(req) && !ptlrpc_req_is_disconnect(req))
+		min_xid = ptlrpc_known_replied_xid(imp);
 	spin_unlock(&imp->imp_lock);
 
+	lustre_msg_set_last_xid(req->rq_reqmsg, min_xid);
+
 	lustre_msg_set_status(req->rq_reqmsg, current_pid());
 
 	rc = sptlrpc_req_refresh_ctx(req, -1);
@@ -1438,6 +1584,16 @@ static int ptlrpc_send_new_req(struct ptlrpc_request *req)
 	       lustre_msg_get_opc(req->rq_reqmsg));
 
 	rc = ptl_send_rpc(req, 0);
+	if (rc == -ENOMEM) {
+		spin_lock(&imp->imp_lock);
+		if (!list_empty(&req->rq_list)) {
+			list_del_init(&req->rq_list);
+			atomic_dec(&req->rq_import->imp_inflight);
+		}
+		spin_unlock(&imp->imp_lock);
+		ptlrpc_rqphase_move(req, RQ_PHASE_NEW);
+		return rc;
+	}
 	if (rc) {
 		DEBUG_REQ(D_HA, req, "send failed (%d); expect timeout", rc);
 		spin_lock(&req->rq_lock);
@@ -1688,18 +1844,9 @@ int ptlrpc_check_set(const struct lu_env *env, struct ptlrpc_request_set *set)
 					spin_lock(&req->rq_lock);
 					req->rq_resend = 1;
 					spin_unlock(&req->rq_lock);
-					if (req->rq_bulk) {
-						__u64 old_xid;
-
-						if (!ptlrpc_unregister_bulk(req, 1))
-							continue;
-
-						/* ensure previous bulk fails */
-						old_xid = req->rq_xid;
-						req->rq_xid = ptlrpc_next_xid();
-						CDEBUG(D_HA, "resend bulk old x%llu new x%llu\n",
-						       old_xid, req->rq_xid);
-					}
+					if (req->rq_bulk &&
+					    !ptlrpc_unregister_bulk(req, 1))
+						continue;
 				}
 				/*
 				 * rq_wait_ctx is only touched by ptlrpcd,
@@ -1727,6 +1874,14 @@ int ptlrpc_check_set(const struct lu_env *env, struct ptlrpc_request_set *set)
 				}
 
 				rc = ptl_send_rpc(req, 0);
+				if (rc == -ENOMEM) {
+					spin_lock(&imp->imp_lock);
+					if (!list_empty(&req->rq_list))
+						list_del_init(&req->rq_list);
+					spin_unlock(&imp->imp_lock);
+					ptlrpc_rqphase_move(req, RQ_PHASE_NEW);
+					continue;
+				}
 				if (rc) {
 					DEBUG_REQ(D_HA, req,
 						  "send failed: rc = %d", rc);
@@ -1850,6 +2005,7 @@ interpret:
 			list_del_init(&req->rq_list);
 			atomic_dec(&imp->imp_inflight);
 		}
+		list_del_init(&req->rq_unreplied_list);
 		spin_unlock(&imp->imp_lock);
 
 		atomic_dec(&set->set_remaining);
@@ -2247,6 +2403,7 @@ static void __ptlrpc_free_req(struct ptlrpc_request *request, int locked)
 		if (!locked)
 			spin_lock(&request->rq_import->imp_lock);
 		list_del_init(&request->rq_replay_list);
+		list_del_init(&request->rq_unreplied_list);
 		if (!locked)
 			spin_unlock(&request->rq_import->imp_lock);
 	}
@@ -2266,7 +2423,7 @@ static void __ptlrpc_free_req(struct ptlrpc_request *request, int locked)
 		request->rq_import = NULL;
 	}
 	if (request->rq_bulk)
-		ptlrpc_free_bulk_pin(request->rq_bulk);
+		ptlrpc_free_bulk(request->rq_bulk);
 
 	if (request->rq_reqbuf || request->rq_clrbuf)
 		sptlrpc_cli_free_reqbuf(request);
@@ -2542,14 +2699,6 @@ void ptlrpc_resend_req(struct ptlrpc_request *req)
 	req->rq_resend = 1;
 	req->rq_net_err = 0;
 	req->rq_timedout = 0;
-	if (req->rq_bulk) {
-		__u64 old_xid = req->rq_xid;
-
-		/* ensure previous bulk fails */
-		req->rq_xid = ptlrpc_next_xid();
-		CDEBUG(D_HA, "resend bulk old x%llu new x%llu\n",
-		       old_xid, req->rq_xid);
-	}
 	ptlrpc_client_wake_req(req);
 	spin_unlock(&req->rq_lock);
 }
@@ -2592,6 +2741,10 @@ void ptlrpc_retain_replayable_request(struct ptlrpc_request *req,
 
 	lustre_msg_add_flags(req->rq_reqmsg, MSG_REPLAY);
 
+	spin_lock(&req->rq_lock);
+	req->rq_resend = 0;
+	spin_unlock(&req->rq_lock);
+
 	LASSERT(imp->imp_replayable);
 	/* Balanced in ptlrpc_free_committed, usually. */
 	ptlrpc_request_addref(req);
@@ -2667,8 +2820,15 @@ static int ptlrpc_replay_interpret(const struct lu_env *env,
 
 	atomic_dec(&imp->imp_replay_inflight);
 
-	if (!ptlrpc_client_replied(req)) {
-		CERROR("request replay timed out, restarting recovery\n");
+	/*
+	 * Note: if it is bulk replay (MDS-MDS replay), then even if
+	 * server got the request, but bulk transfer timeout, let's
+	 * replay the bulk req again
+	 */
+	if (!ptlrpc_client_replied(req) ||
+	    (req->rq_bulk &&
+	     lustre_msg_get_status(req->rq_repmsg) == -ETIMEDOUT)) {
+		DEBUG_REQ(D_ERROR, req, "request replay timed out.\n");
 		rc = -ETIMEDOUT;
 		goto out;
 	}
@@ -2939,6 +3099,48 @@ __u64 ptlrpc_next_xid(void)
 }
 
 /**
+ * If request has a new allocated XID (new request or EINPROGRESS resend),
+ * use this XID as matchbits of bulk, otherwise allocate a new matchbits for
+ * request to ensure previous bulk fails and avoid problems with lost replies
+ * and therefore several transfers landing into the same buffer from different
+ * sending attempts.
+ */
+void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req)
+{
+	struct ptlrpc_bulk_desc *bd = req->rq_bulk;
+
+	LASSERT(bd);
+
+	if (!req->rq_resend) {
+		/* this request has a new xid, just use it as bulk matchbits */
+		req->rq_mbits = req->rq_xid;
+
+	} else { /* needs to generate a new matchbits for resend */
+		u64 old_mbits = req->rq_mbits;
+
+		if ((bd->bd_import->imp_connect_data.ocd_connect_flags &
+		     OBD_CONNECT_BULK_MBITS)) {
+			req->rq_mbits = ptlrpc_next_xid();
+		} else {
+			/* old version transfers rq_xid to peer as matchbits */
+			req->rq_mbits = ptlrpc_next_xid();
+			req->rq_xid = req->rq_mbits;
+		}
+
+		CDEBUG(D_HA, "resend bulk old x%llu new x%llu\n",
+		       old_mbits, req->rq_mbits);
+	}
+
+	/*
+	 * For multi-bulk RPCs, rq_mbits is the last mbits needed for bulks so
+	 * that server can infer the number of bulks that were prepared,
+	 * see LU-1431
+	 */
+	req->rq_mbits += ((bd->bd_iov_count + LNET_MAX_IOV - 1) /
+			  LNET_MAX_IOV) - 1;
+}
+
+/**
  * Get a glimpse at what next xid value might have been.
  * Returns possible next xid.
  */
diff --git a/drivers/staging/lustre/lustre/ptlrpc/connection.c b/drivers/staging/lustre/lustre/ptlrpc/connection.c
index 7b020d60c9e5..6c7c8b68a909 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/connection.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/connection.c
@@ -152,8 +152,8 @@ void ptlrpc_connection_fini(void)
 /*
  * Hash operations for net_peer<->connection
  */
-static unsigned
-conn_hashfn(struct cfs_hash *hs, const void *key, unsigned mask)
+static unsigned int
+conn_hashfn(struct cfs_hash *hs, const void *key, unsigned int mask)
 {
 	return cfs_hash_djb2_hash(key, sizeof(lnet_process_id_t), mask);
 }
diff --git a/drivers/staging/lustre/lustre/ptlrpc/events.c b/drivers/staging/lustre/lustre/ptlrpc/events.c
index 283dfb296d35..49f3e6368415 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/events.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/events.c
@@ -182,9 +182,9 @@ void client_bulk_callback(lnet_event_t *ev)
 	struct ptlrpc_bulk_desc *desc = cbid->cbid_arg;
 	struct ptlrpc_request *req;
 
-	LASSERT((desc->bd_type == BULK_PUT_SINK &&
+	LASSERT((ptlrpc_is_bulk_put_sink(desc->bd_type) &&
 		 ev->type == LNET_EVENT_PUT) ||
-		(desc->bd_type == BULK_GET_SOURCE &&
+		(ptlrpc_is_bulk_get_source(desc->bd_type) &&
 		 ev->type == LNET_EVENT_GET) ||
 		ev->type == LNET_EVENT_UNLINK);
 	LASSERT(ev->unlinked);
diff --git a/drivers/staging/lustre/lustre/ptlrpc/import.c b/drivers/staging/lustre/lustre/ptlrpc/import.c
index a23d0a05b574..e8280194001c 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/import.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/import.c
@@ -396,7 +396,7 @@ void ptlrpc_activate_import(struct obd_import *imp)
 }
 EXPORT_SYMBOL(ptlrpc_activate_import);
 
-static void ptlrpc_pinger_force(struct obd_import *imp)
+void ptlrpc_pinger_force(struct obd_import *imp)
 {
 	CDEBUG(D_HA, "%s: waking up pinger s:%s\n", obd2cli_tgt(imp->imp_obd),
 	       ptlrpc_import_state_name(imp->imp_state));
@@ -408,6 +408,7 @@ static void ptlrpc_pinger_force(struct obd_import *imp)
 	if (imp->imp_state != LUSTRE_IMP_CONNECTING)
 		ptlrpc_pinger_wake_up();
 }
+EXPORT_SYMBOL(ptlrpc_pinger_force);
 
 void ptlrpc_fail_import(struct obd_import *imp, __u32 conn_cnt)
 {
@@ -621,7 +622,8 @@ int ptlrpc_connect_import(struct obd_import *imp)
 		spin_unlock(&imp->imp_lock);
 		CERROR("already connected\n");
 		return 0;
-	} else if (imp->imp_state == LUSTRE_IMP_CONNECTING) {
+	} else if (imp->imp_state == LUSTRE_IMP_CONNECTING ||
+		   imp->imp_connected) {
 		spin_unlock(&imp->imp_lock);
 		CERROR("already connecting\n");
 		return -EALREADY;
@@ -691,8 +693,6 @@ int ptlrpc_connect_import(struct obd_import *imp)
 	request->rq_timeout = INITIAL_CONNECT_TIMEOUT;
 	lustre_msg_set_timeout(request->rq_reqmsg, request->rq_timeout);
 
-	lustre_msg_add_op_flags(request->rq_reqmsg, MSG_CONNECT_NEXT_VER);
-
 	request->rq_no_resend = 1;
 	request->rq_no_delay = 1;
 	request->rq_send_state = LUSTRE_IMP_CONNECTING;
@@ -859,6 +859,17 @@ static int ptlrpc_connect_set_flags(struct obd_import *imp,
 	client_adjust_max_dirty(cli);
 
 	/*
+	 * Update client max modify RPCs in flight with value returned
+	 * by the server
+	 */
+	if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS)
+		cli->cl_max_mod_rpcs_in_flight = min(
+					cli->cl_max_mod_rpcs_in_flight,
+					ocd->ocd_maxmodrpcs);
+	else
+		cli->cl_max_mod_rpcs_in_flight = 1;
+
+	/*
 	 * Reset ns_connect_flags only for initial connect. It might be
 	 * changed in while using FS and if we reset it in reconnect
 	 * this leads to losing user settings done before such as
@@ -873,8 +884,7 @@ static int ptlrpc_connect_set_flags(struct obd_import *imp,
 			ocd->ocd_connect_flags;
 	}
 
-	if ((ocd->ocd_connect_flags & OBD_CONNECT_AT) &&
-	    (imp->imp_msg_magic == LUSTRE_MSG_MAGIC_V2))
+	if (ocd->ocd_connect_flags & OBD_CONNECT_AT)
 		/*
 		 * We need a per-message support flag, because
 		 * a. we don't know if the incoming connect reply
@@ -889,16 +899,45 @@ static int ptlrpc_connect_set_flags(struct obd_import *imp,
 	else
 		imp->imp_msghdr_flags &= ~MSGHDR_AT_SUPPORT;
 
-	if ((ocd->ocd_connect_flags & OBD_CONNECT_FULL20) &&
-	    (imp->imp_msg_magic == LUSTRE_MSG_MAGIC_V2))
-		imp->imp_msghdr_flags |= MSGHDR_CKSUM_INCOMPAT18;
-	else
-		imp->imp_msghdr_flags &= ~MSGHDR_CKSUM_INCOMPAT18;
+	imp->imp_msghdr_flags |= MSGHDR_CKSUM_INCOMPAT18;
 
 	return 0;
 }
 
 /**
+ * Add all replay requests back to unreplied list before start replay,
+ * so that we can make sure the known replied XID is always increased
+ * only even if when replaying requests.
+ */
+static void ptlrpc_prepare_replay(struct obd_import *imp)
+{
+	struct ptlrpc_request *req;
+
+	if (imp->imp_state != LUSTRE_IMP_REPLAY ||
+	    imp->imp_resend_replay)
+		return;
+
+	/*
+	 * If the server was restart during repaly, the requests may
+	 * have been added to the unreplied list in former replay.
+	 */
+	spin_lock(&imp->imp_lock);
+
+	list_for_each_entry(req, &imp->imp_committed_list, rq_replay_list) {
+		if (list_empty(&req->rq_unreplied_list))
+			ptlrpc_add_unreplied(req);
+	}
+
+	list_for_each_entry(req, &imp->imp_replay_list, rq_replay_list) {
+		if (list_empty(&req->rq_unreplied_list))
+			ptlrpc_add_unreplied(req);
+	}
+
+	imp->imp_known_replied_xid = ptlrpc_known_replied_xid(imp);
+	spin_unlock(&imp->imp_lock);
+}
+
+/**
  * interpret_reply callback for connect RPCs.
  * Looks into returned status of connect operation and decides
  * what to do with the import - i.e enter recovery, promote it to
@@ -933,6 +972,13 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
 		ptlrpc_maybe_ping_import_soon(imp);
 		goto out;
 	}
+
+	/*
+	 * LU-7558: indicate that we are interpretting connect reply,
+	 * pltrpc_connect_import() will not try to reconnect until
+	 * interpret will finish.
+	 */
+	imp->imp_connected = 1;
 	spin_unlock(&imp->imp_lock);
 
 	LASSERT(imp->imp_conn_current);
@@ -967,6 +1013,16 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
 
 	spin_unlock(&imp->imp_lock);
 
+	if (!exp) {
+		/* This could happen if export is cleaned during the
+		 * connect attempt
+		 */
+		CERROR("%s: missing export after connect\n",
+		       imp->imp_obd->obd_name);
+		rc = -ENODEV;
+		goto out;
+	}
+
 	/* check that server granted subset of flags we asked for. */
 	if ((ocd->ocd_connect_flags & imp->imp_connect_flags_orig) !=
 	    ocd->ocd_connect_flags) {
@@ -977,15 +1033,6 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
 		goto out;
 	}
 
-	if (!exp) {
-		/* This could happen if export is cleaned during the
-		 * connect attempt
-		 */
-		CERROR("%s: missing export after connect\n",
-		       imp->imp_obd->obd_name);
-		rc = -ENODEV;
-		goto out;
-	}
 	old_connect_flags = exp_connect_flags(exp);
 	exp->exp_connect_data = *ocd;
 	imp->imp_obd->obd_self_export->exp_connect_data = *ocd;
@@ -1124,6 +1171,7 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
 		imp->imp_remote_handle =
 				*lustre_msg_get_handle(request->rq_repmsg);
 		imp->imp_last_replay_transno = 0;
+		imp->imp_replay_cursor = &imp->imp_committed_list;
 		IMPORT_SET_STATE(imp, LUSTRE_IMP_REPLAY);
 	} else {
 		DEBUG_REQ(D_HA, request, "%s: evicting (reconnect/recover flags not set: %x)",
@@ -1147,18 +1195,25 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
 	}
 
 finish:
+	ptlrpc_prepare_replay(imp);
 	rc = ptlrpc_import_recovery_state_machine(imp);
 	if (rc == -ENOTCONN) {
 		CDEBUG(D_HA, "evicted/aborted by %s@%s during recovery; invalidating and reconnecting\n",
 		       obd2cli_tgt(imp->imp_obd),
 		       imp->imp_connection->c_remote_uuid.uuid);
 		ptlrpc_connect_import(imp);
+		spin_lock(&imp->imp_lock);
+		imp->imp_connected = 0;
 		imp->imp_connect_tried = 1;
+		spin_unlock(&imp->imp_lock);
 		return 0;
 	}
 
 out:
+	spin_lock(&imp->imp_lock);
+	imp->imp_connected = 0;
 	imp->imp_connect_tried = 1;
+	spin_unlock(&imp->imp_lock);
 
 	if (rc != 0) {
 		IMPORT_SET_STATE(imp, LUSTRE_IMP_DISCON);
diff --git a/drivers/staging/lustre/lustre/ptlrpc/layout.c b/drivers/staging/lustre/lustre/ptlrpc/layout.c
index 839ef3e80c1a..99d7c667df28 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/layout.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/layout.c
@@ -48,14 +48,14 @@
 
 #include <linux/module.h>
 
-/* LUSTRE_VERSION_CODE */
-#include "../include/lustre_ver.h"
-
-#include "../include/obd_support.h"
-/* lustre_swab_mdt_body */
 #include "../include/lustre/lustre_idl.h"
-/* obd2cli_tgt() (required by DEBUG_REQ()) */
+
+#include "../include/llog_swab.h"
+#include "../include/lustre_debug.h"
+#include "../include/lustre_swab.h"
+#include "../include/lustre_ver.h"
 #include "../include/obd.h"
+#include "../include/obd_support.h"
 
 /* __REQ_LAYOUT_USER__ */
 #endif
@@ -121,7 +121,7 @@ static const struct req_msg_field *mdt_close_client[] = {
 	&RMF_CAPA1
 };
 
-static const struct req_msg_field *mdt_release_close_client[] = {
+static const struct req_msg_field *mdt_intent_close_client[] = {
 	&RMF_PTLRPC_BODY,
 	&RMF_MDT_EPOCH,
 	&RMF_REC_REINT,
@@ -257,6 +257,18 @@ static const struct req_msg_field *mds_reint_rename_client[] = {
 	&RMF_DLM_REQ
 };
 
+static const struct req_msg_field *mds_reint_migrate_client[] = {
+	&RMF_PTLRPC_BODY,
+	&RMF_REC_REINT,
+	&RMF_CAPA1,
+	&RMF_CAPA2,
+	&RMF_NAME,
+	&RMF_SYMTGT,
+	&RMF_DLM_REQ,
+	&RMF_MDT_EPOCH,
+	&RMF_CLOSE_DATA
+};
+
 static const struct req_msg_field *mds_last_unlink_server[] = {
 	&RMF_PTLRPC_BODY,
 	&RMF_MDT_BODY,
@@ -666,10 +678,9 @@ static struct req_format *req_formats[] = {
 	&RQF_MDS_GETXATTR,
 	&RQF_MDS_SYNC,
 	&RQF_MDS_CLOSE,
-	&RQF_MDS_RELEASE_CLOSE,
+	&RQF_MDS_INTENT_CLOSE,
 	&RQF_MDS_READPAGE,
 	&RQF_MDS_WRITEPAGE,
-	&RQF_MDS_DONE_WRITING,
 	&RQF_MDS_REINT,
 	&RQF_MDS_REINT_CREATE,
 	&RQF_MDS_REINT_CREATE_ACL,
@@ -679,9 +690,9 @@ static struct req_format *req_formats[] = {
 	&RQF_MDS_REINT_UNLINK,
 	&RQF_MDS_REINT_LINK,
 	&RQF_MDS_REINT_RENAME,
+	&RQF_MDS_REINT_MIGRATE,
 	&RQF_MDS_REINT_SETATTR,
 	&RQF_MDS_REINT_SETXATTR,
-	&RQF_MDS_QUOTACHECK,
 	&RQF_MDS_QUOTACTL,
 	&RQF_MDS_HSM_PROGRESS,
 	&RQF_MDS_HSM_CT_REGISTER,
@@ -691,10 +702,8 @@ static struct req_format *req_formats[] = {
 	&RQF_MDS_HSM_ACTION,
 	&RQF_MDS_HSM_REQUEST,
 	&RQF_MDS_SWAP_LAYOUTS,
-	&RQF_QC_CALLBACK,
 	&RQF_OST_CONNECT,
 	&RQF_OST_DISCONNECT,
-	&RQF_OST_QUOTACHECK,
 	&RQF_OST_QUOTACTL,
 	&RQF_OST_GETATTR,
 	&RQF_OST_SETATTR,
@@ -1180,14 +1189,6 @@ struct req_format RQF_LOG_CANCEL =
 	DEFINE_REQ_FMT0("OBD_LOG_CANCEL", log_cancel_client, empty);
 EXPORT_SYMBOL(RQF_LOG_CANCEL);
 
-struct req_format RQF_MDS_QUOTACHECK =
-	DEFINE_REQ_FMT0("MDS_QUOTACHECK", quotactl_only, empty);
-EXPORT_SYMBOL(RQF_MDS_QUOTACHECK);
-
-struct req_format RQF_OST_QUOTACHECK =
-	DEFINE_REQ_FMT0("OST_QUOTACHECK", quotactl_only, empty);
-EXPORT_SYMBOL(RQF_OST_QUOTACHECK);
-
 struct req_format RQF_MDS_QUOTACTL =
 	DEFINE_REQ_FMT0("MDS_QUOTACTL", quotactl_only, quotactl_only);
 EXPORT_SYMBOL(RQF_MDS_QUOTACTL);
@@ -1196,10 +1197,6 @@ struct req_format RQF_OST_QUOTACTL =
 	DEFINE_REQ_FMT0("OST_QUOTACTL", quotactl_only, quotactl_only);
 EXPORT_SYMBOL(RQF_OST_QUOTACTL);
 
-struct req_format RQF_QC_CALLBACK =
-	DEFINE_REQ_FMT0("QC_CALLBACK", quotactl_only, empty);
-EXPORT_SYMBOL(RQF_QC_CALLBACK);
-
 struct req_format RQF_MDS_GETSTATUS =
 	DEFINE_REQ_FMT0("MDS_GETSTATUS", mdt_body_only, mdt_body_capa);
 EXPORT_SYMBOL(RQF_MDS_GETSTATUS);
@@ -1270,6 +1267,11 @@ struct req_format RQF_MDS_REINT_RENAME =
 			mds_last_unlink_server);
 EXPORT_SYMBOL(RQF_MDS_REINT_RENAME);
 
+struct req_format RQF_MDS_REINT_MIGRATE =
+	DEFINE_REQ_FMT0("MDS_REINT_MIGRATE", mds_reint_migrate_client,
+			mds_last_unlink_server);
+EXPORT_SYMBOL(RQF_MDS_REINT_MIGRATE);
+
 struct req_format RQF_MDS_REINT_SETATTR =
 	DEFINE_REQ_FMT0("MDS_REINT_SETATTR",
 			mds_reint_setattr_client, mds_setattr_server);
@@ -1381,15 +1383,10 @@ struct req_format RQF_MDS_CLOSE =
 			mdt_close_client, mds_last_unlink_server);
 EXPORT_SYMBOL(RQF_MDS_CLOSE);
 
-struct req_format RQF_MDS_RELEASE_CLOSE =
+struct req_format RQF_MDS_INTENT_CLOSE =
 	DEFINE_REQ_FMT0("MDS_CLOSE",
-			mdt_release_close_client, mds_last_unlink_server);
-EXPORT_SYMBOL(RQF_MDS_RELEASE_CLOSE);
-
-struct req_format RQF_MDS_DONE_WRITING =
-	DEFINE_REQ_FMT0("MDS_DONE_WRITING",
-			mdt_close_client, mdt_body_only);
-EXPORT_SYMBOL(RQF_MDS_DONE_WRITING);
+			mdt_intent_close_client, mds_last_unlink_server);
+EXPORT_SYMBOL(RQF_MDS_INTENT_CLOSE);
 
 struct req_format RQF_MDS_READPAGE =
 	DEFINE_REQ_FMT0("MDS_READPAGE",
@@ -1874,13 +1871,14 @@ static void *__req_capsule_get(struct req_capsule *pill,
 	getter = (field->rmf_flags & RMF_F_STRING) ?
 		(typeof(getter))lustre_msg_string : lustre_msg_buf;
 
-	if (field->rmf_flags & RMF_F_STRUCT_ARRAY) {
+	if (field->rmf_flags & (RMF_F_STRUCT_ARRAY | RMF_F_NO_SIZE_CHECK)) {
 		/*
 		 * We've already asserted that field->rmf_size > 0 in
 		 * req_layout_init().
 		 */
 		len = lustre_msg_buflen(msg, offset);
-		if ((len % field->rmf_size) != 0) {
+		if (!(field->rmf_flags & RMF_F_NO_SIZE_CHECK) &&
+		    (len % field->rmf_size)) {
 			CERROR("%s: array field size mismatch %d modulo %u != 0 (%d)\n",
 			       field->rmf_name, len, field->rmf_size, loc);
 			return NULL;
diff --git a/drivers/staging/lustre/lustre/ptlrpc/llog_client.c b/drivers/staging/lustre/lustre/ptlrpc/llog_client.c
index 0f55c01feba8..110d9f505787 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/llog_client.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/llog_client.c
@@ -287,8 +287,13 @@ static int llog_client_read_header(const struct lu_env *env,
 		goto out;
 	}
 
-	memcpy(handle->lgh_hdr, hdr, sizeof(*hdr));
-	handle->lgh_last_idx = handle->lgh_hdr->llh_tail.lrt_index;
+	if (handle->lgh_hdr_size < hdr->llh_hdr.lrh_len) {
+		rc = -EFAULT;
+		goto out;
+	}
+
+	memcpy(handle->lgh_hdr, hdr, hdr->llh_hdr.lrh_len);
+	handle->lgh_last_idx = LLOG_HDR_TAIL(handle->lgh_hdr)->lrt_index;
 
 	/* sanity checks */
 	llh_hdr = &handle->lgh_hdr->llh_hdr;
@@ -296,9 +301,14 @@ static int llog_client_read_header(const struct lu_env *env,
 		CERROR("bad log header magic: %#x (expecting %#x)\n",
 		       llh_hdr->lrh_type, LLOG_HDR_MAGIC);
 		rc = -EIO;
-	} else if (llh_hdr->lrh_len != LLOG_CHUNK_SIZE) {
-		CERROR("incorrectly sized log header: %#x (expecting %#x)\n",
-		       llh_hdr->lrh_len, LLOG_CHUNK_SIZE);
+	} else if (llh_hdr->lrh_len !=
+		   LLOG_HDR_TAIL(handle->lgh_hdr)->lrt_len ||
+		   (llh_hdr->lrh_len & (llh_hdr->lrh_len - 1)) ||
+		   llh_hdr->lrh_len < LLOG_MIN_CHUNK_SIZE ||
+		   llh_hdr->lrh_len > handle->lgh_hdr_size) {
+		CERROR("incorrectly sized log header: %#x (expecting %#x) (power of two > 8192)\n",
+		       llh_hdr->lrh_len,
+		       LLOG_HDR_TAIL(handle->lgh_hdr)->lrt_len);
 		CERROR("you may need to re-run lconf --write_conf.\n");
 		rc = -EIO;
 	}
diff --git a/drivers/staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c b/drivers/staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c
index 9bad57d65db4..f87478180013 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c
@@ -479,8 +479,8 @@ static int ptlrpc_lprocfs_nrs_seq_show(struct seq_file *m, void *n)
 	struct ptlrpc_nrs_policy *policy;
 	struct ptlrpc_nrs_pol_info *infos;
 	struct ptlrpc_nrs_pol_info tmp;
-	unsigned num_pols;
-	unsigned pol_idx = 0;
+	unsigned int num_pols;
+	unsigned int pol_idx = 0;
 	bool hp = false;
 	int i;
 	int rc = 0;
diff --git a/drivers/staging/lustre/lustre/ptlrpc/niobuf.c b/drivers/staging/lustre/lustre/ptlrpc/niobuf.c
index 9c937398a085..da1209e40f03 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/niobuf.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/niobuf.c
@@ -114,7 +114,7 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req)
 	int rc2;
 	int posted_md;
 	int total_md;
-	__u64 xid;
+	u64 mbits;
 	lnet_handle_me_t me_h;
 	lnet_md_t md;
 
@@ -127,8 +127,7 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req)
 	LASSERT(desc->bd_md_max_brw <= PTLRPC_BULK_OPS_COUNT);
 	LASSERT(desc->bd_iov_count <= PTLRPC_MAX_BRW_PAGES);
 	LASSERT(desc->bd_req);
-	LASSERT(desc->bd_type == BULK_PUT_SINK ||
-		desc->bd_type == BULK_GET_SOURCE);
+	LASSERT(ptlrpc_is_bulk_op_passive(desc->bd_type));
 
 	/* cleanup the state of the bulk for it will be reused */
 	if (req->rq_resend || req->rq_send_state == LUSTRE_IMP_REPLAY)
@@ -143,40 +142,37 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req)
 	LASSERT(desc->bd_cbid.cbid_fn == client_bulk_callback);
 	LASSERT(desc->bd_cbid.cbid_arg == desc);
 
-	/* An XID is only used for a single request from the client.
-	 * For retried bulk transfers, a new XID will be allocated in
-	 * in ptlrpc_check_set() if it needs to be resent, so it is not
-	 * using the same RDMA match bits after an error.
-	 *
-	 * For multi-bulk RPCs, rq_xid is the last XID needed for bulks. The
-	 * first bulk XID is power-of-two aligned before rq_xid. LU-1431
-	 */
-	xid = req->rq_xid & ~((__u64)desc->bd_md_max_brw - 1);
+	total_md = (desc->bd_iov_count + LNET_MAX_IOV - 1) / LNET_MAX_IOV;
+	/* rq_mbits is matchbits of the final bulk */
+	mbits = req->rq_mbits - total_md + 1;
+
+	LASSERTF(mbits == (req->rq_mbits & PTLRPC_BULK_OPS_MASK),
+		 "first mbits = x%llu, last mbits = x%llu\n",
+		 mbits, req->rq_mbits);
 	LASSERTF(!(desc->bd_registered &&
 		   req->rq_send_state != LUSTRE_IMP_REPLAY) ||
-		 xid != desc->bd_last_xid,
-		 "registered: %d  rq_xid: %llu bd_last_xid: %llu\n",
-		 desc->bd_registered, xid, desc->bd_last_xid);
+		 mbits != desc->bd_last_mbits,
+		 "registered: %d  rq_mbits: %llu bd_last_mbits: %llu\n",
+		 desc->bd_registered, mbits, desc->bd_last_mbits);
 
-	total_md = (desc->bd_iov_count + LNET_MAX_IOV - 1) / LNET_MAX_IOV;
 	desc->bd_registered = 1;
-	desc->bd_last_xid = xid;
+	desc->bd_last_mbits = mbits;
 	desc->bd_md_count = total_md;
 	md.user_ptr = &desc->bd_cbid;
 	md.eq_handle = ptlrpc_eq_h;
 	md.threshold = 1;		       /* PUT or GET */
 
-	for (posted_md = 0; posted_md < total_md; posted_md++, xid++) {
+	for (posted_md = 0; posted_md < total_md; posted_md++, mbits++) {
 		md.options = PTLRPC_MD_OPTIONS |
-			     ((desc->bd_type == BULK_GET_SOURCE) ?
+			     (ptlrpc_is_bulk_op_get(desc->bd_type) ?
 			      LNET_MD_OP_GET : LNET_MD_OP_PUT);
 		ptlrpc_fill_bulk_md(&md, desc, posted_md);
 
-		rc = LNetMEAttach(desc->bd_portal, peer, xid, 0,
+		rc = LNetMEAttach(desc->bd_portal, peer, mbits, 0,
 				  LNET_UNLINK, LNET_INS_AFTER, &me_h);
 		if (rc != 0) {
 			CERROR("%s: LNetMEAttach failed x%llu/%d: rc = %d\n",
-			       desc->bd_import->imp_obd->obd_name, xid,
+			       desc->bd_import->imp_obd->obd_name, mbits,
 			       posted_md, rc);
 			break;
 		}
@@ -186,7 +182,7 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req)
 				  &desc->bd_mds[posted_md]);
 		if (rc != 0) {
 			CERROR("%s: LNetMDAttach failed x%llu/%d: rc = %d\n",
-			       desc->bd_import->imp_obd->obd_name, xid,
+			       desc->bd_import->imp_obd->obd_name, mbits,
 			       posted_md, rc);
 			rc2 = LNetMEUnlink(me_h);
 			LASSERT(rc2 == 0);
@@ -205,27 +201,19 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req)
 		return -ENOMEM;
 	}
 
-	/* Set rq_xid to matchbits of the final bulk so that server can
-	 * infer the number of bulks that were prepared
-	 */
-	req->rq_xid = --xid;
-	LASSERTF(desc->bd_last_xid == (req->rq_xid & PTLRPC_BULK_OPS_MASK),
-		 "bd_last_xid = x%llu, rq_xid = x%llu\n",
-		 desc->bd_last_xid, req->rq_xid);
-
 	spin_lock(&desc->bd_lock);
-	/* Holler if peer manages to touch buffers before he knows the xid */
+	/* Holler if peer manages to touch buffers before he knows the mbits */
 	if (desc->bd_md_count != total_md)
 		CWARN("%s: Peer %s touched %d buffers while I registered\n",
 		      desc->bd_import->imp_obd->obd_name, libcfs_id2str(peer),
 		      total_md - desc->bd_md_count);
 	spin_unlock(&desc->bd_lock);
 
-	CDEBUG(D_NET, "Setup %u bulk %s buffers: %u pages %u bytes, xid x%#llx-%#llx, portal %u\n",
+	CDEBUG(D_NET, "Setup %u bulk %s buffers: %u pages %u bytes, mbits x%#llx-%#llx, portal %u\n",
 	       desc->bd_md_count,
-	       desc->bd_type == BULK_GET_SOURCE ? "get-source" : "put-sink",
+	       ptlrpc_is_bulk_op_get(desc->bd_type) ? "get-source" : "put-sink",
 	       desc->bd_iov_count, desc->bd_nob,
-	       desc->bd_last_xid, req->rq_xid, desc->bd_portal);
+	       desc->bd_last_mbits, req->rq_mbits, desc->bd_portal);
 
 	return 0;
 }
@@ -521,6 +509,39 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 	lustre_msg_set_conn_cnt(request->rq_reqmsg, imp->imp_conn_cnt);
 	lustre_msghdr_set_flags(request->rq_reqmsg, imp->imp_msghdr_flags);
 
+	/*
+	 * If it's the first time to resend the request for EINPROGRESS,
+	 * we need to allocate a new XID (see after_reply()), it's different
+	 * from the resend for reply timeout.
+	 */
+	if (request->rq_nr_resend && list_empty(&request->rq_unreplied_list)) {
+		__u64 min_xid = 0;
+		/*
+		 * resend for EINPROGRESS, allocate new xid to avoid reply
+		 * reconstruction
+		 */
+		spin_lock(&imp->imp_lock);
+		ptlrpc_assign_next_xid_nolock(request);
+		request->rq_mbits = request->rq_xid;
+		min_xid = ptlrpc_known_replied_xid(imp);
+		spin_unlock(&imp->imp_lock);
+
+		lustre_msg_set_last_xid(request->rq_reqmsg, min_xid);
+		DEBUG_REQ(D_RPCTRACE, request, "Allocating new xid for resend on EINPROGRESS");
+	} else if (request->rq_bulk) {
+		ptlrpc_set_bulk_mbits(request);
+		lustre_msg_set_mbits(request->rq_reqmsg, request->rq_mbits);
+	}
+
+	if (list_empty(&request->rq_unreplied_list) ||
+	    request->rq_xid <= imp->imp_known_replied_xid) {
+		DEBUG_REQ(D_ERROR, request,
+			  "xid: %llu, replied: %llu, list_empty:%d\n",
+			  request->rq_xid, imp->imp_known_replied_xid,
+			  list_empty(&request->rq_unreplied_list));
+		LBUG();
+	}
+
 	/**
 	 * For enabled AT all request should have AT_SUPPORT in the
 	 * FULL import state when OBD_CONNECT_AT is set
@@ -537,8 +558,15 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 		mpflag = cfs_memory_pressure_get_and_set();
 
 	rc = sptlrpc_cli_wrap_request(request);
-	if (rc)
+	if (rc) {
+		/*
+		 * set rq_sent so that this request is treated
+		 * as a delayed send in the upper layers
+		 */
+		if (rc == -ENOMEM)
+			request->rq_sent = ktime_get_seconds();
 		goto out;
+	}
 
 	/* bulk register should be done after wrap_request() */
 	if (request->rq_bulk) {
diff --git a/drivers/staging/lustre/lustre/ptlrpc/nrs.c b/drivers/staging/lustre/lustre/ptlrpc/nrs.c
index d88faf61e740..7b6ffb195834 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/nrs.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/nrs.c
@@ -82,16 +82,9 @@ static int nrs_policy_ctl_locked(struct ptlrpc_nrs_policy *policy,
 
 static void nrs_policy_stop0(struct ptlrpc_nrs_policy *policy)
 {
-	struct ptlrpc_nrs *nrs = policy->pol_nrs;
-
-	if (policy->pol_desc->pd_ops->op_policy_stop) {
-		spin_unlock(&nrs->nrs_lock);
-
+	if (policy->pol_desc->pd_ops->op_policy_stop)
 		policy->pol_desc->pd_ops->op_policy_stop(policy);
 
-		spin_lock(&nrs->nrs_lock);
-	}
-
 	LASSERT(list_empty(&policy->pol_list_queued));
 	LASSERT(policy->pol_req_queued == 0 &&
 		policy->pol_req_started == 0);
@@ -619,6 +612,12 @@ static int nrs_policy_ctl(struct ptlrpc_nrs *nrs, char *name,
 		goto out;
 	}
 
+	if (policy->pol_state != NRS_POL_STATE_STARTED &&
+	    policy->pol_state != NRS_POL_STATE_STOPPED) {
+		rc = -EAGAIN;
+		goto out;
+	}
+
 	switch (opc) {
 		/**
 		 * Unknown opcode, pass it down to the policy-specific control
diff --git a/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c b/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c
index 871768511e8c..13f00b7cbbe5 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c
@@ -42,11 +42,14 @@
 
 #include "../../include/linux/libcfs/libcfs.h"
 
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
+#include "../include/lustre/ll_fiemap.h"
+
+#include "../include/llog_swab.h"
 #include "../include/lustre_net.h"
+#include "../include/lustre_swab.h"
 #include "../include/obd_cksum.h"
-#include "../include/lustre/ll_fiemap.h"
+#include "../include/obd_support.h"
+#include "../include/obd_class.h"
 
 #include "ptlrpc_internal.h"
 
@@ -942,6 +945,25 @@ __u32 lustre_msg_get_opc(struct lustre_msg *msg)
 }
 EXPORT_SYMBOL(lustre_msg_get_opc);
 
+__u16 lustre_msg_get_tag(struct lustre_msg *msg)
+{
+	switch (msg->lm_magic) {
+	case LUSTRE_MSG_MAGIC_V2: {
+		struct ptlrpc_body *pb = lustre_msg_ptlrpc_body(msg);
+
+		if (!pb) {
+			CERROR("invalid msg %p: no ptlrpc body!\n", msg);
+			return 0;
+		}
+		return pb->pb_tag;
+	}
+	default:
+		CERROR("incorrect message magic: %08x\n", msg->lm_magic);
+		return 0;
+	}
+}
+EXPORT_SYMBOL(lustre_msg_get_tag);
+
 __u64 lustre_msg_get_last_committed(struct lustre_msg *msg)
 {
 	switch (msg->lm_magic) {
@@ -1236,6 +1258,37 @@ void lustre_msg_set_opc(struct lustre_msg *msg, __u32 opc)
 	}
 }
 
+void lustre_msg_set_last_xid(struct lustre_msg *msg, u64 last_xid)
+{
+	switch (msg->lm_magic) {
+	case LUSTRE_MSG_MAGIC_V2: {
+		struct ptlrpc_body *pb = lustre_msg_ptlrpc_body(msg);
+
+		LASSERTF(pb, "invalid msg %p: no ptlrpc body!\n", msg);
+		pb->pb_last_xid = last_xid;
+		return;
+	}
+	default:
+		LASSERTF(0, "incorrect message magic: %08x\n", msg->lm_magic);
+	}
+}
+
+void lustre_msg_set_tag(struct lustre_msg *msg, __u16 tag)
+{
+	switch (msg->lm_magic) {
+	case LUSTRE_MSG_MAGIC_V2: {
+		struct ptlrpc_body *pb = lustre_msg_ptlrpc_body(msg);
+
+		LASSERTF(pb, "invalid msg %p: no ptlrpc body!\n", msg);
+		pb->pb_tag = tag;
+		return;
+	}
+	default:
+		LASSERTF(0, "incorrect message magic: %08x\n", msg->lm_magic);
+	}
+}
+EXPORT_SYMBOL(lustre_msg_set_tag);
+
 void lustre_msg_set_versions(struct lustre_msg *msg, __u64 *versions)
 {
 	switch (msg->lm_magic) {
@@ -1373,6 +1426,21 @@ void lustre_msg_set_cksum(struct lustre_msg *msg, __u32 cksum)
 	}
 }
 
+void lustre_msg_set_mbits(struct lustre_msg *msg, __u64 mbits)
+{
+	switch (msg->lm_magic) {
+	case LUSTRE_MSG_MAGIC_V2: {
+		struct ptlrpc_body *pb = lustre_msg_ptlrpc_body(msg);
+
+		LASSERTF(pb, "invalid msg %p: no ptlrpc body!\n", msg);
+		pb->pb_mbits = mbits;
+		return;
+	}
+	default:
+		LASSERTF(0, "incorrect message magic: %08x\n", msg->lm_magic);
+	}
+}
+
 void ptlrpc_request_set_replen(struct ptlrpc_request *req)
 {
 	int count = req_capsule_filled_sizes(&req->rq_pill, RCL_SERVER);
@@ -1442,7 +1510,7 @@ void lustre_swab_ptlrpc_body(struct ptlrpc_body *b)
 	__swab32s(&b->pb_opc);
 	__swab32s(&b->pb_status);
 	__swab64s(&b->pb_last_xid);
-	__swab64s(&b->pb_last_seen);
+	__swab16s(&b->pb_tag);
 	__swab64s(&b->pb_last_committed);
 	__swab64s(&b->pb_transno);
 	__swab32s(&b->pb_flags);
@@ -1456,7 +1524,12 @@ void lustre_swab_ptlrpc_body(struct ptlrpc_body *b)
 	__swab64s(&b->pb_pre_versions[1]);
 	__swab64s(&b->pb_pre_versions[2]);
 	__swab64s(&b->pb_pre_versions[3]);
-	CLASSERT(offsetof(typeof(*b), pb_padding) != 0);
+	__swab64s(&b->pb_mbits);
+	CLASSERT(offsetof(typeof(*b), pb_padding0) != 0);
+	CLASSERT(offsetof(typeof(*b), pb_padding1) != 0);
+	CLASSERT(offsetof(typeof(*b), pb_padding64_0) != 0);
+	CLASSERT(offsetof(typeof(*b), pb_padding64_1) != 0);
+	CLASSERT(offsetof(typeof(*b), pb_padding64_2) != 0);
 	/* While we need to maintain compatibility between
 	 * clients and servers without ptlrpc_body_v2 (< 2.3)
 	 * do not swab any fields beyond pb_jobid, as we are
@@ -1492,8 +1565,12 @@ void lustre_swab_connect(struct obd_connect_data *ocd)
 		__swab32s(&ocd->ocd_max_easize);
 	if (ocd->ocd_connect_flags & OBD_CONNECT_MAXBYTES)
 		__swab64s(&ocd->ocd_maxbytes);
+	if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS)
+		__swab16s(&ocd->ocd_maxmodrpcs);
+	CLASSERT(offsetof(typeof(*ocd), padding0));
 	CLASSERT(offsetof(typeof(*ocd), padding1) != 0);
-	CLASSERT(offsetof(typeof(*ocd), padding2) != 0);
+	if (ocd->ocd_connect_flags & OBD_CONNECT_FLAGS2)
+		__swab64s(&ocd->ocd_connect_flags2);
 	CLASSERT(offsetof(typeof(*ocd), padding3) != 0);
 	CLASSERT(offsetof(typeof(*ocd), padding4) != 0);
 	CLASSERT(offsetof(typeof(*ocd), padding5) != 0);
@@ -1666,7 +1743,7 @@ void lustre_swab_mdt_body(struct mdt_body *b)
 	__swab32s(&b->mbo_eadatasize);
 	__swab32s(&b->mbo_aclsize);
 	__swab32s(&b->mbo_max_mdsize);
-	__swab32s(&b->mbo_max_cookiesize);
+	CLASSERT(offsetof(typeof(*b), mbo_unused3));
 	__swab32s(&b->mbo_uid_h);
 	__swab32s(&b->mbo_gid_h);
 	CLASSERT(offsetof(typeof(*b), mbo_padding_5) != 0);
@@ -1675,9 +1752,10 @@ void lustre_swab_mdt_body(struct mdt_body *b)
 void lustre_swab_mdt_ioepoch(struct mdt_ioepoch *b)
 {
 	/* handle is opaque */
-	 __swab64s(&b->ioepoch);
-	 __swab32s(&b->flags);
-	 CLASSERT(offsetof(typeof(*b), padding) != 0);
+	/* mio_handle is opaque */
+	CLASSERT(offsetof(typeof(*b), mio_unused1));
+	CLASSERT(offsetof(typeof(*b), mio_unused2));
+	CLASSERT(offsetof(typeof(*b), mio_padding));
 }
 
 void lustre_swab_mgs_target_info(struct mgs_target_info *mti)
@@ -1772,7 +1850,7 @@ void lustre_swab_fid2path(struct getinfo_fid2path *gf)
 }
 EXPORT_SYMBOL(lustre_swab_fid2path);
 
-static void lustre_swab_fiemap_extent(struct ll_fiemap_extent *fm_extent)
+static void lustre_swab_fiemap_extent(struct fiemap_extent *fm_extent)
 {
 	__swab64s(&fm_extent->fe_logical);
 	__swab64s(&fm_extent->fe_physical);
@@ -1781,7 +1859,7 @@ static void lustre_swab_fiemap_extent(struct ll_fiemap_extent *fm_extent)
 	__swab32s(&fm_extent->fe_device);
 }
 
-void lustre_swab_fiemap(struct ll_user_fiemap *fiemap)
+void lustre_swab_fiemap(struct fiemap *fiemap)
 {
 	__u32 i;
 
@@ -1938,7 +2016,7 @@ static void lustre_swab_ldlm_res_id(struct ldlm_res_id *id)
 		__swab64s(&id->name[i]);
 }
 
-static void lustre_swab_ldlm_policy_data(ldlm_wire_policy_data_t *d)
+static void lustre_swab_ldlm_policy_data(union ldlm_wire_policy_data *d)
 {
 	/* the lock data is a union and the first two fields are always an
 	 * extent so it's ok to process an LDLM_EXTENT and LDLM_FLOCK lock
@@ -2062,8 +2140,6 @@ static void dump_obdo(struct obdo *oa)
 	if (valid & OBD_MD_FLHANDLE)
 		CDEBUG(D_RPCTRACE, "obdo: o_handle = %lld\n",
 		       oa->o_handle.cookie);
-	if (valid & OBD_MD_FLCOOKIE)
-		CDEBUG(D_RPCTRACE, "obdo: o_lcookie = (llog_cookie dumping not yet implemented)\n");
 }
 
 void dump_ost_body(struct ost_body *ob)
diff --git a/drivers/staging/lustre/lustre/ptlrpc/pers.c b/drivers/staging/lustre/lustre/ptlrpc/pers.c
index 5b9fb11c0b6b..94e9fa85d774 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/pers.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/pers.c
@@ -43,6 +43,8 @@
 void ptlrpc_fill_bulk_md(lnet_md_t *md, struct ptlrpc_bulk_desc *desc,
 			 int mdidx)
 {
+	int offset = mdidx * LNET_MAX_IOV;
+
 	CLASSERT(PTLRPC_MAX_BRW_PAGES < LI_POISON);
 
 	LASSERT(mdidx < desc->bd_md_max_brw);
@@ -50,23 +52,20 @@ void ptlrpc_fill_bulk_md(lnet_md_t *md, struct ptlrpc_bulk_desc *desc,
 	LASSERT(!(md->options & (LNET_MD_IOVEC | LNET_MD_KIOV |
 				 LNET_MD_PHYS)));
 
-	md->options |= LNET_MD_KIOV;
 	md->length = max(0, desc->bd_iov_count - mdidx * LNET_MAX_IOV);
 	md->length = min_t(unsigned int, LNET_MAX_IOV, md->length);
-	if (desc->bd_enc_iov)
-		md->start = &desc->bd_enc_iov[mdidx * LNET_MAX_IOV];
-	else
-		md->start = &desc->bd_iov[mdidx * LNET_MAX_IOV];
-}
-
-void ptlrpc_add_bulk_page(struct ptlrpc_bulk_desc *desc, struct page *page,
-			  int pageoffset, int len)
-{
-	lnet_kiov_t *kiov = &desc->bd_iov[desc->bd_iov_count];
-
-	kiov->bv_page = page;
-	kiov->bv_offset = pageoffset;
-	kiov->bv_len = len;
 
-	desc->bd_iov_count++;
+	if (ptlrpc_is_bulk_desc_kiov(desc->bd_type)) {
+		md->options |= LNET_MD_KIOV;
+		if (GET_ENC_KIOV(desc))
+			md->start = &BD_GET_ENC_KIOV(desc, offset);
+		else
+			md->start = &BD_GET_KIOV(desc, offset);
+	} else {
+		md->options |= LNET_MD_IOVEC;
+		if (GET_ENC_KVEC(desc))
+			md->start = &BD_GET_ENC_KVEC(desc, offset);
+		else
+			md->start = &BD_GET_KVEC(desc, offset);
+	}
 }
diff --git a/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_internal.h b/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_internal.h
index f14d193287da..e0f859ca6223 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_internal.h
+++ b/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_internal.h
@@ -55,8 +55,11 @@ int ptlrpcd_start(struct ptlrpcd_ctl *pc);
 /* client.c */
 void ptlrpc_at_adj_net_latency(struct ptlrpc_request *req,
 			       unsigned int service_time);
-struct ptlrpc_bulk_desc *ptlrpc_new_bulk(unsigned npages, unsigned max_brw,
-					 unsigned type, unsigned portal);
+struct ptlrpc_bulk_desc *ptlrpc_new_bulk(unsigned int nfrags,
+					 unsigned int max_brw,
+					 enum ptlrpc_bulk_op_type type,
+					 unsigned int portal,
+					 const struct ptlrpc_bulk_frag_ops *ops);
 int ptlrpc_request_cache_init(void);
 void ptlrpc_request_cache_fini(void);
 struct ptlrpc_request *ptlrpc_request_cache_alloc(gfp_t flags);
@@ -67,6 +70,10 @@ void ptlrpc_set_add_new_req(struct ptlrpcd_ctl *pc,
 int ptlrpc_expired_set(void *data);
 int ptlrpc_set_next_timeout(struct ptlrpc_request_set *);
 void ptlrpc_resend_req(struct ptlrpc_request *request);
+void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req);
+void ptlrpc_assign_next_xid_nolock(struct ptlrpc_request *req);
+__u64 ptlrpc_known_replied_xid(struct obd_import *imp);
+void ptlrpc_add_unreplied(struct ptlrpc_request *req);
 
 /* events.c */
 int ptlrpc_init_portals(void);
@@ -226,8 +233,6 @@ int ptlrpc_expire_one_request(struct ptlrpc_request *req, int async_unlink);
 /* pers.c */
 void ptlrpc_fill_bulk_md(lnet_md_t *md, struct ptlrpc_bulk_desc *desc,
 			 int mdcnt);
-void ptlrpc_add_bulk_page(struct ptlrpc_bulk_desc *desc, struct page *page,
-			  int pageoffset, int len);
 
 /* pack_generic.c */
 struct ptlrpc_reply_state *
@@ -322,6 +327,7 @@ static inline void ptlrpc_cli_req_init(struct ptlrpc_request *req)
 
 	INIT_LIST_HEAD(&cr->cr_set_chain);
 	INIT_LIST_HEAD(&cr->cr_ctx_chain);
+	INIT_LIST_HEAD(&cr->cr_unreplied_list);
 	init_waitqueue_head(&cr->cr_reply_waitq);
 	init_waitqueue_head(&cr->cr_set_waitq);
 }
@@ -338,4 +344,24 @@ static inline void ptlrpc_srv_req_init(struct ptlrpc_request *req)
 	INIT_LIST_HEAD(&sr->sr_hist_list);
 }
 
+static inline bool ptlrpc_req_is_connect(struct ptlrpc_request *req)
+{
+	if (lustre_msg_get_opc(req->rq_reqmsg) == MDS_CONNECT ||
+	    lustre_msg_get_opc(req->rq_reqmsg) == OST_CONNECT ||
+	    lustre_msg_get_opc(req->rq_reqmsg) == MGS_CONNECT)
+		return true;
+	else
+		return false;
+}
+
+static inline bool ptlrpc_req_is_disconnect(struct ptlrpc_request *req)
+{
+	if (lustre_msg_get_opc(req->rq_reqmsg) == MDS_DISCONNECT ||
+	    lustre_msg_get_opc(req->rq_reqmsg) == OST_DISCONNECT ||
+	    lustre_msg_get_opc(req->rq_reqmsg) == MGS_DISCONNECT)
+		return true;
+	else
+		return false;
+}
+
 #endif /* PTLRPC_INTERNAL_H */
diff --git a/drivers/staging/lustre/lustre/ptlrpc/recover.c b/drivers/staging/lustre/lustre/ptlrpc/recover.c
index 405faf0dc9fc..c00449036884 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/recover.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/recover.c
@@ -111,7 +111,9 @@ int ptlrpc_replay_next(struct obd_import *imp, int *inflight)
 			 * all of it's requests being replayed, it's safe to
 			 * use a cursor to accelerate the search
 			 */
-			imp->imp_replay_cursor = imp->imp_replay_cursor->next;
+			if (!imp->imp_resend_replay ||
+			    imp->imp_replay_cursor == &imp->imp_committed_list)
+				imp->imp_replay_cursor = imp->imp_replay_cursor->next;
 
 			while (imp->imp_replay_cursor !=
 			       &imp->imp_committed_list) {
@@ -155,10 +157,24 @@ int ptlrpc_replay_next(struct obd_import *imp, int *inflight)
 		lustre_msg_add_flags(req->rq_reqmsg, MSG_RESENT);
 
 	spin_lock(&imp->imp_lock);
+	/* The resend replay request may have been removed from the
+	 * unreplied list.
+	 */
+	if (req && imp->imp_resend_replay &&
+	    list_empty(&req->rq_unreplied_list)) {
+		ptlrpc_add_unreplied(req);
+		imp->imp_known_replied_xid = ptlrpc_known_replied_xid(imp);
+	}
+
 	imp->imp_resend_replay = 0;
 	spin_unlock(&imp->imp_lock);
 
 	if (req) {
+		/* The request should have been added back in unreplied list
+		 * by ptlrpc_prepare_replay().
+		 */
+		LASSERT(!list_empty(&req->rq_unreplied_list));
+
 		rc = ptlrpc_replay_req(req);
 		if (rc) {
 			CERROR("recovery replay error %d for req %llu\n",
@@ -194,7 +210,13 @@ int ptlrpc_resend(struct obd_import *imp)
 		LASSERTF((long)req > PAGE_SIZE && req != LP_POISON,
 			 "req %p bad\n", req);
 		LASSERTF(req->rq_type != LI_POISON, "req %p freed\n", req);
-		if (!ptlrpc_no_resend(req))
+
+		/*
+		 * If the request is allowed to be sent during replay and it
+		 * is not timeout yet, then it does not need to be resent.
+		 */
+		if (!ptlrpc_no_resend(req) &&
+		    (req->rq_timedout || !req->rq_allow_replay))
 			ptlrpc_resend_req(req);
 	}
 	spin_unlock(&imp->imp_lock);
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec.c b/drivers/staging/lustre/lustre/ptlrpc/sec.c
index a7416cd9ac71..e860df7c45a2 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec.c
@@ -379,7 +379,7 @@ int sptlrpc_req_get_ctx(struct ptlrpc_request *req)
 
 	if (!req->rq_cli_ctx) {
 		CERROR("req %p: fail to get context\n", req);
-		return -ENOMEM;
+		return -ECONNREFUSED;
 	}
 
 	return 0;
@@ -515,6 +515,13 @@ static int sptlrpc_req_replace_dead_ctx(struct ptlrpc_request *req)
 
 		set_current_state(TASK_INTERRUPTIBLE);
 		schedule_timeout(msecs_to_jiffies(MSEC_PER_SEC));
+	} else if (unlikely(!test_bit(PTLRPC_CTX_UPTODATE_BIT, &newctx->cc_flags))) {
+		/*
+		 * new ctx not up to date yet
+		 */
+		CDEBUG(D_SEC,
+		       "ctx (%p, fl %lx) doesn't switch, not up to date yet\n",
+		       newctx, newctx->cc_flags);
 	} else {
 		/*
 		 * it's possible newctx == oldctx if we're switching
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c b/drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c
index b2cc5ea6cb93..2fe9085e2034 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c
@@ -108,6 +108,7 @@ static struct ptlrpc_enc_page_pool {
 	unsigned long    epp_st_lowfree;	/* lowest free pages reached */
 	unsigned int     epp_st_max_wqlen;      /* highest waitqueue length */
 	unsigned long       epp_st_max_wait;       /* in jiffies */
+	unsigned long	 epp_st_outofmem;	/* # of out of mem requests */
 	/*
 	 * pointers to pools
 	 */
@@ -139,7 +140,8 @@ int sptlrpc_proc_enc_pool_seq_show(struct seq_file *m, void *v)
 		   "cache missing:	   %lu\n"
 		   "low free mark:	   %lu\n"
 		   "max waitqueue depth:     %u\n"
-		   "max wait time:	   %ld/%lu\n",
+		   "max wait time:	   %ld/%lu\n"
+		   "out of mem:		 %lu\n",
 		   totalram_pages,
 		   PAGES_PER_POOL,
 		   page_pools.epp_max_pages,
@@ -158,7 +160,8 @@ int sptlrpc_proc_enc_pool_seq_show(struct seq_file *m, void *v)
 		   page_pools.epp_st_lowfree,
 		   page_pools.epp_st_max_wqlen,
 		   page_pools.epp_st_max_wait,
-		   msecs_to_jiffies(MSEC_PER_SEC));
+		   msecs_to_jiffies(MSEC_PER_SEC),
+		   page_pools.epp_st_outofmem);
 
 	spin_unlock(&page_pools.epp_lock);
 
@@ -306,12 +309,30 @@ static inline void enc_pools_wakeup(void)
 	}
 }
 
+/*
+ * Export the number of free pages in the pool
+ */
+int get_free_pages_in_pool(void)
+{
+	return page_pools.epp_free_pages;
+}
+
+/*
+ * Let outside world know if enc_pool full capacity is reached
+ */
+int pool_is_at_full_capacity(void)
+{
+	return (page_pools.epp_total_pages == page_pools.epp_max_pages);
+}
+
 void sptlrpc_enc_pool_put_pages(struct ptlrpc_bulk_desc *desc)
 {
 	int p_idx, g_idx;
 	int i;
 
-	if (!desc->bd_enc_iov)
+	LASSERT(ptlrpc_is_bulk_desc_kiov(desc->bd_type));
+
+	if (!GET_ENC_KIOV(desc))
 		return;
 
 	LASSERT(desc->bd_iov_count > 0);
@@ -326,12 +347,12 @@ void sptlrpc_enc_pool_put_pages(struct ptlrpc_bulk_desc *desc)
 	LASSERT(page_pools.epp_pools[p_idx]);
 
 	for (i = 0; i < desc->bd_iov_count; i++) {
-		LASSERT(desc->bd_enc_iov[i].bv_page);
+		LASSERT(BD_GET_ENC_KIOV(desc, i).bv_page);
 		LASSERT(g_idx != 0 || page_pools.epp_pools[p_idx]);
 		LASSERT(!page_pools.epp_pools[p_idx][g_idx]);
 
 		page_pools.epp_pools[p_idx][g_idx] =
-					desc->bd_enc_iov[i].bv_page;
+			BD_GET_ENC_KIOV(desc, i).bv_page;
 
 		if (++g_idx == PAGES_PER_POOL) {
 			p_idx++;
@@ -345,8 +366,8 @@ void sptlrpc_enc_pool_put_pages(struct ptlrpc_bulk_desc *desc)
 
 	spin_unlock(&page_pools.epp_lock);
 
-	kfree(desc->bd_enc_iov);
-	desc->bd_enc_iov = NULL;
+	kfree(GET_ENC_KIOV(desc));
+	GET_ENC_KIOV(desc) = NULL;
 }
 
 static inline void enc_pools_alloc(void)
@@ -404,6 +425,7 @@ int sptlrpc_enc_pool_init(void)
 	page_pools.epp_st_lowfree = 0;
 	page_pools.epp_st_max_wqlen = 0;
 	page_pools.epp_st_max_wait = 0;
+	page_pools.epp_st_outofmem = 0;
 
 	enc_pools_alloc();
 	if (!page_pools.epp_pools)
@@ -431,13 +453,14 @@ void sptlrpc_enc_pool_fini(void)
 
 	if (page_pools.epp_st_access > 0) {
 		CDEBUG(D_SEC,
-		       "max pages %lu, grows %u, grow fails %u, shrinks %u, access %lu, missing %lu, max qlen %u, max wait %ld/%ld\n",
+		       "max pages %lu, grows %u, grow fails %u, shrinks %u, access %lu, missing %lu, max qlen %u, max wait %ld/%ld, out of mem %lu\n",
 		       page_pools.epp_st_max_pages, page_pools.epp_st_grows,
 		       page_pools.epp_st_grow_fails,
 		       page_pools.epp_st_shrinks, page_pools.epp_st_access,
 		       page_pools.epp_st_missings, page_pools.epp_st_max_wqlen,
 		       page_pools.epp_st_max_wait,
-		       msecs_to_jiffies(MSEC_PER_SEC));
+		       msecs_to_jiffies(MSEC_PER_SEC),
+		       page_pools.epp_st_outofmem);
 	}
 }
 
@@ -520,10 +543,11 @@ int sptlrpc_get_bulk_checksum(struct ptlrpc_bulk_desc *desc, __u8 alg,
 	hashsize = cfs_crypto_hash_digestsize(cfs_hash_alg_id[alg]);
 
 	for (i = 0; i < desc->bd_iov_count; i++) {
-		cfs_crypto_hash_update_page(hdesc, desc->bd_iov[i].bv_page,
-					    desc->bd_iov[i].bv_offset &
+		cfs_crypto_hash_update_page(hdesc,
+					    BD_GET_KIOV(desc, i).bv_page,
+					    BD_GET_KIOV(desc, i).bv_offset &
 					    ~PAGE_MASK,
-					    desc->bd_iov[i].bv_len);
+					    BD_GET_KIOV(desc, i).bv_len);
 	}
 
 	if (hashsize > buflen) {
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec_plain.c b/drivers/staging/lustre/lustre/ptlrpc/sec_plain.c
index cd305bcb334a..c5e7a2309fce 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec_plain.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec_plain.c
@@ -153,14 +153,16 @@ static void corrupt_bulk_data(struct ptlrpc_bulk_desc *desc)
 	char *ptr;
 	unsigned int off, i;
 
+	LASSERT(ptlrpc_is_bulk_desc_kiov(desc->bd_type));
+
 	for (i = 0; i < desc->bd_iov_count; i++) {
-		if (desc->bd_iov[i].bv_len == 0)
+		if (!BD_GET_KIOV(desc, i).bv_len)
 			continue;
 
-		ptr = kmap(desc->bd_iov[i].bv_page);
-		off = desc->bd_iov[i].bv_offset & ~PAGE_MASK;
+		ptr = kmap(BD_GET_KIOV(desc, i).bv_page);
+		off = BD_GET_KIOV(desc, i).bv_offset & ~PAGE_MASK;
 		ptr[off] ^= 0x1;
-		kunmap(desc->bd_iov[i].bv_page);
+		kunmap(BD_GET_KIOV(desc, i).bv_page);
 		return;
 	}
 }
@@ -352,11 +354,11 @@ int plain_cli_unwrap_bulk(struct ptlrpc_cli_ctx *ctx,
 
 	/* fix the actual data size */
 	for (i = 0, nob = 0; i < desc->bd_iov_count; i++) {
-		if (desc->bd_iov[i].bv_len + nob > desc->bd_nob_transferred) {
-			desc->bd_iov[i].bv_len =
-				desc->bd_nob_transferred - nob;
-		}
-		nob += desc->bd_iov[i].bv_len;
+		struct bio_vec bv_desc = BD_GET_KIOV(desc, i);
+
+		if (bv_desc.bv_len + nob > desc->bd_nob_transferred)
+			bv_desc.bv_len = desc->bd_nob_transferred - nob;
+		nob += bv_desc.bv_len;
 	}
 
 	rc = plain_verify_bulk_csum(desc, req->rq_flvr.u_bulk.hash.hash_alg,
diff --git a/drivers/staging/lustre/lustre/ptlrpc/service.c b/drivers/staging/lustre/lustre/ptlrpc/service.c
index 72f39308eebb..70c70558e177 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/service.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/service.c
@@ -343,9 +343,9 @@ ptlrpc_server_nthreads_check(struct ptlrpc_service *svc,
 			     struct ptlrpc_service_conf *conf)
 {
 	struct ptlrpc_service_thr_conf *tc = &conf->psc_thr;
-	unsigned init;
-	unsigned total;
-	unsigned nthrs;
+	unsigned int init;
+	unsigned int total;
+	unsigned int nthrs;
 	int weight;
 
 	/*
@@ -2541,8 +2541,9 @@ int ptlrpc_hr_init(void)
 
 		hrp->hrp_nthrs = cfs_cpt_weight(ptlrpc_hr.hr_cpt_table, i);
 		hrp->hrp_nthrs /= weight;
+		if (hrp->hrp_nthrs == 0)
+			hrp->hrp_nthrs = 1;
 
-		LASSERT(hrp->hrp_nthrs > 0);
 		hrp->hrp_thrs =
 			kzalloc_node(hrp->hrp_nthrs * sizeof(*hrt), GFP_NOFS,
 				     cfs_cpt_spread_node(ptlrpc_hr.hr_cpt_table,
diff --git a/drivers/staging/lustre/lustre/ptlrpc/wiretest.c b/drivers/staging/lustre/lustre/ptlrpc/wiretest.c
index b05b1f935e4c..a04e36cf6dd4 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/wiretest.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/wiretest.c
@@ -195,49 +195,29 @@ void lustre_assert_wire_constants(void)
 	LASSERTF(REINT_MAX == 10, "found %lld\n",
 		 (long long)REINT_MAX);
 	LASSERTF(DISP_IT_EXECD == 0x00000001UL, "found 0x%.8xUL\n",
-		 (unsigned)DISP_IT_EXECD);
+		 (unsigned int)DISP_IT_EXECD);
 	LASSERTF(DISP_LOOKUP_EXECD == 0x00000002UL, "found 0x%.8xUL\n",
-		 (unsigned)DISP_LOOKUP_EXECD);
+		 (unsigned int)DISP_LOOKUP_EXECD);
 	LASSERTF(DISP_LOOKUP_NEG == 0x00000004UL, "found 0x%.8xUL\n",
-		 (unsigned)DISP_LOOKUP_NEG);
+		 (unsigned int)DISP_LOOKUP_NEG);
 	LASSERTF(DISP_LOOKUP_POS == 0x00000008UL, "found 0x%.8xUL\n",
-		 (unsigned)DISP_LOOKUP_POS);
+		 (unsigned int)DISP_LOOKUP_POS);
 	LASSERTF(DISP_OPEN_CREATE == 0x00000010UL, "found 0x%.8xUL\n",
-		 (unsigned)DISP_OPEN_CREATE);
+		 (unsigned int)DISP_OPEN_CREATE);
 	LASSERTF(DISP_OPEN_OPEN == 0x00000020UL, "found 0x%.8xUL\n",
-		 (unsigned)DISP_OPEN_OPEN);
+		 (unsigned int)DISP_OPEN_OPEN);
 	LASSERTF(DISP_ENQ_COMPLETE == 0x00400000UL, "found 0x%.8xUL\n",
-		 (unsigned)DISP_ENQ_COMPLETE);
+		 (unsigned int)DISP_ENQ_COMPLETE);
 	LASSERTF(DISP_ENQ_OPEN_REF == 0x00800000UL, "found 0x%.8xUL\n",
-		 (unsigned)DISP_ENQ_OPEN_REF);
+		 (unsigned int)DISP_ENQ_OPEN_REF);
 	LASSERTF(DISP_ENQ_CREATE_REF == 0x01000000UL, "found 0x%.8xUL\n",
-		 (unsigned)DISP_ENQ_CREATE_REF);
+		 (unsigned int)DISP_ENQ_CREATE_REF);
 	LASSERTF(DISP_OPEN_LOCK == 0x02000000UL, "found 0x%.8xUL\n",
-		 (unsigned)DISP_OPEN_LOCK);
+		 (unsigned int)DISP_OPEN_LOCK);
 	LASSERTF(MDS_STATUS_CONN == 1, "found %lld\n",
 		 (long long)MDS_STATUS_CONN);
 	LASSERTF(MDS_STATUS_LOV == 2, "found %lld\n",
 		 (long long)MDS_STATUS_LOV);
-	LASSERTF(LUSTRE_BFLAG_UNCOMMITTED_WRITES == 1, "found %lld\n",
-		 (long long)LUSTRE_BFLAG_UNCOMMITTED_WRITES);
-	LASSERTF(MF_SOM_CHANGE == 0x00000001UL, "found 0x%.8xUL\n",
-		 (unsigned)MF_SOM_CHANGE);
-	LASSERTF(MF_EPOCH_OPEN == 0x00000002UL, "found 0x%.8xUL\n",
-		 (unsigned)MF_EPOCH_OPEN);
-	LASSERTF(MF_EPOCH_CLOSE == 0x00000004UL, "found 0x%.8xUL\n",
-		 (unsigned)MF_EPOCH_CLOSE);
-	LASSERTF(MF_MDC_CANCEL_FID1 == 0x00000008UL, "found 0x%.8xUL\n",
-		 (unsigned)MF_MDC_CANCEL_FID1);
-	LASSERTF(MF_MDC_CANCEL_FID2 == 0x00000010UL, "found 0x%.8xUL\n",
-		 (unsigned)MF_MDC_CANCEL_FID2);
-	LASSERTF(MF_MDC_CANCEL_FID3 == 0x00000020UL, "found 0x%.8xUL\n",
-		 (unsigned)MF_MDC_CANCEL_FID3);
-	LASSERTF(MF_MDC_CANCEL_FID4 == 0x00000040UL, "found 0x%.8xUL\n",
-		 (unsigned)MF_MDC_CANCEL_FID4);
-	LASSERTF(MF_SOM_AU == 0x00000080UL, "found 0x%.8xUL\n",
-		 (unsigned)MF_SOM_AU);
-	LASSERTF(MF_GETATTR_LOCK == 0x00000100UL, "found 0x%.8xUL\n",
-		 (unsigned)MF_GETATTR_LOCK);
 	LASSERTF(MDS_ATTR_MODE == 0x0000000000000001ULL, "found 0x%.16llxULL\n",
 		 (long long)MDS_ATTR_MODE);
 	LASSERTF(MDS_ATTR_UID == 0x0000000000000002ULL, "found 0x%.16llxULL\n",
@@ -420,15 +400,13 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct lustre_mdt_attrs *)0)->lma_self_fid) == 16, "found %lld\n",
 		 (long long)(int)sizeof(((struct lustre_mdt_attrs *)0)->lma_self_fid));
 	LASSERTF(LMAI_RELEASED == 0x00000001UL, "found 0x%.8xUL\n",
-		 (unsigned)LMAI_RELEASED);
+		 (unsigned int)LMAI_RELEASED);
 	LASSERTF(LMAC_HSM == 0x00000001UL, "found 0x%.8xUL\n",
-		 (unsigned)LMAC_HSM);
-	LASSERTF(LMAC_SOM == 0x00000002UL, "found 0x%.8xUL\n",
-		 (unsigned)LMAC_SOM);
+		 (unsigned int)LMAC_HSM);
 	LASSERTF(LMAC_NOT_IN_OI == 0x00000004UL, "found 0x%.8xUL\n",
-		 (unsigned)LMAC_NOT_IN_OI);
+		 (unsigned int)LMAC_NOT_IN_OI);
 	LASSERTF(LMAC_FID_ON_OST == 0x00000008UL, "found 0x%.8xUL\n",
-		 (unsigned)LMAC_FID_ON_OST);
+		 (unsigned int)LMAC_FID_ON_OST);
 
 	/* Checks for struct ost_id */
 	LASSERTF((int)sizeof(struct ost_id) == 16, "found %lld\n",
@@ -478,11 +456,11 @@ void lustre_assert_wire_constants(void)
 	LASSERTF(FID_SEQ_LOV_DEFAULT == 0xffffffffffffffffULL, "found 0x%.16llxULL\n",
 		 (long long)FID_SEQ_LOV_DEFAULT);
 	LASSERTF(FID_OID_SPECIAL_BFL == 0x00000001UL, "found 0x%.8xUL\n",
-		 (unsigned)FID_OID_SPECIAL_BFL);
+		 (unsigned int)FID_OID_SPECIAL_BFL);
 	LASSERTF(FID_OID_DOT_LUSTRE == 0x00000001UL, "found 0x%.8xUL\n",
-		 (unsigned)FID_OID_DOT_LUSTRE);
+		 (unsigned int)FID_OID_DOT_LUSTRE);
 	LASSERTF(FID_OID_DOT_LUSTRE_OBF == 0x00000002UL, "found 0x%.8xUL\n",
-		 (unsigned)FID_OID_DOT_LUSTRE_OBF);
+		 (unsigned int)FID_OID_DOT_LUSTRE_OBF);
 
 	/* Checks for struct lu_dirent */
 	LASSERTF((int)sizeof(struct lu_dirent) == 32, "found %lld\n",
@@ -512,11 +490,11 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct lu_dirent *)0)->lde_name[0]) == 1, "found %lld\n",
 		 (long long)(int)sizeof(((struct lu_dirent *)0)->lde_name[0]));
 	LASSERTF(LUDA_FID == 0x00000001UL, "found 0x%.8xUL\n",
-		 (unsigned)LUDA_FID);
+		 (unsigned int)LUDA_FID);
 	LASSERTF(LUDA_TYPE == 0x00000002UL, "found 0x%.8xUL\n",
-		 (unsigned)LUDA_TYPE);
+		 (unsigned int)LUDA_TYPE);
 	LASSERTF(LUDA_64BITHASH == 0x00000004UL, "found 0x%.8xUL\n",
-		 (unsigned)LUDA_64BITHASH);
+		 (unsigned int)LUDA_64BITHASH);
 
 	/* Checks for struct luda_type */
 	LASSERTF((int)sizeof(struct luda_type) == 2, "found %lld\n",
@@ -635,10 +613,18 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_last_xid));
 	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_last_xid) == 8, "found %lld\n",
 		 (long long)(int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_last_xid));
-	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_last_seen) == 32, "found %lld\n",
-		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_last_seen));
-	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_last_seen) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_last_seen));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_tag) == 32, "found %lld\n",
+		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_tag));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_tag) == 2, "found %lld\n",
+		 (long long)(int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_tag));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_padding0) == 34, "found %lld\n",
+		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_padding0));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding0) == 2, "found %lld\n",
+		 (long long)(int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding0));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_padding1) == 36, "found %lld\n",
+		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_padding1));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding1) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding1));
 	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_last_committed) == 40, "found %lld\n",
 		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_last_committed));
 	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_last_committed) == 8, "found %lld\n",
@@ -680,10 +666,22 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_pre_versions));
 	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_pre_versions) == 32, "found %lld\n",
 		 (long long)(int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_pre_versions));
-	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_padding) == 120, "found %lld\n",
-		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_padding));
-	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding) == 32, "found %lld\n",
-		 (long long)(int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_mbits) == 120, "found %lld\n",
+		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_mbits));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_mbits) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_mbits));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_padding64_0) == 128, "found %lld\n",
+		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_padding64_0));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding64_0) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding64_0));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_padding64_1) == 136, "found %lld\n",
+		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_padding64_1));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding64_1) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding64_1));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_padding64_2) == 144, "found %lld\n",
+		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_padding64_2));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding64_2) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding64_2));
 	CLASSERT(LUSTRE_JOBID_SIZE == 32);
 	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_jobid) == 152, "found %lld\n",
 		 (long long)(int)offsetof(struct ptlrpc_body_v3, pb_jobid));
@@ -713,10 +711,18 @@ void lustre_assert_wire_constants(void)
 		 (int)offsetof(struct ptlrpc_body_v3, pb_last_xid), (int)offsetof(struct ptlrpc_body_v2, pb_last_xid));
 	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_last_xid) == (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_last_xid), "%d != %d\n",
 		 (int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_last_xid), (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_last_xid));
-	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_last_seen) == (int)offsetof(struct ptlrpc_body_v2, pb_last_seen), "%d != %d\n",
-		 (int)offsetof(struct ptlrpc_body_v3, pb_last_seen), (int)offsetof(struct ptlrpc_body_v2, pb_last_seen));
-	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_last_seen) == (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_last_seen), "%d != %d\n",
-		 (int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_last_seen), (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_last_seen));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_tag) == (int)offsetof(struct ptlrpc_body_v2, pb_tag), "%d != %d\n",
+		 (int)offsetof(struct ptlrpc_body_v3, pb_tag), (int)offsetof(struct ptlrpc_body_v2, pb_tag));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_tag) == (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_tag), "%d != %d\n",
+		 (int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_tag), (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_tag));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_padding0) == (int)offsetof(struct ptlrpc_body_v2, pb_padding0), "%d != %d\n",
+		 (int)offsetof(struct ptlrpc_body_v3, pb_padding0), (int)offsetof(struct ptlrpc_body_v2, pb_padding0));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding0) == (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_padding0), "%d != %d\n",
+		 (int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding0), (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_padding0));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_padding1) == (int)offsetof(struct ptlrpc_body_v2, pb_padding1), "%d != %d\n",
+		 (int)offsetof(struct ptlrpc_body_v3, pb_padding1), (int)offsetof(struct ptlrpc_body_v2, pb_padding1));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding1) == (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_padding1), "%d != %d\n",
+		 (int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding1), (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_padding1));
 	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_last_committed) == (int)offsetof(struct ptlrpc_body_v2, pb_last_committed), "%d != %d\n",
 		 (int)offsetof(struct ptlrpc_body_v3, pb_last_committed), (int)offsetof(struct ptlrpc_body_v2, pb_last_committed));
 	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_last_committed) == (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_last_committed), "%d != %d\n",
@@ -757,10 +763,22 @@ void lustre_assert_wire_constants(void)
 		 (int)offsetof(struct ptlrpc_body_v3, pb_pre_versions), (int)offsetof(struct ptlrpc_body_v2, pb_pre_versions));
 	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_pre_versions) == (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_pre_versions), "%d != %d\n",
 		 (int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_pre_versions), (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_pre_versions));
-	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_padding) == (int)offsetof(struct ptlrpc_body_v2, pb_padding), "%d != %d\n",
-		 (int)offsetof(struct ptlrpc_body_v3, pb_padding), (int)offsetof(struct ptlrpc_body_v2, pb_padding));
-	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding) == (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_padding), "%d != %d\n",
-		 (int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding), (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_padding));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_mbits) == (int)offsetof(struct ptlrpc_body_v2, pb_mbits), "%d != %d\n",
+		 (int)offsetof(struct ptlrpc_body_v3, pb_mbits), (int)offsetof(struct ptlrpc_body_v2, pb_mbits));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_mbits) == (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_mbits), "%d != %d\n",
+		 (int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_mbits), (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_mbits));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_padding64_0) == (int)offsetof(struct ptlrpc_body_v2, pb_padding64_0), "%d != %d\n",
+		 (int)offsetof(struct ptlrpc_body_v3, pb_padding64_0), (int)offsetof(struct ptlrpc_body_v2, pb_padding64_0));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding64_0) == (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_padding64_0), "%d != %d\n",
+		 (int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding64_0), (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_padding64_0));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_padding64_1) == (int)offsetof(struct ptlrpc_body_v2, pb_padding64_1), "%d != %d\n",
+		 (int)offsetof(struct ptlrpc_body_v3, pb_padding64_1), (int)offsetof(struct ptlrpc_body_v2, pb_padding64_1));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding64_1) == (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_padding64_1), "%d != %d\n",
+		 (int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding64_1), (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_padding64_1));
+	LASSERTF((int)offsetof(struct ptlrpc_body_v3, pb_padding64_2) == (int)offsetof(struct ptlrpc_body_v2, pb_padding64_2), "%d != %d\n",
+		 (int)offsetof(struct ptlrpc_body_v3, pb_padding64_2), (int)offsetof(struct ptlrpc_body_v2, pb_padding64_2));
+	LASSERTF((int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding64_2) == (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_padding64_2), "%d != %d\n",
+		 (int)sizeof(((struct ptlrpc_body_v3 *)0)->pb_padding64_2), (int)sizeof(((struct ptlrpc_body_v2 *)0)->pb_padding64_2));
 	LASSERTF(MSG_PTLRPC_BODY_OFF == 0, "found %lld\n",
 		 (long long)MSG_PTLRPC_BODY_OFF);
 	LASSERTF(REQ_REC_OFF == 1, "found %lld\n",
@@ -802,41 +820,41 @@ void lustre_assert_wire_constants(void)
 	LASSERTF(MSGHDR_CKSUM_INCOMPAT18 == 2, "found %lld\n",
 		 (long long)MSGHDR_CKSUM_INCOMPAT18);
 	LASSERTF(MSG_OP_FLAG_MASK == 0xffff0000UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_OP_FLAG_MASK);
+		 (unsigned int)MSG_OP_FLAG_MASK);
 	LASSERTF(MSG_OP_FLAG_SHIFT == 16, "found %lld\n",
 		 (long long)MSG_OP_FLAG_SHIFT);
 	LASSERTF(MSG_GEN_FLAG_MASK == 0x0000ffffUL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_GEN_FLAG_MASK);
+		 (unsigned int)MSG_GEN_FLAG_MASK);
 	LASSERTF(MSG_LAST_REPLAY == 0x00000001UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_LAST_REPLAY);
+		 (unsigned int)MSG_LAST_REPLAY);
 	LASSERTF(MSG_RESENT == 0x00000002UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_RESENT);
+		 (unsigned int)MSG_RESENT);
 	LASSERTF(MSG_REPLAY == 0x00000004UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_REPLAY);
+		 (unsigned int)MSG_REPLAY);
 	LASSERTF(MSG_DELAY_REPLAY == 0x00000010UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_DELAY_REPLAY);
+		 (unsigned int)MSG_DELAY_REPLAY);
 	LASSERTF(MSG_VERSION_REPLAY == 0x00000020UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_VERSION_REPLAY);
+		 (unsigned int)MSG_VERSION_REPLAY);
 	LASSERTF(MSG_REQ_REPLAY_DONE == 0x00000040UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_REQ_REPLAY_DONE);
+		 (unsigned int)MSG_REQ_REPLAY_DONE);
 	LASSERTF(MSG_LOCK_REPLAY_DONE == 0x00000080UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_LOCK_REPLAY_DONE);
+		 (unsigned int)MSG_LOCK_REPLAY_DONE);
 	LASSERTF(MSG_CONNECT_RECOVERING == 0x00000001UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_CONNECT_RECOVERING);
+		 (unsigned int)MSG_CONNECT_RECOVERING);
 	LASSERTF(MSG_CONNECT_RECONNECT == 0x00000002UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_CONNECT_RECONNECT);
+		 (unsigned int)MSG_CONNECT_RECONNECT);
 	LASSERTF(MSG_CONNECT_REPLAYABLE == 0x00000004UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_CONNECT_REPLAYABLE);
+		 (unsigned int)MSG_CONNECT_REPLAYABLE);
 	LASSERTF(MSG_CONNECT_LIBCLIENT == 0x00000010UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_CONNECT_LIBCLIENT);
+		 (unsigned int)MSG_CONNECT_LIBCLIENT);
 	LASSERTF(MSG_CONNECT_INITIAL == 0x00000020UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_CONNECT_INITIAL);
+		 (unsigned int)MSG_CONNECT_INITIAL);
 	LASSERTF(MSG_CONNECT_ASYNC == 0x00000040UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_CONNECT_ASYNC);
+		 (unsigned int)MSG_CONNECT_ASYNC);
 	LASSERTF(MSG_CONNECT_NEXT_VER == 0x00000080UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_CONNECT_NEXT_VER);
+		 (unsigned int)MSG_CONNECT_NEXT_VER);
 	LASSERTF(MSG_CONNECT_TRANSNO == 0x00000100UL, "found 0x%.8xUL\n",
-		 (unsigned)MSG_CONNECT_TRANSNO);
+		 (unsigned int)MSG_CONNECT_TRANSNO);
 
 	/* Checks for struct obd_connect_data */
 	LASSERTF((int)sizeof(struct obd_connect_data) == 192, "found %lld\n",
@@ -905,14 +923,22 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct obd_connect_data, ocd_maxbytes));
 	LASSERTF((int)sizeof(((struct obd_connect_data *)0)->ocd_maxbytes) == 8, "found %lld\n",
 		 (long long)(int)sizeof(((struct obd_connect_data *)0)->ocd_maxbytes));
-	LASSERTF((int)offsetof(struct obd_connect_data, padding1) == 72, "found %lld\n",
+	LASSERTF((int)offsetof(struct obd_connect_data, ocd_maxmodrpcs) == 72, "found %lld\n",
+		 (long long)(int)offsetof(struct obd_connect_data, ocd_maxmodrpcs));
+	LASSERTF((int)sizeof(((struct obd_connect_data *)0)->ocd_maxmodrpcs) == 2, "found %lld\n",
+		 (long long)(int)sizeof(((struct obd_connect_data *)0)->ocd_maxmodrpcs));
+	LASSERTF((int)offsetof(struct obd_connect_data, padding0) == 74, "found %lld\n",
+		 (long long)(int)offsetof(struct obd_connect_data, padding0));
+	LASSERTF((int)sizeof(((struct obd_connect_data *)0)->padding0) == 2, "found %lld\n",
+		 (long long)(int)sizeof(((struct obd_connect_data *)0)->padding0));
+	LASSERTF((int)offsetof(struct obd_connect_data, padding1) == 76, "found %lld\n",
 		 (long long)(int)offsetof(struct obd_connect_data, padding1));
-	LASSERTF((int)sizeof(((struct obd_connect_data *)0)->padding1) == 8, "found %lld\n",
+	LASSERTF((int)sizeof(((struct obd_connect_data *)0)->padding1) == 4, "found %lld\n",
 		 (long long)(int)sizeof(((struct obd_connect_data *)0)->padding1));
-	LASSERTF((int)offsetof(struct obd_connect_data, padding2) == 80, "found %lld\n",
-		 (long long)(int)offsetof(struct obd_connect_data, padding2));
-	LASSERTF((int)sizeof(((struct obd_connect_data *)0)->padding2) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct obd_connect_data *)0)->padding2));
+	LASSERTF((int)offsetof(struct obd_connect_data, ocd_connect_flags2) == 80, "found %lld\n",
+		 (long long)(int)offsetof(struct obd_connect_data, ocd_connect_flags2));
+	LASSERTF((int)sizeof(((struct obd_connect_data *)0)->ocd_connect_flags2) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct obd_connect_data *)0)->ocd_connect_flags2));
 	LASSERTF((int)offsetof(struct obd_connect_data, padding3) == 88, "found %lld\n",
 		 (long long)(int)offsetof(struct obd_connect_data, padding3));
 	LASSERTF((int)sizeof(((struct obd_connect_data *)0)->padding3) == 8, "found %lld\n",
@@ -1075,14 +1101,24 @@ void lustre_assert_wire_constants(void)
 		 OBD_CONNECT_LFSCK);
 	LASSERTF(OBD_CONNECT_UNLINK_CLOSE == 0x100000000000000ULL, "found 0x%.16llxULL\n",
 		 OBD_CONNECT_UNLINK_CLOSE);
+	LASSERTF(OBD_CONNECT_MULTIMODRPCS == 0x200000000000000ULL, "found 0x%.16llxULL\n",
+		 OBD_CONNECT_MULTIMODRPCS);
 	LASSERTF(OBD_CONNECT_DIR_STRIPE == 0x400000000000000ULL, "found 0x%.16llxULL\n",
 		 OBD_CONNECT_DIR_STRIPE);
+	LASSERTF(OBD_CONNECT_SUBTREE == 0x800000000000000ULL, "found 0x%.16llxULL\n",
+		 OBD_CONNECT_SUBTREE);
+	LASSERTF(OBD_CONNECT_LOCK_AHEAD == 0x1000000000000000ULL, "found 0x%.16llxULL\n",
+		 OBD_CONNECT_LOCK_AHEAD);
+	LASSERTF(OBD_CONNECT_OBDOPACK == 0x4000000000000000ULL, "found 0x%.16llxULL\n",
+		 OBD_CONNECT_OBDOPACK);
+	LASSERTF(OBD_CONNECT_FLAGS2 == 0x8000000000000000ULL, "found 0x%.16llxULL\n",
+		 OBD_CONNECT_FLAGS2);
 	LASSERTF(OBD_CKSUM_CRC32 == 0x00000001UL, "found 0x%.8xUL\n",
-		 (unsigned)OBD_CKSUM_CRC32);
+		 (unsigned int)OBD_CKSUM_CRC32);
 	LASSERTF(OBD_CKSUM_ADLER == 0x00000002UL, "found 0x%.8xUL\n",
-		 (unsigned)OBD_CKSUM_ADLER);
+		 (unsigned int)OBD_CKSUM_ADLER);
 	LASSERTF(OBD_CKSUM_CRC32C == 0x00000004UL, "found 0x%.8xUL\n",
-		 (unsigned)OBD_CKSUM_CRC32C);
+		 (unsigned int)OBD_CKSUM_CRC32C);
 
 	/* Checks for struct obdo */
 	LASSERTF((int)sizeof(struct obdo) == 208, "found %lld\n",
@@ -1239,8 +1275,6 @@ void lustre_assert_wire_constants(void)
 		 OBD_MD_FLCKSUM);
 	LASSERTF(OBD_MD_FLQOS == (0x00200000ULL), "found 0x%.16llxULL\n",
 		 OBD_MD_FLQOS);
-	LASSERTF(OBD_MD_FLCOOKIE == (0x00800000ULL), "found 0x%.16llxULL\n",
-		 OBD_MD_FLCOOKIE);
 	LASSERTF(OBD_MD_FLGROUP == (0x01000000ULL), "found 0x%.16llxULL\n",
 		 OBD_MD_FLGROUP);
 	LASSERTF(OBD_MD_FLFID == (0x02000000ULL), "found 0x%.16llxULL\n",
@@ -1394,13 +1428,13 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)sizeof(((struct lov_mds_md_v3 *)0)->lmm_objects[0]));
 	CLASSERT(LOV_MAGIC_V3 == (0x0BD30000 | 0x0BD0));
 	LASSERTF(LOV_PATTERN_RAID0 == 0x00000001UL, "found 0x%.8xUL\n",
-		 (unsigned)LOV_PATTERN_RAID0);
+		 (unsigned int)LOV_PATTERN_RAID0);
 	LASSERTF(LOV_PATTERN_RAID1 == 0x00000002UL, "found 0x%.8xUL\n",
-		 (unsigned)LOV_PATTERN_RAID1);
+		 (unsigned int)LOV_PATTERN_RAID1);
 	LASSERTF(LOV_PATTERN_FIRST == 0x00000100UL, "found 0x%.8xUL\n",
-		 (unsigned)LOV_PATTERN_FIRST);
+		 (unsigned int)LOV_PATTERN_FIRST);
 	LASSERTF(LOV_PATTERN_CMOBD == 0x00000200UL, "found 0x%.8xUL\n",
-		 (unsigned)LOV_PATTERN_CMOBD);
+		 (unsigned int)LOV_PATTERN_CMOBD);
 
 	/* Checks for struct lmv_mds_md_v1 */
 	LASSERTF((int)sizeof(struct lmv_mds_md_v1) == 56, "found %lld\n",
@@ -1542,6 +1576,8 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct obd_ioobj, ioo_bufcnt));
 	LASSERTF((int)sizeof(((struct obd_ioobj *)0)->ioo_bufcnt) == 4, "found %lld\n",
 		 (long long)(int)sizeof(((struct obd_ioobj *)0)->ioo_bufcnt));
+	LASSERTF(IOOBJ_MAX_BRW_BITS == 16, "found %lld\n",
+		 (long long)IOOBJ_MAX_BRW_BITS);
 
 	/* Checks for union lquota_id */
 	LASSERTF((int)sizeof(union lquota_id) == 16, "found %lld\n",
@@ -1817,10 +1853,10 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct mdt_body, mbo_max_mdsize));
 	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_max_mdsize) == 4, "found %lld\n",
 		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_max_mdsize));
-	LASSERTF((int)offsetof(struct mdt_body, mbo_max_cookiesize) == 160, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_body, mbo_max_cookiesize));
-	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_max_cookiesize) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_max_cookiesize));
+	LASSERTF((int)offsetof(struct mdt_body, mbo_unused3) == 160, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_body, mbo_unused3));
+	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_unused3) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_body *)0)->mbo_unused3));
 	LASSERTF((int)offsetof(struct mdt_body, mbo_uid_h) == 164, "found %lld\n",
 		 (long long)(int)offsetof(struct mdt_body, mbo_uid_h));
 	LASSERTF((int)sizeof(((struct mdt_body *)0)->mbo_uid_h) == 4, "found %lld\n",
@@ -1857,12 +1893,6 @@ void lustre_assert_wire_constants(void)
 		 MDS_FMODE_CLOSED);
 	LASSERTF(MDS_FMODE_EXEC == 000000000004UL, "found 0%.11oUL\n",
 		 MDS_FMODE_EXEC);
-	LASSERTF(MDS_FMODE_EPOCH == 000001000000UL, "found 0%.11oUL\n",
-		 MDS_FMODE_EPOCH);
-	LASSERTF(MDS_FMODE_TRUNC == 000002000000UL, "found 0%.11oUL\n",
-		 MDS_FMODE_TRUNC);
-	LASSERTF(MDS_FMODE_SOM == 000004000000UL, "found 0%.11oUL\n",
-		 MDS_FMODE_SOM);
 	LASSERTF(MDS_OPEN_CREATED == 000000000010UL, "found 0%.11oUL\n",
 		 MDS_OPEN_CREATED);
 	LASSERTF(MDS_OPEN_CROSS == 000000000020UL, "found 0%.11oUL\n",
@@ -1905,10 +1935,20 @@ void lustre_assert_wire_constants(void)
 		 LUSTRE_IMMUTABLE_FL);
 	LASSERTF(LUSTRE_APPEND_FL == 0x00000020, "found 0x%.8x\n",
 		 LUSTRE_APPEND_FL);
+	LASSERTF(LUSTRE_NODUMP_FL == 0x00000040, "found 0x%.8x\n",
+		 LUSTRE_NODUMP_FL);
 	LASSERTF(LUSTRE_NOATIME_FL == 0x00000080, "found 0x%.8x\n",
 		 LUSTRE_NOATIME_FL);
+	LASSERTF(LUSTRE_INDEX_FL == 0x00001000, "found 0x%.8x\n",
+		 LUSTRE_INDEX_FL);
 	LASSERTF(LUSTRE_DIRSYNC_FL == 0x00010000, "found 0x%.8x\n",
 		 LUSTRE_DIRSYNC_FL);
+	LASSERTF(LUSTRE_TOPDIR_FL == 0x00020000, "found 0x%.8x\n",
+		 LUSTRE_TOPDIR_FL);
+	LASSERTF(LUSTRE_DIRECTIO_FL == 0x00100000, "found 0x%.8x\n",
+		 LUSTRE_DIRECTIO_FL);
+	LASSERTF(LUSTRE_INLINE_DATA_FL == 0x10000000, "found 0x%.8x\n",
+		 LUSTRE_INLINE_DATA_FL);
 	LASSERTF(MDS_INODELOCK_LOOKUP == 0x000001, "found 0x%.8x\n",
 		 MDS_INODELOCK_LOOKUP);
 	LASSERTF(MDS_INODELOCK_UPDATE == 0x000002, "found 0x%.8x\n",
@@ -1921,22 +1961,22 @@ void lustre_assert_wire_constants(void)
 	/* Checks for struct mdt_ioepoch */
 	LASSERTF((int)sizeof(struct mdt_ioepoch) == 24, "found %lld\n",
 		 (long long)(int)sizeof(struct mdt_ioepoch));
-	LASSERTF((int)offsetof(struct mdt_ioepoch, handle) == 0, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_ioepoch, handle));
-	LASSERTF((int)sizeof(((struct mdt_ioepoch *)0)->handle) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_ioepoch *)0)->handle));
-	LASSERTF((int)offsetof(struct mdt_ioepoch, ioepoch) == 8, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_ioepoch, ioepoch));
-	LASSERTF((int)sizeof(((struct mdt_ioepoch *)0)->ioepoch) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_ioepoch *)0)->ioepoch));
-	LASSERTF((int)offsetof(struct mdt_ioepoch, flags) == 16, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_ioepoch, flags));
-	LASSERTF((int)sizeof(((struct mdt_ioepoch *)0)->flags) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_ioepoch *)0)->flags));
-	LASSERTF((int)offsetof(struct mdt_ioepoch, padding) == 20, "found %lld\n",
-		 (long long)(int)offsetof(struct mdt_ioepoch, padding));
-	LASSERTF((int)sizeof(((struct mdt_ioepoch *)0)->padding) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct mdt_ioepoch *)0)->padding));
+	LASSERTF((int)offsetof(struct mdt_ioepoch, mio_handle) == 0, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_ioepoch, mio_handle));
+	LASSERTF((int)sizeof(((struct mdt_ioepoch *)0)->mio_handle) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_ioepoch *)0)->mio_handle));
+	LASSERTF((int)offsetof(struct mdt_ioepoch, mio_unused1) == 8, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_ioepoch, mio_unused1));
+	LASSERTF((int)sizeof(((struct mdt_ioepoch *)0)->mio_unused1) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_ioepoch *)0)->mio_unused1));
+	LASSERTF((int)offsetof(struct mdt_ioepoch, mio_unused2) == 16, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_ioepoch, mio_unused2));
+	LASSERTF((int)sizeof(((struct mdt_ioepoch *)0)->mio_unused2) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_ioepoch *)0)->mio_unused2));
+	LASSERTF((int)offsetof(struct mdt_ioepoch, mio_padding) == 20, "found %lld\n",
+		 (long long)(int)offsetof(struct mdt_ioepoch, mio_padding));
+	LASSERTF((int)sizeof(((struct mdt_ioepoch *)0)->mio_padding) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct mdt_ioepoch *)0)->mio_padding));
 
 	/* Checks for struct mdt_rec_setattr */
 	LASSERTF((int)sizeof(struct mdt_rec_setattr) == 136, "found %lld\n",
@@ -3520,21 +3560,21 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct llogd_conn_body *)0)->lgdc_ctxt_idx) == 4, "found %lld\n",
 		 (long long)(int)sizeof(((struct llogd_conn_body *)0)->lgdc_ctxt_idx));
 
-	/* Checks for struct ll_fiemap_info_key */
+	/* Checks for struct fiemap_info_key */
 	LASSERTF((int)sizeof(struct ll_fiemap_info_key) == 248, "found %lld\n",
 		 (long long)(int)sizeof(struct ll_fiemap_info_key));
-	LASSERTF((int)offsetof(struct ll_fiemap_info_key, name[8]) == 8, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_fiemap_info_key, name[8]));
-	LASSERTF((int)sizeof(((struct ll_fiemap_info_key *)0)->name[8]) == 1, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_fiemap_info_key *)0)->name[8]));
-	LASSERTF((int)offsetof(struct ll_fiemap_info_key, oa) == 8, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_fiemap_info_key, oa));
-	LASSERTF((int)sizeof(((struct ll_fiemap_info_key *)0)->oa) == 208, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_fiemap_info_key *)0)->oa));
-	LASSERTF((int)offsetof(struct ll_fiemap_info_key, fiemap) == 216, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_fiemap_info_key, fiemap));
-	LASSERTF((int)sizeof(((struct ll_fiemap_info_key *)0)->fiemap) == 32, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_fiemap_info_key *)0)->fiemap));
+	LASSERTF((int)offsetof(struct ll_fiemap_info_key, lfik_name[8]) == 8, "found %lld\n",
+		 (long long)(int)offsetof(struct ll_fiemap_info_key, lfik_name[8]));
+	LASSERTF((int)sizeof(((struct ll_fiemap_info_key *)0)->lfik_name[8]) == 1, "found %lld\n",
+		 (long long)(int)sizeof(((struct ll_fiemap_info_key *)0)->lfik_name[8]));
+	LASSERTF((int)offsetof(struct ll_fiemap_info_key, lfik_oa) == 8, "found %lld\n",
+		 (long long)(int)offsetof(struct ll_fiemap_info_key, lfik_oa));
+	LASSERTF((int)sizeof(((struct ll_fiemap_info_key *)0)->lfik_oa) == 208, "found %lld\n",
+		 (long long)(int)sizeof(((struct ll_fiemap_info_key *)0)->lfik_oa));
+	LASSERTF((int)offsetof(struct ll_fiemap_info_key, lfik_fiemap) == 216, "found %lld\n",
+		 (long long)(int)offsetof(struct ll_fiemap_info_key, lfik_fiemap));
+	LASSERTF((int)sizeof(((struct ll_fiemap_info_key *)0)->lfik_fiemap) == 32, "found %lld\n",
+		 (long long)(int)sizeof(((struct ll_fiemap_info_key *)0)->lfik_fiemap));
 
 	/* Checks for struct mgs_target_info */
 	LASSERTF((int)sizeof(struct mgs_target_info) == 4544, "found %lld\n",
@@ -3670,64 +3710,64 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct getinfo_fid2path *)0)->gf_path[0]) == 1, "found %lld\n",
 		 (long long)(int)sizeof(((struct getinfo_fid2path *)0)->gf_path[0]));
 
-	/* Checks for struct ll_user_fiemap */
-	LASSERTF((int)sizeof(struct ll_user_fiemap) == 32, "found %lld\n",
-		 (long long)(int)sizeof(struct ll_user_fiemap));
-	LASSERTF((int)offsetof(struct ll_user_fiemap, fm_start) == 0, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_user_fiemap, fm_start));
-	LASSERTF((int)sizeof(((struct ll_user_fiemap *)0)->fm_start) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_user_fiemap *)0)->fm_start));
-	LASSERTF((int)offsetof(struct ll_user_fiemap, fm_length) == 8, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_user_fiemap, fm_length));
-	LASSERTF((int)sizeof(((struct ll_user_fiemap *)0)->fm_length) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_user_fiemap *)0)->fm_length));
-	LASSERTF((int)offsetof(struct ll_user_fiemap, fm_flags) == 16, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_user_fiemap, fm_flags));
-	LASSERTF((int)sizeof(((struct ll_user_fiemap *)0)->fm_flags) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_user_fiemap *)0)->fm_flags));
-	LASSERTF((int)offsetof(struct ll_user_fiemap, fm_mapped_extents) == 20, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_user_fiemap, fm_mapped_extents));
-	LASSERTF((int)sizeof(((struct ll_user_fiemap *)0)->fm_mapped_extents) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_user_fiemap *)0)->fm_mapped_extents));
-	LASSERTF((int)offsetof(struct ll_user_fiemap, fm_extent_count) == 24, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_user_fiemap, fm_extent_count));
-	LASSERTF((int)sizeof(((struct ll_user_fiemap *)0)->fm_extent_count) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_user_fiemap *)0)->fm_extent_count));
-	LASSERTF((int)offsetof(struct ll_user_fiemap, fm_reserved) == 28, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_user_fiemap, fm_reserved));
-	LASSERTF((int)sizeof(((struct ll_user_fiemap *)0)->fm_reserved) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_user_fiemap *)0)->fm_reserved));
-	LASSERTF((int)offsetof(struct ll_user_fiemap, fm_extents) == 32, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_user_fiemap, fm_extents));
-	LASSERTF((int)sizeof(((struct ll_user_fiemap *)0)->fm_extents) == 0, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_user_fiemap *)0)->fm_extents));
+	/* Checks for struct fiemap */
+	LASSERTF((int)sizeof(struct fiemap) == 32, "found %lld\n",
+		 (long long)(int)sizeof(struct fiemap));
+	LASSERTF((int)offsetof(struct fiemap, fm_start) == 0, "found %lld\n",
+		 (long long)(int)offsetof(struct fiemap, fm_start));
+	LASSERTF((int)sizeof(((struct fiemap *)0)->fm_start) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct fiemap *)0)->fm_start));
+	LASSERTF((int)offsetof(struct fiemap, fm_length) == 8, "found %lld\n",
+		 (long long)(int)offsetof(struct fiemap, fm_length));
+	LASSERTF((int)sizeof(((struct fiemap *)0)->fm_length) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct fiemap *)0)->fm_length));
+	LASSERTF((int)offsetof(struct fiemap, fm_flags) == 16, "found %lld\n",
+		 (long long)(int)offsetof(struct fiemap, fm_flags));
+	LASSERTF((int)sizeof(((struct fiemap *)0)->fm_flags) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct fiemap *)0)->fm_flags));
+	LASSERTF((int)offsetof(struct fiemap, fm_mapped_extents) == 20, "found %lld\n",
+		 (long long)(int)offsetof(struct fiemap, fm_mapped_extents));
+	LASSERTF((int)sizeof(((struct fiemap *)0)->fm_mapped_extents) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct fiemap *)0)->fm_mapped_extents));
+	LASSERTF((int)offsetof(struct fiemap, fm_extent_count) == 24, "found %lld\n",
+		 (long long)(int)offsetof(struct fiemap, fm_extent_count));
+	LASSERTF((int)sizeof(((struct fiemap *)0)->fm_extent_count) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct fiemap *)0)->fm_extent_count));
+	LASSERTF((int)offsetof(struct fiemap, fm_reserved) == 28, "found %lld\n",
+		 (long long)(int)offsetof(struct fiemap, fm_reserved));
+	LASSERTF((int)sizeof(((struct fiemap *)0)->fm_reserved) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct fiemap *)0)->fm_reserved));
+	LASSERTF((int)offsetof(struct fiemap, fm_extents) == 32, "found %lld\n",
+		 (long long)(int)offsetof(struct fiemap, fm_extents));
+	LASSERTF((int)sizeof(((struct fiemap *)0)->fm_extents) == 0, "found %lld\n",
+		 (long long)(int)sizeof(((struct fiemap *)0)->fm_extents));
 	CLASSERT(FIEMAP_FLAG_SYNC == 0x00000001);
 	CLASSERT(FIEMAP_FLAG_XATTR == 0x00000002);
 	CLASSERT(FIEMAP_FLAG_DEVICE_ORDER == 0x40000000);
 
-	/* Checks for struct ll_fiemap_extent */
-	LASSERTF((int)sizeof(struct ll_fiemap_extent) == 56, "found %lld\n",
-		 (long long)(int)sizeof(struct ll_fiemap_extent));
-	LASSERTF((int)offsetof(struct ll_fiemap_extent, fe_logical) == 0, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_fiemap_extent, fe_logical));
-	LASSERTF((int)sizeof(((struct ll_fiemap_extent *)0)->fe_logical) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_fiemap_extent *)0)->fe_logical));
-	LASSERTF((int)offsetof(struct ll_fiemap_extent, fe_physical) == 8, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_fiemap_extent, fe_physical));
-	LASSERTF((int)sizeof(((struct ll_fiemap_extent *)0)->fe_physical) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_fiemap_extent *)0)->fe_physical));
-	LASSERTF((int)offsetof(struct ll_fiemap_extent, fe_length) == 16, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_fiemap_extent, fe_length));
-	LASSERTF((int)sizeof(((struct ll_fiemap_extent *)0)->fe_length) == 8, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_fiemap_extent *)0)->fe_length));
-	LASSERTF((int)offsetof(struct ll_fiemap_extent, fe_flags) == 40, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_fiemap_extent, fe_flags));
-	LASSERTF((int)sizeof(((struct ll_fiemap_extent *)0)->fe_flags) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_fiemap_extent *)0)->fe_flags));
-	LASSERTF((int)offsetof(struct ll_fiemap_extent, fe_device) == 44, "found %lld\n",
-		 (long long)(int)offsetof(struct ll_fiemap_extent, fe_device));
-	LASSERTF((int)sizeof(((struct ll_fiemap_extent *)0)->fe_device) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct ll_fiemap_extent *)0)->fe_device));
+	/* Checks for struct fiemap_extent */
+	LASSERTF((int)sizeof(struct fiemap_extent) == 56, "found %lld\n",
+		 (long long)(int)sizeof(struct fiemap_extent));
+	LASSERTF((int)offsetof(struct fiemap_extent, fe_logical) == 0, "found %lld\n",
+		 (long long)(int)offsetof(struct fiemap_extent, fe_logical));
+	LASSERTF((int)sizeof(((struct fiemap_extent *)0)->fe_logical) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct fiemap_extent *)0)->fe_logical));
+	LASSERTF((int)offsetof(struct fiemap_extent, fe_physical) == 8, "found %lld\n",
+		 (long long)(int)offsetof(struct fiemap_extent, fe_physical));
+	LASSERTF((int)sizeof(((struct fiemap_extent *)0)->fe_physical) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct fiemap_extent *)0)->fe_physical));
+	LASSERTF((int)offsetof(struct fiemap_extent, fe_length) == 16, "found %lld\n",
+		 (long long)(int)offsetof(struct fiemap_extent, fe_length));
+	LASSERTF((int)sizeof(((struct fiemap_extent *)0)->fe_length) == 8, "found %lld\n",
+		 (long long)(int)sizeof(((struct fiemap_extent *)0)->fe_length));
+	LASSERTF((int)offsetof(struct fiemap_extent, fe_flags) == 40, "found %lld\n",
+		 (long long)(int)offsetof(struct fiemap_extent, fe_flags));
+	LASSERTF((int)sizeof(((struct fiemap_extent *)0)->fe_flags) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct fiemap_extent *)0)->fe_flags));
+	LASSERTF((int)offsetof(struct fiemap_extent, fe_reserved[0]) == 44, "found %lld\n",
+		 (long long)(int)offsetof(struct fiemap_extent, fe_reserved[0]));
+	LASSERTF((int)sizeof(((struct fiemap_extent *)0)->fe_reserved[0]) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct fiemap_extent *)0)->fe_reserved[0]));
 	CLASSERT(FIEMAP_EXTENT_LAST == 0x00000001);
 	CLASSERT(FIEMAP_EXTENT_UNKNOWN == 0x00000002);
 	CLASSERT(FIEMAP_EXTENT_DELALLOC == 0x00000004);
@@ -4093,9 +4133,9 @@ void lustre_assert_wire_constants(void)
 	LASSERTF((int)sizeof(((struct hsm_request *)0)->hr_data_len) == 4, "found %lld\n",
 		 (long long)(int)sizeof(((struct hsm_request *)0)->hr_data_len));
 	LASSERTF(HSM_FORCE_ACTION == 0x00000001UL, "found 0x%.8xUL\n",
-		 (unsigned)HSM_FORCE_ACTION);
+		 (unsigned int)HSM_FORCE_ACTION);
 	LASSERTF(HSM_GHOST_COPY == 0x00000002UL, "found 0x%.8xUL\n",
-		 (unsigned)HSM_GHOST_COPY);
+		 (unsigned int)HSM_GHOST_COPY);
 
 	/* Checks for struct hsm_user_request */
 	LASSERTF((int)sizeof(struct hsm_user_request) == 24, "found %lld\n",
diff --git a/drivers/staging/lustre/sysfs-fs-lustre b/drivers/staging/lustre/sysfs-fs-lustre
index 20206ba965af..8691c6543a9c 100644
--- a/drivers/staging/lustre/sysfs-fs-lustre
+++ b/drivers/staging/lustre/sysfs-fs-lustre
@@ -11,7 +11,7 @@ Description:
 		Shows if the lustre module has pinger support.
 		"on" means yes and "off" means no.
 
-What:		/sys/fs/lustre/health
+What:		/sys/fs/lustre/health_check
 Date:		May 2015
 Contact:	"Oleg Drokin" <oleg.drokin@intel.com>
 Description:
diff --git a/drivers/staging/media/bcm2048/radio-bcm2048.c b/drivers/staging/media/bcm2048/radio-bcm2048.c
index 0a9f8589ecfb..37bd439ee08b 100644
--- a/drivers/staging/media/bcm2048/radio-bcm2048.c
+++ b/drivers/staging/media/bcm2048/radio-bcm2048.c
@@ -2538,7 +2538,7 @@ static int bcm2048_vidioc_s_hw_freq_seek(struct file *file, void *priv,
 	return err;
 }
 
-static struct v4l2_ioctl_ops bcm2048_ioctl_ops = {
+static const struct v4l2_ioctl_ops bcm2048_ioctl_ops = {
 	.vidioc_querycap	= bcm2048_vidioc_querycap,
 	.vidioc_g_input		= bcm2048_vidioc_g_input,
 	.vidioc_s_input		= bcm2048_vidioc_s_input,
diff --git a/drivers/staging/media/cxd2099/cxd2099.c b/drivers/staging/media/cxd2099/cxd2099.c
index fedeb3c3549e..c72c3f09f175 100644
--- a/drivers/staging/media/cxd2099/cxd2099.c
+++ b/drivers/staging/media/cxd2099/cxd2099.c
@@ -336,7 +336,8 @@ static int init(struct cxd *ci)
 			break;
 #endif
 		/* TOSTRT = 8, Mode B (gated clock), falling Edge,
-		 * Serial, POL=HIGH, MSB */
+		 * Serial, POL=HIGH, MSB
+		 */
 		status = write_reg(ci, 0x0A, 0xA7);
 		if (status < 0)
 			break;
diff --git a/drivers/staging/media/lirc/lirc_sasem.c b/drivers/staging/media/lirc/lirc_sasem.c
index 4fd810bc46de..b0c176e14b6b 100644
--- a/drivers/staging/media/lirc/lirc_sasem.c
+++ b/drivers/staging/media/lirc/lirc_sasem.c
@@ -103,7 +103,8 @@ struct sasem_context {
 
 	struct tx_t {
 		unsigned char data_buf[SASEM_DATA_BUF_SZ]; /* user data
-							    * buffer */
+							    * buffer
+							    */
 		struct completion finished;  /* wait for write to finish  */
 		atomic_t busy;		     /* write in progress */
 		int status;		     /* status of tx completion */
@@ -295,7 +296,8 @@ static int vfd_close(struct inode *inode, struct file *file)
 		if (!context->dev_present && !context->ir_isopen) {
 			/* Device disconnected before close and IR port is
 			 * not open. If IR port is open, context will be
-			 * deleted by ir_close. */
+			 * deleted by ir_close.
+			 */
 			mutex_unlock(&context->ctx_lock);
 			delete_context(context);
 			return retval;
@@ -396,7 +398,8 @@ static ssize_t vfd_write(struct file *file, const char __user *buf,
 
 	/* Nine 8 byte packets to be sent */
 	/* NOTE: "\x07\x01\0\0\0\0\0\0" or "\x0c\0\0\0\0\0\0\0"
-	 *       will clear the VFD */
+	 *       will clear the VFD
+	 */
 	for (i = 0; i < 9; i++) {
 		switch (i) {
 		case 0:
diff --git a/drivers/staging/media/lirc/lirc_zilog.c b/drivers/staging/media/lirc/lirc_zilog.c
index 3551aed589c0..34aac3e2eb87 100644
--- a/drivers/staging/media/lirc/lirc_zilog.c
+++ b/drivers/staging/media/lirc/lirc_zilog.c
@@ -1157,8 +1157,8 @@ static ssize_t write(struct file *filep, const char __user *buf, size_t n,
 
 		/* Send the code */
 		if (ret == 0) {
-			ret = send_code(tx, (unsigned)command >> 16,
-					    (unsigned)command & 0xFFFF);
+			ret = send_code(tx, (unsigned int)command >> 16,
+					    (unsigned int)command & 0xFFFF);
 			if (ret == -EPROTO) {
 				mutex_unlock(&ir->ir_lock);
 				mutex_unlock(&tx->client_lock);
diff --git a/drivers/staging/media/omap4iss/iss_csi2.c b/drivers/staging/media/omap4iss/iss_csi2.c
index aaca39d751a5..f71d5f2f179f 100644
--- a/drivers/staging/media/omap4iss/iss_csi2.c
+++ b/drivers/staging/media/omap4iss/iss_csi2.c
@@ -224,7 +224,7 @@ static u16 csi2_ctx_map_format(struct iss_csi2_device *csi2)
 		fmtidx = 3;
 		break;
 	default:
-		WARN(1, KERN_ERR "CSI2: pixel format %08x unsupported!\n",
+		WARN(1, "CSI2: pixel format %08x unsupported!\n",
 		     fmt->code);
 		return 0;
 	}
diff --git a/drivers/staging/most/aim-network/networking.c b/drivers/staging/most/aim-network/networking.c
index 4659a6450c04..ce1764cba5f0 100644
--- a/drivers/staging/most/aim-network/networking.c
+++ b/drivers/staging/most/aim-network/networking.c
@@ -67,10 +67,10 @@ struct net_dev_context {
 	struct most_interface *iface;
 	bool channels_opened;
 	bool is_mamac;
-	unsigned char link_stat;
 	struct net_device *dev;
 	struct net_dev_channel rx;
 	struct net_dev_channel tx;
+	struct completion mac_compl;
 	struct list_head list;
 };
 
@@ -181,6 +181,7 @@ static int most_nd_set_mac_address(struct net_device *dev, void *p)
 static int most_nd_open(struct net_device *dev)
 {
 	struct net_dev_context *nd = dev->ml_priv;
+	long ret;
 
 	netdev_info(dev, "open net device\n");
 
@@ -202,16 +203,30 @@ static int most_nd_open(struct net_device *dev)
 		return -EBUSY;
 	}
 
-	nd->channels_opened = true;
-
-	if (nd->is_mamac) {
-		nd->link_stat = 1;
-		netif_wake_queue(dev);
-	} else {
+	if (!is_valid_ether_addr(dev->dev_addr)) {
 		nd->iface->request_netinfo(nd->iface, nd->tx.ch_id);
+		ret = wait_for_completion_interruptible_timeout(
+			      &nd->mac_compl, msecs_to_jiffies(5000));
+		if (!ret) {
+			netdev_err(dev, "mac timeout\n");
+			ret = -EBUSY;
+			goto err;
+		}
+
+		if (ret < 0) {
+			netdev_warn(dev, "mac waiting interrupted\n");
+			goto err;
+		}
 	}
 
+	nd->channels_opened = true;
+	netif_wake_queue(dev);
 	return 0;
+
+err:
+	most_stop_channel(nd->iface, nd->tx.ch_id, &aim);
+	most_stop_channel(nd->iface, nd->rx.ch_id, &aim);
+	return ret;
 }
 
 static int most_nd_stop(struct net_device *dev)
@@ -277,7 +292,6 @@ static const struct net_device_ops most_nd_ops = {
 
 static void most_nd_setup(struct net_device *dev)
 {
-	netdev_info(dev, "setup net device\n");
 	ether_setup(dev);
 	dev->netdev_ops = &most_nd_ops;
 }
@@ -332,6 +346,7 @@ static int aim_probe_channel(struct most_interface *iface, int channel_idx,
 		if (!nd)
 			return -ENOMEM;
 
+		init_completion(&nd->mac_compl);
 		nd->iface = iface;
 
 		spin_lock_irqsave(&list_lock, flags);
@@ -548,8 +563,7 @@ void most_deliver_netinfo(struct most_interface *iface,
 {
 	struct net_dev_context *nd;
 	struct net_device *dev;
-
-	pr_info("Received netinfo from %s\n", iface->description);
+	const u8 *m = mac_addr;
 
 	nd = get_net_dev_context(iface);
 	if (!nd)
@@ -559,15 +573,16 @@ void most_deliver_netinfo(struct most_interface *iface,
 	if (!dev)
 		return;
 
-	if (mac_addr)
-		ether_addr_copy(dev->dev_addr, mac_addr);
-
-	if (nd->link_stat != link_stat) {
-		nd->link_stat = link_stat;
-		if (nd->link_stat)
-			netif_wake_queue(dev);
-		else
-			netif_stop_queue(dev);
+	if (m && is_valid_ether_addr(m)) {
+		if (!is_valid_ether_addr(dev->dev_addr)) {
+			netdev_info(dev, "set mac %02x-%02x-%02x-%02x-%02x-%02x\n",
+				    m[0], m[1], m[2], m[3], m[4], m[5]);
+			ether_addr_copy(dev->dev_addr, m);
+			complete(&nd->mac_compl);
+		} else if (!ether_addr_equal(dev->dev_addr, m)) {
+			netdev_warn(dev, "reject mac %02x-%02x-%02x-%02x-%02x-%02x\n",
+				    m[0], m[1], m[2], m[3], m[4], m[5]);
+		}
 	}
 }
 EXPORT_SYMBOL(most_deliver_netinfo);
diff --git a/drivers/staging/most/hdm-dim2/dim2_hdm.c b/drivers/staging/most/hdm-dim2/dim2_hdm.c
index 78b2c3dd9bb3..35aee9fbbf02 100644
--- a/drivers/staging/most/hdm-dim2/dim2_hdm.c
+++ b/drivers/staging/most/hdm-dim2/dim2_hdm.c
@@ -306,14 +306,11 @@ static int deliver_netinfo_thread(void *data)
 static void retrieve_netinfo(struct dim2_hdm *dev, struct mbo *mbo)
 {
 	u8 *data = mbo->virt_address;
-	u8 *mac = dev->mac_addrs;
 
 	pr_info("Node Address: 0x%03x\n", (u16)data[16] << 8 | data[17]);
 	dev->link_state = data[18];
 	pr_info("NIState: %d\n", dev->link_state);
-	memcpy(mac, data + 19, 6);
-	pr_info("MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n",
-		mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
+	memcpy(dev->mac_addrs, data + 19, 6);
 	dev->deliver_netinfo++;
 	wake_up_interruptible(&dev->netinfo_waitq);
 }
diff --git a/drivers/staging/most/hdm-usb/hdm_usb.c b/drivers/staging/most/hdm-usb/hdm_usb.c
index 26c9adb29308..d6db0bd65be0 100644
--- a/drivers/staging/most/hdm-usb/hdm_usb.c
+++ b/drivers/staging/most/hdm-usb/hdm_usb.c
@@ -97,9 +97,7 @@ struct clear_hold_work {
  * @cap: channel capabilities
  * @conf: channel configuration
  * @dci: direct communication interface of hardware
- * @hw_addr: MAC address of hardware
  * @ep_address: endpoint address table
- * @link_stat: link status of hardware
  * @description: device description
  * @suffix: suffix for channel name
  * @channel_lock: synchronize channel access
@@ -117,9 +115,7 @@ struct most_dev {
 	struct most_channel_capability *cap;
 	struct most_channel_config *conf;
 	struct most_dci_obj *dci;
-	u8 hw_addr[6];
 	u8 *ep_address;
-	u16 link_stat;
 	char description[MAX_STRING_LEN];
 	char suffix[MAX_NUM_ENDPOINTS][MAX_SUFFIX_LEN];
 	spinlock_t channel_lock[MAX_NUM_ENDPOINTS]; /* sync channel access */
@@ -186,28 +182,9 @@ static inline int drci_wr_reg(struct usb_device *dev, u16 reg, u16 data)
 			       5 * HZ);
 }
 
-/**
- * free_anchored_buffers - free device's anchored items
- * @mdev: the device
- * @channel: channel ID
- * @status: status of MBO termination
- */
-static void free_anchored_buffers(struct most_dev *mdev, unsigned int channel,
-				  enum mbo_status_flags status)
+static inline int start_sync_ep(struct usb_device *usb_dev, u16 ep)
 {
-	struct mbo *mbo;
-	struct urb *urb;
-
-	while ((urb = usb_get_from_anchor(&mdev->busy_urbs[channel]))) {
-		mbo = urb->context;
-		usb_kill_urb(urb);
-		if (mbo && mbo->complete) {
-			mbo->status = status;
-			mbo->processed_length = 0;
-			mbo->complete(mbo);
-		}
-		usb_free_urb(urb);
-	}
+	return drci_wr_reg(usb_dev, DRCI_REG_BASE + DRCI_COMMAND + ep * 16, 1);
 }
 
 /**
@@ -278,7 +255,7 @@ static int hdm_poison_channel(struct most_interface *iface, int channel)
 	cancel_work_sync(&mdev->clear_work[channel].ws);
 
 	mutex_lock(&mdev->io_mutex);
-	free_anchored_buffers(mdev, channel, MBO_E_CLOSE);
+	usb_kill_anchored_urbs(&mdev->busy_urbs[channel]);
 	if (mdev->padding_active[channel])
 		mdev->padding_active[channel] = false;
 
@@ -377,33 +354,27 @@ static void hdm_write_completion(struct urb *urb)
 	unsigned long flags;
 
 	spin_lock_irqsave(lock, flags);
-	if (urb->status == -ENOENT || urb->status == -ECONNRESET ||
-	    !mdev->is_channel_healthy[channel]) {
-		spin_unlock_irqrestore(lock, flags);
-		return;
-	}
 
-	if (unlikely(urb->status && urb->status != -ESHUTDOWN)) {
-		mbo->processed_length = 0;
+	mbo->processed_length = 0;
+	mbo->status = MBO_E_INVAL;
+	if (likely(mdev->is_channel_healthy[channel])) {
 		switch (urb->status) {
+		case 0:
+		case -ESHUTDOWN:
+			mbo->processed_length = urb->actual_length;
+			mbo->status = MBO_SUCCESS;
+			break;
 		case -EPIPE:
 			dev_warn(dev, "Broken OUT pipe detected\n");
 			mdev->is_channel_healthy[channel] = false;
-			spin_unlock_irqrestore(lock, flags);
 			mdev->clear_work[channel].pipe = urb->pipe;
 			schedule_work(&mdev->clear_work[channel].ws);
-			return;
+			break;
 		case -ENODEV:
 		case -EPROTO:
 			mbo->status = MBO_E_CLOSE;
 			break;
-		default:
-			mbo->status = MBO_E_INVAL;
-			break;
 		}
-	} else {
-		mbo->status = MBO_SUCCESS;
-		mbo->processed_length = urb->actual_length;
 	}
 
 	spin_unlock_irqrestore(lock, flags);
@@ -531,40 +502,35 @@ static void hdm_read_completion(struct urb *urb)
 	unsigned long flags;
 
 	spin_lock_irqsave(lock, flags);
-	if (urb->status == -ENOENT || urb->status == -ECONNRESET ||
-	    !mdev->is_channel_healthy[channel]) {
-		spin_unlock_irqrestore(lock, flags);
-		return;
-	}
 
-	if (unlikely(urb->status && urb->status != -ESHUTDOWN)) {
-		mbo->processed_length = 0;
+	mbo->processed_length = 0;
+	mbo->status = MBO_E_INVAL;
+	if (likely(mdev->is_channel_healthy[channel])) {
 		switch (urb->status) {
+		case 0:
+		case -ESHUTDOWN:
+			mbo->processed_length = urb->actual_length;
+			mbo->status = MBO_SUCCESS;
+			if (mdev->padding_active[channel] &&
+			    hdm_remove_padding(mdev, channel, mbo)) {
+				mbo->processed_length = 0;
+				mbo->status = MBO_E_INVAL;
+			}
+			break;
 		case -EPIPE:
 			dev_warn(dev, "Broken IN pipe detected\n");
 			mdev->is_channel_healthy[channel] = false;
-			spin_unlock_irqrestore(lock, flags);
 			mdev->clear_work[channel].pipe = urb->pipe;
 			schedule_work(&mdev->clear_work[channel].ws);
-			return;
+			break;
 		case -ENODEV:
 		case -EPROTO:
 			mbo->status = MBO_E_CLOSE;
 			break;
 		case -EOVERFLOW:
 			dev_warn(dev, "Babble on IN pipe detected\n");
-		default:
-			mbo->status = MBO_E_INVAL;
 			break;
 		}
-	} else {
-		mbo->processed_length = urb->actual_length;
-		mbo->status = MBO_SUCCESS;
-		if (mdev->padding_active[channel] &&
-		    hdm_remove_padding(mdev, channel, mbo)) {
-			mbo->processed_length = 0;
-			mbo->status = MBO_E_INVAL;
-		}
 	}
 
 	spin_unlock_irqrestore(lock, flags);
@@ -668,6 +634,15 @@ _error:
  * @iface: interface
  * @channel: channel ID
  * @conf: structure that holds the configuration information
+ *
+ * The attached network interface controller (NIC) supports a padding mode
+ * to avoid short packets on USB, hence increasing the performance due to a
+ * lower interrupt load. This mode is default for synchronous data and can
+ * be switched on for isochronous data. In case padding is active the
+ * driver needs to know the frame size of the payload in order to calculate
+ * the number of bytes it needs to pad when transmitting or to cut off when
+ * receiving data.
+ *
  */
 static int hdm_configure_channel(struct most_interface *iface, int channel,
 				 struct most_channel_config *conf)
@@ -701,6 +676,11 @@ static int hdm_configure_channel(struct most_interface *iface, int channel,
 	    !(conf->data_type == MOST_CH_ISOC &&
 	      conf->packets_per_xact != 0xFF)) {
 		mdev->padding_active[channel] = false;
+		/*
+		 * Since the NIC's padding mode is not going to be
+		 * used, we can skip the frame size calculations and
+		 * move directly on to exit.
+		 */
 		goto exit;
 	}
 
@@ -734,56 +714,12 @@ static int hdm_configure_channel(struct most_interface *iface, int channel,
 			  - conf->buffer_size;
 exit:
 	mdev->conf[channel] = *conf;
-	return 0;
-}
+	if (conf->data_type == MOST_CH_ASYNC) {
+		u16 ep = mdev->ep_address[channel];
 
-/**
- * hdm_update_netinfo - retrieve latest networking information
- * @mdev: device interface
- *
- * This triggers the USB vendor requests to read the hardware address and
- * the current link status of the attached device.
- */
-static int hdm_update_netinfo(struct most_dev *mdev)
-{
-	struct usb_device *usb_device = mdev->usb_device;
-	struct device *dev = &usb_device->dev;
-	u16 hi, mi, lo, link;
-
-	if (!is_valid_ether_addr(mdev->hw_addr)) {
-		if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_HI, &hi) < 0) {
-			dev_err(dev, "Vendor request \"hw_addr_hi\" failed\n");
-			return -EFAULT;
-		}
-
-		if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_MI, &mi) < 0) {
-			dev_err(dev, "Vendor request \"hw_addr_mid\" failed\n");
-			return -EFAULT;
-		}
-
-		if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_LO, &lo) < 0) {
-			dev_err(dev, "Vendor request \"hw_addr_low\" failed\n");
-			return -EFAULT;
-		}
-
-		mutex_lock(&mdev->io_mutex);
-		mdev->hw_addr[0] = hi >> 8;
-		mdev->hw_addr[1] = hi;
-		mdev->hw_addr[2] = mi >> 8;
-		mdev->hw_addr[3] = mi;
-		mdev->hw_addr[4] = lo >> 8;
-		mdev->hw_addr[5] = lo;
-		mutex_unlock(&mdev->io_mutex);
-	}
-
-	if (drci_rd_reg(usb_device, DRCI_REG_NI_STATE, &link) < 0) {
-		dev_err(dev, "Vendor request \"link status\" failed\n");
-		return -EFAULT;
+		if (start_sync_ep(mdev->usb_device, ep) < 0)
+			dev_warn(dev, "sync for ep%02x failed", ep);
 	}
-
-	mutex_lock(&mdev->io_mutex);
-	mdev->link_stat = link;
-	mutex_unlock(&mdev->io_mutex);
 	return 0;
 }
 
@@ -807,7 +743,7 @@ static void hdm_request_netinfo(struct most_interface *iface, int channel)
 }
 
 /**
- * link_stat_timer_handler - add work to link_stat work queue
+ * link_stat_timer_handler - schedule work obtaining mac address and link status
  * @data: pointer to USB device instance
  *
  * The handler runs in interrupt context. That's why we need to defer the
@@ -823,33 +759,47 @@ static void link_stat_timer_handler(unsigned long data)
 }
 
 /**
- * wq_netinfo - work queue function
+ * wq_netinfo - work queue function to deliver latest networking information
  * @wq_obj: object that holds data for our deferred work to do
  *
  * This retrieves the network interface status of the USB INIC
- * and compares it with the current status. If the status has
- * changed, it updates the status of the core.
  */
 static void wq_netinfo(struct work_struct *wq_obj)
 {
 	struct most_dev *mdev = to_mdev_from_work(wq_obj);
-	int i, prev_link_stat = mdev->link_stat;
-	u8 prev_hw_addr[6];
+	struct usb_device *usb_device = mdev->usb_device;
+	struct device *dev = &usb_device->dev;
+	u16 hi, mi, lo, link;
+	u8 hw_addr[6];
 
-	for (i = 0; i < 6; i++)
-		prev_hw_addr[i] = mdev->hw_addr[i];
+	if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_HI, &hi) < 0) {
+		dev_err(dev, "Vendor request 'hw_addr_hi' failed\n");
+		return;
+	}
+
+	if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_MI, &mi) < 0) {
+		dev_err(dev, "Vendor request 'hw_addr_mid' failed\n");
+		return;
+	}
 
-	if (hdm_update_netinfo(mdev) < 0)
+	if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_LO, &lo) < 0) {
+		dev_err(dev, "Vendor request 'hw_addr_low' failed\n");
+		return;
+	}
+
+	if (drci_rd_reg(usb_device, DRCI_REG_NI_STATE, &link) < 0) {
+		dev_err(dev, "Vendor request 'link status' failed\n");
 		return;
-	if (prev_link_stat != mdev->link_stat ||
-	    prev_hw_addr[0] != mdev->hw_addr[0] ||
-	    prev_hw_addr[1] != mdev->hw_addr[1] ||
-	    prev_hw_addr[2] != mdev->hw_addr[2] ||
-	    prev_hw_addr[3] != mdev->hw_addr[3] ||
-	    prev_hw_addr[4] != mdev->hw_addr[4] ||
-	    prev_hw_addr[5] != mdev->hw_addr[5])
-		most_deliver_netinfo(&mdev->iface, mdev->link_stat,
-				     &mdev->hw_addr[0]);
+	}
+
+	hw_addr[0] = hi >> 8;
+	hw_addr[1] = hi;
+	hw_addr[2] = mi >> 8;
+	hw_addr[3] = mi;
+	hw_addr[4] = lo >> 8;
+	hw_addr[5] = lo;
+
+	most_deliver_netinfo(&mdev->iface, link, hw_addr);
 }
 
 /**
@@ -867,7 +817,7 @@ static void wq_clear_halt(struct work_struct *wq_obj)
 
 	mutex_lock(&mdev->io_mutex);
 	most_stop_enqueue(&mdev->iface, channel);
-	free_anchored_buffers(mdev, channel, MBO_E_INVAL);
+	usb_kill_anchored_urbs(&mdev->busy_urbs[channel]);
 	if (usb_clear_halt(mdev->usb_device, pipe))
 		dev_warn(&mdev->usb_device->dev, "Failed to reset endpoint.\n");
 
@@ -1053,6 +1003,7 @@ static ssize_t store_value(struct most_dci_obj *dci_obj,
 	u16 val;
 	u16 reg_addr;
 	const char *name = attr->attr.name;
+	struct usb_device *usb_dev = dci_obj->usb_device;
 	int err = kstrtou16(buf, 16, &val);
 
 	if (err)
@@ -1063,18 +1014,15 @@ static ssize_t store_value(struct most_dci_obj *dci_obj,
 		return count;
 	}
 
-	if (!strcmp(name, "arb_value")) {
-		reg_addr = dci_obj->reg_addr;
-	} else if (!strcmp(name, "sync_ep")) {
-		u16 ep = val;
-
-		reg_addr = DRCI_REG_BASE + DRCI_COMMAND + ep * 16;
-		val = 1;
-	} else if (get_static_reg_addr(ro_regs, name, &reg_addr)) {
+	if (!strcmp(name, "arb_value"))
+		err = drci_wr_reg(usb_dev, dci_obj->reg_addr, val);
+	else if (!strcmp(name, "sync_ep"))
+		err = start_sync_ep(usb_dev, val);
+	else if (!get_static_reg_addr(ro_regs, name, &reg_addr))
+		err = drci_wr_reg(usb_dev, reg_addr, val);
+	else
 		return -EFAULT;
-	}
 
-	err = drci_wr_reg(dci_obj->usb_device, reg_addr, val);
 	if (err < 0)
 		return err;
 
@@ -1186,7 +1134,6 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id)
 	struct most_channel_capability *tmp_cap;
 	struct usb_endpoint_descriptor *ep_desc;
 	int ret = 0;
-	int err;
 
 	if (!mdev)
 		goto exit_ENOMEM;
@@ -1262,13 +1209,6 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id)
 		tmp_cap++;
 		init_usb_anchor(&mdev->busy_urbs[i]);
 		spin_lock_init(&mdev->channel_lock[i]);
-		err = drci_wr_reg(usb_dev,
-				  DRCI_REG_BASE + DRCI_COMMAND +
-				  ep_desc->bEndpointAddress * 16,
-				  1);
-		if (err < 0)
-			dev_warn(dev, "DCI Sync for EP %02x failed",
-				 ep_desc->bEndpointAddress);
 	}
 	dev_notice(dev, "claimed gadget: Vendor=%4.4x ProdID=%4.4x Bus=%02x Device=%02x\n",
 		   le16_to_cpu(usb_dev->descriptor.idVendor),
diff --git a/drivers/staging/most/mostcore/core.c b/drivers/staging/most/mostcore/core.c
index 329109c0024f..191404bc5906 100644
--- a/drivers/staging/most/mostcore/core.c
+++ b/drivers/staging/most/mostcore/core.c
@@ -342,7 +342,7 @@ static ssize_t show_channel_starving(struct most_c_obj *c,
 }
 
 #define create_show_channel_attribute(val) \
-	static MOST_CHNL_ATTR(val, S_IRUGO, show_##val, NULL)
+	static MOST_CHNL_ATTR(val, 0444, show_##val, NULL)
 
 create_show_channel_attribute(available_directions);
 create_show_channel_attribute(available_datatypes);
@@ -494,9 +494,7 @@ static ssize_t store_set_packets_per_xact(struct most_c_obj *c,
 }
 
 #define create_channel_attribute(value) \
-	static MOST_CHNL_ATTR(value, S_IRUGO | S_IWUSR, \
-			      show_##value, \
-			      store_##value)
+	static MOST_CHNL_ATTR(value, 0644, show_##value, store_##value)
 
 create_channel_attribute(set_buffer_size);
 create_channel_attribute(set_number_of_buffers);
@@ -690,7 +688,7 @@ static ssize_t show_interface(struct most_inst_obj *instance_obj,
 }
 
 #define create_inst_attribute(value) \
-	static MOST_INST_ATTR(value, S_IRUGO, show_##value, NULL)
+	static MOST_INST_ATTR(value, 0444, show_##value, NULL)
 
 create_inst_attribute(description);
 create_inst_attribute(interface);
@@ -763,8 +761,6 @@ struct most_aim_obj {
 	struct kobject kobj;
 	struct list_head list;
 	struct most_aim *driver;
-	char add_link[STRING_SIZE];
-	char remove_link[STRING_SIZE];
 };
 
 #define to_aim_obj(d) container_of(d, struct most_aim_obj, kobj)
@@ -851,7 +847,7 @@ static void most_aim_release(struct kobject *kobj)
 	kfree(aim_obj);
 }
 
-static ssize_t show_add_link(struct most_aim_obj *aim_obj,
+static ssize_t add_link_show(struct most_aim_obj *aim_obj,
 			     struct most_aim_attribute *attr,
 			     char *buf)
 {
@@ -885,16 +881,16 @@ static ssize_t show_add_link(struct most_aim_obj *aim_obj,
  *
  * Examples:
  *
- * Input: "mdev0:ch0@ep_81:my_channel\n" or
- *        "mdev0:ch0@ep_81:my_channel"
+ * Input: "mdev0:ch6:my_channel\n" or
+ *        "mdev0:ch6:my_channel"
  *
- * Output: *a -> "mdev0", *b -> "ch0@ep_81", *c -> "my_channel"
+ * Output: *a -> "mdev0", *b -> "ch6", *c -> "my_channel"
  *
- * Input: "mdev0:ch0@ep_81\n"
- * Output: *a -> "mdev0", *b -> "ch0@ep_81", *c -> ""
+ * Input: "mdev1:ep81\n"
+ * Output: *a -> "mdev1", *b -> "ep81", *c -> ""
  *
- * Input: "mdev0:ch0@ep_81"
- * Output: *a -> "mdev0", *b -> "ch0@ep_81", *c == NULL
+ * Input: "mdev1:ep81"
+ * Output: *a -> "mdev1", *b -> "ep81", *c == NULL
  */
 static int split_string(char *buf, char **a, char **b, char **c)
 {
@@ -962,13 +958,13 @@ most_c_obj *get_channel_by_name(char *mdev, char *mdev_ch)
  * Searches for a pair of device and channel and probes the AIM
  *
  * Example:
- * (1) echo -n -e "mdev0:ch0@ep_81:my_rxchannel\n" >add_link
- * (2) echo -n -e "mdev0:ch0@ep_81\n" >add_link
+ * (1) echo "mdev0:ch6:my_rxchannel" >add_link
+ * (2) echo "mdev1:ep81" >add_link
  *
  * (1) would create the device node /dev/my_rxchannel
- * (2) would create the device node /dev/mdev0-ch0@ep_81
+ * (2) would create the device node /dev/mdev1-ep81
  */
-static ssize_t store_add_link(struct most_aim_obj *aim_obj,
+static ssize_t add_link_store(struct most_aim_obj *aim_obj,
 			      struct most_aim_attribute *attr,
 			      const char *buf,
 			      size_t len)
@@ -984,7 +980,6 @@ static ssize_t store_add_link(struct most_aim_obj *aim_obj,
 	size_t max_len = min_t(size_t, len + 1, STRING_SIZE);
 
 	strlcpy(buffer, buf, max_len);
-	strlcpy(aim_obj->add_link, buf, max_len);
 
 	ret = split_string(buffer, &mdev, &mdev_ch, &mdev_devnod);
 	if (ret)
@@ -1019,14 +1014,7 @@ static ssize_t store_add_link(struct most_aim_obj *aim_obj,
 }
 
 static struct most_aim_attribute most_aim_attr_add_link =
-	__ATTR(add_link, S_IRUGO | S_IWUSR, show_add_link, store_add_link);
-
-static ssize_t show_remove_link(struct most_aim_obj *aim_obj,
-				struct most_aim_attribute *attr,
-				char *buf)
-{
-	return snprintf(buf, PAGE_SIZE, "%s\n", aim_obj->remove_link);
-}
+	__ATTR_RW(add_link);
 
 /**
  * store_remove_link - store function for remove_link attribute
@@ -1036,9 +1024,9 @@ static ssize_t show_remove_link(struct most_aim_obj *aim_obj,
  * @len: buffer length
  *
  * Example:
- * echo -n -e "mdev0:ch0@ep_81\n" >remove_link
+ * echo "mdev0:ep81" >remove_link
  */
-static ssize_t store_remove_link(struct most_aim_obj *aim_obj,
+static ssize_t remove_link_store(struct most_aim_obj *aim_obj,
 				 struct most_aim_attribute *attr,
 				 const char *buf,
 				 size_t len)
@@ -1051,7 +1039,6 @@ static ssize_t store_remove_link(struct most_aim_obj *aim_obj,
 	size_t max_len = min_t(size_t, len + 1, STRING_SIZE);
 
 	strlcpy(buffer, buf, max_len);
-	strlcpy(aim_obj->remove_link, buf, max_len);
 	ret = split_string(buffer, &mdev, &mdev_ch, NULL);
 	if (ret)
 		return ret;
@@ -1070,8 +1057,7 @@ static ssize_t store_remove_link(struct most_aim_obj *aim_obj,
 }
 
 static struct most_aim_attribute most_aim_attr_remove_link =
-	__ATTR(remove_link, S_IRUGO | S_IWUSR, show_remove_link,
-	       store_remove_link);
+	__ATTR_WO(remove_link);
 
 static struct attribute *most_aim_def_attrs[] = {
 	&most_aim_attr_add_link.attr,
@@ -1761,9 +1747,6 @@ struct kobject *most_register_interface(struct most_interface *iface)
 
 		if (!name_suffix)
 			snprintf(channel_name, STRING_SIZE, "ch%d", i);
-		else if (name_suffix[0] == '@')
-			snprintf(channel_name, STRING_SIZE, "ch%d%s", i,
-				 name_suffix);
 		else
 			snprintf(channel_name, STRING_SIZE, "%s", name_suffix);
 
diff --git a/drivers/staging/netlogic/xlr_net.c b/drivers/staging/netlogic/xlr_net.c
index 552a7dcbf50b..fb0928a4fb97 100644
--- a/drivers/staging/netlogic/xlr_net.c
+++ b/drivers/staging/netlogic/xlr_net.c
@@ -172,29 +172,31 @@ static struct phy_device *xlr_get_phydev(struct xlr_net_priv *priv)
 /*
  * Ethtool operation
  */
-static int xlr_get_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
+static int xlr_get_link_ksettings(struct net_device *ndev,
+				  struct ethtool_link_ksettings *ecmd)
 {
 	struct xlr_net_priv *priv = netdev_priv(ndev);
 	struct phy_device *phydev = xlr_get_phydev(priv);
 
 	if (!phydev)
 		return -ENODEV;
-	return phy_ethtool_gset(phydev, ecmd);
+	return phy_ethtool_ksettings_get(phydev, ecmd);
 }
 
-static int xlr_set_settings(struct net_device *ndev, struct ethtool_cmd *ecmd)
+static int xlr_set_link_ksettings(struct net_device *ndev,
+				  const struct ethtool_link_ksettings *ecmd)
 {
 	struct xlr_net_priv *priv = netdev_priv(ndev);
 	struct phy_device *phydev = xlr_get_phydev(priv);
 
 	if (!phydev)
 		return -ENODEV;
-	return phy_ethtool_sset(phydev, ecmd);
+	return phy_ethtool_ksettings_set(phydev, ecmd);
 }
 
 static const struct ethtool_ops xlr_ethtool_ops = {
-	.get_settings = xlr_get_settings,
-	.set_settings = xlr_set_settings,
+	.get_link_ksettings = xlr_get_link_ksettings,
+	.set_link_ksettings = xlr_set_link_ksettings,
 };
 
 /*
@@ -1005,10 +1007,8 @@ static int xlr_net_probe(struct platform_device *pdev)
 	 */
 	adapter = (struct xlr_adapter *)
 		devm_kzalloc(&pdev->dev, sizeof(*adapter), GFP_KERNEL);
-	if (!adapter) {
-		err = -ENOMEM;
-		return err;
-	}
+	if (!adapter)
+		return -ENOMEM;
 
 	/*
 	 * XLR and XLS have 1 and 2 NAE controller respectively
diff --git a/drivers/staging/rtl8188eu/Makefile b/drivers/staging/rtl8188eu/Makefile
index 29b9834870fd..27af86e05098 100644
--- a/drivers/staging/rtl8188eu/Makefile
+++ b/drivers/staging/rtl8188eu/Makefile
@@ -53,4 +53,4 @@ r8188eu-y :=				\
 
 obj-$(CONFIG_R8188EU)	:= r8188eu.o
 
-ccflags-y += -D__CHECK_ENDIAN__ -I$(srctree)/$(src)/include
+ccflags-y += -I$(srctree)/$(src)/include
diff --git a/drivers/staging/rtl8188eu/core/rtw_cmd.c b/drivers/staging/rtl8188eu/core/rtw_cmd.c
index f1f4788dbd86..36109cec8706 100644
--- a/drivers/staging/rtl8188eu/core/rtw_cmd.c
+++ b/drivers/staging/rtl8188eu/core/rtw_cmd.c
@@ -308,7 +308,7 @@ u8 rtw_sitesurvey_cmd(struct adapter  *padapter, struct ndis_802_11_ssid *ssid,
 		mod_timer(&pmlmepriv->scan_to_timer,
 			  jiffies + msecs_to_jiffies(SCANNING_TIMEOUT));
 
-		rtw_led_control(padapter, LED_CTL_SITE_SURVEY);
+		LedControl8188eu(padapter, LED_CTL_SITE_SURVEY);
 
 		pmlmepriv->scan_interval = SCAN_INTERVAL;/*  30*2 sec = 60sec */
 	} else {
@@ -335,7 +335,7 @@ u8 rtw_createbss_cmd(struct adapter  *padapter)
 	u8	res = _SUCCESS;
 
 
-	rtw_led_control(padapter, LED_CTL_START_TO_LINK);
+	LedControl8188eu(padapter, LED_CTL_START_TO_LINK);
 
 	if (pmlmepriv->assoc_ssid.SsidLength == 0)
 		RT_TRACE(_module_rtl871x_cmd_c_, _drv_info_, (" createbss for Any SSid:%s\n", pmlmepriv->assoc_ssid.Ssid));
@@ -379,7 +379,7 @@ u8 rtw_joinbss_cmd(struct adapter  *padapter, struct wlan_network *pnetwork)
 	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
 
 
-	rtw_led_control(padapter, LED_CTL_START_TO_LINK);
+	LedControl8188eu(padapter, LED_CTL_START_TO_LINK);
 
 	if (pmlmepriv->assoc_ssid.SsidLength == 0)
 		RT_TRACE(_module_rtl871x_cmd_c_, _drv_info_, ("+Join cmd: Any SSid\n"));
diff --git a/drivers/staging/rtl8188eu/core/rtw_led.c b/drivers/staging/rtl8188eu/core/rtw_led.c
index 14461cf34037..c1478cff5854 100644
--- a/drivers/staging/rtl8188eu/core/rtw_led.c
+++ b/drivers/staging/rtl8188eu/core/rtw_led.c
@@ -30,7 +30,7 @@ void BlinkTimerCallback(unsigned long data)
 	if ((padapter->bSurpriseRemoved) || (padapter->bDriverStopped))
 		return;
 
-	schedule_work(&(pLed->BlinkWorkItem));
+	schedule_work(&pLed->BlinkWorkItem);
 }
 
 /*  */
@@ -60,7 +60,6 @@ void ResetLedStatus(struct LED_871x *pLed)
 
 	pLed->bLedNoLinkBlinkInProgress = false;
 	pLed->bLedLinkBlinkInProgress = false;
-	pLed->bLedStartToLinkBlinkInProgress = false;
 	pLed->bLedScanBlinkInProgress = false;
 }
 
@@ -72,10 +71,10 @@ void InitLed871x(struct adapter *padapter, struct LED_871x *pLed)
 
 	ResetLedStatus(pLed);
 
-	setup_timer(&(pLed->BlinkTimer), BlinkTimerCallback,
+	setup_timer(&pLed->BlinkTimer, BlinkTimerCallback,
 		    (unsigned long)pLed);
 
-	INIT_WORK(&(pLed->BlinkWorkItem), BlinkWorkItemCallback);
+	INIT_WORK(&pLed->BlinkWorkItem, BlinkWorkItemCallback);
 }
 
 
@@ -85,8 +84,8 @@ void InitLed871x(struct adapter *padapter, struct LED_871x *pLed)
 /*  */
 void DeInitLed871x(struct LED_871x *pLed)
 {
-	cancel_work_sync(&(pLed->BlinkWorkItem));
-	del_timer_sync(&(pLed->BlinkTimer));
+	cancel_work_sync(&pLed->BlinkWorkItem);
+	del_timer_sync(&pLed->BlinkTimer);
 	ResetLedStatus(pLed);
 }
 
@@ -99,7 +98,7 @@ void DeInitLed871x(struct LED_871x *pLed)
 static void SwLedBlink1(struct LED_871x *pLed)
 {
 	struct adapter *padapter = pLed->padapter;
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	u8 bStopBlinking = false;
 
 	/*  Change LED according to BlinkingLedState specified. */
@@ -247,9 +246,9 @@ static void SwLedBlink1(struct LED_871x *pLed)
  /* ALPHA, added by chiyoko, 20090106 */
 static void SwLedControlMode1(struct adapter *padapter, enum LED_CTL_MODE LedAction)
 {
-	struct led_priv *ledpriv = &(padapter->ledpriv);
-	struct LED_871x *pLed = &(ledpriv->SwLed0);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct led_priv *ledpriv = &padapter->ledpriv;
+	struct LED_871x *pLed = &ledpriv->SwLed0;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
 	switch (LedAction) {
 	case LED_CTL_POWER_ON:
@@ -259,11 +258,11 @@ static void SwLedControlMode1(struct adapter *padapter, enum LED_CTL_MODE LedAct
 			if (pLed->CurrLedState == LED_BLINK_SCAN || IS_LED_WPS_BLINKING(pLed))
 				return;
 			if (pLed->bLedLinkBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				pLed->bLedLinkBlinkInProgress = false;
 			}
 			if (pLed->bLedBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				pLed->bLedBlinkInProgress = false;
 			}
 
@@ -282,11 +281,11 @@ static void SwLedControlMode1(struct adapter *padapter, enum LED_CTL_MODE LedAct
 			if (pLed->CurrLedState == LED_BLINK_SCAN || IS_LED_WPS_BLINKING(pLed))
 				return;
 			if (pLed->bLedNoLinkBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				pLed->bLedNoLinkBlinkInProgress = false;
 			}
 			if (pLed->bLedBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				pLed->bLedBlinkInProgress = false;
 			}
 			pLed->bLedLinkBlinkInProgress = true;
@@ -306,15 +305,15 @@ static void SwLedControlMode1(struct adapter *padapter, enum LED_CTL_MODE LedAct
 			if (IS_LED_WPS_BLINKING(pLed))
 				return;
 			if (pLed->bLedNoLinkBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				pLed->bLedNoLinkBlinkInProgress = false;
 			}
 			if (pLed->bLedLinkBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				 pLed->bLedLinkBlinkInProgress = false;
 			}
 			if (pLed->bLedBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				pLed->bLedBlinkInProgress = false;
 			}
 			pLed->bLedScanBlinkInProgress = true;
@@ -326,7 +325,7 @@ static void SwLedControlMode1(struct adapter *padapter, enum LED_CTL_MODE LedAct
 				pLed->BlinkingLedState = RTW_LED_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
-		 }
+		}
 		break;
 	case LED_CTL_TX:
 	case LED_CTL_RX:
@@ -334,11 +333,11 @@ static void SwLedControlMode1(struct adapter *padapter, enum LED_CTL_MODE LedAct
 			if (pLed->CurrLedState == LED_BLINK_SCAN || IS_LED_WPS_BLINKING(pLed))
 				return;
 			if (pLed->bLedNoLinkBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				pLed->bLedNoLinkBlinkInProgress = false;
 			}
 			if (pLed->bLedLinkBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				pLed->bLedLinkBlinkInProgress = false;
 			}
 			pLed->bLedBlinkInProgress = true;
@@ -354,21 +353,21 @@ static void SwLedControlMode1(struct adapter *padapter, enum LED_CTL_MODE LedAct
 		break;
 	case LED_CTL_START_WPS: /* wait until xinpin finish */
 	case LED_CTL_START_WPS_BOTTON:
-		 if (!pLed->bLedWPSBlinkInProgress) {
+		if (!pLed->bLedWPSBlinkInProgress) {
 			if (pLed->bLedNoLinkBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				pLed->bLedNoLinkBlinkInProgress = false;
 			}
 			if (pLed->bLedLinkBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				 pLed->bLedLinkBlinkInProgress = false;
 			}
 			if (pLed->bLedBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				pLed->bLedBlinkInProgress = false;
 			}
 			if (pLed->bLedScanBlinkInProgress) {
-				del_timer_sync(&(pLed->BlinkTimer));
+				del_timer_sync(&pLed->BlinkTimer);
 				pLed->bLedScanBlinkInProgress = false;
 			}
 			pLed->bLedWPSBlinkInProgress = true;
@@ -379,27 +378,27 @@ static void SwLedControlMode1(struct adapter *padapter, enum LED_CTL_MODE LedAct
 				pLed->BlinkingLedState = RTW_LED_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
-		 }
+		}
 		break;
 	case LED_CTL_STOP_WPS:
 		if (pLed->bLedNoLinkBlinkInProgress) {
-			del_timer_sync(&(pLed->BlinkTimer));
+			del_timer_sync(&pLed->BlinkTimer);
 			pLed->bLedNoLinkBlinkInProgress = false;
 		}
 		if (pLed->bLedLinkBlinkInProgress) {
-			del_timer_sync(&(pLed->BlinkTimer));
+			del_timer_sync(&pLed->BlinkTimer);
 			 pLed->bLedLinkBlinkInProgress = false;
 		}
 		if (pLed->bLedBlinkInProgress) {
-			del_timer_sync(&(pLed->BlinkTimer));
+			del_timer_sync(&pLed->BlinkTimer);
 			pLed->bLedBlinkInProgress = false;
 		}
 		if (pLed->bLedScanBlinkInProgress) {
-			del_timer_sync(&(pLed->BlinkTimer));
+			del_timer_sync(&pLed->BlinkTimer);
 			pLed->bLedScanBlinkInProgress = false;
 		}
 		if (pLed->bLedWPSBlinkInProgress)
-			del_timer_sync(&(pLed->BlinkTimer));
+			del_timer_sync(&pLed->BlinkTimer);
 		else
 			pLed->bLedWPSBlinkInProgress = true;
 		pLed->CurrLedState = LED_BLINK_WPS_STOP;
@@ -415,7 +414,7 @@ static void SwLedControlMode1(struct adapter *padapter, enum LED_CTL_MODE LedAct
 		break;
 	case LED_CTL_STOP_WPS_FAIL:
 		if (pLed->bLedWPSBlinkInProgress) {
-			del_timer_sync(&(pLed->BlinkTimer));
+			del_timer_sync(&pLed->BlinkTimer);
 			pLed->bLedWPSBlinkInProgress = false;
 		}
 		pLed->bLedNoLinkBlinkInProgress = true;
@@ -431,23 +430,23 @@ static void SwLedControlMode1(struct adapter *padapter, enum LED_CTL_MODE LedAct
 		pLed->CurrLedState = RTW_LED_OFF;
 		pLed->BlinkingLedState = RTW_LED_OFF;
 		if (pLed->bLedNoLinkBlinkInProgress) {
-			del_timer_sync(&(pLed->BlinkTimer));
+			del_timer_sync(&pLed->BlinkTimer);
 			pLed->bLedNoLinkBlinkInProgress = false;
 		}
 		if (pLed->bLedLinkBlinkInProgress) {
-			del_timer_sync(&(pLed->BlinkTimer));
+			del_timer_sync(&pLed->BlinkTimer);
 			pLed->bLedLinkBlinkInProgress = false;
 		}
 		if (pLed->bLedBlinkInProgress) {
-			del_timer_sync(&(pLed->BlinkTimer));
+			del_timer_sync(&pLed->BlinkTimer);
 			pLed->bLedBlinkInProgress = false;
 		}
 		if (pLed->bLedWPSBlinkInProgress) {
-			del_timer_sync(&(pLed->BlinkTimer));
+			del_timer_sync(&pLed->BlinkTimer);
 			pLed->bLedWPSBlinkInProgress = false;
 		}
 		if (pLed->bLedScanBlinkInProgress) {
-			del_timer_sync(&(pLed->BlinkTimer));
+			del_timer_sync(&pLed->BlinkTimer);
 			pLed->bLedScanBlinkInProgress = false;
 		}
 		SwLedOff(padapter, pLed);
@@ -475,15 +474,10 @@ void BlinkHandler(struct LED_871x *pLed)
 
 void LedControl8188eu(struct adapter *padapter, enum LED_CTL_MODE LedAction)
 {
-	struct led_priv *ledpriv = &(padapter->ledpriv);
-
 	if ((padapter->bSurpriseRemoved) || (padapter->bDriverStopped) ||
 	   (!padapter->hw_init_completed))
 		return;
 
-	if (!ledpriv->bRegUseLed)
-		return;
-
 	if ((padapter->pwrctrlpriv.rf_pwrstate != rf_on &&
 	     padapter->pwrctrlpriv.rfoff_reason > RF_CHANGE_BY_PS) &&
 	    (LedAction == LED_CTL_TX || LedAction == LED_CTL_RX ||
diff --git a/drivers/staging/rtl8188eu/core/rtw_mlme.c b/drivers/staging/rtl8188eu/core/rtw_mlme.c
index ee2dcd05010f..032f783b0d83 100644
--- a/drivers/staging/rtl8188eu/core/rtw_mlme.c
+++ b/drivers/staging/rtl8188eu/core/rtw_mlme.c
@@ -801,7 +801,7 @@ void rtw_indicate_connect(struct adapter *padapter)
 	if (!check_fwstate(&padapter->mlmepriv, _FW_LINKED)) {
 		set_fwstate(pmlmepriv, _FW_LINKED);
 
-		rtw_led_control(padapter, LED_CTL_LINK);
+		LedControl8188eu(padapter, LED_CTL_LINK);
 
 		rtw_os_indicate_connect(padapter);
 	}
@@ -833,7 +833,7 @@ void rtw_indicate_disconnect(struct adapter *padapter)
 		rtw_os_indicate_disconnect(padapter);
 
 		_clr_fwstate_(pmlmepriv, _FW_LINKED);
-		rtw_led_control(padapter, LED_CTL_NO_LINK);
+		LedControl8188eu(padapter, LED_CTL_NO_LINK);
 		rtw_clear_scan_deny(padapter);
 	}
 
diff --git a/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c b/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c
index fb13df586441..d9c114776cab 100644
--- a/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c
+++ b/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c
@@ -133,7 +133,9 @@ static struct rt_channel_plan_map	RTW_ChannelPlanMap[RT_CHANNEL_DOMAIN_MAX] = {
 	{0x03},	/* 0x41, RT_CHANNEL_DOMAIN_GLOBAL_DOAMIN_2G */
 };
 
-static struct rt_channel_plan_map RTW_CHANNEL_PLAN_MAP_REALTEK_DEFINE = {0x03}; /* use the combination for max channel numbers */
+static const struct rt_channel_plan_map RTW_CHANNEL_PLAN_MAP_REALTEK_DEFINE = {
+	0x03
+}; /* use the combination for max channel numbers */
 
 /*
  * Search the @param channel_num in given @param channel_set
@@ -667,10 +669,10 @@ static int issue_probereq(struct adapter *padapter, struct ndis_802_11_ssid *pss
 	get_rate_set(padapter, bssrate, &bssrate_len);
 
 	if (bssrate_len > 8) {
-		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_ , 8, bssrate, &(pattrib->pktlen));
-		pframe = rtw_set_ie(pframe, _EXT_SUPPORTEDRATES_IE_ , (bssrate_len - 8), (bssrate + 8), &(pattrib->pktlen));
+		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_, 8, bssrate, &(pattrib->pktlen));
+		pframe = rtw_set_ie(pframe, _EXT_SUPPORTEDRATES_IE_, (bssrate_len - 8), (bssrate + 8), &(pattrib->pktlen));
 	} else {
-		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_ , bssrate_len , bssrate, &(pattrib->pktlen));
+		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_, bssrate_len, bssrate, &(pattrib->pktlen));
 	}
 
 	/* add wps_ie for wps2.0 */
@@ -999,7 +1001,7 @@ static void issue_asocrsp(struct adapter *padapter, unsigned short status,
 	}
 
 	if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_REALTEK)
-		pframe = rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, 6 , REALTEK_96B_IE, &(pattrib->pktlen));
+		pframe = rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, 6, REALTEK_96B_IE, &(pattrib->pktlen));
 
 	/* add WPS IE ie for wps 2.0 */
 	if (pmlmepriv->wps_assoc_resp_ie && pmlmepriv->wps_assoc_resp_ie_len > 0) {
@@ -1120,10 +1122,10 @@ static void issue_assocreq(struct adapter *padapter)
 
 
 	if (bssrate_len > 8) {
-		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_ , 8, bssrate, &(pattrib->pktlen));
-		pframe = rtw_set_ie(pframe, _EXT_SUPPORTEDRATES_IE_ , (bssrate_len - 8), (bssrate + 8), &(pattrib->pktlen));
+		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_, 8, bssrate, &(pattrib->pktlen));
+		pframe = rtw_set_ie(pframe, _EXT_SUPPORTEDRATES_IE_, (bssrate_len - 8), (bssrate + 8), &(pattrib->pktlen));
 	} else {
-		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_ , bssrate_len , bssrate, &(pattrib->pktlen));
+		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_, bssrate_len, bssrate, &(pattrib->pktlen));
 	}
 
 	/* RSN */
@@ -1165,7 +1167,7 @@ static void issue_assocreq(struct adapter *padapter)
 				memcpy(&pmlmeinfo->HT_caps.mcs, MCS_rate_2R, 16);
 				break;
 			}
-			pframe = rtw_set_ie(pframe, _HT_CAPABILITY_IE_, ie_len , (u8 *)(&(pmlmeinfo->HT_caps)), &(pattrib->pktlen));
+			pframe = rtw_set_ie(pframe, _HT_CAPABILITY_IE_, ie_len, (u8 *)(&(pmlmeinfo->HT_caps)), &(pattrib->pktlen));
 		}
 	}
 
@@ -1194,7 +1196,7 @@ static void issue_assocreq(struct adapter *padapter)
 	}
 
 	if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_REALTEK)
-		pframe = rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, 6 , REALTEK_96B_IE, &(pattrib->pktlen));
+		pframe = rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, 6, REALTEK_96B_IE, &(pattrib->pktlen));
 
 	pattrib->last_txcmdsz = pattrib->pktlen;
 	dump_mgntframe(padapter, pmgntframe);
@@ -2644,7 +2646,7 @@ static unsigned int OnBeacon(struct adapter *padapter,
 				ret = rtw_check_bcn_info(padapter, pframe, len);
 				if (!ret) {
 						DBG_88E_LEVEL(_drv_info_, "ap has changed, disconnect now\n ");
-						receive_disconnect(padapter, pmlmeinfo->network.MacAddress , 65535);
+						receive_disconnect(padapter, pmlmeinfo->network.MacAddress, 65535);
 						return _SUCCESS;
 				}
 				/* update WMM, ERP in the beacon */
@@ -2802,7 +2804,7 @@ static unsigned int OnAuth(struct adapter *padapter,
 			/* checking for challenging txt... */
 			DBG_88E("checking for challenging txt...\n");
 
-			p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + 4 + _AUTH_IE_OFFSET_ , _CHLGETXT_IE_, (int *)&ie_len,
+			p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + 4 + _AUTH_IE_OFFSET_, _CHLGETXT_IE_, (int *)&ie_len,
 					len - WLAN_HDR_A3_LEN - _AUTH_IE_OFFSET_ - 4);
 
 			if ((p == NULL) || (ie_len <= 0)) {
@@ -3046,7 +3048,7 @@ static unsigned int OnAssocReq(struct adapter *padapter,
 		memcpy(supportRate, p+2, ie_len);
 		supportRateNum = ie_len;
 
-		p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + ie_offset, _EXT_SUPPORTEDRATES_IE_ , &ie_len,
+		p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + ie_offset, _EXT_SUPPORTEDRATES_IE_, &ie_len,
 				pkt_len - WLAN_HDR_A3_LEN - ie_offset);
 		if (p !=  NULL) {
 			if (supportRateNum <= sizeof(supportRate)) {
@@ -3146,7 +3148,7 @@ static unsigned int OnAssocReq(struct adapter *padapter,
 			if (pmlmepriv->wps_beacon_ie) {
 				u8 selected_registrar = 0;
 
-				rtw_get_wps_attr_content(pmlmepriv->wps_beacon_ie, pmlmepriv->wps_beacon_ie_len, WPS_ATTR_SELECTED_REGISTRAR , &selected_registrar, NULL);
+				rtw_get_wps_attr_content(pmlmepriv->wps_beacon_ie, pmlmepriv->wps_beacon_ie_len, WPS_ATTR_SELECTED_REGISTRAR, &selected_registrar, NULL);
 
 				if (!selected_registrar) {
 					DBG_88E("selected_registrar is false , or AP is not ready to do WPS\n");
@@ -3511,7 +3513,7 @@ static unsigned int OnDeAuth(struct adapter *padapter,
 		DBG_88E_LEVEL(_drv_always_, "sta recv deauth reason code(%d) sta:%pM\n",
 			      reason, GetAddr3Ptr(pframe));
 
-		receive_disconnect(padapter, GetAddr3Ptr(pframe) , reason);
+		receive_disconnect(padapter, GetAddr3Ptr(pframe), reason);
 	}
 	pmlmepriv->LinkDetectInfo.bBusyTraffic = false;
 	return _SUCCESS;
diff --git a/drivers/staging/rtl8188eu/core/rtw_pwrctrl.c b/drivers/staging/rtl8188eu/core/rtw_pwrctrl.c
index 0b70fe7d3b72..4032121a06f3 100644
--- a/drivers/staging/rtl8188eu/core/rtw_pwrctrl.c
+++ b/drivers/staging/rtl8188eu/core/rtw_pwrctrl.c
@@ -56,7 +56,7 @@ static int rtw_hw_suspend(struct adapter *padapter)
 		if (check_fwstate(pmlmepriv, _FW_LINKED)) {
 			_clr_fwstate_(pmlmepriv, _FW_LINKED);
 
-			rtw_led_control(padapter, LED_CTL_NO_LINK);
+			LedControl8188eu(padapter, LED_CTL_NO_LINK);
 
 			rtw_os_indicate_disconnect(padapter);
 
@@ -94,7 +94,7 @@ static int rtw_hw_resume(struct adapter *padapter)
 	pwrpriv->bips_processing = true;
 	rtw_reset_drv_sw(padapter);
 
-	if (pm_netdev_open(pnetdev, false) != 0) {
+	if (ips_netdrv_open((struct adapter *)rtw_netdev_priv(pnetdev)) != _SUCCESS) {
 		mutex_unlock(&pwrpriv->mutex_lock);
 		goto error_exit;
 	}
diff --git a/drivers/staging/rtl8188eu/core/rtw_recv.c b/drivers/staging/rtl8188eu/core/rtw_recv.c
index b87cbbbee054..3e6edb63d36b 100644
--- a/drivers/staging/rtl8188eu/core/rtw_recv.c
+++ b/drivers/staging/rtl8188eu/core/rtw_recv.c
@@ -66,16 +66,12 @@ int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *padapter)
 
 	precvpriv->adapter = padapter;
 
-	precvpriv->free_recvframe_cnt = NR_RECVFRAME;
-
 	precvpriv->pallocated_frame_buf = vzalloc(NR_RECVFRAME * sizeof(struct recv_frame) + RXFRAME_ALIGN_SZ);
 
 	if (!precvpriv->pallocated_frame_buf)
 		return _FAIL;
 
-	precvpriv->precv_frame_buf = PTR_ALIGN(precvpriv->pallocated_frame_buf, RXFRAME_ALIGN_SZ);
-
-	precvframe = (struct recv_frame *)precvpriv->precv_frame_buf;
+	precvframe = PTR_ALIGN(precvpriv->pallocated_frame_buf, RXFRAME_ALIGN_SZ);
 
 	for (i = 0; i < NR_RECVFRAME; i++) {
 		INIT_LIST_HEAD(&(precvframe->list));
@@ -83,15 +79,12 @@ int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *padapter)
 		list_add_tail(&(precvframe->list),
 				     &(precvpriv->free_recv_queue.queue));
 
-		rtw_os_recv_resource_alloc(precvframe);
-
+		precvframe->pkt = NULL;
 		precvframe->len = 0;
 
 		precvframe->adapter = padapter;
 		precvframe++;
 	}
-	precvpriv->rx_pending_cnt = 1;
-
 	res = rtw_hal_init_recv_priv(padapter);
 
 	setup_timer(&precvpriv->signal_stat_timer,
@@ -120,20 +113,11 @@ void _rtw_free_recv_priv(struct recv_priv *precvpriv)
 struct recv_frame *_rtw_alloc_recvframe(struct __queue *pfree_recv_queue)
 {
 	struct recv_frame *hdr;
-	struct adapter *padapter;
-	struct recv_priv *precvpriv;
 
 	hdr = list_first_entry_or_null(&pfree_recv_queue->queue,
 				       struct recv_frame, list);
-	if (hdr) {
+	if (hdr)
 		list_del_init(&hdr->list);
-		padapter = hdr->adapter;
-		if (padapter) {
-			precvpriv = &padapter->recvpriv;
-			if (pfree_recv_queue == &precvpriv->free_recv_queue)
-				precvpriv->free_recvframe_cnt--;
-		}
-	}
 
 	return hdr;
 }
@@ -154,13 +138,8 @@ struct recv_frame *rtw_alloc_recvframe(struct __queue *pfree_recv_queue)
 int rtw_free_recvframe(struct recv_frame *precvframe,
 		       struct __queue *pfree_recv_queue)
 {
-	struct adapter *padapter;
-	struct recv_priv *precvpriv;
-
 	if (!precvframe)
 		return _FAIL;
-	padapter = precvframe->adapter;
-	precvpriv = &padapter->recvpriv;
 	if (precvframe->pkt) {
 		dev_kfree_skb_any(precvframe->pkt);/* free skb by driver */
 		precvframe->pkt = NULL;
@@ -174,29 +153,16 @@ int rtw_free_recvframe(struct recv_frame *precvframe,
 
 	list_add_tail(&(precvframe->list), get_list_head(pfree_recv_queue));
 
-	if (padapter != NULL) {
-		if (pfree_recv_queue == &precvpriv->free_recv_queue)
-				precvpriv->free_recvframe_cnt++;
-	}
-
-      spin_unlock_bh(&pfree_recv_queue->lock);
+	spin_unlock_bh(&pfree_recv_queue->lock);
 
 	return _SUCCESS;
 }
 
 int _rtw_enqueue_recvframe(struct recv_frame *precvframe, struct __queue *queue)
 {
-	struct adapter *padapter = precvframe->adapter;
-	struct recv_priv *precvpriv = &padapter->recvpriv;
-
 	list_del_init(&(precvframe->list));
 	list_add_tail(&(precvframe->list), get_list_head(queue));
 
-	if (padapter != NULL) {
-		if (queue == &precvpriv->free_recv_queue)
-			precvpriv->free_recvframe_cnt++;
-	}
-
 	return _SUCCESS;
 }
 
@@ -1294,7 +1260,7 @@ static int validate_recv_frame(struct adapter *adapter,
 		retval = _FAIL; /*  only data frame return _SUCCESS */
 		break;
 	case WIFI_DATA_TYPE: /* data */
-		rtw_led_control(adapter, LED_CTL_RX);
+		LedControl8188eu(adapter, LED_CTL_RX);
 		pattrib->qos = (subtype & BIT(7)) ? 1 : 0;
 		retval = validate_recv_data_frame(adapter, precv_frame);
 		if (retval == _FAIL) {
@@ -1989,7 +1955,7 @@ static int recv_func_posthandle(struct adapter *padapter,
 	struct __queue *pfree_recv_queue = &padapter->recvpriv.free_recv_queue;
 
 	/*  DATA FRAME */
-	rtw_led_control(padapter, LED_CTL_RX);
+	LedControl8188eu(padapter, LED_CTL_RX);
 
 	prframe = decryptor(padapter, prframe);
 	if (prframe == NULL) {
diff --git a/drivers/staging/rtl8188eu/core/rtw_sta_mgt.c b/drivers/staging/rtl8188eu/core/rtw_sta_mgt.c
index a71e25294add..941d1a069d20 100644
--- a/drivers/staging/rtl8188eu/core/rtw_sta_mgt.c
+++ b/drivers/staging/rtl8188eu/core/rtw_sta_mgt.c
@@ -310,7 +310,6 @@ u32	rtw_free_stainfo(struct adapter *padapter, struct sta_info *psta)
 	/* for A-MPDU Rx reordering buffer control, cancel reordering_ctrl_timer */
 	for (i = 0; i < 16; i++) {
 		struct list_head *phead, *plist;
-		struct recv_frame *prhdr;
 		struct recv_frame *prframe;
 		struct __queue *ppending_recvframe_queue;
 		struct __queue *pfree_recv_queue = &padapter->recvpriv.free_recv_queue;
@@ -327,8 +326,7 @@ u32	rtw_free_stainfo(struct adapter *padapter, struct sta_info *psta)
 		plist = phead->next;
 
 		while (!list_empty(phead)) {
-			prhdr = container_of(plist, struct recv_frame, list);
-			prframe = (struct recv_frame *)prhdr;
+			prframe = container_of(plist, struct recv_frame, list);
 
 			plist = plist->next;
 
diff --git a/drivers/staging/rtl8188eu/core/rtw_xmit.c b/drivers/staging/rtl8188eu/core/rtw_xmit.c
index 0f8b8e0bffdf..b60b126b860e 100644
--- a/drivers/staging/rtl8188eu/core/rtw_xmit.c
+++ b/drivers/staging/rtl8188eu/core/rtw_xmit.c
@@ -220,7 +220,6 @@ void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv)
 	struct adapter *padapter = pxmitpriv->adapter;
 	struct xmit_frame *pxmitframe = (struct xmit_frame *)pxmitpriv->pxmit_frame_buf;
 	struct xmit_buf *pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmitbuf;
-	u32 max_xmit_extbuf_size = MAX_XMIT_EXTBUF_SZ;
 	u32 num_xmit_extbuf = NR_XMIT_EXTBUFF;
 
 	if (pxmitpriv->pxmit_frame_buf == NULL)
@@ -233,7 +232,7 @@ void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv)
 	}
 
 	for (i = 0; i < NR_XMITBUFF; i++) {
-		rtw_os_xmit_resource_free(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ));
+		rtw_os_xmit_resource_free(pxmitbuf);
 		pxmitbuf++;
 	}
 
@@ -243,7 +242,7 @@ void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv)
 	/*  free xmit extension buff */
 	pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmit_extbuf;
 	for (i = 0; i < num_xmit_extbuf; i++) {
-		rtw_os_xmit_resource_free(padapter, pxmitbuf, (max_xmit_extbuf_size + XMITBUF_ALIGN_SZ));
+		rtw_os_xmit_resource_free(pxmitbuf);
 		pxmitbuf++;
 	}
 
@@ -1064,7 +1063,7 @@ s32 rtw_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct
 
 		frg_inx++;
 
-		if (bmcst || rtw_endofpktfile(&pktfile)) {
+		if (bmcst || pktfile.pkt_len == 0) {
 			pattrib->nr_frags = frg_inx;
 
 			pattrib->last_txcmdsz = pattrib->hdrlen + pattrib->iv_len + ((pattrib->nr_frags == 1) ? llc_sz : 0) +
@@ -1677,7 +1676,7 @@ s32 rtw_xmit(struct adapter *padapter, struct sk_buff **ppkt)
 	}
 	pxmitframe->pkt = *ppkt;
 
-	rtw_led_control(padapter, LED_CTL_TX);
+	LedControl8188eu(padapter, LED_CTL_TX);
 
 	pxmitframe->attrib.qsel = pxmitframe->attrib.priority;
 
diff --git a/drivers/staging/rtl8188eu/hal/odm.c b/drivers/staging/rtl8188eu/hal/odm.c
index d983a8029f4c..16476e735011 100644
--- a/drivers/staging/rtl8188eu/hal/odm.c
+++ b/drivers/staging/rtl8188eu/hal/odm.c
@@ -991,7 +991,6 @@ void odm_TXPowerTrackingThermalMeterInit(struct odm_dm_struct *pDM_Odm)
 {
 	pDM_Odm->RFCalibrateInfo.bTXPowerTracking = true;
 	pDM_Odm->RFCalibrateInfo.TXPowercount = 0;
-	pDM_Odm->RFCalibrateInfo.bTXPowerTrackingInit = false;
 	if (*(pDM_Odm->mp_mode) != 1)
 		pDM_Odm->RFCalibrateInfo.TxPowerTrackControl = true;
 	MSG_88E("pDM_Odm TxPowerTrackControl = %d\n", pDM_Odm->RFCalibrateInfo.TxPowerTrackControl);
diff --git a/drivers/staging/rtl8188eu/hal/phy.c b/drivers/staging/rtl8188eu/hal/phy.c
index 5192ef70bcfc..35c91e06cc47 100644
--- a/drivers/staging/rtl8188eu/hal/phy.c
+++ b/drivers/staging/rtl8188eu/hal/phy.c
@@ -40,12 +40,11 @@ static u32 cal_bit_shift(u32 bitmask)
 
 u32 phy_query_bb_reg(struct adapter *adapt, u32 regaddr, u32 bitmask)
 {
-	u32 return_value = 0, original_value, bit_shift;
+	u32 original_value, bit_shift;
 
 	original_value = usb_read32(adapt, regaddr);
 	bit_shift = cal_bit_shift(bitmask);
-	return_value = (original_value & bitmask) >> bit_shift;
-	return return_value;
+	return (original_value & bitmask) >> bit_shift;
 }
 
 void phy_set_bb_reg(struct adapter *adapt, u32 regaddr, u32 bitmask, u32 data)
@@ -119,12 +118,11 @@ static void rf_serial_write(struct adapter *adapt,
 u32 rtw_hal_read_rfreg(struct adapter *adapt, enum rf_radio_path rf_path,
 		     u32 reg_addr, u32 bit_mask)
 {
-	u32 original_value, readback_value, bit_shift;
+	u32 original_value, bit_shift;
 
 	original_value = rf_serial_read(adapt, rf_path, reg_addr);
 	bit_shift =  cal_bit_shift(bit_mask);
-	readback_value = (original_value & bit_mask) >> bit_shift;
-	return readback_value;
+	return (original_value & bit_mask) >> bit_shift;
 }
 
 void phy_set_rf_reg(struct adapter *adapt, enum rf_radio_path rf_path,
@@ -210,13 +208,6 @@ static void phy_set_bw_mode_callback(struct adapter *adapt)
 	u8 reg_bw_opmode;
 	u8 reg_prsr_rsc;
 
-	if (hal_data->rf_chip == RF_PSEUDO_11N)
-		return;
-
-	/*  There is no 40MHz mode in RF_8225. */
-	if (hal_data->rf_chip == RF_8225)
-		return;
-
 	if (adapt->bDriverStopped)
 		return;
 
@@ -265,8 +256,7 @@ static void phy_set_bw_mode_callback(struct adapter *adapt)
 	}
 
 	/* Set RF related register */
-	if (hal_data->rf_chip == RF_6052)
-		rtl88eu_phy_rf6052_set_bandwidth(adapt, hal_data->CurrentChannelBW);
+	rtl88eu_phy_rf6052_set_bandwidth(adapt, hal_data->CurrentChannelBW);
 }
 
 void rtw_hal_set_bwmode(struct adapter *adapt, enum ht_channel_width bandwidth,
@@ -286,7 +276,6 @@ void rtw_hal_set_bwmode(struct adapter *adapt, enum ht_channel_width bandwidth,
 
 static void phy_sw_chnl_callback(struct adapter *adapt, u8 channel)
 {
-	u8 rf_path;
 	u32 param1, param2;
 	struct hal_data_8188e *hal_data = adapt->HalData;
 
@@ -294,12 +283,10 @@ static void phy_sw_chnl_callback(struct adapter *adapt, u8 channel)
 
 	param1 = RF_CHNLBW;
 	param2 = channel;
-	for (rf_path = 0; rf_path < hal_data->NumTotalRFPath; rf_path++) {
-		hal_data->RfRegChnlVal[rf_path] = (hal_data->RfRegChnlVal[rf_path] &
-						  0xfffffc00) | param2;
-		phy_set_rf_reg(adapt, (enum rf_radio_path)rf_path, param1,
-			       bRFRegOffsetMask, hal_data->RfRegChnlVal[rf_path]);
-	}
+	hal_data->RfRegChnlVal[0] = (hal_data->RfRegChnlVal[0] &
+					  0xfffffc00) | param2;
+	phy_set_rf_reg(adapt, 0, param1,
+		       bRFRegOffsetMask, hal_data->RfRegChnlVal[0]);
 }
 
 void rtw_hal_set_chan(struct adapter *adapt, u8 channel)
@@ -307,9 +294,6 @@ void rtw_hal_set_chan(struct adapter *adapt, u8 channel)
 	struct hal_data_8188e *hal_data = adapt->HalData;
 	u8 tmpchannel = hal_data->CurrentChannel;
 
-	if (hal_data->rf_chip == RF_PSEUDO_11N)
-		return;
-
 	if (channel == 0)
 		channel = 1;
 
@@ -407,9 +391,8 @@ void rtl88eu_dm_txpower_tracking_callback_thermalmeter(struct adapter *adapt)
 	s8 ofdm_index[2], cck_index = 0;
 	s8 ofdm_index_old[2] = {0, 0}, cck_index_old = 0;
 	u32 i = 0, j = 0;
-	bool is2t = false;
 
-	u8 ofdm_min_index = 6, rf; /* OFDM BB Swing should be less than +3.0dB */
+	u8 ofdm_min_index = 6; /* OFDM BB Swing should be less than +3.0dB */
 	s8 ofdm_index_mapping[2][index_mapping_NUM_88E] = {
 		/* 2.4G, decrease power */
 		{0, 0, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11},
@@ -427,18 +410,12 @@ void rtl88eu_dm_txpower_tracking_callback_thermalmeter(struct adapter *adapt)
 	dm_txpwr_track_setpwr(dm_odm);
 
 	dm_odm->RFCalibrateInfo.TXPowerTrackingCallbackCnt++;
-	dm_odm->RFCalibrateInfo.bTXPowerTrackingInit = true;
 
 	dm_odm->RFCalibrateInfo.RegA24 = 0x090e1317;
 
 	thermal_val = (u8)rtw_hal_read_rfreg(adapt, RF_PATH_A,
 					   RF_T_METER_88E, 0xfc00);
 
-	if (is2t)
-		rf = 2;
-	else
-		rf = 1;
-
 	if (thermal_val) {
 		/* Query OFDM path A default setting */
 		ele_d = phy_query_bb_reg(adapt, rOFDM0_XATxIQImbalance, bMaskDWord)&bMaskOFDM_D;
@@ -450,17 +427,6 @@ void rtl88eu_dm_txpower_tracking_callback_thermalmeter(struct adapter *adapt)
 			}
 		}
 
-		/* Query OFDM path B default setting */
-		if (is2t) {
-			ele_d = phy_query_bb_reg(adapt, rOFDM0_XBTxIQImbalance, bMaskDWord)&bMaskOFDM_D;
-			for (i = 0; i < OFDM_TABLE_SIZE_92D; i++) {
-				if (ele_d == (OFDMSwingTable[i]&bMaskOFDM_D)) {
-					ofdm_index_old[1] = (u8)i;
-					break;
-				}
-			}
-		}
-
 		/* Query CCK default setting From 0xa24 */
 		temp_cck = dm_odm->RFCalibrateInfo.RegA24;
 
@@ -479,8 +445,7 @@ void rtl88eu_dm_txpower_tracking_callback_thermalmeter(struct adapter *adapt)
 			dm_odm->RFCalibrateInfo.ThermalValue_LCK = thermal_val;
 			dm_odm->RFCalibrateInfo.ThermalValue_IQK = thermal_val;
 
-			for (i = 0; i < rf; i++)
-				dm_odm->RFCalibrateInfo.OFDM_index[i] = ofdm_index_old[i];
+			dm_odm->RFCalibrateInfo.OFDM_index[0] = ofdm_index_old[0];
 			dm_odm->RFCalibrateInfo.CCK_index = cck_index_old;
 		}
 
@@ -539,13 +504,11 @@ void rtl88eu_dm_txpower_tracking_callback_thermalmeter(struct adapter *adapt)
 				offset = index_mapping_NUM_88E-1;
 
 			/* Updating ofdm_index values with new OFDM / CCK offset */
-			for (i = 0; i < rf; i++) {
-				ofdm_index[i] = dm_odm->RFCalibrateInfo.OFDM_index[i] + ofdm_index_mapping[j][offset];
-				if (ofdm_index[i] > OFDM_TABLE_SIZE_92D-1)
-					ofdm_index[i] = OFDM_TABLE_SIZE_92D-1;
-				else if (ofdm_index[i] < ofdm_min_index)
-					ofdm_index[i] = ofdm_min_index;
-			}
+			ofdm_index[0] = dm_odm->RFCalibrateInfo.OFDM_index[0] + ofdm_index_mapping[j][offset];
+			if (ofdm_index[0] > OFDM_TABLE_SIZE_92D-1)
+				ofdm_index[0] = OFDM_TABLE_SIZE_92D-1;
+			else if (ofdm_index[0] < ofdm_min_index)
+				ofdm_index[0] = ofdm_min_index;
 
 			cck_index = dm_odm->RFCalibrateInfo.CCK_index + ofdm_index_mapping[j][offset];
 			if (cck_index > CCK_TABLE_SIZE-1)
diff --git a/drivers/staging/rtl8188eu/hal/rf.c b/drivers/staging/rtl8188eu/hal/rf.c
index 2f3edf0f850a..8f8c9de6a9bc 100644
--- a/drivers/staging/rtl8188eu/hal/rf.c
+++ b/drivers/staging/rtl8188eu/hal/rf.c
@@ -61,8 +61,6 @@ void rtl88eu_phy_rf6052_set_cck_txpower(struct adapter *adapt, u8 *powerlevel)
 				      (powerlevel[idx1]<<8) |
 				      (powerlevel[idx1]<<16) |
 				      (powerlevel[idx1]<<24);
-			if (tx_agc[idx1] > 0x20 && hal_data->ExternalPA)
-				tx_agc[idx1] = 0x20;
 		}
 	} else {
 		if (pdmpriv->DynamicTxHighPowerLvl == TxHighPwrLevel_Level1) {
@@ -139,17 +137,15 @@ static void getpowerbase88e(struct adapter *adapt, u8 *pwr_level_ofdm,
 			     (powerbase0<<8) | powerbase0;
 		*(ofdmbase+i) = powerbase0;
 	}
-	for (i = 0; i < adapt->HalData->NumTotalRFPath; i++) {
-		/* Check HT20 to HT40 diff */
-		if (adapt->HalData->CurrentChannelBW == HT_CHANNEL_WIDTH_20)
-			powerlevel[i] = pwr_level_bw20[i];
-		else
-			powerlevel[i] = pwr_level_bw40[i];
-		powerbase1 = powerlevel[i];
-		powerbase1 = (powerbase1<<24) | (powerbase1<<16) |
-			     (powerbase1<<8) | powerbase1;
-		*(mcs_base+i) = powerbase1;
-	}
+	/* Check HT20 to HT40 diff */
+	if (adapt->HalData->CurrentChannelBW == HT_CHANNEL_WIDTH_20)
+		powerlevel[0] = pwr_level_bw20[0];
+	else
+		powerlevel[0] = pwr_level_bw40[0];
+	powerbase1 = powerlevel[0];
+	powerbase1 = (powerbase1<<24) | (powerbase1<<16) |
+		     (powerbase1<<8) | powerbase1;
+	*mcs_base = powerbase1;
 }
 static void get_rx_power_val_by_reg(struct adapter *adapt, u8 channel,
 				    u8 index, u32 *powerbase0, u32 *powerbase1,
diff --git a/drivers/staging/rtl8188eu/hal/rf_cfg.c b/drivers/staging/rtl8188eu/hal/rf_cfg.c
index dde64417e66a..9712d7b74345 100644
--- a/drivers/staging/rtl8188eu/hal/rf_cfg.c
+++ b/drivers/staging/rtl8188eu/hal/rf_cfg.c
@@ -230,79 +230,33 @@ static bool rf6052_conf_para(struct adapter *adapt)
 {
 	struct hal_data_8188e *hal_data = adapt->HalData;
 	u32 u4val = 0;
-	u8 rfpath;
 	bool rtstatus = true;
 	struct bb_reg_def *pphyreg;
 
-	for (rfpath = 0; rfpath < hal_data->NumTotalRFPath; rfpath++) {
-		pphyreg = &hal_data->PHYRegDef[rfpath];
+	pphyreg = &hal_data->PHYRegDef[RF90_PATH_A];
+	u4val = phy_query_bb_reg(adapt, pphyreg->rfintfs, BRFSI_RFENV);
 
-		switch (rfpath) {
-		case RF90_PATH_A:
-		case RF90_PATH_C:
-			u4val = phy_query_bb_reg(adapt, pphyreg->rfintfs,
-						 BRFSI_RFENV);
-			break;
-		case RF90_PATH_B:
-		case RF90_PATH_D:
-			u4val = phy_query_bb_reg(adapt, pphyreg->rfintfs,
-						 BRFSI_RFENV << 16);
-			break;
-		}
+	phy_set_bb_reg(adapt, pphyreg->rfintfe, BRFSI_RFENV << 16, 0x1);
+	udelay(1);
 
-		phy_set_bb_reg(adapt, pphyreg->rfintfe, BRFSI_RFENV << 16, 0x1);
-		udelay(1);
+	phy_set_bb_reg(adapt, pphyreg->rfintfo, BRFSI_RFENV, 0x1);
+	udelay(1);
 
-		phy_set_bb_reg(adapt, pphyreg->rfintfo, BRFSI_RFENV, 0x1);
-		udelay(1);
+	phy_set_bb_reg(adapt, pphyreg->rfHSSIPara2, B3WIREADDREAALENGTH, 0x0);
+	udelay(1);
 
-		phy_set_bb_reg(adapt, pphyreg->rfHSSIPara2,
-			      B3WIREADDREAALENGTH, 0x0);
-		udelay(1);
+	phy_set_bb_reg(adapt, pphyreg->rfHSSIPara2, B3WIREDATALENGTH, 0x0);
+	udelay(1);
 
-		phy_set_bb_reg(adapt, pphyreg->rfHSSIPara2,
-			       B3WIREDATALENGTH, 0x0);
-		udelay(1);
+	rtstatus = rtl88e_phy_config_rf_with_headerfile(adapt);
 
-		switch (rfpath) {
-		case RF90_PATH_A:
-			rtstatus = rtl88e_phy_config_rf_with_headerfile(adapt);
-			break;
-		case RF90_PATH_B:
-			rtstatus = rtl88e_phy_config_rf_with_headerfile(adapt);
-			break;
-		case RF90_PATH_C:
-			break;
-		case RF90_PATH_D:
-			break;
-		}
-
-		switch (rfpath) {
-		case RF90_PATH_A:
-		case RF90_PATH_C:
-			phy_set_bb_reg(adapt, pphyreg->rfintfs,
-				       BRFSI_RFENV, u4val);
-			break;
-		case RF90_PATH_B:
-		case RF90_PATH_D:
-			phy_set_bb_reg(adapt, pphyreg->rfintfs,
-				       BRFSI_RFENV << 16, u4val);
-			break;
-		}
-
-		if (!rtstatus)
-			return false;
-	}
+	phy_set_bb_reg(adapt, pphyreg->rfintfs, BRFSI_RFENV, u4val);
 
 	return rtstatus;
 }
 
 static bool rtl88e_phy_rf6052_config(struct adapter *adapt)
 {
-	struct hal_data_8188e *hal_data = adapt->HalData;
-
-	hal_data->NumTotalRFPath = 1;
-
 	return rf6052_conf_para(adapt);
 }
 
diff --git a/drivers/staging/rtl8188eu/hal/rtl8188e_hal_init.c b/drivers/staging/rtl8188eu/hal/rtl8188e_hal_init.c
index 385bc2f56f2f..0ce7db723a5d 100644
--- a/drivers/staging/rtl8188eu/hal/rtl8188e_hal_init.c
+++ b/drivers/staging/rtl8188eu/hal/rtl8188e_hal_init.c
@@ -135,7 +135,6 @@ void rtw_hal_read_chip_version(struct adapter *padapter)
 	dump_chip_info(ChipVersion);
 
 	pHalData->VersionID = ChipVersion;
-	pHalData->NumTotalRFPath = 1;
 }
 
 void rtw_hal_set_odm_var(struct adapter *Adapter, enum hal_odm_variable eVariable, void *pValue1, bool bSet)
@@ -470,7 +469,7 @@ void Hal_ReadTxPowerInfo88E(struct adapter *padapter, u8 *PROMContent, bool Auto
 {
 	struct hal_data_8188e *pHalData = padapter->HalData;
 	struct txpowerinfo24g pwrInfo24G;
-	u8 rfPath, ch, group;
+	u8 ch, group;
 	u8 bIn24G, TxCount;
 
 	Hal_ReadPowerValueFromPROM_8188E(&pwrInfo24G, PROMContent, AutoLoadFail);
@@ -478,34 +477,32 @@ void Hal_ReadTxPowerInfo88E(struct adapter *padapter, u8 *PROMContent, bool Auto
 	if (!AutoLoadFail)
 		pHalData->bTXPowerDataReadFromEEPORM = true;
 
-	for (rfPath = 0; rfPath < pHalData->NumTotalRFPath; rfPath++) {
-		for (ch = 0; ch < CHANNEL_MAX_NUMBER; ch++) {
-			bIn24G = Hal_GetChnlGroup88E(ch, &group);
-			if (bIn24G) {
-				pHalData->Index24G_CCK_Base[rfPath][ch] = pwrInfo24G.IndexCCK_Base[rfPath][group];
-				if (ch == 14)
-					pHalData->Index24G_BW40_Base[rfPath][ch] = pwrInfo24G.IndexBW40_Base[rfPath][4];
-				else
-					pHalData->Index24G_BW40_Base[rfPath][ch] = pwrInfo24G.IndexBW40_Base[rfPath][group];
-			}
-			if (bIn24G) {
-				DBG_88E("======= Path %d, Channel %d =======\n", rfPath, ch);
-				DBG_88E("Index24G_CCK_Base[%d][%d] = 0x%x\n", rfPath, ch , pHalData->Index24G_CCK_Base[rfPath][ch]);
-				DBG_88E("Index24G_BW40_Base[%d][%d] = 0x%x\n", rfPath, ch , pHalData->Index24G_BW40_Base[rfPath][ch]);
-			}
+	for (ch = 0; ch < CHANNEL_MAX_NUMBER; ch++) {
+		bIn24G = Hal_GetChnlGroup88E(ch, &group);
+		if (bIn24G) {
+			pHalData->Index24G_CCK_Base[0][ch] = pwrInfo24G.IndexCCK_Base[0][group];
+			if (ch == 14)
+				pHalData->Index24G_BW40_Base[0][ch] = pwrInfo24G.IndexBW40_Base[0][4];
+			else
+				pHalData->Index24G_BW40_Base[0][ch] = pwrInfo24G.IndexBW40_Base[0][group];
 		}
-		for (TxCount = 0; TxCount < MAX_TX_COUNT; TxCount++) {
-			pHalData->CCK_24G_Diff[rfPath][TxCount] = pwrInfo24G.CCK_Diff[rfPath][TxCount];
-			pHalData->OFDM_24G_Diff[rfPath][TxCount] = pwrInfo24G.OFDM_Diff[rfPath][TxCount];
-			pHalData->BW20_24G_Diff[rfPath][TxCount] = pwrInfo24G.BW20_Diff[rfPath][TxCount];
-			pHalData->BW40_24G_Diff[rfPath][TxCount] = pwrInfo24G.BW40_Diff[rfPath][TxCount];
-			DBG_88E("======= TxCount %d =======\n", TxCount);
-			DBG_88E("CCK_24G_Diff[%d][%d] = %d\n", rfPath, TxCount, pHalData->CCK_24G_Diff[rfPath][TxCount]);
-			DBG_88E("OFDM_24G_Diff[%d][%d] = %d\n", rfPath, TxCount, pHalData->OFDM_24G_Diff[rfPath][TxCount]);
-			DBG_88E("BW20_24G_Diff[%d][%d] = %d\n", rfPath, TxCount, pHalData->BW20_24G_Diff[rfPath][TxCount]);
-			DBG_88E("BW40_24G_Diff[%d][%d] = %d\n", rfPath, TxCount, pHalData->BW40_24G_Diff[rfPath][TxCount]);
+		if (bIn24G) {
+			DBG_88E("======= Path %d, Channel %d =======\n", 0, ch);
+			DBG_88E("Index24G_CCK_Base[%d][%d] = 0x%x\n", 0, ch, pHalData->Index24G_CCK_Base[0][ch]);
+			DBG_88E("Index24G_BW40_Base[%d][%d] = 0x%x\n", 0, ch, pHalData->Index24G_BW40_Base[0][ch]);
 		}
 	}
+	for (TxCount = 0; TxCount < MAX_TX_COUNT; TxCount++) {
+		pHalData->CCK_24G_Diff[0][TxCount] = pwrInfo24G.CCK_Diff[0][TxCount];
+		pHalData->OFDM_24G_Diff[0][TxCount] = pwrInfo24G.OFDM_Diff[0][TxCount];
+		pHalData->BW20_24G_Diff[0][TxCount] = pwrInfo24G.BW20_Diff[0][TxCount];
+		pHalData->BW40_24G_Diff[0][TxCount] = pwrInfo24G.BW40_Diff[0][TxCount];
+		DBG_88E("======= TxCount %d =======\n", TxCount);
+		DBG_88E("CCK_24G_Diff[%d][%d] = %d\n", 0, TxCount, pHalData->CCK_24G_Diff[0][TxCount]);
+		DBG_88E("OFDM_24G_Diff[%d][%d] = %d\n", 0, TxCount, pHalData->OFDM_24G_Diff[0][TxCount]);
+		DBG_88E("BW20_24G_Diff[%d][%d] = %d\n", 0, TxCount, pHalData->BW20_24G_Diff[0][TxCount]);
+		DBG_88E("BW40_24G_Diff[%d][%d] = %d\n", 0, TxCount, pHalData->BW40_24G_Diff[0][TxCount]);
+	}
 
 	/*  2010/10/19 MH Add Regulator recognize for CU. */
 	if (!AutoLoadFail) {
diff --git a/drivers/staging/rtl8188eu/hal/rtl8188eu_led.c b/drivers/staging/rtl8188eu/hal/rtl8188eu_led.c
index 780666a755ee..12879afb992e 100644
--- a/drivers/staging/rtl8188eu/hal/rtl8188eu_led.c
+++ b/drivers/staging/rtl8188eu/hal/rtl8188eu_led.c
@@ -46,16 +46,12 @@ void SwLedOff(struct adapter *padapter, struct LED_871x *pLed)
 
 	LedCfg = usb_read8(padapter, REG_LEDCFG2);/* 0x4E */
 
-	if (padapter->HalData->bLedOpenDrain) {
-			/*  Open-drain arrangement for controlling the LED) */
-		LedCfg &= 0x90; /*  Set to software control. */
-		usb_write8(padapter, REG_LEDCFG2, (LedCfg | BIT(3)));
-		LedCfg = usb_read8(padapter, REG_MAC_PINMUX_CFG);
-		LedCfg &= 0xFE;
-		usb_write8(padapter, REG_MAC_PINMUX_CFG, LedCfg);
-	} else {
-		usb_write8(padapter, REG_LEDCFG2, (LedCfg | BIT(3) | BIT(5) | BIT(6)));
-	}
+	/*  Open-drain arrangement for controlling the LED) */
+	LedCfg &= 0x90; /*  Set to software control. */
+	usb_write8(padapter, REG_LEDCFG2, (LedCfg | BIT(3)));
+	LedCfg = usb_read8(padapter, REG_MAC_PINMUX_CFG);
+	LedCfg &= 0xFE;
+	usb_write8(padapter, REG_MAC_PINMUX_CFG, LedCfg);
 exit:
 	pLed->bLedOn = false;
 }
@@ -69,10 +65,6 @@ void rtw_hal_sw_led_init(struct adapter *padapter)
 {
 	struct led_priv *pledpriv = &(padapter->ledpriv);
 
-	pledpriv->bRegUseLed = true;
-	pledpriv->LedControlHandler = LedControl8188eu;
-	padapter->HalData->bLedOpenDrain = true;
-
 	InitLed871x(padapter, &(pledpriv->SwLed0));
 }
 
diff --git a/drivers/staging/rtl8188eu/hal/rtl8188eu_recv.c b/drivers/staging/rtl8188eu/hal/rtl8188eu_recv.c
index d0495a16ff79..0fc093eb7a77 100644
--- a/drivers/staging/rtl8188eu/hal/rtl8188eu_recv.c
+++ b/drivers/staging/rtl8188eu/hal/rtl8188eu_recv.c
@@ -37,19 +37,15 @@ int	rtw_hal_init_recv_priv(struct adapter *padapter)
 	/* init recv_buf */
 	_rtw_init_queue(&precvpriv->free_recv_buf_queue);
 
-	precvpriv->pallocated_recv_buf =
+	precvpriv->precv_buf =
 		kcalloc(NR_RECVBUFF, sizeof(struct recv_buf), GFP_KERNEL);
-	if (!precvpriv->pallocated_recv_buf) {
+	if (!precvpriv->precv_buf) {
 		res = _FAIL;
 		RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
 				("alloc recv_buf fail!\n"));
 		goto exit;
 	}
-
-	precvpriv->precv_buf = precvpriv->pallocated_recv_buf;
-
-
-	precvbuf = (struct recv_buf *)precvpriv->precv_buf;
+	precvbuf = precvpriv->precv_buf;
 
 	for (i = 0; i < NR_RECVBUFF; i++) {
 		res = rtw_os_recvbuf_resource_alloc(padapter, precvbuf);
@@ -58,27 +54,18 @@ int	rtw_hal_init_recv_priv(struct adapter *padapter)
 		precvbuf->adapter = padapter;
 		precvbuf++;
 	}
-	precvpriv->free_recv_buf_queue_cnt = NR_RECVBUFF;
 	skb_queue_head_init(&precvpriv->rx_skb_queue);
 	{
 		int i;
-		size_t tmpaddr = 0;
-		size_t alignm = 0;
 		struct sk_buff *pskb = NULL;
 
 		skb_queue_head_init(&precvpriv->free_recv_skb_queue);
 
 		for (i = 0; i < NR_PREALLOC_RECV_SKB; i++) {
 			pskb = __netdev_alloc_skb(padapter->pnetdev,
-					MAX_RECVBUF_SZ + RECVBUFF_ALIGN_SZ,
-					GFP_KERNEL);
+					MAX_RECVBUF_SZ, GFP_KERNEL);
 			if (pskb) {
 				kmemleak_not_leak(pskb);
-				pskb->dev = padapter->pnetdev;
-				tmpaddr = (size_t)pskb->data;
-				alignm = tmpaddr & (RECVBUFF_ALIGN_SZ-1);
-				skb_reserve(pskb, (RECVBUFF_ALIGN_SZ - alignm));
-
 				skb_queue_tail(&precvpriv->free_recv_skb_queue,
 						pskb);
 			}
@@ -95,14 +82,14 @@ void rtw_hal_free_recv_priv(struct adapter *padapter)
 	struct recv_buf	*precvbuf;
 	struct recv_priv	*precvpriv = &padapter->recvpriv;
 
-	precvbuf = (struct recv_buf *)precvpriv->precv_buf;
+	precvbuf = precvpriv->precv_buf;
 
 	for (i = 0; i < NR_RECVBUFF; i++) {
 		usb_free_urb(precvbuf->purb);
 		precvbuf++;
 	}
 
-	kfree(precvpriv->pallocated_recv_buf);
+	kfree(precvpriv->precv_buf);
 
 	if (skb_queue_len(&precvpriv->rx_skb_queue))
 		DBG_88E(KERN_WARNING "rx_skb_queue not empty\n");
diff --git a/drivers/staging/rtl8188eu/hal/usb_halinit.c b/drivers/staging/rtl8188eu/hal/usb_halinit.c
index 7692ca495ee5..3675edb61942 100644
--- a/drivers/staging/rtl8188eu/hal/usb_halinit.c
+++ b/drivers/staging/rtl8188eu/hal/usb_halinit.c
@@ -562,9 +562,6 @@ static void InitUsbAggregationSetting(struct adapter *Adapter)
 
 	/*  Rx aggregation setting */
 	usb_AggSettingRxUpdate(Adapter);
-
-	/*  201/12/10 MH Add for USB agg mode dynamic switch. */
-	Adapter->HalData->UsbRxHighSpeedMode = false;
 }
 
 static void _InitBeaconParameters(struct adapter *Adapter)
@@ -604,11 +601,6 @@ static void _BBTurnOnBlock(struct adapter *Adapter)
 	phy_set_bb_reg(Adapter, rFPGA0_RFMOD, bOFDMEn, 0x1);
 }
 
-enum {
-	Antenna_Lfet = 1,
-	Antenna_Right = 2,
-};
-
 static void _InitAntenna_Selection(struct adapter *Adapter)
 {
 	struct hal_data_8188e *haldata = Adapter->HalData;
@@ -994,19 +986,16 @@ u32 rtw_hal_inirp_init(struct adapter *Adapter)
 	RT_TRACE(_module_hci_hal_init_c_, _drv_info_,
 		 ("===> usb_inirp_init\n"));
 
-	precvpriv->ff_hwaddr = RECV_BULK_IN_ADDR;
-
 	/* issue Rx irp to receive data */
-	precvbuf = (struct recv_buf *)precvpriv->precv_buf;
+	precvbuf = precvpriv->precv_buf;
 	for (i = 0; i < NR_RECVBUFF; i++) {
-		if (usb_read_port(Adapter, precvpriv->ff_hwaddr, 0, (unsigned char *)precvbuf) == false) {
+		if (usb_read_port(Adapter, RECV_BULK_IN_ADDR, precvbuf) == false) {
 			RT_TRACE(_module_hci_hal_init_c_, _drv_err_, ("usb_rx_init: usb_read_port error\n"));
 			status = _FAIL;
 			goto exit;
 		}
 
 		precvbuf++;
-		precvpriv->free_recv_buf_queue_cnt--;
 	}
 
 exit:
@@ -1107,18 +1096,12 @@ static void _ReadPROMContent(
 	readAdapterInfo_8188EU(Adapter);
 }
 
-static void _ReadRFType(struct adapter *Adapter)
-{
-	Adapter->HalData->rf_chip = RF_6052;
-}
-
 void rtw_hal_read_chip_info(struct adapter *Adapter)
 {
 	unsigned long start = jiffies;
 
 	MSG_88E("====> %s\n", __func__);
 
-	_ReadRFType(Adapter);/* rf_chip -> _InitRFType() */
 	_ReadPROMContent(Adapter);
 
 	MSG_88E("<==== %s in %d ms\n", __func__,
diff --git a/drivers/staging/rtl8188eu/include/Hal8188EPhyCfg.h b/drivers/staging/rtl8188eu/include/Hal8188EPhyCfg.h
index 0976a761b280..550ad62e7064 100644
--- a/drivers/staging/rtl8188eu/include/Hal8188EPhyCfg.h
+++ b/drivers/staging/rtl8188eu/include/Hal8188EPhyCfg.h
@@ -99,17 +99,6 @@ enum phy_rate_tx_offset_area {
 	RA_OFFSET_HT_CCK,
 };
 
-/* BB/RF related */
-enum RF_TYPE_8190P {
-	RF_TYPE_MIN,		/*  0 */
-	RF_8225 = 1,		/*  1 11b/g RF for verification only */
-	RF_8256 = 2,		/*  2 11b/g/n */
-	RF_8258 = 3,		/*  3 11a/b/g/n RF */
-	RF_6052 = 4,		/*  4 11b/g/n RF */
-	/*  TODO: We should remove this psudo PHY RF after we get new RF. */
-	RF_PSEUDO_11N = 5,	/*  5, It is a temporality RF. */
-};
-
 struct bb_reg_def {
 	u32 rfintfs;		/*  set software control: */
 				/*	0x870~0x877[8 bytes] */
diff --git a/drivers/staging/rtl8188eu/include/drv_types.h b/drivers/staging/rtl8188eu/include/drv_types.h
index 32326fd1dd24..e86419e525d8 100644
--- a/drivers/staging/rtl8188eu/include/drv_types.h
+++ b/drivers/staging/rtl8188eu/include/drv_types.h
@@ -156,8 +156,6 @@ struct adapter {
 	u8	hw_init_completed;
 
 	void *cmdThread;
-	void (*intf_start)(struct adapter *adapter);
-	void (*intf_stop)(struct adapter *adapter);
 	struct  net_device *pnetdev;
 	struct  net_device *pmondev;
 
diff --git a/drivers/staging/rtl8188eu/include/hal_intf.h b/drivers/staging/rtl8188eu/include/hal_intf.h
index fa032b0c12ff..e1114a95d442 100644
--- a/drivers/staging/rtl8188eu/include/hal_intf.h
+++ b/drivers/staging/rtl8188eu/include/hal_intf.h
@@ -190,6 +190,7 @@ void rtw_hal_set_odm_var(struct adapter *padapter,
 
 u32	rtw_hal_inirp_init(struct adapter *padapter);
 void	rtw_hal_inirp_deinit(struct adapter *padapter);
+void usb_intf_stop(struct adapter *padapter);
 
 s32	rtw_hal_xmit(struct adapter *padapter, struct xmit_frame *pxmitframe);
 s32	rtw_hal_mgnt_xmit(struct adapter *padapter,
diff --git a/drivers/staging/rtl8188eu/include/odm.h b/drivers/staging/rtl8188eu/include/odm.h
index 805f52e108b2..4fb3bb07ceaa 100644
--- a/drivers/staging/rtl8188eu/include/odm.h
+++ b/drivers/staging/rtl8188eu/include/odm.h
@@ -80,11 +80,6 @@
 #define DM_DIG_FA_TH2_LPS		30 /*  30 lps */
 #define RSSI_OFFSET_DIG			0x05;
 
-/* ANT Test */
-#define ANTTESTALL		0x00	/* Ant A or B will be Testing */
-#define ANTTESTA		0x01	/* Ant A will be Testing */
-#define ANTTESTB		0x02	/* Ant B will be testing */
-
 struct rtw_dig {
 	u8		Dig_Enable_Flag;
 	u8		Dig_Ext_Port_Stage;
@@ -590,7 +585,6 @@ struct odm_rf_cal {
 	s32	RegEBC;
 
 	u8	TXPowercount;
-	bool	bTXPowerTrackingInit;
 	bool	bTXPowerTracking;
 	u8	TxPowerTrackControl; /* for mp mode, turn off txpwrtracking
 				      * as default */
diff --git a/drivers/staging/rtl8188eu/include/osdep_intf.h b/drivers/staging/rtl8188eu/include/osdep_intf.h
index dbd7dc4f87dd..97d3d8504184 100644
--- a/drivers/staging/rtl8188eu/include/osdep_intf.h
+++ b/drivers/staging/rtl8188eu/include/osdep_intf.h
@@ -35,7 +35,8 @@ int rtw_init_netdev_name(struct net_device *pnetdev, const char *ifname);
 struct net_device *rtw_init_netdev(struct adapter *padapter);
 u16 rtw_recv_select_queue(struct sk_buff *skb);
 
-int pm_netdev_open(struct net_device *pnetdev, u8 bnormal);
+int netdev_open(struct net_device *pnetdev);
+int ips_netdrv_open(struct adapter *padapter);
 void rtw_ips_dev_unload(struct adapter *padapter);
 int rtw_ips_pwr_up(struct adapter *padapter);
 void rtw_ips_pwr_down(struct adapter *padapter);
diff --git a/drivers/staging/rtl8188eu/include/recv_osdep.h b/drivers/staging/rtl8188eu/include/recv_osdep.h
index 7550d58f6b5b..9b43a1314bd5 100644
--- a/drivers/staging/rtl8188eu/include/recv_osdep.h
+++ b/drivers/staging/rtl8188eu/include/recv_osdep.h
@@ -29,8 +29,6 @@ int rtw_recv_indicatepkt(struct adapter *adapter,
 
 void rtw_handle_tkip_mic_err(struct adapter *padapter, u8 bgroup);
 
-void rtw_os_recv_resource_alloc(struct recv_frame *recvfr);
-
 int rtw_os_recvbuf_resource_alloc(struct adapter *adapt, struct recv_buf *buf);
 
 void rtw_init_recv_timer(struct recv_reorder_ctrl *preorder_ctrl);
diff --git a/drivers/staging/rtl8188eu/include/rtl8188e_hal.h b/drivers/staging/rtl8188eu/include/rtl8188e_hal.h
index 7c81e3f3d12e..9330361da4ad 100644
--- a/drivers/staging/rtl8188eu/include/rtl8188e_hal.h
+++ b/drivers/staging/rtl8188eu/include/rtl8188e_hal.h
@@ -200,10 +200,6 @@ struct hal_data_8188e {
 
 	u16	BasicRateSet;
 
-	/* rf_ctrl */
-	u8	rf_chip;
-	u8	NumTotalRFPath;
-
 	u8	BoardType;
 
 	/*  EEPROM setting. */
@@ -265,14 +261,6 @@ struct hal_data_8188e {
 	u32	CCKTxPowerLevelOriginalOffset;
 
 	u8	CrystalCap;
-	u32	AntennaTxPath;			/*  Antenna path Tx */
-	u32	AntennaRxPath;			/*  Antenna path Rx */
-	u8	BluetoothCoexist;
-	u8	ExternalPA;
-
-	u8	bLedOpenDrain; /* Open-drain support for controlling the LED.*/
-
-	u8	b1x1RecvCombine;	/*  for 1T1R receive combining */
 
 	u32	AcParam_BE; /* Original parameter for BE, use for EDCA turbo. */
 
@@ -316,14 +304,6 @@ struct hal_data_8188e {
 	u8	OutEpQueueSel;
 	u8	OutEpNumber;
 
-	/*  Add for USB aggreation mode dynamic shceme. */
-	bool		UsbRxHighSpeedMode;
-
-	/*  2010/11/22 MH Add for slim combo debug mode selective. */
-	/*  This is used for fix the drawback of CU TSMC-A/UMC-A cut.
-	 * HW auto suspend ability. Close BT clock. */
-	bool		SlimComboDbg;
-
 	u16	EfuseUsedBytes;
 
 	/*  Auto FSM to Turn On, include clock, isolation, power control
diff --git a/drivers/staging/rtl8188eu/include/rtl8188e_recv.h b/drivers/staging/rtl8188eu/include/rtl8188e_recv.h
index 80832a5f0732..0d8bf51c72a9 100644
--- a/drivers/staging/rtl8188eu/include/rtl8188e_recv.h
+++ b/drivers/staging/rtl8188eu/include/rtl8188e_recv.h
@@ -51,9 +51,7 @@ enum rx_packet_type {
 };
 
 #define INTERRUPT_MSG_FORMAT_LEN 60
-void rtl8188eu_recv_hdl(struct adapter *padapter, struct recv_buf *precvbuf);
 void rtl8188eu_recv_tasklet(void *priv);
-void rtl8188e_query_rx_phy_status(struct recv_frame *fr, struct phy_stat *phy);
 void rtl8188e_process_phy_info(struct adapter *padapter,
 			       struct recv_frame *prframe);
 void update_recvframe_phyinfo_88e(struct recv_frame *fra, struct phy_stat *phy);
diff --git a/drivers/staging/rtl8188eu/include/rtw_led.h b/drivers/staging/rtl8188eu/include/rtw_led.h
index f2054ef70358..607d1ba56a46 100644
--- a/drivers/staging/rtl8188eu/include/rtw_led.h
+++ b/drivers/staging/rtl8188eu/include/rtw_led.h
@@ -70,12 +70,9 @@ struct LED_871x {
 
 	struct timer_list BlinkTimer; /*  Timer object for led blinking. */
 
-	u8 bSWLedCtrl;
-
 	/*  ALPHA, added by chiyoko, 20090106 */
 	u8 bLedNoLinkBlinkInProgress;
 	u8 bLedLinkBlinkInProgress;
-	u8 bLedStartToLinkBlinkInProgress;
 	u8 bLedScanBlinkInProgress;
 	struct work_struct BlinkWorkItem; /* Workitem used by BlinkTimer to
 					   * manipulate H/W to blink LED. */
@@ -91,18 +88,9 @@ void LedControl8188eu(struct adapter *padapter, enum LED_CTL_MODE	LedAction);
 struct led_priv {
 	/* add for led control */
 	struct LED_871x			SwLed0;
-	u8	bRegUseLed;
-	void (*LedControlHandler)(struct adapter *padapter,
-				  enum LED_CTL_MODE LedAction);
 	/* add for led control */
 };
 
-#define rtw_led_control(adapt, action) \
-	do { \
-		if ((adapt)->ledpriv.LedControlHandler) \
-			(adapt)->ledpriv.LedControlHandler((adapt), (action)); \
-	} while (0)
-
 void BlinkTimerCallback(unsigned long data);
 void BlinkWorkItemCallback(struct work_struct *work);
 
diff --git a/drivers/staging/rtl8188eu/include/rtw_mlme.h b/drivers/staging/rtl8188eu/include/rtw_mlme.h
index 9434b869c5e9..18fb7e7b2273 100644
--- a/drivers/staging/rtl8188eu/include/rtw_mlme.h
+++ b/drivers/staging/rtl8188eu/include/rtw_mlme.h
@@ -504,7 +504,7 @@ void rtw_scan_abort(struct adapter *adapter);
 int rtw_restruct_sec_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie,
 			uint in_len);
 int rtw_restruct_wmm_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie,
-		        uint in_len, uint initial_out_len);
+			uint in_len, uint initial_out_len);
 void rtw_init_registrypriv_dev_network(struct adapter *adapter);
 
 void rtw_update_registrypriv_dev_network(struct adapter *adapter);
diff --git a/drivers/staging/rtl8188eu/include/rtw_recv.h b/drivers/staging/rtl8188eu/include/rtw_recv.h
index 49d973881a04..052af7b891da 100644
--- a/drivers/staging/rtl8188eu/include/rtw_recv.h
+++ b/drivers/staging/rtl8188eu/include/rtw_recv.h
@@ -139,8 +139,6 @@ struct rx_pkt_attrib {
 #define SN_EQUAL(a, b)	(a == b)
 #define REORDER_WAIT_TIME	(50) /*  (ms) */
 
-#define RECVBUFF_ALIGN_SZ 8
-
 #define RXDESC_SIZE	24
 #define RXDESC_OFFSET RXDESC_SIZE
 
@@ -166,9 +164,7 @@ struct recv_priv {
 	struct __queue free_recv_queue;
 	struct __queue recv_pending_queue;
 	struct __queue uc_swdec_pending_queue;
-	u8 *pallocated_frame_buf;
-	u8 *precv_frame_buf;
-	uint free_recvframe_cnt;
+	void *pallocated_frame_buf;
 	struct adapter	*adapter;
 	u32	bIsAnyNonBEPkts;
 	u64	rx_bytes;
@@ -176,17 +172,12 @@ struct recv_priv {
 	u64	rx_drop;
 	u64	last_rx_bytes;
 
-	uint	ff_hwaddr;
-	u8	rx_pending_cnt;
-
 	struct tasklet_struct irq_prepare_beacon_tasklet;
 	struct tasklet_struct recv_tasklet;
 	struct sk_buff_head free_recv_skb_queue;
 	struct sk_buff_head rx_skb_queue;
-	u8 *pallocated_recv_buf;
-	u8 *precv_buf;    /*  4 alignment */
+	struct recv_buf *precv_buf;    /*  4 alignment */
 	struct __queue free_recv_buf_queue;
-	u32	free_recv_buf_queue_cnt;
 	/* For display the phy informatiom */
 	u8 is_signal_dbg;	/*  for debug */
 	u8 signal_strength_dbg;	/*  for debug */
diff --git a/drivers/staging/rtl8188eu/include/usb_ops_linux.h b/drivers/staging/rtl8188eu/include/usb_ops_linux.h
index 78d9b6e035bf..fb586365d2e5 100644
--- a/drivers/staging/rtl8188eu/include/usb_ops_linux.h
+++ b/drivers/staging/rtl8188eu/include/usb_ops_linux.h
@@ -53,7 +53,7 @@ u8 usb_read8(struct adapter *adapter, u32 addr);
 u16 usb_read16(struct adapter *adapter, u32 addr);
 u32 usb_read32(struct adapter *adapter, u32 addr);
 
-u32 usb_read_port(struct adapter *adapter, u32 addr, u32 cnt, u8 *pmem);
+u32 usb_read_port(struct adapter *adapter, u32 addr, struct recv_buf *precvbuf);
 void usb_read_port_cancel(struct adapter *adapter);
 
 int usb_write8(struct adapter *adapter, u32 addr, u8 val);
diff --git a/drivers/staging/rtl8188eu/include/xmit_osdep.h b/drivers/staging/rtl8188eu/include/xmit_osdep.h
index f96ca6af934d..959ef4b3066c 100644
--- a/drivers/staging/rtl8188eu/include/xmit_osdep.h
+++ b/drivers/staging/rtl8188eu/include/xmit_osdep.h
@@ -41,13 +41,11 @@ void rtw_os_xmit_schedule(struct adapter *padapter);
 
 int rtw_os_xmit_resource_alloc(struct adapter *padapter,
 			       struct xmit_buf *pxmitbuf, u32 alloc_sz);
-void rtw_os_xmit_resource_free(struct adapter *padapter,
-			       struct xmit_buf *pxmitbuf, u32 free_sz);
+void rtw_os_xmit_resource_free(struct xmit_buf *pxmitbuf);
 
 uint rtw_remainder_len(struct pkt_file *pfile);
 void _rtw_open_pktfile(struct sk_buff *pkt, struct pkt_file *pfile);
 uint _rtw_pktfile_read(struct pkt_file *pfile, u8 *rmem, uint rlen);
-int rtw_endofpktfile(struct pkt_file *pfile);
 
 void rtw_os_pkt_complete(struct adapter *padapter, struct sk_buff *pkt);
 void rtw_os_xmit_complete(struct adapter *padapter,
diff --git a/drivers/staging/rtl8188eu/os_dep/os_intfs.c b/drivers/staging/rtl8188eu/os_dep/os_intfs.c
index 40691f1ec507..8fc3fadf065f 100644
--- a/drivers/staging/rtl8188eu/os_dep/os_intfs.c
+++ b/drivers/staging/rtl8188eu/os_dep/os_intfs.c
@@ -144,7 +144,6 @@ static bool rtw_monitor_enable;
 module_param_named(monitor_enable, rtw_monitor_enable, bool, 0444);
 MODULE_PARM_DESC(monitor_enable, "Enable monitor inferface (default: false)");
 
-static int netdev_open(struct net_device *pnetdev);
 static int netdev_close(struct net_device *pnetdev);
 
 static void loadparam(struct adapter *padapter, struct net_device *pnetdev)
@@ -596,10 +595,9 @@ static int _netdev_open(struct net_device *pnetdev)
 			pr_info("can't init mlme_ext_priv\n");
 			goto netdev_open_error;
 		}
-		if (padapter->intf_start)
-			padapter->intf_start(padapter);
+		rtw_hal_inirp_init(padapter);
 
-		rtw_led_control(padapter, LED_CTL_NO_LINK);
+		LedControl8188eu(padapter, LED_CTL_NO_LINK);
 
 		padapter->bup = true;
 	}
@@ -630,7 +628,7 @@ netdev_open_error:
 	return -1;
 }
 
-static int netdev_open(struct net_device *pnetdev)
+int netdev_open(struct net_device *pnetdev)
 {
 	int ret;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(pnetdev);
@@ -642,7 +640,7 @@ static int netdev_open(struct net_device *pnetdev)
 	return ret;
 }
 
-static int  ips_netdrv_open(struct adapter *padapter)
+int  ips_netdrv_open(struct adapter *padapter)
 {
 	int status = _SUCCESS;
 
@@ -658,8 +656,7 @@ static int  ips_netdrv_open(struct adapter *padapter)
 		goto netdev_open_error;
 	}
 
-	if (padapter->intf_start)
-		padapter->intf_start(padapter);
+	rtw_hal_inirp_init(padapter);
 
 	rtw_set_pwr_state_check_timer(&padapter->pwrctrlpriv);
 	mod_timer(&padapter->mlmepriv.dynamic_chk_timer,
@@ -684,7 +681,7 @@ int rtw_ips_pwr_up(struct adapter *padapter)
 
 	result = ips_netdrv_open(padapter);
 
-	rtw_led_control(padapter, LED_CTL_NO_LINK);
+	LedControl8188eu(padapter, LED_CTL_NO_LINK);
 
 	DBG_88E("<===  rtw_ips_pwr_up.............. in %dms\n",
 		jiffies_to_msecs(jiffies - start_time));
@@ -699,7 +696,7 @@ void rtw_ips_pwr_down(struct adapter *padapter)
 
 	padapter->net_closed = true;
 
-	rtw_led_control(padapter, LED_CTL_POWER_OFF);
+	LedControl8188eu(padapter, LED_CTL_POWER_OFF);
 
 	rtw_ips_dev_unload(padapter);
 	DBG_88E("<=== rtw_ips_pwr_down..................... in %dms\n",
@@ -712,25 +709,13 @@ void rtw_ips_dev_unload(struct adapter *padapter)
 
 	rtw_hal_set_hwreg(padapter, HW_VAR_FIFO_CLEARN_UP, NULL);
 
-	if (padapter->intf_stop)
-		padapter->intf_stop(padapter);
+	usb_intf_stop(padapter);
 
 	/* s5. */
 	if (!padapter->bSurpriseRemoved)
 		rtw_hal_deinit(padapter);
 }
 
-int pm_netdev_open(struct net_device *pnetdev, u8 bnormal)
-{
-	int status;
-
-	if (bnormal)
-		status = netdev_open(pnetdev);
-	else
-		status =  (_SUCCESS == ips_netdrv_open((struct adapter *)rtw_netdev_priv(pnetdev))) ? (0) : (-1);
-	return status;
-}
-
 static int netdev_close(struct net_device *pnetdev)
 {
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(pnetdev);
@@ -763,7 +748,7 @@ static int netdev_close(struct net_device *pnetdev)
 		/* s2-4. */
 		rtw_free_network_queue(padapter, true);
 		/*  Close LED */
-		rtw_led_control(padapter, LED_CTL_POWER_OFF);
+		LedControl8188eu(padapter, LED_CTL_POWER_OFF);
 	}
 
 	RT_TRACE(_module_os_intfs_c_, _drv_info_, ("-88eu_drv - drv_close\n"));
diff --git a/drivers/staging/rtl8188eu/os_dep/osdep_service.c b/drivers/staging/rtl8188eu/os_dep/osdep_service.c
index 7cd2655f27fe..6ff836f481da 100644
--- a/drivers/staging/rtl8188eu/os_dep/osdep_service.c
+++ b/drivers/staging/rtl8188eu/os_dep/osdep_service.c
@@ -12,8 +12,6 @@
  * more details.
  *
  ******************************************************************************/
-
-
 #define _OSDEP_SERVICE_C_
 
 #include <osdep_service.h>
@@ -24,9 +22,10 @@
 #include <rtw_ioctl_set.h>
 
 /*
-* Translate the OS dependent @param error_code to OS independent RTW_STATUS_CODE
-* @return: one of RTW_STATUS_CODE
-*/
+ * Translate the OS dependent @param error_code to OS independent
+ * RTW_STATUS_CODE
+ * @return: one of RTW_STATUS_CODE
+ */
 inline int RTW_STATUS_CODE(int error_code)
 {
 	if (error_code >= 0)
@@ -43,22 +42,20 @@ void *rtw_malloc2d(int h, int w, int size)
 {
 	int j;
 
-	void **a = kzalloc(h*sizeof(void *) + h*w*size, GFP_KERNEL);
-	if (!a) {
-		pr_info("%s: alloc memory fail!\n", __func__);
-		return NULL;
-	}
+	void **a = kzalloc(h * sizeof(void *) + h * w * size, GFP_KERNEL);
+	if (!a)
+		goto out;
 
 	for (j = 0; j < h; j++)
-		a[j] = ((char *)(a+h)) + j*w*size;
-
+		a[j] = ((char *)(a + h)) + j * w * size;
+out:
 	return a;
 }
 
-void	_rtw_init_queue(struct __queue *pqueue)
+void _rtw_init_queue(struct __queue *pqueue)
 {
-	INIT_LIST_HEAD(&(pqueue->queue));
-	spin_lock_init(&(pqueue->lock));
+	INIT_LIST_HEAD(&pqueue->queue);
+	spin_lock_init(&pqueue->lock);
 }
 
 struct net_device *rtw_alloc_etherdev_with_old_priv(void *old_priv)
diff --git a/drivers/staging/rtl8188eu/os_dep/recv_linux.c b/drivers/staging/rtl8188eu/os_dep/recv_linux.c
index 103cdb4ed073..b85824ec5354 100644
--- a/drivers/staging/rtl8188eu/os_dep/recv_linux.c
+++ b/drivers/staging/rtl8188eu/os_dep/recv_linux.c
@@ -21,12 +21,6 @@
 #include <osdep_intf.h>
 #include <usb_ops_linux.h>
 
-/* alloc os related resource in struct recv_frame */
-void rtw_os_recv_resource_alloc(struct recv_frame *precvframe)
-{
-	precvframe->pkt = NULL;
-}
-
 /* alloc os related resource in struct recv_buf */
 int rtw_os_recvbuf_resource_alloc(struct adapter *padapter,
 				  struct recv_buf *precvbuf)
diff --git a/drivers/staging/rtl8188eu/os_dep/usb_intf.c b/drivers/staging/rtl8188eu/os_dep/usb_intf.c
index 68e1e6bbe87f..c6316ffa64d3 100644
--- a/drivers/staging/rtl8188eu/os_dep/usb_intf.c
+++ b/drivers/staging/rtl8188eu/os_dep/usb_intf.c
@@ -141,16 +141,7 @@ static void usb_dvobj_deinit(struct usb_interface *usb_intf)
 
 }
 
-static void usb_intf_start(struct adapter *padapter)
-{
-	RT_TRACE(_module_hci_intfs_c_, _drv_err_, ("+usb_intf_start\n"));
-
-	rtw_hal_inirp_init(padapter);
-
-	RT_TRACE(_module_hci_intfs_c_, _drv_err_, ("-usb_intf_start\n"));
-}
-
-static void usb_intf_stop(struct adapter *padapter)
+void usb_intf_stop(struct adapter *padapter)
 {
 	RT_TRACE(_module_hci_intfs_c_, _drv_err_, ("+usb_intf_stop\n"));
 
@@ -183,8 +174,7 @@ static void rtw_dev_unload(struct adapter *padapter)
 		if (padapter->xmitpriv.ack_tx)
 			rtw_ack_tx_done(&padapter->xmitpriv, RTW_SCTX_DONE_DRV_STOP);
 		/* s3. */
-		if (padapter->intf_stop)
-			padapter->intf_stop(padapter);
+		usb_intf_stop(padapter);
 		/* s4. */
 		if (!padapter->pwrctrlpriv.bInternalAutoSuspend)
 			rtw_stop_drv_threads(padapter);
@@ -294,7 +284,7 @@ static int rtw_resume_process(struct adapter *padapter)
 	pwrpriv->bkeepfwalive = false;
 
 	pr_debug("bkeepfwalive(%x)\n", pwrpriv->bkeepfwalive);
-	if (pm_netdev_open(pnetdev, true) != 0) {
+	if (netdev_open(pnetdev) != 0) {
 		mutex_unlock(&pwrpriv->mutex_lock);
 		goto exit;
 	}
@@ -366,9 +356,6 @@ static struct adapter *rtw_usb_if1_init(struct dvobj_priv *dvobj,
 	if (!padapter->HalData)
 		DBG_88E("cant not alloc memory for HAL DATA\n");
 
-	padapter->intf_start = &usb_intf_start;
-	padapter->intf_stop = &usb_intf_stop;
-
 	/* step read_chip_version */
 	rtw_hal_read_chip_version(padapter);
 
diff --git a/drivers/staging/rtl8188eu/os_dep/usb_ops_linux.c b/drivers/staging/rtl8188eu/os_dep/usb_ops_linux.c
index d0d591501b73..e2dbe1b4afd3 100644
--- a/drivers/staging/rtl8188eu/os_dep/usb_ops_linux.c
+++ b/drivers/staging/rtl8188eu/os_dep/usb_ops_linux.c
@@ -167,27 +167,26 @@ static int recvbuf2recvframe(struct adapter *adapt, struct sk_buff *pskb)
 		}
 		if (pattrib->pkt_rpt_type == NORMAL_RX) { /* Normal rx packet */
 			if (pattrib->physt)
-				update_recvframe_phyinfo_88e(precvframe, (struct phy_stat *)pphy_status);
+				update_recvframe_phyinfo_88e(precvframe, pphy_status);
 			if (rtw_recv_entry(precvframe) != _SUCCESS) {
 				RT_TRACE(_module_rtl871x_recv_c_, _drv_err_,
 					("recvbuf2recvframe: rtw_recv_entry(precvframe) != _SUCCESS\n"));
 			}
-		} else {
-			/* enqueue recvframe to txrtp queue */
-			if (pattrib->pkt_rpt_type == TX_REPORT1) {
-				/* CCX-TXRPT ack for xmit mgmt frames. */
-				handle_txrpt_ccx_88e(adapt, precvframe->rx_data);
-			} else if (pattrib->pkt_rpt_type == TX_REPORT2) {
-				ODM_RA_TxRPT2Handle_8188E(
-							&haldata->odmpriv,
-							precvframe->rx_data,
-							pattrib->pkt_len,
-							pattrib->MacIDValidEntry[0],
-							pattrib->MacIDValidEntry[1]
-							);
-			} else if (pattrib->pkt_rpt_type == HIS_REPORT) {
-				interrupt_handler_8188eu(adapt, pattrib->pkt_len, precvframe->rx_data);
-			}
+		} else if (pattrib->pkt_rpt_type == TX_REPORT1) {
+			/* CCX-TXRPT ack for xmit mgmt frames. */
+			handle_txrpt_ccx_88e(adapt, precvframe->rx_data);
+			rtw_free_recvframe(precvframe, pfree_recv_queue);
+		} else if (pattrib->pkt_rpt_type == TX_REPORT2) {
+			ODM_RA_TxRPT2Handle_8188E(
+						&haldata->odmpriv,
+						precvframe->rx_data,
+						pattrib->pkt_len,
+						pattrib->MacIDValidEntry[0],
+						pattrib->MacIDValidEntry[1]
+						);
+			rtw_free_recvframe(precvframe, pfree_recv_queue);
+		} else if (pattrib->pkt_rpt_type == HIS_REPORT) {
+			interrupt_handler_8188eu(adapt, pattrib->pkt_len, precvframe->rx_data);
 			rtw_free_recvframe(precvframe, pfree_recv_queue);
 		}
 		pkt_cnt--;
@@ -253,7 +252,7 @@ static int usbctrl_vendorreq(struct adapter *adapt, u8 request, u16 value, u16 i
 	/*  Acquire IO memory for vendorreq */
 	pIo_buf = kmalloc(MAX_USB_IO_CTL_SIZE, GFP_ATOMIC);
 
-	if (pIo_buf == NULL) {
+	if (!pIo_buf) {
 		DBG_88E("[%s] pIo_buf == NULL\n", __func__);
 		status = -ENOMEM;
 		goto release_mutex;
@@ -384,8 +383,6 @@ static void usb_read_port_complete(struct urb *purb, struct pt_regs *regs)
 
 	RT_TRACE(_module_hci_ops_os_c_, _drv_err_, ("usb_read_port_complete!!!\n"));
 
-	precvpriv->rx_pending_cnt--;
-
 	if (adapt->bSurpriseRemoved || adapt->bDriverStopped || adapt->bReadPortCancel) {
 		RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
 			 ("usb_read_port_complete:bDriverStopped(%d) OR bSurpriseRemoved(%d)\n",
@@ -403,7 +400,7 @@ static void usb_read_port_complete(struct urb *purb, struct pt_regs *regs)
 			RT_TRACE(_module_hci_ops_os_c_, _drv_err_,
 				 ("usb_read_port_complete: (purb->actual_length > MAX_RECVBUF_SZ) || (purb->actual_length < RXDESC_SIZE)\n"));
 			precvbuf->reuse = true;
-			usb_read_port(adapt, precvpriv->ff_hwaddr, 0, (unsigned char *)precvbuf);
+			usb_read_port(adapt, RECV_BULK_IN_ADDR, precvbuf);
 			DBG_88E("%s()-%d: RX Warning!\n", __func__, __LINE__);
 		} else {
 			skb_put(precvbuf->pskb, purb->actual_length);
@@ -414,7 +411,7 @@ static void usb_read_port_complete(struct urb *purb, struct pt_regs *regs)
 
 			precvbuf->pskb = NULL;
 			precvbuf->reuse = false;
-			usb_read_port(adapt, precvpriv->ff_hwaddr, 0, (unsigned char *)precvbuf);
+			usb_read_port(adapt, RECV_BULK_IN_ADDR, precvbuf);
 		}
 	} else {
 		RT_TRACE(_module_hci_ops_os_c_, _drv_err_, ("usb_read_port_complete : purb->status(%d) != 0\n", purb->status));
@@ -437,7 +434,7 @@ static void usb_read_port_complete(struct urb *purb, struct pt_regs *regs)
 		case -EOVERFLOW:
 			adapt->HalData->srestpriv.Wifi_Error_Status = USB_READ_PORT_FAIL;
 			precvbuf->reuse = true;
-			usb_read_port(adapt, precvpriv->ff_hwaddr, 0, (unsigned char *)precvbuf);
+			usb_read_port(adapt, RECV_BULK_IN_ADDR, precvbuf);
 			break;
 		case -EINPROGRESS:
 			DBG_88E("ERROR: URB IS IN PROGRESS!\n");
@@ -448,17 +445,14 @@ static void usb_read_port_complete(struct urb *purb, struct pt_regs *regs)
 	}
 }
 
-u32 usb_read_port(struct adapter *adapter, u32 addr, u32 cnt, u8 *rmem)
+u32 usb_read_port(struct adapter *adapter, u32 addr, struct recv_buf *precvbuf)
 {
 	struct urb *purb = NULL;
-	struct recv_buf	*precvbuf = (struct recv_buf *)rmem;
 	struct dvobj_priv	*pdvobj = adapter_to_dvobj(adapter);
 	struct recv_priv	*precvpriv = &adapter->recvpriv;
 	struct usb_device	*pusbd = pdvobj->pusbdev;
 	int err;
 	unsigned int pipe;
-	size_t tmpaddr = 0;
-	size_t alignment = 0;
 	u32 ret = _SUCCESS;
 
 
@@ -483,22 +477,16 @@ u32 usb_read_port(struct adapter *adapter, u32 addr, u32 cnt, u8 *rmem)
 
 	/* re-assign for linux based on skb */
 	if ((!precvbuf->reuse) || (precvbuf->pskb == NULL)) {
-		precvbuf->pskb = netdev_alloc_skb(adapter->pnetdev, MAX_RECVBUF_SZ + RECVBUFF_ALIGN_SZ);
+		precvbuf->pskb = netdev_alloc_skb(adapter->pnetdev, MAX_RECVBUF_SZ);
 		if (precvbuf->pskb == NULL) {
 			RT_TRACE(_module_hci_ops_os_c_, _drv_err_, ("init_recvbuf(): alloc_skb fail!\n"));
 			DBG_88E("#### usb_read_port() alloc_skb fail!#####\n");
 			return _FAIL;
 		}
-
-		tmpaddr = (size_t)precvbuf->pskb->data;
-		alignment = tmpaddr & (RECVBUFF_ALIGN_SZ-1);
-		skb_reserve(precvbuf->pskb, (RECVBUFF_ALIGN_SZ - alignment));
 	} else { /* reuse skb */
 		precvbuf->reuse = false;
 	}
 
-	precvpriv->rx_pending_cnt++;
-
 	purb = precvbuf->purb;
 
 	/* translate DMA FIFO addr to pipehandle */
@@ -528,7 +516,7 @@ void rtw_hal_inirp_deinit(struct adapter *padapter)
 	int i;
 	struct recv_buf *precvbuf;
 
-	precvbuf = (struct recv_buf *)padapter->recvpriv.precv_buf;
+	precvbuf = padapter->recvpriv.precv_buf;
 
 	DBG_88E("%s\n", __func__);
 
diff --git a/drivers/staging/rtl8188eu/os_dep/xmit_linux.c b/drivers/staging/rtl8188eu/os_dep/xmit_linux.c
index 4b1b04e00715..e097c619ed1b 100644
--- a/drivers/staging/rtl8188eu/os_dep/xmit_linux.c
+++ b/drivers/staging/rtl8188eu/os_dep/xmit_linux.c
@@ -59,11 +59,6 @@ uint _rtw_pktfile_read(struct pkt_file *pfile, u8 *rmem, uint rlen)
 	return len;
 }
 
-int rtw_endofpktfile(struct pkt_file *pfile)
-{
-	return pfile->pkt_len == 0;
-}
-
 int rtw_os_xmit_resource_alloc(struct adapter *padapter, struct xmit_buf *pxmitbuf, u32 alloc_sz)
 {
 	int i;
@@ -85,8 +80,7 @@ int rtw_os_xmit_resource_alloc(struct adapter *padapter, struct xmit_buf *pxmitb
 	return _SUCCESS;
 }
 
-void rtw_os_xmit_resource_free(struct adapter *padapter,
-			       struct xmit_buf *pxmitbuf, u32 free_sz)
+void rtw_os_xmit_resource_free(struct xmit_buf *pxmitbuf)
 {
 	int i;
 
diff --git a/drivers/staging/rtl8192e/Makefile b/drivers/staging/rtl8192e/Makefile
index cb18db74d78c..7101fcc8871b 100644
--- a/drivers/staging/rtl8192e/Makefile
+++ b/drivers/staging/rtl8192e/Makefile
@@ -17,5 +17,3 @@ obj-$(CONFIG_RTLLIB_CRYPTO_TKIP) += rtllib_crypt_tkip.o
 obj-$(CONFIG_RTLLIB_CRYPTO_WEP) += rtllib_crypt_wep.o
 
 obj-$(CONFIG_RTL8192E) += rtl8192e/
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/staging/rtl8192e/dot11d.c b/drivers/staging/rtl8192e/dot11d.c
index 25725b158eca..017fe04ebe2d 100644
--- a/drivers/staging/rtl8192e/dot11d.c
+++ b/drivers/staging/rtl8192e/dot11d.c
@@ -11,7 +11,7 @@
  *
  * Contact Information:
  * wlanfae <wlanfae@realtek.com>
-******************************************************************************/
+ ******************************************************************************/
 #include "dot11d.h"
 
 struct channel_list {
diff --git a/drivers/staging/rtl8192e/rtl8192e/Makefile b/drivers/staging/rtl8192e/rtl8192e/Makefile
index a2c4fb4ba1af..176a4a2b8b20 100644
--- a/drivers/staging/rtl8192e/rtl8192e/Makefile
+++ b/drivers/staging/rtl8192e/rtl8192e/Makefile
@@ -16,5 +16,3 @@ r8192e_pci-objs :=		\
 	rtl_wx.o		\
 
 obj-$(CONFIG_RTL8192E) += r8192e_pci.o
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_cmdpkt.c b/drivers/staging/rtl8192e/rtl8192e/r8192E_cmdpkt.c
index f9003a28cae2..757ffd4f2f89 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_cmdpkt.c
+++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_cmdpkt.c
@@ -49,7 +49,7 @@ bool rtl92e_send_cmd_pkt(struct net_device *dev, u32 type, const void *data,
 		else
 			skb = dev_alloc_skb(frag_length + 4);
 
-		if (skb == NULL) {
+		if (!skb) {
 			rt_status = false;
 			goto Failed;
 		}
diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_firmware.c b/drivers/staging/rtl8192e/rtl8192e/r8192E_firmware.c
index 9aaa85526eb8..bbe399010be1 100644
--- a/drivers/staging/rtl8192e/rtl8192e/r8192E_firmware.c
+++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_firmware.c
@@ -202,7 +202,5 @@ bool rtl92e_init_fw(struct net_device *dev)
 
 download_firmware_fail:
 	netdev_err(dev, "%s: Failed to initialize firmware.\n", __func__);
-	rt_status = false;
-	return rt_status;
-
+	return false;
 }
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
index 5f53fbd565ef..8a9172aa8178 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
@@ -367,7 +367,7 @@ static void _rtl92e_update_cap(struct net_device *dev, u16 cap)
 	}
 }
 
-static struct rtllib_qos_parameters def_qos_parameters = {
+static const struct rtllib_qos_parameters def_qos_parameters = {
 	{cpu_to_le16(3), cpu_to_le16(3), cpu_to_le16(3), cpu_to_le16(3)},
 	{cpu_to_le16(7), cpu_to_le16(7), cpu_to_le16(7), cpu_to_le16(7)},
 	{2, 2, 2, 2},
diff --git a/drivers/staging/rtl8192e/rtl819x_BAProc.c b/drivers/staging/rtl8192e/rtl819x_BAProc.c
index c7fd1b1653d6..20260af49ee7 100644
--- a/drivers/staging/rtl8192e/rtl819x_BAProc.c
+++ b/drivers/staging/rtl8192e/rtl819x_BAProc.c
@@ -11,7 +11,7 @@
  *
  * Contact Information:
  * wlanfae <wlanfae@realtek.com>
-******************************************************************************/
+ ******************************************************************************/
 #include <asm/byteorder.h>
 #include <asm/unaligned.h>
 #include <linux/etherdevice.h>
diff --git a/drivers/staging/rtl8192e/rtl819x_HTProc.c b/drivers/staging/rtl8192e/rtl819x_HTProc.c
index dd9c0c868361..cded0f43cd33 100644
--- a/drivers/staging/rtl8192e/rtl819x_HTProc.c
+++ b/drivers/staging/rtl8192e/rtl819x_HTProc.c
@@ -11,7 +11,7 @@
  *
  * Contact Information:
  * wlanfae <wlanfae@realtek.com>
-******************************************************************************/
+ ******************************************************************************/
 #include "rtllib.h"
 #include "rtl819x_HT.h"
 u8 MCS_FILTER_ALL[16] = {
diff --git a/drivers/staging/rtl8192e/rtl819x_TSProc.c b/drivers/staging/rtl8192e/rtl819x_TSProc.c
index a966a8e490ab..48bbd9e8a52f 100644
--- a/drivers/staging/rtl8192e/rtl819x_TSProc.c
+++ b/drivers/staging/rtl8192e/rtl819x_TSProc.c
@@ -11,7 +11,7 @@
  *
  * Contact Information:
  * wlanfae <wlanfae@realtek.com>
-******************************************************************************/
+ ******************************************************************************/
 #include "rtllib.h"
 #include <linux/etherdevice.h>
 #include "rtl819x_TS.h"
diff --git a/drivers/staging/rtl8192e/rtllib_rx.c b/drivers/staging/rtl8192e/rtllib_rx.c
index c743182b933e..e5ba7d1a809f 100644
--- a/drivers/staging/rtl8192e/rtllib_rx.c
+++ b/drivers/staging/rtl8192e/rtllib_rx.c
@@ -130,7 +130,7 @@ rtllib_frag_cache_get(struct rtllib_device *ieee,
 				    ETH_ALEN /* WDS */ +
 				    /* QOS Control */
 				    (RTLLIB_QOS_HAS_SEQ(fc) ? 2 : 0));
-		if (skb == NULL)
+		if (!skb)
 			return NULL;
 
 		entry = &ieee->frag_cache[tid][ieee->frag_next_idx[tid]];
@@ -986,7 +986,7 @@ static void rtllib_rx_extract_addr(struct rtllib_device *ieee,
 		ether_addr_copy(src, hdr->addr4);
 		ether_addr_copy(bssid, ieee->current_network.bssid);
 		break;
-	case 0:
+	default:
 		ether_addr_copy(dst, hdr->addr1);
 		ether_addr_copy(src, hdr->addr2);
 		ether_addr_copy(bssid, hdr->addr3);
@@ -1201,6 +1201,7 @@ static int rtllib_rx_decrypt(struct rtllib_device *ieee, struct sk_buff *skb,
 	if (crypt && !(fc & RTLLIB_FCTL_WEP) &&
 	    rtllib_is_eapol_frame(ieee, skb, hdrlen)) {
 		struct eapol *eap = (struct eapol *)(skb->data + 24);
+
 		netdev_dbg(ieee->dev, "RX: IEEE 802.1X EAPOL frame: %s\n",
 			   eap_get_type(eap->type));
 	}
@@ -1430,7 +1431,7 @@ static int rtllib_rx_InfraAdhoc(struct rtllib_device *ieee, struct sk_buff *skb,
 	/* skb: hdr + (possible reassembled) full plaintext payload */
 	payload = skb->data + hdrlen;
 	rxb = kmalloc(sizeof(struct rtllib_rxb), GFP_ATOMIC);
-	if (rxb == NULL)
+	if (!rxb)
 		goto rx_dropped;
 
 	/* to parse amsdu packets */
diff --git a/drivers/staging/rtl8192e/rtllib_softmac.c b/drivers/staging/rtl8192e/rtllib_softmac.c
index da74dc49b95e..1430ba27b049 100644
--- a/drivers/staging/rtl8192e/rtllib_softmac.c
+++ b/drivers/staging/rtl8192e/rtllib_softmac.c
@@ -1524,6 +1524,7 @@ static void rtllib_associate_complete_wq(void *data)
 				     struct rtllib_device,
 				     associate_complete_wq);
 	struct rt_pwr_save_ctrl *pPSC = &(ieee->PowerSaveControl);
+
 	netdev_info(ieee->dev, "Associated successfully\n");
 	if (!ieee->is_silent_reset) {
 		netdev_info(ieee->dev, "normal associate\n");
diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c
index 6fa96d57d316..e68850897adf 100644
--- a/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c
+++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_crypt_tkip.c
@@ -553,7 +553,7 @@ static void michael_mic_hdr(struct sk_buff *skb, u8 *hdr)
 		memcpy(hdr, hdr11->addr3, ETH_ALEN); /* DA */
 		memcpy(hdr + ETH_ALEN, hdr11->addr4, ETH_ALEN); /* SA */
 		break;
-	case 0:
+	default:
 		memcpy(hdr, hdr11->addr1, ETH_ALEN); /* DA */
 		memcpy(hdr + ETH_ALEN, hdr11->addr2, ETH_ALEN); /* SA */
 		break;
diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
index 89cbc077a48d..82f654305414 100644
--- a/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
+++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
@@ -129,7 +129,7 @@ ieee80211_frag_cache_get(struct ieee80211_device *ieee,
 				    8 /* WEP */ +
 				    ETH_ALEN /* WDS */ +
 				    (IEEE80211_QOS_HAS_SEQ(fc)?2:0) /* QOS Control */);
-		if (skb == NULL)
+		if (!skb)
 			return NULL;
 
 		entry = &ieee->frag_cache[tid][ieee->frag_next_idx[tid]];
@@ -1079,7 +1079,7 @@ int ieee80211_rx(struct ieee80211_device *ieee, struct sk_buff *skb,
 		memcpy(src, hdr->addr4, ETH_ALEN);
 		memcpy(bssid, ieee->current_network.bssid, ETH_ALEN);
 		break;
-	case 0:
+	default:
 		memcpy(dst, hdr->addr1, ETH_ALEN);
 		memcpy(src, hdr->addr2, ETH_ALEN);
 		memcpy(bssid, hdr->addr3, ETH_ALEN);
diff --git a/drivers/staging/rtl8712/osdep_service.h b/drivers/staging/rtl8712/osdep_service.h
index c9ea50daffff..b8a170978434 100644
--- a/drivers/staging/rtl8712/osdep_service.h
+++ b/drivers/staging/rtl8712/osdep_service.h
@@ -63,15 +63,6 @@ static inline u32 end_of_queue_search(struct list_head *head,
 	return (head == plist);
 }
 
-static inline void sleep_schedulable(int ms)
-{
-	u32 delta;
-
-	delta = msecs_to_jiffies(ms);/*(ms)*/
-	set_current_state(TASK_INTERRUPTIBLE);
-	schedule_timeout(delta);
-}
-
 static inline void flush_signals_thread(void)
 {
 	if (signal_pending(current))
diff --git a/drivers/staging/rtl8712/rtl8712_hal.h b/drivers/staging/rtl8712/rtl8712_hal.h
index 57d5d2db3c77..84456bb560ef 100644
--- a/drivers/staging/rtl8712/rtl8712_hal.h
+++ b/drivers/staging/rtl8712/rtl8712_hal.h
@@ -68,14 +68,14 @@ struct fw_priv {   /*8-bytes alignment required*/
 	unsigned char signature_0;  /*0x12: CE product, 0x92: IT product*/
 	unsigned char signature_1;  /*0x87: CE product, 0x81: IT product*/
 	unsigned char hci_sel; /*0x81: PCI-AP, 01:PCIe, 02: 92S-U, 0x82: USB-AP,
-			    * 0x12: 72S-U, 03:SDIO
-			    */
+				* 0x12: 72S-U, 03:SDIO
+				*/
 	unsigned char chip_version; /*the same value as register value*/
 	unsigned char customer_ID_0; /*customer  ID low byte*/
 	unsigned char customer_ID_1; /*customer  ID high byte*/
 	unsigned char rf_config;  /*0x11:  1T1R, 0x12: 1T2R, 0x92: 1T2R turbo,
-			     * 0x22: 2T2R
-			     */
+				   * 0x22: 2T2R
+				   */
 	unsigned char usb_ep_num;  /* 4: 4EP, 6: 6EP, 11: 11EP*/
 	/*--- long word 1 ----*/
 	unsigned char regulatory_class_0; /*regulatory class bit map 0*/
@@ -99,8 +99,8 @@ struct fw_priv {   /*8-bytes alignment required*/
 	unsigned char qos_en;    /*1: QoS enable*/
 	unsigned char bw_40MHz_en;   /*1: 40MHz BW enable*/
 	unsigned char AMSDU2AMPDU_en;   /*1: 4181 convert AMSDU to AMPDU,
-				   * 0: disable
-				   */
+					 * 0: disable
+					 */
 	unsigned char AMPDU_en;   /*1: 11n AMPDU enable*/
 	unsigned char rate_control_offload; /*1: FW offloads,0: driver handles*/
 	unsigned char aggregation_offload;  /*1: FW offloads,0: driver handles*/
diff --git a/drivers/staging/rtl8712/rtl8712_led.c b/drivers/staging/rtl8712/rtl8712_led.c
index a8e237e480c9..317aeeed38e8 100644
--- a/drivers/staging/rtl8712/rtl8712_led.c
+++ b/drivers/staging/rtl8712/rtl8712_led.c
@@ -355,7 +355,7 @@ static void SwLedBlink1(struct LED_871x *pLed)
 			}
 			pLed->bLedScanBlinkInProgress = false;
 		} else {
-			 if (pLed->bLedOn)
+			if (pLed->bLedOn)
 				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
 				pLed->BlinkingLedState = LED_STATE_ON;
@@ -390,7 +390,7 @@ static void SwLedBlink1(struct LED_871x *pLed)
 			pLed->BlinkTimes = 0;
 			pLed->bLedBlinkInProgress = false;
 		} else {
-			 if (pLed->bLedOn)
+			if (pLed->bLedOn)
 				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
 				pLed->BlinkingLedState = LED_STATE_ON;
@@ -460,7 +460,7 @@ static void SwLedBlink2(struct LED_871x *pLed)
 			}
 			pLed->bLedScanBlinkInProgress = false;
 		} else {
-			 if (pLed->bLedOn)
+			if (pLed->bLedOn)
 				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
 				pLed->BlinkingLedState = LED_STATE_ON;
@@ -667,7 +667,7 @@ static void SwLedBlink4(struct LED_871x *pLed)
 				  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 			pLed->bLedBlinkInProgress = false;
 		} else {
-			 if (pLed->bLedOn)
+			if (pLed->bLedOn)
 				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
 				pLed->BlinkingLedState = LED_STATE_ON;
@@ -764,7 +764,7 @@ static void SwLedBlink5(struct LED_871x *pLed)
 					  msecs_to_jiffies(LED_BLINK_FASTER_INTERVAL_ALPHA));
 			pLed->bLedBlinkInProgress = false;
 		} else {
-			 if (pLed->bLedOn)
+			if (pLed->bLedOn)
 				pLed->BlinkingLedState = LED_STATE_OFF;
 			else
 				pLed->BlinkingLedState = LED_STATE_ON;
@@ -946,7 +946,7 @@ static void SwLedControlMode1(struct _adapter *padapter,
 		if (psitesurveyctrl->traffic_busy &&
 		    check_fwstate(pmlmepriv, _FW_LINKED))
 			; /* dummy branch */
-		 else if (!pLed->bLedScanBlinkInProgress) {
+		else if (!pLed->bLedScanBlinkInProgress) {
 			if (IS_LED_WPS_BLINKING(pLed))
 				return;
 			if (pLed->bLedNoLinkBlinkInProgress) {
@@ -970,7 +970,7 @@ static void SwLedControlMode1(struct _adapter *padapter,
 				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
-		 }
+		}
 		break;
 	case LED_CTL_TX:
 	case LED_CTL_RX:
@@ -1000,7 +1000,7 @@ static void SwLedControlMode1(struct _adapter *padapter,
 
 	case LED_CTL_START_WPS: /*wait until xinpin finish */
 	case LED_CTL_START_WPS_BOTTON:
-		 if (!pLed->bLedWPSBlinkInProgress) {
+		if (!pLed->bLedWPSBlinkInProgress) {
 			if (pLed->bLedNoLinkBlinkInProgress) {
 				del_timer(&pLed->BlinkTimer);
 				pLed->bLedNoLinkBlinkInProgress = false;
@@ -1113,9 +1113,9 @@ static void SwLedControlMode2(struct _adapter *padapter,
 
 	switch (LedAction) {
 	case LED_CTL_SITE_SURVEY:
-		 if (pmlmepriv->sitesurveyctrl.traffic_busy)
+		if (pmlmepriv->sitesurveyctrl.traffic_busy)
 			; /* dummy branch */
-		 else if (!pLed->bLedScanBlinkInProgress) {
+		else if (!pLed->bLedScanBlinkInProgress) {
 			if (IS_LED_WPS_BLINKING(pLed))
 				return;
 
@@ -1132,7 +1132,7 @@ static void SwLedControlMode2(struct _adapter *padapter,
 				pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer, jiffies +
 				  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
-		 }
+		}
 		break;
 
 	case LED_CTL_TX:
@@ -1186,7 +1186,7 @@ static void SwLedControlMode2(struct _adapter *padapter,
 			pLed->BlinkingLedState = LED_STATE_ON;
 			mod_timer(&pLed->BlinkTimer,
 				  jiffies + msecs_to_jiffies(0));
-		 }
+		}
 		break;
 
 	case LED_CTL_STOP_WPS:
diff --git a/drivers/staging/rtl8712/rtl871x_cmd.c b/drivers/staging/rtl8712/rtl871x_cmd.c
index b7ee5e63af33..04638f1e4e88 100644
--- a/drivers/staging/rtl8712/rtl871x_cmd.c
+++ b/drivers/staging/rtl8712/rtl871x_cmd.c
@@ -72,8 +72,11 @@ static sint _init_cmd_priv(struct cmd_priv *pcmdpriv)
 			    ((addr_t)(pcmdpriv->cmd_allocated_buf) &
 			    (CMDBUFF_ALIGN_SZ - 1));
 	pcmdpriv->rsp_allocated_buf = kmalloc(MAX_RSPSZ + 4, GFP_ATOMIC);
-	if (!pcmdpriv->rsp_allocated_buf)
+	if (!pcmdpriv->rsp_allocated_buf) {
+		kfree(pcmdpriv->cmd_allocated_buf);
+		pcmdpriv->cmd_allocated_buf = NULL;
 		return _FAIL;
+	}
 	pcmdpriv->rsp_buf = pcmdpriv->rsp_allocated_buf  +  4 -
 			    ((addr_t)(pcmdpriv->rsp_allocated_buf) & 3);
 	pcmdpriv->cmd_issued_cnt = 0;
diff --git a/drivers/staging/rtl8712/rtl871x_ioctl_linux.c b/drivers/staging/rtl8712/rtl871x_ioctl_linux.c
index 475e7904fe45..590acb5aea3d 100644
--- a/drivers/staging/rtl8712/rtl871x_ioctl_linux.c
+++ b/drivers/staging/rtl8712/rtl871x_ioctl_linux.c
@@ -588,9 +588,9 @@ static int r871x_set_wpa_ie(struct _adapter *padapter, char *pie,
 					netdev_info(padapter->pnetdev, "r8712u: SET WPS_IE, wps_phase==true\n");
 					cnt += buf[cnt + 1] + 2;
 					break;
-				} else {
-					cnt += buf[cnt + 1] + 2;
 				}
+
+				cnt += buf[cnt + 1] + 2;
 			}
 		}
 	}
diff --git a/drivers/staging/rtl8712/rtl871x_ioctl_set.c b/drivers/staging/rtl8712/rtl871x_ioctl_set.c
index 0aaf2aab6dd0..01a150446f5a 100644
--- a/drivers/staging/rtl8712/rtl871x_ioctl_set.c
+++ b/drivers/staging/rtl8712/rtl871x_ioctl_set.c
@@ -139,9 +139,10 @@ u8 r8712_set_802_11_bssid(struct _adapter *padapter, u8 *bssid)
 		if (!memcmp(&pmlmepriv->cur_network.network.MacAddress, bssid,
 		    ETH_ALEN)) {
 			if (!check_fwstate(pmlmepriv, WIFI_STATION_STATE))
-				goto _Abort_Set_BSSID; /* driver is in
-						* WIFI_ADHOC_MASTER_STATE
-						*/
+				/* driver is in
+				 * WIFI_ADHOC_MASTER_STATE
+				 */
+				goto _Abort_Set_BSSID;
 		} else {
 			r8712_disassoc_cmd(padapter);
 			if (check_fwstate(pmlmepriv, _FW_LINKED))
@@ -203,9 +204,10 @@ void r8712_set_802_11_ssid(struct _adapter *padapter,
 							    WIFI_ADHOC_STATE);
 					}
 				} else {
-					goto _Abort_Set_SSID; /* driver is in
-						* WIFI_ADHOC_MASTER_STATE
-						*/
+					/* driver is in
+					 * WIFI_ADHOC_MASTER_STATE
+					 */
+					goto _Abort_Set_SSID;
 				}
 			}
 		} else {
diff --git a/drivers/staging/rtl8712/rtl871x_mlme.c b/drivers/staging/rtl8712/rtl871x_mlme.c
index c1feef3da26c..35cbdc71cad4 100644
--- a/drivers/staging/rtl8712/rtl871x_mlme.c
+++ b/drivers/staging/rtl8712/rtl871x_mlme.c
@@ -137,11 +137,10 @@ static void free_network_nolock(struct mlme_priv *pmlmepriv,
 }
 
 
-/*
-	return the wlan_network with the matching addr
-	Shall be called under atomic context...
-	to avoid possible racing condition...
-*/
+/* return the wlan_network with the matching addr
+ * Shall be called under atomic context...
+ * to avoid possible racing condition...
+ */
 static struct wlan_network *_r8712_find_network(struct  __queue *scanned_queue,
 					 u8 *addr)
 {
@@ -239,11 +238,10 @@ void r8712_free_network_queue(struct _adapter *dev)
 }
 
 /*
-	return the wlan_network with the matching addr
-
-	Shall be called under atomic context...
-	to avoid possible racing condition...
-*/
+ * return the wlan_network with the matching addr
+ * Shall be called under atomic context...
+ * to avoid possible racing condition...
+ */
 static struct wlan_network *r8712_find_network(struct  __queue *scanned_queue,
 					       u8 *addr)
 {
@@ -369,9 +367,7 @@ static void update_current_network(struct _adapter *adapter,
 	}
 }
 
-/*
-Caller must hold pmlmepriv->lock first.
-*/
+/* Caller must hold pmlmepriv->lock first */
 static void update_scanned_network(struct _adapter *adapter,
 			    struct wlan_bssid_ex *target)
 {
@@ -651,8 +647,8 @@ void r8712_free_assoc_resources(struct _adapter *adapter)
 }
 
 /*
-*r8712_indicate_connect: the caller has to lock pmlmepriv->lock
-*/
+ * r8712_indicate_connect: the caller has to lock pmlmepriv->lock
+ */
 void r8712_indicate_connect(struct _adapter *padapter)
 {
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
@@ -668,8 +664,8 @@ void r8712_indicate_connect(struct _adapter *padapter)
 
 
 /*
-*r8712_ind_disconnect: the caller has to lock pmlmepriv->lock
-*/
+ * r8712_ind_disconnect: the caller has to lock pmlmepriv->lock
+ */
 void r8712_ind_disconnect(struct _adapter *padapter)
 {
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
@@ -1347,8 +1343,8 @@ static int SecIsInPMKIDList(struct _adapter *Adapter, u8 *bssid)
 		   (!memcmp(psecuritypriv->PMKIDList[i].Bssid,
 			    bssid, ETH_ALEN)))
 			break;
-		else
-			i++;
+		i++;
+
 	} while (i < NUM_PMKID_CACHE);
 
 	if (i == NUM_PMKID_CACHE) {
diff --git a/drivers/staging/rtl8712/rtl871x_mlme.h b/drivers/staging/rtl8712/rtl871x_mlme.h
index ddaaab058b2f..53a23234c598 100644
--- a/drivers/staging/rtl8712/rtl871x_mlme.h
+++ b/drivers/staging/rtl8712/rtl871x_mlme.h
@@ -162,24 +162,6 @@ static inline void clr_fwstate(struct mlme_priv *pmlmepriv, sint state)
 	spin_unlock_irqrestore(&pmlmepriv->lock, irqL);
 }
 
-static inline void up_scanned_network(struct mlme_priv *pmlmepriv)
-{
-	unsigned long irqL;
-
-	spin_lock_irqsave(&pmlmepriv->lock, irqL);
-	pmlmepriv->num_of_scanned++;
-	spin_unlock_irqrestore(&pmlmepriv->lock, irqL);
-}
-
-static inline void down_scanned_network(struct mlme_priv *pmlmepriv)
-{
-	unsigned long irqL;
-
-	spin_lock_irqsave(&pmlmepriv->lock, irqL);
-	pmlmepriv->num_of_scanned--;
-	spin_unlock_irqrestore(&pmlmepriv->lock, irqL);
-}
-
 static inline void set_scanned_network_val(struct mlme_priv *pmlmepriv,
 					     sint val)
 {
diff --git a/drivers/staging/rtl8712/rtl871x_pwrctrl.c b/drivers/staging/rtl8712/rtl871x_pwrctrl.c
index d464c136dd98..e42fc1404c35 100644
--- a/drivers/staging/rtl8712/rtl871x_pwrctrl.c
+++ b/drivers/staging/rtl8712/rtl871x_pwrctrl.c
@@ -190,19 +190,15 @@ void r8712_init_pwrctrl_priv(struct _adapter *padapter)
 }
 
 /*
-Caller: r8712_cmd_thread
-
-Check if the fw_pwrstate is okay for issuing cmd.
-If not (cpwm should be is less than P2 state), then the sub-routine
-will raise the cpwm to be greater than or equal to P2.
-
-Calling Context: Passive
-
-Return Value:
-
-_SUCCESS: r8712_cmd_thread can issue cmds to firmware afterwards.
-_FAIL: r8712_cmd_thread can not do anything.
-*/
+ * Caller: r8712_cmd_thread
+ * Check if the fw_pwrstate is okay for issuing cmd.
+ * If not (cpwm should be is less than P2 state), then the sub-routine
+ * will raise the cpwm to be greater than or equal to P2.
+ * Calling Context: Passive
+ * Return Value:
+ * _SUCCESS: r8712_cmd_thread can issue cmds to firmware afterwards.
+ * _FAIL: r8712_cmd_thread can not do anything.
+ */
 sint r8712_register_cmd_alive(struct _adapter *padapter)
 {
 	uint res = _SUCCESS;
@@ -219,13 +215,11 @@ sint r8712_register_cmd_alive(struct _adapter *padapter)
 }
 
 /*
-Caller: ISR
-
-If ISR's txdone,
-No more pkts for TX,
-Then driver shall call this fun. to power down firmware again.
-*/
-
+ * Caller: ISR
+ * If ISR's txdone,
+ * No more pkts for TX,
+ * Then driver shall call this fun. to power down firmware again.
+ */
 void r8712_unregister_cmd_alive(struct _adapter *padapter)
 {
 	struct pwrctrl_priv *pwrctrl = &padapter->pwrctrlpriv;
diff --git a/drivers/staging/rtl8712/rtl871x_recv.c b/drivers/staging/rtl8712/rtl871x_recv.c
index cbd2e51ba42b..35c721a50598 100644
--- a/drivers/staging/rtl8712/rtl871x_recv.c
+++ b/drivers/staging/rtl8712/rtl871x_recv.c
@@ -125,13 +125,10 @@ union recv_frame *r8712_alloc_recvframe(struct __queue *pfree_recv_queue)
 }
 
 /*
-caller : defrag; recvframe_chk_defrag in recv_thread  (passive)
-pframequeue: defrag_queue : will be accessed in recv_thread  (passive)
-
-using spin_lock to protect
-
-*/
-
+ * caller : defrag; recvframe_chk_defrag in recv_thread  (passive)
+ * pframequeue: defrag_queue : will be accessed in recv_thread  (passive)
+ * using spin_lock to protect
+ */
 void r8712_free_recvframe_queue(struct  __queue *pframequeue,
 				struct  __queue *pfree_recv_queue)
 {
@@ -405,7 +402,7 @@ static sint ap2sta_data_frame(struct _adapter *adapter,
 		}
 
 		/* filter packets that SA is myself or multicast or broadcast */
-	       if (!memcmp(myhwaddr, pattrib->src, ETH_ALEN))
+		if (!memcmp(myhwaddr, pattrib->src, ETH_ALEN))
 			return _FAIL;
 
 		/* da should be for me */
diff --git a/drivers/staging/rtl8712/rtl871x_security.c b/drivers/staging/rtl8712/rtl871x_security.c
index 09242425dad4..a7f04a4b089d 100644
--- a/drivers/staging/rtl8712/rtl871x_security.c
+++ b/drivers/staging/rtl8712/rtl871x_security.c
@@ -159,8 +159,8 @@ static u32 getcrc32(u8 *buf, u32 len)
 }
 
 /*
-	Need to consider the fragment  situation
-*/
+ * Need to consider the fragment situation
+ */
 void r8712_wep_encrypt(struct _adapter *padapter, u8 *pxmitframe)
 {	/* exclude ICV */
 	unsigned char	crc[4];
@@ -467,22 +467,22 @@ static const unsigned short Sbox1[2][256] = {/* Sbox for hash (can be in ROM) */
 };
 
 /*
-**********************************************************************
-* Routine: Phase 1 -- generate P1K, given TA, TK, IV32
-*
-* Inputs:
-*     tk[]      = temporal key                         [128 bits]
-*     ta[]      = transmitter's MAC address            [ 48 bits]
-*     iv32      = upper 32 bits of IV                  [ 32 bits]
-* Output:
-*     p1k[]     = Phase 1 key                          [ 80 bits]
-*
-* Note:
-*     This function only needs to be called every 2**16 packets,
-*     although in theory it could be called every packet.
-*
-**********************************************************************
-*/
+ **********************************************************************
+ * Routine: Phase 1 -- generate P1K, given TA, TK, IV32
+ *
+ * Inputs:
+ *     tk[]      = temporal key                         [128 bits]
+ *     ta[]      = transmitter's MAC address            [ 48 bits]
+ *     iv32      = upper 32 bits of IV                  [ 32 bits]
+ * Output:
+ *     p1k[]     = Phase 1 key                          [ 80 bits]
+ *
+ * Note:
+ *     This function only needs to be called every 2**16 packets,
+ *     although in theory it could be called every packet.
+ *
+ **********************************************************************
+ */
 static void phase1(u16 *p1k, const u8 *tk, const u8 *ta, u32 iv32)
 {
 	sint  i;
@@ -506,28 +506,28 @@ static void phase1(u16 *p1k, const u8 *tk, const u8 *ta, u32 iv32)
 }
 
 /*
-**********************************************************************
-* Routine: Phase 2 -- generate RC4KEY, given TK, P1K, IV16
-*
-* Inputs:
-*     tk[]      = Temporal key                         [128 bits]
-*     p1k[]     = Phase 1 output key                   [ 80 bits]
-*     iv16      = low 16 bits of IV counter            [ 16 bits]
-* Output:
-*     rc4key[]  = the key used to encrypt the packet   [128 bits]
-*
-* Note:
-*     The value {TA,IV32,IV16} for Phase1/Phase2 must be unique
-*     across all packets using the same key TK value. Then, for a
-*     given value of TK[], this TKIP48 construction guarantees that
-*     the final RC4KEY value is unique across all packets.
-*
-* Suggested implementation optimization: if PPK[] is "overlaid"
-*     appropriately on RC4KEY[], there is no need for the final
-*     for loop below that copies the PPK[] result into RC4KEY[].
-*
-**********************************************************************
-*/
+ **********************************************************************
+ * Routine: Phase 2 -- generate RC4KEY, given TK, P1K, IV16
+ *
+ * Inputs:
+ *     tk[]      = Temporal key                         [128 bits]
+ *     p1k[]     = Phase 1 output key                   [ 80 bits]
+ *     iv16      = low 16 bits of IV counter            [ 16 bits]
+ * Output:
+ *     rc4key[]  = the key used to encrypt the packet   [128 bits]
+ *
+ * Note:
+ *     The value {TA,IV32,IV16} for Phase1/Phase2 must be unique
+ *     across all packets using the same key TK value. Then, for a
+ *     given value of TK[], this TKIP48 construction guarantees that
+ *     the final RC4KEY value is unique across all packets.
+ *
+ * Suggested implementation optimization: if PPK[] is "overlaid"
+ *     appropriately on RC4KEY[], there is no need for the final
+ *     for loop below that copies the PPK[] result into RC4KEY[].
+ *
+ **********************************************************************
+ */
 static void phase2(u8 *rc4key, const u8 *tk, const u16 *p1k, u16 iv16)
 {
 	sint  i;
diff --git a/drivers/staging/rtl8712/rtl871x_xmit.c b/drivers/staging/rtl8712/rtl871x_xmit.c
index be38364c8a7c..4ab82ba9bb3f 100644
--- a/drivers/staging/rtl8712/rtl871x_xmit.c
+++ b/drivers/staging/rtl8712/rtl871x_xmit.c
@@ -71,8 +71,8 @@ sint _r8712_init_xmit_priv(struct xmit_priv *pxmitpriv,
 	memset((unsigned char *)pxmitpriv, 0, sizeof(struct xmit_priv));
 	spin_lock_init(&pxmitpriv->lock);
 	/*
-	Please insert all the queue initialization using _init_queue below
-	*/
+	 *Please insert all the queue initialization using _init_queue below
+	 */
 	pxmitpriv->adapter = padapter;
 	_init_queue(&pxmitpriv->be_pending);
 	_init_queue(&pxmitpriv->bk_pending);
@@ -83,10 +83,10 @@ sint _r8712_init_xmit_priv(struct xmit_priv *pxmitpriv,
 	_init_queue(&pxmitpriv->apsd_queue);
 	_init_queue(&pxmitpriv->free_xmit_queue);
 	/*
-	Please allocate memory with the sz = (struct xmit_frame) * NR_XMITFRAME,
-	and initialize free_xmit_frame below.
-	Please also apply  free_txobj to link_up all the xmit_frames...
-	*/
+	 * Please allocate memory with the sz = (struct xmit_frame) * NR_XMITFRAME,
+	 * and initialize free_xmit_frame below.
+	 * Please also apply  free_txobj to link_up all the xmit_frames...
+	 */
 	pxmitpriv->pallocated_frame_buf = kmalloc(NR_XMITFRAME * sizeof(struct xmit_frame) + 4,
 						  GFP_ATOMIC);
 	if (!pxmitpriv->pallocated_frame_buf) {
@@ -109,8 +109,8 @@ sint _r8712_init_xmit_priv(struct xmit_priv *pxmitpriv,
 	}
 	pxmitpriv->free_xmitframe_cnt = NR_XMITFRAME;
 	/*
-		init xmit hw_txqueue
-	*/
+	 * init xmit hw_txqueue
+	 */
 	_r8712_init_hw_txqueue(&pxmitpriv->be_txqueue, BE_QUEUE_INX);
 	_r8712_init_hw_txqueue(&pxmitpriv->bk_txqueue, BK_QUEUE_INX);
 	_r8712_init_hw_txqueue(&pxmitpriv->vi_txqueue, VI_QUEUE_INX);
@@ -128,8 +128,11 @@ sint _r8712_init_xmit_priv(struct xmit_priv *pxmitpriv,
 	_init_queue(&pxmitpriv->pending_xmitbuf_queue);
 	pxmitpriv->pallocated_xmitbuf = kmalloc(NR_XMITBUFF * sizeof(struct xmit_buf) + 4,
 						GFP_ATOMIC);
-	if (!pxmitpriv->pallocated_xmitbuf)
+	if (!pxmitpriv->pallocated_xmitbuf) {
+		kfree(pxmitpriv->pallocated_frame_buf);
+		pxmitpriv->pallocated_frame_buf = NULL;
 		return _FAIL;
+	}
 	pxmitpriv->pxmitbuf = pxmitpriv->pallocated_xmitbuf + 4 -
 			      ((addr_t)(pxmitpriv->pallocated_xmitbuf) & 3);
 	pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmitbuf;
@@ -777,24 +780,23 @@ int r8712_free_xmitbuf(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf)
 }
 
 /*
-Calling context:
-1. OS_TXENTRY
-2. RXENTRY (rx_thread or RX_ISR/RX_CallBack)
-
-If we turn on USE_RXTHREAD, then, no need for critical section.
-Otherwise, we must use _enter/_exit critical to protect free_xmit_queue...
-
-Must be very very cautious...
-
-*/
-
+ * Calling context:
+ * 1. OS_TXENTRY
+ * 2. RXENTRY (rx_thread or RX_ISR/RX_CallBack)
+ *
+ * If we turn on USE_RXTHREAD, then, no need for critical section.
+ * Otherwise, we must use _enter/_exit critical to protect free_xmit_queue...
+ *
+ * Must be very very cautious...
+ *
+ */
 struct xmit_frame *r8712_alloc_xmitframe(struct xmit_priv *pxmitpriv)
 {
 	/*
-		Please remember to use all the osdep_service api,
-		and lock/unlock or _enter/_exit critical to protect
-		pfree_xmit_queue
-	*/
+	 * Please remember to use all the osdep_service api,
+	 * and lock/unlock or _enter/_exit critical to protect
+	 * pfree_xmit_queue
+	 */
 	unsigned long irqL;
 	struct xmit_frame *pxframe;
 	struct  __queue *pfree_xmit_queue = &pxmitpriv->free_xmit_queue;
diff --git a/drivers/staging/rtl8712/rtl871x_xmit.h b/drivers/staging/rtl8712/rtl871x_xmit.h
index d899d0c6d3a6..40927277f498 100644
--- a/drivers/staging/rtl8712/rtl871x_xmit.h
+++ b/drivers/staging/rtl8712/rtl871x_xmit.h
@@ -261,12 +261,6 @@ struct	xmit_priv {
 	uint free_xmitbuf_cnt;
 };
 
-static inline struct  __queue *get_free_xmit_queue(
-				struct xmit_priv *pxmitpriv)
-{
-	return &(pxmitpriv->free_xmit_queue);
-}
-
 int r8712_free_xmitbuf(struct xmit_priv *pxmitpriv,
 		       struct xmit_buf *pxmitbuf);
 struct xmit_buf *r8712_alloc_xmitbuf(struct xmit_priv *pxmitpriv);
diff --git a/drivers/staging/rts5208/ms.c b/drivers/staging/rts5208/ms.c
index f27df0b4cb44..28d56c5d1449 100644
--- a/drivers/staging/rts5208/ms.c
+++ b/drivers/staging/rts5208/ms.c
@@ -432,31 +432,36 @@ static int ms_pull_ctl_disable(struct rtsx_chip *chip)
 
 	if (CHECK_PID(chip, 0x5208)) {
 		retval = rtsx_write_register(chip, CARD_PULL_CTL1, 0xFF,
-					     MS_D1_PD | MS_D2_PD | MS_CLK_PD | MS_D6_PD);
+					     MS_D1_PD | MS_D2_PD | MS_CLK_PD |
+					     MS_D6_PD);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 		retval = rtsx_write_register(chip, CARD_PULL_CTL2, 0xFF,
-					     MS_D3_PD | MS_D0_PD | MS_BS_PD | XD_D4_PD);
+					     MS_D3_PD | MS_D0_PD | MS_BS_PD |
+					     XD_D4_PD);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 		retval = rtsx_write_register(chip, CARD_PULL_CTL3, 0xFF,
-					     MS_D7_PD | XD_CE_PD | XD_CLE_PD | XD_CD_PU);
+					     MS_D7_PD | XD_CE_PD | XD_CLE_PD |
+					     XD_CD_PU);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 		retval = rtsx_write_register(chip, CARD_PULL_CTL4, 0xFF,
-					     XD_RDY_PD | SD_D3_PD | SD_D2_PD | XD_ALE_PD);
+					     XD_RDY_PD | SD_D3_PD | SD_D2_PD |
+					     XD_ALE_PD);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 		retval = rtsx_write_register(chip, CARD_PULL_CTL5, 0xFF,
-					     MS_INS_PU | SD_WP_PD | SD_CD_PU | SD_CMD_PD);
+					     MS_INS_PU | SD_WP_PD | SD_CD_PU |
+					     SD_CMD_PD);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
@@ -507,17 +512,17 @@ static int ms_pull_ctl_enable(struct rtsx_chip *chip)
 
 	if (CHECK_PID(chip, 0x5208)) {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL1, 0xFF,
-			MS_D1_PD | MS_D2_PD | MS_CLK_NP | MS_D6_PD);
+			     MS_D1_PD | MS_D2_PD | MS_CLK_NP | MS_D6_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL2, 0xFF,
-			MS_D3_PD | MS_D0_PD | MS_BS_NP | XD_D4_PD);
+			     MS_D3_PD | MS_D0_PD | MS_BS_NP | XD_D4_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL3, 0xFF,
-			MS_D7_PD | XD_CE_PD | XD_CLE_PD | XD_CD_PU);
+			     MS_D7_PD | XD_CE_PD | XD_CLE_PD | XD_CD_PU);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL4, 0xFF,
-			XD_RDY_PD | SD_D3_PD | SD_D2_PD | XD_ALE_PD);
+			     XD_RDY_PD | SD_D3_PD | SD_D2_PD | XD_ALE_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL5, 0xFF,
-			MS_INS_PU | SD_WP_PD | SD_CD_PU | SD_CMD_PD);
+			     MS_INS_PU | SD_WP_PD | SD_CD_PU | SD_CMD_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL6, 0xFF,
-			MS_D5_PD | MS_D4_PD);
+			     MS_D5_PD | MS_D4_PD);
 	} else if (CHECK_PID(chip, 0x5288)) {
 		if (CHECK_BARO_PKG(chip, QFN)) {
 			rtsx_add_cmd(chip, WRITE_REG_CMD,
@@ -616,14 +621,20 @@ static int ms_prepare_reset(struct rtsx_chip *chip)
 
 	if (chip->asic_code) {
 		retval = rtsx_write_register(chip, MS_CFG, 0xFF,
-					     SAMPLE_TIME_RISING | PUSH_TIME_DEFAULT | NO_EXTEND_TOGGLE | MS_BUS_WIDTH_1);
+					     SAMPLE_TIME_RISING |
+					     PUSH_TIME_DEFAULT |
+					     NO_EXTEND_TOGGLE |
+					     MS_BUS_WIDTH_1);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 	} else {
 		retval = rtsx_write_register(chip, MS_CFG, 0xFF,
-					     SAMPLE_TIME_FALLING | PUSH_TIME_DEFAULT | NO_EXTEND_TOGGLE | MS_BUS_WIDTH_1);
+					     SAMPLE_TIME_FALLING |
+					     PUSH_TIME_DEFAULT |
+					     NO_EXTEND_TOGGLE |
+					     MS_BUS_WIDTH_1);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
@@ -665,7 +676,7 @@ static int ms_identify_media_type(struct rtsx_chip *chip, int switch_8bit_bus)
 
 	for (i = 0; i < MS_MAX_RETRY_COUNT; i++) {
 		retval = ms_transfer_tpc(chip, MS_TM_READ_BYTES, READ_REG,
-					6, NO_WAIT_INT);
+					 6, NO_WAIT_INT);
 		if (retval == STATUS_SUCCESS)
 			break;
 	}
@@ -765,7 +776,7 @@ static int ms_confirm_cpu_startup(struct rtsx_chip *chip)
 
 		for (i = 0; i < MS_MAX_RETRY_COUNT; i++) {
 			retval = ms_read_bytes(chip, GET_INT, 1,
-					NO_WAIT_INT, &val, 1);
+					       NO_WAIT_INT, &val, 1);
 			if (retval == STATUS_SUCCESS)
 				break;
 		}
@@ -794,9 +805,9 @@ static int ms_confirm_cpu_startup(struct rtsx_chip *chip)
 	}
 
 	if (val & INT_REG_ERR) {
-		if (val & INT_REG_CMDNK)
+		if (val & INT_REG_CMDNK) {
 			chip->card_wp |= (MS_CARD);
-		else {
+		} else {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
@@ -861,7 +872,7 @@ static int ms_switch_8bit_bus(struct rtsx_chip *chip)
 
 	for (i = 0; i < MS_MAX_RETRY_COUNT; i++) {
 		retval = ms_transfer_tpc(chip, MS_TM_READ_BYTES, GET_INT,
-					1, NO_WAIT_INT);
+					 1, NO_WAIT_INT);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -1061,8 +1072,8 @@ static int ms_read_attribute_info(struct rtsx_chip *chip)
 			return STATUS_FAIL;
 		}
 		retval = ms_transfer_data(chip, MS_TM_AUTO_READ,
-					PRO_READ_LONG_DATA, 0x40, WAIT_INT,
-					0, 0, buf, 64 * 512);
+					  PRO_READ_LONG_DATA, 0x40, WAIT_INT,
+					  0, 0, buf, 64 * 512);
 		if (retval == STATUS_SUCCESS)
 			break;
 
@@ -1087,7 +1098,7 @@ static int ms_read_attribute_info(struct rtsx_chip *chip)
 			break;
 
 		retval = ms_transfer_tpc(chip, MS_TM_NORMAL_READ,
-					PRO_READ_LONG_DATA, 0, WAIT_INT);
+					 PRO_READ_LONG_DATA, 0, WAIT_INT);
 		if (retval != STATUS_SUCCESS) {
 			kfree(buf);
 			rtsx_trace(chip);
@@ -1121,7 +1132,7 @@ static int ms_read_attribute_info(struct rtsx_chip *chip)
 
 #ifdef SUPPORT_MSXC
 		if ((buf[cur_addr_off + 8] == 0x10) ||
-			(buf[cur_addr_off + 8] == 0x13)) {
+		    (buf[cur_addr_off + 8] == 0x13)) {
 #else
 		if (buf[cur_addr_off + 8] == 0x10) {
 #endif
@@ -1264,7 +1275,7 @@ static int ms_read_attribute_info(struct rtsx_chip *chip)
 
 	if (device_type != 0x00) {
 		if ((device_type == 0x01) || (device_type == 0x02) ||
-				(device_type == 0x03)) {
+		    (device_type == 0x03)) {
 			chip->card_wp |= MS_CARD;
 		} else {
 			rtsx_trace(chip);
@@ -1298,7 +1309,7 @@ static int ms_read_attribute_info(struct rtsx_chip *chip)
 
 #ifdef SUPPORT_MAGIC_GATE
 static int mg_set_tpc_para_sub(struct rtsx_chip *chip,
-			int type, u8 mg_entry_num);
+			       int type, u8 mg_entry_num);
 #endif
 
 static int reset_ms_pro(struct rtsx_chip *chip)
@@ -1317,7 +1328,7 @@ static int reset_ms_pro(struct rtsx_chip *chip)
 #endif
 
 #ifdef XC_POWERCLASS
-Retry:
+retry:
 #endif
 	retval = ms_pro_reset_flow(chip, 1);
 	if (retval != STATUS_SUCCESS) {
@@ -1365,10 +1376,10 @@ Retry:
 				change_power_class = power_class_mode;
 			if (change_power_class) {
 				retval = msxc_change_power(chip,
-							change_power_class);
+							   change_power_class);
 				if (retval != STATUS_SUCCESS) {
 					change_power_class--;
-					goto Retry;
+					goto retry;
 				}
 			}
 		}
@@ -1418,14 +1429,14 @@ static int ms_read_status_reg(struct rtsx_chip *chip)
 }
 
 static int ms_read_extra_data(struct rtsx_chip *chip,
-		u16 block_addr, u8 page_num, u8 *buf, int buf_len)
+			      u16 block_addr, u8 page_num, u8 *buf, int buf_len)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	int retval, i;
 	u8 val, data[10];
 
 	retval = ms_set_rw_reg_addr(chip, OverwriteFlag, MS_EXTRA_SIZE,
-				SystemParm, 6);
+				    SystemParm, 6);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -1488,7 +1499,8 @@ static int ms_read_extra_data(struct rtsx_chip *chip,
 			}
 
 			retval = ms_set_rw_reg_addr(chip, OverwriteFlag,
-						MS_EXTRA_SIZE, SystemParm, 6);
+						    MS_EXTRA_SIZE, SystemParm,
+						    6);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
 				return STATUS_FAIL;
@@ -1497,7 +1509,7 @@ static int ms_read_extra_data(struct rtsx_chip *chip,
 	}
 
 	retval = ms_read_bytes(chip, READ_REG, MS_EXTRA_SIZE, NO_WAIT_INT,
-			data, MS_EXTRA_SIZE);
+			       data, MS_EXTRA_SIZE);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -1512,8 +1524,8 @@ static int ms_read_extra_data(struct rtsx_chip *chip,
 	return STATUS_SUCCESS;
 }
 
-static int ms_write_extra_data(struct rtsx_chip *chip,
-		u16 block_addr, u8 page_num, u8 *buf, int buf_len)
+static int ms_write_extra_data(struct rtsx_chip *chip, u16 block_addr,
+			       u8 page_num, u8 *buf, int buf_len)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	int retval, i;
@@ -1525,7 +1537,7 @@ static int ms_write_extra_data(struct rtsx_chip *chip,
 	}
 
 	retval = ms_set_rw_reg_addr(chip, OverwriteFlag, MS_EXTRA_SIZE,
-				SystemParm, 6 + MS_EXTRA_SIZE);
+				    SystemParm, 6 + MS_EXTRA_SIZE);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -1588,7 +1600,7 @@ static int ms_read_page(struct rtsx_chip *chip, u16 block_addr, u8 page_num)
 	u8 val, data[6];
 
 	retval = ms_set_rw_reg_addr(chip, OverwriteFlag, MS_EXTRA_SIZE,
-				SystemParm, 6);
+				    SystemParm, 6);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -1651,7 +1663,7 @@ static int ms_read_page(struct rtsx_chip *chip, u16 block_addr, u8 page_num)
 	}
 
 	retval = ms_transfer_tpc(chip, MS_TM_NORMAL_READ, READ_PAGE_DATA,
-				0, NO_WAIT_INT);
+				 0, NO_WAIT_INT);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -1678,7 +1690,7 @@ static int ms_set_bad_block(struct rtsx_chip *chip, u16 phy_blk)
 	}
 
 	retval = ms_set_rw_reg_addr(chip, OverwriteFlag, MS_EXTRA_SIZE,
-				SystemParm, 7);
+				    SystemParm, 7);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -1742,7 +1754,7 @@ static int ms_erase_block(struct rtsx_chip *chip, u16 phy_blk)
 	u8 val, data[6];
 
 	retval = ms_set_rw_reg_addr(chip, OverwriteFlag, MS_EXTRA_SIZE,
-				SystemParm, 6);
+				    SystemParm, 6);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -1844,7 +1856,7 @@ static int ms_init_page(struct rtsx_chip *chip, u16 phy_blk, u16 log_blk,
 		}
 
 		retval = ms_write_extra_data(chip, phy_blk, i,
-					extra, MS_EXTRA_SIZE);
+					     extra, MS_EXTRA_SIZE);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -1855,7 +1867,7 @@ static int ms_init_page(struct rtsx_chip *chip, u16 phy_blk, u16 log_blk,
 }
 
 static int ms_copy_page(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
-		u16 log_blk, u8 start_page, u8 end_page)
+			u16 log_blk, u8 start_page, u8 end_page)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	bool uncorrect_flag = false;
@@ -1915,7 +1927,7 @@ static int ms_copy_page(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
 		ms_read_extra_data(chip, old_blk, i, extra, MS_EXTRA_SIZE);
 
 		retval = ms_set_rw_reg_addr(chip, OverwriteFlag,
-					MS_EXTRA_SIZE, SystemParm, 6);
+					    MS_EXTRA_SIZE, SystemParm, 6);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -1971,9 +1983,9 @@ static int ms_copy_page(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
 				}
 
 				retval = ms_transfer_tpc(chip,
-							MS_TM_NORMAL_READ,
-							READ_PAGE_DATA,
-							0, NO_WAIT_INT);
+							 MS_TM_NORMAL_READ,
+							 READ_PAGE_DATA,
+							 0, NO_WAIT_INT);
 				if (retval != STATUS_SUCCESS) {
 					rtsx_trace(chip);
 					return STATUS_FAIL;
@@ -1981,20 +1993,24 @@ static int ms_copy_page(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
 
 				if (uncorrect_flag) {
 					ms_set_page_status(log_blk, setPS_NG,
-							extra, MS_EXTRA_SIZE);
+							   extra,
+							   MS_EXTRA_SIZE);
 					if (i == 0)
 						extra[0] &= 0xEF;
 
 					ms_write_extra_data(chip, old_blk, i,
-							extra, MS_EXTRA_SIZE);
+							    extra,
+							    MS_EXTRA_SIZE);
 					dev_dbg(rtsx_dev(chip), "page %d : extra[0] = 0x%x\n",
 						i, extra[0]);
 					MS_SET_BAD_BLOCK_FLG(ms_card);
 
 					ms_set_page_status(log_blk, setPS_Error,
-							extra, MS_EXTRA_SIZE);
+							   extra,
+							   MS_EXTRA_SIZE);
 					ms_write_extra_data(chip, new_blk, i,
-							extra, MS_EXTRA_SIZE);
+							    extra,
+							    MS_EXTRA_SIZE);
 					continue;
 				}
 
@@ -2021,8 +2037,8 @@ static int ms_copy_page(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
 			}
 		}
 
-		retval = ms_set_rw_reg_addr(chip, OverwriteFlag,
-				MS_EXTRA_SIZE, SystemParm, (6 + MS_EXTRA_SIZE));
+		retval = ms_set_rw_reg_addr(chip, OverwriteFlag, MS_EXTRA_SIZE,
+					    SystemParm, (6 + MS_EXTRA_SIZE));
 
 		ms_set_err_code(chip, MS_NO_ERROR);
 
@@ -2085,7 +2101,8 @@ static int ms_copy_page(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
 
 		if (i == 0) {
 			retval = ms_set_rw_reg_addr(chip, OverwriteFlag,
-						MS_EXTRA_SIZE, SystemParm, 7);
+						    MS_EXTRA_SIZE, SystemParm,
+						    7);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
 				return STATUS_FAIL;
@@ -2121,7 +2138,7 @@ static int ms_copy_page(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
 
 			ms_set_err_code(chip, MS_NO_ERROR);
 			retval = ms_read_bytes(chip, GET_INT, 1,
-					NO_WAIT_INT, &val, 1);
+					       NO_WAIT_INT, &val, 1);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
 				return STATUS_FAIL;
@@ -2361,7 +2378,7 @@ RE_SEARCH:
 		}
 
 		retval = ms_transfer_tpc(chip, MS_TM_WRITE_BYTES, WRITE_REG, 1,
-					NO_WAIT_INT);
+					 NO_WAIT_INT);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -2369,7 +2386,9 @@ RE_SEARCH:
 
 		retval = rtsx_write_register(chip, MS_CFG,
 					     0x58 | MS_NO_CHECK_INT,
-					     MS_BUS_WIDTH_4 | PUSH_TIME_ODD | MS_NO_CHECK_INT);
+					     MS_BUS_WIDTH_4 |
+					     PUSH_TIME_ODD |
+					     MS_NO_CHECK_INT);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
@@ -2474,7 +2493,7 @@ static u16 ms_get_l2p_tbl(struct rtsx_chip *chip, int seg_no, u16 log_off)
 }
 
 static void ms_set_l2p_tbl(struct rtsx_chip *chip,
-			int seg_no, u16 log_off, u16 phy_blk)
+			   int seg_no, u16 log_off, u16 phy_blk)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	struct zone_entry *segment;
@@ -2530,7 +2549,7 @@ static const unsigned short ms_start_idx[] = {0, 494, 990, 1486, 1982, 2478,
 					      7934};
 
 static int ms_arbitrate_l2p(struct rtsx_chip *chip, u16 phy_blk,
-			u16 log_off, u8 us1, u8 us2)
+			    u16 log_off, u8 us1, u8 us2)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	struct zone_entry *segment;
@@ -2627,7 +2646,8 @@ static int ms_build_l2p_tbl(struct rtsx_chip *chip, int seg_no)
 
 	disable_cnt = segment->disable_count;
 
-	segment->get_index = segment->set_index = 0;
+	segment->get_index = 0;
+	segment->set_index = 0;
 	segment->unused_blk_cnt = 0;
 
 	for (phy_blk = start; phy_blk < end; phy_blk++) {
@@ -2646,7 +2666,7 @@ static int ms_build_l2p_tbl(struct rtsx_chip *chip, int seg_no)
 		}
 
 		retval = ms_read_extra_data(chip, phy_blk, 0,
-					extra, MS_EXTRA_SIZE);
+					    extra, MS_EXTRA_SIZE);
 		if (retval != STATUS_SUCCESS) {
 			dev_dbg(rtsx_dev(chip), "read extra data fail\n");
 			ms_set_bad_block(chip, phy_blk);
@@ -2685,7 +2705,7 @@ static int ms_build_l2p_tbl(struct rtsx_chip *chip, int seg_no)
 		}
 
 		if ((log_blk < ms_start_idx[seg_no]) ||
-				(log_blk >= ms_start_idx[seg_no + 1])) {
+		    (log_blk >= ms_start_idx[seg_no + 1])) {
 			if (!(chip->card_wp & MS_CARD)) {
 				retval = ms_erase_block(chip, phy_blk);
 				if (retval != STATUS_SUCCESS)
@@ -2705,7 +2725,7 @@ static int ms_build_l2p_tbl(struct rtsx_chip *chip, int seg_no)
 		us1 = extra[0] & 0x10;
 		tmp_blk = segment->l2p_table[idx];
 		retval = ms_read_extra_data(chip, tmp_blk, 0,
-					extra, MS_EXTRA_SIZE);
+					    extra, MS_EXTRA_SIZE);
 		if (retval != STATUS_SUCCESS)
 			continue;
 		us2 = extra[0] & 0x10;
@@ -2774,7 +2794,8 @@ static int ms_build_l2p_tbl(struct rtsx_chip *chip, int seg_no)
 
 				phy_blk = ms_get_unused_block(chip, 0);
 				retval = ms_copy_page(chip, tmp_blk, phy_blk,
-						log_blk, 0, ms_card->page_off + 1);
+						      log_blk, 0,
+						      ms_card->page_off + 1);
 				if (retval != STATUS_SUCCESS) {
 					rtsx_trace(chip);
 					return STATUS_FAIL;
@@ -2861,7 +2882,7 @@ int reset_ms_card(struct rtsx_chip *chip)
 }
 
 static int mspro_set_rw_cmd(struct rtsx_chip *chip,
-			u32 start_sec, u16 sec_cnt, u8 cmd)
+			    u32 start_sec, u16 sec_cnt, u8 cmd)
 {
 	int retval, i;
 	u8 data[8];
@@ -2932,8 +2953,8 @@ static inline int ms_auto_tune_clock(struct rtsx_chip *chip)
 }
 
 static int mspro_rw_multi_sector(struct scsi_cmnd *srb,
-				struct rtsx_chip *chip, u32 start_sector,
-				u16 sector_cnt)
+				 struct rtsx_chip *chip, u32 start_sector,
+				 u16 sector_cnt)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	bool mode_2k = false;
@@ -2992,12 +3013,13 @@ static int mspro_rw_multi_sector(struct scsi_cmnd *srb,
 	}
 
 	if (ms_card->seq_mode) {
-		if ((ms_card->pre_dir != srb->sc_data_direction)
-				|| ((ms_card->pre_sec_addr + ms_card->pre_sec_cnt) != start_sector)
-				|| (mode_2k && (ms_card->seq_mode & MODE_512_SEQ))
-				|| (!mode_2k && (ms_card->seq_mode & MODE_2K_SEQ))
-				|| !(val & MS_INT_BREQ)
-				|| ((ms_card->total_sec_cnt + sector_cnt) > 0xFE00)) {
+		if ((ms_card->pre_dir != srb->sc_data_direction) ||
+		    ((ms_card->pre_sec_addr + ms_card->pre_sec_cnt) !=
+		     start_sector) ||
+		    (mode_2k && (ms_card->seq_mode & MODE_512_SEQ)) ||
+		    (!mode_2k && (ms_card->seq_mode & MODE_2K_SEQ)) ||
+		    !(val & MS_INT_BREQ) ||
+		    ((ms_card->total_sec_cnt + sector_cnt) > 0xFE00)) {
 			ms_card->seq_mode = 0;
 			ms_card->total_sec_cnt = 0;
 			if (val & MS_INT_BREQ) {
@@ -3007,7 +3029,8 @@ static int mspro_rw_multi_sector(struct scsi_cmnd *srb,
 					return STATUS_FAIL;
 				}
 
-				rtsx_write_register(chip, RBCTL, RB_FLUSH, RB_FLUSH);
+				rtsx_write_register(chip, RBCTL, RB_FLUSH,
+						    RB_FLUSH);
 			}
 		}
 	}
@@ -3038,8 +3061,8 @@ static int mspro_rw_multi_sector(struct scsi_cmnd *srb,
 	}
 
 	retval = ms_transfer_data(chip, trans_mode, rw_tpc, sector_cnt,
-				WAIT_INT, mode_2k, scsi_sg_count(srb),
-				scsi_sglist(srb), scsi_bufflen(srb));
+				  WAIT_INT, mode_2k, scsi_sg_count(srb),
+				  scsi_sglist(srb), scsi_bufflen(srb));
 	if (retval != STATUS_SUCCESS) {
 		ms_card->seq_mode = 0;
 		rtsx_read_register(chip, MS_TRANS_CFG, &val);
@@ -3076,7 +3099,7 @@ static int mspro_rw_multi_sector(struct scsi_cmnd *srb,
 }
 
 static int mspro_read_format_progress(struct rtsx_chip *chip,
-				const int short_data_len)
+				      const int short_data_len)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	int retval, i;
@@ -3102,7 +3125,8 @@ static int mspro_read_format_progress(struct rtsx_chip *chip,
 	}
 
 	if (!(tmp & MS_INT_BREQ)) {
-		if ((tmp & (MS_INT_CED | MS_INT_BREQ | MS_INT_CMDNK | MS_INT_ERR)) == MS_INT_CED) {
+		if ((tmp & (MS_INT_CED | MS_INT_BREQ | MS_INT_CMDNK |
+			    MS_INT_ERR)) == MS_INT_CED) {
 			ms_card->format_status = FORMAT_SUCCESS;
 			return STATUS_SUCCESS;
 		}
@@ -3117,7 +3141,7 @@ static int mspro_read_format_progress(struct rtsx_chip *chip,
 		cnt = (u8)short_data_len;
 
 	retval = rtsx_write_register(chip, MS_CFG, MS_NO_CHECK_INT,
-				MS_NO_CHECK_INT);
+				     MS_NO_CHECK_INT);
 	if (retval != STATUS_SUCCESS) {
 		ms_card->format_status = FORMAT_FAIL;
 		rtsx_trace(chip);
@@ -3125,7 +3149,7 @@ static int mspro_read_format_progress(struct rtsx_chip *chip,
 	}
 
 	retval = ms_read_bytes(chip, PRO_READ_SHORT_DATA, cnt, WAIT_INT,
-			data, 8);
+			       data, 8);
 	if (retval != STATUS_SUCCESS) {
 		ms_card->format_status = FORMAT_FAIL;
 		rtsx_trace(chip);
@@ -3204,7 +3228,7 @@ void mspro_polling_format_status(struct rtsx_chip *chip)
 	int i;
 
 	if (ms_card->pro_under_formatting &&
-		(rtsx_get_stat(chip) != RTSX_STAT_SS)) {
+	    (rtsx_get_stat(chip) != RTSX_STAT_SS)) {
 		rtsx_set_stat(chip, RTSX_STAT_RUN);
 
 		for (i = 0; i < 65535; i++) {
@@ -3216,7 +3240,7 @@ void mspro_polling_format_status(struct rtsx_chip *chip)
 }
 
 int mspro_format(struct scsi_cmnd *srb, struct rtsx_chip *chip,
-		int short_data_len, bool quick_format)
+		 int short_data_len, bool quick_format)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	int retval, i;
@@ -3305,9 +3329,9 @@ int mspro_format(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 }
 
 static int ms_read_multiple_pages(struct rtsx_chip *chip, u16 phy_blk,
-				u16 log_blk, u8 start_page, u8 end_page,
-				u8 *buf, unsigned int *index,
-				unsigned int *offset)
+				  u16 log_blk, u8 start_page, u8 end_page,
+				  u8 *buf, unsigned int *index,
+				  unsigned int *offset)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	int retval, i;
@@ -3315,7 +3339,7 @@ static int ms_read_multiple_pages(struct rtsx_chip *chip, u16 phy_blk,
 	u8 *ptr;
 
 	retval = ms_read_extra_data(chip, phy_blk, start_page,
-				extra, MS_EXTRA_SIZE);
+				    extra, MS_EXTRA_SIZE);
 	if (retval == STATUS_SUCCESS) {
 		if ((extra[1] & 0x30) != 0x30) {
 			ms_set_err_code(chip, MS_FLASH_READ_ERROR);
@@ -3325,7 +3349,7 @@ static int ms_read_multiple_pages(struct rtsx_chip *chip, u16 phy_blk,
 	}
 
 	retval = ms_set_rw_reg_addr(chip, OverwriteFlag, MS_EXTRA_SIZE,
-				SystemParm, 6);
+				    SystemParm, 6);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -3389,11 +3413,17 @@ static int ms_read_multiple_pages(struct rtsx_chip *chip, u16 phy_blk,
 				if (retval != STATUS_SUCCESS) {
 					if (!(chip->card_wp & MS_CARD)) {
 						reset_ms(chip);
-						ms_set_page_status(log_blk, setPS_NG, extra, MS_EXTRA_SIZE);
-						ms_write_extra_data(chip, phy_blk,
-								page_addr, extra, MS_EXTRA_SIZE);
+						ms_set_page_status
+							(log_blk, setPS_NG,
+							 extra,
+							 MS_EXTRA_SIZE);
+						ms_write_extra_data
+							(chip, phy_blk,
+							 page_addr, extra,
+							 MS_EXTRA_SIZE);
 					}
-					ms_set_err_code(chip, MS_FLASH_READ_ERROR);
+					ms_set_err_code(chip,
+							MS_FLASH_READ_ERROR);
 					rtsx_trace(chip);
 					return STATUS_FAIL;
 				}
@@ -3420,7 +3450,7 @@ static int ms_read_multiple_pages(struct rtsx_chip *chip, u16 phy_blk,
 			}
 
 			retval = ms_read_bytes(chip, GET_INT, 1, NO_WAIT_INT,
-					&val, 1);
+					       &val, 1);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
 				return STATUS_FAIL;
@@ -3441,23 +3471,24 @@ static int ms_read_multiple_pages(struct rtsx_chip *chip, u16 phy_blk,
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TPC, 0xFF, READ_PAGE_DATA);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TRANS_CFG,
-			0xFF, trans_cfg);
+			     0xFF, trans_cfg);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE,
-			0x01, RING_BUFFER);
+			     0x01, RING_BUFFER);
 
 		trans_dma_enable(DMA_FROM_DEVICE, chip, 512, DMA_512);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TRANSFER, 0xFF,
-				MS_TRANSFER_START |  MS_TM_NORMAL_READ);
+			     MS_TRANSFER_START |  MS_TM_NORMAL_READ);
 		rtsx_add_cmd(chip, CHECK_REG_CMD, MS_TRANSFER,
-			MS_TRANSFER_END, MS_TRANSFER_END);
+			     MS_TRANSFER_END, MS_TRANSFER_END);
 
 		rtsx_send_cmd_no_wait(chip);
 
-		retval = rtsx_transfer_data_partial(chip, MS_CARD, ptr,
-						512, scsi_sg_count(chip->srb),
-						index, offset, DMA_FROM_DEVICE,
-						chip->ms_timeout);
+		retval = rtsx_transfer_data_partial(chip, MS_CARD, ptr, 512,
+						    scsi_sg_count(chip->srb),
+						    index, offset,
+						    DMA_FROM_DEVICE,
+						    chip->ms_timeout);
 		if (retval < 0) {
 			if (retval == -ETIMEDOUT) {
 				ms_set_err_code(chip, MS_TO_ERROR);
@@ -3489,7 +3520,7 @@ static int ms_read_multiple_pages(struct rtsx_chip *chip, u16 phy_blk,
 }
 
 static int ms_write_multiple_pages(struct rtsx_chip *chip, u16 old_blk,
-				u16 new_blk, u16 log_blk, u8 start_page,
+				   u16 new_blk, u16 log_blk, u8 start_page,
 				u8 end_page, u8 *buf, unsigned int *index,
 				unsigned int *offset)
 {
@@ -3500,7 +3531,7 @@ static int ms_write_multiple_pages(struct rtsx_chip *chip, u16 old_blk,
 
 	if (!start_page) {
 		retval = ms_set_rw_reg_addr(chip, OverwriteFlag, MS_EXTRA_SIZE,
-					SystemParm, 7);
+					    SystemParm, 7);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -3534,7 +3565,7 @@ static int ms_write_multiple_pages(struct rtsx_chip *chip, u16 old_blk,
 
 		ms_set_err_code(chip, MS_NO_ERROR);
 		retval = ms_transfer_tpc(chip, MS_TM_READ_BYTES, GET_INT, 1,
-					NO_WAIT_INT);
+					 NO_WAIT_INT);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -3542,7 +3573,7 @@ static int ms_write_multiple_pages(struct rtsx_chip *chip, u16 old_blk,
 	}
 
 	retval = ms_set_rw_reg_addr(chip, OverwriteFlag, MS_EXTRA_SIZE,
-				SystemParm, (6 + MS_EXTRA_SIZE));
+				    SystemParm, (6 + MS_EXTRA_SIZE));
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -3630,25 +3661,26 @@ static int ms_write_multiple_pages(struct rtsx_chip *chip, u16 old_blk,
 		rtsx_init_cmd(chip);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TPC,
-			0xFF, WRITE_PAGE_DATA);
+			     0xFF, WRITE_PAGE_DATA);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TRANS_CFG,
-			0xFF, WAIT_INT);
+			     0xFF, WAIT_INT);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE,
-			0x01, RING_BUFFER);
+			     0x01, RING_BUFFER);
 
 		trans_dma_enable(DMA_TO_DEVICE, chip, 512, DMA_512);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TRANSFER, 0xFF,
-				MS_TRANSFER_START |  MS_TM_NORMAL_WRITE);
+			     MS_TRANSFER_START |  MS_TM_NORMAL_WRITE);
 		rtsx_add_cmd(chip, CHECK_REG_CMD, MS_TRANSFER,
-			MS_TRANSFER_END, MS_TRANSFER_END);
+			     MS_TRANSFER_END, MS_TRANSFER_END);
 
 		rtsx_send_cmd_no_wait(chip);
 
-		retval = rtsx_transfer_data_partial(chip, MS_CARD, ptr,
-						512, scsi_sg_count(chip->srb),
-						index, offset, DMA_TO_DEVICE,
-						chip->ms_timeout);
+		retval = rtsx_transfer_data_partial(chip, MS_CARD, ptr,	512,
+						    scsi_sg_count(chip->srb),
+						    index, offset,
+						    DMA_TO_DEVICE,
+						    chip->ms_timeout);
 		if (retval < 0) {
 			ms_set_err_code(chip, MS_TO_ERROR);
 			rtsx_clear_ms_error(chip);
@@ -3677,7 +3709,7 @@ static int ms_write_multiple_pages(struct rtsx_chip *chip, u16 old_blk,
 			if (page_addr == (end_page - 1)) {
 				if (!(val & INT_REG_CED)) {
 					retval = ms_send_cmd(chip, BLOCK_END,
-							WAIT_INT);
+							     WAIT_INT);
 					if (retval != STATUS_SUCCESS) {
 						rtsx_trace(chip);
 						return STATUS_FAIL;
@@ -3685,7 +3717,7 @@ static int ms_write_multiple_pages(struct rtsx_chip *chip, u16 old_blk,
 				}
 
 				retval = ms_read_bytes(chip, GET_INT, 1,
-						NO_WAIT_INT, &val, 1);
+						       NO_WAIT_INT, &val, 1);
 				if (retval != STATUS_SUCCESS) {
 					rtsx_trace(chip);
 					return STATUS_FAIL;
@@ -3693,7 +3725,7 @@ static int ms_write_multiple_pages(struct rtsx_chip *chip, u16 old_blk,
 			}
 
 			if ((page_addr == (end_page - 1)) ||
-				(page_addr == ms_card->page_off)) {
+			    (page_addr == ms_card->page_off)) {
 				if (!(val & INT_REG_CED)) {
 					ms_set_err_code(chip,
 							MS_FLASH_WRITE_ERROR);
@@ -3711,13 +3743,13 @@ static int ms_write_multiple_pages(struct rtsx_chip *chip, u16 old_blk,
 }
 
 static int ms_finish_write(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
-		u16 log_blk, u8 page_off)
+			   u16 log_blk, u8 page_off)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	int retval, seg_no;
 
 	retval = ms_copy_page(chip, old_blk, new_blk, log_blk,
-			page_off, ms_card->page_off + 1);
+			      page_off, ms_card->page_off + 1);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -3740,13 +3772,13 @@ static int ms_finish_write(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
 }
 
 static int ms_prepare_write(struct rtsx_chip *chip, u16 old_blk, u16 new_blk,
-		u16 log_blk, u8 start_page)
+			    u16 log_blk, u8 start_page)
 {
 	int retval;
 
 	if (start_page) {
 		retval = ms_copy_page(chip, old_blk, new_blk, log_blk,
-				0, start_page);
+				      0, start_page);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -3772,7 +3804,7 @@ int ms_delay_write(struct rtsx_chip *chip)
 
 		delay_write->delay_write_flag = 0;
 		retval = ms_finish_write(chip,
-					delay_write->old_phyblock,
+					 delay_write->old_phyblock,
 					delay_write->new_phyblock,
 					delay_write->logblock,
 					delay_write->pageoff);
@@ -3790,13 +3822,13 @@ static inline void ms_rw_fail(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
 	if (srb->sc_data_direction == DMA_FROM_DEVICE)
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+			       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 	else
 		set_sense_type(chip, SCSI_LUN(srb), SENSE_TYPE_MEDIA_WRITE_ERR);
 }
 
 static int ms_rw_multi_sector(struct scsi_cmnd *srb, struct rtsx_chip *chip,
-			u32 start_sector, u16 sector_cnt)
+			      u32 start_sector, u16 sector_cnt)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	unsigned int lun = SCSI_LUN(srb);
@@ -3843,16 +3875,17 @@ static int ms_rw_multi_sector(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 	if (srb->sc_data_direction == DMA_TO_DEVICE) {
 #ifdef MS_DELAY_WRITE
 		if (delay_write->delay_write_flag &&
-				(delay_write->logblock == log_blk) &&
-				(start_page > delay_write->pageoff)) {
+		    (delay_write->logblock == log_blk) &&
+		    (start_page > delay_write->pageoff)) {
 			delay_write->delay_write_flag = 0;
 			retval = ms_copy_page(chip,
-				delay_write->old_phyblock,
-				delay_write->new_phyblock, log_blk,
-				delay_write->pageoff, start_page);
+					      delay_write->old_phyblock,
+					      delay_write->new_phyblock,
+					      log_blk,
+					      delay_write->pageoff, start_page);
 			if (retval != STATUS_SUCCESS) {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
@@ -3868,32 +3901,35 @@ static int ms_rw_multi_sector(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 			retval = ms_delay_write(chip);
 			if (retval != STATUS_SUCCESS) {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
 #endif
-			old_blk = ms_get_l2p_tbl(chip, seg_no,
-						log_blk - ms_start_idx[seg_no]);
+			old_blk = ms_get_l2p_tbl
+					(chip, seg_no,
+					 log_blk - ms_start_idx[seg_no]);
 			new_blk  = ms_get_unused_block(chip, seg_no);
 			if ((old_blk == 0xFFFF) || (new_blk == 0xFFFF)) {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
 
 			retval = ms_prepare_write(chip, old_blk, new_blk,
-						log_blk, start_page);
+						  log_blk, start_page);
 			if (retval != STATUS_SUCCESS) {
-				if (detect_card_cd(chip, MS_CARD) != STATUS_SUCCESS) {
-					set_sense_type(chip, lun,
+				if (detect_card_cd(chip, MS_CARD) !=
+				    STATUS_SUCCESS) {
+					set_sense_type
+						(chip, lun,
 						SENSE_TYPE_MEDIA_NOT_PRESENT);
 					rtsx_trace(chip);
 					return STATUS_FAIL;
 				}
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
@@ -3906,21 +3942,21 @@ static int ms_rw_multi_sector(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 		if (retval != STATUS_SUCCESS) {
 			if (detect_card_cd(chip, MS_CARD) != STATUS_SUCCESS) {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_NOT_PRESENT);
+					       SENSE_TYPE_MEDIA_NOT_PRESENT);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
 #endif
 		old_blk = ms_get_l2p_tbl(chip, seg_no,
-					log_blk - ms_start_idx[seg_no]);
+					 log_blk - ms_start_idx[seg_no]);
 		if (old_blk == 0xFFFF) {
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
@@ -3942,19 +3978,21 @@ static int ms_rw_multi_sector(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 
 		if (srb->sc_data_direction == DMA_FROM_DEVICE) {
 			retval = ms_read_multiple_pages(chip,
-				old_blk, log_blk, start_page, end_page,
-				ptr, &index, &offset);
+							old_blk, log_blk,
+							start_page, end_page,
+							ptr, &index, &offset);
 		} else {
-			retval = ms_write_multiple_pages(chip, old_blk,
-				new_blk, log_blk, start_page, end_page,
-				ptr, &index, &offset);
+			retval = ms_write_multiple_pages(chip, old_blk, new_blk,
+							 log_blk, start_page,
+							 end_page, ptr, &index,
+							 &offset);
 		}
 
 		if (retval != STATUS_SUCCESS) {
 			toggle_gpio(chip, 1);
 			if (detect_card_cd(chip, MS_CARD) != STATUS_SUCCESS) {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_NOT_PRESENT);
+					       SENSE_TYPE_MEDIA_NOT_PRESENT);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
@@ -3970,8 +4008,8 @@ static int ms_rw_multi_sector(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 					ms_set_unused_block(chip, old_blk);
 
 				ms_set_l2p_tbl(chip, seg_no,
-					log_blk - ms_start_idx[seg_no],
-					new_blk);
+					       log_blk - ms_start_idx[seg_no],
+					       new_blk);
 			}
 		}
 
@@ -3995,14 +4033,14 @@ static int ms_rw_multi_sector(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 			if (retval != STATUS_SUCCESS) {
 				chip->card_fail |= MS_CARD;
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_NOT_PRESENT);
+					       SENSE_TYPE_MEDIA_NOT_PRESENT);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
 		}
 
 		old_blk = ms_get_l2p_tbl(chip, seg_no,
-					log_blk - ms_start_idx[seg_no]);
+					 log_blk - ms_start_idx[seg_no]);
 		if (old_blk == 0xFFFF) {
 			ms_rw_fail(srb, chip);
 			rtsx_trace(chip);
@@ -4034,10 +4072,12 @@ static int ms_rw_multi_sector(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 			delay_write->pageoff = end_page;
 #else
 			retval = ms_finish_write(chip, old_blk, new_blk,
-						log_blk, end_page);
+						 log_blk, end_page);
 			if (retval != STATUS_SUCCESS) {
-				if (detect_card_cd(chip, MS_CARD) != STATUS_SUCCESS) {
-					set_sense_type(chip, lun,
+				if (detect_card_cd(chip, MS_CARD) !=
+				    STATUS_SUCCESS) {
+					set_sense_type
+						(chip, lun,
 						SENSE_TYPE_MEDIA_NOT_PRESENT);
 					rtsx_trace(chip);
 					return STATUS_FAIL;
@@ -4057,17 +4097,17 @@ static int ms_rw_multi_sector(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 }
 
 int ms_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
-	u32 start_sector, u16 sector_cnt)
+	  u32 start_sector, u16 sector_cnt)
 {
 	struct ms_info *ms_card = &chip->ms_card;
 	int retval;
 
 	if (CHK_MSPRO(ms_card))
 		retval = mspro_rw_multi_sector(srb, chip, start_sector,
-					sector_cnt);
+					       sector_cnt);
 	else
 		retval = ms_rw_multi_sector(srb, chip, start_sector,
-					sector_cnt);
+					    sector_cnt);
 
 	return retval;
 }
@@ -4189,7 +4229,7 @@ static int mg_send_ex_cmd(struct rtsx_chip *chip, u8 cmd, u8 entry_num)
 }
 
 static int mg_set_tpc_para_sub(struct rtsx_chip *chip, int type,
-			u8 mg_entry_num)
+			       u8 mg_entry_num)
 {
 	int retval;
 	u8 buf[6];
@@ -4306,7 +4346,7 @@ int mg_get_local_EKB(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	}
 
 	retval = ms_transfer_data(chip, MS_TM_AUTO_READ, PRO_READ_LONG_DATA,
-				3, WAIT_INT, 0, 0, buf + 4, 1536);
+				  3, WAIT_INT, 0, 0, buf + 4, 1536);
 	if (retval != STATUS_SUCCESS) {
 		set_sense_type(chip, lun, SENSE_TYPE_MG_KEY_FAIL_NOT_AUTHEN);
 		rtsx_clear_ms_error(chip);
@@ -4354,7 +4394,7 @@ int mg_chg(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	}
 
 	retval = ms_read_bytes(chip, PRO_READ_SHORT_DATA, 32, WAIT_INT,
-			buf, 32);
+			       buf, 32);
 	if (retval != STATUS_SUCCESS) {
 		set_sense_type(chip, lun, SENSE_TYPE_MG_INCOMPATIBLE_MEDIUM);
 		rtsx_trace(chip);
@@ -4437,7 +4477,7 @@ int mg_get_rsp_chg(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	}
 
 	retval = ms_read_bytes(chip, PRO_READ_SHORT_DATA, 32, WAIT_INT,
-			buf1, 32);
+			       buf1, 32);
 	if (retval != STATUS_SUCCESS) {
 		set_sense_type(chip, lun, SENSE_TYPE_MG_KEY_FAIL_NOT_AUTHEN);
 		rtsx_trace(chip);
@@ -4560,7 +4600,7 @@ int mg_get_ICV(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	}
 
 	retval = ms_transfer_data(chip, MS_TM_AUTO_READ, PRO_READ_LONG_DATA,
-				2, WAIT_INT, 0, 0, buf + 4, 1024);
+				  2, WAIT_INT, 0, 0, buf + 4, 1024);
 	if (retval != STATUS_SUCCESS) {
 		set_sense_type(chip, lun, SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 		rtsx_clear_ms_error(chip);
@@ -4615,11 +4655,12 @@ int mg_set_ICV(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	if (retval != STATUS_SUCCESS) {
 		if (ms_card->mg_auth == 0) {
 			if ((buf[5] & 0xC0) != 0)
-				set_sense_type(chip, lun,
+				set_sense_type
+					(chip, lun,
 					SENSE_TYPE_MG_KEY_FAIL_NOT_ESTAB);
 			else
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MG_WRITE_ERR);
+					       SENSE_TYPE_MG_WRITE_ERR);
 		} else {
 			set_sense_type(chip, lun, SENSE_TYPE_MG_WRITE_ERR);
 		}
@@ -4634,17 +4675,17 @@ int mg_set_ICV(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		rtsx_init_cmd(chip);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TPC,
-			0xFF, PRO_WRITE_LONG_DATA);
+			     0xFF, PRO_WRITE_LONG_DATA);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TRANS_CFG, 0xFF, WAIT_INT);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE,
-			0x01, RING_BUFFER);
+			     0x01, RING_BUFFER);
 
 		trans_dma_enable(DMA_TO_DEVICE, chip, 512, DMA_512);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, MS_TRANSFER, 0xFF,
-				MS_TRANSFER_START |  MS_TM_NORMAL_WRITE);
+			     MS_TRANSFER_START |  MS_TM_NORMAL_WRITE);
 		rtsx_add_cmd(chip, CHECK_REG_CMD, MS_TRANSFER,
-			MS_TRANSFER_END, MS_TRANSFER_END);
+			     MS_TRANSFER_END, MS_TRANSFER_END);
 
 		rtsx_send_cmd_no_wait(chip);
 
@@ -4654,13 +4695,15 @@ int mg_set_ICV(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 			rtsx_clear_ms_error(chip);
 			if (ms_card->mg_auth == 0) {
 				if ((buf[5] & 0xC0) != 0)
-					set_sense_type(chip, lun, SENSE_TYPE_MG_KEY_FAIL_NOT_ESTAB);
+					set_sense_type
+					    (chip, lun,
+					     SENSE_TYPE_MG_KEY_FAIL_NOT_ESTAB);
 				else
 					set_sense_type(chip, lun,
-						SENSE_TYPE_MG_WRITE_ERR);
+						       SENSE_TYPE_MG_WRITE_ERR);
 			} else {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MG_WRITE_ERR);
+					       SENSE_TYPE_MG_WRITE_ERR);
 			}
 			retval = STATUS_FAIL;
 			rtsx_trace(chip);
@@ -4669,16 +4712,17 @@ int mg_set_ICV(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	}
 #else
 	retval = ms_transfer_data(chip, MS_TM_AUTO_WRITE, PRO_WRITE_LONG_DATA,
-				2, WAIT_INT, 0, 0, buf + 4, 1024);
+				  2, WAIT_INT, 0, 0, buf + 4, 1024);
 	if ((retval != STATUS_SUCCESS) || check_ms_err(chip)) {
 		rtsx_clear_ms_error(chip);
 		if (ms_card->mg_auth == 0) {
 			if ((buf[5] & 0xC0) != 0)
-				set_sense_type(chip, lun,
-					SENSE_TYPE_MG_KEY_FAIL_NOT_ESTAB);
+				set_sense_type
+				    (chip, lun,
+				     SENSE_TYPE_MG_KEY_FAIL_NOT_ESTAB);
 			else
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MG_WRITE_ERR);
+					       SENSE_TYPE_MG_WRITE_ERR);
 		} else {
 			set_sense_type(chip, lun, SENSE_TYPE_MG_WRITE_ERR);
 		}
@@ -4706,11 +4750,12 @@ void ms_cleanup_work(struct rtsx_chip *chip)
 		}
 		if (CHK_MSHG(ms_card)) {
 			rtsx_write_register(chip, MS_CFG,
-				MS_2K_SECTOR_MODE, 0x00);
+					    MS_2K_SECTOR_MODE, 0x00);
 		}
 	}
 #ifdef MS_DELAY_WRITE
-	else if ((!CHK_MSPRO(ms_card)) && ms_card->delay_write.delay_write_flag) {
+	else if ((!CHK_MSPRO(ms_card)) &&
+		 ms_card->delay_write.delay_write_flag) {
 		dev_dbg(rtsx_dev(chip), "MS: delay write\n");
 		ms_delay_write(chip);
 		ms_card->cleanup_counter = 0;
diff --git a/drivers/staging/rts5208/ms.h b/drivers/staging/rts5208/ms.h
index d7686399df97..71f98cc03eed 100644
--- a/drivers/staging/rts5208/ms.h
+++ b/drivers/staging/rts5208/ms.h
@@ -202,9 +202,9 @@ void mspro_polling_format_status(struct rtsx_chip *chip);
 void mspro_stop_seq_mode(struct rtsx_chip *chip);
 int reset_ms_card(struct rtsx_chip *chip);
 int ms_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
-	u32 start_sector, u16 sector_cnt);
+	  u32 start_sector, u16 sector_cnt);
 int mspro_format(struct scsi_cmnd *srb, struct rtsx_chip *chip,
-		int short_data_len, bool quick_format);
+		 int short_data_len, bool quick_format);
 void ms_free_l2p_tbl(struct rtsx_chip *chip);
 void ms_cleanup_work(struct rtsx_chip *chip);
 int ms_power_off_card3v3(struct rtsx_chip *chip);
diff --git a/drivers/staging/rts5208/rtsx.c b/drivers/staging/rts5208/rtsx.c
index 5d65a5cdc748..68d75d0d5efd 100644
--- a/drivers/staging/rts5208/rtsx.c
+++ b/drivers/staging/rts5208/rtsx.c
@@ -107,8 +107,10 @@ static int slave_configure(struct scsi_device *sdev)
 	 * the actual value or the modified one, depending on where the
 	 * data comes from.
 	 */
-	if (sdev->scsi_level < SCSI_2)
-		sdev->scsi_level = sdev->sdev_target->scsi_level = SCSI_2;
+	if (sdev->scsi_level < SCSI_2) {
+		sdev->scsi_level = SCSI_2;
+		sdev->sdev_target->scsi_level = SCSI_2;
+	}
 
 	return 0;
 }
@@ -120,12 +122,15 @@ static int slave_configure(struct scsi_device *sdev)
 /* we use this macro to help us write into the buffer */
 #undef SPRINTF
 #define SPRINTF(args...) \
-	do { if (pos < buffer+length) pos += sprintf(pos, ## args); } while (0)
+	do { \
+		if (pos < buffer + length) \
+			pos += sprintf(pos, ## args); \
+	} while (0)
 
 /* queue a command */
 /* This is always called with scsi_lock(host) held */
 static int queuecommand_lck(struct scsi_cmnd *srb,
-			void (*done)(struct scsi_cmnd *))
+			    void (*done)(struct scsi_cmnd *))
 {
 	struct rtsx_dev *dev = host_to_rtsx(srb->device->host);
 	struct rtsx_chip *chip = dev->chip;
@@ -313,7 +318,7 @@ static int rtsx_suspend(struct pci_dev *pci, pm_message_t state)
 		return 0;
 
 	/* lock the device pointers */
-	mutex_lock(&(dev->dev_mutex));
+	mutex_lock(&dev->dev_mutex);
 
 	chip = dev->chip;
 
@@ -349,7 +354,7 @@ static int rtsx_resume(struct pci_dev *pci)
 	chip = dev->chip;
 
 	/* lock the device pointers */
-	mutex_lock(&(dev->dev_mutex));
+	mutex_lock(&dev->dev_mutex);
 
 	pci_set_power_state(pci, PCI_D0);
 	pci_restore_state(pci);
@@ -418,7 +423,7 @@ static int rtsx_control_thread(void *__dev)
 			break;
 
 		/* lock the device pointers */
-		mutex_lock(&(dev->dev_mutex));
+		mutex_lock(&dev->dev_mutex);
 
 		/* if the device has disconnected, we are free to exit */
 		if (rtsx_chk_stat(chip, RTSX_STAT_DISCONNECT)) {
@@ -433,7 +438,7 @@ static int rtsx_control_thread(void *__dev)
 		/* has the command aborted ? */
 		if (rtsx_chk_stat(chip, RTSX_STAT_ABORT)) {
 			chip->srb->result = DID_ABORT << 16;
-			goto SkipForAbort;
+			goto skip_for_abort;
 		}
 
 		scsi_unlock(host);
@@ -480,12 +485,12 @@ static int rtsx_control_thread(void *__dev)
 		else if (chip->srb->result != DID_ABORT << 16) {
 			chip->srb->scsi_done(chip->srb);
 		} else {
-SkipForAbort:
+skip_for_abort:
 			dev_err(&dev->pci->dev, "scsi command aborted\n");
 		}
 
 		if (rtsx_chk_stat(chip, RTSX_STAT_ABORT)) {
-			complete(&(dev->notify));
+			complete(&dev->notify);
 
 			rtsx_set_stat(chip, RTSX_STAT_IDLE);
 		}
@@ -519,9 +524,9 @@ static int rtsx_polling_thread(void *__dev)
 {
 	struct rtsx_dev *dev = __dev;
 	struct rtsx_chip *chip = dev->chip;
-	struct sd_info *sd_card = &(chip->sd_card);
-	struct xd_info *xd_card = &(chip->xd_card);
-	struct ms_info *ms_card = &(chip->ms_card);
+	struct sd_info *sd_card = &chip->sd_card;
+	struct xd_info *xd_card = &chip->xd_card;
+	struct ms_info *ms_card = &chip->ms_card;
 
 	sd_card->cleanup_counter = 0;
 	xd_card->cleanup_counter = 0;
@@ -531,12 +536,11 @@ static int rtsx_polling_thread(void *__dev)
 	wait_timeout((delay_use + 5) * 1000);
 
 	for (;;) {
-
 		set_current_state(TASK_INTERRUPTIBLE);
 		schedule_timeout(msecs_to_jiffies(POLLING_INTERVAL));
 
 		/* lock the device pointers */
-		mutex_lock(&(dev->dev_mutex));
+		mutex_lock(&dev->dev_mutex);
 
 		/* if the device has disconnected, we are free to exit */
 		if (rtsx_chk_stat(chip, RTSX_STAT_DISCONNECT)) {
@@ -550,7 +554,7 @@ static int rtsx_polling_thread(void *__dev)
 		mspro_polling_format_status(chip);
 
 		/* lock the device pointers */
-		mutex_lock(&(dev->dev_mutex));
+		mutex_lock(&dev->dev_mutex);
 
 		rtsx_polling_func(chip);
 
@@ -597,7 +601,7 @@ static irqreturn_t rtsx_interrupt(int irq, void *dev_id)
 			dev->trans_result = TRANS_RESULT_FAIL;
 			if (dev->done)
 				complete(dev->done);
-			goto Exit;
+			goto exit;
 		}
 	}
 
@@ -619,7 +623,7 @@ static irqreturn_t rtsx_interrupt(int irq, void *dev_id)
 		}
 	}
 
-Exit:
+exit:
 	spin_unlock(&dev->reg_lock);
 	return IRQ_HANDLED;
 }
@@ -724,9 +728,10 @@ static int rtsx_scan_thread(void *__dev)
 		dev_info(&dev->pci->dev,
 			 "%s: waiting for device to settle before scanning\n",
 			 CR_DRIVER_NAME);
-		wait_event_interruptible_timeout(dev->delay_wait,
-				rtsx_chk_stat(chip, RTSX_STAT_DISCONNECT),
-				delay_use * HZ);
+		wait_event_interruptible_timeout
+			(dev->delay_wait,
+			 rtsx_chk_stat(chip, RTSX_STAT_DISCONNECT),
+			 delay_use * HZ);
 	}
 
 	/* If the device is still connected, perform the scanning */
@@ -844,7 +849,7 @@ static void rtsx_init_options(struct rtsx_chip *chip)
 }
 
 static int rtsx_probe(struct pci_dev *pci,
-				const struct pci_device_id *pci_id)
+		      const struct pci_device_id *pci_id)
 {
 	struct Scsi_Host *host;
 	struct rtsx_dev *dev;
@@ -879,18 +884,18 @@ static int rtsx_probe(struct pci_dev *pci,
 	dev = host_to_rtsx(host);
 	memset(dev, 0, sizeof(struct rtsx_dev));
 
-	dev->chip = kzalloc(sizeof(struct rtsx_chip), GFP_KERNEL);
+	dev->chip = kzalloc(sizeof(*dev->chip), GFP_KERNEL);
 	if (!dev->chip) {
 		err = -ENOMEM;
 		goto errout;
 	}
 
 	spin_lock_init(&dev->reg_lock);
-	mutex_init(&(dev->dev_mutex));
+	mutex_init(&dev->dev_mutex);
 	init_completion(&dev->cmnd_ready);
 	init_completion(&dev->control_exit);
 	init_completion(&dev->polling_exit);
-	init_completion(&(dev->notify));
+	init_completion(&dev->notify);
 	init_completion(&dev->scanning_done);
 	init_waitqueue_head(&dev->delay_wait);
 
diff --git a/drivers/staging/rts5208/rtsx.h b/drivers/staging/rts5208/rtsx.h
index e725b10ed087..575e5734f2a5 100644
--- a/drivers/staging/rts5208/rtsx.h
+++ b/drivers/staging/rts5208/rtsx.h
@@ -149,7 +149,7 @@ static inline void get_current_time(u8 *timeval_buf, int buf_len)
 
 	getnstimeofday64(&ts64);
 
-	tv_usec = ts64.tv_nsec/NSEC_PER_USEC;
+	tv_usec = ts64.tv_nsec / NSEC_PER_USEC;
 
 	timeval_buf[0] = (u8)(ts64.tv_sec >> 24);
 	timeval_buf[1] = (u8)(ts64.tv_sec >> 16);
diff --git a/drivers/staging/rts5208/rtsx_card.c b/drivers/staging/rts5208/rtsx_card.c
index 97717744962d..a6b7bffc6714 100644
--- a/drivers/staging/rts5208/rtsx_card.c
+++ b/drivers/staging/rts5208/rtsx_card.c
@@ -33,11 +33,11 @@
 
 void do_remaining_work(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 #ifdef XD_DELAY_WRITE
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 #endif
-	struct ms_info *ms_card = &(chip->ms_card);
+	struct ms_info *ms_card = &chip->ms_card;
 
 	if (chip->card_ready & SD_CARD) {
 		if (sd_card->seq_mode) {
@@ -100,9 +100,9 @@ void try_to_switch_sdio_ctrl(struct rtsx_chip *chip)
 	if ((reg1 & 0xC0) && (reg2 & 0xC0)) {
 		chip->sd_int = 1;
 		rtsx_write_register(chip, SDIO_CTRL, 0xFF,
-				SDIO_BUS_CTRL | SDIO_CD_CTRL);
+				    SDIO_BUS_CTRL | SDIO_CD_CTRL);
 		rtsx_write_register(chip, PWR_GATE_CTRL,
-				LDO3318_PWR_MASK, LDO_ON);
+				    LDO3318_PWR_MASK, LDO_ON);
 	}
 }
 
@@ -133,7 +133,7 @@ void dynamic_configure_sdio_aspm(struct rtsx_chip *chip)
 		if (!chip->sdio_aspm) {
 			dev_dbg(rtsx_dev(chip), "SDIO enter ASPM!\n");
 			rtsx_write_register(chip, ASPM_FORCE_CTL, 0xFC,
-					0x30 | (chip->aspm_level[1] << 2));
+					    0x30 | (chip->aspm_level[1] << 2));
 			chip->sdio_aspm = 1;
 		}
 	} else {
@@ -154,7 +154,7 @@ void do_reset_sd_card(struct rtsx_chip *chip)
 		chip->sd_reset_counter, chip->card2lun[SD_CARD]);
 
 	if (chip->card2lun[SD_CARD] >= MAX_ALLOWED_LUN_CNT) {
-		clear_bit(SD_NR, &(chip->need_reset));
+		clear_bit(SD_NR, &chip->need_reset);
 		chip->sd_reset_counter = 0;
 		chip->sd_show_cnt = 0;
 		return;
@@ -169,7 +169,7 @@ void do_reset_sd_card(struct rtsx_chip *chip)
 	if (chip->need_release & SD_CARD)
 		return;
 	if (retval == STATUS_SUCCESS) {
-		clear_bit(SD_NR, &(chip->need_reset));
+		clear_bit(SD_NR, &chip->need_reset);
 		chip->sd_reset_counter = 0;
 		chip->sd_show_cnt = 0;
 		chip->card_ready |= SD_CARD;
@@ -177,7 +177,7 @@ void do_reset_sd_card(struct rtsx_chip *chip)
 		chip->rw_card[chip->card2lun[SD_CARD]] = sd_rw;
 	} else {
 		if (chip->sd_io || (chip->sd_reset_counter >= MAX_RESET_CNT)) {
-			clear_bit(SD_NR, &(chip->need_reset));
+			clear_bit(SD_NR, &chip->need_reset);
 			chip->sd_reset_counter = 0;
 			chip->sd_show_cnt = 0;
 		} else {
@@ -208,7 +208,7 @@ void do_reset_xd_card(struct rtsx_chip *chip)
 		chip->xd_reset_counter, chip->card2lun[XD_CARD]);
 
 	if (chip->card2lun[XD_CARD] >= MAX_ALLOWED_LUN_CNT) {
-		clear_bit(XD_NR, &(chip->need_reset));
+		clear_bit(XD_NR, &chip->need_reset);
 		chip->xd_reset_counter = 0;
 		chip->xd_show_cnt = 0;
 		return;
@@ -223,14 +223,14 @@ void do_reset_xd_card(struct rtsx_chip *chip)
 	if (chip->need_release & XD_CARD)
 		return;
 	if (retval == STATUS_SUCCESS) {
-		clear_bit(XD_NR, &(chip->need_reset));
+		clear_bit(XD_NR, &chip->need_reset);
 		chip->xd_reset_counter = 0;
 		chip->card_ready |= XD_CARD;
 		chip->card_fail &= ~XD_CARD;
 		chip->rw_card[chip->card2lun[XD_CARD]] = xd_rw;
 	} else {
 		if (chip->xd_reset_counter >= MAX_RESET_CNT) {
-			clear_bit(XD_NR, &(chip->need_reset));
+			clear_bit(XD_NR, &chip->need_reset);
 			chip->xd_reset_counter = 0;
 			chip->xd_show_cnt = 0;
 		} else {
@@ -256,7 +256,7 @@ void do_reset_ms_card(struct rtsx_chip *chip)
 		chip->ms_reset_counter, chip->card2lun[MS_CARD]);
 
 	if (chip->card2lun[MS_CARD] >= MAX_ALLOWED_LUN_CNT) {
-		clear_bit(MS_NR, &(chip->need_reset));
+		clear_bit(MS_NR, &chip->need_reset);
 		chip->ms_reset_counter = 0;
 		chip->ms_show_cnt = 0;
 		return;
@@ -271,14 +271,14 @@ void do_reset_ms_card(struct rtsx_chip *chip)
 	if (chip->need_release & MS_CARD)
 		return;
 	if (retval == STATUS_SUCCESS) {
-		clear_bit(MS_NR, &(chip->need_reset));
+		clear_bit(MS_NR, &chip->need_reset);
 		chip->ms_reset_counter = 0;
 		chip->card_ready |= MS_CARD;
 		chip->card_fail &= ~MS_CARD;
 		chip->rw_card[chip->card2lun[MS_CARD]] = ms_rw;
 	} else {
 		if (chip->ms_reset_counter >= MAX_RESET_CNT) {
-			clear_bit(MS_NR, &(chip->need_reset));
+			clear_bit(MS_NR, &chip->need_reset);
 			chip->ms_reset_counter = 0;
 			chip->ms_show_cnt = 0;
 		} else {
@@ -300,7 +300,7 @@ static void release_sdio(struct rtsx_chip *chip)
 {
 	if (chip->sd_io) {
 		rtsx_write_register(chip, CARD_STOP, SD_STOP | SD_CLR_ERR,
-				SD_STOP | SD_CLR_ERR);
+				    SD_STOP | SD_CLR_ERR);
 
 		if (chip->chip_insert_with_sdio) {
 			chip->chip_insert_with_sdio = 0;
@@ -369,7 +369,7 @@ void rtsx_reset_cards(struct rtsx_chip *chip)
 	rtsx_disable_aspm(chip);
 
 	if ((chip->need_reset & SD_CARD) && chip->chip_insert_with_sdio)
-		clear_bit(SD_NR, &(chip->need_reset));
+		clear_bit(SD_NR, &chip->need_reset);
 
 	if (chip->need_reset & XD_CARD) {
 		chip->card_exist |= XD_CARD;
@@ -381,8 +381,8 @@ void rtsx_reset_cards(struct rtsx_chip *chip)
 	}
 	if (CHECK_PID(chip, 0x5288) && CHECK_BARO_PKG(chip, QFN)) {
 		if (chip->card_exist & XD_CARD) {
-			clear_bit(SD_NR, &(chip->need_reset));
-			clear_bit(MS_NR, &(chip->need_reset));
+			clear_bit(SD_NR, &chip->need_reset);
+			clear_bit(MS_NR, &chip->need_reset);
 		}
 	}
 	if (chip->need_reset & SD_CARD) {
@@ -449,7 +449,7 @@ void rtsx_reinit_cards(struct rtsx_chip *chip, int reset_chip)
 
 #ifdef DISABLE_CARD_INT
 void card_cd_debounce(struct rtsx_chip *chip, unsigned long *need_reset,
-		unsigned long *need_release)
+		      unsigned long *need_release)
 {
 	u8 release_map = 0, reset_map = 0;
 
@@ -502,13 +502,13 @@ void card_cd_debounce(struct rtsx_chip *chip, unsigned long *need_reset,
 
 		reset_map = 0;
 		if (!(chip->card_exist & XD_CARD) &&
-				(xd_cnt > (DEBOUNCE_CNT-1)))
+		    (xd_cnt > (DEBOUNCE_CNT - 1)))
 			reset_map |= XD_CARD;
 		if (!(chip->card_exist & SD_CARD) &&
-				(sd_cnt > (DEBOUNCE_CNT-1)))
+		    (sd_cnt > (DEBOUNCE_CNT - 1)))
 			reset_map |= SD_CARD;
 		if (!(chip->card_exist & MS_CARD) &&
-				(ms_cnt > (DEBOUNCE_CNT-1)))
+		    (ms_cnt > (DEBOUNCE_CNT - 1)))
 			reset_map |= MS_CARD;
 	}
 
@@ -531,23 +531,23 @@ void rtsx_init_cards(struct rtsx_chip *chip)
 	}
 
 #ifdef DISABLE_CARD_INT
-	card_cd_debounce(chip, &(chip->need_reset), &(chip->need_release));
+	card_cd_debounce(chip, &chip->need_reset, &chip->need_release);
 #endif
 
 	if (chip->need_release) {
 		if (CHECK_PID(chip, 0x5288) && CHECK_BARO_PKG(chip, QFN)) {
 			if (chip->int_reg & XD_EXIST) {
-				clear_bit(SD_NR, &(chip->need_release));
-				clear_bit(MS_NR, &(chip->need_release));
+				clear_bit(SD_NR, &chip->need_release);
+				clear_bit(MS_NR, &chip->need_release);
 			}
 		}
 
 		if (!(chip->card_exist & SD_CARD) && !chip->sd_io)
-			clear_bit(SD_NR, &(chip->need_release));
+			clear_bit(SD_NR, &chip->need_release);
 		if (!(chip->card_exist & XD_CARD))
-			clear_bit(XD_NR, &(chip->need_release));
+			clear_bit(XD_NR, &chip->need_release);
 		if (!(chip->card_exist & MS_CARD))
-			clear_bit(MS_NR, &(chip->need_release));
+			clear_bit(MS_NR, &chip->need_release);
 
 		dev_dbg(rtsx_dev(chip), "chip->need_release = 0x%x\n",
 			(unsigned int)(chip->need_release));
@@ -556,8 +556,10 @@ void rtsx_init_cards(struct rtsx_chip *chip)
 		if (chip->need_release) {
 			if (chip->ocp_stat & (CARD_OC_NOW | CARD_OC_EVER))
 				rtsx_write_register(chip, OCPCLR,
-						CARD_OC_INT_CLR | CARD_OC_CLR,
-						CARD_OC_INT_CLR | CARD_OC_CLR);
+						    CARD_OC_INT_CLR |
+						    CARD_OC_CLR,
+						    CARD_OC_INT_CLR |
+						    CARD_OC_CLR);
 			chip->ocp_stat = 0;
 		}
 #endif
@@ -567,7 +569,7 @@ void rtsx_init_cards(struct rtsx_chip *chip)
 		}
 
 		if (chip->need_release & SD_CARD) {
-			clear_bit(SD_NR, &(chip->need_release));
+			clear_bit(SD_NR, &chip->need_release);
 			chip->card_exist &= ~SD_CARD;
 			chip->card_ejected &= ~SD_CARD;
 			chip->card_fail &= ~SD_CARD;
@@ -580,7 +582,7 @@ void rtsx_init_cards(struct rtsx_chip *chip)
 		}
 
 		if (chip->need_release & XD_CARD) {
-			clear_bit(XD_NR, &(chip->need_release));
+			clear_bit(XD_NR, &chip->need_release);
 			chip->card_exist &= ~XD_CARD;
 			chip->card_ejected &= ~XD_CARD;
 			chip->card_fail &= ~XD_CARD;
@@ -590,13 +592,13 @@ void rtsx_init_cards(struct rtsx_chip *chip)
 			release_xd_card(chip);
 
 			if (CHECK_PID(chip, 0x5288) &&
-					CHECK_BARO_PKG(chip, QFN))
+			    CHECK_BARO_PKG(chip, QFN))
 				rtsx_write_register(chip, HOST_SLEEP_STATE,
-						0xC0, 0xC0);
+						    0xC0, 0xC0);
 		}
 
 		if (chip->need_release & MS_CARD) {
-			clear_bit(MS_NR, &(chip->need_release));
+			clear_bit(MS_NR, &chip->need_release);
 			chip->card_exist &= ~MS_CARD;
 			chip->card_ejected &= ~MS_CARD;
 			chip->card_fail &= ~MS_CARD;
@@ -650,7 +652,7 @@ int switch_ssc_clock(struct rtsx_chip *chip, int clk)
 		return STATUS_FAIL;
 	}
 
-	mcu_cnt = (u8)(125/clk + 3);
+	mcu_cnt = (u8)(125 / clk + 3);
 	if (mcu_cnt > 7)
 		mcu_cnt = 7;
 
@@ -681,9 +683,9 @@ int switch_ssc_clock(struct rtsx_chip *chip, int clk)
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SSC_CTL1, SSC_RSTB, SSC_RSTB);
 	if (sd_vpclk_phase_reset) {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SD_VPCLK0_CTL,
-			PHASE_NOT_RESET, 0);
+			     PHASE_NOT_RESET, 0);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SD_VPCLK0_CTL,
-			PHASE_NOT_RESET, PHASE_NOT_RESET);
+			     PHASE_NOT_RESET, PHASE_NOT_RESET);
 	}
 
 	retval = rtsx_send_cmd(chip, 0, WAIT_TIME);
@@ -850,7 +852,7 @@ int switch_normal_clock(struct rtsx_chip *chip, int clk)
 }
 
 void trans_dma_enable(enum dma_data_direction dir, struct rtsx_chip *chip,
-		u32 byte_cnt, u8 pack_size)
+		      u32 byte_cnt, u8 pack_size)
 {
 	if (pack_size > DMA_1024)
 		pack_size = DMA_512;
@@ -864,11 +866,11 @@ void trans_dma_enable(enum dma_data_direction dir, struct rtsx_chip *chip,
 
 	if (dir == DMA_FROM_DEVICE) {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, DMACTL,
-			0x03 | DMA_PACK_SIZE_MASK,
+			     0x03 | DMA_PACK_SIZE_MASK,
 			     DMA_DIR_FROM_CARD | DMA_EN | pack_size);
 	} else {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, DMACTL,
-			0x03 | DMA_PACK_SIZE_MASK,
+			     0x03 | DMA_PACK_SIZE_MASK,
 			     DMA_DIR_TO_CARD | DMA_EN | pack_size);
 	}
 
@@ -978,13 +980,13 @@ int card_power_off(struct rtsx_chip *chip, u8 card)
 }
 
 int card_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
-	u32 sec_addr, u16 sec_cnt)
+	    u32 sec_addr, u16 sec_cnt)
 {
 	int retval;
 	unsigned int lun = SCSI_LUN(srb);
 	int i;
 
-	if (chip->rw_card[lun] == NULL) {
+	if (!chip->rw_card[lun]) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
 	}
@@ -1115,7 +1117,7 @@ void turn_on_led(struct rtsx_chip *chip, u8 gpio)
 {
 	if (CHECK_PID(chip, 0x5288))
 		rtsx_write_register(chip, CARD_GPIO, (u8)(1 << gpio),
-				(u8)(1 << gpio));
+				    (u8)(1 << gpio));
 	else
 		rtsx_write_register(chip, CARD_GPIO, (u8)(1 << gpio), 0);
 }
@@ -1126,7 +1128,7 @@ void turn_off_led(struct rtsx_chip *chip, u8 gpio)
 		rtsx_write_register(chip, CARD_GPIO, (u8)(1 << gpio), 0);
 	else
 		rtsx_write_register(chip, CARD_GPIO, (u8)(1 << gpio),
-				(u8)(1 << gpio));
+				    (u8)(1 << gpio));
 }
 
 int detect_card_cd(struct rtsx_chip *chip, int card)
diff --git a/drivers/staging/rts5208/rtsx_card.h b/drivers/staging/rts5208/rtsx_card.h
index 56df9a431d6d..aa37705bae39 100644
--- a/drivers/staging/rts5208/rtsx_card.h
+++ b/drivers/staging/rts5208/rtsx_card.h
@@ -1011,9 +1011,9 @@ int switch_normal_clock(struct rtsx_chip *chip, int clk);
 int enable_card_clock(struct rtsx_chip *chip, u8 card);
 int disable_card_clock(struct rtsx_chip *chip, u8 card);
 int card_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
-	u32 sec_addr, u16 sec_cnt);
+	    u32 sec_addr, u16 sec_cnt);
 void trans_dma_enable(enum dma_data_direction dir,
-		struct rtsx_chip *chip, u32 byte_cnt, u8 pack_size);
+		      struct rtsx_chip *chip, u32 byte_cnt, u8 pack_size);
 void toggle_gpio(struct rtsx_chip *chip, u8 gpio);
 void turn_on_led(struct rtsx_chip *chip, u8 gpio);
 void turn_off_led(struct rtsx_chip *chip, u8 gpio);
@@ -1030,10 +1030,10 @@ u8 get_lun_card(struct rtsx_chip *chip, unsigned int lun);
 static inline u32 get_card_size(struct rtsx_chip *chip, unsigned int lun)
 {
 #ifdef SUPPORT_SD_LOCK
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 
 	if ((get_lun_card(chip, lun) == SD_CARD) &&
-		(sd_card->sd_lock_status & SD_LOCKED))
+	    (sd_card->sd_lock_status & SD_LOCKED))
 		return 0;
 
 	return chip->capacity[lun];
@@ -1073,25 +1073,25 @@ static inline int card_power_off_all(struct rtsx_chip *chip)
 static inline void rtsx_clear_xd_error(struct rtsx_chip *chip)
 {
 	rtsx_write_register(chip, CARD_STOP, XD_STOP | XD_CLR_ERR,
-			XD_STOP | XD_CLR_ERR);
+			    XD_STOP | XD_CLR_ERR);
 }
 
 static inline void rtsx_clear_sd_error(struct rtsx_chip *chip)
 {
 	rtsx_write_register(chip, CARD_STOP, SD_STOP | SD_CLR_ERR,
-			SD_STOP | SD_CLR_ERR);
+			    SD_STOP | SD_CLR_ERR);
 }
 
 static inline void rtsx_clear_ms_error(struct rtsx_chip *chip)
 {
 	rtsx_write_register(chip, CARD_STOP, MS_STOP | MS_CLR_ERR,
-			MS_STOP | MS_CLR_ERR);
+			    MS_STOP | MS_CLR_ERR);
 }
 
 static inline void rtsx_clear_spi_error(struct rtsx_chip *chip)
 {
 	rtsx_write_register(chip, CARD_STOP, SPI_STOP | SPI_CLR_ERR,
-			SPI_STOP | SPI_CLR_ERR);
+			    SPI_STOP | SPI_CLR_ERR);
 }
 
 #ifdef SUPPORT_SDIO_ASPM
diff --git a/drivers/staging/rts5208/rtsx_chip.c b/drivers/staging/rts5208/rtsx_chip.c
index a10dd6220a7b..3511157a2c78 100644
--- a/drivers/staging/rts5208/rtsx_chip.c
+++ b/drivers/staging/rts5208/rtsx_chip.c
@@ -114,7 +114,8 @@ static int rtsx_pre_handle_sdio_old(struct rtsx_chip *chip)
 		if (chip->asic_code) {
 			retval = rtsx_write_register(chip, CARD_PULL_CTL5,
 						     0xFF,
-						     MS_INS_PU | SD_WP_PU | SD_CD_PU | SD_CMD_PU);
+						     MS_INS_PU | SD_WP_PU |
+						     SD_CD_PU | SD_CMD_PU);
 			if (retval) {
 				rtsx_trace(chip);
 				return retval;
@@ -240,10 +241,10 @@ static int rtsx_pre_handle_sdio_new(struct rtsx_chip *chip)
 					return STATUS_FAIL;
 				}
 			} else {
-				retval = rtsx_write_register(chip,
-							     FPGA_PULL_CTL,
-							     FPGA_SD_PULL_CTL_BIT | 0x20,
-							     0);
+				retval = rtsx_write_register
+						(chip, FPGA_PULL_CTL,
+						 FPGA_SD_PULL_CTL_BIT | 0x20,
+						 0);
 				if (retval) {
 					rtsx_trace(chip);
 					return retval;
@@ -713,7 +714,8 @@ nextcard:
 
 	if (chip->ft2_fast_mode) {
 		retval = rtsx_write_register(chip, CARD_PWR_CTL, 0xFF,
-					     MS_PARTIAL_POWER_ON | SD_PARTIAL_POWER_ON);
+					     MS_PARTIAL_POWER_ON |
+					     SD_PARTIAL_POWER_ON);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
@@ -1567,7 +1569,8 @@ int rtsx_write_cfg_dw(struct rtsx_chip *chip, u8 func_no, u16 addr, u32 mask,
 		}
 
 		retval = rtsx_write_register(chip, CFGRWCTL, 0xFF,
-					     0x80 | mode | ((func_no & 0x03) << 4));
+					     0x80 | mode |
+					     ((func_no & 0x03) << 4));
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
diff --git a/drivers/staging/rts5208/rtsx_chip.h b/drivers/staging/rts5208/rtsx_chip.h
index f36642817c6e..4f6e3c1c4621 100644
--- a/drivers/staging/rts5208/rtsx_chip.h
+++ b/drivers/staging/rts5208/rtsx_chip.h
@@ -130,16 +130,20 @@
 #define PRDCT_REV_LEN           4               /* Product LOT Length       */
 
 /* Dynamic flag definitions: used in set_bit() etc. */
-#define RTSX_FLIDX_TRANS_ACTIVE		18  /* 0x00040000 transfer is active	 */
-#define RTSX_FLIDX_ABORTING		20  /* 0x00100000 abort is in progress	 */
-#define RTSX_FLIDX_DISCONNECTING	21  /* 0x00200000 disconnect in progress */
+/* 0x00040000 transfer is active */
+#define RTSX_FLIDX_TRANS_ACTIVE		18
+/* 0x00100000 abort is in progress */
+#define RTSX_FLIDX_ABORTING		20
+/* 0x00200000 disconnect in progress */
+#define RTSX_FLIDX_DISCONNECTING	21
 
 #define ABORTING_OR_DISCONNECTING	((1UL << US_FLIDX_ABORTING) | \
 					 (1UL << US_FLIDX_DISCONNECTING))
 
-#define RTSX_FLIDX_RESETTING		22  /* 0x00400000 device reset in progress */
-#define RTSX_FLIDX_TIMED_OUT		23  /* 0x00800000 SCSI midlayer timed out  */
-
+/* 0x00400000 device reset in progress */
+#define RTSX_FLIDX_RESETTING		22
+/* 0x00800000 SCSI midlayer timed out  */
+#define RTSX_FLIDX_TIMED_OUT		23
 #define DRCT_ACCESS_DEV         0x00    /* Direct Access Device      */
 #define RMB_DISC                0x80    /* The Device is Removable   */
 #define ANSI_SCSI2              0x02    /* Based on ANSI-SCSI2       */
@@ -285,23 +289,24 @@ struct sense_data_t {
 
 #define CARD_INT		(XD_INT | MS_INT | SD_INT)
 #define NEED_COMPLETE_INT	(DATA_DONE_INT | TRANS_OK_INT | TRANS_FAIL_INT)
-#define RTSX_INT		(CMD_DONE_INT | NEED_COMPLETE_INT | CARD_INT | GPIO0_INT | OC_INT)
+#define RTSX_INT		(CMD_DONE_INT | NEED_COMPLETE_INT | CARD_INT | \
+				 GPIO0_INT | OC_INT)
 
 #define CARD_EXIST		(XD_EXIST | MS_EXIST | SD_EXIST)
 
 /* Bus interrupt enable register */
-#define CMD_DONE_INT_EN		(1 << 31)
-#define DATA_DONE_INT_EN	(1 << 30)
-#define TRANS_OK_INT_EN		(1 << 29)
-#define TRANS_FAIL_INT_EN	(1 << 28)
-#define XD_INT_EN		(1 << 27)
-#define MS_INT_EN		(1 << 26)
-#define SD_INT_EN		(1 << 25)
-#define GPIO0_INT_EN		(1 << 24)
-#define OC_INT_EN		(1 << 23)
+#define CMD_DONE_INT_EN		BIT(31)
+#define DATA_DONE_INT_EN	BIT(30)
+#define TRANS_OK_INT_EN		BIT(29)
+#define TRANS_FAIL_INT_EN	BIT(28)
+#define XD_INT_EN		BIT(27)
+#define MS_INT_EN		BIT(26)
+#define SD_INT_EN		BIT(25)
+#define GPIO0_INT_EN		BIT(24)
+#define OC_INT_EN		BIT(23)
 #define DELINK_INT_EN		GPIO0_INT_EN
-#define MS_OC_INT_EN		(1 << 23)
-#define SD_OC_INT_EN		(1 << 22)
+#define MS_OC_INT_EN		BIT(23)
+#define SD_OC_INT_EN		BIT(22)
 
 #define READ_REG_CMD		0
 #define WRITE_REG_CMD		1
@@ -318,10 +323,10 @@ struct sense_data_t {
 #define MS_NR		3
 #define XD_NR		4
 #define SPI_NR		7
-#define SD_CARD		(1 << SD_NR)
-#define MS_CARD		(1 << MS_NR)
-#define XD_CARD		(1 << XD_NR)
-#define SPI_CARD	(1 << SPI_NR)
+#define SD_CARD		BIT(SD_NR)
+#define MS_CARD		BIT(MS_NR)
+#define XD_CARD		BIT(XD_NR)
+#define SPI_CARD	BIT(SPI_NR)
 
 #define MAX_ALLOWED_LUN_CNT	8
 
@@ -393,14 +398,23 @@ struct zone_entry {
 
 /* SD card */
 #define CHK_SD(sd_card)			(((sd_card)->sd_type & 0xFF) == TYPE_SD)
-#define CHK_SD_HS(sd_card)		(CHK_SD(sd_card) && ((sd_card)->sd_type & SD_HS))
-#define CHK_SD_SDR50(sd_card)		(CHK_SD(sd_card) && ((sd_card)->sd_type & SD_SDR50))
-#define CHK_SD_DDR50(sd_card)		(CHK_SD(sd_card) && ((sd_card)->sd_type & SD_DDR50))
-#define CHK_SD_SDR104(sd_card)		(CHK_SD(sd_card) && ((sd_card)->sd_type & SD_SDR104))
-#define CHK_SD_HCXC(sd_card)		(CHK_SD(sd_card) && ((sd_card)->sd_type & SD_HCXC))
-#define CHK_SD_HC(sd_card)		(CHK_SD_HCXC(sd_card) && ((sd_card)->capacity <= 0x4000000))
-#define CHK_SD_XC(sd_card)		(CHK_SD_HCXC(sd_card) && ((sd_card)->capacity > 0x4000000))
-#define CHK_SD30_SPEED(sd_card)		(CHK_SD_SDR50(sd_card) || CHK_SD_DDR50(sd_card) || CHK_SD_SDR104(sd_card))
+#define CHK_SD_HS(sd_card)		(CHK_SD(sd_card) && \
+					 ((sd_card)->sd_type & SD_HS))
+#define CHK_SD_SDR50(sd_card)		(CHK_SD(sd_card) && \
+					 ((sd_card)->sd_type & SD_SDR50))
+#define CHK_SD_DDR50(sd_card)		(CHK_SD(sd_card) && \
+					 ((sd_card)->sd_type & SD_DDR50))
+#define CHK_SD_SDR104(sd_card)		(CHK_SD(sd_card) && \
+					 ((sd_card)->sd_type & SD_SDR104))
+#define CHK_SD_HCXC(sd_card)		(CHK_SD(sd_card) && \
+					 ((sd_card)->sd_type & SD_HCXC))
+#define CHK_SD_HC(sd_card)		(CHK_SD_HCXC(sd_card) && \
+					 ((sd_card)->capacity <= 0x4000000))
+#define CHK_SD_XC(sd_card)		(CHK_SD_HCXC(sd_card) && \
+					 ((sd_card)->capacity > 0x4000000))
+#define CHK_SD30_SPEED(sd_card)		(CHK_SD_SDR50(sd_card) || \
+					 CHK_SD_DDR50(sd_card) || \
+					 CHK_SD_SDR104(sd_card))
 
 #define SET_SD(sd_card)			((sd_card)->sd_type = TYPE_SD)
 #define SET_SD_HS(sd_card)		((sd_card)->sd_type |= SD_HS)
@@ -416,13 +430,20 @@ struct zone_entry {
 #define CLR_SD_HCXC(sd_card)		((sd_card)->sd_type &= ~SD_HCXC)
 
 /* MMC card */
-#define CHK_MMC(sd_card)		(((sd_card)->sd_type & 0xFF) == TYPE_MMC)
-#define CHK_MMC_26M(sd_card)		(CHK_MMC(sd_card) && ((sd_card)->sd_type & MMC_26M))
-#define CHK_MMC_52M(sd_card)		(CHK_MMC(sd_card) && ((sd_card)->sd_type & MMC_52M))
-#define CHK_MMC_4BIT(sd_card)		(CHK_MMC(sd_card) && ((sd_card)->sd_type & MMC_4BIT))
-#define CHK_MMC_8BIT(sd_card)		(CHK_MMC(sd_card) && ((sd_card)->sd_type & MMC_8BIT))
-#define CHK_MMC_SECTOR_MODE(sd_card)	(CHK_MMC(sd_card) && ((sd_card)->sd_type & MMC_SECTOR_MODE))
-#define CHK_MMC_DDR52(sd_card)		(CHK_MMC(sd_card) && ((sd_card)->sd_type & MMC_DDR52))
+#define CHK_MMC(sd_card)		(((sd_card)->sd_type & 0xFF) == \
+					 TYPE_MMC)
+#define CHK_MMC_26M(sd_card)		(CHK_MMC(sd_card) && \
+					 ((sd_card)->sd_type & MMC_26M))
+#define CHK_MMC_52M(sd_card)		(CHK_MMC(sd_card) && \
+					 ((sd_card)->sd_type & MMC_52M))
+#define CHK_MMC_4BIT(sd_card)		(CHK_MMC(sd_card) && \
+					 ((sd_card)->sd_type & MMC_4BIT))
+#define CHK_MMC_8BIT(sd_card)		(CHK_MMC(sd_card) && \
+					 ((sd_card)->sd_type & MMC_8BIT))
+#define CHK_MMC_SECTOR_MODE(sd_card)	(CHK_MMC(sd_card) && \
+					 ((sd_card)->sd_type & MMC_SECTOR_MODE))
+#define CHK_MMC_DDR52(sd_card)		(CHK_MMC(sd_card) && \
+					 ((sd_card)->sd_type & MMC_DDR52))
 
 #define SET_MMC(sd_card)		((sd_card)->sd_type = TYPE_MMC)
 #define SET_MMC_26M(sd_card)		((sd_card)->sd_type |= MMC_26M)
@@ -439,7 +460,8 @@ struct zone_entry {
 #define CLR_MMC_SECTOR_MODE(sd_card)	((sd_card)->sd_type &= ~MMC_SECTOR_MODE)
 #define CLR_MMC_DDR52(sd_card)		((sd_card)->sd_type &= ~MMC_DDR52)
 
-#define CHK_MMC_HS(sd_card)		(CHK_MMC_52M(sd_card) && CHK_MMC_26M(sd_card))
+#define CHK_MMC_HS(sd_card)		(CHK_MMC_52M(sd_card) && \
+					 CHK_MMC_26M(sd_card))
 #define CLR_MMC_HS(sd_card)			\
 do {						\
 	CLR_MMC_DDR52(sd_card);			\
@@ -450,12 +472,18 @@ do {						\
 #define SD_SUPPORT_CLASS_TEN		0x01
 #define SD_SUPPORT_1V8			0x02
 
-#define SD_SET_CLASS_TEN(sd_card)	((sd_card)->sd_setting |= SD_SUPPORT_CLASS_TEN)
-#define SD_CHK_CLASS_TEN(sd_card)	((sd_card)->sd_setting & SD_SUPPORT_CLASS_TEN)
-#define SD_CLR_CLASS_TEN(sd_card)	((sd_card)->sd_setting &= ~SD_SUPPORT_CLASS_TEN)
-#define SD_SET_1V8(sd_card)		((sd_card)->sd_setting |= SD_SUPPORT_1V8)
-#define SD_CHK_1V8(sd_card)		((sd_card)->sd_setting & SD_SUPPORT_1V8)
-#define SD_CLR_1V8(sd_card)		((sd_card)->sd_setting &= ~SD_SUPPORT_1V8)
+#define SD_SET_CLASS_TEN(sd_card)	((sd_card)->sd_setting |= \
+					 SD_SUPPORT_CLASS_TEN)
+#define SD_CHK_CLASS_TEN(sd_card)	((sd_card)->sd_setting & \
+					 SD_SUPPORT_CLASS_TEN)
+#define SD_CLR_CLASS_TEN(sd_card)	((sd_card)->sd_setting &= \
+					 ~SD_SUPPORT_CLASS_TEN)
+#define SD_SET_1V8(sd_card)		((sd_card)->sd_setting |= \
+					 SD_SUPPORT_1V8)
+#define SD_CHK_1V8(sd_card)		((sd_card)->sd_setting & \
+					 SD_SUPPORT_1V8)
+#define SD_CLR_1V8(sd_card)		((sd_card)->sd_setting &= \
+					 ~SD_SUPPORT_1V8)
 
 struct sd_info {
 	u16 sd_type;
@@ -544,9 +572,12 @@ struct xd_info {
 #define HG8BIT			(MS_HG | MS_8BIT)
 
 #define CHK_MSPRO(ms_card)	(((ms_card)->ms_type & 0xFF) == TYPE_MSPRO)
-#define CHK_HG8BIT(ms_card)	(CHK_MSPRO(ms_card) && (((ms_card)->ms_type & HG8BIT) == HG8BIT))
-#define CHK_MSXC(ms_card)	(CHK_MSPRO(ms_card) && ((ms_card)->ms_type & MS_XC))
-#define CHK_MSHG(ms_card)	(CHK_MSPRO(ms_card) && ((ms_card)->ms_type & MS_HG))
+#define CHK_HG8BIT(ms_card)	(CHK_MSPRO(ms_card) && \
+				 (((ms_card)->ms_type & HG8BIT) == HG8BIT))
+#define CHK_MSXC(ms_card)	(CHK_MSPRO(ms_card) && \
+				 ((ms_card)->ms_type & MS_XC))
+#define CHK_MSHG(ms_card)	(CHK_MSPRO(ms_card) && \
+				 ((ms_card)->ms_type & MS_HG))
 
 #define CHK_MS8BIT(ms_card)	(((ms_card)->ms_type & MS_8BIT))
 #define CHK_MS4BIT(ms_card)	(((ms_card)->ms_type & MS_4BIT))
@@ -679,8 +710,10 @@ struct trace_msg_t {
 #define CLR_SDIO_EXIST(chip)		((chip)->sdio_func_exist &= ~SDIO_EXIST)
 
 #define CHK_SDIO_IGNORED(chip)		((chip)->sdio_func_exist & SDIO_IGNORED)
-#define SET_SDIO_IGNORED(chip)		((chip)->sdio_func_exist |= SDIO_IGNORED)
-#define CLR_SDIO_IGNORED(chip)		((chip)->sdio_func_exist &= ~SDIO_IGNORED)
+#define SET_SDIO_IGNORED(chip)		((chip)->sdio_func_exist |= \
+					 SDIO_IGNORED)
+#define CLR_SDIO_IGNORED(chip)		((chip)->sdio_func_exist &= \
+					 ~SDIO_IGNORED)
 
 struct rtsx_chip {
 	struct rtsx_dev	*rtsx;
@@ -957,12 +990,12 @@ void rtsx_stop_cmd(struct rtsx_chip *chip, int card);
 int rtsx_write_register(struct rtsx_chip *chip, u16 addr, u8 mask, u8 data);
 int rtsx_read_register(struct rtsx_chip *chip, u16 addr, u8 *data);
 int rtsx_write_cfg_dw(struct rtsx_chip *chip,
-		u8 func_no, u16 addr, u32 mask, u32 val);
+		      u8 func_no, u16 addr, u32 mask, u32 val);
 int rtsx_read_cfg_dw(struct rtsx_chip *chip, u8 func_no, u16 addr, u32 *val);
 int rtsx_write_cfg_seq(struct rtsx_chip *chip,
-		u8 func, u16 addr, u8 *buf, int len);
+		       u8 func, u16 addr, u8 *buf, int len);
 int rtsx_read_cfg_seq(struct rtsx_chip *chip,
-		u8 func, u16 addr, u8 *buf, int len);
+		      u8 func, u16 addr, u8 *buf, int len);
 int rtsx_write_phy_register(struct rtsx_chip *chip, u8 addr, u16 val);
 int rtsx_read_phy_register(struct rtsx_chip *chip, u8 addr, u16 *val);
 int rtsx_read_efuse(struct rtsx_chip *chip, u8 addr, u8 *val);
diff --git a/drivers/staging/rts5208/rtsx_scsi.c b/drivers/staging/rts5208/rtsx_scsi.c
index becb4bba166c..a95c5de1aa00 100644
--- a/drivers/staging/rts5208/rtsx_scsi.c
+++ b/drivers/staging/rts5208/rtsx_scsi.c
@@ -354,7 +354,7 @@ void set_sense_type(struct rtsx_chip *chip, unsigned int lun, int sense_type)
 
 	case SENSE_TYPE_MEDIA_INVALID_CMD_FIELD:
 		set_sense_data(chip, lun, CUR_ERR, ILGAL_REQ, 0,
-				ASC_INVLD_CDB, ASCQ_INVLD_CDB, CDB_ILLEGAL, 1);
+			       ASC_INVLD_CDB, ASCQ_INVLD_CDB, CDB_ILLEGAL, 1);
 		break;
 
 	case SENSE_TYPE_FORMAT_IN_PROGRESS:
@@ -397,10 +397,10 @@ void set_sense_type(struct rtsx_chip *chip, unsigned int lun, int sense_type)
 }
 
 void set_sense_data(struct rtsx_chip *chip, unsigned int lun, u8 err_code,
-		u8 sense_key, u32 info, u8 asc, u8 ascq, u8 sns_key_info0,
+		    u8 sense_key, u32 info, u8 asc, u8 ascq, u8 sns_key_info0,
 		u16 sns_key_info1)
 {
-	struct sense_data_t *sense = &(chip->sense_buffer[lun]);
+	struct sense_data_t *sense = &chip->sense_buffer[lun];
 
 	sense->err_code = err_code;
 	sense->sense_key = sense_key;
@@ -436,7 +436,7 @@ static int test_unit_ready(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 #ifdef SUPPORT_SD_LOCK
 	if (get_lun_card(chip, SCSI_LUN(srb)) == SD_CARD) {
-		struct sd_info *sd_card = &(chip->sd_card);
+		struct sd_info *sd_card = &chip->sd_card;
 
 		if (sd_card->sd_lock_notify) {
 			sd_card->sd_lock_notify = 0;
@@ -444,7 +444,7 @@ static int test_unit_ready(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 			return TRANSPORT_FAILED;
 		} else if (sd_card->sd_lock_status & SD_LOCKED) {
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_READ_FORBIDDEN);
+				       SENSE_TYPE_MEDIA_READ_FORBIDDEN);
 			return TRANSPORT_FAILED;
 		}
 	}
@@ -514,7 +514,7 @@ static int inquiry(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 #ifdef SUPPORT_MAGIC_GATE
 	if ((chip->mspro_formatter_enable) &&
-			(chip->lun2card[lun] & MS_CARD))
+	    (chip->lun2card[lun] & MS_CARD))
 #else
 	if (chip->mspro_formatter_enable)
 #endif
@@ -603,7 +603,7 @@ static int allow_medium_removal(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	if (prevent) {
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+			       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -615,13 +615,13 @@ static int request_sense(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
 	struct sense_data_t *sense;
 	unsigned int lun = SCSI_LUN(srb);
-	struct ms_info *ms_card = &(chip->ms_card);
+	struct ms_info *ms_card = &chip->ms_card;
 	unsigned char *tmp, *buf;
 
-	sense = &(chip->sense_buffer[lun]);
+	sense = &chip->sense_buffer[lun];
 
 	if ((get_lun_card(chip, lun) == MS_CARD) &&
-		ms_card->pro_under_formatting) {
+	    ms_card->pro_under_formatting) {
 		if (ms_card->format_status == FORMAT_SUCCESS) {
 			set_sense_type(chip, lun, SENSE_TYPE_NO_SENSE);
 			ms_card->pro_under_formatting = 0;
@@ -629,7 +629,7 @@ static int request_sense(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		} else if (ms_card->format_status == FORMAT_IN_PROGRESS) {
 			/* Logical Unit Not Ready Format in Progress */
 			set_sense_data(chip, lun, CUR_ERR, 0x02, 0, 0x04, 0x04,
-					0, (u16)(ms_card->progress));
+				       0, (u16)(ms_card->progress));
 		} else {
 			/* Format Command Failed */
 			set_sense_type(chip, lun, SENSE_TYPE_FORMAT_CMD_FAILED);
@@ -659,9 +659,9 @@ static int request_sense(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 }
 
 static void ms_mode_sense(struct rtsx_chip *chip, u8 cmd,
-		int lun, u8 *buf, int buf_len)
+			  int lun, u8 *buf, int buf_len)
 {
-	struct ms_info *ms_card = &(chip->ms_card);
+	struct ms_info *ms_card = &chip->ms_card;
 	int sys_info_offset;
 	int data_size = buf_len;
 	bool support_format = false;
@@ -754,10 +754,10 @@ static void ms_mode_sense(struct rtsx_chip *chip, u8 cmd,
 static int mode_sense(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
 	unsigned int lun = SCSI_LUN(srb);
-	unsigned int dataSize;
+	unsigned int data_size;
 	int status;
 	bool pro_formatter_flag;
-	unsigned char pageCode, *buf;
+	unsigned char page_code, *buf;
 	u8 card = get_lun_card(chip, lun);
 
 #ifndef SUPPORT_MAGIC_GATE
@@ -770,11 +770,11 @@ static int mode_sense(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 #endif
 
 	pro_formatter_flag = false;
-	dataSize = 8;
+	data_size = 8;
 #ifdef SUPPORT_MAGIC_GATE
 	if ((chip->lun2card[lun] & MS_CARD)) {
 		if (!card || (card == MS_CARD)) {
-			dataSize = 108;
+			data_size = 108;
 			if (chip->mspro_formatter_enable)
 				pro_formatter_flag = true;
 		}
@@ -783,28 +783,28 @@ static int mode_sense(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	if (card == MS_CARD) {
 		if (chip->mspro_formatter_enable) {
 			pro_formatter_flag = true;
-			dataSize = 108;
+			data_size = 108;
 		}
 	}
 #endif
 
-	buf = kmalloc(dataSize, GFP_KERNEL);
+	buf = kmalloc(data_size, GFP_KERNEL);
 	if (!buf) {
 		rtsx_trace(chip);
 		return TRANSPORT_ERROR;
 	}
 
-	pageCode = srb->cmnd[2] & 0x3f;
+	page_code = srb->cmnd[2] & 0x3f;
 
-	if ((pageCode == 0x3F) || (pageCode == 0x1C) ||
-		(pageCode == 0x00) ||
-		(pro_formatter_flag && (pageCode == 0x20))) {
+	if ((page_code == 0x3F) || (page_code == 0x1C) ||
+	    (page_code == 0x00) ||
+		(pro_formatter_flag && (page_code == 0x20))) {
 		if (srb->cmnd[0] == MODE_SENSE) {
-			if ((pageCode == 0x3F) || (pageCode == 0x20)) {
+			if ((page_code == 0x3F) || (page_code == 0x20)) {
 				ms_mode_sense(chip, srb->cmnd[0],
-					      lun, buf, dataSize);
+					      lun, buf, data_size);
 			} else {
-				dataSize = 4;
+				data_size = 4;
 				buf[0] = 0x03;
 				buf[1] = 0x00;
 				if (check_card_wp(chip, lun))
@@ -815,11 +815,11 @@ static int mode_sense(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 				buf[3] = 0x00;
 			}
 		} else {
-			if ((pageCode == 0x3F) || (pageCode == 0x20)) {
+			if ((page_code == 0x3F) || (page_code == 0x20)) {
 				ms_mode_sense(chip, srb->cmnd[0],
-					      lun, buf, dataSize);
+					      lun, buf, data_size);
 			} else {
-				dataSize = 8;
+				data_size = 8;
 				buf[0] = 0x00;
 				buf[1] = 0x06;
 				buf[2] = 0x00;
@@ -842,7 +842,7 @@ static int mode_sense(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	if (status == TRANSPORT_GOOD) {
 		unsigned int len = min_t(unsigned int, scsi_bufflen(srb),
-					dataSize);
+					data_size);
 		rtsx_stor_set_xfer_buf(buf, len, srb);
 		scsi_set_resid(srb, scsi_bufflen(srb) - len);
 	}
@@ -854,7 +854,7 @@ static int mode_sense(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 static int read_write(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
 #ifdef SUPPORT_SD_LOCK
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 #endif
 	unsigned int lun = SCSI_LUN(srb);
 	int retval;
@@ -896,7 +896,7 @@ static int read_write(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		if (sd_card->sd_lock_status & SD_LOCKED) {
 			dev_dbg(rtsx_dev(chip), "SD card locked!\n");
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_READ_FORBIDDEN);
+				       SENSE_TYPE_MEDIA_READ_FORBIDDEN);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -932,7 +932,7 @@ static int read_write(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	 * need to judge start_sec at first
 	 */
 	if ((start_sec > get_card_size(chip, lun)) ||
-			((start_sec + sec_cnt) > get_card_size(chip, lun))) {
+	    ((start_sec + sec_cnt) > get_card_size(chip, lun))) {
 		set_sense_type(chip, lun, SENSE_TYPE_MEDIA_LBA_OVER_RANGE);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
@@ -947,7 +947,7 @@ static int read_write(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		dev_dbg(rtsx_dev(chip), "read/write fail three times in succession\n");
 		if (srb->sc_data_direction == DMA_FROM_DEVICE)
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 		else
 			set_sense_type(chip, lun, SENSE_TYPE_MEDIA_WRITE_ERR);
 
@@ -959,7 +959,7 @@ static int read_write(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		if (check_card_wp(chip, lun)) {
 			dev_dbg(rtsx_dev(chip), "Write protected card!\n");
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_WRITE_PROTECT);
+				       SENSE_TYPE_MEDIA_WRITE_PROTECT);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -973,15 +973,16 @@ static int read_write(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		} else {
 			chip->rw_fail_cnt[lun]++;
 			if (srb->sc_data_direction == DMA_FROM_DEVICE)
-				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				set_sense_type
+					(chip, lun,
+					 SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 			else
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 		}
 		retval = TRANSPORT_FAILED;
 		rtsx_trace(chip);
-		goto Exit;
+		goto exit;
 	} else {
 		chip->rw_fail_cnt[lun] = 0;
 		retval = TRANSPORT_GOOD;
@@ -989,7 +990,7 @@ static int read_write(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	scsi_set_resid(srb, 0);
 
-Exit:
+exit:
 	return retval;
 }
 
@@ -1025,8 +1026,8 @@ static int read_format_capacity(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	/* Capacity List Length */
 	if ((buf_len > 12) && chip->mspro_formatter_enable &&
-			(chip->lun2card[lun] & MS_CARD) &&
-			(!card || (card == MS_CARD))) {
+	    (chip->lun2card[lun] & MS_CARD) &&
+	    (!card || (card == MS_CARD))) {
 		buf[i++] = 0x10;
 		desc_cnt = 2;
 	} else {
@@ -1143,7 +1144,7 @@ static int read_eeprom(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	if (retval != STATUS_SUCCESS) {
 		vfree(buf);
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+			       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -1153,7 +1154,7 @@ static int read_eeprom(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		if (retval != STATUS_SUCCESS) {
 			vfree(buf);
 			set_sense_type(chip, SCSI_LUN(srb),
-				SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -1195,7 +1196,7 @@ static int write_eeprom(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		retval = spi_erase_eeprom_chip(chip);
 		if (retval != STATUS_SUCCESS) {
 			set_sense_type(chip, SCSI_LUN(srb),
-				SENSE_TYPE_MEDIA_WRITE_ERR);
+				       SENSE_TYPE_MEDIA_WRITE_ERR);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -1216,7 +1217,7 @@ static int write_eeprom(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 			if (retval != STATUS_SUCCESS) {
 				vfree(buf);
 				set_sense_type(chip, SCSI_LUN(srb),
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 				rtsx_trace(chip);
 				return TRANSPORT_FAILED;
 			}
@@ -1247,7 +1248,7 @@ static int read_mem(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	if (addr < 0xFC00) {
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+			       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -1271,7 +1272,7 @@ static int read_mem(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		if (retval != STATUS_SUCCESS) {
 			vfree(buf);
 			set_sense_type(chip, SCSI_LUN(srb),
-				SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -1305,7 +1306,7 @@ static int write_mem(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	if (addr < 0xFC00) {
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+			       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -1333,7 +1334,7 @@ static int write_mem(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		if (retval != STATUS_SUCCESS) {
 			vfree(buf);
 			set_sense_type(chip, SCSI_LUN(srb),
-				SENSE_TYPE_MEDIA_WRITE_ERR);
+				       SENSE_TYPE_MEDIA_WRITE_ERR);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -1346,7 +1347,7 @@ static int write_mem(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 static int get_sd_csd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	unsigned int lun = SCSI_LUN(srb);
 
 	if (!check_card_ready(chip, lun)) {
@@ -1399,7 +1400,7 @@ static int trace_msg_cmd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	if ((scsi_bufflen(srb) < buf_len) || !scsi_sglist(srb)) {
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+			       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -1522,9 +1523,9 @@ static int set_variable(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	if (srb->cmnd[3] == 1) {
 		/* Variable Clock */
-		struct xd_info *xd_card = &(chip->xd_card);
-		struct sd_info *sd_card = &(chip->sd_card);
-		struct ms_info *ms_card = &(chip->ms_card);
+		struct xd_info *xd_card = &chip->xd_card;
+		struct sd_info *sd_card = &chip->sd_card;
+		struct ms_info *ms_card = &chip->ms_card;
 
 		switch (srb->cmnd[4]) {
 		case XD_CARD:
@@ -1541,7 +1542,7 @@ static int set_variable(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		default:
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+				       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -1556,7 +1557,7 @@ static int set_variable(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 			rtsx_disable_aspm(chip);
 
 			if (chip->ss_en &&
-				(rtsx_get_stat(chip) == RTSX_STAT_SS)) {
+			    (rtsx_get_stat(chip) == RTSX_STAT_SS)) {
 				rtsx_exit_ss(chip);
 				wait_timeout(100);
 			}
@@ -1565,7 +1566,7 @@ static int set_variable(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 			retval = rtsx_force_power_on(chip, SSC_PDCTL);
 			if (retval != STATUS_SUCCESS) {
 				set_sense_type(chip, SCSI_LUN(srb),
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 				rtsx_trace(chip);
 				return TRANSPORT_FAILED;
 			}
@@ -1586,9 +1587,9 @@ static int get_variable(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	unsigned int lun = SCSI_LUN(srb);
 
 	if (srb->cmnd[3] == 1) {
-		struct xd_info *xd_card = &(chip->xd_card);
-		struct sd_info *sd_card = &(chip->sd_card);
-		struct ms_info *ms_card = &(chip->ms_card);
+		struct xd_info *xd_card = &chip->xd_card;
+		struct sd_info *sd_card = &chip->sd_card;
+		struct ms_info *ms_card = &chip->ms_card;
 		u8 tmp;
 
 		switch (srb->cmnd[4]) {
@@ -1606,7 +1607,7 @@ static int get_variable(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		default:
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+				       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -1648,14 +1649,15 @@ static int dma_access_ring_buffer(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		dev_dbg(rtsx_dev(chip), "Write to device\n");
 
 	retval = rtsx_transfer_data(chip, 0, scsi_sglist(srb), len,
-			scsi_sg_count(srb), srb->sc_data_direction, 1000);
+				    scsi_sg_count(srb), srb->sc_data_direction,
+				    1000);
 	if (retval < 0) {
 		if (srb->sc_data_direction == DMA_FROM_DEVICE)
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 		else
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_WRITE_ERR);
+				       SENSE_TYPE_MEDIA_WRITE_ERR);
 
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
@@ -1667,8 +1669,8 @@ static int dma_access_ring_buffer(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 static int get_dev_status(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
-	struct ms_info *ms_card = &(chip->ms_card);
+	struct sd_info *sd_card = &chip->sd_card;
+	struct ms_info *ms_card = &chip->ms_card;
 	int buf_len;
 	unsigned int lun = SCSI_LUN(srb);
 	u8 card = get_lun_card(chip, lun);
@@ -1699,8 +1701,8 @@ static int get_dev_status(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 #ifdef SUPPORT_OCP
 	status[8] = 0;
-	if (CHECK_LUN_MODE(chip,
-		SD_MS_2LUN) && (chip->lun2card[lun] == MS_CARD)) {
+	if (CHECK_LUN_MODE(chip, SD_MS_2LUN) &&
+	    (chip->lun2card[lun] == MS_CARD)) {
 		oc_now_mask = MS_OC_NOW;
 		oc_ever_mask = MS_OC_EVER;
 	} else {
@@ -1804,7 +1806,7 @@ static int set_chip_mode(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	if (!CHECK_PID(chip, 0x5208)) {
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+			       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -1884,7 +1886,7 @@ static int rw_mem_cmd_buf(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		cmd_type = srb->cmnd[4];
 		if (cmd_type > 2) {
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+				       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -1903,7 +1905,7 @@ static int rw_mem_cmd_buf(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		value = *(rtsx_get_cmd_data(chip) + idx);
 		if (scsi_bufflen(srb) < 1) {
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+				       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -1971,7 +1973,7 @@ static int read_phy_register(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		if (retval != STATUS_SUCCESS) {
 			vfree(buf);
 			set_sense_type(chip, SCSI_LUN(srb),
-				SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -1980,8 +1982,9 @@ static int read_phy_register(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 			retval = rtsx_read_phy_register(chip, addr + i, &val);
 			if (retval != STATUS_SUCCESS) {
 				vfree(buf);
-				set_sense_type(chip, SCSI_LUN(srb),
-					SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				set_sense_type
+					(chip, SCSI_LUN(srb),
+					 SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 				rtsx_trace(chip);
 				return TRANSPORT_FAILED;
 			}
@@ -2039,7 +2042,7 @@ static int write_phy_register(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		if (retval != STATUS_SUCCESS) {
 			vfree(buf);
 			set_sense_type(chip, SCSI_LUN(srb),
-				SENSE_TYPE_MEDIA_WRITE_ERR);
+				       SENSE_TYPE_MEDIA_WRITE_ERR);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -2050,7 +2053,7 @@ static int write_phy_register(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 			if (retval != STATUS_SUCCESS) {
 				vfree(buf);
 				set_sense_type(chip, SCSI_LUN(srb),
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 				rtsx_trace(chip);
 				return TRANSPORT_FAILED;
 			}
@@ -2090,7 +2093,7 @@ static int erase_eeprom2(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		retval = spi_erase_eeprom_chip(chip);
 		if (retval != STATUS_SUCCESS) {
 			set_sense_type(chip, SCSI_LUN(srb),
-				SENSE_TYPE_MEDIA_WRITE_ERR);
+				       SENSE_TYPE_MEDIA_WRITE_ERR);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -2098,13 +2101,13 @@ static int erase_eeprom2(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		retval = spi_erase_eeprom_byte(chip, addr);
 		if (retval != STATUS_SUCCESS) {
 			set_sense_type(chip, SCSI_LUN(srb),
-				SENSE_TYPE_MEDIA_WRITE_ERR);
+				       SENSE_TYPE_MEDIA_WRITE_ERR);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
 	} else {
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+			       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -2139,7 +2142,7 @@ static int read_eeprom2(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	if (retval != STATUS_SUCCESS) {
 		vfree(buf);
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+			       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -2149,7 +2152,7 @@ static int read_eeprom2(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		if (retval != STATUS_SUCCESS) {
 			vfree(buf);
 			set_sense_type(chip, SCSI_LUN(srb),
-				SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -2204,7 +2207,7 @@ static int write_eeprom2(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		if (retval != STATUS_SUCCESS) {
 			vfree(buf);
 			set_sense_type(chip, SCSI_LUN(srb),
-				SENSE_TYPE_MEDIA_WRITE_ERR);
+				       SENSE_TYPE_MEDIA_WRITE_ERR);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -2242,7 +2245,7 @@ static int read_efuse(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	if (retval != STATUS_SUCCESS) {
 		vfree(buf);
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+			       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -2252,7 +2255,7 @@ static int read_efuse(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		if (retval != STATUS_SUCCESS) {
 			vfree(buf);
 			set_sense_type(chip, SCSI_LUN(srb),
-				SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -2311,7 +2314,7 @@ static int write_efuse(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		}
 
 		retval = rtsx_write_register(chip, PWR_GATE_CTRL,
-					LDO3318_PWR_MASK, LDO_OFF);
+					     LDO3318_PWR_MASK, LDO_OFF);
 		if (retval != STATUS_SUCCESS) {
 			vfree(buf);
 			rtsx_trace(chip);
@@ -2321,7 +2324,7 @@ static int write_efuse(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		wait_timeout(600);
 
 		retval = rtsx_write_phy_register(chip, 0x08,
-						0x4C00 | chip->phy_voltage);
+						 0x4C00 | chip->phy_voltage);
 		if (retval != STATUS_SUCCESS) {
 			vfree(buf);
 			rtsx_trace(chip);
@@ -2329,7 +2332,7 @@ static int write_efuse(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		}
 
 		retval = rtsx_write_register(chip, PWR_GATE_CTRL,
-					LDO3318_PWR_MASK, LDO_ON);
+					     LDO3318_PWR_MASK, LDO_ON);
 		if (retval != STATUS_SUCCESS) {
 			vfree(buf);
 			rtsx_trace(chip);
@@ -2352,14 +2355,14 @@ static int write_efuse(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		retval = rtsx_write_efuse(chip, addr + i, buf[i]);
 		if (retval != STATUS_SUCCESS) {
 			set_sense_type(chip, SCSI_LUN(srb),
-				SENSE_TYPE_MEDIA_WRITE_ERR);
+				       SENSE_TYPE_MEDIA_WRITE_ERR);
 			result = TRANSPORT_FAILED;
 			rtsx_trace(chip);
-			goto Exit;
+			goto exit;
 		}
 	}
 
-Exit:
+exit:
 	vfree(buf);
 
 	retval = card_power_off(chip, SPI_CARD);
@@ -2370,7 +2373,7 @@ Exit:
 
 	if (chip->asic_code) {
 		retval = rtsx_write_register(chip, PWR_GATE_CTRL,
-					LDO3318_PWR_MASK, LDO_OFF);
+					     LDO3318_PWR_MASK, LDO_OFF);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return TRANSPORT_ERROR;
@@ -2385,7 +2388,7 @@ Exit:
 		}
 
 		retval = rtsx_write_register(chip, PWR_GATE_CTRL,
-					LDO3318_PWR_MASK, LDO_ON);
+					     LDO3318_PWR_MASK, LDO_ON);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return TRANSPORT_ERROR;
@@ -2425,7 +2428,7 @@ static int read_cfg_byte(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	if (func > func_max) {
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+			       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -2439,7 +2442,7 @@ static int read_cfg_byte(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	retval = rtsx_read_cfg_seq(chip, func, addr, buf, len);
 	if (retval != STATUS_SUCCESS) {
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+			       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 		vfree(buf);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
@@ -2484,7 +2487,7 @@ static int write_cfg_byte(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	if (func > func_max) {
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+			       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -2593,7 +2596,7 @@ static int app_cmd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	default:
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+			       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -2670,7 +2673,7 @@ static int read_status(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	if (get_lun_card(chip, lun) == XD_CARD) {
 		rtsx_status[13] = 0x40;
 	} else if (get_lun_card(chip, lun) == SD_CARD) {
-		struct sd_info *sd_card = &(chip->sd_card);
+		struct sd_info *sd_card = &chip->sd_card;
 
 		rtsx_status[13] = 0x20;
 		if (CHK_SD(sd_card)) {
@@ -2686,7 +2689,7 @@ static int read_status(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 				rtsx_status[13] |= 0x04;
 		}
 	} else if (get_lun_card(chip, lun) == MS_CARD) {
-		struct ms_info *ms_card = &(chip->ms_card);
+		struct ms_info *ms_card = &chip->ms_card;
 
 		if (CHK_MSPRO(ms_card)) {
 			rtsx_status[13] = 0x38;
@@ -2881,7 +2884,7 @@ static int vendor_cmnd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	default:
 		set_sense_type(chip, SCSI_LUN(srb),
-			SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+			       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
@@ -2895,14 +2898,15 @@ void led_shine(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	unsigned int lun = SCSI_LUN(srb);
 	u16 sec_cnt;
 
-	if ((srb->cmnd[0] == READ_10) || (srb->cmnd[0] == WRITE_10))
+	if ((srb->cmnd[0] == READ_10) || (srb->cmnd[0] == WRITE_10)) {
 		sec_cnt = ((u16)(srb->cmnd[7]) << 8) | srb->cmnd[8];
-	else if ((srb->cmnd[0] == READ_6) || (srb->cmnd[0] == WRITE_6)) {
+	} else if ((srb->cmnd[0] == READ_6) || (srb->cmnd[0] == WRITE_6)) {
 		sec_cnt = srb->cmnd[4];
 		if (sec_cnt == 0)
 			sec_cnt = 256;
-	} else
+	} else {
 		return;
+	}
 
 	if (chip->rw_cap[lun] >= GPIO_TOGGLE_THRESHOLD) {
 		toggle_gpio(chip, LED_GPIO);
@@ -2915,7 +2919,7 @@ void led_shine(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 static int ms_format_cmnd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct ms_info *ms_card = &(chip->ms_card);
+	struct ms_info *ms_card = &chip->ms_card;
 	unsigned int lun = SCSI_LUN(srb);
 	bool quick_format;
 	int retval;
@@ -2927,7 +2931,7 @@ static int ms_format_cmnd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	}
 
 	if ((srb->cmnd[3] != 0x4D) || (srb->cmnd[4] != 0x47) ||
-		(srb->cmnd[5] != 0x66) || (srb->cmnd[6] != 0x6D) ||
+	    (srb->cmnd[5] != 0x66) || (srb->cmnd[6] != 0x6D) ||
 		(srb->cmnd[7] != 0x74)) {
 		set_sense_type(chip, lun, SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
@@ -2941,7 +2945,7 @@ static int ms_format_cmnd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		wait_timeout(100);
 
 		if (!check_card_ready(chip, lun) ||
-				(get_card_size(chip, lun) == 0)) {
+		    (get_card_size(chip, lun) == 0)) {
 			set_sense_type(chip, lun, SENSE_TYPE_MEDIA_NOT_PRESENT);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
@@ -2986,7 +2990,7 @@ static int ms_format_cmnd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 #ifdef SUPPORT_PCGL_1P18
 static int get_ms_information(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct ms_info *ms_card = &(chip->ms_card);
+	struct ms_info *ms_card = &chip->ms_card;
 	unsigned int lun = SCSI_LUN(srb);
 	u8 dev_info_id, data_len;
 	u8 *buf;
@@ -3005,8 +3009,8 @@ static int get_ms_information(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	}
 
 	if ((srb->cmnd[2] != 0xB0) || (srb->cmnd[4] != 0x4D) ||
-		(srb->cmnd[5] != 0x53) || (srb->cmnd[6] != 0x49) ||
-		(srb->cmnd[7] != 0x44)) {
+	    (srb->cmnd[5] != 0x53) || (srb->cmnd[6] != 0x49) ||
+	    (srb->cmnd[7] != 0x44)) {
 		set_sense_type(chip, lun, SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
@@ -3014,17 +3018,20 @@ static int get_ms_information(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	dev_info_id = srb->cmnd[3];
 	if ((CHK_MSXC(ms_card) && (dev_info_id == 0x10)) ||
-			(!CHK_MSXC(ms_card) && (dev_info_id == 0x13)) ||
-			!CHK_MSPRO(ms_card)) {
+	    (!CHK_MSXC(ms_card) && (dev_info_id == 0x13)) ||
+	    !CHK_MSPRO(ms_card)) {
 		set_sense_type(chip, lun, SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
 	}
 
-	if (dev_info_id == 0x15)
-		buf_len = data_len = 0x3A;
-	else
-		buf_len = data_len = 0x6A;
+	if (dev_info_id == 0x15) {
+		buf_len = 0x3A;
+		data_len = 0x3A;
+	} else {
+		buf_len = 0x6A;
+		data_len = 0x6A;
+	}
 
 	buf = kmalloc(buf_len, GFP_KERNEL);
 	if (!buf) {
@@ -3100,7 +3107,7 @@ static int ms_sp_cmnd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 }
 
 #ifdef SUPPORT_CPRM
-static int sd_extention_cmnd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
+static int sd_extension_cmnd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
 	unsigned int lun = SCSI_LUN(srb);
 	int result;
@@ -3164,7 +3171,7 @@ static int sd_extention_cmnd(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 #ifdef SUPPORT_MAGIC_GATE
 static int mg_report_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct ms_info *ms_card = &(chip->ms_card);
+	struct ms_info *ms_card = &chip->ms_card;
 	unsigned int lun = SCSI_LUN(srb);
 	int retval;
 	u8 key_format;
@@ -3208,8 +3215,8 @@ static int mg_report_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	switch (key_format) {
 	case KF_GET_LOC_EKB:
 		if ((scsi_bufflen(srb) == 0x41C) &&
-			(srb->cmnd[8] == 0x04) &&
-			(srb->cmnd[9] == 0x1C)) {
+		    (srb->cmnd[8] == 0x04) &&
+		    (srb->cmnd[9] == 0x1C)) {
 			retval = mg_get_local_EKB(srb, chip);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
@@ -3218,7 +3225,7 @@ static int mg_report_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		} else {
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+				       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -3226,8 +3233,8 @@ static int mg_report_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	case KF_RSP_CHG:
 		if ((scsi_bufflen(srb) == 0x24) &&
-			(srb->cmnd[8] == 0x00) &&
-			(srb->cmnd[9] == 0x24)) {
+		    (srb->cmnd[8] == 0x00) &&
+		    (srb->cmnd[9] == 0x24)) {
 			retval = mg_get_rsp_chg(srb, chip);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
@@ -3236,7 +3243,7 @@ static int mg_report_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		} else {
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+				       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -3245,12 +3252,12 @@ static int mg_report_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	case KF_GET_ICV:
 		ms_card->mg_entry_num = srb->cmnd[5];
 		if ((scsi_bufflen(srb) == 0x404) &&
-			(srb->cmnd[8] == 0x04) &&
-			(srb->cmnd[9] == 0x04) &&
-			(srb->cmnd[2] == 0x00) &&
-			(srb->cmnd[3] == 0x00) &&
-			(srb->cmnd[4] == 0x00) &&
-			(srb->cmnd[5] < 32)) {
+		    (srb->cmnd[8] == 0x04) &&
+		    (srb->cmnd[9] == 0x04) &&
+		    (srb->cmnd[2] == 0x00) &&
+		    (srb->cmnd[3] == 0x00) &&
+		    (srb->cmnd[4] == 0x00) &&
+		    (srb->cmnd[5] < 32)) {
 			retval = mg_get_ICV(srb, chip);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
@@ -3259,7 +3266,7 @@ static int mg_report_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		} else {
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+				       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -3277,7 +3284,7 @@ static int mg_report_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 static int mg_send_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct ms_info *ms_card = &(chip->ms_card);
+	struct ms_info *ms_card = &chip->ms_card;
 	unsigned int lun = SCSI_LUN(srb);
 	int retval;
 	u8 key_format;
@@ -3326,8 +3333,8 @@ static int mg_send_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	switch (key_format) {
 	case KF_SET_LEAF_ID:
 		if ((scsi_bufflen(srb) == 0x0C) &&
-			(srb->cmnd[8] == 0x00) &&
-			(srb->cmnd[9] == 0x0C)) {
+		    (srb->cmnd[8] == 0x00) &&
+		    (srb->cmnd[9] == 0x0C)) {
 			retval = mg_set_leaf_id(srb, chip);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
@@ -3336,7 +3343,7 @@ static int mg_send_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		} else {
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+				       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -3344,8 +3351,8 @@ static int mg_send_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	case KF_CHG_HOST:
 		if ((scsi_bufflen(srb) == 0x0C) &&
-			(srb->cmnd[8] == 0x00) &&
-			(srb->cmnd[9] == 0x0C)) {
+		    (srb->cmnd[8] == 0x00) &&
+		    (srb->cmnd[9] == 0x0C)) {
 			retval = mg_chg(srb, chip);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
@@ -3354,7 +3361,7 @@ static int mg_send_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		} else {
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+				       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -3362,8 +3369,8 @@ static int mg_send_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	case KF_RSP_HOST:
 		if ((scsi_bufflen(srb) == 0x0C) &&
-			(srb->cmnd[8] == 0x00) &&
-			(srb->cmnd[9] == 0x0C)) {
+		    (srb->cmnd[8] == 0x00) &&
+		    (srb->cmnd[9] == 0x0C)) {
 			retval = mg_rsp(srb, chip);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
@@ -3372,7 +3379,7 @@ static int mg_send_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		} else {
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+				       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -3381,12 +3388,12 @@ static int mg_send_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	case KF_SET_ICV:
 		ms_card->mg_entry_num = srb->cmnd[5];
 		if ((scsi_bufflen(srb) == 0x404) &&
-			(srb->cmnd[8] == 0x04) &&
-			(srb->cmnd[9] == 0x04) &&
-			(srb->cmnd[2] == 0x00) &&
-			(srb->cmnd[3] == 0x00) &&
-			(srb->cmnd[4] == 0x00) &&
-			(srb->cmnd[5] < 32)) {
+		    (srb->cmnd[8] == 0x04) &&
+		    (srb->cmnd[9] == 0x04) &&
+		    (srb->cmnd[2] == 0x00) &&
+		    (srb->cmnd[3] == 0x00) &&
+		    (srb->cmnd[4] == 0x00) &&
+		    (srb->cmnd[5] < 32)) {
 			retval = mg_set_ICV(srb, chip);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
@@ -3395,7 +3402,7 @@ static int mg_send_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		} else {
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
+				       SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -3415,9 +3422,9 @@ static int mg_send_key(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 int rtsx_scsi_handler(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
 #ifdef SUPPORT_SD_LOCK
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 #endif
-	struct ms_info *ms_card = &(chip->ms_card);
+	struct ms_info *ms_card = &chip->ms_card;
 	unsigned int lun = SCSI_LUN(srb);
 	int result;
 
@@ -3427,9 +3434,9 @@ int rtsx_scsi_handler(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		 * REQUEST_SENSE and rs_ppstatus
 		 */
 		if (!((srb->cmnd[0] == VENDOR_CMND) &&
-				(srb->cmnd[1] == SCSI_APP_CMD) &&
-				(srb->cmnd[2] == GET_DEV_STATUS)) &&
-				(srb->cmnd[0] != REQUEST_SENSE)) {
+		      (srb->cmnd[1] == SCSI_APP_CMD) &&
+		      (srb->cmnd[2] == GET_DEV_STATUS)) &&
+		      (srb->cmnd[0] != REQUEST_SENSE)) {
 			/* Logical Unit Not Ready Format in Progress */
 			set_sense_data(chip, lun, CUR_ERR,
 				       0x02, 0, 0x04, 0x04, 0, 0);
@@ -3440,12 +3447,12 @@ int rtsx_scsi_handler(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 #endif
 
 	if ((get_lun_card(chip, lun) == MS_CARD) &&
-			(ms_card->format_status == FORMAT_IN_PROGRESS)) {
+	    (ms_card->format_status == FORMAT_IN_PROGRESS)) {
 		if ((srb->cmnd[0] != REQUEST_SENSE) &&
-			(srb->cmnd[0] != INQUIRY)) {
+		    (srb->cmnd[0] != INQUIRY)) {
 			/* Logical Unit Not Ready Format in Progress */
 			set_sense_data(chip, lun, CUR_ERR, 0x02, 0, 0x04, 0x04,
-					0, (u16)(ms_card->progress));
+				       0, (u16)(ms_card->progress));
 			rtsx_trace(chip);
 			return TRANSPORT_FAILED;
 		}
@@ -3510,7 +3517,7 @@ int rtsx_scsi_handler(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	case SD_EXECUTE_WRITE:
 	case SD_GET_RSP:
 	case SD_HW_RST:
-		result = sd_extention_cmnd(srb, chip);
+		result = sd_extension_cmnd(srb, chip);
 		break;
 #endif
 
diff --git a/drivers/staging/rts5208/rtsx_scsi.h b/drivers/staging/rts5208/rtsx_scsi.h
index 03dd76d6c859..30f3724848fe 100644
--- a/drivers/staging/rts5208/rtsx_scsi.h
+++ b/drivers/staging/rts5208/rtsx_scsi.h
@@ -136,8 +136,8 @@
 void scsi_show_command(struct rtsx_chip *chip);
 void set_sense_type(struct rtsx_chip *chip, unsigned int lun, int sense_type);
 void set_sense_data(struct rtsx_chip *chip, unsigned int lun, u8 err_code,
-		u8 sense_key, u32 info, u8 asc, u8 ascq,
-		u8 sns_key_info0, u16 sns_key_info1);
+		    u8 sense_key, u32 info, u8 asc, u8 ascq,
+		    u8 sns_key_info0, u16 sns_key_info1);
 int rtsx_scsi_handler(struct scsi_cmnd *srb, struct rtsx_chip *chip);
 
 #endif   /* __REALTEK_RTSX_SCSI_H */
diff --git a/drivers/staging/rts5208/rtsx_sys.h b/drivers/staging/rts5208/rtsx_sys.h
index f49bed9ec76a..817700c0d794 100644
--- a/drivers/staging/rts5208/rtsx_sys.h
+++ b/drivers/staging/rts5208/rtsx_sys.h
@@ -32,9 +32,9 @@ static inline void rtsx_exclusive_enter_ss(struct rtsx_chip *chip)
 {
 	struct rtsx_dev *dev = chip->rtsx;
 
-	spin_lock(&(dev->reg_lock));
+	spin_lock(&dev->reg_lock);
 	rtsx_enter_ss(chip);
-	spin_unlock(&(dev->reg_lock));
+	spin_unlock(&dev->reg_lock);
 }
 
 static inline void rtsx_reset_detected_cards(struct rtsx_chip *chip, int flag)
diff --git a/drivers/staging/rts5208/rtsx_transport.h b/drivers/staging/rts5208/rtsx_transport.h
index 479137398c3d..99740c33f2fb 100644
--- a/drivers/staging/rts5208/rtsx_transport.h
+++ b/drivers/staging/rts5208/rtsx_transport.h
@@ -30,18 +30,21 @@
 #define WAIT_TIME	2000
 
 unsigned int rtsx_stor_access_xfer_buf(unsigned char *buffer,
-	unsigned int buflen, struct scsi_cmnd *srb, unsigned int *index,
-	unsigned int *offset, enum xfer_buf_dir dir);
-void rtsx_stor_set_xfer_buf(unsigned char *buffer,
-	unsigned int buflen, struct scsi_cmnd *srb);
-void rtsx_stor_get_xfer_buf(unsigned char *buffer,
-	unsigned int buflen, struct scsi_cmnd *srb);
+				       unsigned int buflen,
+				       struct scsi_cmnd *srb,
+				       unsigned int *index,
+				       unsigned int *offset,
+				       enum xfer_buf_dir dir);
+void rtsx_stor_set_xfer_buf(unsigned char *buffer, unsigned int buflen,
+			    struct scsi_cmnd *srb);
+void rtsx_stor_get_xfer_buf(unsigned char *buffer, unsigned int buflen,
+			    struct scsi_cmnd *srb);
 void rtsx_invoke_transport(struct scsi_cmnd *srb, struct rtsx_chip *chip);
 
 #define rtsx_init_cmd(chip)			((chip)->ci = 0)
 
-void rtsx_add_cmd(struct rtsx_chip *chip,
-		u8 cmd_type, u16 reg_addr, u8 mask, u8 data);
+void rtsx_add_cmd(struct rtsx_chip *chip, u8 cmd_type, u16 reg_addr, u8 mask,
+		  u8 data);
 void rtsx_send_cmd_no_wait(struct rtsx_chip *chip);
 int rtsx_send_cmd(struct rtsx_chip *chip, u8 card, int timeout);
 
@@ -55,11 +58,12 @@ static inline u8 *rtsx_get_cmd_data(struct rtsx_chip *chip)
 }
 
 int rtsx_transfer_data(struct rtsx_chip *chip, u8 card, void *buf, size_t len,
-		int use_sg, enum dma_data_direction dma_dir, int timeout);
+		       int use_sg, enum dma_data_direction dma_dir,
+		       int timeout);
 
-int rtsx_transfer_data_partial(struct rtsx_chip *chip, u8 card,
-			void *buf, size_t len,
-			int use_sg, unsigned int *index, unsigned int *offset,
-			enum dma_data_direction dma_dir, int timeout);
+int rtsx_transfer_data_partial(struct rtsx_chip *chip, u8 card,	void *buf,
+			       size_t len, int use_sg, unsigned int *index,
+			       unsigned int *offset,
+			       enum dma_data_direction dma_dir, int timeout);
 
 #endif   /* __REALTEK_RTSX_TRANSPORT_H */
diff --git a/drivers/staging/rts5208/sd.c b/drivers/staging/rts5208/sd.c
index b0bbb36f8988..bdd35b611f27 100644
--- a/drivers/staging/rts5208/sd.c
+++ b/drivers/staging/rts5208/sd.c
@@ -56,21 +56,21 @@ static u16 REG_SD_DCMPS1_CTL;
 
 static inline void sd_set_err_code(struct rtsx_chip *chip, u8 err_code)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 
 	sd_card->err_code |= err_code;
 }
 
 static inline void sd_clr_err_code(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 
 	sd_card->err_code = 0;
 }
 
 static inline int sd_check_err_code(struct rtsx_chip *chip, u8 err_code)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 
 	return sd_card->err_code & err_code;
 }
@@ -124,9 +124,9 @@ static int sd_check_data0_status(struct rtsx_chip *chip)
 }
 
 static int sd_send_cmd_get_rsp(struct rtsx_chip *chip, u8 cmd_idx,
-		u32 arg, u8 rsp_type, u8 *rsp, int rsp_len)
+			       u32 arg, u8 rsp_type, u8 *rsp, int rsp_len)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	int timeout = 100;
 	u16 reg_addr;
@@ -153,11 +153,12 @@ RTY_SEND_CMD:
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG2, 0xFF, rsp_type);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE,
-			0x01, PINGPONG_BUFFER);
+		     0x01, PINGPONG_BUFFER);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_TRANSFER,
-			0xFF, SD_TM_CMD_RSP | SD_TRANSFER_START);
+		     0xFF, SD_TM_CMD_RSP | SD_TRANSFER_START);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, REG_SD_TRANSFER,
-		SD_TRANSFER_END | SD_STAT_IDLE, SD_TRANSFER_END | SD_STAT_IDLE);
+		     SD_TRANSFER_END | SD_STAT_IDLE, SD_TRANSFER_END |
+		     SD_STAT_IDLE);
 
 	if (rsp_type == SD_RSP_TYPE_R2) {
 		for (reg_addr = PPBUF_BASE2; reg_addr < PPBUF_BASE2 + 16;
@@ -238,7 +239,7 @@ RTY_SEND_CMD:
 
 	if ((rsp_type == SD_RSP_TYPE_R1) || (rsp_type == SD_RSP_TYPE_R1b)) {
 		if ((cmd_idx != SEND_RELATIVE_ADDR) &&
-			(cmd_idx != SEND_IF_COND)) {
+		    (cmd_idx != SEND_IF_COND)) {
 			if (cmd_idx != STOP_TRANSMISSION) {
 				if (ptr[1] & 0x80) {
 					rtsx_trace(chip);
@@ -285,7 +286,7 @@ static int sd_read_data(struct rtsx_chip *chip,
 			u16 blk_cnt, u8 bus_width, u8 *buf, int buf_len,
 			int timeout)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	int i;
 
@@ -308,27 +309,27 @@ static int sd_read_data(struct rtsx_chip *chip,
 				     0xFF, cmd[i]);
 	}
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_L, 0xFF,
-		(u8)byte_cnt);
+		     (u8)byte_cnt);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_H, 0xFF,
-		(u8)(byte_cnt >> 8));
+		     (u8)(byte_cnt >> 8));
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_L, 0xFF,
-		(u8)blk_cnt);
+		     (u8)blk_cnt);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_H, 0xFF,
-		(u8)(blk_cnt >> 8));
+		     (u8)(blk_cnt >> 8));
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG1, 0x03, bus_width);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG2, 0xFF,
-		SD_CALCULATE_CRC7 | SD_CHECK_CRC16 | SD_NO_WAIT_BUSY_END|
-		SD_CHECK_CRC7 | SD_RSP_LEN_6);
+		     SD_CALCULATE_CRC7 | SD_CHECK_CRC16 | SD_NO_WAIT_BUSY_END |
+		     SD_CHECK_CRC7 | SD_RSP_LEN_6);
 	if (trans_mode != SD_TM_AUTO_TUNING)
 		rtsx_add_cmd(chip, WRITE_REG_CMD,
-			CARD_DATA_SOURCE, 0x01, PINGPONG_BUFFER);
+			     CARD_DATA_SOURCE, 0x01, PINGPONG_BUFFER);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_TRANSFER, 0xFF,
-		trans_mode | SD_TRANSFER_START);
+		     trans_mode | SD_TRANSFER_START);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, REG_SD_TRANSFER, SD_TRANSFER_END,
-		SD_TRANSFER_END);
+		     SD_TRANSFER_END);
 
 	retval = rtsx_send_cmd(chip, SD_CARD, timeout);
 	if (retval < 0) {
@@ -353,10 +354,10 @@ static int sd_read_data(struct rtsx_chip *chip,
 }
 
 static int sd_write_data(struct rtsx_chip *chip, u8 trans_mode,
-		u8 *cmd, int cmd_len, u16 byte_cnt, u16 blk_cnt, u8 bus_width,
-		u8 *buf, int buf_len, int timeout)
+			 u8 *cmd, int cmd_len, u16 byte_cnt, u16 blk_cnt,
+			 u8 bus_width, u8 *buf, int buf_len, int timeout)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	int i;
 
@@ -389,30 +390,30 @@ static int sd_write_data(struct rtsx_chip *chip, u8 trans_mode,
 		}
 	}
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_L, 0xFF,
-		(u8)byte_cnt);
+		     (u8)byte_cnt);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_H, 0xFF,
-		(u8)(byte_cnt >> 8));
+		     (u8)(byte_cnt >> 8));
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_L, 0xFF,
-		(u8)blk_cnt);
+		     (u8)blk_cnt);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_H, 0xFF,
-		(u8)(blk_cnt >> 8));
+		     (u8)(blk_cnt >> 8));
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG1, 0x03, bus_width);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG2, 0xFF,
-		SD_CALCULATE_CRC7 | SD_CHECK_CRC16 | SD_NO_WAIT_BUSY_END |
-		SD_CHECK_CRC7 | SD_RSP_LEN_6);
+		     SD_CALCULATE_CRC7 | SD_CHECK_CRC16 | SD_NO_WAIT_BUSY_END |
+		     SD_CHECK_CRC7 | SD_RSP_LEN_6);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_TRANSFER, 0xFF,
-		trans_mode | SD_TRANSFER_START);
+		     trans_mode | SD_TRANSFER_START);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, REG_SD_TRANSFER, SD_TRANSFER_END,
-		SD_TRANSFER_END);
+		     SD_TRANSFER_END);
 
 	retval = rtsx_send_cmd(chip, SD_CARD, timeout);
 	if (retval < 0) {
 		if (retval == -ETIMEDOUT) {
-			sd_send_cmd_get_rsp(chip, SEND_STATUS,
-				sd_card->sd_addr, SD_RSP_TYPE_R1, NULL, 0);
+			sd_send_cmd_get_rsp(chip, SEND_STATUS, sd_card->sd_addr,
+					    SD_RSP_TYPE_R1, NULL, 0);
 		}
 
 		rtsx_trace(chip);
@@ -424,7 +425,7 @@ static int sd_write_data(struct rtsx_chip *chip, u8 trans_mode,
 
 static int sd_check_csd(struct rtsx_chip *chip, char check_wp)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	int i;
 	u8 csd_ver, trans_speed;
@@ -438,7 +439,7 @@ static int sd_check_csd(struct rtsx_chip *chip, char check_wp)
 		}
 
 		retval = sd_send_cmd_get_rsp(chip, SEND_CSD, sd_card->sd_addr,
-					SD_RSP_TYPE_R2, rsp, 16);
+					     SD_RSP_TYPE_R2, rsp, 16);
 		if (retval == STATUS_SUCCESS)
 			break;
 	}
@@ -534,7 +535,7 @@ static int sd_check_csd(struct rtsx_chip *chip, char check_wp)
 static int sd_set_sample_push_timing(struct rtsx_chip *chip)
 {
 	int retval;
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	u8 val = 0;
 
 	if ((chip->sd_ctl & SD_PUSH_POINT_CTL_MASK) == SD_PUSH_POINT_DELAY)
@@ -573,7 +574,7 @@ static int sd_set_sample_push_timing(struct rtsx_chip *chip)
 
 static void sd_choose_proper_clock(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 
 	if (CHK_SD_SDR104(sd_card)) {
 		if (chip->asic_code)
@@ -637,7 +638,7 @@ static int sd_set_clock_divider(struct rtsx_chip *chip, u8 clk_div)
 
 static int sd_set_init_para(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 
 	retval = sd_set_sample_push_timing(chip);
@@ -659,7 +660,7 @@ static int sd_set_init_para(struct rtsx_chip *chip)
 
 int sd_select_card(struct rtsx_chip *chip, int select)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	u8 cmd_idx, cmd_type;
 	u32 addr;
@@ -686,12 +687,12 @@ int sd_select_card(struct rtsx_chip *chip, int select)
 #ifdef SUPPORT_SD_LOCK
 static int sd_update_lock_status(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	u8 rsp[5];
 
 	retval = sd_send_cmd_get_rsp(chip, SEND_STATUS, sd_card->sd_addr,
-				SD_RSP_TYPE_R1, rsp, 5);
+				     SD_RSP_TYPE_R1, rsp, 5);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -715,23 +716,23 @@ static int sd_update_lock_status(struct rtsx_chip *chip)
 #endif
 
 static int sd_wait_state_data_ready(struct rtsx_chip *chip, u8 state,
-				u8 data_ready, int polling_cnt)
+				    u8 data_ready, int polling_cnt)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval, i;
 	u8 rsp[5];
 
 	for (i = 0; i < polling_cnt; i++) {
 		retval = sd_send_cmd_get_rsp(chip, SEND_STATUS,
-					sd_card->sd_addr, SD_RSP_TYPE_R1, rsp,
-					5);
+					     sd_card->sd_addr, SD_RSP_TYPE_R1,
+					     rsp, 5);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
 
 		if (((rsp[3] & 0x1E) == state) &&
-			((rsp[3] & 0x01) == data_ready))
+		    ((rsp[3] & 0x01) == data_ready))
 			return STATUS_SUCCESS;
 	}
 
@@ -746,8 +747,8 @@ static int sd_change_bank_voltage(struct rtsx_chip *chip, u8 voltage)
 	if (voltage == SD_IO_3V3) {
 		if (chip->asic_code) {
 			retval = rtsx_write_phy_register(chip, 0x08,
-							0x4FC0 |
-							chip->phy_voltage);
+							 0x4FC0 |
+							 chip->phy_voltage);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
 				return STATUS_FAIL;
@@ -763,8 +764,8 @@ static int sd_change_bank_voltage(struct rtsx_chip *chip, u8 voltage)
 	} else if (voltage == SD_IO_1V8) {
 		if (chip->asic_code) {
 			retval = rtsx_write_phy_register(chip, 0x08,
-							0x4C40 |
-							chip->phy_voltage);
+							 0x4C40 |
+							 chip->phy_voltage);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
 				return STATUS_FAIL;
@@ -800,7 +801,7 @@ static int sd_voltage_switch(struct rtsx_chip *chip)
 	}
 
 	retval = sd_send_cmd_get_rsp(chip, VOLTAGE_SWITCH, 0, SD_RSP_TYPE_R1,
-				NULL, 0);
+				     NULL, 0);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -851,8 +852,8 @@ static int sd_voltage_switch(struct rtsx_chip *chip)
 			(SD_CMD_STATUS | SD_DAT3_STATUS | SD_DAT2_STATUS |
 				SD_DAT1_STATUS | SD_DAT0_STATUS)) {
 		dev_dbg(rtsx_dev(chip), "SD_BUS_STAT: 0x%x\n", stat);
-		rtsx_write_register(chip, SD_BUS_STAT,
-				SD_CLK_TOGGLE_EN | SD_CLK_FORCE_STOP, 0);
+		rtsx_write_register(chip, SD_BUS_STAT, SD_CLK_TOGGLE_EN |
+				    SD_CLK_FORCE_STOP, 0);
 		rtsx_write_register(chip, CARD_CLK_EN, 0xFF, 0);
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -903,7 +904,7 @@ static int sd_reset_dcm(struct rtsx_chip *chip, u8 tune_dir)
 
 static int sd_change_phase(struct rtsx_chip *chip, u8 sample_point, u8 tune_dir)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	u16 SD_VP_CTL, SD_DCMPS_CTL;
 	u8 val;
 	int retval;
@@ -968,7 +969,9 @@ static int sd_change_phase(struct rtsx_chip *chip, u8 sample_point, u8 tune_dir)
 			}
 			udelay(50);
 			retval = rtsx_write_register(chip, SD_VP_CTL, 0xFF,
-						     PHASE_CHANGE | PHASE_NOT_RESET | sample_point);
+						     PHASE_CHANGE |
+						     PHASE_NOT_RESET |
+						     sample_point);
 			if (retval) {
 				rtsx_trace(chip);
 				return retval;
@@ -982,7 +985,8 @@ static int sd_change_phase(struct rtsx_chip *chip, u8 sample_point, u8 tune_dir)
 			}
 			udelay(50);
 			retval = rtsx_write_register(chip, SD_VP_CTL, 0xFF,
-						     PHASE_NOT_RESET | sample_point);
+						     PHASE_NOT_RESET |
+						     sample_point);
 			if (retval) {
 				rtsx_trace(chip);
 				return retval;
@@ -992,24 +996,24 @@ static int sd_change_phase(struct rtsx_chip *chip, u8 sample_point, u8 tune_dir)
 
 		rtsx_init_cmd(chip);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SD_DCMPS_CTL, DCMPS_CHANGE,
-			DCMPS_CHANGE);
+			     DCMPS_CHANGE);
 		rtsx_add_cmd(chip, CHECK_REG_CMD, SD_DCMPS_CTL,
-			DCMPS_CHANGE_DONE, DCMPS_CHANGE_DONE);
+			     DCMPS_CHANGE_DONE, DCMPS_CHANGE_DONE);
 		retval = rtsx_send_cmd(chip, SD_CARD, 100);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto Fail;
+			goto fail;
 		}
 
 		val = *rtsx_get_cmd_data(chip);
 		if (val & DCMPS_ERROR) {
 			rtsx_trace(chip);
-			goto Fail;
+			goto fail;
 		}
 
 		if ((val & DCMPS_CURRENT_PHASE) != sample_point) {
 			rtsx_trace(chip);
-			goto Fail;
+			goto fail;
 		}
 
 		retval = rtsx_write_register(chip, SD_DCMPS_CTL,
@@ -1045,7 +1049,7 @@ static int sd_change_phase(struct rtsx_chip *chip, u8 sample_point, u8 tune_dir)
 
 	return STATUS_SUCCESS;
 
-Fail:
+fail:
 	rtsx_read_register(chip, SD_VP_CTL, &val);
 	dev_dbg(rtsx_dev(chip), "SD_VP_CTL: 0x%x\n", val);
 	rtsx_read_register(chip, SD_DCMPS_CTL, &val);
@@ -1060,12 +1064,12 @@ Fail:
 
 static int sd_check_spec(struct rtsx_chip *chip, u8 bus_width)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	u8 cmd[5], buf[8];
 
 	retval = sd_send_cmd_get_rsp(chip, APP_CMD, sd_card->sd_addr,
-				SD_RSP_TYPE_R1, NULL, 0);
+				     SD_RSP_TYPE_R1, NULL, 0);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -1078,7 +1082,7 @@ static int sd_check_spec(struct rtsx_chip *chip, u8 bus_width)
 	cmd[4] = 0;
 
 	retval = sd_read_data(chip, SD_TM_NORMAL_READ, cmd, 5, 8, 1, bus_width,
-			buf, 8, 250);
+			      buf, 8, 250);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_clear_sd_error(chip);
 		rtsx_trace(chip);
@@ -1096,7 +1100,7 @@ static int sd_check_spec(struct rtsx_chip *chip, u8 bus_width)
 }
 
 static int sd_query_switch_result(struct rtsx_chip *chip, u8 func_group,
-				u8 func_to_switch, u8 *buf, int buf_len)
+				  u8 func_to_switch, u8 *buf, int buf_len)
 {
 	u8 support_mask = 0, query_switch = 0, switch_busy = 0;
 	int support_offset = 0, query_switch_offset = 0, check_busy_offset = 0;
@@ -1198,7 +1202,7 @@ static int sd_query_switch_result(struct rtsx_chip *chip, u8 func_group,
 
 	if (func_group == SD_FUNC_GROUP_1) {
 		if (!(buf[support_offset] & support_mask) ||
-			((buf[query_switch_offset] & 0x0F) != query_switch)) {
+		    ((buf[query_switch_offset] & 0x0F) != query_switch)) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
@@ -1206,7 +1210,7 @@ static int sd_query_switch_result(struct rtsx_chip *chip, u8 func_group,
 
 	/* Check 'Busy Status' */
 	if ((buf[DATA_STRUCTURE_VER_OFFSET] == 0x01) &&
-		    ((buf[check_busy_offset] & switch_busy) == switch_busy)) {
+	    ((buf[check_busy_offset] & switch_busy) == switch_busy)) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
 	}
@@ -1214,10 +1218,10 @@ static int sd_query_switch_result(struct rtsx_chip *chip, u8 func_group,
 	return STATUS_SUCCESS;
 }
 
-static int sd_check_switch_mode(struct rtsx_chip *chip, u8 mode,
-		u8 func_group, u8 func_to_switch, u8 bus_width)
+static int sd_check_switch_mode(struct rtsx_chip *chip, u8 mode, u8 func_group,
+				u8 func_to_switch, u8 bus_width)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	u8 cmd[5], buf[64];
 
@@ -1247,7 +1251,7 @@ static int sd_check_switch_mode(struct rtsx_chip *chip, u8 mode,
 	}
 
 	retval = sd_read_data(chip, SD_TM_NORMAL_READ, cmd, 5, 64, 1, bus_width,
-			buf, 64, 250);
+			      buf, 64, 250);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_clear_sd_error(chip);
 		rtsx_trace(chip);
@@ -1326,7 +1330,7 @@ static u8 downgrade_switch_mode(u8 func_group, u8 func_to_switch)
 }
 
 static int sd_check_switch(struct rtsx_chip *chip,
-		u8 func_group, u8 func_to_switch, u8 bus_width)
+			   u8 func_group, u8 func_to_switch, u8 bus_width)
 {
 	int retval;
 	int i;
@@ -1340,12 +1344,14 @@ static int sd_check_switch(struct rtsx_chip *chip,
 		}
 
 		retval = sd_check_switch_mode(chip, SD_CHECK_MODE, func_group,
-				func_to_switch, bus_width);
+					      func_to_switch, bus_width);
 		if (retval == STATUS_SUCCESS) {
 			u8 stat;
 
 			retval = sd_check_switch_mode(chip, SD_SWITCH_MODE,
-					func_group, func_to_switch, bus_width);
+						      func_group,
+						      func_to_switch,
+						      bus_width);
 			if (retval == STATUS_SUCCESS) {
 				switch_good = true;
 				break;
@@ -1364,7 +1370,7 @@ static int sd_check_switch(struct rtsx_chip *chip,
 		}
 
 		func_to_switch = downgrade_switch_mode(func_group,
-						func_to_switch);
+						       func_to_switch);
 
 		wait_timeout(20);
 	}
@@ -1379,14 +1385,14 @@ static int sd_check_switch(struct rtsx_chip *chip,
 
 static int sd_switch_function(struct rtsx_chip *chip, u8 bus_width)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	int i;
 	u8 func_to_switch = 0;
 
 	/* Get supported functions */
-	retval = sd_check_switch_mode(chip, SD_CHECK_MODE,
-			NO_ARGUMENT, NO_ARGUMENT, bus_width);
+	retval = sd_check_switch_mode(chip, SD_CHECK_MODE, NO_ARGUMENT,
+				      NO_ARGUMENT, bus_width);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -1396,24 +1402,24 @@ static int sd_switch_function(struct rtsx_chip *chip, u8 bus_width)
 
 	/* Function Group 1: Access Mode */
 	for (i = 0; i < 4; i++) {
-		switch ((u8)(chip->sd_speed_prior >> (i*8))) {
+		switch ((u8)(chip->sd_speed_prior >> (i * 8))) {
 		case SDR104_SUPPORT:
-			if ((sd_card->func_group1_mask & SDR104_SUPPORT_MASK)
-					&& chip->sdr104_en) {
+			if ((sd_card->func_group1_mask & SDR104_SUPPORT_MASK) &&
+			    chip->sdr104_en) {
 				func_to_switch = SDR104_SUPPORT;
 			}
 			break;
 
 		case DDR50_SUPPORT:
-			if ((sd_card->func_group1_mask & DDR50_SUPPORT_MASK)
-					&& chip->ddr50_en) {
+			if ((sd_card->func_group1_mask & DDR50_SUPPORT_MASK) &&
+			    chip->ddr50_en) {
 				func_to_switch = DDR50_SUPPORT;
 			}
 			break;
 
 		case SDR50_SUPPORT:
-			if ((sd_card->func_group1_mask & SDR50_SUPPORT_MASK)
-					&& chip->sdr50_en) {
+			if ((sd_card->func_group1_mask & SDR50_SUPPORT_MASK) &&
+			    chip->sdr50_en) {
 				func_to_switch = SDR50_SUPPORT;
 			}
 			break;
@@ -1430,7 +1436,6 @@ static int sd_switch_function(struct rtsx_chip *chip, u8 bus_width)
 
 		if (func_to_switch)
 			break;
-
 	}
 	dev_dbg(rtsx_dev(chip), "SD_FUNC_GROUP_1: func_to_switch = 0x%02x",
 		func_to_switch);
@@ -1446,7 +1451,7 @@ static int sd_switch_function(struct rtsx_chip *chip, u8 bus_width)
 
 	if (func_to_switch) {
 		retval = sd_check_switch(chip, SD_FUNC_GROUP_1, func_to_switch,
-					bus_width);
+					 bus_width);
 		if (retval != STATUS_SUCCESS) {
 			if (func_to_switch == SDR104_SUPPORT) {
 				sd_card->sd_switch_fail = SDR104_SUPPORT_MASK;
@@ -1496,7 +1501,7 @@ static int sd_switch_function(struct rtsx_chip *chip, u8 bus_width)
 	func_to_switch = 0xFF;
 
 	for (i = 0; i < 4; i++) {
-		switch ((u8)(chip->sd_current_prior >> (i*8))) {
+		switch ((u8)(chip->sd_current_prior >> (i * 8))) {
 		case CURRENT_LIMIT_800:
 			if (sd_card->func_group4_mask & CURRENT_LIMIT_800_MASK)
 				func_to_switch = CURRENT_LIMIT_800;
@@ -1534,7 +1539,7 @@ static int sd_switch_function(struct rtsx_chip *chip, u8 bus_width)
 
 	if (func_to_switch <= CURRENT_LIMIT_800) {
 		retval = sd_check_switch(chip, SD_FUNC_GROUP_4, func_to_switch,
-					bus_width);
+					 bus_width);
 		if (retval != STATUS_SUCCESS) {
 			if (sd_check_err_code(chip, SD_NO_CARD)) {
 				rtsx_trace(chip);
@@ -1596,8 +1601,8 @@ static int sd_sdr_tuning_rx_cmd(struct rtsx_chip *chip, u8 sample_point)
 	cmd[3] = 0;
 	cmd[4] = 0;
 
-	retval = sd_read_data(chip, SD_TM_AUTO_TUNING,
-			cmd, 5, 0x40, 1, SD_BUS_WIDTH_4, NULL, 0, 100);
+	retval = sd_read_data(chip, SD_TM_AUTO_TUNING, cmd, 5, 0x40, 1,
+			      SD_BUS_WIDTH_4, NULL, 0, 100);
 	if (retval != STATUS_SUCCESS) {
 		(void)sd_wait_data_idle(chip);
 
@@ -1611,7 +1616,7 @@ static int sd_sdr_tuning_rx_cmd(struct rtsx_chip *chip, u8 sample_point)
 
 static int sd_ddr_tuning_rx_cmd(struct rtsx_chip *chip, u8 sample_point)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	u8 cmd[5];
 
@@ -1624,7 +1629,7 @@ static int sd_ddr_tuning_rx_cmd(struct rtsx_chip *chip, u8 sample_point)
 	dev_dbg(rtsx_dev(chip), "sd ddr tuning rx\n");
 
 	retval = sd_send_cmd_get_rsp(chip, APP_CMD, sd_card->sd_addr,
-				SD_RSP_TYPE_R1, NULL, 0);
+				     SD_RSP_TYPE_R1, NULL, 0);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -1636,8 +1641,8 @@ static int sd_ddr_tuning_rx_cmd(struct rtsx_chip *chip, u8 sample_point)
 	cmd[3] = 0;
 	cmd[4] = 0;
 
-	retval = sd_read_data(chip, SD_TM_NORMAL_READ,
-			cmd, 5, 64, 1, SD_BUS_WIDTH_4, NULL, 0, 100);
+	retval = sd_read_data(chip, SD_TM_NORMAL_READ, cmd, 5, 64, 1,
+			      SD_BUS_WIDTH_4, NULL, 0, 100);
 	if (retval != STATUS_SUCCESS) {
 		(void)sd_wait_data_idle(chip);
 
@@ -1651,7 +1656,7 @@ static int sd_ddr_tuning_rx_cmd(struct rtsx_chip *chip, u8 sample_point)
 
 static int mmc_ddr_tunning_rx_cmd(struct rtsx_chip *chip, u8 sample_point)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	u8 cmd[5], bus_width;
 
@@ -1676,8 +1681,8 @@ static int mmc_ddr_tunning_rx_cmd(struct rtsx_chip *chip, u8 sample_point)
 	cmd[3] = 0;
 	cmd[4] = 0;
 
-	retval = sd_read_data(chip, SD_TM_NORMAL_READ,
-			cmd, 5, 0x200, 1, bus_width, NULL, 0, 100);
+	retval = sd_read_data(chip, SD_TM_NORMAL_READ, cmd, 5, 0x200, 1,
+			      bus_width, NULL, 0, 100);
 	if (retval != STATUS_SUCCESS) {
 		(void)sd_wait_data_idle(chip);
 
@@ -1691,7 +1696,7 @@ static int mmc_ddr_tunning_rx_cmd(struct rtsx_chip *chip, u8 sample_point)
 
 static int sd_sdr_tuning_tx_cmd(struct rtsx_chip *chip, u8 sample_point)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 
 	retval = sd_change_phase(chip, sample_point, TUNE_TX);
@@ -1708,11 +1713,11 @@ static int sd_sdr_tuning_tx_cmd(struct rtsx_chip *chip, u8 sample_point)
 	}
 
 	retval = sd_send_cmd_get_rsp(chip, SEND_STATUS, sd_card->sd_addr,
-		SD_RSP_TYPE_R1, NULL, 0);
+				     SD_RSP_TYPE_R1, NULL, 0);
 	if (retval != STATUS_SUCCESS) {
 		if (sd_check_err_code(chip, SD_RSP_TIMEOUT)) {
 			rtsx_write_register(chip, SD_CFG3,
-					SD_RSP_80CLK_TIMEOUT_EN, 0);
+					    SD_RSP_80CLK_TIMEOUT_EN, 0);
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
@@ -1730,7 +1735,7 @@ static int sd_sdr_tuning_tx_cmd(struct rtsx_chip *chip, u8 sample_point)
 
 static int sd_ddr_tuning_tx_cmd(struct rtsx_chip *chip, u8 sample_point)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	u8 cmd[5], bus_width;
 
@@ -1770,8 +1775,8 @@ static int sd_ddr_tuning_tx_cmd(struct rtsx_chip *chip, u8 sample_point)
 	cmd[3] = 0;
 	cmd[4] = 0;
 
-	retval = sd_write_data(chip, SD_TM_AUTO_WRITE_2,
-			cmd, 5, 16, 1, bus_width, sd_card->raw_csd, 16, 100);
+	retval = sd_write_data(chip, SD_TM_AUTO_WRITE_2, cmd, 5, 16, 1,
+			       bus_width, sd_card->raw_csd, 16, 100);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_clear_sd_error(chip);
 		rtsx_write_register(chip, SD_CFG3, SD_RSP_80CLK_TIMEOUT_EN, 0);
@@ -1787,7 +1792,7 @@ static int sd_ddr_tuning_tx_cmd(struct rtsx_chip *chip, u8 sample_point)
 	}
 
 	sd_send_cmd_get_rsp(chip, SEND_STATUS, sd_card->sd_addr, SD_RSP_TYPE_R1,
-			NULL, 0);
+			    NULL, 0);
 
 	return STATUS_SUCCESS;
 }
@@ -1795,7 +1800,7 @@ static int sd_ddr_tuning_tx_cmd(struct rtsx_chip *chip, u8 sample_point)
 static u8 sd_search_final_phase(struct rtsx_chip *chip, u32 phase_map,
 				u8 tune_dir)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	struct timing_phase_path path[MAX_PHASE + 1];
 	int i, j, cont_path_cnt;
 	bool new_block;
@@ -1808,7 +1813,7 @@ static u8 sd_search_final_phase(struct rtsx_chip *chip, u32 phase_map,
 		else
 			final_phase = (u8)chip->sd_default_tx_phase;
 
-		goto Search_Finish;
+		goto search_finish;
 	}
 
 	cont_path_cnt = 0;
@@ -1839,7 +1844,7 @@ static u8 sd_search_final_phase(struct rtsx_chip *chip, u32 phase_map,
 
 	if (cont_path_cnt == 0) {
 		dev_dbg(rtsx_dev(chip), "No continuous phase path\n");
-		goto Search_Finish;
+		goto search_finish;
 	} else {
 		int idx = cont_path_cnt - 1;
 
@@ -1848,7 +1853,7 @@ static u8 sd_search_final_phase(struct rtsx_chip *chip, u32 phase_map,
 	}
 
 	if ((path[0].start == 0) &&
-		(path[cont_path_cnt - 1].end == MAX_PHASE)) {
+	    (path[cont_path_cnt - 1].end == MAX_PHASE)) {
 		path[0].start = path[cont_path_cnt - 1].start - MAX_PHASE - 1;
 		path[0].len += path[cont_path_cnt - 1].len;
 		path[0].mid = path[0].start + path[0].len / 2;
@@ -1906,14 +1911,14 @@ static u8 sd_search_final_phase(struct rtsx_chip *chip, u32 phase_map,
 		}
 	}
 
-Search_Finish:
+search_finish:
 	dev_dbg(rtsx_dev(chip), "Final chosen phase: %d\n", final_phase);
 	return final_phase;
 }
 
 static int sd_tuning_rx(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	int i, j;
 	u32 raw_phase_map[3], phase_map;
@@ -1974,7 +1979,7 @@ static int sd_tuning_rx(struct rtsx_chip *chip)
 
 static int sd_ddr_pre_tuning_tx(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	int i;
 	u32 phase_map;
@@ -1992,7 +1997,7 @@ static int sd_ddr_pre_tuning_tx(struct rtsx_chip *chip)
 		if (detect_card_cd(chip, SD_CARD) != STATUS_SUCCESS) {
 			sd_set_err_code(chip, SD_NO_CARD);
 			rtsx_write_register(chip, SD_CFG3,
-						SD_RSP_80CLK_TIMEOUT_EN, 0);
+					    SD_RSP_80CLK_TIMEOUT_EN, 0);
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
@@ -2002,10 +2007,10 @@ static int sd_ddr_pre_tuning_tx(struct rtsx_chip *chip)
 			continue;
 
 		retval = sd_send_cmd_get_rsp(chip, SEND_STATUS,
-					sd_card->sd_addr, SD_RSP_TYPE_R1, NULL,
-					0);
+					     sd_card->sd_addr, SD_RSP_TYPE_R1,
+					     NULL, 0);
 		if ((retval == STATUS_SUCCESS) ||
-			!sd_check_err_code(chip, SD_RSP_TIMEOUT))
+		    !sd_check_err_code(chip, SD_RSP_TIMEOUT))
 			phase_map |= 1 << i;
 	}
 
@@ -2039,7 +2044,7 @@ static int sd_ddr_pre_tuning_tx(struct rtsx_chip *chip)
 
 static int sd_tuning_tx(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	int i, j;
 	u32 raw_phase_map[3], phase_map;
@@ -2131,7 +2136,7 @@ static int sd_ddr_tuning(struct rtsx_chip *chip)
 		}
 	} else {
 		retval = sd_change_phase(chip, (u8)chip->sd_ddr_tx_phase,
-					TUNE_TX);
+					 TUNE_TX);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -2167,7 +2172,7 @@ static int mmc_ddr_tuning(struct rtsx_chip *chip)
 		}
 	} else {
 		retval = sd_change_phase(chip, (u8)chip->mmc_ddr_tx_phase,
-					TUNE_TX);
+					 TUNE_TX);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -2193,7 +2198,7 @@ static int mmc_ddr_tuning(struct rtsx_chip *chip)
 
 int sd_switch_clock(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	int re_tuning = 0;
 
@@ -2231,7 +2236,7 @@ int sd_switch_clock(struct rtsx_chip *chip)
 
 static int sd_prepare_reset(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 
 	if (chip->asic_code)
@@ -2286,31 +2291,36 @@ static int sd_pull_ctl_disable(struct rtsx_chip *chip)
 
 	if (CHECK_PID(chip, 0x5208)) {
 		retval = rtsx_write_register(chip, CARD_PULL_CTL1, 0xFF,
-					     XD_D3_PD | SD_D7_PD | SD_CLK_PD | SD_D5_PD);
+					     XD_D3_PD | SD_D7_PD | SD_CLK_PD |
+					     SD_D5_PD);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 		retval = rtsx_write_register(chip, CARD_PULL_CTL2, 0xFF,
-					     SD_D6_PD | SD_D0_PD | SD_D1_PD | XD_D5_PD);
+					     SD_D6_PD | SD_D0_PD | SD_D1_PD |
+					     XD_D5_PD);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 		retval = rtsx_write_register(chip, CARD_PULL_CTL3, 0xFF,
-					     SD_D4_PD | XD_CE_PD | XD_CLE_PD | XD_CD_PU);
+					     SD_D4_PD | XD_CE_PD | XD_CLE_PD |
+					     XD_CD_PU);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 		retval = rtsx_write_register(chip, CARD_PULL_CTL4, 0xFF,
-					     XD_RDY_PD | SD_D3_PD | SD_D2_PD | XD_ALE_PD);
+					     XD_RDY_PD | SD_D3_PD | SD_D2_PD |
+					     XD_ALE_PD);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 		retval = rtsx_write_register(chip, CARD_PULL_CTL5, 0xFF,
-					     MS_INS_PU | SD_WP_PD | SD_CD_PU | SD_CMD_PD);
+					     MS_INS_PU | SD_WP_PD | SD_CD_PU |
+					     SD_CMD_PD);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
@@ -2361,27 +2371,27 @@ int sd_pull_ctl_enable(struct rtsx_chip *chip)
 
 	if (CHECK_PID(chip, 0x5208)) {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL1, 0xFF,
-			XD_D3_PD | SD_DAT7_PU | SD_CLK_NP | SD_D5_PU);
+			     XD_D3_PD | SD_DAT7_PU | SD_CLK_NP | SD_D5_PU);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL2, 0xFF,
-			SD_D6_PU | SD_D0_PU | SD_D1_PU | XD_D5_PD);
+			     SD_D6_PU | SD_D0_PU | SD_D1_PU | XD_D5_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL3, 0xFF,
-			SD_D4_PU | XD_CE_PD | XD_CLE_PD | XD_CD_PU);
+			     SD_D4_PU | XD_CE_PD | XD_CLE_PD | XD_CD_PU);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL4, 0xFF,
-			XD_RDY_PD | SD_D3_PU | SD_D2_PU | XD_ALE_PD);
+			     XD_RDY_PD | SD_D3_PU | SD_D2_PU | XD_ALE_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL5, 0xFF,
-			MS_INS_PU | SD_WP_PU | SD_CD_PU | SD_CMD_PU);
+			     MS_INS_PU | SD_WP_PU | SD_CD_PU | SD_CMD_PU);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL6, 0xFF,
-			MS_D5_PD | MS_D4_PD);
+			     MS_D5_PD | MS_D4_PD);
 	} else if (CHECK_PID(chip, 0x5288)) {
 		if (CHECK_BARO_PKG(chip, QFN)) {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL1, 0xFF,
-				0xA8);
+				     0xA8);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL2, 0xFF,
-				0x5A);
+				     0x5A);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL3, 0xFF,
-				0x95);
+				     0x95);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL4, 0xFF,
-				0xAA);
+				     0xAA);
 		}
 	}
 
@@ -2478,7 +2488,7 @@ static int sd_dummy_clock(struct rtsx_chip *chip)
 
 static int sd_read_lba0(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	u8 cmd[5], bus_width;
 
@@ -2499,8 +2509,8 @@ static int sd_read_lba0(struct rtsx_chip *chip)
 			bus_width = SD_BUS_WIDTH_1;
 	}
 
-	retval = sd_read_data(chip, SD_TM_NORMAL_READ, cmd,
-		5, 512, 1, bus_width, NULL, 0, 100);
+	retval = sd_read_data(chip, SD_TM_NORMAL_READ, cmd, 5, 512, 1,
+			      bus_width, NULL, 0, 100);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_clear_sd_error(chip);
 		rtsx_trace(chip);
@@ -2512,14 +2522,14 @@ static int sd_read_lba0(struct rtsx_chip *chip)
 
 static int sd_check_wp_state(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	u32 val;
 	u16 sd_card_type;
 	u8 cmd[5], buf[64];
 
-	retval = sd_send_cmd_get_rsp(chip, APP_CMD,
-			sd_card->sd_addr, SD_RSP_TYPE_R1, NULL, 0);
+	retval = sd_send_cmd_get_rsp(chip, APP_CMD, sd_card->sd_addr,
+				     SD_RSP_TYPE_R1, NULL, 0);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -2532,12 +2542,12 @@ static int sd_check_wp_state(struct rtsx_chip *chip)
 	cmd[4] = 0;
 
 	retval = sd_read_data(chip, SD_TM_NORMAL_READ, cmd, 5, 64, 1,
-			SD_BUS_WIDTH_4, buf, 64, 250);
+			      SD_BUS_WIDTH_4, buf, 64, 250);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_clear_sd_error(chip);
 
 		sd_send_cmd_get_rsp(chip, SEND_STATUS, sd_card->sd_addr,
-				SD_RSP_TYPE_R1, NULL, 0);
+				    SD_RSP_TYPE_R1, NULL, 0);
 		rtsx_trace(chip);
 		return STATUS_FAIL;
 	}
@@ -2562,7 +2572,7 @@ static int sd_check_wp_state(struct rtsx_chip *chip)
 
 static int reset_sd(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	bool hi_cap_flow = false;
 	int retval, i = 0, j = 0, k = 0;
 	bool sd_dont_switch = false;
@@ -2575,7 +2585,7 @@ static int reset_sd(struct rtsx_chip *chip)
 
 	SET_SD(sd_card);
 
-Switch_Fail:
+switch_fail:
 
 	i = 0;
 	j = 0;
@@ -2589,11 +2599,11 @@ Switch_Fail:
 
 	retval = sd_prepare_reset(chip);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 	retval = sd_dummy_clock(chip);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 	if (CHK_SDIO_EXIST(chip) && !CHK_SDIO_IGNORED(chip) && try_sdio) {
 		int rty_cnt = 0;
@@ -2601,11 +2611,11 @@ Switch_Fail:
 		for (; rty_cnt < chip->sdio_retry_cnt; rty_cnt++) {
 			if (detect_card_cd(chip, SD_CARD) != STATUS_SUCCESS) {
 				sd_set_err_code(chip, SD_NO_CARD);
-				goto Status_Fail;
+				goto status_fail;
 			}
 
 			retval = sd_send_cmd_get_rsp(chip, IO_SEND_OP_COND, 0,
-						SD_RSP_TYPE_R4, rsp, 5);
+						     SD_RSP_TYPE_R4, rsp, 5);
 			if (retval == STATUS_SUCCESS) {
 				int func_num = (rsp[1] >> 4) & 0x07;
 
@@ -2613,7 +2623,7 @@ Switch_Fail:
 					dev_dbg(rtsx_dev(chip), "SD_IO card (Function number: %d)!\n",
 						func_num);
 					chip->sd_io = 1;
-					goto Status_Fail;
+					goto status_fail;
 				}
 
 				break;
@@ -2630,14 +2640,14 @@ Switch_Fail:
 	/* Start Initialization Process of SD Card */
 RTY_SD_RST:
 	retval = sd_send_cmd_get_rsp(chip, GO_IDLE_STATE, 0, SD_RSP_TYPE_R0,
-				NULL, 0);
+				     NULL, 0);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 	wait_timeout(20);
 
 	retval = sd_send_cmd_get_rsp(chip, SEND_IF_COND, 0x000001AA,
-				SD_RSP_TYPE_R7, rsp, 5);
+				     SD_RSP_TYPE_R7, rsp, 5);
 	if (retval == STATUS_SUCCESS) {
 		if ((rsp[4] == 0xAA) && ((rsp[3] & 0x0f) == 0x01)) {
 			hi_cap_flow = true;
@@ -2649,37 +2659,37 @@ RTY_SD_RST:
 		voltage = SUPPORT_VOLTAGE;
 
 		retval = sd_send_cmd_get_rsp(chip, GO_IDLE_STATE, 0,
-					SD_RSP_TYPE_R0, NULL, 0);
+					     SD_RSP_TYPE_R0, NULL, 0);
 		if (retval != STATUS_SUCCESS)
-			goto Status_Fail;
+			goto status_fail;
 
 		wait_timeout(20);
 	}
 
 	do {
 		retval = sd_send_cmd_get_rsp(chip, APP_CMD, 0, SD_RSP_TYPE_R1,
-					NULL, 0);
+					     NULL, 0);
 		if (retval != STATUS_SUCCESS) {
 			if (detect_card_cd(chip, SD_CARD) != STATUS_SUCCESS) {
 				sd_set_err_code(chip, SD_NO_CARD);
-				goto Status_Fail;
+				goto status_fail;
 			}
 
 			j++;
 			if (j < 3)
 				goto RTY_SD_RST;
 			else
-				goto Status_Fail;
+				goto status_fail;
 		}
 
 		retval = sd_send_cmd_get_rsp(chip, SD_APP_OP_COND, voltage,
-					SD_RSP_TYPE_R3, rsp, 5);
+					     SD_RSP_TYPE_R3, rsp, 5);
 		if (retval != STATUS_SUCCESS) {
 			k++;
 			if (k < 3)
 				goto RTY_SD_RST;
 			else
-				goto Status_Fail;
+				goto status_fail;
 		}
 
 		i++;
@@ -2687,7 +2697,7 @@ RTY_SD_RST:
 	} while (!(rsp[1] & 0x80) && (i < 255));
 
 	if (i == 255)
-		goto Status_Fail;
+		goto status_fail;
 
 	if (hi_cap_flow) {
 		if (rsp[1] & 0x40)
@@ -2705,19 +2715,19 @@ RTY_SD_RST:
 	if (support_1v8) {
 		retval = sd_voltage_switch(chip);
 		if (retval != STATUS_SUCCESS)
-			goto Status_Fail;
+			goto status_fail;
 	}
 
 	retval = sd_send_cmd_get_rsp(chip, ALL_SEND_CID, 0, SD_RSP_TYPE_R2,
-				NULL, 0);
+				     NULL, 0);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 	for (i = 0; i < 3; i++) {
 		retval = sd_send_cmd_get_rsp(chip, SEND_RELATIVE_ADDR, 0,
-					SD_RSP_TYPE_R6, rsp, 5);
+					     SD_RSP_TYPE_R6, rsp, 5);
 		if (retval != STATUS_SUCCESS)
-			goto Status_Fail;
+			goto status_fail;
 
 		sd_card->sd_addr = (u32)rsp[1] << 24;
 		sd_card->sd_addr += (u32)rsp[2] << 16;
@@ -2728,17 +2738,17 @@ RTY_SD_RST:
 
 	retval = sd_check_csd(chip, 1);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 	retval = sd_select_card(chip, 1);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 #ifdef SUPPORT_SD_LOCK
 SD_UNLOCK_ENTRY:
 	retval = sd_update_lock_status(chip);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 	if (sd_card->sd_lock_status & SD_LOCKED) {
 		sd_card->sd_lock_status |= (SD_LOCK_1BIT_MODE | SD_PWD_EXIST);
@@ -2749,25 +2759,25 @@ SD_UNLOCK_ENTRY:
 #endif
 
 	retval = sd_send_cmd_get_rsp(chip, APP_CMD, sd_card->sd_addr,
-				SD_RSP_TYPE_R1, NULL, 0);
+				     SD_RSP_TYPE_R1, NULL, 0);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 	retval = sd_send_cmd_get_rsp(chip, SET_CLR_CARD_DETECT, 0,
-				SD_RSP_TYPE_R1, NULL, 0);
+				     SD_RSP_TYPE_R1, NULL, 0);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 	if (support_1v8) {
 		retval = sd_send_cmd_get_rsp(chip, APP_CMD, sd_card->sd_addr,
-					SD_RSP_TYPE_R1, NULL, 0);
+					     SD_RSP_TYPE_R1, NULL, 0);
 		if (retval != STATUS_SUCCESS)
-			goto Status_Fail;
+			goto status_fail;
 
 		retval = sd_send_cmd_get_rsp(chip, SET_BUS_WIDTH, 2,
-					SD_RSP_TYPE_R1, NULL, 0);
+					     SD_RSP_TYPE_R1, NULL, 0);
 		if (retval != STATUS_SUCCESS)
-			goto Status_Fail;
+			goto status_fail;
 
 		switch_bus_width = SD_BUS_WIDTH_4;
 	} else {
@@ -2775,13 +2785,13 @@ SD_UNLOCK_ENTRY:
 	}
 
 	retval = sd_send_cmd_get_rsp(chip, SET_BLOCKLEN, 0x200, SD_RSP_TYPE_R1,
-				NULL, 0);
+				     NULL, 0);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 	retval = sd_set_clock_divider(chip, SD_CLK_DIVIDE_0);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 	if (!(sd_card->raw_csd[4] & 0x40))
 		sd_dont_switch = true;
@@ -2804,7 +2814,7 @@ SD_UNLOCK_ENTRY:
 				sd_dont_switch = true;
 				try_sdio = false;
 
-				goto Switch_Fail;
+				goto switch_fail;
 			}
 		} else {
 			if (support_1v8) {
@@ -2812,21 +2822,21 @@ SD_UNLOCK_ENTRY:
 				sd_dont_switch = true;
 				try_sdio = false;
 
-				goto Switch_Fail;
+				goto switch_fail;
 			}
 		}
 	}
 
 	if (!support_1v8) {
 		retval = sd_send_cmd_get_rsp(chip, APP_CMD, sd_card->sd_addr,
-					SD_RSP_TYPE_R1, NULL, 0);
+					     SD_RSP_TYPE_R1, NULL, 0);
 		if (retval != STATUS_SUCCESS)
-			goto Status_Fail;
+			goto status_fail;
 
 		retval = sd_send_cmd_get_rsp(chip, SET_BUS_WIDTH, 2,
-					SD_RSP_TYPE_R1, NULL, 0);
+					     SD_RSP_TYPE_R1, NULL, 0);
 		if (retval != STATUS_SUCCESS)
-			goto Status_Fail;
+			goto status_fail;
 	}
 
 #ifdef SUPPORT_SD_LOCK
@@ -2845,7 +2855,7 @@ SD_UNLOCK_ENTRY:
 
 		retval = sd_set_init_para(chip);
 		if (retval != STATUS_SUCCESS)
-			goto Status_Fail;
+			goto status_fail;
 
 		if (CHK_SD_DDR50(sd_card))
 			retval = sd_ddr_tuning(chip);
@@ -2854,20 +2864,20 @@ SD_UNLOCK_ENTRY:
 
 		if (retval != STATUS_SUCCESS) {
 			if (sd20_mode) {
-				goto Status_Fail;
+				goto status_fail;
 			} else {
 				retval = sd_init_power(chip);
 				if (retval != STATUS_SUCCESS)
-					goto Status_Fail;
+					goto status_fail;
 
 				try_sdio = false;
 				sd20_mode = true;
-				goto Switch_Fail;
+				goto switch_fail;
 			}
 		}
 
 		sd_send_cmd_get_rsp(chip, SEND_STATUS, sd_card->sd_addr,
-				SD_RSP_TYPE_R1, NULL, 0);
+				    SD_RSP_TYPE_R1, NULL, 0);
 
 		if (CHK_SD_DDR50(sd_card)) {
 			retval = sd_wait_state_data_ready(chip, 0x08, 1, 1000);
@@ -2879,15 +2889,15 @@ SD_UNLOCK_ENTRY:
 			retval = sd_read_lba0(chip);
 			if (retval != STATUS_SUCCESS) {
 				if (sd20_mode) {
-					goto Status_Fail;
+					goto status_fail;
 				} else {
 					retval = sd_init_power(chip);
 					if (retval != STATUS_SUCCESS)
-						goto Status_Fail;
+						goto status_fail;
 
 					try_sdio = false;
 					sd20_mode = true;
-					goto Switch_Fail;
+					goto switch_fail;
 				}
 			}
 		}
@@ -2895,7 +2905,7 @@ SD_UNLOCK_ENTRY:
 
 	retval = sd_check_wp_state(chip);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 	chip->card_bus_width[chip->card2lun[SD_CARD]] = 4;
 
@@ -2918,21 +2928,21 @@ SD_UNLOCK_ENTRY:
 
 	return STATUS_SUCCESS;
 
-Status_Fail:
+status_fail:
 	rtsx_trace(chip);
 	return STATUS_FAIL;
 }
 
 static int mmc_test_switch_bus(struct rtsx_chip *chip, u8 width)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	u8 buf[8] = {0}, bus_width, *ptr;
 	u16 byte_cnt;
 	int len;
 
 	retval = sd_send_cmd_get_rsp(chip, BUSTEST_W, 0, SD_RSP_TYPE_R1, NULL,
-				0);
+				     0);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return SWITCH_FAIL;
@@ -2957,8 +2967,8 @@ static int mmc_test_switch_bus(struct rtsx_chip *chip, u8 width)
 		return SWITCH_ERR;
 	}
 
-	retval = sd_write_data(chip, SD_TM_AUTO_WRITE_3,
-			NULL, 0, byte_cnt, 1, bus_width, buf, len, 100);
+	retval = sd_write_data(chip, SD_TM_AUTO_WRITE_3, NULL, 0, byte_cnt, 1,
+			       bus_width, buf, len, 100);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_clear_sd_error(chip);
 		rtsx_write_register(chip, REG_SD_CFG3, 0x02, 0);
@@ -2980,23 +2990,23 @@ static int mmc_test_switch_bus(struct rtsx_chip *chip, u8 width)
 
 	if (width == MMC_8BIT_BUS)
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_L,
-			0xFF, 0x08);
+			     0xFF, 0x08);
 	else
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_L,
-			0xFF, 0x04);
+			     0xFF, 0x04);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_L, 0xFF, 1);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_H, 0xFF, 0);
 
-	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG2, 0xFF,
-		SD_CALCULATE_CRC7 | SD_NO_CHECK_CRC16 | SD_NO_WAIT_BUSY_END|
-		SD_CHECK_CRC7 | SD_RSP_LEN_6);
+	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG2, 0xFF, SD_CALCULATE_CRC7 |
+		     SD_NO_CHECK_CRC16 | SD_NO_WAIT_BUSY_END |
+		     SD_CHECK_CRC7 | SD_RSP_LEN_6);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE, 0x01,
-		PINGPONG_BUFFER);
+		     PINGPONG_BUFFER);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_TRANSFER, 0xFF,
-		SD_TM_NORMAL_READ | SD_TRANSFER_START);
+		     SD_TM_NORMAL_READ | SD_TRANSFER_START);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, REG_SD_TRANSFER, SD_TRANSFER_END,
-		SD_TRANSFER_END);
+		     SD_TRANSFER_END);
 
 	rtsx_add_cmd(chip, READ_REG_CMD, PPBUF_BASE2, 0, 0);
 	if (width == MMC_8BIT_BUS)
@@ -3024,9 +3034,9 @@ static int mmc_test_switch_bus(struct rtsx_chip *chip, u8 width)
 				arg = 0x03B70200;
 
 			retval = sd_send_cmd_get_rsp(chip, SWITCH, arg,
-						SD_RSP_TYPE_R1b, rsp, 5);
+						     SD_RSP_TYPE_R1b, rsp, 5);
 			if ((retval == STATUS_SUCCESS) &&
-				!(rsp[4] & MMC_SWITCH_ERR))
+			    !(rsp[4] & MMC_SWITCH_ERR))
 				return SWITCH_SUCCESS;
 		}
 	} else {
@@ -3041,9 +3051,9 @@ static int mmc_test_switch_bus(struct rtsx_chip *chip, u8 width)
 				arg = 0x03B70100;
 
 			retval = sd_send_cmd_get_rsp(chip, SWITCH, arg,
-						SD_RSP_TYPE_R1b, rsp, 5);
+						     SD_RSP_TYPE_R1b, rsp, 5);
 			if ((retval == STATUS_SUCCESS) &&
-				!(rsp[4] & MMC_SWITCH_ERR))
+			    !(rsp[4] & MMC_SWITCH_ERR))
 				return SWITCH_SUCCESS;
 		}
 	}
@@ -3054,7 +3064,7 @@ static int mmc_test_switch_bus(struct rtsx_chip *chip, u8 width)
 
 static int mmc_switch_timing_bus(struct rtsx_chip *chip, bool switch_ddr)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 	u8 *ptr, card_type, card_type_mask = 0;
 
@@ -3065,7 +3075,7 @@ static int mmc_switch_timing_bus(struct rtsx_chip *chip, bool switch_ddr)
 	rtsx_init_cmd(chip);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD0, 0xFF,
-		0x40 | SEND_EXT_CSD);
+		     0x40 | SEND_EXT_CSD);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD1, 0xFF, 0);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD2, 0xFF, 0);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD3, 0xFF, 0);
@@ -3077,14 +3087,14 @@ static int mmc_switch_timing_bus(struct rtsx_chip *chip, bool switch_ddr)
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_H, 0xFF, 0);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG2, 0xFF,
-		SD_CALCULATE_CRC7 | SD_CHECK_CRC16 | SD_NO_WAIT_BUSY_END|
-		SD_CHECK_CRC7 | SD_RSP_LEN_6);
+		     SD_CALCULATE_CRC7 | SD_CHECK_CRC16 | SD_NO_WAIT_BUSY_END |
+		     SD_CHECK_CRC7 | SD_RSP_LEN_6);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE, 0x01,
-		PINGPONG_BUFFER);
+		     PINGPONG_BUFFER);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_TRANSFER, 0xFF,
-		SD_TM_NORMAL_READ | SD_TRANSFER_START);
+		     SD_TM_NORMAL_READ | SD_TRANSFER_START);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, REG_SD_TRANSFER, SD_TRANSFER_END,
-		SD_TRANSFER_END);
+		     SD_TRANSFER_END);
 
 	rtsx_add_cmd(chip, READ_REG_CMD, PPBUF_BASE2 + 196, 0xFF, 0);
 	rtsx_add_cmd(chip, READ_REG_CMD, PPBUF_BASE2 + 212, 0xFF, 0);
@@ -3097,7 +3107,7 @@ static int mmc_switch_timing_bus(struct rtsx_chip *chip, bool switch_ddr)
 		if (retval == -ETIMEDOUT) {
 			rtsx_clear_sd_error(chip);
 			sd_send_cmd_get_rsp(chip, SEND_STATUS, sd_card->sd_addr,
-					SD_RSP_TYPE_R1, NULL, 0);
+					    SD_RSP_TYPE_R1, NULL, 0);
 		}
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -3106,7 +3116,7 @@ static int mmc_switch_timing_bus(struct rtsx_chip *chip, bool switch_ddr)
 	ptr = rtsx_get_cmd_data(chip);
 	if (ptr[0] & SD_TRANSFER_ERR) {
 		sd_send_cmd_get_rsp(chip, SEND_STATUS, sd_card->sd_addr,
-				SD_RSP_TYPE_R1, NULL, 0);
+				    SD_RSP_TYPE_R1, NULL, 0);
 		rtsx_trace(chip);
 		return STATUS_FAIL;
 	}
@@ -3132,8 +3142,8 @@ static int mmc_switch_timing_bus(struct rtsx_chip *chip, bool switch_ddr)
 			SET_MMC_26M(sd_card);
 		}
 
-		retval = sd_send_cmd_get_rsp(chip, SWITCH,
-				0x03B90100, SD_RSP_TYPE_R1b, rsp, 5);
+		retval = sd_send_cmd_get_rsp(chip, SWITCH, 0x03B90100,
+					     SD_RSP_TYPE_R1b, rsp, 5);
 		if ((retval != STATUS_SUCCESS) || (rsp[4] & MMC_SWITCH_ERR))
 			CLR_MMC_HS(sd_card);
 	}
@@ -3178,7 +3188,7 @@ static int mmc_switch_timing_bus(struct rtsx_chip *chip, bool switch_ddr)
 
 static int reset_mmc(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval, i = 0, j = 0, k = 0;
 	bool switch_ddr = true;
 	u8 rsp[16];
@@ -3190,7 +3200,7 @@ static int reset_mmc(struct rtsx_chip *chip)
 		goto MMC_UNLOCK_ENTRY;
 #endif
 
-Switch_Fail:
+switch_fail:
 	retval = sd_prepare_reset(chip);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
@@ -3201,7 +3211,7 @@ Switch_Fail:
 
 RTY_MMC_RST:
 	retval = sd_send_cmd_get_rsp(chip, GO_IDLE_STATE, 0, SD_RSP_TYPE_R0,
-				NULL, 0);
+				     NULL, 0);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -3215,11 +3225,11 @@ RTY_MMC_RST:
 		}
 
 		retval = sd_send_cmd_get_rsp(chip, SEND_OP_COND,
-					(SUPPORT_VOLTAGE | 0x40000000),
-					SD_RSP_TYPE_R3, rsp, 5);
+					     (SUPPORT_VOLTAGE | 0x40000000),
+					     SD_RSP_TYPE_R3, rsp, 5);
 		if (retval != STATUS_SUCCESS) {
 			if (sd_check_err_code(chip, SD_BUSY) ||
-				sd_check_err_code(chip, SD_TO_ERR)) {
+			    sd_check_err_code(chip, SD_TO_ERR)) {
 				k++;
 				if (k < 20) {
 					sd_clr_err_code(chip);
@@ -3255,7 +3265,7 @@ RTY_MMC_RST:
 		CLR_MMC_SECTOR_MODE(sd_card);
 
 	retval = sd_send_cmd_get_rsp(chip, ALL_SEND_CID, 0, SD_RSP_TYPE_R2,
-				NULL, 0);
+				     NULL, 0);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -3263,7 +3273,7 @@ RTY_MMC_RST:
 
 	sd_card->sd_addr = 0x00100000;
 	retval = sd_send_cmd_get_rsp(chip, SET_RELATIVE_ADDR, sd_card->sd_addr,
-				SD_RSP_TYPE_R6, rsp, 5);
+				     SD_RSP_TYPE_R6, rsp, 5);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -3284,7 +3294,7 @@ RTY_MMC_RST:
 	}
 
 	retval = sd_send_cmd_get_rsp(chip, SET_BLOCKLEN, 0x200, SD_RSP_TYPE_R1,
-				NULL, 0);
+				     NULL, 0);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
 		return STATUS_FAIL;
@@ -3319,7 +3329,7 @@ MMC_UNLOCK_ENTRY:
 				}
 				sd_card->mmc_dont_switch_bus = 1;
 				rtsx_trace(chip);
-				goto Switch_Fail;
+				goto switch_fail;
 			}
 		}
 
@@ -3345,7 +3355,7 @@ MMC_UNLOCK_ENTRY:
 
 				switch_ddr = false;
 				rtsx_trace(chip);
-				goto Switch_Fail;
+				goto switch_fail;
 			}
 
 			retval = sd_wait_state_data_ready(chip, 0x08, 1, 1000);
@@ -3360,7 +3370,7 @@ MMC_UNLOCK_ENTRY:
 
 					switch_ddr = false;
 					rtsx_trace(chip);
-					goto Switch_Fail;
+					goto switch_fail;
 				}
 			}
 		}
@@ -3392,7 +3402,7 @@ MMC_UNLOCK_ENTRY:
 
 int reset_sd_card(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 
 	sd_init_reg_addr(chip);
@@ -3407,7 +3417,7 @@ int reset_sd_card(struct rtsx_chip *chip)
 	}
 
 	if (chip->ignore_sd && CHK_SDIO_EXIST(chip) &&
-		!CHK_SDIO_IGNORED(chip)) {
+	    !CHK_SDIO_IGNORED(chip)) {
 		if (chip->asic_code) {
 			retval = sd_pull_ctl_enable(chip);
 			if (retval != STATUS_SUCCESS) {
@@ -3416,7 +3426,8 @@ int reset_sd_card(struct rtsx_chip *chip)
 			}
 		} else {
 			retval = rtsx_write_register(chip, FPGA_PULL_CTL,
-						FPGA_SD_PULL_CTL_BIT | 0x20, 0);
+						     FPGA_SD_PULL_CTL_BIT |
+						     0x20, 0);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
 				return STATUS_FAIL;
@@ -3505,7 +3516,7 @@ int reset_sd_card(struct rtsx_chip *chip)
 
 static int reset_mmc_only(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 
 	sd_card->sd_type = 0;
@@ -3574,7 +3585,7 @@ static int reset_mmc_only(struct rtsx_chip *chip)
 
 static int wait_data_buf_ready(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int i, retval;
 
 	for (i = 0; i < WAIT_DATA_READY_RTY_CNT; i++) {
@@ -3587,7 +3598,8 @@ static int wait_data_buf_ready(struct rtsx_chip *chip)
 		sd_card->sd_data_buf_ready = 0;
 
 		retval = sd_send_cmd_get_rsp(chip, SEND_STATUS,
-				sd_card->sd_addr, SD_RSP_TYPE_R1, NULL, 0);
+					     sd_card->sd_addr, SD_RSP_TYPE_R1,
+					     NULL, 0);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -3607,7 +3619,7 @@ static int wait_data_buf_ready(struct rtsx_chip *chip)
 
 void sd_stop_seq_mode(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 
 	if (sd_card->seq_mode) {
@@ -3616,7 +3628,7 @@ void sd_stop_seq_mode(struct rtsx_chip *chip)
 			return;
 
 		retval = sd_send_cmd_get_rsp(chip, STOP_TRANSMISSION, 0,
-				SD_RSP_TYPE_R1b, NULL, 0);
+					     SD_RSP_TYPE_R1b, NULL, 0);
 		if (retval != STATUS_SUCCESS)
 			sd_set_err_code(chip, SD_STS_ERR);
 
@@ -3632,7 +3644,7 @@ void sd_stop_seq_mode(struct rtsx_chip *chip)
 
 static inline int sd_auto_tune_clock(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 
 	if (chip->asic_code) {
@@ -3679,9 +3691,9 @@ static inline int sd_auto_tune_clock(struct rtsx_chip *chip)
 }
 
 int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip, u32 start_sector,
-	u16 sector_cnt)
+	  u16 sector_cnt)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	u32 data_addr;
 	u8 cfg2;
 	int retval;
@@ -3730,20 +3742,20 @@ int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip, u32 start_sector,
 	}
 
 	if (sd_card->seq_mode &&
-		((sd_card->pre_dir != srb->sc_data_direction) ||
-			((sd_card->pre_sec_addr + sd_card->pre_sec_cnt) !=
-				start_sector))) {
-		if ((sd_card->pre_sec_cnt < 0x80)
-				&& (sd_card->pre_dir == DMA_FROM_DEVICE)
-				&& !CHK_SD30_SPEED(sd_card)
-				&& !CHK_SD_HS(sd_card)
-				&& !CHK_MMC_HS(sd_card)) {
+	    ((sd_card->pre_dir != srb->sc_data_direction) ||
+	    ((sd_card->pre_sec_addr + sd_card->pre_sec_cnt) !=
+	    start_sector))) {
+		if ((sd_card->pre_sec_cnt < 0x80) &&
+		    (sd_card->pre_dir == DMA_FROM_DEVICE) &&
+		    !CHK_SD30_SPEED(sd_card) &&
+		    !CHK_SD_HS(sd_card) &&
+		    !CHK_MMC_HS(sd_card)) {
 			sd_send_cmd_get_rsp(chip, SEND_STATUS, sd_card->sd_addr,
-					SD_RSP_TYPE_R1, NULL, 0);
+					    SD_RSP_TYPE_R1, NULL, 0);
 		}
 
-		retval = sd_send_cmd_get_rsp(chip, STOP_TRANSMISSION,
-				0, SD_RSP_TYPE_R1b, NULL, 0);
+		retval = sd_send_cmd_get_rsp(chip, STOP_TRANSMISSION, 0,
+					     SD_RSP_TYPE_R1b, NULL, 0);
 		if (retval != STATUS_SUCCESS) {
 			chip->rw_need_retry = 1;
 			sd_set_err_code(chip, SD_STS_ERR);
@@ -3760,12 +3772,12 @@ int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip, u32 start_sector,
 			goto RW_FAIL;
 		}
 
-		if ((sd_card->pre_sec_cnt < 0x80)
-				&& !CHK_SD30_SPEED(sd_card)
-				&& !CHK_SD_HS(sd_card)
-				&& !CHK_MMC_HS(sd_card)) {
+		if ((sd_card->pre_sec_cnt < 0x80) &&
+		    !CHK_SD30_SPEED(sd_card) &&
+		    !CHK_SD_HS(sd_card) &&
+		    !CHK_MMC_HS(sd_card)) {
 			sd_send_cmd_get_rsp(chip, SEND_STATUS, sd_card->sd_addr,
-					SD_RSP_TYPE_R1, NULL, 0);
+					    SD_RSP_TYPE_R1, NULL, 0);
 		}
 	}
 
@@ -3774,30 +3786,30 @@ int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip, u32 start_sector,
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_L, 0xFF, 0x00);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_H, 0xFF, 0x02);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_L, 0xFF,
-		(u8)sector_cnt);
+		     (u8)sector_cnt);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_H, 0xFF,
-		(u8)(sector_cnt >> 8));
+		     (u8)(sector_cnt >> 8));
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE, 0x01, RING_BUFFER);
 
 	if (CHK_MMC_8BIT(sd_card))
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG1,
-			0x03, SD_BUS_WIDTH_8);
+			     0x03, SD_BUS_WIDTH_8);
 	else if (CHK_MMC_4BIT(sd_card) || CHK_SD(sd_card))
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG1,
-			0x03, SD_BUS_WIDTH_4);
+			     0x03, SD_BUS_WIDTH_4);
 	else
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG1,
-			0x03, SD_BUS_WIDTH_1);
+			     0x03, SD_BUS_WIDTH_1);
 
 	if (sd_card->seq_mode) {
-		cfg2 = SD_NO_CALCULATE_CRC7 | SD_CHECK_CRC16|
+		cfg2 = SD_NO_CALCULATE_CRC7 | SD_CHECK_CRC16 |
 			SD_NO_WAIT_BUSY_END | SD_NO_CHECK_CRC7 |
 			SD_RSP_LEN_0;
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG2, 0xFF, cfg2);
 
 		trans_dma_enable(srb->sc_data_direction, chip, sector_cnt * 512,
-				DMA_512);
+				 DMA_512);
 
 		if (srb->sc_data_direction == DMA_FROM_DEVICE) {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_TRANSFER, 0xFF,
@@ -3808,7 +3820,7 @@ int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip, u32 start_sector,
 		}
 
 		rtsx_add_cmd(chip, CHECK_REG_CMD, REG_SD_TRANSFER,
-			SD_TRANSFER_END, SD_TRANSFER_END);
+			     SD_TRANSFER_END, SD_TRANSFER_END);
 
 		rtsx_send_cmd_no_wait(chip);
 	} else {
@@ -3818,22 +3830,22 @@ int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip, u32 start_sector,
 			rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD0, 0xFF,
 				     0x40 | READ_MULTIPLE_BLOCK);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD1, 0xFF,
-				(u8)(data_addr >> 24));
+				     (u8)(data_addr >> 24));
 			rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD2, 0xFF,
-				(u8)(data_addr >> 16));
+				     (u8)(data_addr >> 16));
 			rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD3, 0xFF,
-				(u8)(data_addr >> 8));
+				     (u8)(data_addr >> 8));
 			rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD4, 0xFF,
-				(u8)data_addr);
+				     (u8)data_addr);
 
 			cfg2 = SD_CALCULATE_CRC7 | SD_CHECK_CRC16 |
 				SD_NO_WAIT_BUSY_END | SD_CHECK_CRC7 |
 				SD_RSP_LEN_6;
 			rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG2, 0xFF,
-				cfg2);
+				     cfg2);
 
 			trans_dma_enable(srb->sc_data_direction, chip,
-					sector_cnt * 512, DMA_512);
+					 sector_cnt * 512, DMA_512);
 
 			rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_TRANSFER, 0xFF,
 				     SD_TM_AUTO_READ_2 | SD_TRANSFER_START);
@@ -3861,7 +3873,8 @@ int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip, u32 start_sector,
 			}
 
 			retval = sd_send_cmd_get_rsp(chip, WRITE_MULTIPLE_BLOCK,
-					data_addr, SD_RSP_TYPE_R1, NULL, 0);
+						     data_addr, SD_RSP_TYPE_R1,
+						     NULL, 0);
 			if (retval != STATUS_SUCCESS) {
 				chip->rw_need_retry = 1;
 				rtsx_trace(chip);
@@ -3874,10 +3887,10 @@ int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip, u32 start_sector,
 				SD_NO_WAIT_BUSY_END |
 				SD_NO_CHECK_CRC7 | SD_RSP_LEN_0;
 			rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG2, 0xFF,
-				cfg2);
+				     cfg2);
 
 			trans_dma_enable(srb->sc_data_direction, chip,
-					sector_cnt * 512, DMA_512);
+					 sector_cnt * 512, DMA_512);
 
 			rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_TRANSFER, 0xFF,
 				     SD_TM_AUTO_WRITE_3 | SD_TRANSFER_START);
@@ -3891,7 +3904,7 @@ int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip, u32 start_sector,
 	}
 
 	retval = rtsx_transfer_data(chip, SD_CARD, scsi_sglist(srb),
-				scsi_bufflen(srb), scsi_sg_count(srb),
+				    scsi_bufflen(srb), scsi_sg_count(srb),
 				srb->sc_data_direction, chip->sd_timeout);
 	if (retval < 0) {
 		u8 stat = 0;
@@ -3916,7 +3929,7 @@ int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip, u32 start_sector,
 		chip->rw_need_retry = 1;
 
 		retval = sd_send_cmd_get_rsp(chip, STOP_TRANSMISSION, 0,
-					SD_RSP_TYPE_R1b, NULL, 0);
+					     SD_RSP_TYPE_R1b, NULL, 0);
 		if (retval != STATUS_SUCCESS) {
 			sd_set_err_code(chip, SD_STS_ERR);
 			rtsx_trace(chip);
@@ -3984,8 +3997,9 @@ int soft_reset_sd_card(struct rtsx_chip *chip)
 	return reset_sd(chip);
 }
 
-int ext_sd_send_cmd_get_rsp(struct rtsx_chip *chip, u8 cmd_idx,
-		u32 arg, u8 rsp_type, u8 *rsp, int rsp_len, bool special_check)
+int ext_sd_send_cmd_get_rsp(struct rtsx_chip *chip, u8 cmd_idx, u32 arg,
+			    u8 rsp_type, u8 *rsp, int rsp_len,
+			    bool special_check)
 {
 	int retval;
 	int timeout = 100;
@@ -4011,11 +4025,11 @@ RTY_SEND_CMD:
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG2, 0xFF, rsp_type);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE,
-			0x01, PINGPONG_BUFFER);
+		     0x01, PINGPONG_BUFFER);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_TRANSFER,
-			0xFF, SD_TM_CMD_RSP | SD_TRANSFER_START);
+		     0xFF, SD_TM_CMD_RSP | SD_TRANSFER_START);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, REG_SD_TRANSFER, SD_TRANSFER_END,
-		SD_TRANSFER_END);
+		     SD_TRANSFER_END);
 
 	if (rsp_type == SD_RSP_TYPE_R2) {
 		for (reg_addr = PPBUF_BASE2; reg_addr < PPBUF_BASE2 + 16;
@@ -4084,7 +4098,7 @@ RTY_SEND_CMD:
 	}
 
 	if ((cmd_idx == SELECT_CARD) || (cmd_idx == APP_CMD) ||
-		(cmd_idx == SEND_STATUS) || (cmd_idx == STOP_TRANSMISSION)) {
+	    (cmd_idx == SEND_STATUS) || (cmd_idx == STOP_TRANSMISSION)) {
 		if ((cmd_idx != STOP_TRANSMISSION) && !special_check) {
 			if (ptr[1] & 0x80) {
 				rtsx_trace(chip);
@@ -4172,7 +4186,7 @@ int ext_sd_get_rsp(struct rtsx_chip *chip, int len, u8 *rsp, u8 rsp_type)
 
 int sd_pass_thru_mode(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	unsigned int lun = SCSI_LUN(srb);
 	int len;
 	u8 buf[18] = {
@@ -4206,9 +4220,9 @@ int sd_pass_thru_mode(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	}
 
 	if ((srb->cmnd[2] != 0x53) || (srb->cmnd[3] != 0x44) ||
-		(srb->cmnd[4] != 0x20) || (srb->cmnd[5] != 0x43) ||
-		(srb->cmnd[6] != 0x61) || (srb->cmnd[7] != 0x72) ||
-		(srb->cmnd[8] != 0x64)) {
+	    (srb->cmnd[4] != 0x20) || (srb->cmnd[5] != 0x43) ||
+	    (srb->cmnd[6] != 0x61) || (srb->cmnd[7] != 0x72) ||
+	    (srb->cmnd[8] != 0x64)) {
 		set_sense_type(chip, lun, SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
@@ -4245,7 +4259,7 @@ int sd_pass_thru_mode(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 }
 
 static inline int get_rsp_type(struct scsi_cmnd *srb, u8 *rsp_type,
-			int *rsp_len)
+			       int *rsp_len)
 {
 	if (!rsp_type || !rsp_len)
 		return STATUS_FAIL;
@@ -4285,7 +4299,7 @@ static inline int get_rsp_type(struct scsi_cmnd *srb, u8 *rsp_type,
 
 int sd_execute_no_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	unsigned int lun = SCSI_LUN(srb);
 	int retval, rsp_len;
 	u8 cmd_idx, rsp_type;
@@ -4339,7 +4353,7 @@ int sd_execute_no_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	if ((sd_card->sd_lock_status & SD_LOCK_1BIT_MODE) == 0) {
 		if (CHK_MMC_8BIT(sd_card)) {
 			retval = rtsx_write_register(chip, REG_SD_CFG1, 0x03,
-						SD_BUS_WIDTH_8);
+						     SD_BUS_WIDTH_8);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
 				return TRANSPORT_FAILED;
@@ -4347,7 +4361,7 @@ int sd_execute_no_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		} else if (CHK_SD(sd_card) || CHK_MMC_4BIT(sd_card)) {
 			retval = rtsx_write_register(chip, REG_SD_CFG1, 0x03,
-						SD_BUS_WIDTH_4);
+						     SD_BUS_WIDTH_4);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
 				return TRANSPORT_FAILED;
@@ -4366,32 +4380,33 @@ int sd_execute_no_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		retval = sd_select_card(chip, 0);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Cmd_Failed;
+			goto sd_execute_cmd_failed;
 		}
 	}
 
 	if (acmd) {
 		retval = ext_sd_send_cmd_get_rsp(chip, APP_CMD,
-						sd_card->sd_addr,
-						SD_RSP_TYPE_R1, NULL, 0, false);
+						 sd_card->sd_addr,
+						 SD_RSP_TYPE_R1, NULL, 0,
+						 false);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Cmd_Failed;
+			goto sd_execute_cmd_failed;
 		}
 	}
 
 	retval = ext_sd_send_cmd_get_rsp(chip, cmd_idx, arg, rsp_type,
-			sd_card->rsp, rsp_len, false);
+					 sd_card->rsp, rsp_len, false);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
-		goto SD_Execute_Cmd_Failed;
+		goto sd_execute_cmd_failed;
 	}
 
 	if (standby) {
 		retval = sd_select_card(chip, 1);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Cmd_Failed;
+			goto sd_execute_cmd_failed;
 		}
 	}
 
@@ -4399,14 +4414,14 @@ int sd_execute_no_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	retval = sd_update_lock_status(chip);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
-		goto SD_Execute_Cmd_Failed;
+		goto sd_execute_cmd_failed;
 	}
 #endif
 
 	scsi_set_resid(srb, 0);
 	return TRANSPORT_GOOD;
 
-SD_Execute_Cmd_Failed:
+sd_execute_cmd_failed:
 	sd_card->pre_cmd_err = 1;
 	set_sense_type(chip, lun, SENSE_TYPE_NO_SENSE);
 	release_sd_card(chip);
@@ -4420,7 +4435,7 @@ SD_Execute_Cmd_Failed:
 
 int sd_execute_read_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	unsigned int lun = SCSI_LUN(srb);
 	int retval, rsp_len, i;
 	bool read_err = false, cmd13_checkbit = false;
@@ -4492,10 +4507,11 @@ int sd_execute_read_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	if (data_len < 512) {
 		retval = ext_sd_send_cmd_get_rsp(chip, SET_BLOCKLEN, data_len,
-				SD_RSP_TYPE_R1, NULL, 0, false);
+						 SD_RSP_TYPE_R1, NULL, 0,
+						 false);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Read_Cmd_Failed;
+			goto sd_execute_read_cmd_failed;
 		}
 	}
 
@@ -4503,17 +4519,18 @@ int sd_execute_read_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		retval = sd_select_card(chip, 0);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Read_Cmd_Failed;
+			goto sd_execute_read_cmd_failed;
 		}
 	}
 
 	if (acmd) {
 		retval = ext_sd_send_cmd_get_rsp(chip, APP_CMD,
-						sd_card->sd_addr,
-						SD_RSP_TYPE_R1, NULL, 0, false);
+						 sd_card->sd_addr,
+						 SD_RSP_TYPE_R1, NULL, 0,
+						 false);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Read_Cmd_Failed;
+			goto sd_execute_read_cmd_failed;
 		}
 	}
 
@@ -4539,13 +4556,13 @@ int sd_execute_read_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		}
 
 		retval = sd_read_data(chip, SD_TM_NORMAL_READ, cmd, 5, byte_cnt,
-				       blk_cnt, bus_width, buf, data_len, 2000);
+				      blk_cnt, bus_width, buf, data_len, 2000);
 		if (retval != STATUS_SUCCESS) {
 			read_err = true;
 			kfree(buf);
 			rtsx_clear_sd_error(chip);
 			rtsx_trace(chip);
-			goto SD_Execute_Read_Cmd_Failed;
+			goto sd_execute_read_cmd_failed;
 		}
 
 		min_len = min(data_len, scsi_bufflen(srb));
@@ -4558,24 +4575,24 @@ int sd_execute_read_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		trans_dma_enable(DMA_FROM_DEVICE, chip, data_len, DMA_512);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_H, 0xFF,
-			0x02);
+			     0x02);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_L, 0xFF,
-			0x00);
+			     0x00);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_H,
-				0xFF, (srb->cmnd[7] & 0xFE) >> 1);
+			     0xFF, (srb->cmnd[7] & 0xFE) >> 1);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_L,
-				0xFF, (u8)((data_len & 0x0001FE00) >> 9));
+			     0xFF, (u8)((data_len & 0x0001FE00) >> 9));
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD0, 0xFF,
-			0x40 | cmd_idx);
+			     0x40 | cmd_idx);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD1, 0xFF,
-			srb->cmnd[3]);
+			     srb->cmnd[3]);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD2, 0xFF,
-			srb->cmnd[4]);
+			     srb->cmnd[4]);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD3, 0xFF,
-			srb->cmnd[5]);
+			     srb->cmnd[5]);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CMD4, 0xFF,
-			srb->cmnd[6]);
+			     srb->cmnd[6]);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG1, 0x03, bus_width);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_CFG2, 0xFF, rsp_type);
@@ -4583,66 +4600,69 @@ int sd_execute_read_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_TRANSFER,
 			     0xFF, SD_TM_AUTO_READ_2 | SD_TRANSFER_START);
 		rtsx_add_cmd(chip, CHECK_REG_CMD, REG_SD_TRANSFER,
-			SD_TRANSFER_END, SD_TRANSFER_END);
+			     SD_TRANSFER_END, SD_TRANSFER_END);
 
 		rtsx_send_cmd_no_wait(chip);
 
 		retval = rtsx_transfer_data(chip, SD_CARD, scsi_sglist(srb),
-					scsi_bufflen(srb), scsi_sg_count(srb),
-					DMA_FROM_DEVICE, 10000);
+					    scsi_bufflen(srb),
+					    scsi_sg_count(srb),
+					    DMA_FROM_DEVICE, 10000);
 		if (retval < 0) {
 			read_err = true;
 			rtsx_clear_sd_error(chip);
 			rtsx_trace(chip);
-			goto SD_Execute_Read_Cmd_Failed;
+			goto sd_execute_read_cmd_failed;
 		}
 
 	} else {
 		rtsx_trace(chip);
-		goto SD_Execute_Read_Cmd_Failed;
+		goto sd_execute_read_cmd_failed;
 	}
 
 	retval = ext_sd_get_rsp(chip, rsp_len, sd_card->rsp, rsp_type);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
-		goto SD_Execute_Read_Cmd_Failed;
+		goto sd_execute_read_cmd_failed;
 	}
 
 	if (standby) {
 		retval = sd_select_card(chip, 1);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Read_Cmd_Failed;
+			goto sd_execute_read_cmd_failed;
 		}
 	}
 
 	if (send_cmd12) {
-		retval = ext_sd_send_cmd_get_rsp(chip, STOP_TRANSMISSION,
-				0, SD_RSP_TYPE_R1b, NULL, 0, false);
+		retval = ext_sd_send_cmd_get_rsp(chip, STOP_TRANSMISSION, 0,
+						 SD_RSP_TYPE_R1b, NULL, 0,
+						 false);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Read_Cmd_Failed;
+			goto sd_execute_read_cmd_failed;
 		}
 	}
 
 	if (data_len < 512) {
 		retval = ext_sd_send_cmd_get_rsp(chip, SET_BLOCKLEN, 0x200,
-				SD_RSP_TYPE_R1, NULL, 0, false);
+						 SD_RSP_TYPE_R1, NULL, 0,
+						 false);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Read_Cmd_Failed;
+			goto sd_execute_read_cmd_failed;
 		}
 
 		retval = rtsx_write_register(chip, SD_BYTE_CNT_H, 0xFF, 0x02);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Read_Cmd_Failed;
+			goto sd_execute_read_cmd_failed;
 		}
 
 		retval = rtsx_write_register(chip, SD_BYTE_CNT_L, 0xFF, 0x00);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Read_Cmd_Failed;
+			goto sd_execute_read_cmd_failed;
 		}
 	}
 
@@ -4651,7 +4671,7 @@ int sd_execute_read_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	for (i = 0; i < 3; i++) {
 		retval = ext_sd_send_cmd_get_rsp(chip, SEND_STATUS,
-						sd_card->sd_addr,
+						 sd_card->sd_addr,
 						SD_RSP_TYPE_R1, NULL, 0,
 						cmd13_checkbit);
 		if (retval == STATUS_SUCCESS)
@@ -4659,13 +4679,13 @@ int sd_execute_read_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	}
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
-		goto SD_Execute_Read_Cmd_Failed;
+		goto sd_execute_read_cmd_failed;
 	}
 
 	scsi_set_resid(srb, 0);
 	return TRANSPORT_GOOD;
 
-SD_Execute_Read_Cmd_Failed:
+sd_execute_read_cmd_failed:
 	sd_card->pre_cmd_err = 1;
 	set_sense_type(chip, lun, SENSE_TYPE_NO_SENSE);
 	if (read_err)
@@ -4682,7 +4702,7 @@ SD_Execute_Read_Cmd_Failed:
 
 int sd_execute_write_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	unsigned int lun = SCSI_LUN(srb);
 	int retval, rsp_len, i;
 	bool write_err = false, cmd13_checkbit = false;
@@ -4754,7 +4774,7 @@ int sd_execute_write_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	if ((sd_card->sd_lock_status & SD_LOCK_1BIT_MODE) == 0) {
 		if (CHK_MMC_8BIT(sd_card)) {
 			retval = rtsx_write_register(chip, REG_SD_CFG1, 0x03,
-						SD_BUS_WIDTH_8);
+						     SD_BUS_WIDTH_8);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
 				return TRANSPORT_FAILED;
@@ -4762,7 +4782,7 @@ int sd_execute_write_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		} else if (CHK_SD(sd_card) || CHK_MMC_4BIT(sd_card)) {
 			retval = rtsx_write_register(chip, REG_SD_CFG1, 0x03,
-						SD_BUS_WIDTH_4);
+						     SD_BUS_WIDTH_4);
 			if (retval != STATUS_SUCCESS) {
 				rtsx_trace(chip);
 				return TRANSPORT_FAILED;
@@ -4779,10 +4799,11 @@ int sd_execute_write_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	if (data_len < 512) {
 		retval = ext_sd_send_cmd_get_rsp(chip, SET_BLOCKLEN, data_len,
-				SD_RSP_TYPE_R1, NULL, 0, false);
+						 SD_RSP_TYPE_R1, NULL, 0,
+						 false);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Write_Cmd_Failed;
+			goto sd_execute_write_cmd_failed;
 		}
 	}
 
@@ -4790,25 +4811,26 @@ int sd_execute_write_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		retval = sd_select_card(chip, 0);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Write_Cmd_Failed;
+			goto sd_execute_write_cmd_failed;
 		}
 	}
 
 	if (acmd) {
 		retval = ext_sd_send_cmd_get_rsp(chip, APP_CMD,
-						sd_card->sd_addr,
-						SD_RSP_TYPE_R1, NULL, 0, false);
+						 sd_card->sd_addr,
+						 SD_RSP_TYPE_R1, NULL, 0,
+						 false);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Write_Cmd_Failed;
+			goto sd_execute_write_cmd_failed;
 		}
 	}
 
 	retval = ext_sd_send_cmd_get_rsp(chip, cmd_idx, arg, rsp_type,
-			sd_card->rsp, rsp_len, false);
+					 sd_card->rsp, rsp_len, false);
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
-		goto SD_Execute_Write_Cmd_Failed;
+		goto sd_execute_write_cmd_failed;
 	}
 
 	if (data_len <= 512) {
@@ -4832,37 +4854,37 @@ int sd_execute_write_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 			rtsx_init_cmd(chip);
 			for (i = 0; i < 256; i++) {
 				rtsx_add_cmd(chip, WRITE_REG_CMD,
-						PPBUF_BASE2 + i, 0xFF, buf[i]);
+					     PPBUF_BASE2 + i, 0xFF, buf[i]);
 			}
 			retval = rtsx_send_cmd(chip, 0, 250);
 			if (retval != STATUS_SUCCESS) {
 				kfree(buf);
 				rtsx_trace(chip);
-				goto SD_Execute_Write_Cmd_Failed;
+				goto sd_execute_write_cmd_failed;
 			}
 
 			rtsx_init_cmd(chip);
 			for (i = 256; i < data_len; i++) {
 				rtsx_add_cmd(chip, WRITE_REG_CMD,
-						PPBUF_BASE2 + i, 0xFF, buf[i]);
+					     PPBUF_BASE2 + i, 0xFF, buf[i]);
 			}
 			retval = rtsx_send_cmd(chip, 0, 250);
 			if (retval != STATUS_SUCCESS) {
 				kfree(buf);
 				rtsx_trace(chip);
-				goto SD_Execute_Write_Cmd_Failed;
+				goto sd_execute_write_cmd_failed;
 			}
 		} else {
 			rtsx_init_cmd(chip);
 			for (i = 0; i < data_len; i++) {
 				rtsx_add_cmd(chip, WRITE_REG_CMD,
-						PPBUF_BASE2 + i, 0xFF, buf[i]);
+					     PPBUF_BASE2 + i, 0xFF, buf[i]);
 			}
 			retval = rtsx_send_cmd(chip, 0, 250);
 			if (retval != STATUS_SUCCESS) {
 				kfree(buf);
 				rtsx_trace(chip);
-				goto SD_Execute_Write_Cmd_Failed;
+				goto sd_execute_write_cmd_failed;
 			}
 		}
 
@@ -4871,20 +4893,20 @@ int sd_execute_write_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		rtsx_init_cmd(chip);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_H, 0xFF,
-			srb->cmnd[8] & 0x03);
+			     srb->cmnd[8] & 0x03);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_L, 0xFF,
-			srb->cmnd[9]);
+			     srb->cmnd[9]);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_H, 0xFF,
-			0x00);
+			     0x00);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_L, 0xFF,
-			0x01);
+			     0x01);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE, 0x01,
-			PINGPONG_BUFFER);
+			     PINGPONG_BUFFER);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_TRANSFER, 0xFF,
 			     SD_TM_AUTO_WRITE_3 | SD_TRANSFER_START);
 		rtsx_add_cmd(chip, CHECK_REG_CMD, REG_SD_TRANSFER,
-			SD_TRANSFER_END, SD_TRANSFER_END);
+			     SD_TRANSFER_END, SD_TRANSFER_END);
 
 		retval = rtsx_send_cmd(chip, SD_CARD, 250);
 	} else if (!(data_len & 0x1FF)) {
@@ -4893,35 +4915,36 @@ int sd_execute_write_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		trans_dma_enable(DMA_TO_DEVICE, chip, data_len, DMA_512);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_H, 0xFF,
-			0x02);
+			     0x02);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BYTE_CNT_L, 0xFF,
-			0x00);
+			     0x00);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_H,
-				0xFF, (srb->cmnd[7] & 0xFE) >> 1);
+			     0xFF, (srb->cmnd[7] & 0xFE) >> 1);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_BLOCK_CNT_L,
-				0xFF, (u8)((data_len & 0x0001FE00) >> 9));
+			     0xFF, (u8)((data_len & 0x0001FE00) >> 9));
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, REG_SD_TRANSFER, 0xFF,
-			SD_TM_AUTO_WRITE_3 | SD_TRANSFER_START);
+			     SD_TM_AUTO_WRITE_3 | SD_TRANSFER_START);
 		rtsx_add_cmd(chip, CHECK_REG_CMD, REG_SD_TRANSFER,
-			SD_TRANSFER_END, SD_TRANSFER_END);
+			     SD_TRANSFER_END, SD_TRANSFER_END);
 
 		rtsx_send_cmd_no_wait(chip);
 
 		retval = rtsx_transfer_data(chip, SD_CARD, scsi_sglist(srb),
-					scsi_bufflen(srb), scsi_sg_count(srb),
-					DMA_TO_DEVICE, 10000);
+					    scsi_bufflen(srb),
+					    scsi_sg_count(srb),
+					    DMA_TO_DEVICE, 10000);
 
 	} else {
 		rtsx_trace(chip);
-		goto SD_Execute_Write_Cmd_Failed;
+		goto sd_execute_write_cmd_failed;
 	}
 
 	if (retval < 0) {
 		write_err = true;
 		rtsx_clear_sd_error(chip);
 		rtsx_trace(chip);
-		goto SD_Execute_Write_Cmd_Failed;
+		goto sd_execute_write_cmd_failed;
 	}
 
 #ifdef SUPPORT_SD_LOCK
@@ -4949,37 +4972,39 @@ int sd_execute_write_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		retval = sd_select_card(chip, 1);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Write_Cmd_Failed;
+			goto sd_execute_write_cmd_failed;
 		}
 	}
 
 	if (send_cmd12) {
-		retval = ext_sd_send_cmd_get_rsp(chip, STOP_TRANSMISSION,
-				0, SD_RSP_TYPE_R1b, NULL, 0, false);
+		retval = ext_sd_send_cmd_get_rsp(chip, STOP_TRANSMISSION, 0,
+						 SD_RSP_TYPE_R1b, NULL, 0,
+						 false);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Write_Cmd_Failed;
+			goto sd_execute_write_cmd_failed;
 		}
 	}
 
 	if (data_len < 512) {
 		retval = ext_sd_send_cmd_get_rsp(chip, SET_BLOCKLEN, 0x200,
-				SD_RSP_TYPE_R1, NULL, 0, false);
+						 SD_RSP_TYPE_R1, NULL, 0,
+						 false);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Write_Cmd_Failed;
+			goto sd_execute_write_cmd_failed;
 		}
 
 		retval = rtsx_write_register(chip, SD_BYTE_CNT_H, 0xFF, 0x02);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Write_Cmd_Failed;
+			goto sd_execute_write_cmd_failed;
 		}
 
 		rtsx_write_register(chip, SD_BYTE_CNT_L, 0xFF, 0x00);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
-			goto SD_Execute_Write_Cmd_Failed;
+			goto sd_execute_write_cmd_failed;
 		}
 	}
 
@@ -4988,15 +5013,15 @@ int sd_execute_write_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 	for (i = 0; i < 3; i++) {
 		retval = ext_sd_send_cmd_get_rsp(chip, SEND_STATUS,
-						sd_card->sd_addr,
-						SD_RSP_TYPE_R1, NULL, 0,
-						cmd13_checkbit);
+						 sd_card->sd_addr,
+						 SD_RSP_TYPE_R1, NULL, 0,
+						 cmd13_checkbit);
 		if (retval == STATUS_SUCCESS)
 			break;
 	}
 	if (retval != STATUS_SUCCESS) {
 		rtsx_trace(chip);
-		goto SD_Execute_Write_Cmd_Failed;
+		goto sd_execute_write_cmd_failed;
 	}
 
 #ifdef SUPPORT_SD_LOCK
@@ -5024,7 +5049,7 @@ int sd_execute_write_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 						if (retval != STATUS_SUCCESS) {
 							sd_card->sd_lock_status &= ~(SD_UNLOCK_POW_ON | SD_SDR_RST);
 							rtsx_trace(chip);
-							goto SD_Execute_Write_Cmd_Failed;
+							goto sd_execute_write_cmd_failed;
 						}
 					}
 
@@ -5045,7 +5070,7 @@ int sd_execute_write_data(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	scsi_set_resid(srb, 0);
 	return TRANSPORT_GOOD;
 
-SD_Execute_Write_Cmd_Failed:
+sd_execute_write_cmd_failed:
 	sd_card->pre_cmd_err = 1;
 	set_sense_type(chip, lun, SENSE_TYPE_NO_SENSE);
 	if (write_err)
@@ -5062,7 +5087,7 @@ SD_Execute_Write_Cmd_Failed:
 
 int sd_get_cmd_rsp(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	unsigned int lun = SCSI_LUN(srb);
 	int count;
 	u16 data_len;
@@ -5104,7 +5129,7 @@ int sd_get_cmd_rsp(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 int sd_hw_rst(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	unsigned int lun = SCSI_LUN(srb);
 	int retval;
 
@@ -5122,9 +5147,9 @@ int sd_hw_rst(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	}
 
 	if ((srb->cmnd[2] != 0x53) || (srb->cmnd[3] != 0x44) ||
-		(srb->cmnd[4] != 0x20) || (srb->cmnd[5] != 0x43) ||
-		(srb->cmnd[6] != 0x61) || (srb->cmnd[7] != 0x72) ||
-		(srb->cmnd[8] != 0x64)) {
+	    (srb->cmnd[4] != 0x20) || (srb->cmnd[5] != 0x43) ||
+	    (srb->cmnd[6] != 0x61) || (srb->cmnd[7] != 0x72) ||
+	    (srb->cmnd[8] != 0x64)) {
 		set_sense_type(chip, lun, SENSE_TYPE_MEDIA_INVALID_CMD_FIELD);
 		rtsx_trace(chip);
 		return TRANSPORT_FAILED;
@@ -5174,7 +5199,7 @@ int sd_hw_rst(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 void sd_cleanup_work(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 
 	if (sd_card->seq_mode) {
 		dev_dbg(rtsx_dev(chip), "SD: stop transmission\n");
@@ -5230,7 +5255,7 @@ int sd_power_off_card3v3(struct rtsx_chip *chip)
 
 int release_sd_card(struct rtsx_chip *chip)
 {
-	struct sd_info *sd_card = &(chip->sd_card);
+	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
 
 	chip->card_ready &= ~SD_CARD;
diff --git a/drivers/staging/rts5208/sd.h b/drivers/staging/rts5208/sd.h
index 60b79280fb5f..55764e16b93a 100644
--- a/drivers/staging/rts5208/sd.h
+++ b/drivers/staging/rts5208/sd.h
@@ -280,14 +280,15 @@ int reset_sd_card(struct rtsx_chip *chip);
 int sd_switch_clock(struct rtsx_chip *chip);
 void sd_stop_seq_mode(struct rtsx_chip *chip);
 int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
-	u32 start_sector, u16 sector_cnt);
+	  u32 start_sector, u16 sector_cnt);
 void sd_cleanup_work(struct rtsx_chip *chip);
 int sd_power_off_card3v3(struct rtsx_chip *chip);
 int release_sd_card(struct rtsx_chip *chip);
 #ifdef SUPPORT_CPRM
 int soft_reset_sd_card(struct rtsx_chip *chip);
 int ext_sd_send_cmd_get_rsp(struct rtsx_chip *chip, u8 cmd_idx,
-		u32 arg, u8 rsp_type, u8 *rsp, int rsp_len, bool special_check);
+			    u32 arg, u8 rsp_type, u8 *rsp, int rsp_len,
+			    bool special_check);
 int ext_sd_get_rsp(struct rtsx_chip *chip, int len, u8 *rsp, u8 rsp_type);
 
 int sd_pass_thru_mode(struct scsi_cmnd *srb, struct rtsx_chip *chip);
diff --git a/drivers/staging/rts5208/spi.c b/drivers/staging/rts5208/spi.c
index 13c539c83838..8b8cd955dfeb 100644
--- a/drivers/staging/rts5208/spi.c
+++ b/drivers/staging/rts5208/spi.c
@@ -29,7 +29,7 @@
 
 static inline void spi_set_err_code(struct rtsx_chip *chip, u8 err_code)
 {
-	struct spi_info *spi = &(chip->spi);
+	struct spi_info *spi = &chip->spi;
 
 	spi->err_code = err_code;
 }
@@ -57,7 +57,7 @@ static int spi_init(struct rtsx_chip *chip)
 
 static int spi_set_init_para(struct rtsx_chip *chip)
 {
-	struct spi_info *spi = &(chip->spi);
+	struct spi_info *spi = &chip->spi;
 	int retval;
 
 	retval = rtsx_write_register(chip, SPI_CLK_DIVIDER1, 0xFF,
@@ -117,9 +117,9 @@ static int sf_polling_status(struct rtsx_chip *chip, int msec)
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_COMMAND, 0xFF, SPI_RDSR);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-		SPI_TRANSFER0_START | SPI_POLLING_MODE0);
+		     SPI_TRANSFER0_START | SPI_POLLING_MODE0);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0, SPI_TRANSFER0_END,
-		SPI_TRANSFER0_END);
+		     SPI_TRANSFER0_END);
 
 	retval = rtsx_send_cmd(chip, 0, msec);
 	if (retval < 0) {
@@ -134,7 +134,7 @@ static int sf_polling_status(struct rtsx_chip *chip, int msec)
 
 static int sf_enable_write(struct rtsx_chip *chip, u8 ins)
 {
-	struct spi_info *spi = &(chip->spi);
+	struct spi_info *spi = &chip->spi;
 	int retval;
 
 	if (!spi->write_en)
@@ -144,11 +144,11 @@ static int sf_enable_write(struct rtsx_chip *chip, u8 ins)
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_COMMAND, 0xFF, ins);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF,
-		SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
+		     SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-		SPI_TRANSFER0_START | SPI_C_MODE0);
+		     SPI_TRANSFER0_START | SPI_C_MODE0);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0, SPI_TRANSFER0_END,
-		SPI_TRANSFER0_END);
+		     SPI_TRANSFER0_END);
 
 	retval = rtsx_send_cmd(chip, 0, 100);
 	if (retval < 0) {
@@ -163,7 +163,7 @@ static int sf_enable_write(struct rtsx_chip *chip, u8 ins)
 
 static int sf_disable_write(struct rtsx_chip *chip, u8 ins)
 {
-	struct spi_info *spi = &(chip->spi);
+	struct spi_info *spi = &chip->spi;
 	int retval;
 
 	if (!spi->write_en)
@@ -173,11 +173,11 @@ static int sf_disable_write(struct rtsx_chip *chip, u8 ins)
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_COMMAND, 0xFF, ins);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF,
-		SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
+		     SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-		SPI_TRANSFER0_START | SPI_C_MODE0);
+		     SPI_TRANSFER0_START | SPI_C_MODE0);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0, SPI_TRANSFER0_END,
-		SPI_TRANSFER0_END);
+		     SPI_TRANSFER0_END);
 
 	retval = rtsx_send_cmd(chip, 0, 100);
 	if (retval < 0) {
@@ -191,27 +191,27 @@ static int sf_disable_write(struct rtsx_chip *chip, u8 ins)
 }
 
 static void sf_program(struct rtsx_chip *chip, u8 ins, u8 addr_mode, u32 addr,
-		u16 len)
+		       u16 len)
 {
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_COMMAND, 0xFF, ins);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF,
-		SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
+		     SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_LENGTH0, 0xFF, (u8)len);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_LENGTH1, 0xFF, (u8)(len >> 8));
 	if (addr_mode) {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR0, 0xFF, (u8)addr);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR1, 0xFF,
-			(u8)(addr >> 8));
+			     (u8)(addr >> 8));
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR2, 0xFF,
-			(u8)(addr >> 16));
+			     (u8)(addr >> 16));
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-			SPI_TRANSFER0_START | SPI_CADO_MODE0);
+			     SPI_TRANSFER0_START | SPI_CADO_MODE0);
 	} else {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-			SPI_TRANSFER0_START | SPI_CDO_MODE0);
+			     SPI_TRANSFER0_START | SPI_CDO_MODE0);
 	}
 	rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0, SPI_TRANSFER0_END,
-		SPI_TRANSFER0_END);
+		     SPI_TRANSFER0_END);
 }
 
 static int sf_erase(struct rtsx_chip *chip, u8 ins, u8 addr_mode, u32 addr)
@@ -222,21 +222,21 @@ static int sf_erase(struct rtsx_chip *chip, u8 ins, u8 addr_mode, u32 addr)
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_COMMAND, 0xFF, ins);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF,
-		SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
+		     SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
 	if (addr_mode) {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR0, 0xFF, (u8)addr);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR1, 0xFF,
-			(u8)(addr >> 8));
+			     (u8)(addr >> 8));
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR2, 0xFF,
-			(u8)(addr >> 16));
+			     (u8)(addr >> 16));
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-			SPI_TRANSFER0_START | SPI_CA_MODE0);
+			     SPI_TRANSFER0_START | SPI_CA_MODE0);
 	} else {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-			SPI_TRANSFER0_START | SPI_C_MODE0);
+			     SPI_TRANSFER0_START | SPI_C_MODE0);
 	}
 	rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0, SPI_TRANSFER0_END,
-		SPI_TRANSFER0_END);
+		     SPI_TRANSFER0_END);
 
 	retval = rtsx_send_cmd(chip, 0, 100);
 	if (retval < 0) {
@@ -322,9 +322,9 @@ static int spi_eeprom_program_enable(struct rtsx_chip *chip)
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF, 0x86);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_COMMAND, 0xFF, 0x13);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-		SPI_TRANSFER0_START | SPI_CA_MODE0);
+		     SPI_TRANSFER0_START | SPI_CA_MODE0);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0, SPI_TRANSFER0_END,
-		SPI_TRANSFER0_END);
+		     SPI_TRANSFER0_END);
 
 	retval = rtsx_send_cmd(chip, 0, 100);
 	if (retval < 0) {
@@ -358,9 +358,9 @@ int spi_erase_eeprom_chip(struct rtsx_chip *chip)
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_COMMAND, 0xFF, 0x12);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF, 0x84);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-		SPI_TRANSFER0_START | SPI_CA_MODE0);
+		     SPI_TRANSFER0_START | SPI_CA_MODE0);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0, SPI_TRANSFER0_END,
-		SPI_TRANSFER0_END);
+		     SPI_TRANSFER0_END);
 
 	retval = rtsx_send_cmd(chip, 0, 100);
 	if (retval < 0) {
@@ -402,9 +402,9 @@ int spi_erase_eeprom_byte(struct rtsx_chip *chip, u16 addr)
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR1, 0xFF, (u8)(addr >> 8));
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF, 0x46);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-		SPI_TRANSFER0_START | SPI_CA_MODE0);
+		     SPI_TRANSFER0_START | SPI_CA_MODE0);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0, SPI_TRANSFER0_END,
-		SPI_TRANSFER0_END);
+		     SPI_TRANSFER0_END);
 
 	retval = rtsx_send_cmd(chip, 0, 100);
 	if (retval < 0) {
@@ -442,9 +442,9 @@ int spi_read_eeprom(struct rtsx_chip *chip, u16 addr, u8 *val)
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF, 0x46);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_LENGTH0, 0xFF, 1);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-		SPI_TRANSFER0_START | SPI_CADI_MODE0);
+		     SPI_TRANSFER0_START | SPI_CADI_MODE0);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0, SPI_TRANSFER0_END,
-		SPI_TRANSFER0_END);
+		     SPI_TRANSFER0_END);
 
 	retval = rtsx_send_cmd(chip, 0, 100);
 	if (retval < 0) {
@@ -497,9 +497,9 @@ int spi_write_eeprom(struct rtsx_chip *chip, u16 addr, u8 val)
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR2, 0xFF, (u8)(addr >> 8));
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF, 0x4E);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-		SPI_TRANSFER0_START | SPI_CA_MODE0);
+		     SPI_TRANSFER0_START | SPI_CA_MODE0);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0, SPI_TRANSFER0_END,
-		SPI_TRANSFER0_END);
+		     SPI_TRANSFER0_END);
 
 	retval = rtsx_send_cmd(chip, 0, 100);
 	if (retval < 0) {
@@ -518,12 +518,12 @@ int spi_write_eeprom(struct rtsx_chip *chip, u16 addr, u8 val)
 
 int spi_get_status(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct spi_info *spi = &(chip->spi);
+	struct spi_info *spi = &chip->spi;
 
 	dev_dbg(rtsx_dev(chip), "spi_get_status: err_code = 0x%x\n",
 		spi->err_code);
-	rtsx_stor_set_xfer_buf(&(spi->err_code),
-			min_t(int, scsi_bufflen(srb), 1), srb);
+	rtsx_stor_set_xfer_buf(&spi->err_code,
+			       min_t(int, scsi_bufflen(srb), 1), srb);
 	scsi_set_resid(srb, scsi_bufflen(srb) - 1);
 
 	return STATUS_SUCCESS;
@@ -531,7 +531,7 @@ int spi_get_status(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 int spi_set_parameter(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
-	struct spi_info *spi = &(chip->spi);
+	struct spi_info *spi = &chip->spi;
 
 	spi_set_err_code(chip, SPI_NO_ERR);
 
@@ -574,37 +574,37 @@ int spi_read_flash_id(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	rtsx_init_cmd(chip);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE, 0x01,
-		PINGPONG_BUFFER);
+		     PINGPONG_BUFFER);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_COMMAND, 0xFF, srb->cmnd[3]);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR2, 0xFF, srb->cmnd[4]);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR1, 0xFF, srb->cmnd[5]);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR0, 0xFF, srb->cmnd[6]);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF,
-		SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
+		     SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_LENGTH1, 0xFF, srb->cmnd[7]);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_LENGTH0, 0xFF, srb->cmnd[8]);
 
 	if (len == 0) {
 		if (srb->cmnd[9]) {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0,
-				      0xFF, SPI_TRANSFER0_START | SPI_CA_MODE0);
+				     0xFF, SPI_TRANSFER0_START | SPI_CA_MODE0);
 		} else {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0,
-				      0xFF, SPI_TRANSFER0_START | SPI_C_MODE0);
+				     0xFF, SPI_TRANSFER0_START | SPI_C_MODE0);
 		}
 	} else {
 		if (srb->cmnd[9]) {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-				SPI_TRANSFER0_START | SPI_CADI_MODE0);
+				     SPI_TRANSFER0_START | SPI_CADI_MODE0);
 		} else {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-				SPI_TRANSFER0_START | SPI_CDI_MODE0);
+				     SPI_TRANSFER0_START | SPI_CDI_MODE0);
 		}
 	}
 
 	rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0, SPI_TRANSFER0_END,
-		SPI_TRANSFER0_END);
+		     SPI_TRANSFER0_END);
 
 	retval = rtsx_send_cmd(chip, 0, 100);
 	if (retval < 0) {
@@ -682,38 +682,38 @@ int spi_read_flash(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		if (slow_read) {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR0, 0xFF,
-				(u8)addr);
+				     (u8)addr);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR1, 0xFF,
-				(u8)(addr >> 8));
+				     (u8)(addr >> 8));
 			rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR2, 0xFF,
-				(u8)(addr >> 16));
+				     (u8)(addr >> 16));
 			rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF,
-				SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
+				     SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
 		} else {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR1, 0xFF,
-				(u8)addr);
+				     (u8)addr);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR2, 0xFF,
-				(u8)(addr >> 8));
+				     (u8)(addr >> 8));
 			rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_ADDR3, 0xFF,
-				(u8)(addr >> 16));
+				     (u8)(addr >> 16));
 			rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF,
-				SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_32);
+				     SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_32);
 		}
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_LENGTH1, 0xFF,
-			(u8)(pagelen >> 8));
+			     (u8)(pagelen >> 8));
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_LENGTH0, 0xFF,
-			(u8)pagelen);
+			     (u8)pagelen);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-			SPI_TRANSFER0_START | SPI_CADI_MODE0);
+			     SPI_TRANSFER0_START | SPI_CADI_MODE0);
 		rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0,
-			SPI_TRANSFER0_END, SPI_TRANSFER0_END);
+			     SPI_TRANSFER0_END, SPI_TRANSFER0_END);
 
 		rtsx_send_cmd_no_wait(chip);
 
 		retval = rtsx_transfer_data(chip, 0, buf, pagelen, 0,
-					DMA_FROM_DEVICE, 10000);
+					    DMA_FROM_DEVICE, 10000);
 		if (retval < 0) {
 			kfree(buf);
 			rtsx_clear_spi_error(chip);
@@ -723,7 +723,7 @@ int spi_read_flash(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 		}
 
 		rtsx_stor_access_xfer_buf(buf, pagelen, srb, &index, &offset,
-					TO_XFER_BUF);
+					  TO_XFER_BUF);
 
 		addr += pagelen;
 		len -= pagelen;
@@ -775,14 +775,14 @@ int spi_write_flash(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 			}
 
 			rtsx_stor_access_xfer_buf(buf, 1, srb, &index, &offset,
-						FROM_XFER_BUF);
+						  FROM_XFER_BUF);
 
 			rtsx_init_cmd(chip);
 
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE,
-				0x01, PINGPONG_BUFFER);
+				     0x01, PINGPONG_BUFFER);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, PPBUF_BASE2, 0xFF,
-				buf[0]);
+				     buf[0]);
 			sf_program(chip, ins, 1, addr, 1);
 
 			retval = rtsx_send_cmd(chip, 0, 100);
@@ -824,14 +824,14 @@ int spi_write_flash(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 
 		while (len) {
 			rtsx_stor_access_xfer_buf(buf, 1, srb, &index, &offset,
-						FROM_XFER_BUF);
+						  FROM_XFER_BUF);
 
 			rtsx_init_cmd(chip);
 
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE,
-				0x01, PINGPONG_BUFFER);
+				     0x01, PINGPONG_BUFFER);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, PPBUF_BASE2, 0xFF,
-				buf[0]);
+				     buf[0]);
 			if (first_byte) {
 				sf_program(chip, ins, 1, addr, 1);
 				first_byte = 0;
@@ -899,10 +899,10 @@ int spi_write_flash(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 			rtsx_send_cmd_no_wait(chip);
 
 			rtsx_stor_access_xfer_buf(buf, pagelen, srb, &index,
-						&offset, FROM_XFER_BUF);
+						  &offset, FROM_XFER_BUF);
 
 			retval = rtsx_transfer_data(chip, 0, buf, pagelen, 0,
-						DMA_TO_DEVICE, 100);
+						    DMA_TO_DEVICE, 100);
 			if (retval < 0) {
 				kfree(buf);
 				rtsx_clear_spi_error(chip);
@@ -1010,18 +1010,18 @@ int spi_write_flash_status(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	rtsx_init_cmd(chip);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE, 0x01,
-		PINGPONG_BUFFER);
+		     PINGPONG_BUFFER);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_COMMAND, 0xFF, ins);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_CA_NUMBER, 0xFF,
-		SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
+		     SPI_COMMAND_BIT_8 | SPI_ADDRESS_BIT_24);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_LENGTH1, 0xFF, 0);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_LENGTH0, 0xFF, 1);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, PPBUF_BASE2, 0xFF, status);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, SPI_TRANSFER0, 0xFF,
-		SPI_TRANSFER0_START | SPI_CDO_MODE0);
+		     SPI_TRANSFER0_START | SPI_CDO_MODE0);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, SPI_TRANSFER0, SPI_TRANSFER0_END,
-		SPI_TRANSFER0_END);
+		     SPI_TRANSFER0_END);
 
 	retval = rtsx_send_cmd(chip, 0, 100);
 	if (retval != STATUS_SUCCESS) {
diff --git a/drivers/staging/rts5208/xd.c b/drivers/staging/rts5208/xd.c
index 1de02bb98839..85aba05acbc1 100644
--- a/drivers/staging/rts5208/xd.c
+++ b/drivers/staging/rts5208/xd.c
@@ -37,21 +37,21 @@ static int xd_init_page(struct rtsx_chip *chip, u32 phy_blk, u16 logoff,
 
 static inline void xd_set_err_code(struct rtsx_chip *chip, u8 err_code)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 
 	xd_card->err_code = err_code;
 }
 
 static inline int xd_check_err_code(struct rtsx_chip *chip, u8 err_code)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 
 	return (xd_card->err_code == err_code);
 }
 
 static int xd_set_init_para(struct rtsx_chip *chip)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	int retval;
 
 	if (chip->asic_code)
@@ -70,7 +70,7 @@ static int xd_set_init_para(struct rtsx_chip *chip)
 
 static int xd_switch_clock(struct rtsx_chip *chip)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	int retval;
 
 	retval = select_card(chip, XD_CARD);
@@ -97,9 +97,9 @@ static int xd_read_id(struct rtsx_chip *chip, u8 id_cmd, u8 *id_buf, u8 buf_len)
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_DAT, 0xFF, id_cmd);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER, 0xFF,
-		XD_TRANSFER_START | XD_READ_ID);
+		     XD_TRANSFER_START | XD_READ_ID);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER, XD_TRANSFER_END,
-		XD_TRANSFER_END);
+		     XD_TRANSFER_END);
 
 	for (i = 0; i < 4; i++)
 		rtsx_add_cmd(chip, READ_REG_CMD, (u16)(XD_ADDRESS1 + i), 0, 0);
@@ -122,28 +122,30 @@ static int xd_read_id(struct rtsx_chip *chip, u8 id_cmd, u8 *id_buf, u8 buf_len)
 
 static void xd_assign_phy_addr(struct rtsx_chip *chip, u32 addr, u8 mode)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 
 	switch (mode) {
 	case XD_RW_ADDR:
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_ADDRESS0, 0xFF, 0);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_ADDRESS1, 0xFF, (u8)addr);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_ADDRESS2,
-			0xFF, (u8)(addr >> 8));
+			     0xFF, (u8)(addr >> 8));
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_ADDRESS3,
-			0xFF, (u8)(addr >> 16));
+			     0xFF, (u8)(addr >> 16));
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_CFG, 0xFF,
-			xd_card->addr_cycle | XD_CALC_ECC | XD_BA_NO_TRANSFORM);
+			     xd_card->addr_cycle |
+			     XD_CALC_ECC |
+			     XD_BA_NO_TRANSFORM);
 		break;
 
 	case XD_ERASE_ADDR:
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_ADDRESS0, 0xFF, (u8)addr);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_ADDRESS1,
-			0xFF, (u8)(addr >> 8));
+			     0xFF, (u8)(addr >> 8));
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_ADDRESS2,
-			0xFF, (u8)(addr >> 16));
+			     0xFF, (u8)(addr >> 16));
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_CFG, 0xFF,
-			(xd_card->addr_cycle - 1) | XD_CALC_ECC |
+			     (xd_card->addr_cycle - 1) | XD_CALC_ECC |
 			XD_BA_NO_TRANSFORM);
 		break;
 
@@ -153,7 +155,7 @@ static void xd_assign_phy_addr(struct rtsx_chip *chip, u32 addr, u8 mode)
 }
 
 static int xd_read_redundant(struct rtsx_chip *chip, u32 page_addr,
-			u8 *buf, int buf_len)
+			     u8 *buf, int buf_len)
 {
 	int retval, i;
 
@@ -162,16 +164,16 @@ static int xd_read_redundant(struct rtsx_chip *chip, u32 page_addr,
 	xd_assign_phy_addr(chip, page_addr, XD_RW_ADDR);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER,
-		0xFF, XD_TRANSFER_START | XD_READ_REDUNDANT);
+		     0xFF, XD_TRANSFER_START | XD_READ_REDUNDANT);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER,
-		XD_TRANSFER_END, XD_TRANSFER_END);
+		     XD_TRANSFER_END, XD_TRANSFER_END);
 
 	for (i = 0; i < 6; i++)
 		rtsx_add_cmd(chip, READ_REG_CMD, (u16)(XD_PAGE_STATUS + i),
-			0, 0);
+			     0, 0);
 	for (i = 0; i < 4; i++)
 		rtsx_add_cmd(chip, READ_REG_CMD, (u16)(XD_RESERVED0 + i),
-			0, 0);
+			     0, 0);
 	rtsx_add_cmd(chip, READ_REG_CMD, XD_PARITY, 0, 0);
 
 	retval = rtsx_send_cmd(chip, XD_CARD, 500);
@@ -192,7 +194,7 @@ static int xd_read_redundant(struct rtsx_chip *chip, u32 page_addr,
 }
 
 static int xd_read_data_from_ppb(struct rtsx_chip *chip, int offset,
-				u8 *buf, int buf_len)
+				 u8 *buf, int buf_len)
 {
 	int retval, i;
 
@@ -205,7 +207,7 @@ static int xd_read_data_from_ppb(struct rtsx_chip *chip, int offset,
 
 	for (i = 0; i < buf_len; i++)
 		rtsx_add_cmd(chip, READ_REG_CMD, PPBUF_BASE2 + offset + i,
-			0, 0);
+			     0, 0);
 
 	retval = rtsx_send_cmd(chip, 0, 250);
 	if (retval < 0) {
@@ -220,7 +222,7 @@ static int xd_read_data_from_ppb(struct rtsx_chip *chip, int offset,
 }
 
 static int xd_read_cis(struct rtsx_chip *chip, u32 page_addr, u8 *buf,
-		int buf_len)
+		       int buf_len)
 {
 	int retval;
 	u8 reg;
@@ -235,15 +237,15 @@ static int xd_read_cis(struct rtsx_chip *chip, u32 page_addr, u8 *buf,
 	xd_assign_phy_addr(chip, page_addr, XD_RW_ADDR);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE,
-		0x01, PINGPONG_BUFFER);
+		     0x01, PINGPONG_BUFFER);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_PAGE_CNT, 0xFF, 1);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_CHK_DATA_STATUS,
-		XD_AUTO_CHK_DATA_STATUS, XD_AUTO_CHK_DATA_STATUS);
+		     XD_AUTO_CHK_DATA_STATUS, XD_AUTO_CHK_DATA_STATUS);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER, 0xFF,
-		XD_TRANSFER_START | XD_READ_PAGES);
+		     XD_TRANSFER_START | XD_READ_PAGES);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER, XD_TRANSFER_END,
-		XD_TRANSFER_END);
+		     XD_TRANSFER_END);
 
 	retval = rtsx_send_cmd(chip, XD_CARD, 250);
 	if (retval == -ETIMEDOUT) {
@@ -347,27 +349,27 @@ static void xd_fill_pull_ctl_disable(struct rtsx_chip *chip)
 {
 	if (CHECK_PID(chip, 0x5208)) {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL1, 0xFF,
-			XD_D3_PD | XD_D2_PD | XD_D1_PD | XD_D0_PD);
+			     XD_D3_PD | XD_D2_PD | XD_D1_PD | XD_D0_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL2, 0xFF,
-			XD_D7_PD | XD_D6_PD | XD_D5_PD | XD_D4_PD);
+			     XD_D7_PD | XD_D6_PD | XD_D5_PD | XD_D4_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL3, 0xFF,
-			XD_WP_PD | XD_CE_PD | XD_CLE_PD | XD_CD_PU);
+			     XD_WP_PD | XD_CE_PD | XD_CLE_PD | XD_CD_PU);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL4, 0xFF,
-			XD_RDY_PD | XD_WE_PD | XD_RE_PD | XD_ALE_PD);
+			     XD_RDY_PD | XD_WE_PD | XD_RE_PD | XD_ALE_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL5, 0xFF,
-			MS_INS_PU | SD_WP_PD | SD_CD_PU | SD_CMD_PD);
+			     MS_INS_PU | SD_WP_PD | SD_CD_PU | SD_CMD_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL6, 0xFF,
-			MS_D5_PD | MS_D4_PD);
+			     MS_D5_PD | MS_D4_PD);
 	} else if (CHECK_PID(chip, 0x5288)) {
 		if (CHECK_BARO_PKG(chip, QFN)) {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL1,
-				0xFF, 0x55);
+				     0xFF, 0x55);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL2,
-				0xFF, 0x55);
+				     0xFF, 0x55);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL3,
-				0xFF, 0x4B);
+				     0xFF, 0x4B);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL4,
-				0xFF, 0x69);
+				     0xFF, 0x69);
 		}
 	}
 }
@@ -386,27 +388,27 @@ static void xd_fill_pull_ctl_enable(struct rtsx_chip *chip)
 {
 	if (CHECK_PID(chip, 0x5208)) {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL1, 0xFF,
-			XD_D3_PD | XD_D2_PD | XD_D1_PD | XD_D0_PD);
+			     XD_D3_PD | XD_D2_PD | XD_D1_PD | XD_D0_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL2, 0xFF,
-			XD_D7_PD | XD_D6_PD | XD_D5_PD | XD_D4_PD);
+			     XD_D7_PD | XD_D6_PD | XD_D5_PD | XD_D4_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL3, 0xFF,
-			XD_WP_PD | XD_CE_PU | XD_CLE_PD | XD_CD_PU);
+			     XD_WP_PD | XD_CE_PU | XD_CLE_PD | XD_CD_PU);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL4, 0xFF,
-			XD_RDY_PU | XD_WE_PU | XD_RE_PU | XD_ALE_PD);
+			     XD_RDY_PU | XD_WE_PU | XD_RE_PU | XD_ALE_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL5, 0xFF,
-			MS_INS_PU | SD_WP_PD | SD_CD_PU | SD_CMD_PD);
+			     MS_INS_PU | SD_WP_PD | SD_CD_PU | SD_CMD_PD);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL6, 0xFF,
-			MS_D5_PD | MS_D4_PD);
+			     MS_D5_PD | MS_D4_PD);
 	} else if (CHECK_PID(chip, 0x5288)) {
 		if (CHECK_BARO_PKG(chip, QFN)) {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL1,
-				0xFF, 0x55);
+				     0xFF, 0x55);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL2,
-				0xFF, 0x55);
+				     0xFF, 0x55);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL3,
-				0xFF, 0x53);
+				     0xFF, 0x53);
 			rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_PULL_CTL4,
-				0xFF, 0xA9);
+				     0xFF, 0xA9);
 		}
 	}
 }
@@ -417,31 +419,46 @@ static int xd_pull_ctl_disable(struct rtsx_chip *chip)
 
 	if (CHECK_PID(chip, 0x5208)) {
 		retval = rtsx_write_register(chip, CARD_PULL_CTL1, 0xFF,
-					     XD_D3_PD | XD_D2_PD | XD_D1_PD | XD_D0_PD);
+					     XD_D3_PD |
+					     XD_D2_PD |
+					     XD_D1_PD |
+					     XD_D0_PD);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 		retval = rtsx_write_register(chip, CARD_PULL_CTL2, 0xFF,
-					     XD_D7_PD | XD_D6_PD | XD_D5_PD | XD_D4_PD);
+					     XD_D7_PD |
+					     XD_D6_PD |
+					     XD_D5_PD |
+					     XD_D4_PD);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 		retval = rtsx_write_register(chip, CARD_PULL_CTL3, 0xFF,
-					     XD_WP_PD | XD_CE_PD | XD_CLE_PD | XD_CD_PU);
+					     XD_WP_PD |
+					     XD_CE_PD |
+					     XD_CLE_PD |
+					     XD_CD_PU);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 		retval = rtsx_write_register(chip, CARD_PULL_CTL4, 0xFF,
-					     XD_RDY_PD | XD_WE_PD | XD_RE_PD | XD_ALE_PD);
+					     XD_RDY_PD |
+					     XD_WE_PD |
+					     XD_RE_PD |
+					     XD_ALE_PD);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
 		}
 		retval = rtsx_write_register(chip, CARD_PULL_CTL5, 0xFF,
-					     MS_INS_PU | SD_WP_PD | SD_CD_PU | SD_CMD_PD);
+					     MS_INS_PU |
+					     SD_WP_PD |
+					     SD_CD_PU |
+					     SD_CMD_PD);
 		if (retval) {
 			rtsx_trace(chip);
 			return retval;
@@ -486,7 +503,7 @@ static int xd_pull_ctl_disable(struct rtsx_chip *chip)
 
 static int reset_xd(struct rtsx_chip *chip)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	int retval, i, j;
 	u8 *ptr, id_buf[4], redunt[11];
 
@@ -499,7 +516,7 @@ static int reset_xd(struct rtsx_chip *chip)
 	rtsx_init_cmd(chip);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_CHK_DATA_STATUS, 0xFF,
-		XD_PGSTS_NOT_FF);
+		     XD_PGSTS_NOT_FF);
 	if (chip->asic_code) {
 		if (!CHECK_PID(chip, 0x5288))
 			xd_fill_pull_ctl_disable(chip);
@@ -507,12 +524,13 @@ static int reset_xd(struct rtsx_chip *chip)
 			xd_fill_pull_ctl_stage1_barossa(chip);
 	} else {
 		rtsx_add_cmd(chip, WRITE_REG_CMD, FPGA_PULL_CTL, 0xFF,
-			(FPGA_XD_PULL_CTL_EN1 & FPGA_XD_PULL_CTL_EN3) | 0x20);
+			     (FPGA_XD_PULL_CTL_EN1 & FPGA_XD_PULL_CTL_EN3) |
+			     0x20);
 	}
 
 	if (!chip->ft2_fast_mode)
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_INIT,
-			XD_NO_AUTO_PWR_OFF, 0);
+			     XD_NO_AUTO_PWR_OFF, 0);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_OE, XD_OUTPUT_EN, 0);
 
@@ -537,8 +555,9 @@ static int reset_xd(struct rtsx_chip *chip)
 			xd_fill_pull_ctl_enable(chip);
 		} else {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, FPGA_PULL_CTL, 0xFF,
-				(FPGA_XD_PULL_CTL_EN1 & FPGA_XD_PULL_CTL_EN2) |
-				0x20);
+				     (FPGA_XD_PULL_CTL_EN1 &
+				      FPGA_XD_PULL_CTL_EN2) |
+				     0x20);
 		}
 
 		retval = rtsx_send_cmd(chip, XD_CARD, 100);
@@ -571,8 +590,9 @@ static int reset_xd(struct rtsx_chip *chip)
 			xd_fill_pull_ctl_enable(chip);
 		} else {
 			rtsx_add_cmd(chip, WRITE_REG_CMD, FPGA_PULL_CTL, 0xFF,
-				(FPGA_XD_PULL_CTL_EN1 & FPGA_XD_PULL_CTL_EN2) |
-				0x20);
+				     (FPGA_XD_PULL_CTL_EN1 &
+				      FPGA_XD_PULL_CTL_EN2) |
+				     0x20);
 		}
 	}
 
@@ -599,16 +619,17 @@ static int reset_xd(struct rtsx_chip *chip)
 		rtsx_init_cmd(chip);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_DTCTL, 0xFF,
-			XD_TIME_SETUP_STEP * 3 +
-			XD_TIME_RW_STEP * (2 + i) + XD_TIME_RWN_STEP * i);
+			     XD_TIME_SETUP_STEP * 3 +
+			     XD_TIME_RW_STEP * (2 + i) + XD_TIME_RWN_STEP * i);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_CATCTL, 0xFF,
-			XD_TIME_SETUP_STEP * 3 + XD_TIME_RW_STEP * (4 + i) +
-			XD_TIME_RWN_STEP * (3 + i));
+			     XD_TIME_SETUP_STEP * 3 +
+			     XD_TIME_RW_STEP * (4 + i) +
+			     XD_TIME_RWN_STEP * (3 + i));
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER, 0xFF,
-			XD_TRANSFER_START | XD_RESET);
+			     XD_TRANSFER_START | XD_RESET);
 		rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER,
-			XD_TRANSFER_END, XD_TRANSFER_END);
+			     XD_TRANSFER_END, XD_TRANSFER_END);
 
 		rtsx_add_cmd(chip, READ_REG_CMD, XD_DAT, 0, 0);
 		rtsx_add_cmd(chip, READ_REG_CMD, XD_CTL, 0, 0);
@@ -625,7 +646,7 @@ static int reset_xd(struct rtsx_chip *chip)
 			ptr[0], ptr[1]);
 
 		if (((ptr[0] & READY_FLAG) != READY_STATE) ||
-			!(ptr[1] & XD_RDY))
+		    !(ptr[1] & XD_RDY))
 			continue;
 
 		retval = xd_read_id(chip, READ_ID, id_buf, 4);
@@ -773,7 +794,7 @@ static int reset_xd(struct rtsx_chip *chip)
 		if (redunt[PAGE_STATUS] != XD_GPG) {
 			for (j = 1; j <= 8; j++) {
 				retval = xd_read_redundant(chip, page_addr + j,
-							redunt, 11);
+							   redunt, 11);
 				if (retval == STATUS_SUCCESS) {
 					if (redunt[PAGE_STATUS] == XD_GPG)
 						break;
@@ -786,7 +807,7 @@ static int reset_xd(struct rtsx_chip *chip)
 
 		/* Check CIS data */
 		if ((redunt[BLOCK_STATUS] == XD_GBLK) &&
-			(redunt[PARITY] & XD_BA1_ALL0)) {
+		    (redunt[PARITY] & XD_BA1_ALL0)) {
 			u8 buf[10];
 
 			page_addr += j;
@@ -798,11 +819,11 @@ static int reset_xd(struct rtsx_chip *chip)
 			}
 
 			if ((buf[0] == 0x01) && (buf[1] == 0x03) &&
-				(buf[2] == 0xD9)
-					&& (buf[3] == 0x01) && (buf[4] == 0xFF)
-					&& (buf[5] == 0x18) && (buf[6] == 0x02)
-					&& (buf[7] == 0xDF) && (buf[8] == 0x01)
-					&& (buf[9] == 0x20)) {
+			    (buf[2] == 0xD9) &&
+			    (buf[3] == 0x01) && (buf[4] == 0xFF) &&
+			    (buf[5] == 0x18) && (buf[6] == 0x02) &&
+			    (buf[7] == 0xDF) && (buf[8] == 0x01) &&
+			    (buf[9] == 0x20)) {
 				xd_card->cis_block = (u16)i;
 			}
 		}
@@ -861,7 +882,7 @@ static u16 xd_load_log_block_addr(u8 *redunt)
 
 static int xd_init_l2p_tbl(struct rtsx_chip *chip)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	int size, i;
 
 	dev_dbg(rtsx_dev(chip), "xd_init_l2p_tbl: zone_cnt = %d\n",
@@ -910,7 +931,7 @@ static inline void free_zone(struct zone_entry *zone)
 
 static void xd_set_unused_block(struct rtsx_chip *chip, u32 phy_blk)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	struct zone_entry *zone;
 	int zone_no;
 
@@ -920,15 +941,15 @@ static void xd_set_unused_block(struct rtsx_chip *chip, u32 phy_blk)
 			zone_no, xd_card->zone_cnt);
 		return;
 	}
-	zone = &(xd_card->zone[zone_no]);
+	zone = &xd_card->zone[zone_no];
 
-	if (zone->free_table == NULL) {
+	if (!zone->free_table) {
 		if (xd_build_l2p_tbl(chip, zone_no) != STATUS_SUCCESS)
 			return;
 	}
 
-	if ((zone->set_index >= XD_FREE_TABLE_CNT)
-			|| (zone->set_index < 0)) {
+	if ((zone->set_index >= XD_FREE_TABLE_CNT) ||
+	    (zone->set_index < 0)) {
 		free_zone(zone);
 		dev_dbg(rtsx_dev(chip), "Set unused block fail, invalid set_index\n");
 		return;
@@ -945,7 +966,7 @@ static void xd_set_unused_block(struct rtsx_chip *chip, u32 phy_blk)
 
 static u32 xd_get_unused_block(struct rtsx_chip *chip, int zone_no)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	struct zone_entry *zone;
 	u32 phy_blk;
 
@@ -954,10 +975,10 @@ static u32 xd_get_unused_block(struct rtsx_chip *chip, int zone_no)
 			zone_no, xd_card->zone_cnt);
 		return BLK_NOT_FOUND;
 	}
-	zone = &(xd_card->zone[zone_no]);
+	zone = &xd_card->zone[zone_no];
 
 	if ((zone->unused_blk_cnt == 0) ||
-		(zone->set_index == zone->get_index)) {
+	    (zone->set_index == zone->get_index)) {
 		free_zone(zone);
 		dev_dbg(rtsx_dev(chip), "Get unused block fail, no unused block available\n");
 		return BLK_NOT_FOUND;
@@ -982,22 +1003,22 @@ static u32 xd_get_unused_block(struct rtsx_chip *chip, int zone_no)
 }
 
 static void xd_set_l2p_tbl(struct rtsx_chip *chip,
-			int zone_no, u16 log_off, u16 phy_off)
+			   int zone_no, u16 log_off, u16 phy_off)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	struct zone_entry *zone;
 
-	zone = &(xd_card->zone[zone_no]);
+	zone = &xd_card->zone[zone_no];
 	zone->l2p_table[log_off] = phy_off;
 }
 
 static u32 xd_get_l2p_tbl(struct rtsx_chip *chip, int zone_no, u16 log_off)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	struct zone_entry *zone;
 	int retval;
 
-	zone = &(xd_card->zone[zone_no]);
+	zone = &xd_card->zone[zone_no];
 	if (zone->l2p_table[log_off] == 0xFFFF) {
 		u32 phy_blk = 0;
 		int i;
@@ -1023,7 +1044,7 @@ static u32 xd_get_l2p_tbl(struct rtsx_chip *chip, int zone_no, u16 log_off)
 			}
 
 			retval = xd_init_page(chip, phy_blk, log_off,
-					0, xd_card->page_off + 1);
+					      0, xd_card->page_off + 1);
 			if (retval == STATUS_SUCCESS)
 				break;
 		}
@@ -1041,7 +1062,7 @@ static u32 xd_get_l2p_tbl(struct rtsx_chip *chip, int zone_no, u16 log_off)
 
 int reset_xd_card(struct rtsx_chip *chip)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	int retval;
 
 	memset(xd_card, 0, sizeof(struct xd_info));
@@ -1077,7 +1098,7 @@ int reset_xd_card(struct rtsx_chip *chip)
 
 static int xd_mark_bad_block(struct rtsx_chip *chip, u32 phy_blk)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	int retval;
 	u32 page_addr;
 	u8 reg = 0;
@@ -1107,12 +1128,12 @@ static int xd_mark_bad_block(struct rtsx_chip *chip, u32 phy_blk)
 	xd_assign_phy_addr(chip, page_addr, XD_RW_ADDR);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_PAGE_CNT, 0xFF,
-		xd_card->page_off + 1);
+		     xd_card->page_off + 1);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER, 0xFF,
-		XD_TRANSFER_START | XD_WRITE_REDUNDANT);
+		     XD_TRANSFER_START | XD_WRITE_REDUNDANT);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER,
-		XD_TRANSFER_END, XD_TRANSFER_END);
+		     XD_TRANSFER_END, XD_TRANSFER_END);
 
 	retval = rtsx_send_cmd(chip, XD_CARD, 500);
 	if (retval < 0) {
@@ -1132,7 +1153,7 @@ static int xd_mark_bad_block(struct rtsx_chip *chip, u32 phy_blk)
 static int xd_init_page(struct rtsx_chip *chip, u32 phy_blk,
 			u16 logoff, u8 start_page, u8 end_page)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	int retval;
 	u32 page_addr;
 	u8 reg = 0;
@@ -1153,7 +1174,7 @@ static int xd_init_page(struct rtsx_chip *chip, u32 phy_blk,
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_PAGE_STATUS, 0xFF, 0xFF);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_BLOCK_STATUS, 0xFF, 0xFF);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_BLOCK_ADDR1_H,
-		0xFF, (u8)(logoff >> 8));
+		     0xFF, (u8)(logoff >> 8));
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_BLOCK_ADDR1_L, 0xFF, (u8)logoff);
 
 	page_addr = (phy_blk << xd_card->block_shift) + start_page;
@@ -1161,15 +1182,15 @@ static int xd_init_page(struct rtsx_chip *chip, u32 phy_blk,
 	xd_assign_phy_addr(chip, page_addr, XD_RW_ADDR);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_CFG,
-		XD_BA_TRANSFORM, XD_BA_TRANSFORM);
+		     XD_BA_TRANSFORM, XD_BA_TRANSFORM);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_PAGE_CNT,
-		0xFF, (end_page - start_page));
+		     0xFF, (end_page - start_page));
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER,
-		0xFF, XD_TRANSFER_START | XD_WRITE_REDUNDANT);
+		     0xFF, XD_TRANSFER_START | XD_WRITE_REDUNDANT);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER,
-		XD_TRANSFER_END, XD_TRANSFER_END);
+		     XD_TRANSFER_END, XD_TRANSFER_END);
 
 	retval = rtsx_send_cmd(chip, XD_CARD, 500);
 	if (retval < 0) {
@@ -1191,7 +1212,7 @@ static int xd_init_page(struct rtsx_chip *chip, u32 phy_blk,
 static int xd_copy_page(struct rtsx_chip *chip, u32 old_blk, u32 new_blk,
 			u8 start_page, u8 end_page)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	u32 old_page, new_page;
 	u8 i, reg = 0;
 	int retval;
@@ -1235,11 +1256,11 @@ static int xd_copy_page(struct rtsx_chip *chip, u32 old_blk, u32 new_blk,
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_PAGE_CNT, 0xFF, 1);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_CHK_DATA_STATUS,
-			XD_AUTO_CHK_DATA_STATUS, 0);
+			     XD_AUTO_CHK_DATA_STATUS, 0);
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER, 0xFF,
-			XD_TRANSFER_START | XD_READ_PAGES);
+			     XD_TRANSFER_START | XD_READ_PAGES);
 		rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER,
-			XD_TRANSFER_END, XD_TRANSFER_END);
+			     XD_TRANSFER_END, XD_TRANSFER_END);
 
 		retval = rtsx_send_cmd(chip, XD_CARD, 500);
 		if (retval < 0) {
@@ -1250,22 +1271,24 @@ static int xd_copy_page(struct rtsx_chip *chip, u32 old_blk, u32 new_blk,
 				wait_timeout(100);
 
 				if (detect_card_cd(chip,
-					XD_CARD) != STATUS_SUCCESS) {
+						   XD_CARD) != STATUS_SUCCESS) {
 					xd_set_err_code(chip, XD_NO_CARD);
 					rtsx_trace(chip);
 					return STATUS_FAIL;
 				}
 
 				if (((reg & (XD_ECC1_ERROR | XD_ECC1_UNCORRECTABLE)) ==
-						(XD_ECC1_ERROR | XD_ECC1_UNCORRECTABLE))
-					|| ((reg & (XD_ECC2_ERROR | XD_ECC2_UNCORRECTABLE)) ==
+						(XD_ECC1_ERROR | XD_ECC1_UNCORRECTABLE)) ||
+					((reg & (XD_ECC2_ERROR | XD_ECC2_UNCORRECTABLE)) ==
 						(XD_ECC2_ERROR | XD_ECC2_UNCORRECTABLE))) {
 					rtsx_write_register(chip,
-							XD_PAGE_STATUS, 0xFF,
-							XD_BPG);
+							    XD_PAGE_STATUS,
+							    0xFF,
+							    XD_BPG);
 					rtsx_write_register(chip,
-							XD_BLOCK_STATUS, 0xFF,
-							XD_GBLK);
+							    XD_BLOCK_STATUS,
+							    0xFF,
+							    XD_GBLK);
 					XD_SET_BAD_OLDBLK(xd_card);
 					dev_dbg(rtsx_dev(chip), "old block 0x%x ecc error\n",
 						old_blk);
@@ -1287,7 +1310,7 @@ static int xd_copy_page(struct rtsx_chip *chip, u32 old_blk, u32 new_blk,
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER, 0xFF,
 			     XD_TRANSFER_START | XD_WRITE_PAGES);
 		rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER,
-			XD_TRANSFER_END, XD_TRANSFER_END);
+			     XD_TRANSFER_END, XD_TRANSFER_END);
 
 		retval = rtsx_send_cmd(chip, XD_CARD, 300);
 		if (retval < 0) {
@@ -1320,9 +1343,9 @@ static int xd_reset_cmd(struct rtsx_chip *chip)
 	rtsx_init_cmd(chip);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER,
-		0xFF, XD_TRANSFER_START | XD_RESET);
+		     0xFF, XD_TRANSFER_START | XD_RESET);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER,
-		XD_TRANSFER_END, XD_TRANSFER_END);
+		     XD_TRANSFER_END, XD_TRANSFER_END);
 	rtsx_add_cmd(chip, READ_REG_CMD, XD_DAT, 0, 0);
 	rtsx_add_cmd(chip, READ_REG_CMD, XD_CTL, 0, 0);
 
@@ -1342,7 +1365,7 @@ static int xd_reset_cmd(struct rtsx_chip *chip)
 
 static int xd_erase_block(struct rtsx_chip *chip, u32 phy_blk)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	u32 page_addr;
 	u8 reg = 0, *ptr;
 	int i, retval;
@@ -1360,9 +1383,9 @@ static int xd_erase_block(struct rtsx_chip *chip, u32 phy_blk)
 		xd_assign_phy_addr(chip, page_addr, XD_ERASE_ADDR);
 
 		rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER, 0xFF,
-			XD_TRANSFER_START | XD_ERASE);
+			     XD_TRANSFER_START | XD_ERASE);
 		rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER,
-			XD_TRANSFER_END, XD_TRANSFER_END);
+			     XD_TRANSFER_END, XD_TRANSFER_END);
 		rtsx_add_cmd(chip, READ_REG_CMD, XD_DAT, 0, 0);
 
 		retval = rtsx_send_cmd(chip, XD_CARD, 250);
@@ -1403,7 +1426,7 @@ static int xd_erase_block(struct rtsx_chip *chip, u32 phy_blk)
 
 static int xd_build_l2p_tbl(struct rtsx_chip *chip, int zone_no)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	struct zone_entry *zone;
 	int retval;
 	u32 start, end, i;
@@ -1413,7 +1436,7 @@ static int xd_build_l2p_tbl(struct rtsx_chip *chip, int zone_no)
 
 	dev_dbg(rtsx_dev(chip), "xd_build_l2p_tbl: %d\n", zone_no);
 
-	if (xd_card->zone == NULL) {
+	if (!xd_card->zone) {
 		retval = xd_init_l2p_tbl(chip);
 		if (retval != STATUS_SUCCESS)
 			return retval;
@@ -1425,22 +1448,22 @@ static int xd_build_l2p_tbl(struct rtsx_chip *chip, int zone_no)
 		return STATUS_SUCCESS;
 	}
 
-	zone = &(xd_card->zone[zone_no]);
+	zone = &xd_card->zone[zone_no];
 
-	if (zone->l2p_table == NULL) {
+	if (!zone->l2p_table) {
 		zone->l2p_table = vmalloc(2000);
 		if (!zone->l2p_table) {
 			rtsx_trace(chip);
-			goto Build_Fail;
+			goto build_fail;
 		}
 	}
 	memset((u8 *)(zone->l2p_table), 0xff, 2000);
 
-	if (zone->free_table == NULL) {
+	if (!zone->free_table) {
 		zone->free_table = vmalloc(XD_FREE_TABLE_CNT * 2);
 		if (!zone->free_table) {
 			rtsx_trace(chip);
-			goto Build_Fail;
+			goto build_fail;
 		}
 	}
 	memset((u8 *)(zone->free_table), 0xff, XD_FREE_TABLE_CNT * 2);
@@ -1466,7 +1489,8 @@ static int xd_build_l2p_tbl(struct rtsx_chip *chip, int zone_no)
 	dev_dbg(rtsx_dev(chip), "start block 0x%x, end block 0x%x\n",
 		start, end);
 
-	zone->set_index = zone->get_index = 0;
+	zone->set_index = 0;
+	zone->get_index = 0;
 	zone->unused_blk_cnt = 0;
 
 	for (i = start; i < end; i++) {
@@ -1490,7 +1514,7 @@ static int xd_build_l2p_tbl(struct rtsx_chip *chip, int zone_no)
 
 		cur_fst_page_logoff = xd_load_log_block_addr(redunt);
 		if ((cur_fst_page_logoff == 0xFFFF) ||
-			(cur_fst_page_logoff > max_logoff)) {
+		    (cur_fst_page_logoff > max_logoff)) {
 			retval = xd_erase_block(chip, i);
 			if (retval == STATUS_SUCCESS)
 				xd_set_unused_block(chip, i);
@@ -1498,7 +1522,7 @@ static int xd_build_l2p_tbl(struct rtsx_chip *chip, int zone_no)
 		}
 
 		if ((zone_no == 0) && (cur_fst_page_logoff == 0) &&
-			(redunt[PAGE_STATUS] != XD_GPG))
+		    (redunt[PAGE_STATUS] != XD_GPG))
 			XD_SET_MBR_FAIL(xd_card);
 
 		if (zone->l2p_table[cur_fst_page_logoff] == 0xFFFF) {
@@ -1524,7 +1548,7 @@ static int xd_build_l2p_tbl(struct rtsx_chip *chip, int zone_no)
 
 			for (m = 0; m < 3; m++) {
 				retval = xd_read_redundant(chip, page_addr,
-							redunt, 11);
+							   redunt, 11);
 				if (retval == STATUS_SUCCESS)
 					break;
 			}
@@ -1581,7 +1605,7 @@ static int xd_build_l2p_tbl(struct rtsx_chip *chip, int zone_no)
 
 	return STATUS_SUCCESS;
 
-Build_Fail:
+build_fail:
 	vfree(zone->l2p_table);
 	zone->l2p_table = NULL;
 	vfree(zone->free_table);
@@ -1598,9 +1622,9 @@ static int xd_send_cmd(struct rtsx_chip *chip, u8 cmd)
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_DAT, 0xFF, cmd);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER, 0xFF,
-		XD_TRANSFER_START | XD_SET_CMD);
+		     XD_TRANSFER_START | XD_SET_CMD);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER,
-		XD_TRANSFER_END, XD_TRANSFER_END);
+		     XD_TRANSFER_END, XD_TRANSFER_END);
 
 	retval = rtsx_send_cmd(chip, XD_CARD, 200);
 	if (retval < 0) {
@@ -1612,18 +1636,18 @@ static int xd_send_cmd(struct rtsx_chip *chip, u8 cmd)
 }
 
 static int xd_read_multiple_pages(struct rtsx_chip *chip, u32 phy_blk,
-				u32 log_blk, u8 start_page, u8 end_page,
-				u8 *buf, unsigned int *index,
-				unsigned int *offset)
+				  u32 log_blk, u8 start_page, u8 end_page,
+				  u8 *buf, unsigned int *index,
+				  unsigned int *offset)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	u32 page_addr, new_blk;
 	u16 log_off;
 	u8 reg_val, page_cnt;
 	int zone_no, retval, i;
 
 	if (start_page > end_page)
-		goto Status_Fail;
+		goto status_fail;
 
 	page_cnt = end_page - start_page;
 	zone_no = (int)(log_blk / 1000);
@@ -1639,7 +1663,7 @@ static int xd_read_multiple_pages(struct rtsx_chip *chip, u32 phy_blk,
 
 			if (detect_card_cd(chip, XD_CARD) != STATUS_SUCCESS) {
 				xd_set_err_code(chip, XD_NO_CARD);
-				goto Status_Fail;
+				goto status_fail;
 			}
 		}
 	}
@@ -1653,37 +1677,38 @@ static int xd_read_multiple_pages(struct rtsx_chip *chip, u32 phy_blk,
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE, 0x01, RING_BUFFER);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_PAGE_CNT, 0xFF, page_cnt);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_CHK_DATA_STATUS,
-			XD_AUTO_CHK_DATA_STATUS, XD_AUTO_CHK_DATA_STATUS);
+		     XD_AUTO_CHK_DATA_STATUS, XD_AUTO_CHK_DATA_STATUS);
 
 	trans_dma_enable(chip->srb->sc_data_direction, chip,
-			page_cnt * 512, DMA_512);
+			 page_cnt * 512, DMA_512);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER, 0xFF,
-		XD_TRANSFER_START | XD_READ_PAGES);
+		     XD_TRANSFER_START | XD_READ_PAGES);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER,
-		XD_TRANSFER_END | XD_PPB_EMPTY, XD_TRANSFER_END | XD_PPB_EMPTY);
+		     XD_TRANSFER_END | XD_PPB_EMPTY,
+		     XD_TRANSFER_END | XD_PPB_EMPTY);
 
 	rtsx_send_cmd_no_wait(chip);
 
 	retval = rtsx_transfer_data_partial(chip, XD_CARD, buf, page_cnt * 512,
-					scsi_sg_count(chip->srb),
-					index, offset, DMA_FROM_DEVICE,
-					chip->xd_timeout);
+					    scsi_sg_count(chip->srb),
+					    index, offset, DMA_FROM_DEVICE,
+					    chip->xd_timeout);
 	if (retval < 0) {
 		rtsx_clear_xd_error(chip);
 
 		if (retval == -ETIMEDOUT) {
 			xd_set_err_code(chip, XD_TO_ERROR);
-			goto Status_Fail;
+			goto status_fail;
 		} else {
 			rtsx_trace(chip);
-			goto Fail;
+			goto fail;
 		}
 	}
 
 	return STATUS_SUCCESS;
 
-Fail:
+fail:
 	retval = rtsx_read_register(chip, XD_PAGE_STATUS, &reg_val);
 	if (retval) {
 		rtsx_trace(chip);
@@ -1699,15 +1724,15 @@ Fail:
 		return retval;
 	}
 
-	if (((reg_val & (XD_ECC1_ERROR | XD_ECC1_UNCORRECTABLE))
-				== (XD_ECC1_ERROR | XD_ECC1_UNCORRECTABLE))
-		|| ((reg_val & (XD_ECC2_ERROR | XD_ECC2_UNCORRECTABLE))
-			== (XD_ECC2_ERROR | XD_ECC2_UNCORRECTABLE))) {
+	if (((reg_val & (XD_ECC1_ERROR | XD_ECC1_UNCORRECTABLE)) ==
+				(XD_ECC1_ERROR | XD_ECC1_UNCORRECTABLE)) ||
+		((reg_val & (XD_ECC2_ERROR | XD_ECC2_UNCORRECTABLE)) ==
+			(XD_ECC2_ERROR | XD_ECC2_UNCORRECTABLE))) {
 		wait_timeout(100);
 
 		if (detect_card_cd(chip, XD_CARD) != STATUS_SUCCESS) {
 			xd_set_err_code(chip, XD_NO_CARD);
-			goto Status_Fail;
+			goto status_fail;
 		}
 
 		xd_set_err_code(chip, XD_ECC_ERROR);
@@ -1715,11 +1740,11 @@ Fail:
 		new_blk = xd_get_unused_block(chip, zone_no);
 		if (new_blk == NO_NEW_BLK) {
 			XD_CLR_BAD_OLDBLK(xd_card);
-			goto Status_Fail;
+			goto status_fail;
 		}
 
 		retval = xd_copy_page(chip, phy_blk, new_blk, 0,
-				xd_card->page_off + 1);
+				      xd_card->page_off + 1);
 		if (retval != STATUS_SUCCESS) {
 			if (!XD_CHK_BAD_NEWBLK(xd_card)) {
 				retval = xd_erase_block(chip, new_blk);
@@ -1729,7 +1754,7 @@ Fail:
 				XD_CLR_BAD_NEWBLK(xd_card);
 			}
 			XD_CLR_BAD_OLDBLK(xd_card);
-			goto Status_Fail;
+			goto status_fail;
 		}
 		xd_set_l2p_tbl(chip, zone_no, log_off, (u16)(new_blk & 0x3FF));
 		xd_erase_block(chip, phy_blk);
@@ -1737,15 +1762,15 @@ Fail:
 		XD_CLR_BAD_OLDBLK(xd_card);
 	}
 
-Status_Fail:
+status_fail:
 	rtsx_trace(chip);
 	return STATUS_FAIL;
 }
 
 static int xd_finish_write(struct rtsx_chip *chip,
-		u32 old_blk, u32 new_blk, u32 log_blk, u8 page_off)
+			   u32 old_blk, u32 new_blk, u32 log_blk, u8 page_off)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	int retval, zone_no;
 	u16 log_off;
 
@@ -1762,7 +1787,7 @@ static int xd_finish_write(struct rtsx_chip *chip,
 
 	if (old_blk == BLK_NOT_FOUND) {
 		retval = xd_init_page(chip, new_blk, log_off,
-				page_off, xd_card->page_off + 1);
+				      page_off, xd_card->page_off + 1);
 		if (retval != STATUS_SUCCESS) {
 			retval = xd_erase_block(chip, new_blk);
 			if (retval == STATUS_SUCCESS)
@@ -1772,7 +1797,7 @@ static int xd_finish_write(struct rtsx_chip *chip,
 		}
 	} else {
 		retval = xd_copy_page(chip, old_blk, new_blk,
-				page_off, xd_card->page_off + 1);
+				      page_off, xd_card->page_off + 1);
 		if (retval != STATUS_SUCCESS) {
 			if (!XD_CHK_BAD_NEWBLK(xd_card)) {
 				retval = xd_erase_block(chip, new_blk);
@@ -1804,7 +1829,7 @@ static int xd_finish_write(struct rtsx_chip *chip,
 }
 
 static int xd_prepare_write(struct rtsx_chip *chip,
-		u32 old_blk, u32 new_blk, u32 log_blk, u8 page_off)
+			    u32 old_blk, u32 new_blk, u32 log_blk, u8 page_off)
 {
 	int retval;
 
@@ -1823,11 +1848,11 @@ static int xd_prepare_write(struct rtsx_chip *chip,
 }
 
 static int xd_write_multiple_pages(struct rtsx_chip *chip, u32 old_blk,
-				u32 new_blk, u32 log_blk, u8 start_page,
-				u8 end_page, u8 *buf, unsigned int *index,
-				unsigned int *offset)
+				   u32 new_blk, u32 log_blk, u8 start_page,
+				   u8 end_page, u8 *buf, unsigned int *index,
+				   unsigned int *offset)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	u32 page_addr;
 	int zone_no, retval;
 	u16 log_off;
@@ -1837,7 +1862,7 @@ static int xd_write_multiple_pages(struct rtsx_chip *chip, u32 old_blk,
 		__func__, old_blk, new_blk, log_blk);
 
 	if (start_page > end_page)
-		goto Status_Fail;
+		goto status_fail;
 
 	page_cnt = end_page - start_page;
 	zone_no = (int)(log_blk / 1000);
@@ -1847,12 +1872,12 @@ static int xd_write_multiple_pages(struct rtsx_chip *chip, u32 old_blk,
 
 	retval = xd_send_cmd(chip, READ1_1);
 	if (retval != STATUS_SUCCESS)
-		goto Status_Fail;
+		goto status_fail;
 
 	rtsx_init_cmd(chip);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_BLOCK_ADDR1_H,
-		0xFF, (u8)(log_off >> 8));
+		     0xFF, (u8)(log_off >> 8));
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_BLOCK_ADDR1_L, 0xFF, (u8)log_off);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_BLOCK_STATUS, 0xFF, XD_GBLK);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_PAGE_STATUS, 0xFF, XD_GPG);
@@ -1860,32 +1885,32 @@ static int xd_write_multiple_pages(struct rtsx_chip *chip, u32 old_blk,
 	xd_assign_phy_addr(chip, page_addr, XD_RW_ADDR);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_CFG, XD_BA_TRANSFORM,
-		XD_BA_TRANSFORM);
+		     XD_BA_TRANSFORM);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_PAGE_CNT, 0xFF, page_cnt);
 	rtsx_add_cmd(chip, WRITE_REG_CMD, CARD_DATA_SOURCE, 0x01, RING_BUFFER);
 
 	trans_dma_enable(chip->srb->sc_data_direction, chip,
-			page_cnt * 512, DMA_512);
+			 page_cnt * 512, DMA_512);
 
 	rtsx_add_cmd(chip, WRITE_REG_CMD, XD_TRANSFER,
-		0xFF, XD_TRANSFER_START | XD_WRITE_PAGES);
+		     0xFF, XD_TRANSFER_START | XD_WRITE_PAGES);
 	rtsx_add_cmd(chip, CHECK_REG_CMD, XD_TRANSFER,
-		XD_TRANSFER_END, XD_TRANSFER_END);
+		     XD_TRANSFER_END, XD_TRANSFER_END);
 
 	rtsx_send_cmd_no_wait(chip);
 
 	retval = rtsx_transfer_data_partial(chip, XD_CARD, buf, page_cnt * 512,
-					scsi_sg_count(chip->srb),
-			index, offset, DMA_TO_DEVICE, chip->xd_timeout);
+					    scsi_sg_count(chip->srb),
+					    index, offset, DMA_TO_DEVICE, chip->xd_timeout);
 	if (retval < 0) {
 		rtsx_clear_xd_error(chip);
 
 		if (retval == -ETIMEDOUT) {
 			xd_set_err_code(chip, XD_TO_ERROR);
-			goto Status_Fail;
+			goto status_fail;
 		} else {
 			rtsx_trace(chip);
-			goto Fail;
+			goto fail;
 		}
 	}
 
@@ -1911,7 +1936,7 @@ static int xd_write_multiple_pages(struct rtsx_chip *chip, u32 old_blk,
 
 	return STATUS_SUCCESS;
 
-Fail:
+fail:
 	retval = rtsx_read_register(chip, XD_DAT, &reg_val);
 	if (retval) {
 		rtsx_trace(chip);
@@ -1922,7 +1947,7 @@ Fail:
 		xd_mark_bad_block(chip, new_blk);
 	}
 
-Status_Fail:
+status_fail:
 	rtsx_trace(chip);
 	return STATUS_FAIL;
 }
@@ -1930,8 +1955,8 @@ Status_Fail:
 #ifdef XD_DELAY_WRITE
 int xd_delay_write(struct rtsx_chip *chip)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
-	struct xd_delay_write_tag *delay_write = &(xd_card->delay_write);
+	struct xd_info *xd_card = &chip->xd_card;
+	struct xd_delay_write_tag *delay_write = &xd_card->delay_write;
 	int retval;
 
 	if (delay_write->delay_write_flag) {
@@ -1944,9 +1969,10 @@ int xd_delay_write(struct rtsx_chip *chip)
 
 		delay_write->delay_write_flag = 0;
 		retval = xd_finish_write(chip,
-				delay_write->old_phyblock,
-					delay_write->new_phyblock,
-				delay_write->logblock, delay_write->pageoff);
+					 delay_write->old_phyblock,
+					 delay_write->new_phyblock,
+					 delay_write->logblock,
+					 delay_write->pageoff);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -1958,12 +1984,12 @@ int xd_delay_write(struct rtsx_chip *chip)
 #endif
 
 int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
-	u32 start_sector, u16 sector_cnt)
+	  u32 start_sector, u16 sector_cnt)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	unsigned int lun = SCSI_LUN(srb);
 #ifdef XD_DELAY_WRITE
-	struct xd_delay_write_tag *delay_write = &(xd_card->delay_write);
+	struct xd_delay_write_tag *delay_write = &xd_card->delay_write;
 #endif
 	int retval, zone_no;
 	unsigned int index = 0, offset = 0;
@@ -2012,17 +2038,18 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 	if (srb->sc_data_direction == DMA_TO_DEVICE) {
 #ifdef XD_DELAY_WRITE
 		if (delay_write->delay_write_flag &&
-				(delay_write->logblock == log_blk) &&
-				(start_page > delay_write->pageoff)) {
+		    (delay_write->logblock == log_blk) &&
+		    (start_page > delay_write->pageoff)) {
 			delay_write->delay_write_flag = 0;
 			if (delay_write->old_phyblock != BLK_NOT_FOUND) {
 				retval = xd_copy_page(chip,
-					delay_write->old_phyblock,
-					delay_write->new_phyblock,
-					delay_write->pageoff, start_page);
+						      delay_write->old_phyblock,
+						      delay_write->new_phyblock,
+						      delay_write->pageoff,
+						      start_page);
 				if (retval != STATUS_SUCCESS) {
 					set_sense_type(chip, lun,
-						SENSE_TYPE_MEDIA_WRITE_ERR);
+						       SENSE_TYPE_MEDIA_WRITE_ERR);
 					rtsx_trace(chip);
 					return STATUS_FAIL;
 				}
@@ -2039,7 +2066,7 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 			retval = xd_delay_write(chip);
 			if (retval != STATUS_SUCCESS) {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
@@ -2047,25 +2074,25 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 			old_blk = xd_get_l2p_tbl(chip, zone_no, log_off);
 			new_blk  = xd_get_unused_block(chip, zone_no);
 			if ((old_blk == BLK_NOT_FOUND) ||
-				(new_blk == BLK_NOT_FOUND)) {
+			    (new_blk == BLK_NOT_FOUND)) {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
 
 			retval = xd_prepare_write(chip, old_blk, new_blk,
-						log_blk, start_page);
+						  log_blk, start_page);
 			if (retval != STATUS_SUCCESS) {
 				if (detect_card_cd(chip, XD_CARD) !=
 					STATUS_SUCCESS) {
 					set_sense_type(chip, lun,
-						SENSE_TYPE_MEDIA_NOT_PRESENT);
+						       SENSE_TYPE_MEDIA_NOT_PRESENT);
 					rtsx_trace(chip);
 					return STATUS_FAIL;
 				}
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
@@ -2078,12 +2105,12 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 		if (retval != STATUS_SUCCESS) {
 			if (detect_card_cd(chip, XD_CARD) != STATUS_SUCCESS) {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_NOT_PRESENT);
+					       SENSE_TYPE_MEDIA_NOT_PRESENT);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
@@ -2092,7 +2119,7 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 		old_blk = xd_get_l2p_tbl(chip, zone_no, log_off);
 		if (old_blk == BLK_NOT_FOUND) {
 			set_sense_type(chip, lun,
-				SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+				       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
@@ -2116,22 +2143,22 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 		page_cnt = end_page - start_page;
 		if (srb->sc_data_direction == DMA_FROM_DEVICE) {
 			retval = xd_read_multiple_pages(chip, old_blk, log_blk,
-					start_page, end_page, ptr,
-							&index, &offset);
+							start_page, end_page,
+							ptr, &index, &offset);
 			if (retval != STATUS_SUCCESS) {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+					       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
 		} else {
 			retval = xd_write_multiple_pages(chip, old_blk,
-							new_blk, log_blk,
-					start_page, end_page, ptr,
-							&index, &offset);
+							 new_blk, log_blk,
+							 start_page, end_page,
+							 ptr, &index, &offset);
 			if (retval != STATUS_SUCCESS) {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
@@ -2153,7 +2180,7 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 			if (retval != STATUS_SUCCESS) {
 				chip->card_fail |= XD_CARD;
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_NOT_PRESENT);
+					       SENSE_TYPE_MEDIA_NOT_PRESENT);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
@@ -2163,10 +2190,10 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 		if (old_blk == BLK_NOT_FOUND) {
 			if (srb->sc_data_direction == DMA_FROM_DEVICE)
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
+					       SENSE_TYPE_MEDIA_UNRECOVER_READ_ERR);
 			else
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -2176,7 +2203,7 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 			new_blk = xd_get_unused_block(chip, zone_no);
 			if (new_blk == BLK_NOT_FOUND) {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_WRITE_ERR);
+					       SENSE_TYPE_MEDIA_WRITE_ERR);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
@@ -2186,7 +2213,7 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 	}
 
 	if ((srb->sc_data_direction == DMA_TO_DEVICE) &&
-			(end_page != (xd_card->page_off + 1))) {
+	    (end_page != (xd_card->page_off + 1))) {
 #ifdef XD_DELAY_WRITE
 		delay_write->delay_write_flag = 1;
 		delay_write->old_phyblock = old_blk;
@@ -2202,11 +2229,11 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 		}
 
 		retval = xd_finish_write(chip, old_blk, new_blk,
-					log_blk, end_page);
+					 log_blk, end_page);
 		if (retval != STATUS_SUCCESS) {
 			if (detect_card_cd(chip, XD_CARD) != STATUS_SUCCESS) {
 				set_sense_type(chip, lun,
-					SENSE_TYPE_MEDIA_NOT_PRESENT);
+					       SENSE_TYPE_MEDIA_NOT_PRESENT);
 				rtsx_trace(chip);
 				return STATUS_FAIL;
 			}
@@ -2224,10 +2251,10 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 
 void xd_free_l2p_tbl(struct rtsx_chip *chip)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	int i = 0;
 
-	if (xd_card->zone != NULL) {
+	if (xd_card->zone) {
 		for (i = 0; i < xd_card->zone_cnt; i++) {
 			vfree(xd_card->zone[i].l2p_table);
 			xd_card->zone[i].l2p_table = NULL;
@@ -2242,7 +2269,7 @@ void xd_free_l2p_tbl(struct rtsx_chip *chip)
 void xd_cleanup_work(struct rtsx_chip *chip)
 {
 #ifdef XD_DELAY_WRITE
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 
 	if (xd_card->delay_write.delay_write_flag) {
 		dev_dbg(rtsx_dev(chip), "xD: delay write\n");
@@ -2297,7 +2324,7 @@ int xd_power_off_card3v3(struct rtsx_chip *chip)
 
 int release_xd_card(struct rtsx_chip *chip)
 {
-	struct xd_info *xd_card = &(chip->xd_card);
+	struct xd_info *xd_card = &chip->xd_card;
 	int retval;
 
 	chip->card_ready &= ~XD_CARD;
diff --git a/drivers/staging/rts5208/xd.h b/drivers/staging/rts5208/xd.h
index 938138c50bb5..d5f10880efb7 100644
--- a/drivers/staging/rts5208/xd.h
+++ b/drivers/staging/rts5208/xd.h
@@ -179,7 +179,7 @@ int reset_xd_card(struct rtsx_chip *chip);
 int xd_delay_write(struct rtsx_chip *chip);
 #endif
 int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
-	u32 start_sector, u16 sector_cnt);
+	  u32 start_sector, u16 sector_cnt);
 void xd_free_l2p_tbl(struct rtsx_chip *chip);
 void xd_cleanup_work(struct rtsx_chip *chip);
 int xd_power_off_card3v3(struct rtsx_chip *chip);
diff --git a/drivers/staging/skein/skein_api.c b/drivers/staging/skein/skein_api.c
index cab26e736111..c6526b6fbfb4 100644
--- a/drivers/staging/skein/skein_api.c
+++ b/drivers/staging/skein/skein_api.c
@@ -98,19 +98,16 @@ int skein_mac_init(struct skein_ctx *ctx, const u8 *key, size_t key_len,
 	switch (ctx->skein_size) {
 	case SKEIN_256:
 		ret = skein_256_init_ext(&ctx->m.s256, hash_bit_len,
-					 tree_info,
-					 (const u8 *)key, key_len);
+					 tree_info, key, key_len);
 
 		break;
 	case SKEIN_512:
 		ret = skein_512_init_ext(&ctx->m.s512, hash_bit_len,
-					 tree_info,
-					 (const u8 *)key, key_len);
+					 tree_info, key, key_len);
 		break;
 	case SKEIN_1024:
 		ret = skein_1024_init_ext(&ctx->m.s1024, hash_bit_len,
-					  tree_info,
-					  (const u8 *)key, key_len);
+					  tree_info, key, key_len);
 
 		break;
 	}
@@ -152,16 +149,13 @@ int skein_update(struct skein_ctx *ctx, const u8 *msg,
 
 	switch (ctx->skein_size) {
 	case SKEIN_256:
-		ret = skein_256_update(&ctx->m.s256, (const u8 *)msg,
-				       msg_byte_cnt);
+		ret = skein_256_update(&ctx->m.s256, msg, msg_byte_cnt);
 		break;
 	case SKEIN_512:
-		ret = skein_512_update(&ctx->m.s512, (const u8 *)msg,
-				       msg_byte_cnt);
+		ret = skein_512_update(&ctx->m.s512, msg, msg_byte_cnt);
 		break;
 	case SKEIN_1024:
-		ret = skein_1024_update(&ctx->m.s1024, (const u8 *)msg,
-					msg_byte_cnt);
+		ret = skein_1024_update(&ctx->m.s1024, msg, msg_byte_cnt);
 		break;
 	}
 	return ret;
@@ -211,7 +205,7 @@ int skein_update_bits(struct skein_ctx *ctx, const u8 *msg,
 	/* partial byte bit mask */
 	mask = (u8)(1u << (7 - (msg_bit_cnt & 7)));
 	/* apply bit padding on final byte (in the buffer) */
-	up[length - 1]  = (u8)((up[length - 1] & (0 - mask)) | mask);
+	up[length - 1]  = (up[length - 1] & (0 - mask)) | mask;
 
 	return SKEIN_SUCCESS;
 }
@@ -224,13 +218,13 @@ int skein_final(struct skein_ctx *ctx, u8 *hash)
 
 	switch (ctx->skein_size) {
 	case SKEIN_256:
-		ret = skein_256_final(&ctx->m.s256, (u8 *)hash);
+		ret = skein_256_final(&ctx->m.s256, hash);
 		break;
 	case SKEIN_512:
-		ret = skein_512_final(&ctx->m.s512, (u8 *)hash);
+		ret = skein_512_final(&ctx->m.s512, hash);
 		break;
 	case SKEIN_1024:
-		ret = skein_1024_final(&ctx->m.s1024, (u8 *)hash);
+		ret = skein_1024_final(&ctx->m.s1024, hash);
 		break;
 	}
 	return ret;
diff --git a/drivers/staging/skein/threefish_block.c b/drivers/staging/skein/threefish_block.c
index a95563fad071..50640656c10d 100644
--- a/drivers/staging/skein/threefish_block.c
+++ b/drivers/staging/skein/threefish_block.c
@@ -64,7 +64,6 @@ void threefish_encrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 += b1;
 	b1 = rol64(b1, 32) ^ b2;
 
-
 	b1 += k3 + t2;
 	b0 += b1 + k2;
 	b1 = rol64(b1, 14) ^ b0;
@@ -117,7 +116,6 @@ void threefish_encrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 += b1;
 	b1 = rol64(b1, 32) ^ b2;
 
-
 	b1 += k0 + t1;
 	b0 += b1 + k4;
 	b1 = rol64(b1, 14) ^ b0;
@@ -170,7 +168,6 @@ void threefish_encrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 += b1;
 	b1 = rol64(b1, 32) ^ b2;
 
-
 	b1 += k2 + t0;
 	b0 += b1 + k1;
 	b1 = rol64(b1, 14) ^ b0;
@@ -223,7 +220,6 @@ void threefish_encrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 += b1;
 	b1 = rol64(b1, 32) ^ b2;
 
-
 	b1 += k4 + t2;
 	b0 += b1 + k3;
 	b1 = rol64(b1, 14) ^ b0;
@@ -276,7 +272,6 @@ void threefish_encrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 += b1;
 	b1 = rol64(b1, 32) ^ b2;
 
-
 	b1 += k1 + t1;
 	b0 += b1 + k0;
 	b1 = rol64(b1, 14) ^ b0;
@@ -329,7 +324,6 @@ void threefish_encrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 += b1;
 	b1 = rol64(b1, 32) ^ b2;
 
-
 	b1 += k3 + t0;
 	b0 += b1 + k2;
 	b1 = rol64(b1, 14) ^ b0;
@@ -382,7 +376,6 @@ void threefish_encrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 += b1;
 	b1 = rol64(b1, 32) ^ b2;
 
-
 	b1 += k0 + t2;
 	b0 += b1 + k4;
 	b1 = rol64(b1, 14) ^ b0;
@@ -435,7 +428,6 @@ void threefish_encrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 += b1;
 	b1 = rol64(b1, 32) ^ b2;
 
-
 	b1 += k2 + t1;
 	b0 += b1 + k1;
 	b1 = rol64(b1, 14) ^ b0;
@@ -579,7 +571,6 @@ void threefish_decrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 -= b3 + k3 + t2;
 	b3 -= k4 + 16;
 
-
 	tmp = b3 ^ b0;
 	b3 = ror64(tmp, 32);
 	b0 -= b3;
@@ -648,7 +639,6 @@ void threefish_decrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 -= b3 + k1 + t0;
 	b3 -= k2 + 14;
 
-
 	tmp = b3 ^ b0;
 	b3 = ror64(tmp, 32);
 	b0 -= b3;
@@ -717,7 +707,6 @@ void threefish_decrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 -= b3 + k4 + t1;
 	b3 -= k0 + 12;
 
-
 	tmp = b3 ^ b0;
 	b3 = ror64(tmp, 32);
 	b0 -= b3;
@@ -786,7 +775,6 @@ void threefish_decrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 -= b3 + k2 + t2;
 	b3 -= k3 + 10;
 
-
 	tmp = b3 ^ b0;
 	b3 = ror64(tmp, 32);
 	b0 -= b3;
@@ -855,7 +843,6 @@ void threefish_decrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 -= b3 + k0 + t0;
 	b3 -= k1 + 8;
 
-
 	tmp = b3 ^ b0;
 	b3 = ror64(tmp, 32);
 	b0 -= b3;
@@ -924,7 +911,6 @@ void threefish_decrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 -= b3 + k3 + t1;
 	b3 -= k4 + 6;
 
-
 	tmp = b3 ^ b0;
 	b3 = ror64(tmp, 32);
 	b0 -= b3;
@@ -993,7 +979,6 @@ void threefish_decrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 -= b3 + k1 + t2;
 	b3 -= k2 + 4;
 
-
 	tmp = b3 ^ b0;
 	b3 = ror64(tmp, 32);
 	b0 -= b3;
@@ -1062,7 +1047,6 @@ void threefish_decrypt_256(struct threefish_key *key_ctx, u64 *input,
 	b2 -= b3 + k4 + t0;
 	b3 -= k0 + 2;
 
-
 	tmp = b3 ^ b0;
 	b3 = ror64(tmp, 32);
 	b0 -= b3;
diff --git a/drivers/staging/slicoss/Kconfig b/drivers/staging/slicoss/Kconfig
deleted file mode 100644
index 5c2a15b42dfe..000000000000
--- a/drivers/staging/slicoss/Kconfig
+++ /dev/null
@@ -1,14 +0,0 @@
-config SLICOSS
-	tristate "Alacritech Gigabit IS-NIC support"
-	depends on PCI && X86 && NET
-	default n
-	help
-	  This driver supports Alacritech's IS-NIC gigabit ethernet cards.
-
-	  This includes the following devices:
-	    Mojave cards (single port PCI Gigabit) both copper and fiber
-	    Oasis cards (single and dual port PCI-x Gigabit) copper and fiber
-	    Kalahari cards (dual and quad port PCI-e Gigabit) copper and fiber
-
-	  To compile this driver as a module, choose M here: the module
-	  will be called slicoss.
diff --git a/drivers/staging/slicoss/Makefile b/drivers/staging/slicoss/Makefile
deleted file mode 100644
index 7bc9e9b9d3ab..000000000000
--- a/drivers/staging/slicoss/Makefile
+++ /dev/null
@@ -1 +0,0 @@
-obj-$(CONFIG_SLICOSS) += slicoss.o
diff --git a/drivers/staging/slicoss/README b/drivers/staging/slicoss/README
deleted file mode 100644
index 4fa50e73ce86..000000000000
--- a/drivers/staging/slicoss/README
+++ /dev/null
@@ -1,7 +0,0 @@
-This driver is supposed to support:
-
-	Mojave cards (single port PCI Gigabit) both copper and fiber
-	Oasis cards (single and dual port PCI-x Gigabit) copper and fiber
-	Kalahari cards (dual and quad port PCI-e Gigabit) copper and fiber
-
-The driver was actually tested on Oasis and Kalahari cards.
diff --git a/drivers/staging/slicoss/TODO b/drivers/staging/slicoss/TODO
deleted file mode 100644
index 9019729b7be6..000000000000
--- a/drivers/staging/slicoss/TODO
+++ /dev/null
@@ -1,36 +0,0 @@
-TODO:
-	- move firmware loading to request_firmware()
-	- remove direct memory access of structures
-	- any remaining sparse and checkpatch.pl warnings
-
-	- use net_device_ops
-	- use dev->stats rather than adapter->stats
-	- don't cast netdev_priv it is already void
-	- GET RID OF MACROS
-	- work on all architectures
-	   - without CONFIG_X86_64 confusion
-	   - do 64 bit correctly
-	   - don't depend on order of union
-	- get rid of ASSERT(), use BUG() instead but only where necessary
-	  looks like most aren't really useful
-	- no new SIOCDEVPRIVATE ioctl allowed
-	- don't use module_param for configuring interrupt mitigation
-	  use ethtool instead
-	- reorder code to elminate use of forward declarations
-	- don't keep private linked list of drivers.
-	- use PCI_DEVICE()
-	- do ethtool correctly using ethtool_ops
-	- NAPI?
-	- wasted overhead of extra stats
-	- state variables for things that are
-	  easily available and shouldn't be kept in card structure, cardnum, ...
-	  slotnumber, events, ...
-	- volatile == bad design => bad code
-	- locking too fine grained, not designed just throw more locks
-	  at problem
-
-Please send patches to:
-        Greg Kroah-Hartman <gregkh@linuxfoundation.org>
-and Cc: Lior Dotan <liodot@gmail.com> and Christopher Harrer
-<charrer@alacritech.com> as well as they are also able to test out any
-changes.
diff --git a/drivers/staging/slicoss/slic.h b/drivers/staging/slicoss/slic.h
deleted file mode 100644
index 420546d43002..000000000000
--- a/drivers/staging/slicoss/slic.h
+++ /dev/null
@@ -1,573 +0,0 @@
-/**************************************************************************
- *
- * Copyright (c) 2000-2002 Alacritech, Inc.  All rights reserved.
- *
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above
- *    copyright notice, this list of conditions and the following
- *    disclaimer in the documentation and/or other materials provided
- *    with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY ALACRITECH, INC. ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL ALACRITECH, INC. OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * The views and conclusions contained in the software and documentation
- * are those of the authors and should not be interpreted as representing
- * official policies, either expressed or implied, of Alacritech, Inc.
- *
- **************************************************************************/
-
-/*
- * FILENAME: slic.h
- *
- * This is the base set of header definitions for the SLICOSS driver.
- */
-#ifndef __SLIC_DRIVER_H__
-#define __SLIC_DRIVER_H__
-
-/* firmware stuff */
-#define OASIS_UCODE_VERS_STRING	"1.2"
-#define OASIS_UCODE_VERS_DATE	"2006/03/27 15:10:37"
-#define OASIS_UCODE_HOSTIF_ID	3
-
-#define MOJAVE_UCODE_VERS_STRING	"1.2"
-#define MOJAVE_UCODE_VERS_DATE		"2006/03/27 15:12:22"
-#define MOJAVE_UCODE_HOSTIF_ID		3
-
-#define GB_RCVUCODE_VERS_STRING	"1.2"
-#define GB_RCVUCODE_VERS_DATE	"2006/03/27 15:12:15"
-static u32 OasisRcvUCodeLen = 512;
-static u32 GBRcvUCodeLen = 512;
-#define SECTION_SIZE 65536
-
-#define SLIC_RSPQ_PAGES_GB        10
-#define SLIC_RSPQ_BUFSINPAGE      (PAGE_SIZE / SLIC_RSPBUF_SIZE)
-
-struct slic_rspqueue {
-	u32             offset;
-	u32             pageindex;
-	u32             num_pages;
-	struct slic_rspbuf *rspbuf;
-	u32 *vaddr[SLIC_RSPQ_PAGES_GB];
-	dma_addr_t          paddr[SLIC_RSPQ_PAGES_GB];
-};
-
-#define SLIC_RCVQ_EXPANSION         1
-#define SLIC_RCVQ_ENTRIES           (256 * SLIC_RCVQ_EXPANSION)
-#define SLIC_RCVQ_MINENTRIES        (SLIC_RCVQ_ENTRIES / 2)
-#define SLIC_RCVQ_MAX_PROCESS_ISR   ((SLIC_RCVQ_ENTRIES * 4))
-#define SLIC_RCVQ_RCVBUFSIZE        2048
-#define SLIC_RCVQ_FILLENTRIES       (16 * SLIC_RCVQ_EXPANSION)
-#define SLIC_RCVQ_FILLTHRESH        (SLIC_RCVQ_ENTRIES - SLIC_RCVQ_FILLENTRIES)
-
-struct slic_rcvqueue {
-	struct sk_buff    *head;
-	struct sk_buff    *tail;
-	u32            count;
-	u32            size;
-	u32            errors;
-};
-
-struct slic_rcvbuf_info {
-	u32     id;
-	u32     starttime;
-	u32     stoptime;
-	u32     slicworld;
-	u32     lasttime;
-	u32     lastid;
-};
-
-/*
- * SLIC Handle structure.  Used to restrict handle values to
- * 32 bits by using an index rather than an address.
- * Simplifies ucode in 64-bit systems
- */
-struct slic_handle_word {
-	union {
-		struct {
-			ushort      index;
-			ushort      bottombits; /* to denote num bufs to card */
-		}  parts;
-		u32         whole;
-	}  handle;
-};
-
-struct slic_handle {
-	struct slic_handle_word  token;  /* token passed between host and card*/
-	ushort                      type;
-	void *address;    /* actual address of the object*/
-	ushort                      offset;
-	struct slic_handle       *other_handle;
-	struct slic_handle       *next;
-};
-
-#define SLIC_HANDLE_FREE        0x0000
-#define SLIC_HANDLE_DATA        0x0001
-#define SLIC_HANDLE_CMD         0x0002
-#define SLIC_HANDLE_CONTEXT     0x0003
-#define SLIC_HANDLE_TEAM        0x0004
-
-#define handle_index        handle.parts.index
-#define handle_bottom       handle.parts.bottombits
-#define handle_token        handle.whole
-
-#define SLIC_HOSTCMD_SIZE    512
-
-struct slic_hostcmd {
-	struct slic_host64_cmd  cmd64;
-	u32                    type;
-	struct sk_buff            *skb;
-	u32                    paddrl;
-	u32                    paddrh;
-	u32                    busy;
-	u32                    cmdsize;
-	ushort                     numbufs;
-	struct slic_handle    *pslic_handle;/* handle associated with command */
-	struct slic_hostcmd    *next;
-	struct slic_hostcmd    *next_all;
-};
-
-#define SLIC_CMDQ_CMDSINPAGE    (PAGE_SIZE / SLIC_HOSTCMD_SIZE)
-#define SLIC_CMD_DUMB            3
-#define SLIC_CMDQ_INITCMDS       256
-#define SLIC_CMDQ_MAXCMDS        256
-#define SLIC_CMDQ_MAXOUTSTAND    SLIC_CMDQ_MAXCMDS
-#define SLIC_CMDQ_MAXPAGES       (SLIC_CMDQ_MAXCMDS / SLIC_CMDQ_CMDSINPAGE)
-#define SLIC_CMDQ_INITPAGES      (SLIC_CMDQ_INITCMDS / SLIC_CMDQ_CMDSINPAGE)
-
-struct slic_cmdqmem {
-	int pagecnt;
-	u32 *pages[SLIC_CMDQ_MAXPAGES];
-	dma_addr_t dma_pages[SLIC_CMDQ_MAXPAGES];
-};
-
-struct slic_cmdqueue {
-	struct slic_hostcmd *head;
-	struct slic_hostcmd *tail;
-	int count;
-	spinlock_t lock;
-};
-
-#define SLIC_MAX_CARDS              32
-#define SLIC_MAX_PORTS              4        /* Max # of ports per card   */
-
-struct mcast_address {
-	unsigned char address[6];
-	struct mcast_address *next;
-};
-
-#define CARD_DOWN        0x00000000
-#define CARD_UP          0x00000001
-#define CARD_FAIL        0x00000002
-#define CARD_DIAG        0x00000003
-#define CARD_SLEEP       0x00000004
-
-#define ADAPT_DOWN             0x00
-#define ADAPT_UP               0x01
-#define ADAPT_FAIL             0x02
-#define ADAPT_RESET            0x03
-#define ADAPT_SLEEP            0x04
-
-#define ADAPT_FLAGS_BOOTTIME            0x0001
-#define ADAPT_FLAGS_IS64BIT             0x0002
-#define ADAPT_FLAGS_PENDINGLINKDOWN     0x0004
-#define ADAPT_FLAGS_FIBERMEDIA          0x0008
-#define ADAPT_FLAGS_LOCKS_ALLOCED       0x0010
-#define ADAPT_FLAGS_INT_REGISTERED      0x0020
-#define ADAPT_FLAGS_LOAD_TIMER_SET      0x0040
-#define ADAPT_FLAGS_STATS_TIMER_SET     0x0080
-#define ADAPT_FLAGS_RESET_TIMER_SET     0x0100
-
-#define LINK_DOWN              0x00
-#define LINK_CONFIG            0x01
-#define LINK_UP                0x02
-
-#define LINK_10MB              0x00
-#define LINK_100MB             0x01
-#define LINK_AUTOSPEED         0x02
-#define LINK_1000MB            0x03
-#define LINK_10000MB           0x04
-
-#define LINK_HALFD             0x00
-#define LINK_FULLD             0x01
-#define LINK_AUTOD             0x02
-
-#define MAC_DIRECTED     0x00000001
-#define MAC_BCAST        0x00000002
-#define MAC_MCAST        0x00000004
-#define MAC_PROMISC      0x00000008
-#define MAC_LOOPBACK     0x00000010
-#define MAC_ALLMCAST     0x00000020
-
-#define SLIC_DUPLEX(x)    ((x == LINK_FULLD) ? "FDX" : "HDX")
-#define SLIC_SPEED(x)     ((x == LINK_100MB) ? "100Mb" : ((x == LINK_1000MB) ?\
-				"1000Mb" : " 10Mb"))
-#define SLIC_LINKSTATE(x) ((x == LINK_DOWN) ? "Down" : "Up  ")
-#define SLIC_ADAPTER_STATE(x) ((x == ADAPT_UP) ? "UP" : "Down")
-#define SLIC_CARD_STATE(x)    ((x == CARD_UP) ? "UP" : "Down")
-
-struct slic_iface_stats {
-	/*
-	 * Stats
-	 */
-	u64        xmt_bytes;
-	u64        xmt_ucast;
-	u64        xmt_mcast;
-	u64        xmt_bcast;
-	u64        xmt_errors;
-	u64        xmt_discards;
-	u64        xmit_collisions;
-	u64        xmit_excess_xmit_collisions;
-	u64        rcv_bytes;
-	u64        rcv_ucast;
-	u64        rcv_mcast;
-	u64        rcv_bcast;
-	u64        rcv_errors;
-	u64        rcv_discards;
-};
-
-struct sliccp_stats {
-	u64        xmit_tcp_segs;
-	u64        xmit_tcp_bytes;
-	u64        rcv_tcp_segs;
-	u64        rcv_tcp_bytes;
-};
-
-struct slicnet_stats {
-	struct sliccp_stats        tcp;
-	struct slic_iface_stats      iface;
-};
-
-#define SLIC_LOADTIMER_PERIOD     1
-#define SLIC_INTAGG_DEFAULT       200
-#define SLIC_LOAD_0               0
-#define SLIC_INTAGG_0             0
-#define SLIC_LOAD_1               8000
-#define SLIC_LOAD_2               10000
-#define SLIC_LOAD_3               12000
-#define SLIC_LOAD_4               14000
-#define SLIC_LOAD_5               16000
-#define SLIC_INTAGG_1             50
-#define SLIC_INTAGG_2             100
-#define SLIC_INTAGG_3             150
-#define SLIC_INTAGG_4             200
-#define SLIC_INTAGG_5             250
-#define SLIC_LOAD_1GB             3000
-#define SLIC_LOAD_2GB             6000
-#define SLIC_LOAD_3GB             12000
-#define SLIC_LOAD_4GB             24000
-#define SLIC_LOAD_5GB             48000
-#define SLIC_INTAGG_1GB           50
-#define SLIC_INTAGG_2GB           75
-#define SLIC_INTAGG_3GB           100
-#define SLIC_INTAGG_4GB           100
-#define SLIC_INTAGG_5GB           100
-
-struct ether_header {
-	unsigned char    ether_dhost[6];
-	unsigned char    ether_shost[6];
-	ushort   ether_type;
-};
-
-struct sliccard {
-	uint              busnumber;
-	uint              slotnumber;
-	uint              state;
-	uint              cardnum;
-	uint              card_size;
-	uint              adapters_activated;
-	uint              adapters_allocated;
-	uint              adapters_sleeping;
-	uint              gennumber;
-	u32           events;
-	u32           loadlevel_current;
-	u32           load;
-	uint              reset_in_progress;
-	u32           pingstatus;
-	u32           bad_pingstatus;
-	struct timer_list loadtimer;
-	u32           loadtimerset;
-	uint              config_set;
-	struct slic_config  config;
-	struct adapter  *master;
-	struct adapter  *adapter[SLIC_MAX_PORTS];
-	struct sliccard *next;
-	u32             error_interrupts;
-	u32             error_rmiss_interrupts;
-	u32             rcv_interrupts;
-	u32             xmit_interrupts;
-	u32             num_isrs;
-	u32             false_interrupts;
-	u32             max_isr_rcvs;
-	u32             max_isr_xmits;
-	u32             rcv_interrupt_yields;
-	u32             tx_packets;
-	u32             debug_ix;
-	ushort              reg_type[32];
-	ushort              reg_offset[32];
-	u32             reg_value[32];
-	u32             reg_valueh[32];
-};
-
-#define NUM_CFG_SPACES      2
-#define NUM_CFG_REGS        64
-#define NUM_CFG_REG_ULONGS  (NUM_CFG_REGS / sizeof(u32))
-
-struct physcard {
-	struct adapter  *adapter[SLIC_MAX_PORTS];
-	struct physcard *next;
-	uint                adapters_allocd;
-
-/*
- * the following is not currently needed
- *	u32              bridge_busnum;
- *	u32              bridge_cfg[NUM_CFG_SPACES][NUM_CFG_REG_ULONGS];
- */
-};
-
-struct base_driver {
-	spinlock_t       driver_lock;
-	u32              num_slic_cards;
-	u32              num_slic_ports;
-	u32              num_slic_ports_active;
-	u32              dynamic_intagg;
-	struct sliccard  *slic_card;
-	struct physcard  *phys_card;
-	uint                 cardnuminuse[SLIC_MAX_CARDS];
-};
-
-struct slic_stats {
-	/* xmit stats */
-	u64 xmit_tcp_bytes;
-	u64 xmit_tcp_segs;
-	u64 xmit_bytes;
-	u64 xmit_collisions;
-	u64 xmit_unicasts;
-	u64 xmit_other_error;
-	u64 xmit_excess_collisions;
-	/* rcv stats */
-	u64 rcv_tcp_bytes;
-	u64 rcv_tcp_segs;
-	u64 rcv_bytes;
-	u64 rcv_unicasts;
-	u64 rcv_other_error;
-	u64 rcv_drops;
-};
-
-struct slic_shmem_data {
-	u32 isr;
-	u32 lnkstatus;
-	struct slic_stats stats;
-};
-
-struct slic_shmemory {
-	dma_addr_t isr_phaddr;
-	dma_addr_t lnkstatus_phaddr;
-	dma_addr_t stats_phaddr;
-	struct slic_shmem_data __iomem *shmem_data;
-};
-
-struct slic_upr {
-	uint               adapter;
-	u32            upr_request;
-	u32            upr_data;
-	u32            upr_data_h;
-	u32            upr_buffer;
-	u32            upr_buffer_h;
-	struct slic_upr *next;
-};
-
-struct slic_ifevents {
-	uint        oflow802;
-	uint        uflow802;
-	uint        Tprtoflow;
-	uint        rcvearly;
-	uint        Bufov;
-	uint        Carre;
-	uint        Longe;
-	uint        Invp;
-	uint        Crc;
-	uint        Drbl;
-	uint        Code;
-	uint        IpHlen;
-	uint        IpLen;
-	uint        IpCsum;
-	uint        TpCsum;
-	uint        TpHlen;
-};
-
-struct adapter {
-	void *ifp;
-	struct sliccard *card;
-	uint                port;
-	struct physcard *physcard;
-	uint                physport;
-	uint                cardindex;
-	uint                card_size;
-	uint                chipid;
-	struct net_device  *netdev;
-	spinlock_t          adapter_lock;
-	spinlock_t          reset_lock;
-	struct pci_dev     *pcidev;
-	uint                busnumber;
-	uint                slotnumber;
-	uint                functionnumber;
-	ushort              vendid;
-	ushort              devid;
-	ushort              subsysid;
-	u32             irq;
-	u32             drambase;
-	u32             dramlength;
-	uint                queues_initialized;
-	uint                allocated;
-	uint                activated;
-	u32             intrregistered;
-	uint                isp_initialized;
-	uint                gennumber;
-	struct slic_shmemory shmem;
-	dma_addr_t          phys_shmem;
-	void __iomem *regs;
-	unsigned char               state;
-	unsigned char               linkstate;
-	unsigned char               linkspeed;
-	unsigned char               linkduplex;
-	uint                flags;
-	unsigned char               macaddr[6];
-	unsigned char               currmacaddr[6];
-	u32             macopts;
-	ushort              devflags_prev;
-	u64             mcastmask;
-	struct mcast_address   *mcastaddrs;
-	struct slic_upr   *upr_list;
-	uint                upr_busy;
-	struct timer_list   pingtimer;
-	u32             pingtimerset;
-	struct timer_list   loadtimer;
-	u32             loadtimerset;
-	spinlock_t               upr_lock;
-	spinlock_t               bit64reglock;
-	struct slic_rspqueue     rspqueue;
-	struct slic_rcvqueue     rcvqueue;
-	struct slic_cmdqueue     cmdq_free;
-	struct slic_cmdqueue     cmdq_done;
-	struct slic_cmdqueue     cmdq_all;
-	struct slic_cmdqmem      cmdqmem;
-	/*
-	 * SLIC Handles
-	 */
-	/* Object handles*/
-	struct slic_handle slic_handles[SLIC_CMDQ_MAXCMDS + 1];
-	/* Free object handles*/
-	struct slic_handle *pfree_slic_handles;
-	/* Object handle list lock*/
-	spinlock_t          handle_lock;
-	ushort              slic_handle_ix;
-
-	u32             xmitq_full;
-	u32             all_reg_writes;
-	u32             icr_reg_writes;
-	u32             isr_reg_writes;
-	u32             error_interrupts;
-	u32             error_rmiss_interrupts;
-	u32             rx_errors;
-	u32             rcv_drops;
-	u32             rcv_interrupts;
-	u32             xmit_interrupts;
-	u32             linkevent_interrupts;
-	u32             upr_interrupts;
-	u32             num_isrs;
-	u32             false_interrupts;
-	u32             tx_packets;
-	u32             xmit_completes;
-	u32             tx_drops;
-	u32             rcv_broadcasts;
-	u32             rcv_multicasts;
-	u32             rcv_unicasts;
-	u32             max_isr_rcvs;
-	u32             max_isr_xmits;
-	u32             rcv_interrupt_yields;
-	u32             intagg_period;
-	u32             intagg_delay;
-	u32             dynamic_intagg;
-	struct inicpm_state    *inicpm_info;
-	void *pinicpm_info;
-	struct slic_ifevents  if_events;
-	struct slic_stats        inicstats_prev;
-	struct slicnet_stats     slic_stats;
-};
-
-static inline u32 slic_read32(struct adapter *adapter, unsigned int reg)
-{
-	return ioread32(adapter->regs + reg);
-}
-
-static inline void slic_write32(struct adapter *adapter, unsigned int reg,
-				u32 val)
-{
-	iowrite32(val, adapter->regs + reg);
-}
-
-static inline void slic_write64(struct adapter *adapter, unsigned int reg,
-				u32 val, u32 hiaddr)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&adapter->bit64reglock, flags);
-	slic_write32(adapter, SLIC_REG_ADDR_UPPER, hiaddr);
-	slic_write32(adapter, reg, val);
-	mmiowb();
-	spin_unlock_irqrestore(&adapter->bit64reglock, flags);
-}
-
-static inline void slic_flush_write(struct adapter *adapter)
-{
-	ioread32(adapter->regs + SLIC_REG_HOSTID);
-}
-
-#define UPDATE_STATS(largestat, newstat, oldstat)                        \
-{                                                                        \
-	if ((newstat) < (oldstat))                                       \
-		(largestat) += ((newstat) + (0xFFFFFFFF - oldstat + 1)); \
-	else                                                             \
-		(largestat) += ((newstat) - (oldstat));                  \
-}
-
-#define UPDATE_STATS_GB(largestat, newstat, oldstat)                     \
-{                                                                        \
-	(largestat) += ((newstat) - (oldstat));                          \
-}
-
-#if BITS_PER_LONG == 64
-#define   SLIC_GET_ADDR_LOW(_addr)  (u32)((u64)(_addr) & \
-	0x00000000FFFFFFFF)
-#define   SLIC_GET_ADDR_HIGH(_addr)  (u32)(((u64)(_addr) >> 32) & \
-	0x00000000FFFFFFFF)
-#elif BITS_PER_LONG == 32
-#define   SLIC_GET_ADDR_LOW(_addr)   (u32)(_addr)
-#define   SLIC_GET_ADDR_HIGH(_addr)  (u32)0
-#else
-#error BITS_PER_LONG must be 32 or 64
-#endif
-
-#define FLUSH		true
-#define DONT_FLUSH	false
-
-#define SIOCSLICSETINTAGG        (SIOCDEVPRIVATE + 10)
-
-#endif /*  __SLIC_DRIVER_H__ */
diff --git a/drivers/staging/slicoss/slichw.h b/drivers/staging/slicoss/slichw.h
deleted file mode 100644
index 49cb91aa02bb..000000000000
--- a/drivers/staging/slicoss/slichw.h
+++ /dev/null
@@ -1,652 +0,0 @@
-/**************************************************************************
- *
- * Copyright (c) 2000-2002 Alacritech, Inc.  All rights reserved.
- *
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above
- *    copyright notice, this list of conditions and the following
- *    disclaimer in the documentation and/or other materials provided
- *    with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY ALACRITECH, INC. ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL ALACRITECH, INC. OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * The views and conclusions contained in the software and documentation
- * are those of the authors and should not be interpreted as representing
- * official policies, either expressed or implied, of Alacritech, Inc.
- *
- **************************************************************************/
-
-/*
- * FILENAME: slichw.h
- *
- * This header file contains definitions that are common to our hardware.
- */
-#ifndef __SLICHW_H__
-#define __SLICHW_H__
-
-#define PCI_VENDOR_ID_ALACRITECH	0x139A
-#define SLIC_1GB_DEVICE_ID		0x0005
-#define SLIC_2GB_DEVICE_ID		0x0007	/* Oasis Device ID */
-
-#define SLIC_1GB_CICADA_SUBSYS_ID	0x0008
-
-#define SLIC_NBR_MACS		4
-
-#define SLIC_RCVBUF_SIZE	2048
-#define SLIC_RCVBUF_HEADSIZE	34
-#define SLIC_RCVBUF_TAILSIZE	0
-#define SLIC_RCVBUF_DATASIZE	(SLIC_RCVBUF_SIZE -		\
-				 (SLIC_RCVBUF_HEADSIZE +	\
-				  SLIC_RCVBUF_TAILSIZE))
-
-#define VGBSTAT_XPERR		0x40000000
-#define VGBSTAT_XERRSHFT	25
-#define VGBSTAT_XCSERR		0x23
-#define VGBSTAT_XUFLOW		0x22
-#define VGBSTAT_XHLEN		0x20
-#define VGBSTAT_NETERR		0x01000000
-#define VGBSTAT_NERRSHFT	16
-#define VGBSTAT_NERRMSK		0x1ff
-#define VGBSTAT_NCSERR		0x103
-#define VGBSTAT_NUFLOW		0x102
-#define VGBSTAT_NHLEN		0x100
-#define VGBSTAT_LNKERR		0x00000080
-#define VGBSTAT_LERRMSK		0xff
-#define VGBSTAT_LDEARLY		0x86
-#define VGBSTAT_LBOFLO		0x85
-#define VGBSTAT_LCODERR		0x84
-#define VGBSTAT_LDBLNBL		0x83
-#define VGBSTAT_LCRCERR		0x82
-#define VGBSTAT_LOFLO		0x81
-#define VGBSTAT_LUFLO		0x80
-#define IRHDDR_FLEN_MSK		0x0000ffff
-#define IRHDDR_SVALID		0x80000000
-#define IRHDDR_ERR		0x10000000
-#define VRHSTAT_802OE		0x80000000
-#define VRHSTAT_TPOFLO		0x10000000
-#define VRHSTATB_802UE		0x80000000
-#define VRHSTATB_RCVE		0x40000000
-#define VRHSTATB_BUFF		0x20000000
-#define VRHSTATB_CARRE		0x08000000
-#define VRHSTATB_LONGE		0x02000000
-#define VRHSTATB_PREA		0x01000000
-#define VRHSTATB_CRC		0x00800000
-#define VRHSTATB_DRBL		0x00400000
-#define VRHSTATB_CODE		0x00200000
-#define VRHSTATB_TPCSUM		0x00100000
-#define VRHSTATB_TPHLEN		0x00080000
-#define VRHSTATB_IPCSUM		0x00040000
-#define VRHSTATB_IPLERR		0x00020000
-#define VRHSTATB_IPHERR		0x00010000
-#define SLIC_MAX64_BCNT		23
-#define SLIC_MAX32_BCNT		26
-#define IHCMD_XMT_REQ		0x01
-#define IHFLG_IFSHFT		2
-#define SLIC_RSPBUF_SIZE	32
-
-#define SLIC_RESET_MAGIC	0xDEAD
-#define ICR_INT_OFF		0
-#define ICR_INT_ON		1
-#define ICR_INT_MASK		2
-
-#define ISR_ERR			0x80000000
-#define ISR_RCV			0x40000000
-#define ISR_CMD			0x20000000
-#define ISR_IO			0x60000000
-#define ISR_UPC			0x10000000
-#define ISR_LEVENT		0x08000000
-#define ISR_RMISS		0x02000000
-#define ISR_UPCERR		0x01000000
-#define ISR_XDROP		0x00800000
-#define ISR_UPCBSY		0x00020000
-#define ISR_EVMSK		0xffff0000
-#define ISR_PINGMASK		0x00700000
-#define ISR_PINGDSMASK		0x00710000
-#define ISR_UPCMASK		0x11000000
-#define SLIC_WCS_START		0x80000000
-#define SLIC_WCS_COMPARE	0x40000000
-#define SLIC_RCVWCS_BEGIN	0x40000000
-#define SLIC_RCVWCS_FINISH	0x80000000
-#define SLIC_PM_MAXPATTERNS	6
-#define SLIC_PM_PATTERNSIZE	128
-#define SLIC_PMCAPS_WAKEONLAN	0x00000001
-#define MIICR_REG_PCR		0x00000000
-#define MIICR_REG_4		0x00040000
-#define MIICR_REG_9		0x00090000
-#define MIICR_REG_16		0x00100000
-#define PCR_RESET		0x8000
-#define PCR_POWERDOWN		0x0800
-#define PCR_SPEED_100		0x2000
-#define PCR_SPEED_1000		0x0040
-#define PCR_AUTONEG		0x1000
-#define PCR_AUTONEG_RST		0x0200
-#define PCR_DUPLEX_FULL		0x0100
-#define PSR_LINKUP		0x0004
-
-#define PAR_ADV100FD		0x0100
-#define PAR_ADV100HD		0x0080
-#define PAR_ADV10FD		0x0040
-#define PAR_ADV10HD		0x0020
-#define PAR_ASYMPAUSE		0x0C00
-#define PAR_802_3		0x0001
-
-#define PAR_ADV1000XFD		0x0020
-#define PAR_ADV1000XHD		0x0040
-#define PAR_ASYMPAUSE_FIBER	0x0180
-
-#define PGC_ADV1000FD		0x0200
-#define PGC_ADV1000HD		0x0100
-#define SEEQ_LINKFAIL		0x4000
-#define SEEQ_SPEED		0x0080
-#define SEEQ_DUPLEX		0x0040
-#define TDK_DUPLEX		0x0800
-#define TDK_SPEED		0x0400
-#define MRV_REG16_XOVERON	0x0068
-#define MRV_REG16_XOVEROFF	0x0008
-#define MRV_SPEED_1000		0x8000
-#define MRV_SPEED_100		0x4000
-#define MRV_SPEED_10		0x0000
-#define MRV_FULLDUPLEX		0x2000
-#define MRV_LINKUP		0x0400
-
-#define GIG_LINKUP		0x0001
-#define GIG_FULLDUPLEX		0x0002
-#define GIG_SPEED_MASK		0x000C
-#define GIG_SPEED_1000		0x0008
-#define GIG_SPEED_100		0x0004
-#define GIG_SPEED_10		0x0000
-
-#define MCR_RESET		0x80000000
-#define MCR_CRCEN		0x40000000
-#define MCR_FULLD		0x10000000
-#define MCR_PAD			0x02000000
-#define MCR_RETRYLATE		0x01000000
-#define MCR_BOL_SHIFT		21
-#define MCR_IPG1_SHIFT		14
-#define MCR_IPG2_SHIFT		7
-#define MCR_IPG3_SHIFT		0
-#define GMCR_RESET		0x80000000
-#define GMCR_GBIT		0x20000000
-#define GMCR_FULLD		0x10000000
-#define GMCR_GAPBB_SHIFT	14
-#define GMCR_GAPR1_SHIFT	7
-#define GMCR_GAPR2_SHIFT	0
-#define GMCR_GAPBB_1000		0x60
-#define GMCR_GAPR1_1000		0x2C
-#define GMCR_GAPR2_1000		0x40
-#define GMCR_GAPBB_100		0x70
-#define GMCR_GAPR1_100		0x2C
-#define GMCR_GAPR2_100		0x40
-#define XCR_RESET		0x80000000
-#define XCR_XMTEN		0x40000000
-#define XCR_PAUSEEN		0x20000000
-#define XCR_LOADRNG		0x10000000
-#define RCR_RESET		0x80000000
-#define RCR_RCVEN		0x40000000
-#define RCR_RCVALL		0x20000000
-#define RCR_RCVBAD		0x10000000
-#define RCR_CTLEN		0x08000000
-#define RCR_ADDRAEN		0x02000000
-#define GXCR_RESET		0x80000000
-#define GXCR_XMTEN		0x40000000
-#define GXCR_PAUSEEN		0x20000000
-#define GRCR_RESET		0x80000000
-#define GRCR_RCVEN		0x40000000
-#define GRCR_RCVALL		0x20000000
-#define GRCR_RCVBAD		0x10000000
-#define GRCR_CTLEN		0x08000000
-#define GRCR_ADDRAEN		0x02000000
-#define GRCR_HASHSIZE_SHIFT	17
-#define GRCR_HASHSIZE		14
-
-#define SLIC_EEPROM_ID		0xA5A5
-#define SLIC_SRAM_SIZE2GB	(64 * 1024)
-#define SLIC_SRAM_SIZE1GB	(32 * 1024)
-#define SLIC_HOSTID_DEFAULT	0xFFFF		/* uninitialized hostid */
-#define SLIC_NBR_MACS		4
-
-struct slic_rcvbuf {
-	u8 pad1[6];
-	u16 pad2;
-	u32 pad3;
-	u32 pad4;
-	u32 buffer;
-	u32 length;
-	u32 status;
-	u32 pad5;
-	u16 pad6;
-	u8 data[SLIC_RCVBUF_DATASIZE];
-};
-
-struct slic_hddr_wds {
-	union {
-		struct {
-			u32 frame_status;
-			u32 frame_status_b;
-			u32 time_stamp;
-			u32 checksum;
-		} hdrs_14port;
-		struct {
-			u32 frame_status;
-			u16 ByteCnt;
-			u16 TpChksum;
-			u16 CtxHash;
-			u16 MacHash;
-			u32 BufLnk;
-		} hdrs_gbit;
-	} u0;
-};
-
-#define frame_status14		u0.hdrs_14port.frame_status
-#define frame_status_b14	u0.hdrs_14port.frame_status_b
-#define frame_statusGB		u0.hdrs_gbit.frame_status
-
-struct slic_host64sg {
-	u32 paddrl;
-	u32 paddrh;
-	u32 length;
-};
-
-struct slic_host64_cmd {
-	u32 hosthandle;
-	u32 RSVD;
-	u8 command;
-	u8 flags;
-	union {
-		u16 rsv1;
-		u16 rsv2;
-	} u0;
-	union {
-		struct {
-			u32 totlen;
-			struct slic_host64sg bufs[SLIC_MAX64_BCNT];
-		} slic_buffers;
-	} u;
-};
-
-struct slic_rspbuf {
-	u32 hosthandle;
-	u32 pad0;
-	u32 pad1;
-	u32 status;
-	u32 pad2[4];
-};
-
-/* Reset Register */
-#define SLIC_REG_RESET		0x0000
-/* Interrupt Control Register */
-#define SLIC_REG_ICR		0x0008
-/* Interrupt status pointer */
-#define SLIC_REG_ISP		0x0010
-/* Interrupt status */
-#define SLIC_REG_ISR		0x0018
-/*
- * Header buffer address reg
- * 31-8 - phy addr of set of contiguous hdr buffers
- *  7-0 - number of buffers passed
- * Buffers are 256 bytes long on 256-byte boundaries.
- */
-#define SLIC_REG_HBAR		0x0020
-/*
- * Data buffer handle & address reg
- * 4 sets of registers; Buffers are 2K bytes long 2 per 4K page.
- */
-#define SLIC_REG_DBAR		0x0028
-/*
- * Xmt Cmd buf addr regs.
- * 1 per XMT interface
- * 31-5 - phy addr of host command buffer
- *  4-0 - length of cmd in multiples of 32 bytes
- * Buffers are 32 bytes up to 512 bytes long
- */
-#define SLIC_REG_CBAR		0x0030
-/* Write control store */
-#define	SLIC_REG_WCS		0x0034
-/*
- * Response buffer address reg.
- * 31-8 - phy addr of set of contiguous response buffers
- * 7-0 - number of buffers passed
- * Buffers are 32 bytes long on 32-byte boundaries.
- */
-#define	SLIC_REG_RBAR		0x0038
-/* Read statistics (UPR) */
-#define	SLIC_REG_RSTAT		0x0040
-/* Read link status */
-#define	SLIC_REG_LSTAT		0x0048
-/* Write Mac Config */
-#define	SLIC_REG_WMCFG		0x0050
-/* Write phy register */
-#define SLIC_REG_WPHY		0x0058
-/* Rcv Cmd buf addr reg */
-#define	SLIC_REG_RCBAR		0x0060
-/* Read SLIC Config*/
-#define SLIC_REG_RCONFIG	0x0068
-/* Interrupt aggregation time */
-#define SLIC_REG_INTAGG		0x0070
-/* Write XMIT config reg */
-#define	SLIC_REG_WXCFG		0x0078
-/* Write RCV config reg */
-#define	SLIC_REG_WRCFG		0x0080
-/* Write rcv addr a low */
-#define	SLIC_REG_WRADDRAL	0x0088
-/* Write rcv addr a high */
-#define	SLIC_REG_WRADDRAH	0x0090
-/* Write rcv addr b low */
-#define	SLIC_REG_WRADDRBL	0x0098
-/* Write rcv addr b high */
-#define	SLIC_REG_WRADDRBH	0x00a0
-/* Low bits of mcast mask */
-#define	SLIC_REG_MCASTLOW	0x00a8
-/* High bits of mcast mask */
-#define	SLIC_REG_MCASTHIGH	0x00b0
-/* Ping the card */
-#define SLIC_REG_PING		0x00b8
-/* Dump command */
-#define SLIC_REG_DUMP_CMD	0x00c0
-/* Dump data pointer */
-#define SLIC_REG_DUMP_DATA	0x00c8
-/* Read card's pci_status register */
-#define	SLIC_REG_PCISTATUS	0x00d0
-/* Write hostid field */
-#define SLIC_REG_WRHOSTID	0x00d8
-/* Put card in a low power state */
-#define SLIC_REG_LOW_POWER	0x00e0
-/* Force slic into quiescent state  before soft reset */
-#define SLIC_REG_QUIESCE	0x00e8
-/* Reset interface queues */
-#define SLIC_REG_RESET_IFACE	0x00f0
-/*
- * Register is only written when it has changed.
- * Bits 63-32 for host i/f addrs.
- */
-#define SLIC_REG_ADDR_UPPER	0x00f8
-/* 64 bit Header buffer address reg */
-#define SLIC_REG_HBAR64		0x0100
-/* 64 bit Data buffer handle & address reg */
-#define SLIC_REG_DBAR64		0x0108
-/* 64 bit Xmt Cmd buf addr regs. */
-#define SLIC_REG_CBAR64		0x0110
-/* 64 bit Response buffer address reg.*/
-#define SLIC_REG_RBAR64		0x0118
-/* 64 bit Rcv Cmd buf addr reg*/
-#define	SLIC_REG_RCBAR64	0x0120
-/* Read statistics (64 bit UPR) */
-#define	SLIC_REG_RSTAT64	0x0128
-/* Download Gigabit RCV sequencer ucode */
-#define SLIC_REG_RCV_WCS	0x0130
-/* Write VlanId field */
-#define SLIC_REG_WRVLANID	0x0138
-/* Read Transformer info */
-#define SLIC_REG_READ_XF_INFO	0x0140
-/* Write Transformer info */
-#define SLIC_REG_WRITE_XF_INFO	0x0148
-/* Write card ticks per second */
-#define SLIC_REG_TICKS_PER_SEC	0x0170
-
-#define SLIC_REG_HOSTID		0x1554
-
-enum UPR_REQUEST {
-	SLIC_UPR_STATS,
-	SLIC_UPR_RLSR,
-	SLIC_UPR_WCFG,
-	SLIC_UPR_RCONFIG,
-	SLIC_UPR_RPHY,
-	SLIC_UPR_ENLB,
-	SLIC_UPR_ENCT,
-	SLIC_UPR_PDWN,
-	SLIC_UPR_PING,
-	SLIC_UPR_DUMP,
-};
-
-struct inicpm_wakepattern {
-	u32 patternlength;
-	u8 pattern[SLIC_PM_PATTERNSIZE];
-	u8 mask[SLIC_PM_PATTERNSIZE];
-};
-
-struct inicpm_state {
-	u32 powercaps;
-	u32 powerstate;
-	u32 wake_linkstatus;
-	u32 wake_magicpacket;
-	u32 wake_framepattern;
-	struct inicpm_wakepattern wakepattern[SLIC_PM_MAXPATTERNS];
-};
-
-struct slicpm_packet_pattern {
-	u32 priority;
-	u32 reserved;
-	u32 masksize;
-	u32 patternoffset;
-	u32 patternsize;
-	u32 patternflags;
-};
-
-enum slicpm_power_state {
-	slicpm_state_unspecified = 0,
-	slicpm_state_d0,
-	slicpm_state_d1,
-	slicpm_state_d2,
-	slicpm_state_d3,
-	slicpm_state_maximum
-};
-
-struct slicpm_wakeup_capabilities {
-	enum slicpm_power_state min_magic_packet_wakeup;
-	enum slicpm_power_state min_pattern_wakeup;
-	enum slicpm_power_state min_link_change_wakeup;
-};
-
-struct slic_pnp_capabilities {
-	u32 flags;
-	struct slicpm_wakeup_capabilities wakeup_capabilities;
-};
-
-struct slic_config_mac {
-	u8 macaddrA[6];
-};
-
-#define ATK_FRU_FORMAT		0x00
-#define VENDOR1_FRU_FORMAT	0x01
-#define VENDOR2_FRU_FORMAT	0x02
-#define VENDOR3_FRU_FORMAT	0x03
-#define VENDOR4_FRU_FORMAT	0x04
-#define NO_FRU_FORMAT		0xFF
-
-struct atk_fru {
-	u8 assembly[6];
-	u8 revision[2];
-	u8 serial[14];
-	u8 pad[3];
-};
-
-struct vendor1_fru {
-	u8 commodity;
-	u8 assembly[4];
-	u8 revision[2];
-	u8 supplier[2];
-	u8 date[2];
-	u8 sequence[3];
-	u8 pad[13];
-};
-
-struct vendor2_fru {
-	u8 part[8];
-	u8 supplier[5];
-	u8 date[3];
-	u8 sequence[4];
-	u8 pad[7];
-};
-
-struct vendor3_fru {
-	u8 assembly[6];
-	u8 revision[2];
-	u8 serial[14];
-	u8 pad[3];
-};
-
-struct vendor4_fru {
-	u8 number[8];
-	u8 part[8];
-	u8 version[8];
-	u8 pad[3];
-};
-
-union oemfru {
-	struct vendor1_fru vendor1_fru;
-	struct vendor2_fru vendor2_fru;
-	struct vendor3_fru vendor3_fru;
-	struct vendor4_fru vendor4_fru;
-};
-
-/*
- * SLIC EEPROM structure for Mojave
- */
-struct slic_eeprom {
-	u16 Id;			/* 00 EEPROM/FLASH Magic code 'A5A5'*/
-	u16 EecodeSize;		/* 01 Size of EEPROM Codes (bytes * 4)*/
-	u16 FlashSize;		/* 02 Flash size */
-	u16 EepromSize;		/* 03 EEPROM Size */
-	u16 VendorId;		/* 04 Vendor ID */
-	u16 DeviceId;		/* 05 Device ID */
-	u8 RevisionId;		/* 06 Revision ID */
-	u8 ClassCode[3];	/* 07 Class Code */
-	u8 DbgIntPin;		/* 08 Debug Interrupt pin */
-	u8 NetIntPin0;		/*    Network Interrupt Pin */
-	u8 MinGrant;		/* 09 Minimum grant */
-	u8 MaxLat;		/*    Maximum Latency */
-	u16 PciStatus;		/* 10 PCI Status */
-	u16 SubSysVId;		/* 11 Subsystem Vendor Id */
-	u16 SubSysId;		/* 12 Subsystem ID */
-	u16 DbgDevId;		/* 13 Debug Device Id */
-	u16 DramRomFn;		/* 14 Dram/Rom function */
-	u16 DSize2Pci;		/* 15 DRAM size to PCI (bytes * 64K) */
-	u16 RSize2Pci;		/* 16 ROM extension size to PCI (bytes * 4k) */
-	u8 NetIntPin1;		/* 17 Network Interface Pin 1
-				 *  (simba/leone only)
-				 */
-	u8 NetIntPin2;		/* Network Interface Pin 2 (simba/leone only)*/
-	union {
-		u8 NetIntPin3;	/* 18 Network Interface Pin 3 (simba only) */
-		u8 FreeTime;	/* FreeTime setting (leone/mojave only) */
-	} u1;
-	u8 TBIctl;		/* 10-bit interface control (Mojave only) */
-	u16 DramSize;		/* 19 DRAM size (bytes * 64k) */
-	union {
-		struct {
-			/* Mac Interface Specific portions */
-			struct slic_config_mac	MacInfo[SLIC_NBR_MACS];
-		} mac;				/* MAC access for all boards */
-		struct {
-			/* use above struct for MAC access */
-			struct slic_config_mac	pad[SLIC_NBR_MACS - 1];
-			u16 DeviceId2;	/* Device ID for 2nd PCI function */
-			u8 IntPin2;	/* Interrupt pin for 2nd PCI function */
-			u8 ClassCode2[3]; /* Class Code for 2nd PCI function */
-		} mojave;	/* 2nd function access for gigabit board */
-	} u2;
-	u16 CfgByte6;		/* Config Byte 6 */
-	u16 PMECapab;		/* Power Mgment capabilities */
-	u16 NwClkCtrls;		/* NetworkClockControls */
-	u8 FruFormat;		/* Alacritech FRU format type */
-	struct atk_fru  AtkFru;	/* Alacritech FRU information */
-	u8 OemFruFormat;	/* optional OEM FRU format type */
-	union oemfru OemFru;	/* optional OEM FRU information */
-	u8	Pad[4];		/* Pad to 128 bytes - includes 2 cksum bytes
-				 * (if OEM FRU info exists) and two unusable
-				 * bytes at the end
-				 */
-};
-
-/* SLIC EEPROM structure for Oasis */
-struct oslic_eeprom {
-	u16 Id;			/* 00 EEPROM/FLASH Magic code 'A5A5' */
-	u16 EecodeSize;		/* 01 Size of EEPROM Codes (bytes * 4)*/
-	u16 FlashConfig0;	/* 02 Flash Config for SPI device 0 */
-	u16 FlashConfig1;	/* 03 Flash Config for SPI device 1 */
-	u16 VendorId;		/* 04 Vendor ID */
-	u16 DeviceId;		/* 05 Device ID (function 0) */
-	u8 RevisionId;		/* 06 Revision ID */
-	u8 ClassCode[3];	/* 07 Class Code for PCI function 0 */
-	u8 IntPin1;		/* 08 Interrupt pin for PCI function 1*/
-	u8 ClassCode2[3];	/* 09 Class Code for PCI function 1 */
-	u8 IntPin2;		/* 10 Interrupt pin for PCI function 2*/
-	u8 IntPin0;		/*    Interrupt pin for PCI function 0*/
-	u8 MinGrant;		/* 11 Minimum grant */
-	u8 MaxLat;		/*    Maximum Latency */
-	u16 SubSysVId;		/* 12 Subsystem Vendor Id */
-	u16 SubSysId;		/* 13 Subsystem ID */
-	u16 FlashSize;		/* 14 Flash size (bytes / 4K) */
-	u16 DSize2Pci;		/* 15 DRAM size to PCI (bytes / 64K) */
-	u16 RSize2Pci;		/* 16 Flash (ROM extension) size to PCI
-				 *	(bytes / 4K)
-				 */
-	u16 DeviceId1;		/* 17 Device Id (function 1) */
-	u16 DeviceId2;		/* 18 Device Id (function 2) */
-	u16 CfgByte6;		/* 19 Device Status Config Bytes 6-7 */
-	u16 PMECapab;		/* 20 Power Mgment capabilities */
-	u8 MSICapab;		/* 21 MSI capabilities */
-	u8 ClockDivider;	/*    Clock divider */
-	u16 PciStatusLow;	/* 22 PCI Status bits 15:0 */
-	u16 PciStatusHigh;	/* 23 PCI Status bits 31:16 */
-	u16 DramConfigLow;	/* 24 DRAM Configuration bits 15:0 */
-	u16 DramConfigHigh;	/* 25 DRAM Configuration bits 31:16 */
-	u16 DramSize;		/* 26 DRAM size (bytes / 64K) */
-	u16 GpioTbiCtl;		/* 27 GPIO/TBI controls for functions 1/0 */
-	u16 EepromSize;		/* 28 EEPROM Size */
-	struct slic_config_mac MacInfo[2];	/* 29 MAC addresses (2 ports) */
-	u8 FruFormat;		/* 35 Alacritech FRU format type */
-	struct atk_fru	AtkFru;	/* Alacritech FRU information */
-	u8 OemFruFormat;	/* optional OEM FRU format type */
-	union oemfru OemFru;	/* optional OEM FRU information */
-	u8 Pad[4];		/* Pad to 128 bytes - includes 2 checksum bytes
-				 * (if OEM FRU info exists) and two unusable
-				 * bytes at the end
-				 */
-};
-
-#define	MAX_EECODE_SIZE	sizeof(struct slic_eeprom)
-#define MIN_EECODE_SIZE	0x62	/* code size without optional OEM FRU stuff */
-
-/*
- * SLIC CONFIG structure
- *
- * This structure lives in the CARD structure and is valid for all board types.
- * It is filled in from the appropriate EEPROM structure by
- * SlicGetConfigData()
- */
-struct slic_config {
-	bool EepromValid;	/* Valid EEPROM flag (checksum good?) */
-	u16 DramSize;		/* DRAM size (bytes / 64K) */
-	struct slic_config_mac MacInfo[SLIC_NBR_MACS]; /* MAC addresses */
-	u8 FruFormat;		/* Alacritech FRU format type */
-	struct atk_fru	AtkFru;	/* Alacritech FRU information */
-	u8 OemFruFormat;	/* optional OEM FRU format type */
-	union {
-		struct vendor1_fru vendor1_fru;
-		struct vendor2_fru vendor2_fru;
-		struct vendor3_fru vendor3_fru;
-		struct vendor4_fru vendor4_fru;
-	} OemFru;
-};
-
-#pragma pack()
-
-#endif
diff --git a/drivers/staging/slicoss/slicoss.c b/drivers/staging/slicoss/slicoss.c
deleted file mode 100644
index 2802b900f8ee..000000000000
--- a/drivers/staging/slicoss/slicoss.c
+++ /dev/null
@@ -1,3131 +0,0 @@
-/**************************************************************************
- *
- * Copyright  2000-2006 Alacritech, Inc.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above
- *    copyright notice, this list of conditions and the following
- *    disclaimer in the documentation and/or other materials provided
- *    with the distribution.
- *
- * Alternatively, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") version 2 as published by the Free
- * Software Foundation.
- *
- * THIS SOFTWARE IS PROVIDED BY ALACRITECH, INC. ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL ALACRITECH, INC. OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * The views and conclusions contained in the software and documentation
- * are those of the authors and should not be interpreted as representing
- * official policies, either expressed or implied, of Alacritech, Inc.
- *
- **************************************************************************/
-
-/*
- * FILENAME: slicoss.c
- *
- * The SLICOSS driver for Alacritech's IS-NIC products.
- *
- * This driver is supposed to support:
- *
- *      Mojave cards (single port PCI Gigabit) both copper and fiber
- *      Oasis cards (single and dual port PCI-x Gigabit) copper and fiber
- *      Kalahari cards (dual and quad port PCI-e Gigabit) copper and fiber
- *
- * The driver was actually tested on Oasis and Kalahari cards.
- *
- *
- * NOTE: This is the standard, non-accelerated version of Alacritech's
- *       IS-NIC driver.
- */
-
-#define KLUDGE_FOR_4GB_BOUNDARY         1
-#define DEBUG_MICROCODE                 1
-#define DBG                             1
-#define SLIC_INTERRUPT_PROCESS_LIMIT	1
-#define SLIC_OFFLOAD_IP_CHECKSUM	1
-#define STATS_TIMER_INTERVAL		2
-#define PING_TIMER_INTERVAL		1
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/kernel.h>
-#include <linux/string.h>
-#include <linux/errno.h>
-#include <linux/ioport.h>
-#include <linux/slab.h>
-#include <linux/interrupt.h>
-#include <linux/timer.h>
-#include <linux/pci.h>
-#include <linux/spinlock.h>
-#include <linux/init.h>
-#include <linux/bitops.h>
-#include <linux/io.h>
-#include <linux/netdevice.h>
-#include <linux/crc32.h>
-#include <linux/etherdevice.h>
-#include <linux/skbuff.h>
-#include <linux/delay.h>
-#include <linux/seq_file.h>
-#include <linux/kthread.h>
-#include <linux/module.h>
-
-#include <linux/firmware.h>
-#include <linux/types.h>
-#include <linux/dma-mapping.h>
-#include <linux/mii.h>
-#include <linux/if_vlan.h>
-#include <asm/unaligned.h>
-
-#include <linux/ethtool.h>
-#include <linux/uaccess.h>
-#include "slichw.h"
-#include "slic.h"
-
-static uint slic_first_init = 1;
-static char *slic_banner = "Alacritech SLIC Technology(tm) Server and Storage Accelerator (Non-Accelerated)";
-
-static char *slic_proc_version = "2.0.351  2006/07/14 12:26:00";
-
-static struct base_driver slic_global = { {}, 0, 0, 0, 1, NULL, NULL };
-#define DEFAULT_INTAGG_DELAY 100
-static unsigned int rcv_count;
-
-#define DRV_NAME          "slicoss"
-#define DRV_VERSION       "2.0.1"
-#define DRV_AUTHOR        "Alacritech, Inc. Engineering"
-#define DRV_DESCRIPTION   "Alacritech SLIC Techonology(tm) "\
-		"Non-Accelerated Driver"
-#define DRV_COPYRIGHT     "Copyright  2000-2006 Alacritech, Inc. "\
-		"All rights reserved."
-#define PFX		   DRV_NAME " "
-
-MODULE_AUTHOR(DRV_AUTHOR);
-MODULE_DESCRIPTION(DRV_DESCRIPTION);
-MODULE_LICENSE("Dual BSD/GPL");
-
-static const struct pci_device_id slic_pci_tbl[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_ALACRITECH, SLIC_1GB_DEVICE_ID) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_ALACRITECH, SLIC_2GB_DEVICE_ID) },
-	{ 0 }
-};
-
-static const struct ethtool_ops slic_ethtool_ops;
-
-MODULE_DEVICE_TABLE(pci, slic_pci_tbl);
-
-static void slic_mcast_set_bit(struct adapter *adapter, char *address)
-{
-	unsigned char crcpoly;
-
-	/* Get the CRC polynomial for the mac address */
-	/*
-	 * we use bits 1-8 (lsb), bitwise reversed,
-	 * msb (= lsb bit 0 before bitrev) is automatically discarded
-	 */
-	crcpoly = ether_crc(ETH_ALEN, address) >> 23;
-
-	/*
-	 * We only have space on the SLIC for 64 entries.  Lop
-	 * off the top two bits. (2^6 = 64)
-	 */
-	crcpoly &= 0x3F;
-
-	/* OR in the new bit into our 64 bit mask. */
-	adapter->mcastmask |= (u64)1 << crcpoly;
-}
-
-static void slic_mcast_set_mask(struct adapter *adapter)
-{
-	if (adapter->macopts & (MAC_ALLMCAST | MAC_PROMISC)) {
-		/*
-		 * Turn on all multicast addresses. We have to do this for
-		 * promiscuous mode as well as ALLMCAST mode.  It saves the
-		 * Microcode from having to keep state about the MAC
-		 * configuration.
-		 */
-		slic_write32(adapter, SLIC_REG_MCASTLOW, 0xFFFFFFFF);
-		slic_write32(adapter, SLIC_REG_MCASTHIGH, 0xFFFFFFFF);
-	} else {
-		/*
-		 * Commit our multicast mast to the SLIC by writing to the
-		 * multicast address mask registers
-		 */
-		slic_write32(adapter, SLIC_REG_MCASTLOW,
-			     (u32)(adapter->mcastmask & 0xFFFFFFFF));
-		slic_write32(adapter, SLIC_REG_MCASTHIGH,
-			     (u32)((adapter->mcastmask >> 32) & 0xFFFFFFFF));
-	}
-}
-
-static void slic_timer_ping(ulong dev)
-{
-	struct adapter *adapter;
-	struct sliccard *card;
-
-	adapter = netdev_priv((struct net_device *)dev);
-	card = adapter->card;
-
-	adapter->pingtimer.expires = jiffies + (PING_TIMER_INTERVAL * HZ);
-	add_timer(&adapter->pingtimer);
-}
-
-/*
- *  slic_link_config
- *
- *  Write phy control to configure link duplex/speed
- *
- */
-static void slic_link_config(struct adapter *adapter,
-		      u32 linkspeed, u32 linkduplex)
-{
-	u32 speed;
-	u32 duplex;
-	u32 phy_config;
-	u32 phy_advreg;
-	u32 phy_gctlreg;
-
-	if (adapter->state != ADAPT_UP)
-		return;
-
-	if (linkspeed > LINK_1000MB)
-		linkspeed = LINK_AUTOSPEED;
-	if (linkduplex > LINK_AUTOD)
-		linkduplex = LINK_AUTOD;
-
-	if ((linkspeed == LINK_AUTOSPEED) || (linkspeed == LINK_1000MB)) {
-		if (adapter->flags & ADAPT_FLAGS_FIBERMEDIA) {
-			/*
-			 * We've got a fiber gigabit interface, and register
-			 *  4 is different in fiber mode than in copper mode
-			 */
-
-			/* advertise FD only @1000 Mb */
-			phy_advreg = (MIICR_REG_4 | (PAR_ADV1000XFD));
-			/* enable PAUSE frames        */
-			phy_advreg |= PAR_ASYMPAUSE_FIBER;
-			slic_write32(adapter, SLIC_REG_WPHY, phy_advreg);
-
-			if (linkspeed == LINK_AUTOSPEED) {
-				/* reset phy, enable auto-neg  */
-				phy_config =
-				    (MIICR_REG_PCR |
-				     (PCR_RESET | PCR_AUTONEG |
-				      PCR_AUTONEG_RST));
-				slic_write32(adapter, SLIC_REG_WPHY,
-					     phy_config);
-			} else {	/* forced 1000 Mb FD*/
-				/*
-				 * power down phy to break link
-				 * this may not work)
-				 */
-				phy_config = (MIICR_REG_PCR | PCR_POWERDOWN);
-				slic_write32(adapter, SLIC_REG_WPHY,
-					     phy_config);
-				slic_flush_write(adapter);
-				/*
-				 * wait, Marvell says 1 sec,
-				 * try to get away with 10 ms
-				 */
-				mdelay(10);
-
-				/*
-				 * disable auto-neg, set speed/duplex,
-				 * soft reset phy, powerup
-				 */
-				phy_config =
-				    (MIICR_REG_PCR |
-				     (PCR_RESET | PCR_SPEED_1000 |
-				      PCR_DUPLEX_FULL));
-				slic_write32(adapter, SLIC_REG_WPHY,
-					     phy_config);
-			}
-		} else {	/* copper gigabit */
-
-			/*
-			 * Auto-Negotiate or 1000 Mb must be auto negotiated
-			 * We've got a copper gigabit interface, and
-			 * register 4 is different in copper mode than
-			 * in fiber mode
-			 */
-			if (linkspeed == LINK_AUTOSPEED) {
-				/* advertise 10/100 Mb modes   */
-				phy_advreg =
-				    (MIICR_REG_4 |
-				     (PAR_ADV100FD | PAR_ADV100HD | PAR_ADV10FD
-				      | PAR_ADV10HD));
-			} else {
-			/*
-			 * linkspeed == LINK_1000MB -
-			 * don't advertise 10/100 Mb modes
-			 */
-				phy_advreg = MIICR_REG_4;
-			}
-			/* enable PAUSE frames  */
-			phy_advreg |= PAR_ASYMPAUSE;
-			/* required by the Cicada PHY  */
-			phy_advreg |= PAR_802_3;
-			slic_write32(adapter, SLIC_REG_WPHY, phy_advreg);
-			/* advertise FD only @1000 Mb  */
-			phy_gctlreg = (MIICR_REG_9 | (PGC_ADV1000FD));
-			slic_write32(adapter, SLIC_REG_WPHY, phy_gctlreg);
-
-			if (adapter->subsysid != SLIC_1GB_CICADA_SUBSYS_ID) {
-				/*
-				 * if a Marvell PHY
-				 * enable auto crossover
-				 */
-				phy_config =
-				    (MIICR_REG_16 | (MRV_REG16_XOVERON));
-				slic_write32(adapter, SLIC_REG_WPHY,
-					     phy_config);
-
-				/* reset phy, enable auto-neg  */
-				phy_config =
-				    (MIICR_REG_PCR |
-				     (PCR_RESET | PCR_AUTONEG |
-				      PCR_AUTONEG_RST));
-				slic_write32(adapter, SLIC_REG_WPHY,
-					     phy_config);
-			} else {	/* it's a Cicada PHY  */
-				/* enable and restart auto-neg (don't reset)  */
-				phy_config =
-				    (MIICR_REG_PCR |
-				     (PCR_AUTONEG | PCR_AUTONEG_RST));
-				slic_write32(adapter, SLIC_REG_WPHY,
-					     phy_config);
-			}
-		}
-	} else {
-		/* Forced 10/100  */
-		if (linkspeed == LINK_10MB)
-			speed = 0;
-		else
-			speed = PCR_SPEED_100;
-		if (linkduplex == LINK_HALFD)
-			duplex = 0;
-		else
-			duplex = PCR_DUPLEX_FULL;
-
-		if (adapter->subsysid != SLIC_1GB_CICADA_SUBSYS_ID) {
-			/*
-			 * if a Marvell PHY
-			 * disable auto crossover
-			 */
-			phy_config = (MIICR_REG_16 | (MRV_REG16_XOVEROFF));
-			slic_write32(adapter, SLIC_REG_WPHY, phy_config);
-		}
-
-		/* power down phy to break link (this may not work)  */
-		phy_config = (MIICR_REG_PCR | (PCR_POWERDOWN | speed | duplex));
-		slic_write32(adapter, SLIC_REG_WPHY, phy_config);
-		slic_flush_write(adapter);
-		/* wait, Marvell says 1 sec, try to get away with 10 ms */
-		mdelay(10);
-
-		if (adapter->subsysid != SLIC_1GB_CICADA_SUBSYS_ID) {
-			/*
-			 * if a Marvell PHY
-			 * disable auto-neg, set speed,
-			 * soft reset phy, powerup
-			 */
-			phy_config =
-			    (MIICR_REG_PCR | (PCR_RESET | speed | duplex));
-			slic_write32(adapter, SLIC_REG_WPHY, phy_config);
-		} else {	/* it's a Cicada PHY  */
-			/* disable auto-neg, set speed, powerup  */
-			phy_config = (MIICR_REG_PCR | (speed | duplex));
-			slic_write32(adapter, SLIC_REG_WPHY, phy_config);
-		}
-	}
-}
-
-static int slic_card_download_gbrcv(struct adapter *adapter)
-{
-	const struct firmware *fw;
-	const char *file = "";
-	int ret;
-	u32 codeaddr;
-	u32 instruction;
-	int index = 0;
-	u32 rcvucodelen = 0;
-
-	switch (adapter->devid) {
-	case SLIC_2GB_DEVICE_ID:
-		file = "slicoss/oasisrcvucode.sys";
-		break;
-	case SLIC_1GB_DEVICE_ID:
-		file = "slicoss/gbrcvucode.sys";
-		break;
-	default:
-		return -ENOENT;
-	}
-
-	ret = request_firmware(&fw, file, &adapter->pcidev->dev);
-	if (ret) {
-		dev_err(&adapter->pcidev->dev,
-			"Failed to load firmware %s\n", file);
-		return ret;
-	}
-
-	rcvucodelen = *(u32 *)(fw->data + index);
-	index += 4;
-	switch (adapter->devid) {
-	case SLIC_2GB_DEVICE_ID:
-		if (rcvucodelen != OasisRcvUCodeLen) {
-			release_firmware(fw);
-			return -EINVAL;
-		}
-		break;
-	case SLIC_1GB_DEVICE_ID:
-		if (rcvucodelen != GBRcvUCodeLen) {
-			release_firmware(fw);
-			return -EINVAL;
-		}
-		break;
-	}
-	/* start download */
-	slic_write32(adapter, SLIC_REG_RCV_WCS, SLIC_RCVWCS_BEGIN);
-	/* download the rcv sequencer ucode */
-	for (codeaddr = 0; codeaddr < rcvucodelen; codeaddr++) {
-		/* write out instruction address */
-		slic_write32(adapter, SLIC_REG_RCV_WCS, codeaddr);
-
-		instruction = *(u32 *)(fw->data + index);
-		index += 4;
-		/* write out the instruction data low addr */
-		slic_write32(adapter, SLIC_REG_RCV_WCS, instruction);
-
-		instruction = *(u8 *)(fw->data + index);
-		index++;
-		/* write out the instruction data high addr */
-		slic_write32(adapter, SLIC_REG_RCV_WCS, instruction);
-	}
-
-	/* download finished */
-	release_firmware(fw);
-	slic_write32(adapter, SLIC_REG_RCV_WCS, SLIC_RCVWCS_FINISH);
-	slic_flush_write(adapter);
-
-	return 0;
-}
-
-MODULE_FIRMWARE("slicoss/oasisrcvucode.sys");
-MODULE_FIRMWARE("slicoss/gbrcvucode.sys");
-
-static int slic_card_download(struct adapter *adapter)
-{
-	const struct firmware *fw;
-	const char *file = "";
-	int ret;
-	u32 section;
-	int thissectionsize;
-	int codeaddr;
-	u32 instruction;
-	u32 baseaddress;
-	u32 i;
-	u32 numsects = 0;
-	u32 sectsize[3];
-	u32 sectstart[3];
-	int ucode_start, index = 0;
-
-	switch (adapter->devid) {
-	case SLIC_2GB_DEVICE_ID:
-		file = "slicoss/oasisdownload.sys";
-		break;
-	case SLIC_1GB_DEVICE_ID:
-		file = "slicoss/gbdownload.sys";
-		break;
-	default:
-		return -ENOENT;
-	}
-	ret = request_firmware(&fw, file, &adapter->pcidev->dev);
-	if (ret) {
-		dev_err(&adapter->pcidev->dev,
-			"Failed to load firmware %s\n", file);
-		return ret;
-	}
-	numsects = *(u32 *)(fw->data + index);
-	index += 4;
-	for (i = 0; i < numsects; i++) {
-		sectsize[i] = *(u32 *)(fw->data + index);
-		index += 4;
-	}
-	for (i = 0; i < numsects; i++) {
-		sectstart[i] = *(u32 *)(fw->data + index);
-		index += 4;
-	}
-	ucode_start = index;
-	instruction = *(u32 *)(fw->data + index);
-	index += 4;
-	for (section = 0; section < numsects; section++) {
-		baseaddress = sectstart[section];
-		thissectionsize = sectsize[section] >> 3;
-
-		for (codeaddr = 0; codeaddr < thissectionsize; codeaddr++) {
-			/* Write out instruction address */
-			slic_write32(adapter, SLIC_REG_WCS,
-				     baseaddress + codeaddr);
-			/* Write out instruction to low addr */
-			slic_write32(adapter, SLIC_REG_WCS,
-				     instruction);
-			instruction = *(u32 *)(fw->data + index);
-			index += 4;
-
-			/* Write out instruction to high addr */
-			slic_write32(adapter, SLIC_REG_WCS,
-				     instruction);
-			instruction = *(u32 *)(fw->data + index);
-			index += 4;
-		}
-	}
-	index = ucode_start;
-	for (section = 0; section < numsects; section++) {
-		instruction = *(u32 *)(fw->data + index);
-		baseaddress = sectstart[section];
-		if (baseaddress < 0x8000)
-			continue;
-		thissectionsize = sectsize[section] >> 3;
-
-		for (codeaddr = 0; codeaddr < thissectionsize; codeaddr++) {
-			/* Write out instruction address */
-			slic_write32(adapter, SLIC_REG_WCS,
-				     SLIC_WCS_COMPARE | (baseaddress +
-							 codeaddr));
-			/* Write out instruction to low addr */
-			slic_write32(adapter, SLIC_REG_WCS, instruction);
-			instruction = *(u32 *)(fw->data + index);
-			index += 4;
-			/* Write out instruction to high addr */
-			slic_write32(adapter, SLIC_REG_WCS, instruction);
-			instruction = *(u32 *)(fw->data + index);
-			index += 4;
-		}
-	}
-	release_firmware(fw);
-	/* Everything OK, kick off the card */
-	mdelay(10);
-
-	slic_write32(adapter, SLIC_REG_WCS, SLIC_WCS_START);
-	slic_flush_write(adapter);
-	/*
-	 * stall for 20 ms, long enough for ucode to init card
-	 * and reach mainloop
-	 */
-	mdelay(20);
-
-	return 0;
-}
-
-MODULE_FIRMWARE("slicoss/oasisdownload.sys");
-MODULE_FIRMWARE("slicoss/gbdownload.sys");
-
-static void slic_adapter_set_hwaddr(struct adapter *adapter)
-{
-	struct sliccard *card = adapter->card;
-
-	if ((adapter->card) && (card->config_set)) {
-		memcpy(adapter->macaddr,
-		       card->config.MacInfo[adapter->functionnumber].macaddrA,
-		       sizeof(struct slic_config_mac));
-		if (is_zero_ether_addr(adapter->currmacaddr))
-			memcpy(adapter->currmacaddr, adapter->macaddr,
-			       ETH_ALEN);
-		if (adapter->netdev)
-			memcpy(adapter->netdev->dev_addr, adapter->currmacaddr,
-			       ETH_ALEN);
-	}
-}
-
-static void slic_intagg_set(struct adapter *adapter, u32 value)
-{
-	slic_write32(adapter, SLIC_REG_INTAGG, value);
-	adapter->card->loadlevel_current = value;
-}
-
-static void slic_soft_reset(struct adapter *adapter)
-{
-	if (adapter->card->state == CARD_UP) {
-		slic_write32(adapter, SLIC_REG_QUIESCE, 0);
-		slic_flush_write(adapter);
-		mdelay(1);
-	}
-
-	slic_write32(adapter, SLIC_REG_RESET, SLIC_RESET_MAGIC);
-	slic_flush_write(adapter);
-
-	mdelay(1);
-}
-
-static void slic_mac_address_config(struct adapter *adapter)
-{
-	u32 value;
-	u32 value2;
-
-	value = ntohl(*(__be32 *)&adapter->currmacaddr[2]);
-	slic_write32(adapter, SLIC_REG_WRADDRAL, value);
-	slic_write32(adapter, SLIC_REG_WRADDRBL, value);
-
-	value2 = (u32)((adapter->currmacaddr[0] << 8 |
-			     adapter->currmacaddr[1]) & 0xFFFF);
-
-	slic_write32(adapter, SLIC_REG_WRADDRAH, value2);
-	slic_write32(adapter, SLIC_REG_WRADDRBH, value2);
-
-	/*
-	 * Write our multicast mask out to the card.  This is done
-	 * here in addition to the slic_mcast_addr_set routine
-	 * because ALL_MCAST may have been enabled or disabled
-	 */
-	slic_mcast_set_mask(adapter);
-}
-
-static void slic_mac_config(struct adapter *adapter)
-{
-	u32 value;
-
-	/* Setup GMAC gaps */
-	if (adapter->linkspeed == LINK_1000MB) {
-		value = ((GMCR_GAPBB_1000 << GMCR_GAPBB_SHIFT) |
-			 (GMCR_GAPR1_1000 << GMCR_GAPR1_SHIFT) |
-			 (GMCR_GAPR2_1000 << GMCR_GAPR2_SHIFT));
-	} else {
-		value = ((GMCR_GAPBB_100 << GMCR_GAPBB_SHIFT) |
-			 (GMCR_GAPR1_100 << GMCR_GAPR1_SHIFT) |
-			 (GMCR_GAPR2_100 << GMCR_GAPR2_SHIFT));
-	}
-
-	/* enable GMII */
-	if (adapter->linkspeed == LINK_1000MB)
-		value |= GMCR_GBIT;
-
-	/* enable fullduplex */
-	if ((adapter->linkduplex == LINK_FULLD)
-	    || (adapter->macopts & MAC_LOOPBACK)) {
-		value |= GMCR_FULLD;
-	}
-
-	/* write mac config */
-	slic_write32(adapter, SLIC_REG_WMCFG, value);
-
-	/* setup mac addresses */
-	slic_mac_address_config(adapter);
-}
-
-static void slic_config_set(struct adapter *adapter, bool linkchange)
-{
-	u32 value;
-	u32 RcrReset;
-
-	if (linkchange) {
-		/* Setup MAC */
-		slic_mac_config(adapter);
-		RcrReset = GRCR_RESET;
-	} else {
-		slic_mac_address_config(adapter);
-		RcrReset = 0;
-	}
-
-	if (adapter->linkduplex == LINK_FULLD) {
-		/* setup xmtcfg */
-		value = (GXCR_RESET |	/* Always reset     */
-			 GXCR_XMTEN |	/* Enable transmit  */
-			 GXCR_PAUSEEN);	/* Enable pause     */
-
-		slic_write32(adapter, SLIC_REG_WXCFG, value);
-
-		/* Setup rcvcfg last */
-		value = (RcrReset |	/* Reset, if linkchange */
-			 GRCR_CTLEN |	/* Enable CTL frames    */
-			 GRCR_ADDRAEN |	/* Address A enable     */
-			 GRCR_RCVBAD |	/* Rcv bad frames       */
-			 (GRCR_HASHSIZE << GRCR_HASHSIZE_SHIFT));
-	} else {
-		/* setup xmtcfg */
-		value = (GXCR_RESET |	/* Always reset     */
-			 GXCR_XMTEN);	/* Enable transmit  */
-
-		slic_write32(adapter, SLIC_REG_WXCFG, value);
-
-		/* Setup rcvcfg last */
-		value = (RcrReset |	/* Reset, if linkchange */
-			 GRCR_ADDRAEN |	/* Address A enable     */
-			 GRCR_RCVBAD |	/* Rcv bad frames       */
-			 (GRCR_HASHSIZE << GRCR_HASHSIZE_SHIFT));
-	}
-
-	if (adapter->state != ADAPT_DOWN) {
-		/* Only enable receive if we are restarting or running */
-		value |= GRCR_RCVEN;
-	}
-
-	if (adapter->macopts & MAC_PROMISC)
-		value |= GRCR_RCVALL;
-
-	slic_write32(adapter, SLIC_REG_WRCFG, value);
-}
-
-/*
- *  Turn off RCV and XMT, power down PHY
- */
-static void slic_config_clear(struct adapter *adapter)
-{
-	u32 value;
-	u32 phy_config;
-
-	/* Setup xmtcfg */
-	value = (GXCR_RESET |	/* Always reset */
-		 GXCR_PAUSEEN);	/* Enable pause */
-
-	slic_write32(adapter, SLIC_REG_WXCFG, value);
-
-	value = (GRCR_RESET |	/* Always reset      */
-		 GRCR_CTLEN |	/* Enable CTL frames */
-		 GRCR_ADDRAEN |	/* Address A enable  */
-		 (GRCR_HASHSIZE << GRCR_HASHSIZE_SHIFT));
-
-	slic_write32(adapter, SLIC_REG_WRCFG, value);
-
-	/* power down phy */
-	phy_config = (MIICR_REG_PCR | (PCR_POWERDOWN));
-	slic_write32(adapter, SLIC_REG_WPHY, phy_config);
-}
-
-static bool slic_mac_filter(struct adapter *adapter,
-			struct ether_header *ether_frame)
-{
-	struct net_device *netdev = adapter->netdev;
-	u32 opts = adapter->macopts;
-
-	if (opts & MAC_PROMISC)
-		return true;
-
-	if (is_broadcast_ether_addr(ether_frame->ether_dhost)) {
-		if (opts & MAC_BCAST) {
-			adapter->rcv_broadcasts++;
-			return true;
-		}
-
-		return false;
-	}
-
-	if (is_multicast_ether_addr(ether_frame->ether_dhost)) {
-		if (opts & MAC_ALLMCAST) {
-			adapter->rcv_multicasts++;
-			netdev->stats.multicast++;
-			return true;
-		}
-		if (opts & MAC_MCAST) {
-			struct mcast_address *mcaddr = adapter->mcastaddrs;
-
-			while (mcaddr) {
-				if (ether_addr_equal(mcaddr->address,
-						     ether_frame->ether_dhost)) {
-					adapter->rcv_multicasts++;
-					netdev->stats.multicast++;
-					return true;
-				}
-				mcaddr = mcaddr->next;
-			}
-
-			return false;
-		}
-
-		return false;
-	}
-	if (opts & MAC_DIRECTED) {
-		adapter->rcv_unicasts++;
-		return true;
-	}
-	return false;
-}
-
-static int slic_mac_set_address(struct net_device *dev, void *ptr)
-{
-	struct adapter *adapter = netdev_priv(dev);
-	struct sockaddr *addr = ptr;
-
-	if (netif_running(dev))
-		return -EBUSY;
-	if (!adapter)
-		return -EBUSY;
-
-	if (!is_valid_ether_addr(addr->sa_data))
-		return -EINVAL;
-
-	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
-	memcpy(adapter->currmacaddr, addr->sa_data, dev->addr_len);
-
-	slic_config_set(adapter, true);
-	return 0;
-}
-
-static void slic_timer_load_check(ulong cardaddr)
-{
-	struct sliccard *card = (struct sliccard *)cardaddr;
-	struct adapter *adapter = card->master;
-	u32 load = card->events;
-	u32 level = 0;
-
-	if ((adapter) && (adapter->state == ADAPT_UP) &&
-	    (card->state == CARD_UP) && (slic_global.dynamic_intagg)) {
-		if (adapter->devid == SLIC_1GB_DEVICE_ID) {
-			if (adapter->linkspeed == LINK_1000MB)
-				level = 100;
-			else {
-				if (load > SLIC_LOAD_5)
-					level = SLIC_INTAGG_5;
-				else if (load > SLIC_LOAD_4)
-					level = SLIC_INTAGG_4;
-				else if (load > SLIC_LOAD_3)
-					level = SLIC_INTAGG_3;
-				else if (load > SLIC_LOAD_2)
-					level = SLIC_INTAGG_2;
-				else if (load > SLIC_LOAD_1)
-					level = SLIC_INTAGG_1;
-				else
-					level = SLIC_INTAGG_0;
-			}
-			if (card->loadlevel_current != level) {
-				card->loadlevel_current = level;
-				slic_write32(adapter, SLIC_REG_INTAGG, level);
-			}
-		} else {
-			if (load > SLIC_LOAD_5)
-				level = SLIC_INTAGG_5;
-			else if (load > SLIC_LOAD_4)
-				level = SLIC_INTAGG_4;
-			else if (load > SLIC_LOAD_3)
-				level = SLIC_INTAGG_3;
-			else if (load > SLIC_LOAD_2)
-				level = SLIC_INTAGG_2;
-			else if (load > SLIC_LOAD_1)
-				level = SLIC_INTAGG_1;
-			else
-				level = SLIC_INTAGG_0;
-			if (card->loadlevel_current != level) {
-				card->loadlevel_current = level;
-				slic_write32(adapter, SLIC_REG_INTAGG, level);
-			}
-		}
-	}
-	card->events = 0;
-	card->loadtimer.expires = jiffies + (SLIC_LOADTIMER_PERIOD * HZ);
-	add_timer(&card->loadtimer);
-}
-
-static int slic_upr_queue_request(struct adapter *adapter,
-			   u32 upr_request,
-			   u32 upr_data,
-			   u32 upr_data_h,
-			   u32 upr_buffer, u32 upr_buffer_h)
-{
-	struct slic_upr *upr;
-	struct slic_upr *uprqueue;
-
-	upr = kmalloc(sizeof(*upr), GFP_ATOMIC);
-	if (!upr)
-		return -ENOMEM;
-
-	upr->adapter = adapter->port;
-	upr->upr_request = upr_request;
-	upr->upr_data = upr_data;
-	upr->upr_buffer = upr_buffer;
-	upr->upr_data_h = upr_data_h;
-	upr->upr_buffer_h = upr_buffer_h;
-	upr->next = NULL;
-	if (adapter->upr_list) {
-		uprqueue = adapter->upr_list;
-
-		while (uprqueue->next)
-			uprqueue = uprqueue->next;
-		uprqueue->next = upr;
-	} else {
-		adapter->upr_list = upr;
-	}
-	return 0;
-}
-
-static void slic_upr_start(struct adapter *adapter)
-{
-	struct slic_upr *upr;
-
-	upr = adapter->upr_list;
-	if (!upr)
-		return;
-	if (adapter->upr_busy)
-		return;
-	adapter->upr_busy = 1;
-
-	switch (upr->upr_request) {
-	case SLIC_UPR_STATS:
-		if (upr->upr_data_h == 0) {
-			slic_write32(adapter, SLIC_REG_RSTAT, upr->upr_data);
-		} else {
-			slic_write64(adapter, SLIC_REG_RSTAT64, upr->upr_data,
-				     upr->upr_data_h);
-		}
-		break;
-
-	case SLIC_UPR_RLSR:
-		slic_write64(adapter, SLIC_REG_LSTAT, upr->upr_data,
-			     upr->upr_data_h);
-		break;
-
-	case SLIC_UPR_RCONFIG:
-		slic_write64(adapter, SLIC_REG_RCONFIG, upr->upr_data,
-			     upr->upr_data_h);
-		break;
-	case SLIC_UPR_PING:
-		slic_write32(adapter, SLIC_REG_PING, 1);
-		break;
-	}
-	slic_flush_write(adapter);
-}
-
-static int slic_upr_request(struct adapter *adapter,
-		     u32 upr_request,
-		     u32 upr_data,
-		     u32 upr_data_h,
-		     u32 upr_buffer, u32 upr_buffer_h)
-{
-	unsigned long flags;
-	int rc;
-
-	spin_lock_irqsave(&adapter->upr_lock, flags);
-	rc = slic_upr_queue_request(adapter,
-					upr_request,
-					upr_data,
-					upr_data_h, upr_buffer, upr_buffer_h);
-	if (rc)
-		goto err_unlock_irq;
-
-	slic_upr_start(adapter);
-err_unlock_irq:
-	spin_unlock_irqrestore(&adapter->upr_lock, flags);
-	return rc;
-}
-
-static void slic_link_upr_complete(struct adapter *adapter, u32 isr)
-{
-	struct slic_shmemory *sm = &adapter->shmem;
-	struct slic_shmem_data *sm_data = sm->shmem_data;
-	u32 lst = sm_data->lnkstatus;
-	uint linkup;
-	unsigned char linkspeed;
-	unsigned char linkduplex;
-
-	if ((isr & ISR_UPCERR) || (isr & ISR_UPCBSY)) {
-		dma_addr_t phaddr = sm->lnkstatus_phaddr;
-
-		slic_upr_queue_request(adapter, SLIC_UPR_RLSR,
-				       cpu_to_le32(lower_32_bits(phaddr)),
-				       cpu_to_le32(upper_32_bits(phaddr)),
-				       0, 0);
-		return;
-	}
-	if (adapter->state != ADAPT_UP)
-		return;
-
-	linkup = lst & GIG_LINKUP ? LINK_UP : LINK_DOWN;
-	if (lst & GIG_SPEED_1000)
-		linkspeed = LINK_1000MB;
-	else if (lst & GIG_SPEED_100)
-		linkspeed = LINK_100MB;
-	else
-		linkspeed = LINK_10MB;
-
-	if (lst & GIG_FULLDUPLEX)
-		linkduplex = LINK_FULLD;
-	else
-		linkduplex = LINK_HALFD;
-
-	if ((adapter->linkstate == LINK_DOWN) && (linkup == LINK_DOWN))
-		return;
-
-	/* link up event, but nothing has changed */
-	if ((adapter->linkstate == LINK_UP) &&
-	    (linkup == LINK_UP) &&
-	    (adapter->linkspeed == linkspeed) &&
-	    (adapter->linkduplex == linkduplex))
-		return;
-
-	/* link has changed at this point */
-
-	/* link has gone from up to down */
-	if (linkup == LINK_DOWN) {
-		adapter->linkstate = LINK_DOWN;
-		netif_carrier_off(adapter->netdev);
-		return;
-	}
-
-	/* link has gone from down to up */
-	adapter->linkspeed = linkspeed;
-	adapter->linkduplex = linkduplex;
-
-	if (adapter->linkstate != LINK_UP) {
-		/* setup the mac */
-		slic_config_set(adapter, true);
-		adapter->linkstate = LINK_UP;
-		netif_carrier_on(adapter->netdev);
-	}
-}
-
-static void slic_upr_request_complete(struct adapter *adapter, u32 isr)
-{
-	struct sliccard *card = adapter->card;
-	struct slic_upr *upr;
-	unsigned long flags;
-
-	spin_lock_irqsave(&adapter->upr_lock, flags);
-	upr = adapter->upr_list;
-	if (!upr) {
-		spin_unlock_irqrestore(&adapter->upr_lock, flags);
-		return;
-	}
-	adapter->upr_list = upr->next;
-	upr->next = NULL;
-	adapter->upr_busy = 0;
-	switch (upr->upr_request) {
-	case SLIC_UPR_STATS: {
-		struct slic_shmemory *sm = &adapter->shmem;
-		struct slic_shmem_data *sm_data = sm->shmem_data;
-		struct slic_stats *stats = &sm_data->stats;
-		struct slic_stats *old = &adapter->inicstats_prev;
-		struct slicnet_stats *stst = &adapter->slic_stats;
-
-		if (isr & ISR_UPCERR) {
-			dev_err(&adapter->netdev->dev,
-				"SLIC_UPR_STATS command failed isr[%x]\n", isr);
-			break;
-		}
-
-		UPDATE_STATS_GB(stst->tcp.xmit_tcp_segs, stats->xmit_tcp_segs,
-				old->xmit_tcp_segs);
-
-		UPDATE_STATS_GB(stst->tcp.xmit_tcp_bytes, stats->xmit_tcp_bytes,
-				old->xmit_tcp_bytes);
-
-		UPDATE_STATS_GB(stst->tcp.rcv_tcp_segs, stats->rcv_tcp_segs,
-				old->rcv_tcp_segs);
-
-		UPDATE_STATS_GB(stst->tcp.rcv_tcp_bytes, stats->rcv_tcp_bytes,
-				old->rcv_tcp_bytes);
-
-		UPDATE_STATS_GB(stst->iface.xmt_bytes, stats->xmit_bytes,
-				old->xmit_bytes);
-
-		UPDATE_STATS_GB(stst->iface.xmt_ucast, stats->xmit_unicasts,
-				old->xmit_unicasts);
-
-		UPDATE_STATS_GB(stst->iface.rcv_bytes, stats->rcv_bytes,
-				old->rcv_bytes);
-
-		UPDATE_STATS_GB(stst->iface.rcv_ucast, stats->rcv_unicasts,
-				old->rcv_unicasts);
-
-		UPDATE_STATS_GB(stst->iface.xmt_errors, stats->xmit_collisions,
-				old->xmit_collisions);
-
-		UPDATE_STATS_GB(stst->iface.xmt_errors,
-				stats->xmit_excess_collisions,
-				old->xmit_excess_collisions);
-
-		UPDATE_STATS_GB(stst->iface.xmt_errors, stats->xmit_other_error,
-				old->xmit_other_error);
-
-		UPDATE_STATS_GB(stst->iface.rcv_errors, stats->rcv_other_error,
-				old->rcv_other_error);
-
-		UPDATE_STATS_GB(stst->iface.rcv_discards, stats->rcv_drops,
-				old->rcv_drops);
-
-		if (stats->rcv_drops > old->rcv_drops)
-			adapter->rcv_drops += (stats->rcv_drops -
-					       old->rcv_drops);
-		memcpy_fromio(old, stats, sizeof(*stats));
-		break;
-	}
-	case SLIC_UPR_RLSR:
-		slic_link_upr_complete(adapter, isr);
-		break;
-	case SLIC_UPR_RCONFIG:
-		break;
-	case SLIC_UPR_PING:
-		card->pingstatus |= (isr & ISR_PINGDSMASK);
-		break;
-	}
-	kfree(upr);
-	slic_upr_start(adapter);
-	spin_unlock_irqrestore(&adapter->upr_lock, flags);
-}
-
-static int slic_config_get(struct adapter *adapter, u32 config, u32 config_h)
-{
-	return slic_upr_request(adapter, SLIC_UPR_RCONFIG, config, config_h,
-				0, 0);
-}
-
-/*
- * Compute a checksum of the EEPROM according to RFC 1071.
- */
-static u16 slic_eeprom_cksum(void *eeprom, unsigned int len)
-{
-	u16 *wp = eeprom;
-	u32 checksum = 0;
-
-	while (len > 1) {
-		checksum += *(wp++);
-		len -= 2;
-	}
-
-	if (len > 0)
-		checksum += *(u8 *)wp;
-
-	while (checksum >> 16)
-		checksum = (checksum & 0xFFFF) + ((checksum >> 16) & 0xFFFF);
-
-	return ~checksum;
-}
-
-static void slic_rspqueue_free(struct adapter *adapter)
-{
-	int i;
-	struct slic_rspqueue *rspq = &adapter->rspqueue;
-
-	for (i = 0; i < rspq->num_pages; i++) {
-		if (rspq->vaddr[i]) {
-			pci_free_consistent(adapter->pcidev, PAGE_SIZE,
-					    rspq->vaddr[i], rspq->paddr[i]);
-		}
-		rspq->vaddr[i] = NULL;
-		rspq->paddr[i] = 0;
-	}
-	rspq->offset = 0;
-	rspq->pageindex = 0;
-	rspq->rspbuf = NULL;
-}
-
-static int slic_rspqueue_init(struct adapter *adapter)
-{
-	int i;
-	struct slic_rspqueue *rspq = &adapter->rspqueue;
-	u32 paddrh = 0;
-
-	memset(rspq, 0, sizeof(struct slic_rspqueue));
-
-	rspq->num_pages = SLIC_RSPQ_PAGES_GB;
-
-	for (i = 0; i < rspq->num_pages; i++) {
-		rspq->vaddr[i] = pci_zalloc_consistent(adapter->pcidev,
-						       PAGE_SIZE,
-						       &rspq->paddr[i]);
-		if (!rspq->vaddr[i]) {
-			dev_err(&adapter->pcidev->dev,
-				"pci_alloc_consistent failed\n");
-			slic_rspqueue_free(adapter);
-			return -ENOMEM;
-		}
-
-		if (paddrh == 0) {
-			slic_write32(adapter, SLIC_REG_RBAR,
-				     rspq->paddr[i] | SLIC_RSPQ_BUFSINPAGE);
-		} else {
-			slic_write64(adapter, SLIC_REG_RBAR64,
-				     rspq->paddr[i] | SLIC_RSPQ_BUFSINPAGE,
-				     paddrh);
-		}
-	}
-	rspq->offset = 0;
-	rspq->pageindex = 0;
-	rspq->rspbuf = (struct slic_rspbuf *)rspq->vaddr[0];
-	return 0;
-}
-
-static struct slic_rspbuf *slic_rspqueue_getnext(struct adapter *adapter)
-{
-	struct slic_rspqueue *rspq = &adapter->rspqueue;
-	struct slic_rspbuf *buf;
-
-	if (!(rspq->rspbuf->status))
-		return NULL;
-
-	buf = rspq->rspbuf;
-	if (++rspq->offset < SLIC_RSPQ_BUFSINPAGE) {
-		rspq->rspbuf++;
-	} else {
-		slic_write64(adapter, SLIC_REG_RBAR64,
-			     rspq->paddr[rspq->pageindex] |
-			     SLIC_RSPQ_BUFSINPAGE, 0);
-		rspq->pageindex = (rspq->pageindex + 1) % rspq->num_pages;
-		rspq->offset = 0;
-		rspq->rspbuf = (struct slic_rspbuf *)
-						rspq->vaddr[rspq->pageindex];
-	}
-
-	return buf;
-}
-
-static void slic_cmdqmem_free(struct adapter *adapter)
-{
-	struct slic_cmdqmem *cmdqmem = &adapter->cmdqmem;
-	int i;
-
-	for (i = 0; i < SLIC_CMDQ_MAXPAGES; i++) {
-		if (cmdqmem->pages[i]) {
-			pci_free_consistent(adapter->pcidev,
-					    PAGE_SIZE,
-					    (void *)cmdqmem->pages[i],
-					    cmdqmem->dma_pages[i]);
-		}
-	}
-	memset(cmdqmem, 0, sizeof(struct slic_cmdqmem));
-}
-
-static u32 *slic_cmdqmem_addpage(struct adapter *adapter)
-{
-	struct slic_cmdqmem *cmdqmem = &adapter->cmdqmem;
-	u32 *pageaddr;
-
-	if (cmdqmem->pagecnt >= SLIC_CMDQ_MAXPAGES)
-		return NULL;
-	pageaddr = pci_alloc_consistent(adapter->pcidev,
-					PAGE_SIZE,
-					&cmdqmem->dma_pages[cmdqmem->pagecnt]);
-	if (!pageaddr)
-		return NULL;
-
-	cmdqmem->pages[cmdqmem->pagecnt] = pageaddr;
-	cmdqmem->pagecnt++;
-	return pageaddr;
-}
-
-static void slic_cmdq_free(struct adapter *adapter)
-{
-	struct slic_hostcmd *cmd;
-
-	cmd = adapter->cmdq_all.head;
-	while (cmd) {
-		if (cmd->busy) {
-			struct sk_buff *tempskb;
-
-			tempskb = cmd->skb;
-			if (tempskb) {
-				cmd->skb = NULL;
-				dev_kfree_skb_irq(tempskb);
-			}
-		}
-		cmd = cmd->next_all;
-	}
-	memset(&adapter->cmdq_all, 0, sizeof(struct slic_cmdqueue));
-	memset(&adapter->cmdq_free, 0, sizeof(struct slic_cmdqueue));
-	memset(&adapter->cmdq_done, 0, sizeof(struct slic_cmdqueue));
-	slic_cmdqmem_free(adapter);
-}
-
-static void slic_cmdq_addcmdpage(struct adapter *adapter, u32 *page)
-{
-	struct slic_hostcmd *cmd;
-	struct slic_hostcmd *prev;
-	struct slic_hostcmd *tail;
-	struct slic_cmdqueue *cmdq;
-	int cmdcnt;
-	void *cmdaddr;
-	ulong phys_addr;
-	u32 phys_addrl;
-	u32 phys_addrh;
-	struct slic_handle *pslic_handle;
-	unsigned long flags;
-
-	cmdaddr = page;
-	cmd = cmdaddr;
-	cmdcnt = 0;
-
-	phys_addr = virt_to_bus((void *)page);
-	phys_addrl = SLIC_GET_ADDR_LOW(phys_addr);
-	phys_addrh = SLIC_GET_ADDR_HIGH(phys_addr);
-
-	prev = NULL;
-	tail = cmd;
-	while ((cmdcnt < SLIC_CMDQ_CMDSINPAGE) &&
-	       (adapter->slic_handle_ix < 256)) {
-		/* Allocate and initialize a SLIC_HANDLE for this command */
-		spin_lock_irqsave(&adapter->handle_lock, flags);
-		pslic_handle  =  adapter->pfree_slic_handles;
-		adapter->pfree_slic_handles = pslic_handle->next;
-		spin_unlock_irqrestore(&adapter->handle_lock, flags);
-		pslic_handle->type = SLIC_HANDLE_CMD;
-		pslic_handle->address = (void *)cmd;
-		pslic_handle->offset = (ushort)adapter->slic_handle_ix++;
-		pslic_handle->other_handle = NULL;
-		pslic_handle->next = NULL;
-
-		cmd->pslic_handle = pslic_handle;
-		cmd->cmd64.hosthandle = pslic_handle->token.handle_token;
-		cmd->busy = false;
-		cmd->paddrl = phys_addrl;
-		cmd->paddrh = phys_addrh;
-		cmd->next_all = prev;
-		cmd->next = prev;
-		prev = cmd;
-		phys_addrl += SLIC_HOSTCMD_SIZE;
-		cmdaddr += SLIC_HOSTCMD_SIZE;
-
-		cmd = cmdaddr;
-		cmdcnt++;
-	}
-
-	cmdq = &adapter->cmdq_all;
-	cmdq->count += cmdcnt;	/*  SLIC_CMDQ_CMDSINPAGE;   mooktodo */
-	tail->next_all = cmdq->head;
-	cmdq->head = prev;
-	cmdq = &adapter->cmdq_free;
-	spin_lock_irqsave(&cmdq->lock, flags);
-	cmdq->count += cmdcnt;	/*  SLIC_CMDQ_CMDSINPAGE;   mooktodo */
-	tail->next = cmdq->head;
-	cmdq->head = prev;
-	spin_unlock_irqrestore(&cmdq->lock, flags);
-}
-
-static int slic_cmdq_init(struct adapter *adapter)
-{
-	int i;
-	u32 *pageaddr;
-
-	memset(&adapter->cmdq_all, 0, sizeof(struct slic_cmdqueue));
-	memset(&adapter->cmdq_free, 0, sizeof(struct slic_cmdqueue));
-	memset(&adapter->cmdq_done, 0, sizeof(struct slic_cmdqueue));
-	spin_lock_init(&adapter->cmdq_all.lock);
-	spin_lock_init(&adapter->cmdq_free.lock);
-	spin_lock_init(&adapter->cmdq_done.lock);
-	memset(&adapter->cmdqmem, 0, sizeof(struct slic_cmdqmem));
-	adapter->slic_handle_ix = 1;
-	for (i = 0; i < SLIC_CMDQ_INITPAGES; i++) {
-		pageaddr = slic_cmdqmem_addpage(adapter);
-		if (!pageaddr) {
-			slic_cmdq_free(adapter);
-			return -ENOMEM;
-		}
-		slic_cmdq_addcmdpage(adapter, pageaddr);
-	}
-	adapter->slic_handle_ix = 1;
-
-	return 0;
-}
-
-static void slic_cmdq_reset(struct adapter *adapter)
-{
-	struct slic_hostcmd *hcmd;
-	struct sk_buff *skb;
-	u32 outstanding;
-	unsigned long flags;
-
-	spin_lock_irqsave(&adapter->cmdq_free.lock, flags);
-	spin_lock(&adapter->cmdq_done.lock);
-	outstanding = adapter->cmdq_all.count - adapter->cmdq_done.count;
-	outstanding -= adapter->cmdq_free.count;
-	hcmd = adapter->cmdq_all.head;
-	while (hcmd) {
-		if (hcmd->busy) {
-			skb = hcmd->skb;
-			hcmd->busy = 0;
-			hcmd->skb = NULL;
-			dev_kfree_skb_irq(skb);
-		}
-		hcmd = hcmd->next_all;
-	}
-	adapter->cmdq_free.count = 0;
-	adapter->cmdq_free.head = NULL;
-	adapter->cmdq_free.tail = NULL;
-	adapter->cmdq_done.count = 0;
-	adapter->cmdq_done.head = NULL;
-	adapter->cmdq_done.tail = NULL;
-	adapter->cmdq_free.head = adapter->cmdq_all.head;
-	hcmd = adapter->cmdq_all.head;
-	while (hcmd) {
-		adapter->cmdq_free.count++;
-		hcmd->next = hcmd->next_all;
-		hcmd = hcmd->next_all;
-	}
-	if (adapter->cmdq_free.count != adapter->cmdq_all.count) {
-		dev_err(&adapter->netdev->dev,
-			"free_count %d != all count %d\n",
-			adapter->cmdq_free.count, adapter->cmdq_all.count);
-	}
-	spin_unlock(&adapter->cmdq_done.lock);
-	spin_unlock_irqrestore(&adapter->cmdq_free.lock, flags);
-}
-
-static void slic_cmdq_getdone(struct adapter *adapter)
-{
-	struct slic_cmdqueue *done_cmdq = &adapter->cmdq_done;
-	struct slic_cmdqueue *free_cmdq = &adapter->cmdq_free;
-	unsigned long flags;
-
-	spin_lock_irqsave(&done_cmdq->lock, flags);
-
-	free_cmdq->head = done_cmdq->head;
-	free_cmdq->count = done_cmdq->count;
-	done_cmdq->head = NULL;
-	done_cmdq->tail = NULL;
-	done_cmdq->count = 0;
-	spin_unlock_irqrestore(&done_cmdq->lock, flags);
-}
-
-static struct slic_hostcmd *slic_cmdq_getfree(struct adapter *adapter)
-{
-	struct slic_cmdqueue *cmdq = &adapter->cmdq_free;
-	struct slic_hostcmd *cmd = NULL;
-	unsigned long flags;
-
-lock_and_retry:
-	spin_lock_irqsave(&cmdq->lock, flags);
-retry:
-	cmd = cmdq->head;
-	if (cmd) {
-		cmdq->head = cmd->next;
-		cmdq->count--;
-		spin_unlock_irqrestore(&cmdq->lock, flags);
-	} else {
-		slic_cmdq_getdone(adapter);
-		cmd = cmdq->head;
-		if (cmd) {
-			goto retry;
-		} else {
-			u32 *pageaddr;
-
-			spin_unlock_irqrestore(&cmdq->lock, flags);
-			pageaddr = slic_cmdqmem_addpage(adapter);
-			if (pageaddr) {
-				slic_cmdq_addcmdpage(adapter, pageaddr);
-				goto lock_and_retry;
-			}
-		}
-	}
-	return cmd;
-}
-
-static void slic_cmdq_putdone_irq(struct adapter *adapter,
-				struct slic_hostcmd *cmd)
-{
-	struct slic_cmdqueue *cmdq = &adapter->cmdq_done;
-
-	spin_lock(&cmdq->lock);
-	cmd->busy = 0;
-	cmd->next = cmdq->head;
-	cmdq->head = cmd;
-	cmdq->count++;
-	if ((adapter->xmitq_full) && (cmdq->count > 10))
-		netif_wake_queue(adapter->netdev);
-	spin_unlock(&cmdq->lock);
-}
-
-static int slic_rcvqueue_fill(struct adapter *adapter)
-{
-	void *paddr;
-	u32 paddrl;
-	u32 paddrh;
-	struct slic_rcvqueue *rcvq = &adapter->rcvqueue;
-	int i = 0;
-	struct device *dev = &adapter->netdev->dev;
-
-	while (i < SLIC_RCVQ_FILLENTRIES) {
-		struct slic_rcvbuf *rcvbuf;
-		struct sk_buff *skb;
-#ifdef KLUDGE_FOR_4GB_BOUNDARY
-retry_rcvqfill:
-#endif
-		skb = alloc_skb(SLIC_RCVQ_RCVBUFSIZE, GFP_ATOMIC);
-		if (skb) {
-			paddr = (void *)(unsigned long)
-				pci_map_single(adapter->pcidev,
-					       skb->data,
-					       SLIC_RCVQ_RCVBUFSIZE,
-					       PCI_DMA_FROMDEVICE);
-			paddrl = SLIC_GET_ADDR_LOW(paddr);
-			paddrh = SLIC_GET_ADDR_HIGH(paddr);
-
-			skb->len = SLIC_RCVBUF_HEADSIZE;
-			rcvbuf = (struct slic_rcvbuf *)skb->head;
-			rcvbuf->status = 0;
-			skb->next = NULL;
-#ifdef KLUDGE_FOR_4GB_BOUNDARY
-			if (paddrl == 0) {
-				dev_err(dev, "%s: LOW 32bits PHYSICAL ADDRESS == 0\n",
-					__func__);
-				dev_err(dev, "skb[%p] PROBLEM\n", skb);
-				dev_err(dev, "         skbdata[%p]\n",
-						skb->data);
-				dev_err(dev, "         skblen[%x]\n", skb->len);
-				dev_err(dev, "         paddr[%p]\n", paddr);
-				dev_err(dev, "         paddrl[%x]\n", paddrl);
-				dev_err(dev, "         paddrh[%x]\n", paddrh);
-				dev_err(dev, "         rcvq->head[%p]\n",
-						rcvq->head);
-				dev_err(dev, "         rcvq->tail[%p]\n",
-						rcvq->tail);
-				dev_err(dev, "         rcvq->count[%x]\n",
-						rcvq->count);
-				dev_err(dev, "SKIP THIS SKB!!!!!!!!\n");
-				goto retry_rcvqfill;
-			}
-#else
-			if (paddrl == 0) {
-				dev_err(dev, "%s: LOW 32bits PHYSICAL ADDRESS == 0\n",
-					__func__);
-				dev_err(dev, "skb[%p] PROBLEM\n", skb);
-				dev_err(dev, "         skbdata[%p]\n",
-						skb->data);
-				dev_err(dev, "         skblen[%x]\n", skb->len);
-				dev_err(dev, "         paddr[%p]\n", paddr);
-				dev_err(dev, "         paddrl[%x]\n", paddrl);
-				dev_err(dev, "         paddrh[%x]\n", paddrh);
-				dev_err(dev, "         rcvq->head[%p]\n",
-						rcvq->head);
-				dev_err(dev, "         rcvq->tail[%p]\n",
-						rcvq->tail);
-				dev_err(dev, "         rcvq->count[%x]\n",
-						rcvq->count);
-				dev_err(dev, "GIVE TO CARD ANYWAY\n");
-			}
-#endif
-			if (paddrh == 0) {
-				slic_write32(adapter, SLIC_REG_HBAR,
-					     (u32)paddrl);
-			} else {
-				slic_write64(adapter, SLIC_REG_HBAR64, paddrl,
-					     paddrh);
-			}
-			if (rcvq->head)
-				rcvq->tail->next = skb;
-			else
-				rcvq->head = skb;
-			rcvq->tail = skb;
-			rcvq->count++;
-			i++;
-		} else {
-			dev_err(&adapter->netdev->dev,
-				"slic_rcvqueue_fill could only get [%d] skbuffs\n",
-				i);
-			break;
-		}
-	}
-	return i;
-}
-
-static void slic_rcvqueue_free(struct adapter *adapter)
-{
-	struct slic_rcvqueue *rcvq = &adapter->rcvqueue;
-	struct sk_buff *skb;
-
-	while (rcvq->head) {
-		skb = rcvq->head;
-		rcvq->head = rcvq->head->next;
-		dev_kfree_skb(skb);
-	}
-	rcvq->tail = NULL;
-	rcvq->head = NULL;
-	rcvq->count = 0;
-}
-
-static int slic_rcvqueue_init(struct adapter *adapter)
-{
-	int i, count;
-	struct slic_rcvqueue *rcvq = &adapter->rcvqueue;
-
-	rcvq->tail = NULL;
-	rcvq->head = NULL;
-	rcvq->size = SLIC_RCVQ_ENTRIES;
-	rcvq->errors = 0;
-	rcvq->count = 0;
-	i = SLIC_RCVQ_ENTRIES / SLIC_RCVQ_FILLENTRIES;
-	count = 0;
-	while (i) {
-		count += slic_rcvqueue_fill(adapter);
-		i--;
-	}
-	if (rcvq->count < SLIC_RCVQ_MINENTRIES) {
-		slic_rcvqueue_free(adapter);
-		return -ENOMEM;
-	}
-	return 0;
-}
-
-static struct sk_buff *slic_rcvqueue_getnext(struct adapter *adapter)
-{
-	struct slic_rcvqueue *rcvq = &adapter->rcvqueue;
-	struct sk_buff *skb;
-	struct slic_rcvbuf *rcvbuf;
-	int count;
-
-	if (rcvq->count) {
-		skb = rcvq->head;
-		rcvbuf = (struct slic_rcvbuf *)skb->head;
-
-		if (rcvbuf->status & IRHDDR_SVALID) {
-			rcvq->head = rcvq->head->next;
-			skb->next = NULL;
-			rcvq->count--;
-		} else {
-			skb = NULL;
-		}
-	} else {
-		dev_err(&adapter->netdev->dev,
-			"RcvQ Empty!! rcvq[%p] count[%x]\n", rcvq, rcvq->count);
-		skb = NULL;
-	}
-	while (rcvq->count < SLIC_RCVQ_FILLTHRESH) {
-		count = slic_rcvqueue_fill(adapter);
-		if (!count)
-			break;
-	}
-	if (skb)
-		rcvq->errors = 0;
-	return skb;
-}
-
-static u32 slic_rcvqueue_reinsert(struct adapter *adapter, struct sk_buff *skb)
-{
-	struct slic_rcvqueue *rcvq = &adapter->rcvqueue;
-	void *paddr;
-	u32 paddrl;
-	u32 paddrh;
-	struct slic_rcvbuf *rcvbuf = (struct slic_rcvbuf *)skb->head;
-	struct device *dev;
-
-	paddr = (void *)(unsigned long)
-		pci_map_single(adapter->pcidev, skb->head,
-			       SLIC_RCVQ_RCVBUFSIZE, PCI_DMA_FROMDEVICE);
-	rcvbuf->status = 0;
-	skb->next = NULL;
-
-	paddrl = SLIC_GET_ADDR_LOW(paddr);
-	paddrh = SLIC_GET_ADDR_HIGH(paddr);
-
-	if (paddrl == 0) {
-		dev = &adapter->netdev->dev;
-		dev_err(dev, "%s: LOW 32bits PHYSICAL ADDRESS == 0\n",
-			__func__);
-		dev_err(dev, "skb[%p] PROBLEM\n", skb);
-		dev_err(dev, "         skbdata[%p]\n", skb->data);
-		dev_err(dev, "         skblen[%x]\n", skb->len);
-		dev_err(dev, "         paddr[%p]\n", paddr);
-		dev_err(dev, "         paddrl[%x]\n", paddrl);
-		dev_err(dev, "         paddrh[%x]\n", paddrh);
-		dev_err(dev, "         rcvq->head[%p]\n", rcvq->head);
-		dev_err(dev, "         rcvq->tail[%p]\n", rcvq->tail);
-		dev_err(dev, "         rcvq->count[%x]\n", rcvq->count);
-	}
-	if (paddrh == 0)
-		slic_write32(adapter, SLIC_REG_HBAR, (u32)paddrl);
-	else
-		slic_write64(adapter, SLIC_REG_HBAR64, paddrl, paddrh);
-	if (rcvq->head)
-		rcvq->tail->next = skb;
-	else
-		rcvq->head = skb;
-	rcvq->tail = skb;
-	rcvq->count++;
-	return rcvq->count;
-}
-
-/*
- * slic_link_event_handler -
- *
- * Initiate a link configuration sequence.  The link configuration begins
- * by issuing a READ_LINK_STATUS command to the Utility Processor on the
- * SLIC.  Since the command finishes asynchronously, the slic_upr_comlete
- * routine will follow it up witha UP configuration write command, which
- * will also complete asynchronously.
- *
- */
-static int slic_link_event_handler(struct adapter *adapter)
-{
-	int status;
-	struct slic_shmemory *sm = &adapter->shmem;
-	dma_addr_t phaddr = sm->lnkstatus_phaddr;
-
-	if (adapter->state != ADAPT_UP) {
-		/* Adapter is not operational.  Ignore.  */
-		return -ENODEV;
-	}
-	/* no 4GB wrap guaranteed */
-	status = slic_upr_request(adapter, SLIC_UPR_RLSR,
-				  cpu_to_le32(lower_32_bits(phaddr)),
-				  cpu_to_le32(upper_32_bits(phaddr)), 0, 0);
-	return status;
-}
-
-static void slic_init_cleanup(struct adapter *adapter)
-{
-	if (adapter->intrregistered) {
-		adapter->intrregistered = 0;
-		free_irq(adapter->netdev->irq, adapter->netdev);
-	}
-
-	if (adapter->shmem.shmem_data) {
-		struct slic_shmemory *sm = &adapter->shmem;
-		struct slic_shmem_data *sm_data = sm->shmem_data;
-
-		pci_free_consistent(adapter->pcidev, sizeof(*sm_data), sm_data,
-				    sm->isr_phaddr);
-	}
-
-	if (adapter->pingtimerset) {
-		adapter->pingtimerset = 0;
-		del_timer(&adapter->pingtimer);
-	}
-
-	slic_rspqueue_free(adapter);
-	slic_cmdq_free(adapter);
-	slic_rcvqueue_free(adapter);
-}
-
-/*
- *  Allocate a mcast_address structure to hold the multicast address.
- *  Link it in.
- */
-static int slic_mcast_add_list(struct adapter *adapter, char *address)
-{
-	struct mcast_address *mcaddr, *mlist;
-
-	/* Check to see if it already exists */
-	mlist = adapter->mcastaddrs;
-	while (mlist) {
-		if (ether_addr_equal(mlist->address, address))
-			return 0;
-		mlist = mlist->next;
-	}
-
-	/* Doesn't already exist.  Allocate a structure to hold it */
-	mcaddr = kmalloc(sizeof(*mcaddr), GFP_ATOMIC);
-	if (!mcaddr)
-		return 1;
-
-	ether_addr_copy(mcaddr->address, address);
-
-	mcaddr->next = adapter->mcastaddrs;
-	adapter->mcastaddrs = mcaddr;
-
-	return 0;
-}
-
-static void slic_mcast_set_list(struct net_device *dev)
-{
-	struct adapter *adapter = netdev_priv(dev);
-	int status = 0;
-	char *addresses;
-	struct netdev_hw_addr *ha;
-
-	netdev_for_each_mc_addr(ha, dev) {
-		addresses = (char *)&ha->addr;
-		status = slic_mcast_add_list(adapter, addresses);
-		if (status != 0)
-			break;
-		slic_mcast_set_bit(adapter, addresses);
-	}
-
-	if (adapter->devflags_prev != dev->flags) {
-		adapter->macopts = MAC_DIRECTED;
-		if (dev->flags) {
-			if (dev->flags & IFF_BROADCAST)
-				adapter->macopts |= MAC_BCAST;
-			if (dev->flags & IFF_PROMISC)
-				adapter->macopts |= MAC_PROMISC;
-			if (dev->flags & IFF_ALLMULTI)
-				adapter->macopts |= MAC_ALLMCAST;
-			if (dev->flags & IFF_MULTICAST)
-				adapter->macopts |= MAC_MCAST;
-		}
-		adapter->devflags_prev = dev->flags;
-		slic_config_set(adapter, true);
-	} else {
-		if (status == 0)
-			slic_mcast_set_mask(adapter);
-	}
-}
-
-#define  XMIT_FAIL_LINK_STATE               1
-#define  XMIT_FAIL_ZERO_LENGTH              2
-#define  XMIT_FAIL_HOSTCMD_FAIL             3
-
-static void slic_xmit_build_request(struct adapter *adapter,
-			     struct slic_hostcmd *hcmd, struct sk_buff *skb)
-{
-	struct slic_host64_cmd *ihcmd;
-	ulong phys_addr;
-
-	ihcmd = &hcmd->cmd64;
-
-	ihcmd->flags = adapter->port << IHFLG_IFSHFT;
-	ihcmd->command = IHCMD_XMT_REQ;
-	ihcmd->u.slic_buffers.totlen = skb->len;
-	phys_addr = pci_map_single(adapter->pcidev, skb->data, skb->len,
-			PCI_DMA_TODEVICE);
-	if (pci_dma_mapping_error(adapter->pcidev, phys_addr)) {
-		kfree_skb(skb);
-		dev_err(&adapter->pcidev->dev, "DMA mapping error\n");
-		return;
-	}
-	ihcmd->u.slic_buffers.bufs[0].paddrl = SLIC_GET_ADDR_LOW(phys_addr);
-	ihcmd->u.slic_buffers.bufs[0].paddrh = SLIC_GET_ADDR_HIGH(phys_addr);
-	ihcmd->u.slic_buffers.bufs[0].length = skb->len;
-#if BITS_PER_LONG == 64
-	hcmd->cmdsize = (u32)((((u64)&ihcmd->u.slic_buffers.bufs[1] -
-				     (u64)hcmd) + 31) >> 5);
-#else
-	hcmd->cmdsize = (((u32)&ihcmd->u.slic_buffers.bufs[1] -
-				       (u32)hcmd) + 31) >> 5;
-#endif
-}
-
-static void slic_xmit_fail(struct adapter *adapter,
-		    struct sk_buff *skb,
-		    void *cmd, u32 skbtype, u32 status)
-{
-	if (adapter->xmitq_full)
-		netif_stop_queue(adapter->netdev);
-	if ((!cmd) && (status <= XMIT_FAIL_HOSTCMD_FAIL)) {
-		switch (status) {
-		case XMIT_FAIL_LINK_STATE:
-			dev_err(&adapter->netdev->dev,
-				"reject xmit skb[%p: %x] linkstate[%s] adapter[%s:%d] card[%s:%d]\n",
-				skb, skb->pkt_type,
-				SLIC_LINKSTATE(adapter->linkstate),
-				SLIC_ADAPTER_STATE(adapter->state),
-				adapter->state,
-				SLIC_CARD_STATE(adapter->card->state),
-				adapter->card->state);
-			break;
-		case XMIT_FAIL_ZERO_LENGTH:
-			dev_err(&adapter->netdev->dev,
-				"xmit_start skb->len == 0 skb[%p] type[%x]\n",
-				skb, skb->pkt_type);
-			break;
-		case XMIT_FAIL_HOSTCMD_FAIL:
-			dev_err(&adapter->netdev->dev,
-				"xmit_start skb[%p] type[%x] No host commands available\n",
-				skb, skb->pkt_type);
-			break;
-		}
-	}
-	dev_kfree_skb(skb);
-	adapter->netdev->stats.tx_dropped++;
-}
-
-static void slic_rcv_handle_error(struct adapter *adapter,
-					struct slic_rcvbuf *rcvbuf)
-{
-	struct slic_hddr_wds *hdr = (struct slic_hddr_wds *)rcvbuf->data;
-	struct net_device *netdev = adapter->netdev;
-
-	if (adapter->devid != SLIC_1GB_DEVICE_ID) {
-		if (hdr->frame_status14 & VRHSTAT_802OE)
-			adapter->if_events.oflow802++;
-		if (hdr->frame_status14 & VRHSTAT_TPOFLO)
-			adapter->if_events.Tprtoflow++;
-		if (hdr->frame_status_b14 & VRHSTATB_802UE)
-			adapter->if_events.uflow802++;
-		if (hdr->frame_status_b14 & VRHSTATB_RCVE) {
-			adapter->if_events.rcvearly++;
-			netdev->stats.rx_fifo_errors++;
-		}
-		if (hdr->frame_status_b14 & VRHSTATB_BUFF) {
-			adapter->if_events.Bufov++;
-			netdev->stats.rx_over_errors++;
-		}
-		if (hdr->frame_status_b14 & VRHSTATB_CARRE) {
-			adapter->if_events.Carre++;
-			netdev->stats.tx_carrier_errors++;
-		}
-		if (hdr->frame_status_b14 & VRHSTATB_LONGE)
-			adapter->if_events.Longe++;
-		if (hdr->frame_status_b14 & VRHSTATB_PREA)
-			adapter->if_events.Invp++;
-		if (hdr->frame_status_b14 & VRHSTATB_CRC) {
-			adapter->if_events.Crc++;
-			netdev->stats.rx_crc_errors++;
-		}
-		if (hdr->frame_status_b14 & VRHSTATB_DRBL)
-			adapter->if_events.Drbl++;
-		if (hdr->frame_status_b14 & VRHSTATB_CODE)
-			adapter->if_events.Code++;
-		if (hdr->frame_status_b14 & VRHSTATB_TPCSUM)
-			adapter->if_events.TpCsum++;
-		if (hdr->frame_status_b14 & VRHSTATB_TPHLEN)
-			adapter->if_events.TpHlen++;
-		if (hdr->frame_status_b14 & VRHSTATB_IPCSUM)
-			adapter->if_events.IpCsum++;
-		if (hdr->frame_status_b14 & VRHSTATB_IPLERR)
-			adapter->if_events.IpLen++;
-		if (hdr->frame_status_b14 & VRHSTATB_IPHERR)
-			adapter->if_events.IpHlen++;
-	} else {
-		if (hdr->frame_statusGB & VGBSTAT_XPERR) {
-			u32 xerr = hdr->frame_statusGB >> VGBSTAT_XERRSHFT;
-
-			if (xerr == VGBSTAT_XCSERR)
-				adapter->if_events.TpCsum++;
-			if (xerr == VGBSTAT_XUFLOW)
-				adapter->if_events.Tprtoflow++;
-			if (xerr == VGBSTAT_XHLEN)
-				adapter->if_events.TpHlen++;
-		}
-		if (hdr->frame_statusGB & VGBSTAT_NETERR) {
-			u32 nerr =
-			    (hdr->
-			     frame_statusGB >> VGBSTAT_NERRSHFT) &
-			    VGBSTAT_NERRMSK;
-			if (nerr == VGBSTAT_NCSERR)
-				adapter->if_events.IpCsum++;
-			if (nerr == VGBSTAT_NUFLOW)
-				adapter->if_events.IpLen++;
-			if (nerr == VGBSTAT_NHLEN)
-				adapter->if_events.IpHlen++;
-		}
-		if (hdr->frame_statusGB & VGBSTAT_LNKERR) {
-			u32 lerr = hdr->frame_statusGB & VGBSTAT_LERRMSK;
-
-			if (lerr == VGBSTAT_LDEARLY)
-				adapter->if_events.rcvearly++;
-			if (lerr == VGBSTAT_LBOFLO)
-				adapter->if_events.Bufov++;
-			if (lerr == VGBSTAT_LCODERR)
-				adapter->if_events.Code++;
-			if (lerr == VGBSTAT_LDBLNBL)
-				adapter->if_events.Drbl++;
-			if (lerr == VGBSTAT_LCRCERR)
-				adapter->if_events.Crc++;
-			if (lerr == VGBSTAT_LOFLO)
-				adapter->if_events.oflow802++;
-			if (lerr == VGBSTAT_LUFLO)
-				adapter->if_events.uflow802++;
-		}
-	}
-}
-
-#define TCP_OFFLOAD_FRAME_PUSHFLAG  0x10000000
-#define M_FAST_PATH                 0x0040
-
-static void slic_rcv_handler(struct adapter *adapter)
-{
-	struct net_device *netdev = adapter->netdev;
-	struct sk_buff *skb;
-	struct slic_rcvbuf *rcvbuf;
-	u32 frames = 0;
-
-	while ((skb = slic_rcvqueue_getnext(adapter))) {
-		u32 rx_bytes;
-
-		rcvbuf = (struct slic_rcvbuf *)skb->head;
-		adapter->card->events++;
-		if (rcvbuf->status & IRHDDR_ERR) {
-			adapter->rx_errors++;
-			slic_rcv_handle_error(adapter, rcvbuf);
-			slic_rcvqueue_reinsert(adapter, skb);
-			continue;
-		}
-
-		if (!slic_mac_filter(adapter, (struct ether_header *)
-					rcvbuf->data)) {
-			slic_rcvqueue_reinsert(adapter, skb);
-			continue;
-		}
-		skb_pull(skb, SLIC_RCVBUF_HEADSIZE);
-		rx_bytes = (rcvbuf->length & IRHDDR_FLEN_MSK);
-		skb_put(skb, rx_bytes);
-		netdev->stats.rx_packets++;
-		netdev->stats.rx_bytes += rx_bytes;
-#if SLIC_OFFLOAD_IP_CHECKSUM
-		skb->ip_summed = CHECKSUM_UNNECESSARY;
-#endif
-
-		skb->dev = adapter->netdev;
-		skb->protocol = eth_type_trans(skb, skb->dev);
-		netif_rx(skb);
-
-		++frames;
-#if SLIC_INTERRUPT_PROCESS_LIMIT
-		if (frames >= SLIC_RCVQ_MAX_PROCESS_ISR) {
-			adapter->rcv_interrupt_yields++;
-			break;
-		}
-#endif
-	}
-	adapter->max_isr_rcvs = max(adapter->max_isr_rcvs, frames);
-}
-
-static void slic_xmit_complete(struct adapter *adapter)
-{
-	struct slic_hostcmd *hcmd;
-	struct slic_rspbuf *rspbuf;
-	u32 frames = 0;
-	struct slic_handle_word slic_handle_word;
-
-	do {
-		rspbuf = slic_rspqueue_getnext(adapter);
-		if (!rspbuf)
-			break;
-		adapter->xmit_completes++;
-		adapter->card->events++;
-		/*
-		 * Get the complete host command buffer
-		 */
-		slic_handle_word.handle_token = rspbuf->hosthandle;
-		hcmd =
-			adapter->slic_handles[slic_handle_word.handle_index].
-									address;
-/*      hcmd = (struct slic_hostcmd *) rspbuf->hosthandle; */
-		if (hcmd->type == SLIC_CMD_DUMB) {
-			if (hcmd->skb)
-				dev_kfree_skb_irq(hcmd->skb);
-			slic_cmdq_putdone_irq(adapter, hcmd);
-		}
-		rspbuf->status = 0;
-		rspbuf->hosthandle = 0;
-		frames++;
-	} while (1);
-	adapter->max_isr_xmits = max(adapter->max_isr_xmits, frames);
-}
-
-static void slic_interrupt_card_up(u32 isr, struct adapter *adapter,
-			struct net_device *dev)
-{
-	if (isr & ~ISR_IO) {
-		if (isr & ISR_ERR) {
-			adapter->error_interrupts++;
-			if (isr & ISR_RMISS) {
-				int count;
-				int pre_count;
-				int errors;
-
-				struct slic_rcvqueue *rcvq =
-					&adapter->rcvqueue;
-
-				adapter->error_rmiss_interrupts++;
-
-				if (!rcvq->errors)
-					rcv_count = rcvq->count;
-				pre_count = rcvq->count;
-				errors = rcvq->errors;
-
-				while (rcvq->count < SLIC_RCVQ_FILLTHRESH) {
-					count = slic_rcvqueue_fill(adapter);
-					if (!count)
-						break;
-				}
-			} else if (isr & ISR_XDROP) {
-				dev_err(&dev->dev,
-						"isr & ISR_ERR [%x] ISR_XDROP\n",
-						isr);
-			} else {
-				dev_err(&dev->dev,
-						"isr & ISR_ERR [%x]\n",
-						isr);
-			}
-		}
-
-		if (isr & ISR_LEVENT) {
-			adapter->linkevent_interrupts++;
-			if (slic_link_event_handler(adapter))
-				adapter->linkevent_interrupts--;
-		}
-
-		if ((isr & ISR_UPC) || (isr & ISR_UPCERR) ||
-		    (isr & ISR_UPCBSY)) {
-			adapter->upr_interrupts++;
-			slic_upr_request_complete(adapter, isr);
-		}
-	}
-
-	if (isr & ISR_RCV) {
-		adapter->rcv_interrupts++;
-		slic_rcv_handler(adapter);
-	}
-
-	if (isr & ISR_CMD) {
-		adapter->xmit_interrupts++;
-		slic_xmit_complete(adapter);
-	}
-}
-
-static irqreturn_t slic_interrupt(int irq, void *dev_id)
-{
-	struct net_device *dev = dev_id;
-	struct adapter *adapter = netdev_priv(dev);
-	struct slic_shmemory *sm = &adapter->shmem;
-	struct slic_shmem_data *sm_data = sm->shmem_data;
-	u32 isr;
-
-	if (sm_data->isr) {
-		slic_write32(adapter, SLIC_REG_ICR, ICR_INT_MASK);
-		slic_flush_write(adapter);
-
-		isr = sm_data->isr;
-		sm_data->isr = 0;
-		adapter->num_isrs++;
-		switch (adapter->card->state) {
-		case CARD_UP:
-			slic_interrupt_card_up(isr, adapter, dev);
-			break;
-
-		case CARD_DOWN:
-			if ((isr & ISR_UPC) ||
-			    (isr & ISR_UPCERR) || (isr & ISR_UPCBSY)) {
-				adapter->upr_interrupts++;
-				slic_upr_request_complete(adapter, isr);
-			}
-			break;
-		}
-
-		adapter->all_reg_writes += 2;
-		adapter->isr_reg_writes++;
-		slic_write32(adapter, SLIC_REG_ISR, 0);
-	} else {
-		adapter->false_interrupts++;
-	}
-	return IRQ_HANDLED;
-}
-
-#define NORMAL_ETHFRAME     0
-
-static netdev_tx_t slic_xmit_start(struct sk_buff *skb, struct net_device *dev)
-{
-	struct sliccard *card;
-	struct adapter *adapter = netdev_priv(dev);
-	struct slic_hostcmd *hcmd = NULL;
-	u32 status = 0;
-	void *offloadcmd = NULL;
-
-	card = adapter->card;
-	if ((adapter->linkstate != LINK_UP) ||
-	    (adapter->state != ADAPT_UP) || (card->state != CARD_UP)) {
-		status = XMIT_FAIL_LINK_STATE;
-		goto xmit_fail;
-
-	} else if (skb->len == 0) {
-		status = XMIT_FAIL_ZERO_LENGTH;
-		goto xmit_fail;
-	}
-
-	hcmd = slic_cmdq_getfree(adapter);
-	if (!hcmd) {
-		adapter->xmitq_full = 1;
-		status = XMIT_FAIL_HOSTCMD_FAIL;
-		goto xmit_fail;
-	}
-	hcmd->skb = skb;
-	hcmd->busy = 1;
-	hcmd->type = SLIC_CMD_DUMB;
-	slic_xmit_build_request(adapter, hcmd, skb);
-	dev->stats.tx_packets++;
-	dev->stats.tx_bytes += skb->len;
-
-#ifdef DEBUG_DUMP
-	if (adapter->kill_card) {
-		struct slic_host64_cmd ihcmd;
-
-		ihcmd = &hcmd->cmd64;
-
-		ihcmd->flags |= 0x40;
-		adapter->kill_card = 0;	/* only do this once */
-	}
-#endif
-	if (hcmd->paddrh == 0) {
-		slic_write32(adapter, SLIC_REG_CBAR, (hcmd->paddrl |
-						      hcmd->cmdsize));
-	} else {
-		slic_write64(adapter, SLIC_REG_CBAR64,
-			     hcmd->paddrl | hcmd->cmdsize, hcmd->paddrh);
-	}
-xmit_done:
-	return NETDEV_TX_OK;
-xmit_fail:
-	slic_xmit_fail(adapter, skb, offloadcmd, NORMAL_ETHFRAME, status);
-	goto xmit_done;
-}
-
-static void slic_adapter_freeresources(struct adapter *adapter)
-{
-	slic_init_cleanup(adapter);
-	adapter->error_interrupts = 0;
-	adapter->rcv_interrupts = 0;
-	adapter->xmit_interrupts = 0;
-	adapter->linkevent_interrupts = 0;
-	adapter->upr_interrupts = 0;
-	adapter->num_isrs = 0;
-	adapter->xmit_completes = 0;
-	adapter->rcv_broadcasts = 0;
-	adapter->rcv_multicasts = 0;
-	adapter->rcv_unicasts = 0;
-}
-
-static int slic_adapter_allocresources(struct adapter *adapter,
-				       unsigned long *flags)
-{
-	if (!adapter->intrregistered) {
-		int retval;
-
-		spin_unlock_irqrestore(&slic_global.driver_lock, *flags);
-
-		retval = request_irq(adapter->netdev->irq,
-				     &slic_interrupt,
-				     IRQF_SHARED,
-				     adapter->netdev->name, adapter->netdev);
-
-		spin_lock_irqsave(&slic_global.driver_lock, *flags);
-
-		if (retval) {
-			dev_err(&adapter->netdev->dev,
-				"request_irq (%s) FAILED [%x]\n",
-				adapter->netdev->name, retval);
-			return retval;
-		}
-		adapter->intrregistered = 1;
-	}
-	return 0;
-}
-
-/*
- *  slic_if_init
- *
- *  Perform initialization of our slic interface.
- *
- */
-static int slic_if_init(struct adapter *adapter, unsigned long *flags)
-{
-	struct sliccard *card = adapter->card;
-	struct net_device *dev = adapter->netdev;
-	struct slic_shmemory *sm = &adapter->shmem;
-	struct slic_shmem_data *sm_data = sm->shmem_data;
-	int rc;
-
-	/* adapter should be down at this point */
-	if (adapter->state != ADAPT_DOWN) {
-		dev_err(&dev->dev, "%s: adapter->state != ADAPT_DOWN\n",
-			__func__);
-		rc = -EIO;
-		goto err;
-	}
-
-	adapter->devflags_prev = dev->flags;
-	adapter->macopts = MAC_DIRECTED;
-	if (dev->flags) {
-		if (dev->flags & IFF_BROADCAST)
-			adapter->macopts |= MAC_BCAST;
-		if (dev->flags & IFF_PROMISC)
-			adapter->macopts |= MAC_PROMISC;
-		if (dev->flags & IFF_ALLMULTI)
-			adapter->macopts |= MAC_ALLMCAST;
-		if (dev->flags & IFF_MULTICAST)
-			adapter->macopts |= MAC_MCAST;
-	}
-	rc = slic_adapter_allocresources(adapter, flags);
-	if (rc) {
-		dev_err(&dev->dev, "slic_adapter_allocresources FAILED %x\n",
-			rc);
-		slic_adapter_freeresources(adapter);
-		goto err;
-	}
-
-	if (!adapter->queues_initialized) {
-		rc = slic_rspqueue_init(adapter);
-		if (rc)
-			goto err;
-		rc = slic_cmdq_init(adapter);
-		if (rc)
-			goto err;
-		rc = slic_rcvqueue_init(adapter);
-		if (rc)
-			goto err;
-		adapter->queues_initialized = 1;
-	}
-
-	slic_write32(adapter, SLIC_REG_ICR, ICR_INT_OFF);
-	slic_flush_write(adapter);
-	mdelay(1);
-
-	if (!adapter->isp_initialized) {
-		unsigned long flags;
-
-		spin_lock_irqsave(&adapter->bit64reglock, flags);
-		slic_write32(adapter, SLIC_REG_ADDR_UPPER,
-			     cpu_to_le32(upper_32_bits(sm->isr_phaddr)));
-		slic_write32(adapter, SLIC_REG_ISP,
-			     cpu_to_le32(lower_32_bits(sm->isr_phaddr)));
-		spin_unlock_irqrestore(&adapter->bit64reglock, flags);
-
-		adapter->isp_initialized = 1;
-	}
-
-	adapter->state = ADAPT_UP;
-	if (!card->loadtimerset) {
-		setup_timer(&card->loadtimer, &slic_timer_load_check,
-			    (ulong)card);
-		card->loadtimer.expires =
-		    jiffies + (SLIC_LOADTIMER_PERIOD * HZ);
-		add_timer(&card->loadtimer);
-
-		card->loadtimerset = 1;
-	}
-
-	if (!adapter->pingtimerset) {
-		setup_timer(&adapter->pingtimer, &slic_timer_ping, (ulong)dev);
-		adapter->pingtimer.expires =
-		    jiffies + (PING_TIMER_INTERVAL * HZ);
-		add_timer(&adapter->pingtimer);
-		adapter->pingtimerset = 1;
-		adapter->card->pingstatus = ISR_PINGMASK;
-	}
-
-	/*
-	 *    clear any pending events, then enable interrupts
-	 */
-	sm_data->isr = 0;
-	slic_write32(adapter, SLIC_REG_ISR, 0);
-	slic_write32(adapter, SLIC_REG_ICR, ICR_INT_ON);
-
-	slic_link_config(adapter, LINK_AUTOSPEED, LINK_AUTOD);
-	slic_flush_write(adapter);
-
-	rc = slic_link_event_handler(adapter);
-	if (rc) {
-		/* disable interrupts then clear pending events */
-		slic_write32(adapter, SLIC_REG_ICR, ICR_INT_OFF);
-		slic_write32(adapter, SLIC_REG_ISR, 0);
-		slic_flush_write(adapter);
-
-		if (adapter->pingtimerset) {
-			del_timer(&adapter->pingtimer);
-			adapter->pingtimerset = 0;
-		}
-		if (card->loadtimerset) {
-			del_timer(&card->loadtimer);
-			card->loadtimerset = 0;
-		}
-		adapter->state = ADAPT_DOWN;
-		slic_adapter_freeresources(adapter);
-	}
-
-err:
-	return rc;
-}
-
-static int slic_entry_open(struct net_device *dev)
-{
-	struct adapter *adapter = netdev_priv(dev);
-	struct sliccard *card = adapter->card;
-	unsigned long flags;
-	int status;
-
-	netif_carrier_off(dev);
-
-	spin_lock_irqsave(&slic_global.driver_lock, flags);
-	if (!adapter->activated) {
-		card->adapters_activated++;
-		slic_global.num_slic_ports_active++;
-		adapter->activated = 1;
-	}
-	status = slic_if_init(adapter, &flags);
-
-	if (status != 0) {
-		if (adapter->activated) {
-			card->adapters_activated--;
-			slic_global.num_slic_ports_active--;
-			adapter->activated = 0;
-		}
-		goto spin_unlock;
-	}
-	if (!card->master)
-		card->master = adapter;
-
-spin_unlock:
-	spin_unlock_irqrestore(&slic_global.driver_lock, flags);
-
-	netif_start_queue(adapter->netdev);
-
-	return status;
-}
-
-static void slic_card_cleanup(struct sliccard *card)
-{
-	if (card->loadtimerset) {
-		card->loadtimerset = 0;
-		del_timer_sync(&card->loadtimer);
-	}
-
-	kfree(card);
-}
-
-static void slic_entry_remove(struct pci_dev *pcidev)
-{
-	struct net_device *dev = pci_get_drvdata(pcidev);
-	struct adapter *adapter = netdev_priv(dev);
-	struct sliccard *card;
-	struct mcast_address *mcaddr, *mlist;
-
-	unregister_netdev(dev);
-
-	slic_adapter_freeresources(adapter);
-	iounmap(adapter->regs);
-
-	/* free multicast addresses */
-	mlist = adapter->mcastaddrs;
-	while (mlist) {
-		mcaddr = mlist;
-		mlist = mlist->next;
-		kfree(mcaddr);
-	}
-	card = adapter->card;
-	card->adapters_allocated--;
-	adapter->allocated = 0;
-	if (!card->adapters_allocated) {
-		struct sliccard *curr_card = slic_global.slic_card;
-
-		if (curr_card == card) {
-			slic_global.slic_card = card->next;
-		} else {
-			while (curr_card->next != card)
-				curr_card = curr_card->next;
-			curr_card->next = card->next;
-		}
-		slic_global.num_slic_cards--;
-		slic_card_cleanup(card);
-	}
-	free_netdev(dev);
-	pci_release_regions(pcidev);
-	pci_disable_device(pcidev);
-}
-
-static int slic_entry_halt(struct net_device *dev)
-{
-	struct adapter *adapter = netdev_priv(dev);
-	struct sliccard *card = adapter->card;
-	unsigned long flags;
-
-	spin_lock_irqsave(&slic_global.driver_lock, flags);
-	netif_stop_queue(adapter->netdev);
-	adapter->state = ADAPT_DOWN;
-	adapter->linkstate = LINK_DOWN;
-	adapter->upr_list = NULL;
-	adapter->upr_busy = 0;
-	adapter->devflags_prev = 0;
-	slic_write32(adapter, SLIC_REG_ICR, ICR_INT_OFF);
-	adapter->all_reg_writes++;
-	adapter->icr_reg_writes++;
-	slic_config_clear(adapter);
-	if (adapter->activated) {
-		card->adapters_activated--;
-		slic_global.num_slic_ports_active--;
-		adapter->activated = 0;
-	}
-#ifdef AUTOMATIC_RESET
-	slic_write32(adapter, SLIC_REG_RESET_IFACE, 0);
-#endif
-	slic_flush_write(adapter);
-
-	/*
-	 *  Reset the adapter's cmd queues
-	 */
-	slic_cmdq_reset(adapter);
-
-#ifdef AUTOMATIC_RESET
-	if (!card->adapters_activated)
-		slic_card_init(card, adapter);
-#endif
-
-	spin_unlock_irqrestore(&slic_global.driver_lock, flags);
-
-	netif_carrier_off(dev);
-
-	return 0;
-}
-
-static struct net_device_stats *slic_get_stats(struct net_device *dev)
-{
-	struct adapter *adapter = netdev_priv(dev);
-
-	dev->stats.collisions = adapter->slic_stats.iface.xmit_collisions;
-	dev->stats.rx_errors = adapter->slic_stats.iface.rcv_errors;
-	dev->stats.tx_errors = adapter->slic_stats.iface.xmt_errors;
-	dev->stats.rx_missed_errors = adapter->slic_stats.iface.rcv_discards;
-	dev->stats.tx_heartbeat_errors = 0;
-	dev->stats.tx_aborted_errors = 0;
-	dev->stats.tx_window_errors = 0;
-	dev->stats.tx_fifo_errors = 0;
-	dev->stats.rx_frame_errors = 0;
-	dev->stats.rx_length_errors = 0;
-
-	return &dev->stats;
-}
-
-static int slic_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
-{
-	struct adapter *adapter = netdev_priv(dev);
-	struct ethtool_cmd edata;
-	struct ethtool_cmd ecmd;
-	u32 data[7];
-	u32 intagg;
-
-	switch (cmd) {
-	case SIOCSLICSETINTAGG:
-		if (copy_from_user(data, rq->ifr_data, 28))
-			return -EFAULT;
-		intagg = data[0];
-		dev_err(&dev->dev, "set interrupt aggregation to %d\n",
-			intagg);
-		slic_intagg_set(adapter, intagg);
-		return 0;
-
-	case SIOCETHTOOL:
-		if (copy_from_user(&ecmd, rq->ifr_data, sizeof(ecmd)))
-			return -EFAULT;
-
-		if (ecmd.cmd == ETHTOOL_GSET) {
-			memset(&edata, 0, sizeof(edata));
-			edata.supported = (SUPPORTED_10baseT_Half |
-					   SUPPORTED_10baseT_Full |
-					   SUPPORTED_100baseT_Half |
-					   SUPPORTED_100baseT_Full |
-					   SUPPORTED_Autoneg | SUPPORTED_MII);
-			edata.port = PORT_MII;
-			edata.transceiver = XCVR_INTERNAL;
-			edata.phy_address = 0;
-			if (adapter->linkspeed == LINK_100MB)
-				edata.speed = SPEED_100;
-			else if (adapter->linkspeed == LINK_10MB)
-				edata.speed = SPEED_10;
-			else
-				edata.speed = 0;
-
-			if (adapter->linkduplex == LINK_FULLD)
-				edata.duplex = DUPLEX_FULL;
-			else
-				edata.duplex = DUPLEX_HALF;
-
-			edata.autoneg = AUTONEG_ENABLE;
-			edata.maxtxpkt = 1;
-			edata.maxrxpkt = 1;
-			if (copy_to_user(rq->ifr_data, &edata, sizeof(edata)))
-				return -EFAULT;
-
-		} else if (ecmd.cmd == ETHTOOL_SSET) {
-			if (!capable(CAP_NET_ADMIN))
-				return -EPERM;
-
-			if (adapter->linkspeed == LINK_100MB)
-				edata.speed = SPEED_100;
-			else if (adapter->linkspeed == LINK_10MB)
-				edata.speed = SPEED_10;
-			else
-				edata.speed = 0;
-
-			if (adapter->linkduplex == LINK_FULLD)
-				edata.duplex = DUPLEX_FULL;
-			else
-				edata.duplex = DUPLEX_HALF;
-
-			edata.autoneg = AUTONEG_ENABLE;
-			edata.maxtxpkt = 1;
-			edata.maxrxpkt = 1;
-			if ((ecmd.speed != edata.speed) ||
-			    (ecmd.duplex != edata.duplex)) {
-				u32 speed;
-				u32 duplex;
-
-				if (ecmd.speed == SPEED_10)
-					speed = 0;
-				else
-					speed = PCR_SPEED_100;
-				if (ecmd.duplex == DUPLEX_FULL)
-					duplex = PCR_DUPLEX_FULL;
-				else
-					duplex = 0;
-				slic_link_config(adapter, speed, duplex);
-				if (slic_link_event_handler(adapter))
-					return -EFAULT;
-			}
-		}
-		return 0;
-	default:
-		return -EOPNOTSUPP;
-	}
-}
-
-static void slic_config_pci(struct pci_dev *pcidev)
-{
-	u16 pci_command;
-	u16 new_command;
-
-	pci_read_config_word(pcidev, PCI_COMMAND, &pci_command);
-
-	new_command = pci_command | PCI_COMMAND_MASTER
-	    | PCI_COMMAND_MEMORY
-	    | PCI_COMMAND_INVALIDATE
-	    | PCI_COMMAND_PARITY | PCI_COMMAND_SERR | PCI_COMMAND_FAST_BACK;
-	if (pci_command != new_command)
-		pci_write_config_word(pcidev, PCI_COMMAND, new_command);
-}
-
-static int slic_card_init(struct sliccard *card, struct adapter *adapter)
-{
-	struct slic_shmemory *sm = &adapter->shmem;
-	struct slic_shmem_data *sm_data = sm->shmem_data;
-	struct slic_eeprom *peeprom;
-	struct oslic_eeprom *pOeeprom;
-	dma_addr_t phys_config;
-	u32 phys_configh;
-	u32 phys_configl;
-	u32 i = 0;
-	int status;
-	uint macaddrs = card->card_size;
-	ushort eecodesize;
-	ushort dramsize;
-	ushort ee_chksum;
-	ushort calc_chksum;
-	struct slic_config_mac *pmac;
-	unsigned char fruformat;
-	unsigned char oemfruformat;
-	struct atk_fru *patkfru;
-	union oemfru *poemfru;
-	unsigned long flags;
-
-	/* Reset everything except PCI configuration space */
-	slic_soft_reset(adapter);
-
-	/* Download the microcode */
-	status = slic_card_download(adapter);
-	if (status)
-		return status;
-
-	if (!card->config_set) {
-		peeprom = pci_alloc_consistent(adapter->pcidev,
-					       sizeof(struct slic_eeprom),
-					       &phys_config);
-
-		if (!peeprom) {
-			dev_err(&adapter->pcidev->dev,
-				"Failed to allocate DMA memory for EEPROM.\n");
-			return -ENOMEM;
-		}
-
-		phys_configl = SLIC_GET_ADDR_LOW(phys_config);
-		phys_configh = SLIC_GET_ADDR_HIGH(phys_config);
-
-		memset(peeprom, 0, sizeof(struct slic_eeprom));
-
-		slic_write32(adapter, SLIC_REG_ICR, ICR_INT_OFF);
-		slic_flush_write(adapter);
-		mdelay(1);
-
-		spin_lock_irqsave(&adapter->bit64reglock, flags);
-		slic_write32(adapter, SLIC_REG_ADDR_UPPER,
-			     cpu_to_le32(upper_32_bits(sm->isr_phaddr)));
-		slic_write32(adapter, SLIC_REG_ISP,
-			     cpu_to_le32(lower_32_bits(sm->isr_phaddr)));
-		spin_unlock_irqrestore(&adapter->bit64reglock, flags);
-
-		status = slic_config_get(adapter, phys_configl, phys_configh);
-		if (status) {
-			dev_err(&adapter->pcidev->dev,
-				"Failed to fetch config data from device.\n");
-			goto card_init_err;
-		}
-
-		for (;;) {
-			if (sm_data->isr) {
-				if (sm_data->isr & ISR_UPC) {
-					sm_data->isr = 0;
-					slic_write64(adapter, SLIC_REG_ISP, 0,
-						     0);
-					slic_write32(adapter, SLIC_REG_ISR, 0);
-					slic_flush_write(adapter);
-
-					slic_upr_request_complete(adapter, 0);
-					break;
-				}
-
-				sm_data->isr = 0;
-				slic_write32(adapter, SLIC_REG_ISR, 0);
-				slic_flush_write(adapter);
-			} else {
-				mdelay(1);
-				i++;
-				if (i > 5000) {
-					dev_err(&adapter->pcidev->dev,
-						"Fetch of config data timed out.\n");
-					slic_write64(adapter, SLIC_REG_ISP,
-						     0, 0);
-					slic_flush_write(adapter);
-
-					status = -EINVAL;
-					goto card_init_err;
-				}
-			}
-		}
-
-		switch (adapter->devid) {
-		/* Oasis card */
-		case SLIC_2GB_DEVICE_ID:
-			/* extract EEPROM data and pointers to EEPROM data */
-			pOeeprom = (struct oslic_eeprom *)peeprom;
-			eecodesize = pOeeprom->EecodeSize;
-			dramsize = pOeeprom->DramSize;
-			pmac = pOeeprom->MacInfo;
-			fruformat = pOeeprom->FruFormat;
-			patkfru = &pOeeprom->AtkFru;
-			oemfruformat = pOeeprom->OemFruFormat;
-			poemfru = &pOeeprom->OemFru;
-			macaddrs = 2;
-			/*
-			 * Minor kludge for Oasis card
-			 * get 2 MAC addresses from the
-			 * EEPROM to ensure that function 1
-			 * gets the Port 1 MAC address
-			 */
-			break;
-		default:
-			/* extract EEPROM data and pointers to EEPROM data */
-			eecodesize = peeprom->EecodeSize;
-			dramsize = peeprom->DramSize;
-			pmac = peeprom->u2.mac.MacInfo;
-			fruformat = peeprom->FruFormat;
-			patkfru = &peeprom->AtkFru;
-			oemfruformat = peeprom->OemFruFormat;
-			poemfru = &peeprom->OemFru;
-			break;
-		}
-
-		card->config.EepromValid = false;
-
-		/*  see if the EEPROM is valid by checking it's checksum */
-		if ((eecodesize <= MAX_EECODE_SIZE) &&
-		    (eecodesize >= MIN_EECODE_SIZE)) {
-			ee_chksum =
-			    *(u16 *)((char *)peeprom + (eecodesize - 2));
-			/*
-			 *  calculate the EEPROM checksum
-			 */
-			calc_chksum = slic_eeprom_cksum(peeprom,
-							eecodesize - 2);
-			/*
-			 *  if the ucdoe chksum flag bit worked,
-			 *  we wouldn't need this
-			 */
-			if (ee_chksum == calc_chksum)
-				card->config.EepromValid = true;
-		}
-		/*  copy in the DRAM size */
-		card->config.DramSize = dramsize;
-
-		/*  copy in the MAC address(es) */
-		for (i = 0; i < macaddrs; i++) {
-			memcpy(&card->config.MacInfo[i],
-			       &pmac[i], sizeof(struct slic_config_mac));
-		}
-
-		/*  copy the Alacritech FRU information */
-		card->config.FruFormat = fruformat;
-		memcpy(&card->config.AtkFru, patkfru,
-						sizeof(struct atk_fru));
-
-		pci_free_consistent(adapter->pcidev,
-				    sizeof(struct slic_eeprom),
-				    peeprom, phys_config);
-
-		if (!card->config.EepromValid) {
-			slic_write64(adapter, SLIC_REG_ISP, 0, 0);
-			slic_flush_write(adapter);
-			dev_err(&adapter->pcidev->dev, "EEPROM invalid.\n");
-			return -EINVAL;
-		}
-
-		card->config_set = 1;
-	}
-
-	status = slic_card_download_gbrcv(adapter);
-	if (status)
-		return status;
-
-	if (slic_global.dynamic_intagg)
-		slic_intagg_set(adapter, 0);
-	else
-		slic_intagg_set(adapter, adapter->intagg_delay);
-
-	/*
-	 *  Initialize ping status to "ok"
-	 */
-	card->pingstatus = ISR_PINGMASK;
-
-	/*
-	 * Lastly, mark our card state as up and return success
-	 */
-	card->state = CARD_UP;
-	card->reset_in_progress = 0;
-
-	return 0;
-
-card_init_err:
-	pci_free_consistent(adapter->pcidev, sizeof(struct slic_eeprom),
-			    peeprom, phys_config);
-	return status;
-}
-
-static int slic_get_coalesce(struct net_device *dev,
-			     struct ethtool_coalesce *coalesce)
-{
-	struct adapter *adapter = netdev_priv(dev);
-
-	adapter->intagg_delay = coalesce->rx_coalesce_usecs;
-	adapter->dynamic_intagg = coalesce->use_adaptive_rx_coalesce;
-	return 0;
-}
-
-static int slic_set_coalesce(struct net_device *dev,
-			     struct ethtool_coalesce *coalesce)
-{
-	struct adapter *adapter = netdev_priv(dev);
-
-	coalesce->rx_coalesce_usecs = adapter->intagg_delay;
-	coalesce->use_adaptive_rx_coalesce = adapter->dynamic_intagg;
-	return 0;
-}
-
-static void slic_init_driver(void)
-{
-	if (slic_first_init) {
-		slic_first_init = 0;
-		spin_lock_init(&slic_global.driver_lock);
-	}
-}
-
-static int slic_init_adapter(struct net_device *netdev,
-			     struct pci_dev *pcidev,
-			     const struct pci_device_id *pci_tbl_entry,
-			     void __iomem *memaddr, int chip_idx)
-{
-	ushort index;
-	struct slic_handle *pslic_handle;
-	struct adapter *adapter = netdev_priv(netdev);
-	struct slic_shmemory *sm = &adapter->shmem;
-	struct slic_shmem_data *sm_data;
-	dma_addr_t phaddr;
-
-/*	adapter->pcidev = pcidev;*/
-	adapter->vendid = pci_tbl_entry->vendor;
-	adapter->devid = pci_tbl_entry->device;
-	adapter->subsysid = pci_tbl_entry->subdevice;
-	adapter->busnumber = pcidev->bus->number;
-	adapter->slotnumber = ((pcidev->devfn >> 3) & 0x1F);
-	adapter->functionnumber = (pcidev->devfn & 0x7);
-	adapter->regs = memaddr;
-	adapter->irq = pcidev->irq;
-	adapter->chipid = chip_idx;
-	adapter->port = 0;
-	adapter->cardindex = adapter->port;
-	spin_lock_init(&adapter->upr_lock);
-	spin_lock_init(&adapter->bit64reglock);
-	spin_lock_init(&adapter->adapter_lock);
-	spin_lock_init(&adapter->reset_lock);
-	spin_lock_init(&adapter->handle_lock);
-
-	adapter->card_size = 1;
-	/*
-	 * Initialize slic_handle array
-	 */
-	/*
-	 * Start with 1.  0 is an invalid host handle.
-	 */
-	for (index = 1, pslic_handle = &adapter->slic_handles[1];
-	     index < SLIC_CMDQ_MAXCMDS; index++, pslic_handle++) {
-		pslic_handle->token.handle_index = index;
-		pslic_handle->type = SLIC_HANDLE_FREE;
-		pslic_handle->next = adapter->pfree_slic_handles;
-		adapter->pfree_slic_handles = pslic_handle;
-	}
-	sm_data = pci_zalloc_consistent(adapter->pcidev, sizeof(*sm_data),
-					&phaddr);
-	if (!sm_data)
-		return -ENOMEM;
-
-	sm->shmem_data = sm_data;
-	sm->isr_phaddr = phaddr;
-	sm->lnkstatus_phaddr = phaddr + offsetof(struct slic_shmem_data,
-						 lnkstatus);
-	sm->stats_phaddr = phaddr + offsetof(struct slic_shmem_data, stats);
-
-	return 0;
-}
-
-static const struct net_device_ops slic_netdev_ops = {
-	.ndo_open		= slic_entry_open,
-	.ndo_stop		= slic_entry_halt,
-	.ndo_start_xmit		= slic_xmit_start,
-	.ndo_do_ioctl		= slic_ioctl,
-	.ndo_set_mac_address	= slic_mac_set_address,
-	.ndo_get_stats		= slic_get_stats,
-	.ndo_set_rx_mode	= slic_mcast_set_list,
-	.ndo_validate_addr	= eth_validate_addr,
-};
-
-static u32 slic_card_locate(struct adapter *adapter)
-{
-	struct sliccard *card = slic_global.slic_card;
-	struct physcard *physcard = slic_global.phys_card;
-	ushort card_hostid;
-	uint i;
-
-	card_hostid = slic_read32(adapter, SLIC_REG_HOSTID);
-
-	/* Initialize a new card structure if need be */
-	if (card_hostid == SLIC_HOSTID_DEFAULT) {
-		card = kzalloc(sizeof(*card), GFP_KERNEL);
-		if (!card)
-			return -ENOMEM;
-
-		card->next = slic_global.slic_card;
-		slic_global.slic_card = card;
-		card->busnumber = adapter->busnumber;
-		card->slotnumber = adapter->slotnumber;
-
-		/* Find an available cardnum */
-		for (i = 0; i < SLIC_MAX_CARDS; i++) {
-			if (slic_global.cardnuminuse[i] == 0) {
-				slic_global.cardnuminuse[i] = 1;
-				card->cardnum = i;
-				break;
-			}
-		}
-		slic_global.num_slic_cards++;
-	} else {
-		/* Card exists, find the card this adapter belongs to */
-		while (card) {
-			if (card->cardnum == card_hostid)
-				break;
-			card = card->next;
-		}
-	}
-
-	if (!card)
-		return -ENXIO;
-	/* Put the adapter in the card's adapter list */
-	if (!card->adapter[adapter->port]) {
-		card->adapter[adapter->port] = adapter;
-		adapter->card = card;
-	}
-
-	card->card_size = 1;	/* one port per *logical* card */
-
-	while (physcard) {
-		for (i = 0; i < SLIC_MAX_PORTS; i++) {
-			if (physcard->adapter[i])
-				break;
-		}
-		if (i == SLIC_MAX_PORTS)
-			break;
-
-		if (physcard->adapter[i]->slotnumber == adapter->slotnumber)
-			break;
-		physcard = physcard->next;
-	}
-	if (!physcard) {
-		/* no structure allocated for this physical card yet */
-		physcard = kzalloc(sizeof(*physcard), GFP_ATOMIC);
-		if (!physcard) {
-			if (card_hostid == SLIC_HOSTID_DEFAULT)
-				kfree(card);
-			return -ENOMEM;
-		}
-
-		physcard->next = slic_global.phys_card;
-		slic_global.phys_card = physcard;
-		physcard->adapters_allocd = 1;
-	} else {
-		physcard->adapters_allocd++;
-	}
-	/* Note - this is ZERO relative */
-	adapter->physport = physcard->adapters_allocd - 1;
-
-	physcard->adapter[adapter->physport] = adapter;
-	adapter->physcard = physcard;
-
-	return 0;
-}
-
-static int slic_entry_probe(struct pci_dev *pcidev,
-			       const struct pci_device_id *pci_tbl_entry)
-{
-	static int cards_found;
-	static int did_version;
-	int err = -ENODEV;
-	struct net_device *netdev;
-	struct adapter *adapter;
-	void __iomem *memmapped_ioaddr = NULL;
-	ulong mmio_start = 0;
-	ulong mmio_len = 0;
-	struct sliccard *card = NULL;
-	int pci_using_dac = 0;
-
-	err = pci_enable_device(pcidev);
-
-	if (err)
-		return err;
-
-	if (did_version++ == 0) {
-		dev_info(&pcidev->dev, "%s\n", slic_banner);
-		dev_info(&pcidev->dev, "%s\n", slic_proc_version);
-	}
-
-	if (!pci_set_dma_mask(pcidev, DMA_BIT_MASK(64))) {
-		pci_using_dac = 1;
-		err = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64));
-		if (err) {
-			dev_err(&pcidev->dev, "unable to obtain 64-bit DMA for consistent allocations\n");
-			goto err_out_disable_pci;
-		}
-	} else {
-		err = pci_set_dma_mask(pcidev, DMA_BIT_MASK(32));
-		if (err) {
-			dev_err(&pcidev->dev, "no usable DMA configuration\n");
-			goto err_out_disable_pci;
-		}
-		pci_using_dac = 0;
-		pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
-	}
-
-	err = pci_request_regions(pcidev, DRV_NAME);
-	if (err) {
-		dev_err(&pcidev->dev, "can't obtain PCI resources\n");
-		goto err_out_disable_pci;
-	}
-
-	pci_set_master(pcidev);
-
-	netdev = alloc_etherdev(sizeof(struct adapter));
-	if (!netdev) {
-		err = -ENOMEM;
-		goto err_out_exit_slic_probe;
-	}
-
-	netdev->ethtool_ops = &slic_ethtool_ops;
-	SET_NETDEV_DEV(netdev, &pcidev->dev);
-
-	pci_set_drvdata(pcidev, netdev);
-	adapter = netdev_priv(netdev);
-	adapter->netdev = netdev;
-	adapter->pcidev = pcidev;
-	slic_global.dynamic_intagg = adapter->dynamic_intagg;
-	if (pci_using_dac)
-		netdev->features |= NETIF_F_HIGHDMA;
-
-	mmio_start = pci_resource_start(pcidev, 0);
-	mmio_len = pci_resource_len(pcidev, 0);
-
-	memmapped_ioaddr = ioremap_nocache(mmio_start, mmio_len);
-	if (!memmapped_ioaddr) {
-		dev_err(&pcidev->dev, "cannot remap MMIO region %lx @ %lx\n",
-			mmio_len, mmio_start);
-		err = -ENOMEM;
-		goto err_out_free_netdev;
-	}
-
-	slic_config_pci(pcidev);
-
-	slic_init_driver();
-
-	err = slic_init_adapter(netdev, pcidev, pci_tbl_entry, memmapped_ioaddr,
-				cards_found);
-	if (err) {
-		dev_err(&pcidev->dev, "failed to init adapter: %i\n", err);
-		goto err_out_unmap;
-	}
-
-	err = slic_card_locate(adapter);
-	if (err) {
-		dev_err(&pcidev->dev, "cannot locate card\n");
-		goto err_clean_init;
-	}
-
-	card = adapter->card;
-
-	if (!adapter->allocated) {
-		card->adapters_allocated++;
-		adapter->allocated = 1;
-	}
-
-	err = slic_card_init(card, adapter);
-	if (err)
-		goto err_clean_init;
-
-	slic_adapter_set_hwaddr(adapter);
-
-	netdev->base_addr = (unsigned long)memmapped_ioaddr;
-	netdev->irq = adapter->irq;
-	netdev->netdev_ops = &slic_netdev_ops;
-
-	netif_carrier_off(netdev);
-
-	strcpy(netdev->name, "eth%d");
-	err = register_netdev(netdev);
-	if (err) {
-		dev_err(&pcidev->dev, "Cannot register net device, aborting.\n");
-		goto err_clean_init;
-	}
-
-	cards_found++;
-
-	return 0;
-
-err_clean_init:
-	slic_init_cleanup(adapter);
-err_out_unmap:
-	iounmap(memmapped_ioaddr);
-err_out_free_netdev:
-	free_netdev(netdev);
-err_out_exit_slic_probe:
-	pci_release_regions(pcidev);
-err_out_disable_pci:
-	pci_disable_device(pcidev);
-	return err;
-}
-
-static struct pci_driver slic_driver = {
-	.name = DRV_NAME,
-	.id_table = slic_pci_tbl,
-	.probe = slic_entry_probe,
-	.remove = slic_entry_remove,
-};
-
-static int __init slic_module_init(void)
-{
-	slic_init_driver();
-
-	return pci_register_driver(&slic_driver);
-}
-
-static void __exit slic_module_cleanup(void)
-{
-	pci_unregister_driver(&slic_driver);
-}
-
-static const struct ethtool_ops slic_ethtool_ops = {
-	.get_coalesce = slic_get_coalesce,
-	.set_coalesce = slic_set_coalesce
-};
-
-module_init(slic_module_init);
-module_exit(slic_module_cleanup);
diff --git a/drivers/staging/sm750fb/Makefile b/drivers/staging/sm750fb/Makefile
index dcce3f487ed5..4d781f78b95c 100644
--- a/drivers/staging/sm750fb/Makefile
+++ b/drivers/staging/sm750fb/Makefile
@@ -1,4 +1,4 @@
 obj-$(CONFIG_FB_SM750)	+= sm750fb.o
 
 sm750fb-objs		:= sm750.o sm750_hw.o sm750_accel.o sm750_cursor.o ddk750_chip.o ddk750_power.o ddk750_mode.o
-sm750fb-objs		+= ddk750_display.o ddk750_help.o ddk750_swi2c.o ddk750_sii164.o ddk750_dvi.o ddk750_hwi2c.o
+sm750fb-objs		+= ddk750_display.o ddk750_swi2c.o ddk750_sii164.o ddk750_dvi.o ddk750_hwi2c.o
diff --git a/drivers/staging/sm750fb/ddk750.h b/drivers/staging/sm750fb/ddk750.h
index 2c10a08ed964..734010324a8f 100644
--- a/drivers/staging/sm750fb/ddk750.h
+++ b/drivers/staging/sm750fb/ddk750.h
@@ -1,22 +1,21 @@
+/*
+ *         Copyright (c) 2007 by Silicon Motion, Inc. (SMI)
+ *
+ *  All rights are reserved. Reproduction or in part is prohibited
+ *  without the written consent of the copyright owner.
+ *
+ *  RegSC.h --- SM718 SDK
+ *  This file contains the definitions for the System Configuration registers.
+ */
+
 #ifndef DDK750_H__
 #define DDK750_H__
-/*******************************************************************
-*
-*         Copyright (c) 2007 by Silicon Motion, Inc. (SMI)
-*
-*  All rights are reserved. Reproduction or in part is prohibited
-*  without the written consent of the copyright owner.
-*
-*  RegSC.h --- SM718 SDK
-*  This file contains the definitions for the System Configuration registers.
-*
-*******************************************************************/
+
 #include "ddk750_reg.h"
 #include "ddk750_mode.h"
 #include "ddk750_chip.h"
 #include "ddk750_display.h"
 #include "ddk750_power.h"
-#include "ddk750_help.h"
 #ifdef USE_HW_I2C
 #include "ddk750_hwi2c.h"
 #endif
diff --git a/drivers/staging/sm750fb/ddk750_chip.c b/drivers/staging/sm750fb/ddk750_chip.c
index 839d6730bde9..f59ce5c0867d 100644
--- a/drivers/staging/sm750fb/ddk750_chip.c
+++ b/drivers/staging/sm750fb/ddk750_chip.c
@@ -1,33 +1,32 @@
 #include <linux/kernel.h>
 #include <linux/sizes.h>
 
-#include "ddk750_help.h"
 #include "ddk750_reg.h"
 #include "ddk750_chip.h"
 #include "ddk750_power.h"
 
 #define MHz(x) ((x) * 1000000)
 
+static logical_chip_type_t chip;
+
 logical_chip_type_t sm750_get_chip_type(void)
 {
-	unsigned short physicalID;
-	char physicalRev;
-	logical_chip_type_t chip;
-
-	physicalID = devId750; /* either 0x718 or 0x750 */
-	physicalRev = revId750;
+	return chip;
+}
 
-	if (physicalID == 0x718)
+void sm750_set_chip_type(unsigned short devId, u8 revId)
+{
+	if (devId == 0x718)
 		chip = SM718;
-	else if (physicalID == 0x750) {
+	else if (devId == 0x750) {
 		chip = SM750;
 		/* SM750 and SM750LE are different in their revision ID only. */
-		if (physicalRev == SM750LE_REVISION_ID)
+		if (revId == SM750LE_REVISION_ID) {
 			chip = SM750LE;
+			pr_info("found sm750le\n");
+		}
 	} else
 		chip = SM_UNKNOWN;
-
-	return chip;
 }
 
 static unsigned int get_mxclk_freq(void)
@@ -52,9 +51,9 @@ static unsigned int get_mxclk_freq(void)
  *
  * Input: Frequency to be set.
  */
-static void setChipClock(unsigned int frequency)
+static void set_chip_clock(unsigned int frequency)
 {
-	pll_value_t pll;
+	struct pll_value pll;
 	unsigned int ulActualMxClk;
 
 	/* Cheok_0509: For SM750LE, the chip clock is fixed. Nothing to set. */
@@ -63,29 +62,31 @@ static void setChipClock(unsigned int frequency)
 
 	if (frequency) {
 		/*
-		* Set up PLL, a structure to hold the value to be set in clocks.
-		*/
+		 * Set up PLL structure to hold the value to be set in clocks.
+		 */
 		pll.inputFreq = DEFAULT_INPUT_CLOCK; /* Defined in CLOCK.H */
 		pll.clockType = MXCLK_PLL;
 
 		/*
-		* Call calcPllValue() to fill the other fields of PLL structure.
-		* Sometime, the chip cannot set up the exact clock
-		* required by the User.
-		* Return value of calcPllValue gives the actual possible clock.
-		*/
-		ulActualMxClk = calcPllValue(frequency, &pll);
+		 * Call sm750_calc_pll_value() to fill the other fields of the PLL
+		 * structure. Sometimes, the chip cannot set up the exact
+		 * clock required by the User.
+		 * Return value of sm750_calc_pll_value gives the actual possible
+		 * clock.
+		 */
+		ulActualMxClk = sm750_calc_pll_value(frequency, &pll);
 
 		/* Master Clock Control: MXCLK_PLL */
-		POKE32(MXCLK_PLL_CTRL, formatPllReg(&pll));
+		POKE32(MXCLK_PLL_CTRL, sm750_format_pll_reg(&pll));
 	}
 }
 
-static void setMemoryClock(unsigned int frequency)
+static void set_memory_clock(unsigned int frequency)
 {
 	unsigned int reg, divisor;
 
-	/* Cheok_0509: For SM750LE, the memory clock is fixed.
+	/*
+	 * Cheok_0509: For SM750LE, the memory clock is fixed.
 	 * Nothing to set.
 	 */
 	if (sm750_get_chip_type() == SM750LE)
@@ -120,7 +121,7 @@ static void setMemoryClock(unsigned int frequency)
 			break;
 		}
 
-		setCurrentGate(reg);
+		sm750_set_current_gate(reg);
 	}
 }
 
@@ -132,18 +133,20 @@ static void setMemoryClock(unsigned int frequency)
  * NOTE:
  *      The maximum frequency the engine can run is 168MHz.
  */
-static void setMasterClock(unsigned int frequency)
+static void set_master_clock(unsigned int frequency)
 {
 	unsigned int reg, divisor;
 
-	/* Cheok_0509: For SM750LE, the memory clock is fixed.
+	/*
+	 * Cheok_0509: For SM750LE, the memory clock is fixed.
 	 * Nothing to set.
 	 */
 	if (sm750_get_chip_type() == SM750LE)
 		return;
 
 	if (frequency) {
-		/* Set the frequency to the maximum frequency
+		/*
+		 * Set the frequency to the maximum frequency
 		 * that the SM750 engine can run, which is about 190 MHz.
 		 */
 		if (frequency > MHz(190))
@@ -170,11 +173,11 @@ static void setMasterClock(unsigned int frequency)
 			break;
 		}
 
-		setCurrentGate(reg);
+		sm750_set_current_gate(reg);
 		}
 }
 
-unsigned int ddk750_getVMSize(void)
+unsigned int ddk750_get_vm_size(void)
 {
 	unsigned int reg;
 	unsigned int data;
@@ -206,18 +209,18 @@ unsigned int ddk750_getVMSize(void)
 	return data;
 }
 
-int ddk750_initHw(initchip_param_t *pInitParam)
+int ddk750_init_hw(struct initchip_param *pInitParam)
 {
 	unsigned int reg;
 
 	if (pInitParam->powerMode != 0)
 		pInitParam->powerMode = 0;
-	setPowerMode(pInitParam->powerMode);
+	sm750_set_power_mode(pInitParam->powerMode);
 
 	/* Enable display power gate & LOCALMEM power gate*/
 	reg = PEEK32(CURRENT_GATE);
 	reg |= (CURRENT_GATE_DISPLAY | CURRENT_GATE_LOCALMEM);
-	setCurrentGate(reg);
+	sm750_set_current_gate(reg);
 
 	if (sm750_get_chip_type() != SM750LE) {
 		/*	set panel pll and graphic mode via mmio_88 */
@@ -233,16 +236,17 @@ int ddk750_initHw(initchip_param_t *pInitParam)
 	}
 
 	/* Set the Main Chip Clock */
-	setChipClock(MHz((unsigned int)pInitParam->chipClock));
+	set_chip_clock(MHz((unsigned int)pInitParam->chipClock));
 
 	/* Set up memory clock. */
-	setMemoryClock(MHz(pInitParam->memClock));
+	set_memory_clock(MHz(pInitParam->memClock));
 
 	/* Set up master clock */
-	setMasterClock(MHz(pInitParam->masterClock));
+	set_master_clock(MHz(pInitParam->masterClock));
 
 
-	/* Reset the memory controller.
+	/*
+	 * Reset the memory controller.
 	 * If the memory controller is not reset in SM750,
 	 * the system might hang when sw accesses the memory.
 	 * The memory should be resetted after changing the MXCLK.
@@ -257,7 +261,7 @@ int ddk750_initHw(initchip_param_t *pInitParam)
 	}
 
 	if (pInitParam->setAllEngOff == 1) {
-		enable2DEngine(0);
+		sm750_enable_2d_engine(0);
 
 		/* Disable Overlay, if a former application left it on */
 		reg = PEEK32(VIDEO_DISPLAY_CTRL);
@@ -280,7 +284,7 @@ int ddk750_initHw(initchip_param_t *pInitParam)
 		POKE32(DMA_ABORT_INTERRUPT, reg);
 
 		/* Disable DMA Power, if a former application left it on */
-		enableDMA(0);
+		sm750_enable_dma(0);
 	}
 
 	/* We can add more initialization as needed. */
@@ -305,9 +309,10 @@ int ddk750_initHw(initchip_param_t *pInitParam)
  * M = {1,...,255}
  * N = {2,...,15}
  */
-unsigned int calcPllValue(unsigned int request_orig, pll_value_t *pll)
+unsigned int sm750_calc_pll_value(unsigned int request_orig, struct pll_value *pll)
 {
-	/* as sm750 register definition,
+	/*
+	 * as sm750 register definition,
 	 * N located in 2,15 and M located in 1,255
 	 */
 	int N, M, X, d;
@@ -319,7 +324,8 @@ unsigned int calcPllValue(unsigned int request_orig, pll_value_t *pll)
 	int max_d = 6;
 
 	if (sm750_get_chip_type() == SM750LE) {
-		/* SM750LE don't have
+		/*
+		 * SM750LE don't have
 		 * programmable PLL and M/N values to work on.
 		 * Just return the requested clock.
 		 */
@@ -331,14 +337,16 @@ unsigned int calcPllValue(unsigned int request_orig, pll_value_t *pll)
 	request = request_orig / 1000;
 	input = pll->inputFreq / 1000;
 
-	/* for MXCLK register,
+	/*
+	 * for MXCLK register,
 	 * no POD provided, so need be treated differently
 	 */
 	if (pll->clockType == MXCLK_PLL)
 		max_d = 3;
 
 	for (N = 15; N > 1; N--) {
-		/* RN will not exceed maximum long
+		/*
+		 * RN will not exceed maximum long
 		 * if @request <= 285 MHZ (for 32bit cpu)
 		 */
 		RN = N * request;
@@ -373,7 +381,7 @@ unsigned int calcPllValue(unsigned int request_orig, pll_value_t *pll)
 	return ret;
 }
 
-unsigned int formatPllReg(pll_value_t *pPLL)
+unsigned int sm750_format_pll_reg(struct pll_value *pPLL)
 {
 #ifndef VALIDATION_CHIP
 	unsigned int POD = pPLL->POD;
diff --git a/drivers/staging/sm750fb/ddk750_chip.h b/drivers/staging/sm750fb/ddk750_chip.h
index 14357fd1cc6b..e63b8b293816 100644
--- a/drivers/staging/sm750fb/ddk750_chip.h
+++ b/drivers/staging/sm750fb/ddk750_chip.h
@@ -6,6 +6,14 @@
 #endif
 
 #include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/uaccess.h>
+
+/* software control endianness */
+#define PEEK32(addr) readl(addr + mmio750)
+#define POKE32(addr, data) writel(data, addr + mmio750)
+
+extern void __iomem *mmio750;
 
 /* This is all the chips recognized by this library */
 typedef enum _logical_chip_type_t {
@@ -25,7 +33,7 @@ typedef enum _clock_type_t {
 }
 clock_type_t;
 
-typedef struct _pll_value_t {
+struct pll_value {
 	clock_type_t clockType;
 	unsigned long inputFreq; /* Input clock frequency to the PLL */
 
@@ -34,46 +42,55 @@ typedef struct _pll_value_t {
 	unsigned long N;
 	unsigned long OD;
 	unsigned long POD;
-}
-pll_value_t;
+};
 
 /* input struct to initChipParam() function */
-typedef struct _initchip_param_t {
-	unsigned short powerMode;    /* Use power mode 0 or 1 */
-	unsigned short chipClock;    /**
-				      * Speed of main chip clock in MHz unit
-				      * 0 = keep the current clock setting
-				      * Others = the new main chip clock
-				      */
-	unsigned short memClock;     /**
-				      * Speed of memory clock in MHz unit
-				      * 0 = keep the current clock setting
-				      * Others = the new memory clock
-				      */
-	unsigned short masterClock;  /**
-				      * Speed of master clock in MHz unit
-				      * 0 = keep the current clock setting
-				      * Others = the new master clock
-				      */
-	unsigned short setAllEngOff; /**
-				      * 0 = leave all engine state untouched.
-				      * 1 = make sure they are off: 2D, Overlay,
-				      * video alpha, alpha, hardware cursors
-				      */
-	unsigned char resetMemory;   /**
-				      * 0 = Do not reset the memory controller
-				      * 1 = Reset the memory controller
-				      */
+struct initchip_param {
+	/* Use power mode 0 or 1 */
+	unsigned short powerMode;
+
+	/*
+	 * Speed of main chip clock in MHz unit
+	 * 0 = keep the current clock setting
+	 * Others = the new main chip clock
+	 */
+	unsigned short chipClock;
+
+	/*
+	 * Speed of memory clock in MHz unit
+	 * 0 = keep the current clock setting
+	 * Others = the new memory clock
+	 */
+	unsigned short memClock;
+
+	/*
+	 * Speed of master clock in MHz unit
+	 * 0 = keep the current clock setting
+	 * Others = the new master clock
+	 */
+	unsigned short masterClock;
+
+	/*
+	 * 0 = leave all engine state untouched.
+	 * 1 = make sure they are off: 2D, Overlay,
+	 * video alpha, alpha, hardware cursors
+	 */
+	unsigned short setAllEngOff;
+
+	/*
+	 * 0 = Do not reset the memory controller
+	 * 1 = Reset the memory controller
+	 */
+	unsigned char resetMemory;
 
 	/* More initialization parameter can be added if needed */
-}
-initchip_param_t;
+};
 
 logical_chip_type_t sm750_get_chip_type(void);
-unsigned int calcPllValue(unsigned int request, pll_value_t *pll);
-unsigned int formatPllReg(pll_value_t *pPLL);
-void ddk750_set_mmio(void __iomem *, unsigned short, char);
-unsigned int ddk750_getVMSize(void);
-int ddk750_initHw(initchip_param_t *);
+void sm750_set_chip_type(unsigned short devId, u8 revId);
+unsigned int sm750_calc_pll_value(unsigned int request, struct  pll_value *pll);
+unsigned int sm750_format_pll_reg(struct pll_value *pPLL);
+unsigned int ddk750_get_vm_size(void);
+int ddk750_init_hw(struct initchip_param *);
 
 #endif
diff --git a/drivers/staging/sm750fb/ddk750_display.c b/drivers/staging/sm750fb/ddk750_display.c
index 4023c476b9e4..c347803f7e19 100644
--- a/drivers/staging/sm750fb/ddk750_display.c
+++ b/drivers/staging/sm750fb/ddk750_display.c
@@ -1,11 +1,9 @@
 #include "ddk750_reg.h"
-#include "ddk750_help.h"
+#include "ddk750_chip.h"
 #include "ddk750_display.h"
 #include "ddk750_power.h"
 #include "ddk750_dvi.h"
 
-#define primaryWaitVerticalSync(delay) waitNextVerticalSync(0, delay)
-
 static void setDisplayControl(int ctrl, int disp_state)
 {
 	/* state != 0 means turn on both timing & plane en_bit */
@@ -61,55 +59,28 @@ static void setDisplayControl(int ctrl, int disp_state)
 	}
 }
 
-static void waitNextVerticalSync(int ctrl, int delay)
+static void primary_wait_vertical_sync(int delay)
 {
 	unsigned int status;
 
-	if (!ctrl) {
-		/* primary controller */
+	/*
+	 * Do not wait when the Primary PLL is off or display control is
+	 * already off. This will prevent the software to wait forever.
+	 */
+	if (!(PEEK32(PANEL_PLL_CTRL) & PLL_CTRL_POWER) ||
+	    !(PEEK32(PANEL_DISPLAY_CTRL) & DISPLAY_CTRL_TIMING))
+		return;
 
-		/*
-		 * Do not wait when the Primary PLL is off or display control is
-		 * already off. This will prevent the software to wait forever.
-		 */
-		if (!(PEEK32(PANEL_PLL_CTRL) & PLL_CTRL_POWER) ||
-		    !(PEEK32(PANEL_DISPLAY_CTRL) & DISPLAY_CTRL_TIMING)) {
-			return;
-		}
-
-		while (delay-- > 0) {
-			/* Wait for end of vsync. */
-			do {
-				status = PEEK32(SYSTEM_CTRL);
-			} while (status & SYSTEM_CTRL_PANEL_VSYNC_ACTIVE);
-
-			/* Wait for start of vsync. */
-			do {
-				status = PEEK32(SYSTEM_CTRL);
-			} while (!(status & SYSTEM_CTRL_PANEL_VSYNC_ACTIVE));
-		}
+	while (delay-- > 0) {
+		/* Wait for end of vsync. */
+		do {
+			status = PEEK32(SYSTEM_CTRL);
+		} while (status & SYSTEM_CTRL_PANEL_VSYNC_ACTIVE);
 
-	} else {
-		/*
-		 * Do not wait when the Primary PLL is off or display control is
-		 * already off. This will prevent the software to wait forever.
-		 */
-		if (!(PEEK32(CRT_PLL_CTRL) & PLL_CTRL_POWER) ||
-		    !(PEEK32(CRT_DISPLAY_CTRL) & DISPLAY_CTRL_TIMING)) {
-			return;
-		}
-
-		while (delay-- > 0) {
-			/* Wait for end of vsync. */
-			do {
-				status = PEEK32(SYSTEM_CTRL);
-			} while (status & SYSTEM_CTRL_PANEL_VSYNC_ACTIVE);
-
-			/* Wait for start of vsync. */
-			do {
-				status = PEEK32(SYSTEM_CTRL);
-			} while (!(status & SYSTEM_CTRL_PANEL_VSYNC_ACTIVE));
-		}
+		/* Wait for start of vsync. */
+		do {
+			status = PEEK32(SYSTEM_CTRL);
+		} while (!(status & SYSTEM_CTRL_PANEL_VSYNC_ACTIVE));
 	}
 }
 
@@ -121,22 +92,22 @@ static void swPanelPowerSequence(int disp, int delay)
 	reg = PEEK32(PANEL_DISPLAY_CTRL);
 	reg |= (disp ? PANEL_DISPLAY_CTRL_FPEN : 0);
 	POKE32(PANEL_DISPLAY_CTRL, reg);
-	primaryWaitVerticalSync(delay);
+	primary_wait_vertical_sync(delay);
 
 	reg = PEEK32(PANEL_DISPLAY_CTRL);
 	reg |= (disp ? PANEL_DISPLAY_CTRL_DATA : 0);
 	POKE32(PANEL_DISPLAY_CTRL, reg);
-	primaryWaitVerticalSync(delay);
+	primary_wait_vertical_sync(delay);
 
 	reg = PEEK32(PANEL_DISPLAY_CTRL);
 	reg |= (disp ? PANEL_DISPLAY_CTRL_VBIASEN : 0);
 	POKE32(PANEL_DISPLAY_CTRL, reg);
-	primaryWaitVerticalSync(delay);
+	primary_wait_vertical_sync(delay);
 
 	reg = PEEK32(PANEL_DISPLAY_CTRL);
 	reg |= (disp ? PANEL_DISPLAY_CTRL_FPEN : 0);
 	POKE32(PANEL_DISPLAY_CTRL, reg);
-	primaryWaitVerticalSync(delay);
+	primary_wait_vertical_sync(delay);
 }
 
 void ddk750_setLogicalDispOut(disp_output_t output)
@@ -182,5 +153,5 @@ void ddk750_setLogicalDispOut(disp_output_t output)
 		setDAC((output & DAC_MASK) >> DAC_OFFSET);
 
 	if (output & DPMS_USAGE)
-		ddk750_setDPMS((output & DPMS_MASK) >> DPMS_OFFSET);
+		ddk750_set_dpms((output & DPMS_MASK) >> DPMS_OFFSET);
 }
diff --git a/drivers/staging/sm750fb/ddk750_display.h b/drivers/staging/sm750fb/ddk750_display.h
index e3fde428c52b..8abca88f089e 100644
--- a/drivers/staging/sm750fb/ddk750_display.h
+++ b/drivers/staging/sm750fb/ddk750_display.h
@@ -1,7 +1,8 @@
 #ifndef DDK750_DISPLAY_H__
 #define DDK750_DISPLAY_H__
 
-/* panel path select
+/*
+ * panel path select
  *	80000[29:28]
  */
 
@@ -12,7 +13,8 @@
 #define PNL_2_SEC	((2 << PNL_2_OFFSET) | PNL_2_USAGE)
 
 
-/* primary timing & plane enable bit
+/*
+ * primary timing & plane enable bit
  *	1: 80000[8] & 80000[2] on
  *	0: both off
  */
@@ -23,7 +25,8 @@
 #define PRI_TP_OFF ((0x0 << PRI_TP_OFFSET) | PRI_TP_USAGE)
 
 
-/* panel sequency status
+/*
+ * panel sequency status
  *	80000[27:24]
  */
 #define PNL_SEQ_OFFSET 6
@@ -32,7 +35,8 @@
 #define PNL_SEQ_ON (BIT(PNL_SEQ_OFFSET) | PNL_SEQ_USAGE)
 #define PNL_SEQ_OFF ((0 << PNL_SEQ_OFFSET) | PNL_SEQ_USAGE)
 
-/* dual digital output
+/*
+ * dual digital output
  *	80000[19]
  */
 #define DUAL_TFT_OFFSET 8
@@ -41,7 +45,8 @@
 #define DUAL_TFT_ON (BIT(DUAL_TFT_OFFSET) | DUAL_TFT_USAGE)
 #define DUAL_TFT_OFF ((0 << DUAL_TFT_OFFSET) | DUAL_TFT_USAGE)
 
-/* secondary timing & plane enable bit
+/*
+ * secondary timing & plane enable bit
  *	1:80200[8] & 80200[2] on
  *	0: both off
  */
@@ -51,7 +56,8 @@
 #define SEC_TP_ON  ((0x1 << SEC_TP_OFFSET) | SEC_TP_USAGE)
 #define SEC_TP_OFF ((0x0 << SEC_TP_OFFSET) | SEC_TP_USAGE)
 
-/* crt path select
+/*
+ * crt path select
  *	80200[19:18]
  */
 #define CRT_2_OFFSET 2
@@ -61,7 +67,8 @@
 #define CRT_2_SEC ((0x2 << CRT_2_OFFSET) | CRT_2_USAGE)
 
 
-/* DAC affect both DVI and DSUB
+/*
+ * DAC affect both DVI and DSUB
  *	4[20]
  */
 #define DAC_OFFSET 7
@@ -70,7 +77,8 @@
 #define DAC_ON ((0x0 << DAC_OFFSET) | DAC_USAGE)
 #define DAC_OFF ((0x1 << DAC_OFFSET) | DAC_USAGE)
 
-/* DPMS only affect D-SUB head
+/*
+ * DPMS only affect D-SUB head
  *	0[31:30]
  */
 #define DPMS_OFFSET 9
@@ -81,7 +89,8 @@
 
 
 
-/* LCD1 means panel path TFT1  & panel path DVI (so enable DAC)
+/*
+ * LCD1 means panel path TFT1  & panel path DVI (so enable DAC)
  * CRT means crt path DSUB
  */
 typedef enum _disp_output_t {
@@ -89,7 +98,8 @@ typedef enum _disp_output_t {
 	do_LCD1_SEC = PNL_2_SEC | SEC_TP_ON | PNL_SEQ_ON | DAC_ON,
 	do_LCD2_PRI = CRT_2_PRI | PRI_TP_ON | DUAL_TFT_ON,
 	do_LCD2_SEC = CRT_2_SEC | SEC_TP_ON | DUAL_TFT_ON,
-	/* do_DSUB_PRI = CRT_2_PRI | PRI_TP_ON | DPMS_ON|DAC_ON,
+	/*
+	 * do_DSUB_PRI = CRT_2_PRI | PRI_TP_ON | DPMS_ON|DAC_ON,
 	 * do_DSUB_SEC = CRT_2_SEC | SEC_TP_ON | DPMS_ON|DAC_ON,
 	 */
 	do_CRT_PRI = CRT_2_PRI | PRI_TP_ON | DPMS_ON | DAC_ON,
diff --git a/drivers/staging/sm750fb/ddk750_dvi.c b/drivers/staging/sm750fb/ddk750_dvi.c
index 8252f771ef9e..250c2f478778 100644
--- a/drivers/staging/sm750fb/ddk750_dvi.c
+++ b/drivers/staging/sm750fb/ddk750_dvi.c
@@ -1,6 +1,6 @@
 #define USE_DVICHIP
 #ifdef USE_DVICHIP
-#include "ddk750_help.h"
+#include "ddk750_chip.h"
 #include "ddk750_reg.h"
 #include "ddk750_dvi.h"
 #include "ddk750_sii164.h"
diff --git a/drivers/staging/sm750fb/ddk750_help.c b/drivers/staging/sm750fb/ddk750_help.c
deleted file mode 100644
index 9637dd30d037..000000000000
--- a/drivers/staging/sm750fb/ddk750_help.c
+++ /dev/null
@@ -1,17 +0,0 @@
-#include "ddk750_help.h"
-
-void __iomem *mmio750;
-char revId750;
-unsigned short devId750;
-
-/* after driver mapped io registers, use this function first */
-void ddk750_set_mmio(void __iomem *addr, unsigned short devId, char revId)
-{
-	mmio750 = addr;
-	devId750 = devId;
-	revId750 = revId;
-	if (revId == 0xfe)
-		printk("found sm750le\n");
-}
-
-
diff --git a/drivers/staging/sm750fb/ddk750_help.h b/drivers/staging/sm750fb/ddk750_help.h
deleted file mode 100644
index 009db9213a73..000000000000
--- a/drivers/staging/sm750fb/ddk750_help.h
+++ /dev/null
@@ -1,21 +0,0 @@
-#ifndef DDK750_HELP_H__
-#define DDK750_HELP_H__
-#include "ddk750_chip.h"
-#ifndef USE_INTERNAL_REGISTER_ACCESS
-
-#include <linux/ioport.h>
-#include <linux/io.h>
-#include <linux/uaccess.h>
-
-/* software control endianness */
-#define PEEK32(addr) readl(addr + mmio750)
-#define POKE32(addr, data) writel(data, addr + mmio750)
-
-extern void __iomem *mmio750;
-extern char revId750;
-extern unsigned short devId750;
-#else
-/* implement if you want use it*/
-#endif
-
-#endif
diff --git a/drivers/staging/sm750fb/ddk750_hwi2c.c b/drivers/staging/sm750fb/ddk750_hwi2c.c
index d391c127ead7..05d4a73aa1d4 100644
--- a/drivers/staging/sm750fb/ddk750_hwi2c.c
+++ b/drivers/staging/sm750fb/ddk750_hwi2c.c
@@ -1,6 +1,6 @@
 #define USE_HW_I2C
 #ifdef USE_HW_I2C
-#include "ddk750_help.h"
+#include "ddk750_chip.h"
 #include "ddk750_reg.h"
 #include "ddk750_hwi2c.h"
 #include "ddk750_power.h"
@@ -20,10 +20,11 @@ unsigned char bus_speed_mode
 	value |= (GPIO_MUX_30 | GPIO_MUX_31);
 	POKE32(GPIO_MUX, value);
 
-	/* Enable Hardware I2C power.
+	/*
+	 * Enable Hardware I2C power.
 	 * TODO: Check if we need to enable GPIO power?
 	 */
-	enableI2C(1);
+	sm750_enable_i2c(1);
 
 	/* Enable the I2C Controller and set the bus speed mode */
 	value = PEEK32(I2C_CTRL) & ~(I2C_CTRL_MODE | I2C_CTRL_EN);
@@ -44,7 +45,7 @@ void sm750_hw_i2c_close(void)
 	POKE32(I2C_CTRL, value);
 
 	/* Disable I2C Power */
-	enableI2C(0);
+	sm750_enable_i2c(0);
 
 	/* Set GPIO 30 & 31 back as GPIO pins */
 	value = PEEK32(GPIO_MUX);
@@ -92,7 +93,8 @@ static unsigned int hw_i2c_write_data(
 	/* Set the Device Address */
 	POKE32(I2C_SLAVE_ADDRESS, addr & ~0x01);
 
-	/* Write data.
+	/*
+	 * Write data.
 	 * Note:
 	 *      Only 16 byte can be accessed per i2c start instruction.
 	 */
@@ -158,7 +160,8 @@ static unsigned int hw_i2c_read_data(
 	/* Set the Device Address */
 	POKE32(I2C_SLAVE_ADDRESS, addr | 0x01);
 
-	/* Read data and save them to the buffer.
+	/*
+	 * Read data and save them to the buffer.
 	 * Note:
 	 *      Only 16 byte can be accessed per i2c start instruction.
 	 */
diff --git a/drivers/staging/sm750fb/ddk750_mode.c b/drivers/staging/sm750fb/ddk750_mode.c
index 05b83646c2d5..4a4b1de97a87 100644
--- a/drivers/staging/sm750fb/ddk750_mode.c
+++ b/drivers/staging/sm750fb/ddk750_mode.c
@@ -1,10 +1,10 @@
 
-#include "ddk750_help.h"
 #include "ddk750_reg.h"
 #include "ddk750_mode.h"
 #include "ddk750_chip.h"
 
-/* SM750LE only:
+/*
+ * SM750LE only:
  * This function takes care extra registers and bit fields required to set
  * up a mode in SM750LE
  *
@@ -19,7 +19,8 @@ static unsigned long displayControlAdjust_SM750LE(mode_parameter_t *pModeParam,
 	x = pModeParam->horizontal_display_end;
 	y = pModeParam->vertical_display_end;
 
-	/* SM750LE has to set up the top-left and bottom-right
+	/*
+	 * SM750LE has to set up the top-left and bottom-right
 	 * registers as well.
 	 * Note that normal SM750/SM718 only use those two register for
 	 * auto-centering mode.
@@ -31,7 +32,8 @@ static unsigned long displayControlAdjust_SM750LE(mode_parameter_t *pModeParam,
 			CRT_AUTO_CENTERING_BR_BOTTOM_MASK) |
 		((x - 1) & CRT_AUTO_CENTERING_BR_RIGHT_MASK));
 
-	/* Assume common fields in dispControl have been properly set before
+	/*
+	 * Assume common fields in dispControl have been properly set before
 	 * calling this function.
 	 * This function only sets the extra fields in dispControl.
 	 */
@@ -72,7 +74,8 @@ static unsigned long displayControlAdjust_SM750LE(mode_parameter_t *pModeParam,
 
 
 /* only timing related registers will be  programed */
-static int programModeRegisters(mode_parameter_t *pModeParam, pll_value_t *pll)
+static int programModeRegisters(mode_parameter_t *pModeParam,
+						struct pll_value *pll)
 {
 	int ret = 0;
 	int cnt = 0;
@@ -80,7 +83,7 @@ static int programModeRegisters(mode_parameter_t *pModeParam, pll_value_t *pll)
 
 	if (pll->clockType == SECONDARY_PLL) {
 		/* programe secondary pixel clock */
-		POKE32(CRT_PLL_CTRL, formatPllReg(pll));
+		POKE32(CRT_PLL_CTRL, sm750_format_pll_reg(pll));
 		POKE32(CRT_HORIZONTAL_TOTAL,
 			(((pModeParam->horizontal_total - 1) <<
 				CRT_HORIZONTAL_TOTAL_TOTAL_SHIFT) &
@@ -130,7 +133,7 @@ static int programModeRegisters(mode_parameter_t *pModeParam, pll_value_t *pll)
 	} else if (pll->clockType == PRIMARY_PLL) {
 		unsigned int reserved;
 
-		POKE32(PANEL_PLL_CTRL, formatPllReg(pll));
+		POKE32(PANEL_PLL_CTRL, sm750_format_pll_reg(pll));
 
 		reg = ((pModeParam->horizontal_total - 1) <<
 			PANEL_HORIZONTAL_TOTAL_TOTAL_SHIFT) &
@@ -176,14 +179,14 @@ static int programModeRegisters(mode_parameter_t *pModeParam, pll_value_t *pll)
 			  DISPLAY_CTRL_HSYNC_PHASE | DISPLAY_CTRL_TIMING |
 			  DISPLAY_CTRL_PLANE);
 
-		/* May a hardware bug or just my test chip (not confirmed).
-		* PANEL_DISPLAY_CTRL register seems requiring few writes
-		* before a value can be successfully written in.
-		* Added some masks to mask out the reserved bits.
-		* Note: This problem happens by design. The hardware will wait for the
-		*       next vertical sync to turn on/off the plane.
-		*/
-
+		/*
+		 * May a hardware bug or just my test chip (not confirmed).
+		 * PANEL_DISPLAY_CTRL register seems requiring few writes
+		 * before a value can be successfully written in.
+		 * Added some masks to mask out the reserved bits.
+		 * Note: This problem happens by design. The hardware will wait
+		 *       for the next vertical sync to turn on/off the plane.
+		 */
 		POKE32(PANEL_DISPLAY_CTRL, tmp | reg);
 
 		while ((PEEK32(PANEL_DISPLAY_CTRL) & ~reserved) !=
@@ -201,13 +204,13 @@ static int programModeRegisters(mode_parameter_t *pModeParam, pll_value_t *pll)
 
 int ddk750_setModeTiming(mode_parameter_t *parm, clock_type_t clock)
 {
-	pll_value_t pll;
+	struct pll_value pll;
 	unsigned int uiActualPixelClk;
 
 	pll.inputFreq = DEFAULT_INPUT_CLOCK;
 	pll.clockType = clock;
 
-	uiActualPixelClk = calcPllValue(parm->pixel_clock, &pll);
+	uiActualPixelClk = sm750_calc_pll_value(parm->pixel_clock, &pll);
 	if (sm750_get_chip_type() == SM750LE) {
 		/* set graphic mode via IO method */
 		outb_p(0x88, 0x3d4);
diff --git a/drivers/staging/sm750fb/ddk750_power.c b/drivers/staging/sm750fb/ddk750_power.c
index 7cc6169f884e..6167e30e8e01 100644
--- a/drivers/staging/sm750fb/ddk750_power.c
+++ b/drivers/staging/sm750fb/ddk750_power.c
@@ -1,8 +1,8 @@
-#include "ddk750_help.h"
+#include "ddk750_chip.h"
 #include "ddk750_reg.h"
 #include "ddk750_power.h"
 
-void ddk750_setDPMS(DPMS_t state)
+void ddk750_set_dpms(DPMS_t state)
 {
 	unsigned int value;
 
@@ -17,7 +17,7 @@ void ddk750_setDPMS(DPMS_t state)
 	}
 }
 
-static unsigned int getPowerMode(void)
+static unsigned int get_power_mode(void)
 {
 	if (sm750_get_chip_type() == SM750LE)
 		return 0;
@@ -29,26 +29,26 @@ static unsigned int getPowerMode(void)
  * SM50x can operate in one of three modes: 0, 1 or Sleep.
  * On hardware reset, power mode 0 is default.
  */
-void setPowerMode(unsigned int powerMode)
+void sm750_set_power_mode(unsigned int mode)
 {
-	unsigned int control_value = 0;
+	unsigned int ctrl = 0;
 
-	control_value = PEEK32(POWER_MODE_CTRL) & ~POWER_MODE_CTRL_MODE_MASK;
+	ctrl = PEEK32(POWER_MODE_CTRL) & ~POWER_MODE_CTRL_MODE_MASK;
 
 	if (sm750_get_chip_type() == SM750LE)
 		return;
 
-	switch (powerMode) {
+	switch (mode) {
 	case POWER_MODE_CTRL_MODE_MODE0:
-		control_value |= POWER_MODE_CTRL_MODE_MODE0;
+		ctrl |= POWER_MODE_CTRL_MODE_MODE0;
 		break;
 
 	case POWER_MODE_CTRL_MODE_MODE1:
-		control_value |= POWER_MODE_CTRL_MODE_MODE1;
+		ctrl |= POWER_MODE_CTRL_MODE_MODE1;
 		break;
 
 	case POWER_MODE_CTRL_MODE_SLEEP:
-		control_value |= POWER_MODE_CTRL_MODE_SLEEP;
+		ctrl |= POWER_MODE_CTRL_MODE_SLEEP;
 		break;
 
 	default:
@@ -56,44 +56,28 @@ void setPowerMode(unsigned int powerMode)
 	}
 
 	/* Set up other fields in Power Control Register */
-	if (powerMode == POWER_MODE_CTRL_MODE_SLEEP) {
-		control_value &= ~POWER_MODE_CTRL_OSC_INPUT;
+	if (mode == POWER_MODE_CTRL_MODE_SLEEP) {
+		ctrl &= ~POWER_MODE_CTRL_OSC_INPUT;
 #ifdef VALIDATION_CHIP
-		control_value &= ~POWER_MODE_CTRL_336CLK;
+		ctrl &= ~POWER_MODE_CTRL_336CLK;
 #endif
 	} else {
-		control_value |= POWER_MODE_CTRL_OSC_INPUT;
+		ctrl |= POWER_MODE_CTRL_OSC_INPUT;
 #ifdef VALIDATION_CHIP
-		control_value |= POWER_MODE_CTRL_336CLK;
+		ctrl |= POWER_MODE_CTRL_336CLK;
 #endif
 	}
 
 	/* Program new power mode. */
-	POKE32(POWER_MODE_CTRL, control_value);
+	POKE32(POWER_MODE_CTRL, ctrl);
 }
 
-void setCurrentGate(unsigned int gate)
+void sm750_set_current_gate(unsigned int gate)
 {
-	unsigned int gate_reg;
-	unsigned int mode;
-
-	/* Get current power mode. */
-	mode = getPowerMode();
-
-	switch (mode) {
-	case POWER_MODE_CTRL_MODE_MODE0:
-		gate_reg = MODE0_GATE;
-		break;
-
-	case POWER_MODE_CTRL_MODE_MODE1:
-		gate_reg = MODE1_GATE;
-		break;
-
-	default:
-		gate_reg = MODE0_GATE;
-		break;
-	}
-	POKE32(gate_reg, gate);
+	if (get_power_mode() == POWER_MODE_CTRL_MODE_MODE1)
+		POKE32(MODE1_GATE, gate);
+	else
+		POKE32(MODE0_GATE, gate);
 }
 
 
@@ -101,7 +85,7 @@ void setCurrentGate(unsigned int gate)
 /*
  * This function enable/disable the 2D engine.
  */
-void enable2DEngine(unsigned int enable)
+void sm750_enable_2d_engine(unsigned int enable)
 {
 	u32 gate;
 
@@ -111,10 +95,10 @@ void enable2DEngine(unsigned int enable)
 	else
 		gate &= ~(CURRENT_GATE_DE | CURRENT_GATE_CSC);
 
-	setCurrentGate(gate);
+	sm750_set_current_gate(gate);
 }
 
-void enableDMA(unsigned int enable)
+void sm750_enable_dma(unsigned int enable)
 {
 	u32 gate;
 
@@ -125,13 +109,13 @@ void enableDMA(unsigned int enable)
 	else
 		gate &= ~CURRENT_GATE_DMA;
 
-	setCurrentGate(gate);
+	sm750_set_current_gate(gate);
 }
 
 /*
  * This function enable/disable the GPIO Engine
  */
-void enableGPIO(unsigned int enable)
+void sm750_enable_gpio(unsigned int enable)
 {
 	u32 gate;
 
@@ -142,13 +126,13 @@ void enableGPIO(unsigned int enable)
 	else
 		gate &= ~CURRENT_GATE_GPIO;
 
-	setCurrentGate(gate);
+	sm750_set_current_gate(gate);
 }
 
 /*
  * This function enable/disable the I2C Engine
  */
-void enableI2C(unsigned int enable)
+void sm750_enable_i2c(unsigned int enable)
 {
 	u32 gate;
 
@@ -159,7 +143,7 @@ void enableI2C(unsigned int enable)
 	else
 		gate &= ~CURRENT_GATE_I2C;
 
-	setCurrentGate(gate);
+	sm750_set_current_gate(gate);
 }
 
 
diff --git a/drivers/staging/sm750fb/ddk750_power.h b/drivers/staging/sm750fb/ddk750_power.h
index 5963691f9a68..eb088b0d805f 100644
--- a/drivers/staging/sm750fb/ddk750_power.h
+++ b/drivers/staging/sm750fb/ddk750_power.h
@@ -14,37 +14,29 @@ DPMS_t;
 	       (PEEK32(MISC_CTRL) & ~MISC_CTRL_DAC_POWER_OFF) | (off)); \
 }
 
-void ddk750_setDPMS(DPMS_t);
-
-/*
- * This function sets the current power mode
- */
-void setPowerMode(unsigned int powerMode);
-
-/*
- * This function sets current gate
- */
-void setCurrentGate(unsigned int gate);
+void ddk750_set_dpms(DPMS_t);
+void sm750_set_power_mode(unsigned int powerMode);
+void sm750_set_current_gate(unsigned int gate);
 
 /*
  * This function enable/disable the 2D engine.
  */
-void enable2DEngine(unsigned int enable);
+void sm750_enable_2d_engine(unsigned int enable);
 
 /*
  * This function enable/disable the DMA Engine
  */
-void enableDMA(unsigned int enable);
+void sm750_enable_dma(unsigned int enable);
 
 /*
  * This function enable/disable the GPIO Engine
  */
-void enableGPIO(unsigned int enable);
+void sm750_enable_gpio(unsigned int enable);
 
 /*
  * This function enable/disable the I2C Engine
  */
-void enableI2C(unsigned int enable);
+void sm750_enable_i2c(unsigned int enable);
 
 
 #endif
diff --git a/drivers/staging/sm750fb/ddk750_sii164.c b/drivers/staging/sm750fb/ddk750_sii164.c
index 99a8683e6383..259006ace219 100644
--- a/drivers/staging/sm750fb/ddk750_sii164.c
+++ b/drivers/staging/sm750fb/ddk750_sii164.c
@@ -173,7 +173,8 @@ long sii164InitChip(
 
 		i2cWriteReg(SII164_I2C_ADDRESS, SII164_CONFIGURATION, config);
 
-		/* De-skew enabled with default 111b value.
+		/*
+		 * De-skew enabled with default 111b value.
 		 * This fixes some artifacts problem in some mode on board 2.2.
 		 * Somehow this fix does not affect board 2.1.
 		 */
diff --git a/drivers/staging/sm750fb/ddk750_swi2c.c b/drivers/staging/sm750fb/ddk750_swi2c.c
index 72a42330e7a1..b8a4e44359af 100644
--- a/drivers/staging/sm750fb/ddk750_swi2c.c
+++ b/drivers/staging/sm750fb/ddk750_swi2c.c
@@ -1,21 +1,20 @@
-/*******************************************************************
-*
-*         Copyright (c) 2007 by Silicon Motion, Inc. (SMI)
-*
-*  All rights are reserved. Reproduction or in part is prohibited
-*  without the written consent of the copyright owner.
-*
-*  swi2c.c --- SM750/SM718 DDK
-*  This file contains the source code for I2C using software
-*  implementation.
-*
-*******************************************************************/
-#include "ddk750_help.h"
+/*
+ *         Copyright (c) 2007 by Silicon Motion, Inc. (SMI)
+ *
+ *  All rights are reserved. Reproduction or in part is prohibited
+ *  without the written consent of the copyright owner.
+ *
+ *  swi2c.c --- SM750/SM718 DDK
+ *  This file contains the source code for I2C using software
+ *  implementation.
+ */
+
+#include "ddk750_chip.h"
 #include "ddk750_reg.h"
 #include "ddk750_swi2c.h"
 #include "ddk750_power.h"
 
-/*******************************************************************
+/*
  * I2C Software Master Driver:
  * ===========================
  * Each i2c cycle is split into 4 sections. Each of these section marks
@@ -51,7 +50,7 @@
  *                            SCL | L |   | H |   |
  *                 ---------------+---+---+---+---+
  *
- ******************************************************************/
+ */
 
 /* GPIO pins used for this I2C. It ranges from 0 to 63. */
 static unsigned char sw_i2c_clk_gpio = DEFAULT_I2C_SCL;
@@ -429,7 +428,7 @@ long sm750_sw_i2c_init(
 	       PEEK32(sw_i2c_data_gpio_mux_reg) & ~(1 << sw_i2c_data_gpio));
 
 	/* Enable GPIO power */
-	enableGPIO(1);
+	sm750_enable_gpio(1);
 
 	/* Clear the i2c lines. */
 	for (i = 0; i < 9; i++)
diff --git a/drivers/staging/sm750fb/ddk750_swi2c.h b/drivers/staging/sm750fb/ddk750_swi2c.h
index b53629cda095..5a9466efc7bd 100644
--- a/drivers/staging/sm750fb/ddk750_swi2c.h
+++ b/drivers/staging/sm750fb/ddk750_swi2c.h
@@ -1,15 +1,15 @@
-/*******************************************************************
-*
-*         Copyright (c) 2007 by Silicon Motion, Inc. (SMI)
-*
-*  All rights are reserved. Reproduction or in part is prohibited
-*  without the written consent of the copyright owner.
-*
-*  swi2c.h --- SM750/SM718 DDK
-*  This file contains the definitions for i2c using software
-*  implementation.
-*
-*******************************************************************/
+/*
+ *         Copyright (c) 2007 by Silicon Motion, Inc. (SMI)
+ *
+ *  All rights are reserved. Reproduction or in part is prohibited
+ *  without the written consent of the copyright owner.
+ *
+ *  swi2c.h --- SM750/SM718 DDK
+ *  This file contains the definitions for i2c using software
+ *  implementation.
+ *
+ */
+
 #ifndef _SWI2C_H_
 #define _SWI2C_H_
 
diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c
index 7d90e250142c..e9632f162f99 100644
--- a/drivers/staging/sm750fb/sm750.c
+++ b/drivers/staging/sm750fb/sm750.c
@@ -118,14 +118,14 @@ static int lynxfb_ops_cursor(struct fb_info *info, struct fb_cursor *fbcursor)
 		return -ENXIO;
 	}
 
-	hw_cursor_disable(cursor);
+	sm750_hw_cursor_disable(cursor);
 	if (fbcursor->set & FB_CUR_SETSIZE)
-		hw_cursor_setSize(cursor,
+		sm750_hw_cursor_setSize(cursor,
 				  fbcursor->image.width,
 				  fbcursor->image.height);
 
 	if (fbcursor->set & FB_CUR_SETPOS)
-		hw_cursor_setPos(cursor,
+		sm750_hw_cursor_setPos(cursor,
 				 fbcursor->image.dx - info->var.xoffset,
 				 fbcursor->image.dy - info->var.yoffset);
 
@@ -141,18 +141,18 @@ static int lynxfb_ops_cursor(struct fb_info *info, struct fb_cursor *fbcursor)
 		      ((info->cmap.green[fbcursor->image.bg_color] & 0xfc00) >> 5) |
 		      ((info->cmap.blue[fbcursor->image.bg_color] & 0xf800) >> 11);
 
-		hw_cursor_setColor(cursor, fg, bg);
+		sm750_hw_cursor_setColor(cursor, fg, bg);
 	}
 
 	if (fbcursor->set & (FB_CUR_SETSHAPE | FB_CUR_SETIMAGE)) {
-		hw_cursor_setData(cursor,
+		sm750_hw_cursor_setData(cursor,
 				  fbcursor->rop,
 				  fbcursor->image.data,
 				  fbcursor->mask);
 	}
 
 	if (fbcursor->enable)
-		hw_cursor_enable(cursor);
+		sm750_hw_cursor_enable(cursor);
 
 	return 0;
 }
@@ -575,11 +575,11 @@ static int lynxfb_ops_check_var(struct fb_var_screeninfo *var,
 	return hw_sm750_crtc_checkMode(crtc, var);
 }
 
-static int lynxfb_ops_setcolreg(unsigned regno,
-				unsigned red,
-				unsigned green,
-				unsigned blue,
-				unsigned transp,
+static int lynxfb_ops_setcolreg(unsigned int regno,
+				unsigned int red,
+				unsigned int green,
+				unsigned int blue,
+				unsigned int transp,
 				struct fb_info *info)
 {
 	struct lynxfb_par *par;
@@ -788,7 +788,7 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index)
 	memset_io(crtc->cursor.vstart, 0, crtc->cursor.size);
 	if (!g_hwcursor) {
 		lynxfb_ops.fb_cursor = NULL;
-		hw_cursor_disable(&crtc->cursor);
+		sm750_hw_cursor_disable(&crtc->cursor);
 	}
 
 	/* set info->fbops, must be set before fb_find_mode */
@@ -947,13 +947,13 @@ static void sm750fb_setup(struct sm750_dev *sm750_dev, char *src)
 	g_hwcursor = 3;
 
 	if (!src || !*src) {
-		pr_warn("no specific g_option.\n");
+		dev_warn(&sm750_dev->pdev->dev, "no specific g_option.\n");
 		goto NO_PARAM;
 	}
 
 	while ((opt = strsep(&src, ":")) != NULL && *opt != 0) {
-		pr_info("opt=%s\n", opt);
-		pr_info("src=%s\n", src);
+		dev_info(&sm750_dev->pdev->dev, "opt=%s\n", opt);
+		dev_info(&sm750_dev->pdev->dev, "src=%s\n", src);
 
 		if (!strncmp(opt, "swap", strlen("swap")))
 			swap = 1;
@@ -974,12 +974,12 @@ static void sm750fb_setup(struct sm750_dev *sm750_dev, char *src)
 		else {
 			if (!g_fbmode[0]) {
 				g_fbmode[0] = opt;
-				pr_info("find fbmode0 : %s\n", g_fbmode[0]);
+				dev_info(&sm750_dev->pdev->dev, "find fbmode0 : %s\n", g_fbmode[0]);
 			} else if (!g_fbmode[1]) {
 				g_fbmode[1] = opt;
-				pr_info("find fbmode1 : %s\n", g_fbmode[1]);
+				dev_info(&sm750_dev->pdev->dev, "find fbmode1 : %s\n", g_fbmode[1]);
 			} else {
-				pr_warn("How many view you wann set?\n");
+				dev_warn(&sm750_dev->pdev->dev, "How many view you wann set?\n");
 			}
 		}
 	}
@@ -1083,10 +1083,10 @@ static int lynxfb_pci_probe(struct pci_dev *pdev,
 		 * if some chip need specific function,
 		 * please hook it in smXXX_set_drv routine
 		 */
-		sm750_dev->accel.de_init = hw_de_init;
-		sm750_dev->accel.de_fillrect = hw_fillrect;
-		sm750_dev->accel.de_copyarea = hw_copyarea;
-		sm750_dev->accel.de_imageblit = hw_imageblit;
+		sm750_dev->accel.de_init = sm750_hw_de_init;
+		sm750_dev->accel.de_fillrect = sm750_hw_fillrect;
+		sm750_dev->accel.de_copyarea = sm750_hw_copyarea;
+		sm750_dev->accel.de_imageblit = sm750_hw_imageblit;
 	}
 
 	/* call chip specific setup routine  */
@@ -1188,7 +1188,7 @@ static int __init lynxfb_setup(char *options)
 	return 0;
 }
 
-static struct pci_device_id smi_pci_table[] = {
+static const struct pci_device_id smi_pci_table[] = {
 	{ PCI_DEVICE(0x126f, 0x0750), },
 	{0,}
 };
@@ -1209,7 +1209,6 @@ static struct pci_driver lynxfb_driver = {
 static int __init lynxfb_init(void)
 {
 	char *option;
-	int ret;
 
 #ifdef MODULE
 	option = g_option;
@@ -1219,8 +1218,7 @@ static int __init lynxfb_init(void)
 #endif
 
 	lynxfb_setup(option);
-	ret = pci_register_driver(&lynxfb_driver);
-	return ret;
+	return pci_register_driver(&lynxfb_driver);
 }
 module_init(lynxfb_init);
 
@@ -1245,4 +1243,4 @@ MODULE_PARM_DESC(g_option,
 MODULE_AUTHOR("monk liu <monk.liu@siliconmotion.com>");
 MODULE_AUTHOR("Sudip Mukherjee <sudip@vectorindia.org>");
 MODULE_DESCRIPTION("Frame buffer driver for SM750 chipset");
-MODULE_LICENSE("GPL v2");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/staging/sm750fb/sm750.h b/drivers/staging/sm750fb/sm750.h
index ff31c5c9cc6f..28f4b9b4f95f 100644
--- a/drivers/staging/sm750fb/sm750.h
+++ b/drivers/staging/sm750fb/sm750.h
@@ -146,14 +146,16 @@ struct lynxfb_crtc {
 struct lynxfb_output {
 	int dpms;
 	int paths;
-	/* which paths(s) this output stands for,for sm750:
+	/*
+	 * which paths(s) this output stands for,for sm750:
 	 * paths=1:means output for panel paths
 	 * paths=2:means output for crt paths
 	 * paths=3:means output for both panel and crt paths
 	 */
 
 	int *channel;
-	/* which channel these outputs linked with,for sm750:
+	/*
+	 * which channel these outputs linked with,for sm750:
 	 * *channel=0 means primary channel
 	 * *channel=1 means secondary channel
 	 * output->channel ==> &crtc->channel
diff --git a/drivers/staging/sm750fb/sm750_accel.c b/drivers/staging/sm750fb/sm750_accel.c
index 38adae6b5d83..af0db5789c53 100644
--- a/drivers/staging/sm750fb/sm750_accel.c
+++ b/drivers/staging/sm750fb/sm750_accel.c
@@ -32,7 +32,7 @@ static inline void write_dpPort(struct lynx_accel *accel, u32 data)
 	writel(data, accel->dpPortBase);
 }
 
-void hw_de_init(struct lynx_accel *accel)
+void sm750_hw_de_init(struct lynx_accel *accel)
 {
 	/* setup 2d engine registers */
 	u32 reg, clr;
@@ -65,12 +65,13 @@ void hw_de_init(struct lynx_accel *accel)
 	write_dpr(accel, DE_CONTROL, read_dpr(accel, DE_CONTROL) & ~clr);
 }
 
-/* set2dformat only be called from setmode functions
+/*
+ * set2dformat only be called from setmode functions
  * but if you need dual framebuffer driver,need call set2dformat
  * every time you use 2d function
  */
 
-void hw_set2dformat(struct lynx_accel *accel, int fmt)
+void sm750_hw_set2dformat(struct lynx_accel *accel, int fmt)
 {
 	u32 reg;
 
@@ -82,7 +83,7 @@ void hw_set2dformat(struct lynx_accel *accel, int fmt)
 	write_dpr(accel, DE_STRETCH_FORMAT, reg);
 }
 
-int hw_fillrect(struct lynx_accel *accel,
+int sm750_hw_fillrect(struct lynx_accel *accel,
 				u32 base, u32 pitch, u32 Bpp,
 				u32 x, u32 y, u32 width, u32 height,
 				u32 color, u32 rop)
@@ -90,7 +91,8 @@ int hw_fillrect(struct lynx_accel *accel,
 	u32 deCtrl;
 
 	if (accel->de_wait() != 0) {
-		/* int time wait and always busy,seems hardware
+		/*
+		 * int time wait and always busy,seems hardware
 		 * got something error
 		 */
 		pr_debug("De engine always busy\n");
@@ -126,7 +128,7 @@ int hw_fillrect(struct lynx_accel *accel,
 	return 0;
 }
 
-int hw_copyarea(
+int sm750_hw_copyarea(
 struct lynx_accel *accel,
 unsigned int sBase,  /* Address of source: offset in frame buffer */
 unsigned int sPitch, /* Pitch value of source surface in BYTE */
@@ -213,25 +215,29 @@ unsigned int rop2)   /* ROP value */
 		opSign = (-1);
 	}
 
-	/* Note:
+	/*
+	 * Note:
 	 * DE_FOREGROUND are DE_BACKGROUND are don't care.
 	 * DE_COLOR_COMPARE and DE_COLOR_COMPARE_MAKS
 	 * are set by set deSetTransparency().
 	 */
 
-	/* 2D Source Base.
+	/*
+	 * 2D Source Base.
 	 * It is an address offset (128 bit aligned)
 	 * from the beginning of frame buffer.
 	 */
 	write_dpr(accel, DE_WINDOW_SOURCE_BASE, sBase); /* dpr40 */
 
-	/* 2D Destination Base.
+	/*
+	 * 2D Destination Base.
 	 * It is an address offset (128 bit aligned)
 	 * from the beginning of frame buffer.
 	 */
 	write_dpr(accel, DE_WINDOW_DESTINATION_BASE, dBase); /* dpr44 */
 
-    /* Program pitch (distance between the 1st points of two adjacent lines).
+    /*
+     * Program pitch (distance between the 1st points of two adjacent lines).
      * Note that input pitch is BYTE value, but the 2D Pitch register uses
      * pixel values. Need Byte to pixel conversion.
      */
@@ -240,7 +246,8 @@ unsigned int rop2)   /* ROP value */
 		   DE_PITCH_DESTINATION_MASK) |
 		  (sPitch / Bpp & DE_PITCH_SOURCE_MASK)); /* dpr10 */
 
-    /* Screen Window width in Pixels.
+    /*
+     * Screen Window width in Pixels.
      * 2D engine uses this value to calculate the linear address in frame buffer
      * for a given point.
      */
@@ -286,7 +293,7 @@ static unsigned int deGetTransparency(struct lynx_accel *accel)
 	return de_ctrl;
 }
 
-int hw_imageblit(struct lynx_accel *accel,
+int sm750_hw_imageblit(struct lynx_accel *accel,
 		 const char *pSrcbuf, /* pointer to start of source buffer in system memory */
 		 u32 srcDelta,          /* Pitch value (in bytes) of the source buffer, +ive means top down and -ive mean button up */
 		 u32 startBit, /* Mono data can start at any bit in a byte, this value should be 0 to 7 */
@@ -316,7 +323,8 @@ int hw_imageblit(struct lynx_accel *accel,
 	if (accel->de_wait() != 0)
 		return -1;
 
-	/* 2D Source Base.
+	/*
+	 * 2D Source Base.
 	 * Use 0 for HOST Blt.
 	 */
 	write_dpr(accel, DE_WINDOW_SOURCE_BASE, 0);
@@ -326,16 +334,19 @@ int hw_imageblit(struct lynx_accel *accel,
 	 * from the beginning of frame buffer.
 	 */
 	write_dpr(accel, DE_WINDOW_DESTINATION_BASE, dBase);
-    /* Program pitch (distance between the 1st points of two adjacent lines).
-     * Note that input pitch is BYTE value, but the 2D Pitch register uses
-     * pixel values. Need Byte to pixel conversion.
-     */
+
+	/*
+	 * Program pitch (distance between the 1st points of two adjacent
+	 * lines). Note that input pitch is BYTE value, but the 2D Pitch
+	 * register uses pixel values. Need Byte to pixel conversion.
+	 */
 	write_dpr(accel, DE_PITCH,
 		  ((dPitch / bytePerPixel << DE_PITCH_DESTINATION_SHIFT) &
 		   DE_PITCH_DESTINATION_MASK) |
 		  (dPitch / bytePerPixel & DE_PITCH_SOURCE_MASK)); /* dpr10 */
 
-	/* Screen Window width in Pixels.
+	/*
+	 * Screen Window width in Pixels.
 	 * 2D engine uses this value to calculate the linear address
 	 * in frame buffer for a given point.
 	 */
@@ -344,7 +355,8 @@ int hw_imageblit(struct lynx_accel *accel,
 		   DE_WINDOW_WIDTH_DST_MASK) |
 		  (dPitch / bytePerPixel & DE_WINDOW_WIDTH_SRC_MASK));
 
-	 /* Note: For 2D Source in Host Write, only X_K1_MONO field is needed,
+	 /*
+	  * Note: For 2D Source in Host Write, only X_K1_MONO field is needed,
 	  * and Y_K2 field is not used.
 	  * For mono bitmap, use startBit for X_K1.
 	  */
@@ -383,6 +395,6 @@ int hw_imageblit(struct lynx_accel *accel,
 		pSrcbuf += srcDelta;
 	}
 
-	    return 0;
+	return 0;
 }
 
diff --git a/drivers/staging/sm750fb/sm750_accel.h b/drivers/staging/sm750fb/sm750_accel.h
index d59d005e0add..4b0ff8feb9a0 100644
--- a/drivers/staging/sm750fb/sm750_accel.h
+++ b/drivers/staging/sm750fb/sm750_accel.h
@@ -184,16 +184,16 @@
 #define BOTTOM_TO_TOP 1
 #define RIGHT_TO_LEFT 1
 
-void hw_set2dformat(struct lynx_accel *accel, int fmt);
+void sm750_hw_set2dformat(struct lynx_accel *accel, int fmt);
 
-void hw_de_init(struct lynx_accel *accel);
+void sm750_hw_de_init(struct lynx_accel *accel);
 
-int hw_fillrect(struct lynx_accel *accel,
+int sm750_hw_fillrect(struct lynx_accel *accel,
 				u32 base, u32 pitch, u32 Bpp,
 				u32 x, u32 y, u32 width, u32 height,
 				u32 color, u32 rop);
 
-int hw_copyarea(
+int sm750_hw_copyarea(
 struct lynx_accel *accel,
 unsigned int sBase,  /* Address of source: offset in frame buffer */
 unsigned int sPitch, /* Pitch value of source surface in BYTE */
@@ -208,7 +208,7 @@ unsigned int width,
 unsigned int height, /* width and height of rectangle in pixel value */
 unsigned int rop2);
 
-int hw_imageblit(struct lynx_accel *accel,
+int sm750_hw_imageblit(struct lynx_accel *accel,
 		 const char *pSrcbuf, /* pointer to start of source buffer in system memory */
 		 u32 srcDelta,          /* Pitch value (in bytes) of the source buffer, +ive means top down and -ive mean button up */
 		 u32 startBit, /* Mono data can start at any bit in a byte, this value should be 0 to 7 */
diff --git a/drivers/staging/sm750fb/sm750_cursor.c b/drivers/staging/sm750fb/sm750_cursor.c
index d622d65b6cee..2a13353fc492 100644
--- a/drivers/staging/sm750fb/sm750_cursor.c
+++ b/drivers/staging/sm750fb/sm750_cursor.c
@@ -47,25 +47,25 @@ writel((data), cursor->mmio + (addr))
 
 
 /* hw_cursor_xxx works for voyager,718 and 750 */
-void hw_cursor_enable(struct lynx_cursor *cursor)
+void sm750_hw_cursor_enable(struct lynx_cursor *cursor)
 {
 	u32 reg;
 
 	reg = (cursor->offset & HWC_ADDRESS_ADDRESS_MASK) | HWC_ADDRESS_ENABLE;
 	POKE32(HWC_ADDRESS, reg);
 }
-void hw_cursor_disable(struct lynx_cursor *cursor)
+void sm750_hw_cursor_disable(struct lynx_cursor *cursor)
 {
 	POKE32(HWC_ADDRESS, 0);
 }
 
-void hw_cursor_setSize(struct lynx_cursor *cursor,
+void sm750_hw_cursor_setSize(struct lynx_cursor *cursor,
 						int w, int h)
 {
 	cursor->w = w;
 	cursor->h = h;
 }
-void hw_cursor_setPos(struct lynx_cursor *cursor,
+void sm750_hw_cursor_setPos(struct lynx_cursor *cursor,
 						int x, int y)
 {
 	u32 reg;
@@ -74,7 +74,7 @@ void hw_cursor_setPos(struct lynx_cursor *cursor,
 		(x & HWC_LOCATION_X_MASK));
 	POKE32(HWC_LOCATION, reg);
 }
-void hw_cursor_setColor(struct lynx_cursor *cursor,
+void sm750_hw_cursor_setColor(struct lynx_cursor *cursor,
 						u32 fg, u32 bg)
 {
 	u32 reg = (fg << HWC_COLOR_12_2_RGB565_SHIFT) &
@@ -84,7 +84,7 @@ void hw_cursor_setColor(struct lynx_cursor *cursor,
 	POKE32(HWC_COLOR_3, 0xffe0);
 }
 
-void hw_cursor_setData(struct lynx_cursor *cursor,
+void sm750_hw_cursor_setData(struct lynx_cursor *cursor,
 			u16 rop, const u8 *pcol, const u8 *pmsk)
 {
 	int i, j, count, pitch, offset;
@@ -138,7 +138,7 @@ void hw_cursor_setData(struct lynx_cursor *cursor,
 }
 
 
-void hw_cursor_setData2(struct lynx_cursor *cursor,
+void sm750_hw_cursor_setData2(struct lynx_cursor *cursor,
 			u16 rop, const u8 *pcol, const u8 *pmsk)
 {
 	int i, j, count, pitch, offset;
diff --git a/drivers/staging/sm750fb/sm750_cursor.h b/drivers/staging/sm750fb/sm750_cursor.h
index 6c4fc9b73489..c7b86ae235b4 100644
--- a/drivers/staging/sm750fb/sm750_cursor.h
+++ b/drivers/staging/sm750fb/sm750_cursor.h
@@ -2,16 +2,16 @@
 #define LYNX_CURSOR_H__
 
 /* hw_cursor_xxx works for voyager,718 and 750 */
-void hw_cursor_enable(struct lynx_cursor *cursor);
-void hw_cursor_disable(struct lynx_cursor *cursor);
-void hw_cursor_setSize(struct lynx_cursor *cursor,
+void sm750_hw_cursor_enable(struct lynx_cursor *cursor);
+void sm750_hw_cursor_disable(struct lynx_cursor *cursor);
+void sm750_hw_cursor_setSize(struct lynx_cursor *cursor,
 						int w, int h);
-void hw_cursor_setPos(struct lynx_cursor *cursor,
+void sm750_hw_cursor_setPos(struct lynx_cursor *cursor,
 						int x, int y);
-void hw_cursor_setColor(struct lynx_cursor *cursor,
+void sm750_hw_cursor_setColor(struct lynx_cursor *cursor,
 						u32 fg, u32 bg);
-void hw_cursor_setData(struct lynx_cursor *cursor,
+void sm750_hw_cursor_setData(struct lynx_cursor *cursor,
 			u16 rop, const u8 *data, const u8 *mask);
-void hw_cursor_setData2(struct lynx_cursor *cursor,
+void sm750_hw_cursor_setData2(struct lynx_cursor *cursor,
 			u16 rop, const u8 *data, const u8 *mask);
 #endif
diff --git a/drivers/staging/sm750fb/sm750_hw.c b/drivers/staging/sm750fb/sm750_hw.c
index 7dd208caa5eb..b6af3b53076b 100644
--- a/drivers/staging/sm750fb/sm750_hw.c
+++ b/drivers/staging/sm750fb/sm750_hw.c
@@ -23,6 +23,8 @@
 #include "ddk750.h"
 #include "sm750_accel.h"
 
+void __iomem *mmio750;
+
 int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 {
 	int ret;
@@ -34,7 +36,8 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 
 	pr_info("mmio phyAddr = %lx\n", sm750_dev->vidreg_start);
 
-	/* reserve the vidreg space of smi adaptor
+	/*
+	 * reserve the vidreg space of smi adaptor
 	 * if you do this, you need to add release region code
 	 * in lynxfb_remove, or memory will not be mapped again
 	 * successfully
@@ -59,15 +62,17 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 	sm750_dev->accel.dprBase = sm750_dev->pvReg + DE_BASE_ADDR_TYPE1;
 	sm750_dev->accel.dpPortBase = sm750_dev->pvReg + DE_PORT_ADDR_TYPE1;
 
-	ddk750_set_mmio(sm750_dev->pvReg, sm750_dev->devid, sm750_dev->revid);
+	mmio750 = sm750_dev->pvReg;
+	sm750_set_chip_type(sm750_dev->devid, sm750_dev->revid);
 
 	sm750_dev->vidmem_start = pci_resource_start(pdev, 0);
-	/* don't use pdev_resource[x].end - resource[x].start to
+	/*
+	 * don't use pdev_resource[x].end - resource[x].start to
 	 * calculate the resource size, it's only the maximum available
 	 * size but not the actual size, using
-	 * @ddk750_getVMSize function can be safe.
+	 * @ddk750_get_vm_size function can be safe.
 	 */
-	sm750_dev->vidmem_size = ddk750_getVMSize();
+	sm750_dev->vidmem_size = ddk750_get_vm_size();
 	pr_info("video memory phyAddr = %lx, size = %u bytes\n",
 		sm750_dev->vidmem_start, sm750_dev->vidmem_size);
 
@@ -100,7 +105,7 @@ int hw_sm750_inithw(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 	if (parm->master_clk == 0)
 		parm->master_clk = parm->chip_clk / 3;
 
-	ddk750_initHw((initchip_param_t *)&sm750_dev->initParm);
+	ddk750_init_hw((struct initchip_param *)&sm750_dev->initParm);
 	/* for sm718, open pci burst */
 	if (sm750_dev->devid == 0x718) {
 		POKE32(SYSTEM_CTRL,
@@ -141,7 +146,8 @@ int hw_sm750_inithw(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 		}
 		POKE32(PANEL_DISPLAY_CTRL, val);
 	} else {
-		/* for 750LE, no DVI chip initialization
+		/*
+		 * for 750LE, no DVI chip initialization
 		 * makes Monitor no signal
 		 *
 		 * Set up GPIO for software I2C to program DVI chip in the
@@ -149,13 +155,15 @@ int hw_sm750_inithw(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 		 */
 		sm750_sw_i2c_init(0, 1);
 
-		/* Customer may NOT use CH7301 DVI chip, which has to be
+		/*
+		 * Customer may NOT use CH7301 DVI chip, which has to be
 		 * initialized differently.
 		 */
 		if (sm750_sw_i2c_read_reg(0xec, 0x4a) == 0x95) {
-		/* The following register values for CH7301 are from
-		 * Chrontel app note and our experiment.
-		 */
+			/*
+			 * The following register values for CH7301 are from
+			 * Chrontel app note and our experiment.
+			 */
 			pr_info("yes,CH7301 DVI chip found\n");
 			sm750_sw_i2c_write_reg(0xec, 0x1d, 0x16);
 			sm750_sw_i2c_write_reg(0xec, 0x21, 0x9);
@@ -267,7 +275,7 @@ int hw_sm750_crtc_setMode(struct lynxfb_crtc *crtc,
 			fmt = 2;
 			break;
 		}
-		hw_set2dformat(&sm750_dev->accel, fmt);
+		sm750_hw_set2dformat(&sm750_dev->accel, fmt);
 	}
 
 	/* set timing */
@@ -308,7 +316,8 @@ int hw_sm750_crtc_setMode(struct lynxfb_crtc *crtc,
 		       crtc->oScreen & PANEL_FB_ADDRESS_ADDRESS_MASK);
 
 		reg = var->xres * (var->bits_per_pixel >> 3);
-		/* crtc->channel is not equal to par->index on numeric,
+		/*
+		 * crtc->channel is not equal to par->index on numeric,
 		 * be aware of that
 		 */
 		reg = ALIGN(reg, crtc->line_pad);
@@ -342,7 +351,8 @@ int hw_sm750_crtc_setMode(struct lynxfb_crtc *crtc,
 		/* not implemented now */
 		POKE32(CRT_FB_ADDRESS, crtc->oScreen);
 		reg = var->xres * (var->bits_per_pixel >> 3);
-		/* crtc->channel is not equal to par->index on numeric,
+		/*
+		 * crtc->channel is not equal to par->index on numeric,
 		 * be aware of that
 		 */
 		reg = ALIGN(reg, crtc->line_pad) << CRT_FB_WIDTH_WIDTH_SHIFT;
@@ -469,7 +479,7 @@ void hw_sm750_initAccel(struct sm750_dev *sm750_dev)
 {
 	u32 reg;
 
-	enable2DEngine(1);
+	sm750_enable_2d_engine(1);
 
 	if (sm750_get_chip_type() == SM750LE) {
 		reg = PEEK32(DE_STATE1);
diff --git a/drivers/staging/speakup/TODO b/drivers/staging/speakup/TODO
index 3094799cf6a0..993410c3e531 100644
--- a/drivers/staging/speakup/TODO
+++ b/drivers/staging/speakup/TODO
@@ -42,6 +42,6 @@ We prefer that you contact us on the mailing list; however, if you do
 not want to subscribe to a mailing list, send your email to all of the
 following:
 
-w.d.hubbs@gmail.com, chris@the-brannons.com, kirk@braille.uwo.ca and
+w.d.hubbs@gmail.com, chris@the-brannons.com, kirk@reisers.ca and
 samuel.thibault@ens-lyon.org.
 
diff --git a/drivers/staging/speakup/main.c b/drivers/staging/speakup/main.c
index 97ca4ecca8a9..5c192042eeac 100644
--- a/drivers/staging/speakup/main.c
+++ b/drivers/staging/speakup/main.c
@@ -351,14 +351,14 @@ static void speakup_cut(struct vc_data *vc)
 
 	if (!mark_cut_flag) {
 		mark_cut_flag = 1;
-		spk_xs = (u_short) spk_x;
-		spk_ys = (u_short) spk_y;
+		spk_xs = (u_short)spk_x;
+		spk_ys = (u_short)spk_y;
 		spk_sel_cons = vc;
 		synth_printf("%s\n", spk_msg_get(MSG_MARK));
 		return;
 	}
-	spk_xe = (u_short) spk_x;
-	spk_ye = (u_short) spk_y;
+	spk_xe = (u_short)spk_x;
+	spk_ye = (u_short)spk_y;
 	mark_cut_flag = 0;
 	synth_printf("%s\n", spk_msg_get(MSG_CUT));
 
@@ -489,7 +489,7 @@ static void say_char(struct vc_data *vc)
 	u_short ch;
 
 	spk_old_attr = spk_attr;
-	ch = get_char(vc, (u_short *) spk_pos, &spk_attr);
+	ch = get_char(vc, (u_short *)spk_pos, &spk_attr);
 	if (spk_attr != spk_old_attr) {
 		if (spk_attrib_bleep & 1)
 			bleep(spk_y);
@@ -504,7 +504,7 @@ static void say_phonetic_char(struct vc_data *vc)
 	u_short ch;
 
 	spk_old_attr = spk_attr;
-	ch = get_char(vc, (u_short *) spk_pos, &spk_attr);
+	ch = get_char(vc, (u_short *)spk_pos, &spk_attr);
 	if (isascii(ch) && isalpha(ch)) {
 		ch &= 0x1f;
 		synth_printf("%s\n", phonetic[--ch]);
@@ -556,7 +556,7 @@ static u_long get_word(struct vc_data *vc)
 	u_char temp;
 
 	spk_old_attr = spk_attr;
-	ch = (char)get_char(vc, (u_short *) tmp_pos, &temp);
+	ch = (char)get_char(vc, (u_short *)tmp_pos, &temp);
 
 /* decided to take out the sayword if on a space (mis-information */
 	if (spk_say_word_ctl && ch == SPACE) {
@@ -565,26 +565,26 @@ static u_long get_word(struct vc_data *vc)
 		return 0;
 	} else if ((tmpx < vc->vc_cols - 2)
 		   && (ch == SPACE || ch == 0 || IS_WDLM(ch))
-		   && ((char)get_char(vc, (u_short *) &tmp_pos + 1, &temp) >
+		   && ((char)get_char(vc, (u_short *)&tmp_pos + 1, &temp) >
 		       SPACE)) {
 		tmp_pos += 2;
 		tmpx++;
 	} else
 		while (tmpx > 0) {
-			ch = (char)get_char(vc, (u_short *) tmp_pos - 1, &temp);
+			ch = (char)get_char(vc, (u_short *)tmp_pos - 1, &temp);
 			if ((ch == SPACE || ch == 0 || IS_WDLM(ch))
-			    && ((char)get_char(vc, (u_short *) tmp_pos, &temp) >
+			    && ((char)get_char(vc, (u_short *)tmp_pos, &temp) >
 				SPACE))
 				break;
 			tmp_pos -= 2;
 			tmpx--;
 		}
-	attr_ch = get_char(vc, (u_short *) tmp_pos, &spk_attr);
+	attr_ch = get_char(vc, (u_short *)tmp_pos, &spk_attr);
 	buf[cnt++] = attr_ch & 0xff;
 	while (tmpx < vc->vc_cols - 1) {
 		tmp_pos += 2;
 		tmpx++;
-		ch = (char)get_char(vc, (u_short *) tmp_pos, &temp);
+		ch = (char)get_char(vc, (u_short *)tmp_pos, &temp);
 		if ((ch == SPACE) || ch == 0
 		    || (IS_WDLM(buf[cnt - 1]) && (ch > SPACE)))
 			break;
@@ -639,7 +639,7 @@ static void say_prev_word(struct vc_data *vc)
 		} else
 			spk_x--;
 		spk_pos -= 2;
-		ch = (char)get_char(vc, (u_short *) spk_pos, &temp);
+		ch = (char)get_char(vc, (u_short *)spk_pos, &temp);
 		if (ch == SPACE || ch == 0)
 			state = 0;
 		else if (IS_WDLM(ch))
@@ -672,7 +672,7 @@ static void say_next_word(struct vc_data *vc)
 		return;
 	}
 	while (1) {
-		ch = (char)get_char(vc, (u_short *) spk_pos, &temp);
+		ch = (char)get_char(vc, (u_short *)spk_pos, &temp);
 		if (ch == SPACE || ch == 0)
 			state = 0;
 		else if (IS_WDLM(ch))
@@ -709,7 +709,7 @@ static void spell_word(struct vc_data *vc)
 
 	if (!get_word(vc))
 		return;
-	while ((ch = (u_char) *cp)) {
+	while ((ch = (u_char)*cp)) {
 		if (cp != buf)
 			synth_printf(" %s ", delay_str[spk_spell_delay]);
 		if (IS_CHAR(ch, B_CAP)) {
@@ -751,7 +751,7 @@ static int get_line(struct vc_data *vc)
 	spk_old_attr = spk_attr;
 	spk_attr = get_attributes(vc, (u_short *)spk_pos);
 	for (i = 0; i < vc->vc_cols; i++) {
-		buf[i] = (u_char) get_char(vc, (u_short *) tmp, &tmp2);
+		buf[i] = (u_char)get_char(vc, (u_short *)tmp, &tmp2);
 		tmp += 2;
 	}
 	for (--i; i >= 0; i--)
@@ -816,7 +816,7 @@ static int say_from_to(struct vc_data *vc, u_long from, u_long to,
 	spk_old_attr = spk_attr;
 	spk_attr = get_attributes(vc, (u_short *)from);
 	while (from < to) {
-		buf[i++] = (char)get_char(vc, (u_short *) from, &tmp);
+		buf[i++] = (char)get_char(vc, (u_short *)from, &tmp);
 		from += 2;
 		if (i >= vc->vc_size_row)
 			break;
@@ -892,7 +892,7 @@ static int get_sentence_buf(struct vc_data *vc, int read_punc)
 	spk_attr = get_attributes(vc, (u_short *)start);
 
 	while (start < end) {
-		sentbuf[bn][i] = (char)get_char(vc, (u_short *) start, &tmp);
+		sentbuf[bn][i] = (char)get_char(vc, (u_short *)start, &tmp);
 		if (i > 0) {
 			if (sentbuf[bn][i] == SPACE && sentbuf[bn][i - 1] == '.'
 			    && numsentences[bn] < 9) {
@@ -1040,7 +1040,7 @@ static void say_position(struct vc_data *vc)
 static void say_char_num(struct vc_data *vc)
 {
 	u_char tmp;
-	u_short ch = get_char(vc, (u_short *) spk_pos, &tmp);
+	u_short ch = get_char(vc, (u_short *)spk_pos, &tmp);
 
 	ch &= 0xff;
 	synth_printf(spk_msg_get(MSG_CHAR_INFO), ch, ch);
@@ -1085,7 +1085,7 @@ static void spkup_write(const char *in_buf, int count)
 			    (currsentence <= numsentences[bn]))
 				synth_insert_next_index(currsentence++);
 		}
-		ch = (u_char) *in_buf++;
+		ch = (u_char)*in_buf++;
 		char_type = spk_chartab[ch];
 		if (ch == old_ch && !(char_type & B_NUM)) {
 			if (++rep_count > 2)
@@ -1579,7 +1579,7 @@ static int count_highlight_color(struct vc_data *vc)
 	int cc;
 	int vc_num = vc->vc_num;
 	u16 ch;
-	u16 *start = (u16 *) vc->vc_origin;
+	u16 *start = (u16 *)vc->vc_origin;
 
 	for (i = 0; i < 8; i++)
 		speakup_console[vc_num]->ht.bgcount[i] = 0;
diff --git a/drivers/staging/speakup/selection.c b/drivers/staging/speakup/selection.c
index 0149edc1e0ae..aeb2b865615a 100644
--- a/drivers/staging/speakup/selection.c
+++ b/drivers/staging/speakup/selection.c
@@ -137,7 +137,7 @@ static void __speakup_paste_selection(struct work_struct *work)
 	struct speakup_paste_work *spw =
 		container_of(work, struct speakup_paste_work, work);
 	struct tty_struct *tty = xchg(&spw->tty, NULL);
-	struct vc_data *vc = (struct vc_data *) tty->driver_data;
+	struct vc_data *vc = (struct vc_data *)tty->driver_data;
 	int pasted = 0, count;
 	struct tty_ldisc *ld;
 	DECLARE_WAITQUEUE(wait, current);
diff --git a/drivers/staging/speakup/serialio.c b/drivers/staging/speakup/serialio.c
index c2c435cc3d63..ef89dc1c21c8 100644
--- a/drivers/staging/speakup/serialio.c
+++ b/drivers/staging/speakup/serialio.c
@@ -99,7 +99,7 @@ static irqreturn_t synth_readbuf_handler(int irq, void *dev_id)
 	while (inb_p(speakup_info.port_tts + UART_LSR) & UART_LSR_DR) {
 
 		c = inb_p(speakup_info.port_tts+UART_RX);
-		synth->read_buff_add((u_char) c);
+		synth->read_buff_add((u_char)c);
 	}
 	spin_unlock_irqrestore(&speakup_info.spinlock, flags);
 	return IRQ_HANDLED;
@@ -113,7 +113,7 @@ static void start_serial_interrupt(int irq)
 		return;
 
 	rv = request_irq(irq, synth_readbuf_handler, IRQF_SHARED,
-			 "serial", (void *) synth_readbuf_handler);
+			 "serial", (void *)synth_readbuf_handler);
 
 	if (rv)
 		pr_err("Unable to request Speakup serial I R Q\n");
@@ -141,7 +141,7 @@ void spk_stop_serial_interrupt(void)
 	/* Turn off interrupts */
 	outb(0, speakup_info.port_tts+UART_IER);
 	/* Free IRQ */
-	free_irq(serstate->irq, (void *) synth_readbuf_handler);
+	free_irq(serstate->irq, (void *)synth_readbuf_handler);
 }
 
 int spk_wait_for_xmitr(void)
diff --git a/drivers/staging/speakup/speakup_soft.c b/drivers/staging/speakup/speakup_soft.c
index 6b1d0f538bbd..ed3e4282f41c 100644
--- a/drivers/staging/speakup/speakup_soft.c
+++ b/drivers/staging/speakup/speakup_soft.c
@@ -20,8 +20,8 @@
  */
 
 #include <linux/unistd.h>
-#include <linux/miscdevice.h> /* for misc_register, and SYNTH_MINOR */
-#include <linux/poll.h> /* for poll_wait() */
+#include <linux/miscdevice.h>	/* for misc_register, and SYNTH_MINOR */
+#include <linux/poll.h>		/* for poll_wait() */
 #include <linux/sched.h> /* schedule(), signal_pending(), TASK_INTERRUPTIBLE */
 
 #include "spk_priv.h"
@@ -55,27 +55,26 @@ static struct var_t vars[] = {
 	V_LAST_VAR
 };
 
-/*
- * These attributes will appear in /sys/accessibility/speakup/soft.
- */
+/* These attributes will appear in /sys/accessibility/speakup/soft. */
+
 static struct kobj_attribute caps_start_attribute =
-	__ATTR(caps_start, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(caps_start, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute caps_stop_attribute =
-	__ATTR(caps_stop, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(caps_stop, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute freq_attribute =
-	__ATTR(freq, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(freq, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute pitch_attribute =
-	__ATTR(pitch, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(pitch, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute punct_attribute =
-	__ATTR(punct, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(punct, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute rate_attribute =
-	__ATTR(rate, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(rate, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute tone_attribute =
-	__ATTR(tone, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(tone, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute voice_attribute =
-	__ATTR(voice, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(voice, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute vol_attribute =
-	__ATTR(vol, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(vol, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 
 /*
  * We should uncomment the following definition, when we agree on a
@@ -85,15 +84,15 @@ static struct kobj_attribute vol_attribute =
  */
 
 static struct kobj_attribute delay_time_attribute =
-	__ATTR(delay_time, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(delay_time, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute direct_attribute =
-	__ATTR(direct, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(direct, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute full_time_attribute =
-	__ATTR(full_time, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(full_time, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute jiffy_delta_attribute =
-	__ATTR(jiffy_delta, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(jiffy_delta, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute trigger_time_attribute =
-	__ATTR(trigger_time, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(trigger_time, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 
 /*
  * Create a group of attributes so that we can create and destroy them all
@@ -162,8 +161,8 @@ static char *get_initstring(void)
 	cp = buf;
 	var = synth_soft.vars;
 	while (var->var_id != MAXVARS) {
-		if (var->var_id != CAPS_START && var->var_id != CAPS_STOP
-			&& var->var_id != DIRECT)
+		if (var->var_id != CAPS_START && var->var_id != CAPS_STOP &&
+		    var->var_id != DIRECT)
 			cp = cp + sprintf(cp, var->u.n.synth_fmt,
 					  var->u.n.value);
 		var++;
@@ -277,8 +276,7 @@ static ssize_t softsynth_write(struct file *fp, const char __user *buf,
 	return count;
 }
 
-static unsigned int softsynth_poll(struct file *fp,
-		struct poll_table_struct *wait)
+static unsigned int softsynth_poll(struct file *fp, struct poll_table_struct *wait)
 {
 	unsigned long flags;
 	int ret = 0;
@@ -310,10 +308,8 @@ static const struct file_operations softsynth_fops = {
 	.release = softsynth_close,
 };
 
-
 static int softsynth_probe(struct spk_synth *synth)
 {
-
 	if (misc_registered != 0)
 		return 0;
 	memset(&synth_device, 0, sizeof(synth_device));
diff --git a/drivers/staging/speakup/speakup_spkout.c b/drivers/staging/speakup/speakup_spkout.c
index e449f2770c1f..586890908826 100644
--- a/drivers/staging/speakup/speakup_spkout.c
+++ b/drivers/staging/speakup/speakup_spkout.c
@@ -1,6 +1,6 @@
 /*
  * originally written by: Kirk Reiser <kirk@braille.uwo.ca>
-* this version considerably modified by David Borowski, david575@rogers.com
+ * this version considerably modified by David Borowski, david575@rogers.com
  *
  * Copyright (C) 1998-99  Kirk Reiser.
  * Copyright (C) 2003 David Borowski.
@@ -40,34 +40,33 @@ static struct var_t vars[] = {
 	V_LAST_VAR
 };
 
-/*
- * These attributes will appear in /sys/accessibility/speakup/spkout.
- */
+/* These attributes will appear in /sys/accessibility/speakup/spkout. */
+
 static struct kobj_attribute caps_start_attribute =
-	__ATTR(caps_start, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(caps_start, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute caps_stop_attribute =
-	__ATTR(caps_stop, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(caps_stop, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute pitch_attribute =
-	__ATTR(pitch, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(pitch, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute punct_attribute =
-	__ATTR(punct, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(punct, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute rate_attribute =
-	__ATTR(rate, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(rate, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute tone_attribute =
-	__ATTR(tone, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(tone, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute vol_attribute =
-	__ATTR(vol, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(vol, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 
 static struct kobj_attribute delay_time_attribute =
-	__ATTR(delay_time, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(delay_time, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute direct_attribute =
-	__ATTR(direct, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(direct, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute full_time_attribute =
-	__ATTR(full_time, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(full_time, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute jiffy_delta_attribute =
-	__ATTR(jiffy_delta, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(jiffy_delta, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute trigger_time_attribute =
-	__ATTR(trigger_time, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(trigger_time, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 
 /*
  * Create a group of attributes so that we can create and destroy them all
diff --git a/drivers/staging/speakup/speakup_txprt.c b/drivers/staging/speakup/speakup_txprt.c
index fd98d4ffcb3e..b3d2cfd20ac8 100644
--- a/drivers/staging/speakup/speakup_txprt.c
+++ b/drivers/staging/speakup/speakup_txprt.c
@@ -1,6 +1,6 @@
 /*
  * originally written by: Kirk Reiser <kirk@braille.uwo.ca>
-* this version considerably modified by David Borowski, david575@rogers.com
+ * this version considerably modified by David Borowski, david575@rogers.com
  *
  * Copyright (C) 1998-99  Kirk Reiser.
  * Copyright (C) 2003 David Borowski.
@@ -36,32 +36,31 @@ static struct var_t vars[] = {
 	V_LAST_VAR
 	 };
 
-/*
- * These attributes will appear in /sys/accessibility/speakup/txprt.
- */
+/* These attributes will appear in /sys/accessibility/speakup/txprt. */
+
 static struct kobj_attribute caps_start_attribute =
-	__ATTR(caps_start, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(caps_start, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute caps_stop_attribute =
-	__ATTR(caps_stop, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(caps_stop, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute pitch_attribute =
-	__ATTR(pitch, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(pitch, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute rate_attribute =
-	__ATTR(rate, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(rate, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute tone_attribute =
-	__ATTR(tone, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(tone, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute vol_attribute =
-	__ATTR(vol, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(vol, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 
 static struct kobj_attribute delay_time_attribute =
-	__ATTR(delay_time, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(delay_time, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute direct_attribute =
-	__ATTR(direct, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(direct, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute full_time_attribute =
-	__ATTR(full_time, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(full_time, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute jiffy_delta_attribute =
-	__ATTR(jiffy_delta, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(jiffy_delta, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 static struct kobj_attribute trigger_time_attribute =
-	__ATTR(trigger_time, S_IWUSR|S_IRUGO, spk_var_show, spk_var_store);
+	__ATTR(trigger_time, S_IWUSR | S_IRUGO, spk_var_show, spk_var_store);
 
 /*
  * Create a group of attributes so that we can create and destroy them all
diff --git a/drivers/staging/speakup/spk_priv_keyinfo.h b/drivers/staging/speakup/spk_priv_keyinfo.h
index 130e9cb0118b..c95b68ebd8e7 100644
--- a/drivers/staging/speakup/spk_priv_keyinfo.h
+++ b/drivers/staging/speakup/spk_priv_keyinfo.h
@@ -23,84 +23,82 @@
 
 #define FIRST_SYNTH_VAR RATE
 /* 0 is reserved for no remap */
-#define SPEAKUP_GOTO 0x01
-#define SPEECH_KILL 0x02
-#define SPEAKUP_QUIET 0x03
-#define SPEAKUP_CUT 0x04
-#define SPEAKUP_PASTE 0x05
-#define SAY_FIRST_CHAR 0x06
-#define SAY_LAST_CHAR 0x07
-#define SAY_CHAR 0x08
-#define SAY_PREV_CHAR 0x09
-#define SAY_NEXT_CHAR 0x0a
-#define SAY_WORD 0x0b
-#define SAY_PREV_WORD 0x0c
-#define SAY_NEXT_WORD 0x0d
-#define SAY_LINE 0x0e
-#define SAY_PREV_LINE 0x0f
-#define SAY_NEXT_LINE 0x10
-#define TOP_EDGE 0x11
-#define BOTTOM_EDGE 0x12
-#define LEFT_EDGE 0x13
-#define RIGHT_EDGE 0x14
-#define SPELL_PHONETIC 0x15
-#define SPELL_WORD 0x16
-#define SAY_SCREEN 0x17
-#define SAY_POSITION 0x18
-#define SAY_ATTRIBUTES 0x19
-#define SPEAKUP_OFF 0x1a
-#define SPEAKUP_PARKED 0x1b
-#define SAY_LINE_INDENT 0x1c
-#define SAY_FROM_TOP 0x1d
-#define SAY_TO_BOTTOM 0x1e
-#define SAY_FROM_LEFT 0x1f
-#define SAY_TO_RIGHT 0x20
-#define SAY_CHAR_NUM 0x21
-#define EDIT_SOME 0x22
-#define EDIT_MOST 0x23
-#define SAY_PHONETIC_CHAR 0x24
-#define EDIT_DELIM 0x25
-#define EDIT_REPEAT 0x26
-#define EDIT_EXNUM 0x27
-#define SET_WIN 0x28
-#define CLEAR_WIN 0x29
-#define ENABLE_WIN 0x2a
-#define SAY_WIN 0x2b
-#define SPK_LOCK 0x2c
-#define SPEAKUP_HELP 0x2d
-#define TOGGLE_CURSORING 0x2e
-#define READ_ALL_DOC 0x2f
-#define SPKUP_MAX_FUNC 0x30 /* one greater than the last func handler */
-
-#define SPK_KEY 0x80
-#define FIRST_EDIT_BITS 0x22
-
+#define SPEAKUP_GOTO		0x01
+#define SPEECH_KILL		0x02
+#define SPEAKUP_QUIET		0x03
+#define SPEAKUP_CUT		0x04
+#define SPEAKUP_PASTE		0x05
+#define SAY_FIRST_CHAR		0x06
+#define SAY_LAST_CHAR		0x07
+#define SAY_CHAR		0x08
+#define SAY_PREV_CHAR		0x09
+#define SAY_NEXT_CHAR		0x0a
+#define SAY_WORD		0x0b
+#define SAY_PREV_WORD		0x0c
+#define SAY_NEXT_WORD		0x0d
+#define SAY_LINE		0x0e
+#define SAY_PREV_LINE		0x0f
+#define SAY_NEXT_LINE		0x10
+#define TOP_EDGE		0x11
+#define BOTTOM_EDGE		0x12
+#define LEFT_EDGE		0x13
+#define RIGHT_EDGE		0x14
+#define SPELL_PHONETIC		0x15
+#define SPELL_WORD		0x16
+#define SAY_SCREEN		0x17
+#define SAY_POSITION		0x18
+#define SAY_ATTRIBUTES		0x19
+#define SPEAKUP_OFF		0x1a
+#define SPEAKUP_PARKED		0x1b
+#define SAY_LINE_INDENT	0x1c
+#define SAY_FROM_TOP		0x1d
+#define SAY_TO_BOTTOM		0x1e
+#define SAY_FROM_LEFT		0x1f
+#define SAY_TO_RIGHT		0x20
+#define SAY_CHAR_NUM		0x21
+#define EDIT_SOME		0x22
+#define EDIT_MOST		0x23
+#define SAY_PHONETIC_CHAR	0x24
+#define EDIT_DELIM		0x25
+#define EDIT_REPEAT		0x26
+#define EDIT_EXNUM		0x27
+#define SET_WIN		0x28
+#define CLEAR_WIN		0x29
+#define ENABLE_WIN		0x2a
+#define SAY_WIN		0x2b
+#define SPK_LOCK		0x2c
+#define SPEAKUP_HELP		0x2d
+#define TOGGLE_CURSORING	0x2e
+#define READ_ALL_DOC		0x2f
+#define SPKUP_MAX_FUNC		0x30 /* one greater than the last func handler */
+#define SPK_KEY		0x80
+#define FIRST_EDIT_BITS	0x22
 #define FIRST_SET_VAR SPELL_DELAY
-#define VAR_START 0x40 /* increase if adding more than 0x3f functions */
+#define VAR_START		0x40 /* increase if adding more than 0x3f functions */
 
 /* keys for setting variables, must be ordered same as the enum for var_ids */
 /* with dec being even and inc being 1 greater */
-#define SPELL_DELAY_DEC (VAR_START+0)
-#define SPELL_DELAY_INC (SPELL_DELAY_DEC+1)
-#define PUNC_LEVEL_DEC (SPELL_DELAY_DEC+2)
-#define PUNC_LEVEL_INC (PUNC_LEVEL_DEC+1)
-#define READING_PUNC_DEC (PUNC_LEVEL_DEC+2)
-#define READING_PUNC_INC (READING_PUNC_DEC+1)
-#define ATTRIB_BLEEP_DEC (READING_PUNC_DEC+2)
-#define ATTRIB_BLEEP_INC (ATTRIB_BLEEP_DEC+1)
-#define BLEEPS_DEC (ATTRIB_BLEEP_DEC+2)
-#define BLEEPS_INC (BLEEPS_DEC+1)
-#define RATE_DEC (BLEEPS_DEC+2)
-#define RATE_INC (RATE_DEC+1)
-#define PITCH_DEC (RATE_DEC+2)
-#define PITCH_INC (PITCH_DEC+1)
-#define VOL_DEC (PITCH_DEC+2)
-#define VOL_INC (VOL_DEC+1)
-#define TONE_DEC (VOL_DEC+2)
-#define TONE_INC (TONE_DEC+1)
-#define PUNCT_DEC (TONE_DEC+2)
-#define PUNCT_INC (PUNCT_DEC+1)
-#define VOICE_DEC (PUNCT_DEC+2)
-#define VOICE_INC (VOICE_DEC+1)
+#define SPELL_DELAY_DEC (VAR_START + 0)
+#define SPELL_DELAY_INC (SPELL_DELAY_DEC + 1)
+#define PUNC_LEVEL_DEC (SPELL_DELAY_DEC + 2)
+#define PUNC_LEVEL_INC (PUNC_LEVEL_DEC + 1)
+#define READING_PUNC_DEC (PUNC_LEVEL_DEC + 2)
+#define READING_PUNC_INC (READING_PUNC_DEC + 1)
+#define ATTRIB_BLEEP_DEC (READING_PUNC_DEC + 2)
+#define ATTRIB_BLEEP_INC (ATTRIB_BLEEP_DEC + 1)
+#define BLEEPS_DEC (ATTRIB_BLEEP_DEC + 2)
+#define BLEEPS_INC (BLEEPS_DEC + 1)
+#define RATE_DEC (BLEEPS_DEC + 2)
+#define RATE_INC (RATE_DEC + 1)
+#define PITCH_DEC (RATE_DEC + 2)
+#define PITCH_INC (PITCH_DEC + 1)
+#define VOL_DEC (PITCH_DEC + 2)
+#define VOL_INC (VOL_DEC + 1)
+#define TONE_DEC (VOL_DEC + 2)
+#define TONE_INC (TONE_DEC + 1)
+#define PUNCT_DEC (TONE_DEC + 2)
+#define PUNCT_INC (PUNCT_DEC + 1)
+#define VOICE_DEC (PUNCT_DEC + 2)
+#define VOICE_INC (VOICE_DEC + 1)
 
 #endif
diff --git a/drivers/staging/speakup/spk_types.h b/drivers/staging/speakup/spk_types.h
index e8ff5d7d6419..b07f6cc4f284 100644
--- a/drivers/staging/speakup/spk_types.h
+++ b/drivers/staging/speakup/spk_types.h
@@ -1,16 +1,14 @@
 #ifndef SPEAKUP_TYPES_H
 #define SPEAKUP_TYPES_H
 
-/*
- * This file includes all of the typedefs and structs used in speakup.
- */
+/* This file includes all of the typedefs and structs used in speakup. */
 
 #include <linux/types.h>
 #include <linux/fs.h>
 #include <linux/errno.h>
 #include <linux/delay.h>
 #include <linux/wait.h>		/* for wait_queue */
-#include <linux/init.h> /* for __init */
+#include <linux/init.h>		/* for __init */
 #include <linux/module.h>
 #include <linux/vt_kern.h>
 #include <linux/spinlock.h>
@@ -105,7 +103,7 @@ struct st_var_header {
 	enum var_id_t var_id;
 	enum var_type_t var_type;
 	void *p_val; /* ptr to programs variable to store value */
-	void *data; /* ptr to the vars data */
+	void *data;  /* ptr to the vars data */
 };
 
 struct num_var_t {
@@ -114,8 +112,8 @@ struct num_var_t {
 	int low;
 	int high;
 	short offset, multiplier; /* for fiddling rates etc. */
-	char *out_str; /* if synth needs char representation of number */
-	int value; /* current value */
+	char *out_str;  /* if synth needs char representation of number */
+	int value;	/* current value */
 };
 
 struct punc_var_t {
@@ -169,7 +167,7 @@ struct spk_synth {
 	int (*probe)(struct spk_synth *synth);
 	void (*release)(void);
 	const char *(*synth_immediate)(struct spk_synth *synth,
-					const char *buff);
+				       const char *buff);
 	void (*catch_up)(struct spk_synth *synth);
 	void (*flush)(struct spk_synth *synth);
 	int (*is_alive)(struct spk_synth *synth);
@@ -181,7 +179,7 @@ struct spk_synth {
 	struct attribute_group attributes;
 };
 
-/**
+/*
  * module_spk_synth() - Helper macro for registering a speakup driver
  * @__spk_synth: spk_synth struct
  * Helper macro for speakup drivers which do not do anything special in module
diff --git a/drivers/staging/speakup/synth.c b/drivers/staging/speakup/synth.c
index 54b2f3918628..a61c02ba06da 100644
--- a/drivers/staging/speakup/synth.c
+++ b/drivers/staging/speakup/synth.c
@@ -8,7 +8,7 @@
 #include <linux/delay.h>	/* for loops_per_sec */
 #include <linux/kmod.h>
 #include <linux/jiffies.h>
-#include <linux/uaccess.h> /* for copy_from_user */
+#include <linux/uaccess.h>	/* for copy_from_user */
 #include <linux/sched.h>
 #include <linux/timer.h>
 #include <linux/kthread.h>
@@ -67,13 +67,14 @@ int spk_serial_synth_probe(struct spk_synth *synth)
 		return -ENODEV;
 	}
 	pr_info("%s: ttyS%i, Driver Version %s\n",
-			synth->long_name, synth->ser, synth->version);
+		synth->long_name, synth->ser, synth->version);
 	synth->alive = 1;
 	return 0;
 }
 EXPORT_SYMBOL_GPL(spk_serial_synth_probe);
 
-/* Main loop of the progression thread: keep eating from the buffer
+/*
+ * Main loop of the progression thread: keep eating from the buffer
  * and push to the serial port, waiting as needed
  *
  * For devices that have a "full" notification mechanism, the driver can
@@ -303,12 +304,11 @@ void spk_get_index_count(int *linecount, int *sentcount)
 		sentence_count = ind % 10;
 
 		if ((ind / 10) <= synth->indexing.currindex)
-			index_count = synth->indexing.currindex-(ind/10);
+			index_count = synth->indexing.currindex - (ind / 10);
 		else
 			index_count = synth->indexing.currindex
-				-synth->indexing.lowindex
-				+ synth->indexing.highindex-(ind/10)+1;
-
+				- synth->indexing.lowindex
+				+ synth->indexing.highindex - (ind / 10) + 1;
 	}
 	*sentcount = sentence_count;
 	*linecount = index_count;
@@ -406,8 +406,8 @@ static int do_synth_init(struct spk_synth *in_synth)
 		speakup_register_var(var);
 	if (!spk_quiet_boot)
 		synth_printf("%s found\n", synth->long_name);
-	if (synth->attributes.name
-	&& sysfs_create_group(speakup_kobj, &synth->attributes) < 0)
+	if (synth->attributes.name && sysfs_create_group(speakup_kobj,
+							 &synth->attributes) < 0)
 		return -ENOMEM;
 	synth_flags = synth->flags;
 	wake_up_interruptible_all(&speakup_event);
@@ -476,10 +476,10 @@ void synth_remove(struct spk_synth *in_synth)
 			break;
 	}
 	for ( ; synths[i] != NULL; i++) /* compress table */
-		synths[i] = synths[i+1];
+		synths[i] = synths[i + 1];
 	module_status = 0;
 	mutex_unlock(&spk_mutex);
 }
 EXPORT_SYMBOL_GPL(synth_remove);
 
-short spk_punc_masks[] = { 0, SOME, MOST, PUNC, PUNC|B_SYM };
+short spk_punc_masks[] = { 0, SOME, MOST, PUNC, PUNC | B_SYM };
diff --git a/drivers/staging/speakup/thread.c b/drivers/staging/speakup/thread.c
index 90c383ee7c3f..8c64f1ada6e0 100644
--- a/drivers/staging/speakup/thread.c
+++ b/drivers/staging/speakup/thread.c
@@ -27,7 +27,7 @@ int speakup_thread(void *data)
 			our_sound = spk_unprocessed_sound;
 			spk_unprocessed_sound.active = 0;
 			prepare_to_wait(&speakup_event, &wait,
-				TASK_INTERRUPTIBLE);
+					TASK_INTERRUPTIBLE);
 			should_break = kthread_should_stop() ||
 				our_sound.active ||
 				(synth && synth->catch_up && synth->alive &&
@@ -47,7 +47,8 @@ int speakup_thread(void *data)
 		if (our_sound.active)
 			kd_mksound(our_sound.freq, our_sound.jiffies);
 		if (synth && synth->catch_up && synth->alive) {
-			/* It is up to the callee to take the lock, so that it
+			/*
+			 * It is up to the callee to take the lock, so that it
 			 * can sleep whenever it likes
 			 */
 			synth->catch_up(synth);
diff --git a/drivers/staging/speakup/varhandlers.c b/drivers/staging/speakup/varhandlers.c
index 21186e3dc7ad..cc984196020f 100644
--- a/drivers/staging/speakup/varhandlers.c
+++ b/drivers/staging/speakup/varhandlers.c
@@ -237,8 +237,7 @@ int spk_set_num_var(int input, struct st_var_header *var, int how)
 	if (!var_data->u.n.out_str)
 		l = sprintf(cp, var_data->u.n.synth_fmt, (int)val);
 	else
-		l = sprintf(cp,
-			var_data->u.n.synth_fmt, var_data->u.n.out_str[val]);
+		l = sprintf(cp,	var_data->u.n.synth_fmt, var_data->u.n.out_str[val]);
 	synth_printf("%s", cp);
 	return 0;
 }
@@ -266,7 +265,8 @@ int spk_set_string_var(const char *page, struct st_var_header *var, int len)
 	return 0;
 }
 
-/* spk_set_mask_bits sets or clears the punc/delim/repeat bits,
+/*
+ * spk_set_mask_bits sets or clears the punc/delim/repeat bits,
  * if input is null uses the defaults.
  * values for how: 0 clears bits of chars supplied,
  * 1 clears allk, 2 sets bits for chars
diff --git a/drivers/staging/unisys/include/iochannel.h b/drivers/staging/unisys/include/iochannel.h
index 9081b3f8779c..54f490090a59 100644
--- a/drivers/staging/unisys/include/iochannel.h
+++ b/drivers/staging/unisys/include/iochannel.h
@@ -1,20 +1,20 @@
-/* Copyright (C) 2010 - 2013 UNISYS CORPORATION */
+/* Copyright (C) 2010 - 2016 UNISYS CORPORATION */
 /* All rights reserved. */
 #ifndef __IOCHANNEL_H__
 #define __IOCHANNEL_H__
 
 /*
  * Everything needed for IOPart-GuestPart communication is define in
- * this file.  Note: Everything is OS-independent because this file is
+ * this file. Note: Everything is OS-independent because this file is
  * used by Windows, Linux and possible EFI drivers.
  */
 
 /*
  * Communication flow between the IOPart and GuestPart uses the channel headers
- * channel state.  The following states are currently being used:
+ * channel state. The following states are currently being used:
  *       UNINIT(All Zeroes), CHANNEL_ATTACHING, CHANNEL_ATTACHED, CHANNEL_OPENED
  *
- * additional states will be used later.  No locking is needed to switch between
+ * Additional states will be used later. No locking is needed to switch between
  * states due to the following rules:
  *
  *      1.  IOPart is only the only partition allowed to change from UNIT
@@ -39,10 +39,11 @@
 #define ULTRA_VSWITCH_CHANNEL_PROTOCOL_SIGNATURE \
 	ULTRA_CHANNEL_PROTOCOL_SIGNATURE
 
-/* Must increment these whenever you insert or delete fields within this channel
- * struct.  Also increment whenever you change the meaning of fields within this
- * channel struct so as to break pre-existing software.  Note that you can
- * usually add fields to the END of the channel struct withOUT needing to
+/*
+ * Must increment these whenever you insert or delete fields within this channel
+ * struct. Also increment whenever you change the meaning of fields within this
+ * channel struct so as to break pre-existing software. Note that you can
+ * usually add fields to the END of the channel struct without needing to
  * increment this.
  */
 #define ULTRA_VHBA_CHANNEL_PROTOCOL_VERSIONID 2
@@ -70,59 +71,62 @@
 #define MINNUM(a, b) (((a) < (b)) ? (a) : (b))
 #define MAXNUM(a, b) (((a) > (b)) ? (a) : (b))
 
-/* define the two queues per data channel between iopart and ioguestparts */
-/* used by ioguestpart to 'insert' signals to iopart */
+/* Define the two queues per data channel between iopart and ioguestparts. */
+/* Used by ioguestpart to 'insert' signals to iopart. */
 #define IOCHAN_TO_IOPART 0
-/* used by ioguestpart to 'remove' signals from iopart, same previous queue */
+/* Used by ioguestpart to 'remove' signals from iopart, same previous queue. */
 #define IOCHAN_FROM_IOPART 1
 
-/* size of cdb - i.e., scsi cmnd */
+/* Size of cdb - i.e., SCSI cmnd */
 #define MAX_CMND_SIZE 16
 
 #define MAX_SENSE_SIZE 64
 
 #define MAX_PHYS_INFO 64
 
-/* various types of network packets that can be sent in cmdrsp */
+/* Various types of network packets that can be sent in cmdrsp. */
 enum net_types {
-	NET_RCV_POST = 0,	/* submit buffer to hold receiving
+	NET_RCV_POST = 0,	/*
+				 * Submit buffer to hold receiving
 				 * incoming packet
 				 */
-	/* virtnic -> uisnic */
+	/* visornic -> uisnic */
 	NET_RCV,		/* incoming packet received */
 	/* uisnic -> virtpci */
 	NET_XMIT,		/* for outgoing net packets */
-	/* virtnic -> uisnic */
+	/* visornic -> uisnic */
 	NET_XMIT_DONE,		/* outgoing packet xmitted */
 	/* uisnic -> virtpci */
 	NET_RCV_ENBDIS,		/* enable/disable packet reception */
-	/* virtnic -> uisnic */
+	/* visornic -> uisnic */
 	NET_RCV_ENBDIS_ACK,	/* acknowledge enable/disable packet */
 				/* reception */
-	/* uisnic -> virtnic */
+	/* uisnic -> visornic */
 	NET_RCV_PROMISC,	/* enable/disable promiscuous mode */
-	/* virtnic -> uisnic */
-	NET_CONNECT_STATUS,	/* indicate the loss or restoration of a network
+	/* visornic -> uisnic */
+	NET_CONNECT_STATUS,	/*
+				 * indicate the loss or restoration of a network
 				 * connection
 				 */
-	/* uisnic -> virtnic */
-	NET_MACADDR,		/* indicates the client has requested to update
-				 * its MAC addr
+	/* uisnic -> visornic */
+	NET_MACADDR,		/*
+				 * Indicates the client has requested to update
+				 * it's MAC address
 				 */
-	NET_MACADDR_ACK,	/* MAC address */
+	NET_MACADDR_ACK,	/* MAC address acknowledge */
 
 };
 
-#define		ETH_MIN_DATA_SIZE 46	/* minimum eth data size */
-#define		ETH_MIN_PACKET_SIZE (ETH_HLEN + ETH_MIN_DATA_SIZE)
+#define ETH_MIN_DATA_SIZE 46	/* minimum eth data size */
+#define ETH_MIN_PACKET_SIZE (ETH_HLEN + ETH_MIN_DATA_SIZE)
 
-#define		VISOR_ETH_MAX_MTU 16384	/* maximum data size */
+#define VISOR_ETH_MAX_MTU 16384	/* maximum data size */
 
 #ifndef MAX_MACADDR_LEN
 #define MAX_MACADDR_LEN 6	/* number of bytes in MAC address */
-#endif				/* MAX_MACADDR_LEN */
+#endif
 
-/* various types of scsi task mgmt commands  */
+/* Various types of scsi task mgmt commands. */
 enum task_mgmt_types {
 	TASK_MGMT_ABORT_TASK = 1,
 	TASK_MGMT_BUS_RESET,
@@ -130,7 +134,7 @@ enum task_mgmt_types {
 	TASK_MGMT_TARGET_RESET,
 };
 
-/* various types of vdisk mgmt commands  */
+/* Various types of vdisk mgmt commands. */
 enum vdisk_mgmt_types {
 	VDISK_MGMT_ACQUIRE = 1,
 	VDISK_MGMT_RELEASE,
@@ -144,7 +148,7 @@ struct phys_info {
 
 #define MIN_NUMSIGNALS 64
 
-/* structs with pragma pack  */
+/* Structs with pragma pack. */
 
 struct guest_phys_info {
 	u64 address;
@@ -154,9 +158,9 @@ struct guest_phys_info {
 #define GPI_ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(struct guest_phys_info))
 
 struct uisscsi_dest {
-	u32 channel;		/* channel == bus number */
-	u32 id;			/* id == target number */
-	u32 lun;		/* lun == logical unit number */
+	u32 channel;	/* channel == bus number */
+	u32 id;		/* id == target number */
+	u32 lun;	/* lun == logical unit number */
 } __packed;
 
 struct vhba_wwnn {
@@ -164,7 +168,8 @@ struct vhba_wwnn {
 	u32 wwnn2;
 } __packed;
 
-/* WARNING: Values stired in this structure must contain maximum counts (not
+/*
+ * WARNING: Values stired in this structure must contain maximum counts (not
  * maximum values).
  */
 struct vhba_config_max {/* 20 bytes */
@@ -187,23 +192,24 @@ struct uiscmdrsp_scsi {
 							 * information for each
 							 * fragment
 							 */
-	enum dma_data_direction  data_dir; /* direction of the data, if any */
+	enum dma_data_direction data_dir; /* direction of the data, if any */
 	struct uisscsi_dest vdest;	/* identifies the virtual hba, id, */
 					/* channel, lun to which cmd was sent */
 
-	/* Needed to queue the rsp back to cmd originator */
-	int linuxstat;		/* original Linux status used by linux vdisk */
+	/* Needed to queue the rsp back to cmd originator. */
+	int linuxstat;		/* original Linux status used by Linux vdisk */
 	u8 scsistat;		/* the scsi status */
 	u8 addlstat;		/* non-scsi status */
 #define ADDL_SEL_TIMEOUT	4
 
-	/* the following fields are need to determine the result of command */
+	/* The following fields are need to determine the result of command. */
 	 u8 sensebuf[MAX_SENSE_SIZE];	/* sense info in case cmd failed; */
-	/* it holds the sense_data struct; */
-	/* see that struct for details. */
-	void *vdisk; /* pointer to the vdisk to clean up when IO completes. */
+	/* sensebuf holds the sense_data struct; */
+	/* See sense_data struct for more details. */
+	void *vdisk; /* Pointer to the vdisk to clean up when IO completes. */
 	int no_disk_result;
-	/* used to return no disk inquiry result
+	/*
+	 * Used to return no disk inquiry result
 	 * when no_disk_result is set to 1,
 	 * scsi.scsistat is SAM_STAT_GOOD
 	 * scsi.addlstat is 0
@@ -212,35 +218,44 @@ struct uiscmdrsp_scsi {
 	 */
 } __packed;
 
-/* Defines to support sending correct inquiry result when no disk is
+/*
+ * Defines to support sending correct inquiry result when no disk is
  * configured.
  */
 
-/* From SCSI SPC2 -
+/*
+ * From SCSI SPC2 -
  *
  * If the target is not capable of supporting a device on this logical unit, the
  * device server shall set this field to 7Fh (PERIPHERAL QUALIFIER set to 011b
  * and PERIPHERAL DEVICE TYPE set to 1Fh).
  *
- *The device server is capable of supporting the specified peripheral device
- *type on this logical unit. However, the physical device is not currently
- *connected to this logical unit.
+ * The device server is capable of supporting the specified peripheral device
+ * type on this logical unit. However, the physical device is not currently
+ * connected to this logical unit.
  */
 
-#define DEV_NOT_CAPABLE 0x7f	/* peripheral qualifier of 0x3  */
-				/* peripheral type of 0x1f */
-				/* specifies no device but target present */
+#define DEV_NOT_CAPABLE 0x7f	/*
+				 * peripheral qualifier of 0x3
+				 * peripheral type of 0x1f
+				 * specifies no device but target present
+				 */
 
-#define DEV_DISK_CAPABLE_NOT_PRESENT 0x20 /* peripheral qualifier of 0x1 */
-    /* peripheral type of 0 - disk */
-    /* specifies device capable, but not present */
+#define DEV_DISK_CAPABLE_NOT_PRESENT 0x20	/* peripheral qualifier of 0x1
+						 * peripheral type of 0 - disk
+						 * Specifies device capable, but
+						 * not present
+						 */
 
-#define DEV_HISUPPORT 0x10	/* HiSup = 1; shows support for report luns */
-				/* must be returned for lun 0. */
+#define DEV_HISUPPORT 0x10	/*
+				 * HiSup = 1; shows support for report luns
+				 * must be returned for lun 0.
+				 */
 
-/* NOTE: Linux code assumes inquiry contains 36 bytes. Without checking length
- * in buf[4] some linux code accesses bytes beyond 5 to retrieve vendor, product
- * & revision.  Yikes! So let us always send back 36 bytes, the minimum for
+/*
+ * NOTE: Linux code assumes inquiry contains 36 bytes. Without checking length
+ * in buf[4] some Linux code accesses bytes beyond 5 to retrieve vendor, product
+ * and revision. Yikes! So let us always send back 36 bytes, the minimum for
  * inquiry result.
  */
 #define NO_DISK_INQUIRY_RESULT_LEN 36
@@ -248,11 +263,12 @@ struct uiscmdrsp_scsi {
 #define MIN_INQUIRY_RESULT_LEN 5 /* 5 bytes minimum for inquiry result */
 
 /* SCSI device version for no disk inquiry result */
-#define SCSI_SPC2_VER 4		/* indicates SCSI SPC2 (SPC3 is 5) */
+#define SCSI_SPC2_VER 4	/* indicates SCSI SPC2 (SPC3 is 5) */
 
-/* Struct & Defines to support sense information. */
+/* Struct and Defines to support sense information. */
 
-/* The following struct is returned in sensebuf field in uiscmdrsp_scsi.  It is
+/*
+ * The following struct is returned in sensebuf field in uiscmdrsp_scsi. It is
  * initialized in exactly the manner that is recommended in Windows (hence the
  * odd values).
  * When set, these fields will have the following values:
@@ -288,7 +304,7 @@ struct net_pkt_xmt {
 	struct phys_info frags[MAX_PHYS_INFO];	/* physical page information */
 	char ethhdr[ETH_HLEN];	/* the ethernet header  */
 	struct {
-		/* these are needed for csum at uisnic end */
+		/* These are needed for csum at uisnic end */
 		u8 valid;	/* 1 = struct is valid - else ignore */
 		u8 hrawoffv;	/* 1 = hwrafoff is valid */
 		u8 nhrawoffv;	/* 1 = nhwrafoff is valid */
@@ -300,7 +316,8 @@ struct net_pkt_xmt {
 		/* nhrawoff points to the start of the NETWORK LAYER HEADER */
 	} lincsum;
 
-	    /* **** NOTE ****
+	    /*
+	     * NOTE:
 	     * The full packet is described in frags but the ethernet header is
 	     * separately kept in ethhdr so that uisnic doesn't have "MAP" the
 	     * guest memory to get to the header. uisnic needs ethhdr to
@@ -309,14 +326,15 @@ struct net_pkt_xmt {
 } __packed;
 
 struct net_pkt_xmtdone {
-	u32 xmt_done_result;	/* result of NET_XMIT */
+	u32 xmt_done_result; /* result of NET_XMIT */
 } __packed;
 
-/* RCVPOST_BUF_SIZe must be at most page_size(4096) - cache_line_size (64) The
+/*
+ * RCVPOST_BUF_SIZE must be at most page_size(4096) - cache_line_size (64) The
  * reason is because dev_skb_alloc which is used to generate RCV_POST skbs in
- * virtnic requires that there is "overhead" in the buffer, and pads 16 bytes. I
- * prefer to use 1 full cache line size for "overhead" so that transfers are
- * better.  IOVM requires that a buffer be represented by 1 phys_info structure
+ * visornic requires that there is "overhead" in the buffer, and pads 16 bytes.
+ * Use 1 full cache line size for "overhead" so that transfers are optimized.
+ * IOVM requires that a buffer be represented by 1 phys_info structure
  * which can only cover page_size.
  */
 #define RCVPOST_BUF_SIZE 4032
@@ -324,26 +342,38 @@ struct net_pkt_xmtdone {
 	((VISOR_ETH_MAX_MTU + ETH_HLEN + RCVPOST_BUF_SIZE - 1) \
 	/ RCVPOST_BUF_SIZE)
 
+/*
+ * rcv buf size must be large enough to include ethernet data len + ethernet
+ * header len - we are choosing 2K because it is guaranteed to be describable.
+ */
 struct net_pkt_rcvpost {
-	    /* rcv buf size must be large enough to include ethernet data len +
-	     * ethernet header len - we are choosing 2K because it is guaranteed
-	     * to be describable
-	     */
-	    struct phys_info frag;	/* physical page information for the */
-					/* single fragment 2K rcv buf */
-	    u64 unique_num;
-	    /* unique_num ensure that receive posts are returned to */
-	    /* the Adapter which we sent them originally. */
+	/* Physical page information for the single fragment 2K rcv buf */
+	struct phys_info frag;
+
+	/*
+	 * Ensures that receive posts are returned to the adapter which we sent
+	 * them from originally.
+	 */
+	u64 unique_num;
+
 } __packed;
 
+/*
+ * The number of rcvbuf that can be chained is based on max mtu and size of each
+ * rcvbuf.
+ */
 struct net_pkt_rcv {
-	/* the number of receive buffers that can be chained  */
-	/* is based on max mtu and size of each rcv buf */
-	u32 rcv_done_len;	/* length of received data */
-	u8 numrcvbufs;		/* number of receive buffers that contain the */
-	/* incoming data; guest end MUST chain these together. */
-	void *rcvbuf[MAX_NET_RCV_CHAIN];	/* list of chained rcvbufs */
-	/* each entry is a receive buffer provided by NET_RCV_POST. */
+	u32 rcv_done_len; /* length of received data */
+
+	/*
+	 * numrcvbufs: contain the incoming data; guest side MUST chain these
+	 * together.
+	 */
+	u8 numrcvbufs;
+
+	void *rcvbuf[MAX_NET_RCV_CHAIN]; /* list of chained rcvbufs */
+
+	/* Each entry is a receive buffer provided by NET_RCV_POST. */
 	/* NOTE: first rcvbuf in the chain will also be provided in net.buf. */
 	u64 unique_num;
 	u32 rcvs_dropped_delta;
@@ -351,12 +381,12 @@ struct net_pkt_rcv {
 
 struct net_pkt_enbdis {
 	void *context;
-	u16 enable;		/* 1 = enable, 0 = disable */
+	u16 enable; /* 1 = enable, 0 = disable */
 } __packed;
 
 struct net_pkt_macaddr {
 	void *context;
-	u8 macaddr[MAX_MACADDR_LEN];	/* 6 bytes */
+	u8 macaddr[MAX_MACADDR_LEN]; /* 6 bytes */
 } __packed;
 
 /* cmd rsp packet used for VNIC network traffic  */
@@ -377,41 +407,44 @@ struct uiscmdrsp_net {
 } __packed;
 
 struct uiscmdrsp_scsitaskmgmt {
+	/* The type of task. */
 	enum task_mgmt_types tasktype;
 
-	    /* the type of task */
+	/* The vdisk for which this task mgmt is generated. */
 	struct uisscsi_dest vdest;
 
-	    /* the vdisk for which this task mgmt is generated */
+	/*
+	 * This is a handle that the guest has saved off for its own use.
+	 * The handle value is preserved by iopart and returned as in task
+	 * mgmt rsp.
+	 */
 	u64 handle;
 
-	    /* This is a handle that the guest has saved off for its own use.
-	     * Its value is preserved by iopart & returned as is in the task
-	     * mgmt rsp.
-	     */
+	/*
+	 * For Linux guests, this is a pointer to wait_queue_head that a
+	 * thread is waiting on to see if the taskmgmt command has completed.
+	 * When the rsp is received by guest, the thread receiving the
+	 * response uses this to notify the thread waiting for taskmgmt
+	 * command completion. It's value is preserved by iopart and returned
+	 * as in the task mgmt rsp.
+	 */
 	u64 notify_handle;
 
-	   /* For linux guests, this is a pointer to wait_queue_head that a
-	    * thread is waiting on to see if the taskmgmt command has completed.
-	    * When the rsp is received by guest, the thread receiving the
-	    * response uses this to notify the thread waiting for taskmgmt
-	    * command completion.  Its value is preserved by iopart & returned
-	    * as is in the task mgmt rsp.
-	    */
+	/*
+	 * This is a handle to the location in the guest where the result of
+	 * the taskmgmt command (result field) is saved to when the response
+	 * is handled. It's value is preserved by iopart and returned as in
+	 * the task mgmt rsp.
+	 */
 	u64 notifyresult_handle;
 
-	    /* this is a handle to location in guest where the result of the
-	     * taskmgmt command (result field) is to saved off when the response
-	     * is handled.  Its value is preserved by iopart & returned as is in
-	     * the task mgmt rsp.
-	     */
+	/* Result of taskmgmt command - set by IOPart - values are: */
 	char result;
 
-	    /* result of taskmgmt command - set by IOPart - values are: */
 #define TASK_MGMT_FAILED  0
 } __packed;
 
-/* Used by uissd to send disk add/remove notifications to Guest */
+/* Used by uissd to send disk add/remove notifications to Guest. */
 /* Note that the vHba pointer is not used by the Client/Guest side. */
 struct uiscmdrsp_disknotify {
 	u8 add;			/* 0-remove, 1-add */
@@ -419,49 +452,50 @@ struct uiscmdrsp_disknotify {
 	u32 channel, id, lun;	/* SCSI Path of Disk to added or removed */
 } __packed;
 
-/* The following is used by virthba/vSCSI to send the Acquire/Release commands
+/*
+ * The following is used by virthba/vSCSI to send the Acquire/Release commands
  * to the IOVM.
  */
 struct uiscmdrsp_vdiskmgmt {
+	/* The type of task */
 	enum vdisk_mgmt_types vdisktype;
 
-	    /* the type of task */
+	/* The vdisk for which this task mgmt is generated */
 	struct uisscsi_dest vdest;
 
-	    /* the vdisk for which this task mgmt is generated */
+	/*
+	 * This is a handle that the guest has saved off for its own use. It's
+	 * value is preserved by iopart and returned as in the task mgmt rsp.
+	 */
 	u64 handle;
 
-	    /* This is a handle that the guest has saved off for its own use.
-	     * Its value is preserved by iopart & returned as is in the task
-	     * mgmt rsp.
-	     */
+	/*
+	 * For Linux guests, this is a pointer to wait_queue_head that a
+	 * thread is waiting on to see if the tskmgmt command has completed.
+	 * When the rsp is received by guest, the thread receiving the
+	 * response uses this to notify the thread waiting for taskmgmt
+	 * command completion. It's value is preserved by iopart and returned
+	 * as in the task mgmt rsp.
+	 */
 	u64 notify_handle;
 
-	    /* For linux guests, this is a pointer to wait_queue_head that a
-	     * thread is waiting on to see if the tskmgmt command has completed.
-	     * When the rsp is received by guest, the thread receiving the
-	     * response uses this to notify the thread waiting for taskmgmt
-	     * command completion.  Its value is preserved by iopart & returned
-	     * as is in the task mgmt rsp.
-	     */
+	/*
+	 * Handle to the location in guest where the result of the
+	 * taskmgmt command (result field) is saved to when the response
+	 * is handled. It's value is preserved by iopart and returned as in
+	 * the task mgmt rsp.
+	 */
 	u64 notifyresult_handle;
 
-	    /* this is a handle to location in guest where the result of the
-	     * taskmgmt command (result field) is to saved off when the response
-	     * is handled.  Its value is preserved by iopart & returned as is in
-	     * the task mgmt rsp.
-	     */
+	/* Result of taskmgmt command - set by IOPart - values are: */
 	char result;
-
-	    /* result of taskmgmt command - set by IOPart - values are: */
-#define VDISK_MGMT_FAILED  0
 } __packed;
 
-/* keeping cmd & rsp info in one structure for now cmd rsp packet for scsi */
+/* Keeping cmd and rsp info in one structure for now cmd rsp packet for SCSI */
 struct uiscmdrsp {
 	char cmdtype;
 
-/* describes what type of information is in the struct */
+/* Describes what type of information is in the struct */
 #define CMD_SCSI_TYPE		1
 #define CMD_NET_TYPE		2
 #define CMD_SCSITASKMGMT_TYPE	3
@@ -474,11 +508,11 @@ struct uiscmdrsp {
 		struct uiscmdrsp_disknotify disknotify;
 		struct uiscmdrsp_vdiskmgmt vdiskmgmt;
 	};
-	void *private_data;	/* send the response when the cmd is */
-				/* done (scsi & scsittaskmgmt). */
+	/* Send the response when the cmd is done (scsi and scsittaskmgmt). */
+	void *private_data;
 	struct uiscmdrsp *next;	/* General Purpose Queue Link */
-	struct uiscmdrsp *activeQ_next;	/* Used to track active commands */
-	struct uiscmdrsp *activeQ_prev;	/* Used to track active commands */
+	struct uiscmdrsp *activeQ_next;	/* Pointer to the nextactive commands */
+	struct uiscmdrsp *activeQ_prev;	/* Pointer to the prevactive commands */
 } __packed;
 
 struct iochannel_vhba {
@@ -491,7 +525,8 @@ struct iochannel_vnic {
 	u32 mtu;			/* 4 bytes */
 	uuid_le zone_uuid;		/* 16 bytes */
 } __packed;
-/* This is just the header of the IO channel.  It is assumed that directly after
+/*
+ * This is just the header of the IO channel. It is assumed that directly after
  * this header there is a large region of memory which contains the command and
  * response queues as specified in cmd_q and rsp_q SIGNAL_QUEUE_HEADERS.
  */
@@ -505,31 +540,19 @@ struct spar_io_channel_protocol {
 	} __packed;
 
 #define MAX_CLIENTSTRING_LEN 1024
-	/* client_string is NULL termimated so holds max -1 bytes */
+	/* client_string is NULL termimated so holds max-1 bytes */
 	 u8 client_string[MAX_CLIENTSTRING_LEN];
 } __packed;
 
-/* INLINE functions for initializing and accessing I/O data channels */
-#define SIZEOF_PROTOCOL (COVER(sizeof(struct spar_io_channel_protocol), 64))
+/* INLINE functions for initializing and accessing I/O data channels. */
 #define SIZEOF_CMDRSP (COVER(sizeof(struct uiscmdrsp), 64))
 
-#define MIN_IO_CHANNEL_SIZE COVER(SIZEOF_PROTOCOL + \
-				  2 * MIN_NUMSIGNALS * SIZEOF_CMDRSP, 4096)
-
-/*
- * INLINE function for expanding a guest's pfn-off-size into multiple 4K page
- * pfn-off-size entires.
- */
-
-/* use 4K page sizes when we it comes to passing page information between */
-/* Guest and IOPartition. */
+/* Use 4K page sizes when passing page info between Guest and IOPartition. */
 #define PI_PAGE_SIZE  0x1000
 #define PI_PAGE_MASK  0x0FFF
 
-/* returns next non-zero index on success or zero on failure (i.e. out of
- * room)
- */
-static inline  u16
+/* Returns next non-zero index on success or 0 on failure (i.e. out of room). */
+static inline u16
 add_physinfo_entries(u64 inp_pfn, u16 inp_off, u32 inp_len, u16 index,
 		     u16 max_pi_arr_entries, struct phys_info pi_arr[])
 {
@@ -538,7 +561,7 @@ add_physinfo_entries(u64 inp_pfn, u16 inp_off, u32 inp_len, u16 index,
 
 	firstlen = PI_PAGE_SIZE - inp_off;
 	if (inp_len <= firstlen) {
-		/* the input entry spans only one page - add as is */
+		/* The input entry spans only one page - add as is. */
 		if (index >= max_pi_arr_entries)
 			return 0;
 		pi_arr[index].pi_pfn = inp_pfn;
@@ -547,7 +570,7 @@ add_physinfo_entries(u64 inp_pfn, u16 inp_off, u32 inp_len, u16 index,
 		return index + 1;
 	}
 
-	/* this entry spans multiple pages */
+	/* This entry spans multiple pages. */
 	for (len = inp_len, i = 0; len;
 		len -= pi_arr[index + i].pi_len, i++) {
 		if (index + i >= max_pi_arr_entries)
@@ -565,4 +588,4 @@ add_physinfo_entries(u64 inp_pfn, u16 inp_off, u32 inp_len, u16 index,
 	return index + i;
 }
 
-#endif				/* __IOCHANNEL_H__ */
+#endif /* __IOCHANNEL_H__ */
diff --git a/drivers/staging/unisys/include/visorbus.h b/drivers/staging/unisys/include/visorbus.h
index 677627c72c4c..03d56f818a86 100644
--- a/drivers/staging/unisys/include/visorbus.h
+++ b/drivers/staging/unisys/include/visorbus.h
@@ -166,6 +166,8 @@ struct visor_device {
 	struct controlvm_message_header *pending_msg_hdr;
 	void *vbus_hdr_info;
 	uuid_le partition_uuid;
+	struct dentry *debugfs_dir;
+	struct dentry *debugfs_client_bus_info;
 };
 
 #define to_visor_device(x) container_of(x, struct visor_device, device)
diff --git a/drivers/staging/unisys/visorbus/vbuschannel.h b/drivers/staging/unisys/visorbus/vbuschannel.h
index e97917522f6a..b0df26155d02 100644
--- a/drivers/staging/unisys/visorbus/vbuschannel.h
+++ b/drivers/staging/unisys/visorbus/vbuschannel.h
@@ -23,6 +23,7 @@
  *  the client devices and client drivers for the server end to see.
  */
 #include <linux/uuid.h>
+#include <linux/ctype.h>
 #include "channel.h"
 
 /* {193b331b-c58f-11da-95a9-00e08161165f} */
@@ -50,12 +51,6 @@ static const uuid_le spar_vbus_channel_protocol_uuid =
 				   SPAR_VBUS_CHANNEL_PROTOCOL_VERSIONID, \
 				   SPAR_VBUS_CHANNEL_PROTOCOL_SIGNATURE)
 
-#define SPAR_VBUS_CHANNEL_OK_SERVER(actual_bytes)    \
-	(spar_check_channel_server(spar_vbus_channel_protocol_uuid,	\
-				   "vbus",				\
-				   sizeof(struct spar_vbus_channel_protocol),\
-				   actual_bytes))
-
 #pragma pack(push, 1)		/* both GCC and VC now allow this pragma */
 
 /*
@@ -72,199 +67,38 @@ struct ultra_vbus_deviceinfo {
 };
 
 /**
- * vbuschannel_sanitize_buffer() - remove non-printable chars from buffer
- * @p: destination buffer where chars are written to
- * @remain: number of bytes that can be written starting at #p
- * @src: pointer to source buffer
- * @srcmax: number of valid characters at #src
- *
- * Reads chars from the buffer at @src for @srcmax bytes, and writes to
- * the buffer at @p, which is @remain bytes long, ensuring never to
- * overflow the buffer at @p, using the following rules:
- * - printable characters are simply copied from the buffer at @src to the
- *   buffer at @p
- * - intervening streaks of non-printable characters in the buffer at @src
- *   are replaced with a single space in the buffer at @p
- * Note that we pay no attention to '\0'-termination.
- *
- * Pass @p == NULL and @remain == 0 for this special behavior -- In this
- * case, we simply return the number of bytes that WOULD HAVE been written
- * to a buffer at @p, had it been infinitely big.
- *
- * Return: the number of bytes written to @p (or WOULD HAVE been written to
- *         @p, as described in the previous paragraph)
- */
-static inline int
-vbuschannel_sanitize_buffer(char *p, int remain, char *src, int srcmax)
-{
-	int chars = 0;
-	int nonprintable_streak = 0;
-
-	while (srcmax > 0) {
-		if ((*src >= ' ') && (*src < 0x7f)) {
-			if (nonprintable_streak) {
-				if (remain > 0) {
-					*p = ' ';
-					p++;
-					remain--;
-					chars++;
-				} else if (!p) {
-					chars++;
-				}
-				nonprintable_streak = 0;
-			}
-			if (remain > 0) {
-				*p = *src;
-				p++;
-				remain--;
-				chars++;
-			} else if (!p) {
-				chars++;
-			}
-		} else {
-			nonprintable_streak = 1;
-		}
-		src++;
-		srcmax--;
-	}
-	return chars;
-}
-
-#define VBUSCHANNEL_ADDACHAR(ch, p, remain, chars) \
-	do {					   \
-		if (remain <= 0)		   \
-			break;			   \
-		*p = ch;			   \
-		p++;  chars++;  remain--;	   \
-	} while (0)
-
-/**
- * vbuschannel_itoa() - convert non-negative int to string
- * @p: destination string
- * @remain: max number of bytes that can be written to @p
- * @num: input int to convert
- *
- * Converts the non-negative value at @num to an ascii decimal string
- * at @p, writing at most @remain bytes.  Note there is NO '\0' termination
- * written to @p.
- *
- * Return: number of bytes written to @p
- *
- */
-static inline int
-vbuschannel_itoa(char *p, int remain, int num)
-{
-	int digits = 0;
-	char s[32];
-	int i;
-
-	if (num == 0) {
-		/* '0' is a special case */
-		if (remain <= 0)
-			return 0;
-		*p = '0';
-		return 1;
-	}
-	/* form a backwards decimal ascii string in <s> */
-	while (num > 0) {
-		if (digits >= (int)sizeof(s))
-			return 0;
-		s[digits++] = (num % 10) + '0';
-		num = num / 10;
-	}
-	if (remain < digits) {
-		/* not enough room left at <p> to hold number, so fill with
-		 * '?'
-		 */
-		for (i = 0; i < remain; i++, p++)
-			*p = '?';
-		return remain;
-	}
-	/* plug in the decimal ascii string representing the number, by */
-	/* reversing the string we just built in <s> */
-	i = digits;
-	while (i > 0) {
-		i--;
-		*p = s[i];
-		p++;
-	}
-	return digits;
-}
-
-/**
- * vbuschannel_devinfo_to_string() - format a struct ultra_vbus_deviceinfo
- *                                   to a printable string
+ * vbuschannel_print_devinfo() - format a struct ultra_vbus_deviceinfo
+ *                               and write it to a seq_file
  * @devinfo: the struct ultra_vbus_deviceinfo to format
- * @p: destination string area
- * @remain: size of destination string area in bytes
+ * @seq: seq_file to write to
  * @devix: the device index to be included in the output data, or -1 if no
  *         device index is to be included
  *
- * Reads @devInfo, and converts its contents to a printable string at @p,
- * writing at most @remain bytes. Note there is NO '\0' termination
- * written to @p.
- *
- * Return: number of bytes written to @p
+ * Reads @devInfo, and writes it in human-readable notation to @seq.
  */
-static inline int
-vbuschannel_devinfo_to_string(struct ultra_vbus_deviceinfo *devinfo,
-			      char *p, int remain, int devix)
+static inline void
+vbuschannel_print_devinfo(struct ultra_vbus_deviceinfo *devinfo,
+			  struct seq_file *seq, int devix)
 {
-	char *psrc;
-	int nsrc, x, i, pad;
-	int chars = 0;
-
-	psrc = &devinfo->devtype[0];
-	nsrc = sizeof(devinfo->devtype);
-	if (vbuschannel_sanitize_buffer(NULL, 0, psrc, nsrc) <= 0)
-		return 0;
-
-	/* emit device index */
-	if (devix >= 0) {
-		VBUSCHANNEL_ADDACHAR('[', p, remain, chars);
-		x = vbuschannel_itoa(p, remain, devix);
-		p += x;
-		remain -= x;
-		chars += x;
-		VBUSCHANNEL_ADDACHAR(']', p, remain, chars);
-	} else {
-		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-	}
-
-	/* emit device type */
-	x = vbuschannel_sanitize_buffer(p, remain, psrc, nsrc);
-	p += x;
-	remain -= x;
-	chars += x;
-	pad = 15 - x;		/* pad device type to be exactly 15 chars */
-	for (i = 0; i < pad; i++)
-		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-	VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-
-	/* emit driver name */
-	psrc = &devinfo->drvname[0];
-	nsrc = sizeof(devinfo->drvname);
-	x = vbuschannel_sanitize_buffer(p, remain, psrc, nsrc);
-	p += x;
-	remain -= x;
-	chars += x;
-	pad = 15 - x;		/* pad driver name to be exactly 15 chars */
-	for (i = 0; i < pad; i++)
-		VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-	VBUSCHANNEL_ADDACHAR(' ', p, remain, chars);
-
-	/* emit strings */
-	psrc = &devinfo->infostrs[0];
-	nsrc = sizeof(devinfo->infostrs);
-	x = vbuschannel_sanitize_buffer(p, remain, psrc, nsrc);
-	p += x;
-	remain -= x;
-	chars += x;
-	VBUSCHANNEL_ADDACHAR('\n', p, remain, chars);
-
-	return chars;
+	if (!isprint(devinfo->devtype[0]))
+		return; /* uninitialized vbus device entry */
+
+	if (devix >= 0)
+		seq_printf(seq, "[%d]", devix);
+	else
+		/* vbus device entry is for bus or chipset */
+		seq_puts(seq, "   ");
+
+	/*
+	 * Note: because the s-Par back-end is free to scribble in this area,
+	 * we never assume '\0'-termination.
+	 */
+	seq_printf(seq, "%-*.*s ", (int)sizeof(devinfo->devtype),
+		   (int)sizeof(devinfo->devtype), devinfo->devtype);
+	seq_printf(seq, "%-*.*s ", (int)sizeof(devinfo->drvname),
+		   (int)sizeof(devinfo->drvname), devinfo->drvname);
+	seq_printf(seq, "%.*s\n", (int)sizeof(devinfo->infostrs),
+		   devinfo->infostrs);
 }
 
 struct spar_vbus_headerinfo {
@@ -293,11 +127,6 @@ struct spar_vbus_channel_protocol {
 	/* describes client device and driver for each device on the bus */
 };
 
-#define VBUS_CH_SIZE_EXACT(MAXDEVICES) \
-	(sizeof(ULTRA_VBUS_CHANNEL_PROTOCOL) + ((MAXDEVICES) * \
-						sizeof(ULTRA_VBUS_DEVICEINFO)))
-#define VBUS_CH_SIZE(MAXDEVICES) COVER(VBUS_CH_SIZE_EXACT(MAXDEVICES), 4096)
-
 #pragma pack(pop)
 
 #endif
diff --git a/drivers/staging/unisys/visorbus/visorbus_main.c b/drivers/staging/unisys/visorbus/visorbus_main.c
index fec0a54916fe..3457ef338e1e 100644
--- a/drivers/staging/unisys/visorbus/visorbus_main.c
+++ b/drivers/staging/unisys/visorbus/visorbus_main.c
@@ -14,6 +14,7 @@
  * details.
  */
 
+#include <linux/debugfs.h>
 #include <linux/uuid.h>
 
 #include "visorbus.h"
@@ -33,6 +34,7 @@ static int visorbus_forcenomatch;
 #define POLLJIFFIES_NORMALCHANNEL     10
 
 static int busreg_rc = -ENODEV; /* stores the result from bus registration */
+static struct dentry *visorbus_debugfs_dir;
 
 /*
  * DEVICE type attributes
@@ -151,6 +153,8 @@ visorbus_release_busdevice(struct device *xdev)
 {
 	struct visor_device *dev = dev_get_drvdata(xdev);
 
+	debugfs_remove(dev->debugfs_client_bus_info);
+	debugfs_remove_recursive(dev->debugfs_dir);
 	kfree(dev);
 }
 
@@ -186,6 +190,7 @@ static ssize_t physaddr_show(struct device *dev, struct device_attribute *attr,
 	return snprintf(buf, PAGE_SIZE, "0x%llx\n",
 			visorchannel_get_physaddr(vdev->visorchannel));
 }
+static DEVICE_ATTR_RO(physaddr);
 
 static ssize_t nbytes_show(struct device *dev, struct device_attribute *attr,
 			   char *buf)
@@ -197,6 +202,7 @@ static ssize_t nbytes_show(struct device *dev, struct device_attribute *attr,
 	return snprintf(buf, PAGE_SIZE, "0x%lx\n",
 			visorchannel_get_nbytes(vdev->visorchannel));
 }
+static DEVICE_ATTR_RO(nbytes);
 
 static ssize_t clientpartition_show(struct device *dev,
 				    struct device_attribute *attr, char *buf)
@@ -208,6 +214,7 @@ static ssize_t clientpartition_show(struct device *dev,
 	return snprintf(buf, PAGE_SIZE, "0x%llx\n",
 			visorchannel_get_clientpartition(vdev->visorchannel));
 }
+static DEVICE_ATTR_RO(clientpartition);
 
 static ssize_t typeguid_show(struct device *dev, struct device_attribute *attr,
 			     char *buf)
@@ -220,6 +227,7 @@ static ssize_t typeguid_show(struct device *dev, struct device_attribute *attr,
 	return snprintf(buf, PAGE_SIZE, "%s\n",
 			visorchannel_id(vdev->visorchannel, typeid));
 }
+static DEVICE_ATTR_RO(typeguid);
 
 static ssize_t zoneguid_show(struct device *dev, struct device_attribute *attr,
 			     char *buf)
@@ -232,6 +240,7 @@ static ssize_t zoneguid_show(struct device *dev, struct device_attribute *attr,
 	return snprintf(buf, PAGE_SIZE, "%s\n",
 			visorchannel_zoneid(vdev->visorchannel, zoneid));
 }
+static DEVICE_ATTR_RO(zoneguid);
 
 static ssize_t typename_show(struct device *dev, struct device_attribute *attr,
 			     char *buf)
@@ -250,12 +259,6 @@ static ssize_t typename_show(struct device *dev, struct device_attribute *attr,
 	drv = to_visor_driver(xdrv);
 	return snprintf(buf, PAGE_SIZE, "%s\n", drv->channel_types[i - 1].name);
 }
-
-static DEVICE_ATTR_RO(physaddr);
-static DEVICE_ATTR_RO(nbytes);
-static DEVICE_ATTR_RO(clientpartition);
-static DEVICE_ATTR_RO(typeguid);
-static DEVICE_ATTR_RO(zoneguid);
 static DEVICE_ATTR_RO(typename);
 
 static struct attribute *channel_attrs[] = {
@@ -295,6 +298,7 @@ static ssize_t partition_handle_show(struct device *dev,
 
 	return snprintf(buf, PAGE_SIZE, "0x%llx\n", handle);
 }
+static DEVICE_ATTR_RO(partition_handle);
 
 static ssize_t partition_guid_show(struct device *dev,
 				   struct device_attribute *attr,
@@ -303,6 +307,7 @@ static ssize_t partition_guid_show(struct device *dev,
 
 	return snprintf(buf, PAGE_SIZE, "{%pUb}\n", &vdev->partition_uuid);
 }
+static DEVICE_ATTR_RO(partition_guid);
 
 static ssize_t partition_name_show(struct device *dev,
 				   struct device_attribute *attr,
@@ -311,6 +316,7 @@ static ssize_t partition_name_show(struct device *dev,
 
 	return snprintf(buf, PAGE_SIZE, "%s\n", vdev->name);
 }
+static DEVICE_ATTR_RO(partition_name);
 
 static ssize_t channel_addr_show(struct device *dev,
 				 struct device_attribute *attr,
@@ -320,6 +326,7 @@ static ssize_t channel_addr_show(struct device *dev,
 
 	return snprintf(buf, PAGE_SIZE, "0x%llx\n", addr);
 }
+static DEVICE_ATTR_RO(channel_addr);
 
 static ssize_t channel_bytes_show(struct device *dev,
 				  struct device_attribute *attr,
@@ -329,6 +336,7 @@ static ssize_t channel_bytes_show(struct device *dev,
 
 	return snprintf(buf, PAGE_SIZE, "0x%llx\n", nbytes);
 }
+static DEVICE_ATTR_RO(channel_bytes);
 
 static ssize_t channel_id_show(struct device *dev,
 			       struct device_attribute *attr,
@@ -343,77 +351,7 @@ static ssize_t channel_id_show(struct device *dev,
 	}
 	return len;
 }
-
-static ssize_t client_bus_info_show(struct device *dev,
-				    struct device_attribute *attr,
-				    char *buf) {
-	struct visor_device *vdev = to_visor_device(dev);
-	struct visorchannel *channel = vdev->visorchannel;
-
-	int i, shift, remain = PAGE_SIZE;
-	unsigned long off;
-	char *pos = buf;
-	u8 *partition_name;
-	struct ultra_vbus_deviceinfo dev_info;
-
-	partition_name = "";
-	if (channel) {
-		if (vdev->name)
-			partition_name = vdev->name;
-		shift = snprintf(pos, remain,
-				 "Client device / client driver info for %s partition (vbus #%u):\n",
-				 partition_name, vdev->chipset_bus_no);
-		pos += shift;
-		remain -= shift;
-		shift = visorchannel_read(channel,
-					  offsetof(struct
-						   spar_vbus_channel_protocol,
-						   chp_info),
-					  &dev_info, sizeof(dev_info));
-		if (shift >= 0) {
-			shift = vbuschannel_devinfo_to_string(&dev_info, pos,
-							      remain, -1);
-			pos += shift;
-			remain -= shift;
-		}
-		shift = visorchannel_read(channel,
-					  offsetof(struct
-						   spar_vbus_channel_protocol,
-						   bus_info),
-					  &dev_info, sizeof(dev_info));
-		if (shift >= 0) {
-			shift = vbuschannel_devinfo_to_string(&dev_info, pos,
-							      remain, -1);
-			pos += shift;
-			remain -= shift;
-		}
-		off = offsetof(struct spar_vbus_channel_protocol, dev_info);
-		i = 0;
-		while (off + sizeof(dev_info) <=
-		       visorchannel_get_nbytes(channel)) {
-			shift = visorchannel_read(channel,
-						  off, &dev_info,
-						  sizeof(dev_info));
-			if (shift >= 0) {
-				shift = vbuschannel_devinfo_to_string
-				    (&dev_info, pos, remain, i);
-				pos += shift;
-				remain -= shift;
-			}
-			off += sizeof(dev_info);
-			i++;
-		}
-	}
-	return PAGE_SIZE - remain;
-}
-
-static DEVICE_ATTR_RO(partition_handle);
-static DEVICE_ATTR_RO(partition_guid);
-static DEVICE_ATTR_RO(partition_name);
-static DEVICE_ATTR_RO(channel_addr);
-static DEVICE_ATTR_RO(channel_bytes);
 static DEVICE_ATTR_RO(channel_id);
-static DEVICE_ATTR_RO(client_bus_info);
 
 static struct attribute *dev_attrs[] = {
 		&dev_attr_partition_handle.attr,
@@ -422,7 +360,6 @@ static struct attribute *dev_attrs[] = {
 		&dev_attr_channel_addr.attr,
 		&dev_attr_channel_bytes.attr,
 		&dev_attr_channel_id.attr,
-		&dev_attr_client_bus_info.attr,
 		NULL
 };
 
@@ -435,6 +372,66 @@ static const struct attribute_group *visorbus_groups[] = {
 		NULL
 };
 
+/*
+ *  BUS debugfs entries
+ *
+ *  define & implement display of debugfs attributes under
+ *  /sys/kernel/debug/visorbus/visorbus<n>.
+ */
+
+static int client_bus_info_debugfs_show(struct seq_file *seq, void *v)
+{
+	struct visor_device *vdev = seq->private;
+	struct visorchannel *channel = vdev->visorchannel;
+
+	int i;
+	unsigned long off;
+	struct ultra_vbus_deviceinfo dev_info;
+
+	if (!channel)
+		return 0;
+
+	seq_printf(seq,
+		   "Client device / client driver info for %s partition (vbus #%u):\n",
+		   ((vdev->name) ? (char *)(vdev->name) : ""),
+		   vdev->chipset_bus_no);
+	if (visorchannel_read(channel,
+			      offsetof(struct spar_vbus_channel_protocol,
+				       chp_info),
+			      &dev_info, sizeof(dev_info)) >= 0)
+		vbuschannel_print_devinfo(&dev_info, seq, -1);
+	if (visorchannel_read(channel,
+			      offsetof(struct spar_vbus_channel_protocol,
+				       bus_info),
+			      &dev_info, sizeof(dev_info)) >= 0)
+		vbuschannel_print_devinfo(&dev_info, seq, -1);
+	off = offsetof(struct spar_vbus_channel_protocol, dev_info);
+	i = 0;
+	while (off + sizeof(dev_info) <= visorchannel_get_nbytes(channel)) {
+		if (visorchannel_read(channel, off, &dev_info,
+				      sizeof(dev_info)) >= 0)
+			vbuschannel_print_devinfo(&dev_info, seq, i);
+		off += sizeof(dev_info);
+		i++;
+	}
+
+	return 0;
+}
+
+static int client_bus_info_debugfs_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, client_bus_info_debugfs_show,
+			   inode->i_private);
+}
+
+static const struct file_operations client_bus_info_debugfs_fops = {
+	.owner = THIS_MODULE,
+	.open = client_bus_info_debugfs_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
 static void
 dev_periodic_work(unsigned long __opaque)
 {
@@ -610,8 +607,8 @@ create_visor_device(struct visor_device *dev)
 	u32 chipset_bus_no = dev->chipset_bus_no;
 	u32 chipset_dev_no = dev->chipset_dev_no;
 
-	POSTCODE_LINUX_4(DEVICE_CREATE_ENTRY_PC, chipset_dev_no, chipset_bus_no,
-			 POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(DEVICE_CREATE_ENTRY_PC, chipset_dev_no, chipset_bus_no,
+		       DIAG_SEVERITY_PRINT);
 
 	mutex_init(&dev->visordriver_callback_lock);
 	dev->device.bus = &visorbus_type;
@@ -651,8 +648,8 @@ create_visor_device(struct visor_device *dev)
 	 */
 	err = device_add(&dev->device);
 	if (err < 0) {
-		POSTCODE_LINUX_3(DEVICE_ADD_PC, chipset_bus_no,
-				 DIAG_SEVERITY_ERR);
+		POSTCODE_LINUX(DEVICE_ADD_PC, 0, chipset_bus_no,
+			       DIAG_SEVERITY_ERR);
 		goto err_put;
 	}
 
@@ -966,9 +963,10 @@ static int
 create_bus_instance(struct visor_device *dev)
 {
 	int id = dev->chipset_bus_no;
+	int err;
 	struct spar_vbus_headerinfo *hdr_info;
 
-	POSTCODE_LINUX_2(BUS_CREATE_ENTRY_PC, POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(BUS_CREATE_ENTRY_PC, 0, 0, DIAG_SEVERITY_PRINT);
 
 	hdr_info = kzalloc(sizeof(*hdr_info), GFP_KERNEL);
 	if (!hdr_info)
@@ -979,11 +977,26 @@ create_bus_instance(struct visor_device *dev)
 	dev->device.groups = visorbus_groups;
 	dev->device.release = visorbus_release_busdevice;
 
+	dev->debugfs_dir = debugfs_create_dir(dev_name(&dev->device),
+					      visorbus_debugfs_dir);
+	if (!dev->debugfs_dir) {
+		err = -ENOMEM;
+		goto err_hdr_info;
+	}
+	dev->debugfs_client_bus_info =
+		debugfs_create_file("client_bus_info", S_IRUSR | S_IRGRP,
+				    dev->debugfs_dir, dev,
+				    &client_bus_info_debugfs_fops);
+	if (!dev->debugfs_client_bus_info) {
+		err = -ENOMEM;
+		goto err_debugfs_dir;
+	}
+
 	if (device_register(&dev->device) < 0) {
-		POSTCODE_LINUX_3(DEVICE_CREATE_FAILURE_PC, id,
-				 POSTCODE_SEVERITY_ERR);
-		kfree(hdr_info);
-		return -ENODEV;
+		POSTCODE_LINUX(DEVICE_CREATE_FAILURE_PC, 0, id,
+			       DIAG_SEVERITY_ERR);
+		err = -ENODEV;
+		goto err_debugfs_created;
 	}
 
 	if (get_vbus_header_info(dev->visorchannel, hdr_info) >= 0) {
@@ -998,6 +1011,16 @@ create_bus_instance(struct visor_device *dev)
 	list_add_tail(&dev->list_all, &list_all_bus_instances);
 	dev_set_drvdata(&dev->device, dev);
 	return 0;
+
+err_debugfs_created:
+	debugfs_remove(dev->debugfs_client_bus_info);
+
+err_debugfs_dir:
+	debugfs_remove_recursive(dev->debugfs_dir);
+
+err_hdr_info:
+	kfree(hdr_info);
+	return err;
 }
 
 /**
@@ -1069,16 +1092,16 @@ chipset_bus_create(struct visor_device *dev)
 	int rc;
 	u32 bus_no = dev->chipset_bus_no;
 
-	POSTCODE_LINUX_3(BUS_CREATE_ENTRY_PC, bus_no, POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(BUS_CREATE_ENTRY_PC, 0, bus_no, DIAG_SEVERITY_PRINT);
 	rc = create_bus_instance(dev);
-	POSTCODE_LINUX_3(BUS_CREATE_EXIT_PC, bus_no, POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(BUS_CREATE_EXIT_PC, 0, bus_no, DIAG_SEVERITY_PRINT);
 
 	if (rc < 0)
-		POSTCODE_LINUX_3(BUS_CREATE_FAILURE_PC, bus_no,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(BUS_CREATE_FAILURE_PC, 0, bus_no,
+			       DIAG_SEVERITY_ERR);
 	else
-		POSTCODE_LINUX_3(CHIPSET_INIT_SUCCESS_PC, bus_no,
-				 POSTCODE_SEVERITY_INFO);
+		POSTCODE_LINUX(CHIPSET_INIT_SUCCESS_PC, 0, bus_no,
+			       DIAG_SEVERITY_PRINT);
 
 	bus_create_response(dev, rc);
 }
@@ -1097,18 +1120,18 @@ chipset_device_create(struct visor_device *dev_info)
 	u32 bus_no = dev_info->chipset_bus_no;
 	u32 dev_no = dev_info->chipset_dev_no;
 
-	POSTCODE_LINUX_4(DEVICE_CREATE_ENTRY_PC, dev_no, bus_no,
-			 POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(DEVICE_CREATE_ENTRY_PC, dev_no, bus_no,
+		       DIAG_SEVERITY_PRINT);
 
 	rc = create_visor_device(dev_info);
 	device_create_response(dev_info, rc);
 
 	if (rc < 0)
-		POSTCODE_LINUX_4(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
+			       DIAG_SEVERITY_ERR);
 	else
-		POSTCODE_LINUX_4(DEVICE_CREATE_SUCCESS_PC, dev_no, bus_no,
-				 POSTCODE_SEVERITY_INFO);
+		POSTCODE_LINUX(DEVICE_CREATE_SUCCESS_PC, dev_no, bus_no,
+			       DIAG_SEVERITY_PRINT);
 }
 
 void
@@ -1274,12 +1297,17 @@ visorbus_init(void)
 {
 	int err;
 
-	POSTCODE_LINUX_3(DRIVER_ENTRY_PC, 0, POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(DRIVER_ENTRY_PC, 0, 0, DIAG_SEVERITY_PRINT);
+
+	visorbus_debugfs_dir = debugfs_create_dir("visorbus", NULL);
+	if (!visorbus_debugfs_dir)
+		return -ENOMEM;
+
 	bus_device_info_init(&clientbus_driverinfo, "clientbus", "visorbus");
 
 	err = create_bus_type();
 	if (err < 0) {
-		POSTCODE_LINUX_2(BUS_CREATE_ENTRY_PC, DIAG_SEVERITY_ERR);
+		POSTCODE_LINUX(BUS_CREATE_ENTRY_PC, 0, 0, DIAG_SEVERITY_ERR);
 		goto error;
 	}
 
@@ -1288,7 +1316,7 @@ visorbus_init(void)
 	return 0;
 
 error:
-	POSTCODE_LINUX_3(CHIPSET_INIT_FAILURE_PC, err, POSTCODE_SEVERITY_ERR);
+	POSTCODE_LINUX(CHIPSET_INIT_FAILURE_PC, 0, err, DIAG_SEVERITY_ERR);
 	return err;
 }
 
@@ -1306,6 +1334,7 @@ visorbus_exit(void)
 		remove_bus_instance(dev);
 	}
 	remove_bus_type();
+	debugfs_remove_recursive(visorbus_debugfs_dir);
 }
 
 module_param_named(forcematch, visorbus_forcematch, int, S_IRUGO);
diff --git a/drivers/staging/unisys/visorbus/visorbus_private.h b/drivers/staging/unisys/visorbus/visorbus_private.h
index 15403fb52847..49bec1763e33 100644
--- a/drivers/staging/unisys/visorbus/visorbus_private.h
+++ b/drivers/staging/unisys/visorbus/visorbus_private.h
@@ -70,9 +70,9 @@ struct visorchannel *visorchannel_create_with_lock(u64 physaddr,
 						   gfp_t gfp, uuid_le guid);
 void visorchannel_destroy(struct visorchannel *channel);
 int visorchannel_read(struct visorchannel *channel, ulong offset,
-		      void *local, ulong nbytes);
+		      void *dest, ulong nbytes);
 int visorchannel_write(struct visorchannel *channel, ulong offset,
-		       void *local, ulong nbytes);
+		       void *dest, ulong nbytes);
 u64 visorchannel_get_physaddr(struct visorchannel *channel);
 ulong visorchannel_get_nbytes(struct visorchannel *channel);
 char *visorchannel_id(struct visorchannel *channel, char *s);
diff --git a/drivers/staging/unisys/visorbus/visorchannel.c b/drivers/staging/unisys/visorbus/visorchannel.c
index 300a65dc5c6c..f51a7258bef0 100644
--- a/drivers/staging/unisys/visorbus/visorchannel.c
+++ b/drivers/staging/unisys/visorbus/visorchannel.c
@@ -23,6 +23,7 @@
 #include <linux/io.h>
 
 #include "visorbus.h"
+#include "visorbus_private.h"
 #include "controlvmchannel.h"
 
 #define MYDRVNAME "visorchannel"
@@ -127,19 +128,19 @@ EXPORT_SYMBOL_GPL(visorchannel_get_uuid);
 
 int
 visorchannel_read(struct visorchannel *channel, ulong offset,
-		  void *local, ulong nbytes)
+		  void *dest, ulong nbytes)
 {
 	if (offset + nbytes > channel->nbytes)
 		return -EIO;
 
-	memcpy(local, channel->mapped + offset, nbytes);
+	memcpy(dest, channel->mapped + offset, nbytes);
 
 	return 0;
 }
 
 int
 visorchannel_write(struct visorchannel *channel, ulong offset,
-		   void *local, ulong nbytes)
+		   void *dest, ulong nbytes)
 {
 	size_t chdr_size = sizeof(struct channel_header);
 	size_t copy_size;
@@ -150,10 +151,10 @@ visorchannel_write(struct visorchannel *channel, ulong offset,
 	if (offset < chdr_size) {
 		copy_size = min(chdr_size - offset, nbytes);
 		memcpy(((char *)(&channel->chan_hdr)) + offset,
-		       local, copy_size);
+		       dest, copy_size);
 	}
 
-	memcpy(channel->mapped + offset, local, nbytes);
+	memcpy(channel->mapped + offset, dest, nbytes);
 
 	return 0;
 }
@@ -236,8 +237,9 @@ signalremove_inner(struct visorchannel *channel, u32 queue, void *msg)
 	if (error)
 		return error;
 
+	/* No signals to remove; have caller try again. */
 	if (sig_hdr.head == sig_hdr.tail)
-		return -EIO;	/* no signals to remove */
+		return -EAGAIN;
 
 	sig_hdr.tail = (sig_hdr.tail + 1) % sig_hdr.max_slots;
 
@@ -299,22 +301,30 @@ EXPORT_SYMBOL_GPL(visorchannel_signalremove);
  * Return: boolean indicating whether any messages in the designated
  *         channel/queue are present
  */
+
+static bool
+queue_empty(struct visorchannel *channel, u32 queue)
+{
+	struct signal_queue_header sig_hdr;
+
+	if (sig_read_header(channel, queue, &sig_hdr))
+		return true;
+
+	return (sig_hdr.head == sig_hdr.tail);
+}
+
 bool
 visorchannel_signalempty(struct visorchannel *channel, u32 queue)
 {
-	unsigned long flags = 0;
-	struct signal_queue_header sig_hdr;
-	bool rc = false;
+	bool rc;
+	unsigned long flags;
 
-	if (channel->needs_lock)
-		spin_lock_irqsave(&channel->remove_lock, flags);
+	if (!channel->needs_lock)
+		return queue_empty(channel, queue);
 
-	if (sig_read_header(channel, queue, &sig_hdr))
-		rc = true;
-	if (sig_hdr.head == sig_hdr.tail)
-		rc = true;
-	if (channel->needs_lock)
-		spin_unlock_irqrestore(&channel->remove_lock, flags);
+	spin_lock_irqsave(&channel->remove_lock, flags);
+	rc = queue_empty(channel, queue);
+	spin_unlock_irqrestore(&channel->remove_lock, flags);
 
 	return rc;
 }
diff --git a/drivers/staging/unisys/visorbus/visorchipset.c b/drivers/staging/unisys/visorbus/visorchipset.c
index 59871495ea85..d7148c351d3f 100644
--- a/drivers/staging/unisys/visorbus/visorchipset.c
+++ b/drivers/staging/unisys/visorbus/visorchipset.c
@@ -29,7 +29,7 @@
 #include "visorbus_private.h"
 #include "vmcallinterface.h"
 
-#define CURRENT_FILE_PC VISOR_CHIPSET_PC_visorchipset_main_c
+#define CURRENT_FILE_PC VISOR_BUS_PC_visorchipset_c
 
 #define POLLJIFFIES_CONTROLVMCHANNEL_FAST   1
 #define POLLJIFFIES_CONTROLVMCHANNEL_SLOW 100
@@ -57,7 +57,6 @@ visorchipset_open(struct inode *inode, struct file *file)
 
 	if (minor_number)
 		return -ENODEV;
-	file->private_data = NULL;
 	return 0;
 }
 
@@ -499,7 +498,7 @@ controlvm_init_response(struct controlvm_message *msg,
 	}
 }
 
-static void
+static int
 controlvm_respond_chipset_init(struct controlvm_message_header *msg_hdr,
 			       int response,
 			       enum ultra_chipset_feature features)
@@ -508,34 +507,33 @@ controlvm_respond_chipset_init(struct controlvm_message_header *msg_hdr,
 
 	controlvm_init_response(&outmsg, msg_hdr, response);
 	outmsg.cmd.init_chipset.features = features;
-	if (visorchannel_signalinsert(controlvm_channel,
-				      CONTROLVM_QUEUE_REQUEST, &outmsg)) {
-		return;
-	}
+	return visorchannel_signalinsert(controlvm_channel,
+					 CONTROLVM_QUEUE_REQUEST, &outmsg);
 }
 
-static void
+static int
 chipset_init(struct controlvm_message *inmsg)
 {
 	static int chipset_inited;
 	enum ultra_chipset_feature features = 0;
 	int rc = CONTROLVM_RESP_SUCCESS;
+	int res = 0;
 
-	POSTCODE_LINUX_2(CHIPSET_INIT_ENTRY_PC, POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(CHIPSET_INIT_ENTRY_PC, 0, 0, DIAG_SEVERITY_PRINT);
 	if (chipset_inited) {
 		rc = -CONTROLVM_RESP_ERROR_ALREADY_DONE;
+		res = -EIO;
 		goto out_respond;
 	}
 	chipset_inited = 1;
-	POSTCODE_LINUX_2(CHIPSET_INIT_EXIT_PC, POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(CHIPSET_INIT_EXIT_PC, 0, 0, DIAG_SEVERITY_PRINT);
 
 	/*
 	 * Set features to indicate we support parahotplug (if Command
 	 * also supports it).
 	 */
-	features =
-	    inmsg->cmd.init_chipset.
-	    features & ULTRA_CHIPSET_FEATURE_PARA_HOTPLUG;
+	features = inmsg->cmd.init_chipset.features &
+		   ULTRA_CHIPSET_FEATURE_PARA_HOTPLUG;
 
 	/*
 	 * Set the "reply" bit so Command knows this is a
@@ -545,25 +543,25 @@ chipset_init(struct controlvm_message *inmsg)
 
 out_respond:
 	if (inmsg->hdr.flags.response_expected)
-		controlvm_respond_chipset_init(&inmsg->hdr, rc, features);
+		res = controlvm_respond_chipset_init(&inmsg->hdr, rc, features);
+
+	return res;
 }
 
-static void
+static int
 controlvm_respond(struct controlvm_message_header *msg_hdr, int response)
 {
 	struct controlvm_message outmsg;
 
 	controlvm_init_response(&outmsg, msg_hdr, response);
 	if (outmsg.hdr.flags.test_message == 1)
-		return;
+		return -EINVAL;
 
-	if (visorchannel_signalinsert(controlvm_channel,
-				      CONTROLVM_QUEUE_REQUEST, &outmsg)) {
-		return;
-	}
+	return visorchannel_signalinsert(controlvm_channel,
+					 CONTROLVM_QUEUE_REQUEST, &outmsg);
 }
 
-static void controlvm_respond_physdev_changestate(
+static int controlvm_respond_physdev_changestate(
 		struct controlvm_message_header *msg_hdr, int response,
 		struct spar_segment_state state)
 {
@@ -572,10 +570,8 @@ static void controlvm_respond_physdev_changestate(
 	controlvm_init_response(&outmsg, msg_hdr, response);
 	outmsg.cmd.device_change_state.state = state;
 	outmsg.cmd.device_change_state.flags.phys_device = 1;
-	if (visorchannel_signalinsert(controlvm_channel,
-				      CONTROLVM_QUEUE_REQUEST, &outmsg)) {
-		return;
-	}
+	return visorchannel_signalinsert(controlvm_channel,
+					 CONTROLVM_QUEUE_REQUEST, &outmsg);
 }
 
 enum crash_obj_type {
@@ -583,74 +579,80 @@ enum crash_obj_type {
 	CRASH_BUS,
 };
 
-static void
+static int
 save_crash_message(struct controlvm_message *msg, enum crash_obj_type typ)
 {
 	u32 local_crash_msg_offset;
 	u16 local_crash_msg_count;
+	int err;
 
-	if (visorchannel_read(controlvm_channel,
-			      offsetof(struct spar_controlvm_channel_protocol,
-				       saved_crash_message_count),
-			      &local_crash_msg_count, sizeof(u16)) < 0) {
-		POSTCODE_LINUX_2(CRASH_DEV_CTRL_RD_FAILURE_PC,
-				 POSTCODE_SEVERITY_ERR);
-		return;
+	err = visorchannel_read(controlvm_channel,
+				offsetof(struct spar_controlvm_channel_protocol,
+					 saved_crash_message_count),
+				&local_crash_msg_count, sizeof(u16));
+	if (err) {
+		POSTCODE_LINUX(CRASH_DEV_CTRL_RD_FAILURE_PC, 0, 0,
+			       DIAG_SEVERITY_ERR);
+		return err;
 	}
 
 	if (local_crash_msg_count != CONTROLVM_CRASHMSG_MAX) {
-		POSTCODE_LINUX_3(CRASH_DEV_COUNT_FAILURE_PC,
-				 local_crash_msg_count,
-				 POSTCODE_SEVERITY_ERR);
-		return;
+		POSTCODE_LINUX(CRASH_DEV_COUNT_FAILURE_PC, 0,
+			       local_crash_msg_count,
+			       DIAG_SEVERITY_ERR);
+		return -EIO;
 	}
 
-	if (visorchannel_read(controlvm_channel,
-			      offsetof(struct spar_controlvm_channel_protocol,
-				       saved_crash_message_offset),
-			      &local_crash_msg_offset, sizeof(u32)) < 0) {
-		POSTCODE_LINUX_2(CRASH_DEV_CTRL_RD_FAILURE_PC,
-				 POSTCODE_SEVERITY_ERR);
-		return;
+	err = visorchannel_read(controlvm_channel,
+				offsetof(struct spar_controlvm_channel_protocol,
+					 saved_crash_message_offset),
+				&local_crash_msg_offset, sizeof(u32));
+	if (err) {
+		POSTCODE_LINUX(CRASH_DEV_CTRL_RD_FAILURE_PC, 0, 0,
+			       DIAG_SEVERITY_ERR);
+		return err;
 	}
 
 	if (typ == CRASH_BUS) {
-		if (visorchannel_write(controlvm_channel,
-				       local_crash_msg_offset,
-				       msg,
-				       sizeof(struct controlvm_message)) < 0) {
-			POSTCODE_LINUX_2(SAVE_MSG_BUS_FAILURE_PC,
-					 POSTCODE_SEVERITY_ERR);
-			return;
+		err = visorchannel_write(controlvm_channel,
+					 local_crash_msg_offset,
+					 msg,
+					sizeof(struct controlvm_message));
+		if (err) {
+			POSTCODE_LINUX(SAVE_MSG_BUS_FAILURE_PC, 0, 0,
+				       DIAG_SEVERITY_ERR);
+			return err;
 		}
 	} else {
 		local_crash_msg_offset += sizeof(struct controlvm_message);
-		if (visorchannel_write(controlvm_channel,
-				       local_crash_msg_offset,
-				       msg,
-				       sizeof(struct controlvm_message)) < 0) {
-			POSTCODE_LINUX_2(SAVE_MSG_DEV_FAILURE_PC,
-					 POSTCODE_SEVERITY_ERR);
-			return;
+		err = visorchannel_write(controlvm_channel,
+					 local_crash_msg_offset,
+					 msg,
+					 sizeof(struct controlvm_message));
+		if (err) {
+			POSTCODE_LINUX(SAVE_MSG_DEV_FAILURE_PC, 0, 0,
+				       DIAG_SEVERITY_ERR);
+			return err;
 		}
 	}
+	return 0;
 }
 
-static void
+static int
 bus_responder(enum controlvm_id cmd_id,
 	      struct controlvm_message_header *pending_msg_hdr,
 	      int response)
 {
 	if (!pending_msg_hdr)
-		return;		/* no controlvm response needed */
+		return -EIO;
 
 	if (pending_msg_hdr->id != (u32)cmd_id)
-		return;
+		return -EINVAL;
 
-	controlvm_respond(pending_msg_hdr, response);
+	return controlvm_respond(pending_msg_hdr, response);
 }
 
-static void
+static int
 device_changestate_responder(enum controlvm_id cmd_id,
 			     struct visor_device *p, int response,
 			     struct spar_segment_state response_state)
@@ -660,9 +662,9 @@ device_changestate_responder(enum controlvm_id cmd_id,
 	u32 dev_no = p->chipset_dev_no;
 
 	if (!p->pending_msg_hdr)
-		return;		/* no controlvm response needed */
+		return -EIO;
 	if (p->pending_msg_hdr->id != cmd_id)
-		return;
+		return -EINVAL;
 
 	controlvm_init_response(&outmsg, p->pending_msg_hdr, response);
 
@@ -670,175 +672,74 @@ device_changestate_responder(enum controlvm_id cmd_id,
 	outmsg.cmd.device_change_state.dev_no = dev_no;
 	outmsg.cmd.device_change_state.state = response_state;
 
-	if (visorchannel_signalinsert(controlvm_channel,
-				      CONTROLVM_QUEUE_REQUEST, &outmsg))
-		return;
+	return visorchannel_signalinsert(controlvm_channel,
+					 CONTROLVM_QUEUE_REQUEST, &outmsg);
 }
 
-static void
+static int
 device_responder(enum controlvm_id cmd_id,
 		 struct controlvm_message_header *pending_msg_hdr,
 		 int response)
 {
 	if (!pending_msg_hdr)
-		return;		/* no controlvm response needed */
+		return -EIO;
 
 	if (pending_msg_hdr->id != (u32)cmd_id)
-		return;
-
-	controlvm_respond(pending_msg_hdr, response);
-}
-
-static void
-bus_epilog(struct visor_device *bus_info,
-	   u32 cmd, struct controlvm_message_header *msg_hdr,
-	   int response, bool need_response)
-{
-	struct controlvm_message_header *pmsg_hdr = NULL;
-
-	if (!bus_info) {
-		/*
-		 * relying on a valid passed in response code
-		 * be lazy and re-use msg_hdr for this failure, is this ok??
-		 */
-		pmsg_hdr = msg_hdr;
-		goto out_respond;
-	}
-
-	if (bus_info->pending_msg_hdr) {
-		/* only non-NULL if dev is still waiting on a response */
-		response = -CONTROLVM_RESP_ERROR_MESSAGE_ID_INVALID_FOR_CLIENT;
-		pmsg_hdr = bus_info->pending_msg_hdr;
-		goto out_respond;
-	}
-
-	if (need_response) {
-		pmsg_hdr = kzalloc(sizeof(*pmsg_hdr), GFP_KERNEL);
-		if (!pmsg_hdr) {
-			POSTCODE_LINUX_4(MALLOC_FAILURE_PC, cmd,
-					 bus_info->chipset_bus_no,
-					 POSTCODE_SEVERITY_ERR);
-			return;
-		}
-
-		memcpy(pmsg_hdr, msg_hdr,
-		       sizeof(struct controlvm_message_header));
-		bus_info->pending_msg_hdr = pmsg_hdr;
-	}
-
-	if (response == CONTROLVM_RESP_SUCCESS) {
-		switch (cmd) {
-		case CONTROLVM_BUS_CREATE:
-			chipset_bus_create(bus_info);
-			break;
-		case CONTROLVM_BUS_DESTROY:
-			chipset_bus_destroy(bus_info);
-			break;
-		}
-	}
-
-out_respond:
-	bus_responder(cmd, pmsg_hdr, response);
-}
-
-static void
-device_epilog(struct visor_device *dev_info,
-	      struct spar_segment_state state, u32 cmd,
-	      struct controlvm_message_header *msg_hdr, int response,
-	      bool need_response, bool for_visorbus)
-{
-	struct controlvm_message_header *pmsg_hdr = NULL;
-
-	if (!dev_info) {
-		/*
-		 * relying on a valid passed in response code
-		 * be lazy and re-use msg_hdr for this failure, is this ok??
-		 */
-		pmsg_hdr = msg_hdr;
-		goto out_respond;
-	}
-
-	if (dev_info->pending_msg_hdr) {
-		/* only non-NULL if dev is still waiting on a response */
-		response = -CONTROLVM_RESP_ERROR_MESSAGE_ID_INVALID_FOR_CLIENT;
-		pmsg_hdr = dev_info->pending_msg_hdr;
-		goto out_respond;
-	}
-
-	if (need_response) {
-		pmsg_hdr = kzalloc(sizeof(*pmsg_hdr), GFP_KERNEL);
-		if (!pmsg_hdr) {
-			response = -CONTROLVM_RESP_ERROR_KMALLOC_FAILED;
-			goto out_respond;
-		}
-
-		memcpy(pmsg_hdr, msg_hdr,
-		       sizeof(struct controlvm_message_header));
-		dev_info->pending_msg_hdr = pmsg_hdr;
-	}
-
-	if (response >= 0) {
-		switch (cmd) {
-		case CONTROLVM_DEVICE_CREATE:
-			chipset_device_create(dev_info);
-			break;
-		case CONTROLVM_DEVICE_CHANGESTATE:
-			/* ServerReady / ServerRunning / SegmentStateRunning */
-			if (state.alive == segment_state_running.alive &&
-			    state.operating ==
-				segment_state_running.operating) {
-				chipset_device_resume(dev_info);
-			}
-			/* ServerNotReady / ServerLost / SegmentStateStandby */
-			else if (state.alive == segment_state_standby.alive &&
-				 state.operating ==
-				 segment_state_standby.operating) {
-				/*
-				 * technically this is standby case
-				 * where server is lost
-				 */
-				chipset_device_pause(dev_info);
-			}
-			break;
-		case CONTROLVM_DEVICE_DESTROY:
-			chipset_device_destroy(dev_info);
-			break;
-		}
-	}
+		return -EINVAL;
 
-out_respond:
-	device_responder(cmd, pmsg_hdr, response);
+	return controlvm_respond(pending_msg_hdr, response);
 }
 
-static void
+static int
 bus_create(struct controlvm_message *inmsg)
 {
 	struct controlvm_message_packet *cmd = &inmsg->cmd;
+	struct controlvm_message_header *pmsg_hdr = NULL;
 	u32 bus_no = cmd->create_bus.bus_no;
-	int rc = CONTROLVM_RESP_SUCCESS;
 	struct visor_device *bus_info;
 	struct visorchannel *visorchannel;
+	int err;
 
 	bus_info = visorbus_get_device_by_id(bus_no, BUS_ROOT_DEVICE, NULL);
 	if (bus_info && (bus_info->state.created == 1)) {
-		POSTCODE_LINUX_3(BUS_CREATE_FAILURE_PC, bus_no,
-				 POSTCODE_SEVERITY_ERR);
-		rc = -CONTROLVM_RESP_ERROR_ALREADY_DONE;
-		goto out_bus_epilog;
+		POSTCODE_LINUX(BUS_CREATE_FAILURE_PC, 0, bus_no,
+			       DIAG_SEVERITY_ERR);
+		err = -EEXIST;
+		goto err_respond;
 	}
+
 	bus_info = kzalloc(sizeof(*bus_info), GFP_KERNEL);
 	if (!bus_info) {
-		POSTCODE_LINUX_3(BUS_CREATE_FAILURE_PC, bus_no,
-				 POSTCODE_SEVERITY_ERR);
-		rc = -CONTROLVM_RESP_ERROR_KMALLOC_FAILED;
-		goto out_bus_epilog;
+		POSTCODE_LINUX(BUS_CREATE_FAILURE_PC, 0, bus_no,
+			       DIAG_SEVERITY_ERR);
+		err = -ENOMEM;
+		goto err_respond;
 	}
 
 	INIT_LIST_HEAD(&bus_info->list_all);
 	bus_info->chipset_bus_no = bus_no;
 	bus_info->chipset_dev_no = BUS_ROOT_DEVICE;
 
-	POSTCODE_LINUX_3(BUS_CREATE_ENTRY_PC, bus_no, POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(BUS_CREATE_ENTRY_PC, 0, bus_no, DIAG_SEVERITY_PRINT);
+
+	if (uuid_le_cmp(cmd->create_bus.bus_inst_uuid, spar_siovm_uuid) == 0)
+		save_crash_message(inmsg, CRASH_BUS);
+
+	if (inmsg->hdr.flags.response_expected == 1) {
+		pmsg_hdr = kzalloc(sizeof(*pmsg_hdr),
+				   GFP_KERNEL);
+		if (!pmsg_hdr) {
+			POSTCODE_LINUX(MALLOC_FAILURE_PC, cmd,
+				       bus_info->chipset_bus_no,
+				       DIAG_SEVERITY_ERR);
+			err = -ENOMEM;
+			goto err_free_bus_info;
+		}
+
+		memcpy(pmsg_hdr, &inmsg->hdr,
+		       sizeof(struct controlvm_message_header));
+		bus_info->pending_msg_hdr = pmsg_hdr;
+	}
 
 	visorchannel = visorchannel_create(cmd->create_bus.channel_addr,
 					   cmd->create_bus.channel_bytes,
@@ -846,89 +747,138 @@ bus_create(struct controlvm_message *inmsg)
 					   cmd->create_bus.bus_data_type_uuid);
 
 	if (!visorchannel) {
-		POSTCODE_LINUX_3(BUS_CREATE_FAILURE_PC, bus_no,
-				 POSTCODE_SEVERITY_ERR);
-		rc = -CONTROLVM_RESP_ERROR_KMALLOC_FAILED;
-		kfree(bus_info);
-		bus_info = NULL;
-		goto out_bus_epilog;
+		POSTCODE_LINUX(BUS_CREATE_FAILURE_PC, 0, bus_no,
+			       DIAG_SEVERITY_ERR);
+		err = -ENOMEM;
+		goto err_free_pending_msg;
 	}
 	bus_info->visorchannel = visorchannel;
-	if (uuid_le_cmp(cmd->create_bus.bus_inst_uuid, spar_siovm_uuid) == 0)
-		save_crash_message(inmsg, CRASH_BUS);
 
-	POSTCODE_LINUX_3(BUS_CREATE_EXIT_PC, bus_no, POSTCODE_SEVERITY_INFO);
+	/* Response will be handled by chipset_bus_create */
+	chipset_bus_create(bus_info);
+
+	POSTCODE_LINUX(BUS_CREATE_EXIT_PC, 0, bus_no, DIAG_SEVERITY_PRINT);
+	return 0;
+
+err_free_pending_msg:
+	kfree(bus_info->pending_msg_hdr);
+
+err_free_bus_info:
+	kfree(bus_info);
 
-out_bus_epilog:
-	bus_epilog(bus_info, CONTROLVM_BUS_CREATE, &inmsg->hdr,
-		   rc, inmsg->hdr.flags.response_expected == 1);
+err_respond:
+	if (inmsg->hdr.flags.response_expected == 1)
+		bus_responder(inmsg->hdr.id, &inmsg->hdr, err);
+	return err;
 }
 
-static void
+static int
 bus_destroy(struct controlvm_message *inmsg)
 {
 	struct controlvm_message_packet *cmd = &inmsg->cmd;
+	struct controlvm_message_header *pmsg_hdr = NULL;
 	u32 bus_no = cmd->destroy_bus.bus_no;
 	struct visor_device *bus_info;
-	int rc = CONTROLVM_RESP_SUCCESS;
+	int err;
 
 	bus_info = visorbus_get_device_by_id(bus_no, BUS_ROOT_DEVICE, NULL);
-	if (!bus_info)
-		rc = -CONTROLVM_RESP_ERROR_BUS_INVALID;
-	else if (bus_info->state.created == 0)
-		rc = -CONTROLVM_RESP_ERROR_ALREADY_DONE;
+	if (!bus_info) {
+		err = -ENODEV;
+		goto err_respond;
+	}
+	if (bus_info->state.created == 0) {
+		err = -ENOENT;
+		goto err_respond;
+	}
+	if (bus_info->pending_msg_hdr) {
+		/* only non-NULL if dev is still waiting on a response */
+		err = -EEXIST;
+		goto err_respond;
+	}
+	if (inmsg->hdr.flags.response_expected == 1) {
+		pmsg_hdr = kzalloc(sizeof(*pmsg_hdr), GFP_KERNEL);
+		if (!pmsg_hdr) {
+			POSTCODE_LINUX(MALLOC_FAILURE_PC, cmd,
+				       bus_info->chipset_bus_no,
+				       DIAG_SEVERITY_ERR);
+			err = -ENOMEM;
+			goto err_respond;
+		}
 
-	bus_epilog(bus_info, CONTROLVM_BUS_DESTROY, &inmsg->hdr,
-		   rc, inmsg->hdr.flags.response_expected == 1);
+		memcpy(pmsg_hdr, &inmsg->hdr,
+		       sizeof(struct controlvm_message_header));
+		bus_info->pending_msg_hdr = pmsg_hdr;
+	}
 
-	/* bus_info is freed as part of the busdevice_release function */
+	/* Response will be handled by chipset_bus_destroy */
+	chipset_bus_destroy(bus_info);
+	return 0;
+
+err_respond:
+	if (inmsg->hdr.flags.response_expected == 1)
+		bus_responder(inmsg->hdr.id, &inmsg->hdr, err);
+	return err;
 }
 
-static void
+static int
 bus_configure(struct controlvm_message *inmsg,
 	      struct parser_context *parser_ctx)
 {
 	struct controlvm_message_packet *cmd = &inmsg->cmd;
 	u32 bus_no;
 	struct visor_device *bus_info;
-	int rc = CONTROLVM_RESP_SUCCESS;
+	int err = 0;
 
 	bus_no = cmd->configure_bus.bus_no;
-	POSTCODE_LINUX_3(BUS_CONFIGURE_ENTRY_PC, bus_no,
-			 POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(BUS_CONFIGURE_ENTRY_PC, 0, bus_no,
+		       DIAG_SEVERITY_PRINT);
 
 	bus_info = visorbus_get_device_by_id(bus_no, BUS_ROOT_DEVICE, NULL);
 	if (!bus_info) {
-		POSTCODE_LINUX_3(BUS_CONFIGURE_FAILURE_PC, bus_no,
-				 POSTCODE_SEVERITY_ERR);
-		rc = -CONTROLVM_RESP_ERROR_BUS_INVALID;
+		POSTCODE_LINUX(BUS_CONFIGURE_FAILURE_PC, 0, bus_no,
+			       DIAG_SEVERITY_ERR);
+		err = -EINVAL;
+		goto err_respond;
 	} else if (bus_info->state.created == 0) {
-		POSTCODE_LINUX_3(BUS_CONFIGURE_FAILURE_PC, bus_no,
-				 POSTCODE_SEVERITY_ERR);
-		rc = -CONTROLVM_RESP_ERROR_BUS_INVALID;
+		POSTCODE_LINUX(BUS_CONFIGURE_FAILURE_PC, 0, bus_no,
+			       DIAG_SEVERITY_ERR);
+		err = -EINVAL;
+		goto err_respond;
 	} else if (bus_info->pending_msg_hdr) {
-		POSTCODE_LINUX_3(BUS_CONFIGURE_FAILURE_PC, bus_no,
-				 POSTCODE_SEVERITY_ERR);
-		rc = -CONTROLVM_RESP_ERROR_MESSAGE_ID_INVALID_FOR_CLIENT;
-	} else {
-		visorchannel_set_clientpartition
-			(bus_info->visorchannel,
-			 cmd->configure_bus.guest_handle);
-		bus_info->partition_uuid = parser_id_get(parser_ctx);
-		parser_param_start(parser_ctx, PARSERSTRING_NAME);
-		bus_info->name = parser_string_get(parser_ctx);
-
-		POSTCODE_LINUX_3(BUS_CONFIGURE_EXIT_PC, bus_no,
-				 POSTCODE_SEVERITY_INFO);
+		POSTCODE_LINUX(BUS_CONFIGURE_FAILURE_PC, 0, bus_no,
+			       DIAG_SEVERITY_ERR);
+		err = -EIO;
+		goto err_respond;
 	}
-	bus_epilog(bus_info, CONTROLVM_BUS_CONFIGURE, &inmsg->hdr,
-		   rc, inmsg->hdr.flags.response_expected == 1);
+
+	err = visorchannel_set_clientpartition
+		(bus_info->visorchannel,
+		 cmd->configure_bus.guest_handle);
+	if (err)
+		goto err_respond;
+
+	bus_info->partition_uuid = parser_id_get(parser_ctx);
+	parser_param_start(parser_ctx, PARSERSTRING_NAME);
+	bus_info->name = parser_string_get(parser_ctx);
+
+	POSTCODE_LINUX(BUS_CONFIGURE_EXIT_PC, 0, bus_no,
+		       DIAG_SEVERITY_PRINT);
+
+	if (inmsg->hdr.flags.response_expected == 1)
+		bus_responder(inmsg->hdr.id, &inmsg->hdr, err);
+	return 0;
+
+err_respond:
+	if (inmsg->hdr.flags.response_expected == 1)
+		bus_responder(inmsg->hdr.id, &inmsg->hdr, err);
+	return err;
 }
 
 static void
 my_device_create(struct controlvm_message *inmsg)
 {
 	struct controlvm_message_packet *cmd = &inmsg->cmd;
+	struct controlvm_message_header *pmsg_hdr = NULL;
 	u32 bus_no = cmd->create_device.bus_no;
 	u32 dev_no = cmd->create_device.dev_no;
 	struct visor_device *dev_info = NULL;
@@ -938,31 +888,31 @@ my_device_create(struct controlvm_message *inmsg)
 
 	bus_info = visorbus_get_device_by_id(bus_no, BUS_ROOT_DEVICE, NULL);
 	if (!bus_info) {
-		POSTCODE_LINUX_4(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
+			       DIAG_SEVERITY_ERR);
 		rc = -CONTROLVM_RESP_ERROR_BUS_INVALID;
 		goto out_respond;
 	}
 
 	if (bus_info->state.created == 0) {
-		POSTCODE_LINUX_4(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
+			       DIAG_SEVERITY_ERR);
 		rc = -CONTROLVM_RESP_ERROR_BUS_INVALID;
 		goto out_respond;
 	}
 
 	dev_info = visorbus_get_device_by_id(bus_no, dev_no, NULL);
 	if (dev_info && (dev_info->state.created == 1)) {
-		POSTCODE_LINUX_4(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
+			       DIAG_SEVERITY_ERR);
 		rc = -CONTROLVM_RESP_ERROR_ALREADY_DONE;
 		goto out_respond;
 	}
 
 	dev_info = kzalloc(sizeof(*dev_info), GFP_KERNEL);
 	if (!dev_info) {
-		POSTCODE_LINUX_4(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
+			       DIAG_SEVERITY_ERR);
 		rc = -CONTROLVM_RESP_ERROR_KMALLOC_FAILED;
 		goto out_respond;
 	}
@@ -974,8 +924,8 @@ my_device_create(struct controlvm_message *inmsg)
 	/* not sure where the best place to set the 'parent' */
 	dev_info->device.parent = &bus_info->device;
 
-	POSTCODE_LINUX_4(DEVICE_CREATE_ENTRY_PC, dev_no, bus_no,
-			 POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(DEVICE_CREATE_ENTRY_PC, dev_no, bus_no,
+		       DIAG_SEVERITY_PRINT);
 
 	visorchannel =
 	       visorchannel_create_with_lock(cmd->create_device.channel_addr,
@@ -984,12 +934,10 @@ my_device_create(struct controlvm_message *inmsg)
 					     cmd->create_device.data_type_uuid);
 
 	if (!visorchannel) {
-		POSTCODE_LINUX_4(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(DEVICE_CREATE_FAILURE_PC, dev_no, bus_no,
+			       DIAG_SEVERITY_ERR);
 		rc = -CONTROLVM_RESP_ERROR_KMALLOC_FAILED;
-		kfree(dev_info);
-		dev_info = NULL;
-		goto out_respond;
+		goto out_free_dev_info;
 	}
 	dev_info->visorchannel = visorchannel;
 	dev_info->channel_type_guid = cmd->create_device.data_type_uuid;
@@ -997,18 +945,36 @@ my_device_create(struct controlvm_message *inmsg)
 			spar_vhba_channel_protocol_uuid) == 0)
 		save_crash_message(inmsg, CRASH_DEV);
 
-	POSTCODE_LINUX_4(DEVICE_CREATE_EXIT_PC, dev_no, bus_no,
-			 POSTCODE_SEVERITY_INFO);
+	if (inmsg->hdr.flags.response_expected == 1) {
+		pmsg_hdr = kzalloc(sizeof(*pmsg_hdr), GFP_KERNEL);
+		if (!pmsg_hdr) {
+			rc = -CONTROLVM_RESP_ERROR_KMALLOC_FAILED;
+			goto out_free_dev_info;
+		}
+
+		memcpy(pmsg_hdr, &inmsg->hdr,
+		       sizeof(struct controlvm_message_header));
+		dev_info->pending_msg_hdr = pmsg_hdr;
+	}
+	/* Chipset_device_create will send response */
+	chipset_device_create(dev_info);
+	POSTCODE_LINUX(DEVICE_CREATE_EXIT_PC, dev_no, bus_no,
+		       DIAG_SEVERITY_PRINT);
+	return;
+
+out_free_dev_info:
+	kfree(dev_info);
+
 out_respond:
-	device_epilog(dev_info, segment_state_running,
-		      CONTROLVM_DEVICE_CREATE, &inmsg->hdr, rc,
-		      inmsg->hdr.flags.response_expected == 1, 1);
+	if (inmsg->hdr.flags.response_expected == 1)
+		device_responder(inmsg->hdr.id, &inmsg->hdr, rc);
 }
 
 static void
 my_device_changestate(struct controlvm_message *inmsg)
 {
 	struct controlvm_message_packet *cmd = &inmsg->cmd;
+	struct controlvm_message_header *pmsg_hdr = NULL;
 	u32 bus_no = cmd->device_change_state.bus_no;
 	u32 dev_no = cmd->device_change_state.dev_no;
 	struct spar_segment_state state = cmd->device_change_state.state;
@@ -1017,39 +983,97 @@ my_device_changestate(struct controlvm_message *inmsg)
 
 	dev_info = visorbus_get_device_by_id(bus_no, dev_no, NULL);
 	if (!dev_info) {
-		POSTCODE_LINUX_4(DEVICE_CHANGESTATE_FAILURE_PC, dev_no, bus_no,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(DEVICE_CHANGESTATE_FAILURE_PC, dev_no, bus_no,
+			       DIAG_SEVERITY_ERR);
 		rc = -CONTROLVM_RESP_ERROR_DEVICE_INVALID;
-	} else if (dev_info->state.created == 0) {
-		POSTCODE_LINUX_4(DEVICE_CHANGESTATE_FAILURE_PC, dev_no, bus_no,
-				 POSTCODE_SEVERITY_ERR);
+		goto err_respond;
+	}
+	if (dev_info->state.created == 0) {
+		POSTCODE_LINUX(DEVICE_CHANGESTATE_FAILURE_PC, dev_no, bus_no,
+			       DIAG_SEVERITY_ERR);
 		rc = -CONTROLVM_RESP_ERROR_DEVICE_INVALID;
+		goto err_respond;
+	}
+	if (dev_info->pending_msg_hdr) {
+		/* only non-NULL if dev is still waiting on a response */
+		rc = -CONTROLVM_RESP_ERROR_MESSAGE_ID_INVALID_FOR_CLIENT;
+		goto err_respond;
 	}
-	if ((rc >= CONTROLVM_RESP_SUCCESS) && dev_info)
-		device_epilog(dev_info, state,
-			      CONTROLVM_DEVICE_CHANGESTATE, &inmsg->hdr, rc,
-			      inmsg->hdr.flags.response_expected == 1, 1);
+	if (inmsg->hdr.flags.response_expected == 1) {
+		pmsg_hdr = kzalloc(sizeof(*pmsg_hdr), GFP_KERNEL);
+		if (!pmsg_hdr) {
+			rc = -CONTROLVM_RESP_ERROR_KMALLOC_FAILED;
+			goto err_respond;
+		}
+
+		memcpy(pmsg_hdr, &inmsg->hdr,
+		       sizeof(struct controlvm_message_header));
+		dev_info->pending_msg_hdr = pmsg_hdr;
+	}
+
+	if (state.alive == segment_state_running.alive &&
+	    state.operating == segment_state_running.operating)
+		/* Response will be sent from chipset_device_resume */
+		chipset_device_resume(dev_info);
+	/* ServerNotReady / ServerLost / SegmentStateStandby */
+	else if (state.alive == segment_state_standby.alive &&
+		 state.operating == segment_state_standby.operating)
+		/*
+		 * technically this is standby case where server is lost.
+		 * Response will be sent from chipset_device_pause.
+		 */
+		chipset_device_pause(dev_info);
+
+	return;
+
+err_respond:
+	if (inmsg->hdr.flags.response_expected == 1)
+		device_responder(inmsg->hdr.id, &inmsg->hdr, rc);
 }
 
 static void
 my_device_destroy(struct controlvm_message *inmsg)
 {
 	struct controlvm_message_packet *cmd = &inmsg->cmd;
+	struct controlvm_message_header *pmsg_hdr = NULL;
 	u32 bus_no = cmd->destroy_device.bus_no;
 	u32 dev_no = cmd->destroy_device.dev_no;
 	struct visor_device *dev_info;
 	int rc = CONTROLVM_RESP_SUCCESS;
 
 	dev_info = visorbus_get_device_by_id(bus_no, dev_no, NULL);
-	if (!dev_info)
+	if (!dev_info) {
 		rc = -CONTROLVM_RESP_ERROR_DEVICE_INVALID;
-	else if (dev_info->state.created == 0)
+		goto err_respond;
+	}
+	if (dev_info->state.created == 0) {
 		rc = -CONTROLVM_RESP_ERROR_ALREADY_DONE;
+		goto err_respond;
+	}
 
-	if ((rc >= CONTROLVM_RESP_SUCCESS) && dev_info)
-		device_epilog(dev_info, segment_state_running,
-			      CONTROLVM_DEVICE_DESTROY, &inmsg->hdr, rc,
-			      inmsg->hdr.flags.response_expected == 1, 1);
+	if (dev_info->pending_msg_hdr) {
+		/* only non-NULL if dev is still waiting on a response */
+		rc = -CONTROLVM_RESP_ERROR_MESSAGE_ID_INVALID_FOR_CLIENT;
+		goto err_respond;
+	}
+	if (inmsg->hdr.flags.response_expected == 1) {
+		pmsg_hdr = kzalloc(sizeof(*pmsg_hdr), GFP_KERNEL);
+		if (!pmsg_hdr) {
+			rc = -CONTROLVM_RESP_ERROR_KMALLOC_FAILED;
+			goto err_respond;
+		}
+
+		memcpy(pmsg_hdr, &inmsg->hdr,
+		       sizeof(struct controlvm_message_header));
+		dev_info->pending_msg_hdr = pmsg_hdr;
+	}
+
+	chipset_device_destroy(dev_info);
+	return;
+
+err_respond:
+	if (inmsg->hdr.flags.response_expected == 1)
+		device_responder(inmsg->hdr.id, &inmsg->hdr, rc);
 }
 
 /**
@@ -1075,7 +1099,6 @@ initialize_controlvm_payload_info(u64 phys_addr, u64 offset, u32 bytes,
 	if (!info)
 		return -CONTROLVM_RESP_ERROR_PAYLOAD_INVALID;
 
-	memset(info, 0, sizeof(struct visor_controlvm_payload_info));
 	if ((offset == 0) || (bytes == 0))
 		return -CONTROLVM_RESP_ERROR_PAYLOAD_INVALID;
 
@@ -1083,6 +1106,7 @@ initialize_controlvm_payload_info(u64 phys_addr, u64 offset, u32 bytes,
 	if (!payload)
 		return -CONTROLVM_RESP_ERROR_IOREMAP_FAILED;
 
+	memset(info, 0, sizeof(struct visor_controlvm_payload_info));
 	info->offset = offset;
 	info->bytes = bytes;
 	info->ptr = payload;
@@ -1111,16 +1135,16 @@ initialize_controlvm_payload(void)
 			      offsetof(struct spar_controlvm_channel_protocol,
 				       request_payload_offset),
 			      &payload_offset, sizeof(payload_offset)) < 0) {
-		POSTCODE_LINUX_2(CONTROLVM_INIT_FAILURE_PC,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(CONTROLVM_INIT_FAILURE_PC, 0, 0,
+			       DIAG_SEVERITY_ERR);
 		return;
 	}
 	if (visorchannel_read(controlvm_channel,
 			      offsetof(struct spar_controlvm_channel_protocol,
 				       request_payload_bytes),
 			      &payload_bytes, sizeof(payload_bytes)) < 0) {
-		POSTCODE_LINUX_2(CONTROLVM_INIT_FAILURE_PC,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(CONTROLVM_INIT_FAILURE_PC, 0, 0,
+			       DIAG_SEVERITY_ERR);
 		return;
 	}
 	initialize_controlvm_payload_info(phys_addr,
@@ -1317,7 +1341,7 @@ static struct attribute *visorchipset_install_attrs[] = {
 	NULL
 };
 
-static struct attribute_group visorchipset_install_group = {
+static const struct attribute_group visorchipset_install_group = {
 	.name = "install",
 	.attrs = visorchipset_install_attrs
 };
@@ -1540,7 +1564,7 @@ setup_crash_devices_work_queue(struct work_struct *work)
 	u32 local_crash_msg_offset;
 	u16 local_crash_msg_count;
 
-	POSTCODE_LINUX_2(CRASH_DEV_ENTRY_PC, POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(CRASH_DEV_ENTRY_PC, 0, 0, DIAG_SEVERITY_PRINT);
 
 	/* send init chipset msg */
 	msg.hdr.id = CONTROLVM_CHIPSET_INIT;
@@ -1554,15 +1578,15 @@ setup_crash_devices_work_queue(struct work_struct *work)
 			      offsetof(struct spar_controlvm_channel_protocol,
 				       saved_crash_message_count),
 			      &local_crash_msg_count, sizeof(u16)) < 0) {
-		POSTCODE_LINUX_2(CRASH_DEV_CTRL_RD_FAILURE_PC,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(CRASH_DEV_CTRL_RD_FAILURE_PC, 0, 0,
+			       DIAG_SEVERITY_ERR);
 		return;
 	}
 
 	if (local_crash_msg_count != CONTROLVM_CRASHMSG_MAX) {
-		POSTCODE_LINUX_3(CRASH_DEV_COUNT_FAILURE_PC,
-				 local_crash_msg_count,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(CRASH_DEV_COUNT_FAILURE_PC, 0,
+			       local_crash_msg_count,
+			       DIAG_SEVERITY_ERR);
 		return;
 	}
 
@@ -1571,8 +1595,8 @@ setup_crash_devices_work_queue(struct work_struct *work)
 			      offsetof(struct spar_controlvm_channel_protocol,
 				       saved_crash_message_offset),
 			      &local_crash_msg_offset, sizeof(u32)) < 0) {
-		POSTCODE_LINUX_2(CRASH_DEV_CTRL_RD_FAILURE_PC,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(CRASH_DEV_CTRL_RD_FAILURE_PC, 0, 0,
+			       DIAG_SEVERITY_ERR);
 		return;
 	}
 
@@ -1581,8 +1605,8 @@ setup_crash_devices_work_queue(struct work_struct *work)
 			      local_crash_msg_offset,
 			      &local_crash_bus_msg,
 			      sizeof(struct controlvm_message)) < 0) {
-		POSTCODE_LINUX_2(CRASH_DEV_RD_BUS_FAIULRE_PC,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(CRASH_DEV_RD_BUS_FAILURE_PC, 0, 0,
+			       DIAG_SEVERITY_ERR);
 		return;
 	}
 
@@ -1592,8 +1616,8 @@ setup_crash_devices_work_queue(struct work_struct *work)
 			      sizeof(struct controlvm_message),
 			      &local_crash_dev_msg,
 			      sizeof(struct controlvm_message)) < 0) {
-		POSTCODE_LINUX_2(CRASH_DEV_RD_DEV_FAIULRE_PC,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(CRASH_DEV_RD_DEV_FAILURE_PC, 0, 0,
+			       DIAG_SEVERITY_ERR);
 		return;
 	}
 
@@ -1601,8 +1625,8 @@ setup_crash_devices_work_queue(struct work_struct *work)
 	if (local_crash_bus_msg.cmd.create_bus.channel_addr) {
 		bus_create(&local_crash_bus_msg);
 	} else {
-		POSTCODE_LINUX_2(CRASH_DEV_BUS_NULL_FAILURE_PC,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(CRASH_DEV_BUS_NULL_FAILURE_PC, 0, 0,
+			       DIAG_SEVERITY_ERR);
 		return;
 	}
 
@@ -1610,11 +1634,11 @@ setup_crash_devices_work_queue(struct work_struct *work)
 	if (local_crash_dev_msg.cmd.create_device.channel_addr) {
 		my_device_create(&local_crash_dev_msg);
 	} else {
-		POSTCODE_LINUX_2(CRASH_DEV_DEV_NULL_FAILURE_PC,
-				 POSTCODE_SEVERITY_ERR);
+		POSTCODE_LINUX(CRASH_DEV_DEV_NULL_FAILURE_PC, 0, 0,
+			       DIAG_SEVERITY_ERR);
 		return;
 	}
-	POSTCODE_LINUX_2(CRASH_DEV_EXIT_PC, POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(CRASH_DEV_EXIT_PC, 0, 0, DIAG_SEVERITY_PRINT);
 }
 
 void
@@ -2119,8 +2143,6 @@ visorchipset_init(struct acpi_device *acpi_device)
 	if (!addr)
 		goto error;
 
-	memset(&controlvm_payload_info, 0, sizeof(controlvm_payload_info));
-
 	controlvm_channel = visorchannel_create_with_lock(addr, 0,
 							  GFP_KERNEL, uuid);
 	if (!controlvm_channel)
@@ -2152,11 +2174,12 @@ visorchipset_init(struct acpi_device *acpi_device)
 
 	visorchipset_platform_device.dev.devt = major_dev;
 	if (platform_device_register(&visorchipset_platform_device) < 0) {
-		POSTCODE_LINUX_2(DEVICE_REGISTER_FAILURE_PC, DIAG_SEVERITY_ERR);
+		POSTCODE_LINUX(DEVICE_REGISTER_FAILURE_PC, 0, 0,
+			       DIAG_SEVERITY_ERR);
 		err = -ENODEV;
 		goto error_cancel_work;
 	}
-	POSTCODE_LINUX_2(CHIPSET_INIT_SUCCESS_PC, POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(CHIPSET_INIT_SUCCESS_PC, 0, 0, DIAG_SEVERITY_PRINT);
 
 	err = visorbus_init();
 	if (err < 0)
@@ -2178,14 +2201,14 @@ error_destroy_channel:
 	visorchannel_destroy(controlvm_channel);
 
 error:
-	POSTCODE_LINUX_3(CHIPSET_INIT_FAILURE_PC, err, POSTCODE_SEVERITY_ERR);
+	POSTCODE_LINUX(CHIPSET_INIT_FAILURE_PC, 0, err, DIAG_SEVERITY_ERR);
 	return err;
 }
 
 static int
 visorchipset_exit(struct acpi_device *acpi_device)
 {
-	POSTCODE_LINUX_2(DRIVER_EXIT_PC, POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(DRIVER_EXIT_PC, 0, 0, DIAG_SEVERITY_PRINT);
 
 	visorbus_exit();
 
@@ -2196,7 +2219,7 @@ visorchipset_exit(struct acpi_device *acpi_device)
 
 	visorchipset_file_cleanup(visorchipset_platform_device.dev.devt);
 	platform_device_unregister(&visorchipset_platform_device);
-	POSTCODE_LINUX_2(DRIVER_EXIT_PC, POSTCODE_SEVERITY_INFO);
+	POSTCODE_LINUX(DRIVER_EXIT_PC, 0, 0, DIAG_SEVERITY_PRINT);
 
 	return 0;
 }
diff --git a/drivers/staging/unisys/visorbus/vmcallinterface.h b/drivers/staging/unisys/visorbus/vmcallinterface.h
index 86e695d5a441..674a88b657d3 100644
--- a/drivers/staging/unisys/visorbus/vmcallinterface.h
+++ b/drivers/staging/unisys/visorbus/vmcallinterface.h
@@ -92,15 +92,6 @@ enum vmcall_monitor_interface_method_tuple { /* VMCALL identification tuples  */
 #define ISSUE_IO_VMCALL(method, param, result) \
 	(result = unisys_vmcall(method, (param) & 0xFFFFFFFF,	\
 				(param) >> 32))
-#define ISSUE_IO_EXTENDED_VMCALL(method, param1, param2, param3) \
-	unisys_extended_vmcall(method, param1, param2, param3)
-
-    /* The following uses VMCALL_POST_CODE_LOGEVENT interface but is currently
-     * not used much
-     */
-#define ISSUE_IO_VMCALL_POSTCODE_SEVERITY(postcode, severity)		\
-	ISSUE_IO_EXTENDED_VMCALL(VMCALL_POST_CODE_LOGEVENT, severity,	\
-				 MDS_APPOS, postcode)
 
 /* Structures for IO VMCALLs */
 
@@ -117,118 +108,53 @@ struct vmcall_io_controlvm_addr_params {
 
 /******* INFO ON ISSUE_POSTCODE_LINUX() BELOW *******/
 enum driver_pc {		/* POSTCODE driver identifier tuples */
-	/* visorchipset driver files */
-	VISOR_CHIPSET_PC = 0xA0,
-	VISOR_CHIPSET_PC_controlvm_c = 0xA1,
-	VISOR_CHIPSET_PC_controlvm_cm2 = 0xA2,
-	VISOR_CHIPSET_PC_controlvm_direct_c = 0xA3,
-	VISOR_CHIPSET_PC_file_c = 0xA4,
-	VISOR_CHIPSET_PC_parser_c = 0xA5,
-	VISOR_CHIPSET_PC_testing_c = 0xA6,
-	VISOR_CHIPSET_PC_visorchipset_main_c = 0xA7,
-	VISOR_CHIPSET_PC_visorswitchbus_c = 0xA8,
 	/* visorbus driver files */
-	VISOR_BUS_PC = 0xB0,
-	VISOR_BUS_PC_businst_attr_c = 0xB1,
-	VISOR_BUS_PC_channel_attr_c = 0xB2,
-	VISOR_BUS_PC_devmajorminor_attr_c = 0xB3,
-	VISOR_BUS_PC_visorbus_main_c = 0xB4,
-	/* visorclientbus driver files */
-	VISOR_CLIENT_BUS_PC = 0xC0,
-	VISOR_CLIENT_BUS_PC_visorclientbus_main_c = 0xC1,
-	/* virt hba driver files */
-	VIRT_HBA_PC = 0xC2,
-	VIRT_HBA_PC_virthba_c = 0xC3,
-	/* virtpci driver files */
-	VIRT_PCI_PC = 0xC4,
-	VIRT_PCI_PC_virtpci_c = 0xC5,
-	/* virtnic driver files */
-	VIRT_NIC_PC = 0xC6,
-	VIRT_NIC_P_virtnic_c = 0xC7,
-	/* uislib driver files */
-	UISLIB_PC = 0xD0,
-	UISLIB_PC_uislib_c = 0xD1,
-	UISLIB_PC_uisqueue_c = 0xD2,
-	/* 0xD3 RESERVED */
-	UISLIB_PC_uisutils_c = 0xD4,
+	VISOR_BUS_PC = 0xF0,
+	VISOR_BUS_PC_visorbus_main_c = 0xFF,
+	VISOR_BUS_PC_visorchipset_c = 0xFE,
 };
 
 enum event_pc {			/* POSTCODE event identifier tuples */
-	ATTACH_PORT_ENTRY_PC = 0x001,
-	ATTACH_PORT_FAILURE_PC = 0x002,
-	ATTACH_PORT_SUCCESS_PC = 0x003,
-	BUS_FAILURE_PC = 0x004,
-	BUS_CREATE_ENTRY_PC = 0x005,
-	BUS_CREATE_FAILURE_PC = 0x006,
-	BUS_CREATE_EXIT_PC = 0x007,
-	BUS_CONFIGURE_ENTRY_PC = 0x008,
-	BUS_CONFIGURE_FAILURE_PC = 0x009,
-	BUS_CONFIGURE_EXIT_PC = 0x00A,
-	CHIPSET_INIT_ENTRY_PC = 0x00B,
-	CHIPSET_INIT_SUCCESS_PC = 0x00C,
-	CHIPSET_INIT_FAILURE_PC = 0x00D,
-	CHIPSET_INIT_EXIT_PC = 0x00E,
-	CREATE_WORKQUEUE_PC = 0x00F,
-	CREATE_WORKQUEUE_FAILED_PC = 0x0A0,
-	CONTROLVM_INIT_FAILURE_PC = 0x0A1,
-	DEVICE_CREATE_ENTRY_PC = 0x0A2,
-	DEVICE_CREATE_FAILURE_PC = 0x0A3,
-	DEVICE_CREATE_SUCCESS_PC = 0x0A4,
-	DEVICE_CREATE_EXIT_PC = 0x0A5,
-	DEVICE_ADD_PC = 0x0A6,
-	DEVICE_REGISTER_FAILURE_PC = 0x0A7,
-	DEVICE_CHANGESTATE_ENTRY_PC = 0x0A8,
-	DEVICE_CHANGESTATE_FAILURE_PC = 0x0A9,
-	DEVICE_CHANGESTATE_EXIT_PC = 0x0AA,
-	DRIVER_ENTRY_PC = 0x0AB,
-	DRIVER_EXIT_PC = 0x0AC,
-	MALLOC_FAILURE_PC = 0x0AD,
-	QUEUE_DELAYED_WORK_PC = 0x0AE,
-	/* 0x0B7 RESERVED */
-	VBUS_CHANNEL_ENTRY_PC = 0x0B8,
-	VBUS_CHANNEL_FAILURE_PC = 0x0B9,
-	VBUS_CHANNEL_EXIT_PC = 0x0BA,
-	VHBA_CREATE_ENTRY_PC = 0x0BB,
-	VHBA_CREATE_FAILURE_PC = 0x0BC,
-	VHBA_CREATE_EXIT_PC = 0x0BD,
-	VHBA_CREATE_SUCCESS_PC = 0x0BE,
-	VHBA_COMMAND_HANDLER_PC = 0x0BF,
-	VHBA_PROBE_ENTRY_PC = 0x0C0,
-	VHBA_PROBE_FAILURE_PC = 0x0C1,
-	VHBA_PROBE_EXIT_PC = 0x0C2,
-	VNIC_CREATE_ENTRY_PC = 0x0C3,
-	VNIC_CREATE_FAILURE_PC = 0x0C4,
-	VNIC_CREATE_SUCCESS_PC = 0x0C5,
-	VNIC_PROBE_ENTRY_PC = 0x0C6,
-	VNIC_PROBE_FAILURE_PC = 0x0C7,
-	VNIC_PROBE_EXIT_PC = 0x0C8,
-	VPCI_CREATE_ENTRY_PC = 0x0C9,
-	VPCI_CREATE_FAILURE_PC = 0x0CA,
-	VPCI_CREATE_EXIT_PC = 0x0CB,
-	VPCI_PROBE_ENTRY_PC = 0x0CC,
-	VPCI_PROBE_FAILURE_PC = 0x0CD,
-	VPCI_PROBE_EXIT_PC = 0x0CE,
-	CRASH_DEV_ENTRY_PC = 0x0CF,
-	CRASH_DEV_EXIT_PC = 0x0D0,
-	CRASH_DEV_HADDR_NULL = 0x0D1,
-	CRASH_DEV_CONTROLVM_NULL = 0x0D2,
-	CRASH_DEV_RD_BUS_FAIULRE_PC = 0x0D3,
-	CRASH_DEV_RD_DEV_FAIULRE_PC = 0x0D4,
-	CRASH_DEV_BUS_NULL_FAILURE_PC = 0x0D5,
-	CRASH_DEV_DEV_NULL_FAILURE_PC = 0x0D6,
-	CRASH_DEV_CTRL_RD_FAILURE_PC = 0x0D7,
-	CRASH_DEV_COUNT_FAILURE_PC = 0x0D8,
-	SAVE_MSG_BUS_FAILURE_PC = 0x0D9,
-	SAVE_MSG_DEV_FAILURE_PC = 0x0DA,
-	CALLHOME_INIT_FAILURE_PC = 0x0DB
+	BUS_CREATE_ENTRY_PC = 0x001,
+	BUS_CREATE_FAILURE_PC = 0x002,
+	BUS_CREATE_EXIT_PC = 0x003,
+	BUS_CONFIGURE_ENTRY_PC = 0x004,
+	BUS_CONFIGURE_FAILURE_PC = 0x005,
+	BUS_CONFIGURE_EXIT_PC = 0x006,
+	CHIPSET_INIT_ENTRY_PC = 0x007,
+	CHIPSET_INIT_SUCCESS_PC = 0x008,
+	CHIPSET_INIT_FAILURE_PC = 0x009,
+	CHIPSET_INIT_EXIT_PC = 0x00A,
+	CONTROLVM_INIT_FAILURE_PC = 0x00B,
+	DEVICE_CREATE_ENTRY_PC = 0x00C,
+	DEVICE_CREATE_FAILURE_PC = 0x00D,
+	DEVICE_CREATE_SUCCESS_PC = 0x00E,
+	DEVICE_CREATE_EXIT_PC = 0x00F,
+	DEVICE_ADD_PC = 0x010,
+	DEVICE_REGISTER_FAILURE_PC = 0x011,
+	DEVICE_CHANGESTATE_FAILURE_PC = 0x012,
+	DRIVER_ENTRY_PC = 0x013,
+	DRIVER_EXIT_PC = 0x014,
+	MALLOC_FAILURE_PC = 0x015,
+	CRASH_DEV_ENTRY_PC = 0x016,
+	CRASH_DEV_EXIT_PC = 0x017,
+	CRASH_DEV_RD_BUS_FAILURE_PC = 0x018,
+	CRASH_DEV_RD_DEV_FAILURE_PC = 0x019,
+	CRASH_DEV_BUS_NULL_FAILURE_PC = 0x01A,
+	CRASH_DEV_DEV_NULL_FAILURE_PC = 0x01B,
+	CRASH_DEV_CTRL_RD_FAILURE_PC = 0x01C,
+	CRASH_DEV_COUNT_FAILURE_PC = 0x01D,
+	SAVE_MSG_BUS_FAILURE_PC = 0x01E,
+	SAVE_MSG_DEV_FAILURE_PC = 0x01F,
 };
 
-#define POSTCODE_SEVERITY_ERR DIAG_SEVERITY_ERR
-#define POSTCODE_SEVERITY_WARNING DIAG_SEVERITY_WARNING
-/* TODO-> Info currently doesn't show, so we set info=warning */
-#define POSTCODE_SEVERITY_INFO DIAG_SEVERITY_PRINT
-
-/* example call of POSTCODE_LINUX_2(VISOR_CHIPSET_PC, POSTCODE_SEVERITY_ERR);
+/* Write a 64-bit value to the hypervisor's log file
+ * POSTCODE_LINUX generates a value in the form 0xAABBBCCCDDDDEEEE where
+ *	A is an identifier for the file logging the postcode
+ *	B is an identifier for the event logging the postcode
+ *	C is the line logging the postcode
+ *	D is additional information the caller wants to log
+ *	E is additional information the caller wants to log
  * Please also note that the resulting postcode is in hex, so if you are
  * searching for the __LINE__ number, convert it first to decimal.  The line
  * number combined with driver and type of call, will allow you to track down
@@ -236,35 +162,16 @@ enum event_pc {			/* POSTCODE event identifier tuples */
  * entered/exited from.
  */
 
-/* BASE FUNCTIONS */
-#define POSTCODE_LINUX_A(DRIVER_PC, EVENT_PC, pc32bit, severity)	\
-do {									\
-	unsigned long long post_code_temp;				\
-	post_code_temp = (((u64)DRIVER_PC) << 56) | (((u64)EVENT_PC) << 44) | \
-		((((u64)__LINE__) & 0xFFF) << 32) |			\
-		(((u64)pc32bit) & 0xFFFFFFFF);				\
-	ISSUE_IO_VMCALL_POSTCODE_SEVERITY(post_code_temp, severity);	\
-} while (0)
-
-#define POSTCODE_LINUX_B(DRIVER_PC, EVENT_PC, pc16bit1, pc16bit2, severity) \
+#define POSTCODE_LINUX(EVENT_PC, pc16bit1, pc16bit2, severity)		\
 do {									\
 	unsigned long long post_code_temp;				\
-	post_code_temp = (((u64)DRIVER_PC) << 56) | (((u64)EVENT_PC) << 44) | \
+	post_code_temp = (((u64)CURRENT_FILE_PC) << 56) |		\
+		(((u64)EVENT_PC) << 44) |				\
 		((((u64)__LINE__) & 0xFFF) << 32) |			\
 		((((u64)pc16bit1) & 0xFFFF) << 16) |			\
 		(((u64)pc16bit2) & 0xFFFF);				\
-	ISSUE_IO_VMCALL_POSTCODE_SEVERITY(post_code_temp, severity);	\
+	unisys_extended_vmcall(VMCALL_POST_CODE_LOGEVENT, severity,     \
+			       MDS_APPOS, post_code_temp);              \
 } while (0)
 
-/* MOST COMMON */
-#define POSTCODE_LINUX_2(EVENT_PC, severity)				\
-	POSTCODE_LINUX_A(CURRENT_FILE_PC, EVENT_PC, 0x0000, severity)
-
-#define POSTCODE_LINUX_3(EVENT_PC, pc32bit, severity)			\
-	POSTCODE_LINUX_A(CURRENT_FILE_PC, EVENT_PC, pc32bit, severity)
-
-#define POSTCODE_LINUX_4(EVENT_PC, pc16bit1, pc16bit2, severity)	\
-	POSTCODE_LINUX_B(CURRENT_FILE_PC, EVENT_PC, pc16bit1,		\
-			 pc16bit2, severity)
-
 #endif /* __IOMONINTF_H__ */
diff --git a/drivers/staging/unisys/visorinput/visorinput.c b/drivers/staging/unisys/visorinput/visorinput.c
index 6f94b646f7c5..949cce680b29 100644
--- a/drivers/staging/unisys/visorinput/visorinput.c
+++ b/drivers/staging/unisys/visorinput/visorinput.c
@@ -409,6 +409,9 @@ devdata_create(struct visor_device *dev, enum visorinput_device_type devtype)
 		if (!devdata->visorinput_dev)
 			goto cleanups_register;
 		break;
+	default:
+		/* No other input devices supported */
+		break;
 	}
 
 	dev_set_drvdata(&dev->device, devdata);
@@ -653,6 +656,9 @@ visorinput_channel_interrupt(struct visor_device *dev)
 			input_report_rel(visorinput_dev, REL_WHEEL, -1);
 			input_sync(visorinput_dev);
 			break;
+		default:
+			/* Unsupported input action */
+			break;
 		}
 	}
 }
diff --git a/drivers/staging/unisys/visornic/visornic_main.c b/drivers/staging/unisys/visornic/visornic_main.c
index f8a584bf4a77..c1f674f5268c 100644
--- a/drivers/staging/unisys/visornic/visornic_main.c
+++ b/drivers/staging/unisys/visornic/visornic_main.c
@@ -1371,7 +1371,7 @@ static ssize_t info_debugfs_read(struct file *file, char __user *buf,
 				     " num_rcv_bufs = %d\n",
 				     devdata->num_rcv_bufs);
 		str_pos += scnprintf(vbuf + str_pos, len - str_pos,
-				     " max_oustanding_next_xmits = %lu\n",
+				     " max_outstanding_next_xmits = %lu\n",
 				    devdata->max_outstanding_net_xmits);
 		str_pos += scnprintf(vbuf + str_pos, len - str_pos,
 				     " upper_threshold_net_xmits = %lu\n",
diff --git a/drivers/staging/vc04_services/Kconfig b/drivers/staging/vc04_services/Kconfig
index 9676fb29075a..e61e4ca064a8 100644
--- a/drivers/staging/vc04_services/Kconfig
+++ b/drivers/staging/vc04_services/Kconfig
@@ -1,9 +1,10 @@
-config BCM2708_VCHIQ
+config BCM2835_VCHIQ
 	tristate "Videocore VCHIQ"
-	depends on RASPBERRYPI_FIRMWARE && BROKEN
+	depends on HAS_DMA
+	depends on RASPBERRYPI_FIRMWARE || (COMPILE_TEST && !RASPBERRYPI_FIRMWARE)
 	default y
 	help
 		Kernel to VideoCore communication interface for the
-		BCM2708 family of products.
+		BCM2835 family of products.
 		Defaults to Y when the Broadcom Videocore services
 		are included in the build, N otherwise.
diff --git a/drivers/staging/vc04_services/Makefile b/drivers/staging/vc04_services/Makefile
index 90ab4781df2c..1a9e742ee40d 100644
--- a/drivers/staging/vc04_services/Makefile
+++ b/drivers/staging/vc04_services/Makefile
@@ -1,4 +1,4 @@
-obj-$(CONFIG_BCM2708_VCHIQ)	+= vchiq.o
+obj-$(CONFIG_BCM2835_VCHIQ)	+= vchiq.o
 
 vchiq-objs := \
    interface/vchiq_arm/vchiq_core.o  \
diff --git a/drivers/staging/vc04_services/interface/vchi/TODO b/drivers/staging/vc04_services/interface/vchi/TODO
new file mode 100644
index 000000000000..03aa65183b25
--- /dev/null
+++ b/drivers/staging/vc04_services/interface/vchi/TODO
@@ -0,0 +1,50 @@
+1) Port to aarch64
+
+This driver won't be very useful unless we also have it working on
+Raspberry Pi 3.  This requires, at least:
+
+  - Figure out an alternative to the dmac_map_area() hack.
+
+  - Decide what to use instead of dsb().
+
+  - Do something about (int) cast of bulk->data in
+    vchiq_bulk_transfer().
+
+    bulk->data is a bus address going across to the firmware.  We know
+    our bus addresses are <32bit.
+
+2) Write a DT binding doc and get the corresponding DT node merged to
+   bcm2835.
+
+This will let the driver probe when enabled.
+
+3) Import drivers using VCHI.
+
+VCHI is just a tool to let drivers talk to the firmware.  Here are
+some of the ones we want:
+
+  - vc_mem (https://github.com/raspberrypi/linux/blob/rpi-4.4.y/drivers/char/broadcom/vc_mem.c)
+
+  This driver is what the vcdbg userspace program uses to set up its
+  requests to the firmware, which are transmitted across VCHIQ.  vcdbg
+  is really useful for debugging firmware interactions.
+
+  - bcm2835-camera (https://github.com/raspberrypi/linux/tree/rpi-4.4.y/drivers/media/platform/bcm2835)
+
+  This driver will let us get images from the camera using the MMAL
+  protocol over VCHI.
+
+  - VCSM (https://github.com/raspberrypi/linux/tree/rpi-4.4.y/drivers/char/broadcom/vc_sm)
+
+  This driver is used for talking about regions of VC memory across
+  firmware protocols including VCHI.  We'll want to extend this driver
+  to manage these buffers as dmabufs so that we can zero-copy import
+  camera images into vc4 for rendering/display.
+
+4) Garbage-collect unused code
+
+One of the reasons this driver wasn't upstreamed previously was that
+there's a lot code that got built that's probably unnecessary these
+days.  Once we have the set of VCHI-using drivers we want in tree, we
+should be able to do a sweep of the code to see what's left that's
+unused.
diff --git a/drivers/staging/vc04_services/interface/vchi/vchi.h b/drivers/staging/vc04_services/interface/vchi/vchi.h
index 1b17e98f7379..d6937288210c 100644
--- a/drivers/staging/vc04_services/interface/vchi/vchi.h
+++ b/drivers/staging/vc04_services/interface/vchi/vchi.h
@@ -226,25 +226,12 @@ extern int32_t vchi_service_set_option( const VCHI_SERVICE_HANDLE_T handle,
 					int value);
 
 // Routine to send a message across a service
-extern int32_t vchi_msg_queue( VCHI_SERVICE_HANDLE_T handle,
-                               const void *data,
-                               uint32_t data_size,
-                               VCHI_FLAGS_T flags,
-                               void *msg_handle );
-
-// scatter-gather (vector) and send message
-int32_t vchi_msg_queuev_ex( VCHI_SERVICE_HANDLE_T handle,
-                            VCHI_MSG_VECTOR_EX_T *vector,
-                            uint32_t count,
-                            VCHI_FLAGS_T flags,
-                            void *msg_handle );
-
-// legacy scatter-gather (vector) and send message, only handles pointers
-int32_t vchi_msg_queuev( VCHI_SERVICE_HANDLE_T handle,
-                         VCHI_MSG_VECTOR_T *vector,
-                         uint32_t count,
-                         VCHI_FLAGS_T flags,
-                         void *msg_handle );
+extern int32_t
+	vchi_msg_queue(VCHI_SERVICE_HANDLE_T handle,
+		       ssize_t (*copy_callback)(void *context, void *dest,
+						size_t offset, size_t maxsize),
+		       void *context,
+		       uint32_t data_size);
 
 // Routine to receive a msg from a service
 // Dequeue is equivalent to hold, copy into client buffer, release
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq.h
index ad398bae6ee4..21adf89a9065 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq.h
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq.h
@@ -37,4 +37,15 @@
 #include "vchiq_if.h"
 #include "vchiq_util.h"
 
+/* Do this so that we can test-build the code on non-rpi systems */
+#if IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE)
+
+#else
+
+#ifndef dsb
+#define dsb(a)
+#endif
+
+#endif	/* IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) */
+
 #endif
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c
index 1091b9f1dd07..2b500d85cebc 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c
@@ -45,16 +45,8 @@
 #include <asm/pgtable.h>
 #include <soc/bcm2835/raspberrypi-firmware.h>
 
-#define dmac_map_area			__glue(_CACHE,_dma_map_area)
-#define dmac_unmap_area 		__glue(_CACHE,_dma_unmap_area)
-
-extern void dmac_map_area(const void *, size_t, int);
-extern void dmac_unmap_area(const void *, size_t, int);
-
 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
 
-#define VCHIQ_ARM_ADDRESS(x) ((void *)((char *)x + g_virt_to_bus_offset))
-
 #include "vchiq_arm.h"
 #include "vchiq_2835.h"
 #include "vchiq_connected.h"
@@ -70,13 +62,25 @@ typedef struct vchiq_2835_state_struct {
    VCHIQ_ARM_STATE_T arm_state;
 } VCHIQ_2835_ARM_STATE_T;
 
+struct vchiq_pagelist_info {
+	PAGELIST_T *pagelist;
+	size_t pagelist_buffer_size;
+	dma_addr_t dma_addr;
+	enum dma_data_direction dma_dir;
+	unsigned int num_pages;
+	unsigned int pages_need_release;
+	struct page **pages;
+	struct scatterlist *scatterlist;
+	unsigned int scatterlist_mapped;
+};
+
 static void __iomem *g_regs;
 static unsigned int g_cache_line_size = sizeof(CACHE_LINE_SIZE);
 static unsigned int g_fragments_size;
 static char *g_fragments_base;
 static char *g_free_fragments;
 static struct semaphore g_free_fragments_sema;
-static unsigned long g_virt_to_bus_offset;
+static struct device *g_dev;
 
 extern int vchiq_arm_log_level;
 
@@ -85,12 +89,13 @@ static DEFINE_SEMAPHORE(g_free_fragments_mutex);
 static irqreturn_t
 vchiq_doorbell_irq(int irq, void *dev_id);
 
-static int
+static struct vchiq_pagelist_info *
 create_pagelist(char __user *buf, size_t count, unsigned short type,
-                struct task_struct *task, PAGELIST_T ** ppagelist);
+		struct task_struct *task);
 
 static void
-free_pagelist(PAGELIST_T *pagelist, int actual);
+free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
+	      int actual);
 
 int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state)
 {
@@ -104,7 +109,14 @@ int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state)
 	int slot_mem_size, frag_mem_size;
 	int err, irq, i;
 
-	g_virt_to_bus_offset = virt_to_dma(dev, (void *)0);
+	/*
+	 * VCHI messages between the CPU and firmware use
+	 * 32-bit bus addresses.
+	 */
+	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+
+	if (err < 0)
+		return err;
 
 	(void)of_property_read_u32(dev->of_node, "cache-line-size",
 				   &g_cache_line_size);
@@ -121,7 +133,7 @@ int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state)
 		return -ENOMEM;
 	}
 
-	WARN_ON(((int)slot_mem & (PAGE_SIZE - 1)) != 0);
+	WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0);
 
 	vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
 	if (!vchiq_slot_zero)
@@ -173,9 +185,10 @@ int vchiq_platform_init(struct platform_device *pdev, VCHIQ_STATE_T *state)
 		return err ? : -ENXIO;
 	}
 
+	g_dev = dev;
 	vchiq_log_info(vchiq_arm_log_level,
-		"vchiq_init - done (slots %x, phys %pad)",
-		(unsigned int)vchiq_slot_zero, &slot_phys);
+		"vchiq_init - done (slots %pK, phys %pad)",
+		vchiq_slot_zero, &slot_phys);
 
 	vchiq_call_connected_callbacks();
 
@@ -213,47 +226,37 @@ remote_event_signal(REMOTE_EVENT_T *event)
 
 	event->fired = 1;
 
-	dsb();         /* data barrier operation */
+	dsb(sy);         /* data barrier operation */
 
 	if (event->armed)
 		writel(0, g_regs + BELL2); /* trigger vc interrupt */
 }
 
-int
-vchiq_copy_from_user(void *dst, const void *src, int size)
-{
-	if ((uint32_t)src < TASK_SIZE) {
-		return copy_from_user(dst, src, size);
-	} else {
-		memcpy(dst, src, size);
-		return 0;
-	}
-}
-
 VCHIQ_STATUS_T
 vchiq_prepare_bulk_data(VCHIQ_BULK_T *bulk, VCHI_MEM_HANDLE_T memhandle,
 	void *offset, int size, int dir)
 {
-	PAGELIST_T *pagelist;
-	int ret;
+	struct vchiq_pagelist_info *pagelistinfo;
 
 	WARN_ON(memhandle != VCHI_MEM_HANDLE_INVALID);
 
-	ret = create_pagelist((char __user *)offset, size,
-			(dir == VCHIQ_BULK_RECEIVE)
-			? PAGELIST_READ
-			: PAGELIST_WRITE,
-			current,
-			&pagelist);
-	if (ret != 0)
+	pagelistinfo = create_pagelist((char __user *)offset, size,
+				       (dir == VCHIQ_BULK_RECEIVE)
+				       ? PAGELIST_READ
+				       : PAGELIST_WRITE,
+				       current);
+
+	if (!pagelistinfo)
 		return VCHIQ_ERROR;
 
 	bulk->handle = memhandle;
-	bulk->data = VCHIQ_ARM_ADDRESS(pagelist);
+	bulk->data = (void *)(unsigned long)pagelistinfo->dma_addr;
 
-	/* Store the pagelist address in remote_data, which isn't used by the
-	   slave. */
-	bulk->remote_data = pagelist;
+	/*
+	 * Store the pagelistinfo address in remote_data,
+	 * which isn't used by the slave.
+	 */
+	bulk->remote_data = pagelistinfo;
 
 	return VCHIQ_SUCCESS;
 }
@@ -262,7 +265,8 @@ void
 vchiq_complete_bulk(VCHIQ_BULK_T *bulk)
 {
 	if (bulk && bulk->remote_data && bulk->actual)
-		free_pagelist((PAGELIST_T *)bulk->remote_data, bulk->actual);
+		free_pagelist((struct vchiq_pagelist_info *)bulk->remote_data,
+			      bulk->actual);
 }
 
 void
@@ -350,57 +354,93 @@ vchiq_doorbell_irq(int irq, void *dev_id)
 	return ret;
 }
 
+static void
+cleaup_pagelistinfo(struct vchiq_pagelist_info *pagelistinfo)
+{
+	if (pagelistinfo->scatterlist_mapped) {
+		dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
+			     pagelistinfo->num_pages, pagelistinfo->dma_dir);
+	}
+
+	if (pagelistinfo->pages_need_release) {
+		unsigned int i;
+
+		for (i = 0; i < pagelistinfo->num_pages; i++)
+			put_page(pagelistinfo->pages[i]);
+	}
+
+	dma_free_coherent(g_dev, pagelistinfo->pagelist_buffer_size,
+			  pagelistinfo->pagelist, pagelistinfo->dma_addr);
+}
+
 /* There is a potential problem with partial cache lines (pages?)
 ** at the ends of the block when reading. If the CPU accessed anything in
 ** the same line (page?) then it may have pulled old data into the cache,
 ** obscuring the new data underneath. We can solve this by transferring the
 ** partial cache lines separately, and allowing the ARM to copy into the
 ** cached area.
-
-** N.B. This implementation plays slightly fast and loose with the Linux
-** driver programming rules, e.g. its use of dmac_map_area instead of
-** dma_map_single, but it isn't a multi-platform driver and it benefits
-** from increased speed as a result.
 */
 
-static int
+static struct vchiq_pagelist_info *
 create_pagelist(char __user *buf, size_t count, unsigned short type,
-	struct task_struct *task, PAGELIST_T ** ppagelist)
+		struct task_struct *task)
 {
 	PAGELIST_T *pagelist;
+	struct vchiq_pagelist_info *pagelistinfo;
 	struct page **pages;
-	unsigned long *addrs;
-	unsigned int num_pages, offset, i;
-	char *addr, *base_addr, *next_addr;
-	int run, addridx, actual_pages;
-        unsigned long *need_release;
-
-	offset = (unsigned int)buf & (PAGE_SIZE - 1);
+	u32 *addrs;
+	unsigned int num_pages, offset, i, k;
+	int actual_pages;
+	size_t pagelist_size;
+	struct scatterlist *scatterlist, *sg;
+	int dma_buffers;
+	dma_addr_t dma_addr;
+
+	offset = ((unsigned int)(unsigned long)buf & (PAGE_SIZE - 1));
 	num_pages = (count + offset + PAGE_SIZE - 1) / PAGE_SIZE;
 
-	*ppagelist = NULL;
+	pagelist_size = sizeof(PAGELIST_T) +
+			(num_pages * sizeof(u32)) +
+			(num_pages * sizeof(pages[0]) +
+			(num_pages * sizeof(struct scatterlist))) +
+			sizeof(struct vchiq_pagelist_info);
 
 	/* Allocate enough storage to hold the page pointers and the page
 	** list
 	*/
-	pagelist = kmalloc(sizeof(PAGELIST_T) +
-                           (num_pages * sizeof(unsigned long)) +
-                           sizeof(unsigned long) +
-                           (num_pages * sizeof(pages[0])),
-                           GFP_KERNEL);
-
-	vchiq_log_trace(vchiq_arm_log_level,
-		"create_pagelist - %x", (unsigned int)pagelist);
+	pagelist = dma_zalloc_coherent(g_dev,
+				       pagelist_size,
+				       &dma_addr,
+				       GFP_KERNEL);
+
+	vchiq_log_trace(vchiq_arm_log_level, "create_pagelist - %pK",
+			pagelist);
 	if (!pagelist)
-		return -ENOMEM;
+		return NULL;
 
-	addrs = pagelist->addrs;
-        need_release = (unsigned long *)(addrs + num_pages);
-	pages = (struct page **)(addrs + num_pages + 1);
+	addrs		= pagelist->addrs;
+	pages		= (struct page **)(addrs + num_pages);
+	scatterlist	= (struct scatterlist *)(pages + num_pages);
+	pagelistinfo	= (struct vchiq_pagelist_info *)
+			  (scatterlist + num_pages);
+
+	pagelist->length = count;
+	pagelist->type = type;
+	pagelist->offset = offset;
+
+	/* Populate the fields of the pagelistinfo structure */
+	pagelistinfo->pagelist = pagelist;
+	pagelistinfo->pagelist_buffer_size = pagelist_size;
+	pagelistinfo->dma_addr = dma_addr;
+	pagelistinfo->dma_dir =  (type == PAGELIST_WRITE) ?
+				  DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	pagelistinfo->num_pages = num_pages;
+	pagelistinfo->pages_need_release = 0;
+	pagelistinfo->pages = pages;
+	pagelistinfo->scatterlist = scatterlist;
+	pagelistinfo->scatterlist_mapped = 0;
 
 	if (is_vmalloc_addr(buf)) {
-		int dir = (type == PAGELIST_WRITE) ?
-			DMA_TO_DEVICE : DMA_FROM_DEVICE;
 		unsigned long length = count;
 		unsigned int off = offset;
 
@@ -413,14 +453,13 @@ create_pagelist(char __user *buf, size_t count, unsigned short type,
 			if (bytes > length)
 				bytes = length;
 			pages[actual_pages] = pg;
-			dmac_map_area(page_address(pg) + off, bytes, dir);
 			length -= bytes;
 			off = 0;
 		}
-		*need_release = 0; /* do not try and release vmalloc pages */
+		/* do not try and release vmalloc pages */
 	} else {
 		down_read(&task->mm->mmap_sem);
-		actual_pages = get_user_pages(task, task->mm,
+		actual_pages = get_user_pages(
 				          (unsigned long)buf & ~(PAGE_SIZE - 1),
 					  num_pages,
 					  (type == PAGELIST_READ) ? FOLL_WRITE : 0,
@@ -438,44 +477,59 @@ create_pagelist(char __user *buf, size_t count, unsigned short type,
 			while (actual_pages > 0)
 			{
 				actual_pages--;
-				page_cache_release(pages[actual_pages]);
+				put_page(pages[actual_pages]);
 			}
-			kfree(pagelist);
-			if (actual_pages == 0)
-				actual_pages = -ENOMEM;
-			return actual_pages;
+			cleaup_pagelistinfo(pagelistinfo);
+			return NULL;
 		}
-		*need_release = 1; /* release user pages */
+		 /* release user pages */
+		pagelistinfo->pages_need_release = 1;
 	}
 
-	pagelist->length = count;
-	pagelist->type = type;
-	pagelist->offset = offset;
-
-	/* Group the pages into runs of contiguous pages */
-
-	base_addr = VCHIQ_ARM_ADDRESS(page_address(pages[0]));
-	next_addr = base_addr + PAGE_SIZE;
-	addridx = 0;
-	run = 0;
+	/*
+	 * Initialize the scatterlist so that the magic cookie
+	 *  is filled if debugging is enabled
+	 */
+	sg_init_table(scatterlist, num_pages);
+	/* Now set the pages for each scatterlist */
+	for (i = 0; i < num_pages; i++)
+		sg_set_page(scatterlist + i, pages[i], PAGE_SIZE, 0);
+
+	dma_buffers = dma_map_sg(g_dev,
+				 scatterlist,
+				 num_pages,
+				 pagelistinfo->dma_dir);
+
+	if (dma_buffers == 0) {
+		cleaup_pagelistinfo(pagelistinfo);
+		return NULL;
+	}
 
-	for (i = 1; i < num_pages; i++) {
-		addr = VCHIQ_ARM_ADDRESS(page_address(pages[i]));
-		if ((addr == next_addr) && (run < (PAGE_SIZE - 1))) {
-			next_addr += PAGE_SIZE;
-			run++;
+	pagelistinfo->scatterlist_mapped = 1;
+
+	/* Combine adjacent blocks for performance */
+	k = 0;
+	for_each_sg(scatterlist, sg, dma_buffers, i) {
+		u32 len = sg_dma_len(sg);
+		u32 addr = sg_dma_address(sg);
+
+		/* Note: addrs is the address + page_count - 1
+		 * The firmware expects the block to be page
+		 * aligned and a multiple of the page size
+		 */
+		WARN_ON(len == 0);
+		WARN_ON(len & ~PAGE_MASK);
+		WARN_ON(addr & ~PAGE_MASK);
+		if (k > 0 &&
+		    ((addrs[k - 1] & PAGE_MASK) |
+			((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT)
+		    == addr) {
+			addrs[k - 1] += (len >> PAGE_SHIFT);
 		} else {
-			addrs[addridx] = (unsigned long)base_addr + run;
-			addridx++;
-			base_addr = addr;
-			next_addr = addr + PAGE_SIZE;
-			run = 0;
+			addrs[k++] = addr | ((len >> PAGE_SHIFT) - 1);
 		}
 	}
 
-	addrs[addridx] = (unsigned long)base_addr + run;
-	addridx++;
-
 	/* Partial cache lines (fragments) require special measures */
 	if ((type == PAGELIST_READ) &&
 		((pagelist->offset & (g_cache_line_size - 1)) ||
@@ -484,8 +538,8 @@ create_pagelist(char __user *buf, size_t count, unsigned short type,
 		char *fragments;
 
 		if (down_interruptible(&g_free_fragments_sema) != 0) {
-			kfree(pagelist);
-			return -EINTR;
+			cleaup_pagelistinfo(pagelistinfo);
+			return NULL;
 		}
 
 		WARN_ON(g_free_fragments == NULL);
@@ -499,29 +553,28 @@ create_pagelist(char __user *buf, size_t count, unsigned short type,
 			(fragments - g_fragments_base) / g_fragments_size;
 	}
 
-	dmac_flush_range(pagelist, addrs + num_pages);
-
-	*ppagelist = pagelist;
-
-	return 0;
+	return pagelistinfo;
 }
 
 static void
-free_pagelist(PAGELIST_T *pagelist, int actual)
+free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
+	      int actual)
 {
-        unsigned long *need_release;
-	struct page **pages;
-	unsigned int num_pages, i;
-
-	vchiq_log_trace(vchiq_arm_log_level,
-		"free_pagelist - %x, %d", (unsigned int)pagelist, actual);
+	unsigned int i;
+	PAGELIST_T *pagelist   = pagelistinfo->pagelist;
+	struct page **pages    = pagelistinfo->pages;
+	unsigned int num_pages = pagelistinfo->num_pages;
 
-	num_pages =
-		(pagelist->length + pagelist->offset + PAGE_SIZE - 1) /
-		PAGE_SIZE;
+	vchiq_log_trace(vchiq_arm_log_level, "free_pagelist - %pK, %d",
+			pagelistinfo->pagelist, actual);
 
-        need_release = (unsigned long *)(pagelist->addrs + num_pages);
-	pages = (struct page **)(pagelist->addrs + num_pages + 1);
+	/*
+	 * NOTE: dma_unmap_sg must be called before the
+	 * cpu can touch any of the data/pages.
+	 */
+	dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
+		     pagelistinfo->num_pages, pagelistinfo->dma_dir);
+	pagelistinfo->scatterlist_mapped = 0;
 
 	/* Deal with any partial cache lines (fragments) */
 	if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
@@ -559,27 +612,12 @@ free_pagelist(PAGELIST_T *pagelist, int actual)
 		up(&g_free_fragments_sema);
 	}
 
-	if (*need_release) {
-		unsigned int length = pagelist->length;
-		unsigned int offset = pagelist->offset;
-
-		for (i = 0; i < num_pages; i++) {
-			struct page *pg = pages[i];
-
-			if (pagelist->type != PAGELIST_WRITE) {
-				unsigned int bytes = PAGE_SIZE - offset;
-
-				if (bytes > length)
-					bytes = length;
-				dmac_unmap_area(page_address(pg) + offset,
-						bytes, DMA_FROM_DEVICE);
-				length -= bytes;
-				offset = 0;
-				set_page_dirty(pg);
-			}
-			page_cache_release(pg);
-		}
+	/* Need to mark all the pages dirty. */
+	if (pagelist->type != PAGELIST_WRITE &&
+	    pagelistinfo->pages_need_release) {
+		for (i = 0; i < num_pages; i++)
+			set_page_dirty(pages[i]);
 	}
 
-	kfree(pagelist);
+	cleaup_pagelistinfo(pagelistinfo);
 }
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
index 7b6cd4d80621..0d987898b4f8 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
@@ -190,8 +190,8 @@ static const char *const ioctl_names[] = {
 	"CLOSE_DELIVERED"
 };
 
-vchiq_static_assert((sizeof(ioctl_names)/sizeof(ioctl_names[0])) ==
-	(VCHIQ_IOC_MAX + 1));
+vchiq_static_assert(ARRAY_SIZE(ioctl_names) ==
+		    (VCHIQ_IOC_MAX + 1));
 
 static void
 dump_phys_mem(void *virt_addr, uint32_t num_bytes);
@@ -402,6 +402,107 @@ static void close_delivered(USER_SERVICE_T *user_service)
 	}
 }
 
+struct vchiq_io_copy_callback_context {
+	VCHIQ_ELEMENT_T *current_element;
+	size_t current_element_offset;
+	unsigned long elements_to_go;
+	size_t current_offset;
+};
+
+static ssize_t
+vchiq_ioc_copy_element_data(
+	void *context,
+	void *dest,
+	size_t offset,
+	size_t maxsize)
+{
+	long res;
+	size_t bytes_this_round;
+	struct vchiq_io_copy_callback_context *copy_context =
+		(struct vchiq_io_copy_callback_context *)context;
+
+	if (offset != copy_context->current_offset)
+		return 0;
+
+	if (!copy_context->elements_to_go)
+		return 0;
+
+	/*
+	 * Complex logic here to handle the case of 0 size elements
+	 * in the middle of the array of elements.
+	 *
+	 * Need to skip over these 0 size elements.
+	 */
+	while (1) {
+		bytes_this_round = min(copy_context->current_element->size -
+				       copy_context->current_element_offset,
+				       maxsize);
+
+		if (bytes_this_round)
+			break;
+
+		copy_context->elements_to_go--;
+		copy_context->current_element++;
+		copy_context->current_element_offset = 0;
+
+		if (!copy_context->elements_to_go)
+			return 0;
+	}
+
+	res = copy_from_user(dest,
+			     copy_context->current_element->data +
+			     copy_context->current_element_offset,
+			     bytes_this_round);
+
+	if (res != 0)
+		return -EFAULT;
+
+	copy_context->current_element_offset += bytes_this_round;
+	copy_context->current_offset += bytes_this_round;
+
+	/*
+	 * Check if done with current element, and if so advance to the next.
+	 */
+	if (copy_context->current_element_offset ==
+	    copy_context->current_element->size) {
+		copy_context->elements_to_go--;
+		copy_context->current_element++;
+		copy_context->current_element_offset = 0;
+	}
+
+	return bytes_this_round;
+}
+
+/**************************************************************************
+ *
+ *   vchiq_ioc_queue_message
+ *
+ **************************************************************************/
+static VCHIQ_STATUS_T
+vchiq_ioc_queue_message(VCHIQ_SERVICE_HANDLE_T handle,
+			VCHIQ_ELEMENT_T *elements,
+			unsigned long count)
+{
+	struct vchiq_io_copy_callback_context context;
+	unsigned long i;
+	size_t total_size = 0;
+
+	context.current_element = elements;
+	context.current_element_offset = 0;
+	context.elements_to_go = count;
+	context.current_offset = 0;
+
+	for (i = 0; i < count; i++) {
+		if (!elements[i].data && elements[i].size != 0)
+			return -EFAULT;
+
+		total_size += elements[i].size;
+	}
+
+	return vchiq_queue_message(handle, vchiq_ioc_copy_element_data,
+				   &context, total_size);
+}
+
 /****************************************************************************
 *
 *   vchiq_ioctl
@@ -418,8 +519,8 @@ vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 	DEBUG_INITIALISE(g_state.local)
 
 	vchiq_log_trace(vchiq_arm_log_level,
-		 "vchiq_ioctl - instance %x, cmd %s, arg %lx",
-		(unsigned int)instance,
+		"vchiq_ioctl - instance %pK, cmd %s, arg %lx",
+		instance,
 		((_IOC_TYPE(cmd) == VCHIQ_IOC_MAGIC) &&
 		(_IOC_NR(cmd) <= VCHIQ_IOC_MAX)) ?
 		ioctl_names[_IOC_NR(cmd)] : "<invalid>", arg);
@@ -453,7 +554,7 @@ vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 			ret = -EINVAL;
 			break;
 		}
-		rc = mutex_lock_interruptible(&instance->state->mutex);
+		rc = mutex_lock_killable(&instance->state->mutex);
 		if (rc != 0) {
 			vchiq_log_error(vchiq_arm_log_level,
 				"vchiq: connect: could not lock mutex for "
@@ -651,7 +752,7 @@ vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 			VCHIQ_ELEMENT_T elements[MAX_ELEMENTS];
 			if (copy_from_user(elements, args.elements,
 				args.count * sizeof(VCHIQ_ELEMENT_T)) == 0)
-				status = vchiq_queue_message
+				status = vchiq_ioc_queue_message
 					(args.handle,
 					elements, args.count);
 			else
@@ -713,8 +814,8 @@ vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 				break;
 			}
 			vchiq_log_info(vchiq_arm_log_level,
-				"found bulk_waiter %x for pid %d",
-				(unsigned int)waiter, current->pid);
+				"found bulk_waiter %pK for pid %d", waiter,
+				current->pid);
 			args.userdata = &waiter->bulk_waiter;
 		}
 		status = vchiq_bulk_transfer
@@ -743,8 +844,8 @@ vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 			list_add(&waiter->list, &instance->bulk_waiter_list);
 			mutex_unlock(&instance->bulk_waiter_list_mutex);
 			vchiq_log_info(vchiq_arm_log_level,
-				"saved bulk_waiter %x for pid %d",
-				(unsigned int)waiter, current->pid);
+				"saved bulk_waiter %pK for pid %d",
+				waiter, current->pid);
 
 			if (copy_to_user((void __user *)
 				&(((VCHIQ_QUEUE_BULK_TRANSFER_T __user *)
@@ -826,10 +927,8 @@ vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 					if (args.msgbufsize < msglen) {
 						vchiq_log_error(
 							vchiq_arm_log_level,
-							"header %x: msgbufsize"
-							" %x < msglen %x",
-							(unsigned int)header,
-							args.msgbufsize,
+							"header %pK: msgbufsize %x < msglen %x",
+							header, args.msgbufsize,
 							msglen);
 						WARN(1, "invalid message "
 							"size\n");
@@ -980,9 +1079,8 @@ vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 				ret = -EFAULT;
 		} else {
 			vchiq_log_error(vchiq_arm_log_level,
-				"header %x: bufsize %x < size %x",
-				(unsigned int)header, args.bufsize,
-				header->size);
+				"header %pK: bufsize %x < size %x",
+				header, args.bufsize, header->size);
 			WARN(1, "invalid size\n");
 			ret = -EMSGSIZE;
 		}
@@ -1284,9 +1382,8 @@ vchiq_release(struct inode *inode, struct file *file)
 					list);
 				list_del(pos);
 				vchiq_log_info(vchiq_arm_log_level,
-					"bulk_waiter - cleaned up %x "
-					"for pid %d",
-					(unsigned int)waiter, waiter->pid);
+					"bulk_waiter - cleaned up %pK for pid %d",
+					waiter, waiter->pid);
 				kfree(waiter);
 			}
 		}
@@ -1385,9 +1482,8 @@ vchiq_dump_platform_instances(void *dump_context)
 			instance = service->instance;
 			if (instance && !instance->mark) {
 				len = snprintf(buf, sizeof(buf),
-					"Instance %x: pid %d,%s completions "
-						"%d/%d",
-					(unsigned int)instance, instance->pid,
+					"Instance %pK: pid %d,%s completions %d/%d",
+					instance, instance->pid,
 					instance->connected ? " connected, " :
 						"",
 					instance->completion_insert -
@@ -1415,8 +1511,7 @@ vchiq_dump_platform_service_state(void *dump_context, VCHIQ_SERVICE_T *service)
 	char buf[80];
 	int len;
 
-	len = snprintf(buf, sizeof(buf), "  instance %x",
-		(unsigned int)service->instance);
+	len = snprintf(buf, sizeof(buf), "  instance %pK", service->instance);
 
 	if ((service->base.callback == service_callback) &&
 		user_service->is_vchi) {
@@ -1473,8 +1568,7 @@ dump_phys_mem(void *virt_addr, uint32_t num_bytes)
 	}
 
 	down_read(&current->mm->mmap_sem);
-	rc = get_user_pages(current,      /* task */
-		current->mm,              /* mm */
+	rc = get_user_pages(
 		(unsigned long)virt_addr, /* start */
 		num_pages,                /* len */
 		0,                        /* gup_flags */
@@ -1485,6 +1579,12 @@ dump_phys_mem(void *virt_addr, uint32_t num_bytes)
 	prev_idx = -1;
 	page = NULL;
 
+	if (rc < 0) {
+		vchiq_log_error(vchiq_arm_log_level,
+				"Failed to get user pages: %d\n", rc);
+		goto out;
+	}
+
 	while (offset < end_offset) {
 
 		int page_offset = offset % PAGE_SIZE;
@@ -1508,11 +1608,13 @@ dump_phys_mem(void *virt_addr, uint32_t num_bytes)
 
 		offset += 16;
 	}
+
+out:
 	if (page != NULL)
 		kunmap(page);
 
 	for (page_idx = 0; page_idx < num_pages; page_idx++)
-		page_cache_release(pages[page_idx]);
+		put_page(pages[page_idx]);
 
 	kfree(pages);
 }
@@ -1683,8 +1785,6 @@ exit:
 VCHIQ_STATUS_T
 vchiq_arm_init_state(VCHIQ_STATE_T *state, VCHIQ_ARM_STATE_T *arm_state)
 {
-	VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
-
 	if (arm_state) {
 		rwlock_init(&arm_state->susp_res_lock);
 
@@ -1712,14 +1812,13 @@ vchiq_arm_init_state(VCHIQ_STATE_T *state, VCHIQ_ARM_STATE_T *arm_state)
 
 		arm_state->suspend_timer_timeout = SUSPEND_TIMER_TIMEOUT_MS;
 		arm_state->suspend_timer_running = 0;
-		init_timer(&arm_state->suspend_timer);
-		arm_state->suspend_timer.data = (unsigned long)(state);
-		arm_state->suspend_timer.function = suspend_timer_callback;
+		setup_timer(&arm_state->suspend_timer, suspend_timer_callback,
+			    (unsigned long)(state));
 
 		arm_state->first_connect = 0;
 
 	}
-	return status;
+	return VCHIQ_SUCCESS;
 }
 
 /*
@@ -2032,20 +2131,20 @@ static void
 output_timeout_error(VCHIQ_STATE_T *state)
 {
 	VCHIQ_ARM_STATE_T *arm_state = vchiq_platform_get_arm_state(state);
-	char service_err[50] = "";
+	char err[50] = "";
 	int vc_use_count = arm_state->videocore_use_count;
 	int active_services = state->unused_service;
 	int i;
 
 	if (!arm_state->videocore_use_count) {
-		snprintf(service_err, 50, " Videocore usecount is 0");
+		snprintf(err, sizeof(err), " Videocore usecount is 0");
 		goto output_msg;
 	}
 	for (i = 0; i < active_services; i++) {
 		VCHIQ_SERVICE_T *service_ptr = state->services[i];
 		if (service_ptr && service_ptr->service_use_count &&
 			(service_ptr->srvstate != VCHIQ_SRVSTATE_FREE)) {
-			snprintf(service_err, 50, " %c%c%c%c(%d) service has "
+			snprintf(err, sizeof(err), " %c%c%c%c(%d) service has "
 				"use count %d%s", VCHIQ_FOURCC_AS_4CHARS(
 					service_ptr->base.fourcc),
 				 service_ptr->client_id,
@@ -2059,7 +2158,7 @@ output_timeout_error(VCHIQ_STATE_T *state)
 output_msg:
 	vchiq_log_error(vchiq_susp_log_level,
 		"timed out waiting for vc suspend (%d).%s",
-		 arm_state->autosuspend_override, service_err);
+		 arm_state->autosuspend_override, err);
 
 }
 
@@ -2780,7 +2879,7 @@ void vchiq_platform_conn_state_changed(VCHIQ_STATE_T *state,
 				&vchiq_keepalive_thread_func,
 				(void *)state,
 				threadname);
-			if (arm_state->ka_thread == NULL) {
+			if (IS_ERR(arm_state->ka_thread)) {
 				vchiq_log_error(vchiq_susp_log_level,
 					"vchiq: FATAL: couldn't create thread %s",
 					threadname);
@@ -2800,28 +2899,27 @@ static int vchiq_probe(struct platform_device *pdev)
 	void *ptr_err;
 
 	fw_node = of_parse_phandle(pdev->dev.of_node, "firmware", 0);
-/* Remove comment when booting without Device Tree is no longer supported
 	if (!fw_node) {
 		dev_err(&pdev->dev, "Missing firmware node\n");
 		return -ENOENT;
 	}
-*/
+
 	fw = rpi_firmware_get(fw_node);
+	of_node_put(fw_node);
 	if (!fw)
 		return -EPROBE_DEFER;
 
 	platform_set_drvdata(pdev, fw);
 
-	/* create debugfs entries */
-	err = vchiq_debugfs_init();
+	err = vchiq_platform_init(pdev, &g_state);
 	if (err != 0)
-		goto failed_debugfs_init;
+		goto failed_platform_init;
 
 	err = alloc_chrdev_region(&vchiq_devid, VCHIQ_MINOR, 1, DEVICE_NAME);
 	if (err != 0) {
 		vchiq_log_error(vchiq_arm_log_level,
 			"Unable to allocate device number");
-		goto failed_alloc_chrdev;
+		goto failed_platform_init;
 	}
 	cdev_init(&vchiq_cdev, &vchiq_fops);
 	vchiq_cdev.owner = THIS_MODULE;
@@ -2844,9 +2942,10 @@ static int vchiq_probe(struct platform_device *pdev)
 	if (IS_ERR(ptr_err))
 		goto failed_device_create;
 
-	err = vchiq_platform_init(pdev, &g_state);
+	/* create debugfs entries */
+	err = vchiq_debugfs_init();
 	if (err != 0)
-		goto failed_platform_init;
+		goto failed_debugfs_init;
 
 	vchiq_log_info(vchiq_arm_log_level,
 		"vchiq: initialised - version %d (min %d), device %d.%d",
@@ -2855,7 +2954,7 @@ static int vchiq_probe(struct platform_device *pdev)
 
 	return 0;
 
-failed_platform_init:
+failed_debugfs_init:
 	device_destroy(vchiq_class, vchiq_devid);
 failed_device_create:
 	class_destroy(vchiq_class);
@@ -2864,15 +2963,14 @@ failed_class_create:
 	err = PTR_ERR(ptr_err);
 failed_cdev_add:
 	unregister_chrdev_region(vchiq_devid, 1);
-failed_alloc_chrdev:
-	vchiq_debugfs_deinit();
-failed_debugfs_init:
+failed_platform_init:
 	vchiq_log_warning(vchiq_arm_log_level, "could not load vchiq");
 	return err;
 }
 
 static int vchiq_remove(struct platform_device *pdev)
 {
+	vchiq_debugfs_deinit();
 	device_destroy(vchiq_class, vchiq_devid);
 	class_destroy(vchiq_class);
 	cdev_del(&vchiq_cdev);
@@ -2890,7 +2988,6 @@ MODULE_DEVICE_TABLE(of, vchiq_of_match);
 static struct platform_driver vchiq_driver = {
 	.driver = {
 		.name = "bcm2835_vchiq",
-		.owner = THIS_MODULE,
 		.of_match_table = vchiq_of_match,
 	},
 	.probe = vchiq_probe,
@@ -2899,4 +2996,5 @@ static struct platform_driver vchiq_driver = {
 module_platform_driver(vchiq_driver);
 
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Videocore VCHIQ driver");
 MODULE_AUTHOR("Broadcom Corporation");
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.c
index 5efc62ffb2f5..7ea29665bd0c 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.c
@@ -72,7 +72,7 @@ void vchiq_add_connected_callback(VCHIQ_CONNECTED_CALLBACK_T callback)
 {
 	connected_init();
 
-	if (mutex_lock_interruptible(&g_connected_mutex) != 0)
+	if (mutex_lock_killable(&g_connected_mutex) != 0)
 		return;
 
 	if (g_connected)
@@ -107,7 +107,7 @@ void vchiq_call_connected_callbacks(void)
 
 	connected_init();
 
-	if (mutex_lock_interruptible(&g_connected_mutex) != 0)
+	if (mutex_lock_killable(&g_connected_mutex) != 0)
 		return;
 
 	for (i = 0; i <  g_num_deferred_callbacks; i++)
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c
index 2c98da4307df..028e90bc1cdc 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c
@@ -296,12 +296,13 @@ lock_service(VCHIQ_SERVICE_T *service)
 void
 unlock_service(VCHIQ_SERVICE_T *service)
 {
-	VCHIQ_STATE_T *state = service->state;
 	spin_lock(&service_spinlock);
 	BUG_ON(!service || (service->ref_count == 0));
 	if (service && service->ref_count) {
 		service->ref_count--;
 		if (!service->ref_count) {
+			VCHIQ_STATE_T *state = service->state;
+
 			BUG_ON(service->srvstate != VCHIQ_SRVSTATE_FREE);
 			state->services[service->localport] = NULL;
 		} else
@@ -380,9 +381,9 @@ make_service_callback(VCHIQ_SERVICE_T *service, VCHIQ_REASON_T reason,
 	VCHIQ_HEADER_T *header, void *bulk_userdata)
 {
 	VCHIQ_STATUS_T status;
-	vchiq_log_trace(vchiq_core_log_level, "%d: callback:%d (%s, %x, %x)",
+	vchiq_log_trace(vchiq_core_log_level, "%d: callback:%d (%s, %pK, %pK)",
 		service->state->id, service->localport, reason_names[reason],
-		(unsigned int)header, (unsigned int)bulk_userdata);
+		header, bulk_userdata);
 	status = service->base.callback(reason, header, service->handle,
 		bulk_userdata);
 	if (status == VCHIQ_ERROR) {
@@ -406,28 +407,24 @@ vchiq_set_conn_state(VCHIQ_STATE_T *state, VCHIQ_CONNSTATE_T newstate)
 }
 
 static inline void
-remote_event_create(REMOTE_EVENT_T *event)
+remote_event_create(VCHIQ_STATE_T *state, REMOTE_EVENT_T *event)
 {
 	event->armed = 0;
 	/* Don't clear the 'fired' flag because it may already have been set
 	** by the other side. */
-	sema_init(event->event, 0);
-}
-
-static inline void
-remote_event_destroy(REMOTE_EVENT_T *event)
-{
-	(void)event;
+	sema_init((struct semaphore *)((char *)state + event->event), 0);
 }
 
 static inline int
-remote_event_wait(REMOTE_EVENT_T *event)
+remote_event_wait(VCHIQ_STATE_T *state, REMOTE_EVENT_T *event)
 {
 	if (!event->fired) {
 		event->armed = 1;
-		dsb();
+		dsb(sy);
 		if (!event->fired) {
-			if (down_interruptible(event->event) != 0) {
+			if (down_interruptible(
+					(struct semaphore *)
+					((char *)state + event->event)) != 0) {
 				event->armed = 0;
 				return 0;
 			}
@@ -441,34 +438,34 @@ remote_event_wait(REMOTE_EVENT_T *event)
 }
 
 static inline void
-remote_event_signal_local(REMOTE_EVENT_T *event)
+remote_event_signal_local(VCHIQ_STATE_T *state, REMOTE_EVENT_T *event)
 {
 	event->armed = 0;
-	up(event->event);
+	up((struct semaphore *)((char *)state + event->event));
 }
 
 static inline void
-remote_event_poll(REMOTE_EVENT_T *event)
+remote_event_poll(VCHIQ_STATE_T *state, REMOTE_EVENT_T *event)
 {
 	if (event->fired && event->armed)
-		remote_event_signal_local(event);
+		remote_event_signal_local(state, event);
 }
 
 void
 remote_event_pollall(VCHIQ_STATE_T *state)
 {
-	remote_event_poll(&state->local->sync_trigger);
-	remote_event_poll(&state->local->sync_release);
-	remote_event_poll(&state->local->trigger);
-	remote_event_poll(&state->local->recycle);
+	remote_event_poll(state, &state->local->sync_trigger);
+	remote_event_poll(state, &state->local->sync_release);
+	remote_event_poll(state, &state->local->trigger);
+	remote_event_poll(state, &state->local->recycle);
 }
 
 /* Round up message sizes so that any space at the end of a slot is always big
 ** enough for a header. This relies on header size being a power of two, which
 ** has been verified earlier by a static assertion. */
 
-static inline unsigned int
-calc_stride(unsigned int size)
+static inline size_t
+calc_stride(size_t size)
 {
 	/* Allow room for the header */
 	size += sizeof(VCHIQ_HEADER_T);
@@ -541,13 +538,13 @@ request_poll(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service, int poll_type)
 	wmb();
 
 	/* ... and ensure the slot handler runs. */
-	remote_event_signal_local(&state->local->trigger);
+	remote_event_signal_local(state, &state->local->trigger);
 }
 
 /* Called from queue_message, by the slot handler and application threads,
 ** with slot_mutex held */
 static VCHIQ_HEADER_T *
-reserve_space(VCHIQ_STATE_T *state, int space, int is_blocking)
+reserve_space(VCHIQ_STATE_T *state, size_t space, int is_blocking)
 {
 	VCHIQ_SHARED_STATE_T *local = state->local;
 	int tx_pos = state->local_tx_pos;
@@ -626,8 +623,8 @@ process_free_queue(VCHIQ_STATE_T *state)
 		char *data = (char *)SLOT_DATA_FROM_INDEX(state, slot_index);
 		int data_found = 0;
 
-		vchiq_log_trace(vchiq_core_log_level, "%d: pfq %d=%x %x %x",
-			state->id, slot_index, (unsigned int)data,
+		vchiq_log_trace(vchiq_core_log_level, "%d: pfq %d=%pK %x %x",
+			state->id, slot_index, data,
 			local->slot_queue_recycle, slot_queue_available);
 
 		/* Initialise the bitmask for services which have used this
@@ -659,16 +656,10 @@ process_free_queue(VCHIQ_STATE_T *state)
 					up(&service_quota->quota_event);
 				else if (count == 0) {
 					vchiq_log_error(vchiq_core_log_level,
-						"service %d "
-						"message_use_count=%d "
-						"(header %x, msgid %x, "
-						"header->msgid %x, "
-						"header->size %x)",
+						"service %d message_use_count=%d (header %pK, msgid %x, header->msgid %x, header->size %x)",
 						port,
-						service_quota->
-							message_use_count,
-						(unsigned int)header, msgid,
-						header->msgid,
+						service_quota->message_use_count,
+						header, msgid, header->msgid,
 						header->size);
 					WARN(1, "invalid message use count\n");
 				}
@@ -690,26 +681,16 @@ process_free_queue(VCHIQ_STATE_T *state)
 						up(&service_quota->quota_event);
 						vchiq_log_trace(
 							vchiq_core_log_level,
-							"%d: pfq:%d %x@%x - "
-							"slot_use->%d",
+							"%d: pfq:%d %x@%pK - slot_use->%d",
 							state->id, port,
-							header->size,
-							(unsigned int)header,
+							header->size, header,
 							count - 1);
 					} else {
 						vchiq_log_error(
 							vchiq_core_log_level,
-								"service %d "
-								"slot_use_count"
-								"=%d (header %x"
-								", msgid %x, "
-								"header->msgid"
-								" %x, header->"
-								"size %x)",
-							port, count,
-							(unsigned int)header,
-							msgid,
-							header->msgid,
+								"service %d slot_use_count=%d (header %pK, msgid %x, header->msgid %x, header->size %x)",
+							port, count, header,
+							msgid, header->msgid,
 							header->size);
 						WARN(1, "bad slot use count\n");
 					}
@@ -721,10 +702,9 @@ process_free_queue(VCHIQ_STATE_T *state)
 			pos += calc_stride(header->size);
 			if (pos > VCHIQ_SLOT_SIZE) {
 				vchiq_log_error(vchiq_core_log_level,
-					"pfq - pos %x: header %x, msgid %x, "
-					"header->msgid %x, header->size %x",
-					pos, (unsigned int)header, msgid,
-					header->msgid, header->size);
+					"pfq - pos %x: header %pK, msgid %x, header->msgid %x, header->size %x",
+					pos, header, msgid, header->msgid,
+					header->size);
 				WARN(1, "invalid slot position\n");
 			}
 		}
@@ -746,18 +726,66 @@ process_free_queue(VCHIQ_STATE_T *state)
 	}
 }
 
+static ssize_t
+memcpy_copy_callback(
+	void *context, void *dest,
+	size_t offset, size_t maxsize)
+{
+	void *src = context;
+
+	memcpy(dest + offset, src + offset, maxsize);
+	return maxsize;
+}
+
+static ssize_t
+copy_message_data(
+	ssize_t (*copy_callback)(void *context, void *dest,
+				 size_t offset, size_t maxsize),
+	void *context,
+	void *dest,
+	size_t size)
+{
+	size_t pos = 0;
+
+	while (pos < size) {
+		ssize_t callback_result;
+		size_t max_bytes = size - pos;
+
+		callback_result =
+			copy_callback(context, dest + pos,
+				      pos, max_bytes);
+
+		if (callback_result < 0)
+			return callback_result;
+
+		if (!callback_result)
+			return -EIO;
+
+		if (callback_result > max_bytes)
+			return -EIO;
+
+		pos += callback_result;
+	}
+
+	return size;
+}
+
 /* Called by the slot handler and application threads */
 static VCHIQ_STATUS_T
 queue_message(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
-	int msgid, const VCHIQ_ELEMENT_T *elements,
-	int count, int size, int flags)
+	int msgid,
+	ssize_t (*copy_callback)(void *context, void *dest,
+				 size_t offset, size_t maxsize),
+	void *context,
+	size_t size,
+	int flags)
 {
 	VCHIQ_SHARED_STATE_T *local;
 	VCHIQ_SERVICE_QUOTA_T *service_quota = NULL;
 	VCHIQ_HEADER_T *header;
 	int type = VCHIQ_MSG_TYPE(msgid);
 
-	unsigned int stride;
+	size_t stride;
 
 	local = state->local;
 
@@ -766,7 +794,7 @@ queue_message(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 	WARN_ON(!(stride <= VCHIQ_SLOT_SIZE));
 
 	if (!(flags & QMFLAGS_NO_MUTEX_LOCK) &&
-		(mutex_lock_interruptible(&state->slot_mutex) != 0))
+		(mutex_lock_killable(&state->slot_mutex) != 0))
 		return VCHIQ_RETRY;
 
 	if (type == VCHIQ_MSG_DATA) {
@@ -822,7 +850,7 @@ queue_message(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 				service_quota->slot_quota))) {
 			spin_unlock(&quota_spinlock);
 			vchiq_log_trace(vchiq_core_log_level,
-				"%d: qm:%d %s,%x - quota stall "
+				"%d: qm:%d %s,%zx - quota stall "
 				"(msg %d, slot %d)",
 				state->id, service->localport,
 				msg_type_str(type), size,
@@ -835,7 +863,7 @@ queue_message(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 				return VCHIQ_RETRY;
 			if (service->closing)
 				return VCHIQ_ERROR;
-			if (mutex_lock_interruptible(&state->slot_mutex) != 0)
+			if (mutex_lock_killable(&state->slot_mutex) != 0)
 				return VCHIQ_RETRY;
 			if (service->srvstate != VCHIQ_SRVSTATE_OPEN) {
 				/* The service has been closed */
@@ -863,43 +891,37 @@ queue_message(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 	}
 
 	if (type == VCHIQ_MSG_DATA) {
-		int i, pos;
+		ssize_t callback_result;
 		int tx_end_index;
 		int slot_use_count;
 
 		vchiq_log_info(vchiq_core_log_level,
-			"%d: qm %s@%x,%x (%d->%d)",
-			state->id,
-			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
-			(unsigned int)header, size,
-			VCHIQ_MSG_SRCPORT(msgid),
+			"%d: qm %s@%pK,%zx (%d->%d)",
+			state->id, msg_type_str(VCHIQ_MSG_TYPE(msgid)),
+			header, size, VCHIQ_MSG_SRCPORT(msgid),
 			VCHIQ_MSG_DSTPORT(msgid));
 
 		BUG_ON(!service);
 		BUG_ON((flags & (QMFLAGS_NO_MUTEX_LOCK |
 				 QMFLAGS_NO_MUTEX_UNLOCK)) != 0);
 
-		for (i = 0, pos = 0; i < (unsigned int)count;
-			pos += elements[i++].size)
-			if (elements[i].size) {
-				if (vchiq_copy_from_user
-					(header->data + pos, elements[i].data,
-					(size_t) elements[i].size) !=
-					VCHIQ_SUCCESS) {
-					mutex_unlock(&state->slot_mutex);
-					VCHIQ_SERVICE_STATS_INC(service,
+		callback_result =
+			copy_message_data(copy_callback, context,
+					  header->data, size);
+
+		if (callback_result < 0) {
+			mutex_unlock(&state->slot_mutex);
+			VCHIQ_SERVICE_STATS_INC(service,
 						error_count);
-					return VCHIQ_ERROR;
-				}
-				if (i == 0) {
-					if (SRVTRACE_ENABLED(service,
-							VCHIQ_LOG_INFO))
-						vchiq_log_dump_mem("Sent", 0,
-							header->data + pos,
-							min(64u,
-							elements[0].size));
-				}
-			}
+			return VCHIQ_ERROR;
+		}
+
+		if (SRVTRACE_ENABLED(service,
+				     VCHIQ_LOG_INFO))
+			vchiq_log_dump_mem("Sent", 0,
+					   header->data,
+					   min((size_t)64,
+					       (size_t)callback_result));
 
 		spin_lock(&quota_spinlock);
 		service_quota->message_use_count++;
@@ -927,7 +949,7 @@ queue_message(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 
 		if (slot_use_count)
 			vchiq_log_trace(vchiq_core_log_level,
-				"%d: qm:%d %s,%x - slot_use->%d (hdr %p)",
+				"%d: qm:%d %s,%zx - slot_use->%d (hdr %p)",
 				state->id, service->localport,
 				msg_type_str(VCHIQ_MSG_TYPE(msgid)), size,
 				slot_use_count, header);
@@ -936,15 +958,22 @@ queue_message(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 		VCHIQ_SERVICE_STATS_ADD(service, ctrl_tx_bytes, size);
 	} else {
 		vchiq_log_info(vchiq_core_log_level,
-			"%d: qm %s@%x,%x (%d->%d)", state->id,
+			"%d: qm %s@%pK,%zx (%d->%d)", state->id,
 			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
-			(unsigned int)header, size,
-			VCHIQ_MSG_SRCPORT(msgid),
+			header, size, VCHIQ_MSG_SRCPORT(msgid),
 			VCHIQ_MSG_DSTPORT(msgid));
 		if (size != 0) {
-			WARN_ON(!((count == 1) && (size == elements[0].size)));
-			memcpy(header->data, elements[0].data,
-				elements[0].size);
+			/* It is assumed for now that this code path
+			 * only happens from calls inside this file.
+			 *
+			 * External callers are through the vchiq_queue_message
+			 * path which always sets the type to be VCHIQ_MSG_DATA
+			 *
+			 * At first glance this appears to be correct but
+			 * more review is needed.
+			 */
+			copy_message_data(copy_callback, context,
+					  header->data, size);
 		}
 		VCHIQ_STATS_INC(state, ctrl_tx_count);
 	}
@@ -960,7 +989,7 @@ queue_message(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 			: VCHIQ_MAKE_FOURCC('?', '?', '?', '?');
 
 		vchiq_log_info(SRVTRACE_LEVEL(service),
-			"Sent Msg %s(%u) to %c%c%c%c s:%u d:%d len:%d",
+			"Sent Msg %s(%u) to %c%c%c%c s:%u d:%d len:%zu",
 			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
 			VCHIQ_MSG_TYPE(msgid),
 			VCHIQ_FOURCC_AS_4CHARS(svc_fourcc),
@@ -990,19 +1019,24 @@ queue_message(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 /* Called by the slot handler and application threads */
 static VCHIQ_STATUS_T
 queue_message_sync(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
-	int msgid, const VCHIQ_ELEMENT_T *elements,
-	int count, int size, int is_blocking)
+	int msgid,
+	ssize_t (*copy_callback)(void *context, void *dest,
+				 size_t offset, size_t maxsize),
+	void *context,
+	int size,
+	int is_blocking)
 {
 	VCHIQ_SHARED_STATE_T *local;
 	VCHIQ_HEADER_T *header;
+	ssize_t callback_result;
 
 	local = state->local;
 
 	if ((VCHIQ_MSG_TYPE(msgid) != VCHIQ_MSG_RESUME) &&
-		(mutex_lock_interruptible(&state->sync_mutex) != 0))
+		(mutex_lock_killable(&state->sync_mutex) != 0))
 		return VCHIQ_RETRY;
 
-	remote_event_wait(&local->sync_release);
+	remote_event_wait(state, &local->sync_release);
 
 	rmb();
 
@@ -1017,52 +1051,34 @@ queue_message_sync(VCHIQ_STATE_T *state, VCHIQ_SERVICE_T *service,
 				state->id, oldmsgid);
 	}
 
-	if (service) {
-		int i, pos;
+	vchiq_log_info(vchiq_sync_log_level,
+		       "%d: qms %s@%pK,%x (%d->%d)", state->id,
+		       msg_type_str(VCHIQ_MSG_TYPE(msgid)),
+		       header, size, VCHIQ_MSG_SRCPORT(msgid),
+		       VCHIQ_MSG_DSTPORT(msgid));
 
-		vchiq_log_info(vchiq_sync_log_level,
-			"%d: qms %s@%x,%x (%d->%d)", state->id,
-			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
-			(unsigned int)header, size,
-			VCHIQ_MSG_SRCPORT(msgid),
-			VCHIQ_MSG_DSTPORT(msgid));
+	callback_result =
+		copy_message_data(copy_callback, context,
+				  header->data, size);
 
-		for (i = 0, pos = 0; i < (unsigned int)count;
-			pos += elements[i++].size)
-			if (elements[i].size) {
-				if (vchiq_copy_from_user
-					(header->data + pos, elements[i].data,
-					(size_t) elements[i].size) !=
-					VCHIQ_SUCCESS) {
-					mutex_unlock(&state->sync_mutex);
-					VCHIQ_SERVICE_STATS_INC(service,
-						error_count);
-					return VCHIQ_ERROR;
-				}
-				if (i == 0) {
-					if (vchiq_sync_log_level >=
-						VCHIQ_LOG_TRACE)
-						vchiq_log_dump_mem("Sent Sync",
-							0, header->data + pos,
-							min(64u,
-							elements[0].size));
-				}
-			}
+	if (callback_result < 0) {
+		mutex_unlock(&state->slot_mutex);
+		VCHIQ_SERVICE_STATS_INC(service,
+					error_count);
+		return VCHIQ_ERROR;
+	}
+
+	if (service) {
+		if (SRVTRACE_ENABLED(service,
+				     VCHIQ_LOG_INFO))
+			vchiq_log_dump_mem("Sent", 0,
+					   header->data,
+					   min((size_t)64,
+					       (size_t)callback_result));
 
 		VCHIQ_SERVICE_STATS_INC(service, ctrl_tx_count);
 		VCHIQ_SERVICE_STATS_ADD(service, ctrl_tx_bytes, size);
 	} else {
-		vchiq_log_info(vchiq_sync_log_level,
-			"%d: qms %s@%x,%x (%d->%d)", state->id,
-			msg_type_str(VCHIQ_MSG_TYPE(msgid)),
-			(unsigned int)header, size,
-			VCHIQ_MSG_SRCPORT(msgid),
-			VCHIQ_MSG_DSTPORT(msgid));
-		if (size != 0) {
-			WARN_ON(!((count == 1) && (size == elements[0].size)));
-			memcpy(header->data, elements[0].data,
-				elements[0].size);
-		}
 		VCHIQ_STATS_INC(state, ctrl_tx_count);
 	}
 
@@ -1175,11 +1191,16 @@ notify_bulks(VCHIQ_SERVICE_T *service, VCHIQ_BULK_QUEUE_T *queue,
 				VCHIQ_MSG_BULK_RX_DONE : VCHIQ_MSG_BULK_TX_DONE;
 			int msgid = VCHIQ_MAKE_MSG(msgtype, service->localport,
 				service->remoteport);
-			VCHIQ_ELEMENT_T element = { &bulk->actual, 4 };
 			/* Only reply to non-dummy bulk requests */
 			if (bulk->remote_data) {
-				status = queue_message(service->state, NULL,
-					msgid, &element, 1, 4, 0);
+				status = queue_message(
+						service->state,
+						NULL,
+						msgid,
+						memcpy_copy_callback,
+						&bulk->actual,
+						4,
+						0);
 				if (status != VCHIQ_SUCCESS)
 					break;
 			}
@@ -1344,7 +1365,7 @@ resolve_bulks(VCHIQ_SERVICE_T *service, VCHIQ_BULK_QUEUE_T *queue)
 		WARN_ON(!((int)(queue->local_insert - queue->process) > 0));
 		WARN_ON(!((int)(queue->remote_insert - queue->process) > 0));
 
-		rc = mutex_lock_interruptible(&state->bulk_transfer_mutex);
+		rc = mutex_lock_killable(&state->bulk_transfer_mutex);
 		if (rc != 0)
 			break;
 
@@ -1356,26 +1377,22 @@ resolve_bulks(VCHIQ_SERVICE_T *service, VCHIQ_BULK_QUEUE_T *queue)
 				"Send Bulk to" : "Recv Bulk from";
 			if (bulk->actual != VCHIQ_BULK_ACTUAL_ABORTED)
 				vchiq_log_info(SRVTRACE_LEVEL(service),
-					"%s %c%c%c%c d:%d len:%d %x<->%x",
+					"%s %c%c%c%c d:%d len:%d %pK<->%pK",
 					header,
 					VCHIQ_FOURCC_AS_4CHARS(
 						service->base.fourcc),
-					service->remoteport,
-					bulk->size,
-					(unsigned int)bulk->data,
-					(unsigned int)bulk->remote_data);
+					service->remoteport, bulk->size,
+					bulk->data, bulk->remote_data);
 			else
 				vchiq_log_info(SRVTRACE_LEVEL(service),
 					"%s %c%c%c%c d:%d ABORTED - tx len:%d,"
-					" rx len:%d %x<->%x",
+					" rx len:%d %pK<->%pK",
 					header,
 					VCHIQ_FOURCC_AS_4CHARS(
 						service->base.fourcc),
 					service->remoteport,
-					bulk->size,
-					bulk->remote_size,
-					(unsigned int)bulk->data,
-					(unsigned int)bulk->remote_data);
+					bulk->size, bulk->remote_size,
+					bulk->data, bulk->remote_data);
 		}
 
 		vchiq_complete_bulk(bulk);
@@ -1511,9 +1528,8 @@ parse_open(VCHIQ_STATE_T *state, VCHIQ_HEADER_T *header)
 
 		fourcc = payload->fourcc;
 		vchiq_log_info(vchiq_core_log_level,
-			"%d: prs OPEN@%x (%d->'%c%c%c%c')",
-			state->id, (unsigned int)header,
-			localport,
+			"%d: prs OPEN@%pK (%d->'%c%c%c%c')",
+			state->id, header, localport,
 			VCHIQ_FOURCC_AS_4CHARS(fourcc));
 
 		service = get_listening_service(state, fourcc);
@@ -1544,10 +1560,6 @@ parse_open(VCHIQ_STATE_T *state, VCHIQ_HEADER_T *header)
 				struct vchiq_openack_payload ack_payload = {
 					service->version
 				};
-				VCHIQ_ELEMENT_T body = {
-					&ack_payload,
-					sizeof(ack_payload)
-				};
 
 				if (state->version_common <
 				    VCHIQ_VERSION_SYNCHRONOUS_MODE)
@@ -1557,21 +1569,28 @@ parse_open(VCHIQ_STATE_T *state, VCHIQ_HEADER_T *header)
 				if (service->sync &&
 				    (state->version_common >=
 				     VCHIQ_VERSION_SYNCHRONOUS_MODE)) {
-					if (queue_message_sync(state, NULL,
+					if (queue_message_sync(
+						state,
+						NULL,
 						VCHIQ_MAKE_MSG(
 							VCHIQ_MSG_OPENACK,
 							service->localport,
 							remoteport),
-						&body, 1, sizeof(ack_payload),
+						memcpy_copy_callback,
+						&ack_payload,
+						sizeof(ack_payload),
 						0) == VCHIQ_RETRY)
 						goto bail_not_ready;
 				} else {
-					if (queue_message(state, NULL,
-						VCHIQ_MAKE_MSG(
+					if (queue_message(state,
+							NULL,
+							VCHIQ_MAKE_MSG(
 							VCHIQ_MSG_OPENACK,
 							service->localport,
 							remoteport),
-						&body, 1, sizeof(ack_payload),
+						memcpy_copy_callback,
+						&ack_payload,
+						sizeof(ack_payload),
 						0) == VCHIQ_RETRY)
 						goto bail_not_ready;
 				}
@@ -1650,7 +1669,7 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 
 		header = (VCHIQ_HEADER_T *)(state->rx_data +
 			(state->rx_pos & VCHIQ_SLOT_MASK));
-		DEBUG_VALUE(PARSE_HEADER, (int)header);
+		DEBUG_VALUE(PARSE_HEADER, (int)(long)header);
 		msgid = header->msgid;
 		DEBUG_VALUE(PARSE_MSGID, msgid);
 		size = header->size;
@@ -1684,21 +1703,18 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 					remoteport);
 				if (service)
 					vchiq_log_warning(vchiq_core_log_level,
-						"%d: prs %s@%x (%d->%d) - "
-						"found connected service %d",
+						"%d: prs %s@%pK (%d->%d) - found connected service %d",
 						state->id, msg_type_str(type),
-						(unsigned int)header,
-						remoteport, localport,
+						header, remoteport, localport,
 						service->localport);
 			}
 
 			if (!service) {
 				vchiq_log_error(vchiq_core_log_level,
-					"%d: prs %s@%x (%d->%d) - "
-					"invalid/closed service %d",
+					"%d: prs %s@%pK (%d->%d) - invalid/closed service %d",
 					state->id, msg_type_str(type),
-					(unsigned int)header,
-					remoteport, localport, localport);
+					header, remoteport, localport,
+					localport);
 				goto skip_message;
 			}
 			break;
@@ -1723,12 +1739,11 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 					min(64, size));
 		}
 
-		if (((unsigned int)header & VCHIQ_SLOT_MASK) + calc_stride(size)
-			> VCHIQ_SLOT_SIZE) {
+		if (((unsigned long)header & VCHIQ_SLOT_MASK) +
+		    calc_stride(size) > VCHIQ_SLOT_SIZE) {
 			vchiq_log_error(vchiq_core_log_level,
-				"header %x (msgid %x) - size %x too big for "
-				"slot",
-				(unsigned int)header, (unsigned int)msgid,
+				"header %pK (msgid %x) - size %x too big for slot",
+				header, (unsigned int)msgid,
 				(unsigned int)size);
 			WARN(1, "oversized for slot\n");
 		}
@@ -1747,9 +1762,9 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 				service->peer_version = payload->version;
 			}
 			vchiq_log_info(vchiq_core_log_level,
-				"%d: prs OPENACK@%x,%x (%d->%d) v:%d",
-				state->id, (unsigned int)header, size,
-				remoteport, localport, service->peer_version);
+				"%d: prs OPENACK@%pK,%x (%d->%d) v:%d",
+				state->id, header, size, remoteport, localport,
+				service->peer_version);
 			if (service->srvstate ==
 				VCHIQ_SRVSTATE_OPENING) {
 				service->remoteport = remoteport;
@@ -1765,9 +1780,8 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 			WARN_ON(size != 0); /* There should be no data */
 
 			vchiq_log_info(vchiq_core_log_level,
-				"%d: prs CLOSE@%x (%d->%d)",
-				state->id, (unsigned int)header,
-				remoteport, localport);
+				"%d: prs CLOSE@%pK (%d->%d)",
+				state->id, header, remoteport, localport);
 
 			mark_service_closing_internal(service, 1);
 
@@ -1783,9 +1797,8 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 			break;
 		case VCHIQ_MSG_DATA:
 			vchiq_log_info(vchiq_core_log_level,
-				"%d: prs DATA@%x,%x (%d->%d)",
-				state->id, (unsigned int)header, size,
-				remoteport, localport);
+				"%d: prs DATA@%pK,%x (%d->%d)",
+				state->id, header, size, remoteport, localport);
 
 			if ((service->remoteport == remoteport)
 				&& (service->srvstate ==
@@ -1808,8 +1821,7 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 			break;
 		case VCHIQ_MSG_CONNECT:
 			vchiq_log_info(vchiq_core_log_level,
-				"%d: prs CONNECT@%x",
-				state->id, (unsigned int)header);
+				"%d: prs CONNECT@%pK", state->id, header);
 			state->version_common = ((VCHIQ_SLOT_ZERO_T *)
 						 state->slot_data)->version;
 			up(&state->connect);
@@ -1827,7 +1839,7 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 				int resolved = 0;
 
 				DEBUG_TRACE(PARSE_LINE);
-				if (mutex_lock_interruptible(
+				if (mutex_lock_killable(
 					&service->bulk_mutex) != 0) {
 					DEBUG_TRACE(PARSE_LINE);
 					goto bail_not_ready;
@@ -1838,17 +1850,15 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 				bulk = &queue->bulks[
 					BULK_INDEX(queue->remote_insert)];
 				bulk->remote_data =
-					(void *)((int *)header->data)[0];
+					(void *)(long)((int *)header->data)[0];
 				bulk->remote_size = ((int *)header->data)[1];
 				wmb();
 
 				vchiq_log_info(vchiq_core_log_level,
-					"%d: prs %s@%x (%d->%d) %x@%x",
+					"%d: prs %s@%pK (%d->%d) %x@%pK",
 					state->id, msg_type_str(type),
-					(unsigned int)header,
-					remoteport, localport,
-					bulk->remote_size,
-					(unsigned int)bulk->remote_data);
+					header, remoteport, localport,
+					bulk->remote_size, bulk->remote_data);
 
 				queue->remote_insert++;
 
@@ -1893,7 +1903,7 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 					&service->bulk_rx : &service->bulk_tx;
 
 				DEBUG_TRACE(PARSE_LINE);
-				if (mutex_lock_interruptible(
+				if (mutex_lock_killable(
 					&service->bulk_mutex) != 0) {
 					DEBUG_TRACE(PARSE_LINE);
 					goto bail_not_ready;
@@ -1901,11 +1911,10 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 				if ((int)(queue->remote_insert -
 					queue->local_insert) >= 0) {
 					vchiq_log_error(vchiq_core_log_level,
-						"%d: prs %s@%x (%d->%d) "
+						"%d: prs %s@%pK (%d->%d) "
 						"unexpected (ri=%d,li=%d)",
 						state->id, msg_type_str(type),
-						(unsigned int)header,
-						remoteport, localport,
+						header, remoteport, localport,
 						queue->remote_insert,
 						queue->local_insert);
 					mutex_unlock(&service->bulk_mutex);
@@ -1921,11 +1930,10 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 				queue->remote_insert++;
 
 				vchiq_log_info(vchiq_core_log_level,
-					"%d: prs %s@%x (%d->%d) %x@%x",
+					"%d: prs %s@%pK (%d->%d) %x@%pK",
 					state->id, msg_type_str(type),
-					(unsigned int)header,
-					remoteport, localport,
-					bulk->actual, (unsigned int)bulk->data);
+					header, remoteport, localport,
+					bulk->actual, bulk->data);
 
 				vchiq_log_trace(vchiq_core_log_level,
 					"%d: prs:%d %cx li=%x ri=%x p=%x",
@@ -1947,14 +1955,14 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 			break;
 		case VCHIQ_MSG_PADDING:
 			vchiq_log_trace(vchiq_core_log_level,
-				"%d: prs PADDING@%x,%x",
-				state->id, (unsigned int)header, size);
+				"%d: prs PADDING@%pK,%x",
+				state->id, header, size);
 			break;
 		case VCHIQ_MSG_PAUSE:
 			/* If initiated, signal the application thread */
 			vchiq_log_trace(vchiq_core_log_level,
-				"%d: prs PAUSE@%x,%x",
-				state->id, (unsigned int)header, size);
+				"%d: prs PAUSE@%pK,%x",
+				state->id, header, size);
 			if (state->conn_state == VCHIQ_CONNSTATE_PAUSED) {
 				vchiq_log_error(vchiq_core_log_level,
 					"%d: PAUSE received in state PAUSED",
@@ -1977,8 +1985,8 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 			break;
 		case VCHIQ_MSG_RESUME:
 			vchiq_log_trace(vchiq_core_log_level,
-				"%d: prs RESUME@%x,%x",
-				state->id, (unsigned int)header, size);
+				"%d: prs RESUME@%pK,%x",
+				state->id, header, size);
 			/* Release the slot mutex */
 			mutex_unlock(&state->slot_mutex);
 			if (state->is_master)
@@ -1999,8 +2007,8 @@ parse_rx_slots(VCHIQ_STATE_T *state)
 
 		default:
 			vchiq_log_error(vchiq_core_log_level,
-				"%d: prs invalid msgid %x@%x,%x",
-				state->id, msgid, (unsigned int)header, size);
+				"%d: prs invalid msgid %x@%pK,%x",
+				state->id, msgid, header, size);
 			WARN(1, "invalid message\n");
 			break;
 		}
@@ -2039,7 +2047,7 @@ slot_handler_func(void *v)
 	while (1) {
 		DEBUG_COUNT(SLOT_HANDLER_COUNT);
 		DEBUG_TRACE(SLOT_HANDLER_LINE);
-		remote_event_wait(&local->trigger);
+		remote_event_wait(state, &local->trigger);
 
 		rmb();
 
@@ -2128,7 +2136,7 @@ recycle_func(void *v)
 	VCHIQ_SHARED_STATE_T *local = state->local;
 
 	while (1) {
-		remote_event_wait(&local->recycle);
+		remote_event_wait(state, &local->recycle);
 
 		process_free_queue(state);
 	}
@@ -2151,7 +2159,7 @@ sync_func(void *v)
 		int type;
 		unsigned int localport, remoteport;
 
-		remote_event_wait(&local->sync_trigger);
+		remote_event_wait(state, &local->sync_trigger);
 
 		rmb();
 
@@ -2165,11 +2173,9 @@ sync_func(void *v)
 
 		if (!service) {
 			vchiq_log_error(vchiq_sync_log_level,
-				"%d: sf %s@%x (%d->%d) - "
-				"invalid/closed service %d",
+				"%d: sf %s@%pK (%d->%d) - invalid/closed service %d",
 				state->id, msg_type_str(type),
-				(unsigned int)header,
-				remoteport, localport, localport);
+				header, remoteport, localport, localport);
 			release_message_sync(state, header);
 			continue;
 		}
@@ -2199,9 +2205,9 @@ sync_func(void *v)
 				service->peer_version = payload->version;
 			}
 			vchiq_log_info(vchiq_sync_log_level,
-				"%d: sf OPENACK@%x,%x (%d->%d) v:%d",
-				state->id, (unsigned int)header, size,
-				remoteport, localport, service->peer_version);
+				"%d: sf OPENACK@%pK,%x (%d->%d) v:%d",
+				state->id, header, size, remoteport, localport,
+				service->peer_version);
 			if (service->srvstate == VCHIQ_SRVSTATE_OPENING) {
 				service->remoteport = remoteport;
 				vchiq_set_service_state(service,
@@ -2214,9 +2220,8 @@ sync_func(void *v)
 
 		case VCHIQ_MSG_DATA:
 			vchiq_log_trace(vchiq_sync_log_level,
-				"%d: sf DATA@%x,%x (%d->%d)",
-				state->id, (unsigned int)header, size,
-				remoteport, localport);
+				"%d: sf DATA@%pK,%x (%d->%d)",
+				state->id, header, size, remoteport, localport);
 
 			if ((service->remoteport == remoteport) &&
 				(service->srvstate ==
@@ -2234,8 +2239,8 @@ sync_func(void *v)
 
 		default:
 			vchiq_log_error(vchiq_sync_log_level,
-				"%d: sf unexpected msgid %x@%x,%x",
-				state->id, msgid, (unsigned int)header, size);
+				"%d: sf unexpected msgid %x@%pK,%x",
+				state->id, msgid, header, size);
 			release_message_sync(state, header);
 			break;
 		}
@@ -2268,7 +2273,8 @@ get_conn_state_name(VCHIQ_CONNSTATE_T conn_state)
 VCHIQ_SLOT_ZERO_T *
 vchiq_init_slots(void *mem_base, int mem_size)
 {
-	int mem_align = (VCHIQ_SLOT_SIZE - (int)mem_base) & VCHIQ_SLOT_MASK;
+	int mem_align =
+		(int)((VCHIQ_SLOT_SIZE - (long)mem_base) & VCHIQ_SLOT_MASK);
 	VCHIQ_SLOT_ZERO_T *slot_zero =
 		(VCHIQ_SLOT_ZERO_T *)((char *)mem_base + mem_align);
 	int num_slots = (mem_size - mem_align)/VCHIQ_SLOT_SIZE;
@@ -2316,16 +2322,16 @@ vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
 	int i;
 
 	vchiq_log_warning(vchiq_core_log_level,
-		"%s: slot_zero = 0x%08lx, is_master = %d",
-		__func__, (unsigned long)slot_zero, is_master);
+		"%s: slot_zero = %pK, is_master = %d",
+		__func__, slot_zero, is_master);
 
 	/* Check the input configuration */
 
 	if (slot_zero->magic != VCHIQ_MAGIC) {
 		vchiq_loud_error_header();
 		vchiq_loud_error("Invalid VCHIQ magic value found.");
-		vchiq_loud_error("slot_zero=%x: magic=%x (expected %x)",
-			(unsigned int)slot_zero, slot_zero->magic, VCHIQ_MAGIC);
+		vchiq_loud_error("slot_zero=%pK: magic=%x (expected %x)",
+			slot_zero, slot_zero->magic, VCHIQ_MAGIC);
 		vchiq_loud_error_footer();
 		return VCHIQ_ERROR;
 	}
@@ -2333,10 +2339,8 @@ vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
 	if (slot_zero->version < VCHIQ_VERSION_MIN) {
 		vchiq_loud_error_header();
 		vchiq_loud_error("Incompatible VCHIQ versions found.");
-		vchiq_loud_error("slot_zero=%x: VideoCore version=%d "
-			"(minimum %d)",
-			(unsigned int)slot_zero, slot_zero->version,
-			VCHIQ_VERSION_MIN);
+		vchiq_loud_error("slot_zero=%pK: VideoCore version=%d (minimum %d)",
+			slot_zero, slot_zero->version, VCHIQ_VERSION_MIN);
 		vchiq_loud_error("Restart with a newer VideoCore image.");
 		vchiq_loud_error_footer();
 		return VCHIQ_ERROR;
@@ -2345,10 +2349,8 @@ vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
 	if (VCHIQ_VERSION < slot_zero->version_min) {
 		vchiq_loud_error_header();
 		vchiq_loud_error("Incompatible VCHIQ versions found.");
-		vchiq_loud_error("slot_zero=%x: version=%d (VideoCore "
-			"minimum %d)",
-			(unsigned int)slot_zero, VCHIQ_VERSION,
-			slot_zero->version_min);
+		vchiq_loud_error("slot_zero=%pK: version=%d (VideoCore minimum %d)",
+			slot_zero, VCHIQ_VERSION, slot_zero->version_min);
 		vchiq_loud_error("Restart with a newer kernel.");
 		vchiq_loud_error_footer();
 		return VCHIQ_ERROR;
@@ -2360,26 +2362,20 @@ vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
 		 (slot_zero->max_slots_per_side != VCHIQ_MAX_SLOTS_PER_SIDE)) {
 		vchiq_loud_error_header();
 		if (slot_zero->slot_zero_size != sizeof(VCHIQ_SLOT_ZERO_T))
-			vchiq_loud_error("slot_zero=%x: slot_zero_size=%x "
-				"(expected %x)",
-				(unsigned int)slot_zero,
-				slot_zero->slot_zero_size,
-				sizeof(VCHIQ_SLOT_ZERO_T));
+			vchiq_loud_error("slot_zero=%pK: slot_zero_size=%d (expected %d)",
+				slot_zero, slot_zero->slot_zero_size,
+				(int)sizeof(VCHIQ_SLOT_ZERO_T));
 		if (slot_zero->slot_size != VCHIQ_SLOT_SIZE)
-			vchiq_loud_error("slot_zero=%x: slot_size=%d "
-				"(expected %d",
-				(unsigned int)slot_zero, slot_zero->slot_size,
+			vchiq_loud_error("slot_zero=%pK: slot_size=%d (expected %d)",
+				slot_zero, slot_zero->slot_size,
 				VCHIQ_SLOT_SIZE);
 		if (slot_zero->max_slots != VCHIQ_MAX_SLOTS)
-			vchiq_loud_error("slot_zero=%x: max_slots=%d "
-				"(expected %d)",
-				(unsigned int)slot_zero, slot_zero->max_slots,
+			vchiq_loud_error("slot_zero=%pK: max_slots=%d (expected %d)",
+				slot_zero, slot_zero->max_slots,
 				VCHIQ_MAX_SLOTS);
 		if (slot_zero->max_slots_per_side != VCHIQ_MAX_SLOTS_PER_SIDE)
-			vchiq_loud_error("slot_zero=%x: max_slots_per_side=%d "
-				"(expected %d)",
-				(unsigned int)slot_zero,
-				slot_zero->max_slots_per_side,
+			vchiq_loud_error("slot_zero=%pK: max_slots_per_side=%d (expected %d)",
+				slot_zero, slot_zero->max_slots_per_side,
 				VCHIQ_MAX_SLOTS_PER_SIDE);
 		vchiq_loud_error_footer();
 		return VCHIQ_ERROR;
@@ -2463,24 +2459,24 @@ vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
 	state->data_use_count = 0;
 	state->data_quota = state->slot_queue_available - 1;
 
-	local->trigger.event = &state->trigger_event;
-	remote_event_create(&local->trigger);
+	local->trigger.event = offsetof(VCHIQ_STATE_T, trigger_event);
+	remote_event_create(state, &local->trigger);
 	local->tx_pos = 0;
 
-	local->recycle.event = &state->recycle_event;
-	remote_event_create(&local->recycle);
+	local->recycle.event = offsetof(VCHIQ_STATE_T, recycle_event);
+	remote_event_create(state, &local->recycle);
 	local->slot_queue_recycle = state->slot_queue_available;
 
-	local->sync_trigger.event = &state->sync_trigger_event;
-	remote_event_create(&local->sync_trigger);
+	local->sync_trigger.event = offsetof(VCHIQ_STATE_T, sync_trigger_event);
+	remote_event_create(state, &local->sync_trigger);
 
-	local->sync_release.event = &state->sync_release_event;
-	remote_event_create(&local->sync_release);
+	local->sync_release.event = offsetof(VCHIQ_STATE_T, sync_release_event);
+	remote_event_create(state, &local->sync_release);
 
 	/* At start-of-day, the slot is empty and available */
 	((VCHIQ_HEADER_T *)SLOT_DATA_FROM_INDEX(state, local->slot_sync))->msgid
 		= VCHIQ_MSGID_PADDING;
-	remote_event_signal_local(&local->sync_release);
+	remote_event_signal_local(state, &local->sync_release);
 
 	local->debug[DEBUG_ENTRIES] = DEBUG_MAX;
 
@@ -2494,7 +2490,7 @@ vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
 		(void *)state,
 		threadname);
 
-	if (state->slot_handler_thread == NULL) {
+	if (IS_ERR(state->slot_handler_thread)) {
 		vchiq_loud_error_header();
 		vchiq_loud_error("couldn't create thread %s", threadname);
 		vchiq_loud_error_footer();
@@ -2507,7 +2503,7 @@ vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
 	state->recycle_thread = kthread_create(&recycle_func,
 		(void *)state,
 		threadname);
-	if (state->recycle_thread == NULL) {
+	if (IS_ERR(state->recycle_thread)) {
 		vchiq_loud_error_header();
 		vchiq_loud_error("couldn't create thread %s", threadname);
 		vchiq_loud_error_footer();
@@ -2520,7 +2516,7 @@ vchiq_init_state(VCHIQ_STATE_T *state, VCHIQ_SLOT_ZERO_T *slot_zero,
 	state->sync_thread = kthread_create(&sync_func,
 		(void *)state,
 		threadname);
-	if (state->sync_thread == NULL) {
+	if (IS_ERR(state->sync_thread)) {
 		vchiq_loud_error_header();
 		vchiq_loud_error("couldn't create thread %s", threadname);
 		vchiq_loud_error_footer();
@@ -2684,14 +2680,19 @@ vchiq_open_service_internal(VCHIQ_SERVICE_T *service, int client_id)
 		service->version,
 		service->version_min
 	};
-	VCHIQ_ELEMENT_T body = { &payload, sizeof(payload) };
 	VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
 
 	service->client_id = client_id;
 	vchiq_use_service_internal(service);
-	status = queue_message(service->state, NULL,
-		VCHIQ_MAKE_MSG(VCHIQ_MSG_OPEN, service->localport, 0),
-		&body, 1, sizeof(payload), QMFLAGS_IS_BLOCKING);
+	status = queue_message(service->state,
+			       NULL,
+			       VCHIQ_MAKE_MSG(VCHIQ_MSG_OPEN,
+					      service->localport,
+					      0),
+			       memcpy_copy_callback,
+			       &payload,
+			       sizeof(payload),
+			       QMFLAGS_IS_BLOCKING);
 	if (status == VCHIQ_SUCCESS) {
 		/* Wait for the ACK/NAK */
 		if (down_interruptible(&service->remove_event) != 0) {
@@ -2756,20 +2757,16 @@ release_service_messages(VCHIQ_SERVICE_T *service)
 				if ((port == service->localport) &&
 					(msgid & VCHIQ_MSGID_CLAIMED)) {
 					vchiq_log_info(vchiq_core_log_level,
-						"  fsi - hdr %x",
-						(unsigned int)header);
+						"  fsi - hdr %pK", header);
 					release_slot(state, slot_info, header,
 						NULL);
 				}
 				pos += calc_stride(header->size);
 				if (pos > VCHIQ_SLOT_SIZE) {
 					vchiq_log_error(vchiq_core_log_level,
-						"fsi - pos %x: header %x, "
-						"msgid %x, header->msgid %x, "
-						"header->size %x",
-						pos, (unsigned int)header,
-						msgid, header->msgid,
-						header->size);
+						"fsi - pos %x: header %pK, msgid %x, header->msgid %x, header->size %x",
+						pos, header, msgid,
+						header->msgid, header->size);
 					WARN(1, "invalid slot position\n");
 				}
 			}
@@ -2783,7 +2780,7 @@ do_abort_bulks(VCHIQ_SERVICE_T *service)
 	VCHIQ_STATUS_T status;
 
 	/* Abort any outstanding bulk transfers */
-	if (mutex_lock_interruptible(&service->bulk_mutex) != 0)
+	if (mutex_lock_killable(&service->bulk_mutex) != 0)
 		return 0;
 	abort_outstanding_bulks(service, &service->bulk_tx);
 	abort_outstanding_bulks(service, &service->bulk_rx);
@@ -3303,7 +3300,7 @@ vchiq_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle,
 	queue = (dir == VCHIQ_BULK_TRANSMIT) ?
 		&service->bulk_tx : &service->bulk_rx;
 
-	if (mutex_lock_interruptible(&service->bulk_mutex) != 0) {
+	if (mutex_lock_killable(&service->bulk_mutex) != 0) {
 		status = VCHIQ_RETRY;
 		goto error_exit;
 	}
@@ -3317,7 +3314,7 @@ vchiq_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle,
 				status = VCHIQ_RETRY;
 				goto error_exit;
 			}
-			if (mutex_lock_interruptible(&service->bulk_mutex)
+			if (mutex_lock_killable(&service->bulk_mutex)
 				!= 0) {
 				status = VCHIQ_RETRY;
 				goto error_exit;
@@ -3341,14 +3338,13 @@ vchiq_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle,
 	wmb();
 
 	vchiq_log_info(vchiq_core_log_level,
-		"%d: bt (%d->%d) %cx %x@%x %x",
-		state->id,
-		service->localport, service->remoteport, dir_char,
-		size, (unsigned int)bulk->data, (unsigned int)userdata);
+		"%d: bt (%d->%d) %cx %x@%pK %pK",
+		state->id, service->localport, service->remoteport, dir_char,
+		size, bulk->data, userdata);
 
 	/* The slot mutex must be held when the service is being closed, so
 	   claim it here to ensure that isn't happening */
-	if (mutex_lock_interruptible(&state->slot_mutex) != 0) {
+	if (mutex_lock_killable(&state->slot_mutex) != 0) {
 		status = VCHIQ_RETRY;
 		goto cancel_bulk_error_exit;
 	}
@@ -3363,16 +3359,19 @@ vchiq_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle,
 				(dir == VCHIQ_BULK_TRANSMIT) ?
 				VCHIQ_POLL_TXNOTIFY : VCHIQ_POLL_RXNOTIFY);
 	} else {
-		int payload[2] = { (int)bulk->data, bulk->size };
-		VCHIQ_ELEMENT_T element = { payload, sizeof(payload) };
-
-		status = queue_message(state, NULL,
-			VCHIQ_MAKE_MSG(dir_msgtype,
-				service->localport, service->remoteport),
-			&element, 1, sizeof(payload),
-			QMFLAGS_IS_BLOCKING |
-			QMFLAGS_NO_MUTEX_LOCK |
-			QMFLAGS_NO_MUTEX_UNLOCK);
+		int payload[2] = { (int)(long)bulk->data, bulk->size };
+
+		status = queue_message(state,
+				       NULL,
+				       VCHIQ_MAKE_MSG(dir_msgtype,
+						      service->localport,
+						      service->remoteport),
+				       memcpy_copy_callback,
+				       &payload,
+				       sizeof(payload),
+				       QMFLAGS_IS_BLOCKING |
+				       QMFLAGS_NO_MUTEX_LOCK |
+				       QMFLAGS_NO_MUTEX_UNLOCK);
 		if (status != VCHIQ_SUCCESS) {
 			goto unlock_both_error_exit;
 		}
@@ -3418,26 +3417,22 @@ error_exit:
 
 VCHIQ_STATUS_T
 vchiq_queue_message(VCHIQ_SERVICE_HANDLE_T handle,
-	const VCHIQ_ELEMENT_T *elements, unsigned int count)
+		    ssize_t (*copy_callback)(void *context, void *dest,
+					     size_t offset, size_t maxsize),
+		    void *context,
+		    size_t size)
 {
 	VCHIQ_SERVICE_T *service = find_service_by_handle(handle);
 	VCHIQ_STATUS_T status = VCHIQ_ERROR;
 
-	unsigned int size = 0;
-	unsigned int i;
-
 	if (!service ||
 		(vchiq_check_service(service) != VCHIQ_SUCCESS))
 		goto error_exit;
 
-	for (i = 0; i < (unsigned int)count; i++) {
-		if (elements[i].size) {
-			if (elements[i].data == NULL) {
-				VCHIQ_SERVICE_STATS_INC(service, error_count);
-				goto error_exit;
-			}
-			size += elements[i].size;
-		}
+	if (!size) {
+		VCHIQ_SERVICE_STATS_INC(service, error_count);
+		goto error_exit;
+
 	}
 
 	if (size > VCHIQ_MAX_MSG_SIZE) {
@@ -3451,14 +3446,14 @@ vchiq_queue_message(VCHIQ_SERVICE_HANDLE_T handle,
 				VCHIQ_MAKE_MSG(VCHIQ_MSG_DATA,
 					service->localport,
 					service->remoteport),
-				elements, count, size, 1);
+				copy_callback, context, size, 1);
 		break;
 	case VCHIQ_SRVSTATE_OPENSYNC:
 		status = queue_message_sync(service->state, service,
 				VCHIQ_MAKE_MSG(VCHIQ_MSG_DATA,
 					service->localport,
 					service->remoteport),
-				elements, count, size, 1);
+				copy_callback, context, size, 1);
 		break;
 	default:
 		status = VCHIQ_ERROR;
@@ -3691,13 +3686,11 @@ vchiq_dump_state(void *dump_context, VCHIQ_STATE_T *state)
 	vchiq_dump(dump_context, buf, len + 1);
 
 	len = snprintf(buf, sizeof(buf),
-		"  tx_pos=%x(@%x), rx_pos=%x(@%x)",
+		"  tx_pos=%x(@%pK), rx_pos=%x(@%pK)",
 		state->local->tx_pos,
-		(uint32_t)state->tx_data +
-			(state->local_tx_pos & VCHIQ_SLOT_MASK),
+		state->tx_data + (state->local_tx_pos & VCHIQ_SLOT_MASK),
 		state->rx_pos,
-		(uint32_t)state->rx_data +
-			(state->rx_pos & VCHIQ_SLOT_MASK));
+		state->rx_data + (state->rx_pos & VCHIQ_SLOT_MASK));
 	vchiq_dump(dump_context, buf, len + 1);
 
 	len = snprintf(buf, sizeof(buf),
@@ -3747,7 +3740,7 @@ vchiq_dump_service_state(void *dump_context, VCHIQ_SERVICE_T *service)
 	char buf[80];
 	int len;
 
-	len = snprintf(buf, sizeof(buf), "Service %d: %s (ref %u)",
+	len = snprintf(buf, sizeof(buf), "Service %u: %s (ref %u)",
 		service->localport, srvstate_names[service->srvstate],
 		service->ref_count - 1); /*Don't include the lock just taken*/
 
@@ -3759,7 +3752,7 @@ vchiq_dump_service_state(void *dump_context, VCHIQ_SERVICE_T *service)
 		int tx_pending, rx_pending;
 		if (service->remoteport != VCHIQ_PORT_FREE) {
 			int len2 = snprintf(remoteport, sizeof(remoteport),
-				"%d", service->remoteport);
+				"%u", service->remoteport);
 			if (service->public_fourcc != VCHIQ_FOURCC_INVALID)
 				snprintf(remoteport + len2,
 					sizeof(remoteport) - len2,
@@ -3888,26 +3881,26 @@ VCHIQ_STATUS_T vchiq_send_remote_use_active(VCHIQ_STATE_T *state)
 	return status;
 }
 
-void vchiq_log_dump_mem(const char *label, uint32_t addr, const void *voidMem,
-	size_t numBytes)
+void vchiq_log_dump_mem(const char *label, uint32_t addr, const void *void_mem,
+	size_t num_bytes)
 {
-	const uint8_t  *mem = (const uint8_t *)voidMem;
+	const uint8_t  *mem = (const uint8_t *)void_mem;
 	size_t          offset;
-	char            lineBuf[100];
+	char            line_buf[100];
 	char           *s;
 
-	while (numBytes > 0) {
-		s = lineBuf;
+	while (num_bytes > 0) {
+		s = line_buf;
 
 		for (offset = 0; offset < 16; offset++) {
-			if (offset < numBytes)
+			if (offset < num_bytes)
 				s += snprintf(s, 4, "%02x ", mem[offset]);
 			else
 				s += snprintf(s, 4, "   ");
 		}
 
 		for (offset = 0; offset < 16; offset++) {
-			if (offset < numBytes) {
+			if (offset < num_bytes) {
 				uint8_t ch = mem[offset];
 
 				if ((ch < ' ') || (ch > '~'))
@@ -3919,16 +3912,16 @@ void vchiq_log_dump_mem(const char *label, uint32_t addr, const void *voidMem,
 
 		if ((label != NULL) && (*label != '\0'))
 			vchiq_log_trace(VCHIQ_LOG_TRACE,
-				"%s: %08x: %s", label, addr, lineBuf);
+				"%s: %08x: %s", label, addr, line_buf);
 		else
 			vchiq_log_trace(VCHIQ_LOG_TRACE,
-				"%08x: %s", addr, lineBuf);
+				"%08x: %s", addr, line_buf);
 
 		addr += 16;
 		mem += 16;
-		if (numBytes > 16)
-			numBytes -= 16;
+		if (num_bytes > 16)
+			num_bytes -= 16;
 		else
-			numBytes = 0;
+			num_bytes = 0;
 	}
 }
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h
index 9be484c776d0..9e164652548a 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h
@@ -184,11 +184,11 @@ enum {
 
 #define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug;
 #define DEBUG_TRACE(d) \
-	do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(); } while (0)
+	do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0)
 #define DEBUG_VALUE(d, v) \
-	do { debug_ptr[DEBUG_ ## d] = (v); dsb(); } while (0)
+	do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0)
 #define DEBUG_COUNT(d) \
-	do { debug_ptr[DEBUG_ ## d]++; dsb(); } while (0)
+	do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0)
 
 #else /* VCHIQ_ENABLE_DEBUG */
 
@@ -264,7 +264,8 @@ typedef struct vchiq_bulk_queue_struct {
 typedef struct remote_event_struct {
 	int armed;
 	int fired;
-	struct semaphore *event;
+	/* Contains offset from the beginning of the VCHIQ_STATE_T structure */
+	u32 event;
 } REMOTE_EVENT_T;
 
 typedef struct opaque_platform_state_t *VCHIQ_PLATFORM_STATE_T;
@@ -633,9 +634,6 @@ vchiq_transfer_bulk(VCHIQ_BULK_T *bulk);
 extern void
 vchiq_complete_bulk(VCHIQ_BULK_T *bulk);
 
-extern VCHIQ_STATUS_T
-vchiq_copy_from_user(void *dst, const void *src, int size);
-
 extern void
 remote_event_signal(REMOTE_EVENT_T *event);
 
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_debugfs.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_debugfs.c
index 7e032130d967..f07cd4448ddf 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_debugfs.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_debugfs.c
@@ -120,7 +120,7 @@ static int debugfs_log_open(struct inode *inode, struct file *file)
 	return single_open(file, debugfs_log_show, inode->i_private);
 }
 
-static int debugfs_log_write(struct file *file,
+static ssize_t debugfs_log_write(struct file *file,
 	const char __user *buffer,
 	size_t count, loff_t *ppos)
 {
@@ -229,7 +229,7 @@ static int debugfs_trace_open(struct inode *inode, struct file *file)
 	return single_open(file, debugfs_trace_show, inode->i_private);
 }
 
-static int debugfs_trace_write(struct file *file,
+static ssize_t debugfs_trace_write(struct file *file,
 	const char __user *buffer,
 	size_t count, loff_t *ppos)
 {
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_if.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_if.h
index 8067bbe7ce8d..377e8e48bb54 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_if.h
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_if.h
@@ -141,9 +141,12 @@ extern VCHIQ_STATUS_T vchiq_use_service(VCHIQ_SERVICE_HANDLE_T service);
 extern VCHIQ_STATUS_T vchiq_use_service_no_resume(
 	VCHIQ_SERVICE_HANDLE_T service);
 extern VCHIQ_STATUS_T vchiq_release_service(VCHIQ_SERVICE_HANDLE_T service);
-
-extern VCHIQ_STATUS_T vchiq_queue_message(VCHIQ_SERVICE_HANDLE_T service,
-	const VCHIQ_ELEMENT_T *elements, unsigned int count);
+extern VCHIQ_STATUS_T
+vchiq_queue_message(VCHIQ_SERVICE_HANDLE_T handle,
+		    ssize_t (*copy_callback)(void *context, void *dest,
+					     size_t offset, size_t maxsize),
+		    void *context,
+		    size_t size);
 extern void           vchiq_release_message(VCHIQ_SERVICE_HANDLE_T service,
 	VCHIQ_HEADER_T *header);
 extern VCHIQ_STATUS_T vchiq_queue_bulk_transmit(VCHIQ_SERVICE_HANDLE_T service,
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_kern_lib.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_kern_lib.c
index 25e7011edc50..e93922a87263 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_kern_lib.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_kern_lib.c
@@ -70,7 +70,7 @@ vchiq_blocking_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle, void *data,
 *
 ***************************************************************************/
 #define VCHIQ_INIT_RETRIES 10
-VCHIQ_STATUS_T vchiq_initialise(VCHIQ_INSTANCE_T *instanceOut)
+VCHIQ_STATUS_T vchiq_initialise(VCHIQ_INSTANCE_T *instance_out)
 {
 	VCHIQ_STATUS_T status = VCHIQ_ERROR;
 	VCHIQ_STATE_T *state;
@@ -108,7 +108,7 @@ VCHIQ_STATUS_T vchiq_initialise(VCHIQ_INSTANCE_T *instanceOut)
 	mutex_init(&instance->bulk_waiter_list_mutex);
 	INIT_LIST_HEAD(&instance->bulk_waiter_list);
 
-	*instanceOut = instance;
+	*instance_out = instance;
 
 	status = VCHIQ_SUCCESS;
 
@@ -134,7 +134,7 @@ VCHIQ_STATUS_T vchiq_shutdown(VCHIQ_INSTANCE_T instance)
 	vchiq_log_trace(vchiq_core_log_level,
 		"%s(%p) called", __func__, instance);
 
-	if (mutex_lock_interruptible(&state->mutex) != 0)
+	if (mutex_lock_killable(&state->mutex) != 0)
 		return VCHIQ_RETRY;
 
 	/* Remove all services */
@@ -155,9 +155,8 @@ VCHIQ_STATUS_T vchiq_shutdown(VCHIQ_INSTANCE_T instance)
 					list);
 			list_del(pos);
 			vchiq_log_info(vchiq_arm_log_level,
-					"bulk_waiter - cleaned up %x "
-					"for pid %d",
-					(unsigned int)waiter, waiter->pid);
+					"bulk_waiter - cleaned up %pK for pid %d",
+					waiter, waiter->pid);
 			kfree(waiter);
 		}
 		kfree(instance);
@@ -192,7 +191,7 @@ VCHIQ_STATUS_T vchiq_connect(VCHIQ_INSTANCE_T instance)
 	vchiq_log_trace(vchiq_core_log_level,
 		"%s(%p) called", __func__, instance);
 
-	if (mutex_lock_interruptible(&state->mutex) != 0) {
+	if (mutex_lock_killable(&state->mutex) != 0) {
 		vchiq_log_trace(vchiq_core_log_level,
 			"%s: call to mutex_lock failed", __func__);
 		status = VCHIQ_RETRY;
@@ -450,8 +449,8 @@ vchiq_blocking_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle, void *data,
 		list_add(&waiter->list, &instance->bulk_waiter_list);
 		mutex_unlock(&instance->bulk_waiter_list_mutex);
 		vchiq_log_info(vchiq_arm_log_level,
-				"saved bulk_waiter %x for pid %d",
-				(unsigned int)waiter, current->pid);
+				"saved bulk_waiter %pK for pid %d",
+				waiter, current->pid);
 	}
 
 	return status;
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_killable.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_killable.h
index 335446e05476..778063ba312a 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_killable.h
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_killable.h
@@ -52,18 +52,4 @@ static inline int __must_check down_interruptible_killable(struct semaphore *sem
 }
 #define down_interruptible down_interruptible_killable
 
-
-static inline int __must_check mutex_lock_interruptible_killable(struct mutex *lock)
-{
-	/* Allow interception of killable signals only. We don't want to be interrupted by harmless signals like SIGALRM */
-	int ret;
-	sigset_t blocked, oldset;
-	siginitsetinv(&blocked, SHUTDOWN_SIGS);
-	sigprocmask(SIG_SETMASK, &blocked, &oldset);
-	ret = mutex_lock_interruptible(lock);
-	sigprocmask(SIG_SETMASK, &oldset, NULL);
-	return ret;
-}
-#define mutex_lock_interruptible mutex_lock_interruptible_killable
-
 #endif
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_memdrv.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_memdrv.h
index d02e7764bd0d..dd43458f306f 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_memdrv.h
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_memdrv.h
@@ -42,13 +42,13 @@
 /* ---- Constants and Types ---------------------------------------------- */
 
 typedef struct {
-	 void                   *armSharedMemVirt;
-	 dma_addr_t              armSharedMemPhys;
-	 size_t                  armSharedMemSize;
+	 void                   *arm_shared_mem_virt;
+	 dma_addr_t              arm_shared_mem_phys;
+	 size_t                  arm_shared_mem_size;
 
-	 void                   *vcSharedMemVirt;
-	 dma_addr_t              vcSharedMemPhys;
-	 size_t                  vcSharedMemSize;
+	 void                   *vc_shared_mem_virt;
+	 dma_addr_t              vc_shared_mem_phys;
+	 size_t                  vc_shared_mem_size;
 } VCHIQ_SHARED_MEM_INFO_T;
 
 /* ---- Variable Externs ------------------------------------------------- */
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_pagelist.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_pagelist.h
index 54a3ecec69ef..12c304ceb952 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_pagelist.h
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_pagelist.h
@@ -43,11 +43,13 @@
 #define PAGELIST_READ_WITH_FRAGMENTS 2
 
 typedef struct pagelist_struct {
-	unsigned long length;
-	unsigned short type;
-	unsigned short offset;
-	unsigned long addrs[1];	/* N.B. 12 LSBs hold the number of following
-				   pages at consecutive addresses. */
+	u32 length;
+	u16 type;
+	u16 offset;
+	u32 addrs[1];	/* N.B. 12 LSBs hold the number
+			 * of following pages at consecutive
+			 * addresses.
+			 */
 } PAGELIST_T;
 
 typedef struct fragments_struct {
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_shim.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_shim.c
index 8072ff613636..d9771394a041 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_shim.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_shim.c
@@ -148,10 +148,10 @@ EXPORT_SYMBOL(vchi_msg_remove);
  * Name: vchi_msg_queue
  *
  * Arguments:  VCHI_SERVICE_HANDLE_T handle,
- *             const void *data,
- *             uint32_t data_size,
- *             VCHI_FLAGS_T flags,
- *             void *msg_handle,
+ *             ssize_t (*copy_callback)(void *context, void *dest,
+ *				        size_t offset, size_t maxsize),
+ *	       void *context,
+ *             uint32_t data_size
  *
  * Description: Thin wrapper to queue a message onto a connection
  *
@@ -159,28 +159,29 @@ EXPORT_SYMBOL(vchi_msg_remove);
  *
  ***********************************************************/
 int32_t vchi_msg_queue(VCHI_SERVICE_HANDLE_T handle,
-	const void *data,
-	uint32_t data_size,
-	VCHI_FLAGS_T flags,
-	void *msg_handle)
+	ssize_t (*copy_callback)(void *context, void *dest,
+				 size_t offset, size_t maxsize),
+	void *context,
+	uint32_t data_size)
 {
 	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
-	VCHIQ_ELEMENT_T element = {data, data_size};
 	VCHIQ_STATUS_T status;
 
-	(void)msg_handle;
-
-	WARN_ON(flags != VCHI_FLAGS_BLOCK_UNTIL_QUEUED);
+	while (1) {
+		status = vchiq_queue_message(service->handle,
+					     copy_callback,
+					     context,
+					     data_size);
 
-	status = vchiq_queue_message(service->handle, &element, 1);
+		/*
+		 * vchiq_queue_message() may return VCHIQ_RETRY, so we need to
+		 * implement a retry mechanism since this function is supposed
+		 * to block until queued
+		 */
+		if (status != VCHIQ_RETRY)
+			break;
 
-	/* vchiq_queue_message() may return VCHIQ_RETRY, so we need to
-	** implement a retry mechanism since this function is supposed
-	** to block until queued
-	*/
-	while (status == VCHIQ_RETRY) {
 		msleep(1);
-		status = vchiq_queue_message(service->handle, &element, 1);
 	}
 
 	return vchiq_status_to_vchi(status);
@@ -229,17 +230,18 @@ int32_t vchi_bulk_queue_receive(VCHI_SERVICE_HANDLE_T handle,
 		return vchiq_status_to_vchi(VCHIQ_ERROR);
 	}
 
-	status = vchiq_bulk_receive(service->handle, data_dst, data_size,
-		bulk_handle, mode);
-
-	/* vchiq_bulk_receive() may return VCHIQ_RETRY, so we need to
-	** implement a retry mechanism since this function is supposed
-	** to block until queued
-	*/
-	while (status == VCHIQ_RETRY) {
-		msleep(1);
+	while (1) {
 		status = vchiq_bulk_receive(service->handle, data_dst,
 			data_size, bulk_handle, mode);
+		/*
+		 * vchiq_bulk_receive() may return VCHIQ_RETRY, so we need to
+		 * implement a retry mechanism since this function is supposed
+		 * to block until queued
+		 */
+		if (status != VCHIQ_RETRY)
+			break;
+
+		msleep(1);
 	}
 
 	return vchiq_status_to_vchi(status);
@@ -289,17 +291,19 @@ int32_t vchi_bulk_queue_transmit(VCHI_SERVICE_HANDLE_T handle,
 		return vchiq_status_to_vchi(VCHIQ_ERROR);
 	}
 
-	status = vchiq_bulk_transmit(service->handle, data_src, data_size,
-		bulk_handle, mode);
-
-	/* vchiq_bulk_transmit() may return VCHIQ_RETRY, so we need to
-	** implement a retry mechanism since this function is supposed
-	** to block until queued
-	*/
-	while (status == VCHIQ_RETRY) {
-		msleep(1);
+	while (1) {
 		status = vchiq_bulk_transmit(service->handle, data_src,
 			data_size, bulk_handle, mode);
+
+		/*
+		 * vchiq_bulk_transmit() may return VCHIQ_RETRY, so we need to
+		 * implement a retry mechanism since this function is supposed
+		 * to block until queued
+		 */
+		if (status != VCHIQ_RETRY)
+			break;
+
+		msleep(1);
 	}
 
 	return vchiq_status_to_vchi(status);
@@ -350,44 +354,6 @@ int32_t vchi_msg_dequeue(VCHI_SERVICE_HANDLE_T handle,
 EXPORT_SYMBOL(vchi_msg_dequeue);
 
 /***********************************************************
- * Name: vchi_msg_queuev
- *
- * Arguments:  VCHI_SERVICE_HANDLE_T handle,
- *             VCHI_MSG_VECTOR_T *vector,
- *             uint32_t count,
- *             VCHI_FLAGS_T flags,
- *             void *msg_handle
- *
- * Description: Thin wrapper to queue a message onto a connection
- *
- * Returns: int32_t - success == 0
- *
- ***********************************************************/
-
-vchiq_static_assert(sizeof(VCHI_MSG_VECTOR_T) == sizeof(VCHIQ_ELEMENT_T));
-vchiq_static_assert(offsetof(VCHI_MSG_VECTOR_T, vec_base) ==
-	offsetof(VCHIQ_ELEMENT_T, data));
-vchiq_static_assert(offsetof(VCHI_MSG_VECTOR_T, vec_len) ==
-	offsetof(VCHIQ_ELEMENT_T, size));
-
-int32_t vchi_msg_queuev(VCHI_SERVICE_HANDLE_T handle,
-	VCHI_MSG_VECTOR_T *vector,
-	uint32_t count,
-	VCHI_FLAGS_T flags,
-	void *msg_handle)
-{
-	SHIM_SERVICE_T *service = (SHIM_SERVICE_T *)handle;
-
-	(void)msg_handle;
-
-	WARN_ON(flags != VCHI_FLAGS_BLOCK_UNTIL_QUEUED);
-
-	return vchiq_status_to_vchi(vchiq_queue_message(service->handle,
-		(const VCHIQ_ELEMENT_T *)vector, count));
-}
-EXPORT_SYMBOL(vchi_msg_queuev);
-
-/***********************************************************
  * Name: vchi_held_msg_release
  *
  * Arguments:  VCHI_HELD_MSG_T *message
@@ -400,8 +366,16 @@ EXPORT_SYMBOL(vchi_msg_queuev);
  ***********************************************************/
 int32_t vchi_held_msg_release(VCHI_HELD_MSG_T *message)
 {
-	vchiq_release_message((VCHIQ_SERVICE_HANDLE_T)message->service,
-		(VCHIQ_HEADER_T *)message->message);
+	/*
+	 * Convert the service field pointer back to an
+	 * VCHIQ_SERVICE_HANDLE_T which is an int.
+	 * This pointer is opaque to everything except
+	 * vchi_msg_hold which simply upcasted the int
+	 * to a pointer.
+	 */
+
+	vchiq_release_message((VCHIQ_SERVICE_HANDLE_T)(long)message->service,
+			      (VCHIQ_HEADER_T *)message->message);
 
 	return 0;
 }
@@ -445,8 +419,16 @@ int32_t vchi_msg_hold(VCHI_SERVICE_HANDLE_T handle,
 	*data = header->data;
 	*msg_size = header->size;
 
+	/*
+	 * upcast the VCHIQ_SERVICE_HANDLE_T which is an int
+	 * to a pointer and stuff it in the held message.
+	 * This pointer is opaque to everything except
+	 * vchi_held_msg_release which simply downcasts it back
+	 * to an int.
+	 */
+
 	message_handle->service =
-		(struct opaque_vchi_service_t *)service->handle;
+		(struct opaque_vchi_service_t *)(long)service->handle;
 	message_handle->message = header;
 
 	return 0;
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.c
index 384acb8d2eae..f76f4d790532 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.c
@@ -61,8 +61,7 @@ int vchiu_queue_init(VCHIU_QUEUE_T *queue, int size)
 
 void vchiu_queue_delete(VCHIU_QUEUE_T *queue)
 {
-	if (queue->storage != NULL)
-		kfree(queue->storage);
+	kfree(queue->storage);
 }
 
 int vchiu_queue_is_empty(VCHIU_QUEUE_T *queue)
diff --git a/drivers/staging/vme/devices/vme_pio2.h b/drivers/staging/vme/devices/vme_pio2.h
index d5d94c43c074..5577df3199e7 100644
--- a/drivers/staging/vme/devices/vme_pio2.h
+++ b/drivers/staging/vme/devices/vme_pio2.h
@@ -48,8 +48,6 @@ static const int PIO2_REGS_INT_MASK[8] = { PIO2_REGS_INT_MASK0,
 					PIO2_REGS_INT_MASK6,
 					PIO2_REGS_INT_MASK7 };
 
-
-
 #define PIO2_REGS_CTRL			0x18
 #define PIO2_REGS_VME_VECTOR		0x19
 #define PIO2_REGS_CNTR0			0x20
@@ -63,7 +61,6 @@ static const int PIO2_REGS_INT_MASK[8] = { PIO2_REGS_INT_MASK0,
 
 #define PIO2_REGS_ID			0x30
 
-
 /* PIO2_REGS_DATAx (0x0 - 0x3) */
 
 static const int PIO2_CHANNEL_BANK[32] = { 0, 0, 0, 0, 0, 0, 0, 0,
@@ -204,8 +201,6 @@ static const int PIO2_CNTR_SC_DEV[6] = { PIO2_CNTR_SC_DEV0, PIO2_CNTR_SC_DEV1,
 
 #define PIO2_CNTR_BCD			1
 
-
-
 enum pio2_bank_config { NOFIT, INPUT, OUTPUT, BOTH };
 enum pio2_int_config { NONE = 0, LOW2HIGH = 1, HIGH2LOW = 2, EITHER = 4 };
 
@@ -240,10 +235,10 @@ struct pio2_card {
 	struct pio2_cntr cntr[6];
 };
 
-int pio2_cntr_reset(struct pio2_card *);
+int pio2_cntr_reset(struct pio2_card *card);
 
-int pio2_gpio_reset(struct pio2_card *);
-int pio2_gpio_init(struct pio2_card *);
-void pio2_gpio_exit(struct pio2_card *);
+int pio2_gpio_reset(struct pio2_card *card);
+int pio2_gpio_init(struct pio2_card *card);
+void pio2_gpio_exit(struct pio2_card *card);
 
 #endif /* _VME_PIO2_H_ */
diff --git a/drivers/staging/vme/devices/vme_pio2_core.c b/drivers/staging/vme/devices/vme_pio2_core.c
index 8e66a520266c..20a2d835fdaa 100644
--- a/drivers/staging/vme/devices/vme_pio2_core.c
+++ b/drivers/staging/vme/devices/vme_pio2_core.c
@@ -365,7 +365,7 @@ static int pio2_probe(struct vme_dev *vdev)
 		vec = card->irq_vector | PIO2_VECTOR_CNTR[i];
 
 		retval = vme_irq_request(vdev, card->irq_level, vec,
-			&pio2_int, card);
+					 &pio2_int, card);
 		if (retval < 0) {
 			dev_err(&card->vdev->dev,
 				"Unable to attach VME interrupt vector0x%x, level 0x%x\n",
diff --git a/drivers/staging/vme/devices/vme_user.c b/drivers/staging/vme/devices/vme_user.c
index d84dffb894f4..87aa5174df22 100644
--- a/drivers/staging/vme/devices/vme_user.c
+++ b/drivers/staging/vme/devices/vme_user.c
@@ -661,7 +661,7 @@ err_sysfs:
 	}
 	class_destroy(vme_user_sysfs_class);
 
-	/* Ensure counter set correcty to unalloc all master windows */
+	/* Ensure counter set correctly to unalloc all master windows */
 	i = MASTER_MAX + 1;
 err_master:
 	while (i > MASTER_MINOR) {
@@ -671,7 +671,7 @@ err_master:
 	}
 
 	/*
-	 * Ensure counter set correcty to unalloc all slave windows and buffers
+	 * Ensure counter set correctly to unalloc all slave windows and buffers
 	 */
 	i = SLAVE_MAX + 1;
 err_slave:
@@ -716,7 +716,7 @@ static int vme_user_remove(struct vme_dev *dev)
 	/* Unregister device driver */
 	cdev_del(vme_user_cdev);
 
-	/* Unregiser the major and minor device numbers */
+	/* Unregister the major and minor device numbers */
 	unregister_chrdev_region(MKDEV(VME_MAJOR, 0), VME_DEVS);
 
 	return 0;
diff --git a/drivers/staging/vt6655/baseband.c b/drivers/staging/vt6655/baseband.c
index de503a316e71..44dfa5421374 100644
--- a/drivers/staging/vt6655/baseband.c
+++ b/drivers/staging/vt6655/baseband.c
@@ -12,11 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- *
  * File: baseband.c
  *
  * Purpose: Implement functions to access baseband
@@ -1916,7 +1911,7 @@ void vnt_get_phy_field(struct vnt_private *priv, u32 frame_length,
  *
  * Parameters:
  *  In:
- *      dwIoBase    - I/O base address
+ *      iobase      - I/O base address
  *      byBBAddr    - address of register in Baseband
  *  Out:
  *      pbyData     - data read
@@ -1927,24 +1922,24 @@ void vnt_get_phy_field(struct vnt_private *priv, u32 frame_length,
 bool BBbReadEmbedded(struct vnt_private *priv,
 		     unsigned char byBBAddr, unsigned char *pbyData)
 {
-	void __iomem *dwIoBase = priv->PortOffset;
+	void __iomem *iobase = priv->PortOffset;
 	unsigned short ww;
 	unsigned char byValue;
 
 	/* BB reg offset */
-	VNSvOutPortB(dwIoBase + MAC_REG_BBREGADR, byBBAddr);
+	VNSvOutPortB(iobase + MAC_REG_BBREGADR, byBBAddr);
 
 	/* turn on REGR */
-	MACvRegBitsOn(dwIoBase, MAC_REG_BBREGCTL, BBREGCTL_REGR);
+	MACvRegBitsOn(iobase, MAC_REG_BBREGCTL, BBREGCTL_REGR);
 	/* W_MAX_TIMEOUT is the timeout period */
 	for (ww = 0; ww < W_MAX_TIMEOUT; ww++) {
-		VNSvInPortB(dwIoBase + MAC_REG_BBREGCTL, &byValue);
+		VNSvInPortB(iobase + MAC_REG_BBREGCTL, &byValue);
 		if (byValue & BBREGCTL_DONE)
 			break;
 	}
 
 	/* get BB data */
-	VNSvInPortB(dwIoBase + MAC_REG_BBREGDATA, pbyData);
+	VNSvInPortB(iobase + MAC_REG_BBREGDATA, pbyData);
 
 	if (ww == W_MAX_TIMEOUT) {
 		pr_debug(" DBG_PORT80(0x30)\n");
@@ -1958,7 +1953,7 @@ bool BBbReadEmbedded(struct vnt_private *priv,
  *
  * Parameters:
  *  In:
- *      dwIoBase    - I/O base address
+ *      iobase      - I/O base address
  *      byBBAddr    - address of register in Baseband
  *      byData      - data to write
  *  Out:
@@ -1970,20 +1965,20 @@ bool BBbReadEmbedded(struct vnt_private *priv,
 bool BBbWriteEmbedded(struct vnt_private *priv,
 		      unsigned char byBBAddr, unsigned char byData)
 {
-	void __iomem *dwIoBase = priv->PortOffset;
+	void __iomem *iobase = priv->PortOffset;
 	unsigned short ww;
 	unsigned char byValue;
 
 	/* BB reg offset */
-	VNSvOutPortB(dwIoBase + MAC_REG_BBREGADR, byBBAddr);
+	VNSvOutPortB(iobase + MAC_REG_BBREGADR, byBBAddr);
 	/* set BB data */
-	VNSvOutPortB(dwIoBase + MAC_REG_BBREGDATA, byData);
+	VNSvOutPortB(iobase + MAC_REG_BBREGDATA, byData);
 
 	/* turn on BBREGCTL_REGW */
-	MACvRegBitsOn(dwIoBase, MAC_REG_BBREGCTL, BBREGCTL_REGW);
+	MACvRegBitsOn(iobase, MAC_REG_BBREGCTL, BBREGCTL_REGW);
 	/* W_MAX_TIMEOUT is the timeout period */
 	for (ww = 0; ww < W_MAX_TIMEOUT; ww++) {
-		VNSvInPortB(dwIoBase + MAC_REG_BBREGCTL, &byValue);
+		VNSvInPortB(iobase + MAC_REG_BBREGCTL, &byValue);
 		if (byValue & BBREGCTL_DONE)
 			break;
 	}
@@ -2000,7 +1995,7 @@ bool BBbWriteEmbedded(struct vnt_private *priv,
  *
  * Parameters:
  *  In:
- *      dwIoBase    - I/O base address
+ *      iobase      - I/O base address
  *      byRevId     - Revision ID
  *      byRFType    - RF type
  *  Out:
@@ -2014,7 +2009,7 @@ bool BBbVT3253Init(struct vnt_private *priv)
 {
 	bool bResult = true;
 	int        ii;
-	void __iomem *dwIoBase = priv->PortOffset;
+	void __iomem *iobase = priv->PortOffset;
 	unsigned char byRFType = priv->byRFType;
 	unsigned char byLocalID = priv->byLocalID;
 
@@ -2036,8 +2031,8 @@ bool BBbVT3253Init(struct vnt_private *priv)
 					byVT3253B0_AGC4_RFMD2959[ii][0],
 					byVT3253B0_AGC4_RFMD2959[ii][1]);
 
-			VNSvOutPortD(dwIoBase + MAC_REG_ITRTMSET, 0x23);
-			MACvRegBitsOn(dwIoBase, MAC_REG_PAPEDELAY, BIT(0));
+			VNSvOutPortD(iobase + MAC_REG_ITRTMSET, 0x23);
+			MACvRegBitsOn(iobase, MAC_REG_PAPEDELAY, BIT(0));
 		}
 		priv->abyBBVGA[0] = 0x18;
 		priv->abyBBVGA[1] = 0x0A;
@@ -2076,8 +2071,8 @@ bool BBbVT3253Init(struct vnt_private *priv)
 				byVT3253B0_AGC[ii][0],
 				byVT3253B0_AGC[ii][1]);
 
-		VNSvOutPortB(dwIoBase + MAC_REG_ITRTMSET, 0x23);
-		MACvRegBitsOn(dwIoBase, MAC_REG_PAPEDELAY, BIT(0));
+		VNSvOutPortB(iobase + MAC_REG_ITRTMSET, 0x23);
+		MACvRegBitsOn(iobase, MAC_REG_PAPEDELAY, BIT(0));
 
 		priv->abyBBVGA[0] = 0x14;
 		priv->abyBBVGA[1] = 0x0A;
@@ -2098,7 +2093,7 @@ bool BBbVT3253Init(struct vnt_private *priv)
 		 * 0x45->0x41(VC1/VC2 define, make the ANT_A, ANT_B inverted)
 		 */
 
-		/*bResult &= BBbWriteEmbedded(dwIoBase,0x09,0x41);*/
+		/*bResult &= BBbWriteEmbedded(iobase,0x09,0x41);*/
 
 		/* Init ANT B select,
 		 * RX Config CR10 = 0x28->0x2A,
@@ -2106,7 +2101,7 @@ bool BBbVT3253Init(struct vnt_private *priv)
 		 * make the ANT_A, ANT_B inverted)
 		 */
 
-		/*bResult &= BBbWriteEmbedded(dwIoBase,0x0a,0x28);*/
+		/*bResult &= BBbWriteEmbedded(iobase,0x0a,0x28);*/
 		/* Select VC1/VC2, CR215 = 0x02->0x06 */
 		bResult &= BBbWriteEmbedded(priv, 0xd7, 0x06);
 
@@ -2154,7 +2149,7 @@ bool BBbVT3253Init(struct vnt_private *priv)
 		priv->ldBmThreshold[2] = 0;
 		priv->ldBmThreshold[3] = 0;
 		/* Fix VT3226 DFC system timing issue */
-		MACvSetRFLE_LatchBase(dwIoBase);
+		MACvSetRFLE_LatchBase(iobase);
 		/* {{ RobertYu: 20050104 */
 	} else if (byRFType == RF_AIROHA7230) {
 		for (ii = 0; ii < CB_VT3253B0_INIT_FOR_AIROHA2230; ii++)
@@ -2162,16 +2157,15 @@ bool BBbVT3253Init(struct vnt_private *priv)
 				byVT3253B0_AIROHA2230[ii][0],
 				byVT3253B0_AIROHA2230[ii][1]);
 
-
 		/* {{ RobertYu:20050223, request by JerryChung */
 		/* Init ANT B select,TX Config CR09 = 0x61->0x45,
 		 * 0x45->0x41(VC1/VC2 define, make the ANT_A, ANT_B inverted)
 		 */
-		/*bResult &= BBbWriteEmbedded(dwIoBase,0x09,0x41);*/
+		/*bResult &= BBbWriteEmbedded(iobase,0x09,0x41);*/
 		/* Init ANT B select,RX Config CR10 = 0x28->0x2A,
 		 * 0x2A->0x28(VC1/VC2 define, make the ANT_A, ANT_B inverted)
 		 */
-		/*bResult &= BBbWriteEmbedded(dwIoBase,0x0a,0x28);*/
+		/*bResult &= BBbWriteEmbedded(iobase,0x0a,0x28);*/
 		/* Select VC1/VC2, CR215 = 0x02->0x06 */
 		bResult &= BBbWriteEmbedded(priv, 0xd7, 0x06);
 		/* }} */
@@ -2259,7 +2253,7 @@ void BBvSetVGAGainOffset(struct vnt_private *priv, unsigned char byData)
  *
  * Parameters:
  *  In:
- *      dwIoBase    - I/O base address
+ *      iobase      - I/O base address
  *  Out:
  *      none
  *
@@ -2280,7 +2274,7 @@ BBvSoftwareReset(struct vnt_private *priv)
  *
  * Parameters:
  *  In:
- *      dwIoBase    - I/O base address
+ *      iobase      - I/O base address
  *  Out:
  *      none
  *
@@ -2302,7 +2296,7 @@ BBvPowerSaveModeON(struct vnt_private *priv)
  *
  * Parameters:
  *  In:
- *      dwIoBase    - I/O base address
+ *      iobase      - I/O base address
  *  Out:
  *      none
  *
diff --git a/drivers/staging/vt6655/baseband.h b/drivers/staging/vt6655/baseband.h
index b4e8c43180ec..8a567c9155b4 100644
--- a/drivers/staging/vt6655/baseband.h
+++ b/drivers/staging/vt6655/baseband.h
@@ -12,11 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- *
  * File: baseband.h
  *
  * Purpose: Implement functions to access baseband
@@ -60,12 +55,6 @@
 #define TOP_RATE_2M         0x00200000
 #define TOP_RATE_1M         0x00100000
 
-#define BBvClearFOE(dwIoBase)				\
-	BBbWriteEmbedded(dwIoBase, 0xB1, 0)
-
-#define BBvSetFOE(dwIoBase)				\
-	BBbWriteEmbedded(dwIoBase, 0xB1, 0x0C)
-
 unsigned int
 BBuGetFrameTime(
 	unsigned char byPreambleType,
diff --git a/drivers/staging/vt6655/card.c b/drivers/staging/vt6655/card.c
index dbcea4434725..e0c92818ed70 100644
--- a/drivers/staging/vt6655/card.c
+++ b/drivers/staging/vt6655/card.c
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: card.c
  * Purpose: Provide functions to setup NIC operation mode
  * Functions:
@@ -36,7 +32,7 @@
  *
  * Revision History:
  *      06-10-2003 Bryan YC Fan:  Re-write codes to support VT3253 spec.
- *      08-26-2003 Kyle Hsu:      Modify the defination type of dwIoBase.
+ *      08-26-2003 Kyle Hsu:      Modify the defination type of iobase.
  *      09-01-2003 Bryan YC Fan:  Add vUpdateIFS().
  *
  */
@@ -261,7 +257,7 @@ bool CARDbSetPhyParameter(struct vnt_private *priv, u8 bb_type)
 		BBbWriteEmbedded(priv, 0x88, 0x02);
 		bySlot = C_SLOT_LONG;
 		bySIFS = C_SIFS_BG;
-		byDIFS = C_SIFS_BG + 2*C_SLOT_LONG;
+		byDIFS = C_SIFS_BG + 2 * C_SLOT_LONG;
 		byCWMaxMin = 0xA5;
 	} else { /* PK_TYPE_11GA & PK_TYPE_11GB */
 		MACvSetBBType(priv->PortOffset, BB_TYPE_11G);
@@ -289,7 +285,7 @@ bool CARDbSetPhyParameter(struct vnt_private *priv, u8 bb_type)
 			byDIFS = C_SIFS_BG + 2 * C_SLOT_SHORT;
 		} else {
 			bySlot = C_SLOT_LONG;
-			byDIFS = C_SIFS_BG + 2*C_SLOT_LONG;
+			byDIFS = C_SIFS_BG + 2 * C_SLOT_LONG;
 		}
 
 		byCWMaxMin = 0xa4;
@@ -528,8 +524,11 @@ CARDvSafeResetTx(
 	struct vnt_tx_desc *pCurrTD;
 
 	/* initialize TD index */
-	priv->apTailTD[0] = priv->apCurrTD[0] = &(priv->apTD0Rings[0]);
-	priv->apTailTD[1] = priv->apCurrTD[1] = &(priv->apTD1Rings[0]);
+	priv->apTailTD[0] = &(priv->apTD0Rings[0]);
+	priv->apCurrTD[0] = &(priv->apTD0Rings[0]);
+
+	priv->apTailTD[1] = &(priv->apTD1Rings[0]);
+	priv->apCurrTD[1] = &(priv->apTD1Rings[0]);
 
 	for (uu = 0; uu < TYPE_MAXTD; uu++)
 		priv->iTDUsed[uu] = 0;
@@ -938,20 +937,20 @@ u64 CARDqGetTSFOffset(unsigned char byRxRate, u64 qwTSF1, u64 qwTSF2)
  */
 bool CARDbGetCurrentTSF(struct vnt_private *priv, u64 *pqwCurrTSF)
 {
-	void __iomem *dwIoBase = priv->PortOffset;
+	void __iomem *iobase = priv->PortOffset;
 	unsigned short ww;
 	unsigned char byData;
 
-	MACvRegBitsOn(dwIoBase, MAC_REG_TFTCTL, TFTCTL_TSFCNTRRD);
+	MACvRegBitsOn(iobase, MAC_REG_TFTCTL, TFTCTL_TSFCNTRRD);
 	for (ww = 0; ww < W_MAX_TIMEOUT; ww++) {
-		VNSvInPortB(dwIoBase + MAC_REG_TFTCTL, &byData);
+		VNSvInPortB(iobase + MAC_REG_TFTCTL, &byData);
 		if (!(byData & TFTCTL_TSFCNTRRD))
 			break;
 	}
 	if (ww == W_MAX_TIMEOUT)
 		return false;
-	VNSvInPortD(dwIoBase + MAC_REG_TSFCNTR, (u32 *)pqwCurrTSF);
-	VNSvInPortD(dwIoBase + MAC_REG_TSFCNTR + 4, (u32 *)pqwCurrTSF + 1);
+	VNSvInPortD(iobase + MAC_REG_TSFCNTR, (u32 *)pqwCurrTSF);
+	VNSvInPortD(iobase + MAC_REG_TSFCNTR + 4, (u32 *)pqwCurrTSF + 1);
 
 	return true;
 }
@@ -989,7 +988,7 @@ u64 CARDqGetNextTBTT(u64 qwTSF, unsigned short wBeaconInterval)
  *
  * Parameters:
  *  In:
- *      dwIoBase        - IO Base
+ *      iobase          - IO Base
  *      wBeaconInterval - Beacon Interval
  *  Out:
  *      none
@@ -999,16 +998,16 @@ u64 CARDqGetNextTBTT(u64 qwTSF, unsigned short wBeaconInterval)
 void CARDvSetFirstNextTBTT(struct vnt_private *priv,
 			   unsigned short wBeaconInterval)
 {
-	void __iomem *dwIoBase = priv->PortOffset;
+	void __iomem *iobase = priv->PortOffset;
 	u64 qwNextTBTT = 0;
 
 	CARDbGetCurrentTSF(priv, &qwNextTBTT); /* Get Local TSF counter */
 
 	qwNextTBTT = CARDqGetNextTBTT(qwNextTBTT, wBeaconInterval);
 	/* Set NextTBTT */
-	VNSvOutPortD(dwIoBase + MAC_REG_NEXTTBTT, (u32)qwNextTBTT);
-	VNSvOutPortD(dwIoBase + MAC_REG_NEXTTBTT + 4, (u32)(qwNextTBTT >> 32));
-	MACvRegBitsOn(dwIoBase, MAC_REG_TFTCTL, TFTCTL_TBTTSYNCEN);
+	VNSvOutPortD(iobase + MAC_REG_NEXTTBTT, (u32)qwNextTBTT);
+	VNSvOutPortD(iobase + MAC_REG_NEXTTBTT + 4, (u32)(qwNextTBTT >> 32));
+	MACvRegBitsOn(iobase, MAC_REG_TFTCTL, TFTCTL_TBTTSYNCEN);
 }
 
 /*
@@ -1028,12 +1027,12 @@ void CARDvSetFirstNextTBTT(struct vnt_private *priv,
 void CARDvUpdateNextTBTT(struct vnt_private *priv, u64 qwTSF,
 			 unsigned short wBeaconInterval)
 {
-	void __iomem *dwIoBase = priv->PortOffset;
+	void __iomem *iobase = priv->PortOffset;
 
 	qwTSF = CARDqGetNextTBTT(qwTSF, wBeaconInterval);
 	/* Set NextTBTT */
-	VNSvOutPortD(dwIoBase + MAC_REG_NEXTTBTT, (u32)qwTSF);
-	VNSvOutPortD(dwIoBase + MAC_REG_NEXTTBTT + 4, (u32)(qwTSF >> 32));
-	MACvRegBitsOn(dwIoBase, MAC_REG_TFTCTL, TFTCTL_TBTTSYNCEN);
+	VNSvOutPortD(iobase + MAC_REG_NEXTTBTT, (u32)qwTSF);
+	VNSvOutPortD(iobase + MAC_REG_NEXTTBTT + 4, (u32)(qwTSF >> 32));
+	MACvRegBitsOn(iobase, MAC_REG_TFTCTL, TFTCTL_TBTTSYNCEN);
 	pr_debug("Card:Update Next TBTT[%8llx]\n", qwTSF);
 }
diff --git a/drivers/staging/vt6655/card.h b/drivers/staging/vt6655/card.h
index 0203c7fd91a2..44420b5a445f 100644
--- a/drivers/staging/vt6655/card.h
+++ b/drivers/staging/vt6655/card.h
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: card.h
  *
  * Purpose: Provide functions to setup NIC operation mode
@@ -50,7 +46,7 @@
 
 #define CB_MAX_CHANNEL_24G      14
 #define CB_MAX_CHANNEL_5G       42
-#define CB_MAX_CHANNEL          (CB_MAX_CHANNEL_24G+CB_MAX_CHANNEL_5G)
+#define CB_MAX_CHANNEL          (CB_MAX_CHANNEL_24G + CB_MAX_CHANNEL_5G)
 
 typedef enum _CARD_PKT_TYPE {
 	PKT_TYPE_802_11_BCN,
diff --git a/drivers/staging/vt6655/channel.c b/drivers/staging/vt6655/channel.c
index 029a8df4ca1c..ab89956511a0 100644
--- a/drivers/staging/vt6655/channel.c
+++ b/drivers/staging/vt6655/channel.c
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: channel.c
  *
  */
diff --git a/drivers/staging/vt6655/channel.h b/drivers/staging/vt6655/channel.h
index 2d613e7f169c..2621dfabff06 100644
--- a/drivers/staging/vt6655/channel.h
+++ b/drivers/staging/vt6655/channel.h
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: channel.h
  *
  */
diff --git a/drivers/staging/vt6655/desc.h b/drivers/staging/vt6655/desc.h
index 2d7f6ae89164..2fee6e759ad8 100644
--- a/drivers/staging/vt6655/desc.h
+++ b/drivers/staging/vt6655/desc.h
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: desc.h
  *
  * Purpose:The header file of descriptor
diff --git a/drivers/staging/vt6655/device.h b/drivers/staging/vt6655/device.h
index 55405e058196..3ae40d846a09 100644
--- a/drivers/staging/vt6655/device.h
+++ b/drivers/staging/vt6655/device.h
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: device.h
  *
  * Purpose: MAC Data structure
@@ -283,12 +279,12 @@ struct vnt_private {
 	unsigned char byOFDMPwrG;
 	unsigned char byCurPwr;
 	char	 byCurPwrdBm;
-	unsigned char abyCCKPwrTbl[CB_MAX_CHANNEL_24G+1];
-	unsigned char abyOFDMPwrTbl[CB_MAX_CHANNEL+1];
-	char	abyCCKDefaultPwr[CB_MAX_CHANNEL_24G+1];
-	char	abyOFDMDefaultPwr[CB_MAX_CHANNEL+1];
-	char	abyRegPwr[CB_MAX_CHANNEL+1];
-	char	abyLocalPwr[CB_MAX_CHANNEL+1];
+	unsigned char abyCCKPwrTbl[CB_MAX_CHANNEL_24G + 1];
+	unsigned char abyOFDMPwrTbl[CB_MAX_CHANNEL + 1];
+	char	abyCCKDefaultPwr[CB_MAX_CHANNEL_24G + 1];
+	char	abyOFDMDefaultPwr[CB_MAX_CHANNEL + 1];
+	char	abyRegPwr[CB_MAX_CHANNEL + 1];
+	char	abyLocalPwr[CB_MAX_CHANNEL + 1];
 
 	/* BaseBand Loopback Use */
 	unsigned char byBBCR4d;
diff --git a/drivers/staging/vt6655/device_cfg.h b/drivers/staging/vt6655/device_cfg.h
index b4c9547d3138..0298ea923f97 100644
--- a/drivers/staging/vt6655/device_cfg.h
+++ b/drivers/staging/vt6655/device_cfg.h
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: device_cfg.h
  *
  * Purpose: Driver configuration header
diff --git a/drivers/staging/vt6655/device_main.c b/drivers/staging/vt6655/device_main.c
index f109eeac358d..da0f71191009 100644
--- a/drivers/staging/vt6655/device_main.c
+++ b/drivers/staging/vt6655/device_main.c
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: device_main.c
  *
  * Purpose: driver entry for initial, open, close, tx and rx.
@@ -314,7 +310,7 @@ static void device_init_registers(struct vnt_private *priv)
 			SROMbyReadEmbedded(priv->PortOffset,
 					   (unsigned char)(ii + EEP_OFS_CCK_PWR_TBL));
 		if (priv->abyCCKPwrTbl[ii + 1] == 0)
-			priv->abyCCKPwrTbl[ii+1] = priv->byCCKPwr;
+			priv->abyCCKPwrTbl[ii + 1] = priv->byCCKPwr;
 
 		priv->abyOFDMPwrTbl[ii + 1] =
 			SROMbyReadEmbedded(priv->PortOffset,
@@ -556,7 +552,7 @@ static void device_init_rd0_ring(struct vnt_private *priv)
 		if (!device_alloc_rx_buf(priv, desc))
 			dev_err(&priv->pcid->dev, "can not alloc rx bufs\n");
 
-		desc->next = &(priv->aRD0Ring[(i+1) % priv->opts.rx_descs0]);
+		desc->next = &(priv->aRD0Ring[(i + 1) % priv->opts.rx_descs0]);
 		desc->next_desc = cpu_to_le32(curr + sizeof(struct vnt_rx_desc));
 	}
 
@@ -1272,7 +1268,6 @@ static void vnt_remove_interface(struct ieee80211_hw *hw,
 	priv->op_mode = NL80211_IFTYPE_UNSPECIFIED;
 }
 
-
 static int vnt_config(struct ieee80211_hw *hw, u32 changed)
 {
 	struct vnt_private *priv = hw->priv;
diff --git a/drivers/staging/vt6655/dpc.c b/drivers/staging/vt6655/dpc.c
index 700032e9c477..9b3fa779258a 100644
--- a/drivers/staging/vt6655/dpc.c
+++ b/drivers/staging/vt6655/dpc.c
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: dpc.c
  *
  * Purpose: handle dpc rx functions
diff --git a/drivers/staging/vt6655/dpc.h b/drivers/staging/vt6655/dpc.h
index e80b30816968..6e75fa9c5618 100644
--- a/drivers/staging/vt6655/dpc.h
+++ b/drivers/staging/vt6655/dpc.h
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: dpc.h
  *
  * Purpose:
diff --git a/drivers/staging/vt6655/key.c b/drivers/staging/vt6655/key.c
index e161d5d9aebb..dad9e292d4da 100644
--- a/drivers/staging/vt6655/key.c
+++ b/drivers/staging/vt6655/key.c
@@ -12,11 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- *
  * File: key.c
  *
  * Purpose: Implement functions for 802.11i Key management
diff --git a/drivers/staging/vt6655/key.h b/drivers/staging/vt6655/key.h
index d72719741a56..a5024611af60 100644
--- a/drivers/staging/vt6655/key.h
+++ b/drivers/staging/vt6655/key.h
@@ -12,11 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- *
  * File: key.h
  *
  * Purpose: Implement functions for 802.11i Key management
diff --git a/drivers/staging/vt6655/mac.c b/drivers/staging/vt6655/mac.c
index 8e13f7f41415..4aaa99bafcda 100644
--- a/drivers/staging/vt6655/mac.c
+++ b/drivers/staging/vt6655/mac.c
@@ -12,11 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- *
  * File: mac.c
  *
  * Purpose:  MAC routines
@@ -147,7 +142,6 @@ void MACvSetShortRetryLimit(struct vnt_private *priv,
 	iowrite8(byRetryLimit, io_base + MAC_REG_SRT);
 }
 
-
 /*
  * Description:
  *      Set 802.11 Long Retry Limit
@@ -321,7 +315,7 @@ bool MACbSoftwareReset(struct vnt_private *priv)
  */
 bool MACbSafeSoftwareReset(struct vnt_private *priv)
 {
-	unsigned char abyTmpRegData[MAC_MAX_CONTEXT_SIZE_PAGE0+MAC_MAX_CONTEXT_SIZE_PAGE1];
+	unsigned char abyTmpRegData[MAC_MAX_CONTEXT_SIZE_PAGE0 + MAC_MAX_CONTEXT_SIZE_PAGE1];
 	bool bRetVal;
 
 	/* PATCH....
diff --git a/drivers/staging/vt6655/mac.h b/drivers/staging/vt6655/mac.h
index 030f529c339b..33b758cb79d4 100644
--- a/drivers/staging/vt6655/mac.h
+++ b/drivers/staging/vt6655/mac.h
@@ -12,11 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- *
  * File: mac.h
  *
  * Purpose: MAC routines
@@ -554,341 +549,341 @@
 
 /*---------------------  Export Macros ------------------------------*/
 
-#define MACvRegBitsOn(dwIoBase, byRegOfs, byBits)			\
+#define MACvRegBitsOn(iobase, byRegOfs, byBits)			\
 do {									\
 	unsigned char byData;						\
-	VNSvInPortB(dwIoBase + byRegOfs, &byData);			\
-	VNSvOutPortB(dwIoBase + byRegOfs, byData | (byBits));		\
+	VNSvInPortB(iobase + byRegOfs, &byData);			\
+	VNSvOutPortB(iobase + byRegOfs, byData | (byBits));		\
 } while (0)
 
-#define MACvWordRegBitsOn(dwIoBase, byRegOfs, wBits)			\
+#define MACvWordRegBitsOn(iobase, byRegOfs, wBits)			\
 do {									\
 	unsigned short wData;						\
-	VNSvInPortW(dwIoBase + byRegOfs, &wData);			\
-	VNSvOutPortW(dwIoBase + byRegOfs, wData | (wBits));		\
+	VNSvInPortW(iobase + byRegOfs, &wData);			\
+	VNSvOutPortW(iobase + byRegOfs, wData | (wBits));		\
 } while (0)
 
-#define MACvDWordRegBitsOn(dwIoBase, byRegOfs, dwBits)			\
+#define MACvDWordRegBitsOn(iobase, byRegOfs, dwBits)			\
 do {									\
 	unsigned long dwData;						\
-	VNSvInPortD(dwIoBase + byRegOfs, &dwData);			\
-	VNSvOutPortD(dwIoBase + byRegOfs, dwData | (dwBits));		\
+	VNSvInPortD(iobase + byRegOfs, &dwData);			\
+	VNSvOutPortD(iobase + byRegOfs, dwData | (dwBits));		\
 } while (0)
 
-#define MACvRegBitsOnEx(dwIoBase, byRegOfs, byMask, byBits)		\
+#define MACvRegBitsOnEx(iobase, byRegOfs, byMask, byBits)		\
 do {									\
 	unsigned char byData;						\
-	VNSvInPortB(dwIoBase + byRegOfs, &byData);			\
+	VNSvInPortB(iobase + byRegOfs, &byData);			\
 	byData &= byMask;						\
-	VNSvOutPortB(dwIoBase + byRegOfs, byData | (byBits));		\
+	VNSvOutPortB(iobase + byRegOfs, byData | (byBits));		\
 } while (0)
 
-#define MACvRegBitsOff(dwIoBase, byRegOfs, byBits)			\
+#define MACvRegBitsOff(iobase, byRegOfs, byBits)			\
 do {									\
 	unsigned char byData;						\
-	VNSvInPortB(dwIoBase + byRegOfs, &byData);			\
-	VNSvOutPortB(dwIoBase + byRegOfs, byData & ~(byBits));		\
+	VNSvInPortB(iobase + byRegOfs, &byData);			\
+	VNSvOutPortB(iobase + byRegOfs, byData & ~(byBits));		\
 } while (0)
 
-#define MACvWordRegBitsOff(dwIoBase, byRegOfs, wBits)			\
+#define MACvWordRegBitsOff(iobase, byRegOfs, wBits)			\
 do {									\
 	unsigned short wData;						\
-	VNSvInPortW(dwIoBase + byRegOfs, &wData);			\
-	VNSvOutPortW(dwIoBase + byRegOfs, wData & ~(wBits));		\
+	VNSvInPortW(iobase + byRegOfs, &wData);			\
+	VNSvOutPortW(iobase + byRegOfs, wData & ~(wBits));		\
 } while (0)
 
-#define MACvDWordRegBitsOff(dwIoBase, byRegOfs, dwBits)			\
+#define MACvDWordRegBitsOff(iobase, byRegOfs, dwBits)			\
 do {									\
 	unsigned long dwData;						\
-	VNSvInPortD(dwIoBase + byRegOfs, &dwData);			\
-	VNSvOutPortD(dwIoBase + byRegOfs, dwData & ~(dwBits));		\
+	VNSvInPortD(iobase + byRegOfs, &dwData);			\
+	VNSvOutPortD(iobase + byRegOfs, dwData & ~(dwBits));		\
 } while (0)
 
-#define MACvGetCurrRx0DescAddr(dwIoBase, pdwCurrDescAddr)	\
-	VNSvInPortD(dwIoBase + MAC_REG_RXDMAPTR0,		\
+#define MACvGetCurrRx0DescAddr(iobase, pdwCurrDescAddr)	\
+	VNSvInPortD(iobase + MAC_REG_RXDMAPTR0,		\
 		    (unsigned long *)pdwCurrDescAddr)
 
-#define MACvGetCurrRx1DescAddr(dwIoBase, pdwCurrDescAddr)	\
-	VNSvInPortD(dwIoBase + MAC_REG_RXDMAPTR1,		\
+#define MACvGetCurrRx1DescAddr(iobase, pdwCurrDescAddr)	\
+	VNSvInPortD(iobase + MAC_REG_RXDMAPTR1,		\
 		    (unsigned long *)pdwCurrDescAddr)
 
-#define MACvGetCurrTx0DescAddr(dwIoBase, pdwCurrDescAddr)	\
-	VNSvInPortD(dwIoBase + MAC_REG_TXDMAPTR0,		\
+#define MACvGetCurrTx0DescAddr(iobase, pdwCurrDescAddr)	\
+	VNSvInPortD(iobase + MAC_REG_TXDMAPTR0,		\
 		    (unsigned long *)pdwCurrDescAddr)
 
-#define MACvGetCurrAC0DescAddr(dwIoBase, pdwCurrDescAddr)	\
-	VNSvInPortD(dwIoBase + MAC_REG_AC0DMAPTR,		\
+#define MACvGetCurrAC0DescAddr(iobase, pdwCurrDescAddr)	\
+	VNSvInPortD(iobase + MAC_REG_AC0DMAPTR,		\
 		    (unsigned long *)pdwCurrDescAddr)
 
-#define MACvGetCurrSyncDescAddr(dwIoBase, pdwCurrDescAddr)	\
-	VNSvInPortD(dwIoBase + MAC_REG_SYNCDMAPTR,		\
+#define MACvGetCurrSyncDescAddr(iobase, pdwCurrDescAddr)	\
+	VNSvInPortD(iobase + MAC_REG_SYNCDMAPTR,		\
 		    (unsigned long *)pdwCurrDescAddr)
 
-#define MACvGetCurrATIMDescAddr(dwIoBase, pdwCurrDescAddr)	\
-	VNSvInPortD(dwIoBase + MAC_REG_ATIMDMAPTR,		\
+#define MACvGetCurrATIMDescAddr(iobase, pdwCurrDescAddr)	\
+	VNSvInPortD(iobase + MAC_REG_ATIMDMAPTR,		\
 		    (unsigned long *)pdwCurrDescAddr)
 
 /* set the chip with current BCN tx descriptor address */
-#define MACvSetCurrBCNTxDescAddr(dwIoBase, dwCurrDescAddr)	\
-	VNSvOutPortD(dwIoBase + MAC_REG_BCNDMAPTR,		\
+#define MACvSetCurrBCNTxDescAddr(iobase, dwCurrDescAddr)	\
+	VNSvOutPortD(iobase + MAC_REG_BCNDMAPTR,		\
 		     dwCurrDescAddr)
 
 /* set the chip with current BCN length */
-#define MACvSetCurrBCNLength(dwIoBase, wCurrBCNLength)		\
-	VNSvOutPortW(dwIoBase + MAC_REG_BCNDMACTL+2,		\
+#define MACvSetCurrBCNLength(iobase, wCurrBCNLength)		\
+	VNSvOutPortW(iobase + MAC_REG_BCNDMACTL+2,		\
 		     wCurrBCNLength)
 
-#define MACvReadBSSIDAddress(dwIoBase, pbyEtherAddr)		\
+#define MACvReadBSSIDAddress(iobase, pbyEtherAddr)		\
 do {								\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAGE1SEL, 1);		\
-	VNSvInPortB(dwIoBase + MAC_REG_BSSID0,			\
+	VNSvOutPortB(iobase + MAC_REG_PAGE1SEL, 1);		\
+	VNSvInPortB(iobase + MAC_REG_BSSID0,			\
 		    (unsigned char *)pbyEtherAddr);		\
-	VNSvInPortB(dwIoBase + MAC_REG_BSSID0 + 1,		\
+	VNSvInPortB(iobase + MAC_REG_BSSID0 + 1,		\
 		    pbyEtherAddr + 1);				\
-	VNSvInPortB(dwIoBase + MAC_REG_BSSID0 + 2,		\
+	VNSvInPortB(iobase + MAC_REG_BSSID0 + 2,		\
 		    pbyEtherAddr + 2);				\
-	VNSvInPortB(dwIoBase + MAC_REG_BSSID0 + 3,		\
+	VNSvInPortB(iobase + MAC_REG_BSSID0 + 3,		\
 		    pbyEtherAddr + 3);				\
-	VNSvInPortB(dwIoBase + MAC_REG_BSSID0 + 4,		\
+	VNSvInPortB(iobase + MAC_REG_BSSID0 + 4,		\
 		    pbyEtherAddr + 4);				\
-	VNSvInPortB(dwIoBase + MAC_REG_BSSID0 + 5,		\
+	VNSvInPortB(iobase + MAC_REG_BSSID0 + 5,		\
 		    pbyEtherAddr + 5);				\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAGE1SEL, 0);		\
+	VNSvOutPortB(iobase + MAC_REG_PAGE1SEL, 0);		\
 } while (0)
 
-#define MACvWriteBSSIDAddress(dwIoBase, pbyEtherAddr)		\
+#define MACvWriteBSSIDAddress(iobase, pbyEtherAddr)		\
 do {								\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAGE1SEL, 1);		\
-	VNSvOutPortB(dwIoBase + MAC_REG_BSSID0,			\
+	VNSvOutPortB(iobase + MAC_REG_PAGE1SEL, 1);		\
+	VNSvOutPortB(iobase + MAC_REG_BSSID0,			\
 		     *(pbyEtherAddr));				\
-	VNSvOutPortB(dwIoBase + MAC_REG_BSSID0 + 1,		\
+	VNSvOutPortB(iobase + MAC_REG_BSSID0 + 1,		\
 		     *(pbyEtherAddr + 1));			\
-	VNSvOutPortB(dwIoBase + MAC_REG_BSSID0 + 2,		\
+	VNSvOutPortB(iobase + MAC_REG_BSSID0 + 2,		\
 		     *(pbyEtherAddr + 2));			\
-	VNSvOutPortB(dwIoBase + MAC_REG_BSSID0 + 3,		\
+	VNSvOutPortB(iobase + MAC_REG_BSSID0 + 3,		\
 		     *(pbyEtherAddr + 3));			\
-	VNSvOutPortB(dwIoBase + MAC_REG_BSSID0 + 4,		\
+	VNSvOutPortB(iobase + MAC_REG_BSSID0 + 4,		\
 		     *(pbyEtherAddr + 4));			\
-	VNSvOutPortB(dwIoBase + MAC_REG_BSSID0 + 5,		\
+	VNSvOutPortB(iobase + MAC_REG_BSSID0 + 5,		\
 		     *(pbyEtherAddr + 5));			\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAGE1SEL, 0);		\
+	VNSvOutPortB(iobase + MAC_REG_PAGE1SEL, 0);		\
 } while (0)
 
-#define MACvReadEtherAddress(dwIoBase, pbyEtherAddr)		\
+#define MACvReadEtherAddress(iobase, pbyEtherAddr)		\
 do {								\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAGE1SEL, 1);		\
-	VNSvInPortB(dwIoBase + MAC_REG_PAR0,			\
+	VNSvOutPortB(iobase + MAC_REG_PAGE1SEL, 1);		\
+	VNSvInPortB(iobase + MAC_REG_PAR0,			\
 		    (unsigned char *)pbyEtherAddr);		\
-	VNSvInPortB(dwIoBase + MAC_REG_PAR0 + 1,		\
+	VNSvInPortB(iobase + MAC_REG_PAR0 + 1,		\
 		    pbyEtherAddr + 1);				\
-	VNSvInPortB(dwIoBase + MAC_REG_PAR0 + 2,		\
+	VNSvInPortB(iobase + MAC_REG_PAR0 + 2,		\
 		    pbyEtherAddr + 2);				\
-	VNSvInPortB(dwIoBase + MAC_REG_PAR0 + 3,		\
+	VNSvInPortB(iobase + MAC_REG_PAR0 + 3,		\
 		    pbyEtherAddr + 3);				\
-	VNSvInPortB(dwIoBase + MAC_REG_PAR0 + 4,		\
+	VNSvInPortB(iobase + MAC_REG_PAR0 + 4,		\
 		    pbyEtherAddr + 4);				\
-	VNSvInPortB(dwIoBase + MAC_REG_PAR0 + 5,		\
+	VNSvInPortB(iobase + MAC_REG_PAR0 + 5,		\
 		    pbyEtherAddr + 5);				\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAGE1SEL, 0);		\
+	VNSvOutPortB(iobase + MAC_REG_PAGE1SEL, 0);		\
 } while (0)
 
-#define MACvWriteEtherAddress(dwIoBase, pbyEtherAddr)		\
+#define MACvWriteEtherAddress(iobase, pbyEtherAddr)		\
 do {								\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAGE1SEL, 1);		\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAR0,			\
+	VNSvOutPortB(iobase + MAC_REG_PAGE1SEL, 1);		\
+	VNSvOutPortB(iobase + MAC_REG_PAR0,			\
 		     *pbyEtherAddr);				\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAR0 + 1,		\
+	VNSvOutPortB(iobase + MAC_REG_PAR0 + 1,		\
 		     *(pbyEtherAddr + 1));			\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAR0 + 2,		\
+	VNSvOutPortB(iobase + MAC_REG_PAR0 + 2,		\
 		     *(pbyEtherAddr + 2));			\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAR0 + 3,		\
+	VNSvOutPortB(iobase + MAC_REG_PAR0 + 3,		\
 		     *(pbyEtherAddr + 3));			\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAR0 + 4,		\
+	VNSvOutPortB(iobase + MAC_REG_PAR0 + 4,		\
 		     *(pbyEtherAddr + 4));			\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAR0 + 5,		\
+	VNSvOutPortB(iobase + MAC_REG_PAR0 + 5,		\
 		     *(pbyEtherAddr + 5));			\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAGE1SEL, 0);		\
+	VNSvOutPortB(iobase + MAC_REG_PAGE1SEL, 0);		\
 } while (0)
 
-#define MACvClearISR(dwIoBase)						\
-	VNSvOutPortD(dwIoBase + MAC_REG_ISR, IMR_MASK_VALUE)
+#define MACvClearISR(iobase)						\
+	VNSvOutPortD(iobase + MAC_REG_ISR, IMR_MASK_VALUE)
 
-#define MACvStart(dwIoBase)						\
-	VNSvOutPortB(dwIoBase + MAC_REG_HOSTCR,				\
+#define MACvStart(iobase)						\
+	VNSvOutPortB(iobase + MAC_REG_HOSTCR,				\
 		     (HOSTCR_MACEN | HOSTCR_RXON | HOSTCR_TXON))
 
-#define MACvRx0PerPktMode(dwIoBase)					\
-	VNSvOutPortD(dwIoBase + MAC_REG_RXDMACTL0, RX_PERPKT)
+#define MACvRx0PerPktMode(iobase)					\
+	VNSvOutPortD(iobase + MAC_REG_RXDMACTL0, RX_PERPKT)
 
-#define MACvRx0BufferFillMode(dwIoBase)					\
-	VNSvOutPortD(dwIoBase + MAC_REG_RXDMACTL0, RX_PERPKTCLR)
+#define MACvRx0BufferFillMode(iobase)					\
+	VNSvOutPortD(iobase + MAC_REG_RXDMACTL0, RX_PERPKTCLR)
 
-#define MACvRx1PerPktMode(dwIoBase)					\
-	VNSvOutPortD(dwIoBase + MAC_REG_RXDMACTL1, RX_PERPKT)
+#define MACvRx1PerPktMode(iobase)					\
+	VNSvOutPortD(iobase + MAC_REG_RXDMACTL1, RX_PERPKT)
 
-#define MACvRx1BufferFillMode(dwIoBase)					\
-	VNSvOutPortD(dwIoBase + MAC_REG_RXDMACTL1, RX_PERPKTCLR)
+#define MACvRx1BufferFillMode(iobase)					\
+	VNSvOutPortD(iobase + MAC_REG_RXDMACTL1, RX_PERPKTCLR)
 
-#define MACvRxOn(dwIoBase)						\
-	MACvRegBitsOn(dwIoBase, MAC_REG_HOSTCR, HOSTCR_RXON)
+#define MACvRxOn(iobase)						\
+	MACvRegBitsOn(iobase, MAC_REG_HOSTCR, HOSTCR_RXON)
 
-#define MACvReceive0(dwIoBase)						\
+#define MACvReceive0(iobase)						\
 do {									\
 	unsigned long dwData;						\
-	VNSvInPortD(dwIoBase + MAC_REG_RXDMACTL0, &dwData);		\
+	VNSvInPortD(iobase + MAC_REG_RXDMACTL0, &dwData);		\
 	if (dwData & DMACTL_RUN)					\
-		VNSvOutPortD(dwIoBase + MAC_REG_RXDMACTL0, DMACTL_WAKE); \
+		VNSvOutPortD(iobase + MAC_REG_RXDMACTL0, DMACTL_WAKE); \
 	else								\
-		VNSvOutPortD(dwIoBase + MAC_REG_RXDMACTL0, DMACTL_RUN); \
+		VNSvOutPortD(iobase + MAC_REG_RXDMACTL0, DMACTL_RUN); \
 } while (0)
 
-#define MACvReceive1(dwIoBase)						\
+#define MACvReceive1(iobase)						\
 do {									\
 	unsigned long dwData;						\
-	VNSvInPortD(dwIoBase + MAC_REG_RXDMACTL1, &dwData);		\
+	VNSvInPortD(iobase + MAC_REG_RXDMACTL1, &dwData);		\
 	if (dwData & DMACTL_RUN)					\
-		VNSvOutPortD(dwIoBase + MAC_REG_RXDMACTL1, DMACTL_WAKE); \
+		VNSvOutPortD(iobase + MAC_REG_RXDMACTL1, DMACTL_WAKE); \
 	else								\
-		VNSvOutPortD(dwIoBase + MAC_REG_RXDMACTL1, DMACTL_RUN); \
+		VNSvOutPortD(iobase + MAC_REG_RXDMACTL1, DMACTL_RUN); \
 } while (0)
 
-#define MACvTxOn(dwIoBase)						\
-	MACvRegBitsOn(dwIoBase, MAC_REG_HOSTCR, HOSTCR_TXON)
+#define MACvTxOn(iobase)						\
+	MACvRegBitsOn(iobase, MAC_REG_HOSTCR, HOSTCR_TXON)
 
-#define MACvTransmit0(dwIoBase)						\
+#define MACvTransmit0(iobase)						\
 do {									\
 	unsigned long dwData;						\
-	VNSvInPortD(dwIoBase + MAC_REG_TXDMACTL0, &dwData);		\
+	VNSvInPortD(iobase + MAC_REG_TXDMACTL0, &dwData);		\
 	if (dwData & DMACTL_RUN)					\
-		VNSvOutPortD(dwIoBase + MAC_REG_TXDMACTL0, DMACTL_WAKE); \
+		VNSvOutPortD(iobase + MAC_REG_TXDMACTL0, DMACTL_WAKE); \
 	else								\
-		VNSvOutPortD(dwIoBase + MAC_REG_TXDMACTL0, DMACTL_RUN); \
+		VNSvOutPortD(iobase + MAC_REG_TXDMACTL0, DMACTL_RUN); \
 } while (0)
 
-#define MACvTransmitAC0(dwIoBase)					\
+#define MACvTransmitAC0(iobase)					\
 do {									\
 	unsigned long dwData;						\
-	VNSvInPortD(dwIoBase + MAC_REG_AC0DMACTL, &dwData);		\
+	VNSvInPortD(iobase + MAC_REG_AC0DMACTL, &dwData);		\
 	if (dwData & DMACTL_RUN)					\
-		VNSvOutPortD(dwIoBase + MAC_REG_AC0DMACTL, DMACTL_WAKE); \
+		VNSvOutPortD(iobase + MAC_REG_AC0DMACTL, DMACTL_WAKE); \
 	else								\
-		VNSvOutPortD(dwIoBase + MAC_REG_AC0DMACTL, DMACTL_RUN); \
+		VNSvOutPortD(iobase + MAC_REG_AC0DMACTL, DMACTL_RUN); \
 } while (0)
 
-#define MACvTransmitSYNC(dwIoBase)					\
+#define MACvTransmitSYNC(iobase)					\
 do {									\
 	unsigned long dwData;						\
-	VNSvInPortD(dwIoBase + MAC_REG_SYNCDMACTL, &dwData);		\
+	VNSvInPortD(iobase + MAC_REG_SYNCDMACTL, &dwData);		\
 	if (dwData & DMACTL_RUN)					\
-		VNSvOutPortD(dwIoBase + MAC_REG_SYNCDMACTL, DMACTL_WAKE); \
+		VNSvOutPortD(iobase + MAC_REG_SYNCDMACTL, DMACTL_WAKE); \
 	else								\
-		VNSvOutPortD(dwIoBase + MAC_REG_SYNCDMACTL, DMACTL_RUN); \
+		VNSvOutPortD(iobase + MAC_REG_SYNCDMACTL, DMACTL_RUN); \
 } while (0)
 
-#define MACvTransmitATIM(dwIoBase)					\
+#define MACvTransmitATIM(iobase)					\
 do {									\
 	unsigned long dwData;						\
-	VNSvInPortD(dwIoBase + MAC_REG_ATIMDMACTL, &dwData);		\
+	VNSvInPortD(iobase + MAC_REG_ATIMDMACTL, &dwData);		\
 	if (dwData & DMACTL_RUN)					\
-		VNSvOutPortD(dwIoBase + MAC_REG_ATIMDMACTL, DMACTL_WAKE); \
+		VNSvOutPortD(iobase + MAC_REG_ATIMDMACTL, DMACTL_WAKE); \
 	else								\
-		VNSvOutPortD(dwIoBase + MAC_REG_ATIMDMACTL, DMACTL_RUN); \
+		VNSvOutPortD(iobase + MAC_REG_ATIMDMACTL, DMACTL_RUN); \
 } while (0)
 
-#define MACvTransmitBCN(dwIoBase)					\
-	VNSvOutPortB(dwIoBase + MAC_REG_BCNDMACTL, BEACON_READY)
+#define MACvTransmitBCN(iobase)					\
+	VNSvOutPortB(iobase + MAC_REG_BCNDMACTL, BEACON_READY)
 
-#define MACvClearStckDS(dwIoBase)					\
+#define MACvClearStckDS(iobase)					\
 do {									\
 	unsigned char byOrgValue;					\
-	VNSvInPortB(dwIoBase + MAC_REG_STICKHW, &byOrgValue);		\
+	VNSvInPortB(iobase + MAC_REG_STICKHW, &byOrgValue);		\
 	byOrgValue = byOrgValue & 0xFC;					\
-	VNSvOutPortB(dwIoBase + MAC_REG_STICKHW, byOrgValue);		\
+	VNSvOutPortB(iobase + MAC_REG_STICKHW, byOrgValue);		\
 } while (0)
 
-#define MACvReadISR(dwIoBase, pdwValue)				\
-	VNSvInPortD(dwIoBase + MAC_REG_ISR, pdwValue)
+#define MACvReadISR(iobase, pdwValue)				\
+	VNSvInPortD(iobase + MAC_REG_ISR, pdwValue)
 
-#define MACvWriteISR(dwIoBase, dwValue)				\
-	VNSvOutPortD(dwIoBase + MAC_REG_ISR, dwValue)
+#define MACvWriteISR(iobase, dwValue)				\
+	VNSvOutPortD(iobase + MAC_REG_ISR, dwValue)
 
-#define MACvIntEnable(dwIoBase, dwMask)				\
-	VNSvOutPortD(dwIoBase + MAC_REG_IMR, dwMask)
+#define MACvIntEnable(iobase, dwMask)				\
+	VNSvOutPortD(iobase + MAC_REG_IMR, dwMask)
 
-#define MACvIntDisable(dwIoBase)				\
-	VNSvOutPortD(dwIoBase + MAC_REG_IMR, 0)
+#define MACvIntDisable(iobase)				\
+	VNSvOutPortD(iobase + MAC_REG_IMR, 0)
 
-#define MACvSelectPage0(dwIoBase)				\
-		VNSvOutPortB(dwIoBase + MAC_REG_PAGE1SEL, 0)
+#define MACvSelectPage0(iobase)				\
+		VNSvOutPortB(iobase + MAC_REG_PAGE1SEL, 0)
 
-#define MACvSelectPage1(dwIoBase)				\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAGE1SEL, 1)
+#define MACvSelectPage1(iobase)				\
+	VNSvOutPortB(iobase + MAC_REG_PAGE1SEL, 1)
 
-#define MACvReadMIBCounter(dwIoBase, pdwCounter)			\
-	VNSvInPortD(dwIoBase + MAC_REG_MIBCNTR, pdwCounter)
+#define MACvReadMIBCounter(iobase, pdwCounter)			\
+	VNSvInPortD(iobase + MAC_REG_MIBCNTR, pdwCounter)
 
-#define MACvPwrEvntDisable(dwIoBase)					\
-	VNSvOutPortW(dwIoBase + MAC_REG_WAKEUPEN0, 0x0000)
+#define MACvPwrEvntDisable(iobase)					\
+	VNSvOutPortW(iobase + MAC_REG_WAKEUPEN0, 0x0000)
 
-#define MACvEnableProtectMD(dwIoBase)					\
+#define MACvEnableProtectMD(iobase)					\
 do {									\
 	unsigned long dwOrgValue;					\
-	VNSvInPortD(dwIoBase + MAC_REG_ENCFG, &dwOrgValue);		\
+	VNSvInPortD(iobase + MAC_REG_ENCFG, &dwOrgValue);		\
 	dwOrgValue = dwOrgValue | EnCFG_ProtectMd;			\
-	VNSvOutPortD(dwIoBase + MAC_REG_ENCFG, dwOrgValue);		\
+	VNSvOutPortD(iobase + MAC_REG_ENCFG, dwOrgValue);		\
 } while (0)
 
-#define MACvDisableProtectMD(dwIoBase)					\
+#define MACvDisableProtectMD(iobase)					\
 do {									\
 	unsigned long dwOrgValue;					\
-	VNSvInPortD(dwIoBase + MAC_REG_ENCFG, &dwOrgValue);		\
+	VNSvInPortD(iobase + MAC_REG_ENCFG, &dwOrgValue);		\
 	dwOrgValue = dwOrgValue & ~EnCFG_ProtectMd;			\
-	VNSvOutPortD(dwIoBase + MAC_REG_ENCFG, dwOrgValue);		\
+	VNSvOutPortD(iobase + MAC_REG_ENCFG, dwOrgValue);		\
 } while (0)
 
-#define MACvEnableBarkerPreambleMd(dwIoBase)				\
+#define MACvEnableBarkerPreambleMd(iobase)				\
 do {									\
 	unsigned long dwOrgValue;					\
-	VNSvInPortD(dwIoBase + MAC_REG_ENCFG, &dwOrgValue);		\
+	VNSvInPortD(iobase + MAC_REG_ENCFG, &dwOrgValue);		\
 	dwOrgValue = dwOrgValue | EnCFG_BarkerPream;			\
-	VNSvOutPortD(dwIoBase + MAC_REG_ENCFG, dwOrgValue);		\
+	VNSvOutPortD(iobase + MAC_REG_ENCFG, dwOrgValue);		\
 } while (0)
 
-#define MACvDisableBarkerPreambleMd(dwIoBase)				\
+#define MACvDisableBarkerPreambleMd(iobase)				\
 do {									\
 	unsigned long dwOrgValue;					\
-	VNSvInPortD(dwIoBase + MAC_REG_ENCFG, &dwOrgValue);		\
+	VNSvInPortD(iobase + MAC_REG_ENCFG, &dwOrgValue);		\
 	dwOrgValue = dwOrgValue & ~EnCFG_BarkerPream;			\
-	VNSvOutPortD(dwIoBase + MAC_REG_ENCFG, dwOrgValue);		\
+	VNSvOutPortD(iobase + MAC_REG_ENCFG, dwOrgValue);		\
 } while (0)
 
-#define MACvSetBBType(dwIoBase, byTyp)					\
+#define MACvSetBBType(iobase, byTyp)					\
 do {									\
 	unsigned long dwOrgValue;					\
-	VNSvInPortD(dwIoBase + MAC_REG_ENCFG, &dwOrgValue);		\
+	VNSvInPortD(iobase + MAC_REG_ENCFG, &dwOrgValue);		\
 	dwOrgValue = dwOrgValue & ~EnCFG_BBType_MASK;			\
 	dwOrgValue = dwOrgValue | (unsigned long)byTyp;			\
-	VNSvOutPortD(dwIoBase + MAC_REG_ENCFG, dwOrgValue);		\
+	VNSvOutPortD(iobase + MAC_REG_ENCFG, dwOrgValue);		\
 } while (0)
 
-#define MACvReadATIMW(dwIoBase, pwCounter)				\
-	VNSvInPortW(dwIoBase + MAC_REG_AIDATIM, pwCounter)
+#define MACvReadATIMW(iobase, pwCounter)				\
+	VNSvInPortW(iobase + MAC_REG_AIDATIM, pwCounter)
 
-#define MACvWriteATIMW(dwIoBase, wCounter)				\
-	VNSvOutPortW(dwIoBase + MAC_REG_AIDATIM, wCounter)
+#define MACvWriteATIMW(iobase, wCounter)				\
+	VNSvOutPortW(iobase + MAC_REG_AIDATIM, wCounter)
 
-#define MACvWriteCRC16_128(dwIoBase, byRegOfs, wCRC)		\
+#define MACvWriteCRC16_128(iobase, byRegOfs, wCRC)		\
 do {								\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAGE1SEL, 1);		\
-	VNSvOutPortW(dwIoBase + byRegOfs, wCRC);		\
-	VNSvOutPortB(dwIoBase + MAC_REG_PAGE1SEL, 0);		\
+	VNSvOutPortB(iobase + MAC_REG_PAGE1SEL, 1);		\
+	VNSvOutPortW(iobase + byRegOfs, wCRC);		\
+	VNSvOutPortB(iobase + MAC_REG_PAGE1SEL, 0);		\
 } while (0)
 
-#define MACvGPIOIn(dwIoBase, pbyValue)					\
-	VNSvInPortB(dwIoBase + MAC_REG_GPIOCTL1, pbyValue)
+#define MACvGPIOIn(iobase, pbyValue)					\
+	VNSvInPortB(iobase + MAC_REG_GPIOCTL1, pbyValue)
 
-#define MACvSetRFLE_LatchBase(dwIoBase)                                 \
-	MACvWordRegBitsOn(dwIoBase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_RFLEOPT)
+#define MACvSetRFLE_LatchBase(iobase)                                 \
+	MACvWordRegBitsOn(iobase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_RFLEOPT)
 
 bool MACbIsRegBitsOn(struct vnt_private *, unsigned char byRegOfs,
 		     unsigned char byTestBits);
diff --git a/drivers/staging/vt6655/power.c b/drivers/staging/vt6655/power.c
index 7d6e7464ae51..716d2a80f840 100644
--- a/drivers/staging/vt6655/power.c
+++ b/drivers/staging/vt6655/power.c
@@ -12,11 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- *
  * File: power.c
  *
  * Purpose: Handles 802.11 power management  functions
@@ -133,7 +128,6 @@ PSvDisablePowerSaving(
 	priv->bPWBitOn = false;
 }
 
-
 /*
  *
  * Routine Description:
diff --git a/drivers/staging/vt6655/power.h b/drivers/staging/vt6655/power.h
index d82dd8d6d68b..dfcb0ca8b448 100644
--- a/drivers/staging/vt6655/power.h
+++ b/drivers/staging/vt6655/power.h
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: power.h
  *
  * Purpose: Handles 802.11 power management  functions
@@ -46,7 +42,6 @@ PSvEnablePowerSaving(
 	unsigned short wListenInterval
 );
 
-
 bool
 PSbIsNextTBTTWakeUp(
 	struct vnt_private *
diff --git a/drivers/staging/vt6655/rf.c b/drivers/staging/vt6655/rf.c
index 447882c7a6be..edf7db9d53b3 100644
--- a/drivers/staging/vt6655/rf.c
+++ b/drivers/staging/vt6655/rf.c
@@ -12,11 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- *
  * File: rf.c
  *
  * Purpose: rf function code
@@ -50,359 +45,362 @@
 #define AL7230_PWR_IDX_LEN    64
 
 static const unsigned long dwAL2230InitTable[CB_AL2230_INIT_SEQ] = {
-	0x03F79000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x03333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x01A00200+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x00FFF300+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0005A400+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0F4DC500+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0805B600+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0146C700+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x00068800+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0403B900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x00DBBA00+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x00099B00+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0BDFFC00+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x00000D00+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x00580F00+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW
+	0x03F79000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x03333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x01A00200 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x00FFF300 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0005A400 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0F4DC500 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0805B600 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0146C700 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x00068800 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0403B900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x00DBBA00 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x00099B00 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0BDFFC00 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x00000D00 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x00580F00 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW
 };
 
 static const unsigned long dwAL2230ChannelTable0[CB_MAX_CHANNEL] = {
-	0x03F79000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 1, Tf = 2412MHz */
-	0x03F79000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 2, Tf = 2417MHz */
-	0x03E79000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 3, Tf = 2422MHz */
-	0x03E79000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 4, Tf = 2427MHz */
-	0x03F7A000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 5, Tf = 2432MHz */
-	0x03F7A000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 6, Tf = 2437MHz */
-	0x03E7A000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 7, Tf = 2442MHz */
-	0x03E7A000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 8, Tf = 2447MHz */
-	0x03F7B000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 9, Tf = 2452MHz */
-	0x03F7B000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 10, Tf = 2457MHz */
-	0x03E7B000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 11, Tf = 2462MHz */
-	0x03E7B000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 12, Tf = 2467MHz */
-	0x03F7C000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 13, Tf = 2472MHz */
-	0x03E7C000+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW  /* channel = 14, Tf = 2412M */
+	0x03F79000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 1, Tf = 2412MHz */
+	0x03F79000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 2, Tf = 2417MHz */
+	0x03E79000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 3, Tf = 2422MHz */
+	0x03E79000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 4, Tf = 2427MHz */
+	0x03F7A000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 5, Tf = 2432MHz */
+	0x03F7A000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 6, Tf = 2437MHz */
+	0x03E7A000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 7, Tf = 2442MHz */
+	0x03E7A000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 8, Tf = 2447MHz */
+	0x03F7B000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 9, Tf = 2452MHz */
+	0x03F7B000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 10, Tf = 2457MHz */
+	0x03E7B000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 11, Tf = 2462MHz */
+	0x03E7B000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 12, Tf = 2467MHz */
+	0x03F7C000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 13, Tf = 2472MHz */
+	0x03E7C000 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW  /* channel = 14, Tf = 2412M */
 };
 
 static const unsigned long dwAL2230ChannelTable1[CB_MAX_CHANNEL] = {
-	0x03333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 1, Tf = 2412MHz */
-	0x0B333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 2, Tf = 2417MHz */
-	0x03333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 3, Tf = 2422MHz */
-	0x0B333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 4, Tf = 2427MHz */
-	0x03333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 5, Tf = 2432MHz */
-	0x0B333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 6, Tf = 2437MHz */
-	0x03333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 7, Tf = 2442MHz */
-	0x0B333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 8, Tf = 2447MHz */
-	0x03333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 9, Tf = 2452MHz */
-	0x0B333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 10, Tf = 2457MHz */
-	0x03333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 11, Tf = 2462MHz */
-	0x0B333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 12, Tf = 2467MHz */
-	0x03333100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 13, Tf = 2472MHz */
-	0x06666100+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW  /* channel = 14, Tf = 2412M */
+	0x03333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 1, Tf = 2412MHz */
+	0x0B333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 2, Tf = 2417MHz */
+	0x03333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 3, Tf = 2422MHz */
+	0x0B333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 4, Tf = 2427MHz */
+	0x03333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 5, Tf = 2432MHz */
+	0x0B333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 6, Tf = 2437MHz */
+	0x03333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 7, Tf = 2442MHz */
+	0x0B333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 8, Tf = 2447MHz */
+	0x03333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 9, Tf = 2452MHz */
+	0x0B333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 10, Tf = 2457MHz */
+	0x03333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 11, Tf = 2462MHz */
+	0x0B333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 12, Tf = 2467MHz */
+	0x03333100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 13, Tf = 2472MHz */
+	0x06666100 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW  /* channel = 14, Tf = 2412M */
 };
 
 static unsigned long dwAL2230PowerTable[AL2230_PWR_IDX_LEN] = {
-	0x04040900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04041900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04042900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04043900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04044900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04045900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04046900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04047900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04048900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04049900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0404A900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0404B900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0404C900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0404D900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0404E900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0404F900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04050900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04051900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04052900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04053900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04054900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04055900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04056900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04057900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04058900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04059900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0405A900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0405B900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0405C900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0405D900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0405E900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0405F900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04060900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04061900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04062900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04063900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04064900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04065900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04066900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04067900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04068900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04069900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0406A900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0406B900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0406C900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0406D900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0406E900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0406F900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04070900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04071900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04072900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04073900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04074900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04075900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04076900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04077900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04078900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x04079900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0407A900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0407B900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0407C900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0407D900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0407E900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x0407F900+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW
+	0x04040900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04041900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04042900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04043900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04044900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04045900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04046900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04047900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04048900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04049900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0404A900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0404B900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0404C900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0404D900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0404E900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0404F900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04050900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04051900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04052900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04053900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04054900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04055900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04056900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04057900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04058900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04059900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0405A900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0405B900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0405C900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0405D900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0405E900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0405F900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04060900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04061900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04062900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04063900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04064900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04065900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04066900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04067900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04068900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04069900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0406A900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0406B900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0406C900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0406D900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0406E900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0406F900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04070900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04071900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04072900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04073900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04074900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04075900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04076900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04077900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04078900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x04079900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0407A900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0407B900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0407C900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0407D900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0407E900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x0407F900 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW
 };
 
 /* 40MHz reference frequency
  * Need to Pull PLLON(PE3) low when writing channel registers through 3-wire.
  */
 static const unsigned long dwAL7230InitTable[CB_AL7230_INIT_SEQ] = {
-	0x00379000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Channel1 // Need modify for 11a */
-	0x13333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Channel1 // Need modify for 11a */
-	0x841FF200+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Need modify for 11a: 451FE2 */
-	0x3FDFA300+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Need modify for 11a: 5FDFA3 */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* 11b/g    // Need modify for 11a */
+	0x00379000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Channel1 // Need modify for 11a */
+	0x13333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Channel1 // Need modify for 11a */
+	0x841FF200 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Need modify for 11a: 451FE2 */
+	0x3FDFA300 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Need modify for 11a: 5FDFA3 */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* 11b/g    // Need modify for 11a */
 	/* RoberYu:20050113, Rev0.47 Regsiter Setting Guide */
-	0x802B5500+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Need modify for 11a: 8D1B55 */
-	0x56AF3600+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW,
-	0xCE020700+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Need modify for 11a: 860207 */
-	0x6EBC0800+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x221BB900+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW,
-	0xE0000A00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Need modify for 11a: E0600A */
-	0x08031B00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* init 0x080B1B00 => 0x080F1B00 for 3 wire control TxGain(D10) */
+	0x802B5500 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Need modify for 11a: 8D1B55 */
+	0x56AF3600 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW,
+	0xCE020700 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Need modify for 11a: 860207 */
+	0x6EBC0800 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x221BB900 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW,
+	0xE0000A00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Need modify for 11a: E0600A */
+	0x08031B00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* init 0x080B1B00 => 0x080F1B00 for 3 wire control TxGain(D10) */
 	/* RoberYu:20050113, Rev0.47 Regsiter Setting Guide */
-	0x000A3C00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Need modify for 11a: 00143C */
-	0xFFFFFD00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x00000E00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x1ABA8F00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW  /* Need modify for 11a: 12BACF */
+	0x000A3C00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Need modify for 11a: 00143C */
+	0xFFFFFD00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x00000E00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x1ABA8F00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW  /* Need modify for 11a: 12BACF */
 };
 
 static const unsigned long dwAL7230InitTableAMode[CB_AL7230_INIT_SEQ] = {
-	0x0FF52000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Channel184 // Need modify for 11b/g */
-	0x00000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Channel184 // Need modify for 11b/g */
-	0x451FE200+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Need modify for 11b/g */
-	0x5FDFA300+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Need modify for 11b/g */
-	0x67F78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* 11a    // Need modify for 11b/g */
-	0x853F5500+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Need modify for 11b/g, RoberYu:20050113 */
-	0x56AF3600+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW,
-	0xCE020700+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Need modify for 11b/g */
-	0x6EBC0800+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x221BB900+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW,
-	0xE0600A00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Need modify for 11b/g */
-	0x08031B00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* init 0x080B1B00 => 0x080F1B00 for 3 wire control TxGain(D10) */
-	0x00147C00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* Need modify for 11b/g */
-	0xFFFFFD00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x00000E00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW,
-	0x12BACF00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW  /* Need modify for 11b/g */
+	0x0FF52000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Channel184 // Need modify for 11b/g */
+	0x00000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Channel184 // Need modify for 11b/g */
+	0x451FE200 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Need modify for 11b/g */
+	0x5FDFA300 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Need modify for 11b/g */
+	0x67F78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* 11a    // Need modify for 11b/g */
+	0x853F5500 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Need modify for 11b/g, RoberYu:20050113 */
+	0x56AF3600 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW,
+	0xCE020700 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Need modify for 11b/g */
+	0x6EBC0800 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x221BB900 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW,
+	0xE0600A00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Need modify for 11b/g */
+	0x08031B00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* init 0x080B1B00 => 0x080F1B00 for 3 wire control TxGain(D10) */
+	0x00147C00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* Need modify for 11b/g */
+	0xFFFFFD00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x00000E00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW,
+	0x12BACF00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW  /* Need modify for 11b/g */
 };
 
 static const unsigned long dwAL7230ChannelTable0[CB_MAX_CHANNEL] = {
-	0x00379000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  1, Tf = 2412MHz */
-	0x00379000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  2, Tf = 2417MHz */
-	0x00379000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  3, Tf = 2422MHz */
-	0x00379000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  4, Tf = 2427MHz */
-	0x0037A000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  5, Tf = 2432MHz */
-	0x0037A000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  6, Tf = 2437MHz */
-	0x0037A000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  7, Tf = 2442MHz */
-	0x0037A000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  8, Tf = 2447MHz //RobertYu: 20050218, update for APNode 0.49 */
-	0x0037B000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  9, Tf = 2452MHz //RobertYu: 20050218, update for APNode 0.49 */
-	0x0037B000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 10, Tf = 2457MHz //RobertYu: 20050218, update for APNode 0.49 */
-	0x0037B000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 11, Tf = 2462MHz //RobertYu: 20050218, update for APNode 0.49 */
-	0x0037B000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 12, Tf = 2467MHz //RobertYu: 20050218, update for APNode 0.49 */
-	0x0037C000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 13, Tf = 2472MHz //RobertYu: 20050218, update for APNode 0.49 */
-	0x0037C000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 14, Tf = 2484MHz */
+	0x00379000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  1, Tf = 2412MHz */
+	0x00379000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  2, Tf = 2417MHz */
+	0x00379000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  3, Tf = 2422MHz */
+	0x00379000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  4, Tf = 2427MHz */
+	0x0037A000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  5, Tf = 2432MHz */
+	0x0037A000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  6, Tf = 2437MHz */
+	0x0037A000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  7, Tf = 2442MHz */
+	0x0037A000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  8, Tf = 2447MHz //RobertYu: 20050218, update for APNode 0.49 */
+	0x0037B000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  9, Tf = 2452MHz //RobertYu: 20050218, update for APNode 0.49 */
+	0x0037B000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 10, Tf = 2457MHz //RobertYu: 20050218, update for APNode 0.49 */
+	0x0037B000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 11, Tf = 2462MHz //RobertYu: 20050218, update for APNode 0.49 */
+	0x0037B000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 12, Tf = 2467MHz //RobertYu: 20050218, update for APNode 0.49 */
+	0x0037C000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 13, Tf = 2472MHz //RobertYu: 20050218, update for APNode 0.49 */
+	0x0037C000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 14, Tf = 2484MHz */
 
 	/* 4.9G => Ch 183, 184, 185, 187, 188, 189, 192, 196  (Value:15 ~ 22) */
-	0x0FF52000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 183, Tf = 4915MHz (15) */
-	0x0FF52000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 184, Tf = 4920MHz (16) */
-	0x0FF52000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 185, Tf = 4925MHz (17) */
-	0x0FF52000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 187, Tf = 4935MHz (18) */
-	0x0FF52000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 188, Tf = 4940MHz (19) */
-	0x0FF52000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 189, Tf = 4945MHz (20) */
-	0x0FF53000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 192, Tf = 4960MHz (21) */
-	0x0FF53000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 196, Tf = 4980MHz (22) */
+	0x0FF52000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 183, Tf = 4915MHz (15) */
+	0x0FF52000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 184, Tf = 4920MHz (16) */
+	0x0FF52000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 185, Tf = 4925MHz (17) */
+	0x0FF52000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 187, Tf = 4935MHz (18) */
+	0x0FF52000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 188, Tf = 4940MHz (19) */
+	0x0FF52000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 189, Tf = 4945MHz (20) */
+	0x0FF53000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 192, Tf = 4960MHz (21) */
+	0x0FF53000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 196, Tf = 4980MHz (22) */
 
 	/* 5G => Ch 7, 8, 9, 11, 12, 16, 34, 36, 38, 40, 42, 44, 46, 48, 52, 56, 60, 64,
-	 * 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165  (Value 23 ~ 56) */
-
-	0x0FF54000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =   7, Tf = 5035MHz (23) */
-	0x0FF54000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =   8, Tf = 5040MHz (24) */
-	0x0FF54000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =   9, Tf = 5045MHz (25) */
-	0x0FF54000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  11, Tf = 5055MHz (26) */
-	0x0FF54000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  12, Tf = 5060MHz (27) */
-	0x0FF55000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  16, Tf = 5080MHz (28) */
-	0x0FF56000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  34, Tf = 5170MHz (29) */
-	0x0FF56000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  36, Tf = 5180MHz (30) */
-	0x0FF57000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  38, Tf = 5190MHz (31) //RobertYu: 20050218, update for APNode 0.49 */
-	0x0FF57000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  40, Tf = 5200MHz (32) */
-	0x0FF57000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  42, Tf = 5210MHz (33) */
-	0x0FF57000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  44, Tf = 5220MHz (34) */
-	0x0FF57000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  46, Tf = 5230MHz (35) */
-	0x0FF57000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  48, Tf = 5240MHz (36) */
-	0x0FF58000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  52, Tf = 5260MHz (37) */
-	0x0FF58000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  56, Tf = 5280MHz (38) */
-	0x0FF58000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  60, Tf = 5300MHz (39) */
-	0x0FF59000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  64, Tf = 5320MHz (40) */
-
-	0x0FF5C000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 100, Tf = 5500MHz (41) */
-	0x0FF5C000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 104, Tf = 5520MHz (42) */
-	0x0FF5C000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 108, Tf = 5540MHz (43) */
-	0x0FF5D000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 112, Tf = 5560MHz (44) */
-	0x0FF5D000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 116, Tf = 5580MHz (45) */
-	0x0FF5D000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 120, Tf = 5600MHz (46) */
-	0x0FF5E000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 124, Tf = 5620MHz (47) */
-	0x0FF5E000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 128, Tf = 5640MHz (48) */
-	0x0FF5E000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 132, Tf = 5660MHz (49) */
-	0x0FF5F000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 136, Tf = 5680MHz (50) */
-	0x0FF5F000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 140, Tf = 5700MHz (51) */
-	0x0FF60000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 149, Tf = 5745MHz (52) */
-	0x0FF60000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 153, Tf = 5765MHz (53) */
-	0x0FF60000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 157, Tf = 5785MHz (54) */
-	0x0FF61000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 161, Tf = 5805MHz (55) */
-	0x0FF61000+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW  /* channel = 165, Tf = 5825MHz (56) */
+	 * 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165  (Value 23 ~ 56)
+	 */
+
+	0x0FF54000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =   7, Tf = 5035MHz (23) */
+	0x0FF54000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =   8, Tf = 5040MHz (24) */
+	0x0FF54000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =   9, Tf = 5045MHz (25) */
+	0x0FF54000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  11, Tf = 5055MHz (26) */
+	0x0FF54000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  12, Tf = 5060MHz (27) */
+	0x0FF55000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  16, Tf = 5080MHz (28) */
+	0x0FF56000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  34, Tf = 5170MHz (29) */
+	0x0FF56000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  36, Tf = 5180MHz (30) */
+	0x0FF57000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  38, Tf = 5190MHz (31) //RobertYu: 20050218, update for APNode 0.49 */
+	0x0FF57000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  40, Tf = 5200MHz (32) */
+	0x0FF57000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  42, Tf = 5210MHz (33) */
+	0x0FF57000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  44, Tf = 5220MHz (34) */
+	0x0FF57000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  46, Tf = 5230MHz (35) */
+	0x0FF57000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  48, Tf = 5240MHz (36) */
+	0x0FF58000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  52, Tf = 5260MHz (37) */
+	0x0FF58000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  56, Tf = 5280MHz (38) */
+	0x0FF58000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  60, Tf = 5300MHz (39) */
+	0x0FF59000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  64, Tf = 5320MHz (40) */
+
+	0x0FF5C000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 100, Tf = 5500MHz (41) */
+	0x0FF5C000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 104, Tf = 5520MHz (42) */
+	0x0FF5C000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 108, Tf = 5540MHz (43) */
+	0x0FF5D000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 112, Tf = 5560MHz (44) */
+	0x0FF5D000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 116, Tf = 5580MHz (45) */
+	0x0FF5D000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 120, Tf = 5600MHz (46) */
+	0x0FF5E000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 124, Tf = 5620MHz (47) */
+	0x0FF5E000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 128, Tf = 5640MHz (48) */
+	0x0FF5E000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 132, Tf = 5660MHz (49) */
+	0x0FF5F000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 136, Tf = 5680MHz (50) */
+	0x0FF5F000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 140, Tf = 5700MHz (51) */
+	0x0FF60000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 149, Tf = 5745MHz (52) */
+	0x0FF60000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 153, Tf = 5765MHz (53) */
+	0x0FF60000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 157, Tf = 5785MHz (54) */
+	0x0FF61000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 161, Tf = 5805MHz (55) */
+	0x0FF61000 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW  /* channel = 165, Tf = 5825MHz (56) */
 };
 
 static const unsigned long dwAL7230ChannelTable1[CB_MAX_CHANNEL] = {
-	0x13333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  1, Tf = 2412MHz */
-	0x1B333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  2, Tf = 2417MHz */
-	0x03333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  3, Tf = 2422MHz */
-	0x0B333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  4, Tf = 2427MHz */
-	0x13333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  5, Tf = 2432MHz */
-	0x1B333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  6, Tf = 2437MHz */
-	0x03333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  7, Tf = 2442MHz */
-	0x0B333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  8, Tf = 2447MHz */
-	0x13333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  9, Tf = 2452MHz */
-	0x1B333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 10, Tf = 2457MHz */
-	0x03333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 11, Tf = 2462MHz */
-	0x0B333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 12, Tf = 2467MHz */
-	0x13333100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 13, Tf = 2472MHz */
-	0x06666100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 14, Tf = 2484MHz */
+	0x13333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  1, Tf = 2412MHz */
+	0x1B333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  2, Tf = 2417MHz */
+	0x03333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  3, Tf = 2422MHz */
+	0x0B333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  4, Tf = 2427MHz */
+	0x13333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  5, Tf = 2432MHz */
+	0x1B333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  6, Tf = 2437MHz */
+	0x03333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  7, Tf = 2442MHz */
+	0x0B333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  8, Tf = 2447MHz */
+	0x13333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  9, Tf = 2452MHz */
+	0x1B333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 10, Tf = 2457MHz */
+	0x03333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 11, Tf = 2462MHz */
+	0x0B333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 12, Tf = 2467MHz */
+	0x13333100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 13, Tf = 2472MHz */
+	0x06666100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 14, Tf = 2484MHz */
 
 	/* 4.9G => Ch 183, 184, 185, 187, 188, 189, 192, 196  (Value:15 ~ 22) */
-	0x1D555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 183, Tf = 4915MHz (15) */
-	0x00000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 184, Tf = 4920MHz (16) */
-	0x02AAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 185, Tf = 4925MHz (17) */
-	0x08000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 187, Tf = 4935MHz (18) */
-	0x0AAAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 188, Tf = 4940MHz (19) */
-	0x0D555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 189, Tf = 4945MHz (20) */
-	0x15555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 192, Tf = 4960MHz (21) */
-	0x00000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 196, Tf = 4980MHz (22) */
+	0x1D555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 183, Tf = 4915MHz (15) */
+	0x00000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 184, Tf = 4920MHz (16) */
+	0x02AAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 185, Tf = 4925MHz (17) */
+	0x08000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 187, Tf = 4935MHz (18) */
+	0x0AAAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 188, Tf = 4940MHz (19) */
+	0x0D555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 189, Tf = 4945MHz (20) */
+	0x15555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 192, Tf = 4960MHz (21) */
+	0x00000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 196, Tf = 4980MHz (22) */
 
 	/* 5G => Ch 7, 8, 9, 11, 12, 16, 34, 36, 38, 40, 42, 44, 46, 48, 52, 56, 60, 64,
-	 * 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165  (Value 23 ~ 56) */
-	0x1D555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =   7, Tf = 5035MHz (23) */
-	0x00000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =   8, Tf = 5040MHz (24) */
-	0x02AAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =   9, Tf = 5045MHz (25) */
-	0x08000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  11, Tf = 5055MHz (26) */
-	0x0AAAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  12, Tf = 5060MHz (27) */
-	0x15555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  16, Tf = 5080MHz (28) */
-	0x05555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  34, Tf = 5170MHz (29) */
-	0x0AAAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  36, Tf = 5180MHz (30) */
-	0x10000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  38, Tf = 5190MHz (31) */
-	0x15555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  40, Tf = 5200MHz (32) */
-	0x1AAAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  42, Tf = 5210MHz (33) */
-	0x00000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  44, Tf = 5220MHz (34) */
-	0x05555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  46, Tf = 5230MHz (35) */
-	0x0AAAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  48, Tf = 5240MHz (36) */
-	0x15555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  52, Tf = 5260MHz (37) */
-	0x00000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  56, Tf = 5280MHz (38) */
-	0x0AAAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  60, Tf = 5300MHz (39) */
-	0x15555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  64, Tf = 5320MHz (40) */
-	0x15555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 100, Tf = 5500MHz (41) */
-	0x00000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 104, Tf = 5520MHz (42) */
-	0x0AAAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 108, Tf = 5540MHz (43) */
-	0x15555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 112, Tf = 5560MHz (44) */
-	0x00000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 116, Tf = 5580MHz (45) */
-	0x0AAAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 120, Tf = 5600MHz (46) */
-	0x15555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 124, Tf = 5620MHz (47) */
-	0x00000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 128, Tf = 5640MHz (48) */
-	0x0AAAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 132, Tf = 5660MHz (49) */
-	0x15555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 136, Tf = 5680MHz (50) */
-	0x00000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 140, Tf = 5700MHz (51) */
-	0x18000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 149, Tf = 5745MHz (52) */
-	0x02AAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 153, Tf = 5765MHz (53) */
-	0x0D555100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 157, Tf = 5785MHz (54) */
-	0x18000100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 161, Tf = 5805MHz (55) */
-	0x02AAA100+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW  /* channel = 165, Tf = 5825MHz (56) */
+	 * 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165  (Value 23 ~ 56)
+	 */
+	0x1D555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =   7, Tf = 5035MHz (23) */
+	0x00000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =   8, Tf = 5040MHz (24) */
+	0x02AAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =   9, Tf = 5045MHz (25) */
+	0x08000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  11, Tf = 5055MHz (26) */
+	0x0AAAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  12, Tf = 5060MHz (27) */
+	0x15555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  16, Tf = 5080MHz (28) */
+	0x05555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  34, Tf = 5170MHz (29) */
+	0x0AAAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  36, Tf = 5180MHz (30) */
+	0x10000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  38, Tf = 5190MHz (31) */
+	0x15555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  40, Tf = 5200MHz (32) */
+	0x1AAAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  42, Tf = 5210MHz (33) */
+	0x00000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  44, Tf = 5220MHz (34) */
+	0x05555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  46, Tf = 5230MHz (35) */
+	0x0AAAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  48, Tf = 5240MHz (36) */
+	0x15555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  52, Tf = 5260MHz (37) */
+	0x00000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  56, Tf = 5280MHz (38) */
+	0x0AAAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  60, Tf = 5300MHz (39) */
+	0x15555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  64, Tf = 5320MHz (40) */
+	0x15555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 100, Tf = 5500MHz (41) */
+	0x00000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 104, Tf = 5520MHz (42) */
+	0x0AAAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 108, Tf = 5540MHz (43) */
+	0x15555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 112, Tf = 5560MHz (44) */
+	0x00000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 116, Tf = 5580MHz (45) */
+	0x0AAAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 120, Tf = 5600MHz (46) */
+	0x15555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 124, Tf = 5620MHz (47) */
+	0x00000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 128, Tf = 5640MHz (48) */
+	0x0AAAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 132, Tf = 5660MHz (49) */
+	0x15555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 136, Tf = 5680MHz (50) */
+	0x00000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 140, Tf = 5700MHz (51) */
+	0x18000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 149, Tf = 5745MHz (52) */
+	0x02AAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 153, Tf = 5765MHz (53) */
+	0x0D555100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 157, Tf = 5785MHz (54) */
+	0x18000100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 161, Tf = 5805MHz (55) */
+	0x02AAA100 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW  /* channel = 165, Tf = 5825MHz (56) */
 };
 
 static const unsigned long dwAL7230ChannelTable2[CB_MAX_CHANNEL] = {
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  1, Tf = 2412MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  2, Tf = 2417MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  3, Tf = 2422MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  4, Tf = 2427MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  5, Tf = 2432MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  6, Tf = 2437MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  7, Tf = 2442MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  8, Tf = 2447MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  9, Tf = 2452MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 10, Tf = 2457MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 11, Tf = 2462MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 12, Tf = 2467MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 13, Tf = 2472MHz */
-	0x7FD78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 14, Tf = 2484MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  1, Tf = 2412MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  2, Tf = 2417MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  3, Tf = 2422MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  4, Tf = 2427MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  5, Tf = 2432MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  6, Tf = 2437MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  7, Tf = 2442MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  8, Tf = 2447MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  9, Tf = 2452MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 10, Tf = 2457MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 11, Tf = 2462MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 12, Tf = 2467MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 13, Tf = 2472MHz */
+	0x7FD78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 14, Tf = 2484MHz */
 
 	/* 4.9G => Ch 183, 184, 185, 187, 188, 189, 192, 196  (Value:15 ~ 22) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 183, Tf = 4915MHz (15) */
-	0x67D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 184, Tf = 4920MHz (16) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 185, Tf = 4925MHz (17) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 187, Tf = 4935MHz (18) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 188, Tf = 4940MHz (19) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 189, Tf = 4945MHz (20) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 192, Tf = 4960MHz (21) */
-	0x67D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 196, Tf = 4980MHz (22) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 183, Tf = 4915MHz (15) */
+	0x67D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 184, Tf = 4920MHz (16) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 185, Tf = 4925MHz (17) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 187, Tf = 4935MHz (18) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 188, Tf = 4940MHz (19) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 189, Tf = 4945MHz (20) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 192, Tf = 4960MHz (21) */
+	0x67D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 196, Tf = 4980MHz (22) */
 
 	/* 5G => Ch 7, 8, 9, 11, 12, 16, 34, 36, 38, 40, 42, 44, 46, 48, 52, 56, 60, 64,
-	 * 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165  (Value 23 ~ 56) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =   7, Tf = 5035MHz (23) */
-	0x67D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =   8, Tf = 5040MHz (24) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =   9, Tf = 5045MHz (25) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  11, Tf = 5055MHz (26) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  12, Tf = 5060MHz (27) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  16, Tf = 5080MHz (28) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  34, Tf = 5170MHz (29) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  36, Tf = 5180MHz (30) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  38, Tf = 5190MHz (31) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  40, Tf = 5200MHz (32) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  42, Tf = 5210MHz (33) */
-	0x67D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  44, Tf = 5220MHz (34) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  46, Tf = 5230MHz (35) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  48, Tf = 5240MHz (36) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  52, Tf = 5260MHz (37) */
-	0x67D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  56, Tf = 5280MHz (38) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  60, Tf = 5300MHz (39) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel =  64, Tf = 5320MHz (40) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 100, Tf = 5500MHz (41) */
-	0x67D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 104, Tf = 5520MHz (42) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 108, Tf = 5540MHz (43) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 112, Tf = 5560MHz (44) */
-	0x67D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 116, Tf = 5580MHz (45) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 120, Tf = 5600MHz (46) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 124, Tf = 5620MHz (47) */
-	0x67D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 128, Tf = 5640MHz (48) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 132, Tf = 5660MHz (49) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 136, Tf = 5680MHz (50) */
-	0x67D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 140, Tf = 5700MHz (51) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 149, Tf = 5745MHz (52) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 153, Tf = 5765MHz (53) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 157, Tf = 5785MHz (54) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW, /* channel = 161, Tf = 5805MHz (55) */
-	0x77D78400+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW  /* channel = 165, Tf = 5825MHz (56) */
+	 * 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165  (Value 23 ~ 56)
+	 */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =   7, Tf = 5035MHz (23) */
+	0x67D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =   8, Tf = 5040MHz (24) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =   9, Tf = 5045MHz (25) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  11, Tf = 5055MHz (26) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  12, Tf = 5060MHz (27) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  16, Tf = 5080MHz (28) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  34, Tf = 5170MHz (29) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  36, Tf = 5180MHz (30) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  38, Tf = 5190MHz (31) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  40, Tf = 5200MHz (32) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  42, Tf = 5210MHz (33) */
+	0x67D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  44, Tf = 5220MHz (34) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  46, Tf = 5230MHz (35) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  48, Tf = 5240MHz (36) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  52, Tf = 5260MHz (37) */
+	0x67D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  56, Tf = 5280MHz (38) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  60, Tf = 5300MHz (39) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel =  64, Tf = 5320MHz (40) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 100, Tf = 5500MHz (41) */
+	0x67D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 104, Tf = 5520MHz (42) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 108, Tf = 5540MHz (43) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 112, Tf = 5560MHz (44) */
+	0x67D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 116, Tf = 5580MHz (45) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 120, Tf = 5600MHz (46) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 124, Tf = 5620MHz (47) */
+	0x67D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 128, Tf = 5640MHz (48) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 132, Tf = 5660MHz (49) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 136, Tf = 5680MHz (50) */
+	0x67D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 140, Tf = 5700MHz (51) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 149, Tf = 5745MHz (52) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 153, Tf = 5765MHz (53) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 157, Tf = 5785MHz (54) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW, /* channel = 161, Tf = 5805MHz (55) */
+	0x77D78400 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW  /* channel = 165, Tf = 5825MHz (56) */
 };
 
 /*
@@ -410,7 +408,7 @@ static const unsigned long dwAL7230ChannelTable2[CB_MAX_CHANNEL] = {
  *
  * Parameters:
  *  In:
- *      dwIoBase    - I/O base address
+ *      iobase      - I/O base address
  *  Out:
  *      none
  *
@@ -419,16 +417,16 @@ static const unsigned long dwAL7230ChannelTable2[CB_MAX_CHANNEL] = {
  */
 static bool s_bAL7230Init(struct vnt_private *priv)
 {
-	void __iomem *dwIoBase = priv->PortOffset;
+	void __iomem *iobase = priv->PortOffset;
 	int     ii;
 	bool ret;
 
 	ret = true;
 
 	/* 3-wire control for normal mode */
-	VNSvOutPortB(dwIoBase + MAC_REG_SOFTPWRCTL, 0);
+	VNSvOutPortB(iobase + MAC_REG_SOFTPWRCTL, 0);
 
-	MACvWordRegBitsOn(dwIoBase, MAC_REG_SOFTPWRCTL, (SOFTPWRCTL_SWPECTI  |
+	MACvWordRegBitsOn(iobase, MAC_REG_SOFTPWRCTL, (SOFTPWRCTL_SWPECTI  |
 							 SOFTPWRCTL_TXPEINV));
 	BBvPowerSaveModeOFF(priv); /* RobertYu:20050106, have DC value for Calibration */
 
@@ -436,20 +434,20 @@ static bool s_bAL7230Init(struct vnt_private *priv)
 		ret &= IFRFbWriteEmbedded(priv, dwAL7230InitTable[ii]);
 
 	/* PLL On */
-	MACvWordRegBitsOn(dwIoBase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_SWPE3);
+	MACvWordRegBitsOn(iobase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_SWPE3);
 
 	/* Calibration */
 	MACvTimer0MicroSDelay(priv, 150);/* 150us */
 	/* TXDCOC:active, RCK:disable */
-	ret &= IFRFbWriteEmbedded(priv, (0x9ABA8F00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW));
+	ret &= IFRFbWriteEmbedded(priv, (0x9ABA8F00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW));
 	MACvTimer0MicroSDelay(priv, 30);/* 30us */
 	/* TXDCOC:disable, RCK:active */
-	ret &= IFRFbWriteEmbedded(priv, (0x3ABA8F00+(BY_AL7230_REG_LEN<<3)+IFREGCTL_REGW));
+	ret &= IFRFbWriteEmbedded(priv, (0x3ABA8F00 + (BY_AL7230_REG_LEN << 3) + IFREGCTL_REGW));
 	MACvTimer0MicroSDelay(priv, 30);/* 30us */
 	/* TXDCOC:disable, RCK:disable */
 	ret &= IFRFbWriteEmbedded(priv, dwAL7230InitTable[CB_AL7230_INIT_SEQ-1]);
 
-	MACvWordRegBitsOn(dwIoBase, MAC_REG_SOFTPWRCTL, (SOFTPWRCTL_SWPE3    |
+	MACvWordRegBitsOn(iobase, MAC_REG_SOFTPWRCTL, (SOFTPWRCTL_SWPE3    |
 							 SOFTPWRCTL_SWPE2    |
 							 SOFTPWRCTL_SWPECTI  |
 							 SOFTPWRCTL_TXPEINV));
@@ -458,7 +456,7 @@ static bool s_bAL7230Init(struct vnt_private *priv)
 
 	/* PE1: TX_ON, PE2: RX_ON, PE3: PLLON */
 	/* 3-wire control for power saving mode */
-	VNSvOutPortB(dwIoBase + MAC_REG_PSPWRSIG, (PSSIG_WPE3 | PSSIG_WPE2)); /* 1100 0000 */
+	VNSvOutPortB(iobase + MAC_REG_PSPWRSIG, (PSSIG_WPE3 | PSSIG_WPE2)); /* 1100 0000 */
 
 	return ret;
 }
@@ -468,26 +466,26 @@ static bool s_bAL7230Init(struct vnt_private *priv)
  */
 static bool s_bAL7230SelectChannel(struct vnt_private *priv, unsigned char byChannel)
 {
-	void __iomem *dwIoBase = priv->PortOffset;
+	void __iomem *iobase = priv->PortOffset;
 	bool ret;
 
 	ret = true;
 
 	/* PLLON Off */
-	MACvWordRegBitsOff(dwIoBase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_SWPE3);
+	MACvWordRegBitsOff(iobase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_SWPE3);
 
 	ret &= IFRFbWriteEmbedded(priv, dwAL7230ChannelTable0[byChannel - 1]);
 	ret &= IFRFbWriteEmbedded(priv, dwAL7230ChannelTable1[byChannel - 1]);
 	ret &= IFRFbWriteEmbedded(priv, dwAL7230ChannelTable2[byChannel - 1]);
 
 	/* PLLOn On */
-	MACvWordRegBitsOn(dwIoBase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_SWPE3);
+	MACvWordRegBitsOn(iobase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_SWPE3);
 
 	/* Set Channel[7] = 0 to tell H/W channel is changing now. */
-	VNSvOutPortB(dwIoBase + MAC_REG_CHANNEL, (byChannel & 0x7F));
+	VNSvOutPortB(iobase + MAC_REG_CHANNEL, (byChannel & 0x7F));
 	MACvTimer0MicroSDelay(priv, SWITCH_CHANNEL_DELAY_AL7230);
 	/* Set Channel[7] = 1 to tell H/W channel change is done. */
-	VNSvOutPortB(dwIoBase + MAC_REG_CHANNEL, (byChannel | 0x80));
+	VNSvOutPortB(iobase + MAC_REG_CHANNEL, (byChannel | 0x80));
 
 	return ret;
 }
@@ -497,7 +495,7 @@ static bool s_bAL7230SelectChannel(struct vnt_private *priv, unsigned char byCha
  *
  * Parameters:
  *  In:
- *      dwIoBase    - I/O base address
+ *      iobase      - I/O base address
  *      dwData      - data to write
  *  Out:
  *      none
@@ -507,15 +505,15 @@ static bool s_bAL7230SelectChannel(struct vnt_private *priv, unsigned char byCha
  */
 bool IFRFbWriteEmbedded(struct vnt_private *priv, unsigned long dwData)
 {
-	void __iomem *dwIoBase = priv->PortOffset;
+	void __iomem *iobase = priv->PortOffset;
 	unsigned short ww;
 	unsigned long dwValue;
 
-	VNSvOutPortD(dwIoBase + MAC_REG_IFREGCTL, dwData);
+	VNSvOutPortD(iobase + MAC_REG_IFREGCTL, dwData);
 
 	/* W_MAX_TIMEOUT is the timeout period */
 	for (ww = 0; ww < W_MAX_TIMEOUT; ww++) {
-		VNSvInPortD(dwIoBase + MAC_REG_IFREGCTL, &dwValue);
+		VNSvInPortD(iobase + MAC_REG_IFREGCTL, &dwValue);
 		if (dwValue & IFREGCTL_DONE)
 			break;
 	}
@@ -531,7 +529,7 @@ bool IFRFbWriteEmbedded(struct vnt_private *priv, unsigned long dwData)
  *
  * Parameters:
  *  In:
- *      dwIoBase    - I/O base address
+ *      iobase      - I/O base address
  *  Out:
  *      none
  *
@@ -540,51 +538,51 @@ bool IFRFbWriteEmbedded(struct vnt_private *priv, unsigned long dwData)
  */
 static bool RFbAL2230Init(struct vnt_private *priv)
 {
-	void __iomem *dwIoBase = priv->PortOffset;
+	void __iomem *iobase = priv->PortOffset;
 	int     ii;
 	bool ret;
 
 	ret = true;
 
 	/* 3-wire control for normal mode */
-	VNSvOutPortB(dwIoBase + MAC_REG_SOFTPWRCTL, 0);
+	VNSvOutPortB(iobase + MAC_REG_SOFTPWRCTL, 0);
 
-	MACvWordRegBitsOn(dwIoBase, MAC_REG_SOFTPWRCTL, (SOFTPWRCTL_SWPECTI  |
+	MACvWordRegBitsOn(iobase, MAC_REG_SOFTPWRCTL, (SOFTPWRCTL_SWPECTI  |
 							 SOFTPWRCTL_TXPEINV));
 	/* PLL  Off */
-	MACvWordRegBitsOff(dwIoBase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_SWPE3);
+	MACvWordRegBitsOff(iobase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_SWPE3);
 
 	/* patch abnormal AL2230 frequency output */
-	IFRFbWriteEmbedded(priv, (0x07168700+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW));
+	IFRFbWriteEmbedded(priv, (0x07168700 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW));
 
 	for (ii = 0; ii < CB_AL2230_INIT_SEQ; ii++)
 		ret &= IFRFbWriteEmbedded(priv, dwAL2230InitTable[ii]);
 	MACvTimer0MicroSDelay(priv, 30); /* delay 30 us */
 
 	/* PLL On */
-	MACvWordRegBitsOn(dwIoBase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_SWPE3);
+	MACvWordRegBitsOn(iobase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_SWPE3);
 
 	MACvTimer0MicroSDelay(priv, 150);/* 150us */
-	ret &= IFRFbWriteEmbedded(priv, (0x00d80f00+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW));
+	ret &= IFRFbWriteEmbedded(priv, (0x00d80f00 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW));
 	MACvTimer0MicroSDelay(priv, 30);/* 30us */
-	ret &= IFRFbWriteEmbedded(priv, (0x00780f00+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW));
+	ret &= IFRFbWriteEmbedded(priv, (0x00780f00 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW));
 	MACvTimer0MicroSDelay(priv, 30);/* 30us */
 	ret &= IFRFbWriteEmbedded(priv, dwAL2230InitTable[CB_AL2230_INIT_SEQ-1]);
 
-	MACvWordRegBitsOn(dwIoBase, MAC_REG_SOFTPWRCTL, (SOFTPWRCTL_SWPE3    |
+	MACvWordRegBitsOn(iobase, MAC_REG_SOFTPWRCTL, (SOFTPWRCTL_SWPE3    |
 							 SOFTPWRCTL_SWPE2    |
 							 SOFTPWRCTL_SWPECTI  |
 							 SOFTPWRCTL_TXPEINV));
 
 	/* 3-wire control for power saving mode */
-	VNSvOutPortB(dwIoBase + MAC_REG_PSPWRSIG, (PSSIG_WPE3 | PSSIG_WPE2)); /* 1100 0000 */
+	VNSvOutPortB(iobase + MAC_REG_PSPWRSIG, (PSSIG_WPE3 | PSSIG_WPE2)); /* 1100 0000 */
 
 	return ret;
 }
 
 static bool RFbAL2230SelectChannel(struct vnt_private *priv, unsigned char byChannel)
 {
-	void __iomem *dwIoBase = priv->PortOffset;
+	void __iomem *iobase = priv->PortOffset;
 	bool ret;
 
 	ret = true;
@@ -593,10 +591,10 @@ static bool RFbAL2230SelectChannel(struct vnt_private *priv, unsigned char byCha
 	ret &= IFRFbWriteEmbedded(priv, dwAL2230ChannelTable1[byChannel - 1]);
 
 	/* Set Channel[7] = 0 to tell H/W channel is changing now. */
-	VNSvOutPortB(dwIoBase + MAC_REG_CHANNEL, (byChannel & 0x7F));
+	VNSvOutPortB(iobase + MAC_REG_CHANNEL, (byChannel & 0x7F));
 	MACvTimer0MicroSDelay(priv, SWITCH_CHANNEL_DELAY_AL2230);
 	/* Set Channel[7] = 1 to tell H/W channel change is done. */
-	VNSvOutPortB(dwIoBase + MAC_REG_CHANNEL, (byChannel | 0x80));
+	VNSvOutPortB(iobase + MAC_REG_CHANNEL, (byChannel | 0x80));
 
 	return ret;
 }
@@ -681,7 +679,7 @@ bool RFbSelectChannel(struct vnt_private *priv, unsigned char byRFType,
  *
  * Parameters:
  *  In:
- *      dwIoBase    - I/O base address
+ *      iobase      - I/O base address
  *      uChannel    - channel number
  *      bySleepCnt  - SleepProgSyn count
  *
@@ -691,12 +689,12 @@ bool RFbSelectChannel(struct vnt_private *priv, unsigned char byRFType,
 bool RFvWriteWakeProgSyn(struct vnt_private *priv, unsigned char byRFType,
 			 u16 uChannel)
 {
-	void __iomem *dwIoBase = priv->PortOffset;
+	void __iomem *iobase = priv->PortOffset;
 	int   ii;
 	unsigned char byInitCount = 0;
 	unsigned char bySleepCount = 0;
 
-	VNSvOutPortW(dwIoBase + MAC_REG_MISCFFNDEX, 0);
+	VNSvOutPortW(iobase + MAC_REG_MISCFFNDEX, 0);
 	switch (byRFType) {
 	case RF_AIROHA:
 	case RF_AL2230S:
@@ -758,7 +756,7 @@ bool RFvWriteWakeProgSyn(struct vnt_private *priv, unsigned char byRFType,
  *
  * Parameters:
  *  In:
- *      dwIoBase       - I/O base address
+ *      iobase         - I/O base address
  *      dwRFPowerTable - RF Tx Power Setting
  *  Out:
  *      none
@@ -830,7 +828,7 @@ bool RFbSetPower(
  *
  * Parameters:
  *  In:
- *      dwIoBase       - I/O base address
+ *      iobase         - I/O base address
  *      dwRFPowerTable - RF Tx Power Setting
  *  Out:
  *      none
@@ -855,20 +853,20 @@ bool RFbRawSetPower(
 	case RF_AIROHA:
 		ret &= IFRFbWriteEmbedded(priv, dwAL2230PowerTable[byPwr]);
 		if (rate <= RATE_11M)
-			ret &= IFRFbWriteEmbedded(priv, 0x0001B400+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW);
+			ret &= IFRFbWriteEmbedded(priv, 0x0001B400 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW);
 		else
-			ret &= IFRFbWriteEmbedded(priv, 0x0005A400+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW);
+			ret &= IFRFbWriteEmbedded(priv, 0x0005A400 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW);
 
 		break;
 
 	case RF_AL2230S:
 		ret &= IFRFbWriteEmbedded(priv, dwAL2230PowerTable[byPwr]);
 		if (rate <= RATE_11M) {
-			ret &= IFRFbWriteEmbedded(priv, 0x040C1400+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW);
-			ret &= IFRFbWriteEmbedded(priv, 0x00299B00+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW);
+			ret &= IFRFbWriteEmbedded(priv, 0x040C1400 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW);
+			ret &= IFRFbWriteEmbedded(priv, 0x00299B00 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW);
 		} else {
-			ret &= IFRFbWriteEmbedded(priv, 0x0005A400+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW);
-			ret &= IFRFbWriteEmbedded(priv, 0x00099B00+(BY_AL2230_REG_LEN<<3)+IFREGCTL_REGW);
+			ret &= IFRFbWriteEmbedded(priv, 0x0005A400 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW);
+			ret &= IFRFbWriteEmbedded(priv, 0x00099B00 + (BY_AL2230_REG_LEN << 3) + IFREGCTL_REGW);
 		}
 
 		break;
diff --git a/drivers/staging/vt6655/rf.h b/drivers/staging/vt6655/rf.h
index e9c786995506..b6e853784a26 100644
--- a/drivers/staging/vt6655/rf.h
+++ b/drivers/staging/vt6655/rf.h
@@ -12,11 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- *
  * File: rf.h
  *
  * Purpose:
diff --git a/drivers/staging/vt6655/rxtx.c b/drivers/staging/vt6655/rxtx.c
index 7e69bc99d60f..3efe19a1b13f 100644
--- a/drivers/staging/vt6655/rxtx.c
+++ b/drivers/staging/vt6655/rxtx.c
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: rxtx.c
  *
  * Purpose: handle WMAC/802.3/802.11 rx & tx functions
@@ -1086,8 +1082,8 @@ s_cbFillTxBufHead(struct vnt_private *pDevice, unsigned char byPktType,
 	}
 
 	/*
-	* Use for AUTO FALL BACK
-	*/
+	 * Use for AUTO FALL BACK
+	 */
 	if (fifo_ctl & FIFOCTL_AUTO_FB_0)
 		byFBOption = AUTO_FB_0;
 	else if (fifo_ctl & FIFOCTL_AUTO_FB_1)
diff --git a/drivers/staging/vt6655/rxtx.h b/drivers/staging/vt6655/rxtx.h
index 1e30ecb5c63c..89de67115826 100644
--- a/drivers/staging/vt6655/rxtx.h
+++ b/drivers/staging/vt6655/rxtx.h
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: rxtx.h
  *
  * Purpose:
diff --git a/drivers/staging/vt6655/srom.c b/drivers/staging/vt6655/srom.c
index ee992772066f..635f271595f6 100644
--- a/drivers/staging/vt6655/srom.c
+++ b/drivers/staging/vt6655/srom.c
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: srom.c
  *
  * Purpose:Implement functions to access eeprom
@@ -64,7 +60,7 @@
  *
  * Parameters:
  *  In:
- *      dwIoBase        - I/O base address
+ *      iobase          - I/O base address
  *      byContntOffset  - address of EEPROM
  *  Out:
  *      none
@@ -72,7 +68,7 @@
  * Return Value: data read
  *
  */
-unsigned char SROMbyReadEmbedded(void __iomem *dwIoBase,
+unsigned char SROMbyReadEmbedded(void __iomem *iobase,
 				 unsigned char byContntOffset)
 {
 	unsigned short wDelay, wNoACK;
@@ -81,18 +77,18 @@ unsigned char SROMbyReadEmbedded(void __iomem *dwIoBase,
 	unsigned char byOrg;
 
 	byData = 0xFF;
-	VNSvInPortB(dwIoBase + MAC_REG_I2MCFG, &byOrg);
+	VNSvInPortB(iobase + MAC_REG_I2MCFG, &byOrg);
 	/* turn off hardware retry for getting NACK */
-	VNSvOutPortB(dwIoBase + MAC_REG_I2MCFG, (byOrg & (~I2MCFG_NORETRY)));
+	VNSvOutPortB(iobase + MAC_REG_I2MCFG, (byOrg & (~I2MCFG_NORETRY)));
 	for (wNoACK = 0; wNoACK < W_MAX_I2CRETRY; wNoACK++) {
-		VNSvOutPortB(dwIoBase + MAC_REG_I2MTGID, EEP_I2C_DEV_ID);
-		VNSvOutPortB(dwIoBase + MAC_REG_I2MTGAD, byContntOffset);
+		VNSvOutPortB(iobase + MAC_REG_I2MTGID, EEP_I2C_DEV_ID);
+		VNSvOutPortB(iobase + MAC_REG_I2MTGAD, byContntOffset);
 
 		/* issue read command */
-		VNSvOutPortB(dwIoBase + MAC_REG_I2MCSR, I2MCSR_EEMR);
+		VNSvOutPortB(iobase + MAC_REG_I2MCSR, I2MCSR_EEMR);
 		/* wait DONE be set */
 		for (wDelay = 0; wDelay < W_MAX_TIMEOUT; wDelay++) {
-			VNSvInPortB(dwIoBase + MAC_REG_I2MCSR, &byWait);
+			VNSvInPortB(iobase + MAC_REG_I2MCSR, &byWait);
 			if (byWait & (I2MCSR_DONE | I2MCSR_NACK))
 				break;
 			PCAvDelayByIO(CB_DELAY_LOOP_WAIT);
@@ -102,8 +98,8 @@ unsigned char SROMbyReadEmbedded(void __iomem *dwIoBase,
 			break;
 		}
 	}
-	VNSvInPortB(dwIoBase + MAC_REG_I2MDIPT, &byData);
-	VNSvOutPortB(dwIoBase + MAC_REG_I2MCFG, byOrg);
+	VNSvInPortB(iobase + MAC_REG_I2MDIPT, &byData);
+	VNSvOutPortB(iobase + MAC_REG_I2MCFG, byOrg);
 	return byData;
 }
 
@@ -112,20 +108,20 @@ unsigned char SROMbyReadEmbedded(void __iomem *dwIoBase,
  *
  * Parameters:
  *  In:
- *      dwIoBase        - I/O base address
+ *      iobase          - I/O base address
  *  Out:
  *      pbyEepromRegs   - EEPROM content Buffer
  *
  * Return Value: none
  *
  */
-void SROMvReadAllContents(void __iomem *dwIoBase, unsigned char *pbyEepromRegs)
+void SROMvReadAllContents(void __iomem *iobase, unsigned char *pbyEepromRegs)
 {
 	int     ii;
 
 	/* ii = Rom Address */
 	for (ii = 0; ii < EEP_MAX_CONTEXT_SIZE; ii++) {
-		*pbyEepromRegs = SROMbyReadEmbedded(dwIoBase,
+		*pbyEepromRegs = SROMbyReadEmbedded(iobase,
 						    (unsigned char)ii);
 		pbyEepromRegs++;
 	}
@@ -136,21 +132,21 @@ void SROMvReadAllContents(void __iomem *dwIoBase, unsigned char *pbyEepromRegs)
  *
  * Parameters:
  *  In:
- *      dwIoBase        - I/O base address
+ *      iobase          - I/O base address
  *  Out:
  *      pbyEtherAddress - Ethernet Address buffer
  *
  * Return Value: none
  *
  */
-void SROMvReadEtherAddress(void __iomem *dwIoBase,
+void SROMvReadEtherAddress(void __iomem *iobase,
 			   unsigned char *pbyEtherAddress)
 {
 	unsigned char ii;
 
 	/* ii = Rom Address */
 	for (ii = 0; ii < ETH_ALEN; ii++) {
-		*pbyEtherAddress = SROMbyReadEmbedded(dwIoBase, ii);
+		*pbyEtherAddress = SROMbyReadEmbedded(iobase, ii);
 		pbyEtherAddress++;
 	}
 }
diff --git a/drivers/staging/vt6655/srom.h b/drivers/staging/vt6655/srom.h
index 531bf0069373..6e03ab6dfa9d 100644
--- a/drivers/staging/vt6655/srom.h
+++ b/drivers/staging/vt6655/srom.h
@@ -12,11 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- *
  * File: srom.h
  *
  * Purpose: Implement functions to access eeprom
@@ -90,12 +85,12 @@
 
 /*---------------------  Export Functions  --------------------------*/
 
-unsigned char SROMbyReadEmbedded(void __iomem *dwIoBase,
+unsigned char SROMbyReadEmbedded(void __iomem *iobase,
 				 unsigned char byContntOffset);
 
-void SROMvReadAllContents(void __iomem *dwIoBase, unsigned char *pbyEepromRegs);
+void SROMvReadAllContents(void __iomem *iobase, unsigned char *pbyEepromRegs);
 
-void SROMvReadEtherAddress(void __iomem *dwIoBase,
+void SROMvReadEtherAddress(void __iomem *iobase,
 			   unsigned char *pbyEtherAddress);
 
 #endif /* __EEPROM_H__*/
diff --git a/drivers/staging/vt6655/tmacro.h b/drivers/staging/vt6655/tmacro.h
index 597efefc017f..d6a0563ad55c 100644
--- a/drivers/staging/vt6655/tmacro.h
+++ b/drivers/staging/vt6655/tmacro.h
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: tmacro.h
  *
  * Purpose: define basic common types and macros
diff --git a/drivers/staging/vt6655/upc.h b/drivers/staging/vt6655/upc.h
index 85fe0464cfb3..9806b5989014 100644
--- a/drivers/staging/vt6655/upc.h
+++ b/drivers/staging/vt6655/upc.h
@@ -12,10 +12,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
  * File: upc.h
  *
  * Purpose: Macros to access device
diff --git a/drivers/staging/vt6656/baseband.h b/drivers/staging/vt6656/baseband.h
index 7cc13874f8f1..fe1c25c64cca 100644
--- a/drivers/staging/vt6656/baseband.h
+++ b/drivers/staging/vt6656/baseband.h
@@ -86,15 +86,15 @@ struct vnt_phy_field {
 unsigned int vnt_get_frame_time(u8 preamble_type, u8 pkt_type,
 				unsigned int frame_length, u16 tx_rate);
 
-void vnt_get_phy_field(struct vnt_private *, u32 frame_length,
-		       u16 tx_rate, u8 pkt_type, struct vnt_phy_field *);
-
-void vnt_set_short_slot_time(struct vnt_private *);
-void vnt_set_vga_gain_offset(struct vnt_private *, u8);
-void vnt_set_antenna_mode(struct vnt_private *, u8);
-int vnt_vt3184_init(struct vnt_private *);
-void vnt_set_deep_sleep(struct vnt_private *);
-void vnt_exit_deep_sleep(struct vnt_private *);
-void vnt_update_pre_ed_threshold(struct vnt_private *, int scanning);
+void vnt_get_phy_field(struct vnt_private *priv, u32 frame_length,
+		       u16 tx_rate, u8 pkt_type, struct vnt_phy_field *phy);
+
+void vnt_set_short_slot_time(struct vnt_private *priv);
+void vnt_set_vga_gain_offset(struct vnt_private *priv, u8 data);
+void vnt_set_antenna_mode(struct vnt_private *priv, u8 antenna_mode);
+int vnt_vt3184_init(struct vnt_private *priv);
+void vnt_set_deep_sleep(struct vnt_private *priv);
+void vnt_exit_deep_sleep(struct vnt_private *priv);
+void vnt_update_pre_ed_threshold(struct vnt_private *priv, int scanning);
 
 #endif /* __BASEBAND_H__ */
diff --git a/drivers/staging/vt6656/card.c b/drivers/staging/vt6656/card.c
index 53b469c71dc2..0e5a99375099 100644
--- a/drivers/staging/vt6656/card.c
+++ b/drivers/staging/vt6656/card.c
@@ -501,16 +501,7 @@ u8 vnt_get_pkt_type(struct vnt_private *priv)
  */
 u64 vnt_get_tsf_offset(u8 rx_rate, u64 tsf1, u64 tsf2)
 {
-	u64 tsf_offset = 0;
-	u16 rx_bcn_offset;
-
-	rx_bcn_offset = cw_rxbcntsf_off[rx_rate % MAX_RATE];
-
-	tsf2 += (u64)rx_bcn_offset;
-
-	tsf_offset = tsf1 - tsf2;
-
-	return tsf_offset;
+	return tsf1 - tsf2 - (u64)cw_rxbcntsf_off[rx_rate % MAX_RATE];
 }
 
 /*
@@ -610,8 +601,8 @@ u64 vnt_get_next_tbtt(u64 tsf, u16 beacon_interval)
 	beacon_int = beacon_interval * 1024;
 
 	/* Next TBTT =
-	*	((local_current_TSF / beacon_interval) + 1) * beacon_interval
-	*/
+	 *	((local_current_TSF / beacon_interval) + 1) * beacon_interval
+	 */
 	if (beacon_int) {
 		do_div(tsf, beacon_int);
 		tsf += 1;
diff --git a/drivers/staging/vt6656/mac.c b/drivers/staging/vt6656/mac.c
index eeed16e9124e..611da4929ddc 100644
--- a/drivers/staging/vt6656/mac.c
+++ b/drivers/staging/vt6656/mac.c
@@ -121,7 +121,7 @@ void vnt_mac_set_keyentry(struct vnt_private *priv, u16 key_ctl, u32 entry_idx,
 	u16 offset;
 
 	offset = MISCFIFO_KEYETRY0;
-	offset += (entry_idx * MISCFIFO_KEYENTRYSIZE);
+	offset += entry_idx * MISCFIFO_KEYENTRYSIZE;
 
 	set_key.u.write.key_ctl = cpu_to_le16(key_ctl);
 	ether_addr_copy(set_key.u.write.addr, addr);
diff --git a/drivers/staging/vt6656/main_usb.c b/drivers/staging/vt6656/main_usb.c
index 0594828bdabf..50d02d9aa535 100644
--- a/drivers/staging/vt6656/main_usb.c
+++ b/drivers/staging/vt6656/main_usb.c
@@ -85,7 +85,7 @@ MODULE_PARM_DESC(tx_buffers, "Number of receive usb tx buffers");
  * Static vars definitions
  */
 
-static struct usb_device_id vt6656_table[] = {
+static const struct usb_device_id vt6656_table[] = {
 	{USB_DEVICE(VNT_USB_VENDOR_ID, VNT_USB_PRODUCT_ID)},
 	{}
 };
@@ -326,9 +326,9 @@ static int vnt_init_registers(struct vnt_private *priv)
 		priv->current_net_addr);
 
 	/*
-	* set BB and packet type at the same time
-	* set Short Slot Time, xIFS, and RSPINF
-	*/
+	 * set BB and packet type at the same time
+	 * set Short Slot Time, xIFS, and RSPINF
+	 */
 	if (priv->bb_type == BB_TYPE_11A)
 		priv->short_slot_time = true;
 	else
diff --git a/drivers/staging/vt6656/rf.c b/drivers/staging/vt6656/rf.c
index 79a3108719a6..6101a35582b6 100644
--- a/drivers/staging/vt6656/rf.c
+++ b/drivers/staging/vt6656/rf.c
@@ -730,9 +730,9 @@ int vnt_rf_set_txpower(struct vnt_private *priv, u8 power, u32 rate)
 			return false;
 
 		/*
-		* 0x080F1B00 for 3 wire control TxGain(D10)
-		* and 0x31 as TX Gain value
-		*/
+		 * 0x080F1B00 for 3 wire control TxGain(D10)
+		 * and 0x31 as TX Gain value
+		 */
 		power_setting = 0x080c0b00 | (power << 12);
 
 		ret &= vnt_rf_write_embedded(priv, power_setting);
@@ -800,8 +800,8 @@ int vnt_rf_set_txpower(struct vnt_private *priv, u8 power, u32 rate)
 /* Convert rssi to dbm */
 void vnt_rf_rssi_to_dbm(struct vnt_private *priv, u8 rssi, long *dbm)
 {
-	u8 idx = (((rssi & 0xc0) >> 6) & 0x03);
-	long b = (rssi & 0x3f);
+	u8 idx = ((rssi & 0xc0) >> 6) & 0x03;
+	long b = rssi & 0x3f;
 	long a = 0;
 	u8 airoharf[4] = {0, 18, 0, 40};
 
diff --git a/drivers/staging/wilc1000/coreconfigurator.c b/drivers/staging/wilc1000/coreconfigurator.c
index 4b51c0ac27ac..622994795222 100644
--- a/drivers/staging/wilc1000/coreconfigurator.c
+++ b/drivers/staging/wilc1000/coreconfigurator.c
@@ -224,9 +224,7 @@ static inline u16 get_asoc_status(u8 *data)
 	u16 asoc_status;
 
 	asoc_status = data[3];
-	asoc_status = (asoc_status << 8) | data[2];
-
-	return asoc_status;
+	return (asoc_status << 8) | data[2];
 }
 
 static inline u16 get_asoc_id(u8 *data)
diff --git a/drivers/staging/wilc1000/host_interface.c b/drivers/staging/wilc1000/host_interface.c
index 6ab7443eabde..b00ea75524e4 100644
--- a/drivers/staging/wilc1000/host_interface.c
+++ b/drivers/staging/wilc1000/host_interface.c
@@ -1722,10 +1722,8 @@ _WPAPtk_end_case_:
 
 	case PMKSA:
 		pu8keybuf = kmalloc((pstrHostIFkeyAttr->attr.pmkid.numpmkid * PMKSA_KEY_LEN) + 1, GFP_KERNEL);
-		if (!pu8keybuf) {
-			netdev_err(vif->ndev, "No buffer to send PMKSA Key\n");
+		if (!pu8keybuf)
 			return -ENOMEM;
-		}
 
 		pu8keybuf[0] = pstrHostIFkeyAttr->attr.pmkid.numpmkid;
 
@@ -1932,7 +1930,7 @@ static s32 Handle_Get_InActiveTime(struct wilc_vif *vif,
 	wid.val = kmalloc(wid.size, GFP_KERNEL);
 
 	stamac = wid.val;
-	memcpy(stamac, strHostIfStaInactiveT->mac, ETH_ALEN);
+	ether_addr_copy(stamac, strHostIfStaInactiveT->mac);
 
 	result = wilc_send_config_pkt(vif, SET_CFG, &wid, 1,
 				      wilc_get_vif_idx(vif));
@@ -2168,7 +2166,7 @@ static void Handle_DelStation(struct wilc_vif *vif,
 
 	pu8CurrByte = wid.val;
 
-	memcpy(pu8CurrByte, pstrDelStaParam->mac_addr, ETH_ALEN);
+	ether_addr_copy(pu8CurrByte, pstrDelStaParam->mac_addr);
 
 	result = wilc_send_config_pkt(vif, SET_CFG, &wid, 1,
 				      wilc_get_vif_idx(vif));
@@ -2322,10 +2320,8 @@ static u32 Handle_ListenStateExpired(struct wilc_vif *vif,
 		wid.size = 2;
 		wid.val = kmalloc(wid.size, GFP_KERNEL);
 
-		if (!wid.val) {
-			netdev_err(vif->ndev, "Failed to allocate memory\n");
+		if (!wid.val)
 			return -ENOMEM;
-		}
 
 		wid.val[0] = u8remain_on_chan_flag;
 		wid.val[1] = FALSE_FRMWR_CHANNEL;
diff --git a/drivers/staging/wilc1000/host_interface.h b/drivers/staging/wilc1000/host_interface.h
index ddfea29df2a7..f36d3b5a0370 100644
--- a/drivers/staging/wilc1000/host_interface.h
+++ b/drivers/staging/wilc1000/host_interface.h
@@ -367,7 +367,6 @@ extern u8 wilc_connected_ssid[6];
 extern u8 wilc_multicast_mac_addr_list[WILC_MULTICAST_TABLE_SIZE][ETH_ALEN];
 
 extern int wilc_connecting;
-extern u8 wilc_initialized;
 extern struct timer_list wilc_during_ip_timer;
 
 #endif
diff --git a/drivers/staging/wilc1000/linux_wlan.c b/drivers/staging/wilc1000/linux_wlan.c
index 6370a5efe343..3775706578b2 100644
--- a/drivers/staging/wilc1000/linux_wlan.c
+++ b/drivers/staging/wilc1000/linux_wlan.c
@@ -37,6 +37,8 @@ static void linux_wlan_tx_complete(void *priv, int status);
 static int  mac_init_fn(struct net_device *ndev);
 static struct net_device_stats *mac_stats(struct net_device *dev);
 static int  mac_ioctl(struct net_device *ndev, struct ifreq *req, int cmd);
+static int wilc_mac_open(struct net_device *ndev);
+static int wilc_mac_close(struct net_device *ndev);
 static void wilc_set_multicast_list(struct net_device *dev);
 
 bool wilc_enable_ps = true;
@@ -218,17 +220,6 @@ static void deinit_irq(struct net_device *dev)
 	}
 }
 
-int wilc_lock_timeout(struct wilc *nic, void *vp, u32 timeout)
-{
-	/* FIXME: replace with mutex_lock or wait_for_completion */
-	int error = -1;
-
-	if (vp)
-		error = down_timeout(vp,
-				     msecs_to_jiffies(timeout));
-	return error;
-}
-
 void wilc_mac_indicate(struct wilc *wilc, int flag)
 {
 	int status;
@@ -269,23 +260,12 @@ static struct net_device *get_if_handler(struct wilc *wilc, u8 *mac_header)
 
 int wilc_wlan_set_bssid(struct net_device *wilc_netdev, u8 *bssid, u8 mode)
 {
-	int i = 0;
-	int ret = -1;
-	struct wilc_vif *vif;
-	struct wilc *wilc;
-
-	vif = netdev_priv(wilc_netdev);
-	wilc = vif->wilc;
+	struct wilc_vif *vif = netdev_priv(wilc_netdev);
 
-	for (i = 0; i < wilc->vif_num; i++)
-		if (wilc->vif[i]->ndev == wilc_netdev) {
-			memcpy(wilc->vif[i]->bssid, bssid, 6);
-			wilc->vif[i]->mode = mode;
-			ret = 0;
-			break;
-		}
+	memcpy(vif->bssid, bssid, 6);
+	vif->mode = mode;
 
-	return ret;
+	return 0;
 }
 
 int wilc_wlan_get_num_conn_ifcs(struct wilc *wilc)
@@ -847,7 +827,7 @@ static int mac_init_fn(struct net_device *ndev)
 	return 0;
 }
 
-int wilc_mac_open(struct net_device *ndev)
+static int wilc_mac_open(struct net_device *ndev)
 {
 	struct wilc_vif *vif;
 
@@ -1038,7 +1018,7 @@ int wilc_mac_xmit(struct sk_buff *skb, struct net_device *ndev)
 	return 0;
 }
 
-int wilc_mac_close(struct net_device *ndev)
+static int wilc_mac_close(struct net_device *ndev)
 {
 	struct wilc_priv *priv;
 	struct wilc_vif *vif;
@@ -1212,16 +1192,11 @@ void WILC_WFI_mgmt_rx(struct wilc *wilc, u8 *buff, u32 size)
 
 void wilc_netdev_cleanup(struct wilc *wilc)
 {
-	int i = 0;
-	struct wilc_vif *vif[NUM_CONCURRENT_IFC];
+	int i;
 
-	if (wilc && (wilc->vif[0]->ndev || wilc->vif[1]->ndev)) {
+	if (wilc && (wilc->vif[0]->ndev || wilc->vif[1]->ndev))
 		unregister_inetaddr_notifier(&g_dev_notifier);
 
-		for (i = 0; i < NUM_CONCURRENT_IFC; i++)
-			vif[i] = netdev_priv(wilc->vif[i]->ndev);
-	}
-
 	if (wilc && wilc->firmware) {
 		release_firmware(wilc->firmware);
 		wilc->firmware = NULL;
@@ -1230,7 +1205,7 @@ void wilc_netdev_cleanup(struct wilc *wilc)
 	if (wilc && (wilc->vif[0]->ndev || wilc->vif[1]->ndev)) {
 		for (i = 0; i < NUM_CONCURRENT_IFC; i++)
 			if (wilc->vif[i]->ndev)
-				if (vif[i]->mac_opened)
+				if (wilc->vif[i]->mac_opened)
 					wilc_mac_close(wilc->vif[i]->ndev);
 
 		for (i = 0; i < NUM_CONCURRENT_IFC; i++) {
@@ -1278,9 +1253,9 @@ int wilc_netdev_init(struct wilc **wilc, struct device *dev, int io_type,
 
 		vif->idx = wl->vif_num;
 		vif->wilc = *wilc;
+		vif->ndev = ndev;
 		wl->vif[i] = vif;
-		wl->vif[wl->vif_num]->ndev = ndev;
-		wl->vif_num++;
+		wl->vif_num = i;
 		ndev->netdev_ops = &wilc_netdev_ops;
 
 		{
diff --git a/drivers/staging/wilc1000/wilc_debugfs.c b/drivers/staging/wilc1000/wilc_debugfs.c
index 802bb1d5e207..07260c497db4 100644
--- a/drivers/staging/wilc1000/wilc_debugfs.c
+++ b/drivers/staging/wilc1000/wilc_debugfs.c
@@ -62,16 +62,16 @@ static ssize_t wilc_debug_level_write(struct file *filp, const char __user *buf,
 		return ret;
 
 	if (flag > DBG_LEVEL_ALL) {
-		printk("%s, value (0x%08x) is out of range, stay previous flag (0x%08x)\n", __func__, flag, atomic_read(&WILC_DEBUG_LEVEL));
+		pr_info("%s, value (0x%08x) is out of range, stay previous flag (0x%08x)\n", __func__, flag, atomic_read(&WILC_DEBUG_LEVEL));
 		return -EINVAL;
 	}
 
 	atomic_set(&WILC_DEBUG_LEVEL, (int)flag);
 
 	if (flag == 0)
-		printk(KERN_INFO "Debug-level disabled\n");
+		pr_info("Debug-level disabled\n");
 	else
-		printk(KERN_INFO "Debug-level enabled\n");
+		pr_info("Debug-level enabled\n");
 
 	return count;
 }
diff --git a/drivers/staging/wilc1000/wilc_sdio.c b/drivers/staging/wilc1000/wilc_sdio.c
index 39b73fb27398..3ad7cec4662d 100644
--- a/drivers/staging/wilc1000/wilc_sdio.c
+++ b/drivers/staging/wilc1000/wilc_sdio.c
@@ -39,6 +39,7 @@ struct wilc_sdio {
 };
 
 static struct wilc_sdio g_sdio;
+static const struct wilc_hif_func wilc_hif_sdio;
 
 static int sdio_write_reg(struct wilc *wilc, u32 addr, u32 data);
 static int sdio_read_reg(struct wilc *wilc, u32 addr, u32 *data);
@@ -1100,7 +1101,7 @@ static int sdio_sync_ext(struct wilc *wilc, int nint)
  *
  ********************************************/
 
-const struct wilc_hif_func wilc_hif_sdio = {
+static const struct wilc_hif_func wilc_hif_sdio = {
 	.hif_init = sdio_init,
 	.hif_deinit = sdio_deinit,
 	.hif_read_reg = sdio_read_reg,
diff --git a/drivers/staging/wilc1000/wilc_spi.c b/drivers/staging/wilc1000/wilc_spi.c
index f08cf6d9e1af..55d53c3a95df 100644
--- a/drivers/staging/wilc1000/wilc_spi.c
+++ b/drivers/staging/wilc1000/wilc_spi.c
@@ -30,6 +30,7 @@ struct wilc_spi {
 };
 
 static struct wilc_spi g_spi;
+static const struct wilc_hif_func wilc_hif_spi;
 
 static int wilc_spi_read(struct wilc *wilc, u32, u8 *, u32);
 static int wilc_spi_write(struct wilc *wilc, u32, u8 *, u32);
@@ -858,7 +859,8 @@ static int wilc_spi_init(struct wilc *wilc, bool resume)
 	/* the SPI to it's initial value. */
 	if (!spi_internal_read(wilc, WILC_SPI_PROTOCOL_OFFSET, &reg)) {
 		/* Read failed. Try with CRC off. This might happen when module
-		 * is removed but chip isn't reset*/
+		 * is removed but chip isn't reset
+		 */
 		g_spi.crc_off = 1;
 		dev_err(&spi->dev, "Failed internal read protocol with CRC on, retrying with CRC off...\n");
 		if (!spi_internal_read(wilc, WILC_SPI_PROTOCOL_OFFSET, &reg)) {
@@ -1133,7 +1135,7 @@ static int wilc_spi_sync_ext(struct wilc *wilc, int nint)
  *      Global spi HIF function table
  *
  ********************************************/
-const struct wilc_hif_func wilc_hif_spi = {
+static const struct wilc_hif_func wilc_hif_spi = {
 	.hif_init = wilc_spi_init,
 	.hif_deinit = _wilc_spi_deinit,
 	.hif_read_reg = wilc_spi_read_reg,
diff --git a/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c b/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c
index 60d8b055bb2f..c1a24f7bc85f 100644
--- a/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c
+++ b/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c
@@ -90,17 +90,12 @@ static const struct wiphy_wowlan_support wowlan_support = {
 #define IS_MGMT_STATUS_SUCCES			0x040
 #define GET_PKT_OFFSET(a) (((a) >> 22) & 0x1ff)
 
-extern int wilc_mac_open(struct net_device *ndev);
-extern int wilc_mac_close(struct net_device *ndev);
-
 static struct network_info last_scanned_shadow[MAX_NUM_SCANNED_NETWORKS_SHADOW];
 static u32 last_scanned_cnt;
 struct timer_list wilc_during_ip_timer;
 static struct timer_list hAgingTimer;
 static u8 op_ifcs;
 
-u8 wilc_initialized = 1;
-
 #define CHAN2G(_channel, _freq, _flags) {	 \
 		.band             = NL80211_BAND_2GHZ, \
 		.center_freq      = (_freq),		 \
@@ -1193,6 +1188,7 @@ static int get_station(struct wiphy *wiphy, struct net_device *dev,
 	u32 i = 0;
 	u32 associatedsta = ~0;
 	u32 inactive_time = 0;
+
 	priv = wiphy_priv(wiphy);
 	vif = netdev_priv(dev);
 
@@ -1590,28 +1586,25 @@ static int remain_on_channel(struct wiphy *wiphy,
 	priv->strRemainOnChanParams.u32ListenDuration = duration;
 	priv->strRemainOnChanParams.u32ListenSessionID++;
 
-	s32Error = wilc_remain_on_channel(vif,
+	return wilc_remain_on_channel(vif,
 				priv->strRemainOnChanParams.u32ListenSessionID,
 				duration, chan->hw_value,
 				WILC_WFI_RemainOnChannelExpired,
 				WILC_WFI_RemainOnChannelReady, (void *)priv);
-
-	return s32Error;
 }
 
 static int cancel_remain_on_channel(struct wiphy *wiphy,
 				    struct wireless_dev *wdev,
 				    u64 cookie)
 {
-	s32 s32Error = 0;
 	struct wilc_priv *priv;
 	struct wilc_vif *vif;
 
 	priv = wiphy_priv(wiphy);
 	vif = netdev_priv(priv->dev);
 
-	s32Error = wilc_listen_state_expired(vif, priv->strRemainOnChanParams.u32ListenSessionID);
-	return s32Error;
+	return wilc_listen_state_expired(vif,
+			priv->strRemainOnChanParams.u32ListenSessionID);
 }
 
 static int mgmt_tx(struct wiphy *wiphy,
@@ -1935,12 +1928,10 @@ static int start_ap(struct wiphy *wiphy, struct net_device *dev,
 	wilc_wlan_set_bssid(dev, wl->vif[vif->idx]->src_addr, AP_MODE);
 	wilc_set_power_mgmt(vif, 0, 0);
 
-	s32Error = wilc_add_beacon(vif, settings->beacon_interval,
+	return wilc_add_beacon(vif, settings->beacon_interval,
 				   settings->dtim_period, beacon->head_len,
 				   (u8 *)beacon->head, beacon->tail_len,
 				   (u8 *)beacon->tail);
-
-	return s32Error;
 }
 
 static int change_beacon(struct wiphy *wiphy, struct net_device *dev,
@@ -1948,16 +1939,13 @@ static int change_beacon(struct wiphy *wiphy, struct net_device *dev,
 {
 	struct wilc_priv *priv;
 	struct wilc_vif *vif;
-	s32 s32Error = 0;
 
 	priv = wiphy_priv(wiphy);
 	vif = netdev_priv(priv->dev);
 
-	s32Error = wilc_add_beacon(vif, 0, 0, beacon->head_len,
+	return wilc_add_beacon(vif, 0, 0, beacon->head_len,
 				   (u8 *)beacon->head, beacon->tail_len,
 				   (u8 *)beacon->tail);
-
-	return s32Error;
 }
 
 static int stop_ap(struct wiphy *wiphy, struct net_device *dev)
diff --git a/drivers/staging/wilc1000/wilc_wfi_netdevice.h b/drivers/staging/wilc1000/wilc_wfi_netdevice.h
index ec6b1674cf38..d431673bc46c 100644
--- a/drivers/staging/wilc1000/wilc_wfi_netdevice.h
+++ b/drivers/staging/wilc1000/wilc_wfi_netdevice.h
@@ -225,7 +225,6 @@ int wilc1000_wlan_init(struct net_device *dev, struct wilc_vif *vif);
 
 void wilc_frmw_to_linux(struct wilc *wilc, u8 *buff, u32 size, u32 pkt_offset);
 void wilc_mac_indicate(struct wilc *wilc, int flag);
-int wilc_lock_timeout(struct wilc *wilc, void *, u32 timeout);
 void wilc_netdev_cleanup(struct wilc *wilc);
 int wilc_netdev_init(struct wilc **wilc, struct device *, int io_type, int gpio,
 		     const struct wilc_hif_func *ops);
diff --git a/drivers/staging/wilc1000/wilc_wlan.h b/drivers/staging/wilc1000/wilc_wlan.h
index de6c4ddbf45a..11365efcc5d0 100644
--- a/drivers/staging/wilc1000/wilc_wlan.h
+++ b/drivers/staging/wilc1000/wilc_wlan.h
@@ -248,9 +248,6 @@ struct wilc_hif_func {
 	void (*disable_interrupt)(struct wilc *nic);
 };
 
-extern const struct wilc_hif_func wilc_hif_spi;
-extern const struct wilc_hif_func wilc_hif_sdio;
-
 /********************************************
  *
  *      Configuration Structure
@@ -297,9 +294,6 @@ void wilc_enable_tcp_ack_filter(bool value);
 int wilc_wlan_get_num_conn_ifcs(struct wilc *);
 int wilc_mac_xmit(struct sk_buff *skb, struct net_device *dev);
 
-int wilc_mac_open(struct net_device *ndev);
-int wilc_mac_close(struct net_device *ndev);
-
 void WILC_WFI_p2p_rx(struct net_device *dev, u8 *buff, u32 size);
 void host_wakeup_notify(struct wilc *wilc);
 void host_sleep_notify(struct wilc *wilc);
diff --git a/drivers/staging/wlan-ng/cfg80211.c b/drivers/staging/wlan-ng/cfg80211.c
index 182b2d564627..aa0e5a3d4a89 100644
--- a/drivers/staging/wlan-ng/cfg80211.c
+++ b/drivers/staging/wlan-ng/cfg80211.c
@@ -323,7 +323,7 @@ static int prism2_scan(struct wiphy *wiphy,
 
 	priv->scan_request = request;
 
-	memset(&msg1, 0x00, sizeof(struct p80211msg_dot11req_scan));
+	memset(&msg1, 0x00, sizeof(msg1));
 	msg1.msgcode = DIDmsg_dot11req_scan;
 	msg1.bsstype.data = P80211ENUM_bsstype_any;
 
@@ -375,13 +375,13 @@ static int prism2_scan(struct wiphy *wiphy,
 		ie_buf[0] = WLAN_EID_SSID;
 		ie_buf[1] = msg2.ssid.data.len;
 		ie_len = ie_buf[1] + 2;
-		memcpy(&ie_buf[2], &(msg2.ssid.data.data), msg2.ssid.data.len);
+		memcpy(&ie_buf[2], &msg2.ssid.data.data, msg2.ssid.data.len);
 		freq = ieee80211_channel_to_frequency(msg2.dschannel.data,
 						      NL80211_BAND_2GHZ);
 		bss = cfg80211_inform_bss(wiphy,
 			ieee80211_get_channel(wiphy, freq),
 			CFG80211_BSS_FTYPE_UNKNOWN,
-			(const u8 *)&(msg2.bssid.data.data),
+			(const u8 *)&msg2.bssid.data.data,
 			msg2.timestamp.data, msg2.capinfo.data,
 			msg2.beaconperiod.data,
 			ie_buf,
diff --git a/drivers/staging/wlan-ng/hfa384x.h b/drivers/staging/wlan-ng/hfa384x.h
index 43c299c3b631..60caf9c37727 100644
--- a/drivers/staging/wlan-ng/hfa384x.h
+++ b/drivers/staging/wlan-ng/hfa384x.h
@@ -137,21 +137,11 @@
 #define		HFA384x_DLSTATE_FLASHENABLED		2
 
 /*--- Register Field Masks --------------------------*/
-#define		HFA384x_CMD_AINFO		((u16)(BIT(14) | BIT(13) \
-							| BIT(12) | BIT(11) \
-							| BIT(10) | BIT(9) \
-							| BIT(8)))
-#define		HFA384x_CMD_MACPORT		((u16)(BIT(10) | BIT(9) | \
-							BIT(8)))
-#define		HFA384x_CMD_PROGMODE		((u16)(BIT(9) | BIT(8)))
-#define		HFA384x_CMD_CMDCODE		((u16)(BIT(5) | BIT(4) | \
-							BIT(3) | BIT(2) | \
-							BIT(1) | BIT(0)))
-
-#define		HFA384x_STATUS_RESULT		((u16)(BIT(14) | BIT(13) \
-							| BIT(12) | BIT(11) \
-							| BIT(10) | BIT(9) \
-							| BIT(8)))
+#define		HFA384x_CMD_AINFO		((u16)GENMASK(14, 8))
+#define		HFA384x_CMD_MACPORT		((u16)GENMASK(10, 8))
+#define		HFA384x_CMD_PROGMODE		((u16)GENMASK(9, 8))
+#define		HFA384x_CMD_CMDCODE		((u16)GENMASK(5, 0))
+#define		HFA384x_STATUS_RESULT		((u16)GENMASK(14, 8))
 
 /*--- Command Code Constants --------------------------*/
 /*--- Controller Commands --------------------------*/
@@ -266,7 +256,7 @@
 #define		HFA384x_RID_DBMCOMMSQUALITY_LEN	 \
 	((u16)sizeof(struct hfa384x_dbmcommsquality))
 #define		HFA384x_RID_JOINREQUEST_LEN \
-	((u16)sizeof(struct hfa384x_JoinRequest_data))
+	((u16)sizeof(struct hfa384x_join_request_data))
 
 /*--------------------------------------------------------------------
  * Information RIDs:  Modem Information
@@ -286,7 +276,7 @@
 #define		HFA384x_RID_CNFWEPFLAGS		((u16)0xFC28)
 #define		HFA384x_RID_CNFAUTHENTICATION	((u16)0xFC2A)
 #define		HFA384x_RID_CNFROAMINGMODE	((u16)0xFC2D)
-#define		HFA384x_RID_CNFAPBCNint		((u16)0xFC33)
+#define		HFA384x_RID_CNFAPBCNINT		((u16)0xFC33)
 #define		HFA384x_RID_CNFDBMADJUST	((u16)0xFC46)
 #define		HFA384x_RID_CNFWPADATA		((u16)0xFC48)
 #define		HFA384x_RID_CNFBASICRATES	((u16)0xFCB3)
@@ -408,27 +398,27 @@ struct hfa384x_caplevel {
 #define HFA384x_CREATEIBSS_JOINCREATEIBSS          0
 
 /*-- Configuration Record: HostScanRequest (data portion only) --*/
-struct hfa384x_HostScanRequest_data {
-	u16 channelList;
-	u16 txRate;
+struct hfa384x_host_scan_request_data {
+	u16 channel_list;
+	u16 tx_rate;
 	struct hfa384x_bytestr32 ssid;
 } __packed;
 
 /*-- Configuration Record: JoinRequest (data portion only) --*/
-struct hfa384x_JoinRequest_data {
+struct hfa384x_join_request_data {
 	u8 bssid[WLAN_BSSID_LEN];
 	u16 channel;
 } __packed;
 
 /*-- Configuration Record: authenticateStation (data portion only) --*/
-struct hfa384x_authenticateStation_data {
+struct hfa384x_authenticate_station_data {
 	u8 address[ETH_ALEN];
 	u16 status;
 	u16 algorithm;
 } __packed;
 
 /*-- Configuration Record: WPAData       (data portion only) --*/
-struct hfa384x_WPAData {
+struct hfa384x_wpa_data {
 	u16 datalen;
 	u8 data[0];		/* max 80 */
 } __packed;
@@ -455,16 +445,16 @@ struct hfa384x_downloadbuffer {
 
 /*-- Information Record: commsquality --*/
 struct hfa384x_commsquality {
-	u16 CQ_currBSS;
-	u16 ASL_currBSS;
-	u16 ANL_currFC;
+	u16 cq_curr_bss;
+	u16 asl_curr_bss;
+	u16 anl_curr_fc;
 } __packed;
 
 /*-- Information Record: dmbcommsquality --*/
 struct hfa384x_dbmcommsquality {
-	u16 CQdbm_currBSS;
-	u16 ASLdbm_currBSS;
-	u16 ANLdbm_currFC;
+	u16 cq_dbm_curr_bss;
+	u16 asl_dbm_curr_bss;
+	u16 anl_dbm_curr_fc;
 } __packed;
 
 /*--------------------------------------------------------------------
@@ -511,9 +501,8 @@ struct hfa384x_tx_frame {
 #define		HFA384x_TXSTATUS_AGEDERR		((u16)BIT(1))
 #define		HFA384x_TXSTATUS_RETRYERR		((u16)BIT(0))
 /*-- Transmit Control Field --*/
-#define		HFA384x_TX_MACPORT			((u16)(BIT(10) | \
-							  BIT(9) | BIT(8)))
-#define		HFA384x_TX_STRUCTYPE			((u16)(BIT(4) | BIT(3)))
+#define		HFA384x_TX_MACPORT			((u16)GENMASK(10, 8))
+#define		HFA384x_TX_STRUCTYPE			((u16)GENMASK(4, 3))
 #define		HFA384x_TX_TXEX				((u16)BIT(2))
 #define		HFA384x_TX_TXOK				((u16)BIT(1))
 /*--------------------------------------------------------------------
@@ -571,9 +560,7 @@ struct hfa384x_rx_frame {
  */
 
 /*-- Status Fields --*/
-#define		HFA384x_RXSTATUS_MACPORT		((u16)(BIT(10) | \
-								BIT(9) | \
-								BIT(8)))
+#define		HFA384x_RXSTATUS_MACPORT		((u16)GENMASK(10, 8))
 #define		HFA384x_RXSTATUS_FCSERR			((u16)BIT(0))
 /*--------------------------------------------------------------------
  * Communication Frames: Test/Get/Set Field Values for Receive Frames
@@ -610,7 +597,7 @@ struct hfa384x_rx_frame {
  */
 
 /*--  Inquiry Frame, Diagnose: Communication Tallies --*/
-struct hfa384x_CommTallies16 {
+struct hfa384x_comm_tallies_16 {
 	u16 txunicastframes;
 	u16 txmulticastframes;
 	u16 txfragments;
@@ -634,7 +621,7 @@ struct hfa384x_CommTallies16 {
 	u16 rxmsginbadmsgfrag;
 } __packed;
 
-struct hfa384x_CommTallies32 {
+struct hfa384x_comm_tallies_32 {
 	u32 txunicastframes;
 	u32 txmulticastframes;
 	u32 txfragments;
@@ -659,7 +646,7 @@ struct hfa384x_CommTallies32 {
 } __packed;
 
 /*--  Inquiry Frame, Diagnose: Scan Results & Subfields--*/
-struct hfa384x_ScanResultSub {
+struct hfa384x_scan_result_sub {
 	u16 chid;
 	u16 anl;
 	u16 sl;
@@ -671,14 +658,14 @@ struct hfa384x_ScanResultSub {
 	u16 proberesp_rate;
 } __packed;
 
-struct hfa384x_ScanResult {
+struct hfa384x_scan_result {
 	u16 rsvd;
 	u16 scanreason;
-	struct hfa384x_ScanResultSub result[HFA384x_SCANRESULT_MAX];
+	struct hfa384x_scan_result_sub result[HFA384x_SCANRESULT_MAX];
 } __packed;
 
 /*--  Inquiry Frame, Diagnose: ChInfo Results & Subfields--*/
-struct hfa384x_ChInfoResultSub {
+struct hfa384x_ch_info_result_sub {
 	u16 chid;
 	u16 anl;
 	u16 pnl;
@@ -688,13 +675,13 @@ struct hfa384x_ChInfoResultSub {
 #define HFA384x_CHINFORESULT_BSSACTIVE	BIT(0)
 #define HFA384x_CHINFORESULT_PCFACTIVE	BIT(1)
 
-struct hfa384x_ChInfoResult {
+struct hfa384x_ch_info_result {
 	u16 scanchannels;
-	struct hfa384x_ChInfoResultSub result[HFA384x_CHINFORESULT_MAX];
+	struct hfa384x_ch_info_result_sub result[HFA384x_CHINFORESULT_MAX];
 } __packed;
 
 /*--  Inquiry Frame, Diagnose: Host Scan Results & Subfields--*/
-struct hfa384x_HScanResultSub {
+struct hfa384x_hscan_result_sub {
 	u16 chid;
 	u16 anl;
 	u16 sl;
@@ -707,10 +694,10 @@ struct hfa384x_HScanResultSub {
 	u16 atim;
 } __packed;
 
-struct hfa384x_HScanResult {
+struct hfa384x_hscan_result {
 	u16 nresult;
 	u16 rsvd;
-	struct hfa384x_HScanResultSub result[HFA384x_HSCANRESULT_MAX];
+	struct hfa384x_hscan_result_sub result[HFA384x_HSCANRESULT_MAX];
 } __packed;
 
 /*--  Unsolicited Frame, MAC Mgmt: LinkStatus --*/
@@ -723,7 +710,7 @@ struct hfa384x_HScanResult {
 #define HFA384x_LINK_AP_INRANGE		((u16)5)
 #define HFA384x_LINK_ASSOCFAIL		((u16)6)
 
-struct hfa384x_LinkStatus {
+struct hfa384x_link_status {
 	u16 linkstatus;
 } __packed;
 
@@ -733,7 +720,7 @@ struct hfa384x_LinkStatus {
 #define HFA384x_ASSOCSTATUS_REASSOC	((u16)2)
 #define HFA384x_ASSOCSTATUS_AUTHFAIL	((u16)5)
 
-struct hfa384x_AssocStatus {
+struct hfa384x_assoc_status {
 	u16 assocstatus;
 	u8 sta_addr[ETH_ALEN];
 	/* old_ap_addr is only valid if assocstatus == 2 */
@@ -744,37 +731,37 @@ struct hfa384x_AssocStatus {
 
 /*--  Unsolicited Frame, MAC Mgmt: AuthRequest (AP Only) --*/
 
-struct hfa384x_AuthRequest {
+struct hfa384x_auth_request {
 	u8 sta_addr[ETH_ALEN];
 	u16 algorithm;
 } __packed;
 
 /*--  Unsolicited Frame, MAC Mgmt: PSUserCount (AP Only) --*/
 
-struct hfa384x_PSUserCount {
+struct hfa384x_ps_user_count {
 	u16 usercnt;
 } __packed;
 
-struct hfa384x_KeyIDChanged {
+struct hfa384x_key_id_changed {
 	u8 sta_addr[ETH_ALEN];
 	u16 keyid;
 } __packed;
 
 /*--  Collection of all Inf frames ---------------*/
 union hfa384x_infodata {
-	struct hfa384x_CommTallies16 commtallies16;
-	struct hfa384x_CommTallies32 commtallies32;
-	struct hfa384x_ScanResult scanresult;
-	struct hfa384x_ChInfoResult chinforesult;
-	struct hfa384x_HScanResult hscanresult;
-	struct hfa384x_LinkStatus linkstatus;
-	struct hfa384x_AssocStatus assocstatus;
-	struct hfa384x_AuthRequest authreq;
-	struct hfa384x_PSUserCount psusercnt;
-	struct hfa384x_KeyIDChanged keyidchanged;
-} __packed;
-
-struct hfa384x_InfFrame {
+	struct hfa384x_comm_tallies_16 commtallies16;
+	struct hfa384x_comm_tallies_32 commtallies32;
+	struct hfa384x_scan_result scanresult;
+	struct hfa384x_ch_info_result chinforesult;
+	struct hfa384x_hscan_result hscanresult;
+	struct hfa384x_link_status linkstatus;
+	struct hfa384x_assoc_status assocstatus;
+	struct hfa384x_auth_request authreq;
+	struct hfa384x_ps_user_count psusercnt;
+	struct hfa384x_key_id_changed keyidchanged;
+} __packed;
+
+struct hfa384x_inf_frame {
 	u16 framelen;
 	u16 infotype;
 	union hfa384x_infodata info;
@@ -862,7 +849,7 @@ struct hfa384x_usb_rxfrm {
 
 struct hfa384x_usb_infofrm {
 	u16 type;
-	struct hfa384x_InfFrame info;
+	struct hfa384x_inf_frame info;
 } __packed;
 
 struct hfa384x_usb_statusresp {
@@ -1169,7 +1156,6 @@ enum ctlx_state {
 	CTLX_REQ_COMPLETE,	/* OUT URB complete */
 	CTLX_RESP_COMPLETE	/* IN URB received */
 };
-typedef enum ctlx_state CTLX_STATE;
 
 struct hfa384x_usbctlx;
 struct hfa384x;
@@ -1186,7 +1172,7 @@ struct hfa384x_usbctlx {
 	union hfa384x_usbout outbuf;	/* pkt buf for OUT */
 	union hfa384x_usbin inbuf;	/* pkt buf for IN(a copy) */
 
-	CTLX_STATE state;	/* Tracks running state */
+	enum ctlx_state state;	/* Tracks running state */
 
 	struct completion done;
 	volatile int reapable;	/* Food for the reaper task */
@@ -1294,7 +1280,7 @@ struct hfa384x {
 	int scanflag;		/* to signal scan complete */
 	int join_ap;		/* are we joined to a specific ap */
 	int join_retries;	/* number of join retries till we fail */
-	struct hfa384x_JoinRequest_data joinreq;	/* join request saved data */
+	struct hfa384x_join_request_data joinreq;/* join request saved data */
 
 	struct wlandevice *wlandev;
 	/* Timer to allow for the deferred processing of linkstatus messages */
@@ -1360,17 +1346,17 @@ struct hfa384x {
 	struct hfa384x_caplevel cap_act_ap_mfi;	/* ap f/w to modem interface */
 
 	u32 psusercount;	/* Power save user count. */
-	struct hfa384x_CommTallies32 tallies;	/* Communication tallies. */
+	struct hfa384x_comm_tallies_32 tallies;	/* Communication tallies. */
 	u8 comment[WLAN_COMMENT_MAX + 1];	/* User comment */
 
 	/* Channel Info request results (AP only) */
 	struct {
 		atomic_t done;
 		u8 count;
-		struct hfa384x_ChInfoResult results;
+		struct hfa384x_ch_info_result results;
 	} channel_info;
 
-	struct hfa384x_InfFrame *scanresults;
+	struct hfa384x_inf_frame *scanresults;
 
 	struct prism2sta_authlist authlist;	/* Authenticated station list. */
 	unsigned int accessmode;		/* Access mode. */
diff --git a/drivers/staging/wlan-ng/hfa384x_usb.c b/drivers/staging/wlan-ng/hfa384x_usb.c
index 6a107f8a06e2..4fe037aeef12 100644
--- a/drivers/staging/wlan-ng/hfa384x_usb.c
+++ b/drivers/staging/wlan-ng/hfa384x_usb.c
@@ -1,114 +1,114 @@
 /* src/prism2/driver/hfa384x_usb.c
-*
-* Functions that talk to the USB variantof the Intersil hfa384x MAC
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-* This file implements functions that correspond to the prism2/hfa384x
-* 802.11 MAC hardware and firmware host interface.
-*
-* The functions can be considered to represent several levels of
-* abstraction.  The lowest level functions are simply C-callable wrappers
-* around the register accesses.  The next higher level represents C-callable
-* prism2 API functions that match the Intersil documentation as closely
-* as is reasonable.  The next higher layer implements common sequences
-* of invocations of the API layer (e.g. write to bap, followed by cmd).
-*
-* Common sequences:
-* hfa384x_drvr_xxx	Highest level abstractions provided by the
-*			hfa384x code.  They are driver defined wrappers
-*			for common sequences.  These functions generally
-*			use the services of the lower levels.
-*
-* hfa384x_drvr_xxxconfig  An example of the drvr level abstraction. These
-*			functions are wrappers for the RID get/set
-*			sequence. They call copy_[to|from]_bap() and
-*			cmd_access(). These functions operate on the
-*			RIDs and buffers without validation. The caller
-*			is responsible for that.
-*
-* API wrapper functions:
-* hfa384x_cmd_xxx	functions that provide access to the f/w commands.
-*			The function arguments correspond to each command
-*			argument, even command arguments that get packed
-*			into single registers.  These functions _just_
-*			issue the command by setting the cmd/parm regs
-*			& reading the status/resp regs.  Additional
-*			activities required to fully use a command
-*			(read/write from/to bap, get/set int status etc.)
-*			are implemented separately.  Think of these as
-*			C-callable prism2 commands.
-*
-* Lowest Layer Functions:
-* hfa384x_docmd_xxx	These functions implement the sequence required
-*			to issue any prism2 command.  Primarily used by the
-*			hfa384x_cmd_xxx functions.
-*
-* hfa384x_bap_xxx	BAP read/write access functions.
-*			Note: we usually use BAP0 for non-interrupt context
-*			 and BAP1 for interrupt context.
-*
-* hfa384x_dl_xxx	download related functions.
-*
-* Driver State Issues:
-* Note that there are two pairs of functions that manage the
-* 'initialized' and 'running' states of the hw/MAC combo.  The four
-* functions are create(), destroy(), start(), and stop().  create()
-* sets up the data structures required to support the hfa384x_*
-* functions and destroy() cleans them up.  The start() function gets
-* the actual hardware running and enables the interrupts.  The stop()
-* function shuts the hardware down.  The sequence should be:
-* create()
-* start()
-*  .
-*  .  Do interesting things w/ the hardware
-*  .
-* stop()
-* destroy()
-*
-* Note that destroy() can be called without calling stop() first.
-* --------------------------------------------------------------------
-*/
+ *
+ * Functions that talk to the USB variantof the Intersil hfa384x MAC
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ * This file implements functions that correspond to the prism2/hfa384x
+ * 802.11 MAC hardware and firmware host interface.
+ *
+ * The functions can be considered to represent several levels of
+ * abstraction.  The lowest level functions are simply C-callable wrappers
+ * around the register accesses.  The next higher level represents C-callable
+ * prism2 API functions that match the Intersil documentation as closely
+ * as is reasonable.  The next higher layer implements common sequences
+ * of invocations of the API layer (e.g. write to bap, followed by cmd).
+ *
+ * Common sequences:
+ * hfa384x_drvr_xxx	Highest level abstractions provided by the
+ *			hfa384x code.  They are driver defined wrappers
+ *			for common sequences.  These functions generally
+ *			use the services of the lower levels.
+ *
+ * hfa384x_drvr_xxxconfig  An example of the drvr level abstraction. These
+ *			functions are wrappers for the RID get/set
+ *			sequence. They call copy_[to|from]_bap() and
+ *			cmd_access(). These functions operate on the
+ *			RIDs and buffers without validation. The caller
+ *			is responsible for that.
+ *
+ * API wrapper functions:
+ * hfa384x_cmd_xxx	functions that provide access to the f/w commands.
+ *			The function arguments correspond to each command
+ *			argument, even command arguments that get packed
+ *			into single registers.  These functions _just_
+ *			issue the command by setting the cmd/parm regs
+ *			& reading the status/resp regs.  Additional
+ *			activities required to fully use a command
+ *			(read/write from/to bap, get/set int status etc.)
+ *			are implemented separately.  Think of these as
+ *			C-callable prism2 commands.
+ *
+ * Lowest Layer Functions:
+ * hfa384x_docmd_xxx	These functions implement the sequence required
+ *			to issue any prism2 command.  Primarily used by the
+ *			hfa384x_cmd_xxx functions.
+ *
+ * hfa384x_bap_xxx	BAP read/write access functions.
+ *			Note: we usually use BAP0 for non-interrupt context
+ *			 and BAP1 for interrupt context.
+ *
+ * hfa384x_dl_xxx	download related functions.
+ *
+ * Driver State Issues:
+ * Note that there are two pairs of functions that manage the
+ * 'initialized' and 'running' states of the hw/MAC combo.  The four
+ * functions are create(), destroy(), start(), and stop().  create()
+ * sets up the data structures required to support the hfa384x_*
+ * functions and destroy() cleans them up.  The start() function gets
+ * the actual hardware running and enables the interrupts.  The stop()
+ * function shuts the hardware down.  The sequence should be:
+ * create()
+ * start()
+ *  .
+ *  .  Do interesting things w/ the hardware
+ *  .
+ * stop()
+ * destroy()
+ *
+ * Note that destroy() can be called without calling stop() first.
+ * --------------------------------------------------------------------
+ */
 
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -153,8 +153,8 @@ enum cmd_mode {
 static void dbprint_urb(struct urb *urb);
 #endif
 
-static void
-hfa384x_int_rxmonitor(struct wlandevice *wlandev, struct hfa384x_usb_rxfrm *rxfrm);
+static void hfa384x_int_rxmonitor(struct wlandevice *wlandev,
+				  struct hfa384x_usb_rxfrm *rxfrm);
 
 static void hfa384x_usb_defer(struct work_struct *data);
 
@@ -173,7 +173,8 @@ hfa384x_usbin_txcompl(struct wlandevice *wlandev, union hfa384x_usbin *usbin);
 
 static void hfa384x_usbin_rx(struct wlandevice *wlandev, struct sk_buff *skb);
 
-static void hfa384x_usbin_info(struct wlandevice *wlandev, union hfa384x_usbin *usbin);
+static void hfa384x_usbin_info(struct wlandevice *wlandev,
+			       union hfa384x_usbin *usbin);
 
 static void hfa384x_usbin_ctlx(struct hfa384x *hw, union hfa384x_usbin *usbin,
 			       int urb_status);
@@ -193,9 +194,11 @@ static void hfa384x_usbctlx_completion_task(unsigned long data);
 
 static void hfa384x_usbctlx_reaper_task(unsigned long data);
 
-static int hfa384x_usbctlx_submit(struct hfa384x *hw, struct hfa384x_usbctlx *ctlx);
+static int hfa384x_usbctlx_submit(struct hfa384x *hw,
+				  struct hfa384x_usbctlx *ctlx);
 
-static void unlocked_usbctlx_complete(struct hfa384x *hw, struct hfa384x_usbctlx *ctlx);
+static void unlocked_usbctlx_complete(struct hfa384x *hw,
+				      struct hfa384x_usbctlx *ctlx);
 
 struct usbctlx_completor {
 	int (*complete)(struct usbctlx_completor *);
@@ -209,7 +212,8 @@ hfa384x_usbctlx_complete_sync(struct hfa384x *hw,
 static int
 unlocked_usbctlx_cancel_async(struct hfa384x *hw, struct hfa384x_usbctlx *ctlx);
 
-static void hfa384x_cb_status(struct hfa384x *hw, const struct hfa384x_usbctlx *ctlx);
+static void hfa384x_cb_status(struct hfa384x *hw,
+			      const struct hfa384x_usbctlx *ctlx);
 
 static int
 usbctlx_get_status(const struct hfa384x_usb_statusresp *cmdresp,
@@ -263,7 +267,7 @@ hfa384x_dowmem(struct hfa384x *hw,
 
 static int hfa384x_isgood_pdrcode(u16 pdrcode);
 
-static inline const char *ctlxstr(CTLX_STATE s)
+static inline const char *ctlxstr(enum ctlx_state s)
 {
 	static const char * const ctlx_str[] = {
 		"Initial state",
@@ -307,21 +311,22 @@ void dbprint_urb(struct urb *urb)
 #endif
 
 /*----------------------------------------------------------------
-* submit_rx_urb
-*
-* Listen for input data on the BULK-IN pipe. If the pipe has
-* stalled then schedule it to be reset.
-*
-* Arguments:
-*	hw		device struct
-*	memflags	memory allocation flags
-*
-* Returns:
-*	error code from submission
-*
-* Call context:
-*	Any
-----------------------------------------------------------------*/
+ * submit_rx_urb
+ *
+ * Listen for input data on the BULK-IN pipe. If the pipe has
+ * stalled then schedule it to be reset.
+ *
+ * Arguments:
+ *	hw		device struct
+ *	memflags	memory allocation flags
+ *
+ * Returns:
+ *	error code from submission
+ *
+ * Call context:
+ *	Any
+ *----------------------------------------------------------------
+ */
 static int submit_rx_urb(struct hfa384x *hw, gfp_t memflags)
 {
 	struct sk_buff *skb;
@@ -367,23 +372,24 @@ done:
 }
 
 /*----------------------------------------------------------------
-* submit_tx_urb
-*
-* Prepares and submits the URB of transmitted data. If the
-* submission fails then it will schedule the output pipe to
-* be reset.
-*
-* Arguments:
-*	hw		device struct
-*	tx_urb		URB of data for transmission
-*	memflags	memory allocation flags
-*
-* Returns:
-*	error code from submission
-*
-* Call context:
-*	Any
-----------------------------------------------------------------*/
+ * submit_tx_urb
+ *
+ * Prepares and submits the URB of transmitted data. If the
+ * submission fails then it will schedule the output pipe to
+ * be reset.
+ *
+ * Arguments:
+ *	hw		device struct
+ *	tx_urb		URB of data for transmission
+ *	memflags	memory allocation flags
+ *
+ * Returns:
+ *	error code from submission
+ *
+ * Call context:
+ *	Any
+ *----------------------------------------------------------------
+ */
 static int submit_tx_urb(struct hfa384x *hw, struct urb *tx_urb, gfp_t memflags)
 {
 	struct net_device *netdev = hw->wlandev->netdev;
@@ -412,21 +418,22 @@ static int submit_tx_urb(struct hfa384x *hw, struct urb *tx_urb, gfp_t memflags)
 }
 
 /*----------------------------------------------------------------
-* hfa394x_usb_defer
-*
-* There are some things that the USB stack cannot do while
-* in interrupt context, so we arrange this function to run
-* in process context.
-*
-* Arguments:
-*	hw	device structure
-*
-* Returns:
-*	nothing
-*
-* Call context:
-*	process (by design)
-----------------------------------------------------------------*/
+ * hfa394x_usb_defer
+ *
+ * There are some things that the USB stack cannot do while
+ * in interrupt context, so we arrange this function to run
+ * in process context.
+ *
+ * Arguments:
+ *	hw	device structure
+ *
+ * Returns:
+ *	nothing
+ *
+ * Call context:
+ *	process (by design)
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usb_defer(struct work_struct *data)
 {
 	struct hfa384x *hw = container_of(data, struct hfa384x, usb_work);
@@ -501,29 +508,30 @@ static void hfa384x_usb_defer(struct work_struct *data)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_create
-*
-* Sets up the struct hfa384x data structure for use.  Note this
-* does _not_ initialize the actual hardware, just the data structures
-* we use to keep track of its state.
-*
-* Arguments:
-*	hw		device structure
-*	irq		device irq number
-*	iobase		i/o base address for register access
-*	membase		memory base address for register access
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_create
+ *
+ * Sets up the struct hfa384x data structure for use.  Note this
+ * does _not_ initialize the actual hardware, just the data structures
+ * we use to keep track of its state.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	irq		device irq number
+ *	iobase		i/o base address for register access
+ *	membase		memory base address for register access
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 void hfa384x_create(struct hfa384x *hw, struct usb_device *usb)
 {
-	memset(hw, 0, sizeof(struct hfa384x));
+	memset(hw, 0, sizeof(*hw));
 	hw->usb = usb;
 
 	/* set up the endpoints */
@@ -571,27 +579,28 @@ void hfa384x_create(struct hfa384x *hw, struct usb_device *usb)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_destroy
-*
-* Partner to hfa384x_create().  This function cleans up the hw
-* structure so that it can be freed by the caller using a simple
-* kfree.  Currently, this function is just a placeholder.  If, at some
-* point in the future, an hw in the 'shutdown' state requires a 'deep'
-* kfree, this is where it should be done.  Note that if this function
-* is called on a _running_ hw structure, the drvr_stop() function is
-* called.
-*
-* Arguments:
-*	hw		device structure
-*
-* Returns:
-*	nothing, this function is not allowed to fail.
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_destroy
+ *
+ * Partner to hfa384x_create().  This function cleans up the hw
+ * structure so that it can be freed by the caller using a simple
+ * kfree.  Currently, this function is just a placeholder.  If, at some
+ * point in the future, an hw in the 'shutdown' state requires a 'deep'
+ * kfree, this is where it should be done.  Note that if this function
+ * is called on a _running_ hw structure, the drvr_stop() function is
+ * called.
+ *
+ * Arguments:
+ *	hw		device structure
+ *
+ * Returns:
+ *	nothing, this function is not allowed to fail.
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 void hfa384x_destroy(struct hfa384x *hw)
 {
 	struct sk_buff *skb;
@@ -645,10 +654,11 @@ usbctlx_get_rridresult(const struct hfa384x_usb_rridresp *rridresp,
 }
 
 /*----------------------------------------------------------------
-* Completor object:
-* This completor must be passed to hfa384x_usbctlx_complete_sync()
-* when processing a CTLX that returns a struct hfa384x_cmdresult structure.
-----------------------------------------------------------------*/
+ * Completor object:
+ * This completor must be passed to hfa384x_usbctlx_complete_sync()
+ * when processing a CTLX that returns a struct hfa384x_cmdresult structure.
+ *----------------------------------------------------------------
+ */
 struct usbctlx_cmd_completor {
 	struct usbctlx_completor head;
 
@@ -664,24 +674,23 @@ static inline int usbctlx_cmd_completor_fn(struct usbctlx_completor *head)
 	return usbctlx_get_status(complete->cmdresp, complete->result);
 }
 
-static inline struct usbctlx_completor *init_cmd_completor(
-						struct usbctlx_cmd_completor
-							*completor,
-						const struct hfa384x_usb_statusresp
-							*cmdresp,
-						struct hfa384x_cmdresult *result)
+static inline struct usbctlx_completor *
+init_cmd_completor(struct usbctlx_cmd_completor *completor,
+		   const struct hfa384x_usb_statusresp *cmdresp,
+		   struct hfa384x_cmdresult *result)
 {
 	completor->head.complete = usbctlx_cmd_completor_fn;
 	completor->cmdresp = cmdresp;
 	completor->result = result;
-	return &(completor->head);
+	return &completor->head;
 }
 
 /*----------------------------------------------------------------
-* Completor object:
-* This completor must be passed to hfa384x_usbctlx_complete_sync()
-* when processing a CTLX that reads a RID.
-----------------------------------------------------------------*/
+ * Completor object:
+ * This completor must be passed to hfa384x_usbctlx_complete_sync()
+ * when processing a CTLX that reads a RID.
+ *----------------------------------------------------------------
+ */
 struct usbctlx_rrid_completor {
 	struct usbctlx_completor head;
 
@@ -710,37 +719,38 @@ static int usbctlx_rrid_completor_fn(struct usbctlx_completor *head)
 	return 0;
 }
 
-static inline struct usbctlx_completor *init_rrid_completor(
-						struct usbctlx_rrid_completor
-							*completor,
-						const struct hfa384x_usb_rridresp
-							*rridresp,
-						void *riddata,
-						unsigned int riddatalen)
+static inline struct usbctlx_completor *
+init_rrid_completor(struct usbctlx_rrid_completor *completor,
+		    const struct hfa384x_usb_rridresp *rridresp,
+		    void *riddata,
+		    unsigned int riddatalen)
 {
 	completor->head.complete = usbctlx_rrid_completor_fn;
 	completor->rridresp = rridresp;
 	completor->riddata = riddata;
 	completor->riddatalen = riddatalen;
-	return &(completor->head);
+	return &completor->head;
 }
 
 /*----------------------------------------------------------------
-* Completor object:
-* Interprets the results of a synchronous RID-write
-----------------------------------------------------------------*/
+ * Completor object:
+ * Interprets the results of a synchronous RID-write
+ *----------------------------------------------------------------
+ */
 #define init_wrid_completor  init_cmd_completor
 
 /*----------------------------------------------------------------
-* Completor object:
-* Interprets the results of a synchronous memory-write
-----------------------------------------------------------------*/
+ * Completor object:
+ * Interprets the results of a synchronous memory-write
+ *----------------------------------------------------------------
+ */
 #define init_wmem_completor  init_cmd_completor
 
 /*----------------------------------------------------------------
-* Completor object:
-* Interprets the results of a synchronous memory-read
-----------------------------------------------------------------*/
+ * Completor object:
+ * Interprets the results of a synchronous memory-read
+ *----------------------------------------------------------------
+ */
 struct usbctlx_rmem_completor {
 	struct usbctlx_completor head;
 
@@ -759,43 +769,43 @@ static int usbctlx_rmem_completor_fn(struct usbctlx_completor *head)
 	return 0;
 }
 
-static inline struct usbctlx_completor *init_rmem_completor(
-						struct usbctlx_rmem_completor
-							*completor,
-						struct hfa384x_usb_rmemresp
-							*rmemresp,
-						void *data,
-						unsigned int len)
+static inline struct usbctlx_completor *
+init_rmem_completor(struct usbctlx_rmem_completor *completor,
+		    struct hfa384x_usb_rmemresp *rmemresp,
+		    void *data,
+		    unsigned int len)
 {
 	completor->head.complete = usbctlx_rmem_completor_fn;
 	completor->rmemresp = rmemresp;
 	completor->data = data;
 	completor->len = len;
-	return &(completor->head);
+	return &completor->head;
 }
 
 /*----------------------------------------------------------------
-* hfa384x_cb_status
-*
-* Ctlx_complete handler for async CMD type control exchanges.
-* mark the hw struct as such.
-*
-* Note: If the handling is changed here, it should probably be
-*       changed in docmd as well.
-*
-* Arguments:
-*	hw		hw struct
-*	ctlx		completed CTLX
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	interrupt
-----------------------------------------------------------------*/
-static void hfa384x_cb_status(struct hfa384x *hw, const struct hfa384x_usbctlx *ctlx)
+ * hfa384x_cb_status
+ *
+ * Ctlx_complete handler for async CMD type control exchanges.
+ * mark the hw struct as such.
+ *
+ * Note: If the handling is changed here, it should probably be
+ *       changed in docmd as well.
+ *
+ * Arguments:
+ *	hw		hw struct
+ *	ctlx		completed CTLX
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt
+ *----------------------------------------------------------------
+ */
+static void hfa384x_cb_status(struct hfa384x *hw,
+			      const struct hfa384x_usbctlx *ctlx)
 {
 	if (ctlx->usercb) {
 		struct hfa384x_cmdresult cmdresult;
@@ -812,7 +822,8 @@ static void hfa384x_cb_status(struct hfa384x *hw, const struct hfa384x_usbctlx *
 	}
 }
 
-static inline int hfa384x_docmd_wait(struct hfa384x *hw, struct hfa384x_metacmd *cmd)
+static inline int hfa384x_docmd_wait(struct hfa384x *hw,
+				     struct hfa384x_metacmd *cmd)
 {
 	return hfa384x_docmd(hw, DOWAIT, cmd, NULL, NULL, NULL);
 }
@@ -905,24 +916,25 @@ hfa384x_dowmem_async(struct hfa384x *hw,
 }
 
 /*----------------------------------------------------------------
-* hfa384x_cmd_initialize
-*
-* Issues the initialize command and sets the hw->state based
-* on the result.
-*
-* Arguments:
-*	hw		device structure
-*
-* Returns:
-*	0		success
-*	>0		f/w reported error - f/w status code
-*	<0		driver reported error
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_cmd_initialize
+ *
+ * Issues the initialize command and sets the hw->state based
+ * on the result.
+ *
+ * Arguments:
+ *	hw		device structure
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported error - f/w status code
+ *	<0		driver reported error
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_cmd_initialize(struct hfa384x *hw)
 {
 	int result = 0;
@@ -950,25 +962,26 @@ int hfa384x_cmd_initialize(struct hfa384x *hw)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_cmd_disable
-*
-* Issues the disable command to stop communications on one of
-* the MACs 'ports'.
-*
-* Arguments:
-*	hw		device structure
-*	macport		MAC port number (host order)
-*
-* Returns:
-*	0		success
-*	>0		f/w reported failure - f/w status code
-*	<0		driver reported error (timeout|bad arg)
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_cmd_disable
+ *
+ * Issues the disable command to stop communications on one of
+ * the MACs 'ports'.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	macport		MAC port number (host order)
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported failure - f/w status code
+ *	<0		driver reported error (timeout|bad arg)
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_cmd_disable(struct hfa384x *hw, u16 macport)
 {
 	struct hfa384x_metacmd cmd;
@@ -983,25 +996,26 @@ int hfa384x_cmd_disable(struct hfa384x *hw, u16 macport)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_cmd_enable
-*
-* Issues the enable command to enable communications on one of
-* the MACs 'ports'.
-*
-* Arguments:
-*	hw		device structure
-*	macport		MAC port number
-*
-* Returns:
-*	0		success
-*	>0		f/w reported failure - f/w status code
-*	<0		driver reported error (timeout|bad arg)
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_cmd_enable
+ *
+ * Issues the enable command to enable communications on one of
+ * the MACs 'ports'.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	macport		MAC port number
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported failure - f/w status code
+ *	<0		driver reported error (timeout|bad arg)
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_cmd_enable(struct hfa384x *hw, u16 macport)
 {
 	struct hfa384x_metacmd cmd;
@@ -1016,34 +1030,35 @@ int hfa384x_cmd_enable(struct hfa384x *hw, u16 macport)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_cmd_monitor
-*
-* Enables the 'monitor mode' of the MAC.  Here's the description of
-* monitor mode that I've received thus far:
-*
-*  "The "monitor mode" of operation is that the MAC passes all
-*  frames for which the PLCP checks are correct. All received
-*  MPDUs are passed to the host with MAC Port = 7, with a
-*  receive status of good, FCS error, or undecryptable. Passing
-*  certain MPDUs is a violation of the 802.11 standard, but useful
-*  for a debugging tool."  Normal communication is not possible
-*  while monitor mode is enabled.
-*
-* Arguments:
-*	hw		device structure
-*	enable		a code (0x0b|0x0f) that enables/disables
-*			monitor mode. (host order)
-*
-* Returns:
-*	0		success
-*	>0		f/w reported failure - f/w status code
-*	<0		driver reported error (timeout|bad arg)
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_cmd_monitor
+ *
+ * Enables the 'monitor mode' of the MAC.  Here's the description of
+ * monitor mode that I've received thus far:
+ *
+ *  "The "monitor mode" of operation is that the MAC passes all
+ *  frames for which the PLCP checks are correct. All received
+ *  MPDUs are passed to the host with MAC Port = 7, with a
+ *  receive status of good, FCS error, or undecryptable. Passing
+ *  certain MPDUs is a violation of the 802.11 standard, but useful
+ *  for a debugging tool."  Normal communication is not possible
+ *  while monitor mode is enabled.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	enable		a code (0x0b|0x0f) that enables/disables
+ *			monitor mode. (host order)
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported failure - f/w status code
+ *	<0		driver reported error (timeout|bad arg)
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_cmd_monitor(struct hfa384x *hw, u16 enable)
 {
 	struct hfa384x_metacmd cmd;
@@ -1058,43 +1073,44 @@ int hfa384x_cmd_monitor(struct hfa384x *hw, u16 enable)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_cmd_download
-*
-* Sets the controls for the MAC controller code/data download
-* process.  The arguments set the mode and address associated
-* with a download.  Note that the aux registers should be enabled
-* prior to setting one of the download enable modes.
-*
-* Arguments:
-*	hw		device structure
-*	mode		0 - Disable programming and begin code exec
-*			1 - Enable volatile mem programming
-*			2 - Enable non-volatile mem programming
-*			3 - Program non-volatile section from NV download
-*			    buffer.
-*			(host order)
-*	lowaddr
-*	highaddr	For mode 1, sets the high & low order bits of
-*			the "destination address".  This address will be
-*			the execution start address when download is
-*			subsequently disabled.
-*			For mode 2, sets the high & low order bits of
-*			the destination in NV ram.
-*			For modes 0 & 3, should be zero. (host order)
-*			NOTE: these are CMD format.
-*	codelen		Length of the data to write in mode 2,
-*			zero otherwise. (host order)
-*
-* Returns:
-*	0		success
-*	>0		f/w reported failure - f/w status code
-*	<0		driver reported error (timeout|bad arg)
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_cmd_download
+ *
+ * Sets the controls for the MAC controller code/data download
+ * process.  The arguments set the mode and address associated
+ * with a download.  Note that the aux registers should be enabled
+ * prior to setting one of the download enable modes.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	mode		0 - Disable programming and begin code exec
+ *			1 - Enable volatile mem programming
+ *			2 - Enable non-volatile mem programming
+ *			3 - Program non-volatile section from NV download
+ *			    buffer.
+ *			(host order)
+ *	lowaddr
+ *	highaddr	For mode 1, sets the high & low order bits of
+ *			the "destination address".  This address will be
+ *			the execution start address when download is
+ *			subsequently disabled.
+ *			For mode 2, sets the high & low order bits of
+ *			the destination in NV ram.
+ *			For modes 0 & 3, should be zero. (host order)
+ *			NOTE: these are CMD format.
+ *	codelen		Length of the data to write in mode 2,
+ *			zero otherwise. (host order)
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported failure - f/w status code
+ *	<0		driver reported error (timeout|bad arg)
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_cmd_download(struct hfa384x *hw, u16 mode, u16 lowaddr,
 			 u16 highaddr, u16 codelen)
 {
@@ -1114,29 +1130,31 @@ int hfa384x_cmd_download(struct hfa384x *hw, u16 mode, u16 lowaddr,
 }
 
 /*----------------------------------------------------------------
-* hfa384x_corereset
-*
-* Perform a reset of the hfa38xx MAC core.  We assume that the hw
-* structure is in its "created" state.  That is, it is initialized
-* with proper values.  Note that if a reset is done after the
-* device has been active for awhile, the caller might have to clean
-* up some leftover cruft in the hw structure.
-*
-* Arguments:
-*	hw		device structure
-*	holdtime	how long (in ms) to hold the reset
-*	settletime	how long (in ms) to wait after releasing
-*			the reset
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
-int hfa384x_corereset(struct hfa384x *hw, int holdtime, int settletime, int genesis)
+ * hfa384x_corereset
+ *
+ * Perform a reset of the hfa38xx MAC core.  We assume that the hw
+ * structure is in its "created" state.  That is, it is initialized
+ * with proper values.  Note that if a reset is done after the
+ * device has been active for awhile, the caller might have to clean
+ * up some leftover cruft in the hw structure.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	holdtime	how long (in ms) to hold the reset
+ *	settletime	how long (in ms) to wait after releasing
+ *			the reset
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
+int hfa384x_corereset(struct hfa384x *hw, int holdtime,
+		      int settletime, int genesis)
 {
 	int result;
 
@@ -1150,29 +1168,30 @@ int hfa384x_corereset(struct hfa384x *hw, int holdtime, int settletime, int gene
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbctlx_complete_sync
-*
-* Waits for a synchronous CTLX object to complete,
-* and then handles the response.
-*
-* Arguments:
-*	hw		device structure
-*	ctlx		CTLX ptr
-*	completor	functor object to decide what to
-*			do with the CTLX's result.
-*
-* Returns:
-*	0		Success
-*	-ERESTARTSYS	Interrupted by a signal
-*	-EIO		CTLX failed
-*	-ENODEV		Adapter was unplugged
-*	???		Result from completor
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_usbctlx_complete_sync
+ *
+ * Waits for a synchronous CTLX object to complete,
+ * and then handles the response.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	ctlx		CTLX ptr
+ *	completor	functor object to decide what to
+ *			do with the CTLX's result.
+ *
+ * Returns:
+ *	0		Success
+ *	-ERESTARTSYS	Interrupted by a signal
+ *	-EIO		CTLX failed
+ *	-ENODEV		Adapter was unplugged
+ *	???		Result from completor
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 static int hfa384x_usbctlx_complete_sync(struct hfa384x *hw,
 					 struct hfa384x_usbctlx *ctlx,
 					 struct usbctlx_completor *completor)
@@ -1257,37 +1276,38 @@ cleanup:
 }
 
 /*----------------------------------------------------------------
-* hfa384x_docmd
-*
-* Constructs a command CTLX and submits it.
-*
-* NOTE: Any changes to the 'post-submit' code in this function
-*       need to be carried over to hfa384x_cbcmd() since the handling
-*       is virtually identical.
-*
-* Arguments:
-*	hw		device structure
-*	mode		DOWAIT or DOASYNC
-*       cmd             cmd structure.  Includes all arguments and result
-*                       data points.  All in host order. in host order
-*	cmdcb		command-specific callback
-*	usercb		user callback for async calls, NULL for DOWAIT calls
-*	usercb_data	user supplied data pointer for async calls, NULL
-*			for DOASYNC calls
-*
-* Returns:
-*	0		success
-*	-EIO		CTLX failure
-*	-ERESTARTSYS	Awakened on signal
-*	>0		command indicated error, Status and Resp0-2 are
-*			in hw structure.
-*
-* Side effects:
-*
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_docmd
+ *
+ * Constructs a command CTLX and submits it.
+ *
+ * NOTE: Any changes to the 'post-submit' code in this function
+ *       need to be carried over to hfa384x_cbcmd() since the handling
+ *       is virtually identical.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	mode		DOWAIT or DOASYNC
+ *       cmd             cmd structure.  Includes all arguments and result
+ *                       data points.  All in host order. in host order
+ *	cmdcb		command-specific callback
+ *	usercb		user callback for async calls, NULL for DOWAIT calls
+ *	usercb_data	user supplied data pointer for async calls, NULL
+ *			for DOASYNC calls
+ *
+ * Returns:
+ *	0		success
+ *	-EIO		CTLX failure
+ *	-ERESTARTSYS	Awakened on signal
+ *	>0		command indicated error, Status and Resp0-2 are
+ *			in hw structure.
+ *
+ * Side effects:
+ *
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 static int
 hfa384x_docmd(struct hfa384x *hw,
 	      enum cmd_mode mode,
@@ -1341,41 +1361,42 @@ done:
 }
 
 /*----------------------------------------------------------------
-* hfa384x_dorrid
-*
-* Constructs a read rid CTLX and issues it.
-*
-* NOTE: Any changes to the 'post-submit' code in this function
-*       need to be carried over to hfa384x_cbrrid() since the handling
-*       is virtually identical.
-*
-* Arguments:
-*	hw		device structure
-*	mode		DOWAIT or DOASYNC
-*	rid		Read RID number (host order)
-*	riddata		Caller supplied buffer that MAC formatted RID.data
-*			record will be written to for DOWAIT calls. Should
-*			be NULL for DOASYNC calls.
-*	riddatalen	Buffer length for DOWAIT calls. Zero for DOASYNC calls.
-*	cmdcb		command callback for async calls, NULL for DOWAIT calls
-*	usercb		user callback for async calls, NULL for DOWAIT calls
-*	usercb_data	user supplied data pointer for async calls, NULL
-*			for DOWAIT calls
-*
-* Returns:
-*	0		success
-*	-EIO		CTLX failure
-*	-ERESTARTSYS	Awakened on signal
-*	-ENODATA	riddatalen != macdatalen
-*	>0		command indicated error, Status and Resp0-2 are
-*			in hw structure.
-*
-* Side effects:
-*
-* Call context:
-*	interrupt (DOASYNC)
-*	process (DOWAIT or DOASYNC)
-----------------------------------------------------------------*/
+ * hfa384x_dorrid
+ *
+ * Constructs a read rid CTLX and issues it.
+ *
+ * NOTE: Any changes to the 'post-submit' code in this function
+ *       need to be carried over to hfa384x_cbrrid() since the handling
+ *       is virtually identical.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	mode		DOWAIT or DOASYNC
+ *	rid		Read RID number (host order)
+ *	riddata		Caller supplied buffer that MAC formatted RID.data
+ *			record will be written to for DOWAIT calls. Should
+ *			be NULL for DOASYNC calls.
+ *	riddatalen	Buffer length for DOWAIT calls. Zero for DOASYNC calls.
+ *	cmdcb		command callback for async calls, NULL for DOWAIT calls
+ *	usercb		user callback for async calls, NULL for DOWAIT calls
+ *	usercb_data	user supplied data pointer for async calls, NULL
+ *			for DOWAIT calls
+ *
+ * Returns:
+ *	0		success
+ *	-EIO		CTLX failure
+ *	-ERESTARTSYS	Awakened on signal
+ *	-ENODATA	riddatalen != macdatalen
+ *	>0		command indicated error, Status and Resp0-2 are
+ *			in hw structure.
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt (DOASYNC)
+ *	process (DOWAIT or DOASYNC)
+ *----------------------------------------------------------------
+ */
 static int
 hfa384x_dorrid(struct hfa384x *hw,
 	       enum cmd_mode mode,
@@ -1426,37 +1447,38 @@ done:
 }
 
 /*----------------------------------------------------------------
-* hfa384x_dowrid
-*
-* Constructs a write rid CTLX and issues it.
-*
-* NOTE: Any changes to the 'post-submit' code in this function
-*       need to be carried over to hfa384x_cbwrid() since the handling
-*       is virtually identical.
-*
-* Arguments:
-*	hw		device structure
-*	enum cmd_mode	DOWAIT or DOASYNC
-*	rid		RID code
-*	riddata		Data portion of RID formatted for MAC
-*	riddatalen	Length of the data portion in bytes
-*       cmdcb           command callback for async calls, NULL for DOWAIT calls
-*	usercb		user callback for async calls, NULL for DOWAIT calls
-*	usercb_data	user supplied data pointer for async calls
-*
-* Returns:
-*	0		success
-*	-ETIMEDOUT	timed out waiting for register ready or
-*			command completion
-*	>0		command indicated error, Status and Resp0-2 are
-*			in hw structure.
-*
-* Side effects:
-*
-* Call context:
-*	interrupt (DOASYNC)
-*	process (DOWAIT or DOASYNC)
-----------------------------------------------------------------*/
+ * hfa384x_dowrid
+ *
+ * Constructs a write rid CTLX and issues it.
+ *
+ * NOTE: Any changes to the 'post-submit' code in this function
+ *       need to be carried over to hfa384x_cbwrid() since the handling
+ *       is virtually identical.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	enum cmd_mode	DOWAIT or DOASYNC
+ *	rid		RID code
+ *	riddata		Data portion of RID formatted for MAC
+ *	riddatalen	Length of the data portion in bytes
+ *       cmdcb           command callback for async calls, NULL for DOWAIT calls
+ *	usercb		user callback for async calls, NULL for DOWAIT calls
+ *	usercb_data	user supplied data pointer for async calls
+ *
+ * Returns:
+ *	0		success
+ *	-ETIMEDOUT	timed out waiting for register ready or
+ *			command completion
+ *	>0		command indicated error, Status and Resp0-2 are
+ *			in hw structure.
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt (DOASYNC)
+ *	process (DOWAIT or DOASYNC)
+ *----------------------------------------------------------------
+ */
 static int
 hfa384x_dowrid(struct hfa384x *hw,
 	       enum cmd_mode mode,
@@ -1512,38 +1534,39 @@ done:
 }
 
 /*----------------------------------------------------------------
-* hfa384x_dormem
-*
-* Constructs a readmem CTLX and issues it.
-*
-* NOTE: Any changes to the 'post-submit' code in this function
-*       need to be carried over to hfa384x_cbrmem() since the handling
-*       is virtually identical.
-*
-* Arguments:
-*	hw		device structure
-*	mode		DOWAIT or DOASYNC
-*	page		MAC address space page (CMD format)
-*	offset		MAC address space offset
-*	data		Ptr to data buffer to receive read
-*	len		Length of the data to read (max == 2048)
-*	cmdcb		command callback for async calls, NULL for DOWAIT calls
-*	usercb		user callback for async calls, NULL for DOWAIT calls
-*	usercb_data	user supplied data pointer for async calls
-*
-* Returns:
-*	0		success
-*	-ETIMEDOUT	timed out waiting for register ready or
-*			command completion
-*	>0		command indicated error, Status and Resp0-2 are
-*			in hw structure.
-*
-* Side effects:
-*
-* Call context:
-*	interrupt (DOASYNC)
-*	process (DOWAIT or DOASYNC)
-----------------------------------------------------------------*/
+ * hfa384x_dormem
+ *
+ * Constructs a readmem CTLX and issues it.
+ *
+ * NOTE: Any changes to the 'post-submit' code in this function
+ *       need to be carried over to hfa384x_cbrmem() since the handling
+ *       is virtually identical.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	mode		DOWAIT or DOASYNC
+ *	page		MAC address space page (CMD format)
+ *	offset		MAC address space offset
+ *	data		Ptr to data buffer to receive read
+ *	len		Length of the data to read (max == 2048)
+ *	cmdcb		command callback for async calls, NULL for DOWAIT calls
+ *	usercb		user callback for async calls, NULL for DOWAIT calls
+ *	usercb_data	user supplied data pointer for async calls
+ *
+ * Returns:
+ *	0		success
+ *	-ETIMEDOUT	timed out waiting for register ready or
+ *			command completion
+ *	>0		command indicated error, Status and Resp0-2 are
+ *			in hw structure.
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt (DOASYNC)
+ *	process (DOWAIT or DOASYNC)
+ *----------------------------------------------------------------
+ */
 static int
 hfa384x_dormem(struct hfa384x *hw,
 	       enum cmd_mode mode,
@@ -1603,38 +1626,39 @@ done:
 }
 
 /*----------------------------------------------------------------
-* hfa384x_dowmem
-*
-* Constructs a writemem CTLX and issues it.
-*
-* NOTE: Any changes to the 'post-submit' code in this function
-*       need to be carried over to hfa384x_cbwmem() since the handling
-*       is virtually identical.
-*
-* Arguments:
-*	hw		device structure
-*	mode		DOWAIT or DOASYNC
-*	page		MAC address space page (CMD format)
-*	offset		MAC address space offset
-*	data		Ptr to data buffer containing write data
-*	len		Length of the data to read (max == 2048)
-*	cmdcb		command callback for async calls, NULL for DOWAIT calls
-*	usercb		user callback for async calls, NULL for DOWAIT calls
-*	usercb_data	user supplied data pointer for async calls.
-*
-* Returns:
-*	0		success
-*	-ETIMEDOUT	timed out waiting for register ready or
-*			command completion
-*	>0		command indicated error, Status and Resp0-2 are
-*			in hw structure.
-*
-* Side effects:
-*
-* Call context:
-*	interrupt (DOWAIT)
-*	process (DOWAIT or DOASYNC)
-----------------------------------------------------------------*/
+ * hfa384x_dowmem
+ *
+ * Constructs a writemem CTLX and issues it.
+ *
+ * NOTE: Any changes to the 'post-submit' code in this function
+ *       need to be carried over to hfa384x_cbwmem() since the handling
+ *       is virtually identical.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	mode		DOWAIT or DOASYNC
+ *	page		MAC address space page (CMD format)
+ *	offset		MAC address space offset
+ *	data		Ptr to data buffer containing write data
+ *	len		Length of the data to read (max == 2048)
+ *	cmdcb		command callback for async calls, NULL for DOWAIT calls
+ *	usercb		user callback for async calls, NULL for DOWAIT calls
+ *	usercb_data	user supplied data pointer for async calls.
+ *
+ * Returns:
+ *	0		success
+ *	-ETIMEDOUT	timed out waiting for register ready or
+ *			command completion
+ *	>0		command indicated error, Status and Resp0-2 are
+ *			in hw structure.
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt (DOWAIT)
+ *	process (DOWAIT or DOASYNC)
+ *----------------------------------------------------------------
+ */
 static int
 hfa384x_dowmem(struct hfa384x *hw,
 	       enum cmd_mode mode,
@@ -1694,27 +1718,28 @@ done:
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_disable
-*
-* Issues the disable command to stop communications on one of
-* the MACs 'ports'.  Only macport 0 is valid  for stations.
-* APs may also disable macports 1-6.  Only ports that have been
-* previously enabled may be disabled.
-*
-* Arguments:
-*	hw		device structure
-*	macport		MAC port number (host order)
-*
-* Returns:
-*	0		success
-*	>0		f/w reported failure - f/w status code
-*	<0		driver reported error (timeout|bad arg)
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_drvr_disable
+ *
+ * Issues the disable command to stop communications on one of
+ * the MACs 'ports'.  Only macport 0 is valid  for stations.
+ * APs may also disable macports 1-6.  Only ports that have been
+ * previously enabled may be disabled.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	macport		MAC port number (host order)
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported failure - f/w status code
+ *	<0		driver reported error (timeout|bad arg)
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_disable(struct hfa384x *hw, u16 macport)
 {
 	int result = 0;
@@ -1732,27 +1757,28 @@ int hfa384x_drvr_disable(struct hfa384x *hw, u16 macport)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_enable
-*
-* Issues the enable command to enable communications on one of
-* the MACs 'ports'.  Only macport 0 is valid  for stations.
-* APs may also enable macports 1-6.  Only ports that are currently
-* disabled may be enabled.
-*
-* Arguments:
-*	hw		device structure
-*	macport		MAC port number
-*
-* Returns:
-*	0		success
-*	>0		f/w reported failure - f/w status code
-*	<0		driver reported error (timeout|bad arg)
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_drvr_enable
+ *
+ * Issues the enable command to enable communications on one of
+ * the MACs 'ports'.  Only macport 0 is valid  for stations.
+ * APs may also enable macports 1-6.  Only ports that are currently
+ * disabled may be enabled.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	macport		MAC port number
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported failure - f/w status code
+ *	<0		driver reported error (timeout|bad arg)
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_enable(struct hfa384x *hw, u16 macport)
 {
 	int result = 0;
@@ -1770,26 +1796,27 @@ int hfa384x_drvr_enable(struct hfa384x *hw, u16 macport)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_flashdl_enable
-*
-* Begins the flash download state.  Checks to see that we're not
-* already in a download state and that a port isn't enabled.
-* Sets the download state and retrieves the flash download
-* buffer location, buffer size, and timeout length.
-*
-* Arguments:
-*	hw		device structure
-*
-* Returns:
-*	0		success
-*	>0		f/w reported error - f/w status code
-*	<0		driver reported error
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_drvr_flashdl_enable
+ *
+ * Begins the flash download state.  Checks to see that we're not
+ * already in a download state and that a port isn't enabled.
+ * Sets the download state and retrieves the flash download
+ * buffer location, buffer size, and timeout length.
+ *
+ * Arguments:
+ *	hw		device structure
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported error - f/w status code
+ *	<0		driver reported error
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_flashdl_enable(struct hfa384x *hw)
 {
 	int result = 0;
@@ -1809,7 +1836,7 @@ int hfa384x_drvr_flashdl_enable(struct hfa384x *hw)
 
 	/* Retrieve the buffer loc&size and timeout */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_DOWNLOADBUFFER,
-					&(hw->bufinfo), sizeof(hw->bufinfo));
+					&hw->bufinfo, sizeof(hw->bufinfo));
 	if (result)
 		return result;
 
@@ -1817,7 +1844,7 @@ int hfa384x_drvr_flashdl_enable(struct hfa384x *hw)
 	hw->bufinfo.offset = le16_to_cpu(hw->bufinfo.offset);
 	hw->bufinfo.len = le16_to_cpu(hw->bufinfo.len);
 	result = hfa384x_drvr_getconfig16(hw, HFA384x_RID_MAXLOADTIME,
-					  &(hw->dltimeout));
+					  &hw->dltimeout);
 	if (result)
 		return result;
 
@@ -1831,24 +1858,25 @@ int hfa384x_drvr_flashdl_enable(struct hfa384x *hw)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_flashdl_disable
-*
-* Ends the flash download state.  Note that this will cause the MAC
-* firmware to restart.
-*
-* Arguments:
-*	hw		device structure
-*
-* Returns:
-*	0		success
-*	>0		f/w reported error - f/w status code
-*	<0		driver reported error
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_drvr_flashdl_disable
+ *
+ * Ends the flash download state.  Note that this will cause the MAC
+ * firmware to restart.
+ *
+ * Arguments:
+ *	hw		device structure
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported error - f/w status code
+ *	<0		driver reported error
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_flashdl_disable(struct hfa384x *hw)
 {
 	/* Check that we're already in the download state */
@@ -1866,35 +1894,37 @@ int hfa384x_drvr_flashdl_disable(struct hfa384x *hw)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_flashdl_write
-*
-* Performs a FLASH download of a chunk of data. First checks to see
-* that we're in the FLASH download state, then sets the download
-* mode, uses the aux functions to 1) copy the data to the flash
-* buffer, 2) sets the download 'write flash' mode, 3) readback and
-* compare.  Lather rinse, repeat as many times an necessary to get
-* all the given data into flash.
-* When all data has been written using this function (possibly
-* repeatedly), call drvr_flashdl_disable() to end the download state
-* and restart the MAC.
-*
-* Arguments:
-*	hw		device structure
-*	daddr		Card address to write to. (host order)
-*	buf		Ptr to data to write.
-*	len		Length of data (host order).
-*
-* Returns:
-*	0		success
-*	>0		f/w reported error - f/w status code
-*	<0		driver reported error
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
-int hfa384x_drvr_flashdl_write(struct hfa384x *hw, u32 daddr, void *buf, u32 len)
+ * hfa384x_drvr_flashdl_write
+ *
+ * Performs a FLASH download of a chunk of data. First checks to see
+ * that we're in the FLASH download state, then sets the download
+ * mode, uses the aux functions to 1) copy the data to the flash
+ * buffer, 2) sets the download 'write flash' mode, 3) readback and
+ * compare.  Lather rinse, repeat as many times an necessary to get
+ * all the given data into flash.
+ * When all data has been written using this function (possibly
+ * repeatedly), call drvr_flashdl_disable() to end the download state
+ * and restart the MAC.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	daddr		Card address to write to. (host order)
+ *	buf		Ptr to data to write.
+ *	len		Length of data (host order).
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported error - f/w status code
+ *	<0		driver reported error
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
+int hfa384x_drvr_flashdl_write(struct hfa384x *hw, u32 daddr,
+			       void *buf, u32 len)
 {
 	int result = 0;
 	u32 dlbufaddr;
@@ -2008,30 +2038,31 @@ exit_proc:
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_getconfig
-*
-* Performs the sequence necessary to read a config/info item.
-*
-* Arguments:
-*	hw		device structure
-*	rid		config/info record id (host order)
-*	buf		host side record buffer.  Upon return it will
-*			contain the body portion of the record (minus the
-*			RID and len).
-*	len		buffer length (in bytes, should match record length)
-*
-* Returns:
-*	0		success
-*	>0		f/w reported error - f/w status code
-*	<0		driver reported error
-*	-ENODATA	length mismatch between argument and retrieved
-*			record.
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_drvr_getconfig
+ *
+ * Performs the sequence necessary to read a config/info item.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	rid		config/info record id (host order)
+ *	buf		host side record buffer.  Upon return it will
+ *			contain the body portion of the record (minus the
+ *			RID and len).
+ *	len		buffer length (in bytes, should match record length)
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported error - f/w status code
+ *	<0		driver reported error
+ *	-ENODATA	length mismatch between argument and retrieved
+ *			record.
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_getconfig(struct hfa384x *hw, u16 rid, void *buf, u16 len)
 {
 	return hfa384x_dorrid_wait(hw, rid, buf, len);
@@ -2059,7 +2090,8 @@ int hfa384x_drvr_getconfig(struct hfa384x *hw, u16 rid, void *buf, u16 len)
  *
  * Call context:
  *       process
- ----------------------------------------------------------------*/
+ *----------------------------------------------------------------
+ */
 int
 hfa384x_drvr_setconfig_async(struct hfa384x *hw,
 			     u16 rid,
@@ -2071,23 +2103,24 @@ hfa384x_drvr_setconfig_async(struct hfa384x *hw,
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_ramdl_disable
-*
-* Ends the ram download state.
-*
-* Arguments:
-*	hw		device structure
-*
-* Returns:
-*	0		success
-*	>0		f/w reported error - f/w status code
-*	<0		driver reported error
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_drvr_ramdl_disable
+ *
+ * Ends the ram download state.
+ *
+ * Arguments:
+ *	hw		device structure
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported error - f/w status code
+ *	<0		driver reported error
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_ramdl_disable(struct hfa384x *hw)
 {
 	/* Check that we're already in the download state */
@@ -2105,29 +2138,30 @@ int hfa384x_drvr_ramdl_disable(struct hfa384x *hw)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_ramdl_enable
-*
-* Begins the ram download state.  Checks to see that we're not
-* already in a download state and that a port isn't enabled.
-* Sets the download state and calls cmd_download with the
-* ENABLE_VOLATILE subcommand and the exeaddr argument.
-*
-* Arguments:
-*	hw		device structure
-*	exeaddr		the card execution address that will be
-*                       jumped to when ramdl_disable() is called
-*			(host order).
-*
-* Returns:
-*	0		success
-*	>0		f/w reported error - f/w status code
-*	<0		driver reported error
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_drvr_ramdl_enable
+ *
+ * Begins the ram download state.  Checks to see that we're not
+ * already in a download state and that a port isn't enabled.
+ * Sets the download state and calls cmd_download with the
+ * ENABLE_VOLATILE subcommand and the exeaddr argument.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	exeaddr		the card execution address that will be
+ *                       jumped to when ramdl_disable() is called
+ *			(host order).
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported error - f/w status code
+ *	<0		driver reported error
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_ramdl_enable(struct hfa384x *hw, u32 exeaddr)
 {
 	int result = 0;
@@ -2146,7 +2180,8 @@ int hfa384x_drvr_ramdl_enable(struct hfa384x *hw, u32 exeaddr)
 
 	/* Check that we're not already in a download state */
 	if (hw->dlstate != HFA384x_DLSTATE_DISABLED) {
-		netdev_err(hw->wlandev->netdev, "Download state not disabled.\n");
+		netdev_err(hw->wlandev->netdev,
+			   "Download state not disabled.\n");
 		return -EINVAL;
 	}
 
@@ -2171,31 +2206,32 @@ int hfa384x_drvr_ramdl_enable(struct hfa384x *hw, u32 exeaddr)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_ramdl_write
-*
-* Performs a RAM download of a chunk of data. First checks to see
-* that we're in the RAM download state, then uses the [read|write]mem USB
-* commands to 1) copy the data, 2) readback and compare.  The download
-* state is unaffected.  When all data has been written using
-* this function, call drvr_ramdl_disable() to end the download state
-* and restart the MAC.
-*
-* Arguments:
-*	hw		device structure
-*	daddr		Card address to write to. (host order)
-*	buf		Ptr to data to write.
-*	len		Length of data (host order).
-*
-* Returns:
-*	0		success
-*	>0		f/w reported error - f/w status code
-*	<0		driver reported error
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_drvr_ramdl_write
+ *
+ * Performs a RAM download of a chunk of data. First checks to see
+ * that we're in the RAM download state, then uses the [read|write]mem USB
+ * commands to 1) copy the data, 2) readback and compare.  The download
+ * state is unaffected.  When all data has been written using
+ * this function, call drvr_ramdl_disable() to end the download state
+ * and restart the MAC.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	daddr		Card address to write to. (host order)
+ *	buf		Ptr to data to write.
+ *	len		Length of data (host order).
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported error - f/w status code
+ *	<0		driver reported error
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_ramdl_write(struct hfa384x *hw, u32 daddr, void *buf, u32 len)
 {
 	int result = 0;
@@ -2246,36 +2282,37 @@ int hfa384x_drvr_ramdl_write(struct hfa384x *hw, u32 daddr, void *buf, u32 len)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_readpda
-*
-* Performs the sequence to read the PDA space.  Note there is no
-* drvr_writepda() function.  Writing a PDA is
-* generally implemented by a calling component via calls to
-* cmd_download and writing to the flash download buffer via the
-* aux regs.
-*
-* Arguments:
-*	hw		device structure
-*	buf		buffer to store PDA in
-*	len		buffer length
-*
-* Returns:
-*	0		success
-*	>0		f/w reported error - f/w status code
-*	<0		driver reported error
-*	-ETIMEDOUT	timeout waiting for the cmd regs to become
-*			available, or waiting for the control reg
-*			to indicate the Aux port is enabled.
-*	-ENODATA	the buffer does NOT contain a valid PDA.
-*			Either the card PDA is bad, or the auxdata
-*			reads are giving us garbage.
-
-*
-* Side effects:
-*
-* Call context:
-*	process or non-card interrupt.
-----------------------------------------------------------------*/
+ * hfa384x_drvr_readpda
+ *
+ * Performs the sequence to read the PDA space.  Note there is no
+ * drvr_writepda() function.  Writing a PDA is
+ * generally implemented by a calling component via calls to
+ * cmd_download and writing to the flash download buffer via the
+ * aux regs.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	buf		buffer to store PDA in
+ *	len		buffer length
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported error - f/w status code
+ *	<0		driver reported error
+ *	-ETIMEDOUT	timeout waiting for the cmd regs to become
+ *			available, or waiting for the control reg
+ *			to indicate the Aux port is enabled.
+ *	-ENODATA	the buffer does NOT contain a valid PDA.
+ *			Either the card PDA is bad, or the auxdata
+ *			reads are giving us garbage.
+ *
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process or non-card interrupt.
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_readpda(struct hfa384x *hw, void *buf, unsigned int len)
 {
 	int result = 0;
@@ -2306,7 +2343,7 @@ int hfa384x_drvr_readpda(struct hfa384x *hw, void *buf, unsigned int len)
 
 		/* units of bytes */
 		result = hfa384x_dormem_wait(hw, currpage, curroffset, buf,
-						len);
+					     len);
 
 		if (result) {
 			netdev_warn(hw->wlandev->netdev,
@@ -2366,51 +2403,52 @@ int hfa384x_drvr_readpda(struct hfa384x *hw, void *buf, unsigned int len)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_setconfig
-*
-* Performs the sequence necessary to write a config/info item.
-*
-* Arguments:
-*	hw		device structure
-*	rid		config/info record id (in host order)
-*	buf		host side record buffer
-*	len		buffer length (in bytes)
-*
-* Returns:
-*	0		success
-*	>0		f/w reported error - f/w status code
-*	<0		driver reported error
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_drvr_setconfig
+ *
+ * Performs the sequence necessary to write a config/info item.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	rid		config/info record id (in host order)
+ *	buf		host side record buffer
+ *	len		buffer length (in bytes)
+ *
+ * Returns:
+ *	0		success
+ *	>0		f/w reported error - f/w status code
+ *	<0		driver reported error
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_setconfig(struct hfa384x *hw, u16 rid, void *buf, u16 len)
 {
 	return hfa384x_dowrid_wait(hw, rid, buf, len);
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_start
-*
-* Issues the MAC initialize command, sets up some data structures,
-* and enables the interrupts.  After this function completes, the
-* low-level stuff should be ready for any/all commands.
-*
-* Arguments:
-*	hw		device structure
-* Returns:
-*	0		success
-*	>0		f/w reported error - f/w status code
-*	<0		driver reported error
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
-
+ * hfa384x_drvr_start
+ *
+ * Issues the MAC initialize command, sets up some data structures,
+ * and enables the interrupts.  After this function completes, the
+ * low-level stuff should be ready for any/all commands.
+ *
+ * Arguments:
+ *	hw		device structure
+ * Returns:
+ *	0		success
+ *	>0		f/w reported error - f/w status code
+ *	<0		driver reported error
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_start(struct hfa384x *hw)
 {
 	int result, result1, result2;
@@ -2494,24 +2532,25 @@ done:
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_stop
-*
-* Shuts down the MAC to the point where it is safe to unload the
-* driver.  Any subsystem that may be holding a data or function
-* ptr into the driver must be cleared/deinitialized.
-*
-* Arguments:
-*	hw		device structure
-* Returns:
-*	0		success
-*	>0		f/w reported error - f/w status code
-*	<0		driver reported error
-*
-* Side effects:
-*
-* Call context:
-*	process
-----------------------------------------------------------------*/
+ * hfa384x_drvr_stop
+ *
+ * Shuts down the MAC to the point where it is safe to unload the
+ * driver.  Any subsystem that may be holding a data or function
+ * ptr into the driver must be cleared/deinitialized.
+ *
+ * Arguments:
+ *	hw		device structure
+ * Returns:
+ *	0		success
+ *	>0		f/w reported error - f/w status code
+ *	<0		driver reported error
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_stop(struct hfa384x *hw)
 {
 	int i;
@@ -2542,26 +2581,27 @@ int hfa384x_drvr_stop(struct hfa384x *hw)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_drvr_txframe
-*
-* Takes a frame from prism2sta and queues it for transmission.
-*
-* Arguments:
-*	hw		device structure
-*	skb		packet buffer struct.  Contains an 802.11
-*			data frame.
-*       p80211_hdr      points to the 802.11 header for the packet.
-* Returns:
-*	0		Success and more buffs available
-*	1		Success but no more buffs
-*	2		Allocation failure
-*	4		Buffer full or queue busy
-*
-* Side effects:
-*
-* Call context:
-*	interrupt
-----------------------------------------------------------------*/
+ * hfa384x_drvr_txframe
+ *
+ * Takes a frame from prism2sta and queues it for transmission.
+ *
+ * Arguments:
+ *	hw		device structure
+ *	skb		packet buffer struct.  Contains an 802.11
+ *			data frame.
+ *       p80211_hdr      points to the 802.11 header for the packet.
+ * Returns:
+ *	0		Success and more buffs available
+ *	1		Success but no more buffs
+ *	2		Allocation failure
+ *	4		Buffer full or queue busy
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 int hfa384x_drvr_txframe(struct hfa384x *hw, struct sk_buff *skb,
 			 union p80211_hdr *p80211_hdr,
 			 struct p80211_metawep *p80211_wep)
@@ -2608,7 +2648,7 @@ int hfa384x_drvr_txframe(struct hfa384x *hw, struct sk_buff *skb,
 	    cpu_to_le16(hw->txbuff.txfrm.desc.tx_control);
 
 	/* copy the header over to the txdesc */
-	memcpy(&(hw->txbuff.txfrm.desc.frame_control), p80211_hdr,
+	memcpy(&hw->txbuff.txfrm.desc.frame_control, p80211_hdr,
 	       sizeof(union p80211_hdr));
 
 	/* if we're using host WEP, increase size by IV+ICV */
@@ -2638,9 +2678,9 @@ int hfa384x_drvr_txframe(struct hfa384x *hw, struct sk_buff *skb,
 		memcpy(ptr, p80211_wep->icv, sizeof(p80211_wep->icv));
 
 	/* Send the USB packet */
-	usb_fill_bulk_urb(&(hw->tx_urb), hw->usb,
+	usb_fill_bulk_urb(&hw->tx_urb, hw->usb,
 			  hw->endp_out,
-			  &(hw->txbuff), ROUNDUP64(usbpktlen),
+			  &hw->txbuff, ROUNDUP64(usbpktlen),
 			  hfa384x_usbout_callback, hw->wlandev);
 	hw->tx_urb.transfer_flags |= USB_QUEUE_BULK;
 
@@ -2676,18 +2716,19 @@ void hfa384x_tx_timeout(struct wlandevice *wlandev)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbctlx_reaper_task
-*
-* Tasklet to delete dead CTLX objects
-*
-* Arguments:
-*	data	ptr to a struct hfa384x
-*
-* Returns:
-*
-* Call context:
-*	Interrupt
-----------------------------------------------------------------*/
+ * hfa384x_usbctlx_reaper_task
+ *
+ * Tasklet to delete dead CTLX objects
+ *
+ * Arguments:
+ *	data	ptr to a struct hfa384x
+ *
+ * Returns:
+ *
+ * Call context:
+ *	Interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usbctlx_reaper_task(unsigned long data)
 {
 	struct hfa384x *hw = (struct hfa384x *)data;
@@ -2708,19 +2749,20 @@ static void hfa384x_usbctlx_reaper_task(unsigned long data)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbctlx_completion_task
-*
-* Tasklet to call completion handlers for returned CTLXs
-*
-* Arguments:
-*	data	ptr to struct hfa384x
-*
-* Returns:
-*	Nothing
-*
-* Call context:
-*	Interrupt
-----------------------------------------------------------------*/
+ * hfa384x_usbctlx_completion_task
+ *
+ * Tasklet to call completion handlers for returned CTLXs
+ *
+ * Arguments:
+ *	data	ptr to struct hfa384x
+ *
+ * Returns:
+ *	Nothing
+ *
+ * Call context:
+ *	Interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usbctlx_completion_task(unsigned long data)
 {
 	struct hfa384x *hw = (struct hfa384x *)data;
@@ -2781,22 +2823,23 @@ static void hfa384x_usbctlx_completion_task(unsigned long data)
 }
 
 /*----------------------------------------------------------------
-* unlocked_usbctlx_cancel_async
-*
-* Mark the CTLX dead asynchronously, and ensure that the
-* next command on the queue is run afterwards.
-*
-* Arguments:
-*	hw	ptr to the struct hfa384x structure
-*	ctlx	ptr to a CTLX structure
-*
-* Returns:
-*	0	the CTLX's URB is inactive
-* -EINPROGRESS	the URB is currently being unlinked
-*
-* Call context:
-*	Either process or interrupt, but presumably interrupt
-----------------------------------------------------------------*/
+ * unlocked_usbctlx_cancel_async
+ *
+ * Mark the CTLX dead asynchronously, and ensure that the
+ * next command on the queue is run afterwards.
+ *
+ * Arguments:
+ *	hw	ptr to the struct hfa384x structure
+ *	ctlx	ptr to a CTLX structure
+ *
+ * Returns:
+ *	0	the CTLX's URB is inactive
+ * -EINPROGRESS	the URB is currently being unlinked
+ *
+ * Call context:
+ *	Either process or interrupt, but presumably interrupt
+ *----------------------------------------------------------------
+ */
 static int unlocked_usbctlx_cancel_async(struct hfa384x *hw,
 					 struct hfa384x_usbctlx *ctlx)
 {
@@ -2826,28 +2869,30 @@ static int unlocked_usbctlx_cancel_async(struct hfa384x *hw,
 }
 
 /*----------------------------------------------------------------
-* unlocked_usbctlx_complete
-*
-* A CTLX has completed.  It may have been successful, it may not
-* have been. At this point, the CTLX should be quiescent.  The URBs
-* aren't active and the timers should have been stopped.
-*
-* The CTLX is migrated to the "completing" queue, and the completing
-* tasklet is scheduled.
-*
-* Arguments:
-*	hw		ptr to a struct hfa384x structure
-*	ctlx		ptr to a ctlx structure
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	Either, assume interrupt
-----------------------------------------------------------------*/
-static void unlocked_usbctlx_complete(struct hfa384x *hw, struct hfa384x_usbctlx *ctlx)
+ * unlocked_usbctlx_complete
+ *
+ * A CTLX has completed.  It may have been successful, it may not
+ * have been. At this point, the CTLX should be quiescent.  The URBs
+ * aren't active and the timers should have been stopped.
+ *
+ * The CTLX is migrated to the "completing" queue, and the completing
+ * tasklet is scheduled.
+ *
+ * Arguments:
+ *	hw		ptr to a struct hfa384x structure
+ *	ctlx		ptr to a ctlx structure
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	Either, assume interrupt
+ *----------------------------------------------------------------
+ */
+static void unlocked_usbctlx_complete(struct hfa384x *hw,
+				      struct hfa384x_usbctlx *ctlx)
 {
 	/* Timers have been stopped, and ctlx should be in
 	 * a terminal state. Retire it from the "active"
@@ -2871,21 +2916,22 @@ static void unlocked_usbctlx_complete(struct hfa384x *hw, struct hfa384x_usbctlx
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbctlxq_run
-*
-* Checks to see if the head item is running.  If not, starts it.
-*
-* Arguments:
-*	hw	ptr to struct hfa384x
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	any
-----------------------------------------------------------------*/
+ * hfa384x_usbctlxq_run
+ *
+ * Checks to see if the head item is running.  If not, starts it.
+ *
+ * Arguments:
+ *	hw	ptr to struct hfa384x
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	any
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usbctlxq_run(struct hfa384x *hw)
 {
 	unsigned long flags;
@@ -2916,9 +2962,9 @@ static void hfa384x_usbctlxq_run(struct hfa384x *hw)
 		list_move_tail(&head->list, &hw->ctlxq.active);
 
 		/* Fill the out packet */
-		usb_fill_bulk_urb(&(hw->ctlx_urb), hw->usb,
+		usb_fill_bulk_urb(&hw->ctlx_urb, hw->usb,
 				  hw->endp_out,
-				  &(head->outbuf), ROUNDUP64(head->outbufsize),
+				  &head->outbuf, ROUNDUP64(head->outbufsize),
 				  hfa384x_ctlxout_callback, hw);
 		hw->ctlx_urb.transfer_flags |= USB_QUEUE_BULK;
 
@@ -2971,26 +3017,27 @@ unlock:
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbin_callback
-*
-* Callback for URBs on the BULKIN endpoint.
-*
-* Arguments:
-*	urb		ptr to the completed urb
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	interrupt
-----------------------------------------------------------------*/
+ * hfa384x_usbin_callback
+ *
+ * Callback for URBs on the BULKIN endpoint.
+ *
+ * Arguments:
+ *	urb		ptr to the completed urb
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usbin_callback(struct urb *urb)
 {
 	struct wlandevice *wlandev = urb->context;
 	struct hfa384x *hw;
-	union hfa384x_usbin *usbin = (union hfa384x_usbin *)urb->transfer_buffer;
+	union hfa384x_usbin *usbin;
 	struct sk_buff *skb = NULL;
 	int result;
 	int urb_status;
@@ -3010,7 +3057,10 @@ static void hfa384x_usbin_callback(struct urb *urb)
 		goto exit;
 
 	skb = hw->rx_urb_skb;
-	BUG_ON(!skb || (skb->data != urb->transfer_buffer));
+	if (!skb || (skb->data != urb->transfer_buffer)) {
+		WARN_ON(1);
+		return;
+	}
 
 	hw->rx_urb_skb = NULL;
 
@@ -3089,6 +3139,7 @@ static void hfa384x_usbin_callback(struct urb *urb)
 	/* Note: the check of the sw_support field, the type field doesn't
 	 *       have bit 12 set like the docs suggest.
 	 */
+	usbin = (union hfa384x_usbin *)urb->transfer_buffer;
 	type = le16_to_cpu(usbin->type);
 	if (HFA384x_USB_ISRXFRM(type)) {
 		if (action == HANDLE) {
@@ -3147,25 +3198,26 @@ exit:
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbin_ctlx
-*
-* We've received a URB containing a Prism2 "response" message.
-* This message needs to be matched up with a CTLX on the active
-* queue and our state updated accordingly.
-*
-* Arguments:
-*	hw		ptr to struct hfa384x
-*	usbin		ptr to USB IN packet
-*	urb_status	status of this Bulk-In URB
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	interrupt
-----------------------------------------------------------------*/
+ * hfa384x_usbin_ctlx
+ *
+ * We've received a URB containing a Prism2 "response" message.
+ * This message needs to be matched up with a CTLX on the active
+ * queue and our state updated accordingly.
+ *
+ * Arguments:
+ *	hw		ptr to struct hfa384x
+ *	usbin		ptr to USB IN packet
+ *	urb_status	status of this Bulk-In URB
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usbin_ctlx(struct hfa384x *hw, union hfa384x_usbin *usbin,
 			       int urb_status)
 {
@@ -3269,22 +3321,23 @@ unlock:
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbin_txcompl
-*
-* At this point we have the results of a previous transmit.
-*
-* Arguments:
-*	wlandev		wlan device
-*	usbin		ptr to the usb transfer buffer
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	interrupt
-----------------------------------------------------------------*/
+ * hfa384x_usbin_txcompl
+ *
+ * At this point we have the results of a previous transmit.
+ *
+ * Arguments:
+ *	wlandev		wlan device
+ *	usbin		ptr to the usb transfer buffer
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usbin_txcompl(struct wlandevice *wlandev,
 				  union hfa384x_usbin *usbin)
 {
@@ -3300,22 +3353,23 @@ static void hfa384x_usbin_txcompl(struct wlandevice *wlandev,
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbin_rx
-*
-* At this point we have a successful received a rx frame packet.
-*
-* Arguments:
-*	wlandev		wlan device
-*	usbin		ptr to the usb transfer buffer
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	interrupt
-----------------------------------------------------------------*/
+ * hfa384x_usbin_rx
+ *
+ * At this point we have a successful received a rx frame packet.
+ *
+ * Arguments:
+ *	wlandev		wlan device
+ *	usbin		ptr to the usb transfer buffer
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usbin_rx(struct wlandevice *wlandev, struct sk_buff *skb)
 {
 	union hfa384x_usbin *usbin = (union hfa384x_usbin *)skb->data;
@@ -3396,30 +3450,31 @@ static void hfa384x_usbin_rx(struct wlandevice *wlandev, struct sk_buff *skb)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_int_rxmonitor
-*
-* Helper function for int_rx.  Handles monitor frames.
-* Note that this function allocates space for the FCS and sets it
-* to 0xffffffff.  The hfa384x doesn't give us the FCS value but the
-* higher layers expect it.  0xffffffff is used as a flag to indicate
-* the FCS is bogus.
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	rxfrm		rx descriptor read from card in int_rx
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*	Allocates an skb and passes it up via the PF_PACKET interface.
-* Call context:
-*	interrupt
-----------------------------------------------------------------*/
+ * hfa384x_int_rxmonitor
+ *
+ * Helper function for int_rx.  Handles monitor frames.
+ * Note that this function allocates space for the FCS and sets it
+ * to 0xffffffff.  The hfa384x doesn't give us the FCS value but the
+ * higher layers expect it.  0xffffffff is used as a flag to indicate
+ * the FCS is bogus.
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	rxfrm		rx descriptor read from card in int_rx
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *	Allocates an skb and passes it up via the PF_PACKET interface.
+ * Call context:
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_int_rxmonitor(struct wlandevice *wlandev,
 				  struct hfa384x_usb_rxfrm *rxfrm)
 {
-	struct hfa384x_rx_frame *rxdesc = &(rxfrm->desc);
+	struct hfa384x_rx_frame *rxdesc = &rxfrm->desc;
 	unsigned int hdrlen = 0;
 	unsigned int datalen = 0;
 	unsigned int skblen = 0;
@@ -3474,9 +3529,10 @@ static void hfa384x_int_rxmonitor(struct wlandevice *wlandev,
 	}
 
 	/* Copy the 802.11 header to the skb
-	   (ctl frames may be less than a full header) */
+	 * (ctl frames may be less than a full header)
+	 */
 	datap = skb_put(skb, hdrlen);
-	memcpy(datap, &(rxdesc->frame_control), hdrlen);
+	memcpy(datap, &rxdesc->frame_control, hdrlen);
 
 	/* If any, copy the data from the card to the skb */
 	if (datalen > 0) {
@@ -3501,22 +3557,23 @@ static void hfa384x_int_rxmonitor(struct wlandevice *wlandev,
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbin_info
-*
-* At this point we have a successful received a Prism2 info frame.
-*
-* Arguments:
-*	wlandev		wlan device
-*	usbin		ptr to the usb transfer buffer
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	interrupt
-----------------------------------------------------------------*/
+ * hfa384x_usbin_info
+ *
+ * At this point we have a successful received a Prism2 info frame.
+ *
+ * Arguments:
+ *	wlandev		wlan device
+ *	usbin		ptr to the usb transfer buffer
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usbin_info(struct wlandevice *wlandev,
 			       union hfa384x_usbin *usbin)
 {
@@ -3526,21 +3583,22 @@ static void hfa384x_usbin_info(struct wlandevice *wlandev,
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbout_callback
-*
-* Callback for URBs on the BULKOUT endpoint.
-*
-* Arguments:
-*	urb		ptr to the completed urb
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	interrupt
-----------------------------------------------------------------*/
+ * hfa384x_usbout_callback
+ *
+ * Callback for URBs on the BULKOUT endpoint.
+ *
+ * Arguments:
+ *	urb		ptr to the completed urb
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usbout_callback(struct urb *urb)
 {
 	struct wlandevice *wlandev = urb->context;
@@ -3601,21 +3659,22 @@ static void hfa384x_usbout_callback(struct urb *urb)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_ctlxout_callback
-*
-* Callback for control data on the BULKOUT endpoint.
-*
-* Arguments:
-*	urb		ptr to the completed urb
-*
-* Returns:
-* nothing
-*
-* Side effects:
-*
-* Call context:
-* interrupt
-----------------------------------------------------------------*/
+ * hfa384x_ctlxout_callback
+ *
+ * Callback for control data on the BULKOUT endpoint.
+ *
+ * Arguments:
+ *	urb		ptr to the completed urb
+ *
+ * Returns:
+ * nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ * interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_ctlxout_callback(struct urb *urb)
 {
 	struct hfa384x *hw = urb->context;
@@ -3730,23 +3789,24 @@ delresp:
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbctlx_reqtimerfn
-*
-* Timer response function for CTLX request timeouts.  If this
-* function is called, it means that the callback for the OUT
-* URB containing a Prism2.x XXX_Request was never called.
-*
-* Arguments:
-*	data		a ptr to the struct hfa384x
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	interrupt
-----------------------------------------------------------------*/
+ * hfa384x_usbctlx_reqtimerfn
+ *
+ * Timer response function for CTLX request timeouts.  If this
+ * function is called, it means that the callback for the OUT
+ * URB containing a Prism2.x XXX_Request was never called.
+ *
+ * Arguments:
+ *	data		a ptr to the struct hfa384x
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usbctlx_reqtimerfn(unsigned long data)
 {
 	struct hfa384x *hw = (struct hfa384x *)data;
@@ -3788,23 +3848,24 @@ static void hfa384x_usbctlx_reqtimerfn(unsigned long data)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbctlx_resptimerfn
-*
-* Timer response function for CTLX response timeouts.  If this
-* function is called, it means that the callback for the IN
-* URB containing a Prism2.x XXX_Response was never called.
-*
-* Arguments:
-*	data		a ptr to the struct hfa384x
-*
-* Returns:
-*	nothing
-*
-* Side effects:
-*
-* Call context:
-*	interrupt
-----------------------------------------------------------------*/
+ * hfa384x_usbctlx_resptimerfn
+ *
+ * Timer response function for CTLX response timeouts.  If this
+ * function is called, it means that the callback for the IN
+ * URB containing a Prism2.x XXX_Response was never called.
+ *
+ * Arguments:
+ *	data		a ptr to the struct hfa384x
+ *
+ * Returns:
+ *	nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usbctlx_resptimerfn(unsigned long data)
 {
 	struct hfa384x *hw = (struct hfa384x *)data;
@@ -3830,20 +3891,21 @@ static void hfa384x_usbctlx_resptimerfn(unsigned long data)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usb_throttlefn
-*
-*
-* Arguments:
-*	data	ptr to hw
-*
-* Returns:
-*	Nothing
-*
-* Side effects:
-*
-* Call context:
-*	Interrupt
-----------------------------------------------------------------*/
+ * hfa384x_usb_throttlefn
+ *
+ *
+ * Arguments:
+ *	data	ptr to hw
+ *
+ * Returns:
+ *	Nothing
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	Interrupt
+ *----------------------------------------------------------------
+ */
 static void hfa384x_usb_throttlefn(unsigned long data)
 {
 	struct hfa384x *hw = (struct hfa384x *)data;
@@ -3869,24 +3931,26 @@ static void hfa384x_usb_throttlefn(unsigned long data)
 }
 
 /*----------------------------------------------------------------
-* hfa384x_usbctlx_submit
-*
-* Called from the doxxx functions to submit a CTLX to the queue
-*
-* Arguments:
-*	hw		ptr to the hw struct
-*	ctlx		ctlx structure to enqueue
-*
-* Returns:
-*	-ENODEV if the adapter is unplugged
-*	0
-*
-* Side effects:
-*
-* Call context:
-*	process or interrupt
-----------------------------------------------------------------*/
-static int hfa384x_usbctlx_submit(struct hfa384x *hw, struct hfa384x_usbctlx *ctlx)
+ * hfa384x_usbctlx_submit
+ *
+ * Called from the doxxx functions to submit a CTLX to the queue
+ *
+ * Arguments:
+ *	hw		ptr to the hw struct
+ *	ctlx		ctlx structure to enqueue
+ *
+ * Returns:
+ *	-ENODEV if the adapter is unplugged
+ *	0
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	process or interrupt
+ *----------------------------------------------------------------
+ */
+static int hfa384x_usbctlx_submit(struct hfa384x *hw,
+				  struct hfa384x_usbctlx *ctlx)
 {
 	unsigned long flags;
 
@@ -3906,21 +3970,22 @@ static int hfa384x_usbctlx_submit(struct hfa384x *hw, struct hfa384x_usbctlx *ct
 }
 
 /*----------------------------------------------------------------
-* hfa384x_isgood_pdrcore
-*
-* Quick check of PDR codes.
-*
-* Arguments:
-*	pdrcode		PDR code number (host order)
-*
-* Returns:
-*	zero		not good.
-*	one		is good.
-*
-* Side effects:
-*
-* Call context:
-----------------------------------------------------------------*/
+ * hfa384x_isgood_pdrcore
+ *
+ * Quick check of PDR codes.
+ *
+ * Arguments:
+ *	pdrcode		PDR code number (host order)
+ *
+ * Returns:
+ *	zero		not good.
+ *	one		is good.
+ *
+ * Side effects:
+ *
+ * Call context:
+ *----------------------------------------------------------------
+ */
 static int hfa384x_isgood_pdrcode(u16 pdrcode)
 {
 	switch (pdrcode) {
diff --git a/drivers/staging/wlan-ng/p80211conv.c b/drivers/staging/wlan-ng/p80211conv.c
index 0247cbc29145..8387e6a3031a 100644
--- a/drivers/staging/wlan-ng/p80211conv.c
+++ b/drivers/staging/wlan-ng/p80211conv.c
@@ -1,56 +1,56 @@
 /* src/p80211/p80211conv.c
-*
-* Ether/802.11 conversions and packet buffer routines
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-* This file defines the functions that perform Ethernet to/from
-* 802.11 frame conversions.
-*
-* --------------------------------------------------------------------
-*
-*================================================================
-*/
+ *
+ * Ether/802.11 conversions and packet buffer routines
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ * This file defines the functions that perform Ethernet to/from
+ * 802.11 frame conversions.
+ *
+ * --------------------------------------------------------------------
+ *
+ *================================================================
+ */
 
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -79,31 +79,31 @@ static const u8 oui_rfc1042[] = { 0x00, 0x00, 0x00 };
 static const u8 oui_8021h[] = { 0x00, 0x00, 0xf8 };
 
 /*----------------------------------------------------------------
-* p80211pb_ether_to_80211
-*
-* Uses the contents of the ether frame and the etherconv setting
-* to build the elements of the 802.11 frame.
-*
-* We don't actually set
-* up the frame header here.  That's the MAC's job.  We're only handling
-* conversion of DIXII or 802.3+LLC frames to something that works
-* with 802.11.
-*
-* Note -- 802.11 header is NOT part of the skb.  Likewise, the 802.11
-*         FCS is also not present and will need to be added elsewhere.
-*
-* Arguments:
-*	ethconv		Conversion type to perform
-*	skb		skbuff containing the ether frame
-*       p80211_hdr      802.11 header
-*
-* Returns:
-*	0 on success, non-zero otherwise
-*
-* Call context:
-*	May be called in interrupt or non-interrupt context
-*----------------------------------------------------------------
-*/
+ * p80211pb_ether_to_80211
+ *
+ * Uses the contents of the ether frame and the etherconv setting
+ * to build the elements of the 802.11 frame.
+ *
+ * We don't actually set
+ * up the frame header here.  That's the MAC's job.  We're only handling
+ * conversion of DIXII or 802.3+LLC frames to something that works
+ * with 802.11.
+ *
+ * Note -- 802.11 header is NOT part of the skb.  Likewise, the 802.11
+ *         FCS is also not present and will need to be added elsewhere.
+ *
+ * Arguments:
+ *	ethconv		Conversion type to perform
+ *	skb		skbuff containing the ether frame
+ *       p80211_hdr      802.11 header
+ *
+ * Returns:
+ *	0 on success, non-zero otherwise
+ *
+ * Call context:
+ *	May be called in interrupt or non-interrupt context
+ *----------------------------------------------------------------
+ */
 int skb_ether_to_p80211(struct wlandevice *wlandev, u32 ethconv,
 			struct sk_buff *skb, union p80211_hdr *p80211_hdr,
 			struct p80211_metawep *p80211_wep)
@@ -255,25 +255,25 @@ static void orinoco_spy_gather(struct wlandevice *wlandev, char *mac,
 }
 
 /*----------------------------------------------------------------
-* p80211pb_80211_to_ether
-*
-* Uses the contents of a received 802.11 frame and the etherconv
-* setting to build an ether frame.
-*
-* This function extracts the src and dest address from the 802.11
-* frame to use in the construction of the eth frame.
-*
-* Arguments:
-*	ethconv		Conversion type to perform
-*	skb		Packet buffer containing the 802.11 frame
-*
-* Returns:
-*	0 on success, non-zero otherwise
-*
-* Call context:
-*	May be called in interrupt or non-interrupt context
-*----------------------------------------------------------------
-*/
+ * p80211pb_80211_to_ether
+ *
+ * Uses the contents of a received 802.11 frame and the etherconv
+ * setting to build an ether frame.
+ *
+ * This function extracts the src and dest address from the 802.11
+ * frame to use in the construction of the eth frame.
+ *
+ * Arguments:
+ *	ethconv		Conversion type to perform
+ *	skb		Packet buffer containing the 802.11 frame
+ *
+ * Returns:
+ *	0 on success, non-zero otherwise
+ *
+ * Call context:
+ *	May be called in interrupt or non-interrupt context
+ *----------------------------------------------------------------
+ */
 int skb_p80211_to_ether(struct wlandevice *wlandev, u32 ethconv,
 			struct sk_buff *skb)
 {
@@ -508,22 +508,22 @@ int skb_p80211_to_ether(struct wlandevice *wlandev, u32 ethconv,
 }
 
 /*----------------------------------------------------------------
-* p80211_stt_findproto
-*
-* Searches the 802.1h Selective Translation Table for a given
-* protocol.
-*
-* Arguments:
-*	proto	protocol number (in host order) to search for.
-*
-* Returns:
-*	1 - if the table is empty or a match is found.
-*	0 - if the table is non-empty and a match is not found.
-*
-* Call context:
-*	May be called in interrupt or non-interrupt context
-*----------------------------------------------------------------
-*/
+ * p80211_stt_findproto
+ *
+ * Searches the 802.1h Selective Translation Table for a given
+ * protocol.
+ *
+ * Arguments:
+ *	proto	protocol number (in host order) to search for.
+ *
+ * Returns:
+ *	1 - if the table is empty or a match is found.
+ *	0 - if the table is non-empty and a match is not found.
+ *
+ * Call context:
+ *	May be called in interrupt or non-interrupt context
+ *----------------------------------------------------------------
+ */
 int p80211_stt_findproto(u16 proto)
 {
 	/* Always return found for now.  This is the behavior used by the */
@@ -540,21 +540,21 @@ int p80211_stt_findproto(u16 proto)
 }
 
 /*----------------------------------------------------------------
-* p80211skb_rxmeta_detach
-*
-* Disconnects the frmmeta and rxmeta from an skb.
-*
-* Arguments:
-*	wlandev		The wlandev this skb belongs to.
-*	skb		The skb we're attaching to.
-*
-* Returns:
-*	0 on success, non-zero otherwise
-*
-* Call context:
-*	May be called in interrupt or non-interrupt context
-*----------------------------------------------------------------
-*/
+ * p80211skb_rxmeta_detach
+ *
+ * Disconnects the frmmeta and rxmeta from an skb.
+ *
+ * Arguments:
+ *	wlandev		The wlandev this skb belongs to.
+ *	skb		The skb we're attaching to.
+ *
+ * Returns:
+ *	0 on success, non-zero otherwise
+ *
+ * Call context:
+ *	May be called in interrupt or non-interrupt context
+ *----------------------------------------------------------------
+ */
 void p80211skb_rxmeta_detach(struct sk_buff *skb)
 {
 	struct p80211_rxmeta *rxmeta;
@@ -584,22 +584,22 @@ void p80211skb_rxmeta_detach(struct sk_buff *skb)
 }
 
 /*----------------------------------------------------------------
-* p80211skb_rxmeta_attach
-*
-* Allocates a p80211rxmeta structure, initializes it, and attaches
-* it to an skb.
-*
-* Arguments:
-*	wlandev		The wlandev this skb belongs to.
-*	skb		The skb we're attaching to.
-*
-* Returns:
-*	0 on success, non-zero otherwise
-*
-* Call context:
-*	May be called in interrupt or non-interrupt context
-*----------------------------------------------------------------
-*/
+ * p80211skb_rxmeta_attach
+ *
+ * Allocates a p80211rxmeta structure, initializes it, and attaches
+ * it to an skb.
+ *
+ * Arguments:
+ *	wlandev		The wlandev this skb belongs to.
+ *	skb		The skb we're attaching to.
+ *
+ * Returns:
+ *	0 on success, non-zero otherwise
+ *
+ * Call context:
+ *	May be called in interrupt or non-interrupt context
+ *----------------------------------------------------------------
+ */
 int p80211skb_rxmeta_attach(struct wlandevice *wlandev, struct sk_buff *skb)
 {
 	int result = 0;
@@ -615,11 +615,9 @@ int p80211skb_rxmeta_attach(struct wlandevice *wlandev, struct sk_buff *skb)
 	}
 
 	/* Allocate the rxmeta */
-	rxmeta = kzalloc(sizeof(struct p80211_rxmeta), GFP_ATOMIC);
+	rxmeta = kzalloc(sizeof(*rxmeta), GFP_ATOMIC);
 
 	if (!rxmeta) {
-		netdev_err(wlandev->netdev,
-			   "%s: Failed to allocate rxmeta.\n", wlandev->name);
 		result = 1;
 		goto exit;
 	}
@@ -638,22 +636,22 @@ exit:
 }
 
 /*----------------------------------------------------------------
-* p80211skb_free
-*
-* Frees an entire p80211skb by checking and freeing the meta struct
-* and then freeing the skb.
-*
-* Arguments:
-*	wlandev		The wlandev this skb belongs to.
-*	skb		The skb we're attaching to.
-*
-* Returns:
-*	0 on success, non-zero otherwise
-*
-* Call context:
-*	May be called in interrupt or non-interrupt context
-*----------------------------------------------------------------
-*/
+ * p80211skb_free
+ *
+ * Frees an entire p80211skb by checking and freeing the meta struct
+ * and then freeing the skb.
+ *
+ * Arguments:
+ *	wlandev		The wlandev this skb belongs to.
+ *	skb		The skb we're attaching to.
+ *
+ * Returns:
+ *	0 on success, non-zero otherwise
+ *
+ * Call context:
+ *	May be called in interrupt or non-interrupt context
+ *----------------------------------------------------------------
+ */
 void p80211skb_free(struct wlandevice *wlandev, struct sk_buff *skb)
 {
 	struct p80211_frmmeta *meta;
diff --git a/drivers/staging/wlan-ng/p80211conv.h b/drivers/staging/wlan-ng/p80211conv.h
index 8c10357bedf0..ed70d98e5cf1 100644
--- a/drivers/staging/wlan-ng/p80211conv.h
+++ b/drivers/staging/wlan-ng/p80211conv.h
@@ -1,54 +1,54 @@
 /* p80211conv.h
-*
-* Ether/802.11 conversions and packet buffer routines
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-* This file declares the functions, types and macros that perform
-* Ethernet to/from 802.11 frame conversions.
-*
-* --------------------------------------------------------------------
-*/
+ *
+ * Ether/802.11 conversions and packet buffer routines
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ * This file declares the functions, types and macros that perform
+ * Ethernet to/from 802.11 frame conversions.
+ *
+ * --------------------------------------------------------------------
+ */
 
 #ifndef _LINUX_P80211CONV_H
 #define _LINUX_P80211CONV_H
diff --git a/drivers/staging/wlan-ng/p80211hdr.h b/drivers/staging/wlan-ng/p80211hdr.h
index 79d9b20b364d..2c44c613a586 100644
--- a/drivers/staging/wlan-ng/p80211hdr.h
+++ b/drivers/staging/wlan-ng/p80211hdr.h
@@ -1,61 +1,61 @@
 /* p80211hdr.h
-*
-* Macros, types, and functions for handling 802.11 MAC headers
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-* This file declares the constants and types used in the interface
-* between a wlan driver and the user mode utilities.
-*
-* Note:
-*  - Constant values are always in HOST byte order.  To assign
-*    values to multi-byte fields they _must_ be converted to
-*    ieee byte order.  To retrieve multi-byte values from incoming
-*    frames, they must be converted to host order.
-*
-* All functions declared here are implemented in p80211.c
-* --------------------------------------------------------------------
-*/
+ *
+ * Macros, types, and functions for handling 802.11 MAC headers
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ * This file declares the constants and types used in the interface
+ * between a wlan driver and the user mode utilities.
+ *
+ * Note:
+ *  - Constant values are always in HOST byte order.  To assign
+ *    values to multi-byte fields they _must_ be converted to
+ *    ieee byte order.  To retrieve multi-byte values from incoming
+ *    frames, they must be converted to host order.
+ *
+ * All functions declared here are implemented in p80211.c
+ * --------------------------------------------------------------------
+ */
 
 #ifndef _P80211HDR_H
 #define _P80211HDR_H
@@ -131,8 +131,8 @@
 /*                        SET_FC_FSTYPE(WLAN_FSTYPE_RTS) );   */
 /*------------------------------------------------------------*/
 
-#define WLAN_GET_FC_FTYPE(n)	((((u16)(n)) & (BIT(2) | BIT(3))) >> 2)
-#define WLAN_GET_FC_FSTYPE(n)	((((u16)(n)) & (BIT(4)|BIT(5)|BIT(6)|BIT(7))) >> 4)
+#define WLAN_GET_FC_FTYPE(n)	((((u16)(n)) & GENMASK(3, 2)) >> 2)
+#define WLAN_GET_FC_FSTYPE(n)	((((u16)(n)) & GENMASK(7, 4)) >> 4)
 #define WLAN_GET_FC_TODS(n)	((((u16)(n)) & (BIT(8))) >> 8)
 #define WLAN_GET_FC_FROMDS(n)	((((u16)(n)) & (BIT(9))) >> 9)
 #define WLAN_GET_FC_ISWEP(n)	((((u16)(n)) & (BIT(14))) >> 14)
diff --git a/drivers/staging/wlan-ng/p80211ioctl.h b/drivers/staging/wlan-ng/p80211ioctl.h
index 06c5e36649a7..ab6067e65050 100644
--- a/drivers/staging/wlan-ng/p80211ioctl.h
+++ b/drivers/staging/wlan-ng/p80211ioctl.h
@@ -1,64 +1,64 @@
 /* p80211ioctl.h
-*
-* Declares constants and types for the p80211 ioctls
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-*  While this file is called 'ioctl' is purpose goes a little beyond
-*  that.  This file defines the types and contants used to implement
-*  the p80211 request/confirm/indicate interfaces on Linux.  The
-*  request/confirm interface is, in fact, normally implemented as an
-*  ioctl.  The indicate interface on the other hand, is implemented
-*  using the Linux 'netlink' interface.
-*
-*  The reason I say that request/confirm is 'normally' implemented
-*  via ioctl is that we're reserving the right to be able to send
-*  request commands via the netlink interface.  This will be necessary
-*  if we ever need to send request messages when there aren't any
-*  wlan network devices present (i.e. sending a message that only p80211
-*  cares about.
-* --------------------------------------------------------------------
-*/
+ *
+ * Declares constants and types for the p80211 ioctls
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ *  While this file is called 'ioctl' is purpose goes a little beyond
+ *  that.  This file defines the types and contants used to implement
+ *  the p80211 request/confirm/indicate interfaces on Linux.  The
+ *  request/confirm interface is, in fact, normally implemented as an
+ *  ioctl.  The indicate interface on the other hand, is implemented
+ *  using the Linux 'netlink' interface.
+ *
+ *  The reason I say that request/confirm is 'normally' implemented
+ *  via ioctl is that we're reserving the right to be able to send
+ *  request commands via the netlink interface.  This will be necessary
+ *  if we ever need to send request messages when there aren't any
+ *  wlan network devices present (i.e. sending a message that only p80211
+ *  cares about.
+ * --------------------------------------------------------------------
+ */
 
 #ifndef _P80211IOCTL_H
 #define _P80211IOCTL_H
diff --git a/drivers/staging/wlan-ng/p80211metadef.h b/drivers/staging/wlan-ng/p80211metadef.h
index b0d3567ca0ad..ea3d9ce222b9 100644
--- a/drivers/staging/wlan-ng/p80211metadef.h
+++ b/drivers/staging/wlan-ng/p80211metadef.h
@@ -1,48 +1,48 @@
 /* This file is GENERATED AUTOMATICALLY.  DO NOT EDIT OR MODIFY.
-* --------------------------------------------------------------------
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*/
+ * --------------------------------------------------------------------
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ */
 
 #ifndef _P80211MKMETADEF_H
 #define _P80211MKMETADEF_H
diff --git a/drivers/staging/wlan-ng/p80211mgmt.h b/drivers/staging/wlan-ng/p80211mgmt.h
index 3dd066ac034e..653950fd9843 100644
--- a/drivers/staging/wlan-ng/p80211mgmt.h
+++ b/drivers/staging/wlan-ng/p80211mgmt.h
@@ -1,101 +1,101 @@
 /* p80211mgmt.h
-*
-* Macros, types, and functions to handle 802.11 mgmt frames
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-* This file declares the constants and types used in the interface
-* between a wlan driver and the user mode utilities.
-*
-* Notes:
-*  - Constant values are always in HOST byte order.  To assign
-*    values to multi-byte fields they _must_ be converted to
-*    ieee byte order.  To retrieve multi-byte values from incoming
-*    frames, they must be converted to host order.
-*
-*  - The len member of the frame structure does NOT!!! include
-*    the MAC CRC.  Therefore, the len field on rx'd frames should
-*    have 4 subtracted from it.
-*
-* All functions declared here are implemented in p80211.c
-*
-* The types, macros, and functions defined here are primarily
-* used for encoding and decoding management frames.  They are
-* designed to follow these patterns of use:
-*
-* DECODE:
-* 1) a frame of length len is received into buffer b
-* 2) using the hdr structure and macros, we determine the type
-* 3) an appropriate mgmt frame structure, mf, is allocated and zeroed
-* 4) mf.hdr = b
-*    mf.buf = b
-*    mf.len = len
-* 5) call mgmt_decode( mf )
-* 6) the frame field pointers in mf are now set.  Note that any
-*    multi-byte frame field values accessed using the frame field
-*    pointers are in ieee byte order and will have to be converted
-*    to host order.
-*
-* ENCODE:
-* 1) Library client allocates buffer space for maximum length
-*    frame of the desired type
-* 2) Library client allocates a mgmt frame structure, called mf,
-*    of the desired type
-* 3) Set the following:
-*    mf.type = <desired type>
-*    mf.buf = <allocated buffer address>
-* 4) call mgmt_encode( mf )
-* 5) all of the fixed field pointers and fixed length information element
-*    pointers in mf are now set to their respective locations in the
-*    allocated space (fortunately, all variable length information elements
-*    fall at the end of their respective frames).
-* 5a) The length field is set to include the last of the fixed and fixed
-*     length fields.  It may have to be updated for optional or variable
-*	length information elements.
-* 6) Optional and variable length information elements are special cases
-*    and must be handled individually by the client code.
-* --------------------------------------------------------------------
-*/
+ *
+ * Macros, types, and functions to handle 802.11 mgmt frames
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ * This file declares the constants and types used in the interface
+ * between a wlan driver and the user mode utilities.
+ *
+ * Notes:
+ *  - Constant values are always in HOST byte order.  To assign
+ *    values to multi-byte fields they _must_ be converted to
+ *    ieee byte order.  To retrieve multi-byte values from incoming
+ *    frames, they must be converted to host order.
+ *
+ *  - The len member of the frame structure does NOT!!! include
+ *    the MAC CRC.  Therefore, the len field on rx'd frames should
+ *    have 4 subtracted from it.
+ *
+ * All functions declared here are implemented in p80211.c
+ *
+ * The types, macros, and functions defined here are primarily
+ * used for encoding and decoding management frames.  They are
+ * designed to follow these patterns of use:
+ *
+ * DECODE:
+ * 1) a frame of length len is received into buffer b
+ * 2) using the hdr structure and macros, we determine the type
+ * 3) an appropriate mgmt frame structure, mf, is allocated and zeroed
+ * 4) mf.hdr = b
+ *    mf.buf = b
+ *    mf.len = len
+ * 5) call mgmt_decode( mf )
+ * 6) the frame field pointers in mf are now set.  Note that any
+ *    multi-byte frame field values accessed using the frame field
+ *    pointers are in ieee byte order and will have to be converted
+ *    to host order.
+ *
+ * ENCODE:
+ * 1) Library client allocates buffer space for maximum length
+ *    frame of the desired type
+ * 2) Library client allocates a mgmt frame structure, called mf,
+ *    of the desired type
+ * 3) Set the following:
+ *    mf.type = <desired type>
+ *    mf.buf = <allocated buffer address>
+ * 4) call mgmt_encode( mf )
+ * 5) all of the fixed field pointers and fixed length information element
+ *    pointers in mf are now set to their respective locations in the
+ *    allocated space (fortunately, all variable length information elements
+ *    fall at the end of their respective frames).
+ * 5a) The length field is set to include the last of the fixed and fixed
+ *     length fields.  It may have to be updated for optional or variable
+ *	length information elements.
+ * 6) Optional and variable length information elements are special cases
+ *    and must be handled individually by the client code.
+ * --------------------------------------------------------------------
+ */
 
 #ifndef _P80211MGMT_H
 #define _P80211MGMT_H
diff --git a/drivers/staging/wlan-ng/p80211msg.h b/drivers/staging/wlan-ng/p80211msg.h
index 43d2f971e2cd..40c5cf5997c7 100644
--- a/drivers/staging/wlan-ng/p80211msg.h
+++ b/drivers/staging/wlan-ng/p80211msg.h
@@ -1,49 +1,49 @@
 /* p80211msg.h
-*
-* Macros, constants, types, and funcs for req and ind messages
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*/
+ *
+ * Macros, constants, types, and funcs for req and ind messages
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ */
 
 #ifndef _P80211MSG_H
 #define _P80211MSG_H
diff --git a/drivers/staging/wlan-ng/p80211netdev.c b/drivers/staging/wlan-ng/p80211netdev.c
index 4762d38a720e..73fcf07254fe 100644
--- a/drivers/staging/wlan-ng/p80211netdev.c
+++ b/drivers/staging/wlan-ng/p80211netdev.c
@@ -1,53 +1,53 @@
 /* src/p80211/p80211knetdev.c
-*
-* Linux Kernel net device interface
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-* The functions required for a Linux network device are defined here.
-*
-* --------------------------------------------------------------------
-*/
+ *
+ * Linux Kernel net device interface
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ * The functions required for a Linux network device are defined here.
+ *
+ * --------------------------------------------------------------------
+ */
 
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -112,17 +112,18 @@ module_param(wlan_wext_write, int, 0644);
 MODULE_PARM_DESC(wlan_wext_write, "enable write wireless extensions");
 
 /*----------------------------------------------------------------
-* p80211knetdev_init
-*
-* Init method for a Linux netdevice.  Called in response to
-* register_netdev.
-*
-* Arguments:
-*	none
-*
-* Returns:
-*	nothing
-----------------------------------------------------------------*/
+ * p80211knetdev_init
+ *
+ * Init method for a Linux netdevice.  Called in response to
+ * register_netdev.
+ *
+ * Arguments:
+ *	none
+ *
+ * Returns:
+ *	nothing
+ *----------------------------------------------------------------
+ */
 static int p80211knetdev_init(struct net_device *netdev)
 {
 	/* Called in response to register_netdev */
@@ -133,19 +134,20 @@ static int p80211knetdev_init(struct net_device *netdev)
 }
 
 /*----------------------------------------------------------------
-* p80211knetdev_open
-*
-* Linux netdevice open method.  Following a successful call here,
-* the device is supposed to be ready for tx and rx.  In our
-* situation that may not be entirely true due to the state of the
-* MAC below.
-*
-* Arguments:
-*	netdev		Linux network device structure
-*
-* Returns:
-*	zero on success, non-zero otherwise
-----------------------------------------------------------------*/
+ * p80211knetdev_open
+ *
+ * Linux netdevice open method.  Following a successful call here,
+ * the device is supposed to be ready for tx and rx.  In our
+ * situation that may not be entirely true due to the state of the
+ * MAC below.
+ *
+ * Arguments:
+ *	netdev		Linux network device structure
+ *
+ * Returns:
+ *	zero on success, non-zero otherwise
+ *----------------------------------------------------------------
+ */
 static int p80211knetdev_open(struct net_device *netdev)
 {
 	int result = 0;		/* success */
@@ -170,17 +172,18 @@ static int p80211knetdev_open(struct net_device *netdev)
 }
 
 /*----------------------------------------------------------------
-* p80211knetdev_stop
-*
-* Linux netdevice stop (close) method.  Following this call,
-* no frames should go up or down through this interface.
-*
-* Arguments:
-*	netdev		Linux network device structure
-*
-* Returns:
-*	zero on success, non-zero otherwise
-----------------------------------------------------------------*/
+ * p80211knetdev_stop
+ *
+ * Linux netdevice stop (close) method.  Following this call,
+ * no frames should go up or down through this interface.
+ *
+ * Arguments:
+ *	netdev		Linux network device structure
+ *
+ * Returns:
+ *	zero on success, non-zero otherwise
+ *----------------------------------------------------------------
+ */
 static int p80211knetdev_stop(struct net_device *netdev)
 {
 	int result = 0;
@@ -196,18 +199,19 @@ static int p80211knetdev_stop(struct net_device *netdev)
 }
 
 /*----------------------------------------------------------------
-* p80211netdev_rx
-*
-* Frame receive function called by the mac specific driver.
-*
-* Arguments:
-*	wlandev		WLAN network device structure
-*	skb		skbuff containing a full 802.11 frame.
-* Returns:
-*	nothing
-* Side effects:
-*
-----------------------------------------------------------------*/
+ * p80211netdev_rx
+ *
+ * Frame receive function called by the mac specific driver.
+ *
+ * Arguments:
+ *	wlandev		WLAN network device structure
+ *	skb		skbuff containing a full 802.11 frame.
+ * Returns:
+ *	nothing
+ * Side effects:
+ *
+ *----------------------------------------------------------------
+ */
 void p80211netdev_rx(struct wlandevice *wlandev, struct sk_buff *skb)
 {
 	/* Enqueue for post-irq processing */
@@ -227,7 +231,8 @@ void p80211netdev_rx(struct wlandevice *wlandev, struct sk_buff *skb)
  *	    CONV_TO_ETHER_FAILED if conversion failed
  *	    CONV_TO_ETHER_SKIPPED if frame is ignored
  */
-static int p80211_convert_to_ether(struct wlandevice *wlandev, struct sk_buff *skb)
+static int p80211_convert_to_ether(struct wlandevice *wlandev,
+				   struct sk_buff *skb)
 {
 	struct p80211_hdr_a3 *hdr;
 
@@ -272,7 +277,6 @@ static void p80211netdev_rx_bh(unsigned long arg)
 	/* Let's empty our our queue */
 	while ((skb = skb_dequeue(&wlandev->nsd_rxq))) {
 		if (wlandev->state == WLAN_DEVICE_OPEN) {
-
 			if (dev->type != ARPHRD_ETHER) {
 				/* RAW frame; we shouldn't convert it */
 				/* XXX Append the Prism Header here instead. */
@@ -299,24 +303,25 @@ static void p80211netdev_rx_bh(unsigned long arg)
 }
 
 /*----------------------------------------------------------------
-* p80211knetdev_hard_start_xmit
-*
-* Linux netdevice method for transmitting a frame.
-*
-* Arguments:
-*	skb	Linux sk_buff containing the frame.
-*	netdev	Linux netdevice.
-*
-* Side effects:
-*	If the lower layers report that buffers are full. netdev->tbusy
-*	will be set to prevent higher layers from sending more traffic.
-*
-*	Note: If this function returns non-zero, higher layers retain
-*	      ownership of the skb.
-*
-* Returns:
-*	zero on success, non-zero on failure.
-----------------------------------------------------------------*/
+ * p80211knetdev_hard_start_xmit
+ *
+ * Linux netdevice method for transmitting a frame.
+ *
+ * Arguments:
+ *	skb	Linux sk_buff containing the frame.
+ *	netdev	Linux netdevice.
+ *
+ * Side effects:
+ *	If the lower layers report that buffers are full. netdev->tbusy
+ *	will be set to prevent higher layers from sending more traffic.
+ *
+ *	Note: If this function returns non-zero, higher layers retain
+ *	      ownership of the skb.
+ *
+ * Returns:
+ *	zero on success, non-zero on failure.
+ *----------------------------------------------------------------
+ */
 static int p80211knetdev_hard_start_xmit(struct sk_buff *skb,
 					 struct net_device *netdev)
 {
@@ -336,8 +341,8 @@ static int p80211knetdev_hard_start_xmit(struct sk_buff *skb,
 		goto failed;
 	}
 
-	memset(&p80211_hdr, 0, sizeof(union p80211_hdr));
-	memset(&p80211_wep, 0, sizeof(struct p80211_metawep));
+	memset(&p80211_hdr, 0, sizeof(p80211_hdr));
+	memset(&p80211_wep, 0, sizeof(p80211_wep));
 
 	if (netif_queue_stopped(netdev)) {
 		netdev_dbg(netdev, "called when queue stopped.\n");
@@ -375,8 +380,8 @@ static int p80211knetdev_hard_start_xmit(struct sk_buff *skb,
 			goto failed;
 		}
 		/* move the header over */
-		memcpy(&p80211_hdr, skb->data, sizeof(union p80211_hdr));
-		skb_pull(skb, sizeof(union p80211_hdr));
+		memcpy(&p80211_hdr, skb->data, sizeof(p80211_hdr));
+		skb_pull(skb, sizeof(p80211_hdr));
 	} else {
 		if (skb_ether_to_p80211
 		    (wlandev, wlandev->ethconv, skb, &p80211_hdr,
@@ -435,17 +440,18 @@ failed:
 }
 
 /*----------------------------------------------------------------
-* p80211knetdev_set_multicast_list
-*
-* Called from higher layers whenever there's a need to set/clear
-* promiscuous mode or rewrite the multicast list.
-*
-* Arguments:
-*	none
-*
-* Returns:
-*	nothing
-----------------------------------------------------------------*/
+ * p80211knetdev_set_multicast_list
+ *
+ * Called from higher layers whenever there's a need to set/clear
+ * promiscuous mode or rewrite the multicast list.
+ *
+ * Arguments:
+ *	none
+ *
+ * Returns:
+ *	nothing
+ *----------------------------------------------------------------
+ */
 static void p80211knetdev_set_multicast_list(struct net_device *dev)
 {
 	struct wlandevice *wlandev = dev->ml_priv;
@@ -454,12 +460,12 @@ static void p80211knetdev_set_multicast_list(struct net_device *dev)
 
 	if (wlandev->set_multicast_list)
 		wlandev->set_multicast_list(wlandev, dev);
-
 }
 
 #ifdef SIOCETHTOOL
 
-static int p80211netdev_ethtool(struct wlandevice *wlandev, void __user *useraddr)
+static int p80211netdev_ethtool(struct wlandevice *wlandev,
+				void __user *useraddr)
 {
 	u32 ethcmd;
 	struct ethtool_drvinfo info;
@@ -505,33 +511,35 @@ static int p80211netdev_ethtool(struct wlandevice *wlandev, void __user *useradd
 #endif
 
 /*----------------------------------------------------------------
-* p80211knetdev_do_ioctl
-*
-* Handle an ioctl call on one of our devices.  Everything Linux
-* ioctl specific is done here.  Then we pass the contents of the
-* ifr->data to the request message handler.
-*
-* Arguments:
-*	dev	Linux kernel netdevice
-*	ifr	Our private ioctl request structure, typed for the
-*		generic struct ifreq so we can use ptr to func
-*		w/o cast.
-*
-* Returns:
-*	zero on success, a negative errno on failure.  Possible values:
-*		-ENETDOWN Device isn't up.
-*		-EBUSY	cmd already in progress
-*		-ETIME	p80211 cmd timed out (MSD may have its own timers)
-*		-EFAULT memory fault copying msg from user buffer
-*		-ENOMEM unable to allocate kernel msg buffer
-*		-ENOSYS	bad magic, it the cmd really for us?
-*		-EintR	sleeping on cmd, awakened by signal, cmd cancelled.
-*
-* Call Context:
-*	Process thread (ioctl caller).  TODO: SMP support may require
-*	locks.
-----------------------------------------------------------------*/
-static int p80211knetdev_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
+ * p80211knetdev_do_ioctl
+ *
+ * Handle an ioctl call on one of our devices.  Everything Linux
+ * ioctl specific is done here.  Then we pass the contents of the
+ * ifr->data to the request message handler.
+ *
+ * Arguments:
+ *	dev	Linux kernel netdevice
+ *	ifr	Our private ioctl request structure, typed for the
+ *		generic struct ifreq so we can use ptr to func
+ *		w/o cast.
+ *
+ * Returns:
+ *	zero on success, a negative errno on failure.  Possible values:
+ *		-ENETDOWN Device isn't up.
+ *		-EBUSY	cmd already in progress
+ *		-ETIME	p80211 cmd timed out (MSD may have its own timers)
+ *		-EFAULT memory fault copying msg from user buffer
+ *		-ENOMEM unable to allocate kernel msg buffer
+ *		-EINVAL	bad magic, it the cmd really for us?
+ *		-EintR	sleeping on cmd, awakened by signal, cmd cancelled.
+ *
+ * Call Context:
+ *	Process thread (ioctl caller).  TODO: SMP support may require
+ *	locks.
+ *----------------------------------------------------------------
+ */
+static int p80211knetdev_do_ioctl(struct net_device *dev,
+				  struct ifreq *ifr, int cmd)
 {
 	int result = 0;
 	struct p80211ioctl_req *req = (struct p80211ioctl_req *)ifr;
@@ -550,7 +558,7 @@ static int p80211knetdev_do_ioctl(struct net_device *dev, struct ifreq *ifr, int
 
 	/* Test the magic, assume ifr is good if it's there */
 	if (req->magic != P80211_IOCTL_MAGIC) {
-		result = -ENOSYS;
+		result = -EINVAL;
 		goto bail;
 	}
 
@@ -558,7 +566,7 @@ static int p80211knetdev_do_ioctl(struct net_device *dev, struct ifreq *ifr, int
 		result = 0;
 		goto bail;
 	} else if (cmd != P80211_IFREQ) {
-		result = -ENOSYS;
+		result = -EINVAL;
 		goto bail;
 	}
 
@@ -586,30 +594,31 @@ bail:
 }
 
 /*----------------------------------------------------------------
-* p80211knetdev_set_mac_address
-*
-* Handles the ioctl for changing the MACAddress of a netdevice
-*
-* references: linux/netdevice.h and drivers/net/net_init.c
-*
-* NOTE: [MSM] We only prevent address changes when the netdev is
-* up.  We don't control anything based on dot11 state.  If the
-* address is changed on a STA that's currently associated, you
-* will probably lose the ability to send and receive data frames.
-* Just be aware.  Therefore, this should usually only be done
-* prior to scan/join/auth/assoc.
-*
-* Arguments:
-*	dev	netdevice struct
-*	addr	the new MACAddress (a struct)
-*
-* Returns:
-*	zero on success, a negative errno on failure.  Possible values:
-*		-EBUSY	device is bussy (cmd not possible)
-*		-and errors returned by: p80211req_dorequest(..)
-*
-* by: Collin R. Mulliner <collin@mulliner.org>
-----------------------------------------------------------------*/
+ * p80211knetdev_set_mac_address
+ *
+ * Handles the ioctl for changing the MACAddress of a netdevice
+ *
+ * references: linux/netdevice.h and drivers/net/net_init.c
+ *
+ * NOTE: [MSM] We only prevent address changes when the netdev is
+ * up.  We don't control anything based on dot11 state.  If the
+ * address is changed on a STA that's currently associated, you
+ * will probably lose the ability to send and receive data frames.
+ * Just be aware.  Therefore, this should usually only be done
+ * prior to scan/join/auth/assoc.
+ *
+ * Arguments:
+ *	dev	netdevice struct
+ *	addr	the new MACAddress (a struct)
+ *
+ * Returns:
+ *	zero on success, a negative errno on failure.  Possible values:
+ *		-EBUSY	device is bussy (cmd not possible)
+ *		-and errors returned by: p80211req_dorequest(..)
+ *
+ * by: Collin R. Mulliner <collin@mulliner.org>
+ *----------------------------------------------------------------
+ */
 static int p80211knetdev_set_mac_address(struct net_device *dev, void *addr)
 {
 	struct sockaddr *new_addr = addr;
@@ -629,9 +638,9 @@ static int p80211knetdev_set_mac_address(struct net_device *dev, void *addr)
 	resultcode = &dot11req.resultcode;
 
 	/* Set up a dot11req_mibset */
-	memset(&dot11req, 0, sizeof(struct p80211msg_dot11req_mibset));
+	memset(&dot11req, 0, sizeof(dot11req));
 	dot11req.msgcode = DIDmsg_dot11req_mibset;
-	dot11req.msglen = sizeof(struct p80211msg_dot11req_mibset);
+	dot11req.msglen = sizeof(dot11req);
 	memcpy(dot11req.devname,
 	       ((struct wlandevice *)dev->ml_priv)->name, WLAN_DEVNAMELEN_MAX - 1);
 
@@ -682,28 +691,29 @@ static const struct net_device_ops p80211_netdev_ops = {
 };
 
 /*----------------------------------------------------------------
-* wlan_setup
-*
-* Roughly matches the functionality of ether_setup.  Here
-* we set up any members of the wlandevice structure that are common
-* to all devices.  Additionally, we allocate a linux 'struct device'
-* and perform the same setup as ether_setup.
-*
-* Note: It's important that the caller have setup the wlandev->name
-*	ptr prior to calling this function.
-*
-* Arguments:
-*	wlandev		ptr to the wlandev structure for the
-*			interface.
-*	physdev		ptr to usb device
-* Returns:
-*	zero on success, non-zero otherwise.
-* Call Context:
-*	Should be process thread.  We'll assume it might be
-*	interrupt though.  When we add support for statically
-*	compiled drivers, this function will be called in the
-*	context of the kernel startup code.
-----------------------------------------------------------------*/
+ * wlan_setup
+ *
+ * Roughly matches the functionality of ether_setup.  Here
+ * we set up any members of the wlandevice structure that are common
+ * to all devices.  Additionally, we allocate a linux 'struct device'
+ * and perform the same setup as ether_setup.
+ *
+ * Note: It's important that the caller have setup the wlandev->name
+ *	ptr prior to calling this function.
+ *
+ * Arguments:
+ *	wlandev		ptr to the wlandev structure for the
+ *			interface.
+ *	physdev		ptr to usb device
+ * Returns:
+ *	zero on success, non-zero otherwise.
+ * Call Context:
+ *	Should be process thread.  We'll assume it might be
+ *	interrupt though.  When we add support for statically
+ *	compiled drivers, this function will be called in the
+ *	context of the kernel startup code.
+ *----------------------------------------------------------------
+ */
 int wlan_setup(struct wlandevice *wlandev, struct device *physdev)
 {
 	int result = 0;
@@ -757,24 +767,25 @@ int wlan_setup(struct wlandevice *wlandev, struct device *physdev)
 }
 
 /*----------------------------------------------------------------
-* wlan_unsetup
-*
-* This function is paired with the wlan_setup routine.  It should
-* be called after unregister_wlandev.  Basically, all it does is
-* free the 'struct device' that's associated with the wlandev.
-* We do it here because the 'struct device' isn't allocated
-* explicitly in the driver code, it's done in wlan_setup.  To
-* do the free in the driver might seem like 'magic'.
-*
-* Arguments:
-*	wlandev		ptr to the wlandev structure for the
-*			interface.
-* Call Context:
-*	Should be process thread.  We'll assume it might be
-*	interrupt though.  When we add support for statically
-*	compiled drivers, this function will be called in the
-*	context of the kernel startup code.
-----------------------------------------------------------------*/
+ * wlan_unsetup
+ *
+ * This function is paired with the wlan_setup routine.  It should
+ * be called after unregister_wlandev.  Basically, all it does is
+ * free the 'struct device' that's associated with the wlandev.
+ * We do it here because the 'struct device' isn't allocated
+ * explicitly in the driver code, it's done in wlan_setup.  To
+ * do the free in the driver might seem like 'magic'.
+ *
+ * Arguments:
+ *	wlandev		ptr to the wlandev structure for the
+ *			interface.
+ * Call Context:
+ *	Should be process thread.  We'll assume it might be
+ *	interrupt though.  When we add support for statically
+ *	compiled drivers, this function will be called in the
+ *	context of the kernel startup code.
+ *----------------------------------------------------------------
+ */
 void wlan_unsetup(struct wlandevice *wlandev)
 {
 	struct wireless_dev *wdev;
@@ -791,46 +802,48 @@ void wlan_unsetup(struct wlandevice *wlandev)
 }
 
 /*----------------------------------------------------------------
-* register_wlandev
-*
-* Roughly matches the functionality of register_netdev.  This function
-* is called after the driver has successfully probed and set up the
-* resources for the device.  It's now ready to become a named device
-* in the Linux system.
-*
-* First we allocate a name for the device (if not already set), then
-* we call the Linux function register_netdevice.
-*
-* Arguments:
-*	wlandev		ptr to the wlandev structure for the
-*			interface.
-* Returns:
-*	zero on success, non-zero otherwise.
-* Call Context:
-*	Can be either interrupt or not.
-----------------------------------------------------------------*/
+ * register_wlandev
+ *
+ * Roughly matches the functionality of register_netdev.  This function
+ * is called after the driver has successfully probed and set up the
+ * resources for the device.  It's now ready to become a named device
+ * in the Linux system.
+ *
+ * First we allocate a name for the device (if not already set), then
+ * we call the Linux function register_netdevice.
+ *
+ * Arguments:
+ *	wlandev		ptr to the wlandev structure for the
+ *			interface.
+ * Returns:
+ *	zero on success, non-zero otherwise.
+ * Call Context:
+ *	Can be either interrupt or not.
+ *----------------------------------------------------------------
+ */
 int register_wlandev(struct wlandevice *wlandev)
 {
 	return register_netdev(wlandev->netdev);
 }
 
 /*----------------------------------------------------------------
-* unregister_wlandev
-*
-* Roughly matches the functionality of unregister_netdev.  This
-* function is called to remove a named device from the system.
-*
-* First we tell linux that the device should no longer exist.
-* Then we remove it from the list of known wlan devices.
-*
-* Arguments:
-*	wlandev		ptr to the wlandev structure for the
-*			interface.
-* Returns:
-*	zero on success, non-zero otherwise.
-* Call Context:
-*	Can be either interrupt or not.
-----------------------------------------------------------------*/
+ * unregister_wlandev
+ *
+ * Roughly matches the functionality of unregister_netdev.  This
+ * function is called to remove a named device from the system.
+ *
+ * First we tell linux that the device should no longer exist.
+ * Then we remove it from the list of known wlan devices.
+ *
+ * Arguments:
+ *	wlandev		ptr to the wlandev structure for the
+ *			interface.
+ * Returns:
+ *	zero on success, non-zero otherwise.
+ * Call Context:
+ *	Can be either interrupt or not.
+ *----------------------------------------------------------------
+ */
 int unregister_wlandev(struct wlandevice *wlandev)
 {
 	struct sk_buff *skb;
@@ -845,35 +858,36 @@ int unregister_wlandev(struct wlandevice *wlandev)
 }
 
 /*----------------------------------------------------------------
-* p80211netdev_hwremoved
-*
-* Hardware removed notification. This function should be called
-* immediately after an MSD has detected that the underlying hardware
-* has been yanked out from under us.  The primary things we need
-* to do are:
-*   - Mark the wlandev
-*   - Prevent any further traffic from the knetdev i/f
-*   - Prevent any further requests from mgmt i/f
-*   - If there are any waitq'd mgmt requests or mgmt-frame exchanges,
-*     shut them down.
-*   - Call the MSD hwremoved function.
-*
-* The remainder of the cleanup will be handled by unregister().
-* Our primary goal here is to prevent as much tickling of the MSD
-* as possible since the MSD is already in a 'wounded' state.
-*
-* TODO: As new features are added, this function should be
-*       updated.
-*
-* Arguments:
-*	wlandev		WLAN network device structure
-* Returns:
-*	nothing
-* Side effects:
-*
-* Call context:
-*	Usually interrupt.
-----------------------------------------------------------------*/
+ * p80211netdev_hwremoved
+ *
+ * Hardware removed notification. This function should be called
+ * immediately after an MSD has detected that the underlying hardware
+ * has been yanked out from under us.  The primary things we need
+ * to do are:
+ *   - Mark the wlandev
+ *   - Prevent any further traffic from the knetdev i/f
+ *   - Prevent any further requests from mgmt i/f
+ *   - If there are any waitq'd mgmt requests or mgmt-frame exchanges,
+ *     shut them down.
+ *   - Call the MSD hwremoved function.
+ *
+ * The remainder of the cleanup will be handled by unregister().
+ * Our primary goal here is to prevent as much tickling of the MSD
+ * as possible since the MSD is already in a 'wounded' state.
+ *
+ * TODO: As new features are added, this function should be
+ *       updated.
+ *
+ * Arguments:
+ *	wlandev		WLAN network device structure
+ * Returns:
+ *	nothing
+ * Side effects:
+ *
+ * Call context:
+ *	Usually interrupt.
+ *----------------------------------------------------------------
+ */
 void p80211netdev_hwremoved(struct wlandevice *wlandev)
 {
 	wlandev->hwremoved = 1;
@@ -884,26 +898,27 @@ void p80211netdev_hwremoved(struct wlandevice *wlandev)
 }
 
 /*----------------------------------------------------------------
-* p80211_rx_typedrop
-*
-* Classifies the frame, increments the appropriate counter, and
-* returns 0|1|2 indicating whether the driver should handle, ignore, or
-* drop the frame
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	fc		frame control field
-*
-* Returns:
-*	zero if the frame should be handled by the driver,
-*       one if the frame should be ignored
-*       anything else means we drop it.
-*
-* Side effects:
-*
-* Call context:
-*	interrupt
-----------------------------------------------------------------*/
+ * p80211_rx_typedrop
+ *
+ * Classifies the frame, increments the appropriate counter, and
+ * returns 0|1|2 indicating whether the driver should handle, ignore, or
+ * drop the frame
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	fc		frame control field
+ *
+ * Returns:
+ *	zero if the frame should be handled by the driver,
+ *       one if the frame should be ignored
+ *       anything else means we drop it.
+ *
+ * Side effects:
+ *
+ * Call context:
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 static int p80211_rx_typedrop(struct wlandevice *wlandev, u16 fc)
 {
 	u16 ftype;
diff --git a/drivers/staging/wlan-ng/p80211netdev.h b/drivers/staging/wlan-ng/p80211netdev.h
index 1e6a774fc7c5..8e0d08298c8b 100644
--- a/drivers/staging/wlan-ng/p80211netdev.h
+++ b/drivers/staging/wlan-ng/p80211netdev.h
@@ -1,54 +1,54 @@
 /* p80211netdev.h
-*
-* WLAN net device structure and functions
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-* This file declares the structure type that represents each wlan
-* interface.
-*
-* --------------------------------------------------------------------
-*/
+ *
+ * WLAN net device structure and functions
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ * This file declares the structure type that represents each wlan
+ * interface.
+ *
+ * --------------------------------------------------------------------
+ */
 
 #ifndef _LINUX_P80211NETDEV_H
 #define _LINUX_P80211NETDEV_H
@@ -143,7 +143,7 @@ extern struct iw_handler_def p80211wext_handler_def;
 #define NUM_WEPKEYS 4
 #define MAX_KEYLEN 32
 
-#define HOSTWEP_DEFAULTKEY_MASK (BIT(1)|BIT(0))
+#define HOSTWEP_DEFAULTKEY_MASK GENMASK(1, 0)
 #define HOSTWEP_SHAREDKEY BIT(3)
 #define HOSTWEP_DECRYPT  BIT(4)
 #define HOSTWEP_ENCRYPT  BIT(5)
diff --git a/drivers/staging/wlan-ng/p80211req.c b/drivers/staging/wlan-ng/p80211req.c
index d43e85b5d49b..621df98183bf 100644
--- a/drivers/staging/wlan-ng/p80211req.c
+++ b/drivers/staging/wlan-ng/p80211req.c
@@ -1,54 +1,54 @@
 /* src/p80211/p80211req.c
-*
-* Request/Indication/MacMgmt interface handling functions
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-* This file contains the functions, types, and macros to support the
-* MLME request interface that's implemented via the device ioctls.
-*
-* --------------------------------------------------------------------
-*/
+ *
+ * Request/Indication/MacMgmt interface handling functions
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ * This file contains the functions, types, and macros to support the
+ * MLME request interface that's implemented via the device ioctls.
+ *
+ * --------------------------------------------------------------------
+ */
 
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -72,10 +72,11 @@
 #include "p80211metastruct.h"
 #include "p80211req.h"
 
-static void p80211req_handlemsg(struct wlandevice *wlandev, struct p80211msg *msg);
+static void p80211req_handlemsg(struct wlandevice *wlandev,
+				struct p80211msg *msg);
 static void p80211req_mibset_mibget(struct wlandevice *wlandev,
-				   struct p80211msg_dot11req_mibget *mib_msg,
-				   int isget);
+				    struct p80211msg_dot11req_mibget *mib_msg,
+				    int isget);
 
 static void p80211req_handle_action(struct wlandevice *wlandev, u32 *data,
 				    int isget, u32 flag)
@@ -93,21 +94,22 @@ static void p80211req_handle_action(struct wlandevice *wlandev, u32 *data,
 }
 
 /*----------------------------------------------------------------
-* p80211req_dorequest
-*
-* Handles an MLME request/confirm message.
-*
-* Arguments:
-*	wlandev		WLAN device struct
-*	msgbuf		Buffer containing a request message
-*
-* Returns:
-*	0 on success, an errno otherwise
-*
-* Call context:
-*	Potentially blocks the caller, so it's a good idea to
-*	not call this function from an interrupt context.
-----------------------------------------------------------------*/
+ * p80211req_dorequest
+ *
+ * Handles an MLME request/confirm message.
+ *
+ * Arguments:
+ *	wlandev		WLAN device struct
+ *	msgbuf		Buffer containing a request message
+ *
+ * Returns:
+ *	0 on success, an errno otherwise
+ *
+ * Call context:
+ *	Potentially blocks the caller, so it's a good idea to
+ *	not call this function from an interrupt context.
+ *----------------------------------------------------------------
+ */
 int p80211req_dorequest(struct wlandevice *wlandev, u8 *msgbuf)
 {
 	struct p80211msg *msg = (struct p80211msg *)msgbuf;
@@ -122,7 +124,7 @@ int p80211req_dorequest(struct wlandevice *wlandev, u8 *msgbuf)
 
 	/* Check Permissions */
 	if (!capable(CAP_NET_ADMIN) &&
-	(msg->msgcode != DIDmsg_dot11req_mibget)) {
+	    (msg->msgcode != DIDmsg_dot11req_mibget)) {
 		netdev_err(wlandev->netdev,
 			   "%s: only dot11req_mibget allowed for non-root.\n",
 			   wlandev->name);
@@ -130,7 +132,7 @@ int p80211req_dorequest(struct wlandevice *wlandev, u8 *msgbuf)
 	}
 
 	/* Check for busy status */
-	if (test_and_set_bit(1, &(wlandev->request_pending)))
+	if (test_and_set_bit(1, &wlandev->request_pending))
 		return -EBUSY;
 
 	/* Allow p80211 to look at msg and handle if desired. */
@@ -139,35 +141,36 @@ int p80211req_dorequest(struct wlandevice *wlandev, u8 *msgbuf)
 	p80211req_handlemsg(wlandev, msg);
 
 	/* Pass it down to wlandev via wlandev->mlmerequest */
-	if (wlandev->mlmerequest != NULL)
+	if (wlandev->mlmerequest)
 		wlandev->mlmerequest(wlandev, msg);
 
-	clear_bit(1, &(wlandev->request_pending));
+	clear_bit(1, &wlandev->request_pending);
 	return 0;	/* if result==0, msg->status still may contain an err */
 }
 
 /*----------------------------------------------------------------
-* p80211req_handlemsg
-*
-* p80211 message handler.  Primarily looks for messages that
-* belong to p80211 and then dispatches the appropriate response.
-* TODO: we don't do anything yet.  Once the linuxMIB is better
-*	defined we'll need a get/set handler.
-*
-* Arguments:
-*	wlandev		WLAN device struct
-*	msg		message structure
-*
-* Returns:
-*	nothing (any results are set in the status field of the msg)
-*
-* Call context:
-*	Process thread
-----------------------------------------------------------------*/
-static void p80211req_handlemsg(struct wlandevice *wlandev, struct p80211msg *msg)
+ * p80211req_handlemsg
+ *
+ * p80211 message handler.  Primarily looks for messages that
+ * belong to p80211 and then dispatches the appropriate response.
+ * TODO: we don't do anything yet.  Once the linuxMIB is better
+ *	defined we'll need a get/set handler.
+ *
+ * Arguments:
+ *	wlandev		WLAN device struct
+ *	msg		message structure
+ *
+ * Returns:
+ *	nothing (any results are set in the status field of the msg)
+ *
+ * Call context:
+ *	Process thread
+ *----------------------------------------------------------------
+ */
+static void p80211req_handlemsg(struct wlandevice *wlandev,
+				struct p80211msg *msg)
 {
 	switch (msg->msgcode) {
-
 	case DIDmsg_lnxreq_hostwep:{
 		struct p80211msg_lnxreq_hostwep *req =
 			(struct p80211msg_lnxreq_hostwep *)msg;
@@ -192,8 +195,8 @@ static void p80211req_handlemsg(struct wlandevice *wlandev, struct p80211msg *ms
 }
 
 static void p80211req_mibset_mibget(struct wlandevice *wlandev,
-				   struct p80211msg_dot11req_mibget *mib_msg,
-				   int isget)
+				    struct p80211msg_dot11req_mibget *mib_msg,
+				    int isget)
 {
 	struct p80211itemd *mibitem = (struct p80211itemd *)mib_msg->mibattribute.data;
 	struct p80211pstrd *pstr = (struct p80211pstrd *)mibitem->data;
diff --git a/drivers/staging/wlan-ng/p80211req.h b/drivers/staging/wlan-ng/p80211req.h
index 8d3054c22a05..6c72f59993e0 100644
--- a/drivers/staging/wlan-ng/p80211req.h
+++ b/drivers/staging/wlan-ng/p80211req.h
@@ -1,49 +1,49 @@
 /* p80211req.h
-*
-* Request handling functions
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*/
+ *
+ * Request handling functions
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ */
 
 #ifndef _LINUX_P80211REQ_H
 #define _LINUX_P80211REQ_H
diff --git a/drivers/staging/wlan-ng/p80211wep.c b/drivers/staging/wlan-ng/p80211wep.c
index 23b183738037..6492ffe59085 100644
--- a/drivers/staging/wlan-ng/p80211wep.c
+++ b/drivers/staging/wlan-ng/p80211wep.c
@@ -1,49 +1,49 @@
 /* src/p80211/p80211wep.c
-*
-* WEP encode/decode for P80211.
-*
-* Copyright (C) 2002 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*/
+ *
+ * WEP encode/decode for P80211.
+ *
+ * Copyright (C) 2002 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ */
 
 /*================================================================*/
 /* System Includes */
@@ -52,8 +52,6 @@
 #include <linux/wireless.h>
 #include <linux/random.h>
 #include <linux/kernel.h>
-
-
 #include "p80211hdr.h"
 #include "p80211types.h"
 #include "p80211msg.h"
@@ -125,14 +123,13 @@ int wep_change_key(struct wlandevice *wlandev, int keynum, u8 *key, int keylen)
 		return -1;
 	if (keylen >= MAX_KEYLEN)
 		return -1;
-	if (key == NULL)
+	if (!key)
 		return -1;
 	if (keynum < 0)
 		return -1;
 	if (keynum >= NUM_WEPKEYS)
 		return -1;
 
-
 	wlandev->wep_keylens[keynum] = keylen;
 	memcpy(wlandev->wep_keys[keynum], key, keylen);
 
@@ -176,7 +173,6 @@ int wep_decrypt(struct wlandevice *wlandev, u8 *buf, u32 len, int key_override,
 
 	keylen += 3;		/* add in IV bytes */
 
-
 	/* set up the RC4 state */
 	for (i = 0; i < 256; i++)
 		s[i] = i;
@@ -217,8 +213,8 @@ int wep_decrypt(struct wlandevice *wlandev, u8 *buf, u32 len, int key_override,
 }
 
 /* encrypts in-place. */
-int wep_encrypt(struct wlandevice *wlandev, u8 *buf, u8 *dst, u32 len, int keynum,
-		u8 *iv, u8 *icv)
+int wep_encrypt(struct wlandevice *wlandev, u8 *buf,
+		u8 *dst, u32 len, int keynum, u8 *iv, u8 *icv)
 {
 	u32 i, j, k, crc, keylen;
 	u8 s[256], key[64];
diff --git a/drivers/staging/wlan-ng/prism2fw.c b/drivers/staging/wlan-ng/prism2fw.c
index 96aa21188669..2e349f87e738 100644
--- a/drivers/staging/wlan-ng/prism2fw.c
+++ b/drivers/staging/wlan-ng/prism2fw.c
@@ -1,49 +1,49 @@
 /* from src/prism2/download/prism2dl.c
-*
-* utility for downloading prism2 images moved into kernelspace
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*/
+ *
+ * utility for downloading prism2 images moved into kernelspace
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ */
 
 /*================================================================*/
 /* System Includes */
@@ -124,7 +124,7 @@ struct imgchunk {
 
 /* Data records */
 static unsigned int ns3data;
-static struct s3datarec s3data[S3DATA_MAX];
+static struct s3datarec *s3data;
 
 /* Plug records */
 static unsigned int ns3plug;
@@ -161,7 +161,7 @@ static struct hfa384x_caplevel priid;
 /* Local Function Declarations */
 
 static int prism2_fwapply(const struct ihex_binrec *rfptr,
-struct wlandevice *wlandev);
+			  struct wlandevice *wlandev);
 
 static int read_fwfile(const struct ihex_binrec *rfptr);
 
@@ -172,13 +172,15 @@ static int read_cardpda(struct pda *pda, struct wlandevice *wlandev);
 static int mkpdrlist(struct pda *pda);
 
 static int plugimage(struct imgchunk *fchunk, unsigned int nfchunks,
-	      struct s3plugrec *s3plug, unsigned int ns3plug, struct pda *pda);
+		     struct s3plugrec *s3plug, unsigned int ns3plug,
+		     struct pda *pda);
 
 static int crcimage(struct imgchunk *fchunk, unsigned int nfchunks,
-	     struct s3crcrec *s3crc, unsigned int ns3crc);
+		    struct s3crcrec *s3crc, unsigned int ns3crc);
 
 static int writeimage(struct wlandevice *wlandev, struct imgchunk *fchunk,
-	       unsigned int nfchunks);
+		      unsigned int nfchunks);
+
 static void free_chunks(struct imgchunk *fchunk, unsigned int *nfchunks);
 
 static void free_srecs(void);
@@ -189,30 +191,31 @@ static int validate_identity(void);
 /* Function Definitions */
 
 /*----------------------------------------------------------------
-* prism2_fwtry
-*
-* Try and get firmware into memory
-*
-* Arguments:
-*	udev	usb device structure
-*	wlandev wlan device structure
-*
-* Returns:
-*	0	- success
-*	~0	- failure
-----------------------------------------------------------------*/
+ * prism2_fwtry
+ *
+ * Try and get firmware into memory
+ *
+ * Arguments:
+ *	udev	usb device structure
+ *	wlandev wlan device structure
+ *
+ * Returns:
+ *	0	- success
+ *	~0	- failure
+ *----------------------------------------------------------------
+ */
 static int prism2_fwtry(struct usb_device *udev, struct wlandevice *wlandev)
 {
 	const struct firmware *fw_entry = NULL;
 
 	netdev_info(wlandev->netdev, "prism2_usb: Checking for firmware %s\n",
-	       PRISM2_USB_FWFILE);
+		    PRISM2_USB_FWFILE);
 	if (request_ihex_firmware(&fw_entry,
 				  PRISM2_USB_FWFILE, &udev->dev) != 0) {
 		netdev_info(wlandev->netdev,
-		       "prism2_usb: Firmware not available, but not essential\n");
+			    "prism2_usb: Firmware not available, but not essential\n");
 		netdev_info(wlandev->netdev,
-		       "prism2_usb: can continue to use card anyway.\n");
+			    "prism2_usb: can continue to use card anyway.\n");
 		return 1;
 	}
 
@@ -226,18 +229,19 @@ static int prism2_fwtry(struct usb_device *udev, struct wlandevice *wlandev)
 }
 
 /*----------------------------------------------------------------
-* prism2_fwapply
-*
-* Apply the firmware loaded into memory
-*
-* Arguments:
-*	rfptr	firmware image in kernel memory
-*	wlandev device
-*
-* Returns:
-*	0	- success
-*	~0	- failure
-----------------------------------------------------------------*/
+ * prism2_fwapply
+ *
+ * Apply the firmware loaded into memory
+ *
+ * Arguments:
+ *	rfptr	firmware image in kernel memory
+ *	wlandev device
+ *
+ * Returns:
+ *	0	- success
+ *	~0	- failure
+ *----------------------------------------------------------------
+ */
 static int prism2_fwapply(const struct ihex_binrec *rfptr,
 			  struct wlandevice *wlandev)
 {
@@ -248,7 +252,12 @@ static int prism2_fwapply(const struct ihex_binrec *rfptr,
 
 	/* Initialize the data structures */
 	ns3data = 0;
-	memset(s3data, 0, sizeof(s3data));
+	s3data = kcalloc(S3DATA_MAX, sizeof(*s3data), GFP_KERNEL);
+	if (!s3data) {
+		result = -ENOMEM;
+		goto out;
+	}
+
 	ns3plug = 0;
 	memset(s3plug, 0, sizeof(s3plug));
 	ns3crc = 0;
@@ -372,24 +381,25 @@ out:
 }
 
 /*----------------------------------------------------------------
-* crcimage
-*
-* Adds a CRC16 in the two bytes prior to each block identified by
-* an S3 CRC record.  Currently, we don't actually do a CRC we just
-* insert the value 0xC0DE in hfa384x order.
-*
-* Arguments:
-*	fchunk		Array of image chunks
-*	nfchunks	Number of image chunks
-*	s3crc		Array of crc records
-*	ns3crc		Number of crc records
-*
-* Returns:
-*	0	success
-*	~0	failure
-----------------------------------------------------------------*/
+ * crcimage
+ *
+ * Adds a CRC16 in the two bytes prior to each block identified by
+ * an S3 CRC record.  Currently, we don't actually do a CRC we just
+ * insert the value 0xC0DE in hfa384x order.
+ *
+ * Arguments:
+ *	fchunk		Array of image chunks
+ *	nfchunks	Number of image chunks
+ *	s3crc		Array of crc records
+ *	ns3crc		Number of crc records
+ *
+ * Returns:
+ *	0	success
+ *	~0	failure
+ *----------------------------------------------------------------
+ */
 static int crcimage(struct imgchunk *fchunk, unsigned int nfchunks,
-	     struct s3crcrec *s3crc, unsigned int ns3crc)
+		    struct s3crcrec *s3crc, unsigned int ns3crc)
 {
 	int result = 0;
 	int i;
@@ -433,22 +443,22 @@ static int crcimage(struct imgchunk *fchunk, unsigned int nfchunks,
 		dest = fchunk[c].data + chunkoff;
 		*dest = 0xde;
 		*(dest + 1) = 0xc0;
-
 	}
 	return result;
 }
 
 /*----------------------------------------------------------------
-* free_chunks
-*
-* Clears the chunklist data structures in preparation for a new file.
-*
-* Arguments:
-*	none
-*
-* Returns:
-*	nothing
-----------------------------------------------------------------*/
+ * free_chunks
+ *
+ * Clears the chunklist data structures in preparation for a new file.
+ *
+ * Arguments:
+ *	none
+ *
+ * Returns:
+ *	nothing
+ *----------------------------------------------------------------
+ */
 static void free_chunks(struct imgchunk *fchunk, unsigned int *nfchunks)
 {
 	int i;
@@ -458,24 +468,24 @@ static void free_chunks(struct imgchunk *fchunk, unsigned int *nfchunks)
 
 	*nfchunks = 0;
 	memset(fchunk, 0, sizeof(*fchunk));
-
 }
 
 /*----------------------------------------------------------------
-* free_srecs
-*
-* Clears the srec data structures in preparation for a new file.
-*
-* Arguments:
-*	none
-*
-* Returns:
-*	nothing
-----------------------------------------------------------------*/
+ * free_srecs
+ *
+ * Clears the srec data structures in preparation for a new file.
+ *
+ * Arguments:
+ *	none
+ *
+ * Returns:
+ *	nothing
+ *----------------------------------------------------------------
+ */
 static void free_srecs(void)
 {
 	ns3data = 0;
-	memset(s3data, 0, sizeof(s3data));
+	kfree(s3data);
 	ns3plug = 0;
 	memset(s3plug, 0, sizeof(s3plug));
 	ns3crc = 0;
@@ -486,19 +496,20 @@ static void free_srecs(void)
 }
 
 /*----------------------------------------------------------------
-* mkimage
-*
-* Scans the currently loaded set of S records for data residing
-* in contiguous memory regions.  Each contiguous region is then
-* made into a 'chunk'.  This function assumes that we're building
-* a new chunk list.  Assumes the s3data items are in sorted order.
-*
-* Arguments:	none
-*
-* Returns:
-*	0	- success
-*	~0	- failure (probably an errno)
-----------------------------------------------------------------*/
+ * mkimage
+ *
+ * Scans the currently loaded set of S records for data residing
+ * in contiguous memory regions.  Each contiguous region is then
+ * made into a 'chunk'.  This function assumes that we're building
+ * a new chunk list.  Assumes the s3data items are in sorted order.
+ *
+ * Arguments:	none
+ *
+ * Returns:
+ *	0	- success
+ *	~0	- failure (probably an errno)
+ *----------------------------------------------------------------
+ */
 static int mkimage(struct imgchunk *clist, unsigned int *ccnt)
 {
 	int result = 0;
@@ -577,19 +588,20 @@ static int mkimage(struct imgchunk *clist, unsigned int *ccnt)
 }
 
 /*----------------------------------------------------------------
-* mkpdrlist
-*
-* Reads a raw PDA and builds an array of pdrec_t structures.
-*
-* Arguments:
-*	pda	buffer containing raw PDA bytes
-*	pdrec	ptr to an array of pdrec_t's.  Will be filled on exit.
-*	nrec	ptr to a variable that will contain the count of PDRs
-*
-* Returns:
-*	0	- success
-*	~0	- failure (probably an errno)
-----------------------------------------------------------------*/
+ * mkpdrlist
+ *
+ * Reads a raw PDA and builds an array of pdrec_t structures.
+ *
+ * Arguments:
+ *	pda	buffer containing raw PDA bytes
+ *	pdrec	ptr to an array of pdrec_t's.  Will be filled on exit.
+ *	nrec	ptr to a variable that will contain the count of PDRs
+ *
+ * Returns:
+ *	0	- success
+ *	~0	- failure (probably an errno)
+ *----------------------------------------------------------------
+ */
 static int mkpdrlist(struct pda *pda)
 {
 	u16 *pda16 = (u16 *)pda->buf;
@@ -599,7 +611,7 @@ static int mkpdrlist(struct pda *pda)
 	curroff = 0;
 	while (curroff < (HFA384x_PDA_LEN_MAX / 2 - 1) &&
 	       le16_to_cpu(pda16[curroff + 1]) != HFA384x_PDR_END_OF_PDA) {
-		pda->rec[pda->nrec] = (struct hfa384x_pdrec *)&(pda16[curroff]);
+		pda->rec[pda->nrec] = (struct hfa384x_pdrec *)&pda16[curroff];
 
 		if (le16_to_cpu(pda->rec[pda->nrec]->code) ==
 		    HFA384x_PDR_NICID) {
@@ -631,37 +643,38 @@ static int mkpdrlist(struct pda *pda)
 
 		(pda->nrec)++;
 		curroff += le16_to_cpu(pda16[curroff]) + 1;
-
 	}
 	if (curroff >= (HFA384x_PDA_LEN_MAX / 2 - 1)) {
 		pr_err("no end record found or invalid lengths in PDR data, exiting. %x %d\n",
 		       curroff, pda->nrec);
 		return 1;
 	}
-	pda->rec[pda->nrec] = (struct hfa384x_pdrec *)&(pda16[curroff]);
+	pda->rec[pda->nrec] = (struct hfa384x_pdrec *)&pda16[curroff];
 	(pda->nrec)++;
 	return 0;
 }
 
 /*----------------------------------------------------------------
-* plugimage
-*
-* Plugs the given image using the given plug records from the given
-* PDA and filename.
-*
-* Arguments:
-*	fchunk		Array of image chunks
-*	nfchunks	Number of image chunks
-*	s3plug		Array of plug records
-*	ns3plug		Number of plug records
-*	pda		Current pda data
-*
-* Returns:
-*	0	success
-*	~0	failure
-----------------------------------------------------------------*/
+ * plugimage
+ *
+ * Plugs the given image using the given plug records from the given
+ * PDA and filename.
+ *
+ * Arguments:
+ *	fchunk		Array of image chunks
+ *	nfchunks	Number of image chunks
+ *	s3plug		Array of plug records
+ *	ns3plug		Number of plug records
+ *	pda		Current pda data
+ *
+ * Returns:
+ *	0	success
+ *	~0	failure
+ *----------------------------------------------------------------
+ */
 static int plugimage(struct imgchunk *fchunk, unsigned int nfchunks,
-	      struct s3plugrec *s3plug, unsigned int ns3plug, struct pda *pda)
+		     struct s3plugrec *s3plug, unsigned int ns3plug,
+		     struct pda *pda)
 {
 	int result = 0;
 	int i;			/* plug index */
@@ -741,31 +754,31 @@ static int plugimage(struct imgchunk *fchunk, unsigned int nfchunks,
 			memset(dest, 0, s3plug[i].len);
 			strncpy(dest, PRISM2_USB_FWFILE, s3plug[i].len - 1);
 		} else {	/* plug a PDR */
-			memcpy(dest, &(pda->rec[j]->data), s3plug[i].len);
+			memcpy(dest, &pda->rec[j]->data, s3plug[i].len);
 		}
 	}
 	return result;
-
 }
 
 /*----------------------------------------------------------------
-* read_cardpda
-*
-* Sends the command for the driver to read the pda from the card
-* named in the device variable.  Upon success, the card pda is
-* stored in the "cardpda" variables.  Note that the pda structure
-* is considered 'well formed' after this function.  That means
-* that the nrecs is valid, the rec array has been set up, and there's
-* a valid PDAEND record in the raw PDA data.
-*
-* Arguments:
-*	pda		pda structure
-*	wlandev		device
-*
-* Returns:
-*	0	- success
-*	~0	- failure (probably an errno)
-----------------------------------------------------------------*/
+ * read_cardpda
+ *
+ * Sends the command for the driver to read the pda from the card
+ * named in the device variable.  Upon success, the card pda is
+ * stored in the "cardpda" variables.  Note that the pda structure
+ * is considered 'well formed' after this function.  That means
+ * that the nrecs is valid, the rec array has been set up, and there's
+ * a valid PDAEND record in the raw PDA data.
+ *
+ * Arguments:
+ *	pda		pda structure
+ *	wlandev		device
+ *
+ * Returns:
+ *	0	- success
+ *	~0	- failure (probably an errno)
+ *----------------------------------------------------------------
+ */
 static int read_cardpda(struct pda *pda, struct wlandevice *wlandev)
 {
 	int result = 0;
@@ -802,65 +815,66 @@ static int read_cardpda(struct pda *pda, struct wlandevice *wlandev)
 }
 
 /*----------------------------------------------------------------
-* read_fwfile
-*
-* Reads the given fw file which should have been compiled from an srec
-* file. Each record in the fw file will either be a plain data record,
-* a start address record, or other records used for plugging.
-*
-* Note that data records are expected to be sorted into
-* ascending address order in the fw file.
-*
-* Note also that the start address record, originally an S7 record in
-* the srec file, is expected in the fw file to be like a data record but
-* with a certain address to make it identifiable.
-*
-* Here's the SREC format that the fw should have come from:
-* S[37]nnaaaaaaaaddd...dddcc
-*
-*       nn - number of bytes starting with the address field
-* aaaaaaaa - address in readable (or big endian) format
-* dd....dd - 0-245 data bytes (two chars per byte)
-*       cc - checksum
-*
-* The S7 record's (there should be only one) address value gets
-* converted to an S3 record with address of 0xff400000, with the
-* start address being stored as a 4 byte data word. That address is
-* the start execution address used for RAM downloads.
-*
-* The S3 records have a collection of subformats indicated by the
-* value of aaaaaaaa:
-*   0xff000000 - Plug record, data field format:
-*                xxxxxxxxaaaaaaaassssssss
-*                x - PDR code number (little endian)
-*                a - Address in load image to plug (little endian)
-*                s - Length of plug data area (little endian)
-*
-*   0xff100000 - CRC16 generation record, data field format:
-*                aaaaaaaassssssssbbbbbbbb
-*                a - Start address for CRC calculation (little endian)
-*                s - Length of data to  calculate over (little endian)
-*                b - Boolean, true=write crc, false=don't write
-*
-*   0xff200000 - Info record, data field format:
-*                ssssttttdd..dd
-*                s - Size in words (little endian)
-*                t - Info type (little endian), see #defines and
-*                    struct s3inforec for details about types.
-*                d - (s - 1) little endian words giving the contents of
-*                    the given info type.
-*
-*   0xff400000 - Start address record, data field format:
-*                aaaaaaaa
-*                a - Address in load image to plug (little endian)
-*
-* Arguments:
-*	record	firmware image (ihex record structure) in kernel memory
-*
-* Returns:
-*	0	- success
-*	~0	- failure (probably an errno)
-----------------------------------------------------------------*/
+ * read_fwfile
+ *
+ * Reads the given fw file which should have been compiled from an srec
+ * file. Each record in the fw file will either be a plain data record,
+ * a start address record, or other records used for plugging.
+ *
+ * Note that data records are expected to be sorted into
+ * ascending address order in the fw file.
+ *
+ * Note also that the start address record, originally an S7 record in
+ * the srec file, is expected in the fw file to be like a data record but
+ * with a certain address to make it identifiable.
+ *
+ * Here's the SREC format that the fw should have come from:
+ * S[37]nnaaaaaaaaddd...dddcc
+ *
+ *       nn - number of bytes starting with the address field
+ * aaaaaaaa - address in readable (or big endian) format
+ * dd....dd - 0-245 data bytes (two chars per byte)
+ *       cc - checksum
+ *
+ * The S7 record's (there should be only one) address value gets
+ * converted to an S3 record with address of 0xff400000, with the
+ * start address being stored as a 4 byte data word. That address is
+ * the start execution address used for RAM downloads.
+ *
+ * The S3 records have a collection of subformats indicated by the
+ * value of aaaaaaaa:
+ *   0xff000000 - Plug record, data field format:
+ *                xxxxxxxxaaaaaaaassssssss
+ *                x - PDR code number (little endian)
+ *                a - Address in load image to plug (little endian)
+ *                s - Length of plug data area (little endian)
+ *
+ *   0xff100000 - CRC16 generation record, data field format:
+ *                aaaaaaaassssssssbbbbbbbb
+ *                a - Start address for CRC calculation (little endian)
+ *                s - Length of data to  calculate over (little endian)
+ *                b - Boolean, true=write crc, false=don't write
+ *
+ *   0xff200000 - Info record, data field format:
+ *                ssssttttdd..dd
+ *                s - Size in words (little endian)
+ *                t - Info type (little endian), see #defines and
+ *                    struct s3inforec for details about types.
+ *                d - (s - 1) little endian words giving the contents of
+ *                    the given info type.
+ *
+ *   0xff400000 - Start address record, data field format:
+ *                aaaaaaaa
+ *                a - Address in load image to plug (little endian)
+ *
+ * Arguments:
+ *	record	firmware image (ihex record structure) in kernel memory
+ *
+ * Returns:
+ *	0	- success
+ *	~0	- failure (probably an errno)
+ *----------------------------------------------------------------
+ */
 static int read_fwfile(const struct ihex_binrec *record)
 {
 	int		i;
@@ -872,7 +886,6 @@ static int read_fwfile(const struct ihex_binrec *record)
 	pr_debug("Reading fw file ...\n");
 
 	while (record) {
-
 		rcnt++;
 
 		len = be16_to_cpu(record->len);
@@ -887,8 +900,8 @@ static int read_fwfile(const struct ihex_binrec *record)
 		case S3ADDR_START:
 			startaddr = *ptr32;
 			pr_debug("  S7 start addr, record=%d addr=0x%08x\n",
-				      rcnt,
-				      startaddr);
+				 rcnt,
+				 startaddr);
 			break;
 		case S3ADDR_PLUG:
 			s3plug[ns3plug].itemcode = *ptr32;
@@ -896,10 +909,10 @@ static int read_fwfile(const struct ihex_binrec *record)
 			s3plug[ns3plug].len = *(ptr32 + 2);
 
 			pr_debug("  S3 plugrec, record=%d itemcode=0x%08x addr=0x%08x len=%d\n",
-				      rcnt,
-				      s3plug[ns3plug].itemcode,
-				      s3plug[ns3plug].addr,
-				      s3plug[ns3plug].len);
+				 rcnt,
+				 s3plug[ns3plug].itemcode,
+				 s3plug[ns3plug].addr,
+				 s3plug[ns3plug].len);
 
 			ns3plug++;
 			if (ns3plug == S3PLUG_MAX) {
@@ -913,10 +926,10 @@ static int read_fwfile(const struct ihex_binrec *record)
 			s3crc[ns3crc].dowrite = *(ptr32 + 2);
 
 			pr_debug("  S3 crcrec, record=%d addr=0x%08x len=%d write=0x%08x\n",
-				      rcnt,
-				      s3crc[ns3crc].addr,
-				      s3crc[ns3crc].len,
-				      s3crc[ns3crc].dowrite);
+				 rcnt,
+				 s3crc[ns3crc].addr,
+				 s3crc[ns3crc].len,
+				 s3crc[ns3crc].dowrite);
 			ns3crc++;
 			if (ns3crc == S3CRC_MAX) {
 				pr_err("S3 crcrec limit reached - aborting\n");
@@ -928,16 +941,16 @@ static int read_fwfile(const struct ihex_binrec *record)
 			s3info[ns3info].type = *(ptr16 + 1);
 
 			pr_debug("  S3 inforec, record=%d len=0x%04x type=0x%04x\n",
-				      rcnt,
-				      s3info[ns3info].len,
-				      s3info[ns3info].type);
+				 rcnt,
+				 s3info[ns3info].len,
+				 s3info[ns3info].type);
 			if (((s3info[ns3info].len - 1) * sizeof(u16)) >
 			   sizeof(s3info[ns3info].info)) {
 				pr_err("S3 inforec length too long - aborting\n");
 				return 1;
 			}
 
-			tmpinfo = (u16 *)&(s3info[ns3info].info.version);
+			tmpinfo = (u16 *)&s3info[ns3info].info.version;
 			pr_debug("            info=");
 			for (i = 0; i < s3info[ns3info].len - 1; i++) {
 				tmpinfo[i] = *(ptr16 + 2 + i);
@@ -968,22 +981,23 @@ static int read_fwfile(const struct ihex_binrec *record)
 }
 
 /*----------------------------------------------------------------
-* writeimage
-*
-* Takes the chunks, builds p80211 messages and sends them down
-* to the driver for writing to the card.
-*
-* Arguments:
-*	wlandev		device
-*	fchunk		Array of image chunks
-*	nfchunks	Number of image chunks
-*
-* Returns:
-*	0	success
-*	~0	failure
-----------------------------------------------------------------*/
+ * writeimage
+ *
+ * Takes the chunks, builds p80211 messages and sends them down
+ * to the driver for writing to the card.
+ *
+ * Arguments:
+ *	wlandev		device
+ *	fchunk		Array of image chunks
+ *	nfchunks	Number of image chunks
+ *
+ * Returns:
+ *	0	success
+ *	~0	failure
+ *----------------------------------------------------------------
+ */
 static int writeimage(struct wlandevice *wlandev, struct imgchunk *fchunk,
-	       unsigned int nfchunks)
+		      unsigned int nfchunks)
 {
 	int result = 0;
 	struct p80211msg_p2req_ramdl_state *rstmsg;
@@ -1099,7 +1113,6 @@ static int writeimage(struct wlandevice *wlandev, struct imgchunk *fchunk,
 				result = 1;
 				goto free_result;
 			}
-
 		}
 	}
 
diff --git a/drivers/staging/wlan-ng/prism2mgmt.c b/drivers/staging/wlan-ng/prism2mgmt.c
index 170de1c9eac4..c558ad656c49 100644
--- a/drivers/staging/wlan-ng/prism2mgmt.c
+++ b/drivers/staging/wlan-ng/prism2mgmt.c
@@ -1,61 +1,61 @@
 /* src/prism2/driver/prism2mgmt.c
-*
-* Management request handler functions.
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-* The functions in this file handle management requests sent from
-* user mode.
-*
-* Most of these functions have two separate blocks of code that are
-* conditional on whether this is a station or an AP.  This is used
-* to separate out the STA and AP responses to these management primitives.
-* It's a choice (good, bad, indifferent?) to have the code in the same
-* place so it's clear that the same primitive is implemented in both
-* cases but has different behavior.
-*
-* --------------------------------------------------------------------
-*/
+ *
+ * Management request handler functions.
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ * The functions in this file handle management requests sent from
+ * user mode.
+ *
+ * Most of these functions have two separate blocks of code that are
+ * conditional on whether this is a station or an AP.  This is used
+ * to separate out the STA and AP responses to these management primitives.
+ * It's a choice (good, bad, indifferent?) to have the code in the same
+ * place so it's clear that the same primitive is implemented in both
+ * cases but has different behavior.
+ *
+ * --------------------------------------------------------------------
+ */
 
 #include <linux/if_arp.h>
 #include <linux/module.h>
@@ -84,35 +84,36 @@
 #include "prism2mgmt.h"
 
 /* Converts 802.11 format rate specifications to prism2 */
-#define p80211rate_to_p2bit(n)	((((n)&~BIT(7)) == 2) ? BIT(0) :  \
-				 (((n)&~BIT(7)) == 4) ? BIT(1) : \
-				 (((n)&~BIT(7)) == 11) ? BIT(2) : \
-				 (((n)&~BIT(7)) == 22) ? BIT(3) : 0)
+#define p80211rate_to_p2bit(n)	((((n) & ~BIT(7)) == 2) ? BIT(0) :  \
+				 (((n) & ~BIT(7)) == 4) ? BIT(1) : \
+				 (((n) & ~BIT(7)) == 11) ? BIT(2) : \
+				 (((n) & ~BIT(7)) == 22) ? BIT(3) : 0)
 
 /*----------------------------------------------------------------
-* prism2mgmt_scan
-*
-* Initiate a scan for BSSs.
-*
-* This function corresponds to MLME-scan.request and part of
-* MLME-scan.confirm.  As far as I can tell in the standard, there
-* are no restrictions on when a scan.request may be issued.  We have
-* to handle in whatever state the driver/MAC happen to be.
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	msgp		ptr to msg buffer
-*
-* Returns:
-*	0	success and done
-*	<0	success, but we're waiting for something to finish.
-*	>0	an error occurred while handling the message.
-* Side effects:
-*
-* Call context:
-*	process thread  (usually)
-*	interrupt
-----------------------------------------------------------------*/
+ * prism2mgmt_scan
+ *
+ * Initiate a scan for BSSs.
+ *
+ * This function corresponds to MLME-scan.request and part of
+ * MLME-scan.confirm.  As far as I can tell in the standard, there
+ * are no restrictions on when a scan.request may be issued.  We have
+ * to handle in whatever state the driver/MAC happen to be.
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	msgp		ptr to msg buffer
+ *
+ * Returns:
+ *	0	success and done
+ *	<0	success, but we're waiting for something to finish.
+ *	>0	an error occurred while handling the message.
+ * Side effects:
+ *
+ * Call context:
+ *	process thread  (usually)
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 int prism2mgmt_scan(struct wlandevice *wlandev, void *msgp)
 {
 	int result = 0;
@@ -122,7 +123,7 @@ int prism2mgmt_scan(struct wlandevice *wlandev, void *msgp)
 	int i, timeout;
 	int istmpenable = 0;
 
-	struct hfa384x_HostScanRequest_data scanreq;
+	struct hfa384x_host_scan_request_data scanreq;
 
 	/* gatekeeper check */
 	if (HFA384x_FIRMWARE_VERSION(hw->ident_sta_fw.major,
@@ -184,7 +185,7 @@ int prism2mgmt_scan(struct wlandevice *wlandev, void *msgp)
 
 	/* set up the txrate to be 2MBPS. Should be fastest basicrate... */
 	word = HFA384x_RATEBIT_2;
-	scanreq.txRate = cpu_to_le16(word);
+	scanreq.tx_rate = cpu_to_le16(word);
 
 	/* set up the channel list */
 	word = 0;
@@ -196,7 +197,7 @@ int prism2mgmt_scan(struct wlandevice *wlandev, void *msgp)
 		/* channel 1 is BIT 0 ... channel 14 is BIT 13 */
 		word |= (1 << (channel - 1));
 	}
-	scanreq.channelList = cpu_to_le16(word);
+	scanreq.channel_list = cpu_to_le16(word);
 
 	/* set up the ssid, if present. */
 	scanreq.ssid.len = cpu_to_le16(msg->ssid.data.len);
@@ -292,7 +293,7 @@ int prism2mgmt_scan(struct wlandevice *wlandev, void *msgp)
 
 	result = hfa384x_drvr_setconfig(hw,
 					HFA384x_RID_HOSTSCAN, &scanreq,
-					sizeof(struct hfa384x_HostScanRequest_data));
+					sizeof(scanreq));
 	if (result) {
 		netdev_err(wlandev->netdev,
 			   "setconfig(SCANREQUEST) failed. result=%d\n",
@@ -347,31 +348,32 @@ exit:
 }
 
 /*----------------------------------------------------------------
-* prism2mgmt_scan_results
-*
-* Retrieve the BSS description for one of the BSSs identified in
-* a scan.
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	msgp		ptr to msg buffer
-*
-* Returns:
-*	0	success and done
-*	<0	success, but we're waiting for something to finish.
-*	>0	an error occurred while handling the message.
-* Side effects:
-*
-* Call context:
-*	process thread  (usually)
-*	interrupt
-----------------------------------------------------------------*/
+ * prism2mgmt_scan_results
+ *
+ * Retrieve the BSS description for one of the BSSs identified in
+ * a scan.
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	msgp		ptr to msg buffer
+ *
+ * Returns:
+ *	0	success and done
+ *	<0	success, but we're waiting for something to finish.
+ *	>0	an error occurred while handling the message.
+ * Side effects:
+ *
+ * Call context:
+ *	process thread  (usually)
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 int prism2mgmt_scan_results(struct wlandevice *wlandev, void *msgp)
 {
 	int result = 0;
 	struct p80211msg_dot11req_scan_results *req;
 	struct hfa384x *hw = wlandev->priv;
-	struct hfa384x_HScanResultSub *item = NULL;
+	struct hfa384x_hscan_result_sub *item = NULL;
 
 	int count;
 
@@ -425,8 +427,8 @@ int prism2mgmt_scan_results(struct wlandevice *wlandev, void *msgp)
 #define REQBASICRATE(N) \
 	do { \
 		if ((count >= N) && DOT11_RATE5_ISBASIC_GET( \
-			item->supprates[(N)-1])) { \
-			req->basicrate ## N .data = item->supprates[(N)-1]; \
+			item->supprates[(N) - 1])) { \
+			req->basicrate ## N .data = item->supprates[(N) - 1]; \
 			req->basicrate ## N .status = \
 				P80211ENUM_msgitem_status_data_ok; \
 		} \
@@ -444,7 +446,7 @@ int prism2mgmt_scan_results(struct wlandevice *wlandev, void *msgp)
 #define REQSUPPRATE(N) \
 	do { \
 		if (count >= N) { \
-			req->supprate ## N .data = item->supprates[(N)-1]; \
+			req->supprate ## N .data = item->supprates[(N) - 1]; \
 			req->supprate ## N .status = \
 				P80211ENUM_msgitem_status_data_ok; \
 		} \
@@ -507,24 +509,25 @@ exit:
 }
 
 /*----------------------------------------------------------------
-* prism2mgmt_start
-*
-* Start a BSS.  Any station can do this for IBSS, only AP for ESS.
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	msgp		ptr to msg buffer
-*
-* Returns:
-*	0	success and done
-*	<0	success, but we're waiting for something to finish.
-*	>0	an error occurred while handling the message.
-* Side effects:
-*
-* Call context:
-*	process thread  (usually)
-*	interrupt
-----------------------------------------------------------------*/
+ * prism2mgmt_start
+ *
+ * Start a BSS.  Any station can do this for IBSS, only AP for ESS.
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	msgp		ptr to msg buffer
+ *
+ * Returns:
+ *	0	success and done
+ *	<0	success, but we're waiting for something to finish.
+ *	>0	an error occurred while handling the message.
+ * Side effects:
+ *
+ * Call context:
+ *	process thread  (usually)
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 int prism2mgmt_start(struct wlandevice *wlandev, void *msgp)
 {
 	int result = 0;
@@ -580,7 +583,7 @@ int prism2mgmt_start(struct wlandevice *wlandev, void *msgp)
 
 	/* beacon period */
 	word = msg->beaconperiod.data;
-	result = hfa384x_drvr_setconfig16(hw, HFA384x_RID_CNFAPBCNint, word);
+	result = hfa384x_drvr_setconfig16(hw, HFA384x_RID_CNFAPBCNINT, word);
 	if (result) {
 		netdev_err(wlandev->netdev,
 			   "Failed to set beacon period=%d.\n", word);
@@ -689,23 +692,24 @@ done:
 }
 
 /*----------------------------------------------------------------
-* prism2mgmt_readpda
-*
-* Collect the PDA data and put it in the message.
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	msgp		ptr to msg buffer
-*
-* Returns:
-*	0	success and done
-*	<0	success, but we're waiting for something to finish.
-*	>0	an error occurred while handling the message.
-* Side effects:
-*
-* Call context:
-*	process thread  (usually)
-----------------------------------------------------------------*/
+ * prism2mgmt_readpda
+ *
+ * Collect the PDA data and put it in the message.
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	msgp		ptr to msg buffer
+ *
+ * Returns:
+ *	0	success and done
+ *	<0	success, but we're waiting for something to finish.
+ *	>0	an error occurred while handling the message.
+ * Side effects:
+ *
+ * Call context:
+ *	process thread  (usually)
+ *----------------------------------------------------------------
+ */
 int prism2mgmt_readpda(struct wlandevice *wlandev, void *msgp)
 {
 	struct hfa384x *hw = wlandev->priv;
@@ -748,30 +752,31 @@ int prism2mgmt_readpda(struct wlandevice *wlandev, void *msgp)
 }
 
 /*----------------------------------------------------------------
-* prism2mgmt_ramdl_state
-*
-* Establishes the beginning/end of a card RAM download session.
-*
-* It is expected that the ramdl_write() function will be called
-* one or more times between the 'enable' and 'disable' calls to
-* this function.
-*
-* Note: This function should not be called when a mac comm port
-*       is active.
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	msgp		ptr to msg buffer
-*
-* Returns:
-*	0	success and done
-*	<0	success, but we're waiting for something to finish.
-*	>0	an error occurred while handling the message.
-* Side effects:
-*
-* Call context:
-*	process thread  (usually)
-----------------------------------------------------------------*/
+ * prism2mgmt_ramdl_state
+ *
+ * Establishes the beginning/end of a card RAM download session.
+ *
+ * It is expected that the ramdl_write() function will be called
+ * one or more times between the 'enable' and 'disable' calls to
+ * this function.
+ *
+ * Note: This function should not be called when a mac comm port
+ *       is active.
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	msgp		ptr to msg buffer
+ *
+ * Returns:
+ *	0	success and done
+ *	<0	success, but we're waiting for something to finish.
+ *	>0	an error occurred while handling the message.
+ * Side effects:
+ *
+ * Call context:
+ *	process thread  (usually)
+ *----------------------------------------------------------------
+ */
 int prism2mgmt_ramdl_state(struct wlandevice *wlandev, void *msgp)
 {
 	struct hfa384x *hw = wlandev->priv;
@@ -808,25 +813,26 @@ int prism2mgmt_ramdl_state(struct wlandevice *wlandev, void *msgp)
 }
 
 /*----------------------------------------------------------------
-* prism2mgmt_ramdl_write
-*
-* Writes a buffer to the card RAM using the download state.  This
-* is for writing code to card RAM.  To just read or write raw data
-* use the aux functions.
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	msgp		ptr to msg buffer
-*
-* Returns:
-*	0	success and done
-*	<0	success, but we're waiting for something to finish.
-*	>0	an error occurred while handling the message.
-* Side effects:
-*
-* Call context:
-*	process thread  (usually)
-----------------------------------------------------------------*/
+ * prism2mgmt_ramdl_write
+ *
+ * Writes a buffer to the card RAM using the download state.  This
+ * is for writing code to card RAM.  To just read or write raw data
+ * use the aux functions.
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	msgp		ptr to msg buffer
+ *
+ * Returns:
+ *	0	success and done
+ *	<0	success, but we're waiting for something to finish.
+ *	>0	an error occurred while handling the message.
+ * Side effects:
+ *
+ * Call context:
+ *	process thread  (usually)
+ *----------------------------------------------------------------
+ */
 int prism2mgmt_ramdl_write(struct wlandevice *wlandev, void *msgp)
 {
 	struct hfa384x *hw = wlandev->priv;
@@ -864,30 +870,31 @@ int prism2mgmt_ramdl_write(struct wlandevice *wlandev, void *msgp)
 }
 
 /*----------------------------------------------------------------
-* prism2mgmt_flashdl_state
-*
-* Establishes the beginning/end of a card Flash download session.
-*
-* It is expected that the flashdl_write() function will be called
-* one or more times between the 'enable' and 'disable' calls to
-* this function.
-*
-* Note: This function should not be called when a mac comm port
-*       is active.
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	msgp		ptr to msg buffer
-*
-* Returns:
-*	0	success and done
-*	<0	success, but we're waiting for something to finish.
-*	>0	an error occurred while handling the message.
-* Side effects:
-*
-* Call context:
-*	process thread  (usually)
-----------------------------------------------------------------*/
+ * prism2mgmt_flashdl_state
+ *
+ * Establishes the beginning/end of a card Flash download session.
+ *
+ * It is expected that the flashdl_write() function will be called
+ * one or more times between the 'enable' and 'disable' calls to
+ * this function.
+ *
+ * Note: This function should not be called when a mac comm port
+ *       is active.
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	msgp		ptr to msg buffer
+ *
+ * Returns:
+ *	0	success and done
+ *	<0	success, but we're waiting for something to finish.
+ *	>0	an error occurred while handling the message.
+ * Side effects:
+ *
+ * Call context:
+ *	process thread  (usually)
+ *----------------------------------------------------------------
+ */
 int prism2mgmt_flashdl_state(struct wlandevice *wlandev, void *msgp)
 {
 	int result = 0;
@@ -942,23 +949,24 @@ int prism2mgmt_flashdl_state(struct wlandevice *wlandev, void *msgp)
 }
 
 /*----------------------------------------------------------------
-* prism2mgmt_flashdl_write
-*
-*
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	msgp		ptr to msg buffer
-*
-* Returns:
-*	0	success and done
-*	<0	success, but we're waiting for something to finish.
-*	>0	an error occurred while handling the message.
-* Side effects:
-*
-* Call context:
-*	process thread  (usually)
-----------------------------------------------------------------*/
+ * prism2mgmt_flashdl_write
+ *
+ *
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	msgp		ptr to msg buffer
+ *
+ * Returns:
+ *	0	success and done
+ *	<0	success, but we're waiting for something to finish.
+ *	>0	an error occurred while handling the message.
+ * Side effects:
+ *
+ * Call context:
+ *	process thread  (usually)
+ *----------------------------------------------------------------
+ */
 int prism2mgmt_flashdl_write(struct wlandevice *wlandev, void *msgp)
 {
 	struct hfa384x *hw = wlandev->priv;
@@ -1001,24 +1009,25 @@ int prism2mgmt_flashdl_write(struct wlandevice *wlandev, void *msgp)
 }
 
 /*----------------------------------------------------------------
-* prism2mgmt_autojoin
-*
-* Associate with an ESS.
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	msgp		ptr to msg buffer
-*
-* Returns:
-*	0	success and done
-*	<0	success, but we're waiting for something to finish.
-*	>0	an error occurred while handling the message.
-* Side effects:
-*
-* Call context:
-*	process thread  (usually)
-*	interrupt
-----------------------------------------------------------------*/
+ * prism2mgmt_autojoin
+ *
+ * Associate with an ESS.
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	msgp		ptr to msg buffer
+ *
+ * Returns:
+ *	0	success and done
+ *	<0	success, but we're waiting for something to finish.
+ *	>0	an error occurred while handling the message.
+ * Side effects:
+ *
+ * Call context:
+ *	process thread  (usually)
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 int prism2mgmt_autojoin(struct wlandevice *wlandev, void *msgp)
 {
 	struct hfa384x *hw = wlandev->priv;
@@ -1072,24 +1081,25 @@ int prism2mgmt_autojoin(struct wlandevice *wlandev, void *msgp)
 }
 
 /*----------------------------------------------------------------
-* prism2mgmt_wlansniff
-*
-* Start or stop sniffing.
-*
-* Arguments:
-*	wlandev		wlan device structure
-*	msgp		ptr to msg buffer
-*
-* Returns:
-*	0	success and done
-*	<0	success, but we're waiting for something to finish.
-*	>0	an error occurred while handling the message.
-* Side effects:
-*
-* Call context:
-*	process thread  (usually)
-*	interrupt
-----------------------------------------------------------------*/
+ * prism2mgmt_wlansniff
+ *
+ * Start or stop sniffing.
+ *
+ * Arguments:
+ *	wlandev		wlan device structure
+ *	msgp		ptr to msg buffer
+ *
+ * Returns:
+ *	0	success and done
+ *	<0	success, but we're waiting for something to finish.
+ *	>0	an error occurred while handling the message.
+ * Side effects:
+ *
+ * Call context:
+ *	process thread  (usually)
+ *	interrupt
+ *----------------------------------------------------------------
+ */
 int prism2mgmt_wlansniff(struct wlandevice *wlandev, void *msgp)
 {
 	int result = 0;
diff --git a/drivers/staging/wlan-ng/prism2mgmt.h b/drivers/staging/wlan-ng/prism2mgmt.h
index cc1ac7a60dfe..88b979ff68b3 100644
--- a/drivers/staging/wlan-ng/prism2mgmt.h
+++ b/drivers/staging/wlan-ng/prism2mgmt.h
@@ -1,61 +1,61 @@
 /* prism2mgmt.h
-*
-* Declares the mgmt command handler functions
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-* This file contains the constants and data structures for interaction
-* with the hfa384x Wireless LAN (WLAN) Media Access Controller (MAC).
-* The hfa384x is a portion of the Harris PRISM(tm) WLAN chipset.
-*
-* [Implementation and usage notes]
-*
-* [References]
-*   CW10 Programmer's Manual v1.5
-*   IEEE 802.11 D10.0
-*
-*    --------------------------------------------------------------------
-*/
+ *
+ * Declares the mgmt command handler functions
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ * This file contains the constants and data structures for interaction
+ * with the hfa384x Wireless LAN (WLAN) Media Access Controller (MAC).
+ * The hfa384x is a portion of the Harris PRISM(tm) WLAN chipset.
+ *
+ * [Implementation and usage notes]
+ *
+ * [References]
+ *   CW10 Programmer's Manual v1.5
+ *   IEEE 802.11 D10.0
+ *
+ *    --------------------------------------------------------------------
+ */
 
 #ifndef _PRISM2MGMT_H
 #define _PRISM2MGMT_H
@@ -65,7 +65,8 @@ extern int prism2_reset_settletime;
 
 u32 prism2sta_ifstate(struct wlandevice *wlandev, u32 ifstate);
 
-void prism2sta_ev_info(struct wlandevice *wlandev, struct hfa384x_InfFrame *inf);
+void prism2sta_ev_info(struct wlandevice *wlandev,
+		       struct hfa384x_inf_frame *inf);
 void prism2sta_ev_txexc(struct wlandevice *wlandev, u16 status);
 void prism2sta_ev_tx(struct wlandevice *wlandev, u16 status);
 void prism2sta_ev_alloc(struct wlandevice *wlandev);
@@ -83,9 +84,11 @@ int prism2mgmt_flashdl_write(struct wlandevice *wlandev, void *msgp);
 int prism2mgmt_autojoin(struct wlandevice *wlandev, void *msgp);
 
 /*---------------------------------------------------------------
-* conversion functions going between wlan message data types and
-* Prism2 data types
----------------------------------------------------------------*/
+ * conversion functions going between wlan message data types and
+ * Prism2 data types
+ *---------------------------------------------------------------
+ */
+
 /* byte area conversion functions*/
 void prism2mgmt_bytearea2pstr(u8 *bytearea, struct p80211pstrd *pstr, int len);
 
diff --git a/drivers/staging/wlan-ng/prism2mib.c b/drivers/staging/wlan-ng/prism2mib.c
index 63ab6bc88654..8ea6a647d037 100644
--- a/drivers/staging/wlan-ng/prism2mib.c
+++ b/drivers/staging/wlan-ng/prism2mib.c
@@ -1,54 +1,54 @@
 /* src/prism2/driver/prism2mib.c
-*
-* Management request for mibset/mibget
-*
-* Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
-* --------------------------------------------------------------------
-*
-* linux-wlan
-*
-*   The contents of this file are subject to the Mozilla Public
-*   License Version 1.1 (the "License"); you may not use this file
-*   except in compliance with the License. You may obtain a copy of
-*   the License at http://www.mozilla.org/MPL/
-*
-*   Software distributed under the License is distributed on an "AS
-*   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
-*   implied. See the License for the specific language governing
-*   rights and limitations under the License.
-*
-*   Alternatively, the contents of this file may be used under the
-*   terms of the GNU Public License version 2 (the "GPL"), in which
-*   case the provisions of the GPL are applicable instead of the
-*   above.  If you wish to allow the use of your version of this file
-*   only under the terms of the GPL and not to allow others to use
-*   your version of this file under the MPL, indicate your decision
-*   by deleting the provisions above and replace them with the notice
-*   and other provisions required by the GPL.  If you do not delete
-*   the provisions above, a recipient may use your version of this
-*   file under either the MPL or the GPL.
-*
-* --------------------------------------------------------------------
-*
-* Inquiries regarding the linux-wlan Open Source project can be
-* made directly to:
-*
-* AbsoluteValue Systems Inc.
-* info@linux-wlan.com
-* http://www.linux-wlan.com
-*
-* --------------------------------------------------------------------
-*
-* Portions of the development of this software were funded by
-* Intersil Corporation as part of PRISM(R) chipset product development.
-*
-* --------------------------------------------------------------------
-*
-* The functions in this file handle the mibset/mibget management
-* functions.
-*
-* --------------------------------------------------------------------
-*/
+ *
+ * Management request for mibset/mibget
+ *
+ * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
+ * --------------------------------------------------------------------
+ *
+ * linux-wlan
+ *
+ *   The contents of this file are subject to the Mozilla Public
+ *   License Version 1.1 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.mozilla.org/MPL/
+ *
+ *   Software distributed under the License is distributed on an "AS
+ *   IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
+ *   implied. See the License for the specific language governing
+ *   rights and limitations under the License.
+ *
+ *   Alternatively, the contents of this file may be used under the
+ *   terms of the GNU Public License version 2 (the "GPL"), in which
+ *   case the provisions of the GPL are applicable instead of the
+ *   above.  If you wish to allow the use of your version of this file
+ *   only under the terms of the GPL and not to allow others to use
+ *   your version of this file under the MPL, indicate your decision
+ *   by deleting the provisions above and replace them with the notice
+ *   and other provisions required by the GPL.  If you do not delete
+ *   the provisions above, a recipient may use your version of this
+ *   file under either the MPL or the GPL.
+ *
+ * --------------------------------------------------------------------
+ *
+ * Inquiries regarding the linux-wlan Open Source project can be
+ * made directly to:
+ *
+ * AbsoluteValue Systems Inc.
+ * info@linux-wlan.com
+ * http://www.linux-wlan.com
+ *
+ * --------------------------------------------------------------------
+ *
+ * Portions of the development of this software were funded by
+ * Intersil Corporation as part of PRISM(R) chipset product development.
+ *
+ * --------------------------------------------------------------------
+ *
+ * The functions in this file handle the mibset/mibget management
+ * functions.
+ *
+ * --------------------------------------------------------------------
+ */
 
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -709,7 +709,7 @@ static int prism2mib_priv(struct mibrec *mib,
 
 	switch (mib->did) {
 	case DIDmib_lnx_lnxConfigTable_lnxRSNAIE:{
-			struct hfa384x_WPAData wpa;
+			struct hfa384x_wpa_data wpa;
 
 			if (isget) {
 				hfa384x_drvr_getconfig(hw,
diff --git a/drivers/staging/wlan-ng/prism2sta.c b/drivers/staging/wlan-ng/prism2sta.c
index e1b4a94292ff..984804b92e05 100644
--- a/drivers/staging/wlan-ng/prism2sta.c
+++ b/drivers/staging/wlan-ng/prism2sta.c
@@ -104,32 +104,33 @@ static void prism2sta_reset(struct wlandevice *wlandev);
 static int prism2sta_txframe(struct wlandevice *wlandev, struct sk_buff *skb,
 			     union p80211_hdr *p80211_hdr,
 			     struct p80211_metawep *p80211_wep);
-static int prism2sta_mlmerequest(struct wlandevice *wlandev, struct p80211msg *msg);
+static int prism2sta_mlmerequest(struct wlandevice *wlandev,
+				 struct p80211msg *msg);
 static int prism2sta_getcardinfo(struct wlandevice *wlandev);
 static int prism2sta_globalsetup(struct wlandevice *wlandev);
 static int prism2sta_setmulticast(struct wlandevice *wlandev,
 				  struct net_device *dev);
 
 static void prism2sta_inf_handover(struct wlandevice *wlandev,
-				   struct hfa384x_InfFrame *inf);
+				   struct hfa384x_inf_frame *inf);
 static void prism2sta_inf_tallies(struct wlandevice *wlandev,
-				  struct hfa384x_InfFrame *inf);
+				  struct hfa384x_inf_frame *inf);
 static void prism2sta_inf_hostscanresults(struct wlandevice *wlandev,
-					  struct hfa384x_InfFrame *inf);
+					  struct hfa384x_inf_frame *inf);
 static void prism2sta_inf_scanresults(struct wlandevice *wlandev,
-				      struct hfa384x_InfFrame *inf);
+				      struct hfa384x_inf_frame *inf);
 static void prism2sta_inf_chinforesults(struct wlandevice *wlandev,
-					struct hfa384x_InfFrame *inf);
+					struct hfa384x_inf_frame *inf);
 static void prism2sta_inf_linkstatus(struct wlandevice *wlandev,
-				     struct hfa384x_InfFrame *inf);
+				     struct hfa384x_inf_frame *inf);
 static void prism2sta_inf_assocstatus(struct wlandevice *wlandev,
-				      struct hfa384x_InfFrame *inf);
+				      struct hfa384x_inf_frame *inf);
 static void prism2sta_inf_authreq(struct wlandevice *wlandev,
-				  struct hfa384x_InfFrame *inf);
+				  struct hfa384x_inf_frame *inf);
 static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
-					struct hfa384x_InfFrame *inf);
+					struct hfa384x_inf_frame *inf);
 static void prism2sta_inf_psusercnt(struct wlandevice *wlandev,
-				    struct hfa384x_InfFrame *inf);
+				    struct hfa384x_inf_frame *inf);
 
 /*
  * prism2sta_open
@@ -278,7 +279,8 @@ static int prism2sta_txframe(struct wlandevice *wlandev, struct sk_buff *skb,
  * Call context:
  *	process thread
  */
-static int prism2sta_mlmerequest(struct wlandevice *wlandev, struct p80211msg *msg)
+static int prism2sta_mlmerequest(struct wlandevice *wlandev,
+				 struct p80211msg *msg)
 {
 	struct hfa384x *hw = wlandev->priv;
 
@@ -370,9 +372,10 @@ static int prism2sta_mlmerequest(struct wlandevice *wlandev, struct p80211msg *m
 			qualmsg->noise.status =
 			    P80211ENUM_msgitem_status_data_ok;
 
-			qualmsg->link.data = le16_to_cpu(hw->qual.CQ_currBSS);
-			qualmsg->level.data = le16_to_cpu(hw->qual.ASL_currBSS);
-			qualmsg->noise.data = le16_to_cpu(hw->qual.ANL_currFC);
+			qualmsg->link.data = le16_to_cpu(hw->qual.cq_curr_bss);
+			qualmsg->level.data =
+				le16_to_cpu(hw->qual.asl_curr_bss);
+			qualmsg->noise.data = le16_to_cpu(hw->qual.anl_curr_fc);
 			qualmsg->txrate.data = hw->txrate;
 
 			break;
@@ -606,8 +609,8 @@ static int prism2sta_getcardinfo(struct wlandevice *wlandev)
 	hw->ident_nic.minor = le16_to_cpu(hw->ident_nic.minor);
 
 	netdev_info(wlandev->netdev, "ident: nic h/w: id=0x%02x %d.%d.%d\n",
-	       hw->ident_nic.id, hw->ident_nic.major,
-	       hw->ident_nic.minor, hw->ident_nic.variant);
+		    hw->ident_nic.id, hw->ident_nic.major,
+		    hw->ident_nic.minor, hw->ident_nic.variant);
 
 	/* Primary f/w identity */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_PRIIDENTITY,
@@ -625,8 +628,8 @@ static int prism2sta_getcardinfo(struct wlandevice *wlandev)
 	hw->ident_pri_fw.minor = le16_to_cpu(hw->ident_pri_fw.minor);
 
 	netdev_info(wlandev->netdev, "ident: pri f/w: id=0x%02x %d.%d.%d\n",
-	       hw->ident_pri_fw.id, hw->ident_pri_fw.major,
-	       hw->ident_pri_fw.minor, hw->ident_pri_fw.variant);
+		    hw->ident_pri_fw.id, hw->ident_pri_fw.major,
+		    hw->ident_pri_fw.minor, hw->ident_pri_fw.variant);
 
 	/* Station (Secondary?) f/w identity */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_STAIDENTITY,
@@ -639,7 +642,7 @@ static int prism2sta_getcardinfo(struct wlandevice *wlandev)
 
 	if (hw->ident_nic.id < 0x8000) {
 		netdev_err(wlandev->netdev,
-		       "FATAL: Card is not an Intersil Prism2/2.5/3\n");
+			   "FATAL: Card is not an Intersil Prism2/2.5/3\n");
 		result = -1;
 		goto failed;
 	}
@@ -651,19 +654,19 @@ static int prism2sta_getcardinfo(struct wlandevice *wlandev)
 	hw->ident_sta_fw.minor = le16_to_cpu(hw->ident_sta_fw.minor);
 
 	/* strip out the 'special' variant bits */
-	hw->mm_mods = hw->ident_sta_fw.variant & (BIT(14) | BIT(15));
-	hw->ident_sta_fw.variant &= ~((u16)(BIT(14) | BIT(15)));
+	hw->mm_mods = hw->ident_sta_fw.variant & GENMASK(15, 14);
+	hw->ident_sta_fw.variant &= ~((u16)GENMASK(15, 14));
 
 	if (hw->ident_sta_fw.id == 0x1f) {
 		netdev_info(wlandev->netdev,
-		       "ident: sta f/w: id=0x%02x %d.%d.%d\n",
-		       hw->ident_sta_fw.id, hw->ident_sta_fw.major,
-		       hw->ident_sta_fw.minor, hw->ident_sta_fw.variant);
+			    "ident: sta f/w: id=0x%02x %d.%d.%d\n",
+			    hw->ident_sta_fw.id, hw->ident_sta_fw.major,
+			    hw->ident_sta_fw.minor, hw->ident_sta_fw.variant);
 	} else {
 		netdev_info(wlandev->netdev,
-		       "ident:  ap f/w: id=0x%02x %d.%d.%d\n",
-		       hw->ident_sta_fw.id, hw->ident_sta_fw.major,
-		       hw->ident_sta_fw.minor, hw->ident_sta_fw.variant);
+			    "ident:  ap f/w: id=0x%02x %d.%d.%d\n",
+			    hw->ident_sta_fw.id, hw->ident_sta_fw.major,
+			    hw->ident_sta_fw.minor, hw->ident_sta_fw.variant);
 		netdev_err(wlandev->netdev, "Unsupported Tertiary AP firmware loaded!\n");
 		goto failed;
 	}
@@ -687,10 +690,10 @@ static int prism2sta_getcardinfo(struct wlandevice *wlandev)
 	hw->cap_sup_mfi.top = le16_to_cpu(hw->cap_sup_mfi.top);
 
 	netdev_info(wlandev->netdev,
-	       "MFI:SUP:role=0x%02x:id=0x%02x:var=0x%02x:b/t=%d/%d\n",
-	       hw->cap_sup_mfi.role, hw->cap_sup_mfi.id,
-	       hw->cap_sup_mfi.variant, hw->cap_sup_mfi.bottom,
-	       hw->cap_sup_mfi.top);
+		    "MFI:SUP:role=0x%02x:id=0x%02x:var=0x%02x:b/t=%d/%d\n",
+		    hw->cap_sup_mfi.role, hw->cap_sup_mfi.id,
+		    hw->cap_sup_mfi.variant, hw->cap_sup_mfi.bottom,
+		    hw->cap_sup_mfi.top);
 
 	/* Compatibility range, Controller supplier */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_CFISUPRANGE,
@@ -711,10 +714,10 @@ static int prism2sta_getcardinfo(struct wlandevice *wlandev)
 	hw->cap_sup_cfi.top = le16_to_cpu(hw->cap_sup_cfi.top);
 
 	netdev_info(wlandev->netdev,
-	       "CFI:SUP:role=0x%02x:id=0x%02x:var=0x%02x:b/t=%d/%d\n",
-	       hw->cap_sup_cfi.role, hw->cap_sup_cfi.id,
-	       hw->cap_sup_cfi.variant, hw->cap_sup_cfi.bottom,
-	       hw->cap_sup_cfi.top);
+		    "CFI:SUP:role=0x%02x:id=0x%02x:var=0x%02x:b/t=%d/%d\n",
+		    hw->cap_sup_cfi.role, hw->cap_sup_cfi.id,
+		    hw->cap_sup_cfi.variant, hw->cap_sup_cfi.bottom,
+		    hw->cap_sup_cfi.top);
 
 	/* Compatibility range, Primary f/w supplier */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_PRISUPRANGE,
@@ -735,10 +738,10 @@ static int prism2sta_getcardinfo(struct wlandevice *wlandev)
 	hw->cap_sup_pri.top = le16_to_cpu(hw->cap_sup_pri.top);
 
 	netdev_info(wlandev->netdev,
-	       "PRI:SUP:role=0x%02x:id=0x%02x:var=0x%02x:b/t=%d/%d\n",
-	       hw->cap_sup_pri.role, hw->cap_sup_pri.id,
-	       hw->cap_sup_pri.variant, hw->cap_sup_pri.bottom,
-	       hw->cap_sup_pri.top);
+		    "PRI:SUP:role=0x%02x:id=0x%02x:var=0x%02x:b/t=%d/%d\n",
+		    hw->cap_sup_pri.role, hw->cap_sup_pri.id,
+		    hw->cap_sup_pri.variant, hw->cap_sup_pri.bottom,
+		    hw->cap_sup_pri.top);
 
 	/* Compatibility range, Station f/w supplier */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_STASUPRANGE,
@@ -791,10 +794,10 @@ static int prism2sta_getcardinfo(struct wlandevice *wlandev)
 	hw->cap_act_pri_cfi.top = le16_to_cpu(hw->cap_act_pri_cfi.top);
 
 	netdev_info(wlandev->netdev,
-	       "PRI-CFI:ACT:role=0x%02x:id=0x%02x:var=0x%02x:b/t=%d/%d\n",
-	       hw->cap_act_pri_cfi.role, hw->cap_act_pri_cfi.id,
-	       hw->cap_act_pri_cfi.variant, hw->cap_act_pri_cfi.bottom,
-	       hw->cap_act_pri_cfi.top);
+		    "PRI-CFI:ACT:role=0x%02x:id=0x%02x:var=0x%02x:b/t=%d/%d\n",
+		    hw->cap_act_pri_cfi.role, hw->cap_act_pri_cfi.id,
+		    hw->cap_act_pri_cfi.variant, hw->cap_act_pri_cfi.bottom,
+		    hw->cap_act_pri_cfi.top);
 
 	/* Compatibility range, sta f/w actor, CFI supplier */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_STA_CFIACTRANGES,
@@ -815,10 +818,10 @@ static int prism2sta_getcardinfo(struct wlandevice *wlandev)
 	hw->cap_act_sta_cfi.top = le16_to_cpu(hw->cap_act_sta_cfi.top);
 
 	netdev_info(wlandev->netdev,
-	       "STA-CFI:ACT:role=0x%02x:id=0x%02x:var=0x%02x:b/t=%d/%d\n",
-	       hw->cap_act_sta_cfi.role, hw->cap_act_sta_cfi.id,
-	       hw->cap_act_sta_cfi.variant, hw->cap_act_sta_cfi.bottom,
-	       hw->cap_act_sta_cfi.top);
+		    "STA-CFI:ACT:role=0x%02x:id=0x%02x:var=0x%02x:b/t=%d/%d\n",
+		    hw->cap_act_sta_cfi.role, hw->cap_act_sta_cfi.id,
+		    hw->cap_act_sta_cfi.variant, hw->cap_act_sta_cfi.bottom,
+		    hw->cap_act_sta_cfi.top);
 
 	/* Compatibility range, sta f/w actor, MFI supplier */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_STA_MFIACTRANGES,
@@ -839,10 +842,10 @@ static int prism2sta_getcardinfo(struct wlandevice *wlandev)
 	hw->cap_act_sta_mfi.top = le16_to_cpu(hw->cap_act_sta_mfi.top);
 
 	netdev_info(wlandev->netdev,
-	       "STA-MFI:ACT:role=0x%02x:id=0x%02x:var=0x%02x:b/t=%d/%d\n",
-	       hw->cap_act_sta_mfi.role, hw->cap_act_sta_mfi.id,
-	       hw->cap_act_sta_mfi.variant, hw->cap_act_sta_mfi.bottom,
-	       hw->cap_act_sta_mfi.top);
+		    "STA-MFI:ACT:role=0x%02x:id=0x%02x:var=0x%02x:b/t=%d/%d\n",
+		    hw->cap_act_sta_mfi.role, hw->cap_act_sta_mfi.id,
+		    hw->cap_act_sta_mfi.variant, hw->cap_act_sta_mfi.bottom,
+		    hw->cap_act_sta_mfi.top);
 
 	/* Serial Number */
 	result = hfa384x_drvr_getconfig(hw, HFA384x_RID_NICSERIALNUMBER,
@@ -920,7 +923,7 @@ static int prism2sta_globalsetup(struct wlandevice *wlandev)
 }
 
 static int prism2sta_setmulticast(struct wlandevice *wlandev,
-					struct net_device *dev)
+				  struct net_device *dev)
 {
 	int result = 0;
 	struct hfa384x *hw = wlandev->priv;
@@ -962,7 +965,7 @@ exit:
  *	interrupt
  */
 static void prism2sta_inf_handover(struct wlandevice *wlandev,
-				   struct hfa384x_InfFrame *inf)
+				   struct hfa384x_inf_frame *inf)
 {
 	pr_debug("received infoframe:HANDOVER (unhandled)\n");
 }
@@ -985,7 +988,7 @@ static void prism2sta_inf_handover(struct wlandevice *wlandev,
  *	interrupt
  */
 static void prism2sta_inf_tallies(struct wlandevice *wlandev,
-				  struct hfa384x_InfFrame *inf)
+				  struct hfa384x_inf_frame *inf)
 {
 	struct hfa384x *hw = wlandev->priv;
 	u16 *src16;
@@ -999,7 +1002,7 @@ static void prism2sta_inf_tallies(struct wlandevice *wlandev,
 	 * record length of the info record.
 	 */
 
-	cnt = sizeof(struct hfa384x_CommTallies32) / sizeof(u32);
+	cnt = sizeof(struct hfa384x_comm_tallies_32) / sizeof(u32);
 	if (inf->framelen > 22) {
 		dst = (u32 *)&hw->tallies;
 		src32 = (u32 *)&inf->info.commtallies32;
@@ -1031,19 +1034,19 @@ static void prism2sta_inf_tallies(struct wlandevice *wlandev,
  *	interrupt
  */
 static void prism2sta_inf_scanresults(struct wlandevice *wlandev,
-				      struct hfa384x_InfFrame *inf)
+				      struct hfa384x_inf_frame *inf)
 {
 	struct hfa384x *hw = wlandev->priv;
 	int nbss;
-	struct hfa384x_ScanResult *sr = &(inf->info.scanresult);
+	struct hfa384x_scan_result *sr = &inf->info.scanresult;
 	int i;
-	struct hfa384x_JoinRequest_data joinreq;
+	struct hfa384x_join_request_data joinreq;
 	int result;
 
 	/* Get the number of results, first in bytes, then in results */
 	nbss = (inf->framelen * sizeof(u16)) -
 	    sizeof(inf->infotype) - sizeof(inf->info.scanresult.scanreason);
-	nbss /= sizeof(struct hfa384x_ScanResultSub);
+	nbss /= sizeof(struct hfa384x_scan_result_sub);
 
 	/* Print em */
 	pr_debug("rx scanresults, reason=%d, nbss=%d:\n",
@@ -1064,7 +1067,7 @@ static void prism2sta_inf_scanresults(struct wlandevice *wlandev,
 					&joinreq, HFA384x_RID_JOINREQUEST_LEN);
 	if (result) {
 		netdev_err(wlandev->netdev, "setconfig(joinreq) failed, result=%d\n",
-		       result);
+			   result);
 	}
 }
 
@@ -1086,7 +1089,7 @@ static void prism2sta_inf_scanresults(struct wlandevice *wlandev,
  *	interrupt
  */
 static void prism2sta_inf_hostscanresults(struct wlandevice *wlandev,
-					  struct hfa384x_InfFrame *inf)
+					  struct hfa384x_inf_frame *inf)
 {
 	struct hfa384x *hw = wlandev->priv;
 	int nbss;
@@ -1099,7 +1102,7 @@ static void prism2sta_inf_hostscanresults(struct wlandevice *wlandev,
 
 	kfree(hw->scanresults);
 
-	hw->scanresults = kmemdup(inf, sizeof(struct hfa384x_InfFrame), GFP_ATOMIC);
+	hw->scanresults = kmemdup(inf, sizeof(*inf), GFP_ATOMIC);
 
 	if (nbss == 0)
 		nbss = -1;
@@ -1127,7 +1130,7 @@ static void prism2sta_inf_hostscanresults(struct wlandevice *wlandev,
  *	interrupt
  */
 static void prism2sta_inf_chinforesults(struct wlandevice *wlandev,
-					struct hfa384x_InfFrame *inf)
+					struct hfa384x_inf_frame *inf)
 {
 	struct hfa384x *hw = wlandev->priv;
 	unsigned int i, n;
@@ -1136,8 +1139,8 @@ static void prism2sta_inf_chinforesults(struct wlandevice *wlandev,
 	    le16_to_cpu(inf->info.chinforesult.scanchannels);
 
 	for (i = 0, n = 0; i < HFA384x_CHINFORESULT_MAX; i++) {
-		struct hfa384x_ChInfoResultSub *result;
-		struct hfa384x_ChInfoResultSub *chinforesult;
+		struct hfa384x_ch_info_result_sub *result;
+		struct hfa384x_ch_info_result_sub *chinforesult;
 		int chan;
 
 		if (!(hw->channel_info.results.scanchannels & (1 << i)))
@@ -1179,10 +1182,10 @@ void prism2sta_processing_defer(struct work_struct *data)
 	/* First let's process the auth frames */
 	{
 		struct sk_buff *skb;
-		struct hfa384x_InfFrame *inf;
+		struct hfa384x_inf_frame *inf;
 
 		while ((skb = skb_dequeue(&hw->authq))) {
-			inf = (struct hfa384x_InfFrame *)skb->data;
+			inf = (struct hfa384x_inf_frame *)skb->data;
 			prism2sta_inf_authreq_defer(wlandev, inf);
 		}
 
@@ -1294,7 +1297,7 @@ void prism2sta_processing_defer(struct work_struct *data)
 		 */
 		if (wlandev->netdev->type == ARPHRD_ETHER)
 			netdev_info(wlandev->netdev,
-			       "linkstatus=DISCONNECTED (unhandled)\n");
+				    "linkstatus=DISCONNECTED (unhandled)\n");
 		wlandev->macmode = WLAN_MACMODE_NONE;
 
 		netif_carrier_off(wlandev->netdev);
@@ -1391,7 +1394,7 @@ void prism2sta_processing_defer(struct work_struct *data)
 		 * Disable Transmits, Ignore receives of data frames
 		 */
 		if (hw->join_ap && --hw->join_retries > 0) {
-			struct hfa384x_JoinRequest_data joinreq;
+			struct hfa384x_join_request_data joinreq;
 
 			joinreq = hw->joinreq;
 			/* Send the join request */
@@ -1415,7 +1418,7 @@ void prism2sta_processing_defer(struct work_struct *data)
 	default:
 		/* This is bad, IO port problems? */
 		netdev_warn(wlandev->netdev,
-		       "unknown linkstatus=0x%02x\n", hw->link_status);
+			    "unknown linkstatus=0x%02x\n", hw->link_status);
 		return;
 	}
 
@@ -1440,7 +1443,7 @@ void prism2sta_processing_defer(struct work_struct *data)
  *	interrupt
  */
 static void prism2sta_inf_linkstatus(struct wlandevice *wlandev,
-				     struct hfa384x_InfFrame *inf)
+				     struct hfa384x_inf_frame *inf)
 {
 	struct hfa384x *hw = wlandev->priv;
 
@@ -1468,10 +1471,10 @@ static void prism2sta_inf_linkstatus(struct wlandevice *wlandev,
  *	interrupt
  */
 static void prism2sta_inf_assocstatus(struct wlandevice *wlandev,
-				      struct hfa384x_InfFrame *inf)
+				      struct hfa384x_inf_frame *inf)
 {
 	struct hfa384x *hw = wlandev->priv;
-	struct hfa384x_AssocStatus rec;
+	struct hfa384x_assoc_status rec;
 	int i;
 
 	memcpy(&rec, &inf->info.assocstatus, sizeof(rec));
@@ -1529,7 +1532,7 @@ static void prism2sta_inf_assocstatus(struct wlandevice *wlandev,
  *
  */
 static void prism2sta_inf_authreq(struct wlandevice *wlandev,
-				  struct hfa384x_InfFrame *inf)
+				  struct hfa384x_inf_frame *inf)
 {
 	struct hfa384x *hw = wlandev->priv;
 	struct sk_buff *skb;
@@ -1544,10 +1547,10 @@ static void prism2sta_inf_authreq(struct wlandevice *wlandev,
 }
 
 static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
-					struct hfa384x_InfFrame *inf)
+					struct hfa384x_inf_frame *inf)
 {
 	struct hfa384x *hw = wlandev->priv;
-	struct hfa384x_authenticateStation_data rec;
+	struct hfa384x_authenticate_station_data rec;
 
 	int i, added, result, cnt;
 	u8 *addr;
@@ -1718,7 +1721,7 @@ static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
  *	interrupt
  */
 static void prism2sta_inf_psusercnt(struct wlandevice *wlandev,
-				    struct hfa384x_InfFrame *inf)
+				    struct hfa384x_inf_frame *inf)
 {
 	struct hfa384x *hw = wlandev->priv;
 
@@ -1742,7 +1745,8 @@ static void prism2sta_inf_psusercnt(struct wlandevice *wlandev,
  * Call context:
  *	interrupt
  */
-void prism2sta_ev_info(struct wlandevice *wlandev, struct hfa384x_InfFrame *inf)
+void prism2sta_ev_info(struct wlandevice *wlandev,
+		       struct hfa384x_inf_frame *inf)
 {
 	inf->infotype = le16_to_cpu(inf->infotype);
 	/* Dispatch */
@@ -1785,7 +1789,7 @@ void prism2sta_ev_info(struct wlandevice *wlandev, struct hfa384x_InfFrame *inf)
 		break;
 	default:
 		netdev_warn(wlandev->netdev,
-		       "Unknown info type=0x%02x\n", inf->infotype);
+			    "Unknown info type=0x%02x\n", inf->infotype);
 		break;
 	}
 }
@@ -1859,32 +1863,32 @@ void prism2sta_ev_alloc(struct wlandevice *wlandev)
 }
 
 /*
-* create_wlan
-*
-* Called at module init time.  This creates the struct wlandevice structure
-* and initializes it with relevant bits.
-*
-* Arguments:
-*	none
-*
-* Returns:
-*	the created struct wlandevice structure.
-*
-* Side effects:
-*	also allocates the priv/hw structures.
-*
-* Call context:
-*	process thread
-*
-*/
+ * create_wlan
+ *
+ * Called at module init time.  This creates the struct wlandevice structure
+ * and initializes it with relevant bits.
+ *
+ * Arguments:
+ *	none
+ *
+ * Returns:
+ *	the created struct wlandevice structure.
+ *
+ * Side effects:
+ *	also allocates the priv/hw structures.
+ *
+ * Call context:
+ *	process thread
+ *
+ */
 static struct wlandevice *create_wlan(void)
 {
 	struct wlandevice *wlandev = NULL;
 	struct hfa384x *hw = NULL;
 
 	/* Alloc our structures */
-	wlandev = kzalloc(sizeof(struct wlandevice), GFP_KERNEL);
-	hw = kzalloc(sizeof(struct hfa384x), GFP_KERNEL);
+	wlandev = kzalloc(sizeof(*wlandev), GFP_KERNEL);
+	hw = kzalloc(sizeof(*hw), GFP_KERNEL);
 
 	if (!wlandev || !hw) {
 		kfree(wlandev);
@@ -1943,9 +1947,9 @@ void prism2sta_commsqual_defer(struct work_struct *data)
 		}
 
 		pr_debug("commsqual %d %d %d\n",
-			 le16_to_cpu(hw->qual.CQ_currBSS),
-			 le16_to_cpu(hw->qual.ASL_currBSS),
-			 le16_to_cpu(hw->qual.ANL_currFC));
+			 le16_to_cpu(hw->qual.cq_curr_bss),
+			 le16_to_cpu(hw->qual.asl_curr_bss),
+			 le16_to_cpu(hw->qual.anl_curr_fc));
 	}
 
 	/* Get the signal rate */
diff --git a/drivers/staging/xgifb/XGI_main.h b/drivers/staging/xgifb/XGI_main.h
index 85079fea7152..7a80a90f229f 100644
--- a/drivers/staging/xgifb/XGI_main.h
+++ b/drivers/staging/xgifb/XGI_main.h
@@ -139,7 +139,7 @@ static const struct _XGIbios_mode {
 
 static const unsigned short XGI310paneltype[] = {
 	 LCD_UNKNOWN,   LCD_800x600, LCD_1024x768, LCD_1280x1024,
-	 LCD_640x480,  LCD_1024x600, LCD_1152x864, LCD_1280x960,
+	 LCD_640x480,  LCD_1024x600, LCD_1152x864,  LCD_1280x960,
 	LCD_1152x768, LCD_1400x1050, LCD_1280x768, LCD_1600x1200,
 	LCD_1024x768,  LCD_1024x768, LCD_1024x768};
 
@@ -174,7 +174,7 @@ static const struct _XGI_tvtype {
 	{"NTSC",	2},
 	{"pal",		1},
 	{"ntsc",	2},
-	{"\0",		-1}
+	{"\0",	       -1}
 };
 
 static const struct _XGI_vrate {
@@ -183,44 +183,44 @@ static const struct _XGI_vrate {
 	u16 yres;
 	u16 refresh;
 } XGIfb_vrate[] = {
-	{1,  640,  480, 60}, {2,  640,  480,  72},
-	{3, 640,   480,  75}, {4,  640, 480,  85},
+	{1,  640,  480,  60}, {2,  640,  480,  72},
+	{3,  640,  480,  75}, {4,  640,  480,  85},
 
 	{5,  640,  480, 100}, {6,  640,  480, 120},
-	{7, 640,   480, 160}, {8,  640, 480, 200},
+	{7,  640,  480, 160}, {8,  640,  480, 200},
 
-	{1,  720,  480, 60},
-	{1,  720,  576, 58},
-	{1,  800,  480, 60}, {2,  800,  480,  75}, {3, 800,   480,  85},
-	{1,  800,  600,  60}, {2, 800,   600,  72}, {3,  800, 600,  75},
-	{4,  800,  600, 85}, {5,  800,  600, 100},
-	{6, 800,   600, 120}, {7,  800, 600, 160},
+	{1,  720,  480,  60},
+	{1,  720,  576,  58},
+	{1,  800,  480,  60}, {2,  800,  480,  75}, {3,  800,  480,  85},
+	{1,  800,  600,  60}, {2,  800,  600,  72}, {3,  800,  600,  75},
+	{4,  800,  600,  85}, {5,  800,  600, 100},
+	{6,  800,  600, 120}, {7,  800,  600, 160},
 
-	{1, 1024,  768,  60}, {2, 1024,  768,  70}, {3, 1024, 768,  75},
-	{4, 1024,  768, 85}, {5, 1024,  768, 100}, {6, 1024,  768, 120},
-	{1, 1024,  576, 60}, {2, 1024,  576,  75}, {3, 1024,  576,  85},
-	{1, 1024,  600, 60},
-	{1, 1152,  768, 60},
-	{1, 1280,  720, 60}, {2, 1280,  720,  75}, {3, 1280,  720,  85},
-	{1, 1280,  768, 60},
+	{1, 1024,  768,  60}, {2, 1024,  768,  70}, {3, 1024,  768,  75},
+	{4, 1024,  768,  85}, {5, 1024,  768, 100}, {6, 1024,  768, 120},
+	{1, 1024,  576,  60}, {2, 1024,  576,  75}, {3, 1024,  576,  85},
+	{1, 1024,  600,  60},
+	{1, 1152,  768,  60},
+	{1, 1280,  720,  60}, {2, 1280,  720,  75}, {3, 1280,  720,  85},
+	{1, 1280,  768,  60},
 	{1, 1280, 1024,  60}, {2, 1280, 1024,  75}, {3, 1280, 1024,  85},
-	{1, 1280,  960, 70},
-	{1, 1400, 1050, 60},
-	{1, 1600, 1200, 60}, {2, 1600, 1200,  65},
+	{1, 1280,  960,  70},
+	{1, 1400, 1050,  60},
+	{1, 1600, 1200,  60}, {2, 1600, 1200,  65},
 	{3, 1600, 1200,  70}, {4, 1600, 1200,  75},
 
-	{5, 1600, 1200, 85}, {6, 1600, 1200, 100},
+	{5, 1600, 1200,  85}, {6, 1600, 1200, 100},
 	{7, 1600, 1200, 120},
 
-	{1, 1920, 1440, 60}, {2, 1920, 1440,  65},
+	{1, 1920, 1440,  60}, {2, 1920, 1440,  65},
 	{3, 1920, 1440,  70}, {4, 1920, 1440,  75},
 
-	{5, 1920, 1440, 85}, {6, 1920, 1440, 100},
-	{1, 2048, 1536, 60}, {2, 2048, 1536,  65},
+	{5, 1920, 1440,  85}, {6, 1920, 1440, 100},
+	{1, 2048, 1536,  60}, {2, 2048, 1536,  65},
 	{3, 2048, 1536,  70}, {4, 2048, 1536,  75},
 
-	{5, 2048, 1536, 85},
-	{0, 0, 0, 0}
+	{5, 2048, 1536,  85},
+	{0,    0,    0,   0}
 };
 
 static const struct _XGI_TV_filter {
diff --git a/drivers/staging/xgifb/XGI_main_26.c b/drivers/staging/xgifb/XGI_main_26.c
index 0c78491ff5a1..777cd6e11694 100644
--- a/drivers/staging/xgifb/XGI_main_26.c
+++ b/drivers/staging/xgifb/XGI_main_26.c
@@ -56,8 +56,8 @@ static inline void dumpVGAReg(struct xgifb_video_info *xgifb_info)
 /* --------------- Hardware Access Routines -------------------------- */
 
 static int XGIfb_mode_rate_to_dclock(struct vb_device_info *XGI_Pr,
-		struct xgi_hw_device_info *HwDeviceExtension,
-		unsigned char modeno)
+				     struct xgi_hw_device_info *HwDeviceExtension,
+				     unsigned char modeno)
 {
 	unsigned short ModeNo = modeno;
 	unsigned short ModeIdIndex = 0, ClockIndex = 0;
@@ -68,7 +68,7 @@ static int XGIfb_mode_rate_to_dclock(struct vb_device_info *XGI_Pr,
 	XGI_SearchModeID(ModeNo, &ModeIdIndex);
 
 	RefreshRateTableIndex = XGI_GetRatePtrCRT2(HwDeviceExtension, ModeNo,
-			ModeIdIndex, XGI_Pr);
+						   ModeIdIndex, XGI_Pr);
 
 	ClockIndex = XGI330_RefIndex[RefreshRateTableIndex].Ext_CRTVCLK;
 
@@ -76,11 +76,11 @@ static int XGIfb_mode_rate_to_dclock(struct vb_device_info *XGI_Pr,
 }
 
 static int XGIfb_mode_rate_to_ddata(struct vb_device_info *XGI_Pr,
-		struct xgi_hw_device_info *HwDeviceExtension,
-		unsigned char modeno,
-		u32 *left_margin, u32 *right_margin, u32 *upper_margin,
-		u32 *lower_margin, u32 *hsync_len, u32 *vsync_len, u32 *sync,
-		u32 *vmode)
+				    struct xgi_hw_device_info *HwDeviceExtension,
+				    unsigned char modeno, u32 *left_margin,
+				    u32 *right_margin, u32 *upper_margin,
+				    u32 *lower_margin, u32 *hsync_len,
+				    u32 *vsync_len, u32 *sync, u32 *vmode)
 {
 	unsigned short ModeNo = modeno;
 	unsigned short ModeIdIndex, index = 0;
@@ -95,7 +95,7 @@ static int XGIfb_mode_rate_to_ddata(struct vb_device_info *XGI_Pr,
 	if (!XGI_SearchModeID(ModeNo, &ModeIdIndex))
 		return 0;
 	RefreshRateTableIndex = XGI_GetRatePtrCRT2(HwDeviceExtension, ModeNo,
-			ModeIdIndex, XGI_Pr);
+						   ModeIdIndex, XGI_Pr);
 	index = XGI330_RefIndex[RefreshRateTableIndex].Ext_CRT1CRTC;
 
 	sr_data = XGI_CRT1Table[index].CR[5];
@@ -105,7 +105,7 @@ static int XGIfb_mode_rate_to_ddata(struct vb_device_info *XGI_Pr,
 	cr_data = XGI_CRT1Table[index].CR[3];
 
 	/* Horizontal retrace (=sync) start */
-	HRS = (cr_data & 0xff) | ((unsigned short) (sr_data & 0xC0) << 2);
+	HRS = (cr_data & 0xff) | ((unsigned short)(sr_data & 0xC0) << 2);
 	F = HRS - HDE - 3;
 
 	sr_data = XGI_CRT1Table[index].CR[6];
@@ -115,8 +115,8 @@ static int XGIfb_mode_rate_to_ddata(struct vb_device_info *XGI_Pr,
 	cr_data2 = XGI_CRT1Table[index].CR[4];
 
 	/* Horizontal blank end */
-	HBE = (cr_data & 0x1f) | ((unsigned short) (cr_data2 & 0x80) >> 2)
-			| ((unsigned short) (sr_data & 0x03) << 6);
+	HBE = (cr_data & 0x1f) | ((unsigned short)(cr_data2 & 0x80) >> 2)
+			| ((unsigned short)(sr_data & 0x03) << 6);
 
 	/* Horizontal retrace (=sync) end */
 	HRE = (cr_data2 & 0x1f) | ((sr_data & 0x04) << 3);
@@ -142,15 +142,15 @@ static int XGIfb_mode_rate_to_ddata(struct vb_device_info *XGI_Pr,
 	cr_data = XGI_CRT1Table[index].CR[10];
 
 	/* Vertical retrace (=sync) start */
-	VRS = (cr_data & 0xff) | ((unsigned short) (cr_data2 & 0x04) << 6)
-			| ((unsigned short) (cr_data2 & 0x80) << 2)
-			| ((unsigned short) (sr_data & 0x08) << 7);
+	VRS = (cr_data & 0xff) | ((unsigned short)(cr_data2 & 0x04) << 6)
+			| ((unsigned short)(cr_data2 & 0x80) << 2)
+			| ((unsigned short)(sr_data & 0x08) << 7);
 	F = VRS + 1 - VDE;
 
 	cr_data = XGI_CRT1Table[index].CR[13];
 
 	/* Vertical blank end */
-	VBE = (cr_data & 0xff) | ((unsigned short) (sr_data & 0x10) << 4);
+	VBE = (cr_data & 0xff) | ((unsigned short)(sr_data & 0x10) << 4);
 	temp = VBE - ((VDE - 1) & 511);
 	B = (temp > 0) ? temp : (temp + 512);
 
@@ -231,11 +231,11 @@ static int XGIfb_GetXG21DefaultLVDSModeIdx(struct xgifb_video_info *xgifb_info)
 {
 	int i = 0;
 
-	while ((XGIbios_mode[i].mode_no != 0)
-	       && (XGIbios_mode[i].xres <= xgifb_info->lvds_data.LVDSHDE)) {
-		if ((XGIbios_mode[i].xres == xgifb_info->lvds_data.LVDSHDE)
-		    && (XGIbios_mode[i].yres == xgifb_info->lvds_data.LVDSVDE)
-		    && (XGIbios_mode[i].bpp == 8)) {
+	while ((XGIbios_mode[i].mode_no != 0) &&
+	       (XGIbios_mode[i].xres <= xgifb_info->lvds_data.LVDSHDE)) {
+		if ((XGIbios_mode[i].xres == xgifb_info->lvds_data.LVDSHDE) &&
+		    (XGIbios_mode[i].yres == xgifb_info->lvds_data.LVDSVDE) &&
+		    (XGIbios_mode[i].bpp == 8)) {
 			return i;
 		}
 		i++;
@@ -384,9 +384,8 @@ static int XGIfb_validate_mode(struct xgifb_video_info *xgifb_info, int myindex)
 					return -1;
 				break;
 			case 640:
-				if ((XGIbios_mode[myindex].yres != 400)
-						&& (XGIbios_mode[myindex].yres
-								!= 480))
+				if ((XGIbios_mode[myindex].yres != 400) &&
+				    (XGIbios_mode[myindex].yres	!= 480))
 					return -1;
 				break;
 			case 800:
@@ -518,7 +517,7 @@ static void XGIfb_search_crt2type(const char *name)
 {
 	int i = 0;
 
-	if (name == NULL)
+	if (!name)
 		return;
 
 	while (XGI_crt2type[i].type_no != -1) {
@@ -562,7 +561,7 @@ static u8 XGIfb_search_refresh_rate(struct xgifb_video_info *xgifb_info,
 						!= 1)) {
 					pr_debug("Adjusting rate from %d down to %d\n",
 						 rate,
-						 XGIfb_vrate[i-1].refresh);
+						 XGIfb_vrate[i - 1].refresh);
 					xgifb_info->rate_idx =
 						XGIfb_vrate[i - 1].idx;
 					xgifb_info->refresh_rate =
@@ -589,7 +588,7 @@ static void XGIfb_search_tvstd(const char *name)
 {
 	int i = 0;
 
-	if (name == NULL)
+	if (!name)
 		return;
 
 	while (XGI_tvtype[i].type_no != -1) {
@@ -683,7 +682,7 @@ static void XGIfb_pre_setmode(struct xgifb_video_info *xgifb_info)
 	xgifb_reg_set(XGICR, IND_XGI_SCRATCH_REG_CR30, cr30);
 	xgifb_reg_set(XGICR, IND_XGI_SCRATCH_REG_CR31, cr31);
 	xgifb_reg_set(XGICR, IND_XGI_SCRATCH_REG_CR33,
-						(xgifb_info->rate_idx & 0x0F));
+		      (xgifb_info->rate_idx & 0x0F));
 }
 
 static void XGIfb_post_setmode(struct xgifb_video_info *xgifb_info)
@@ -730,7 +729,6 @@ static void XGIfb_post_setmode(struct xgifb_video_info *xgifb_info)
 
 	if (xgifb_info->display2 == XGIFB_DISP_TV &&
 	    xgifb_info->hasVB == HASVB_301) {
-
 		reg = xgifb_reg_get(XGIPART4, 0x01);
 
 		if (reg < 0xB0) { /* Set filter for XGI301 */
@@ -763,16 +761,13 @@ static void XGIfb_post_setmode(struct xgifb_video_info *xgifb_info)
 				     0x01);
 
 			if (xgifb_info->TV_type == TVMODE_NTSC) {
-
 				xgifb_reg_and(XGIPART2, 0x3a, 0x1f);
 
 				if (xgifb_info->TV_plug == TVPLUG_SVIDEO) {
-
 					xgifb_reg_and(XGIPART2, 0x30, 0xdf);
 
 				} else if (xgifb_info->TV_plug
 						== TVPLUG_COMPOSITE) {
-
 					xgifb_reg_or(XGIPART2, 0x30, 0x20);
 
 					switch (xgifb_info->video_width) {
@@ -822,16 +817,13 @@ static void XGIfb_post_setmode(struct xgifb_video_info *xgifb_info)
 				}
 
 			} else if (xgifb_info->TV_type == TVMODE_PAL) {
-
 				xgifb_reg_and(XGIPART2, 0x3A, 0x1F);
 
 				if (xgifb_info->TV_plug == TVPLUG_SVIDEO) {
-
 					xgifb_reg_and(XGIPART2, 0x30, 0xDF);
 
 				} else if (xgifb_info->TV_plug
 						== TVPLUG_COMPOSITE) {
-
 					xgifb_reg_or(XGIPART2, 0x30, 0x20);
 
 					switch (xgifb_info->video_width) {
@@ -912,7 +904,7 @@ static void XGIfb_post_setmode(struct xgifb_video_info *xgifb_info)
 }
 
 static int XGIfb_do_set_var(struct fb_var_screeninfo *var, int isactive,
-		struct fb_info *info)
+			    struct fb_info *info)
 {
 	struct xgifb_video_info *xgifb_info = info->par;
 	struct xgi_hw_device_info *hw_info = &xgifb_info->hw_info;
@@ -945,17 +937,15 @@ static int XGIfb_do_set_var(struct fb_var_screeninfo *var, int isactive,
 	if (var->pixclock) {
 		drate = 1000000000 / var->pixclock;
 		hrate = (drate * 1000) / htotal;
-		xgifb_info->refresh_rate = (unsigned int) (hrate * 2
+		xgifb_info->refresh_rate = (unsigned int)(hrate * 2
 				/ vtotal);
 	} else {
 		xgifb_info->refresh_rate = 60;
 	}
 
 	pr_debug("Change mode to %dx%dx%d-%dHz\n",
-	       var->xres,
-	       var->yres,
-	       var->bits_per_pixel,
-	       xgifb_info->refresh_rate);
+		 var->xres, var->yres, var->bits_per_pixel,
+		 xgifb_info->refresh_rate);
 
 	old_mode = xgifb_info->mode_idx;
 	xgifb_info->mode_idx = 0;
@@ -992,7 +982,6 @@ static int XGIfb_do_set_var(struct fb_var_screeninfo *var, int isactive,
 	}
 
 	if (isactive) {
-
 		XGIfb_pre_setmode(xgifb_info);
 		if (XGISetModeNew(xgifb_info, hw_info,
 				  XGIbios_mode[xgifb_info->mode_idx].mode_no)
@@ -1064,7 +1053,7 @@ static int XGIfb_do_set_var(struct fb_var_screeninfo *var, int isactive,
 			break;
 		}
 	}
-	XGIfb_bpp_to_var(xgifb_info, var); /*update ARGB info*/
+	XGIfb_bpp_to_var(xgifb_info, var); /* update ARGB info */
 
 	dumpVGAReg(xgifb_info);
 	return 0;
@@ -1150,7 +1139,7 @@ static int XGIfb_setcolreg(unsigned int regno, unsigned int red,
 		}
 		break;
 	case 16:
-		((u32 *) (info->pseudo_palette))[regno] = ((red & 0xf800))
+		((u32 *)(info->pseudo_palette))[regno] = ((red & 0xf800))
 				| ((green & 0xfc00) >> 5) | ((blue & 0xf800)
 				>> 11);
 		break;
@@ -1158,7 +1147,7 @@ static int XGIfb_setcolreg(unsigned int regno, unsigned int red,
 		red >>= 8;
 		green >>= 8;
 		blue >>= 8;
-		((u32 *) (info->pseudo_palette))[regno] = (red << 16) | (green
+		((u32 *)(info->pseudo_palette))[regno] = (red << 16) | (green
 				<< 8) | (blue);
 		break;
 	}
@@ -1168,7 +1157,7 @@ static int XGIfb_setcolreg(unsigned int regno, unsigned int red,
 /* ----------- FBDev related routines for all series ---------- */
 
 static int XGIfb_get_fix(struct fb_fix_screeninfo *fix, int con,
-		struct fb_info *info)
+			 struct fb_info *info)
 {
 	struct xgifb_video_info *xgifb_info = info->par;
 
@@ -1250,7 +1239,7 @@ static int XGIfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 		drate = 1000000000 / var->pixclock;
 		hrate = (drate * 1000) / htotal;
 		xgifb_info->refresh_rate =
-			(unsigned int) (hrate * 2 / vtotal);
+			(unsigned int)(hrate * 2 / vtotal);
 		pr_debug(
 			"%s: pixclock = %d ,htotal=%d, vtotal=%d\n"
 			"%s: drate=%d, hrate=%d, refresh_rate=%d\n",
@@ -1262,10 +1251,10 @@ static int XGIfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 
 	search_idx = 0;
 	while ((XGIbios_mode[search_idx].mode_no != 0) &&
-		(XGIbios_mode[search_idx].xres <= var->xres)) {
+	       (XGIbios_mode[search_idx].xres <= var->xres)) {
 		if ((XGIbios_mode[search_idx].xres == var->xres) &&
-			(XGIbios_mode[search_idx].yres == var->yres) &&
-			(XGIbios_mode[search_idx].bpp == var->bits_per_pixel)) {
+		    (XGIbios_mode[search_idx].yres == var->yres) &&
+		    (XGIbios_mode[search_idx].bpp == var->bits_per_pixel)) {
 			if (XGIfb_validate_mode(xgifb_info, search_idx) > 0) {
 				found_mode = 1;
 				break;
@@ -1275,9 +1264,8 @@ static int XGIfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 	}
 
 	if (!found_mode) {
-
 		pr_err("%dx%dx%d is no valid mode\n",
-			var->xres, var->yres, var->bits_per_pixel);
+		       var->xres, var->yres, var->bits_per_pixel);
 		search_idx = 0;
 		while (XGIbios_mode[search_idx].mode_no != 0) {
 			if ((var->xres <= XGIbios_mode[search_idx].xres) &&
@@ -1296,11 +1284,11 @@ static int XGIfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 			var->xres = XGIbios_mode[search_idx].xres;
 			var->yres = XGIbios_mode[search_idx].yres;
 			pr_debug("Adapted to mode %dx%dx%d\n",
-				var->xres, var->yres, var->bits_per_pixel);
+				 var->xres, var->yres, var->bits_per_pixel);
 
 		} else {
 			pr_err("Failed to find similar mode to %dx%dx%d\n",
-				var->xres, var->yres, var->bits_per_pixel);
+			       var->xres, var->yres, var->bits_per_pixel);
 			return -EINVAL;
 		}
 	}
@@ -1332,7 +1320,7 @@ static int XGIfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 }
 
 static int XGIfb_pan_display(struct fb_var_screeninfo *var,
-		struct fb_info *info)
+			     struct fb_info *info)
 {
 	int err;
 
@@ -1344,9 +1332,8 @@ static int XGIfb_pan_display(struct fb_var_screeninfo *var,
 	if (var->vmode & FB_VMODE_YWRAP) {
 		if (var->yoffset >= info->var.yres_virtual || var->xoffset)
 			return -EINVAL;
-	} else if (var->xoffset + info->var.xres > info->var.xres_virtual
-				|| var->yoffset + info->var.yres
-						> info->var.yres_virtual) {
+	} else if (var->xoffset + info->var.xres > info->var.xres_virtual ||
+		   var->yoffset + info->var.yres > info->var.yres_virtual) {
 		return -EINVAL;
 	}
 	err = XGIfb_pan_var(var, info);
@@ -1401,7 +1388,6 @@ static struct fb_ops XGIfb_ops = {
 
 static int XGIfb_get_dram_size(struct xgifb_video_info *xgifb_info)
 {
-
 	u8 ChannelNum, tmp;
 	u8 reg = 0;
 
@@ -1474,10 +1460,8 @@ static int XGIfb_get_dram_size(struct xgifb_video_info *xgifb_info)
 	xgifb_info->video_size = xgifb_info->video_size * ChannelNum;
 
 	pr_info("SR14=%x DramSzie %x ChannelNum %x\n",
-	       reg,
-	       xgifb_info->video_size, ChannelNum);
+		reg, xgifb_info->video_size, ChannelNum);
 	return 0;
-
 }
 
 static void XGIfb_detect_VB(struct xgifb_video_info *xgifb_info)
@@ -1597,7 +1581,6 @@ static int __init XGIfb_setup(char *options)
 	pr_info("Options: %s\n", options);
 
 	while ((this_opt = strsep(&options, ",")) != NULL) {
-
 		if (!*this_opt)
 			continue;
 
@@ -1634,8 +1617,7 @@ static int __init XGIfb_setup(char *options)
 	return 0;
 }
 
-static int xgifb_probe(struct pci_dev *pdev,
-		const struct pci_device_id *ent)
+static int xgifb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	u8 reg, reg1;
 	u8 CR48, CR38;
@@ -1670,7 +1652,7 @@ static int xgifb_probe(struct pci_dev *pdev,
 	xgifb_info->mmio_size = pci_resource_len(pdev, 1);
 	xgifb_info->vga_base = pci_resource_start(pdev, 2) + 0x30;
 	dev_info(&pdev->dev, "Relocate IO address: %Lx [%08lx]\n",
-		 (u64) pci_resource_start(pdev, 2),
+		 (u64)pci_resource_start(pdev, 2),
 		 xgifb_info->vga_base);
 
 	if (pci_enable_device(pdev)) {
@@ -1688,7 +1670,7 @@ static int xgifb_probe(struct pci_dev *pdev,
 	xgifb_reg_set(XGISR, IND_SIS_PASSWORD, SIS_PASSWORD);
 	reg1 = xgifb_reg_get(XGISR, IND_SIS_PASSWORD);
 
-	if (reg1 != 0xa1) { /*I/O error */
+	if (reg1 != 0xa1) { /* I/O error */
 		dev_err(&pdev->dev, "I/O error\n");
 		ret = -EIO;
 		goto error_disable;
@@ -1698,7 +1680,7 @@ static int xgifb_probe(struct pci_dev *pdev,
 	case PCI_DEVICE_ID_XGI_20:
 		xgifb_reg_or(XGICR, Index_CR_GPIO_Reg3, GPIOG_EN);
 		CR48 = xgifb_reg_get(XGICR, Index_CR_GPIO_Reg1);
-		if (CR48&GPIOG_READ)
+		if (CR48 & GPIOG_READ)
 			xgifb_info->chip = XG21;
 		else
 			xgifb_info->chip = XG20;
@@ -1727,7 +1709,7 @@ static int xgifb_probe(struct pci_dev *pdev,
 		xgifb_info->video_size = video_size_max;
 	}
 
-	/* Enable PCI_LINEAR_ADDRESSING and MMIO_ENABLE  */
+	/* Enable PCI_LINEAR_ADDRESSING and MMIO_ENABLE */
 	xgifb_reg_or(XGISR,
 		     IND_SIS_PCI_ADDRESS_SET,
 		     (SIS_PCI_ADDR_ENABLE | SIS_MEM_MAP_IO_ENABLE));
@@ -1740,7 +1722,7 @@ static int xgifb_probe(struct pci_dev *pdev,
 				xgifb_info->video_size,
 				"XGIfb FB")) {
 		dev_err(&pdev->dev, "Unable request memory size %x\n",
-		       xgifb_info->video_size);
+			xgifb_info->video_size);
 		dev_err(&pdev->dev,
 			"Fatal error: Unable to reserve frame buffer memory. Is there another framebuffer driver active?\n");
 		ret = -ENODEV;
@@ -1763,13 +1745,13 @@ static int xgifb_probe(struct pci_dev *pdev,
 
 	dev_info(&pdev->dev,
 		 "Framebuffer at 0x%Lx, mapped to 0x%p, size %dk\n",
-		 (u64) xgifb_info->video_base,
+		 (u64)xgifb_info->video_base,
 		 xgifb_info->video_vbase,
 		 xgifb_info->video_size / 1024);
 
 	dev_info(&pdev->dev,
 		 "MMIO at 0x%Lx, mapped to 0x%p, size %ldk\n",
-		 (u64) xgifb_info->mmio_base, xgifb_info->mmio_vbase,
+		 (u64)xgifb_info->mmio_base, xgifb_info->mmio_vbase,
 		 xgifb_info->mmio_size / 1024);
 
 	pci_set_drvdata(pdev, xgifb_info);
@@ -1784,9 +1766,9 @@ static int xgifb_probe(struct pci_dev *pdev,
 		xgifb_info->hasVB = HASVB_NONE;
 	} else if (xgifb_info->chip == XG21) {
 		CR38 = xgifb_reg_get(XGICR, 0x38);
-		if ((CR38&0xE0) == 0xC0)
+		if ((CR38 & 0xE0) == 0xC0)
 			xgifb_info->display2 = XGIFB_DISP_LCD;
-		else if ((CR38&0xE0) == 0x60)
+		else if ((CR38 & 0xE0) == 0x60)
 			xgifb_info->hasVB = HASVB_CHRONTEL;
 		else
 			xgifb_info->hasVB = HASVB_NONE;
@@ -1903,8 +1885,7 @@ static int xgifb_probe(struct pci_dev *pdev,
 	xgifb_info->refresh_rate = refresh_rate;
 	if (xgifb_info->refresh_rate == 0)
 		xgifb_info->refresh_rate = 60;
-	if (XGIfb_search_refresh_rate(xgifb_info,
-			xgifb_info->refresh_rate) == 0) {
+	if (XGIfb_search_refresh_rate(xgifb_info, xgifb_info->refresh_rate) == 0) {
 		xgifb_info->rate_idx = 1;
 		xgifb_info->refresh_rate = 60;
 	}
@@ -1939,15 +1920,13 @@ static int xgifb_probe(struct pci_dev *pdev,
 	default:
 		xgifb_info->video_cmap_len = 16;
 		pr_info("Unsupported depth %d\n",
-		       xgifb_info->video_bpp);
+			xgifb_info->video_bpp);
 		break;
 	}
 
 	pr_info("Default mode is %dx%dx%d (%dHz)\n",
-	       xgifb_info->video_width,
-	       xgifb_info->video_height,
-	       xgifb_info->video_bpp,
-	       xgifb_info->refresh_rate);
+		xgifb_info->video_width, xgifb_info->video_height,
+		xgifb_info->video_bpp, xgifb_info->refresh_rate);
 
 	fb_info->var.red.length		= 8;
 	fb_info->var.green.length	= 8;
@@ -1964,22 +1943,20 @@ static int xgifb_probe(struct pci_dev *pdev,
 
 	XGIfb_bpp_to_var(xgifb_info, &fb_info->var);
 
-	fb_info->var.pixclock = (u32) (1000000000 /
-			XGIfb_mode_rate_to_dclock(&xgifb_info->dev_info,
-				hw_info,
-				XGIbios_mode[xgifb_info->mode_idx].mode_no));
-
-	if (XGIfb_mode_rate_to_ddata(&xgifb_info->dev_info, hw_info,
-		XGIbios_mode[xgifb_info->mode_idx].mode_no,
-		&fb_info->var.left_margin,
-		&fb_info->var.right_margin,
-		&fb_info->var.upper_margin,
-		&fb_info->var.lower_margin,
-		&fb_info->var.hsync_len,
-		&fb_info->var.vsync_len,
-		&fb_info->var.sync,
-		&fb_info->var.vmode)) {
-
+	fb_info->var.pixclock = (u32)(1000000000 / XGIfb_mode_rate_to_dclock
+				      (&xgifb_info->dev_info, hw_info,
+				       XGIbios_mode[xgifb_info->mode_idx].mode_no));
+
+	if (XGIfb_mode_rate_to_ddata(&xgifb_info->dev_info,
+				     hw_info, XGIbios_mode[xgifb_info->mode_idx].mode_no,
+				     &fb_info->var.left_margin,
+				     &fb_info->var.right_margin,
+				     &fb_info->var.upper_margin,
+				     &fb_info->var.lower_margin,
+				     &fb_info->var.hsync_len,
+				     &fb_info->var.vsync_len,
+				     &fb_info->var.sync,
+				     &fb_info->var.vmode)) {
 		if ((fb_info->var.vmode & FB_VMODE_MASK) ==
 		    FB_VMODE_INTERLACED) {
 			fb_info->var.yres <<= 1;
@@ -1990,7 +1967,6 @@ static int xgifb_probe(struct pci_dev *pdev,
 			fb_info->var.yres >>= 1;
 			fb_info->var.yres_virtual >>= 1;
 		}
-
 	}
 
 	fb_info->flags = FBINFO_FLAG_DEFAULT;
@@ -2028,9 +2004,7 @@ error:
 	return ret;
 }
 
-/*****************************************************/
-/*                PCI DEVICE HANDLING                */
-/*****************************************************/
+/* -------------------- PCI DEVICE HANDLING -------------------- */
 
 static void xgifb_remove(struct pci_dev *pdev)
 {
@@ -2054,25 +2028,23 @@ static struct pci_driver xgifb_driver = {
 	.remove = xgifb_remove
 };
 
-/*****************************************************/
-/*                      MODULE                       */
-/*****************************************************/
+/* -------------------- MODULE -------------------- */
 
-module_param(mode, charp, 0);
+module_param(mode, charp, 0000);
 MODULE_PARM_DESC(mode,
-	"Selects the desired default display mode in the format XxYxDepth (eg. 1024x768x16).");
+		 "Selects the desired default display mode in the format XxYxDepth (eg. 1024x768x16).");
 
-module_param(forcecrt2type, charp, 0);
+module_param(forcecrt2type, charp, 0000);
 MODULE_PARM_DESC(forcecrt2type,
-	"Force the second display output type. Possible values are NONE, LCD, TV, VGA, SVIDEO or COMPOSITE.");
+		 "Force the second display output type. Possible values are NONE, LCD, TV, VGA, SVIDEO or COMPOSITE.");
 
-module_param(vesa, int, 0);
+module_param(vesa, int, 0000);
 MODULE_PARM_DESC(vesa,
-	"Selects the desired default display mode by VESA mode number (eg. 0x117).");
+		 "Selects the desired default display mode by VESA mode number (eg. 0x117).");
 
-module_param(filter, int, 0);
+module_param(filter, int, 0000);
 MODULE_PARM_DESC(filter,
-	"Selects TV flicker filter type (only for systems with a SiS301 video bridge). Possible values 0-7. Default: [no filter]).");
+		 "Selects TV flicker filter type (only for systems with a SiS301 video bridge). Possible values 0-7. Default: [no filter]).");
 
 static int __init xgifb_init(void)
 {
diff --git a/drivers/staging/xgifb/vb_init.c b/drivers/staging/xgifb/vb_init.c
index 062ece22ed84..14af157958cd 100644
--- a/drivers/staging/xgifb/vb_init.c
+++ b/drivers/staging/xgifb/vb_init.c
@@ -55,8 +55,9 @@ XGINew_GetXG20DRAMType(struct xgi_hw_device_info *HwDeviceExtension,
 		xgifb_reg_or(pVBInfo->P3d4, 0x4A, 0x80); /* Enable GPIOH read */
 		/* GPIOF 0:DVI 1:DVO */
 		data = xgifb_reg_get(pVBInfo->P3d4, 0x48);
-		/* HOTPLUG_SUPPORT */
-		/* for current XG20 & XG21, GPIOH is floating, driver will
+		/*
+		 * HOTPLUG_SUPPORT
+		 * for current XG20 & XG21, GPIOH is floating, driver will
 		 * fix DDR temporarily
 		 */
 		/* DVI read GPIOH */
@@ -199,7 +200,8 @@ static void XGINew_DDRII_Bootup_XG27(
 }
 
 static void XGINew_DDR2_MRS_XG20(struct xgi_hw_device_info *HwDeviceExtension,
-		unsigned long P3c4, struct vb_device_info *pVBInfo)
+				 unsigned long P3c4,
+				 struct vb_device_info *pVBInfo)
 {
 	unsigned long P3d4 = P3c4 + 0x10;
 
@@ -353,8 +355,8 @@ static void XGINew_DDR2_DefaultRegister(
 		unsigned long Port, struct vb_device_info *pVBInfo)
 {
 	unsigned long P3d4 = Port, P3c4 = Port - 0x10;
-
-	/* keep following setting sequence, each setting in
+	/*
+	 * keep following setting sequence, each setting in
 	 * the same reg insert idle
 	 */
 	xgifb_reg_set(P3d4, 0x82, 0x77);
@@ -387,7 +389,7 @@ static void XGINew_DDR2_DefaultRegister(
 }
 
 static void XGI_SetDRAM_Helper(unsigned long P3d4, u8 seed, u8 temp2, u8 reg,
-	u8 shift_factor, u8 mask1, u8 mask2)
+			       u8 shift_factor, u8 mask1, u8 mask2)
 {
 	u8 j;
 
@@ -460,15 +462,15 @@ static void XGINew_SetDRAMDefaultRegister340(
 
 	for (j = 0; j <= 6; j++) /* CR90 - CR96 */
 		xgifb_reg_set(P3d4, (0x90 + j),
-				pVBInfo->CR40[14 + j][pVBInfo->ram_type]);
+			      pVBInfo->CR40[14 + j][pVBInfo->ram_type]);
 
 	for (j = 0; j <= 2; j++) /* CRC3 - CRC5 */
 		xgifb_reg_set(P3d4, (0xC3 + j),
-				pVBInfo->CR40[21 + j][pVBInfo->ram_type]);
+			      pVBInfo->CR40[21 + j][pVBInfo->ram_type]);
 
 	for (j = 0; j < 2; j++) /* CR8A - CR8B */
 		xgifb_reg_set(P3d4, (0x8A + j),
-				pVBInfo->CR40[1 + j][pVBInfo->ram_type]);
+			      pVBInfo->CR40[1 + j][pVBInfo->ram_type]);
 
 	if (HwDeviceExtension->jChipType == XG42)
 		xgifb_reg_set(P3d4, 0x8C, 0x87);
@@ -539,7 +541,8 @@ static unsigned short XGINew_SetDRAMSize20Reg(
 }
 
 static int XGINew_ReadWriteRest(unsigned short StopAddr,
-		unsigned short StartAddr, struct vb_device_info *pVBInfo)
+				unsigned short StartAddr,
+				struct vb_device_info *pVBInfo)
 {
 	int i;
 	unsigned long Position = 0;
@@ -583,7 +586,7 @@ static unsigned char XGINew_CheckFrequence(struct vb_device_info *pVBInfo)
 }
 
 static void XGINew_CheckChannel(struct xgi_hw_device_info *HwDeviceExtension,
-		struct vb_device_info *pVBInfo)
+				struct vb_device_info *pVBInfo)
 {
 	unsigned char data;
 
@@ -647,7 +650,7 @@ static void XGINew_CheckChannel(struct xgi_hw_device_info *HwDeviceExtension,
 				pVBInfo->ram_bus = 16; /* 16 bits */
 				/* (0x31:12x8x2) 22bit + 2 rank */
 				xgifb_reg_set(pVBInfo->P3c4, 0x13, 0xB1);
-				/* 0x41:16Mx16 bit*/
+				/* 0x41:16Mx16 bit */
 				xgifb_reg_set(pVBInfo->P3c4, 0x14, 0x41);
 				usleep_range(15, 1015);
 
@@ -660,7 +663,7 @@ static void XGINew_CheckChannel(struct xgi_hw_device_info *HwDeviceExtension,
 					xgifb_reg_set(pVBInfo->P3c4,
 						      0x13,
 						      0x31);
-					/* 0x31:8Mx16 bit*/
+					/* 0x31:8Mx16 bit */
 					xgifb_reg_set(pVBInfo->P3c4,
 						      0x14,
 						      0x31);
@@ -678,7 +681,7 @@ static void XGINew_CheckChannel(struct xgi_hw_device_info *HwDeviceExtension,
 				pVBInfo->ram_bus = 8; /* 8 bits */
 				/* (0x31:12x8x2) 22bit + 2 rank */
 				xgifb_reg_set(pVBInfo->P3c4, 0x13, 0xB1);
-				/* 0x30:8Mx8 bit*/
+				/* 0x30:8Mx8 bit */
 				xgifb_reg_set(pVBInfo->P3c4, 0x14, 0x30);
 				usleep_range(15, 1015);
 
@@ -697,7 +700,7 @@ static void XGINew_CheckChannel(struct xgi_hw_device_info *HwDeviceExtension,
 	case XG27:
 		pVBInfo->ram_bus = 16; /* 16 bits */
 		pVBInfo->ram_channel = 1; /* Single channel */
-		xgifb_reg_set(pVBInfo->P3c4, 0x14, 0x51); /* 32Mx16 bit*/
+		xgifb_reg_set(pVBInfo->P3c4, 0x14, 0x51); /* 32Mx16 bit */
 		break;
 	case XG42:
 		/*
@@ -785,7 +788,7 @@ static void XGINew_CheckChannel(struct xgi_hw_device_info *HwDeviceExtension,
 }
 
 static int XGINew_DDRSizing340(struct xgi_hw_device_info *HwDeviceExtension,
-		struct vb_device_info *pVBInfo)
+			       struct vb_device_info *pVBInfo)
 {
 	u8 i, size;
 	unsigned short memsize, start_addr;
@@ -827,8 +830,8 @@ static int XGINew_DDRSizing340(struct xgi_hw_device_info *HwDeviceExtension,
 }
 
 static void XGINew_SetDRAMSize_340(struct xgifb_video_info *xgifb_info,
-		struct xgi_hw_device_info *HwDeviceExtension,
-		struct vb_device_info *pVBInfo)
+				   struct xgi_hw_device_info *HwDeviceExtension,
+				   struct vb_device_info *pVBInfo)
 {
 	unsigned short data;
 
@@ -905,9 +908,9 @@ static bool xgifb_read_vbios(struct pci_dev *pdev)
 		goto error;
 	if (j == 0xff)
 		j = 1;
-	/*
-	 * Read the LVDS table index scratch register set by the BIOS.
-	 */
+
+	/* Read the LVDS table index scratch register set by the BIOS. */
+
 	entry = xgifb_reg_get(xgifb_info->dev_info.P3d4, 0x36);
 	if (entry >= j)
 		entry = 0;
@@ -1039,8 +1042,9 @@ static void XGINew_SetModeScratch(struct vb_device_info *pVBInfo)
 	}
 
 	tempcl |= SetSimuScanMode;
-	if ((!(temp & ActiveCRT1)) && ((temp & ActiveLCD) || (temp & ActiveTV)
-			|| (temp & ActiveCRT2)))
+	if ((!(temp & ActiveCRT1)) && ((temp & ActiveLCD) ||
+				       (temp & ActiveTV) ||
+				       (temp & ActiveCRT2)))
 		tempcl ^= (SetSimuScanMode | SwitchCRT2);
 	if ((temp & ActiveLCD) && (temp & ActiveTV))
 		tempcl ^= (SetSimuScanMode | SwitchCRT2);
@@ -1085,7 +1089,7 @@ static unsigned short XGINew_SenseLCD(struct xgi_hw_device_info
 }
 
 static void XGINew_GetXG21Sense(struct pci_dev *pdev,
-		struct vb_device_info *pVBInfo)
+				struct vb_device_info *pVBInfo)
 {
 	struct xgifb_video_info *xgifb_info = pci_get_drvdata(pdev);
 	unsigned char Temp;
@@ -1095,7 +1099,7 @@ static void XGINew_GetXG21Sense(struct pci_dev *pdev,
 		/* LVDS on chip */
 		xgifb_reg_and_or(pVBInfo->P3d4, 0x38, ~0xE0, 0xC0);
 	} else {
-		/* Enable GPIOA/B read  */
+		/* Enable GPIOA/B read */
 		xgifb_reg_and_or(pVBInfo->P3d4, 0x4A, ~0x03, 0x03);
 		Temp = xgifb_reg_get(pVBInfo->P3d4, 0x48) & 0xC0;
 		if (Temp == 0xC0) { /* DVI & DVO GPIOA/B pull high */
@@ -1119,7 +1123,7 @@ static void XGINew_GetXG27Sense(struct vb_device_info *pVBInfo)
 	unsigned char Temp, bCR4A;
 
 	bCR4A = xgifb_reg_get(pVBInfo->P3d4, 0x4A);
-	/* Enable GPIOA/B/C read  */
+	/* Enable GPIOA/B/C read */
 	xgifb_reg_and_or(pVBInfo->P3d4, 0x4A, ~0x07, 0x07);
 	Temp = xgifb_reg_get(pVBInfo->P3d4, 0x48) & 0x07;
 	xgifb_reg_set(pVBInfo->P3d4, 0x4A, bCR4A);
diff --git a/drivers/staging/xgifb/vb_setmode.c b/drivers/staging/xgifb/vb_setmode.c
index d8010c5c1a70..7c7c8c8f1df3 100644
--- a/drivers/staging/xgifb/vb_setmode.c
+++ b/drivers/staging/xgifb/vb_setmode.c
@@ -55,7 +55,7 @@ void InitTo330Pointer(unsigned char ChipType, struct vb_device_info *pVBInfo)
 		pVBInfo->XGINew_CR97 = 0xc1;
 		pVBInfo->SR18 = XG27_SR18;
 
-		/*Z11m DDR*/
+		/* Z11m DDR */
 		temp = xgifb_reg_get(pVBInfo->P3c4, 0x3B);
 		/* SR3B[7][3]MAA15 MAA11 (Power on Trapping) */
 		if (((temp & 0x88) == 0x80) || ((temp & 0x88) == 0x08))
@@ -73,7 +73,7 @@ static void XGI_SetSeqRegs(struct vb_device_info *pVBInfo)
 		/* Get SR1,2,3,4 from file */
 		/* SR1 is with screen off 0x20 */
 		SRdata = XGI330_StandTable.SR[i];
-		xgifb_reg_set(pVBInfo->P3c4, i+1, SRdata); /* Set SR 1 2 3 4 */
+		xgifb_reg_set(pVBInfo->P3c4, i + 1, SRdata); /* Set SR 1 2 3 4 */
 	}
 }
 
@@ -167,7 +167,8 @@ static unsigned char XGI_SetDefaultVCLK(struct vb_device_info *pVBInfo)
 }
 
 static unsigned char XGI_AjustCRT2Rate(unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex, unsigned short *i,
+				       unsigned short RefreshRateTableIndex,
+				       unsigned short *i,
 		struct vb_device_info *pVBInfo)
 {
 	unsigned short tempax, tempbx, resinfo, modeflag, infoflag;
@@ -244,7 +245,7 @@ static unsigned char XGI_AjustCRT2Rate(unsigned short ModeIdIndex,
 }
 
 static void XGI_SetSync(unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+			struct vb_device_info *pVBInfo)
 {
 	unsigned short sync, temp;
 
@@ -257,7 +258,7 @@ static void XGI_SetSync(unsigned short RefreshRateTableIndex,
 }
 
 static void XGI_SetCRT1Timing_H(struct vb_device_info *pVBInfo,
-		struct xgi_hw_device_info *HwDeviceExtension)
+				struct xgi_hw_device_info *HwDeviceExtension)
 {
 	unsigned char data, data1, pushax;
 	unsigned short i, j;
@@ -359,9 +360,9 @@ static void XGI_SetCRT1Timing_V(unsigned short ModeIdIndex,
 }
 
 static void XGI_SetCRT1CRTC(unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo,
-		struct xgi_hw_device_info *HwDeviceExtension)
+			    unsigned short RefreshRateTableIndex,
+			    struct vb_device_info *pVBInfo,
+			    struct xgi_hw_device_info *HwDeviceExtension)
 {
 	unsigned char index, data;
 	unsigned short i;
@@ -390,14 +391,14 @@ static void XGI_SetCRT1CRTC(unsigned short ModeIdIndex,
 		xgifb_reg_set(pVBInfo->P3d4, 0x14, 0x4F);
 }
 
-/* --------------------------------------------------------------------- */
-/* Function : XGI_SetXG21CRTC */
-/* Input : Stand or enhance CRTC table */
-/* Output : Fill CRT Hsync/Vsync to SR2E/SR2F/SR30/SR33/SR34/SR3F */
-/* Description : Set LCD timing */
-/* --------------------------------------------------------------------- */
+/*
+ * Function : XGI_SetXG21CRTC
+ * Input : Stand or enhance CRTC table
+ * Output : Fill CRT Hsync/Vsync to SR2E/SR2F/SR30/SR33/SR34/SR3F
+ * Description : Set LCD timing
+ */
 static void XGI_SetXG21CRTC(unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+			    struct vb_device_info *pVBInfo)
 {
 	unsigned char index, Tempax, Tempbx, Tempcx, Tempdx;
 	unsigned short Temp1, Temp2, Temp3;
@@ -506,8 +507,8 @@ static void XGI_SetXG27CRTC(unsigned short RefreshRateTableIndex,
 
 	/* SR0B */
 	Tempax = XGI_CRT1Table[index].CR[5];
-	Tempax &= 0xC0; /* Tempax[7:6]: SR0B[7:6]: HRS[9:8]*/
-	Tempbx |= (Tempax << 2); /* Tempbx: HRS[9:0] */
+	Tempax &= 0xC0; /* Tempax[7:6]: SR0B[7:6]: HRS[9:8] */
+	Tempbx |= Tempax << 2; /* Tempbx: HRS[9:0] */
 
 	Tempax = XGI_CRT1Table[index].CR[4]; /* CR5 HRE */
 	Tempax &= 0x1F; /* Tempax[4:0]: HRE[4:0] */
@@ -530,7 +531,7 @@ static void XGI_SetXG27CRTC(unsigned short RefreshRateTableIndex,
 	Tempax = XGI_CRT1Table[index].CR[5]; /* SR0B */
 	Tempax &= 0xC0; /* Tempax[7:6]: SR0B[7:6]: HRS[9:8]*/
 	Tempax >>= 6; /* Tempax[1:0]: HRS[9:8]*/
-	Tempax |= ((Tempbx << 2) & 0xFF); /* Tempax[7:2]: HRE[5:0] */
+	Tempax |= (Tempbx << 2) & 0xFF; /* Tempax[7:2]: HRE[5:0] */
 	/* SR2F [7:2][1:0]: HRE[5:0]HRS[9:8] */
 	xgifb_reg_set(pVBInfo->P3c4, 0x2F, Tempax);
 	xgifb_reg_and_or(pVBInfo->P3c4, 0x30, 0xE3, 00);
@@ -548,12 +549,12 @@ static void XGI_SetXG27CRTC(unsigned short RefreshRateTableIndex,
 	Tempax >>= 2; /* Tempax[0]: VRS[8] */
 	/* SR35[0]: VRS[8] */
 	xgifb_reg_and_or(pVBInfo->P3c4, 0x35, ~0x01, Tempax);
-	Tempcx |= (Tempax << 8); /* Tempcx <= VRS[8:0] */
-	Tempcx |= ((Tempbx & 0x80) << 2); /* Tempcx <= VRS[9:0] */
+	Tempcx |= Tempax << 8; /* Tempcx <= VRS[8:0] */
+	Tempcx |= (Tempbx & 0x80) << 2; /* Tempcx <= VRS[9:0] */
 	/* Tempax: SR0A */
 	Tempax = XGI_CRT1Table[index].CR[14];
 	Tempax &= 0x08; /* SR0A[3] VRS[10] */
-	Tempcx |= (Tempax << 7); /* Tempcx <= VRS[10:0] */
+	Tempcx |= Tempax << 7; /* Tempcx <= VRS[10:0] */
 
 	/* Tempax: CR11 VRE */
 	Tempax = XGI_CRT1Table[index].CR[11];
@@ -636,12 +637,12 @@ static void xgifb_set_lcd(int chip_id,
 		xgifb_reg_or(pVBInfo->P3c4, 0x35, 0x80);
 }
 
-/* --------------------------------------------------------------------- */
-/* Function : XGI_UpdateXG21CRTC */
-/* Input : */
-/* Output : CRT1 CRTC */
-/* Description : Modify CRT1 Hsync/Vsync to fix LCD mode timing */
-/* --------------------------------------------------------------------- */
+/*
+ * Function : XGI_UpdateXG21CRTC
+ * Input :
+ * Output : CRT1 CRTC
+ * Description : Modify CRT1 Hsync/Vsync to fix LCD mode timing
+ */
 static void XGI_UpdateXG21CRTC(unsigned short ModeNo,
 			       struct vb_device_info *pVBInfo,
 			       unsigned short RefreshRateTableIndex)
@@ -665,19 +666,19 @@ static void XGI_UpdateXG21CRTC(unsigned short ModeNo,
 
 	if (index != -1) {
 		xgifb_reg_set(pVBInfo->P3d4, 0x02,
-				XGI_UpdateCRT1Table[index].CR02);
+			      XGI_UpdateCRT1Table[index].CR02);
 		xgifb_reg_set(pVBInfo->P3d4, 0x03,
-				XGI_UpdateCRT1Table[index].CR03);
+			      XGI_UpdateCRT1Table[index].CR03);
 		xgifb_reg_set(pVBInfo->P3d4, 0x15,
-				XGI_UpdateCRT1Table[index].CR15);
+			      XGI_UpdateCRT1Table[index].CR15);
 		xgifb_reg_set(pVBInfo->P3d4, 0x16,
-				XGI_UpdateCRT1Table[index].CR16);
+			      XGI_UpdateCRT1Table[index].CR16);
 	}
 }
 
 static void XGI_SetCRT1DE(unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+			  unsigned short RefreshRateTableIndex,
+			  struct vb_device_info *pVBInfo)
 {
 	unsigned short resindex, tempax, tempbx, tempcx, temp, modeflag;
 
@@ -715,7 +716,7 @@ static void XGI_SetCRT1DE(unsigned short ModeIdIndex,
 	xgifb_reg_set(pVBInfo->P3d4, 0x11, data); /* Unlock CRTC */
 	xgifb_reg_set(pVBInfo->P3d4, 0x01, (unsigned short)(tempcx & 0xff));
 	xgifb_reg_and_or(pVBInfo->P3d4, 0x0b, ~0x0c,
-			(unsigned short)((tempcx & 0x0ff00) >> 10));
+			 (unsigned short)((tempcx & 0x0ff00) >> 10));
 	xgifb_reg_set(pVBInfo->P3d4, 0x12, (unsigned short)(tempbx & 0xff));
 	tempax = 0;
 	tempbx >>= 8;
@@ -796,7 +797,7 @@ static void XGI_SetCRT1Offset(unsigned short ModeNo,
 	i |= temp;
 	xgifb_reg_set(pVBInfo->P3c4, 0x0E, i);
 
-	temp = (unsigned char) temp2;
+	temp = (unsigned char)temp2;
 	temp &= 0xFF; /* al */
 	xgifb_reg_set(pVBInfo->P3d4, 0x13, temp);
 
@@ -822,15 +823,15 @@ static void XGI_SetCRT1Offset(unsigned short ModeNo,
 }
 
 static unsigned short XGI_GetVCLK2Ptr(unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+				      unsigned short RefreshRateTableIndex,
+				      struct vb_device_info *pVBInfo)
 {
 	unsigned short VCLKIndex, modeflag;
 
 	/* si+Ext_ResInfo */
 	modeflag = XGI330_EModeIDTable[ModeIdIndex].Ext_ModeFlag;
 
-	if (pVBInfo->VBInfo & (SetCRT2ToLCD | XGI_SetCRT2ToLCDA)) { /*301b*/
+	if (pVBInfo->VBInfo & (SetCRT2ToLCD | XGI_SetCRT2ToLCDA)) { /* 301b */
 		if (pVBInfo->LCDResInfo != Panel_1024x768)
 			/* LCDXlat2VCLK */
 			VCLKIndex = VCLK108_2_315 + 5;
@@ -951,8 +952,8 @@ static void XGI_SetCRT1FIFO(struct xgi_hw_device_info *HwDeviceExtension,
 }
 
 static void XGI_SetVCLKState(struct xgi_hw_device_info *HwDeviceExtension,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+			     unsigned short RefreshRateTableIndex,
+			     struct vb_device_info *pVBInfo)
 {
 	unsigned short data, data2 = 0;
 	short VCLK;
@@ -989,9 +990,9 @@ static void XGI_SetVCLKState(struct xgi_hw_device_info *HwDeviceExtension,
 }
 
 static void XGI_SetCRT1ModeRegs(struct xgi_hw_device_info *HwDeviceExtension,
-		unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+				unsigned short ModeIdIndex,
+				unsigned short RefreshRateTableIndex,
+				struct vb_device_info *pVBInfo)
 {
 	unsigned short data, data2, data3, infoflag = 0, modeflag, resindex,
 			xres;
@@ -1087,9 +1088,9 @@ static void XGI_WriteDAC(unsigned short dl,
 		else
 			swap(bl, bh);
 	}
-	outb((unsigned short) dh, pVBInfo->P3c9);
-	outb((unsigned short) bh, pVBInfo->P3c9);
-	outb((unsigned short) bl, pVBInfo->P3c9);
+	outb((unsigned short)dh, pVBInfo->P3c9);
+	outb((unsigned short)bh, pVBInfo->P3c9);
+	outb((unsigned short)bl, pVBInfo->P3c9);
 }
 
 static void XGI_LoadDAC(struct vb_device_info *pVBInfo)
@@ -1187,8 +1188,8 @@ static void XGI_GetLVDSResInfo(unsigned short ModeIdIndex,
 }
 
 static void const *XGI_GetLcdPtr(struct XGI330_LCDDataTablStruct const *table,
-		unsigned short ModeIdIndex,
-		struct vb_device_info *pVBInfo)
+				 unsigned short ModeIdIndex,
+				 struct vb_device_info *pVBInfo)
 {
 	unsigned short i, tempdx, tempbx, modeflag;
 
@@ -1201,12 +1202,12 @@ static void const *XGI_GetLcdPtr(struct XGI330_LCDDataTablStruct const *table,
 	while (table[i].PANELID != 0xff) {
 		tempdx = pVBInfo->LCDResInfo;
 		if (tempbx & 0x0080) { /* OEMUtil */
-			tempbx &= (~0x0080);
+			tempbx &= ~0x0080;
 			tempdx = pVBInfo->LCDTypeInfo;
 		}
 
 		if (pVBInfo->LCDInfo & EnableScalingLCD)
-			tempdx &= (~PanelResInfo);
+			tempdx &= ~PanelResInfo;
 
 		if (table[i].PANELID == tempdx) {
 			tempbx = table[i].MASK;
@@ -1226,8 +1227,8 @@ static void const *XGI_GetLcdPtr(struct XGI330_LCDDataTablStruct const *table,
 }
 
 static struct SiS_TVData const *XGI_GetTVPtr(unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+					     unsigned short RefreshRateTableIndex,
+					     struct vb_device_info *pVBInfo)
 {
 	unsigned short i, tempdx, tempal, modeflag;
 
@@ -1441,9 +1442,9 @@ static void XGI_SetLVDSRegs(unsigned short ModeIdIndex,
 	tempbx >>= 3;
 
 	xgifb_reg_set(pVBInfo->Part1Port, 0x16,
-			(unsigned short) (tempbx & 0xff));
+		      (unsigned short)(tempbx & 0xff));
 	xgifb_reg_set(pVBInfo->Part1Port, 0x17,
-			(unsigned short) (tempcx & 0xff));
+		      (unsigned short)(tempcx & 0xff));
 
 	tempax = pVBInfo->HT;
 
@@ -1469,7 +1470,7 @@ static void XGI_SetLVDSRegs(unsigned short ModeIdIndex,
 
 	xgifb_reg_set(pVBInfo->Part1Port, 0x15, tempax);
 	xgifb_reg_set(pVBInfo->Part1Port, 0x14,
-			(unsigned short) (tempbx & 0xff));
+		      (unsigned short)(tempbx & 0xff));
 
 	tempax = pVBInfo->VT;
 	tempbx = LCDPtr1->LCDVDES;
@@ -1480,17 +1481,14 @@ static void XGI_SetLVDSRegs(unsigned short ModeIdIndex,
 	if (tempcx >= tempax)
 		tempcx -= tempax;
 
-	xgifb_reg_set(pVBInfo->Part1Port, 0x1b,
-			(unsigned short) (tempbx & 0xff));
-	xgifb_reg_set(pVBInfo->Part1Port, 0x1c,
-			(unsigned short) (tempcx & 0xff));
+	xgifb_reg_set(pVBInfo->Part1Port, 0x1b,	(unsigned short)(tempbx & 0xff));
+	xgifb_reg_set(pVBInfo->Part1Port, 0x1c,	(unsigned short)(tempcx & 0xff));
 
 	tempbx = (tempbx >> 8) & 0x07;
 	tempcx = (tempcx >> 8) & 0x07;
 
-	xgifb_reg_set(pVBInfo->Part1Port, 0x1d,
-			(unsigned short) ((tempcx << 3)
-					| tempbx));
+	xgifb_reg_set(pVBInfo->Part1Port, 0x1d,	(unsigned short)((tempcx << 3) |
+		      tempbx));
 
 	tempax = pVBInfo->VT;
 	tempbx = LCDPtr1->LCDVRS;
@@ -1504,10 +1502,8 @@ static void XGI_SetLVDSRegs(unsigned short ModeIdIndex,
 	if (tempcx >= tempax)
 		tempcx -= tempax;
 
-	xgifb_reg_set(pVBInfo->Part1Port, 0x18,
-			(unsigned short) (tempbx & 0xff));
-	xgifb_reg_and_or(pVBInfo->Part1Port, 0x19, ~0x0f,
-			(unsigned short) (tempcx & 0x0f));
+	xgifb_reg_set(pVBInfo->Part1Port, 0x18,	(unsigned short)(tempbx & 0xff));
+	xgifb_reg_and_or(pVBInfo->Part1Port, 0x19, ~0x0f, (unsigned short)(tempcx & 0x0f));
 
 	tempax = ((tempbx >> 8) & 0x07) << 3;
 
@@ -1518,8 +1514,7 @@ static void XGI_SetLVDSRegs(unsigned short ModeIdIndex,
 	if (pVBInfo->LCDInfo & XGI_EnableLVDSDDA)
 		tempax |= 0x40;
 
-	xgifb_reg_and_or(pVBInfo->Part1Port, 0x1a, 0x07,
-				tempax);
+	xgifb_reg_and_or(pVBInfo->Part1Port, 0x1a, 0x07, tempax);
 
 	tempbx = pVBInfo->VDE;
 	tempax = pVBInfo->VGAVDE;
@@ -1527,7 +1522,7 @@ static void XGI_SetLVDSRegs(unsigned short ModeIdIndex,
 	temp = tempax; /* 0430 ylshieh */
 	temp1 = (temp << 18) / tempbx;
 
-	tempdx = (unsigned short) ((temp << 18) % tempbx);
+	tempdx = (unsigned short)((temp << 18) % tempbx);
 
 	if (tempdx != 0)
 		temp1 += 1;
@@ -1535,12 +1530,10 @@ static void XGI_SetLVDSRegs(unsigned short ModeIdIndex,
 	temp2 = temp1;
 	push3 = temp2;
 
-	xgifb_reg_set(pVBInfo->Part1Port, 0x37,
-			(unsigned short) (temp2 & 0xff));
-	xgifb_reg_set(pVBInfo->Part1Port, 0x36,
-			(unsigned short) ((temp2 >> 8) & 0xff));
+	xgifb_reg_set(pVBInfo->Part1Port, 0x37,	(unsigned short)(temp2 & 0xff));
+	xgifb_reg_set(pVBInfo->Part1Port, 0x36,	(unsigned short)((temp2 >> 8) & 0xff));
 
-	tempbx = (unsigned short) (temp2 >> 16);
+	tempbx = (unsigned short)(temp2 >> 16);
 	tempax = tempbx & 0x03;
 
 	tempbx = pVBInfo->VGAVDE;
@@ -1553,24 +1546,20 @@ static void XGI_SetLVDSRegs(unsigned short ModeIdIndex,
 		temp2 = push3;
 		xgifb_reg_set(pVBInfo->Part4Port,
 			      0x3c,
-			      (unsigned short) (temp2 & 0xff));
+			      (unsigned short)(temp2 & 0xff));
 		xgifb_reg_set(pVBInfo->Part4Port,
 			      0x3b,
-			      (unsigned short) ((temp2 >> 8) &
+			      (unsigned short)((temp2 >> 8) &
 			      0xff));
-		tempbx = (unsigned short) (temp2 >> 16);
-		xgifb_reg_and_or(pVBInfo->Part4Port, 0x3a,
-				~0xc0,
-				(unsigned short) ((tempbx &
-						   0xff) << 6));
+		tempbx = (unsigned short)(temp2 >> 16);
+		xgifb_reg_and_or(pVBInfo->Part4Port, 0x3a, ~0xc0,
+				 (unsigned short)((tempbx & 0xff) << 6));
 
 		tempcx = pVBInfo->VGAVDE;
 		if (tempcx == pVBInfo->VDE)
-			xgifb_reg_and_or(pVBInfo->Part4Port,
-					0x30, ~0x0c, 0x00);
+			xgifb_reg_and_or(pVBInfo->Part4Port, 0x30, ~0x0c, 0x00);
 		else
-			xgifb_reg_and_or(pVBInfo->Part4Port,
-					0x30, ~0x0c, 0x08);
+			xgifb_reg_and_or(pVBInfo->Part4Port, 0x30, ~0x0c, 0x08);
 	}
 
 	tempcx = pVBInfo->VGAHDE;
@@ -1578,7 +1567,7 @@ static void XGI_SetLVDSRegs(unsigned short ModeIdIndex,
 
 	temp1 = tempcx << 16;
 
-	tempax = (unsigned short) (temp1 / tempbx);
+	tempax = (unsigned short)(temp1 / tempbx);
 
 	if ((tempbx & 0xffff) == (tempcx & 0xffff))
 		tempax = 65535;
@@ -1592,42 +1581,38 @@ static void XGI_SetLVDSRegs(unsigned short ModeIdIndex,
 
 	temp3 = (temp3 & 0xffff0000) + (temp1 & 0xffff);
 
-	tempax = (unsigned short) (temp3 & 0xff);
+	tempax = (unsigned short)(temp3 & 0xff);
 	xgifb_reg_set(pVBInfo->Part1Port, 0x1f, tempax);
 
 	temp1 = pVBInfo->VGAVDE << 18;
 	temp1 = temp1 / push3;
-	tempbx = (unsigned short) (temp1 & 0xffff);
+	tempbx = (unsigned short)(temp1 & 0xffff);
 
 	if (pVBInfo->LCDResInfo == Panel_1024x768)
 		tempbx -= 1;
 
 	tempax = ((tempbx >> 8) & 0xff) << 3;
-	tempax |= (unsigned short) ((temp3 >> 8) & 0x07);
-	xgifb_reg_set(pVBInfo->Part1Port, 0x20,
-			(unsigned short) (tempax & 0xff));
-	xgifb_reg_set(pVBInfo->Part1Port, 0x21,
-			(unsigned short) (tempbx & 0xff));
+	tempax |= (unsigned short)((temp3 >> 8) & 0x07);
+	xgifb_reg_set(pVBInfo->Part1Port, 0x20, (unsigned short)(tempax & 0xff));
+	xgifb_reg_set(pVBInfo->Part1Port, 0x21, (unsigned short)(tempbx & 0xff));
 
 	temp3 >>= 16;
 
 	if (modeflag & HalfDCLK)
 		temp3 >>= 1;
 
-	xgifb_reg_set(pVBInfo->Part1Port, 0x22,
-			(unsigned short) ((temp3 >> 8) & 0xff));
-	xgifb_reg_set(pVBInfo->Part1Port, 0x23,
-			(unsigned short) (temp3 & 0xff));
+	xgifb_reg_set(pVBInfo->Part1Port, 0x22,	(unsigned short)((temp3 >> 8) & 0xff));
+	xgifb_reg_set(pVBInfo->Part1Port, 0x23,	(unsigned short)(temp3 & 0xff));
 }
 
-/* --------------------------------------------------------------------- */
-/* Function : XGI_GETLCDVCLKPtr */
-/* Input : */
-/* Output : al -> VCLK Index */
-/* Description : */
-/* --------------------------------------------------------------------- */
+/*
+ * Function : XGI_GETLCDVCLKPtr
+ * Input :
+ * Output : al -> VCLK Index
+ * Description :
+ */
 static void XGI_GetLCDVCLKPtr(unsigned char *di_0, unsigned char *di_1,
-		struct vb_device_info *pVBInfo)
+			      struct vb_device_info *pVBInfo)
 {
 	unsigned short index;
 
@@ -1645,7 +1630,8 @@ static void XGI_GetLCDVCLKPtr(unsigned char *di_0, unsigned char *di_1,
 }
 
 static unsigned char XGI_GetVCLKPtr(unsigned short RefreshRateTableIndex,
-		unsigned short ModeIdIndex, struct vb_device_info *pVBInfo)
+				    unsigned short ModeIdIndex,
+				    struct vb_device_info *pVBInfo)
 {
 	unsigned short index, modeflag;
 	unsigned char tempal;
@@ -1681,15 +1667,11 @@ static unsigned char XGI_GetVCLKPtr(unsigned short RefreshRateTableIndex,
 				return tempal;
 			}
 
-			if (pVBInfo->TVInfo & TVSetYPbPr750p) {
-				tempal = XGI_YPbPr750pVCLK;
-				return tempal;
-			}
+			if (pVBInfo->TVInfo & TVSetYPbPr750p)
+				return XGI_YPbPr750pVCLK;
 
-			if (pVBInfo->TVInfo & TVSetYPbPr525p) {
-				tempal = YPbPr525pVCLK;
-				return tempal;
-			}
+			if (pVBInfo->TVInfo & TVSetYPbPr525p)
+				return YPbPr525pVCLK;
 
 			tempal = NTSC1024VCLK;
 
@@ -1705,12 +1687,11 @@ static unsigned char XGI_GetVCLKPtr(unsigned short RefreshRateTableIndex,
 	} /* {End of VB} */
 
 	inb((pVBInfo->P3ca + 0x02));
-	tempal = XGI330_RefIndex[RefreshRateTableIndex].Ext_CRTVCLK;
-	return tempal;
+	return XGI330_RefIndex[RefreshRateTableIndex].Ext_CRTVCLK;
 }
 
 static void XGI_GetVCLKLen(unsigned char tempal, unsigned char *di_0,
-		unsigned char *di_1, struct vb_device_info *pVBInfo)
+			   unsigned char *di_1, struct vb_device_info *pVBInfo)
 {
 	if (pVBInfo->VBType & (VB_SIS301 | VB_SIS301B | VB_SIS302B
 			| VB_SIS301LV | VB_SIS302LV | VB_XGI301C)) {
@@ -1726,8 +1707,8 @@ static void XGI_GetVCLKLen(unsigned char tempal, unsigned char *di_0,
 }
 
 static void XGI_SetCRT2ECLK(unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+			    unsigned short RefreshRateTableIndex,
+			    struct vb_device_info *pVBInfo)
 {
 	unsigned char di_0, di_1, tempal;
 	int i;
@@ -1738,7 +1719,7 @@ static void XGI_SetCRT2ECLK(unsigned short ModeIdIndex,
 
 	for (i = 0; i < 4; i++) {
 		xgifb_reg_and_or(pVBInfo->P3d4, 0x31, ~0x30,
-				(unsigned short) (0x10 * i));
+				 (unsigned short)(0x10 * i));
 		if (!(pVBInfo->VBInfo & XGI_SetCRT2ToLCDA) &&
 		    !(pVBInfo->VBInfo & SetInSlaveMode)) {
 			xgifb_reg_set(pVBInfo->P3c4, 0x2e, di_0);
@@ -1876,8 +1857,7 @@ finish:
 	pVBInfo->VBType = tempbx;
 }
 
-static void XGI_GetVBInfo(unsigned short ModeIdIndex,
-		struct vb_device_info *pVBInfo)
+static void XGI_GetVBInfo(unsigned short ModeIdIndex, struct vb_device_info *pVBInfo)
 {
 	unsigned short tempax, push, tempbx, temp, modeflag;
 
@@ -1921,7 +1901,7 @@ static void XGI_GetVBInfo(unsigned short ModeIdIndex,
 			tempbx |= SetCRT2ToHiVision;
 
 			if (temp != YPbPrMode1080i) {
-				tempbx &= (~SetCRT2ToHiVision);
+				tempbx &= ~SetCRT2ToHiVision;
 				tempbx |= SetCRT2ToYPbPr525750;
 			}
 		}
@@ -2002,8 +1982,7 @@ static void XGI_GetVBInfo(unsigned short ModeIdIndex,
 	pVBInfo->VBInfo = tempbx;
 }
 
-static void XGI_GetTVInfo(unsigned short ModeIdIndex,
-		struct vb_device_info *pVBInfo)
+static void XGI_GetTVInfo(unsigned short ModeIdIndex, struct vb_device_info *pVBInfo)
 {
 	unsigned short tempbx = 0, resinfo = 0, modeflag, index1;
 
@@ -2078,7 +2057,7 @@ static unsigned char XGI_GetLCDInfo(unsigned short ModeIdIndex,
 	pVBInfo->LCDTypeInfo = 0;
 	pVBInfo->LCDInfo = 0;
 
-	/* si+Ext_ResInfo // */
+	/* si+Ext_ResInfo */
 	resinfo = XGI330_EModeIDTable[ModeIdIndex].Ext_RESINFO;
 	temp = xgifb_reg_get(pVBInfo->P3d4, 0x36); /* Get LCD Res.Info */
 	tempbx = temp & 0x0F;
@@ -2175,12 +2154,12 @@ static unsigned char XG21GPIODataTransfer(unsigned char ujDate)
 	return ujRet;
 }
 
-/*----------------------------------------------------------------------------*/
-/* output                                                                     */
-/*      bl[5] : LVDS signal                                                   */
-/*      bl[1] : LVDS backlight                                                */
-/*      bl[0] : LVDS VDD                                                      */
-/*----------------------------------------------------------------------------*/
+/*
+ * output
+ *	bl[5] : LVDS signal
+ *	bl[1] : LVDS backlight
+ *	bl[0] : LVDS VDD
+ */
 static unsigned char XGI_XG21GetPSCValue(struct vb_device_info *pVBInfo)
 {
 	unsigned char CR4A, temp;
@@ -2196,12 +2175,12 @@ static unsigned char XGI_XG21GetPSCValue(struct vb_device_info *pVBInfo)
 	return temp;
 }
 
-/*----------------------------------------------------------------------------*/
-/* output                                                                     */
-/*      bl[5] : LVDS signal                                                   */
-/*      bl[1] : LVDS backlight                                                */
-/*      bl[0] : LVDS VDD                                                      */
-/*----------------------------------------------------------------------------*/
+/*
+ * output
+ *	bl[5] : LVDS signal
+ *	bl[1] : LVDS backlight
+ *	bl[0] : LVDS VDD
+ */
 static unsigned char XGI_XG27GetPSCValue(struct vb_device_info *pVBInfo)
 {
 	unsigned char CR4A, CRB4, temp;
@@ -2219,17 +2198,17 @@ static unsigned char XGI_XG27GetPSCValue(struct vb_device_info *pVBInfo)
 	return temp;
 }
 
-/*----------------------------------------------------------------------------*/
-/* input                                                                      */
-/*      bl[5] : 1;LVDS signal on                                              */
-/*      bl[1] : 1;LVDS backlight on                                           */
-/*      bl[0] : 1:LVDS VDD on                                                 */
-/*      bh: 100000b : clear bit 5, to set bit5                                */
-/*          000010b : clear bit 1, to set bit1                                */
-/*          000001b : clear bit 0, to set bit0                                */
-/*----------------------------------------------------------------------------*/
+/*
+ * input
+ *	bl[5] : 1;LVDS signal on
+ *	bl[1] : 1;LVDS backlight on
+ *	bl[0] : 1:LVDS VDD on
+ *	bh: 100000b : clear bit 5, to set bit5
+ *	    000010b : clear bit 1, to set bit1
+ *	    000001b : clear bit 0, to set bit0
+ */
 static void XGI_XG21BLSignalVDD(unsigned short tempbh, unsigned short tempbl,
-		struct vb_device_info *pVBInfo)
+				struct vb_device_info *pVBInfo)
 {
 	unsigned char CR4A, temp;
 
@@ -2254,7 +2233,7 @@ static void XGI_XG21BLSignalVDD(unsigned short tempbh, unsigned short tempbl,
 }
 
 static void XGI_XG27BLSignalVDD(unsigned short tempbh, unsigned short tempbl,
-		struct vb_device_info *pVBInfo)
+				struct vb_device_info *pVBInfo)
 {
 	unsigned char CR4A, temp;
 	unsigned short tempbh0, tempbl0;
@@ -2284,8 +2263,8 @@ static void XGI_XG27BLSignalVDD(unsigned short tempbh, unsigned short tempbl,
 }
 
 static void XGI_DisplayOn(struct xgifb_video_info *xgifb_info,
-		struct xgi_hw_device_info *pXGIHWDE,
-		struct vb_device_info *pVBInfo)
+			  struct xgi_hw_device_info *pXGIHWDE,
+			  struct vb_device_info *pVBInfo)
 {
 	xgifb_reg_and_or(pVBInfo->P3c4, 0x01, 0xDF, 0x00);
 	if (pXGIHWDE->jChipType == XG21) {
@@ -2328,8 +2307,8 @@ static void XGI_DisplayOn(struct xgifb_video_info *xgifb_info,
 }
 
 void XGI_DisplayOff(struct xgifb_video_info *xgifb_info,
-		struct xgi_hw_device_info *pXGIHWDE,
-		struct vb_device_info *pVBInfo)
+		    struct xgi_hw_device_info *pXGIHWDE,
+		    struct vb_device_info *pVBInfo)
 {
 	if (pXGIHWDE->jChipType == XG21) {
 		if (pVBInfo->IF_DEF_LVDS == 1) {
@@ -2448,7 +2427,7 @@ exit:
 static unsigned char XGI_IsLCDDualLink(struct vb_device_info *pVBInfo)
 {
 	if ((pVBInfo->VBInfo & (SetCRT2ToLCD | XGI_SetCRT2ToLCDA)) &&
-			(pVBInfo->LCDInfo & SetLCDDualLink)) /* shampoo0129 */
+	    (pVBInfo->LCDInfo & SetLCDDualLink)) /* shampoo0129 */
 		return 1;
 
 	return 0;
@@ -2466,16 +2445,15 @@ static void XGI_GetRAMDAC2DATA(unsigned short ModeIdIndex,
 	modeflag = XGI330_EModeIDTable[ModeIdIndex].Ext_ModeFlag;
 	CRT1Index = XGI330_RefIndex[RefreshRateTableIndex].Ext_CRT1CRTC;
 	CRT1Index &= IndexMask;
-	temp1 = (unsigned short) XGI_CRT1Table[CRT1Index].CR[0];
-	temp2 = (unsigned short) XGI_CRT1Table[CRT1Index].CR[5];
+	temp1 = (unsigned short)XGI_CRT1Table[CRT1Index].CR[0];
+	temp2 = (unsigned short)XGI_CRT1Table[CRT1Index].CR[5];
 	tempax = (temp1 & 0xFF) | ((temp2 & 0x03) << 8);
-	tempbx = (unsigned short) XGI_CRT1Table[CRT1Index].CR[8];
-	tempcx = (unsigned short)
-			XGI_CRT1Table[CRT1Index].CR[14] << 8;
+	tempbx = (unsigned short)XGI_CRT1Table[CRT1Index].CR[8];
+	tempcx = (unsigned short)XGI_CRT1Table[CRT1Index].CR[14] << 8;
 	tempcx &= 0x0100;
 	tempcx <<= 2;
 	tempbx |= tempcx;
-	temp1 = (unsigned short) XGI_CRT1Table[CRT1Index].CR[9];
+	temp1 = (unsigned short)XGI_CRT1Table[CRT1Index].CR[9];
 
 	if (temp1 & 0x01)
 		tempbx |= 0x0100;
@@ -2497,8 +2475,8 @@ static void XGI_GetRAMDAC2DATA(unsigned short ModeIdIndex,
 }
 
 static void XGI_GetCRT2Data(unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+			    unsigned short RefreshRateTableIndex,
+			    struct vb_device_info *pVBInfo)
 {
 	unsigned short tempax = 0, tempbx = 0, modeflag, resinfo;
 
@@ -2667,8 +2645,8 @@ static void XGI_GetCRT2Data(unsigned short ModeIdIndex,
 }
 
 static void XGI_SetCRT2VCLK(unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+			    unsigned short RefreshRateTableIndex,
+			    struct vb_device_info *pVBInfo)
 {
 	unsigned char di_0, di_1, tempal;
 
@@ -2739,9 +2717,9 @@ static unsigned short XGI_GetOffset(unsigned short ModeNo,
 }
 
 static void XGI_SetCRT2Offset(unsigned short ModeNo,
-		unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+			      unsigned short ModeIdIndex,
+			      unsigned short RefreshRateTableIndex,
+			      struct vb_device_info *pVBInfo)
 {
 	unsigned short offset;
 	unsigned char temp;
@@ -2750,11 +2728,11 @@ static void XGI_SetCRT2Offset(unsigned short ModeNo,
 		return;
 
 	offset = XGI_GetOffset(ModeNo, ModeIdIndex, RefreshRateTableIndex);
-	temp = (unsigned char) (offset & 0xFF);
+	temp = (unsigned char)(offset & 0xFF);
 	xgifb_reg_set(pVBInfo->Part1Port, 0x07, temp);
-	temp = (unsigned char) ((offset & 0xFF00) >> 8);
+	temp = (unsigned char)((offset & 0xFF00) >> 8);
 	xgifb_reg_set(pVBInfo->Part1Port, 0x09, temp);
-	temp = (unsigned char) (((offset >> 3) & 0xFF) + 1);
+	temp = (unsigned char)(((offset >> 3) & 0xFF) + 1);
 	xgifb_reg_set(pVBInfo->Part1Port, 0x03, temp);
 }
 
@@ -2767,8 +2745,8 @@ static void XGI_SetCRT2FIFO(struct vb_device_info *pVBInfo)
 }
 
 static void XGI_PreSetGroup1(unsigned short ModeNo, unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+			     unsigned short RefreshRateTableIndex,
+			     struct vb_device_info *pVBInfo)
 {
 	u8 tempcx;
 
@@ -2783,8 +2761,8 @@ static void XGI_PreSetGroup1(unsigned short ModeNo, unsigned short ModeIdIndex,
 }
 
 static void XGI_SetGroup1(unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+			  unsigned short RefreshRateTableIndex,
+			  struct vb_device_info *pVBInfo)
 {
 	unsigned short temp = 0, tempax = 0, tempbx = 0, tempcx = 0,
 			pushbx = 0, CRT1Index, modeflag;
@@ -2933,11 +2911,11 @@ static unsigned short XGI_GetVGAHT2(struct vb_device_info *pVBInfo)
 	tempax = (pVBInfo->VT - pVBInfo->VDE) * pVBInfo->RVBHCFACT;
 	tempax = (tempax * pVBInfo->HT) / tempbx;
 
-	return (unsigned short) tempax;
+	return (unsigned short)tempax;
 }
 
 static void XGI_SetLockRegs(unsigned short ModeNo, unsigned short ModeIdIndex,
-		struct vb_device_info *pVBInfo)
+			    struct vb_device_info *pVBInfo)
 {
 	unsigned short push1, push2, tempax, tempbx = 0, tempcx, temp, resinfo,
 			modeflag;
@@ -3044,14 +3022,14 @@ static void XGI_SetLockRegs(unsigned short ModeNo, unsigned short ModeIdIndex,
 			if (ModeNo == 0x50) {
 				if (pVBInfo->TVInfo == SetNTSCTV) {
 					xgifb_reg_set(pVBInfo->Part1Port,
-							0x07, 0x30);
+						      0x07, 0x30);
 					xgifb_reg_set(pVBInfo->Part1Port,
-							0x08, 0x03);
+						      0x08, 0x03);
 				} else {
 					xgifb_reg_set(pVBInfo->Part1Port,
-							0x07, 0x2f);
+						      0x07, 0x2f);
 					xgifb_reg_set(pVBInfo->Part1Port,
-							0x08, 0x02);
+						      0x08, 0x02);
 				}
 			}
 		}
@@ -3064,7 +3042,7 @@ static void XGI_SetLockRegs(unsigned short ModeNo, unsigned short ModeIdIndex,
 	tempbx = pVBInfo->VGAVT;
 	push1 = tempbx;
 	tempcx = 0x121;
-	tempbx = pVBInfo->VGAVDE; /* 0x0E Virtical Display End */
+	tempbx = pVBInfo->VGAVDE; /* 0x0E Vertical Display End */
 
 	if (tempbx == 357)
 		tempbx = 350;
@@ -3116,7 +3094,7 @@ static void XGI_SetLockRegs(unsigned short ModeNo, unsigned short ModeIdIndex,
 	if (tempbx & 0x0400)
 		tempcx |= 0x0600;
 
-	/* 0x11 Vertival Blank End */
+	/* 0x11 Vertical Blank End */
 	xgifb_reg_set(pVBInfo->Part1Port, 0x11, 0x00);
 
 	tempax = push1;
@@ -3227,7 +3205,7 @@ static void XGI_SetLockRegs(unsigned short ModeNo, unsigned short ModeIdIndex,
 }
 
 static void XGI_SetGroup2(unsigned short ModeNo, unsigned short ModeIdIndex,
-		struct vb_device_info *pVBInfo)
+			  struct vb_device_info *pVBInfo)
 {
 	unsigned short i, j, tempax, tempbx, tempcx, temp, push1, push2,
 			modeflag;
@@ -3315,7 +3293,7 @@ static void XGI_SetGroup2(unsigned short ModeNo, unsigned short ModeIdIndex,
 		tempax = (tempax & 0x00FF) | ((tempax & 0x00FF) << 8);
 		push1 = tempax;
 		temp = (tempax & 0xFF00) >> 8;
-		temp += (unsigned short) TimingPoint[0];
+		temp += (unsigned short)TimingPoint[0];
 
 		if (pVBInfo->VBType & (VB_SIS301B | VB_SIS302B | VB_SIS301LV
 				| VB_SIS302LV | VB_XGI301C)) {
@@ -3526,7 +3504,7 @@ static void XGI_SetGroup2(unsigned short ModeNo, unsigned short ModeIdIndex,
 
 	tempcx = 0x0101;
 
-	if (pVBInfo->VBInfo & SetCRT2ToTV) { /*301b*/
+	if (pVBInfo->VBInfo & SetCRT2ToTV) { /* 301b */
 		if (pVBInfo->VGAHDE >= 1024) {
 			tempcx = 0x1920;
 			if (pVBInfo->VGAHDE >= 1280) {
@@ -3562,7 +3540,7 @@ static void XGI_SetGroup2(unsigned short ModeNo, unsigned short ModeIdIndex,
 		if (temp2 != 0)
 			tempeax += 1;
 
-		tempax = (unsigned short) tempeax;
+		tempax = (unsigned short)tempeax;
 
 		/* 301b */
 		if (pVBInfo->VBType & (VB_SIS301B | VB_SIS302B | VB_SIS301LV
@@ -3572,9 +3550,9 @@ static void XGI_SetGroup2(unsigned short ModeNo, unsigned short ModeIdIndex,
 		/* end 301b */
 
 		tempbx = push1;
-		tempbx = (unsigned short) (((tempeax & 0x0000FF00) & 0x1F00)
+		tempbx = (unsigned short)(((tempeax & 0x0000FF00) & 0x1F00)
 				| (tempbx & 0x00FF));
-		tempax = (unsigned short) (((tempeax & 0x000000FF) << 8)
+		tempax = (unsigned short)(((tempeax & 0x000000FF) << 8)
 				| (tempax & 0x00FF));
 		temp = (tempax & 0xFF00) >> 8;
 	} else {
@@ -3622,14 +3600,14 @@ static void XGI_SetGroup2(unsigned short ModeNo, unsigned short ModeIdIndex,
 
 	xgifb_reg_set(pVBInfo->Part2Port, 0x4d, temp);
 	temp = xgifb_reg_get(pVBInfo->Part2Port, 0x43); /* 301b change */
-	xgifb_reg_set(pVBInfo->Part2Port, 0x43, (unsigned short) (temp - 3));
+	xgifb_reg_set(pVBInfo->Part2Port, 0x43, (unsigned short)(temp - 3));
 
 	if (!(pVBInfo->TVInfo & (TVSetYPbPr525p | TVSetYPbPr750p))) {
 		if (pVBInfo->TVInfo & NTSC1024x768) {
 			TimingPoint = XGI_NTSC1024AdjTime;
 			for (i = 0x1c, j = 0; i <= 0x30; i++, j++) {
 				xgifb_reg_set(pVBInfo->Part2Port, i,
-						TimingPoint[j]);
+					      TimingPoint[j]);
 			}
 			xgifb_reg_set(pVBInfo->Part2Port, 0x43, 0x72);
 		}
@@ -3639,7 +3617,7 @@ static void XGI_SetGroup2(unsigned short ModeNo, unsigned short ModeIdIndex,
 	if (pVBInfo->VBType & VB_XGI301C) {
 		if (pVBInfo->TVInfo & TVSetPALM)
 			xgifb_reg_and_or(pVBInfo->Part2Port, 0x4E, ~0x08,
-					0x08); /* PALM Mode */
+					 0x08); /* PALM Mode */
 	}
 
 	if (pVBInfo->TVInfo & TVSetPALM) {
@@ -3656,8 +3634,7 @@ static void XGI_SetGroup2(unsigned short ModeNo, unsigned short ModeIdIndex,
 	}
 }
 
-static void XGI_SetLCDRegs(unsigned short ModeIdIndex,
-		struct vb_device_info *pVBInfo)
+static void XGI_SetLCDRegs(unsigned short ModeIdIndex, struct vb_device_info *pVBInfo)
 {
 	unsigned short pushbx, tempax, tempbx, tempcx, temp, tempah,
 			tempbh, tempch;
@@ -3853,12 +3830,12 @@ static void XGI_SetLCDRegs(unsigned short ModeIdIndex,
 	}
 }
 
-/* --------------------------------------------------------------------- */
-/* Function : XGI_GetTap4Ptr */
-/* Input : */
-/* Output : di -> Tap4 Reg. Setting Pointer */
-/* Description : */
-/* --------------------------------------------------------------------- */
+/*
+ * Function : XGI_GetTap4Ptr
+ * Input :
+ * Output : di -> Tap4 Reg. Setting Pointer
+ * Description :
+ */
 static struct XGI301C_Tap4TimingStruct const
 *XGI_GetTap4Ptr(unsigned short tempcx, struct vb_device_info *pVBInfo)
 {
@@ -3882,7 +3859,7 @@ static struct XGI301C_Tap4TimingStruct const
 
 	if (pVBInfo->VBInfo & SetCRT2ToYPbPr525750) {
 		if ((pVBInfo->TVInfo & TVSetYPbPr525i) ||
-			(pVBInfo->TVInfo & TVSetYPbPr525p))
+		    (pVBInfo->TVInfo & TVSetYPbPr525p))
 			Tap4TimingPtr = xgifb_ntsc_525_tap4_timing;
 		if (pVBInfo->TVInfo & TVSetYPbPr750p)
 			Tap4TimingPtr = YPbPr750pTap4Timing;
@@ -3988,8 +3965,8 @@ static void XGI_SetGroup3(unsigned short ModeIdIndex,
 }
 
 static void XGI_SetGroup4(unsigned short ModeIdIndex,
-		unsigned short RefreshRateTableIndex,
-		struct vb_device_info *pVBInfo)
+			  unsigned short RefreshRateTableIndex,
+			  struct vb_device_info *pVBInfo)
 {
 	unsigned short tempax, tempcx, tempbx, modeflag, temp, temp2;
 
@@ -4080,12 +4057,12 @@ static void XGI_SetGroup4(unsigned short ModeIdIndex,
 	if (templong != 0)
 		tempebx++;
 
-	temp = (unsigned short) (tempebx & 0x000000FF);
+	temp = (unsigned short)(tempebx & 0x000000FF);
 	xgifb_reg_set(pVBInfo->Part4Port, 0x1B, temp);
 
-	temp = (unsigned short) ((tempebx & 0x0000FF00) >> 8);
+	temp = (unsigned short)((tempebx & 0x0000FF00) >> 8);
 	xgifb_reg_set(pVBInfo->Part4Port, 0x1A, temp);
-	tempbx = (unsigned short) (tempebx >> 16);
+	tempbx = (unsigned short)(tempebx >> 16);
 	temp = tempbx & 0x00FF;
 	temp <<= 4;
 	temp |= ((tempcx & 0xFF00) >> 8);
@@ -4132,8 +4109,7 @@ static void XGI_SetGroup4(unsigned short ModeIdIndex,
 					| TVSetHiVision))) {
 				temp |= 0x0001;
 				if ((pVBInfo->VBInfo & SetInSlaveMode) &&
-						!(pVBInfo->TVInfo
-								& TVSimuMode))
+				    !(pVBInfo->TVInfo & TVSimuMode))
 					temp &= (~0x0001);
 			}
 		}
@@ -4174,7 +4150,8 @@ static void XGI_DisableGatingCRT(struct vb_device_info *pVBInfo)
 }
 
 static unsigned char XGI_XG21CheckLVDSMode(struct xgifb_video_info *xgifb_info,
-		unsigned short ModeNo, unsigned short ModeIdIndex)
+					   unsigned short ModeNo,
+					   unsigned short ModeIdIndex)
 {
 	unsigned short xres, yres, colordepth, modeflag, resindex;
 
@@ -4221,7 +4198,7 @@ static void xgifb_set_lvds(struct xgifb_video_info *xgifb_info,
 	unsigned short LVDSVT, LVDSVBS, LVDSVRS, LVDSVRE, LVDSVBE;
 	unsigned short value;
 
-	temp = (unsigned char) ((xgifb_info->lvds_data.LVDS_Capability &
+	temp = (unsigned char)((xgifb_info->lvds_data.LVDS_Capability &
 				(LCDPolarity << 8)) >> 8);
 	temp &= LCDPolarity;
 	Miscdata = inb(pVBInfo->P3cc);
@@ -4354,12 +4331,12 @@ static void xgifb_set_lvds(struct xgifb_video_info *xgifb_info,
 	if (chip_id == XG27) {
 		/* Panel VRS SR35[2:0] SR34[7:0] */
 		xgifb_reg_and_or(pVBInfo->P3c4, 0x35, ~0x07,
-					(value & 0x700) >> 8);
+				 (value & 0x700) >> 8);
 		xgifb_reg_set(pVBInfo->P3c4, 0x34, value & 0xFF);
 	} else {
 		/* Panel VRS SR3F[1:0] SR34[7:0] SR33[0] */
 		xgifb_reg_and_or(pVBInfo->P3c4, 0x3F, ~0x03,
-					(value & 0x600) >> 9);
+				 (value & 0x600) >> 9);
 		xgifb_reg_set(pVBInfo->P3c4, 0x34, (value >> 1) & 0xFF);
 		xgifb_reg_and_or(pVBInfo->P3d4, 0x33, ~0x01, value & 0x01);
 	}
@@ -4372,11 +4349,11 @@ static void xgifb_set_lvds(struct xgifb_video_info *xgifb_info,
 	/* Panel VRE SR3F[7:2] */
 	if (chip_id == XG27)
 		xgifb_reg_and_or(pVBInfo->P3c4, 0x3F, ~0xFC,
-					(value << 2) & 0xFC);
+				 (value << 2) & 0xFC);
 	else
 		/* SR3F[7] has to be 0, h/w bug */
 		xgifb_reg_and_or(pVBInfo->P3c4, 0x3F, ~0xFC,
-					(value << 2) & 0x7C);
+				 (value << 2) & 0x7C);
 
 	for (temp = 0, value = 0; temp < 3; temp++) {
 		xgifb_reg_and_or(pVBInfo->P3c4, 0x31, ~0x30, value);
@@ -4400,13 +4377,13 @@ static void xgifb_set_lvds(struct xgifb_video_info *xgifb_info,
 	}
 }
 
-/* --------------------------------------------------------------------- */
-/* Function : XGI_IsLCDON */
-/* Input : */
-/* Output : 0 : Skip PSC Control */
-/* 1: Disable PSC */
-/* Description : */
-/* --------------------------------------------------------------------- */
+/*
+ * Function : XGI_IsLCDON
+ * Input :
+ * Output : 0 : Skip PSC Control
+ * 1: Disable PSC
+ * Description :
+ */
 static unsigned char XGI_IsLCDON(struct vb_device_info *pVBInfo)
 {
 	unsigned short tempax;
@@ -4421,8 +4398,8 @@ static unsigned char XGI_IsLCDON(struct vb_device_info *pVBInfo)
 }
 
 static void XGI_DisableBridge(struct xgifb_video_info *xgifb_info,
-		struct xgi_hw_device_info *HwDeviceExtension,
-		struct vb_device_info *pVBInfo)
+			      struct xgi_hw_device_info *HwDeviceExtension,
+			      struct vb_device_info *pVBInfo)
 {
 	unsigned short tempah = 0;
 
@@ -4498,23 +4475,23 @@ static void XGI_DisableBridge(struct xgifb_video_info *xgifb_info,
 	}
 }
 
-/* --------------------------------------------------------------------- */
-/* Function : XGI_GetTVPtrIndex */
-/* Input : */
-/* Output : */
-/* Description : bx 0 : ExtNTSC */
-/* 1 : StNTSC */
-/* 2 : ExtPAL */
-/* 3 : StPAL */
-/* 4 : ExtHiTV */
-/* 5 : StHiTV */
-/* 6 : Ext525i */
-/* 7 : St525i */
-/* 8 : Ext525p */
-/* 9 : St525p */
-/* A : Ext750p */
-/* B : St750p */
-/* --------------------------------------------------------------------- */
+/*
+ * Function : XGI_GetTVPtrIndex
+ * Input :
+ * Output :
+ * Description : bx 0 : ExtNTSC
+ * 1 : StNTSC
+ * 2 : ExtPAL
+ * 3 : StPAL
+ * 4 : ExtHiTV
+ * 5 : StHiTV
+ * 6 : Ext525i
+ * 7 : St525i
+ * 8 : Ext525p
+ * 9 : St525p
+ * A : Ext750p
+ * B : St750p
+ */
 static unsigned short XGI_GetTVPtrIndex(struct vb_device_info *pVBInfo)
 {
 	unsigned short tempbx = 0;
@@ -4535,24 +4512,24 @@ static unsigned short XGI_GetTVPtrIndex(struct vb_device_info *pVBInfo)
 	return tempbx;
 }
 
-/* --------------------------------------------------------------------- */
-/* Function : XGI_GetTVPtrIndex2 */
-/* Input : */
-/* Output : bx 0 : NTSC */
-/* 1 : PAL */
-/* 2 : PALM */
-/* 3 : PALN */
-/* 4 : NTSC1024x768 */
-/* 5 : PAL-M 1024x768 */
-/* 6-7: reserved */
-/* cl 0 : YFilter1 */
-/* 1 : YFilter2 */
-/* ch 0 : 301A */
-/* 1 : 301B/302B/301LV/302LV */
-/* Description : */
-/* --------------------------------------------------------------------- */
+/*
+ * Function : XGI_GetTVPtrIndex2
+ * Input :
+ * Output : bx 0 : NTSC
+ * 1 : PAL
+ * 2 : PALM
+ * 3 : PALN
+ * 4 : NTSC1024x768
+ * 5 : PAL-M 1024x768
+ * 6-7: reserved
+ * cl 0 : YFilter1
+ * 1 : YFilter2
+ * ch 0 : 301A
+ * 1 : 301B/302B/301LV/302LV
+ * Description :
+ */
 static void XGI_GetTVPtrIndex2(unsigned short *tempbx, unsigned char *tempcl,
-		unsigned char *tempch, struct vb_device_info *pVBInfo)
+			       unsigned char *tempch, struct vb_device_info *pVBInfo)
 {
 	*tempbx = 0;
 	*tempcl = 0;
@@ -4637,33 +4614,32 @@ static void XGI_SetLCDCap_A(unsigned short tempcx,
 
 	if (temp & LCDRGB18Bit) {
 		xgifb_reg_and_or(pVBInfo->Part1Port, 0x19, 0x0F,
-				/* Enable Dither */
-				(unsigned short) (0x20 | (tempcx & 0x00C0)));
+				 /* Enable Dither */
+				 (unsigned short)(0x20 | (tempcx & 0x00C0)));
 		xgifb_reg_and_or(pVBInfo->Part1Port, 0x1A, 0x7F, 0x80);
 	} else {
 		xgifb_reg_and_or(pVBInfo->Part1Port, 0x19, 0x0F,
-				(unsigned short) (0x30 | (tempcx & 0x00C0)));
+				 (unsigned short)(0x30 | (tempcx & 0x00C0)));
 		xgifb_reg_and_or(pVBInfo->Part1Port, 0x1A, 0x7F, 0x00);
 	}
 }
 
-/* --------------------------------------------------------------------- */
-/* Function : XGI_SetLCDCap_B */
-/* Input : cx -> LCD Capability */
-/* Output : */
-/* Description : */
-/* --------------------------------------------------------------------- */
+/*
+ * Function : XGI_SetLCDCap_B
+ * Input : cx -> LCD Capability
+ * Output :
+ * Description :
+ */
 static void XGI_SetLCDCap_B(unsigned short tempcx,
 			    struct vb_device_info *pVBInfo)
 {
 	if (tempcx & EnableLCD24bpp) /* 24bits */
 		xgifb_reg_and_or(pVBInfo->Part2Port, 0x1A, 0xE0,
-				(unsigned short) (((tempcx & 0x00ff) >> 6)
-						| 0x0c));
+				 (unsigned short)(((tempcx & 0x00ff) >> 6) | 0x0c));
 	else
 		xgifb_reg_and_or(pVBInfo->Part2Port, 0x1A, 0xE0,
-				(unsigned short) (((tempcx & 0x00ff) >> 6)
-						| 0x18)); /* Enable Dither */
+				 (unsigned short)(((tempcx & 0x00ff) >> 6) | 0x18));
+				  /* Enable Dither */
 }
 
 static void XGI_LongWait(struct vb_device_info *pVBInfo)
@@ -4698,13 +4674,13 @@ static void SetSpectrum(struct vb_device_info *pVBInfo)
 	XGI_LongWait(pVBInfo);
 
 	xgifb_reg_set(pVBInfo->Part4Port, 0x31,
-			pVBInfo->LCDCapList[index].Spectrum_31);
+		      pVBInfo->LCDCapList[index].Spectrum_31);
 	xgifb_reg_set(pVBInfo->Part4Port, 0x32,
-			pVBInfo->LCDCapList[index].Spectrum_32);
+		      pVBInfo->LCDCapList[index].Spectrum_32);
 	xgifb_reg_set(pVBInfo->Part4Port, 0x33,
-			pVBInfo->LCDCapList[index].Spectrum_33);
+		      pVBInfo->LCDCapList[index].Spectrum_33);
 	xgifb_reg_set(pVBInfo->Part4Port, 0x34,
-			pVBInfo->LCDCapList[index].Spectrum_34);
+		      pVBInfo->LCDCapList[index].Spectrum_34);
 	XGI_LongWait(pVBInfo);
 	xgifb_reg_or(pVBInfo->Part4Port, 0x30, 0x40); /* enable spectrum */
 }
@@ -4721,13 +4697,13 @@ static void XGI_SetLCDCap(struct vb_device_info *pVBInfo)
 		    (VB_SIS301LV | VB_SIS302LV | VB_XGI301C)) {
 			/* Set 301LV Capability */
 			xgifb_reg_set(pVBInfo->Part4Port, 0x24,
-					(unsigned char) (tempcx & 0x1F));
+				      (unsigned char)(tempcx & 0x1F));
 		}
 		/* VB Driving */
 		xgifb_reg_and_or(pVBInfo->Part4Port, 0x0D,
-				~((EnableVBCLKDRVLOW | EnablePLLSPLOW) >> 8),
-				(unsigned short) ((tempcx & (EnableVBCLKDRVLOW
-						| EnablePLLSPLOW)) >> 8));
+				 ~((EnableVBCLKDRVLOW | EnablePLLSPLOW) >> 8),
+				 (unsigned short)((tempcx & (EnableVBCLKDRVLOW |
+				 EnablePLLSPLOW)) >> 8));
 
 		if (pVBInfo->VBInfo & SetCRT2ToLCD)
 			XGI_SetLCDCap_B(tempcx, pVBInfo);
@@ -4744,12 +4720,12 @@ static void XGI_SetLCDCap(struct vb_device_info *pVBInfo)
 	}
 }
 
-/* --------------------------------------------------------------------- */
-/* Function : XGI_SetAntiFlicker */
-/* Input : */
-/* Output : */
-/* Description : Set TV Customized Param. */
-/* --------------------------------------------------------------------- */
+/*
+ * Function : XGI_SetAntiFlicker
+ * Input :
+ * Output :
+ * Description : Set TV Customized Param.
+ */
 static void XGI_SetAntiFlicker(struct vb_device_info *pVBInfo)
 {
 	unsigned short tempbx;
@@ -4792,13 +4768,13 @@ static void XGI_SetPhaseIncr(struct vb_device_info *pVBInfo)
 	XGI_GetTVPtrIndex2(&tempbx, &tempcl, &tempch, pVBInfo); /* bx, cl, ch */
 	tempData = TVPhaseList[tempbx];
 
-	xgifb_reg_set(pVBInfo->Part2Port, 0x31, (unsigned short) (tempData
+	xgifb_reg_set(pVBInfo->Part2Port, 0x31, (unsigned short)(tempData
 			& 0x000000FF));
-	xgifb_reg_set(pVBInfo->Part2Port, 0x32, (unsigned short) ((tempData
+	xgifb_reg_set(pVBInfo->Part2Port, 0x32, (unsigned short)((tempData
 			& 0x0000FF00) >> 8));
-	xgifb_reg_set(pVBInfo->Part2Port, 0x33, (unsigned short) ((tempData
+	xgifb_reg_set(pVBInfo->Part2Port, 0x33, (unsigned short)((tempData
 			& 0x00FF0000) >> 16));
-	xgifb_reg_set(pVBInfo->Part2Port, 0x34, (unsigned short) ((tempData
+	xgifb_reg_set(pVBInfo->Part2Port, 0x34, (unsigned short)((tempData
 			& 0xFF000000) >> 24));
 }
 
@@ -4866,12 +4842,12 @@ static void XGI_SetYFilter(unsigned short ModeIdIndex,
 	}
 }
 
-/* --------------------------------------------------------------------- */
-/* Function : XGI_OEM310Setting */
-/* Input : */
-/* Output : */
-/* Description : Customized Param. for 301 */
-/* --------------------------------------------------------------------- */
+/*
+ * Function : XGI_OEM310Setting
+ * Input :
+ * Output :
+ * Description : Customized Param. for 301
+ */
 static void XGI_OEM310Setting(unsigned short ModeIdIndex,
 			      struct vb_device_info *pVBInfo)
 {
@@ -4890,12 +4866,12 @@ static void XGI_OEM310Setting(unsigned short ModeIdIndex,
 	}
 }
 
-/* --------------------------------------------------------------------- */
-/* Function : XGI_SetCRT2ModeRegs */
-/* Input : */
-/* Output : */
-/* Description : Origin code for crt2group */
-/* --------------------------------------------------------------------- */
+/*
+ * Function : XGI_SetCRT2ModeRegs
+ * Input :
+ * Output :
+ * Description : Origin code for crt2group
+ */
 static void XGI_SetCRT2ModeRegs(struct vb_device_info *pVBInfo)
 {
 	unsigned short tempbl;
@@ -4999,8 +4975,8 @@ reg_and_or:
 				tempah |= 0x40;
 		}
 
-		if ((pVBInfo->LCDResInfo == Panel_1280x1024)
-				|| (pVBInfo->LCDResInfo == Panel_1280x1024x75))
+		if ((pVBInfo->LCDResInfo == Panel_1280x1024) ||
+		    (pVBInfo->LCDResInfo == Panel_1280x1024x75))
 			tempah |= 0x80;
 
 		if (pVBInfo->LCDResInfo == Panel_1280x960)
@@ -5068,8 +5044,9 @@ void XGI_LockCRT2(struct vb_device_info *pVBInfo)
 }
 
 unsigned short XGI_GetRatePtrCRT2(struct xgi_hw_device_info *pXGIHWDE,
-		unsigned short ModeNo, unsigned short ModeIdIndex,
-		struct vb_device_info *pVBInfo)
+				  unsigned short ModeNo,
+				  unsigned short ModeIdIndex,
+				  struct vb_device_info *pVBInfo)
 {
 	const u8 LCDARefreshIndex[] = {
 		0x00, 0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x00 };
@@ -5143,14 +5120,14 @@ unsigned short XGI_GetRatePtrCRT2(struct xgi_hw_device_info *pXGIHWDE,
 }
 
 static void XGI_SetLCDAGroup(unsigned short ModeNo, unsigned short ModeIdIndex,
-		struct xgi_hw_device_info *HwDeviceExtension,
-		struct vb_device_info *pVBInfo)
+			     struct xgi_hw_device_info *HwDeviceExtension,
+			     struct vb_device_info *pVBInfo)
 {
 	unsigned short RefreshRateTableIndex;
 
 	pVBInfo->SetFlag |= ProgrammingCRT2;
 	RefreshRateTableIndex = XGI_GetRatePtrCRT2(HwDeviceExtension, ModeNo,
-			ModeIdIndex, pVBInfo);
+						   ModeIdIndex, pVBInfo);
 	XGI_GetLVDSResInfo(ModeIdIndex, pVBInfo);
 	XGI_GetLVDSData(ModeIdIndex, pVBInfo);
 	XGI_ModCRT1Regs(ModeIdIndex, HwDeviceExtension, pVBInfo);
@@ -5159,8 +5136,8 @@ static void XGI_SetLCDAGroup(unsigned short ModeNo, unsigned short ModeIdIndex,
 }
 
 static unsigned char XGI_SetCRT2Group301(unsigned short ModeNo,
-		struct xgi_hw_device_info *HwDeviceExtension,
-		struct vb_device_info *pVBInfo)
+					 struct xgi_hw_device_info *HwDeviceExtension,
+					 struct vb_device_info *pVBInfo)
 {
 	unsigned short ModeIdIndex, RefreshRateTableIndex;
 
@@ -5168,7 +5145,7 @@ static unsigned char XGI_SetCRT2Group301(unsigned short ModeNo,
 	XGI_SearchModeID(ModeNo, &ModeIdIndex);
 	pVBInfo->SelectCRT2Rate = 4;
 	RefreshRateTableIndex = XGI_GetRatePtrCRT2(HwDeviceExtension, ModeNo,
-			ModeIdIndex, pVBInfo);
+						   ModeIdIndex, pVBInfo);
 	XGI_SaveCRT2Info(ModeNo, pVBInfo);
 	XGI_GetCRT2ResInfo(ModeIdIndex, pVBInfo);
 	XGI_GetCRT2Data(ModeIdIndex, RefreshRateTableIndex, pVBInfo);
@@ -5210,39 +5187,39 @@ void XGI_SenseCRT1(struct vb_device_info *pVBInfo)
 	CR63 = xgifb_reg_get(pVBInfo->P3d4, 0x63);
 	SR01 = xgifb_reg_get(pVBInfo->P3c4, 0x01);
 
-	xgifb_reg_set(pVBInfo->P3c4, 0x01, (unsigned char) (SR01 & 0xDF));
-	xgifb_reg_set(pVBInfo->P3d4, 0x63, (unsigned char) (CR63 & 0xBF));
+	xgifb_reg_set(pVBInfo->P3c4, 0x01, (unsigned char)(SR01 & 0xDF));
+	xgifb_reg_set(pVBInfo->P3d4, 0x63, (unsigned char)(CR63 & 0xBF));
 
 	CR17 = xgifb_reg_get(pVBInfo->P3d4, 0x17);
-	xgifb_reg_set(pVBInfo->P3d4, 0x17, (unsigned char) (CR17 | 0x80));
+	xgifb_reg_set(pVBInfo->P3d4, 0x17, (unsigned char)(CR17 | 0x80));
 
 	SR1F = xgifb_reg_get(pVBInfo->P3c4, 0x1F);
-	xgifb_reg_set(pVBInfo->P3c4, 0x1F, (unsigned char) (SR1F | 0x04));
+	xgifb_reg_set(pVBInfo->P3c4, 0x1F, (unsigned char)(SR1F | 0x04));
 
 	SR07 = xgifb_reg_get(pVBInfo->P3c4, 0x07);
-	xgifb_reg_set(pVBInfo->P3c4, 0x07, (unsigned char) (SR07 & 0xFB));
+	xgifb_reg_set(pVBInfo->P3c4, 0x07, (unsigned char)(SR07 & 0xFB));
 	SR06 = xgifb_reg_get(pVBInfo->P3c4, 0x06);
-	xgifb_reg_set(pVBInfo->P3c4, 0x06, (unsigned char) (SR06 & 0xC3));
+	xgifb_reg_set(pVBInfo->P3c4, 0x06, (unsigned char)(SR06 & 0xC3));
 
 	xgifb_reg_set(pVBInfo->P3d4, 0x11, 0x00);
 
 	for (i = 0; i < 8; i++)
-		xgifb_reg_set(pVBInfo->P3d4, (unsigned short) i, CRTCData[i]);
+		xgifb_reg_set(pVBInfo->P3d4, (unsigned short)i, CRTCData[i]);
 
 	for (i = 8; i < 11; i++)
-		xgifb_reg_set(pVBInfo->P3d4, (unsigned short) (i + 8),
-				CRTCData[i]);
+		xgifb_reg_set(pVBInfo->P3d4, (unsigned short)(i + 8),
+			      CRTCData[i]);
 
 	for (i = 11; i < 13; i++)
-		xgifb_reg_set(pVBInfo->P3d4, (unsigned short) (i + 4),
-				CRTCData[i]);
+		xgifb_reg_set(pVBInfo->P3d4, (unsigned short)(i + 4),
+			      CRTCData[i]);
 
 	for (i = 13; i < 16; i++)
-		xgifb_reg_set(pVBInfo->P3c4, (unsigned short) (i - 3),
-				CRTCData[i]);
+		xgifb_reg_set(pVBInfo->P3c4, (unsigned short)(i - 3),
+			      CRTCData[i]);
 
-	xgifb_reg_set(pVBInfo->P3c4, 0x0E, (unsigned char) (CRTCData[16]
-			& 0xE0));
+	xgifb_reg_set(pVBInfo->P3c4, 0x0E, (unsigned char)(CRTCData[16]
+		      & 0xE0));
 
 	xgifb_reg_set(pVBInfo->P3c4, 0x31, 0x00);
 	xgifb_reg_set(pVBInfo->P3c4, 0x2B, 0x1B);
@@ -5275,12 +5252,12 @@ void XGI_SenseCRT1(struct vb_device_info *pVBInfo)
 
 	xgifb_reg_set(pVBInfo->P3d4, 0x53, (xgifb_reg_get(
 			pVBInfo->P3d4, 0x53) & 0xFD));
-	xgifb_reg_set(pVBInfo->P3c4, 0x1F, (unsigned char) SR1F);
+	xgifb_reg_set(pVBInfo->P3c4, 0x1F, (unsigned char)SR1F);
 }
 
 static void XGI_EnableBridge(struct xgifb_video_info *xgifb_info,
-		struct xgi_hw_device_info *HwDeviceExtension,
-		struct vb_device_info *pVBInfo)
+			     struct xgi_hw_device_info *HwDeviceExtension,
+			     struct vb_device_info *pVBInfo)
 {
 	unsigned short tempah;
 
@@ -5310,11 +5287,11 @@ static void XGI_EnableBridge(struct xgifb_video_info *xgifb_info,
 
 		if (!(pVBInfo->VBInfo & DisableCRT2Display)) {
 			xgifb_reg_and_or(pVBInfo->Part2Port, 0x00, ~0xE0,
-					0x20); /* shampoo 0129 */
+					 0x20); /* shampoo 0129 */
 			if (pVBInfo->VBType & (VB_SIS302LV | VB_XGI301C)) {
 				if (pVBInfo->VBInfo &
 					(SetCRT2ToLCD | XGI_SetCRT2ToLCDA))
-					/* LVDS PLL power on */
+					 /* LVDS PLL power on */
 					xgifb_reg_and(pVBInfo->Part4Port, 0x2A,
 						      0x7F);
 				/* LVDS Driver power on */
@@ -5358,9 +5335,9 @@ static void XGI_EnableBridge(struct xgifb_video_info *xgifb_info,
 }
 
 static void XGI_SetCRT1Group(struct xgifb_video_info *xgifb_info,
-		struct xgi_hw_device_info *HwDeviceExtension,
-		unsigned short ModeNo, unsigned short ModeIdIndex,
-		struct vb_device_info *pVBInfo)
+			     struct xgi_hw_device_info *HwDeviceExtension,
+			     unsigned short ModeNo, unsigned short ModeIdIndex,
+			     struct vb_device_info *pVBInfo)
 {
 	unsigned short RefreshRateTableIndex, temp;
 
@@ -5389,14 +5366,14 @@ static void XGI_SetCRT1Group(struct xgifb_video_info *xgifb_info,
 	}
 
 	RefreshRateTableIndex = XGI_GetRatePtrCRT2(HwDeviceExtension, ModeNo,
-			ModeIdIndex, pVBInfo);
+						   ModeIdIndex, pVBInfo);
 	if (RefreshRateTableIndex != 0xFFFF) {
 		XGI_SetSync(RefreshRateTableIndex, pVBInfo);
 		XGI_SetCRT1CRTC(ModeIdIndex, RefreshRateTableIndex,
 				pVBInfo, HwDeviceExtension);
 		XGI_SetCRT1DE(ModeIdIndex, RefreshRateTableIndex, pVBInfo);
 		XGI_SetCRT1Offset(ModeNo, ModeIdIndex, RefreshRateTableIndex,
-				HwDeviceExtension, pVBInfo);
+				  HwDeviceExtension, pVBInfo);
 		XGI_SetCRT1VCLK(ModeIdIndex, HwDeviceExtension,
 				RefreshRateTableIndex, pVBInfo);
 	}
@@ -5410,15 +5387,15 @@ static void XGI_SetCRT1Group(struct xgifb_video_info *xgifb_info,
 				XGI_SetXG21CRTC(RefreshRateTableIndex, pVBInfo);
 
 			XGI_UpdateXG21CRTC(ModeNo, pVBInfo,
-					RefreshRateTableIndex);
+					   RefreshRateTableIndex);
 
 			xgifb_set_lcd(HwDeviceExtension->jChipType,
 				      pVBInfo, RefreshRateTableIndex);
 
 			if (pVBInfo->IF_DEF_LVDS == 1)
 				xgifb_set_lvds(xgifb_info,
-						HwDeviceExtension->jChipType,
-						ModeIdIndex, pVBInfo);
+					       HwDeviceExtension->jChipType,
+					       ModeIdIndex, pVBInfo);
 		}
 	}
 
@@ -5430,8 +5407,8 @@ static void XGI_SetCRT1Group(struct xgifb_video_info *xgifb_info,
 }
 
 unsigned char XGISetModeNew(struct xgifb_video_info *xgifb_info,
-			struct xgi_hw_device_info *HwDeviceExtension,
-			unsigned short ModeNo)
+			    struct xgi_hw_device_info *HwDeviceExtension,
+			    unsigned short ModeNo)
 {
 	unsigned short ModeIdIndex;
 	struct vb_device_info VBINF;
@@ -5440,7 +5417,7 @@ unsigned char XGISetModeNew(struct xgifb_video_info *xgifb_info,
 	pVBInfo->IF_DEF_LVDS = 0;
 
 	if (HwDeviceExtension->jChipType >= XG20)
-		pVBInfo->VBType = 0; /*set VBType default 0*/
+		pVBInfo->VBType = 0; /* set VBType default 0 */
 
 	XGIRegInit(pVBInfo, xgifb_info->vga_base);
 
@@ -5473,13 +5450,13 @@ unsigned char XGISetModeNew(struct xgifb_video_info *xgifb_info,
 		XGI_DisableBridge(xgifb_info, HwDeviceExtension, pVBInfo);
 
 		if (pVBInfo->VBInfo & (SetSimuScanMode | XGI_SetCRT2ToLCDA) ||
-			!(pVBInfo->VBInfo & SwitchCRT2)) {
+		    !(pVBInfo->VBInfo & SwitchCRT2)) {
 			XGI_SetCRT1Group(xgifb_info, HwDeviceExtension, ModeNo,
-					ModeIdIndex, pVBInfo);
+					 ModeIdIndex, pVBInfo);
 
 			if (pVBInfo->VBInfo & XGI_SetCRT2ToLCDA) {
 				XGI_SetLCDAGroup(ModeNo, ModeIdIndex,
-						HwDeviceExtension, pVBInfo);
+						 HwDeviceExtension, pVBInfo);
 			}
 		}
 
@@ -5488,7 +5465,7 @@ unsigned char XGISetModeNew(struct xgifb_video_info *xgifb_info,
 			case VB_CHIP_301: /* fall through */
 			case VB_CHIP_302:
 				XGI_SetCRT2Group301(ModeNo, HwDeviceExtension,
-						pVBInfo); /*add for CRT2 */
+						    pVBInfo); /* add for CRT2 */
 				break;
 
 			default:
@@ -5497,7 +5474,7 @@ unsigned char XGISetModeNew(struct xgifb_video_info *xgifb_info,
 		}
 
 		XGI_SetCRT2ModeRegs(pVBInfo);
-		XGI_OEM310Setting(ModeIdIndex, pVBInfo); /*0212*/
+		XGI_OEM310Setting(ModeIdIndex, pVBInfo); /* 0212 */
 		XGI_EnableBridge(xgifb_info, HwDeviceExtension, pVBInfo);
 	} /* !XG20 */
 	else {
@@ -5515,7 +5492,7 @@ unsigned char XGISetModeNew(struct xgifb_video_info *xgifb_info,
 		XGI_DisplayOff(xgifb_info, HwDeviceExtension, pVBInfo);
 
 		XGI_SetCRT1Group(xgifb_info, HwDeviceExtension, ModeNo,
-				ModeIdIndex, pVBInfo);
+				 ModeIdIndex, pVBInfo);
 
 		XGI_DisplayOn(xgifb_info, HwDeviceExtension, pVBInfo);
 	}
diff --git a/drivers/staging/xgifb/vb_table.h b/drivers/staging/xgifb/vb_table.h
index c801deb142f6..f9f98e06e6d5 100644
--- a/drivers/staging/xgifb/vb_table.h
+++ b/drivers/staging/xgifb/vb_table.h
@@ -1701,6 +1701,7 @@ static const struct XGI_LVDSCRT1VDataStruct XGI_LVDSCRT11280x1024_1_Vx75[] = {
 	{ {0x28, 0xF5, 0x00, 0x84, 0xFF, 0x29, 0x90} },/* ; 04 (x768) */
 	{ {0x28, 0x5A, 0x13, 0x87, 0xFF, 0x29, 0xA9} } /* ; 05 (x1024) */
 };
+
 /* CR00,CR02,CR03,CR04,CR05,SR0B,SR0C,SR0E */
 static const struct XGI_LVDSCRT1HDataStruct XGI_LVDSCRT11280x1024_2_Hx75[] = {
 	{ {0x7E, 0x3B, 0x9A, 0x44, 0x12, 0x00, 0x01, 0x00} },/* ; 00 (320x) */
@@ -1886,17 +1887,17 @@ static const struct XGI330_LCDCapStruct XGI_LCDDLCapList[] = {
 	0x6C, 0xC3, 0x35, 0x62,
 	0x0A, 0xC0, 0x28, 0x10},
 /* LCDCap1280x1024 */
-	{Panel_1280x1024, XGI_LCDDualLink+DefaultLCDCap,
+	{Panel_1280x1024, XGI_LCDDualLink + DefaultLCDCap,
 	0x70, 0x03, VCLK108_2_315,
 	0x70, 0x44, 0xF8, 0x2F,
 	0x0A, 0xC0, 0x30, 0x10},
 /* LCDCap1400x1050 */
-	{Panel_1400x1050, XGI_LCDDualLink+DefaultLCDCap,
+	{Panel_1400x1050, XGI_LCDDualLink + DefaultLCDCap,
 	0x70, 0x03, VCLK108_2_315,
 	 0x70, 0x44, 0xF8, 0x2F,
 	 0x0A, 0xC0, 0x30, 0x10},
 /* LCDCap1600x1200 */
-	{Panel_1600x1200, XGI_LCDDualLink+DefaultLCDCap,
+	{Panel_1600x1200, XGI_LCDDualLink + DefaultLCDCap,
 	0xC0, 0x03, VCLK162,
 	 0x43, 0x22, 0x70, 0x24,
 	 0x0A, 0xC0, 0x30, 0x10},
@@ -1905,7 +1906,7 @@ static const struct XGI330_LCDCapStruct XGI_LCDDLCapList[] = {
 	 0x2B, 0x61, 0x2B, 0x61,
 	 0x0A, 0xC0, 0x28, 0x10},
 /* LCDCap1280x1024x75 */
-	{Panel_1280x1024x75, XGI_LCDDualLink+DefaultLCDCap,
+	{Panel_1280x1024x75, XGI_LCDDualLink + DefaultLCDCap,
 	 0x90, 0x03, VCLK135_5,
 	 0x54, 0x42, 0x4A, 0x61,
 	 0x0A, 0xC0, 0x30, 0x10},
diff --git a/drivers/staging/xgifb/vb_util.h b/drivers/staging/xgifb/vb_util.h
index 08db58b396b2..052694e75053 100644
--- a/drivers/staging/xgifb/vb_util.h
+++ b/drivers/staging/xgifb/vb_util.h
@@ -18,7 +18,7 @@ static inline void xgifb_reg_and_or(unsigned long port, u8 index,
 	u8 temp;
 
 	temp = xgifb_reg_get(port, index);
-	temp = (u8) ((temp & data_and) | data_or);
+	temp = (u8)((temp & data_and) | data_or);
 	xgifb_reg_set(port, index, temp);
 }
 
@@ -28,7 +28,7 @@ static inline void xgifb_reg_and(unsigned long port, u8 index,
 	u8 temp;
 
 	temp = xgifb_reg_get(port, index);
-	temp = (u8) (temp & data_and);
+	temp = (u8)(temp & data_and);
 	xgifb_reg_set(port, index, temp);
 }
 
diff --git a/drivers/target/target_core_fabric_configfs.c b/drivers/target/target_core_fabric_configfs.c
index 31a096aa16ab..d8a16ca6baa5 100644
--- a/drivers/target/target_core_fabric_configfs.c
+++ b/drivers/target/target_core_fabric_configfs.c
@@ -137,7 +137,7 @@ static int target_fabric_mappedlun_link(
 	return core_dev_add_initiator_node_lun_acl(se_tpg, lacl, lun, lun_access_ro);
 }
 
-static int target_fabric_mappedlun_unlink(
+static void target_fabric_mappedlun_unlink(
 	struct config_item *lun_acl_ci,
 	struct config_item *lun_ci)
 {
@@ -146,7 +146,7 @@ static int target_fabric_mappedlun_unlink(
 	struct se_lun *lun = container_of(to_config_group(lun_ci),
 			struct se_lun, lun_group);
 
-	return core_dev_del_initiator_node_lun_acl(lun, lacl);
+	core_dev_del_initiator_node_lun_acl(lun, lacl);
 }
 
 static struct se_lun_acl *item_to_lun_acl(struct config_item *item)
@@ -669,7 +669,7 @@ out:
 	return ret;
 }
 
-static int target_fabric_port_unlink(
+static void target_fabric_port_unlink(
 	struct config_item *lun_ci,
 	struct config_item *se_dev_ci)
 {
@@ -688,7 +688,6 @@ static int target_fabric_port_unlink(
 	}
 
 	core_dev_del_lun(se_tpg, lun);
-	return 0;
 }
 
 static void target_fabric_port_release(struct config_item *item)
diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c
index 372d744315f3..d316ed537d59 100644
--- a/drivers/target/target_core_iblock.c
+++ b/drivers/target/target_core_iblock.c
@@ -388,7 +388,7 @@ iblock_execute_sync_cache(struct se_cmd *cmd)
 	bio = bio_alloc(GFP_KERNEL, 0);
 	bio->bi_end_io = iblock_end_io_flush;
 	bio->bi_bdev = ib_dev->ibd_bd;
-	bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_FLUSH);
+	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 	if (!immed)
 		bio->bi_private = cmd;
 	submit_bio(bio);
@@ -686,15 +686,15 @@ iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
 		struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
 		struct request_queue *q = bdev_get_queue(ib_dev->ibd_bd);
 		/*
-		 * Force writethrough using WRITE_FUA if a volatile write cache
+		 * Force writethrough using REQ_FUA if a volatile write cache
 		 * is not enabled, or if initiator set the Force Unit Access bit.
 		 */
 		op = REQ_OP_WRITE;
 		if (test_bit(QUEUE_FLAG_FUA, &q->queue_flags)) {
 			if (cmd->se_cmd_flags & SCF_FUA)
-				op_flags = WRITE_FUA;
+				op_flags = REQ_FUA;
 			else if (!test_bit(QUEUE_FLAG_WC, &q->queue_flags))
-				op_flags = WRITE_FUA;
+				op_flags = REQ_FUA;
 		}
 	} else {
 		op = REQ_OP_READ;
diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c
index 9125d9358dea..04d7aa7390d0 100644
--- a/drivers/target/target_core_pscsi.c
+++ b/drivers/target/target_core_pscsi.c
@@ -935,13 +935,9 @@ pscsi_map_sg(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
 
 			rc = bio_add_pc_page(pdv->pdv_sd->request_queue,
 					bio, page, bytes, off);
-			if (rc != bytes)
-				goto fail;
-
 			pr_debug("PSCSI: bio->bi_vcnt: %d nr_vecs: %d\n",
-				bio->bi_vcnt, nr_vecs);
-
-			if (bio->bi_vcnt > nr_vecs) {
+				bio_segments(bio), nr_vecs);
+			if (rc != bytes) {
 				pr_debug("PSCSI: Reached bio->bi_vcnt max:"
 					" %d i: %d bio: %p, allocating another"
 					" bio\n", bio->bi_vcnt, i, bio);
diff --git a/drivers/target/tcm_fc/tfc_cmd.c b/drivers/target/tcm_fc/tfc_cmd.c
index ff5de9a96643..9af7842b8178 100644
--- a/drivers/target/tcm_fc/tfc_cmd.c
+++ b/drivers/target/tcm_fc/tfc_cmd.c
@@ -92,7 +92,7 @@ static void ft_free_cmd(struct ft_cmd *cmd)
 	fp = cmd->req_frame;
 	lport = fr_dev(fp);
 	if (fr_seq(fp))
-		lport->tt.seq_release(fr_seq(fp));
+		fc_seq_release(fr_seq(fp));
 	fc_frame_free(fp);
 	percpu_ida_free(&sess->se_sess->sess_tag_pool, cmd->se_cmd.map_tag);
 	ft_sess_put(sess);	/* undo get from lookup at recv */
@@ -161,11 +161,11 @@ int ft_queue_status(struct se_cmd *se_cmd)
 	/*
 	 * Send response.
 	 */
-	cmd->seq = lport->tt.seq_start_next(cmd->seq);
+	cmd->seq = fc_seq_start_next(cmd->seq);
 	fc_fill_fc_hdr(fp, FC_RCTL_DD_CMD_STATUS, ep->did, ep->sid, FC_TYPE_FCP,
 		       FC_FC_EX_CTX | FC_FC_LAST_SEQ | FC_FC_END_SEQ, 0);
 
-	rc = lport->tt.seq_send(lport, cmd->seq, fp);
+	rc = fc_seq_send(lport, cmd->seq, fp);
 	if (rc) {
 		pr_info_ratelimited("%s: Failed to send response frame %p, "
 				    "xid <0x%x>\n", __func__, fp, ep->xid);
@@ -177,7 +177,7 @@ int ft_queue_status(struct se_cmd *se_cmd)
 		se_cmd->scsi_status = SAM_STAT_TASK_SET_FULL;
 		return -ENOMEM;
 	}
-	lport->tt.exch_done(cmd->seq);
+	fc_exch_done(cmd->seq);
 	/*
 	 * Drop the extra ACK_KREF reference taken by target_submit_cmd()
 	 * ahead of ft_check_stop_free() -> transport_generic_free_cmd()
@@ -221,7 +221,7 @@ int ft_write_pending(struct se_cmd *se_cmd)
 	memset(txrdy, 0, sizeof(*txrdy));
 	txrdy->ft_burst_len = htonl(se_cmd->data_length);
 
-	cmd->seq = lport->tt.seq_start_next(cmd->seq);
+	cmd->seq = fc_seq_start_next(cmd->seq);
 	fc_fill_fc_hdr(fp, FC_RCTL_DD_DATA_DESC, ep->did, ep->sid, FC_TYPE_FCP,
 		       FC_FC_EX_CTX | FC_FC_END_SEQ | FC_FC_SEQ_INIT, 0);
 
@@ -242,7 +242,7 @@ int ft_write_pending(struct se_cmd *se_cmd)
 				cmd->was_ddp_setup = 1;
 		}
 	}
-	lport->tt.seq_send(lport, cmd->seq, fp);
+	fc_seq_send(lport, cmd->seq, fp);
 	return 0;
 }
 
@@ -323,8 +323,8 @@ static void ft_send_resp_status(struct fc_lport *lport,
 	fc_fill_reply_hdr(fp, rx_fp, FC_RCTL_DD_CMD_STATUS, 0);
 	sp = fr_seq(fp);
 	if (sp) {
-		lport->tt.seq_send(lport, sp, fp);
-		lport->tt.exch_done(sp);
+		fc_seq_send(lport, sp, fp);
+		fc_exch_done(sp);
 	} else {
 		lport->tt.frame_send(lport, fp);
 	}
@@ -461,7 +461,7 @@ static void ft_recv_cmd(struct ft_sess *sess, struct fc_frame *fp)
 
 	cmd->se_cmd.map_tag = tag;
 	cmd->sess = sess;
-	cmd->seq = lport->tt.seq_assign(lport, fp);
+	cmd->seq = fc_seq_assign(lport, fp);
 	if (!cmd->seq) {
 		percpu_ida_free(&se_sess->sess_tag_pool, tag);
 		goto busy;
@@ -563,7 +563,7 @@ static void ft_send_work(struct work_struct *work)
 		task_attr = TCM_SIMPLE_TAG;
 	}
 
-	fc_seq_exch(cmd->seq)->lp->tt.seq_set_resp(cmd->seq, ft_recv_seq, cmd);
+	fc_seq_set_resp(cmd->seq, ft_recv_seq, cmd);
 	cmd->se_cmd.tag = fc_seq_exch(cmd->seq)->rxid;
 	/*
 	 * Use a single se_cmd->cmd_kref as we expect to release se_cmd
diff --git a/drivers/target/tcm_fc/tfc_io.c b/drivers/target/tcm_fc/tfc_io.c
index 6f7c65abfe2a..1eb1f58e00e4 100644
--- a/drivers/target/tcm_fc/tfc_io.c
+++ b/drivers/target/tcm_fc/tfc_io.c
@@ -82,7 +82,7 @@ int ft_queue_data_in(struct se_cmd *se_cmd)
 
 	ep = fc_seq_exch(cmd->seq);
 	lport = ep->lp;
-	cmd->seq = lport->tt.seq_start_next(cmd->seq);
+	cmd->seq = fc_seq_start_next(cmd->seq);
 
 	remaining = se_cmd->data_length;
 
@@ -174,7 +174,7 @@ int ft_queue_data_in(struct se_cmd *se_cmd)
 			f_ctl |= FC_FC_END_SEQ;
 		fc_fill_fc_hdr(fp, FC_RCTL_DD_SOL_DATA, ep->did, ep->sid,
 			       FC_TYPE_FCP, f_ctl, fh_off);
-		error = lport->tt.seq_send(lport, seq, fp);
+		error = fc_seq_send(lport, seq, fp);
 		if (error) {
 			pr_info_ratelimited("%s: Failed to send frame %p, "
 						"xid <0x%x>, remaining %zu, "
diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig
index a13541bdc726..c2c056cc7ea5 100644
--- a/drivers/thermal/Kconfig
+++ b/drivers/thermal/Kconfig
@@ -177,8 +177,10 @@ config THERMAL_EMULATION
 
 config HISI_THERMAL
 	tristate "Hisilicon thermal driver"
-	depends on (ARCH_HISI && CPU_THERMAL && OF) || COMPILE_TEST
+	depends on ARCH_HISI || COMPILE_TEST
 	depends on HAS_IOMEM
+	depends on OF
+	default y
 	help
 	  Enable this to plug hisilicon's thermal sensor driver into the Linux
 	  thermal framework. cpufreq is used as the cooling device to throttle
diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile
index c92eb22a41ff..6a3d7b573036 100644
--- a/drivers/thermal/Makefile
+++ b/drivers/thermal/Makefile
@@ -3,7 +3,8 @@
 #
 
 obj-$(CONFIG_THERMAL)		+= thermal_sys.o
-thermal_sys-y			+= thermal_core.o
+thermal_sys-y			+= thermal_core.o thermal_sysfs.o \
+					thermal_helpers.o
 
 # interface to/from other layers providing sensors
 thermal_sys-$(CONFIG_THERMAL_HWMON)		+= thermal_hwmon.o
diff --git a/drivers/thermal/db8500_thermal.c b/drivers/thermal/db8500_thermal.c
index e776cea80cfc..f491faf16592 100644
--- a/drivers/thermal/db8500_thermal.c
+++ b/drivers/thermal/db8500_thermal.c
@@ -512,6 +512,7 @@ static const struct of_device_id db8500_thermal_match[] = {
 	{ .compatible = "stericsson,db8500-thermal" },
 	{},
 };
+MODULE_DEVICE_TABLE(of, db8500_thermal_match);
 #endif
 
 static struct platform_driver db8500_thermal_driver = {
diff --git a/drivers/thermal/devfreq_cooling.c b/drivers/thermal/devfreq_cooling.c
index 81631b110e17..5a737fd5f1aa 100644
--- a/drivers/thermal/devfreq_cooling.c
+++ b/drivers/thermal/devfreq_cooling.c
@@ -238,7 +238,7 @@ get_static_power(struct devfreq_cooling_device *dfc, unsigned long freq)
 		return 0;
 	}
 
-	return dfc->power_ops->get_static_power(voltage);
+	return dfc->power_ops->get_static_power(df, voltage);
 }
 
 /**
@@ -262,7 +262,8 @@ get_dynamic_power(struct devfreq_cooling_device *dfc, unsigned long freq,
 	struct devfreq_cooling_power *dfc_power = dfc->power_ops;
 
 	if (dfc_power->get_dynamic_power)
-		return dfc_power->get_dynamic_power(freq, voltage);
+		return dfc_power->get_dynamic_power(dfc->devfreq, freq,
+						    voltage);
 
 	freq_mhz = freq / 1000000;
 	power = (u64)dfc_power->dyn_power_coeff * freq_mhz * voltage * voltage;
diff --git a/drivers/thermal/int340x_thermal/int3400_thermal.c b/drivers/thermal/int340x_thermal/int3400_thermal.c
index 5836e5554433..9413c4abf0b9 100644
--- a/drivers/thermal/int340x_thermal/int3400_thermal.c
+++ b/drivers/thermal/int340x_thermal/int3400_thermal.c
@@ -96,7 +96,7 @@ static ssize_t current_uuid_store(struct device *dev,
 	return -EINVAL;
 }
 
-static DEVICE_ATTR(current_uuid, 0644, current_uuid_show, current_uuid_store);
+static DEVICE_ATTR_RW(current_uuid);
 static DEVICE_ATTR_RO(available_uuids);
 static struct attribute *uuid_attrs[] = {
 	&dev_attr_available_uuids.attr,
diff --git a/drivers/thermal/intel_pch_thermal.c b/drivers/thermal/intel_pch_thermal.c
index 19bf2028e508..2b49e8d0fe9e 100644
--- a/drivers/thermal/intel_pch_thermal.c
+++ b/drivers/thermal/intel_pch_thermal.c
@@ -29,6 +29,7 @@
 #define PCH_THERMAL_DID_HSW_2	0x8C24 /* Haswell PCH */
 #define PCH_THERMAL_DID_WPT	0x9CA4 /* Wildcat Point */
 #define PCH_THERMAL_DID_SKL	0x9D31 /* Skylake PCH */
+#define PCH_THERMAL_DID_SKL_H	0xA131 /* Skylake PCH 100 series */
 
 /* Wildcat Point-LP  PCH Thermal registers */
 #define WPT_TEMP	0x0000	/* Temperature */
@@ -273,37 +274,44 @@ static struct thermal_zone_device_ops tzd_ops = {
 	.get_trip_temp = pch_get_trip_temp,
 };
 
+enum board_ids {
+	board_hsw,
+	board_wpt,
+	board_skl,
+};
+
+static const struct board_info {
+	const char *name;
+	const struct pch_dev_ops *ops;
+} board_info[] = {
+	[board_hsw] = {
+		.name = "pch_haswell",
+		.ops = &pch_dev_ops_wpt,
+	},
+	[board_wpt] = {
+		.name = "pch_wildcat_point",
+		.ops = &pch_dev_ops_wpt,
+	},
+	[board_skl] = {
+		.name = "pch_skylake",
+		.ops = &pch_dev_ops_wpt,
+	},
+};
 
 static int intel_pch_thermal_probe(struct pci_dev *pdev,
 				   const struct pci_device_id *id)
 {
+	enum board_ids board_id = id->driver_data;
+	const struct board_info *bi = &board_info[board_id];
 	struct pch_thermal_device *ptd;
 	int err;
 	int nr_trips;
-	char *dev_name;
 
 	ptd = devm_kzalloc(&pdev->dev, sizeof(*ptd), GFP_KERNEL);
 	if (!ptd)
 		return -ENOMEM;
 
-	switch (pdev->device) {
-	case PCH_THERMAL_DID_WPT:
-		ptd->ops = &pch_dev_ops_wpt;
-		dev_name = "pch_wildcat_point";
-		break;
-	case PCH_THERMAL_DID_SKL:
-		ptd->ops = &pch_dev_ops_wpt;
-		dev_name = "pch_skylake";
-		break;
-	case PCH_THERMAL_DID_HSW_1:
-	case PCH_THERMAL_DID_HSW_2:
-		ptd->ops = &pch_dev_ops_wpt;
-		dev_name = "pch_haswell";
-		break;
-	default:
-		dev_err(&pdev->dev, "unknown pch thermal device\n");
-		return -ENODEV;
-	}
+	ptd->ops = bi->ops;
 
 	pci_set_drvdata(pdev, ptd);
 	ptd->pdev = pdev;
@@ -331,11 +339,11 @@ static int intel_pch_thermal_probe(struct pci_dev *pdev,
 	if (err)
 		goto error_cleanup;
 
-	ptd->tzd = thermal_zone_device_register(dev_name, nr_trips, 0, ptd,
+	ptd->tzd = thermal_zone_device_register(bi->name, nr_trips, 0, ptd,
 						&tzd_ops, NULL, 0, 0);
 	if (IS_ERR(ptd->tzd)) {
 		dev_err(&pdev->dev, "Failed to register thermal zone %s\n",
-			dev_name);
+			bi->name);
 		err = PTR_ERR(ptd->tzd);
 		goto error_cleanup;
 	}
@@ -380,10 +388,16 @@ static int intel_pch_thermal_resume(struct device *device)
 }
 
 static struct pci_device_id intel_pch_thermal_id[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_WPT) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_SKL) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_HSW_1) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_HSW_2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_HSW_1),
+		.driver_data = board_hsw, },
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_HSW_2),
+		.driver_data = board_hsw, },
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_WPT),
+		.driver_data = board_wpt, },
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_SKL),
+		.driver_data = board_skl, },
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_SKL_H),
+		.driver_data = board_skl, },
 	{ 0, },
 };
 MODULE_DEVICE_TABLE(pci, intel_pch_thermal_id);
diff --git a/drivers/thermal/intel_powerclamp.c b/drivers/thermal/intel_powerclamp.c
index 350cb5e22ff3..df64692e9e64 100644
--- a/drivers/thermal/intel_powerclamp.c
+++ b/drivers/thermal/intel_powerclamp.c
@@ -43,7 +43,6 @@
 #include <linux/kernel.h>
 #include <linux/delay.h>
 #include <linux/kthread.h>
-#include <linux/freezer.h>
 #include <linux/cpu.h>
 #include <linux/thermal.h>
 #include <linux/slab.h>
@@ -85,11 +84,26 @@ static unsigned int control_cpu; /* The cpu assigned to collect stat and update
 				  */
 static bool clamping;
 
+static const struct sched_param sparam = {
+	.sched_priority = MAX_USER_RT_PRIO / 2,
+};
+struct powerclamp_worker_data {
+	struct kthread_worker *worker;
+	struct kthread_work balancing_work;
+	struct kthread_delayed_work idle_injection_work;
+	unsigned int cpu;
+	unsigned int count;
+	unsigned int guard;
+	unsigned int window_size_now;
+	unsigned int target_ratio;
+	unsigned int duration_jiffies;
+	bool clamping;
+};
 
-static struct task_struct * __percpu *powerclamp_thread;
+static struct powerclamp_worker_data * __percpu worker_data;
 static struct thermal_cooling_device *cooling_dev;
 static unsigned long *cpu_clamping_mask;  /* bit map for tracking per cpu
-					   * clamping thread
+					   * clamping kthread worker
 					   */
 
 static unsigned int duration;
@@ -261,11 +275,6 @@ static u64 pkg_state_counter(void)
 	return count;
 }
 
-static void noop_timer(unsigned long foo)
-{
-	/* empty... just the fact that we get the interrupt wakes us up */
-}
-
 static unsigned int get_compensation(int ratio)
 {
 	unsigned int comp = 0;
@@ -367,103 +376,79 @@ static bool powerclamp_adjust_controls(unsigned int target_ratio,
 	return set_target_ratio + guard <= current_ratio;
 }
 
-static int clamp_thread(void *arg)
+static void clamp_balancing_func(struct kthread_work *work)
 {
-	int cpunr = (unsigned long)arg;
-	DEFINE_TIMER(wakeup_timer, noop_timer, 0, 0);
-	static const struct sched_param param = {
-		.sched_priority = MAX_USER_RT_PRIO/2,
-	};
-	unsigned int count = 0;
-	unsigned int target_ratio;
+	struct powerclamp_worker_data *w_data;
+	int sleeptime;
+	unsigned long target_jiffies;
+	unsigned int compensated_ratio;
+	int interval; /* jiffies to sleep for each attempt */
 
-	set_bit(cpunr, cpu_clamping_mask);
-	set_freezable();
-	init_timer_on_stack(&wakeup_timer);
-	sched_setscheduler(current, SCHED_FIFO, &param);
-
-	while (true == clamping && !kthread_should_stop() &&
-		cpu_online(cpunr)) {
-		int sleeptime;
-		unsigned long target_jiffies;
-		unsigned int guard;
-		unsigned int compensated_ratio;
-		int interval; /* jiffies to sleep for each attempt */
-		unsigned int duration_jiffies = msecs_to_jiffies(duration);
-		unsigned int window_size_now;
-
-		try_to_freeze();
-		/*
-		 * make sure user selected ratio does not take effect until
-		 * the next round. adjust target_ratio if user has changed
-		 * target such that we can converge quickly.
-		 */
-		target_ratio = set_target_ratio;
-		guard = 1 + target_ratio/20;
-		window_size_now = window_size;
-		count++;
-
-		/*
-		 * systems may have different ability to enter package level
-		 * c-states, thus we need to compensate the injected idle ratio
-		 * to achieve the actual target reported by the HW.
-		 */
-		compensated_ratio = target_ratio +
-			get_compensation(target_ratio);
-		if (compensated_ratio <= 0)
-			compensated_ratio = 1;
-		interval = duration_jiffies * 100 / compensated_ratio;
-
-		/* align idle time */
-		target_jiffies = roundup(jiffies, interval);
-		sleeptime = target_jiffies - jiffies;
-		if (sleeptime <= 0)
-			sleeptime = 1;
-		schedule_timeout_interruptible(sleeptime);
-		/*
-		 * only elected controlling cpu can collect stats and update
-		 * control parameters.
-		 */
-		if (cpunr == control_cpu && !(count%window_size_now)) {
-			should_skip =
-				powerclamp_adjust_controls(target_ratio,
-							guard, window_size_now);
-			smp_mb();
-		}
+	w_data = container_of(work, struct powerclamp_worker_data,
+			      balancing_work);
 
-		if (should_skip)
-			continue;
-
-		target_jiffies = jiffies + duration_jiffies;
-		mod_timer(&wakeup_timer, target_jiffies);
-		if (unlikely(local_softirq_pending()))
-			continue;
-		/*
-		 * stop tick sched during idle time, interrupts are still
-		 * allowed. thus jiffies are updated properly.
-		 */
-		preempt_disable();
-		/* mwait until target jiffies is reached */
-		while (time_before(jiffies, target_jiffies)) {
-			unsigned long ecx = 1;
-			unsigned long eax = target_mwait;
-
-			/*
-			 * REVISIT: may call enter_idle() to notify drivers who
-			 * can save power during cpu idle. same for exit_idle()
-			 */
-			local_touch_nmi();
-			stop_critical_timings();
-			mwait_idle_with_hints(eax, ecx);
-			start_critical_timings();
-			atomic_inc(&idle_wakeup_counter);
-		}
-		preempt_enable();
+	/*
+	 * make sure user selected ratio does not take effect until
+	 * the next round. adjust target_ratio if user has changed
+	 * target such that we can converge quickly.
+	 */
+	w_data->target_ratio = READ_ONCE(set_target_ratio);
+	w_data->guard = 1 + w_data->target_ratio / 20;
+	w_data->window_size_now = window_size;
+	w_data->duration_jiffies = msecs_to_jiffies(duration);
+	w_data->count++;
+
+	/*
+	 * systems may have different ability to enter package level
+	 * c-states, thus we need to compensate the injected idle ratio
+	 * to achieve the actual target reported by the HW.
+	 */
+	compensated_ratio = w_data->target_ratio +
+		get_compensation(w_data->target_ratio);
+	if (compensated_ratio <= 0)
+		compensated_ratio = 1;
+	interval = w_data->duration_jiffies * 100 / compensated_ratio;
+
+	/* align idle time */
+	target_jiffies = roundup(jiffies, interval);
+	sleeptime = target_jiffies - jiffies;
+	if (sleeptime <= 0)
+		sleeptime = 1;
+
+	if (clamping && w_data->clamping && cpu_online(w_data->cpu))
+		kthread_queue_delayed_work(w_data->worker,
+					   &w_data->idle_injection_work,
+					   sleeptime);
+}
+
+static void clamp_idle_injection_func(struct kthread_work *work)
+{
+	struct powerclamp_worker_data *w_data;
+
+	w_data = container_of(work, struct powerclamp_worker_data,
+			      idle_injection_work.work);
+
+	/*
+	 * only elected controlling cpu can collect stats and update
+	 * control parameters.
+	 */
+	if (w_data->cpu == control_cpu &&
+	    !(w_data->count % w_data->window_size_now)) {
+		should_skip =
+			powerclamp_adjust_controls(w_data->target_ratio,
+						   w_data->guard,
+						   w_data->window_size_now);
+		smp_mb();
 	}
-	del_timer_sync(&wakeup_timer);
-	clear_bit(cpunr, cpu_clamping_mask);
 
-	return 0;
+	if (should_skip)
+		goto balance;
+
+	play_idle(jiffies_to_msecs(w_data->duration_jiffies));
+
+balance:
+	if (clamping && w_data->clamping && cpu_online(w_data->cpu))
+		kthread_queue_work(w_data->worker, &w_data->balancing_work);
 }
 
 /*
@@ -507,10 +492,60 @@ static void poll_pkg_cstate(struct work_struct *dummy)
 		schedule_delayed_work(&poll_pkg_cstate_work, HZ);
 }
 
+static void start_power_clamp_worker(unsigned long cpu)
+{
+	struct powerclamp_worker_data *w_data = per_cpu_ptr(worker_data, cpu);
+	struct kthread_worker *worker;
+
+	worker = kthread_create_worker_on_cpu(cpu, 0, "kidle_inject/%ld", cpu);
+	if (IS_ERR(worker))
+		return;
+
+	w_data->worker = worker;
+	w_data->count = 0;
+	w_data->cpu = cpu;
+	w_data->clamping = true;
+	set_bit(cpu, cpu_clamping_mask);
+	sched_setscheduler(worker->task, SCHED_FIFO, &sparam);
+	kthread_init_work(&w_data->balancing_work, clamp_balancing_func);
+	kthread_init_delayed_work(&w_data->idle_injection_work,
+				  clamp_idle_injection_func);
+	kthread_queue_work(w_data->worker, &w_data->balancing_work);
+}
+
+static void stop_power_clamp_worker(unsigned long cpu)
+{
+	struct powerclamp_worker_data *w_data = per_cpu_ptr(worker_data, cpu);
+
+	if (!w_data->worker)
+		return;
+
+	w_data->clamping = false;
+	/*
+	 * Make sure that all works that get queued after this point see
+	 * the clamping disabled. The counter part is not needed because
+	 * there is an implicit memory barrier when the queued work
+	 * is proceed.
+	 */
+	smp_wmb();
+	kthread_cancel_work_sync(&w_data->balancing_work);
+	kthread_cancel_delayed_work_sync(&w_data->idle_injection_work);
+	/*
+	 * The balancing work still might be queued here because
+	 * the handling of the "clapming" variable, cancel, and queue
+	 * operations are not synchronized via a lock. But it is not
+	 * a big deal. The balancing work is fast and destroy kthread
+	 * will wait for it.
+	 */
+	clear_bit(w_data->cpu, cpu_clamping_mask);
+	kthread_destroy_worker(w_data->worker);
+
+	w_data->worker = NULL;
+}
+
 static int start_power_clamp(void)
 {
 	unsigned long cpu;
-	struct task_struct *thread;
 
 	set_target_ratio = clamp(set_target_ratio, 0U, MAX_TARGET_RATIO - 1);
 	/* prevent cpu hotplug */
@@ -524,22 +559,9 @@ static int start_power_clamp(void)
 	clamping = true;
 	schedule_delayed_work(&poll_pkg_cstate_work, 0);
 
-	/* start one thread per online cpu */
+	/* start one kthread worker per online cpu */
 	for_each_online_cpu(cpu) {
-		struct task_struct **p =
-			per_cpu_ptr(powerclamp_thread, cpu);
-
-		thread = kthread_create_on_node(clamp_thread,
-						(void *) cpu,
-						cpu_to_node(cpu),
-						"kidle_inject/%ld", cpu);
-		/* bind to cpu here */
-		if (likely(!IS_ERR(thread))) {
-			kthread_bind(thread, cpu);
-			wake_up_process(thread);
-			*p = thread;
-		}
-
+		start_power_clamp_worker(cpu);
 	}
 	put_online_cpus();
 
@@ -549,71 +571,49 @@ static int start_power_clamp(void)
 static void end_power_clamp(void)
 {
 	int i;
-	struct task_struct *thread;
 
-	clamping = false;
 	/*
-	 * make clamping visible to other cpus and give per cpu clamping threads
-	 * sometime to exit, or gets killed later.
+	 * Block requeuing in all the kthread workers. They will flush and
+	 * stop faster.
 	 */
-	smp_mb();
-	msleep(20);
+	clamping = false;
 	if (bitmap_weight(cpu_clamping_mask, num_possible_cpus())) {
 		for_each_set_bit(i, cpu_clamping_mask, num_possible_cpus()) {
-			pr_debug("clamping thread for cpu %d alive, kill\n", i);
-			thread = *per_cpu_ptr(powerclamp_thread, i);
-			kthread_stop(thread);
+			pr_debug("clamping worker for cpu %d alive, destroy\n",
+				 i);
+			stop_power_clamp_worker(i);
 		}
 	}
 }
 
-static int powerclamp_cpu_callback(struct notifier_block *nfb,
-				unsigned long action, void *hcpu)
+static int powerclamp_cpu_online(unsigned int cpu)
 {
-	unsigned long cpu = (unsigned long)hcpu;
-	struct task_struct *thread;
-	struct task_struct **percpu_thread =
-		per_cpu_ptr(powerclamp_thread, cpu);
-
-	if (false == clamping)
-		goto exit_ok;
-
-	switch (action) {
-	case CPU_ONLINE:
-		thread = kthread_create_on_node(clamp_thread,
-						(void *) cpu,
-						cpu_to_node(cpu),
-						"kidle_inject/%lu", cpu);
-		if (likely(!IS_ERR(thread))) {
-			kthread_bind(thread, cpu);
-			wake_up_process(thread);
-			*percpu_thread = thread;
-		}
-		/* prefer BSP as controlling CPU */
-		if (cpu == 0) {
-			control_cpu = 0;
-			smp_mb();
-		}
-		break;
-	case CPU_DEAD:
-		if (test_bit(cpu, cpu_clamping_mask)) {
-			pr_err("cpu %lu dead but powerclamping thread is not\n",
-				cpu);
-			kthread_stop(*percpu_thread);
-		}
-		if (cpu == control_cpu) {
-			control_cpu = smp_processor_id();
-			smp_mb();
-		}
+	if (clamping == false)
+		return 0;
+	start_power_clamp_worker(cpu);
+	/* prefer BSP as controlling CPU */
+	if (cpu == 0) {
+		control_cpu = 0;
+		smp_mb();
 	}
-
-exit_ok:
-	return NOTIFY_OK;
+	return 0;
 }
 
-static struct notifier_block powerclamp_cpu_notifier = {
-	.notifier_call = powerclamp_cpu_callback,
-};
+static int powerclamp_cpu_predown(unsigned int cpu)
+{
+	if (clamping == false)
+		return 0;
+
+	stop_power_clamp_worker(cpu);
+	if (cpu != control_cpu)
+		return 0;
+
+	control_cpu = cpumask_first(cpu_online_mask);
+	if (control_cpu == cpu)
+		control_cpu = cpumask_next(cpu, cpu_online_mask);
+	smp_mb();
+	return 0;
+}
 
 static int powerclamp_get_max_state(struct thermal_cooling_device *cdev,
 				 unsigned long *state)
@@ -741,6 +741,8 @@ file_error:
 	debugfs_remove_recursive(debug_dir);
 }
 
+static enum cpuhp_state hp_state;
+
 static int __init powerclamp_init(void)
 {
 	int retval;
@@ -758,10 +760,17 @@ static int __init powerclamp_init(void)
 
 	/* set default limit, maybe adjusted during runtime based on feedback */
 	window_size = 2;
-	register_hotcpu_notifier(&powerclamp_cpu_notifier);
+	retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					   "thermal/intel_powerclamp:online",
+					   powerclamp_cpu_online,
+					   powerclamp_cpu_predown);
+	if (retval < 0)
+		goto exit_free;
+
+	hp_state = retval;
 
-	powerclamp_thread = alloc_percpu(struct task_struct *);
-	if (!powerclamp_thread) {
+	worker_data = alloc_percpu(struct powerclamp_worker_data);
+	if (!worker_data) {
 		retval = -ENOMEM;
 		goto exit_unregister;
 	}
@@ -781,9 +790,9 @@ static int __init powerclamp_init(void)
 	return 0;
 
 exit_free_thread:
-	free_percpu(powerclamp_thread);
+	free_percpu(worker_data);
 exit_unregister:
-	unregister_hotcpu_notifier(&powerclamp_cpu_notifier);
+	cpuhp_remove_state_nocalls(hp_state);
 exit_free:
 	kfree(cpu_clamping_mask);
 	return retval;
@@ -792,9 +801,9 @@ module_init(powerclamp_init);
 
 static void __exit powerclamp_exit(void)
 {
-	unregister_hotcpu_notifier(&powerclamp_cpu_notifier);
 	end_power_clamp();
-	free_percpu(powerclamp_thread);
+	cpuhp_remove_state_nocalls(hp_state);
+	free_percpu(worker_data);
 	thermal_cooling_device_unregister(cooling_dev);
 	kfree(cpu_clamping_mask);
 
diff --git a/drivers/thermal/max77620_thermal.c b/drivers/thermal/max77620_thermal.c
index 83905ff46e40..e9a1fe342760 100644
--- a/drivers/thermal/max77620_thermal.c
+++ b/drivers/thermal/max77620_thermal.c
@@ -149,6 +149,7 @@ static struct platform_device_id max77620_thermal_devtype[] = {
 	{ .name = "max77620-thermal", },
 	{},
 };
+MODULE_DEVICE_TABLE(platform, max77620_thermal_devtype);
 
 static struct platform_driver max77620_thermal_driver = {
 	.driver = {
diff --git a/drivers/thermal/qcom-spmi-temp-alarm.c b/drivers/thermal/qcom-spmi-temp-alarm.c
index 819c6d5d7aa7..f50241962ad2 100644
--- a/drivers/thermal/qcom-spmi-temp-alarm.c
+++ b/drivers/thermal/qcom-spmi-temp-alarm.c
@@ -200,7 +200,7 @@ static int qpnp_tm_probe(struct platform_device *pdev)
 	struct qpnp_tm_chip *chip;
 	struct device_node *node;
 	u8 type, subtype;
-	u32 res[2];
+	u32 res;
 	int ret, irq;
 
 	node = pdev->dev.of_node;
@@ -215,7 +215,7 @@ static int qpnp_tm_probe(struct platform_device *pdev)
 	if (!chip->map)
 		return -ENXIO;
 
-	ret = of_property_read_u32_array(node, "reg", res, 2);
+	ret = of_property_read_u32(node, "reg", &res);
 	if (ret < 0)
 		return ret;
 
@@ -228,7 +228,7 @@ static int qpnp_tm_probe(struct platform_device *pdev)
 	if (PTR_ERR(chip->adc) == -EPROBE_DEFER)
 		return PTR_ERR(chip->adc);
 
-	chip->base = res[0];
+	chip->base = res;
 
 	ret = qpnp_tm_read(chip, QPNP_TM_REG_TYPE, &type);
 	if (ret < 0) {
diff --git a/drivers/thermal/rockchip_thermal.c b/drivers/thermal/rockchip_thermal.c
index e227a9f0acf7..b811b0fb61b1 100644
--- a/drivers/thermal/rockchip_thermal.c
+++ b/drivers/thermal/rockchip_thermal.c
@@ -524,11 +524,6 @@ static void rk_tsadcv2_initialize(struct regmap *grf, void __iomem *regs,
 		       regs + TSADCV2_AUTO_PERIOD_HT);
 	writel_relaxed(TSADCV2_HIGHT_TSHUT_DEBOUNCE_COUNT,
 		       regs + TSADCV2_HIGHT_TSHUT_DEBOUNCE);
-
-	if (IS_ERR(grf)) {
-		pr_warn("%s: Missing rockchip,grf property\n", __func__);
-		return;
-	}
 }
 
 /**
@@ -971,6 +966,8 @@ static int rockchip_configure_from_dt(struct device *dev,
 	 * need this property.
 	 */
 	thermal->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
+	if (IS_ERR(thermal->grf))
+		dev_warn(dev, "Missing rockchip,grf property\n");
 
 	return 0;
 }
diff --git a/drivers/thermal/tango_thermal.c b/drivers/thermal/tango_thermal.c
index 201304aeafeb..4e67795cb6ce 100644
--- a/drivers/thermal/tango_thermal.c
+++ b/drivers/thermal/tango_thermal.c
@@ -107,6 +107,7 @@ static const struct of_device_id tango_sensor_ids[] = {
 	},
 	{ /* sentinel */ }
 };
+MODULE_DEVICE_TABLE(of, tango_sensor_ids);
 
 static struct platform_driver tango_thermal_driver = {
 	.probe	= tango_thermal_probe,
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index 911fd964c742..641faab6e24b 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -5,22 +5,9 @@
  *  Copyright (C) 2008 Zhang Rui <rui.zhang@intel.com>
  *  Copyright (C) 2008 Sujith Thomas <sujith.thomas@intel.com>
  *
- *  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
  *  the Free Software Foundation; version 2 of the License.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License along
- *  with this program; if not, write to the Free Software Foundation, Inc.,
- *  59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
- *
- * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -64,6 +51,13 @@ static atomic_t in_suspend;
 
 static struct thermal_governor *def_governor;
 
+/*
+ * Governor section: set of functions to handle thermal governors
+ *
+ * Functions to help in the life cycle of thermal governors within
+ * the thermal core and by the thermal governor code.
+ */
+
 static struct thermal_governor *__find_governor(const char *name)
 {
 	struct thermal_governor *pos;
@@ -142,11 +136,16 @@ int thermal_register_governor(struct thermal_governor *governor)
 	mutex_lock(&thermal_governor_lock);
 
 	err = -EBUSY;
-	if (__find_governor(governor->name) == NULL) {
+	if (!__find_governor(governor->name)) {
+		bool match_default;
+
 		err = 0;
 		list_add(&governor->governor_list, &thermal_governor_list);
-		if (!def_governor && !strncmp(governor->name,
-			DEFAULT_THERMAL_GOVERNOR, THERMAL_NAME_LENGTH))
+		match_default = !strncmp(governor->name,
+					 DEFAULT_THERMAL_GOVERNOR,
+					 THERMAL_NAME_LENGTH);
+
+		if (!def_governor && match_default)
 			def_governor = governor;
 	}
 
@@ -188,14 +187,14 @@ void thermal_unregister_governor(struct thermal_governor *governor)
 
 	mutex_lock(&thermal_governor_lock);
 
-	if (__find_governor(governor->name) == NULL)
+	if (!__find_governor(governor->name))
 		goto exit;
 
 	mutex_lock(&thermal_list_lock);
 
 	list_for_each_entry(pos, &thermal_tz_list, node) {
 		if (!strncasecmp(pos->governor->name, governor->name,
-						THERMAL_NAME_LENGTH))
+				 THERMAL_NAME_LENGTH))
 			thermal_set_governor(pos, NULL);
 	}
 
@@ -203,195 +202,92 @@ void thermal_unregister_governor(struct thermal_governor *governor)
 	list_del(&governor->governor_list);
 exit:
 	mutex_unlock(&thermal_governor_lock);
-	return;
 }
 
-static int get_idr(struct idr *idr, struct mutex *lock, int *id)
+int thermal_zone_device_set_policy(struct thermal_zone_device *tz,
+				   char *policy)
 {
-	int ret;
-
-	if (lock)
-		mutex_lock(lock);
-	ret = idr_alloc(idr, NULL, 0, 0, GFP_KERNEL);
-	if (lock)
-		mutex_unlock(lock);
-	if (unlikely(ret < 0))
-		return ret;
-	*id = ret;
-	return 0;
-}
-
-static void release_idr(struct idr *idr, struct mutex *lock, int id)
-{
-	if (lock)
-		mutex_lock(lock);
-	idr_remove(idr, id);
-	if (lock)
-		mutex_unlock(lock);
-}
-
-int get_tz_trend(struct thermal_zone_device *tz, int trip)
-{
-	enum thermal_trend trend;
-
-	if (tz->emul_temperature || !tz->ops->get_trend ||
-	    tz->ops->get_trend(tz, trip, &trend)) {
-		if (tz->temperature > tz->last_temperature)
-			trend = THERMAL_TREND_RAISING;
-		else if (tz->temperature < tz->last_temperature)
-			trend = THERMAL_TREND_DROPPING;
-		else
-			trend = THERMAL_TREND_STABLE;
-	}
-
-	return trend;
-}
-EXPORT_SYMBOL(get_tz_trend);
-
-struct thermal_instance *get_thermal_instance(struct thermal_zone_device *tz,
-			struct thermal_cooling_device *cdev, int trip)
-{
-	struct thermal_instance *pos = NULL;
-	struct thermal_instance *target_instance = NULL;
+	struct thermal_governor *gov;
+	int ret = -EINVAL;
 
+	mutex_lock(&thermal_governor_lock);
 	mutex_lock(&tz->lock);
-	mutex_lock(&cdev->lock);
 
-	list_for_each_entry(pos, &tz->thermal_instances, tz_node) {
-		if (pos->tz == tz && pos->trip == trip && pos->cdev == cdev) {
-			target_instance = pos;
-			break;
-		}
-	}
+	gov = __find_governor(strim(policy));
+	if (!gov)
+		goto exit;
 
-	mutex_unlock(&cdev->lock);
-	mutex_unlock(&tz->lock);
+	ret = thermal_set_governor(tz, gov);
 
-	return target_instance;
-}
-EXPORT_SYMBOL(get_thermal_instance);
+exit:
+	mutex_unlock(&tz->lock);
+	mutex_unlock(&thermal_governor_lock);
 
-static void print_bind_err_msg(struct thermal_zone_device *tz,
-			struct thermal_cooling_device *cdev, int ret)
-{
-	dev_err(&tz->device, "binding zone %s with cdev %s failed:%d\n",
-				tz->type, cdev->type, ret);
+	return ret;
 }
 
-static void __bind(struct thermal_zone_device *tz, int mask,
-			struct thermal_cooling_device *cdev,
-			unsigned long *limits,
-			unsigned int weight)
+int thermal_build_list_of_policies(char *buf)
 {
-	int i, ret;
+	struct thermal_governor *pos;
+	ssize_t count = 0;
+	ssize_t size = PAGE_SIZE;
 
-	for (i = 0; i < tz->trips; i++) {
-		if (mask & (1 << i)) {
-			unsigned long upper, lower;
+	mutex_lock(&thermal_governor_lock);
 
-			upper = THERMAL_NO_LIMIT;
-			lower = THERMAL_NO_LIMIT;
-			if (limits) {
-				lower = limits[i * 2];
-				upper = limits[i * 2 + 1];
-			}
-			ret = thermal_zone_bind_cooling_device(tz, i, cdev,
-							       upper, lower,
-							       weight);
-			if (ret)
-				print_bind_err_msg(tz, cdev, ret);
-		}
+	list_for_each_entry(pos, &thermal_governor_list, governor_list) {
+		size = PAGE_SIZE - count;
+		count += scnprintf(buf + count, size, "%s ", pos->name);
 	}
-}
+	count += scnprintf(buf + count, size, "\n");
 
-static void __unbind(struct thermal_zone_device *tz, int mask,
-			struct thermal_cooling_device *cdev)
-{
-	int i;
+	mutex_unlock(&thermal_governor_lock);
 
-	for (i = 0; i < tz->trips; i++)
-		if (mask & (1 << i))
-			thermal_zone_unbind_cooling_device(tz, i, cdev);
+	return count;
 }
 
-static void bind_cdev(struct thermal_cooling_device *cdev)
+static int __init thermal_register_governors(void)
 {
-	int i, ret;
-	const struct thermal_zone_params *tzp;
-	struct thermal_zone_device *pos = NULL;
-
-	mutex_lock(&thermal_list_lock);
+	int result;
 
-	list_for_each_entry(pos, &thermal_tz_list, node) {
-		if (!pos->tzp && !pos->ops->bind)
-			continue;
+	result = thermal_gov_step_wise_register();
+	if (result)
+		return result;
 
-		if (pos->ops->bind) {
-			ret = pos->ops->bind(pos, cdev);
-			if (ret)
-				print_bind_err_msg(pos, cdev, ret);
-			continue;
-		}
+	result = thermal_gov_fair_share_register();
+	if (result)
+		return result;
 
-		tzp = pos->tzp;
-		if (!tzp || !tzp->tbp)
-			continue;
+	result = thermal_gov_bang_bang_register();
+	if (result)
+		return result;
 
-		for (i = 0; i < tzp->num_tbps; i++) {
-			if (tzp->tbp[i].cdev || !tzp->tbp[i].match)
-				continue;
-			if (tzp->tbp[i].match(pos, cdev))
-				continue;
-			tzp->tbp[i].cdev = cdev;
-			__bind(pos, tzp->tbp[i].trip_mask, cdev,
-			       tzp->tbp[i].binding_limits,
-			       tzp->tbp[i].weight);
-		}
-	}
+	result = thermal_gov_user_space_register();
+	if (result)
+		return result;
 
-	mutex_unlock(&thermal_list_lock);
+	return thermal_gov_power_allocator_register();
 }
 
-static void bind_tz(struct thermal_zone_device *tz)
+static void thermal_unregister_governors(void)
 {
-	int i, ret;
-	struct thermal_cooling_device *pos = NULL;
-	const struct thermal_zone_params *tzp = tz->tzp;
-
-	if (!tzp && !tz->ops->bind)
-		return;
-
-	mutex_lock(&thermal_list_lock);
-
-	/* If there is ops->bind, try to use ops->bind */
-	if (tz->ops->bind) {
-		list_for_each_entry(pos, &thermal_cdev_list, node) {
-			ret = tz->ops->bind(tz, pos);
-			if (ret)
-				print_bind_err_msg(tz, pos, ret);
-		}
-		goto exit;
-	}
-
-	if (!tzp || !tzp->tbp)
-		goto exit;
-
-	list_for_each_entry(pos, &thermal_cdev_list, node) {
-		for (i = 0; i < tzp->num_tbps; i++) {
-			if (tzp->tbp[i].cdev || !tzp->tbp[i].match)
-				continue;
-			if (tzp->tbp[i].match(tz, pos))
-				continue;
-			tzp->tbp[i].cdev = pos;
-			__bind(tz, tzp->tbp[i].trip_mask, pos,
-			       tzp->tbp[i].binding_limits,
-			       tzp->tbp[i].weight);
-		}
-	}
-exit:
-	mutex_unlock(&thermal_list_lock);
+	thermal_gov_step_wise_unregister();
+	thermal_gov_fair_share_unregister();
+	thermal_gov_bang_bang_unregister();
+	thermal_gov_user_space_unregister();
+	thermal_gov_power_allocator_unregister();
 }
 
+/*
+ * Zone update section: main control loop applied to each zone while monitoring
+ *
+ * in polling mode. The monitoring is done using a workqueue.
+ * Same update may be done on a zone by calling thermal_zone_device_update().
+ *
+ * An update means:
+ * - Non-critical trips will invoke the governor responsible for that zone;
+ * - Hot trips will produce a notification to userspace;
+ * - Critical trip point will cause a system shutdown.
+ */
 static void thermal_zone_device_set_polling(struct thermal_zone_device *tz,
 					    int delay)
 {
@@ -420,14 +316,15 @@ static void monitor_thermal_zone(struct thermal_zone_device *tz)
 }
 
 static void handle_non_critical_trips(struct thermal_zone_device *tz,
-			int trip, enum thermal_trip_type trip_type)
+				      int trip,
+				      enum thermal_trip_type trip_type)
 {
 	tz->governor ? tz->governor->throttle(tz, trip) :
 		       def_governor->throttle(tz, trip);
 }
 
 static void handle_critical_trips(struct thermal_zone_device *tz,
-				int trip, enum thermal_trip_type trip_type)
+				  int trip, enum thermal_trip_type trip_type)
 {
 	int trip_temp;
 
@@ -471,105 +368,6 @@ static void handle_thermal_trip(struct thermal_zone_device *tz, int trip)
 	monitor_thermal_zone(tz);
 }
 
-/**
- * thermal_zone_get_temp() - returns the temperature of a thermal zone
- * @tz: a valid pointer to a struct thermal_zone_device
- * @temp: a valid pointer to where to store the resulting temperature.
- *
- * When a valid thermal zone reference is passed, it will fetch its
- * temperature and fill @temp.
- *
- * Return: On success returns 0, an error code otherwise
- */
-int thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp)
-{
-	int ret = -EINVAL;
-	int count;
-	int crit_temp = INT_MAX;
-	enum thermal_trip_type type;
-
-	if (!tz || IS_ERR(tz) || !tz->ops->get_temp)
-		goto exit;
-
-	mutex_lock(&tz->lock);
-
-	ret = tz->ops->get_temp(tz, temp);
-
-	if (IS_ENABLED(CONFIG_THERMAL_EMULATION) && tz->emul_temperature) {
-		for (count = 0; count < tz->trips; count++) {
-			ret = tz->ops->get_trip_type(tz, count, &type);
-			if (!ret && type == THERMAL_TRIP_CRITICAL) {
-				ret = tz->ops->get_trip_temp(tz, count,
-						&crit_temp);
-				break;
-			}
-		}
-
-		/*
-		 * Only allow emulating a temperature when the real temperature
-		 * is below the critical temperature so that the emulation code
-		 * cannot hide critical conditions.
-		 */
-		if (!ret && *temp < crit_temp)
-			*temp = tz->emul_temperature;
-	}
- 
-	mutex_unlock(&tz->lock);
-exit:
-	return ret;
-}
-EXPORT_SYMBOL_GPL(thermal_zone_get_temp);
-
-void thermal_zone_set_trips(struct thermal_zone_device *tz)
-{
-	int low = -INT_MAX;
-	int high = INT_MAX;
-	int trip_temp, hysteresis;
-	int i, ret;
-
-	mutex_lock(&tz->lock);
-
-	if (!tz->ops->set_trips || !tz->ops->get_trip_hyst)
-		goto exit;
-
-	for (i = 0; i < tz->trips; i++) {
-		int trip_low;
-
-		tz->ops->get_trip_temp(tz, i, &trip_temp);
-		tz->ops->get_trip_hyst(tz, i, &hysteresis);
-
-		trip_low = trip_temp - hysteresis;
-
-		if (trip_low < tz->temperature && trip_low > low)
-			low = trip_low;
-
-		if (trip_temp > tz->temperature && trip_temp < high)
-			high = trip_temp;
-	}
-
-	/* No need to change trip points */
-	if (tz->prev_low_trip == low && tz->prev_high_trip == high)
-		goto exit;
-
-	tz->prev_low_trip = low;
-	tz->prev_high_trip = high;
-
-	dev_dbg(&tz->device,
-		"new temperature boundaries: %d < x < %d\n", low, high);
-
-	/*
-	 * Set a temperature window. When this window is left the driver
-	 * must inform the thermal core via thermal_zone_device_update.
-	 */
-	ret = tz->ops->set_trips(tz, low, high);
-	if (ret)
-		dev_err(&tz->device, "Failed to set trips: %d\n", ret);
-
-exit:
-	mutex_unlock(&tz->lock);
-}
-EXPORT_SYMBOL_GPL(thermal_zone_set_trips);
-
 static void update_temperature(struct thermal_zone_device *tz)
 {
 	int temp, ret;
@@ -629,6 +427,24 @@ void thermal_zone_device_update(struct thermal_zone_device *tz,
 }
 EXPORT_SYMBOL_GPL(thermal_zone_device_update);
 
+/**
+ * thermal_notify_framework - Sensor drivers use this API to notify framework
+ * @tz:		thermal zone device
+ * @trip:	indicates which trip point has been crossed
+ *
+ * This function handles the trip events from sensor drivers. It starts
+ * throttling the cooling devices according to the policy configured.
+ * For CRITICAL and HOT trip points, this notifies the respective drivers,
+ * and does actual throttling for other trip points i.e ACTIVE and PASSIVE.
+ * The throttling policy is based on the configured platform data; if no
+ * platform data is provided, this uses the step_wise throttling policy.
+ */
+void thermal_notify_framework(struct thermal_zone_device *tz, int trip)
+{
+	handle_thermal_trip(tz, trip);
+}
+EXPORT_SYMBOL_GPL(thermal_notify_framework);
+
 static void thermal_zone_device_check(struct work_struct *work)
 {
 	struct thermal_zone_device *tz = container_of(work, struct
@@ -637,445 +453,12 @@ static void thermal_zone_device_check(struct work_struct *work)
 	thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
 }
 
-/* sys I/F for thermal zone */
-
-#define to_thermal_zone(_dev) \
-	container_of(_dev, struct thermal_zone_device, device)
-
-static ssize_t
-type_show(struct device *dev, struct device_attribute *attr, char *buf)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-
-	return sprintf(buf, "%s\n", tz->type);
-}
-
-static ssize_t
-temp_show(struct device *dev, struct device_attribute *attr, char *buf)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	int temperature, ret;
-
-	ret = thermal_zone_get_temp(tz, &temperature);
-
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%d\n", temperature);
-}
-
-static ssize_t
-mode_show(struct device *dev, struct device_attribute *attr, char *buf)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	enum thermal_device_mode mode;
-	int result;
-
-	if (!tz->ops->get_mode)
-		return -EPERM;
-
-	result = tz->ops->get_mode(tz, &mode);
-	if (result)
-		return result;
-
-	return sprintf(buf, "%s\n", mode == THERMAL_DEVICE_ENABLED ? "enabled"
-		       : "disabled");
-}
-
-static ssize_t
-mode_store(struct device *dev, struct device_attribute *attr,
-	   const char *buf, size_t count)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	int result;
-
-	if (!tz->ops->set_mode)
-		return -EPERM;
-
-	if (!strncmp(buf, "enabled", sizeof("enabled") - 1))
-		result = tz->ops->set_mode(tz, THERMAL_DEVICE_ENABLED);
-	else if (!strncmp(buf, "disabled", sizeof("disabled") - 1))
-		result = tz->ops->set_mode(tz, THERMAL_DEVICE_DISABLED);
-	else
-		result = -EINVAL;
-
-	if (result)
-		return result;
-
-	return count;
-}
-
-static ssize_t
-trip_point_type_show(struct device *dev, struct device_attribute *attr,
-		     char *buf)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	enum thermal_trip_type type;
-	int trip, result;
-
-	if (!tz->ops->get_trip_type)
-		return -EPERM;
-
-	if (!sscanf(attr->attr.name, "trip_point_%d_type", &trip))
-		return -EINVAL;
-
-	result = tz->ops->get_trip_type(tz, trip, &type);
-	if (result)
-		return result;
-
-	switch (type) {
-	case THERMAL_TRIP_CRITICAL:
-		return sprintf(buf, "critical\n");
-	case THERMAL_TRIP_HOT:
-		return sprintf(buf, "hot\n");
-	case THERMAL_TRIP_PASSIVE:
-		return sprintf(buf, "passive\n");
-	case THERMAL_TRIP_ACTIVE:
-		return sprintf(buf, "active\n");
-	default:
-		return sprintf(buf, "unknown\n");
-	}
-}
-
-static ssize_t
-trip_point_temp_store(struct device *dev, struct device_attribute *attr,
-		     const char *buf, size_t count)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	int trip, ret;
-	int temperature;
-
-	if (!tz->ops->set_trip_temp)
-		return -EPERM;
-
-	if (!sscanf(attr->attr.name, "trip_point_%d_temp", &trip))
-		return -EINVAL;
-
-	if (kstrtoint(buf, 10, &temperature))
-		return -EINVAL;
-
-	ret = tz->ops->set_trip_temp(tz, trip, temperature);
-	if (ret)
-		return ret;
-
-	thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
-
-	return count;
-}
-
-static ssize_t
-trip_point_temp_show(struct device *dev, struct device_attribute *attr,
-		     char *buf)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	int trip, ret;
-	int temperature;
-
-	if (!tz->ops->get_trip_temp)
-		return -EPERM;
-
-	if (!sscanf(attr->attr.name, "trip_point_%d_temp", &trip))
-		return -EINVAL;
-
-	ret = tz->ops->get_trip_temp(tz, trip, &temperature);
-
-	if (ret)
-		return ret;
-
-	return sprintf(buf, "%d\n", temperature);
-}
-
-static ssize_t
-trip_point_hyst_store(struct device *dev, struct device_attribute *attr,
-			const char *buf, size_t count)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	int trip, ret;
-	int temperature;
-
-	if (!tz->ops->set_trip_hyst)
-		return -EPERM;
-
-	if (!sscanf(attr->attr.name, "trip_point_%d_hyst", &trip))
-		return -EINVAL;
-
-	if (kstrtoint(buf, 10, &temperature))
-		return -EINVAL;
-
-	/*
-	 * We are not doing any check on the 'temperature' value
-	 * here. The driver implementing 'set_trip_hyst' has to
-	 * take care of this.
-	 */
-	ret = tz->ops->set_trip_hyst(tz, trip, temperature);
-
-	if (!ret)
-		thermal_zone_set_trips(tz);
-
-	return ret ? ret : count;
-}
-
-static ssize_t
-trip_point_hyst_show(struct device *dev, struct device_attribute *attr,
-			char *buf)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	int trip, ret;
-	int temperature;
-
-	if (!tz->ops->get_trip_hyst)
-		return -EPERM;
-
-	if (!sscanf(attr->attr.name, "trip_point_%d_hyst", &trip))
-		return -EINVAL;
-
-	ret = tz->ops->get_trip_hyst(tz, trip, &temperature);
-
-	return ret ? ret : sprintf(buf, "%d\n", temperature);
-}
-
-static ssize_t
-passive_store(struct device *dev, struct device_attribute *attr,
-		    const char *buf, size_t count)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	struct thermal_cooling_device *cdev = NULL;
-	int state;
-
-	if (!sscanf(buf, "%d\n", &state))
-		return -EINVAL;
-
-	/* sanity check: values below 1000 millicelcius don't make sense
-	 * and can cause the system to go into a thermal heart attack
-	 */
-	if (state && state < 1000)
-		return -EINVAL;
-
-	if (state && !tz->forced_passive) {
-		mutex_lock(&thermal_list_lock);
-		list_for_each_entry(cdev, &thermal_cdev_list, node) {
-			if (!strncmp("Processor", cdev->type,
-				     sizeof("Processor")))
-				thermal_zone_bind_cooling_device(tz,
-						THERMAL_TRIPS_NONE, cdev,
-						THERMAL_NO_LIMIT,
-						THERMAL_NO_LIMIT,
-						THERMAL_WEIGHT_DEFAULT);
-		}
-		mutex_unlock(&thermal_list_lock);
-		if (!tz->passive_delay)
-			tz->passive_delay = 1000;
-	} else if (!state && tz->forced_passive) {
-		mutex_lock(&thermal_list_lock);
-		list_for_each_entry(cdev, &thermal_cdev_list, node) {
-			if (!strncmp("Processor", cdev->type,
-				     sizeof("Processor")))
-				thermal_zone_unbind_cooling_device(tz,
-								   THERMAL_TRIPS_NONE,
-								   cdev);
-		}
-		mutex_unlock(&thermal_list_lock);
-		tz->passive_delay = 0;
-	}
-
-	tz->forced_passive = state;
-
-	thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
-
-	return count;
-}
-
-static ssize_t
-passive_show(struct device *dev, struct device_attribute *attr,
-		   char *buf)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-
-	return sprintf(buf, "%d\n", tz->forced_passive);
-}
-
-static ssize_t
-policy_store(struct device *dev, struct device_attribute *attr,
-		    const char *buf, size_t count)
-{
-	int ret = -EINVAL;
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	struct thermal_governor *gov;
-	char name[THERMAL_NAME_LENGTH];
-
-	snprintf(name, sizeof(name), "%s", buf);
-
-	mutex_lock(&thermal_governor_lock);
-	mutex_lock(&tz->lock);
-
-	gov = __find_governor(strim(name));
-	if (!gov)
-		goto exit;
-
-	ret = thermal_set_governor(tz, gov);
-	if (!ret)
-		ret = count;
-
-exit:
-	mutex_unlock(&tz->lock);
-	mutex_unlock(&thermal_governor_lock);
-	return ret;
-}
-
-static ssize_t
-policy_show(struct device *dev, struct device_attribute *devattr, char *buf)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-
-	return sprintf(buf, "%s\n", tz->governor->name);
-}
-
-static ssize_t
-available_policies_show(struct device *dev, struct device_attribute *devattr,
-			char *buf)
-{
-	struct thermal_governor *pos;
-	ssize_t count = 0;
-	ssize_t size = PAGE_SIZE;
-
-	mutex_lock(&thermal_governor_lock);
-
-	list_for_each_entry(pos, &thermal_governor_list, governor_list) {
-		size = PAGE_SIZE - count;
-		count += scnprintf(buf + count, size, "%s ", pos->name);
-	}
-	count += scnprintf(buf + count, size, "\n");
-
-	mutex_unlock(&thermal_governor_lock);
-
-	return count;
-}
-
-static ssize_t
-emul_temp_store(struct device *dev, struct device_attribute *attr,
-		     const char *buf, size_t count)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	int ret = 0;
-	int temperature;
-
-	if (kstrtoint(buf, 10, &temperature))
-		return -EINVAL;
-
-	if (!tz->ops->set_emul_temp) {
-		mutex_lock(&tz->lock);
-		tz->emul_temperature = temperature;
-		mutex_unlock(&tz->lock);
-	} else {
-		ret = tz->ops->set_emul_temp(tz, temperature);
-	}
-
-	if (!ret)
-		thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
-
-	return ret ? ret : count;
-}
-static DEVICE_ATTR(emul_temp, S_IWUSR, NULL, emul_temp_store);
-
-static ssize_t
-sustainable_power_show(struct device *dev, struct device_attribute *devattr,
-		       char *buf)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-
-	if (tz->tzp)
-		return sprintf(buf, "%u\n", tz->tzp->sustainable_power);
-	else
-		return -EIO;
-}
-
-static ssize_t
-sustainable_power_store(struct device *dev, struct device_attribute *devattr,
-			const char *buf, size_t count)
-{
-	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	u32 sustainable_power;
-
-	if (!tz->tzp)
-		return -EIO;
-
-	if (kstrtou32(buf, 10, &sustainable_power))
-		return -EINVAL;
-
-	tz->tzp->sustainable_power = sustainable_power;
-
-	return count;
-}
-static DEVICE_ATTR(sustainable_power, S_IWUSR | S_IRUGO, sustainable_power_show,
-		sustainable_power_store);
-
-#define create_s32_tzp_attr(name)					\
-	static ssize_t							\
-	name##_show(struct device *dev, struct device_attribute *devattr, \
-		char *buf)						\
-	{								\
-	struct thermal_zone_device *tz = to_thermal_zone(dev);		\
-									\
-	if (tz->tzp)							\
-		return sprintf(buf, "%d\n", tz->tzp->name);		\
-	else								\
-		return -EIO;						\
-	}								\
-									\
-	static ssize_t							\
-	name##_store(struct device *dev, struct device_attribute *devattr, \
-		const char *buf, size_t count)				\
-	{								\
-		struct thermal_zone_device *tz = to_thermal_zone(dev);	\
-		s32 value;						\
-									\
-		if (!tz->tzp)						\
-			return -EIO;					\
-									\
-		if (kstrtos32(buf, 10, &value))				\
-			return -EINVAL;					\
-									\
-		tz->tzp->name = value;					\
-									\
-		return count;						\
-	}								\
-	static DEVICE_ATTR(name, S_IWUSR | S_IRUGO, name##_show, name##_store)
-
-create_s32_tzp_attr(k_po);
-create_s32_tzp_attr(k_pu);
-create_s32_tzp_attr(k_i);
-create_s32_tzp_attr(k_d);
-create_s32_tzp_attr(integral_cutoff);
-create_s32_tzp_attr(slope);
-create_s32_tzp_attr(offset);
-#undef create_s32_tzp_attr
-
-static struct device_attribute *dev_tzp_attrs[] = {
-	&dev_attr_sustainable_power,
-	&dev_attr_k_po,
-	&dev_attr_k_pu,
-	&dev_attr_k_i,
-	&dev_attr_k_d,
-	&dev_attr_integral_cutoff,
-	&dev_attr_slope,
-	&dev_attr_offset,
-};
-
-static int create_tzp_attrs(struct device *dev)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(dev_tzp_attrs); i++) {
-		int ret;
-		struct device_attribute *dev_attr = dev_tzp_attrs[i];
-
-		ret = device_create_file(dev, dev_attr);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
+/*
+ * Power actor section: interface to power actors to estimate power
+ *
+ * Set of functions used to interact to cooling devices that know
+ * how to estimate their devices power consumption.
+ */
 
 /**
  * power_actor_get_max_power() - get the maximum power that a cdev can consume
@@ -1127,12 +510,13 @@ int power_actor_get_min_power(struct thermal_cooling_device *cdev,
 }
 
 /**
- * power_actor_set_power() - limit the maximum power that a cooling device can consume
+ * power_actor_set_power() - limit the maximum power a cooling device consumes
  * @cdev:	pointer to &thermal_cooling_device
  * @instance:	thermal instance to update
  * @power:	the power in milliwatts
  *
- * Set the cooling device to consume at most @power milliwatts.
+ * Set the cooling device to consume at most @power milliwatts. The limit is
+ * expected to be a cap at the maximum power consumption.
  *
  * Return: 0 on success, -EINVAL if the cooling device does not
  * implement the power actor API or -E* for other failures.
@@ -1159,143 +543,75 @@ int power_actor_set_power(struct thermal_cooling_device *cdev,
 	return 0;
 }
 
-static DEVICE_ATTR(type, 0444, type_show, NULL);
-static DEVICE_ATTR(temp, 0444, temp_show, NULL);
-static DEVICE_ATTR(mode, 0644, mode_show, mode_store);
-static DEVICE_ATTR(passive, S_IRUGO | S_IWUSR, passive_show, passive_store);
-static DEVICE_ATTR(policy, S_IRUGO | S_IWUSR, policy_show, policy_store);
-static DEVICE_ATTR(available_policies, S_IRUGO, available_policies_show, NULL);
-
-/* sys I/F for cooling device */
-#define to_cooling_device(_dev)	\
-	container_of(_dev, struct thermal_cooling_device, device)
-
-static ssize_t
-thermal_cooling_device_type_show(struct device *dev,
-				 struct device_attribute *attr, char *buf)
+void thermal_zone_device_rebind_exception(struct thermal_zone_device *tz,
+					  const char *cdev_type, size_t size)
 {
-	struct thermal_cooling_device *cdev = to_cooling_device(dev);
+	struct thermal_cooling_device *cdev = NULL;
+
+	mutex_lock(&thermal_list_lock);
+	list_for_each_entry(cdev, &thermal_cdev_list, node) {
+		/* skip non matching cdevs */
+		if (strncmp(cdev_type, cdev->type, size))
+			continue;
 
-	return sprintf(buf, "%s\n", cdev->type);
+		/* re binding the exception matching the type pattern */
+		thermal_zone_bind_cooling_device(tz, THERMAL_TRIPS_NONE, cdev,
+						 THERMAL_NO_LIMIT,
+						 THERMAL_NO_LIMIT,
+						 THERMAL_WEIGHT_DEFAULT);
+	}
+	mutex_unlock(&thermal_list_lock);
 }
 
-static ssize_t
-thermal_cooling_device_max_state_show(struct device *dev,
-				      struct device_attribute *attr, char *buf)
+void thermal_zone_device_unbind_exception(struct thermal_zone_device *tz,
+					  const char *cdev_type, size_t size)
 {
-	struct thermal_cooling_device *cdev = to_cooling_device(dev);
-	unsigned long state;
-	int ret;
+	struct thermal_cooling_device *cdev = NULL;
 
-	ret = cdev->ops->get_max_state(cdev, &state);
-	if (ret)
-		return ret;
-	return sprintf(buf, "%ld\n", state);
+	mutex_lock(&thermal_list_lock);
+	list_for_each_entry(cdev, &thermal_cdev_list, node) {
+		/* skip non matching cdevs */
+		if (strncmp(cdev_type, cdev->type, size))
+			continue;
+		/* unbinding the exception matching the type pattern */
+		thermal_zone_unbind_cooling_device(tz, THERMAL_TRIPS_NONE,
+						   cdev);
+	}
+	mutex_unlock(&thermal_list_lock);
 }
 
-static ssize_t
-thermal_cooling_device_cur_state_show(struct device *dev,
-				      struct device_attribute *attr, char *buf)
+/*
+ * Device management section: cooling devices, zones devices, and binding
+ *
+ * Set of functions provided by the thermal core for:
+ * - cooling devices lifecycle: registration, unregistration,
+ *				binding, and unbinding.
+ * - thermal zone devices lifecycle: registration, unregistration,
+ *				     binding, and unbinding.
+ */
+static int get_idr(struct idr *idr, struct mutex *lock, int *id)
 {
-	struct thermal_cooling_device *cdev = to_cooling_device(dev);
-	unsigned long state;
 	int ret;
 
-	ret = cdev->ops->get_cur_state(cdev, &state);
-	if (ret)
+	if (lock)
+		mutex_lock(lock);
+	ret = idr_alloc(idr, NULL, 0, 0, GFP_KERNEL);
+	if (lock)
+		mutex_unlock(lock);
+	if (unlikely(ret < 0))
 		return ret;
-	return sprintf(buf, "%ld\n", state);
-}
-
-static ssize_t
-thermal_cooling_device_cur_state_store(struct device *dev,
-				       struct device_attribute *attr,
-				       const char *buf, size_t count)
-{
-	struct thermal_cooling_device *cdev = to_cooling_device(dev);
-	unsigned long state;
-	int result;
-
-	if (!sscanf(buf, "%ld\n", &state))
-		return -EINVAL;
-
-	if ((long)state < 0)
-		return -EINVAL;
-
-	result = cdev->ops->set_cur_state(cdev, state);
-	if (result)
-		return result;
-	return count;
-}
-
-static struct device_attribute dev_attr_cdev_type =
-__ATTR(type, 0444, thermal_cooling_device_type_show, NULL);
-static DEVICE_ATTR(max_state, 0444,
-		   thermal_cooling_device_max_state_show, NULL);
-static DEVICE_ATTR(cur_state, 0644,
-		   thermal_cooling_device_cur_state_show,
-		   thermal_cooling_device_cur_state_store);
-
-static ssize_t
-thermal_cooling_device_trip_point_show(struct device *dev,
-				       struct device_attribute *attr, char *buf)
-{
-	struct thermal_instance *instance;
-
-	instance =
-	    container_of(attr, struct thermal_instance, attr);
-
-	if (instance->trip == THERMAL_TRIPS_NONE)
-		return sprintf(buf, "-1\n");
-	else
-		return sprintf(buf, "%d\n", instance->trip);
-}
-
-static struct attribute *cooling_device_attrs[] = {
-	&dev_attr_cdev_type.attr,
-	&dev_attr_max_state.attr,
-	&dev_attr_cur_state.attr,
-	NULL,
-};
-
-static const struct attribute_group cooling_device_attr_group = {
-	.attrs = cooling_device_attrs,
-};
-
-static const struct attribute_group *cooling_device_attr_groups[] = {
-	&cooling_device_attr_group,
-	NULL,
-};
-
-static ssize_t
-thermal_cooling_device_weight_show(struct device *dev,
-				   struct device_attribute *attr, char *buf)
-{
-	struct thermal_instance *instance;
-
-	instance = container_of(attr, struct thermal_instance, weight_attr);
-
-	return sprintf(buf, "%d\n", instance->weight);
+	*id = ret;
+	return 0;
 }
 
-static ssize_t
-thermal_cooling_device_weight_store(struct device *dev,
-				    struct device_attribute *attr,
-				    const char *buf, size_t count)
+static void release_idr(struct idr *idr, struct mutex *lock, int id)
 {
-	struct thermal_instance *instance;
-	int ret, weight;
-
-	ret = kstrtoint(buf, 0, &weight);
-	if (ret)
-		return ret;
-
-	instance = container_of(attr, struct thermal_instance, weight_attr);
-	instance->weight = weight;
-
-	return count;
+	if (lock)
+		mutex_lock(lock);
+	idr_remove(idr, id);
+	if (lock)
+		mutex_unlock(lock);
 }
-/* Device management */
 
 /**
  * thermal_zone_bind_cooling_device() - bind a cooling device to a thermal zone
@@ -1358,8 +674,7 @@ int thermal_zone_bind_cooling_device(struct thermal_zone_device *tz,
 	if (lower > upper || upper > max_state)
 		return -EINVAL;
 
-	dev =
-	    kzalloc(sizeof(struct thermal_instance), GFP_KERNEL);
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 	if (!dev)
 		return -ENOMEM;
 	dev->tz = tz;
@@ -1402,10 +717,10 @@ int thermal_zone_bind_cooling_device(struct thermal_zone_device *tz,
 	mutex_lock(&tz->lock);
 	mutex_lock(&cdev->lock);
 	list_for_each_entry(pos, &tz->thermal_instances, tz_node)
-	    if (pos->tz == tz && pos->trip == trip && pos->cdev == cdev) {
-		result = -EEXIST;
-		break;
-	}
+		if (pos->tz == tz && pos->trip == trip && pos->cdev == cdev) {
+			result = -EEXIST;
+			break;
+		}
 	if (!result) {
 		list_add_tail(&dev->tz_node, &tz->thermal_instances);
 		list_add_tail(&dev->cdev_node, &cdev->thermal_instances);
@@ -1485,8 +800,8 @@ static void thermal_release(struct device *dev)
 		     sizeof("thermal_zone") - 1)) {
 		tz = to_thermal_zone(dev);
 		kfree(tz);
-	} else if(!strncmp(dev_name(dev), "cooling_device",
-			sizeof("cooling_device") - 1)){
+	} else if (!strncmp(dev_name(dev), "cooling_device",
+			    sizeof("cooling_device") - 1)) {
 		cdev = to_cooling_device(dev);
 		kfree(cdev);
 	}
@@ -1497,6 +812,78 @@ static struct class thermal_class = {
 	.dev_release = thermal_release,
 };
 
+static inline
+void print_bind_err_msg(struct thermal_zone_device *tz,
+			struct thermal_cooling_device *cdev, int ret)
+{
+	dev_err(&tz->device, "binding zone %s with cdev %s failed:%d\n",
+		tz->type, cdev->type, ret);
+}
+
+static void __bind(struct thermal_zone_device *tz, int mask,
+		   struct thermal_cooling_device *cdev,
+		   unsigned long *limits,
+		   unsigned int weight)
+{
+	int i, ret;
+
+	for (i = 0; i < tz->trips; i++) {
+		if (mask & (1 << i)) {
+			unsigned long upper, lower;
+
+			upper = THERMAL_NO_LIMIT;
+			lower = THERMAL_NO_LIMIT;
+			if (limits) {
+				lower = limits[i * 2];
+				upper = limits[i * 2 + 1];
+			}
+			ret = thermal_zone_bind_cooling_device(tz, i, cdev,
+							       upper, lower,
+							       weight);
+			if (ret)
+				print_bind_err_msg(tz, cdev, ret);
+		}
+	}
+}
+
+static void bind_cdev(struct thermal_cooling_device *cdev)
+{
+	int i, ret;
+	const struct thermal_zone_params *tzp;
+	struct thermal_zone_device *pos = NULL;
+
+	mutex_lock(&thermal_list_lock);
+
+	list_for_each_entry(pos, &thermal_tz_list, node) {
+		if (!pos->tzp && !pos->ops->bind)
+			continue;
+
+		if (pos->ops->bind) {
+			ret = pos->ops->bind(pos, cdev);
+			if (ret)
+				print_bind_err_msg(pos, cdev, ret);
+			continue;
+		}
+
+		tzp = pos->tzp;
+		if (!tzp || !tzp->tbp)
+			continue;
+
+		for (i = 0; i < tzp->num_tbps; i++) {
+			if (tzp->tbp[i].cdev || !tzp->tbp[i].match)
+				continue;
+			if (tzp->tbp[i].match(pos, cdev))
+				continue;
+			tzp->tbp[i].cdev = cdev;
+			__bind(pos, tzp->tbp[i].trip_mask, cdev,
+			       tzp->tbp[i].binding_limits,
+			       tzp->tbp[i].weight);
+		}
+	}
+
+	mutex_unlock(&thermal_list_lock);
+}
+
 /**
  * __thermal_cooling_device_register() - register a new thermal cooling device
  * @np:		a pointer to a device tree node.
@@ -1529,7 +916,7 @@ __thermal_cooling_device_register(struct device_node *np,
 	    !ops->set_cur_state)
 		return ERR_PTR(-EINVAL);
 
-	cdev = kzalloc(sizeof(struct thermal_cooling_device), GFP_KERNEL);
+	cdev = kzalloc(sizeof(*cdev), GFP_KERNEL);
 	if (!cdev)
 		return ERR_PTR(-ENOMEM);
 
@@ -1546,7 +933,7 @@ __thermal_cooling_device_register(struct device_node *np,
 	cdev->ops = ops;
 	cdev->updated = false;
 	cdev->device.class = &thermal_class;
-	cdev->device.groups = cooling_device_attr_groups;
+	thermal_cooling_device_setup_sysfs(cdev);
 	cdev->devdata = devdata;
 	dev_set_name(&cdev->device, "cooling_device%d", cdev->id);
 	result = device_register(&cdev->device);
@@ -1619,12 +1006,22 @@ thermal_of_cooling_device_register(struct device_node *np,
 }
 EXPORT_SYMBOL_GPL(thermal_of_cooling_device_register);
 
+static void __unbind(struct thermal_zone_device *tz, int mask,
+		     struct thermal_cooling_device *cdev)
+{
+	int i;
+
+	for (i = 0; i < tz->trips; i++)
+		if (mask & (1 << i))
+			thermal_zone_unbind_cooling_device(tz, i, cdev);
+}
+
 /**
- * thermal_cooling_device_unregister - removes the registered thermal cooling device
+ * thermal_cooling_device_unregister - removes a thermal cooling device
  * @cdev:	the thermal cooling device to remove.
  *
- * thermal_cooling_device_unregister() must be called when the device is no
- * longer needed.
+ * thermal_cooling_device_unregister() must be called when a registered
+ * thermal cooling device is no longer needed.
  */
 void thermal_cooling_device_unregister(struct thermal_cooling_device *cdev)
 {
@@ -1638,8 +1035,8 @@ void thermal_cooling_device_unregister(struct thermal_cooling_device *cdev)
 
 	mutex_lock(&thermal_list_lock);
 	list_for_each_entry(pos, &thermal_cdev_list, node)
-	    if (pos == cdev)
-		break;
+		if (pos == cdev)
+			break;
 	if (pos != cdev) {
 		/* thermal cooling device not found */
 		mutex_unlock(&thermal_list_lock);
@@ -1668,171 +1065,49 @@ void thermal_cooling_device_unregister(struct thermal_cooling_device *cdev)
 
 	mutex_unlock(&thermal_list_lock);
 
-	if (cdev->type[0])
-		device_remove_file(&cdev->device, &dev_attr_cdev_type);
-	device_remove_file(&cdev->device, &dev_attr_max_state);
-	device_remove_file(&cdev->device, &dev_attr_cur_state);
-
 	release_idr(&thermal_cdev_idr, &thermal_idr_lock, cdev->id);
 	device_unregister(&cdev->device);
-	return;
 }
 EXPORT_SYMBOL_GPL(thermal_cooling_device_unregister);
 
-void thermal_cdev_update(struct thermal_cooling_device *cdev)
+static void bind_tz(struct thermal_zone_device *tz)
 {
-	struct thermal_instance *instance;
-	unsigned long target = 0;
+	int i, ret;
+	struct thermal_cooling_device *pos = NULL;
+	const struct thermal_zone_params *tzp = tz->tzp;
 
-	mutex_lock(&cdev->lock);
-	/* cooling device is updated*/
-	if (cdev->updated) {
-		mutex_unlock(&cdev->lock);
+	if (!tzp && !tz->ops->bind)
 		return;
-	}
-
-	/* Make sure cdev enters the deepest cooling state */
-	list_for_each_entry(instance, &cdev->thermal_instances, cdev_node) {
-		dev_dbg(&cdev->device, "zone%d->target=%lu\n",
-				instance->tz->id, instance->target);
-		if (instance->target == THERMAL_NO_TARGET)
-			continue;
-		if (instance->target > target)
-			target = instance->target;
-	}
-	cdev->ops->set_cur_state(cdev, target);
-	cdev->updated = true;
-	mutex_unlock(&cdev->lock);
-	trace_cdev_update(cdev, target);
-	dev_dbg(&cdev->device, "set to state %lu\n", target);
-}
-EXPORT_SYMBOL(thermal_cdev_update);
 
-/**
- * thermal_notify_framework - Sensor drivers use this API to notify framework
- * @tz:		thermal zone device
- * @trip:	indicates which trip point has been crossed
- *
- * This function handles the trip events from sensor drivers. It starts
- * throttling the cooling devices according to the policy configured.
- * For CRITICAL and HOT trip points, this notifies the respective drivers,
- * and does actual throttling for other trip points i.e ACTIVE and PASSIVE.
- * The throttling policy is based on the configured platform data; if no
- * platform data is provided, this uses the step_wise throttling policy.
- */
-void thermal_notify_framework(struct thermal_zone_device *tz, int trip)
-{
-	handle_thermal_trip(tz, trip);
-}
-EXPORT_SYMBOL_GPL(thermal_notify_framework);
-
-/**
- * create_trip_attrs() - create attributes for trip points
- * @tz:		the thermal zone device
- * @mask:	Writeable trip point bitmap.
- *
- * helper function to instantiate sysfs entries for every trip
- * point and its properties of a struct thermal_zone_device.
- *
- * Return: 0 on success, the proper error value otherwise.
- */
-static int create_trip_attrs(struct thermal_zone_device *tz, int mask)
-{
-	int indx;
-	int size = sizeof(struct thermal_attr) * tz->trips;
-
-	tz->trip_type_attrs = kzalloc(size, GFP_KERNEL);
-	if (!tz->trip_type_attrs)
-		return -ENOMEM;
-
-	tz->trip_temp_attrs = kzalloc(size, GFP_KERNEL);
-	if (!tz->trip_temp_attrs) {
-		kfree(tz->trip_type_attrs);
-		return -ENOMEM;
-	}
+	mutex_lock(&thermal_list_lock);
 
-	if (tz->ops->get_trip_hyst) {
-		tz->trip_hyst_attrs = kzalloc(size, GFP_KERNEL);
-		if (!tz->trip_hyst_attrs) {
-			kfree(tz->trip_type_attrs);
-			kfree(tz->trip_temp_attrs);
-			return -ENOMEM;
+	/* If there is ops->bind, try to use ops->bind */
+	if (tz->ops->bind) {
+		list_for_each_entry(pos, &thermal_cdev_list, node) {
+			ret = tz->ops->bind(tz, pos);
+			if (ret)
+				print_bind_err_msg(tz, pos, ret);
 		}
+		goto exit;
 	}
 
+	if (!tzp || !tzp->tbp)
+		goto exit;
 
-	for (indx = 0; indx < tz->trips; indx++) {
-		/* create trip type attribute */
-		snprintf(tz->trip_type_attrs[indx].name, THERMAL_NAME_LENGTH,
-			 "trip_point_%d_type", indx);
-
-		sysfs_attr_init(&tz->trip_type_attrs[indx].attr.attr);
-		tz->trip_type_attrs[indx].attr.attr.name =
-						tz->trip_type_attrs[indx].name;
-		tz->trip_type_attrs[indx].attr.attr.mode = S_IRUGO;
-		tz->trip_type_attrs[indx].attr.show = trip_point_type_show;
-
-		device_create_file(&tz->device,
-				   &tz->trip_type_attrs[indx].attr);
-
-		/* create trip temp attribute */
-		snprintf(tz->trip_temp_attrs[indx].name, THERMAL_NAME_LENGTH,
-			 "trip_point_%d_temp", indx);
-
-		sysfs_attr_init(&tz->trip_temp_attrs[indx].attr.attr);
-		tz->trip_temp_attrs[indx].attr.attr.name =
-						tz->trip_temp_attrs[indx].name;
-		tz->trip_temp_attrs[indx].attr.attr.mode = S_IRUGO;
-		tz->trip_temp_attrs[indx].attr.show = trip_point_temp_show;
-		if (IS_ENABLED(CONFIG_THERMAL_WRITABLE_TRIPS) &&
-		    mask & (1 << indx)) {
-			tz->trip_temp_attrs[indx].attr.attr.mode |= S_IWUSR;
-			tz->trip_temp_attrs[indx].attr.store =
-							trip_point_temp_store;
-		}
-
-		device_create_file(&tz->device,
-				   &tz->trip_temp_attrs[indx].attr);
-
-		/* create Optional trip hyst attribute */
-		if (!tz->ops->get_trip_hyst)
-			continue;
-		snprintf(tz->trip_hyst_attrs[indx].name, THERMAL_NAME_LENGTH,
-			 "trip_point_%d_hyst", indx);
-
-		sysfs_attr_init(&tz->trip_hyst_attrs[indx].attr.attr);
-		tz->trip_hyst_attrs[indx].attr.attr.name =
-					tz->trip_hyst_attrs[indx].name;
-		tz->trip_hyst_attrs[indx].attr.attr.mode = S_IRUGO;
-		tz->trip_hyst_attrs[indx].attr.show = trip_point_hyst_show;
-		if (tz->ops->set_trip_hyst) {
-			tz->trip_hyst_attrs[indx].attr.attr.mode |= S_IWUSR;
-			tz->trip_hyst_attrs[indx].attr.store =
-					trip_point_hyst_store;
+	list_for_each_entry(pos, &thermal_cdev_list, node) {
+		for (i = 0; i < tzp->num_tbps; i++) {
+			if (tzp->tbp[i].cdev || !tzp->tbp[i].match)
+				continue;
+			if (tzp->tbp[i].match(tz, pos))
+				continue;
+			tzp->tbp[i].cdev = pos;
+			__bind(tz, tzp->tbp[i].trip_mask, pos,
+			       tzp->tbp[i].binding_limits,
+			       tzp->tbp[i].weight);
 		}
-
-		device_create_file(&tz->device,
-				   &tz->trip_hyst_attrs[indx].attr);
-	}
-	return 0;
-}
-
-static void remove_trip_attrs(struct thermal_zone_device *tz)
-{
-	int indx;
-
-	for (indx = 0; indx < tz->trips; indx++) {
-		device_remove_file(&tz->device,
-				   &tz->trip_type_attrs[indx].attr);
-		device_remove_file(&tz->device,
-				   &tz->trip_temp_attrs[indx].attr);
-		if (tz->ops->get_trip_hyst)
-			device_remove_file(&tz->device,
-				  &tz->trip_hyst_attrs[indx].attr);
 	}
-	kfree(tz->trip_type_attrs);
-	kfree(tz->trip_temp_attrs);
-	kfree(tz->trip_hyst_attrs);
+exit:
+	mutex_unlock(&thermal_list_lock);
 }
 
 /**
@@ -1859,20 +1134,22 @@ static void remove_trip_attrs(struct thermal_zone_device *tz)
  * in case of error, an ERR_PTR. Caller must check return value with
  * IS_ERR*() helpers.
  */
-struct thermal_zone_device *thermal_zone_device_register(const char *type,
-	int trips, int mask, void *devdata,
-	struct thermal_zone_device_ops *ops,
-	struct thermal_zone_params *tzp,
-	int passive_delay, int polling_delay)
+struct thermal_zone_device *
+thermal_zone_device_register(const char *type, int trips, int mask,
+			     void *devdata, struct thermal_zone_device_ops *ops,
+			     struct thermal_zone_params *tzp, int passive_delay,
+			     int polling_delay)
 {
 	struct thermal_zone_device *tz;
 	enum thermal_trip_type trip_type;
 	int trip_temp;
 	int result;
 	int count;
-	int passive = 0;
 	struct thermal_governor *governor;
 
+	if (!type || strlen(type) == 0)
+		return ERR_PTR(-EINVAL);
+
 	if (type && strlen(type) >= THERMAL_NAME_LENGTH)
 		return ERR_PTR(-EINVAL);
 
@@ -1885,7 +1162,7 @@ struct thermal_zone_device *thermal_zone_device_register(const char *type,
 	if (trips > 0 && (!ops->get_trip_type || !ops->get_trip_temp))
 		return ERR_PTR(-EINVAL);
 
-	tz = kzalloc(sizeof(struct thermal_zone_device), GFP_KERNEL);
+	tz = kzalloc(sizeof(*tz), GFP_KERNEL);
 	if (!tz)
 		return ERR_PTR(-ENOMEM);
 
@@ -1898,7 +1175,7 @@ struct thermal_zone_device *thermal_zone_device_register(const char *type,
 		return ERR_PTR(result);
 	}
 
-	strlcpy(tz->type, type ? : "", sizeof(tz->type));
+	strlcpy(tz->type, type, sizeof(tz->type));
 	tz->ops = ops;
 	tz->tzp = tzp;
 	tz->device.class = &thermal_class;
@@ -1906,6 +1183,13 @@ struct thermal_zone_device *thermal_zone_device_register(const char *type,
 	tz->trips = trips;
 	tz->passive_delay = passive_delay;
 	tz->polling_delay = polling_delay;
+
+	/* sys I/F */
+	/* Add nodes that are always present via .groups */
+	result = thermal_zone_create_device_groups(tz, mask);
+	if (result)
+		goto unregister;
+
 	/* A new thermal zone needs to be updated anyway. */
 	atomic_set(&tz->need_update, 1);
 
@@ -1917,32 +1201,9 @@ struct thermal_zone_device *thermal_zone_device_register(const char *type,
 		return ERR_PTR(result);
 	}
 
-	/* sys I/F */
-	if (type) {
-		result = device_create_file(&tz->device, &dev_attr_type);
-		if (result)
-			goto unregister;
-	}
-
-	result = device_create_file(&tz->device, &dev_attr_temp);
-	if (result)
-		goto unregister;
-
-	if (ops->get_mode) {
-		result = device_create_file(&tz->device, &dev_attr_mode);
-		if (result)
-			goto unregister;
-	}
-
-	result = create_trip_attrs(tz, mask);
-	if (result)
-		goto unregister;
-
 	for (count = 0; count < trips; count++) {
 		if (tz->ops->get_trip_type(tz, count, &trip_type))
 			set_bit(count, &tz->trips_disabled);
-		if (trip_type == THERMAL_TRIP_PASSIVE)
-			passive = 1;
 		if (tz->ops->get_trip_temp(tz, count, &trip_temp))
 			set_bit(count, &tz->trips_disabled);
 		/* Check for bogus trip points */
@@ -1950,33 +1211,6 @@ struct thermal_zone_device *thermal_zone_device_register(const char *type,
 			set_bit(count, &tz->trips_disabled);
 	}
 
-	if (!passive) {
-		result = device_create_file(&tz->device, &dev_attr_passive);
-		if (result)
-			goto unregister;
-	}
-
-	if (IS_ENABLED(CONFIG_THERMAL_EMULATION)) {
-		result = device_create_file(&tz->device, &dev_attr_emul_temp);
-		if (result)
-			goto unregister;
-	}
-
-	/* Create policy attribute */
-	result = device_create_file(&tz->device, &dev_attr_policy);
-	if (result)
-		goto unregister;
-
-	/* Add thermal zone params */
-	result = create_tzp_attrs(&tz->device);
-	if (result)
-		goto unregister;
-
-	/* Create available_policies attribute */
-	result = device_create_file(&tz->device, &dev_attr_available_policies);
-	if (result)
-		goto unregister;
-
 	/* Update 'this' zone's governor information */
 	mutex_lock(&thermal_governor_lock);
 
@@ -2006,7 +1240,7 @@ struct thermal_zone_device *thermal_zone_device_register(const char *type,
 	/* Bind cooling devices for this zone */
 	bind_tz(tz);
 
-	INIT_DELAYED_WORK(&(tz->poll_queue), thermal_zone_device_check);
+	INIT_DELAYED_WORK(&tz->poll_queue, thermal_zone_device_check);
 
 	thermal_zone_device_reset(tz);
 	/* Update the new thermal zone and mark it as already updated. */
@@ -2040,8 +1274,8 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz)
 
 	mutex_lock(&thermal_list_lock);
 	list_for_each_entry(pos, &thermal_tz_list, node)
-	    if (pos == tz)
-		break;
+		if (pos == tz)
+			break;
 	if (pos != tz) {
 		/* thermal zone device not found */
 		mutex_unlock(&thermal_list_lock);
@@ -2071,14 +1305,10 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz)
 
 	thermal_zone_device_set_polling(tz, 0);
 
-	if (tz->type[0])
-		device_remove_file(&tz->device, &dev_attr_type);
-	device_remove_file(&tz->device, &dev_attr_temp);
-	if (tz->ops->get_mode)
-		device_remove_file(&tz->device, &dev_attr_mode);
-	device_remove_file(&tz->device, &dev_attr_policy);
-	device_remove_file(&tz->device, &dev_attr_available_policies);
-	remove_trip_attrs(tz);
+	kfree(tz->trip_type_attrs);
+	kfree(tz->trip_temp_attrs);
+	kfree(tz->trip_hyst_attrs);
+	kfree(tz->trips_attribute_group.attrs);
 	thermal_set_governor(tz, NULL);
 
 	thermal_remove_hwmon_sysfs(tz);
@@ -2086,7 +1316,7 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz)
 	idr_destroy(&tz->idr);
 	mutex_destroy(&tz->lock);
 	device_unregister(&tz->device);
-	return;
+	kfree(tz->device.groups);
 }
 EXPORT_SYMBOL_GPL(thermal_zone_device_unregister);
 
@@ -2128,36 +1358,6 @@ exit:
 }
 EXPORT_SYMBOL_GPL(thermal_zone_get_zone_by_name);
 
-/**
- * thermal_zone_get_slope - return the slope attribute of the thermal zone
- * @tz: thermal zone device with the slope attribute
- *
- * Return: If the thermal zone device has a slope attribute, return it, else
- * return 1.
- */
-int thermal_zone_get_slope(struct thermal_zone_device *tz)
-{
-	if (tz && tz->tzp)
-		return tz->tzp->slope;
-	return 1;
-}
-EXPORT_SYMBOL_GPL(thermal_zone_get_slope);
-
-/**
- * thermal_zone_get_offset - return the offset attribute of the thermal zone
- * @tz: thermal zone device with the offset attribute
- *
- * Return: If the thermal zone device has a offset attribute, return it, else
- * return 0.
- */
-int thermal_zone_get_offset(struct thermal_zone_device *tz)
-{
-	if (tz && tz->tzp)
-		return tz->tzp->offset;
-	return 0;
-}
-EXPORT_SYMBOL_GPL(thermal_zone_get_offset);
-
 #ifdef CONFIG_NET
 static const struct genl_multicast_group thermal_event_mcgrps[] = {
 	{ .name = THERMAL_GENL_MCAST_GROUP_NAME, },
@@ -2173,7 +1373,7 @@ static struct genl_family thermal_event_genl_family __ro_after_init = {
 };
 
 int thermal_generate_netlink_event(struct thermal_zone_device *tz,
-					enum events event)
+				   enum events event)
 {
 	struct sk_buff *skb;
 	struct nlattr *attr;
@@ -2249,40 +1449,8 @@ static inline int genetlink_init(void) { return 0; }
 static inline void genetlink_exit(void) {}
 #endif /* !CONFIG_NET */
 
-static int __init thermal_register_governors(void)
-{
-	int result;
-
-	result = thermal_gov_step_wise_register();
-	if (result)
-		return result;
-
-	result = thermal_gov_fair_share_register();
-	if (result)
-		return result;
-
-	result = thermal_gov_bang_bang_register();
-	if (result)
-		return result;
-
-	result = thermal_gov_user_space_register();
-	if (result)
-		return result;
-
-	return thermal_gov_power_allocator_register();
-}
-
-static void thermal_unregister_governors(void)
-{
-	thermal_gov_step_wise_unregister();
-	thermal_gov_fair_share_unregister();
-	thermal_gov_bang_bang_unregister();
-	thermal_gov_user_space_unregister();
-	thermal_gov_power_allocator_unregister();
-}
-
 static int thermal_pm_notify(struct notifier_block *nb,
-				unsigned long mode, void *_unused)
+			     unsigned long mode, void *_unused)
 {
 	struct thermal_zone_device *tz;
 
diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h
index 749d41abfbab..2412b3759e16 100644
--- a/drivers/thermal/thermal_core.h
+++ b/drivers/thermal/thermal_core.h
@@ -54,8 +54,34 @@ struct thermal_instance {
 	unsigned int weight; /* The weight of the cooling device */
 };
 
+#define to_thermal_zone(_dev) \
+	container_of(_dev, struct thermal_zone_device, device)
+
+#define to_cooling_device(_dev)	\
+	container_of(_dev, struct thermal_cooling_device, device)
+
 int thermal_register_governor(struct thermal_governor *);
 void thermal_unregister_governor(struct thermal_governor *);
+void thermal_zone_device_rebind_exception(struct thermal_zone_device *,
+					  const char *, size_t);
+void thermal_zone_device_unbind_exception(struct thermal_zone_device *,
+					  const char *, size_t);
+int thermal_zone_device_set_policy(struct thermal_zone_device *, char *);
+int thermal_build_list_of_policies(char *buf);
+
+/* sysfs I/F */
+int thermal_zone_create_device_groups(struct thermal_zone_device *, int);
+void thermal_cooling_device_setup_sysfs(struct thermal_cooling_device *);
+/* used only at binding time */
+ssize_t
+thermal_cooling_device_trip_point_show(struct device *,
+				       struct device_attribute *, char *);
+ssize_t thermal_cooling_device_weight_show(struct device *,
+					   struct device_attribute *, char *);
+
+ssize_t thermal_cooling_device_weight_store(struct device *,
+					    struct device_attribute *,
+					    const char *, size_t);
 
 #ifdef CONFIG_THERMAL_GOV_STEP_WISE
 int thermal_gov_step_wise_register(void);
diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c
new file mode 100644
index 000000000000..8cdf75adcce1
--- /dev/null
+++ b/drivers/thermal/thermal_helpers.c
@@ -0,0 +1,226 @@
+/*
+ *  thermal_helpers.c - helper functions to handle thermal devices
+ *
+ *  Copyright (C) 2016 Eduardo Valentin <edubezval@gmail.com>
+ *
+ *  Highly based on original thermal_core.c
+ *  Copyright (C) 2008 Intel Corp
+ *  Copyright (C) 2008 Zhang Rui <rui.zhang@intel.com>
+ *  Copyright (C) 2008 Sujith Thomas <sujith.thomas@intel.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/sysfs.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include <trace/events/thermal.h>
+
+#include "thermal_core.h"
+
+int get_tz_trend(struct thermal_zone_device *tz, int trip)
+{
+	enum thermal_trend trend;
+
+	if (tz->emul_temperature || !tz->ops->get_trend ||
+	    tz->ops->get_trend(tz, trip, &trend)) {
+		if (tz->temperature > tz->last_temperature)
+			trend = THERMAL_TREND_RAISING;
+		else if (tz->temperature < tz->last_temperature)
+			trend = THERMAL_TREND_DROPPING;
+		else
+			trend = THERMAL_TREND_STABLE;
+	}
+
+	return trend;
+}
+EXPORT_SYMBOL(get_tz_trend);
+
+struct thermal_instance *
+get_thermal_instance(struct thermal_zone_device *tz,
+		     struct thermal_cooling_device *cdev, int trip)
+{
+	struct thermal_instance *pos = NULL;
+	struct thermal_instance *target_instance = NULL;
+
+	mutex_lock(&tz->lock);
+	mutex_lock(&cdev->lock);
+
+	list_for_each_entry(pos, &tz->thermal_instances, tz_node) {
+		if (pos->tz == tz && pos->trip == trip && pos->cdev == cdev) {
+			target_instance = pos;
+			break;
+		}
+	}
+
+	mutex_unlock(&cdev->lock);
+	mutex_unlock(&tz->lock);
+
+	return target_instance;
+}
+EXPORT_SYMBOL(get_thermal_instance);
+
+/**
+ * thermal_zone_get_temp() - returns the temperature of a thermal zone
+ * @tz: a valid pointer to a struct thermal_zone_device
+ * @temp: a valid pointer to where to store the resulting temperature.
+ *
+ * When a valid thermal zone reference is passed, it will fetch its
+ * temperature and fill @temp.
+ *
+ * Return: On success returns 0, an error code otherwise
+ */
+int thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp)
+{
+	int ret = -EINVAL;
+	int count;
+	int crit_temp = INT_MAX;
+	enum thermal_trip_type type;
+
+	if (!tz || IS_ERR(tz) || !tz->ops->get_temp)
+		goto exit;
+
+	mutex_lock(&tz->lock);
+
+	ret = tz->ops->get_temp(tz, temp);
+
+	if (IS_ENABLED(CONFIG_THERMAL_EMULATION) && tz->emul_temperature) {
+		for (count = 0; count < tz->trips; count++) {
+			ret = tz->ops->get_trip_type(tz, count, &type);
+			if (!ret && type == THERMAL_TRIP_CRITICAL) {
+				ret = tz->ops->get_trip_temp(tz, count,
+						&crit_temp);
+				break;
+			}
+		}
+
+		/*
+		 * Only allow emulating a temperature when the real temperature
+		 * is below the critical temperature so that the emulation code
+		 * cannot hide critical conditions.
+		 */
+		if (!ret && *temp < crit_temp)
+			*temp = tz->emul_temperature;
+	}
+
+	mutex_unlock(&tz->lock);
+exit:
+	return ret;
+}
+EXPORT_SYMBOL_GPL(thermal_zone_get_temp);
+
+void thermal_zone_set_trips(struct thermal_zone_device *tz)
+{
+	int low = -INT_MAX;
+	int high = INT_MAX;
+	int trip_temp, hysteresis;
+	int i, ret;
+
+	mutex_lock(&tz->lock);
+
+	if (!tz->ops->set_trips || !tz->ops->get_trip_hyst)
+		goto exit;
+
+	for (i = 0; i < tz->trips; i++) {
+		int trip_low;
+
+		tz->ops->get_trip_temp(tz, i, &trip_temp);
+		tz->ops->get_trip_hyst(tz, i, &hysteresis);
+
+		trip_low = trip_temp - hysteresis;
+
+		if (trip_low < tz->temperature && trip_low > low)
+			low = trip_low;
+
+		if (trip_temp > tz->temperature && trip_temp < high)
+			high = trip_temp;
+	}
+
+	/* No need to change trip points */
+	if (tz->prev_low_trip == low && tz->prev_high_trip == high)
+		goto exit;
+
+	tz->prev_low_trip = low;
+	tz->prev_high_trip = high;
+
+	dev_dbg(&tz->device,
+		"new temperature boundaries: %d < x < %d\n", low, high);
+
+	/*
+	 * Set a temperature window. When this window is left the driver
+	 * must inform the thermal core via thermal_zone_device_update.
+	 */
+	ret = tz->ops->set_trips(tz, low, high);
+	if (ret)
+		dev_err(&tz->device, "Failed to set trips: %d\n", ret);
+
+exit:
+	mutex_unlock(&tz->lock);
+}
+EXPORT_SYMBOL_GPL(thermal_zone_set_trips);
+
+void thermal_cdev_update(struct thermal_cooling_device *cdev)
+{
+	struct thermal_instance *instance;
+	unsigned long target = 0;
+
+	mutex_lock(&cdev->lock);
+	/* cooling device is updated*/
+	if (cdev->updated) {
+		mutex_unlock(&cdev->lock);
+		return;
+	}
+
+	/* Make sure cdev enters the deepest cooling state */
+	list_for_each_entry(instance, &cdev->thermal_instances, cdev_node) {
+		dev_dbg(&cdev->device, "zone%d->target=%lu\n",
+			instance->tz->id, instance->target);
+		if (instance->target == THERMAL_NO_TARGET)
+			continue;
+		if (instance->target > target)
+			target = instance->target;
+	}
+	cdev->ops->set_cur_state(cdev, target);
+	cdev->updated = true;
+	mutex_unlock(&cdev->lock);
+	trace_cdev_update(cdev, target);
+	dev_dbg(&cdev->device, "set to state %lu\n", target);
+}
+EXPORT_SYMBOL(thermal_cdev_update);
+
+/**
+ * thermal_zone_get_slope - return the slope attribute of the thermal zone
+ * @tz: thermal zone device with the slope attribute
+ *
+ * Return: If the thermal zone device has a slope attribute, return it, else
+ * return 1.
+ */
+int thermal_zone_get_slope(struct thermal_zone_device *tz)
+{
+	if (tz && tz->tzp)
+		return tz->tzp->slope;
+	return 1;
+}
+EXPORT_SYMBOL_GPL(thermal_zone_get_slope);
+
+/**
+ * thermal_zone_get_offset - return the offset attribute of the thermal zone
+ * @tz: thermal zone device with the offset attribute
+ *
+ * Return: If the thermal zone device has a offset attribute, return it, else
+ * return 0.
+ */
+int thermal_zone_get_offset(struct thermal_zone_device *tz)
+{
+	if (tz && tz->tzp)
+		return tz->tzp->offset;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(thermal_zone_get_offset);
diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c
index c41c7742903a..541af5946203 100644
--- a/drivers/thermal/thermal_hwmon.c
+++ b/drivers/thermal/thermal_hwmon.c
@@ -64,7 +64,7 @@ name_show(struct device *dev, struct device_attribute *attr, char *buf)
 	struct thermal_hwmon_device *hwmon = dev_get_drvdata(dev);
 	return sprintf(buf, "%s\n", hwmon->type);
 }
-static DEVICE_ATTR(name, 0444, name_show, NULL);
+static DEVICE_ATTR_RO(name);
 
 static ssize_t
 temp_input_show(struct device *dev, struct device_attribute *attr, char *buf)
@@ -98,7 +98,7 @@ temp_crit_show(struct device *dev, struct device_attribute *attr, char *buf)
 	int temperature;
 	int ret;
 
-	ret = tz->ops->get_trip_temp(tz, 0, &temperature);
+	ret = tz->ops->get_crit_temp(tz, &temperature);
 	if (ret)
 		return ret;
 
diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c
new file mode 100644
index 000000000000..a694de907a26
--- /dev/null
+++ b/drivers/thermal/thermal_sysfs.c
@@ -0,0 +1,771 @@
+/*
+ *  thermal.c - sysfs interface of thermal devices
+ *
+ *  Copyright (C) 2016 Eduardo Valentin <edubezval@gmail.com>
+ *
+ *  Highly based on original thermal_core.c
+ *  Copyright (C) 2008 Intel Corp
+ *  Copyright (C) 2008 Zhang Rui <rui.zhang@intel.com>
+ *  Copyright (C) 2008 Sujith Thomas <sujith.thomas@intel.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; version 2 of the License.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/sysfs.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "thermal_core.h"
+
+/* sys I/F for thermal zone */
+
+static ssize_t
+type_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+
+	return sprintf(buf, "%s\n", tz->type);
+}
+
+static ssize_t
+temp_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+	int temperature, ret;
+
+	ret = thermal_zone_get_temp(tz, &temperature);
+
+	if (ret)
+		return ret;
+
+	return sprintf(buf, "%d\n", temperature);
+}
+
+static ssize_t
+mode_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+	enum thermal_device_mode mode;
+	int result;
+
+	if (!tz->ops->get_mode)
+		return -EPERM;
+
+	result = tz->ops->get_mode(tz, &mode);
+	if (result)
+		return result;
+
+	return sprintf(buf, "%s\n", mode == THERMAL_DEVICE_ENABLED ? "enabled"
+		       : "disabled");
+}
+
+static ssize_t
+mode_store(struct device *dev, struct device_attribute *attr,
+	   const char *buf, size_t count)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+	int result;
+
+	if (!tz->ops->set_mode)
+		return -EPERM;
+
+	if (!strncmp(buf, "enabled", sizeof("enabled") - 1))
+		result = tz->ops->set_mode(tz, THERMAL_DEVICE_ENABLED);
+	else if (!strncmp(buf, "disabled", sizeof("disabled") - 1))
+		result = tz->ops->set_mode(tz, THERMAL_DEVICE_DISABLED);
+	else
+		result = -EINVAL;
+
+	if (result)
+		return result;
+
+	return count;
+}
+
+static ssize_t
+trip_point_type_show(struct device *dev, struct device_attribute *attr,
+		     char *buf)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+	enum thermal_trip_type type;
+	int trip, result;
+
+	if (!tz->ops->get_trip_type)
+		return -EPERM;
+
+	if (sscanf(attr->attr.name, "trip_point_%d_type", &trip) != 1)
+		return -EINVAL;
+
+	result = tz->ops->get_trip_type(tz, trip, &type);
+	if (result)
+		return result;
+
+	switch (type) {
+	case THERMAL_TRIP_CRITICAL:
+		return sprintf(buf, "critical\n");
+	case THERMAL_TRIP_HOT:
+		return sprintf(buf, "hot\n");
+	case THERMAL_TRIP_PASSIVE:
+		return sprintf(buf, "passive\n");
+	case THERMAL_TRIP_ACTIVE:
+		return sprintf(buf, "active\n");
+	default:
+		return sprintf(buf, "unknown\n");
+	}
+}
+
+static ssize_t
+trip_point_temp_store(struct device *dev, struct device_attribute *attr,
+		      const char *buf, size_t count)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+	int trip, ret;
+	int temperature;
+
+	if (!tz->ops->set_trip_temp)
+		return -EPERM;
+
+	if (sscanf(attr->attr.name, "trip_point_%d_temp", &trip) != 1)
+		return -EINVAL;
+
+	if (kstrtoint(buf, 10, &temperature))
+		return -EINVAL;
+
+	ret = tz->ops->set_trip_temp(tz, trip, temperature);
+	if (ret)
+		return ret;
+
+	thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
+
+	return count;
+}
+
+static ssize_t
+trip_point_temp_show(struct device *dev, struct device_attribute *attr,
+		     char *buf)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+	int trip, ret;
+	int temperature;
+
+	if (!tz->ops->get_trip_temp)
+		return -EPERM;
+
+	if (sscanf(attr->attr.name, "trip_point_%d_temp", &trip) != 1)
+		return -EINVAL;
+
+	ret = tz->ops->get_trip_temp(tz, trip, &temperature);
+
+	if (ret)
+		return ret;
+
+	return sprintf(buf, "%d\n", temperature);
+}
+
+static ssize_t
+trip_point_hyst_store(struct device *dev, struct device_attribute *attr,
+		      const char *buf, size_t count)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+	int trip, ret;
+	int temperature;
+
+	if (!tz->ops->set_trip_hyst)
+		return -EPERM;
+
+	if (sscanf(attr->attr.name, "trip_point_%d_hyst", &trip) != 1)
+		return -EINVAL;
+
+	if (kstrtoint(buf, 10, &temperature))
+		return -EINVAL;
+
+	/*
+	 * We are not doing any check on the 'temperature' value
+	 * here. The driver implementing 'set_trip_hyst' has to
+	 * take care of this.
+	 */
+	ret = tz->ops->set_trip_hyst(tz, trip, temperature);
+
+	if (!ret)
+		thermal_zone_set_trips(tz);
+
+	return ret ? ret : count;
+}
+
+static ssize_t
+trip_point_hyst_show(struct device *dev, struct device_attribute *attr,
+		     char *buf)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+	int trip, ret;
+	int temperature;
+
+	if (!tz->ops->get_trip_hyst)
+		return -EPERM;
+
+	if (sscanf(attr->attr.name, "trip_point_%d_hyst", &trip) != 1)
+		return -EINVAL;
+
+	ret = tz->ops->get_trip_hyst(tz, trip, &temperature);
+
+	return ret ? ret : sprintf(buf, "%d\n", temperature);
+}
+
+static ssize_t
+passive_store(struct device *dev, struct device_attribute *attr,
+	      const char *buf, size_t count)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+	int state;
+
+	if (sscanf(buf, "%d\n", &state) != 1)
+		return -EINVAL;
+
+	/* sanity check: values below 1000 millicelcius don't make sense
+	 * and can cause the system to go into a thermal heart attack
+	 */
+	if (state && state < 1000)
+		return -EINVAL;
+
+	if (state && !tz->forced_passive) {
+		if (!tz->passive_delay)
+			tz->passive_delay = 1000;
+		thermal_zone_device_rebind_exception(tz, "Processor",
+						     sizeof("Processor"));
+	} else if (!state && tz->forced_passive) {
+		tz->passive_delay = 0;
+		thermal_zone_device_unbind_exception(tz, "Processor",
+						     sizeof("Processor"));
+	}
+
+	tz->forced_passive = state;
+
+	thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
+
+	return count;
+}
+
+static ssize_t
+passive_show(struct device *dev, struct device_attribute *attr,
+	     char *buf)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+
+	return sprintf(buf, "%d\n", tz->forced_passive);
+}
+
+static ssize_t
+policy_store(struct device *dev, struct device_attribute *attr,
+	     const char *buf, size_t count)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+	char name[THERMAL_NAME_LENGTH];
+	int ret;
+
+	snprintf(name, sizeof(name), "%s", buf);
+
+	ret = thermal_zone_device_set_policy(tz, name);
+	if (!ret)
+		ret = count;
+
+	return ret;
+}
+
+static ssize_t
+policy_show(struct device *dev, struct device_attribute *devattr, char *buf)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+
+	return sprintf(buf, "%s\n", tz->governor->name);
+}
+
+static ssize_t
+available_policies_show(struct device *dev, struct device_attribute *devattr,
+			char *buf)
+{
+	return thermal_build_list_of_policies(buf);
+}
+
+#if (IS_ENABLED(CONFIG_THERMAL_EMULATION))
+static ssize_t
+emul_temp_store(struct device *dev, struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+	int ret = 0;
+	int temperature;
+
+	if (kstrtoint(buf, 10, &temperature))
+		return -EINVAL;
+
+	if (!tz->ops->set_emul_temp) {
+		mutex_lock(&tz->lock);
+		tz->emul_temperature = temperature;
+		mutex_unlock(&tz->lock);
+	} else {
+		ret = tz->ops->set_emul_temp(tz, temperature);
+	}
+
+	if (!ret)
+		thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
+
+	return ret ? ret : count;
+}
+static DEVICE_ATTR(emul_temp, S_IWUSR, NULL, emul_temp_store);
+#endif
+
+static ssize_t
+sustainable_power_show(struct device *dev, struct device_attribute *devattr,
+		       char *buf)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+
+	if (tz->tzp)
+		return sprintf(buf, "%u\n", tz->tzp->sustainable_power);
+	else
+		return -EIO;
+}
+
+static ssize_t
+sustainable_power_store(struct device *dev, struct device_attribute *devattr,
+			const char *buf, size_t count)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+	u32 sustainable_power;
+
+	if (!tz->tzp)
+		return -EIO;
+
+	if (kstrtou32(buf, 10, &sustainable_power))
+		return -EINVAL;
+
+	tz->tzp->sustainable_power = sustainable_power;
+
+	return count;
+}
+
+#define create_s32_tzp_attr(name)					\
+	static ssize_t							\
+	name##_show(struct device *dev, struct device_attribute *devattr, \
+		char *buf)						\
+	{								\
+	struct thermal_zone_device *tz = to_thermal_zone(dev);		\
+									\
+	if (tz->tzp)							\
+		return sprintf(buf, "%d\n", tz->tzp->name);		\
+	else								\
+		return -EIO;						\
+	}								\
+									\
+	static ssize_t							\
+	name##_store(struct device *dev, struct device_attribute *devattr, \
+		const char *buf, size_t count)				\
+	{								\
+		struct thermal_zone_device *tz = to_thermal_zone(dev);	\
+		s32 value;						\
+									\
+		if (!tz->tzp)						\
+			return -EIO;					\
+									\
+		if (kstrtos32(buf, 10, &value))				\
+			return -EINVAL;					\
+									\
+		tz->tzp->name = value;					\
+									\
+		return count;						\
+	}								\
+	static DEVICE_ATTR(name, S_IWUSR | S_IRUGO, name##_show, name##_store)
+
+create_s32_tzp_attr(k_po);
+create_s32_tzp_attr(k_pu);
+create_s32_tzp_attr(k_i);
+create_s32_tzp_attr(k_d);
+create_s32_tzp_attr(integral_cutoff);
+create_s32_tzp_attr(slope);
+create_s32_tzp_attr(offset);
+#undef create_s32_tzp_attr
+
+/*
+ * These are thermal zone device attributes that will always be present.
+ * All the attributes created for tzp (create_s32_tzp_attr) also are always
+ * present on the sysfs interface.
+ */
+static DEVICE_ATTR(type, 0444, type_show, NULL);
+static DEVICE_ATTR(temp, 0444, temp_show, NULL);
+static DEVICE_ATTR(policy, S_IRUGO | S_IWUSR, policy_show, policy_store);
+static DEVICE_ATTR(available_policies, S_IRUGO, available_policies_show, NULL);
+static DEVICE_ATTR(sustainable_power, S_IWUSR | S_IRUGO, sustainable_power_show,
+		   sustainable_power_store);
+
+/* These thermal zone device attributes are created based on conditions */
+static DEVICE_ATTR(mode, 0644, mode_show, mode_store);
+static DEVICE_ATTR(passive, S_IRUGO | S_IWUSR, passive_show, passive_store);
+
+/* These attributes are unconditionally added to a thermal zone */
+static struct attribute *thermal_zone_dev_attrs[] = {
+	&dev_attr_type.attr,
+	&dev_attr_temp.attr,
+#if (IS_ENABLED(CONFIG_THERMAL_EMULATION))
+	&dev_attr_emul_temp.attr,
+#endif
+	&dev_attr_policy.attr,
+	&dev_attr_available_policies.attr,
+	&dev_attr_sustainable_power.attr,
+	&dev_attr_k_po.attr,
+	&dev_attr_k_pu.attr,
+	&dev_attr_k_i.attr,
+	&dev_attr_k_d.attr,
+	&dev_attr_integral_cutoff.attr,
+	&dev_attr_slope.attr,
+	&dev_attr_offset.attr,
+	NULL,
+};
+
+static struct attribute_group thermal_zone_attribute_group = {
+	.attrs = thermal_zone_dev_attrs,
+};
+
+/* We expose mode only if .get_mode is present */
+static struct attribute *thermal_zone_mode_attrs[] = {
+	&dev_attr_mode.attr,
+	NULL,
+};
+
+static umode_t thermal_zone_mode_is_visible(struct kobject *kobj,
+					    struct attribute *attr,
+					    int attrno)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct thermal_zone_device *tz;
+
+	tz = container_of(dev, struct thermal_zone_device, device);
+
+	if (tz->ops->get_mode)
+		return attr->mode;
+
+	return 0;
+}
+
+static struct attribute_group thermal_zone_mode_attribute_group = {
+	.attrs = thermal_zone_mode_attrs,
+	.is_visible = thermal_zone_mode_is_visible,
+};
+
+/* We expose passive only if passive trips are present */
+static struct attribute *thermal_zone_passive_attrs[] = {
+	&dev_attr_passive.attr,
+	NULL,
+};
+
+static umode_t thermal_zone_passive_is_visible(struct kobject *kobj,
+					       struct attribute *attr,
+					       int attrno)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct thermal_zone_device *tz;
+	enum thermal_trip_type trip_type;
+	int count, passive = 0;
+
+	tz = container_of(dev, struct thermal_zone_device, device);
+
+	for (count = 0; count < tz->trips && !passive; count++) {
+		tz->ops->get_trip_type(tz, count, &trip_type);
+
+		if (trip_type == THERMAL_TRIP_PASSIVE)
+			passive = 1;
+	}
+
+	if (!passive)
+		return attr->mode;
+
+	return 0;
+}
+
+static struct attribute_group thermal_zone_passive_attribute_group = {
+	.attrs = thermal_zone_passive_attrs,
+	.is_visible = thermal_zone_passive_is_visible,
+};
+
+static const struct attribute_group *thermal_zone_attribute_groups[] = {
+	&thermal_zone_attribute_group,
+	&thermal_zone_mode_attribute_group,
+	&thermal_zone_passive_attribute_group,
+	/* This is not NULL terminated as we create the group dynamically */
+};
+
+/**
+ * create_trip_attrs() - create attributes for trip points
+ * @tz:		the thermal zone device
+ * @mask:	Writeable trip point bitmap.
+ *
+ * helper function to instantiate sysfs entries for every trip
+ * point and its properties of a struct thermal_zone_device.
+ *
+ * Return: 0 on success, the proper error value otherwise.
+ */
+static int create_trip_attrs(struct thermal_zone_device *tz, int mask)
+{
+	struct attribute **attrs;
+	int indx;
+
+	/* This function works only for zones with at least one trip */
+	if (tz->trips <= 0)
+		return -EINVAL;
+
+	tz->trip_type_attrs = kcalloc(tz->trips, sizeof(*tz->trip_type_attrs),
+				      GFP_KERNEL);
+	if (!tz->trip_type_attrs)
+		return -ENOMEM;
+
+	tz->trip_temp_attrs = kcalloc(tz->trips, sizeof(*tz->trip_temp_attrs),
+				      GFP_KERNEL);
+	if (!tz->trip_temp_attrs) {
+		kfree(tz->trip_type_attrs);
+		return -ENOMEM;
+	}
+
+	if (tz->ops->get_trip_hyst) {
+		tz->trip_hyst_attrs = kcalloc(tz->trips,
+					      sizeof(*tz->trip_hyst_attrs),
+					      GFP_KERNEL);
+		if (!tz->trip_hyst_attrs) {
+			kfree(tz->trip_type_attrs);
+			kfree(tz->trip_temp_attrs);
+			return -ENOMEM;
+		}
+	}
+
+	attrs = kcalloc(tz->trips * 3 + 1, sizeof(*attrs), GFP_KERNEL);
+	if (!attrs) {
+		kfree(tz->trip_type_attrs);
+		kfree(tz->trip_temp_attrs);
+		if (tz->ops->get_trip_hyst)
+			kfree(tz->trip_hyst_attrs);
+		return -ENOMEM;
+	}
+
+	for (indx = 0; indx < tz->trips; indx++) {
+		/* create trip type attribute */
+		snprintf(tz->trip_type_attrs[indx].name, THERMAL_NAME_LENGTH,
+			 "trip_point_%d_type", indx);
+
+		sysfs_attr_init(&tz->trip_type_attrs[indx].attr.attr);
+		tz->trip_type_attrs[indx].attr.attr.name =
+						tz->trip_type_attrs[indx].name;
+		tz->trip_type_attrs[indx].attr.attr.mode = S_IRUGO;
+		tz->trip_type_attrs[indx].attr.show = trip_point_type_show;
+		attrs[indx] = &tz->trip_type_attrs[indx].attr.attr;
+
+		/* create trip temp attribute */
+		snprintf(tz->trip_temp_attrs[indx].name, THERMAL_NAME_LENGTH,
+			 "trip_point_%d_temp", indx);
+
+		sysfs_attr_init(&tz->trip_temp_attrs[indx].attr.attr);
+		tz->trip_temp_attrs[indx].attr.attr.name =
+						tz->trip_temp_attrs[indx].name;
+		tz->trip_temp_attrs[indx].attr.attr.mode = S_IRUGO;
+		tz->trip_temp_attrs[indx].attr.show = trip_point_temp_show;
+		if (IS_ENABLED(CONFIG_THERMAL_WRITABLE_TRIPS) &&
+		    mask & (1 << indx)) {
+			tz->trip_temp_attrs[indx].attr.attr.mode |= S_IWUSR;
+			tz->trip_temp_attrs[indx].attr.store =
+							trip_point_temp_store;
+		}
+		attrs[indx + tz->trips] = &tz->trip_temp_attrs[indx].attr.attr;
+
+		/* create Optional trip hyst attribute */
+		if (!tz->ops->get_trip_hyst)
+			continue;
+		snprintf(tz->trip_hyst_attrs[indx].name, THERMAL_NAME_LENGTH,
+			 "trip_point_%d_hyst", indx);
+
+		sysfs_attr_init(&tz->trip_hyst_attrs[indx].attr.attr);
+		tz->trip_hyst_attrs[indx].attr.attr.name =
+					tz->trip_hyst_attrs[indx].name;
+		tz->trip_hyst_attrs[indx].attr.attr.mode = S_IRUGO;
+		tz->trip_hyst_attrs[indx].attr.show = trip_point_hyst_show;
+		if (tz->ops->set_trip_hyst) {
+			tz->trip_hyst_attrs[indx].attr.attr.mode |= S_IWUSR;
+			tz->trip_hyst_attrs[indx].attr.store =
+					trip_point_hyst_store;
+		}
+		attrs[indx + tz->trips * 2] =
+					&tz->trip_hyst_attrs[indx].attr.attr;
+	}
+	attrs[tz->trips * 3] = NULL;
+
+	tz->trips_attribute_group.attrs = attrs;
+
+	return 0;
+}
+
+int thermal_zone_create_device_groups(struct thermal_zone_device *tz,
+				      int mask)
+{
+	const struct attribute_group **groups;
+	int i, size, result;
+
+	/* we need one extra for trips and the NULL to terminate the array */
+	size = ARRAY_SIZE(thermal_zone_attribute_groups) + 2;
+	/* This also takes care of API requirement to be NULL terminated */
+	groups = kcalloc(size, sizeof(*groups), GFP_KERNEL);
+	if (!groups)
+		return -ENOMEM;
+
+	for (i = 0; i < size - 2; i++)
+		groups[i] = thermal_zone_attribute_groups[i];
+
+	if (tz->trips) {
+		result = create_trip_attrs(tz, mask);
+		if (result) {
+			kfree(groups);
+
+			return result;
+		}
+
+		groups[size - 2] = &tz->trips_attribute_group;
+	}
+
+	tz->device.groups = groups;
+
+	return 0;
+}
+
+/* sys I/F for cooling device */
+static ssize_t
+thermal_cooling_device_type_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct thermal_cooling_device *cdev = to_cooling_device(dev);
+
+	return sprintf(buf, "%s\n", cdev->type);
+}
+
+static ssize_t
+thermal_cooling_device_max_state_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct thermal_cooling_device *cdev = to_cooling_device(dev);
+	unsigned long state;
+	int ret;
+
+	ret = cdev->ops->get_max_state(cdev, &state);
+	if (ret)
+		return ret;
+	return sprintf(buf, "%ld\n", state);
+}
+
+static ssize_t
+thermal_cooling_device_cur_state_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct thermal_cooling_device *cdev = to_cooling_device(dev);
+	unsigned long state;
+	int ret;
+
+	ret = cdev->ops->get_cur_state(cdev, &state);
+	if (ret)
+		return ret;
+	return sprintf(buf, "%ld\n", state);
+}
+
+static ssize_t
+thermal_cooling_device_cur_state_store(struct device *dev,
+				       struct device_attribute *attr,
+				       const char *buf, size_t count)
+{
+	struct thermal_cooling_device *cdev = to_cooling_device(dev);
+	unsigned long state;
+	int result;
+
+	if (sscanf(buf, "%ld\n", &state) != 1)
+		return -EINVAL;
+
+	if ((long)state < 0)
+		return -EINVAL;
+
+	result = cdev->ops->set_cur_state(cdev, state);
+	if (result)
+		return result;
+	return count;
+}
+
+static struct device_attribute dev_attr_cdev_type =
+__ATTR(type, 0444, thermal_cooling_device_type_show, NULL);
+static DEVICE_ATTR(max_state, 0444,
+		   thermal_cooling_device_max_state_show, NULL);
+static DEVICE_ATTR(cur_state, 0644,
+		   thermal_cooling_device_cur_state_show,
+		   thermal_cooling_device_cur_state_store);
+
+static struct attribute *cooling_device_attrs[] = {
+	&dev_attr_cdev_type.attr,
+	&dev_attr_max_state.attr,
+	&dev_attr_cur_state.attr,
+	NULL,
+};
+
+static const struct attribute_group cooling_device_attr_group = {
+	.attrs = cooling_device_attrs,
+};
+
+static const struct attribute_group *cooling_device_attr_groups[] = {
+	&cooling_device_attr_group,
+	NULL,
+};
+
+void thermal_cooling_device_setup_sysfs(struct thermal_cooling_device *cdev)
+{
+	cdev->device.groups = cooling_device_attr_groups;
+}
+
+/* these helper will be used only at the time of bindig */
+ssize_t
+thermal_cooling_device_trip_point_show(struct device *dev,
+				       struct device_attribute *attr, char *buf)
+{
+	struct thermal_instance *instance;
+
+	instance =
+	    container_of(attr, struct thermal_instance, attr);
+
+	if (instance->trip == THERMAL_TRIPS_NONE)
+		return sprintf(buf, "-1\n");
+	else
+		return sprintf(buf, "%d\n", instance->trip);
+}
+
+ssize_t
+thermal_cooling_device_weight_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	struct thermal_instance *instance;
+
+	instance = container_of(attr, struct thermal_instance, weight_attr);
+
+	return sprintf(buf, "%d\n", instance->weight);
+}
+
+ssize_t
+thermal_cooling_device_weight_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	struct thermal_instance *instance;
+	int ret, weight;
+
+	ret = kstrtoint(buf, 0, &weight);
+	if (ret)
+		return ret;
+
+	instance = container_of(attr, struct thermal_instance, weight_attr);
+	instance->weight = weight;
+
+	return count;
+}
diff --git a/drivers/thermal/ti-soc-thermal/ti-bandgap.c b/drivers/thermal/ti-soc-thermal/ti-bandgap.c
index 06ea9766a70a..ba9c302454fb 100644
--- a/drivers/thermal/ti-soc-thermal/ti-bandgap.c
+++ b/drivers/thermal/ti-soc-thermal/ti-bandgap.c
@@ -1298,7 +1298,7 @@ int ti_bandgap_probe(struct platform_device *pdev)
 	if (IS_ERR(bgp->div_clk)) {
 		dev_err(&pdev->dev, "failed to request div_ts_ck clock ref\n");
 		ret = PTR_ERR(bgp->div_clk);
-		goto free_irqs;
+		goto put_fclock;
 	}
 
 	for (i = 0; i < bgp->conf->sensor_count; i++) {
@@ -1430,8 +1430,9 @@ disable_clk:
 	if (TI_BANDGAP_HAS(bgp, CLK_CTRL))
 		clk_disable_unprepare(bgp->fclock);
 put_clks:
-	clk_put(bgp->fclock);
 	clk_put(bgp->div_clk);
+put_fclock:
+	clk_put(bgp->fclock);
 free_irqs:
 	if (TI_BANDGAP_HAS(bgp, TSHUT)) {
 		free_irq(gpio_to_irq(bgp->tshut_gpio), NULL);
diff --git a/drivers/thermal/x86_pkg_temp_thermal.c b/drivers/thermal/x86_pkg_temp_thermal.c
index 95f4c1bcdb4c..d93eee2f101b 100644
--- a/drivers/thermal/x86_pkg_temp_thermal.c
+++ b/drivers/thermal/x86_pkg_temp_thermal.c
@@ -54,37 +54,33 @@ MODULE_PARM_DESC(notify_delay_ms,
 * is some wrong values returned by cpuid for number of thresholds.
 */
 #define MAX_NUMBER_OF_TRIPS	2
-/* Limit number of package temp zones */
-#define MAX_PKG_TEMP_ZONE_IDS	256
-
-struct phy_dev_entry {
-	struct list_head list;
-	u16 phys_proc_id;
-	u16 first_cpu;
-	u32 tj_max;
-	int ref_cnt;
-	u32 start_pkg_therm_low;
-	u32 start_pkg_therm_high;
-	struct thermal_zone_device *tzone;
+
+struct pkg_device {
+	int				cpu;
+	bool				work_scheduled;
+	u32				tj_max;
+	u32				msr_pkg_therm_low;
+	u32				msr_pkg_therm_high;
+	struct delayed_work		work;
+	struct thermal_zone_device	*tzone;
+	struct cpumask			cpumask;
 };
 
 static struct thermal_zone_params pkg_temp_tz_params = {
 	.no_hwmon	= true,
 };
 
-/* List maintaining number of package instances */
-static LIST_HEAD(phy_dev_list);
-static DEFINE_MUTEX(phy_dev_list_mutex);
-
-/* Interrupt to work function schedule queue */
-static DEFINE_PER_CPU(struct delayed_work, pkg_temp_thermal_threshold_work);
+/* Keep track of how many package pointers we allocated in init() */
+static int max_packages __read_mostly;
+/* Array of package pointers */
+static struct pkg_device **packages;
+/* Serializes interrupt notification, work and hotplug */
+static DEFINE_SPINLOCK(pkg_temp_lock);
+/* Protects zone operation in the work function against hotplug removal */
+static DEFINE_MUTEX(thermal_zone_mutex);
 
-/* To track if the work is already scheduled on a package */
-static u8 *pkg_work_scheduled;
-
-/* Spin lock to prevent races with pkg_work_scheduled */
-static spinlock_t pkg_work_lock;
-static u16 max_phy_id;
+/* The dynamically assigned cpu hotplug state for module_exit() */
+static enum cpuhp_state pkg_thermal_hp_state __read_mostly;
 
 /* Debug counters to show using debugfs */
 static struct dentry *debugfs;
@@ -116,22 +112,20 @@ err_out:
 	return -ENOENT;
 }
 
-static struct phy_dev_entry
-			*pkg_temp_thermal_get_phy_entry(unsigned int cpu)
+/*
+ * Protection:
+ *
+ * - cpu hotplug: Read serialized by cpu hotplug lock
+ *		  Write must hold pkg_temp_lock
+ *
+ * - Other callsites: Must hold pkg_temp_lock
+ */
+static struct pkg_device *pkg_temp_thermal_get_dev(unsigned int cpu)
 {
-	u16 phys_proc_id = topology_physical_package_id(cpu);
-	struct phy_dev_entry *phy_ptr;
-
-	mutex_lock(&phy_dev_list_mutex);
-
-	list_for_each_entry(phy_ptr, &phy_dev_list, list)
-		if (phy_ptr->phys_proc_id == phys_proc_id) {
-			mutex_unlock(&phy_dev_list_mutex);
-			return phy_ptr;
-		}
-
-	mutex_unlock(&phy_dev_list_mutex);
+	int pkgid = topology_logical_package_id(cpu);
 
+	if (pkgid >= 0 && pkgid < max_packages)
+		return packages[pkgid];
 	return NULL;
 }
 
@@ -141,61 +135,44 @@ static struct phy_dev_entry
 */
 static int get_tj_max(int cpu, u32 *tj_max)
 {
-	u32 eax, edx;
-	u32 val;
+	u32 eax, edx, val;
 	int err;
 
 	err = rdmsr_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &eax, &edx);
 	if (err)
-		goto err_ret;
-	else {
-		val = (eax >> 16) & 0xff;
-		if (val)
-			*tj_max = val * 1000;
-		else {
-			err = -EINVAL;
-			goto err_ret;
-		}
-	}
+		return err;
 
-	return 0;
-err_ret:
-	*tj_max = 0;
-	return err;
+	val = (eax >> 16) & 0xff;
+	*tj_max = val * 1000;
+
+	return val ? 0 : -EINVAL;
 }
 
 static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
 {
+	struct pkg_device *pkgdev = tzd->devdata;
 	u32 eax, edx;
-	struct phy_dev_entry *phy_dev_entry;
 
-	phy_dev_entry = tzd->devdata;
-	rdmsr_on_cpu(phy_dev_entry->first_cpu, MSR_IA32_PACKAGE_THERM_STATUS,
-			&eax, &edx);
+	rdmsr_on_cpu(pkgdev->cpu, MSR_IA32_PACKAGE_THERM_STATUS, &eax, &edx);
 	if (eax & 0x80000000) {
-		*temp = phy_dev_entry->tj_max -
-				((eax >> 16) & 0x7f) * 1000;
+		*temp = pkgdev->tj_max - ((eax >> 16) & 0x7f) * 1000;
 		pr_debug("sys_get_curr_temp %d\n", *temp);
 		return 0;
 	}
-
 	return -EINVAL;
 }
 
 static int sys_get_trip_temp(struct thermal_zone_device *tzd,
-		int trip, int *temp)
+			     int trip, int *temp)
 {
-	u32 eax, edx;
-	struct phy_dev_entry *phy_dev_entry;
-	u32 mask, shift;
+	struct pkg_device *pkgdev = tzd->devdata;
 	unsigned long thres_reg_value;
+	u32 mask, shift, eax, edx;
 	int ret;
 
 	if (trip >= MAX_NUMBER_OF_TRIPS)
 		return -EINVAL;
 
-	phy_dev_entry = tzd->devdata;
-
 	if (trip) {
 		mask = THERM_MASK_THRESHOLD1;
 		shift = THERM_SHIFT_THRESHOLD1;
@@ -204,14 +181,14 @@ static int sys_get_trip_temp(struct thermal_zone_device *tzd,
 		shift = THERM_SHIFT_THRESHOLD0;
 	}
 
-	ret = rdmsr_on_cpu(phy_dev_entry->first_cpu,
-				MSR_IA32_PACKAGE_THERM_INTERRUPT, &eax, &edx);
+	ret = rdmsr_on_cpu(pkgdev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
+			   &eax, &edx);
 	if (ret < 0)
-		return -EINVAL;
+		return ret;
 
 	thres_reg_value = (eax & mask) >> shift;
 	if (thres_reg_value)
-		*temp = phy_dev_entry->tj_max - thres_reg_value * 1000;
+		*temp = pkgdev->tj_max - thres_reg_value * 1000;
 	else
 		*temp = 0;
 	pr_debug("sys_get_trip_temp %d\n", *temp);
@@ -219,24 +196,20 @@ static int sys_get_trip_temp(struct thermal_zone_device *tzd,
 	return 0;
 }
 
-static int sys_set_trip_temp(struct thermal_zone_device *tzd, int trip,
-							int temp)
+static int
+sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp)
 {
-	u32 l, h;
-	struct phy_dev_entry *phy_dev_entry;
-	u32 mask, shift, intr;
+	struct pkg_device *pkgdev = tzd->devdata;
+	u32 l, h, mask, shift, intr;
 	int ret;
 
-	phy_dev_entry = tzd->devdata;
-
-	if (trip >= MAX_NUMBER_OF_TRIPS || temp >= phy_dev_entry->tj_max)
+	if (trip >= MAX_NUMBER_OF_TRIPS || temp >= pkgdev->tj_max)
 		return -EINVAL;
 
-	ret = rdmsr_on_cpu(phy_dev_entry->first_cpu,
-					MSR_IA32_PACKAGE_THERM_INTERRUPT,
-					&l, &h);
+	ret = rdmsr_on_cpu(pkgdev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
+			   &l, &h);
 	if (ret < 0)
-		return -EINVAL;
+		return ret;
 
 	if (trip) {
 		mask = THERM_MASK_THRESHOLD1;
@@ -252,24 +225,20 @@ static int sys_set_trip_temp(struct thermal_zone_device *tzd, int trip,
 	* When users space sets a trip temperature == 0, which is indication
 	* that, it is no longer interested in receiving notifications.
 	*/
-	if (!temp)
+	if (!temp) {
 		l &= ~intr;
-	else {
-		l |= (phy_dev_entry->tj_max - temp)/1000 << shift;
+	} else {
+		l |= (pkgdev->tj_max - temp)/1000 << shift;
 		l |= intr;
 	}
 
-	return wrmsr_on_cpu(phy_dev_entry->first_cpu,
-					MSR_IA32_PACKAGE_THERM_INTERRUPT,
-					l, h);
+	return wrmsr_on_cpu(pkgdev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
 }
 
-static int sys_get_trip_type(struct thermal_zone_device *thermal,
-		int trip, enum thermal_trip_type *type)
+static int sys_get_trip_type(struct thermal_zone_device *thermal, int trip,
+			     enum thermal_trip_type *type)
 {
-
 	*type = THERMAL_TRIP_PASSIVE;
-
 	return 0;
 }
 
@@ -281,7 +250,7 @@ static struct thermal_zone_device_ops tzone_ops = {
 	.set_trip_temp = sys_set_trip_temp,
 };
 
-static bool pkg_temp_thermal_platform_thermal_rate_control(void)
+static bool pkg_thermal_rate_control(void)
 {
 	return true;
 }
@@ -289,8 +258,8 @@ static bool pkg_temp_thermal_platform_thermal_rate_control(void)
 /* Enable threshold interrupt on local package/cpu */
 static inline void enable_pkg_thres_interrupt(void)
 {
-	u32 l, h;
 	u8 thres_0, thres_1;
+	u32 l, h;
 
 	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
 	/* only enable/disable if it had valid threshold value */
@@ -307,271 +276,232 @@ static inline void enable_pkg_thres_interrupt(void)
 static inline void disable_pkg_thres_interrupt(void)
 {
 	u32 l, h;
+
 	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
-	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
-			l & (~THERM_INT_THRESHOLD0_ENABLE) &
-				(~THERM_INT_THRESHOLD1_ENABLE), h);
+
+	l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
+	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
 }
 
 static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
 {
-	__u64 msr_val;
+	struct thermal_zone_device *tzone = NULL;
 	int cpu = smp_processor_id();
-	int phy_id = topology_physical_package_id(cpu);
-	struct phy_dev_entry *phdev = pkg_temp_thermal_get_phy_entry(cpu);
-	bool notify = false;
-	unsigned long flags;
-
-	if (!phdev)
-		return;
+	struct pkg_device *pkgdev;
+	u64 msr_val, wr_val;
 
-	spin_lock_irqsave(&pkg_work_lock, flags);
+	mutex_lock(&thermal_zone_mutex);
+	spin_lock_irq(&pkg_temp_lock);
 	++pkg_work_cnt;
-	if (unlikely(phy_id > max_phy_id)) {
-		spin_unlock_irqrestore(&pkg_work_lock, flags);
+
+	pkgdev = pkg_temp_thermal_get_dev(cpu);
+	if (!pkgdev) {
+		spin_unlock_irq(&pkg_temp_lock);
+		mutex_unlock(&thermal_zone_mutex);
 		return;
 	}
-	pkg_work_scheduled[phy_id] = 0;
-	spin_unlock_irqrestore(&pkg_work_lock, flags);
+	pkgdev->work_scheduled = false;
 
-	enable_pkg_thres_interrupt();
 	rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val);
-	if (msr_val & THERM_LOG_THRESHOLD0) {
-		wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS,
-				msr_val & ~THERM_LOG_THRESHOLD0);
-		notify = true;
-	}
-	if (msr_val & THERM_LOG_THRESHOLD1) {
-		wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS,
-				msr_val & ~THERM_LOG_THRESHOLD1);
-		notify = true;
-	}
-	if (notify) {
-		pr_debug("thermal_zone_device_update\n");
-		thermal_zone_device_update(phdev->tzone,
-					   THERMAL_EVENT_UNSPECIFIED);
+	wr_val = msr_val & ~(THERM_LOG_THRESHOLD0 | THERM_LOG_THRESHOLD1);
+	if (wr_val != msr_val) {
+		wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS, wr_val);
+		tzone = pkgdev->tzone;
 	}
+
+	enable_pkg_thres_interrupt();
+	spin_unlock_irq(&pkg_temp_lock);
+
+	/*
+	 * If tzone is not NULL, then thermal_zone_mutex will prevent the
+	 * concurrent removal in the cpu offline callback.
+	 */
+	if (tzone)
+		thermal_zone_device_update(tzone, THERMAL_EVENT_UNSPECIFIED);
+
+	mutex_unlock(&thermal_zone_mutex);
 }
 
-static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val)
+static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
+{
+	unsigned long ms = msecs_to_jiffies(notify_delay_ms);
+
+	schedule_delayed_work_on(cpu, work, ms);
+}
+
+static int pkg_thermal_notify(u64 msr_val)
 {
-	unsigned long flags;
 	int cpu = smp_processor_id();
-	int phy_id = topology_physical_package_id(cpu);
+	struct pkg_device *pkgdev;
+	unsigned long flags;
 
-	/*
-	* When a package is in interrupted state, all CPU's in that package
-	* are in the same interrupt state. So scheduling on any one CPU in
-	* the package is enough and simply return for others.
-	*/
-	spin_lock_irqsave(&pkg_work_lock, flags);
+	spin_lock_irqsave(&pkg_temp_lock, flags);
 	++pkg_interrupt_cnt;
-	if (unlikely(phy_id > max_phy_id) || unlikely(!pkg_work_scheduled) ||
-			pkg_work_scheduled[phy_id]) {
-		disable_pkg_thres_interrupt();
-		spin_unlock_irqrestore(&pkg_work_lock, flags);
-		return -EINVAL;
-	}
-	pkg_work_scheduled[phy_id] = 1;
-	spin_unlock_irqrestore(&pkg_work_lock, flags);
 
 	disable_pkg_thres_interrupt();
-	schedule_delayed_work_on(cpu,
-				&per_cpu(pkg_temp_thermal_threshold_work, cpu),
-				msecs_to_jiffies(notify_delay_ms));
-	return 0;
-}
 
-static int find_siblings_cpu(int cpu)
-{
-	int i;
-	int id = topology_physical_package_id(cpu);
-
-	for_each_online_cpu(i)
-		if (i != cpu && topology_physical_package_id(i) == id)
-			return i;
+	/* Work is per package, so scheduling it once is enough. */
+	pkgdev = pkg_temp_thermal_get_dev(cpu);
+	if (pkgdev && !pkgdev->work_scheduled) {
+		pkgdev->work_scheduled = true;
+		pkg_thermal_schedule_work(pkgdev->cpu, &pkgdev->work);
+	}
 
+	spin_unlock_irqrestore(&pkg_temp_lock, flags);
 	return 0;
 }
 
 static int pkg_temp_thermal_device_add(unsigned int cpu)
 {
-	int err;
-	u32 tj_max;
-	struct phy_dev_entry *phy_dev_entry;
-	int thres_count;
-	u32 eax, ebx, ecx, edx;
-	u8 *temp;
-	unsigned long flags;
+	int pkgid = topology_logical_package_id(cpu);
+	u32 tj_max, eax, ebx, ecx, edx;
+	struct pkg_device *pkgdev;
+	int thres_count, err;
+
+	if (pkgid >= max_packages)
+		return -ENOMEM;
 
 	cpuid(6, &eax, &ebx, &ecx, &edx);
 	thres_count = ebx & 0x07;
 	if (!thres_count)
 		return -ENODEV;
 
-	if (topology_physical_package_id(cpu) > MAX_PKG_TEMP_ZONE_IDS)
-		return -ENODEV;
-
 	thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS);
 
 	err = get_tj_max(cpu, &tj_max);
 	if (err)
-		goto err_ret;
-
-	mutex_lock(&phy_dev_list_mutex);
+		return err;
 
-	phy_dev_entry = kzalloc(sizeof(*phy_dev_entry), GFP_KERNEL);
-	if (!phy_dev_entry) {
-		err = -ENOMEM;
-		goto err_ret_unlock;
-	}
+	pkgdev = kzalloc(sizeof(*pkgdev), GFP_KERNEL);
+	if (!pkgdev)
+		return -ENOMEM;
 
-	spin_lock_irqsave(&pkg_work_lock, flags);
-	if (topology_physical_package_id(cpu) > max_phy_id)
-		max_phy_id = topology_physical_package_id(cpu);
-	temp = krealloc(pkg_work_scheduled,
-			(max_phy_id+1) * sizeof(u8), GFP_ATOMIC);
-	if (!temp) {
-		spin_unlock_irqrestore(&pkg_work_lock, flags);
-		err = -ENOMEM;
-		goto err_ret_free;
-	}
-	pkg_work_scheduled = temp;
-	pkg_work_scheduled[topology_physical_package_id(cpu)] = 0;
-	spin_unlock_irqrestore(&pkg_work_lock, flags);
-
-	phy_dev_entry->phys_proc_id = topology_physical_package_id(cpu);
-	phy_dev_entry->first_cpu = cpu;
-	phy_dev_entry->tj_max = tj_max;
-	phy_dev_entry->ref_cnt = 1;
-	phy_dev_entry->tzone = thermal_zone_device_register("x86_pkg_temp",
+	INIT_DELAYED_WORK(&pkgdev->work, pkg_temp_thermal_threshold_work_fn);
+	pkgdev->cpu = cpu;
+	pkgdev->tj_max = tj_max;
+	pkgdev->tzone = thermal_zone_device_register("x86_pkg_temp",
 			thres_count,
-			(thres_count == MAX_NUMBER_OF_TRIPS) ?
-				0x03 : 0x01,
-			phy_dev_entry, &tzone_ops, &pkg_temp_tz_params, 0, 0);
-	if (IS_ERR(phy_dev_entry->tzone)) {
-		err = PTR_ERR(phy_dev_entry->tzone);
-		goto err_ret_free;
+			(thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01,
+			pkgdev, &tzone_ops, &pkg_temp_tz_params, 0, 0);
+	if (IS_ERR(pkgdev->tzone)) {
+		err = PTR_ERR(pkgdev->tzone);
+		kfree(pkgdev);
+		return err;
 	}
 	/* Store MSR value for package thermal interrupt, to restore at exit */
-	rdmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
-				&phy_dev_entry->start_pkg_therm_low,
-				&phy_dev_entry->start_pkg_therm_high);
-
-	list_add_tail(&phy_dev_entry->list, &phy_dev_list);
-	pr_debug("pkg_temp_thermal_device_add :phy_id %d cpu %d\n",
-			phy_dev_entry->phys_proc_id, cpu);
-
-	mutex_unlock(&phy_dev_list_mutex);
+	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, pkgdev->msr_pkg_therm_low,
+	      pkgdev->msr_pkg_therm_high);
 
+	cpumask_set_cpu(cpu, &pkgdev->cpumask);
+	spin_lock_irq(&pkg_temp_lock);
+	packages[pkgid] = pkgdev;
+	spin_unlock_irq(&pkg_temp_lock);
 	return 0;
-
-err_ret_free:
-	kfree(phy_dev_entry);
-err_ret_unlock:
-	mutex_unlock(&phy_dev_list_mutex);
-
-err_ret:
-	return err;
 }
 
-static int pkg_temp_thermal_device_remove(unsigned int cpu)
+static int pkg_thermal_cpu_offline(unsigned int cpu)
 {
-	struct phy_dev_entry *n;
-	u16 phys_proc_id = topology_physical_package_id(cpu);
-	struct phy_dev_entry *phdev =
-			pkg_temp_thermal_get_phy_entry(cpu);
+	struct pkg_device *pkgdev = pkg_temp_thermal_get_dev(cpu);
+	bool lastcpu, was_target;
+	int target;
 
-	if (!phdev)
-		return -ENODEV;
+	if (!pkgdev)
+		return 0;
 
-	mutex_lock(&phy_dev_list_mutex);
-	/* If we are loosing the first cpu for this package, we need change */
-	if (phdev->first_cpu == cpu) {
-		phdev->first_cpu = find_siblings_cpu(cpu);
-		pr_debug("thermal_device_remove: first cpu switched %d\n",
-					phdev->first_cpu);
+	target = cpumask_any_but(&pkgdev->cpumask, cpu);
+	cpumask_clear_cpu(cpu, &pkgdev->cpumask);
+	lastcpu = target >= nr_cpu_ids;
+	/*
+	 * Remove the sysfs files, if this is the last cpu in the package
+	 * before doing further cleanups.
+	 */
+	if (lastcpu) {
+		struct thermal_zone_device *tzone = pkgdev->tzone;
+
+		/*
+		 * We must protect against a work function calling
+		 * thermal_zone_update, after/while unregister. We null out
+		 * the pointer under the zone mutex, so the worker function
+		 * won't try to call.
+		 */
+		mutex_lock(&thermal_zone_mutex);
+		pkgdev->tzone = NULL;
+		mutex_unlock(&thermal_zone_mutex);
+
+		thermal_zone_device_unregister(tzone);
 	}
+
+	/* Protect against work and interrupts */
+	spin_lock_irq(&pkg_temp_lock);
+
 	/*
-	* It is possible that no siblings left as this was the last cpu
-	* going offline. We don't need to worry about this assignment
-	* as the phydev entry will be removed in this case and
-	* thermal zone is removed.
-	*/
-	--phdev->ref_cnt;
-	pr_debug("thermal_device_remove: pkg: %d cpu %d ref_cnt %d\n",
-					phys_proc_id, cpu, phdev->ref_cnt);
-	if (!phdev->ref_cnt)
-		list_for_each_entry_safe(phdev, n, &phy_dev_list, list) {
-			if (phdev->phys_proc_id == phys_proc_id) {
-				thermal_zone_device_unregister(phdev->tzone);
-				list_del(&phdev->list);
-				kfree(phdev);
-				break;
-			}
-		}
-	mutex_unlock(&phy_dev_list_mutex);
+	 * Check whether this cpu was the current target and store the new
+	 * one. When we drop the lock, then the interrupt notify function
+	 * will see the new target.
+	 */
+	was_target = pkgdev->cpu == cpu;
+	pkgdev->cpu = target;
 
-	return 0;
-}
+	/*
+	 * If this is the last CPU in the package remove the package
+	 * reference from the array and restore the interrupt MSR. When we
+	 * drop the lock neither the interrupt notify function nor the
+	 * worker will see the package anymore.
+	 */
+	if (lastcpu) {
+		packages[topology_logical_package_id(cpu)] = NULL;
+		/* After this point nothing touches the MSR anymore. */
+		wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
+		      pkgdev->msr_pkg_therm_low, pkgdev->msr_pkg_therm_high);
+	}
 
-static int get_core_online(unsigned int cpu)
-{
-	struct cpuinfo_x86 *c = &cpu_data(cpu);
-	struct phy_dev_entry *phdev = pkg_temp_thermal_get_phy_entry(cpu);
-
-	/* Check if there is already an instance for this package */
-	if (!phdev) {
-		if (!cpu_has(c, X86_FEATURE_DTHERM) ||
-					!cpu_has(c, X86_FEATURE_PTS))
-			return -ENODEV;
-		if (pkg_temp_thermal_device_add(cpu))
-			return -ENODEV;
-	} else {
-		mutex_lock(&phy_dev_list_mutex);
-		++phdev->ref_cnt;
-		pr_debug("get_core_online: cpu %d ref_cnt %d\n",
-						cpu, phdev->ref_cnt);
-		mutex_unlock(&phy_dev_list_mutex);
+	/*
+	 * Check whether there is work scheduled and whether the work is
+	 * targeted at the outgoing CPU.
+	 */
+	if (pkgdev->work_scheduled && was_target) {
+		/*
+		 * To cancel the work we need to drop the lock, otherwise
+		 * we might deadlock if the work needs to be flushed.
+		 */
+		spin_unlock_irq(&pkg_temp_lock);
+		cancel_delayed_work_sync(&pkgdev->work);
+		spin_lock_irq(&pkg_temp_lock);
+		/*
+		 * If this is not the last cpu in the package and the work
+		 * did not run after we dropped the lock above, then we
+		 * need to reschedule the work, otherwise the interrupt
+		 * stays disabled forever.
+		 */
+		if (!lastcpu && pkgdev->work_scheduled)
+			pkg_thermal_schedule_work(target, &pkgdev->work);
 	}
-	INIT_DELAYED_WORK(&per_cpu(pkg_temp_thermal_threshold_work, cpu),
-			pkg_temp_thermal_threshold_work_fn);
 
-	pr_debug("get_core_online: cpu %d successful\n", cpu);
+	spin_unlock_irq(&pkg_temp_lock);
 
+	/* Final cleanup if this is the last cpu */
+	if (lastcpu)
+		kfree(pkgdev);
 	return 0;
 }
 
-static void put_core_offline(unsigned int cpu)
+static int pkg_thermal_cpu_online(unsigned int cpu)
 {
-	if (!pkg_temp_thermal_device_remove(cpu))
-		cancel_delayed_work_sync(
-			&per_cpu(pkg_temp_thermal_threshold_work, cpu));
+	struct pkg_device *pkgdev = pkg_temp_thermal_get_dev(cpu);
+	struct cpuinfo_x86 *c = &cpu_data(cpu);
 
-	pr_debug("put_core_offline: cpu %d\n", cpu);
-}
+	/* Paranoia check */
+	if (!cpu_has(c, X86_FEATURE_DTHERM) || !cpu_has(c, X86_FEATURE_PTS))
+		return -ENODEV;
 
-static int pkg_temp_thermal_cpu_callback(struct notifier_block *nfb,
-				 unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long) hcpu;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		get_core_online(cpu);
-		break;
-	case CPU_DOWN_PREPARE:
-		put_core_offline(cpu);
-		break;
+	/* If the package exists, nothing to do */
+	if (pkgdev) {
+		cpumask_set_cpu(cpu, &pkgdev->cpumask);
+		return 0;
 	}
-	return NOTIFY_OK;
+	return pkg_temp_thermal_device_add(cpu);
 }
 
-static struct notifier_block pkg_temp_thermal_notifier __refdata = {
-	.notifier_call = pkg_temp_thermal_cpu_callback,
-};
-
 static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
 	{ X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_PTS },
 	{}
@@ -580,71 +510,46 @@ MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);
 
 static int __init pkg_temp_thermal_init(void)
 {
-	int i;
+	int ret;
 
 	if (!x86_match_cpu(pkg_temp_thermal_ids))
 		return -ENODEV;
 
-	spin_lock_init(&pkg_work_lock);
-	platform_thermal_package_notify =
-			pkg_temp_thermal_platform_thermal_notify;
-	platform_thermal_package_rate_control =
-			pkg_temp_thermal_platform_thermal_rate_control;
+	max_packages = topology_max_packages();
+	packages = kzalloc(max_packages * sizeof(struct pkg_device *), GFP_KERNEL);
+	if (!packages)
+		return -ENOMEM;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i)
-		if (get_core_online(i))
-			goto err_ret;
-	__register_hotcpu_notifier(&pkg_temp_thermal_notifier);
-	cpu_notifier_register_done();
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
+				pkg_thermal_cpu_online,	pkg_thermal_cpu_offline);
+	if (ret < 0)
+		goto err;
 
-	pkg_temp_debugfs_init(); /* Don't care if fails */
+	/* Store the state for module exit */
+	pkg_thermal_hp_state = ret;
 
-	return 0;
+	platform_thermal_package_notify = pkg_thermal_notify;
+	platform_thermal_package_rate_control = pkg_thermal_rate_control;
 
-err_ret:
-	for_each_online_cpu(i)
-		put_core_offline(i);
-	cpu_notifier_register_done();
-	kfree(pkg_work_scheduled);
-	platform_thermal_package_notify = NULL;
-	platform_thermal_package_rate_control = NULL;
+	 /* Don't care if it fails */
+	pkg_temp_debugfs_init();
+	return 0;
 
-	return -ENODEV;
+err:
+	kfree(packages);
+	return ret;
 }
+module_init(pkg_temp_thermal_init)
 
 static void __exit pkg_temp_thermal_exit(void)
 {
-	struct phy_dev_entry *phdev, *n;
-	int i;
-
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&pkg_temp_thermal_notifier);
-	mutex_lock(&phy_dev_list_mutex);
-	list_for_each_entry_safe(phdev, n, &phy_dev_list, list) {
-		/* Retore old MSR value for package thermal interrupt */
-		wrmsr_on_cpu(phdev->first_cpu,
-			MSR_IA32_PACKAGE_THERM_INTERRUPT,
-			phdev->start_pkg_therm_low,
-			phdev->start_pkg_therm_high);
-		thermal_zone_device_unregister(phdev->tzone);
-		list_del(&phdev->list);
-		kfree(phdev);
-	}
-	mutex_unlock(&phy_dev_list_mutex);
 	platform_thermal_package_notify = NULL;
 	platform_thermal_package_rate_control = NULL;
-	for_each_online_cpu(i)
-		cancel_delayed_work_sync(
-			&per_cpu(pkg_temp_thermal_threshold_work, i));
-	cpu_notifier_register_done();
-
-	kfree(pkg_work_scheduled);
 
+	cpuhp_remove_state(pkg_thermal_hp_state);
 	debugfs_remove_recursive(debugfs);
+	kfree(packages);
 }
-
-module_init(pkg_temp_thermal_init)
 module_exit(pkg_temp_thermal_exit)
 
 MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver");
diff --git a/drivers/thunderbolt/nhi_regs.h b/drivers/thunderbolt/nhi_regs.h
index 86b996c702a0..75cf0691e6c5 100644
--- a/drivers/thunderbolt/nhi_regs.h
+++ b/drivers/thunderbolt/nhi_regs.h
@@ -1,11 +1,11 @@
 /*
- * Thunderbolt Cactus Ridge driver - NHI registers
+ * Thunderbolt driver - NHI registers
  *
  * Copyright (c) 2014 Andreas Noever <andreas.noever@gmail.com>
  */
 
-#ifndef DSL3510_REGS_H_
-#define DSL3510_REGS_H_
+#ifndef NHI_REGS_H_
+#define NHI_REGS_H_
 
 #include <linux/types.h>
 
diff --git a/drivers/tty/amiserial.c b/drivers/tty/amiserial.c
index 208f573495dc..dfbb974927f2 100644
--- a/drivers/tty/amiserial.c
+++ b/drivers/tty/amiserial.c
@@ -1012,8 +1012,6 @@ static int get_serial_info(struct tty_struct *tty, struct serial_state *state,
 {
 	struct serial_struct tmp;
    
-	if (!retinfo)
-		return -EFAULT;
 	memset(&tmp, 0, sizeof(tmp));
 	tty_lock(tty);
 	tmp.line = tty->index;
diff --git a/drivers/tty/nozomi.c b/drivers/tty/nozomi.c
index d6fd0e802ef5..39b3723a32a6 100644
--- a/drivers/tty/nozomi.c
+++ b/drivers/tty/nozomi.c
@@ -63,44 +63,23 @@
 
 #define VERSION_STRING DRIVER_DESC " 2.1d"
 
-/*    Macros definitions */
-
 /* Default debug printout level */
 #define NOZOMI_DEBUG_LEVEL 0x00
-
-#define P_BUF_SIZE 128
-#define NFO(_err_flag_, args...)				\
-do {								\
-	char tmp[P_BUF_SIZE];					\
-	snprintf(tmp, sizeof(tmp), ##args);			\
-	printk(_err_flag_ "[%d] %s(): %s\n", __LINE__,		\
-		__func__, tmp);				\
-} while (0)
-
-#define DBG1(args...) D_(0x01, ##args)
-#define DBG2(args...) D_(0x02, ##args)
-#define DBG3(args...) D_(0x04, ##args)
-#define DBG4(args...) D_(0x08, ##args)
-#define DBG5(args...) D_(0x10, ##args)
-#define DBG6(args...) D_(0x20, ##args)
-#define DBG7(args...) D_(0x40, ##args)
-#define DBG8(args...) D_(0x80, ##args)
-
-#ifdef DEBUG
-/* Do we need this settable at runtime? */
 static int debug = NOZOMI_DEBUG_LEVEL;
+module_param(debug, int, S_IRUGO | S_IWUSR);
 
-#define D(lvl, args...)  do \
-			{if (lvl & debug) NFO(KERN_DEBUG, ##args); } \
-			while (0)
-#define D_(lvl, args...) D(lvl, ##args)
-
-/* These printouts are always printed */
+/*    Macros definitions */
+#define DBG_(lvl, fmt, args...)				\
+do {							\
+	if (lvl & debug)				\
+		pr_debug("[%d] %s(): " fmt "\n",	\
+			 __LINE__, __func__,  ##args);	\
+} while (0)
 
-#else
-static int debug;
-#define D_(lvl, args...)
-#endif
+#define DBG1(args...) DBG_(0x01, ##args)
+#define DBG2(args...) DBG_(0x02, ##args)
+#define DBG3(args...) DBG_(0x04, ##args)
+#define DBG4(args...) DBG_(0x08, ##args)
 
 /* TODO: rewrite to optimize macros... */
 
@@ -1320,7 +1299,7 @@ static ssize_t card_type_show(struct device *dev, struct device_attribute *attr,
 
 	return sprintf(buf, "%d\n", dc->card_type);
 }
-static DEVICE_ATTR(card_type, S_IRUGO, card_type_show, NULL);
+static DEVICE_ATTR_RO(card_type);
 
 static ssize_t open_ttys_show(struct device *dev, struct device_attribute *attr,
 			  char *buf)
@@ -1329,7 +1308,7 @@ static ssize_t open_ttys_show(struct device *dev, struct device_attribute *attr,
 
 	return sprintf(buf, "%u\n", dc->open_ttys);
 }
-static DEVICE_ATTR(open_ttys, S_IRUGO, open_ttys_show, NULL);
+static DEVICE_ATTR_RO(open_ttys);
 
 static void make_sysfs_files(struct nozomi *dc)
 {
@@ -1943,7 +1922,5 @@ static __exit void nozomi_exit(void)
 module_init(nozomi_init);
 module_exit(nozomi_exit);
 
-module_param(debug, int, S_IRUGO | S_IWUSR);
-
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION(DRIVER_DESC);
diff --git a/drivers/tty/rocket.c b/drivers/tty/rocket.c
index b0cc47c77b40..d66c1edd9892 100644
--- a/drivers/tty/rocket.c
+++ b/drivers/tty/rocket.c
@@ -1189,8 +1189,6 @@ static int get_config(struct r_port *info, struct rocket_config __user *retinfo)
 {
 	struct rocket_config tmp;
 
-	if (!retinfo)
-		return -EFAULT;
 	memset(&tmp, 0, sizeof (tmp));
 	mutex_lock(&info->port.mutex);
 	tmp.line = info->line;
@@ -1255,8 +1253,6 @@ static int get_ports(struct r_port *info, struct rocket_ports __user *retports)
 	struct rocket_ports tmp;
 	int board;
 
-	if (!retports)
-		return -EFAULT;
 	memset(&tmp, 0, sizeof (tmp));
 	tmp.tty_major = rocket_driver->major;
 
diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h
index a697a8585ddc..ce8d4ffcc425 100644
--- a/drivers/tty/serial/8250/8250.h
+++ b/drivers/tty/serial/8250/8250.h
@@ -80,6 +80,7 @@ struct serial8250_config {
 #define UART_CAP_RTOIE	(1 << 13)	/* UART needs IER bit 4 set (Xscale, Tegra) */
 #define UART_CAP_HFIFO	(1 << 14)	/* UART has a "hidden" FIFO */
 #define UART_CAP_RPM	(1 << 15)	/* Runtime PM is active while idle */
+#define UART_CAP_IRDA	(1 << 16)	/* UART supports IrDA line discipline */
 
 #define UART_BUG_QUOT	(1 << 0)	/* UART has buggy quot LSB */
 #define UART_BUG_TXEN	(1 << 1)	/* UART has buggy TX IIR status */
@@ -129,8 +130,13 @@ static inline void serial_dl_write(struct uart_8250_port *up, int value)
 }
 
 struct uart_8250_port *serial8250_get_port(int line);
+
 void serial8250_rpm_get(struct uart_8250_port *p);
 void serial8250_rpm_put(struct uart_8250_port *p);
+
+void serial8250_rpm_get_tx(struct uart_8250_port *p);
+void serial8250_rpm_put_tx(struct uart_8250_port *p);
+
 int serial8250_em485_init(struct uart_8250_port *p);
 void serial8250_em485_destroy(struct uart_8250_port *p);
 
diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c
index 240a361b674f..61569a765d9e 100644
--- a/drivers/tty/serial/8250/8250_core.c
+++ b/drivers/tty/serial/8250/8250_core.c
@@ -425,10 +425,10 @@ struct uart_8250_port *serial8250_get_port(int line)
 EXPORT_SYMBOL_GPL(serial8250_get_port);
 
 static void (*serial8250_isa_config)(int port, struct uart_port *up,
-	unsigned short *capabilities);
+	u32 *capabilities);
 
 void serial8250_set_isa_configurator(
-	void (*v)(int port, struct uart_port *up, unsigned short *capabilities))
+	void (*v)(int port, struct uart_port *up, u32 *capabilities))
 {
 	serial8250_isa_config = v;
 }
@@ -830,6 +830,7 @@ static int serial8250_probe(struct platform_device *dev)
 		uart.port.handle_irq	= p->handle_irq;
 		uart.port.handle_break	= p->handle_break;
 		uart.port.set_termios	= p->set_termios;
+		uart.port.set_ldisc	= p->set_ldisc;
 		uart.port.get_mctrl	= p->get_mctrl;
 		uart.port.pm		= p->pm;
 		uart.port.dev		= &dev->dev;
@@ -1023,6 +1024,8 @@ int serial8250_register_8250_port(struct uart_8250_port *up)
 		/*  Possibly override set_termios call */
 		if (up->port.set_termios)
 			uart->port.set_termios = up->port.set_termios;
+		if (up->port.set_ldisc)
+			uart->port.set_ldisc = up->port.set_ldisc;
 		if (up->port.get_mctrl)
 			uart->port.get_mctrl = up->port.get_mctrl;
 		if (up->port.set_mctrl)
diff --git a/drivers/tty/serial/8250/8250_dma.c b/drivers/tty/serial/8250/8250_dma.c
index fdbddbc6375d..26f17456b0d7 100644
--- a/drivers/tty/serial/8250/8250_dma.c
+++ b/drivers/tty/serial/8250/8250_dma.c
@@ -72,10 +72,15 @@ int serial8250_tx_dma(struct uart_8250_port *p)
 	struct dma_async_tx_descriptor	*desc;
 	int ret;
 
-	if (uart_tx_stopped(&p->port) || dma->tx_running ||
-	    uart_circ_empty(xmit))
+	if (dma->tx_running)
 		return 0;
 
+	if (uart_tx_stopped(&p->port) || uart_circ_empty(xmit)) {
+		/* We have been called from __dma_tx_complete() */
+		serial8250_rpm_put_tx(p);
+		return 0;
+	}
+
 	dma->tx_size = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 
 	desc = dmaengine_prep_slave_single(dma->txchan,
diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c
index 459d726f9d59..c89fafc972b6 100644
--- a/drivers/tty/serial/8250/8250_dw.c
+++ b/drivers/tty/serial/8250/8250_dw.c
@@ -53,6 +53,8 @@
 /* Helper for fifo size calculation */
 #define DW_UART_CPR_FIFO_SIZE(a)	(((a >> 16) & 0xff) * 16)
 
+/* DesignWare specific register fields */
+#define DW_UART_MCR_SIRE		BIT(6)
 
 struct dw8250_data {
 	u8			usr_reg;
@@ -254,6 +256,22 @@ out:
 	serial8250_do_set_termios(p, termios, old);
 }
 
+static void dw8250_set_ldisc(struct uart_port *p, struct ktermios *termios)
+{
+	struct uart_8250_port *up = up_to_u8250p(p);
+	unsigned int mcr = p->serial_in(p, UART_MCR);
+
+	if (up->capabilities & UART_CAP_IRDA) {
+		if (termios->c_line == N_IRDA)
+			mcr |= DW_UART_MCR_SIRE;
+		else
+			mcr &= ~DW_UART_MCR_SIRE;
+
+		p->serial_out(p, UART_MCR, mcr);
+	}
+	serial8250_do_set_ldisc(p, termios);
+}
+
 /*
  * dw8250_fallback_dma_filter will prevent the UART from getting just any free
  * channel on platforms that have DMA engines, but don't have any channels
@@ -357,6 +375,9 @@ static void dw8250_setup_port(struct uart_port *p)
 
 	if (reg & DW_UART_CPR_AFCE_MODE)
 		up->capabilities |= UART_CAP_AFE;
+
+	if (reg & DW_UART_CPR_SIR_MODE)
+		up->capabilities |= UART_CAP_IRDA;
 }
 
 static int dw8250_probe(struct platform_device *pdev)
@@ -392,6 +413,7 @@ static int dw8250_probe(struct platform_device *pdev)
 	p->iotype	= UPIO_MEM;
 	p->serial_in	= dw8250_serial_in;
 	p->serial_out	= dw8250_serial_out;
+	p->set_ldisc	= dw8250_set_ldisc;
 
 	p->membase = devm_ioremap(dev, regs->start, resource_size(regs));
 	if (!p->membase)
diff --git a/drivers/tty/serial/8250/8250_fintek.c b/drivers/tty/serial/8250/8250_fintek.c
index 0facc789fe7d..b67e7a544935 100644
--- a/drivers/tty/serial/8250/8250_fintek.c
+++ b/drivers/tty/serial/8250/8250_fintek.c
@@ -21,8 +21,11 @@
 #define EXIT_KEY 0xAA
 #define CHIP_ID1  0x20
 #define CHIP_ID2  0x21
-#define CHIP_ID_0 0x1602
-#define CHIP_ID_1 0x0501
+#define CHIP_ID_F81865 0x0407
+#define CHIP_ID_F81866 0x1010
+#define CHIP_ID_F81216AD 0x1602
+#define CHIP_ID_F81216H 0x0501
+#define CHIP_ID_F81216 0x0802
 #define VENDOR_ID1 0x23
 #define VENDOR_ID1_VAL 0x19
 #define VENDOR_ID2 0x24
@@ -43,12 +46,60 @@
 #define RXW4C_IRA BIT(3)
 #define TXW4C_IRA BIT(2)
 
+#define FIFO_CTRL		0xF6
+#define FIFO_MODE_MASK		(BIT(1) | BIT(0))
+#define FIFO_MODE_128		(BIT(1) | BIT(0))
+#define RXFTHR_MODE_MASK	(BIT(5) | BIT(4))
+#define RXFTHR_MODE_4X		BIT(5)
+
+#define F81216_LDN_LOW	0x0
+#define F81216_LDN_HIGH	0x4
+
+/*
+ * F81866 registers
+ *
+ * The IRQ setting mode of F81866 is not the same with F81216 series.
+ *	Level/Low: IRQ_MODE0:0, IRQ_MODE1:0
+ *	Edge/High: IRQ_MODE0:1, IRQ_MODE1:0
+ */
+#define F81866_IRQ_MODE		0xf0
+#define F81866_IRQ_SHARE	BIT(0)
+#define F81866_IRQ_MODE0	BIT(1)
+
+#define F81866_FIFO_CTRL	FIFO_CTRL
+#define F81866_IRQ_MODE1	BIT(3)
+
+#define F81866_LDN_LOW		0x10
+#define F81866_LDN_HIGH		0x16
+
 struct fintek_8250 {
+	u16 pid;
 	u16 base_port;
 	u8 index;
 	u8 key;
 };
 
+static u8 sio_read_reg(struct fintek_8250 *pdata, u8 reg)
+{
+	outb(reg, pdata->base_port + ADDR_PORT);
+	return inb(pdata->base_port + DATA_PORT);
+}
+
+static void sio_write_reg(struct fintek_8250 *pdata, u8 reg, u8 data)
+{
+	outb(reg, pdata->base_port + ADDR_PORT);
+	outb(data, pdata->base_port + DATA_PORT);
+}
+
+static void sio_write_mask_reg(struct fintek_8250 *pdata, u8 reg, u8 mask,
+			       u8 data)
+{
+	u8 tmp;
+
+	tmp = (sio_read_reg(pdata, reg) & ~mask) | (mask & data);
+	sio_write_reg(pdata, reg, tmp);
+}
+
 static int fintek_8250_enter_key(u16 base_port, u8 key)
 {
 	if (!request_muxed_region(base_port, 2, "8250_fintek"))
@@ -66,29 +117,55 @@ static void fintek_8250_exit_key(u16 base_port)
 	release_region(base_port + ADDR_PORT, 2);
 }
 
-static int fintek_8250_check_id(u16 base_port)
+static int fintek_8250_check_id(struct fintek_8250 *pdata)
 {
 	u16 chip;
 
-	outb(VENDOR_ID1, base_port + ADDR_PORT);
-	if (inb(base_port + DATA_PORT) != VENDOR_ID1_VAL)
+	if (sio_read_reg(pdata, VENDOR_ID1) != VENDOR_ID1_VAL)
 		return -ENODEV;
 
-	outb(VENDOR_ID2, base_port + ADDR_PORT);
-	if (inb(base_port + DATA_PORT) != VENDOR_ID2_VAL)
+	if (sio_read_reg(pdata, VENDOR_ID2) != VENDOR_ID2_VAL)
 		return -ENODEV;
 
-	outb(CHIP_ID1, base_port + ADDR_PORT);
-	chip = inb(base_port + DATA_PORT);
-	outb(CHIP_ID2, base_port + ADDR_PORT);
-	chip |= inb(base_port + DATA_PORT) << 8;
-
-	if (chip != CHIP_ID_0 && chip != CHIP_ID_1)
+	chip = sio_read_reg(pdata, CHIP_ID1);
+	chip |= sio_read_reg(pdata, CHIP_ID2) << 8;
+
+	switch (chip) {
+	case CHIP_ID_F81865:
+	case CHIP_ID_F81866:
+	case CHIP_ID_F81216AD:
+	case CHIP_ID_F81216H:
+	case CHIP_ID_F81216:
+		break;
+	default:
 		return -ENODEV;
+	}
 
+	pdata->pid = chip;
 	return 0;
 }
 
+static int fintek_8250_get_ldn_range(struct fintek_8250 *pdata, int *min,
+				     int *max)
+{
+	switch (pdata->pid) {
+	case CHIP_ID_F81865:
+	case CHIP_ID_F81866:
+		*min = F81866_LDN_LOW;
+		*max = F81866_LDN_HIGH;
+		return 0;
+
+	case CHIP_ID_F81216AD:
+	case CHIP_ID_F81216H:
+	case CHIP_ID_F81216:
+		*min = F81216_LDN_LOW;
+		*max = F81216_LDN_HIGH;
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
 static int fintek_8250_rs485_config(struct uart_port *port,
 			      struct serial_rs485 *rs485)
 {
@@ -128,10 +205,8 @@ static int fintek_8250_rs485_config(struct uart_port *port,
 	if (fintek_8250_enter_key(pdata->base_port, pdata->key))
 		return -EBUSY;
 
-	outb(LDN, pdata->base_port + ADDR_PORT);
-	outb(pdata->index, pdata->base_port + DATA_PORT);
-	outb(RS485, pdata->base_port + ADDR_PORT);
-	outb(config, pdata->base_port + DATA_PORT);
+	sio_write_reg(pdata, LDN, pdata->index);
+	sio_write_reg(pdata, RS485, config);
 	fintek_8250_exit_key(pdata->base_port);
 
 	port->rs485 = *rs485;
@@ -139,40 +214,90 @@ static int fintek_8250_rs485_config(struct uart_port *port,
 	return 0;
 }
 
-static int find_base_port(struct fintek_8250 *pdata, u16 io_address)
+static void fintek_8250_set_irq_mode(struct fintek_8250 *pdata, bool is_level)
+{
+	sio_write_reg(pdata, LDN, pdata->index);
+
+	switch (pdata->pid) {
+	case CHIP_ID_F81866:
+		sio_write_mask_reg(pdata, F81866_FIFO_CTRL, F81866_IRQ_MODE1,
+				   0);
+		/* fall through */
+	case CHIP_ID_F81865:
+		sio_write_mask_reg(pdata, F81866_IRQ_MODE, F81866_IRQ_SHARE,
+				   F81866_IRQ_SHARE);
+		sio_write_mask_reg(pdata, F81866_IRQ_MODE, F81866_IRQ_MODE0,
+				   is_level ? 0 : F81866_IRQ_MODE0);
+		break;
+
+	case CHIP_ID_F81216AD:
+	case CHIP_ID_F81216H:
+	case CHIP_ID_F81216:
+		sio_write_mask_reg(pdata, FINTEK_IRQ_MODE, IRQ_SHARE,
+				   IRQ_SHARE);
+		sio_write_mask_reg(pdata, FINTEK_IRQ_MODE, IRQ_MODE_MASK,
+				   is_level ? IRQ_LEVEL_LOW : IRQ_EDGE_HIGH);
+		break;
+	}
+}
+
+static void fintek_8250_set_max_fifo(struct fintek_8250 *pdata)
+{
+	switch (pdata->pid) {
+	case CHIP_ID_F81216H: /* 128Bytes FIFO */
+	case CHIP_ID_F81866:
+		sio_write_mask_reg(pdata, FIFO_CTRL,
+				   FIFO_MODE_MASK | RXFTHR_MODE_MASK,
+				   FIFO_MODE_128 | RXFTHR_MODE_4X);
+		break;
+
+	default: /* Default 16Bytes FIFO */
+		break;
+	}
+}
+
+static int probe_setup_port(struct fintek_8250 *pdata, u16 io_address,
+			  unsigned int irq)
 {
 	static const u16 addr[] = {0x4e, 0x2e};
 	static const u8 keys[] = {0x77, 0xa0, 0x87, 0x67};
-	int i, j, k;
+	struct irq_data *irq_data;
+	bool level_mode = false;
+	int i, j, k, min, max;
 
 	for (i = 0; i < ARRAY_SIZE(addr); i++) {
 		for (j = 0; j < ARRAY_SIZE(keys); j++) {
+			pdata->base_port = addr[i];
+			pdata->key = keys[j];
 
 			if (fintek_8250_enter_key(addr[i], keys[j]))
 				continue;
-			if (fintek_8250_check_id(addr[i])) {
+			if (fintek_8250_check_id(pdata) ||
+			    fintek_8250_get_ldn_range(pdata, &min, &max)) {
 				fintek_8250_exit_key(addr[i]);
 				continue;
 			}
 
-			for (k = 0; k < 4; k++) {
+			for (k = min; k < max; k++) {
 				u16 aux;
 
-				outb(LDN, addr[i] + ADDR_PORT);
-				outb(k, addr[i] + DATA_PORT);
-
-				outb(IO_ADDR1, addr[i] + ADDR_PORT);
-				aux = inb(addr[i] + DATA_PORT);
-				outb(IO_ADDR2, addr[i] + ADDR_PORT);
-				aux |= inb(addr[i] + DATA_PORT) << 8;
+				sio_write_reg(pdata, LDN, k);
+				aux = sio_read_reg(pdata, IO_ADDR1);
+				aux |= sio_read_reg(pdata, IO_ADDR2) << 8;
 				if (aux != io_address)
 					continue;
 
-				fintek_8250_exit_key(addr[i]);
-				pdata->key = keys[j];
-				pdata->base_port = addr[i];
 				pdata->index = k;
 
+				irq_data = irq_get_irq_data(irq);
+				if (irq_data)
+					level_mode =
+						irqd_is_level_type(irq_data);
+
+				fintek_8250_set_irq_mode(pdata, level_mode);
+				fintek_8250_set_max_fifo(pdata);
+				fintek_8250_exit_key(addr[i]);
+
 				return 0;
 			}
 
@@ -183,39 +308,29 @@ static int find_base_port(struct fintek_8250 *pdata, u16 io_address)
 	return -ENODEV;
 }
 
-static int fintek_8250_set_irq_mode(struct fintek_8250 *pdata, bool level_mode)
+static void fintek_8250_set_rs485_handler(struct uart_8250_port *uart)
 {
-	int status;
-	u8 tmp;
-
-	status = fintek_8250_enter_key(pdata->base_port, pdata->key);
-	if (status)
-		return status;
-
-	outb(LDN, pdata->base_port + ADDR_PORT);
-	outb(pdata->index, pdata->base_port + DATA_PORT);
-
-	outb(FINTEK_IRQ_MODE, pdata->base_port + ADDR_PORT);
-	tmp = inb(pdata->base_port + DATA_PORT);
-
-	tmp &= ~IRQ_MODE_MASK;
-	tmp |= IRQ_SHARE;
-	if (!level_mode)
-		tmp |= IRQ_EDGE_HIGH;
-
-	outb(tmp, pdata->base_port + DATA_PORT);
-	fintek_8250_exit_key(pdata->base_port);
-	return 0;
+	struct fintek_8250 *pdata = uart->port.private_data;
+
+	switch (pdata->pid) {
+	case CHIP_ID_F81216AD:
+	case CHIP_ID_F81216H:
+	case CHIP_ID_F81866:
+	case CHIP_ID_F81865:
+		uart->port.rs485_config = fintek_8250_rs485_config;
+		break;
+
+	default: /* No RS485 Auto direction functional */
+		break;
+	}
 }
 
 int fintek_8250_probe(struct uart_8250_port *uart)
 {
 	struct fintek_8250 *pdata;
 	struct fintek_8250 probe_data;
-	struct irq_data *irq_data = irq_get_irq_data(uart->port.irq);
-	bool level_mode = irqd_is_level_type(irq_data);
 
-	if (find_base_port(&probe_data, uart->port.iobase))
+	if (probe_setup_port(&probe_data, uart->port.iobase, uart->port.irq))
 		return -ENODEV;
 
 	pdata = devm_kzalloc(uart->port.dev, sizeof(*pdata), GFP_KERNEL);
@@ -223,8 +338,8 @@ int fintek_8250_probe(struct uart_8250_port *uart)
 		return -ENOMEM;
 
 	memcpy(pdata, &probe_data, sizeof(probe_data));
-	uart->port.rs485_config = fintek_8250_rs485_config;
 	uart->port.private_data = pdata;
+	fintek_8250_set_rs485_handler(uart);
 
-	return fintek_8250_set_irq_mode(pdata, level_mode);
+	return 0;
 }
diff --git a/drivers/tty/serial/8250/8250_lpss.c b/drivers/tty/serial/8250/8250_lpss.c
index b9923464599f..58cbb30a9401 100644
--- a/drivers/tty/serial/8250/8250_lpss.c
+++ b/drivers/tty/serial/8250/8250_lpss.c
@@ -157,12 +157,12 @@ static int byt_serial_setup(struct lpss8250 *lpss, struct uart_port *port)
 static const struct dw_dma_platform_data qrk_serial_dma_pdata = {
 	.nr_channels = 2,
 	.is_private = true,
-	.is_nollp = true,
 	.chan_allocation_order = CHAN_ALLOCATION_ASCENDING,
 	.chan_priority = CHAN_PRIORITY_ASCENDING,
 	.block_size = 4095,
 	.nr_masters = 1,
 	.data_width = {4},
+	.multi_block = {0},
 };
 
 static void qrk_serial_setup_dma(struct lpss8250 *lpss, struct uart_port *port)
@@ -174,7 +174,7 @@ static void qrk_serial_setup_dma(struct lpss8250 *lpss, struct uart_port *port)
 	int ret;
 
 	chip->dev = &pdev->dev;
-	chip->irq = pdev->irq;
+	chip->irq = pci_irq_vector(pdev, 0);
 	chip->regs = pci_ioremap_bar(pdev, 1);
 	chip->pdata = &qrk_serial_dma_pdata;
 
@@ -183,6 +183,9 @@ static void qrk_serial_setup_dma(struct lpss8250 *lpss, struct uart_port *port)
 	if (ret)
 		return;
 
+	pci_set_master(pdev);
+	pci_try_set_mwi(pdev);
+
 	/* Special DMA address for UART */
 	dma->rx_dma_addr = 0xfffff000;
 	dma->tx_dma_addr = 0xfffff000;
@@ -280,8 +283,6 @@ static int lpss8250_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (ret)
 		return ret;
 
-	pci_set_master(pdev);
-
 	lpss = devm_kzalloc(&pdev->dev, sizeof(*lpss), GFP_KERNEL);
 	if (!lpss)
 		return -ENOMEM;
diff --git a/drivers/tty/serial/8250/8250_mid.c b/drivers/tty/serial/8250/8250_mid.c
index 39c2324484dd..ac013edf4992 100644
--- a/drivers/tty/serial/8250/8250_mid.c
+++ b/drivers/tty/serial/8250/8250_mid.c
@@ -303,10 +303,10 @@ static void mid8250_remove(struct pci_dev *pdev)
 {
 	struct mid8250 *mid = pci_get_drvdata(pdev);
 
+	serial8250_unregister_port(mid->line);
+
 	if (mid->board->exit)
 		mid->board->exit(mid);
-
-	serial8250_unregister_port(mid->line);
 }
 
 static const struct mid8250_board pnw_board = {
diff --git a/drivers/tty/serial/8250/8250_of.c b/drivers/tty/serial/8250/8250_of.c
index 7a8b5fc81a19..d25ab1cd4295 100644
--- a/drivers/tty/serial/8250/8250_of.c
+++ b/drivers/tty/serial/8250/8250_of.c
@@ -332,8 +332,6 @@ static const struct of_device_id of_platform_serial_table[] = {
 		.data = (void *)PORT_ALTR_16550_F128, },
 	{ .compatible = "mrvl,mmp-uart",
 		.data = (void *)PORT_XSCALE, },
-	{ .compatible = "mrvl,pxa-uart",
-		.data = (void *)PORT_XSCALE, },
 	{ /* end of list */ },
 };
 MODULE_DEVICE_TABLE(of, of_platform_serial_table);
diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c
index b98c1578f45a..aa0166b6d450 100644
--- a/drivers/tty/serial/8250/8250_pci.c
+++ b/drivers/tty/serial/8250/8250_pci.c
@@ -52,6 +52,7 @@ struct serial_private {
 	struct pci_dev		*dev;
 	unsigned int		nr;
 	struct pci_serial_quirk	*quirk;
+	const struct pciserial_board *board;
 	int			line[0];
 };
 
@@ -1329,6 +1330,30 @@ static int pci_default_setup(struct serial_private *priv,
 	return setup_port(priv, port, bar, offset, board->reg_shift);
 }
 
+static int pci_pericom_setup(struct serial_private *priv,
+		  const struct pciserial_board *board,
+		  struct uart_8250_port *port, int idx)
+{
+	unsigned int bar, offset = board->first_offset, maxnr;
+
+	bar = FL_GET_BASE(board->flags);
+	if (board->flags & FL_BASE_BARS)
+		bar += idx;
+	else
+		offset += idx * board->uart_offset;
+
+	if (idx==3)
+		offset = 0x38;
+
+	maxnr = (pci_resource_len(priv->dev, bar) - board->first_offset) >>
+		(board->reg_shift + 3);
+
+	if (board->flags & FL_REGION_SZ_CAP && idx >= maxnr)
+		return 1;
+
+	return setup_port(priv, port, bar, offset, board->reg_shift);
+}
+
 static int
 ce4100_serial_setup(struct serial_private *priv,
 		  const struct pciserial_board *board,
@@ -2096,6 +2121,16 @@ static struct pci_serial_quirk pci_serial_quirks[] __refdata = {
 		.exit		= pci_plx9050_exit,
 	},
 	/*
+	 * Pericom (Only 7954 - It have a offset jump for port 4)
+	 */
+	{
+		.vendor		= PCI_VENDOR_ID_PERICOM,
+		.device		= PCI_DEVICE_ID_PERICOM_PI7C9X7954,
+		.subvendor	= PCI_ANY_ID,
+		.subdevice	= PCI_ANY_ID,
+		.setup		= pci_pericom_setup,
+	},
+	/*
 	 * PLX
 	 */
 	{
@@ -3862,6 +3897,7 @@ pciserial_init_ports(struct pci_dev *dev, const struct pciserial_board *board)
 		}
 	}
 	priv->nr = i;
+	priv->board = board;
 	return priv;
 
 err_deinit:
@@ -3872,7 +3908,7 @@ err_out:
 }
 EXPORT_SYMBOL_GPL(pciserial_init_ports);
 
-void pciserial_remove_ports(struct serial_private *priv)
+void pciserial_detach_ports(struct serial_private *priv)
 {
 	struct pci_serial_quirk *quirk;
 	int i;
@@ -3886,7 +3922,11 @@ void pciserial_remove_ports(struct serial_private *priv)
 	quirk = find_quirk(priv->dev);
 	if (quirk->exit)
 		quirk->exit(priv->dev);
+}
 
+void pciserial_remove_ports(struct serial_private *priv)
+{
+	pciserial_detach_ports(priv);
 	kfree(priv);
 }
 EXPORT_SYMBOL_GPL(pciserial_remove_ports);
@@ -5577,7 +5617,7 @@ static pci_ers_result_t serial8250_io_error_detected(struct pci_dev *dev,
 		return PCI_ERS_RESULT_DISCONNECT;
 
 	if (priv)
-		pciserial_suspend_ports(priv);
+		pciserial_detach_ports(priv);
 
 	pci_disable_device(dev);
 
@@ -5602,9 +5642,18 @@ static pci_ers_result_t serial8250_io_slot_reset(struct pci_dev *dev)
 static void serial8250_io_resume(struct pci_dev *dev)
 {
 	struct serial_private *priv = pci_get_drvdata(dev);
+	const struct pciserial_board *board;
 
-	if (priv)
-		pciserial_resume_ports(priv);
+	if (!priv)
+		return;
+
+	board = priv->board;
+	kfree(priv);
+	priv = pciserial_init_ports(dev, board);
+
+	if (!IS_ERR(priv)) {
+		pci_set_drvdata(dev, priv);
+	}
 }
 
 static const struct pci_error_handlers serial8250_err_handler = {
diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
index 1731b98d2471..fe4399b41df6 100644
--- a/drivers/tty/serial/8250/8250_port.c
+++ b/drivers/tty/serial/8250/8250_port.c
@@ -636,7 +636,7 @@ EXPORT_SYMBOL_GPL(serial8250_em485_destroy);
  * once and disable_runtime_pm_tx() will still disable RPM because the fifo is
  * empty and the HW can idle again.
  */
-static void serial8250_rpm_get_tx(struct uart_8250_port *p)
+void serial8250_rpm_get_tx(struct uart_8250_port *p)
 {
 	unsigned char rpm_active;
 
@@ -648,8 +648,9 @@ static void serial8250_rpm_get_tx(struct uart_8250_port *p)
 		return;
 	pm_runtime_get_sync(p->port.dev);
 }
+EXPORT_SYMBOL_GPL(serial8250_rpm_get_tx);
 
-static void serial8250_rpm_put_tx(struct uart_8250_port *p)
+void serial8250_rpm_put_tx(struct uart_8250_port *p)
 {
 	unsigned char rpm_active;
 
@@ -662,6 +663,7 @@ static void serial8250_rpm_put_tx(struct uart_8250_port *p)
 	pm_runtime_mark_last_busy(p->port.dev);
 	pm_runtime_put_autosuspend(p->port.dev);
 }
+EXPORT_SYMBOL_GPL(serial8250_rpm_put_tx);
 
 /*
  * IER sleep support.  UARTs which have EFRs need the "extended
@@ -2691,8 +2693,7 @@ serial8250_set_termios(struct uart_port *port, struct ktermios *termios,
 		serial8250_do_set_termios(port, termios, old);
 }
 
-static void
-serial8250_set_ldisc(struct uart_port *port, struct ktermios *termios)
+void serial8250_do_set_ldisc(struct uart_port *port, struct ktermios *termios)
 {
 	if (termios->c_line == N_PPS) {
 		port->flags |= UPF_HARDPPS_CD;
@@ -2708,7 +2709,16 @@ serial8250_set_ldisc(struct uart_port *port, struct ktermios *termios)
 		}
 	}
 }
+EXPORT_SYMBOL_GPL(serial8250_do_set_ldisc);
 
+static void
+serial8250_set_ldisc(struct uart_port *port, struct ktermios *termios)
+{
+	if (port->set_ldisc)
+		port->set_ldisc(port, termios);
+	else
+		serial8250_do_set_ldisc(port, termios);
+}
 
 void serial8250_do_pm(struct uart_port *port, unsigned int state,
 		      unsigned int oldstate)
diff --git a/drivers/tty/serial/8250/8250_pxa.c b/drivers/tty/serial/8250/8250_pxa.c
new file mode 100644
index 000000000000..4d68731af534
--- /dev/null
+++ b/drivers/tty/serial/8250/8250_pxa.c
@@ -0,0 +1,190 @@
+/*
+ *  drivers/tty/serial/8250/8250_pxa.c -- driver for PXA on-board UARTS
+ *  Copyright:	(C) 2013 Sergei Ianovich <ynvich@gmail.com>
+ *
+ *  replaces drivers/serial/pxa.c by Nicolas Pitre
+ *  Created:	Feb 20, 2003
+ *  Copyright:	(C) 2003 Monta Vista Software, Inc.
+ *
+ *  Based on drivers/serial/8250.c by Russell King.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/serial_8250.h>
+#include <linux/serial_core.h>
+#include <linux/serial_reg.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/clk.h>
+#include <linux/pm_runtime.h>
+
+#include "8250.h"
+
+struct pxa8250_data {
+	int			line;
+	struct clk		*clk;
+};
+
+static int __maybe_unused serial_pxa_suspend(struct device *dev)
+{
+	struct pxa8250_data *data = dev_get_drvdata(dev);
+
+	serial8250_suspend_port(data->line);
+
+	return 0;
+}
+
+static int __maybe_unused serial_pxa_resume(struct device *dev)
+{
+	struct pxa8250_data *data = dev_get_drvdata(dev);
+
+	serial8250_resume_port(data->line);
+
+	return 0;
+}
+
+static const struct dev_pm_ops serial_pxa_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(serial_pxa_suspend, serial_pxa_resume)
+};
+
+static const struct of_device_id serial_pxa_dt_ids[] = {
+	{ .compatible = "mrvl,pxa-uart", },
+	{ .compatible = "mrvl,mmp-uart", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, serial_pxa_dt_ids);
+
+/* Uart divisor latch write */
+static void serial_pxa_dl_write(struct uart_8250_port *up, int value)
+{
+	unsigned int dll;
+
+	serial_out(up, UART_DLL, value & 0xff);
+	/*
+	 * work around Erratum #74 according to Marvel(R) PXA270M Processor
+	 * Specification Update (April 19, 2010)
+	 */
+	dll = serial_in(up, UART_DLL);
+	WARN_ON(dll != (value & 0xff));
+
+	serial_out(up, UART_DLM, value >> 8 & 0xff);
+}
+
+
+static void serial_pxa_pm(struct uart_port *port, unsigned int state,
+	      unsigned int oldstate)
+{
+	struct pxa8250_data *data = port->private_data;
+
+	if (!state)
+		clk_prepare_enable(data->clk);
+	else
+		clk_disable_unprepare(data->clk);
+}
+
+static int serial_pxa_probe(struct platform_device *pdev)
+{
+	struct uart_8250_port uart = {};
+	struct pxa8250_data *data;
+	struct resource *mmres, *irqres;
+	int ret;
+
+	mmres = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	irqres = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+	if (!mmres || !irqres)
+		return -ENODEV;
+
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(data->clk))
+		return PTR_ERR(data->clk);
+
+	ret = clk_prepare(data->clk);
+	if (ret)
+		return ret;
+
+	uart.port.type = PORT_XSCALE;
+	uart.port.iotype = UPIO_MEM32;
+	uart.port.mapbase = mmres->start;
+	uart.port.regshift = 2;
+	uart.port.irq = irqres->start;
+	uart.port.fifosize = 64;
+	uart.port.flags = UPF_IOREMAP | UPF_SKIP_TEST;
+	uart.port.dev = &pdev->dev;
+	uart.port.uartclk = clk_get_rate(data->clk);
+	uart.port.pm = serial_pxa_pm;
+	uart.port.private_data = data;
+	uart.dl_write = serial_pxa_dl_write;
+
+	ret = serial8250_register_8250_port(&uart);
+	if (ret < 0)
+		goto err_clk;
+
+	data->line = ret;
+
+	platform_set_drvdata(pdev, data);
+
+	return 0;
+
+ err_clk:
+	clk_unprepare(data->clk);
+	return ret;
+}
+
+static int serial_pxa_remove(struct platform_device *pdev)
+{
+	struct pxa8250_data *data = platform_get_drvdata(pdev);
+
+	serial8250_unregister_port(data->line);
+
+	clk_unprepare(data->clk);
+
+	return 0;
+}
+
+static struct platform_driver serial_pxa_driver = {
+	.probe          = serial_pxa_probe,
+	.remove         = serial_pxa_remove,
+
+	.driver		= {
+		.name	= "pxa2xx-uart",
+		.pm	= &serial_pxa_pm_ops,
+		.of_match_table = serial_pxa_dt_ids,
+	},
+};
+
+module_platform_driver(serial_pxa_driver);
+
+#ifdef CONFIG_SERIAL_8250_CONSOLE
+static int __init early_serial_pxa_setup(struct earlycon_device *device,
+				  const char *options)
+{
+	struct uart_port *port = &device->port;
+
+	if (!(device->port.membase || device->port.iobase))
+		return -ENODEV;
+
+	port->regshift = 2;
+	return early_serial8250_setup(device, NULL);
+}
+OF_EARLYCON_DECLARE(early_pxa, "mrvl,pxa-uart", early_serial_pxa_setup);
+#endif
+
+MODULE_AUTHOR("Sergei Ianovich");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:pxa2xx-uart");
diff --git a/drivers/tty/serial/8250/8250_uniphier.c b/drivers/tty/serial/8250/8250_uniphier.c
index 417d9e7038e1..746680ebf90c 100644
--- a/drivers/tty/serial/8250/8250_uniphier.c
+++ b/drivers/tty/serial/8250/8250_uniphier.c
@@ -24,10 +24,22 @@
 /* Most (but not all) of UniPhier UART devices have 64-depth FIFO. */
 #define UNIPHIER_UART_DEFAULT_FIFO_SIZE	64
 
-#define UNIPHIER_UART_CHAR_FCR	3	/* Character / FIFO Control Register */
-#define UNIPHIER_UART_LCR_MCR	4	/* Line/Modem Control Register */
-#define   UNIPHIER_UART_LCR_SHIFT	8
-#define UNIPHIER_UART_DLR	9	/* Divisor Latch Register */
+/*
+ * This hardware is similar to 8250, but its register map is a bit different:
+ *   - MMIO32 (regshift = 2)
+ *   - FCR is not at 2, but 3
+ *   - LCR and MCR are not at 3 and 4, they share 4
+ *   - Divisor latch at 9, no divisor latch access bit
+ */
+
+#define UNIPHIER_UART_REGSHIFT		2
+
+/* bit[15:8] = CHAR (not used), bit[7:0] = FCR */
+#define UNIPHIER_UART_CHAR_FCR		(3 << (UNIPHIER_UART_REGSHIFT))
+/* bit[15:8] = LCR, bit[7:0] = MCR */
+#define UNIPHIER_UART_LCR_MCR		(4 << (UNIPHIER_UART_REGSHIFT))
+/* Divisor Latch Register */
+#define UNIPHIER_UART_DLR		(9 << (UNIPHIER_UART_REGSHIFT))
 
 struct uniphier8250_priv {
 	int line;
@@ -44,7 +56,7 @@ static int __init uniphier_early_console_setup(struct earlycon_device *device,
 
 	/* This hardware always expects MMIO32 register interface. */
 	device->port.iotype = UPIO_MEM32;
-	device->port.regshift = 2;
+	device->port.regshift = UNIPHIER_UART_REGSHIFT;
 
 	/*
 	 * Do not touch the divisor register in early_serial8250_setup();
@@ -68,17 +80,16 @@ static unsigned int uniphier_serial_in(struct uart_port *p, int offset)
 
 	switch (offset) {
 	case UART_LCR:
-		valshift = UNIPHIER_UART_LCR_SHIFT;
+		valshift = 8;
 		/* fall through */
 	case UART_MCR:
 		offset = UNIPHIER_UART_LCR_MCR;
 		break;
 	default:
+		offset <<= UNIPHIER_UART_REGSHIFT;
 		break;
 	}
 
-	offset <<= p->regshift;
-
 	/*
 	 * The return value must be masked with 0xff because LCR and MCR reside
 	 * in the same register that must be accessed by 32-bit write/read.
@@ -90,27 +101,26 @@ static unsigned int uniphier_serial_in(struct uart_port *p, int offset)
 static void uniphier_serial_out(struct uart_port *p, int offset, int value)
 {
 	unsigned int valshift = 0;
-	bool normal = false;
+	bool normal = true;
 
 	switch (offset) {
 	case UART_FCR:
 		offset = UNIPHIER_UART_CHAR_FCR;
 		break;
 	case UART_LCR:
-		valshift = UNIPHIER_UART_LCR_SHIFT;
+		valshift = 8;
 		/* Divisor latch access bit does not exist. */
 		value &= ~UART_LCR_DLAB;
 		/* fall through */
 	case UART_MCR:
 		offset = UNIPHIER_UART_LCR_MCR;
+		normal = false;
 		break;
 	default:
-		normal = true;
+		offset <<= UNIPHIER_UART_REGSHIFT;
 		break;
 	}
 
-	offset <<= p->regshift;
-
 	if (normal) {
 		writel(value, p->membase + offset);
 	} else {
@@ -139,16 +149,12 @@ static void uniphier_serial_out(struct uart_port *p, int offset, int value)
  */
 static int uniphier_serial_dl_read(struct uart_8250_port *up)
 {
-	int offset = UNIPHIER_UART_DLR << up->port.regshift;
-
-	return readl(up->port.membase + offset);
+	return readl(up->port.membase + UNIPHIER_UART_DLR);
 }
 
 static void uniphier_serial_dl_write(struct uart_8250_port *up, int value)
 {
-	int offset = UNIPHIER_UART_DLR << up->port.regshift;
-
-	writel(value, up->port.membase + offset);
+	writel(value, up->port.membase + UNIPHIER_UART_DLR);
 }
 
 static int uniphier_of_serial_setup(struct device *dev, struct uart_port *port,
@@ -234,7 +240,7 @@ static int uniphier_uart_probe(struct platform_device *pdev)
 
 	up.port.type = PORT_16550A;
 	up.port.iotype = UPIO_MEM32;
-	up.port.regshift = 2;
+	up.port.regshift = UNIPHIER_UART_REGSHIFT;
 	up.port.flags = UPF_FIXED_PORT | UPF_FIXED_TYPE;
 	up.capabilities = UART_CAP_FIFO;
 
diff --git a/drivers/tty/serial/8250/Kconfig b/drivers/tty/serial/8250/Kconfig
index 899834776b36..0b8b6740ba43 100644
--- a/drivers/tty/serial/8250/Kconfig
+++ b/drivers/tty/serial/8250/Kconfig
@@ -439,6 +439,16 @@ config SERIAL_8250_MOXA
 	  This driver can also be built as a module. The module will be called
 	  8250_moxa. If you want to do that, say M here.
 
+config SERIAL_8250_PXA
+	tristate "PXA serial port support"
+	depends on SERIAL_8250
+	depends on ARCH_PXA || ARCH_MMP
+	help
+	  If you have a machine based on an Intel XScale PXA2xx CPU you can
+	  enable its onboard serial ports by enabling this option. The option is
+	  applicable to both devicetree and legacy boards, and early console is
+	  part of its support.
+
 config SERIAL_OF_PLATFORM
 	tristate "Devicetree based probing for 8250 ports"
 	depends on SERIAL_8250 && OF
diff --git a/drivers/tty/serial/8250/Makefile b/drivers/tty/serial/8250/Makefile
index 276c6fb60337..850e721877a9 100644
--- a/drivers/tty/serial/8250/Makefile
+++ b/drivers/tty/serial/8250/Makefile
@@ -31,6 +31,7 @@ obj-$(CONFIG_SERIAL_8250_INGENIC)	+= 8250_ingenic.o
 obj-$(CONFIG_SERIAL_8250_LPSS)		+= 8250_lpss.o
 obj-$(CONFIG_SERIAL_8250_MID)		+= 8250_mid.o
 obj-$(CONFIG_SERIAL_8250_MOXA)		+= 8250_moxa.o
+obj-$(CONFIG_SERIAL_8250_PXA)		+= 8250_pxa.o
 obj-$(CONFIG_SERIAL_OF_PLATFORM)	+= 8250_of.o
 
 CFLAGS_8250_ingenic.o += -I$(srctree)/scripts/dtc/libfdt
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index 25c1d7bc0100..e9cf5b67f1b7 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -438,17 +438,27 @@ config SERIAL_MPSC_CONSOLE
 	  Say Y here if you want to support a serial console on a Marvell MPSC.
 
 config SERIAL_PXA
-	bool "PXA serial port support"
+	bool "PXA serial port support (DEPRECATED)"
 	depends on ARCH_PXA || ARCH_MMP
 	select SERIAL_CORE
+	select SERIAL_8250_PXA if SERIAL_8250=y
+	select SERIAL_PXA_NON8250 if !SERIAL_8250=y
 	help
 	  If you have a machine based on an Intel XScale PXA2xx CPU you
 	  can enable its onboard serial ports by enabling this option.
 
+	  Unless you have a specific need, you should use SERIAL_8250_PXA
+	  instead of this.
+
+config SERIAL_PXA_NON8250
+	bool
+	depends on !SERIAL_8250
+
 config SERIAL_PXA_CONSOLE
-	bool "Console on PXA serial port"
+	bool "Console on PXA serial port (DEPRECATED)"
 	depends on SERIAL_PXA
 	select SERIAL_CORE_CONSOLE
+	select SERIAL_8250_CONSOLE if SERIAL_8250=y
 	help
 	  If you have enabled the serial port on the Intel XScale PXA
 	  CPU you can make it the console by answering Y to this option.
@@ -460,6 +470,9 @@ config SERIAL_PXA_CONSOLE
 	  your boot loader (lilo or loadlin) about how to pass options to the
 	  kernel at boot time.)
 
+	  Unless you have a specific need, you should use SERIAL_8250_PXA
+	  and SERIAL_8250_CONSOLE instead of this.
+
 config SERIAL_SA1100
 	bool "SA1100 serial port support"
 	depends on ARCH_SA1100
@@ -1626,7 +1639,7 @@ config SERIAL_STM32
 	tristate "STMicroelectronics STM32 serial port support"
 	select SERIAL_CORE
 	depends on HAS_DMA
-	depends on ARM || COMPILE_TEST
+	depends on ARCH_STM32 || COMPILE_TEST
 	help
 	  This driver is for the on-chip Serial Controller on
 	  STMicroelectronics STM32 MCUs.
diff --git a/drivers/tty/serial/Makefile b/drivers/tty/serial/Makefile
index 1278d376da50..2d6288bc4554 100644
--- a/drivers/tty/serial/Makefile
+++ b/drivers/tty/serial/Makefile
@@ -23,7 +23,7 @@ obj-$(CONFIG_SERIAL_8250) += 8250/
 obj-$(CONFIG_SERIAL_AMBA_PL010) += amba-pl010.o
 obj-$(CONFIG_SERIAL_AMBA_PL011) += amba-pl011.o
 obj-$(CONFIG_SERIAL_CLPS711X) += clps711x.o
-obj-$(CONFIG_SERIAL_PXA) += pxa.o
+obj-$(CONFIG_SERIAL_PXA_NON8250) += pxa.o
 obj-$(CONFIG_SERIAL_PNX8XXX) += pnx8xxx_uart.o
 obj-$(CONFIG_SERIAL_SA1100) += sa1100.o
 obj-$(CONFIG_SERIAL_BCM63XX) += bcm63xx_uart.o
@@ -62,13 +62,11 @@ obj-$(CONFIG_SERIAL_ATMEL) += atmel_serial.o
 obj-$(CONFIG_SERIAL_UARTLITE) += uartlite.o
 obj-$(CONFIG_SERIAL_MSM) += msm_serial.o
 obj-$(CONFIG_SERIAL_NETX) += netx-serial.o
-obj-$(CONFIG_SERIAL_KGDB_NMI) += kgdb_nmi.o
 obj-$(CONFIG_SERIAL_KS8695) += serial_ks8695.o
 obj-$(CONFIG_SERIAL_OMAP) += omap-serial.o
 obj-$(CONFIG_SERIAL_ALTERA_UART) += altera_uart.o
 obj-$(CONFIG_SERIAL_ST_ASC) += st-asc.o
 obj-$(CONFIG_SERIAL_TILEGX) += tilegx.o
-obj-$(CONFIG_KGDB_SERIAL_CONSOLE) += kgdboc.o
 obj-$(CONFIG_SERIAL_QE) += ucc_uart.o
 obj-$(CONFIG_SERIAL_TIMBERDALE)	+= timbuart.o
 obj-$(CONFIG_SERIAL_GRLIB_GAISLER_APBUART) += apbuart.o
@@ -96,3 +94,6 @@ obj-$(CONFIG_SERIAL_MPS2_UART)	+= mps2-uart.o
 
 # GPIOLIB helpers for modem control lines
 obj-$(CONFIG_SERIAL_MCTRL_GPIO)	+= serial_mctrl_gpio.o
+
+obj-$(CONFIG_SERIAL_KGDB_NMI) += kgdb_nmi.o
+obj-$(CONFIG_KGDB_SERIAL_CONSOLE) += kgdboc.o
diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c
index e2c33b9528d8..d4171d71a258 100644
--- a/drivers/tty/serial/amba-pl011.c
+++ b/drivers/tty/serial/amba-pl011.c
@@ -2315,12 +2315,67 @@ static int __init pl011_console_setup(struct console *co, char *options)
 	return uart_set_options(&uap->port, co, baud, parity, bits, flow);
 }
 
+/**
+ *	pl011_console_match - non-standard console matching
+ *	@co:	  registering console
+ *	@name:	  name from console command line
+ *	@idx:	  index from console command line
+ *	@options: ptr to option string from console command line
+ *
+ *	Only attempts to match console command lines of the form:
+ *	    console=pl011,mmio|mmio32,<addr>[,<options>]
+ *	    console=pl011,0x<addr>[,<options>]
+ *	This form is used to register an initial earlycon boot console and
+ *	replace it with the amba_console at pl011 driver init.
+ *
+ *	Performs console setup for a match (as required by interface)
+ *	If no <options> are specified, then assume the h/w is already setup.
+ *
+ *	Returns 0 if console matches; otherwise non-zero to use default matching
+ */
+static int __init pl011_console_match(struct console *co, char *name, int idx,
+				      char *options)
+{
+	unsigned char iotype;
+	resource_size_t addr;
+	int i;
+
+	if (strcmp(name, "pl011") != 0)
+		return -ENODEV;
+
+	if (uart_parse_earlycon(options, &iotype, &addr, &options))
+		return -ENODEV;
+
+	if (iotype != UPIO_MEM && iotype != UPIO_MEM32)
+		return -ENODEV;
+
+	/* try to match the port specified on the command line */
+	for (i = 0; i < ARRAY_SIZE(amba_ports); i++) {
+		struct uart_port *port;
+
+		if (!amba_ports[i])
+			continue;
+
+		port = &amba_ports[i]->port;
+
+		if (port->mapbase != addr)
+			continue;
+
+		co->index = i;
+		port->cons = co;
+		return pl011_console_setup(co, options);
+	}
+
+	return -ENODEV;
+}
+
 static struct uart_driver amba_reg;
 static struct console amba_console = {
 	.name		= "ttyAMA",
 	.write		= pl011_console_write,
 	.device		= uart_console_device,
 	.setup		= pl011_console_setup,
+	.match		= pl011_console_match,
 	.flags		= CON_PRINTBUFFER,
 	.index		= -1,
 	.data		= &amba_reg,
@@ -2357,6 +2412,7 @@ static int __init pl011_early_console_setup(struct earlycon_device *device,
 	return 0;
 }
 OF_EARLYCON_DECLARE(pl011, "arm,pl011", pl011_early_console_setup);
+OF_EARLYCON_DECLARE(pl011, "arm,sbsa-uart", pl011_early_console_setup);
 
 #else
 #define AMBA_CONSOLE	NULL
diff --git a/drivers/tty/serial/crisv10.c b/drivers/tty/serial/crisv10.c
index 6450a38cb1aa..e92c23470e51 100644
--- a/drivers/tty/serial/crisv10.c
+++ b/drivers/tty/serial/crisv10.c
@@ -3213,8 +3213,6 @@ get_serial_info(struct e100_serial * info,
 	 * should set them to something else than 0.
 	 */
 
-	if (!retinfo)
-		return -EFAULT;
 	memset(&tmp, 0, sizeof(tmp));
 	tmp.type = info->type;
 	tmp.line = info->line;
diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c
index 76103f2c4a80..a1c6519837a4 100644
--- a/drivers/tty/serial/fsl_lpuart.c
+++ b/drivers/tty/serial/fsl_lpuart.c
@@ -430,6 +430,65 @@ static void lpuart_flush_buffer(struct uart_port *port)
 	}
 }
 
+#if defined(CONFIG_CONSOLE_POLL)
+
+static int lpuart_poll_init(struct uart_port *port)
+{
+	struct lpuart_port *sport = container_of(port,
+					struct lpuart_port, port);
+	unsigned long flags;
+	unsigned char temp;
+
+	sport->port.fifosize = 0;
+
+	spin_lock_irqsave(&sport->port.lock, flags);
+	/* Disable Rx & Tx */
+	writeb(0, sport->port.membase + UARTCR2);
+
+	temp = readb(sport->port.membase + UARTPFIFO);
+	/* Enable Rx and Tx FIFO */
+	writeb(temp | UARTPFIFO_RXFE | UARTPFIFO_TXFE,
+			sport->port.membase + UARTPFIFO);
+
+	/* flush Tx and Rx FIFO */
+	writeb(UARTCFIFO_TXFLUSH | UARTCFIFO_RXFLUSH,
+			sport->port.membase + UARTCFIFO);
+
+	/* explicitly clear RDRF */
+	if (readb(sport->port.membase + UARTSR1) & UARTSR1_RDRF) {
+		readb(sport->port.membase + UARTDR);
+		writeb(UARTSFIFO_RXUF, sport->port.membase + UARTSFIFO);
+	}
+
+	writeb(0, sport->port.membase + UARTTWFIFO);
+	writeb(1, sport->port.membase + UARTRWFIFO);
+
+	/* Enable Rx and Tx */
+	writeb(UARTCR2_RE | UARTCR2_TE, sport->port.membase + UARTCR2);
+	spin_unlock_irqrestore(&sport->port.lock, flags);
+
+	return 0;
+}
+
+static void lpuart_poll_put_char(struct uart_port *port, unsigned char c)
+{
+	/* drain */
+	while (!(readb(port->membase + UARTSR1) & UARTSR1_TDRE))
+		barrier();
+
+	writeb(c, port->membase + UARTDR);
+}
+
+static int lpuart_poll_get_char(struct uart_port *port)
+{
+	if (!(readb(port->membase + UARTSR1) & UARTSR1_RDRF))
+		return NO_POLL_CHAR;
+
+	return readb(port->membase + UARTDR);
+}
+
+#endif
+
 static inline void lpuart_transmit_buffer(struct lpuart_port *sport)
 {
 	struct circ_buf *xmit = &sport->port.state->xmit;
@@ -1595,6 +1654,11 @@ static const struct uart_ops lpuart_pops = {
 	.config_port	= lpuart_config_port,
 	.verify_port	= lpuart_verify_port,
 	.flush_buffer	= lpuart_flush_buffer,
+#if defined(CONFIG_CONSOLE_POLL)
+	.poll_init	= lpuart_poll_init,
+	.poll_get_char	= lpuart_poll_get_char,
+	.poll_put_char	= lpuart_poll_put_char,
+#endif
 };
 
 static const struct uart_ops lpuart32_pops = {
diff --git a/drivers/tty/serial/ifx6x60.c b/drivers/tty/serial/ifx6x60.c
index d386346248de..157883653256 100644
--- a/drivers/tty/serial/ifx6x60.c
+++ b/drivers/tty/serial/ifx6x60.c
@@ -1042,6 +1042,7 @@ static int ifx_spi_spi_probe(struct spi_device *spi)
 	ret = spi_setup(spi);
 	if (ret) {
 		dev_err(&spi->dev, "SPI setup wasn't successful %d", ret);
+		kfree(ifx_dev);
 		return -ENODEV;
 	}
 
diff --git a/drivers/tty/serial/ioc4_serial.c b/drivers/tty/serial/ioc4_serial.c
index e5c42fef69d2..3be051abb2a2 100644
--- a/drivers/tty/serial/ioc4_serial.c
+++ b/drivers/tty/serial/ioc4_serial.c
@@ -1082,7 +1082,7 @@ static int inline ioc4_attach_local(struct ioc4_driver_data *idd)
 		if (!port) {
 			printk(KERN_WARNING
 				"IOC4 serial memory not available for port\n");
-			return -ENOMEM;
+			goto free;
 		}
 		spin_lock_init(&port->ip_lock);
 
@@ -1190,6 +1190,11 @@ static int inline ioc4_attach_local(struct ioc4_driver_data *idd)
 				handle_dma_error_intr, port);
 	}
 	return 0;
+
+free:
+	while (port_number)
+		kfree(ports[--port_number]);
+	return -ENOMEM;
 }
 
 /**
diff --git a/drivers/tty/serial/mxs-auart.c b/drivers/tty/serial/mxs-auart.c
index 770454e0dfa3..8c1c9112b3fd 100644
--- a/drivers/tty/serial/mxs-auart.c
+++ b/drivers/tty/serial/mxs-auart.c
@@ -1016,7 +1016,7 @@ static void mxs_auart_settermios(struct uart_port *u,
 			ctrl |= AUART_LINECTRL_EPS;
 	}
 
-	u->read_status_mask = 0;
+	u->read_status_mask = AUART_STAT_OERR;
 
 	if (termios->c_iflag & INPCK)
 		u->read_status_mask |= AUART_STAT_PERR;
diff --git a/drivers/tty/serial/pxa.c b/drivers/tty/serial/pxa.c
index cd9d9e878475..75952811c0da 100644
--- a/drivers/tty/serial/pxa.c
+++ b/drivers/tty/serial/pxa.c
@@ -925,6 +925,8 @@ static struct platform_driver serial_pxa_driver = {
 	},
 };
 
+
+/* 8250 driver for PXA serial ports should be used */
 static int __init serial_pxa_init(void)
 {
 	int ret;
diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c
index fb0672554123..793395451982 100644
--- a/drivers/tty/serial/sc16is7xx.c
+++ b/drivers/tty/serial/sc16is7xx.c
@@ -1264,7 +1264,7 @@ static int sc16is7xx_probe(struct device *dev,
 
 	/* Setup interrupt */
 	ret = devm_request_irq(dev, irq, sc16is7xx_irq,
-			       IRQF_ONESHOT | flags, dev_name(dev), s);
+			       flags, dev_name(dev), s);
 	if (!ret)
 		return 0;
 
diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c
index f2303f390345..d0847375ea64 100644
--- a/drivers/tty/serial/serial_core.c
+++ b/drivers/tty/serial/serial_core.c
@@ -73,7 +73,7 @@ static inline struct uart_port *uart_port_ref(struct uart_state *state)
 
 static inline void uart_port_deref(struct uart_port *uport)
 {
-	if (uport && atomic_dec_and_test(&uport->state->refcount))
+	if (atomic_dec_and_test(&uport->state->refcount))
 		wake_up(&uport->state->remove_wait);
 }
 
@@ -88,9 +88,10 @@ static inline void uart_port_deref(struct uart_port *uport)
 #define uart_port_unlock(uport, flags)					\
 	({								\
 		struct uart_port *__uport = uport;			\
-		if (__uport)						\
+		if (__uport) {						\
 			spin_unlock_irqrestore(&__uport->lock, flags);	\
-		uart_port_deref(__uport);				\
+			uart_port_deref(__uport);			\
+		}							\
 	})
 
 static inline struct uart_port *uart_port_check(struct uart_state *state)
@@ -1515,7 +1516,10 @@ static void uart_wait_until_sent(struct tty_struct *tty, int timeout)
 	unsigned long char_time, expire;
 
 	port = uart_port_ref(state);
-	if (!port || port->type == PORT_UNKNOWN || port->fifosize == 0) {
+	if (!port)
+		return;
+
+	if (port->type == PORT_UNKNOWN || port->fifosize == 0) {
 		uart_port_deref(port);
 		return;
 	}
@@ -2365,9 +2369,10 @@ static int uart_poll_get_char(struct tty_driver *driver, int line)
 
 	if (state) {
 		port = uart_port_ref(state);
-		if (port)
+		if (port) {
 			ret = port->ops->poll_get_char(port);
-		uart_port_deref(port);
+			uart_port_deref(port);
+		}
 	}
 	return ret;
 }
diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c
index 4b26252c2885..91e7dddbf72c 100644
--- a/drivers/tty/serial/sh-sci.c
+++ b/drivers/tty/serial/sh-sci.c
@@ -1142,11 +1142,8 @@ static int sci_dma_rx_push(struct sci_port *s, void *buf, size_t count)
 	int copied;
 
 	copied = tty_insert_flip_string(tport, buf, count);
-	if (copied < count) {
-		dev_warn(port->dev, "Rx overrun: dropping %zu bytes\n",
-			 count - copied);
+	if (copied < count)
 		port->icount.buf_overrun++;
-	}
 
 	port->icount.rx += copied;
 
@@ -1161,8 +1158,6 @@ static int sci_dma_rx_find_active(struct sci_port *s)
 		if (s->active_rx == s->cookie_rx[i])
 			return i;
 
-	dev_err(s->port.dev, "%s: Rx cookie %d not found!\n", __func__,
-		s->active_rx);
 	return -1;
 }
 
@@ -1223,9 +1218,9 @@ static void sci_dma_rx_complete(void *arg)
 
 	dma_async_issue_pending(chan);
 
+	spin_unlock_irqrestore(&port->lock, flags);
 	dev_dbg(port->dev, "%s: cookie %d #%d, new active cookie %d\n",
 		__func__, s->cookie_rx[active], active, s->active_rx);
-	spin_unlock_irqrestore(&port->lock, flags);
 	return;
 
 fail:
@@ -1273,8 +1268,6 @@ static void sci_submit_rx(struct sci_port *s)
 		if (dma_submit_error(s->cookie_rx[i]))
 			goto fail;
 
-		dev_dbg(s->port.dev, "%s(): cookie %d to #%d\n", __func__,
-			s->cookie_rx[i], i);
 	}
 
 	s->active_rx = s->cookie_rx[0];
@@ -1288,7 +1281,6 @@ fail:
 	for (i = 0; i < 2; i++)
 		s->cookie_rx[i] = -EINVAL;
 	s->active_rx = -EINVAL;
-	dev_warn(s->port.dev, "Failed to re-start Rx DMA, using PIO\n");
 	sci_rx_dma_release(s, true);
 }
 
@@ -1358,10 +1350,10 @@ static void rx_timer_fn(unsigned long arg)
 	int active, count;
 	u16 scr;
 
-	spin_lock_irqsave(&port->lock, flags);
-
 	dev_dbg(port->dev, "DMA Rx timed out\n");
 
+	spin_lock_irqsave(&port->lock, flags);
+
 	active = sci_dma_rx_find_active(s);
 	if (active < 0) {
 		spin_unlock_irqrestore(&port->lock, flags);
@@ -1370,9 +1362,9 @@ static void rx_timer_fn(unsigned long arg)
 
 	status = dmaengine_tx_status(s->chan_rx, s->active_rx, &state);
 	if (status == DMA_COMPLETE) {
+		spin_unlock_irqrestore(&port->lock, flags);
 		dev_dbg(port->dev, "Cookie %d #%d has already completed\n",
 			s->active_rx, active);
-		spin_unlock_irqrestore(&port->lock, flags);
 
 		/* Let packet complete handler take care of the packet */
 		return;
@@ -1396,8 +1388,6 @@ static void rx_timer_fn(unsigned long arg)
 	/* Handle incomplete DMA receive */
 	dmaengine_terminate_all(s->chan_rx);
 	read = sg_dma_len(&s->sg_rx[active]) - state.residue;
-	dev_dbg(port->dev, "Read %u bytes with cookie %d\n", read,
-		s->active_rx);
 
 	if (read) {
 		count = sci_dma_rx_push(s, s->rx_buf[active], read);
diff --git a/drivers/tty/serial/sunhv.c b/drivers/tty/serial/sunhv.c
index 4e603d060e80..99ef5c6e4766 100644
--- a/drivers/tty/serial/sunhv.c
+++ b/drivers/tty/serial/sunhv.c
@@ -598,7 +598,8 @@ static int hv_remove(struct platform_device *dev)
 	uart_remove_one_port(&sunhv_reg, port);
 
 	sunserial_unregister_minors(&sunhv_reg, 1);
-
+	kfree(con_read_page);
+	kfree(con_write_page);
 	kfree(port);
 	sunhv_port = NULL;
 
diff --git a/drivers/tty/serial/sunsu.c b/drivers/tty/serial/sunsu.c
index 9ad98eaa35bf..72df2e1b88af 100644
--- a/drivers/tty/serial/sunsu.c
+++ b/drivers/tty/serial/sunsu.c
@@ -1500,6 +1500,7 @@ static int su_probe(struct platform_device *op)
 
 out_unmap:
 	of_iounmap(&op->resource[0], up->port.membase, up->reg_size);
+	kfree(up);
 	return err;
 }
 
diff --git a/drivers/tty/vt/consolemap.c b/drivers/tty/vt/consolemap.c
index 9d7ab7b66a8a..71e81406ef71 100644
--- a/drivers/tty/vt/consolemap.c
+++ b/drivers/tty/vt/consolemap.c
@@ -9,6 +9,17 @@
  * Support for multiple unimaps by Jakub Jelinek <jj@ultra.linux.cz>, July 1998
  *
  * Fix bug in inverse translation. Stanislav Voronyi <stas@cnti.uanet.kharkov.ua>, Dec 1998
+ *
+ * In order to prevent the following circular lock dependency:
+ *   &mm->mmap_sem --> cpu_hotplug.lock --> console_lock --> &mm->mmap_sem
+ *
+ * We cannot allow page fault to happen while holding the console_lock.
+ * Therefore, all the userspace copy operations have to be done outside
+ * the console_lock critical sections.
+ *
+ * As all the affected functions are all called directly from vt_ioctl(), we
+ * can allocate some small buffers directly on stack without worrying about
+ * stack overflow.
  */
 
 #include <linux/module.h>
@@ -22,6 +33,7 @@
 #include <linux/console.h>
 #include <linux/consolemap.h>
 #include <linux/vt_kern.h>
+#include <linux/string.h>
 
 static unsigned short translations[][256] = {
   /* 8-bit Latin-1 mapped to Unicode -- trivial mapping */
@@ -309,18 +321,19 @@ static void update_user_maps(void)
 int con_set_trans_old(unsigned char __user * arg)
 {
 	int i;
-	unsigned short *p = translations[USER_MAP];
+	unsigned short inbuf[E_TABSZ];
 
 	if (!access_ok(VERIFY_READ, arg, E_TABSZ))
 		return -EFAULT;
 
-	console_lock();
-	for (i=0; i<E_TABSZ ; i++) {
+	for (i = 0; i < E_TABSZ ; i++) {
 		unsigned char uc;
 		__get_user(uc, arg+i);
-		p[i] = UNI_DIRECT_BASE | uc;
+		inbuf[i] = UNI_DIRECT_BASE | uc;
 	}
 
+	console_lock();
+	memcpy(translations[USER_MAP], inbuf, sizeof(inbuf));
 	update_user_maps();
 	console_unlock();
 	return 0;
@@ -330,35 +343,37 @@ int con_get_trans_old(unsigned char __user * arg)
 {
 	int i, ch;
 	unsigned short *p = translations[USER_MAP];
+	unsigned char outbuf[E_TABSZ];
 
 	if (!access_ok(VERIFY_WRITE, arg, E_TABSZ))
 		return -EFAULT;
 
 	console_lock();
-	for (i=0; i<E_TABSZ ; i++)
+	for (i = 0; i < E_TABSZ ; i++)
 	{
 		ch = conv_uni_to_pc(vc_cons[fg_console].d, p[i]);
-		__put_user((ch & ~0xff) ? 0 : ch, arg+i);
+		outbuf[i] = (ch & ~0xff) ? 0 : ch;
 	}
 	console_unlock();
+
+	for (i = 0; i < E_TABSZ ; i++)
+		__put_user(outbuf[i], arg+i);
 	return 0;
 }
 
 int con_set_trans_new(ushort __user * arg)
 {
 	int i;
-	unsigned short *p = translations[USER_MAP];
+	unsigned short inbuf[E_TABSZ];
 
 	if (!access_ok(VERIFY_READ, arg, E_TABSZ*sizeof(unsigned short)))
 		return -EFAULT;
 
-	console_lock();
-	for (i=0; i<E_TABSZ ; i++) {
-		unsigned short us;
-		__get_user(us, arg+i);
-		p[i] = us;
-	}
+	for (i = 0; i < E_TABSZ ; i++)
+		__get_user(inbuf[i], arg+i);
 
+	console_lock();
+	memcpy(translations[USER_MAP], inbuf, sizeof(inbuf));
 	update_user_maps();
 	console_unlock();
 	return 0;
@@ -367,16 +382,17 @@ int con_set_trans_new(ushort __user * arg)
 int con_get_trans_new(ushort __user * arg)
 {
 	int i;
-	unsigned short *p = translations[USER_MAP];
+	unsigned short outbuf[E_TABSZ];
 
 	if (!access_ok(VERIFY_WRITE, arg, E_TABSZ*sizeof(unsigned short)))
 		return -EFAULT;
 
 	console_lock();
-	for (i=0; i<E_TABSZ ; i++)
-	  __put_user(p[i], arg+i);
+	memcpy(outbuf, translations[USER_MAP], sizeof(outbuf));
 	console_unlock();
-	
+
+	for (i = 0; i < E_TABSZ ; i++)
+		__put_user(outbuf[i], arg+i);
 	return 0;
 }
 
@@ -536,10 +552,20 @@ int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list)
 {
 	int err = 0, err1, i;
 	struct uni_pagedir *p, *q;
+	struct unipair *unilist, *plist;
 
 	if (!ct)
 		return 0;
 
+	unilist = kmalloc_array(ct, sizeof(struct unipair), GFP_KERNEL);
+	if (!unilist)
+		return -ENOMEM;
+
+	for (i = ct, plist = unilist; i; i--, plist++, list++) {
+		__get_user(plist->unicode, &list->unicode);
+		__get_user(plist->fontpos, &list->fontpos);
+	}
+
 	console_lock();
 
 	/* Save original vc_unipagdir_loc in case we allocate a new one */
@@ -557,8 +583,8 @@ int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list)
 		
 		err1 = con_do_clear_unimap(vc);
 		if (err1) {
-			console_unlock();
-			return err1;
+			err = err1;
+			goto out_unlock;
 		}
 		
 		/*
@@ -592,8 +618,8 @@ int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list)
 						*vc->vc_uni_pagedir_loc = p;
 						con_release_unimap(q);
 						kfree(q);
-						console_unlock();
-						return err1; 
+						err = err1;
+						goto out_unlock;
 					}
 				}
 			} else {
@@ -617,22 +643,17 @@ int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list)
 	/*
 	 * Insert user specified unicode pairs into new table.
 	 */
-	while (ct--) {
-		unsigned short unicode, fontpos;
-		__get_user(unicode, &list->unicode);
-		__get_user(fontpos, &list->fontpos);
-		if ((err1 = con_insert_unipair(p, unicode,fontpos)) != 0)
+	for (plist = unilist; ct; ct--, plist++) {
+		err1 = con_insert_unipair(p, plist->unicode, plist->fontpos);
+		if (err1)
 			err = err1;
-		list++;
 	}
 	
 	/*
 	 * Merge with fontmaps of any other virtual consoles.
 	 */
-	if (con_unify_unimap(vc, p)) {
-		console_unlock();
-		return err;
-	}
+	if (con_unify_unimap(vc, p))
+		goto out_unlock;
 
 	for (i = 0; i <= 3; i++)
 		set_inverse_transl(vc, p, i); /* Update inverse translations */
@@ -640,6 +661,7 @@ int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list)
 
 out_unlock:
 	console_unlock();
+	kfree(unilist);
 	return err;
 }
 
@@ -735,9 +757,15 @@ EXPORT_SYMBOL(con_copy_unimap);
  */
 int con_get_unimap(struct vc_data *vc, ushort ct, ushort __user *uct, struct unipair __user *list)
 {
-	int i, j, k, ect;
+	int i, j, k;
+	ushort ect;
 	u16 **p1, *p2;
 	struct uni_pagedir *p;
+	struct unipair *unilist, *plist;
+
+	unilist = kmalloc_array(ct, sizeof(struct unipair), GFP_KERNEL);
+	if (!unilist)
+		return -ENOMEM;
 
 	console_lock();
 
@@ -750,21 +778,26 @@ int con_get_unimap(struct vc_data *vc, ushort ct, ushort __user *uct, struct uni
 			for (j = 0; j < 32; j++) {
 			p2 = *(p1++);
 			if (p2)
-				for (k = 0; k < 64; k++) {
-					if (*p2 < MAX_GLYPH && ect++ < ct) {
-						__put_user((u_short)((i<<11)+(j<<6)+k),
-							   &list->unicode);
-						__put_user((u_short) *p2, 
-							   &list->fontpos);
-						list++;
+				for (k = 0; k < 64; k++, p2++) {
+					if (*p2 >= MAX_GLYPH)
+						continue;
+					if (ect < ct) {
+						unilist[ect].unicode =
+							(i<<11)+(j<<6)+k;
+						unilist[ect].fontpos = *p2;
 					}
-					p2++;
+					ect++;
 				}
 			}
 		}
 	}
-	__put_user(ect, uct);
 	console_unlock();
+	for (i = min(ect, ct), plist = unilist; i; i--, list++, plist++) {
+		__put_user(plist->unicode, &list->unicode);
+		__put_user(plist->fontpos, &list->fontpos);
+	}
+	__put_user(ect, uct);
+	kfree(unilist);
 	return ((ect <= ct) ? 0 : -ENOMEM);
 }
 
diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c
index 0f8caae4267d..3dd6a491cdba 100644
--- a/drivers/tty/vt/keyboard.c
+++ b/drivers/tty/vt/keyboard.c
@@ -982,7 +982,7 @@ static void kbd_led_trigger_activate(struct led_classdev *cdev)
 	KBD_LED_TRIGGER((_led_bit) + 8, _name)
 
 static struct kbd_led_trigger kbd_led_triggers[] = {
-	KBD_LED_TRIGGER(VC_SCROLLOCK, "kbd-scrollock"),
+	KBD_LED_TRIGGER(VC_SCROLLOCK, "kbd-scrolllock"),
 	KBD_LED_TRIGGER(VC_NUMLOCK,   "kbd-numlock"),
 	KBD_LED_TRIGGER(VC_CAPSLOCK,  "kbd-capslock"),
 	KBD_LED_TRIGGER(VC_KANALOCK,  "kbd-kanalock"),
@@ -1256,7 +1256,7 @@ static int emulate_raw(struct vc_data *vc, unsigned int keycode,
 	case KEY_SYSRQ:
 		/*
 		 * Real AT keyboards (that's what we're trying
-		 * to emulate here emit 0xe0 0x2a 0xe0 0x37 when
+		 * to emulate here) emit 0xe0 0x2a 0xe0 0x37 when
 		 * pressing PrtSc/SysRq alone, but simply 0x54
 		 * when pressing Alt+PrtSc/SysRq.
 		 */
diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c
index 8c3bf3d613c0..4c10a9df3b91 100644
--- a/drivers/tty/vt/vt.c
+++ b/drivers/tty/vt/vt.c
@@ -315,38 +315,27 @@ void schedule_console_callback(void)
 	schedule_work(&console_work);
 }
 
-static void scrup(struct vc_data *vc, unsigned int t, unsigned int b, int nr)
+static void con_scroll(struct vc_data *vc, unsigned int t, unsigned int b,
+		enum con_scroll dir, unsigned int nr)
 {
-	unsigned short *d, *s;
+	u16 *clear, *d, *s;
 
-	if (t+nr >= b)
+	if (t + nr >= b)
 		nr = b - t - 1;
 	if (b > vc->vc_rows || t >= b || nr < 1)
 		return;
-	if (con_is_visible(vc) && vc->vc_sw->con_scroll(vc, t, b, SM_UP, nr))
+	if (con_is_visible(vc) && vc->vc_sw->con_scroll(vc, t, b, dir, nr))
 		return;
-	d = (unsigned short *)(vc->vc_origin + vc->vc_size_row * t);
-	s = (unsigned short *)(vc->vc_origin + vc->vc_size_row * (t + nr));
-	scr_memmovew(d, s, (b - t - nr) * vc->vc_size_row);
-	scr_memsetw(d + (b - t - nr) * vc->vc_cols, vc->vc_video_erase_char,
-		    vc->vc_size_row * nr);
-}
 
-static void scrdown(struct vc_data *vc, unsigned int t, unsigned int b, int nr)
-{
-	unsigned short *s;
-	unsigned int step;
+	s = clear = (u16 *)(vc->vc_origin + vc->vc_size_row * t);
+	d = (u16 *)(vc->vc_origin + vc->vc_size_row * (t + nr));
 
-	if (t+nr >= b)
-		nr = b - t - 1;
-	if (b > vc->vc_rows || t >= b || nr < 1)
-		return;
-	if (con_is_visible(vc) && vc->vc_sw->con_scroll(vc, t, b, SM_DOWN, nr))
-		return;
-	s = (unsigned short *)(vc->vc_origin + vc->vc_size_row * t);
-	step = vc->vc_cols * nr;
-	scr_memmovew(s + step, s, (b - t - nr) * vc->vc_size_row);
-	scr_memsetw(s, vc->vc_video_erase_char, 2 * step);
+	if (dir == SM_UP) {
+		clear = s + (b - t - nr) * vc->vc_cols;
+		swap(s, d);
+	}
+	scr_memmovew(d, s, (b - t - nr) * vc->vc_size_row);
+	scr_memsetw(clear, vc->vc_video_erase_char, vc->vc_size_row * nr);
 }
 
 static void do_update_region(struct vc_data *vc, unsigned long start, int count)
@@ -1120,7 +1109,7 @@ static void lf(struct vc_data *vc)
 	 * if below scrolling region
 	 */
     	if (vc->vc_y + 1 == vc->vc_bottom)
-		scrup(vc, vc->vc_top, vc->vc_bottom, 1);
+		con_scroll(vc, vc->vc_top, vc->vc_bottom, SM_UP, 1);
 	else if (vc->vc_y < vc->vc_rows - 1) {
 	    	vc->vc_y++;
 		vc->vc_pos += vc->vc_size_row;
@@ -1135,7 +1124,7 @@ static void ri(struct vc_data *vc)
 	 * if above scrolling region
 	 */
 	if (vc->vc_y == vc->vc_top)
-		scrdown(vc, vc->vc_top, vc->vc_bottom, 1);
+		con_scroll(vc, vc->vc_top, vc->vc_bottom, SM_DOWN, 1);
 	else if (vc->vc_y > 0) {
 		vc->vc_y--;
 		vc->vc_pos -= vc->vc_size_row;
@@ -1631,7 +1620,7 @@ static void csi_L(struct vc_data *vc, unsigned int nr)
 		nr = vc->vc_rows - vc->vc_y;
 	else if (!nr)
 		nr = 1;
-	scrdown(vc, vc->vc_y, vc->vc_bottom, nr);
+	con_scroll(vc, vc->vc_y, vc->vc_bottom, SM_DOWN, nr);
 	vc->vc_need_wrap = 0;
 }
 
@@ -1652,7 +1641,7 @@ static void csi_M(struct vc_data *vc, unsigned int nr)
 		nr = vc->vc_rows - vc->vc_y;
 	else if (!nr)
 		nr=1;
-	scrup(vc, vc->vc_y, vc->vc_bottom, nr);
+	con_scroll(vc, vc->vc_y, vc->vc_bottom, SM_UP, nr);
 	vc->vc_need_wrap = 0;
 }
 
@@ -3934,10 +3923,6 @@ void unblank_screen(void)
  */
 static void blank_screen_t(unsigned long dummy)
 {
-	if (unlikely(!keventd_up())) {
-		mod_timer(&console_timer, jiffies + (blankinterval * HZ));
-		return;
-	}
 	blank_timer_expired = 1;
 	schedule_work(&console_work);
 }
@@ -4295,6 +4280,46 @@ void vcs_scr_updated(struct vc_data *vc)
 	notify_update(vc);
 }
 
+void vc_scrolldelta_helper(struct vc_data *c, int lines,
+		unsigned int rolled_over, void *base, unsigned int size)
+{
+	unsigned long ubase = (unsigned long)base;
+	ptrdiff_t scr_end = (void *)c->vc_scr_end - base;
+	ptrdiff_t vorigin = (void *)c->vc_visible_origin - base;
+	ptrdiff_t origin = (void *)c->vc_origin - base;
+	int margin = c->vc_size_row * 4;
+	int from, wrap, from_off, avail;
+
+	/* Turn scrollback off */
+	if (!lines) {
+		c->vc_visible_origin = c->vc_origin;
+		return;
+	}
+
+	/* Do we have already enough to allow jumping from 0 to the end? */
+	if (rolled_over > scr_end + margin) {
+		from = scr_end;
+		wrap = rolled_over + c->vc_size_row;
+	} else {
+		from = 0;
+		wrap = size;
+	}
+
+	from_off = (vorigin - from + wrap) % wrap + lines * c->vc_size_row;
+	avail = (origin - from + wrap) % wrap;
+
+	/* Only a little piece would be left? Show all incl. the piece! */
+	if (avail < 2 * margin)
+		margin = 0;
+	if (from_off < margin)
+		from_off = 0;
+	if (from_off > avail - margin)
+		from_off = avail;
+
+	c->vc_visible_origin = ubase + (from + from_off) % wrap;
+}
+EXPORT_SYMBOL_GPL(vc_scrolldelta_helper);
+
 /*
  *	Visible symbols for modules
  */
diff --git a/drivers/uio/Kconfig b/drivers/uio/Kconfig
index 52c98ce1b6fe..7e8dc78a9796 100644
--- a/drivers/uio/Kconfig
+++ b/drivers/uio/Kconfig
@@ -155,4 +155,13 @@ config UIO_MF624
 
 	  If you compile this as a module, it will be called uio_mf624.
 
+config UIO_HV_GENERIC
+	tristate "Generic driver for Hyper-V VMBus"
+	depends on HYPERV
+	help
+	  Generic driver that you can bind, dynamically, to any
+	  Hyper-V VMBus device. It is useful to provide direct access
+	  to network and storage devices from userspace.
+
+	  If you compile this as a module, it will be called uio_hv_generic.
 endif
diff --git a/drivers/uio/Makefile b/drivers/uio/Makefile
index 8560dad52d0f..e9663bb8a4c7 100644
--- a/drivers/uio/Makefile
+++ b/drivers/uio/Makefile
@@ -9,3 +9,4 @@ obj-$(CONFIG_UIO_NETX)	+= uio_netx.o
 obj-$(CONFIG_UIO_PRUSS)         += uio_pruss.o
 obj-$(CONFIG_UIO_MF624)         += uio_mf624.o
 obj-$(CONFIG_UIO_FSL_ELBC_GPCM)	+= uio_fsl_elbc_gpcm.o
+obj-$(CONFIG_UIO_HV_GENERIC)	+= uio_hv_generic.o
diff --git a/drivers/uio/uio_hv_generic.c b/drivers/uio/uio_hv_generic.c
new file mode 100644
index 000000000000..50958f167305
--- /dev/null
+++ b/drivers/uio/uio_hv_generic.c
@@ -0,0 +1,218 @@
+/*
+ * uio_hv_generic - generic UIO driver for VMBus
+ *
+ * Copyright (c) 2013-2016 Brocade Communications Systems, Inc.
+ * Copyright (c) 2016, Microsoft Corporation.
+ *
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ *
+ * Since the driver does not declare any device ids, you must allocate
+ * id and bind the device to the driver yourself.  For example:
+ *
+ * # echo "f8615163-df3e-46c5-913f-f2d2f965ed0e" \
+ *    > /sys/bus/vmbus/drivers/uio_hv_generic
+ * # echo -n vmbus-ed963694-e847-4b2a-85af-bc9cfc11d6f3 \
+ *    > /sys/bus/vmbus/drivers/hv_netvsc/unbind
+ * # echo -n vmbus-ed963694-e847-4b2a-85af-bc9cfc11d6f3 \
+ *    > /sys/bus/vmbus/drivers/uio_hv_generic/bind
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/uio_driver.h>
+#include <linux/netdevice.h>
+#include <linux/if_ether.h>
+#include <linux/skbuff.h>
+#include <linux/hyperv.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+
+#include "../hv/hyperv_vmbus.h"
+
+#define DRIVER_VERSION	"0.02.0"
+#define DRIVER_AUTHOR	"Stephen Hemminger <sthemmin at microsoft.com>"
+#define DRIVER_DESC	"Generic UIO driver for VMBus devices"
+
+/*
+ * List of resources to be mapped to user space
+ * can be extended up to MAX_UIO_MAPS(5) items
+ */
+enum hv_uio_map {
+	TXRX_RING_MAP = 0,
+	INT_PAGE_MAP,
+	MON_PAGE_MAP,
+};
+
+#define HV_RING_SIZE	512
+
+struct hv_uio_private_data {
+	struct uio_info info;
+	struct hv_device *device;
+};
+
+static int
+hv_uio_mmap(struct uio_info *info, struct vm_area_struct *vma)
+{
+	int mi;
+
+	if (vma->vm_pgoff >= MAX_UIO_MAPS)
+		return -EINVAL;
+
+	if (info->mem[vma->vm_pgoff].size == 0)
+		return -EINVAL;
+
+	mi = (int)vma->vm_pgoff;
+
+	return remap_pfn_range(vma, vma->vm_start,
+			info->mem[mi].addr >> PAGE_SHIFT,
+			vma->vm_end - vma->vm_start, vma->vm_page_prot);
+}
+
+/*
+ * This is the irqcontrol callback to be registered to uio_info.
+ * It can be used to disable/enable interrupt from user space processes.
+ *
+ * @param info
+ *  pointer to uio_info.
+ * @param irq_state
+ *  state value. 1 to enable interrupt, 0 to disable interrupt.
+ */
+static int
+hv_uio_irqcontrol(struct uio_info *info, s32 irq_state)
+{
+	struct hv_uio_private_data *pdata = info->priv;
+	struct hv_device *dev = pdata->device;
+
+	dev->channel->inbound.ring_buffer->interrupt_mask = !irq_state;
+	virt_mb();
+
+	return 0;
+}
+
+/*
+ * Callback from vmbus_event when something is in inbound ring.
+ */
+static void hv_uio_channel_cb(void *context)
+{
+	struct hv_uio_private_data *pdata = context;
+	struct hv_device *dev = pdata->device;
+
+	dev->channel->inbound.ring_buffer->interrupt_mask = 1;
+	virt_mb();
+
+	uio_event_notify(&pdata->info);
+}
+
+static int
+hv_uio_probe(struct hv_device *dev,
+	     const struct hv_vmbus_device_id *dev_id)
+{
+	struct hv_uio_private_data *pdata;
+	int ret;
+
+	pdata = kzalloc(sizeof(*pdata), GFP_KERNEL);
+	if (!pdata)
+		return -ENOMEM;
+
+	ret = vmbus_open(dev->channel, HV_RING_SIZE * PAGE_SIZE,
+			 HV_RING_SIZE * PAGE_SIZE, NULL, 0,
+			 hv_uio_channel_cb, pdata);
+	if (ret)
+		goto fail;
+
+	dev->channel->inbound.ring_buffer->interrupt_mask = 1;
+	dev->channel->batched_reading = false;
+
+	/* Fill general uio info */
+	pdata->info.name = "uio_hv_generic";
+	pdata->info.version = DRIVER_VERSION;
+	pdata->info.irqcontrol = hv_uio_irqcontrol;
+	pdata->info.mmap = hv_uio_mmap;
+	pdata->info.irq = UIO_IRQ_CUSTOM;
+
+	/* mem resources */
+	pdata->info.mem[TXRX_RING_MAP].name = "txrx_rings";
+	pdata->info.mem[TXRX_RING_MAP].addr
+		= virt_to_phys(dev->channel->ringbuffer_pages);
+	pdata->info.mem[TXRX_RING_MAP].size
+		= dev->channel->ringbuffer_pagecount * PAGE_SIZE;
+	pdata->info.mem[TXRX_RING_MAP].memtype = UIO_MEM_LOGICAL;
+
+	pdata->info.mem[INT_PAGE_MAP].name = "int_page";
+	pdata->info.mem[INT_PAGE_MAP].addr =
+		virt_to_phys(vmbus_connection.int_page);
+	pdata->info.mem[INT_PAGE_MAP].size = PAGE_SIZE;
+	pdata->info.mem[INT_PAGE_MAP].memtype = UIO_MEM_LOGICAL;
+
+	pdata->info.mem[MON_PAGE_MAP].name = "monitor_pages";
+	pdata->info.mem[MON_PAGE_MAP].addr =
+		virt_to_phys(vmbus_connection.monitor_pages[1]);
+	pdata->info.mem[MON_PAGE_MAP].size = PAGE_SIZE;
+	pdata->info.mem[MON_PAGE_MAP].memtype = UIO_MEM_LOGICAL;
+
+	pdata->info.priv = pdata;
+	pdata->device = dev;
+
+	ret = uio_register_device(&dev->device, &pdata->info);
+	if (ret) {
+		dev_err(&dev->device, "hv_uio register failed\n");
+		goto fail_close;
+	}
+
+	hv_set_drvdata(dev, pdata);
+
+	return 0;
+
+fail_close:
+	vmbus_close(dev->channel);
+fail:
+	kfree(pdata);
+
+	return ret;
+}
+
+static int
+hv_uio_remove(struct hv_device *dev)
+{
+	struct hv_uio_private_data *pdata = hv_get_drvdata(dev);
+
+	if (!pdata)
+		return 0;
+
+	uio_unregister_device(&pdata->info);
+	hv_set_drvdata(dev, NULL);
+	vmbus_close(dev->channel);
+	kfree(pdata);
+	return 0;
+}
+
+static struct hv_driver hv_uio_drv = {
+	.name = "uio_hv_generic",
+	.id_table = NULL, /* only dynamic id's */
+	.probe = hv_uio_probe,
+	.remove = hv_uio_remove,
+};
+
+static int __init
+hyperv_module_init(void)
+{
+	return vmbus_driver_register(&hv_uio_drv);
+}
+
+static void __exit
+hyperv_module_exit(void)
+{
+	vmbus_driver_unregister(&hv_uio_drv);
+}
+
+module_init(hyperv_module_init);
+module_exit(hyperv_module_exit);
+
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESC);
diff --git a/drivers/uio/uio_pruss.c b/drivers/uio/uio_pruss.c
index ca9e2fafb0b6..31d5b1d3b5af 100644
--- a/drivers/uio/uio_pruss.c
+++ b/drivers/uio/uio_pruss.c
@@ -111,6 +111,7 @@ static void pruss_cleanup(struct device *dev, struct uio_pruss_dev *gdev)
 			      gdev->sram_vaddr,
 			      sram_pool_sz);
 	kfree(gdev->info);
+	clk_disable(gdev->pruss_clk);
 	clk_put(gdev->pruss_clk);
 	kfree(gdev);
 }
@@ -143,7 +144,14 @@ static int pruss_probe(struct platform_device *pdev)
 		kfree(gdev);
 		return ret;
 	} else {
-		clk_enable(gdev->pruss_clk);
+		ret = clk_enable(gdev->pruss_clk);
+		if (ret) {
+			dev_err(dev, "Failed to enable clock\n");
+			clk_put(gdev->pruss_clk);
+			kfree(gdev->info);
+			kfree(gdev);
+			return ret;
+		}
 	}
 
 	regs_prussio = platform_get_resource(pdev, IORESOURCE_MEM, 0);
diff --git a/drivers/usb/Kconfig b/drivers/usb/Kconfig
index 644e978cbd3e..fbe493d44e81 100644
--- a/drivers/usb/Kconfig
+++ b/drivers/usb/Kconfig
@@ -95,6 +95,8 @@ source "drivers/usb/usbip/Kconfig"
 
 endif
 
+source "drivers/usb/mtu3/Kconfig"
+
 source "drivers/usb/musb/Kconfig"
 
 source "drivers/usb/dwc3/Kconfig"
diff --git a/drivers/usb/Makefile b/drivers/usb/Makefile
index dca78565eb55..7791af6c102c 100644
--- a/drivers/usb/Makefile
+++ b/drivers/usb/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_USB_DWC2)		+= dwc2/
 obj-$(CONFIG_USB_ISP1760)	+= isp1760/
 
 obj-$(CONFIG_USB_MON)		+= mon/
+obj-$(CONFIG_USB_MTU3)		+= mtu3/
 
 obj-$(CONFIG_PCI)		+= host/
 obj-$(CONFIG_USB_EHCI_HCD)	+= host/
diff --git a/drivers/usb/chipidea/ci_hdrc_imx.c b/drivers/usb/chipidea/ci_hdrc_imx.c
index 099179457f60..5f4a8157fad8 100644
--- a/drivers/usb/chipidea/ci_hdrc_imx.c
+++ b/drivers/usb/chipidea/ci_hdrc_imx.c
@@ -18,6 +18,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/dma-mapping.h>
 #include <linux/usb/chipidea.h>
+#include <linux/usb/of.h>
 #include <linux/clk.h>
 
 #include "ci.h"
@@ -146,6 +147,9 @@ static struct imx_usbmisc_data *usbmisc_get_init_data(struct device *dev)
 	if (of_find_property(np, "external-vbus-divider", NULL))
 		data->evdo = 1;
 
+	if (of_usb_get_phy_mode(np) == USBPHY_INTERFACE_MODE_ULPI)
+		data->ulpi = 1;
+
 	return data;
 }
 
diff --git a/drivers/usb/chipidea/ci_hdrc_imx.h b/drivers/usb/chipidea/ci_hdrc_imx.h
index 409aa5ca8dda..d666c9f036ba 100644
--- a/drivers/usb/chipidea/ci_hdrc_imx.h
+++ b/drivers/usb/chipidea/ci_hdrc_imx.h
@@ -19,6 +19,7 @@ struct imx_usbmisc_data {
 	unsigned int disable_oc:1; /* over current detect disabled */
 	unsigned int oc_polarity:1; /* over current polarity if oc enabled */
 	unsigned int evdo:1; /* set external vbus divider option */
+	unsigned int ulpi:1; /* connected to an ULPI phy */
 };
 
 int imx_usbmisc_init(struct imx_usbmisc_data *);
diff --git a/drivers/usb/chipidea/udc.c b/drivers/usb/chipidea/udc.c
index c9e80ad48fdc..cf132f057137 100644
--- a/drivers/usb/chipidea/udc.c
+++ b/drivers/usb/chipidea/udc.c
@@ -365,7 +365,7 @@ static int add_td_to_list(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq,
 		if (hwreq->req.length == 0
 				|| hwreq->req.length % hwep->ep.maxpacket)
 			mul++;
-		node->ptr->token |= mul << __ffs(TD_MULTO);
+		node->ptr->token |= cpu_to_le32(mul << __ffs(TD_MULTO));
 	}
 
 	temp = (u32) (hwreq->req.dma + hwreq->req.actual);
@@ -504,7 +504,7 @@ static int _hardware_enqueue(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq)
 		if (hwreq->req.length == 0
 				|| hwreq->req.length % hwep->ep.maxpacket)
 			mul++;
-		hwep->qh.ptr->cap |= mul << __ffs(QH_MULT);
+		hwep->qh.ptr->cap |= cpu_to_le32(mul << __ffs(QH_MULT));
 	}
 
 	ret = hw_ep_prime(ci, hwep->num, hwep->dir,
@@ -529,7 +529,7 @@ static void free_pending_td(struct ci_hw_ep *hwep)
 static int reprime_dtd(struct ci_hdrc *ci, struct ci_hw_ep *hwep,
 					   struct td_node *node)
 {
-	hwep->qh.ptr->td.next = node->dma;
+	hwep->qh.ptr->td.next = cpu_to_le32(node->dma);
 	hwep->qh.ptr->td.token &=
 		cpu_to_le32(~(TD_STATUS_HALTED | TD_STATUS_ACTIVE));
 
@@ -821,7 +821,7 @@ static int _ep_queue(struct usb_ep *ep, struct usb_request *req,
 	}
 
 	if (usb_endpoint_xfer_isoc(hwep->ep.desc) &&
-	    hwreq->req.length > (1 + hwep->ep.mult) * hwep->ep.maxpacket) {
+	    hwreq->req.length > hwep->ep.mult * hwep->ep.maxpacket) {
 		dev_err(hwep->ci->dev, "request length too big for isochronous\n");
 		return -EMSGSIZE;
 	}
@@ -1253,8 +1253,8 @@ static int ep_enable(struct usb_ep *ep,
 	hwep->num  = usb_endpoint_num(desc);
 	hwep->type = usb_endpoint_type(desc);
 
-	hwep->ep.maxpacket = usb_endpoint_maxp(desc) & 0x07ff;
-	hwep->ep.mult = QH_ISO_MULT(usb_endpoint_maxp(desc));
+	hwep->ep.maxpacket = usb_endpoint_maxp(desc);
+	hwep->ep.mult = usb_endpoint_maxp_mult(desc);
 
 	if (hwep->type == USB_ENDPOINT_XFER_CONTROL)
 		cap |= QH_IOS;
diff --git a/drivers/usb/chipidea/udc.h b/drivers/usb/chipidea/udc.h
index e66df0020bd4..2ecd1174d66c 100644
--- a/drivers/usb/chipidea/udc.h
+++ b/drivers/usb/chipidea/udc.h
@@ -22,11 +22,11 @@
 /* DMA layout of transfer descriptors */
 struct ci_hw_td {
 	/* 0 */
-	u32 next;
+	__le32 next;
 #define TD_TERMINATE          BIT(0)
 #define TD_ADDR_MASK          (0xFFFFFFEUL << 5)
 	/* 1 */
-	u32 token;
+	__le32 token;
 #define TD_STATUS             (0x00FFUL <<  0)
 #define TD_STATUS_TR_ERR      BIT(3)
 #define TD_STATUS_DT_ERR      BIT(5)
@@ -36,7 +36,7 @@ struct ci_hw_td {
 #define TD_IOC                BIT(15)
 #define TD_TOTAL_BYTES        (0x7FFFUL << 16)
 	/* 2 */
-	u32 page[5];
+	__le32 page[5];
 #define TD_CURR_OFFSET        (0x0FFFUL <<  0)
 #define TD_FRAME_NUM          (0x07FFUL <<  0)
 #define TD_RESERVED_MASK      (0x0FFFUL <<  0)
@@ -45,18 +45,18 @@ struct ci_hw_td {
 /* DMA layout of queue heads */
 struct ci_hw_qh {
 	/* 0 */
-	u32 cap;
+	__le32 cap;
 #define QH_IOS                BIT(15)
 #define QH_MAX_PKT            (0x07FFUL << 16)
 #define QH_ZLT                BIT(29)
 #define QH_MULT               (0x0003UL << 30)
 #define QH_ISO_MULT(x)		((x >> 11) & 0x03)
 	/* 1 */
-	u32 curr;
+	__le32 curr;
 	/* 2 - 8 */
 	struct ci_hw_td		td;
 	/* 9 */
-	u32 RESERVED;
+	__le32 RESERVED;
 	struct usb_ctrlrequest   setup;
 } __attribute__ ((packed, aligned(4)));
 
diff --git a/drivers/usb/chipidea/usbmisc_imx.c b/drivers/usb/chipidea/usbmisc_imx.c
index 20d02a5e418d..e77a4ed4f021 100644
--- a/drivers/usb/chipidea/usbmisc_imx.c
+++ b/drivers/usb/chipidea/usbmisc_imx.c
@@ -46,11 +46,23 @@
 
 #define MX53_USB_OTG_PHY_CTRL_0_OFFSET	0x08
 #define MX53_USB_OTG_PHY_CTRL_1_OFFSET	0x0c
+#define MX53_USB_CTRL_1_OFFSET	        0x10
+#define MX53_USB_CTRL_1_H2_XCVR_CLK_SEL_MASK (0x11 << 2)
+#define MX53_USB_CTRL_1_H2_XCVR_CLK_SEL_ULPI BIT(2)
+#define MX53_USB_CTRL_1_H3_XCVR_CLK_SEL_MASK (0x11 << 6)
+#define MX53_USB_CTRL_1_H3_XCVR_CLK_SEL_ULPI BIT(6)
 #define MX53_USB_UH2_CTRL_OFFSET	0x14
 #define MX53_USB_UH3_CTRL_OFFSET	0x18
+#define MX53_USB_CLKONOFF_CTRL_OFFSET	0x24
+#define MX53_USB_CLKONOFF_CTRL_H2_INT60CKOFF BIT(21)
+#define MX53_USB_CLKONOFF_CTRL_H3_INT60CKOFF BIT(22)
 #define MX53_BM_OVER_CUR_DIS_H1		BIT(5)
 #define MX53_BM_OVER_CUR_DIS_OTG	BIT(8)
 #define MX53_BM_OVER_CUR_DIS_UHx	BIT(30)
+#define MX53_USB_CTRL_1_UH2_ULPI_EN	BIT(26)
+#define MX53_USB_CTRL_1_UH3_ULPI_EN	BIT(27)
+#define MX53_USB_UHx_CTRL_WAKE_UP_EN	BIT(7)
+#define MX53_USB_UHx_CTRL_ULPI_INT_EN	BIT(8)
 #define MX53_USB_PHYCTRL1_PLLDIV_MASK	0x3
 #define MX53_USB_PLL_DIV_24_MHZ		0x01
 
@@ -199,31 +211,77 @@ static int usbmisc_imx53_init(struct imx_usbmisc_data *data)
 	val |= MX53_USB_PLL_DIV_24_MHZ;
 	writel(val, usbmisc->base + MX53_USB_OTG_PHY_CTRL_1_OFFSET);
 
-	if (data->disable_oc) {
-		spin_lock_irqsave(&usbmisc->lock, flags);
-		switch (data->index) {
-		case 0:
+	spin_lock_irqsave(&usbmisc->lock, flags);
+
+	switch (data->index) {
+	case 0:
+		if (data->disable_oc) {
 			reg = usbmisc->base + MX53_USB_OTG_PHY_CTRL_0_OFFSET;
 			val = readl(reg) | MX53_BM_OVER_CUR_DIS_OTG;
-			break;
-		case 1:
+			writel(val, reg);
+		}
+		break;
+	case 1:
+		if (data->disable_oc) {
 			reg = usbmisc->base + MX53_USB_OTG_PHY_CTRL_0_OFFSET;
 			val = readl(reg) | MX53_BM_OVER_CUR_DIS_H1;
-			break;
-		case 2:
+			writel(val, reg);
+		}
+		break;
+	case 2:
+		if (data->ulpi) {
+			/* set USBH2 into ULPI-mode. */
+			reg = usbmisc->base + MX53_USB_CTRL_1_OFFSET;
+			val = readl(reg) | MX53_USB_CTRL_1_UH2_ULPI_EN;
+			/* select ULPI clock */
+			val &= ~MX53_USB_CTRL_1_H2_XCVR_CLK_SEL_MASK;
+			val |= MX53_USB_CTRL_1_H2_XCVR_CLK_SEL_ULPI;
+			writel(val, reg);
+			/* Set interrupt wake up enable */
+			reg = usbmisc->base + MX53_USB_UH2_CTRL_OFFSET;
+			val = readl(reg) | MX53_USB_UHx_CTRL_WAKE_UP_EN
+				| MX53_USB_UHx_CTRL_ULPI_INT_EN;
+			writel(val, reg);
+			/* Disable internal 60Mhz clock */
+			reg = usbmisc->base + MX53_USB_CLKONOFF_CTRL_OFFSET;
+			val = readl(reg) | MX53_USB_CLKONOFF_CTRL_H2_INT60CKOFF;
+			writel(val, reg);
+		}
+		if (data->disable_oc) {
 			reg = usbmisc->base + MX53_USB_UH2_CTRL_OFFSET;
 			val = readl(reg) | MX53_BM_OVER_CUR_DIS_UHx;
-			break;
-		case 3:
+			writel(val, reg);
+		}
+		break;
+	case 3:
+		if (data->ulpi) {
+			/* set USBH3 into ULPI-mode. */
+			reg = usbmisc->base + MX53_USB_CTRL_1_OFFSET;
+			val = readl(reg) | MX53_USB_CTRL_1_UH3_ULPI_EN;
+			/* select ULPI clock */
+			val &= ~MX53_USB_CTRL_1_H3_XCVR_CLK_SEL_MASK;
+			val |= MX53_USB_CTRL_1_H3_XCVR_CLK_SEL_ULPI;
+			writel(val, reg);
+			/* Set interrupt wake up enable */
 			reg = usbmisc->base + MX53_USB_UH3_CTRL_OFFSET;
-			val = readl(reg) | MX53_BM_OVER_CUR_DIS_UHx;
-			break;
+			val = readl(reg) | MX53_USB_UHx_CTRL_WAKE_UP_EN
+				| MX53_USB_UHx_CTRL_ULPI_INT_EN;
+			writel(val, reg);
+			/* Disable internal 60Mhz clock */
+			reg = usbmisc->base + MX53_USB_CLKONOFF_CTRL_OFFSET;
+			val = readl(reg) | MX53_USB_CLKONOFF_CTRL_H3_INT60CKOFF;
+			writel(val, reg);
 		}
-		if (reg && val)
+		if (data->disable_oc) {
+			reg = usbmisc->base + MX53_USB_UH3_CTRL_OFFSET;
+			val = readl(reg) | MX53_BM_OVER_CUR_DIS_UHx;
 			writel(val, reg);
-		spin_unlock_irqrestore(&usbmisc->lock, flags);
+		}
+		break;
 	}
 
+	spin_unlock_irqrestore(&usbmisc->lock, flags);
+
 	return 0;
 }
 
diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c
index fada988512a1..e35b1508d3eb 100644
--- a/drivers/usb/class/cdc-acm.c
+++ b/drivers/usb/class/cdc-acm.c
@@ -133,8 +133,8 @@ static int acm_ctrl_msg(struct acm *acm, int request, int value,
 		buf, len, 5000);
 
 	dev_dbg(&acm->control->dev,
-			"%s - rq 0x%02x, val %#x, len %#x, result %d\n",
-			__func__, request, value, len, retval);
+		"%s - rq 0x%02x, val %#x, len %#x, result %d\n",
+		__func__, request, value, len, retval);
 
 	usb_autopm_put_interface(acm->control);
 
@@ -158,6 +158,17 @@ static inline int acm_set_control(struct acm *acm, int control)
 #define acm_send_break(acm, ms) \
 	acm_ctrl_msg(acm, USB_CDC_REQ_SEND_BREAK, ms, NULL, 0)
 
+static void acm_kill_urbs(struct acm *acm)
+{
+	int i;
+
+	usb_kill_urb(acm->ctrlurb);
+	for (i = 0; i < ACM_NW; i++)
+		usb_kill_urb(acm->wb[i].urb);
+	for (i = 0; i < acm->rx_buflimit; i++)
+		usb_kill_urb(acm->read_urbs[i]);
+}
+
 /*
  * Write buffer management.
  * All of these assume proper locks taken by the caller.
@@ -291,13 +302,13 @@ static void acm_ctrl_irq(struct urb *urb)
 	case -ESHUTDOWN:
 		/* this urb is terminated, clean up */
 		dev_dbg(&acm->control->dev,
-				"%s - urb shutting down with status: %d\n",
-				__func__, status);
+			"%s - urb shutting down with status: %d\n",
+			__func__, status);
 		return;
 	default:
 		dev_dbg(&acm->control->dev,
-				"%s - nonzero urb status received: %d\n",
-				__func__, status);
+			"%s - nonzero urb status received: %d\n",
+			__func__, status);
 		goto exit;
 	}
 
@@ -306,16 +317,16 @@ static void acm_ctrl_irq(struct urb *urb)
 	data = (unsigned char *)(dr + 1);
 	switch (dr->bNotificationType) {
 	case USB_CDC_NOTIFY_NETWORK_CONNECTION:
-		dev_dbg(&acm->control->dev, "%s - network connection: %d\n",
-							__func__, dr->wValue);
+		dev_dbg(&acm->control->dev,
+			"%s - network connection: %d\n", __func__, dr->wValue);
 		break;
 
 	case USB_CDC_NOTIFY_SERIAL_STATE:
 		newctrl = get_unaligned_le16(data);
 
 		if (!acm->clocal && (acm->ctrlin & ~newctrl & ACM_CTRL_DCD)) {
-			dev_dbg(&acm->control->dev, "%s - calling hangup\n",
-					__func__);
+			dev_dbg(&acm->control->dev,
+				"%s - calling hangup\n", __func__);
 			tty_port_tty_hangup(&acm->port, false);
 		}
 
@@ -357,8 +368,8 @@ static void acm_ctrl_irq(struct urb *urb)
 exit:
 	retval = usb_submit_urb(urb, GFP_ATOMIC);
 	if (retval && retval != -EPERM)
-		dev_err(&acm->control->dev, "%s - usb_submit_urb failed: %d\n",
-							__func__, retval);
+		dev_err(&acm->control->dev,
+			"%s - usb_submit_urb failed: %d\n", __func__, retval);
 }
 
 static int acm_submit_read_urb(struct acm *acm, int index, gfp_t mem_flags)
@@ -372,8 +383,8 @@ static int acm_submit_read_urb(struct acm *acm, int index, gfp_t mem_flags)
 	if (res) {
 		if (res != -EPERM) {
 			dev_err(&acm->data->dev,
-					"urb %d failed submission with %d\n",
-					index, res);
+				"urb %d failed submission with %d\n",
+				index, res);
 		}
 		set_bit(index, &acm->read_urbs_free);
 		return res;
@@ -416,30 +427,43 @@ static void acm_read_bulk_callback(struct urb *urb)
 	int status = urb->status;
 
 	dev_vdbg(&acm->data->dev, "got urb %d, len %d, status %d\n",
-					rb->index, urb->actual_length,
-					status);
+		rb->index, urb->actual_length, status);
+
+	set_bit(rb->index, &acm->read_urbs_free);
 
 	if (!acm->dev) {
-		set_bit(rb->index, &acm->read_urbs_free);
 		dev_dbg(&acm->data->dev, "%s - disconnected\n", __func__);
 		return;
 	}
 
-	if (status) {
-		set_bit(rb->index, &acm->read_urbs_free);
-		if ((status != -ENOENT) || (urb->actual_length == 0))
-			return;
+	switch (status) {
+	case 0:
+		usb_mark_last_busy(acm->dev);
+		acm_process_read_urb(acm, urb);
+		break;
+	case -EPIPE:
+		set_bit(EVENT_RX_STALL, &acm->flags);
+		schedule_work(&acm->work);
+		return;
+	case -ENOENT:
+	case -ECONNRESET:
+	case -ESHUTDOWN:
+		dev_dbg(&acm->data->dev,
+			"%s - urb shutting down with status: %d\n",
+			__func__, status);
+		return;
+	default:
+		dev_dbg(&acm->data->dev,
+			"%s - nonzero urb status received: %d\n",
+			__func__, status);
+		break;
 	}
 
-	usb_mark_last_busy(acm->dev);
-
-	acm_process_read_urb(acm, urb);
 	/*
 	 * Unthrottle may run on another CPU which needs to see events
 	 * in the same order. Submission has an implict barrier
 	 */
 	smp_mb__before_atomic();
-	set_bit(rb->index, &acm->read_urbs_free);
 
 	/* throttle device if requested by tty */
 	spin_lock_irqsave(&acm->read_lock, flags);
@@ -469,14 +493,30 @@ static void acm_write_bulk(struct urb *urb)
 	spin_lock_irqsave(&acm->write_lock, flags);
 	acm_write_done(acm, wb);
 	spin_unlock_irqrestore(&acm->write_lock, flags);
+	set_bit(EVENT_TTY_WAKEUP, &acm->flags);
 	schedule_work(&acm->work);
 }
 
 static void acm_softint(struct work_struct *work)
 {
+	int i;
 	struct acm *acm = container_of(work, struct acm, work);
 
-	tty_port_tty_wakeup(&acm->port);
+	if (test_bit(EVENT_RX_STALL, &acm->flags)) {
+		if (!(usb_autopm_get_interface(acm->data))) {
+			for (i = 0; i < acm->rx_buflimit; i++)
+				usb_kill_urb(acm->read_urbs[i]);
+			usb_clear_halt(acm->dev, acm->in);
+			acm_submit_read_urbs(acm, GFP_KERNEL);
+			usb_autopm_put_interface(acm->data);
+		}
+		clear_bit(EVENT_RX_STALL, &acm->flags);
+	}
+
+	if (test_bit(EVENT_TTY_WAKEUP, &acm->flags)) {
+		tty_port_tty_wakeup(&acm->port);
+		clear_bit(EVENT_TTY_WAKEUP, &acm->flags);
+	}
 }
 
 /*
@@ -608,7 +648,6 @@ static void acm_port_shutdown(struct tty_port *port)
 	struct acm *acm = container_of(port, struct acm, port);
 	struct urb *urb;
 	struct acm_wb *wb;
-	int i;
 
 	/*
 	 * Need to grab write_lock to prevent race with resume, but no need to
@@ -630,11 +669,7 @@ static void acm_port_shutdown(struct tty_port *port)
 		usb_autopm_put_interface_async(acm->control);
 	}
 
-	usb_kill_urb(acm->ctrlurb);
-	for (i = 0; i < ACM_NW; i++)
-		usb_kill_urb(acm->wb[i].urb);
-	for (i = 0; i < acm->rx_buflimit; i++)
-		usb_kill_urb(acm->read_urbs[i]);
+	acm_kill_urbs(acm);
 }
 
 static void acm_tty_cleanup(struct tty_struct *tty)
@@ -837,8 +872,8 @@ static int acm_tty_break_ctl(struct tty_struct *tty, int state)
 
 	retval = acm_send_break(acm, state ? 0xffff : 0);
 	if (retval < 0)
-		dev_dbg(&acm->control->dev, "%s - send break failed\n",
-								__func__);
+		dev_dbg(&acm->control->dev,
+			"%s - send break failed\n", __func__);
 	return retval;
 }
 
@@ -877,9 +912,6 @@ static int get_serial_info(struct acm *acm, struct serial_struct __user *info)
 {
 	struct serial_struct tmp;
 
-	if (!info)
-		return -EINVAL;
-
 	memset(&tmp, 0, sizeof(tmp));
 	tmp.flags = ASYNC_LOW_LATENCY;
 	tmp.xmit_fifo_size = acm->writesize;
@@ -969,25 +1001,20 @@ static int wait_serial_change(struct acm *acm, unsigned long arg)
 	return rv;
 }
 
-static int get_serial_usage(struct acm *acm,
-			    struct serial_icounter_struct __user *count)
+static int acm_tty_get_icount(struct tty_struct *tty,
+					struct serial_icounter_struct *icount)
 {
-	struct serial_icounter_struct icount;
-	int rv = 0;
-
-	memset(&icount, 0, sizeof(icount));
-	icount.dsr = acm->iocount.dsr;
-	icount.rng = acm->iocount.rng;
-	icount.dcd = acm->iocount.dcd;
-	icount.frame = acm->iocount.frame;
-	icount.overrun = acm->iocount.overrun;
-	icount.parity = acm->iocount.parity;
-	icount.brk = acm->iocount.brk;
+	struct acm *acm = tty->driver_data;
 
-	if (copy_to_user(count, &icount, sizeof(icount)) > 0)
-		rv = -EFAULT;
+	icount->dsr = acm->iocount.dsr;
+	icount->rng = acm->iocount.rng;
+	icount->dcd = acm->iocount.dcd;
+	icount->frame = acm->iocount.frame;
+	icount->overrun = acm->iocount.overrun;
+	icount->parity = acm->iocount.parity;
+	icount->brk = acm->iocount.brk;
 
-	return rv;
+	return 0;
 }
 
 static int acm_tty_ioctl(struct tty_struct *tty,
@@ -1012,9 +1039,6 @@ static int acm_tty_ioctl(struct tty_struct *tty,
 		rv = wait_serial_change(acm, arg);
 		usb_autopm_put_interface(acm->control);
 		break;
-	case TIOCGICOUNT:
-		rv = get_serial_usage(acm, (struct serial_icounter_struct __user *) arg);
-		break;
 	}
 
 	return rv;
@@ -1088,19 +1112,17 @@ static void acm_write_buffers_free(struct acm *acm)
 {
 	int i;
 	struct acm_wb *wb;
-	struct usb_device *usb_dev = interface_to_usbdev(acm->control);
 
 	for (wb = &acm->wb[0], i = 0; i < ACM_NW; i++, wb++)
-		usb_free_coherent(usb_dev, acm->writesize, wb->buf, wb->dmah);
+		usb_free_coherent(acm->dev, acm->writesize, wb->buf, wb->dmah);
 }
 
 static void acm_read_buffers_free(struct acm *acm)
 {
-	struct usb_device *usb_dev = interface_to_usbdev(acm->control);
 	int i;
 
 	for (i = 0; i < acm->rx_buflimit; i++)
-		usb_free_coherent(usb_dev, acm->readsize,
+		usb_free_coherent(acm->dev, acm->readsize,
 			  acm->read_buffers[i].base, acm->read_buffers[i].dma);
 }
 
@@ -1345,9 +1367,16 @@ made_compressed_probe:
 	spin_lock_init(&acm->write_lock);
 	spin_lock_init(&acm->read_lock);
 	mutex_init(&acm->mutex);
-	acm->is_int_ep = usb_endpoint_xfer_int(epread);
-	if (acm->is_int_ep)
+	if (usb_endpoint_xfer_int(epread)) {
 		acm->bInterval = epread->bInterval;
+		acm->in = usb_rcvintpipe(usb_dev, epread->bEndpointAddress);
+	} else {
+		acm->in = usb_rcvbulkpipe(usb_dev, epread->bEndpointAddress);
+	}
+	if (usb_endpoint_xfer_int(epwrite))
+		acm->out = usb_sndintpipe(usb_dev, epwrite->bEndpointAddress);
+	else
+		acm->out = usb_sndbulkpipe(usb_dev, epwrite->bEndpointAddress);
 	tty_port_init(&acm->port);
 	acm->port.ops = &acm_port_ops;
 	init_usb_anchor(&acm->delayed);
@@ -1382,20 +1411,15 @@ made_compressed_probe:
 
 		urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
 		urb->transfer_dma = rb->dma;
-		if (acm->is_int_ep) {
-			usb_fill_int_urb(urb, acm->dev,
-					 usb_rcvintpipe(usb_dev, epread->bEndpointAddress),
-					 rb->base,
+		if (usb_endpoint_xfer_int(epread))
+			usb_fill_int_urb(urb, acm->dev, acm->in, rb->base,
 					 acm->readsize,
 					 acm_read_bulk_callback, rb,
 					 acm->bInterval);
-		} else {
-			usb_fill_bulk_urb(urb, acm->dev,
-					  usb_rcvbulkpipe(usb_dev, epread->bEndpointAddress),
-					  rb->base,
+		else
+			usb_fill_bulk_urb(urb, acm->dev, acm->in, rb->base,
 					  acm->readsize,
 					  acm_read_bulk_callback, rb);
-		}
 
 		acm->read_urbs[i] = urb;
 		__set_bit(i, &acm->read_urbs_free);
@@ -1408,12 +1432,10 @@ made_compressed_probe:
 			goto alloc_fail7;
 
 		if (usb_endpoint_xfer_int(epwrite))
-			usb_fill_int_urb(snd->urb, usb_dev,
-				usb_sndintpipe(usb_dev, epwrite->bEndpointAddress),
+			usb_fill_int_urb(snd->urb, usb_dev, acm->out,
 				NULL, acm->writesize, acm_write_bulk, snd, epwrite->bInterval);
 		else
-			usb_fill_bulk_urb(snd->urb, usb_dev,
-				usb_sndbulkpipe(usb_dev, epwrite->bEndpointAddress),
+			usb_fill_bulk_urb(snd->urb, usb_dev, acm->out,
 				NULL, acm->writesize, acm_write_bulk, snd);
 		snd->urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
 		if (quirks & SEND_ZERO_PACKET)
@@ -1485,8 +1507,8 @@ skip_countries:
 	}
 
 	if (quirks & CLEAR_HALT_CONDITIONS) {
-		usb_clear_halt(usb_dev, usb_rcvbulkpipe(usb_dev, epread->bEndpointAddress));
-		usb_clear_halt(usb_dev, usb_sndbulkpipe(usb_dev, epwrite->bEndpointAddress));
+		usb_clear_halt(usb_dev, acm->in);
+		usb_clear_halt(usb_dev, acm->out);
 	}
 
 	return 0;
@@ -1520,25 +1542,10 @@ alloc_fail:
 	return rv;
 }
 
-static void stop_data_traffic(struct acm *acm)
-{
-	int i;
-
-	usb_kill_urb(acm->ctrlurb);
-	for (i = 0; i < ACM_NW; i++)
-		usb_kill_urb(acm->wb[i].urb);
-	for (i = 0; i < acm->rx_buflimit; i++)
-		usb_kill_urb(acm->read_urbs[i]);
-
-	cancel_work_sync(&acm->work);
-}
-
 static void acm_disconnect(struct usb_interface *intf)
 {
 	struct acm *acm = usb_get_intfdata(intf);
-	struct usb_device *usb_dev = interface_to_usbdev(intf);
 	struct tty_struct *tty;
-	int i;
 
 	/* sibling interface is already cleaning up */
 	if (!acm)
@@ -1564,17 +1571,13 @@ static void acm_disconnect(struct usb_interface *intf)
 		tty_kref_put(tty);
 	}
 
-	stop_data_traffic(acm);
+	acm_kill_urbs(acm);
+	cancel_work_sync(&acm->work);
 
 	tty_unregister_device(acm_tty_driver, acm->minor);
 
-	usb_free_urb(acm->ctrlurb);
-	for (i = 0; i < ACM_NW; i++)
-		usb_free_urb(acm->wb[i].urb);
-	for (i = 0; i < acm->rx_buflimit; i++)
-		usb_free_urb(acm->read_urbs[i]);
 	acm_write_buffers_free(acm);
-	usb_free_coherent(usb_dev, acm->ctrlsize, acm->ctrl_buffer, acm->ctrl_dma);
+	usb_free_coherent(acm->dev, acm->ctrlsize, acm->ctrl_buffer, acm->ctrl_dma);
 	acm_read_buffers_free(acm);
 
 	if (!acm->combined_interfaces)
@@ -1603,7 +1606,8 @@ static int acm_suspend(struct usb_interface *intf, pm_message_t message)
 	if (cnt)
 		return 0;
 
-	stop_data_traffic(acm);
+	acm_kill_urbs(acm);
+	cancel_work_sync(&acm->work);
 
 	return 0;
 }
@@ -1657,6 +1661,15 @@ static int acm_reset_resume(struct usb_interface *intf)
 
 #endif /* CONFIG_PM */
 
+static int acm_pre_reset(struct usb_interface *intf)
+{
+	struct acm *acm = usb_get_intfdata(intf);
+
+	clear_bit(EVENT_RX_STALL, &acm->flags);
+
+	return 0;
+}
+
 #define NOKIA_PCSUITE_ACM_INFO(x) \
 		USB_DEVICE_AND_INTERFACE_INFO(0x0421, x, \
 		USB_CLASS_COMM, USB_CDC_SUBCLASS_ACM, \
@@ -1719,6 +1732,7 @@ static const struct usb_device_id acm_ids[] = {
 	{ USB_DEVICE(0x20df, 0x0001), /* Simtec Electronics Entropy Key */
 	.driver_info = QUIRK_CONTROL_LINE_STATE, },
 	{ USB_DEVICE(0x2184, 0x001c) },	/* GW Instek AFG-2225 */
+	{ USB_DEVICE(0x2184, 0x0036) },	/* GW Instek AFG-125 */
 	{ USB_DEVICE(0x22b8, 0x6425), /* Motorola MOTOMAGX phones */
 	},
 	/* Motorola H24 HSPA module: */
@@ -1898,6 +1912,7 @@ static struct usb_driver acm_driver = {
 	.resume =	acm_resume,
 	.reset_resume =	acm_reset_resume,
 #endif
+	.pre_reset =	acm_pre_reset,
 	.id_table =	acm_ids,
 #ifdef CONFIG_PM
 	.supports_autosuspend = 1,
@@ -1927,6 +1942,7 @@ static const struct tty_operations acm_ops = {
 	.set_termios =		acm_tty_set_termios,
 	.tiocmget =		acm_tty_tiocmget,
 	.tiocmset =		acm_tty_tiocmset,
+	.get_icount =		acm_tty_get_icount,
 };
 
 /*
diff --git a/drivers/usb/class/cdc-acm.h b/drivers/usb/class/cdc-acm.h
index 1f1eabfd8462..c980f11cdf56 100644
--- a/drivers/usb/class/cdc-acm.h
+++ b/drivers/usb/class/cdc-acm.h
@@ -83,6 +83,7 @@ struct acm {
 	struct usb_device *dev;				/* the corresponding usb device */
 	struct usb_interface *control;			/* control interface */
 	struct usb_interface *data;			/* data interface */
+	unsigned in, out;				/* i/o pipes */
 	struct tty_port port;			 	/* our tty port data */
 	struct urb *ctrlurb;				/* urbs */
 	u8 *ctrl_buffer;				/* buffers of urbs */
@@ -102,6 +103,9 @@ struct acm {
 	spinlock_t write_lock;
 	struct mutex mutex;
 	bool disconnected;
+	unsigned long flags;
+#		define EVENT_TTY_WAKEUP	0
+#		define EVENT_RX_STALL	1
 	struct usb_cdc_line_coding line;		/* bits, stop, parity */
 	struct work_struct work;			/* work queue entry for line discipline waking up */
 	unsigned int ctrlin;				/* input control lines (DCD, DSR, RI, break, overruns) */
@@ -116,7 +120,6 @@ struct acm {
 	unsigned int ctrl_caps;				/* control capabilities from the class specific header */
 	unsigned int susp_count;			/* number of suspended interfaces */
 	unsigned int combined_interfaces:1;		/* control and data collapsed */
-	unsigned int is_int_ep:1;			/* interrupt endpoints contrary to spec used */
 	unsigned int throttled:1;			/* actually throttled */
 	unsigned int throttle_req:1;			/* throttle requested */
 	u8 bInterval;
diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c
index a6c1fae7d52a..f03692ec5520 100644
--- a/drivers/usb/class/usbtmc.c
+++ b/drivers/usb/class/usbtmc.c
@@ -157,6 +157,7 @@ static int usbtmc_open(struct inode *inode, struct file *filp)
 	}
 
 	data = usb_get_intfdata(intf);
+	/* Protect reference to data from file structure until release */
 	kref_get(&data->kref);
 
 	/* Store pointer in file structure's private data field */
@@ -531,7 +532,7 @@ static int usbtmc488_ioctl_simple(struct usbtmc_device_data *data,
 }
 
 /*
- * Sends a REQUEST_DEV_DEP_MSG_IN message on the Bulk-IN endpoint.
+ * Sends a REQUEST_DEV_DEP_MSG_IN message on the Bulk-OUT endpoint.
  * @transfer_size: number of bytes to request from the device.
  *
  * See the USBTMC specification, Table 4.
@@ -1471,7 +1472,7 @@ static int usbtmc_probe(struct usb_interface *intf,
 		if (!data->iin_urb)
 			goto error_register;
 
-		/* will reference data in int urb */
+		/* Protect interrupt in endpoint data until iin_urb is freed */
 		kref_get(&data->kref);
 
 		/* allocate buffer for interrupt in */
diff --git a/drivers/usb/core/buffer.c b/drivers/usb/core/buffer.c
index 98e39f91723a..b9bf6e2eb6fe 100644
--- a/drivers/usb/core/buffer.c
+++ b/drivers/usb/core/buffer.c
@@ -3,6 +3,9 @@
  *
  * This implementation plugs in through generic "usb_bus" level methods,
  * and should work with all USB controllers, regardless of bus type.
+ *
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
  */
 
 #include <linux/module.h>
diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c
index a2d90aca779f..0aa9e7d697a5 100644
--- a/drivers/usb/core/config.c
+++ b/drivers/usb/core/config.c
@@ -1,3 +1,8 @@
+/*
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
+ */
+
 #include <linux/usb.h>
 #include <linux/usb/ch9.h>
 #include <linux/usb/hcd.h>
diff --git a/drivers/usb/core/devices.c b/drivers/usb/core/devices.c
index ef04b50e6bbb..f2987ddb1cde 100644
--- a/drivers/usb/core/devices.c
+++ b/drivers/usb/core/devices.c
@@ -182,14 +182,8 @@ static char *usb_dump_endpoint_descriptor(int speed, char *start, char *end,
 
 	dir = usb_endpoint_dir_in(desc) ? 'I' : 'O';
 
-	if (speed == USB_SPEED_HIGH) {
-		switch (usb_endpoint_maxp(desc) & (0x03 << 11)) {
-		case 1 << 11:
-			bandwidth = 2; break;
-		case 2 << 11:
-			bandwidth = 3; break;
-		}
-	}
+	if (speed == USB_SPEED_HIGH)
+		bandwidth = usb_endpoint_maxp_mult(desc);
 
 	/* this isn't checking for illegal values */
 	switch (usb_endpoint_type(desc)) {
@@ -233,7 +227,7 @@ static char *usb_dump_endpoint_descriptor(int speed, char *start, char *end,
 
 	start += sprintf(start, format_endpt, desc->bEndpointAddress, dir,
 			 desc->bmAttributes, type,
-			 (usb_endpoint_maxp(desc) & 0x07ff) *
+			 usb_endpoint_maxp(desc) *
 			 bandwidth,
 			 interval, unit);
 	return start;
diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c
index dadd1e8dfe09..cdee5130638b 100644
--- a/drivers/usb/core/driver.c
+++ b/drivers/usb/core/driver.c
@@ -15,6 +15,9 @@
  *		(usb_device_id matching changes by Adam J. Richter)
  *	(C) Copyright Greg Kroah-Hartman 2002-2003
  *
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
+ *
  * NOTE! This is not actually a driver at all, rather this is
  * just a collection of helper routines that implement the
  * matching, probing, releasing, suspending and resuming for
diff --git a/drivers/usb/core/endpoint.c b/drivers/usb/core/endpoint.c
index 101983b7e8d2..a60bc830a056 100644
--- a/drivers/usb/core/endpoint.c
+++ b/drivers/usb/core/endpoint.c
@@ -5,8 +5,10 @@
  * (C) Copyright 2002,2004 IBM Corp.
  * (C) Copyright 2006 Novell Inc.
  *
- * Endpoint sysfs stuff
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
  *
+ * Endpoint sysfs stuff
  */
 
 #include <linux/kernel.h>
@@ -50,8 +52,7 @@ static ssize_t wMaxPacketSize_show(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
 	struct ep_device *ep = to_ep_device(dev);
-	return sprintf(buf, "%04x\n",
-			usb_endpoint_maxp(ep->desc) & 0x07ff);
+	return sprintf(buf, "%04x\n", usb_endpoint_maxp(ep->desc));
 }
 static DEVICE_ATTR_RO(wMaxPacketSize);
 
diff --git a/drivers/usb/core/file.c b/drivers/usb/core/file.c
index 822ced9639aa..e26bd5e773ad 100644
--- a/drivers/usb/core/file.c
+++ b/drivers/usb/core/file.c
@@ -13,6 +13,8 @@
  *     (usb_device_id matching changes by Adam J. Richter)
  * (C) Copyright Greg Kroah-Hartman 2002-2003
  *
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
  */
 
 #include <linux/module.h>
diff --git a/drivers/usb/core/generic.c b/drivers/usb/core/generic.c
index 358ca8dd784f..bd3e0c5a6db2 100644
--- a/drivers/usb/core/generic.c
+++ b/drivers/usb/core/generic.c
@@ -15,6 +15,8 @@
  *		(usb_device_id matching changes by Adam J. Richter)
  *	(C) Copyright Greg Kroah-Hartman 2002-2003
  *
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
  */
 
 #include <linux/usb.h>
diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c
index cbb146736f57..143454ea385b 100644
--- a/drivers/usb/core/hub.c
+++ b/drivers/usb/core/hub.c
@@ -6,6 +6,8 @@
  * (C) Copyright 1999 Gregory P. Smith
  * (C) Copyright 2001 Brad Hards (bhards@bigpond.net.au)
  *
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
  */
 
 #include <linux/kernel.h>
@@ -101,6 +103,8 @@ EXPORT_SYMBOL_GPL(ehci_cf_port_reset_rwsem);
 
 static void hub_release(struct kref *kref);
 static int usb_reset_and_verify_device(struct usb_device *udev);
+static void hub_usb3_port_prepare_disable(struct usb_hub *hub,
+					  struct usb_port *port_dev);
 
 static inline char *portspeed(struct usb_hub *hub, int portstatus)
 {
@@ -899,82 +903,28 @@ static int hub_set_port_link_state(struct usb_hub *hub, int port1,
 }
 
 /*
- * If USB 3.0 ports are placed into the Disabled state, they will no longer
- * detect any device connects or disconnects.  This is generally not what the
- * USB core wants, since it expects a disabled port to produce a port status
- * change event when a new device connects.
- *
- * Instead, set the link state to Disabled, wait for the link to settle into
- * that state, clear any change bits, and then put the port into the RxDetect
- * state.
+ * USB-3 does not have a similar link state as USB-2 that will avoid negotiating
+ * a connection with a plugged-in cable but will signal the host when the cable
+ * is unplugged. Disable remote wake and set link state to U3 for USB-3 devices
  */
-static int hub_usb3_port_disable(struct usb_hub *hub, int port1)
-{
-	int ret;
-	int total_time;
-	u16 portchange, portstatus;
-
-	if (!hub_is_superspeed(hub->hdev))
-		return -EINVAL;
-
-	ret = hub_port_status(hub, port1, &portstatus, &portchange);
-	if (ret < 0)
-		return ret;
-
-	/*
-	 * USB controller Advanced Micro Devices, Inc. [AMD] FCH USB XHCI
-	 * Controller [1022:7814] will have spurious result making the following
-	 * usb 3.0 device hotplugging route to the 2.0 root hub and recognized
-	 * as high-speed device if we set the usb 3.0 port link state to
-	 * Disabled. Since it's already in USB_SS_PORT_LS_RX_DETECT state, we
-	 * check the state here to avoid the bug.
-	 */
-	if ((portstatus & USB_PORT_STAT_LINK_STATE) ==
-				USB_SS_PORT_LS_RX_DETECT) {
-		dev_dbg(&hub->ports[port1 - 1]->dev,
-			 "Not disabling port; link state is RxDetect\n");
-		return ret;
-	}
-
-	ret = hub_set_port_link_state(hub, port1, USB_SS_PORT_LS_SS_DISABLED);
-	if (ret)
-		return ret;
-
-	/* Wait for the link to enter the disabled state. */
-	for (total_time = 0; ; total_time += HUB_DEBOUNCE_STEP) {
-		ret = hub_port_status(hub, port1, &portstatus, &portchange);
-		if (ret < 0)
-			return ret;
-
-		if ((portstatus & USB_PORT_STAT_LINK_STATE) ==
-				USB_SS_PORT_LS_SS_DISABLED)
-			break;
-		if (total_time >= HUB_DEBOUNCE_TIMEOUT)
-			break;
-		msleep(HUB_DEBOUNCE_STEP);
-	}
-	if (total_time >= HUB_DEBOUNCE_TIMEOUT)
-		dev_warn(&hub->ports[port1 - 1]->dev,
-				"Could not disable after %d ms\n", total_time);
-
-	return hub_set_port_link_state(hub, port1, USB_SS_PORT_LS_RX_DETECT);
-}
-
 static int hub_port_disable(struct usb_hub *hub, int port1, int set_state)
 {
 	struct usb_port *port_dev = hub->ports[port1 - 1];
 	struct usb_device *hdev = hub->hdev;
 	int ret = 0;
 
-	if (port_dev->child && set_state)
-		usb_set_device_state(port_dev->child, USB_STATE_NOTATTACHED);
 	if (!hub->error) {
-		if (hub_is_superspeed(hub->hdev))
-			ret = hub_usb3_port_disable(hub, port1);
-		else
+		if (hub_is_superspeed(hub->hdev)) {
+			hub_usb3_port_prepare_disable(hub, port_dev);
+			ret = hub_set_port_link_state(hub, port_dev->portnum,
+						      USB_SS_PORT_LS_U3);
+		} else {
 			ret = usb_clear_port_feature(hdev, port1,
 					USB_PORT_FEAT_ENABLE);
+		}
 	}
+	if (port_dev->child && set_state)
+		usb_set_device_state(port_dev->child, USB_STATE_NOTATTACHED);
 	if (ret && ret != -ENODEV)
 		dev_err(&port_dev->dev, "cannot disable (err = %d)\n", ret);
 	return ret;
@@ -2731,8 +2681,15 @@ static int hub_port_wait_reset(struct usb_hub *hub, int port1,
 		if (ret < 0)
 			return ret;
 
-		/* The port state is unknown until the reset completes. */
-		if (!(portstatus & USB_PORT_STAT_RESET))
+		/*
+		 * The port state is unknown until the reset completes.
+		 *
+		 * On top of that, some chips may require additional time
+		 * to re-establish a connection after the reset is complete,
+		 * so also wait for the connection to be re-established.
+		 */
+		if (!(portstatus & USB_PORT_STAT_RESET) &&
+		    (portstatus & USB_PORT_STAT_CONNECTION))
 			break;
 
 		/* switch to the long delay after two short delay failures */
@@ -4140,6 +4097,26 @@ void usb_unlocked_enable_lpm(struct usb_device *udev)
 }
 EXPORT_SYMBOL_GPL(usb_unlocked_enable_lpm);
 
+/* usb3 devices use U3 for disabled, make sure remote wakeup is disabled */
+static void hub_usb3_port_prepare_disable(struct usb_hub *hub,
+					  struct usb_port *port_dev)
+{
+	struct usb_device *udev = port_dev->child;
+	int ret;
+
+	if (udev && udev->port_is_suspended && udev->do_remote_wakeup) {
+		ret = hub_set_port_link_state(hub, port_dev->portnum,
+					      USB_SS_PORT_LS_U0);
+		if (!ret) {
+			msleep(USB_RESUME_TIMEOUT);
+			ret = usb_disable_remote_wakeup(udev);
+		}
+		if (ret)
+			dev_warn(&udev->dev,
+				 "Port disable: can't disable remote wake\n");
+		udev->do_remote_wakeup = 0;
+	}
+}
 
 #else	/* CONFIG_PM */
 
@@ -4147,6 +4124,9 @@ EXPORT_SYMBOL_GPL(usb_unlocked_enable_lpm);
 #define hub_resume		NULL
 #define hub_reset_resume	NULL
 
+static inline void hub_usb3_port_prepare_disable(struct usb_hub *hub,
+						 struct usb_port *port_dev) { }
+
 int usb_disable_lpm(struct usb_device *udev)
 {
 	return 0;
diff --git a/drivers/usb/core/ledtrig-usbport.c b/drivers/usb/core/ledtrig-usbport.c
index 3ed5162677ad..1713248ab15a 100644
--- a/drivers/usb/core/ledtrig-usbport.c
+++ b/drivers/usb/core/ledtrig-usbport.c
@@ -74,8 +74,7 @@ static void usbport_trig_update_count(struct usbport_trig_data *usbport_data)
 
 	usbport_data->count = 0;
 	usb_for_each_dev(usbport_data, usbport_trig_usb_dev_check);
-	led_cdev->brightness_set(led_cdev,
-				 usbport_data->count ? LED_FULL : LED_OFF);
+	led_set_brightness(led_cdev, usbport_data->count ? LED_FULL : LED_OFF);
 }
 
 /***************************************
@@ -228,12 +227,12 @@ static int usbport_trig_notify(struct notifier_block *nb, unsigned long action,
 	case USB_DEVICE_ADD:
 		usbport_trig_add_usb_dev_ports(usb_dev, usbport_data);
 		if (observed && usbport_data->count++ == 0)
-			led_cdev->brightness_set(led_cdev, LED_FULL);
+			led_set_brightness(led_cdev, LED_FULL);
 		return NOTIFY_OK;
 	case USB_DEVICE_REMOVE:
 		usbport_trig_remove_usb_dev_ports(usbport_data, usb_dev);
 		if (observed && --usbport_data->count == 0)
-			led_cdev->brightness_set(led_cdev, LED_OFF);
+			led_set_brightness(led_cdev, LED_OFF);
 		return NOTIFY_OK;
 	}
 
diff --git a/drivers/usb/core/message.c b/drivers/usb/core/message.c
index 3a4707746157..dea55914d641 100644
--- a/drivers/usb/core/message.c
+++ b/drivers/usb/core/message.c
@@ -1,5 +1,8 @@
 /*
  * message.c - synchronous message handling
+ *
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
  */
 
 #include <linux/pci.h>	/* for scatterlist macros */
diff --git a/drivers/usb/core/notify.c b/drivers/usb/core/notify.c
index 7728c91dfa2e..b12a463a3e22 100644
--- a/drivers/usb/core/notify.c
+++ b/drivers/usb/core/notify.c
@@ -6,6 +6,8 @@
  * notifier functions originally based on those in kernel/sys.c
  * but fixed up to not be so broken.
  *
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
  */
 
 
diff --git a/drivers/usb/core/sysfs.c b/drivers/usb/core/sysfs.c
index c953a0f1c695..dfc68ed24db1 100644
--- a/drivers/usb/core/sysfs.c
+++ b/drivers/usb/core/sysfs.c
@@ -7,6 +7,8 @@
  *
  * All of the sysfs file attributes for usb devices and interfaces.
  *
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
  */
 
 
@@ -14,6 +16,7 @@
 #include <linux/string.h>
 #include <linux/usb.h>
 #include <linux/usb/quirks.h>
+#include <linux/of.h>
 #include "usb.h"
 
 /* Active configuration fields */
@@ -104,6 +107,17 @@ static ssize_t bConfigurationValue_store(struct device *dev,
 static DEVICE_ATTR_IGNORE_LOCKDEP(bConfigurationValue, S_IRUGO | S_IWUSR,
 		bConfigurationValue_show, bConfigurationValue_store);
 
+#ifdef CONFIG_OF
+static ssize_t devspec_show(struct device *dev, struct device_attribute *attr,
+			    char *buf)
+{
+	struct device_node *of_node = dev->of_node;
+
+	return sprintf(buf, "%s\n", of_node_full_name(of_node));
+}
+static DEVICE_ATTR_RO(devspec);
+#endif
+
 /* String fields */
 #define usb_string_attr(name)						\
 static ssize_t  name##_show(struct device *dev,				\
@@ -786,6 +800,9 @@ static struct attribute *dev_attrs[] = {
 	&dev_attr_remove.attr,
 	&dev_attr_removable.attr,
 	&dev_attr_ltm_capable.attr,
+#ifdef CONFIG_OF
+	&dev_attr_devspec.attr,
+#endif
 	NULL,
 };
 static struct attribute_group dev_attr_grp = {
diff --git a/drivers/usb/core/urb.c b/drivers/usb/core/urb.c
index a9039696476e..d75cb8c0f7df 100644
--- a/drivers/usb/core/urb.c
+++ b/drivers/usb/core/urb.c
@@ -1,3 +1,8 @@
+/*
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
+ */
+
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/bitops.h>
@@ -407,11 +412,8 @@ int usb_submit_urb(struct urb *urb, gfp_t mem_flags)
 		}
 
 		/* "high bandwidth" mode, 1-3 packets/uframe? */
-		if (dev->speed == USB_SPEED_HIGH) {
-			int	mult = 1 + ((max >> 11) & 0x03);
-			max &= 0x07ff;
-			max *= mult;
-		}
+		if (dev->speed == USB_SPEED_HIGH)
+			max *= usb_endpoint_maxp_mult(&ep->desc);
 
 		if (urb->number_of_packets <= 0)
 			return -EINVAL;
diff --git a/drivers/usb/core/usb.c b/drivers/usb/core/usb.c
index 592151461017..a2ccc69fb45c 100644
--- a/drivers/usb/core/usb.c
+++ b/drivers/usb/core/usb.c
@@ -12,6 +12,9 @@
  *     (usb_device_id matching changes by Adam J. Richter)
  * (C) Copyright Greg Kroah-Hartman 2002-2003
  *
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
+ *
  * NOTE! This is not actually a driver at all, rather this is
  * just a collection of helper routines that implement the
  * generic USB things that the real drivers can use..
diff --git a/drivers/usb/core/usb.h b/drivers/usb/core/usb.h
index 53318126ed91..dc6949248823 100644
--- a/drivers/usb/core/usb.h
+++ b/drivers/usb/core/usb.h
@@ -1,3 +1,8 @@
+/*
+ * Released under the GPLv2 only.
+ * SPDX-License-Identifier: GPL-2.0
+ */
+
 #include <linux/pm.h>
 #include <linux/acpi.h>
 
diff --git a/drivers/usb/dwc2/Makefile b/drivers/usb/dwc2/Makefile
index 50fdaace1e73..b9237e1e45d0 100644
--- a/drivers/usb/dwc2/Makefile
+++ b/drivers/usb/dwc2/Makefile
@@ -3,6 +3,7 @@ ccflags-$(CONFIG_USB_DWC2_VERBOSE)	+= -DVERBOSE_DEBUG
 
 obj-$(CONFIG_USB_DWC2)			+= dwc2.o
 dwc2-y					:= core.o core_intr.o platform.o
+dwc2-y					+= params.o
 
 ifneq ($(filter y,$(CONFIG_USB_DWC2_HOST) $(CONFIG_USB_DWC2_DUAL_ROLE)),)
 	dwc2-y				+= hcd.o hcd_intr.o
diff --git a/drivers/usb/dwc2/core.c b/drivers/usb/dwc2/core.c
index 4c0fa0b17353..11d8ae9aead1 100644
--- a/drivers/usb/dwc2/core.c
+++ b/drivers/usb/dwc2/core.c
@@ -135,7 +135,7 @@ int dwc2_exit_hibernation(struct dwc2_hsotg *hsotg, bool restore)
 	u32 pcgcctl;
 	int ret = 0;
 
-	if (!hsotg->core_params->hibernation)
+	if (!hsotg->params.hibernation)
 		return -ENOTSUPP;
 
 	pcgcctl = dwc2_readl(hsotg->regs + PCGCTL);
@@ -188,7 +188,7 @@ int dwc2_enter_hibernation(struct dwc2_hsotg *hsotg)
 	u32 pcgcctl;
 	int ret = 0;
 
-	if (!hsotg->core_params->hibernation)
+	if (!hsotg->params.hibernation)
 		return -ENOTSUPP;
 
 	/* Backup all registers */
@@ -445,7 +445,7 @@ static bool dwc2_force_mode(struct dwc2_hsotg *hsotg, bool host)
  * the force mode. We only need to call this once during probe if
  * dr_mode == OTG.
  */
-static void dwc2_clear_force_mode(struct dwc2_hsotg *hsotg)
+void dwc2_clear_force_mode(struct dwc2_hsotg *hsotg)
 {
 	u32 gusbcfg;
 
@@ -541,7 +541,7 @@ void dwc2_dump_host_registers(struct dwc2_hsotg *hsotg)
 	addr = hsotg->regs + HAINTMSK;
 	dev_dbg(hsotg->dev, "HAINTMSK	 @0x%08lX : 0x%08X\n",
 		(unsigned long)addr, dwc2_readl(addr));
-	if (hsotg->core_params->dma_desc_enable > 0) {
+	if (hsotg->params.dma_desc_enable > 0) {
 		addr = hsotg->regs + HFLBADDR;
 		dev_dbg(hsotg->dev, "HFLBADDR @0x%08lX : 0x%08X\n",
 			(unsigned long)addr, dwc2_readl(addr));
@@ -551,7 +551,7 @@ void dwc2_dump_host_registers(struct dwc2_hsotg *hsotg)
 	dev_dbg(hsotg->dev, "HPRT0	 @0x%08lX : 0x%08X\n",
 		(unsigned long)addr, dwc2_readl(addr));
 
-	for (i = 0; i < hsotg->core_params->host_channels; i++) {
+	for (i = 0; i < hsotg->params.host_channels; i++) {
 		dev_dbg(hsotg->dev, "Host Channel %d Specific Registers\n", i);
 		addr = hsotg->regs + HCCHAR(i);
 		dev_dbg(hsotg->dev, "HCCHAR	 @0x%08lX : 0x%08X\n",
@@ -571,7 +571,7 @@ void dwc2_dump_host_registers(struct dwc2_hsotg *hsotg)
 		addr = hsotg->regs + HCDMA(i);
 		dev_dbg(hsotg->dev, "HCDMA	 @0x%08lX : 0x%08X\n",
 			(unsigned long)addr, dwc2_readl(addr));
-		if (hsotg->core_params->dma_desc_enable > 0) {
+		if (hsotg->params.dma_desc_enable > 0) {
 			addr = hsotg->regs + HCDMAB(i);
 			dev_dbg(hsotg->dev, "HCDMAB	 @0x%08lX : 0x%08X\n",
 				(unsigned long)addr, dwc2_readl(addr));
@@ -735,704 +735,13 @@ void dwc2_flush_rx_fifo(struct dwc2_hsotg *hsotg)
 	udelay(1);
 }
 
-#define DWC2_OUT_OF_BOUNDS(a, b, c)	((a) < (b) || (a) > (c))
-
-/* Parameter access functions */
-void dwc2_set_param_otg_cap(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	switch (val) {
-	case DWC2_CAP_PARAM_HNP_SRP_CAPABLE:
-		if (hsotg->hw_params.op_mode != GHWCFG2_OP_MODE_HNP_SRP_CAPABLE)
-			valid = 0;
-		break;
-	case DWC2_CAP_PARAM_SRP_ONLY_CAPABLE:
-		switch (hsotg->hw_params.op_mode) {
-		case GHWCFG2_OP_MODE_HNP_SRP_CAPABLE:
-		case GHWCFG2_OP_MODE_SRP_ONLY_CAPABLE:
-		case GHWCFG2_OP_MODE_SRP_CAPABLE_DEVICE:
-		case GHWCFG2_OP_MODE_SRP_CAPABLE_HOST:
-			break;
-		default:
-			valid = 0;
-			break;
-		}
-		break;
-	case DWC2_CAP_PARAM_NO_HNP_SRP_CAPABLE:
-		/* always valid */
-		break;
-	default:
-		valid = 0;
-		break;
-	}
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for otg_cap parameter. Check HW configuration.\n",
-				val);
-		switch (hsotg->hw_params.op_mode) {
-		case GHWCFG2_OP_MODE_HNP_SRP_CAPABLE:
-			val = DWC2_CAP_PARAM_HNP_SRP_CAPABLE;
-			break;
-		case GHWCFG2_OP_MODE_SRP_ONLY_CAPABLE:
-		case GHWCFG2_OP_MODE_SRP_CAPABLE_DEVICE:
-		case GHWCFG2_OP_MODE_SRP_CAPABLE_HOST:
-			val = DWC2_CAP_PARAM_SRP_ONLY_CAPABLE;
-			break;
-		default:
-			val = DWC2_CAP_PARAM_NO_HNP_SRP_CAPABLE;
-			break;
-		}
-		dev_dbg(hsotg->dev, "Setting otg_cap to %d\n", val);
-	}
-
-	hsotg->core_params->otg_cap = val;
-}
-
-void dwc2_set_param_dma_enable(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (val > 0 && hsotg->hw_params.arch == GHWCFG2_SLAVE_ONLY_ARCH)
-		valid = 0;
-	if (val < 0)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for dma_enable parameter. Check HW configuration.\n",
-				val);
-		val = hsotg->hw_params.arch != GHWCFG2_SLAVE_ONLY_ARCH;
-		dev_dbg(hsotg->dev, "Setting dma_enable to %d\n", val);
-	}
-
-	hsotg->core_params->dma_enable = val;
-}
-
-void dwc2_set_param_dma_desc_enable(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (val > 0 && (hsotg->core_params->dma_enable <= 0 ||
-			!hsotg->hw_params.dma_desc_enable))
-		valid = 0;
-	if (val < 0)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for dma_desc_enable parameter. Check HW configuration.\n",
-				val);
-		val = (hsotg->core_params->dma_enable > 0 &&
-			hsotg->hw_params.dma_desc_enable);
-		dev_dbg(hsotg->dev, "Setting dma_desc_enable to %d\n", val);
-	}
-
-	hsotg->core_params->dma_desc_enable = val;
-}
-
-void dwc2_set_param_dma_desc_fs_enable(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (val > 0 && (hsotg->core_params->dma_enable <= 0 ||
-			!hsotg->hw_params.dma_desc_enable))
-		valid = 0;
-	if (val < 0)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for dma_desc_fs_enable parameter. Check HW configuration.\n",
-				val);
-		val = (hsotg->core_params->dma_enable > 0 &&
-			hsotg->hw_params.dma_desc_enable);
-	}
-
-	hsotg->core_params->dma_desc_fs_enable = val;
-	dev_dbg(hsotg->dev, "Setting dma_desc_fs_enable to %d\n", val);
-}
-
-void dwc2_set_param_host_support_fs_ls_low_power(struct dwc2_hsotg *hsotg,
-						 int val)
-{
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev,
-				"Wrong value for host_support_fs_low_power\n");
-			dev_err(hsotg->dev,
-				"host_support_fs_low_power must be 0 or 1\n");
-		}
-		val = 0;
-		dev_dbg(hsotg->dev,
-			"Setting host_support_fs_low_power to %d\n", val);
-	}
-
-	hsotg->core_params->host_support_fs_ls_low_power = val;
-}
-
-void dwc2_set_param_enable_dynamic_fifo(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (val > 0 && !hsotg->hw_params.enable_dynamic_fifo)
-		valid = 0;
-	if (val < 0)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for enable_dynamic_fifo parameter. Check HW configuration.\n",
-				val);
-		val = hsotg->hw_params.enable_dynamic_fifo;
-		dev_dbg(hsotg->dev, "Setting enable_dynamic_fifo to %d\n", val);
-	}
-
-	hsotg->core_params->enable_dynamic_fifo = val;
-}
-
-void dwc2_set_param_host_rx_fifo_size(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (val < 16 || val > hsotg->hw_params.host_rx_fifo_size)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for host_rx_fifo_size. Check HW configuration.\n",
-				val);
-		val = hsotg->hw_params.host_rx_fifo_size;
-		dev_dbg(hsotg->dev, "Setting host_rx_fifo_size to %d\n", val);
-	}
-
-	hsotg->core_params->host_rx_fifo_size = val;
-}
-
-void dwc2_set_param_host_nperio_tx_fifo_size(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (val < 16 || val > hsotg->hw_params.host_nperio_tx_fifo_size)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for host_nperio_tx_fifo_size. Check HW configuration.\n",
-				val);
-		val = hsotg->hw_params.host_nperio_tx_fifo_size;
-		dev_dbg(hsotg->dev, "Setting host_nperio_tx_fifo_size to %d\n",
-			val);
-	}
-
-	hsotg->core_params->host_nperio_tx_fifo_size = val;
-}
-
-void dwc2_set_param_host_perio_tx_fifo_size(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (val < 16 || val > hsotg->hw_params.host_perio_tx_fifo_size)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for host_perio_tx_fifo_size. Check HW configuration.\n",
-				val);
-		val = hsotg->hw_params.host_perio_tx_fifo_size;
-		dev_dbg(hsotg->dev, "Setting host_perio_tx_fifo_size to %d\n",
-			val);
-	}
-
-	hsotg->core_params->host_perio_tx_fifo_size = val;
-}
-
-void dwc2_set_param_max_transfer_size(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (val < 2047 || val > hsotg->hw_params.max_transfer_size)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for max_transfer_size. Check HW configuration.\n",
-				val);
-		val = hsotg->hw_params.max_transfer_size;
-		dev_dbg(hsotg->dev, "Setting max_transfer_size to %d\n", val);
-	}
-
-	hsotg->core_params->max_transfer_size = val;
-}
-
-void dwc2_set_param_max_packet_count(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (val < 15 || val > hsotg->hw_params.max_packet_count)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for max_packet_count. Check HW configuration.\n",
-				val);
-		val = hsotg->hw_params.max_packet_count;
-		dev_dbg(hsotg->dev, "Setting max_packet_count to %d\n", val);
-	}
-
-	hsotg->core_params->max_packet_count = val;
-}
-
-void dwc2_set_param_host_channels(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (val < 1 || val > hsotg->hw_params.host_channels)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for host_channels. Check HW configuration.\n",
-				val);
-		val = hsotg->hw_params.host_channels;
-		dev_dbg(hsotg->dev, "Setting host_channels to %d\n", val);
-	}
-
-	hsotg->core_params->host_channels = val;
-}
-
-void dwc2_set_param_phy_type(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 0;
-	u32 hs_phy_type, fs_phy_type;
-
-	if (DWC2_OUT_OF_BOUNDS(val, DWC2_PHY_TYPE_PARAM_FS,
-			       DWC2_PHY_TYPE_PARAM_ULPI)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev, "Wrong value for phy_type\n");
-			dev_err(hsotg->dev, "phy_type must be 0, 1 or 2\n");
-		}
-
-		valid = 0;
-	}
-
-	hs_phy_type = hsotg->hw_params.hs_phy_type;
-	fs_phy_type = hsotg->hw_params.fs_phy_type;
-	if (val == DWC2_PHY_TYPE_PARAM_UTMI &&
-	    (hs_phy_type == GHWCFG2_HS_PHY_TYPE_UTMI ||
-	     hs_phy_type == GHWCFG2_HS_PHY_TYPE_UTMI_ULPI))
-		valid = 1;
-	else if (val == DWC2_PHY_TYPE_PARAM_ULPI &&
-		 (hs_phy_type == GHWCFG2_HS_PHY_TYPE_ULPI ||
-		  hs_phy_type == GHWCFG2_HS_PHY_TYPE_UTMI_ULPI))
-		valid = 1;
-	else if (val == DWC2_PHY_TYPE_PARAM_FS &&
-		 fs_phy_type == GHWCFG2_FS_PHY_TYPE_DEDICATED)
-		valid = 1;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for phy_type. Check HW configuration.\n",
-				val);
-		val = DWC2_PHY_TYPE_PARAM_FS;
-		if (hs_phy_type != GHWCFG2_HS_PHY_TYPE_NOT_SUPPORTED) {
-			if (hs_phy_type == GHWCFG2_HS_PHY_TYPE_UTMI ||
-			    hs_phy_type == GHWCFG2_HS_PHY_TYPE_UTMI_ULPI)
-				val = DWC2_PHY_TYPE_PARAM_UTMI;
-			else
-				val = DWC2_PHY_TYPE_PARAM_ULPI;
-		}
-		dev_dbg(hsotg->dev, "Setting phy_type to %d\n", val);
-	}
-
-	hsotg->core_params->phy_type = val;
-}
-
-static int dwc2_get_param_phy_type(struct dwc2_hsotg *hsotg)
-{
-	return hsotg->core_params->phy_type;
-}
-
-void dwc2_set_param_speed(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev, "Wrong value for speed parameter\n");
-			dev_err(hsotg->dev, "max_speed parameter must be 0 or 1\n");
-		}
-		valid = 0;
-	}
-
-	if (val == DWC2_SPEED_PARAM_HIGH &&
-	    dwc2_get_param_phy_type(hsotg) == DWC2_PHY_TYPE_PARAM_FS)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for speed parameter. Check HW configuration.\n",
-				val);
-		val = dwc2_get_param_phy_type(hsotg) == DWC2_PHY_TYPE_PARAM_FS ?
-				DWC2_SPEED_PARAM_FULL : DWC2_SPEED_PARAM_HIGH;
-		dev_dbg(hsotg->dev, "Setting speed to %d\n", val);
-	}
-
-	hsotg->core_params->speed = val;
-}
-
-void dwc2_set_param_host_ls_low_power_phy_clk(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (DWC2_OUT_OF_BOUNDS(val, DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_48MHZ,
-			       DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_6MHZ)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev,
-				"Wrong value for host_ls_low_power_phy_clk parameter\n");
-			dev_err(hsotg->dev,
-				"host_ls_low_power_phy_clk must be 0 or 1\n");
-		}
-		valid = 0;
-	}
-
-	if (val == DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_48MHZ &&
-	    dwc2_get_param_phy_type(hsotg) == DWC2_PHY_TYPE_PARAM_FS)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for host_ls_low_power_phy_clk. Check HW configuration.\n",
-				val);
-		val = dwc2_get_param_phy_type(hsotg) == DWC2_PHY_TYPE_PARAM_FS
-			? DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_6MHZ
-			: DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_48MHZ;
-		dev_dbg(hsotg->dev, "Setting host_ls_low_power_phy_clk to %d\n",
-			val);
-	}
-
-	hsotg->core_params->host_ls_low_power_phy_clk = val;
-}
-
-void dwc2_set_param_phy_ulpi_ddr(struct dwc2_hsotg *hsotg, int val)
-{
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev, "Wrong value for phy_ulpi_ddr\n");
-			dev_err(hsotg->dev, "phy_upli_ddr must be 0 or 1\n");
-		}
-		val = 0;
-		dev_dbg(hsotg->dev, "Setting phy_upli_ddr to %d\n", val);
-	}
-
-	hsotg->core_params->phy_ulpi_ddr = val;
-}
-
-void dwc2_set_param_phy_ulpi_ext_vbus(struct dwc2_hsotg *hsotg, int val)
-{
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev,
-				"Wrong value for phy_ulpi_ext_vbus\n");
-			dev_err(hsotg->dev,
-				"phy_ulpi_ext_vbus must be 0 or 1\n");
-		}
-		val = 0;
-		dev_dbg(hsotg->dev, "Setting phy_ulpi_ext_vbus to %d\n", val);
-	}
-
-	hsotg->core_params->phy_ulpi_ext_vbus = val;
-}
-
-void dwc2_set_param_phy_utmi_width(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 0;
-
-	switch (hsotg->hw_params.utmi_phy_data_width) {
-	case GHWCFG4_UTMI_PHY_DATA_WIDTH_8:
-		valid = (val == 8);
-		break;
-	case GHWCFG4_UTMI_PHY_DATA_WIDTH_16:
-		valid = (val == 16);
-		break;
-	case GHWCFG4_UTMI_PHY_DATA_WIDTH_8_OR_16:
-		valid = (val == 8 || val == 16);
-		break;
-	}
-
-	if (!valid) {
-		if (val >= 0) {
-			dev_err(hsotg->dev,
-				"%d invalid for phy_utmi_width. Check HW configuration.\n",
-				val);
-		}
-		val = (hsotg->hw_params.utmi_phy_data_width ==
-		       GHWCFG4_UTMI_PHY_DATA_WIDTH_8) ? 8 : 16;
-		dev_dbg(hsotg->dev, "Setting phy_utmi_width to %d\n", val);
-	}
-
-	hsotg->core_params->phy_utmi_width = val;
-}
-
-void dwc2_set_param_ulpi_fs_ls(struct dwc2_hsotg *hsotg, int val)
-{
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev, "Wrong value for ulpi_fs_ls\n");
-			dev_err(hsotg->dev, "ulpi_fs_ls must be 0 or 1\n");
-		}
-		val = 0;
-		dev_dbg(hsotg->dev, "Setting ulpi_fs_ls to %d\n", val);
-	}
-
-	hsotg->core_params->ulpi_fs_ls = val;
-}
-
-void dwc2_set_param_ts_dline(struct dwc2_hsotg *hsotg, int val)
-{
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev, "Wrong value for ts_dline\n");
-			dev_err(hsotg->dev, "ts_dline must be 0 or 1\n");
-		}
-		val = 0;
-		dev_dbg(hsotg->dev, "Setting ts_dline to %d\n", val);
-	}
-
-	hsotg->core_params->ts_dline = val;
-}
-
-void dwc2_set_param_i2c_enable(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev, "Wrong value for i2c_enable\n");
-			dev_err(hsotg->dev, "i2c_enable must be 0 or 1\n");
-		}
-
-		valid = 0;
-	}
-
-	if (val == 1 && !(hsotg->hw_params.i2c_enable))
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for i2c_enable. Check HW configuration.\n",
-				val);
-		val = hsotg->hw_params.i2c_enable;
-		dev_dbg(hsotg->dev, "Setting i2c_enable to %d\n", val);
-	}
-
-	hsotg->core_params->i2c_enable = val;
-}
-
-void dwc2_set_param_en_multiple_tx_fifo(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev,
-				"Wrong value for en_multiple_tx_fifo,\n");
-			dev_err(hsotg->dev,
-				"en_multiple_tx_fifo must be 0 or 1\n");
-		}
-		valid = 0;
-	}
-
-	if (val == 1 && !hsotg->hw_params.en_multiple_tx_fifo)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for parameter en_multiple_tx_fifo. Check HW configuration.\n",
-				val);
-		val = hsotg->hw_params.en_multiple_tx_fifo;
-		dev_dbg(hsotg->dev, "Setting en_multiple_tx_fifo to %d\n", val);
-	}
-
-	hsotg->core_params->en_multiple_tx_fifo = val;
-}
-
-void dwc2_set_param_reload_ctl(struct dwc2_hsotg *hsotg, int val)
-{
-	int valid = 1;
-
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev,
-				"'%d' invalid for parameter reload_ctl\n", val);
-			dev_err(hsotg->dev, "reload_ctl must be 0 or 1\n");
-		}
-		valid = 0;
-	}
-
-	if (val == 1 && hsotg->hw_params.snpsid < DWC2_CORE_REV_2_92a)
-		valid = 0;
-
-	if (!valid) {
-		if (val >= 0)
-			dev_err(hsotg->dev,
-				"%d invalid for parameter reload_ctl. Check HW configuration.\n",
-				val);
-		val = hsotg->hw_params.snpsid >= DWC2_CORE_REV_2_92a;
-		dev_dbg(hsotg->dev, "Setting reload_ctl to %d\n", val);
-	}
-
-	hsotg->core_params->reload_ctl = val;
-}
-
-void dwc2_set_param_ahbcfg(struct dwc2_hsotg *hsotg, int val)
-{
-	if (val != -1)
-		hsotg->core_params->ahbcfg = val;
-	else
-		hsotg->core_params->ahbcfg = GAHBCFG_HBSTLEN_INCR4 <<
-						GAHBCFG_HBSTLEN_SHIFT;
-}
-
-void dwc2_set_param_otg_ver(struct dwc2_hsotg *hsotg, int val)
-{
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev,
-				"'%d' invalid for parameter otg_ver\n", val);
-			dev_err(hsotg->dev,
-				"otg_ver must be 0 (for OTG 1.3 support) or 1 (for OTG 2.0 support)\n");
-		}
-		val = 0;
-		dev_dbg(hsotg->dev, "Setting otg_ver to %d\n", val);
-	}
-
-	hsotg->core_params->otg_ver = val;
-}
-
-static void dwc2_set_param_uframe_sched(struct dwc2_hsotg *hsotg, int val)
-{
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev,
-				"'%d' invalid for parameter uframe_sched\n",
-				val);
-			dev_err(hsotg->dev, "uframe_sched must be 0 or 1\n");
-		}
-		val = 1;
-		dev_dbg(hsotg->dev, "Setting uframe_sched to %d\n", val);
-	}
-
-	hsotg->core_params->uframe_sched = val;
-}
-
-static void dwc2_set_param_external_id_pin_ctl(struct dwc2_hsotg *hsotg,
-		int val)
-{
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev,
-				"'%d' invalid for parameter external_id_pin_ctl\n",
-				val);
-			dev_err(hsotg->dev, "external_id_pin_ctl must be 0 or 1\n");
-		}
-		val = 0;
-		dev_dbg(hsotg->dev, "Setting external_id_pin_ctl to %d\n", val);
-	}
-
-	hsotg->core_params->external_id_pin_ctl = val;
-}
-
-static void dwc2_set_param_hibernation(struct dwc2_hsotg *hsotg,
-		int val)
-{
-	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
-		if (val >= 0) {
-			dev_err(hsotg->dev,
-				"'%d' invalid for parameter hibernation\n",
-				val);
-			dev_err(hsotg->dev, "hibernation must be 0 or 1\n");
-		}
-		val = 0;
-		dev_dbg(hsotg->dev, "Setting hibernation to %d\n", val);
-	}
-
-	hsotg->core_params->hibernation = val;
-}
-
-/*
- * This function is called during module intialization to pass module parameters
- * for the DWC_otg core.
- */
-void dwc2_set_parameters(struct dwc2_hsotg *hsotg,
-			 const struct dwc2_core_params *params)
-{
-	dev_dbg(hsotg->dev, "%s()\n", __func__);
-
-	dwc2_set_param_otg_cap(hsotg, params->otg_cap);
-	dwc2_set_param_dma_enable(hsotg, params->dma_enable);
-	dwc2_set_param_dma_desc_enable(hsotg, params->dma_desc_enable);
-	dwc2_set_param_dma_desc_fs_enable(hsotg, params->dma_desc_fs_enable);
-	dwc2_set_param_host_support_fs_ls_low_power(hsotg,
-			params->host_support_fs_ls_low_power);
-	dwc2_set_param_enable_dynamic_fifo(hsotg,
-			params->enable_dynamic_fifo);
-	dwc2_set_param_host_rx_fifo_size(hsotg,
-			params->host_rx_fifo_size);
-	dwc2_set_param_host_nperio_tx_fifo_size(hsotg,
-			params->host_nperio_tx_fifo_size);
-	dwc2_set_param_host_perio_tx_fifo_size(hsotg,
-			params->host_perio_tx_fifo_size);
-	dwc2_set_param_max_transfer_size(hsotg,
-			params->max_transfer_size);
-	dwc2_set_param_max_packet_count(hsotg,
-			params->max_packet_count);
-	dwc2_set_param_host_channels(hsotg, params->host_channels);
-	dwc2_set_param_phy_type(hsotg, params->phy_type);
-	dwc2_set_param_speed(hsotg, params->speed);
-	dwc2_set_param_host_ls_low_power_phy_clk(hsotg,
-			params->host_ls_low_power_phy_clk);
-	dwc2_set_param_phy_ulpi_ddr(hsotg, params->phy_ulpi_ddr);
-	dwc2_set_param_phy_ulpi_ext_vbus(hsotg,
-			params->phy_ulpi_ext_vbus);
-	dwc2_set_param_phy_utmi_width(hsotg, params->phy_utmi_width);
-	dwc2_set_param_ulpi_fs_ls(hsotg, params->ulpi_fs_ls);
-	dwc2_set_param_ts_dline(hsotg, params->ts_dline);
-	dwc2_set_param_i2c_enable(hsotg, params->i2c_enable);
-	dwc2_set_param_en_multiple_tx_fifo(hsotg,
-			params->en_multiple_tx_fifo);
-	dwc2_set_param_reload_ctl(hsotg, params->reload_ctl);
-	dwc2_set_param_ahbcfg(hsotg, params->ahbcfg);
-	dwc2_set_param_otg_ver(hsotg, params->otg_ver);
-	dwc2_set_param_uframe_sched(hsotg, params->uframe_sched);
-	dwc2_set_param_external_id_pin_ctl(hsotg, params->external_id_pin_ctl);
-	dwc2_set_param_hibernation(hsotg, params->hibernation);
-}
-
 /*
  * Forces either host or device mode if the controller is not
  * currently in that mode.
  *
  * Returns true if the mode was forced.
  */
-static bool dwc2_force_mode_if_needed(struct dwc2_hsotg *hsotg, bool host)
+bool dwc2_force_mode_if_needed(struct dwc2_hsotg *hsotg, bool host)
 {
 	if (host && dwc2_is_host_mode(hsotg))
 		return false;
@@ -1442,232 +751,9 @@ static bool dwc2_force_mode_if_needed(struct dwc2_hsotg *hsotg, bool host)
 	return dwc2_force_mode(hsotg, host);
 }
 
-/*
- * Gets host hardware parameters. Forces host mode if not currently in
- * host mode. Should be called immediately after a core soft reset in
- * order to get the reset values.
- */
-static void dwc2_get_host_hwparams(struct dwc2_hsotg *hsotg)
-{
-	struct dwc2_hw_params *hw = &hsotg->hw_params;
-	u32 gnptxfsiz;
-	u32 hptxfsiz;
-	bool forced;
-
-	if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL)
-		return;
-
-	forced = dwc2_force_mode_if_needed(hsotg, true);
-
-	gnptxfsiz = dwc2_readl(hsotg->regs + GNPTXFSIZ);
-	hptxfsiz = dwc2_readl(hsotg->regs + HPTXFSIZ);
-	dev_dbg(hsotg->dev, "gnptxfsiz=%08x\n", gnptxfsiz);
-	dev_dbg(hsotg->dev, "hptxfsiz=%08x\n", hptxfsiz);
-
-	if (forced)
-		dwc2_clear_force_mode(hsotg);
-
-	hw->host_nperio_tx_fifo_size = (gnptxfsiz & FIFOSIZE_DEPTH_MASK) >>
-				       FIFOSIZE_DEPTH_SHIFT;
-	hw->host_perio_tx_fifo_size = (hptxfsiz & FIFOSIZE_DEPTH_MASK) >>
-				      FIFOSIZE_DEPTH_SHIFT;
-}
-
-/*
- * Gets device hardware parameters. Forces device mode if not
- * currently in device mode. Should be called immediately after a core
- * soft reset in order to get the reset values.
- */
-static void dwc2_get_dev_hwparams(struct dwc2_hsotg *hsotg)
-{
-	struct dwc2_hw_params *hw = &hsotg->hw_params;
-	bool forced;
-	u32 gnptxfsiz;
-
-	if (hsotg->dr_mode == USB_DR_MODE_HOST)
-		return;
-
-	forced = dwc2_force_mode_if_needed(hsotg, false);
-
-	gnptxfsiz = dwc2_readl(hsotg->regs + GNPTXFSIZ);
-	dev_dbg(hsotg->dev, "gnptxfsiz=%08x\n", gnptxfsiz);
-
-	if (forced)
-		dwc2_clear_force_mode(hsotg);
-
-	hw->dev_nperio_tx_fifo_size = (gnptxfsiz & FIFOSIZE_DEPTH_MASK) >>
-				       FIFOSIZE_DEPTH_SHIFT;
-}
-
-/**
- * During device initialization, read various hardware configuration
- * registers and interpret the contents.
- */
-int dwc2_get_hwparams(struct dwc2_hsotg *hsotg)
-{
-	struct dwc2_hw_params *hw = &hsotg->hw_params;
-	unsigned width;
-	u32 hwcfg1, hwcfg2, hwcfg3, hwcfg4;
-	u32 grxfsiz;
-
-	/*
-	 * Attempt to ensure this device is really a DWC_otg Controller.
-	 * Read and verify the GSNPSID register contents. The value should be
-	 * 0x45f42xxx or 0x45f43xxx, which corresponds to either "OT2" or "OT3",
-	 * as in "OTG version 2.xx" or "OTG version 3.xx".
-	 */
-	hw->snpsid = dwc2_readl(hsotg->regs + GSNPSID);
-	if ((hw->snpsid & 0xfffff000) != 0x4f542000 &&
-	    (hw->snpsid & 0xfffff000) != 0x4f543000) {
-		dev_err(hsotg->dev, "Bad value for GSNPSID: 0x%08x\n",
-			hw->snpsid);
-		return -ENODEV;
-	}
-
-	dev_dbg(hsotg->dev, "Core Release: %1x.%1x%1x%1x (snpsid=%x)\n",
-		hw->snpsid >> 12 & 0xf, hw->snpsid >> 8 & 0xf,
-		hw->snpsid >> 4 & 0xf, hw->snpsid & 0xf, hw->snpsid);
-
-	hwcfg1 = dwc2_readl(hsotg->regs + GHWCFG1);
-	hwcfg2 = dwc2_readl(hsotg->regs + GHWCFG2);
-	hwcfg3 = dwc2_readl(hsotg->regs + GHWCFG3);
-	hwcfg4 = dwc2_readl(hsotg->regs + GHWCFG4);
-	grxfsiz = dwc2_readl(hsotg->regs + GRXFSIZ);
-
-	dev_dbg(hsotg->dev, "hwcfg1=%08x\n", hwcfg1);
-	dev_dbg(hsotg->dev, "hwcfg2=%08x\n", hwcfg2);
-	dev_dbg(hsotg->dev, "hwcfg3=%08x\n", hwcfg3);
-	dev_dbg(hsotg->dev, "hwcfg4=%08x\n", hwcfg4);
-	dev_dbg(hsotg->dev, "grxfsiz=%08x\n", grxfsiz);
-
-	/*
-	 * Host specific hardware parameters. Reading these parameters
-	 * requires the controller to be in host mode. The mode will
-	 * be forced, if necessary, to read these values.
-	 */
-	dwc2_get_host_hwparams(hsotg);
-	dwc2_get_dev_hwparams(hsotg);
-
-	/* hwcfg1 */
-	hw->dev_ep_dirs = hwcfg1;
-
-	/* hwcfg2 */
-	hw->op_mode = (hwcfg2 & GHWCFG2_OP_MODE_MASK) >>
-		      GHWCFG2_OP_MODE_SHIFT;
-	hw->arch = (hwcfg2 & GHWCFG2_ARCHITECTURE_MASK) >>
-		   GHWCFG2_ARCHITECTURE_SHIFT;
-	hw->enable_dynamic_fifo = !!(hwcfg2 & GHWCFG2_DYNAMIC_FIFO);
-	hw->host_channels = 1 + ((hwcfg2 & GHWCFG2_NUM_HOST_CHAN_MASK) >>
-				GHWCFG2_NUM_HOST_CHAN_SHIFT);
-	hw->hs_phy_type = (hwcfg2 & GHWCFG2_HS_PHY_TYPE_MASK) >>
-			  GHWCFG2_HS_PHY_TYPE_SHIFT;
-	hw->fs_phy_type = (hwcfg2 & GHWCFG2_FS_PHY_TYPE_MASK) >>
-			  GHWCFG2_FS_PHY_TYPE_SHIFT;
-	hw->num_dev_ep = (hwcfg2 & GHWCFG2_NUM_DEV_EP_MASK) >>
-			 GHWCFG2_NUM_DEV_EP_SHIFT;
-	hw->nperio_tx_q_depth =
-		(hwcfg2 & GHWCFG2_NONPERIO_TX_Q_DEPTH_MASK) >>
-		GHWCFG2_NONPERIO_TX_Q_DEPTH_SHIFT << 1;
-	hw->host_perio_tx_q_depth =
-		(hwcfg2 & GHWCFG2_HOST_PERIO_TX_Q_DEPTH_MASK) >>
-		GHWCFG2_HOST_PERIO_TX_Q_DEPTH_SHIFT << 1;
-	hw->dev_token_q_depth =
-		(hwcfg2 & GHWCFG2_DEV_TOKEN_Q_DEPTH_MASK) >>
-		GHWCFG2_DEV_TOKEN_Q_DEPTH_SHIFT;
-
-	/* hwcfg3 */
-	width = (hwcfg3 & GHWCFG3_XFER_SIZE_CNTR_WIDTH_MASK) >>
-		GHWCFG3_XFER_SIZE_CNTR_WIDTH_SHIFT;
-	hw->max_transfer_size = (1 << (width + 11)) - 1;
-	width = (hwcfg3 & GHWCFG3_PACKET_SIZE_CNTR_WIDTH_MASK) >>
-		GHWCFG3_PACKET_SIZE_CNTR_WIDTH_SHIFT;
-	hw->max_packet_count = (1 << (width + 4)) - 1;
-	hw->i2c_enable = !!(hwcfg3 & GHWCFG3_I2C);
-	hw->total_fifo_size = (hwcfg3 & GHWCFG3_DFIFO_DEPTH_MASK) >>
-			      GHWCFG3_DFIFO_DEPTH_SHIFT;
-
-	/* hwcfg4 */
-	hw->en_multiple_tx_fifo = !!(hwcfg4 & GHWCFG4_DED_FIFO_EN);
-	hw->num_dev_perio_in_ep = (hwcfg4 & GHWCFG4_NUM_DEV_PERIO_IN_EP_MASK) >>
-				  GHWCFG4_NUM_DEV_PERIO_IN_EP_SHIFT;
-	hw->dma_desc_enable = !!(hwcfg4 & GHWCFG4_DESC_DMA);
-	hw->power_optimized = !!(hwcfg4 & GHWCFG4_POWER_OPTIMIZ);
-	hw->utmi_phy_data_width = (hwcfg4 & GHWCFG4_UTMI_PHY_DATA_WIDTH_MASK) >>
-				  GHWCFG4_UTMI_PHY_DATA_WIDTH_SHIFT;
-
-	/* fifo sizes */
-	hw->host_rx_fifo_size = (grxfsiz & GRXFSIZ_DEPTH_MASK) >>
-				GRXFSIZ_DEPTH_SHIFT;
-
-	dev_dbg(hsotg->dev, "Detected values from hardware:\n");
-	dev_dbg(hsotg->dev, "  op_mode=%d\n",
-		hw->op_mode);
-	dev_dbg(hsotg->dev, "  arch=%d\n",
-		hw->arch);
-	dev_dbg(hsotg->dev, "  dma_desc_enable=%d\n",
-		hw->dma_desc_enable);
-	dev_dbg(hsotg->dev, "  power_optimized=%d\n",
-		hw->power_optimized);
-	dev_dbg(hsotg->dev, "  i2c_enable=%d\n",
-		hw->i2c_enable);
-	dev_dbg(hsotg->dev, "  hs_phy_type=%d\n",
-		hw->hs_phy_type);
-	dev_dbg(hsotg->dev, "  fs_phy_type=%d\n",
-		hw->fs_phy_type);
-	dev_dbg(hsotg->dev, "  utmi_phy_data_width=%d\n",
-		hw->utmi_phy_data_width);
-	dev_dbg(hsotg->dev, "  num_dev_ep=%d\n",
-		hw->num_dev_ep);
-	dev_dbg(hsotg->dev, "  num_dev_perio_in_ep=%d\n",
-		hw->num_dev_perio_in_ep);
-	dev_dbg(hsotg->dev, "  host_channels=%d\n",
-		hw->host_channels);
-	dev_dbg(hsotg->dev, "  max_transfer_size=%d\n",
-		hw->max_transfer_size);
-	dev_dbg(hsotg->dev, "  max_packet_count=%d\n",
-		hw->max_packet_count);
-	dev_dbg(hsotg->dev, "  nperio_tx_q_depth=0x%0x\n",
-		hw->nperio_tx_q_depth);
-	dev_dbg(hsotg->dev, "  host_perio_tx_q_depth=0x%0x\n",
-		hw->host_perio_tx_q_depth);
-	dev_dbg(hsotg->dev, "  dev_token_q_depth=0x%0x\n",
-		hw->dev_token_q_depth);
-	dev_dbg(hsotg->dev, "  enable_dynamic_fifo=%d\n",
-		hw->enable_dynamic_fifo);
-	dev_dbg(hsotg->dev, "  en_multiple_tx_fifo=%d\n",
-		hw->en_multiple_tx_fifo);
-	dev_dbg(hsotg->dev, "  total_fifo_size=%d\n",
-		hw->total_fifo_size);
-	dev_dbg(hsotg->dev, "  host_rx_fifo_size=%d\n",
-		hw->host_rx_fifo_size);
-	dev_dbg(hsotg->dev, "  host_nperio_tx_fifo_size=%d\n",
-		hw->host_nperio_tx_fifo_size);
-	dev_dbg(hsotg->dev, "  host_perio_tx_fifo_size=%d\n",
-		hw->host_perio_tx_fifo_size);
-	dev_dbg(hsotg->dev, "\n");
-
-	return 0;
-}
-
-/*
- * Sets all parameters to the given value.
- *
- * Assumes that the dwc2_core_params struct contains only integers.
- */
-void dwc2_set_all_params(struct dwc2_core_params *params, int value)
-{
-	int *p = (int *)params;
-	size_t size = sizeof(*params) / sizeof(*p);
-	int i;
-
-	for (i = 0; i < size; i++)
-		p[i] = value;
-}
-
-
 u16 dwc2_get_otg_version(struct dwc2_hsotg *hsotg)
 {
-	return hsotg->core_params->otg_ver == 1 ? 0x0200 : 0x0103;
+	return hsotg->params.otg_ver == 1 ? 0x0200 : 0x0103;
 }
 
 bool dwc2_is_controller_alive(struct dwc2_hsotg *hsotg)
diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h
index 2a21a0414b1d..9548d3e03453 100644
--- a/drivers/usb/dwc2/core.h
+++ b/drivers/usb/dwc2/core.h
@@ -172,6 +172,11 @@ struct dwc2_hsotg_req;
  * @periodic: Set if this is a periodic ep, such as Interrupt
  * @isochronous: Set if this is a isochronous ep
  * @send_zlp: Set if we need to send a zero-length packet.
+ * @desc_list_dma: The DMA address of descriptor chain currently in use.
+ * @desc_list: Pointer to descriptor DMA chain head currently in use.
+ * @desc_count: Count of entries within the DMA descriptor chain of EP.
+ * @isoc_chain_num: Number of ISOC chain currently in use - either 0 or 1.
+ * @next_desc: index of next free descriptor in the ISOC chain under SW control.
  * @total_data: The total number of data bytes done.
  * @fifo_size: The size of the FIFO (for periodic IN endpoints)
  * @fifo_load: The amount of data loaded into the FIFO (periodic IN)
@@ -219,6 +224,13 @@ struct dwc2_hsotg_ep {
 #define TARGET_FRAME_INITIAL   0xFFFFFFFF
 	bool			frame_overrun;
 
+	dma_addr_t		desc_list_dma;
+	struct dwc2_dma_desc	*desc_list;
+	u8			desc_count;
+
+	unsigned char		isoc_chain_num;
+	unsigned int		next_desc;
+
 	char                    name[10];
 };
 
@@ -286,7 +298,7 @@ enum dwc2_ep0_state {
  * @otg_ver:            OTG version supported
  *                       0 - 1.3 (default)
  *                       1 - 2.0
- * @dma_enable:         Specifies whether to use slave or DMA mode for accessing
+ * @host_dma:           Specifies whether to use slave or DMA mode for accessing
  *                      the data FIFOs. The driver will automatically detect the
  *                      value for this parameter if none is specified.
  *                       0 - Slave (always available)
@@ -314,7 +326,8 @@ enum dwc2_ep0_state {
  * @enable_dynamic_fifo: 0 - Use coreConsultant-specified FIFO size parameters
  *                       1 - Allow dynamic FIFO sizing (default, if available)
  * @en_multiple_tx_fifo: Specifies whether dedicated per-endpoint transmit FIFOs
- *                      are enabled
+ *                      are enabled for non-periodic IN endpoints in device
+ *                      mode.
  * @host_rx_fifo_size:  Number of 4-byte words in the Rx FIFO in host mode when
  *                      dynamic FIFO sizing is enabled
  *                       16 to 32768
@@ -417,6 +430,20 @@ enum dwc2_ep0_state {
  *			needed.
  *			0 - No (default)
  *			1 - Yes
+ * @g_dma:              Enables gadget dma usage (default: autodetect).
+ * @g_dma_desc:         Enables gadget descriptor DMA (default: autodetect).
+ * @g_rx_fifo_size:	The periodic rx fifo size for the device, in
+ *			DWORDS from 16-32768 (default: 2048 if
+ *			possible, otherwise autodetect).
+ * @g_np_tx_fifo_size:	The non-periodic tx fifo size for the device in
+ *			DWORDS from 16-32768 (default: 1024 if
+ *			possible, otherwise autodetect).
+ * @g_tx_fifo_size:	An array of TX fifo sizes in dedicated fifo
+ *			mode. Each value corresponds to one EP
+ *			starting from EP1 (max 15 values). Sizes are
+ *			in DWORDS with possible values from from
+ *			16-32768 (default: 256, 256, 256, 256, 768,
+ *			768, 768, 768, 0, 0, 0, 0, 0, 0, 0).
  *
  * The following parameters may be specified when starting the module. These
  * parameters define how the DWC_otg controller should be configured. A
@@ -430,11 +457,18 @@ struct dwc2_core_params {
 	 * dwc2_set_all_params!
 	 */
 	int otg_cap;
+#define DWC2_CAP_PARAM_HNP_SRP_CAPABLE		0
+#define DWC2_CAP_PARAM_SRP_ONLY_CAPABLE		1
+#define DWC2_CAP_PARAM_NO_HNP_SRP_CAPABLE	2
+
 	int otg_ver;
-	int dma_enable;
 	int dma_desc_enable;
 	int dma_desc_fs_enable;
 	int speed;
+#define DWC2_SPEED_PARAM_HIGH	0
+#define DWC2_SPEED_PARAM_FULL	1
+#define DWC2_SPEED_PARAM_LOW	2
+
 	int enable_dynamic_fifo;
 	int en_multiple_tx_fifo;
 	int host_rx_fifo_size;
@@ -444,19 +478,44 @@ struct dwc2_core_params {
 	int max_packet_count;
 	int host_channels;
 	int phy_type;
+#define DWC2_PHY_TYPE_PARAM_FS		0
+#define DWC2_PHY_TYPE_PARAM_UTMI	1
+#define DWC2_PHY_TYPE_PARAM_ULPI	2
+
 	int phy_utmi_width;
 	int phy_ulpi_ddr;
 	int phy_ulpi_ext_vbus;
+#define DWC2_PHY_ULPI_INTERNAL_VBUS	0
+#define DWC2_PHY_ULPI_EXTERNAL_VBUS	1
+
 	int i2c_enable;
 	int ulpi_fs_ls;
 	int host_support_fs_ls_low_power;
 	int host_ls_low_power_phy_clk;
+#define DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_48MHZ	0
+#define DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_6MHZ	1
+
 	int ts_dline;
 	int reload_ctl;
 	int ahbcfg;
 	int uframe_sched;
 	int external_id_pin_ctl;
 	int hibernation;
+
+	/*
+	 * The following parameters are *only* set via device
+	 * properties and cannot be set directly in this structure.
+	 */
+
+	/* Host parameters */
+	bool host_dma;
+
+	/* Gadget parameters */
+	bool g_dma;
+	bool g_dma_desc;
+	u16 g_rx_fifo_size;
+	u16 g_np_tx_fifo_size;
+	u32 g_tx_fifo_size[MAX_EPS_CHANNELS];
 };
 
 /**
@@ -516,10 +575,9 @@ struct dwc2_hw_params {
 	unsigned op_mode:3;
 	unsigned arch:2;
 	unsigned dma_desc_enable:1;
-	unsigned dma_desc_fs_enable:1;
 	unsigned enable_dynamic_fifo:1;
 	unsigned en_multiple_tx_fifo:1;
-	unsigned host_rx_fifo_size:16;
+	unsigned rx_fifo_size:16;
 	unsigned host_nperio_tx_fifo_size:16;
 	unsigned dev_nperio_tx_fifo_size:16;
 	unsigned host_perio_tx_fifo_size:16;
@@ -839,11 +897,13 @@ struct dwc2_hregs_backup {
  * @ctrl_req:           Request for EP0 control packets.
  * @ep0_state:          EP0 control transfers state
  * @test_mode:          USB test mode requested by the host
+ * @setup_desc_dma:	EP0 setup stage desc chain DMA address
+ * @setup_desc:		EP0 setup stage desc chain pointer
+ * @ctrl_in_desc_dma:	EP0 IN data phase desc chain DMA address
+ * @ctrl_in_desc:	EP0 IN data phase desc chain pointer
+ * @ctrl_out_desc_dma:	EP0 OUT data phase desc chain DMA address
+ * @ctrl_out_desc:	EP0 OUT data phase desc chain pointer
  * @eps:                The endpoints being supplied to the gadget framework
- * @g_using_dma:          Indicate if dma usage is enabled
- * @g_rx_fifo_sz:         Contains rx fifo size value
- * @g_np_g_tx_fifo_sz:      Contains Non-Periodic tx fifo size value
- * @g_tx_fifo_sz:         Contains tx fifo size value per endpoints
  */
 struct dwc2_hsotg {
 	struct device *dev;
@@ -851,7 +911,7 @@ struct dwc2_hsotg {
 	/** Params detected from hardware */
 	struct dwc2_hw_params hw_params;
 	/** Params to actually use */
-	struct dwc2_core_params *core_params;
+	struct dwc2_core_params params;
 	enum usb_otg_state op_state;
 	enum usb_dr_mode dr_mode;
 	unsigned int hcd_enabled:1;
@@ -891,6 +951,8 @@ struct dwc2_hsotg {
 #define DWC2_CORE_REV_2_94a	0x4f54294a
 #define DWC2_CORE_REV_3_00a	0x4f54300a
 #define DWC2_CORE_REV_3_10a	0x4f54310a
+#define DWC2_FS_IOT_REV_1_00a	0x5531100a
+#define DWC2_HS_IOT_REV_1_00a	0x5532100a
 
 #if IS_ENABLED(CONFIG_USB_DWC2_HOST) || IS_ENABLED(CONFIG_USB_DWC2_DUAL_ROLE)
 	union dwc2_hcd_internal_flags {
@@ -986,15 +1048,18 @@ struct dwc2_hsotg {
 	enum dwc2_ep0_state ep0_state;
 	u8 test_mode;
 
+	dma_addr_t setup_desc_dma[2];
+	struct dwc2_dma_desc *setup_desc[2];
+	dma_addr_t ctrl_in_desc_dma;
+	struct dwc2_dma_desc *ctrl_in_desc;
+	dma_addr_t ctrl_out_desc_dma;
+	struct dwc2_dma_desc *ctrl_out_desc;
+
 	struct usb_gadget gadget;
 	unsigned int enabled:1;
 	unsigned int connected:1;
 	struct dwc2_hsotg_ep *eps_in[MAX_EPS_CHANNELS];
 	struct dwc2_hsotg_ep *eps_out[MAX_EPS_CHANNELS];
-	u32 g_using_dma;
-	u32 g_rx_fifo_sz;
-	u32 g_np_g_tx_fifo_sz;
-	u32 g_tx_fifo_sz[MAX_EPS_CHANNELS];
 #endif /* CONFIG_USB_DWC2_PERIPHERAL || CONFIG_USB_DWC2_DUAL_ROLE */
 };
 
@@ -1016,6 +1081,22 @@ enum dwc2_halt_status {
 	DWC2_HC_XFER_URB_DEQUEUE,
 };
 
+/* Core version information */
+static inline bool dwc2_is_iot(struct dwc2_hsotg *hsotg)
+{
+	return (hsotg->hw_params.snpsid & 0xfff00000) == 0x55300000;
+}
+
+static inline bool dwc2_is_fs_iot(struct dwc2_hsotg *hsotg)
+{
+	return (hsotg->hw_params.snpsid & 0xffff0000) == 0x55310000;
+}
+
+static inline bool dwc2_is_hs_iot(struct dwc2_hsotg *hsotg)
+{
+	return (hsotg->hw_params.snpsid & 0xffff0000) == 0x55320000;
+}
+
 /*
  * The following functions support initialization of the core driver component
  * and the DWC_otg controller
@@ -1025,6 +1106,8 @@ extern int dwc2_core_reset_and_force_dr_mode(struct dwc2_hsotg *hsotg);
 extern int dwc2_enter_hibernation(struct dwc2_hsotg *hsotg);
 extern int dwc2_exit_hibernation(struct dwc2_hsotg *hsotg, bool restore);
 
+bool dwc2_force_mode_if_needed(struct dwc2_hsotg *hsotg, bool host);
+void dwc2_clear_force_mode(struct dwc2_hsotg *hsotg);
 void dwc2_force_dr_mode(struct dwc2_hsotg *hsotg);
 
 extern bool dwc2_is_controller_alive(struct dwc2_hsotg *hsotg);
@@ -1044,217 +1127,16 @@ extern void dwc2_disable_global_interrupts(struct dwc2_hsotg *hcd);
 /* This function should be called on every hardware interrupt. */
 extern irqreturn_t dwc2_handle_common_intr(int irq, void *dev);
 
-/* OTG Core Parameters */
-
-/*
- * Specifies the OTG capabilities. The driver will automatically
- * detect the value for this parameter if none is specified.
- * 0 - HNP and SRP capable (default)
- * 1 - SRP Only capable
- * 2 - No HNP/SRP capable
- */
-extern void dwc2_set_param_otg_cap(struct dwc2_hsotg *hsotg, int val);
-#define DWC2_CAP_PARAM_HNP_SRP_CAPABLE		0
-#define DWC2_CAP_PARAM_SRP_ONLY_CAPABLE		1
-#define DWC2_CAP_PARAM_NO_HNP_SRP_CAPABLE	2
-
-/*
- * Specifies whether to use slave or DMA mode for accessing the data
- * FIFOs. The driver will automatically detect the value for this
- * parameter if none is specified.
- * 0 - Slave
- * 1 - DMA (default, if available)
- */
-extern void dwc2_set_param_dma_enable(struct dwc2_hsotg *hsotg, int val);
-
-/*
- * When DMA mode is enabled specifies whether to use
- * address DMA or DMA Descritor mode for accessing the data
- * FIFOs in device mode. The driver will automatically detect
- * the value for this parameter if none is specified.
- * 0 - address DMA
- * 1 - DMA Descriptor(default, if available)
- */
-extern void dwc2_set_param_dma_desc_enable(struct dwc2_hsotg *hsotg, int val);
-
-/*
- * When DMA mode is enabled specifies whether to use
- * address DMA or DMA Descritor mode with full speed devices
- * for accessing the data FIFOs in host mode.
- * 0 - address DMA
- * 1 - FS DMA Descriptor(default, if available)
- */
-extern void dwc2_set_param_dma_desc_fs_enable(struct dwc2_hsotg *hsotg,
-					      int val);
-
-/*
- * Specifies the maximum speed of operation in host and device mode.
- * The actual speed depends on the speed of the attached device and
- * the value of phy_type. The actual speed depends on the speed of the
- * attached device.
- * 0 - High Speed (default)
- * 1 - Full Speed
- */
-extern void dwc2_set_param_speed(struct dwc2_hsotg *hsotg, int val);
-#define DWC2_SPEED_PARAM_HIGH	0
-#define DWC2_SPEED_PARAM_FULL	1
-
-/*
- * Specifies whether low power mode is supported when attached
- * to a Full Speed or Low Speed device in host mode.
- *
- * 0 - Don't support low power mode (default)
- * 1 - Support low power mode
- */
-extern void dwc2_set_param_host_support_fs_ls_low_power(
-		struct dwc2_hsotg *hsotg, int val);
-
-/*
- * Specifies the PHY clock rate in low power mode when connected to a
- * Low Speed device in host mode. This parameter is applicable only if
- * HOST_SUPPORT_FS_LS_LOW_POWER is enabled. If PHY_TYPE is set to FS
- * then defaults to 6 MHZ otherwise 48 MHZ.
- *
- * 0 - 48 MHz
- * 1 - 6 MHz
- */
-extern void dwc2_set_param_host_ls_low_power_phy_clk(struct dwc2_hsotg *hsotg,
-						     int val);
-#define DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_48MHZ	0
-#define DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_6MHZ	1
-
-/*
- * 0 - Use cC FIFO size parameters
- * 1 - Allow dynamic FIFO sizing (default)
- */
-extern void dwc2_set_param_enable_dynamic_fifo(struct dwc2_hsotg *hsotg,
-					       int val);
-
-/*
- * Number of 4-byte words in the Rx FIFO in host mode when dynamic
- * FIFO sizing is enabled.
- * 16 to 32768 (default 1024)
- */
-extern void dwc2_set_param_host_rx_fifo_size(struct dwc2_hsotg *hsotg, int val);
-
-/*
- * Number of 4-byte words in the non-periodic Tx FIFO in host mode
- * when Dynamic FIFO sizing is enabled in the core.
- * 16 to 32768 (default 256)
- */
-extern void dwc2_set_param_host_nperio_tx_fifo_size(struct dwc2_hsotg *hsotg,
-						    int val);
-
-/*
- * Number of 4-byte words in the host periodic Tx FIFO when dynamic
- * FIFO sizing is enabled.
- * 16 to 32768 (default 256)
- */
-extern void dwc2_set_param_host_perio_tx_fifo_size(struct dwc2_hsotg *hsotg,
-						   int val);
-
-/*
- * The maximum transfer size supported in bytes.
- * 2047 to 65,535  (default 65,535)
- */
-extern void dwc2_set_param_max_transfer_size(struct dwc2_hsotg *hsotg, int val);
-
-/*
- * The maximum number of packets in a transfer.
- * 15 to 511  (default 511)
- */
-extern void dwc2_set_param_max_packet_count(struct dwc2_hsotg *hsotg, int val);
-
-/*
- * The number of host channel registers to use.
- * 1 to 16 (default 11)
- * Note: The FPGA configuration supports a maximum of 11 host channels.
- */
-extern void dwc2_set_param_host_channels(struct dwc2_hsotg *hsotg, int val);
-
-/*
- * Specifies the type of PHY interface to use. By default, the driver
- * will automatically detect the phy_type.
- *
- * 0 - Full Speed PHY
- * 1 - UTMI+ (default)
- * 2 - ULPI
- */
-extern void dwc2_set_param_phy_type(struct dwc2_hsotg *hsotg, int val);
-#define DWC2_PHY_TYPE_PARAM_FS		0
-#define DWC2_PHY_TYPE_PARAM_UTMI	1
-#define DWC2_PHY_TYPE_PARAM_ULPI	2
-
-/*
- * Specifies the UTMI+ Data Width. This parameter is
- * applicable for a PHY_TYPE of UTMI+ or ULPI. (For a ULPI
- * PHY_TYPE, this parameter indicates the data width between
- * the MAC and the ULPI Wrapper.) Also, this parameter is
- * applicable only if the OTG_HSPHY_WIDTH cC parameter was set
- * to "8 and 16 bits", meaning that the core has been
- * configured to work at either data path width.
- *
- * 8 or 16 bits (default 16)
- */
-extern void dwc2_set_param_phy_utmi_width(struct dwc2_hsotg *hsotg, int val);
-
-/*
- * Specifies whether the ULPI operates at double or single
- * data rate. This parameter is only applicable if PHY_TYPE is
- * ULPI.
- *
- * 0 - single data rate ULPI interface with 8 bit wide data
- * bus (default)
- * 1 - double data rate ULPI interface with 4 bit wide data
- * bus
- */
-extern void dwc2_set_param_phy_ulpi_ddr(struct dwc2_hsotg *hsotg, int val);
-
-/*
- * Specifies whether to use the internal or external supply to
- * drive the vbus with a ULPI phy.
- */
-extern void dwc2_set_param_phy_ulpi_ext_vbus(struct dwc2_hsotg *hsotg, int val);
-#define DWC2_PHY_ULPI_INTERNAL_VBUS	0
-#define DWC2_PHY_ULPI_EXTERNAL_VBUS	1
-
-/*
- * Specifies whether to use the I2Cinterface for full speed PHY. This
- * parameter is only applicable if PHY_TYPE is FS.
- * 0 - No (default)
- * 1 - Yes
- */
-extern void dwc2_set_param_i2c_enable(struct dwc2_hsotg *hsotg, int val);
-
-extern void dwc2_set_param_ulpi_fs_ls(struct dwc2_hsotg *hsotg, int val);
-
-extern void dwc2_set_param_ts_dline(struct dwc2_hsotg *hsotg, int val);
-
-/*
- * Specifies whether dedicated transmit FIFOs are
- * enabled for non periodic IN endpoints in device mode
- * 0 - No
- * 1 - Yes
- */
-extern void dwc2_set_param_en_multiple_tx_fifo(struct dwc2_hsotg *hsotg,
-					       int val);
-
-extern void dwc2_set_param_reload_ctl(struct dwc2_hsotg *hsotg, int val);
-
-extern void dwc2_set_param_ahbcfg(struct dwc2_hsotg *hsotg, int val);
-
-extern void dwc2_set_param_otg_ver(struct dwc2_hsotg *hsotg, int val);
-
-extern void dwc2_set_parameters(struct dwc2_hsotg *hsotg,
-				const struct dwc2_core_params *params);
-
-extern void dwc2_set_all_params(struct dwc2_core_params *params, int value);
-
-extern int dwc2_get_hwparams(struct dwc2_hsotg *hsotg);
+/* The device ID match table */
+extern const struct of_device_id dwc2_of_match_table[];
 
 extern int dwc2_lowlevel_hw_enable(struct dwc2_hsotg *hsotg);
 extern int dwc2_lowlevel_hw_disable(struct dwc2_hsotg *hsotg);
 
+/* Parameters */
+int dwc2_get_hwparams(struct dwc2_hsotg *hsotg);
+int dwc2_init_params(struct dwc2_hsotg *hsotg);
+
 /*
  * The following functions check the controller's OTG operation mode
  * capability (GHWCFG2.OTG_MODE).
diff --git a/drivers/usb/dwc2/core_intr.c b/drivers/usb/dwc2/core_intr.c
index d85c5c9f96c1..5b228ba6045f 100644
--- a/drivers/usb/dwc2/core_intr.c
+++ b/drivers/usb/dwc2/core_intr.c
@@ -159,9 +159,9 @@ static void dwc2_handle_otg_intr(struct dwc2_hsotg *hsotg)
 			" ++OTG Interrupt: Session Request Success Status Change++\n");
 		gotgctl = dwc2_readl(hsotg->regs + GOTGCTL);
 		if (gotgctl & GOTGCTL_SESREQSCS) {
-			if (hsotg->core_params->phy_type ==
+			if (hsotg->params.phy_type ==
 					DWC2_PHY_TYPE_PARAM_FS
-			    && hsotg->core_params->i2c_enable > 0) {
+			    && hsotg->params.i2c_enable > 0) {
 				hsotg->srp_success = 1;
 			} else {
 				/* Clear Session Request */
@@ -370,7 +370,7 @@ static void dwc2_handle_wakeup_detected_intr(struct dwc2_hsotg *hsotg)
 		/* Change to L0 state */
 		hsotg->lx_state = DWC2_L0;
 	} else {
-		if (hsotg->core_params->hibernation)
+		if (hsotg->params.hibernation)
 			return;
 
 		if (hsotg->lx_state != DWC2_L1) {
diff --git a/drivers/usb/dwc2/debugfs.c b/drivers/usb/dwc2/debugfs.c
index 55d91f24f94a..0a130916a91c 100644
--- a/drivers/usb/dwc2/debugfs.c
+++ b/drivers/usb/dwc2/debugfs.c
@@ -213,7 +213,7 @@ static int fifo_show(struct seq_file *seq, void *v)
 	val = dwc2_readl(regs + GNPTXFSIZ);
 	seq_printf(seq, "NPTXFIFO: Size %d, Start 0x%08x\n",
 		   val >> FIFOSIZE_DEPTH_SHIFT,
-		   val & FIFOSIZE_DEPTH_MASK);
+		   val & FIFOSIZE_STARTADDR_MASK);
 
 	seq_puts(seq, "\nPeriodic TXFIFOs:\n");
 
diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c
index 24fbebc9b409..b95930f20d90 100644
--- a/drivers/usb/dwc2/gadget.c
+++ b/drivers/usb/dwc2/gadget.c
@@ -93,7 +93,18 @@ static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg);
  */
 static inline bool using_dma(struct dwc2_hsotg *hsotg)
 {
-	return hsotg->g_using_dma;
+	return hsotg->params.g_dma;
+}
+
+/*
+ * using_desc_dma - return the descriptor DMA status of the driver.
+ * @hsotg: The driver state.
+ *
+ * Return true if we're using descriptor DMA.
+ */
+static inline bool using_desc_dma(struct dwc2_hsotg *hsotg)
+{
+	return hsotg->params.g_dma_desc;
 }
 
 /**
@@ -190,16 +201,17 @@ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg)
 	unsigned int addr;
 	int timeout;
 	u32 val;
+	u32 *txfsz = hsotg->params.g_tx_fifo_size;
 
 	/* Reset fifo map if not correctly cleared during previous session */
 	WARN_ON(hsotg->fifo_map);
 	hsotg->fifo_map = 0;
 
 	/* set RX/NPTX FIFO sizes */
-	dwc2_writel(hsotg->g_rx_fifo_sz, hsotg->regs + GRXFSIZ);
-	dwc2_writel((hsotg->g_rx_fifo_sz << FIFOSIZE_STARTADDR_SHIFT) |
-		(hsotg->g_np_g_tx_fifo_sz << FIFOSIZE_DEPTH_SHIFT),
-		hsotg->regs + GNPTXFSIZ);
+	dwc2_writel(hsotg->params.g_rx_fifo_size, hsotg->regs + GRXFSIZ);
+	dwc2_writel((hsotg->params.g_rx_fifo_size << FIFOSIZE_STARTADDR_SHIFT) |
+		    (hsotg->params.g_np_tx_fifo_size << FIFOSIZE_DEPTH_SHIFT),
+		    hsotg->regs + GNPTXFSIZ);
 
 	/*
 	 * arange all the rest of the TX FIFOs, as some versions of this
@@ -209,7 +221,7 @@ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg)
 	 */
 
 	/* start at the end of the GNPTXFSIZ, rounded up */
-	addr = hsotg->g_rx_fifo_sz + hsotg->g_np_g_tx_fifo_sz;
+	addr = hsotg->params.g_rx_fifo_size + hsotg->params.g_np_tx_fifo_size;
 
 	/*
 	 * Configure fifos sizes from provided configuration and assign
@@ -217,15 +229,16 @@ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg)
 	 * given endpoint.
 	 */
 	for (ep = 1; ep < MAX_EPS_CHANNELS; ep++) {
-		if (!hsotg->g_tx_fifo_sz[ep])
+		if (!txfsz[ep])
 			continue;
 		val = addr;
-		val |= hsotg->g_tx_fifo_sz[ep] << FIFOSIZE_DEPTH_SHIFT;
-		WARN_ONCE(addr + hsotg->g_tx_fifo_sz[ep] > hsotg->fifo_mem,
+		val |= txfsz[ep] << FIFOSIZE_DEPTH_SHIFT;
+		WARN_ONCE(addr + txfsz[ep] > hsotg->fifo_mem,
 			  "insufficient fifo memory");
-		addr += hsotg->g_tx_fifo_sz[ep];
+		addr += txfsz[ep];
 
 		dwc2_writel(val, hsotg->regs + DPTXFSIZN(ep));
+		val = dwc2_readl(hsotg->regs + DPTXFSIZN(ep));
 	}
 
 	/*
@@ -303,12 +316,55 @@ static void dwc2_hsotg_unmap_dma(struct dwc2_hsotg *hsotg,
 				struct dwc2_hsotg_req *hs_req)
 {
 	struct usb_request *req = &hs_req->req;
+	usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->dir_in);
+}
 
-	/* ignore this if we're not moving any data */
-	if (hs_req->req.length == 0)
-		return;
+/*
+ * dwc2_gadget_alloc_ctrl_desc_chains - allocate DMA descriptor chains
+ * for Control endpoint
+ * @hsotg: The device state.
+ *
+ * This function will allocate 4 descriptor chains for EP 0: 2 for
+ * Setup stage, per one for IN and OUT data/status transactions.
+ */
+static int dwc2_gadget_alloc_ctrl_desc_chains(struct dwc2_hsotg *hsotg)
+{
+	hsotg->setup_desc[0] =
+		dmam_alloc_coherent(hsotg->dev,
+				    sizeof(struct dwc2_dma_desc),
+				    &hsotg->setup_desc_dma[0],
+				    GFP_KERNEL);
+	if (!hsotg->setup_desc[0])
+		goto fail;
+
+	hsotg->setup_desc[1] =
+		dmam_alloc_coherent(hsotg->dev,
+				    sizeof(struct dwc2_dma_desc),
+				    &hsotg->setup_desc_dma[1],
+				    GFP_KERNEL);
+	if (!hsotg->setup_desc[1])
+		goto fail;
+
+	hsotg->ctrl_in_desc =
+		dmam_alloc_coherent(hsotg->dev,
+				    sizeof(struct dwc2_dma_desc),
+				    &hsotg->ctrl_in_desc_dma,
+				    GFP_KERNEL);
+	if (!hsotg->ctrl_in_desc)
+		goto fail;
+
+	hsotg->ctrl_out_desc =
+		dmam_alloc_coherent(hsotg->dev,
+				    sizeof(struct dwc2_dma_desc),
+				    &hsotg->ctrl_out_desc_dma,
+				    GFP_KERNEL);
+	if (!hsotg->ctrl_out_desc)
+		goto fail;
 
-	usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->dir_in);
+	return 0;
+
+fail:
+	return -ENOMEM;
 }
 
 /**
@@ -541,6 +597,273 @@ static u32 dwc2_hsotg_read_frameno(struct dwc2_hsotg *hsotg)
 }
 
 /**
+ * dwc2_gadget_get_chain_limit - get the maximum data payload value of the
+ * DMA descriptor chain prepared for specific endpoint
+ * @hs_ep: The endpoint
+ *
+ * Return the maximum data that can be queued in one go on a given endpoint
+ * depending on its descriptor chain capacity so that transfers that
+ * are too long can be split.
+ */
+static unsigned int dwc2_gadget_get_chain_limit(struct dwc2_hsotg_ep *hs_ep)
+{
+	int is_isoc = hs_ep->isochronous;
+	unsigned int maxsize;
+
+	if (is_isoc)
+		maxsize = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT :
+					   DEV_DMA_ISOC_RX_NBYTES_LIMIT;
+	else
+		maxsize = DEV_DMA_NBYTES_LIMIT;
+
+	/* Above size of one descriptor was chosen, multiple it */
+	maxsize *= MAX_DMA_DESC_NUM_GENERIC;
+
+	return maxsize;
+}
+
+/*
+ * dwc2_gadget_get_desc_params - get DMA descriptor parameters.
+ * @hs_ep: The endpoint
+ * @mask: RX/TX bytes mask to be defined
+ *
+ * Returns maximum data payload for one descriptor after analyzing endpoint
+ * characteristics.
+ * DMA descriptor transfer bytes limit depends on EP type:
+ * Control out - MPS,
+ * Isochronous - descriptor rx/tx bytes bitfield limit,
+ * Control In/Bulk/Interrupt - multiple of mps. This will allow to not
+ * have concatenations from various descriptors within one packet.
+ *
+ * Selects corresponding mask for RX/TX bytes as well.
+ */
+static u32 dwc2_gadget_get_desc_params(struct dwc2_hsotg_ep *hs_ep, u32 *mask)
+{
+	u32 mps = hs_ep->ep.maxpacket;
+	int dir_in = hs_ep->dir_in;
+	u32 desc_size = 0;
+
+	if (!hs_ep->index && !dir_in) {
+		desc_size = mps;
+		*mask = DEV_DMA_NBYTES_MASK;
+	} else if (hs_ep->isochronous) {
+		if (dir_in) {
+			desc_size = DEV_DMA_ISOC_TX_NBYTES_LIMIT;
+			*mask = DEV_DMA_ISOC_TX_NBYTES_MASK;
+		} else {
+			desc_size = DEV_DMA_ISOC_RX_NBYTES_LIMIT;
+			*mask = DEV_DMA_ISOC_RX_NBYTES_MASK;
+		}
+	} else {
+		desc_size = DEV_DMA_NBYTES_LIMIT;
+		*mask = DEV_DMA_NBYTES_MASK;
+
+		/* Round down desc_size to be mps multiple */
+		desc_size -= desc_size % mps;
+	}
+
+	return desc_size;
+}
+
+/*
+ * dwc2_gadget_config_nonisoc_xfer_ddma - prepare non ISOC DMA desc chain.
+ * @hs_ep: The endpoint
+ * @dma_buff: DMA address to use
+ * @len: Length of the transfer
+ *
+ * This function will iterate over descriptor chain and fill its entries
+ * with corresponding information based on transfer data.
+ */
+static void dwc2_gadget_config_nonisoc_xfer_ddma(struct dwc2_hsotg_ep *hs_ep,
+						 dma_addr_t dma_buff,
+						 unsigned int len)
+{
+	struct dwc2_hsotg *hsotg = hs_ep->parent;
+	int dir_in = hs_ep->dir_in;
+	struct dwc2_dma_desc *desc = hs_ep->desc_list;
+	u32 mps = hs_ep->ep.maxpacket;
+	u32 maxsize = 0;
+	u32 offset = 0;
+	u32 mask = 0;
+	int i;
+
+	maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask);
+
+	hs_ep->desc_count = (len / maxsize) +
+				((len % maxsize) ? 1 : 0);
+	if (len == 0)
+		hs_ep->desc_count = 1;
+
+	for (i = 0; i < hs_ep->desc_count; ++i) {
+		desc->status = 0;
+		desc->status |= (DEV_DMA_BUFF_STS_HBUSY
+				 << DEV_DMA_BUFF_STS_SHIFT);
+
+		if (len > maxsize) {
+			if (!hs_ep->index && !dir_in)
+				desc->status |= (DEV_DMA_L | DEV_DMA_IOC);
+
+			desc->status |= (maxsize <<
+						DEV_DMA_NBYTES_SHIFT & mask);
+			desc->buf = dma_buff + offset;
+
+			len -= maxsize;
+			offset += maxsize;
+		} else {
+			desc->status |= (DEV_DMA_L | DEV_DMA_IOC);
+
+			if (dir_in)
+				desc->status |= (len % mps) ? DEV_DMA_SHORT :
+					((hs_ep->send_zlp) ? DEV_DMA_SHORT : 0);
+			if (len > maxsize)
+				dev_err(hsotg->dev, "wrong len %d\n", len);
+
+			desc->status |=
+				len << DEV_DMA_NBYTES_SHIFT & mask;
+			desc->buf = dma_buff + offset;
+		}
+
+		desc->status &= ~DEV_DMA_BUFF_STS_MASK;
+		desc->status |= (DEV_DMA_BUFF_STS_HREADY
+				 << DEV_DMA_BUFF_STS_SHIFT);
+		desc++;
+	}
+}
+
+/*
+ * dwc2_gadget_fill_isoc_desc - fills next isochronous descriptor in chain.
+ * @hs_ep: The isochronous endpoint.
+ * @dma_buff: usb requests dma buffer.
+ * @len: usb request transfer length.
+ *
+ * Finds out index of first free entry either in the bottom or up half of
+ * descriptor chain depend on which is under SW control and not processed
+ * by HW. Then fills that descriptor with the data of the arrived usb request,
+ * frame info, sets Last and IOC bits increments next_desc. If filled
+ * descriptor is not the first one, removes L bit from the previous descriptor
+ * status.
+ */
+static int dwc2_gadget_fill_isoc_desc(struct dwc2_hsotg_ep *hs_ep,
+				      dma_addr_t dma_buff, unsigned int len)
+{
+	struct dwc2_dma_desc *desc;
+	struct dwc2_hsotg *hsotg = hs_ep->parent;
+	u32 index;
+	u32 maxsize = 0;
+	u32 mask = 0;
+
+	maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask);
+	if (len > maxsize) {
+		dev_err(hsotg->dev, "wrong len %d\n", len);
+		return -EINVAL;
+	}
+
+	/*
+	 * If SW has already filled half of chain, then return and wait for
+	 * the other chain to be processed by HW.
+	 */
+	if (hs_ep->next_desc == MAX_DMA_DESC_NUM_GENERIC / 2)
+		return -EBUSY;
+
+	/* Increment frame number by interval for IN */
+	if (hs_ep->dir_in)
+		dwc2_gadget_incr_frame_num(hs_ep);
+
+	index = (MAX_DMA_DESC_NUM_GENERIC / 2) * hs_ep->isoc_chain_num +
+		 hs_ep->next_desc;
+
+	/* Sanity check of calculated index */
+	if ((hs_ep->isoc_chain_num && index > MAX_DMA_DESC_NUM_GENERIC) ||
+	    (!hs_ep->isoc_chain_num && index > MAX_DMA_DESC_NUM_GENERIC / 2)) {
+		dev_err(hsotg->dev, "wrong index %d for iso chain\n", index);
+		return -EINVAL;
+	}
+
+	desc = &hs_ep->desc_list[index];
+
+	/* Clear L bit of previous desc if more than one entries in the chain */
+	if (hs_ep->next_desc)
+		hs_ep->desc_list[index - 1].status &= ~DEV_DMA_L;
+
+	dev_dbg(hsotg->dev, "%s: Filling ep %d, dir %s isoc desc # %d\n",
+		__func__, hs_ep->index, hs_ep->dir_in ? "in" : "out", index);
+
+	desc->status = 0;
+	desc->status |= (DEV_DMA_BUFF_STS_HBUSY	<< DEV_DMA_BUFF_STS_SHIFT);
+
+	desc->buf = dma_buff;
+	desc->status |= (DEV_DMA_L | DEV_DMA_IOC |
+			 ((len << DEV_DMA_NBYTES_SHIFT) & mask));
+
+	if (hs_ep->dir_in) {
+		desc->status |= ((hs_ep->mc << DEV_DMA_ISOC_PID_SHIFT) &
+				 DEV_DMA_ISOC_PID_MASK) |
+				((len % hs_ep->ep.maxpacket) ?
+				 DEV_DMA_SHORT : 0) |
+				((hs_ep->target_frame <<
+				  DEV_DMA_ISOC_FRNUM_SHIFT) &
+				 DEV_DMA_ISOC_FRNUM_MASK);
+	}
+
+	desc->status &= ~DEV_DMA_BUFF_STS_MASK;
+	desc->status |= (DEV_DMA_BUFF_STS_HREADY << DEV_DMA_BUFF_STS_SHIFT);
+
+	/* Update index of last configured entry in the chain */
+	hs_ep->next_desc++;
+
+	return 0;
+}
+
+/*
+ * dwc2_gadget_start_isoc_ddma - start isochronous transfer in DDMA
+ * @hs_ep: The isochronous endpoint.
+ *
+ * Prepare first descriptor chain for isochronous endpoints. Afterwards
+ * write DMA address to HW and enable the endpoint.
+ *
+ * Switch between descriptor chains via isoc_chain_num to give SW opportunity
+ * to prepare second descriptor chain while first one is being processed by HW.
+ */
+static void dwc2_gadget_start_isoc_ddma(struct dwc2_hsotg_ep *hs_ep)
+{
+	struct dwc2_hsotg *hsotg = hs_ep->parent;
+	struct dwc2_hsotg_req *hs_req, *treq;
+	int index = hs_ep->index;
+	int ret;
+	u32 dma_reg;
+	u32 depctl;
+	u32 ctrl;
+
+	if (list_empty(&hs_ep->queue)) {
+		dev_dbg(hsotg->dev, "%s: No requests in queue\n", __func__);
+		return;
+	}
+
+	list_for_each_entry_safe(hs_req, treq, &hs_ep->queue, queue) {
+		ret = dwc2_gadget_fill_isoc_desc(hs_ep, hs_req->req.dma,
+						 hs_req->req.length);
+		if (ret) {
+			dev_dbg(hsotg->dev, "%s: desc chain full\n", __func__);
+			break;
+		}
+	}
+
+	depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
+	dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index);
+
+	/* write descriptor chain address to control register */
+	dwc2_writel(hs_ep->desc_list_dma, hsotg->regs + dma_reg);
+
+	ctrl = dwc2_readl(hsotg->regs + depctl);
+	ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK;
+	dwc2_writel(ctrl, hsotg->regs + depctl);
+
+	/* Switch ISOC descriptor chain number being processed by SW*/
+	hs_ep->isoc_chain_num = (hs_ep->isoc_chain_num ^ 1) & 0x1;
+	hs_ep->next_desc = 0;
+}
+
+/**
  * dwc2_hsotg_start_req - start a USB request from an endpoint's queue
  * @hsotg: The controller state.
  * @hs_ep: The endpoint to process a request for
@@ -565,6 +888,7 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg,
 	unsigned length;
 	unsigned packets;
 	unsigned maxreq;
+	unsigned int dma_reg;
 
 	if (index != 0) {
 		if (hs_ep->req && !continuing) {
@@ -579,6 +903,7 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg,
 		}
 	}
 
+	dma_reg = dir_in ? DIEPDMA(index) : DOEPDMA(index);
 	epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
 	epsize_reg = dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);
 
@@ -598,7 +923,11 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg,
 	dev_dbg(hsotg->dev, "ureq->length:%d ureq->actual:%d\n",
 		ureq->length, ureq->actual);
 
-	maxreq = get_ep_limit(hs_ep);
+	if (!using_desc_dma(hsotg))
+		maxreq = get_ep_limit(hs_ep);
+	else
+		maxreq = dwc2_gadget_get_chain_limit(hs_ep);
+
 	if (length > maxreq) {
 		int round = maxreq % hs_ep->ep.maxpacket;
 
@@ -650,22 +979,51 @@ static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg,
 	/* store the request as the current one we're doing */
 	hs_ep->req = hs_req;
 
-	/* write size / packets */
-	dwc2_writel(epsize, hsotg->regs + epsize_reg);
+	if (using_desc_dma(hsotg)) {
+		u32 offset = 0;
+		u32 mps = hs_ep->ep.maxpacket;
 
-	if (using_dma(hsotg) && !continuing) {
-		unsigned int dma_reg;
+		/* Adjust length: EP0 - MPS, other OUT EPs - multiple of MPS */
+		if (!dir_in) {
+			if (!index)
+				length = mps;
+			else if (length % mps)
+				length += (mps - (length % mps));
+		}
 
 		/*
-		 * write DMA address to control register, buffer already
-		 * synced by dwc2_hsotg_ep_queue().
+		 * If more data to send, adjust DMA for EP0 out data stage.
+		 * ureq->dma stays unchanged, hence increment it by already
+		 * passed passed data count before starting new transaction.
 		 */
+		if (!index && hsotg->ep0_state == DWC2_EP0_DATA_OUT &&
+		    continuing)
+			offset = ureq->actual;
 
-		dma_reg = dir_in ? DIEPDMA(index) : DOEPDMA(index);
-		dwc2_writel(ureq->dma, hsotg->regs + dma_reg);
+		/* Fill DDMA chain entries */
+		dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, ureq->dma + offset,
+						     length);
 
-		dev_dbg(hsotg->dev, "%s: %pad => 0x%08x\n",
-			__func__, &ureq->dma, dma_reg);
+		/* write descriptor chain address to control register */
+		dwc2_writel(hs_ep->desc_list_dma, hsotg->regs + dma_reg);
+
+		dev_dbg(hsotg->dev, "%s: %08x pad => 0x%08x\n",
+			__func__, (u32)hs_ep->desc_list_dma, dma_reg);
+	} else {
+		/* write size / packets */
+		dwc2_writel(epsize, hsotg->regs + epsize_reg);
+
+		if (using_dma(hsotg) && !continuing && (length != 0)) {
+			/*
+			 * write DMA address to control register, buffer
+			 * already synced by dwc2_hsotg_ep_queue().
+			 */
+
+			dwc2_writel(ureq->dma, hsotg->regs + dma_reg);
+
+			dev_dbg(hsotg->dev, "%s: %pad => 0x%08x\n",
+				__func__, &ureq->dma, dma_reg);
+		}
 	}
 
 	if (hs_ep->isochronous && hs_ep->interval == 1) {
@@ -738,13 +1096,8 @@ static int dwc2_hsotg_map_dma(struct dwc2_hsotg *hsotg,
 			     struct dwc2_hsotg_ep *hs_ep,
 			     struct usb_request *req)
 {
-	struct dwc2_hsotg_req *hs_req = our_req(req);
 	int ret;
 
-	/* if the length is zero, ignore the DMA data */
-	if (hs_req->req.length == 0)
-		return 0;
-
 	ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in);
 	if (ret)
 		goto dma_error;
@@ -835,6 +1188,41 @@ static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep)
 	return false;
 }
 
+/*
+ * dwc2_gadget_set_ep0_desc_chain - Set EP's desc chain pointers
+ * @hsotg: The driver state
+ * @hs_ep: the ep descriptor chain is for
+ *
+ * Called to update EP0 structure's pointers depend on stage of
+ * control transfer.
+ */
+static int dwc2_gadget_set_ep0_desc_chain(struct dwc2_hsotg *hsotg,
+					  struct dwc2_hsotg_ep *hs_ep)
+{
+	switch (hsotg->ep0_state) {
+	case DWC2_EP0_SETUP:
+	case DWC2_EP0_STATUS_OUT:
+		hs_ep->desc_list = hsotg->setup_desc[0];
+		hs_ep->desc_list_dma = hsotg->setup_desc_dma[0];
+		break;
+	case DWC2_EP0_DATA_IN:
+	case DWC2_EP0_STATUS_IN:
+		hs_ep->desc_list = hsotg->ctrl_in_desc;
+		hs_ep->desc_list_dma = hsotg->ctrl_in_desc_dma;
+		break;
+	case DWC2_EP0_DATA_OUT:
+		hs_ep->desc_list = hsotg->ctrl_out_desc;
+		hs_ep->desc_list_dma = hsotg->ctrl_out_desc_dma;
+		break;
+	default:
+		dev_err(hsotg->dev, "invalid EP 0 state in queue %d\n",
+			hsotg->ep0_state);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req,
 			      gfp_t gfp_flags)
 {
@@ -870,10 +1258,32 @@ static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req,
 		if (ret)
 			return ret;
 	}
+	/* If using descriptor DMA configure EP0 descriptor chain pointers */
+	if (using_desc_dma(hs) && !hs_ep->index) {
+		ret = dwc2_gadget_set_ep0_desc_chain(hs, hs_ep);
+		if (ret)
+			return ret;
+	}
 
 	first = list_empty(&hs_ep->queue);
 	list_add_tail(&hs_req->queue, &hs_ep->queue);
 
+	/*
+	 * Handle DDMA isochronous transfers separately - just add new entry
+	 * to the half of descriptor chain that is not processed by HW.
+	 * Transfer will be started once SW gets either one of NAK or
+	 * OutTknEpDis interrupts.
+	 */
+	if (using_desc_dma(hs) && hs_ep->isochronous &&
+	    hs_ep->target_frame != TARGET_FRAME_INITIAL) {
+		ret = dwc2_gadget_fill_isoc_desc(hs_ep, hs_req->req.dma,
+						 hs_req->req.length);
+		if (ret)
+			dev_dbg(hs->dev, "%s: ISO desc chain full\n", __func__);
+
+		return 0;
+	}
+
 	if (first) {
 		if (!hs_ep->isochronous) {
 			dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);
@@ -1099,10 +1509,8 @@ static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now);
  */
 static struct dwc2_hsotg_req *get_ep_head(struct dwc2_hsotg_ep *hs_ep)
 {
-	if (list_empty(&hs_ep->queue))
-		return NULL;
-
-	return list_first_entry(&hs_ep->queue, struct dwc2_hsotg_req, queue);
+	return list_first_entry_or_null(&hs_ep->queue, struct dwc2_hsotg_req,
+					queue);
 }
 
 /**
@@ -1440,14 +1848,21 @@ static void dwc2_hsotg_program_zlp(struct dwc2_hsotg *hsotg,
 
 	if (hs_ep->dir_in)
 		dev_dbg(hsotg->dev, "Sending zero-length packet on ep%d\n",
-									index);
+			index);
 	else
 		dev_dbg(hsotg->dev, "Receiving zero-length packet on ep%d\n",
-									index);
+			index);
+	if (using_desc_dma(hsotg)) {
+		/* Not specific buffer needed for ep0 ZLP */
+		dma_addr_t dma = hs_ep->desc_list_dma;
 
-	dwc2_writel(DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) |
-		    DXEPTSIZ_XFERSIZE(0), hsotg->regs +
-		    epsiz_reg);
+		dwc2_gadget_set_ep0_desc_chain(hsotg, hs_ep);
+		dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, dma, 0);
+	} else {
+		dwc2_writel(DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) |
+			    DXEPTSIZ_XFERSIZE(0), hsotg->regs +
+			    epsiz_reg);
+	}
 
 	ctrl = dwc2_readl(hsotg->regs + epctl_reg);
 	ctrl |= DXEPCTL_CNAK;  /* clear NAK set by core */
@@ -1510,6 +1925,10 @@ static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg,
 		spin_lock(&hsotg->lock);
 	}
 
+	/* In DDMA don't need to proceed to starting of next ISOC request */
+	if (using_desc_dma(hsotg) && hs_ep->isochronous)
+		return;
+
 	/*
 	 * Look to see if there is anything else to do. Note, the completion
 	 * of the previous request may have caused a new request to be started
@@ -1521,6 +1940,115 @@ static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg,
 	}
 }
 
+/*
+ * dwc2_gadget_complete_isoc_request_ddma - complete an isoc request in DDMA
+ * @hs_ep: The endpoint the request was on.
+ *
+ * Get first request from the ep queue, determine descriptor on which complete
+ * happened. SW based on isoc_chain_num discovers which half of the descriptor
+ * chain is currently in use by HW, adjusts dma_address and calculates index
+ * of completed descriptor based on the value of DEPDMA register. Update actual
+ * length of request, giveback to gadget.
+ */
+static void dwc2_gadget_complete_isoc_request_ddma(struct dwc2_hsotg_ep *hs_ep)
+{
+	struct dwc2_hsotg *hsotg = hs_ep->parent;
+	struct dwc2_hsotg_req *hs_req;
+	struct usb_request *ureq;
+	int index;
+	dma_addr_t dma_addr;
+	u32 dma_reg;
+	u32 depdma;
+	u32 desc_sts;
+	u32 mask;
+
+	hs_req = get_ep_head(hs_ep);
+	if (!hs_req) {
+		dev_warn(hsotg->dev, "%s: ISOC EP queue empty\n", __func__);
+		return;
+	}
+	ureq = &hs_req->req;
+
+	dma_addr = hs_ep->desc_list_dma;
+
+	/*
+	 * If lower half of  descriptor chain is currently use by SW,
+	 * that means higher half is being processed by HW, so shift
+	 * DMA address to higher half of descriptor chain.
+	 */
+	if (!hs_ep->isoc_chain_num)
+		dma_addr += sizeof(struct dwc2_dma_desc) *
+			    (MAX_DMA_DESC_NUM_GENERIC / 2);
+
+	dma_reg = hs_ep->dir_in ? DIEPDMA(hs_ep->index) : DOEPDMA(hs_ep->index);
+	depdma = dwc2_readl(hsotg->regs + dma_reg);
+
+	index = (depdma - dma_addr) / sizeof(struct dwc2_dma_desc) - 1;
+	desc_sts = hs_ep->desc_list[index].status;
+
+	mask = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_MASK :
+	       DEV_DMA_ISOC_RX_NBYTES_MASK;
+	ureq->actual = ureq->length -
+		       ((desc_sts & mask) >> DEV_DMA_ISOC_NBYTES_SHIFT);
+
+	/* Adjust actual length for ISOC Out if length is not align of 4 */
+	if (!hs_ep->dir_in && ureq->length & 0x3)
+		ureq->actual += 4 - (ureq->length & 0x3);
+
+	dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
+}
+
+/*
+ * dwc2_gadget_start_next_isoc_ddma - start next isoc request, if any.
+ * @hs_ep: The isochronous endpoint to be re-enabled.
+ *
+ * If ep has been disabled due to last descriptor servicing (IN endpoint) or
+ * BNA (OUT endpoint) check the status of other half of descriptor chain that
+ * was under SW control till HW was busy and restart the endpoint if needed.
+ */
+static void dwc2_gadget_start_next_isoc_ddma(struct dwc2_hsotg_ep *hs_ep)
+{
+	struct dwc2_hsotg *hsotg = hs_ep->parent;
+	u32 depctl;
+	u32 dma_reg;
+	u32 ctrl;
+	u32 dma_addr = hs_ep->desc_list_dma;
+	unsigned char index = hs_ep->index;
+
+	dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index);
+	depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
+
+	ctrl = dwc2_readl(hsotg->regs + depctl);
+
+	/*
+	 * EP was disabled if HW has processed last descriptor or BNA was set.
+	 * So restart ep if SW has prepared new descriptor chain in ep_queue
+	 * routine while HW was busy.
+	 */
+	if (!(ctrl & DXEPCTL_EPENA)) {
+		if (!hs_ep->next_desc) {
+			dev_dbg(hsotg->dev, "%s: No more ISOC requests\n",
+				__func__);
+			return;
+		}
+
+		dma_addr += sizeof(struct dwc2_dma_desc) *
+			    (MAX_DMA_DESC_NUM_GENERIC / 2) *
+			    hs_ep->isoc_chain_num;
+		dwc2_writel(dma_addr, hsotg->regs + dma_reg);
+
+		ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK;
+		dwc2_writel(ctrl, hsotg->regs + depctl);
+
+		/* Switch ISOC descriptor chain number being processed by SW*/
+		hs_ep->isoc_chain_num = (hs_ep->isoc_chain_num ^ 1) & 0x1;
+		hs_ep->next_desc = 0;
+
+		dev_dbg(hsotg->dev, "%s: Restarted isochronous endpoint\n",
+			__func__);
+	}
+}
+
 /**
  * dwc2_hsotg_rx_data - receive data from the FIFO for an endpoint
  * @hsotg: The device state.
@@ -1618,6 +2146,36 @@ static void dwc2_hsotg_change_ep_iso_parity(struct dwc2_hsotg *hsotg,
 	dwc2_writel(ctrl, hsotg->regs + epctl_reg);
 }
 
+/*
+ * dwc2_gadget_get_xfersize_ddma - get transferred bytes amount from desc
+ * @hs_ep - The endpoint on which transfer went
+ *
+ * Iterate over endpoints descriptor chain and get info on bytes remained
+ * in DMA descriptors after transfer has completed. Used for non isoc EPs.
+ */
+static unsigned int dwc2_gadget_get_xfersize_ddma(struct dwc2_hsotg_ep *hs_ep)
+{
+	struct dwc2_hsotg *hsotg = hs_ep->parent;
+	unsigned int bytes_rem = 0;
+	struct dwc2_dma_desc *desc = hs_ep->desc_list;
+	int i;
+	u32 status;
+
+	if (!desc)
+		return -EINVAL;
+
+	for (i = 0; i < hs_ep->desc_count; ++i) {
+		status = desc->status;
+		bytes_rem += status & DEV_DMA_NBYTES_MASK;
+
+		if (status & DEV_DMA_STS_MASK)
+			dev_err(hsotg->dev, "descriptor %d closed with %x\n",
+				i, status & DEV_DMA_STS_MASK);
+	}
+
+	return bytes_rem;
+}
+
 /**
  * dwc2_hsotg_handle_outdone - handle receiving OutDone/SetupDone from RXFIFO
  * @hsotg: The device instance
@@ -1648,6 +2206,9 @@ static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum)
 		return;
 	}
 
+	if (using_desc_dma(hsotg))
+		size_left = dwc2_gadget_get_xfersize_ddma(hs_ep);
+
 	if (using_dma(hsotg)) {
 		unsigned size_done;
 
@@ -1682,7 +2243,9 @@ static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum)
 		 */
 	}
 
-	if (epnum == 0 && hsotg->ep0_state == DWC2_EP0_DATA_OUT) {
+	/* DDMA IN status phase will start from StsPhseRcvd interrupt */
+	if (!using_desc_dma(hsotg) && epnum == 0 &&
+	    hsotg->ep0_state == DWC2_EP0_DATA_OUT) {
 		/* Move to STATUS IN */
 		dwc2_hsotg_ep0_zlp(hsotg, true);
 		return;
@@ -1812,17 +2375,17 @@ static u32 dwc2_hsotg_ep0_mps(unsigned int mps)
  * @hsotg: The driver state.
  * @ep: The index number of the endpoint
  * @mps: The maximum packet size in bytes
+ * @mc: The multicount value
  *
  * Configure the maximum packet size for the given endpoint, updating
  * the hardware control registers to reflect this.
  */
 static void dwc2_hsotg_set_ep_maxpacket(struct dwc2_hsotg *hsotg,
-			unsigned int ep, unsigned int mps, unsigned int dir_in)
+					unsigned int ep, unsigned int mps,
+					unsigned int mc, unsigned int dir_in)
 {
 	struct dwc2_hsotg_ep *hs_ep;
 	void __iomem *regs = hsotg->regs;
-	u32 mpsval;
-	u32 mcval;
 	u32 reg;
 
 	hs_ep = index_to_ep(hsotg, ep, dir_in);
@@ -1830,32 +2393,32 @@ static void dwc2_hsotg_set_ep_maxpacket(struct dwc2_hsotg *hsotg,
 		return;
 
 	if (ep == 0) {
+		u32 mps_bytes = mps;
+
 		/* EP0 is a special case */
-		mpsval = dwc2_hsotg_ep0_mps(mps);
-		if (mpsval > 3)
+		mps = dwc2_hsotg_ep0_mps(mps_bytes);
+		if (mps > 3)
 			goto bad_mps;
-		hs_ep->ep.maxpacket = mps;
+		hs_ep->ep.maxpacket = mps_bytes;
 		hs_ep->mc = 1;
 	} else {
-		mpsval = mps & DXEPCTL_MPS_MASK;
-		if (mpsval > 1024)
+		if (mps > 1024)
 			goto bad_mps;
-		mcval = ((mps >> 11) & 0x3) + 1;
-		hs_ep->mc = mcval;
-		if (mcval > 3)
+		hs_ep->mc = mc;
+		if (mc > 3)
 			goto bad_mps;
-		hs_ep->ep.maxpacket = mpsval;
+		hs_ep->ep.maxpacket = mps;
 	}
 
 	if (dir_in) {
 		reg = dwc2_readl(regs + DIEPCTL(ep));
 		reg &= ~DXEPCTL_MPS_MASK;
-		reg |= mpsval;
+		reg |= mps;
 		dwc2_writel(reg, regs + DIEPCTL(ep));
 	} else {
 		reg = dwc2_readl(regs + DOEPCTL(ep));
 		reg &= ~DXEPCTL_MPS_MASK;
-		reg |= mpsval;
+		reg |= mps;
 		dwc2_writel(reg, regs + DOEPCTL(ep));
 	}
 
@@ -1954,6 +2517,13 @@ static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg,
 	/* Finish ZLP handling for IN EP0 transactions */
 	if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) {
 		dev_dbg(hsotg->dev, "zlp packet sent\n");
+
+		/*
+		 * While send zlp for DWC2_EP0_STATUS_IN EP direction was
+		 * changed to IN. Change back to complete OUT transfer request
+		 */
+		hs_ep->dir_in = 0;
+
 		dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
 		if (hsotg->test_mode) {
 			int ret;
@@ -1979,8 +2549,14 @@ static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg,
 	 * past the end of the buffer (DMA transfers are always 32bit
 	 * aligned).
 	 */
-
-	size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
+	if (using_desc_dma(hsotg)) {
+		size_left = dwc2_gadget_get_xfersize_ddma(hs_ep);
+		if (size_left < 0)
+			dev_err(hsotg->dev, "error parsing DDMA results %d\n",
+				size_left);
+	} else {
+		size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
+	}
 
 	size_done = hs_ep->size_loaded - size_left;
 	size_done += hs_ep->last_load;
@@ -2128,12 +2704,28 @@ static void dwc2_gadget_handle_out_token_ep_disabled(struct dwc2_hsotg_ep *ep)
 	struct dwc2_hsotg *hsotg = ep->parent;
 	int dir_in = ep->dir_in;
 	u32 doepmsk;
+	u32 tmp;
 
 	if (dir_in || !ep->isochronous)
 		return;
 
+	/*
+	 * Store frame in which irq was asserted here, as
+	 * it can change while completing request below.
+	 */
+	tmp = dwc2_hsotg_read_frameno(hsotg);
+
 	dwc2_hsotg_complete_request(hsotg, ep, get_ep_head(ep), -ENODATA);
 
+	if (using_desc_dma(hsotg)) {
+		if (ep->target_frame == TARGET_FRAME_INITIAL) {
+			/* Start first ISO Out */
+			ep->target_frame = tmp;
+			dwc2_gadget_start_isoc_ddma(ep);
+		}
+		return;
+	}
+
 	if (ep->interval > 1 &&
 	    ep->target_frame == TARGET_FRAME_INITIAL) {
 		u32 dsts;
@@ -2182,6 +2774,12 @@ static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep)
 
 	if (hs_ep->target_frame == TARGET_FRAME_INITIAL) {
 		hs_ep->target_frame = dwc2_hsotg_read_frameno(hsotg);
+
+		if (using_desc_dma(hsotg)) {
+			dwc2_gadget_start_isoc_ddma(hs_ep);
+			return;
+		}
+
 		if (hs_ep->interval > 1) {
 			u32 ctrl = dwc2_readl(hsotg->regs +
 					      DIEPCTL(hs_ep->index));
@@ -2237,8 +2835,15 @@ static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx,
 	if (idx == 0 && (ints & (DXEPINT_SETUP | DXEPINT_SETUP_RCVD)))
 		ints &= ~DXEPINT_XFERCOMPL;
 
-	if (ints & DXEPINT_STSPHSERCVD)
-		dev_dbg(hsotg->dev, "%s: StsPhseRcvd asserted\n", __func__);
+	/*
+	 * Don't process XferCompl interrupt in DDMA if EP0 is still in SETUP
+	 * stage and xfercomplete was generated without SETUP phase done
+	 * interrupt. SW should parse received setup packet only after host's
+	 * exit from setup phase of control transfer.
+	 */
+	if (using_desc_dma(hsotg) && idx == 0 && !hs_ep->dir_in &&
+	    hsotg->ep0_state == DWC2_EP0_SETUP && !(ints & DXEPINT_SETUP))
+		ints &= ~DXEPINT_XFERCOMPL;
 
 	if (ints & DXEPINT_XFERCOMPL) {
 		dev_dbg(hsotg->dev,
@@ -2246,11 +2851,17 @@ static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx,
 			__func__, dwc2_readl(hsotg->regs + epctl_reg),
 			dwc2_readl(hsotg->regs + epsiz_reg));
 
-		/*
-		 * we get OutDone from the FIFO, so we only need to look
-		 * at completing IN requests here
-		 */
-		if (dir_in) {
+		/* In DDMA handle isochronous requests separately */
+		if (using_desc_dma(hsotg) && hs_ep->isochronous) {
+			dwc2_gadget_complete_isoc_request_ddma(hs_ep);
+			/* Try to start next isoc request */
+			dwc2_gadget_start_next_isoc_ddma(hs_ep);
+		} else if (dir_in) {
+			/*
+			 * We get OutDone from the FIFO, so we only
+			 * need to look at completing IN requests here
+			 * if operating slave mode
+			 */
 			if (hs_ep->isochronous && hs_ep->interval > 1)
 				dwc2_gadget_incr_frame_num(hs_ep);
 
@@ -2302,9 +2913,30 @@ static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx,
 		}
 	}
 
+	if (ints & DXEPINT_STSPHSERCVD) {
+		dev_dbg(hsotg->dev, "%s: StsPhseRcvd\n", __func__);
+
+		/* Move to STATUS IN for DDMA */
+		if (using_desc_dma(hsotg))
+			dwc2_hsotg_ep0_zlp(hsotg, true);
+	}
+
 	if (ints & DXEPINT_BACK2BACKSETUP)
 		dev_dbg(hsotg->dev, "%s: B2BSetup/INEPNakEff\n", __func__);
 
+	if (ints & DXEPINT_BNAINTR) {
+		dev_dbg(hsotg->dev, "%s: BNA interrupt\n", __func__);
+
+		/*
+		 * Try to start next isoc request, if any.
+		 * Sometimes the endpoint remains enabled after BNA interrupt
+		 * assertion, which is not expected, hence we can enter here
+		 * couple of times.
+		 */
+		if (hs_ep->isochronous)
+			dwc2_gadget_start_next_isoc_ddma(hs_ep);
+	}
+
 	if (dir_in && !hs_ep->isochronous) {
 		/* not sure if this is important, but we'll clear it anyway */
 		if (ints & DXEPINT_INTKNTXFEMP) {
@@ -2372,6 +3004,8 @@ static void dwc2_hsotg_irq_enumdone(struct dwc2_hsotg *hsotg)
 
 	case DSTS_ENUMSPD_LS:
 		hsotg->gadget.speed = USB_SPEED_LOW;
+		ep0_mps = 8;
+		ep_mps = 8;
 		/*
 		 * note, we don't actually support LS in this driver at the
 		 * moment, and the documentation seems to imply that it isn't
@@ -2390,13 +3024,15 @@ static void dwc2_hsotg_irq_enumdone(struct dwc2_hsotg *hsotg)
 	if (ep0_mps) {
 		int i;
 		/* Initialize ep0 for both in and out directions */
-		dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 1);
-		dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0);
+		dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 1);
+		dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 0);
 		for (i = 1; i < hsotg->num_of_eps; i++) {
 			if (hsotg->eps_in[i])
-				dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps, 1);
+				dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps,
+							    0, 1);
 			if (hsotg->eps_out[i])
-				dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps, 0);
+				dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps,
+							    0, 0);
 		}
 	}
 
@@ -2516,6 +3152,7 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg,
 	u32 intmsk;
 	u32 val;
 	u32 usbcfg;
+	u32 dcfg = 0;
 
 	/* Kill any ep0 requests as controller will be reinitialized */
 	kill_all_requests(hsotg, hsotg->eps_out[0], -ECONNRESET);
@@ -2534,10 +3171,17 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg,
 	usbcfg &= ~(GUSBCFG_TOUTCAL_MASK | GUSBCFG_PHYIF16 | GUSBCFG_SRPCAP |
 		GUSBCFG_HNPCAP);
 
-	/* set the PLL on, remove the HNP/SRP and set the PHY */
-	val = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5;
-	usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) |
-		(val << GUSBCFG_USBTRDTIM_SHIFT);
+	if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS &&
+	    (hsotg->params.speed == DWC2_SPEED_PARAM_FULL ||
+	     hsotg->params.speed == DWC2_SPEED_PARAM_LOW)) {
+		/* FS/LS Dedicated Transceiver Interface */
+		usbcfg |= GUSBCFG_PHYSEL;
+	} else {
+		/* set the PLL on, remove the HNP/SRP and set the PHY */
+		val = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5;
+		usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) |
+			(val << GUSBCFG_USBTRDTIM_SHIFT);
+	}
 	dwc2_writel(usbcfg, hsotg->regs + GUSBCFG);
 
 	dwc2_hsotg_init_fifo(hsotg);
@@ -2545,7 +3189,23 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg,
 	if (!is_usb_reset)
 		__orr32(hsotg->regs + DCTL, DCTL_SFTDISCON);
 
-	dwc2_writel(DCFG_EPMISCNT(1) | DCFG_DEVSPD_HS,  hsotg->regs + DCFG);
+	dcfg |= DCFG_EPMISCNT(1);
+
+	switch (hsotg->params.speed) {
+	case DWC2_SPEED_PARAM_LOW:
+		dcfg |= DCFG_DEVSPD_LS;
+		break;
+	case DWC2_SPEED_PARAM_FULL:
+		if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS)
+			dcfg |= DCFG_DEVSPD_FS48;
+		else
+			dcfg |= DCFG_DEVSPD_FS;
+		break;
+	default:
+		dcfg |= DCFG_DEVSPD_HS;
+	}
+
+	dwc2_writel(dcfg,  hsotg->regs + DCFG);
 
 	/* Clear any pending OTG interrupts */
 	dwc2_writel(0xffffffff, hsotg->regs + GOTGINT);
@@ -2556,23 +3216,31 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg,
 		GINTSTS_GOUTNAKEFF | GINTSTS_GINNAKEFF |
 		GINTSTS_USBRST | GINTSTS_RESETDET |
 		GINTSTS_ENUMDONE | GINTSTS_OTGINT |
-		GINTSTS_USBSUSP | GINTSTS_WKUPINT |
-		GINTSTS_INCOMPL_SOIN | GINTSTS_INCOMPL_SOOUT;
+		GINTSTS_USBSUSP | GINTSTS_WKUPINT;
 
-	if (hsotg->core_params->external_id_pin_ctl <= 0)
+	if (!using_desc_dma(hsotg))
+		intmsk |= GINTSTS_INCOMPL_SOIN | GINTSTS_INCOMPL_SOOUT;
+
+	if (hsotg->params.external_id_pin_ctl <= 0)
 		intmsk |= GINTSTS_CONIDSTSCHNG;
 
 	dwc2_writel(intmsk, hsotg->regs + GINTMSK);
 
-	if (using_dma(hsotg))
+	if (using_dma(hsotg)) {
 		dwc2_writel(GAHBCFG_GLBL_INTR_EN | GAHBCFG_DMA_EN |
 			    (GAHBCFG_HBSTLEN_INCR4 << GAHBCFG_HBSTLEN_SHIFT),
 			    hsotg->regs + GAHBCFG);
-	else
+
+		/* Set DDMA mode support in the core if needed */
+		if (using_desc_dma(hsotg))
+			__orr32(hsotg->regs + DCFG, DCFG_DESCDMA_EN);
+
+	} else {
 		dwc2_writel(((hsotg->dedicated_fifos) ?
 						(GAHBCFG_NP_TXF_EMP_LVL |
 						 GAHBCFG_P_TXF_EMP_LVL) : 0) |
 			    GAHBCFG_GLBL_INTR_EN, hsotg->regs + GAHBCFG);
+	}
 
 	/*
 	 * If INTknTXFEmpMsk is enabled, it's important to disable ep interrupts
@@ -2588,13 +3256,18 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg,
 
 	/*
 	 * don't need XferCompl, we get that from RXFIFO in slave mode. In
-	 * DMA mode we may need this.
+	 * DMA mode we may need this and StsPhseRcvd.
 	 */
-	dwc2_writel((using_dma(hsotg) ? (DIEPMSK_XFERCOMPLMSK) : 0) |
+	dwc2_writel((using_dma(hsotg) ? (DIEPMSK_XFERCOMPLMSK |
+		DOEPMSK_STSPHSERCVDMSK) : 0) |
 		DOEPMSK_EPDISBLDMSK | DOEPMSK_AHBERRMSK |
-		DOEPMSK_SETUPMSK | DOEPMSK_STSPHSERCVDMSK,
+		DOEPMSK_SETUPMSK,
 		hsotg->regs + DOEPMSK);
 
+	/* Enable BNA interrupt for DDMA */
+	if (using_desc_dma(hsotg))
+		__orr32(hsotg->regs + DOEPMSK, DOEPMSK_BNAMSK);
+
 	dwc2_writel(0, hsotg->regs + DAINTMSK);
 
 	dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
@@ -2935,6 +3608,95 @@ irq_retry:
 	return IRQ_HANDLED;
 }
 
+static int dwc2_hsotg_wait_bit_set(struct dwc2_hsotg *hs_otg, u32 reg,
+				   u32 bit, u32 timeout)
+{
+	u32 i;
+
+	for (i = 0; i < timeout; i++) {
+		if (dwc2_readl(hs_otg->regs + reg) & bit)
+			return 0;
+		udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg,
+				   struct dwc2_hsotg_ep *hs_ep)
+{
+	u32 epctrl_reg;
+	u32 epint_reg;
+
+	epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) :
+		DOEPCTL(hs_ep->index);
+	epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) :
+		DOEPINT(hs_ep->index);
+
+	dev_dbg(hsotg->dev, "%s: stopping transfer on %s\n", __func__,
+		hs_ep->name);
+
+	if (hs_ep->dir_in) {
+		if (hsotg->dedicated_fifos || hs_ep->periodic) {
+			__orr32(hsotg->regs + epctrl_reg, DXEPCTL_SNAK);
+			/* Wait for Nak effect */
+			if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg,
+						    DXEPINT_INEPNAKEFF, 100))
+				dev_warn(hsotg->dev,
+					 "%s: timeout DIEPINT.NAKEFF\n",
+					 __func__);
+		} else {
+			__orr32(hsotg->regs + DCTL, DCTL_SGNPINNAK);
+			/* Wait for Nak effect */
+			if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
+						    GINTSTS_GINNAKEFF, 100))
+				dev_warn(hsotg->dev,
+					 "%s: timeout GINTSTS.GINNAKEFF\n",
+					 __func__);
+		}
+	} else {
+		if (!(dwc2_readl(hsotg->regs + GINTSTS) & GINTSTS_GOUTNAKEFF))
+			__orr32(hsotg->regs + DCTL, DCTL_SGOUTNAK);
+
+		/* Wait for global nak to take effect */
+		if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
+					    GINTSTS_GOUTNAKEFF, 100))
+			dev_warn(hsotg->dev, "%s: timeout GINTSTS.GOUTNAKEFF\n",
+				 __func__);
+	}
+
+	/* Disable ep */
+	__orr32(hsotg->regs + epctrl_reg, DXEPCTL_EPDIS | DXEPCTL_SNAK);
+
+	/* Wait for ep to be disabled */
+	if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, DXEPINT_EPDISBLD, 100))
+		dev_warn(hsotg->dev,
+			 "%s: timeout DOEPCTL.EPDisable\n", __func__);
+
+	/* Clear EPDISBLD interrupt */
+	__orr32(hsotg->regs + epint_reg, DXEPINT_EPDISBLD);
+
+	if (hs_ep->dir_in) {
+		unsigned short fifo_index;
+
+		if (hsotg->dedicated_fifos || hs_ep->periodic)
+			fifo_index = hs_ep->fifo_index;
+		else
+			fifo_index = 0;
+
+		/* Flush TX FIFO */
+		dwc2_flush_tx_fifo(hsotg, fifo_index);
+
+		/* Clear Global In NP NAK in Shared FIFO for non periodic ep */
+		if (!hsotg->dedicated_fifos && !hs_ep->periodic)
+			__orr32(hsotg->regs + DCTL, DCTL_CGNPINNAK);
+
+	} else {
+		/* Remove global NAKs */
+		__orr32(hsotg->regs + DCTL, DCTL_CGOUTNAK);
+	}
+}
+
 /**
  * dwc2_hsotg_ep_enable - enable the given endpoint
  * @ep: The USB endpint to configure
@@ -2952,6 +3714,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
 	u32 epctrl_reg;
 	u32 epctrl;
 	u32 mps;
+	u32 mc;
 	u32 mask;
 	unsigned int dir_in;
 	unsigned int i, val, size;
@@ -2975,6 +3738,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
 	}
 
 	mps = usb_endpoint_maxp(desc);
+	mc = usb_endpoint_maxp_mult(desc);
 
 	/* note, we handle this here instead of dwc2_hsotg_set_ep_maxpacket */
 
@@ -2984,6 +3748,18 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
 	dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x from 0x%08x\n",
 		__func__, epctrl, epctrl_reg);
 
+	/* Allocate DMA descriptor chain for non-ctrl endpoints */
+	if (using_desc_dma(hsotg)) {
+		hs_ep->desc_list = dma_alloc_coherent(hsotg->dev,
+			MAX_DMA_DESC_NUM_GENERIC *
+			sizeof(struct dwc2_dma_desc),
+			&hs_ep->desc_list_dma, GFP_KERNEL);
+		if (!hs_ep->desc_list) {
+			ret = -ENOMEM;
+			goto error2;
+		}
+	}
+
 	spin_lock_irqsave(&hsotg->lock, flags);
 
 	epctrl &= ~(DXEPCTL_EPTYPE_MASK | DXEPCTL_MPS_MASK);
@@ -2996,7 +3772,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
 	epctrl |= DXEPCTL_USBACTEP;
 
 	/* update the endpoint state */
-	dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, dir_in);
+	dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, mc, dir_in);
 
 	/* default, set to non-periodic */
 	hs_ep->isochronous = 0;
@@ -3011,6 +3787,8 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
 		hs_ep->isochronous = 1;
 		hs_ep->interval = 1 << (desc->bInterval - 1);
 		hs_ep->target_frame = TARGET_FRAME_INITIAL;
+		hs_ep->isoc_chain_num = 0;
+		hs_ep->next_desc = 0;
 		if (dir_in) {
 			hs_ep->periodic = 1;
 			mask = dwc2_readl(hsotg->regs + DIEPMSK);
@@ -3067,7 +3845,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
 			dev_err(hsotg->dev,
 				"%s: No suitable fifo found\n", __func__);
 			ret = -ENOMEM;
-			goto error;
+			goto error1;
 		}
 		hsotg->fifo_map |= 1 << fifo_index;
 		epctrl |= DXEPCTL_TXFNUM(fifo_index);
@@ -3089,8 +3867,17 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
 	/* enable the endpoint interrupt */
 	dwc2_hsotg_ctrl_epint(hsotg, index, dir_in, 1);
 
-error:
+error1:
 	spin_unlock_irqrestore(&hsotg->lock, flags);
+
+error2:
+	if (ret && using_desc_dma(hsotg) && hs_ep->desc_list) {
+		dma_free_coherent(hsotg->dev, MAX_DMA_DESC_NUM_GENERIC *
+			sizeof(struct dwc2_dma_desc),
+			hs_ep->desc_list, hs_ep->desc_list_dma);
+		hs_ep->desc_list = NULL;
+	}
+
 	return ret;
 }
 
@@ -3115,11 +3902,23 @@ static int dwc2_hsotg_ep_disable(struct usb_ep *ep)
 		return -EINVAL;
 	}
 
+	/* Remove DMA memory allocated for non-control Endpoints */
+	if (using_desc_dma(hsotg)) {
+		dma_free_coherent(hsotg->dev, MAX_DMA_DESC_NUM_GENERIC *
+				  sizeof(struct dwc2_dma_desc),
+				  hs_ep->desc_list, hs_ep->desc_list_dma);
+		hs_ep->desc_list = NULL;
+	}
+
 	epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
 
 	spin_lock_irqsave(&hsotg->lock, flags);
 
 	ctrl = dwc2_readl(hsotg->regs + epctrl_reg);
+
+	if (ctrl & DXEPCTL_EPENA)
+		dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep);
+
 	ctrl &= ~DXEPCTL_EPENA;
 	ctrl &= ~DXEPCTL_USBACTEP;
 	ctrl |= DXEPCTL_SNAK;
@@ -3158,77 +3957,6 @@ static bool on_list(struct dwc2_hsotg_ep *ep, struct dwc2_hsotg_req *test)
 	return false;
 }
 
-static int dwc2_hsotg_wait_bit_set(struct dwc2_hsotg *hs_otg, u32 reg,
-							u32 bit, u32 timeout)
-{
-	u32 i;
-
-	for (i = 0; i < timeout; i++) {
-		if (dwc2_readl(hs_otg->regs + reg) & bit)
-			return 0;
-		udelay(1);
-	}
-
-	return -ETIMEDOUT;
-}
-
-static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg,
-						struct dwc2_hsotg_ep *hs_ep)
-{
-	u32 epctrl_reg;
-	u32 epint_reg;
-
-	epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) :
-		DOEPCTL(hs_ep->index);
-	epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) :
-		DOEPINT(hs_ep->index);
-
-	dev_dbg(hsotg->dev, "%s: stopping transfer on %s\n", __func__,
-			hs_ep->name);
-	if (hs_ep->dir_in) {
-		__orr32(hsotg->regs + epctrl_reg, DXEPCTL_SNAK);
-		/* Wait for Nak effect */
-		if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg,
-						DXEPINT_INEPNAKEFF, 100))
-			dev_warn(hsotg->dev,
-				"%s: timeout DIEPINT.NAKEFF\n", __func__);
-	} else {
-		if (!(dwc2_readl(hsotg->regs + GINTSTS) & GINTSTS_GOUTNAKEFF))
-			__orr32(hsotg->regs + DCTL, DCTL_SGOUTNAK);
-
-		/* Wait for global nak to take effect */
-		if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
-						GINTSTS_GOUTNAKEFF, 100))
-			dev_warn(hsotg->dev,
-				"%s: timeout GINTSTS.GOUTNAKEFF\n", __func__);
-	}
-
-	/* Disable ep */
-	__orr32(hsotg->regs + epctrl_reg, DXEPCTL_EPDIS | DXEPCTL_SNAK);
-
-	/* Wait for ep to be disabled */
-	if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, DXEPINT_EPDISBLD, 100))
-		dev_warn(hsotg->dev,
-			"%s: timeout DOEPCTL.EPDisable\n", __func__);
-
-	if (hs_ep->dir_in) {
-		if (hsotg->dedicated_fifos) {
-			dwc2_writel(GRSTCTL_TXFNUM(hs_ep->fifo_index) |
-				GRSTCTL_TXFFLSH, hsotg->regs + GRSTCTL);
-			/* Wait for fifo flush */
-			if (dwc2_hsotg_wait_bit_set(hsotg, GRSTCTL,
-							GRSTCTL_TXFFLSH, 100))
-				dev_warn(hsotg->dev,
-					"%s: timeout flushing fifos\n",
-					__func__);
-		}
-		/* TODO: Flush shared tx fifo */
-	} else {
-		/* Remove global NAKs */
-		__bic32(hsotg->regs + DCTL, DCTL_SGOUTNAK);
-	}
-}
-
 /**
  * dwc2_hsotg_ep_dequeue - dequeue given endpoint
  * @ep: The endpoint to dequeue.
@@ -3665,14 +4393,21 @@ static void dwc2_hsotg_initep(struct dwc2_hsotg *hsotg,
 
 	hs_ep->parent = hsotg;
 	hs_ep->ep.name = hs_ep->name;
-	usb_ep_set_maxpacket_limit(&hs_ep->ep, epnum ? 1024 : EP0_MPS_LIMIT);
+
+	if (hsotg->params.speed == DWC2_SPEED_PARAM_LOW)
+		usb_ep_set_maxpacket_limit(&hs_ep->ep, 8);
+	else
+		usb_ep_set_maxpacket_limit(&hs_ep->ep,
+					   epnum ? 1024 : EP0_MPS_LIMIT);
 	hs_ep->ep.ops = &dwc2_hsotg_ep_ops;
 
 	if (epnum == 0) {
 		hs_ep->ep.caps.type_control = true;
 	} else {
-		hs_ep->ep.caps.type_iso = true;
-		hs_ep->ep.caps.type_bulk = true;
+		if (hsotg->params.speed != DWC2_SPEED_PARAM_LOW) {
+			hs_ep->ep.caps.type_iso = true;
+			hs_ep->ep.caps.type_bulk = true;
+		}
 		hs_ep->ep.caps.type_int = true;
 	}
 
@@ -3802,51 +4537,6 @@ static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg)
 #endif
 }
 
-#ifdef CONFIG_OF
-static void dwc2_hsotg_of_probe(struct dwc2_hsotg *hsotg)
-{
-	struct device_node *np = hsotg->dev->of_node;
-	u32 len = 0;
-	u32 i = 0;
-
-	/* Enable dma if requested in device tree */
-	hsotg->g_using_dma = of_property_read_bool(np, "g-use-dma");
-
-	/*
-	* Register TX periodic fifo size per endpoint.
-	* EP0 is excluded since it has no fifo configuration.
-	*/
-	if (!of_find_property(np, "g-tx-fifo-size", &len))
-		goto rx_fifo;
-
-	len /= sizeof(u32);
-
-	/* Read tx fifo sizes other than ep0 */
-	if (of_property_read_u32_array(np, "g-tx-fifo-size",
-						&hsotg->g_tx_fifo_sz[1], len))
-		goto rx_fifo;
-
-	/* Add ep0 */
-	len++;
-
-	/* Make remaining TX fifos unavailable */
-	if (len < MAX_EPS_CHANNELS) {
-		for (i = len; i < MAX_EPS_CHANNELS; i++)
-			hsotg->g_tx_fifo_sz[i] = 0;
-	}
-
-rx_fifo:
-	/* Register RX fifo size */
-	of_property_read_u32(np, "g-rx-fifo-size", &hsotg->g_rx_fifo_sz);
-
-	/* Register NPTX fifo size */
-	of_property_read_u32(np, "g-np-tx-fifo-size",
-						&hsotg->g_np_g_tx_fifo_sz);
-}
-#else
-static inline void dwc2_hsotg_of_probe(struct dwc2_hsotg *hsotg) { }
-#endif
-
 /**
  * dwc2_gadget_init - init function for gadget
  * @dwc2: The data structure for the DWC2 driver.
@@ -3857,33 +4547,11 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq)
 	struct device *dev = hsotg->dev;
 	int epnum;
 	int ret;
-	int i;
-	u32 p_tx_fifo[] = DWC2_G_P_LEGACY_TX_FIFO_SIZE;
-
-	/* Initialize to legacy fifo configuration values */
-	hsotg->g_rx_fifo_sz = 2048;
-	hsotg->g_np_g_tx_fifo_sz = 1024;
-	memcpy(&hsotg->g_tx_fifo_sz[1], p_tx_fifo, sizeof(p_tx_fifo));
-	/* Device tree specific probe */
-	dwc2_hsotg_of_probe(hsotg);
-
-	/* Check against largest possible value. */
-	if (hsotg->g_np_g_tx_fifo_sz >
-	    hsotg->hw_params.dev_nperio_tx_fifo_size) {
-		dev_warn(dev, "Specified GNPTXFDEP=%d > %d\n",
-			 hsotg->g_np_g_tx_fifo_sz,
-			 hsotg->hw_params.dev_nperio_tx_fifo_size);
-		hsotg->g_np_g_tx_fifo_sz =
-			hsotg->hw_params.dev_nperio_tx_fifo_size;
-	}
 
 	/* Dump fifo information */
 	dev_dbg(dev, "NonPeriodic TXFIFO size: %d\n",
-						hsotg->g_np_g_tx_fifo_sz);
-	dev_dbg(dev, "RXFIFO size: %d\n", hsotg->g_rx_fifo_sz);
-	for (i = 0; i < MAX_EPS_CHANNELS; i++)
-		dev_dbg(dev, "Periodic TXFIFO%2d size: %d\n", i,
-						hsotg->g_tx_fifo_sz[i]);
+		hsotg->params.g_np_tx_fifo_size);
+	dev_dbg(dev, "RXFIFO size: %d\n", hsotg->params.g_rx_fifo_size);
 
 	hsotg->gadget.max_speed = USB_SPEED_HIGH;
 	hsotg->gadget.ops = &dwc2_hsotg_gadget_ops;
@@ -3909,6 +4577,12 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq)
 	if (!hsotg->ep0_buff)
 		return -ENOMEM;
 
+	if (using_desc_dma(hsotg)) {
+		ret = dwc2_gadget_alloc_ctrl_desc_chains(hsotg);
+		if (ret < 0)
+			return ret;
+	}
+
 	ret = devm_request_irq(hsotg->dev, irq, dwc2_hsotg_irq, IRQF_SHARED,
 				dev_name(hsotg->dev), hsotg);
 	if (ret < 0) {
diff --git a/drivers/usb/dwc2/hcd.c b/drivers/usb/dwc2/hcd.c
index df5a06578005..911c3b36ac06 100644
--- a/drivers/usb/dwc2/hcd.c
+++ b/drivers/usb/dwc2/hcd.c
@@ -79,9 +79,9 @@ static void dwc2_enable_common_interrupts(struct dwc2_hsotg *hsotg)
 	/* Enable the interrupts in the GINTMSK */
 	intmsk = GINTSTS_MODEMIS | GINTSTS_OTGINT;
 
-	if (hsotg->core_params->dma_enable <= 0)
+	if (hsotg->params.host_dma <= 0)
 		intmsk |= GINTSTS_RXFLVL;
-	if (hsotg->core_params->external_id_pin_ctl <= 0)
+	if (hsotg->params.external_id_pin_ctl <= 0)
 		intmsk |= GINTSTS_CONIDSTSCHNG;
 
 	intmsk |= GINTSTS_WKUPINT | GINTSTS_USBSUSP |
@@ -100,8 +100,8 @@ static void dwc2_init_fs_ls_pclk_sel(struct dwc2_hsotg *hsotg)
 
 	if ((hsotg->hw_params.hs_phy_type == GHWCFG2_HS_PHY_TYPE_ULPI &&
 	     hsotg->hw_params.fs_phy_type == GHWCFG2_FS_PHY_TYPE_DEDICATED &&
-	     hsotg->core_params->ulpi_fs_ls > 0) ||
-	    hsotg->core_params->phy_type == DWC2_PHY_TYPE_PARAM_FS) {
+	     hsotg->params.ulpi_fs_ls > 0) ||
+	    hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS) {
 		/* Full speed PHY */
 		val = HCFG_FSLSPCLKSEL_48_MHZ;
 	} else {
@@ -152,7 +152,7 @@ static int dwc2_fs_phy_init(struct dwc2_hsotg *hsotg, bool select_phy)
 	if (dwc2_is_host_mode(hsotg))
 		dwc2_init_fs_ls_pclk_sel(hsotg);
 
-	if (hsotg->core_params->i2c_enable > 0) {
+	if (hsotg->params.i2c_enable > 0) {
 		dev_dbg(hsotg->dev, "FS PHY enabling I2C\n");
 
 		/* Program GUSBCFG.OtgUtmiFsSel to I2C */
@@ -189,20 +189,20 @@ static int dwc2_hs_phy_init(struct dwc2_hsotg *hsotg, bool select_phy)
 	 * so only program the first time. Do a soft reset immediately after
 	 * setting phyif.
 	 */
-	switch (hsotg->core_params->phy_type) {
+	switch (hsotg->params.phy_type) {
 	case DWC2_PHY_TYPE_PARAM_ULPI:
 		/* ULPI interface */
 		dev_dbg(hsotg->dev, "HS ULPI PHY selected\n");
 		usbcfg |= GUSBCFG_ULPI_UTMI_SEL;
 		usbcfg &= ~(GUSBCFG_PHYIF16 | GUSBCFG_DDRSEL);
-		if (hsotg->core_params->phy_ulpi_ddr > 0)
+		if (hsotg->params.phy_ulpi_ddr > 0)
 			usbcfg |= GUSBCFG_DDRSEL;
 		break;
 	case DWC2_PHY_TYPE_PARAM_UTMI:
 		/* UTMI+ interface */
 		dev_dbg(hsotg->dev, "HS UTMI+ PHY selected\n");
 		usbcfg &= ~(GUSBCFG_ULPI_UTMI_SEL | GUSBCFG_PHYIF16);
-		if (hsotg->core_params->phy_utmi_width == 16)
+		if (hsotg->params.phy_utmi_width == 16)
 			usbcfg |= GUSBCFG_PHYIF16;
 		break;
 	default:
@@ -230,9 +230,10 @@ static int dwc2_phy_init(struct dwc2_hsotg *hsotg, bool select_phy)
 	u32 usbcfg;
 	int retval = 0;
 
-	if (hsotg->core_params->speed == DWC2_SPEED_PARAM_FULL &&
-	    hsotg->core_params->phy_type == DWC2_PHY_TYPE_PARAM_FS) {
-		/* If FS mode with FS PHY */
+	if ((hsotg->params.speed == DWC2_SPEED_PARAM_FULL ||
+	     hsotg->params.speed == DWC2_SPEED_PARAM_LOW) &&
+	    hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS) {
+		/* If FS/LS mode with FS/LS PHY */
 		retval = dwc2_fs_phy_init(hsotg, select_phy);
 		if (retval)
 			return retval;
@@ -245,7 +246,7 @@ static int dwc2_phy_init(struct dwc2_hsotg *hsotg, bool select_phy)
 
 	if (hsotg->hw_params.hs_phy_type == GHWCFG2_HS_PHY_TYPE_ULPI &&
 	    hsotg->hw_params.fs_phy_type == GHWCFG2_FS_PHY_TYPE_DEDICATED &&
-	    hsotg->core_params->ulpi_fs_ls > 0) {
+	    hsotg->params.ulpi_fs_ls > 0) {
 		dev_dbg(hsotg->dev, "Setting ULPI FSLS\n");
 		usbcfg = dwc2_readl(hsotg->regs + GUSBCFG);
 		usbcfg |= GUSBCFG_ULPI_FS_LS;
@@ -272,9 +273,9 @@ static int dwc2_gahbcfg_init(struct dwc2_hsotg *hsotg)
 
 	case GHWCFG2_INT_DMA_ARCH:
 		dev_dbg(hsotg->dev, "Internal DMA Mode\n");
-		if (hsotg->core_params->ahbcfg != -1) {
+		if (hsotg->params.ahbcfg != -1) {
 			ahbcfg &= GAHBCFG_CTRL_MASK;
-			ahbcfg |= hsotg->core_params->ahbcfg &
+			ahbcfg |= hsotg->params.ahbcfg &
 				  ~GAHBCFG_CTRL_MASK;
 		}
 		break;
@@ -285,21 +286,21 @@ static int dwc2_gahbcfg_init(struct dwc2_hsotg *hsotg)
 		break;
 	}
 
-	dev_dbg(hsotg->dev, "dma_enable:%d dma_desc_enable:%d\n",
-		hsotg->core_params->dma_enable,
-		hsotg->core_params->dma_desc_enable);
+	dev_dbg(hsotg->dev, "host_dma:%d dma_desc_enable:%d\n",
+		hsotg->params.host_dma,
+		hsotg->params.dma_desc_enable);
 
-	if (hsotg->core_params->dma_enable > 0) {
-		if (hsotg->core_params->dma_desc_enable > 0)
+	if (hsotg->params.host_dma > 0) {
+		if (hsotg->params.dma_desc_enable > 0)
 			dev_dbg(hsotg->dev, "Using Descriptor DMA mode\n");
 		else
 			dev_dbg(hsotg->dev, "Using Buffer DMA mode\n");
 	} else {
 		dev_dbg(hsotg->dev, "Using Slave mode\n");
-		hsotg->core_params->dma_desc_enable = 0;
+		hsotg->params.dma_desc_enable = 0;
 	}
 
-	if (hsotg->core_params->dma_enable > 0)
+	if (hsotg->params.host_dma > 0)
 		ahbcfg |= GAHBCFG_DMA_EN;
 
 	dwc2_writel(ahbcfg, hsotg->regs + GAHBCFG);
@@ -316,10 +317,10 @@ static void dwc2_gusbcfg_init(struct dwc2_hsotg *hsotg)
 
 	switch (hsotg->hw_params.op_mode) {
 	case GHWCFG2_OP_MODE_HNP_SRP_CAPABLE:
-		if (hsotg->core_params->otg_cap ==
+		if (hsotg->params.otg_cap ==
 				DWC2_CAP_PARAM_HNP_SRP_CAPABLE)
 			usbcfg |= GUSBCFG_HNPCAP;
-		if (hsotg->core_params->otg_cap !=
+		if (hsotg->params.otg_cap !=
 				DWC2_CAP_PARAM_NO_HNP_SRP_CAPABLE)
 			usbcfg |= GUSBCFG_SRPCAP;
 		break;
@@ -327,7 +328,7 @@ static void dwc2_gusbcfg_init(struct dwc2_hsotg *hsotg)
 	case GHWCFG2_OP_MODE_SRP_ONLY_CAPABLE:
 	case GHWCFG2_OP_MODE_SRP_CAPABLE_DEVICE:
 	case GHWCFG2_OP_MODE_SRP_CAPABLE_HOST:
-		if (hsotg->core_params->otg_cap !=
+		if (hsotg->params.otg_cap !=
 				DWC2_CAP_PARAM_NO_HNP_SRP_CAPABLE)
 			usbcfg |= GUSBCFG_SRPCAP;
 		break;
@@ -390,7 +391,7 @@ static void dwc2_disable_host_interrupts(struct dwc2_hsotg *hsotg)
  */
 static void dwc2_calculate_dynamic_fifo(struct dwc2_hsotg *hsotg)
 {
-	struct dwc2_core_params *params = hsotg->core_params;
+	struct dwc2_core_params *params = &hsotg->params;
 	struct dwc2_hw_params *hw = &hsotg->hw_params;
 	u32 rxfsiz, nptxfsiz, ptxfsiz, total_fifo_size;
 
@@ -449,7 +450,7 @@ static void dwc2_calculate_dynamic_fifo(struct dwc2_hsotg *hsotg)
 
 static void dwc2_config_fifos(struct dwc2_hsotg *hsotg)
 {
-	struct dwc2_core_params *params = hsotg->core_params;
+	struct dwc2_core_params *params = &hsotg->params;
 	u32 nptxfsiz, hptxfsiz, dfifocfg, grxfsiz;
 
 	if (!params->enable_dynamic_fifo)
@@ -490,7 +491,7 @@ static void dwc2_config_fifos(struct dwc2_hsotg *hsotg)
 	dev_dbg(hsotg->dev, "new hptxfsiz=%08x\n",
 		dwc2_readl(hsotg->regs + HPTXFSIZ));
 
-	if (hsotg->core_params->en_multiple_tx_fifo > 0 &&
+	if (hsotg->params.en_multiple_tx_fifo > 0 &&
 	    hsotg->hw_params.snpsid <= DWC2_CORE_REV_2_94a) {
 		/*
 		 * Global DFIFOCFG calculation for Host mode -
@@ -598,7 +599,7 @@ static void dwc2_dump_channel_info(struct dwc2_hsotg *hsotg,
 				   struct dwc2_host_chan *chan)
 {
 #ifdef VERBOSE_DEBUG
-	int num_channels = hsotg->core_params->host_channels;
+	int num_channels = hsotg->params.host_channels;
 	struct dwc2_qh *qh;
 	u32 hcchar;
 	u32 hcsplt;
@@ -648,6 +649,35 @@ static void dwc2_dump_channel_info(struct dwc2_hsotg *hsotg,
 #endif /* VERBOSE_DEBUG */
 }
 
+static int _dwc2_hcd_start(struct usb_hcd *hcd);
+
+static void dwc2_host_start(struct dwc2_hsotg *hsotg)
+{
+	struct usb_hcd *hcd = dwc2_hsotg_to_hcd(hsotg);
+
+	hcd->self.is_b_host = dwc2_hcd_is_b_host(hsotg);
+	_dwc2_hcd_start(hcd);
+}
+
+static void dwc2_host_disconnect(struct dwc2_hsotg *hsotg)
+{
+	struct usb_hcd *hcd = dwc2_hsotg_to_hcd(hsotg);
+
+	hcd->self.is_b_host = 0;
+}
+
+static void dwc2_host_hub_info(struct dwc2_hsotg *hsotg, void *context,
+			       int *hub_addr, int *hub_port)
+{
+	struct urb *urb = context;
+
+	if (urb->dev->tt)
+		*hub_addr = urb->dev->tt->hub->devnum;
+	else
+		*hub_addr = 0;
+	*hub_port = urb->dev->ttport;
+}
+
 /*
  * =========================================================================
  *  Low Level Host Channel Access Functions
@@ -741,7 +771,7 @@ static void dwc2_hc_enable_dma_ints(struct dwc2_hsotg *hsotg,
 	 * For Descriptor DMA mode core halts the channel on AHB error.
 	 * Interrupt is not required.
 	 */
-	if (hsotg->core_params->dma_desc_enable <= 0) {
+	if (hsotg->params.dma_desc_enable <= 0) {
 		if (dbg_hc(chan))
 			dev_vdbg(hsotg->dev, "desc DMA disabled\n");
 		hcintmsk |= HCINTMSK_AHBERR;
@@ -774,7 +804,7 @@ static void dwc2_hc_enable_ints(struct dwc2_hsotg *hsotg,
 {
 	u32 intmsk;
 
-	if (hsotg->core_params->dma_enable > 0) {
+	if (hsotg->params.host_dma > 0) {
 		if (dbg_hc(chan))
 			dev_vdbg(hsotg->dev, "DMA enabled\n");
 		dwc2_hc_enable_dma_ints(hsotg, chan);
@@ -994,7 +1024,7 @@ void dwc2_hc_halt(struct dwc2_hsotg *hsotg, struct dwc2_host_chan *chan,
 
 	/* No need to set the bit in DDMA for disabling the channel */
 	/* TODO check it everywhere channel is disabled */
-	if (hsotg->core_params->dma_desc_enable <= 0) {
+	if (hsotg->params.dma_desc_enable <= 0) {
 		if (dbg_hc(chan))
 			dev_vdbg(hsotg->dev, "desc DMA disabled\n");
 		hcchar |= HCCHAR_CHENA;
@@ -1004,7 +1034,7 @@ void dwc2_hc_halt(struct dwc2_hsotg *hsotg, struct dwc2_host_chan *chan,
 	}
 	hcchar |= HCCHAR_CHDIS;
 
-	if (hsotg->core_params->dma_enable <= 0) {
+	if (hsotg->params.host_dma <= 0) {
 		if (dbg_hc(chan))
 			dev_vdbg(hsotg->dev, "DMA not enabled\n");
 		hcchar |= HCCHAR_CHENA;
@@ -1143,7 +1173,7 @@ static void dwc2_hc_set_even_odd_frame(struct dwc2_hsotg *hsotg,
 		fifo_space = (dwc2_readl(hsotg->regs + HPTXSTS) &
 			      TXSTS_FSPCAVAIL_MASK) >> TXSTS_FSPCAVAIL_SHIFT;
 		bytes_in_fifo = sizeof(u32) *
-				(hsotg->core_params->host_perio_tx_fifo_size -
+				(hsotg->params.host_perio_tx_fifo_size -
 				 fifo_space);
 
 		/*
@@ -1339,8 +1369,8 @@ static void dwc2_hc_do_ping(struct dwc2_hsotg *hsotg,
 static void dwc2_hc_start_transfer(struct dwc2_hsotg *hsotg,
 				   struct dwc2_host_chan *chan)
 {
-	u32 max_hc_xfer_size = hsotg->core_params->max_transfer_size;
-	u16 max_hc_pkt_count = hsotg->core_params->max_packet_count;
+	u32 max_hc_xfer_size = hsotg->params.max_transfer_size;
+	u16 max_hc_pkt_count = hsotg->params.max_packet_count;
 	u32 hcchar;
 	u32 hctsiz = 0;
 	u16 num_packets;
@@ -1350,7 +1380,7 @@ static void dwc2_hc_start_transfer(struct dwc2_hsotg *hsotg,
 		dev_vdbg(hsotg->dev, "%s()\n", __func__);
 
 	if (chan->do_ping) {
-		if (hsotg->core_params->dma_enable <= 0) {
+		if (hsotg->params.host_dma <= 0) {
 			if (dbg_hc(chan))
 				dev_vdbg(hsotg->dev, "ping, no DMA\n");
 			dwc2_hc_do_ping(hsotg, chan);
@@ -1478,7 +1508,7 @@ static void dwc2_hc_start_transfer(struct dwc2_hsotg *hsotg,
 			 TSIZ_SC_MC_PID_SHIFT);
 	}
 
-	if (hsotg->core_params->dma_enable > 0) {
+	if (hsotg->params.host_dma > 0) {
 		dwc2_writel((u32)chan->xfer_dma,
 			    hsotg->regs + HCDMA(chan->hc_num));
 		if (dbg_hc(chan))
@@ -1521,7 +1551,7 @@ static void dwc2_hc_start_transfer(struct dwc2_hsotg *hsotg,
 	chan->xfer_started = 1;
 	chan->requests++;
 
-	if (hsotg->core_params->dma_enable <= 0 &&
+	if (hsotg->params.host_dma <= 0 &&
 	    !chan->ep_is_in && chan->xfer_len > 0)
 		/* Load OUT packet into the appropriate Tx FIFO */
 		dwc2_hc_write_packet(hsotg, chan);
@@ -1799,12 +1829,12 @@ void dwc2_hcd_start(struct dwc2_hsotg *hsotg)
 /* Must be called with interrupt disabled and spinlock held */
 static void dwc2_hcd_cleanup_channels(struct dwc2_hsotg *hsotg)
 {
-	int num_channels = hsotg->core_params->host_channels;
+	int num_channels = hsotg->params.host_channels;
 	struct dwc2_host_chan *channel;
 	u32 hcchar;
 	int i;
 
-	if (hsotg->core_params->dma_enable <= 0) {
+	if (hsotg->params.host_dma <= 0) {
 		/* Flush out any channel requests in slave mode */
 		for (i = 0; i < num_channels; i++) {
 			channel = hsotg->hc_ptr_array[i];
@@ -1840,9 +1870,9 @@ static void dwc2_hcd_cleanup_channels(struct dwc2_hsotg *hsotg)
 		channel->qh = NULL;
 	}
 	/* All channels have been freed, mark them available */
-	if (hsotg->core_params->uframe_sched > 0) {
+	if (hsotg->params.uframe_sched > 0) {
 		hsotg->available_host_channels =
-			hsotg->core_params->host_channels;
+			hsotg->params.host_channels;
 	} else {
 		hsotg->non_periodic_channels = 0;
 		hsotg->periodic_channels = 0;
@@ -2077,7 +2107,7 @@ static int dwc2_hcd_urb_dequeue(struct dwc2_hsotg *hsotg,
 	 * Free the QTD and clean up the associated QH. Leave the QH in the
 	 * schedule if it has any remaining QTDs.
 	 */
-	if (hsotg->core_params->dma_desc_enable <= 0) {
+	if (hsotg->params.dma_desc_enable <= 0) {
 		u8 in_process = urb_qtd->in_process;
 
 		dwc2_hcd_qtd_unlink_and_free(hsotg, urb_qtd, qh);
@@ -2185,13 +2215,13 @@ static int dwc2_core_init(struct dwc2_hsotg *hsotg, bool initial_setup)
 
 	/* Set ULPI External VBUS bit if needed */
 	usbcfg &= ~GUSBCFG_ULPI_EXT_VBUS_DRV;
-	if (hsotg->core_params->phy_ulpi_ext_vbus ==
+	if (hsotg->params.phy_ulpi_ext_vbus ==
 				DWC2_PHY_ULPI_EXTERNAL_VBUS)
 		usbcfg |= GUSBCFG_ULPI_EXT_VBUS_DRV;
 
 	/* Set external TS Dline pulsing bit if needed */
 	usbcfg &= ~GUSBCFG_TERMSELDLPULSE;
-	if (hsotg->core_params->ts_dline > 0)
+	if (hsotg->params.ts_dline > 0)
 		usbcfg |= GUSBCFG_TERMSELDLPULSE;
 
 	dwc2_writel(usbcfg, hsotg->regs + GUSBCFG);
@@ -2230,10 +2260,10 @@ static int dwc2_core_init(struct dwc2_hsotg *hsotg, bool initial_setup)
 	/* Program the GOTGCTL register */
 	otgctl = dwc2_readl(hsotg->regs + GOTGCTL);
 	otgctl &= ~GOTGCTL_OTGVER;
-	if (hsotg->core_params->otg_ver > 0)
+	if (hsotg->params.otg_ver > 0)
 		otgctl |= GOTGCTL_OTGVER;
 	dwc2_writel(otgctl, hsotg->regs + GOTGCTL);
-	dev_dbg(hsotg->dev, "OTG VER PARAM: %d\n", hsotg->core_params->otg_ver);
+	dev_dbg(hsotg->dev, "OTG VER PARAM: %d\n", hsotg->params.otg_ver);
 
 	/* Clear the SRP success bit for FS-I2c */
 	hsotg->srp_success = 0;
@@ -2277,7 +2307,8 @@ static void dwc2_core_host_init(struct dwc2_hsotg *hsotg)
 
 	/* Initialize Host Configuration Register */
 	dwc2_init_fs_ls_pclk_sel(hsotg);
-	if (hsotg->core_params->speed == DWC2_SPEED_PARAM_FULL) {
+	if (hsotg->params.speed == DWC2_SPEED_PARAM_FULL ||
+	    hsotg->params.speed == DWC2_SPEED_PARAM_LOW) {
 		hcfg = dwc2_readl(hsotg->regs + HCFG);
 		hcfg |= HCFG_FSLSSUPP;
 		dwc2_writel(hcfg, hsotg->regs + HCFG);
@@ -2288,13 +2319,13 @@ static void dwc2_core_host_init(struct dwc2_hsotg *hsotg)
 	 * runtime. This bit needs to be programmed during initial configuration
 	 * and its value must not be changed during runtime.
 	 */
-	if (hsotg->core_params->reload_ctl > 0) {
+	if (hsotg->params.reload_ctl > 0) {
 		hfir = dwc2_readl(hsotg->regs + HFIR);
 		hfir |= HFIR_RLDCTRL;
 		dwc2_writel(hfir, hsotg->regs + HFIR);
 	}
 
-	if (hsotg->core_params->dma_desc_enable > 0) {
+	if (hsotg->params.dma_desc_enable > 0) {
 		u32 op_mode = hsotg->hw_params.op_mode;
 
 		if (hsotg->hw_params.snpsid < DWC2_CORE_REV_2_90a ||
@@ -2306,7 +2337,7 @@ static void dwc2_core_host_init(struct dwc2_hsotg *hsotg)
 				"Hardware does not support descriptor DMA mode -\n");
 			dev_err(hsotg->dev,
 				"falling back to buffer DMA mode.\n");
-			hsotg->core_params->dma_desc_enable = 0;
+			hsotg->params.dma_desc_enable = 0;
 		} else {
 			hcfg = dwc2_readl(hsotg->regs + HCFG);
 			hcfg |= HCFG_DESCDMA;
@@ -2332,12 +2363,12 @@ static void dwc2_core_host_init(struct dwc2_hsotg *hsotg)
 	otgctl &= ~GOTGCTL_HSTSETHNPEN;
 	dwc2_writel(otgctl, hsotg->regs + GOTGCTL);
 
-	if (hsotg->core_params->dma_desc_enable <= 0) {
+	if (hsotg->params.dma_desc_enable <= 0) {
 		int num_channels, i;
 		u32 hcchar;
 
 		/* Flush out any leftover queued requests */
-		num_channels = hsotg->core_params->host_channels;
+		num_channels = hsotg->params.host_channels;
 		for (i = 0; i < num_channels; i++) {
 			hcchar = dwc2_readl(hsotg->regs + HCCHAR(i));
 			hcchar &= ~HCCHAR_CHENA;
@@ -2399,9 +2430,9 @@ static void dwc2_hcd_reinit(struct dwc2_hsotg *hsotg)
 	hsotg->flags.d32 = 0;
 	hsotg->non_periodic_qh_ptr = &hsotg->non_periodic_sched_active;
 
-	if (hsotg->core_params->uframe_sched > 0) {
+	if (hsotg->params.uframe_sched > 0) {
 		hsotg->available_host_channels =
-			hsotg->core_params->host_channels;
+			hsotg->params.host_channels;
 	} else {
 		hsotg->non_periodic_channels = 0;
 		hsotg->periodic_channels = 0;
@@ -2415,7 +2446,7 @@ static void dwc2_hcd_reinit(struct dwc2_hsotg *hsotg)
 				 hc_list_entry)
 		list_del_init(&chan->hc_list_entry);
 
-	num_channels = hsotg->core_params->host_channels;
+	num_channels = hsotg->params.host_channels;
 	for (i = 0; i < num_channels; i++) {
 		chan = hsotg->hc_ptr_array[i];
 		list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);
@@ -2457,7 +2488,7 @@ static void dwc2_hc_init_xfer(struct dwc2_hsotg *hsotg,
 			chan->do_ping = 0;
 			chan->ep_is_in = 0;
 			chan->data_pid_start = DWC2_HC_PID_SETUP;
-			if (hsotg->core_params->dma_enable > 0)
+			if (hsotg->params.host_dma > 0)
 				chan->xfer_dma = urb->setup_dma;
 			else
 				chan->xfer_buf = urb->setup_packet;
@@ -2484,7 +2515,7 @@ static void dwc2_hc_init_xfer(struct dwc2_hsotg *hsotg,
 				chan->do_ping = 0;
 			chan->data_pid_start = DWC2_HC_PID_DATA1;
 			chan->xfer_len = 0;
-			if (hsotg->core_params->dma_enable > 0)
+			if (hsotg->params.host_dma > 0)
 				chan->xfer_dma = hsotg->status_buf_dma;
 			else
 				chan->xfer_buf = hsotg->status_buf;
@@ -2502,13 +2533,13 @@ static void dwc2_hc_init_xfer(struct dwc2_hsotg *hsotg,
 
 	case USB_ENDPOINT_XFER_ISOC:
 		chan->ep_type = USB_ENDPOINT_XFER_ISOC;
-		if (hsotg->core_params->dma_desc_enable > 0)
+		if (hsotg->params.dma_desc_enable > 0)
 			break;
 
 		frame_desc = &urb->iso_descs[qtd->isoc_frame_index];
 		frame_desc->status = 0;
 
-		if (hsotg->core_params->dma_enable > 0) {
+		if (hsotg->params.host_dma > 0) {
 			chan->xfer_dma = urb->dma;
 			chan->xfer_dma += frame_desc->offset +
 					qtd->isoc_split_offset;
@@ -2690,7 +2721,7 @@ static int dwc2_assign_and_init_hc(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
 		!dwc2_hcd_is_pipe_in(&urb->pipe_info))
 		urb->actual_length = urb->length;
 
-	if (hsotg->core_params->dma_enable > 0)
+	if (hsotg->params.host_dma > 0)
 		chan->xfer_dma = urb->dma + urb->actual_length;
 	else
 		chan->xfer_buf = (u8 *)urb->buf + urb->actual_length;
@@ -2715,7 +2746,7 @@ static int dwc2_assign_and_init_hc(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
 		 */
 		chan->multi_count = dwc2_hb_mult(qh->maxp);
 
-	if (hsotg->core_params->dma_desc_enable > 0) {
+	if (hsotg->params.dma_desc_enable > 0) {
 		chan->desc_list_addr = qh->desc_list_dma;
 		chan->desc_list_sz = qh->desc_list_sz;
 	}
@@ -2752,7 +2783,7 @@ enum dwc2_transaction_type dwc2_hcd_select_transactions(
 	while (qh_ptr != &hsotg->periodic_sched_ready) {
 		if (list_empty(&hsotg->free_hc_list))
 			break;
-		if (hsotg->core_params->uframe_sched > 0) {
+		if (hsotg->params.uframe_sched > 0) {
 			if (hsotg->available_host_channels <= 1)
 				break;
 			hsotg->available_host_channels--;
@@ -2776,17 +2807,17 @@ enum dwc2_transaction_type dwc2_hcd_select_transactions(
 	 * schedule. Some free host channels may not be used if they are
 	 * reserved for periodic transfers.
 	 */
-	num_channels = hsotg->core_params->host_channels;
+	num_channels = hsotg->params.host_channels;
 	qh_ptr = hsotg->non_periodic_sched_inactive.next;
 	while (qh_ptr != &hsotg->non_periodic_sched_inactive) {
-		if (hsotg->core_params->uframe_sched <= 0 &&
+		if (hsotg->params.uframe_sched <= 0 &&
 		    hsotg->non_periodic_channels >= num_channels -
 						hsotg->periodic_channels)
 			break;
 		if (list_empty(&hsotg->free_hc_list))
 			break;
 		qh = list_entry(qh_ptr, struct dwc2_qh, qh_list_entry);
-		if (hsotg->core_params->uframe_sched > 0) {
+		if (hsotg->params.uframe_sched > 0) {
 			if (hsotg->available_host_channels < 1)
 				break;
 			hsotg->available_host_channels--;
@@ -2808,7 +2839,7 @@ enum dwc2_transaction_type dwc2_hcd_select_transactions(
 		else
 			ret_val = DWC2_TRANSACTION_ALL;
 
-		if (hsotg->core_params->uframe_sched <= 0)
+		if (hsotg->params.uframe_sched <= 0)
 			hsotg->non_periodic_channels++;
 	}
 
@@ -2847,8 +2878,8 @@ static int dwc2_queue_transaction(struct dwc2_hsotg *hsotg,
 		list_move_tail(&chan->split_order_list_entry,
 			       &hsotg->split_order);
 
-	if (hsotg->core_params->dma_enable > 0) {
-		if (hsotg->core_params->dma_desc_enable > 0) {
+	if (hsotg->params.host_dma > 0) {
+		if (hsotg->params.dma_desc_enable > 0) {
 			if (!chan->xfer_started ||
 			    chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
 				dwc2_hcd_start_xfer_ddma(hsotg, chan->qh);
@@ -2957,7 +2988,7 @@ static void dwc2_process_periodic_channels(struct dwc2_hsotg *hsotg)
 		 * The flag prevents any halts to get into the request queue in
 		 * the middle of multiple high-bandwidth packets getting queued.
 		 */
-		if (hsotg->core_params->dma_enable <= 0 &&
+		if (hsotg->params.host_dma <= 0 &&
 				qh->channel->multi_count > 1)
 			hsotg->queuing_high_bandwidth = 1;
 
@@ -2976,7 +3007,7 @@ static void dwc2_process_periodic_channels(struct dwc2_hsotg *hsotg)
 		 * controller automatically handles multiple packets for
 		 * high-bandwidth transfers.
 		 */
-		if (hsotg->core_params->dma_enable > 0 || status == 0 ||
+		if (hsotg->params.host_dma > 0 || status == 0 ||
 		    qh->channel->requests == qh->channel->multi_count) {
 			qh_ptr = qh_ptr->next;
 			/*
@@ -2993,7 +3024,7 @@ static void dwc2_process_periodic_channels(struct dwc2_hsotg *hsotg)
 
 exit:
 	if (no_queue_space || no_fifo_space ||
-	    (hsotg->core_params->dma_enable <= 0 &&
+	    (hsotg->params.host_dma <= 0 &&
 	     !list_empty(&hsotg->periodic_sched_assigned))) {
 		/*
 		 * May need to queue more transactions as the request
@@ -3073,7 +3104,7 @@ static void dwc2_process_non_periodic_channels(struct dwc2_hsotg *hsotg)
 		tx_status = dwc2_readl(hsotg->regs + GNPTXSTS);
 		qspcavail = (tx_status & TXSTS_QSPCAVAIL_MASK) >>
 			    TXSTS_QSPCAVAIL_SHIFT;
-		if (hsotg->core_params->dma_enable <= 0 && qspcavail == 0) {
+		if (hsotg->params.host_dma <= 0 && qspcavail == 0) {
 			no_queue_space = 1;
 			break;
 		}
@@ -3106,7 +3137,7 @@ next:
 					hsotg->non_periodic_qh_ptr->next;
 	} while (hsotg->non_periodic_qh_ptr != orig_qh_ptr);
 
-	if (hsotg->core_params->dma_enable <= 0) {
+	if (hsotg->params.host_dma <= 0) {
 		tx_status = dwc2_readl(hsotg->regs + GNPTXSTS);
 		qspcavail = (tx_status & TXSTS_QSPCAVAIL_MASK) >>
 			    TXSTS_QSPCAVAIL_SHIFT;
@@ -3307,7 +3338,7 @@ static void dwc2_port_suspend(struct dwc2_hsotg *hsotg, u16 windex)
 	 * If hibernation is supported, Phy clock will be suspended
 	 * after registers are backuped.
 	 */
-	if (!hsotg->core_params->hibernation) {
+	if (!hsotg->params.hibernation) {
 		/* Suspend the Phy Clock */
 		pcgctl = dwc2_readl(hsotg->regs + PCGCTL);
 		pcgctl |= PCGCTL_STOPPCLK;
@@ -3342,7 +3373,7 @@ static void dwc2_port_resume(struct dwc2_hsotg *hsotg)
 	 * If hibernation is supported, Phy clock is already resumed
 	 * after registers restore.
 	 */
-	if (!hsotg->core_params->hibernation) {
+	if (!hsotg->params.hibernation) {
 		pcgctl = dwc2_readl(hsotg->regs + PCGCTL);
 		pcgctl &= ~PCGCTL_STOPPCLK;
 		dwc2_writel(pcgctl, hsotg->regs + PCGCTL);
@@ -3569,7 +3600,7 @@ static int dwc2_hcd_hub_control(struct dwc2_hsotg *hsotg, u16 typereq,
 			port_status |= USB_PORT_STAT_TEST;
 		/* USB_PORT_FEAT_INDICATOR unsupported always 0 */
 
-		if (hsotg->core_params->dma_desc_fs_enable) {
+		if (hsotg->params.dma_desc_fs_enable) {
 			/*
 			 * Enable descriptor DMA only if a full speed
 			 * device is connected.
@@ -3583,7 +3614,7 @@ static int dwc2_hcd_hub_control(struct dwc2_hsotg *hsotg, u16 typereq,
 				u32 hcfg;
 
 				dev_info(hsotg->dev, "Enabling descriptor DMA mode\n");
-				hsotg->core_params->dma_desc_enable = 1;
+				hsotg->params.dma_desc_enable = 1;
 				hcfg = dwc2_readl(hsotg->regs + HCFG);
 				hcfg |= HCFG_DESCDMA;
 				dwc2_writel(hcfg, hsotg->regs + HCFG);
@@ -3824,7 +3855,7 @@ void dwc2_hcd_dump_state(struct dwc2_hsotg *hsotg)
 	u32 p_tx_status;
 	int i;
 
-	num_channels = hsotg->core_params->host_channels;
+	num_channels = hsotg->params.host_channels;
 	dev_dbg(hsotg->dev, "\n");
 	dev_dbg(hsotg->dev,
 		"************************************************************\n");
@@ -4020,35 +4051,6 @@ static struct dwc2_hsotg *dwc2_hcd_to_hsotg(struct usb_hcd *hcd)
 	return p->hsotg;
 }
 
-static int _dwc2_hcd_start(struct usb_hcd *hcd);
-
-void dwc2_host_start(struct dwc2_hsotg *hsotg)
-{
-	struct usb_hcd *hcd = dwc2_hsotg_to_hcd(hsotg);
-
-	hcd->self.is_b_host = dwc2_hcd_is_b_host(hsotg);
-	_dwc2_hcd_start(hcd);
-}
-
-void dwc2_host_disconnect(struct dwc2_hsotg *hsotg)
-{
-	struct usb_hcd *hcd = dwc2_hsotg_to_hcd(hsotg);
-
-	hcd->self.is_b_host = 0;
-}
-
-void dwc2_host_hub_info(struct dwc2_hsotg *hsotg, void *context, int *hub_addr,
-			int *hub_port)
-{
-	struct urb *urb = context;
-
-	if (urb->dev->tt)
-		*hub_addr = urb->dev->tt->hub->devnum;
-	else
-		*hub_addr = 0;
-	*hub_port = urb->dev->ttport;
-}
-
 /**
  * dwc2_host_get_tt_info() - Get the dwc2_tt associated with context
  *
@@ -4365,7 +4367,7 @@ static int _dwc2_hcd_suspend(struct usb_hcd *hcd)
 	if (!HCD_HW_ACCESSIBLE(hcd))
 		goto unlock;
 
-	if (!hsotg->core_params->hibernation)
+	if (!hsotg->params.hibernation)
 		goto skip_power_saving;
 
 	/*
@@ -4417,7 +4419,7 @@ static int _dwc2_hcd_resume(struct usb_hcd *hcd)
 	if (hsotg->lx_state != DWC2_L2)
 		goto unlock;
 
-	if (!hsotg->core_params->hibernation) {
+	if (!hsotg->params.hibernation) {
 		hsotg->lx_state = DWC2_L0;
 		goto unlock;
 	}
@@ -4510,9 +4512,6 @@ static void dwc2_dump_urb_info(struct usb_hcd *hcd, struct urb *urb,
 	case PIPE_ISOCHRONOUS:
 		pipetype = "ISOCHRONOUS";
 		break;
-	default:
-		pipetype = "UNKNOWN";
-		break;
 	}
 
 	dev_vdbg(hsotg->dev, "  Endpoint type: %s %s (%s)\n", pipetype,
@@ -4609,8 +4608,6 @@ static int _dwc2_hcd_urb_enqueue(struct usb_hcd *hcd, struct urb *urb,
 	case PIPE_INTERRUPT:
 		ep_type = USB_ENDPOINT_XFER_INT;
 		break;
-	default:
-		dev_warn(hsotg->dev, "Wrong ep type\n");
 	}
 
 	dwc2_urb = dwc2_hcd_urb_alloc(hsotg, urb->number_of_packets,
@@ -4919,7 +4916,7 @@ static void dwc2_hcd_free(struct dwc2_hsotg *hsotg)
 		}
 	}
 
-	if (hsotg->core_params->dma_enable > 0) {
+	if (hsotg->params.host_dma > 0) {
 		if (hsotg->status_buf) {
 			dma_free_coherent(hsotg->dev, DWC2_HCD_STATUS_BUF_SIZE,
 					  hsotg->status_buf,
@@ -4999,16 +4996,16 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg, int irq)
 	hsotg->last_frame_num = HFNUM_MAX_FRNUM;
 
 	/* Check if the bus driver or platform code has setup a dma_mask */
-	if (hsotg->core_params->dma_enable > 0 &&
+	if (hsotg->params.host_dma > 0 &&
 	    hsotg->dev->dma_mask == NULL) {
 		dev_warn(hsotg->dev,
 			 "dma_mask not set, disabling DMA\n");
-		hsotg->core_params->dma_enable = 0;
-		hsotg->core_params->dma_desc_enable = 0;
+		hsotg->params.host_dma = 0;
+		hsotg->params.dma_desc_enable = 0;
 	}
 
 	/* Set device flags indicating whether the HCD supports DMA */
-	if (hsotg->core_params->dma_enable > 0) {
+	if (hsotg->params.host_dma > 0) {
 		if (dma_set_mask(hsotg->dev, DMA_BIT_MASK(32)) < 0)
 			dev_warn(hsotg->dev, "can't set DMA mask\n");
 		if (dma_set_coherent_mask(hsotg->dev, DMA_BIT_MASK(32)) < 0)
@@ -5019,7 +5016,7 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg, int irq)
 	if (!hcd)
 		goto error1;
 
-	if (hsotg->core_params->dma_enable <= 0)
+	if (hsotg->params.host_dma <= 0)
 		hcd->self.uses_dma = 0;
 
 	hcd->has_tt = 1;
@@ -5067,7 +5064,7 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg, int irq)
 	 * in the controller. Initialize the channel descriptor array.
 	 */
 	INIT_LIST_HEAD(&hsotg->free_hc_list);
-	num_channels = hsotg->core_params->host_channels;
+	num_channels = hsotg->params.host_channels;
 	memset(&hsotg->hc_ptr_array[0], 0, sizeof(hsotg->hc_ptr_array));
 
 	for (i = 0; i < num_channels; i++) {
@@ -5091,7 +5088,7 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg, int irq)
 	 * done after usb_add_hcd since that function allocates the DMA buffer
 	 * pool.
 	 */
-	if (hsotg->core_params->dma_enable > 0)
+	if (hsotg->params.host_dma > 0)
 		hsotg->status_buf = dma_alloc_coherent(hsotg->dev,
 					DWC2_HCD_STATUS_BUF_SIZE,
 					&hsotg->status_buf_dma, GFP_KERNEL);
@@ -5107,10 +5104,10 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg, int irq)
 	 * DMA mode.
 	 * Alignment must be set to 512 bytes.
 	 */
-	if (hsotg->core_params->dma_desc_enable ||
-	    hsotg->core_params->dma_desc_fs_enable) {
+	if (hsotg->params.dma_desc_enable ||
+	    hsotg->params.dma_desc_fs_enable) {
 		hsotg->desc_gen_cache = kmem_cache_create("dwc2-gen-desc",
-				sizeof(struct dwc2_hcd_dma_desc) *
+				sizeof(struct dwc2_dma_desc) *
 				MAX_DMA_DESC_NUM_GENERIC, 512, SLAB_CACHE_DMA,
 				NULL);
 		if (!hsotg->desc_gen_cache) {
@@ -5121,12 +5118,12 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg, int irq)
 			 * Disable descriptor dma mode since it will not be
 			 * usable.
 			 */
-			hsotg->core_params->dma_desc_enable = 0;
-			hsotg->core_params->dma_desc_fs_enable = 0;
+			hsotg->params.dma_desc_enable = 0;
+			hsotg->params.dma_desc_fs_enable = 0;
 		}
 
 		hsotg->desc_hsisoc_cache = kmem_cache_create("dwc2-hsisoc-desc",
-				sizeof(struct dwc2_hcd_dma_desc) *
+				sizeof(struct dwc2_dma_desc) *
 				MAX_DMA_DESC_NUM_HS_ISOC, 512, 0, NULL);
 		if (!hsotg->desc_hsisoc_cache) {
 			dev_err(hsotg->dev,
@@ -5138,8 +5135,8 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg, int irq)
 			 * Disable descriptor dma mode since it will not be
 			 * usable.
 			 */
-			hsotg->core_params->dma_desc_enable = 0;
-			hsotg->core_params->dma_desc_fs_enable = 0;
+			hsotg->params.dma_desc_enable = 0;
+			hsotg->params.dma_desc_fs_enable = 0;
 		}
 	}
 
@@ -5184,7 +5181,6 @@ error3:
 error2:
 	usb_put_hcd(hcd);
 error1:
-	kfree(hsotg->core_params);
 
 #ifdef CONFIG_USB_DWC2_TRACK_MISSED_SOFS
 	kfree(hsotg->last_frame_num_array);
@@ -5250,7 +5246,7 @@ int dwc2_backup_host_registers(struct dwc2_hsotg *hsotg)
 	hr = &hsotg->hr_backup;
 	hr->hcfg = dwc2_readl(hsotg->regs + HCFG);
 	hr->haintmsk = dwc2_readl(hsotg->regs + HAINTMSK);
-	for (i = 0; i < hsotg->core_params->host_channels; ++i)
+	for (i = 0; i < hsotg->params.host_channels; ++i)
 		hr->hcintmsk[i] = dwc2_readl(hsotg->regs + HCINTMSK(i));
 
 	hr->hprt0 = dwc2_read_hprt0(hsotg);
@@ -5286,7 +5282,7 @@ int dwc2_restore_host_registers(struct dwc2_hsotg *hsotg)
 	dwc2_writel(hr->hcfg, hsotg->regs + HCFG);
 	dwc2_writel(hr->haintmsk, hsotg->regs + HAINTMSK);
 
-	for (i = 0; i < hsotg->core_params->host_channels; ++i)
+	for (i = 0; i < hsotg->params.host_channels; ++i)
 		dwc2_writel(hr->hcintmsk[i], hsotg->regs + HCINTMSK(i));
 
 	dwc2_writel(hr->hprt0, hsotg->regs + HPRT0);
diff --git a/drivers/usb/dwc2/hcd.h b/drivers/usb/dwc2/hcd.h
index 7758bfb644ff..1ed5fa2beff4 100644
--- a/drivers/usb/dwc2/hcd.h
+++ b/drivers/usb/dwc2/hcd.h
@@ -348,7 +348,7 @@ struct dwc2_qh {
 	struct list_head qtd_list;
 	struct dwc2_host_chan *channel;
 	struct list_head qh_list_entry;
-	struct dwc2_hcd_dma_desc *desc_list;
+	struct dwc2_dma_desc *desc_list;
 	dma_addr_t desc_list_dma;
 	u32 desc_list_sz;
 	u32 *n_bytes;
@@ -793,11 +793,6 @@ extern void dwc2_hcd_dump_frrem(struct dwc2_hsotg *hsotg);
 #define URB_SEND_ZERO_PACKET	0x2
 
 /* Host driver callbacks */
-
-extern void dwc2_host_start(struct dwc2_hsotg *hsotg);
-extern void dwc2_host_disconnect(struct dwc2_hsotg *hsotg);
-extern void dwc2_host_hub_info(struct dwc2_hsotg *hsotg, void *context,
-			       int *hub_addr, int *hub_port);
 extern struct dwc2_tt *dwc2_host_get_tt_info(struct dwc2_hsotg *hsotg,
 					     void *context, gfp_t mem_flags,
 					     int *ttport);
diff --git a/drivers/usb/dwc2/hcd_ddma.c b/drivers/usb/dwc2/hcd_ddma.c
index 0e1d42b5dec5..cf0367768cb3 100644
--- a/drivers/usb/dwc2/hcd_ddma.c
+++ b/drivers/usb/dwc2/hcd_ddma.c
@@ -95,7 +95,7 @@ static int dwc2_desc_list_alloc(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
 	else
 		desc_cache = hsotg->desc_gen_cache;
 
-	qh->desc_list_sz = sizeof(struct dwc2_hcd_dma_desc) *
+	qh->desc_list_sz = sizeof(struct dwc2_dma_desc) *
 						dwc2_max_desc_num(qh);
 
 	qh->desc_list = kmem_cache_zalloc(desc_cache, flags | GFP_DMA);
@@ -297,7 +297,7 @@ static void dwc2_release_channel_ddma(struct dwc2_hsotg *hsotg,
 	struct dwc2_host_chan *chan = qh->channel;
 
 	if (dwc2_qh_is_non_per(qh)) {
-		if (hsotg->core_params->uframe_sched > 0)
+		if (hsotg->params.uframe_sched > 0)
 			hsotg->available_host_channels++;
 		else
 			hsotg->non_periodic_channels--;
@@ -322,7 +322,7 @@ static void dwc2_release_channel_ddma(struct dwc2_hsotg *hsotg,
 	qh->ntd = 0;
 
 	if (qh->desc_list)
-		memset(qh->desc_list, 0, sizeof(struct dwc2_hcd_dma_desc) *
+		memset(qh->desc_list, 0, sizeof(struct dwc2_dma_desc) *
 		       dwc2_max_desc_num(qh));
 }
 
@@ -404,7 +404,7 @@ void dwc2_hcd_qh_free_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
 
 	if ((qh->ep_type == USB_ENDPOINT_XFER_ISOC ||
 	     qh->ep_type == USB_ENDPOINT_XFER_INT) &&
-	    (hsotg->core_params->uframe_sched > 0 ||
+	    (hsotg->params.uframe_sched > 0 ||
 	     !hsotg->periodic_channels) && hsotg->frame_list) {
 		dwc2_per_sched_disable(hsotg);
 		dwc2_frame_list_free(hsotg);
@@ -542,7 +542,7 @@ static void dwc2_fill_host_isoc_dma_desc(struct dwc2_hsotg *hsotg,
 					 struct dwc2_qh *qh, u32 max_xfer_size,
 					 u16 idx)
 {
-	struct dwc2_hcd_dma_desc *dma_desc = &qh->desc_list[idx];
+	struct dwc2_dma_desc *dma_desc = &qh->desc_list[idx];
 	struct dwc2_hcd_iso_packet_desc *frame_desc;
 
 	memset(dma_desc, 0, sizeof(*dma_desc));
@@ -571,8 +571,8 @@ static void dwc2_fill_host_isoc_dma_desc(struct dwc2_hsotg *hsotg,
 
 	dma_sync_single_for_device(hsotg->dev,
 			qh->desc_list_dma +
-			(idx * sizeof(struct dwc2_hcd_dma_desc)),
-			sizeof(struct dwc2_hcd_dma_desc),
+			(idx * sizeof(struct dwc2_dma_desc)),
+			sizeof(struct dwc2_dma_desc),
 			DMA_TO_DEVICE);
 }
 
@@ -645,8 +645,8 @@ static void dwc2_init_isoc_dma_desc(struct dwc2_hsotg *hsotg,
 		qh->desc_list[idx].status |= HOST_DMA_IOC;
 		dma_sync_single_for_device(hsotg->dev,
 					   qh->desc_list_dma + (idx *
-					   sizeof(struct dwc2_hcd_dma_desc)),
-					   sizeof(struct dwc2_hcd_dma_desc),
+					   sizeof(struct dwc2_dma_desc)),
+					   sizeof(struct dwc2_dma_desc),
 					   DMA_TO_DEVICE);
 	}
 #else
@@ -679,8 +679,8 @@ static void dwc2_init_isoc_dma_desc(struct dwc2_hsotg *hsotg,
 	qh->desc_list[idx].status |= HOST_DMA_IOC;
 	dma_sync_single_for_device(hsotg->dev,
 				   qh->desc_list_dma +
-				   (idx * sizeof(struct dwc2_hcd_dma_desc)),
-				   sizeof(struct dwc2_hcd_dma_desc),
+				   (idx * sizeof(struct dwc2_dma_desc)),
+				   sizeof(struct dwc2_dma_desc),
 				   DMA_TO_DEVICE);
 #endif
 }
@@ -690,11 +690,11 @@ static void dwc2_fill_host_dma_desc(struct dwc2_hsotg *hsotg,
 				    struct dwc2_qtd *qtd, struct dwc2_qh *qh,
 				    int n_desc)
 {
-	struct dwc2_hcd_dma_desc *dma_desc = &qh->desc_list[n_desc];
+	struct dwc2_dma_desc *dma_desc = &qh->desc_list[n_desc];
 	int len = chan->xfer_len;
 
-	if (len > MAX_DMA_DESC_SIZE - (chan->max_packet - 1))
-		len = MAX_DMA_DESC_SIZE - (chan->max_packet - 1);
+	if (len > HOST_DMA_NBYTES_LIMIT - (chan->max_packet - 1))
+		len = HOST_DMA_NBYTES_LIMIT - (chan->max_packet - 1);
 
 	if (chan->ep_is_in) {
 		int num_packets;
@@ -721,8 +721,8 @@ static void dwc2_fill_host_dma_desc(struct dwc2_hsotg *hsotg,
 
 	dma_sync_single_for_device(hsotg->dev,
 				   qh->desc_list_dma +
-				   (n_desc * sizeof(struct dwc2_hcd_dma_desc)),
-				   sizeof(struct dwc2_hcd_dma_desc),
+				   (n_desc * sizeof(struct dwc2_dma_desc)),
+				   sizeof(struct dwc2_dma_desc),
 				   DMA_TO_DEVICE);
 
 	/*
@@ -778,8 +778,8 @@ static void dwc2_init_non_isoc_dma_desc(struct dwc2_hsotg *hsotg,
 				dma_sync_single_for_device(hsotg->dev,
 					qh->desc_list_dma +
 					((n_desc - 1) *
-					sizeof(struct dwc2_hcd_dma_desc)),
-					sizeof(struct dwc2_hcd_dma_desc),
+					sizeof(struct dwc2_dma_desc)),
+					sizeof(struct dwc2_dma_desc),
 					DMA_TO_DEVICE);
 			}
 			dwc2_fill_host_dma_desc(hsotg, chan, qtd, qh, n_desc);
@@ -808,8 +808,8 @@ static void dwc2_init_non_isoc_dma_desc(struct dwc2_hsotg *hsotg,
 			 n_desc - 1, &qh->desc_list[n_desc - 1]);
 		dma_sync_single_for_device(hsotg->dev,
 					   qh->desc_list_dma + (n_desc - 1) *
-					   sizeof(struct dwc2_hcd_dma_desc),
-					   sizeof(struct dwc2_hcd_dma_desc),
+					   sizeof(struct dwc2_dma_desc),
+					   sizeof(struct dwc2_dma_desc),
 					   DMA_TO_DEVICE);
 		if (n_desc > 1) {
 			qh->desc_list[0].status |= HOST_DMA_A;
@@ -817,7 +817,7 @@ static void dwc2_init_non_isoc_dma_desc(struct dwc2_hsotg *hsotg,
 				 &qh->desc_list[0]);
 			dma_sync_single_for_device(hsotg->dev,
 					qh->desc_list_dma,
-					sizeof(struct dwc2_hcd_dma_desc),
+					sizeof(struct dwc2_dma_desc),
 					DMA_TO_DEVICE);
 		}
 		chan->ntd = n_desc;
@@ -893,7 +893,7 @@ static int dwc2_cmpl_host_isoc_dma_desc(struct dwc2_hsotg *hsotg,
 					struct dwc2_qtd *qtd,
 					struct dwc2_qh *qh, u16 idx)
 {
-	struct dwc2_hcd_dma_desc *dma_desc;
+	struct dwc2_dma_desc *dma_desc;
 	struct dwc2_hcd_iso_packet_desc *frame_desc;
 	u16 remain = 0;
 	int rc = 0;
@@ -902,8 +902,8 @@ static int dwc2_cmpl_host_isoc_dma_desc(struct dwc2_hsotg *hsotg,
 		return -EINVAL;
 
 	dma_sync_single_for_cpu(hsotg->dev, qh->desc_list_dma + (idx *
-				sizeof(struct dwc2_hcd_dma_desc)),
-				sizeof(struct dwc2_hcd_dma_desc),
+				sizeof(struct dwc2_dma_desc)),
+				sizeof(struct dwc2_dma_desc),
 				DMA_FROM_DEVICE);
 
 	dma_desc = &qh->desc_list[idx];
@@ -1066,7 +1066,7 @@ stop_scan:
 static int dwc2_update_non_isoc_urb_state_ddma(struct dwc2_hsotg *hsotg,
 					struct dwc2_host_chan *chan,
 					struct dwc2_qtd *qtd,
-					struct dwc2_hcd_dma_desc *dma_desc,
+					struct dwc2_dma_desc *dma_desc,
 					enum dwc2_halt_status halt_status,
 					u32 n_bytes, int *xfer_done)
 {
@@ -1154,7 +1154,7 @@ static int dwc2_process_non_isoc_desc(struct dwc2_hsotg *hsotg,
 {
 	struct dwc2_qh *qh = chan->qh;
 	struct dwc2_hcd_urb *urb = qtd->urb;
-	struct dwc2_hcd_dma_desc *dma_desc;
+	struct dwc2_dma_desc *dma_desc;
 	u32 n_bytes;
 	int failed;
 
@@ -1165,8 +1165,8 @@ static int dwc2_process_non_isoc_desc(struct dwc2_hsotg *hsotg,
 
 	dma_sync_single_for_cpu(hsotg->dev,
 				qh->desc_list_dma + (desc_num *
-				sizeof(struct dwc2_hcd_dma_desc)),
-				sizeof(struct dwc2_hcd_dma_desc),
+				sizeof(struct dwc2_dma_desc)),
+				sizeof(struct dwc2_dma_desc),
 				DMA_FROM_DEVICE);
 
 	dma_desc = &qh->desc_list[desc_num];
diff --git a/drivers/usb/dwc2/hcd_intr.c b/drivers/usb/dwc2/hcd_intr.c
index 906f223542ee..b8f4b6aaf1d0 100644
--- a/drivers/usb/dwc2/hcd_intr.c
+++ b/drivers/usb/dwc2/hcd_intr.c
@@ -256,7 +256,7 @@ static void dwc2_perio_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
 static void dwc2_hprt0_enable(struct dwc2_hsotg *hsotg, u32 hprt0,
 			      u32 *hprt0_modify)
 {
-	struct dwc2_core_params *params = hsotg->core_params;
+	struct dwc2_core_params *params = &hsotg->params;
 	int do_reset = 0;
 	u32 usbcfg;
 	u32 prtspd;
@@ -395,10 +395,10 @@ static void dwc2_port_intr(struct dwc2_hsotg *hsotg)
 			dwc2_hprt0_enable(hsotg, hprt0, &hprt0_modify);
 		} else {
 			hsotg->flags.b.port_enable_change = 1;
-			if (hsotg->core_params->dma_desc_fs_enable) {
+			if (hsotg->params.dma_desc_fs_enable) {
 				u32 hcfg;
 
-				hsotg->core_params->dma_desc_enable = 0;
+				hsotg->params.dma_desc_enable = 0;
 				hsotg->new_connection = false;
 				hcfg = dwc2_readl(hsotg->regs + HCFG);
 				hcfg &= ~HCFG_DESCDMA;
@@ -604,7 +604,7 @@ static enum dwc2_halt_status dwc2_update_isoc_urb_state(
 		/* Skip whole frame */
 		if (chan->qh->do_split &&
 		    chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
-		    hsotg->core_params->dma_enable > 0) {
+		    hsotg->params.host_dma > 0) {
 			qtd->complete_split = 0;
 			qtd->isoc_split_offset = 0;
 		}
@@ -743,7 +743,7 @@ cleanup:
 	dwc2_hc_cleanup(hsotg, chan);
 	list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);
 
-	if (hsotg->core_params->uframe_sched > 0) {
+	if (hsotg->params.uframe_sched > 0) {
 		hsotg->available_host_channels++;
 	} else {
 		switch (chan->ep_type) {
@@ -789,7 +789,7 @@ static void dwc2_halt_channel(struct dwc2_hsotg *hsotg,
 	if (dbg_hc(chan))
 		dev_vdbg(hsotg->dev, "%s()\n", __func__);
 
-	if (hsotg->core_params->dma_enable > 0) {
+	if (hsotg->params.host_dma > 0) {
 		if (dbg_hc(chan))
 			dev_vdbg(hsotg->dev, "DMA enabled\n");
 		dwc2_release_channel(hsotg, chan, qtd, halt_status);
@@ -915,6 +915,8 @@ static int dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg *hsotg,
 {
 	struct dwc2_hcd_iso_packet_desc *frame_desc;
 	u32 len;
+	u32 hctsiz;
+	u32 pid;
 
 	if (!qtd->urb)
 		return 0;
@@ -932,7 +934,10 @@ static int dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg *hsotg,
 
 	qtd->isoc_split_offset += len;
 
-	if (frame_desc->actual_length >= frame_desc->length) {
+	hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
+	pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
+
+	if (frame_desc->actual_length >= frame_desc->length || pid == 0) {
 		frame_desc->status = 0;
 		qtd->isoc_frame_index++;
 		qtd->complete_split = 0;
@@ -974,7 +979,7 @@ static void dwc2_hc_xfercomp_intr(struct dwc2_hsotg *hsotg,
 
 	pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
 
-	if (hsotg->core_params->dma_desc_enable > 0) {
+	if (hsotg->params.dma_desc_enable > 0) {
 		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum, halt_status);
 		if (pipe_type == USB_ENDPOINT_XFER_ISOC)
 			/* Do not disable the interrupt, just clear it */
@@ -985,7 +990,7 @@ static void dwc2_hc_xfercomp_intr(struct dwc2_hsotg *hsotg,
 	/* Handle xfer complete on CSPLIT */
 	if (chan->qh->do_split) {
 		if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
-		    hsotg->core_params->dma_enable > 0) {
+		    hsotg->params.host_dma > 0) {
 			if (qtd->complete_split &&
 			    dwc2_xfercomp_isoc_split_in(hsotg, chan, chnum,
 							qtd))
@@ -1097,7 +1102,7 @@ static void dwc2_hc_stall_intr(struct dwc2_hsotg *hsotg,
 	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: STALL Received--\n",
 		chnum);
 
-	if (hsotg->core_params->dma_desc_enable > 0) {
+	if (hsotg->params.dma_desc_enable > 0) {
 		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
 					    DWC2_HC_XFER_STALL);
 		goto handle_stall_done;
@@ -1207,7 +1212,7 @@ static void dwc2_hc_nak_intr(struct dwc2_hsotg *hsotg,
 	switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
 	case USB_ENDPOINT_XFER_CONTROL:
 	case USB_ENDPOINT_XFER_BULK:
-		if (hsotg->core_params->dma_enable > 0 && chan->ep_is_in) {
+		if (hsotg->params.host_dma > 0 && chan->ep_is_in) {
 			/*
 			 * NAK interrupts are enabled on bulk/control IN
 			 * transfers in DMA mode for the sole purpose of
@@ -1353,7 +1358,7 @@ static void dwc2_hc_nyet_intr(struct dwc2_hsotg *hsotg,
 	 */
 	if (chan->do_split && chan->complete_split) {
 		if (chan->ep_is_in && chan->ep_type == USB_ENDPOINT_XFER_ISOC &&
-		    hsotg->core_params->dma_enable > 0) {
+		    hsotg->params.host_dma > 0) {
 			qtd->complete_split = 0;
 			qtd->isoc_split_offset = 0;
 			qtd->isoc_frame_index++;
@@ -1374,7 +1379,7 @@ static void dwc2_hc_nyet_intr(struct dwc2_hsotg *hsotg,
 			struct dwc2_qh *qh = chan->qh;
 			bool past_end;
 
-			if (hsotg->core_params->uframe_sched <= 0) {
+			if (hsotg->params.uframe_sched <= 0) {
 				int frnum = dwc2_hcd_get_frame_number(hsotg);
 
 				/* Don't have num_hs_transfers; simple logic */
@@ -1467,7 +1472,7 @@ static void dwc2_hc_babble_intr(struct dwc2_hsotg *hsotg,
 
 	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
 
-	if (hsotg->core_params->dma_desc_enable > 0) {
+	if (hsotg->params.dma_desc_enable > 0) {
 		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
 					    DWC2_HC_XFER_BABBLE_ERR);
 		goto disable_int;
@@ -1572,7 +1577,7 @@ static void dwc2_hc_ahberr_intr(struct dwc2_hsotg *hsotg,
 	dev_err(hsotg->dev, "  Interval: %d\n", urb->interval);
 
 	/* Core halts the channel for Descriptor DMA mode */
-	if (hsotg->core_params->dma_desc_enable > 0) {
+	if (hsotg->params.dma_desc_enable > 0) {
 		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
 					    DWC2_HC_XFER_AHB_ERR);
 		goto handle_ahberr_done;
@@ -1604,7 +1609,7 @@ static void dwc2_hc_xacterr_intr(struct dwc2_hsotg *hsotg,
 
 	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
 
-	if (hsotg->core_params->dma_desc_enable > 0) {
+	if (hsotg->params.dma_desc_enable > 0) {
 		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
 					    DWC2_HC_XFER_XACT_ERR);
 		goto handle_xacterr_done;
@@ -1798,8 +1803,8 @@ static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
 
 	if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE ||
 	    (chan->halt_status == DWC2_HC_XFER_AHB_ERR &&
-	     hsotg->core_params->dma_desc_enable <= 0)) {
-		if (hsotg->core_params->dma_desc_enable > 0)
+	     hsotg->params.dma_desc_enable <= 0)) {
+		if (hsotg->params.dma_desc_enable > 0)
 			dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
 						    chan->halt_status);
 		else
@@ -1830,7 +1835,7 @@ static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
 	} else if (chan->hcint & HCINTMSK_STALL) {
 		dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
 	} else if ((chan->hcint & HCINTMSK_XACTERR) &&
-		   hsotg->core_params->dma_desc_enable <= 0) {
+		   hsotg->params.dma_desc_enable <= 0) {
 		if (out_nak_enh) {
 			if (chan->hcint &
 			    (HCINTMSK_NYET | HCINTMSK_NAK | HCINTMSK_ACK)) {
@@ -1850,10 +1855,10 @@ static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
 		 */
 		dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
 	} else if ((chan->hcint & HCINTMSK_XCS_XACT) &&
-		   hsotg->core_params->dma_desc_enable > 0) {
+		   hsotg->params.dma_desc_enable > 0) {
 		dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
 	} else if ((chan->hcint & HCINTMSK_AHBERR) &&
-		   hsotg->core_params->dma_desc_enable > 0) {
+		   hsotg->params.dma_desc_enable > 0) {
 		dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
 	} else if (chan->hcint & HCINTMSK_BBLERR) {
 		dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
@@ -1946,7 +1951,7 @@ static void dwc2_hc_chhltd_intr(struct dwc2_hsotg *hsotg,
 		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: Channel Halted--\n",
 			 chnum);
 
-	if (hsotg->core_params->dma_enable > 0) {
+	if (hsotg->params.host_dma > 0) {
 		dwc2_hc_chhltd_intr_dma(hsotg, chan, chnum, qtd);
 	} else {
 		if (!dwc2_halt_status_ok(hsotg, chan, chnum, qtd))
@@ -2023,7 +2028,7 @@ static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
 		 * interrupt unmasked
 		 */
 		WARN_ON(hcint != HCINTMSK_CHHLTD);
-		if (hsotg->core_params->dma_desc_enable > 0)
+		if (hsotg->params.dma_desc_enable > 0)
 			dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
 						    chan->halt_status);
 		else
@@ -2051,7 +2056,7 @@ static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
 	qtd = list_first_entry(&chan->qh->qtd_list, struct dwc2_qtd,
 			       qtd_list_entry);
 
-	if (hsotg->core_params->dma_enable <= 0) {
+	if (hsotg->params.host_dma <= 0) {
 		if ((hcint & HCINTMSK_CHHLTD) && hcint != HCINTMSK_CHHLTD)
 			hcint &= ~HCINTMSK_CHHLTD;
 	}
@@ -2156,7 +2161,7 @@ static void dwc2_hc_intr(struct dwc2_hsotg *hsotg)
 		}
 	}
 
-	for (i = 0; i < hsotg->core_params->host_channels; i++) {
+	for (i = 0; i < hsotg->params.host_channels; i++) {
 		if (haint & (1 << i))
 			dwc2_hc_n_intr(hsotg, i);
 	}
diff --git a/drivers/usb/dwc2/hcd_queue.c b/drivers/usb/dwc2/hcd_queue.c
index 13754353251f..5713f03a4e56 100644
--- a/drivers/usb/dwc2/hcd_queue.c
+++ b/drivers/usb/dwc2/hcd_queue.c
@@ -75,7 +75,7 @@ static int dwc2_periodic_channel_available(struct dwc2_hsotg *hsotg)
 	int status;
 	int num_channels;
 
-	num_channels = hsotg->core_params->host_channels;
+	num_channels = hsotg->params.host_channels;
 	if (hsotg->periodic_channels + hsotg->non_periodic_channels <
 								num_channels
 	    && hsotg->periodic_channels < num_channels - 1) {
@@ -355,6 +355,37 @@ static void pmap_unschedule(unsigned long *map, int bits_per_period,
 	}
 }
 
+/**
+ * dwc2_get_ls_map() - Get the map used for the given qh
+ *
+ * @hsotg: The HCD state structure for the DWC OTG controller.
+ * @qh:    QH for the periodic transfer.
+ *
+ * We'll always get the periodic map out of our TT.  Note that even if we're
+ * running the host straight in low speed / full speed mode it appears as if
+ * a TT is allocated for us, so we'll use it.  If that ever changes we can
+ * add logic here to get a map out of "hsotg" if !qh->do_split.
+ *
+ * Returns: the map or NULL if a map couldn't be found.
+ */
+static unsigned long *dwc2_get_ls_map(struct dwc2_hsotg *hsotg,
+				      struct dwc2_qh *qh)
+{
+	unsigned long *map;
+
+	/* Don't expect to be missing a TT and be doing low speed scheduling */
+	if (WARN_ON(!qh->dwc_tt))
+		return NULL;
+
+	/* Get the map and adjust if this is a multi_tt hub */
+	map = qh->dwc_tt->periodic_bitmaps;
+	if (qh->dwc_tt->usb_tt->multi)
+		map += DWC2_ELEMENTS_PER_LS_BITMAP * qh->ttport;
+
+	return map;
+}
+
+#ifdef DWC2_PRINT_SCHEDULE
 /*
  * cat_printf() - A printf() + strcat() helper
  *
@@ -454,35 +485,6 @@ static void pmap_print(unsigned long *map, int bits_per_period,
 	}
 }
 
-/**
- * dwc2_get_ls_map() - Get the map used for the given qh
- *
- * @hsotg: The HCD state structure for the DWC OTG controller.
- * @qh:    QH for the periodic transfer.
- *
- * We'll always get the periodic map out of our TT.  Note that even if we're
- * running the host straight in low speed / full speed mode it appears as if
- * a TT is allocated for us, so we'll use it.  If that ever changes we can
- * add logic here to get a map out of "hsotg" if !qh->do_split.
- *
- * Returns: the map or NULL if a map couldn't be found.
- */
-static unsigned long *dwc2_get_ls_map(struct dwc2_hsotg *hsotg,
-				      struct dwc2_qh *qh)
-{
-	unsigned long *map;
-
-	/* Don't expect to be missing a TT and be doing low speed scheduling */
-	if (WARN_ON(!qh->dwc_tt))
-		return NULL;
-
-	/* Get the map and adjust if this is a multi_tt hub */
-	map = qh->dwc_tt->periodic_bitmaps;
-	if (qh->dwc_tt->usb_tt->multi)
-		map += DWC2_ELEMENTS_PER_LS_BITMAP * qh->ttport;
-
-	return map;
-}
 
 struct dwc2_qh_print_data {
 	struct dwc2_hsotg *hsotg;
@@ -519,9 +521,6 @@ static void dwc2_qh_schedule_print(struct dwc2_hsotg *hsotg,
 	 * If we don't have tracing turned on, don't run unless the special
 	 * define is turned on.
 	 */
-#ifndef DWC2_PRINT_SCHEDULE
-	return;
-#endif
 
 	if (qh->schedule_low_speed) {
 		unsigned long *map = dwc2_get_ls_map(hsotg, qh);
@@ -559,8 +558,12 @@ static void dwc2_qh_schedule_print(struct dwc2_hsotg *hsotg,
 			   DWC2_HS_SCHEDULE_UFRAMES, "uFrame", "us",
 			   dwc2_qh_print, &print_data);
 	}
-
+	return;
 }
+#else
+static inline void dwc2_qh_schedule_print(struct dwc2_hsotg *hsotg,
+					  struct dwc2_qh *qh) {};
+#endif
 
 /**
  * dwc2_ls_pmap_schedule() - Schedule a low speed QH
@@ -1104,7 +1107,7 @@ static void dwc2_pick_first_frame(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
 	next_active_frame = earliest_frame;
 
 	/* Get the "no microframe schduler" out of the way... */
-	if (hsotg->core_params->uframe_sched <= 0) {
+	if (hsotg->params.uframe_sched <= 0) {
 		if (qh->do_split)
 			/* Splits are active at microframe 0 minus 1 */
 			next_active_frame |= 0x7;
@@ -1197,7 +1200,7 @@ static int dwc2_do_reserve(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
 {
 	int status;
 
-	if (hsotg->core_params->uframe_sched > 0) {
+	if (hsotg->params.uframe_sched > 0) {
 		status = dwc2_uframe_schedule(hsotg, qh);
 	} else {
 		status = dwc2_periodic_channel_available(hsotg);
@@ -1218,7 +1221,7 @@ static int dwc2_do_reserve(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
 		return status;
 	}
 
-	if (hsotg->core_params->uframe_sched <= 0)
+	if (hsotg->params.uframe_sched <= 0)
 		/* Reserve periodic channel */
 		hsotg->periodic_channels++;
 
@@ -1254,7 +1257,7 @@ static void dwc2_do_unreserve(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
 	/* Update claimed usecs per (micro)frame */
 	hsotg->periodic_usecs -= qh->host_us;
 
-	if (hsotg->core_params->uframe_sched > 0) {
+	if (hsotg->params.uframe_sched > 0) {
 		dwc2_uframe_unschedule(hsotg, qh);
 	} else {
 		/* Release periodic channel reservation */
@@ -1328,7 +1331,7 @@ static int dwc2_check_max_xfer_size(struct dwc2_hsotg *hsotg,
 	int status = 0;
 
 	max_xfer_size = dwc2_max_packet(qh->maxp) * dwc2_hb_mult(qh->maxp);
-	max_channel_xfer_size = hsotg->core_params->max_transfer_size;
+	max_channel_xfer_size = hsotg->params.max_transfer_size;
 
 	if (max_xfer_size > max_channel_xfer_size) {
 		dev_err(hsotg->dev,
@@ -1391,7 +1394,7 @@ static int dwc2_schedule_periodic(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
 
 	qh->unreserve_pending = 0;
 
-	if (hsotg->core_params->dma_desc_enable > 0)
+	if (hsotg->params.dma_desc_enable > 0)
 		/* Don't rely on SOF and start in ready schedule */
 		list_add_tail(&qh->qh_list_entry, &hsotg->periodic_sched_ready);
 	else
@@ -1599,7 +1602,7 @@ struct dwc2_qh *dwc2_hcd_qh_create(struct dwc2_hsotg *hsotg,
 
 	dwc2_qh_init(hsotg, qh, urb, mem_flags);
 
-	if (hsotg->core_params->dma_desc_enable > 0 &&
+	if (hsotg->params.dma_desc_enable > 0 &&
 	    dwc2_hcd_qh_init_ddma(hsotg, qh, mem_flags) < 0) {
 		dwc2_hcd_qh_free(hsotg, qh);
 		return NULL;
@@ -1711,7 +1714,7 @@ void dwc2_hcd_qh_unlink(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
 	dwc2_deschedule_periodic(hsotg, qh);
 	hsotg->periodic_qh_count--;
 	if (!hsotg->periodic_qh_count &&
-	    hsotg->core_params->dma_desc_enable <= 0) {
+	    hsotg->params.dma_desc_enable <= 0) {
 		intr_mask = dwc2_readl(hsotg->regs + GINTMSK);
 		intr_mask &= ~GINTSTS_SOF;
 		dwc2_writel(intr_mask, hsotg->regs + GINTMSK);
diff --git a/drivers/usb/dwc2/hw.h b/drivers/usb/dwc2/hw.h
index 91058441e62a..5be056b39e5c 100644
--- a/drivers/usb/dwc2/hw.h
+++ b/drivers/usb/dwc2/hw.h
@@ -412,6 +412,7 @@
 /* Device mode registers */
 
 #define DCFG				HSOTG_REG(0x800)
+#define DCFG_DESCDMA_EN			(1 << 23)
 #define DCFG_EPMISCNT_MASK		(0x1f << 18)
 #define DCFG_EPMISCNT_SHIFT		18
 #define DCFG_EPMISCNT_LIMIT		0x1f
@@ -473,6 +474,7 @@
 #define DIEPMSK_XFERCOMPLMSK		(1 << 0)
 
 #define DOEPMSK				HSOTG_REG(0x814)
+#define DOEPMSK_BNAMSK			(1 << 9)
 #define DOEPMSK_BACK2BACKSETUP		(1 << 6)
 #define DOEPMSK_STSPHSERCVDMSK		(1 << 5)
 #define DOEPMSK_OUTTKNEPDISMSK		(1 << 4)
@@ -790,7 +792,8 @@
 #define HCFIFO(_ch)			HSOTG_REG(0x1000 + 0x1000 * (_ch))
 
 /**
- * struct dwc2_hcd_dma_desc - Host-mode DMA descriptor structure
+ * struct dwc2_dma_desc - DMA descriptor structure,
+ * used for both host and gadget modes
  *
  * @status: DMA descriptor status quadlet
  * @buf:    DMA descriptor data buffer pointer
@@ -798,10 +801,12 @@
  * DMA Descriptor structure contains two quadlets:
  * Status quadlet and Data buffer pointer.
  */
-struct dwc2_hcd_dma_desc {
+struct dwc2_dma_desc {
 	u32 status;
 	u32 buf;
-};
+} __packed;
+
+/* Host Mode DMA descriptor status quadlet */
 
 #define HOST_DMA_A			(1 << 31)
 #define HOST_DMA_STS_MASK		(0x3 << 28)
@@ -817,8 +822,43 @@ struct dwc2_hcd_dma_desc {
 #define HOST_DMA_ISOC_NBYTES_SHIFT	0
 #define HOST_DMA_NBYTES_MASK		(0x1ffff << 0)
 #define HOST_DMA_NBYTES_SHIFT		0
+#define HOST_DMA_NBYTES_LIMIT		131071
+
+/* Device Mode DMA descriptor status quadlet */
+
+#define DEV_DMA_BUFF_STS_MASK		(0x3 << 30)
+#define DEV_DMA_BUFF_STS_SHIFT		30
+#define DEV_DMA_BUFF_STS_HREADY		0
+#define DEV_DMA_BUFF_STS_DMABUSY	1
+#define DEV_DMA_BUFF_STS_DMADONE	2
+#define DEV_DMA_BUFF_STS_HBUSY		3
+#define DEV_DMA_STS_MASK		(0x3 << 28)
+#define DEV_DMA_STS_SHIFT		28
+#define DEV_DMA_STS_SUCC		0
+#define DEV_DMA_STS_BUFF_FLUSH		1
+#define DEV_DMA_STS_BUFF_ERR		3
+#define DEV_DMA_L			(1 << 27)
+#define DEV_DMA_SHORT			(1 << 26)
+#define DEV_DMA_IOC			(1 << 25)
+#define DEV_DMA_SR			(1 << 24)
+#define DEV_DMA_MTRF			(1 << 23)
+#define DEV_DMA_ISOC_PID_MASK		(0x3 << 23)
+#define DEV_DMA_ISOC_PID_SHIFT		23
+#define DEV_DMA_ISOC_PID_DATA0		0
+#define DEV_DMA_ISOC_PID_DATA2		1
+#define DEV_DMA_ISOC_PID_DATA1		2
+#define DEV_DMA_ISOC_PID_MDATA		3
+#define DEV_DMA_ISOC_FRNUM_MASK		(0x7ff << 12)
+#define DEV_DMA_ISOC_FRNUM_SHIFT	12
+#define DEV_DMA_ISOC_TX_NBYTES_MASK	(0xfff << 0)
+#define DEV_DMA_ISOC_TX_NBYTES_LIMIT	0xfff
+#define DEV_DMA_ISOC_RX_NBYTES_MASK	(0x7ff << 0)
+#define DEV_DMA_ISOC_RX_NBYTES_LIMIT	0x7ff
+#define DEV_DMA_ISOC_NBYTES_SHIFT	0
+#define DEV_DMA_NBYTES_MASK		(0xffff << 0)
+#define DEV_DMA_NBYTES_SHIFT		0
+#define DEV_DMA_NBYTES_LIMIT		0xffff
 
-#define MAX_DMA_DESC_SIZE		131071
 #define MAX_DMA_DESC_NUM_GENERIC	64
 #define MAX_DMA_DESC_NUM_HS_ISOC	256
 
diff --git a/drivers/usb/dwc2/params.c b/drivers/usb/dwc2/params.c
new file mode 100644
index 000000000000..a786256535b6
--- /dev/null
+++ b/drivers/usb/dwc2/params.c
@@ -0,0 +1,1435 @@
+/*
+ * Copyright (C) 2004-2016 Synopsys, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The names of the above-listed copyright holders may not be used
+ *    to endorse or promote products derived from this software without
+ *    specific prior written permission.
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation; either version 2 of the License, or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+
+#include "core.h"
+
+static const struct dwc2_core_params params_hi6220 = {
+	.otg_cap			= 2,	/* No HNP/SRP capable */
+	.otg_ver			= 0,	/* 1.3 */
+	.dma_desc_enable		= 0,
+	.dma_desc_fs_enable		= 0,
+	.speed				= 0,	/* High Speed */
+	.enable_dynamic_fifo		= 1,
+	.en_multiple_tx_fifo		= 1,
+	.host_rx_fifo_size		= 512,
+	.host_nperio_tx_fifo_size	= 512,
+	.host_perio_tx_fifo_size	= 512,
+	.max_transfer_size		= 65535,
+	.max_packet_count		= 511,
+	.host_channels			= 16,
+	.phy_type			= 1,	/* UTMI */
+	.phy_utmi_width			= 8,
+	.phy_ulpi_ddr			= 0,	/* Single */
+	.phy_ulpi_ext_vbus		= 0,
+	.i2c_enable			= 0,
+	.ulpi_fs_ls			= 0,
+	.host_support_fs_ls_low_power	= 0,
+	.host_ls_low_power_phy_clk	= 0,	/* 48 MHz */
+	.ts_dline			= 0,
+	.reload_ctl			= 0,
+	.ahbcfg				= GAHBCFG_HBSTLEN_INCR16 <<
+					  GAHBCFG_HBSTLEN_SHIFT,
+	.uframe_sched			= 0,
+	.external_id_pin_ctl		= -1,
+	.hibernation			= -1,
+};
+
+static const struct dwc2_core_params params_bcm2835 = {
+	.otg_cap			= 0,	/* HNP/SRP capable */
+	.otg_ver			= 0,	/* 1.3 */
+	.dma_desc_enable		= 0,
+	.dma_desc_fs_enable		= 0,
+	.speed				= 0,	/* High Speed */
+	.enable_dynamic_fifo		= 1,
+	.en_multiple_tx_fifo		= 1,
+	.host_rx_fifo_size		= 774,	/* 774 DWORDs */
+	.host_nperio_tx_fifo_size	= 256,	/* 256 DWORDs */
+	.host_perio_tx_fifo_size	= 512,	/* 512 DWORDs */
+	.max_transfer_size		= 65535,
+	.max_packet_count		= 511,
+	.host_channels			= 8,
+	.phy_type			= 1,	/* UTMI */
+	.phy_utmi_width			= 8,	/* 8 bits */
+	.phy_ulpi_ddr			= 0,	/* Single */
+	.phy_ulpi_ext_vbus		= 0,
+	.i2c_enable			= 0,
+	.ulpi_fs_ls			= 0,
+	.host_support_fs_ls_low_power	= 0,
+	.host_ls_low_power_phy_clk	= 0,	/* 48 MHz */
+	.ts_dline			= 0,
+	.reload_ctl			= 0,
+	.ahbcfg				= 0x10,
+	.uframe_sched			= 0,
+	.external_id_pin_ctl		= -1,
+	.hibernation			= -1,
+};
+
+static const struct dwc2_core_params params_rk3066 = {
+	.otg_cap			= 2,	/* non-HNP/non-SRP */
+	.otg_ver			= -1,
+	.dma_desc_enable		= 0,
+	.dma_desc_fs_enable		= 0,
+	.speed				= -1,
+	.enable_dynamic_fifo		= 1,
+	.en_multiple_tx_fifo		= -1,
+	.host_rx_fifo_size		= 525,	/* 525 DWORDs */
+	.host_nperio_tx_fifo_size	= 128,	/* 128 DWORDs */
+	.host_perio_tx_fifo_size	= 256,	/* 256 DWORDs */
+	.max_transfer_size		= -1,
+	.max_packet_count		= -1,
+	.host_channels			= -1,
+	.phy_type			= -1,
+	.phy_utmi_width			= -1,
+	.phy_ulpi_ddr			= -1,
+	.phy_ulpi_ext_vbus		= -1,
+	.i2c_enable			= -1,
+	.ulpi_fs_ls			= -1,
+	.host_support_fs_ls_low_power	= -1,
+	.host_ls_low_power_phy_clk	= -1,
+	.ts_dline			= -1,
+	.reload_ctl			= -1,
+	.ahbcfg				= GAHBCFG_HBSTLEN_INCR16 <<
+					  GAHBCFG_HBSTLEN_SHIFT,
+	.uframe_sched			= -1,
+	.external_id_pin_ctl		= -1,
+	.hibernation			= -1,
+};
+
+static const struct dwc2_core_params params_ltq = {
+	.otg_cap			= 2,	/* non-HNP/non-SRP */
+	.otg_ver			= -1,
+	.dma_desc_enable		= -1,
+	.dma_desc_fs_enable		= -1,
+	.speed				= -1,
+	.enable_dynamic_fifo		= -1,
+	.en_multiple_tx_fifo		= -1,
+	.host_rx_fifo_size		= 288,	/* 288 DWORDs */
+	.host_nperio_tx_fifo_size	= 128,	/* 128 DWORDs */
+	.host_perio_tx_fifo_size	= 96,	/* 96 DWORDs */
+	.max_transfer_size		= 65535,
+	.max_packet_count		= 511,
+	.host_channels			= -1,
+	.phy_type			= -1,
+	.phy_utmi_width			= -1,
+	.phy_ulpi_ddr			= -1,
+	.phy_ulpi_ext_vbus		= -1,
+	.i2c_enable			= -1,
+	.ulpi_fs_ls			= -1,
+	.host_support_fs_ls_low_power	= -1,
+	.host_ls_low_power_phy_clk	= -1,
+	.ts_dline			= -1,
+	.reload_ctl			= -1,
+	.ahbcfg				= GAHBCFG_HBSTLEN_INCR16 <<
+					  GAHBCFG_HBSTLEN_SHIFT,
+	.uframe_sched			= -1,
+	.external_id_pin_ctl		= -1,
+	.hibernation			= -1,
+};
+
+static const struct dwc2_core_params params_amlogic = {
+	.otg_cap			= DWC2_CAP_PARAM_NO_HNP_SRP_CAPABLE,
+	.otg_ver			= -1,
+	.dma_desc_enable		= 0,
+	.dma_desc_fs_enable		= 0,
+	.speed				= DWC2_SPEED_PARAM_HIGH,
+	.enable_dynamic_fifo		= 1,
+	.en_multiple_tx_fifo		= -1,
+	.host_rx_fifo_size		= 512,
+	.host_nperio_tx_fifo_size	= 500,
+	.host_perio_tx_fifo_size	= 500,
+	.max_transfer_size		= -1,
+	.max_packet_count		= -1,
+	.host_channels			= 16,
+	.phy_type			= DWC2_PHY_TYPE_PARAM_UTMI,
+	.phy_utmi_width			= -1,
+	.phy_ulpi_ddr			= -1,
+	.phy_ulpi_ext_vbus		= -1,
+	.i2c_enable			= -1,
+	.ulpi_fs_ls			= -1,
+	.host_support_fs_ls_low_power	= -1,
+	.host_ls_low_power_phy_clk	= -1,
+	.ts_dline			= -1,
+	.reload_ctl			= 1,
+	.ahbcfg				= GAHBCFG_HBSTLEN_INCR8 <<
+					  GAHBCFG_HBSTLEN_SHIFT,
+	.uframe_sched			= 0,
+	.external_id_pin_ctl		= -1,
+	.hibernation			= -1,
+};
+
+static const struct dwc2_core_params params_default = {
+	.otg_cap			= -1,
+	.otg_ver			= -1,
+
+	/*
+	 * Disable descriptor dma mode by default as the HW can support
+	 * it, but does not support it for SPLIT transactions.
+	 * Disable it for FS devices as well.
+	 */
+	.dma_desc_enable		= 0,
+	.dma_desc_fs_enable		= 0,
+
+	.speed				= -1,
+	.enable_dynamic_fifo		= -1,
+	.en_multiple_tx_fifo		= -1,
+	.host_rx_fifo_size		= -1,
+	.host_nperio_tx_fifo_size	= -1,
+	.host_perio_tx_fifo_size	= -1,
+	.max_transfer_size		= -1,
+	.max_packet_count		= -1,
+	.host_channels			= -1,
+	.phy_type			= -1,
+	.phy_utmi_width			= -1,
+	.phy_ulpi_ddr			= -1,
+	.phy_ulpi_ext_vbus		= -1,
+	.i2c_enable			= -1,
+	.ulpi_fs_ls			= -1,
+	.host_support_fs_ls_low_power	= -1,
+	.host_ls_low_power_phy_clk	= -1,
+	.ts_dline			= -1,
+	.reload_ctl			= -1,
+	.ahbcfg				= -1,
+	.uframe_sched			= -1,
+	.external_id_pin_ctl		= -1,
+	.hibernation			= -1,
+};
+
+const struct of_device_id dwc2_of_match_table[] = {
+	{ .compatible = "brcm,bcm2835-usb", .data = &params_bcm2835 },
+	{ .compatible = "hisilicon,hi6220-usb", .data = &params_hi6220 },
+	{ .compatible = "rockchip,rk3066-usb", .data = &params_rk3066 },
+	{ .compatible = "lantiq,arx100-usb", .data = &params_ltq },
+	{ .compatible = "lantiq,xrx200-usb", .data = &params_ltq },
+	{ .compatible = "snps,dwc2", .data = NULL },
+	{ .compatible = "samsung,s3c6400-hsotg", .data = NULL},
+	{ .compatible = "amlogic,meson8b-usb", .data = &params_amlogic },
+	{ .compatible = "amlogic,meson-gxbb-usb", .data = &params_amlogic },
+	{ .compatible = "amcc,dwc-otg", .data = NULL },
+	{},
+};
+MODULE_DEVICE_TABLE(of, dwc2_of_match_table);
+
+static void dwc2_get_device_property(struct dwc2_hsotg *hsotg,
+				     char *property, u8 size, u64 *value)
+{
+	u8 val8;
+	u16 val16;
+	u32 val32;
+
+	switch (size) {
+	case 0:
+		*value = device_property_read_bool(hsotg->dev, property);
+		break;
+	case 1:
+		if (device_property_read_u8(hsotg->dev, property, &val8))
+			return;
+
+		*value = val8;
+		break;
+	case 2:
+		if (device_property_read_u16(hsotg->dev, property, &val16))
+			return;
+
+		*value = val16;
+		break;
+	case 4:
+		if (device_property_read_u32(hsotg->dev, property, &val32))
+			return;
+
+		*value = val32;
+		break;
+	case 8:
+		if (device_property_read_u64(hsotg->dev, property, value))
+			return;
+
+		break;
+	default:
+		/*
+		 * The size is checked by the only function that calls
+		 * this so this should never happen.
+		 */
+		WARN_ON(1);
+		return;
+	}
+}
+
+static void dwc2_set_core_param(void *param, u8 size, u64 value)
+{
+	switch (size) {
+	case 0:
+		*((bool *)param) = !!value;
+		break;
+	case 1:
+		*((u8 *)param) = (u8)value;
+		break;
+	case 2:
+		*((u16 *)param) = (u16)value;
+		break;
+	case 4:
+		*((u32 *)param) = (u32)value;
+		break;
+	case 8:
+		*((u64 *)param) = (u64)value;
+		break;
+	default:
+		/*
+		 * The size is checked by the only function that calls
+		 * this so this should never happen.
+		 */
+		WARN_ON(1);
+		return;
+	}
+}
+
+/**
+ * dwc2_set_param() - Set a core parameter
+ *
+ * @hsotg: Programming view of the DWC_otg controller
+ * @param: Pointer to the parameter to set
+ * @lookup: True if the property should be looked up
+ * @property: The device property to read
+ * @legacy: The param value to set if @property is not available. This
+ *          will typically be the legacy value set in the static
+ *          params structure.
+ * @def: The default value
+ * @min: The minimum value
+ * @max: The maximum value
+ * @size: The size of the core parameter in bytes, or 0 for bool.
+ *
+ * This function looks up @property and sets the @param to that value.
+ * If the property doesn't exist it uses the passed-in @value. It will
+ * verify that the value falls between @min and @max. If it doesn't,
+ * it will output an error and set the parameter to either @def or,
+ * failing that, to @min.
+ *
+ * The @size is used to write to @param and to query the device
+ * properties so that this same function can be used with different
+ * types of parameters.
+ */
+static void dwc2_set_param(struct dwc2_hsotg *hsotg, void *param,
+			   bool lookup, char *property, u64 legacy,
+			   u64 def, u64 min, u64 max, u8 size)
+{
+	u64 sizemax;
+	u64 value;
+
+	if (WARN_ON(!hsotg || !param || !property))
+		return;
+
+	if (WARN((size > 8) || ((size & (size - 1)) != 0),
+		 "Invalid size %d for %s\n", size, property))
+		return;
+
+	dev_vdbg(hsotg->dev, "%s: Setting %s: legacy=%llu, def=%llu, min=%llu, max=%llu, size=%d\n",
+		 __func__, property, legacy, def, min, max, size);
+
+	sizemax = (1ULL << (size * 8)) - 1;
+	value = legacy;
+
+	/* Override legacy settings. */
+	if (lookup)
+		dwc2_get_device_property(hsotg, property, size, &value);
+
+	/*
+	 * While the value is not valid, try setting it to the default
+	 * value, and failing that, set it to the minimum.
+	 */
+	while ((value < min) || (value > max)) {
+		/* Print an error unless the value is set to auto. */
+		if (value != sizemax)
+			dev_err(hsotg->dev, "Invalid value %llu for param %s\n",
+				value, property);
+
+		/*
+		 * If we are already the default, just set it to the
+		 * minimum.
+		 */
+		if (value == def) {
+			dev_vdbg(hsotg->dev, "%s: setting value to min=%llu\n",
+				 __func__, min);
+			value = min;
+			break;
+		}
+
+		/* Try the default value */
+		dev_vdbg(hsotg->dev, "%s: setting value to default=%llu\n",
+			 __func__, def);
+		value = def;
+	}
+
+	dev_dbg(hsotg->dev, "Setting %s to %llu\n", property, value);
+	dwc2_set_core_param(param, size, value);
+}
+
+/**
+ * dwc2_set_param_u16() - Set a u16 parameter
+ *
+ * See dwc2_set_param().
+ */
+static void dwc2_set_param_u16(struct dwc2_hsotg *hsotg, u16 *param,
+			       bool lookup, char *property, u16 legacy,
+			       u16 def, u16 min, u16 max)
+{
+	dwc2_set_param(hsotg, param, lookup, property,
+		       legacy, def, min, max, 2);
+}
+
+/**
+ * dwc2_set_param_bool() - Set a bool parameter
+ *
+ * See dwc2_set_param().
+ *
+ * Note: there is no 'legacy' argument here because there is no legacy
+ * source of bool params.
+ */
+static void dwc2_set_param_bool(struct dwc2_hsotg *hsotg, bool *param,
+				bool lookup, char *property,
+				bool def, bool min, bool max)
+{
+	dwc2_set_param(hsotg, param, lookup, property,
+		       def, def, min, max, 0);
+}
+
+#define DWC2_OUT_OF_BOUNDS(a, b, c)	((a) < (b) || (a) > (c))
+
+/* Parameter access functions */
+static void dwc2_set_param_otg_cap(struct dwc2_hsotg *hsotg, int val)
+{
+	int valid = 1;
+
+	switch (val) {
+	case DWC2_CAP_PARAM_HNP_SRP_CAPABLE:
+		if (hsotg->hw_params.op_mode != GHWCFG2_OP_MODE_HNP_SRP_CAPABLE)
+			valid = 0;
+		break;
+	case DWC2_CAP_PARAM_SRP_ONLY_CAPABLE:
+		switch (hsotg->hw_params.op_mode) {
+		case GHWCFG2_OP_MODE_HNP_SRP_CAPABLE:
+		case GHWCFG2_OP_MODE_SRP_ONLY_CAPABLE:
+		case GHWCFG2_OP_MODE_SRP_CAPABLE_DEVICE:
+		case GHWCFG2_OP_MODE_SRP_CAPABLE_HOST:
+			break;
+		default:
+			valid = 0;
+			break;
+		}
+		break;
+	case DWC2_CAP_PARAM_NO_HNP_SRP_CAPABLE:
+		/* always valid */
+		break;
+	default:
+		valid = 0;
+		break;
+	}
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for otg_cap parameter. Check HW configuration.\n",
+				val);
+		switch (hsotg->hw_params.op_mode) {
+		case GHWCFG2_OP_MODE_HNP_SRP_CAPABLE:
+			val = DWC2_CAP_PARAM_HNP_SRP_CAPABLE;
+			break;
+		case GHWCFG2_OP_MODE_SRP_ONLY_CAPABLE:
+		case GHWCFG2_OP_MODE_SRP_CAPABLE_DEVICE:
+		case GHWCFG2_OP_MODE_SRP_CAPABLE_HOST:
+			val = DWC2_CAP_PARAM_SRP_ONLY_CAPABLE;
+			break;
+		default:
+			val = DWC2_CAP_PARAM_NO_HNP_SRP_CAPABLE;
+			break;
+		}
+		dev_dbg(hsotg->dev, "Setting otg_cap to %d\n", val);
+	}
+
+	hsotg->params.otg_cap = val;
+}
+
+static void dwc2_set_param_dma_desc_enable(struct dwc2_hsotg *hsotg, int val)
+{
+	int valid = 1;
+
+	if (val > 0 && (hsotg->params.host_dma <= 0 ||
+			!hsotg->hw_params.dma_desc_enable))
+		valid = 0;
+	if (val < 0)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for dma_desc_enable parameter. Check HW configuration.\n",
+				val);
+		val = (hsotg->params.host_dma > 0 &&
+			hsotg->hw_params.dma_desc_enable);
+		dev_dbg(hsotg->dev, "Setting dma_desc_enable to %d\n", val);
+	}
+
+	hsotg->params.dma_desc_enable = val;
+}
+
+static void dwc2_set_param_dma_desc_fs_enable(struct dwc2_hsotg *hsotg, int val)
+{
+	int valid = 1;
+
+	if (val > 0 && (hsotg->params.host_dma <= 0 ||
+			!hsotg->hw_params.dma_desc_enable))
+		valid = 0;
+	if (val < 0)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for dma_desc_fs_enable parameter. Check HW configuration.\n",
+				val);
+		val = (hsotg->params.host_dma > 0 &&
+			hsotg->hw_params.dma_desc_enable);
+	}
+
+	hsotg->params.dma_desc_fs_enable = val;
+	dev_dbg(hsotg->dev, "Setting dma_desc_fs_enable to %d\n", val);
+}
+
+static void
+dwc2_set_param_host_support_fs_ls_low_power(struct dwc2_hsotg *hsotg,
+					    int val)
+{
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev,
+				"Wrong value for host_support_fs_low_power\n");
+			dev_err(hsotg->dev,
+				"host_support_fs_low_power must be 0 or 1\n");
+		}
+		val = 0;
+		dev_dbg(hsotg->dev,
+			"Setting host_support_fs_low_power to %d\n", val);
+	}
+
+	hsotg->params.host_support_fs_ls_low_power = val;
+}
+
+static void dwc2_set_param_enable_dynamic_fifo(struct dwc2_hsotg *hsotg,
+					       int val)
+{
+	int valid = 1;
+
+	if (val > 0 && !hsotg->hw_params.enable_dynamic_fifo)
+		valid = 0;
+	if (val < 0)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for enable_dynamic_fifo parameter. Check HW configuration.\n",
+				val);
+		val = hsotg->hw_params.enable_dynamic_fifo;
+		dev_dbg(hsotg->dev, "Setting enable_dynamic_fifo to %d\n", val);
+	}
+
+	hsotg->params.enable_dynamic_fifo = val;
+}
+
+static void dwc2_set_param_host_rx_fifo_size(struct dwc2_hsotg *hsotg, int val)
+{
+	int valid = 1;
+
+	if (val < 16 || val > hsotg->hw_params.rx_fifo_size)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for host_rx_fifo_size. Check HW configuration.\n",
+				val);
+		val = hsotg->hw_params.rx_fifo_size;
+		dev_dbg(hsotg->dev, "Setting host_rx_fifo_size to %d\n", val);
+	}
+
+	hsotg->params.host_rx_fifo_size = val;
+}
+
+static void dwc2_set_param_host_nperio_tx_fifo_size(struct dwc2_hsotg *hsotg,
+						    int val)
+{
+	int valid = 1;
+
+	if (val < 16 || val > hsotg->hw_params.host_nperio_tx_fifo_size)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for host_nperio_tx_fifo_size. Check HW configuration.\n",
+				val);
+		val = hsotg->hw_params.host_nperio_tx_fifo_size;
+		dev_dbg(hsotg->dev, "Setting host_nperio_tx_fifo_size to %d\n",
+			val);
+	}
+
+	hsotg->params.host_nperio_tx_fifo_size = val;
+}
+
+static void dwc2_set_param_host_perio_tx_fifo_size(struct dwc2_hsotg *hsotg,
+						   int val)
+{
+	int valid = 1;
+
+	if (val < 16 || val > hsotg->hw_params.host_perio_tx_fifo_size)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for host_perio_tx_fifo_size. Check HW configuration.\n",
+				val);
+		val = hsotg->hw_params.host_perio_tx_fifo_size;
+		dev_dbg(hsotg->dev, "Setting host_perio_tx_fifo_size to %d\n",
+			val);
+	}
+
+	hsotg->params.host_perio_tx_fifo_size = val;
+}
+
+static void dwc2_set_param_max_transfer_size(struct dwc2_hsotg *hsotg, int val)
+{
+	int valid = 1;
+
+	if (val < 2047 || val > hsotg->hw_params.max_transfer_size)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for max_transfer_size. Check HW configuration.\n",
+				val);
+		val = hsotg->hw_params.max_transfer_size;
+		dev_dbg(hsotg->dev, "Setting max_transfer_size to %d\n", val);
+	}
+
+	hsotg->params.max_transfer_size = val;
+}
+
+static void dwc2_set_param_max_packet_count(struct dwc2_hsotg *hsotg, int val)
+{
+	int valid = 1;
+
+	if (val < 15 || val > hsotg->hw_params.max_packet_count)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for max_packet_count. Check HW configuration.\n",
+				val);
+		val = hsotg->hw_params.max_packet_count;
+		dev_dbg(hsotg->dev, "Setting max_packet_count to %d\n", val);
+	}
+
+	hsotg->params.max_packet_count = val;
+}
+
+static void dwc2_set_param_host_channels(struct dwc2_hsotg *hsotg, int val)
+{
+	int valid = 1;
+
+	if (val < 1 || val > hsotg->hw_params.host_channels)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for host_channels. Check HW configuration.\n",
+				val);
+		val = hsotg->hw_params.host_channels;
+		dev_dbg(hsotg->dev, "Setting host_channels to %d\n", val);
+	}
+
+	hsotg->params.host_channels = val;
+}
+
+static void dwc2_set_param_phy_type(struct dwc2_hsotg *hsotg, int val)
+{
+	int valid = 0;
+	u32 hs_phy_type, fs_phy_type;
+
+	if (DWC2_OUT_OF_BOUNDS(val, DWC2_PHY_TYPE_PARAM_FS,
+			       DWC2_PHY_TYPE_PARAM_ULPI)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev, "Wrong value for phy_type\n");
+			dev_err(hsotg->dev, "phy_type must be 0, 1 or 2\n");
+		}
+
+		valid = 0;
+	}
+
+	hs_phy_type = hsotg->hw_params.hs_phy_type;
+	fs_phy_type = hsotg->hw_params.fs_phy_type;
+	if (val == DWC2_PHY_TYPE_PARAM_UTMI &&
+	    (hs_phy_type == GHWCFG2_HS_PHY_TYPE_UTMI ||
+	     hs_phy_type == GHWCFG2_HS_PHY_TYPE_UTMI_ULPI))
+		valid = 1;
+	else if (val == DWC2_PHY_TYPE_PARAM_ULPI &&
+		 (hs_phy_type == GHWCFG2_HS_PHY_TYPE_ULPI ||
+		  hs_phy_type == GHWCFG2_HS_PHY_TYPE_UTMI_ULPI))
+		valid = 1;
+	else if (val == DWC2_PHY_TYPE_PARAM_FS &&
+		 fs_phy_type == GHWCFG2_FS_PHY_TYPE_DEDICATED)
+		valid = 1;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for phy_type. Check HW configuration.\n",
+				val);
+		val = DWC2_PHY_TYPE_PARAM_FS;
+		if (hs_phy_type != GHWCFG2_HS_PHY_TYPE_NOT_SUPPORTED) {
+			if (hs_phy_type == GHWCFG2_HS_PHY_TYPE_UTMI ||
+			    hs_phy_type == GHWCFG2_HS_PHY_TYPE_UTMI_ULPI)
+				val = DWC2_PHY_TYPE_PARAM_UTMI;
+			else
+				val = DWC2_PHY_TYPE_PARAM_ULPI;
+		}
+		dev_dbg(hsotg->dev, "Setting phy_type to %d\n", val);
+	}
+
+	hsotg->params.phy_type = val;
+}
+
+static int dwc2_get_param_phy_type(struct dwc2_hsotg *hsotg)
+{
+	return hsotg->params.phy_type;
+}
+
+static void dwc2_set_param_speed(struct dwc2_hsotg *hsotg, int val)
+{
+	int valid = 1;
+
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 2)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev, "Wrong value for speed parameter\n");
+			dev_err(hsotg->dev, "max_speed parameter must be 0, 1, or 2\n");
+		}
+		valid = 0;
+	}
+
+	if (dwc2_is_hs_iot(hsotg) &&
+	    val == DWC2_SPEED_PARAM_LOW)
+		valid = 0;
+
+	if (val == DWC2_SPEED_PARAM_HIGH &&
+	    dwc2_get_param_phy_type(hsotg) == DWC2_PHY_TYPE_PARAM_FS)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for speed parameter. Check HW configuration.\n",
+				val);
+		val = dwc2_get_param_phy_type(hsotg) == DWC2_PHY_TYPE_PARAM_FS ?
+				DWC2_SPEED_PARAM_FULL : DWC2_SPEED_PARAM_HIGH;
+		dev_dbg(hsotg->dev, "Setting speed to %d\n", val);
+	}
+
+	hsotg->params.speed = val;
+}
+
+static void dwc2_set_param_host_ls_low_power_phy_clk(struct dwc2_hsotg *hsotg,
+						     int val)
+{
+	int valid = 1;
+
+	if (DWC2_OUT_OF_BOUNDS(val, DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_48MHZ,
+			       DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_6MHZ)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev,
+				"Wrong value for host_ls_low_power_phy_clk parameter\n");
+			dev_err(hsotg->dev,
+				"host_ls_low_power_phy_clk must be 0 or 1\n");
+		}
+		valid = 0;
+	}
+
+	if (val == DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_48MHZ &&
+	    dwc2_get_param_phy_type(hsotg) == DWC2_PHY_TYPE_PARAM_FS)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for host_ls_low_power_phy_clk. Check HW configuration.\n",
+				val);
+		val = dwc2_get_param_phy_type(hsotg) == DWC2_PHY_TYPE_PARAM_FS
+			? DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_6MHZ
+			: DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_48MHZ;
+		dev_dbg(hsotg->dev, "Setting host_ls_low_power_phy_clk to %d\n",
+			val);
+	}
+
+	hsotg->params.host_ls_low_power_phy_clk = val;
+}
+
+static void dwc2_set_param_phy_ulpi_ddr(struct dwc2_hsotg *hsotg, int val)
+{
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev, "Wrong value for phy_ulpi_ddr\n");
+			dev_err(hsotg->dev, "phy_upli_ddr must be 0 or 1\n");
+		}
+		val = 0;
+		dev_dbg(hsotg->dev, "Setting phy_upli_ddr to %d\n", val);
+	}
+
+	hsotg->params.phy_ulpi_ddr = val;
+}
+
+static void dwc2_set_param_phy_ulpi_ext_vbus(struct dwc2_hsotg *hsotg, int val)
+{
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev,
+				"Wrong value for phy_ulpi_ext_vbus\n");
+			dev_err(hsotg->dev,
+				"phy_ulpi_ext_vbus must be 0 or 1\n");
+		}
+		val = 0;
+		dev_dbg(hsotg->dev, "Setting phy_ulpi_ext_vbus to %d\n", val);
+	}
+
+	hsotg->params.phy_ulpi_ext_vbus = val;
+}
+
+static void dwc2_set_param_phy_utmi_width(struct dwc2_hsotg *hsotg, int val)
+{
+	int valid = 0;
+
+	switch (hsotg->hw_params.utmi_phy_data_width) {
+	case GHWCFG4_UTMI_PHY_DATA_WIDTH_8:
+		valid = (val == 8);
+		break;
+	case GHWCFG4_UTMI_PHY_DATA_WIDTH_16:
+		valid = (val == 16);
+		break;
+	case GHWCFG4_UTMI_PHY_DATA_WIDTH_8_OR_16:
+		valid = (val == 8 || val == 16);
+		break;
+	}
+
+	if (!valid) {
+		if (val >= 0) {
+			dev_err(hsotg->dev,
+				"%d invalid for phy_utmi_width. Check HW configuration.\n",
+				val);
+		}
+		val = (hsotg->hw_params.utmi_phy_data_width ==
+		       GHWCFG4_UTMI_PHY_DATA_WIDTH_8) ? 8 : 16;
+		dev_dbg(hsotg->dev, "Setting phy_utmi_width to %d\n", val);
+	}
+
+	hsotg->params.phy_utmi_width = val;
+}
+
+static void dwc2_set_param_ulpi_fs_ls(struct dwc2_hsotg *hsotg, int val)
+{
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev, "Wrong value for ulpi_fs_ls\n");
+			dev_err(hsotg->dev, "ulpi_fs_ls must be 0 or 1\n");
+		}
+		val = 0;
+		dev_dbg(hsotg->dev, "Setting ulpi_fs_ls to %d\n", val);
+	}
+
+	hsotg->params.ulpi_fs_ls = val;
+}
+
+static void dwc2_set_param_ts_dline(struct dwc2_hsotg *hsotg, int val)
+{
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev, "Wrong value for ts_dline\n");
+			dev_err(hsotg->dev, "ts_dline must be 0 or 1\n");
+		}
+		val = 0;
+		dev_dbg(hsotg->dev, "Setting ts_dline to %d\n", val);
+	}
+
+	hsotg->params.ts_dline = val;
+}
+
+static void dwc2_set_param_i2c_enable(struct dwc2_hsotg *hsotg, int val)
+{
+	int valid = 1;
+
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev, "Wrong value for i2c_enable\n");
+			dev_err(hsotg->dev, "i2c_enable must be 0 or 1\n");
+		}
+
+		valid = 0;
+	}
+
+	if (val == 1 && !(hsotg->hw_params.i2c_enable))
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for i2c_enable. Check HW configuration.\n",
+				val);
+		val = hsotg->hw_params.i2c_enable;
+		dev_dbg(hsotg->dev, "Setting i2c_enable to %d\n", val);
+	}
+
+	hsotg->params.i2c_enable = val;
+}
+
+static void dwc2_set_param_en_multiple_tx_fifo(struct dwc2_hsotg *hsotg,
+					       int val)
+{
+	int valid = 1;
+
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev,
+				"Wrong value for en_multiple_tx_fifo,\n");
+			dev_err(hsotg->dev,
+				"en_multiple_tx_fifo must be 0 or 1\n");
+		}
+		valid = 0;
+	}
+
+	if (val == 1 && !hsotg->hw_params.en_multiple_tx_fifo)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for parameter en_multiple_tx_fifo. Check HW configuration.\n",
+				val);
+		val = hsotg->hw_params.en_multiple_tx_fifo;
+		dev_dbg(hsotg->dev, "Setting en_multiple_tx_fifo to %d\n", val);
+	}
+
+	hsotg->params.en_multiple_tx_fifo = val;
+}
+
+static void dwc2_set_param_reload_ctl(struct dwc2_hsotg *hsotg, int val)
+{
+	int valid = 1;
+
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev,
+				"'%d' invalid for parameter reload_ctl\n", val);
+			dev_err(hsotg->dev, "reload_ctl must be 0 or 1\n");
+		}
+		valid = 0;
+	}
+
+	if (val == 1 && hsotg->hw_params.snpsid < DWC2_CORE_REV_2_92a)
+		valid = 0;
+
+	if (!valid) {
+		if (val >= 0)
+			dev_err(hsotg->dev,
+				"%d invalid for parameter reload_ctl. Check HW configuration.\n",
+				val);
+		val = hsotg->hw_params.snpsid >= DWC2_CORE_REV_2_92a;
+		dev_dbg(hsotg->dev, "Setting reload_ctl to %d\n", val);
+	}
+
+	hsotg->params.reload_ctl = val;
+}
+
+static void dwc2_set_param_ahbcfg(struct dwc2_hsotg *hsotg, int val)
+{
+	if (val != -1)
+		hsotg->params.ahbcfg = val;
+	else
+		hsotg->params.ahbcfg = GAHBCFG_HBSTLEN_INCR4 <<
+						GAHBCFG_HBSTLEN_SHIFT;
+}
+
+static void dwc2_set_param_otg_ver(struct dwc2_hsotg *hsotg, int val)
+{
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev,
+				"'%d' invalid for parameter otg_ver\n", val);
+			dev_err(hsotg->dev,
+				"otg_ver must be 0 (for OTG 1.3 support) or 1 (for OTG 2.0 support)\n");
+		}
+		val = 0;
+		dev_dbg(hsotg->dev, "Setting otg_ver to %d\n", val);
+	}
+
+	hsotg->params.otg_ver = val;
+}
+
+static void dwc2_set_param_uframe_sched(struct dwc2_hsotg *hsotg, int val)
+{
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev,
+				"'%d' invalid for parameter uframe_sched\n",
+				val);
+			dev_err(hsotg->dev, "uframe_sched must be 0 or 1\n");
+		}
+		val = 1;
+		dev_dbg(hsotg->dev, "Setting uframe_sched to %d\n", val);
+	}
+
+	hsotg->params.uframe_sched = val;
+}
+
+static void dwc2_set_param_external_id_pin_ctl(struct dwc2_hsotg *hsotg,
+					       int val)
+{
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev,
+				"'%d' invalid for parameter external_id_pin_ctl\n",
+				val);
+			dev_err(hsotg->dev, "external_id_pin_ctl must be 0 or 1\n");
+		}
+		val = 0;
+		dev_dbg(hsotg->dev, "Setting external_id_pin_ctl to %d\n", val);
+	}
+
+	hsotg->params.external_id_pin_ctl = val;
+}
+
+static void dwc2_set_param_hibernation(struct dwc2_hsotg *hsotg,
+				       int val)
+{
+	if (DWC2_OUT_OF_BOUNDS(val, 0, 1)) {
+		if (val >= 0) {
+			dev_err(hsotg->dev,
+				"'%d' invalid for parameter hibernation\n",
+				val);
+			dev_err(hsotg->dev, "hibernation must be 0 or 1\n");
+		}
+		val = 0;
+		dev_dbg(hsotg->dev, "Setting hibernation to %d\n", val);
+	}
+
+	hsotg->params.hibernation = val;
+}
+
+static void dwc2_set_param_tx_fifo_sizes(struct dwc2_hsotg *hsotg)
+{
+	int i;
+	int num;
+	char *property = "g-tx-fifo-size";
+	struct dwc2_core_params *p = &hsotg->params;
+
+	memset(p->g_tx_fifo_size, 0, sizeof(p->g_tx_fifo_size));
+
+	/* Read tx fifo sizes */
+	num = device_property_read_u32_array(hsotg->dev, property, NULL, 0);
+
+	if (num > 0) {
+		device_property_read_u32_array(hsotg->dev, property,
+					       &p->g_tx_fifo_size[1],
+					       num);
+	} else {
+		u32 p_tx_fifo[] = DWC2_G_P_LEGACY_TX_FIFO_SIZE;
+
+		memcpy(&p->g_tx_fifo_size[1],
+		       p_tx_fifo,
+		       sizeof(p_tx_fifo));
+
+		num = ARRAY_SIZE(p_tx_fifo);
+	}
+
+	for (i = 0; i < num; i++) {
+		if ((i + 1) >= ARRAY_SIZE(p->g_tx_fifo_size))
+			break;
+
+		dev_dbg(hsotg->dev, "Setting %s[%d] to %d\n",
+			property, i + 1, p->g_tx_fifo_size[i + 1]);
+	}
+}
+
+static void dwc2_set_gadget_dma(struct dwc2_hsotg *hsotg)
+{
+	struct dwc2_hw_params *hw = &hsotg->hw_params;
+	struct dwc2_core_params *p = &hsotg->params;
+	bool dma_capable = !(hw->arch == GHWCFG2_SLAVE_ONLY_ARCH);
+
+	/* Buffer DMA */
+	dwc2_set_param_bool(hsotg, &p->g_dma,
+			    false, "gadget-dma",
+			    true, false,
+			    dma_capable);
+
+	/* DMA Descriptor */
+	dwc2_set_param_bool(hsotg, &p->g_dma_desc, false,
+			    "gadget-dma-desc",
+			    p->g_dma, false,
+			    !!hw->dma_desc_enable);
+}
+
+/**
+ * dwc2_set_parameters() - Set all core parameters.
+ *
+ * @hsotg: Programming view of the DWC_otg controller
+ * @params: The parameters to set
+ */
+static void dwc2_set_parameters(struct dwc2_hsotg *hsotg,
+				const struct dwc2_core_params *params)
+{
+	struct dwc2_hw_params *hw = &hsotg->hw_params;
+	struct dwc2_core_params *p = &hsotg->params;
+	bool dma_capable = !(hw->arch == GHWCFG2_SLAVE_ONLY_ARCH);
+
+	dwc2_set_param_otg_cap(hsotg, params->otg_cap);
+	if ((hsotg->dr_mode == USB_DR_MODE_HOST) ||
+	    (hsotg->dr_mode == USB_DR_MODE_OTG)) {
+		dev_dbg(hsotg->dev, "Setting HOST parameters\n");
+
+		dwc2_set_param_bool(hsotg, &p->host_dma,
+				    false, "host-dma",
+				    true, false,
+				    dma_capable);
+	}
+	dwc2_set_param_dma_desc_enable(hsotg, params->dma_desc_enable);
+	dwc2_set_param_dma_desc_fs_enable(hsotg, params->dma_desc_fs_enable);
+
+	dwc2_set_param_host_support_fs_ls_low_power(hsotg,
+			params->host_support_fs_ls_low_power);
+	dwc2_set_param_enable_dynamic_fifo(hsotg,
+			params->enable_dynamic_fifo);
+	dwc2_set_param_host_rx_fifo_size(hsotg,
+			params->host_rx_fifo_size);
+	dwc2_set_param_host_nperio_tx_fifo_size(hsotg,
+			params->host_nperio_tx_fifo_size);
+	dwc2_set_param_host_perio_tx_fifo_size(hsotg,
+			params->host_perio_tx_fifo_size);
+	dwc2_set_param_max_transfer_size(hsotg,
+			params->max_transfer_size);
+	dwc2_set_param_max_packet_count(hsotg,
+			params->max_packet_count);
+	dwc2_set_param_host_channels(hsotg, params->host_channels);
+	dwc2_set_param_phy_type(hsotg, params->phy_type);
+	dwc2_set_param_speed(hsotg, params->speed);
+	dwc2_set_param_host_ls_low_power_phy_clk(hsotg,
+			params->host_ls_low_power_phy_clk);
+	dwc2_set_param_phy_ulpi_ddr(hsotg, params->phy_ulpi_ddr);
+	dwc2_set_param_phy_ulpi_ext_vbus(hsotg,
+			params->phy_ulpi_ext_vbus);
+	dwc2_set_param_phy_utmi_width(hsotg, params->phy_utmi_width);
+	dwc2_set_param_ulpi_fs_ls(hsotg, params->ulpi_fs_ls);
+	dwc2_set_param_ts_dline(hsotg, params->ts_dline);
+	dwc2_set_param_i2c_enable(hsotg, params->i2c_enable);
+	dwc2_set_param_en_multiple_tx_fifo(hsotg,
+			params->en_multiple_tx_fifo);
+	dwc2_set_param_reload_ctl(hsotg, params->reload_ctl);
+	dwc2_set_param_ahbcfg(hsotg, params->ahbcfg);
+	dwc2_set_param_otg_ver(hsotg, params->otg_ver);
+	dwc2_set_param_uframe_sched(hsotg, params->uframe_sched);
+	dwc2_set_param_external_id_pin_ctl(hsotg, params->external_id_pin_ctl);
+	dwc2_set_param_hibernation(hsotg, params->hibernation);
+
+	/*
+	 * Set devicetree-only parameters. These parameters do not
+	 * take any values from @params.
+	 */
+	if ((hsotg->dr_mode == USB_DR_MODE_PERIPHERAL) ||
+	    (hsotg->dr_mode == USB_DR_MODE_OTG)) {
+		dev_dbg(hsotg->dev, "Setting peripheral device properties\n");
+
+		dwc2_set_gadget_dma(hsotg);
+
+		/*
+		 * The values for g_rx_fifo_size (2048) and
+		 * g_np_tx_fifo_size (1024) come from the legacy s3c
+		 * gadget driver. These defaults have been hard-coded
+		 * for some time so many platforms depend on these
+		 * values. Leave them as defaults for now and only
+		 * auto-detect if the hardware does not support the
+		 * default.
+		 */
+		dwc2_set_param_u16(hsotg, &p->g_rx_fifo_size,
+				   true, "g-rx-fifo-size", 2048,
+				   hw->rx_fifo_size,
+				   16, hw->rx_fifo_size);
+
+		dwc2_set_param_u16(hsotg, &p->g_np_tx_fifo_size,
+				   true, "g-np-tx-fifo-size", 1024,
+				   hw->dev_nperio_tx_fifo_size,
+				   16, hw->dev_nperio_tx_fifo_size);
+
+		dwc2_set_param_tx_fifo_sizes(hsotg);
+	}
+}
+
+/*
+ * Gets host hardware parameters. Forces host mode if not currently in
+ * host mode. Should be called immediately after a core soft reset in
+ * order to get the reset values.
+ */
+static void dwc2_get_host_hwparams(struct dwc2_hsotg *hsotg)
+{
+	struct dwc2_hw_params *hw = &hsotg->hw_params;
+	u32 gnptxfsiz;
+	u32 hptxfsiz;
+	bool forced;
+
+	if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL)
+		return;
+
+	forced = dwc2_force_mode_if_needed(hsotg, true);
+
+	gnptxfsiz = dwc2_readl(hsotg->regs + GNPTXFSIZ);
+	hptxfsiz = dwc2_readl(hsotg->regs + HPTXFSIZ);
+	dev_dbg(hsotg->dev, "gnptxfsiz=%08x\n", gnptxfsiz);
+	dev_dbg(hsotg->dev, "hptxfsiz=%08x\n", hptxfsiz);
+
+	if (forced)
+		dwc2_clear_force_mode(hsotg);
+
+	hw->host_nperio_tx_fifo_size = (gnptxfsiz & FIFOSIZE_DEPTH_MASK) >>
+				       FIFOSIZE_DEPTH_SHIFT;
+	hw->host_perio_tx_fifo_size = (hptxfsiz & FIFOSIZE_DEPTH_MASK) >>
+				      FIFOSIZE_DEPTH_SHIFT;
+}
+
+/*
+ * Gets device hardware parameters. Forces device mode if not
+ * currently in device mode. Should be called immediately after a core
+ * soft reset in order to get the reset values.
+ */
+static void dwc2_get_dev_hwparams(struct dwc2_hsotg *hsotg)
+{
+	struct dwc2_hw_params *hw = &hsotg->hw_params;
+	bool forced;
+	u32 gnptxfsiz;
+
+	if (hsotg->dr_mode == USB_DR_MODE_HOST)
+		return;
+
+	forced = dwc2_force_mode_if_needed(hsotg, false);
+
+	gnptxfsiz = dwc2_readl(hsotg->regs + GNPTXFSIZ);
+	dev_dbg(hsotg->dev, "gnptxfsiz=%08x\n", gnptxfsiz);
+
+	if (forced)
+		dwc2_clear_force_mode(hsotg);
+
+	hw->dev_nperio_tx_fifo_size = (gnptxfsiz & FIFOSIZE_DEPTH_MASK) >>
+				       FIFOSIZE_DEPTH_SHIFT;
+}
+
+/**
+ * During device initialization, read various hardware configuration
+ * registers and interpret the contents.
+ */
+int dwc2_get_hwparams(struct dwc2_hsotg *hsotg)
+{
+	struct dwc2_hw_params *hw = &hsotg->hw_params;
+	unsigned int width;
+	u32 hwcfg1, hwcfg2, hwcfg3, hwcfg4;
+	u32 grxfsiz;
+
+	/*
+	 * Attempt to ensure this device is really a DWC_otg Controller.
+	 * Read and verify the GSNPSID register contents. The value should be
+	 * 0x45f42xxx or 0x45f43xxx, which corresponds to either "OT2" or "OT3",
+	 * as in "OTG version 2.xx" or "OTG version 3.xx".
+	 */
+	hw->snpsid = dwc2_readl(hsotg->regs + GSNPSID);
+	if ((hw->snpsid & 0xfffff000) != 0x4f542000 &&
+	    (hw->snpsid & 0xfffff000) != 0x4f543000 &&
+	    (hw->snpsid & 0xffff0000) != 0x55310000 &&
+	    (hw->snpsid & 0xffff0000) != 0x55320000) {
+		dev_err(hsotg->dev, "Bad value for GSNPSID: 0x%08x\n",
+			hw->snpsid);
+		return -ENODEV;
+	}
+
+	dev_dbg(hsotg->dev, "Core Release: %1x.%1x%1x%1x (snpsid=%x)\n",
+		hw->snpsid >> 12 & 0xf, hw->snpsid >> 8 & 0xf,
+		hw->snpsid >> 4 & 0xf, hw->snpsid & 0xf, hw->snpsid);
+
+	hwcfg1 = dwc2_readl(hsotg->regs + GHWCFG1);
+	hwcfg2 = dwc2_readl(hsotg->regs + GHWCFG2);
+	hwcfg3 = dwc2_readl(hsotg->regs + GHWCFG3);
+	hwcfg4 = dwc2_readl(hsotg->regs + GHWCFG4);
+	grxfsiz = dwc2_readl(hsotg->regs + GRXFSIZ);
+
+	dev_dbg(hsotg->dev, "hwcfg1=%08x\n", hwcfg1);
+	dev_dbg(hsotg->dev, "hwcfg2=%08x\n", hwcfg2);
+	dev_dbg(hsotg->dev, "hwcfg3=%08x\n", hwcfg3);
+	dev_dbg(hsotg->dev, "hwcfg4=%08x\n", hwcfg4);
+	dev_dbg(hsotg->dev, "grxfsiz=%08x\n", grxfsiz);
+
+	/*
+	 * Host specific hardware parameters. Reading these parameters
+	 * requires the controller to be in host mode. The mode will
+	 * be forced, if necessary, to read these values.
+	 */
+	dwc2_get_host_hwparams(hsotg);
+	dwc2_get_dev_hwparams(hsotg);
+
+	/* hwcfg1 */
+	hw->dev_ep_dirs = hwcfg1;
+
+	/* hwcfg2 */
+	hw->op_mode = (hwcfg2 & GHWCFG2_OP_MODE_MASK) >>
+		      GHWCFG2_OP_MODE_SHIFT;
+	hw->arch = (hwcfg2 & GHWCFG2_ARCHITECTURE_MASK) >>
+		   GHWCFG2_ARCHITECTURE_SHIFT;
+	hw->enable_dynamic_fifo = !!(hwcfg2 & GHWCFG2_DYNAMIC_FIFO);
+	hw->host_channels = 1 + ((hwcfg2 & GHWCFG2_NUM_HOST_CHAN_MASK) >>
+				GHWCFG2_NUM_HOST_CHAN_SHIFT);
+	hw->hs_phy_type = (hwcfg2 & GHWCFG2_HS_PHY_TYPE_MASK) >>
+			  GHWCFG2_HS_PHY_TYPE_SHIFT;
+	hw->fs_phy_type = (hwcfg2 & GHWCFG2_FS_PHY_TYPE_MASK) >>
+			  GHWCFG2_FS_PHY_TYPE_SHIFT;
+	hw->num_dev_ep = (hwcfg2 & GHWCFG2_NUM_DEV_EP_MASK) >>
+			 GHWCFG2_NUM_DEV_EP_SHIFT;
+	hw->nperio_tx_q_depth =
+		(hwcfg2 & GHWCFG2_NONPERIO_TX_Q_DEPTH_MASK) >>
+		GHWCFG2_NONPERIO_TX_Q_DEPTH_SHIFT << 1;
+	hw->host_perio_tx_q_depth =
+		(hwcfg2 & GHWCFG2_HOST_PERIO_TX_Q_DEPTH_MASK) >>
+		GHWCFG2_HOST_PERIO_TX_Q_DEPTH_SHIFT << 1;
+	hw->dev_token_q_depth =
+		(hwcfg2 & GHWCFG2_DEV_TOKEN_Q_DEPTH_MASK) >>
+		GHWCFG2_DEV_TOKEN_Q_DEPTH_SHIFT;
+
+	/* hwcfg3 */
+	width = (hwcfg3 & GHWCFG3_XFER_SIZE_CNTR_WIDTH_MASK) >>
+		GHWCFG3_XFER_SIZE_CNTR_WIDTH_SHIFT;
+	hw->max_transfer_size = (1 << (width + 11)) - 1;
+	width = (hwcfg3 & GHWCFG3_PACKET_SIZE_CNTR_WIDTH_MASK) >>
+		GHWCFG3_PACKET_SIZE_CNTR_WIDTH_SHIFT;
+	hw->max_packet_count = (1 << (width + 4)) - 1;
+	hw->i2c_enable = !!(hwcfg3 & GHWCFG3_I2C);
+	hw->total_fifo_size = (hwcfg3 & GHWCFG3_DFIFO_DEPTH_MASK) >>
+			      GHWCFG3_DFIFO_DEPTH_SHIFT;
+
+	/* hwcfg4 */
+	hw->en_multiple_tx_fifo = !!(hwcfg4 & GHWCFG4_DED_FIFO_EN);
+	hw->num_dev_perio_in_ep = (hwcfg4 & GHWCFG4_NUM_DEV_PERIO_IN_EP_MASK) >>
+				  GHWCFG4_NUM_DEV_PERIO_IN_EP_SHIFT;
+	hw->dma_desc_enable = !!(hwcfg4 & GHWCFG4_DESC_DMA);
+	hw->power_optimized = !!(hwcfg4 & GHWCFG4_POWER_OPTIMIZ);
+	hw->utmi_phy_data_width = (hwcfg4 & GHWCFG4_UTMI_PHY_DATA_WIDTH_MASK) >>
+				  GHWCFG4_UTMI_PHY_DATA_WIDTH_SHIFT;
+
+	/* fifo sizes */
+	hw->rx_fifo_size = (grxfsiz & GRXFSIZ_DEPTH_MASK) >>
+				GRXFSIZ_DEPTH_SHIFT;
+
+	dev_dbg(hsotg->dev, "Detected values from hardware:\n");
+	dev_dbg(hsotg->dev, "  op_mode=%d\n",
+		hw->op_mode);
+	dev_dbg(hsotg->dev, "  arch=%d\n",
+		hw->arch);
+	dev_dbg(hsotg->dev, "  dma_desc_enable=%d\n",
+		hw->dma_desc_enable);
+	dev_dbg(hsotg->dev, "  power_optimized=%d\n",
+		hw->power_optimized);
+	dev_dbg(hsotg->dev, "  i2c_enable=%d\n",
+		hw->i2c_enable);
+	dev_dbg(hsotg->dev, "  hs_phy_type=%d\n",
+		hw->hs_phy_type);
+	dev_dbg(hsotg->dev, "  fs_phy_type=%d\n",
+		hw->fs_phy_type);
+	dev_dbg(hsotg->dev, "  utmi_phy_data_width=%d\n",
+		hw->utmi_phy_data_width);
+	dev_dbg(hsotg->dev, "  num_dev_ep=%d\n",
+		hw->num_dev_ep);
+	dev_dbg(hsotg->dev, "  num_dev_perio_in_ep=%d\n",
+		hw->num_dev_perio_in_ep);
+	dev_dbg(hsotg->dev, "  host_channels=%d\n",
+		hw->host_channels);
+	dev_dbg(hsotg->dev, "  max_transfer_size=%d\n",
+		hw->max_transfer_size);
+	dev_dbg(hsotg->dev, "  max_packet_count=%d\n",
+		hw->max_packet_count);
+	dev_dbg(hsotg->dev, "  nperio_tx_q_depth=0x%0x\n",
+		hw->nperio_tx_q_depth);
+	dev_dbg(hsotg->dev, "  host_perio_tx_q_depth=0x%0x\n",
+		hw->host_perio_tx_q_depth);
+	dev_dbg(hsotg->dev, "  dev_token_q_depth=0x%0x\n",
+		hw->dev_token_q_depth);
+	dev_dbg(hsotg->dev, "  enable_dynamic_fifo=%d\n",
+		hw->enable_dynamic_fifo);
+	dev_dbg(hsotg->dev, "  en_multiple_tx_fifo=%d\n",
+		hw->en_multiple_tx_fifo);
+	dev_dbg(hsotg->dev, "  total_fifo_size=%d\n",
+		hw->total_fifo_size);
+	dev_dbg(hsotg->dev, "  rx_fifo_size=%d\n",
+		hw->rx_fifo_size);
+	dev_dbg(hsotg->dev, "  host_nperio_tx_fifo_size=%d\n",
+		hw->host_nperio_tx_fifo_size);
+	dev_dbg(hsotg->dev, "  host_perio_tx_fifo_size=%d\n",
+		hw->host_perio_tx_fifo_size);
+	dev_dbg(hsotg->dev, "\n");
+
+	return 0;
+}
+
+int dwc2_init_params(struct dwc2_hsotg *hsotg)
+{
+	const struct of_device_id *match;
+	struct dwc2_core_params params;
+
+	match = of_match_device(dwc2_of_match_table, hsotg->dev);
+	if (match && match->data)
+		params = *((struct dwc2_core_params *)match->data);
+	else
+		params = params_default;
+
+	if (dwc2_is_fs_iot(hsotg)) {
+		params.speed = DWC2_SPEED_PARAM_FULL;
+		params.phy_type = DWC2_PHY_TYPE_PARAM_FS;
+	}
+
+	dwc2_set_parameters(hsotg, &params);
+
+	return 0;
+}
diff --git a/drivers/usb/dwc2/pci.c b/drivers/usb/dwc2/pci.c
index ae419615a176..a23329e3d7cd 100644
--- a/drivers/usb/dwc2/pci.c
+++ b/drivers/usb/dwc2/pci.c
@@ -62,6 +62,20 @@ struct dwc2_pci_glue {
 	struct platform_device *phy;
 };
 
+static int dwc2_pci_quirks(struct pci_dev *pdev, struct platform_device *dwc2)
+{
+	if (pdev->vendor == PCI_VENDOR_ID_SYNOPSYS &&
+	    pdev->device == PCI_PRODUCT_ID_HAPS_HSOTG) {
+		struct property_entry properties[] = {
+			{ },
+		};
+
+		return platform_device_add_properties(dwc2, properties);
+	}
+
+	return 0;
+}
+
 static void dwc2_pci_remove(struct pci_dev *pci)
 {
 	struct dwc2_pci_glue *glue = pci_get_drvdata(pci);
@@ -122,6 +136,10 @@ static int dwc2_pci_probe(struct pci_dev *pci,
 		return PTR_ERR(phy);
 	}
 
+	ret = dwc2_pci_quirks(pci, dwc2);
+	if (ret)
+		goto err;
+
 	ret = platform_device_add(dwc2);
 	if (ret) {
 		dev_err(dev, "failed to register dwc2 device\n");
diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c
index 8e1728b39a49..4fc8c603afb8 100644
--- a/drivers/usb/dwc2/platform.c
+++ b/drivers/usb/dwc2/platform.c
@@ -55,165 +55,6 @@
 
 static const char dwc2_driver_name[] = "dwc2";
 
-static const struct dwc2_core_params params_hi6220 = {
-	.otg_cap			= 2,	/* No HNP/SRP capable */
-	.otg_ver			= 0,	/* 1.3 */
-	.dma_enable			= 1,
-	.dma_desc_enable		= 0,
-	.dma_desc_fs_enable		= 0,
-	.speed				= 0,	/* High Speed */
-	.enable_dynamic_fifo		= 1,
-	.en_multiple_tx_fifo		= 1,
-	.host_rx_fifo_size		= 512,
-	.host_nperio_tx_fifo_size	= 512,
-	.host_perio_tx_fifo_size	= 512,
-	.max_transfer_size		= 65535,
-	.max_packet_count		= 511,
-	.host_channels			= 16,
-	.phy_type			= 1,	/* UTMI */
-	.phy_utmi_width			= 8,
-	.phy_ulpi_ddr			= 0,	/* Single */
-	.phy_ulpi_ext_vbus		= 0,
-	.i2c_enable			= 0,
-	.ulpi_fs_ls			= 0,
-	.host_support_fs_ls_low_power	= 0,
-	.host_ls_low_power_phy_clk	= 0,	/* 48 MHz */
-	.ts_dline			= 0,
-	.reload_ctl			= 0,
-	.ahbcfg				= GAHBCFG_HBSTLEN_INCR16 <<
-					  GAHBCFG_HBSTLEN_SHIFT,
-	.uframe_sched			= 0,
-	.external_id_pin_ctl		= -1,
-	.hibernation			= -1,
-};
-
-static const struct dwc2_core_params params_bcm2835 = {
-	.otg_cap			= 0,	/* HNP/SRP capable */
-	.otg_ver			= 0,	/* 1.3 */
-	.dma_enable			= 1,
-	.dma_desc_enable		= 0,
-	.dma_desc_fs_enable		= 0,
-	.speed				= 0,	/* High Speed */
-	.enable_dynamic_fifo		= 1,
-	.en_multiple_tx_fifo		= 1,
-	.host_rx_fifo_size		= 774,	/* 774 DWORDs */
-	.host_nperio_tx_fifo_size	= 256,	/* 256 DWORDs */
-	.host_perio_tx_fifo_size	= 512,	/* 512 DWORDs */
-	.max_transfer_size		= 65535,
-	.max_packet_count		= 511,
-	.host_channels			= 8,
-	.phy_type			= 1,	/* UTMI */
-	.phy_utmi_width			= 8,	/* 8 bits */
-	.phy_ulpi_ddr			= 0,	/* Single */
-	.phy_ulpi_ext_vbus		= 0,
-	.i2c_enable			= 0,
-	.ulpi_fs_ls			= 0,
-	.host_support_fs_ls_low_power	= 0,
-	.host_ls_low_power_phy_clk	= 0,	/* 48 MHz */
-	.ts_dline			= 0,
-	.reload_ctl			= 0,
-	.ahbcfg				= 0x10,
-	.uframe_sched			= 0,
-	.external_id_pin_ctl		= -1,
-	.hibernation			= -1,
-};
-
-static const struct dwc2_core_params params_rk3066 = {
-	.otg_cap			= 2,	/* non-HNP/non-SRP */
-	.otg_ver			= -1,
-	.dma_enable			= -1,
-	.dma_desc_enable		= 0,
-	.dma_desc_fs_enable		= 0,
-	.speed				= -1,
-	.enable_dynamic_fifo		= 1,
-	.en_multiple_tx_fifo		= -1,
-	.host_rx_fifo_size		= 525,	/* 525 DWORDs */
-	.host_nperio_tx_fifo_size	= 128,	/* 128 DWORDs */
-	.host_perio_tx_fifo_size	= 256,	/* 256 DWORDs */
-	.max_transfer_size		= -1,
-	.max_packet_count		= -1,
-	.host_channels			= -1,
-	.phy_type			= -1,
-	.phy_utmi_width			= -1,
-	.phy_ulpi_ddr			= -1,
-	.phy_ulpi_ext_vbus		= -1,
-	.i2c_enable			= -1,
-	.ulpi_fs_ls			= -1,
-	.host_support_fs_ls_low_power	= -1,
-	.host_ls_low_power_phy_clk	= -1,
-	.ts_dline			= -1,
-	.reload_ctl			= -1,
-	.ahbcfg				= GAHBCFG_HBSTLEN_INCR16 <<
-					  GAHBCFG_HBSTLEN_SHIFT,
-	.uframe_sched			= -1,
-	.external_id_pin_ctl		= -1,
-	.hibernation			= -1,
-};
-
-static const struct dwc2_core_params params_ltq = {
-	.otg_cap			= 2,	/* non-HNP/non-SRP */
-	.otg_ver			= -1,
-	.dma_enable			= -1,
-	.dma_desc_enable		= -1,
-	.dma_desc_fs_enable		= -1,
-	.speed				= -1,
-	.enable_dynamic_fifo		= -1,
-	.en_multiple_tx_fifo		= -1,
-	.host_rx_fifo_size		= 288,	/* 288 DWORDs */
-	.host_nperio_tx_fifo_size	= 128,	/* 128 DWORDs */
-	.host_perio_tx_fifo_size	= 96,	/* 96 DWORDs */
-	.max_transfer_size		= 65535,
-	.max_packet_count		= 511,
-	.host_channels			= -1,
-	.phy_type			= -1,
-	.phy_utmi_width			= -1,
-	.phy_ulpi_ddr			= -1,
-	.phy_ulpi_ext_vbus		= -1,
-	.i2c_enable			= -1,
-	.ulpi_fs_ls			= -1,
-	.host_support_fs_ls_low_power	= -1,
-	.host_ls_low_power_phy_clk	= -1,
-	.ts_dline			= -1,
-	.reload_ctl			= -1,
-	.ahbcfg				= GAHBCFG_HBSTLEN_INCR16 <<
-					  GAHBCFG_HBSTLEN_SHIFT,
-	.uframe_sched			= -1,
-	.external_id_pin_ctl		= -1,
-	.hibernation			= -1,
-};
-
-static const struct dwc2_core_params params_amlogic = {
-	.otg_cap			= DWC2_CAP_PARAM_NO_HNP_SRP_CAPABLE,
-	.otg_ver			= -1,
-	.dma_enable			= 1,
-	.dma_desc_enable		= 0,
-	.dma_desc_fs_enable		= 0,
-	.speed				= DWC2_SPEED_PARAM_HIGH,
-	.enable_dynamic_fifo		= 1,
-	.en_multiple_tx_fifo		= -1,
-	.host_rx_fifo_size		= 512,
-	.host_nperio_tx_fifo_size	= 500,
-	.host_perio_tx_fifo_size	= 500,
-	.max_transfer_size		= -1,
-	.max_packet_count		= -1,
-	.host_channels			= 16,
-	.phy_type			= DWC2_PHY_TYPE_PARAM_UTMI,
-	.phy_utmi_width			= -1,
-	.phy_ulpi_ddr			= -1,
-	.phy_ulpi_ext_vbus		= -1,
-	.i2c_enable			= -1,
-	.ulpi_fs_ls			= -1,
-	.host_support_fs_ls_low_power	= -1,
-	.host_ls_low_power_phy_clk	= -1,
-	.ts_dline			= -1,
-	.reload_ctl			= 1,
-	.ahbcfg				= GAHBCFG_HBSTLEN_INCR8 <<
-					  GAHBCFG_HBSTLEN_SHIFT,
-	.uframe_sched			= 0,
-	.external_id_pin_ctl		= -1,
-	.hibernation			= -1,
-};
-
 /*
  * Check the dr_mode against the module configuration and hardware
  * capabilities.
@@ -510,20 +351,6 @@ static void dwc2_driver_shutdown(struct platform_device *dev)
 	disable_irq(hsotg->irq);
 }
 
-static const struct of_device_id dwc2_of_match_table[] = {
-	{ .compatible = "brcm,bcm2835-usb", .data = &params_bcm2835 },
-	{ .compatible = "hisilicon,hi6220-usb", .data = &params_hi6220 },
-	{ .compatible = "rockchip,rk3066-usb", .data = &params_rk3066 },
-	{ .compatible = "lantiq,arx100-usb", .data = &params_ltq },
-	{ .compatible = "lantiq,xrx200-usb", .data = &params_ltq },
-	{ .compatible = "snps,dwc2", .data = NULL },
-	{ .compatible = "samsung,s3c6400-hsotg", .data = NULL},
-	{ .compatible = "amlogic,meson8b-usb", .data = &params_amlogic },
-	{ .compatible = "amlogic,meson-gxbb-usb", .data = &params_amlogic },
-	{},
-};
-MODULE_DEVICE_TABLE(of, dwc2_of_match_table);
-
 /**
  * dwc2_driver_probe() - Called when the DWC_otg core is bound to the DWC_otg
  * driver
@@ -538,30 +365,10 @@ MODULE_DEVICE_TABLE(of, dwc2_of_match_table);
  */
 static int dwc2_driver_probe(struct platform_device *dev)
 {
-	const struct of_device_id *match;
-	const struct dwc2_core_params *params;
-	struct dwc2_core_params defparams;
 	struct dwc2_hsotg *hsotg;
 	struct resource *res;
 	int retval;
 
-	match = of_match_device(dwc2_of_match_table, &dev->dev);
-	if (match && match->data) {
-		params = match->data;
-	} else {
-		/* Default all params to autodetect */
-		dwc2_set_all_params(&defparams, -1);
-		params = &defparams;
-
-		/*
-		 * Disable descriptor dma mode by default as the HW can support
-		 * it, but does not support it for SPLIT transactions.
-		 * Disable it for FS devices as well.
-		 */
-		defparams.dma_desc_enable = 0;
-		defparams.dma_desc_fs_enable = 0;
-	}
-
 	hsotg = devm_kzalloc(&dev->dev, sizeof(*hsotg), GFP_KERNEL);
 	if (!hsotg)
 		return -ENOMEM;
@@ -591,13 +398,6 @@ static int dwc2_driver_probe(struct platform_device *dev)
 
 	spin_lock_init(&hsotg->lock);
 
-	hsotg->core_params = devm_kzalloc(&dev->dev,
-				sizeof(*hsotg->core_params), GFP_KERNEL);
-	if (!hsotg->core_params)
-		return -ENOMEM;
-
-	dwc2_set_all_params(hsotg->core_params, -1);
-
 	hsotg->irq = platform_get_irq(dev, 0);
 	if (hsotg->irq < 0) {
 		dev_err(&dev->dev, "missing IRQ resource\n");
@@ -631,11 +431,12 @@ static int dwc2_driver_probe(struct platform_device *dev)
 	if (retval)
 		goto error;
 
-	/* Validate parameter values */
-	dwc2_set_parameters(hsotg, params);
-
 	dwc2_force_dr_mode(hsotg);
 
+	retval = dwc2_init_params(hsotg);
+	if (retval)
+		goto error;
+
 	if (hsotg->dr_mode != USB_DR_MODE_HOST) {
 		retval = dwc2_gadget_init(hsotg, hsotg->irq);
 		if (retval)
diff --git a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig
index b97cde76914d..c5aa235863e8 100644
--- a/drivers/usb/dwc3/Kconfig
+++ b/drivers/usb/dwc3/Kconfig
@@ -62,7 +62,7 @@ config USB_DWC3_OMAP
 
 config USB_DWC3_EXYNOS
 	tristate "Samsung Exynos Platform"
-	depends on ARCH_EXYNOS && OF || COMPILE_TEST
+	depends on (ARCH_EXYNOS || COMPILE_TEST) && OF
 	default USB_DWC3
 	help
 	  Recent Exynos5 SoCs ship with one DesignWare Core USB3 IP inside,
@@ -70,7 +70,7 @@ config USB_DWC3_EXYNOS
 
 config USB_DWC3_PCI
 	tristate "PCIe-based Platforms"
-	depends on PCI
+	depends on PCI && ACPI
 	default USB_DWC3
 	help
 	  If you're using the DesignWare Core IP with a PCIe, please say
@@ -98,7 +98,7 @@ config USB_DWC3_OF_SIMPLE
 
 config USB_DWC3_ST
 	tristate "STMicroelectronics Platforms"
-	depends on ARCH_STI && OF
+	depends on (ARCH_STI || COMPILE_TEST) && OF
 	default USB_DWC3
 	help
 	  STMicroelectronics SoCs with one DesignWare Core USB3 IP
diff --git a/drivers/usb/dwc3/Makefile b/drivers/usb/dwc3/Makefile
index 22420e17d68b..ffca34029b21 100644
--- a/drivers/usb/dwc3/Makefile
+++ b/drivers/usb/dwc3/Makefile
@@ -3,7 +3,11 @@ CFLAGS_trace.o				:= -I$(src)
 
 obj-$(CONFIG_USB_DWC3)			+= dwc3.o
 
-dwc3-y					:= core.o debug.o trace.o
+dwc3-y					:= core.o
+
+ifneq ($(CONFIG_FTRACE),)
+	dwc3-y				+= trace.o
+endif
 
 ifneq ($(filter y,$(CONFIG_USB_DWC3_HOST) $(CONFIG_USB_DWC3_DUAL_ROLE)),)
 	dwc3-y				+= host.o
diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c
index fea446900cad..369bab16a824 100644
--- a/drivers/usb/dwc3/core.c
+++ b/drivers/usb/dwc3/core.c
@@ -169,33 +169,6 @@ static int dwc3_core_soft_reset(struct dwc3 *dwc)
 	return -ETIMEDOUT;
 }
 
-/**
- * dwc3_soft_reset - Issue soft reset
- * @dwc: Pointer to our controller context structure
- */
-static int dwc3_soft_reset(struct dwc3 *dwc)
-{
-	unsigned long timeout;
-	u32 reg;
-
-	timeout = jiffies + msecs_to_jiffies(500);
-	dwc3_writel(dwc->regs, DWC3_DCTL, DWC3_DCTL_CSFTRST);
-	do {
-		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
-		if (!(reg & DWC3_DCTL_CSFTRST))
-			break;
-
-		if (time_after(jiffies, timeout)) {
-			dev_err(dwc->dev, "Reset Timed Out\n");
-			return -ETIMEDOUT;
-		}
-
-		cpu_relax();
-	} while (true);
-
-	return 0;
-}
-
 /*
  * dwc3_frame_length_adjustment - Adjusts frame length if required
  * @dwc3: Pointer to our controller context structure
@@ -229,7 +202,7 @@ static void dwc3_frame_length_adjustment(struct dwc3 *dwc)
 static void dwc3_free_one_event_buffer(struct dwc3 *dwc,
 		struct dwc3_event_buffer *evt)
 {
-	dma_free_coherent(dwc->dev, evt->length, evt->buf, evt->dma);
+	dma_free_coherent(dwc->sysdev, evt->length, evt->buf, evt->dma);
 }
 
 /**
@@ -251,7 +224,11 @@ static struct dwc3_event_buffer *dwc3_alloc_one_event_buffer(struct dwc3 *dwc,
 
 	evt->dwc	= dwc;
 	evt->length	= length;
-	evt->buf	= dma_alloc_coherent(dwc->dev, length,
+	evt->cache	= devm_kzalloc(dwc->dev, length, GFP_KERNEL);
+	if (!evt->cache)
+		return ERR_PTR(-ENOMEM);
+
+	evt->buf	= dma_alloc_coherent(dwc->sysdev, length,
 			&evt->dma, GFP_KERNEL);
 	if (!evt->buf)
 		return ERR_PTR(-ENOMEM);
@@ -305,13 +282,7 @@ static int dwc3_event_buffers_setup(struct dwc3 *dwc)
 	struct dwc3_event_buffer	*evt;
 
 	evt = dwc->ev_buf;
-	dwc3_trace(trace_dwc3_core,
-			"Event buf %p dma %08llx length %d\n",
-			evt->buf, (unsigned long long) evt->dma,
-			evt->length);
-
 	evt->lpos = 0;
-
 	dwc3_writel(dwc->regs, DWC3_GEVNTADRLO(0),
 			lower_32_bits(evt->dma));
 	dwc3_writel(dwc->regs, DWC3_GEVNTADRHI(0),
@@ -370,11 +341,11 @@ static int dwc3_setup_scratch_buffers(struct dwc3 *dwc)
 	if (!WARN_ON(dwc->scratchbuf))
 		return 0;
 
-	scratch_addr = dma_map_single(dwc->dev, dwc->scratchbuf,
+	scratch_addr = dma_map_single(dwc->sysdev, dwc->scratchbuf,
 			dwc->nr_scratch * DWC3_SCRATCHBUF_SIZE,
 			DMA_BIDIRECTIONAL);
-	if (dma_mapping_error(dwc->dev, scratch_addr)) {
-		dev_err(dwc->dev, "failed to map scratch buffer\n");
+	if (dma_mapping_error(dwc->sysdev, scratch_addr)) {
+		dev_err(dwc->sysdev, "failed to map scratch buffer\n");
 		ret = -EFAULT;
 		goto err0;
 	}
@@ -398,7 +369,7 @@ static int dwc3_setup_scratch_buffers(struct dwc3 *dwc)
 	return 0;
 
 err1:
-	dma_unmap_single(dwc->dev, dwc->scratch_addr, dwc->nr_scratch *
+	dma_unmap_single(dwc->sysdev, dwc->scratch_addr, dwc->nr_scratch *
 			DWC3_SCRATCHBUF_SIZE, DMA_BIDIRECTIONAL);
 
 err0:
@@ -417,7 +388,7 @@ static void dwc3_free_scratch_buffers(struct dwc3 *dwc)
 	if (!WARN_ON(dwc->scratchbuf))
 		return;
 
-	dma_unmap_single(dwc->dev, dwc->scratch_addr, dwc->nr_scratch *
+	dma_unmap_single(dwc->sysdev, dwc->scratch_addr, dwc->nr_scratch *
 			DWC3_SCRATCHBUF_SIZE, DMA_BIDIRECTIONAL);
 	kfree(dwc->scratchbuf);
 }
@@ -428,9 +399,6 @@ static void dwc3_core_num_eps(struct dwc3 *dwc)
 
 	dwc->num_in_eps = DWC3_NUM_IN_EPS(parms);
 	dwc->num_out_eps = DWC3_NUM_EPS(parms) - dwc->num_in_eps;
-
-	dwc3_trace(trace_dwc3_core, "found %d IN and %d OUT endpoints",
-			dwc->num_in_eps, dwc->num_out_eps);
 }
 
 static void dwc3_cache_hwparams(struct dwc3 *dwc)
@@ -524,13 +492,6 @@ static int dwc3_phy_setup(struct dwc3 *dwc)
 		}
 		/* FALLTHROUGH */
 	case DWC3_GHWPARAMS3_HSPHY_IFC_ULPI:
-		/* Making sure the interface and PHY are operational */
-		ret = dwc3_soft_reset(dwc);
-		if (ret)
-			return ret;
-
-		udelay(1);
-
 		ret = dwc3_ulpi_init(dwc);
 		if (ret)
 			return ret;
@@ -594,19 +555,12 @@ static void dwc3_core_exit(struct dwc3 *dwc)
 	phy_power_off(dwc->usb3_generic_phy);
 }
 
-/**
- * dwc3_core_init - Low-level initialization of DWC3 Core
- * @dwc: Pointer to our controller context structure
- *
- * Returns 0 on success otherwise negative errno.
- */
-static int dwc3_core_init(struct dwc3 *dwc)
+static bool dwc3_core_is_valid(struct dwc3 *dwc)
 {
-	u32			hwparams4 = dwc->hwparams.hwparams4;
-	u32			reg;
-	int			ret;
+	u32 reg;
 
 	reg = dwc3_readl(dwc->regs, DWC3_GSNPSID);
+
 	/* This should read as U3 followed by revision number */
 	if ((reg & DWC3_GSNPSID_MASK) == 0x55330000) {
 		/* Detected DWC_usb3 IP */
@@ -616,36 +570,16 @@ static int dwc3_core_init(struct dwc3 *dwc)
 		dwc->revision = dwc3_readl(dwc->regs, DWC3_VER_NUMBER);
 		dwc->revision |= DWC3_REVISION_IS_DWC31;
 	} else {
-		dev_err(dwc->dev, "this is not a DesignWare USB3 DRD Core\n");
-		ret = -ENODEV;
-		goto err0;
+		return false;
 	}
 
-	/*
-	 * Write Linux Version Code to our GUID register so it's easy to figure
-	 * out which kernel version a bug was found.
-	 */
-	dwc3_writel(dwc->regs, DWC3_GUID, LINUX_VERSION_CODE);
-
-	/* Handle USB2.0-only core configuration */
-	if (DWC3_GHWPARAMS3_SSPHY_IFC(dwc->hwparams.hwparams3) ==
-			DWC3_GHWPARAMS3_SSPHY_IFC_DIS) {
-		if (dwc->maximum_speed == USB_SPEED_SUPER)
-			dwc->maximum_speed = USB_SPEED_HIGH;
-	}
-
-	/* issue device SoftReset too */
-	ret = dwc3_soft_reset(dwc);
-	if (ret)
-		goto err0;
-
-	ret = dwc3_core_soft_reset(dwc);
-	if (ret)
-		goto err0;
+	return true;
+}
 
-	ret = dwc3_phy_setup(dwc);
-	if (ret)
-		goto err0;
+static void dwc3_core_setup_global_control(struct dwc3 *dwc)
+{
+	u32 hwparams4 = dwc->hwparams.hwparams4;
+	u32 reg;
 
 	reg = dwc3_readl(dwc->regs, DWC3_GCTL);
 	reg &= ~DWC3_GCTL_SCALEDOWN_MASK;
@@ -683,13 +617,13 @@ static int dwc3_core_init(struct dwc3 *dwc)
 		reg |= DWC3_GCTL_GBLHIBERNATIONEN;
 		break;
 	default:
-		dwc3_trace(trace_dwc3_core, "No power optimization available\n");
+		/* nothing */
+		break;
 	}
 
 	/* check if current dwc3 is on simulation board */
 	if (dwc->hwparams.hwparams6 & DWC3_GHWPARAMS6_EN_FPGA) {
-		dwc3_trace(trace_dwc3_core,
-				"running on FPGA platform\n");
+		dev_info(dwc->dev, "Running with FPGA optmizations\n");
 		dwc->is_fpga = true;
 	}
 
@@ -714,7 +648,47 @@ static int dwc3_core_init(struct dwc3 *dwc)
 		reg |= DWC3_GCTL_U2RSTECN;
 
 	dwc3_writel(dwc->regs, DWC3_GCTL, reg);
+}
+
+/**
+ * dwc3_core_init - Low-level initialization of DWC3 Core
+ * @dwc: Pointer to our controller context structure
+ *
+ * Returns 0 on success otherwise negative errno.
+ */
+static int dwc3_core_init(struct dwc3 *dwc)
+{
+	u32			reg;
+	int			ret;
+
+	if (!dwc3_core_is_valid(dwc)) {
+		dev_err(dwc->dev, "this is not a DesignWare USB3 DRD Core\n");
+		ret = -ENODEV;
+		goto err0;
+	}
+
+	/*
+	 * Write Linux Version Code to our GUID register so it's easy to figure
+	 * out which kernel version a bug was found.
+	 */
+	dwc3_writel(dwc->regs, DWC3_GUID, LINUX_VERSION_CODE);
+
+	/* Handle USB2.0-only core configuration */
+	if (DWC3_GHWPARAMS3_SSPHY_IFC(dwc->hwparams.hwparams3) ==
+			DWC3_GHWPARAMS3_SSPHY_IFC_DIS) {
+		if (dwc->maximum_speed == USB_SPEED_SUPER)
+			dwc->maximum_speed = USB_SPEED_HIGH;
+	}
+
+	ret = dwc3_core_soft_reset(dwc);
+	if (ret)
+		goto err0;
 
+	ret = dwc3_phy_setup(dwc);
+	if (ret)
+		goto err0;
+
+	dwc3_core_setup_global_control(dwc);
 	dwc3_core_num_eps(dwc);
 
 	ret = dwc3_setup_scratch_buffers(dwc);
@@ -766,6 +740,16 @@ static int dwc3_core_init(struct dwc3 *dwc)
 		dwc3_writel(dwc->regs, DWC3_GUCTL2, reg);
 	}
 
+	/*
+	 * Enable hardware control of sending remote wakeup in HS when
+	 * the device is in the L1 state.
+	 */
+	if (dwc->revision >= DWC3_REVISION_290A) {
+		reg = dwc3_readl(dwc->regs, DWC3_GUCTL1);
+		reg |= DWC3_GUCTL1_DEV_L1_EXIT_BY_HW;
+		dwc3_writel(dwc->regs, DWC3_GUCTL1, reg);
+	}
+
 	return 0;
 
 err4:
@@ -919,57 +903,13 @@ static void dwc3_core_exit_mode(struct dwc3 *dwc)
 	}
 }
 
-#define DWC3_ALIGN_MASK		(16 - 1)
-
-static int dwc3_probe(struct platform_device *pdev)
+static void dwc3_get_properties(struct dwc3 *dwc)
 {
-	struct device		*dev = &pdev->dev;
-	struct resource		*res;
-	struct dwc3		*dwc;
+	struct device		*dev = dwc->dev;
 	u8			lpm_nyet_threshold;
 	u8			tx_de_emphasis;
 	u8			hird_threshold;
 
-	int			ret;
-
-	void __iomem		*regs;
-	void			*mem;
-
-	mem = devm_kzalloc(dev, sizeof(*dwc) + DWC3_ALIGN_MASK, GFP_KERNEL);
-	if (!mem)
-		return -ENOMEM;
-
-	dwc = PTR_ALIGN(mem, DWC3_ALIGN_MASK + 1);
-	dwc->mem = mem;
-	dwc->dev = dev;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		dev_err(dev, "missing memory resource\n");
-		return -ENODEV;
-	}
-
-	dwc->xhci_resources[0].start = res->start;
-	dwc->xhci_resources[0].end = dwc->xhci_resources[0].start +
-					DWC3_XHCI_REGS_END;
-	dwc->xhci_resources[0].flags = res->flags;
-	dwc->xhci_resources[0].name = res->name;
-
-	res->start += DWC3_GLOBALS_REGS_START;
-
-	/*
-	 * Request memory region but exclude xHCI regs,
-	 * since it will be requested by the xhci-plat driver.
-	 */
-	regs = devm_ioremap_resource(dev, res);
-	if (IS_ERR(regs)) {
-		ret = PTR_ERR(regs);
-		goto err0;
-	}
-
-	dwc->regs	= regs;
-	dwc->regs_size	= resource_size(res);
-
 	/* default to highest possible threshold */
 	lpm_nyet_threshold = 0xff;
 
@@ -986,6 +926,13 @@ static int dwc3_probe(struct platform_device *pdev)
 	dwc->dr_mode = usb_get_dr_mode(dev);
 	dwc->hsphy_mode = of_usb_get_phy_mode(dev->of_node);
 
+	dwc->sysdev_is_parent = device_property_read_bool(dev,
+				"linux,sysdev_is_parent");
+	if (dwc->sysdev_is_parent)
+		dwc->sysdev = dwc->dev->parent;
+	else
+		dwc->sysdev = dwc->dev;
+
 	dwc->has_lpm_erratum = device_property_read_bool(dev,
 				"snps,has-lpm-erratum");
 	device_property_read_u8(dev, "snps,lpm-nyet-threshold",
@@ -1041,6 +988,112 @@ static int dwc3_probe(struct platform_device *pdev)
 	dwc->hird_threshold = hird_threshold
 		| (dwc->is_utmi_l1_suspend << 4);
 
+	dwc->imod_interval = 0;
+}
+
+/* check whether the core supports IMOD */
+bool dwc3_has_imod(struct dwc3 *dwc)
+{
+	return ((dwc3_is_usb3(dwc) &&
+		 dwc->revision >= DWC3_REVISION_300A) ||
+		(dwc3_is_usb31(dwc) &&
+		 dwc->revision >= DWC3_USB31_REVISION_120A));
+}
+
+static void dwc3_check_params(struct dwc3 *dwc)
+{
+	struct device *dev = dwc->dev;
+
+	/* Check for proper value of imod_interval */
+	if (dwc->imod_interval && !dwc3_has_imod(dwc)) {
+		dev_warn(dwc->dev, "Interrupt moderation not supported\n");
+		dwc->imod_interval = 0;
+	}
+
+	/*
+	 * Workaround for STAR 9000961433 which affects only version
+	 * 3.00a of the DWC_usb3 core. This prevents the controller
+	 * interrupt from being masked while handling events. IMOD
+	 * allows us to work around this issue. Enable it for the
+	 * affected version.
+	 */
+	if (!dwc->imod_interval &&
+	    (dwc->revision == DWC3_REVISION_300A))
+		dwc->imod_interval = 1;
+
+	/* Check the maximum_speed parameter */
+	switch (dwc->maximum_speed) {
+	case USB_SPEED_LOW:
+	case USB_SPEED_FULL:
+	case USB_SPEED_HIGH:
+	case USB_SPEED_SUPER:
+	case USB_SPEED_SUPER_PLUS:
+		break;
+	default:
+		dev_err(dev, "invalid maximum_speed parameter %d\n",
+			dwc->maximum_speed);
+		/* fall through */
+	case USB_SPEED_UNKNOWN:
+		/* default to superspeed */
+		dwc->maximum_speed = USB_SPEED_SUPER;
+
+		/*
+		 * default to superspeed plus if we are capable.
+		 */
+		if (dwc3_is_usb31(dwc) &&
+		    (DWC3_GHWPARAMS3_SSPHY_IFC(dwc->hwparams.hwparams3) ==
+		     DWC3_GHWPARAMS3_SSPHY_IFC_GEN2))
+			dwc->maximum_speed = USB_SPEED_SUPER_PLUS;
+
+		break;
+	}
+}
+
+static int dwc3_probe(struct platform_device *pdev)
+{
+	struct device		*dev = &pdev->dev;
+	struct resource		*res;
+	struct dwc3		*dwc;
+
+	int			ret;
+
+	void __iomem		*regs;
+
+	dwc = devm_kzalloc(dev, sizeof(*dwc), GFP_KERNEL);
+	if (!dwc)
+		return -ENOMEM;
+
+	dwc->dev = dev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(dev, "missing memory resource\n");
+		return -ENODEV;
+	}
+
+	dwc->xhci_resources[0].start = res->start;
+	dwc->xhci_resources[0].end = dwc->xhci_resources[0].start +
+					DWC3_XHCI_REGS_END;
+	dwc->xhci_resources[0].flags = res->flags;
+	dwc->xhci_resources[0].name = res->name;
+
+	res->start += DWC3_GLOBALS_REGS_START;
+
+	/*
+	 * Request memory region but exclude xHCI regs,
+	 * since it will be requested by the xhci-plat driver.
+	 */
+	regs = devm_ioremap_resource(dev, res);
+	if (IS_ERR(regs)) {
+		ret = PTR_ERR(regs);
+		goto err0;
+	}
+
+	dwc->regs	= regs;
+	dwc->regs_size	= resource_size(res);
+
+	dwc3_get_properties(dwc);
+
 	platform_set_drvdata(pdev, dwc);
 	dwc3_cache_hwparams(dwc);
 
@@ -1050,12 +1103,6 @@ static int dwc3_probe(struct platform_device *pdev)
 
 	spin_lock_init(&dwc->lock);
 
-	if (!dev->dma_mask) {
-		dev->dma_mask = dev->parent->dma_mask;
-		dev->dma_parms = dev->parent->dma_parms;
-		dma_set_coherent_mask(dev, dev->parent->coherent_dma_mask);
-	}
-
 	pm_runtime_set_active(dev);
 	pm_runtime_use_autosuspend(dev);
 	pm_runtime_set_autosuspend_delay(dev, DWC3_DEFAULT_AUTOSUSPEND_DELAY);
@@ -1087,32 +1134,7 @@ static int dwc3_probe(struct platform_device *pdev)
 		goto err4;
 	}
 
-	/* Check the maximum_speed parameter */
-	switch (dwc->maximum_speed) {
-	case USB_SPEED_LOW:
-	case USB_SPEED_FULL:
-	case USB_SPEED_HIGH:
-	case USB_SPEED_SUPER:
-	case USB_SPEED_SUPER_PLUS:
-		break;
-	default:
-		dev_err(dev, "invalid maximum_speed parameter %d\n",
-			dwc->maximum_speed);
-		/* fall through */
-	case USB_SPEED_UNKNOWN:
-		/* default to superspeed */
-		dwc->maximum_speed = USB_SPEED_SUPER;
-
-		/*
-		 * default to superspeed plus if we are capable.
-		 */
-		if (dwc3_is_usb31(dwc) &&
-		    (DWC3_GHWPARAMS3_SSPHY_IFC(dwc->hwparams.hwparams3) ==
-		     DWC3_GHWPARAMS3_SSPHY_IFC_GEN2))
-			dwc->maximum_speed = USB_SPEED_SUPER_PLUS;
-
-		break;
-	}
+	dwc3_check_params(dwc);
 
 	ret = dwc3_core_init_mode(dwc);
 	if (ret)
diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h
index 6b60e42626a2..de5a8570be04 100644
--- a/drivers/usb/dwc3/core.h
+++ b/drivers/usb/dwc3/core.h
@@ -26,6 +26,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/mm.h>
 #include <linux/debugfs.h>
+#include <linux/wait.h>
 
 #include <linux/usb/ch9.h>
 #include <linux/usb/gadget.h>
@@ -37,6 +38,7 @@
 #define DWC3_MSG_MAX	500
 
 /* Global constants */
+#define DWC3_PULL_UP_TIMEOUT	500	/* ms */
 #define DWC3_ZLP_BUF_SIZE	1024	/* size of a superspeed bulk */
 #define DWC3_EP0_BOUNCE_SIZE	512
 #define DWC3_ENDPOINTS_NUM	32
@@ -65,6 +67,7 @@
 #define DWC3_DEVICE_EVENT_OVERFLOW		11
 
 #define DWC3_GEVNTCOUNT_MASK	0xfffc
+#define DWC3_GEVNTCOUNT_EHB	(1 << 31)
 #define DWC3_GSNPSID_MASK	0xffff0000
 #define DWC3_GSNPSREV_MASK	0xffff
 
@@ -147,6 +150,8 @@
 #define DWC3_DEPCMDPAR0		0x08
 #define DWC3_DEPCMD		0x0c
 
+#define DWC3_DEV_IMOD(n)	(0xca00 + (n * 0x4))
+
 /* OTG Registers */
 #define DWC3_OCFG		0xcc00
 #define DWC3_OCTL		0xcc04
@@ -198,6 +203,9 @@
 #define DWC3_GCTL_GBLHIBERNATIONEN	(1 << 1)
 #define DWC3_GCTL_DSBLCLKGTNG		(1 << 0)
 
+/* Global User Control 1 Register */
+#define DWC3_GUCTL1_DEV_L1_EXIT_BY_HW	(1 << 24)
+
 /* Global USB2 PHY Configuration Register */
 #define DWC3_GUSB2PHYCFG_PHYSOFTRST	(1 << 31)
 #define DWC3_GUSB2PHYCFG_U2_FREECLK_EXISTS	(1 << 30)
@@ -450,6 +458,8 @@
 #define DWC3_DEPCMD_SETTRANSFRESOURCE	(0x02 << 0)
 #define DWC3_DEPCMD_SETEPCONFIG		(0x01 << 0)
 
+#define DWC3_DEPCMD_CMD(x)		((x) & 0xf)
+
 /* The EP number goes 0..31 so ep0 is always out and ep1 is always in */
 #define DWC3_DALEPENA_EP(n)		(1 << n)
 
@@ -458,6 +468,11 @@
 #define DWC3_DEPCMD_TYPE_BULK		2
 #define DWC3_DEPCMD_TYPE_INTR		3
 
+#define DWC3_DEV_IMOD_COUNT_SHIFT	16
+#define DWC3_DEV_IMOD_COUNT_MASK	(0xffff << 16)
+#define DWC3_DEV_IMOD_INTERVAL_SHIFT	0
+#define DWC3_DEV_IMOD_INTERVAL_MASK	(0xffff << 0)
+
 /* Structures */
 
 struct dwc3_trb;
@@ -465,6 +480,7 @@ struct dwc3_trb;
 /**
  * struct dwc3_event_buffer - Software event buffer representation
  * @buf: _THE_ buffer
+ * @cache: The buffer cache used in the threaded interrupt
  * @length: size of this buffer
  * @lpos: event offset
  * @count: cache of last read event count register
@@ -474,6 +490,7 @@ struct dwc3_trb;
  */
 struct dwc3_event_buffer {
 	void			*buf;
+	void			*cache;
 	unsigned		length;
 	unsigned int		lpos;
 	unsigned int		count;
@@ -499,6 +516,7 @@ struct dwc3_event_buffer {
  * @endpoint: usb endpoint
  * @pending_list: list of pending requests for this endpoint
  * @started_list: list of started requests on this endpoint
+ * @wait_end_transfer: wait_queue_head_t for waiting on End Transfer complete
  * @lock: spinlock for endpoint request queue traversal
  * @regs: pointer to first endpoint register
  * @trb_pool: array of transaction buffers
@@ -524,12 +542,13 @@ struct dwc3_ep {
 	struct list_head	pending_list;
 	struct list_head	started_list;
 
+	wait_queue_head_t	wait_end_transfer;
+
 	spinlock_t		lock;
 	void __iomem		*regs;
 
 	struct dwc3_trb		*trb_pool;
 	dma_addr_t		trb_pool_dma;
-	const struct usb_ss_ep_comp_descriptor *comp_desc;
 	struct dwc3		*dwc;
 
 	u32			saved_state;
@@ -540,6 +559,8 @@ struct dwc3_ep {
 #define DWC3_EP_BUSY		(1 << 4)
 #define DWC3_EP_PENDING_REQUEST	(1 << 5)
 #define DWC3_EP_MISSED_ISOC	(1 << 6)
+#define DWC3_EP_END_TRANSFER_PENDING	(1 << 7)
+#define DWC3_EP_TRANSFER_STARTED (1 << 8)
 
 	/* This last one is specific to EP0 */
 #define DWC3_EP0_DIR_IN		(1 << 31)
@@ -703,7 +724,7 @@ struct dwc3_hwparams {
  * @dep: struct dwc3_ep owning this request
  * @sg: pointer to first incomplete sg
  * @num_pending_sgs: counter to pending sgs
- * @first_trb_index: index to first trb used by this request
+ * @remaining: amount of data remaining
  * @epnum: endpoint number to which this request refers
  * @trb: pointer to struct dwc3_trb
  * @trb_dma: DMA address of @trb
@@ -718,7 +739,7 @@ struct dwc3_request {
 	struct scatterlist	*sg;
 
 	unsigned		num_pending_sgs;
-	u8			first_trb_index;
+	unsigned		remaining;
 	u8			epnum;
 	struct dwc3_trb		*trb;
 	dma_addr_t		trb_dma;
@@ -748,6 +769,7 @@ struct dwc3_scratchpad_array {
  * @ep0_usb_req: dummy req used while handling STD USB requests
  * @ep0_bounce_addr: dma address of ep0_bounce
  * @scratch_addr: dma address of scratchbuf
+ * @ep0_in_setup: one control transfer is completed and enter setup phase
  * @lock: for synchronizing
  * @dev: pointer to our struct device
  * @xhci: pointer to our xHCI child
@@ -784,7 +806,6 @@ struct dwc3_scratchpad_array {
  * @ep0state: state of endpoint zero
  * @link_state: link state
  * @speed: device speed (super, high, full, low)
- * @mem: points to start of memory which is used for this struct.
  * @hwparams: copy of hwparams registers
  * @root: debugfs root folder pointer
  * @regset: debugfs pointer to regdump file
@@ -798,6 +819,7 @@ struct dwc3_scratchpad_array {
  * @ep0_bounced: true when we used bounce buffer
  * @ep0_expect_in: true when we expect a DATA IN transfer
  * @has_hibernation: true when dwc3 was configured with Hibernation
+ * @sysdev_is_parent: true when dwc3 device has a parent driver
  * @has_lpm_erratum: true when core was configured with LPM Erratum. Note that
  *			there's now way for software to detect this in runtime.
  * @is_utmi_l1_suspend: the core asserts output signal
@@ -833,6 +855,8 @@ struct dwc3_scratchpad_array {
  * 	1	- -3.5dB de-emphasis
  * 	2	- No de-emphasis
  * 	3	- Reserved
+ * @imod_interval: set the interrupt moderation interval in 250ns
+ *                 increments or 0 to disable.
  */
 struct dwc3 {
 	struct usb_ctrlrequest	*ctrl_req;
@@ -846,11 +870,13 @@ struct dwc3 {
 	dma_addr_t		ep0_bounce_addr;
 	dma_addr_t		scratch_addr;
 	struct dwc3_request	ep0_usb_req;
+	struct completion	ep0_in_setup;
 
 	/* device lock */
 	spinlock_t		lock;
 
 	struct device		*dev;
+	struct device		*sysdev;
 
 	struct platform_device	*xhci;
 	struct resource		xhci_resources[DWC3_XHCI_RESOURCES_NUM];
@@ -909,6 +935,7 @@ struct dwc3 {
 #define DWC3_REVISION_260A	0x5533260a
 #define DWC3_REVISION_270A	0x5533270a
 #define DWC3_REVISION_280A	0x5533280a
+#define DWC3_REVISION_290A	0x5533290a
 #define DWC3_REVISION_300A	0x5533300a
 #define DWC3_REVISION_310A	0x5533310a
 
@@ -918,6 +945,7 @@ struct dwc3 {
  */
 #define DWC3_REVISION_IS_DWC31		0x80000000
 #define DWC3_USB31_REVISION_110A	(0x3131302a | DWC3_REVISION_IS_DWC31)
+#define DWC3_USB31_REVISION_120A	(0x3132302a | DWC3_REVISION_IS_DWC31)
 
 	enum dwc3_ep0_next	ep0_next_event;
 	enum dwc3_ep0_state	ep0state;
@@ -934,8 +962,6 @@ struct dwc3 {
 	u8			num_out_eps;
 	u8			num_in_eps;
 
-	void			*mem;
-
 	struct dwc3_hwparams	hwparams;
 	struct dentry		*root;
 	struct debugfs_regset32	*regset;
@@ -952,6 +978,7 @@ struct dwc3 {
 	unsigned		ep0_bounced:1;
 	unsigned		ep0_expect_in:1;
 	unsigned		has_hibernation:1;
+	unsigned		sysdev_is_parent:1;
 	unsigned		has_lpm_erratum:1;
 	unsigned		is_utmi_l1_suspend:1;
 	unsigned		is_fpga:1;
@@ -978,6 +1005,8 @@ struct dwc3 {
 
 	unsigned		tx_de_emphasis_quirk:1;
 	unsigned		tx_de_emphasis:2;
+
+	u16			imod_interval;
 };
 
 /* -------------------------------------------------------------------------- */
@@ -1039,12 +1068,16 @@ struct dwc3_event_depevt {
 /* Control-only Status */
 #define DEPEVT_STATUS_CONTROL_DATA	1
 #define DEPEVT_STATUS_CONTROL_STATUS	2
+#define DEPEVT_STATUS_CONTROL_PHASE(n)	((n) & 3)
 
 /* In response to Start Transfer */
 #define DEPEVT_TRANSFER_NO_RESOURCE	1
 #define DEPEVT_TRANSFER_BUS_EXPIRY	2
 
 	u32	parameters:16;
+
+/* For Command Complete Events */
+#define DEPEVT_PARAMETER_CMD(n)	(((n) & (0xf << 8)) >> 8)
 } __packed;
 
 /**
@@ -1133,12 +1166,20 @@ struct dwc3_gadget_ep_cmd_params {
 void dwc3_set_mode(struct dwc3 *dwc, u32 mode);
 u32 dwc3_core_fifo_space(struct dwc3_ep *dep, u8 type);
 
+/* check whether we are on the DWC_usb3 core */
+static inline bool dwc3_is_usb3(struct dwc3 *dwc)
+{
+	return !(dwc->revision & DWC3_REVISION_IS_DWC31);
+}
+
 /* check whether we are on the DWC_usb31 core */
 static inline bool dwc3_is_usb31(struct dwc3 *dwc)
 {
 	return !!(dwc->revision & DWC3_REVISION_IS_DWC31);
 }
 
+bool dwc3_has_imod(struct dwc3 *dwc);
+
 #if IS_ENABLED(CONFIG_USB_DWC3_HOST) || IS_ENABLED(CONFIG_USB_DWC3_DUAL_ROLE)
 int dwc3_host_init(struct dwc3 *dwc);
 void dwc3_host_exit(struct dwc3 *dwc);
diff --git a/drivers/usb/dwc3/debug.c b/drivers/usb/dwc3/debug.c
deleted file mode 100644
index 0be6885bc370..000000000000
--- a/drivers/usb/dwc3/debug.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * debug.c - DesignWare USB3 DRD Controller Debug/Trace Support
- *
- * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com
- *
- * Author: Felipe Balbi <balbi@ti.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2  of
- * the License as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include "debug.h"
-
-void dwc3_trace(void (*trace)(struct va_format *), const char *fmt, ...)
-{
-	struct va_format vaf;
-	va_list args;
-
-	va_start(args, fmt);
-	vaf.fmt = fmt;
-	vaf.va = &args;
-
-	trace(&vaf);
-
-	va_end(args);
-}
diff --git a/drivers/usb/dwc3/debug.h b/drivers/usb/dwc3/debug.h
index 33ab2a203c1b..eeed4ffd8131 100644
--- a/drivers/usb/dwc3/debug.h
+++ b/drivers/usb/dwc3/debug.h
@@ -124,6 +124,22 @@ dwc3_gadget_link_string(enum dwc3_link_state link_state)
 	}
 }
 
+static inline const char *dwc3_ep0_state_string(enum dwc3_ep0_state state)
+{
+	switch (state) {
+	case EP0_UNCONNECTED:
+		return "Unconnected";
+	case EP0_SETUP_PHASE:
+		return "Setup Phase";
+	case EP0_DATA_PHASE:
+		return "Data Phase";
+	case EP0_STATUS_PHASE:
+		return "Status Phase";
+	default:
+		return "UNKNOWN";
+	}
+}
+
 /**
  * dwc3_gadget_event_string - returns event name
  * @event: the event code
@@ -184,10 +200,11 @@ dwc3_gadget_event_string(const struct dwc3_event_devt *event)
  * @event: then event code
  */
 static inline const char *
-dwc3_ep_event_string(const struct dwc3_event_depevt *event)
+dwc3_ep_event_string(const struct dwc3_event_depevt *event, u32 ep0state)
 {
 	u8 epnum = event->endpoint_number;
 	static char str[256];
+	size_t len;
 	int status;
 	int ret;
 
@@ -199,6 +216,10 @@ dwc3_ep_event_string(const struct dwc3_event_depevt *event)
 	switch (event->endpoint_event) {
 	case DWC3_DEPEVT_XFERCOMPLETE:
 		strcat(str, "Transfer Complete");
+		len = strlen(str);
+
+		if (epnum <= 1)
+			sprintf(str + len, " [%s]", dwc3_ep0_state_string(ep0state));
 		break;
 	case DWC3_DEPEVT_XFERINPROGRESS:
 		strcat(str, "Transfer In-Progress");
@@ -207,6 +228,19 @@ dwc3_ep_event_string(const struct dwc3_event_depevt *event)
 		strcat(str, "Transfer Not Ready");
 		status = event->status & DEPEVT_STATUS_TRANSFER_ACTIVE;
 		strcat(str, status ? " (Active)" : " (Not Active)");
+
+		/* Control Endpoints */
+		if (epnum <= 1) {
+			int phase = DEPEVT_STATUS_CONTROL_PHASE(event->status);
+
+			switch (phase) {
+			case DEPEVT_STATUS_CONTROL_DATA:
+				strcat(str, " [Data Phase]");
+				break;
+			case DEPEVT_STATUS_CONTROL_STATUS:
+				strcat(str, " [Status Phase]");
+			}
+		}
 		break;
 	case DWC3_DEPEVT_RXTXFIFOEVT:
 		strcat(str, "FIFO");
@@ -270,14 +304,14 @@ static inline const char *dwc3_gadget_event_type_string(u8 event)
 	}
 }
 
-static inline const char *dwc3_decode_event(u32 event)
+static inline const char *dwc3_decode_event(u32 event, u32 ep0state)
 {
 	const union dwc3_event evt = (union dwc3_event) event;
 
 	if (evt.type.is_devspec)
 		return dwc3_gadget_event_string(&evt.devt);
 	else
-		return dwc3_ep_event_string(&evt.depevt);
+		return dwc3_ep_event_string(&evt.depevt, ep0state);
 }
 
 static inline const char *dwc3_ep_cmd_status_string(int status)
@@ -310,7 +344,6 @@ static inline const char *dwc3_gadget_generic_cmd_status_string(int status)
 	}
 }
 
-void dwc3_trace(void (*trace)(struct va_format *), const char *fmt, ...);
 
 #ifdef CONFIG_DEBUG_FS
 extern void dwc3_debugfs_init(struct dwc3 *);
diff --git a/drivers/usb/dwc3/dwc3-exynos.c b/drivers/usb/dwc3/dwc3-exynos.c
index 2f1fb7e7aa54..e27899bb5706 100644
--- a/drivers/usb/dwc3/dwc3-exynos.c
+++ b/drivers/usb/dwc3/dwc3-exynos.c
@@ -20,7 +20,6 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/platform_device.h>
-#include <linux/dma-mapping.h>
 #include <linux/clk.h>
 #include <linux/usb/otg.h>
 #include <linux/usb/usb_phy_generic.h>
@@ -117,15 +116,6 @@ static int dwc3_exynos_probe(struct platform_device *pdev)
 	if (!exynos)
 		return -ENOMEM;
 
-	/*
-	 * Right now device-tree probed devices don't get dma_mask set.
-	 * Since shared usb code relies on it, set it here for now.
-	 * Once we move to full device tree support this will vanish off.
-	 */
-	ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32));
-	if (ret)
-		return ret;
-
 	platform_set_drvdata(pdev, exynos);
 
 	exynos->dev	= dev;
diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c
index 6df0f5dad9a4..2b73339f286b 100644
--- a/drivers/usb/dwc3/dwc3-pci.c
+++ b/drivers/usb/dwc3/dwc3-pci.c
@@ -39,6 +39,27 @@
 #define PCI_DEVICE_ID_INTEL_APL			0x5aaa
 #define PCI_DEVICE_ID_INTEL_KBP			0xa2b0
 
+#define PCI_INTEL_BXT_DSM_UUID		"732b85d5-b7a7-4a1b-9ba0-4bbd00ffd511"
+#define PCI_INTEL_BXT_FUNC_PMU_PWR	4
+#define PCI_INTEL_BXT_STATE_D0		0
+#define PCI_INTEL_BXT_STATE_D3		3
+
+/**
+ * struct dwc3_pci - Driver private structure
+ * @dwc3: child dwc3 platform_device
+ * @pci: our link to PCI bus
+ * @uuid: _DSM UUID
+ * @has_dsm_for_pm: true for devices which need to run _DSM on runtime PM
+ */
+struct dwc3_pci {
+	struct platform_device *dwc3;
+	struct pci_dev *pci;
+
+	u8 uuid[16];
+
+	unsigned int has_dsm_for_pm:1;
+};
+
 static const struct acpi_gpio_params reset_gpios = { 0, 0, false };
 static const struct acpi_gpio_params cs_gpios = { 1, 0, false };
 
@@ -48,8 +69,21 @@ static const struct acpi_gpio_mapping acpi_dwc3_byt_gpios[] = {
 	{ },
 };
 
-static int dwc3_pci_quirks(struct pci_dev *pdev, struct platform_device *dwc3)
+static int dwc3_pci_quirks(struct dwc3_pci *dwc)
 {
+	struct platform_device		*dwc3 = dwc->dwc3;
+	struct pci_dev			*pdev = dwc->pci;
+	int				ret;
+
+	struct property_entry sysdev_property[] = {
+		PROPERTY_ENTRY_BOOL("linux,sysdev_is_parent"),
+		{ },
+	};
+
+	ret = platform_device_add_properties(dwc3, sysdev_property);
+	if (ret)
+		return ret;
+
 	if (pdev->vendor == PCI_VENDOR_ID_AMD &&
 	    pdev->device == PCI_DEVICE_ID_AMD_NL_USB) {
 		struct property_entry properties[] = {
@@ -89,6 +123,12 @@ static int dwc3_pci_quirks(struct pci_dev *pdev, struct platform_device *dwc3)
 		if (ret < 0)
 			return ret;
 
+		if (pdev->device == PCI_DEVICE_ID_INTEL_BXT ||
+				pdev->device == PCI_DEVICE_ID_INTEL_BXT_M) {
+			acpi_str_to_uuid(PCI_INTEL_BXT_DSM_UUID, dwc->uuid);
+			dwc->has_dsm_for_pm = true;
+		}
+
 		if (pdev->device == PCI_DEVICE_ID_INTEL_BYT) {
 			struct gpio_desc *gpio;
 
@@ -139,8 +179,8 @@ static int dwc3_pci_quirks(struct pci_dev *pdev, struct platform_device *dwc3)
 static int dwc3_pci_probe(struct pci_dev *pci,
 		const struct pci_device_id *id)
 {
+	struct dwc3_pci		*dwc;
 	struct resource		res[2];
-	struct platform_device	*dwc3;
 	int			ret;
 	struct device		*dev = &pci->dev;
 
@@ -152,11 +192,13 @@ static int dwc3_pci_probe(struct pci_dev *pci,
 
 	pci_set_master(pci);
 
-	dwc3 = platform_device_alloc("dwc3", PLATFORM_DEVID_AUTO);
-	if (!dwc3) {
-		dev_err(dev, "couldn't allocate dwc3 device\n");
+	dwc = devm_kzalloc(dev, sizeof(*dwc), GFP_KERNEL);
+	if (!dwc)
+		return -ENOMEM;
+
+	dwc->dwc3 = platform_device_alloc("dwc3", PLATFORM_DEVID_AUTO);
+	if (!dwc->dwc3)
 		return -ENOMEM;
-	}
 
 	memset(res, 0x00, sizeof(struct resource) * ARRAY_SIZE(res));
 
@@ -169,20 +211,21 @@ static int dwc3_pci_probe(struct pci_dev *pci,
 	res[1].name	= "dwc_usb3";
 	res[1].flags	= IORESOURCE_IRQ;
 
-	ret = platform_device_add_resources(dwc3, res, ARRAY_SIZE(res));
+	ret = platform_device_add_resources(dwc->dwc3, res, ARRAY_SIZE(res));
 	if (ret) {
 		dev_err(dev, "couldn't add resources to dwc3 device\n");
 		return ret;
 	}
 
-	dwc3->dev.parent = dev;
-	ACPI_COMPANION_SET(&dwc3->dev, ACPI_COMPANION(dev));
+	dwc->pci = pci;
+	dwc->dwc3->dev.parent = dev;
+	ACPI_COMPANION_SET(&dwc->dwc3->dev, ACPI_COMPANION(dev));
 
-	ret = dwc3_pci_quirks(pci, dwc3);
+	ret = dwc3_pci_quirks(dwc);
 	if (ret)
 		goto err;
 
-	ret = platform_device_add(dwc3);
+	ret = platform_device_add(dwc->dwc3);
 	if (ret) {
 		dev_err(dev, "failed to register dwc3 device\n");
 		goto err;
@@ -190,21 +233,23 @@ static int dwc3_pci_probe(struct pci_dev *pci,
 
 	device_init_wakeup(dev, true);
 	device_set_run_wake(dev, true);
-	pci_set_drvdata(pci, dwc3);
+	pci_set_drvdata(pci, dwc);
 	pm_runtime_put(dev);
 
 	return 0;
 err:
-	platform_device_put(dwc3);
+	platform_device_put(dwc->dwc3);
 	return ret;
 }
 
 static void dwc3_pci_remove(struct pci_dev *pci)
 {
+	struct dwc3_pci		*dwc = pci_get_drvdata(pci);
+
 	device_init_wakeup(&pci->dev, false);
 	pm_runtime_get(&pci->dev);
 	acpi_dev_remove_driver_gpios(ACPI_COMPANION(&pci->dev));
-	platform_device_unregister(pci_get_drvdata(pci));
+	platform_device_unregister(dwc->dwc3);
 }
 
 static const struct pci_device_id dwc3_pci_id_table[] = {
@@ -234,40 +279,75 @@ static const struct pci_device_id dwc3_pci_id_table[] = {
 };
 MODULE_DEVICE_TABLE(pci, dwc3_pci_id_table);
 
+#if defined(CONFIG_PM) || defined(CONFIG_PM_SLEEP)
+static int dwc3_pci_dsm(struct dwc3_pci *dwc, int param)
+{
+	union acpi_object *obj;
+	union acpi_object tmp;
+	union acpi_object argv4 = ACPI_INIT_DSM_ARGV4(1, &tmp);
+
+	if (!dwc->has_dsm_for_pm)
+		return 0;
+
+	tmp.type = ACPI_TYPE_INTEGER;
+	tmp.integer.value = param;
+
+	obj = acpi_evaluate_dsm(ACPI_HANDLE(&dwc->pci->dev), dwc->uuid,
+			1, PCI_INTEL_BXT_FUNC_PMU_PWR, &argv4);
+	if (!obj) {
+		dev_err(&dwc->pci->dev, "failed to evaluate _DSM\n");
+		return -EIO;
+	}
+
+	ACPI_FREE(obj);
+
+	return 0;
+}
+#endif /* CONFIG_PM || CONFIG_PM_SLEEP */
+
 #ifdef CONFIG_PM
 static int dwc3_pci_runtime_suspend(struct device *dev)
 {
+	struct dwc3_pci		*dwc = dev_get_drvdata(dev);
+
 	if (device_run_wake(dev))
-		return 0;
+		return dwc3_pci_dsm(dwc, PCI_INTEL_BXT_STATE_D3);
 
 	return -EBUSY;
 }
 
 static int dwc3_pci_runtime_resume(struct device *dev)
 {
-	struct platform_device *dwc3 = dev_get_drvdata(dev);
+	struct dwc3_pci		*dwc = dev_get_drvdata(dev);
+	struct platform_device	*dwc3 = dwc->dwc3;
+	int			ret;
+
+	ret = dwc3_pci_dsm(dwc, PCI_INTEL_BXT_STATE_D0);
+	if (ret)
+		return ret;
 
 	return pm_runtime_get(&dwc3->dev);
 }
 #endif /* CONFIG_PM */
 
 #ifdef CONFIG_PM_SLEEP
-static int dwc3_pci_pm_dummy(struct device *dev)
+static int dwc3_pci_suspend(struct device *dev)
 {
-	/*
-	 * There's nothing to do here. No, seriously. Everything is either taken
-	 * care either by PCI subsystem or dwc3/core.c, so we have nothing
-	 * missing here.
-	 *
-	 * So you'd think we didn't need this at all, but PCI subsystem will
-	 * bail out if we don't have a valid callback :-s
-	 */
-	return 0;
+	struct dwc3_pci		*dwc = dev_get_drvdata(dev);
+
+	return dwc3_pci_dsm(dwc, PCI_INTEL_BXT_STATE_D3);
+}
+
+static int dwc3_pci_resume(struct device *dev)
+{
+	struct dwc3_pci		*dwc = dev_get_drvdata(dev);
+
+	return dwc3_pci_dsm(dwc, PCI_INTEL_BXT_STATE_D0);
 }
 #endif /* CONFIG_PM_SLEEP */
 
 static struct dev_pm_ops dwc3_pci_dev_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(dwc3_pci_pm_dummy, dwc3_pci_pm_dummy)
+	SET_SYSTEM_SLEEP_PM_OPS(dwc3_pci_suspend, dwc3_pci_resume)
 	SET_RUNTIME_PM_OPS(dwc3_pci_runtime_suspend, dwc3_pci_runtime_resume,
 		NULL)
 };
diff --git a/drivers/usb/dwc3/dwc3-st.c b/drivers/usb/dwc3/dwc3-st.c
index aaaf256f71dd..dfbf464eb88c 100644
--- a/drivers/usb/dwc3/dwc3-st.c
+++ b/drivers/usb/dwc3/dwc3-st.c
@@ -219,7 +219,6 @@ static int st_dwc3_probe(struct platform_device *pdev)
 	if (IS_ERR(regmap))
 		return PTR_ERR(regmap);
 
-	dma_set_coherent_mask(dev, dev->coherent_dma_mask);
 	dwc3_data->dev = dev;
 	dwc3_data->regmap = regmap;
 
diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c
index fe79d771dee4..4878d187c7d4 100644
--- a/drivers/usb/dwc3/ep0.c
+++ b/drivers/usb/dwc3/ep0.c
@@ -39,22 +39,6 @@ static void __dwc3_ep0_do_control_status(struct dwc3 *dwc, struct dwc3_ep *dep);
 static void __dwc3_ep0_do_control_data(struct dwc3 *dwc,
 		struct dwc3_ep *dep, struct dwc3_request *req);
 
-static const char *dwc3_ep0_state_string(enum dwc3_ep0_state state)
-{
-	switch (state) {
-	case EP0_UNCONNECTED:
-		return "Unconnected";
-	case EP0_SETUP_PHASE:
-		return "Setup Phase";
-	case EP0_DATA_PHASE:
-		return "Data Phase";
-	case EP0_STATUS_PHASE:
-		return "Status Phase";
-	default:
-		return "UNKNOWN";
-	}
-}
-
 static int dwc3_ep0_start_trans(struct dwc3 *dwc, u8 epnum, dma_addr_t buf_dma,
 		u32 len, u32 type, bool chain)
 {
@@ -65,10 +49,8 @@ static int dwc3_ep0_start_trans(struct dwc3 *dwc, u8 epnum, dma_addr_t buf_dma,
 	int				ret;
 
 	dep = dwc->eps[epnum];
-	if (dep->flags & DWC3_EP_BUSY) {
-		dwc3_trace(trace_dwc3_ep0, "%s still busy", dep->name);
+	if (dep->flags & DWC3_EP_BUSY)
 		return 0;
-	}
 
 	trb = &dwc->ep0_trb[dep->trb_enqueue];
 
@@ -99,11 +81,8 @@ static int dwc3_ep0_start_trans(struct dwc3 *dwc, u8 epnum, dma_addr_t buf_dma,
 	trace_dwc3_prepare_trb(dep, trb);
 
 	ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_STARTTRANSFER, &params);
-	if (ret < 0) {
-		dwc3_trace(trace_dwc3_ep0, "%s STARTTRANSFER failed",
-				dep->name);
+	if (ret < 0)
 		return ret;
-	}
 
 	dep->flags |= DWC3_EP_BUSY;
 	dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
@@ -163,9 +142,6 @@ static int __dwc3_gadget_ep0_queue(struct dwc3_ep *dep,
 
 		if (dwc->ep0state == EP0_STATUS_PHASE)
 			__dwc3_ep0_do_control_status(dwc, dwc->eps[direction]);
-		else
-			dwc3_trace(trace_dwc3_ep0,
-					"too early for delayed status");
 
 		return 0;
 	}
@@ -229,9 +205,8 @@ int dwc3_gadget_ep0_queue(struct usb_ep *ep, struct usb_request *request,
 
 	spin_lock_irqsave(&dwc->lock, flags);
 	if (!dep->endpoint.desc) {
-		dwc3_trace(trace_dwc3_ep0,
-				"trying to queue request %p to disabled %s",
-				request, dep->name);
+		dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
+				dep->name);
 		ret = -ESHUTDOWN;
 		goto out;
 	}
@@ -242,11 +217,6 @@ int dwc3_gadget_ep0_queue(struct usb_ep *ep, struct usb_request *request,
 		goto out;
 	}
 
-	dwc3_trace(trace_dwc3_ep0,
-			"queueing request %p to %s length %d state '%s'",
-			request, dep->name, request->length,
-			dwc3_ep0_state_string(dwc->ep0state));
-
 	ret = __dwc3_gadget_ep0_queue(dep, req);
 
 out:
@@ -308,6 +278,8 @@ void dwc3_ep0_out_start(struct dwc3 *dwc)
 {
 	int				ret;
 
+	complete(&dwc->ep0_in_setup);
+
 	ret = dwc3_ep0_start_trans(dwc, 0, dwc->ctrl_req_addr, 8,
 			DWC3_TRBCTL_CONTROL_SETUP, false);
 	WARN_ON(ret < 0);
@@ -395,126 +367,198 @@ static int dwc3_ep0_handle_status(struct dwc3 *dwc,
 	return __dwc3_gadget_ep0_queue(dep, &dwc->ep0_usb_req);
 }
 
-static int dwc3_ep0_handle_feature(struct dwc3 *dwc,
+static int dwc3_ep0_handle_u1(struct dwc3 *dwc, enum usb_device_state state,
+		int set)
+{
+	u32 reg;
+
+	if (state != USB_STATE_CONFIGURED)
+		return -EINVAL;
+	if ((dwc->speed != DWC3_DSTS_SUPERSPEED) &&
+			(dwc->speed != DWC3_DSTS_SUPERSPEED_PLUS))
+		return -EINVAL;
+
+	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
+	if (set)
+		reg |= DWC3_DCTL_INITU1ENA;
+	else
+		reg &= ~DWC3_DCTL_INITU1ENA;
+	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
+
+	return 0;
+}
+
+static int dwc3_ep0_handle_u2(struct dwc3 *dwc, enum usb_device_state state,
+		int set)
+{
+	u32 reg;
+
+
+	if (state != USB_STATE_CONFIGURED)
+		return -EINVAL;
+	if ((dwc->speed != DWC3_DSTS_SUPERSPEED) &&
+			(dwc->speed != DWC3_DSTS_SUPERSPEED_PLUS))
+		return -EINVAL;
+
+	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
+	if (set)
+		reg |= DWC3_DCTL_INITU2ENA;
+	else
+		reg &= ~DWC3_DCTL_INITU2ENA;
+	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
+
+	return 0;
+}
+
+static int dwc3_ep0_handle_test(struct dwc3 *dwc, enum usb_device_state state,
+		u32 wIndex, int set)
+{
+	if ((wIndex & 0xff) != 0)
+		return -EINVAL;
+	if (!set)
+		return -EINVAL;
+
+	switch (wIndex >> 8) {
+	case TEST_J:
+	case TEST_K:
+	case TEST_SE0_NAK:
+	case TEST_PACKET:
+	case TEST_FORCE_EN:
+		dwc->test_mode_nr = wIndex >> 8;
+		dwc->test_mode = true;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int dwc3_ep0_handle_device(struct dwc3 *dwc,
 		struct usb_ctrlrequest *ctrl, int set)
 {
-	struct dwc3_ep		*dep;
-	u32			recip;
+	enum usb_device_state	state;
 	u32			wValue;
 	u32			wIndex;
-	u32			reg;
-	int			ret;
-	enum usb_device_state	state;
+	int			ret = 0;
 
 	wValue = le16_to_cpu(ctrl->wValue);
 	wIndex = le16_to_cpu(ctrl->wIndex);
-	recip = ctrl->bRequestType & USB_RECIP_MASK;
 	state = dwc->gadget.state;
 
-	switch (recip) {
-	case USB_RECIP_DEVICE:
+	switch (wValue) {
+	case USB_DEVICE_REMOTE_WAKEUP:
+		break;
+	/*
+	 * 9.4.1 says only only for SS, in AddressState only for
+	 * default control pipe
+	 */
+	case USB_DEVICE_U1_ENABLE:
+		ret = dwc3_ep0_handle_u1(dwc, state, set);
+		break;
+	case USB_DEVICE_U2_ENABLE:
+		ret = dwc3_ep0_handle_u2(dwc, state, set);
+		break;
+	case USB_DEVICE_LTM_ENABLE:
+		ret = -EINVAL;
+		break;
+	case USB_DEVICE_TEST_MODE:
+		ret = dwc3_ep0_handle_test(dwc, state, wIndex, set);
+		break;
+	default:
+		ret = -EINVAL;
+	}
 
-		switch (wValue) {
-		case USB_DEVICE_REMOTE_WAKEUP:
-			break;
+	return ret;
+}
+
+static int dwc3_ep0_handle_intf(struct dwc3 *dwc,
+		struct usb_ctrlrequest *ctrl, int set)
+{
+	enum usb_device_state	state;
+	u32			wValue;
+	u32			wIndex;
+	int			ret = 0;
+
+	wValue = le16_to_cpu(ctrl->wValue);
+	wIndex = le16_to_cpu(ctrl->wIndex);
+	state = dwc->gadget.state;
+
+	switch (wValue) {
+	case USB_INTRF_FUNC_SUSPEND:
 		/*
-		 * 9.4.1 says only only for SS, in AddressState only for
-		 * default control pipe
+		 * REVISIT: Ideally we would enable some low power mode here,
+		 * however it's unclear what we should be doing here.
+		 *
+		 * For now, we're not doing anything, just making sure we return
+		 * 0 so USB Command Verifier tests pass without any errors.
 		 */
-		case USB_DEVICE_U1_ENABLE:
-			if (state != USB_STATE_CONFIGURED)
-				return -EINVAL;
-			if ((dwc->speed != DWC3_DSTS_SUPERSPEED) &&
-			    (dwc->speed != DWC3_DSTS_SUPERSPEED_PLUS))
-				return -EINVAL;
+		break;
+	default:
+		ret = -EINVAL;
+	}
 
-			reg = dwc3_readl(dwc->regs, DWC3_DCTL);
-			if (set)
-				reg |= DWC3_DCTL_INITU1ENA;
-			else
-				reg &= ~DWC3_DCTL_INITU1ENA;
-			dwc3_writel(dwc->regs, DWC3_DCTL, reg);
-			break;
+	return ret;
+}
 
-		case USB_DEVICE_U2_ENABLE:
-			if (state != USB_STATE_CONFIGURED)
-				return -EINVAL;
-			if ((dwc->speed != DWC3_DSTS_SUPERSPEED) &&
-			    (dwc->speed != DWC3_DSTS_SUPERSPEED_PLUS))
-				return -EINVAL;
+static int dwc3_ep0_handle_endpoint(struct dwc3 *dwc,
+		struct usb_ctrlrequest *ctrl, int set)
+{
+	struct dwc3_ep		*dep;
+	enum usb_device_state	state;
+	u32			wValue;
+	u32			wIndex;
+	int			ret;
 
-			reg = dwc3_readl(dwc->regs, DWC3_DCTL);
-			if (set)
-				reg |= DWC3_DCTL_INITU2ENA;
-			else
-				reg &= ~DWC3_DCTL_INITU2ENA;
-			dwc3_writel(dwc->regs, DWC3_DCTL, reg);
-			break;
+	wValue = le16_to_cpu(ctrl->wValue);
+	wIndex = le16_to_cpu(ctrl->wIndex);
+	state = dwc->gadget.state;
 
-		case USB_DEVICE_LTM_ENABLE:
+	switch (wValue) {
+	case USB_ENDPOINT_HALT:
+		dep = dwc3_wIndex_to_dep(dwc, ctrl->wIndex);
+		if (!dep)
 			return -EINVAL;
 
-		case USB_DEVICE_TEST_MODE:
-			if ((wIndex & 0xff) != 0)
-				return -EINVAL;
-			if (!set)
-				return -EINVAL;
-
-			switch (wIndex >> 8) {
-			case TEST_J:
-			case TEST_K:
-			case TEST_SE0_NAK:
-			case TEST_PACKET:
-			case TEST_FORCE_EN:
-				dwc->test_mode_nr = wIndex >> 8;
-				dwc->test_mode = true;
-				break;
-			default:
-				return -EINVAL;
-			}
+		if (set == 0 && (dep->flags & DWC3_EP_WEDGE))
 			break;
-		default:
+
+		ret = __dwc3_gadget_ep_set_halt(dep, set, true);
+		if (ret)
 			return -EINVAL;
-		}
 		break;
+	default:
+		return -EINVAL;
+	}
 
+	return 0;
+}
+
+static int dwc3_ep0_handle_feature(struct dwc3 *dwc,
+		struct usb_ctrlrequest *ctrl, int set)
+{
+	u32			recip;
+	int			ret;
+	enum usb_device_state	state;
+
+	recip = ctrl->bRequestType & USB_RECIP_MASK;
+	state = dwc->gadget.state;
+
+	switch (recip) {
+	case USB_RECIP_DEVICE:
+		ret = dwc3_ep0_handle_device(dwc, ctrl, set);
+		break;
 	case USB_RECIP_INTERFACE:
-		switch (wValue) {
-		case USB_INTRF_FUNC_SUSPEND:
-			if (wIndex & USB_INTRF_FUNC_SUSPEND_LP)
-				/* XXX enable Low power suspend */
-				;
-			if (wIndex & USB_INTRF_FUNC_SUSPEND_RW)
-				/* XXX enable remote wakeup */
-				;
-			break;
-		default:
-			return -EINVAL;
-		}
+		ret = dwc3_ep0_handle_intf(dwc, ctrl, set);
 		break;
-
 	case USB_RECIP_ENDPOINT:
-		switch (wValue) {
-		case USB_ENDPOINT_HALT:
-			dep = dwc3_wIndex_to_dep(dwc, ctrl->wIndex);
-			if (!dep)
-				return -EINVAL;
-			if (set == 0 && (dep->flags & DWC3_EP_WEDGE))
-				break;
-			ret = __dwc3_gadget_ep_set_halt(dep, set, true);
-			if (ret)
-				return -EINVAL;
-			break;
-		default:
-			return -EINVAL;
-		}
+		ret = dwc3_ep0_handle_endpoint(dwc, ctrl, set);
 		break;
-
 	default:
-		return -EINVAL;
+		ret = -EINVAL;
 	}
 
-	return 0;
+	return ret;
 }
 
 static int dwc3_ep0_set_address(struct dwc3 *dwc, struct usb_ctrlrequest *ctrl)
@@ -525,13 +569,12 @@ static int dwc3_ep0_set_address(struct dwc3 *dwc, struct usb_ctrlrequest *ctrl)
 
 	addr = le16_to_cpu(ctrl->wValue);
 	if (addr > 127) {
-		dwc3_trace(trace_dwc3_ep0, "invalid device address %d", addr);
+		dev_err(dwc->dev, "invalid device address %d\n", addr);
 		return -EINVAL;
 	}
 
 	if (state == USB_STATE_CONFIGURED) {
-		dwc3_trace(trace_dwc3_ep0,
-				"trying to set address when configured");
+		dev_err(dwc->dev, "can't SetAddress() from Configured State\n");
 		return -EINVAL;
 	}
 
@@ -716,35 +759,27 @@ static int dwc3_ep0_std_request(struct dwc3 *dwc, struct usb_ctrlrequest *ctrl)
 
 	switch (ctrl->bRequest) {
 	case USB_REQ_GET_STATUS:
-		dwc3_trace(trace_dwc3_ep0, "USB_REQ_GET_STATUS");
 		ret = dwc3_ep0_handle_status(dwc, ctrl);
 		break;
 	case USB_REQ_CLEAR_FEATURE:
-		dwc3_trace(trace_dwc3_ep0, "USB_REQ_CLEAR_FEATURE");
 		ret = dwc3_ep0_handle_feature(dwc, ctrl, 0);
 		break;
 	case USB_REQ_SET_FEATURE:
-		dwc3_trace(trace_dwc3_ep0, "USB_REQ_SET_FEATURE");
 		ret = dwc3_ep0_handle_feature(dwc, ctrl, 1);
 		break;
 	case USB_REQ_SET_ADDRESS:
-		dwc3_trace(trace_dwc3_ep0, "USB_REQ_SET_ADDRESS");
 		ret = dwc3_ep0_set_address(dwc, ctrl);
 		break;
 	case USB_REQ_SET_CONFIGURATION:
-		dwc3_trace(trace_dwc3_ep0, "USB_REQ_SET_CONFIGURATION");
 		ret = dwc3_ep0_set_config(dwc, ctrl);
 		break;
 	case USB_REQ_SET_SEL:
-		dwc3_trace(trace_dwc3_ep0, "USB_REQ_SET_SEL");
 		ret = dwc3_ep0_set_sel(dwc, ctrl);
 		break;
 	case USB_REQ_SET_ISOCH_DELAY:
-		dwc3_trace(trace_dwc3_ep0, "USB_REQ_SET_ISOCH_DELAY");
 		ret = dwc3_ep0_set_isoch_delay(dwc, ctrl);
 		break;
 	default:
-		dwc3_trace(trace_dwc3_ep0, "Forwarding to gadget driver");
 		ret = dwc3_ep0_delegate_req(dwc, ctrl);
 		break;
 	}
@@ -820,9 +855,6 @@ static void dwc3_ep0_complete_data(struct dwc3 *dwc,
 	status = DWC3_TRB_SIZE_TRBSTS(trb->size);
 	if (status == DWC3_TRBSTS_SETUP_PENDING) {
 		dwc->setup_packet_pending = true;
-
-		dwc3_trace(trace_dwc3_ep0, "Setup Pending received");
-
 		if (r)
 			dwc3_gadget_giveback(ep0, r, -ECONNRESET);
 
@@ -912,7 +944,7 @@ static void dwc3_ep0_complete_status(struct dwc3 *dwc,
 
 		ret = dwc3_gadget_set_test_mode(dwc, dwc->test_mode_nr);
 		if (ret < 0) {
-			dwc3_trace(trace_dwc3_ep0, "Invalid Test #%d",
+			dev_err(dwc->dev, "invalid test #%d\n",
 					dwc->test_mode_nr);
 			dwc3_ep0_stall_and_restart(dwc);
 			return;
@@ -920,10 +952,8 @@ static void dwc3_ep0_complete_status(struct dwc3 *dwc,
 	}
 
 	status = DWC3_TRB_SIZE_TRBSTS(trb->size);
-	if (status == DWC3_TRBSTS_SETUP_PENDING) {
+	if (status == DWC3_TRBSTS_SETUP_PENDING)
 		dwc->setup_packet_pending = true;
-		dwc3_trace(trace_dwc3_ep0, "Setup Pending received");
-	}
 
 	dwc->ep0state = EP0_SETUP_PHASE;
 	dwc3_ep0_out_start(dwc);
@@ -940,17 +970,14 @@ static void dwc3_ep0_xfer_complete(struct dwc3 *dwc,
 
 	switch (dwc->ep0state) {
 	case EP0_SETUP_PHASE:
-		dwc3_trace(trace_dwc3_ep0, "Setup Phase");
 		dwc3_ep0_inspect_setup(dwc, event);
 		break;
 
 	case EP0_DATA_PHASE:
-		dwc3_trace(trace_dwc3_ep0, "Data Phase");
 		dwc3_ep0_complete_data(dwc, event);
 		break;
 
 	case EP0_STATUS_PHASE:
-		dwc3_trace(trace_dwc3_ep0, "Status Phase");
 		dwc3_ep0_complete_status(dwc, event);
 		break;
 	default:
@@ -974,12 +1001,10 @@ static void __dwc3_ep0_do_control_data(struct dwc3 *dwc,
 		u32	transfer_size = 0;
 		u32	maxpacket;
 
-		ret = usb_gadget_map_request(&dwc->gadget, &req->request,
-				dep->number);
-		if (ret) {
-			dwc3_trace(trace_dwc3_ep0, "failed to map request");
+		ret = usb_gadget_map_request_by_dev(dwc->sysdev,
+				&req->request, dep->number);
+		if (ret)
 			return;
-		}
 
 		maxpacket = dep->endpoint.maxpacket;
 
@@ -1002,12 +1027,10 @@ static void __dwc3_ep0_do_control_data(struct dwc3 *dwc,
 				dwc->ep0_bounce_addr, transfer_size,
 				DWC3_TRBCTL_CONTROL_DATA, false);
 	} else {
-		ret = usb_gadget_map_request(&dwc->gadget, &req->request,
-				dep->number);
-		if (ret) {
-			dwc3_trace(trace_dwc3_ep0, "failed to map request");
+		ret = usb_gadget_map_request_by_dev(dwc->sysdev,
+				&req->request, dep->number);
+		if (ret)
 			return;
-		}
 
 		ret = dwc3_ep0_start_trans(dwc, dep->number, req->request.dma,
 				req->request.length, DWC3_TRBCTL_CONTROL_DATA,
@@ -1065,8 +1088,6 @@ static void dwc3_ep0_xfernotready(struct dwc3 *dwc,
 {
 	switch (event->status) {
 	case DEPEVT_STATUS_CONTROL_DATA:
-		dwc3_trace(trace_dwc3_ep0, "Control Data");
-
 		/*
 		 * We already have a DATA transfer in the controller's cache,
 		 * if we receive a XferNotReady(DATA) we will ignore it, unless
@@ -1079,8 +1100,7 @@ static void dwc3_ep0_xfernotready(struct dwc3 *dwc,
 		if (dwc->ep0_expect_in != event->endpoint_number) {
 			struct dwc3_ep	*dep = dwc->eps[dwc->ep0_expect_in];
 
-			dwc3_trace(trace_dwc3_ep0,
-					"Wrong direction for Data phase");
+			dev_err(dwc->dev, "unexpected direction for Data Phase\n");
 			dwc3_ep0_end_control_data(dwc, dep);
 			dwc3_ep0_stall_and_restart(dwc);
 			return;
@@ -1092,13 +1112,10 @@ static void dwc3_ep0_xfernotready(struct dwc3 *dwc,
 		if (dwc->ep0_next_event != DWC3_EP0_NRDY_STATUS)
 			return;
 
-		dwc3_trace(trace_dwc3_ep0, "Control Status");
-
 		dwc->ep0state = EP0_STATUS_PHASE;
 
 		if (dwc->delayed_status) {
 			WARN_ON_ONCE(event->endpoint_number != 1);
-			dwc3_trace(trace_dwc3_ep0, "Delayed Status");
 			return;
 		}
 
@@ -1109,10 +1126,6 @@ static void dwc3_ep0_xfernotready(struct dwc3 *dwc,
 void dwc3_ep0_interrupt(struct dwc3 *dwc,
 		const struct dwc3_event_depevt *event)
 {
-	dwc3_trace(trace_dwc3_ep0, "%s: state '%s'",
-			dwc3_ep_event_string(event),
-			dwc3_ep0_state_string(dwc->ep0state));
-
 	switch (event->endpoint_event) {
 	case DWC3_DEPEVT_XFERCOMPLETE:
 		dwc3_ep0_xfer_complete(dwc, event);
diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index 1dfa56a5f1c5..efddaf5d11d1 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -139,9 +139,6 @@ int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
 		udelay(5);
 	}
 
-	dwc3_trace(trace_dwc3_gadget,
-			"link state change request timed out");
-
 	return -ETIMEDOUT;
 }
 
@@ -178,6 +175,7 @@ void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 	req->started = false;
 	list_del(&req->list);
 	req->trb = NULL;
+	req->remaining = 0;
 
 	if (req->request.status == -EINPROGRESS)
 		req->request.status = status;
@@ -185,8 +183,8 @@ void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 	if (dwc->ep0_bounced && dep->number == 0)
 		dwc->ep0_bounced = false;
 	else
-		usb_gadget_unmap_request(&dwc->gadget, &req->request,
-				req->direction);
+		usb_gadget_unmap_request_by_dev(dwc->sysdev,
+				&req->request, req->direction);
 
 	trace_dwc3_gadget_giveback(req);
 
@@ -216,7 +214,7 @@ int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
 				ret = -EINVAL;
 			break;
 		}
-	} while (timeout--);
+	} while (--timeout);
 
 	if (!timeout) {
 		ret = -ETIMEDOUT;
@@ -233,6 +231,7 @@ static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 		struct dwc3_gadget_ep_cmd_params *params)
 {
+	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 	struct dwc3		*dwc = dep->dwc;
 	u32			timeout = 500;
 	u32			reg;
@@ -258,7 +257,7 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 		}
 	}
 
-	if (cmd == DWC3_DEPCMD_STARTTRANSFER) {
+	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 		int		needs_wakeup;
 
 		needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
@@ -276,7 +275,28 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 	dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
 	dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
 
-	dwc3_writel(dep->regs, DWC3_DEPCMD, cmd | DWC3_DEPCMD_CMDACT);
+	/*
+	 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
+	 * not relying on XferNotReady, we can make use of a special "No
+	 * Response Update Transfer" command where we should clear both CmdAct
+	 * and CmdIOC bits.
+	 *
+	 * With this, we don't need to wait for command completion and can
+	 * straight away issue further commands to the endpoint.
+	 *
+	 * NOTICE: We're making an assumption that control endpoints will never
+	 * make use of Update Transfer command. This is a safe assumption
+	 * because we can never have more than one request at a time with
+	 * Control Endpoints. If anybody changes that assumption, this chunk
+	 * needs to be updated accordingly.
+	 */
+	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
+			!usb_endpoint_xfer_isoc(desc))
+		cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
+	else
+		cmd |= DWC3_DEPCMD_CMDACT;
+
+	dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
 	do {
 		reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
 		if (!(reg & DWC3_DEPCMD_CMDACT)) {
@@ -318,6 +338,20 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 
 	trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
 
+	if (ret == 0) {
+		switch (DWC3_DEPCMD_CMD(cmd)) {
+		case DWC3_DEPCMD_STARTTRANSFER:
+			dep->flags |= DWC3_EP_TRANSFER_STARTED;
+			break;
+		case DWC3_DEPCMD_ENDTRANSFER:
+			dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
+			break;
+		default:
+			/* nothing */
+			break;
+		}
+	}
+
 	if (unlikely(susphy)) {
 		reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 		reg |= DWC3_GUSB2PHYCFG_SUSPHY;
@@ -365,7 +399,7 @@ static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 	if (dep->trb_pool)
 		return 0;
 
-	dep->trb_pool = dma_alloc_coherent(dwc->dev,
+	dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
 			sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 			&dep->trb_pool_dma, GFP_KERNEL);
 	if (!dep->trb_pool) {
@@ -381,7 +415,7 @@ static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 {
 	struct dwc3		*dwc = dep->dwc;
 
-	dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
+	dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 			dep->trb_pool, dep->trb_pool_dma);
 
 	dep->trb_pool = NULL;
@@ -454,16 +488,19 @@ static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
 }
 
 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
-		const struct usb_endpoint_descriptor *desc,
-		const struct usb_ss_ep_comp_descriptor *comp_desc,
 		bool modify, bool restore)
 {
+	const struct usb_ss_ep_comp_descriptor *comp_desc;
+	const struct usb_endpoint_descriptor *desc;
 	struct dwc3_gadget_ep_cmd_params params;
 
 	if (dev_WARN_ONCE(dwc->dev, modify && restore,
 					"Can't modify and restore\n"))
 		return -EINVAL;
 
+	comp_desc = dep->endpoint.comp_desc;
+	desc = dep->endpoint.desc;
+
 	memset(&params, 0x00, sizeof(params));
 
 	params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
@@ -542,24 +579,21 @@ static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
  * Caller should take care of locking
  */
 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
-		const struct usb_endpoint_descriptor *desc,
-		const struct usb_ss_ep_comp_descriptor *comp_desc,
 		bool modify, bool restore)
 {
+	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 	struct dwc3		*dwc = dep->dwc;
+
 	u32			reg;
 	int			ret;
 
-	dwc3_trace(trace_dwc3_gadget, "Enabling %s", dep->name);
-
 	if (!(dep->flags & DWC3_EP_ENABLED)) {
 		ret = dwc3_gadget_start_config(dwc, dep);
 		if (ret)
 			return ret;
 	}
 
-	ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, modify,
-			restore);
+	ret = dwc3_gadget_set_ep_config(dwc, dep, modify, restore);
 	if (ret)
 		return ret;
 
@@ -567,17 +601,18 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 		struct dwc3_trb	*trb_st_hw;
 		struct dwc3_trb	*trb_link;
 
-		dep->endpoint.desc = desc;
-		dep->comp_desc = comp_desc;
 		dep->type = usb_endpoint_type(desc);
 		dep->flags |= DWC3_EP_ENABLED;
+		dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
 
 		reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 		reg |= DWC3_DALEPENA_EP(dep->number);
 		dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 
+		init_waitqueue_head(&dep->wait_end_transfer);
+
 		if (usb_endpoint_xfer_control(desc))
-			return 0;
+			goto out;
 
 		/* Initialize the TRB ring */
 		dep->trb_dequeue = 0;
@@ -595,6 +630,39 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 		trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 	}
 
+	/*
+	 * Issue StartTransfer here with no-op TRB so we can always rely on No
+	 * Response Update Transfer command.
+	 */
+	if (usb_endpoint_xfer_bulk(desc)) {
+		struct dwc3_gadget_ep_cmd_params params;
+		struct dwc3_trb	*trb;
+		dma_addr_t trb_dma;
+		u32 cmd;
+
+		memset(&params, 0, sizeof(params));
+		trb = &dep->trb_pool[0];
+		trb_dma = dwc3_trb_dma_offset(dep, trb);
+
+		params.param0 = upper_32_bits(trb_dma);
+		params.param1 = lower_32_bits(trb_dma);
+
+		cmd = DWC3_DEPCMD_STARTTRANSFER;
+
+		ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
+		if (ret < 0)
+			return ret;
+
+		dep->flags |= DWC3_EP_BUSY;
+
+		dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
+		WARN_ON_ONCE(!dep->resource_index);
+	}
+
+
+out:
+	trace_dwc3_gadget_ep_enable(dep);
+
 	return 0;
 }
 
@@ -632,7 +700,7 @@ static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 	struct dwc3		*dwc = dep->dwc;
 	u32			reg;
 
-	dwc3_trace(trace_dwc3_gadget, "Disabling %s", dep->name);
+	trace_dwc3_gadget_ep_disable(dep);
 
 	dwc3_remove_requests(dwc, dep);
 
@@ -645,10 +713,14 @@ static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 	dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 
 	dep->stream_capable = false;
-	dep->endpoint.desc = NULL;
-	dep->comp_desc = NULL;
 	dep->type = 0;
-	dep->flags = 0;
+	dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
+
+	/* Clear out the ep descriptors for non-ep0 */
+	if (dep->number > 1) {
+		dep->endpoint.comp_desc = NULL;
+		dep->endpoint.desc = NULL;
+	}
 
 	return 0;
 }
@@ -695,7 +767,7 @@ static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 		return 0;
 
 	spin_lock_irqsave(&dwc->lock, flags);
-	ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false, false);
+	ret = __dwc3_gadget_ep_enable(dep, false, false);
 	spin_unlock_irqrestore(&dwc->lock, flags);
 
 	return ret;
@@ -771,10 +843,9 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 		unsigned length, unsigned chain, unsigned node)
 {
 	struct dwc3_trb		*trb;
-
-	dwc3_trace(trace_dwc3_gadget, "%s: req %p dma %08llx length %d%s",
-			dep->name, req, (unsigned long long) dma,
-			length, chain ? " chain" : "");
+	struct dwc3		*dwc = dep->dwc;
+	struct usb_gadget	*gadget = &dwc->gadget;
+	enum usb_device_speed	speed = gadget->speed;
 
 	trb = &dep->trb_pool[dep->trb_enqueue];
 
@@ -782,7 +853,6 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 		dwc3_gadget_move_started_request(req);
 		req->trb = trb;
 		req->trb_dma = dwc3_trb_dma_offset(dep, trb);
-		req->first_trb_index = dep->trb_enqueue;
 		dep->queued_requests++;
 	}
 
@@ -798,10 +868,16 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 		break;
 
 	case USB_ENDPOINT_XFER_ISOC:
-		if (!node)
+		if (!node) {
 			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
-		else
+
+			if (speed == USB_SPEED_HIGH) {
+				struct usb_ep *ep = &dep->endpoint;
+				trb->size |= DWC3_TRB_SIZE_PCM1(ep->mult - 1);
+			}
+		} else {
 			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
+		}
 
 		/* always enable Interrupt on Missed ISOC */
 		trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
@@ -816,15 +892,21 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 		 * This is only possible with faulty memory because we
 		 * checked it already :)
 		 */
-		BUG();
+		dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
+				usb_endpoint_type(dep->endpoint.desc));
 	}
 
 	/* always enable Continue on Short Packet */
-	trb->ctrl |= DWC3_TRB_CTRL_CSP;
+	if (usb_endpoint_dir_out(dep->endpoint.desc)) {
+		trb->ctrl |= DWC3_TRB_CTRL_CSP;
+
+		if (req->request.short_not_ok)
+			trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
+	}
 
 	if ((!req->request.no_interrupt && !chain) ||
 			(dwc3_calc_trbs_left(dep) == 0))
-		trb->ctrl |= DWC3_TRB_CTRL_IOC | DWC3_TRB_CTRL_ISP_IMI;
+		trb->ctrl |= DWC3_TRB_CTRL_IOC;
 
 	if (chain)
 		trb->ctrl |= DWC3_TRB_CTRL_CHN;
@@ -859,6 +941,7 @@ static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
 {
 	struct dwc3_trb		*tmp;
+	struct dwc3		*dwc = dep->dwc;
 	u8			trbs_left;
 
 	/*
@@ -870,7 +953,8 @@ static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
 	 */
 	if (dep->trb_enqueue == dep->trb_dequeue) {
 		tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
-		if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
+		if (dev_WARN_ONCE(dwc->dev, tmp->ctrl & DWC3_TRB_CTRL_HWO,
+				  "%s No TRBS left\n", dep->name))
 			return 0;
 
 		return DWC3_TRB_NUM - 1;
@@ -941,6 +1025,24 @@ static void dwc3_prepare_trbs(struct dwc3_ep *dep)
 	if (!dwc3_calc_trbs_left(dep))
 		return;
 
+	/*
+	 * We can get in a situation where there's a request in the started list
+	 * but there weren't enough TRBs to fully kick it in the first time
+	 * around, so it has been waiting for more TRBs to be freed up.
+	 *
+	 * In that case, we should check if we have a request with pending_sgs
+	 * in the started list and prepare TRBs for that request first,
+	 * otherwise we will prepare TRBs completely out of order and that will
+	 * break things.
+	 */
+	list_for_each_entry(req, &dep->started_list, list) {
+		if (req->num_pending_sgs > 0)
+			dwc3_prepare_one_trb_sg(dep, req);
+
+		if (!dwc3_calc_trbs_left(dep))
+			return;
+	}
+
 	list_for_each_entry_safe(req, n, &dep->pending_list, list) {
 		if (req->num_pending_sgs > 0)
 			dwc3_prepare_one_trb_sg(dep, req);
@@ -956,7 +1058,6 @@ static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param)
 {
 	struct dwc3_gadget_ep_cmd_params params;
 	struct dwc3_request		*req;
-	struct dwc3			*dwc = dep->dwc;
 	int				starting;
 	int				ret;
 	u32				cmd;
@@ -989,9 +1090,10 @@ static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param)
 		 * here and stop, unmap, free and del each of the linked
 		 * requests instead of what we do now.
 		 */
-		usb_gadget_unmap_request(&dwc->gadget, &req->request,
-				req->direction);
-		list_del(&req->list);
+		if (req->trb)
+			memset(req->trb, 0, sizeof(struct dwc3_trb));
+		dep->queued_requests--;
+		dwc3_gadget_giveback(dep, req, ret);
 		return ret;
 	}
 
@@ -1005,14 +1107,21 @@ static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param)
 	return 0;
 }
 
+static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
+{
+	u32			reg;
+
+	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
+	return DWC3_DSTS_SOFFN(reg);
+}
+
 static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
 		struct dwc3_ep *dep, u32 cur_uf)
 {
 	u32 uf;
 
 	if (list_empty(&dep->pending_list)) {
-		dwc3_trace(trace_dwc3_gadget,
-				"ISOC ep %s run out for requests",
+		dev_info(dwc->dev, "%s: ran out of requests\n",
 				dep->name);
 		dep->flags |= DWC3_EP_PENDING_REQUEST;
 		return;
@@ -1041,16 +1150,15 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
 	int			ret;
 
 	if (!dep->endpoint.desc) {
-		dwc3_trace(trace_dwc3_gadget,
-				"trying to queue request %p to disabled %s",
-				&req->request, dep->endpoint.name);
+		dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
+				dep->name);
 		return -ESHUTDOWN;
 	}
 
 	if (WARN(req->dep != dep, "request %p belongs to '%s'\n",
 				&req->request, req->dep->name)) {
-		dwc3_trace(trace_dwc3_gadget, "request %p belongs to '%s'",
-				&req->request, req->dep->name);
+		dev_err(dwc->dev, "%s: request %p belongs to '%s'\n",
+				dep->name, &req->request, req->dep->name);
 		return -EINVAL;
 	}
 
@@ -1063,8 +1171,8 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
 
 	trace_dwc3_ep_queue(req);
 
-	ret = usb_gadget_map_request(&dwc->gadget, &req->request,
-			dep->direction);
+	ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
+					    dep->direction);
 	if (ret)
 		return ret;
 
@@ -1082,10 +1190,17 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
 	 * errors which will force us issue EndTransfer command.
 	 */
 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
-		if ((dep->flags & DWC3_EP_PENDING_REQUEST) &&
-				list_empty(&dep->started_list)) {
-			dwc3_stop_active_transfer(dwc, dep->number, true);
-			dep->flags = DWC3_EP_ENABLED;
+		if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
+			if (dep->flags & DWC3_EP_TRANSFER_STARTED) {
+				dwc3_stop_active_transfer(dwc, dep->number, true);
+				dep->flags = DWC3_EP_ENABLED;
+			} else {
+				u32 cur_uf;
+
+				cur_uf = __dwc3_gadget_get_frame(dwc);
+				__dwc3_gadget_start_isoc(dwc, dep, cur_uf);
+				dep->flags &= ~DWC3_EP_PENDING_REQUEST;
+			}
 		}
 		return 0;
 	}
@@ -1094,10 +1209,6 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
 		return 0;
 
 	ret = __dwc3_gadget_kick_transfer(dep, 0);
-	if (ret && ret != -EBUSY)
-		dwc3_trace(trace_dwc3_gadget,
-				"%s: failed to kick transfers",
-				dep->name);
 	if (ret == -EBUSY)
 		ret = 0;
 
@@ -1116,7 +1227,6 @@ static int __dwc3_gadget_ep_queue_zlp(struct dwc3 *dwc, struct dwc3_ep *dep)
 	struct usb_request		*request;
 	struct usb_ep			*ep = &dep->endpoint;
 
-	dwc3_trace(trace_dwc3_gadget, "queueing ZLP");
 	request = dwc3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
 	if (!request)
 		return -ENOMEM;
@@ -1235,9 +1345,6 @@ int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
 
 		if (!protocol && ((dep->direction && transfer_in_flight) ||
 				(!dep->direction && started))) {
-			dwc3_trace(trace_dwc3_gadget,
-					"%s: pending request, cannot halt",
-					dep->name);
 			return -EAGAIN;
 		}
 
@@ -1331,10 +1438,8 @@ static const struct usb_ep_ops dwc3_gadget_ep_ops = {
 static int dwc3_gadget_get_frame(struct usb_gadget *g)
 {
 	struct dwc3		*dwc = gadget_to_dwc(g);
-	u32			reg;
 
-	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
-	return DWC3_DSTS_SOFFN(reg);
+	return __dwc3_gadget_get_frame(dwc);
 }
 
 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
@@ -1357,10 +1462,8 @@ static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
 
 	speed = reg & DWC3_DSTS_CONNECTSPD;
 	if ((speed == DWC3_DSTS_SUPERSPEED) ||
-	    (speed == DWC3_DSTS_SUPERSPEED_PLUS)) {
-		dwc3_trace(trace_dwc3_gadget, "no wakeup on SuperSpeed");
+	    (speed == DWC3_DSTS_SUPERSPEED_PLUS))
 		return 0;
-	}
 
 	link_state = DWC3_DSTS_USBLNKST(reg);
 
@@ -1369,9 +1472,6 @@ static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
 	case DWC3_LINK_STATE_U3:	/* in HS, means SUSPEND */
 		break;
 	default:
-		dwc3_trace(trace_dwc3_gadget,
-				"can't wakeup from '%s'",
-				dwc3_gadget_link_string(link_state));
 		return -EINVAL;
 	}
 
@@ -1476,11 +1576,6 @@ static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
 	if (!timeout)
 		return -ETIMEDOUT;
 
-	dwc3_trace(trace_dwc3_gadget, "gadget %s data soft-%s",
-			dwc->gadget_driver
-			? dwc->gadget_driver->function : "no-function",
-			is_on ? "connect" : "disconnect");
-
 	return 0;
 }
 
@@ -1492,6 +1587,21 @@ static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
 
 	is_on = !!is_on;
 
+	/*
+	 * Per databook, when we want to stop the gadget, if a control transfer
+	 * is still in process, complete it and get the core into setup phase.
+	 */
+	if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
+		reinit_completion(&dwc->ep0_in_setup);
+
+		ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
+				msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
+		if (ret == 0) {
+			dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
+			return -ETIMEDOUT;
+		}
+	}
+
 	spin_lock_irqsave(&dwc->lock, flags);
 	ret = dwc3_gadget_run_stop(dwc, is_on, false);
 	spin_unlock_irqrestore(&dwc->lock, flags);
@@ -1509,11 +1619,13 @@ static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
 			DWC3_DEVTEN_CMDCMPLTEN |
 			DWC3_DEVTEN_ERRTICERREN |
 			DWC3_DEVTEN_WKUPEVTEN |
-			DWC3_DEVTEN_ULSTCNGEN |
 			DWC3_DEVTEN_CONNECTDONEEN |
 			DWC3_DEVTEN_USBRSTEN |
 			DWC3_DEVTEN_DISCONNEVTEN);
 
+	if (dwc->revision < DWC3_REVISION_250A)
+		reg |= DWC3_DEVTEN_ULSTCNGEN;
+
 	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
 }
 
@@ -1573,6 +1685,17 @@ static int __dwc3_gadget_start(struct dwc3 *dwc)
 	int			ret = 0;
 	u32			reg;
 
+	/*
+	 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
+	 * the core supports IMOD, disable it.
+	 */
+	if (dwc->imod_interval) {
+		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
+		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
+	} else if (dwc3_has_imod(dwc)) {
+		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
+	}
+
 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
 	reg &= ~(DWC3_DCFG_SPEED_MASK);
 
@@ -1633,16 +1756,14 @@ static int __dwc3_gadget_start(struct dwc3 *dwc)
 	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
 
 	dep = dwc->eps[0];
-	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
-			false);
+	ret = __dwc3_gadget_ep_enable(dep, false, false);
 	if (ret) {
 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
 		goto err0;
 	}
 
 	dep = dwc->eps[1];
-	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
-			false);
+	ret = __dwc3_gadget_ep_enable(dep, false, false);
 	if (ret) {
 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
 		goto err1;
@@ -1708,9 +1829,6 @@ err0:
 
 static void __dwc3_gadget_stop(struct dwc3 *dwc)
 {
-	if (pm_runtime_suspended(dwc->dev))
-		return;
-
 	dwc3_gadget_disable_irq(dwc);
 	__dwc3_gadget_ep_disable(dwc->eps[0]);
 	__dwc3_gadget_ep_disable(dwc->eps[1]);
@@ -1720,9 +1838,30 @@ static int dwc3_gadget_stop(struct usb_gadget *g)
 {
 	struct dwc3		*dwc = gadget_to_dwc(g);
 	unsigned long		flags;
+	int			epnum;
 
 	spin_lock_irqsave(&dwc->lock, flags);
+
+	if (pm_runtime_suspended(dwc->dev))
+		goto out;
+
 	__dwc3_gadget_stop(dwc);
+
+	for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
+		struct dwc3_ep  *dep = dwc->eps[epnum];
+
+		if (!dep)
+			continue;
+
+		if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
+			continue;
+
+		wait_event_lock_irq(dep->wait_end_transfer,
+				    !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
+				    dwc->lock);
+	}
+
+out:
 	dwc->gadget_driver	= NULL;
 	spin_unlock_irqrestore(&dwc->lock, flags);
 
@@ -1765,9 +1904,13 @@ static int dwc3_gadget_init_hw_endpoints(struct dwc3 *dwc,
 				(epnum & 1) ? "in" : "out");
 
 		dep->endpoint.name = dep->name;
-		spin_lock_init(&dep->lock);
 
-		dwc3_trace(trace_dwc3_gadget, "initializing %s", dep->name);
+		if (!(dep->number > 1)) {
+			dep->endpoint.desc = &dwc3_gadget_ep0_desc;
+			dep->endpoint.comp_desc = NULL;
+		}
+
+		spin_lock_init(&dep->lock);
 
 		if (epnum == 0 || epnum == 1) {
 			usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
@@ -1815,15 +1958,13 @@ static int dwc3_gadget_init_endpoints(struct dwc3 *dwc)
 
 	ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_out_eps, 0);
 	if (ret < 0) {
-		dwc3_trace(trace_dwc3_gadget,
-				"failed to allocate OUT endpoints");
+		dev_err(dwc->dev, "failed to initialize OUT endpoints\n");
 		return ret;
 	}
 
 	ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_in_eps, 1);
 	if (ret < 0) {
-		dwc3_trace(trace_dwc3_gadget,
-				"failed to allocate IN endpoints");
+		dev_err(dwc->dev, "failed to initialize IN endpoints\n");
 		return ret;
 	}
 
@@ -1892,15 +2033,12 @@ static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
 		return 1;
 
 	count = trb->size & DWC3_TRB_SIZE_MASK;
-	req->request.actual += count;
+	req->remaining += count;
 
 	if (dep->direction) {
 		if (count) {
 			trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
 			if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
-				dwc3_trace(trace_dwc3_gadget,
-						"%s: incomplete IN transfer",
-						dep->name);
 				/*
 				 * If missed isoc occurred and there is
 				 * no request queued then issue END
@@ -1946,11 +2084,10 @@ static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
 	struct dwc3_request	*req, *n;
 	struct dwc3_trb		*trb;
 	bool			ioc = false;
-	int			ret;
+	int			ret = 0;
 
 	list_for_each_entry_safe(req, n, &dep->started_list, list) {
 		unsigned length;
-		unsigned actual;
 		int chain;
 
 		length = req->request.length;
@@ -1964,6 +2101,9 @@ static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
 			for_each_sg(sg, s, pending, i) {
 				trb = &dep->trb_pool[dep->trb_dequeue];
 
+				if (trb->ctrl & DWC3_TRB_CTRL_HWO)
+					break;
+
 				req->sg = sg_next(s);
 				req->num_pending_sgs--;
 
@@ -1978,17 +2118,9 @@ static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
 					event, status, chain);
 		}
 
-		/*
-		 * We assume here we will always receive the entire data block
-		 * which we should receive. Meaning, if we program RX to
-		 * receive 4K but we receive only 2K, we assume that's all we
-		 * should receive and we simply bounce the request back to the
-		 * gadget driver for further processing.
-		 */
-		actual = length - req->request.actual;
-		req->request.actual = actual;
+		req->request.actual = length - req->remaining;
 
-		if (ret && chain && (actual < length) && req->num_pending_sgs)
+		if ((req->request.actual < length) && req->num_pending_sgs)
 			return __dwc3_gadget_kick_transfer(dep, 0);
 
 		dwc3_gadget_giveback(dep, req, status);
@@ -2096,10 +2228,12 @@ static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
 {
 	struct dwc3_ep		*dep;
 	u8			epnum = event->endpoint_number;
+	u8			cmd;
 
 	dep = dwc->eps[epnum];
 
-	if (!(dep->flags & DWC3_EP_ENABLED))
+	if (!(dep->flags & DWC3_EP_ENABLED) &&
+	    !(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
 		return;
 
 	if (epnum == 0 || epnum == 1) {
@@ -2112,9 +2246,7 @@ static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
 		dep->resource_index = 0;
 
 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
-			dwc3_trace(trace_dwc3_gadget,
-					"%s is an Isochronous endpoint",
-					dep->name);
+			dev_err(dwc->dev, "XferComplete for Isochronous endpoint\n");
 			return;
 		}
 
@@ -2127,22 +2259,11 @@ static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
 			dwc3_gadget_start_isoc(dwc, dep, event);
 		} else {
-			int active;
 			int ret;
 
-			active = event->status & DEPEVT_STATUS_TRANSFER_ACTIVE;
-
-			dwc3_trace(trace_dwc3_gadget, "%s: reason %s",
-					dep->name, active ? "Transfer Active"
-					: "Transfer Not Active");
-
 			ret = __dwc3_gadget_kick_transfer(dep, 0);
 			if (!ret || ret == -EBUSY)
 				return;
-
-			dwc3_trace(trace_dwc3_gadget,
-					"%s: failed to kick transfers",
-					dep->name);
 		}
 
 		break;
@@ -2152,26 +2273,16 @@ static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
 					dep->name);
 			return;
 		}
+		break;
+	case DWC3_DEPEVT_EPCMDCMPLT:
+		cmd = DEPEVT_PARAMETER_CMD(event->parameters);
 
-		switch (event->status) {
-		case DEPEVT_STREAMEVT_FOUND:
-			dwc3_trace(trace_dwc3_gadget,
-					"Stream %d found and started",
-					event->parameters);
-
-			break;
-		case DEPEVT_STREAMEVT_NOTFOUND:
-			/* FALLTHROUGH */
-		default:
-			dwc3_trace(trace_dwc3_gadget,
-					"unable to find suitable stream");
+		if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
+			dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
+			wake_up(&dep->wait_end_transfer);
 		}
 		break;
 	case DWC3_DEPEVT_RXTXFIFOEVT:
-		dwc3_trace(trace_dwc3_gadget, "%s FIFO Overrun", dep->name);
-		break;
-	case DWC3_DEPEVT_EPCMDCMPLT:
-		dwc3_trace(trace_dwc3_gadget, "Endpoint Command Complete");
 		break;
 	}
 }
@@ -2224,7 +2335,8 @@ static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
 
 	dep = dwc->eps[epnum];
 
-	if (!dep->resource_index)
+	if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
+	    !dep->resource_index)
 		return;
 
 	/*
@@ -2268,25 +2380,9 @@ static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
 	dep->resource_index = 0;
 	dep->flags &= ~DWC3_EP_BUSY;
 
-	if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A)
+	if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) {
+		dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
 		udelay(100);
-}
-
-static void dwc3_stop_active_transfers(struct dwc3 *dwc)
-{
-	u32 epnum;
-
-	for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
-		struct dwc3_ep *dep;
-
-		dep = dwc->eps[epnum];
-		if (!dep)
-			continue;
-
-		if (!(dep->flags & DWC3_EP_ENABLED))
-			continue;
-
-		dwc3_remove_requests(dwc, dep);
 	}
 }
 
@@ -2375,8 +2471,6 @@ static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
 	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 	dwc->test_mode = false;
-
-	dwc3_stop_active_transfers(dwc);
 	dwc3_clear_stall_all_ep(dwc);
 
 	/* Reset device address to zero */
@@ -2385,32 +2479,6 @@ static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
 }
 
-static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
-{
-	u32 reg;
-	u32 usb30_clock = DWC3_GCTL_CLK_BUS;
-
-	/*
-	 * We change the clock only at SS but I dunno why I would want to do
-	 * this. Maybe it becomes part of the power saving plan.
-	 */
-
-	if ((speed != DWC3_DSTS_SUPERSPEED) &&
-	    (speed != DWC3_DSTS_SUPERSPEED_PLUS))
-		return;
-
-	/*
-	 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
-	 * each time on Connect Done.
-	 */
-	if (!usb30_clock)
-		return;
-
-	reg = dwc3_readl(dwc->regs, DWC3_GCTL);
-	reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
-	dwc3_writel(dwc->regs, DWC3_GCTL, reg);
-}
-
 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
 {
 	struct dwc3_ep		*dep;
@@ -2422,7 +2490,14 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
 	speed = reg & DWC3_DSTS_CONNECTSPD;
 	dwc->speed = speed;
 
-	dwc3_update_ram_clk_sel(dwc, speed);
+	/*
+	 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
+	 * each time on Connect Done.
+	 *
+	 * Currently we always use the reset value. If any platform
+	 * wants to set this to a different value, we need to add a
+	 * setting and update GCTL.RAMCLKSEL here.
+	 */
 
 	switch (speed) {
 	case DWC3_DSTS_SUPERSPEED_PLUS:
@@ -2491,7 +2566,7 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
 		 */
 		WARN_ONCE(dwc->revision < DWC3_REVISION_240A
 				&& dwc->has_lpm_erratum,
-				"LPM Erratum not available on dwc3 revisisions < 2.40a\n");
+				"LPM Erratum not available on dwc3 revisions < 2.40a\n");
 
 		if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
 			reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
@@ -2504,16 +2579,14 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
 	}
 
 	dep = dwc->eps[0];
-	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
-			false);
+	ret = __dwc3_gadget_ep_enable(dep, true, false);
 	if (ret) {
 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
 		return;
 	}
 
 	dep = dwc->eps[1];
-	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
-			false);
+	ret = __dwc3_gadget_ep_enable(dep, true, false);
 	if (ret) {
 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
 		return;
@@ -2570,8 +2643,6 @@ static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
 			(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
 		if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
 				(next == DWC3_LINK_STATE_RESUME)) {
-			dwc3_trace(trace_dwc3_gadget,
-					"ignoring transition U3 -> Resume");
 			return;
 		}
 	}
@@ -2705,11 +2776,7 @@ static void dwc3_gadget_interrupt(struct dwc3 *dwc,
 		break;
 	case DWC3_DEVICE_EVENT_EOPF:
 		/* It changed to be suspend event for version 2.30a and above */
-		if (dwc->revision < DWC3_REVISION_230A) {
-			dwc3_trace(trace_dwc3_gadget, "End of Periodic Frame");
-		} else {
-			dwc3_trace(trace_dwc3_gadget, "U3/L1-L2 Suspend Event");
-
+		if (dwc->revision >= DWC3_REVISION_230A) {
 			/*
 			 * Ignore suspend event until the gadget enters into
 			 * USB_STATE_CONFIGURED state.
@@ -2720,16 +2787,9 @@ static void dwc3_gadget_interrupt(struct dwc3 *dwc,
 		}
 		break;
 	case DWC3_DEVICE_EVENT_SOF:
-		dwc3_trace(trace_dwc3_gadget, "Start of Periodic Frame");
-		break;
 	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
-		dwc3_trace(trace_dwc3_gadget, "Erratic Error");
-		break;
 	case DWC3_DEVICE_EVENT_CMD_CMPL:
-		dwc3_trace(trace_dwc3_gadget, "Command Complete");
-		break;
 	case DWC3_DEVICE_EVENT_OVERFLOW:
-		dwc3_trace(trace_dwc3_gadget, "Overflow");
 		break;
 	default:
 		dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
@@ -2739,7 +2799,7 @@ static void dwc3_gadget_interrupt(struct dwc3 *dwc,
 static void dwc3_process_event_entry(struct dwc3 *dwc,
 		const union dwc3_event *event)
 {
-	trace_dwc3_event(event->raw);
+	trace_dwc3_event(event->raw, dwc);
 
 	/* Endpoint IRQ, handle it and return early */
 	if (event->type.is_devspec == 0) {
@@ -2772,7 +2832,7 @@ static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
 	while (left > 0) {
 		union dwc3_event event;
 
-		event.raw = *(u32 *) (evt->buf + evt->lpos);
+		event.raw = *(u32 *) (evt->cache + evt->lpos);
 
 		dwc3_process_event_entry(dwc, &event);
 
@@ -2785,10 +2845,8 @@ static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
 		 * boundary so I worry about that once we try to handle
 		 * that.
 		 */
-		evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
+		evt->lpos = (evt->lpos + 4) % evt->length;
 		left -= 4;
-
-		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), 4);
 	}
 
 	evt->count = 0;
@@ -2800,6 +2858,11 @@ static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
 	reg &= ~DWC3_GEVNTSIZ_INTMASK;
 	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
 
+	if (dwc->imod_interval) {
+		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
+		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
+	}
+
 	return ret;
 }
 
@@ -2820,6 +2883,7 @@ static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
 {
 	struct dwc3 *dwc = evt->dwc;
+	u32 amount;
 	u32 count;
 	u32 reg;
 
@@ -2843,6 +2907,14 @@ static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
 	reg |= DWC3_GEVNTSIZ_INTMASK;
 	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
 
+	amount = min(count, evt->length - evt->lpos);
+	memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
+
+	if (amount < count)
+		memcpy(evt->cache, evt->buf, count - amount);
+
+	dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
+
 	return IRQ_WAKE_THREAD;
 }
 
@@ -2853,6 +2925,39 @@ static irqreturn_t dwc3_interrupt(int irq, void *_evt)
 	return dwc3_check_event_buf(evt);
 }
 
+static int dwc3_gadget_get_irq(struct dwc3 *dwc)
+{
+	struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
+	int irq;
+
+	irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
+	if (irq > 0)
+		goto out;
+
+	if (irq == -EPROBE_DEFER)
+		goto out;
+
+	irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
+	if (irq > 0)
+		goto out;
+
+	if (irq == -EPROBE_DEFER)
+		goto out;
+
+	irq = platform_get_irq(dwc3_pdev, 0);
+	if (irq > 0)
+		goto out;
+
+	if (irq != -EPROBE_DEFER)
+		dev_err(dwc->dev, "missing peripheral IRQ\n");
+
+	if (!irq)
+		irq = -EINVAL;
+
+out:
+	return irq;
+}
+
 /**
  * dwc3_gadget_init - Initializes gadget related registers
  * @dwc: pointer to our controller context structure
@@ -2861,35 +2966,18 @@ static irqreturn_t dwc3_interrupt(int irq, void *_evt)
  */
 int dwc3_gadget_init(struct dwc3 *dwc)
 {
-	int ret, irq;
-	struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
+	int ret;
+	int irq;
 
-	irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
-	if (irq == -EPROBE_DEFER)
-		return irq;
-
-	if (irq <= 0) {
-		irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
-		if (irq == -EPROBE_DEFER)
-			return irq;
-
-		if (irq <= 0) {
-			irq = platform_get_irq(dwc3_pdev, 0);
-			if (irq <= 0) {
-				if (irq != -EPROBE_DEFER) {
-					dev_err(dwc->dev,
-						"missing peripheral IRQ\n");
-				}
-				if (!irq)
-					irq = -EINVAL;
-				return irq;
-			}
-		}
+	irq = dwc3_gadget_get_irq(dwc);
+	if (irq < 0) {
+		ret = irq;
+		goto err0;
 	}
 
 	dwc->irq_gadget = irq;
 
-	dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
+	dwc->ctrl_req = dma_alloc_coherent(dwc->sysdev, sizeof(*dwc->ctrl_req),
 			&dwc->ctrl_req_addr, GFP_KERNEL);
 	if (!dwc->ctrl_req) {
 		dev_err(dwc->dev, "failed to allocate ctrl request\n");
@@ -2897,8 +2985,9 @@ int dwc3_gadget_init(struct dwc3 *dwc)
 		goto err0;
 	}
 
-	dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2,
-			&dwc->ep0_trb_addr, GFP_KERNEL);
+	dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
+					  sizeof(*dwc->ep0_trb) * 2,
+					  &dwc->ep0_trb_addr, GFP_KERNEL);
 	if (!dwc->ep0_trb) {
 		dev_err(dwc->dev, "failed to allocate ep0 trb\n");
 		ret = -ENOMEM;
@@ -2911,7 +3000,7 @@ int dwc3_gadget_init(struct dwc3 *dwc)
 		goto err2;
 	}
 
-	dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
+	dwc->ep0_bounce = dma_alloc_coherent(dwc->sysdev,
 			DWC3_EP0_BOUNCE_SIZE, &dwc->ep0_bounce_addr,
 			GFP_KERNEL);
 	if (!dwc->ep0_bounce) {
@@ -2926,6 +3015,8 @@ int dwc3_gadget_init(struct dwc3 *dwc)
 		goto err4;
 	}
 
+	init_completion(&dwc->ep0_in_setup);
+
 	dwc->gadget.ops			= &dwc3_gadget_ops;
 	dwc->gadget.speed		= USB_SPEED_UNKNOWN;
 	dwc->gadget.sg_supported	= true;
@@ -2949,8 +3040,7 @@ int dwc3_gadget_init(struct dwc3 *dwc)
 	 * composite.c that we are USB 2.0 + LPM ECN.
 	 */
 	if (dwc->revision < DWC3_REVISION_220A)
-		dwc3_trace(trace_dwc3_gadget,
-				"Changing max_speed on rev %08x",
+		dev_info(dwc->dev, "changing max_speed on rev %08x\n",
 				dwc->revision);
 
 	dwc->gadget.max_speed		= dwc->maximum_speed;
@@ -2983,18 +3073,18 @@ err5:
 
 err4:
 	dwc3_gadget_free_endpoints(dwc);
-	dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
+	dma_free_coherent(dwc->sysdev, DWC3_EP0_BOUNCE_SIZE,
 			dwc->ep0_bounce, dwc->ep0_bounce_addr);
 
 err3:
 	kfree(dwc->setup_buf);
 
 err2:
-	dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2,
+	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
 			dwc->ep0_trb, dwc->ep0_trb_addr);
 
 err1:
-	dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
+	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ctrl_req),
 			dwc->ctrl_req, dwc->ctrl_req_addr);
 
 err0:
@@ -3009,16 +3099,16 @@ void dwc3_gadget_exit(struct dwc3 *dwc)
 
 	dwc3_gadget_free_endpoints(dwc);
 
-	dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
+	dma_free_coherent(dwc->sysdev, DWC3_EP0_BOUNCE_SIZE,
 			dwc->ep0_bounce, dwc->ep0_bounce_addr);
 
 	kfree(dwc->setup_buf);
 	kfree(dwc->zlp_buf);
 
-	dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2,
+	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
 			dwc->ep0_trb, dwc->ep0_trb_addr);
 
-	dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
+	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ctrl_req),
 			dwc->ctrl_req, dwc->ctrl_req_addr);
 }
 
diff --git a/drivers/usb/dwc3/gadget.h b/drivers/usb/dwc3/gadget.h
index e4a1d974a5ae..3129bcf74d7d 100644
--- a/drivers/usb/dwc3/gadget.h
+++ b/drivers/usb/dwc3/gadget.h
@@ -62,10 +62,7 @@ struct dwc3;
 
 static inline struct dwc3_request *next_request(struct list_head *list)
 {
-	if (list_empty(list))
-		return NULL;
-
-	return list_first_entry(list, struct dwc3_request, list);
+	return list_first_entry_or_null(list, struct dwc3_request, list);
 }
 
 static inline void dwc3_gadget_move_started_request(struct dwc3_request *req)
diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c
index f6533c68fed1..487f0ff6ae25 100644
--- a/drivers/usb/dwc3/host.c
+++ b/drivers/usb/dwc3/host.c
@@ -19,6 +19,39 @@
 
 #include "core.h"
 
+static int dwc3_host_get_irq(struct dwc3 *dwc)
+{
+	struct platform_device	*dwc3_pdev = to_platform_device(dwc->dev);
+	int irq;
+
+	irq = platform_get_irq_byname(dwc3_pdev, "host");
+	if (irq > 0)
+		goto out;
+
+	if (irq == -EPROBE_DEFER)
+		goto out;
+
+	irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
+	if (irq > 0)
+		goto out;
+
+	if (irq == -EPROBE_DEFER)
+		goto out;
+
+	irq = platform_get_irq(dwc3_pdev, 0);
+	if (irq > 0)
+		goto out;
+
+	if (irq != -EPROBE_DEFER)
+		dev_err(dwc->dev, "missing host IRQ\n");
+
+	if (!irq)
+		irq = -EINVAL;
+
+out:
+	return irq;
+}
+
 int dwc3_host_init(struct dwc3 *dwc)
 {
 	struct property_entry	props[2];
@@ -27,39 +60,18 @@ int dwc3_host_init(struct dwc3 *dwc)
 	struct resource		*res;
 	struct platform_device	*dwc3_pdev = to_platform_device(dwc->dev);
 
-	irq = platform_get_irq_byname(dwc3_pdev, "host");
-	if (irq == -EPROBE_DEFER)
+	irq = dwc3_host_get_irq(dwc);
+	if (irq < 0)
 		return irq;
 
-	if (irq <= 0) {
-		irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
-		if (irq == -EPROBE_DEFER)
-			return irq;
-
-		if (irq <= 0) {
-			irq = platform_get_irq(dwc3_pdev, 0);
-			if (irq <= 0) {
-				if (irq != -EPROBE_DEFER) {
-					dev_err(dwc->dev,
-						"missing host IRQ\n");
-				}
-				if (!irq)
-					irq = -EINVAL;
-				return irq;
-			} else {
-				res = platform_get_resource(dwc3_pdev,
-							    IORESOURCE_IRQ, 0);
-			}
-		} else {
-			res = platform_get_resource_byname(dwc3_pdev,
-							   IORESOURCE_IRQ,
-							   "dwc_usb3");
-		}
-
-	} else {
+	res = platform_get_resource_byname(dwc3_pdev, IORESOURCE_IRQ, "host");
+	if (!res)
 		res = platform_get_resource_byname(dwc3_pdev, IORESOURCE_IRQ,
-						   "host");
-	}
+				"dwc_usb3");
+	if (!res)
+		res = platform_get_resource(dwc3_pdev, IORESOURCE_IRQ, 0);
+	if (!res)
+		return -ENOMEM;
 
 	dwc->xhci_resources[1].start = irq;
 	dwc->xhci_resources[1].end = irq;
@@ -72,11 +84,7 @@ int dwc3_host_init(struct dwc3 *dwc)
 		return -ENOMEM;
 	}
 
-	dma_set_coherent_mask(&xhci->dev, dwc->dev->coherent_dma_mask);
-
 	xhci->dev.parent	= dwc->dev;
-	xhci->dev.dma_mask	= dwc->dev->dma_mask;
-	xhci->dev.dma_parms	= dwc->dev->dma_parms;
 
 	dwc->xhci = xhci;
 
@@ -99,9 +107,9 @@ int dwc3_host_init(struct dwc3 *dwc)
 	}
 
 	phy_create_lookup(dwc->usb2_generic_phy, "usb2-phy",
-			  dev_name(&xhci->dev));
+			  dev_name(dwc->dev));
 	phy_create_lookup(dwc->usb3_generic_phy, "usb3-phy",
-			  dev_name(&xhci->dev));
+			  dev_name(dwc->dev));
 
 	ret = platform_device_add(xhci);
 	if (ret) {
@@ -112,9 +120,9 @@ int dwc3_host_init(struct dwc3 *dwc)
 	return 0;
 err2:
 	phy_remove_lookup(dwc->usb2_generic_phy, "usb2-phy",
-			  dev_name(&xhci->dev));
+			  dev_name(dwc->dev));
 	phy_remove_lookup(dwc->usb3_generic_phy, "usb3-phy",
-			  dev_name(&xhci->dev));
+			  dev_name(dwc->dev));
 err1:
 	platform_device_put(xhci);
 	return ret;
@@ -123,8 +131,8 @@ err1:
 void dwc3_host_exit(struct dwc3 *dwc)
 {
 	phy_remove_lookup(dwc->usb2_generic_phy, "usb2-phy",
-			  dev_name(&dwc->xhci->dev));
+			  dev_name(dwc->dev));
 	phy_remove_lookup(dwc->usb3_generic_phy, "usb3-phy",
-			  dev_name(&dwc->xhci->dev));
+			  dev_name(dwc->dev));
 	platform_device_unregister(dwc->xhci);
 }
diff --git a/drivers/usb/dwc3/io.h b/drivers/usb/dwc3/io.h
index a06f9a8fecc7..c69b06696824 100644
--- a/drivers/usb/dwc3/io.h
+++ b/drivers/usb/dwc3/io.h
@@ -40,8 +40,7 @@ static inline u32 dwc3_readl(void __iomem *base, u32 offset)
 	 * documentation, so we revert it back to the proper addresses, the
 	 * same way they are described on SNPS documentation
 	 */
-	dwc3_trace(trace_dwc3_readl, "addr %p value %08x",
-			base - DWC3_GLOBALS_REGS_START + offset, value);
+	trace_dwc3_readl(base - DWC3_GLOBALS_REGS_START, offset, value);
 
 	return value;
 }
@@ -60,8 +59,7 @@ static inline void dwc3_writel(void __iomem *base, u32 offset, u32 value)
 	 * documentation, so we revert it back to the proper addresses, the
 	 * same way they are described on SNPS documentation
 	 */
-	dwc3_trace(trace_dwc3_writel, "addr %p value %08x",
-			base - DWC3_GLOBALS_REGS_START + offset, value);
+	trace_dwc3_writel(base - DWC3_GLOBALS_REGS_START, offset, value);
 }
 
 #endif /* __DRIVERS_USB_DWC3_IO_H */
diff --git a/drivers/usb/dwc3/trace.h b/drivers/usb/dwc3/trace.h
index d24cefd191b5..2b124f94d858 100644
--- a/drivers/usb/dwc3/trace.h
+++ b/drivers/usb/dwc3/trace.h
@@ -37,47 +37,66 @@ DECLARE_EVENT_CLASS(dwc3_log_msg,
 	TP_printk("%s", __get_str(msg))
 );
 
-DEFINE_EVENT(dwc3_log_msg, dwc3_readl,
+DEFINE_EVENT(dwc3_log_msg, dwc3_gadget,
 	TP_PROTO(struct va_format *vaf),
 	TP_ARGS(vaf)
 );
 
-DEFINE_EVENT(dwc3_log_msg, dwc3_writel,
+DEFINE_EVENT(dwc3_log_msg, dwc3_core,
 	TP_PROTO(struct va_format *vaf),
 	TP_ARGS(vaf)
 );
 
-DEFINE_EVENT(dwc3_log_msg, dwc3_gadget,
+DEFINE_EVENT(dwc3_log_msg, dwc3_ep0,
 	TP_PROTO(struct va_format *vaf),
 	TP_ARGS(vaf)
 );
 
-DEFINE_EVENT(dwc3_log_msg, dwc3_core,
-	TP_PROTO(struct va_format *vaf),
-	TP_ARGS(vaf)
+DECLARE_EVENT_CLASS(dwc3_log_io,
+	TP_PROTO(void *base, u32 offset, u32 value),
+	TP_ARGS(base, offset, value),
+	TP_STRUCT__entry(
+		__field(void *, base)
+		__field(u32, offset)
+		__field(u32, value)
+	),
+	TP_fast_assign(
+		__entry->base = base;
+		__entry->offset = offset;
+		__entry->value = value;
+	),
+	TP_printk("addr %p value %08x", __entry->base + __entry->offset,
+			__entry->value)
 );
 
-DEFINE_EVENT(dwc3_log_msg, dwc3_ep0,
-	TP_PROTO(struct va_format *vaf),
-	TP_ARGS(vaf)
+DEFINE_EVENT(dwc3_log_io, dwc3_readl,
+	TP_PROTO(void *base, u32 offset, u32 value),
+	TP_ARGS(base, offset, value)
+);
+
+DEFINE_EVENT(dwc3_log_io, dwc3_writel,
+	TP_PROTO(void *base, u32 offset, u32 value),
+	TP_ARGS(base, offset, value)
 );
 
 DECLARE_EVENT_CLASS(dwc3_log_event,
-	TP_PROTO(u32 event),
-	TP_ARGS(event),
+	TP_PROTO(u32 event, struct dwc3 *dwc),
+	TP_ARGS(event, dwc),
 	TP_STRUCT__entry(
 		__field(u32, event)
+		__field(u32, ep0state)
 	),
 	TP_fast_assign(
 		__entry->event = event;
+		__entry->ep0state = dwc->ep0state;
 	),
 	TP_printk("event (%08x): %s", __entry->event,
-			dwc3_decode_event(__entry->event))
+			dwc3_decode_event(__entry->event, __entry->ep0state))
 );
 
 DEFINE_EVENT(dwc3_log_event, dwc3_event,
-	TP_PROTO(u32 event),
-	TP_ARGS(event)
+	TP_PROTO(u32 event, struct dwc3 *dwc),
+	TP_ARGS(event, dwc)
 );
 
 DECLARE_EVENT_CLASS(dwc3_log_ctrl,
@@ -237,6 +256,7 @@ DECLARE_EVENT_CLASS(dwc3_log_trb,
 		__field(u32, bph)
 		__field(u32, size)
 		__field(u32, ctrl)
+		__field(u32, type)
 	),
 	TP_fast_assign(
 		snprintf(__get_str(name), DWC3_MSG_MAX, "%s", dep->name);
@@ -247,11 +267,31 @@ DECLARE_EVENT_CLASS(dwc3_log_trb,
 		__entry->bph = trb->bph;
 		__entry->size = trb->size;
 		__entry->ctrl = trb->ctrl;
+		__entry->type = usb_endpoint_type(dep->endpoint.desc);
 	),
-	TP_printk("%s: %d/%d trb %p buf %08x%08x size %d ctrl %08x (%c%c%c%c:%c%c:%s)",
+	TP_printk("%s: %d/%d trb %p buf %08x%08x size %s%d ctrl %08x (%c%c%c%c:%c%c:%s)",
 		__get_str(name), __entry->queued, __entry->allocated,
 		__entry->trb, __entry->bph, __entry->bpl,
-		__entry->size, __entry->ctrl,
+		({char *s;
+		int pcm = ((__entry->size >> 24) & 3) + 1;
+		switch (__entry->type) {
+		case USB_ENDPOINT_XFER_INT:
+		case USB_ENDPOINT_XFER_ISOC:
+			switch (pcm) {
+			case 1:
+				s = "1x ";
+				break;
+			case 2:
+				s = "2x ";
+				break;
+			case 3:
+				s = "3x ";
+				break;
+			}
+		default:
+			s = "";
+		} s; }),
+		DWC3_TRB_SIZE_LENGTH(__entry->size), __entry->ctrl,
 		__entry->ctrl & DWC3_TRB_CTRL_HWO ? 'H' : 'h',
 		__entry->ctrl & DWC3_TRB_CTRL_LST ? 'L' : 'l',
 		__entry->ctrl & DWC3_TRB_CTRL_CHN ? 'C' : 'c',
@@ -301,6 +341,57 @@ DEFINE_EVENT(dwc3_log_trb, dwc3_complete_trb,
 	TP_ARGS(dep, trb)
 );
 
+DECLARE_EVENT_CLASS(dwc3_log_ep,
+	TP_PROTO(struct dwc3_ep *dep),
+	TP_ARGS(dep),
+	TP_STRUCT__entry(
+		__dynamic_array(char, name, DWC3_MSG_MAX)
+		__field(unsigned, maxpacket)
+		__field(unsigned, maxpacket_limit)
+		__field(unsigned, max_streams)
+		__field(unsigned, maxburst)
+		__field(unsigned, flags)
+		__field(unsigned, direction)
+		__field(u8, trb_enqueue)
+		__field(u8, trb_dequeue)
+	),
+	TP_fast_assign(
+		snprintf(__get_str(name), DWC3_MSG_MAX, "%s", dep->name);
+		__entry->maxpacket = dep->endpoint.maxpacket;
+		__entry->maxpacket_limit = dep->endpoint.maxpacket_limit;
+		__entry->max_streams = dep->endpoint.max_streams;
+		__entry->maxburst = dep->endpoint.maxburst;
+		__entry->flags = dep->flags;
+		__entry->direction = dep->direction;
+		__entry->trb_enqueue = dep->trb_enqueue;
+		__entry->trb_dequeue = dep->trb_dequeue;
+	),
+	TP_printk("%s: mps %d/%d streams %d burst %d ring %d/%d flags %c:%c%c%c%c%c:%c:%c",
+		__get_str(name), __entry->maxpacket,
+		__entry->maxpacket_limit, __entry->max_streams,
+		__entry->maxburst, __entry->trb_enqueue,
+		__entry->trb_dequeue,
+		__entry->flags & DWC3_EP_ENABLED ? 'E' : 'e',
+		__entry->flags & DWC3_EP_STALL ? 'S' : 's',
+		__entry->flags & DWC3_EP_WEDGE ? 'W' : 'w',
+		__entry->flags & DWC3_EP_BUSY ? 'B' : 'b',
+		__entry->flags & DWC3_EP_PENDING_REQUEST ? 'P' : 'p',
+		__entry->flags & DWC3_EP_MISSED_ISOC ? 'M' : 'm',
+		__entry->flags & DWC3_EP_END_TRANSFER_PENDING ? 'E' : 'e',
+		__entry->direction ? '<' : '>'
+	)
+);
+
+DEFINE_EVENT(dwc3_log_ep, dwc3_gadget_ep_enable,
+	TP_PROTO(struct dwc3_ep *dep),
+	TP_ARGS(dep)
+);
+
+DEFINE_EVENT(dwc3_log_ep, dwc3_gadget_ep_disable,
+	TP_PROTO(struct dwc3_ep *dep),
+	TP_ARGS(dep)
+);
+
 #endif /* __DWC3_TRACE_H */
 
 /* this part has to be here */
diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c
index 32176f779861..41ab61f9b6e0 100644
--- a/drivers/usb/gadget/composite.c
+++ b/drivers/usb/gadget/composite.c
@@ -201,7 +201,12 @@ ep_found:
 	_ep->desc = chosen_desc;
 	_ep->comp_desc = NULL;
 	_ep->maxburst = 0;
-	_ep->mult = 0;
+	_ep->mult = 1;
+
+	if (g->speed == USB_SPEED_HIGH && (usb_endpoint_xfer_isoc(_ep->desc) ||
+				usb_endpoint_xfer_int(_ep->desc)))
+		_ep->mult = usb_endpoint_maxp_mult(_ep->desc);
+
 	if (!want_comp_desc)
 		return 0;
 
@@ -218,7 +223,7 @@ ep_found:
 		switch (usb_endpoint_type(_ep->desc)) {
 		case USB_ENDPOINT_XFER_ISOC:
 			/* mult: bits 1:0 of bmAttributes */
-			_ep->mult = comp_desc->bmAttributes & 0x3;
+			_ep->mult = (comp_desc->bmAttributes & 0x3) + 1;
 		case USB_ENDPOINT_XFER_BULK:
 		case USB_ENDPOINT_XFER_INT:
 			_ep->maxburst = comp_desc->bMaxBurst + 1;
@@ -2382,18 +2387,8 @@ EXPORT_SYMBOL_GPL(usb_composite_setup_continue);
 
 static char *composite_default_mfr(struct usb_gadget *gadget)
 {
-	char *mfr;
-	int len;
-
-	len = snprintf(NULL, 0, "%s %s with %s", init_utsname()->sysname,
-			init_utsname()->release, gadget->name);
-	len++;
-	mfr = kmalloc(len, GFP_KERNEL);
-	if (!mfr)
-		return NULL;
-	snprintf(mfr, len, "%s %s with %s", init_utsname()->sysname,
-			init_utsname()->release, gadget->name);
-	return mfr;
+	return kasprintf(GFP_KERNEL, "%s %s with %s", init_utsname()->sysname,
+			 init_utsname()->release, gadget->name);
 }
 
 void usb_composite_overwrite_options(struct usb_composite_dev *cdev,
diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c
index 3984787f8e97..78c44979dde3 100644
--- a/drivers/usb/gadget/configfs.c
+++ b/drivers/usb/gadget/configfs.c
@@ -408,7 +408,7 @@ out:
 	return ret;
 }
 
-static int config_usb_cfg_unlink(
+static void config_usb_cfg_unlink(
 	struct config_item *usb_cfg_ci,
 	struct config_item *usb_func_ci)
 {
@@ -437,12 +437,11 @@ static int config_usb_cfg_unlink(
 			list_del(&f->list);
 			usb_put_function(f);
 			mutex_unlock(&gi->lock);
-			return 0;
+			return;
 		}
 	}
 	mutex_unlock(&gi->lock);
 	WARN(1, "Unable to locate function to unbind\n");
-	return 0;
 }
 
 static struct configfs_item_operations gadget_config_item_ops = {
@@ -865,7 +864,7 @@ out:
 	return ret;
 }
 
-static int os_desc_unlink(struct config_item *os_desc_ci,
+static void os_desc_unlink(struct config_item *os_desc_ci,
 			  struct config_item *usb_cfg_ci)
 {
 	struct gadget_info *gi = container_of(to_config_group(os_desc_ci),
@@ -878,7 +877,6 @@ static int os_desc_unlink(struct config_item *os_desc_ci,
 	cdev->os_desc_config = NULL;
 	WARN_ON(gi->composite.gadget_driver.udc_name);
 	mutex_unlock(&gi->lock);
-	return 0;
 }
 
 static struct configfs_item_operations os_desc_ops = {
diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
index 17989b72cdae..0780d8311ec6 100644
--- a/drivers/usb/gadget/function/f_fs.c
+++ b/drivers/usb/gadget/function/f_fs.c
@@ -266,7 +266,7 @@ static void ffs_ep0_complete(struct usb_ep *ep, struct usb_request *req)
 {
 	struct ffs_data *ffs = req->context;
 
-	complete_all(&ffs->ep0req_completion);
+	complete(&ffs->ep0req_completion);
 }
 
 static int __ffs_ep0_queue_wait(struct ffs_data *ffs, char *data, size_t len)
diff --git a/drivers/usb/gadget/function/f_hid.c b/drivers/usb/gadget/function/f_hid.c
index e2966f87c860..3151d2a0fe59 100644
--- a/drivers/usb/gadget/function/f_hid.c
+++ b/drivers/usb/gadget/function/f_hid.c
@@ -98,6 +98,60 @@ static struct hid_descriptor hidg_desc = {
 	/*.desc[0].wDescriptorLenght	= DYNAMIC */
 };
 
+/* Super-Speed Support */
+
+static struct usb_endpoint_descriptor hidg_ss_in_ep_desc = {
+	.bLength		= USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType	= USB_DT_ENDPOINT,
+	.bEndpointAddress	= USB_DIR_IN,
+	.bmAttributes		= USB_ENDPOINT_XFER_INT,
+	/*.wMaxPacketSize	= DYNAMIC */
+	.bInterval		= 4, /* FIXME: Add this field in the
+				      * HID gadget configuration?
+				      * (struct hidg_func_descriptor)
+				      */
+};
+
+static struct usb_ss_ep_comp_descriptor hidg_ss_in_comp_desc = {
+	.bLength                = sizeof(hidg_ss_in_comp_desc),
+	.bDescriptorType        = USB_DT_SS_ENDPOINT_COMP,
+
+	/* .bMaxBurst           = 0, */
+	/* .bmAttributes        = 0, */
+	/* .wBytesPerInterval   = DYNAMIC */
+};
+
+static struct usb_endpoint_descriptor hidg_ss_out_ep_desc = {
+	.bLength		= USB_DT_ENDPOINT_SIZE,
+	.bDescriptorType	= USB_DT_ENDPOINT,
+	.bEndpointAddress	= USB_DIR_OUT,
+	.bmAttributes		= USB_ENDPOINT_XFER_INT,
+	/*.wMaxPacketSize	= DYNAMIC */
+	.bInterval		= 4, /* FIXME: Add this field in the
+				      * HID gadget configuration?
+				      * (struct hidg_func_descriptor)
+				      */
+};
+
+static struct usb_ss_ep_comp_descriptor hidg_ss_out_comp_desc = {
+	.bLength                = sizeof(hidg_ss_out_comp_desc),
+	.bDescriptorType        = USB_DT_SS_ENDPOINT_COMP,
+
+	/* .bMaxBurst           = 0, */
+	/* .bmAttributes        = 0, */
+	/* .wBytesPerInterval   = DYNAMIC */
+};
+
+static struct usb_descriptor_header *hidg_ss_descriptors[] = {
+	(struct usb_descriptor_header *)&hidg_interface_desc,
+	(struct usb_descriptor_header *)&hidg_desc,
+	(struct usb_descriptor_header *)&hidg_ss_in_ep_desc,
+	(struct usb_descriptor_header *)&hidg_ss_in_comp_desc,
+	(struct usb_descriptor_header *)&hidg_ss_out_ep_desc,
+	(struct usb_descriptor_header *)&hidg_ss_out_comp_desc,
+	NULL,
+};
+
 /* High-Speed Support */
 
 static struct usb_endpoint_descriptor hidg_hs_in_ep_desc = {
@@ -624,8 +678,14 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f)
 	/* set descriptor dynamic values */
 	hidg_interface_desc.bInterfaceSubClass = hidg->bInterfaceSubClass;
 	hidg_interface_desc.bInterfaceProtocol = hidg->bInterfaceProtocol;
+	hidg_ss_in_ep_desc.wMaxPacketSize = cpu_to_le16(hidg->report_length);
+	hidg_ss_in_comp_desc.wBytesPerInterval =
+				cpu_to_le16(hidg->report_length);
 	hidg_hs_in_ep_desc.wMaxPacketSize = cpu_to_le16(hidg->report_length);
 	hidg_fs_in_ep_desc.wMaxPacketSize = cpu_to_le16(hidg->report_length);
+	hidg_ss_out_ep_desc.wMaxPacketSize = cpu_to_le16(hidg->report_length);
+	hidg_ss_out_comp_desc.wBytesPerInterval =
+				cpu_to_le16(hidg->report_length);
 	hidg_hs_out_ep_desc.wMaxPacketSize = cpu_to_le16(hidg->report_length);
 	hidg_fs_out_ep_desc.wMaxPacketSize = cpu_to_le16(hidg->report_length);
 	/*
@@ -641,8 +701,13 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f)
 	hidg_hs_out_ep_desc.bEndpointAddress =
 		hidg_fs_out_ep_desc.bEndpointAddress;
 
+	hidg_ss_in_ep_desc.bEndpointAddress =
+		hidg_fs_in_ep_desc.bEndpointAddress;
+	hidg_ss_out_ep_desc.bEndpointAddress =
+		hidg_fs_out_ep_desc.bEndpointAddress;
+
 	status = usb_assign_descriptors(f, hidg_fs_descriptors,
-			hidg_hs_descriptors, NULL, NULL);
+			hidg_hs_descriptors, hidg_ss_descriptors, NULL);
 	if (status)
 		goto fail;
 
@@ -840,7 +905,7 @@ static void hidg_free_inst(struct usb_function_instance *f)
 	mutex_lock(&hidg_ida_lock);
 
 	hidg_put_minor(opts->minor);
-	if (idr_is_empty(&hidg_ida.idr))
+	if (ida_is_empty(&hidg_ida))
 		ghid_cleanup();
 
 	mutex_unlock(&hidg_ida_lock);
@@ -866,7 +931,7 @@ static struct usb_function_instance *hidg_alloc_inst(void)
 
 	mutex_lock(&hidg_ida_lock);
 
-	if (idr_is_empty(&hidg_ida.idr)) {
+	if (ida_is_empty(&hidg_ida)) {
 		status = ghid_setup(NULL, HIDG_MINORS);
 		if (status)  {
 			ret = ERR_PTR(status);
@@ -879,7 +944,7 @@ static struct usb_function_instance *hidg_alloc_inst(void)
 	if (opts->minor < 0) {
 		ret = ERR_PTR(opts->minor);
 		kfree(opts);
-		if (idr_is_empty(&hidg_ida.idr))
+		if (ida_is_empty(&hidg_ida))
 			ghid_cleanup();
 		goto unlock;
 	}
diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c
index 639603722709..e8008fa35e1e 100644
--- a/drivers/usb/gadget/function/f_ncm.c
+++ b/drivers/usb/gadget/function/f_ncm.c
@@ -998,7 +998,7 @@ static struct sk_buff *package_for_tx(struct f_ncm *ncm)
 	/* Merge the skbs */
 	swap(skb2, ncm->skb_tx_data);
 	if (ncm->skb_tx_data) {
-		dev_kfree_skb_any(ncm->skb_tx_data);
+		dev_consume_skb_any(ncm->skb_tx_data);
 		ncm->skb_tx_data = NULL;
 	}
 
@@ -1009,7 +1009,7 @@ static struct sk_buff *package_for_tx(struct f_ncm *ncm)
 	/* Copy NTB across. */
 	ntb_iter = (void *) skb_put(skb2, ncm->skb_tx_ndp->len);
 	memcpy(ntb_iter, ncm->skb_tx_ndp->data, ncm->skb_tx_ndp->len);
-	dev_kfree_skb_any(ncm->skb_tx_ndp);
+	dev_consume_skb_any(ncm->skb_tx_ndp);
 	ncm->skb_tx_ndp = NULL;
 
 	/* Insert zero'd datagram. */
@@ -1078,6 +1078,7 @@ static struct sk_buff *ncm_wrap_ntb(struct gether *port,
 			if (!ncm->skb_tx_data)
 				goto err;
 
+			ncm->skb_tx_data->dev = ncm->netdev;
 			ntb_data = (void *) skb_put(ncm->skb_tx_data, ncb_len);
 			memset(ntb_data, 0, ncb_len);
 			/* dwSignature */
@@ -1096,6 +1097,8 @@ static struct sk_buff *ncm_wrap_ntb(struct gether *port,
 						    GFP_ATOMIC);
 			if (!ncm->skb_tx_ndp)
 				goto err;
+
+			ncm->skb_tx_ndp->dev = ncm->netdev;
 			ntb_ndp = (void *) skb_put(ncm->skb_tx_ndp,
 						    opts->ndp_size);
 			memset(ntb_ndp, 0, ncb_len);
@@ -1133,7 +1136,7 @@ static struct sk_buff *ncm_wrap_ntb(struct gether *port,
 		memset(ntb_data, 0, dgram_pad);
 		ntb_data = (void *) skb_put(ncm->skb_tx_data, skb->len);
 		memcpy(ntb_data, skb->data, skb->len);
-		dev_kfree_skb_any(skb);
+		dev_consume_skb_any(skb);
 		skb = NULL;
 
 	} else if (ncm->skb_tx_data && ncm->timer_force_tx) {
@@ -1329,7 +1332,7 @@ static int ncm_unwrap_ntb(struct gether *port,
 		} while (ndp_len > 2 * (opts->dgram_item_len * 2));
 	} while (ndp_index);
 
-	dev_kfree_skb_any(skb);
+	dev_consume_skb_any(skb);
 
 	VDBG(port->func.config->cdev,
 	     "Parsed NTB with %d frames\n", dgram_counter);
diff --git a/drivers/usb/gadget/function/f_printer.c b/drivers/usb/gadget/function/f_printer.c
index 0de36cda6e41..8054da9276dd 100644
--- a/drivers/usb/gadget/function/f_printer.c
+++ b/drivers/usb/gadget/function/f_printer.c
@@ -1265,7 +1265,7 @@ static void gprinter_free_inst(struct usb_function_instance *f)
 	mutex_lock(&printer_ida_lock);
 
 	gprinter_put_minor(opts->minor);
-	if (idr_is_empty(&printer_ida.idr))
+	if (ida_is_empty(&printer_ida))
 		gprinter_cleanup();
 
 	mutex_unlock(&printer_ida_lock);
@@ -1289,7 +1289,7 @@ static struct usb_function_instance *gprinter_alloc_inst(void)
 
 	mutex_lock(&printer_ida_lock);
 
-	if (idr_is_empty(&printer_ida.idr)) {
+	if (ida_is_empty(&printer_ida)) {
 		status = gprinter_setup(PRINTER_MINORS);
 		if (status) {
 			ret = ERR_PTR(status);
@@ -1302,7 +1302,7 @@ static struct usb_function_instance *gprinter_alloc_inst(void)
 	if (opts->minor < 0) {
 		ret = ERR_PTR(opts->minor);
 		kfree(opts);
-		if (idr_is_empty(&printer_ida.idr))
+		if (ida_is_empty(&printer_ida))
 			gprinter_cleanup();
 		goto unlock;
 	}
diff --git a/drivers/usb/gadget/function/f_uac2.c b/drivers/usb/gadget/function/f_uac2.c
index cd214ec8a601..969cfe741380 100644
--- a/drivers/usb/gadget/function/f_uac2.c
+++ b/drivers/usb/gadget/function/f_uac2.c
@@ -1067,13 +1067,13 @@ afunc_bind(struct usb_configuration *cfg, struct usb_function *fn)
 	agdev->out_ep = usb_ep_autoconfig(gadget, &fs_epout_desc);
 	if (!agdev->out_ep) {
 		dev_err(dev, "%s:%d Error!\n", __func__, __LINE__);
-		goto err;
+		return ret;
 	}
 
 	agdev->in_ep = usb_ep_autoconfig(gadget, &fs_epin_desc);
 	if (!agdev->in_ep) {
 		dev_err(dev, "%s:%d Error!\n", __func__, __LINE__);
-		goto err;
+		return ret;
 	}
 
 	uac2->p_prm.uac2 = uac2;
@@ -1091,7 +1091,7 @@ afunc_bind(struct usb_configuration *cfg, struct usb_function *fn)
 	ret = usb_assign_descriptors(fn, fs_audio_desc, hs_audio_desc, NULL,
 				     NULL);
 	if (ret)
-		goto err;
+		return ret;
 
 	prm = &agdev->uac2.c_prm;
 	prm->max_psize = hs_epout_desc.wMaxPacketSize;
@@ -1106,19 +1106,19 @@ afunc_bind(struct usb_configuration *cfg, struct usb_function *fn)
 	prm->rbuf = kzalloc(prm->max_psize * USB_XFERS, GFP_KERNEL);
 	if (!prm->rbuf) {
 		prm->max_psize = 0;
-		goto err_free_descs;
+		goto err;
 	}
 
 	ret = alsa_uac2_init(agdev);
 	if (ret)
-		goto err_free_descs;
+		goto err;
 	return 0;
 
-err_free_descs:
-	usb_free_all_descriptors(fn);
 err:
 	kfree(agdev->uac2.p_prm.rbuf);
 	kfree(agdev->uac2.c_prm.rbuf);
+err_free_descs:
+	usb_free_all_descriptors(fn);
 	return -EINVAL;
 }
 
diff --git a/drivers/usb/gadget/function/rndis.c b/drivers/usb/gadget/function/rndis.c
index ab6ac1b74ac0..a3b5e468b116 100644
--- a/drivers/usb/gadget/function/rndis.c
+++ b/drivers/usb/gadget/function/rndis.c
@@ -80,8 +80,7 @@ static const struct file_operations rndis_proc_fops;
 #endif /* CONFIG_USB_GADGET_DEBUG_FILES */
 
 /* supported OIDs */
-static const u32 oid_supported_list[] =
-{
+static const u32 oid_supported_list[] = {
 	/* the general stuff */
 	RNDIS_OID_GEN_SUPPORTED_LIST,
 	RNDIS_OID_GEN_HARDWARE_STATUS,
@@ -474,8 +473,7 @@ static int gen_ndis_query_resp(struct rndis_params *params, u32 OID, u8 *buf,
 		break;
 
 	default:
-		pr_warning("%s: query unknown OID 0x%08X\n",
-			 __func__, OID);
+		pr_warn("%s: query unknown OID 0x%08X\n", __func__, OID);
 	}
 	if (retval < 0)
 		length = 0;
@@ -546,8 +544,8 @@ static int gen_ndis_set_resp(struct rndis_params *params, u32 OID,
 		break;
 
 	default:
-		pr_warning("%s: set unknown OID 0x%08X, size %d\n",
-			 __func__, OID, buf_len);
+		pr_warn("%s: set unknown OID 0x%08X, size %d\n",
+			__func__, OID, buf_len);
 	}
 
 	return retval;
@@ -854,7 +852,7 @@ int rndis_msg_parser(struct rndis_params *params, u8 *buf)
 		 * In one case those messages seemed to relate to the host
 		 * suspending itself.
 		 */
-		pr_warning("%s: unknown RNDIS message 0x%08X len %d\n",
+		pr_warn("%s: unknown RNDIS message 0x%08X len %d\n",
 			__func__, MsgType, MsgLength);
 		print_hex_dump_bytes(__func__, DUMP_PREFIX_OFFSET,
 				     buf, MsgLength);
diff --git a/drivers/usb/gadget/function/rndis.h b/drivers/usb/gadget/function/rndis.h
index ef92eb66d8ad..21e0430ffb98 100644
--- a/drivers/usb/gadget/function/rndis.h
+++ b/drivers/usb/gadget/function/rndis.h
@@ -22,8 +22,7 @@
 #define RNDIS_MAXIMUM_FRAME_SIZE	1518
 #define RNDIS_MAX_TOTAL_SIZE		1558
 
-typedef struct rndis_init_msg_type
-{
+typedef struct rndis_init_msg_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	RequestID;
@@ -32,8 +31,7 @@ typedef struct rndis_init_msg_type
 	__le32	MaxTransferSize;
 } rndis_init_msg_type;
 
-typedef struct rndis_init_cmplt_type
-{
+typedef struct rndis_init_cmplt_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	RequestID;
@@ -49,15 +47,13 @@ typedef struct rndis_init_cmplt_type
 	__le32	AFListSize;
 } rndis_init_cmplt_type;
 
-typedef struct rndis_halt_msg_type
-{
+typedef struct rndis_halt_msg_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	RequestID;
 } rndis_halt_msg_type;
 
-typedef struct rndis_query_msg_type
-{
+typedef struct rndis_query_msg_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	RequestID;
@@ -67,8 +63,7 @@ typedef struct rndis_query_msg_type
 	__le32	DeviceVcHandle;
 } rndis_query_msg_type;
 
-typedef struct rndis_query_cmplt_type
-{
+typedef struct rndis_query_cmplt_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	RequestID;
@@ -77,8 +72,7 @@ typedef struct rndis_query_cmplt_type
 	__le32	InformationBufferOffset;
 } rndis_query_cmplt_type;
 
-typedef struct rndis_set_msg_type
-{
+typedef struct rndis_set_msg_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	RequestID;
@@ -88,31 +82,27 @@ typedef struct rndis_set_msg_type
 	__le32	DeviceVcHandle;
 } rndis_set_msg_type;
 
-typedef struct rndis_set_cmplt_type
-{
+typedef struct rndis_set_cmplt_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	RequestID;
 	__le32	Status;
 } rndis_set_cmplt_type;
 
-typedef struct rndis_reset_msg_type
-{
+typedef struct rndis_reset_msg_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	Reserved;
 } rndis_reset_msg_type;
 
-typedef struct rndis_reset_cmplt_type
-{
+typedef struct rndis_reset_cmplt_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	Status;
 	__le32	AddressingReset;
 } rndis_reset_cmplt_type;
 
-typedef struct rndis_indicate_status_msg_type
-{
+typedef struct rndis_indicate_status_msg_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	Status;
@@ -120,23 +110,20 @@ typedef struct rndis_indicate_status_msg_type
 	__le32	StatusBufferOffset;
 } rndis_indicate_status_msg_type;
 
-typedef struct rndis_keepalive_msg_type
-{
+typedef struct rndis_keepalive_msg_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	RequestID;
 } rndis_keepalive_msg_type;
 
-typedef struct rndis_keepalive_cmplt_type
-{
+typedef struct rndis_keepalive_cmplt_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	RequestID;
 	__le32	Status;
 } rndis_keepalive_cmplt_type;
 
-struct rndis_packet_msg_type
-{
+struct rndis_packet_msg_type {
 	__le32	MessageType;
 	__le32	MessageLength;
 	__le32	DataOffset;
@@ -150,8 +137,7 @@ struct rndis_packet_msg_type
 	__le32	Reserved;
 } __attribute__ ((packed));
 
-struct rndis_config_parameter
-{
+struct rndis_config_parameter {
 	__le32	ParameterNameOffset;
 	__le32	ParameterNameLength;
 	__le32	ParameterType;
@@ -160,23 +146,20 @@ struct rndis_config_parameter
 };
 
 /* implementation specific */
-enum rndis_state
-{
+enum rndis_state {
 	RNDIS_UNINITIALIZED,
 	RNDIS_INITIALIZED,
 	RNDIS_DATA_INITIALIZED,
 };
 
-typedef struct rndis_resp_t
-{
+typedef struct rndis_resp_t {
 	struct list_head	list;
 	u8			*buf;
 	u32			length;
 	int			send;
 } rndis_resp_t;
 
-typedef struct rndis_params
-{
+typedef struct rndis_params {
 	int			confignr;
 	u8			used;
 	u16			saved_filter;
diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c
index 84a1709e0784..b4e5d6dfd549 100644
--- a/drivers/usb/gadget/function/u_ether.c
+++ b/drivers/usb/gadget/function/u_ether.c
@@ -215,7 +215,7 @@ rx_submit(struct eth_dev *dev, struct usb_request *req, gfp_t gfp_flags)
 	if (dev->port_usb->is_fixed)
 		size = max_t(size_t, size, dev->port_usb->fixed_out_len);
 
-	skb = alloc_skb(size + NET_IP_ALIGN, gfp_flags);
+	skb = __netdev_alloc_skb(dev->net, size + NET_IP_ALIGN, gfp_flags);
 	if (skb == NULL) {
 		DBG(dev, "no rx skb\n");
 		goto enomem;
@@ -446,16 +446,17 @@ static void tx_complete(struct usb_ep *ep, struct usb_request *req)
 		/* FALLTHROUGH */
 	case -ECONNRESET:		/* unlink */
 	case -ESHUTDOWN:		/* disconnect etc */
+		dev_kfree_skb_any(skb);
 		break;
 	case 0:
 		dev->net->stats.tx_bytes += skb->len;
+		dev_consume_skb_any(skb);
 	}
 	dev->net->stats.tx_packets++;
 
 	spin_lock(&dev->req_lock);
 	list_add(&req->list, &dev->tx_reqs);
 	spin_unlock(&dev->req_lock);
-	dev_kfree_skb_any(skb);
 
 	atomic_dec(&dev->tx_qlen);
 	if (netif_carrier_ok(dev->net))
diff --git a/drivers/usb/gadget/function/u_serial.c b/drivers/usb/gadget/function/u_serial.c
index e0cd1e4c8892..000677c991b0 100644
--- a/drivers/usb/gadget/function/u_serial.c
+++ b/drivers/usb/gadget/function/u_serial.c
@@ -622,8 +622,8 @@ static void gs_write_complete(struct usb_ep *ep, struct usb_request *req)
 	switch (req->status) {
 	default:
 		/* presumably a transient fault */
-		pr_warning("%s: unexpected %s status %d\n",
-				__func__, ep->name, req->status);
+		pr_warn("%s: unexpected %s status %d\n",
+			__func__, ep->name, req->status);
 		/* FALL THROUGH */
 	case 0:
 		/* normal completion */
@@ -1256,7 +1256,8 @@ static void gserial_console_exit(void)
 	struct gscons_info *info = &gscons_info;
 
 	unregister_console(&gserial_cons);
-	kthread_stop(info->console_thread);
+	if (info->console_thread != NULL)
+		kthread_stop(info->console_thread);
 	gs_buf_free(&info->con_buf);
 }
 
diff --git a/drivers/usb/gadget/function/uvc.h b/drivers/usb/gadget/function/uvc.h
index 7d3bb6272e06..11d70dead32b 100644
--- a/drivers/usb/gadget/function/uvc.h
+++ b/drivers/usb/gadget/function/uvc.h
@@ -26,14 +26,12 @@
 #define UVC_EVENT_DATA			(V4L2_EVENT_PRIVATE_START + 5)
 #define UVC_EVENT_LAST			(V4L2_EVENT_PRIVATE_START + 5)
 
-struct uvc_request_data
-{
+struct uvc_request_data {
 	__s32 length;
 	__u8 data[60];
 };
 
-struct uvc_event
-{
+struct uvc_event {
 	union {
 		enum usb_device_speed speed;
 		struct usb_ctrlrequest req;
@@ -104,8 +102,7 @@ extern unsigned int uvc_gadget_trace_param;
  * Structures
  */
 
-struct uvc_video
-{
+struct uvc_video {
 	struct usb_ep *ep;
 
 	/* Frame parameters */
@@ -134,15 +131,13 @@ struct uvc_video
 	unsigned int fid;
 };
 
-enum uvc_state
-{
+enum uvc_state {
 	UVC_STATE_DISCONNECTED,
 	UVC_STATE_CONNECTED,
 	UVC_STATE_STREAMING,
 };
 
-struct uvc_device
-{
+struct uvc_device {
 	struct video_device vdev;
 	struct v4l2_device v4l2_dev;
 	enum uvc_state state;
@@ -175,8 +170,7 @@ static inline struct uvc_device *to_uvc(struct usb_function *f)
 	return container_of(f, struct uvc_device, func);
 }
 
-struct uvc_file_handle
-{
+struct uvc_file_handle {
 	struct v4l2_fh vfh;
 	struct uvc_video *device;
 };
diff --git a/drivers/usb/gadget/function/uvc_configfs.c b/drivers/usb/gadget/function/uvc_configfs.c
index 31125a4a2658..4e037d2a7a60 100644
--- a/drivers/usb/gadget/function/uvc_configfs.c
+++ b/drivers/usb/gadget/function/uvc_configfs.c
@@ -547,7 +547,7 @@ out:
 	return ret;
 }
 
-static int uvcg_control_class_drop_link(struct config_item *src,
+static void uvcg_control_class_drop_link(struct config_item *src,
 					struct config_item *target)
 {
 	struct config_item *control, *header;
@@ -555,7 +555,6 @@ static int uvcg_control_class_drop_link(struct config_item *src,
 	struct mutex *su_mutex = &src->ci_group->cg_subsys->su_mutex;
 	struct uvc_descriptor_header **class_array;
 	struct uvcg_control_header *target_hdr;
-	int ret = -EINVAL;
 
 	mutex_lock(su_mutex); /* for navigating configfs hierarchy */
 
@@ -569,23 +568,17 @@ static int uvcg_control_class_drop_link(struct config_item *src,
 	mutex_lock(&opts->lock);
 
 	class_array = uvcg_get_ctl_class_arr(src, opts);
-	if (!class_array)
-		goto unlock;
-	if (opts->refcnt) {
-		ret = -EBUSY;
+	if (!class_array || opts->refcnt)
 		goto unlock;
-	}
 
 	target_hdr = to_uvcg_control_header(target);
 	--target_hdr->linked;
 	class_array[0] = NULL;
-	ret = 0;
 
 unlock:
 	mutex_unlock(&opts->lock);
 out:
 	mutex_unlock(su_mutex);
-	return ret;
 }
 
 static struct configfs_item_operations uvcg_control_class_item_ops = {
@@ -777,7 +770,7 @@ out:
 	return ret;
 }
 
-static int uvcg_streaming_header_drop_link(struct config_item *src,
+static void uvcg_streaming_header_drop_link(struct config_item *src,
 					   struct config_item *target)
 {
 	struct mutex *su_mutex = &src->ci_group->cg_subsys->su_mutex;
@@ -786,7 +779,6 @@ static int uvcg_streaming_header_drop_link(struct config_item *src,
 	struct uvcg_streaming_header *src_hdr;
 	struct uvcg_format *target_fmt = NULL;
 	struct uvcg_format_ptr *format_ptr, *tmp;
-	int ret = -EINVAL;
 
 	src_hdr = to_uvcg_streaming_header(src);
 	mutex_lock(su_mutex); /* for navigating configfs hierarchy */
@@ -811,8 +803,6 @@ static int uvcg_streaming_header_drop_link(struct config_item *src,
 out:
 	mutex_unlock(&opts->lock);
 	mutex_unlock(su_mutex);
-	return ret;
-
 }
 
 static struct configfs_item_operations uvcg_streaming_header_item_ops = {
@@ -2051,7 +2041,7 @@ out:
 	return ret;
 }
 
-static int uvcg_streaming_class_drop_link(struct config_item *src,
+static void uvcg_streaming_class_drop_link(struct config_item *src,
 					  struct config_item *target)
 {
 	struct config_item *streaming, *header;
@@ -2059,7 +2049,6 @@ static int uvcg_streaming_class_drop_link(struct config_item *src,
 	struct mutex *su_mutex = &src->ci_group->cg_subsys->su_mutex;
 	struct uvc_descriptor_header ***class_array;
 	struct uvcg_streaming_header *target_hdr;
-	int ret = -EINVAL;
 
 	mutex_lock(su_mutex); /* for navigating configfs hierarchy */
 
@@ -2076,23 +2065,19 @@ static int uvcg_streaming_class_drop_link(struct config_item *src,
 	if (!class_array || !*class_array)
 		goto unlock;
 
-	if (opts->refcnt) {
-		ret = -EBUSY;
+	if (opts->refcnt)
 		goto unlock;
-	}
 
 	target_hdr = to_uvcg_streaming_header(target);
 	--target_hdr->linked;
 	kfree(**class_array);
 	kfree(*class_array);
 	*class_array = NULL;
-	ret = 0;
 
 unlock:
 	mutex_unlock(&opts->lock);
 out:
 	mutex_unlock(su_mutex);
-	return ret;
 }
 
 static struct configfs_item_operations uvcg_streaming_class_item_ops = {
diff --git a/drivers/usb/gadget/function/uvc_v4l2.c b/drivers/usb/gadget/function/uvc_v4l2.c
index f4ccbd56f4d2..3e22b45687d3 100644
--- a/drivers/usb/gadget/function/uvc_v4l2.c
+++ b/drivers/usb/gadget/function/uvc_v4l2.c
@@ -53,8 +53,7 @@ uvc_send_response(struct uvc_device *uvc, struct uvc_request_data *data)
  * V4L2 ioctls
  */
 
-struct uvc_format
-{
+struct uvc_format {
 	u8 bpp;
 	u32 fcc;
 };
diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c
index 3d0d5d94a62f..0f01c04d7cbd 100644
--- a/drivers/usb/gadget/function/uvc_video.c
+++ b/drivers/usb/gadget/function/uvc_video.c
@@ -243,7 +243,7 @@ uvc_video_alloc_requests(struct uvc_video *video)
 
 	req_size = video->ep->maxpacket
 		 * max_t(unsigned int, video->ep->maxburst, 1)
-		 * (video->ep->mult + 1);
+		 * (video->ep->mult);
 
 	for (i = 0; i < UVC_NUM_REQUESTS; ++i) {
 		video->req_buffer[i] = kmalloc(req_size, GFP_KERNEL);
diff --git a/drivers/usb/gadget/udc/at91_udc.h b/drivers/usb/gadget/udc/at91_udc.h
index 0a433e6b346b..9bbe72764f31 100644
--- a/drivers/usb/gadget/udc/at91_udc.h
+++ b/drivers/usb/gadget/udc/at91_udc.h
@@ -175,7 +175,7 @@ struct at91_request {
 #endif
 
 #define ERR(stuff...)		pr_err("udc: " stuff)
-#define WARNING(stuff...)	pr_warning("udc: " stuff)
+#define WARNING(stuff...)	pr_warn("udc: " stuff)
 #define INFO(stuff...)		pr_info("udc: " stuff)
 #define DBG(stuff...)		pr_debug("udc: " stuff)
 
diff --git a/drivers/usb/gadget/udc/atmel_usba_udc.c b/drivers/usb/gadget/udc/atmel_usba_udc.c
index 45bc997d0711..f3212db9bc37 100644
--- a/drivers/usb/gadget/udc/atmel_usba_udc.c
+++ b/drivers/usb/gadget/udc/atmel_usba_udc.c
@@ -529,7 +529,7 @@ usba_ep_enable(struct usb_ep *_ep, const struct usb_endpoint_descriptor *desc)
 
 	DBG(DBG_GADGET, "%s: ep_enable: desc=%p\n", ep->ep.name, desc);
 
-	maxpacket = usb_endpoint_maxp(desc) & 0x7ff;
+	maxpacket = usb_endpoint_maxp(desc);
 
 	if (((desc->bEndpointAddress & USB_ENDPOINT_NUMBER_MASK) != ep->index)
 			|| ep->index == 0
@@ -573,7 +573,7 @@ usba_ep_enable(struct usb_ep *_ep, const struct usb_endpoint_descriptor *desc)
 		 * Bits 11:12 specify number of _additional_
 		 * transactions per microframe.
 		 */
-		nr_trans = ((usb_endpoint_maxp(desc) >> 11) & 3) + 1;
+		nr_trans = usb_endpoint_maxp_mult(desc);
 		if (nr_trans > 3)
 			return -EINVAL;
 
@@ -1464,8 +1464,8 @@ restart:
 		pkt_len = USBA_BFEXT(BYTE_COUNT, usba_ep_readl(ep, STA));
 		DBG(DBG_HW, "Packet length: %u\n", pkt_len);
 		if (pkt_len != sizeof(crq)) {
-			pr_warning("udc: Invalid packet length %u "
-				"(expected %zu)\n", pkt_len, sizeof(crq));
+			pr_warn("udc: Invalid packet length %u (expected %zu)\n",
+				pkt_len, sizeof(crq));
 			set_protocol_stall(udc, ep);
 			return;
 		}
diff --git a/drivers/usb/gadget/udc/bdc/bdc_cmd.c b/drivers/usb/gadget/udc/bdc/bdc_cmd.c
index 4d5e9188beae..6e920f1dce02 100644
--- a/drivers/usb/gadget/udc/bdc/bdc_cmd.c
+++ b/drivers/usb/gadget/udc/bdc/bdc_cmd.c
@@ -182,7 +182,7 @@ int bdc_config_ep(struct bdc *bdc, struct bdc_ep *ep)
 					usb_endpoint_xfer_int(desc)) {
 			param2 |= si;
 
-			mbs = (usb_endpoint_maxp(desc) & 0x1800) >> 11;
+			mbs = usb_endpoint_maxp_mult(desc);
 			param2 |= mbs << MB_SHIFT;
 		}
 		break;
diff --git a/drivers/usb/gadget/udc/bdc/bdc_ep.c b/drivers/usb/gadget/udc/bdc/bdc_ep.c
index ccaa74ab6c0e..ff1ef24d1777 100644
--- a/drivers/usb/gadget/udc/bdc/bdc_ep.c
+++ b/drivers/usb/gadget/udc/bdc/bdc_ep.c
@@ -446,7 +446,7 @@ static int setup_bd_list_xfr(struct bdc *bdc, struct bdc_req *req, int num_bds)
 	bd_xfr->start_bdi = bd_list->eqp_bdi;
 	bd = bdi_to_bd(ep, bd_list->eqp_bdi);
 	req_len = req->usb_req.length;
-	maxp = usb_endpoint_maxp(ep->desc) & 0x7ff;
+	maxp = usb_endpoint_maxp(ep->desc);
 	tfs = roundup(req->usb_req.length, maxp);
 	tfs = tfs/maxp;
 	dev_vdbg(bdc->dev, "%s ep:%s num_bds:%d tfs:%d r_len:%d bd:%p\n",
diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c
index 77d07904f932..02b14e91ae6c 100644
--- a/drivers/usb/gadget/udc/dummy_hcd.c
+++ b/drivers/usb/gadget/udc/dummy_hcd.c
@@ -503,7 +503,7 @@ static int dummy_enable(struct usb_ep *_ep,
 	 * maximum packet size.
 	 * For SS devices the wMaxPacketSize is limited by 1024.
 	 */
-	max = usb_endpoint_maxp(desc) & 0x7ff;
+	max = usb_endpoint_maxp(desc);
 
 	/* drivers must not request bad settings, since lower levels
 	 * (hardware or its drivers) may not check.  some endpoints
@@ -1483,8 +1483,7 @@ static int periodic_bytes(struct dummy *dum, struct dummy_ep *ep)
 		int	tmp;
 
 		/* high bandwidth mode */
-		tmp = usb_endpoint_maxp(ep->desc);
-		tmp = (tmp >> 11) & 0x03;
+		tmp = usb_endpoint_maxp_mult(ep->desc);
 		tmp *= 8 /* applies to entire frame */;
 		limit += limit * tmp;
 	}
diff --git a/drivers/usb/gadget/udc/fsl_udc_core.c b/drivers/usb/gadget/udc/fsl_udc_core.c
index aab5221d6c2e..71094e479a96 100644
--- a/drivers/usb/gadget/udc/fsl_udc_core.c
+++ b/drivers/usb/gadget/udc/fsl_udc_core.c
@@ -585,8 +585,7 @@ static int fsl_ep_enable(struct usb_ep *_ep,
 		break;
 	case USB_ENDPOINT_XFER_ISOC:
 		/* Calculate transactions needed for high bandwidth iso */
-		mult = (unsigned char)(1 + ((max >> 11) & 0x03));
-		max = max & 0x7ff;	/* bit 0~10 */
+		mult = usb_endpoint_maxp_mult(desc);
 		/* 3 transactions at most */
 		if (mult > 3)
 			goto en_done;
diff --git a/drivers/usb/gadget/udc/fsl_usb2_udc.h b/drivers/usb/gadget/udc/fsl_usb2_udc.h
index 84715625b2b3..e92b8408b6f6 100644
--- a/drivers/usb/gadget/udc/fsl_usb2_udc.h
+++ b/drivers/usb/gadget/udc/fsl_usb2_udc.h
@@ -554,7 +554,7 @@ static void dump_msg(const char *label, const u8 * buf, unsigned int length)
 #endif
 
 #define ERR(stuff...)		pr_err("udc: " stuff)
-#define WARNING(stuff...)		pr_warning("udc: " stuff)
+#define WARNING(stuff...)	pr_warn("udc: " stuff)
 #define INFO(stuff...)		pr_info("udc: " stuff)
 
 /*-------------------------------------------------------------------------*/
diff --git a/drivers/usb/gadget/udc/fusb300_udc.c b/drivers/usb/gadget/udc/fusb300_udc.c
index 948845c90e47..42ff308578df 100644
--- a/drivers/usb/gadget/udc/fusb300_udc.c
+++ b/drivers/usb/gadget/udc/fusb300_udc.c
@@ -218,7 +218,7 @@ static int config_ep(struct fusb300_ep *ep,
 	   (info.type == USB_ENDPOINT_XFER_ISOC)) {
 		info.interval = desc->bInterval;
 		if (info.type == USB_ENDPOINT_XFER_ISOC)
-			info.bw_num = ((desc->wMaxPacketSize & 0x1800) >> 11);
+			info.bw_num = usb_endpoint_maxp_mult(desc);
 	}
 
 	ep_fifo_setting(fusb300, info);
diff --git a/drivers/usb/gadget/udc/gr_udc.c b/drivers/usb/gadget/udc/gr_udc.c
index 39b7136d31d9..b16f8af34050 100644
--- a/drivers/usb/gadget/udc/gr_udc.c
+++ b/drivers/usb/gadget/udc/gr_udc.c
@@ -1539,7 +1539,7 @@ static int gr_ep_enable(struct usb_ep *_ep,
 	 * additional transactions.
 	 */
 	max = 0x7ff & usb_endpoint_maxp(desc);
-	nt = 0x3 & (usb_endpoint_maxp(desc) >> 11);
+	nt = usb_endpoint_maxp_mult(desc) - 1;
 	buffer_size = GR_BUFFER_SIZE(epctrl);
 	if (nt && (mode == 0 || mode == 2)) {
 		dev_err(dev->dev,
diff --git a/drivers/usb/gadget/udc/m66592-udc.c b/drivers/usb/gadget/udc/m66592-udc.c
index 6e977dc22570..de3e03483659 100644
--- a/drivers/usb/gadget/udc/m66592-udc.c
+++ b/drivers/usb/gadget/udc/m66592-udc.c
@@ -637,7 +637,7 @@ static void init_controller(struct m66592 *m66592)
 			clock = M66592_XTAL48;
 			break;
 		default:
-			pr_warning("m66592-udc: xtal configuration error\n");
+			pr_warn("m66592-udc: xtal configuration error\n");
 			clock = 0;
 		}
 
@@ -649,7 +649,7 @@ static void init_controller(struct m66592 *m66592)
 			irq_sense = 0;
 			break;
 		default:
-			pr_warning("m66592-udc: irq trigger config error\n");
+			pr_warn("m66592-udc: irq trigger config error\n");
 			irq_sense = 0;
 		}
 
diff --git a/drivers/usb/gadget/udc/mv_u3d_core.c b/drivers/usb/gadget/udc/mv_u3d_core.c
index b9e19a591322..8d726bd767fd 100644
--- a/drivers/usb/gadget/udc/mv_u3d_core.c
+++ b/drivers/usb/gadget/udc/mv_u3d_core.c
@@ -462,6 +462,12 @@ static int mv_u3d_req_to_trb(struct mv_u3d_req *req)
 					req->trb_head->trb_hw,
 					trb_num * sizeof(*trb_hw),
 					DMA_BIDIRECTIONAL);
+		if (dma_mapping_error(u3d->gadget.dev.parent,
+					req->trb_head->trb_dma)) {
+			kfree(req->trb_head->trb_hw);
+			kfree(req->trb_head);
+			return -EFAULT;
+		}
 
 		req->chain = 1;
 	}
@@ -487,30 +493,32 @@ mv_u3d_start_queue(struct mv_u3d_ep *ep)
 	ret = usb_gadget_map_request(&u3d->gadget, &req->req,
 					mv_u3d_ep_dir(ep));
 	if (ret)
-		return ret;
+		goto break_processing;
 
 	req->req.status = -EINPROGRESS;
 	req->req.actual = 0;
 	req->trb_count = 0;
 
-	/* build trbs and push them to device queue */
-	if (!mv_u3d_req_to_trb(req)) {
-		ret = mv_u3d_queue_trb(ep, req);
-		if (ret) {
-			ep->processing = 0;
-			return ret;
-		}
-	} else {
-		ep->processing = 0;
+	/* build trbs */
+	ret = mv_u3d_req_to_trb(req);
+	if (ret) {
 		dev_err(u3d->dev, "%s, mv_u3d_req_to_trb fail\n", __func__);
-		return -ENOMEM;
+		goto break_processing;
 	}
 
+	/* and push them to device queue */
+	ret = mv_u3d_queue_trb(ep, req);
+	if (ret)
+		goto break_processing;
+
 	/* irq handler advances the queue */
-	if (req)
-		list_add_tail(&req->queue, &ep->queue);
+	list_add_tail(&req->queue, &ep->queue);
 
 	return 0;
+
+break_processing:
+	ep->processing = 0;
+	return ret;
 }
 
 static int mv_u3d_ep_enable(struct usb_ep *_ep,
diff --git a/drivers/usb/gadget/udc/mv_udc_core.c b/drivers/usb/gadget/udc/mv_udc_core.c
index ce73b3552269..d82a91bddbd9 100644
--- a/drivers/usb/gadget/udc/mv_udc_core.c
+++ b/drivers/usb/gadget/udc/mv_udc_core.c
@@ -494,8 +494,7 @@ static int mv_ep_enable(struct usb_ep *_ep,
 		break;
 	case USB_ENDPOINT_XFER_ISOC:
 		/* Calculate transactions needed for high bandwidth iso */
-		mult = (unsigned char)(1 + ((max >> 11) & 0x03));
-		max = max & 0x7ff;	/* bit 0~10 */
+		mult = usb_endpoint_maxp_mult(desc);
 		/* 3 transactions at most */
 		if (mult > 3)
 			goto en_done;
diff --git a/drivers/usb/gadget/udc/net2272.c b/drivers/usb/gadget/udc/net2272.c
index 7c6113432093..078c91d546e0 100644
--- a/drivers/usb/gadget/udc/net2272.c
+++ b/drivers/usb/gadget/udc/net2272.c
@@ -202,10 +202,10 @@ net2272_enable(struct usb_ep *_ep, const struct usb_endpoint_descriptor *desc)
 	if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
 		return -ESHUTDOWN;
 
-	max = usb_endpoint_maxp(desc) & 0x1fff;
+	max = usb_endpoint_maxp(desc);
 
 	spin_lock_irqsave(&dev->lock, flags);
-	_ep->maxpacket = max & 0x7fff;
+	_ep->maxpacket = max;
 	ep->desc = desc;
 
 	/* net2272_ep_reset() has already been called */
diff --git a/drivers/usb/gadget/udc/net2280.c b/drivers/usb/gadget/udc/net2280.c
index 61c938c36d88..85504419ab31 100644
--- a/drivers/usb/gadget/udc/net2280.c
+++ b/drivers/usb/gadget/udc/net2280.c
@@ -224,14 +224,14 @@ net2280_enable(struct usb_ep *_ep, const struct usb_endpoint_descriptor *desc)
 	}
 
 	/* sanity check ep-e/ep-f since their fifos are small */
-	max = usb_endpoint_maxp(desc) & 0x1fff;
+	max = usb_endpoint_maxp(desc);
 	if (ep->num > 4 && max > 64 && (dev->quirks & PLX_LEGACY)) {
 		ret = -ERANGE;
 		goto print_err;
 	}
 
 	spin_lock_irqsave(&dev->lock, flags);
-	_ep->maxpacket = max & 0x7ff;
+	_ep->maxpacket = max;
 	ep->desc = desc;
 
 	/* ep_reset() has already been called */
@@ -1839,7 +1839,7 @@ static ssize_t queues_show(struct device *_dev, struct device_attribute *attr,
 				ep->ep.name, t & USB_ENDPOINT_NUMBER_MASK,
 				(t & USB_DIR_IN) ? "in" : "out",
 				type_string(d->bmAttributes),
-				usb_endpoint_maxp(d) & 0x1fff,
+				usb_endpoint_maxp(d),
 				ep->dma ? "dma" : "pio", ep->fifo_size
 				);
 		} else /* ep0 should only have one transfer queued */
diff --git a/drivers/usb/gadget/udc/omap_udc.h b/drivers/usb/gadget/udc/omap_udc.h
index cfadeb5fc5de..26974196cf44 100644
--- a/drivers/usb/gadget/udc/omap_udc.h
+++ b/drivers/usb/gadget/udc/omap_udc.h
@@ -187,7 +187,7 @@ struct omap_udc {
 #endif
 
 #define ERR(stuff...)		pr_err("udc: " stuff)
-#define WARNING(stuff...)	pr_warning("udc: " stuff)
+#define WARNING(stuff...)	pr_warn("udc: " stuff)
 #define INFO(stuff...)		pr_info("udc: " stuff)
 #define DBG(stuff...)		pr_debug("udc: " stuff)
 
diff --git a/drivers/usb/gadget/udc/pxa25x_udc.h b/drivers/usb/gadget/udc/pxa25x_udc.h
index 4b8b72d7ab37..a458bec2536d 100644
--- a/drivers/usb/gadget/udc/pxa25x_udc.h
+++ b/drivers/usb/gadget/udc/pxa25x_udc.h
@@ -248,7 +248,7 @@ dump_state(struct pxa25x_udc *dev)
 #define DBG(lvl, stuff...) do{if ((lvl) <= UDC_DEBUG) DMSG(stuff);}while(0)
 
 #define ERR(stuff...)		pr_err("udc: " stuff)
-#define WARNING(stuff...)	pr_warning("udc: " stuff)
+#define WARNING(stuff...)	pr_warn("udc: " stuff)
 #define INFO(stuff...)		pr_info("udc: " stuff)
 
 
diff --git a/drivers/usb/gadget/udc/s3c2410_udc.c b/drivers/usb/gadget/udc/s3c2410_udc.c
index eb3571ee59e3..4643a01262b4 100644
--- a/drivers/usb/gadget/udc/s3c2410_udc.c
+++ b/drivers/usb/gadget/udc/s3c2410_udc.c
@@ -1047,10 +1047,10 @@ static int s3c2410_udc_ep_enable(struct usb_ep *_ep,
 	if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
 		return -ESHUTDOWN;
 
-	max = usb_endpoint_maxp(desc) & 0x1fff;
+	max = usb_endpoint_maxp(desc);
 
 	local_irq_save(flags);
-	_ep->maxpacket = max & 0x7ff;
+	_ep->maxpacket = max;
 	ep->ep.desc = desc;
 	ep->halted = 0;
 	ep->bEndpointAddress = desc->bEndpointAddress;
diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig
index 0b80cee30da4..6361fc739306 100644
--- a/drivers/usb/host/Kconfig
+++ b/drivers/usb/host/Kconfig
@@ -479,9 +479,10 @@ config USB_OHCI_HCD_OMAP3
 	  OMAP3 and later chips.
 
 config USB_OHCI_HCD_DAVINCI
-	bool "OHCI support for TI DaVinci DA8xx"
+	tristate "OHCI support for TI DaVinci DA8xx"
 	depends on ARCH_DAVINCI_DA8XX
-	depends on USB_OHCI_HCD=y
+	depends on USB_OHCI_HCD
+	select PHY_DA8XX_USB
 	default y
 	help
 	  Enables support for the DaVinci DA8xx integrated OHCI
diff --git a/drivers/usb/host/Makefile b/drivers/usb/host/Makefile
index 6ef785b0ea8f..2644537b7bcf 100644
--- a/drivers/usb/host/Makefile
+++ b/drivers/usb/host/Makefile
@@ -61,6 +61,7 @@ obj-$(CONFIG_USB_OHCI_HCD_AT91)	+= ohci-at91.o
 obj-$(CONFIG_USB_OHCI_HCD_S3C2410)	+= ohci-s3c2410.o
 obj-$(CONFIG_USB_OHCI_HCD_LPC32XX)	+= ohci-nxp.o
 obj-$(CONFIG_USB_OHCI_HCD_PXA27X)	+= ohci-pxa27x.o
+obj-$(CONFIG_USB_OHCI_HCD_DAVINCI)	+= ohci-da8xx.o
 
 obj-$(CONFIG_USB_UHCI_HCD)	+= uhci-hcd.o
 obj-$(CONFIG_USB_FHCI_HCD)	+= fhci.o
diff --git a/drivers/usb/host/ehci-fsl.c b/drivers/usb/host/ehci-fsl.c
index 9f5ffb629973..91701cc68082 100644
--- a/drivers/usb/host/ehci-fsl.c
+++ b/drivers/usb/host/ehci-fsl.c
@@ -286,6 +286,9 @@ static int ehci_fsl_usb_setup(struct ehci_hcd *ehci)
 	if (pdata->has_fsl_erratum_a005275 == 1)
 		ehci->has_fsl_hs_errata = 1;
 
+	if (pdata->has_fsl_erratum_a005697 == 1)
+		ehci->has_fsl_susp_errata = 1;
+
 	if ((pdata->operating_mode == FSL_USB2_DR_HOST) ||
 			(pdata->operating_mode == FSL_USB2_DR_OTG))
 		if (ehci_fsl_setup_phy(hcd, pdata->phy_mode, 0))
diff --git a/drivers/usb/host/ehci-hub.c b/drivers/usb/host/ehci-hub.c
index 74f62d68f013..df169c8e7225 100644
--- a/drivers/usb/host/ehci-hub.c
+++ b/drivers/usb/host/ehci-hub.c
@@ -310,6 +310,14 @@ static int ehci_bus_suspend (struct usb_hcd *hcd)
 	}
 	spin_unlock_irq(&ehci->lock);
 
+	if (changed && ehci_has_fsl_susp_errata(ehci))
+		/*
+		 * Wait for at least 10 millisecondes to ensure the controller
+		 * enter the suspend status before initiating a port resume
+		 * using the Force Port Resume bit (Not-EHCI compatible).
+		 */
+		usleep_range(10000, 20000);
+
 	if ((changed && ehci->has_tdi_phy_lpm) || fs_idle_delay) {
 		/*
 		 * Wait for HCD to enter low-power mode or for the bus
@@ -1200,6 +1208,12 @@ int ehci_hub_control(
 					wIndex, (temp1 & HOSTPC_PHCD) ?
 					"succeeded" : "failed");
 			}
+			if (ehci_has_fsl_susp_errata(ehci)) {
+				/* 10ms for HCD enter suspend */
+				spin_unlock_irqrestore(&ehci->lock, flags);
+				usleep_range(10000, 20000);
+				spin_lock_irqsave(&ehci->lock, flags);
+			}
 			set_bit(wIndex, &ehci->suspended_ports);
 			break;
 		case USB_PORT_FEAT_POWER:
diff --git a/drivers/usb/host/ehci-pci.c b/drivers/usb/host/ehci-pci.c
index 3b3649d88c5f..93326974ff4b 100644
--- a/drivers/usb/host/ehci-pci.c
+++ b/drivers/usb/host/ehci-pci.c
@@ -258,9 +258,8 @@ static int ehci_pci_setup(struct usb_hcd *hcd)
 	/* These workarounds need to be applied after ehci_setup() */
 	switch (pdev->vendor) {
 	case PCI_VENDOR_ID_NEC:
-		ehci->need_io_watchdog = 0;
-		break;
 	case PCI_VENDOR_ID_INTEL:
+	case PCI_VENDOR_ID_AMD:
 		ehci->need_io_watchdog = 0;
 		break;
 	case PCI_VENDOR_ID_NVIDIA:
diff --git a/drivers/usb/host/ehci-q.c b/drivers/usb/host/ehci-q.c
index eca3710d8fc4..8f3f055c05fa 100644
--- a/drivers/usb/host/ehci-q.c
+++ b/drivers/usb/host/ehci-q.c
@@ -550,11 +550,6 @@ qh_completions (struct ehci_hcd *ehci, struct ehci_qh *qh)
 
 /*-------------------------------------------------------------------------*/
 
-// high bandwidth multiplier, as encoded in highspeed endpoint descriptors
-#define hb_mult(wMaxPacketSize) (1 + (((wMaxPacketSize) >> 11) & 0x03))
-// ... and packet size, for any kind of endpoint descriptor
-#define max_packet(wMaxPacketSize) ((wMaxPacketSize) & 0x07ff)
-
 /*
  * reverse of qh_urb_transaction:  free a list of TDs.
  * used for cleanup after errors, before HC sees an URB's TDs.
@@ -651,7 +646,7 @@ qh_urb_transaction (
 		token |= (1 /* "in" */ << 8);
 	/* else it's already initted to "out" pid (0 << 8) */
 
-	maxpacket = max_packet(usb_maxpacket(urb->dev, urb->pipe, !is_input));
+	maxpacket = usb_maxpacket(urb->dev, urb->pipe, !is_input);
 
 	/*
 	 * buffer gets wrapped in one or more qtds;
@@ -770,9 +765,11 @@ qh_make (
 	gfp_t			flags
 ) {
 	struct ehci_qh		*qh = ehci_qh_alloc (ehci, flags);
+	struct usb_host_endpoint *ep;
 	u32			info1 = 0, info2 = 0;
 	int			is_input, type;
 	int			maxp = 0;
+	int			mult;
 	struct usb_tt		*tt = urb->dev->tt;
 	struct ehci_qh_hw	*hw;
 
@@ -787,13 +784,15 @@ qh_make (
 
 	is_input = usb_pipein (urb->pipe);
 	type = usb_pipetype (urb->pipe);
-	maxp = usb_maxpacket (urb->dev, urb->pipe, !is_input);
+	ep = usb_pipe_endpoint (urb->dev, urb->pipe);
+	maxp = usb_endpoint_maxp (&ep->desc);
+	mult = usb_endpoint_maxp_mult (&ep->desc);
 
 	/* 1024 byte maxpacket is a hardware ceiling.  High bandwidth
 	 * acts like up to 3KB, but is built from smaller packets.
 	 */
-	if (max_packet(maxp) > 1024) {
-		ehci_dbg(ehci, "bogus qh maxpacket %d\n", max_packet(maxp));
+	if (maxp > 1024) {
+		ehci_dbg(ehci, "bogus qh maxpacket %d\n", maxp);
 		goto done;
 	}
 
@@ -809,8 +808,7 @@ qh_make (
 		unsigned	tmp;
 
 		qh->ps.usecs = NS_TO_US(usb_calc_bus_time(USB_SPEED_HIGH,
-				is_input, 0,
-				hb_mult(maxp) * max_packet(maxp)));
+				is_input, 0, mult * maxp));
 		qh->ps.phase = NO_FRAME;
 
 		if (urb->dev->speed == USB_SPEED_HIGH) {
@@ -854,7 +852,7 @@ qh_make (
 			think_time = tt ? tt->think_time : 0;
 			qh->ps.tt_usecs = NS_TO_US(think_time +
 					usb_calc_bus_time (urb->dev->speed,
-					is_input, 0, max_packet (maxp)));
+					is_input, 0, maxp));
 			if (urb->interval > ehci->periodic_size)
 				urb->interval = ehci->periodic_size;
 			qh->ps.period = urb->interval;
@@ -925,11 +923,11 @@ qh_make (
 			 * to help them do so.  So now people expect to use
 			 * such nonconformant devices with Linux too; sigh.
 			 */
-			info1 |= max_packet(maxp) << 16;
+			info1 |= maxp << 16;
 			info2 |= (EHCI_TUNE_MULT_HS << 30);
 		} else {		/* PIPE_INTERRUPT */
-			info1 |= max_packet (maxp) << 16;
-			info2 |= hb_mult (maxp) << 30;
+			info1 |= maxp << 16;
+			info2 |= mult << 30;
 		}
 		break;
 	default:
@@ -1221,7 +1219,7 @@ static int submit_single_step_set_feature(
 
 	token |= (1 /* "in" */ << 8);  /*This is IN stage*/
 
-	maxpacket = max_packet(usb_maxpacket(urb->dev, urb->pipe, 0));
+	maxpacket = usb_maxpacket(urb->dev, urb->pipe, 0);
 
 	qtd_fill(ehci, qtd, buf, len, token, maxpacket);
 
diff --git a/drivers/usb/host/ehci-sched.c b/drivers/usb/host/ehci-sched.c
index 1dfe54f14737..980a6b3b2da2 100644
--- a/drivers/usb/host/ehci-sched.c
+++ b/drivers/usb/host/ehci-sched.c
@@ -1064,11 +1064,10 @@ iso_stream_init(
 
 	/* knows about ITD vs SITD */
 	if (dev->speed == USB_SPEED_HIGH) {
-		unsigned multi = hb_mult(maxp);
+		unsigned multi = usb_endpoint_maxp_mult(&urb->ep->desc);
 
 		stream->highspeed = 1;
 
-		maxp = max_packet(maxp);
 		buf1 |= maxp;
 		maxp *= multi;
 
diff --git a/drivers/usb/host/ehci-w90x900.c b/drivers/usb/host/ehci-w90x900.c
index e42a29e8e229..63b9d0c67963 100644
--- a/drivers/usb/host/ehci-w90x900.c
+++ b/drivers/usb/host/ehci-w90x900.c
@@ -33,8 +33,7 @@ static const char hcd_name[] = "ehci-w90x900 ";
 
 static struct hc_driver __read_mostly ehci_w90x900_hc_driver;
 
-static int usb_w90x900_probe(const struct hc_driver *driver,
-		      struct platform_device *pdev)
+static int ehci_w90x900_probe(struct platform_device *pdev)
 {
 	struct usb_hcd *hcd;
 	struct ehci_hcd *ehci;
@@ -42,7 +41,8 @@ static int usb_w90x900_probe(const struct hc_driver *driver,
 	int retval = 0, irq;
 	unsigned long val;
 
-	hcd = usb_create_hcd(driver, &pdev->dev, "w90x900 EHCI");
+	hcd = usb_create_hcd(&ehci_w90x900_hc_driver,
+			&pdev->dev, "w90x900 EHCI");
 	if (!hcd) {
 		retval = -ENOMEM;
 		goto err1;
@@ -63,9 +63,9 @@ static int usb_w90x900_probe(const struct hc_driver *driver,
 		HC_LENGTH(ehci, ehci_readl(ehci, &ehci->caps->hc_capbase));
 
 	/* enable PHY 0,1,the regs only apply to w90p910
-	*  0xA4,0xA8 were offsets of PHY0 and PHY1 controller of
-	*  w90p910 IC relative to ehci->regs.
-	*/
+	 *  0xA4,0xA8 were offsets of PHY0 and PHY1 controller of
+	 *  w90p910 IC relative to ehci->regs.
+	 */
 	val = __raw_readl(ehci->regs+PHY0_CTR);
 	val |= ENPHY;
 	__raw_writel(val, ehci->regs+PHY0_CTR);
@@ -92,26 +92,12 @@ err1:
 	return retval;
 }
 
-static void usb_w90x900_remove(struct usb_hcd *hcd,
-			struct platform_device *pdev)
-{
-	usb_remove_hcd(hcd);
-	usb_put_hcd(hcd);
-}
-
-static int ehci_w90x900_probe(struct platform_device *pdev)
-{
-	if (usb_disabled())
-		return -ENODEV;
-
-	return usb_w90x900_probe(&ehci_w90x900_hc_driver, pdev);
-}
-
 static int ehci_w90x900_remove(struct platform_device *pdev)
 {
 	struct usb_hcd *hcd = platform_get_drvdata(pdev);
 
-	usb_w90x900_remove(hcd, pdev);
+	usb_remove_hcd(hcd);
+	usb_put_hcd(hcd);
 
 	return 0;
 }
diff --git a/drivers/usb/host/ehci.h b/drivers/usb/host/ehci.h
index 3f3b74aeca97..a8e36170d8b8 100644
--- a/drivers/usb/host/ehci.h
+++ b/drivers/usb/host/ehci.h
@@ -219,6 +219,7 @@ struct ehci_hcd {			/* one per controller */
 	unsigned		no_selective_suspend:1;
 	unsigned		has_fsl_port_bug:1; /* FreeScale */
 	unsigned		has_fsl_hs_errata:1;	/* Freescale HS quirk */
+	unsigned		has_fsl_susp_errata:1;	/* NXP SUSP quirk */
 	unsigned		big_endian_mmio:1;
 	unsigned		big_endian_desc:1;
 	unsigned		big_endian_capbase:1;
@@ -710,6 +711,13 @@ ehci_port_speed(struct ehci_hcd *ehci, unsigned int portsc)
 #endif
 
 /*
+ * Some Freescale/NXP processors have an erratum (USB A-005697)
+ * in which we need to wait for 10ms for bus to enter suspend mode
+ * after setting SUSP bit.
+ */
+#define ehci_has_fsl_susp_errata(e)	((e)->has_fsl_susp_errata)
+
+/*
  * While most USB host controllers implement their registers in
  * little-endian format, a minority (celleb companion chip) implement
  * them in big endian format.
diff --git a/drivers/usb/host/fsl-mph-dr-of.c b/drivers/usb/host/fsl-mph-dr-of.c
index f07ccb25bc24..e90ddb530765 100644
--- a/drivers/usb/host/fsl-mph-dr-of.c
+++ b/drivers/usb/host/fsl-mph-dr-of.c
@@ -226,6 +226,8 @@ static int fsl_usb2_mph_dr_of_probe(struct platform_device *ofdev)
 		of_property_read_bool(np, "fsl,usb-erratum-a007792");
 	pdata->has_fsl_erratum_a005275 =
 		of_property_read_bool(np, "fsl,usb-erratum-a005275");
+	pdata->has_fsl_erratum_a005697 =
+		of_property_read_bool(np, "fsl,usb_erratum-a005697");
 
 	/*
 	 * Determine whether phy_clk_valid needs to be checked
diff --git a/drivers/usb/host/isp1362-hcd.c b/drivers/usb/host/isp1362-hcd.c
index 6cf82ee460a6..0f2b4b358e1a 100644
--- a/drivers/usb/host/isp1362-hcd.c
+++ b/drivers/usb/host/isp1362-hcd.c
@@ -147,7 +147,7 @@ static inline struct isp1362_ep_queue *get_ptd_queue(struct isp1362_hcd *isp1362
 	if (epq)
 		DBG(1, "%s: PTD $%04x is on %s queue\n", __func__, offset, epq->name);
 	else
-		pr_warning("%s: invalid PTD $%04x\n", __func__, offset);
+		pr_warn("%s: invalid PTD $%04x\n", __func__, offset);
 
 	return epq;
 }
@@ -157,8 +157,9 @@ static inline int get_ptd_offset(struct isp1362_ep_queue *epq, u8 index)
 	int offset;
 
 	if (index * epq->blk_size > epq->buf_size) {
-		pr_warning("%s: Bad %s index %d(%d)\n", __func__, epq->name, index,
-		     epq->buf_size / epq->blk_size);
+		pr_warn("%s: Bad %s index %d(%d)\n",
+			__func__, epq->name, index,
+			epq->buf_size / epq->blk_size);
 		return -EINVAL;
 	}
 	offset = epq->buf_start + index * epq->blk_size;
@@ -902,8 +903,8 @@ static void start_iso_transfers(struct isp1362_hcd *isp1362_hcd)
 
 			ptd_offset = next_ptd(epq, ep);
 			if (ptd_offset < 0) {
-				pr_warning("%s: req %d No more %s PTD buffers available\n", __func__,
-				     ep->num_req, epq->name);
+				pr_warn("%s: req %d No more %s PTD buffers available\n",
+					__func__, ep->num_req, epq->name);
 				break;
 			}
 		}
@@ -973,8 +974,8 @@ static void finish_transfers(struct isp1362_hcd *isp1362_hcd, unsigned long done
 			break;
 	}
 	if (done_map)
-		pr_warning("%s: done_map not clear: %08lx:%08lx\n", __func__, done_map,
-		     epq->skip_map);
+		pr_warn("%s: done_map not clear: %08lx:%08lx\n",
+			__func__, done_map, epq->skip_map);
 	atomic_dec(&epq->finishing);
 }
 
@@ -1433,7 +1434,7 @@ static int isp1362_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
 		} else
 			DBG(1, "%s: urb %p active; wait4irq\n", __func__, urb);
 	} else {
-		pr_warning("%s: No EP in URB %p\n", __func__, urb);
+		pr_warn("%s: No EP in URB %p\n", __func__, urb);
 		retval = -EINVAL;
 	}
 done:
@@ -1748,10 +1749,10 @@ static int isp1362_bus_suspend(struct usb_hcd *hcd)
 		/* FALL THROUGH */
 	case OHCI_USB_RESET:
 		status = -EBUSY;
-		pr_warning("%s: needs reinit!\n", __func__);
+		pr_warn("%s: needs reinit!\n", __func__);
 		goto done;
 	case OHCI_USB_SUSPEND:
-		pr_warning("%s: already suspended?\n", __func__);
+		pr_warn("%s: already suspended?\n", __func__);
 		goto done;
 	}
 	DBG(0, "%s: suspend root hub\n", __func__);
@@ -1839,7 +1840,7 @@ static int isp1362_bus_resume(struct usb_hcd *hcd)
 	isp1362_hcd->hc_control = isp1362_read_reg32(isp1362_hcd, HCCONTROL);
 	pr_info("%s: HCCONTROL: %08x\n", __func__, isp1362_hcd->hc_control);
 	if (hcd->state == HC_STATE_RESUMING) {
-		pr_warning("%s: duplicate resume\n", __func__);
+		pr_warn("%s: duplicate resume\n", __func__);
 		status = 0;
 	} else
 		switch (isp1362_hcd->hc_control & OHCI_CTRL_HCFS) {
@@ -2474,8 +2475,8 @@ static int isp1362_chip_test(struct isp1362_hcd *isp1362_hcd)
 					    __func__, offset);
 					break;
 				}
-				pr_warning("%s: memory check with offset %02x ok after second read\n",
-				     __func__, offset);
+				pr_warn("%s: memory check with offset %02x ok after second read\n",
+					__func__, offset);
 			}
 		}
 		kfree(ref);
diff --git a/drivers/usb/host/ohci-at91.c b/drivers/usb/host/ohci-at91.c
index b38a228134df..be9e63836881 100644
--- a/drivers/usb/host/ohci-at91.c
+++ b/drivers/usb/host/ohci-at91.c
@@ -14,8 +14,8 @@
 
 #include <linux/clk.h>
 #include <linux/dma-mapping.h>
+#include <linux/gpio/consumer.h>
 #include <linux/of_platform.h>
-#include <linux/of_gpio.h>
 #include <linux/platform_device.h>
 #include <linux/platform_data/atmel.h>
 #include <linux/io.h>
@@ -39,8 +39,8 @@
 
 #define AT91_MAX_USBH_PORTS	3
 struct at91_usbh_data {
-	int vbus_pin[AT91_MAX_USBH_PORTS];	/* port power-control pin */
-	int overcurrent_pin[AT91_MAX_USBH_PORTS];
+	struct gpio_desc *vbus_pin[AT91_MAX_USBH_PORTS];
+	struct gpio_desc *overcurrent_pin[AT91_MAX_USBH_PORTS];
 	u8 ports;				/* number of ports on root hub */
 	u8 overcurrent_supported;
 	u8 vbus_pin_active_low[AT91_MAX_USBH_PORTS];
@@ -68,8 +68,6 @@ static const struct ohci_driver_overrides ohci_at91_drv_overrides __initconst =
 	.extra_priv_size = sizeof(struct ohci_at91_priv),
 };
 
-extern int usb_disabled(void);
-
 /*-------------------------------------------------------------------------*/
 
 static void at91_start_clock(struct ohci_at91_priv *ohci_at91)
@@ -268,11 +266,8 @@ static void ohci_at91_usb_set_power(struct at91_usbh_data *pdata, int port, int
 	if (!valid_port(port))
 		return;
 
-	if (!gpio_is_valid(pdata->vbus_pin[port]))
-		return;
-
-	gpio_set_value(pdata->vbus_pin[port],
-		       pdata->vbus_pin_active_low[port] ^ enable);
+	gpiod_set_value(pdata->vbus_pin[port],
+			pdata->vbus_pin_active_low[port] ^ enable);
 }
 
 static int ohci_at91_usb_get_power(struct at91_usbh_data *pdata, int port)
@@ -280,11 +275,8 @@ static int ohci_at91_usb_get_power(struct at91_usbh_data *pdata, int port)
 	if (!valid_port(port))
 		return -EINVAL;
 
-	if (!gpio_is_valid(pdata->vbus_pin[port]))
-		return -EINVAL;
-
-	return gpio_get_value(pdata->vbus_pin[port]) ^
-		pdata->vbus_pin_active_low[port];
+	return gpiod_get_value(pdata->vbus_pin[port]) ^
+	       pdata->vbus_pin_active_low[port];
 }
 
 /*
@@ -474,16 +466,13 @@ static irqreturn_t ohci_hcd_at91_overcurrent_irq(int irq, void *data)
 {
 	struct platform_device *pdev = data;
 	struct at91_usbh_data *pdata = dev_get_platdata(&pdev->dev);
-	int val, gpio, port;
+	int val, port;
 
 	/* From the GPIO notifying the over-current situation, find
 	 * out the corresponding port */
 	at91_for_each_port(port) {
-		if (gpio_is_valid(pdata->overcurrent_pin[port]) &&
-				gpio_to_irq(pdata->overcurrent_pin[port]) == irq) {
-			gpio = pdata->overcurrent_pin[port];
+		if (gpiod_to_irq(pdata->overcurrent_pin[port]) == irq)
 			break;
-		}
 	}
 
 	if (port == AT91_MAX_USBH_PORTS) {
@@ -491,7 +480,7 @@ static irqreturn_t ohci_hcd_at91_overcurrent_irq(int irq, void *data)
 		return IRQ_HANDLED;
 	}
 
-	val = gpio_get_value(gpio);
+	val = gpiod_get_value(pdata->overcurrent_pin[port]);
 
 	/* When notified of an over-current situation, disable power
 	   on the corresponding port, and mark this port in
@@ -522,9 +511,8 @@ static int ohci_hcd_at91_drv_probe(struct platform_device *pdev)
 	struct device_node *np = pdev->dev.of_node;
 	struct at91_usbh_data	*pdata;
 	int			i;
-	int			gpio;
 	int			ret;
-	enum of_gpio_flags	flags;
+	int			err;
 	u32			ports;
 
 	/* Right now device-tree probed devices don't get dma_mask set.
@@ -545,38 +533,16 @@ static int ohci_hcd_at91_drv_probe(struct platform_device *pdev)
 		pdata->ports = ports;
 
 	at91_for_each_port(i) {
-		/*
-		 * do not configure PIO if not in relation with
-		 * real USB port on board
-		 */
-		if (i >= pdata->ports) {
-			pdata->vbus_pin[i] = -EINVAL;
-			pdata->overcurrent_pin[i] = -EINVAL;
+		pdata->vbus_pin[i] = devm_gpiod_get_optional(&pdev->dev,
+							     "atmel,vbus-gpio",
+							     GPIOD_IN);
+		if (IS_ERR(pdata->vbus_pin[i])) {
+			err = PTR_ERR(pdata->vbus_pin[i]);
+			dev_err(&pdev->dev, "unable to claim gpio \"vbus\": %d\n", err);
 			continue;
 		}
 
-		gpio = of_get_named_gpio_flags(np, "atmel,vbus-gpio", i,
-					       &flags);
-		pdata->vbus_pin[i] = gpio;
-		if (!gpio_is_valid(gpio))
-			continue;
-		pdata->vbus_pin_active_low[i] = flags & OF_GPIO_ACTIVE_LOW;
-
-		ret = gpio_request(gpio, "ohci_vbus");
-		if (ret) {
-			dev_err(&pdev->dev,
-				"can't request vbus gpio %d\n", gpio);
-			continue;
-		}
-		ret = gpio_direction_output(gpio,
-					!pdata->vbus_pin_active_low[i]);
-		if (ret) {
-			dev_err(&pdev->dev,
-				"can't put vbus gpio %d as output %d\n",
-				gpio, !pdata->vbus_pin_active_low[i]);
-			gpio_free(gpio);
-			continue;
-		}
+		pdata->vbus_pin_active_low[i] = gpiod_get_value(pdata->vbus_pin[i]);
 
 		ohci_at91_usb_set_power(pdata, i, 1);
 	}
@@ -586,37 +552,21 @@ static int ohci_hcd_at91_drv_probe(struct platform_device *pdev)
 			break;
 
 		pdata->overcurrent_pin[i] =
-			of_get_named_gpio_flags(np, "atmel,oc-gpio", i, &flags);
-
-		if (!gpio_is_valid(pdata->overcurrent_pin[i]))
-			continue;
-		gpio = pdata->overcurrent_pin[i];
-
-		ret = gpio_request(gpio, "ohci_overcurrent");
-		if (ret) {
-			dev_err(&pdev->dev,
-				"can't request overcurrent gpio %d\n",
-				gpio);
+			devm_gpiod_get_optional(&pdev->dev,
+						"atmel,oc-gpio", GPIOD_IN);
+		if (IS_ERR(pdata->overcurrent_pin[i])) {
+			err = PTR_ERR(pdata->overcurrent_pin[i]);
+			dev_err(&pdev->dev, "unable to claim gpio \"overcurrent\": %d\n", err);
 			continue;
 		}
 
-		ret = gpio_direction_input(gpio);
-		if (ret) {
-			dev_err(&pdev->dev,
-				"can't configure overcurrent gpio %d as input\n",
-				gpio);
-			gpio_free(gpio);
-			continue;
-		}
-
-		ret = request_irq(gpio_to_irq(gpio),
-				  ohci_hcd_at91_overcurrent_irq,
-				  IRQF_SHARED, "ohci_overcurrent", pdev);
-		if (ret) {
-			gpio_free(gpio);
-			dev_err(&pdev->dev,
-				"can't get gpio IRQ for overcurrent\n");
-		}
+		ret = devm_request_irq(&pdev->dev,
+				       gpiod_to_irq(pdata->overcurrent_pin[i]),
+				       ohci_hcd_at91_overcurrent_irq,
+				       IRQF_SHARED,
+				       "ohci_overcurrent", pdev);
+		if (ret)
+			dev_info(&pdev->dev, "failed to request gpio \"overcurrent\" IRQ\n");
 	}
 
 	device_init_wakeup(&pdev->dev, 1);
@@ -629,19 +579,8 @@ static int ohci_hcd_at91_drv_remove(struct platform_device *pdev)
 	int			i;
 
 	if (pdata) {
-		at91_for_each_port(i) {
-			if (!gpio_is_valid(pdata->vbus_pin[i]))
-				continue;
+		at91_for_each_port(i)
 			ohci_at91_usb_set_power(pdata, i, 0);
-			gpio_free(pdata->vbus_pin[i]);
-		}
-
-		at91_for_each_port(i) {
-			if (!gpio_is_valid(pdata->overcurrent_pin[i]))
-				continue;
-			free_irq(gpio_to_irq(pdata->overcurrent_pin[i]), pdev);
-			gpio_free(pdata->overcurrent_pin[i]);
-		}
 	}
 
 	device_init_wakeup(&pdev->dev, 0);
diff --git a/drivers/usb/host/ohci-da8xx.c b/drivers/usb/host/ohci-da8xx.c
index e5c33bc98ea4..05da2cb59612 100644
--- a/drivers/usb/host/ohci-da8xx.c
+++ b/drivers/usb/host/ohci-da8xx.c
@@ -11,62 +11,192 @@
  * kind, whether express or implied.
  */
 
+#include <linux/clk.h>
+#include <linux/io.h>
 #include <linux/interrupt.h>
 #include <linux/jiffies.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/clk.h>
-
-#include <mach/da8xx.h>
+#include <linux/phy/phy.h>
 #include <linux/platform_data/usb-davinci.h>
+#include <linux/regulator/consumer.h>
+#include <linux/usb.h>
+#include <linux/usb/hcd.h>
+#include <asm/unaligned.h>
 
-#ifndef CONFIG_ARCH_DAVINCI_DA8XX
-#error "This file is DA8xx bus glue.  Define CONFIG_ARCH_DAVINCI_DA8XX."
-#endif
+#include "ohci.h"
 
-#define CFGCHIP2	DA8XX_SYSCFG0_VIRT(DA8XX_CFGCHIP2_REG)
+#define DRIVER_DESC "DA8XX"
+#define DRV_NAME "ohci-da8xx"
 
-static struct clk *usb11_clk;
-static struct clk *usb20_clk;
+static struct hc_driver __read_mostly ohci_da8xx_hc_driver;
+
+static int (*orig_ohci_hub_control)(struct usb_hcd  *hcd, u16 typeReq,
+			u16 wValue, u16 wIndex, char *buf, u16 wLength);
+static int (*orig_ohci_hub_status_data)(struct usb_hcd *hcd, char *buf);
+
+struct da8xx_ohci_hcd {
+	struct usb_hcd *hcd;
+	struct clk *usb11_clk;
+	struct phy *usb11_phy;
+	struct regulator *vbus_reg;
+	struct notifier_block nb;
+	unsigned int reg_enabled;
+};
+
+#define to_da8xx_ohci(hcd) (struct da8xx_ohci_hcd *)(hcd_to_ohci(hcd)->priv)
 
 /* Over-current indicator change bitmask */
 static volatile u16 ocic_mask;
 
-static void ohci_da8xx_clock(int on)
+static int ohci_da8xx_enable(struct usb_hcd *hcd)
 {
-	u32 cfgchip2;
-
-	cfgchip2 = __raw_readl(CFGCHIP2);
-	if (on) {
-		clk_enable(usb11_clk);
-
-		/*
-		 * If USB 1.1 reference clock is sourced from USB 2.0 PHY, we
-		 * need to enable the USB 2.0 module clocking, start its PHY,
-		 * and not allow it to stop the clock during USB 2.0 suspend.
-		 */
-		if (!(cfgchip2 & CFGCHIP2_USB1PHYCLKMUX)) {
-			clk_enable(usb20_clk);
-
-			cfgchip2 &= ~(CFGCHIP2_RESET | CFGCHIP2_PHYPWRDN);
-			cfgchip2 |= CFGCHIP2_PHY_PLLON;
-			__raw_writel(cfgchip2, CFGCHIP2);
-
-			pr_info("Waiting for USB PHY clock good...\n");
-			while (!(__raw_readl(CFGCHIP2) & CFGCHIP2_PHYCLKGD))
-				cpu_relax();
-		}
+	struct da8xx_ohci_hcd *da8xx_ohci = to_da8xx_ohci(hcd);
+	int ret;
+
+	ret = clk_prepare_enable(da8xx_ohci->usb11_clk);
+	if (ret)
+		return ret;
+
+	ret = phy_init(da8xx_ohci->usb11_phy);
+	if (ret)
+		goto err_phy_init;
+
+	ret = phy_power_on(da8xx_ohci->usb11_phy);
+	if (ret)
+		goto err_phy_power_on;
+
+	return 0;
+
+err_phy_power_on:
+	phy_exit(da8xx_ohci->usb11_phy);
+err_phy_init:
+	clk_disable_unprepare(da8xx_ohci->usb11_clk);
+
+	return ret;
+}
+
+static void ohci_da8xx_disable(struct usb_hcd *hcd)
+{
+	struct da8xx_ohci_hcd *da8xx_ohci = to_da8xx_ohci(hcd);
+
+	phy_power_off(da8xx_ohci->usb11_phy);
+	phy_exit(da8xx_ohci->usb11_phy);
+	clk_disable_unprepare(da8xx_ohci->usb11_clk);
+}
 
-		/* Enable USB 1.1 PHY */
-		cfgchip2 |= CFGCHIP2_USB1SUSPENDM;
-	} else {
-		clk_disable(usb11_clk);
-		if (!(cfgchip2 & CFGCHIP2_USB1PHYCLKMUX))
-			clk_disable(usb20_clk);
+static int ohci_da8xx_set_power(struct usb_hcd *hcd, int on)
+{
+	struct da8xx_ohci_hcd *da8xx_ohci = to_da8xx_ohci(hcd);
+	struct device *dev		= hcd->self.controller;
+	struct da8xx_ohci_root_hub *hub	= dev_get_platdata(dev);
+	int ret;
+
+	if (hub && hub->set_power)
+		return hub->set_power(1, on);
+
+	if (!da8xx_ohci->vbus_reg)
+		return 0;
 
-		/* Disable USB 1.1 PHY */
-		cfgchip2 &= ~CFGCHIP2_USB1SUSPENDM;
+	if (on && !da8xx_ohci->reg_enabled) {
+		ret = regulator_enable(da8xx_ohci->vbus_reg);
+		if (ret) {
+			dev_err(dev, "Failed to enable regulator: %d\n", ret);
+			return ret;
+		}
+		da8xx_ohci->reg_enabled = 1;
+
+	} else if (!on && da8xx_ohci->reg_enabled) {
+		ret = regulator_disable(da8xx_ohci->vbus_reg);
+		if (ret) {
+			dev_err(dev, "Failed  to disable regulator: %d\n", ret);
+			return ret;
+		}
+		da8xx_ohci->reg_enabled = 0;
 	}
-	__raw_writel(cfgchip2, CFGCHIP2);
+
+	return 0;
+}
+
+static int ohci_da8xx_get_power(struct usb_hcd *hcd)
+{
+	struct da8xx_ohci_hcd *da8xx_ohci = to_da8xx_ohci(hcd);
+	struct device *dev		= hcd->self.controller;
+	struct da8xx_ohci_root_hub *hub	= dev_get_platdata(dev);
+
+	if (hub && hub->get_power)
+		return hub->get_power(1);
+
+	if (da8xx_ohci->vbus_reg)
+		return regulator_is_enabled(da8xx_ohci->vbus_reg);
+
+	return 1;
+}
+
+static int ohci_da8xx_get_oci(struct usb_hcd *hcd)
+{
+	struct da8xx_ohci_hcd *da8xx_ohci = to_da8xx_ohci(hcd);
+	struct device *dev		= hcd->self.controller;
+	struct da8xx_ohci_root_hub *hub	= dev_get_platdata(dev);
+	unsigned int flags;
+	int ret;
+
+	if (hub && hub->get_oci)
+		return hub->get_oci(1);
+
+	if (!da8xx_ohci->vbus_reg)
+		return 0;
+
+	ret = regulator_get_error_flags(da8xx_ohci->vbus_reg, &flags);
+	if (ret)
+		return ret;
+
+	if (flags & REGULATOR_ERROR_OVER_CURRENT)
+		return 1;
+
+	return 0;
+}
+
+static int ohci_da8xx_has_set_power(struct usb_hcd *hcd)
+{
+	struct da8xx_ohci_hcd *da8xx_ohci = to_da8xx_ohci(hcd);
+	struct device *dev		= hcd->self.controller;
+	struct da8xx_ohci_root_hub *hub	= dev_get_platdata(dev);
+
+	if (hub && hub->set_power)
+		return 1;
+
+	if (da8xx_ohci->vbus_reg)
+		return 1;
+
+	return 0;
+}
+
+static int ohci_da8xx_has_oci(struct usb_hcd *hcd)
+{
+	struct da8xx_ohci_hcd *da8xx_ohci = to_da8xx_ohci(hcd);
+	struct device *dev		= hcd->self.controller;
+	struct da8xx_ohci_root_hub *hub	= dev_get_platdata(dev);
+
+	if (hub && hub->get_oci)
+		return 1;
+
+	if (da8xx_ohci->vbus_reg)
+		return 1;
+
+	return 0;
+}
+
+static int ohci_da8xx_has_potpgt(struct usb_hcd *hcd)
+{
+	struct device *dev		= hcd->self.controller;
+	struct da8xx_ohci_root_hub *hub	= dev_get_platdata(dev);
+
+	if (hub && hub->potpgt)
+		return 1;
+
+	return 0;
 }
 
 /*
@@ -82,7 +212,51 @@ static void ohci_da8xx_ocic_handler(struct da8xx_ohci_root_hub *hub,
 		hub->set_power(port, 0);
 }
 
-static int ohci_da8xx_init(struct usb_hcd *hcd)
+static int ohci_da8xx_regulator_event(struct notifier_block *nb,
+				unsigned long event, void *data)
+{
+	struct da8xx_ohci_hcd *da8xx_ohci =
+				container_of(nb, struct da8xx_ohci_hcd, nb);
+
+	if (event & REGULATOR_EVENT_OVER_CURRENT) {
+		ocic_mask |= 1 << 1;
+		ohci_da8xx_set_power(da8xx_ohci->hcd, 0);
+	}
+
+	return 0;
+}
+
+static int ohci_da8xx_register_notify(struct usb_hcd *hcd)
+{
+	struct da8xx_ohci_hcd *da8xx_ohci = to_da8xx_ohci(hcd);
+	struct device *dev		= hcd->self.controller;
+	struct da8xx_ohci_root_hub *hub	= dev_get_platdata(dev);
+	int ret = 0;
+
+	if (hub && hub->ocic_notify) {
+		ret = hub->ocic_notify(ohci_da8xx_ocic_handler);
+	} else if (da8xx_ohci->vbus_reg) {
+		da8xx_ohci->nb.notifier_call = ohci_da8xx_regulator_event;
+		ret = devm_regulator_register_notifier(da8xx_ohci->vbus_reg,
+						&da8xx_ohci->nb);
+	}
+
+	if (ret)
+		dev_err(dev, "Failed to register notifier: %d\n", ret);
+
+	return ret;
+}
+
+static void ohci_da8xx_unregister_notify(struct usb_hcd *hcd)
+{
+	struct device *dev		= hcd->self.controller;
+	struct da8xx_ohci_root_hub *hub	= dev_get_platdata(dev);
+
+	if (hub && hub->ocic_notify)
+		hub->ocic_notify(NULL);
+}
+
+static int ohci_da8xx_reset(struct usb_hcd *hcd)
 {
 	struct device *dev		= hcd->self.controller;
 	struct da8xx_ohci_root_hub *hub	= dev_get_platdata(dev);
@@ -92,7 +266,9 @@ static int ohci_da8xx_init(struct usb_hcd *hcd)
 
 	dev_dbg(dev, "starting USB controller\n");
 
-	ohci_da8xx_clock(1);
+	result = ohci_da8xx_enable(hcd);
+	if (result < 0)
+		return result;
 
 	/*
 	 * DA8xx only have 1 port connected to the pins but the HC root hub
@@ -100,9 +276,11 @@ static int ohci_da8xx_init(struct usb_hcd *hcd)
 	 */
 	ohci->num_ports = 1;
 
-	result = ohci_init(ohci);
-	if (result < 0)
+	result = ohci_setup(hcd);
+	if (result < 0) {
+		ohci_da8xx_disable(hcd);
 		return result;
+	}
 
 	/*
 	 * Since we're providing a board-specific root hub port power control
@@ -111,45 +289,29 @@ static int ohci_da8xx_init(struct usb_hcd *hcd)
 	 * the correct hub descriptor...
 	 */
 	rh_a = ohci_readl(ohci, &ohci->regs->roothub.a);
-	if (hub->set_power) {
+	if (ohci_da8xx_has_set_power(hcd)) {
 		rh_a &= ~RH_A_NPS;
 		rh_a |=  RH_A_PSM;
 	}
-	if (hub->get_oci) {
+	if (ohci_da8xx_has_oci(hcd)) {
 		rh_a &= ~RH_A_NOCP;
 		rh_a |=  RH_A_OCPM;
 	}
-	rh_a &= ~RH_A_POTPGT;
-	rh_a |= hub->potpgt << 24;
+	if (ohci_da8xx_has_potpgt(hcd)) {
+		rh_a &= ~RH_A_POTPGT;
+		rh_a |= hub->potpgt << 24;
+	}
 	ohci_writel(ohci, rh_a, &ohci->regs->roothub.a);
 
 	return result;
 }
 
-static void ohci_da8xx_stop(struct usb_hcd *hcd)
-{
-	ohci_stop(hcd);
-	ohci_da8xx_clock(0);
-}
-
-static int ohci_da8xx_start(struct usb_hcd *hcd)
-{
-	struct ohci_hcd	*ohci		= hcd_to_ohci(hcd);
-	int result;
-
-	result = ohci_run(ohci);
-	if (result < 0)
-		ohci_da8xx_stop(hcd);
-
-	return result;
-}
-
 /*
  * Update the status data from the hub with the over-current indicator change.
  */
 static int ohci_da8xx_hub_status_data(struct usb_hcd *hcd, char *buf)
 {
-	int length		= ohci_hub_status_data(hcd, buf);
+	int length		= orig_ohci_hub_status_data(hcd, buf);
 
 	/* See if we have OCIC bit set on port 1 */
 	if (ocic_mask & (1 << 1)) {
@@ -171,7 +333,6 @@ static int ohci_da8xx_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 				  u16 wIndex, char *buf, u16 wLength)
 {
 	struct device *dev		= hcd->self.controller;
-	struct da8xx_ohci_root_hub *hub	= dev_get_platdata(dev);
 	int temp;
 
 	switch (typeReq) {
@@ -185,11 +346,11 @@ static int ohci_da8xx_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 		temp = roothub_portstatus(hcd_to_ohci(hcd), wIndex - 1);
 
 		/* The port power status (PPS) bit defaults to 1 */
-		if (hub->get_power && hub->get_power(wIndex) == 0)
+		if (!ohci_da8xx_get_power(hcd))
 			temp &= ~RH_PS_PPS;
 
 		/* The port over-current indicator (POCI) bit is always 0 */
-		if (hub->get_oci && hub->get_oci(wIndex) > 0)
+		if (ohci_da8xx_get_oci(hcd) > 0)
 			temp |=  RH_PS_POCI;
 
 		/* The over-current indicator change (OCIC) bit is 0 too */
@@ -214,10 +375,7 @@ check_port:
 			dev_dbg(dev, "%sPortFeature(%u): %s\n",
 				temp ? "Set" : "Clear", wIndex, "POWER");
 
-			if (!hub->set_power)
-				return -EPIPE;
-
-			return hub->set_power(wIndex, temp) ? -EPIPE : 0;
+			return ohci_da8xx_set_power(hcd, temp) ? -EPIPE : 0;
 		case USB_PORT_FEAT_C_OVER_CURRENT:
 			dev_dbg(dev, "%sPortFeature(%u): %s\n",
 				temp ? "Set" : "Clear", wIndex,
@@ -231,86 +389,61 @@ check_port:
 		}
 	}
 
-	return ohci_hub_control(hcd, typeReq, wValue, wIndex, buf, wLength);
+	return orig_ohci_hub_control(hcd, typeReq, wValue,
+			wIndex, buf, wLength);
 }
 
-static const struct hc_driver ohci_da8xx_hc_driver = {
-	.description		= hcd_name,
-	.product_desc		= "DA8xx OHCI",
-	.hcd_priv_size		= sizeof(struct ohci_hcd),
-
-	/*
-	 * generic hardware linkage
-	 */
-	.irq			= ohci_irq,
-	.flags			= HCD_USB11 | HCD_MEMORY,
-
-	/*
-	 * basic lifecycle operations
-	 */
-	.reset			= ohci_da8xx_init,
-	.start			= ohci_da8xx_start,
-	.stop			= ohci_da8xx_stop,
-	.shutdown		= ohci_shutdown,
-
-	/*
-	 * managing i/o requests and associated device resources
-	 */
-	.urb_enqueue		= ohci_urb_enqueue,
-	.urb_dequeue		= ohci_urb_dequeue,
-	.endpoint_disable	= ohci_endpoint_disable,
-
-	/*
-	 * scheduling support
-	 */
-	.get_frame_number	= ohci_get_frame,
-
-	/*
-	 * root hub support
-	 */
-	.hub_status_data	= ohci_da8xx_hub_status_data,
-	.hub_control		= ohci_da8xx_hub_control,
-
-#ifdef	CONFIG_PM
-	.bus_suspend		= ohci_bus_suspend,
-	.bus_resume		= ohci_bus_resume,
-#endif
-	.start_port_reset	= ohci_start_port_reset,
-};
-
 /*-------------------------------------------------------------------------*/
+#ifdef CONFIG_OF
+static const struct of_device_id da8xx_ohci_ids[] = {
+	{ .compatible = "ti,da830-ohci" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, da8xx_ohci_ids);
+#endif
 
-
-/**
- * usb_hcd_da8xx_probe - initialize DA8xx-based HCDs
- * Context: !in_interrupt()
- *
- * Allocates basic resources for this USB host controller, and
- * then invokes the start() method for the HCD associated with it
- * through the hotplug entry's driver_data.
- */
-static int usb_hcd_da8xx_probe(const struct hc_driver *driver,
-			       struct platform_device *pdev)
+static int ohci_da8xx_probe(struct platform_device *pdev)
 {
-	struct da8xx_ohci_root_hub *hub	= dev_get_platdata(&pdev->dev);
+	struct da8xx_ohci_hcd *da8xx_ohci;
 	struct usb_hcd	*hcd;
 	struct resource *mem;
 	int error, irq;
+	hcd = usb_create_hcd(&ohci_da8xx_hc_driver, &pdev->dev,
+				dev_name(&pdev->dev));
+	if (!hcd)
+		return -ENOMEM;
 
-	if (hub == NULL)
-		return -ENODEV;
+	da8xx_ohci = to_da8xx_ohci(hcd);
+	da8xx_ohci->hcd = hcd;
 
-	usb11_clk = devm_clk_get(&pdev->dev, "usb11");
-	if (IS_ERR(usb11_clk))
-		return PTR_ERR(usb11_clk);
+	da8xx_ohci->usb11_clk = devm_clk_get(&pdev->dev, "usb11");
+	if (IS_ERR(da8xx_ohci->usb11_clk)) {
+		error = PTR_ERR(da8xx_ohci->usb11_clk);
+		if (error != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Failed to get clock.\n");
+		goto err;
+	}
 
-	usb20_clk = devm_clk_get(&pdev->dev, "usb20");
-	if (IS_ERR(usb20_clk))
-		return PTR_ERR(usb20_clk);
+	da8xx_ohci->usb11_phy = devm_phy_get(&pdev->dev, "usb-phy");
+	if (IS_ERR(da8xx_ohci->usb11_phy)) {
+		error = PTR_ERR(da8xx_ohci->usb11_phy);
+		if (error != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Failed to get phy.\n");
+		goto err;
+	}
 
-	hcd = usb_create_hcd(driver, &pdev->dev, dev_name(&pdev->dev));
-	if (!hcd)
-		return -ENOMEM;
+	da8xx_ohci->vbus_reg = devm_regulator_get_optional(&pdev->dev, "vbus");
+	if (IS_ERR(da8xx_ohci->vbus_reg)) {
+		error = PTR_ERR(da8xx_ohci->vbus_reg);
+		if (error == -ENODEV) {
+			da8xx_ohci->vbus_reg = NULL;
+		} else if (error == -EPROBE_DEFER) {
+			goto err;
+		} else {
+			dev_err(&pdev->dev, "Failed to get regulator\n");
+			goto err;
+		}
+	}
 
 	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	hcd->regs = devm_ioremap_resource(&pdev->dev, mem);
@@ -321,60 +454,38 @@ static int usb_hcd_da8xx_probe(const struct hc_driver *driver,
 	hcd->rsrc_start = mem->start;
 	hcd->rsrc_len = resource_size(mem);
 
-	ohci_hcd_init(hcd_to_ohci(hcd));
-
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
 		error = -ENODEV;
 		goto err;
 	}
+
 	error = usb_add_hcd(hcd, irq, 0);
 	if (error)
 		goto err;
 
 	device_wakeup_enable(hcd->self.controller);
 
-	if (hub->ocic_notify) {
-		error = hub->ocic_notify(ohci_da8xx_ocic_handler);
-		if (!error)
-			return 0;
-	}
+	error = ohci_da8xx_register_notify(hcd);
+	if (error)
+		goto err_remove_hcd;
+
+	return 0;
 
+err_remove_hcd:
 	usb_remove_hcd(hcd);
 err:
 	usb_put_hcd(hcd);
 	return error;
 }
 
-/**
- * usb_hcd_da8xx_remove - shutdown processing for DA8xx-based HCDs
- * @dev: USB Host Controller being removed
- * Context: !in_interrupt()
- *
- * Reverses the effect of usb_hcd_da8xx_probe(), first invoking
- * the HCD's stop() method.  It is always called from a thread
- * context, normally "rmmod", "apmd", or something similar.
- */
-static inline void
-usb_hcd_da8xx_remove(struct usb_hcd *hcd, struct platform_device *pdev)
+static int ohci_da8xx_remove(struct platform_device *pdev)
 {
-	struct da8xx_ohci_root_hub *hub	= dev_get_platdata(&pdev->dev);
+	struct usb_hcd	*hcd = platform_get_drvdata(pdev);
 
-	hub->ocic_notify(NULL);
+	ohci_da8xx_unregister_notify(hcd);
 	usb_remove_hcd(hcd);
 	usb_put_hcd(hcd);
-}
-
-static int ohci_hcd_da8xx_drv_probe(struct platform_device *dev)
-{
-	return usb_hcd_da8xx_probe(&ohci_da8xx_hc_driver, dev);
-}
-
-static int ohci_hcd_da8xx_drv_remove(struct platform_device *dev)
-{
-	struct usb_hcd	*hcd = platform_get_drvdata(dev);
-
-	usb_hcd_da8xx_remove(hcd, dev);
 
 	return 0;
 }
@@ -397,7 +508,7 @@ static int ohci_da8xx_suspend(struct platform_device *pdev,
 	if (ret)
 		return ret;
 
-	ohci_da8xx_clock(0);
+	ohci_da8xx_disable(hcd);
 	hcd->state = HC_STATE_SUSPENDED;
 
 	return ret;
@@ -407,32 +518,77 @@ static int ohci_da8xx_resume(struct platform_device *dev)
 {
 	struct usb_hcd	*hcd	= platform_get_drvdata(dev);
 	struct ohci_hcd	*ohci	= hcd_to_ohci(hcd);
+	int ret;
 
 	if (time_before(jiffies, ohci->next_statechange))
 		msleep(5);
 	ohci->next_statechange = jiffies;
 
-	ohci_da8xx_clock(1);
-	dev->dev.power.power_state = PMSG_ON;
-	usb_hcd_resume_root_hub(hcd);
+	ret = ohci_da8xx_enable(hcd);
+	if (ret)
+		return ret;
+
+	ohci_resume(hcd, false);
+
 	return 0;
 }
 #endif
 
+static const struct ohci_driver_overrides da8xx_overrides __initconst = {
+	.reset		 = ohci_da8xx_reset,
+	.extra_priv_size = sizeof(struct da8xx_ohci_hcd),
+};
+
 /*
  * Driver definition to register with platform structure.
  */
 static struct platform_driver ohci_hcd_da8xx_driver = {
-	.probe		= ohci_hcd_da8xx_drv_probe,
-	.remove		= ohci_hcd_da8xx_drv_remove,
+	.probe		= ohci_da8xx_probe,
+	.remove		= ohci_da8xx_remove,
 	.shutdown 	= usb_hcd_platform_shutdown,
 #ifdef	CONFIG_PM
 	.suspend	= ohci_da8xx_suspend,
 	.resume		= ohci_da8xx_resume,
 #endif
 	.driver		= {
-		.name	= "ohci",
+		.name	= DRV_NAME,
+		.of_match_table = of_match_ptr(da8xx_ohci_ids),
 	},
 };
 
-MODULE_ALIAS("platform:ohci");
+static int __init ohci_da8xx_init(void)
+{
+
+	if (usb_disabled())
+		return -ENODEV;
+
+	pr_info("%s: " DRIVER_DESC "\n", DRV_NAME);
+	ohci_init_driver(&ohci_da8xx_hc_driver, &da8xx_overrides);
+
+	/*
+	 * The Davinci da8xx HW has some unusual quirks, which require
+	 * da8xx-specific workarounds. We override certain hc_driver
+	 * functions here to achieve that. We explicitly do not enhance
+	 * ohci_driver_overrides to allow this more easily, since this
+	 * is an unusual case, and we don't want to encourage others to
+	 * override these functions by making it too easy.
+	 */
+
+	orig_ohci_hub_control = ohci_da8xx_hc_driver.hub_control;
+	orig_ohci_hub_status_data = ohci_da8xx_hc_driver.hub_status_data;
+
+	ohci_da8xx_hc_driver.hub_status_data     = ohci_da8xx_hub_status_data;
+	ohci_da8xx_hc_driver.hub_control         = ohci_da8xx_hub_control;
+
+	return platform_driver_register(&ohci_hcd_da8xx_driver);
+}
+module_init(ohci_da8xx_init);
+
+static void __exit ohci_da8xx_exit(void)
+{
+	platform_driver_unregister(&ohci_hcd_da8xx_driver);
+}
+module_exit(ohci_da8xx_exit);
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:" DRV_NAME);
diff --git a/drivers/usb/host/ohci-hcd.c b/drivers/usb/host/ohci-hcd.c
index 86612ac3fda2..8685cf3e6292 100644
--- a/drivers/usb/host/ohci-hcd.c
+++ b/drivers/usb/host/ohci-hcd.c
@@ -1219,11 +1219,6 @@ MODULE_LICENSE ("GPL");
 #define SA1111_DRIVER		ohci_hcd_sa1111_driver
 #endif
 
-#ifdef CONFIG_USB_OHCI_HCD_DAVINCI
-#include "ohci-da8xx.c"
-#define DAVINCI_PLATFORM_DRIVER	ohci_hcd_da8xx_driver
-#endif
-
 #ifdef CONFIG_USB_OHCI_HCD_PPC_OF
 #include "ohci-ppc-of.c"
 #define OF_PLATFORM_DRIVER	ohci_hcd_ppc_of_driver
@@ -1303,19 +1298,9 @@ static int __init ohci_hcd_mod_init(void)
 		goto error_tmio;
 #endif
 
-#ifdef DAVINCI_PLATFORM_DRIVER
-	retval = platform_driver_register(&DAVINCI_PLATFORM_DRIVER);
-	if (retval < 0)
-		goto error_davinci;
-#endif
-
 	return retval;
 
 	/* Error path */
-#ifdef DAVINCI_PLATFORM_DRIVER
-	platform_driver_unregister(&DAVINCI_PLATFORM_DRIVER);
- error_davinci:
-#endif
 #ifdef TMIO_OHCI_DRIVER
 	platform_driver_unregister(&TMIO_OHCI_DRIVER);
  error_tmio:
@@ -1351,9 +1336,6 @@ module_init(ohci_hcd_mod_init);
 
 static void __exit ohci_hcd_mod_exit(void)
 {
-#ifdef DAVINCI_PLATFORM_DRIVER
-	platform_driver_unregister(&DAVINCI_PLATFORM_DRIVER);
-#endif
 #ifdef TMIO_OHCI_DRIVER
 	platform_driver_unregister(&TMIO_OHCI_DRIVER);
 #endif
diff --git a/drivers/usb/host/ohci-mem.c b/drivers/usb/host/ohci-mem.c
index c9e315c6808a..ed8a762b8670 100644
--- a/drivers/usb/host/ohci-mem.c
+++ b/drivers/usb/host/ohci-mem.c
@@ -88,10 +88,9 @@ td_alloc (struct ohci_hcd *hc, gfp_t mem_flags)
 	dma_addr_t	dma;
 	struct td	*td;
 
-	td = dma_pool_alloc (hc->td_cache, mem_flags, &dma);
+	td = dma_pool_zalloc (hc->td_cache, mem_flags, &dma);
 	if (td) {
 		/* in case hc fetches it, make it look dead */
-		memset (td, 0, sizeof *td);
 		td->hwNextTD = cpu_to_hc32 (hc, dma);
 		td->td_dma = dma;
 		/* hashed in td_fill */
@@ -122,9 +121,8 @@ ed_alloc (struct ohci_hcd *hc, gfp_t mem_flags)
 	dma_addr_t	dma;
 	struct ed	*ed;
 
-	ed = dma_pool_alloc (hc->ed_cache, mem_flags, &dma);
+	ed = dma_pool_zalloc (hc->ed_cache, mem_flags, &dma);
 	if (ed) {
-		memset (ed, 0, sizeof (*ed));
 		INIT_LIST_HEAD (&ed->td_list);
 		ed->dma = dma;
 	}
diff --git a/drivers/usb/host/ohci-nxp.c b/drivers/usb/host/ohci-nxp.c
index b7d4756232ae..6df8e2ed40fd 100644
--- a/drivers/usb/host/ohci-nxp.c
+++ b/drivers/usb/host/ohci-nxp.c
@@ -56,8 +56,6 @@ static struct hc_driver __read_mostly ohci_nxp_hc_driver;
 
 static struct i2c_client *isp1301_i2c_client;
 
-extern int usb_disabled(void);
-
 static struct clk *usb_host_clk;
 
 static void isp1301_configure_lpc32xx(void)
@@ -127,6 +125,7 @@ static inline void isp1301_vbus_off(void)
 static void ohci_nxp_start_hc(void)
 {
 	unsigned long tmp = __raw_readl(USB_OTG_STAT_CONTROL) | HOST_EN;
+
 	__raw_writel(tmp, USB_OTG_STAT_CONTROL);
 	isp1301_vbus_on();
 }
@@ -134,6 +133,7 @@ static void ohci_nxp_start_hc(void)
 static void ohci_nxp_stop_hc(void)
 {
 	unsigned long tmp;
+
 	isp1301_vbus_off();
 	tmp = __raw_readl(USB_OTG_STAT_CONTROL) & ~HOST_EN;
 	__raw_writel(tmp, USB_OTG_STAT_CONTROL);
@@ -155,9 +155,8 @@ static int ohci_hcd_nxp_probe(struct platform_device *pdev)
 	}
 
 	isp1301_i2c_client = isp1301_get_client(isp1301_node);
-	if (!isp1301_i2c_client) {
+	if (!isp1301_i2c_client)
 		return -EPROBE_DEFER;
-	}
 
 	ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 	if (ret)
diff --git a/drivers/usb/host/ohci-omap.c b/drivers/usb/host/ohci-omap.c
index 495c1454b9e8..b08e385399b9 100644
--- a/drivers/usb/host/ohci-omap.c
+++ b/drivers/usb/host/ohci-omap.c
@@ -68,9 +68,6 @@ static inline int tps65010_set_gpio_out_value(unsigned gpio, unsigned value)
 
 #endif
 
-extern int usb_disabled(void);
-extern int ocpi_enable(void);
-
 static struct clk *usb_host_ck;
 static struct clk *usb_dc_ck;
 
@@ -296,15 +293,14 @@ static int ohci_omap_reset(struct usb_hcd *hcd)
 /*-------------------------------------------------------------------------*/
 
 /**
- * usb_hcd_omap_probe - initialize OMAP-based HCDs
+ * ohci_hcd_omap_probe - initialize OMAP-based HCDs
  * Context: !in_interrupt()
  *
  * Allocates basic resources for this USB host controller, and
  * then invokes the start() method for the HCD associated with it
  * through the hotplug entry's driver_data.
  */
-static int usb_hcd_omap_probe (const struct hc_driver *driver,
-			  struct platform_device *pdev)
+static int ohci_hcd_omap_probe(struct platform_device *pdev)
 {
 	int retval, irq;
 	struct usb_hcd *hcd = 0;
@@ -336,7 +332,8 @@ static int usb_hcd_omap_probe (const struct hc_driver *driver,
 	}
 
 
-	hcd = usb_create_hcd (driver, &pdev->dev, dev_name(&pdev->dev));
+	hcd = usb_create_hcd(&ohci_omap_hc_driver, &pdev->dev,
+			dev_name(&pdev->dev));
 	if (!hcd) {
 		retval = -ENOMEM;
 		goto err0;
@@ -384,17 +381,18 @@ err0:
 /* may be called with controller, bus, and devices active */
 
 /**
- * usb_hcd_omap_remove - shutdown processing for OMAP-based HCDs
+ * ohci_hcd_omap_remove - shutdown processing for OMAP-based HCDs
  * @dev: USB Host Controller being removed
  * Context: !in_interrupt()
  *
- * Reverses the effect of usb_hcd_omap_probe(), first invoking
+ * Reverses the effect of ohci_hcd_omap_probe(), first invoking
  * the HCD's stop() method.  It is always called from a thread
  * context, normally "rmmod", "apmd", or something similar.
  */
-static inline void
-usb_hcd_omap_remove (struct usb_hcd *hcd, struct platform_device *pdev)
+static int ohci_hcd_omap_remove(struct platform_device *pdev)
 {
+	struct usb_hcd	*hcd = platform_get_drvdata(pdev);
+
 	dev_dbg(hcd->self.controller, "stopping USB Controller\n");
 	usb_remove_hcd(hcd);
 	omap_ohci_clock_power(0);
@@ -409,21 +407,6 @@ usb_hcd_omap_remove (struct usb_hcd *hcd, struct platform_device *pdev)
 	usb_put_hcd(hcd);
 	clk_put(usb_dc_ck);
 	clk_put(usb_host_ck);
-}
-
-/*-------------------------------------------------------------------------*/
-
-static int ohci_hcd_omap_drv_probe(struct platform_device *dev)
-{
-	return usb_hcd_omap_probe(&ohci_omap_hc_driver, dev);
-}
-
-static int ohci_hcd_omap_drv_remove(struct platform_device *dev)
-{
-	struct usb_hcd		*hcd = platform_get_drvdata(dev);
-
-	usb_hcd_omap_remove(hcd, dev);
-
 	return 0;
 }
 
@@ -472,8 +455,8 @@ static int ohci_omap_resume(struct platform_device *dev)
  * Driver definition to register with the OMAP bus
  */
 static struct platform_driver ohci_hcd_omap_driver = {
-	.probe		= ohci_hcd_omap_drv_probe,
-	.remove		= ohci_hcd_omap_drv_remove,
+	.probe		= ohci_hcd_omap_probe,
+	.remove		= ohci_hcd_omap_remove,
 	.shutdown	= usb_hcd_platform_shutdown,
 #ifdef	CONFIG_PM
 	.suspend	= ohci_omap_suspend,
diff --git a/drivers/usb/host/ohci-pxa27x.c b/drivers/usb/host/ohci-pxa27x.c
index a667cf2d5788..79efde8f21e0 100644
--- a/drivers/usb/host/ohci-pxa27x.c
+++ b/drivers/usb/host/ohci-pxa27x.c
@@ -404,7 +404,7 @@ static int ohci_pxa_of_init(struct platform_device *pdev)
 
 
 /**
- * usb_hcd_pxa27x_probe - initialize pxa27x-based HCDs
+ * ohci_hcd_pxa27x_probe - initialize pxa27x-based HCDs
  * Context: !in_interrupt()
  *
  * Allocates basic resources for this USB host controller, and
@@ -412,7 +412,7 @@ static int ohci_pxa_of_init(struct platform_device *pdev)
  * through the hotplug entry's driver_data.
  *
  */
-int usb_hcd_pxa27x_probe (const struct hc_driver *driver, struct platform_device *pdev)
+static int ohci_hcd_pxa27x_probe(struct platform_device *pdev)
 {
 	int retval, irq;
 	struct usb_hcd *hcd;
@@ -442,7 +442,7 @@ int usb_hcd_pxa27x_probe (const struct hc_driver *driver, struct platform_device
 	if (IS_ERR(usb_clk))
 		return PTR_ERR(usb_clk);
 
-	hcd = usb_create_hcd (driver, &pdev->dev, "pxa27x");
+	hcd = usb_create_hcd(&ohci_pxa27x_hc_driver, &pdev->dev, "pxa27x");
 	if (!hcd)
 		return -ENOMEM;
 
@@ -503,17 +503,18 @@ int usb_hcd_pxa27x_probe (const struct hc_driver *driver, struct platform_device
 /* may be called with controller, bus, and devices active */
 
 /**
- * usb_hcd_pxa27x_remove - shutdown processing for pxa27x-based HCDs
+ * ohci_hcd_pxa27x_remove - shutdown processing for pxa27x-based HCDs
  * @dev: USB Host Controller being removed
  * Context: !in_interrupt()
  *
- * Reverses the effect of usb_hcd_pxa27x_probe(), first invoking
+ * Reverses the effect of ohci_hcd_pxa27x_probe(), first invoking
  * the HCD's stop() method.  It is always called from a thread
  * context, normally "rmmod", "apmd", or something similar.
  *
  */
-void usb_hcd_pxa27x_remove (struct usb_hcd *hcd, struct platform_device *pdev)
+static int ohci_hcd_pxa27x_remove(struct platform_device *pdev)
 {
+	struct usb_hcd *hcd = platform_get_drvdata(pdev);
 	struct pxa27x_ohci *pxa_ohci = to_pxa27x_ohci(hcd);
 	unsigned int i;
 
@@ -524,28 +525,11 @@ void usb_hcd_pxa27x_remove (struct usb_hcd *hcd, struct platform_device *pdev)
 		pxa27x_ohci_set_vbus_power(pxa_ohci, i, false);
 
 	usb_put_hcd(hcd);
+	return 0;
 }
 
 /*-------------------------------------------------------------------------*/
 
-static int ohci_hcd_pxa27x_drv_probe(struct platform_device *pdev)
-{
-	pr_debug ("In ohci_hcd_pxa27x_drv_probe");
-
-	if (usb_disabled())
-		return -ENODEV;
-
-	return usb_hcd_pxa27x_probe(&ohci_pxa27x_hc_driver, pdev);
-}
-
-static int ohci_hcd_pxa27x_drv_remove(struct platform_device *pdev)
-{
-	struct usb_hcd *hcd = platform_get_drvdata(pdev);
-
-	usb_hcd_pxa27x_remove(hcd, pdev);
-	return 0;
-}
-
 #ifdef CONFIG_PM
 static int ohci_hcd_pxa27x_drv_suspend(struct device *dev)
 {
@@ -598,8 +582,8 @@ static const struct dev_pm_ops ohci_hcd_pxa27x_pm_ops = {
 #endif
 
 static struct platform_driver ohci_hcd_pxa27x_driver = {
-	.probe		= ohci_hcd_pxa27x_drv_probe,
-	.remove		= ohci_hcd_pxa27x_drv_remove,
+	.probe		= ohci_hcd_pxa27x_probe,
+	.remove		= ohci_hcd_pxa27x_remove,
 	.shutdown	= usb_hcd_platform_shutdown,
 	.driver		= {
 		.name	= "pxa27x-ohci",
diff --git a/drivers/usb/host/ohci-s3c2410.c b/drivers/usb/host/ohci-s3c2410.c
index 7a1919ca543a..b006b93126f7 100644
--- a/drivers/usb/host/ohci-s3c2410.c
+++ b/drivers/usb/host/ohci-s3c2410.c
@@ -43,6 +43,8 @@ static const char hcd_name[] = "ohci-s3c2410";
 static struct clk *clk;
 static struct clk *usb_clk;
 
+static struct hc_driver __read_mostly ohci_s3c2410_hc_driver;
+
 /* forward definitions */
 
 static void s3c2410_hcd_oc(struct s3c2410_hcd_info *info, int port_oc);
@@ -321,26 +323,29 @@ static void s3c2410_hcd_oc(struct s3c2410_hcd_info *info, int port_oc)
 /* may be called with controller, bus, and devices active */
 
 /*
- * usb_hcd_s3c2410_remove - shutdown processing for HCD
+ * ohci_hcd_s3c2410_remove - shutdown processing for HCD
  * @dev: USB Host Controller being removed
  * Context: !in_interrupt()
  *
- * Reverses the effect of usb_hcd_3c2410_probe(), first invoking
+ * Reverses the effect of ohci_hcd_3c2410_probe(), first invoking
  * the HCD's stop() method.  It is always called from a thread
  * context, normally "rmmod", "apmd", or something similar.
  *
 */
 
-static void
-usb_hcd_s3c2410_remove(struct usb_hcd *hcd, struct platform_device *dev)
+static int
+ohci_hcd_s3c2410_remove(struct platform_device *dev)
 {
+	struct usb_hcd *hcd = platform_get_drvdata(dev);
+
 	usb_remove_hcd(hcd);
 	s3c2410_stop_hc(dev);
 	usb_put_hcd(hcd);
+	return 0;
 }
 
 /**
- * usb_hcd_s3c2410_probe - initialize S3C2410-based HCDs
+ * ohci_hcd_s3c2410_probe - initialize S3C2410-based HCDs
  * Context: !in_interrupt()
  *
  * Allocates basic resources for this USB host controller, and
@@ -348,8 +353,7 @@ usb_hcd_s3c2410_remove(struct usb_hcd *hcd, struct platform_device *dev)
  * through the hotplug entry's driver_data.
  *
  */
-static int usb_hcd_s3c2410_probe(const struct hc_driver *driver,
-				  struct platform_device *dev)
+static int ohci_hcd_s3c2410_probe(struct platform_device *dev)
 {
 	struct usb_hcd *hcd = NULL;
 	struct s3c2410_hcd_info *info = dev_get_platdata(&dev->dev);
@@ -358,7 +362,7 @@ static int usb_hcd_s3c2410_probe(const struct hc_driver *driver,
 	s3c2410_usb_set_power(info, 1, 1);
 	s3c2410_usb_set_power(info, 2, 1);
 
-	hcd = usb_create_hcd(driver, &dev->dev, "s3c24xx");
+	hcd = usb_create_hcd(&ohci_s3c2410_hc_driver, &dev->dev, "s3c24xx");
 	if (hcd == NULL)
 		return -ENOMEM;
 
@@ -404,21 +408,6 @@ static int usb_hcd_s3c2410_probe(const struct hc_driver *driver,
 
 /*-------------------------------------------------------------------------*/
 
-static struct hc_driver __read_mostly ohci_s3c2410_hc_driver;
-
-static int ohci_hcd_s3c2410_drv_probe(struct platform_device *pdev)
-{
-	return usb_hcd_s3c2410_probe(&ohci_s3c2410_hc_driver, pdev);
-}
-
-static int ohci_hcd_s3c2410_drv_remove(struct platform_device *pdev)
-{
-	struct usb_hcd *hcd = platform_get_drvdata(pdev);
-
-	usb_hcd_s3c2410_remove(hcd, pdev);
-	return 0;
-}
-
 #ifdef CONFIG_PM
 static int ohci_hcd_s3c2410_drv_suspend(struct device *dev)
 {
@@ -457,13 +446,21 @@ static const struct dev_pm_ops ohci_hcd_s3c2410_pm_ops = {
 	.resume		= ohci_hcd_s3c2410_drv_resume,
 };
 
+static const struct of_device_id ohci_hcd_s3c2410_dt_ids[] = {
+	{ .compatible = "samsung,s3c2410-ohci" },
+	{ /* sentinel */ }
+};
+
+MODULE_DEVICE_TABLE(of, ohci_hcd_s3c2410_dt_ids);
+
 static struct platform_driver ohci_hcd_s3c2410_driver = {
-	.probe		= ohci_hcd_s3c2410_drv_probe,
-	.remove		= ohci_hcd_s3c2410_drv_remove,
+	.probe		= ohci_hcd_s3c2410_probe,
+	.remove		= ohci_hcd_s3c2410_remove,
 	.shutdown	= usb_hcd_platform_shutdown,
 	.driver		= {
 		.name	= "s3c2410-ohci",
 		.pm	= &ohci_hcd_s3c2410_pm_ops,
+		.of_match_table	= ohci_hcd_s3c2410_dt_ids,
 	},
 };
 
diff --git a/drivers/usb/host/uhci-pci.c b/drivers/usb/host/uhci-pci.c
index 940304c33224..02260cfdedb1 100644
--- a/drivers/usb/host/uhci-pci.c
+++ b/drivers/usb/host/uhci-pci.c
@@ -129,6 +129,10 @@ static int uhci_pci_init(struct usb_hcd *hcd)
 	if (to_pci_dev(uhci_dev(uhci))->vendor == PCI_VENDOR_ID_HP)
 		uhci->wait_for_hp = 1;
 
+	/* Intel controllers use non-PME wakeup signalling */
+	if (to_pci_dev(uhci_dev(uhci))->vendor == PCI_VENDOR_ID_INTEL)
+		device_set_run_wake(uhci_dev(uhci), 1);
+
 	/* Set up pointers to PCI-specific functions */
 	uhci->reset_hc = uhci_pci_reset_hc;
 	uhci->check_and_reset_hc = uhci_pci_check_and_reset_hc;
diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c
index 6afe32381209..321de2e0161b 100644
--- a/drivers/usb/host/xhci-mem.c
+++ b/drivers/usb/host/xhci-mem.c
@@ -1032,7 +1032,6 @@ int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id,
 		goto fail;
 	dev->num_rings_cached = 0;
 
-	init_completion(&dev->cmd_completion);
 	dev->udev = udev;
 
 	/* Point to output device context in dcbaa. */
@@ -1370,7 +1369,7 @@ static u32 xhci_get_endpoint_max_burst(struct usb_device *udev,
 	if (udev->speed == USB_SPEED_HIGH &&
 	    (usb_endpoint_xfer_isoc(&ep->desc) ||
 	     usb_endpoint_xfer_int(&ep->desc)))
-		return (usb_endpoint_maxp(&ep->desc) & 0x1800) >> 11;
+		return usb_endpoint_maxp_mult(&ep->desc) - 1;
 
 	return 0;
 }
@@ -1415,10 +1414,10 @@ static u32 xhci_get_max_esit_payload(struct usb_device *udev,
 	else if (udev->speed >= USB_SPEED_SUPER)
 		return le16_to_cpu(ep->ss_ep_comp.wBytesPerInterval);
 
-	max_packet = GET_MAX_PACKET(usb_endpoint_maxp(&ep->desc));
-	max_burst = (usb_endpoint_maxp(&ep->desc) & 0x1800) >> 11;
+	max_packet = usb_endpoint_maxp(&ep->desc);
+	max_burst = usb_endpoint_maxp_mult(&ep->desc);
 	/* A 0 in max burst means 1 transfer per ESIT */
-	return max_packet * (max_burst + 1);
+	return max_packet * max_burst;
 }
 
 /* Set up an endpoint with one ring segment.  Do not allocate stream rings.
@@ -1461,7 +1460,7 @@ int xhci_endpoint_init(struct xhci_hcd *xhci,
 	max_esit_payload = xhci_get_max_esit_payload(udev, ep);
 	interval = xhci_get_endpoint_interval(udev, ep);
 	mult = xhci_get_endpoint_mult(udev, ep);
-	max_packet = GET_MAX_PACKET(usb_endpoint_maxp(&ep->desc));
+	max_packet = usb_endpoint_maxp(&ep->desc);
 	max_burst = xhci_get_endpoint_max_burst(udev, ep);
 	avg_trb_len = max_esit_payload;
 
@@ -2384,7 +2383,7 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
 	 * "physically contiguous and 64-byte (cache line) aligned".
 	 */
 	xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma,
-			GFP_KERNEL);
+			flags);
 	if (!xhci->dcbaa)
 		goto fail;
 	memset(xhci->dcbaa, 0, sizeof *(xhci->dcbaa));
@@ -2480,7 +2479,7 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
 
 	xhci->erst.entries = dma_alloc_coherent(dev,
 			sizeof(struct xhci_erst_entry) * ERST_NUM_SEGS, &dma,
-			GFP_KERNEL);
+			flags);
 	if (!xhci->erst.entries)
 		goto fail;
 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
@@ -2536,7 +2535,6 @@ int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
 	 * something other than the default (~1ms minimum between interrupts).
 	 * See section 5.5.1.2.
 	 */
-	init_completion(&xhci->addr_dev);
 	for (i = 0; i < MAX_HC_SLOTS; ++i)
 		xhci->devs[i] = NULL;
 	for (i = 0; i < USB_MAXCHILDREN; ++i) {
diff --git a/drivers/usb/host/xhci-mtk-sch.c b/drivers/usb/host/xhci-mtk-sch.c
index 73f763c4f5f5..6e7ddf6cafae 100644
--- a/drivers/usb/host/xhci-mtk-sch.c
+++ b/drivers/usb/host/xhci-mtk-sch.c
@@ -337,7 +337,7 @@ int xhci_mtk_add_ep_quirk(struct usb_hcd *hcd, struct usb_device *udev,
 
 	xhci_dbg(xhci, "%s() type:%d, speed:%d, mpkt:%d, dir:%d, ep:%p\n",
 		__func__, usb_endpoint_type(&ep->desc), udev->speed,
-		GET_MAX_PACKET(usb_endpoint_maxp(&ep->desc)),
+		usb_endpoint_maxp(&ep->desc),
 		usb_endpoint_dir_in(&ep->desc), ep);
 
 	if (!need_bw_sch(ep, udev->speed, slot_ctx->tt_info & TT_SLOT)) {
@@ -403,7 +403,7 @@ void xhci_mtk_drop_ep_quirk(struct usb_hcd *hcd, struct usb_device *udev,
 
 	xhci_dbg(xhci, "%s() type:%d, speed:%d, mpks:%d, dir:%d, ep:%p\n",
 		__func__, usb_endpoint_type(&ep->desc), udev->speed,
-		GET_MAX_PACKET(usb_endpoint_maxp(&ep->desc)),
+		usb_endpoint_maxp(&ep->desc),
 		usb_endpoint_dir_in(&ep->desc), ep);
 
 	if (!need_bw_sch(ep, udev->speed, slot_ctx->tt_info & TT_SLOT))
diff --git a/drivers/usb/host/xhci-mtk.c b/drivers/usb/host/xhci-mtk.c
index 79959f17c38c..1094ebd2838f 100644
--- a/drivers/usb/host/xhci-mtk.c
+++ b/drivers/usb/host/xhci-mtk.c
@@ -94,6 +94,9 @@ static int xhci_mtk_host_enable(struct xhci_hcd_mtk *mtk)
 	int ret;
 	int i;
 
+	if (!mtk->has_ippc)
+		return 0;
+
 	/* power on host ip */
 	value = readl(&ippc->ip_pw_ctr1);
 	value &= ~CTRL1_IP_HOST_PDN;
@@ -139,6 +142,9 @@ static int xhci_mtk_host_disable(struct xhci_hcd_mtk *mtk)
 	int ret;
 	int i;
 
+	if (!mtk->has_ippc)
+		return 0;
+
 	/* power down all u3 ports */
 	for (i = 0; i < mtk->num_u3_ports; i++) {
 		value = readl(&ippc->u3_ctrl_p[i]);
@@ -173,6 +179,9 @@ static int xhci_mtk_ssusb_config(struct xhci_hcd_mtk *mtk)
 	struct mu3c_ippc_regs __iomem *ippc = mtk->ippc_regs;
 	u32 value;
 
+	if (!mtk->has_ippc)
+		return 0;
+
 	/* reset whole ip */
 	value = readl(&ippc->ip_pw_ctr0);
 	value |= CTRL0_IP_SW_RST;
@@ -475,6 +484,7 @@ static void xhci_mtk_quirks(struct device *dev, struct xhci_hcd *xhci)
 /* called during probe() after chip reset completes */
 static int xhci_mtk_setup(struct usb_hcd *hcd)
 {
+	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
 	struct xhci_hcd_mtk *mtk = hcd_to_mtk(hcd);
 	int ret;
 
@@ -482,12 +492,21 @@ static int xhci_mtk_setup(struct usb_hcd *hcd)
 		ret = xhci_mtk_ssusb_config(mtk);
 		if (ret)
 			return ret;
+	}
+
+	ret = xhci_gen_setup(hcd, xhci_mtk_quirks);
+	if (ret)
+		return ret;
+
+	if (usb_hcd_is_primary_hcd(hcd)) {
+		mtk->num_u3_ports = xhci->num_usb3_ports;
+		mtk->num_u2_ports = xhci->num_usb2_ports;
 		ret = xhci_mtk_sch_init(mtk);
 		if (ret)
 			return ret;
 	}
 
-	return xhci_gen_setup(hcd, xhci_mtk_quirks);
+	return ret;
 }
 
 static int xhci_mtk_probe(struct platform_device *pdev)
@@ -586,7 +605,7 @@ static int xhci_mtk_probe(struct platform_device *pdev)
 	mtk->hcd = platform_get_drvdata(pdev);
 	platform_set_drvdata(pdev, mtk);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mac");
 	hcd->regs = devm_ioremap_resource(dev, res);
 	if (IS_ERR(hcd->regs)) {
 		ret = PTR_ERR(hcd->regs);
@@ -595,11 +614,16 @@ static int xhci_mtk_probe(struct platform_device *pdev)
 	hcd->rsrc_start = res->start;
 	hcd->rsrc_len = resource_size(res);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-	mtk->ippc_regs = devm_ioremap_resource(dev, res);
-	if (IS_ERR(mtk->ippc_regs)) {
-		ret = PTR_ERR(mtk->ippc_regs);
-		goto put_usb2_hcd;
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ippc");
+	if (res) {	/* ippc register is optional */
+		mtk->ippc_regs = devm_ioremap_resource(dev, res);
+		if (IS_ERR(mtk->ippc_regs)) {
+			ret = PTR_ERR(mtk->ippc_regs);
+			goto put_usb2_hcd;
+		}
+		mtk->has_ippc = true;
+	} else {
+		mtk->has_ippc = false;
 	}
 
 	for (phy_num = 0; phy_num < mtk->num_phys; phy_num++) {
diff --git a/drivers/usb/host/xhci-mtk.h b/drivers/usb/host/xhci-mtk.h
index 7da677c79ea8..2845c49efe1b 100644
--- a/drivers/usb/host/xhci-mtk.h
+++ b/drivers/usb/host/xhci-mtk.h
@@ -118,6 +118,7 @@ struct xhci_hcd_mtk {
 	struct usb_hcd *hcd;
 	struct mu3h_sch_bw_info *sch_array;
 	struct mu3c_ippc_regs __iomem *ippc_regs;
+	bool has_ippc;
 	int num_u2_ports;
 	int num_u3_ports;
 	struct regulator *vusb33;
diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c
index ed56bf9ed885..ddfab301e366 100644
--- a/drivers/usb/host/xhci-plat.c
+++ b/drivers/usb/host/xhci-plat.c
@@ -100,6 +100,12 @@ static const struct xhci_plat_priv xhci_plat_renesas_rcar_gen3 = {
 	.plat_start = xhci_rcar_start,
 };
 
+static const struct xhci_plat_priv xhci_plat_renesas_rcar_r8a7796 = {
+	.firmware_name = XHCI_RCAR_FIRMWARE_NAME_V3,
+	.init_quirk = xhci_rcar_init_quirk,
+	.plat_start = xhci_rcar_start,
+};
+
 static const struct of_device_id usb_xhci_of_match[] = {
 	{
 		.compatible = "generic-xhci",
@@ -124,6 +130,9 @@ static const struct of_device_id usb_xhci_of_match[] = {
 		.compatible = "renesas,xhci-r8a7795",
 		.data = &xhci_plat_renesas_rcar_gen3,
 	}, {
+		.compatible = "renesas,xhci-r8a7796",
+		.data = &xhci_plat_renesas_rcar_r8a7796,
+	}, {
 		.compatible = "renesas,rcar-gen2-xhci",
 		.data = &xhci_plat_renesas_rcar_gen2,
 	}, {
diff --git a/drivers/usb/host/xhci-rcar.c b/drivers/usb/host/xhci-rcar.c
index 0e4535e632ec..d28df386e780 100644
--- a/drivers/usb/host/xhci-rcar.c
+++ b/drivers/usb/host/xhci-rcar.c
@@ -19,6 +19,8 @@
 #include "xhci-rcar.h"
 
 /*
+* - The V3 firmware is for r8a7796 (with good performance).
+* - The V2 firmware can be used on both r8a7795 (es1.x) and r8a7796.
 * - The V2 firmware is possible to use on R-Car Gen2. However, the V2 causes
 *   performance degradation. So, this driver continues to use the V1 if R-Car
 *   Gen2.
@@ -26,6 +28,7 @@
 */
 MODULE_FIRMWARE(XHCI_RCAR_FIRMWARE_NAME_V1);
 MODULE_FIRMWARE(XHCI_RCAR_FIRMWARE_NAME_V2);
+MODULE_FIRMWARE(XHCI_RCAR_FIRMWARE_NAME_V3);
 
 /*** Register Offset ***/
 #define RCAR_USB3_INT_ENA	0x224	/* Interrupt Enable */
@@ -92,6 +95,7 @@ static int xhci_rcar_is_gen3(struct device *dev)
 	struct device_node *node = dev->of_node;
 
 	return of_device_is_compatible(node, "renesas,xhci-r8a7795") ||
+		of_device_is_compatible(node, "renesas,xhci-r8a7796") ||
 		of_device_is_compatible(node, "renesas,rcar-gen3-xhci");
 }
 
diff --git a/drivers/usb/host/xhci-rcar.h b/drivers/usb/host/xhci-rcar.h
index 2941a25cfe98..d2ffe20401cf 100644
--- a/drivers/usb/host/xhci-rcar.h
+++ b/drivers/usb/host/xhci-rcar.h
@@ -13,6 +13,7 @@
 
 #define XHCI_RCAR_FIRMWARE_NAME_V1	"r8a779x_usb3_v1.dlmem"
 #define XHCI_RCAR_FIRMWARE_NAME_V2	"r8a779x_usb3_v2.dlmem"
+#define XHCI_RCAR_FIRMWARE_NAME_V3	"r8a779x_usb3_v3.dlmem"
 
 #if IS_ENABLED(CONFIG_USB_XHCI_RCAR)
 void xhci_rcar_start(struct usb_hcd *hcd);
diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c
index 797137e26549..bdf6b13d9b67 100644
--- a/drivers/usb/host/xhci-ring.c
+++ b/drivers/usb/host/xhci-ring.c
@@ -89,6 +89,11 @@ dma_addr_t xhci_trb_virt_to_dma(struct xhci_segment *seg,
 	return seg->dma + (segment_offset * sizeof(*trb));
 }
 
+static bool trb_is_noop(union xhci_trb *trb)
+{
+	return TRB_TYPE_NOOP_LE32(trb->generic.field[3]);
+}
+
 static bool trb_is_link(union xhci_trb *trb)
 {
 	return TRB_TYPE_LINK_LE32(trb->link.control);
@@ -110,6 +115,20 @@ static bool link_trb_toggles_cycle(union xhci_trb *trb)
 	return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
 }
 
+static bool last_td_in_urb(struct xhci_td *td)
+{
+	struct urb_priv *urb_priv = td->urb->hcpriv;
+
+	return urb_priv->td_cnt == urb_priv->length;
+}
+
+static void inc_td_cnt(struct urb *urb)
+{
+	struct urb_priv *urb_priv = urb->hcpriv;
+
+	urb_priv->td_cnt++;
+}
+
 /* Updates trb to point to the next TRB in the ring, and updates seg if the next
  * TRB is in a new segment.  This does not skip over link TRBs, and it does not
  * effect the ring dequeue or enqueue pointers.
@@ -303,7 +322,6 @@ static int xhci_abort_cmd_ring(struct xhci_hcd *xhci)
 				"maybe the host is dead\n");
 		del_timer(&xhci->cmd_timer);
 		xhci->xhc_state |= XHCI_STATE_DYING;
-		xhci_quiesce(xhci);
 		xhci_halt(xhci);
 		return -ESHUTDOWN;
 	}
@@ -473,9 +491,8 @@ void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
 		if (new_deq == cur_td->last_trb)
 			td_last_trb_found = true;
 
-		if (cycle_found &&
-		    TRB_TYPE_LINK_LE32(new_deq->generic.field[3]) &&
-		    new_deq->generic.field[3] & cpu_to_le32(LINK_TOGGLE))
+		if (cycle_found && trb_is_link(new_deq) &&
+		    link_trb_toggles_cycle(new_deq))
 			state->new_cycle_state ^= 0x1;
 
 		next_trb(xhci, ep_ring, &new_seg, &new_deq);
@@ -511,54 +528,32 @@ void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
  * of this TD.)  This is used to remove partially enqueued isoc TDs from a ring.
  */
 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
-		struct xhci_td *cur_td, bool flip_cycle)
+		       struct xhci_td *td, bool flip_cycle)
 {
-	struct xhci_segment *cur_seg;
-	union xhci_trb *cur_trb;
-
-	for (cur_seg = cur_td->start_seg, cur_trb = cur_td->first_trb;
-			true;
-			next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) {
-		if (TRB_TYPE_LINK_LE32(cur_trb->generic.field[3])) {
-			/* Unchain any chained Link TRBs, but
-			 * leave the pointers intact.
-			 */
-			cur_trb->generic.field[3] &= cpu_to_le32(~TRB_CHAIN);
-			/* Flip the cycle bit (link TRBs can't be the first
-			 * or last TRB).
-			 */
-			if (flip_cycle)
-				cur_trb->generic.field[3] ^=
-					cpu_to_le32(TRB_CYCLE);
-			xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
-					"Cancel (unchain) link TRB");
-			xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
-					"Address = %p (0x%llx dma); "
-					"in seg %p (0x%llx dma)",
-					cur_trb,
-					(unsigned long long)xhci_trb_virt_to_dma(cur_seg, cur_trb),
-					cur_seg,
-					(unsigned long long)cur_seg->dma);
+	struct xhci_segment *seg	= td->start_seg;
+	union xhci_trb *trb		= td->first_trb;
+
+	while (1) {
+		if (trb_is_link(trb)) {
+			/* unchain chained link TRBs */
+			trb->link.control &= cpu_to_le32(~TRB_CHAIN);
 		} else {
-			cur_trb->generic.field[0] = 0;
-			cur_trb->generic.field[1] = 0;
-			cur_trb->generic.field[2] = 0;
+			trb->generic.field[0] = 0;
+			trb->generic.field[1] = 0;
+			trb->generic.field[2] = 0;
 			/* Preserve only the cycle bit of this TRB */
-			cur_trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
-			/* Flip the cycle bit except on the first or last TRB */
-			if (flip_cycle && cur_trb != cur_td->first_trb &&
-					cur_trb != cur_td->last_trb)
-				cur_trb->generic.field[3] ^=
-					cpu_to_le32(TRB_CYCLE);
-			cur_trb->generic.field[3] |= cpu_to_le32(
+			trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
+			trb->generic.field[3] |= cpu_to_le32(
 				TRB_TYPE(TRB_TR_NOOP));
-			xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
-					"TRB to noop at offset 0x%llx",
-					(unsigned long long)
-					xhci_trb_virt_to_dma(cur_seg, cur_trb));
 		}
-		if (cur_trb == cur_td->last_trb)
+		/* flip cycle if asked to */
+		if (flip_cycle && trb != td->first_trb && trb != td->last_trb)
+			trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE);
+
+		if (trb == td->last_trb)
 			break;
+
+		next_trb(xhci, ep_ring, &seg, &trb);
 	}
 }
 
@@ -574,39 +569,33 @@ static void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci,
 		ep->stop_cmds_pending--;
 }
 
-/* Must be called with xhci->lock held in interrupt context */
+/*
+ * Must be called with xhci->lock held in interrupt context,
+ * releases and re-acquires xhci->lock
+ */
 static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci,
-		struct xhci_td *cur_td, int status)
+				     struct xhci_td *cur_td, int status)
 {
-	struct usb_hcd *hcd;
-	struct urb	*urb;
-	struct urb_priv	*urb_priv;
+	struct urb	*urb		= cur_td->urb;
+	struct urb_priv	*urb_priv	= urb->hcpriv;
+	struct usb_hcd	*hcd		= bus_to_hcd(urb->dev->bus);
 
-	urb = cur_td->urb;
-	urb_priv = urb->hcpriv;
-	urb_priv->td_cnt++;
-	hcd = bus_to_hcd(urb->dev->bus);
-
-	/* Only giveback urb when this is the last td in urb */
-	if (urb_priv->td_cnt == urb_priv->length) {
-		if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
-			xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
-			if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs	== 0) {
-				if (xhci->quirks & XHCI_AMD_PLL_FIX)
-					usb_amd_quirk_pll_enable();
-			}
+	if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
+		xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
+		if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs	== 0) {
+			if (xhci->quirks & XHCI_AMD_PLL_FIX)
+				usb_amd_quirk_pll_enable();
 		}
-		usb_hcd_unlink_urb_from_ep(hcd, urb);
-
-		spin_unlock(&xhci->lock);
-		usb_hcd_giveback_urb(hcd, urb, status);
-		xhci_urb_free_priv(urb_priv);
-		spin_lock(&xhci->lock);
 	}
+	xhci_urb_free_priv(urb_priv);
+	usb_hcd_unlink_urb_from_ep(hcd, urb);
+	spin_unlock(&xhci->lock);
+	usb_hcd_giveback_urb(hcd, urb, status);
+	spin_lock(&xhci->lock);
 }
 
-void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, struct xhci_ring *ring,
-				 struct xhci_td *td)
+static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci,
+		struct xhci_ring *ring, struct xhci_td *td)
 {
 	struct device *dev = xhci_to_hcd(xhci)->self.controller;
 	struct xhci_segment *seg = td->bounce_seg;
@@ -752,7 +741,9 @@ remove_finished_td:
 		ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
 		if (ep_ring && cur_td->bounce_seg)
 			xhci_unmap_td_bounce_buffer(xhci, ep_ring, cur_td);
-		xhci_giveback_urb_in_irq(xhci, cur_td, 0);
+		inc_td_cnt(cur_td->urb);
+		if (last_td_in_urb(cur_td))
+			xhci_giveback_urb_in_irq(xhci, cur_td, 0);
 
 		/* Stop processing the cancelled list if the watchdog timer is
 		 * running.
@@ -777,7 +768,10 @@ static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring)
 
 		if (cur_td->bounce_seg)
 			xhci_unmap_td_bounce_buffer(xhci, ring, cur_td);
-		xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
+
+		inc_td_cnt(cur_td->urb);
+		if (last_td_in_urb(cur_td))
+			xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
 	}
 }
 
@@ -814,7 +808,10 @@ static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci,
 		cur_td = list_first_entry(&ep->cancelled_td_list,
 				struct xhci_td, cancelled_td_list);
 		list_del_init(&cur_td->cancelled_td_list);
-		xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
+
+		inc_td_cnt(cur_td->urb);
+		if (last_td_in_urb(cur_td))
+			xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
 	}
 }
 
@@ -1003,8 +1000,7 @@ static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
 			break;
 		case COMP_CTX_STATE:
 			xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n");
-			ep_state = le32_to_cpu(ep_ctx->ep_info);
-			ep_state &= EP_STATE_MASK;
+			ep_state = GET_EP_CTX_STATE(ep_ctx);
 			slot_state = le32_to_cpu(slot_ctx->dev_state);
 			slot_state = GET_SLOT_STATE(slot_state);
 			xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
@@ -1096,12 +1092,12 @@ static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
 }
 
 static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id,
-		u32 cmd_comp_code)
+		struct xhci_command *command, u32 cmd_comp_code)
 {
 	if (cmd_comp_code == COMP_SUCCESS)
-		xhci->slot_id = slot_id;
+		command->slot_id = slot_id;
 	else
-		xhci->slot_id = 0;
+		command->slot_id = 0;
 }
 
 static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id)
@@ -1183,7 +1179,7 @@ static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci,
 		struct xhci_event_cmd *event)
 {
 	if (!(xhci->quirks & XHCI_NEC_HOST)) {
-		xhci->error_bitmask |= 1 << 6;
+		xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n");
 		return;
 	}
 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
@@ -1325,14 +1321,13 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
 	cmd_trb = xhci->cmd_ring->dequeue;
 	cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
 			cmd_trb);
-	/* Is the command ring deq ptr out of sync with the deq seg ptr? */
-	if (cmd_dequeue_dma == 0) {
-		xhci->error_bitmask |= 1 << 4;
-		return;
-	}
-	/* Does the DMA address match our internal dequeue pointer address? */
-	if (cmd_dma != (u64) cmd_dequeue_dma) {
-		xhci->error_bitmask |= 1 << 5;
+	/*
+	 * Check whether the completion event is for our internal kept
+	 * command.
+	 */
+	if (!cmd_dequeue_dma || cmd_dma != (u64)cmd_dequeue_dma) {
+		xhci_warn(xhci,
+			  "ERROR mismatched command completion event\n");
 		return;
 	}
 
@@ -1371,7 +1366,7 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
 	cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3]));
 	switch (cmd_type) {
 	case TRB_ENABLE_SLOT:
-		xhci_handle_cmd_enable_slot(xhci, slot_id, cmd_comp_code);
+		xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code);
 		break;
 	case TRB_DISABLE_SLOT:
 		xhci_handle_cmd_disable_slot(xhci, slot_id);
@@ -1418,7 +1413,7 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
 		break;
 	default:
 		/* Skip over unknown commands on the event ring */
-		xhci->error_bitmask |= 1 << 6;
+		xhci_info(xhci, "INFO unknown command type %d\n", cmd_type);
 		break;
 	}
 
@@ -1519,10 +1514,10 @@ static void handle_port_status(struct xhci_hcd *xhci,
 	bool bogus_port_status = false;
 
 	/* Port status change events always have a successful completion code */
-	if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS) {
-		xhci_warn(xhci, "WARN: xHC returned failed port status event\n");
-		xhci->error_bitmask |= 1 << 8;
-	}
+	if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS)
+		xhci_warn(xhci,
+			  "WARN: xHC returned failed port status event\n");
+
 	port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
 	xhci_dbg(xhci, "Port Status Change Event for port %d\n", port_id);
 
@@ -1759,7 +1754,7 @@ struct xhci_segment *trb_in_td(struct xhci_hcd *xhci,
 static void xhci_cleanup_halted_endpoint(struct xhci_hcd *xhci,
 		unsigned int slot_id, unsigned int ep_index,
 		unsigned int stream_id,
-		struct xhci_td *td, union xhci_trb *event_trb)
+		struct xhci_td *td, union xhci_trb *ep_trb)
 {
 	struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
 	struct xhci_command *command;
@@ -1798,8 +1793,7 @@ static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci,
 		 * endpoint anyway.  Check if a babble halted the
 		 * endpoint.
 		 */
-		if ((ep_ctx->ep_info & cpu_to_le32(EP_STATE_MASK)) ==
-		    cpu_to_le32(EP_STATE_HALTED))
+		if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_HALTED)
 			return 1;
 
 	return 0;
@@ -1824,7 +1818,7 @@ int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code)
  * Return 1 if the urb can be given back.
  */
 static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td,
-	union xhci_trb *event_trb, struct xhci_transfer_event *event,
+	union xhci_trb *ep_trb, struct xhci_transfer_event *event,
 	struct xhci_virt_ep *ep, int *status, bool skip)
 {
 	struct xhci_virt_device *xdev;
@@ -1833,7 +1827,6 @@ static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td,
 	int ep_index;
 	struct urb *urb = NULL;
 	struct xhci_ep_ctx *ep_ctx;
-	int ret = 0;
 	struct urb_priv	*urb_priv;
 	u32 trb_comp_code;
 
@@ -1866,7 +1859,7 @@ static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td,
 		 * The class driver clears the device side halt later.
 		 */
 		xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index,
-					ep_ring->stream_id, td, event_trb);
+					ep_ring->stream_id, td, ep_trb);
 	} else {
 		/* Update ring dequeue pointer */
 		while (ep_ring->dequeue != td->last_trb)
@@ -1889,41 +1882,54 @@ td_cleanup:
 	 * unsigned).  Play it safe and say we didn't transfer anything.
 	 */
 	if (urb->actual_length > urb->transfer_buffer_length) {
-		xhci_warn(xhci, "URB transfer length is wrong, xHC issue? req. len = %u, act. len = %u\n",
-			urb->transfer_buffer_length,
-			urb->actual_length);
+		xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n",
+			  urb->transfer_buffer_length, urb->actual_length);
 		urb->actual_length = 0;
-		if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
-			*status = -EREMOTEIO;
-		else
-			*status = 0;
+		*status = 0;
 	}
 	list_del_init(&td->td_list);
 	/* Was this TD slated to be cancelled but completed anyway? */
 	if (!list_empty(&td->cancelled_td_list))
 		list_del_init(&td->cancelled_td_list);
 
-	urb_priv->td_cnt++;
+	inc_td_cnt(urb);
 	/* Giveback the urb when all the tds are completed */
-	if (urb_priv->td_cnt == urb_priv->length) {
-		ret = 1;
-		if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
-			xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
-			if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
-				if (xhci->quirks & XHCI_AMD_PLL_FIX)
-					usb_amd_quirk_pll_enable();
-			}
-		}
+	if (last_td_in_urb(td)) {
+		if ((urb->actual_length != urb->transfer_buffer_length &&
+		     (urb->transfer_flags & URB_SHORT_NOT_OK)) ||
+		    (*status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc)))
+			xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n",
+				 urb, urb->actual_length,
+				 urb->transfer_buffer_length, *status);
+
+		/* set isoc urb status to 0 just as EHCI, UHCI, and OHCI */
+		if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
+			*status = 0;
+		xhci_giveback_urb_in_irq(xhci, td, *status);
 	}
+	return 0;
+}
 
-	return ret;
+/* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */
+static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring,
+			   union xhci_trb *stop_trb)
+{
+	u32 sum;
+	union xhci_trb *trb = ring->dequeue;
+	struct xhci_segment *seg = ring->deq_seg;
+
+	for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) {
+		if (!trb_is_noop(trb) && !trb_is_link(trb))
+			sum += TRB_LEN(le32_to_cpu(trb->generic.field[2]));
+	}
+	return sum;
 }
 
 /*
  * Process control tds, update urb status and actual_length.
  */
 static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
-	union xhci_trb *event_trb, struct xhci_transfer_event *event,
+	union xhci_trb *ep_trb, struct xhci_transfer_event *event,
 	struct xhci_virt_ep *ep, int *status)
 {
 	struct xhci_virt_device *xdev;
@@ -1932,6 +1938,8 @@ static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
 	int ep_index;
 	struct xhci_ep_ctx *ep_ctx;
 	u32 trb_comp_code;
+	u32 remaining, requested;
+	bool on_data_stage;
 
 	slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
 	xdev = xhci->devs[slot_id];
@@ -1939,195 +1947,161 @@ static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
 	ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
 	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
+	requested = td->urb->transfer_buffer_length;
+	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
+
+	/* not setup (dequeue), or status stage means we are at data stage */
+	on_data_stage = (ep_trb != ep_ring->dequeue && ep_trb != td->last_trb);
 
 	switch (trb_comp_code) {
 	case COMP_SUCCESS:
-		if (event_trb == ep_ring->dequeue) {
-			xhci_warn(xhci, "WARN: Success on ctrl setup TRB "
-					"without IOC set??\n");
-			*status = -ESHUTDOWN;
-		} else if (event_trb != td->last_trb) {
-			xhci_warn(xhci, "WARN: Success on ctrl data TRB "
-					"without IOC set??\n");
+		if (ep_trb != td->last_trb) {
+			xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n",
+				  on_data_stage ? "data" : "setup");
 			*status = -ESHUTDOWN;
-		} else {
-			*status = 0;
+			break;
 		}
+		*status = 0;
 		break;
 	case COMP_SHORT_TX:
-		if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
-			*status = -EREMOTEIO;
-		else
-			*status = 0;
+		*status = 0;
 		break;
 	case COMP_STOP_SHORT:
-		if (event_trb == ep_ring->dequeue || event_trb == td->last_trb)
-			xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n");
+		if (on_data_stage)
+			td->urb->actual_length = remaining;
 		else
-			td->urb->actual_length =
-				EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
-
-		return finish_td(xhci, td, event_trb, event, ep, status, false);
+			xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n");
+		goto finish_td;
 	case COMP_STOP:
-		/* Did we stop at data stage? */
-		if (event_trb != ep_ring->dequeue && event_trb != td->last_trb)
-			td->urb->actual_length =
-				td->urb->transfer_buffer_length -
-				EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
-		/* fall through */
+		if (on_data_stage)
+			td->urb->actual_length = requested - remaining;
+		goto finish_td;
 	case COMP_STOP_INVAL:
-		return finish_td(xhci, td, event_trb, event, ep, status, false);
+		goto finish_td;
 	default:
 		if (!xhci_requires_manual_halt_cleanup(xhci,
-					ep_ctx, trb_comp_code))
+						       ep_ctx, trb_comp_code))
 			break;
-		xhci_dbg(xhci, "TRB error code %u, "
-				"halted endpoint index = %u\n",
-				trb_comp_code, ep_index);
+		xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n",
+			 trb_comp_code, ep_index);
 		/* else fall through */
 	case COMP_STALL:
 		/* Did we transfer part of the data (middle) phase? */
-		if (event_trb != ep_ring->dequeue &&
-				event_trb != td->last_trb)
-			td->urb->actual_length =
-				td->urb->transfer_buffer_length -
-				EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
+		if (on_data_stage)
+			td->urb->actual_length = requested - remaining;
 		else if (!td->urb_length_set)
 			td->urb->actual_length = 0;
-
-		return finish_td(xhci, td, event_trb, event, ep, status, false);
+		goto finish_td;
 	}
+
+	/* stopped at setup stage, no data transferred */
+	if (ep_trb == ep_ring->dequeue)
+		goto finish_td;
+
 	/*
-	 * Did we transfer any data, despite the errors that might have
-	 * happened?  I.e. did we get past the setup stage?
+	 * if on data stage then update the actual_length of the URB and flag it
+	 * as set, so it won't be overwritten in the event for the last TRB.
 	 */
-	if (event_trb != ep_ring->dequeue) {
-		/* The event was for the status stage */
-		if (event_trb == td->last_trb) {
-			if (td->urb_length_set) {
-				/* Don't overwrite a previously set error code
-				 */
-				if ((*status == -EINPROGRESS || *status == 0) &&
-						(td->urb->transfer_flags
-						 & URB_SHORT_NOT_OK))
-					/* Did we already see a short data
-					 * stage? */
-					*status = -EREMOTEIO;
-			} else {
-				td->urb->actual_length =
-					td->urb->transfer_buffer_length;
-			}
-		} else {
-			/*
-			 * Maybe the event was for the data stage? If so, update
-			 * already the actual_length of the URB and flag it as
-			 * set, so that it is not overwritten in the event for
-			 * the last TRB.
-			 */
-			td->urb_length_set = true;
-			td->urb->actual_length =
-				td->urb->transfer_buffer_length -
-				EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
-			xhci_dbg(xhci, "Waiting for status "
-					"stage event\n");
-			return 0;
-		}
+	if (on_data_stage) {
+		td->urb_length_set = true;
+		td->urb->actual_length = requested - remaining;
+		xhci_dbg(xhci, "Waiting for status stage event\n");
+		return 0;
 	}
 
-	return finish_td(xhci, td, event_trb, event, ep, status, false);
+	/* at status stage */
+	if (!td->urb_length_set)
+		td->urb->actual_length = requested;
+
+finish_td:
+	return finish_td(xhci, td, ep_trb, event, ep, status, false);
 }
 
 /*
  * Process isochronous tds, update urb packet status and actual_length.
  */
 static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
-	union xhci_trb *event_trb, struct xhci_transfer_event *event,
+	union xhci_trb *ep_trb, struct xhci_transfer_event *event,
 	struct xhci_virt_ep *ep, int *status)
 {
 	struct xhci_ring *ep_ring;
 	struct urb_priv *urb_priv;
 	int idx;
-	int len = 0;
-	union xhci_trb *cur_trb;
-	struct xhci_segment *cur_seg;
 	struct usb_iso_packet_descriptor *frame;
 	u32 trb_comp_code;
-	bool skip_td = false;
+	bool sum_trbs_for_length = false;
+	u32 remaining, requested, ep_trb_len;
+	int short_framestatus;
 
 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
 	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
 	urb_priv = td->urb->hcpriv;
 	idx = urb_priv->td_cnt;
 	frame = &td->urb->iso_frame_desc[idx];
+	requested = frame->length;
+	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
+	ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
+	short_framestatus = td->urb->transfer_flags & URB_SHORT_NOT_OK ?
+		-EREMOTEIO : 0;
 
 	/* handle completion code */
 	switch (trb_comp_code) {
 	case COMP_SUCCESS:
-		if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) == 0) {
-			frame->status = 0;
+		if (remaining) {
+			frame->status = short_framestatus;
+			if (xhci->quirks & XHCI_TRUST_TX_LENGTH)
+				sum_trbs_for_length = true;
 			break;
 		}
-		if ((xhci->quirks & XHCI_TRUST_TX_LENGTH))
-			trb_comp_code = COMP_SHORT_TX;
-	/* fallthrough */
-	case COMP_STOP_SHORT:
+		frame->status = 0;
+		break;
 	case COMP_SHORT_TX:
-		frame->status = td->urb->transfer_flags & URB_SHORT_NOT_OK ?
-				-EREMOTEIO : 0;
+		frame->status = short_framestatus;
+		sum_trbs_for_length = true;
 		break;
 	case COMP_BW_OVER:
 		frame->status = -ECOMM;
-		skip_td = true;
 		break;
 	case COMP_BUFF_OVER:
 	case COMP_BABBLE:
 		frame->status = -EOVERFLOW;
-		skip_td = true;
 		break;
 	case COMP_DEV_ERR:
 	case COMP_STALL:
 		frame->status = -EPROTO;
-		skip_td = true;
 		break;
 	case COMP_TX_ERR:
 		frame->status = -EPROTO;
-		if (event_trb != td->last_trb)
+		if (ep_trb != td->last_trb)
 			return 0;
-		skip_td = true;
 		break;
 	case COMP_STOP:
+		sum_trbs_for_length = true;
+		break;
+	case COMP_STOP_SHORT:
+		/* field normally containing residue now contains tranferred */
+		frame->status = short_framestatus;
+		requested = remaining;
+		break;
 	case COMP_STOP_INVAL:
+		requested = 0;
+		remaining = 0;
 		break;
 	default:
+		sum_trbs_for_length = true;
 		frame->status = -1;
 		break;
 	}
 
-	if (trb_comp_code == COMP_SUCCESS || skip_td) {
-		frame->actual_length = frame->length;
-		td->urb->actual_length += frame->length;
-	} else if (trb_comp_code == COMP_STOP_SHORT) {
-		frame->actual_length =
-			EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
-		td->urb->actual_length += frame->actual_length;
-	} else {
-		for (cur_trb = ep_ring->dequeue,
-		     cur_seg = ep_ring->deq_seg; cur_trb != event_trb;
-		     next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) {
-			if (!TRB_TYPE_NOOP_LE32(cur_trb->generic.field[3]) &&
-			    !TRB_TYPE_LINK_LE32(cur_trb->generic.field[3]))
-				len += TRB_LEN(le32_to_cpu(cur_trb->generic.field[2]));
-		}
-		len += TRB_LEN(le32_to_cpu(cur_trb->generic.field[2])) -
-			EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
+	if (sum_trbs_for_length)
+		frame->actual_length = sum_trb_lengths(xhci, ep_ring, ep_trb) +
+			ep_trb_len - remaining;
+	else
+		frame->actual_length = requested;
 
-		if (trb_comp_code != COMP_STOP_INVAL) {
-			frame->actual_length = len;
-			td->urb->actual_length += len;
-		}
-	}
+	td->urb->actual_length += frame->actual_length;
 
-	return finish_td(xhci, td, event_trb, event, ep, status, false);
+	return finish_td(xhci, td, ep_trb, event, ep, status, false);
 }
 
 static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
@@ -2162,119 +2136,62 @@ static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
  * Process bulk and interrupt tds, update urb status and actual_length.
  */
 static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td,
-	union xhci_trb *event_trb, struct xhci_transfer_event *event,
+	union xhci_trb *ep_trb, struct xhci_transfer_event *event,
 	struct xhci_virt_ep *ep, int *status)
 {
 	struct xhci_ring *ep_ring;
-	union xhci_trb *cur_trb;
-	struct xhci_segment *cur_seg;
 	u32 trb_comp_code;
+	u32 remaining, requested, ep_trb_len;
 
 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
 	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
+	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
+	ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
+	requested = td->urb->transfer_buffer_length;
 
 	switch (trb_comp_code) {
 	case COMP_SUCCESS:
-		/* Double check that the HW transferred everything. */
-		if (event_trb != td->last_trb ||
-		    EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) != 0) {
-			xhci_warn(xhci, "WARN Successful completion "
-					"on short TX\n");
-			if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
-				*status = -EREMOTEIO;
-			else
-				*status = 0;
-			if ((xhci->quirks & XHCI_TRUST_TX_LENGTH))
-				trb_comp_code = COMP_SHORT_TX;
-		} else {
-			*status = 0;
+		/* handle success with untransferred data as short packet */
+		if (ep_trb != td->last_trb || remaining) {
+			xhci_warn(xhci, "WARN Successful completion on short TX\n");
+			xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
+				 td->urb->ep->desc.bEndpointAddress,
+				 requested, remaining);
 		}
+		*status = 0;
 		break;
-	case COMP_STOP_SHORT:
 	case COMP_SHORT_TX:
-		if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
-			*status = -EREMOTEIO;
-		else
-			*status = 0;
+		xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
+			 td->urb->ep->desc.bEndpointAddress,
+			 requested, remaining);
+		*status = 0;
+		break;
+	case COMP_STOP_SHORT:
+		td->urb->actual_length = remaining;
+		goto finish_td;
+	case COMP_STOP_INVAL:
+		/* stopped on ep trb with invalid length, exclude it */
+		ep_trb_len	= 0;
+		remaining	= 0;
 		break;
 	default:
-		/* Others already handled above */
+		/* do nothing */
 		break;
 	}
-	if (trb_comp_code == COMP_SHORT_TX)
-		xhci_dbg(xhci, "ep %#x - asked for %d bytes, "
-				"%d bytes untransferred\n",
-				td->urb->ep->desc.bEndpointAddress,
-				td->urb->transfer_buffer_length,
-				EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)));
-	/* Stopped - short packet completion */
-	if (trb_comp_code == COMP_STOP_SHORT) {
-		td->urb->actual_length =
-			EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
 
-		if (td->urb->transfer_buffer_length <
-				td->urb->actual_length) {
-			xhci_warn(xhci, "HC gave bad length of %d bytes txed\n",
-				EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)));
-			td->urb->actual_length = 0;
-			 /* status will be set by usb core for canceled urbs */
-		}
-	/* Fast path - was this the last TRB in the TD for this URB? */
-	} else if (event_trb == td->last_trb) {
-		if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) != 0) {
-			td->urb->actual_length =
-				td->urb->transfer_buffer_length -
-				EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
-			if (td->urb->transfer_buffer_length <
-					td->urb->actual_length) {
-				xhci_warn(xhci, "HC gave bad length "
-						"of %d bytes left\n",
-					  EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)));
-				td->urb->actual_length = 0;
-				if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
-					*status = -EREMOTEIO;
-				else
-					*status = 0;
-			}
-			/* Don't overwrite a previously set error code */
-			if (*status == -EINPROGRESS) {
-				if (td->urb->transfer_flags & URB_SHORT_NOT_OK)
-					*status = -EREMOTEIO;
-				else
-					*status = 0;
-			}
-		} else {
-			td->urb->actual_length =
-				td->urb->transfer_buffer_length;
-			/* Ignore a short packet completion if the
-			 * untransferred length was zero.
-			 */
-			if (*status == -EREMOTEIO)
-				*status = 0;
-		}
-	} else {
-		/* Slow path - walk the list, starting from the dequeue
-		 * pointer, to get the actual length transferred.
-		 */
+	if (ep_trb == td->last_trb)
+		td->urb->actual_length = requested - remaining;
+	else
+		td->urb->actual_length =
+			sum_trb_lengths(xhci, ep_ring, ep_trb) +
+			ep_trb_len - remaining;
+finish_td:
+	if (remaining > requested) {
+		xhci_warn(xhci, "bad transfer trb length %d in event trb\n",
+			  remaining);
 		td->urb->actual_length = 0;
-		for (cur_trb = ep_ring->dequeue, cur_seg = ep_ring->deq_seg;
-				cur_trb != event_trb;
-				next_trb(xhci, ep_ring, &cur_seg, &cur_trb)) {
-			if (!TRB_TYPE_NOOP_LE32(cur_trb->generic.field[3]) &&
-			    !TRB_TYPE_LINK_LE32(cur_trb->generic.field[3]))
-				td->urb->actual_length +=
-					TRB_LEN(le32_to_cpu(cur_trb->generic.field[2]));
-		}
-		/* If the ring didn't stop on a Link or No-op TRB, add
-		 * in the actual bytes transferred from the Normal TRB
-		 */
-		if (trb_comp_code != COMP_STOP_INVAL)
-			td->urb->actual_length +=
-				TRB_LEN(le32_to_cpu(cur_trb->generic.field[2])) -
-				EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
 	}
-
-	return finish_td(xhci, td, event_trb, event, ep, status, false);
+	return finish_td(xhci, td, ep_trb, event, ep, status, false);
 }
 
 /*
@@ -2293,16 +2210,13 @@ static int handle_tx_event(struct xhci_hcd *xhci,
 	unsigned int slot_id;
 	int ep_index;
 	struct xhci_td *td = NULL;
-	dma_addr_t event_dma;
-	struct xhci_segment *event_seg;
-	union xhci_trb *event_trb;
-	struct urb *urb = NULL;
+	dma_addr_t ep_trb_dma;
+	struct xhci_segment *ep_seg;
+	union xhci_trb *ep_trb;
 	int status = -EINPROGRESS;
-	struct urb_priv *urb_priv;
 	struct xhci_ep_ctx *ep_ctx;
 	struct list_head *tmp;
 	u32 trb_comp_code;
-	int ret = 0;
 	int td_num = 0;
 	bool handling_skipped_tds = false;
 
@@ -2328,9 +2242,7 @@ static int handle_tx_event(struct xhci_hcd *xhci,
 	ep = &xdev->eps[ep_index];
 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
 	ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
-	if (!ep_ring ||
-	    (le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK) ==
-	    EP_STATE_DISABLED) {
+	if (!ep_ring ||  GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) {
 		xhci_err(xhci, "ERROR Transfer event for disabled endpoint "
 				"or incorrect stream ring\n");
 		xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n",
@@ -2352,7 +2264,7 @@ static int handle_tx_event(struct xhci_hcd *xhci,
 			td_num++;
 	}
 
-	event_dma = le64_to_cpu(event->buffer);
+	ep_trb_dma = le64_to_cpu(event->buffer);
 	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
 	/* Look for common error cases */
 	switch (trb_comp_code) {
@@ -2480,7 +2392,6 @@ static int handle_tx_event(struct xhci_hcd *xhci,
 				xhci_dbg(xhci, "td_list is empty while skip "
 						"flag set. Clear skip flag.\n");
 			}
-			ret = 0;
 			goto cleanup;
 		}
 
@@ -2489,7 +2400,6 @@ static int handle_tx_event(struct xhci_hcd *xhci,
 			ep->skip = false;
 			xhci_dbg(xhci, "All tds on the ep_ring skipped. "
 						"Clear skip flag.\n");
-			ret = 0;
 			goto cleanup;
 		}
 
@@ -2498,8 +2408,8 @@ static int handle_tx_event(struct xhci_hcd *xhci,
 			td_num--;
 
 		/* Is this a TRB in the currently executing TD? */
-		event_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue,
-				td->last_trb, event_dma, false);
+		ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue,
+				td->last_trb, ep_trb_dma, false);
 
 		/*
 		 * Skip the Force Stopped Event. The event_trb(event_dma) of FSE
@@ -2509,13 +2419,12 @@ static int handle_tx_event(struct xhci_hcd *xhci,
 		 * last TRB of the previous TD. The command completion handle
 		 * will take care the rest.
 		 */
-		if (!event_seg && (trb_comp_code == COMP_STOP ||
+		if (!ep_seg && (trb_comp_code == COMP_STOP ||
 				   trb_comp_code == COMP_STOP_INVAL)) {
-			ret = 0;
 			goto cleanup;
 		}
 
-		if (!event_seg) {
+		if (!ep_seg) {
 			if (!ep->skip ||
 			    !usb_endpoint_xfer_isoc(&td->urb->ep->desc)) {
 				/* Some host controllers give a spurious
@@ -2525,7 +2434,6 @@ static int handle_tx_event(struct xhci_hcd *xhci,
 				if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) &&
 						ep_ring->last_td_was_short) {
 					ep_ring->last_td_was_short = false;
-					ret = 0;
 					goto cleanup;
 				}
 				/* HC is busted, give up! */
@@ -2536,11 +2444,11 @@ static int handle_tx_event(struct xhci_hcd *xhci,
 					trb_comp_code);
 				trb_in_td(xhci, ep_ring->deq_seg,
 					  ep_ring->dequeue, td->last_trb,
-					  event_dma, true);
+					  ep_trb_dma, true);
 				return -ESHUTDOWN;
 			}
 
-			ret = skip_isoc_td(xhci, td, event, ep, &status);
+			skip_isoc_td(xhci, td, event, ep, &status);
 			goto cleanup;
 		}
 		if (trb_comp_code == COMP_SHORT_TX)
@@ -2553,36 +2461,28 @@ static int handle_tx_event(struct xhci_hcd *xhci,
 			ep->skip = false;
 		}
 
-		event_trb = &event_seg->trbs[(event_dma - event_seg->dma) /
-						sizeof(*event_trb)];
+		ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) /
+						sizeof(*ep_trb)];
 		/*
 		 * No-op TRB should not trigger interrupts.
-		 * If event_trb is a no-op TRB, it means the
+		 * If ep_trb is a no-op TRB, it means the
 		 * corresponding TD has been cancelled. Just ignore
 		 * the TD.
 		 */
-		if (TRB_TYPE_NOOP_LE32(event_trb->generic.field[3])) {
-			xhci_dbg(xhci,
-				 "event_trb is a no-op TRB. Skip it\n");
+		if (trb_is_noop(ep_trb)) {
+			xhci_dbg(xhci, "ep_trb is a no-op TRB. Skip it\n");
 			goto cleanup;
 		}
 
-		/* Now update the urb's actual_length and give back to
-		 * the core
-		 */
+		/* update the urb's actual_length and give back to the core */
 		if (usb_endpoint_xfer_control(&td->urb->ep->desc))
-			ret = process_ctrl_td(xhci, td, event_trb, event, ep,
-						 &status);
+			process_ctrl_td(xhci, td, ep_trb, event, ep, &status);
 		else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc))
-			ret = process_isoc_td(xhci, td, event_trb, event, ep,
-						 &status);
+			process_isoc_td(xhci, td, ep_trb, event, ep, &status);
 		else
-			ret = process_bulk_intr_td(xhci, td, event_trb, event,
-						 ep, &status);
-
+			process_bulk_intr_td(xhci, td, ep_trb, event, ep,
+					     &status);
 cleanup:
-
-
 		handling_skipped_tds = ep->skip &&
 			trb_comp_code != COMP_MISSED_INT &&
 			trb_comp_code != COMP_PING_ERR;
@@ -2594,33 +2494,6 @@ cleanup:
 		if (!handling_skipped_tds)
 			inc_deq(xhci, xhci->event_ring);
 
-		if (ret) {
-			urb = td->urb;
-			urb_priv = urb->hcpriv;
-
-			xhci_urb_free_priv(urb_priv);
-
-			usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb);
-			if ((urb->actual_length != urb->transfer_buffer_length &&
-						(urb->transfer_flags &
-						 URB_SHORT_NOT_OK)) ||
-					(status != 0 &&
-					 !usb_endpoint_xfer_isoc(&urb->ep->desc)))
-				xhci_dbg(xhci, "Giveback URB %p, len = %d, "
-						"expected = %d, status = %d\n",
-						urb, urb->actual_length,
-						urb->transfer_buffer_length,
-						status);
-			spin_unlock(&xhci->lock);
-			/* EHCI, UHCI, and OHCI always unconditionally set the
-			 * urb->status of an isochronous endpoint to 0.
-			 */
-			if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
-				status = 0;
-			usb_hcd_giveback_urb(bus_to_hcd(urb->dev->bus), urb, status);
-			spin_lock(&xhci->lock);
-		}
-
 	/*
 	 * If ep->skip is set, it means there are missed tds on the
 	 * endpoint ring need to take care of.
@@ -2644,18 +2517,17 @@ static int xhci_handle_event(struct xhci_hcd *xhci)
 	int update_ptrs = 1;
 	int ret;
 
+	/* Event ring hasn't been allocated yet. */
 	if (!xhci->event_ring || !xhci->event_ring->dequeue) {
-		xhci->error_bitmask |= 1 << 1;
-		return 0;
+		xhci_err(xhci, "ERROR event ring not ready\n");
+		return -ENOMEM;
 	}
 
 	event = xhci->event_ring->dequeue;
 	/* Does the HC or OS own the TRB? */
 	if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) !=
-	    xhci->event_ring->cycle_state) {
-		xhci->error_bitmask |= 1 << 2;
+	    xhci->event_ring->cycle_state)
 		return 0;
-	}
 
 	/*
 	 * Barrier between reading the TRB_CYCLE (valid) flag above and any
@@ -2663,7 +2535,7 @@ static int xhci_handle_event(struct xhci_hcd *xhci)
 	 */
 	rmb();
 	/* FIXME: Handle more event types. */
-	switch ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK)) {
+	switch (le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) {
 	case TRB_TYPE(TRB_COMPLETION):
 		handle_cmd_completion(xhci, &event->event_cmd);
 		break;
@@ -2673,9 +2545,7 @@ static int xhci_handle_event(struct xhci_hcd *xhci)
 		break;
 	case TRB_TYPE(TRB_TRANSFER):
 		ret = handle_tx_event(xhci, &event->trans_event);
-		if (ret < 0)
-			xhci->error_bitmask |= 1 << 9;
-		else
+		if (ret >= 0)
 			update_ptrs = 0;
 		break;
 	case TRB_TYPE(TRB_DEV_NOTE):
@@ -2686,7 +2556,9 @@ static int xhci_handle_event(struct xhci_hcd *xhci)
 		    TRB_TYPE(48))
 			handle_vendor_event(xhci, event);
 		else
-			xhci->error_bitmask |= 1 << 3;
+			xhci_warn(xhci, "ERROR unknown event type %d\n",
+				  TRB_FIELD_TO_TYPE(
+				  le32_to_cpu(event->event_cmd.flags)));
 	}
 	/* Any of the above functions may drop and re-acquire the lock, so check
 	 * to make sure a watchdog timer didn't mark the host as non-responsive.
@@ -2931,8 +2803,7 @@ static int prepare_transfer(struct xhci_hcd *xhci,
 		return -EINVAL;
 	}
 
-	ret = prepare_ring(xhci, ep_ring,
-			   le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK,
+	ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
 			   num_trbs, mem_flags);
 	if (ret)
 		return ret;
@@ -3120,7 +2991,7 @@ static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred,
 	if (xhci->quirks & XHCI_MTK_HOST)
 		trb_buff_len = 0;
 
-	maxp = GET_MAX_PACKET(usb_endpoint_maxp(&urb->ep->desc));
+	maxp = usb_endpoint_maxp(&urb->ep->desc);
 	total_packet_count = DIV_ROUND_UP(td_total_len, maxp);
 
 	/* Queueing functions don't count the current TRB into transferred */
@@ -3136,7 +3007,7 @@ static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len,
 	unsigned int max_pkt;
 	u32 new_buff_len;
 
-	max_pkt = GET_MAX_PACKET(usb_endpoint_maxp(&urb->ep->desc));
+	max_pkt = usb_endpoint_maxp(&urb->ep->desc);
 	unalign = (enqd_len + *trb_buff_len) % max_pkt;
 
 	/* we got lucky, last normal TRB data on segment is packet aligned */
@@ -3650,7 +3521,7 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
 		addr = start_addr + urb->iso_frame_desc[i].offset;
 		td_len = urb->iso_frame_desc[i].length;
 		td_remain_len = td_len;
-		max_pkt = GET_MAX_PACKET(usb_endpoint_maxp(&urb->ep->desc));
+		max_pkt = usb_endpoint_maxp(&urb->ep->desc);
 		total_pkt_count = DIV_ROUND_UP(td_len, max_pkt);
 
 		/* A zero-length transfer still involves at least one packet. */
@@ -3828,7 +3699,7 @@ int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
 	/* Check the ring to guarantee there is enough room for the whole urb.
 	 * Do not insert any td of the urb to the ring if the check failed.
 	 */
-	ret = prepare_ring(xhci, ep_ring, le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK,
+	ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
 			   num_trbs, mem_flags);
 	if (ret)
 		return ret;
@@ -3841,8 +3712,7 @@ int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
 
 	/* Calculate the start frame and put it in urb->start_frame. */
 	if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) {
-		if ((le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK) ==
-				EP_STATE_RUNNING) {
+		if (GET_EP_CTX_STATE(ep_ctx) ==	EP_STATE_RUNNING) {
 			urb->start_frame = xep->next_frame_id;
 			goto skip_start_over;
 		}
diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c
index 1a4ca02729c2..1cd56417cbec 100644
--- a/drivers/usb/host/xhci.c
+++ b/drivers/usb/host/xhci.c
@@ -113,12 +113,12 @@ int xhci_halt(struct xhci_hcd *xhci)
 
 	ret = xhci_handshake(&xhci->op_regs->status,
 			STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC);
-	if (!ret) {
-		xhci->xhc_state |= XHCI_STATE_HALTED;
-		xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
-	} else
-		xhci_warn(xhci, "Host not halted after %u microseconds.\n",
-				XHCI_MAX_HALT_USEC);
+	if (ret) {
+		xhci_warn(xhci, "Host halt failed, %d\n", ret);
+		return ret;
+	}
+	xhci->xhc_state |= XHCI_STATE_HALTED;
+	xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
 	return ret;
 }
 
@@ -167,6 +167,12 @@ int xhci_reset(struct xhci_hcd *xhci)
 	int ret, i;
 
 	state = readl(&xhci->op_regs->status);
+
+	if (state == ~(u32)0) {
+		xhci_warn(xhci, "Host not accessible, reset failed.\n");
+		return -ENODEV;
+	}
+
 	if ((state & STS_HALT) == 0) {
 		xhci_warn(xhci, "Host controller not halted, aborting reset.\n");
 		return 0;
@@ -690,7 +696,6 @@ void xhci_stop(struct usb_hcd *hcd)
 		xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
 		xhci_halt(xhci);
 		xhci_reset(xhci);
-
 		spin_unlock_irq(&xhci->lock);
 	}
 
@@ -1645,8 +1650,7 @@ int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
 	/* If the HC already knows the endpoint is disabled,
 	 * or the HCD has noted it is disabled, ignore this request
 	 */
-	if (((ep_ctx->ep_info & cpu_to_le32(EP_STATE_MASK)) ==
-	     cpu_to_le32(EP_STATE_DISABLED)) ||
+	if ((GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) ||
 	    le32_to_cpu(ctrl_ctx->drop_flags) &
 	    xhci_get_endpoint_flag(&ep->desc)) {
 		/* Do not warn when called after a usb_device_reset */
@@ -3209,7 +3213,7 @@ int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev,
 
 	for (i = 0; i < num_eps; i++) {
 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
-		max_packet = GET_MAX_PACKET(usb_endpoint_maxp(&eps[i]->desc));
+		max_packet = usb_endpoint_maxp(&eps[i]->desc);
 		vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci,
 				num_stream_ctxs,
 				num_streams,
@@ -3683,27 +3687,26 @@ int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
 	int ret, slot_id;
 	struct xhci_command *command;
 
-	command = xhci_alloc_command(xhci, false, false, GFP_KERNEL);
+	command = xhci_alloc_command(xhci, false, true, GFP_KERNEL);
 	if (!command)
 		return 0;
 
 	/* xhci->slot_id and xhci->addr_dev are not thread-safe */
 	mutex_lock(&xhci->mutex);
 	spin_lock_irqsave(&xhci->lock, flags);
-	command->completion = &xhci->addr_dev;
 	ret = xhci_queue_slot_control(xhci, command, TRB_ENABLE_SLOT, 0);
 	if (ret) {
 		spin_unlock_irqrestore(&xhci->lock, flags);
 		mutex_unlock(&xhci->mutex);
 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
-		kfree(command);
+		xhci_free_command(xhci, command);
 		return 0;
 	}
 	xhci_ring_cmd_db(xhci);
 	spin_unlock_irqrestore(&xhci->lock, flags);
 
 	wait_for_completion(command->completion);
-	slot_id = xhci->slot_id;
+	slot_id = command->slot_id;
 	mutex_unlock(&xhci->mutex);
 
 	if (!slot_id || command->status != COMP_SUCCESS) {
@@ -3711,7 +3714,7 @@ int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
 		xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n",
 				HCS_MAX_SLOTS(
 					readl(&xhci->cap_regs->hcs_params1)));
-		kfree(command);
+		xhci_free_command(xhci, command);
 		return 0;
 	}
 
@@ -3747,7 +3750,7 @@ int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
 #endif
 
 
-	kfree(command);
+	xhci_free_command(xhci, command);
 	/* Is this a LS or FS device under a HS hub? */
 	/* Hub or peripherial? */
 	return 1;
@@ -3755,6 +3758,7 @@ int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
 disable_slot:
 	/* Disable slot, if we can do it without mem alloc */
 	spin_lock_irqsave(&xhci->lock, flags);
+	kfree(command->completion);
 	command->completion = NULL;
 	command->status = 0;
 	if (!xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT,
@@ -3816,14 +3820,13 @@ static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
 		}
 	}
 
-	command = xhci_alloc_command(xhci, false, false, GFP_KERNEL);
+	command = xhci_alloc_command(xhci, false, true, GFP_KERNEL);
 	if (!command) {
 		ret = -ENOMEM;
 		goto out;
 	}
 
 	command->in_ctx = virt_dev->in_ctx;
-	command->completion = &xhci->addr_dev;
 
 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
 	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
@@ -3941,7 +3944,10 @@ static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
 		       le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
 out:
 	mutex_unlock(&xhci->mutex);
-	kfree(command);
+	if (command) {
+		kfree(command->completion);
+		kfree(command);
+	}
 	return ret;
 }
 
diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h
index f945380035d0..8ccc11a974b8 100644
--- a/drivers/usb/host/xhci.h
+++ b/drivers/usb/host/xhci.h
@@ -709,6 +709,8 @@ struct xhci_ep_ctx {
 #define EP_STATE_HALTED		2
 #define EP_STATE_STOPPED	3
 #define EP_STATE_ERROR		4
+#define GET_EP_CTX_STATE(ctx)	(le32_to_cpu((ctx)->ep_info) & EP_STATE_MASK)
+
 /* Mult - Max number of burtst within an interval, in EP companion desc. */
 #define EP_MULT(p)		(((p) & 0x3) << 8)
 #define CTX_TO_EP_MULT(p)	(((p) >> 8) & 0x3)
@@ -747,11 +749,6 @@ struct xhci_ep_ctx {
 #define MAX_PACKET_MASK		(0xffff << 16)
 #define MAX_PACKET_DECODED(p)	(((p) >> 16) & 0xffff)
 
-/* Get max packet size from ep desc. Bit 10..0 specify the max packet size.
- * USB2.0 spec 9.6.6.
- */
-#define GET_MAX_PACKET(p)	((p) & 0x7ff)
-
 /* tx_info bitmasks */
 #define EP_AVG_TRB_LENGTH(p)		((p) & 0xffff)
 #define EP_MAX_ESIT_PAYLOAD_LO(p)	(((p) & 0xffff) << 16)
@@ -789,6 +786,7 @@ struct xhci_command {
 	/* Input context for changing device state */
 	struct xhci_container_ctx	*in_ctx;
 	u32				status;
+	int				slot_id;
 	/* If completion is null, no one is waiting on this command
 	 * and the structure can be freed after the command completes.
 	 */
@@ -997,7 +995,6 @@ struct xhci_virt_device {
 	int				num_rings_cached;
 #define	XHCI_MAX_RINGS_CACHED	31
 	struct xhci_virt_ep		eps[31];
-	struct completion		cmd_completion;
 	u8				fake_port;
 	u8				real_port;
 	struct xhci_interval_bw_table	*bw_table;
@@ -1583,8 +1580,6 @@ struct xhci_hcd {
 	/* slot enabling and address device helpers */
 	/* these are not thread safe so use mutex */
 	struct mutex mutex;
-	struct completion	addr_dev;
-	int slot_id;
 	/* For USB 3.0 LPM enable/disable. */
 	struct xhci_command		*lpm_command;
 	/* Internal mirror of the HW's dcbaa */
@@ -1618,8 +1613,6 @@ struct xhci_hcd {
 #define XHCI_STATE_DYING	(1 << 0)
 #define XHCI_STATE_HALTED	(1 << 1)
 #define XHCI_STATE_REMOVING	(1 << 2)
-	/* Statistics */
-	int			error_bitmask;
 	unsigned int		quirks;
 #define	XHCI_LINK_TRB_QUIRK	(1 << 0)
 #define XHCI_RESET_EP_QUIRK	(1 << 1)
diff --git a/drivers/usb/isp1760/isp1760-if.c b/drivers/usb/isp1760/isp1760-if.c
index 9535b2872183..79205b31e4a9 100644
--- a/drivers/usb/isp1760/isp1760-if.c
+++ b/drivers/usb/isp1760/isp1760-if.c
@@ -197,7 +197,7 @@ static int isp1760_plat_probe(struct platform_device *pdev)
 
 	irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
 	if (!irq_res) {
-		pr_warning("isp1760: IRQ resource not available\n");
+		pr_warn("isp1760: IRQ resource not available\n");
 		return -ENODEV;
 	}
 	irqflags = irq_res->flags & IRQF_TRIGGER_MASK;
diff --git a/drivers/usb/misc/chaoskey.c b/drivers/usb/misc/chaoskey.c
index 6ddd08a32777..aa350dc9eb25 100644
--- a/drivers/usb/misc/chaoskey.c
+++ b/drivers/usb/misc/chaoskey.c
@@ -215,19 +215,7 @@ static int chaoskey_probe(struct usb_interface *interface,
 
 	dev->hwrng.name = dev->name ? dev->name : chaoskey_driver.name;
 	dev->hwrng.read = chaoskey_rng_read;
-
-	/* Set the 'quality' metric.  Quality is measured in units of
-	 * 1/1024's of a bit ("mills"). This should be set to 1024,
-	 * but there is a bug in the hwrng core which masks it with
-	 * 1023.
-	 *
-	 * The patch that has been merged to the crypto development
-	 * tree for that bug limits the value to 1024 at most, so by
-	 * setting this to 1024 + 1023, we get 1023 before the fix is
-	 * merged and 1024 afterwards. We'll patch this driver once
-	 * both bits of code are in the same tree.
-	 */
-	dev->hwrng.quality = 1024 + 1023;
+	dev->hwrng.quality = 1024;
 
 	dev->hwrng_registered = (hwrng_register(&dev->hwrng) == 0);
 	if (!dev->hwrng_registered)
diff --git a/drivers/usb/misc/rio500.c b/drivers/usb/misc/rio500.c
index 13731d512624..fc329c98a6e8 100644
--- a/drivers/usb/misc/rio500.c
+++ b/drivers/usb/misc/rio500.c
@@ -421,7 +421,7 @@ read_rio(struct file *file, char __user *buffer, size_t count, loff_t * ppos)
 		} else if (result != -EREMOTEIO) {
 			mutex_unlock(&(rio->lock));
 			dev_err(&rio->rio_dev->dev,
-				"Read Whoops - result:%u partial:%u this_read:%u\n",
+				"Read Whoops - result:%d partial:%u this_read:%u\n",
 				result, partial, this_read);
 			return -EIO;
 		} else {
diff --git a/drivers/usb/misc/sisusbvga/sisusb_con.c b/drivers/usb/misc/sisusbvga/sisusb_con.c
index 460cebf322e3..4b5777ec1501 100644
--- a/drivers/usb/misc/sisusbvga/sisusb_con.c
+++ b/drivers/usb/misc/sisusbvga/sisusb_con.c
@@ -686,8 +686,6 @@ static void
 sisusbcon_scrolldelta(struct vc_data *c, int lines)
 {
 	struct sisusb_usb_data *sisusb;
-	int margin = c->vc_size_row * 4;
-	int ul, we, p, st;
 
 	sisusb = sisusb_get_sisusb_lock_and_check(c->vc_num);
 	if (!sisusb)
@@ -700,39 +698,8 @@ sisusbcon_scrolldelta(struct vc_data *c, int lines)
 		return;
 	}
 
-	if (!lines)		/* Turn scrollback off */
-		c->vc_visible_origin = c->vc_origin;
-	else {
-
-		if (sisusb->con_rolled_over >
-				(c->vc_scr_end - sisusb->scrbuf) + margin) {
-
-			ul = c->vc_scr_end - sisusb->scrbuf;
-			we = sisusb->con_rolled_over + c->vc_size_row;
-
-		} else {
-
-			ul = 0;
-			we = sisusb->scrbuf_size;
-
-		}
-
-		p = (c->vc_visible_origin - sisusb->scrbuf - ul + we) % we +
-				lines * c->vc_size_row;
-
-		st = (c->vc_origin - sisusb->scrbuf - ul + we) % we;
-
-		if (st < 2 * margin)
-			margin = 0;
-
-		if (p < margin)
-			p = 0;
-
-		if (p > st - margin)
-			p = st;
-
-		c->vc_visible_origin = sisusb->scrbuf + (p + ul) % we;
-	}
+	vc_scrolldelta_helper(c, lines, sisusb->con_rolled_over,
+			(void *)sisusb->scrbuf, sisusb->scrbuf_size);
 
 	sisusbcon_set_start_address(sisusb, c);
 
@@ -808,9 +775,10 @@ sisusbcon_cursor(struct vc_data *c, int mode)
 	mutex_unlock(&sisusb->lock);
 }
 
-static int
+static bool
 sisusbcon_scroll_area(struct vc_data *c, struct sisusb_usb_data *sisusb,
-					int t, int b, int dir, int lines)
+		unsigned int t, unsigned int b, enum con_scroll dir,
+		unsigned int lines)
 {
 	int cols = sisusb->sisusb_num_columns;
 	int length = ((b - t) * cols) * 2;
@@ -852,8 +820,9 @@ sisusbcon_scroll_area(struct vc_data *c, struct sisusb_usb_data *sisusb,
 }
 
 /* Interface routine */
-static int
-sisusbcon_scroll(struct vc_data *c, int t, int b, int dir, int lines)
+static bool
+sisusbcon_scroll(struct vc_data *c, unsigned int t, unsigned int b,
+		enum con_scroll dir, unsigned int lines)
 {
 	struct sisusb_usb_data *sisusb;
 	u16 eattr = c->vc_video_erase_char;
@@ -870,17 +839,17 @@ sisusbcon_scroll(struct vc_data *c, int t, int b, int dir, int lines)
 	 */
 
 	if (!lines)
-		return 1;
+		return true;
 
 	sisusb = sisusb_get_sisusb_lock_and_check(c->vc_num);
 	if (!sisusb)
-		return 0;
+		return false;
 
 	/* sisusb->lock is down */
 
 	if (sisusb_is_inactive(c, sisusb)) {
 		mutex_unlock(&sisusb->lock);
-		return 0;
+		return false;
 	}
 
 	/* Special case */
@@ -971,7 +940,7 @@ sisusbcon_scroll(struct vc_data *c, int t, int b, int dir, int lines)
 
 	mutex_unlock(&sisusb->lock);
 
-	return 1;
+	return true;
 }
 
 /* Interface routine */
diff --git a/drivers/usb/misc/usbtest.c b/drivers/usb/misc/usbtest.c
index 5c8210dc6fd9..3525626bf086 100644
--- a/drivers/usb/misc/usbtest.c
+++ b/drivers/usb/misc/usbtest.c
@@ -1915,7 +1915,7 @@ static struct urb *iso_alloc_urb(
 	if (bytes < 0 || !desc)
 		return NULL;
 	maxp = 0x7ff & usb_endpoint_maxp(desc);
-	maxp *= 1 + (0x3 & (usb_endpoint_maxp(desc) >> 11));
+	maxp *= usb_endpoint_maxp_mult(desc);
 	packets = DIV_ROUND_UP(bytes, maxp);
 
 	urb = usb_alloc_urb(packets, GFP_KERNEL);
@@ -2001,8 +2001,8 @@ test_queue(struct usbtest_dev *dev, struct usbtest_param_32 *param,
 			"iso period %d %sframes, wMaxPacket %d, transactions: %d\n",
 			1 << (desc->bInterval - 1),
 			(udev->speed == USB_SPEED_HIGH) ? "micro" : "",
-			usb_endpoint_maxp(desc) & 0x7ff,
-			1 + (0x3 & (usb_endpoint_maxp(desc) >> 11)));
+			usb_endpoint_maxp(desc),
+			usb_endpoint_maxp_mult(desc));
 
 		dev_info(&dev->intf->dev,
 			"total %lu msec (%lu packets)\n",
diff --git a/drivers/usb/mtu3/Kconfig b/drivers/usb/mtu3/Kconfig
new file mode 100644
index 000000000000..25cd61947bee
--- /dev/null
+++ b/drivers/usb/mtu3/Kconfig
@@ -0,0 +1,54 @@
+# For MTK USB3.0 IP
+
+config USB_MTU3
+	tristate "MediaTek USB3 Dual Role controller"
+	depends on EXTCON && (USB || USB_GADGET) && HAS_DMA
+	depends on ARCH_MEDIATEK || COMPILE_TEST
+	select USB_XHCI_MTK if USB_SUPPORT && USB_XHCI_HCD
+	help
+	  Say Y or M here if your system runs on MediaTek SoCs with
+	  Dual Role SuperSpeed USB controller. You can select usb
+	  mode as peripheral role or host role, or both.
+
+	  If you don't know what this is, please say N.
+
+	  Choose M here to compile this driver as a module, and it
+	  will be called mtu3.ko.
+
+
+if USB_MTU3
+choice
+	bool "MTU3 Mode Selection"
+	default USB_MTU3_DUAL_ROLE if (USB && USB_GADGET)
+	default USB_MTU3_HOST if (USB && !USB_GADGET)
+	default USB_MTU3_GADGET if (!USB && USB_GADGET)
+
+config USB_MTU3_HOST
+	bool "Host only mode"
+	depends on USB=y || USB=USB_MTU3
+	help
+	  Select this when you want to use MTU3 in host mode only,
+	  thereby the gadget feature will be regressed.
+
+config USB_MTU3_GADGET
+	bool "Gadget only mode"
+	depends on USB_GADGET=y || USB_GADGET=USB_MTU3
+	help
+	  Select this when you want to use MTU3 in gadget mode only,
+	  thereby the host feature will be regressed.
+
+config USB_MTU3_DUAL_ROLE
+	bool "Dual Role mode"
+	depends on ((USB=y || USB=USB_MTU3) && (USB_GADGET=y || USB_GADGET=USB_MTU3))
+	help
+	  This is the default mode of working of MTU3 controller where
+	  both host and gadget features are enabled.
+
+endchoice
+
+config USB_MTU3_DEBUG
+	bool "Enable Debugging Messages"
+	help
+	  Say Y here to enable debugging messages in the MTU3 Driver.
+
+endif
diff --git a/drivers/usb/mtu3/Makefile b/drivers/usb/mtu3/Makefile
new file mode 100644
index 000000000000..60e0fff7a847
--- /dev/null
+++ b/drivers/usb/mtu3/Makefile
@@ -0,0 +1,18 @@
+
+ccflags-$(CONFIG_USB_MTU3_DEBUG)	+= -DDEBUG
+
+obj-$(CONFIG_USB_MTU3)	+= mtu3.o
+
+mtu3-y	:= mtu3_plat.o
+
+ifneq ($(filter y,$(CONFIG_USB_MTU3_HOST) $(CONFIG_USB_MTU3_DUAL_ROLE)),)
+	mtu3-y	+= mtu3_host.o
+endif
+
+ifneq ($(filter y,$(CONFIG_USB_MTU3_GADGET) $(CONFIG_USB_MTU3_DUAL_ROLE)),)
+	mtu3-y	+= mtu3_core.o mtu3_gadget_ep0.o mtu3_gadget.o mtu3_qmu.o
+endif
+
+ifneq ($(CONFIG_USB_MTU3_DUAL_ROLE),)
+	mtu3-y	+= mtu3_dr.o
+endif
diff --git a/drivers/usb/mtu3/mtu3.h b/drivers/usb/mtu3/mtu3.h
new file mode 100644
index 000000000000..ba9df719f363
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3.h
@@ -0,0 +1,417 @@
+/*
+ * mtu3.h - MediaTek USB3 DRD header
+ *
+ * Copyright (C) 2016 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef __MTU3_H__
+#define __MTU3_H__
+
+#include <linux/device.h>
+#include <linux/dmapool.h>
+#include <linux/extcon.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/phy/phy.h>
+#include <linux/regulator/consumer.h>
+#include <linux/usb.h>
+#include <linux/usb/ch9.h>
+#include <linux/usb/gadget.h>
+#include <linux/usb/otg.h>
+
+struct mtu3;
+struct mtu3_ep;
+struct mtu3_request;
+
+#include "mtu3_hw_regs.h"
+#include "mtu3_qmu.h"
+
+#define	MU3D_EP_TXCR0(epnum)	(U3D_TX1CSR0 + (((epnum) - 1) * 0x10))
+#define	MU3D_EP_TXCR1(epnum)	(U3D_TX1CSR1 + (((epnum) - 1) * 0x10))
+#define	MU3D_EP_TXCR2(epnum)	(U3D_TX1CSR2 + (((epnum) - 1) * 0x10))
+
+#define	MU3D_EP_RXCR0(epnum)	(U3D_RX1CSR0 + (((epnum) - 1) * 0x10))
+#define	MU3D_EP_RXCR1(epnum)	(U3D_RX1CSR1 + (((epnum) - 1) * 0x10))
+#define	MU3D_EP_RXCR2(epnum)	(U3D_RX1CSR2 + (((epnum) - 1) * 0x10))
+
+#define USB_QMU_RQCSR(epnum)	(U3D_RXQCSR1 + (((epnum) - 1) * 0x10))
+#define USB_QMU_RQSAR(epnum)	(U3D_RXQSAR1 + (((epnum) - 1) * 0x10))
+#define USB_QMU_RQCPR(epnum)	(U3D_RXQCPR1 + (((epnum) - 1) * 0x10))
+
+#define USB_QMU_TQCSR(epnum)	(U3D_TXQCSR1 + (((epnum) - 1) * 0x10))
+#define USB_QMU_TQSAR(epnum)	(U3D_TXQSAR1 + (((epnum) - 1) * 0x10))
+#define USB_QMU_TQCPR(epnum)	(U3D_TXQCPR1 + (((epnum) - 1) * 0x10))
+
+#define SSUSB_U3_CTRL(p)	(U3D_SSUSB_U3_CTRL_0P + ((p) * 0x08))
+#define SSUSB_U2_CTRL(p)	(U3D_SSUSB_U2_CTRL_0P + ((p) * 0x08))
+
+#define MTU3_DRIVER_NAME	"mtu3"
+#define	DMA_ADDR_INVALID	(~(dma_addr_t)0)
+
+#define MTU3_EP_ENABLED		BIT(0)
+#define MTU3_EP_STALL		BIT(1)
+#define MTU3_EP_WEDGE		BIT(2)
+#define MTU3_EP_BUSY		BIT(3)
+
+#define MTU3_U3_IP_SLOT_DEFAULT 2
+#define MTU3_U2_IP_SLOT_DEFAULT 1
+
+/**
+ * Normally the device works on HS or SS, to simplify fifo management,
+ * devide fifo into some 512B parts, use bitmap to manage it; And
+ * 128 bits size of bitmap is large enough, that means it can manage
+ * up to 64KB fifo size.
+ * NOTE: MTU3_EP_FIFO_UNIT should be power of two
+ */
+#define MTU3_EP_FIFO_UNIT		(1 << 9)
+#define MTU3_FIFO_BIT_SIZE		128
+#define MTU3_U2_IP_EP0_FIFO_SIZE	64
+
+/**
+ * Maximum size of ep0 response buffer for ch9 requests,
+ * the SET_SEL request uses 6 so far, and GET_STATUS is 2
+ */
+#define EP0_RESPONSE_BUF  6
+
+/* device operated link and speed got from DEVICE_CONF register */
+enum mtu3_speed {
+	MTU3_SPEED_INACTIVE = 0,
+	MTU3_SPEED_FULL = 1,
+	MTU3_SPEED_HIGH = 3,
+	MTU3_SPEED_SUPER = 4,
+};
+
+/**
+ * @MU3D_EP0_STATE_SETUP: waits for SETUP or received a SETUP
+ *		without data stage.
+ * @MU3D_EP0_STATE_TX: IN data stage
+ * @MU3D_EP0_STATE_RX: OUT data stage
+ * @MU3D_EP0_STATE_TX_END: the last IN data is transferred, and
+ *		waits for its completion interrupt
+ * @MU3D_EP0_STATE_STALL: ep0 is in stall status, will be auto-cleared
+ *		after receives a SETUP.
+ */
+enum mtu3_g_ep0_state {
+	MU3D_EP0_STATE_SETUP = 1,
+	MU3D_EP0_STATE_TX,
+	MU3D_EP0_STATE_RX,
+	MU3D_EP0_STATE_TX_END,
+	MU3D_EP0_STATE_STALL,
+};
+
+/**
+ * @base: the base address of fifo
+ * @limit: the bitmap size in bits
+ * @bitmap: fifo bitmap in unit of @MTU3_EP_FIFO_UNIT
+ */
+struct mtu3_fifo_info {
+	u32 base;
+	u32 limit;
+	DECLARE_BITMAP(bitmap, MTU3_FIFO_BIT_SIZE);
+};
+
+/**
+ * General Purpose Descriptor (GPD):
+ *	The format of TX GPD is a little different from RX one.
+ *	And the size of GPD is 16 bytes.
+ *
+ * @flag:
+ *	bit0: Hardware Own (HWO)
+ *	bit1: Buffer Descriptor Present (BDP), always 0, BD is not supported
+ *	bit2: Bypass (BPS), 1: HW skips this GPD if HWO = 1
+ *	bit7: Interrupt On Completion (IOC)
+ * @chksum: This is used to validate the contents of this GPD;
+ *	If TXQ_CS_EN / RXQ_CS_EN bit is set, an interrupt is issued
+ *	when checksum validation fails;
+ *	Checksum value is calculated over the 16 bytes of the GPD by default;
+ * @data_buf_len (RX ONLY): This value indicates the length of
+ *	the assigned data buffer
+ * @next_gpd: Physical address of the next GPD
+ * @buffer: Physical address of the data buffer
+ * @buf_len:
+ *	(TX): This value indicates the length of the assigned data buffer
+ *	(RX): The total length of data received
+ * @ext_len: reserved
+ * @ext_flag:
+ *	bit5 (TX ONLY): Zero Length Packet (ZLP),
+ */
+struct qmu_gpd {
+	__u8 flag;
+	__u8 chksum;
+	__le16 data_buf_len;
+	__le32 next_gpd;
+	__le32 buffer;
+	__le16 buf_len;
+	__u8 ext_len;
+	__u8 ext_flag;
+} __packed;
+
+/**
+* dma: physical base address of GPD segment
+* start: virtual base address of GPD segment
+* end: the last GPD element
+* enqueue: the first empty GPD to use
+* dequeue: the first completed GPD serviced by ISR
+* NOTE: the size of GPD ring should be >= 2
+*/
+struct mtu3_gpd_ring {
+	dma_addr_t dma;
+	struct qmu_gpd *start;
+	struct qmu_gpd *end;
+	struct qmu_gpd *enqueue;
+	struct qmu_gpd *dequeue;
+};
+
+/**
+* @vbus: vbus 5V used by host mode
+* @edev: external connector used to detect vbus and iddig changes
+* @vbus_nb: notifier for vbus detection
+* @vbus_nb: notifier for iddig(idpin) detection
+* @extcon_reg_dwork: delay work for extcon notifier register, waiting for
+*		xHCI driver initialization, it's necessary for system bootup
+*		as device.
+* @is_u3_drd: whether port0 supports usb3.0 dual-role device or not
+* @id_*: used to maually switch between host and device modes by idpin
+* @manual_drd_enabled: it's true when supports dual-role device by debugfs
+*		to switch host/device modes depending on user input.
+*/
+struct otg_switch_mtk {
+	struct regulator *vbus;
+	struct extcon_dev *edev;
+	struct notifier_block vbus_nb;
+	struct notifier_block id_nb;
+	struct delayed_work extcon_reg_dwork;
+	bool is_u3_drd;
+	/* dual-role switch by debugfs */
+	struct pinctrl *id_pinctrl;
+	struct pinctrl_state *id_float;
+	struct pinctrl_state *id_ground;
+	bool manual_drd_enabled;
+};
+
+/**
+ * @mac_base: register base address of device MAC, exclude xHCI's
+ * @ippc_base: register base address of IP Power and Clock interface (IPPC)
+ * @vusb33: usb3.3V shared by device/host IP
+ * @sys_clk: system clock of mtu3, shared by device/host IP
+ * @dr_mode: works in which mode:
+ *		host only, device only or dual-role mode
+ * @u2_ports: number of usb2.0 host ports
+ * @u3_ports: number of usb3.0 host ports
+ * @dbgfs_root: only used when supports manual dual-role switch via debugfs
+ * @wakeup_en: it's true when supports remote wakeup in host mode
+ * @wk_deb_p0: port0's wakeup debounce clock
+ * @wk_deb_p1: it's optional, and depends on port1 is supported or not
+ */
+struct ssusb_mtk {
+	struct device *dev;
+	struct mtu3 *u3d;
+	void __iomem *mac_base;
+	void __iomem *ippc_base;
+	struct phy **phys;
+	int num_phys;
+	/* common power & clock */
+	struct regulator *vusb33;
+	struct clk *sys_clk;
+	/* otg */
+	struct otg_switch_mtk otg_switch;
+	enum usb_dr_mode dr_mode;
+	bool is_host;
+	int u2_ports;
+	int u3_ports;
+	struct dentry *dbgfs_root;
+	/* usb wakeup for host mode */
+	bool wakeup_en;
+	struct clk *wk_deb_p0;
+	struct clk *wk_deb_p1;
+	struct regmap *pericfg;
+};
+
+/**
+ * @fifo_size: it is (@slot + 1) * @fifo_seg_size
+ * @fifo_seg_size: it is roundup_pow_of_two(@maxp)
+ */
+struct mtu3_ep {
+	struct usb_ep ep;
+	char name[12];
+	struct mtu3 *mtu;
+	u8 epnum;
+	u8 type;
+	u8 is_in;
+	u16 maxp;
+	int slot;
+	u32 fifo_size;
+	u32 fifo_addr;
+	u32 fifo_seg_size;
+	struct mtu3_fifo_info *fifo;
+
+	struct list_head req_list;
+	struct mtu3_gpd_ring gpd_ring;
+	const struct usb_ss_ep_comp_descriptor *comp_desc;
+	const struct usb_endpoint_descriptor *desc;
+
+	int flags;
+	u8 wedged;
+	u8 busy;
+};
+
+struct mtu3_request {
+	struct usb_request request;
+	struct list_head list;
+	struct mtu3_ep *mep;
+	struct mtu3 *mtu;
+	struct qmu_gpd *gpd;
+	int epnum;
+};
+
+static inline struct ssusb_mtk *dev_to_ssusb(struct device *dev)
+{
+	return dev_get_drvdata(dev);
+}
+
+/**
+ * struct mtu3 - device driver instance data.
+ * @slot: MTU3_U2_IP_SLOT_DEFAULT for U2 IP only,
+ *		MTU3_U3_IP_SLOT_DEFAULT for U3 IP
+ * @may_wakeup: means device's remote wakeup is enabled
+ * @is_self_powered: is reported in device status and the config descriptor
+ * @ep0_req: dummy request used while handling standard USB requests
+ *		for GET_STATUS and SET_SEL
+ * @setup_buf: ep0 response buffer for GET_STATUS and SET_SEL requests
+ */
+struct mtu3 {
+	spinlock_t lock;
+	struct ssusb_mtk *ssusb;
+	struct device *dev;
+	void __iomem *mac_base;
+	void __iomem *ippc_base;
+	int irq;
+
+	struct mtu3_fifo_info tx_fifo;
+	struct mtu3_fifo_info rx_fifo;
+
+	struct mtu3_ep *ep_array;
+	struct mtu3_ep *in_eps;
+	struct mtu3_ep *out_eps;
+	struct mtu3_ep *ep0;
+	int num_eps;
+	int slot;
+	int active_ep;
+
+	struct dma_pool	*qmu_gpd_pool;
+	enum mtu3_g_ep0_state ep0_state;
+	struct usb_gadget g;	/* the gadget */
+	struct usb_gadget_driver *gadget_driver;
+	struct mtu3_request ep0_req;
+	u8 setup_buf[EP0_RESPONSE_BUF];
+	u32 max_speed;
+
+	unsigned is_active:1;
+	unsigned may_wakeup:1;
+	unsigned is_self_powered:1;
+	unsigned test_mode:1;
+	unsigned softconnect:1;
+	unsigned u1_enable:1;
+	unsigned u2_enable:1;
+	unsigned is_u3_ip:1;
+
+	u8 address;
+	u8 test_mode_nr;
+	u32 hw_version;
+};
+
+static inline struct mtu3 *gadget_to_mtu3(struct usb_gadget *g)
+{
+	return container_of(g, struct mtu3, g);
+}
+
+static inline int is_first_entry(const struct list_head *list,
+	const struct list_head *head)
+{
+	return list_is_last(head, list);
+}
+
+static inline struct mtu3_request *to_mtu3_request(struct usb_request *req)
+{
+	return req ? container_of(req, struct mtu3_request, request) : NULL;
+}
+
+static inline struct mtu3_ep *to_mtu3_ep(struct usb_ep *ep)
+{
+	return ep ? container_of(ep, struct mtu3_ep, ep) : NULL;
+}
+
+static inline struct mtu3_request *next_request(struct mtu3_ep *mep)
+{
+	struct list_head *queue = &mep->req_list;
+
+	if (list_empty(queue))
+		return NULL;
+
+	return list_first_entry(queue, struct mtu3_request, list);
+}
+
+static inline void mtu3_writel(void __iomem *base, u32 offset, u32 data)
+{
+	writel(data, base + offset);
+}
+
+static inline u32 mtu3_readl(void __iomem *base, u32 offset)
+{
+	return readl(base + offset);
+}
+
+static inline void mtu3_setbits(void __iomem *base, u32 offset, u32 bits)
+{
+	void __iomem *addr = base + offset;
+	u32 tmp = readl(addr);
+
+	writel((tmp | (bits)), addr);
+}
+
+static inline void mtu3_clrbits(void __iomem *base, u32 offset, u32 bits)
+{
+	void __iomem *addr = base + offset;
+	u32 tmp = readl(addr);
+
+	writel((tmp & ~(bits)), addr);
+}
+
+int ssusb_check_clocks(struct ssusb_mtk *ssusb, u32 ex_clks);
+struct usb_request *mtu3_alloc_request(struct usb_ep *ep, gfp_t gfp_flags);
+void mtu3_free_request(struct usb_ep *ep, struct usb_request *req);
+void mtu3_req_complete(struct mtu3_ep *mep,
+		struct usb_request *req, int status);
+
+int mtu3_config_ep(struct mtu3 *mtu, struct mtu3_ep *mep,
+		int interval, int burst, int mult);
+void mtu3_deconfig_ep(struct mtu3 *mtu, struct mtu3_ep *mep);
+void mtu3_ep_stall_set(struct mtu3_ep *mep, bool set);
+void mtu3_ep0_setup(struct mtu3 *mtu);
+void mtu3_start(struct mtu3 *mtu);
+void mtu3_stop(struct mtu3 *mtu);
+void mtu3_dev_on_off(struct mtu3 *mtu, int is_on);
+
+int mtu3_gadget_setup(struct mtu3 *mtu);
+void mtu3_gadget_cleanup(struct mtu3 *mtu);
+void mtu3_gadget_reset(struct mtu3 *mtu);
+void mtu3_gadget_suspend(struct mtu3 *mtu);
+void mtu3_gadget_resume(struct mtu3 *mtu);
+void mtu3_gadget_disconnect(struct mtu3 *mtu);
+
+irqreturn_t mtu3_ep0_isr(struct mtu3 *mtu);
+extern const struct usb_ep_ops mtu3_ep0_ops;
+
+#endif
diff --git a/drivers/usb/mtu3/mtu3_core.c b/drivers/usb/mtu3/mtu3_core.c
new file mode 100644
index 000000000000..99c65b0788ff
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_core.c
@@ -0,0 +1,863 @@
+/*
+ * mtu3_core.c - hardware access layer and gadget init/exit of
+ *                     MediaTek usb3 Dual-Role Controller Driver
+ *
+ * Copyright (C) 2016 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+
+#include "mtu3.h"
+
+static int ep_fifo_alloc(struct mtu3_ep *mep, u32 seg_size)
+{
+	struct mtu3_fifo_info *fifo = mep->fifo;
+	u32 num_bits = DIV_ROUND_UP(seg_size, MTU3_EP_FIFO_UNIT);
+	u32 start_bit;
+
+	/* ensure that @mep->fifo_seg_size is power of two */
+	num_bits = roundup_pow_of_two(num_bits);
+	if (num_bits > fifo->limit)
+		return -EINVAL;
+
+	mep->fifo_seg_size = num_bits * MTU3_EP_FIFO_UNIT;
+	num_bits = num_bits * (mep->slot + 1);
+	start_bit = bitmap_find_next_zero_area(fifo->bitmap,
+			fifo->limit, 0, num_bits, 0);
+	if (start_bit >= fifo->limit)
+		return -EOVERFLOW;
+
+	bitmap_set(fifo->bitmap, start_bit, num_bits);
+	mep->fifo_size = num_bits * MTU3_EP_FIFO_UNIT;
+	mep->fifo_addr = fifo->base + MTU3_EP_FIFO_UNIT * start_bit;
+
+	dev_dbg(mep->mtu->dev, "%s fifo:%#x/%#x, start_bit: %d\n",
+		__func__, mep->fifo_seg_size, mep->fifo_size, start_bit);
+
+	return mep->fifo_addr;
+}
+
+static void ep_fifo_free(struct mtu3_ep *mep)
+{
+	struct mtu3_fifo_info *fifo = mep->fifo;
+	u32 addr = mep->fifo_addr;
+	u32 bits = mep->fifo_size / MTU3_EP_FIFO_UNIT;
+	u32 start_bit;
+
+	if (unlikely(addr < fifo->base || bits > fifo->limit))
+		return;
+
+	start_bit = (addr - fifo->base) / MTU3_EP_FIFO_UNIT;
+	bitmap_clear(fifo->bitmap, start_bit, bits);
+	mep->fifo_size = 0;
+	mep->fifo_seg_size = 0;
+
+	dev_dbg(mep->mtu->dev, "%s size:%#x/%#x, start_bit: %d\n",
+		__func__, mep->fifo_seg_size, mep->fifo_size, start_bit);
+}
+
+/* enable/disable U3D SS function */
+static inline void mtu3_ss_func_set(struct mtu3 *mtu, bool enable)
+{
+	/* If usb3_en==0, LTSSM will go to SS.Disable state */
+	if (enable)
+		mtu3_setbits(mtu->mac_base, U3D_USB3_CONFIG, USB3_EN);
+	else
+		mtu3_clrbits(mtu->mac_base, U3D_USB3_CONFIG, USB3_EN);
+
+	dev_dbg(mtu->dev, "USB3_EN = %d\n", !!enable);
+}
+
+/* set/clear U3D HS device soft connect */
+static inline void mtu3_hs_softconn_set(struct mtu3 *mtu, bool enable)
+{
+	if (enable) {
+		mtu3_setbits(mtu->mac_base, U3D_POWER_MANAGEMENT,
+			SOFT_CONN | SUSPENDM_ENABLE);
+	} else {
+		mtu3_clrbits(mtu->mac_base, U3D_POWER_MANAGEMENT,
+			SOFT_CONN | SUSPENDM_ENABLE);
+	}
+	dev_dbg(mtu->dev, "SOFTCONN = %d\n", !!enable);
+}
+
+/* only port0 of U2/U3 supports device mode */
+static int mtu3_device_enable(struct mtu3 *mtu)
+{
+	void __iomem *ibase = mtu->ippc_base;
+	u32 check_clk = 0;
+
+	mtu3_clrbits(ibase, U3D_SSUSB_IP_PW_CTRL2, SSUSB_IP_DEV_PDN);
+
+	if (mtu->is_u3_ip) {
+		check_clk = SSUSB_U3_MAC_RST_B_STS;
+		mtu3_clrbits(ibase, SSUSB_U3_CTRL(0),
+			(SSUSB_U3_PORT_DIS | SSUSB_U3_PORT_PDN |
+			SSUSB_U3_PORT_HOST_SEL));
+	}
+	mtu3_clrbits(ibase, SSUSB_U2_CTRL(0),
+		(SSUSB_U2_PORT_DIS | SSUSB_U2_PORT_PDN |
+		SSUSB_U2_PORT_HOST_SEL));
+	mtu3_setbits(ibase, SSUSB_U2_CTRL(0), SSUSB_U2_PORT_OTG_SEL);
+
+	return ssusb_check_clocks(mtu->ssusb, check_clk);
+}
+
+static void mtu3_device_disable(struct mtu3 *mtu)
+{
+	void __iomem *ibase = mtu->ippc_base;
+
+	if (mtu->is_u3_ip)
+		mtu3_setbits(ibase, SSUSB_U3_CTRL(0),
+			(SSUSB_U3_PORT_DIS | SSUSB_U3_PORT_PDN));
+
+	mtu3_setbits(ibase, SSUSB_U2_CTRL(0),
+		SSUSB_U2_PORT_DIS | SSUSB_U2_PORT_PDN);
+	mtu3_clrbits(ibase, SSUSB_U2_CTRL(0), SSUSB_U2_PORT_OTG_SEL);
+	mtu3_setbits(ibase, U3D_SSUSB_IP_PW_CTRL2, SSUSB_IP_DEV_PDN);
+}
+
+/* reset U3D's device module. */
+static void mtu3_device_reset(struct mtu3 *mtu)
+{
+	void __iomem *ibase = mtu->ippc_base;
+
+	mtu3_setbits(ibase, U3D_SSUSB_DEV_RST_CTRL, SSUSB_DEV_SW_RST);
+	udelay(1);
+	mtu3_clrbits(ibase, U3D_SSUSB_DEV_RST_CTRL, SSUSB_DEV_SW_RST);
+}
+
+/* disable all interrupts */
+static void mtu3_intr_disable(struct mtu3 *mtu)
+{
+	void __iomem *mbase = mtu->mac_base;
+
+	/* Disable level 1 interrupts */
+	mtu3_writel(mbase, U3D_LV1IECR, ~0x0);
+	/* Disable endpoint interrupts */
+	mtu3_writel(mbase, U3D_EPIECR, ~0x0);
+}
+
+static void mtu3_intr_status_clear(struct mtu3 *mtu)
+{
+	void __iomem *mbase = mtu->mac_base;
+
+	/* Clear EP0 and Tx/Rx EPn interrupts status */
+	mtu3_writel(mbase, U3D_EPISR, ~0x0);
+	/* Clear U2 USB common interrupts status */
+	mtu3_writel(mbase, U3D_COMMON_USB_INTR, ~0x0);
+	/* Clear U3 LTSSM interrupts status */
+	mtu3_writel(mbase, U3D_LTSSM_INTR, ~0x0);
+	/* Clear speed change interrupt status */
+	mtu3_writel(mbase, U3D_DEV_LINK_INTR, ~0x0);
+}
+
+/* enable system global interrupt */
+static void mtu3_intr_enable(struct mtu3 *mtu)
+{
+	void __iomem *mbase = mtu->mac_base;
+	u32 value;
+
+	/*Enable level 1 interrupts (BMU, QMU, MAC3, DMA, MAC2, EPCTL) */
+	value = BMU_INTR | QMU_INTR | MAC3_INTR | MAC2_INTR | EP_CTRL_INTR;
+	mtu3_writel(mbase, U3D_LV1IESR, value);
+
+	/* Enable U2 common USB interrupts */
+	value = SUSPEND_INTR | RESUME_INTR | RESET_INTR;
+	mtu3_writel(mbase, U3D_COMMON_USB_INTR_ENABLE, value);
+
+	if (mtu->is_u3_ip) {
+		/* Enable U3 LTSSM interrupts */
+		value = HOT_RST_INTR | WARM_RST_INTR | VBUS_RISE_INTR |
+		    VBUS_FALL_INTR | ENTER_U3_INTR | EXIT_U3_INTR;
+		mtu3_writel(mbase, U3D_LTSSM_INTR_ENABLE, value);
+	}
+
+	/* Enable QMU interrupts. */
+	value = TXQ_CSERR_INT | TXQ_LENERR_INT | RXQ_CSERR_INT |
+			RXQ_LENERR_INT | RXQ_ZLPERR_INT;
+	mtu3_writel(mbase, U3D_QIESR1, value);
+
+	/* Enable speed change interrupt */
+	mtu3_writel(mbase, U3D_DEV_LINK_INTR_ENABLE, SSUSB_DEV_SPEED_CHG_INTR);
+}
+
+/* set/clear the stall and toggle bits for non-ep0 */
+void mtu3_ep_stall_set(struct mtu3_ep *mep, bool set)
+{
+	struct mtu3 *mtu = mep->mtu;
+	void __iomem *mbase = mtu->mac_base;
+	u8 epnum = mep->epnum;
+	u32 csr;
+
+	if (mep->is_in) {	/* TX */
+		csr = mtu3_readl(mbase, MU3D_EP_TXCR0(epnum)) & TX_W1C_BITS;
+		if (set)
+			csr |= TX_SENDSTALL;
+		else
+			csr = (csr & (~TX_SENDSTALL)) | TX_SENTSTALL;
+		mtu3_writel(mbase, MU3D_EP_TXCR0(epnum), csr);
+	} else {	/* RX */
+		csr = mtu3_readl(mbase, MU3D_EP_RXCR0(epnum)) & RX_W1C_BITS;
+		if (set)
+			csr |= RX_SENDSTALL;
+		else
+			csr = (csr & (~RX_SENDSTALL)) | RX_SENTSTALL;
+		mtu3_writel(mbase, MU3D_EP_RXCR0(epnum), csr);
+	}
+
+	if (!set) {
+		mtu3_setbits(mbase, U3D_EP_RST, EP_RST(mep->is_in, epnum));
+		mtu3_clrbits(mbase, U3D_EP_RST, EP_RST(mep->is_in, epnum));
+		mep->flags &= ~MTU3_EP_STALL;
+	} else {
+		mep->flags |= MTU3_EP_STALL;
+	}
+
+	dev_dbg(mtu->dev, "%s: %s\n", mep->name,
+		set ? "SEND STALL" : "CLEAR STALL, with EP RESET");
+}
+
+void mtu3_dev_on_off(struct mtu3 *mtu, int is_on)
+{
+	if (mtu->is_u3_ip && (mtu->max_speed == USB_SPEED_SUPER))
+		mtu3_ss_func_set(mtu, is_on);
+	else
+		mtu3_hs_softconn_set(mtu, is_on);
+
+	dev_info(mtu->dev, "gadget (%s) pullup D%s\n",
+		usb_speed_string(mtu->max_speed), is_on ? "+" : "-");
+}
+
+void mtu3_start(struct mtu3 *mtu)
+{
+	void __iomem *mbase = mtu->mac_base;
+
+	dev_dbg(mtu->dev, "%s devctl 0x%x\n", __func__,
+		mtu3_readl(mbase, U3D_DEVICE_CONTROL));
+
+	mtu3_clrbits(mtu->ippc_base, U3D_SSUSB_IP_PW_CTRL2, SSUSB_IP_DEV_PDN);
+
+	/*
+	 * When disable U2 port, USB2_CSR's register will be reset to
+	 * default value after re-enable it again(HS is enabled by default).
+	 * So if force mac to work as FS, disable HS function.
+	 */
+	if (mtu->max_speed == USB_SPEED_FULL)
+		mtu3_clrbits(mbase, U3D_POWER_MANAGEMENT, HS_ENABLE);
+
+	/* Initialize the default interrupts */
+	mtu3_intr_enable(mtu);
+	mtu->is_active = 1;
+
+	if (mtu->softconnect)
+		mtu3_dev_on_off(mtu, 1);
+}
+
+void mtu3_stop(struct mtu3 *mtu)
+{
+	dev_dbg(mtu->dev, "%s\n", __func__);
+
+	mtu3_intr_disable(mtu);
+	mtu3_intr_status_clear(mtu);
+
+	if (mtu->softconnect)
+		mtu3_dev_on_off(mtu, 0);
+
+	mtu->is_active = 0;
+	mtu3_setbits(mtu->ippc_base, U3D_SSUSB_IP_PW_CTRL2, SSUSB_IP_DEV_PDN);
+}
+
+/* for non-ep0 */
+int mtu3_config_ep(struct mtu3 *mtu, struct mtu3_ep *mep,
+			int interval, int burst, int mult)
+{
+	void __iomem *mbase = mtu->mac_base;
+	int epnum = mep->epnum;
+	u32 csr0, csr1, csr2;
+	int fifo_sgsz, fifo_addr;
+	int num_pkts;
+
+	fifo_addr = ep_fifo_alloc(mep, mep->maxp);
+	if (fifo_addr < 0) {
+		dev_err(mtu->dev, "alloc ep fifo failed(%d)\n", mep->maxp);
+		return -ENOMEM;
+	}
+	fifo_sgsz = ilog2(mep->fifo_seg_size);
+	dev_dbg(mtu->dev, "%s fifosz: %x(%x/%x)\n", __func__, fifo_sgsz,
+		mep->fifo_seg_size, mep->fifo_size);
+
+	if (mep->is_in) {
+		csr0 = TX_TXMAXPKTSZ(mep->maxp);
+		csr0 |= TX_DMAREQEN;
+
+		num_pkts = (burst + 1) * (mult + 1) - 1;
+		csr1 = TX_SS_BURST(burst) | TX_SLOT(mep->slot);
+		csr1 |= TX_MAX_PKT(num_pkts) | TX_MULT(mult);
+
+		csr2 = TX_FIFOADDR(fifo_addr >> 4);
+		csr2 |= TX_FIFOSEGSIZE(fifo_sgsz);
+
+		switch (mep->type) {
+		case USB_ENDPOINT_XFER_BULK:
+			csr1 |= TX_TYPE(TYPE_BULK);
+			break;
+		case USB_ENDPOINT_XFER_ISOC:
+			csr1 |= TX_TYPE(TYPE_ISO);
+			csr2 |= TX_BINTERVAL(interval);
+			break;
+		case USB_ENDPOINT_XFER_INT:
+			csr1 |= TX_TYPE(TYPE_INT);
+			csr2 |= TX_BINTERVAL(interval);
+			break;
+		}
+
+		/* Enable QMU Done interrupt */
+		mtu3_setbits(mbase, U3D_QIESR0, QMU_TX_DONE_INT(epnum));
+
+		mtu3_writel(mbase, MU3D_EP_TXCR0(epnum), csr0);
+		mtu3_writel(mbase, MU3D_EP_TXCR1(epnum), csr1);
+		mtu3_writel(mbase, MU3D_EP_TXCR2(epnum), csr2);
+
+		dev_dbg(mtu->dev, "U3D_TX%d CSR0:%#x, CSR1:%#x, CSR2:%#x\n",
+			epnum, mtu3_readl(mbase, MU3D_EP_TXCR0(epnum)),
+			mtu3_readl(mbase, MU3D_EP_TXCR1(epnum)),
+			mtu3_readl(mbase, MU3D_EP_TXCR2(epnum)));
+	} else {
+		csr0 = RX_RXMAXPKTSZ(mep->maxp);
+		csr0 |= RX_DMAREQEN;
+
+		num_pkts = (burst + 1) * (mult + 1) - 1;
+		csr1 = RX_SS_BURST(burst) | RX_SLOT(mep->slot);
+		csr1 |= RX_MAX_PKT(num_pkts) | RX_MULT(mult);
+
+		csr2 = RX_FIFOADDR(fifo_addr >> 4);
+		csr2 |= RX_FIFOSEGSIZE(fifo_sgsz);
+
+		switch (mep->type) {
+		case USB_ENDPOINT_XFER_BULK:
+			csr1 |= RX_TYPE(TYPE_BULK);
+			break;
+		case USB_ENDPOINT_XFER_ISOC:
+			csr1 |= RX_TYPE(TYPE_ISO);
+			csr2 |= RX_BINTERVAL(interval);
+			break;
+		case USB_ENDPOINT_XFER_INT:
+			csr1 |= RX_TYPE(TYPE_INT);
+			csr2 |= RX_BINTERVAL(interval);
+			break;
+		}
+
+		/*Enable QMU Done interrupt */
+		mtu3_setbits(mbase, U3D_QIESR0, QMU_RX_DONE_INT(epnum));
+
+		mtu3_writel(mbase, MU3D_EP_RXCR0(epnum), csr0);
+		mtu3_writel(mbase, MU3D_EP_RXCR1(epnum), csr1);
+		mtu3_writel(mbase, MU3D_EP_RXCR2(epnum), csr2);
+
+		dev_dbg(mtu->dev, "U3D_RX%d CSR0:%#x, CSR1:%#x, CSR2:%#x\n",
+			epnum, mtu3_readl(mbase, MU3D_EP_RXCR0(epnum)),
+			mtu3_readl(mbase, MU3D_EP_RXCR1(epnum)),
+			mtu3_readl(mbase, MU3D_EP_RXCR2(epnum)));
+	}
+
+	dev_dbg(mtu->dev, "csr0:%#x, csr1:%#x, csr2:%#x\n", csr0, csr1, csr2);
+	dev_dbg(mtu->dev, "%s: %s, fifo-addr:%#x, fifo-size:%#x(%#x/%#x)\n",
+		__func__, mep->name, mep->fifo_addr, mep->fifo_size,
+		fifo_sgsz, mep->fifo_seg_size);
+
+	return 0;
+}
+
+/* for non-ep0 */
+void mtu3_deconfig_ep(struct mtu3 *mtu, struct mtu3_ep *mep)
+{
+	void __iomem *mbase = mtu->mac_base;
+	int epnum = mep->epnum;
+
+	if (mep->is_in) {
+		mtu3_writel(mbase, MU3D_EP_TXCR0(epnum), 0);
+		mtu3_writel(mbase, MU3D_EP_TXCR1(epnum), 0);
+		mtu3_writel(mbase, MU3D_EP_TXCR2(epnum), 0);
+		mtu3_setbits(mbase, U3D_QIECR0, QMU_TX_DONE_INT(epnum));
+	} else {
+		mtu3_writel(mbase, MU3D_EP_RXCR0(epnum), 0);
+		mtu3_writel(mbase, MU3D_EP_RXCR1(epnum), 0);
+		mtu3_writel(mbase, MU3D_EP_RXCR2(epnum), 0);
+		mtu3_setbits(mbase, U3D_QIECR0, QMU_RX_DONE_INT(epnum));
+	}
+
+	ep_fifo_free(mep);
+
+	dev_dbg(mtu->dev, "%s: %s\n", __func__, mep->name);
+}
+
+/*
+ * Two scenarios:
+ * 1. when device IP supports SS, the fifo of EP0, TX EPs, RX EPs
+ *	are separated;
+ * 2. when supports only HS, the fifo is shared for all EPs, and
+ *	the capability registers of @EPNTXFFSZ or @EPNRXFFSZ indicate
+ *	the total fifo size of non-ep0, and ep0's is fixed to 64B,
+ *	so the total fifo size is 64B + @EPNTXFFSZ;
+ *	Due to the first 64B should be reserved for EP0, non-ep0's fifo
+ *	starts from offset 64 and are divided into two equal parts for
+ *	TX or RX EPs for simplification.
+ */
+static void get_ep_fifo_config(struct mtu3 *mtu)
+{
+	struct mtu3_fifo_info *tx_fifo;
+	struct mtu3_fifo_info *rx_fifo;
+	u32 fifosize;
+
+	if (mtu->is_u3_ip) {
+		fifosize = mtu3_readl(mtu->mac_base, U3D_CAP_EPNTXFFSZ);
+		tx_fifo = &mtu->tx_fifo;
+		tx_fifo->base = 0;
+		tx_fifo->limit = fifosize / MTU3_EP_FIFO_UNIT;
+		bitmap_zero(tx_fifo->bitmap, MTU3_FIFO_BIT_SIZE);
+
+		fifosize = mtu3_readl(mtu->mac_base, U3D_CAP_EPNRXFFSZ);
+		rx_fifo = &mtu->rx_fifo;
+		rx_fifo->base = 0;
+		rx_fifo->limit = fifosize / MTU3_EP_FIFO_UNIT;
+		bitmap_zero(rx_fifo->bitmap, MTU3_FIFO_BIT_SIZE);
+		mtu->slot = MTU3_U3_IP_SLOT_DEFAULT;
+	} else {
+		fifosize = mtu3_readl(mtu->mac_base, U3D_CAP_EPNTXFFSZ);
+		tx_fifo = &mtu->tx_fifo;
+		tx_fifo->base = MTU3_U2_IP_EP0_FIFO_SIZE;
+		tx_fifo->limit = (fifosize / MTU3_EP_FIFO_UNIT) >> 1;
+		bitmap_zero(tx_fifo->bitmap, MTU3_FIFO_BIT_SIZE);
+
+		rx_fifo = &mtu->rx_fifo;
+		rx_fifo->base =
+			tx_fifo->base + tx_fifo->limit * MTU3_EP_FIFO_UNIT;
+		rx_fifo->limit = tx_fifo->limit;
+		bitmap_zero(rx_fifo->bitmap, MTU3_FIFO_BIT_SIZE);
+		mtu->slot = MTU3_U2_IP_SLOT_DEFAULT;
+	}
+
+	dev_dbg(mtu->dev, "%s, TX: base-%d, limit-%d; RX: base-%d, limit-%d\n",
+		__func__, tx_fifo->base, tx_fifo->limit,
+		rx_fifo->base, rx_fifo->limit);
+}
+
+void mtu3_ep0_setup(struct mtu3 *mtu)
+{
+	u32 maxpacket = mtu->g.ep0->maxpacket;
+	u32 csr;
+
+	dev_dbg(mtu->dev, "%s maxpacket: %d\n", __func__, maxpacket);
+
+	csr = mtu3_readl(mtu->mac_base, U3D_EP0CSR);
+	csr &= ~EP0_MAXPKTSZ_MSK;
+	csr |= EP0_MAXPKTSZ(maxpacket);
+	csr &= EP0_W1C_BITS;
+	mtu3_writel(mtu->mac_base, U3D_EP0CSR, csr);
+
+	/* Enable EP0 interrupt */
+	mtu3_writel(mtu->mac_base, U3D_EPIESR, EP0ISR);
+}
+
+static int mtu3_mem_alloc(struct mtu3 *mtu)
+{
+	void __iomem *mbase = mtu->mac_base;
+	struct mtu3_ep *ep_array;
+	int in_ep_num, out_ep_num;
+	u32 cap_epinfo;
+	int ret;
+	int i;
+
+	cap_epinfo = mtu3_readl(mbase, U3D_CAP_EPINFO);
+	in_ep_num = CAP_TX_EP_NUM(cap_epinfo);
+	out_ep_num = CAP_RX_EP_NUM(cap_epinfo);
+
+	dev_info(mtu->dev, "fifosz/epnum: Tx=%#x/%d, Rx=%#x/%d\n",
+		 mtu3_readl(mbase, U3D_CAP_EPNTXFFSZ), in_ep_num,
+		 mtu3_readl(mbase, U3D_CAP_EPNRXFFSZ), out_ep_num);
+
+	/* one for ep0, another is reserved */
+	mtu->num_eps = min(in_ep_num, out_ep_num) + 1;
+	ep_array = kcalloc(mtu->num_eps * 2, sizeof(*ep_array), GFP_KERNEL);
+	if (ep_array == NULL)
+		return -ENOMEM;
+
+	mtu->ep_array = ep_array;
+	mtu->in_eps = ep_array;
+	mtu->out_eps = &ep_array[mtu->num_eps];
+	/* ep0 uses in_eps[0], out_eps[0] is reserved */
+	mtu->ep0 = mtu->in_eps;
+	mtu->ep0->mtu = mtu;
+	mtu->ep0->epnum = 0;
+
+	for (i = 1; i < mtu->num_eps; i++) {
+		struct mtu3_ep *mep = mtu->in_eps + i;
+
+		mep->fifo = &mtu->tx_fifo;
+		mep = mtu->out_eps + i;
+		mep->fifo = &mtu->rx_fifo;
+	}
+
+	get_ep_fifo_config(mtu);
+
+	ret = mtu3_qmu_init(mtu);
+	if (ret)
+		kfree(mtu->ep_array);
+
+	return ret;
+}
+
+static void mtu3_mem_free(struct mtu3 *mtu)
+{
+	mtu3_qmu_exit(mtu);
+	kfree(mtu->ep_array);
+}
+
+static void mtu3_set_speed(struct mtu3 *mtu)
+{
+	void __iomem *mbase = mtu->mac_base;
+
+	if (!mtu->is_u3_ip && (mtu->max_speed > USB_SPEED_HIGH))
+		mtu->max_speed = USB_SPEED_HIGH;
+
+	if (mtu->max_speed == USB_SPEED_FULL) {
+		/* disable U3 SS function */
+		mtu3_clrbits(mbase, U3D_USB3_CONFIG, USB3_EN);
+		/* disable HS function */
+		mtu3_clrbits(mbase, U3D_POWER_MANAGEMENT, HS_ENABLE);
+	} else if (mtu->max_speed == USB_SPEED_HIGH) {
+		mtu3_clrbits(mbase, U3D_USB3_CONFIG, USB3_EN);
+		/* HS/FS detected by HW */
+		mtu3_setbits(mbase, U3D_POWER_MANAGEMENT, HS_ENABLE);
+	}
+
+	dev_info(mtu->dev, "max_speed: %s\n",
+		usb_speed_string(mtu->max_speed));
+}
+
+static void mtu3_regs_init(struct mtu3 *mtu)
+{
+
+	void __iomem *mbase = mtu->mac_base;
+
+	/* be sure interrupts are disabled before registration of ISR */
+	mtu3_intr_disable(mtu);
+	mtu3_intr_status_clear(mtu);
+
+	if (mtu->is_u3_ip) {
+		/* disable LGO_U1/U2 by default */
+		mtu3_clrbits(mbase, U3D_LINK_POWER_CONTROL,
+				SW_U1_ACCEPT_ENABLE | SW_U2_ACCEPT_ENABLE |
+				SW_U1_REQUEST_ENABLE | SW_U2_REQUEST_ENABLE);
+		/* device responses to u3_exit from host automatically */
+		mtu3_clrbits(mbase, U3D_LTSSM_CTRL, SOFT_U3_EXIT_EN);
+		/* automatically build U2 link when U3 detect fail */
+		mtu3_setbits(mbase, U3D_USB2_TEST_MODE, U2U3_AUTO_SWITCH);
+	}
+
+	mtu3_set_speed(mtu);
+
+	/* delay about 0.1us from detecting reset to send chirp-K */
+	mtu3_clrbits(mbase, U3D_LINK_RESET_INFO, WTCHRP_MSK);
+	/* U2/U3 detected by HW */
+	mtu3_writel(mbase, U3D_DEVICE_CONF, 0);
+	/* enable QMU 16B checksum */
+	mtu3_setbits(mbase, U3D_QCR0, QMU_CS16B_EN);
+	/* vbus detected by HW */
+	mtu3_clrbits(mbase, U3D_MISC_CTRL, VBUS_FRC_EN | VBUS_ON);
+}
+
+static irqreturn_t mtu3_link_isr(struct mtu3 *mtu)
+{
+	void __iomem *mbase = mtu->mac_base;
+	enum usb_device_speed udev_speed;
+	u32 maxpkt = 64;
+	u32 link;
+	u32 speed;
+
+	link = mtu3_readl(mbase, U3D_DEV_LINK_INTR);
+	link &= mtu3_readl(mbase, U3D_DEV_LINK_INTR_ENABLE);
+	mtu3_writel(mbase, U3D_DEV_LINK_INTR, link); /* W1C */
+	dev_dbg(mtu->dev, "=== LINK[%x] ===\n", link);
+
+	if (!(link & SSUSB_DEV_SPEED_CHG_INTR))
+		return IRQ_NONE;
+
+	speed = SSUSB_DEV_SPEED(mtu3_readl(mbase, U3D_DEVICE_CONF));
+
+	switch (speed) {
+	case MTU3_SPEED_FULL:
+		udev_speed = USB_SPEED_FULL;
+		/*BESLCK = 4 < BESLCK_U3 = 10 < BESLDCK = 15 */
+		mtu3_writel(mbase, U3D_USB20_LPM_PARAMETER, LPM_BESLDCK(0xf)
+				| LPM_BESLCK(4) | LPM_BESLCK_U3(0xa));
+		mtu3_setbits(mbase, U3D_POWER_MANAGEMENT,
+				LPM_BESL_STALL | LPM_BESLD_STALL);
+		break;
+	case MTU3_SPEED_HIGH:
+		udev_speed = USB_SPEED_HIGH;
+		/*BESLCK = 4 < BESLCK_U3 = 10 < BESLDCK = 15 */
+		mtu3_writel(mbase, U3D_USB20_LPM_PARAMETER, LPM_BESLDCK(0xf)
+				| LPM_BESLCK(4) | LPM_BESLCK_U3(0xa));
+		mtu3_setbits(mbase, U3D_POWER_MANAGEMENT,
+				LPM_BESL_STALL | LPM_BESLD_STALL);
+		break;
+	case MTU3_SPEED_SUPER:
+		udev_speed = USB_SPEED_SUPER;
+		maxpkt = 512;
+		break;
+	default:
+		udev_speed = USB_SPEED_UNKNOWN;
+		break;
+	}
+	dev_dbg(mtu->dev, "%s: %s\n", __func__, usb_speed_string(udev_speed));
+
+	mtu->g.speed = udev_speed;
+	mtu->g.ep0->maxpacket = maxpkt;
+	mtu->ep0_state = MU3D_EP0_STATE_SETUP;
+
+	if (udev_speed == USB_SPEED_UNKNOWN)
+		mtu3_gadget_disconnect(mtu);
+	else
+		mtu3_ep0_setup(mtu);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t mtu3_u3_ltssm_isr(struct mtu3 *mtu)
+{
+	void __iomem *mbase = mtu->mac_base;
+	u32 ltssm;
+
+	ltssm = mtu3_readl(mbase, U3D_LTSSM_INTR);
+	ltssm &= mtu3_readl(mbase, U3D_LTSSM_INTR_ENABLE);
+	mtu3_writel(mbase, U3D_LTSSM_INTR, ltssm); /* W1C */
+	dev_dbg(mtu->dev, "=== LTSSM[%x] ===\n", ltssm);
+
+	if (ltssm & (HOT_RST_INTR | WARM_RST_INTR))
+		mtu3_gadget_reset(mtu);
+
+	if (ltssm & VBUS_FALL_INTR)
+		mtu3_ss_func_set(mtu, false);
+
+	if (ltssm & VBUS_RISE_INTR)
+		mtu3_ss_func_set(mtu, true);
+
+	if (ltssm & EXIT_U3_INTR)
+		mtu3_gadget_resume(mtu);
+
+	if (ltssm & ENTER_U3_INTR)
+		mtu3_gadget_suspend(mtu);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t mtu3_u2_common_isr(struct mtu3 *mtu)
+{
+	void __iomem *mbase = mtu->mac_base;
+	u32 u2comm;
+
+	u2comm = mtu3_readl(mbase, U3D_COMMON_USB_INTR);
+	u2comm &= mtu3_readl(mbase, U3D_COMMON_USB_INTR_ENABLE);
+	mtu3_writel(mbase, U3D_COMMON_USB_INTR, u2comm); /* W1C */
+	dev_dbg(mtu->dev, "=== U2COMM[%x] ===\n", u2comm);
+
+	if (u2comm & SUSPEND_INTR)
+		mtu3_gadget_suspend(mtu);
+
+	if (u2comm & RESUME_INTR)
+		mtu3_gadget_resume(mtu);
+
+	if (u2comm & RESET_INTR)
+		mtu3_gadget_reset(mtu);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t mtu3_irq(int irq, void *data)
+{
+	struct mtu3 *mtu = (struct mtu3 *)data;
+	unsigned long flags;
+	u32 level1;
+
+	spin_lock_irqsave(&mtu->lock, flags);
+
+	/* U3D_LV1ISR is RU */
+	level1 = mtu3_readl(mtu->mac_base, U3D_LV1ISR);
+	level1 &= mtu3_readl(mtu->mac_base, U3D_LV1IER);
+
+	if (level1 & EP_CTRL_INTR)
+		mtu3_link_isr(mtu);
+
+	if (level1 & MAC2_INTR)
+		mtu3_u2_common_isr(mtu);
+
+	if (level1 & MAC3_INTR)
+		mtu3_u3_ltssm_isr(mtu);
+
+	if (level1 & BMU_INTR)
+		mtu3_ep0_isr(mtu);
+
+	if (level1 & QMU_INTR)
+		mtu3_qmu_isr(mtu);
+
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	return IRQ_HANDLED;
+}
+
+static int mtu3_hw_init(struct mtu3 *mtu)
+{
+	u32 cap_dev;
+	int ret;
+
+	mtu->hw_version = mtu3_readl(mtu->ippc_base, U3D_SSUSB_HW_ID);
+
+	cap_dev = mtu3_readl(mtu->ippc_base, U3D_SSUSB_IP_DEV_CAP);
+	mtu->is_u3_ip = !!SSUSB_IP_DEV_U3_PORT_NUM(cap_dev);
+
+	dev_info(mtu->dev, "IP version 0x%x(%s IP)\n", mtu->hw_version,
+		mtu->is_u3_ip ? "U3" : "U2");
+
+	mtu3_device_reset(mtu);
+
+	ret = mtu3_device_enable(mtu);
+	if (ret) {
+		dev_err(mtu->dev, "device enable failed %d\n", ret);
+		return ret;
+	}
+
+	ret = mtu3_mem_alloc(mtu);
+	if (ret)
+		return -ENOMEM;
+
+	mtu3_regs_init(mtu);
+
+	return 0;
+}
+
+static void mtu3_hw_exit(struct mtu3 *mtu)
+{
+	mtu3_device_disable(mtu);
+	mtu3_mem_free(mtu);
+}
+
+/*-------------------------------------------------------------------------*/
+
+int ssusb_gadget_init(struct ssusb_mtk *ssusb)
+{
+	struct device *dev = ssusb->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct mtu3 *mtu = NULL;
+	struct resource *res;
+	int ret = -ENOMEM;
+
+	mtu = devm_kzalloc(dev, sizeof(struct mtu3), GFP_KERNEL);
+	if (mtu == NULL)
+		return -ENOMEM;
+
+	mtu->irq = platform_get_irq(pdev, 0);
+	if (mtu->irq <= 0) {
+		dev_err(dev, "fail to get irq number\n");
+		return -ENODEV;
+	}
+	dev_info(dev, "irq %d\n", mtu->irq);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mac");
+	mtu->mac_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(mtu->mac_base)) {
+		dev_err(dev, "error mapping memory for dev mac\n");
+		return PTR_ERR(mtu->mac_base);
+	}
+
+	spin_lock_init(&mtu->lock);
+	mtu->dev = dev;
+	mtu->ippc_base = ssusb->ippc_base;
+	ssusb->mac_base	= mtu->mac_base;
+	ssusb->u3d = mtu;
+	mtu->ssusb = ssusb;
+	mtu->max_speed = usb_get_maximum_speed(dev);
+
+	/* check the max_speed parameter */
+	switch (mtu->max_speed) {
+	case USB_SPEED_FULL:
+	case USB_SPEED_HIGH:
+	case USB_SPEED_SUPER:
+		break;
+	default:
+		dev_err(dev, "invalid max_speed: %s\n",
+			usb_speed_string(mtu->max_speed));
+		/* fall through */
+	case USB_SPEED_UNKNOWN:
+		/* default as SS */
+		mtu->max_speed = USB_SPEED_SUPER;
+		break;
+	}
+
+	dev_dbg(dev, "mac_base=0x%p, ippc_base=0x%p\n",
+		mtu->mac_base, mtu->ippc_base);
+
+	ret = mtu3_hw_init(mtu);
+	if (ret) {
+		dev_err(dev, "mtu3 hw init failed:%d\n", ret);
+		return ret;
+	}
+
+	ret = devm_request_irq(dev, mtu->irq, mtu3_irq, 0, dev_name(dev), mtu);
+	if (ret) {
+		dev_err(dev, "request irq %d failed!\n", mtu->irq);
+		goto irq_err;
+	}
+
+	device_init_wakeup(dev, true);
+
+	ret = mtu3_gadget_setup(mtu);
+	if (ret) {
+		dev_err(dev, "mtu3 gadget init failed:%d\n", ret);
+		goto gadget_err;
+	}
+
+	/* init as host mode, power down device IP for power saving */
+	if (mtu->ssusb->dr_mode == USB_DR_MODE_OTG)
+		mtu3_stop(mtu);
+
+	dev_dbg(dev, " %s() done...\n", __func__);
+
+	return 0;
+
+gadget_err:
+	device_init_wakeup(dev, false);
+
+irq_err:
+	mtu3_hw_exit(mtu);
+	ssusb->u3d = NULL;
+	dev_err(dev, " %s() fail...\n", __func__);
+
+	return ret;
+}
+
+void ssusb_gadget_exit(struct ssusb_mtk *ssusb)
+{
+	struct mtu3 *mtu = ssusb->u3d;
+
+	mtu3_gadget_cleanup(mtu);
+	device_init_wakeup(ssusb->dev, false);
+	mtu3_hw_exit(mtu);
+}
diff --git a/drivers/usb/mtu3/mtu3_dr.c b/drivers/usb/mtu3/mtu3_dr.c
new file mode 100644
index 000000000000..1a8987e7c5b0
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_dr.c
@@ -0,0 +1,379 @@
+/*
+ * mtu3_dr.c - dual role switch and host glue layer
+ *
+ * Copyright (C) 2016 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/debugfs.h>
+#include <linux/irq.h>
+#include <linux/kernel.h>
+#include <linux/of_device.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
+
+#include "mtu3.h"
+#include "mtu3_dr.h"
+
+#define USB2_PORT 2
+#define USB3_PORT 3
+
+enum mtu3_vbus_id_state {
+	MTU3_ID_FLOAT = 1,
+	MTU3_ID_GROUND,
+	MTU3_VBUS_OFF,
+	MTU3_VBUS_VALID,
+};
+
+static void toggle_opstate(struct ssusb_mtk *ssusb)
+{
+	if (!ssusb->otg_switch.is_u3_drd) {
+		mtu3_setbits(ssusb->mac_base, U3D_DEVICE_CONTROL, DC_SESSION);
+		mtu3_setbits(ssusb->mac_base, U3D_POWER_MANAGEMENT, SOFT_CONN);
+	}
+}
+
+/* only port0 supports dual-role mode */
+static int ssusb_port0_switch(struct ssusb_mtk *ssusb,
+	int version, bool tohost)
+{
+	void __iomem *ibase = ssusb->ippc_base;
+	u32 value;
+
+	dev_dbg(ssusb->dev, "%s (switch u%d port0 to %s)\n", __func__,
+		version, tohost ? "host" : "device");
+
+	if (version == USB2_PORT) {
+		/* 1. power off and disable u2 port0 */
+		value = mtu3_readl(ibase, SSUSB_U2_CTRL(0));
+		value |= SSUSB_U2_PORT_PDN | SSUSB_U2_PORT_DIS;
+		mtu3_writel(ibase, SSUSB_U2_CTRL(0), value);
+
+		/* 2. power on, enable u2 port0 and select its mode */
+		value = mtu3_readl(ibase, SSUSB_U2_CTRL(0));
+		value &= ~(SSUSB_U2_PORT_PDN | SSUSB_U2_PORT_DIS);
+		value = tohost ? (value | SSUSB_U2_PORT_HOST_SEL) :
+			(value & (~SSUSB_U2_PORT_HOST_SEL));
+		mtu3_writel(ibase, SSUSB_U2_CTRL(0), value);
+	} else {
+		/* 1. power off and disable u3 port0 */
+		value = mtu3_readl(ibase, SSUSB_U3_CTRL(0));
+		value |= SSUSB_U3_PORT_PDN | SSUSB_U3_PORT_DIS;
+		mtu3_writel(ibase, SSUSB_U3_CTRL(0), value);
+
+		/* 2. power on, enable u3 port0 and select its mode */
+		value = mtu3_readl(ibase, SSUSB_U3_CTRL(0));
+		value &= ~(SSUSB_U3_PORT_PDN | SSUSB_U3_PORT_DIS);
+		value = tohost ? (value | SSUSB_U3_PORT_HOST_SEL) :
+			(value & (~SSUSB_U3_PORT_HOST_SEL));
+		mtu3_writel(ibase, SSUSB_U3_CTRL(0), value);
+	}
+
+	return 0;
+}
+
+static void switch_port_to_host(struct ssusb_mtk *ssusb)
+{
+	u32 check_clk = 0;
+
+	dev_dbg(ssusb->dev, "%s\n", __func__);
+
+	ssusb_port0_switch(ssusb, USB2_PORT, true);
+
+	if (ssusb->otg_switch.is_u3_drd) {
+		ssusb_port0_switch(ssusb, USB3_PORT, true);
+		check_clk = SSUSB_U3_MAC_RST_B_STS;
+	}
+
+	ssusb_check_clocks(ssusb, check_clk);
+
+	/* after all clocks are stable */
+	toggle_opstate(ssusb);
+}
+
+static void switch_port_to_device(struct ssusb_mtk *ssusb)
+{
+	u32 check_clk = 0;
+
+	dev_dbg(ssusb->dev, "%s\n", __func__);
+
+	ssusb_port0_switch(ssusb, USB2_PORT, false);
+
+	if (ssusb->otg_switch.is_u3_drd) {
+		ssusb_port0_switch(ssusb, USB3_PORT, false);
+		check_clk = SSUSB_U3_MAC_RST_B_STS;
+	}
+
+	ssusb_check_clocks(ssusb, check_clk);
+}
+
+int ssusb_set_vbus(struct otg_switch_mtk *otg_sx, int is_on)
+{
+	struct ssusb_mtk *ssusb =
+		container_of(otg_sx, struct ssusb_mtk, otg_switch);
+	struct regulator *vbus = otg_sx->vbus;
+	int ret;
+
+	/* vbus is optional */
+	if (!vbus)
+		return 0;
+
+	dev_dbg(ssusb->dev, "%s: turn %s\n", __func__, is_on ? "on" : "off");
+
+	if (is_on) {
+		ret = regulator_enable(vbus);
+		if (ret) {
+			dev_err(ssusb->dev, "vbus regulator enable failed\n");
+			return ret;
+		}
+	} else {
+		regulator_disable(vbus);
+	}
+
+	return 0;
+}
+
+/*
+ * switch to host: -> MTU3_VBUS_OFF --> MTU3_ID_GROUND
+ * switch to device: -> MTU3_ID_FLOAT --> MTU3_VBUS_VALID
+ */
+static void ssusb_set_mailbox(struct otg_switch_mtk *otg_sx,
+	enum mtu3_vbus_id_state status)
+{
+	struct ssusb_mtk *ssusb =
+		container_of(otg_sx, struct ssusb_mtk, otg_switch);
+	struct mtu3 *mtu = ssusb->u3d;
+
+	dev_dbg(ssusb->dev, "mailbox state(%d)\n", status);
+
+	switch (status) {
+	case MTU3_ID_GROUND:
+		switch_port_to_host(ssusb);
+		ssusb_set_vbus(otg_sx, 1);
+		ssusb->is_host = true;
+		break;
+	case MTU3_ID_FLOAT:
+		ssusb->is_host = false;
+		ssusb_set_vbus(otg_sx, 0);
+		switch_port_to_device(ssusb);
+		break;
+	case MTU3_VBUS_OFF:
+		mtu3_stop(mtu);
+		pm_relax(ssusb->dev);
+		break;
+	case MTU3_VBUS_VALID:
+		/* avoid suspend when works as device */
+		pm_stay_awake(ssusb->dev);
+		mtu3_start(mtu);
+		break;
+	default:
+		dev_err(ssusb->dev, "invalid state\n");
+	}
+}
+
+static int ssusb_id_notifier(struct notifier_block *nb,
+	unsigned long event, void *ptr)
+{
+	struct otg_switch_mtk *otg_sx =
+		container_of(nb, struct otg_switch_mtk, id_nb);
+
+	if (event)
+		ssusb_set_mailbox(otg_sx, MTU3_ID_GROUND);
+	else
+		ssusb_set_mailbox(otg_sx, MTU3_ID_FLOAT);
+
+	return NOTIFY_DONE;
+}
+
+static int ssusb_vbus_notifier(struct notifier_block *nb,
+	unsigned long event, void *ptr)
+{
+	struct otg_switch_mtk *otg_sx =
+		container_of(nb, struct otg_switch_mtk, vbus_nb);
+
+	if (event)
+		ssusb_set_mailbox(otg_sx, MTU3_VBUS_VALID);
+	else
+		ssusb_set_mailbox(otg_sx, MTU3_VBUS_OFF);
+
+	return NOTIFY_DONE;
+}
+
+static int ssusb_extcon_register(struct otg_switch_mtk *otg_sx)
+{
+	struct ssusb_mtk *ssusb =
+		container_of(otg_sx, struct ssusb_mtk, otg_switch);
+	struct extcon_dev *edev = otg_sx->edev;
+	int ret;
+
+	/* extcon is optional */
+	if (!edev)
+		return 0;
+
+	otg_sx->vbus_nb.notifier_call = ssusb_vbus_notifier;
+	ret = extcon_register_notifier(edev, EXTCON_USB,
+					&otg_sx->vbus_nb);
+	if (ret < 0)
+		dev_err(ssusb->dev, "failed to register notifier for USB\n");
+
+	otg_sx->id_nb.notifier_call = ssusb_id_notifier;
+	ret = extcon_register_notifier(edev, EXTCON_USB_HOST,
+					&otg_sx->id_nb);
+	if (ret < 0)
+		dev_err(ssusb->dev, "failed to register notifier for USB-HOST\n");
+
+	dev_dbg(ssusb->dev, "EXTCON_USB: %d, EXTCON_USB_HOST: %d\n",
+		extcon_get_cable_state_(edev, EXTCON_USB),
+		extcon_get_cable_state_(edev, EXTCON_USB_HOST));
+
+	/* default as host, switch to device mode if needed */
+	if (extcon_get_cable_state_(edev, EXTCON_USB_HOST) == false)
+		ssusb_set_mailbox(otg_sx, MTU3_ID_FLOAT);
+	if (extcon_get_cable_state_(edev, EXTCON_USB) == true)
+		ssusb_set_mailbox(otg_sx, MTU3_VBUS_VALID);
+
+	return 0;
+}
+
+static void extcon_register_dwork(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct otg_switch_mtk *otg_sx =
+	    container_of(dwork, struct otg_switch_mtk, extcon_reg_dwork);
+
+	ssusb_extcon_register(otg_sx);
+}
+
+/*
+ * We provide an interface via debugfs to switch between host and device modes
+ * depending on user input.
+ * This is useful in special cases, such as uses TYPE-A receptacle but also
+ * wants to support dual-role mode.
+ * It generates cable state changes by pulling up/down IDPIN and
+ * notifies driver to switch mode by "extcon-usb-gpio".
+ * NOTE: when use MICRO receptacle, should not enable this interface.
+ */
+static void ssusb_mode_manual_switch(struct ssusb_mtk *ssusb, int to_host)
+{
+	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
+
+	if (to_host)
+		pinctrl_select_state(otg_sx->id_pinctrl, otg_sx->id_ground);
+	else
+		pinctrl_select_state(otg_sx->id_pinctrl, otg_sx->id_float);
+}
+
+
+static int ssusb_mode_show(struct seq_file *sf, void *unused)
+{
+	struct ssusb_mtk *ssusb = sf->private;
+
+	seq_printf(sf, "current mode: %s(%s drd)\n(echo device/host)\n",
+		ssusb->is_host ? "host" : "device",
+		ssusb->otg_switch.manual_drd_enabled ? "manual" : "auto");
+
+	return 0;
+}
+
+static int ssusb_mode_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, ssusb_mode_show, inode->i_private);
+}
+
+static ssize_t ssusb_mode_write(struct file *file,
+	const char __user *ubuf, size_t count, loff_t *ppos)
+{
+	struct seq_file *sf = file->private_data;
+	struct ssusb_mtk *ssusb = sf->private;
+	char buf[16];
+
+	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
+		return -EFAULT;
+
+	if (!strncmp(buf, "host", 4) && !ssusb->is_host) {
+		ssusb_mode_manual_switch(ssusb, 1);
+	} else if (!strncmp(buf, "device", 6) && ssusb->is_host) {
+		ssusb_mode_manual_switch(ssusb, 0);
+	} else {
+		dev_err(ssusb->dev, "wrong or duplicated setting\n");
+		return -EINVAL;
+	}
+
+	return count;
+}
+
+static const struct file_operations ssusb_mode_fops = {
+	.open = ssusb_mode_open,
+	.write = ssusb_mode_write,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static void ssusb_debugfs_init(struct ssusb_mtk *ssusb)
+{
+	struct dentry *root;
+	struct dentry *file;
+
+	root = debugfs_create_dir(dev_name(ssusb->dev), usb_debug_root);
+	if (IS_ERR_OR_NULL(root)) {
+		if (!root)
+			dev_err(ssusb->dev, "create debugfs root failed\n");
+		return;
+	}
+	ssusb->dbgfs_root = root;
+
+	file = debugfs_create_file("mode", S_IRUGO | S_IWUSR, root,
+			ssusb, &ssusb_mode_fops);
+	if (!file)
+		dev_dbg(ssusb->dev, "create debugfs mode failed\n");
+}
+
+static void ssusb_debugfs_exit(struct ssusb_mtk *ssusb)
+{
+	debugfs_remove_recursive(ssusb->dbgfs_root);
+}
+
+int ssusb_otg_switch_init(struct ssusb_mtk *ssusb)
+{
+	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
+
+	INIT_DELAYED_WORK(&otg_sx->extcon_reg_dwork, extcon_register_dwork);
+
+	if (otg_sx->manual_drd_enabled)
+		ssusb_debugfs_init(ssusb);
+
+	/* It is enough to delay 1s for waiting for host initialization */
+	schedule_delayed_work(&otg_sx->extcon_reg_dwork, HZ);
+
+	return 0;
+}
+
+void ssusb_otg_switch_exit(struct ssusb_mtk *ssusb)
+{
+	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
+
+	cancel_delayed_work(&otg_sx->extcon_reg_dwork);
+
+	if (otg_sx->edev) {
+		extcon_unregister_notifier(otg_sx->edev,
+			EXTCON_USB, &otg_sx->vbus_nb);
+		extcon_unregister_notifier(otg_sx->edev,
+			EXTCON_USB_HOST, &otg_sx->id_nb);
+	}
+
+	if (otg_sx->manual_drd_enabled)
+		ssusb_debugfs_exit(ssusb);
+}
diff --git a/drivers/usb/mtu3/mtu3_dr.h b/drivers/usb/mtu3/mtu3_dr.h
new file mode 100644
index 000000000000..9b228b5811b0
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_dr.h
@@ -0,0 +1,108 @@
+/*
+ * mtu3_dr.h - dual role switch and host glue layer header
+ *
+ * Copyright (C) 2016 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef _MTU3_DR_H_
+#define _MTU3_DR_H_
+
+#if IS_ENABLED(CONFIG_USB_MTU3_HOST) || IS_ENABLED(CONFIG_USB_MTU3_DUAL_ROLE)
+
+int ssusb_host_init(struct ssusb_mtk *ssusb, struct device_node *parent_dn);
+void ssusb_host_exit(struct ssusb_mtk *ssusb);
+int ssusb_wakeup_of_property_parse(struct ssusb_mtk *ssusb,
+				struct device_node *dn);
+int ssusb_host_enable(struct ssusb_mtk *ssusb);
+int ssusb_host_disable(struct ssusb_mtk *ssusb, bool suspend);
+int ssusb_wakeup_enable(struct ssusb_mtk *ssusb);
+void ssusb_wakeup_disable(struct ssusb_mtk *ssusb);
+
+#else
+
+static inline int ssusb_host_init(struct ssusb_mtk *ssusb,
+
+	struct device_node *parent_dn)
+{
+	return 0;
+}
+
+static inline void ssusb_host_exit(struct ssusb_mtk *ssusb)
+{}
+
+static inline int ssusb_wakeup_of_property_parse(
+	struct ssusb_mtk *ssusb, struct device_node *dn)
+{
+	return 0;
+}
+
+static inline int ssusb_host_enable(struct ssusb_mtk *ssusb)
+{
+	return 0;
+}
+
+static inline int ssusb_host_disable(struct ssusb_mtk *ssusb, bool suspend)
+{
+	return 0;
+}
+
+static inline int ssusb_wakeup_enable(struct ssusb_mtk *ssusb)
+{
+	return 0;
+}
+
+static inline void ssusb_wakeup_disable(struct ssusb_mtk *ssusb)
+{}
+
+#endif
+
+
+#if IS_ENABLED(CONFIG_USB_MTU3_GADGET) || IS_ENABLED(CONFIG_USB_MTU3_DUAL_ROLE)
+int ssusb_gadget_init(struct ssusb_mtk *ssusb);
+void ssusb_gadget_exit(struct ssusb_mtk *ssusb);
+#else
+static inline int ssusb_gadget_init(struct ssusb_mtk *ssusb)
+{
+	return 0;
+}
+
+static inline void ssusb_gadget_exit(struct ssusb_mtk *ssusb)
+{}
+#endif
+
+
+#if IS_ENABLED(CONFIG_USB_MTU3_DUAL_ROLE)
+int ssusb_otg_switch_init(struct ssusb_mtk *ssusb);
+void ssusb_otg_switch_exit(struct ssusb_mtk *ssusb);
+int ssusb_set_vbus(struct otg_switch_mtk *otg_sx, int is_on);
+
+#else
+
+static inline int ssusb_otg_switch_init(struct ssusb_mtk *ssusb)
+{
+	return 0;
+}
+
+static inline void ssusb_otg_switch_exit(struct ssusb_mtk *ssusb)
+{}
+
+static inline int ssusb_set_vbus(struct otg_switch_mtk *otg_sx, int is_on)
+{
+	return 0;
+}
+
+#endif
+
+#endif		/* _MTU3_DR_H_ */
diff --git a/drivers/usb/mtu3/mtu3_gadget.c b/drivers/usb/mtu3/mtu3_gadget.c
new file mode 100644
index 000000000000..9dd2441b4fa1
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_gadget.c
@@ -0,0 +1,730 @@
+/*
+ * mtu3_gadget.c - MediaTek usb3 DRD peripheral support
+ *
+ * Copyright (C) 2016 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include "mtu3.h"
+
+void mtu3_req_complete(struct mtu3_ep *mep,
+		     struct usb_request *req, int status)
+__releases(mep->mtu->lock)
+__acquires(mep->mtu->lock)
+{
+	struct mtu3_request *mreq;
+	struct mtu3 *mtu;
+	int busy = mep->busy;
+
+	mreq = to_mtu3_request(req);
+	list_del(&mreq->list);
+	if (mreq->request.status == -EINPROGRESS)
+		mreq->request.status = status;
+
+	mtu = mreq->mtu;
+	mep->busy = 1;
+	spin_unlock(&mtu->lock);
+
+	/* ep0 makes use of PIO, needn't unmap it */
+	if (mep->epnum)
+		usb_gadget_unmap_request(&mtu->g, req, mep->is_in);
+
+	dev_dbg(mtu->dev, "%s complete req: %p, sts %d, %d/%d\n", mep->name,
+		req, req->status, mreq->request.actual, mreq->request.length);
+
+	usb_gadget_giveback_request(&mep->ep, &mreq->request);
+
+	spin_lock(&mtu->lock);
+	mep->busy = busy;
+}
+
+static void nuke(struct mtu3_ep *mep, const int status)
+{
+	struct mtu3_request *mreq = NULL;
+
+	mep->busy = 1;
+	if (list_empty(&mep->req_list))
+		return;
+
+	dev_dbg(mep->mtu->dev, "abort %s's req: sts %d\n", mep->name, status);
+
+	/* exclude EP0 */
+	if (mep->epnum)
+		mtu3_qmu_flush(mep);
+
+	while (!list_empty(&mep->req_list)) {
+		mreq = list_first_entry(&mep->req_list,
+					struct mtu3_request, list);
+		mtu3_req_complete(mep, &mreq->request, status);
+	}
+}
+
+static int mtu3_ep_enable(struct mtu3_ep *mep)
+{
+	const struct usb_endpoint_descriptor *desc;
+	const struct usb_ss_ep_comp_descriptor *comp_desc;
+	struct mtu3 *mtu = mep->mtu;
+	u32 interval = 0;
+	u32 mult = 0;
+	u32 burst = 0;
+	int max_packet;
+	int ret;
+
+	desc = mep->desc;
+	comp_desc = mep->comp_desc;
+	mep->type = usb_endpoint_type(desc);
+	max_packet = usb_endpoint_maxp(desc);
+	mep->maxp = max_packet & GENMASK(10, 0);
+
+	switch (mtu->g.speed) {
+	case USB_SPEED_SUPER:
+		if (usb_endpoint_xfer_int(desc) ||
+				usb_endpoint_xfer_isoc(desc)) {
+			interval = desc->bInterval;
+			interval = clamp_val(interval, 1, 16) - 1;
+			if (usb_endpoint_xfer_isoc(desc) && comp_desc)
+				mult = comp_desc->bmAttributes;
+		}
+		if (comp_desc)
+			burst = comp_desc->bMaxBurst;
+
+		break;
+	case USB_SPEED_HIGH:
+		if (usb_endpoint_xfer_isoc(desc) ||
+				usb_endpoint_xfer_int(desc)) {
+			interval = desc->bInterval;
+			interval = clamp_val(interval, 1, 16) - 1;
+			burst = (max_packet & GENMASK(12, 11)) >> 11;
+		}
+		break;
+	default:
+		break; /*others are ignored */
+	}
+
+	dev_dbg(mtu->dev, "%s maxp:%d, interval:%d, burst:%d, mult:%d\n",
+		__func__, mep->maxp, interval, burst, mult);
+
+	mep->ep.maxpacket = mep->maxp;
+	mep->ep.desc = desc;
+	mep->ep.comp_desc = comp_desc;
+
+	/* slot mainly affects bulk/isoc transfer, so ignore int */
+	mep->slot = usb_endpoint_xfer_int(desc) ? 0 : mtu->slot;
+
+	ret = mtu3_config_ep(mtu, mep, interval, burst, mult);
+	if (ret < 0)
+		return ret;
+
+	ret = mtu3_gpd_ring_alloc(mep);
+	if (ret < 0) {
+		mtu3_deconfig_ep(mtu, mep);
+		return ret;
+	}
+
+	mtu3_qmu_start(mep);
+
+	return 0;
+}
+
+static int mtu3_ep_disable(struct mtu3_ep *mep)
+{
+	struct mtu3 *mtu = mep->mtu;
+
+	mtu3_qmu_stop(mep);
+
+	/* abort all pending requests */
+	nuke(mep, -ESHUTDOWN);
+	mtu3_deconfig_ep(mtu, mep);
+	mtu3_gpd_ring_free(mep);
+
+	mep->desc = NULL;
+	mep->ep.desc = NULL;
+	mep->comp_desc = NULL;
+	mep->type = 0;
+	mep->flags = 0;
+
+	return 0;
+}
+
+static int mtu3_gadget_ep_enable(struct usb_ep *ep,
+		const struct usb_endpoint_descriptor *desc)
+{
+	struct mtu3_ep *mep;
+	struct mtu3 *mtu;
+	unsigned long flags;
+	int ret = -EINVAL;
+
+	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
+		pr_debug("%s invalid parameters\n", __func__);
+		return -EINVAL;
+	}
+
+	if (!desc->wMaxPacketSize) {
+		pr_debug("%s missing wMaxPacketSize\n", __func__);
+		return -EINVAL;
+	}
+	mep = to_mtu3_ep(ep);
+	mtu = mep->mtu;
+
+	/* check ep number and direction against endpoint */
+	if (usb_endpoint_num(desc) != mep->epnum)
+		return -EINVAL;
+
+	if (!!usb_endpoint_dir_in(desc) ^ !!mep->is_in)
+		return -EINVAL;
+
+	dev_dbg(mtu->dev, "%s %s\n", __func__, ep->name);
+
+	if (mep->flags & MTU3_EP_ENABLED) {
+		dev_WARN_ONCE(mtu->dev, true, "%s is already enabled\n",
+				mep->name);
+		return 0;
+	}
+
+	spin_lock_irqsave(&mtu->lock, flags);
+	mep->desc = desc;
+	mep->comp_desc = ep->comp_desc;
+
+	ret = mtu3_ep_enable(mep);
+	if (ret)
+		goto error;
+
+	mep->busy = 0;
+	mep->wedged = 0;
+	mep->flags |= MTU3_EP_ENABLED;
+	mtu->active_ep++;
+
+error:
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	dev_dbg(mtu->dev, "%s active_ep=%d\n", __func__, mtu->active_ep);
+
+	return ret;
+}
+
+static int mtu3_gadget_ep_disable(struct usb_ep *ep)
+{
+	struct mtu3_ep *mep = to_mtu3_ep(ep);
+	struct mtu3 *mtu = mep->mtu;
+	unsigned long flags;
+
+	dev_dbg(mtu->dev, "%s %s\n", __func__, mep->name);
+
+	if (!(mep->flags & MTU3_EP_ENABLED)) {
+		dev_warn(mtu->dev, "%s is already disabled\n", mep->name);
+		return 0;
+	}
+
+	spin_lock_irqsave(&mtu->lock, flags);
+	mtu3_ep_disable(mep);
+	mep->flags &= ~MTU3_EP_ENABLED;
+	mtu->active_ep--;
+	spin_unlock_irqrestore(&(mtu->lock), flags);
+
+	dev_dbg(mtu->dev, "%s active_ep=%d, mtu3 is_active=%d\n",
+		__func__, mtu->active_ep, mtu->is_active);
+
+	return 0;
+}
+
+struct usb_request *mtu3_alloc_request(struct usb_ep *ep, gfp_t gfp_flags)
+{
+	struct mtu3_ep *mep = to_mtu3_ep(ep);
+	struct mtu3_request *mreq;
+
+	mreq = kzalloc(sizeof(*mreq), gfp_flags);
+	if (!mreq)
+		return NULL;
+
+	mreq->request.dma = DMA_ADDR_INVALID;
+	mreq->epnum = mep->epnum;
+	mreq->mep = mep;
+
+	return &mreq->request;
+}
+
+void mtu3_free_request(struct usb_ep *ep, struct usb_request *req)
+{
+	kfree(to_mtu3_request(req));
+}
+
+static int mtu3_gadget_queue(struct usb_ep *ep,
+		struct usb_request *req, gfp_t gfp_flags)
+{
+	struct mtu3_ep *mep;
+	struct mtu3_request *mreq;
+	struct mtu3 *mtu;
+	unsigned long flags;
+	int ret = 0;
+
+	if (!ep || !req)
+		return -EINVAL;
+
+	if (!req->buf)
+		return -ENODATA;
+
+	mep = to_mtu3_ep(ep);
+	mtu = mep->mtu;
+	mreq = to_mtu3_request(req);
+	mreq->mtu = mtu;
+
+	if (mreq->mep != mep)
+		return -EINVAL;
+
+	dev_dbg(mtu->dev, "%s %s EP%d(%s), req=%p, maxp=%d, len#%d\n",
+		__func__, mep->is_in ? "TX" : "RX", mreq->epnum, ep->name,
+		mreq, ep->maxpacket, mreq->request.length);
+
+	if (req->length > GPD_BUF_SIZE) {
+		dev_warn(mtu->dev,
+			"req length > supported MAX:%d requested:%d\n",
+			GPD_BUF_SIZE, req->length);
+		return -EOPNOTSUPP;
+	}
+
+	/* don't queue if the ep is down */
+	if (!mep->desc) {
+		dev_dbg(mtu->dev, "req=%p queued to %s while it's disabled\n",
+			req, ep->name);
+		return -ESHUTDOWN;
+	}
+
+	mreq->request.actual = 0;
+	mreq->request.status = -EINPROGRESS;
+
+	ret = usb_gadget_map_request(&mtu->g, req, mep->is_in);
+	if (ret) {
+		dev_err(mtu->dev, "dma mapping failed\n");
+		return ret;
+	}
+
+	spin_lock_irqsave(&mtu->lock, flags);
+
+	if (mtu3_prepare_transfer(mep)) {
+		ret = -EAGAIN;
+		goto error;
+	}
+
+	list_add_tail(&mreq->list, &mep->req_list);
+	mtu3_insert_gpd(mep, mreq);
+	mtu3_qmu_resume(mep);
+
+error:
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	return ret;
+}
+
+static int mtu3_gadget_dequeue(struct usb_ep *ep, struct usb_request *req)
+{
+	struct mtu3_ep *mep = to_mtu3_ep(ep);
+	struct mtu3_request *mreq = to_mtu3_request(req);
+	struct mtu3_request *r;
+	unsigned long flags;
+	int ret = 0;
+	struct mtu3 *mtu = mep->mtu;
+
+	if (!ep || !req || mreq->mep != mep)
+		return -EINVAL;
+
+	dev_dbg(mtu->dev, "%s : req=%p\n", __func__, req);
+
+	spin_lock_irqsave(&mtu->lock, flags);
+
+	list_for_each_entry(r, &mep->req_list, list) {
+		if (r == mreq)
+			break;
+	}
+	if (r != mreq) {
+		dev_dbg(mtu->dev, "req=%p not queued to %s\n", req, ep->name);
+		ret = -EINVAL;
+		goto done;
+	}
+
+	mtu3_qmu_flush(mep);  /* REVISIT: set BPS ?? */
+	mtu3_req_complete(mep, req, -ECONNRESET);
+	mtu3_qmu_start(mep);
+
+done:
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	return ret;
+}
+
+/*
+ * Set or clear the halt bit of an EP.
+ * A halted EP won't TX/RX any data but will queue requests.
+ */
+static int mtu3_gadget_ep_set_halt(struct usb_ep *ep, int value)
+{
+	struct mtu3_ep *mep = to_mtu3_ep(ep);
+	struct mtu3 *mtu = mep->mtu;
+	struct mtu3_request *mreq;
+	unsigned long flags;
+	int ret = 0;
+
+	if (!ep)
+		return -EINVAL;
+
+	dev_dbg(mtu->dev, "%s : %s...", __func__, ep->name);
+
+	spin_lock_irqsave(&mtu->lock, flags);
+
+	if (mep->type == USB_ENDPOINT_XFER_ISOC) {
+		ret = -EINVAL;
+		goto done;
+	}
+
+	mreq = next_request(mep);
+	if (value) {
+		/*
+		 * If there is not request for TX-EP, QMU will not transfer
+		 * data to TX-FIFO, so no need check whether TX-FIFO
+		 * holds bytes or not here
+		 */
+		if (mreq) {
+			dev_dbg(mtu->dev, "req in progress, cannot halt %s\n",
+				ep->name);
+			ret = -EAGAIN;
+			goto done;
+		}
+	} else {
+		mep->wedged = 0;
+	}
+
+	dev_dbg(mtu->dev, "%s %s stall\n", ep->name, value ? "set" : "clear");
+
+	mtu3_ep_stall_set(mep, value);
+
+done:
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	return ret;
+}
+
+/* Sets the halt feature with the clear requests ignored */
+static int mtu3_gadget_ep_set_wedge(struct usb_ep *ep)
+{
+	struct mtu3_ep *mep = to_mtu3_ep(ep);
+
+	if (!ep)
+		return -EINVAL;
+
+	mep->wedged = 1;
+
+	return usb_ep_set_halt(ep);
+}
+
+static const struct usb_ep_ops mtu3_ep_ops = {
+	.enable = mtu3_gadget_ep_enable,
+	.disable = mtu3_gadget_ep_disable,
+	.alloc_request = mtu3_alloc_request,
+	.free_request = mtu3_free_request,
+	.queue = mtu3_gadget_queue,
+	.dequeue = mtu3_gadget_dequeue,
+	.set_halt = mtu3_gadget_ep_set_halt,
+	.set_wedge = mtu3_gadget_ep_set_wedge,
+};
+
+static int mtu3_gadget_get_frame(struct usb_gadget *gadget)
+{
+	struct mtu3 *mtu = gadget_to_mtu3(gadget);
+
+	return (int)mtu3_readl(mtu->mac_base, U3D_USB20_FRAME_NUM);
+}
+
+static int mtu3_gadget_wakeup(struct usb_gadget *gadget)
+{
+	struct mtu3 *mtu = gadget_to_mtu3(gadget);
+	unsigned long flags;
+
+	dev_dbg(mtu->dev, "%s\n", __func__);
+
+	/* remote wakeup feature is not enabled by host */
+	if (!mtu->may_wakeup)
+		return  -EOPNOTSUPP;
+
+	spin_lock_irqsave(&mtu->lock, flags);
+	if (mtu->g.speed == USB_SPEED_SUPER) {
+		mtu3_setbits(mtu->mac_base, U3D_LINK_POWER_CONTROL, UX_EXIT);
+	} else {
+		mtu3_setbits(mtu->mac_base, U3D_POWER_MANAGEMENT, RESUME);
+		spin_unlock_irqrestore(&mtu->lock, flags);
+		usleep_range(10000, 11000);
+		spin_lock_irqsave(&mtu->lock, flags);
+		mtu3_clrbits(mtu->mac_base, U3D_POWER_MANAGEMENT, RESUME);
+	}
+	spin_unlock_irqrestore(&mtu->lock, flags);
+	return 0;
+}
+
+static int mtu3_gadget_set_self_powered(struct usb_gadget *gadget,
+		int is_selfpowered)
+{
+	struct mtu3 *mtu = gadget_to_mtu3(gadget);
+
+	mtu->is_self_powered = !!is_selfpowered;
+	return 0;
+}
+
+static int mtu3_gadget_pullup(struct usb_gadget *gadget, int is_on)
+{
+	struct mtu3 *mtu = gadget_to_mtu3(gadget);
+	unsigned long flags;
+
+	dev_dbg(mtu->dev, "%s (%s) for %sactive device\n", __func__,
+		is_on ? "on" : "off", mtu->is_active ? "" : "in");
+
+	/* we'd rather not pullup unless the device is active. */
+	spin_lock_irqsave(&mtu->lock, flags);
+
+	is_on = !!is_on;
+	if (!mtu->is_active) {
+		/* save it for mtu3_start() to process the request */
+		mtu->softconnect = is_on;
+	} else if (is_on != mtu->softconnect) {
+		mtu->softconnect = is_on;
+		mtu3_dev_on_off(mtu, is_on);
+	}
+
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	return 0;
+}
+
+static int mtu3_gadget_start(struct usb_gadget *gadget,
+		struct usb_gadget_driver *driver)
+{
+	struct mtu3 *mtu = gadget_to_mtu3(gadget);
+	unsigned long flags;
+
+	if (mtu->gadget_driver) {
+		dev_err(mtu->dev, "%s is already bound to %s\n",
+			mtu->g.name, mtu->gadget_driver->driver.name);
+		return -EBUSY;
+	}
+
+	dev_dbg(mtu->dev, "bind driver %s\n", driver->function);
+
+	spin_lock_irqsave(&mtu->lock, flags);
+
+	mtu->softconnect = 0;
+	mtu->gadget_driver = driver;
+
+	if (mtu->ssusb->dr_mode == USB_DR_MODE_PERIPHERAL)
+		mtu3_start(mtu);
+
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	return 0;
+}
+
+static void stop_activity(struct mtu3 *mtu)
+{
+	struct usb_gadget_driver *driver = mtu->gadget_driver;
+	int i;
+
+	/* don't disconnect if it's not connected */
+	if (mtu->g.speed == USB_SPEED_UNKNOWN)
+		driver = NULL;
+	else
+		mtu->g.speed = USB_SPEED_UNKNOWN;
+
+	/* deactivate the hardware */
+	if (mtu->softconnect) {
+		mtu->softconnect = 0;
+		mtu3_dev_on_off(mtu, 0);
+	}
+
+	/*
+	 * killing any outstanding requests will quiesce the driver;
+	 * then report disconnect
+	 */
+	nuke(mtu->ep0, -ESHUTDOWN);
+	for (i = 1; i < mtu->num_eps; i++) {
+		nuke(mtu->in_eps + i, -ESHUTDOWN);
+		nuke(mtu->out_eps + i, -ESHUTDOWN);
+	}
+
+	if (driver) {
+		spin_unlock(&mtu->lock);
+		driver->disconnect(&mtu->g);
+		spin_lock(&mtu->lock);
+	}
+}
+
+static int mtu3_gadget_stop(struct usb_gadget *g)
+{
+	struct mtu3 *mtu = gadget_to_mtu3(g);
+	unsigned long flags;
+
+	dev_dbg(mtu->dev, "%s\n", __func__);
+
+	spin_lock_irqsave(&mtu->lock, flags);
+
+	stop_activity(mtu);
+	mtu->gadget_driver = NULL;
+
+	if (mtu->ssusb->dr_mode == USB_DR_MODE_PERIPHERAL)
+		mtu3_stop(mtu);
+
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	return 0;
+}
+
+static const struct usb_gadget_ops mtu3_gadget_ops = {
+	.get_frame = mtu3_gadget_get_frame,
+	.wakeup = mtu3_gadget_wakeup,
+	.set_selfpowered = mtu3_gadget_set_self_powered,
+	.pullup = mtu3_gadget_pullup,
+	.udc_start = mtu3_gadget_start,
+	.udc_stop = mtu3_gadget_stop,
+};
+
+static void init_hw_ep(struct mtu3 *mtu, struct mtu3_ep *mep,
+		u32 epnum, u32 is_in)
+{
+	mep->epnum = epnum;
+	mep->mtu = mtu;
+	mep->is_in = is_in;
+
+	INIT_LIST_HEAD(&mep->req_list);
+
+	sprintf(mep->name, "ep%d%s", epnum,
+		!epnum ? "" : (is_in ? "in" : "out"));
+
+	mep->ep.name = mep->name;
+	INIT_LIST_HEAD(&mep->ep.ep_list);
+
+	/* initialize maxpacket as SS */
+	if (!epnum) {
+		usb_ep_set_maxpacket_limit(&mep->ep, 512);
+		mep->ep.caps.type_control = true;
+		mep->ep.ops = &mtu3_ep0_ops;
+		mtu->g.ep0 = &mep->ep;
+	} else {
+		usb_ep_set_maxpacket_limit(&mep->ep, 1024);
+		mep->ep.caps.type_iso = true;
+		mep->ep.caps.type_bulk = true;
+		mep->ep.caps.type_int = true;
+		mep->ep.ops = &mtu3_ep_ops;
+		list_add_tail(&mep->ep.ep_list, &mtu->g.ep_list);
+	}
+
+	dev_dbg(mtu->dev, "%s, name=%s, maxp=%d\n", __func__, mep->ep.name,
+		 mep->ep.maxpacket);
+
+	if (!epnum) {
+		mep->ep.caps.dir_in = true;
+		mep->ep.caps.dir_out = true;
+	} else if (is_in) {
+		mep->ep.caps.dir_in = true;
+	} else {
+		mep->ep.caps.dir_out = true;
+	}
+}
+
+static void mtu3_gadget_init_eps(struct mtu3 *mtu)
+{
+	u8 epnum;
+
+	/* initialize endpoint list just once */
+	INIT_LIST_HEAD(&(mtu->g.ep_list));
+
+	dev_dbg(mtu->dev, "%s num_eps(1 for a pair of tx&rx ep)=%d\n",
+		__func__, mtu->num_eps);
+
+	init_hw_ep(mtu, mtu->ep0, 0, 0);
+	for (epnum = 1; epnum < mtu->num_eps; epnum++) {
+		init_hw_ep(mtu, mtu->in_eps + epnum, epnum, 1);
+		init_hw_ep(mtu, mtu->out_eps + epnum, epnum, 0);
+	}
+}
+
+int mtu3_gadget_setup(struct mtu3 *mtu)
+{
+	int ret;
+
+	mtu->g.ops = &mtu3_gadget_ops;
+	mtu->g.max_speed = mtu->max_speed;
+	mtu->g.speed = USB_SPEED_UNKNOWN;
+	mtu->g.sg_supported = 0;
+	mtu->g.name = MTU3_DRIVER_NAME;
+	mtu->is_active = 0;
+
+	mtu3_gadget_init_eps(mtu);
+
+	ret = usb_add_gadget_udc(mtu->dev, &mtu->g);
+	if (ret) {
+		dev_err(mtu->dev, "failed to register udc\n");
+		return ret;
+	}
+
+	usb_gadget_set_state(&mtu->g, USB_STATE_NOTATTACHED);
+
+	return 0;
+}
+
+void mtu3_gadget_cleanup(struct mtu3 *mtu)
+{
+	usb_del_gadget_udc(&mtu->g);
+}
+
+void mtu3_gadget_resume(struct mtu3 *mtu)
+{
+	dev_dbg(mtu->dev, "gadget RESUME\n");
+	if (mtu->gadget_driver && mtu->gadget_driver->resume) {
+		spin_unlock(&mtu->lock);
+		mtu->gadget_driver->resume(&mtu->g);
+		spin_lock(&mtu->lock);
+	}
+}
+
+/* called when SOF packets stop for 3+ msec or enters U3 */
+void mtu3_gadget_suspend(struct mtu3 *mtu)
+{
+	dev_dbg(mtu->dev, "gadget SUSPEND\n");
+	if (mtu->gadget_driver && mtu->gadget_driver->suspend) {
+		spin_unlock(&mtu->lock);
+		mtu->gadget_driver->suspend(&mtu->g);
+		spin_lock(&mtu->lock);
+	}
+}
+
+/* called when VBUS drops below session threshold, and in other cases */
+void mtu3_gadget_disconnect(struct mtu3 *mtu)
+{
+	dev_dbg(mtu->dev, "gadget DISCONNECT\n");
+	if (mtu->gadget_driver && mtu->gadget_driver->disconnect) {
+		spin_unlock(&mtu->lock);
+		mtu->gadget_driver->disconnect(&mtu->g);
+		spin_lock(&mtu->lock);
+	}
+
+	usb_gadget_set_state(&mtu->g, USB_STATE_NOTATTACHED);
+}
+
+void mtu3_gadget_reset(struct mtu3 *mtu)
+{
+	dev_dbg(mtu->dev, "gadget RESET\n");
+
+	/* report disconnect, if we didn't flush EP state */
+	if (mtu->g.speed != USB_SPEED_UNKNOWN)
+		mtu3_gadget_disconnect(mtu);
+
+	mtu->address = 0;
+	mtu->ep0_state = MU3D_EP0_STATE_SETUP;
+	mtu->may_wakeup = 0;
+}
diff --git a/drivers/usb/mtu3/mtu3_gadget_ep0.c b/drivers/usb/mtu3/mtu3_gadget_ep0.c
new file mode 100644
index 000000000000..2d7427b48775
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_gadget_ep0.c
@@ -0,0 +1,881 @@
+/*
+ * mtu3_gadget_ep0.c - MediaTek USB3 DRD peripheral driver ep0 handling
+ *
+ * Copyright (c) 2016 MediaTek Inc.
+ *
+ * Author:  Chunfeng.Yun <chunfeng.yun@mediatek.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include "mtu3.h"
+
+/* ep0 is always mtu3->in_eps[0] */
+#define	next_ep0_request(mtu)	next_request((mtu)->ep0)
+
+/* for high speed test mode; see USB 2.0 spec 7.1.20 */
+static const u8 mtu3_test_packet[53] = {
+	/* implicit SYNC then DATA0 to start */
+
+	/* JKJKJKJK x9 */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	/* JJKKJJKK x8 */
+	0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+	/* JJJJKKKK x8 */
+	0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee, 0xee,
+	/* JJJJJJJKKKKKKK x8 */
+	0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	/* JJJJJJJK x8 */
+	0x7f, 0xbf, 0xdf, 0xef, 0xf7, 0xfb, 0xfd,
+	/* JKKKKKKK x10, JK */
+	0xfc, 0x7e, 0xbf, 0xdf, 0xef, 0xf7, 0xfb, 0xfd, 0x7e,
+	/* implicit CRC16 then EOP to end */
+};
+
+static char *decode_ep0_state(struct mtu3 *mtu)
+{
+	switch (mtu->ep0_state) {
+	case MU3D_EP0_STATE_SETUP:
+		return "SETUP";
+	case MU3D_EP0_STATE_TX:
+		return "IN";
+	case MU3D_EP0_STATE_RX:
+		return "OUT";
+	case MU3D_EP0_STATE_TX_END:
+		return "TX-END";
+	case MU3D_EP0_STATE_STALL:
+		return "STALL";
+	default:
+		return "??";
+	}
+}
+
+static void ep0_req_giveback(struct mtu3 *mtu, struct usb_request *req)
+{
+	mtu3_req_complete(mtu->ep0, req, 0);
+}
+
+static int
+forward_to_driver(struct mtu3 *mtu, const struct usb_ctrlrequest *setup)
+__releases(mtu->lock)
+__acquires(mtu->lock)
+{
+	int ret;
+
+	if (!mtu->gadget_driver)
+		return -EOPNOTSUPP;
+
+	spin_unlock(&mtu->lock);
+	ret = mtu->gadget_driver->setup(&mtu->g, setup);
+	spin_lock(&mtu->lock);
+
+	dev_dbg(mtu->dev, "%s ret %d\n", __func__, ret);
+	return ret;
+}
+
+static void ep0_write_fifo(struct mtu3_ep *mep, const u8 *src, u16 len)
+{
+	void __iomem *fifo = mep->mtu->mac_base + U3D_FIFO0;
+	u16 index = 0;
+
+	dev_dbg(mep->mtu->dev, "%s: ep%din, len=%d, buf=%p\n",
+		__func__, mep->epnum, len, src);
+
+	if (len >= 4) {
+		iowrite32_rep(fifo, src, len >> 2);
+		index = len & ~0x03;
+	}
+	if (len & 0x02) {
+		writew(*(u16 *)&src[index], fifo);
+		index += 2;
+	}
+	if (len & 0x01)
+		writeb(src[index], fifo);
+}
+
+static void ep0_read_fifo(struct mtu3_ep *mep, u8 *dst, u16 len)
+{
+	void __iomem *fifo = mep->mtu->mac_base + U3D_FIFO0;
+	u32 value;
+	u16 index = 0;
+
+	dev_dbg(mep->mtu->dev, "%s: ep%dout len=%d buf=%p\n",
+		 __func__, mep->epnum, len, dst);
+
+	if (len >= 4) {
+		ioread32_rep(fifo, dst, len >> 2);
+		index = len & ~0x03;
+	}
+	if (len & 0x3) {
+		value = readl(fifo);
+		memcpy(&dst[index], &value, len & 0x3);
+	}
+
+}
+
+static void ep0_load_test_packet(struct mtu3 *mtu)
+{
+	/*
+	 * because the length of test packet is less than max packet of HS ep0,
+	 * write it into fifo directly.
+	 */
+	ep0_write_fifo(mtu->ep0, mtu3_test_packet, sizeof(mtu3_test_packet));
+}
+
+/*
+ * A. send STALL for setup transfer without data stage:
+ *		set SENDSTALL and SETUPPKTRDY at the same time;
+ * B. send STALL for other cases:
+ *		set SENDSTALL only.
+ */
+static void ep0_stall_set(struct mtu3_ep *mep0, bool set, u32 pktrdy)
+{
+	struct mtu3 *mtu = mep0->mtu;
+	void __iomem *mbase = mtu->mac_base;
+	u32 csr;
+
+	/* EP0_SENTSTALL is W1C */
+	csr = mtu3_readl(mbase, U3D_EP0CSR) & EP0_W1C_BITS;
+	if (set)
+		csr |= EP0_SENDSTALL | pktrdy;
+	else
+		csr = (csr & ~EP0_SENDSTALL) | EP0_SENTSTALL;
+	mtu3_writel(mtu->mac_base, U3D_EP0CSR, csr);
+
+	mtu->ep0_state = MU3D_EP0_STATE_SETUP;
+
+	dev_dbg(mtu->dev, "ep0: %s STALL, ep0_state: %s\n",
+		set ? "SEND" : "CLEAR", decode_ep0_state(mtu));
+}
+
+static int ep0_queue(struct mtu3_ep *mep0, struct mtu3_request *mreq);
+
+static void ep0_dummy_complete(struct usb_ep *ep, struct usb_request *req)
+{}
+
+static void ep0_set_sel_complete(struct usb_ep *ep, struct usb_request *req)
+{
+	struct mtu3_request *mreq;
+	struct mtu3 *mtu;
+	struct usb_set_sel_req sel;
+
+	memcpy(&sel, req->buf, sizeof(sel));
+
+	mreq = to_mtu3_request(req);
+	mtu = mreq->mtu;
+	dev_dbg(mtu->dev, "u1sel:%d, u1pel:%d, u2sel:%d, u2pel:%d\n",
+		sel.u1_sel, sel.u1_pel, sel.u2_sel, sel.u2_pel);
+}
+
+/* queue data stage to handle 6 byte SET_SEL request */
+static int ep0_set_sel(struct mtu3 *mtu, struct usb_ctrlrequest *setup)
+{
+	int ret;
+	u16 length = le16_to_cpu(setup->wLength);
+
+	if (unlikely(length != 6)) {
+		dev_err(mtu->dev, "%s wrong wLength:%d\n",
+			__func__, length);
+		return -EINVAL;
+	}
+
+	mtu->ep0_req.mep = mtu->ep0;
+	mtu->ep0_req.request.length = 6;
+	mtu->ep0_req.request.buf = mtu->setup_buf;
+	mtu->ep0_req.request.complete = ep0_set_sel_complete;
+	ret = ep0_queue(mtu->ep0, &mtu->ep0_req);
+
+	return ret < 0 ? ret : 1;
+}
+
+static int
+ep0_get_status(struct mtu3 *mtu, const struct usb_ctrlrequest *setup)
+{
+	struct mtu3_ep *mep = NULL;
+	int handled = 1;
+	u8 result[2] = {0, 0};
+	u8 epnum = 0;
+	int is_in;
+
+	switch (setup->bRequestType & USB_RECIP_MASK) {
+	case USB_RECIP_DEVICE:
+		result[0] = mtu->is_self_powered << USB_DEVICE_SELF_POWERED;
+		result[0] |= mtu->may_wakeup << USB_DEVICE_REMOTE_WAKEUP;
+		/* superspeed only */
+		if (mtu->g.speed == USB_SPEED_SUPER) {
+			result[0] |= mtu->u1_enable << USB_DEV_STAT_U1_ENABLED;
+			result[0] |= mtu->u2_enable << USB_DEV_STAT_U2_ENABLED;
+		}
+
+		dev_dbg(mtu->dev, "%s result=%x, U1=%x, U2=%x\n", __func__,
+			result[0], mtu->u1_enable, mtu->u2_enable);
+
+		break;
+	case USB_RECIP_INTERFACE:
+		break;
+	case USB_RECIP_ENDPOINT:
+		epnum = (u8) le16_to_cpu(setup->wIndex);
+		is_in = epnum & USB_DIR_IN;
+		epnum &= USB_ENDPOINT_NUMBER_MASK;
+
+		if (epnum >= mtu->num_eps) {
+			handled = -EINVAL;
+			break;
+		}
+		if (!epnum)
+			break;
+
+		mep = (is_in ? mtu->in_eps : mtu->out_eps) + epnum;
+		if (!mep->desc) {
+			handled = -EINVAL;
+			break;
+		}
+		if (mep->flags & MTU3_EP_STALL)
+			result[0] |= 1 << USB_ENDPOINT_HALT;
+
+		break;
+	default:
+		/* class, vendor, etc ... delegate */
+		handled = 0;
+		break;
+	}
+
+	if (handled > 0) {
+		int ret;
+
+		/* prepare a data stage for GET_STATUS */
+		dev_dbg(mtu->dev, "get_status=%x\n", *(u16 *)result);
+		memcpy(mtu->setup_buf, result, sizeof(result));
+		mtu->ep0_req.mep = mtu->ep0;
+		mtu->ep0_req.request.length = 2;
+		mtu->ep0_req.request.buf = &mtu->setup_buf;
+		mtu->ep0_req.request.complete = ep0_dummy_complete;
+		ret = ep0_queue(mtu->ep0, &mtu->ep0_req);
+		if (ret < 0)
+			handled = ret;
+	}
+	return handled;
+}
+
+static int handle_test_mode(struct mtu3 *mtu, struct usb_ctrlrequest *setup)
+{
+	void __iomem *mbase = mtu->mac_base;
+	int handled = 1;
+
+	switch (le16_to_cpu(setup->wIndex) >> 8) {
+	case TEST_J:
+		dev_dbg(mtu->dev, "TEST_J\n");
+		mtu->test_mode_nr = TEST_J_MODE;
+		break;
+	case TEST_K:
+		dev_dbg(mtu->dev, "TEST_K\n");
+		mtu->test_mode_nr = TEST_K_MODE;
+		break;
+	case TEST_SE0_NAK:
+		dev_dbg(mtu->dev, "TEST_SE0_NAK\n");
+		mtu->test_mode_nr = TEST_SE0_NAK_MODE;
+		break;
+	case TEST_PACKET:
+		dev_dbg(mtu->dev, "TEST_PACKET\n");
+		mtu->test_mode_nr = TEST_PACKET_MODE;
+		break;
+	default:
+		handled = -EINVAL;
+		goto out;
+	}
+
+	mtu->test_mode = true;
+
+	/* no TX completion interrupt, and need restart platform after test */
+	if (mtu->test_mode_nr == TEST_PACKET_MODE)
+		ep0_load_test_packet(mtu);
+
+	mtu3_writel(mbase, U3D_USB2_TEST_MODE, mtu->test_mode_nr);
+
+	mtu->ep0_state = MU3D_EP0_STATE_SETUP;
+
+out:
+	return handled;
+}
+
+static int ep0_handle_feature_dev(struct mtu3 *mtu,
+		struct usb_ctrlrequest *setup, bool set)
+{
+	void __iomem *mbase = mtu->mac_base;
+	int handled = -EINVAL;
+	u32 lpc;
+
+	switch (le16_to_cpu(setup->wValue)) {
+	case USB_DEVICE_REMOTE_WAKEUP:
+		mtu->may_wakeup = !!set;
+		handled = 1;
+		break;
+	case USB_DEVICE_TEST_MODE:
+		if (!set || (mtu->g.speed != USB_SPEED_HIGH) ||
+			(le16_to_cpu(setup->wIndex) & 0xff))
+			break;
+
+		handled = handle_test_mode(mtu, setup);
+		break;
+	case USB_DEVICE_U1_ENABLE:
+		if (mtu->g.speed != USB_SPEED_SUPER ||
+			mtu->g.state != USB_STATE_CONFIGURED)
+			break;
+
+		lpc = mtu3_readl(mbase, U3D_LINK_POWER_CONTROL);
+		if (set)
+			lpc |= SW_U1_ACCEPT_ENABLE;
+		else
+			lpc &= ~SW_U1_ACCEPT_ENABLE;
+		mtu3_writel(mbase, U3D_LINK_POWER_CONTROL, lpc);
+
+		mtu->u1_enable = !!set;
+		handled = 1;
+		break;
+	case USB_DEVICE_U2_ENABLE:
+		if (mtu->g.speed != USB_SPEED_SUPER ||
+			mtu->g.state != USB_STATE_CONFIGURED)
+			break;
+
+		lpc = mtu3_readl(mbase, U3D_LINK_POWER_CONTROL);
+		if (set)
+			lpc |= SW_U2_ACCEPT_ENABLE;
+		else
+			lpc &= ~SW_U2_ACCEPT_ENABLE;
+		mtu3_writel(mbase, U3D_LINK_POWER_CONTROL, lpc);
+
+		mtu->u2_enable = !!set;
+		handled = 1;
+		break;
+	default:
+		handled = -EINVAL;
+		break;
+	}
+	return handled;
+}
+
+static int ep0_handle_feature(struct mtu3 *mtu,
+		struct usb_ctrlrequest *setup, bool set)
+{
+	struct mtu3_ep *mep;
+	int handled = -EINVAL;
+	int is_in;
+	u16 value;
+	u16 index;
+	u8 epnum;
+
+	value = le16_to_cpu(setup->wValue);
+	index = le16_to_cpu(setup->wIndex);
+
+	switch (setup->bRequestType & USB_RECIP_MASK) {
+	case USB_RECIP_DEVICE:
+		handled = ep0_handle_feature_dev(mtu, setup, set);
+		break;
+	case USB_RECIP_INTERFACE:
+		/* superspeed only */
+		if ((value == USB_INTRF_FUNC_SUSPEND)
+			&& (mtu->g.speed == USB_SPEED_SUPER)) {
+			/*
+			 * forward the request because function drivers
+			 * should handle it
+			 */
+			handled = 0;
+		}
+		break;
+	case USB_RECIP_ENDPOINT:
+		epnum = index & USB_ENDPOINT_NUMBER_MASK;
+		if (epnum == 0 || epnum >= mtu->num_eps ||
+			value != USB_ENDPOINT_HALT)
+			break;
+
+		is_in = index & USB_DIR_IN;
+		mep = (is_in ? mtu->in_eps : mtu->out_eps) + epnum;
+		if (!mep->desc)
+			break;
+
+		handled = 1;
+		/* ignore request if endpoint is wedged */
+		if (mep->wedged)
+			break;
+
+		mtu3_ep_stall_set(mep, set);
+		break;
+	default:
+		/* class, vendor, etc ... delegate */
+		handled = 0;
+		break;
+	}
+	return handled;
+}
+
+/*
+ * handle all control requests can be handled
+ * returns:
+ *	negative errno - error happened
+ *	zero - need delegate SETUP to gadget driver
+ *	positive - already handled
+ */
+static int handle_standard_request(struct mtu3 *mtu,
+			  struct usb_ctrlrequest *setup)
+{
+	void __iomem *mbase = mtu->mac_base;
+	enum usb_device_state state = mtu->g.state;
+	int handled = -EINVAL;
+	u32 dev_conf;
+	u16 value;
+
+	value = le16_to_cpu(setup->wValue);
+
+	/* the gadget driver handles everything except what we must handle */
+	switch (setup->bRequest) {
+	case USB_REQ_SET_ADDRESS:
+		/* change it after the status stage */
+		mtu->address = (u8) (value & 0x7f);
+		dev_dbg(mtu->dev, "set address to 0x%x\n", mtu->address);
+
+		dev_conf = mtu3_readl(mbase, U3D_DEVICE_CONF);
+		dev_conf &= ~DEV_ADDR_MSK;
+		dev_conf |= DEV_ADDR(mtu->address);
+		mtu3_writel(mbase, U3D_DEVICE_CONF, dev_conf);
+
+		if (mtu->address)
+			usb_gadget_set_state(&mtu->g, USB_STATE_ADDRESS);
+		else
+			usb_gadget_set_state(&mtu->g, USB_STATE_DEFAULT);
+
+		handled = 1;
+		break;
+	case USB_REQ_SET_CONFIGURATION:
+		if (state == USB_STATE_ADDRESS) {
+			usb_gadget_set_state(&mtu->g,
+					USB_STATE_CONFIGURED);
+		} else if (state == USB_STATE_CONFIGURED) {
+			/*
+			 * USB2 spec sec 9.4.7, if wValue is 0 then dev
+			 * is moved to addressed state
+			 */
+			if (!value)
+				usb_gadget_set_state(&mtu->g,
+						USB_STATE_ADDRESS);
+		}
+		handled = 0;
+		break;
+	case USB_REQ_CLEAR_FEATURE:
+		handled = ep0_handle_feature(mtu, setup, 0);
+		break;
+	case USB_REQ_SET_FEATURE:
+		handled = ep0_handle_feature(mtu, setup, 1);
+		break;
+	case USB_REQ_GET_STATUS:
+		handled = ep0_get_status(mtu, setup);
+		break;
+	case USB_REQ_SET_SEL:
+		handled = ep0_set_sel(mtu, setup);
+		break;
+	case USB_REQ_SET_ISOCH_DELAY:
+		handled = 1;
+		break;
+	default:
+		/* delegate SET_CONFIGURATION, etc */
+		handled = 0;
+	}
+
+	return handled;
+}
+
+/* receive an data packet (OUT) */
+static void ep0_rx_state(struct mtu3 *mtu)
+{
+	struct mtu3_request *mreq;
+	struct usb_request *req;
+	void __iomem *mbase = mtu->mac_base;
+	u32 maxp;
+	u32 csr;
+	u16 count = 0;
+
+	dev_dbg(mtu->dev, "%s\n", __func__);
+
+	csr = mtu3_readl(mbase, U3D_EP0CSR) & EP0_W1C_BITS;
+	mreq = next_ep0_request(mtu);
+	req = &mreq->request;
+
+	/* read packet and ack; or stall because of gadget driver bug */
+	if (req) {
+		void *buf = req->buf + req->actual;
+		unsigned int len = req->length - req->actual;
+
+		/* read the buffer */
+		count = mtu3_readl(mbase, U3D_RXCOUNT0);
+		if (count > len) {
+			req->status = -EOVERFLOW;
+			count = len;
+		}
+		ep0_read_fifo(mtu->ep0, buf, count);
+		req->actual += count;
+		csr |= EP0_RXPKTRDY;
+
+		maxp = mtu->g.ep0->maxpacket;
+		if (count < maxp || req->actual == req->length) {
+			mtu->ep0_state = MU3D_EP0_STATE_SETUP;
+			dev_dbg(mtu->dev, "ep0 state: %s\n",
+				decode_ep0_state(mtu));
+
+			csr |= EP0_DATAEND;
+		} else {
+			req = NULL;
+		}
+	} else {
+		csr |= EP0_RXPKTRDY | EP0_SENDSTALL;
+		dev_dbg(mtu->dev, "%s: SENDSTALL\n", __func__);
+	}
+
+	mtu3_writel(mbase, U3D_EP0CSR, csr);
+
+	/* give back the request if have received all data */
+	if (req)
+		ep0_req_giveback(mtu, req);
+
+}
+
+/* transmitting to the host (IN) */
+static void ep0_tx_state(struct mtu3 *mtu)
+{
+	struct mtu3_request *mreq = next_ep0_request(mtu);
+	struct usb_request *req;
+	u32 csr;
+	u8 *src;
+	u8 count;
+	u32 maxp;
+
+	dev_dbg(mtu->dev, "%s\n", __func__);
+
+	if (!mreq)
+		return;
+
+	maxp = mtu->g.ep0->maxpacket;
+	req = &mreq->request;
+
+	/* load the data */
+	src = (u8 *)req->buf + req->actual;
+	count = min(maxp, req->length - req->actual);
+	if (count)
+		ep0_write_fifo(mtu->ep0, src, count);
+
+	dev_dbg(mtu->dev, "%s act=%d, len=%d, cnt=%d, maxp=%d zero=%d\n",
+		 __func__, req->actual, req->length, count, maxp, req->zero);
+
+	req->actual += count;
+
+	if ((count < maxp)
+		|| ((req->actual == req->length) && !req->zero))
+		mtu->ep0_state = MU3D_EP0_STATE_TX_END;
+
+	/* send it out, triggering a "txpktrdy cleared" irq */
+	csr = mtu3_readl(mtu->mac_base, U3D_EP0CSR) & EP0_W1C_BITS;
+	mtu3_writel(mtu->mac_base, U3D_EP0CSR, csr | EP0_TXPKTRDY);
+
+	dev_dbg(mtu->dev, "%s ep0csr=0x%x\n", __func__,
+		mtu3_readl(mtu->mac_base, U3D_EP0CSR));
+}
+
+static void ep0_read_setup(struct mtu3 *mtu, struct usb_ctrlrequest *setup)
+{
+	struct mtu3_request *mreq;
+	u32 count;
+	u32 csr;
+
+	csr = mtu3_readl(mtu->mac_base, U3D_EP0CSR) & EP0_W1C_BITS;
+	count = mtu3_readl(mtu->mac_base, U3D_RXCOUNT0);
+
+	ep0_read_fifo(mtu->ep0, (u8 *)setup, count);
+
+	dev_dbg(mtu->dev, "SETUP req%02x.%02x v%04x i%04x l%04x\n",
+		 setup->bRequestType, setup->bRequest,
+		 le16_to_cpu(setup->wValue), le16_to_cpu(setup->wIndex),
+		 le16_to_cpu(setup->wLength));
+
+	/* clean up any leftover transfers */
+	mreq = next_ep0_request(mtu);
+	if (mreq)
+		ep0_req_giveback(mtu, &mreq->request);
+
+	if (le16_to_cpu(setup->wLength) == 0) {
+		;	/* no data stage, nothing to do */
+	} else if (setup->bRequestType & USB_DIR_IN) {
+		mtu3_writel(mtu->mac_base, U3D_EP0CSR,
+			csr | EP0_SETUPPKTRDY | EP0_DPHTX);
+		mtu->ep0_state = MU3D_EP0_STATE_TX;
+	} else {
+		mtu3_writel(mtu->mac_base, U3D_EP0CSR,
+			(csr | EP0_SETUPPKTRDY) & (~EP0_DPHTX));
+		mtu->ep0_state = MU3D_EP0_STATE_RX;
+	}
+}
+
+static int ep0_handle_setup(struct mtu3 *mtu)
+__releases(mtu->lock)
+__acquires(mtu->lock)
+{
+	struct usb_ctrlrequest setup;
+	struct mtu3_request *mreq;
+	void __iomem *mbase = mtu->mac_base;
+	int handled = 0;
+
+	ep0_read_setup(mtu, &setup);
+
+	if ((setup.bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
+		handled = handle_standard_request(mtu, &setup);
+
+	dev_dbg(mtu->dev, "handled %d, ep0_state: %s\n",
+		 handled, decode_ep0_state(mtu));
+
+	if (handled < 0)
+		goto stall;
+	else if (handled > 0)
+		goto finish;
+
+	handled = forward_to_driver(mtu, &setup);
+	if (handled < 0) {
+stall:
+		dev_dbg(mtu->dev, "%s stall (%d)\n", __func__, handled);
+
+		ep0_stall_set(mtu->ep0, true,
+			le16_to_cpu(setup.wLength) ? 0 : EP0_SETUPPKTRDY);
+
+		return 0;
+	}
+
+finish:
+	if (mtu->test_mode) {
+		;	/* nothing to do */
+	} else if (le16_to_cpu(setup.wLength) == 0) { /* no data stage */
+
+		mtu3_writel(mbase, U3D_EP0CSR,
+			(mtu3_readl(mbase, U3D_EP0CSR) & EP0_W1C_BITS)
+			| EP0_SETUPPKTRDY | EP0_DATAEND);
+
+		/* complete zlp request directly */
+		mreq = next_ep0_request(mtu);
+		if (mreq && !mreq->request.length)
+			ep0_req_giveback(mtu, &mreq->request);
+	}
+
+	return 0;
+}
+
+irqreturn_t mtu3_ep0_isr(struct mtu3 *mtu)
+{
+	void __iomem *mbase = mtu->mac_base;
+	struct mtu3_request *mreq;
+	u32 int_status;
+	irqreturn_t ret = IRQ_NONE;
+	u32 csr;
+	u32 len;
+
+	int_status = mtu3_readl(mbase, U3D_EPISR);
+	int_status &= mtu3_readl(mbase, U3D_EPIER);
+	mtu3_writel(mbase, U3D_EPISR, int_status); /* W1C */
+
+	/* only handle ep0's */
+	if (!(int_status & EP0ISR))
+		return IRQ_NONE;
+
+	csr = mtu3_readl(mbase, U3D_EP0CSR);
+
+	dev_dbg(mtu->dev, "%s csr=0x%x\n", __func__, csr);
+
+	/* we sent a stall.. need to clear it now.. */
+	if (csr & EP0_SENTSTALL) {
+		ep0_stall_set(mtu->ep0, false, 0);
+		csr = mtu3_readl(mbase, U3D_EP0CSR);
+		ret = IRQ_HANDLED;
+	}
+	dev_dbg(mtu->dev, "ep0_state: %s\n", decode_ep0_state(mtu));
+
+	switch (mtu->ep0_state) {
+	case MU3D_EP0_STATE_TX:
+		/* irq on clearing txpktrdy */
+		if ((csr & EP0_FIFOFULL) == 0) {
+			ep0_tx_state(mtu);
+			ret = IRQ_HANDLED;
+		}
+		break;
+	case MU3D_EP0_STATE_RX:
+		/* irq on set rxpktrdy */
+		if (csr & EP0_RXPKTRDY) {
+			ep0_rx_state(mtu);
+			ret = IRQ_HANDLED;
+		}
+		break;
+	case MU3D_EP0_STATE_TX_END:
+		mtu3_writel(mbase, U3D_EP0CSR,
+			(csr & EP0_W1C_BITS) | EP0_DATAEND);
+
+		mreq = next_ep0_request(mtu);
+		if (mreq)
+			ep0_req_giveback(mtu, &mreq->request);
+
+		mtu->ep0_state = MU3D_EP0_STATE_SETUP;
+		ret = IRQ_HANDLED;
+		dev_dbg(mtu->dev, "ep0_state: %s\n", decode_ep0_state(mtu));
+		break;
+	case MU3D_EP0_STATE_SETUP:
+		if (!(csr & EP0_SETUPPKTRDY))
+			break;
+
+		len = mtu3_readl(mbase, U3D_RXCOUNT0);
+		if (len != 8) {
+			dev_err(mtu->dev, "SETUP packet len %d != 8 ?\n", len);
+			break;
+		}
+
+		ep0_handle_setup(mtu);
+		ret = IRQ_HANDLED;
+		break;
+	default:
+		/* can't happen */
+		ep0_stall_set(mtu->ep0, true, 0);
+		WARN_ON(1);
+		break;
+	}
+
+	return ret;
+}
+
+
+static int mtu3_ep0_enable(struct usb_ep *ep,
+	const struct usb_endpoint_descriptor *desc)
+{
+	/* always enabled */
+	return -EINVAL;
+}
+
+static int mtu3_ep0_disable(struct usb_ep *ep)
+{
+	/* always enabled */
+	return -EINVAL;
+}
+
+static int ep0_queue(struct mtu3_ep *mep, struct mtu3_request *mreq)
+{
+	struct mtu3 *mtu = mep->mtu;
+
+	mreq->mtu = mtu;
+	mreq->request.actual = 0;
+	mreq->request.status = -EINPROGRESS;
+
+	dev_dbg(mtu->dev, "%s %s (ep0_state: %s), len#%d\n", __func__,
+		mep->name, decode_ep0_state(mtu), mreq->request.length);
+
+	if (!list_empty(&mep->req_list))
+		return -EBUSY;
+
+	switch (mtu->ep0_state) {
+	case MU3D_EP0_STATE_SETUP:
+	case MU3D_EP0_STATE_RX:	/* control-OUT data */
+	case MU3D_EP0_STATE_TX:	/* control-IN data */
+		break;
+	default:
+		dev_err(mtu->dev, "%s, error in ep0 state %s\n", __func__,
+			decode_ep0_state(mtu));
+		return -EINVAL;
+	}
+
+	list_add_tail(&mreq->list, &mep->req_list);
+
+	/* sequence #1, IN ... start writing the data */
+	if (mtu->ep0_state == MU3D_EP0_STATE_TX)
+		ep0_tx_state(mtu);
+
+	return 0;
+}
+
+static int mtu3_ep0_queue(struct usb_ep *ep,
+	struct usb_request *req, gfp_t gfp)
+{
+	struct mtu3_ep *mep;
+	struct mtu3_request *mreq;
+	struct mtu3 *mtu;
+	unsigned long flags;
+	int ret = 0;
+
+	if (!ep || !req)
+		return -EINVAL;
+
+	mep = to_mtu3_ep(ep);
+	mtu = mep->mtu;
+	mreq = to_mtu3_request(req);
+
+	spin_lock_irqsave(&mtu->lock, flags);
+	ret = ep0_queue(mep, mreq);
+	spin_unlock_irqrestore(&mtu->lock, flags);
+	return ret;
+}
+
+static int mtu3_ep0_dequeue(struct usb_ep *ep, struct usb_request *req)
+{
+	/* we just won't support this */
+	return -EINVAL;
+}
+
+static int mtu3_ep0_halt(struct usb_ep *ep, int value)
+{
+	struct mtu3_ep *mep;
+	struct mtu3 *mtu;
+	unsigned long flags;
+	int ret = 0;
+
+	if (!ep || !value)
+		return -EINVAL;
+
+	mep = to_mtu3_ep(ep);
+	mtu = mep->mtu;
+
+	dev_dbg(mtu->dev, "%s\n", __func__);
+
+	spin_lock_irqsave(&mtu->lock, flags);
+
+	if (!list_empty(&mep->req_list)) {
+		ret = -EBUSY;
+		goto cleanup;
+	}
+
+	switch (mtu->ep0_state) {
+	/*
+	 * stalls are usually issued after parsing SETUP packet, either
+	 * directly in irq context from setup() or else later.
+	 */
+	case MU3D_EP0_STATE_TX:
+	case MU3D_EP0_STATE_TX_END:
+	case MU3D_EP0_STATE_RX:
+	case MU3D_EP0_STATE_SETUP:
+		ep0_stall_set(mtu->ep0, true, 0);
+		break;
+	default:
+		dev_dbg(mtu->dev, "ep0 can't halt in state %s\n",
+			decode_ep0_state(mtu));
+		ret = -EINVAL;
+	}
+
+cleanup:
+	spin_unlock_irqrestore(&mtu->lock, flags);
+	return ret;
+}
+
+const struct usb_ep_ops mtu3_ep0_ops = {
+	.enable = mtu3_ep0_enable,
+	.disable = mtu3_ep0_disable,
+	.alloc_request = mtu3_alloc_request,
+	.free_request = mtu3_free_request,
+	.queue = mtu3_ep0_queue,
+	.dequeue = mtu3_ep0_dequeue,
+	.set_halt = mtu3_ep0_halt,
+};
diff --git a/drivers/usb/mtu3/mtu3_host.c b/drivers/usb/mtu3/mtu3_host.c
new file mode 100644
index 000000000000..cd4d01087855
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_host.c
@@ -0,0 +1,294 @@
+/*
+ * mtu3_dr.c - dual role switch and host glue layer
+ *
+ * Copyright (C) 2016 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/iopoll.h>
+#include <linux/irq.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+
+#include "mtu3.h"
+#include "mtu3_dr.h"
+
+#define PERI_WK_CTRL1		0x404
+#define UWK_CTL1_IS_C(x)	(((x) & 0xf) << 26)
+#define UWK_CTL1_IS_E		BIT(25)
+#define UWK_CTL1_IDDIG_C(x)	(((x) & 0xf) << 11)  /* cycle debounce */
+#define UWK_CTL1_IDDIG_E	BIT(10) /* enable debounce */
+#define UWK_CTL1_IDDIG_P	BIT(9)  /* polarity */
+#define UWK_CTL1_IS_P		BIT(6)  /* polarity for ip sleep */
+
+/*
+ * ip-sleep wakeup mode:
+ * all clocks can be turn off, but power domain should be kept on
+ */
+static void ssusb_wakeup_ip_sleep_en(struct ssusb_mtk *ssusb)
+{
+	u32 tmp;
+	struct regmap *pericfg = ssusb->pericfg;
+
+	regmap_read(pericfg, PERI_WK_CTRL1, &tmp);
+	tmp &= ~UWK_CTL1_IS_P;
+	tmp &= ~(UWK_CTL1_IS_C(0xf));
+	tmp |= UWK_CTL1_IS_C(0x8);
+	regmap_write(pericfg, PERI_WK_CTRL1, tmp);
+	regmap_write(pericfg, PERI_WK_CTRL1, tmp | UWK_CTL1_IS_E);
+
+	regmap_read(pericfg, PERI_WK_CTRL1, &tmp);
+	dev_dbg(ssusb->dev, "%s(): WK_CTRL1[P6,E25,C26:29]=%#x\n",
+		__func__, tmp);
+}
+
+static void ssusb_wakeup_ip_sleep_dis(struct ssusb_mtk *ssusb)
+{
+	u32 tmp;
+
+	regmap_read(ssusb->pericfg, PERI_WK_CTRL1, &tmp);
+	tmp &= ~UWK_CTL1_IS_E;
+	regmap_write(ssusb->pericfg, PERI_WK_CTRL1, tmp);
+}
+
+int ssusb_wakeup_of_property_parse(struct ssusb_mtk *ssusb,
+				struct device_node *dn)
+{
+	struct device *dev = ssusb->dev;
+
+	/*
+	 * Wakeup function is optional, so it is not an error if this property
+	 * does not exist, and in such case, no need to get relative
+	 * properties anymore.
+	 */
+	ssusb->wakeup_en = of_property_read_bool(dn, "mediatek,enable-wakeup");
+	if (!ssusb->wakeup_en)
+		return 0;
+
+	ssusb->wk_deb_p0 = devm_clk_get(dev, "wakeup_deb_p0");
+	if (IS_ERR(ssusb->wk_deb_p0)) {
+		dev_err(dev, "fail to get wakeup_deb_p0\n");
+		return PTR_ERR(ssusb->wk_deb_p0);
+	}
+
+	if (of_property_read_bool(dn, "wakeup_deb_p1")) {
+		ssusb->wk_deb_p1 = devm_clk_get(dev, "wakeup_deb_p1");
+		if (IS_ERR(ssusb->wk_deb_p1)) {
+			dev_err(dev, "fail to get wakeup_deb_p1\n");
+			return PTR_ERR(ssusb->wk_deb_p1);
+		}
+	}
+
+	ssusb->pericfg = syscon_regmap_lookup_by_phandle(dn,
+						"mediatek,syscon-wakeup");
+	if (IS_ERR(ssusb->pericfg)) {
+		dev_err(dev, "fail to get pericfg regs\n");
+		return PTR_ERR(ssusb->pericfg);
+	}
+
+	return 0;
+}
+
+static int ssusb_wakeup_clks_enable(struct ssusb_mtk *ssusb)
+{
+	int ret;
+
+	ret = clk_prepare_enable(ssusb->wk_deb_p0);
+	if (ret) {
+		dev_err(ssusb->dev, "failed to enable wk_deb_p0\n");
+		goto usb_p0_err;
+	}
+
+	ret = clk_prepare_enable(ssusb->wk_deb_p1);
+	if (ret) {
+		dev_err(ssusb->dev, "failed to enable wk_deb_p1\n");
+		goto usb_p1_err;
+	}
+
+	return 0;
+
+usb_p1_err:
+	clk_disable_unprepare(ssusb->wk_deb_p0);
+usb_p0_err:
+	return -EINVAL;
+}
+
+static void ssusb_wakeup_clks_disable(struct ssusb_mtk *ssusb)
+{
+	clk_disable_unprepare(ssusb->wk_deb_p1);
+	clk_disable_unprepare(ssusb->wk_deb_p0);
+}
+
+static void host_ports_num_get(struct ssusb_mtk *ssusb)
+{
+	u32 xhci_cap;
+
+	xhci_cap = mtu3_readl(ssusb->ippc_base, U3D_SSUSB_IP_XHCI_CAP);
+	ssusb->u2_ports = SSUSB_IP_XHCI_U2_PORT_NUM(xhci_cap);
+	ssusb->u3_ports = SSUSB_IP_XHCI_U3_PORT_NUM(xhci_cap);
+
+	dev_dbg(ssusb->dev, "host - u2_ports:%d, u3_ports:%d\n",
+		 ssusb->u2_ports, ssusb->u3_ports);
+}
+
+/* only configure ports will be used later */
+int ssusb_host_enable(struct ssusb_mtk *ssusb)
+{
+	void __iomem *ibase = ssusb->ippc_base;
+	int num_u3p = ssusb->u3_ports;
+	int num_u2p = ssusb->u2_ports;
+	u32 check_clk;
+	u32 value;
+	int i;
+
+	/* power on host ip */
+	mtu3_clrbits(ibase, U3D_SSUSB_IP_PW_CTRL1, SSUSB_IP_HOST_PDN);
+
+	/* power on and enable all u3 ports */
+	for (i = 0; i < num_u3p; i++) {
+		value = mtu3_readl(ibase, SSUSB_U3_CTRL(i));
+		value &= ~(SSUSB_U3_PORT_PDN | SSUSB_U3_PORT_DIS);
+		value |= SSUSB_U3_PORT_HOST_SEL;
+		mtu3_writel(ibase, SSUSB_U3_CTRL(i), value);
+	}
+
+	/* power on and enable all u2 ports */
+	for (i = 0; i < num_u2p; i++) {
+		value = mtu3_readl(ibase, SSUSB_U2_CTRL(i));
+		value &= ~(SSUSB_U2_PORT_PDN | SSUSB_U2_PORT_DIS);
+		value |= SSUSB_U2_PORT_HOST_SEL;
+		mtu3_writel(ibase, SSUSB_U2_CTRL(i), value);
+	}
+
+	check_clk = SSUSB_XHCI_RST_B_STS;
+	if (num_u3p)
+		check_clk = SSUSB_U3_MAC_RST_B_STS;
+
+	return ssusb_check_clocks(ssusb, check_clk);
+}
+
+int ssusb_host_disable(struct ssusb_mtk *ssusb, bool suspend)
+{
+	void __iomem *ibase = ssusb->ippc_base;
+	int num_u3p = ssusb->u3_ports;
+	int num_u2p = ssusb->u2_ports;
+	u32 value;
+	int ret;
+	int i;
+
+	/* power down and disable all u3 ports */
+	for (i = 0; i < num_u3p; i++) {
+		value = mtu3_readl(ibase, SSUSB_U3_CTRL(i));
+		value |= SSUSB_U3_PORT_PDN;
+		value |= suspend ? 0 : SSUSB_U3_PORT_DIS;
+		mtu3_writel(ibase, SSUSB_U3_CTRL(i), value);
+	}
+
+	/* power down and disable all u2 ports */
+	for (i = 0; i < num_u2p; i++) {
+		value = mtu3_readl(ibase, SSUSB_U2_CTRL(i));
+		value |= SSUSB_U2_PORT_PDN;
+		value |= suspend ? 0 : SSUSB_U2_PORT_DIS;
+		mtu3_writel(ibase, SSUSB_U2_CTRL(i), value);
+	}
+
+	/* power down host ip */
+	mtu3_setbits(ibase, U3D_SSUSB_IP_PW_CTRL1, SSUSB_IP_HOST_PDN);
+
+	if (!suspend)
+		return 0;
+
+	/* wait for host ip to sleep */
+	ret = readl_poll_timeout(ibase + U3D_SSUSB_IP_PW_STS1, value,
+			  (value & SSUSB_IP_SLEEP_STS), 100, 100000);
+	if (ret)
+		dev_err(ssusb->dev, "ip sleep failed!!!\n");
+
+	return ret;
+}
+
+static void ssusb_host_setup(struct ssusb_mtk *ssusb)
+{
+	host_ports_num_get(ssusb);
+
+	/*
+	 * power on host and power on/enable all ports
+	 * if support OTG, gadget driver will switch port0 to device mode
+	 */
+	ssusb_host_enable(ssusb);
+
+	/* if port0 supports dual-role, works as host mode by default */
+	ssusb_set_vbus(&ssusb->otg_switch, 1);
+}
+
+static void ssusb_host_cleanup(struct ssusb_mtk *ssusb)
+{
+	if (ssusb->is_host)
+		ssusb_set_vbus(&ssusb->otg_switch, 0);
+
+	ssusb_host_disable(ssusb, false);
+}
+
+/*
+ * If host supports multiple ports, the VBUSes(5V) of ports except port0
+ * which supports OTG are better to be enabled by default in DTS.
+ * Because the host driver will keep link with devices attached when system
+ * enters suspend mode, so no need to control VBUSes after initialization.
+ */
+int ssusb_host_init(struct ssusb_mtk *ssusb, struct device_node *parent_dn)
+{
+	struct device *parent_dev = ssusb->dev;
+	int ret;
+
+	ssusb_host_setup(ssusb);
+
+	ret = of_platform_populate(parent_dn, NULL, NULL, parent_dev);
+	if (ret) {
+		dev_dbg(parent_dev, "failed to create child devices at %s\n",
+				parent_dn->full_name);
+		return ret;
+	}
+
+	dev_info(parent_dev, "xHCI platform device register success...\n");
+
+	return 0;
+}
+
+void ssusb_host_exit(struct ssusb_mtk *ssusb)
+{
+	of_platform_depopulate(ssusb->dev);
+	ssusb_host_cleanup(ssusb);
+}
+
+int ssusb_wakeup_enable(struct ssusb_mtk *ssusb)
+{
+	int ret = 0;
+
+	if (ssusb->wakeup_en) {
+		ret = ssusb_wakeup_clks_enable(ssusb);
+		ssusb_wakeup_ip_sleep_en(ssusb);
+	}
+	return ret;
+}
+
+void ssusb_wakeup_disable(struct ssusb_mtk *ssusb)
+{
+	if (ssusb->wakeup_en) {
+		ssusb_wakeup_ip_sleep_dis(ssusb);
+		ssusb_wakeup_clks_disable(ssusb);
+	}
+}
diff --git a/drivers/usb/mtu3/mtu3_hw_regs.h b/drivers/usb/mtu3/mtu3_hw_regs.h
new file mode 100644
index 000000000000..212367295276
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_hw_regs.h
@@ -0,0 +1,473 @@
+/*
+ * mtu3_hw_regs.h - MediaTek USB3 DRD register and field definitions
+ *
+ * Copyright (C) 2016 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef _SSUSB_HW_REGS_H_
+#define _SSUSB_HW_REGS_H_
+
+/* segment offset of MAC register */
+#define SSUSB_DEV_BASE		0x0000
+#define SSUSB_EPCTL_CSR_BASE	0x0800
+#define SSUSB_USB3_MAC_CSR_BASE	0x1400
+#define SSUSB_USB3_SYS_CSR_BASE	0x1400
+#define SSUSB_USB2_CSR_BASE	0x2400
+
+/* IPPC register in Infra */
+#define SSUSB_SIFSLV_IPPC_BASE	0x0000
+
+/* --------------- SSUSB_DEV REGISTER DEFINITION --------------- */
+
+#define U3D_LV1ISR		(SSUSB_DEV_BASE + 0x0000)
+#define U3D_LV1IER		(SSUSB_DEV_BASE + 0x0004)
+#define U3D_LV1IESR		(SSUSB_DEV_BASE + 0x0008)
+#define U3D_LV1IECR		(SSUSB_DEV_BASE + 0x000C)
+
+#define U3D_EPISR		(SSUSB_DEV_BASE + 0x0080)
+#define U3D_EPIER		(SSUSB_DEV_BASE + 0x0084)
+#define U3D_EPIESR		(SSUSB_DEV_BASE + 0x0088)
+#define U3D_EPIECR		(SSUSB_DEV_BASE + 0x008C)
+
+#define U3D_EP0CSR		(SSUSB_DEV_BASE + 0x0100)
+#define U3D_RXCOUNT0		(SSUSB_DEV_BASE + 0x0108)
+#define U3D_RESERVED		(SSUSB_DEV_BASE + 0x010C)
+#define U3D_TX1CSR0		(SSUSB_DEV_BASE + 0x0110)
+#define U3D_TX1CSR1		(SSUSB_DEV_BASE + 0x0114)
+#define U3D_TX1CSR2		(SSUSB_DEV_BASE + 0x0118)
+
+#define U3D_RX1CSR0		(SSUSB_DEV_BASE + 0x0210)
+#define U3D_RX1CSR1		(SSUSB_DEV_BASE + 0x0214)
+#define U3D_RX1CSR2		(SSUSB_DEV_BASE + 0x0218)
+
+#define U3D_FIFO0		(SSUSB_DEV_BASE + 0x0300)
+
+#define U3D_QCR0		(SSUSB_DEV_BASE + 0x0400)
+#define U3D_QCR1		(SSUSB_DEV_BASE + 0x0404)
+#define U3D_QCR2		(SSUSB_DEV_BASE + 0x0408)
+#define U3D_QCR3		(SSUSB_DEV_BASE + 0x040C)
+
+#define U3D_TXQCSR1		(SSUSB_DEV_BASE + 0x0510)
+#define U3D_TXQSAR1		(SSUSB_DEV_BASE + 0x0514)
+#define U3D_TXQCPR1		(SSUSB_DEV_BASE + 0x0518)
+
+#define U3D_RXQCSR1		(SSUSB_DEV_BASE + 0x0610)
+#define U3D_RXQSAR1		(SSUSB_DEV_BASE + 0x0614)
+#define U3D_RXQCPR1		(SSUSB_DEV_BASE + 0x0618)
+#define U3D_RXQLDPR1		(SSUSB_DEV_BASE + 0x061C)
+
+#define U3D_QISAR0		(SSUSB_DEV_BASE + 0x0700)
+#define U3D_QIER0		(SSUSB_DEV_BASE + 0x0704)
+#define U3D_QIESR0		(SSUSB_DEV_BASE + 0x0708)
+#define U3D_QIECR0		(SSUSB_DEV_BASE + 0x070C)
+#define U3D_QISAR1		(SSUSB_DEV_BASE + 0x0710)
+#define U3D_QIER1		(SSUSB_DEV_BASE + 0x0714)
+#define U3D_QIESR1		(SSUSB_DEV_BASE + 0x0718)
+#define U3D_QIECR1		(SSUSB_DEV_BASE + 0x071C)
+
+#define U3D_TQERRIR0		(SSUSB_DEV_BASE + 0x0780)
+#define U3D_TQERRIER0		(SSUSB_DEV_BASE + 0x0784)
+#define U3D_TQERRIESR0		(SSUSB_DEV_BASE + 0x0788)
+#define U3D_TQERRIECR0		(SSUSB_DEV_BASE + 0x078C)
+#define U3D_RQERRIR0		(SSUSB_DEV_BASE + 0x07C0)
+#define U3D_RQERRIER0		(SSUSB_DEV_BASE + 0x07C4)
+#define U3D_RQERRIESR0		(SSUSB_DEV_BASE + 0x07C8)
+#define U3D_RQERRIECR0		(SSUSB_DEV_BASE + 0x07CC)
+#define U3D_RQERRIR1		(SSUSB_DEV_BASE + 0x07D0)
+#define U3D_RQERRIER1		(SSUSB_DEV_BASE + 0x07D4)
+#define U3D_RQERRIESR1		(SSUSB_DEV_BASE + 0x07D8)
+#define U3D_RQERRIECR1		(SSUSB_DEV_BASE + 0x07DC)
+
+#define U3D_CAP_EP0FFSZ		(SSUSB_DEV_BASE + 0x0C04)
+#define U3D_CAP_EPNTXFFSZ	(SSUSB_DEV_BASE + 0x0C08)
+#define U3D_CAP_EPNRXFFSZ	(SSUSB_DEV_BASE + 0x0C0C)
+#define U3D_CAP_EPINFO		(SSUSB_DEV_BASE + 0x0C10)
+#define U3D_MISC_CTRL		(SSUSB_DEV_BASE + 0x0C84)
+
+/*---------------- SSUSB_DEV FIELD DEFINITION ---------------*/
+
+/* U3D_LV1ISR */
+#define EP_CTRL_INTR		BIT(5)
+#define MAC2_INTR		BIT(4)
+#define DMA_INTR		BIT(3)
+#define MAC3_INTR		BIT(2)
+#define QMU_INTR		BIT(1)
+#define BMU_INTR		BIT(0)
+
+/* U3D_LV1IECR */
+#define LV1IECR_MSK		GENMASK(31, 0)
+
+/* U3D_EPISR */
+#define EPRISR(x)		(BIT(16) << (x))
+#define EPTISR(x)		(BIT(0) << (x))
+#define EP0ISR			BIT(0)
+
+/* U3D_EP0CSR */
+#define EP0_SENDSTALL		BIT(25)
+#define EP0_FIFOFULL		BIT(23)
+#define EP0_SENTSTALL		BIT(22)
+#define EP0_DPHTX		BIT(20)
+#define EP0_DATAEND		BIT(19)
+#define EP0_TXPKTRDY		BIT(18)
+#define EP0_SETUPPKTRDY		BIT(17)
+#define EP0_RXPKTRDY		BIT(16)
+#define EP0_MAXPKTSZ_MSK	GENMASK(9, 0)
+#define EP0_MAXPKTSZ(x)		((x) & EP0_MAXPKTSZ_MSK)
+#define EP0_W1C_BITS	(~(EP0_RXPKTRDY | EP0_SETUPPKTRDY | EP0_SENTSTALL))
+
+/* U3D_TX1CSR0 */
+#define TX_DMAREQEN		BIT(29)
+#define TX_FIFOFULL		BIT(25)
+#define TX_FIFOEMPTY		BIT(24)
+#define TX_SENTSTALL		BIT(22)
+#define TX_SENDSTALL		BIT(21)
+#define TX_TXPKTRDY		BIT(16)
+#define TX_TXMAXPKTSZ_MSK	GENMASK(10, 0)
+#define TX_TXMAXPKTSZ(x)	((x) & TX_TXMAXPKTSZ_MSK)
+#define TX_W1C_BITS		(~(TX_SENTSTALL))
+
+/* U3D_TX1CSR1 */
+#define TX_MULT(x)		(((x) & 0x3) << 22)
+#define TX_MAX_PKT(x)		(((x) & 0x3f) << 16)
+#define TX_SLOT(x)		(((x) & 0x3f) << 8)
+#define TX_TYPE(x)		(((x) & 0x3) << 4)
+#define TX_SS_BURST(x)		(((x) & 0xf) << 0)
+
+/* for TX_TYPE & RX_TYPE */
+#define TYPE_BULK		(0x0)
+#define TYPE_INT		(0x1)
+#define TYPE_ISO		(0x2)
+#define TYPE_MASK		(0x3)
+
+/* U3D_TX1CSR2 */
+#define TX_BINTERVAL(x)		(((x) & 0xff) << 24)
+#define TX_FIFOSEGSIZE(x)	(((x) & 0xf) << 16)
+#define TX_FIFOADDR(x)		(((x) & 0x1fff) << 0)
+
+/* U3D_RX1CSR0 */
+#define RX_DMAREQEN		BIT(29)
+#define RX_SENTSTALL		BIT(22)
+#define RX_SENDSTALL		BIT(21)
+#define RX_RXPKTRDY		BIT(16)
+#define RX_RXMAXPKTSZ_MSK	GENMASK(10, 0)
+#define RX_RXMAXPKTSZ(x)	((x) & RX_RXMAXPKTSZ_MSK)
+#define RX_W1C_BITS		(~(RX_SENTSTALL | RX_RXPKTRDY))
+
+/* U3D_RX1CSR1 */
+#define RX_MULT(x)		(((x) & 0x3) << 22)
+#define RX_MAX_PKT(x)		(((x) & 0x3f) << 16)
+#define RX_SLOT(x)		(((x) & 0x3f) << 8)
+#define RX_TYPE(x)		(((x) & 0x3) << 4)
+#define RX_SS_BURST(x)		(((x) & 0xf) << 0)
+
+/* U3D_RX1CSR2 */
+#define RX_BINTERVAL(x)		(((x) & 0xff) << 24)
+#define RX_FIFOSEGSIZE(x)	(((x) & 0xf) << 16)
+#define RX_FIFOADDR(x)		(((x) & 0x1fff) << 0)
+
+/* U3D_QCR0 */
+#define QMU_RX_CS_EN(x)		(BIT(16) << (x))
+#define QMU_TX_CS_EN(x)		(BIT(0) << (x))
+#define QMU_CS16B_EN		BIT(0)
+
+/* U3D_QCR1 */
+#define QMU_TX_ZLP(x)		(BIT(0) << (x))
+
+/* U3D_QCR3 */
+#define QMU_RX_COZ(x)		(BIT(16) << (x))
+#define QMU_RX_ZLP(x)		(BIT(0) << (x))
+
+/* U3D_TXQCSR1 */
+/* U3D_RXQCSR1 */
+#define QMU_Q_ACTIVE		BIT(15)
+#define QMU_Q_STOP		BIT(2)
+#define QMU_Q_RESUME		BIT(1)
+#define QMU_Q_START		BIT(0)
+
+/* U3D_QISAR0, U3D_QIER0, U3D_QIESR0, U3D_QIECR0 */
+#define QMU_RX_DONE_INT(x)	(BIT(16) << (x))
+#define QMU_TX_DONE_INT(x)	(BIT(0) << (x))
+
+/* U3D_QISAR1, U3D_QIER1, U3D_QIESR1, U3D_QIECR1 */
+#define RXQ_ZLPERR_INT		BIT(20)
+#define RXQ_LENERR_INT		BIT(18)
+#define RXQ_CSERR_INT		BIT(17)
+#define RXQ_EMPTY_INT		BIT(16)
+#define TXQ_LENERR_INT		BIT(2)
+#define TXQ_CSERR_INT		BIT(1)
+#define TXQ_EMPTY_INT		BIT(0)
+
+/* U3D_TQERRIR0, U3D_TQERRIER0, U3D_TQERRIESR0, U3D_TQERRIECR0 */
+#define QMU_TX_LEN_ERR(x)	(BIT(16) << (x))
+#define QMU_TX_CS_ERR(x)	(BIT(0) << (x))
+
+/* U3D_RQERRIR0, U3D_RQERRIER0, U3D_RQERRIESR0, U3D_RQERRIECR0 */
+#define QMU_RX_LEN_ERR(x)	(BIT(16) << (x))
+#define QMU_RX_CS_ERR(x)	(BIT(0) << (x))
+
+/* U3D_RQERRIR1, U3D_RQERRIER1, U3D_RQERRIESR1, U3D_RQERRIECR1 */
+#define QMU_RX_ZLP_ERR(n)	(BIT(16) << (n))
+
+/* U3D_CAP_EPINFO */
+#define CAP_RX_EP_NUM(x)	(((x) >> 8) & 0x1f)
+#define CAP_TX_EP_NUM(x)	((x) & 0x1f)
+
+/* U3D_MISC_CTRL */
+#define VBUS_ON			BIT(1)
+#define VBUS_FRC_EN		BIT(0)
+
+
+/*---------------- SSUSB_EPCTL_CSR REGISTER DEFINITION ----------------*/
+
+#define U3D_DEVICE_CONF			(SSUSB_EPCTL_CSR_BASE + 0x0000)
+#define U3D_EP_RST			(SSUSB_EPCTL_CSR_BASE + 0x0004)
+
+#define U3D_DEV_LINK_INTR_ENABLE	(SSUSB_EPCTL_CSR_BASE + 0x0050)
+#define U3D_DEV_LINK_INTR		(SSUSB_EPCTL_CSR_BASE + 0x0054)
+
+/*---------------- SSUSB_EPCTL_CSR FIELD DEFINITION ----------------*/
+
+/* U3D_DEVICE_CONF */
+#define DEV_ADDR_MSK		GENMASK(30, 24)
+#define DEV_ADDR(x)		((0x7f & (x)) << 24)
+#define HW_USB2_3_SEL		BIT(18)
+#define SW_USB2_3_SEL_EN	BIT(17)
+#define SW_USB2_3_SEL		BIT(16)
+#define SSUSB_DEV_SPEED(x)	((x) & 0x7)
+
+/* U3D_EP_RST */
+#define EP1_IN_RST		BIT(17)
+#define EP1_OUT_RST		BIT(1)
+#define EP_RST(is_in, epnum)	(((is_in) ? BIT(16) : BIT(0)) << (epnum))
+#define EP0_RST			BIT(0)
+
+/* U3D_DEV_LINK_INTR_ENABLE */
+/* U3D_DEV_LINK_INTR */
+#define SSUSB_DEV_SPEED_CHG_INTR	BIT(0)
+
+
+/*---------------- SSUSB_USB3_MAC_CSR REGISTER DEFINITION ----------------*/
+
+#define U3D_LTSSM_CTRL		(SSUSB_USB3_MAC_CSR_BASE + 0x0010)
+#define U3D_USB3_CONFIG		(SSUSB_USB3_MAC_CSR_BASE + 0x001C)
+
+#define U3D_LTSSM_INTR_ENABLE	(SSUSB_USB3_MAC_CSR_BASE + 0x013C)
+#define U3D_LTSSM_INTR		(SSUSB_USB3_MAC_CSR_BASE + 0x0140)
+
+/*---------------- SSUSB_USB3_MAC_CSR FIELD DEFINITION ----------------*/
+
+/* U3D_LTSSM_CTRL */
+#define FORCE_POLLING_FAIL	BIT(4)
+#define FORCE_RXDETECT_FAIL	BIT(3)
+#define SOFT_U3_EXIT_EN		BIT(2)
+#define COMPLIANCE_EN		BIT(1)
+#define U1_GO_U2_EN		BIT(0)
+
+/* U3D_USB3_CONFIG */
+#define USB3_EN			BIT(0)
+
+/* U3D_LTSSM_INTR_ENABLE */
+/* U3D_LTSSM_INTR */
+#define U3_RESUME_INTR		BIT(18)
+#define U3_LFPS_TMOUT_INTR	BIT(17)
+#define VBUS_FALL_INTR		BIT(16)
+#define VBUS_RISE_INTR		BIT(15)
+#define RXDET_SUCCESS_INTR	BIT(14)
+#define EXIT_U3_INTR		BIT(13)
+#define EXIT_U2_INTR		BIT(12)
+#define EXIT_U1_INTR		BIT(11)
+#define ENTER_U3_INTR		BIT(10)
+#define ENTER_U2_INTR		BIT(9)
+#define ENTER_U1_INTR		BIT(8)
+#define ENTER_U0_INTR		BIT(7)
+#define RECOVERY_INTR		BIT(6)
+#define WARM_RST_INTR		BIT(5)
+#define HOT_RST_INTR		BIT(4)
+#define LOOPBACK_INTR		BIT(3)
+#define COMPLIANCE_INTR		BIT(2)
+#define SS_DISABLE_INTR		BIT(1)
+#define SS_INACTIVE_INTR	BIT(0)
+
+/*---------------- SSUSB_USB3_SYS_CSR REGISTER DEFINITION ----------------*/
+
+#define U3D_LINK_UX_INACT_TIMER	(SSUSB_USB3_SYS_CSR_BASE + 0x020C)
+#define U3D_LINK_POWER_CONTROL	(SSUSB_USB3_SYS_CSR_BASE + 0x0210)
+#define U3D_LINK_ERR_COUNT	(SSUSB_USB3_SYS_CSR_BASE + 0x0214)
+
+/*---------------- SSUSB_USB3_SYS_CSR FIELD DEFINITION ----------------*/
+
+/* U3D_LINK_UX_INACT_TIMER */
+#define DEV_U2_INACT_TIMEOUT_MSK	GENMASK(23, 16)
+#define DEV_U2_INACT_TIMEOUT_VALUE(x)	(((x) & 0xff) << 16)
+#define U2_INACT_TIMEOUT_MSK		GENMASK(15, 8)
+#define U1_INACT_TIMEOUT_MSK		GENMASK(7, 0)
+#define U1_INACT_TIMEOUT_VALUE(x)	((x) & 0xff)
+
+/* U3D_LINK_POWER_CONTROL */
+#define SW_U2_ACCEPT_ENABLE	BIT(9)
+#define SW_U1_ACCEPT_ENABLE	BIT(8)
+#define UX_EXIT			BIT(5)
+#define LGO_U3			BIT(4)
+#define LGO_U2			BIT(3)
+#define LGO_U1			BIT(2)
+#define SW_U2_REQUEST_ENABLE	BIT(1)
+#define SW_U1_REQUEST_ENABLE	BIT(0)
+
+/* U3D_LINK_ERR_COUNT */
+#define CLR_LINK_ERR_CNT	BIT(16)
+#define LINK_ERROR_COUNT	GENMASK(15, 0)
+
+/*---------------- SSUSB_USB2_CSR REGISTER DEFINITION ----------------*/
+
+#define U3D_POWER_MANAGEMENT		(SSUSB_USB2_CSR_BASE + 0x0004)
+#define U3D_DEVICE_CONTROL		(SSUSB_USB2_CSR_BASE + 0x000C)
+#define U3D_USB2_TEST_MODE		(SSUSB_USB2_CSR_BASE + 0x0014)
+#define U3D_COMMON_USB_INTR_ENABLE	(SSUSB_USB2_CSR_BASE + 0x0018)
+#define U3D_COMMON_USB_INTR		(SSUSB_USB2_CSR_BASE + 0x001C)
+#define U3D_LINK_RESET_INFO		(SSUSB_USB2_CSR_BASE + 0x0024)
+#define U3D_USB20_FRAME_NUM		(SSUSB_USB2_CSR_BASE + 0x003C)
+#define U3D_USB20_LPM_PARAMETER		(SSUSB_USB2_CSR_BASE + 0x0044)
+#define U3D_USB20_MISC_CONTROL		(SSUSB_USB2_CSR_BASE + 0x004C)
+
+/*---------------- SSUSB_USB2_CSR FIELD DEFINITION ----------------*/
+
+/* U3D_POWER_MANAGEMENT */
+#define LPM_BESL_STALL		BIT(14)
+#define LPM_BESLD_STALL		BIT(13)
+#define LPM_RWP			BIT(11)
+#define LPM_HRWE		BIT(10)
+#define LPM_MODE(x)		(((x) & 0x3) << 8)
+#define ISO_UPDATE		BIT(7)
+#define SOFT_CONN		BIT(6)
+#define HS_ENABLE		BIT(5)
+#define RESUME			BIT(2)
+#define SUSPENDM_ENABLE		BIT(0)
+
+/* U3D_DEVICE_CONTROL */
+#define DC_HOSTREQ		BIT(1)
+#define DC_SESSION		BIT(0)
+
+/* U3D_USB2_TEST_MODE */
+#define U2U3_AUTO_SWITCH	BIT(10)
+#define LPM_FORCE_STALL		BIT(8)
+#define FIFO_ACCESS		BIT(6)
+#define FORCE_FS		BIT(5)
+#define FORCE_HS		BIT(4)
+#define TEST_PACKET_MODE	BIT(3)
+#define TEST_K_MODE		BIT(2)
+#define TEST_J_MODE		BIT(1)
+#define TEST_SE0_NAK_MODE	BIT(0)
+
+/* U3D_COMMON_USB_INTR_ENABLE */
+/* U3D_COMMON_USB_INTR */
+#define LPM_RESUME_INTR		BIT(9)
+#define LPM_INTR		BIT(8)
+#define DISCONN_INTR		BIT(5)
+#define CONN_INTR		BIT(4)
+#define SOF_INTR		BIT(3)
+#define RESET_INTR		BIT(2)
+#define RESUME_INTR		BIT(1)
+#define SUSPEND_INTR		BIT(0)
+
+/* U3D_LINK_RESET_INFO */
+#define WTCHRP_MSK		GENMASK(19, 16)
+
+/* U3D_USB20_LPM_PARAMETER */
+#define LPM_BESLCK_U3(x)	(((x) & 0xf) << 12)
+#define LPM_BESLCK(x)		(((x) & 0xf) << 8)
+#define LPM_BESLDCK(x)		(((x) & 0xf) << 4)
+#define LPM_BESL		GENMASK(3, 0)
+
+/* U3D_USB20_MISC_CONTROL */
+#define LPM_U3_ACK_EN		BIT(0)
+
+/*---------------- SSUSB_SIFSLV_IPPC REGISTER DEFINITION ----------------*/
+
+#define U3D_SSUSB_IP_PW_CTRL0	(SSUSB_SIFSLV_IPPC_BASE + 0x0000)
+#define U3D_SSUSB_IP_PW_CTRL1	(SSUSB_SIFSLV_IPPC_BASE + 0x0004)
+#define U3D_SSUSB_IP_PW_CTRL2	(SSUSB_SIFSLV_IPPC_BASE + 0x0008)
+#define U3D_SSUSB_IP_PW_CTRL3	(SSUSB_SIFSLV_IPPC_BASE + 0x000C)
+#define U3D_SSUSB_IP_PW_STS1	(SSUSB_SIFSLV_IPPC_BASE + 0x0010)
+#define U3D_SSUSB_IP_PW_STS2	(SSUSB_SIFSLV_IPPC_BASE + 0x0014)
+#define U3D_SSUSB_OTG_STS	(SSUSB_SIFSLV_IPPC_BASE + 0x0018)
+#define U3D_SSUSB_OTG_STS_CLR	(SSUSB_SIFSLV_IPPC_BASE + 0x001C)
+#define U3D_SSUSB_IP_XHCI_CAP	(SSUSB_SIFSLV_IPPC_BASE + 0x0024)
+#define U3D_SSUSB_IP_DEV_CAP	(SSUSB_SIFSLV_IPPC_BASE + 0x0028)
+#define U3D_SSUSB_OTG_INT_EN	(SSUSB_SIFSLV_IPPC_BASE + 0x002C)
+#define U3D_SSUSB_U3_CTRL_0P	(SSUSB_SIFSLV_IPPC_BASE + 0x0030)
+#define U3D_SSUSB_U2_CTRL_0P	(SSUSB_SIFSLV_IPPC_BASE + 0x0050)
+#define U3D_SSUSB_REF_CK_CTRL	(SSUSB_SIFSLV_IPPC_BASE + 0x008C)
+#define U3D_SSUSB_DEV_RST_CTRL	(SSUSB_SIFSLV_IPPC_BASE + 0x0098)
+#define U3D_SSUSB_HW_ID		(SSUSB_SIFSLV_IPPC_BASE + 0x00A0)
+#define U3D_SSUSB_HW_SUB_ID	(SSUSB_SIFSLV_IPPC_BASE + 0x00A4)
+#define U3D_SSUSB_IP_SPARE0	(SSUSB_SIFSLV_IPPC_BASE + 0x00C8)
+
+/*---------------- SSUSB_SIFSLV_IPPC FIELD DEFINITION ----------------*/
+
+/* U3D_SSUSB_IP_PW_CTRL0 */
+#define SSUSB_IP_SW_RST			BIT(0)
+
+/* U3D_SSUSB_IP_PW_CTRL1 */
+#define SSUSB_IP_HOST_PDN		BIT(0)
+
+/* U3D_SSUSB_IP_PW_CTRL2 */
+#define SSUSB_IP_DEV_PDN		BIT(0)
+
+/* U3D_SSUSB_IP_PW_CTRL3 */
+#define SSUSB_IP_PCIE_PDN		BIT(0)
+
+/* U3D_SSUSB_IP_PW_STS1 */
+#define SSUSB_IP_SLEEP_STS		BIT(30)
+#define SSUSB_U3_MAC_RST_B_STS		BIT(16)
+#define SSUSB_XHCI_RST_B_STS		BIT(11)
+#define SSUSB_SYS125_RST_B_STS		BIT(10)
+#define SSUSB_REF_RST_B_STS		BIT(8)
+#define SSUSB_SYSPLL_STABLE		BIT(0)
+
+/* U3D_SSUSB_IP_PW_STS2 */
+#define SSUSB_U2_MAC_SYS_RST_B_STS	BIT(0)
+
+/* U3D_SSUSB_OTG_STS */
+#define SSUSB_VBUS_VALID		BIT(9)
+
+/* U3D_SSUSB_OTG_STS_CLR */
+#define SSUSB_VBUS_INTR_CLR		BIT(6)
+
+/* U3D_SSUSB_IP_XHCI_CAP */
+#define SSUSB_IP_XHCI_U2_PORT_NUM(x)	(((x) >> 8) & 0xff)
+#define SSUSB_IP_XHCI_U3_PORT_NUM(x)	((x) & 0xff)
+
+/* U3D_SSUSB_IP_DEV_CAP */
+#define SSUSB_IP_DEV_U3_PORT_NUM(x)	((x) & 0xff)
+
+/* U3D_SSUSB_OTG_INT_EN */
+#define SSUSB_VBUS_CHG_INT_A_EN		BIT(7)
+#define SSUSB_VBUS_CHG_INT_B_EN		BIT(6)
+
+/* U3D_SSUSB_U3_CTRL_0P */
+#define SSUSB_U3_PORT_HOST_SEL		BIT(2)
+#define SSUSB_U3_PORT_PDN		BIT(1)
+#define SSUSB_U3_PORT_DIS		BIT(0)
+
+/* U3D_SSUSB_U2_CTRL_0P */
+#define SSUSB_U2_PORT_OTG_SEL		BIT(7)
+#define SSUSB_U2_PORT_HOST_SEL		BIT(2)
+#define SSUSB_U2_PORT_PDN		BIT(1)
+#define SSUSB_U2_PORT_DIS		BIT(0)
+
+/* U3D_SSUSB_DEV_RST_CTRL */
+#define SSUSB_DEV_SW_RST		BIT(0)
+
+#endif	/* _SSUSB_HW_REGS_H_ */
diff --git a/drivers/usb/mtu3/mtu3_plat.c b/drivers/usb/mtu3/mtu3_plat.c
new file mode 100644
index 000000000000..783367805c99
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_plat.c
@@ -0,0 +1,484 @@
+/*
+ * Copyright (C) 2016 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/platform_device.h>
+
+#include "mtu3.h"
+#include "mtu3_dr.h"
+
+/* u2-port0 should be powered on and enabled; */
+int ssusb_check_clocks(struct ssusb_mtk *ssusb, u32 ex_clks)
+{
+	void __iomem *ibase = ssusb->ippc_base;
+	u32 value, check_val;
+	int ret;
+
+	check_val = ex_clks | SSUSB_SYS125_RST_B_STS | SSUSB_SYSPLL_STABLE |
+			SSUSB_REF_RST_B_STS;
+
+	ret = readl_poll_timeout(ibase + U3D_SSUSB_IP_PW_STS1, value,
+			(check_val == (value & check_val)), 100, 20000);
+	if (ret) {
+		dev_err(ssusb->dev, "clks of sts1 are not stable!\n");
+		return ret;
+	}
+
+	ret = readl_poll_timeout(ibase + U3D_SSUSB_IP_PW_STS2, value,
+			(value & SSUSB_U2_MAC_SYS_RST_B_STS), 100, 10000);
+	if (ret) {
+		dev_err(ssusb->dev, "mac2 clock is not stable\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ssusb_phy_init(struct ssusb_mtk *ssusb)
+{
+	int i;
+	int ret;
+
+	for (i = 0; i < ssusb->num_phys; i++) {
+		ret = phy_init(ssusb->phys[i]);
+		if (ret)
+			goto exit_phy;
+	}
+	return 0;
+
+exit_phy:
+	for (; i > 0; i--)
+		phy_exit(ssusb->phys[i - 1]);
+
+	return ret;
+}
+
+static int ssusb_phy_exit(struct ssusb_mtk *ssusb)
+{
+	int i;
+
+	for (i = 0; i < ssusb->num_phys; i++)
+		phy_exit(ssusb->phys[i]);
+
+	return 0;
+}
+
+static int ssusb_phy_power_on(struct ssusb_mtk *ssusb)
+{
+	int i;
+	int ret;
+
+	for (i = 0; i < ssusb->num_phys; i++) {
+		ret = phy_power_on(ssusb->phys[i]);
+		if (ret)
+			goto power_off_phy;
+	}
+	return 0;
+
+power_off_phy:
+	for (; i > 0; i--)
+		phy_power_off(ssusb->phys[i - 1]);
+
+	return ret;
+}
+
+static void ssusb_phy_power_off(struct ssusb_mtk *ssusb)
+{
+	unsigned int i;
+
+	for (i = 0; i < ssusb->num_phys; i++)
+		phy_power_off(ssusb->phys[i]);
+}
+
+static int ssusb_rscs_init(struct ssusb_mtk *ssusb)
+{
+	int ret = 0;
+
+	ret = regulator_enable(ssusb->vusb33);
+	if (ret) {
+		dev_err(ssusb->dev, "failed to enable vusb33\n");
+		goto vusb33_err;
+	}
+
+	ret = clk_prepare_enable(ssusb->sys_clk);
+	if (ret) {
+		dev_err(ssusb->dev, "failed to enable sys_clk\n");
+		goto clk_err;
+	}
+
+	ret = ssusb_phy_init(ssusb);
+	if (ret) {
+		dev_err(ssusb->dev, "failed to init phy\n");
+		goto phy_init_err;
+	}
+
+	ret = ssusb_phy_power_on(ssusb);
+	if (ret) {
+		dev_err(ssusb->dev, "failed to power on phy\n");
+		goto phy_err;
+	}
+
+	return 0;
+
+phy_err:
+	ssusb_phy_exit(ssusb);
+phy_init_err:
+	clk_disable_unprepare(ssusb->sys_clk);
+clk_err:
+	regulator_disable(ssusb->vusb33);
+vusb33_err:
+
+	return ret;
+}
+
+static void ssusb_rscs_exit(struct ssusb_mtk *ssusb)
+{
+	clk_disable_unprepare(ssusb->sys_clk);
+	regulator_disable(ssusb->vusb33);
+	ssusb_phy_power_off(ssusb);
+	ssusb_phy_exit(ssusb);
+}
+
+static void ssusb_ip_sw_reset(struct ssusb_mtk *ssusb)
+{
+	/* reset whole ip (xhci & u3d) */
+	mtu3_setbits(ssusb->ippc_base, U3D_SSUSB_IP_PW_CTRL0, SSUSB_IP_SW_RST);
+	udelay(1);
+	mtu3_clrbits(ssusb->ippc_base, U3D_SSUSB_IP_PW_CTRL0, SSUSB_IP_SW_RST);
+}
+
+static int get_iddig_pinctrl(struct ssusb_mtk *ssusb)
+{
+	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
+
+	otg_sx->id_pinctrl = devm_pinctrl_get(ssusb->dev);
+	if (IS_ERR(otg_sx->id_pinctrl)) {
+		dev_err(ssusb->dev, "Cannot find id pinctrl!\n");
+		return PTR_ERR(otg_sx->id_pinctrl);
+	}
+
+	otg_sx->id_float =
+		pinctrl_lookup_state(otg_sx->id_pinctrl, "id_float");
+	if (IS_ERR(otg_sx->id_float)) {
+		dev_err(ssusb->dev, "Cannot find pinctrl id_float!\n");
+		return PTR_ERR(otg_sx->id_float);
+	}
+
+	otg_sx->id_ground =
+		pinctrl_lookup_state(otg_sx->id_pinctrl, "id_ground");
+	if (IS_ERR(otg_sx->id_ground)) {
+		dev_err(ssusb->dev, "Cannot find pinctrl id_ground!\n");
+		return PTR_ERR(otg_sx->id_ground);
+	}
+
+	return 0;
+}
+
+static int get_ssusb_rscs(struct platform_device *pdev, struct ssusb_mtk *ssusb)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
+	struct device *dev = &pdev->dev;
+	struct regulator *vbus;
+	struct resource *res;
+	int i;
+	int ret;
+
+	ssusb->num_phys = of_count_phandle_with_args(node,
+			"phys", "#phy-cells");
+	if (ssusb->num_phys > 0) {
+		ssusb->phys = devm_kcalloc(dev, ssusb->num_phys,
+					sizeof(*ssusb->phys), GFP_KERNEL);
+		if (!ssusb->phys)
+			return -ENOMEM;
+	} else {
+		ssusb->num_phys = 0;
+	}
+
+	for (i = 0; i < ssusb->num_phys; i++) {
+		ssusb->phys[i] = devm_of_phy_get_by_index(dev, node, i);
+		if (IS_ERR(ssusb->phys[i])) {
+			dev_err(dev, "failed to get phy-%d\n", i);
+			return PTR_ERR(ssusb->phys[i]);
+		}
+	}
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ippc");
+	ssusb->ippc_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(ssusb->ippc_base)) {
+		dev_err(dev, "failed to map memory for ippc\n");
+		return PTR_ERR(ssusb->ippc_base);
+	}
+
+	ssusb->vusb33 = devm_regulator_get(&pdev->dev, "vusb33");
+	if (IS_ERR(ssusb->vusb33)) {
+		dev_err(dev, "failed to get vusb33\n");
+		return PTR_ERR(ssusb->vusb33);
+	}
+
+	ssusb->sys_clk = devm_clk_get(dev, "sys_ck");
+	if (IS_ERR(ssusb->sys_clk)) {
+		dev_err(dev, "failed to get sys clock\n");
+		return PTR_ERR(ssusb->sys_clk);
+	}
+
+	ssusb->dr_mode = usb_get_dr_mode(dev);
+	if (ssusb->dr_mode == USB_DR_MODE_UNKNOWN) {
+		dev_err(dev, "dr_mode is error\n");
+		return -EINVAL;
+	}
+
+	if (ssusb->dr_mode == USB_DR_MODE_PERIPHERAL)
+		return 0;
+
+	/* if host role is supported */
+	ret = ssusb_wakeup_of_property_parse(ssusb, node);
+	if (ret)
+		return ret;
+
+	if (ssusb->dr_mode != USB_DR_MODE_OTG)
+		return 0;
+
+	/* if dual-role mode is supported */
+	vbus = devm_regulator_get(&pdev->dev, "vbus");
+	if (IS_ERR(vbus)) {
+		dev_err(dev, "failed to get vbus\n");
+		return PTR_ERR(vbus);
+	}
+	otg_sx->vbus = vbus;
+
+	otg_sx->is_u3_drd = of_property_read_bool(node, "mediatek,usb3-drd");
+	otg_sx->manual_drd_enabled =
+		of_property_read_bool(node, "enable-manual-drd");
+
+	if (of_property_read_bool(node, "extcon")) {
+		otg_sx->edev = extcon_get_edev_by_phandle(ssusb->dev, 0);
+		if (IS_ERR(otg_sx->edev)) {
+			dev_err(ssusb->dev, "couldn't get extcon device\n");
+			return -EPROBE_DEFER;
+		}
+		if (otg_sx->manual_drd_enabled) {
+			ret = get_iddig_pinctrl(ssusb);
+			if (ret)
+				return ret;
+		}
+	}
+
+	dev_info(dev, "dr_mode: %d, is_u3_dr: %d\n",
+		ssusb->dr_mode, otg_sx->is_u3_drd);
+
+	return 0;
+}
+
+static int mtu3_probe(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
+	struct ssusb_mtk *ssusb;
+	int ret = -ENOMEM;
+
+	/* all elements are set to ZERO as default value */
+	ssusb = devm_kzalloc(dev, sizeof(*ssusb), GFP_KERNEL);
+	if (!ssusb)
+		return -ENOMEM;
+
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	if (ret) {
+		dev_err(dev, "No suitable DMA config available\n");
+		return -ENOTSUPP;
+	}
+
+	platform_set_drvdata(pdev, ssusb);
+	ssusb->dev = dev;
+
+	ret = get_ssusb_rscs(pdev, ssusb);
+	if (ret)
+		return ret;
+
+	/* enable power domain */
+	pm_runtime_enable(dev);
+	pm_runtime_get_sync(dev);
+	device_enable_async_suspend(dev);
+
+	ret = ssusb_rscs_init(ssusb);
+	if (ret)
+		goto comm_init_err;
+
+	ssusb_ip_sw_reset(ssusb);
+
+	if (IS_ENABLED(CONFIG_USB_MTU3_HOST))
+		ssusb->dr_mode = USB_DR_MODE_HOST;
+	else if (IS_ENABLED(CONFIG_USB_MTU3_GADGET))
+		ssusb->dr_mode = USB_DR_MODE_PERIPHERAL;
+
+	/* default as host */
+	ssusb->is_host = !(ssusb->dr_mode == USB_DR_MODE_PERIPHERAL);
+
+	switch (ssusb->dr_mode) {
+	case USB_DR_MODE_PERIPHERAL:
+		ret = ssusb_gadget_init(ssusb);
+		if (ret) {
+			dev_err(dev, "failed to initialize gadget\n");
+			goto comm_exit;
+		}
+		break;
+	case USB_DR_MODE_HOST:
+		ret = ssusb_host_init(ssusb, node);
+		if (ret) {
+			dev_err(dev, "failed to initialize host\n");
+			goto comm_exit;
+		}
+		break;
+	case USB_DR_MODE_OTG:
+		ret = ssusb_gadget_init(ssusb);
+		if (ret) {
+			dev_err(dev, "failed to initialize gadget\n");
+			goto comm_exit;
+		}
+
+		ret = ssusb_host_init(ssusb, node);
+		if (ret) {
+			dev_err(dev, "failed to initialize host\n");
+			goto gadget_exit;
+		}
+
+		ssusb_otg_switch_init(ssusb);
+		break;
+	default:
+		dev_err(dev, "unsupported mode: %d\n", ssusb->dr_mode);
+		ret = -EINVAL;
+		goto comm_exit;
+	}
+
+	return 0;
+
+gadget_exit:
+	ssusb_gadget_exit(ssusb);
+comm_exit:
+	ssusb_rscs_exit(ssusb);
+comm_init_err:
+	pm_runtime_put_sync(dev);
+	pm_runtime_disable(dev);
+
+	return ret;
+}
+
+static int mtu3_remove(struct platform_device *pdev)
+{
+	struct ssusb_mtk *ssusb = platform_get_drvdata(pdev);
+
+	switch (ssusb->dr_mode) {
+	case USB_DR_MODE_PERIPHERAL:
+		ssusb_gadget_exit(ssusb);
+		break;
+	case USB_DR_MODE_HOST:
+		ssusb_host_exit(ssusb);
+		break;
+	case USB_DR_MODE_OTG:
+		ssusb_otg_switch_exit(ssusb);
+		ssusb_gadget_exit(ssusb);
+		ssusb_host_exit(ssusb);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ssusb_rscs_exit(ssusb);
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+/*
+ * when support dual-role mode, we reject suspend when
+ * it works as device mode;
+ */
+static int __maybe_unused mtu3_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct ssusb_mtk *ssusb = platform_get_drvdata(pdev);
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	/* REVISIT: disconnect it for only device mode? */
+	if (!ssusb->is_host)
+		return 0;
+
+	ssusb_host_disable(ssusb, true);
+	ssusb_phy_power_off(ssusb);
+	clk_disable_unprepare(ssusb->sys_clk);
+	ssusb_wakeup_enable(ssusb);
+
+	return 0;
+}
+
+static int __maybe_unused mtu3_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct ssusb_mtk *ssusb = platform_get_drvdata(pdev);
+
+	dev_dbg(dev, "%s\n", __func__);
+
+	if (!ssusb->is_host)
+		return 0;
+
+	ssusb_wakeup_disable(ssusb);
+	clk_prepare_enable(ssusb->sys_clk);
+	ssusb_phy_power_on(ssusb);
+	ssusb_host_enable(ssusb);
+
+	return 0;
+}
+
+static const struct dev_pm_ops mtu3_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(mtu3_suspend, mtu3_resume)
+};
+
+#define DEV_PM_OPS (IS_ENABLED(CONFIG_PM) ? &mtu3_pm_ops : NULL)
+
+#ifdef CONFIG_OF
+
+static const struct of_device_id mtu3_of_match[] = {
+	{.compatible = "mediatek,mt8173-mtu3",},
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, mtu3_of_match);
+
+#endif
+
+static struct platform_driver mtu3_driver = {
+	.probe = mtu3_probe,
+	.remove = mtu3_remove,
+	.driver = {
+		.name = MTU3_DRIVER_NAME,
+		.pm = DEV_PM_OPS,
+		.of_match_table = of_match_ptr(mtu3_of_match),
+	},
+};
+module_platform_driver(mtu3_driver);
+
+MODULE_AUTHOR("Chunfeng Yun <chunfeng.yun@mediatek.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("MediaTek USB3 DRD Controller Driver");
diff --git a/drivers/usb/mtu3/mtu3_qmu.c b/drivers/usb/mtu3/mtu3_qmu.c
new file mode 100644
index 000000000000..7d9ba8a52368
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_qmu.c
@@ -0,0 +1,573 @@
+/*
+ * mtu3_qmu.c - Queue Management Unit driver for device controller
+ *
+ * Copyright (C) 2016 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+/*
+ * Queue Management Unit (QMU) is designed to unload SW effort
+ * to serve DMA interrupts.
+ * By preparing General Purpose Descriptor (GPD) and Buffer Descriptor (BD),
+ * SW links data buffers and triggers QMU to send / receive data to
+ * host / from device at a time.
+ * And now only GPD is supported.
+ *
+ * For more detailed information, please refer to QMU Programming Guide
+ */
+
+#include <linux/dmapool.h>
+#include <linux/iopoll.h>
+
+#include "mtu3.h"
+
+#define QMU_CHECKSUM_LEN	16
+
+#define GPD_FLAGS_HWO	BIT(0)
+#define GPD_FLAGS_BDP	BIT(1)
+#define GPD_FLAGS_BPS	BIT(2)
+#define GPD_FLAGS_IOC	BIT(7)
+
+#define GPD_EXT_FLAG_ZLP	BIT(5)
+
+
+static struct qmu_gpd *gpd_dma_to_virt(struct mtu3_gpd_ring *ring,
+		dma_addr_t dma_addr)
+{
+	dma_addr_t dma_base = ring->dma;
+	struct qmu_gpd *gpd_head = ring->start;
+	u32 offset = (dma_addr - dma_base) / sizeof(*gpd_head);
+
+	if (offset >= MAX_GPD_NUM)
+		return NULL;
+
+	return gpd_head + offset;
+}
+
+static dma_addr_t gpd_virt_to_dma(struct mtu3_gpd_ring *ring,
+		struct qmu_gpd *gpd)
+{
+	dma_addr_t dma_base = ring->dma;
+	struct qmu_gpd *gpd_head = ring->start;
+	u32 offset;
+
+	offset = gpd - gpd_head;
+	if (offset >= MAX_GPD_NUM)
+		return 0;
+
+	return dma_base + (offset * sizeof(*gpd));
+}
+
+static void gpd_ring_init(struct mtu3_gpd_ring *ring, struct qmu_gpd *gpd)
+{
+	ring->start = gpd;
+	ring->enqueue = gpd;
+	ring->dequeue = gpd;
+	ring->end = gpd + MAX_GPD_NUM - 1;
+}
+
+static void reset_gpd_list(struct mtu3_ep *mep)
+{
+	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
+	struct qmu_gpd *gpd = ring->start;
+
+	if (gpd) {
+		gpd->flag &= ~GPD_FLAGS_HWO;
+		gpd_ring_init(ring, gpd);
+	}
+}
+
+int mtu3_gpd_ring_alloc(struct mtu3_ep *mep)
+{
+	struct qmu_gpd *gpd;
+	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
+
+	/* software own all gpds as default */
+	gpd = dma_pool_zalloc(mep->mtu->qmu_gpd_pool, GFP_ATOMIC, &ring->dma);
+	if (gpd == NULL)
+		return -ENOMEM;
+
+	gpd_ring_init(ring, gpd);
+
+	return 0;
+}
+
+void mtu3_gpd_ring_free(struct mtu3_ep *mep)
+{
+	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
+
+	dma_pool_free(mep->mtu->qmu_gpd_pool,
+			ring->start, ring->dma);
+	memset(ring, 0, sizeof(*ring));
+}
+
+/*
+ * calculate check sum of a gpd or bd
+ * add "noinline" and "mb" to prevent wrong calculation
+ */
+static noinline u8 qmu_calc_checksum(u8 *data)
+{
+	u8 chksum = 0;
+	int i;
+
+	data[1] = 0x0;  /* set checksum to 0 */
+
+	mb();	/* ensure the gpd/bd is really up-to-date */
+	for (i = 0; i < QMU_CHECKSUM_LEN; i++)
+		chksum += data[i];
+
+	/* Default: HWO=1, @flag[bit0] */
+	chksum += 1;
+
+	return 0xFF - chksum;
+}
+
+void mtu3_qmu_resume(struct mtu3_ep *mep)
+{
+	struct mtu3 *mtu = mep->mtu;
+	void __iomem *mbase = mtu->mac_base;
+	int epnum = mep->epnum;
+	u32 offset;
+
+	offset = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);
+
+	mtu3_writel(mbase, offset, QMU_Q_RESUME);
+	if (!(mtu3_readl(mbase, offset) & QMU_Q_ACTIVE))
+		mtu3_writel(mbase, offset, QMU_Q_RESUME);
+}
+
+static struct qmu_gpd *advance_enq_gpd(struct mtu3_gpd_ring *ring)
+{
+	if (ring->enqueue < ring->end)
+		ring->enqueue++;
+	else
+		ring->enqueue = ring->start;
+
+	return ring->enqueue;
+}
+
+static struct qmu_gpd *advance_deq_gpd(struct mtu3_gpd_ring *ring)
+{
+	if (ring->dequeue < ring->end)
+		ring->dequeue++;
+	else
+		ring->dequeue = ring->start;
+
+	return ring->dequeue;
+}
+
+/* check if a ring is emtpy */
+static int gpd_ring_empty(struct mtu3_gpd_ring *ring)
+{
+	struct qmu_gpd *enq = ring->enqueue;
+	struct qmu_gpd *next;
+
+	if (ring->enqueue < ring->end)
+		next = enq + 1;
+	else
+		next = ring->start;
+
+	/* one gpd is reserved to simplify gpd preparation */
+	return next == ring->dequeue;
+}
+
+int mtu3_prepare_transfer(struct mtu3_ep *mep)
+{
+	return gpd_ring_empty(&mep->gpd_ring);
+}
+
+static int mtu3_prepare_tx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
+{
+	struct qmu_gpd *enq;
+	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
+	struct qmu_gpd *gpd = ring->enqueue;
+	struct usb_request *req = &mreq->request;
+
+	/* set all fields to zero as default value */
+	memset(gpd, 0, sizeof(*gpd));
+
+	gpd->buffer = cpu_to_le32((u32)req->dma);
+	gpd->buf_len = cpu_to_le16(req->length);
+	gpd->flag |= GPD_FLAGS_IOC;
+
+	/* get the next GPD */
+	enq = advance_enq_gpd(ring);
+	dev_dbg(mep->mtu->dev, "TX-EP%d queue gpd=%p, enq=%p\n",
+		mep->epnum, gpd, enq);
+
+	enq->flag &= ~GPD_FLAGS_HWO;
+	gpd->next_gpd = cpu_to_le32((u32)gpd_virt_to_dma(ring, enq));
+
+	if (req->zero)
+		gpd->ext_flag |= GPD_EXT_FLAG_ZLP;
+
+	gpd->chksum = qmu_calc_checksum((u8 *)gpd);
+	gpd->flag |= GPD_FLAGS_HWO;
+
+	mreq->gpd = gpd;
+
+	return 0;
+}
+
+static int mtu3_prepare_rx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
+{
+	struct qmu_gpd *enq;
+	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
+	struct qmu_gpd *gpd = ring->enqueue;
+	struct usb_request *req = &mreq->request;
+
+	/* set all fields to zero as default value */
+	memset(gpd, 0, sizeof(*gpd));
+
+	gpd->buffer = cpu_to_le32((u32)req->dma);
+	gpd->data_buf_len = cpu_to_le16(req->length);
+	gpd->flag |= GPD_FLAGS_IOC;
+
+	/* get the next GPD */
+	enq = advance_enq_gpd(ring);
+	dev_dbg(mep->mtu->dev, "RX-EP%d queue gpd=%p, enq=%p\n",
+		mep->epnum, gpd, enq);
+
+	enq->flag &= ~GPD_FLAGS_HWO;
+	gpd->next_gpd = cpu_to_le32((u32)gpd_virt_to_dma(ring, enq));
+	gpd->chksum = qmu_calc_checksum((u8 *)gpd);
+	gpd->flag |= GPD_FLAGS_HWO;
+
+	mreq->gpd = gpd;
+
+	return 0;
+}
+
+void mtu3_insert_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
+{
+
+	if (mep->is_in)
+		mtu3_prepare_tx_gpd(mep, mreq);
+	else
+		mtu3_prepare_rx_gpd(mep, mreq);
+}
+
+int mtu3_qmu_start(struct mtu3_ep *mep)
+{
+	struct mtu3 *mtu = mep->mtu;
+	void __iomem *mbase = mtu->mac_base;
+	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
+	u8 epnum = mep->epnum;
+
+	if (mep->is_in) {
+		/* set QMU start address */
+		mtu3_writel(mbase, USB_QMU_TQSAR(mep->epnum), ring->dma);
+		mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);
+		mtu3_setbits(mbase, U3D_QCR0, QMU_TX_CS_EN(epnum));
+		/* send zero length packet according to ZLP flag in GPD */
+		mtu3_setbits(mbase, U3D_QCR1, QMU_TX_ZLP(epnum));
+		mtu3_writel(mbase, U3D_TQERRIESR0,
+				QMU_TX_LEN_ERR(epnum) | QMU_TX_CS_ERR(epnum));
+
+		if (mtu3_readl(mbase, USB_QMU_TQCSR(epnum)) & QMU_Q_ACTIVE) {
+			dev_warn(mtu->dev, "Tx %d Active Now!\n", epnum);
+			return 0;
+		}
+		mtu3_writel(mbase, USB_QMU_TQCSR(epnum), QMU_Q_START);
+
+	} else {
+		mtu3_writel(mbase, USB_QMU_RQSAR(mep->epnum), ring->dma);
+		mtu3_setbits(mbase, MU3D_EP_RXCR0(mep->epnum), RX_DMAREQEN);
+		mtu3_setbits(mbase, U3D_QCR0, QMU_RX_CS_EN(epnum));
+		/* don't expect ZLP */
+		mtu3_clrbits(mbase, U3D_QCR3, QMU_RX_ZLP(epnum));
+		/* move to next GPD when receive ZLP */
+		mtu3_setbits(mbase, U3D_QCR3, QMU_RX_COZ(epnum));
+		mtu3_writel(mbase, U3D_RQERRIESR0,
+				QMU_RX_LEN_ERR(epnum) | QMU_RX_CS_ERR(epnum));
+		mtu3_writel(mbase, U3D_RQERRIESR1, QMU_RX_ZLP_ERR(epnum));
+
+		if (mtu3_readl(mbase, USB_QMU_RQCSR(epnum)) & QMU_Q_ACTIVE) {
+			dev_warn(mtu->dev, "Rx %d Active Now!\n", epnum);
+			return 0;
+		}
+		mtu3_writel(mbase, USB_QMU_RQCSR(epnum), QMU_Q_START);
+	}
+
+	return 0;
+}
+
+/* may called in atomic context */
+void mtu3_qmu_stop(struct mtu3_ep *mep)
+{
+	struct mtu3 *mtu = mep->mtu;
+	void __iomem *mbase = mtu->mac_base;
+	int epnum = mep->epnum;
+	u32 value = 0;
+	u32 qcsr;
+	int ret;
+
+	qcsr = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);
+
+	if (!(mtu3_readl(mbase, qcsr) & QMU_Q_ACTIVE)) {
+		dev_dbg(mtu->dev, "%s's qmu is inactive now!\n", mep->name);
+		return;
+	}
+	mtu3_writel(mbase, qcsr, QMU_Q_STOP);
+
+	ret = readl_poll_timeout_atomic(mbase + qcsr, value,
+			!(value & QMU_Q_ACTIVE), 1, 1000);
+	if (ret) {
+		dev_err(mtu->dev, "stop %s's qmu failed\n", mep->name);
+		return;
+	}
+
+	dev_dbg(mtu->dev, "%s's qmu stop now!\n", mep->name);
+}
+
+void mtu3_qmu_flush(struct mtu3_ep *mep)
+{
+
+	dev_dbg(mep->mtu->dev, "%s flush QMU %s\n", __func__,
+		((mep->is_in) ? "TX" : "RX"));
+
+	/*Stop QMU */
+	mtu3_qmu_stop(mep);
+	reset_gpd_list(mep);
+}
+
+/*
+ * QMU can't transfer zero length packet directly (a hardware limit
+ * on old SoCs), so when needs to send ZLP, we intentionally trigger
+ * a length error interrupt, and in the ISR sends a ZLP by BMU.
+ */
+static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum)
+{
+	struct mtu3_ep *mep = mtu->in_eps + epnum;
+	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
+	void __iomem *mbase = mtu->mac_base;
+	struct qmu_gpd *gpd_current = NULL;
+	dma_addr_t gpd_dma = mtu3_readl(mbase, USB_QMU_TQCPR(epnum));
+	struct usb_request *req = NULL;
+	struct mtu3_request *mreq;
+	u32 txcsr = 0;
+	int ret;
+
+	mreq = next_request(mep);
+	if (mreq && mreq->request.length == 0)
+		req = &mreq->request;
+	else
+		return;
+
+	gpd_current = gpd_dma_to_virt(ring, gpd_dma);
+
+	if (le16_to_cpu(gpd_current->buf_len) != 0) {
+		dev_err(mtu->dev, "TX EP%d buffer length error(!=0)\n", epnum);
+		return;
+	}
+
+	dev_dbg(mtu->dev, "%s send ZLP for req=%p\n", __func__, mreq);
+
+	mtu3_clrbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);
+
+	ret = readl_poll_timeout_atomic(mbase + MU3D_EP_TXCR0(mep->epnum),
+			txcsr, !(txcsr & TX_FIFOFULL), 1, 1000);
+	if (ret) {
+		dev_err(mtu->dev, "%s wait for fifo empty fail\n", __func__);
+		return;
+	}
+	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_TXPKTRDY);
+
+	/* by pass the current GDP */
+	gpd_current->flag |= GPD_FLAGS_BPS;
+	gpd_current->chksum = qmu_calc_checksum((u8 *)gpd_current);
+	gpd_current->flag |= GPD_FLAGS_HWO;
+
+	/*enable DMAREQEN, switch back to QMU mode */
+	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);
+	mtu3_qmu_resume(mep);
+}
+
+/*
+ * NOTE: request list maybe is already empty as following case:
+ * queue_tx --> qmu_interrupt(clear interrupt pending, schedule tasklet)-->
+ * queue_tx --> process_tasklet(meanwhile, the second one is transferred,
+ * tasklet process both of them)-->qmu_interrupt for second one.
+ * To avoid upper case, put qmu_done_tx in ISR directly to process it.
+ */
+static void qmu_done_tx(struct mtu3 *mtu, u8 epnum)
+{
+	struct mtu3_ep *mep = mtu->in_eps + epnum;
+	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
+	void __iomem *mbase = mtu->mac_base;
+	struct qmu_gpd *gpd = ring->dequeue;
+	struct qmu_gpd *gpd_current = NULL;
+	dma_addr_t gpd_dma = mtu3_readl(mbase, USB_QMU_TQCPR(epnum));
+	struct usb_request *request = NULL;
+	struct mtu3_request *mreq;
+
+	/*transfer phy address got from QMU register to virtual address */
+	gpd_current = gpd_dma_to_virt(ring, gpd_dma);
+
+	dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
+		__func__, epnum, gpd, gpd_current, ring->enqueue);
+
+	while (gpd != gpd_current && !(gpd->flag & GPD_FLAGS_HWO)) {
+
+		mreq = next_request(mep);
+
+		if (mreq == NULL || mreq->gpd != gpd) {
+			dev_err(mtu->dev, "no correct TX req is found\n");
+			break;
+		}
+
+		request = &mreq->request;
+		request->actual = le16_to_cpu(gpd->buf_len);
+		mtu3_req_complete(mep, request, 0);
+
+		gpd = advance_deq_gpd(ring);
+	}
+
+	dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",
+		__func__, epnum, ring->dequeue, ring->enqueue);
+
+}
+
+static void qmu_done_rx(struct mtu3 *mtu, u8 epnum)
+{
+	struct mtu3_ep *mep = mtu->out_eps + epnum;
+	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
+	void __iomem *mbase = mtu->mac_base;
+	struct qmu_gpd *gpd = ring->dequeue;
+	struct qmu_gpd *gpd_current = NULL;
+	dma_addr_t gpd_dma = mtu3_readl(mbase, USB_QMU_RQCPR(epnum));
+	struct usb_request *req = NULL;
+	struct mtu3_request *mreq;
+
+	gpd_current = gpd_dma_to_virt(ring, gpd_dma);
+
+	dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
+		__func__, epnum, gpd, gpd_current, ring->enqueue);
+
+	while (gpd != gpd_current && !(gpd->flag & GPD_FLAGS_HWO)) {
+
+		mreq = next_request(mep);
+
+		if (mreq == NULL || mreq->gpd != gpd) {
+			dev_err(mtu->dev, "no correct RX req is found\n");
+			break;
+		}
+		req = &mreq->request;
+
+		req->actual = le16_to_cpu(gpd->buf_len);
+		mtu3_req_complete(mep, req, 0);
+
+		gpd = advance_deq_gpd(ring);
+	}
+
+	dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",
+		__func__, epnum, ring->dequeue, ring->enqueue);
+}
+
+static void qmu_done_isr(struct mtu3 *mtu, u32 done_status)
+{
+	int i;
+
+	for (i = 1; i < mtu->num_eps; i++) {
+		if (done_status & QMU_RX_DONE_INT(i))
+			qmu_done_rx(mtu, i);
+		if (done_status & QMU_TX_DONE_INT(i))
+			qmu_done_tx(mtu, i);
+	}
+}
+
+static void qmu_exception_isr(struct mtu3 *mtu, u32 qmu_status)
+{
+	void __iomem *mbase = mtu->mac_base;
+	u32 errval;
+	int i;
+
+	if ((qmu_status & RXQ_CSERR_INT) || (qmu_status & RXQ_LENERR_INT)) {
+		errval = mtu3_readl(mbase, U3D_RQERRIR0);
+		for (i = 1; i < mtu->num_eps; i++) {
+			if (errval & QMU_RX_CS_ERR(i))
+				dev_err(mtu->dev, "Rx %d CS error!\n", i);
+
+			if (errval & QMU_RX_LEN_ERR(i))
+				dev_err(mtu->dev, "RX %d Length error\n", i);
+		}
+		mtu3_writel(mbase, U3D_RQERRIR0, errval);
+	}
+
+	if (qmu_status & RXQ_ZLPERR_INT) {
+		errval = mtu3_readl(mbase, U3D_RQERRIR1);
+		for (i = 1; i < mtu->num_eps; i++) {
+			if (errval & QMU_RX_ZLP_ERR(i))
+				dev_dbg(mtu->dev, "RX EP%d Recv ZLP\n", i);
+		}
+		mtu3_writel(mbase, U3D_RQERRIR1, errval);
+	}
+
+	if ((qmu_status & TXQ_CSERR_INT) || (qmu_status & TXQ_LENERR_INT)) {
+		errval = mtu3_readl(mbase, U3D_TQERRIR0);
+		for (i = 1; i < mtu->num_eps; i++) {
+			if (errval & QMU_TX_CS_ERR(i))
+				dev_err(mtu->dev, "Tx %d checksum error!\n", i);
+
+			if (errval & QMU_TX_LEN_ERR(i))
+				qmu_tx_zlp_error_handler(mtu, i);
+		}
+		mtu3_writel(mbase, U3D_TQERRIR0, errval);
+	}
+}
+
+irqreturn_t mtu3_qmu_isr(struct mtu3 *mtu)
+{
+	void __iomem *mbase = mtu->mac_base;
+	u32 qmu_status;
+	u32 qmu_done_status;
+
+	/* U3D_QISAR1 is read update */
+	qmu_status = mtu3_readl(mbase, U3D_QISAR1);
+	qmu_status &= mtu3_readl(mbase, U3D_QIER1);
+
+	qmu_done_status = mtu3_readl(mbase, U3D_QISAR0);
+	qmu_done_status &= mtu3_readl(mbase, U3D_QIER0);
+	mtu3_writel(mbase, U3D_QISAR0, qmu_done_status); /* W1C */
+	dev_dbg(mtu->dev, "=== QMUdone[tx=%x, rx=%x] QMUexp[%x] ===\n",
+		(qmu_done_status & 0xFFFF), qmu_done_status >> 16,
+		qmu_status);
+
+	if (qmu_done_status)
+		qmu_done_isr(mtu, qmu_done_status);
+
+	if (qmu_status)
+		qmu_exception_isr(mtu, qmu_status);
+
+	return IRQ_HANDLED;
+}
+
+int mtu3_qmu_init(struct mtu3 *mtu)
+{
+
+	compiletime_assert(QMU_GPD_SIZE == 16, "QMU_GPD size SHOULD be 16B");
+
+	mtu->qmu_gpd_pool = dma_pool_create("QMU_GPD", mtu->dev,
+			QMU_GPD_RING_SIZE, QMU_GPD_SIZE, 0);
+
+	if (!mtu->qmu_gpd_pool)
+		return -ENOMEM;
+
+	return 0;
+}
+
+void mtu3_qmu_exit(struct mtu3 *mtu)
+{
+	dma_pool_destroy(mtu->qmu_gpd_pool);
+}
diff --git a/drivers/usb/mtu3/mtu3_qmu.h b/drivers/usb/mtu3/mtu3_qmu.h
new file mode 100644
index 000000000000..4dafa16bf120
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_qmu.h
@@ -0,0 +1,43 @@
+/*
+ * mtu3_qmu.h - Queue Management Unit driver header
+ *
+ * Copyright (C) 2016 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef __MTK_QMU_H__
+#define __MTK_QMU_H__
+
+#define MAX_GPD_NUM		64
+#define QMU_GPD_SIZE		(sizeof(struct qmu_gpd))
+#define QMU_GPD_RING_SIZE	(MAX_GPD_NUM * QMU_GPD_SIZE)
+
+#define GPD_BUF_SIZE		65532
+
+void mtu3_qmu_stop(struct mtu3_ep *mep);
+int mtu3_qmu_start(struct mtu3_ep *mep);
+void mtu3_qmu_resume(struct mtu3_ep *mep);
+void mtu3_qmu_flush(struct mtu3_ep *mep);
+
+void mtu3_insert_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq);
+int mtu3_prepare_transfer(struct mtu3_ep *mep);
+
+int mtu3_gpd_ring_alloc(struct mtu3_ep *mep);
+void mtu3_gpd_ring_free(struct mtu3_ep *mep);
+
+irqreturn_t mtu3_qmu_isr(struct mtu3 *mtu);
+int mtu3_qmu_init(struct mtu3 *mtu);
+void mtu3_qmu_exit(struct mtu3 *mtu);
+
+#endif
diff --git a/drivers/usb/musb/da8xx.c b/drivers/usb/musb/da8xx.c
index 2440f88e07a3..e89708d839e5 100644
--- a/drivers/usb/musb/da8xx.c
+++ b/drivers/usb/musb/da8xx.c
@@ -6,6 +6,9 @@
  * Based on the DaVinci "glue layer" code.
  * Copyright (C) 2005-2006 by Texas Instruments
  *
+ * DT support
+ * Copyright (c) 2016 Petr Kulhavy <petr@barix.com>
+ *
  * This file is part of the Inventra Controller Driver for Linux.
  *
  * The Inventra Controller Driver for Linux is free software; you
@@ -340,6 +343,13 @@ static int da8xx_musb_set_mode(struct musb *musb, u8 musb_mode)
 	struct da8xx_glue *glue = dev_get_drvdata(musb->controller->parent);
 	enum phy_mode phy_mode;
 
+	/*
+	 * The PHY has some issues when it is forced in device or host mode.
+	 * Unless the user request another mode, configure the PHY in OTG mode.
+	 */
+	if (!musb->is_initialized)
+		return phy_set_mode(glue->phy, PHY_MODE_USB_OTG);
+
 	switch (musb_mode) {
 	case MUSB_HOST:		/* Force VBUS valid, ID = 0 */
 		phy_mode = PHY_MODE_USB_HOST;
@@ -366,6 +376,12 @@ static int da8xx_musb_init(struct musb *musb)
 
 	musb->mregs += DA8XX_MENTOR_CORE_OFFSET;
 
+	ret = clk_prepare_enable(glue->clk);
+	if (ret) {
+		dev_err(glue->dev, "failed to enable clock\n");
+		return ret;
+	}
+
 	/* Returns zero if e.g. not clocked */
 	rev = musb_readl(reg_base, DA8XX_USB_REVISION_REG);
 	if (!rev)
@@ -377,12 +393,6 @@ static int da8xx_musb_init(struct musb *musb)
 		goto fail;
 	}
 
-	ret = clk_prepare_enable(glue->clk);
-	if (ret) {
-		dev_err(glue->dev, "failed to enable clock\n");
-		goto fail;
-	}
-
 	setup_timer(&otg_workaround, otg_timer, (unsigned long)musb);
 
 	/* Reset the controller */
@@ -392,7 +402,7 @@ static int da8xx_musb_init(struct musb *musb)
 	ret = phy_init(glue->phy);
 	if (ret) {
 		dev_err(glue->dev, "Failed to init phy.\n");
-		goto err_phy_init;
+		goto fail;
 	}
 
 	ret = phy_power_on(glue->phy);
@@ -412,9 +422,8 @@ static int da8xx_musb_init(struct musb *musb)
 
 err_phy_power_on:
 	phy_exit(glue->phy);
-err_phy_init:
-	clk_disable_unprepare(glue->clk);
 fail:
+	clk_disable_unprepare(glue->clk);
 	return ret;
 }
 
@@ -433,6 +442,21 @@ static int da8xx_musb_exit(struct musb *musb)
 	return 0;
 }
 
+static inline u8 get_vbus_power(struct device *dev)
+{
+	struct regulator *vbus_supply;
+	int current_uA;
+
+	vbus_supply = regulator_get_optional(dev, "vbus");
+	if (IS_ERR(vbus_supply))
+		return 255;
+	current_uA = regulator_get_current_limit(vbus_supply);
+	regulator_put(vbus_supply);
+	if (current_uA <= 0 || current_uA > 510000)
+		return 255;
+	return current_uA / 1000 / 2;
+}
+
 static const struct musb_platform_ops da8xx_ops = {
 	.quirks		= MUSB_DMA_CPPI | MUSB_INDEXED_EP,
 	.init		= da8xx_musb_init,
@@ -458,6 +482,12 @@ static const struct platform_device_info da8xx_dev_info = {
 	.dma_mask	= DMA_BIT_MASK(32),
 };
 
+static const struct musb_hdrc_config da8xx_config = {
+	.ram_bits = 10,
+	.num_eps = 5,
+	.multipoint = 1,
+};
+
 static int da8xx_probe(struct platform_device *pdev)
 {
 	struct resource musb_resources[2];
@@ -465,6 +495,7 @@ static int da8xx_probe(struct platform_device *pdev)
 	struct da8xx_glue		*glue;
 	struct platform_device_info	pinfo;
 	struct clk			*clk;
+	struct device_node		*np = pdev->dev.of_node;
 	int				ret;
 
 	glue = devm_kzalloc(&pdev->dev, sizeof(*glue), GFP_KERNEL);
@@ -487,6 +518,16 @@ static int da8xx_probe(struct platform_device *pdev)
 	glue->dev			= &pdev->dev;
 	glue->clk			= clk;
 
+	if (IS_ENABLED(CONFIG_OF) && np) {
+		pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
+		if (!pdata)
+			return -ENOMEM;
+
+		pdata->config	= &da8xx_config;
+		pdata->mode	= musb_get_mode(&pdev->dev);
+		pdata->power	= get_vbus_power(&pdev->dev);
+	}
+
 	pdata->platform_ops		= &da8xx_ops;
 
 	glue->usb_phy = usb_phy_generic_register();
@@ -537,11 +578,22 @@ static int da8xx_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_OF
+static const struct of_device_id da8xx_id_table[] = {
+	{
+		.compatible = "ti,da830-musb",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, da8xx_id_table);
+#endif
+
 static struct platform_driver da8xx_driver = {
 	.probe		= da8xx_probe,
 	.remove		= da8xx_remove,
 	.driver		= {
 		.name	= "musb-da8xx",
+		.of_match_table = of_match_ptr(da8xx_id_table),
 	},
 };
 
diff --git a/drivers/usb/musb/musb_core.c b/drivers/usb/musb/musb_core.c
index c3e172e15ec3..9e226468a13e 100644
--- a/drivers/usb/musb/musb_core.c
+++ b/drivers/usb/musb/musb_core.c
@@ -100,6 +100,7 @@
 #include <linux/io.h>
 #include <linux/dma-mapping.h>
 #include <linux/usb.h>
+#include <linux/usb/of.h>
 
 #include "musb_core.h"
 #include "musb_trace.h"
@@ -130,6 +131,24 @@ static inline struct musb *dev_to_musb(struct device *dev)
 	return dev_get_drvdata(dev);
 }
 
+enum musb_mode musb_get_mode(struct device *dev)
+{
+	enum usb_dr_mode mode;
+
+	mode = usb_get_dr_mode(dev);
+	switch (mode) {
+	case USB_DR_MODE_HOST:
+		return MUSB_HOST;
+	case USB_DR_MODE_PERIPHERAL:
+		return MUSB_PERIPHERAL;
+	case USB_DR_MODE_OTG:
+	case USB_DR_MODE_UNKNOWN:
+	default:
+		return MUSB_OTG;
+	}
+}
+EXPORT_SYMBOL_GPL(musb_get_mode);
+
 /*-------------------------------------------------------------------------*/
 
 #ifndef CONFIG_BLACKFIN
@@ -569,10 +588,7 @@ static irqreturn_t musb_stage0_irq(struct musb *musb, u8 int_usb,
 		if (devctl & MUSB_DEVCTL_HM) {
 			switch (musb->xceiv->otg->state) {
 			case OTG_STATE_A_SUSPEND:
-				/* remote wakeup?  later, GetPortStatus
-				 * will stop RESUME signaling
-				 */
-
+				/* remote wakeup? */
 				musb->port1_status |=
 						(USB_PORT_STAT_C_SUSPEND << 16)
 						| MUSB_PORT_STAT_RESUME;
@@ -2414,8 +2430,9 @@ fail2:
 	musb_platform_exit(musb);
 
 fail1:
-	dev_err(musb->controller,
-		"musb_init_controller failed with status %d\n", status);
+	if (status != -EPROBE_DEFER)
+		dev_err(musb->controller,
+			"%s failed with status %d\n", __func__, status);
 
 	musb_free(musb);
 
diff --git a/drivers/usb/musb/musb_core.h b/drivers/usb/musb/musb_core.h
index 91817d77d59c..a611e2f67bdc 100644
--- a/drivers/usb/musb/musb_core.h
+++ b/drivers/usb/musb/musb_core.h
@@ -626,4 +626,10 @@ static inline void musb_platform_post_root_reset_end(struct musb *musb)
 		musb->ops->post_root_reset_end(musb);
 }
 
+/*
+ * gets the "dr_mode" property from DT and converts it into musb_mode
+ * if the property is not found or not recognized returns MUSB_OTG
+ */
+extern enum musb_mode musb_get_mode(struct device *dev);
+
 #endif	/* __MUSB_CORE_H__ */
diff --git a/drivers/usb/musb/musb_gadget.c b/drivers/usb/musb/musb_gadget.c
index a55173c9e564..1acc4864f9f6 100644
--- a/drivers/usb/musb/musb_gadget.c
+++ b/drivers/usb/musb/musb_gadget.c
@@ -974,8 +974,8 @@ static int musb_gadget_enable(struct usb_ep *ep,
 		goto fail;
 
 	/* REVISIT this rules out high bandwidth periodic transfers */
-	tmp = usb_endpoint_maxp(desc);
-	if (tmp & ~0x07ff) {
+	tmp = usb_endpoint_maxp_mult(desc) - 1;
+	if (tmp) {
 		int ok;
 
 		if (usb_endpoint_dir_in(desc))
@@ -987,12 +987,12 @@ static int musb_gadget_enable(struct usb_ep *ep,
 			musb_dbg(musb, "no support for high bandwidth ISO");
 			goto fail;
 		}
-		musb_ep->hb_mult = (tmp >> 11) & 3;
+		musb_ep->hb_mult = tmp;
 	} else {
 		musb_ep->hb_mult = 0;
 	}
 
-	musb_ep->packet_sz = tmp & 0x7ff;
+	musb_ep->packet_sz = usb_endpoint_maxp(desc);
 	tmp = musb_ep->packet_sz * (musb_ep->hb_mult + 1);
 
 	/* enable the interrupts for the endpoint, set the endpoint
diff --git a/drivers/usb/musb/musb_host.c b/drivers/usb/musb/musb_host.c
index 53bc4ceefe89..f6cdbad00dac 100644
--- a/drivers/usb/musb/musb_host.c
+++ b/drivers/usb/musb/musb_host.c
@@ -2237,7 +2237,7 @@ static int musb_urb_enqueue(
 	 * Some musb cores don't support high bandwidth ISO transfers; and
 	 * we don't (yet!) support high bandwidth interrupt transfers.
 	 */
-	qh->hb_mult = 1 + ((qh->maxpacket >> 11) & 0x03);
+	qh->hb_mult = usb_endpoint_maxp_mult(epd);
 	if (qh->hb_mult > 1) {
 		int ok = (qh->type == USB_ENDPOINT_XFER_ISOC);
 
diff --git a/drivers/usb/musb/musb_virthub.c b/drivers/usb/musb/musb_virthub.c
index 61b5f1c3c5bc..0b4595439d51 100644
--- a/drivers/usb/musb/musb_virthub.c
+++ b/drivers/usb/musb/musb_virthub.c
@@ -132,7 +132,6 @@ void musb_port_suspend(struct musb *musb, bool do_suspend)
 
 		musb_dbg(musb, "Root port resuming, power %02x", power);
 
-		/* later, GetPortStatus will stop RESUME signaling */
 		musb->port1_status |= MUSB_PORT_STAT_RESUME;
 		schedule_delayed_work(&musb->finish_resume_work,
 				      msecs_to_jiffies(USB_RESUME_TIMEOUT));
diff --git a/drivers/usb/musb/omap2430.c b/drivers/usb/musb/omap2430.c
index e8be8e39ab8f..8b73214a9ea3 100644
--- a/drivers/usb/musb/omap2430.c
+++ b/drivers/usb/musb/omap2430.c
@@ -277,12 +277,12 @@ static int omap2430_musb_init(struct musb *musb)
 		if (status == -ENXIO)
 			return status;
 
-		pr_err("HS USB OTG: no transceiver configured\n");
+		dev_dbg(dev, "HS USB OTG: no transceiver configured\n");
 		return -EPROBE_DEFER;
 	}
 
 	if (IS_ERR(musb->phy)) {
-		pr_err("HS USB OTG: no PHY configured\n");
+		dev_err(dev, "HS USB OTG: no PHY configured\n");
 		return PTR_ERR(musb->phy);
 	}
 	musb->isr = omap2430_musb_interrupt;
@@ -301,7 +301,7 @@ static int omap2430_musb_init(struct musb *musb)
 
 	musb_writel(musb->mregs, OTG_INTERFSEL, l);
 
-	pr_debug("HS USB OTG: revision 0x%x, sysconfig 0x%02x, "
+	dev_dbg(dev, "HS USB OTG: revision 0x%x, sysconfig 0x%02x, "
 			"sysstatus 0x%x, intrfsel 0x%x, simenable  0x%x\n",
 			musb_readl(musb->mregs, OTG_REVISION),
 			musb_readl(musb->mregs, OTG_SYSCONFIG),
diff --git a/drivers/usb/musb/sunxi.c b/drivers/usb/musb/sunxi.c
index 1408245be18e..d0be0eadd0d9 100644
--- a/drivers/usb/musb/sunxi.c
+++ b/drivers/usb/musb/sunxi.c
@@ -186,16 +186,6 @@ static irqreturn_t sunxi_musb_interrupt(int irq, void *__hci)
 	if (musb->int_usb)
 		writeb(musb->int_usb, musb->mregs + SUNXI_MUSB_INTRUSB);
 
-	/*
-	 * sunxi musb often signals babble on low / full speed device
-	 * disconnect, without ever raising MUSB_INTR_DISCONNECT, since
-	 * normally babble never happens treat it as disconnect.
-	 */
-	if ((musb->int_usb & MUSB_INTR_BABBLE) && is_host_active(musb)) {
-		musb->int_usb &= ~MUSB_INTR_BABBLE;
-		musb->int_usb |= MUSB_INTR_DISCONNECT;
-	}
-
 	if ((musb->int_usb & MUSB_INTR_RESET) && !is_host_active(musb)) {
 		/* ep0 FADDR must be 0 when (re)entering peripheral mode */
 		musb_ep_select(musb->mregs, 0);
@@ -390,6 +380,20 @@ static int sunxi_musb_set_mode(struct musb *musb, u8 mode)
 	return 0;
 }
 
+static int sunxi_musb_recover(struct musb *musb)
+{
+	struct sunxi_glue *glue = dev_get_drvdata(musb->controller->parent);
+
+	/*
+	 * Schedule a phy_set_mode with the current glue->phy_mode value,
+	 * this will force end the current session.
+	 */
+	set_bit(SUNXI_MUSB_FL_PHY_MODE_PEND, &glue->flags);
+	schedule_work(&glue->work);
+
+	return 0;
+}
+
 /*
  * sunxi musb register layout
  * 0x00 - 0x17	fifo regs, 1 long per fifo
@@ -618,6 +622,7 @@ static const struct musb_platform_ops sunxi_musb_ops = {
 	.dma_init	= sunxi_musb_dma_controller_create,
 	.dma_exit	= sunxi_musb_dma_controller_destroy,
 	.set_mode	= sunxi_musb_set_mode,
+	.recover	= sunxi_musb_recover,
 	.set_vbus	= sunxi_musb_set_vbus,
 	.pre_root_reset_end = sunxi_musb_pre_root_reset_end,
 	.post_root_reset_end = sunxi_musb_post_root_reset_end,
diff --git a/drivers/usb/phy/Kconfig b/drivers/usb/phy/Kconfig
index b9c409a18faa..61cef7511a50 100644
--- a/drivers/usb/phy/Kconfig
+++ b/drivers/usb/phy/Kconfig
@@ -84,6 +84,7 @@ config SAMSUNG_USBPHY
 config TWL6030_USB
 	tristate "TWL6030 USB Transceiver Driver"
 	depends on TWL4030_CORE && OMAP_USB2 && USB_MUSB_OMAP2PLUS
+	depends on OF
 	help
 	  Enable this to support the USB OTG transceiver on TWL6030
 	  family chips. This TWL6030 transceiver has the VBUS and ID GND
diff --git a/drivers/usb/phy/phy-am335x-control.c b/drivers/usb/phy/phy-am335x-control.c
index 42a1afe36a90..5f5f19813fde 100644
--- a/drivers/usb/phy/phy-am335x-control.c
+++ b/drivers/usb/phy/phy-am335x-control.c
@@ -134,10 +134,12 @@ struct phy_control *am335x_get_phy_control(struct device *dev)
 		return NULL;
 
 	dev = bus_find_device(&platform_bus_type, NULL, node, match);
+	of_node_put(node);
 	if (!dev)
 		return NULL;
 
 	ctrl_usb = dev_get_drvdata(dev);
+	put_device(dev);
 	if (!ctrl_usb)
 		return NULL;
 	return &ctrl_usb->phy_ctrl;
diff --git a/drivers/usb/phy/phy-generic.c b/drivers/usb/phy/phy-generic.c
index 8311ba2968cd..89d6e7a5fdb7 100644
--- a/drivers/usb/phy/phy-generic.c
+++ b/drivers/usb/phy/phy-generic.c
@@ -59,6 +59,15 @@ EXPORT_SYMBOL_GPL(usb_phy_generic_unregister);
 
 static int nop_set_suspend(struct usb_phy *x, int suspend)
 {
+	struct usb_phy_generic *nop = dev_get_drvdata(x->dev);
+
+	if (!IS_ERR(nop->clk)) {
+		if (suspend)
+			clk_disable_unprepare(nop->clk);
+		else
+			clk_prepare_enable(nop->clk);
+	}
+
 	return 0;
 }
 
diff --git a/drivers/usb/phy/phy-isp1301-omap.c b/drivers/usb/phy/phy-isp1301-omap.c
index 8d111ec653e4..042c5a8fd423 100644
--- a/drivers/usb/phy/phy-isp1301-omap.c
+++ b/drivers/usb/phy/phy-isp1301-omap.c
@@ -94,7 +94,7 @@ struct isp1301 {
 
 #if defined(CONFIG_MACH_OMAP_H2) || defined(CONFIG_MACH_OMAP_H3)
 
-#if	defined(CONFIG_TPS65010) || (defined(CONFIG_TPS65010_MODULE) && defined(MODULE))
+#if IS_REACHABLE(CONFIG_TPS65010)
 
 #include <linux/i2c/tps65010.h>
 
diff --git a/drivers/usb/phy/phy-twl6030-usb.c b/drivers/usb/phy/phy-twl6030-usb.c
index a72e8d670adc..628b600b02b1 100644
--- a/drivers/usb/phy/phy-twl6030-usb.c
+++ b/drivers/usb/phy/phy-twl6030-usb.c
@@ -108,7 +108,6 @@ struct twl6030_usb {
 	enum musb_vbus_id_status linkstat;
 	u8			asleep;
 	bool			vbus_enable;
-	const char		*regulator;
 };
 
 #define	comparator_to_twl(x) container_of((x), struct twl6030_usb, comparator)
@@ -166,7 +165,7 @@ static int twl6030_usb_ldo_init(struct twl6030_usb *twl)
 	/* Program MISC2 register and set bit VUSB_IN_VBAT */
 	twl6030_writeb(twl, TWL6030_MODULE_ID0, 0x10, TWL6030_MISC2);
 
-	twl->usb3v3 = regulator_get(twl->dev, twl->regulator);
+	twl->usb3v3 = regulator_get(twl->dev, "usb");
 	if (IS_ERR(twl->usb3v3))
 		return -ENODEV;
 
@@ -341,7 +340,11 @@ static int twl6030_usb_probe(struct platform_device *pdev)
 	int			status, err;
 	struct device_node	*np = pdev->dev.of_node;
 	struct device		*dev = &pdev->dev;
-	struct twl4030_usb_data	*pdata = dev_get_platdata(dev);
+
+	if (!np) {
+		dev_err(dev, "no DT info\n");
+		return -EINVAL;
+	}
 
 	twl = devm_kzalloc(dev, sizeof(*twl), GFP_KERNEL);
 	if (!twl)
@@ -361,18 +364,6 @@ static int twl6030_usb_probe(struct platform_device *pdev)
 		return -EPROBE_DEFER;
 	}
 
-	if (np) {
-		twl->regulator = "usb";
-	} else if (pdata) {
-		if (pdata->features & TWL6032_SUBCLASS)
-			twl->regulator = "ldousb";
-		else
-			twl->regulator = "vusb";
-	} else {
-		dev_err(&pdev->dev, "twl6030 initialized without pdata\n");
-		return -EINVAL;
-	}
-
 	/* init spinlock for workqueue */
 	spin_lock_init(&twl->lock);
 
@@ -436,13 +427,11 @@ static int twl6030_usb_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_OF
 static const struct of_device_id twl6030_usb_id_table[] = {
 	{ .compatible = "ti,twl6030-usb" },
 	{}
 };
 MODULE_DEVICE_TABLE(of, twl6030_usb_id_table);
-#endif
 
 static struct platform_driver twl6030_usb_driver = {
 	.probe		= twl6030_usb_probe,
diff --git a/drivers/usb/renesas_usbhs/fifo.c b/drivers/usb/renesas_usbhs/fifo.c
index 857e78337324..d1af831f43eb 100644
--- a/drivers/usb/renesas_usbhs/fifo.c
+++ b/drivers/usb/renesas_usbhs/fifo.c
@@ -100,10 +100,7 @@ static void __usbhsf_pkt_del(struct usbhs_pkt *pkt)
 
 static struct usbhs_pkt *__usbhsf_pkt_get(struct usbhs_pipe *pipe)
 {
-	if (list_empty(&pipe->list))
-		return NULL;
-
-	return list_first_entry(&pipe->list, struct usbhs_pkt, node);
+	return list_first_entry_or_null(&pipe->list, struct usbhs_pkt, node);
 }
 
 static void usbhsf_fifo_clear(struct usbhs_pipe *pipe,
diff --git a/drivers/usb/serial/Kconfig b/drivers/usb/serial/Kconfig
index 56ecb8b5115d..d9bc8dafe000 100644
--- a/drivers/usb/serial/Kconfig
+++ b/drivers/usb/serial/Kconfig
@@ -255,6 +255,16 @@ config USB_SERIAL_F81232
 	  To compile this driver as a module, choose M here: the
 	  module will be called f81232.
 
+config USB_SERIAL_F8153X
+	tristate "USB Fintek F81532/534 Multi-Ports Serial Driver"
+	help
+	  Say Y here if you want to use the Fintek F81532/534 Multi-Ports
+	  USB to serial adapter.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called f81534.
+
+
 config USB_SERIAL_GARMIN
        tristate "USB Garmin GPS driver"
        help
diff --git a/drivers/usb/serial/Makefile b/drivers/usb/serial/Makefile
index 349d9df0895f..9e43b7b002eb 100644
--- a/drivers/usb/serial/Makefile
+++ b/drivers/usb/serial/Makefile
@@ -23,6 +23,7 @@ obj-$(CONFIG_USB_SERIAL_EDGEPORT)		+= io_edgeport.o
 obj-$(CONFIG_USB_SERIAL_EDGEPORT_TI)		+= io_ti.o
 obj-$(CONFIG_USB_SERIAL_EMPEG)			+= empeg.o
 obj-$(CONFIG_USB_SERIAL_F81232)			+= f81232.o
+obj-$(CONFIG_USB_SERIAL_F8153X)			+= f81534.o
 obj-$(CONFIG_USB_SERIAL_FTDI_SIO)		+= ftdi_sio.o
 obj-$(CONFIG_USB_SERIAL_GARMIN)			+= garmin_gps.o
 obj-$(CONFIG_USB_SERIAL_IPAQ)			+= ipaq.o
diff --git a/drivers/usb/serial/ch341.c b/drivers/usb/serial/ch341.c
index f139488d0816..2597b83a8ae2 100644
--- a/drivers/usb/serial/ch341.c
+++ b/drivers/usb/serial/ch341.c
@@ -61,13 +61,26 @@
  * the Net/FreeBSD uchcom.c driver by Takanori Watanabe.  Domo arigato.
  */
 
+#define CH341_REQ_READ_VERSION 0x5F
 #define CH341_REQ_WRITE_REG    0x9A
 #define CH341_REQ_READ_REG     0x95
-#define CH341_REG_BREAK1       0x05
-#define CH341_REG_BREAK2       0x18
-#define CH341_NBREAK_BITS_REG1 0x01
-#define CH341_NBREAK_BITS_REG2 0x40
-
+#define CH341_REQ_SERIAL_INIT  0xA1
+#define CH341_REQ_MODEM_CTRL   0xA4
+
+#define CH341_REG_BREAK        0x05
+#define CH341_REG_LCR          0x18
+#define CH341_NBREAK_BITS      0x01
+
+#define CH341_LCR_ENABLE_RX    0x80
+#define CH341_LCR_ENABLE_TX    0x40
+#define CH341_LCR_MARK_SPACE   0x20
+#define CH341_LCR_PAR_EVEN     0x10
+#define CH341_LCR_ENABLE_PAR   0x08
+#define CH341_LCR_STOP_BITS_2  0x04
+#define CH341_LCR_CS8          0x03
+#define CH341_LCR_CS7          0x02
+#define CH341_LCR_CS6          0x01
+#define CH341_LCR_CS5          0x00
 
 static const struct usb_device_id id_table[] = {
 	{ USB_DEVICE(0x4348, 0x5523) },
@@ -119,10 +132,10 @@ static int ch341_control_in(struct usb_device *dev,
 	return r;
 }
 
-static int ch341_set_baudrate(struct usb_device *dev,
-			      struct ch341_private *priv)
+static int ch341_init_set_baudrate(struct usb_device *dev,
+				   struct ch341_private *priv, unsigned ctrl)
 {
-	short a, b;
+	short a;
 	int r;
 	unsigned long factor;
 	short divisor;
@@ -142,18 +155,17 @@ static int ch341_set_baudrate(struct usb_device *dev,
 
 	factor = 0x10000 - factor;
 	a = (factor & 0xff00) | divisor;
-	b = factor & 0xff;
 
-	r = ch341_control_out(dev, 0x9a, 0x1312, a);
-	if (!r)
-		r = ch341_control_out(dev, 0x9a, 0x0f2c, b);
+	/* 0x9c is "enable SFR_UART Control register and timer" */
+	r = ch341_control_out(dev, CH341_REQ_SERIAL_INIT,
+			      0x9c | (ctrl << 8), a | 0x80);
 
 	return r;
 }
 
 static int ch341_set_handshake(struct usb_device *dev, u8 control)
 {
-	return ch341_control_out(dev, 0xa4, ~control, 0);
+	return ch341_control_out(dev, CH341_REQ_MODEM_CTRL, ~control, 0);
 }
 
 static int ch341_get_status(struct usb_device *dev, struct ch341_private *priv)
@@ -167,7 +179,7 @@ static int ch341_get_status(struct usb_device *dev, struct ch341_private *priv)
 	if (!buffer)
 		return -ENOMEM;
 
-	r = ch341_control_in(dev, 0x95, 0x0706, 0, buffer, size);
+	r = ch341_control_in(dev, CH341_REQ_READ_REG, 0x0706, 0, buffer, size);
 	if (r < 0)
 		goto out;
 
@@ -197,24 +209,21 @@ static int ch341_configure(struct usb_device *dev, struct ch341_private *priv)
 		return -ENOMEM;
 
 	/* expect two bytes 0x27 0x00 */
-	r = ch341_control_in(dev, 0x5f, 0, 0, buffer, size);
+	r = ch341_control_in(dev, CH341_REQ_READ_VERSION, 0, 0, buffer, size);
 	if (r < 0)
 		goto out;
+	dev_dbg(&dev->dev, "Chip version: 0x%02x\n", buffer[0]);
 
-	r = ch341_control_out(dev, 0xa1, 0, 0);
-	if (r < 0)
-		goto out;
-
-	r = ch341_set_baudrate(dev, priv);
+	r = ch341_control_out(dev, CH341_REQ_SERIAL_INIT, 0, 0);
 	if (r < 0)
 		goto out;
 
 	/* expect two bytes 0x56 0x00 */
-	r = ch341_control_in(dev, 0x95, 0x2518, 0, buffer, size);
+	r = ch341_control_in(dev, CH341_REQ_READ_REG, 0x2518, 0, buffer, size);
 	if (r < 0)
 		goto out;
 
-	r = ch341_control_out(dev, 0x9a, 0x2518, 0x0050);
+	r = ch341_control_out(dev, CH341_REQ_WRITE_REG, 0x2518, 0x0050);
 	if (r < 0)
 		goto out;
 
@@ -223,11 +232,7 @@ static int ch341_configure(struct usb_device *dev, struct ch341_private *priv)
 	if (r < 0)
 		goto out;
 
-	r = ch341_control_out(dev, 0xa1, 0x501f, 0xd90a);
-	if (r < 0)
-		goto out;
-
-	r = ch341_set_baudrate(dev, priv);
+	r = ch341_init_set_baudrate(dev, priv, 0);
 	if (r < 0)
 		goto out;
 
@@ -342,16 +347,53 @@ static void ch341_set_termios(struct tty_struct *tty,
 	struct ch341_private *priv = usb_get_serial_port_data(port);
 	unsigned baud_rate;
 	unsigned long flags;
+	unsigned char ctrl;
+	int r;
+
+	/* redundant changes may cause the chip to lose bytes */
+	if (old_termios && !tty_termios_hw_change(&tty->termios, old_termios))
+		return;
 
 	baud_rate = tty_get_baud_rate(tty);
 
 	priv->baud_rate = baud_rate;
+	ctrl = CH341_LCR_ENABLE_RX | CH341_LCR_ENABLE_TX;
+
+	switch (C_CSIZE(tty)) {
+	case CS5:
+		ctrl |= CH341_LCR_CS5;
+		break;
+	case CS6:
+		ctrl |= CH341_LCR_CS6;
+		break;
+	case CS7:
+		ctrl |= CH341_LCR_CS7;
+		break;
+	case CS8:
+		ctrl |= CH341_LCR_CS8;
+		break;
+	}
+
+	if (C_PARENB(tty)) {
+		ctrl |= CH341_LCR_ENABLE_PAR;
+		if (C_PARODD(tty) == 0)
+			ctrl |= CH341_LCR_PAR_EVEN;
+		if (C_CMSPAR(tty))
+			ctrl |= CH341_LCR_MARK_SPACE;
+	}
+
+	if (C_CSTOPB(tty))
+		ctrl |= CH341_LCR_STOP_BITS_2;
 
 	if (baud_rate) {
 		spin_lock_irqsave(&priv->lock, flags);
 		priv->line_control |= (CH341_BIT_DTR | CH341_BIT_RTS);
 		spin_unlock_irqrestore(&priv->lock, flags);
-		ch341_set_baudrate(port->serial->dev, priv);
+		r = ch341_init_set_baudrate(port->serial->dev, priv, ctrl);
+		if (r < 0 && old_termios) {
+			priv->baud_rate = tty_termios_baud_rate(old_termios);
+			tty_termios_copy_hw(&tty->termios, old_termios);
+		}
 	} else {
 		spin_lock_irqsave(&priv->lock, flags);
 		priv->line_control &= ~(CH341_BIT_DTR | CH341_BIT_RTS);
@@ -360,17 +402,12 @@ static void ch341_set_termios(struct tty_struct *tty,
 
 	ch341_set_handshake(port->serial->dev, priv->line_control);
 
-	/* Unimplemented:
-	 * (cflag & CSIZE) : data bits [5, 8]
-	 * (cflag & PARENB) : parity {NONE, EVEN, ODD}
-	 * (cflag & CSTOPB) : stop bits [1, 2]
-	 */
 }
 
 static void ch341_break_ctl(struct tty_struct *tty, int break_state)
 {
 	const uint16_t ch341_break_reg =
-			((uint16_t) CH341_REG_BREAK2 << 8) | CH341_REG_BREAK1;
+			((uint16_t) CH341_REG_LCR << 8) | CH341_REG_BREAK;
 	struct usb_serial_port *port = tty->driver_data;
 	int r;
 	uint16_t reg_contents;
@@ -391,12 +428,12 @@ static void ch341_break_ctl(struct tty_struct *tty, int break_state)
 		__func__, break_reg[0], break_reg[1]);
 	if (break_state != 0) {
 		dev_dbg(&port->dev, "%s - Enter break state requested\n", __func__);
-		break_reg[0] &= ~CH341_NBREAK_BITS_REG1;
-		break_reg[1] &= ~CH341_NBREAK_BITS_REG2;
+		break_reg[0] &= ~CH341_NBREAK_BITS;
+		break_reg[1] &= ~CH341_LCR_ENABLE_TX;
 	} else {
 		dev_dbg(&port->dev, "%s - Leave break state requested\n", __func__);
-		break_reg[0] |= CH341_NBREAK_BITS_REG1;
-		break_reg[1] |= CH341_NBREAK_BITS_REG2;
+		break_reg[0] |= CH341_NBREAK_BITS;
+		break_reg[1] |= CH341_LCR_ENABLE_TX;
 	}
 	dev_dbg(&port->dev, "%s - New ch341 break register contents - reg1: %x, reg2: %x\n",
 		__func__, break_reg[0], break_reg[1]);
diff --git a/drivers/usb/serial/cp210x.c b/drivers/usb/serial/cp210x.c
index 243ac5ebe46a..fff718352e0c 100644
--- a/drivers/usb/serial/cp210x.c
+++ b/drivers/usb/serial/cp210x.c
@@ -23,6 +23,9 @@
 #include <linux/usb.h>
 #include <linux/uaccess.h>
 #include <linux/usb/serial.h>
+#include <linux/gpio/driver.h>
+#include <linux/bitops.h>
+#include <linux/mutex.h>
 
 #define DRIVER_DESC "Silicon Labs CP210x RS232 serial adaptor driver"
 
@@ -33,7 +36,7 @@ static int cp210x_open(struct tty_struct *tty, struct usb_serial_port *);
 static void cp210x_close(struct usb_serial_port *);
 static void cp210x_get_termios(struct tty_struct *, struct usb_serial_port *);
 static void cp210x_get_termios_port(struct usb_serial_port *port,
-	unsigned int *cflagp, unsigned int *baudp);
+	tcflag_t *cflagp, unsigned int *baudp);
 static void cp210x_change_speed(struct tty_struct *, struct usb_serial_port *,
 							struct ktermios *);
 static void cp210x_set_termios(struct tty_struct *, struct usb_serial_port *,
@@ -44,6 +47,9 @@ static int cp210x_tiocmset(struct tty_struct *, unsigned int, unsigned int);
 static int cp210x_tiocmset_port(struct usb_serial_port *port,
 		unsigned int, unsigned int);
 static void cp210x_break_ctl(struct tty_struct *, int);
+static int cp210x_attach(struct usb_serial *);
+static void cp210x_disconnect(struct usb_serial *);
+static void cp210x_release(struct usb_serial *);
 static int cp210x_port_probe(struct usb_serial_port *);
 static int cp210x_port_remove(struct usb_serial_port *);
 static void cp210x_dtr_rts(struct usb_serial_port *p, int on);
@@ -209,6 +215,16 @@ static const struct usb_device_id id_table[] = {
 
 MODULE_DEVICE_TABLE(usb, id_table);
 
+struct cp210x_serial_private {
+#ifdef CONFIG_GPIOLIB
+	struct gpio_chip	gc;
+	u8			config;
+	u8			gpio_mode;
+	bool			gpio_registered;
+#endif
+	u8			partnum;
+};
+
 struct cp210x_port_private {
 	__u8			bInterfaceNumber;
 	bool			has_swapped_line_ctl;
@@ -230,6 +246,9 @@ static struct usb_serial_driver cp210x_device = {
 	.tx_empty		= cp210x_tx_empty,
 	.tiocmget		= cp210x_tiocmget,
 	.tiocmset		= cp210x_tiocmset,
+	.attach			= cp210x_attach,
+	.disconnect		= cp210x_disconnect,
+	.release		= cp210x_release,
 	.port_probe		= cp210x_port_probe,
 	.port_remove		= cp210x_port_remove,
 	.dtr_rts		= cp210x_dtr_rts
@@ -272,6 +291,7 @@ static struct usb_serial_driver * const serial_drivers[] = {
 #define CP210X_SET_CHARS	0x19
 #define CP210X_GET_BAUDRATE	0x1D
 #define CP210X_SET_BAUDRATE	0x1E
+#define CP210X_VENDOR_SPECIFIC	0xFF
 
 /* CP210X_IFC_ENABLE */
 #define UART_ENABLE		0x0001
@@ -314,6 +334,21 @@ static struct usb_serial_driver * const serial_drivers[] = {
 #define CONTROL_WRITE_DTR	0x0100
 #define CONTROL_WRITE_RTS	0x0200
 
+/* CP210X_VENDOR_SPECIFIC values */
+#define CP210X_READ_LATCH	0x00C2
+#define CP210X_GET_PARTNUM	0x370B
+#define CP210X_GET_PORTCONFIG	0x370C
+#define CP210X_GET_DEVICEMODE	0x3711
+#define CP210X_WRITE_LATCH	0x37E1
+
+/* Part number definitions */
+#define CP210X_PARTNUM_CP2101	0x01
+#define CP210X_PARTNUM_CP2102	0x02
+#define CP210X_PARTNUM_CP2103	0x03
+#define CP210X_PARTNUM_CP2104	0x04
+#define CP210X_PARTNUM_CP2105	0x05
+#define CP210X_PARTNUM_CP2108	0x08
+
 /* CP210X_GET_COMM_STATUS returns these 0x13 bytes */
 struct cp210x_comm_status {
 	__le32   ulErrors;
@@ -369,6 +404,60 @@ struct cp210x_flow_ctl {
 #define CP210X_SERIAL_RTS_ACTIVE	1
 #define CP210X_SERIAL_RTS_FLOW_CTL	2
 
+/* CP210X_VENDOR_SPECIFIC, CP210X_GET_DEVICEMODE call reads these 0x2 bytes. */
+struct cp210x_pin_mode {
+	u8	eci;
+	u8	sci;
+} __packed;
+
+#define CP210X_PIN_MODE_MODEM		0
+#define CP210X_PIN_MODE_GPIO		BIT(0)
+
+/*
+ * CP210X_VENDOR_SPECIFIC, CP210X_GET_PORTCONFIG call reads these 0xf bytes.
+ * Structure needs padding due to unused/unspecified bytes.
+ */
+struct cp210x_config {
+	__le16	gpio_mode;
+	u8	__pad0[2];
+	__le16	reset_state;
+	u8	__pad1[4];
+	__le16	suspend_state;
+	u8	sci_cfg;
+	u8	eci_cfg;
+	u8	device_cfg;
+} __packed;
+
+/* GPIO modes */
+#define CP210X_SCI_GPIO_MODE_OFFSET	9
+#define CP210X_SCI_GPIO_MODE_MASK	GENMASK(11, 9)
+
+#define CP210X_ECI_GPIO_MODE_OFFSET	2
+#define CP210X_ECI_GPIO_MODE_MASK	GENMASK(3, 2)
+
+/* CP2105 port configuration values */
+#define CP2105_GPIO0_TXLED_MODE		BIT(0)
+#define CP2105_GPIO1_RXLED_MODE		BIT(1)
+#define CP2105_GPIO1_RS485_MODE		BIT(2)
+
+/* CP210X_VENDOR_SPECIFIC, CP210X_WRITE_LATCH call writes these 0x2 bytes. */
+struct cp210x_gpio_write {
+	u8	mask;
+	u8	state;
+} __packed;
+
+/*
+ * Helper to get interface number when we only have struct usb_serial.
+ */
+static u8 cp210x_interface_num(struct usb_serial *serial)
+{
+	struct usb_host_interface *cur_altsetting;
+
+	cur_altsetting = serial->interface->cur_altsetting;
+
+	return cur_altsetting->desc.bInterfaceNumber;
+}
+
 /*
  * Reads a variable-sized block of CP210X_ registers, identified by req.
  * Returns data into buf in native USB byte order.
@@ -402,7 +491,7 @@ static int cp210x_read_reg_block(struct usb_serial_port *port, u8 req,
 		dev_err(&port->dev, "failed get req 0x%x size %d status: %d\n",
 				req, bufsize, result);
 		if (result >= 0)
-			result = -EPROTO;
+			result = -EIO;
 
 		/*
 		 * FIXME Some callers don't bother to check for error,
@@ -465,6 +554,40 @@ static int cp210x_read_u8_reg(struct usb_serial_port *port, u8 req, u8 *val)
 }
 
 /*
+ * Reads a variable-sized vendor block of CP210X_ registers, identified by val.
+ * Returns data into buf in native USB byte order.
+ */
+static int cp210x_read_vendor_block(struct usb_serial *serial, u8 type, u16 val,
+				    void *buf, int bufsize)
+{
+	void *dmabuf;
+	int result;
+
+	dmabuf = kmalloc(bufsize, GFP_KERNEL);
+	if (!dmabuf)
+		return -ENOMEM;
+
+	result = usb_control_msg(serial->dev, usb_rcvctrlpipe(serial->dev, 0),
+				 CP210X_VENDOR_SPECIFIC, type, val,
+				 cp210x_interface_num(serial), dmabuf, bufsize,
+				 USB_CTRL_GET_TIMEOUT);
+	if (result == bufsize) {
+		memcpy(buf, dmabuf, bufsize);
+		result = 0;
+	} else {
+		dev_err(&serial->interface->dev,
+			"failed to get vendor val 0x%04x size %d: %d\n", val,
+			bufsize, result);
+		if (result >= 0)
+			result = -EIO;
+	}
+
+	kfree(dmabuf);
+
+	return result;
+}
+
+/*
  * Writes any 16-bit CP210X_ register (req) whose value is passed
  * entirely in the wValue field of the USB request.
  */
@@ -515,7 +638,7 @@ static int cp210x_write_reg_block(struct usb_serial_port *port, u8 req,
 		dev_err(&port->dev, "failed set req 0x%x size %d status: %d\n",
 				req, bufsize, result);
 		if (result >= 0)
-			result = -EPROTO;
+			result = -EIO;
 	}
 
 	return result;
@@ -533,6 +656,42 @@ static int cp210x_write_u32_reg(struct usb_serial_port *port, u8 req, u32 val)
 	return cp210x_write_reg_block(port, req, &le32_val, sizeof(le32_val));
 }
 
+#ifdef CONFIG_GPIOLIB
+/*
+ * Writes a variable-sized vendor block of CP210X_ registers, identified by val.
+ * Data in buf must be in native USB byte order.
+ */
+static int cp210x_write_vendor_block(struct usb_serial *serial, u8 type,
+				     u16 val, void *buf, int bufsize)
+{
+	void *dmabuf;
+	int result;
+
+	dmabuf = kmemdup(buf, bufsize, GFP_KERNEL);
+	if (!dmabuf)
+		return -ENOMEM;
+
+	result = usb_control_msg(serial->dev, usb_sndctrlpipe(serial->dev, 0),
+				 CP210X_VENDOR_SPECIFIC, type, val,
+				 cp210x_interface_num(serial), dmabuf, bufsize,
+				 USB_CTRL_SET_TIMEOUT);
+
+	kfree(dmabuf);
+
+	if (result == bufsize) {
+		result = 0;
+	} else {
+		dev_err(&serial->interface->dev,
+			"failed to set vendor val 0x%04x size %d: %d\n", val,
+			bufsize, result);
+		if (result >= 0)
+			result = -EIO;
+	}
+
+	return result;
+}
+#endif
+
 /*
  * Detect CP2108 GET_LINE_CTL bug and activate workaround.
  * Write a known good value 0x800, read it back.
@@ -683,7 +842,7 @@ static int cp210x_get_tx_queue_byte_count(struct usb_serial_port *port,
 	} else {
 		dev_err(&port->dev, "failed to get comm status: %d\n", result);
 		if (result >= 0)
-			result = -EPROTO;
+			result = -EIO;
 	}
 
 	kfree(sts);
@@ -719,7 +878,7 @@ static void cp210x_get_termios(struct tty_struct *tty,
 			&tty->termios.c_cflag, &baud);
 		tty_encode_baud_rate(tty, baud, baud);
 	} else {
-		unsigned int cflag;
+		tcflag_t cflag;
 		cflag = 0;
 		cp210x_get_termios_port(port, &cflag, &baud);
 	}
@@ -730,10 +889,10 @@ static void cp210x_get_termios(struct tty_struct *tty,
  * This is the heart of cp210x_get_termios which always uses a &usb_serial_port.
  */
 static void cp210x_get_termios_port(struct usb_serial_port *port,
-	unsigned int *cflagp, unsigned int *baudp)
+	tcflag_t *cflagp, unsigned int *baudp)
 {
 	struct device *dev = &port->dev;
-	unsigned int cflag;
+	tcflag_t cflag;
 	struct cp210x_flow_ctl flow_ctl;
 	u32 baud;
 	u16 bits;
@@ -930,16 +1089,9 @@ static void cp210x_set_termios(struct tty_struct *tty,
 			dev_dbg(dev, "%s - data bits = 7\n", __func__);
 			break;
 		case CS8:
-			bits |= BITS_DATA_8;
-			dev_dbg(dev, "%s - data bits = 8\n", __func__);
-			break;
-		/*case CS9:
-			bits |= BITS_DATA_9;
-			dev_dbg(dev, "%s - data bits = 9\n", __func__);
-			break;*/
 		default:
-			dev_dbg(dev, "cp210x driver does not support the number of bits requested, using 8 bit mode\n");
 			bits |= BITS_DATA_8;
+			dev_dbg(dev, "%s - data bits = 8\n", __func__);
 			break;
 		}
 		if (cp210x_write_u16_reg(port, CP210X_SET_LINE_CTL, bits))
@@ -1108,10 +1260,188 @@ static void cp210x_break_ctl(struct tty_struct *tty, int break_state)
 	cp210x_write_u16_reg(port, CP210X_SET_BREAK, state);
 }
 
+#ifdef CONFIG_GPIOLIB
+static int cp210x_gpio_request(struct gpio_chip *gc, unsigned int offset)
+{
+	struct usb_serial *serial = gpiochip_get_data(gc);
+	struct cp210x_serial_private *priv = usb_get_serial_data(serial);
+
+	switch (offset) {
+	case 0:
+		if (priv->config & CP2105_GPIO0_TXLED_MODE)
+			return -ENODEV;
+		break;
+	case 1:
+		if (priv->config & (CP2105_GPIO1_RXLED_MODE |
+				    CP2105_GPIO1_RS485_MODE))
+			return -ENODEV;
+		break;
+	}
+
+	return 0;
+}
+
+static int cp210x_gpio_get(struct gpio_chip *gc, unsigned int gpio)
+{
+	struct usb_serial *serial = gpiochip_get_data(gc);
+	int result;
+	u8 buf;
+
+	result = cp210x_read_vendor_block(serial, REQTYPE_INTERFACE_TO_HOST,
+					  CP210X_READ_LATCH, &buf, sizeof(buf));
+	if (result < 0)
+		return result;
+
+	return !!(buf & BIT(gpio));
+}
+
+static void cp210x_gpio_set(struct gpio_chip *gc, unsigned int gpio, int value)
+{
+	struct usb_serial *serial = gpiochip_get_data(gc);
+	struct cp210x_gpio_write buf;
+
+	if (value == 1)
+		buf.state = BIT(gpio);
+	else
+		buf.state = 0;
+
+	buf.mask = BIT(gpio);
+
+	cp210x_write_vendor_block(serial, REQTYPE_HOST_TO_INTERFACE,
+				  CP210X_WRITE_LATCH, &buf, sizeof(buf));
+}
+
+static int cp210x_gpio_direction_get(struct gpio_chip *gc, unsigned int gpio)
+{
+	/* Hardware does not support an input mode */
+	return 0;
+}
+
+static int cp210x_gpio_direction_input(struct gpio_chip *gc, unsigned int gpio)
+{
+	/* Hardware does not support an input mode */
+	return -ENOTSUPP;
+}
+
+static int cp210x_gpio_direction_output(struct gpio_chip *gc, unsigned int gpio,
+					int value)
+{
+	return 0;
+}
+
+static int cp210x_gpio_set_single_ended(struct gpio_chip *gc, unsigned int gpio,
+					enum single_ended_mode mode)
+{
+	struct usb_serial *serial = gpiochip_get_data(gc);
+	struct cp210x_serial_private *priv = usb_get_serial_data(serial);
+
+	/* Succeed only if in correct mode (this can't be set at runtime) */
+	if ((mode == LINE_MODE_PUSH_PULL) && (priv->gpio_mode & BIT(gpio)))
+		return 0;
+
+	if ((mode == LINE_MODE_OPEN_DRAIN) && !(priv->gpio_mode & BIT(gpio)))
+		return 0;
+
+	return -ENOTSUPP;
+}
+
+/*
+ * This function is for configuring GPIO using shared pins, where other signals
+ * are made unavailable by configuring the use of GPIO. This is believed to be
+ * only applicable to the cp2105 at this point, the other devices supported by
+ * this driver that provide GPIO do so in a way that does not impact other
+ * signals and are thus expected to have very different initialisation.
+ */
+static int cp2105_shared_gpio_init(struct usb_serial *serial)
+{
+	struct cp210x_serial_private *priv = usb_get_serial_data(serial);
+	struct cp210x_pin_mode mode;
+	struct cp210x_config config;
+	u8 intf_num = cp210x_interface_num(serial);
+	int result;
+
+	result = cp210x_read_vendor_block(serial, REQTYPE_DEVICE_TO_HOST,
+					  CP210X_GET_DEVICEMODE, &mode,
+					  sizeof(mode));
+	if (result < 0)
+		return result;
+
+	result = cp210x_read_vendor_block(serial, REQTYPE_DEVICE_TO_HOST,
+					  CP210X_GET_PORTCONFIG, &config,
+					  sizeof(config));
+	if (result < 0)
+		return result;
+
+	/*  2 banks of GPIO - One for the pins taken from each serial port */
+	if (intf_num == 0) {
+		if (mode.eci == CP210X_PIN_MODE_MODEM)
+			return 0;
+
+		priv->config = config.eci_cfg;
+		priv->gpio_mode = (u8)((le16_to_cpu(config.gpio_mode) &
+						CP210X_ECI_GPIO_MODE_MASK) >>
+						CP210X_ECI_GPIO_MODE_OFFSET);
+		priv->gc.ngpio = 2;
+	} else if (intf_num == 1) {
+		if (mode.sci == CP210X_PIN_MODE_MODEM)
+			return 0;
+
+		priv->config = config.sci_cfg;
+		priv->gpio_mode = (u8)((le16_to_cpu(config.gpio_mode) &
+						CP210X_SCI_GPIO_MODE_MASK) >>
+						CP210X_SCI_GPIO_MODE_OFFSET);
+		priv->gc.ngpio = 3;
+	} else {
+		return -ENODEV;
+	}
+
+	priv->gc.label = "cp210x";
+	priv->gc.request = cp210x_gpio_request;
+	priv->gc.get_direction = cp210x_gpio_direction_get;
+	priv->gc.direction_input = cp210x_gpio_direction_input;
+	priv->gc.direction_output = cp210x_gpio_direction_output;
+	priv->gc.get = cp210x_gpio_get;
+	priv->gc.set = cp210x_gpio_set;
+	priv->gc.set_single_ended = cp210x_gpio_set_single_ended;
+	priv->gc.owner = THIS_MODULE;
+	priv->gc.parent = &serial->interface->dev;
+	priv->gc.base = -1;
+	priv->gc.can_sleep = true;
+
+	result = gpiochip_add_data(&priv->gc, serial);
+	if (!result)
+		priv->gpio_registered = true;
+
+	return result;
+}
+
+static void cp210x_gpio_remove(struct usb_serial *serial)
+{
+	struct cp210x_serial_private *priv = usb_get_serial_data(serial);
+
+	if (priv->gpio_registered) {
+		gpiochip_remove(&priv->gc);
+		priv->gpio_registered = false;
+	}
+}
+
+#else
+
+static int cp2105_shared_gpio_init(struct usb_serial *serial)
+{
+	return 0;
+}
+
+static void cp210x_gpio_remove(struct usb_serial *serial)
+{
+	/* Nothing to do */
+}
+
+#endif
+
 static int cp210x_port_probe(struct usb_serial_port *port)
 {
 	struct usb_serial *serial = port->serial;
-	struct usb_host_interface *cur_altsetting;
 	struct cp210x_port_private *port_priv;
 	int ret;
 
@@ -1119,8 +1449,7 @@ static int cp210x_port_probe(struct usb_serial_port *port)
 	if (!port_priv)
 		return -ENOMEM;
 
-	cur_altsetting = serial->interface->cur_altsetting;
-	port_priv->bInterfaceNumber = cur_altsetting->desc.bInterfaceNumber;
+	port_priv->bInterfaceNumber = cp210x_interface_num(serial);
 
 	usb_set_serial_port_data(port, port_priv);
 
@@ -1143,6 +1472,52 @@ static int cp210x_port_remove(struct usb_serial_port *port)
 	return 0;
 }
 
+static int cp210x_attach(struct usb_serial *serial)
+{
+	int result;
+	struct cp210x_serial_private *priv;
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	result = cp210x_read_vendor_block(serial, REQTYPE_DEVICE_TO_HOST,
+					  CP210X_GET_PARTNUM, &priv->partnum,
+					  sizeof(priv->partnum));
+	if (result < 0)
+		goto err_free_priv;
+
+	usb_set_serial_data(serial, priv);
+
+	if (priv->partnum == CP210X_PARTNUM_CP2105) {
+		result = cp2105_shared_gpio_init(serial);
+		if (result < 0) {
+			dev_err(&serial->interface->dev,
+				"GPIO initialisation failed, continuing without GPIO support\n");
+		}
+	}
+
+	return 0;
+err_free_priv:
+	kfree(priv);
+
+	return result;
+}
+
+static void cp210x_disconnect(struct usb_serial *serial)
+{
+	cp210x_gpio_remove(serial);
+}
+
+static void cp210x_release(struct usb_serial *serial)
+{
+	struct cp210x_serial_private *priv = usb_get_serial_data(serial);
+
+	cp210x_gpio_remove(serial);
+
+	kfree(priv);
+}
+
 module_usb_serial_driver(serial_drivers, id_table);
 
 MODULE_DESCRIPTION(DRIVER_DESC);
diff --git a/drivers/usb/serial/f81534.c b/drivers/usb/serial/f81534.c
new file mode 100644
index 000000000000..8282a6a18fee
--- /dev/null
+++ b/drivers/usb/serial/f81534.c
@@ -0,0 +1,1409 @@
+/*
+ * F81532/F81534 USB to Serial Ports Bridge
+ *
+ * F81532 => 2 Serial Ports
+ * F81534 => 4 Serial Ports
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * Copyright (C) 2016 Feature Integration Technology Inc., (Fintek)
+ * Copyright (C) 2016 Tom Tsai (Tom_Tsai@fintek.com.tw)
+ * Copyright (C) 2016 Peter Hong (Peter_Hong@fintek.com.tw)
+ *
+ * The F81532/F81534 had 1 control endpoint for setting, 1 endpoint bulk-out
+ * for all serial port TX and 1 endpoint bulk-in for all serial port read in
+ * (Read Data/MSR/LSR).
+ *
+ * Write URB is fixed with 512bytes, per serial port used 128Bytes.
+ * It can be described by f81534_prepare_write_buffer()
+ *
+ * Read URB is 512Bytes max, per serial port used 128Bytes.
+ * It can be described by f81534_process_read_urb() and maybe received with
+ * 128x1,2,3,4 bytes.
+ *
+ */
+#include <linux/slab.h>
+#include <linux/tty.h>
+#include <linux/tty_flip.h>
+#include <linux/usb.h>
+#include <linux/usb/serial.h>
+#include <linux/serial_reg.h>
+#include <linux/module.h>
+#include <linux/uaccess.h>
+
+/* Serial Port register Address */
+#define F81534_UART_BASE_ADDRESS	0x1200
+#define F81534_UART_OFFSET		0x10
+#define F81534_DIVISOR_LSB_REG		(0x00 + F81534_UART_BASE_ADDRESS)
+#define F81534_DIVISOR_MSB_REG		(0x01 + F81534_UART_BASE_ADDRESS)
+#define F81534_FIFO_CONTROL_REG		(0x02 + F81534_UART_BASE_ADDRESS)
+#define F81534_LINE_CONTROL_REG		(0x03 + F81534_UART_BASE_ADDRESS)
+#define F81534_MODEM_CONTROL_REG	(0x04 + F81534_UART_BASE_ADDRESS)
+#define F81534_MODEM_STATUS_REG		(0x06 + F81534_UART_BASE_ADDRESS)
+#define F81534_CONFIG1_REG		(0x09 + F81534_UART_BASE_ADDRESS)
+
+#define F81534_DEF_CONF_ADDRESS_START	0x3000
+#define F81534_DEF_CONF_SIZE		8
+
+#define F81534_CUSTOM_ADDRESS_START	0x2f00
+#define F81534_CUSTOM_DATA_SIZE		0x10
+#define F81534_CUSTOM_NO_CUSTOM_DATA	0xff
+#define F81534_CUSTOM_VALID_TOKEN	0xf0
+#define F81534_CONF_OFFSET		1
+
+#define F81534_MAX_DATA_BLOCK		64
+#define F81534_MAX_BUS_RETRY		20
+
+/* Default URB timeout for USB operations */
+#define F81534_USB_MAX_RETRY		10
+#define F81534_USB_TIMEOUT		1000
+#define F81534_SET_GET_REGISTER		0xA0
+
+#define F81534_NUM_PORT			4
+#define F81534_UNUSED_PORT		0xff
+#define F81534_WRITE_BUFFER_SIZE	512
+
+#define DRIVER_DESC			"Fintek F81532/F81534"
+#define FINTEK_VENDOR_ID_1		0x1934
+#define FINTEK_VENDOR_ID_2		0x2C42
+#define FINTEK_DEVICE_ID		0x1202
+#define F81534_MAX_TX_SIZE		124
+#define F81534_MAX_RX_SIZE		124
+#define F81534_RECEIVE_BLOCK_SIZE	128
+#define F81534_MAX_RECEIVE_BLOCK_SIZE	512
+
+#define F81534_TOKEN_RECEIVE		0x01
+#define F81534_TOKEN_WRITE		0x02
+#define F81534_TOKEN_TX_EMPTY		0x03
+#define F81534_TOKEN_MSR_CHANGE		0x04
+
+/*
+ * We used interal SPI bus to access FLASH section. We must wait the SPI bus to
+ * idle if we performed any command.
+ *
+ * SPI Bus status register: F81534_BUS_REG_STATUS
+ *	Bit 0/1	: BUSY
+ *	Bit 2	: IDLE
+ */
+#define F81534_BUS_BUSY			(BIT(0) | BIT(1))
+#define F81534_BUS_IDLE			BIT(2)
+#define F81534_BUS_READ_DATA		0x1004
+#define F81534_BUS_REG_STATUS		0x1003
+#define F81534_BUS_REG_START		0x1002
+#define F81534_BUS_REG_END		0x1001
+
+#define F81534_CMD_READ			0x03
+
+#define F81534_DEFAULT_BAUD_RATE	9600
+#define F81534_MAX_BAUDRATE		115200
+
+#define F81534_PORT_CONF_DISABLE_PORT	BIT(3)
+#define F81534_PORT_CONF_NOT_EXIST_PORT	BIT(7)
+#define F81534_PORT_UNAVAILABLE		\
+	(F81534_PORT_CONF_DISABLE_PORT | F81534_PORT_CONF_NOT_EXIST_PORT)
+
+#define F81534_1X_RXTRIGGER		0xc3
+#define F81534_8X_RXTRIGGER		0xcf
+
+static const struct usb_device_id f81534_id_table[] = {
+	{ USB_DEVICE(FINTEK_VENDOR_ID_1, FINTEK_DEVICE_ID) },
+	{ USB_DEVICE(FINTEK_VENDOR_ID_2, FINTEK_DEVICE_ID) },
+	{}			/* Terminating entry */
+};
+
+#define F81534_TX_EMPTY_BIT		0
+
+struct f81534_serial_private {
+	u8 conf_data[F81534_DEF_CONF_SIZE];
+	int tty_idx[F81534_NUM_PORT];
+	u8 setting_idx;
+	int opened_port;
+	struct mutex urb_mutex;
+};
+
+struct f81534_port_private {
+	struct mutex mcr_mutex;
+	unsigned long tx_empty;
+	spinlock_t msr_lock;
+	u8 shadow_mcr;
+	u8 shadow_msr;
+	u8 phy_num;
+};
+
+static int f81534_logic_to_phy_port(struct usb_serial *serial,
+					struct usb_serial_port *port)
+{
+	struct f81534_serial_private *serial_priv =
+			usb_get_serial_data(port->serial);
+	int count = 0;
+	int i;
+
+	for (i = 0; i < F81534_NUM_PORT; ++i) {
+		if (serial_priv->conf_data[i] & F81534_PORT_UNAVAILABLE)
+			continue;
+
+		if (port->port_number == count)
+			return i;
+
+		++count;
+	}
+
+	return -ENODEV;
+}
+
+static int f81534_set_register(struct usb_serial *serial, u16 reg, u8 data)
+{
+	struct usb_interface *interface = serial->interface;
+	struct usb_device *dev = serial->dev;
+	size_t count = F81534_USB_MAX_RETRY;
+	int status;
+	u8 *tmp;
+
+	tmp = kmalloc(sizeof(u8), GFP_KERNEL);
+	if (!tmp)
+		return -ENOMEM;
+
+	*tmp = data;
+
+	/*
+	 * Our device maybe not reply when heavily loading, We'll retry for
+	 * F81534_USB_MAX_RETRY times.
+	 */
+	while (count--) {
+		status = usb_control_msg(dev, usb_sndctrlpipe(dev, 0),
+					 F81534_SET_GET_REGISTER,
+					 USB_TYPE_VENDOR | USB_DIR_OUT,
+					 reg, 0, tmp, sizeof(u8),
+					 F81534_USB_TIMEOUT);
+		if (status > 0) {
+			status = 0;
+			break;
+		} else if (status == 0) {
+			status = -EIO;
+		}
+	}
+
+	if (status < 0) {
+		dev_err(&interface->dev, "%s: reg: %x data: %x failed: %d\n",
+				__func__, reg, data, status);
+	}
+
+	kfree(tmp);
+	return status;
+}
+
+static int f81534_get_register(struct usb_serial *serial, u16 reg, u8 *data)
+{
+	struct usb_interface *interface = serial->interface;
+	struct usb_device *dev = serial->dev;
+	size_t count = F81534_USB_MAX_RETRY;
+	int status;
+	u8 *tmp;
+
+	tmp = kmalloc(sizeof(u8), GFP_KERNEL);
+	if (!tmp)
+		return -ENOMEM;
+
+	/*
+	 * Our device maybe not reply when heavily loading, We'll retry for
+	 * F81534_USB_MAX_RETRY times.
+	 */
+	while (count--) {
+		status = usb_control_msg(dev, usb_rcvctrlpipe(dev, 0),
+					 F81534_SET_GET_REGISTER,
+					 USB_TYPE_VENDOR | USB_DIR_IN,
+					 reg, 0, tmp, sizeof(u8),
+					 F81534_USB_TIMEOUT);
+		if (status > 0) {
+			status = 0;
+			break;
+		} else if (status == 0) {
+			status = -EIO;
+		}
+	}
+
+	if (status < 0) {
+		dev_err(&interface->dev, "%s: reg: %x failed: %d\n", __func__,
+				reg, status);
+		goto end;
+	}
+
+	*data = *tmp;
+
+end:
+	kfree(tmp);
+	return status;
+}
+
+static int f81534_set_port_register(struct usb_serial_port *port, u16 reg,
+					u8 data)
+{
+	struct f81534_port_private *port_priv = usb_get_serial_port_data(port);
+
+	return f81534_set_register(port->serial,
+			reg + port_priv->phy_num * F81534_UART_OFFSET, data);
+}
+
+static int f81534_get_port_register(struct usb_serial_port *port, u16 reg,
+					u8 *data)
+{
+	struct f81534_port_private *port_priv = usb_get_serial_port_data(port);
+
+	return f81534_get_register(port->serial,
+			reg + port_priv->phy_num * F81534_UART_OFFSET, data);
+}
+
+/*
+ * If we try to access the internal flash via SPI bus, we should check the bus
+ * status for every command. e.g., F81534_BUS_REG_START/F81534_BUS_REG_END
+ */
+static int f81534_wait_for_spi_idle(struct usb_serial *serial)
+{
+	size_t count = F81534_MAX_BUS_RETRY;
+	u8 tmp;
+	int status;
+
+	do {
+		status = f81534_get_register(serial, F81534_BUS_REG_STATUS,
+						&tmp);
+		if (status)
+			return status;
+
+		if (tmp & F81534_BUS_BUSY)
+			continue;
+
+		if (tmp & F81534_BUS_IDLE)
+			break;
+
+	} while (--count);
+
+	if (!count) {
+		dev_err(&serial->interface->dev,
+				"%s: timed out waiting for idle SPI bus\n",
+				__func__);
+		return -EIO;
+	}
+
+	return f81534_set_register(serial, F81534_BUS_REG_STATUS,
+				tmp & ~F81534_BUS_IDLE);
+}
+
+static int f81534_get_spi_register(struct usb_serial *serial, u16 reg,
+					u8 *data)
+{
+	int status;
+
+	status = f81534_get_register(serial, reg, data);
+	if (status)
+		return status;
+
+	return f81534_wait_for_spi_idle(serial);
+}
+
+static int f81534_set_spi_register(struct usb_serial *serial, u16 reg, u8 data)
+{
+	int status;
+
+	status = f81534_set_register(serial, reg, data);
+	if (status)
+		return status;
+
+	return f81534_wait_for_spi_idle(serial);
+}
+
+static int f81534_read_flash(struct usb_serial *serial, u32 address,
+				size_t size, u8 *buf)
+{
+	u8 tmp_buf[F81534_MAX_DATA_BLOCK];
+	size_t block = 0;
+	size_t read_size;
+	size_t count;
+	int status;
+	int offset;
+	u16 reg_tmp;
+
+	status = f81534_set_spi_register(serial, F81534_BUS_REG_START,
+					F81534_CMD_READ);
+	if (status)
+		return status;
+
+	status = f81534_set_spi_register(serial, F81534_BUS_REG_START,
+					(address >> 16) & 0xff);
+	if (status)
+		return status;
+
+	status = f81534_set_spi_register(serial, F81534_BUS_REG_START,
+					(address >> 8) & 0xff);
+	if (status)
+		return status;
+
+	status = f81534_set_spi_register(serial, F81534_BUS_REG_START,
+					(address >> 0) & 0xff);
+	if (status)
+		return status;
+
+	/* Continuous read mode */
+	do {
+		read_size = min_t(size_t, F81534_MAX_DATA_BLOCK, size);
+
+		for (count = 0; count < read_size; ++count) {
+			/* To write F81534_BUS_REG_END when final byte */
+			if (size <= F81534_MAX_DATA_BLOCK &&
+					read_size == count + 1)
+				reg_tmp = F81534_BUS_REG_END;
+			else
+				reg_tmp = F81534_BUS_REG_START;
+
+			/*
+			 * Dummy code, force IC to generate a read pulse, the
+			 * set of value 0xf1 is dont care (any value is ok)
+			 */
+			status = f81534_set_spi_register(serial, reg_tmp,
+					0xf1);
+			if (status)
+				return status;
+
+			status = f81534_get_spi_register(serial,
+						F81534_BUS_READ_DATA,
+						&tmp_buf[count]);
+			if (status)
+				return status;
+
+			offset = count + block * F81534_MAX_DATA_BLOCK;
+			buf[offset] = tmp_buf[count];
+		}
+
+		size -= read_size;
+		++block;
+	} while (size);
+
+	return 0;
+}
+
+static void f81534_prepare_write_buffer(struct usb_serial_port *port, u8 *buf)
+{
+	struct f81534_port_private *port_priv = usb_get_serial_port_data(port);
+	int phy_num = port_priv->phy_num;
+	u8 tx_len;
+	int i;
+
+	/*
+	 * The block layout is fixed with 4x128 Bytes, per 128 Bytes a port.
+	 * index 0: port phy idx (e.g., 0,1,2,3)
+	 * index 1: only F81534_TOKEN_WRITE
+	 * index 2: serial TX out length
+	 * index 3: fix to 0
+	 * index 4~127: serial out data block
+	 */
+	for (i = 0; i < F81534_NUM_PORT; ++i) {
+		buf[i * F81534_RECEIVE_BLOCK_SIZE] = i;
+		buf[i * F81534_RECEIVE_BLOCK_SIZE + 1] = F81534_TOKEN_WRITE;
+		buf[i * F81534_RECEIVE_BLOCK_SIZE + 2] = 0;
+		buf[i * F81534_RECEIVE_BLOCK_SIZE + 3] = 0;
+	}
+
+	tx_len = kfifo_out_locked(&port->write_fifo,
+				&buf[phy_num * F81534_RECEIVE_BLOCK_SIZE + 4],
+				F81534_MAX_TX_SIZE, &port->lock);
+
+	buf[phy_num * F81534_RECEIVE_BLOCK_SIZE + 2] = tx_len;
+}
+
+static int f81534_submit_writer(struct usb_serial_port *port, gfp_t mem_flags)
+{
+	struct f81534_port_private *port_priv = usb_get_serial_port_data(port);
+	struct urb *urb;
+	unsigned long flags;
+	int result;
+
+	/* Check is any data in write_fifo */
+	spin_lock_irqsave(&port->lock, flags);
+
+	if (kfifo_is_empty(&port->write_fifo)) {
+		spin_unlock_irqrestore(&port->lock, flags);
+		return 0;
+	}
+
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	/* Check H/W is TXEMPTY */
+	if (!test_and_clear_bit(F81534_TX_EMPTY_BIT, &port_priv->tx_empty))
+		return 0;
+
+	urb = port->write_urbs[0];
+	f81534_prepare_write_buffer(port, port->bulk_out_buffers[0]);
+	urb->transfer_buffer_length = F81534_WRITE_BUFFER_SIZE;
+
+	result = usb_submit_urb(urb, mem_flags);
+	if (result) {
+		set_bit(F81534_TX_EMPTY_BIT, &port_priv->tx_empty);
+		dev_err(&port->dev, "%s: submit failed: %d\n", __func__,
+				result);
+		return result;
+	}
+
+	usb_serial_port_softint(port);
+	return 0;
+}
+
+static u32 f81534_calc_baud_divisor(u32 baudrate, u32 clockrate)
+{
+	if (!baudrate)
+		return 0;
+
+	/* Round to nearest divisor */
+	return DIV_ROUND_CLOSEST(clockrate, baudrate);
+}
+
+static int f81534_set_port_config(struct usb_serial_port *port, u32 baudrate,
+					u8 lcr)
+{
+	u32 divisor;
+	int status;
+	u8 value;
+
+	if (baudrate <= 1200)
+		value = F81534_1X_RXTRIGGER;	/* 128 FIFO & TL: 1x */
+	else
+		value = F81534_8X_RXTRIGGER;	/* 128 FIFO & TL: 8x */
+
+	status = f81534_set_port_register(port, F81534_CONFIG1_REG, value);
+	if (status) {
+		dev_err(&port->dev, "%s: CONFIG1 setting failed\n", __func__);
+		return status;
+	}
+
+	if (baudrate <= 1200)
+		value = UART_FCR_TRIGGER_1 | UART_FCR_ENABLE_FIFO; /* TL: 1 */
+	else
+		value = UART_FCR_R_TRIG_11 | UART_FCR_ENABLE_FIFO; /* TL: 14 */
+
+	status = f81534_set_port_register(port, F81534_FIFO_CONTROL_REG,
+						value);
+	if (status) {
+		dev_err(&port->dev, "%s: FCR setting failed\n", __func__);
+		return status;
+	}
+
+	divisor = f81534_calc_baud_divisor(baudrate, F81534_MAX_BAUDRATE);
+	value = UART_LCR_DLAB;
+	status = f81534_set_port_register(port, F81534_LINE_CONTROL_REG,
+						value);
+	if (status) {
+		dev_err(&port->dev, "%s: set LCR failed\n", __func__);
+		return status;
+	}
+
+	value = divisor & 0xff;
+	status = f81534_set_port_register(port, F81534_DIVISOR_LSB_REG, value);
+	if (status) {
+		dev_err(&port->dev, "%s: set DLAB LSB failed\n", __func__);
+		return status;
+	}
+
+	value = (divisor >> 8) & 0xff;
+	status = f81534_set_port_register(port, F81534_DIVISOR_MSB_REG, value);
+	if (status) {
+		dev_err(&port->dev, "%s: set DLAB MSB failed\n", __func__);
+		return status;
+	}
+
+	status = f81534_set_port_register(port, F81534_LINE_CONTROL_REG, lcr);
+	if (status) {
+		dev_err(&port->dev, "%s: set LCR failed\n", __func__);
+		return status;
+	}
+
+	return 0;
+}
+
+static int f81534_update_mctrl(struct usb_serial_port *port, unsigned int set,
+				unsigned int clear)
+{
+	struct f81534_port_private *port_priv = usb_get_serial_port_data(port);
+	int status;
+	u8 tmp;
+
+	if (((set | clear) & (TIOCM_DTR | TIOCM_RTS)) == 0)
+		return 0;	/* no change */
+
+	mutex_lock(&port_priv->mcr_mutex);
+
+	/* 'Set' takes precedence over 'Clear' */
+	clear &= ~set;
+
+	/* Always enable UART_MCR_OUT2 */
+	tmp = UART_MCR_OUT2 | port_priv->shadow_mcr;
+
+	if (clear & TIOCM_DTR)
+		tmp &= ~UART_MCR_DTR;
+
+	if (clear & TIOCM_RTS)
+		tmp &= ~UART_MCR_RTS;
+
+	if (set & TIOCM_DTR)
+		tmp |= UART_MCR_DTR;
+
+	if (set & TIOCM_RTS)
+		tmp |= UART_MCR_RTS;
+
+	status = f81534_set_port_register(port, F81534_MODEM_CONTROL_REG, tmp);
+	if (status < 0) {
+		dev_err(&port->dev, "%s: MCR write failed\n", __func__);
+		mutex_unlock(&port_priv->mcr_mutex);
+		return status;
+	}
+
+	port_priv->shadow_mcr = tmp;
+	mutex_unlock(&port_priv->mcr_mutex);
+	return 0;
+}
+
+/*
+ * This function will search the data area with token F81534_CUSTOM_VALID_TOKEN
+ * for latest configuration index. If nothing found
+ * (*index = F81534_CUSTOM_NO_CUSTOM_DATA), We'll load default configure in
+ * F81534_DEF_CONF_ADDRESS_START section.
+ *
+ * Due to we only use block0 to save data, so *index should be 0 or
+ * F81534_CUSTOM_NO_CUSTOM_DATA.
+ */
+static int f81534_find_config_idx(struct usb_serial *serial, u8 *index)
+{
+	u8 tmp;
+	int status;
+
+	status = f81534_read_flash(serial, F81534_CUSTOM_ADDRESS_START, 1,
+					&tmp);
+	if (status) {
+		dev_err(&serial->interface->dev, "%s: read failed: %d\n",
+				__func__, status);
+		return status;
+	}
+
+	/* We'll use the custom data when the data is valid. */
+	if (tmp == F81534_CUSTOM_VALID_TOKEN)
+		*index = 0;
+	else
+		*index = F81534_CUSTOM_NO_CUSTOM_DATA;
+
+	return 0;
+}
+
+/*
+ * We had 2 generation of F81532/534 IC. All has an internal storage.
+ *
+ * 1st is pure USB-to-TTL RS232 IC and designed for 4 ports only, no any
+ * internal data will used. All mode and gpio control should manually set
+ * by AP or Driver and all storage space value are 0xff. The
+ * f81534_calc_num_ports() will run to final we marked as "oldest version"
+ * for this IC.
+ *
+ * 2rd is designed to more generic to use any transceiver and this is our
+ * mass production type. We'll save data in F81534_CUSTOM_ADDRESS_START
+ * (0x2f00) with 9bytes. The 1st byte is a indicater. If the token is
+ * F81534_CUSTOM_VALID_TOKEN(0xf0), the IC is 2nd gen type, the following
+ * 4bytes save port mode (0:RS232/1:RS485 Invert/2:RS485), and the last
+ * 4bytes save GPIO state(value from 0~7 to represent 3 GPIO output pin).
+ * The f81534_calc_num_ports() will run to "new style" with checking
+ * F81534_PORT_UNAVAILABLE section.
+ */
+static int f81534_calc_num_ports(struct usb_serial *serial)
+{
+	u8 setting[F81534_CUSTOM_DATA_SIZE];
+	u8 setting_idx;
+	u8 num_port = 0;
+	int status;
+	size_t i;
+
+	/* Check had custom setting */
+	status = f81534_find_config_idx(serial, &setting_idx);
+	if (status) {
+		dev_err(&serial->interface->dev, "%s: find idx failed: %d\n",
+				__func__, status);
+		return 0;
+	}
+
+	/*
+	 * We'll read custom data only when data available, otherwise we'll
+	 * read default value instead.
+	 */
+	if (setting_idx != F81534_CUSTOM_NO_CUSTOM_DATA) {
+		status = f81534_read_flash(serial,
+						F81534_CUSTOM_ADDRESS_START +
+						F81534_CONF_OFFSET,
+						sizeof(setting), setting);
+		if (status) {
+			dev_err(&serial->interface->dev,
+					"%s: get custom data failed: %d\n",
+					__func__, status);
+			return 0;
+		}
+
+		dev_dbg(&serial->interface->dev,
+				"%s: read config from block: %d\n", __func__,
+				setting_idx);
+	} else {
+		/* Read default board setting */
+		status = f81534_read_flash(serial,
+				F81534_DEF_CONF_ADDRESS_START, F81534_NUM_PORT,
+				setting);
+
+		if (status) {
+			dev_err(&serial->interface->dev,
+					"%s: read failed: %d\n", __func__,
+					status);
+			return 0;
+		}
+
+		dev_dbg(&serial->interface->dev, "%s: read default config\n",
+				__func__);
+	}
+
+	/* New style, find all possible ports */
+	for (i = 0; i < F81534_NUM_PORT; ++i) {
+		if (setting[i] & F81534_PORT_UNAVAILABLE)
+			continue;
+
+		++num_port;
+	}
+
+	if (num_port)
+		return num_port;
+
+	dev_warn(&serial->interface->dev, "%s: Read Failed. default 4 ports\n",
+			__func__);
+	return 4;		/* Nothing found, oldest version IC */
+}
+
+static void f81534_set_termios(struct tty_struct *tty,
+				struct usb_serial_port *port,
+				struct ktermios *old_termios)
+{
+	u8 new_lcr = 0;
+	int status;
+	u32 baud;
+
+	if (C_BAUD(tty) == B0)
+		f81534_update_mctrl(port, 0, TIOCM_DTR | TIOCM_RTS);
+	else if (old_termios && (old_termios->c_cflag & CBAUD) == B0)
+		f81534_update_mctrl(port, TIOCM_DTR | TIOCM_RTS, 0);
+
+	if (C_PARENB(tty)) {
+		new_lcr |= UART_LCR_PARITY;
+
+		if (!C_PARODD(tty))
+			new_lcr |= UART_LCR_EPAR;
+
+		if (C_CMSPAR(tty))
+			new_lcr |= UART_LCR_SPAR;
+	}
+
+	if (C_CSTOPB(tty))
+		new_lcr |= UART_LCR_STOP;
+
+	switch (C_CSIZE(tty)) {
+	case CS5:
+		new_lcr |= UART_LCR_WLEN5;
+		break;
+	case CS6:
+		new_lcr |= UART_LCR_WLEN6;
+		break;
+	case CS7:
+		new_lcr |= UART_LCR_WLEN7;
+		break;
+	default:
+	case CS8:
+		new_lcr |= UART_LCR_WLEN8;
+		break;
+	}
+
+	baud = tty_get_baud_rate(tty);
+	if (!baud)
+		return;
+
+	if (baud > F81534_MAX_BAUDRATE) {
+		if (old_termios)
+			baud = tty_termios_baud_rate(old_termios);
+		else
+			baud = F81534_DEFAULT_BAUD_RATE;
+
+		tty_encode_baud_rate(tty, baud, baud);
+	}
+
+	dev_dbg(&port->dev, "%s: baud: %d\n", __func__, baud);
+
+	status = f81534_set_port_config(port, baud, new_lcr);
+	if (status < 0) {
+		dev_err(&port->dev, "%s: set port config failed: %d\n",
+				__func__, status);
+	}
+}
+
+static int f81534_submit_read_urb(struct usb_serial *serial, gfp_t flags)
+{
+	return usb_serial_generic_submit_read_urbs(serial->port[0], flags);
+}
+
+static void f81534_msr_changed(struct usb_serial_port *port, u8 msr)
+{
+	struct f81534_port_private *port_priv = usb_get_serial_port_data(port);
+	struct tty_struct *tty;
+	unsigned long flags;
+	u8 old_msr;
+
+	if (!(msr & UART_MSR_ANY_DELTA))
+		return;
+
+	spin_lock_irqsave(&port_priv->msr_lock, flags);
+	old_msr = port_priv->shadow_msr;
+	port_priv->shadow_msr = msr;
+	spin_unlock_irqrestore(&port_priv->msr_lock, flags);
+
+	dev_dbg(&port->dev, "%s: MSR from %02x to %02x\n", __func__, old_msr,
+			msr);
+
+	/* Update input line counters */
+	if (msr & UART_MSR_DCTS)
+		port->icount.cts++;
+	if (msr & UART_MSR_DDSR)
+		port->icount.dsr++;
+	if (msr & UART_MSR_DDCD)
+		port->icount.dcd++;
+	if (msr & UART_MSR_TERI)
+		port->icount.rng++;
+
+	wake_up_interruptible(&port->port.delta_msr_wait);
+
+	if (!(msr & UART_MSR_DDCD))
+		return;
+
+	dev_dbg(&port->dev, "%s: DCD Changed: phy_num: %d from %x to %x\n",
+			__func__, port_priv->phy_num, old_msr, msr);
+
+	tty = tty_port_tty_get(&port->port);
+	if (!tty)
+		return;
+
+	usb_serial_handle_dcd_change(port, tty, msr & UART_MSR_DCD);
+	tty_kref_put(tty);
+}
+
+static int f81534_read_msr(struct usb_serial_port *port)
+{
+	struct f81534_port_private *port_priv = usb_get_serial_port_data(port);
+	unsigned long flags;
+	int status;
+	u8 msr;
+
+	/* Get MSR initial value */
+	status = f81534_get_port_register(port, F81534_MODEM_STATUS_REG, &msr);
+	if (status)
+		return status;
+
+	/* Force update current state */
+	spin_lock_irqsave(&port_priv->msr_lock, flags);
+	port_priv->shadow_msr = msr;
+	spin_unlock_irqrestore(&port_priv->msr_lock, flags);
+
+	return 0;
+}
+
+static int f81534_open(struct tty_struct *tty, struct usb_serial_port *port)
+{
+	struct f81534_serial_private *serial_priv =
+			usb_get_serial_data(port->serial);
+	struct f81534_port_private *port_priv = usb_get_serial_port_data(port);
+	int status;
+
+	status = f81534_set_port_register(port,
+				F81534_FIFO_CONTROL_REG, UART_FCR_ENABLE_FIFO |
+				UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT);
+	if (status) {
+		dev_err(&port->dev, "%s: Clear FIFO failed: %d\n", __func__,
+				status);
+		return status;
+	}
+
+	if (tty)
+		f81534_set_termios(tty, port, NULL);
+
+	status = f81534_read_msr(port);
+	if (status)
+		return status;
+
+	mutex_lock(&serial_priv->urb_mutex);
+
+	/* Submit Read URBs for first port opened */
+	if (!serial_priv->opened_port) {
+		status = f81534_submit_read_urb(port->serial, GFP_KERNEL);
+		if (status)
+			goto exit;
+	}
+
+	serial_priv->opened_port++;
+
+exit:
+	mutex_unlock(&serial_priv->urb_mutex);
+
+	set_bit(F81534_TX_EMPTY_BIT, &port_priv->tx_empty);
+	return status;
+}
+
+static void f81534_close(struct usb_serial_port *port)
+{
+	struct f81534_serial_private *serial_priv =
+			usb_get_serial_data(port->serial);
+	struct usb_serial_port *port0 = port->serial->port[0];
+	unsigned long flags;
+	size_t i;
+
+	usb_kill_urb(port->write_urbs[0]);
+
+	spin_lock_irqsave(&port->lock, flags);
+	kfifo_reset_out(&port->write_fifo);
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	/* Kill Read URBs when final port closed */
+	mutex_lock(&serial_priv->urb_mutex);
+	serial_priv->opened_port--;
+
+	if (!serial_priv->opened_port) {
+		for (i = 0; i < ARRAY_SIZE(port0->read_urbs); ++i)
+			usb_kill_urb(port0->read_urbs[i]);
+	}
+
+	mutex_unlock(&serial_priv->urb_mutex);
+}
+
+static int f81534_get_serial_info(struct usb_serial_port *port,
+				  struct serial_struct __user *retinfo)
+{
+	struct f81534_port_private *port_priv;
+	struct serial_struct tmp;
+
+	port_priv = usb_get_serial_port_data(port);
+
+	memset(&tmp, 0, sizeof(tmp));
+
+	tmp.type = PORT_16550A;
+	tmp.port = port->port_number;
+	tmp.line = port->minor;
+	tmp.baud_base = F81534_MAX_BAUDRATE;
+
+	if (copy_to_user(retinfo, &tmp, sizeof(*retinfo)))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int f81534_ioctl(struct tty_struct *tty, unsigned int cmd,
+			unsigned long arg)
+{
+	struct usb_serial_port *port = tty->driver_data;
+	struct serial_struct __user *buf = (struct serial_struct __user *)arg;
+
+	switch (cmd) {
+	case TIOCGSERIAL:
+		return f81534_get_serial_info(port, buf);
+	default:
+		break;
+	}
+
+	return -ENOIOCTLCMD;
+}
+
+static void f81534_process_per_serial_block(struct usb_serial_port *port,
+		u8 *data)
+{
+	struct f81534_port_private *port_priv = usb_get_serial_port_data(port);
+	int phy_num = data[0];
+	size_t read_size = 0;
+	size_t i;
+	char tty_flag;
+	int status;
+	u8 lsr;
+
+	/*
+	 * The block layout is 128 Bytes
+	 * index 0: port phy idx (e.g., 0,1,2,3),
+	 * index 1: It's could be
+	 *			F81534_TOKEN_RECEIVE
+	 *			F81534_TOKEN_TX_EMPTY
+	 *			F81534_TOKEN_MSR_CHANGE
+	 * index 2: serial in size (data+lsr, must be even)
+	 *			meaningful for F81534_TOKEN_RECEIVE only
+	 * index 3: current MSR with this device
+	 * index 4~127: serial in data block (data+lsr, must be even)
+	 */
+	switch (data[1]) {
+	case F81534_TOKEN_TX_EMPTY:
+		set_bit(F81534_TX_EMPTY_BIT, &port_priv->tx_empty);
+
+		/* Try to submit writer */
+		status = f81534_submit_writer(port, GFP_ATOMIC);
+		if (status)
+			dev_err(&port->dev, "%s: submit failed\n", __func__);
+		return;
+
+	case F81534_TOKEN_MSR_CHANGE:
+		f81534_msr_changed(port, data[3]);
+		return;
+
+	case F81534_TOKEN_RECEIVE:
+		read_size = data[2];
+		if (read_size > F81534_MAX_RX_SIZE) {
+			dev_err(&port->dev,
+				"%s: phy: %d read_size: %zu larger than: %d\n",
+				__func__, phy_num, read_size,
+				F81534_MAX_RX_SIZE);
+			return;
+		}
+
+		break;
+
+	default:
+		dev_warn(&port->dev, "%s: unknown token: %02x\n", __func__,
+				data[1]);
+		return;
+	}
+
+	for (i = 4; i < 4 + read_size; i += 2) {
+		tty_flag = TTY_NORMAL;
+		lsr = data[i + 1];
+
+		if (lsr & UART_LSR_BRK_ERROR_BITS) {
+			if (lsr & UART_LSR_BI) {
+				tty_flag = TTY_BREAK;
+				port->icount.brk++;
+				usb_serial_handle_break(port);
+			} else if (lsr & UART_LSR_PE) {
+				tty_flag = TTY_PARITY;
+				port->icount.parity++;
+			} else if (lsr & UART_LSR_FE) {
+				tty_flag = TTY_FRAME;
+				port->icount.frame++;
+			}
+
+			if (lsr & UART_LSR_OE) {
+				port->icount.overrun++;
+				tty_insert_flip_char(&port->port, 0,
+						TTY_OVERRUN);
+			}
+		}
+
+		if (port->port.console && port->sysrq) {
+			if (usb_serial_handle_sysrq_char(port, data[i]))
+				continue;
+		}
+
+		tty_insert_flip_char(&port->port, data[i], tty_flag);
+	}
+
+	tty_flip_buffer_push(&port->port);
+}
+
+static void f81534_process_read_urb(struct urb *urb)
+{
+	struct f81534_serial_private *serial_priv;
+	struct usb_serial_port *port;
+	struct usb_serial *serial;
+	u8 *buf;
+	int phy_port_num;
+	int tty_port_num;
+	size_t i;
+
+	if (!urb->actual_length ||
+			urb->actual_length % F81534_RECEIVE_BLOCK_SIZE) {
+		return;
+	}
+
+	port = urb->context;
+	serial = port->serial;
+	buf = urb->transfer_buffer;
+	serial_priv = usb_get_serial_data(serial);
+
+	for (i = 0; i < urb->actual_length; i += F81534_RECEIVE_BLOCK_SIZE) {
+		phy_port_num = buf[i];
+		if (phy_port_num >= F81534_NUM_PORT) {
+			dev_err(&port->dev,
+				"%s: phy_port_num: %d larger than: %d\n",
+				__func__, phy_port_num, F81534_NUM_PORT);
+			continue;
+		}
+
+		tty_port_num = serial_priv->tty_idx[phy_port_num];
+		port = serial->port[tty_port_num];
+
+		if (tty_port_initialized(&port->port))
+			f81534_process_per_serial_block(port, &buf[i]);
+	}
+}
+
+static void f81534_write_usb_callback(struct urb *urb)
+{
+	struct usb_serial_port *port = urb->context;
+
+	switch (urb->status) {
+	case 0:
+		break;
+	case -ENOENT:
+	case -ECONNRESET:
+	case -ESHUTDOWN:
+		dev_dbg(&port->dev, "%s - urb stopped: %d\n",
+				__func__, urb->status);
+		return;
+	case -EPIPE:
+		dev_err(&port->dev, "%s - urb stopped: %d\n",
+				__func__, urb->status);
+		return;
+	default:
+		dev_dbg(&port->dev, "%s - nonzero urb status: %d\n",
+				__func__, urb->status);
+		break;
+	}
+}
+
+static int f81534_setup_ports(struct usb_serial *serial)
+{
+	struct usb_serial_port *port;
+	u8 port0_out_address;
+	int buffer_size;
+	size_t i;
+
+	/*
+	 * In our system architecture, we had 2 or 4 serial ports,
+	 * but only get 1 set of bulk in/out endpoints.
+	 *
+	 * The usb-serial subsystem will generate port 0 data,
+	 * but port 1/2/3 will not. It's will generate write URB and buffer
+	 * by following code and use the port0 read URB for read operation.
+	 */
+	for (i = 1; i < serial->num_ports; ++i) {
+		port0_out_address = serial->port[0]->bulk_out_endpointAddress;
+		buffer_size = serial->port[0]->bulk_out_size;
+		port = serial->port[i];
+
+		if (kfifo_alloc(&port->write_fifo, PAGE_SIZE, GFP_KERNEL))
+			return -ENOMEM;
+
+		port->bulk_out_size = buffer_size;
+		port->bulk_out_endpointAddress = port0_out_address;
+
+		port->write_urbs[0] = usb_alloc_urb(0, GFP_KERNEL);
+		if (!port->write_urbs[0])
+			return -ENOMEM;
+
+		port->bulk_out_buffers[0] = kzalloc(buffer_size, GFP_KERNEL);
+		if (!port->bulk_out_buffers[0])
+			return -ENOMEM;
+
+		usb_fill_bulk_urb(port->write_urbs[0], serial->dev,
+				usb_sndbulkpipe(serial->dev,
+					port0_out_address),
+				port->bulk_out_buffers[0], buffer_size,
+				serial->type->write_bulk_callback, port);
+
+		port->write_urb = port->write_urbs[0];
+		port->bulk_out_buffer = port->bulk_out_buffers[0];
+	}
+
+	return 0;
+}
+
+static int f81534_probe(struct usb_serial *serial,
+					const struct usb_device_id *id)
+{
+	struct usb_endpoint_descriptor *endpoint;
+	struct usb_host_interface *iface_desc;
+	struct device *dev;
+	int num_bulk_in = 0;
+	int num_bulk_out = 0;
+	int size_bulk_in = 0;
+	int size_bulk_out = 0;
+	int i;
+
+	dev = &serial->interface->dev;
+	iface_desc = serial->interface->cur_altsetting;
+
+	for (i = 0; i < iface_desc->desc.bNumEndpoints; ++i) {
+		endpoint = &iface_desc->endpoint[i].desc;
+
+		if (usb_endpoint_is_bulk_in(endpoint)) {
+			++num_bulk_in;
+			size_bulk_in = usb_endpoint_maxp(endpoint);
+		}
+
+		if (usb_endpoint_is_bulk_out(endpoint)) {
+			++num_bulk_out;
+			size_bulk_out = usb_endpoint_maxp(endpoint);
+		}
+	}
+
+	if (num_bulk_in != 1 || num_bulk_out != 1) {
+		dev_err(dev, "expected endpoints not found\n");
+		return -ENODEV;
+	}
+
+	if (size_bulk_out != F81534_WRITE_BUFFER_SIZE ||
+			size_bulk_in != F81534_MAX_RECEIVE_BLOCK_SIZE) {
+		dev_err(dev, "unsupported endpoint max packet size\n");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int f81534_attach(struct usb_serial *serial)
+{
+	struct f81534_serial_private *serial_priv;
+	int index = 0;
+	int status;
+	int i;
+
+	serial_priv = devm_kzalloc(&serial->interface->dev,
+					sizeof(*serial_priv), GFP_KERNEL);
+	if (!serial_priv)
+		return -ENOMEM;
+
+	usb_set_serial_data(serial, serial_priv);
+
+	mutex_init(&serial_priv->urb_mutex);
+
+	status = f81534_setup_ports(serial);
+	if (status)
+		return status;
+
+	/* Check had custom setting */
+	status = f81534_find_config_idx(serial, &serial_priv->setting_idx);
+	if (status) {
+		dev_err(&serial->interface->dev, "%s: find idx failed: %d\n",
+				__func__, status);
+		return status;
+	}
+
+	/*
+	 * We'll read custom data only when data available, otherwise we'll
+	 * read default value instead.
+	 */
+	if (serial_priv->setting_idx == F81534_CUSTOM_NO_CUSTOM_DATA) {
+		/*
+		 * The default configuration layout:
+		 *	byte 0/1/2/3: uart setting
+		 */
+		status = f81534_read_flash(serial,
+					F81534_DEF_CONF_ADDRESS_START,
+					F81534_DEF_CONF_SIZE,
+					serial_priv->conf_data);
+		if (status) {
+			dev_err(&serial->interface->dev,
+					"%s: read reserve data failed: %d\n",
+					__func__, status);
+			return status;
+		}
+	} else {
+		/* Only read 8 bytes for mode & GPIO */
+		status = f81534_read_flash(serial,
+						F81534_CUSTOM_ADDRESS_START +
+						F81534_CONF_OFFSET,
+						sizeof(serial_priv->conf_data),
+						serial_priv->conf_data);
+		if (status) {
+			dev_err(&serial->interface->dev,
+					"%s: idx: %d get data failed: %d\n",
+					__func__, serial_priv->setting_idx,
+					status);
+			return status;
+		}
+	}
+
+	/* Assign phy-to-logic mapping */
+	for (i = 0; i < F81534_NUM_PORT; ++i) {
+		if (serial_priv->conf_data[i] & F81534_PORT_UNAVAILABLE)
+			continue;
+
+		serial_priv->tty_idx[i] = index++;
+		dev_dbg(&serial->interface->dev,
+				"%s: phy_num: %d, tty_idx: %d\n", __func__, i,
+				serial_priv->tty_idx[i]);
+	}
+
+	return 0;
+}
+
+static int f81534_port_probe(struct usb_serial_port *port)
+{
+	struct f81534_port_private *port_priv;
+
+	port_priv = devm_kzalloc(&port->dev, sizeof(*port_priv), GFP_KERNEL);
+	if (!port_priv)
+		return -ENOMEM;
+
+	spin_lock_init(&port_priv->msr_lock);
+	mutex_init(&port_priv->mcr_mutex);
+
+	/* Assign logic-to-phy mapping */
+	port_priv->phy_num = f81534_logic_to_phy_port(port->serial, port);
+	if (port_priv->phy_num < 0 || port_priv->phy_num >= F81534_NUM_PORT)
+		return -ENODEV;
+
+	usb_set_serial_port_data(port, port_priv);
+	dev_dbg(&port->dev, "%s: port_number: %d, phy_num: %d\n", __func__,
+			port->port_number, port_priv->phy_num);
+
+	return 0;
+}
+
+static int f81534_tiocmget(struct tty_struct *tty)
+{
+	struct usb_serial_port *port = tty->driver_data;
+	struct f81534_port_private *port_priv = usb_get_serial_port_data(port);
+	int status;
+	int r;
+	u8 msr;
+	u8 mcr;
+
+	/* Read current MSR from device */
+	status = f81534_get_port_register(port, F81534_MODEM_STATUS_REG, &msr);
+	if (status)
+		return status;
+
+	mutex_lock(&port_priv->mcr_mutex);
+	mcr = port_priv->shadow_mcr;
+	mutex_unlock(&port_priv->mcr_mutex);
+
+	r = (mcr & UART_MCR_DTR ? TIOCM_DTR : 0) |
+	    (mcr & UART_MCR_RTS ? TIOCM_RTS : 0) |
+	    (msr & UART_MSR_CTS ? TIOCM_CTS : 0) |
+	    (msr & UART_MSR_DCD ? TIOCM_CAR : 0) |
+	    (msr & UART_MSR_RI ? TIOCM_RI : 0) |
+	    (msr & UART_MSR_DSR ? TIOCM_DSR : 0);
+
+	return r;
+}
+
+static int f81534_tiocmset(struct tty_struct *tty, unsigned int set,
+				unsigned int clear)
+{
+	struct usb_serial_port *port = tty->driver_data;
+
+	return f81534_update_mctrl(port, set, clear);
+}
+
+static void f81534_dtr_rts(struct usb_serial_port *port, int on)
+{
+	if (on)
+		f81534_update_mctrl(port, TIOCM_DTR | TIOCM_RTS, 0);
+	else
+		f81534_update_mctrl(port, 0, TIOCM_DTR | TIOCM_RTS);
+}
+
+static int f81534_write(struct tty_struct *tty, struct usb_serial_port *port,
+			const u8 *buf, int count)
+{
+	int bytes_out, status;
+
+	if (!count)
+		return 0;
+
+	bytes_out = kfifo_in_locked(&port->write_fifo, buf, count,
+					&port->lock);
+
+	status = f81534_submit_writer(port, GFP_ATOMIC);
+	if (status) {
+		dev_err(&port->dev, "%s: submit failed\n", __func__);
+		return status;
+	}
+
+	return bytes_out;
+}
+
+static bool f81534_tx_empty(struct usb_serial_port *port)
+{
+	struct f81534_port_private *port_priv = usb_get_serial_port_data(port);
+
+	return test_bit(F81534_TX_EMPTY_BIT, &port_priv->tx_empty);
+}
+
+static int f81534_resume(struct usb_serial *serial)
+{
+	struct f81534_serial_private *serial_priv =
+			usb_get_serial_data(serial);
+	struct usb_serial_port *port;
+	int error = 0;
+	int status;
+	size_t i;
+
+	/*
+	 * We'll register port 0 bulkin when port had opened, It'll take all
+	 * port received data, MSR register change and TX_EMPTY information.
+	 */
+	mutex_lock(&serial_priv->urb_mutex);
+
+	if (serial_priv->opened_port) {
+		status = f81534_submit_read_urb(serial, GFP_NOIO);
+		if (status) {
+			mutex_unlock(&serial_priv->urb_mutex);
+			return status;
+		}
+	}
+
+	mutex_unlock(&serial_priv->urb_mutex);
+
+	for (i = 0; i < serial->num_ports; i++) {
+		port = serial->port[i];
+		if (!tty_port_initialized(&port->port))
+			continue;
+
+		status = f81534_submit_writer(port, GFP_NOIO);
+		if (status) {
+			dev_err(&port->dev, "%s: submit failed\n", __func__);
+			++error;
+		}
+	}
+
+	if (error)
+		return -EIO;
+
+	return 0;
+}
+
+static struct usb_serial_driver f81534_device = {
+	.driver = {
+		   .owner = THIS_MODULE,
+		   .name = "f81534",
+	},
+	.description =		DRIVER_DESC,
+	.id_table =		f81534_id_table,
+	.open =			f81534_open,
+	.close =		f81534_close,
+	.write =		f81534_write,
+	.tx_empty =		f81534_tx_empty,
+	.calc_num_ports =	f81534_calc_num_ports,
+	.probe =		f81534_probe,
+	.attach =		f81534_attach,
+	.port_probe =		f81534_port_probe,
+	.dtr_rts =		f81534_dtr_rts,
+	.process_read_urb =	f81534_process_read_urb,
+	.ioctl =		f81534_ioctl,
+	.tiocmget =		f81534_tiocmget,
+	.tiocmset =		f81534_tiocmset,
+	.write_bulk_callback =	f81534_write_usb_callback,
+	.set_termios =		f81534_set_termios,
+	.resume =		f81534_resume,
+};
+
+static struct usb_serial_driver *const serial_drivers[] = {
+	&f81534_device, NULL
+};
+
+module_usb_serial_driver(serial_drivers, f81534_id_table);
+
+MODULE_DEVICE_TABLE(usb, f81534_id_table);
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_AUTHOR("Peter Hong <Peter_Hong@fintek.com.tw>");
+MODULE_AUTHOR("Tom Tsai <Tom_Tsai@fintek.com.tw>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c
index 6e9fc8bcc285..23d14b98ae2a 100644
--- a/drivers/usb/serial/ftdi_sio.c
+++ b/drivers/usb/serial/ftdi_sio.c
@@ -1455,8 +1455,6 @@ static int get_serial_info(struct usb_serial_port *port,
 	struct ftdi_private *priv = usb_get_serial_port_data(port);
 	struct serial_struct tmp;
 
-	if (!retinfo)
-		return -EFAULT;
 	memset(&tmp, 0, sizeof(tmp));
 	tmp.flags = priv->flags;
 	tmp.baud_base = priv->baud_base;
@@ -1538,9 +1536,6 @@ static int get_lsr_info(struct usb_serial_port *port,
 	struct ftdi_private *priv = usb_get_serial_port_data(port);
 	unsigned int result = 0;
 
-	if (!retinfo)
-		return -EFAULT;
-
 	if (priv->transmit_empty)
 		result = TIOCSER_TEMT;
 
diff --git a/drivers/usb/serial/io_edgeport.c b/drivers/usb/serial/io_edgeport.c
index 11c05ce2f35f..dcc0c58aaad5 100644
--- a/drivers/usb/serial/io_edgeport.c
+++ b/drivers/usb/serial/io_edgeport.c
@@ -1554,9 +1554,6 @@ static int get_serial_info(struct edgeport_port *edge_port,
 {
 	struct serial_struct tmp;
 
-	if (!retinfo)
-		return -EFAULT;
-
 	memset(&tmp, 0, sizeof(tmp));
 
 	tmp.type		= PORT_16550A;
diff --git a/drivers/usb/serial/io_ti.c b/drivers/usb/serial/io_ti.c
index fce82fd79f77..c339163698eb 100644
--- a/drivers/usb/serial/io_ti.c
+++ b/drivers/usb/serial/io_ti.c
@@ -2459,9 +2459,6 @@ static int get_serial_info(struct edgeport_port *edge_port,
 	struct serial_struct tmp;
 	unsigned cwait;
 
-	if (!retinfo)
-		return -EFAULT;
-
 	cwait = edge_port->port->port.closing_wait;
 	if (cwait != ASYNC_CLOSING_WAIT_NONE)
 		cwait = jiffies_to_msecs(cwait) / 10;
diff --git a/drivers/usb/serial/kl5kusb105.c b/drivers/usb/serial/kl5kusb105.c
index fc5d3a791e08..0ee190fc1bf8 100644
--- a/drivers/usb/serial/kl5kusb105.c
+++ b/drivers/usb/serial/kl5kusb105.c
@@ -296,7 +296,7 @@ static int  klsi_105_open(struct tty_struct *tty, struct usb_serial_port *port)
 	rc = usb_serial_generic_open(tty, port);
 	if (rc) {
 		retval = rc;
-		goto exit;
+		goto err_free_cfg;
 	}
 
 	rc = usb_control_msg(port->serial->dev,
@@ -311,21 +311,38 @@ static int  klsi_105_open(struct tty_struct *tty, struct usb_serial_port *port)
 	if (rc < 0) {
 		dev_err(&port->dev, "Enabling read failed (error = %d)\n", rc);
 		retval = rc;
+		goto err_generic_close;
 	} else
 		dev_dbg(&port->dev, "%s - enabled reading\n", __func__);
 
 	rc = klsi_105_get_line_state(port, &line_state);
-	if (rc >= 0) {
-		spin_lock_irqsave(&priv->lock, flags);
-		priv->line_state = line_state;
-		spin_unlock_irqrestore(&priv->lock, flags);
-		dev_dbg(&port->dev, "%s - read line state 0x%lx\n", __func__, line_state);
-		retval = 0;
-	} else
+	if (rc < 0) {
 		retval = rc;
+		goto err_disable_read;
+	}
+
+	spin_lock_irqsave(&priv->lock, flags);
+	priv->line_state = line_state;
+	spin_unlock_irqrestore(&priv->lock, flags);
+	dev_dbg(&port->dev, "%s - read line state 0x%lx\n", __func__,
+			line_state);
+
+	return 0;
 
-exit:
+err_disable_read:
+	usb_control_msg(port->serial->dev,
+			     usb_sndctrlpipe(port->serial->dev, 0),
+			     KL5KUSB105A_SIO_CONFIGURE,
+			     USB_TYPE_VENDOR | USB_DIR_OUT,
+			     KL5KUSB105A_SIO_CONFIGURE_READ_OFF,
+			     0, /* index */
+			     NULL, 0,
+			     KLSI_TIMEOUT);
+err_generic_close:
+	usb_serial_generic_close(port);
+err_free_cfg:
 	kfree(cfg);
+
 	return retval;
 }
 
diff --git a/drivers/usb/serial/mos7720.c b/drivers/usb/serial/mos7720.c
index de9992b492b0..d52caa03679c 100644
--- a/drivers/usb/serial/mos7720.c
+++ b/drivers/usb/serial/mos7720.c
@@ -1861,9 +1861,6 @@ static int get_serial_info(struct moschip_port *mos7720_port,
 {
 	struct serial_struct tmp;
 
-	if (!retinfo)
-		return -EFAULT;
-
 	memset(&tmp, 0, sizeof(tmp));
 
 	tmp.type		= PORT_16550A;
diff --git a/drivers/usb/serial/mos7840.c b/drivers/usb/serial/mos7840.c
index 57426d703a09..9a220b8e810f 100644
--- a/drivers/usb/serial/mos7840.c
+++ b/drivers/usb/serial/mos7840.c
@@ -1956,9 +1956,6 @@ static int mos7840_get_serial_info(struct moschip_port *mos7840_port,
 	if (mos7840_port == NULL)
 		return -1;
 
-	if (!retinfo)
-		return -EFAULT;
-
 	memset(&tmp, 0, sizeof(tmp));
 
 	tmp.type = PORT_16550A;
diff --git a/drivers/usb/serial/opticon.c b/drivers/usb/serial/opticon.c
index 4b7bfb394a32..5ded6f524d59 100644
--- a/drivers/usb/serial/opticon.c
+++ b/drivers/usb/serial/opticon.c
@@ -336,9 +336,6 @@ static int get_serial_info(struct usb_serial_port *port,
 {
 	struct serial_struct tmp;
 
-	if (!serial)
-		return -EFAULT;
-
 	memset(&tmp, 0x00, sizeof(tmp));
 
 	/* fake emulate a 16550 uart to make userspace code happy */
diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
index 9894e341c6ac..7ce31a4c7e7f 100644
--- a/drivers/usb/serial/option.c
+++ b/drivers/usb/serial/option.c
@@ -268,6 +268,8 @@ static void option_instat_callback(struct urb *urb);
 #define TELIT_PRODUCT_CC864_SINGLE		0x1006
 #define TELIT_PRODUCT_DE910_DUAL		0x1010
 #define TELIT_PRODUCT_UE910_V2			0x1012
+#define TELIT_PRODUCT_LE922_USBCFG1		0x1040
+#define TELIT_PRODUCT_LE922_USBCFG2		0x1041
 #define TELIT_PRODUCT_LE922_USBCFG0		0x1042
 #define TELIT_PRODUCT_LE922_USBCFG3		0x1043
 #define TELIT_PRODUCT_LE922_USBCFG5		0x1045
@@ -1210,6 +1212,10 @@ static const struct usb_device_id option_ids[] = {
 	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_UE910_V2) },
 	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG0),
 		.driver_info = (kernel_ulong_t)&telit_le922_blacklist_usbcfg0 },
+	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG1),
+		.driver_info = (kernel_ulong_t)&telit_le910_blacklist },
+	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG2),
+		.driver_info = (kernel_ulong_t)&telit_le922_blacklist_usbcfg3 },
 	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG3),
 		.driver_info = (kernel_ulong_t)&telit_le922_blacklist_usbcfg3 },
 	{ USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG5, 0xff),
@@ -1989,6 +1995,7 @@ static const struct usb_device_id option_ids[] = {
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x7d02, 0xff, 0x00, 0x00) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x7d03, 0xff, 0x02, 0x01) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x7d03, 0xff, 0x00, 0x00) },
+	{ USB_DEVICE_INTERFACE_CLASS(0x2001, 0x7d04, 0xff) },			/* D-Link DWM-158 */
 	{ USB_DEVICE_INTERFACE_CLASS(0x2001, 0x7e19, 0xff),			/* D-Link DWM-221 B1 */
 	  .driver_info = (kernel_ulong_t)&net_intf4_blacklist },
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x07d1, 0x3e01, 0xff, 0xff, 0xff) }, /* D-Link DWM-152/C1 */
diff --git a/drivers/usb/serial/quatech2.c b/drivers/usb/serial/quatech2.c
index 85acb50a7ee2..659cb8606bd9 100644
--- a/drivers/usb/serial/quatech2.c
+++ b/drivers/usb/serial/quatech2.c
@@ -463,9 +463,6 @@ static int get_serial_info(struct usb_serial_port *port,
 {
 	struct serial_struct tmp;
 
-	if (!retinfo)
-		return -EFAULT;
-
 	memset(&tmp, 0, sizeof(tmp));
 	tmp.line		= port->minor;
 	tmp.port		= 0;
diff --git a/drivers/usb/serial/ssu100.c b/drivers/usb/serial/ssu100.c
index 70a098de429f..2a156144c76c 100644
--- a/drivers/usb/serial/ssu100.c
+++ b/drivers/usb/serial/ssu100.c
@@ -318,9 +318,6 @@ static int get_serial_info(struct usb_serial_port *port,
 {
 	struct serial_struct tmp;
 
-	if (!retinfo)
-		return -EFAULT;
-
 	memset(&tmp, 0, sizeof(tmp));
 	tmp.line		= port->minor;
 	tmp.port		= 0;
diff --git a/drivers/usb/serial/ti_usb_3410_5052.c b/drivers/usb/serial/ti_usb_3410_5052.c
index a8b9bdba314f..8db9d071d940 100644
--- a/drivers/usb/serial/ti_usb_3410_5052.c
+++ b/drivers/usb/serial/ti_usb_3410_5052.c
@@ -1426,9 +1426,6 @@ static int ti_get_serial_info(struct ti_port *tport,
 	struct serial_struct ret_serial;
 	unsigned cwait;
 
-	if (!ret_arg)
-		return -EFAULT;
-
 	cwait = port->port.closing_wait;
 	if (cwait != ASYNC_CLOSING_WAIT_NONE)
 		cwait = jiffies_to_msecs(cwait) / 10;
diff --git a/drivers/usb/serial/usb_wwan.c b/drivers/usb/serial/usb_wwan.c
index 3dfdfc81254b..59bfcb3da116 100644
--- a/drivers/usb/serial/usb_wwan.c
+++ b/drivers/usb/serial/usb_wwan.c
@@ -140,9 +140,6 @@ static int get_serial_info(struct usb_serial_port *port,
 {
 	struct serial_struct tmp;
 
-	if (!retinfo)
-		return -EFAULT;
-
 	memset(&tmp, 0, sizeof(tmp));
 	tmp.line            = port->minor;
 	tmp.port            = port->port_number;
diff --git a/drivers/usb/storage/usb.c b/drivers/usb/storage/usb.c
index 2cba13a532cd..615bea08ec0a 100644
--- a/drivers/usb/storage/usb.c
+++ b/drivers/usb/storage/usb.c
@@ -52,7 +52,6 @@
 
 #include <linux/sched.h>
 #include <linux/errno.h>
-#include <linux/freezer.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/kthread.h>
diff --git a/drivers/usb/usbip/vhci_hcd.c b/drivers/usb/usbip/vhci_hcd.c
index 03eccf29ace0..c4724fb3a691 100644
--- a/drivers/usb/usbip/vhci_hcd.c
+++ b/drivers/usb/usbip/vhci_hcd.c
@@ -460,13 +460,14 @@ static void vhci_tx_urb(struct urb *urb)
 {
 	struct vhci_device *vdev = get_vdev(urb->dev);
 	struct vhci_priv *priv;
-	struct vhci_hcd *vhci = vdev_to_vhci(vdev);
+	struct vhci_hcd *vhci;
 	unsigned long flags;
 
 	if (!vdev) {
 		pr_err("could not get virtual device");
 		return;
 	}
+	vhci = vdev_to_vhci(vdev);
 
 	priv = kzalloc(sizeof(struct vhci_priv), GFP_ATOMIC);
 	if (!priv) {
diff --git a/drivers/usb/usbip/vhci_sysfs.c b/drivers/usb/usbip/vhci_sysfs.c
index c404017c1b5a..b96e5b189269 100644
--- a/drivers/usb/usbip/vhci_sysfs.c
+++ b/drivers/usb/usbip/vhci_sysfs.c
@@ -361,6 +361,7 @@ static void set_status_attr(int id)
 	status->attr.attr.name = status->name;
 	status->attr.attr.mode = S_IRUGO;
 	status->attr.show = status_show;
+	sysfs_attr_init(&status->attr.attr);
 }
 
 static int init_status_attrs(void)
diff --git a/drivers/usb/usbip/vudc_dev.c b/drivers/usb/usbip/vudc_dev.c
index 7091848df6c8..968471b62cbc 100644
--- a/drivers/usb/usbip/vudc_dev.c
+++ b/drivers/usb/usbip/vudc_dev.c
@@ -242,10 +242,10 @@ static const struct usb_gadget_ops vgadget_ops = {
 static int vep_enable(struct usb_ep *_ep,
 		const struct usb_endpoint_descriptor *desc)
 {
-	struct vep *ep;
-	struct vudc *udc;
-	unsigned maxp;
-	unsigned long flags;
+	struct vep	*ep;
+	struct vudc	*udc;
+	unsigned int	maxp;
+	unsigned long	flags;
 
 	ep = to_vep(_ep);
 	udc = ep_to_vudc(ep);
@@ -259,7 +259,7 @@ static int vep_enable(struct usb_ep *_ep,
 
 	spin_lock_irqsave(&udc->lock, flags);
 
-	maxp = usb_endpoint_maxp(desc) & 0x7ff;
+	maxp = usb_endpoint_maxp(desc);
 	_ep->maxpacket = maxp;
 	ep->desc = desc;
 	ep->type = usb_endpoint_type(desc);
@@ -549,30 +549,34 @@ static int init_vudc_hw(struct vudc *udc)
 		sprintf(ep->name, "ep%d%s", num,
 			i ? (is_out ? "out" : "in") : "");
 		ep->ep.name = ep->name;
+
+		ep->ep.ops = &vep_ops;
+
+		usb_ep_set_maxpacket_limit(&ep->ep, ~0);
+		ep->ep.max_streams = 16;
+		ep->gadget = &udc->gadget;
+		INIT_LIST_HEAD(&ep->req_queue);
+
 		if (i == 0) {
+			/* ep0 */
 			ep->ep.caps.type_control = true;
 			ep->ep.caps.dir_out = true;
 			ep->ep.caps.dir_in = true;
+
+			udc->gadget.ep0 = &ep->ep;
 		} else {
+			/* All other eps */
 			ep->ep.caps.type_iso = true;
 			ep->ep.caps.type_int = true;
 			ep->ep.caps.type_bulk = true;
-		}
 
-		if (is_out)
-			ep->ep.caps.dir_out = true;
-		else
-			ep->ep.caps.dir_in = true;
+			if (is_out)
+				ep->ep.caps.dir_out = true;
+			else
+				ep->ep.caps.dir_in = true;
 
-		ep->ep.ops = &vep_ops;
-		list_add_tail(&ep->ep.ep_list, &udc->gadget.ep_list);
-		ep->halted = ep->wedged = ep->already_seen =
-			ep->setup_stage = 0;
-		usb_ep_set_maxpacket_limit(&ep->ep, ~0);
-		ep->ep.max_streams = 16;
-		ep->gadget = &udc->gadget;
-		ep->desc = NULL;
-		INIT_LIST_HEAD(&ep->req_queue);
+			list_add_tail(&ep->ep.ep_list, &udc->gadget.ep_list);
+		}
 	}
 
 	spin_lock_init(&udc->lock);
@@ -589,9 +593,6 @@ static int init_vudc_hw(struct vudc *udc)
 	ud->eh_ops.reset    = vudc_device_reset;
 	ud->eh_ops.unusable = vudc_device_unusable;
 
-	udc->gadget.ep0 = &udc->ep[0].ep;
-	list_del_init(&udc->ep[0].ep.ep_list);
-
 	v_init_timer(udc);
 	return 0;
 
diff --git a/drivers/usb/usbip/vudc_transfer.c b/drivers/usb/usbip/vudc_transfer.c
index aba6bd478045..4cfd475ee865 100644
--- a/drivers/usb/usbip/vudc_transfer.c
+++ b/drivers/usb/usbip/vudc_transfer.c
@@ -73,8 +73,8 @@ static int handle_control_request(struct vudc *udc, struct urb *urb,
 {
 	struct vep	*ep2;
 	int		ret_val = 1;
-	unsigned	w_index;
-	unsigned	w_value;
+	unsigned int	w_index;
+	unsigned int	w_value;
 
 	w_index = le16_to_cpu(setup->wIndex);
 	w_value = le16_to_cpu(setup->wValue);
@@ -200,7 +200,7 @@ static int transfer(struct vudc *udc,
 top:
 	/* if there's no request queued, the device is NAKing; return */
 	list_for_each_entry(req, &ep->req_queue, req_entry) {
-		unsigned	host_len, dev_len, len;
+		unsigned int	host_len, dev_len, len;
 		void		*ubuf_pos, *rbuf_pos;
 		int		is_short, to_host;
 		int		rescan = 0;
@@ -339,6 +339,8 @@ static void v_timer(unsigned long _vudc)
 		total = timer->frame_limit;
 	}
 
+	/* We have to clear ep0 flags separately as it's not on the list */
+	udc->ep[0].already_seen = 0;
 	list_for_each_entry(_ep, &udc->gadget.ep_list, ep_list) {
 		ep = to_vep(_ep);
 		ep->already_seen = 0;
diff --git a/drivers/usb/wusbcore/dev-sysfs.c b/drivers/usb/wusbcore/dev-sysfs.c
index 415b14002a61..d4de56b93d68 100644
--- a/drivers/usb/wusbcore/dev-sysfs.c
+++ b/drivers/usb/wusbcore/dev-sysfs.c
@@ -53,7 +53,7 @@ static ssize_t wusb_disconnect_store(struct device *dev,
 	wusbhc_put(wusbhc);
 	return size;
 }
-static DEVICE_ATTR(wusb_disconnect, 0200, NULL, wusb_disconnect_store);
+static DEVICE_ATTR_WO(wusb_disconnect);
 
 static ssize_t wusb_cdid_show(struct device *dev,
 			      struct device_attribute *attr, char *buf)
@@ -69,7 +69,7 @@ static ssize_t wusb_cdid_show(struct device *dev,
 	wusb_dev_put(wusb_dev);
 	return result + 1;
 }
-static DEVICE_ATTR(wusb_cdid, 0444, wusb_cdid_show, NULL);
+static DEVICE_ATTR_RO(wusb_cdid);
 
 static ssize_t wusb_ck_store(struct device *dev,
 			     struct device_attribute *attr,
@@ -105,7 +105,7 @@ static ssize_t wusb_ck_store(struct device *dev,
 	wusbhc_put(wusbhc);
 	return result < 0 ? result : size;
 }
-static DEVICE_ATTR(wusb_ck, 0200, NULL, wusb_ck_store);
+static DEVICE_ATTR_WO(wusb_ck);
 
 static struct attribute *wusb_dev_attrs[] = {
 		&dev_attr_wusb_disconnect.attr,
diff --git a/drivers/usb/wusbcore/security.c b/drivers/usb/wusbcore/security.c
index 8c9421b69da0..170f2c38de9b 100644
--- a/drivers/usb/wusbcore/security.c
+++ b/drivers/usb/wusbcore/security.c
@@ -240,6 +240,7 @@ int wusb_dev_sec_add(struct wusbhc *wusbhc,
 	if (new_secd == NULL) {
 		dev_err(dev,
 			"Can't allocate space for security descriptors\n");
+		result = -ENOMEM;
 		goto out;
 	}
 	secd = new_secd;
diff --git a/drivers/usb/wusbcore/wa-nep.c b/drivers/usb/wusbcore/wa-nep.c
index ed4622279c63..e3819fc182b0 100644
--- a/drivers/usb/wusbcore/wa-nep.c
+++ b/drivers/usb/wusbcore/wa-nep.c
@@ -198,6 +198,7 @@ static int wa_nep_queue(struct wahc *wa, size_t size)
 	if (nw == NULL) {
 		if (printk_ratelimit())
 			dev_err(dev, "No memory to queue notification\n");
+		result = -ENOMEM;
 		goto out;
 	}
 	INIT_WORK(&nw->work, wa_notif_dispatch);
diff --git a/drivers/usb/wusbcore/wa-xfer.c b/drivers/usb/wusbcore/wa-xfer.c
index 167fcc71f5f6..e70322b1dd02 100644
--- a/drivers/usb/wusbcore/wa-xfer.c
+++ b/drivers/usb/wusbcore/wa-xfer.c
@@ -1203,6 +1203,7 @@ static int __wa_xfer_setup_segs(struct wa_xfer *xfer, size_t xfer_hdr_size)
 				sizeof(struct wa_xfer_packet_info_hwaiso) +
 				(seg_isoc_frame_count * sizeof(__le16));
 		}
+		result = -ENOMEM;
 		seg = xfer->seg[cnt] = kmalloc(alloc_size + iso_pkt_descr_size,
 						GFP_ATOMIC);
 		if (seg == NULL)
diff --git a/drivers/usb/wusbcore/wusbhc.c b/drivers/usb/wusbcore/wusbhc.c
index 94f401ab859f..a273a91cf667 100644
--- a/drivers/usb/wusbcore/wusbhc.c
+++ b/drivers/usb/wusbcore/wusbhc.c
@@ -84,8 +84,7 @@ static ssize_t wusb_trust_timeout_store(struct device *dev,
 out:
 	return result < 0 ? result : size;
 }
-static DEVICE_ATTR(wusb_trust_timeout, 0644, wusb_trust_timeout_show,
-					     wusb_trust_timeout_store);
+static DEVICE_ATTR_RW(wusb_trust_timeout);
 
 /*
  * Show the current WUSB CHID.
@@ -145,7 +144,7 @@ static ssize_t wusb_chid_store(struct device *dev,
 	result = wusbhc_chid_set(wusbhc, &chid);
 	return result < 0 ? result : size;
 }
-static DEVICE_ATTR(wusb_chid, 0644, wusb_chid_show, wusb_chid_store);
+static DEVICE_ATTR_RW(wusb_chid);
 
 
 static ssize_t wusb_phy_rate_show(struct device *dev,
@@ -174,8 +173,7 @@ static ssize_t wusb_phy_rate_store(struct device *dev,
 	wusbhc->phy_rate = phy_rate;
 	return size;
 }
-static DEVICE_ATTR(wusb_phy_rate, 0644, wusb_phy_rate_show,
-			wusb_phy_rate_store);
+static DEVICE_ATTR_RW(wusb_phy_rate);
 
 static ssize_t wusb_dnts_show(struct device *dev,
 				  struct device_attribute *attr,
@@ -205,7 +203,7 @@ static ssize_t wusb_dnts_store(struct device *dev,
 
 	return size;
 }
-static DEVICE_ATTR(wusb_dnts, 0644, wusb_dnts_show, wusb_dnts_store);
+static DEVICE_ATTR_RW(wusb_dnts);
 
 static ssize_t wusb_retry_count_show(struct device *dev,
 				  struct device_attribute *attr,
@@ -234,8 +232,7 @@ static ssize_t wusb_retry_count_store(struct device *dev,
 
 	return size;
 }
-static DEVICE_ATTR(wusb_retry_count, 0644, wusb_retry_count_show,
-	wusb_retry_count_store);
+static DEVICE_ATTR_RW(wusb_retry_count);
 
 /* Group all the WUSBHC attributes */
 static struct attribute *wusbhc_attrs[] = {
diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig
index da6e2ce77495..23eced02aaf6 100644
--- a/drivers/vfio/Kconfig
+++ b/drivers/vfio/Kconfig
@@ -48,4 +48,5 @@ menuconfig VFIO_NOIOMMU
 
 source "drivers/vfio/pci/Kconfig"
 source "drivers/vfio/platform/Kconfig"
+source "drivers/vfio/mdev/Kconfig"
 source "virt/lib/Kconfig"
diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile
index 7b8a31f63fea..4a23c13b6be4 100644
--- a/drivers/vfio/Makefile
+++ b/drivers/vfio/Makefile
@@ -7,3 +7,4 @@ obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
 obj-$(CONFIG_VFIO_SPAPR_EEH) += vfio_spapr_eeh.o
 obj-$(CONFIG_VFIO_PCI) += pci/
 obj-$(CONFIG_VFIO_PLATFORM) += platform/
+obj-$(CONFIG_VFIO_MDEV) += mdev/
diff --git a/drivers/vfio/mdev/Kconfig b/drivers/vfio/mdev/Kconfig
new file mode 100644
index 000000000000..14fdb106a827
--- /dev/null
+++ b/drivers/vfio/mdev/Kconfig
@@ -0,0 +1,17 @@
+
+config VFIO_MDEV
+	tristate "Mediated device driver framework"
+	depends on VFIO
+	default n
+	help
+	  Provides a framework to virtualize devices.
+	  See Documentation/vfio-mediated-device.txt for more details.
+
+	  If you don't know what do here, say N.
+
+config VFIO_MDEV_DEVICE
+	tristate "VFIO driver for Mediated devices"
+	depends on VFIO && VFIO_MDEV
+	default n
+	help
+	  VFIO based driver for Mediated devices.
diff --git a/drivers/vfio/mdev/Makefile b/drivers/vfio/mdev/Makefile
new file mode 100644
index 000000000000..fa2d5ea466ee
--- /dev/null
+++ b/drivers/vfio/mdev/Makefile
@@ -0,0 +1,5 @@
+
+mdev-y := mdev_core.o mdev_sysfs.o mdev_driver.o
+
+obj-$(CONFIG_VFIO_MDEV) += mdev.o
+obj-$(CONFIG_VFIO_MDEV_DEVICE) += vfio_mdev.o
diff --git a/drivers/vfio/mdev/mdev_core.c b/drivers/vfio/mdev/mdev_core.c
new file mode 100644
index 000000000000..be1ee89ee917
--- /dev/null
+++ b/drivers/vfio/mdev/mdev_core.c
@@ -0,0 +1,385 @@
+/*
+ * Mediated device Core Driver
+ *
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ *     Author: Neo Jia <cjia@nvidia.com>
+ *             Kirti Wankhede <kwankhede@nvidia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/uuid.h>
+#include <linux/sysfs.h>
+#include <linux/mdev.h>
+
+#include "mdev_private.h"
+
+#define DRIVER_VERSION		"0.1"
+#define DRIVER_AUTHOR		"NVIDIA Corporation"
+#define DRIVER_DESC		"Mediated device Core Driver"
+
+static LIST_HEAD(parent_list);
+static DEFINE_MUTEX(parent_list_lock);
+static struct class_compat *mdev_bus_compat_class;
+
+static int _find_mdev_device(struct device *dev, void *data)
+{
+	struct mdev_device *mdev;
+
+	if (!dev_is_mdev(dev))
+		return 0;
+
+	mdev = to_mdev_device(dev);
+
+	if (uuid_le_cmp(mdev->uuid, *(uuid_le *)data) == 0)
+		return 1;
+
+	return 0;
+}
+
+static bool mdev_device_exist(struct parent_device *parent, uuid_le uuid)
+{
+	struct device *dev;
+
+	dev = device_find_child(parent->dev, &uuid, _find_mdev_device);
+	if (dev) {
+		put_device(dev);
+		return true;
+	}
+
+	return false;
+}
+
+/* Should be called holding parent_list_lock */
+static struct parent_device *__find_parent_device(struct device *dev)
+{
+	struct parent_device *parent;
+
+	list_for_each_entry(parent, &parent_list, next) {
+		if (parent->dev == dev)
+			return parent;
+	}
+	return NULL;
+}
+
+static void mdev_release_parent(struct kref *kref)
+{
+	struct parent_device *parent = container_of(kref, struct parent_device,
+						    ref);
+	struct device *dev = parent->dev;
+
+	kfree(parent);
+	put_device(dev);
+}
+
+static
+inline struct parent_device *mdev_get_parent(struct parent_device *parent)
+{
+	if (parent)
+		kref_get(&parent->ref);
+
+	return parent;
+}
+
+static inline void mdev_put_parent(struct parent_device *parent)
+{
+	if (parent)
+		kref_put(&parent->ref, mdev_release_parent);
+}
+
+static int mdev_device_create_ops(struct kobject *kobj,
+				  struct mdev_device *mdev)
+{
+	struct parent_device *parent = mdev->parent;
+	int ret;
+
+	ret = parent->ops->create(kobj, mdev);
+	if (ret)
+		return ret;
+
+	ret = sysfs_create_groups(&mdev->dev.kobj,
+				  parent->ops->mdev_attr_groups);
+	if (ret)
+		parent->ops->remove(mdev);
+
+	return ret;
+}
+
+/*
+ * mdev_device_remove_ops gets called from sysfs's 'remove' and when parent
+ * device is being unregistered from mdev device framework.
+ * - 'force_remove' is set to 'false' when called from sysfs's 'remove' which
+ *   indicates that if the mdev device is active, used by VMM or userspace
+ *   application, vendor driver could return error then don't remove the device.
+ * - 'force_remove' is set to 'true' when called from mdev_unregister_device()
+ *   which indicate that parent device is being removed from mdev device
+ *   framework so remove mdev device forcefully.
+ */
+static int mdev_device_remove_ops(struct mdev_device *mdev, bool force_remove)
+{
+	struct parent_device *parent = mdev->parent;
+	int ret;
+
+	/*
+	 * Vendor driver can return error if VMM or userspace application is
+	 * using this mdev device.
+	 */
+	ret = parent->ops->remove(mdev);
+	if (ret && !force_remove)
+		return -EBUSY;
+
+	sysfs_remove_groups(&mdev->dev.kobj, parent->ops->mdev_attr_groups);
+	return 0;
+}
+
+static int mdev_device_remove_cb(struct device *dev, void *data)
+{
+	if (!dev_is_mdev(dev))
+		return 0;
+
+	return mdev_device_remove(dev, data ? *(bool *)data : true);
+}
+
+/*
+ * mdev_register_device : Register a device
+ * @dev: device structure representing parent device.
+ * @ops: Parent device operation structure to be registered.
+ *
+ * Add device to list of registered parent devices.
+ * Returns a negative value on error, otherwise 0.
+ */
+int mdev_register_device(struct device *dev, const struct parent_ops *ops)
+{
+	int ret;
+	struct parent_device *parent;
+
+	/* check for mandatory ops */
+	if (!ops || !ops->create || !ops->remove || !ops->supported_type_groups)
+		return -EINVAL;
+
+	dev = get_device(dev);
+	if (!dev)
+		return -EINVAL;
+
+	mutex_lock(&parent_list_lock);
+
+	/* Check for duplicate */
+	parent = __find_parent_device(dev);
+	if (parent) {
+		ret = -EEXIST;
+		goto add_dev_err;
+	}
+
+	parent = kzalloc(sizeof(*parent), GFP_KERNEL);
+	if (!parent) {
+		ret = -ENOMEM;
+		goto add_dev_err;
+	}
+
+	kref_init(&parent->ref);
+	mutex_init(&parent->lock);
+
+	parent->dev = dev;
+	parent->ops = ops;
+
+	if (!mdev_bus_compat_class) {
+		mdev_bus_compat_class = class_compat_register("mdev_bus");
+		if (!mdev_bus_compat_class) {
+			ret = -ENOMEM;
+			goto add_dev_err;
+		}
+	}
+
+	ret = parent_create_sysfs_files(parent);
+	if (ret)
+		goto add_dev_err;
+
+	ret = class_compat_create_link(mdev_bus_compat_class, dev, NULL);
+	if (ret)
+		dev_warn(dev, "Failed to create compatibility class link\n");
+
+	list_add(&parent->next, &parent_list);
+	mutex_unlock(&parent_list_lock);
+
+	dev_info(dev, "MDEV: Registered\n");
+	return 0;
+
+add_dev_err:
+	mutex_unlock(&parent_list_lock);
+	if (parent)
+		mdev_put_parent(parent);
+	else
+		put_device(dev);
+	return ret;
+}
+EXPORT_SYMBOL(mdev_register_device);
+
+/*
+ * mdev_unregister_device : Unregister a parent device
+ * @dev: device structure representing parent device.
+ *
+ * Remove device from list of registered parent devices. Give a chance to free
+ * existing mediated devices for given device.
+ */
+
+void mdev_unregister_device(struct device *dev)
+{
+	struct parent_device *parent;
+	bool force_remove = true;
+
+	mutex_lock(&parent_list_lock);
+	parent = __find_parent_device(dev);
+
+	if (!parent) {
+		mutex_unlock(&parent_list_lock);
+		return;
+	}
+	dev_info(dev, "MDEV: Unregistering\n");
+
+	list_del(&parent->next);
+	class_compat_remove_link(mdev_bus_compat_class, dev, NULL);
+
+	device_for_each_child(dev, (void *)&force_remove,
+			      mdev_device_remove_cb);
+
+	parent_remove_sysfs_files(parent);
+
+	mutex_unlock(&parent_list_lock);
+	mdev_put_parent(parent);
+}
+EXPORT_SYMBOL(mdev_unregister_device);
+
+static void mdev_device_release(struct device *dev)
+{
+	struct mdev_device *mdev = to_mdev_device(dev);
+
+	dev_dbg(&mdev->dev, "MDEV: destroying\n");
+	kfree(mdev);
+}
+
+int mdev_device_create(struct kobject *kobj, struct device *dev, uuid_le uuid)
+{
+	int ret;
+	struct mdev_device *mdev;
+	struct parent_device *parent;
+	struct mdev_type *type = to_mdev_type(kobj);
+
+	parent = mdev_get_parent(type->parent);
+	if (!parent)
+		return -EINVAL;
+
+	mutex_lock(&parent->lock);
+
+	/* Check for duplicate */
+	if (mdev_device_exist(parent, uuid)) {
+		ret = -EEXIST;
+		goto create_err;
+	}
+
+	mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
+	if (!mdev) {
+		ret = -ENOMEM;
+		goto create_err;
+	}
+
+	memcpy(&mdev->uuid, &uuid, sizeof(uuid_le));
+	mdev->parent = parent;
+	kref_init(&mdev->ref);
+
+	mdev->dev.parent  = dev;
+	mdev->dev.bus     = &mdev_bus_type;
+	mdev->dev.release = mdev_device_release;
+	dev_set_name(&mdev->dev, "%pUl", uuid.b);
+
+	ret = device_register(&mdev->dev);
+	if (ret) {
+		put_device(&mdev->dev);
+		goto create_err;
+	}
+
+	ret = mdev_device_create_ops(kobj, mdev);
+	if (ret)
+		goto create_failed;
+
+	ret = mdev_create_sysfs_files(&mdev->dev, type);
+	if (ret) {
+		mdev_device_remove_ops(mdev, true);
+		goto create_failed;
+	}
+
+	mdev->type_kobj = kobj;
+	dev_dbg(&mdev->dev, "MDEV: created\n");
+
+	mutex_unlock(&parent->lock);
+	return ret;
+
+create_failed:
+	device_unregister(&mdev->dev);
+
+create_err:
+	mutex_unlock(&parent->lock);
+	mdev_put_parent(parent);
+	return ret;
+}
+
+int mdev_device_remove(struct device *dev, bool force_remove)
+{
+	struct mdev_device *mdev;
+	struct parent_device *parent;
+	struct mdev_type *type;
+	int ret;
+
+	mdev = to_mdev_device(dev);
+	type = to_mdev_type(mdev->type_kobj);
+	parent = mdev->parent;
+	mutex_lock(&parent->lock);
+
+	ret = mdev_device_remove_ops(mdev, force_remove);
+	if (ret) {
+		mutex_unlock(&parent->lock);
+		return ret;
+	}
+
+	mdev_remove_sysfs_files(dev, type);
+	device_unregister(dev);
+	mutex_unlock(&parent->lock);
+	mdev_put_parent(parent);
+	return ret;
+}
+
+static int __init mdev_init(void)
+{
+	int ret;
+
+	ret = mdev_bus_register();
+
+	/*
+	 * Attempt to load known vfio_mdev.  This gives us a working environment
+	 * without the user needing to explicitly load vfio_mdev driver.
+	 */
+	if (!ret)
+		request_module_nowait("vfio_mdev");
+
+	return ret;
+}
+
+static void __exit mdev_exit(void)
+{
+	if (mdev_bus_compat_class)
+		class_compat_unregister(mdev_bus_compat_class);
+
+	mdev_bus_unregister();
+}
+
+module_init(mdev_init)
+module_exit(mdev_exit)
+
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESC);
diff --git a/drivers/vfio/mdev/mdev_driver.c b/drivers/vfio/mdev/mdev_driver.c
new file mode 100644
index 000000000000..6f0391f6f9b6
--- /dev/null
+++ b/drivers/vfio/mdev/mdev_driver.c
@@ -0,0 +1,119 @@
+/*
+ * MDEV driver
+ *
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ *     Author: Neo Jia <cjia@nvidia.com>
+ *             Kirti Wankhede <kwankhede@nvidia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/device.h>
+#include <linux/iommu.h>
+#include <linux/mdev.h>
+
+#include "mdev_private.h"
+
+static int mdev_attach_iommu(struct mdev_device *mdev)
+{
+	int ret;
+	struct iommu_group *group;
+
+	group = iommu_group_alloc();
+	if (IS_ERR(group))
+		return PTR_ERR(group);
+
+	ret = iommu_group_add_device(group, &mdev->dev);
+	if (!ret)
+		dev_info(&mdev->dev, "MDEV: group_id = %d\n",
+			 iommu_group_id(group));
+
+	iommu_group_put(group);
+	return ret;
+}
+
+static void mdev_detach_iommu(struct mdev_device *mdev)
+{
+	iommu_group_remove_device(&mdev->dev);
+	dev_info(&mdev->dev, "MDEV: detaching iommu\n");
+}
+
+static int mdev_probe(struct device *dev)
+{
+	struct mdev_driver *drv = to_mdev_driver(dev->driver);
+	struct mdev_device *mdev = to_mdev_device(dev);
+	int ret;
+
+	ret = mdev_attach_iommu(mdev);
+	if (ret)
+		return ret;
+
+	if (drv && drv->probe) {
+		ret = drv->probe(dev);
+		if (ret)
+			mdev_detach_iommu(mdev);
+	}
+
+	return ret;
+}
+
+static int mdev_remove(struct device *dev)
+{
+	struct mdev_driver *drv = to_mdev_driver(dev->driver);
+	struct mdev_device *mdev = to_mdev_device(dev);
+
+	if (drv && drv->remove)
+		drv->remove(dev);
+
+	mdev_detach_iommu(mdev);
+
+	return 0;
+}
+
+struct bus_type mdev_bus_type = {
+	.name		= "mdev",
+	.probe		= mdev_probe,
+	.remove		= mdev_remove,
+};
+EXPORT_SYMBOL_GPL(mdev_bus_type);
+
+/**
+ * mdev_register_driver - register a new MDEV driver
+ * @drv: the driver to register
+ * @owner: module owner of driver to be registered
+ *
+ * Returns a negative value on error, otherwise 0.
+ **/
+int mdev_register_driver(struct mdev_driver *drv, struct module *owner)
+{
+	/* initialize common driver fields */
+	drv->driver.name = drv->name;
+	drv->driver.bus = &mdev_bus_type;
+	drv->driver.owner = owner;
+
+	/* register with core */
+	return driver_register(&drv->driver);
+}
+EXPORT_SYMBOL(mdev_register_driver);
+
+/*
+ * mdev_unregister_driver - unregister MDEV driver
+ * @drv: the driver to unregister
+ */
+void mdev_unregister_driver(struct mdev_driver *drv)
+{
+	driver_unregister(&drv->driver);
+}
+EXPORT_SYMBOL(mdev_unregister_driver);
+
+int mdev_bus_register(void)
+{
+	return bus_register(&mdev_bus_type);
+}
+
+void mdev_bus_unregister(void)
+{
+	bus_unregister(&mdev_bus_type);
+}
diff --git a/drivers/vfio/mdev/mdev_private.h b/drivers/vfio/mdev/mdev_private.h
new file mode 100644
index 000000000000..d35097cbf3d7
--- /dev/null
+++ b/drivers/vfio/mdev/mdev_private.h
@@ -0,0 +1,41 @@
+/*
+ * Mediated device interal definitions
+ *
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ *     Author: Neo Jia <cjia@nvidia.com>
+ *             Kirti Wankhede <kwankhede@nvidia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef MDEV_PRIVATE_H
+#define MDEV_PRIVATE_H
+
+int  mdev_bus_register(void);
+void mdev_bus_unregister(void);
+
+struct mdev_type {
+	struct kobject kobj;
+	struct kobject *devices_kobj;
+	struct parent_device *parent;
+	struct list_head next;
+	struct attribute_group *group;
+};
+
+#define to_mdev_type_attr(_attr)	\
+	container_of(_attr, struct mdev_type_attribute, attr)
+#define to_mdev_type(_kobj)		\
+	container_of(_kobj, struct mdev_type, kobj)
+
+int  parent_create_sysfs_files(struct parent_device *parent);
+void parent_remove_sysfs_files(struct parent_device *parent);
+
+int  mdev_create_sysfs_files(struct device *dev, struct mdev_type *type);
+void mdev_remove_sysfs_files(struct device *dev, struct mdev_type *type);
+
+int  mdev_device_create(struct kobject *kobj, struct device *dev, uuid_le uuid);
+int  mdev_device_remove(struct device *dev, bool force_remove);
+
+#endif /* MDEV_PRIVATE_H */
diff --git a/drivers/vfio/mdev/mdev_sysfs.c b/drivers/vfio/mdev/mdev_sysfs.c
new file mode 100644
index 000000000000..1a53deb2ee10
--- /dev/null
+++ b/drivers/vfio/mdev/mdev_sysfs.c
@@ -0,0 +1,286 @@
+/*
+ * File attributes for Mediated devices
+ *
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ *     Author: Neo Jia <cjia@nvidia.com>
+ *             Kirti Wankhede <kwankhede@nvidia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/sysfs.h>
+#include <linux/ctype.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/uuid.h>
+#include <linux/mdev.h>
+
+#include "mdev_private.h"
+
+/* Static functions */
+
+static ssize_t mdev_type_attr_show(struct kobject *kobj,
+				     struct attribute *__attr, char *buf)
+{
+	struct mdev_type_attribute *attr = to_mdev_type_attr(__attr);
+	struct mdev_type *type = to_mdev_type(kobj);
+	ssize_t ret = -EIO;
+
+	if (attr->show)
+		ret = attr->show(kobj, type->parent->dev, buf);
+	return ret;
+}
+
+static ssize_t mdev_type_attr_store(struct kobject *kobj,
+				      struct attribute *__attr,
+				      const char *buf, size_t count)
+{
+	struct mdev_type_attribute *attr = to_mdev_type_attr(__attr);
+	struct mdev_type *type = to_mdev_type(kobj);
+	ssize_t ret = -EIO;
+
+	if (attr->store)
+		ret = attr->store(&type->kobj, type->parent->dev, buf, count);
+	return ret;
+}
+
+static const struct sysfs_ops mdev_type_sysfs_ops = {
+	.show = mdev_type_attr_show,
+	.store = mdev_type_attr_store,
+};
+
+static ssize_t create_store(struct kobject *kobj, struct device *dev,
+			    const char *buf, size_t count)
+{
+	char *str;
+	uuid_le uuid;
+	int ret;
+
+	if ((count < UUID_STRING_LEN) || (count > UUID_STRING_LEN + 1))
+		return -EINVAL;
+
+	str = kstrndup(buf, count, GFP_KERNEL);
+	if (!str)
+		return -ENOMEM;
+
+	ret = uuid_le_to_bin(str, &uuid);
+	kfree(str);
+	if (ret)
+		return ret;
+
+	ret = mdev_device_create(kobj, dev, uuid);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+MDEV_TYPE_ATTR_WO(create);
+
+static void mdev_type_release(struct kobject *kobj)
+{
+	struct mdev_type *type = to_mdev_type(kobj);
+
+	pr_debug("Releasing group %s\n", kobj->name);
+	kfree(type);
+}
+
+static struct kobj_type mdev_type_ktype = {
+	.sysfs_ops = &mdev_type_sysfs_ops,
+	.release = mdev_type_release,
+};
+
+struct mdev_type *add_mdev_supported_type(struct parent_device *parent,
+					  struct attribute_group *group)
+{
+	struct mdev_type *type;
+	int ret;
+
+	if (!group->name) {
+		pr_err("%s: Type name empty!\n", __func__);
+		return ERR_PTR(-EINVAL);
+	}
+
+	type = kzalloc(sizeof(*type), GFP_KERNEL);
+	if (!type)
+		return ERR_PTR(-ENOMEM);
+
+	type->kobj.kset = parent->mdev_types_kset;
+
+	ret = kobject_init_and_add(&type->kobj, &mdev_type_ktype, NULL,
+				   "%s-%s", dev_driver_string(parent->dev),
+				   group->name);
+	if (ret) {
+		kfree(type);
+		return ERR_PTR(ret);
+	}
+
+	ret = sysfs_create_file(&type->kobj, &mdev_type_attr_create.attr);
+	if (ret)
+		goto attr_create_failed;
+
+	type->devices_kobj = kobject_create_and_add("devices", &type->kobj);
+	if (!type->devices_kobj) {
+		ret = -ENOMEM;
+		goto attr_devices_failed;
+	}
+
+	ret = sysfs_create_files(&type->kobj,
+				 (const struct attribute **)group->attrs);
+	if (ret) {
+		ret = -ENOMEM;
+		goto attrs_failed;
+	}
+
+	type->group = group;
+	type->parent = parent;
+	return type;
+
+attrs_failed:
+	kobject_put(type->devices_kobj);
+attr_devices_failed:
+	sysfs_remove_file(&type->kobj, &mdev_type_attr_create.attr);
+attr_create_failed:
+	kobject_del(&type->kobj);
+	kobject_put(&type->kobj);
+	return ERR_PTR(ret);
+}
+
+static void remove_mdev_supported_type(struct mdev_type *type)
+{
+	sysfs_remove_files(&type->kobj,
+			   (const struct attribute **)type->group->attrs);
+	kobject_put(type->devices_kobj);
+	sysfs_remove_file(&type->kobj, &mdev_type_attr_create.attr);
+	kobject_del(&type->kobj);
+	kobject_put(&type->kobj);
+}
+
+static int add_mdev_supported_type_groups(struct parent_device *parent)
+{
+	int i;
+
+	for (i = 0; parent->ops->supported_type_groups[i]; i++) {
+		struct mdev_type *type;
+
+		type = add_mdev_supported_type(parent,
+					parent->ops->supported_type_groups[i]);
+		if (IS_ERR(type)) {
+			struct mdev_type *ltype, *tmp;
+
+			list_for_each_entry_safe(ltype, tmp, &parent->type_list,
+						  next) {
+				list_del(&ltype->next);
+				remove_mdev_supported_type(ltype);
+			}
+			return PTR_ERR(type);
+		}
+		list_add(&type->next, &parent->type_list);
+	}
+	return 0;
+}
+
+/* mdev sysfs functions */
+void parent_remove_sysfs_files(struct parent_device *parent)
+{
+	struct mdev_type *type, *tmp;
+
+	list_for_each_entry_safe(type, tmp, &parent->type_list, next) {
+		list_del(&type->next);
+		remove_mdev_supported_type(type);
+	}
+
+	sysfs_remove_groups(&parent->dev->kobj, parent->ops->dev_attr_groups);
+	kset_unregister(parent->mdev_types_kset);
+}
+
+int parent_create_sysfs_files(struct parent_device *parent)
+{
+	int ret;
+
+	parent->mdev_types_kset = kset_create_and_add("mdev_supported_types",
+					       NULL, &parent->dev->kobj);
+
+	if (!parent->mdev_types_kset)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&parent->type_list);
+
+	ret = sysfs_create_groups(&parent->dev->kobj,
+				  parent->ops->dev_attr_groups);
+	if (ret)
+		goto create_err;
+
+	ret = add_mdev_supported_type_groups(parent);
+	if (ret)
+		sysfs_remove_groups(&parent->dev->kobj,
+				    parent->ops->dev_attr_groups);
+	else
+		return ret;
+
+create_err:
+	kset_unregister(parent->mdev_types_kset);
+	return ret;
+}
+
+static ssize_t remove_store(struct device *dev, struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	unsigned long val;
+
+	if (kstrtoul(buf, 0, &val) < 0)
+		return -EINVAL;
+
+	if (val && device_remove_file_self(dev, attr)) {
+		int ret;
+
+		ret = mdev_device_remove(dev, false);
+		if (ret) {
+			device_create_file(dev, attr);
+			return ret;
+		}
+	}
+
+	return count;
+}
+
+static DEVICE_ATTR_WO(remove);
+
+static const struct attribute *mdev_device_attrs[] = {
+	&dev_attr_remove.attr,
+	NULL,
+};
+
+int  mdev_create_sysfs_files(struct device *dev, struct mdev_type *type)
+{
+	int ret;
+
+	ret = sysfs_create_files(&dev->kobj, mdev_device_attrs);
+	if (ret)
+		return ret;
+
+	ret = sysfs_create_link(type->devices_kobj, &dev->kobj, dev_name(dev));
+	if (ret)
+		goto device_link_failed;
+
+	ret = sysfs_create_link(&dev->kobj, &type->kobj, "mdev_type");
+	if (ret)
+		goto type_link_failed;
+
+	return ret;
+
+type_link_failed:
+	sysfs_remove_link(type->devices_kobj, dev_name(dev));
+device_link_failed:
+	sysfs_remove_files(&dev->kobj, mdev_device_attrs);
+	return ret;
+}
+
+void mdev_remove_sysfs_files(struct device *dev, struct mdev_type *type)
+{
+	sysfs_remove_link(&dev->kobj, "mdev_type");
+	sysfs_remove_link(type->devices_kobj, dev_name(dev));
+	sysfs_remove_files(&dev->kobj, mdev_device_attrs);
+}
diff --git a/drivers/vfio/mdev/vfio_mdev.c b/drivers/vfio/mdev/vfio_mdev.c
new file mode 100644
index 000000000000..ffc36758cb84
--- /dev/null
+++ b/drivers/vfio/mdev/vfio_mdev.c
@@ -0,0 +1,148 @@
+/*
+ * VFIO based driver for Mediated device
+ *
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ *     Author: Neo Jia <cjia@nvidia.com>
+ *             Kirti Wankhede <kwankhede@nvidia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/vfio.h>
+#include <linux/mdev.h>
+
+#include "mdev_private.h"
+
+#define DRIVER_VERSION  "0.1"
+#define DRIVER_AUTHOR   "NVIDIA Corporation"
+#define DRIVER_DESC     "VFIO based driver for Mediated device"
+
+static int vfio_mdev_open(void *device_data)
+{
+	struct mdev_device *mdev = device_data;
+	struct parent_device *parent = mdev->parent;
+	int ret;
+
+	if (unlikely(!parent->ops->open))
+		return -EINVAL;
+
+	if (!try_module_get(THIS_MODULE))
+		return -ENODEV;
+
+	ret = parent->ops->open(mdev);
+	if (ret)
+		module_put(THIS_MODULE);
+
+	return ret;
+}
+
+static void vfio_mdev_release(void *device_data)
+{
+	struct mdev_device *mdev = device_data;
+	struct parent_device *parent = mdev->parent;
+
+	if (likely(parent->ops->release))
+		parent->ops->release(mdev);
+
+	module_put(THIS_MODULE);
+}
+
+static long vfio_mdev_unlocked_ioctl(void *device_data,
+				     unsigned int cmd, unsigned long arg)
+{
+	struct mdev_device *mdev = device_data;
+	struct parent_device *parent = mdev->parent;
+
+	if (unlikely(!parent->ops->ioctl))
+		return -EINVAL;
+
+	return parent->ops->ioctl(mdev, cmd, arg);
+}
+
+static ssize_t vfio_mdev_read(void *device_data, char __user *buf,
+			      size_t count, loff_t *ppos)
+{
+	struct mdev_device *mdev = device_data;
+	struct parent_device *parent = mdev->parent;
+
+	if (unlikely(!parent->ops->read))
+		return -EINVAL;
+
+	return parent->ops->read(mdev, buf, count, ppos);
+}
+
+static ssize_t vfio_mdev_write(void *device_data, const char __user *buf,
+			       size_t count, loff_t *ppos)
+{
+	struct mdev_device *mdev = device_data;
+	struct parent_device *parent = mdev->parent;
+
+	if (unlikely(!parent->ops->write))
+		return -EINVAL;
+
+	return parent->ops->write(mdev, buf, count, ppos);
+}
+
+static int vfio_mdev_mmap(void *device_data, struct vm_area_struct *vma)
+{
+	struct mdev_device *mdev = device_data;
+	struct parent_device *parent = mdev->parent;
+
+	if (unlikely(!parent->ops->mmap))
+		return -EINVAL;
+
+	return parent->ops->mmap(mdev, vma);
+}
+
+static const struct vfio_device_ops vfio_mdev_dev_ops = {
+	.name		= "vfio-mdev",
+	.open		= vfio_mdev_open,
+	.release	= vfio_mdev_release,
+	.ioctl		= vfio_mdev_unlocked_ioctl,
+	.read		= vfio_mdev_read,
+	.write		= vfio_mdev_write,
+	.mmap		= vfio_mdev_mmap,
+};
+
+int vfio_mdev_probe(struct device *dev)
+{
+	struct mdev_device *mdev = to_mdev_device(dev);
+
+	return vfio_add_group_dev(dev, &vfio_mdev_dev_ops, mdev);
+}
+
+void vfio_mdev_remove(struct device *dev)
+{
+	vfio_del_group_dev(dev);
+}
+
+struct mdev_driver vfio_mdev_driver = {
+	.name	= "vfio_mdev",
+	.probe	= vfio_mdev_probe,
+	.remove	= vfio_mdev_remove,
+};
+
+static int __init vfio_mdev_init(void)
+{
+	return mdev_register_driver(&vfio_mdev_driver, THIS_MODULE);
+}
+
+static void __exit vfio_mdev_exit(void)
+{
+	mdev_unregister_driver(&vfio_mdev_driver);
+}
+
+module_init(vfio_mdev_init)
+module_exit(vfio_mdev_exit)
+
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESC);
diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
index 031bc08d000d..dcd7c2a99618 100644
--- a/drivers/vfio/pci/vfio_pci.c
+++ b/drivers/vfio/pci/vfio_pci.c
@@ -558,10 +558,9 @@ static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,
 static int msix_sparse_mmap_cap(struct vfio_pci_device *vdev,
 				struct vfio_info_cap *caps)
 {
-	struct vfio_info_cap_header *header;
 	struct vfio_region_info_cap_sparse_mmap *sparse;
 	size_t end, size;
-	int nr_areas = 2, i = 0;
+	int nr_areas = 2, i = 0, ret;
 
 	end = pci_resource_len(vdev->pdev, vdev->msix_bar);
 
@@ -572,13 +571,10 @@ static int msix_sparse_mmap_cap(struct vfio_pci_device *vdev,
 
 	size = sizeof(*sparse) + (nr_areas * sizeof(*sparse->areas));
 
-	header = vfio_info_cap_add(caps, size,
-				   VFIO_REGION_INFO_CAP_SPARSE_MMAP, 1);
-	if (IS_ERR(header))
-		return PTR_ERR(header);
+	sparse = kzalloc(size, GFP_KERNEL);
+	if (!sparse)
+		return -ENOMEM;
 
-	sparse = container_of(header,
-			      struct vfio_region_info_cap_sparse_mmap, header);
 	sparse->nr_areas = nr_areas;
 
 	if (vdev->msix_offset & PAGE_MASK) {
@@ -594,26 +590,11 @@ static int msix_sparse_mmap_cap(struct vfio_pci_device *vdev,
 		i++;
 	}
 
-	return 0;
-}
-
-static int region_type_cap(struct vfio_pci_device *vdev,
-			   struct vfio_info_cap *caps,
-			   unsigned int type, unsigned int subtype)
-{
-	struct vfio_info_cap_header *header;
-	struct vfio_region_info_cap_type *cap;
-
-	header = vfio_info_cap_add(caps, sizeof(*cap),
-				   VFIO_REGION_INFO_CAP_TYPE, 1);
-	if (IS_ERR(header))
-		return PTR_ERR(header);
-
-	cap = container_of(header, struct vfio_region_info_cap_type, header);
-	cap->type = type;
-	cap->subtype = subtype;
+	ret = vfio_info_add_capability(caps, VFIO_REGION_INFO_CAP_SPARSE_MMAP,
+				       sparse);
+	kfree(sparse);
 
-	return 0;
+	return ret;
 }
 
 int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
@@ -752,6 +733,9 @@ static long vfio_pci_ioctl(void *device_data,
 
 			break;
 		default:
+		{
+			struct vfio_region_info_cap_type cap_type;
+
 			if (info.index >=
 			    VFIO_PCI_NUM_REGIONS + vdev->num_regions)
 				return -EINVAL;
@@ -762,11 +746,16 @@ static long vfio_pci_ioctl(void *device_data,
 			info.size = vdev->region[i].size;
 			info.flags = vdev->region[i].flags;
 
-			ret = region_type_cap(vdev, &caps,
-					      vdev->region[i].type,
-					      vdev->region[i].subtype);
+			cap_type.type = vdev->region[i].type;
+			cap_type.subtype = vdev->region[i].subtype;
+
+			ret = vfio_info_add_capability(&caps,
+						      VFIO_REGION_INFO_CAP_TYPE,
+						      &cap_type);
 			if (ret)
 				return ret;
+
+		}
 		}
 
 		if (caps.size) {
@@ -829,45 +818,25 @@ static long vfio_pci_ioctl(void *device_data,
 
 	} else if (cmd == VFIO_DEVICE_SET_IRQS) {
 		struct vfio_irq_set hdr;
-		size_t size;
 		u8 *data = NULL;
 		int max, ret = 0;
+		size_t data_size = 0;
 
 		minsz = offsetofend(struct vfio_irq_set, count);
 
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		if (hdr.argsz < minsz || hdr.index >= VFIO_PCI_NUM_IRQS ||
-		    hdr.count >= (U32_MAX - hdr.start) ||
-		    hdr.flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
-				  VFIO_IRQ_SET_ACTION_TYPE_MASK))
-			return -EINVAL;
-
 		max = vfio_pci_get_irq_count(vdev, hdr.index);
-		if (hdr.start >= max || hdr.start + hdr.count > max)
-			return -EINVAL;
 
-		switch (hdr.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
-		case VFIO_IRQ_SET_DATA_NONE:
-			size = 0;
-			break;
-		case VFIO_IRQ_SET_DATA_BOOL:
-			size = sizeof(uint8_t);
-			break;
-		case VFIO_IRQ_SET_DATA_EVENTFD:
-			size = sizeof(int32_t);
-			break;
-		default:
-			return -EINVAL;
-		}
-
-		if (size) {
-			if (hdr.argsz - minsz < hdr.count * size)
-				return -EINVAL;
+		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+						 VFIO_PCI_NUM_IRQS, &data_size);
+		if (ret)
+			return ret;
 
+		if (data_size) {
 			data = memdup_user((void __user *)(arg + minsz),
-					   hdr.count * size);
+					    data_size);
 			if (IS_ERR(data))
 				return PTR_ERR(data);
 		}
diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c
index 65d4a3015542..330a57024cbc 100644
--- a/drivers/vfio/pci/vfio_pci_config.c
+++ b/drivers/vfio/pci/vfio_pci_config.c
@@ -31,8 +31,6 @@
 
 #include "vfio_pci_private.h"
 
-#define PCI_CFG_SPACE_SIZE	256
-
 /* Fake capability ID for standard config space */
 #define PCI_CAP_ID_BASIC	0
 
@@ -152,7 +150,7 @@ static int vfio_user_config_read(struct pci_dev *pdev, int offset,
 
 	*val = cpu_to_le32(tmp_val);
 
-	return pcibios_err_to_errno(ret);
+	return ret;
 }
 
 static int vfio_user_config_write(struct pci_dev *pdev, int offset,
@@ -173,7 +171,7 @@ static int vfio_user_config_write(struct pci_dev *pdev, int offset,
 		break;
 	}
 
-	return pcibios_err_to_errno(ret);
+	return ret;
 }
 
 static int vfio_default_config_read(struct vfio_pci_device *vdev, int pos,
@@ -257,7 +255,7 @@ static int vfio_direct_config_read(struct vfio_pci_device *vdev, int pos,
 
 	ret = vfio_user_config_read(vdev->pdev, pos, val, count);
 	if (ret)
-		return pcibios_err_to_errno(ret);
+		return ret;
 
 	if (pos >= PCI_CFG_SPACE_SIZE) { /* Extended cap header mangling */
 		if (offset < 4)
@@ -295,7 +293,7 @@ static int vfio_raw_config_read(struct vfio_pci_device *vdev, int pos,
 
 	ret = vfio_user_config_read(vdev->pdev, pos, val, count);
 	if (ret)
-		return pcibios_err_to_errno(ret);
+		return ret;
 
 	return count;
 }
@@ -1089,7 +1087,7 @@ static int vfio_msi_config_write(struct vfio_pci_device *vdev, int pos,
 						 start + PCI_MSI_FLAGS,
 						 flags);
 		if (ret)
-			return pcibios_err_to_errno(ret);
+			return ret;
 	}
 
 	return count;
diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index d78142830754..4c27f4be3c3d 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -364,36 +364,21 @@ static long vfio_platform_ioctl(void *device_data,
 		struct vfio_irq_set hdr;
 		u8 *data = NULL;
 		int ret = 0;
+		size_t data_size = 0;
 
 		minsz = offsetofend(struct vfio_irq_set, count);
 
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		if (hdr.argsz < minsz)
-			return -EINVAL;
-
-		if (hdr.index >= vdev->num_irqs)
-			return -EINVAL;
-
-		if (hdr.flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
-				  VFIO_IRQ_SET_ACTION_TYPE_MASK))
-			return -EINVAL;
-
-		if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
-			size_t size;
-
-			if (hdr.flags & VFIO_IRQ_SET_DATA_BOOL)
-				size = sizeof(uint8_t);
-			else if (hdr.flags & VFIO_IRQ_SET_DATA_EVENTFD)
-				size = sizeof(int32_t);
-			else
-				return -EINVAL;
-
-			if (hdr.argsz - minsz < size)
-				return -EINVAL;
+		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
+						 vdev->num_irqs, &data_size);
+		if (ret)
+			return ret;
 
-			data = memdup_user((void __user *)(arg + minsz), size);
+		if (data_size) {
+			data = memdup_user((void __user *)(arg + minsz),
+					    data_size);
 			if (IS_ERR(data))
 				return PTR_ERR(data);
 		}
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index d1d70e0b011b..9901c4671e2f 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -86,6 +86,8 @@ struct vfio_group {
 	struct mutex			unbound_lock;
 	atomic_t			opened;
 	bool				noiommu;
+	struct kvm			*kvm;
+	struct blocking_notifier_head	notifier;
 };
 
 struct vfio_device {
@@ -339,6 +341,7 @@ static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group)
 #ifdef CONFIG_VFIO_NOIOMMU
 	group->noiommu = (iommu_group_get_iommudata(iommu_group) == &noiommu);
 #endif
+	BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
 
 	group->nb.notifier_call = vfio_iommu_group_notifier;
 
@@ -480,6 +483,21 @@ static struct vfio_group *vfio_group_get_from_minor(int minor)
 	return group;
 }
 
+static struct vfio_group *vfio_group_get_from_dev(struct device *dev)
+{
+	struct iommu_group *iommu_group;
+	struct vfio_group *group;
+
+	iommu_group = iommu_group_get(dev);
+	if (!iommu_group)
+		return NULL;
+
+	group = vfio_group_get_from_iommu(iommu_group);
+	iommu_group_put(iommu_group);
+
+	return group;
+}
+
 /**
  * Device objects - create, release, get, put, search
  */
@@ -811,16 +829,10 @@ EXPORT_SYMBOL_GPL(vfio_add_group_dev);
  */
 struct vfio_device *vfio_device_get_from_dev(struct device *dev)
 {
-	struct iommu_group *iommu_group;
 	struct vfio_group *group;
 	struct vfio_device *device;
 
-	iommu_group = iommu_group_get(dev);
-	if (!iommu_group)
-		return NULL;
-
-	group = vfio_group_get_from_iommu(iommu_group);
-	iommu_group_put(iommu_group);
+	group = vfio_group_get_from_dev(dev);
 	if (!group)
 		return NULL;
 
@@ -1376,6 +1388,23 @@ static bool vfio_group_viable(struct vfio_group *group)
 					 group, vfio_dev_viable) == 0);
 }
 
+static int vfio_group_add_container_user(struct vfio_group *group)
+{
+	if (!atomic_inc_not_zero(&group->container_users))
+		return -EINVAL;
+
+	if (group->noiommu) {
+		atomic_dec(&group->container_users);
+		return -EPERM;
+	}
+	if (!group->container->iommu_driver || !vfio_group_viable(group)) {
+		atomic_dec(&group->container_users);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static const struct file_operations vfio_device_fops;
 
 static int vfio_group_get_device_fd(struct vfio_group *group, char *buf)
@@ -1555,6 +1584,9 @@ static int vfio_group_fops_release(struct inode *inode, struct file *filep)
 
 	filep->private_data = NULL;
 
+	/* Any user didn't unregister? */
+	WARN_ON(group->notifier.head);
+
 	vfio_group_try_dissolve_container(group);
 
 	atomic_dec(&group->opened);
@@ -1685,23 +1717,14 @@ static const struct file_operations vfio_device_fops = {
 struct vfio_group *vfio_group_get_external_user(struct file *filep)
 {
 	struct vfio_group *group = filep->private_data;
+	int ret;
 
 	if (filep->f_op != &vfio_group_fops)
 		return ERR_PTR(-EINVAL);
 
-	if (!atomic_inc_not_zero(&group->container_users))
-		return ERR_PTR(-EINVAL);
-
-	if (group->noiommu) {
-		atomic_dec(&group->container_users);
-		return ERR_PTR(-EPERM);
-	}
-
-	if (!group->container->iommu_driver ||
-			!vfio_group_viable(group)) {
-		atomic_dec(&group->container_users);
-		return ERR_PTR(-EINVAL);
-	}
+	ret = vfio_group_add_container_user(group);
+	if (ret)
+		return ERR_PTR(ret);
 
 	vfio_group_get(group);
 
@@ -1763,7 +1786,7 @@ struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps,
 	header->version = version;
 
 	/* Add to the end of the capability chain */
-	for (tmp = caps->buf; tmp->next; tmp = (void *)tmp + tmp->next)
+	for (tmp = buf; tmp->next; tmp = buf + tmp->next)
 		; /* nothing */
 
 	tmp->next = caps->size;
@@ -1776,11 +1799,403 @@ EXPORT_SYMBOL_GPL(vfio_info_cap_add);
 void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset)
 {
 	struct vfio_info_cap_header *tmp;
+	void *buf = (void *)caps->buf;
 
-	for (tmp = caps->buf; tmp->next; tmp = (void *)tmp + tmp->next - offset)
+	for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset)
 		tmp->next += offset;
 }
-EXPORT_SYMBOL_GPL(vfio_info_cap_shift);
+EXPORT_SYMBOL(vfio_info_cap_shift);
+
+static int sparse_mmap_cap(struct vfio_info_cap *caps, void *cap_type)
+{
+	struct vfio_info_cap_header *header;
+	struct vfio_region_info_cap_sparse_mmap *sparse_cap, *sparse = cap_type;
+	size_t size;
+
+	size = sizeof(*sparse) + sparse->nr_areas *  sizeof(*sparse->areas);
+	header = vfio_info_cap_add(caps, size,
+				   VFIO_REGION_INFO_CAP_SPARSE_MMAP, 1);
+	if (IS_ERR(header))
+		return PTR_ERR(header);
+
+	sparse_cap = container_of(header,
+			struct vfio_region_info_cap_sparse_mmap, header);
+	sparse_cap->nr_areas = sparse->nr_areas;
+	memcpy(sparse_cap->areas, sparse->areas,
+	       sparse->nr_areas * sizeof(*sparse->areas));
+	return 0;
+}
+
+static int region_type_cap(struct vfio_info_cap *caps, void *cap_type)
+{
+	struct vfio_info_cap_header *header;
+	struct vfio_region_info_cap_type *type_cap, *cap = cap_type;
+
+	header = vfio_info_cap_add(caps, sizeof(*cap),
+				   VFIO_REGION_INFO_CAP_TYPE, 1);
+	if (IS_ERR(header))
+		return PTR_ERR(header);
+
+	type_cap = container_of(header, struct vfio_region_info_cap_type,
+				header);
+	type_cap->type = cap->type;
+	type_cap->subtype = cap->subtype;
+	return 0;
+}
+
+int vfio_info_add_capability(struct vfio_info_cap *caps, int cap_type_id,
+			     void *cap_type)
+{
+	int ret = -EINVAL;
+
+	if (!cap_type)
+		return 0;
+
+	switch (cap_type_id) {
+	case VFIO_REGION_INFO_CAP_SPARSE_MMAP:
+		ret = sparse_mmap_cap(caps, cap_type);
+		break;
+
+	case VFIO_REGION_INFO_CAP_TYPE:
+		ret = region_type_cap(caps, cap_type);
+		break;
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(vfio_info_add_capability);
+
+int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs,
+				       int max_irq_type, size_t *data_size)
+{
+	unsigned long minsz;
+	size_t size;
+
+	minsz = offsetofend(struct vfio_irq_set, count);
+
+	if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) ||
+	    (hdr->count >= (U32_MAX - hdr->start)) ||
+	    (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
+				VFIO_IRQ_SET_ACTION_TYPE_MASK)))
+		return -EINVAL;
+
+	if (data_size)
+		*data_size = 0;
+
+	if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs)
+		return -EINVAL;
+
+	switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
+	case VFIO_IRQ_SET_DATA_NONE:
+		size = 0;
+		break;
+	case VFIO_IRQ_SET_DATA_BOOL:
+		size = sizeof(uint8_t);
+		break;
+	case VFIO_IRQ_SET_DATA_EVENTFD:
+		size = sizeof(int32_t);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (size) {
+		if (hdr->argsz - minsz < hdr->count * size)
+			return -EINVAL;
+
+		if (!data_size)
+			return -EINVAL;
+
+		*data_size = hdr->count * size;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);
+
+/*
+ * Pin a set of guest PFNs and return their associated host PFNs for local
+ * domain only.
+ * @dev [in]     : device
+ * @user_pfn [in]: array of user/guest PFNs to be unpinned.
+ * @npage [in]   : count of elements in user_pfn array.  This count should not
+ *		   be greater VFIO_PIN_PAGES_MAX_ENTRIES.
+ * @prot [in]    : protection flags
+ * @phys_pfn[out]: array of host PFNs
+ * Return error or number of pages pinned.
+ */
+int vfio_pin_pages(struct device *dev, unsigned long *user_pfn, int npage,
+		   int prot, unsigned long *phys_pfn)
+{
+	struct vfio_container *container;
+	struct vfio_group *group;
+	struct vfio_iommu_driver *driver;
+	int ret;
+
+	if (!dev || !user_pfn || !phys_pfn || !npage)
+		return -EINVAL;
+
+	if (npage > VFIO_PIN_PAGES_MAX_ENTRIES)
+		return -E2BIG;
+
+	group = vfio_group_get_from_dev(dev);
+	if (!group)
+		return -ENODEV;
+
+	ret = vfio_group_add_container_user(group);
+	if (ret)
+		goto err_pin_pages;
+
+	container = group->container;
+	down_read(&container->group_lock);
+
+	driver = container->iommu_driver;
+	if (likely(driver && driver->ops->pin_pages))
+		ret = driver->ops->pin_pages(container->iommu_data, user_pfn,
+					     npage, prot, phys_pfn);
+	else
+		ret = -ENOTTY;
+
+	up_read(&container->group_lock);
+	vfio_group_try_dissolve_container(group);
+
+err_pin_pages:
+	vfio_group_put(group);
+	return ret;
+}
+EXPORT_SYMBOL(vfio_pin_pages);
+
+/*
+ * Unpin set of host PFNs for local domain only.
+ * @dev [in]     : device
+ * @user_pfn [in]: array of user/guest PFNs to be unpinned. Number of user/guest
+ *		   PFNs should not be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
+ * @npage [in]   : count of elements in user_pfn array.  This count should not
+ *                 be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
+ * Return error or number of pages unpinned.
+ */
+int vfio_unpin_pages(struct device *dev, unsigned long *user_pfn, int npage)
+{
+	struct vfio_container *container;
+	struct vfio_group *group;
+	struct vfio_iommu_driver *driver;
+	int ret;
+
+	if (!dev || !user_pfn || !npage)
+		return -EINVAL;
+
+	if (npage > VFIO_PIN_PAGES_MAX_ENTRIES)
+		return -E2BIG;
+
+	group = vfio_group_get_from_dev(dev);
+	if (!group)
+		return -ENODEV;
+
+	ret = vfio_group_add_container_user(group);
+	if (ret)
+		goto err_unpin_pages;
+
+	container = group->container;
+	down_read(&container->group_lock);
+
+	driver = container->iommu_driver;
+	if (likely(driver && driver->ops->unpin_pages))
+		ret = driver->ops->unpin_pages(container->iommu_data, user_pfn,
+					       npage);
+	else
+		ret = -ENOTTY;
+
+	up_read(&container->group_lock);
+	vfio_group_try_dissolve_container(group);
+
+err_unpin_pages:
+	vfio_group_put(group);
+	return ret;
+}
+EXPORT_SYMBOL(vfio_unpin_pages);
+
+static int vfio_register_iommu_notifier(struct vfio_group *group,
+					unsigned long *events,
+					struct notifier_block *nb)
+{
+	struct vfio_container *container;
+	struct vfio_iommu_driver *driver;
+	int ret;
+
+	ret = vfio_group_add_container_user(group);
+	if (ret)
+		return -EINVAL;
+
+	container = group->container;
+	down_read(&container->group_lock);
+
+	driver = container->iommu_driver;
+	if (likely(driver && driver->ops->register_notifier))
+		ret = driver->ops->register_notifier(container->iommu_data,
+						     events, nb);
+	else
+		ret = -ENOTTY;
+
+	up_read(&container->group_lock);
+	vfio_group_try_dissolve_container(group);
+
+	return ret;
+}
+
+static int vfio_unregister_iommu_notifier(struct vfio_group *group,
+					  struct notifier_block *nb)
+{
+	struct vfio_container *container;
+	struct vfio_iommu_driver *driver;
+	int ret;
+
+	ret = vfio_group_add_container_user(group);
+	if (ret)
+		return -EINVAL;
+
+	container = group->container;
+	down_read(&container->group_lock);
+
+	driver = container->iommu_driver;
+	if (likely(driver && driver->ops->unregister_notifier))
+		ret = driver->ops->unregister_notifier(container->iommu_data,
+						       nb);
+	else
+		ret = -ENOTTY;
+
+	up_read(&container->group_lock);
+	vfio_group_try_dissolve_container(group);
+
+	return ret;
+}
+
+void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm)
+{
+	group->kvm = kvm;
+	blocking_notifier_call_chain(&group->notifier,
+				VFIO_GROUP_NOTIFY_SET_KVM, kvm);
+}
+EXPORT_SYMBOL_GPL(vfio_group_set_kvm);
+
+static int vfio_register_group_notifier(struct vfio_group *group,
+					unsigned long *events,
+					struct notifier_block *nb)
+{
+	struct vfio_container *container;
+	int ret;
+	bool set_kvm = false;
+
+	if (*events & VFIO_GROUP_NOTIFY_SET_KVM)
+		set_kvm = true;
+
+	/* clear known events */
+	*events &= ~VFIO_GROUP_NOTIFY_SET_KVM;
+
+	/* refuse to continue if still events remaining */
+	if (*events)
+		return -EINVAL;
+
+	ret = vfio_group_add_container_user(group);
+	if (ret)
+		return -EINVAL;
+
+	container = group->container;
+	down_read(&container->group_lock);
+
+	ret = blocking_notifier_chain_register(&group->notifier, nb);
+
+	/*
+	 * The attaching of kvm and vfio_group might already happen, so
+	 * here we replay once upon registration.
+	 */
+	if (!ret && set_kvm && group->kvm)
+		blocking_notifier_call_chain(&group->notifier,
+					VFIO_GROUP_NOTIFY_SET_KVM, group->kvm);
+
+	up_read(&container->group_lock);
+	vfio_group_try_dissolve_container(group);
+
+	return ret;
+}
+
+static int vfio_unregister_group_notifier(struct vfio_group *group,
+					 struct notifier_block *nb)
+{
+	struct vfio_container *container;
+	int ret;
+
+	ret = vfio_group_add_container_user(group);
+	if (ret)
+		return -EINVAL;
+
+	container = group->container;
+	down_read(&container->group_lock);
+
+	ret = blocking_notifier_chain_unregister(&group->notifier, nb);
+
+	up_read(&container->group_lock);
+	vfio_group_try_dissolve_container(group);
+
+	return ret;
+}
+
+int vfio_register_notifier(struct device *dev, enum vfio_notify_type type,
+			   unsigned long *events, struct notifier_block *nb)
+{
+	struct vfio_group *group;
+	int ret;
+
+	if (!dev || !nb || !events || (*events == 0))
+		return -EINVAL;
+
+	group = vfio_group_get_from_dev(dev);
+	if (!group)
+		return -ENODEV;
+
+	switch (type) {
+	case VFIO_IOMMU_NOTIFY:
+		ret = vfio_register_iommu_notifier(group, events, nb);
+		break;
+	case VFIO_GROUP_NOTIFY:
+		ret = vfio_register_group_notifier(group, events, nb);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	vfio_group_put(group);
+	return ret;
+}
+EXPORT_SYMBOL(vfio_register_notifier);
+
+int vfio_unregister_notifier(struct device *dev, enum vfio_notify_type type,
+			     struct notifier_block *nb)
+{
+	struct vfio_group *group;
+	int ret;
+
+	if (!dev || !nb)
+		return -EINVAL;
+
+	group = vfio_group_get_from_dev(dev);
+	if (!group)
+		return -ENODEV;
+
+	switch (type) {
+	case VFIO_IOMMU_NOTIFY:
+		ret = vfio_unregister_iommu_notifier(group, nb);
+		break;
+	case VFIO_GROUP_NOTIFY:
+		ret = vfio_unregister_group_notifier(group, nb);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	vfio_group_put(group);
+	return ret;
+}
+EXPORT_SYMBOL(vfio_unregister_notifier);
 
 /**
  * Module/class support
diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c
index 80378ddadc5c..c8823578a1b2 100644
--- a/drivers/vfio/vfio_iommu_spapr_tce.c
+++ b/drivers/vfio/vfio_iommu_spapr_tce.c
@@ -31,49 +31,49 @@
 static void tce_iommu_detach_group(void *iommu_data,
 		struct iommu_group *iommu_group);
 
-static long try_increment_locked_vm(long npages)
+static long try_increment_locked_vm(struct mm_struct *mm, long npages)
 {
 	long ret = 0, locked, lock_limit;
 
-	if (!current || !current->mm)
-		return -ESRCH; /* process exited */
+	if (WARN_ON_ONCE(!mm))
+		return -EPERM;
 
 	if (!npages)
 		return 0;
 
-	down_write(&current->mm->mmap_sem);
-	locked = current->mm->locked_vm + npages;
+	down_write(&mm->mmap_sem);
+	locked = mm->locked_vm + npages;
 	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 	if (locked > lock_limit && !capable(CAP_IPC_LOCK))
 		ret = -ENOMEM;
 	else
-		current->mm->locked_vm += npages;
+		mm->locked_vm += npages;
 
 	pr_debug("[%d] RLIMIT_MEMLOCK +%ld %ld/%ld%s\n", current->pid,
 			npages << PAGE_SHIFT,
-			current->mm->locked_vm << PAGE_SHIFT,
+			mm->locked_vm << PAGE_SHIFT,
 			rlimit(RLIMIT_MEMLOCK),
 			ret ? " - exceeded" : "");
 
-	up_write(&current->mm->mmap_sem);
+	up_write(&mm->mmap_sem);
 
 	return ret;
 }
 
-static void decrement_locked_vm(long npages)
+static void decrement_locked_vm(struct mm_struct *mm, long npages)
 {
-	if (!current || !current->mm || !npages)
-		return; /* process exited */
+	if (!mm || !npages)
+		return;
 
-	down_write(&current->mm->mmap_sem);
-	if (WARN_ON_ONCE(npages > current->mm->locked_vm))
-		npages = current->mm->locked_vm;
-	current->mm->locked_vm -= npages;
+	down_write(&mm->mmap_sem);
+	if (WARN_ON_ONCE(npages > mm->locked_vm))
+		npages = mm->locked_vm;
+	mm->locked_vm -= npages;
 	pr_debug("[%d] RLIMIT_MEMLOCK -%ld %ld/%ld\n", current->pid,
 			npages << PAGE_SHIFT,
-			current->mm->locked_vm << PAGE_SHIFT,
+			mm->locked_vm << PAGE_SHIFT,
 			rlimit(RLIMIT_MEMLOCK));
-	up_write(&current->mm->mmap_sem);
+	up_write(&mm->mmap_sem);
 }
 
 /*
@@ -89,6 +89,15 @@ struct tce_iommu_group {
 };
 
 /*
+ * A container needs to remember which preregistered region  it has
+ * referenced to do proper cleanup at the userspace process exit.
+ */
+struct tce_iommu_prereg {
+	struct list_head next;
+	struct mm_iommu_table_group_mem_t *mem;
+};
+
+/*
  * The container descriptor supports only a single group per container.
  * Required by the API as the container is not supplied with the IOMMU group
  * at the moment of initialization.
@@ -97,24 +106,68 @@ struct tce_container {
 	struct mutex lock;
 	bool enabled;
 	bool v2;
+	bool def_window_pending;
 	unsigned long locked_pages;
+	struct mm_struct *mm;
 	struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES];
 	struct list_head group_list;
+	struct list_head prereg_list;
 };
 
+static long tce_iommu_mm_set(struct tce_container *container)
+{
+	if (container->mm) {
+		if (container->mm == current->mm)
+			return 0;
+		return -EPERM;
+	}
+	BUG_ON(!current->mm);
+	container->mm = current->mm;
+	atomic_inc(&container->mm->mm_count);
+
+	return 0;
+}
+
+static long tce_iommu_prereg_free(struct tce_container *container,
+		struct tce_iommu_prereg *tcemem)
+{
+	long ret;
+
+	ret = mm_iommu_put(container->mm, tcemem->mem);
+	if (ret)
+		return ret;
+
+	list_del(&tcemem->next);
+	kfree(tcemem);
+
+	return 0;
+}
+
 static long tce_iommu_unregister_pages(struct tce_container *container,
 		__u64 vaddr, __u64 size)
 {
 	struct mm_iommu_table_group_mem_t *mem;
+	struct tce_iommu_prereg *tcemem;
+	bool found = false;
 
 	if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK))
 		return -EINVAL;
 
-	mem = mm_iommu_find(vaddr, size >> PAGE_SHIFT);
+	mem = mm_iommu_find(container->mm, vaddr, size >> PAGE_SHIFT);
 	if (!mem)
 		return -ENOENT;
 
-	return mm_iommu_put(mem);
+	list_for_each_entry(tcemem, &container->prereg_list, next) {
+		if (tcemem->mem == mem) {
+			found = true;
+			break;
+		}
+	}
+
+	if (!found)
+		return -ENOENT;
+
+	return tce_iommu_prereg_free(container, tcemem);
 }
 
 static long tce_iommu_register_pages(struct tce_container *container,
@@ -122,22 +175,36 @@ static long tce_iommu_register_pages(struct tce_container *container,
 {
 	long ret = 0;
 	struct mm_iommu_table_group_mem_t *mem = NULL;
+	struct tce_iommu_prereg *tcemem;
 	unsigned long entries = size >> PAGE_SHIFT;
 
 	if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK) ||
 			((vaddr + size) < vaddr))
 		return -EINVAL;
 
-	ret = mm_iommu_get(vaddr, entries, &mem);
+	mem = mm_iommu_find(container->mm, vaddr, entries);
+	if (mem) {
+		list_for_each_entry(tcemem, &container->prereg_list, next) {
+			if (tcemem->mem == mem)
+				return -EBUSY;
+		}
+	}
+
+	ret = mm_iommu_get(container->mm, vaddr, entries, &mem);
 	if (ret)
 		return ret;
 
+	tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL);
+	tcemem->mem = mem;
+	list_add(&tcemem->next, &container->prereg_list);
+
 	container->enabled = true;
 
 	return 0;
 }
 
-static long tce_iommu_userspace_view_alloc(struct iommu_table *tbl)
+static long tce_iommu_userspace_view_alloc(struct iommu_table *tbl,
+		struct mm_struct *mm)
 {
 	unsigned long cb = _ALIGN_UP(sizeof(tbl->it_userspace[0]) *
 			tbl->it_size, PAGE_SIZE);
@@ -146,13 +213,13 @@ static long tce_iommu_userspace_view_alloc(struct iommu_table *tbl)
 
 	BUG_ON(tbl->it_userspace);
 
-	ret = try_increment_locked_vm(cb >> PAGE_SHIFT);
+	ret = try_increment_locked_vm(mm, cb >> PAGE_SHIFT);
 	if (ret)
 		return ret;
 
 	uas = vzalloc(cb);
 	if (!uas) {
-		decrement_locked_vm(cb >> PAGE_SHIFT);
+		decrement_locked_vm(mm, cb >> PAGE_SHIFT);
 		return -ENOMEM;
 	}
 	tbl->it_userspace = uas;
@@ -160,7 +227,8 @@ static long tce_iommu_userspace_view_alloc(struct iommu_table *tbl)
 	return 0;
 }
 
-static void tce_iommu_userspace_view_free(struct iommu_table *tbl)
+static void tce_iommu_userspace_view_free(struct iommu_table *tbl,
+		struct mm_struct *mm)
 {
 	unsigned long cb = _ALIGN_UP(sizeof(tbl->it_userspace[0]) *
 			tbl->it_size, PAGE_SIZE);
@@ -170,7 +238,7 @@ static void tce_iommu_userspace_view_free(struct iommu_table *tbl)
 
 	vfree(tbl->it_userspace);
 	tbl->it_userspace = NULL;
-	decrement_locked_vm(cb >> PAGE_SHIFT);
+	decrement_locked_vm(mm, cb >> PAGE_SHIFT);
 }
 
 static bool tce_page_is_contained(struct page *page, unsigned page_shift)
@@ -230,9 +298,6 @@ static int tce_iommu_enable(struct tce_container *container)
 	struct iommu_table_group *table_group;
 	struct tce_iommu_group *tcegrp;
 
-	if (!current->mm)
-		return -ESRCH; /* process exited */
-
 	if (container->enabled)
 		return -EBUSY;
 
@@ -277,8 +342,12 @@ static int tce_iommu_enable(struct tce_container *container)
 	if (!table_group->tce32_size)
 		return -EPERM;
 
+	ret = tce_iommu_mm_set(container);
+	if (ret)
+		return ret;
+
 	locked = table_group->tce32_size >> PAGE_SHIFT;
-	ret = try_increment_locked_vm(locked);
+	ret = try_increment_locked_vm(container->mm, locked);
 	if (ret)
 		return ret;
 
@@ -296,10 +365,8 @@ static void tce_iommu_disable(struct tce_container *container)
 
 	container->enabled = false;
 
-	if (!current->mm)
-		return;
-
-	decrement_locked_vm(container->locked_pages);
+	BUG_ON(!container->mm);
+	decrement_locked_vm(container->mm, container->locked_pages);
 }
 
 static void *tce_iommu_open(unsigned long arg)
@@ -317,6 +384,7 @@ static void *tce_iommu_open(unsigned long arg)
 
 	mutex_init(&container->lock);
 	INIT_LIST_HEAD_RCU(&container->group_list);
+	INIT_LIST_HEAD_RCU(&container->prereg_list);
 
 	container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU;
 
@@ -326,7 +394,8 @@ static void *tce_iommu_open(unsigned long arg)
 static int tce_iommu_clear(struct tce_container *container,
 		struct iommu_table *tbl,
 		unsigned long entry, unsigned long pages);
-static void tce_iommu_free_table(struct iommu_table *tbl);
+static void tce_iommu_free_table(struct tce_container *container,
+		struct iommu_table *tbl);
 
 static void tce_iommu_release(void *iommu_data)
 {
@@ -351,10 +420,20 @@ static void tce_iommu_release(void *iommu_data)
 			continue;
 
 		tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
-		tce_iommu_free_table(tbl);
+		tce_iommu_free_table(container, tbl);
+	}
+
+	while (!list_empty(&container->prereg_list)) {
+		struct tce_iommu_prereg *tcemem;
+
+		tcemem = list_first_entry(&container->prereg_list,
+				struct tce_iommu_prereg, next);
+		WARN_ON_ONCE(tce_iommu_prereg_free(container, tcemem));
 	}
 
 	tce_iommu_disable(container);
+	if (container->mm)
+		mmdrop(container->mm);
 	mutex_destroy(&container->lock);
 
 	kfree(container);
@@ -369,13 +448,14 @@ static void tce_iommu_unuse_page(struct tce_container *container,
 	put_page(page);
 }
 
-static int tce_iommu_prereg_ua_to_hpa(unsigned long tce, unsigned long size,
+static int tce_iommu_prereg_ua_to_hpa(struct tce_container *container,
+		unsigned long tce, unsigned long size,
 		unsigned long *phpa, struct mm_iommu_table_group_mem_t **pmem)
 {
 	long ret = 0;
 	struct mm_iommu_table_group_mem_t *mem;
 
-	mem = mm_iommu_lookup(tce, size);
+	mem = mm_iommu_lookup(container->mm, tce, size);
 	if (!mem)
 		return -EINVAL;
 
@@ -388,18 +468,18 @@ static int tce_iommu_prereg_ua_to_hpa(unsigned long tce, unsigned long size,
 	return 0;
 }
 
-static void tce_iommu_unuse_page_v2(struct iommu_table *tbl,
-		unsigned long entry)
+static void tce_iommu_unuse_page_v2(struct tce_container *container,
+		struct iommu_table *tbl, unsigned long entry)
 {
 	struct mm_iommu_table_group_mem_t *mem = NULL;
 	int ret;
 	unsigned long hpa = 0;
 	unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry);
 
-	if (!pua || !current || !current->mm)
+	if (!pua)
 		return;
 
-	ret = tce_iommu_prereg_ua_to_hpa(*pua, IOMMU_PAGE_SIZE(tbl),
+	ret = tce_iommu_prereg_ua_to_hpa(container, *pua, IOMMU_PAGE_SIZE(tbl),
 			&hpa, &mem);
 	if (ret)
 		pr_debug("%s: tce %lx at #%lx was not cached, ret=%d\n",
@@ -429,7 +509,7 @@ static int tce_iommu_clear(struct tce_container *container,
 			continue;
 
 		if (container->v2) {
-			tce_iommu_unuse_page_v2(tbl, entry);
+			tce_iommu_unuse_page_v2(container, tbl, entry);
 			continue;
 		}
 
@@ -509,13 +589,19 @@ static long tce_iommu_build_v2(struct tce_container *container,
 	unsigned long hpa;
 	enum dma_data_direction dirtmp;
 
+	if (!tbl->it_userspace) {
+		ret = tce_iommu_userspace_view_alloc(tbl, container->mm);
+		if (ret)
+			return ret;
+	}
+
 	for (i = 0; i < pages; ++i) {
 		struct mm_iommu_table_group_mem_t *mem = NULL;
 		unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl,
 				entry + i);
 
-		ret = tce_iommu_prereg_ua_to_hpa(tce, IOMMU_PAGE_SIZE(tbl),
-				&hpa, &mem);
+		ret = tce_iommu_prereg_ua_to_hpa(container,
+				tce, IOMMU_PAGE_SIZE(tbl), &hpa, &mem);
 		if (ret)
 			break;
 
@@ -536,7 +622,7 @@ static long tce_iommu_build_v2(struct tce_container *container,
 		ret = iommu_tce_xchg(tbl, entry + i, &hpa, &dirtmp);
 		if (ret) {
 			/* dirtmp cannot be DMA_NONE here */
-			tce_iommu_unuse_page_v2(tbl, entry + i);
+			tce_iommu_unuse_page_v2(container, tbl, entry + i);
 			pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
 					__func__, entry << tbl->it_page_shift,
 					tce, ret);
@@ -544,7 +630,7 @@ static long tce_iommu_build_v2(struct tce_container *container,
 		}
 
 		if (dirtmp != DMA_NONE)
-			tce_iommu_unuse_page_v2(tbl, entry + i);
+			tce_iommu_unuse_page_v2(container, tbl, entry + i);
 
 		*pua = tce;
 
@@ -572,7 +658,7 @@ static long tce_iommu_create_table(struct tce_container *container,
 	if (!table_size)
 		return -EINVAL;
 
-	ret = try_increment_locked_vm(table_size >> PAGE_SHIFT);
+	ret = try_increment_locked_vm(container->mm, table_size >> PAGE_SHIFT);
 	if (ret)
 		return ret;
 
@@ -582,25 +668,17 @@ static long tce_iommu_create_table(struct tce_container *container,
 	WARN_ON(!ret && !(*ptbl)->it_ops->free);
 	WARN_ON(!ret && ((*ptbl)->it_allocated_size != table_size));
 
-	if (!ret && container->v2) {
-		ret = tce_iommu_userspace_view_alloc(*ptbl);
-		if (ret)
-			(*ptbl)->it_ops->free(*ptbl);
-	}
-
-	if (ret)
-		decrement_locked_vm(table_size >> PAGE_SHIFT);
-
 	return ret;
 }
 
-static void tce_iommu_free_table(struct iommu_table *tbl)
+static void tce_iommu_free_table(struct tce_container *container,
+		struct iommu_table *tbl)
 {
 	unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
 
-	tce_iommu_userspace_view_free(tbl);
+	tce_iommu_userspace_view_free(tbl, container->mm);
 	tbl->it_ops->free(tbl);
-	decrement_locked_vm(pages);
+	decrement_locked_vm(container->mm, pages);
 }
 
 static long tce_iommu_create_window(struct tce_container *container,
@@ -663,7 +741,7 @@ unset_exit:
 		table_group = iommu_group_get_iommudata(tcegrp->grp);
 		table_group->ops->unset_window(table_group, num);
 	}
-	tce_iommu_free_table(tbl);
+	tce_iommu_free_table(container, tbl);
 
 	return ret;
 }
@@ -701,12 +779,41 @@ static long tce_iommu_remove_window(struct tce_container *container,
 
 	/* Free table */
 	tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
-	tce_iommu_free_table(tbl);
+	tce_iommu_free_table(container, tbl);
 	container->tables[num] = NULL;
 
 	return 0;
 }
 
+static long tce_iommu_create_default_window(struct tce_container *container)
+{
+	long ret;
+	__u64 start_addr = 0;
+	struct tce_iommu_group *tcegrp;
+	struct iommu_table_group *table_group;
+
+	if (!container->def_window_pending)
+		return 0;
+
+	if (!tce_groups_attached(container))
+		return -ENODEV;
+
+	tcegrp = list_first_entry(&container->group_list,
+			struct tce_iommu_group, next);
+	table_group = iommu_group_get_iommudata(tcegrp->grp);
+	if (!table_group)
+		return -ENODEV;
+
+	ret = tce_iommu_create_window(container, IOMMU_PAGE_SHIFT_4K,
+			table_group->tce32_size, 1, &start_addr);
+	WARN_ON_ONCE(!ret && start_addr);
+
+	if (!ret)
+		container->def_window_pending = false;
+
+	return ret;
+}
+
 static long tce_iommu_ioctl(void *iommu_data,
 				 unsigned int cmd, unsigned long arg)
 {
@@ -727,7 +834,17 @@ static long tce_iommu_ioctl(void *iommu_data,
 		}
 
 		return (ret < 0) ? 0 : ret;
+	}
+
+	/*
+	 * Sanity check to prevent one userspace from manipulating
+	 * another userspace mm.
+	 */
+	BUG_ON(!container);
+	if (container->mm && container->mm != current->mm)
+		return -EPERM;
 
+	switch (cmd) {
 	case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
 		struct vfio_iommu_spapr_tce_info info;
 		struct tce_iommu_group *tcegrp;
@@ -797,6 +914,10 @@ static long tce_iommu_ioctl(void *iommu_data,
 				VFIO_DMA_MAP_FLAG_WRITE))
 			return -EINVAL;
 
+		ret = tce_iommu_create_default_window(container);
+		if (ret)
+			return ret;
+
 		num = tce_iommu_find_table(container, param.iova, &tbl);
 		if (num < 0)
 			return -ENXIO;
@@ -860,6 +981,10 @@ static long tce_iommu_ioctl(void *iommu_data,
 		if (param.flags)
 			return -EINVAL;
 
+		ret = tce_iommu_create_default_window(container);
+		if (ret)
+			return ret;
+
 		num = tce_iommu_find_table(container, param.iova, &tbl);
 		if (num < 0)
 			return -ENXIO;
@@ -888,6 +1013,10 @@ static long tce_iommu_ioctl(void *iommu_data,
 		minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
 				size);
 
+		ret = tce_iommu_mm_set(container);
+		if (ret)
+			return ret;
+
 		if (copy_from_user(&param, (void __user *)arg, minsz))
 			return -EFAULT;
 
@@ -911,6 +1040,9 @@ static long tce_iommu_ioctl(void *iommu_data,
 		if (!container->v2)
 			break;
 
+		if (!container->mm)
+			return -EPERM;
+
 		minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
 				size);
 
@@ -969,6 +1101,10 @@ static long tce_iommu_ioctl(void *iommu_data,
 		if (!container->v2)
 			break;
 
+		ret = tce_iommu_mm_set(container);
+		if (ret)
+			return ret;
+
 		if (!tce_groups_attached(container))
 			return -ENXIO;
 
@@ -986,6 +1122,10 @@ static long tce_iommu_ioctl(void *iommu_data,
 
 		mutex_lock(&container->lock);
 
+		ret = tce_iommu_create_default_window(container);
+		if (ret)
+			return ret;
+
 		ret = tce_iommu_create_window(container, create.page_shift,
 				create.window_size, create.levels,
 				&create.start_addr);
@@ -1003,6 +1143,10 @@ static long tce_iommu_ioctl(void *iommu_data,
 		if (!container->v2)
 			break;
 
+		ret = tce_iommu_mm_set(container);
+		if (ret)
+			return ret;
+
 		if (!tce_groups_attached(container))
 			return -ENXIO;
 
@@ -1018,6 +1162,11 @@ static long tce_iommu_ioctl(void *iommu_data,
 		if (remove.flags)
 			return -EINVAL;
 
+		if (container->def_window_pending && !remove.start_addr) {
+			container->def_window_pending = false;
+			return 0;
+		}
+
 		mutex_lock(&container->lock);
 
 		ret = tce_iommu_remove_window(container, remove.start_addr);
@@ -1043,7 +1192,7 @@ static void tce_iommu_release_ownership(struct tce_container *container,
 			continue;
 
 		tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
-		tce_iommu_userspace_view_free(tbl);
+		tce_iommu_userspace_view_free(tbl, container->mm);
 		if (tbl->it_map)
 			iommu_release_ownership(tbl);
 
@@ -1062,10 +1211,7 @@ static int tce_iommu_take_ownership(struct tce_container *container,
 		if (!tbl || !tbl->it_map)
 			continue;
 
-		rc = tce_iommu_userspace_view_alloc(tbl);
-		if (!rc)
-			rc = iommu_take_ownership(tbl);
-
+		rc = iommu_take_ownership(tbl);
 		if (rc) {
 			for (j = 0; j < i; ++j)
 				iommu_release_ownership(
@@ -1100,9 +1246,6 @@ static void tce_iommu_release_ownership_ddw(struct tce_container *container,
 static long tce_iommu_take_ownership_ddw(struct tce_container *container,
 		struct iommu_table_group *table_group)
 {
-	long i, ret = 0;
-	struct iommu_table *tbl = NULL;
-
 	if (!table_group->ops->create_table || !table_group->ops->set_window ||
 			!table_group->ops->release_ownership) {
 		WARN_ON_ONCE(1);
@@ -1111,47 +1254,7 @@ static long tce_iommu_take_ownership_ddw(struct tce_container *container,
 
 	table_group->ops->take_ownership(table_group);
 
-	/*
-	 * If it the first group attached, check if there is
-	 * a default DMA window and create one if none as
-	 * the userspace expects it to exist.
-	 */
-	if (!tce_groups_attached(container) && !container->tables[0]) {
-		ret = tce_iommu_create_table(container,
-				table_group,
-				0, /* window number */
-				IOMMU_PAGE_SHIFT_4K,
-				table_group->tce32_size,
-				1, /* default levels */
-				&tbl);
-		if (ret)
-			goto release_exit;
-		else
-			container->tables[0] = tbl;
-	}
-
-	/* Set all windows to the new group */
-	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
-		tbl = container->tables[i];
-
-		if (!tbl)
-			continue;
-
-		/* Set the default window to a new group */
-		ret = table_group->ops->set_window(table_group, i, tbl);
-		if (ret)
-			goto release_exit;
-	}
-
 	return 0;
-
-release_exit:
-	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
-		table_group->ops->unset_window(table_group, i);
-
-	table_group->ops->release_ownership(table_group);
-
-	return ret;
 }
 
 static int tce_iommu_attach_group(void *iommu_data,
@@ -1203,10 +1306,13 @@ static int tce_iommu_attach_group(void *iommu_data,
 	}
 
 	if (!table_group->ops || !table_group->ops->take_ownership ||
-			!table_group->ops->release_ownership)
+			!table_group->ops->release_ownership) {
 		ret = tce_iommu_take_ownership(container, table_group);
-	else
+	} else {
 		ret = tce_iommu_take_ownership_ddw(container, table_group);
+		if (!tce_groups_attached(container) && !container->tables[0])
+			container->def_window_pending = true;
+	}
 
 	if (!ret) {
 		tcegrp->grp = iommu_group;
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 2ba19424e4a1..f3726ba12aa6 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -36,6 +36,9 @@
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
 #include <linux/workqueue.h>
+#include <linux/pid_namespace.h>
+#include <linux/mdev.h>
+#include <linux/notifier.h>
 
 #define DRIVER_VERSION  "0.2"
 #define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
@@ -55,8 +58,10 @@ MODULE_PARM_DESC(disable_hugepages,
 
 struct vfio_iommu {
 	struct list_head	domain_list;
+	struct vfio_domain	*external_domain; /* domain for external user */
 	struct mutex		lock;
 	struct rb_root		dma_list;
+	struct blocking_notifier_head notifier;
 	bool			v2;
 	bool			nesting;
 };
@@ -75,6 +80,9 @@ struct vfio_dma {
 	unsigned long		vaddr;		/* Process virtual addr */
 	size_t			size;		/* Map size (bytes) */
 	int			prot;		/* IOMMU_READ/WRITE */
+	bool			iommu_mapped;
+	struct task_struct	*task;
+	struct rb_root		pfn_list;	/* Ex-user pinned pfn list */
 };
 
 struct vfio_group {
@@ -83,6 +91,21 @@ struct vfio_group {
 };
 
 /*
+ * Guest RAM pinning working set or DMA target
+ */
+struct vfio_pfn {
+	struct rb_node		node;
+	dma_addr_t		iova;		/* Device address */
+	unsigned long		pfn;		/* Host pfn */
+	atomic_t		ref_count;
+};
+
+#define IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)	\
+					(!list_empty(&iommu->domain_list))
+
+static int put_pfn(unsigned long pfn, int prot);
+
+/*
  * This code handles mapping and unmapping of user data buffers
  * into DMA'ble space using the IOMMU
  */
@@ -130,6 +153,97 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old)
 	rb_erase(&old->node, &iommu->dma_list);
 }
 
+/*
+ * Helper Functions for host iova-pfn list
+ */
+static struct vfio_pfn *vfio_find_vpfn(struct vfio_dma *dma, dma_addr_t iova)
+{
+	struct vfio_pfn *vpfn;
+	struct rb_node *node = dma->pfn_list.rb_node;
+
+	while (node) {
+		vpfn = rb_entry(node, struct vfio_pfn, node);
+
+		if (iova < vpfn->iova)
+			node = node->rb_left;
+		else if (iova > vpfn->iova)
+			node = node->rb_right;
+		else
+			return vpfn;
+	}
+	return NULL;
+}
+
+static void vfio_link_pfn(struct vfio_dma *dma,
+			  struct vfio_pfn *new)
+{
+	struct rb_node **link, *parent = NULL;
+	struct vfio_pfn *vpfn;
+
+	link = &dma->pfn_list.rb_node;
+	while (*link) {
+		parent = *link;
+		vpfn = rb_entry(parent, struct vfio_pfn, node);
+
+		if (new->iova < vpfn->iova)
+			link = &(*link)->rb_left;
+		else
+			link = &(*link)->rb_right;
+	}
+
+	rb_link_node(&new->node, parent, link);
+	rb_insert_color(&new->node, &dma->pfn_list);
+}
+
+static void vfio_unlink_pfn(struct vfio_dma *dma, struct vfio_pfn *old)
+{
+	rb_erase(&old->node, &dma->pfn_list);
+}
+
+static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova,
+				unsigned long pfn)
+{
+	struct vfio_pfn *vpfn;
+
+	vpfn = kzalloc(sizeof(*vpfn), GFP_KERNEL);
+	if (!vpfn)
+		return -ENOMEM;
+
+	vpfn->iova = iova;
+	vpfn->pfn = pfn;
+	atomic_set(&vpfn->ref_count, 1);
+	vfio_link_pfn(dma, vpfn);
+	return 0;
+}
+
+static void vfio_remove_from_pfn_list(struct vfio_dma *dma,
+				      struct vfio_pfn *vpfn)
+{
+	vfio_unlink_pfn(dma, vpfn);
+	kfree(vpfn);
+}
+
+static struct vfio_pfn *vfio_iova_get_vfio_pfn(struct vfio_dma *dma,
+					       unsigned long iova)
+{
+	struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova);
+
+	if (vpfn)
+		atomic_inc(&vpfn->ref_count);
+	return vpfn;
+}
+
+static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn)
+{
+	int ret = 0;
+
+	if (atomic_dec_and_test(&vpfn->ref_count)) {
+		ret = put_pfn(vpfn->pfn, dma->prot);
+		vfio_remove_from_pfn_list(dma, vpfn);
+	}
+	return ret;
+}
+
 struct vwork {
 	struct mm_struct	*mm;
 	long			npage;
@@ -150,17 +264,22 @@ static void vfio_lock_acct_bg(struct work_struct *work)
 	kfree(vwork);
 }
 
-static void vfio_lock_acct(long npage)
+static void vfio_lock_acct(struct task_struct *task, long npage)
 {
 	struct vwork *vwork;
 	struct mm_struct *mm;
 
-	if (!current->mm || !npage)
+	if (!npage)
+		return;
+
+	mm = get_task_mm(task);
+	if (!mm)
 		return; /* process exited or nothing to do */
 
-	if (down_write_trylock(&current->mm->mmap_sem)) {
-		current->mm->locked_vm += npage;
-		up_write(&current->mm->mmap_sem);
+	if (down_write_trylock(&mm->mmap_sem)) {
+		mm->locked_vm += npage;
+		up_write(&mm->mmap_sem);
+		mmput(mm);
 		return;
 	}
 
@@ -170,11 +289,8 @@ static void vfio_lock_acct(long npage)
 	 * wouldn't need this silliness
 	 */
 	vwork = kmalloc(sizeof(struct vwork), GFP_KERNEL);
-	if (!vwork)
-		return;
-	mm = get_task_mm(current);
-	if (!mm) {
-		kfree(vwork);
+	if (!vwork) {
+		mmput(mm);
 		return;
 	}
 	INIT_WORK(&vwork->work, vfio_lock_acct_bg);
@@ -228,20 +344,36 @@ static int put_pfn(unsigned long pfn, int prot)
 	return 0;
 }
 
-static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
+static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
+			 int prot, unsigned long *pfn)
 {
 	struct page *page[1];
 	struct vm_area_struct *vma;
-	int ret = -EFAULT;
+	int ret;
+
+	if (mm == current->mm) {
+		ret = get_user_pages_fast(vaddr, 1, !!(prot & IOMMU_WRITE),
+					  page);
+	} else {
+		unsigned int flags = 0;
+
+		if (prot & IOMMU_WRITE)
+			flags |= FOLL_WRITE;
+
+		down_read(&mm->mmap_sem);
+		ret = get_user_pages_remote(NULL, mm, vaddr, 1, flags, page,
+					    NULL, NULL);
+		up_read(&mm->mmap_sem);
+	}
 
-	if (get_user_pages_fast(vaddr, 1, !!(prot & IOMMU_WRITE), page) == 1) {
+	if (ret == 1) {
 		*pfn = page_to_pfn(page[0]);
 		return 0;
 	}
 
-	down_read(&current->mm->mmap_sem);
+	down_read(&mm->mmap_sem);
 
-	vma = find_vma_intersection(current->mm, vaddr, vaddr + 1);
+	vma = find_vma_intersection(mm, vaddr, vaddr + 1);
 
 	if (vma && vma->vm_flags & VM_PFNMAP) {
 		*pfn = ((vaddr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
@@ -249,8 +381,7 @@ static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
 			ret = 0;
 	}
 
-	up_read(&current->mm->mmap_sem);
-
+	up_read(&mm->mmap_sem);
 	return ret;
 }
 
@@ -259,88 +390,299 @@ static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn)
  * the iommu can only map chunks of consecutive pfns anyway, so get the
  * first page and all consecutive pages with the same locking.
  */
-static long vfio_pin_pages(unsigned long vaddr, long npage,
-			   int prot, unsigned long *pfn_base)
+static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
+				  long npage, unsigned long *pfn_base)
 {
-	unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-	bool lock_cap = capable(CAP_IPC_LOCK);
-	long ret, i;
+	unsigned long limit;
+	bool lock_cap = ns_capable(task_active_pid_ns(dma->task)->user_ns,
+				   CAP_IPC_LOCK);
+	struct mm_struct *mm;
+	long ret, i = 0, lock_acct = 0;
 	bool rsvd;
+	dma_addr_t iova = vaddr - dma->vaddr + dma->iova;
 
-	if (!current->mm)
+	mm = get_task_mm(dma->task);
+	if (!mm)
 		return -ENODEV;
 
-	ret = vaddr_get_pfn(vaddr, prot, pfn_base);
+	ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base);
 	if (ret)
-		return ret;
+		goto pin_pg_remote_exit;
 
 	rsvd = is_invalid_reserved_pfn(*pfn_base);
+	limit = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 
-	if (!rsvd && !lock_cap && current->mm->locked_vm + 1 > limit) {
-		put_pfn(*pfn_base, prot);
-		pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
-			limit << PAGE_SHIFT);
-		return -ENOMEM;
+	/*
+	 * Reserved pages aren't counted against the user, externally pinned
+	 * pages are already counted against the user.
+	 */
+	if (!rsvd && !vfio_find_vpfn(dma, iova)) {
+		if (!lock_cap && mm->locked_vm + 1 > limit) {
+			put_pfn(*pfn_base, dma->prot);
+			pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__,
+					limit << PAGE_SHIFT);
+			ret = -ENOMEM;
+			goto pin_pg_remote_exit;
+		}
+		lock_acct++;
 	}
 
-	if (unlikely(disable_hugepages)) {
-		if (!rsvd)
-			vfio_lock_acct(1);
-		return 1;
-	}
+	i++;
+	if (likely(!disable_hugepages)) {
+		/* Lock all the consecutive pages from pfn_base */
+		for (vaddr += PAGE_SIZE, iova += PAGE_SIZE; i < npage;
+		     i++, vaddr += PAGE_SIZE, iova += PAGE_SIZE) {
+			unsigned long pfn = 0;
 
-	/* Lock all the consecutive pages from pfn_base */
-	for (i = 1, vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) {
-		unsigned long pfn = 0;
+			ret = vaddr_get_pfn(mm, vaddr, dma->prot, &pfn);
+			if (ret)
+				break;
 
-		ret = vaddr_get_pfn(vaddr, prot, &pfn);
-		if (ret)
-			break;
+			if (pfn != *pfn_base + i ||
+			    rsvd != is_invalid_reserved_pfn(pfn)) {
+				put_pfn(pfn, dma->prot);
+				break;
+			}
 
-		if (pfn != *pfn_base + i ||
-		    rsvd != is_invalid_reserved_pfn(pfn)) {
-			put_pfn(pfn, prot);
-			break;
+			if (!rsvd && !vfio_find_vpfn(dma, iova)) {
+				if (!lock_cap &&
+				    mm->locked_vm + lock_acct + 1 > limit) {
+					put_pfn(pfn, dma->prot);
+					pr_warn("%s: RLIMIT_MEMLOCK (%ld) "
+						"exceeded\n", __func__,
+						limit << PAGE_SHIFT);
+					break;
+				}
+				lock_acct++;
+			}
 		}
+	}
 
-		if (!rsvd && !lock_cap &&
-		    current->mm->locked_vm + i + 1 > limit) {
-			put_pfn(pfn, prot);
-			pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n",
-				__func__, limit << PAGE_SHIFT);
-			break;
+	vfio_lock_acct(dma->task, lock_acct);
+	ret = i;
+
+pin_pg_remote_exit:
+	mmput(mm);
+	return ret;
+}
+
+static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
+				    unsigned long pfn, long npage,
+				    bool do_accounting)
+{
+	long unlocked = 0, locked = 0;
+	long i;
+
+	for (i = 0; i < npage; i++) {
+		if (put_pfn(pfn++, dma->prot)) {
+			unlocked++;
+			if (vfio_find_vpfn(dma, iova + (i << PAGE_SHIFT)))
+				locked++;
 		}
 	}
 
-	if (!rsvd)
-		vfio_lock_acct(i);
+	if (do_accounting)
+		vfio_lock_acct(dma->task, locked - unlocked);
 
-	return i;
+	return unlocked;
 }
 
-static long vfio_unpin_pages(unsigned long pfn, long npage,
-			     int prot, bool do_accounting)
+static int vfio_pin_page_external(struct vfio_dma *dma, unsigned long vaddr,
+				  unsigned long *pfn_base, bool do_accounting)
 {
-	unsigned long unlocked = 0;
-	long i;
+	unsigned long limit;
+	bool lock_cap = ns_capable(task_active_pid_ns(dma->task)->user_ns,
+				   CAP_IPC_LOCK);
+	struct mm_struct *mm;
+	int ret;
+	bool rsvd;
 
-	for (i = 0; i < npage; i++)
-		unlocked += put_pfn(pfn++, prot);
+	mm = get_task_mm(dma->task);
+	if (!mm)
+		return -ENODEV;
+
+	ret = vaddr_get_pfn(mm, vaddr, dma->prot, pfn_base);
+	if (ret)
+		goto pin_page_exit;
+
+	rsvd = is_invalid_reserved_pfn(*pfn_base);
+	limit = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+
+	if (!rsvd && !lock_cap && mm->locked_vm + 1 > limit) {
+		put_pfn(*pfn_base, dma->prot);
+		pr_warn("%s: Task %s (%d) RLIMIT_MEMLOCK (%ld) exceeded\n",
+			__func__, dma->task->comm, task_pid_nr(dma->task),
+			limit << PAGE_SHIFT);
+		ret = -ENOMEM;
+		goto pin_page_exit;
+	}
+
+	if (!rsvd && do_accounting)
+		vfio_lock_acct(dma->task, 1);
+	ret = 1;
+
+pin_page_exit:
+	mmput(mm);
+	return ret;
+}
+
+static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova,
+				    bool do_accounting)
+{
+	int unlocked;
+	struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova);
+
+	if (!vpfn)
+		return 0;
+
+	unlocked = vfio_iova_put_vfio_pfn(dma, vpfn);
 
 	if (do_accounting)
-		vfio_lock_acct(-unlocked);
+		vfio_lock_acct(dma->task, -unlocked);
 
 	return unlocked;
 }
 
-static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
+static int vfio_iommu_type1_pin_pages(void *iommu_data,
+				      unsigned long *user_pfn,
+				      int npage, int prot,
+				      unsigned long *phys_pfn)
+{
+	struct vfio_iommu *iommu = iommu_data;
+	int i, j, ret;
+	unsigned long remote_vaddr;
+	struct vfio_dma *dma;
+	bool do_accounting;
+
+	if (!iommu || !user_pfn || !phys_pfn)
+		return -EINVAL;
+
+	/* Supported for v2 version only */
+	if (!iommu->v2)
+		return -EACCES;
+
+	mutex_lock(&iommu->lock);
+
+	/* Fail if notifier list is empty */
+	if ((!iommu->external_domain) || (!iommu->notifier.head)) {
+		ret = -EINVAL;
+		goto pin_done;
+	}
+
+	/*
+	 * If iommu capable domain exist in the container then all pages are
+	 * already pinned and accounted. Accouting should be done if there is no
+	 * iommu capable domain in the container.
+	 */
+	do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
+
+	for (i = 0; i < npage; i++) {
+		dma_addr_t iova;
+		struct vfio_pfn *vpfn;
+
+		iova = user_pfn[i] << PAGE_SHIFT;
+		dma = vfio_find_dma(iommu, iova, PAGE_SIZE);
+		if (!dma) {
+			ret = -EINVAL;
+			goto pin_unwind;
+		}
+
+		if ((dma->prot & prot) != prot) {
+			ret = -EPERM;
+			goto pin_unwind;
+		}
+
+		vpfn = vfio_iova_get_vfio_pfn(dma, iova);
+		if (vpfn) {
+			phys_pfn[i] = vpfn->pfn;
+			continue;
+		}
+
+		remote_vaddr = dma->vaddr + iova - dma->iova;
+		ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn[i],
+					     do_accounting);
+		if (ret <= 0) {
+			WARN_ON(!ret);
+			goto pin_unwind;
+		}
+
+		ret = vfio_add_to_pfn_list(dma, iova, phys_pfn[i]);
+		if (ret) {
+			vfio_unpin_page_external(dma, iova, do_accounting);
+			goto pin_unwind;
+		}
+	}
+
+	ret = i;
+	goto pin_done;
+
+pin_unwind:
+	phys_pfn[i] = 0;
+	for (j = 0; j < i; j++) {
+		dma_addr_t iova;
+
+		iova = user_pfn[j] << PAGE_SHIFT;
+		dma = vfio_find_dma(iommu, iova, PAGE_SIZE);
+		vfio_unpin_page_external(dma, iova, do_accounting);
+		phys_pfn[j] = 0;
+	}
+pin_done:
+	mutex_unlock(&iommu->lock);
+	return ret;
+}
+
+static int vfio_iommu_type1_unpin_pages(void *iommu_data,
+					unsigned long *user_pfn,
+					int npage)
+{
+	struct vfio_iommu *iommu = iommu_data;
+	bool do_accounting;
+	int i;
+
+	if (!iommu || !user_pfn)
+		return -EINVAL;
+
+	/* Supported for v2 version only */
+	if (!iommu->v2)
+		return -EACCES;
+
+	mutex_lock(&iommu->lock);
+
+	if (!iommu->external_domain) {
+		mutex_unlock(&iommu->lock);
+		return -EINVAL;
+	}
+
+	do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
+	for (i = 0; i < npage; i++) {
+		struct vfio_dma *dma;
+		dma_addr_t iova;
+
+		iova = user_pfn[i] << PAGE_SHIFT;
+		dma = vfio_find_dma(iommu, iova, PAGE_SIZE);
+		if (!dma)
+			goto unpin_exit;
+		vfio_unpin_page_external(dma, iova, do_accounting);
+	}
+
+unpin_exit:
+	mutex_unlock(&iommu->lock);
+	return i > npage ? npage : (i > 0 ? i : -EINVAL);
+}
+
+static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma,
+			     bool do_accounting)
 {
 	dma_addr_t iova = dma->iova, end = dma->iova + dma->size;
 	struct vfio_domain *domain, *d;
 	long unlocked = 0;
 
 	if (!dma->size)
-		return;
+		return 0;
+
+	if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu))
+		return 0;
+
 	/*
 	 * We use the IOMMU to track the physical addresses, otherwise we'd
 	 * need a much more complicated tracking system.  Unfortunately that
@@ -382,21 +724,28 @@ static void vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma)
 		if (WARN_ON(!unmapped))
 			break;
 
-		unlocked += vfio_unpin_pages(phys >> PAGE_SHIFT,
-					     unmapped >> PAGE_SHIFT,
-					     dma->prot, false);
+		unlocked += vfio_unpin_pages_remote(dma, iova,
+						    phys >> PAGE_SHIFT,
+						    unmapped >> PAGE_SHIFT,
+						    false);
 		iova += unmapped;
 
 		cond_resched();
 	}
 
-	vfio_lock_acct(-unlocked);
+	dma->iommu_mapped = false;
+	if (do_accounting) {
+		vfio_lock_acct(dma->task, -unlocked);
+		return 0;
+	}
+	return unlocked;
 }
 
 static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
 {
-	vfio_unmap_unpin(iommu, dma);
+	vfio_unmap_unpin(iommu, dma, true);
 	vfio_unlink_dma(iommu, dma);
+	put_task_struct(dma->task);
 	kfree(dma);
 }
 
@@ -430,9 +779,9 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
 			     struct vfio_iommu_type1_dma_unmap *unmap)
 {
 	uint64_t mask;
-	struct vfio_dma *dma;
+	struct vfio_dma *dma, *dma_last = NULL;
 	size_t unmapped = 0;
-	int ret = 0;
+	int ret = 0, retries = 0;
 
 	mask = ((uint64_t)1 << __ffs(vfio_pgsize_bitmap(iommu))) - 1;
 
@@ -442,7 +791,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
 		return -EINVAL;
 
 	WARN_ON(mask & PAGE_MASK);
-
+again:
 	mutex_lock(&iommu->lock);
 
 	/*
@@ -477,7 +826,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
 	 * mappings within the range.
 	 */
 	if (iommu->v2) {
-		dma = vfio_find_dma(iommu, unmap->iova, 0);
+		dma = vfio_find_dma(iommu, unmap->iova, 1);
 		if (dma && dma->iova != unmap->iova) {
 			ret = -EINVAL;
 			goto unlock;
@@ -492,6 +841,38 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
 	while ((dma = vfio_find_dma(iommu, unmap->iova, unmap->size))) {
 		if (!iommu->v2 && unmap->iova > dma->iova)
 			break;
+		/*
+		 * Task with same address space who mapped this iova range is
+		 * allowed to unmap the iova range.
+		 */
+		if (dma->task->mm != current->mm)
+			break;
+
+		if (!RB_EMPTY_ROOT(&dma->pfn_list)) {
+			struct vfio_iommu_type1_dma_unmap nb_unmap;
+
+			if (dma_last == dma) {
+				BUG_ON(++retries > 10);
+			} else {
+				dma_last = dma;
+				retries = 0;
+			}
+
+			nb_unmap.iova = dma->iova;
+			nb_unmap.size = dma->size;
+
+			/*
+			 * Notify anyone (mdev vendor drivers) to invalidate and
+			 * unmap iovas within the range we're about to unmap.
+			 * Vendor drivers MUST unpin pages in response to an
+			 * invalidation.
+			 */
+			mutex_unlock(&iommu->lock);
+			blocking_notifier_call_chain(&iommu->notifier,
+						    VFIO_IOMMU_NOTIFY_DMA_UNMAP,
+						    &nb_unmap);
+			goto again;
+		}
 		unmapped += dma->size;
 		vfio_remove_dma(iommu, dma);
 	}
@@ -558,17 +939,56 @@ unwind:
 	return ret;
 }
 
+static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma,
+			    size_t map_size)
+{
+	dma_addr_t iova = dma->iova;
+	unsigned long vaddr = dma->vaddr;
+	size_t size = map_size;
+	long npage;
+	unsigned long pfn;
+	int ret = 0;
+
+	while (size) {
+		/* Pin a contiguous chunk of memory */
+		npage = vfio_pin_pages_remote(dma, vaddr + dma->size,
+					      size >> PAGE_SHIFT, &pfn);
+		if (npage <= 0) {
+			WARN_ON(!npage);
+			ret = (int)npage;
+			break;
+		}
+
+		/* Map it! */
+		ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage,
+				     dma->prot);
+		if (ret) {
+			vfio_unpin_pages_remote(dma, iova + dma->size, pfn,
+						npage, true);
+			break;
+		}
+
+		size -= npage << PAGE_SHIFT;
+		dma->size += npage << PAGE_SHIFT;
+	}
+
+	dma->iommu_mapped = true;
+
+	if (ret)
+		vfio_remove_dma(iommu, dma);
+
+	return ret;
+}
+
 static int vfio_dma_do_map(struct vfio_iommu *iommu,
 			   struct vfio_iommu_type1_dma_map *map)
 {
 	dma_addr_t iova = map->iova;
 	unsigned long vaddr = map->vaddr;
 	size_t size = map->size;
-	long npage;
 	int ret = 0, prot = 0;
 	uint64_t mask;
 	struct vfio_dma *dma;
-	unsigned long pfn;
 
 	/* Verify that none of our __u64 fields overflow */
 	if (map->size != size || map->vaddr != vaddr || map->iova != iova)
@@ -594,47 +1014,33 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
 	mutex_lock(&iommu->lock);
 
 	if (vfio_find_dma(iommu, iova, size)) {
-		mutex_unlock(&iommu->lock);
-		return -EEXIST;
+		ret = -EEXIST;
+		goto out_unlock;
 	}
 
 	dma = kzalloc(sizeof(*dma), GFP_KERNEL);
 	if (!dma) {
-		mutex_unlock(&iommu->lock);
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto out_unlock;
 	}
 
 	dma->iova = iova;
 	dma->vaddr = vaddr;
 	dma->prot = prot;
+	get_task_struct(current);
+	dma->task = current;
+	dma->pfn_list = RB_ROOT;
 
 	/* Insert zero-sized and grow as we map chunks of it */
 	vfio_link_dma(iommu, dma);
 
-	while (size) {
-		/* Pin a contiguous chunk of memory */
-		npage = vfio_pin_pages(vaddr + dma->size,
-				       size >> PAGE_SHIFT, prot, &pfn);
-		if (npage <= 0) {
-			WARN_ON(!npage);
-			ret = (int)npage;
-			break;
-		}
-
-		/* Map it! */
-		ret = vfio_iommu_map(iommu, iova + dma->size, pfn, npage, prot);
-		if (ret) {
-			vfio_unpin_pages(pfn, npage, prot, true);
-			break;
-		}
-
-		size -= npage << PAGE_SHIFT;
-		dma->size += npage << PAGE_SHIFT;
-	}
-
-	if (ret)
-		vfio_remove_dma(iommu, dma);
+	/* Don't pin and map if container doesn't contain IOMMU capable domain*/
+	if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu))
+		dma->size = size;
+	else
+		ret = vfio_pin_map_dma(iommu, dma, size);
 
+out_unlock:
 	mutex_unlock(&iommu->lock);
 	return ret;
 }
@@ -662,10 +1068,6 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
 	d = list_first_entry(&iommu->domain_list, struct vfio_domain, next);
 	n = rb_first(&iommu->dma_list);
 
-	/* If there's not a domain, there better not be any mappings */
-	if (WARN_ON(n && !d))
-		return -EINVAL;
-
 	for (; n; n = rb_next(n)) {
 		struct vfio_dma *dma;
 		dma_addr_t iova;
@@ -674,21 +1076,49 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
 		iova = dma->iova;
 
 		while (iova < dma->iova + dma->size) {
-			phys_addr_t phys = iommu_iova_to_phys(d->domain, iova);
+			phys_addr_t phys;
 			size_t size;
 
-			if (WARN_ON(!phys)) {
-				iova += PAGE_SIZE;
-				continue;
+			if (dma->iommu_mapped) {
+				phys_addr_t p;
+				dma_addr_t i;
+
+				phys = iommu_iova_to_phys(d->domain, iova);
+
+				if (WARN_ON(!phys)) {
+					iova += PAGE_SIZE;
+					continue;
+				}
+
+				size = PAGE_SIZE;
+				p = phys + size;
+				i = iova + size;
+				while (i < dma->iova + dma->size &&
+				       p == iommu_iova_to_phys(d->domain, i)) {
+					size += PAGE_SIZE;
+					p += PAGE_SIZE;
+					i += PAGE_SIZE;
+				}
+			} else {
+				unsigned long pfn;
+				unsigned long vaddr = dma->vaddr +
+						     (iova - dma->iova);
+				size_t n = dma->iova + dma->size - iova;
+				long npage;
+
+				npage = vfio_pin_pages_remote(dma, vaddr,
+							      n >> PAGE_SHIFT,
+							      &pfn);
+				if (npage <= 0) {
+					WARN_ON(!npage);
+					ret = (int)npage;
+					return ret;
+				}
+
+				phys = pfn << PAGE_SHIFT;
+				size = npage << PAGE_SHIFT;
 			}
 
-			size = PAGE_SIZE;
-
-			while (iova + size < dma->iova + dma->size &&
-			       phys + size == iommu_iova_to_phys(d->domain,
-								 iova + size))
-				size += PAGE_SIZE;
-
 			ret = iommu_map(domain->domain, iova, phys,
 					size, dma->prot | domain->prot);
 			if (ret)
@@ -696,8 +1126,8 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu,
 
 			iova += size;
 		}
+		dma->iommu_mapped = true;
 	}
-
 	return 0;
 }
 
@@ -734,22 +1164,39 @@ static void vfio_test_domain_fgsp(struct vfio_domain *domain)
 	__free_pages(pages, order);
 }
 
+static struct vfio_group *find_iommu_group(struct vfio_domain *domain,
+					   struct iommu_group *iommu_group)
+{
+	struct vfio_group *g;
+
+	list_for_each_entry(g, &domain->group_list, next) {
+		if (g->iommu_group == iommu_group)
+			return g;
+	}
+
+	return NULL;
+}
+
 static int vfio_iommu_type1_attach_group(void *iommu_data,
 					 struct iommu_group *iommu_group)
 {
 	struct vfio_iommu *iommu = iommu_data;
-	struct vfio_group *group, *g;
+	struct vfio_group *group;
 	struct vfio_domain *domain, *d;
-	struct bus_type *bus = NULL;
+	struct bus_type *bus = NULL, *mdev_bus;
 	int ret;
 
 	mutex_lock(&iommu->lock);
 
 	list_for_each_entry(d, &iommu->domain_list, next) {
-		list_for_each_entry(g, &d->group_list, next) {
-			if (g->iommu_group != iommu_group)
-				continue;
+		if (find_iommu_group(d, iommu_group)) {
+			mutex_unlock(&iommu->lock);
+			return -EINVAL;
+		}
+	}
 
+	if (iommu->external_domain) {
+		if (find_iommu_group(iommu->external_domain, iommu_group)) {
 			mutex_unlock(&iommu->lock);
 			return -EINVAL;
 		}
@@ -769,6 +1216,25 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
 	if (ret)
 		goto out_free;
 
+	mdev_bus = symbol_get(mdev_bus_type);
+
+	if (mdev_bus) {
+		if ((bus == mdev_bus) && !iommu_present(bus)) {
+			symbol_put(mdev_bus_type);
+			if (!iommu->external_domain) {
+				INIT_LIST_HEAD(&domain->group_list);
+				iommu->external_domain = domain;
+			} else
+				kfree(domain);
+
+			list_add(&group->next,
+				 &iommu->external_domain->group_list);
+			mutex_unlock(&iommu->lock);
+			return 0;
+		}
+		symbol_put(mdev_bus_type);
+	}
+
 	domain->domain = iommu_domain_alloc(bus);
 	if (!domain->domain) {
 		ret = -EIO;
@@ -859,6 +1325,46 @@ static void vfio_iommu_unmap_unpin_all(struct vfio_iommu *iommu)
 		vfio_remove_dma(iommu, rb_entry(node, struct vfio_dma, node));
 }
 
+static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu)
+{
+	struct rb_node *n, *p;
+
+	n = rb_first(&iommu->dma_list);
+	for (; n; n = rb_next(n)) {
+		struct vfio_dma *dma;
+		long locked = 0, unlocked = 0;
+
+		dma = rb_entry(n, struct vfio_dma, node);
+		unlocked += vfio_unmap_unpin(iommu, dma, false);
+		p = rb_first(&dma->pfn_list);
+		for (; p; p = rb_next(p)) {
+			struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn,
+							 node);
+
+			if (!is_invalid_reserved_pfn(vpfn->pfn))
+				locked++;
+		}
+		vfio_lock_acct(dma->task, locked - unlocked);
+	}
+}
+
+static void vfio_sanity_check_pfn_list(struct vfio_iommu *iommu)
+{
+	struct rb_node *n;
+
+	n = rb_first(&iommu->dma_list);
+	for (; n; n = rb_next(n)) {
+		struct vfio_dma *dma;
+
+		dma = rb_entry(n, struct vfio_dma, node);
+
+		if (WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)))
+			break;
+	}
+	/* mdev vendor driver must unregister notifier */
+	WARN_ON(iommu->notifier.head);
+}
+
 static void vfio_iommu_type1_detach_group(void *iommu_data,
 					  struct iommu_group *iommu_group)
 {
@@ -868,31 +1374,55 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
 
 	mutex_lock(&iommu->lock);
 
-	list_for_each_entry(domain, &iommu->domain_list, next) {
-		list_for_each_entry(group, &domain->group_list, next) {
-			if (group->iommu_group != iommu_group)
-				continue;
-
-			iommu_detach_group(domain->domain, iommu_group);
+	if (iommu->external_domain) {
+		group = find_iommu_group(iommu->external_domain, iommu_group);
+		if (group) {
 			list_del(&group->next);
 			kfree(group);
-			/*
-			 * Group ownership provides privilege, if the group
-			 * list is empty, the domain goes away.  If it's the
-			 * last domain, then all the mappings go away too.
-			 */
-			if (list_empty(&domain->group_list)) {
-				if (list_is_singular(&iommu->domain_list))
+
+			if (list_empty(&iommu->external_domain->group_list)) {
+				vfio_sanity_check_pfn_list(iommu);
+
+				if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu))
 					vfio_iommu_unmap_unpin_all(iommu);
-				iommu_domain_free(domain->domain);
-				list_del(&domain->next);
-				kfree(domain);
+
+				kfree(iommu->external_domain);
+				iommu->external_domain = NULL;
+			}
+			goto detach_group_done;
+		}
+	}
+
+	list_for_each_entry(domain, &iommu->domain_list, next) {
+		group = find_iommu_group(domain, iommu_group);
+		if (!group)
+			continue;
+
+		iommu_detach_group(domain->domain, iommu_group);
+		list_del(&group->next);
+		kfree(group);
+		/*
+		 * Group ownership provides privilege, if the group list is
+		 * empty, the domain goes away. If it's the last domain with
+		 * iommu and external domain doesn't exist, then all the
+		 * mappings go away too. If it's the last domain with iommu and
+		 * external domain exist, update accounting
+		 */
+		if (list_empty(&domain->group_list)) {
+			if (list_is_singular(&iommu->domain_list)) {
+				if (!iommu->external_domain)
+					vfio_iommu_unmap_unpin_all(iommu);
+				else
+					vfio_iommu_unmap_unpin_reaccount(iommu);
 			}
-			goto done;
+			iommu_domain_free(domain->domain);
+			list_del(&domain->next);
+			kfree(domain);
 		}
+		break;
 	}
 
-done:
+detach_group_done:
 	mutex_unlock(&iommu->lock);
 }
 
@@ -920,31 +1450,46 @@ static void *vfio_iommu_type1_open(unsigned long arg)
 	INIT_LIST_HEAD(&iommu->domain_list);
 	iommu->dma_list = RB_ROOT;
 	mutex_init(&iommu->lock);
+	BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier);
 
 	return iommu;
 }
 
+static void vfio_release_domain(struct vfio_domain *domain, bool external)
+{
+	struct vfio_group *group, *group_tmp;
+
+	list_for_each_entry_safe(group, group_tmp,
+				 &domain->group_list, next) {
+		if (!external)
+			iommu_detach_group(domain->domain, group->iommu_group);
+		list_del(&group->next);
+		kfree(group);
+	}
+
+	if (!external)
+		iommu_domain_free(domain->domain);
+}
+
 static void vfio_iommu_type1_release(void *iommu_data)
 {
 	struct vfio_iommu *iommu = iommu_data;
 	struct vfio_domain *domain, *domain_tmp;
-	struct vfio_group *group, *group_tmp;
+
+	if (iommu->external_domain) {
+		vfio_release_domain(iommu->external_domain, true);
+		vfio_sanity_check_pfn_list(iommu);
+		kfree(iommu->external_domain);
+	}
 
 	vfio_iommu_unmap_unpin_all(iommu);
 
 	list_for_each_entry_safe(domain, domain_tmp,
 				 &iommu->domain_list, next) {
-		list_for_each_entry_safe(group, group_tmp,
-					 &domain->group_list, next) {
-			iommu_detach_group(domain->domain, group->iommu_group);
-			list_del(&group->next);
-			kfree(group);
-		}
-		iommu_domain_free(domain->domain);
+		vfio_release_domain(domain, false);
 		list_del(&domain->next);
 		kfree(domain);
 	}
-
 	kfree(iommu);
 }
 
@@ -1040,14 +1585,42 @@ static long vfio_iommu_type1_ioctl(void *iommu_data,
 	return -ENOTTY;
 }
 
+static int vfio_iommu_type1_register_notifier(void *iommu_data,
+					      unsigned long *events,
+					      struct notifier_block *nb)
+{
+	struct vfio_iommu *iommu = iommu_data;
+
+	/* clear known events */
+	*events &= ~VFIO_IOMMU_NOTIFY_DMA_UNMAP;
+
+	/* refuse to register if still events remaining */
+	if (*events)
+		return -EINVAL;
+
+	return blocking_notifier_chain_register(&iommu->notifier, nb);
+}
+
+static int vfio_iommu_type1_unregister_notifier(void *iommu_data,
+						struct notifier_block *nb)
+{
+	struct vfio_iommu *iommu = iommu_data;
+
+	return blocking_notifier_chain_unregister(&iommu->notifier, nb);
+}
+
 static const struct vfio_iommu_driver_ops vfio_iommu_driver_ops_type1 = {
-	.name		= "vfio-iommu-type1",
-	.owner		= THIS_MODULE,
-	.open		= vfio_iommu_type1_open,
-	.release	= vfio_iommu_type1_release,
-	.ioctl		= vfio_iommu_type1_ioctl,
-	.attach_group	= vfio_iommu_type1_attach_group,
-	.detach_group	= vfio_iommu_type1_detach_group,
+	.name			= "vfio-iommu-type1",
+	.owner			= THIS_MODULE,
+	.open			= vfio_iommu_type1_open,
+	.release		= vfio_iommu_type1_release,
+	.ioctl			= vfio_iommu_type1_ioctl,
+	.attach_group		= vfio_iommu_type1_attach_group,
+	.detach_group		= vfio_iommu_type1_detach_group,
+	.pin_pages		= vfio_iommu_type1_pin_pages,
+	.unpin_pages		= vfio_iommu_type1_unpin_pages,
+	.register_notifier	= vfio_iommu_type1_register_notifier,
+	.unregister_notifier	= vfio_iommu_type1_unregister_notifier,
 };
 
 static int __init vfio_iommu_type1_init(void)
diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c
index 6e29d053843d..e2be447752c2 100644
--- a/drivers/vhost/scsi.c
+++ b/drivers/vhost/scsi.c
@@ -1749,7 +1749,6 @@ out:
 static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg,
 				const char *name)
 {
-	struct se_portal_group *se_tpg;
 	struct vhost_scsi_nexus *tv_nexus;
 
 	mutex_lock(&tpg->tv_tpg_mutex);
@@ -1758,7 +1757,6 @@ static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg,
 		pr_debug("tpg->tpg_nexus already exists\n");
 		return -EEXIST;
 	}
-	se_tpg = &tpg->se_tpg;
 
 	tv_nexus = kzalloc(sizeof(struct vhost_scsi_nexus), GFP_KERNEL);
 	if (!tv_nexus) {
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 266354390c8f..c0f81e8cb50c 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -49,7 +49,7 @@ enum {
 
 INTERVAL_TREE_DEFINE(struct vhost_umem_node,
 		     rb, __u64, __subtree_last,
-		     START, LAST, , vhost_umem_interval_tree);
+		     START, LAST, static inline, vhost_umem_interval_tree);
 
 #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
@@ -290,6 +290,7 @@ static void vhost_vq_reset(struct vhost_dev *dev,
 	vq->avail = NULL;
 	vq->used = NULL;
 	vq->last_avail_idx = 0;
+	vq->last_used_event = 0;
 	vq->avail_idx = 0;
 	vq->last_used_idx = 0;
 	vq->signalled_used = 0;
@@ -719,7 +720,7 @@ static int memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem,
 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
 			  struct iovec iov[], int iov_size, int access);
 
-static int vhost_copy_to_user(struct vhost_virtqueue *vq, void *to,
+static int vhost_copy_to_user(struct vhost_virtqueue *vq, void __user *to,
 			      const void *from, unsigned size)
 {
 	int ret;
@@ -749,7 +750,7 @@ out:
 }
 
 static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to,
-				void *from, unsigned size)
+				void __user *from, unsigned size)
 {
 	int ret;
 
@@ -783,7 +784,7 @@ out:
 }
 
 static void __user *__vhost_get_user(struct vhost_virtqueue *vq,
-				     void *addr, unsigned size)
+				     void __user *addr, unsigned size)
 {
 	int ret;
 
@@ -934,8 +935,8 @@ static int umem_access_ok(u64 uaddr, u64 size, int access)
 	return 0;
 }
 
-int vhost_process_iotlb_msg(struct vhost_dev *dev,
-			    struct vhost_iotlb_msg *msg)
+static int vhost_process_iotlb_msg(struct vhost_dev *dev,
+				   struct vhost_iotlb_msg *msg)
 {
 	int ret = 0;
 
@@ -1324,7 +1325,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp)
 			r = -EINVAL;
 			break;
 		}
-		vq->last_avail_idx = s.num;
+		vq->last_avail_idx = vq->last_used_event = s.num;
 		/* Forget the cached index value. */
 		vq->avail_idx = vq->last_avail_idx;
 		break;
@@ -2159,10 +2160,6 @@ static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
 	__u16 old, new;
 	__virtio16 event;
 	bool v;
-	/* Flush out used index updates. This is paired
-	 * with the barrier that the Guest executes when enabling
-	 * interrupts. */
-	smp_mb();
 
 	if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) &&
 	    unlikely(vq->avail_idx == vq->last_avail_idx))
@@ -2170,6 +2167,10 @@ static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
 
 	if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
 		__virtio16 flags;
+		/* Flush out used index updates. This is paired
+		 * with the barrier that the Guest executes when enabling
+		 * interrupts. */
+		smp_mb();
 		if (vhost_get_user(vq, flags, &vq->avail->flags)) {
 			vq_err(vq, "Failed to get flags");
 			return true;
@@ -2184,11 +2185,26 @@ static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
 	if (unlikely(!v))
 		return true;
 
+	/* We're sure if the following conditions are met, there's no
+	 * need to notify guest:
+	 * 1) cached used event is ahead of new
+	 * 2) old to new updating does not cross cached used event. */
+	if (vring_need_event(vq->last_used_event, new + vq->num, new) &&
+	    !vring_need_event(vq->last_used_event, new, old))
+		return false;
+
+	/* Flush out used index updates. This is paired
+	 * with the barrier that the Guest executes when enabling
+	 * interrupts. */
+	smp_mb();
+
 	if (vhost_get_user(vq, event, vhost_used_event(vq))) {
 		vq_err(vq, "Failed to get used event idx");
 		return true;
 	}
-	return vring_need_event(vhost16_to_cpu(vq, event), new, old);
+	vq->last_used_event = vhost16_to_cpu(vq, event);
+
+	return vring_need_event(vq->last_used_event, new, old);
 }
 
 /* This actually signals the guest, using eventfd. */
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index 78f3c5fc02e4..a9cbbb148f46 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -107,6 +107,9 @@ struct vhost_virtqueue {
 	/* Last index we used. */
 	u16 last_used_idx;
 
+	/* Last used evet we've seen */
+	u16 last_used_event;
+
 	/* Used flags */
 	u16 used_flags;
 
diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c
index 3bb02c60a2f5..bb8971f2a634 100644
--- a/drivers/vhost/vringh.c
+++ b/drivers/vhost/vringh.c
@@ -3,6 +3,7 @@
  *
  * Since these may be in userspace, we use (inline) accessors.
  */
+#include <linux/compiler.h>
 #include <linux/module.h>
 #include <linux/vringh.h>
 #include <linux/virtio_ring.h>
@@ -820,13 +821,13 @@ EXPORT_SYMBOL(vringh_need_notify_user);
 static inline int getu16_kern(const struct vringh *vrh,
 			      u16 *val, const __virtio16 *p)
 {
-	*val = vringh16_to_cpu(vrh, ACCESS_ONCE(*p));
+	*val = vringh16_to_cpu(vrh, READ_ONCE(*p));
 	return 0;
 }
 
 static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val)
 {
-	ACCESS_ONCE(*p) = cpu_to_vringh16(vrh, val);
+	WRITE_ONCE(*p, cpu_to_vringh16(vrh, val));
 	return 0;
 }
 
diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c
index e6b70966c19d..bbbf588540ed 100644
--- a/drivers/vhost/vsock.c
+++ b/drivers/vhost/vsock.c
@@ -50,11 +50,10 @@ static u32 vhost_transport_get_local_cid(void)
 	return VHOST_VSOCK_DEFAULT_HOST_CID;
 }
 
-static struct vhost_vsock *vhost_vsock_get(u32 guest_cid)
+static struct vhost_vsock *__vhost_vsock_get(u32 guest_cid)
 {
 	struct vhost_vsock *vsock;
 
-	spin_lock_bh(&vhost_vsock_lock);
 	list_for_each_entry(vsock, &vhost_vsock_list, list) {
 		u32 other_cid = vsock->guest_cid;
 
@@ -63,15 +62,24 @@ static struct vhost_vsock *vhost_vsock_get(u32 guest_cid)
 			continue;
 
 		if (other_cid == guest_cid) {
-			spin_unlock_bh(&vhost_vsock_lock);
 			return vsock;
 		}
 	}
-	spin_unlock_bh(&vhost_vsock_lock);
 
 	return NULL;
 }
 
+static struct vhost_vsock *vhost_vsock_get(u32 guest_cid)
+{
+	struct vhost_vsock *vsock;
+
+	spin_lock_bh(&vhost_vsock_lock);
+	vsock = __vhost_vsock_get(guest_cid);
+	spin_unlock_bh(&vhost_vsock_lock);
+
+	return vsock;
+}
+
 static void
 vhost_transport_do_send_pkt(struct vhost_vsock *vsock,
 			    struct vhost_virtqueue *vq)
@@ -559,11 +567,12 @@ static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid)
 		return -EINVAL;
 
 	/* Refuse if CID is already in use */
-	other = vhost_vsock_get(guest_cid);
-	if (other && other != vsock)
-		return -EADDRINUSE;
-
 	spin_lock_bh(&vhost_vsock_lock);
+	other = __vhost_vsock_get(guest_cid);
+	if (other && other != vsock) {
+		spin_unlock_bh(&vhost_vsock_lock);
+		return -EADDRINUSE;
+	}
 	vsock->guest_cid = guest_cid;
 	spin_unlock_bh(&vhost_vsock_lock);
 
diff --git a/drivers/video/console/fbcon.c b/drivers/video/console/fbcon.c
index b87f5cfdaea5..a44f5627b82a 100644
--- a/drivers/video/console/fbcon.c
+++ b/drivers/video/console/fbcon.c
@@ -164,8 +164,6 @@ static void fbcon_putcs(struct vc_data *vc, const unsigned short *s,
 			int count, int ypos, int xpos);
 static void fbcon_clear_margins(struct vc_data *vc, int bottom_only);
 static void fbcon_cursor(struct vc_data *vc, int mode);
-static int fbcon_scroll(struct vc_data *vc, int t, int b, int dir,
-			int count);
 static void fbcon_bmove(struct vc_data *vc, int sy, int sx, int dy, int dx,
 			int height, int width);
 static int fbcon_switch(struct vc_data *vc);
@@ -1795,15 +1793,15 @@ static inline void fbcon_softback_note(struct vc_data *vc, int t,
 	softback_curr = softback_in;
 }
 
-static int fbcon_scroll(struct vc_data *vc, int t, int b, int dir,
-			int count)
+static bool fbcon_scroll(struct vc_data *vc, unsigned int t, unsigned int b,
+		enum con_scroll dir, unsigned int count)
 {
 	struct fb_info *info = registered_fb[con2fb_map[vc->vc_num]];
 	struct display *p = &fb_display[vc->vc_num];
 	int scroll_partial = info->flags & FBINFO_PARTIAL_PAN_OK;
 
 	if (fbcon_is_inactive(vc, info))
-		return -EINVAL;
+		return true;
 
 	fbcon_cursor(vc, CM_ERASE);
 
@@ -1831,7 +1829,7 @@ static int fbcon_scroll(struct vc_data *vc, int t, int b, int dir,
 							(b - count)),
 				    vc->vc_video_erase_char,
 				    vc->vc_size_row * count);
-			return 1;
+			return true;
 			break;
 
 		case SCROLL_WRAP_MOVE:
@@ -1903,7 +1901,7 @@ static int fbcon_scroll(struct vc_data *vc, int t, int b, int dir,
 							(b - count)),
 				    vc->vc_video_erase_char,
 				    vc->vc_size_row * count);
-			return 1;
+			return true;
 		}
 		break;
 
@@ -1922,7 +1920,7 @@ static int fbcon_scroll(struct vc_data *vc, int t, int b, int dir,
 							t),
 				    vc->vc_video_erase_char,
 				    vc->vc_size_row * count);
-			return 1;
+			return true;
 			break;
 
 		case SCROLL_WRAP_MOVE:
@@ -1992,10 +1990,10 @@ static int fbcon_scroll(struct vc_data *vc, int t, int b, int dir,
 							t),
 				    vc->vc_video_erase_char,
 				    vc->vc_size_row * count);
-			return 1;
+			return true;
 		}
 	}
-	return 0;
+	return false;
 }
 
 
diff --git a/drivers/video/console/mdacon.c b/drivers/video/console/mdacon.c
index bacbb044d77c..ec192a1bf297 100644
--- a/drivers/video/console/mdacon.c
+++ b/drivers/video/console/mdacon.c
@@ -488,12 +488,13 @@ static void mdacon_cursor(struct vc_data *c, int mode)
 	}
 }
 
-static int mdacon_scroll(struct vc_data *c, int t, int b, int dir, int lines)
+static bool mdacon_scroll(struct vc_data *c, unsigned int t, unsigned int b,
+		enum con_scroll dir, unsigned int lines)
 {
 	u16 eattr = mda_convert_attr(c->vc_video_erase_char);
 
 	if (!lines)
-		return 0;
+		return false;
 
 	if (lines > c->vc_rows)   /* maximum realistic size */
 		lines = c->vc_rows;
@@ -514,7 +515,7 @@ static int mdacon_scroll(struct vc_data *c, int t, int b, int dir, int lines)
 		break;
 	}
 
-	return 0;
+	return false;
 }
 
 
diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c
index e3b9521e4ec3..1e11614322fe 100644
--- a/drivers/video/console/newport_con.c
+++ b/drivers/video/console/newport_con.c
@@ -574,8 +574,8 @@ static int newport_font_set(struct vc_data *vc, struct console_font *font, unsig
 	return newport_set_font(vc->vc_num, font);
 }
 
-static int newport_scroll(struct vc_data *vc, int t, int b, int dir,
-			  int lines)
+static bool newport_scroll(struct vc_data *vc, unsigned int t, unsigned int b,
+		enum con_scroll dir, unsigned int lines)
 {
 	int count, x, y;
 	unsigned short *s, *d;
@@ -595,7 +595,7 @@ static int newport_scroll(struct vc_data *vc, int t, int b, int dir,
 					    (vc->vc_color & 0xf0) >> 4);
 		}
 		npregs->cset.topscan = (topscan - 1) & 0x3ff;
-		return 0;
+		return false;
 	}
 
 	count = (b - t - lines) * vc->vc_cols;
@@ -670,7 +670,7 @@ static int newport_scroll(struct vc_data *vc, int t, int b, int dir,
 			}
 		}
 	}
-	return 1;
+	return true;
 }
 
 static int newport_dummy(struct vc_data *c)
diff --git a/drivers/video/console/sticon.c b/drivers/video/console/sticon.c
index 3a10ac19598f..79c9bd8d3025 100644
--- a/drivers/video/console/sticon.c
+++ b/drivers/video/console/sticon.c
@@ -153,12 +153,13 @@ static void sticon_cursor(struct vc_data *conp, int mode)
     }
 }
 
-static int sticon_scroll(struct vc_data *conp, int t, int b, int dir, int count)
+static bool sticon_scroll(struct vc_data *conp, unsigned int t,
+		unsigned int b, enum con_scroll dir, unsigned int count)
 {
     struct sti_struct *sti = sticon_sti;
 
     if (vga_is_gfx)
-        return 0;
+        return false;
 
     sticon_cursor(conp, CM_ERASE);
 
@@ -174,7 +175,7 @@ static int sticon_scroll(struct vc_data *conp, int t, int b, int dir, int count)
 	break;
     }
 
-    return 0;
+    return false;
 }
 
 static void sticon_init(struct vc_data *c, int init)
diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c
index 11576611a974..c22a56232b7c 100644
--- a/drivers/video/console/vgacon.c
+++ b/drivers/video/console/vgacon.c
@@ -60,15 +60,6 @@ static struct vgastate vgastate;
 
 #define BLANK 0x0020
 
-#define CAN_LOAD_EGA_FONTS	/* undefine if the user must not do this */
-#define CAN_LOAD_PALETTE	/* undefine if the user must not do this */
-
-/* You really do _NOT_ want to define this, unless you have buggy
- * Trident VGA which will resize cursor when moving it between column
- * 15 & 16. If you define this and your VGA is OK, inverse bug will
- * appear.
- */
-#undef TRIDENT_GLITCH
 #define VGA_FONTWIDTH       8   /* VGA does not support fontwidths != 8 */
 /*
  *  Interface used by the world
@@ -83,14 +74,12 @@ static int vgacon_blank(struct vc_data *c, int blank, int mode_switch);
 static void vgacon_scrolldelta(struct vc_data *c, int lines);
 static int vgacon_set_origin(struct vc_data *c);
 static void vgacon_save_screen(struct vc_data *c);
-static int vgacon_scroll(struct vc_data *c, int t, int b, int dir,
-			 int lines);
 static void vgacon_invert_region(struct vc_data *c, u16 * p, int count);
 static struct uni_pagedir *vgacon_uni_pagedir;
 static int vgacon_refcount;
 
 /* Description of the hardware situation */
-static int		vga_init_done		__read_mostly;
+static bool		vga_init_done;
 static unsigned long	vga_vram_base		__read_mostly;	/* Base of video memory */
 static unsigned long	vga_vram_end		__read_mostly;	/* End of video memory */
 static unsigned int	vga_vram_size		__read_mostly;	/* Size of video memory */
@@ -98,31 +87,31 @@ static u16		vga_video_port_reg	__read_mostly;	/* Video register select port */
 static u16		vga_video_port_val	__read_mostly;	/* Video register value port */
 static unsigned int	vga_video_num_columns;			/* Number of text columns */
 static unsigned int	vga_video_num_lines;			/* Number of text lines */
-static int		vga_can_do_color	__read_mostly;	/* Do we support colors? */
+static bool		vga_can_do_color;			/* Do we support colors? */
 static unsigned int	vga_default_font_height __read_mostly;	/* Height of default screen font */
 static unsigned char	vga_video_type		__read_mostly;	/* Card type */
-static unsigned char	vga_hardscroll_enabled	__read_mostly;
-static unsigned char	vga_hardscroll_user_enable __read_mostly = 1;
-static unsigned char	vga_font_is_default = 1;
+static bool		vga_font_is_default = true;
 static int		vga_vesa_blanked;
-static int 		vga_palette_blanked;
-static int 		vga_is_gfx;
-static int 		vga_512_chars;
+static bool 		vga_palette_blanked;
+static bool 		vga_is_gfx;
+static bool 		vga_512_chars;
 static int 		vga_video_font_height;
 static int 		vga_scan_lines		__read_mostly;
 static unsigned int 	vga_rolled_over;
 
-static int vgacon_text_mode_force;
+static bool vgacon_text_mode_force;
+static bool vga_hardscroll_enabled;
+static bool vga_hardscroll_user_enable = true;
 
 bool vgacon_text_force(void)
 {
-	return vgacon_text_mode_force ? true : false;
+	return vgacon_text_mode_force;
 }
 EXPORT_SYMBOL(vgacon_text_force);
 
 static int __init text_mode(char *str)
 {
-	vgacon_text_mode_force = 1;
+	vgacon_text_mode_force = true;
 	return 1;
 }
 
@@ -136,7 +125,7 @@ static int __init no_scroll(char *str)
 	 * Braille reader made by F.H. Papenmeier (Germany).
 	 * Use the "no-scroll" bootflag.
 	 */
-	vga_hardscroll_user_enable = vga_hardscroll_enabled = 0;
+	vga_hardscroll_user_enable = vga_hardscroll_enabled = false;
 	return 1;
 }
 
@@ -159,18 +148,10 @@ static inline void write_vga(unsigned char reg, unsigned int val)
 	 * handlers, thus the write has to be IRQ-atomic.
 	 */
 	raw_spin_lock_irqsave(&vga_lock, flags);
-
-#ifndef SLOW_VGA
 	v1 = reg + (val & 0xff00);
 	v2 = reg + 1 + ((val << 8) & 0xff00);
 	outw(v1, vga_video_port_reg);
 	outw(v2, vga_video_port_reg);
-#else
-	outb_p(reg, vga_video_port_reg);
-	outb_p(val >> 8, vga_video_port_val);
-	outb_p(reg + 1, vga_video_port_reg);
-	outb_p(val & 0xff, vga_video_port_val);
-#endif
 	raw_spin_unlock_irqrestore(&vga_lock, flags);
 }
 
@@ -334,31 +315,8 @@ static void vgacon_restore_screen(struct vc_data *c)
 
 static void vgacon_scrolldelta(struct vc_data *c, int lines)
 {
-	if (!lines)		/* Turn scrollback off */
-		c->vc_visible_origin = c->vc_origin;
-	else {
-		int margin = c->vc_size_row * 4;
-		int ul, we, p, st;
-
-		if (vga_rolled_over >
-		    (c->vc_scr_end - vga_vram_base) + margin) {
-			ul = c->vc_scr_end - vga_vram_base;
-			we = vga_rolled_over + c->vc_size_row;
-		} else {
-			ul = 0;
-			we = vga_vram_size;
-		}
-		p = (c->vc_visible_origin - vga_vram_base - ul + we) % we +
-		    lines * c->vc_size_row;
-		st = (c->vc_origin - vga_vram_base - ul + we) % we;
-		if (st < 2 * margin)
-			margin = 0;
-		if (p < margin)
-			p = 0;
-		if (p > st - margin)
-			p = st;
-		c->vc_visible_origin = vga_vram_base + (p + ul) % we;
-	}
+	vc_scrolldelta_helper(c, lines, vga_rolled_over, (void *)vga_vram_base,
+			vga_vram_size);
 	vga_set_mem_top(c);
 }
 #endif /* CONFIG_VGACON_SOFT_SCROLLBACK */
@@ -427,7 +385,7 @@ static const char *vgacon_startup(void)
 		}
 	} else {
 		/* If not, it is color. */
-		vga_can_do_color = 1;
+		vga_can_do_color = true;
 		vga_vram_base = 0xb8000;
 		vga_video_port_reg = VGA_CRT_IC;
 		vga_video_port_val = VGA_CRT_DC;
@@ -451,18 +409,6 @@ static const char *vgacon_startup(void)
 				request_resource(&ioport_resource,
 						 &vga_console_resource);
 
-#ifdef VGA_CAN_DO_64KB
-				/*
-				 * get 64K rather than 32K of video RAM.
-				 * This doesn't actually work on all "VGA"
-				 * controllers (it seems like setting MM=01
-				 * and COE=1 isn't necessarily a good idea)
-				 */
-				vga_vram_base = 0xa0000;
-				vga_vram_size = 0x10000;
-				outb_p(6, VGA_GFX_I);
-				outb_p(6, VGA_GFX_D);
-#endif
 				/*
 				 * Normalise the palette registers, to point
 				 * the 16 screen colours to the first 16
@@ -542,7 +488,7 @@ static const char *vgacon_startup(void)
 
 	if (!vga_init_done) {
 		vgacon_scrollback_startup();
-		vga_init_done = 1;
+		vga_init_done = true;
 	}
 
 	return display_desc;
@@ -634,7 +580,7 @@ static u8 vgacon_build_attr(struct vc_data *c, u8 color, u8 intensity,
 
 static void vgacon_invert_region(struct vc_data *c, u16 * p, int count)
 {
-	int col = vga_can_do_color;
+	const bool col = vga_can_do_color;
 
 	while (count--) {
 		u16 a = scr_readw(p);
@@ -652,11 +598,6 @@ static void vgacon_set_cursor_size(int xpos, int from, int to)
 	unsigned long flags;
 	int curs, cure;
 
-#ifdef TRIDENT_GLITCH
-	if (xpos < 16)
-		from--, to--;
-#endif
-
 	if ((from == cursor_size_lastfrom) && (to == cursor_size_lastto))
 		return;
 	cursor_size_lastfrom = from;
@@ -858,12 +799,10 @@ static void vga_set_palette(struct vc_data *vc, const unsigned char *table)
 
 static void vgacon_set_palette(struct vc_data *vc, const unsigned char *table)
 {
-#ifdef CAN_LOAD_PALETTE
 	if (vga_video_type != VIDEO_TYPE_VGAC || vga_palette_blanked
 	    || !con_is_visible(vc))
 		return;
 	vga_set_palette(vc, table);
-#endif
 }
 
 /* structure holding original VGA register settings */
@@ -1006,24 +945,24 @@ static int vgacon_blank(struct vc_data *c, int blank, int mode_switch)
 		}
 		if (vga_palette_blanked) {
 			vga_set_palette(c, color_table);
-			vga_palette_blanked = 0;
+			vga_palette_blanked = false;
 			return 0;
 		}
-		vga_is_gfx = 0;
+		vga_is_gfx = false;
 		/* Tell console.c that it has to restore the screen itself */
 		return 1;
 	case 1:		/* Normal blanking */
 	case -1:	/* Obsolete */
 		if (!mode_switch && vga_video_type == VIDEO_TYPE_VGAC) {
 			vga_pal_blank(&vgastate);
-			vga_palette_blanked = 1;
+			vga_palette_blanked = true;
 			return 0;
 		}
 		vgacon_set_origin(c);
 		scr_memsetw((void *) vga_vram_base, BLANK,
 			    c->vc_screenbuf_size);
 		if (mode_switch)
-			vga_is_gfx = 1;
+			vga_is_gfx = true;
 		return 1;
 	default:		/* VESA blanking */
 		if (vga_video_type == VIDEO_TYPE_VGAC) {
@@ -1046,15 +985,14 @@ static int vgacon_blank(struct vc_data *c, int blank, int mode_switch)
  * (sizif@botik.yaroslavl.su).
  */
 
-#ifdef CAN_LOAD_EGA_FONTS
-
 #define colourmap 0xa0000
 /* Pauline Middelink <middelin@polyware.iaf.nl> reports that we
    should use 0xA0000 for the bwmap as well.. */
 #define blackwmap 0xa0000
 #define cmapsz 8192
 
-static int vgacon_do_font_op(struct vgastate *state,char *arg,int set,int ch512)
+static int vgacon_do_font_op(struct vgastate *state, char *arg, int set,
+		bool ch512)
 {
 	unsigned short video_port_status = vga_video_port_reg + 6;
 	int font_select = 0x00, beg, i;
@@ -1063,10 +1001,6 @@ static int vgacon_do_font_op(struct vgastate *state,char *arg,int set,int ch512)
 	if (vga_video_type != VIDEO_TYPE_EGAM) {
 		charmap = (char *) VGA_MAP_MEM(colourmap, 0);
 		beg = 0x0e;
-#ifdef VGA_CAN_DO_64KB
-		if (vga_video_type == VIDEO_TYPE_VGAC)
-			beg = 0x06;
-#endif
 	} else {
 		charmap = (char *) VGA_MAP_MEM(blackwmap, 0);
 		beg = 0x0a;
@@ -1080,7 +1014,7 @@ static int vgacon_do_font_op(struct vgastate *state,char *arg,int set,int ch512)
 	if (!arg)
 		return -EINVAL;	/* Return to default font not supported */
 
-	vga_font_is_default = 0;
+	vga_font_is_default = false;
 	font_select = ch512 ? 0x04 : 0x00;
 #else
 	/*
@@ -1091,7 +1025,7 @@ static int vgacon_do_font_op(struct vgastate *state,char *arg,int set,int ch512)
 	if (set) {
 		vga_font_is_default = !arg;
 		if (!arg)
-			ch512 = 0;	/* Default font is always 256 */
+			ch512 = false;	/* Default font is always 256 */
 		font_select = arg ? (ch512 ? 0x0e : 0x0a) : 0x00;
 	}
 
@@ -1295,13 +1229,6 @@ static int vgacon_font_get(struct vc_data *c, struct console_font *font)
 	return vgacon_do_font_op(&vgastate, font->data, 0, vga_512_chars);
 }
 
-#else
-
-#define vgacon_font_set NULL
-#define vgacon_font_get NULL
-
-#endif
-
 static int vgacon_resize(struct vc_data *c, unsigned int width,
 			 unsigned int height, unsigned int user)
 {
@@ -1350,17 +1277,17 @@ static void vgacon_save_screen(struct vc_data *c)
 			    c->vc_screenbuf_size > vga_vram_size ? vga_vram_size : c->vc_screenbuf_size);
 }
 
-static int vgacon_scroll(struct vc_data *c, int t, int b, int dir,
-			 int lines)
+static bool vgacon_scroll(struct vc_data *c, unsigned int t, unsigned int b,
+		enum con_scroll dir, unsigned int lines)
 {
 	unsigned long oldo;
 	unsigned int delta;
 
 	if (t || b != c->vc_rows || vga_is_gfx || c->vc_mode != KD_TEXT)
-		return 0;
+		return false;
 
 	if (!vga_hardscroll_enabled || lines >= c->vc_rows / 2)
-		return 0;
+		return false;
 
 	vgacon_restore_screen(c);
 	oldo = c->vc_origin;
@@ -1396,7 +1323,7 @@ static int vgacon_scroll(struct vc_data *c, int t, int b, int dir,
 	c->vc_visible_origin = c->vc_origin;
 	vga_set_mem_top(c);
 	c->vc_pos = (c->vc_pos - oldo) + c->vc_origin;
-	return 1;
+	return true;
 }
 
 
diff --git a/drivers/video/fbdev/xen-fbfront.c b/drivers/video/fbdev/xen-fbfront.c
index 0567d517eed3..d0115a7af0a9 100644
--- a/drivers/video/fbdev/xen-fbfront.c
+++ b/drivers/video/fbdev/xen-fbfront.c
@@ -633,7 +633,6 @@ static void xenfb_backend_changed(struct xenbus_device *dev,
 				  enum xenbus_state backend_state)
 {
 	struct xenfb_info *info = dev_get_drvdata(&dev->dev);
-	int val;
 
 	switch (backend_state) {
 	case XenbusStateInitialising:
@@ -657,16 +656,12 @@ InitWait:
 		if (dev->state != XenbusStateConnected)
 			goto InitWait; /* no InitWait seen yet, fudge it */
 
-		if (xenbus_scanf(XBT_NIL, info->xbdev->otherend,
-				 "request-update", "%d", &val) < 0)
-			val = 0;
-		if (val)
+		if (xenbus_read_unsigned(info->xbdev->otherend,
+					 "request-update", 0))
 			info->update_wanted = 1;
 
-		if (xenbus_scanf(XBT_NIL, dev->otherend,
-				 "feature-resize", "%d", &val) < 0)
-			val = 0;
-		info->feature_resize = val;
+		info->feature_resize = xenbus_read_unsigned(dev->otherend,
+							"feature-resize", 0);
 		break;
 
 	case XenbusStateClosed:
diff --git a/drivers/video/hdmi.c b/drivers/video/hdmi.c
index 162689227a23..1cf907ecded4 100644
--- a/drivers/video/hdmi.c
+++ b/drivers/video/hdmi.c
@@ -533,6 +533,10 @@ hdmi_picture_aspect_get_name(enum hdmi_picture_aspect picture_aspect)
 		return "4:3";
 	case HDMI_PICTURE_ASPECT_16_9:
 		return "16:9";
+	case HDMI_PICTURE_ASPECT_64_27:
+		return "64:27";
+	case HDMI_PICTURE_ASPECT_256_135:
+		return "256:135";
 	case HDMI_PICTURE_ASPECT_RESERVED:
 		return "Reserved";
 	}
diff --git a/drivers/video/of_display_timing.c b/drivers/video/of_display_timing.c
index 8a1076beecd3..32b0a7543433 100644
--- a/drivers/video/of_display_timing.c
+++ b/drivers/video/of_display_timing.c
@@ -88,6 +88,15 @@ static int of_parse_display_timing(const struct device_node *np,
 		dt->flags |= val ? DISPLAY_FLAGS_PIXDATA_POSEDGE :
 				DISPLAY_FLAGS_PIXDATA_NEGEDGE;
 
+	if (!of_property_read_u32(np, "syncclk-active", &val))
+		dt->flags |= val ? DISPLAY_FLAGS_SYNC_POSEDGE :
+				DISPLAY_FLAGS_SYNC_NEGEDGE;
+	else if (dt->flags & (DISPLAY_FLAGS_PIXDATA_POSEDGE |
+			      DISPLAY_FLAGS_PIXDATA_NEGEDGE))
+		dt->flags |= dt->flags & DISPLAY_FLAGS_PIXDATA_POSEDGE ?
+				DISPLAY_FLAGS_SYNC_POSEDGE :
+				DISPLAY_FLAGS_SYNC_NEGEDGE;
+
 	if (of_property_read_bool(np, "interlaced"))
 		dt->flags |= DISPLAY_FLAGS_INTERLACED;
 	if (of_property_read_bool(np, "doublescan"))
@@ -110,7 +119,7 @@ static int of_parse_display_timing(const struct device_node *np,
  * @name: name of the timing node
  * @dt: display_timing struct to fill
  **/
-int of_get_display_timing(struct device_node *np, const char *name,
+int of_get_display_timing(const struct device_node *np, const char *name,
 		struct display_timing *dt)
 {
 	struct device_node *timing_np;
@@ -133,7 +142,7 @@ EXPORT_SYMBOL_GPL(of_get_display_timing);
  * of_get_display_timings - parse all display_timing entries from a device_node
  * @np: device_node with the subnodes
  **/
-struct display_timings *of_get_display_timings(struct device_node *np)
+struct display_timings *of_get_display_timings(const struct device_node *np)
 {
 	struct device_node *timings_np;
 	struct device_node *entry;
@@ -249,7 +258,7 @@ EXPORT_SYMBOL_GPL(of_get_display_timings);
  * of_display_timings_exist - check if a display-timings node is provided
  * @np: device_node with the timing
  **/
-int of_display_timings_exist(struct device_node *np)
+int of_display_timings_exist(const struct device_node *np)
 {
 	struct device_node *timings_np;
 
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
index 48bfea91dbca..d47a2fcef818 100644
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -489,6 +489,7 @@ static const struct virtio_config_ops virtio_mmio_config_ops = {
 };
 
 
+static void virtio_mmio_release_dev_empty(struct device *_d) {}
 
 /* Platform device */
 
@@ -511,6 +512,7 @@ static int virtio_mmio_probe(struct platform_device *pdev)
 		return  -ENOMEM;
 
 	vm_dev->vdev.dev.parent = &pdev->dev;
+	vm_dev->vdev.dev.release = virtio_mmio_release_dev_empty;
 	vm_dev->vdev.config = &virtio_mmio_config_ops;
 	vm_dev->pdev = pdev;
 	INIT_LIST_HEAD(&vm_dev->virtqueues);
diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c
index d9a905827967..186cbab327b8 100644
--- a/drivers/virtio/virtio_pci_common.c
+++ b/drivers/virtio/virtio_pci_common.c
@@ -37,7 +37,7 @@ void vp_synchronize_vectors(struct virtio_device *vdev)
 		synchronize_irq(vp_dev->pci_dev->irq);
 
 	for (i = 0; i < vp_dev->msix_vectors; ++i)
-		synchronize_irq(vp_dev->msix_entries[i].vector);
+		synchronize_irq(pci_irq_vector(vp_dev->pci_dev, i));
 }
 
 /* the notify function used when creating a virt queue */
@@ -102,41 +102,6 @@ static irqreturn_t vp_interrupt(int irq, void *opaque)
 	return vp_vring_interrupt(irq, opaque);
 }
 
-static void vp_free_vectors(struct virtio_device *vdev)
-{
-	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
-	int i;
-
-	if (vp_dev->intx_enabled) {
-		free_irq(vp_dev->pci_dev->irq, vp_dev);
-		vp_dev->intx_enabled = 0;
-	}
-
-	for (i = 0; i < vp_dev->msix_used_vectors; ++i)
-		free_irq(vp_dev->msix_entries[i].vector, vp_dev);
-
-	for (i = 0; i < vp_dev->msix_vectors; i++)
-		if (vp_dev->msix_affinity_masks[i])
-			free_cpumask_var(vp_dev->msix_affinity_masks[i]);
-
-	if (vp_dev->msix_enabled) {
-		/* Disable the vector used for configuration */
-		vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
-
-		pci_disable_msix(vp_dev->pci_dev);
-		vp_dev->msix_enabled = 0;
-	}
-
-	vp_dev->msix_vectors = 0;
-	vp_dev->msix_used_vectors = 0;
-	kfree(vp_dev->msix_names);
-	vp_dev->msix_names = NULL;
-	kfree(vp_dev->msix_entries);
-	vp_dev->msix_entries = NULL;
-	kfree(vp_dev->msix_affinity_masks);
-	vp_dev->msix_affinity_masks = NULL;
-}
-
 static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
 				   bool per_vq_vectors)
 {
@@ -147,10 +112,6 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
 
 	vp_dev->msix_vectors = nvectors;
 
-	vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
-				       GFP_KERNEL);
-	if (!vp_dev->msix_entries)
-		goto error;
 	vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
 				     GFP_KERNEL);
 	if (!vp_dev->msix_names)
@@ -165,12 +126,9 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
 					GFP_KERNEL))
 			goto error;
 
-	for (i = 0; i < nvectors; ++i)
-		vp_dev->msix_entries[i].entry = i;
-
-	err = pci_enable_msix_exact(vp_dev->pci_dev,
-				    vp_dev->msix_entries, nvectors);
-	if (err)
+	err = pci_alloc_irq_vectors(vp_dev->pci_dev, nvectors, nvectors,
+			PCI_IRQ_MSIX);
+	if (err < 0)
 		goto error;
 	vp_dev->msix_enabled = 1;
 
@@ -178,7 +136,7 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
 	v = vp_dev->msix_used_vectors;
 	snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
 		 "%s-config", name);
-	err = request_irq(vp_dev->msix_entries[v].vector,
+	err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
 			  vp_config_changed, 0, vp_dev->msix_names[v],
 			  vp_dev);
 	if (err)
@@ -197,7 +155,7 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
 		v = vp_dev->msix_used_vectors;
 		snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
 			 "%s-virtqueues", name);
-		err = request_irq(vp_dev->msix_entries[v].vector,
+		err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
 				  vp_vring_interrupt, 0, vp_dev->msix_names[v],
 				  vp_dev);
 		if (err)
@@ -206,19 +164,6 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
 	}
 	return 0;
 error:
-	vp_free_vectors(vdev);
-	return err;
-}
-
-static int vp_request_intx(struct virtio_device *vdev)
-{
-	int err;
-	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
-
-	err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
-			  IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
-	if (!err)
-		vp_dev->intx_enabled = 1;
 	return err;
 }
 
@@ -276,67 +221,88 @@ void vp_del_vqs(struct virtio_device *vdev)
 {
 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 	struct virtqueue *vq, *n;
-	struct virtio_pci_vq_info *info;
+	int i;
 
 	list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
-		info = vp_dev->vqs[vq->index];
-		if (vp_dev->per_vq_vectors &&
-			info->msix_vector != VIRTIO_MSI_NO_VECTOR)
-			free_irq(vp_dev->msix_entries[info->msix_vector].vector,
-				 vq);
+		if (vp_dev->per_vq_vectors) {
+			int v = vp_dev->vqs[vq->index]->msix_vector;
+
+			if (v != VIRTIO_MSI_NO_VECTOR)
+				free_irq(pci_irq_vector(vp_dev->pci_dev, v),
+					vq);
+		}
 		vp_del_vq(vq);
 	}
 	vp_dev->per_vq_vectors = false;
 
-	vp_free_vectors(vdev);
+	if (vp_dev->intx_enabled) {
+		free_irq(vp_dev->pci_dev->irq, vp_dev);
+		vp_dev->intx_enabled = 0;
+	}
+
+	for (i = 0; i < vp_dev->msix_used_vectors; ++i)
+		free_irq(pci_irq_vector(vp_dev->pci_dev, i), vp_dev);
+
+	for (i = 0; i < vp_dev->msix_vectors; i++)
+		if (vp_dev->msix_affinity_masks[i])
+			free_cpumask_var(vp_dev->msix_affinity_masks[i]);
+
+	if (vp_dev->msix_enabled) {
+		/* Disable the vector used for configuration */
+		vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
+
+		pci_free_irq_vectors(vp_dev->pci_dev);
+		vp_dev->msix_enabled = 0;
+	}
+
+	vp_dev->msix_vectors = 0;
+	vp_dev->msix_used_vectors = 0;
+	kfree(vp_dev->msix_names);
+	vp_dev->msix_names = NULL;
+	kfree(vp_dev->msix_affinity_masks);
+	vp_dev->msix_affinity_masks = NULL;
 	kfree(vp_dev->vqs);
 	vp_dev->vqs = NULL;
 }
 
-static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
+static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned nvqs,
 			      struct virtqueue *vqs[],
 			      vq_callback_t *callbacks[],
 			      const char * const names[],
-			      bool use_msix,
 			      bool per_vq_vectors)
 {
 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 	u16 msix_vec;
 	int i, err, nvectors, allocated_vectors;
 
-	vp_dev->vqs = kmalloc(nvqs * sizeof *vp_dev->vqs, GFP_KERNEL);
+	vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL);
 	if (!vp_dev->vqs)
 		return -ENOMEM;
 
-	if (!use_msix) {
-		/* Old style: one normal interrupt for change and all vqs. */
-		err = vp_request_intx(vdev);
-		if (err)
-			goto error_find;
+	if (per_vq_vectors) {
+		/* Best option: one for change interrupt, one per vq. */
+		nvectors = 1;
+		for (i = 0; i < nvqs; ++i)
+			if (callbacks[i])
+				++nvectors;
 	} else {
-		if (per_vq_vectors) {
-			/* Best option: one for change interrupt, one per vq. */
-			nvectors = 1;
-			for (i = 0; i < nvqs; ++i)
-				if (callbacks[i])
-					++nvectors;
-		} else {
-			/* Second best: one for change, shared for all vqs. */
-			nvectors = 2;
-		}
-
-		err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
-		if (err)
-			goto error_find;
+		/* Second best: one for change, shared for all vqs. */
+		nvectors = 2;
 	}
 
+	err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
+	if (err)
+		goto error_find;
+
 	vp_dev->per_vq_vectors = per_vq_vectors;
 	allocated_vectors = vp_dev->msix_used_vectors;
 	for (i = 0; i < nvqs; ++i) {
 		if (!names[i]) {
 			vqs[i] = NULL;
 			continue;
-		} else if (!callbacks[i] || !vp_dev->msix_enabled)
+		}
+
+		if (!callbacks[i])
 			msix_vec = VIRTIO_MSI_NO_VECTOR;
 		else if (vp_dev->per_vq_vectors)
 			msix_vec = allocated_vectors++;
@@ -356,14 +322,12 @@ static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 			 sizeof *vp_dev->msix_names,
 			 "%s-%s",
 			 dev_name(&vp_dev->vdev.dev), names[i]);
-		err = request_irq(vp_dev->msix_entries[msix_vec].vector,
+		err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec),
 				  vring_interrupt, 0,
 				  vp_dev->msix_names[msix_vec],
 				  vqs[i]);
-		if (err) {
-			vp_del_vq(vqs[i]);
+		if (err)
 			goto error_find;
-		}
 	}
 	return 0;
 
@@ -372,6 +336,43 @@ error_find:
 	return err;
 }
 
+static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned nvqs,
+		struct virtqueue *vqs[], vq_callback_t *callbacks[],
+		const char * const names[])
+{
+	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
+	int i, err;
+
+	vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL);
+	if (!vp_dev->vqs)
+		return -ENOMEM;
+
+	err = request_irq(vp_dev->pci_dev->irq, vp_interrupt, IRQF_SHARED,
+			dev_name(&vdev->dev), vp_dev);
+	if (err)
+		goto out_del_vqs;
+
+	vp_dev->intx_enabled = 1;
+	vp_dev->per_vq_vectors = false;
+	for (i = 0; i < nvqs; ++i) {
+		if (!names[i]) {
+			vqs[i] = NULL;
+			continue;
+		}
+		vqs[i] = vp_setup_vq(vdev, i, callbacks[i], names[i],
+				VIRTIO_MSI_NO_VECTOR);
+		if (IS_ERR(vqs[i])) {
+			err = PTR_ERR(vqs[i]);
+			goto out_del_vqs;
+		}
+	}
+
+	return 0;
+out_del_vqs:
+	vp_del_vqs(vdev);
+	return err;
+}
+
 /* the config->find_vqs() implementation */
 int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 		struct virtqueue *vqs[],
@@ -381,17 +382,15 @@ int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 	int err;
 
 	/* Try MSI-X with one vector per queue. */
-	err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
+	err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, true);
 	if (!err)
 		return 0;
 	/* Fallback: MSI-X with one vector for config, one shared for queues. */
-	err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
-				 true, false);
+	err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, false);
 	if (!err)
 		return 0;
 	/* Finally fall back to regular interrupts. */
-	return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
-				  false, false);
+	return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names);
 }
 
 const char *vp_bus_name(struct virtio_device *vdev)
@@ -419,7 +418,7 @@ int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
 
 	if (vp_dev->msix_enabled) {
 		mask = vp_dev->msix_affinity_masks[info->msix_vector];
-		irq = vp_dev->msix_entries[info->msix_vector].vector;
+		irq = pci_irq_vector(vp_dev->pci_dev, info->msix_vector);
 		if (cpu == -1)
 			irq_set_affinity_hint(irq, NULL);
 		else {
diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h
index 28263200ed42..b2f666250ae0 100644
--- a/drivers/virtio/virtio_pci_common.h
+++ b/drivers/virtio/virtio_pci_common.h
@@ -85,7 +85,6 @@ struct virtio_pci_device {
 	/* MSI-X support */
 	int msix_enabled;
 	int intx_enabled;
-	struct msix_entry *msix_entries;
 	cpumask_var_t *msix_affinity_masks;
 	/* Name strings for interrupts. This size should be enough,
 	 * and I'm too lazy to allocate each name separately. */
diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
index e76bd91a29da..4bf7ab375894 100644
--- a/drivers/virtio/virtio_pci_modern.c
+++ b/drivers/virtio/virtio_pci_modern.c
@@ -33,12 +33,12 @@ static inline u8 vp_ioread8(u8 __iomem *addr)
 {
 	return ioread8(addr);
 }
-static inline u16 vp_ioread16 (u16 __iomem *addr)
+static inline u16 vp_ioread16 (__le16 __iomem *addr)
 {
 	return ioread16(addr);
 }
 
-static inline u32 vp_ioread32(u32 __iomem *addr)
+static inline u32 vp_ioread32(__le32 __iomem *addr)
 {
 	return ioread32(addr);
 }
@@ -48,12 +48,12 @@ static inline void vp_iowrite8(u8 value, u8 __iomem *addr)
 	iowrite8(value, addr);
 }
 
-static inline void vp_iowrite16(u16 value, u16 __iomem *addr)
+static inline void vp_iowrite16(u16 value, __le16 __iomem *addr)
 {
 	iowrite16(value, addr);
 }
 
-static inline void vp_iowrite32(u32 value, u32 __iomem *addr)
+static inline void vp_iowrite32(u32 value, __le32 __iomem *addr)
 {
 	iowrite32(value, addr);
 }
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 489bfc61cf30..409aeaa49246 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -420,7 +420,7 @@ unmap_release:
 		if (i == err_idx)
 			break;
 		vring_unmap_one(vq, &desc[i]);
-		i = vq->vring.desc[i].next;
+		i = virtio16_to_cpu(_vq->vdev, vq->vring.desc[i].next);
 	}
 
 	vq->vq.num_free += total_sg;
@@ -601,7 +601,7 @@ EXPORT_SYMBOL_GPL(virtqueue_kick);
 static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
 {
 	unsigned int i, j;
-	u16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
+	__virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
 
 	/* Clear data ptr. */
 	vq->desc_state[head].data = NULL;
@@ -649,7 +649,7 @@ static inline bool more_used(const struct vring_virtqueue *vq)
  * @vq: the struct virtqueue we're talking about.
  * @len: the length written into the buffer
  *
- * If the driver wrote data into the buffer, @len will be set to the
+ * If the device wrote data into the buffer, @len will be set to the
  * amount written.  This means you don't need to clear the buffer
  * beforehand to ensure there's no data leakage in the case of short
  * writes.
diff --git a/drivers/watchdog/mei_wdt.c b/drivers/watchdog/mei_wdt.c
index 630bd189f167..79b35515904e 100644
--- a/drivers/watchdog/mei_wdt.c
+++ b/drivers/watchdog/mei_wdt.c
@@ -410,11 +410,11 @@ static void mei_wdt_unregister_work(struct work_struct *work)
 }
 
 /**
- * mei_wdt_event_rx - callback for data receive
+ * mei_wdt_rx - callback for data receive
  *
  * @cldev: bus device
  */
-static void mei_wdt_event_rx(struct mei_cl_device *cldev)
+static void mei_wdt_rx(struct mei_cl_device *cldev)
 {
 	struct mei_wdt *wdt = mei_cldev_get_drvdata(cldev);
 	struct mei_wdt_start_response res;
@@ -482,11 +482,11 @@ out:
 }
 
 /*
- * mei_wdt_notify_event - callback for event notification
+ * mei_wdt_notif - callback for event notification
  *
  * @cldev: bus device
  */
-static void mei_wdt_notify_event(struct mei_cl_device *cldev)
+static void mei_wdt_notif(struct mei_cl_device *cldev)
 {
 	struct mei_wdt *wdt = mei_cldev_get_drvdata(cldev);
 
@@ -496,23 +496,6 @@ static void mei_wdt_notify_event(struct mei_cl_device *cldev)
 	mei_wdt_register(wdt);
 }
 
-/**
- * mei_wdt_event - callback for event receive
- *
- * @cldev: bus device
- * @events: event mask
- * @context: callback context
- */
-static void mei_wdt_event(struct mei_cl_device *cldev,
-			  u32 events, void *context)
-{
-	if (events & BIT(MEI_CL_EVENT_RX))
-		mei_wdt_event_rx(cldev);
-
-	if (events & BIT(MEI_CL_EVENT_NOTIF))
-		mei_wdt_notify_event(cldev);
-}
-
 #if IS_ENABLED(CONFIG_DEBUG_FS)
 
 static ssize_t mei_dbgfs_read_activation(struct file *file, char __user *ubuf,
@@ -623,16 +606,17 @@ static int mei_wdt_probe(struct mei_cl_device *cldev,
 		goto err_out;
 	}
 
-	ret = mei_cldev_register_event_cb(wdt->cldev,
-					  BIT(MEI_CL_EVENT_RX) |
-					  BIT(MEI_CL_EVENT_NOTIF),
-					  mei_wdt_event, NULL);
+	ret = mei_cldev_register_rx_cb(wdt->cldev, mei_wdt_rx);
+	if (ret) {
+		dev_err(&cldev->dev, "Could not reg rx event ret=%d\n", ret);
+		goto err_disable;
+	}
 
+	ret = mei_cldev_register_notif_cb(wdt->cldev, mei_wdt_notif);
 	/* on legacy devices notification is not supported
-	 * this doesn't fail the registration for RX event
 	 */
 	if (ret && ret != -EOPNOTSUPP) {
-		dev_err(&cldev->dev, "Could not register event ret=%d\n", ret);
+		dev_err(&cldev->dev, "Could not reg notif event ret=%d\n", ret);
 		goto err_disable;
 	}
 
@@ -699,25 +683,7 @@ static struct mei_cl_driver mei_wdt_driver = {
 	.remove = mei_wdt_remove,
 };
 
-static int __init mei_wdt_init(void)
-{
-	int ret;
-
-	ret = mei_cldev_driver_register(&mei_wdt_driver);
-	if (ret) {
-		pr_err(KBUILD_MODNAME ": module registration failed\n");
-		return ret;
-	}
-	return 0;
-}
-
-static void __exit mei_wdt_exit(void)
-{
-	mei_cldev_driver_unregister(&mei_wdt_driver);
-}
-
-module_init(mei_wdt_init);
-module_exit(mei_wdt_exit);
+module_mei_cl_driver(mei_wdt_driver);
 
 MODULE_AUTHOR("Intel Corporation");
 MODULE_LICENSE("GPL");
diff --git a/drivers/watchdog/sa1100_wdt.c b/drivers/watchdog/sa1100_wdt.c
index e1d39a1e9628..8965e3f536c3 100644
--- a/drivers/watchdog/sa1100_wdt.c
+++ b/drivers/watchdog/sa1100_wdt.c
@@ -22,6 +22,7 @@
 
 #include <linux/module.h>
 #include <linux/moduleparam.h>
+#include <linux/clk.h>
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/fs.h>
@@ -155,12 +156,27 @@ static struct miscdevice sa1100dog_miscdev = {
 };
 
 static int margin __initdata = 60;		/* (secs) Default is 1 minute */
+static struct clk *clk;
 
 static int __init sa1100dog_init(void)
 {
 	int ret;
 
-	oscr_freq = get_clock_tick_rate();
+	clk = clk_get(NULL, "OSTIMER0");
+	if (IS_ERR(clk)) {
+		pr_err("SA1100/PXA2xx Watchdog Timer: clock not found: %d\n",
+		       (int) PTR_ERR(clk));
+		return PTR_ERR(clk);
+	}
+
+	ret = clk_prepare_enable(clk);
+	if (ret) {
+		pr_err("SA1100/PXA2xx Watchdog Timer: clock failed to prepare+enable: %d\n",
+		       ret);
+		goto err;
+	}
+
+	oscr_freq = clk_get_rate(clk);
 
 	/*
 	 * Read the reset status, and save it for later.  If
@@ -176,11 +192,17 @@ static int __init sa1100dog_init(void)
 		pr_info("SA1100/PXA2xx Watchdog Timer: timer margin %d sec\n",
 			margin);
 	return ret;
+err:
+	clk_disable_unprepare(clk);
+	clk_put(clk);
+	return ret;
 }
 
 static void __exit sa1100dog_exit(void)
 {
 	misc_deregister(&sa1100dog_miscdev);
+	clk_disable_unprepare(clk);
+	clk_put(clk);
 }
 
 module_init(sa1100dog_init);
diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
index e4db19e88ab1..db107fa50ca1 100644
--- a/drivers/xen/balloon.c
+++ b/drivers/xen/balloon.c
@@ -180,7 +180,6 @@ static void __balloon_append(struct page *page)
 static void balloon_append(struct page *page)
 {
 	__balloon_append(page);
-	adjust_managed_page_count(page, -1);
 }
 
 /* balloon_retrieve: rescue a page from the balloon, if it is not empty. */
@@ -201,8 +200,6 @@ static struct page *balloon_retrieve(bool require_lowmem)
 	else
 		balloon_stats.balloon_low--;
 
-	adjust_managed_page_count(page, 1);
-
 	return page;
 }
 
@@ -478,7 +475,7 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
 #endif
 
 		/* Relinquish the page back to the allocator. */
-		__free_reserved_page(page);
+		free_reserved_page(page);
 	}
 
 	balloon_stats.current_pages += rc;
@@ -509,6 +506,7 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
 			state = BP_EAGAIN;
 			break;
 		}
+		adjust_managed_page_count(page, -1);
 		scrub_page(page);
 		list_add(&page->lru, &pages);
 	}
diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c
index adc19ce3cc66..fd8e872d2943 100644
--- a/drivers/xen/events/events_base.c
+++ b/drivers/xen/events/events_base.c
@@ -947,7 +947,7 @@ static int find_virq(unsigned int virq, unsigned int cpu)
 			continue;
 		if (status.status != EVTCHNSTAT_virq)
 			continue;
-		if (status.u.virq == virq && status.vcpu == cpu) {
+		if (status.u.virq == virq && status.vcpu == xen_vcpu_nr(cpu)) {
 			rc = port;
 			break;
 		}
diff --git a/drivers/xen/gntalloc.c b/drivers/xen/gntalloc.c
index 7a47c4c9fb1b..1bf55a32a4b3 100644
--- a/drivers/xen/gntalloc.c
+++ b/drivers/xen/gntalloc.c
@@ -127,18 +127,21 @@ static int add_grefs(struct ioctl_gntalloc_alloc_gref *op,
 	struct gntalloc_gref *gref, *next;
 
 	readonly = !(op->flags & GNTALLOC_FLAG_WRITABLE);
-	rc = -ENOMEM;
 	for (i = 0; i < op->count; i++) {
 		gref = kzalloc(sizeof(*gref), GFP_KERNEL);
-		if (!gref)
+		if (!gref) {
+			rc = -ENOMEM;
 			goto undo;
+		}
 		list_add_tail(&gref->next_gref, &queue_gref);
 		list_add_tail(&gref->next_file, &queue_file);
 		gref->users = 1;
 		gref->file_index = op->index + i * PAGE_SIZE;
 		gref->page = alloc_page(GFP_KERNEL|__GFP_ZERO);
-		if (!gref->page)
+		if (!gref->page) {
+			rc = -ENOMEM;
 			goto undo;
+		}
 
 		/* Grant foreign access to the page. */
 		rc = gnttab_grant_foreign_access(op->domid,
diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c
index bb952121ea94..2ef2b61b69df 100644
--- a/drivers/xen/gntdev.c
+++ b/drivers/xen/gntdev.c
@@ -1007,7 +1007,7 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma)
 
 	vma->vm_ops = &gntdev_vmops;
 
-	vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP | VM_IO;
+	vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP | VM_MIXEDMAP;
 
 	if (use_ptemod)
 		vma->vm_flags |= VM_DONTCOPY;
diff --git a/drivers/xen/platform-pci.c b/drivers/xen/platform-pci.c
index b59c9455aae1..112ce422dc22 100644
--- a/drivers/xen/platform-pci.c
+++ b/drivers/xen/platform-pci.c
@@ -125,8 +125,4 @@ static struct pci_driver platform_driver = {
 	.id_table =       platform_pci_tbl,
 };
 
-static int __init platform_pci_init(void)
-{
-	return pci_register_driver(&platform_driver);
-}
-device_initcall(platform_pci_init);
+builtin_pci_driver(platform_driver);
diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c
index 702040fe2001..6e3306f4a525 100644
--- a/drivers/xen/privcmd.c
+++ b/drivers/xen/privcmd.c
@@ -602,7 +602,7 @@ static int privcmd_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	printk(KERN_DEBUG "privcmd_fault: vma=%p %lx-%lx, pgoff=%lx, uv=%p\n",
 	       vma, vma->vm_start, vma->vm_end,
-	       vmf->pgoff, vmf->virtual_address);
+	       vmf->pgoff, (void *)vmf->address);
 
 	return VM_FAULT_SIGBUS;
 }
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 87e6035c9e81..478fb91e3df2 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -405,7 +405,8 @@ dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page,
 	 */
 	trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force);
 
-	map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir);
+	map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir,
+				     attrs);
 	if (map == SWIOTLB_MAP_ERROR)
 		return DMA_ERROR_CODE;
 
@@ -416,11 +417,13 @@ dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page,
 	/*
 	 * Ensure that the address returned is DMA'ble
 	 */
-	if (!dma_capable(dev, dev_addr, size)) {
-		swiotlb_tbl_unmap_single(dev, map, size, dir);
-		dev_addr = 0;
-	}
-	return dev_addr;
+	if (dma_capable(dev, dev_addr, size))
+		return dev_addr;
+
+	attrs |= DMA_ATTR_SKIP_CPU_SYNC;
+	swiotlb_tbl_unmap_single(dev, map, size, dir, attrs);
+
+	return DMA_ERROR_CODE;
 }
 EXPORT_SYMBOL_GPL(xen_swiotlb_map_page);
 
@@ -444,7 +447,7 @@ static void xen_unmap_single(struct device *hwdev, dma_addr_t dev_addr,
 
 	/* NOTE: We use dev_addr here, not paddr! */
 	if (is_xen_swiotlb_buffer(dev_addr)) {
-		swiotlb_tbl_unmap_single(hwdev, paddr, size, dir);
+		swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs);
 		return;
 	}
 
@@ -557,11 +560,12 @@ xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
 								 start_dma_addr,
 								 sg_phys(sg),
 								 sg->length,
-								 dir);
+								 dir, attrs);
 			if (map == SWIOTLB_MAP_ERROR) {
 				dev_warn(hwdev, "swiotlb buffer is full\n");
 				/* Don't panic here, we expect map_sg users
 				   to do proper error handling. */
+				attrs |= DMA_ATTR_SKIP_CPU_SYNC;
 				xen_swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
 							   attrs);
 				sg_dma_len(sgl) = 0;
@@ -648,13 +652,6 @@ xen_swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
 }
 EXPORT_SYMBOL_GPL(xen_swiotlb_sync_sg_for_device);
 
-int
-xen_swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
-{
-	return !dma_addr;
-}
-EXPORT_SYMBOL_GPL(xen_swiotlb_dma_mapping_error);
-
 /*
  * Return whether the given device DMA address mask can be supported
  * properly.  For example, if your device can only drive the low 24-bits
diff --git a/drivers/xen/xen-pciback/xenbus.c b/drivers/xen/xen-pciback/xenbus.c
index 5ce878c51d03..3f0aee0a068b 100644
--- a/drivers/xen/xen-pciback/xenbus.c
+++ b/drivers/xen/xen-pciback/xenbus.c
@@ -362,7 +362,7 @@ static int xen_pcibk_reconfigure(struct xen_pcibk_device *pdev)
 	int err = 0;
 	int num_devs;
 	int domain, bus, slot, func;
-	int substate;
+	unsigned int substate;
 	int i, len;
 	char state_str[64];
 	char dev_str[64];
@@ -395,10 +395,8 @@ static int xen_pcibk_reconfigure(struct xen_pcibk_device *pdev)
 					 "configuration");
 			goto out;
 		}
-		err = xenbus_scanf(XBT_NIL, pdev->xdev->nodename, state_str,
-				   "%d", &substate);
-		if (err != 1)
-			substate = XenbusStateUnknown;
+		substate = xenbus_read_unsigned(pdev->xdev->nodename, state_str,
+						XenbusStateUnknown);
 
 		switch (substate) {
 		case XenbusStateInitialising:
diff --git a/drivers/xen/xenbus/xenbus_dev_frontend.c b/drivers/xen/xenbus/xenbus_dev_frontend.c
index 1e8be12ebb55..6c0ead4be784 100644
--- a/drivers/xen/xenbus/xenbus_dev_frontend.c
+++ b/drivers/xen/xenbus/xenbus_dev_frontend.c
@@ -538,6 +538,8 @@ static int xenbus_file_open(struct inode *inode, struct file *filp)
 
 	nonseekable_open(inode, filp);
 
+	filp->f_mode &= ~FMODE_ATOMIC_POS; /* cdev-style semantics */
+
 	u = kzalloc(sizeof(*u), GFP_KERNEL);
 	if (u == NULL)
 		return -ENOMEM;
diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c
index 33a31cfef55d..4bdf654041e9 100644
--- a/drivers/xen/xenbus/xenbus_probe.c
+++ b/drivers/xen/xenbus/xenbus_probe.c
@@ -702,7 +702,7 @@ device_initcall(xenbus_probe_initcall);
  */
 static int __init xenstored_local_init(void)
 {
-	int err = 0;
+	int err = -ENOMEM;
 	unsigned long page = 0;
 	struct evtchn_alloc_unbound alloc_unbound;
 
@@ -826,7 +826,7 @@ static int __init xenbus_init(void)
 	 * Create xenfs mountpoint in /proc for compatibility with
 	 * utilities that expect to find "xenbus" under "/proc/xen".
 	 */
-	proc_mkdir("xen", NULL);
+	proc_create_mount_point("xen");
 #endif
 
 out_error:
diff --git a/drivers/xen/xenbus/xenbus_probe_backend.c b/drivers/xen/xenbus/xenbus_probe_backend.c
index 04f7f85a5edf..37929df829a3 100644
--- a/drivers/xen/xenbus/xenbus_probe_backend.c
+++ b/drivers/xen/xenbus/xenbus_probe_backend.c
@@ -224,13 +224,7 @@ static int read_frontend_details(struct xenbus_device *xendev)
 
 int xenbus_dev_is_online(struct xenbus_device *dev)
 {
-	int rc, val;
-
-	rc = xenbus_scanf(XBT_NIL, dev->nodename, "online", "%d", &val);
-	if (rc != 1)
-		val = 0; /* no online node present */
-
-	return val;
+	return !!xenbus_read_unsigned(dev->nodename, "online", 0);
 }
 EXPORT_SYMBOL_GPL(xenbus_dev_is_online);
 
diff --git a/drivers/xen/xenbus/xenbus_xs.c b/drivers/xen/xenbus/xenbus_xs.c
index 22f7cd711c57..6afb993c5809 100644
--- a/drivers/xen/xenbus/xenbus_xs.c
+++ b/drivers/xen/xenbus/xenbus_xs.c
@@ -559,6 +559,21 @@ int xenbus_scanf(struct xenbus_transaction t,
 }
 EXPORT_SYMBOL_GPL(xenbus_scanf);
 
+/* Read an (optional) unsigned value. */
+unsigned int xenbus_read_unsigned(const char *dir, const char *node,
+				  unsigned int default_val)
+{
+	unsigned int val;
+	int ret;
+
+	ret = xenbus_scanf(XBT_NIL, dir, node, "%u", &val);
+	if (ret <= 0)
+		val = default_val;
+
+	return val;
+}
+EXPORT_SYMBOL_GPL(xenbus_read_unsigned);
+
 /* Single printf and write: returns -errno or 0. */
 int xenbus_printf(struct xenbus_transaction t,
 		  const char *dir, const char *node, const char *fmt, ...)
@@ -672,7 +687,7 @@ static bool xen_strict_xenbus_quirk(void)
 }
 static void xs_reset_watches(void)
 {
-	int err, supported = 0;
+	int err;
 
 	if (!xen_hvm_domain() || xen_initial_domain())
 		return;
@@ -680,9 +695,8 @@ static void xs_reset_watches(void)
 	if (xen_strict_xenbus_quirk())
 		return;
 
-	err = xenbus_scanf(XBT_NIL, "control",
-			"platform-feature-xs_reset_watches", "%d", &supported);
-	if (err != 1 || !supported)
+	if (!xenbus_read_unsigned("control",
+				  "platform-feature-xs_reset_watches", 0))
 		return;
 
 	err = xs_error(xs_single(XBT_NIL, XS_RESET_WATCHES, "", NULL));
diff --git a/fs/9p/vfs_addr.c b/fs/9p/vfs_addr.c
index 6181ad79e1a5..5ca1fb0043f6 100644
--- a/fs/9p/vfs_addr.c
+++ b/fs/9p/vfs_addr.c
@@ -34,6 +34,7 @@
 #include <linux/idr.h>
 #include <linux/sched.h>
 #include <linux/uio.h>
+#include <linux/bvec.h>
 #include <net/9p/9p.h>
 #include <net/9p/client.h>
 
diff --git a/fs/Kconfig b/fs/Kconfig
index 4bd03a2b0518..8e9e5f4104f4 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -55,7 +55,6 @@ config FS_DAX_PMD
 	depends on FS_DAX
 	depends on ZONE_DEVICE
 	depends on TRANSPARENT_HUGEPAGE
-	depends on BROKEN
 
 endif # BLOCK
 
diff --git a/fs/block_dev.c b/fs/block_dev.c
index 05b553368bb4..7c4507224ed6 100644
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -30,6 +30,7 @@
 #include <linux/cleancache.h>
 #include <linux/dax.h>
 #include <linux/badblocks.h>
+#include <linux/task_io_accounting_ops.h>
 #include <linux/falloc.h>
 #include <asm/uaccess.h>
 #include "internal.h"
@@ -175,16 +176,269 @@ static struct inode *bdev_file_inode(struct file *file)
 	return file->f_mapping->host;
 }
 
+static unsigned int dio_bio_write_op(struct kiocb *iocb)
+{
+	unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
+
+	/* avoid the need for a I/O completion work item */
+	if (iocb->ki_flags & IOCB_DSYNC)
+		op |= REQ_FUA;
+	return op;
+}
+
+#define DIO_INLINE_BIO_VECS 4
+
+static void blkdev_bio_end_io_simple(struct bio *bio)
+{
+	struct task_struct *waiter = bio->bi_private;
+
+	WRITE_ONCE(bio->bi_private, NULL);
+	wake_up_process(waiter);
+}
+
 static ssize_t
-blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
+__blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
+		int nr_pages)
+{
+	struct file *file = iocb->ki_filp;
+	struct block_device *bdev = I_BDEV(bdev_file_inode(file));
+	struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs, *bvec;
+	loff_t pos = iocb->ki_pos;
+	bool should_dirty = false;
+	struct bio bio;
+	ssize_t ret;
+	blk_qc_t qc;
+	int i;
+
+	if ((pos | iov_iter_alignment(iter)) &
+	    (bdev_logical_block_size(bdev) - 1))
+		return -EINVAL;
+
+	if (nr_pages <= DIO_INLINE_BIO_VECS)
+		vecs = inline_vecs;
+	else {
+		vecs = kmalloc(nr_pages * sizeof(struct bio_vec), GFP_KERNEL);
+		if (!vecs)
+			return -ENOMEM;
+	}
+
+	bio_init(&bio, vecs, nr_pages);
+	bio.bi_bdev = bdev;
+	bio.bi_iter.bi_sector = pos >> 9;
+	bio.bi_private = current;
+	bio.bi_end_io = blkdev_bio_end_io_simple;
+
+	ret = bio_iov_iter_get_pages(&bio, iter);
+	if (unlikely(ret))
+		return ret;
+	ret = bio.bi_iter.bi_size;
+
+	if (iov_iter_rw(iter) == READ) {
+		bio.bi_opf = REQ_OP_READ;
+		if (iter_is_iovec(iter))
+			should_dirty = true;
+	} else {
+		bio.bi_opf = dio_bio_write_op(iocb);
+		task_io_account_write(ret);
+	}
+
+	qc = submit_bio(&bio);
+	for (;;) {
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		if (!READ_ONCE(bio.bi_private))
+			break;
+		if (!(iocb->ki_flags & IOCB_HIPRI) ||
+		    !blk_mq_poll(bdev_get_queue(bdev), qc))
+			io_schedule();
+	}
+	__set_current_state(TASK_RUNNING);
+
+	bio_for_each_segment_all(bvec, &bio, i) {
+		if (should_dirty && !PageCompound(bvec->bv_page))
+			set_page_dirty_lock(bvec->bv_page);
+		put_page(bvec->bv_page);
+	}
+
+	if (vecs != inline_vecs)
+		kfree(vecs);
+
+	if (unlikely(bio.bi_error))
+		return bio.bi_error;
+	return ret;
+}
+
+struct blkdev_dio {
+	union {
+		struct kiocb		*iocb;
+		struct task_struct	*waiter;
+	};
+	size_t			size;
+	atomic_t		ref;
+	bool			multi_bio : 1;
+	bool			should_dirty : 1;
+	bool			is_sync : 1;
+	struct bio		bio;
+};
+
+static struct bio_set *blkdev_dio_pool __read_mostly;
+
+static void blkdev_bio_end_io(struct bio *bio)
+{
+	struct blkdev_dio *dio = bio->bi_private;
+	bool should_dirty = dio->should_dirty;
+
+	if (dio->multi_bio && !atomic_dec_and_test(&dio->ref)) {
+		if (bio->bi_error && !dio->bio.bi_error)
+			dio->bio.bi_error = bio->bi_error;
+	} else {
+		if (!dio->is_sync) {
+			struct kiocb *iocb = dio->iocb;
+			ssize_t ret = dio->bio.bi_error;
+
+			if (likely(!ret)) {
+				ret = dio->size;
+				iocb->ki_pos += ret;
+			}
+
+			dio->iocb->ki_complete(iocb, ret, 0);
+			bio_put(&dio->bio);
+		} else {
+			struct task_struct *waiter = dio->waiter;
+
+			WRITE_ONCE(dio->waiter, NULL);
+			wake_up_process(waiter);
+		}
+	}
+
+	if (should_dirty) {
+		bio_check_pages_dirty(bio);
+	} else {
+		struct bio_vec *bvec;
+		int i;
+
+		bio_for_each_segment_all(bvec, bio, i)
+			put_page(bvec->bv_page);
+		bio_put(bio);
+	}
+}
+
+static ssize_t
+__blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
 {
 	struct file *file = iocb->ki_filp;
 	struct inode *inode = bdev_file_inode(file);
+	struct block_device *bdev = I_BDEV(inode);
+	struct blkdev_dio *dio;
+	struct bio *bio;
+	bool is_read = (iov_iter_rw(iter) == READ);
+	loff_t pos = iocb->ki_pos;
+	blk_qc_t qc = BLK_QC_T_NONE;
+	int ret;
+
+	if ((pos | iov_iter_alignment(iter)) &
+	    (bdev_logical_block_size(bdev) - 1))
+		return -EINVAL;
+
+	bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, blkdev_dio_pool);
+	bio_get(bio); /* extra ref for the completion handler */
+
+	dio = container_of(bio, struct blkdev_dio, bio);
+	dio->is_sync = is_sync_kiocb(iocb);
+	if (dio->is_sync)
+		dio->waiter = current;
+	else
+		dio->iocb = iocb;
+
+	dio->size = 0;
+	dio->multi_bio = false;
+	dio->should_dirty = is_read && (iter->type == ITER_IOVEC);
+
+	for (;;) {
+		bio->bi_bdev = bdev;
+		bio->bi_iter.bi_sector = pos >> 9;
+		bio->bi_private = dio;
+		bio->bi_end_io = blkdev_bio_end_io;
+
+		ret = bio_iov_iter_get_pages(bio, iter);
+		if (unlikely(ret)) {
+			bio->bi_error = ret;
+			bio_endio(bio);
+			break;
+		}
+
+		if (is_read) {
+			bio->bi_opf = REQ_OP_READ;
+			if (dio->should_dirty)
+				bio_set_pages_dirty(bio);
+		} else {
+			bio->bi_opf = dio_bio_write_op(iocb);
+			task_io_account_write(bio->bi_iter.bi_size);
+		}
+
+		dio->size += bio->bi_iter.bi_size;
+		pos += bio->bi_iter.bi_size;
+
+		nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES);
+		if (!nr_pages) {
+			qc = submit_bio(bio);
+			break;
+		}
+
+		if (!dio->multi_bio) {
+			dio->multi_bio = true;
+			atomic_set(&dio->ref, 2);
+		} else {
+			atomic_inc(&dio->ref);
+		}
 
-	return __blockdev_direct_IO(iocb, inode, I_BDEV(inode), iter,
-				    blkdev_get_block, NULL, NULL,
-				    DIO_SKIP_DIO_COUNT);
+		submit_bio(bio);
+		bio = bio_alloc(GFP_KERNEL, nr_pages);
+	}
+
+	if (!dio->is_sync)
+		return -EIOCBQUEUED;
+
+	for (;;) {
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		if (!READ_ONCE(dio->waiter))
+			break;
+
+		if (!(iocb->ki_flags & IOCB_HIPRI) ||
+		    !blk_mq_poll(bdev_get_queue(bdev), qc))
+			io_schedule();
+	}
+	__set_current_state(TASK_RUNNING);
+
+	ret = dio->bio.bi_error;
+	if (likely(!ret))
+		ret = dio->size;
+
+	bio_put(&dio->bio);
+	return ret;
+}
+
+static ssize_t
+blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
+{
+	int nr_pages;
+
+	nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES + 1);
+	if (!nr_pages)
+		return 0;
+	if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES)
+		return __blkdev_direct_IO_simple(iocb, iter, nr_pages);
+
+	return __blkdev_direct_IO(iocb, iter, min(nr_pages, BIO_MAX_PAGES));
+}
+
+static __init int blkdev_init(void)
+{
+	blkdev_dio_pool = bioset_create(4, offsetof(struct blkdev_dio, bio));
+	if (!blkdev_dio_pool)
+		return -ENOMEM;
+	return 0;
 }
+module_init(blkdev_init);
 
 int __sync_blockdev(struct block_device *bdev, int wait)
 {
@@ -832,7 +1086,7 @@ static bool bd_may_claim(struct block_device *bdev, struct block_device *whole,
 		return true;	 /* already a holder */
 	else if (bdev->bd_holder != NULL)
 		return false; 	 /* held by someone else */
-	else if (bdev->bd_contains == bdev)
+	else if (whole == bdev)
 		return true;  	 /* is a whole device which isn't held */
 
 	else if (whole->bd_holder == bd_may_claim)
@@ -1950,6 +2204,7 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
 	spin_lock(&blockdev_superblock->s_inode_list_lock);
 	list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
 		struct address_space *mapping = inode->i_mapping;
+		struct block_device *bdev;
 
 		spin_lock(&inode->i_lock);
 		if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) ||
@@ -1970,8 +2225,12 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
 		 */
 		iput(old_inode);
 		old_inode = inode;
+		bdev = I_BDEV(inode);
 
-		func(I_BDEV(inode), arg);
+		mutex_lock(&bdev->bd_mutex);
+		if (bdev->bd_openers)
+			func(bdev, arg);
+		mutex_unlock(&bdev->bd_mutex);
 
 		spin_lock(&blockdev_superblock->s_inode_list_lock);
 	}
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 3a57f99d96aa..fe10afd51e02 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -930,7 +930,7 @@ int btrfs_wq_submit_bio(struct btrfs_fs_info *fs_info, struct inode *inode,
 
 	atomic_inc(&fs_info->nr_async_submits);
 
-	if (bio->bi_opf & REQ_SYNC)
+	if (op_is_sync(bio->bi_opf))
 		btrfs_set_work_high_priority(&async->work);
 
 	btrfs_queue_work(fs_info->workers, &async->work);
@@ -3485,9 +3485,9 @@ static int write_dev_supers(struct btrfs_device *device,
 		 * to go down lazy.
 		 */
 		if (i == 0)
-			ret = btrfsic_submit_bh(REQ_OP_WRITE, WRITE_FUA, bh);
+			ret = btrfsic_submit_bh(REQ_OP_WRITE, REQ_FUA, bh);
 		else
-			ret = btrfsic_submit_bh(REQ_OP_WRITE, WRITE_SYNC, bh);
+			ret = btrfsic_submit_bh(REQ_OP_WRITE, REQ_SYNC, bh);
 		if (ret)
 			errors++;
 	}
@@ -3551,7 +3551,7 @@ static int write_dev_flush(struct btrfs_device *device, int wait)
 
 	bio->bi_end_io = btrfs_end_empty_barrier;
 	bio->bi_bdev = device->bdev;
-	bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_FLUSH);
+	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 	init_completion(&device->flush_wait);
 	bio->bi_private = &device->flush_wait;
 	device->flush_bio = bio;
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 8ed05d95584a..1e67723c27a1 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -127,7 +127,7 @@ struct extent_page_data {
 	 */
 	unsigned int extent_locked:1;
 
-	/* tells the submit_bio code to use a WRITE_SYNC */
+	/* tells the submit_bio code to use REQ_SYNC */
 	unsigned int sync_io:1;
 };
 
@@ -2047,7 +2047,7 @@ int repair_io_failure(struct inode *inode, u64 start, u64 length, u64 logical,
 		return -EIO;
 	}
 	bio->bi_bdev = dev->bdev;
-	bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_SYNC);
+	bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
 	bio_add_page(bio, page, length, pg_offset);
 
 	if (btrfsic_submit_bio_wait(bio)) {
@@ -2388,7 +2388,7 @@ static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
 	struct inode *inode = page->mapping->host;
 	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
 	struct bio *bio;
-	int read_mode;
+	int read_mode = 0;
 	int ret;
 
 	BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
@@ -2404,9 +2404,7 @@ static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
 	}
 
 	if (failed_bio->bi_vcnt > 1)
-		read_mode = READ_SYNC | REQ_FAILFAST_DEV;
-	else
-		read_mode = READ_SYNC;
+		read_mode |= REQ_FAILFAST_DEV;
 
 	phy_offset >>= inode->i_sb->s_blocksize_bits;
 	bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
@@ -3484,7 +3482,7 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
 	unsigned long nr_written = 0;
 
 	if (wbc->sync_mode == WB_SYNC_ALL)
-		write_flags = WRITE_SYNC;
+		write_flags = REQ_SYNC;
 
 	trace___extent_writepage(page, inode, wbc);
 
@@ -3729,7 +3727,7 @@ static noinline_for_stack int write_one_eb(struct extent_buffer *eb,
 	unsigned long i, num_pages;
 	unsigned long bio_flags = 0;
 	unsigned long start, end;
-	int write_flags = (epd->sync_io ? WRITE_SYNC : 0) | REQ_META;
+	int write_flags = (epd->sync_io ? REQ_SYNC : 0) | REQ_META;
 	int ret = 0;
 
 	clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
@@ -4076,7 +4074,7 @@ static void flush_epd_write_bio(struct extent_page_data *epd)
 		int ret;
 
 		bio_set_op_attrs(epd->bio, REQ_OP_WRITE,
-				 epd->sync_io ? WRITE_SYNC : 0);
+				 epd->sync_io ? REQ_SYNC : 0);
 
 		ret = submit_one_bio(epd->bio, 0, epd->bio_flags);
 		BUG_ON(ret < 0); /* -ENOMEM */
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 8e3a5a266917..a4c879671b9d 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -7917,7 +7917,7 @@ static int dio_read_error(struct inode *inode, struct bio *failed_bio,
 	struct io_failure_record *failrec;
 	struct bio *bio;
 	int isector;
-	int read_mode;
+	int read_mode = 0;
 	int ret;
 
 	BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
@@ -7936,9 +7936,7 @@ static int dio_read_error(struct inode *inode, struct bio *failed_bio,
 	if ((failed_bio->bi_vcnt > 1)
 		|| (failed_bio->bi_io_vec->bv_len
 			> BTRFS_I(inode)->root->sectorsize))
-		read_mode = READ_SYNC | REQ_FAILFAST_DEV;
-	else
-		read_mode = READ_SYNC;
+		read_mode |= REQ_FAILFAST_DEV;
 
 	isector = start - btrfs_io_bio(failed_bio)->logical;
 	isector >>= inode->i_sb->s_blocksize_bits;
@@ -8427,7 +8425,7 @@ static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip,
 	if (!bio)
 		return -ENOMEM;
 
-	bio_set_op_attrs(bio, bio_op(orig_bio), bio_flags(orig_bio));
+	bio->bi_opf = orig_bio->bi_opf;
 	bio->bi_private = dip;
 	bio->bi_end_io = btrfs_end_dio_bio;
 	btrfs_io_bio(bio)->logical = file_offset;
@@ -8465,8 +8463,7 @@ next_block:
 						  start_sector, GFP_NOFS);
 			if (!bio)
 				goto out_err;
-			bio_set_op_attrs(bio, bio_op(orig_bio),
-					 bio_flags(orig_bio));
+			bio->bi_opf = orig_bio->bi_opf;
 			bio->bi_private = dip;
 			bio->bi_end_io = btrfs_end_dio_bio;
 			btrfs_io_bio(bio)->logical = file_offset;
diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c
index fffb9ab8526e..ff3078234d94 100644
--- a/fs/btrfs/scrub.c
+++ b/fs/btrfs/scrub.c
@@ -4440,7 +4440,7 @@ static int write_page_nocow(struct scrub_ctx *sctx,
 	bio->bi_iter.bi_size = 0;
 	bio->bi_iter.bi_sector = physical_for_dev_replace >> 9;
 	bio->bi_bdev = dev->bdev;
-	bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_SYNC);
+	bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
 	ret = bio_add_page(bio, page, PAGE_SIZE, 0);
 	if (ret != PAGE_SIZE) {
 leave_with_eio:
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
index 180f910339f4..3b713b6fcc26 100644
--- a/fs/btrfs/super.c
+++ b/fs/btrfs/super.c
@@ -202,12 +202,12 @@ static struct ratelimit_state printk_limits[] = {
 void btrfs_printk(const struct btrfs_fs_info *fs_info, const char *fmt, ...)
 {
 	struct super_block *sb = fs_info->sb;
-	char lvl[PRINTK_MAX_SINGLE_HEADER_LEN + 1];
+	char lvl[PRINTK_MAX_SINGLE_HEADER_LEN + 1] = "\0";
 	struct va_format vaf;
 	va_list args;
-	const char *type = NULL;
 	int kern_level;
-	struct ratelimit_state *ratelimit;
+	const char *type = logtypes[4];
+	struct ratelimit_state *ratelimit = &printk_limits[4];
 
 	va_start(args, fmt);
 
@@ -223,12 +223,6 @@ void btrfs_printk(const struct btrfs_fs_info *fs_info, const char *fmt, ...)
 		fmt += size;
 	}
 
-	if (!type) {
-		*lvl = '\0';
-		type = logtypes[4];
-		ratelimit = &printk_limits[4];
-	}
-
 	vaf.fmt = fmt;
 	vaf.va = &args;
 
diff --git a/fs/btrfs/tests/btrfs-tests.c b/fs/btrfs/tests/btrfs-tests.c
index bf62ad919a95..00ee006a8aa2 100644
--- a/fs/btrfs/tests/btrfs-tests.c
+++ b/fs/btrfs/tests/btrfs-tests.c
@@ -162,6 +162,7 @@ void btrfs_free_dummy_fs_info(struct btrfs_fs_info *fs_info)
 				slot = radix_tree_iter_retry(&iter);
 			continue;
 		}
+		slot = radix_tree_iter_resume(slot, &iter);
 		spin_unlock(&fs_info->buffer_lock);
 		free_extent_buffer_stale(eb);
 		spin_lock(&fs_info->buffer_lock);
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index 71a60cc01451..0d7d635d8bfb 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -6023,7 +6023,7 @@ static void btrfs_end_bio(struct bio *bio)
 				else
 					btrfs_dev_stat_inc(dev,
 						BTRFS_DEV_STAT_READ_ERRS);
-				if ((bio->bi_opf & WRITE_FLUSH) == WRITE_FLUSH)
+				if (bio->bi_opf & REQ_PREFLUSH)
 					btrfs_dev_stat_inc(dev,
 						BTRFS_DEV_STAT_FLUSH_ERRS);
 				btrfs_dev_stat_print_on_error(dev);
@@ -6100,7 +6100,7 @@ static noinline void btrfs_schedule_bio(struct btrfs_root *root,
 	bio->bi_next = NULL;
 
 	spin_lock(&device->io_lock);
-	if (bio->bi_opf & REQ_SYNC)
+	if (op_is_sync(bio->bi_opf))
 		pending_bios = &device->pending_sync_bios;
 	else
 		pending_bios = &device->pending_bios;
diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h
index 09ed29c67848..f137ffe6654c 100644
--- a/fs/btrfs/volumes.h
+++ b/fs/btrfs/volumes.h
@@ -62,7 +62,7 @@ struct btrfs_device {
 	int running_pending;
 	/* regular prio bios */
 	struct btrfs_pending_bios pending_bios;
-	/* WRITE_SYNC bios */
+	/* sync bios */
 	struct btrfs_pending_bios pending_sync_bios;
 
 	struct block_device *bdev;
diff --git a/fs/buffer.c b/fs/buffer.c
index 1613656028d6..d21771fcf7d3 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -43,6 +43,7 @@
 #include <linux/bitops.h>
 #include <linux/mpage.h>
 #include <linux/bit_spinlock.h>
+#include <linux/pagevec.h>
 #include <trace/events/block.h>
 
 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
@@ -753,7 +754,7 @@ static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
 				 * still in flight on potentially older
 				 * contents.
 				 */
-				write_dirty_buffer(bh, WRITE_SYNC);
+				write_dirty_buffer(bh, REQ_SYNC);
 
 				/*
 				 * Kick off IO for the previous mapping. Note
@@ -1604,37 +1605,80 @@ void create_empty_buffers(struct page *page,
 }
 EXPORT_SYMBOL(create_empty_buffers);
 
-/*
- * We are taking a block for data and we don't want any output from any
- * buffer-cache aliases starting from return from that function and
- * until the moment when something will explicitly mark the buffer
- * dirty (hopefully that will not happen until we will free that block ;-)
- * We don't even need to mark it not-uptodate - nobody can expect
- * anything from a newly allocated buffer anyway. We used to used
- * unmap_buffer() for such invalidation, but that was wrong. We definitely
- * don't want to mark the alias unmapped, for example - it would confuse
- * anyone who might pick it with bread() afterwards...
- *
- * Also..  Note that bforget() doesn't lock the buffer.  So there can
- * be writeout I/O going on against recently-freed buffers.  We don't
- * wait on that I/O in bforget() - it's more efficient to wait on the I/O
- * only if we really need to.  That happens here.
- */
-void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
+/**
+ * clean_bdev_aliases: clean a range of buffers in block device
+ * @bdev: Block device to clean buffers in
+ * @block: Start of a range of blocks to clean
+ * @len: Number of blocks to clean
+ *
+ * We are taking a range of blocks for data and we don't want writeback of any
+ * buffer-cache aliases starting from return from this function and until the
+ * moment when something will explicitly mark the buffer dirty (hopefully that
+ * will not happen until we will free that block ;-) We don't even need to mark
+ * it not-uptodate - nobody can expect anything from a newly allocated buffer
+ * anyway. We used to use unmap_buffer() for such invalidation, but that was
+ * wrong. We definitely don't want to mark the alias unmapped, for example - it
+ * would confuse anyone who might pick it with bread() afterwards...
+ *
+ * Also..  Note that bforget() doesn't lock the buffer.  So there can be
+ * writeout I/O going on against recently-freed buffers.  We don't wait on that
+ * I/O in bforget() - it's more efficient to wait on the I/O only if we really
+ * need to.  That happens here.
+ */
+void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
 {
-	struct buffer_head *old_bh;
+	struct inode *bd_inode = bdev->bd_inode;
+	struct address_space *bd_mapping = bd_inode->i_mapping;
+	struct pagevec pvec;
+	pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
+	pgoff_t end;
+	int i;
+	struct buffer_head *bh;
+	struct buffer_head *head;
 
-	might_sleep();
+	end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits);
+	pagevec_init(&pvec, 0);
+	while (index <= end && pagevec_lookup(&pvec, bd_mapping, index,
+			min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
+		for (i = 0; i < pagevec_count(&pvec); i++) {
+			struct page *page = pvec.pages[i];
 
-	old_bh = __find_get_block_slow(bdev, block);
-	if (old_bh) {
-		clear_buffer_dirty(old_bh);
-		wait_on_buffer(old_bh);
-		clear_buffer_req(old_bh);
-		__brelse(old_bh);
+			index = page->index;
+			if (index > end)
+				break;
+			if (!page_has_buffers(page))
+				continue;
+			/*
+			 * We use page lock instead of bd_mapping->private_lock
+			 * to pin buffers here since we can afford to sleep and
+			 * it scales better than a global spinlock lock.
+			 */
+			lock_page(page);
+			/* Recheck when the page is locked which pins bhs */
+			if (!page_has_buffers(page))
+				goto unlock_page;
+			head = page_buffers(page);
+			bh = head;
+			do {
+				if (!buffer_mapped(bh))
+					goto next;
+				if (bh->b_blocknr >= block + len)
+					break;
+				clear_buffer_dirty(bh);
+				wait_on_buffer(bh);
+				clear_buffer_req(bh);
+next:
+				bh = bh->b_this_page;
+			} while (bh != head);
+unlock_page:
+			unlock_page(page);
+		}
+		pagevec_release(&pvec);
+		cond_resched();
+		index++;
 	}
 }
-EXPORT_SYMBOL(unmap_underlying_metadata);
+EXPORT_SYMBOL(clean_bdev_aliases);
 
 /*
  * Size is a power-of-two in the range 512..PAGE_SIZE,
@@ -1684,7 +1728,7 @@ static struct buffer_head *create_page_buffers(struct page *page, struct inode *
  * prevents this contention from occurring.
  *
  * If block_write_full_page() is called with wbc->sync_mode ==
- * WB_SYNC_ALL, the writes are posted using WRITE_SYNC; this
+ * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
  * causes the writes to be flagged as synchronous writes.
  */
 int __block_write_full_page(struct inode *inode, struct page *page,
@@ -1697,7 +1741,7 @@ int __block_write_full_page(struct inode *inode, struct page *page,
 	struct buffer_head *bh, *head;
 	unsigned int blocksize, bbits;
 	int nr_underway = 0;
-	int write_flags = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : 0);
+	int write_flags = wbc_to_write_flags(wbc);
 
 	head = create_page_buffers(page, inode,
 					(1 << BH_Dirty)|(1 << BH_Uptodate));
@@ -1745,8 +1789,7 @@ int __block_write_full_page(struct inode *inode, struct page *page,
 			if (buffer_new(bh)) {
 				/* blockdev mappings never come here */
 				clear_buffer_new(bh);
-				unmap_underlying_metadata(bh->b_bdev,
-							bh->b_blocknr);
+				clean_bdev_bh_alias(bh);
 			}
 		}
 		bh = bh->b_this_page;
@@ -1992,8 +2035,7 @@ int __block_write_begin_int(struct page *page, loff_t pos, unsigned len,
 			}
 
 			if (buffer_new(bh)) {
-				unmap_underlying_metadata(bh->b_bdev,
-							bh->b_blocknr);
+				clean_bdev_bh_alias(bh);
 				if (PageUptodate(page)) {
 					clear_buffer_new(bh);
 					set_buffer_uptodate(bh);
@@ -2633,7 +2675,7 @@ int nobh_write_begin(struct address_space *mapping,
 		if (!buffer_mapped(bh))
 			is_mapped_to_disk = 0;
 		if (buffer_new(bh))
-			unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
+			clean_bdev_bh_alias(bh);
 		if (PageUptodate(page)) {
 			set_buffer_uptodate(bh);
 			continue;
@@ -3118,7 +3160,7 @@ EXPORT_SYMBOL(submit_bh);
 /**
  * ll_rw_block: low-level access to block devices (DEPRECATED)
  * @op: whether to %READ or %WRITE
- * @op_flags: rq_flag_bits
+ * @op_flags: req_flag_bits
  * @nr: number of &struct buffer_heads in the array
  * @bhs: array of pointers to &struct buffer_head
  *
@@ -3210,7 +3252,7 @@ EXPORT_SYMBOL(__sync_dirty_buffer);
 
 int sync_dirty_buffer(struct buffer_head *bh)
 {
-	return __sync_dirty_buffer(bh, WRITE_SYNC);
+	return __sync_dirty_buffer(bh, REQ_SYNC);
 }
 EXPORT_SYMBOL(sync_dirty_buffer);
 
diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c
index 4547aeddd12b..f7563c88c917 100644
--- a/fs/cifs/connect.c
+++ b/fs/cifs/connect.c
@@ -41,6 +41,7 @@
 #include <keys/user-type.h>
 #include <net/ipv6.h>
 #include <linux/parser.h>
+#include <linux/bvec.h>
 
 #include "cifspdu.h"
 #include "cifsglob.h"
diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c
index 206a597b2293..5f02edc819af 100644
--- a/fs/cifs/transport.c
+++ b/fs/cifs/transport.c
@@ -28,6 +28,7 @@
 #include <linux/delay.h>
 #include <linux/freezer.h>
 #include <linux/tcp.h>
+#include <linux/bvec.h>
 #include <linux/highmem.h>
 #include <asm/uaccess.h>
 #include <asm/processor.h>
diff --git a/fs/crypto/Kconfig b/fs/crypto/Kconfig
index 92348faf9865..f514978f6688 100644
--- a/fs/crypto/Kconfig
+++ b/fs/crypto/Kconfig
@@ -8,9 +8,7 @@ config FS_ENCRYPTION
 	select CRYPTO_XTS
 	select CRYPTO_CTS
 	select CRYPTO_CTR
-	select CRYPTO_SHA256
 	select KEYS
-	select ENCRYPTED_KEYS
 	help
 	  Enable encryption of files and directories.  This
 	  feature is similar to ecryptfs, but it is more memory
diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c
index 98f87fe8f186..ac8e4f6a3773 100644
--- a/fs/crypto/crypto.c
+++ b/fs/crypto/crypto.c
@@ -27,7 +27,7 @@
 #include <linux/bio.h>
 #include <linux/dcache.h>
 #include <linux/namei.h>
-#include <linux/fscrypto.h>
+#include "fscrypt_private.h"
 
 static unsigned int num_prealloc_crypto_pages = 32;
 static unsigned int num_prealloc_crypto_ctxs = 128;
@@ -63,7 +63,7 @@ void fscrypt_release_ctx(struct fscrypt_ctx *ctx)
 {
 	unsigned long flags;
 
-	if (ctx->flags & FS_WRITE_PATH_FL && ctx->w.bounce_page) {
+	if (ctx->flags & FS_CTX_HAS_BOUNCE_BUFFER_FL && ctx->w.bounce_page) {
 		mempool_free(ctx->w.bounce_page, fscrypt_bounce_page_pool);
 		ctx->w.bounce_page = NULL;
 	}
@@ -88,7 +88,7 @@ EXPORT_SYMBOL(fscrypt_release_ctx);
  * Return: An allocated and initialized encryption context on success; error
  * value or NULL otherwise.
  */
-struct fscrypt_ctx *fscrypt_get_ctx(struct inode *inode, gfp_t gfp_flags)
+struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *inode, gfp_t gfp_flags)
 {
 	struct fscrypt_ctx *ctx = NULL;
 	struct fscrypt_info *ci = inode->i_crypt_info;
@@ -121,7 +121,7 @@ struct fscrypt_ctx *fscrypt_get_ctx(struct inode *inode, gfp_t gfp_flags)
 	} else {
 		ctx->flags &= ~FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
 	}
-	ctx->flags &= ~FS_WRITE_PATH_FL;
+	ctx->flags &= ~FS_CTX_HAS_BOUNCE_BUFFER_FL;
 	return ctx;
 }
 EXPORT_SYMBOL(fscrypt_get_ctx);
@@ -146,9 +146,10 @@ typedef enum {
 	FS_ENCRYPT,
 } fscrypt_direction_t;
 
-static int do_page_crypto(struct inode *inode,
-			fscrypt_direction_t rw, pgoff_t index,
+static int do_page_crypto(const struct inode *inode,
+			fscrypt_direction_t rw, u64 lblk_num,
 			struct page *src_page, struct page *dest_page,
+			unsigned int len, unsigned int offs,
 			gfp_t gfp_flags)
 {
 	struct {
@@ -162,6 +163,8 @@ static int do_page_crypto(struct inode *inode,
 	struct crypto_skcipher *tfm = ci->ci_ctfm;
 	int res = 0;
 
+	BUG_ON(len == 0);
+
 	req = skcipher_request_alloc(tfm, gfp_flags);
 	if (!req) {
 		printk_ratelimited(KERN_ERR
@@ -175,14 +178,14 @@ static int do_page_crypto(struct inode *inode,
 		page_crypt_complete, &ecr);
 
 	BUILD_BUG_ON(sizeof(xts_tweak) != FS_XTS_TWEAK_SIZE);
-	xts_tweak.index = cpu_to_le64(index);
+	xts_tweak.index = cpu_to_le64(lblk_num);
 	memset(xts_tweak.padding, 0, sizeof(xts_tweak.padding));
 
 	sg_init_table(&dst, 1);
-	sg_set_page(&dst, dest_page, PAGE_SIZE, 0);
+	sg_set_page(&dst, dest_page, len, offs);
 	sg_init_table(&src, 1);
-	sg_set_page(&src, src_page, PAGE_SIZE, 0);
-	skcipher_request_set_crypt(req, &src, &dst, PAGE_SIZE, &xts_tweak);
+	sg_set_page(&src, src_page, len, offs);
+	skcipher_request_set_crypt(req, &src, &dst, len, &xts_tweak);
 	if (rw == FS_DECRYPT)
 		res = crypto_skcipher_decrypt(req);
 	else
@@ -207,34 +210,66 @@ static struct page *alloc_bounce_page(struct fscrypt_ctx *ctx, gfp_t gfp_flags)
 	ctx->w.bounce_page = mempool_alloc(fscrypt_bounce_page_pool, gfp_flags);
 	if (ctx->w.bounce_page == NULL)
 		return ERR_PTR(-ENOMEM);
-	ctx->flags |= FS_WRITE_PATH_FL;
+	ctx->flags |= FS_CTX_HAS_BOUNCE_BUFFER_FL;
 	return ctx->w.bounce_page;
 }
 
 /**
  * fscypt_encrypt_page() - Encrypts a page
- * @inode:          The inode for which the encryption should take place
- * @plaintext_page: The page to encrypt. Must be locked.
- * @gfp_flags:      The gfp flag for memory allocation
+ * @inode:     The inode for which the encryption should take place
+ * @page:      The page to encrypt. Must be locked for bounce-page
+ *             encryption.
+ * @len:       Length of data to encrypt in @page and encrypted
+ *             data in returned page.
+ * @offs:      Offset of data within @page and returned
+ *             page holding encrypted data.
+ * @lblk_num:  Logical block number. This must be unique for multiple
+ *             calls with same inode, except when overwriting
+ *             previously written data.
+ * @gfp_flags: The gfp flag for memory allocation
  *
- * Allocates a ciphertext page and encrypts plaintext_page into it using the ctx
- * encryption context.
+ * Encrypts @page using the ctx encryption context. Performs encryption
+ * either in-place or into a newly allocated bounce page.
+ * Called on the page write path.
  *
- * Called on the page write path.  The caller must call
+ * Bounce page allocation is the default.
+ * In this case, the contents of @page are encrypted and stored in an
+ * allocated bounce page. @page has to be locked and the caller must call
  * fscrypt_restore_control_page() on the returned ciphertext page to
  * release the bounce buffer and the encryption context.
  *
- * Return: An allocated page with the encrypted content on success. Else, an
+ * In-place encryption is used by setting the FS_CFLG_OWN_PAGES flag in
+ * fscrypt_operations. Here, the input-page is returned with its content
+ * encrypted.
+ *
+ * Return: A page with the encrypted content on success. Else, an
  * error value or NULL.
  */
-struct page *fscrypt_encrypt_page(struct inode *inode,
-				struct page *plaintext_page, gfp_t gfp_flags)
+struct page *fscrypt_encrypt_page(const struct inode *inode,
+				struct page *page,
+				unsigned int len,
+				unsigned int offs,
+				u64 lblk_num, gfp_t gfp_flags)
+
 {
 	struct fscrypt_ctx *ctx;
-	struct page *ciphertext_page = NULL;
+	struct page *ciphertext_page = page;
 	int err;
 
-	BUG_ON(!PageLocked(plaintext_page));
+	BUG_ON(len % FS_CRYPTO_BLOCK_SIZE != 0);
+
+	if (inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES) {
+		/* with inplace-encryption we just encrypt the page */
+		err = do_page_crypto(inode, FS_ENCRYPT, lblk_num,
+					page, ciphertext_page,
+					len, offs, gfp_flags);
+		if (err)
+			return ERR_PTR(err);
+
+		return ciphertext_page;
+	}
+
+	BUG_ON(!PageLocked(page));
 
 	ctx = fscrypt_get_ctx(inode, gfp_flags);
 	if (IS_ERR(ctx))
@@ -245,10 +280,10 @@ struct page *fscrypt_encrypt_page(struct inode *inode,
 	if (IS_ERR(ciphertext_page))
 		goto errout;
 
-	ctx->w.control_page = plaintext_page;
-	err = do_page_crypto(inode, FS_ENCRYPT, plaintext_page->index,
-					plaintext_page, ciphertext_page,
-					gfp_flags);
+	ctx->w.control_page = page;
+	err = do_page_crypto(inode, FS_ENCRYPT, lblk_num,
+					page, ciphertext_page,
+					len, offs, gfp_flags);
 	if (err) {
 		ciphertext_page = ERR_PTR(err);
 		goto errout;
@@ -265,8 +300,13 @@ errout:
 EXPORT_SYMBOL(fscrypt_encrypt_page);
 
 /**
- * f2crypt_decrypt_page() - Decrypts a page in-place
- * @page: The page to decrypt. Must be locked.
+ * fscrypt_decrypt_page() - Decrypts a page in-place
+ * @inode:     The corresponding inode for the page to decrypt.
+ * @page:      The page to decrypt. Must be locked in case
+ *             it is a writeback page (FS_CFLG_OWN_PAGES unset).
+ * @len:       Number of bytes in @page to be decrypted.
+ * @offs:      Start of data in @page.
+ * @lblk_num:  Logical block number.
  *
  * Decrypts page in-place using the ctx encryption context.
  *
@@ -274,16 +314,18 @@ EXPORT_SYMBOL(fscrypt_encrypt_page);
  *
  * Return: Zero on success, non-zero otherwise.
  */
-int fscrypt_decrypt_page(struct page *page)
+int fscrypt_decrypt_page(const struct inode *inode, struct page *page,
+			unsigned int len, unsigned int offs, u64 lblk_num)
 {
-	BUG_ON(!PageLocked(page));
+	if (!(inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES))
+		BUG_ON(!PageLocked(page));
 
-	return do_page_crypto(page->mapping->host,
-			FS_DECRYPT, page->index, page, page, GFP_NOFS);
+	return do_page_crypto(inode, FS_DECRYPT, lblk_num, page, page, len,
+			offs, GFP_NOFS);
 }
 EXPORT_SYMBOL(fscrypt_decrypt_page);
 
-int fscrypt_zeroout_range(struct inode *inode, pgoff_t lblk,
+int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk,
 				sector_t pblk, unsigned int len)
 {
 	struct fscrypt_ctx *ctx;
@@ -306,7 +348,7 @@ int fscrypt_zeroout_range(struct inode *inode, pgoff_t lblk,
 	while (len--) {
 		err = do_page_crypto(inode, FS_ENCRYPT, lblk,
 					ZERO_PAGE(0), ciphertext_page,
-					GFP_NOFS);
+					PAGE_SIZE, 0, GFP_NOFS);
 		if (err)
 			goto errout;
 
@@ -414,7 +456,8 @@ static void completion_pages(struct work_struct *work)
 
 	bio_for_each_segment_all(bv, bio, i) {
 		struct page *page = bv->bv_page;
-		int ret = fscrypt_decrypt_page(page);
+		int ret = fscrypt_decrypt_page(page->mapping->host, page,
+				PAGE_SIZE, 0, page->index);
 
 		if (ret) {
 			WARN_ON_ONCE(1);
@@ -482,17 +525,22 @@ static void fscrypt_destroy(void)
 
 /**
  * fscrypt_initialize() - allocate major buffers for fs encryption.
+ * @cop_flags:  fscrypt operations flags
  *
  * We only call this when we start accessing encrypted files, since it
  * results in memory getting allocated that wouldn't otherwise be used.
  *
  * Return: Zero on success, non-zero otherwise.
  */
-int fscrypt_initialize(void)
+int fscrypt_initialize(unsigned int cop_flags)
 {
 	int i, res = -ENOMEM;
 
-	if (fscrypt_bounce_page_pool)
+	/*
+	 * No need to allocate a bounce page pool if there already is one or
+	 * this FS won't use it.
+	 */
+	if (cop_flags & FS_CFLG_OWN_PAGES || fscrypt_bounce_page_pool)
 		return 0;
 
 	mutex_lock(&fscrypt_init_mutex);
@@ -521,7 +569,6 @@ fail:
 	mutex_unlock(&fscrypt_init_mutex);
 	return res;
 }
-EXPORT_SYMBOL(fscrypt_initialize);
 
 /**
  * fscrypt_init() - Set up for fs encryption.
diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c
index 9b774f4b50c8..56ad9d195f18 100644
--- a/fs/crypto/fname.c
+++ b/fs/crypto/fname.c
@@ -12,7 +12,7 @@
 
 #include <linux/scatterlist.h>
 #include <linux/ratelimit.h>
-#include <linux/fscrypto.h>
+#include "fscrypt_private.h"
 
 /**
  * fname_crypt_complete() - completion callback for filename crypto
@@ -209,7 +209,7 @@ static int digest_decode(const char *src, int len, char *dst)
 	return cp - dst;
 }
 
-u32 fscrypt_fname_encrypted_size(struct inode *inode, u32 ilen)
+u32 fscrypt_fname_encrypted_size(const struct inode *inode, u32 ilen)
 {
 	int padding = 32;
 	struct fscrypt_info *ci = inode->i_crypt_info;
@@ -227,7 +227,7 @@ EXPORT_SYMBOL(fscrypt_fname_encrypted_size);
  * Allocates an output buffer that is sufficient for the crypto operation
  * specified by the context and the direction.
  */
-int fscrypt_fname_alloc_buffer(struct inode *inode,
+int fscrypt_fname_alloc_buffer(const struct inode *inode,
 				u32 ilen, struct fscrypt_str *crypto_str)
 {
 	unsigned int olen = fscrypt_fname_encrypted_size(inode, ilen);
@@ -350,7 +350,7 @@ int fscrypt_setup_filename(struct inode *dir, const struct qstr *iname,
 		fname->disk_name.len = iname->len;
 		return 0;
 	}
-	ret = get_crypt_info(dir);
+	ret = fscrypt_get_crypt_info(dir);
 	if (ret && ret != -EOPNOTSUPP)
 		return ret;
 
diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h
new file mode 100644
index 000000000000..aeab032d7d35
--- /dev/null
+++ b/fs/crypto/fscrypt_private.h
@@ -0,0 +1,93 @@
+/*
+ * fscrypt_private.h
+ *
+ * Copyright (C) 2015, Google, Inc.
+ *
+ * This contains encryption key functions.
+ *
+ * Written by Michael Halcrow, Ildar Muslukhov, and Uday Savagaonkar, 2015.
+ */
+
+#ifndef _FSCRYPT_PRIVATE_H
+#define _FSCRYPT_PRIVATE_H
+
+#include <linux/fscrypto.h>
+
+#define FS_FNAME_CRYPTO_DIGEST_SIZE	32
+
+/* Encryption parameters */
+#define FS_XTS_TWEAK_SIZE		16
+#define FS_AES_128_ECB_KEY_SIZE		16
+#define FS_AES_256_GCM_KEY_SIZE		32
+#define FS_AES_256_CBC_KEY_SIZE		32
+#define FS_AES_256_CTS_KEY_SIZE		32
+#define FS_AES_256_XTS_KEY_SIZE		64
+#define FS_MAX_KEY_SIZE			64
+
+#define FS_KEY_DESC_PREFIX		"fscrypt:"
+#define FS_KEY_DESC_PREFIX_SIZE		8
+
+#define FS_KEY_DERIVATION_NONCE_SIZE		16
+
+/**
+ * Encryption context for inode
+ *
+ * Protector format:
+ *  1 byte: Protector format (1 = this version)
+ *  1 byte: File contents encryption mode
+ *  1 byte: File names encryption mode
+ *  1 byte: Flags
+ *  8 bytes: Master Key descriptor
+ *  16 bytes: Encryption Key derivation nonce
+ */
+struct fscrypt_context {
+	u8 format;
+	u8 contents_encryption_mode;
+	u8 filenames_encryption_mode;
+	u8 flags;
+	u8 master_key_descriptor[FS_KEY_DESCRIPTOR_SIZE];
+	u8 nonce[FS_KEY_DERIVATION_NONCE_SIZE];
+} __packed;
+
+#define FS_ENCRYPTION_CONTEXT_FORMAT_V1		1
+
+/* This is passed in from userspace into the kernel keyring */
+struct fscrypt_key {
+	u32 mode;
+	u8 raw[FS_MAX_KEY_SIZE];
+	u32 size;
+} __packed;
+
+/*
+ * A pointer to this structure is stored in the file system's in-core
+ * representation of an inode.
+ */
+struct fscrypt_info {
+	u8 ci_data_mode;
+	u8 ci_filename_mode;
+	u8 ci_flags;
+	struct crypto_skcipher *ci_ctfm;
+	struct key *ci_keyring_key;
+	u8 ci_master_key[FS_KEY_DESCRIPTOR_SIZE];
+};
+
+#define FS_CTX_REQUIRES_FREE_ENCRYPT_FL		0x00000001
+#define FS_CTX_HAS_BOUNCE_BUFFER_FL		0x00000002
+
+struct fscrypt_completion_result {
+	struct completion completion;
+	int res;
+};
+
+#define DECLARE_FS_COMPLETION_RESULT(ecr) \
+	struct fscrypt_completion_result ecr = { \
+		COMPLETION_INITIALIZER((ecr).completion), 0 }
+
+
+/* crypto.c */
+int fscrypt_initialize(unsigned int cop_flags);
+
+/* keyinfo.c */
+extern int fscrypt_get_crypt_info(struct inode *);
+
+#endif /* _FSCRYPT_PRIVATE_H */
diff --git a/fs/crypto/keyinfo.c b/fs/crypto/keyinfo.c
index 67fb6d8876d0..6eeea1dcba41 100644
--- a/fs/crypto/keyinfo.c
+++ b/fs/crypto/keyinfo.c
@@ -10,7 +10,7 @@
 
 #include <keys/user-type.h>
 #include <linux/scatterlist.h>
-#include <linux/fscrypto.h>
+#include "fscrypt_private.h"
 
 static void derive_crypt_complete(struct crypto_async_request *req, int rc)
 {
@@ -178,7 +178,7 @@ static void put_crypt_info(struct fscrypt_info *ci)
 	kmem_cache_free(fscrypt_info_cachep, ci);
 }
 
-int get_crypt_info(struct inode *inode)
+int fscrypt_get_crypt_info(struct inode *inode)
 {
 	struct fscrypt_info *crypt_info;
 	struct fscrypt_context ctx;
@@ -188,7 +188,7 @@ int get_crypt_info(struct inode *inode)
 	u8 *raw_key = NULL;
 	int res;
 
-	res = fscrypt_initialize();
+	res = fscrypt_initialize(inode->i_sb->s_cop->flags);
 	if (res)
 		return res;
 
@@ -327,7 +327,7 @@ int fscrypt_get_encryption_info(struct inode *inode)
 		 (ci->ci_keyring_key->flags & ((1 << KEY_FLAG_INVALIDATED) |
 					       (1 << KEY_FLAG_REVOKED) |
 					       (1 << KEY_FLAG_DEAD)))))
-		return get_crypt_info(inode);
+		return fscrypt_get_crypt_info(inode);
 	return 0;
 }
 EXPORT_SYMBOL(fscrypt_get_encryption_info);
diff --git a/fs/crypto/policy.c b/fs/crypto/policy.c
index 6865663aac69..6ed7c2eebeec 100644
--- a/fs/crypto/policy.c
+++ b/fs/crypto/policy.c
@@ -10,8 +10,8 @@
 
 #include <linux/random.h>
 #include <linux/string.h>
-#include <linux/fscrypto.h>
 #include <linux/mount.h>
+#include "fscrypt_private.h"
 
 static int inode_has_encryption_context(struct inode *inode)
 {
@@ -93,16 +93,19 @@ static int create_encryption_context_from_policy(struct inode *inode,
 	return inode->i_sb->s_cop->set_context(inode, &ctx, sizeof(ctx), NULL);
 }
 
-int fscrypt_process_policy(struct file *filp,
-				const struct fscrypt_policy *policy)
+int fscrypt_ioctl_set_policy(struct file *filp, const void __user *arg)
 {
+	struct fscrypt_policy policy;
 	struct inode *inode = file_inode(filp);
 	int ret;
 
+	if (copy_from_user(&policy, arg, sizeof(policy)))
+		return -EFAULT;
+
 	if (!inode_owner_or_capable(inode))
 		return -EACCES;
 
-	if (policy->version != 0)
+	if (policy.version != 0)
 		return -EINVAL;
 
 	ret = mnt_want_write_file(filp);
@@ -120,9 +123,9 @@ int fscrypt_process_policy(struct file *filp,
 			ret = -ENOTEMPTY;
 		else
 			ret = create_encryption_context_from_policy(inode,
-								    policy);
+								    &policy);
 	} else if (!is_encryption_context_consistent_with_policy(inode,
-								 policy)) {
+								 &policy)) {
 		printk(KERN_WARNING
 		       "%s: Policy inconsistent with encryption context\n",
 		       __func__);
@@ -134,11 +137,13 @@ int fscrypt_process_policy(struct file *filp,
 	mnt_drop_write_file(filp);
 	return ret;
 }
-EXPORT_SYMBOL(fscrypt_process_policy);
+EXPORT_SYMBOL(fscrypt_ioctl_set_policy);
 
-int fscrypt_get_policy(struct inode *inode, struct fscrypt_policy *policy)
+int fscrypt_ioctl_get_policy(struct file *filp, void __user *arg)
 {
+	struct inode *inode = file_inode(filp);
 	struct fscrypt_context ctx;
+	struct fscrypt_policy policy;
 	int res;
 
 	if (!inode->i_sb->s_cop->get_context ||
@@ -151,15 +156,18 @@ int fscrypt_get_policy(struct inode *inode, struct fscrypt_policy *policy)
 	if (ctx.format != FS_ENCRYPTION_CONTEXT_FORMAT_V1)
 		return -EINVAL;
 
-	policy->version = 0;
-	policy->contents_encryption_mode = ctx.contents_encryption_mode;
-	policy->filenames_encryption_mode = ctx.filenames_encryption_mode;
-	policy->flags = ctx.flags;
-	memcpy(&policy->master_key_descriptor, ctx.master_key_descriptor,
+	policy.version = 0;
+	policy.contents_encryption_mode = ctx.contents_encryption_mode;
+	policy.filenames_encryption_mode = ctx.filenames_encryption_mode;
+	policy.flags = ctx.flags;
+	memcpy(policy.master_key_descriptor, ctx.master_key_descriptor,
 				FS_KEY_DESCRIPTOR_SIZE);
+
+	if (copy_to_user(arg, &policy, sizeof(policy)))
+		return -EFAULT;
 	return 0;
 }
-EXPORT_SYMBOL(fscrypt_get_policy);
+EXPORT_SYMBOL(fscrypt_ioctl_get_policy);
 
 int fscrypt_has_permitted_context(struct inode *parent, struct inode *child)
 {
diff --git a/fs/dax.c b/fs/dax.c
index 6916ed37d463..a8732fbed381 100644
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -31,28 +31,15 @@
 #include <linux/vmstat.h>
 #include <linux/pfn_t.h>
 #include <linux/sizes.h>
+#include <linux/mmu_notifier.h>
 #include <linux/iomap.h>
 #include "internal.h"
 
-/*
- * We use lowest available bit in exceptional entry for locking, other two
- * bits to determine entry type. In total 3 special bits.
- */
-#define RADIX_DAX_SHIFT	(RADIX_TREE_EXCEPTIONAL_SHIFT + 3)
-#define RADIX_DAX_PTE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1))
-#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2))
-#define RADIX_DAX_TYPE_MASK (RADIX_DAX_PTE | RADIX_DAX_PMD)
-#define RADIX_DAX_TYPE(entry) ((unsigned long)entry & RADIX_DAX_TYPE_MASK)
-#define RADIX_DAX_SECTOR(entry) (((unsigned long)entry >> RADIX_DAX_SHIFT))
-#define RADIX_DAX_ENTRY(sector, pmd) ((void *)((unsigned long)sector << \
-		RADIX_DAX_SHIFT | (pmd ? RADIX_DAX_PMD : RADIX_DAX_PTE) | \
-		RADIX_TREE_EXCEPTIONAL_ENTRY))
-
 /* We choose 4096 entries - same as per-zone page wait tables */
 #define DAX_WAIT_TABLE_BITS 12
 #define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
 
-wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
+static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
 
 static int __init init_dax_wait_table(void)
 {
@@ -64,14 +51,6 @@ static int __init init_dax_wait_table(void)
 }
 fs_initcall(init_dax_wait_table);
 
-static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,
-					      pgoff_t index)
-{
-	unsigned long hash = hash_long((unsigned long)mapping ^ index,
-				       DAX_WAIT_TABLE_BITS);
-	return wait_table + hash;
-}
-
 static long dax_map_atomic(struct block_device *bdev, struct blk_dax_ctl *dax)
 {
 	struct request_queue *q = bdev->bd_queue;
@@ -98,209 +77,52 @@ static void dax_unmap_atomic(struct block_device *bdev,
 	blk_queue_exit(bdev->bd_queue);
 }
 
-struct page *read_dax_sector(struct block_device *bdev, sector_t n)
+static int dax_is_pmd_entry(void *entry)
 {
-	struct page *page = alloc_pages(GFP_KERNEL, 0);
-	struct blk_dax_ctl dax = {
-		.size = PAGE_SIZE,
-		.sector = n & ~((((int) PAGE_SIZE) / 512) - 1),
-	};
-	long rc;
-
-	if (!page)
-		return ERR_PTR(-ENOMEM);
-
-	rc = dax_map_atomic(bdev, &dax);
-	if (rc < 0)
-		return ERR_PTR(rc);
-	memcpy_from_pmem(page_address(page), dax.addr, PAGE_SIZE);
-	dax_unmap_atomic(bdev, &dax);
-	return page;
+	return (unsigned long)entry & RADIX_DAX_PMD;
 }
 
-static bool buffer_written(struct buffer_head *bh)
+static int dax_is_pte_entry(void *entry)
 {
-	return buffer_mapped(bh) && !buffer_unwritten(bh);
+	return !((unsigned long)entry & RADIX_DAX_PMD);
 }
 
-/*
- * When ext4 encounters a hole, it returns without modifying the buffer_head
- * which means that we can't trust b_size.  To cope with this, we set b_state
- * to 0 before calling get_block and, if any bit is set, we know we can trust
- * b_size.  Unfortunate, really, since ext4 knows precisely how long a hole is
- * and would save us time calling get_block repeatedly.
- */
-static bool buffer_size_valid(struct buffer_head *bh)
+static int dax_is_zero_entry(void *entry)
 {
-	return bh->b_state != 0;
+	return (unsigned long)entry & RADIX_DAX_HZP;
 }
 
-
-static sector_t to_sector(const struct buffer_head *bh,
-		const struct inode *inode)
+static int dax_is_empty_entry(void *entry)
 {
-	sector_t sector = bh->b_blocknr << (inode->i_blkbits - 9);
-
-	return sector;
+	return (unsigned long)entry & RADIX_DAX_EMPTY;
 }
 
-static ssize_t dax_io(struct inode *inode, struct iov_iter *iter,
-		      loff_t start, loff_t end, get_block_t get_block,
-		      struct buffer_head *bh)
+struct page *read_dax_sector(struct block_device *bdev, sector_t n)
 {
-	loff_t pos = start, max = start, bh_max = start;
-	bool hole = false;
-	struct block_device *bdev = NULL;
-	int rw = iov_iter_rw(iter), rc;
-	long map_len = 0;
+	struct page *page = alloc_pages(GFP_KERNEL, 0);
 	struct blk_dax_ctl dax = {
-		.addr = ERR_PTR(-EIO),
+		.size = PAGE_SIZE,
+		.sector = n & ~((((int) PAGE_SIZE) / 512) - 1),
 	};
-	unsigned blkbits = inode->i_blkbits;
-	sector_t file_blks = (i_size_read(inode) + (1 << blkbits) - 1)
-								>> blkbits;
-
-	if (rw == READ)
-		end = min(end, i_size_read(inode));
-
-	while (pos < end) {
-		size_t len;
-		if (pos == max) {
-			long page = pos >> PAGE_SHIFT;
-			sector_t block = page << (PAGE_SHIFT - blkbits);
-			unsigned first = pos - (block << blkbits);
-			long size;
-
-			if (pos == bh_max) {
-				bh->b_size = PAGE_ALIGN(end - pos);
-				bh->b_state = 0;
-				rc = get_block(inode, block, bh, rw == WRITE);
-				if (rc)
-					break;
-				if (!buffer_size_valid(bh))
-					bh->b_size = 1 << blkbits;
-				bh_max = pos - first + bh->b_size;
-				bdev = bh->b_bdev;
-				/*
-				 * We allow uninitialized buffers for writes
-				 * beyond EOF as those cannot race with faults
-				 */
-				WARN_ON_ONCE(
-					(buffer_new(bh) && block < file_blks) ||
-					(rw == WRITE && buffer_unwritten(bh)));
-			} else {
-				unsigned done = bh->b_size -
-						(bh_max - (pos - first));
-				bh->b_blocknr += done >> blkbits;
-				bh->b_size -= done;
-			}
-
-			hole = rw == READ && !buffer_written(bh);
-			if (hole) {
-				size = bh->b_size - first;
-			} else {
-				dax_unmap_atomic(bdev, &dax);
-				dax.sector = to_sector(bh, inode);
-				dax.size = bh->b_size;
-				map_len = dax_map_atomic(bdev, &dax);
-				if (map_len < 0) {
-					rc = map_len;
-					break;
-				}
-				dax.addr += first;
-				size = map_len - first;
-			}
-			/*
-			 * pos + size is one past the last offset for IO,
-			 * so pos + size can overflow loff_t at extreme offsets.
-			 * Cast to u64 to catch this and get the true minimum.
-			 */
-			max = min_t(u64, pos + size, end);
-		}
-
-		if (iov_iter_rw(iter) == WRITE) {
-			len = copy_from_iter_pmem(dax.addr, max - pos, iter);
-		} else if (!hole)
-			len = copy_to_iter((void __force *) dax.addr, max - pos,
-					iter);
-		else
-			len = iov_iter_zero(max - pos, iter);
-
-		if (!len) {
-			rc = -EFAULT;
-			break;
-		}
+	long rc;
 
-		pos += len;
-		if (!IS_ERR(dax.addr))
-			dax.addr += len;
-	}
+	if (!page)
+		return ERR_PTR(-ENOMEM);
 
+	rc = dax_map_atomic(bdev, &dax);
+	if (rc < 0)
+		return ERR_PTR(rc);
+	memcpy_from_pmem(page_address(page), dax.addr, PAGE_SIZE);
 	dax_unmap_atomic(bdev, &dax);
-
-	return (pos == start) ? rc : pos - start;
-}
-
-/**
- * dax_do_io - Perform I/O to a DAX file
- * @iocb: The control block for this I/O
- * @inode: The file which the I/O is directed at
- * @iter: The addresses to do I/O from or to
- * @get_block: The filesystem method used to translate file offsets to blocks
- * @end_io: A filesystem callback for I/O completion
- * @flags: See below
- *
- * This function uses the same locking scheme as do_blockdev_direct_IO:
- * If @flags has DIO_LOCKING set, we assume that the i_mutex is held by the
- * caller for writes.  For reads, we take and release the i_mutex ourselves.
- * If DIO_LOCKING is not set, the filesystem takes care of its own locking.
- * As with do_blockdev_direct_IO(), we increment i_dio_count while the I/O
- * is in progress.
- */
-ssize_t dax_do_io(struct kiocb *iocb, struct inode *inode,
-		  struct iov_iter *iter, get_block_t get_block,
-		  dio_iodone_t end_io, int flags)
-{
-	struct buffer_head bh;
-	ssize_t retval = -EINVAL;
-	loff_t pos = iocb->ki_pos;
-	loff_t end = pos + iov_iter_count(iter);
-
-	memset(&bh, 0, sizeof(bh));
-	bh.b_bdev = inode->i_sb->s_bdev;
-
-	if ((flags & DIO_LOCKING) && iov_iter_rw(iter) == READ)
-		inode_lock(inode);
-
-	/* Protects against truncate */
-	if (!(flags & DIO_SKIP_DIO_COUNT))
-		inode_dio_begin(inode);
-
-	retval = dax_io(inode, iter, pos, end, get_block, &bh);
-
-	if ((flags & DIO_LOCKING) && iov_iter_rw(iter) == READ)
-		inode_unlock(inode);
-
-	if (end_io) {
-		int err;
-
-		err = end_io(iocb, pos, retval, bh.b_private);
-		if (err)
-			retval = err;
-	}
-
-	if (!(flags & DIO_SKIP_DIO_COUNT))
-		inode_dio_end(inode);
-	return retval;
+	return page;
 }
-EXPORT_SYMBOL_GPL(dax_do_io);
 
 /*
  * DAX radix tree locking
  */
 struct exceptional_entry_key {
 	struct address_space *mapping;
-	unsigned long index;
+	pgoff_t entry_start;
 };
 
 struct wait_exceptional_entry_queue {
@@ -308,6 +130,26 @@ struct wait_exceptional_entry_queue {
 	struct exceptional_entry_key key;
 };
 
+static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,
+		pgoff_t index, void *entry, struct exceptional_entry_key *key)
+{
+	unsigned long hash;
+
+	/*
+	 * If 'entry' is a PMD, align the 'index' that we use for the wait
+	 * queue to the start of that PMD.  This ensures that all offsets in
+	 * the range covered by the PMD map to the same bit lock.
+	 */
+	if (dax_is_pmd_entry(entry))
+		index &= ~((1UL << (PMD_SHIFT - PAGE_SHIFT)) - 1);
+
+	key->mapping = mapping;
+	key->entry_start = index;
+
+	hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS);
+	return wait_table + hash;
+}
+
 static int wake_exceptional_entry_func(wait_queue_t *wait, unsigned int mode,
 				       int sync, void *keyp)
 {
@@ -316,7 +158,7 @@ static int wake_exceptional_entry_func(wait_queue_t *wait, unsigned int mode,
 		container_of(wait, struct wait_exceptional_entry_queue, wait);
 
 	if (key->mapping != ewait->key.mapping ||
-	    key->index != ewait->key.index)
+	    key->entry_start != ewait->key.entry_start)
 		return 0;
 	return autoremove_wake_function(wait, mode, sync, NULL);
 }
@@ -372,24 +214,24 @@ static inline void *unlock_slot(struct address_space *mapping, void **slot)
 static void *get_unlocked_mapping_entry(struct address_space *mapping,
 					pgoff_t index, void ***slotp)
 {
-	void *ret, **slot;
+	void *entry, **slot;
 	struct wait_exceptional_entry_queue ewait;
-	wait_queue_head_t *wq = dax_entry_waitqueue(mapping, index);
+	wait_queue_head_t *wq;
 
 	init_wait(&ewait.wait);
 	ewait.wait.func = wake_exceptional_entry_func;
-	ewait.key.mapping = mapping;
-	ewait.key.index = index;
 
 	for (;;) {
-		ret = __radix_tree_lookup(&mapping->page_tree, index, NULL,
+		entry = __radix_tree_lookup(&mapping->page_tree, index, NULL,
 					  &slot);
-		if (!ret || !radix_tree_exceptional_entry(ret) ||
+		if (!entry || !radix_tree_exceptional_entry(entry) ||
 		    !slot_locked(mapping, slot)) {
 			if (slotp)
 				*slotp = slot;
-			return ret;
+			return entry;
 		}
+
+		wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key);
 		prepare_to_wait_exclusive(wq, &ewait.wait,
 					  TASK_UNINTERRUPTIBLE);
 		spin_unlock_irq(&mapping->tree_lock);
@@ -399,52 +241,173 @@ static void *get_unlocked_mapping_entry(struct address_space *mapping,
 	}
 }
 
+static void dax_unlock_mapping_entry(struct address_space *mapping,
+				     pgoff_t index)
+{
+	void *entry, **slot;
+
+	spin_lock_irq(&mapping->tree_lock);
+	entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot);
+	if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) ||
+			 !slot_locked(mapping, slot))) {
+		spin_unlock_irq(&mapping->tree_lock);
+		return;
+	}
+	unlock_slot(mapping, slot);
+	spin_unlock_irq(&mapping->tree_lock);
+	dax_wake_mapping_entry_waiter(mapping, index, entry, false);
+}
+
+static void put_locked_mapping_entry(struct address_space *mapping,
+				     pgoff_t index, void *entry)
+{
+	if (!radix_tree_exceptional_entry(entry)) {
+		unlock_page(entry);
+		put_page(entry);
+	} else {
+		dax_unlock_mapping_entry(mapping, index);
+	}
+}
+
+/*
+ * Called when we are done with radix tree entry we looked up via
+ * get_unlocked_mapping_entry() and which we didn't lock in the end.
+ */
+static void put_unlocked_mapping_entry(struct address_space *mapping,
+				       pgoff_t index, void *entry)
+{
+	if (!radix_tree_exceptional_entry(entry))
+		return;
+
+	/* We have to wake up next waiter for the radix tree entry lock */
+	dax_wake_mapping_entry_waiter(mapping, index, entry, false);
+}
+
 /*
  * Find radix tree entry at given index. If it points to a page, return with
  * the page locked. If it points to the exceptional entry, return with the
  * radix tree entry locked. If the radix tree doesn't contain given index,
  * create empty exceptional entry for the index and return with it locked.
  *
+ * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will
+ * either return that locked entry or will return an error.  This error will
+ * happen if there are any 4k entries (either zero pages or DAX entries)
+ * within the 2MiB range that we are requesting.
+ *
+ * We always favor 4k entries over 2MiB entries. There isn't a flow where we
+ * evict 4k entries in order to 'upgrade' them to a 2MiB entry.  A 2MiB
+ * insertion will fail if it finds any 4k entries already in the tree, and a
+ * 4k insertion will cause an existing 2MiB entry to be unmapped and
+ * downgraded to 4k entries.  This happens for both 2MiB huge zero pages as
+ * well as 2MiB empty entries.
+ *
+ * The exception to this downgrade path is for 2MiB DAX PMD entries that have
+ * real storage backing them.  We will leave these real 2MiB DAX entries in
+ * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry.
+ *
  * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
  * persistent memory the benefit is doubtful. We can add that later if we can
  * show it helps.
  */
-static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index)
+static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index,
+		unsigned long size_flag)
 {
-	void *ret, **slot;
+	bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */
+	void *entry, **slot;
 
 restart:
 	spin_lock_irq(&mapping->tree_lock);
-	ret = get_unlocked_mapping_entry(mapping, index, &slot);
+	entry = get_unlocked_mapping_entry(mapping, index, &slot);
+
+	if (entry) {
+		if (size_flag & RADIX_DAX_PMD) {
+			if (!radix_tree_exceptional_entry(entry) ||
+			    dax_is_pte_entry(entry)) {
+				put_unlocked_mapping_entry(mapping, index,
+						entry);
+				entry = ERR_PTR(-EEXIST);
+				goto out_unlock;
+			}
+		} else { /* trying to grab a PTE entry */
+			if (radix_tree_exceptional_entry(entry) &&
+			    dax_is_pmd_entry(entry) &&
+			    (dax_is_zero_entry(entry) ||
+			     dax_is_empty_entry(entry))) {
+				pmd_downgrade = true;
+			}
+		}
+	}
+
 	/* No entry for given index? Make sure radix tree is big enough. */
-	if (!ret) {
+	if (!entry || pmd_downgrade) {
 		int err;
 
+		if (pmd_downgrade) {
+			/*
+			 * Make sure 'entry' remains valid while we drop
+			 * mapping->tree_lock.
+			 */
+			entry = lock_slot(mapping, slot);
+		}
+
 		spin_unlock_irq(&mapping->tree_lock);
+		/*
+		 * Besides huge zero pages the only other thing that gets
+		 * downgraded are empty entries which don't need to be
+		 * unmapped.
+		 */
+		if (pmd_downgrade && dax_is_zero_entry(entry))
+			unmap_mapping_range(mapping,
+				(index << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0);
+
 		err = radix_tree_preload(
 				mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);
-		if (err)
+		if (err) {
+			if (pmd_downgrade)
+				put_locked_mapping_entry(mapping, index, entry);
 			return ERR_PTR(err);
-		ret = (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY |
-			       RADIX_DAX_ENTRY_LOCK);
+		}
 		spin_lock_irq(&mapping->tree_lock);
-		err = radix_tree_insert(&mapping->page_tree, index, ret);
+
+		if (pmd_downgrade) {
+			radix_tree_delete(&mapping->page_tree, index);
+			mapping->nrexceptional--;
+			dax_wake_mapping_entry_waiter(mapping, index, entry,
+					true);
+		}
+
+		entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY);
+
+		err = __radix_tree_insert(&mapping->page_tree, index,
+				dax_radix_order(entry), entry);
 		radix_tree_preload_end();
 		if (err) {
 			spin_unlock_irq(&mapping->tree_lock);
-			/* Someone already created the entry? */
-			if (err == -EEXIST)
+			/*
+			 * Someone already created the entry?  This is a
+			 * normal failure when inserting PMDs in a range
+			 * that already contains PTEs.  In that case we want
+			 * to return -EEXIST immediately.
+			 */
+			if (err == -EEXIST && !(size_flag & RADIX_DAX_PMD))
 				goto restart;
+			/*
+			 * Our insertion of a DAX PMD entry failed, most
+			 * likely because it collided with a PTE sized entry
+			 * at a different index in the PMD range.  We haven't
+			 * inserted anything into the radix tree and have no
+			 * waiters to wake.
+			 */
 			return ERR_PTR(err);
 		}
 		/* Good, we have inserted empty locked entry into the tree. */
 		mapping->nrexceptional++;
 		spin_unlock_irq(&mapping->tree_lock);
-		return ret;
+		return entry;
 	}
 	/* Normal page in radix tree? */
-	if (!radix_tree_exceptional_entry(ret)) {
-		struct page *page = ret;
+	if (!radix_tree_exceptional_entry(entry)) {
+		struct page *page = entry;
 
 		get_page(page);
 		spin_unlock_irq(&mapping->tree_lock);
@@ -457,15 +420,26 @@ restart:
 		}
 		return page;
 	}
-	ret = lock_slot(mapping, slot);
+	entry = lock_slot(mapping, slot);
+ out_unlock:
 	spin_unlock_irq(&mapping->tree_lock);
-	return ret;
+	return entry;
 }
 
+/*
+ * We do not necessarily hold the mapping->tree_lock when we call this
+ * function so it is possible that 'entry' is no longer a valid item in the
+ * radix tree.  This is okay because all we really need to do is to find the
+ * correct waitqueue where tasks might be waiting for that old 'entry' and
+ * wake them.
+ */
 void dax_wake_mapping_entry_waiter(struct address_space *mapping,
-				   pgoff_t index, bool wake_all)
+		pgoff_t index, void *entry, bool wake_all)
 {
-	wait_queue_head_t *wq = dax_entry_waitqueue(mapping, index);
+	struct exceptional_entry_key key;
+	wait_queue_head_t *wq;
+
+	wq = dax_entry_waitqueue(mapping, index, entry, &key);
 
 	/*
 	 * Checking for locked entry and prepare_to_wait_exclusive() happens
@@ -473,54 +447,8 @@ void dax_wake_mapping_entry_waiter(struct address_space *mapping,
 	 * So at this point all tasks that could have seen our entry locked
 	 * must be in the waitqueue and the following check will see them.
 	 */
-	if (waitqueue_active(wq)) {
-		struct exceptional_entry_key key;
-
-		key.mapping = mapping;
-		key.index = index;
+	if (waitqueue_active(wq))
 		__wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
-	}
-}
-
-void dax_unlock_mapping_entry(struct address_space *mapping, pgoff_t index)
-{
-	void *ret, **slot;
-
-	spin_lock_irq(&mapping->tree_lock);
-	ret = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot);
-	if (WARN_ON_ONCE(!ret || !radix_tree_exceptional_entry(ret) ||
-			 !slot_locked(mapping, slot))) {
-		spin_unlock_irq(&mapping->tree_lock);
-		return;
-	}
-	unlock_slot(mapping, slot);
-	spin_unlock_irq(&mapping->tree_lock);
-	dax_wake_mapping_entry_waiter(mapping, index, false);
-}
-
-static void put_locked_mapping_entry(struct address_space *mapping,
-				     pgoff_t index, void *entry)
-{
-	if (!radix_tree_exceptional_entry(entry)) {
-		unlock_page(entry);
-		put_page(entry);
-	} else {
-		dax_unlock_mapping_entry(mapping, index);
-	}
-}
-
-/*
- * Called when we are done with radix tree entry we looked up via
- * get_unlocked_mapping_entry() and which we didn't lock in the end.
- */
-static void put_unlocked_mapping_entry(struct address_space *mapping,
-				       pgoff_t index, void *entry)
-{
-	if (!radix_tree_exceptional_entry(entry))
-		return;
-
-	/* We have to wake up next waiter for the radix tree entry lock */
-	dax_wake_mapping_entry_waiter(mapping, index, false);
 }
 
 /*
@@ -547,7 +475,7 @@ int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
 	radix_tree_delete(&mapping->page_tree, index);
 	mapping->nrexceptional--;
 	spin_unlock_irq(&mapping->tree_lock);
-	dax_wake_mapping_entry_waiter(mapping, index, true);
+	dax_wake_mapping_entry_waiter(mapping, index, entry, true);
 
 	return 1;
 }
@@ -574,10 +502,8 @@ static int dax_load_hole(struct address_space *mapping, void *entry,
 	/* This will replace locked radix tree entry with a hole page */
 	page = find_or_create_page(mapping, vmf->pgoff,
 				   vmf->gfp_mask | __GFP_ZERO);
-	if (!page) {
-		put_locked_mapping_entry(mapping, vmf->pgoff, entry);
+	if (!page)
 		return VM_FAULT_OOM;
-	}
 	vmf->page = page;
 	return VM_FAULT_LOCKED;
 }
@@ -600,11 +526,17 @@ static int copy_user_dax(struct block_device *bdev, sector_t sector, size_t size
 	return 0;
 }
 
-#define DAX_PMD_INDEX(page_index) (page_index & (PMD_MASK >> PAGE_SHIFT))
-
+/*
+ * By this point grab_mapping_entry() has ensured that we have a locked entry
+ * of the appropriate size so we don't have to worry about downgrading PMDs to
+ * PTEs.  If we happen to be trying to insert a PTE and there is a PMD
+ * already in the tree, we will skip the insertion and just dirty the PMD as
+ * appropriate.
+ */
 static void *dax_insert_mapping_entry(struct address_space *mapping,
 				      struct vm_fault *vmf,
-				      void *entry, sector_t sector)
+				      void *entry, sector_t sector,
+				      unsigned long flags)
 {
 	struct radix_tree_root *page_tree = &mapping->page_tree;
 	int error = 0;
@@ -627,22 +559,35 @@ static void *dax_insert_mapping_entry(struct address_space *mapping,
 		error = radix_tree_preload(vmf->gfp_mask & ~__GFP_HIGHMEM);
 		if (error)
 			return ERR_PTR(error);
+	} else if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_HZP)) {
+		/* replacing huge zero page with PMD block mapping */
+		unmap_mapping_range(mapping,
+			(vmf->pgoff << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0);
 	}
 
 	spin_lock_irq(&mapping->tree_lock);
-	new_entry = (void *)((unsigned long)RADIX_DAX_ENTRY(sector, false) |
-		       RADIX_DAX_ENTRY_LOCK);
+	new_entry = dax_radix_locked_entry(sector, flags);
+
 	if (hole_fill) {
 		__delete_from_page_cache(entry, NULL);
 		/* Drop pagecache reference */
 		put_page(entry);
-		error = radix_tree_insert(page_tree, index, new_entry);
+		error = __radix_tree_insert(page_tree, index,
+				dax_radix_order(new_entry), new_entry);
 		if (error) {
 			new_entry = ERR_PTR(error);
 			goto unlock;
 		}
 		mapping->nrexceptional++;
-	} else {
+	} else if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
+		/*
+		 * Only swap our new entry into the radix tree if the current
+		 * entry is a zero page or an empty entry.  If a normal PTE or
+		 * PMD entry is already in the tree, we leave it alone.  This
+		 * means that if we are trying to insert a PTE and the
+		 * existing entry is a PMD, we will just leave the PMD in the
+		 * tree and dirty it if necessary.
+		 */
 		struct radix_tree_node *node;
 		void **slot;
 		void *ret;
@@ -670,63 +615,150 @@ static void *dax_insert_mapping_entry(struct address_space *mapping,
 	return new_entry;
 }
 
+static inline unsigned long
+pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
+{
+	unsigned long address;
+
+	address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
+	VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
+	return address;
+}
+
+/* Walk all mappings of a given index of a file and writeprotect them */
+static void dax_mapping_entry_mkclean(struct address_space *mapping,
+				      pgoff_t index, unsigned long pfn)
+{
+	struct vm_area_struct *vma;
+	pte_t *ptep;
+	pte_t pte;
+	spinlock_t *ptl;
+	bool changed;
+
+	i_mmap_lock_read(mapping);
+	vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
+		unsigned long address;
+
+		cond_resched();
+
+		if (!(vma->vm_flags & VM_SHARED))
+			continue;
+
+		address = pgoff_address(index, vma);
+		changed = false;
+		if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
+			continue;
+		if (pfn != pte_pfn(*ptep))
+			goto unlock;
+		if (!pte_dirty(*ptep) && !pte_write(*ptep))
+			goto unlock;
+
+		flush_cache_page(vma, address, pfn);
+		pte = ptep_clear_flush(vma, address, ptep);
+		pte = pte_wrprotect(pte);
+		pte = pte_mkclean(pte);
+		set_pte_at(vma->vm_mm, address, ptep, pte);
+		changed = true;
+unlock:
+		pte_unmap_unlock(ptep, ptl);
+
+		if (changed)
+			mmu_notifier_invalidate_page(vma->vm_mm, address);
+	}
+	i_mmap_unlock_read(mapping);
+}
+
 static int dax_writeback_one(struct block_device *bdev,
 		struct address_space *mapping, pgoff_t index, void *entry)
 {
 	struct radix_tree_root *page_tree = &mapping->page_tree;
-	int type = RADIX_DAX_TYPE(entry);
-	struct radix_tree_node *node;
 	struct blk_dax_ctl dax;
-	void **slot;
+	void *entry2, **slot;
 	int ret = 0;
 
-	spin_lock_irq(&mapping->tree_lock);
 	/*
-	 * Regular page slots are stabilized by the page lock even
-	 * without the tree itself locked.  These unlocked entries
-	 * need verification under the tree lock.
+	 * A page got tagged dirty in DAX mapping? Something is seriously
+	 * wrong.
 	 */
-	if (!__radix_tree_lookup(page_tree, index, &node, &slot))
-		goto unlock;
-	if (*slot != entry)
-		goto unlock;
-
-	/* another fsync thread may have already written back this entry */
-	if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
-		goto unlock;
+	if (WARN_ON(!radix_tree_exceptional_entry(entry)))
+		return -EIO;
 
-	if (WARN_ON_ONCE(type != RADIX_DAX_PTE && type != RADIX_DAX_PMD)) {
+	spin_lock_irq(&mapping->tree_lock);
+	entry2 = get_unlocked_mapping_entry(mapping, index, &slot);
+	/* Entry got punched out / reallocated? */
+	if (!entry2 || !radix_tree_exceptional_entry(entry2))
+		goto put_unlocked;
+	/*
+	 * Entry got reallocated elsewhere? No need to writeback. We have to
+	 * compare sectors as we must not bail out due to difference in lockbit
+	 * or entry type.
+	 */
+	if (dax_radix_sector(entry2) != dax_radix_sector(entry))
+		goto put_unlocked;
+	if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
+				dax_is_zero_entry(entry))) {
 		ret = -EIO;
-		goto unlock;
+		goto put_unlocked;
 	}
 
-	dax.sector = RADIX_DAX_SECTOR(entry);
-	dax.size = (type == RADIX_DAX_PMD ? PMD_SIZE : PAGE_SIZE);
+	/* Another fsync thread may have already written back this entry */
+	if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
+		goto put_unlocked;
+	/* Lock the entry to serialize with page faults */
+	entry = lock_slot(mapping, slot);
+	/*
+	 * We can clear the tag now but we have to be careful so that concurrent
+	 * dax_writeback_one() calls for the same index cannot finish before we
+	 * actually flush the caches. This is achieved as the calls will look
+	 * at the entry only under tree_lock and once they do that they will
+	 * see the entry locked and wait for it to unlock.
+	 */
+	radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE);
 	spin_unlock_irq(&mapping->tree_lock);
 
 	/*
+	 * Even if dax_writeback_mapping_range() was given a wbc->range_start
+	 * in the middle of a PMD, the 'index' we are given will be aligned to
+	 * the start index of the PMD, as will the sector we pull from
+	 * 'entry'.  This allows us to flush for PMD_SIZE and not have to
+	 * worry about partial PMD writebacks.
+	 */
+	dax.sector = dax_radix_sector(entry);
+	dax.size = PAGE_SIZE << dax_radix_order(entry);
+
+	/*
 	 * We cannot hold tree_lock while calling dax_map_atomic() because it
 	 * eventually calls cond_resched().
 	 */
 	ret = dax_map_atomic(bdev, &dax);
-	if (ret < 0)
+	if (ret < 0) {
+		put_locked_mapping_entry(mapping, index, entry);
 		return ret;
+	}
 
 	if (WARN_ON_ONCE(ret < dax.size)) {
 		ret = -EIO;
 		goto unmap;
 	}
 
+	dax_mapping_entry_mkclean(mapping, index, pfn_t_to_pfn(dax.pfn));
 	wb_cache_pmem(dax.addr, dax.size);
-
+	/*
+	 * After we have flushed the cache, we can clear the dirty tag. There
+	 * cannot be new dirty data in the pfn after the flush has completed as
+	 * the pfn mappings are writeprotected and fault waits for mapping
+	 * entry lock.
+	 */
 	spin_lock_irq(&mapping->tree_lock);
-	radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE);
+	radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY);
 	spin_unlock_irq(&mapping->tree_lock);
  unmap:
 	dax_unmap_atomic(bdev, &dax);
+	put_locked_mapping_entry(mapping, index, entry);
 	return ret;
 
- unlock:
+ put_unlocked:
+	put_unlocked_mapping_entry(mapping, index, entry2);
 	spin_unlock_irq(&mapping->tree_lock);
 	return ret;
 }
@@ -740,12 +772,11 @@ int dax_writeback_mapping_range(struct address_space *mapping,
 		struct block_device *bdev, struct writeback_control *wbc)
 {
 	struct inode *inode = mapping->host;
-	pgoff_t start_index, end_index, pmd_index;
+	pgoff_t start_index, end_index;
 	pgoff_t indices[PAGEVEC_SIZE];
 	struct pagevec pvec;
 	bool done = false;
 	int i, ret = 0;
-	void *entry;
 
 	if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
 		return -EIO;
@@ -755,15 +786,6 @@ int dax_writeback_mapping_range(struct address_space *mapping,
 
 	start_index = wbc->range_start >> PAGE_SHIFT;
 	end_index = wbc->range_end >> PAGE_SHIFT;
-	pmd_index = DAX_PMD_INDEX(start_index);
-
-	rcu_read_lock();
-	entry = radix_tree_lookup(&mapping->page_tree, pmd_index);
-	rcu_read_unlock();
-
-	/* see if the start of our range is covered by a PMD entry */
-	if (entry && RADIX_DAX_TYPE(entry) == RADIX_DAX_PMD)
-		start_index = pmd_index;
 
 	tag_pages_for_writeback(mapping, start_index, end_index);
 
@@ -796,7 +818,7 @@ static int dax_insert_mapping(struct address_space *mapping,
 		struct block_device *bdev, sector_t sector, size_t size,
 		void **entryp, struct vm_area_struct *vma, struct vm_fault *vmf)
 {
-	unsigned long vaddr = (unsigned long)vmf->virtual_address;
+	unsigned long vaddr = vmf->address;
 	struct blk_dax_ctl dax = {
 		.sector = sector,
 		.size = size,
@@ -808,7 +830,7 @@ static int dax_insert_mapping(struct address_space *mapping,
 		return PTR_ERR(dax.addr);
 	dax_unmap_atomic(bdev, &dax);
 
-	ret = dax_insert_mapping_entry(mapping, vmf, entry, dax.sector);
+	ret = dax_insert_mapping_entry(mapping, vmf, entry, dax.sector, 0);
 	if (IS_ERR(ret))
 		return PTR_ERR(ret);
 	*entryp = ret;
@@ -817,323 +839,6 @@ static int dax_insert_mapping(struct address_space *mapping,
 }
 
 /**
- * dax_fault - handle a page fault on a DAX file
- * @vma: The virtual memory area where the fault occurred
- * @vmf: The description of the fault
- * @get_block: The filesystem method used to translate file offsets to blocks
- *
- * When a page fault occurs, filesystems may call this helper in their
- * fault handler for DAX files. dax_fault() assumes the caller has done all
- * the necessary locking for the page fault to proceed successfully.
- */
-int dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
-			get_block_t get_block)
-{
-	struct file *file = vma->vm_file;
-	struct address_space *mapping = file->f_mapping;
-	struct inode *inode = mapping->host;
-	void *entry;
-	struct buffer_head bh;
-	unsigned long vaddr = (unsigned long)vmf->virtual_address;
-	unsigned blkbits = inode->i_blkbits;
-	sector_t block;
-	pgoff_t size;
-	int error;
-	int major = 0;
-
-	/*
-	 * Check whether offset isn't beyond end of file now. Caller is supposed
-	 * to hold locks serializing us with truncate / punch hole so this is
-	 * a reliable test.
-	 */
-	size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
-	if (vmf->pgoff >= size)
-		return VM_FAULT_SIGBUS;
-
-	memset(&bh, 0, sizeof(bh));
-	block = (sector_t)vmf->pgoff << (PAGE_SHIFT - blkbits);
-	bh.b_bdev = inode->i_sb->s_bdev;
-	bh.b_size = PAGE_SIZE;
-
-	entry = grab_mapping_entry(mapping, vmf->pgoff);
-	if (IS_ERR(entry)) {
-		error = PTR_ERR(entry);
-		goto out;
-	}
-
-	error = get_block(inode, block, &bh, 0);
-	if (!error && (bh.b_size < PAGE_SIZE))
-		error = -EIO;		/* fs corruption? */
-	if (error)
-		goto unlock_entry;
-
-	if (vmf->cow_page) {
-		struct page *new_page = vmf->cow_page;
-		if (buffer_written(&bh))
-			error = copy_user_dax(bh.b_bdev, to_sector(&bh, inode),
-					bh.b_size, new_page, vaddr);
-		else
-			clear_user_highpage(new_page, vaddr);
-		if (error)
-			goto unlock_entry;
-		if (!radix_tree_exceptional_entry(entry)) {
-			vmf->page = entry;
-			return VM_FAULT_LOCKED;
-		}
-		vmf->entry = entry;
-		return VM_FAULT_DAX_LOCKED;
-	}
-
-	if (!buffer_mapped(&bh)) {
-		if (vmf->flags & FAULT_FLAG_WRITE) {
-			error = get_block(inode, block, &bh, 1);
-			count_vm_event(PGMAJFAULT);
-			mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
-			major = VM_FAULT_MAJOR;
-			if (!error && (bh.b_size < PAGE_SIZE))
-				error = -EIO;
-			if (error)
-				goto unlock_entry;
-		} else {
-			return dax_load_hole(mapping, entry, vmf);
-		}
-	}
-
-	/* Filesystem should not return unwritten buffers to us! */
-	WARN_ON_ONCE(buffer_unwritten(&bh) || buffer_new(&bh));
-	error = dax_insert_mapping(mapping, bh.b_bdev, to_sector(&bh, inode),
-			bh.b_size, &entry, vma, vmf);
- unlock_entry:
-	put_locked_mapping_entry(mapping, vmf->pgoff, entry);
- out:
-	if (error == -ENOMEM)
-		return VM_FAULT_OOM | major;
-	/* -EBUSY is fine, somebody else faulted on the same PTE */
-	if ((error < 0) && (error != -EBUSY))
-		return VM_FAULT_SIGBUS | major;
-	return VM_FAULT_NOPAGE | major;
-}
-EXPORT_SYMBOL_GPL(dax_fault);
-
-#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
-/*
- * The 'colour' (ie low bits) within a PMD of a page offset.  This comes up
- * more often than one might expect in the below function.
- */
-#define PG_PMD_COLOUR	((PMD_SIZE >> PAGE_SHIFT) - 1)
-
-static void __dax_dbg(struct buffer_head *bh, unsigned long address,
-		const char *reason, const char *fn)
-{
-	if (bh) {
-		char bname[BDEVNAME_SIZE];
-		bdevname(bh->b_bdev, bname);
-		pr_debug("%s: %s addr: %lx dev %s state %lx start %lld "
-			"length %zd fallback: %s\n", fn, current->comm,
-			address, bname, bh->b_state, (u64)bh->b_blocknr,
-			bh->b_size, reason);
-	} else {
-		pr_debug("%s: %s addr: %lx fallback: %s\n", fn,
-			current->comm, address, reason);
-	}
-}
-
-#define dax_pmd_dbg(bh, address, reason)	__dax_dbg(bh, address, reason, "dax_pmd")
-
-/**
- * dax_pmd_fault - handle a PMD fault on a DAX file
- * @vma: The virtual memory area where the fault occurred
- * @vmf: The description of the fault
- * @get_block: The filesystem method used to translate file offsets to blocks
- *
- * When a page fault occurs, filesystems may call this helper in their
- * pmd_fault handler for DAX files.
- */
-int dax_pmd_fault(struct vm_area_struct *vma, unsigned long address,
-		pmd_t *pmd, unsigned int flags, get_block_t get_block)
-{
-	struct file *file = vma->vm_file;
-	struct address_space *mapping = file->f_mapping;
-	struct inode *inode = mapping->host;
-	struct buffer_head bh;
-	unsigned blkbits = inode->i_blkbits;
-	unsigned long pmd_addr = address & PMD_MASK;
-	bool write = flags & FAULT_FLAG_WRITE;
-	struct block_device *bdev;
-	pgoff_t size, pgoff;
-	sector_t block;
-	int result = 0;
-	bool alloc = false;
-
-	/* dax pmd mappings require pfn_t_devmap() */
-	if (!IS_ENABLED(CONFIG_FS_DAX_PMD))
-		return VM_FAULT_FALLBACK;
-
-	/* Fall back to PTEs if we're going to COW */
-	if (write && !(vma->vm_flags & VM_SHARED)) {
-		split_huge_pmd(vma, pmd, address);
-		dax_pmd_dbg(NULL, address, "cow write");
-		return VM_FAULT_FALLBACK;
-	}
-	/* If the PMD would extend outside the VMA */
-	if (pmd_addr < vma->vm_start) {
-		dax_pmd_dbg(NULL, address, "vma start unaligned");
-		return VM_FAULT_FALLBACK;
-	}
-	if ((pmd_addr + PMD_SIZE) > vma->vm_end) {
-		dax_pmd_dbg(NULL, address, "vma end unaligned");
-		return VM_FAULT_FALLBACK;
-	}
-
-	pgoff = linear_page_index(vma, pmd_addr);
-	size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
-	if (pgoff >= size)
-		return VM_FAULT_SIGBUS;
-	/* If the PMD would cover blocks out of the file */
-	if ((pgoff | PG_PMD_COLOUR) >= size) {
-		dax_pmd_dbg(NULL, address,
-				"offset + huge page size > file size");
-		return VM_FAULT_FALLBACK;
-	}
-
-	memset(&bh, 0, sizeof(bh));
-	bh.b_bdev = inode->i_sb->s_bdev;
-	block = (sector_t)pgoff << (PAGE_SHIFT - blkbits);
-
-	bh.b_size = PMD_SIZE;
-
-	if (get_block(inode, block, &bh, 0) != 0)
-		return VM_FAULT_SIGBUS;
-
-	if (!buffer_mapped(&bh) && write) {
-		if (get_block(inode, block, &bh, 1) != 0)
-			return VM_FAULT_SIGBUS;
-		alloc = true;
-		WARN_ON_ONCE(buffer_unwritten(&bh) || buffer_new(&bh));
-	}
-
-	bdev = bh.b_bdev;
-
-	/*
-	 * If the filesystem isn't willing to tell us the length of a hole,
-	 * just fall back to PTEs.  Calling get_block 512 times in a loop
-	 * would be silly.
-	 */
-	if (!buffer_size_valid(&bh) || bh.b_size < PMD_SIZE) {
-		dax_pmd_dbg(&bh, address, "allocated block too small");
-		return VM_FAULT_FALLBACK;
-	}
-
-	/*
-	 * If we allocated new storage, make sure no process has any
-	 * zero pages covering this hole
-	 */
-	if (alloc) {
-		loff_t lstart = pgoff << PAGE_SHIFT;
-		loff_t lend = lstart + PMD_SIZE - 1; /* inclusive */
-
-		truncate_pagecache_range(inode, lstart, lend);
-	}
-
-	if (!write && !buffer_mapped(&bh)) {
-		spinlock_t *ptl;
-		pmd_t entry;
-		struct page *zero_page = mm_get_huge_zero_page(vma->vm_mm);
-
-		if (unlikely(!zero_page)) {
-			dax_pmd_dbg(&bh, address, "no zero page");
-			goto fallback;
-		}
-
-		ptl = pmd_lock(vma->vm_mm, pmd);
-		if (!pmd_none(*pmd)) {
-			spin_unlock(ptl);
-			dax_pmd_dbg(&bh, address, "pmd already present");
-			goto fallback;
-		}
-
-		dev_dbg(part_to_dev(bdev->bd_part),
-				"%s: %s addr: %lx pfn: <zero> sect: %llx\n",
-				__func__, current->comm, address,
-				(unsigned long long) to_sector(&bh, inode));
-
-		entry = mk_pmd(zero_page, vma->vm_page_prot);
-		entry = pmd_mkhuge(entry);
-		set_pmd_at(vma->vm_mm, pmd_addr, pmd, entry);
-		result = VM_FAULT_NOPAGE;
-		spin_unlock(ptl);
-	} else {
-		struct blk_dax_ctl dax = {
-			.sector = to_sector(&bh, inode),
-			.size = PMD_SIZE,
-		};
-		long length = dax_map_atomic(bdev, &dax);
-
-		if (length < 0) {
-			dax_pmd_dbg(&bh, address, "dax-error fallback");
-			goto fallback;
-		}
-		if (length < PMD_SIZE) {
-			dax_pmd_dbg(&bh, address, "dax-length too small");
-			dax_unmap_atomic(bdev, &dax);
-			goto fallback;
-		}
-		if (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR) {
-			dax_pmd_dbg(&bh, address, "pfn unaligned");
-			dax_unmap_atomic(bdev, &dax);
-			goto fallback;
-		}
-
-		if (!pfn_t_devmap(dax.pfn)) {
-			dax_unmap_atomic(bdev, &dax);
-			dax_pmd_dbg(&bh, address, "pfn not in memmap");
-			goto fallback;
-		}
-		dax_unmap_atomic(bdev, &dax);
-
-		/*
-		 * For PTE faults we insert a radix tree entry for reads, and
-		 * leave it clean.  Then on the first write we dirty the radix
-		 * tree entry via the dax_pfn_mkwrite() path.  This sequence
-		 * allows the dax_pfn_mkwrite() call to be simpler and avoid a
-		 * call into get_block() to translate the pgoff to a sector in
-		 * order to be able to create a new radix tree entry.
-		 *
-		 * The PMD path doesn't have an equivalent to
-		 * dax_pfn_mkwrite(), though, so for a read followed by a
-		 * write we traverse all the way through dax_pmd_fault()
-		 * twice.  This means we can just skip inserting a radix tree
-		 * entry completely on the initial read and just wait until
-		 * the write to insert a dirty entry.
-		 */
-		if (write) {
-			/*
-			 * We should insert radix-tree entry and dirty it here.
-			 * For now this is broken...
-			 */
-		}
-
-		dev_dbg(part_to_dev(bdev->bd_part),
-				"%s: %s addr: %lx pfn: %lx sect: %llx\n",
-				__func__, current->comm, address,
-				pfn_t_to_pfn(dax.pfn),
-				(unsigned long long) dax.sector);
-		result |= vmf_insert_pfn_pmd(vma, address, pmd,
-				dax.pfn, write);
-	}
-
- out:
-	return result;
-
- fallback:
-	count_vm_event(THP_FAULT_FALLBACK);
-	result = VM_FAULT_FALLBACK;
-	goto out;
-}
-EXPORT_SYMBOL_GPL(dax_pmd_fault);
-#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
-
-/**
  * dax_pfn_mkwrite - handle first write to DAX page
  * @vma: The virtual memory area where the fault occurred
  * @vmf: The description of the fault
@@ -1142,17 +847,27 @@ int dax_pfn_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct file *file = vma->vm_file;
 	struct address_space *mapping = file->f_mapping;
-	void *entry;
+	void *entry, **slot;
 	pgoff_t index = vmf->pgoff;
 
 	spin_lock_irq(&mapping->tree_lock);
-	entry = get_unlocked_mapping_entry(mapping, index, NULL);
-	if (!entry || !radix_tree_exceptional_entry(entry))
-		goto out;
+	entry = get_unlocked_mapping_entry(mapping, index, &slot);
+	if (!entry || !radix_tree_exceptional_entry(entry)) {
+		if (entry)
+			put_unlocked_mapping_entry(mapping, index, entry);
+		spin_unlock_irq(&mapping->tree_lock);
+		return VM_FAULT_NOPAGE;
+	}
 	radix_tree_tag_set(&mapping->page_tree, index, PAGECACHE_TAG_DIRTY);
-	put_unlocked_mapping_entry(mapping, index, entry);
-out:
+	entry = lock_slot(mapping, slot);
 	spin_unlock_irq(&mapping->tree_lock);
+	/*
+	 * If we race with somebody updating the PTE and finish_mkwrite_fault()
+	 * fails, we don't care. We need to return VM_FAULT_NOPAGE and retry
+	 * the fault in either case.
+	 */
+	finish_mkwrite_fault(vmf);
+	put_locked_mapping_entry(mapping, index, entry);
 	return VM_FAULT_NOPAGE;
 }
 EXPORT_SYMBOL_GPL(dax_pfn_mkwrite);
@@ -1193,62 +908,14 @@ int __dax_zero_page_range(struct block_device *bdev, sector_t sector,
 }
 EXPORT_SYMBOL_GPL(__dax_zero_page_range);
 
-/**
- * dax_zero_page_range - zero a range within a page of a DAX file
- * @inode: The file being truncated
- * @from: The file offset that is being truncated to
- * @length: The number of bytes to zero
- * @get_block: The filesystem method used to translate file offsets to blocks
- *
- * This function can be called by a filesystem when it is zeroing part of a
- * page in a DAX file.  This is intended for hole-punch operations.  If
- * you are truncating a file, the helper function dax_truncate_page() may be
- * more convenient.
- */
-int dax_zero_page_range(struct inode *inode, loff_t from, unsigned length,
-							get_block_t get_block)
-{
-	struct buffer_head bh;
-	pgoff_t index = from >> PAGE_SHIFT;
-	unsigned offset = from & (PAGE_SIZE-1);
-	int err;
-
-	/* Block boundary? Nothing to do */
-	if (!length)
-		return 0;
-	BUG_ON((offset + length) > PAGE_SIZE);
-
-	memset(&bh, 0, sizeof(bh));
-	bh.b_bdev = inode->i_sb->s_bdev;
-	bh.b_size = PAGE_SIZE;
-	err = get_block(inode, index, &bh, 0);
-	if (err < 0 || !buffer_written(&bh))
-		return err;
-
-	return __dax_zero_page_range(bh.b_bdev, to_sector(&bh, inode),
-			offset, length);
-}
-EXPORT_SYMBOL_GPL(dax_zero_page_range);
-
-/**
- * dax_truncate_page - handle a partial page being truncated in a DAX file
- * @inode: The file being truncated
- * @from: The file offset that is being truncated to
- * @get_block: The filesystem method used to translate file offsets to blocks
- *
- * Similar to block_truncate_page(), this function can be called by a
- * filesystem when it is truncating a DAX file to handle the partial page.
- */
-int dax_truncate_page(struct inode *inode, loff_t from, get_block_t get_block)
+#ifdef CONFIG_FS_IOMAP
+static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
 {
-	unsigned length = PAGE_ALIGN(from) - from;
-	return dax_zero_page_range(inode, from, length, get_block);
+	return iomap->blkno + (((pos & PAGE_MASK) - iomap->offset) >> 9);
 }
-EXPORT_SYMBOL_GPL(dax_truncate_page);
 
-#ifdef CONFIG_FS_IOMAP
 static loff_t
-iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
+dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
 		struct iomap *iomap)
 {
 	struct iov_iter *iter = data;
@@ -1272,8 +939,7 @@ iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
 		struct blk_dax_ctl dax = { 0 };
 		ssize_t map_len;
 
-		dax.sector = iomap->blkno +
-			(((pos & PAGE_MASK) - iomap->offset) >> 9);
+		dax.sector = dax_iomap_sector(iomap, pos);
 		dax.size = (length + offset + PAGE_SIZE - 1) & PAGE_MASK;
 		map_len = dax_map_atomic(iomap->bdev, &dax);
 		if (map_len < 0) {
@@ -1305,7 +971,7 @@ iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
 }
 
 /**
- * iomap_dax_rw - Perform I/O to a DAX file
+ * dax_iomap_rw - Perform I/O to a DAX file
  * @iocb:	The control block for this I/O
  * @iter:	The addresses to do I/O from or to
  * @ops:	iomap ops passed from the file system
@@ -1315,7 +981,7 @@ iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
  * and evicting any page cache pages in the region under I/O.
  */
 ssize_t
-iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter,
+dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
 		struct iomap_ops *ops)
 {
 	struct address_space *mapping = iocb->ki_filp->f_mapping;
@@ -1345,7 +1011,7 @@ iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter,
 
 	while (iov_iter_count(iter)) {
 		ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
-				iter, iomap_dax_actor);
+				iter, dax_iomap_actor);
 		if (ret <= 0)
 			break;
 		pos += ret;
@@ -1355,10 +1021,10 @@ iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter,
 	iocb->ki_pos += done;
 	return done ? done : ret;
 }
-EXPORT_SYMBOL_GPL(iomap_dax_rw);
+EXPORT_SYMBOL_GPL(dax_iomap_rw);
 
 /**
- * iomap_dax_fault - handle a page fault on a DAX file
+ * dax_iomap_fault - handle a page fault on a DAX file
  * @vma: The virtual memory area where the fault occurred
  * @vmf: The description of the fault
  * @ops: iomap ops passed from the file system
@@ -1367,17 +1033,18 @@ EXPORT_SYMBOL_GPL(iomap_dax_rw);
  * or mkwrite handler for DAX files. Assumes the caller has done all the
  * necessary locking for the page fault to proceed successfully.
  */
-int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
+int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
 			struct iomap_ops *ops)
 {
 	struct address_space *mapping = vma->vm_file->f_mapping;
 	struct inode *inode = mapping->host;
-	unsigned long vaddr = (unsigned long)vmf->virtual_address;
+	unsigned long vaddr = vmf->address;
 	loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
 	sector_t sector;
 	struct iomap iomap = { 0 };
-	unsigned flags = 0;
+	unsigned flags = IOMAP_FAULT;
 	int error, major = 0;
+	int vmf_ret = 0;
 	void *entry;
 
 	/*
@@ -1388,7 +1055,7 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
 	if (pos >= i_size_read(inode))
 		return VM_FAULT_SIGBUS;
 
-	entry = grab_mapping_entry(mapping, vmf->pgoff);
+	entry = grab_mapping_entry(mapping, vmf->pgoff, 0);
 	if (IS_ERR(entry)) {
 		error = PTR_ERR(entry);
 		goto out;
@@ -1407,10 +1074,10 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
 		goto unlock_entry;
 	if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
 		error = -EIO;		/* fs corruption? */
-		goto unlock_entry;
+		goto finish_iomap;
 	}
 
-	sector = iomap.blkno + (((pos & PAGE_MASK) - iomap.offset) >> 9);
+	sector = dax_iomap_sector(&iomap, pos);
 
 	if (vmf->cow_page) {
 		switch (iomap.type) {
@@ -1429,13 +1096,13 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
 		}
 
 		if (error)
-			goto unlock_entry;
-		if (!radix_tree_exceptional_entry(entry)) {
-			vmf->page = entry;
-			return VM_FAULT_LOCKED;
-		}
-		vmf->entry = entry;
-		return VM_FAULT_DAX_LOCKED;
+			goto finish_iomap;
+
+		__SetPageUptodate(vmf->cow_page);
+		vmf_ret = finish_fault(vmf);
+		if (!vmf_ret)
+			vmf_ret = VM_FAULT_DONE_COW;
+		goto finish_iomap;
 	}
 
 	switch (iomap.type) {
@@ -1450,8 +1117,10 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
 		break;
 	case IOMAP_UNWRITTEN:
 	case IOMAP_HOLE:
-		if (!(vmf->flags & FAULT_FLAG_WRITE))
-			return dax_load_hole(mapping, entry, vmf);
+		if (!(vmf->flags & FAULT_FLAG_WRITE)) {
+			vmf_ret = dax_load_hole(mapping, entry, vmf);
+			break;
+		}
 		/*FALLTHRU*/
 	default:
 		WARN_ON_ONCE(1);
@@ -1459,15 +1128,218 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
 		break;
 	}
 
+ finish_iomap:
+	if (ops->iomap_end) {
+		if (error || (vmf_ret & VM_FAULT_ERROR)) {
+			/* keep previous error */
+			ops->iomap_end(inode, pos, PAGE_SIZE, 0, flags,
+					&iomap);
+		} else {
+			error = ops->iomap_end(inode, pos, PAGE_SIZE,
+					PAGE_SIZE, flags, &iomap);
+		}
+	}
  unlock_entry:
-	put_locked_mapping_entry(mapping, vmf->pgoff, entry);
+	if (vmf_ret != VM_FAULT_LOCKED || error)
+		put_locked_mapping_entry(mapping, vmf->pgoff, entry);
  out:
 	if (error == -ENOMEM)
 		return VM_FAULT_OOM | major;
 	/* -EBUSY is fine, somebody else faulted on the same PTE */
 	if (error < 0 && error != -EBUSY)
 		return VM_FAULT_SIGBUS | major;
+	if (vmf_ret) {
+		WARN_ON_ONCE(error); /* -EBUSY from ops->iomap_end? */
+		return vmf_ret;
+	}
 	return VM_FAULT_NOPAGE | major;
 }
-EXPORT_SYMBOL_GPL(iomap_dax_fault);
+EXPORT_SYMBOL_GPL(dax_iomap_fault);
+
+#ifdef CONFIG_FS_DAX_PMD
+/*
+ * The 'colour' (ie low bits) within a PMD of a page offset.  This comes up
+ * more often than one might expect in the below functions.
+ */
+#define PG_PMD_COLOUR	((PMD_SIZE >> PAGE_SHIFT) - 1)
+
+static int dax_pmd_insert_mapping(struct vm_area_struct *vma, pmd_t *pmd,
+		struct vm_fault *vmf, unsigned long address,
+		struct iomap *iomap, loff_t pos, bool write, void **entryp)
+{
+	struct address_space *mapping = vma->vm_file->f_mapping;
+	struct block_device *bdev = iomap->bdev;
+	struct blk_dax_ctl dax = {
+		.sector = dax_iomap_sector(iomap, pos),
+		.size = PMD_SIZE,
+	};
+	long length = dax_map_atomic(bdev, &dax);
+	void *ret;
+
+	if (length < 0) /* dax_map_atomic() failed */
+		return VM_FAULT_FALLBACK;
+	if (length < PMD_SIZE)
+		goto unmap_fallback;
+	if (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR)
+		goto unmap_fallback;
+	if (!pfn_t_devmap(dax.pfn))
+		goto unmap_fallback;
+
+	dax_unmap_atomic(bdev, &dax);
+
+	ret = dax_insert_mapping_entry(mapping, vmf, *entryp, dax.sector,
+			RADIX_DAX_PMD);
+	if (IS_ERR(ret))
+		return VM_FAULT_FALLBACK;
+	*entryp = ret;
+
+	return vmf_insert_pfn_pmd(vma, address, pmd, dax.pfn, write);
+
+ unmap_fallback:
+	dax_unmap_atomic(bdev, &dax);
+	return VM_FAULT_FALLBACK;
+}
+
+static int dax_pmd_load_hole(struct vm_area_struct *vma, pmd_t *pmd,
+		struct vm_fault *vmf, unsigned long address,
+		struct iomap *iomap, void **entryp)
+{
+	struct address_space *mapping = vma->vm_file->f_mapping;
+	unsigned long pmd_addr = address & PMD_MASK;
+	struct page *zero_page;
+	spinlock_t *ptl;
+	pmd_t pmd_entry;
+	void *ret;
+
+	zero_page = mm_get_huge_zero_page(vma->vm_mm);
+
+	if (unlikely(!zero_page))
+		return VM_FAULT_FALLBACK;
+
+	ret = dax_insert_mapping_entry(mapping, vmf, *entryp, 0,
+			RADIX_DAX_PMD | RADIX_DAX_HZP);
+	if (IS_ERR(ret))
+		return VM_FAULT_FALLBACK;
+	*entryp = ret;
+
+	ptl = pmd_lock(vma->vm_mm, pmd);
+	if (!pmd_none(*pmd)) {
+		spin_unlock(ptl);
+		return VM_FAULT_FALLBACK;
+	}
+
+	pmd_entry = mk_pmd(zero_page, vma->vm_page_prot);
+	pmd_entry = pmd_mkhuge(pmd_entry);
+	set_pmd_at(vma->vm_mm, pmd_addr, pmd, pmd_entry);
+	spin_unlock(ptl);
+	return VM_FAULT_NOPAGE;
+}
+
+int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address,
+		pmd_t *pmd, unsigned int flags, struct iomap_ops *ops)
+{
+	struct address_space *mapping = vma->vm_file->f_mapping;
+	unsigned long pmd_addr = address & PMD_MASK;
+	bool write = flags & FAULT_FLAG_WRITE;
+	unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
+	struct inode *inode = mapping->host;
+	int result = VM_FAULT_FALLBACK;
+	struct iomap iomap = { 0 };
+	pgoff_t max_pgoff, pgoff;
+	struct vm_fault vmf;
+	void *entry;
+	loff_t pos;
+	int error;
+
+	/* Fall back to PTEs if we're going to COW */
+	if (write && !(vma->vm_flags & VM_SHARED))
+		goto fallback;
+
+	/* If the PMD would extend outside the VMA */
+	if (pmd_addr < vma->vm_start)
+		goto fallback;
+	if ((pmd_addr + PMD_SIZE) > vma->vm_end)
+		goto fallback;
+
+	/*
+	 * Check whether offset isn't beyond end of file now. Caller is
+	 * supposed to hold locks serializing us with truncate / punch hole so
+	 * this is a reliable test.
+	 */
+	pgoff = linear_page_index(vma, pmd_addr);
+	max_pgoff = (i_size_read(inode) - 1) >> PAGE_SHIFT;
+
+	if (pgoff > max_pgoff)
+		return VM_FAULT_SIGBUS;
+
+	/* If the PMD would extend beyond the file size */
+	if ((pgoff | PG_PMD_COLOUR) > max_pgoff)
+		goto fallback;
+
+	/*
+	 * grab_mapping_entry() will make sure we get a 2M empty entry, a DAX
+	 * PMD or a HZP entry.  If it can't (because a 4k page is already in
+	 * the tree, for instance), it will return -EEXIST and we just fall
+	 * back to 4k entries.
+	 */
+	entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD);
+	if (IS_ERR(entry))
+		goto fallback;
+
+	/*
+	 * Note that we don't use iomap_apply here.  We aren't doing I/O, only
+	 * setting up a mapping, so really we're using iomap_begin() as a way
+	 * to look up our filesystem block.
+	 */
+	pos = (loff_t)pgoff << PAGE_SHIFT;
+	error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
+	if (error)
+		goto unlock_entry;
+	if (iomap.offset + iomap.length < pos + PMD_SIZE)
+		goto finish_iomap;
+
+	vmf.pgoff = pgoff;
+	vmf.flags = flags;
+	vmf.gfp_mask = mapping_gfp_mask(mapping) | __GFP_IO;
+
+	switch (iomap.type) {
+	case IOMAP_MAPPED:
+		result = dax_pmd_insert_mapping(vma, pmd, &vmf, address,
+				&iomap, pos, write, &entry);
+		break;
+	case IOMAP_UNWRITTEN:
+	case IOMAP_HOLE:
+		if (WARN_ON_ONCE(write))
+			goto finish_iomap;
+		result = dax_pmd_load_hole(vma, pmd, &vmf, address, &iomap,
+				&entry);
+		break;
+	default:
+		WARN_ON_ONCE(1);
+		break;
+	}
+
+ finish_iomap:
+	if (ops->iomap_end) {
+		if (result == VM_FAULT_FALLBACK) {
+			ops->iomap_end(inode, pos, PMD_SIZE, 0, iomap_flags,
+					&iomap);
+		} else {
+			error = ops->iomap_end(inode, pos, PMD_SIZE, PMD_SIZE,
+					iomap_flags, &iomap);
+			if (error)
+				result = VM_FAULT_FALLBACK;
+		}
+	}
+ unlock_entry:
+	put_locked_mapping_entry(mapping, pgoff, entry);
+ fallback:
+	if (result == VM_FAULT_FALLBACK) {
+		split_huge_pmd(vma, pmd, address);
+		count_vm_event(THP_FAULT_FALLBACK);
+	}
+	return result;
+}
+EXPORT_SYMBOL_GPL(dax_iomap_pmd_fault);
+#endif /* CONFIG_FS_DAX_PMD */
 #endif /* CONFIG_FS_IOMAP */
diff --git a/fs/direct-io.c b/fs/direct-io.c
index fb9aa16a7727..aeae8c063451 100644
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -457,7 +457,7 @@ static struct bio *dio_await_one(struct dio *dio)
 		dio->waiter = current;
 		spin_unlock_irqrestore(&dio->bio_lock, flags);
 		if (!(dio->iocb->ki_flags & IOCB_HIPRI) ||
-		    !blk_poll(bdev_get_queue(dio->bio_bdev), dio->bio_cookie))
+		    !blk_mq_poll(bdev_get_queue(dio->bio_bdev), dio->bio_cookie))
 			io_schedule();
 		/* wake up sets us TASK_RUNNING */
 		spin_lock_irqsave(&dio->bio_lock, flags);
@@ -554,7 +554,7 @@ static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio)
  * filesystems that don't need it and also allows us to create the workqueue
  * late enough so the we can include s_id in the name of the workqueue.
  */
-static int sb_init_dio_done_wq(struct super_block *sb)
+int sb_init_dio_done_wq(struct super_block *sb)
 {
 	struct workqueue_struct *old;
 	struct workqueue_struct *wq = alloc_workqueue("dio/%s",
@@ -843,24 +843,6 @@ out:
 }
 
 /*
- * Clean any dirty buffers in the blockdev mapping which alias newly-created
- * file blocks.  Only called for S_ISREG files - blockdevs do not set
- * buffer_new
- */
-static void clean_blockdev_aliases(struct dio *dio, struct buffer_head *map_bh)
-{
-	unsigned i;
-	unsigned nblocks;
-
-	nblocks = map_bh->b_size >> dio->inode->i_blkbits;
-
-	for (i = 0; i < nblocks; i++) {
-		unmap_underlying_metadata(map_bh->b_bdev,
-					  map_bh->b_blocknr + i);
-	}
-}
-
-/*
  * If we are not writing the entire block and get_block() allocated
  * the block for us, we need to fill-in the unused portion of the
  * block with zeros. This happens only if user-buffer, fileoffset or
@@ -960,11 +942,15 @@ static int do_direct_IO(struct dio *dio, struct dio_submit *sdio,
 					goto do_holes;
 
 				sdio->blocks_available =
-						map_bh->b_size >> sdio->blkbits;
+						map_bh->b_size >> blkbits;
 				sdio->next_block_for_io =
 					map_bh->b_blocknr << sdio->blkfactor;
-				if (buffer_new(map_bh))
-					clean_blockdev_aliases(dio, map_bh);
+				if (buffer_new(map_bh)) {
+					clean_bdev_aliases(
+						map_bh->b_bdev,
+						map_bh->b_blocknr,
+						map_bh->b_size >> blkbits);
+				}
 
 				if (!sdio->blkfactor)
 					goto do_holes;
@@ -1209,7 +1195,7 @@ do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
 	dio->inode = inode;
 	if (iov_iter_rw(iter) == WRITE) {
 		dio->op = REQ_OP_WRITE;
-		dio->op_flags = WRITE_ODIRECT;
+		dio->op_flags = REQ_SYNC | REQ_IDLE;
 	} else {
 		dio->op = REQ_OP_READ;
 	}
diff --git a/fs/dlm/ast.c b/fs/dlm/ast.c
index dcea1e37a1b7..07fed838d8fd 100644
--- a/fs/dlm/ast.c
+++ b/fs/dlm/ast.c
@@ -268,7 +268,7 @@ void dlm_callback_work(struct work_struct *work)
 int dlm_callback_start(struct dlm_ls *ls)
 {
 	ls->ls_callback_wq = alloc_workqueue("dlm_callback",
-					     WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
+					     WQ_HIGHPRI | WQ_MEM_RECLAIM, 0);
 	if (!ls->ls_callback_wq) {
 		log_print("can't start dlm_callback workqueue");
 		return -ENOMEM;
diff --git a/fs/dlm/config.c b/fs/dlm/config.c
index df955d2209ce..7211e826d90d 100644
--- a/fs/dlm/config.c
+++ b/fs/dlm/config.c
@@ -12,7 +12,7 @@
 ******************************************************************************/
 
 #include <linux/kernel.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/configfs.h>
 #include <linux/slab.h>
 #include <linux/in.h>
diff --git a/fs/dlm/debug_fs.c b/fs/dlm/debug_fs.c
index 466f7d60edc2..ca7089aeadab 100644
--- a/fs/dlm/debug_fs.c
+++ b/fs/dlm/debug_fs.c
@@ -12,7 +12,7 @@
 
 #include <linux/pagemap.h>
 #include <linux/seq_file.h>
-#include <linux/module.h>
+#include <linux/init.h>
 #include <linux/ctype.h>
 #include <linux/debugfs.h>
 #include <linux/slab.h>
diff --git a/fs/dlm/dlm_internal.h b/fs/dlm/dlm_internal.h
index 216b61604ef9..b670f5601fbb 100644
--- a/fs/dlm/dlm_internal.h
+++ b/fs/dlm/dlm_internal.h
@@ -18,7 +18,6 @@
  * This is the main header file to be included in each DLM source file.
  */
 
-#include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/sched.h>
 #include <linux/types.h>
diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c
index f3e72787e7f9..91592b75c309 100644
--- a/fs/dlm/lockspace.c
+++ b/fs/dlm/lockspace.c
@@ -11,6 +11,8 @@
 *******************************************************************************
 ******************************************************************************/
 
+#include <linux/module.h>
+
 #include "dlm_internal.h"
 #include "lockspace.h"
 #include "member.h"
diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
index 609998de533e..7d398d300e97 100644
--- a/fs/dlm/lowcomms.c
+++ b/fs/dlm/lowcomms.c
@@ -519,29 +519,25 @@ out:
 /* Note: sk_callback_lock must be locked before calling this function. */
 static void save_callbacks(struct connection *con, struct sock *sk)
 {
-	lock_sock(sk);
 	con->orig_data_ready = sk->sk_data_ready;
 	con->orig_state_change = sk->sk_state_change;
 	con->orig_write_space = sk->sk_write_space;
 	con->orig_error_report = sk->sk_error_report;
-	release_sock(sk);
 }
 
 static void restore_callbacks(struct connection *con, struct sock *sk)
 {
 	write_lock_bh(&sk->sk_callback_lock);
-	lock_sock(sk);
 	sk->sk_user_data = NULL;
 	sk->sk_data_ready = con->orig_data_ready;
 	sk->sk_state_change = con->orig_state_change;
 	sk->sk_write_space = con->orig_write_space;
 	sk->sk_error_report = con->orig_error_report;
-	release_sock(sk);
 	write_unlock_bh(&sk->sk_callback_lock);
 }
 
 /* Make a socket active */
-static void add_sock(struct socket *sock, struct connection *con)
+static void add_sock(struct socket *sock, struct connection *con, bool save_cb)
 {
 	struct sock *sk = sock->sk;
 
@@ -549,7 +545,7 @@ static void add_sock(struct socket *sock, struct connection *con)
 	con->sock = sock;
 
 	sk->sk_user_data = con;
-	if (!test_bit(CF_IS_OTHERCON, &con->flags))
+	if (save_cb)
 		save_callbacks(con, sk);
 	/* Install a data_ready callback */
 	sk->sk_data_ready = lowcomms_data_ready;
@@ -806,7 +802,7 @@ static int tcp_accept_from_sock(struct connection *con)
 			newcon->othercon = othercon;
 			othercon->sock = newsock;
 			newsock->sk->sk_user_data = othercon;
-			add_sock(newsock, othercon);
+			add_sock(newsock, othercon, false);
 			addcon = othercon;
 		}
 		else {
@@ -819,7 +815,10 @@ static int tcp_accept_from_sock(struct connection *con)
 	else {
 		newsock->sk->sk_user_data = newcon;
 		newcon->rx_action = receive_from_sock;
-		add_sock(newsock, newcon);
+		/* accept copies the sk after we've saved the callbacks, so we
+		   don't want to save them a second time or comm errors will
+		   result in calling sk_error_report recursively. */
+		add_sock(newsock, newcon, false);
 		addcon = newcon;
 	}
 
@@ -880,7 +879,8 @@ static int sctp_accept_from_sock(struct connection *con)
 	}
 
 	make_sockaddr(&prim.ssp_addr, 0, &addr_len);
-	if (addr_to_nodeid(&prim.ssp_addr, &nodeid)) {
+	ret = addr_to_nodeid(&prim.ssp_addr, &nodeid);
+	if (ret) {
 		unsigned char *b = (unsigned char *)&prim.ssp_addr;
 
 		log_print("reject connect from unknown addr");
@@ -919,7 +919,7 @@ static int sctp_accept_from_sock(struct connection *con)
 			newcon->othercon = othercon;
 			othercon->sock = newsock;
 			newsock->sk->sk_user_data = othercon;
-			add_sock(newsock, othercon);
+			add_sock(newsock, othercon, false);
 			addcon = othercon;
 		} else {
 			printk("Extra connection from node %d attempted\n", nodeid);
@@ -930,7 +930,7 @@ static int sctp_accept_from_sock(struct connection *con)
 	} else {
 		newsock->sk->sk_user_data = newcon;
 		newcon->rx_action = receive_from_sock;
-		add_sock(newsock, newcon);
+		add_sock(newsock, newcon, false);
 		addcon = newcon;
 	}
 
@@ -1058,7 +1058,7 @@ static void sctp_connect_to_sock(struct connection *con)
 	sock->sk->sk_user_data = con;
 	con->rx_action = receive_from_sock;
 	con->connect_action = sctp_connect_to_sock;
-	add_sock(sock, con);
+	add_sock(sock, con, true);
 
 	/* Bind to all addresses. */
 	if (sctp_bind_addrs(con, 0))
@@ -1146,7 +1146,7 @@ static void tcp_connect_to_sock(struct connection *con)
 	sock->sk->sk_user_data = con;
 	con->rx_action = receive_from_sock;
 	con->connect_action = tcp_connect_to_sock;
-	add_sock(sock, con);
+	add_sock(sock, con, true);
 
 	/* Bind to our cluster-known address connecting to avoid
 	   routing problems */
@@ -1366,7 +1366,7 @@ static int tcp_listen_for_all(void)
 
 	sock = tcp_create_listen_sock(con, dlm_local_addr[0]);
 	if (sock) {
-		add_sock(sock, con);
+		add_sock(sock, con, true);
 		result = 0;
 	}
 	else {
diff --git a/fs/dlm/main.c b/fs/dlm/main.c
index 079c0bd71ab7..8e1b618891be 100644
--- a/fs/dlm/main.c
+++ b/fs/dlm/main.c
@@ -11,6 +11,8 @@
 *******************************************************************************
 ******************************************************************************/
 
+#include <linux/module.h>
+
 #include "dlm_internal.h"
 #include "lockspace.h"
 #include "lock.h"
diff --git a/fs/dlm/netlink.c b/fs/dlm/netlink.c
index 0643ae44f342..43a96c330570 100644
--- a/fs/dlm/netlink.c
+++ b/fs/dlm/netlink.c
@@ -65,7 +65,7 @@ static int user_cmd(struct sk_buff *skb, struct genl_info *info)
 	return 0;
 }
 
-static struct genl_ops dlm_nl_ops[] = {
+static const struct genl_ops dlm_nl_ops[] = {
 	{
 		.cmd	= DLM_CMD_HELLO,
 		.doit	= user_cmd,
diff --git a/fs/dlm/user.c b/fs/dlm/user.c
index 58c2f4a21b7f..1ce908c2232c 100644
--- a/fs/dlm/user.c
+++ b/fs/dlm/user.c
@@ -9,7 +9,6 @@
 #include <linux/miscdevice.h>
 #include <linux/init.h>
 #include <linux/wait.h>
-#include <linux/module.h>
 #include <linux/file.h>
 #include <linux/fs.h>
 #include <linux/poll.h>
diff --git a/fs/exec.c b/fs/exec.c
index 923c57d96899..8112eacf10f3 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -209,7 +209,7 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos,
 	 * doing the exec and bprm->mm is the new process's mm.
 	 */
 	ret = get_user_pages_remote(current, bprm->mm, pos, 1, gup_flags,
-			&page, NULL);
+			&page, NULL, NULL);
 	if (ret <= 0)
 		return NULL;
 
@@ -1277,8 +1277,22 @@ EXPORT_SYMBOL(flush_old_exec);
 
 void would_dump(struct linux_binprm *bprm, struct file *file)
 {
-	if (inode_permission(file_inode(file), MAY_READ) < 0)
+	struct inode *inode = file_inode(file);
+	if (inode_permission(inode, MAY_READ) < 0) {
+		struct user_namespace *old, *user_ns;
 		bprm->interp_flags |= BINPRM_FLAGS_ENFORCE_NONDUMP;
+
+		/* Ensure mm->user_ns contains the executable */
+		user_ns = old = bprm->mm->user_ns;
+		while ((user_ns != &init_user_ns) &&
+		       !privileged_wrt_inode_uidgid(user_ns, inode))
+			user_ns = user_ns->parent;
+
+		if (old != user_ns) {
+			bprm->mm->user_ns = get_user_ns(user_ns);
+			put_user_ns(old);
+		}
+	}
 }
 EXPORT_SYMBOL(would_dump);
 
@@ -1308,7 +1322,6 @@ void setup_new_exec(struct linux_binprm * bprm)
 	    !gid_eq(bprm->cred->gid, current_egid())) {
 		current->pdeath_signal = 0;
 	} else {
-		would_dump(bprm, bprm->file);
 		if (bprm->interp_flags & BINPRM_FLAGS_ENFORCE_NONDUMP)
 			set_dumpable(current->mm, suid_dumpable);
 	}
@@ -1408,7 +1421,7 @@ static void check_unsafe_exec(struct linux_binprm *bprm)
 	unsigned n_fs;
 
 	if (p->ptrace) {
-		if (p->ptrace & PT_PTRACE_CAP)
+		if (ptracer_capable(p, current_user_ns()))
 			bprm->unsafe |= LSM_UNSAFE_PTRACE_CAP;
 		else
 			bprm->unsafe |= LSM_UNSAFE_PTRACE;
@@ -1743,6 +1756,8 @@ static int do_execveat_common(int fd, struct filename *filename,
 	if (retval < 0)
 		goto out;
 
+	would_dump(bprm, bprm->file);
+
 	retval = exec_binprm(bprm);
 	if (retval < 0)
 		goto out;
diff --git a/fs/ext2/file.c b/fs/ext2/file.c
index a0e1478dfd04..b0f241528a30 100644
--- a/fs/ext2/file.c
+++ b/fs/ext2/file.c
@@ -38,7 +38,7 @@ static ssize_t ext2_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		return 0; /* skip atime */
 
 	inode_lock_shared(inode);
-	ret = iomap_dax_rw(iocb, to, &ext2_iomap_ops);
+	ret = dax_iomap_rw(iocb, to, &ext2_iomap_ops);
 	inode_unlock_shared(inode);
 
 	file_accessed(iocb->ki_filp);
@@ -62,7 +62,7 @@ static ssize_t ext2_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	if (ret)
 		goto out_unlock;
 
-	ret = iomap_dax_rw(iocb, from, &ext2_iomap_ops);
+	ret = dax_iomap_rw(iocb, from, &ext2_iomap_ops);
 	if (ret > 0 && iocb->ki_pos > i_size_read(inode)) {
 		i_size_write(inode, iocb->ki_pos);
 		mark_inode_dirty(inode);
@@ -99,7 +99,7 @@ static int ext2_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	}
 	down_read(&ei->dax_sem);
 
-	ret = iomap_dax_fault(vma, vmf, &ext2_iomap_ops);
+	ret = dax_iomap_fault(vma, vmf, &ext2_iomap_ops);
 
 	up_read(&ei->dax_sem);
 	if (vmf->flags & FAULT_FLAG_WRITE)
@@ -107,27 +107,6 @@ static int ext2_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	return ret;
 }
 
-static int ext2_dax_pmd_fault(struct vm_area_struct *vma, unsigned long addr,
-						pmd_t *pmd, unsigned int flags)
-{
-	struct inode *inode = file_inode(vma->vm_file);
-	struct ext2_inode_info *ei = EXT2_I(inode);
-	int ret;
-
-	if (flags & FAULT_FLAG_WRITE) {
-		sb_start_pagefault(inode->i_sb);
-		file_update_time(vma->vm_file);
-	}
-	down_read(&ei->dax_sem);
-
-	ret = dax_pmd_fault(vma, addr, pmd, flags, ext2_get_block);
-
-	up_read(&ei->dax_sem);
-	if (flags & FAULT_FLAG_WRITE)
-		sb_end_pagefault(inode->i_sb);
-	return ret;
-}
-
 static int ext2_dax_pfn_mkwrite(struct vm_area_struct *vma,
 		struct vm_fault *vmf)
 {
@@ -154,7 +133,11 @@ static int ext2_dax_pfn_mkwrite(struct vm_area_struct *vma,
 
 static const struct vm_operations_struct ext2_dax_vm_ops = {
 	.fault		= ext2_dax_fault,
-	.pmd_fault	= ext2_dax_pmd_fault,
+	/*
+	 * .pmd_fault is not supported for DAX because allocation in ext2
+	 * cannot be reliably aligned to huge page sizes and so pmd faults
+	 * will always fail and fail back to regular faults.
+	 */
 	.page_mkwrite	= ext2_dax_fault,
 	.pfn_mkwrite	= ext2_dax_pfn_mkwrite,
 };
@@ -166,7 +149,7 @@ static int ext2_file_mmap(struct file *file, struct vm_area_struct *vma)
 
 	file_accessed(file);
 	vma->vm_ops = &ext2_dax_vm_ops;
-	vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
+	vma->vm_flags |= VM_MIXEDMAP;
 	return 0;
 }
 #else
diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c
index 41b8b44a391c..e173afe92661 100644
--- a/fs/ext2/inode.c
+++ b/fs/ext2/inode.c
@@ -732,16 +732,13 @@ static int ext2_get_blocks(struct inode *inode,
 	}
 
 	if (IS_DAX(inode)) {
-		int i;
-
 		/*
 		 * We must unmap blocks before zeroing so that writeback cannot
 		 * overwrite zeros with stale data from block device page cache.
 		 */
-		for (i = 0; i < count; i++) {
-			unmap_underlying_metadata(inode->i_sb->s_bdev,
-					le32_to_cpu(chain[depth-1].key) + i);
-		}
+		clean_bdev_aliases(inode->i_sb->s_bdev,
+				   le32_to_cpu(chain[depth-1].key),
+				   count);
 		/*
 		 * block must be initialised before we put it in the tree
 		 * so that it's not found by another thread before it's
@@ -850,6 +847,9 @@ struct iomap_ops ext2_iomap_ops = {
 	.iomap_begin		= ext2_iomap_begin,
 	.iomap_end		= ext2_iomap_end,
 };
+#else
+/* Define empty ops for !CONFIG_FS_DAX case to avoid ugly ifdefs */
+struct iomap_ops ext2_iomap_ops;
 #endif /* CONFIG_FS_DAX */
 
 int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
@@ -1293,9 +1293,11 @@ static int ext2_setsize(struct inode *inode, loff_t newsize)
 
 	inode_dio_wait(inode);
 
-	if (IS_DAX(inode))
-		error = dax_truncate_page(inode, newsize, ext2_get_block);
-	else if (test_opt(inode->i_sb, NOBH))
+	if (IS_DAX(inode)) {
+		error = iomap_zero_range(inode, newsize,
+					 PAGE_ALIGN(newsize) - newsize, NULL,
+					 &ext2_iomap_ops);
+	} else if (test_opt(inode->i_sb, NOBH))
 		error = nobh_truncate_page(inode->i_mapping,
 				newsize, ext2_get_block);
 	else
diff --git a/fs/ext4/Kconfig b/fs/ext4/Kconfig
index e38039fd96ff..7b90691e98c4 100644
--- a/fs/ext4/Kconfig
+++ b/fs/ext4/Kconfig
@@ -37,6 +37,7 @@ config EXT4_FS
 	select CRC16
 	select CRYPTO
 	select CRYPTO_CRC32C
+	select FS_IOMAP if FS_DAX
 	help
 	  This is the next generation of the ext3 filesystem.
 
diff --git a/fs/ext4/acl.c b/fs/ext4/acl.c
index dfa519979038..fd389935ecd1 100644
--- a/fs/ext4/acl.c
+++ b/fs/ext4/acl.c
@@ -196,7 +196,7 @@ __ext4_set_acl(handle_t *handle, struct inode *inode, int type,
 			error = posix_acl_update_mode(inode, &inode->i_mode, &acl);
 			if (error)
 				return error;
-			inode->i_ctime = ext4_current_time(inode);
+			inode->i_ctime = current_time(inode);
 			ext4_mark_inode_dirty(handle, inode);
 		}
 		break;
diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
index a8a750f59621..2163c1e69f2a 100644
--- a/fs/ext4/ext4.h
+++ b/fs/ext4/ext4.h
@@ -397,8 +397,9 @@ struct flex_groups {
 #define EXT4_RESERVED_FL		0x80000000 /* reserved for ext4 lib */
 
 #define EXT4_FL_USER_VISIBLE		0x304BDFFF /* User visible flags */
-#define EXT4_FL_USER_MODIFIABLE		0x204380FF /* User modifiable flags */
+#define EXT4_FL_USER_MODIFIABLE		0x204BC0FF /* User modifiable flags */
 
+/* Flags we can manipulate with through EXT4_IOC_FSSETXATTR */
 #define EXT4_FL_XFLAG_VISIBLE		(EXT4_SYNC_FL | \
 					 EXT4_IMMUTABLE_FL | \
 					 EXT4_APPEND_FL | \
@@ -1533,12 +1534,6 @@ static inline struct ext4_inode_info *EXT4_I(struct inode *inode)
 	return container_of(inode, struct ext4_inode_info, vfs_inode);
 }
 
-static inline struct timespec ext4_current_time(struct inode *inode)
-{
-	return (inode->i_sb->s_time_gran < NSEC_PER_SEC) ?
-		current_fs_time(inode->i_sb) : CURRENT_TIME_SEC;
-}
-
 static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino)
 {
 	return ino == EXT4_ROOT_INO ||
@@ -2277,11 +2272,6 @@ extern unsigned ext4_free_clusters_after_init(struct super_block *sb,
 					      struct ext4_group_desc *gdp);
 ext4_fsblk_t ext4_inode_to_goal_block(struct inode *);
 
-static inline int ext4_sb_has_crypto(struct super_block *sb)
-{
-	return ext4_has_feature_encrypt(sb);
-}
-
 static inline bool ext4_encrypted_inode(struct inode *inode)
 {
 	return ext4_test_inode_flag(inode, EXT4_INODE_ENCRYPT);
@@ -2339,8 +2329,8 @@ static inline void ext4_fname_free_filename(struct ext4_filename *fname) { }
 #define fscrypt_pullback_bio_page	fscrypt_notsupp_pullback_bio_page
 #define fscrypt_restore_control_page	fscrypt_notsupp_restore_control_page
 #define fscrypt_zeroout_range		fscrypt_notsupp_zeroout_range
-#define fscrypt_process_policy		fscrypt_notsupp_process_policy
-#define fscrypt_get_policy		fscrypt_notsupp_get_policy
+#define fscrypt_ioctl_set_policy	fscrypt_notsupp_ioctl_set_policy
+#define fscrypt_ioctl_get_policy	fscrypt_notsupp_ioctl_get_policy
 #define fscrypt_has_permitted_context	fscrypt_notsupp_has_permitted_context
 #define fscrypt_inherit_context		fscrypt_notsupp_inherit_context
 #define fscrypt_get_encryption_info	fscrypt_notsupp_get_encryption_info
@@ -2458,8 +2448,6 @@ struct buffer_head *ext4_getblk(handle_t *, struct inode *, ext4_lblk_t, int);
 struct buffer_head *ext4_bread(handle_t *, struct inode *, ext4_lblk_t, int);
 int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
 			     struct buffer_head *bh_result, int create);
-int ext4_dax_get_block(struct inode *inode, sector_t iblock,
-		       struct buffer_head *bh_result, int create);
 int ext4_get_block(struct inode *inode, sector_t iblock,
 		   struct buffer_head *bh_result, int create);
 int ext4_dio_get_block(struct inode *inode, sector_t iblock,
@@ -2492,7 +2480,7 @@ extern int ext4_change_inode_journal_flag(struct inode *, int);
 extern int ext4_get_inode_loc(struct inode *, struct ext4_iloc *);
 extern int ext4_inode_attach_jinode(struct inode *inode);
 extern int ext4_can_truncate(struct inode *inode);
-extern void ext4_truncate(struct inode *);
+extern int ext4_truncate(struct inode *);
 extern int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length);
 extern int ext4_truncate_restart_trans(handle_t *, struct inode *, int nblocks);
 extern void ext4_set_inode_flags(struct inode *);
@@ -3129,7 +3117,7 @@ extern int ext4_ext_writepage_trans_blocks(struct inode *, int);
 extern int ext4_ext_index_trans_blocks(struct inode *inode, int extents);
 extern int ext4_ext_map_blocks(handle_t *handle, struct inode *inode,
 			       struct ext4_map_blocks *map, int flags);
-extern void ext4_ext_truncate(handle_t *, struct inode *);
+extern int ext4_ext_truncate(handle_t *, struct inode *);
 extern int ext4_ext_remove_space(struct inode *inode, ext4_lblk_t start,
 				 ext4_lblk_t end);
 extern void ext4_ext_init(struct super_block *);
@@ -3265,12 +3253,7 @@ static inline void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end)
 	}
 }
 
-static inline bool ext4_aligned_io(struct inode *inode, loff_t off, loff_t len)
-{
-	int blksize = 1 << inode->i_blkbits;
-
-	return IS_ALIGNED(off, blksize) && IS_ALIGNED(len, blksize);
-}
+extern struct iomap_ops ext4_iomap_ops;
 
 #endif	/* __KERNEL__ */
 
diff --git a/fs/ext4/ext4_jbd2.h b/fs/ext4/ext4_jbd2.h
index b1d52c14098e..f97611171023 100644
--- a/fs/ext4/ext4_jbd2.h
+++ b/fs/ext4/ext4_jbd2.h
@@ -414,17 +414,19 @@ static inline int ext4_inode_journal_mode(struct inode *inode)
 		return EXT4_INODE_WRITEBACK_DATA_MODE;	/* writeback */
 	/* We do not support data journalling with delayed allocation */
 	if (!S_ISREG(inode->i_mode) ||
-	    test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA)
-		return EXT4_INODE_JOURNAL_DATA_MODE;	/* journal data */
-	if (ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA) &&
-	    !test_opt(inode->i_sb, DELALLOC))
+	    test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA ||
+	    (ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA) &&
+	    !test_opt(inode->i_sb, DELALLOC))) {
+		/* We do not support data journalling for encrypted data */
+		if (S_ISREG(inode->i_mode) && ext4_encrypted_inode(inode))
+			return EXT4_INODE_ORDERED_DATA_MODE;  /* ordered */
 		return EXT4_INODE_JOURNAL_DATA_MODE;	/* journal data */
+	}
 	if (test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA)
 		return EXT4_INODE_ORDERED_DATA_MODE;	/* ordered */
 	if (test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_WRITEBACK_DATA)
 		return EXT4_INODE_WRITEBACK_DATA_MODE;	/* writeback */
-	else
-		BUG();
+	BUG();
 }
 
 static inline int ext4_should_journal_data(struct inode *inode)
diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c
index c930a0110fb4..b1f8416923ab 100644
--- a/fs/ext4/extents.c
+++ b/fs/ext4/extents.c
@@ -3777,14 +3777,6 @@ out:
 	return err;
 }
 
-static void unmap_underlying_metadata_blocks(struct block_device *bdev,
-			sector_t block, int count)
-{
-	int i;
-	for (i = 0; i < count; i++)
-                unmap_underlying_metadata(bdev, block + i);
-}
-
 /*
  * Handle EOFBLOCKS_FL flag, clearing it if necessary
  */
@@ -4121,9 +4113,8 @@ out:
 	 * new.
 	 */
 	if (allocated > map->m_len) {
-		unmap_underlying_metadata_blocks(inode->i_sb->s_bdev,
-					newblock + map->m_len,
-					allocated - map->m_len);
+		clean_bdev_aliases(inode->i_sb->s_bdev, newblock + map->m_len,
+				   allocated - map->m_len);
 		allocated = map->m_len;
 	}
 	map->m_len = allocated;
@@ -4631,7 +4622,7 @@ out2:
 	return err ? err : allocated;
 }
 
-void ext4_ext_truncate(handle_t *handle, struct inode *inode)
+int ext4_ext_truncate(handle_t *handle, struct inode *inode)
 {
 	struct super_block *sb = inode->i_sb;
 	ext4_lblk_t last_block;
@@ -4645,7 +4636,9 @@ void ext4_ext_truncate(handle_t *handle, struct inode *inode)
 
 	/* we have to know where to truncate from in crash case */
 	EXT4_I(inode)->i_disksize = inode->i_size;
-	ext4_mark_inode_dirty(handle, inode);
+	err = ext4_mark_inode_dirty(handle, inode);
+	if (err)
+		return err;
 
 	last_block = (inode->i_size + sb->s_blocksize - 1)
 			>> EXT4_BLOCK_SIZE_BITS(sb);
@@ -4657,12 +4650,9 @@ retry:
 		congestion_wait(BLK_RW_ASYNC, HZ/50);
 		goto retry;
 	}
-	if (err) {
-		ext4_std_error(inode->i_sb, err);
-		return;
-	}
-	err = ext4_ext_remove_space(inode, last_block, EXT_MAX_BLOCKS - 1);
-	ext4_std_error(inode->i_sb, err);
+	if (err)
+		return err;
+	return ext4_ext_remove_space(inode, last_block, EXT_MAX_BLOCKS - 1);
 }
 
 static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset,
@@ -4701,7 +4691,7 @@ retry:
 		/*
 		 * Recalculate credits when extent tree depth changes.
 		 */
-		if (depth >= 0 && depth != ext_depth(inode)) {
+		if (depth != ext_depth(inode)) {
 			credits = ext4_chunk_trans_blocks(inode, len);
 			depth = ext_depth(inode);
 		}
@@ -4725,7 +4715,7 @@ retry:
 		map.m_lblk += ret;
 		map.m_len = len = len - ret;
 		epos = (loff_t)map.m_lblk << inode->i_blkbits;
-		inode->i_ctime = ext4_current_time(inode);
+		inode->i_ctime = current_time(inode);
 		if (new_size) {
 			if (epos > new_size)
 				epos = new_size;
@@ -4853,7 +4843,7 @@ static long ext4_zero_range(struct file *file, loff_t offset,
 		}
 		/* Now release the pages and zero block aligned part of pages */
 		truncate_pagecache_range(inode, start, end - 1);
-		inode->i_mtime = inode->i_ctime = ext4_current_time(inode);
+		inode->i_mtime = inode->i_ctime = current_time(inode);
 
 		ret = ext4_alloc_file_blocks(file, lblk, max_blocks, new_size,
 					     flags, mode);
@@ -4878,7 +4868,7 @@ static long ext4_zero_range(struct file *file, loff_t offset,
 		goto out_dio;
 	}
 
-	inode->i_mtime = inode->i_ctime = ext4_current_time(inode);
+	inode->i_mtime = inode->i_ctime = current_time(inode);
 	if (new_size) {
 		ext4_update_inode_size(inode, new_size);
 	} else {
@@ -5568,7 +5558,7 @@ int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len)
 	up_write(&EXT4_I(inode)->i_data_sem);
 	if (IS_SYNC(inode))
 		ext4_handle_sync(handle);
-	inode->i_mtime = inode->i_ctime = ext4_current_time(inode);
+	inode->i_mtime = inode->i_ctime = current_time(inode);
 	ext4_mark_inode_dirty(handle, inode);
 
 out_stop:
@@ -5678,7 +5668,7 @@ int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len)
 	/* Expand file to avoid data loss if there is error while shifting */
 	inode->i_size += len;
 	EXT4_I(inode)->i_disksize += len;
-	inode->i_mtime = inode->i_ctime = ext4_current_time(inode);
+	inode->i_mtime = inode->i_ctime = current_time(inode);
 	ret = ext4_mark_inode_dirty(handle, inode);
 	if (ret)
 		goto out_stop;
diff --git a/fs/ext4/file.c b/fs/ext4/file.c
index 2a822d30e73f..b5f184493c57 100644
--- a/fs/ext4/file.c
+++ b/fs/ext4/file.c
@@ -31,6 +31,42 @@
 #include "xattr.h"
 #include "acl.h"
 
+#ifdef CONFIG_FS_DAX
+static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)
+{
+	struct inode *inode = file_inode(iocb->ki_filp);
+	ssize_t ret;
+
+	inode_lock_shared(inode);
+	/*
+	 * Recheck under inode lock - at this point we are sure it cannot
+	 * change anymore
+	 */
+	if (!IS_DAX(inode)) {
+		inode_unlock_shared(inode);
+		/* Fallback to buffered IO in case we cannot support DAX */
+		return generic_file_read_iter(iocb, to);
+	}
+	ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops);
+	inode_unlock_shared(inode);
+
+	file_accessed(iocb->ki_filp);
+	return ret;
+}
+#endif
+
+static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
+{
+	if (!iov_iter_count(to))
+		return 0; /* skip atime */
+
+#ifdef CONFIG_FS_DAX
+	if (IS_DAX(file_inode(iocb->ki_filp)))
+		return ext4_dax_read_iter(iocb, to);
+#endif
+	return generic_file_read_iter(iocb, to);
+}
+
 /*
  * Called when an inode is released. Note that this is different
  * from ext4_file_open: open gets called at every open, but release
@@ -88,6 +124,86 @@ ext4_unaligned_aio(struct inode *inode, struct iov_iter *from, loff_t pos)
 	return 0;
 }
 
+/* Is IO overwriting allocated and initialized blocks? */
+static bool ext4_overwrite_io(struct inode *inode, loff_t pos, loff_t len)
+{
+	struct ext4_map_blocks map;
+	unsigned int blkbits = inode->i_blkbits;
+	int err, blklen;
+
+	if (pos + len > i_size_read(inode))
+		return false;
+
+	map.m_lblk = pos >> blkbits;
+	map.m_len = EXT4_MAX_BLOCKS(len, pos, blkbits);
+	blklen = map.m_len;
+
+	err = ext4_map_blocks(NULL, inode, &map, 0);
+	/*
+	 * 'err==len' means that all of the blocks have been preallocated,
+	 * regardless of whether they have been initialized or not. To exclude
+	 * unwritten extents, we need to check m_flags.
+	 */
+	return err == blklen && (map.m_flags & EXT4_MAP_MAPPED);
+}
+
+static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from)
+{
+	struct inode *inode = file_inode(iocb->ki_filp);
+	ssize_t ret;
+
+	ret = generic_write_checks(iocb, from);
+	if (ret <= 0)
+		return ret;
+	/*
+	 * If we have encountered a bitmap-format file, the size limit
+	 * is smaller than s_maxbytes, which is for extent-mapped files.
+	 */
+	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
+		struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
+
+		if (iocb->ki_pos >= sbi->s_bitmap_maxbytes)
+			return -EFBIG;
+		iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos);
+	}
+	return iov_iter_count(from);
+}
+
+#ifdef CONFIG_FS_DAX
+static ssize_t
+ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)
+{
+	struct inode *inode = file_inode(iocb->ki_filp);
+	ssize_t ret;
+	bool overwrite = false;
+
+	inode_lock(inode);
+	ret = ext4_write_checks(iocb, from);
+	if (ret <= 0)
+		goto out;
+	ret = file_remove_privs(iocb->ki_filp);
+	if (ret)
+		goto out;
+	ret = file_update_time(iocb->ki_filp);
+	if (ret)
+		goto out;
+
+	if (ext4_overwrite_io(inode, iocb->ki_pos, iov_iter_count(from))) {
+		overwrite = true;
+		downgrade_write(&inode->i_rwsem);
+	}
+	ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops);
+out:
+	if (!overwrite)
+		inode_unlock(inode);
+	else
+		inode_unlock_shared(inode);
+	if (ret > 0)
+		ret = generic_write_sync(iocb, ret);
+	return ret;
+}
+#endif
+
 static ssize_t
 ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 {
@@ -97,8 +213,13 @@ ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	int overwrite = 0;
 	ssize_t ret;
 
+#ifdef CONFIG_FS_DAX
+	if (IS_DAX(inode))
+		return ext4_dax_write_iter(iocb, from);
+#endif
+
 	inode_lock(inode);
-	ret = generic_write_checks(iocb, from);
+	ret = ext4_write_checks(iocb, from);
 	if (ret <= 0)
 		goto out;
 
@@ -114,53 +235,11 @@ ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 		ext4_unwritten_wait(inode);
 	}
 
-	/*
-	 * If we have encountered a bitmap-format file, the size limit
-	 * is smaller than s_maxbytes, which is for extent-mapped files.
-	 */
-	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
-		struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
-
-		if (iocb->ki_pos >= sbi->s_bitmap_maxbytes) {
-			ret = -EFBIG;
-			goto out;
-		}
-		iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos);
-	}
-
 	iocb->private = &overwrite;
-	if (o_direct) {
-		size_t length = iov_iter_count(from);
-		loff_t pos = iocb->ki_pos;
-
-		/* check whether we do a DIO overwrite or not */
-		if (ext4_should_dioread_nolock(inode) && !unaligned_aio &&
-		    pos + length <= i_size_read(inode)) {
-			struct ext4_map_blocks map;
-			unsigned int blkbits = inode->i_blkbits;
-			int err, len;
-
-			map.m_lblk = pos >> blkbits;
-			map.m_len = EXT4_MAX_BLOCKS(length, pos, blkbits);
-			len = map.m_len;
-
-			err = ext4_map_blocks(NULL, inode, &map, 0);
-			/*
-			 * 'err==len' means that all of blocks has
-			 * been preallocated no matter they are
-			 * initialized or not.  For excluding
-			 * unwritten extents, we need to check
-			 * m_flags.  There are two conditions that
-			 * indicate for initialized extents.  1) If we
-			 * hit extent cache, EXT4_MAP_MAPPED flag is
-			 * returned; 2) If we do a real lookup,
-			 * non-flags are returned.  So we should check
-			 * these two conditions.
-			 */
-			if (err == len && (map.m_flags & EXT4_MAP_MAPPED))
-				overwrite = 1;
-		}
-	}
+	/* Check whether we do a DIO overwrite or not */
+	if (o_direct && ext4_should_dioread_nolock(inode) && !unaligned_aio &&
+	    ext4_overwrite_io(inode, iocb->ki_pos, iov_iter_count(from)))
+		overwrite = 1;
 
 	ret = __generic_file_write_iter(iocb, from);
 	inode_unlock(inode);
@@ -196,7 +275,7 @@ static int ext4_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	if (IS_ERR(handle))
 		result = VM_FAULT_SIGBUS;
 	else
-		result = dax_fault(vma, vmf, ext4_dax_get_block);
+		result = dax_iomap_fault(vma, vmf, &ext4_iomap_ops);
 
 	if (write) {
 		if (!IS_ERR(handle))
@@ -230,9 +309,10 @@ static int ext4_dax_pmd_fault(struct vm_area_struct *vma, unsigned long addr,
 
 	if (IS_ERR(handle))
 		result = VM_FAULT_SIGBUS;
-	else
-		result = dax_pmd_fault(vma, addr, pmd, flags,
-					 ext4_dax_get_block);
+	else {
+		result = dax_iomap_pmd_fault(vma, addr, pmd, flags,
+					     &ext4_iomap_ops);
+	}
 
 	if (write) {
 		if (!IS_ERR(handle))
@@ -687,7 +767,7 @@ loff_t ext4_llseek(struct file *file, loff_t offset, int whence)
 
 const struct file_operations ext4_file_operations = {
 	.llseek		= ext4_llseek,
-	.read_iter	= generic_file_read_iter,
+	.read_iter	= ext4_file_read_iter,
 	.write_iter	= ext4_file_write_iter,
 	.unlocked_ioctl = ext4_ioctl,
 #ifdef CONFIG_COMPAT
diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c
index 170421edfdfe..e57e8d90ea54 100644
--- a/fs/ext4/ialloc.c
+++ b/fs/ext4/ialloc.c
@@ -1039,7 +1039,7 @@ got:
 	/* This is the optimal IO size (for stat), not the fs block size */
 	inode->i_blocks = 0;
 	inode->i_mtime = inode->i_atime = inode->i_ctime = ei->i_crtime =
-						       ext4_current_time(inode);
+						       current_time(inode);
 
 	memset(ei->i_data, 0, sizeof(ei->i_data));
 	ei->i_dir_start_lookup = 0;
@@ -1115,8 +1115,7 @@ got:
 	}
 
 	if (encrypt) {
-		/* give pointer to avoid set_context with journal ops. */
-		err = fscrypt_inherit_context(dir, inode, &encrypt, true);
+		err = fscrypt_inherit_context(dir, inode, handle, true);
 		if (err)
 			goto fail_free_drop;
 	}
diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c
index f74d5ee2cdec..437df6a1a841 100644
--- a/fs/ext4/inline.c
+++ b/fs/ext4/inline.c
@@ -299,6 +299,11 @@ static int ext4_create_inline_data(handle_t *handle,
 	EXT4_I(inode)->i_inline_size = len + EXT4_MIN_INLINE_DATA_SIZE;
 	ext4_clear_inode_flag(inode, EXT4_INODE_EXTENTS);
 	ext4_set_inode_flag(inode, EXT4_INODE_INLINE_DATA);
+	/*
+	 * Propagate changes to inode->i_flags as well - e.g. S_DAX may
+	 * get cleared
+	 */
+	ext4_set_inode_flags(inode);
 	get_bh(is.iloc.bh);
 	error = ext4_mark_iloc_dirty(handle, inode, &is.iloc);
 
@@ -336,8 +341,10 @@ static int ext4_update_inline_data(handle_t *handle, struct inode *inode,
 
 	len -= EXT4_MIN_INLINE_DATA_SIZE;
 	value = kzalloc(len, GFP_NOFS);
-	if (!value)
+	if (!value) {
+		error = -ENOMEM;
 		goto out;
+	}
 
 	error = ext4_xattr_ibody_get(inode, i.name_index, i.name,
 				     value, len);
@@ -442,6 +449,11 @@ static int ext4_destroy_inline_data_nolock(handle_t *handle,
 		}
 	}
 	ext4_clear_inode_flag(inode, EXT4_INODE_INLINE_DATA);
+	/*
+	 * Propagate changes to inode->i_flags as well - e.g. S_DAX may
+	 * get set.
+	 */
+	ext4_set_inode_flags(inode);
 
 	get_bh(is.iloc.bh);
 	error = ext4_mark_iloc_dirty(handle, inode, &is.iloc);
@@ -1028,7 +1040,7 @@ static int ext4_add_dirent_to_inline(handle_t *handle,
 	 * happen is that the times are slightly out of date
 	 * and/or different from the directory change time.
 	 */
-	dir->i_mtime = dir->i_ctime = ext4_current_time(dir);
+	dir->i_mtime = dir->i_ctime = current_time(dir);
 	ext4_update_dx_flag(dir);
 	dir->i_version++;
 	ext4_mark_inode_dirty(handle, dir);
@@ -1971,7 +1983,7 @@ out:
 	if (inode->i_nlink)
 		ext4_orphan_del(handle, inode);
 
-	inode->i_mtime = inode->i_ctime = ext4_current_time(inode);
+	inode->i_mtime = inode->i_ctime = current_time(inode);
 	ext4_mark_inode_dirty(handle, inode);
 	if (IS_SYNC(inode))
 		ext4_handle_sync(handle);
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 9c064727ed62..88d57af1b516 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -37,6 +37,7 @@
 #include <linux/printk.h>
 #include <linux/slab.h>
 #include <linux/bitops.h>
+#include <linux/iomap.h>
 
 #include "ext4_jbd2.h"
 #include "xattr.h"
@@ -71,10 +72,9 @@ static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
 			csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum,
 					   csum_size);
 			offset += csum_size;
-			csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
-					   EXT4_INODE_SIZE(inode->i_sb) -
-					   offset);
 		}
+		csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
+				   EXT4_INODE_SIZE(inode->i_sb) - offset);
 	}
 
 	return csum;
@@ -261,8 +261,15 @@ void ext4_evict_inode(struct inode *inode)
 			     "couldn't mark inode dirty (err %d)", err);
 		goto stop_handle;
 	}
-	if (inode->i_blocks)
-		ext4_truncate(inode);
+	if (inode->i_blocks) {
+		err = ext4_truncate(inode);
+		if (err) {
+			ext4_error(inode->i_sb,
+				   "couldn't truncate inode %lu (err %d)",
+				   inode->i_ino, err);
+			goto stop_handle;
+		}
+	}
 
 	/*
 	 * ext4_ext_truncate() doesn't reserve any slop when it
@@ -654,12 +661,8 @@ found:
 		if (flags & EXT4_GET_BLOCKS_ZERO &&
 		    map->m_flags & EXT4_MAP_MAPPED &&
 		    map->m_flags & EXT4_MAP_NEW) {
-			ext4_lblk_t i;
-
-			for (i = 0; i < map->m_len; i++) {
-				unmap_underlying_metadata(inode->i_sb->s_bdev,
-							  map->m_pblk + i);
-			}
+			clean_bdev_aliases(inode->i_sb->s_bdev, map->m_pblk,
+					   map->m_len);
 			ret = ext4_issue_zeroout(inode, map->m_lblk,
 						 map->m_pblk, map->m_len);
 			if (ret) {
@@ -767,6 +770,9 @@ static int _ext4_get_block(struct inode *inode, sector_t iblock,
 		ext4_update_bh_state(bh, map.m_flags);
 		bh->b_size = inode->i_sb->s_blocksize * map.m_len;
 		ret = 0;
+	} else if (ret == 0) {
+		/* hole case, need to fill in bh->b_size */
+		bh->b_size = inode->i_sb->s_blocksize * map.m_len;
 	}
 	return ret;
 }
@@ -1127,8 +1133,7 @@ static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
 			if (err)
 				break;
 			if (buffer_new(bh)) {
-				unmap_underlying_metadata(bh->b_bdev,
-							  bh->b_blocknr);
+				clean_bdev_bh_alias(bh);
 				if (PageUptodate(page)) {
 					clear_buffer_new(bh);
 					set_buffer_uptodate(bh);
@@ -1166,7 +1171,8 @@ static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
 	if (unlikely(err))
 		page_zero_new_buffers(page, from, to);
 	else if (decrypt)
-		err = fscrypt_decrypt_page(page);
+		err = fscrypt_decrypt_page(page->mapping->host, page,
+				PAGE_SIZE, 0, page->index);
 	return err;
 }
 #endif
@@ -2360,11 +2366,8 @@ static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
 
 	BUG_ON(map->m_len == 0);
 	if (map->m_flags & EXT4_MAP_NEW) {
-		struct block_device *bdev = inode->i_sb->s_bdev;
-		int i;
-
-		for (i = 0; i < map->m_len; i++)
-			unmap_underlying_metadata(bdev, map->m_pblk + i);
+		clean_bdev_aliases(inode->i_sb->s_bdev, map->m_pblk,
+				   map->m_len);
 	}
 	return 0;
 }
@@ -2891,7 +2894,8 @@ static int ext4_da_write_begin(struct file *file, struct address_space *mapping,
 
 	index = pos >> PAGE_SHIFT;
 
-	if (ext4_nonda_switch(inode->i_sb)) {
+	if (ext4_nonda_switch(inode->i_sb) ||
+	    S_ISLNK(inode->i_mode)) {
 		*fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
 		return ext4_write_begin(file, mapping, pos,
 					len, flags, pagep, fsdata);
@@ -3268,53 +3272,159 @@ static int ext4_releasepage(struct page *page, gfp_t wait)
 }
 
 #ifdef CONFIG_FS_DAX
-/*
- * Get block function for DAX IO and mmap faults. It takes care of converting
- * unwritten extents to written ones and initializes new / converted blocks
- * to zeros.
- */
-int ext4_dax_get_block(struct inode *inode, sector_t iblock,
-		       struct buffer_head *bh_result, int create)
+static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
+			    unsigned flags, struct iomap *iomap)
 {
+	unsigned int blkbits = inode->i_blkbits;
+	unsigned long first_block = offset >> blkbits;
+	unsigned long last_block = (offset + length - 1) >> blkbits;
+	struct ext4_map_blocks map;
 	int ret;
 
-	ext4_debug("inode %lu, create flag %d\n", inode->i_ino, create);
-	if (!create)
-		return _ext4_get_block(inode, iblock, bh_result, 0);
+	if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
+		return -ERANGE;
 
-	ret = ext4_get_block_trans(inode, iblock, bh_result,
-				   EXT4_GET_BLOCKS_PRE_IO |
-				   EXT4_GET_BLOCKS_CREATE_ZERO);
-	if (ret < 0)
-		return ret;
+	map.m_lblk = first_block;
+	map.m_len = last_block - first_block + 1;
 
-	if (buffer_unwritten(bh_result)) {
+	if (!(flags & IOMAP_WRITE)) {
+		ret = ext4_map_blocks(NULL, inode, &map, 0);
+	} else {
+		int dio_credits;
+		handle_t *handle;
+		int retries = 0;
+
+		/* Trim mapping request to maximum we can map at once for DIO */
+		if (map.m_len > DIO_MAX_BLOCKS)
+			map.m_len = DIO_MAX_BLOCKS;
+		dio_credits = ext4_chunk_trans_blocks(inode, map.m_len);
+retry:
 		/*
-		 * We are protected by i_mmap_sem or i_mutex so we know block
-		 * cannot go away from under us even though we dropped
-		 * i_data_sem. Convert extent to written and write zeros there.
+		 * Either we allocate blocks and then we don't get unwritten
+		 * extent so we have reserved enough credits, or the blocks
+		 * are already allocated and unwritten and in that case
+		 * extent conversion fits in the credits as well.
 		 */
-		ret = ext4_get_block_trans(inode, iblock, bh_result,
-					   EXT4_GET_BLOCKS_CONVERT |
-					   EXT4_GET_BLOCKS_CREATE_ZERO);
-		if (ret < 0)
+		handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
+					    dio_credits);
+		if (IS_ERR(handle))
+			return PTR_ERR(handle);
+
+		ret = ext4_map_blocks(handle, inode, &map,
+				      EXT4_GET_BLOCKS_CREATE_ZERO);
+		if (ret < 0) {
+			ext4_journal_stop(handle);
+			if (ret == -ENOSPC &&
+			    ext4_should_retry_alloc(inode->i_sb, &retries))
+				goto retry;
 			return ret;
+		}
+
+		/*
+		 * If we added blocks beyond i_size, we need to make sure they
+		 * will get truncated if we crash before updating i_size in
+		 * ext4_iomap_end(). For faults we don't need to do that (and
+		 * even cannot because for orphan list operations inode_lock is
+		 * required) - if we happen to instantiate block beyond i_size,
+		 * it is because we race with truncate which has already added
+		 * the inode to the orphan list.
+		 */
+		if (!(flags & IOMAP_FAULT) && first_block + map.m_len >
+		    (i_size_read(inode) + (1 << blkbits) - 1) >> blkbits) {
+			int err;
+
+			err = ext4_orphan_add(handle, inode);
+			if (err < 0) {
+				ext4_journal_stop(handle);
+				return err;
+			}
+		}
+		ext4_journal_stop(handle);
 	}
-	/*
-	 * At least for now we have to clear BH_New so that DAX code
-	 * doesn't attempt to zero blocks again in a racy way.
-	 */
-	clear_buffer_new(bh_result);
+
+	iomap->flags = 0;
+	iomap->bdev = inode->i_sb->s_bdev;
+	iomap->offset = first_block << blkbits;
+
+	if (ret == 0) {
+		iomap->type = IOMAP_HOLE;
+		iomap->blkno = IOMAP_NULL_BLOCK;
+		iomap->length = (u64)map.m_len << blkbits;
+	} else {
+		if (map.m_flags & EXT4_MAP_MAPPED) {
+			iomap->type = IOMAP_MAPPED;
+		} else if (map.m_flags & EXT4_MAP_UNWRITTEN) {
+			iomap->type = IOMAP_UNWRITTEN;
+		} else {
+			WARN_ON_ONCE(1);
+			return -EIO;
+		}
+		iomap->blkno = (sector_t)map.m_pblk << (blkbits - 9);
+		iomap->length = (u64)map.m_len << blkbits;
+	}
+
+	if (map.m_flags & EXT4_MAP_NEW)
+		iomap->flags |= IOMAP_F_NEW;
 	return 0;
 }
-#else
-/* Just define empty function, it will never get called. */
-int ext4_dax_get_block(struct inode *inode, sector_t iblock,
-		       struct buffer_head *bh_result, int create)
+
+static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length,
+			  ssize_t written, unsigned flags, struct iomap *iomap)
 {
-	BUG();
-	return 0;
+	int ret = 0;
+	handle_t *handle;
+	int blkbits = inode->i_blkbits;
+	bool truncate = false;
+
+	if (!(flags & IOMAP_WRITE) || (flags & IOMAP_FAULT))
+		return 0;
+
+	handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
+	if (IS_ERR(handle)) {
+		ret = PTR_ERR(handle);
+		goto orphan_del;
+	}
+	if (ext4_update_inode_size(inode, offset + written))
+		ext4_mark_inode_dirty(handle, inode);
+	/*
+	 * We may need to truncate allocated but not written blocks beyond EOF.
+	 */
+	if (iomap->offset + iomap->length > 
+	    ALIGN(inode->i_size, 1 << blkbits)) {
+		ext4_lblk_t written_blk, end_blk;
+
+		written_blk = (offset + written) >> blkbits;
+		end_blk = (offset + length) >> blkbits;
+		if (written_blk < end_blk && ext4_can_truncate(inode))
+			truncate = true;
+	}
+	/*
+	 * Remove inode from orphan list if we were extending a inode and
+	 * everything went fine.
+	 */
+	if (!truncate && inode->i_nlink &&
+	    !list_empty(&EXT4_I(inode)->i_orphan))
+		ext4_orphan_del(handle, inode);
+	ext4_journal_stop(handle);
+	if (truncate) {
+		ext4_truncate_failed_write(inode);
+orphan_del:
+		/*
+		 * If truncate failed early the inode might still be on the
+		 * orphan list; we need to make sure the inode is removed from
+		 * the orphan list in that case.
+		 */
+		if (inode->i_nlink)
+			ext4_orphan_del(NULL, inode);
+	}
+	return ret;
 }
+
+struct iomap_ops ext4_iomap_ops = {
+	.iomap_begin		= ext4_iomap_begin,
+	.iomap_end		= ext4_iomap_end,
+};
+
 #endif
 
 static int ext4_end_io_dio(struct kiocb *iocb, loff_t offset,
@@ -3436,19 +3546,7 @@ static ssize_t ext4_direct_IO_write(struct kiocb *iocb, struct iov_iter *iter)
 	iocb->private = NULL;
 	if (overwrite)
 		get_block_func = ext4_dio_get_block_overwrite;
-	else if (IS_DAX(inode)) {
-		/*
-		 * We can avoid zeroing for aligned DAX writes beyond EOF. Other
-		 * writes need zeroing either because they can race with page
-		 * faults or because they use partial blocks.
-		 */
-		if (round_down(offset, 1<<inode->i_blkbits) >= inode->i_size &&
-		    ext4_aligned_io(inode, offset, count))
-			get_block_func = ext4_dio_get_block;
-		else
-			get_block_func = ext4_dax_get_block;
-		dio_flags = DIO_LOCKING;
-	} else if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) ||
+	else if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) ||
 		   round_down(offset, 1 << inode->i_blkbits) >= inode->i_size) {
 		get_block_func = ext4_dio_get_block;
 		dio_flags = DIO_LOCKING | DIO_SKIP_HOLES;
@@ -3462,14 +3560,9 @@ static ssize_t ext4_direct_IO_write(struct kiocb *iocb, struct iov_iter *iter)
 #ifdef CONFIG_EXT4_FS_ENCRYPTION
 	BUG_ON(ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode));
 #endif
-	if (IS_DAX(inode)) {
-		ret = dax_do_io(iocb, inode, iter, get_block_func,
-				ext4_end_io_dio, dio_flags);
-	} else
-		ret = __blockdev_direct_IO(iocb, inode,
-					   inode->i_sb->s_bdev, iter,
-					   get_block_func,
-					   ext4_end_io_dio, NULL, dio_flags);
+	ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, iter,
+				   get_block_func, ext4_end_io_dio, NULL,
+				   dio_flags);
 
 	if (ret > 0 && !overwrite && ext4_test_inode_state(inode,
 						EXT4_STATE_DIO_UNWRITTEN)) {
@@ -3538,6 +3631,7 @@ static ssize_t ext4_direct_IO_read(struct kiocb *iocb, struct iov_iter *iter)
 {
 	struct address_space *mapping = iocb->ki_filp->f_mapping;
 	struct inode *inode = mapping->host;
+	size_t count = iov_iter_count(iter);
 	ssize_t ret;
 
 	/*
@@ -3546,19 +3640,12 @@ static ssize_t ext4_direct_IO_read(struct kiocb *iocb, struct iov_iter *iter)
 	 * we are protected against page writeback as well.
 	 */
 	inode_lock_shared(inode);
-	if (IS_DAX(inode)) {
-		ret = dax_do_io(iocb, inode, iter, ext4_dio_get_block, NULL, 0);
-	} else {
-		size_t count = iov_iter_count(iter);
-
-		ret = filemap_write_and_wait_range(mapping, iocb->ki_pos,
-						   iocb->ki_pos + count);
-		if (ret)
-			goto out_unlock;
-		ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev,
-					   iter, ext4_dio_get_block,
-					   NULL, NULL, 0);
-	}
+	ret = filemap_write_and_wait_range(mapping, iocb->ki_pos,
+					   iocb->ki_pos + count);
+	if (ret)
+		goto out_unlock;
+	ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev,
+				   iter, ext4_dio_get_block, NULL, NULL, 0);
 out_unlock:
 	inode_unlock_shared(inode);
 	return ret;
@@ -3587,6 +3674,10 @@ static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
 	if (ext4_has_inline_data(inode))
 		return 0;
 
+	/* DAX uses iomap path now */
+	if (WARN_ON_ONCE(IS_DAX(inode)))
+		return 0;
+
 	trace_ext4_direct_IO_enter(inode, offset, count, iov_iter_rw(iter));
 	if (iov_iter_rw(iter) == READ)
 		ret = ext4_direct_IO_read(iocb, iter);
@@ -3615,6 +3706,13 @@ static int ext4_journalled_set_page_dirty(struct page *page)
 	return __set_page_dirty_nobuffers(page);
 }
 
+static int ext4_set_page_dirty(struct page *page)
+{
+	WARN_ON_ONCE(!PageLocked(page) && !PageDirty(page));
+	WARN_ON_ONCE(!page_has_buffers(page));
+	return __set_page_dirty_buffers(page);
+}
+
 static const struct address_space_operations ext4_aops = {
 	.readpage		= ext4_readpage,
 	.readpages		= ext4_readpages,
@@ -3622,6 +3720,7 @@ static const struct address_space_operations ext4_aops = {
 	.writepages		= ext4_writepages,
 	.write_begin		= ext4_write_begin,
 	.write_end		= ext4_write_end,
+	.set_page_dirty		= ext4_set_page_dirty,
 	.bmap			= ext4_bmap,
 	.invalidatepage		= ext4_invalidatepage,
 	.releasepage		= ext4_releasepage,
@@ -3654,6 +3753,7 @@ static const struct address_space_operations ext4_da_aops = {
 	.writepages		= ext4_writepages,
 	.write_begin		= ext4_da_write_begin,
 	.write_end		= ext4_da_write_end,
+	.set_page_dirty		= ext4_set_page_dirty,
 	.bmap			= ext4_bmap,
 	.invalidatepage		= ext4_da_invalidatepage,
 	.releasepage		= ext4_releasepage,
@@ -3743,7 +3843,8 @@ static int __ext4_block_zero_page_range(handle_t *handle,
 			/* We expect the key to be set. */
 			BUG_ON(!fscrypt_has_encryption_key(inode));
 			BUG_ON(blocksize != PAGE_SIZE);
-			WARN_ON_ONCE(fscrypt_decrypt_page(page));
+			WARN_ON_ONCE(fscrypt_decrypt_page(page->mapping->host,
+						page, PAGE_SIZE, 0, page->index));
 		}
 	}
 	if (ext4_should_journal_data(inode)) {
@@ -3792,8 +3893,10 @@ static int ext4_block_zero_page_range(handle_t *handle,
 	if (length > max || length < 0)
 		length = max;
 
-	if (IS_DAX(inode))
-		return dax_zero_page_range(inode, from, length, ext4_get_block);
+	if (IS_DAX(inode)) {
+		return iomap_zero_range(inode, from, length, NULL,
+					&ext4_iomap_ops);
+	}
 	return __ext4_block_zero_page_range(handle, mapping, from, length);
 }
 
@@ -4026,7 +4129,7 @@ int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length)
 	if (IS_SYNC(inode))
 		ext4_handle_sync(handle);
 
-	inode->i_mtime = inode->i_ctime = ext4_current_time(inode);
+	inode->i_mtime = inode->i_ctime = current_time(inode);
 	ext4_mark_inode_dirty(handle, inode);
 out_stop:
 	ext4_journal_stop(handle);
@@ -4091,10 +4194,11 @@ int ext4_inode_attach_jinode(struct inode *inode)
  * that's fine - as long as they are linked from the inode, the post-crash
  * ext4_truncate() run will find them and release them.
  */
-void ext4_truncate(struct inode *inode)
+int ext4_truncate(struct inode *inode)
 {
 	struct ext4_inode_info *ei = EXT4_I(inode);
 	unsigned int credits;
+	int err = 0;
 	handle_t *handle;
 	struct address_space *mapping = inode->i_mapping;
 
@@ -4108,7 +4212,7 @@ void ext4_truncate(struct inode *inode)
 	trace_ext4_truncate_enter(inode);
 
 	if (!ext4_can_truncate(inode))
-		return;
+		return 0;
 
 	ext4_clear_inode_flag(inode, EXT4_INODE_EOFBLOCKS);
 
@@ -4120,13 +4224,13 @@ void ext4_truncate(struct inode *inode)
 
 		ext4_inline_data_truncate(inode, &has_inline);
 		if (has_inline)
-			return;
+			return 0;
 	}
 
 	/* If we zero-out tail of the page, we have to create jinode for jbd2 */
 	if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {
 		if (ext4_inode_attach_jinode(inode) < 0)
-			return;
+			return 0;
 	}
 
 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
@@ -4135,10 +4239,8 @@ void ext4_truncate(struct inode *inode)
 		credits = ext4_blocks_for_truncate(inode);
 
 	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
-	if (IS_ERR(handle)) {
-		ext4_std_error(inode->i_sb, PTR_ERR(handle));
-		return;
-	}
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
 
 	if (inode->i_size & (inode->i_sb->s_blocksize - 1))
 		ext4_block_truncate_page(handle, mapping, inode->i_size);
@@ -4152,7 +4254,8 @@ void ext4_truncate(struct inode *inode)
 	 * Implication: the file must always be in a sane, consistent
 	 * truncatable state while each transaction commits.
 	 */
-	if (ext4_orphan_add(handle, inode))
+	err = ext4_orphan_add(handle, inode);
+	if (err)
 		goto out_stop;
 
 	down_write(&EXT4_I(inode)->i_data_sem);
@@ -4160,11 +4263,13 @@ void ext4_truncate(struct inode *inode)
 	ext4_discard_preallocations(inode);
 
 	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
-		ext4_ext_truncate(handle, inode);
+		err = ext4_ext_truncate(handle, inode);
 	else
 		ext4_ind_truncate(handle, inode);
 
 	up_write(&ei->i_data_sem);
+	if (err)
+		goto out_stop;
 
 	if (IS_SYNC(inode))
 		ext4_handle_sync(handle);
@@ -4180,11 +4285,12 @@ out_stop:
 	if (inode->i_nlink)
 		ext4_orphan_del(handle, inode);
 
-	inode->i_mtime = inode->i_ctime = ext4_current_time(inode);
+	inode->i_mtime = inode->i_ctime = current_time(inode);
 	ext4_mark_inode_dirty(handle, inode);
 	ext4_journal_stop(handle);
 
 	trace_ext4_truncate_exit(inode);
+	return err;
 }
 
 /*
@@ -4352,7 +4458,9 @@ void ext4_set_inode_flags(struct inode *inode)
 		new_fl |= S_NOATIME;
 	if (flags & EXT4_DIRSYNC_FL)
 		new_fl |= S_DIRSYNC;
-	if (test_opt(inode->i_sb, DAX) && S_ISREG(inode->i_mode))
+	if (test_opt(inode->i_sb, DAX) && S_ISREG(inode->i_mode) &&
+	    !ext4_should_journal_data(inode) && !ext4_has_inline_data(inode) &&
+	    !ext4_encrypted_inode(inode))
 		new_fl |= S_DAX;
 	inode_set_flags(inode, new_fl,
 			S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX);
@@ -4411,7 +4519,9 @@ static inline void ext4_iget_extra_inode(struct inode *inode,
 {
 	__le32 *magic = (void *)raw_inode +
 			EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;
-	if (*magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
+	if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize + sizeof(__le32) <=
+	    EXT4_INODE_SIZE(inode->i_sb) &&
+	    *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
 		ext4_set_inode_state(inode, EXT4_STATE_XATTR);
 		ext4_find_inline_data_nolock(inode);
 	} else
@@ -4434,6 +4544,7 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino)
 	struct inode *inode;
 	journal_t *journal = EXT4_SB(sb)->s_journal;
 	long ret;
+	loff_t size;
 	int block;
 	uid_t i_uid;
 	gid_t i_gid;
@@ -4456,10 +4567,12 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino)
 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
 		ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
 		if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
-		    EXT4_INODE_SIZE(inode->i_sb)) {
-			EXT4_ERROR_INODE(inode, "bad extra_isize (%u != %u)",
-				EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize,
-				EXT4_INODE_SIZE(inode->i_sb));
+			EXT4_INODE_SIZE(inode->i_sb) ||
+		    (ei->i_extra_isize & 3)) {
+			EXT4_ERROR_INODE(inode,
+					 "bad extra_isize %u (inode size %u)",
+					 ei->i_extra_isize,
+					 EXT4_INODE_SIZE(inode->i_sb));
 			ret = -EFSCORRUPTED;
 			goto bad_inode;
 		}
@@ -4534,6 +4647,11 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino)
 		ei->i_file_acl |=
 			((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
 	inode->i_size = ext4_isize(raw_inode);
+	if ((size = i_size_read(inode)) < 0) {
+		EXT4_ERROR_INODE(inode, "bad i_size value: %lld", size);
+		ret = -EFSCORRUPTED;
+		goto bad_inode;
+	}
 	ei->i_disksize = inode->i_size;
 #ifdef CONFIG_QUOTA
 	ei->i_reserved_quota = 0;
@@ -4577,6 +4695,7 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino)
 	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
 		if (ei->i_extra_isize == 0) {
 			/* The extra space is currently unused. Use it. */
+			BUILD_BUG_ON(sizeof(struct ext4_inode) & 3);
 			ei->i_extra_isize = sizeof(struct ext4_inode) -
 					    EXT4_GOOD_OLD_INODE_SIZE;
 		} else {
@@ -5154,7 +5273,7 @@ int ext4_setattr(struct dentry *dentry, struct iattr *attr)
 			 * update c/mtime in shrink case below
 			 */
 			if (!shrink) {
-				inode->i_mtime = ext4_current_time(inode);
+				inode->i_mtime = current_time(inode);
 				inode->i_ctime = inode->i_mtime;
 			}
 			down_write(&EXT4_I(inode)->i_data_sem);
@@ -5199,12 +5318,15 @@ int ext4_setattr(struct dentry *dentry, struct iattr *attr)
 		 * in data=journal mode to make pages freeable.
 		 */
 		truncate_pagecache(inode, inode->i_size);
-		if (shrink)
-			ext4_truncate(inode);
+		if (shrink) {
+			rc = ext4_truncate(inode);
+			if (rc)
+				error = rc;
+		}
 		up_write(&EXT4_I(inode)->i_mmap_sem);
 	}
 
-	if (!rc) {
+	if (!error) {
 		setattr_copy(inode, attr);
 		mark_inode_dirty(inode);
 	}
@@ -5216,7 +5338,7 @@ int ext4_setattr(struct dentry *dentry, struct iattr *attr)
 	if (orphan && inode->i_nlink)
 		ext4_orphan_del(NULL, inode);
 
-	if (!rc && (ia_valid & ATTR_MODE))
+	if (!error && (ia_valid & ATTR_MODE))
 		rc = posix_acl_chmod(inode, inode->i_mode);
 
 err_out:
@@ -5455,18 +5577,20 @@ int ext4_mark_inode_dirty(handle_t *handle, struct inode *inode)
 	err = ext4_reserve_inode_write(handle, inode, &iloc);
 	if (err)
 		return err;
-	if (ext4_handle_valid(handle) &&
-	    EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize &&
+	if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize &&
 	    !ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
 		/*
-		 * We need extra buffer credits since we may write into EA block
+		 * In nojournal mode, we can immediately attempt to expand
+		 * the inode.  When journaled, we first need to obtain extra
+		 * buffer credits since we may write into the EA block
 		 * with this same handle. If journal_extend fails, then it will
 		 * only result in a minor loss of functionality for that inode.
 		 * If this is felt to be critical, then e2fsck should be run to
 		 * force a large enough s_min_extra_isize.
 		 */
-		if ((jbd2_journal_extend(handle,
-			     EXT4_DATA_TRANS_BLOCKS(inode->i_sb))) == 0) {
+		if (!ext4_handle_valid(handle) ||
+		    jbd2_journal_extend(handle,
+			     EXT4_DATA_TRANS_BLOCKS(inode->i_sb)) == 0) {
 			ret = ext4_expand_extra_isize(inode,
 						      sbi->s_want_extra_isize,
 						      iloc, handle);
@@ -5620,6 +5744,11 @@ int ext4_change_inode_journal_flag(struct inode *inode, int val)
 		ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
 	}
 	ext4_set_aops(inode);
+	/*
+	 * Update inode->i_flags after EXT4_INODE_JOURNAL_DATA was updated.
+	 * E.g. S_DAX may get cleared / set.
+	 */
+	ext4_set_inode_flags(inode);
 
 	jbd2_journal_unlock_updates(journal);
 	percpu_up_write(&sbi->s_journal_flag_rwsem);
diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c
index bf5ae8ebbc97..49fd1371bfa2 100644
--- a/fs/ext4/ioctl.c
+++ b/fs/ext4/ioctl.c
@@ -153,7 +153,7 @@ static long swap_inode_boot_loader(struct super_block *sb,
 
 	swap_inode_data(inode, inode_bl);
 
-	inode->i_ctime = inode_bl->i_ctime = ext4_current_time(inode);
+	inode->i_ctime = inode_bl->i_ctime = current_time(inode);
 
 	spin_lock(&sbi->s_next_gen_lock);
 	inode->i_generation = sbi->s_next_generation++;
@@ -191,6 +191,7 @@ journal_err_out:
 	return err;
 }
 
+#ifdef CONFIG_EXT4_FS_ENCRYPTION
 static int uuid_is_zero(__u8 u[16])
 {
 	int	i;
@@ -200,6 +201,7 @@ static int uuid_is_zero(__u8 u[16])
 			return 0;
 	return 1;
 }
+#endif
 
 static int ext4_ioctl_setflags(struct inode *inode,
 			       unsigned int flags)
@@ -248,8 +250,11 @@ static int ext4_ioctl_setflags(struct inode *inode,
 			err = -EOPNOTSUPP;
 			goto flags_out;
 		}
-	} else if (oldflags & EXT4_EOFBLOCKS_FL)
-		ext4_truncate(inode);
+	} else if (oldflags & EXT4_EOFBLOCKS_FL) {
+		err = ext4_truncate(inode);
+		if (err)
+			goto flags_out;
+	}
 
 	handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
 	if (IS_ERR(handle)) {
@@ -265,6 +270,9 @@ static int ext4_ioctl_setflags(struct inode *inode,
 	for (i = 0, mask = 1; i < 32; i++, mask <<= 1) {
 		if (!(mask & EXT4_FL_USER_MODIFIABLE))
 			continue;
+		/* These flags get special treatment later */
+		if (mask == EXT4_JOURNAL_DATA_FL || mask == EXT4_EXTENTS_FL)
+			continue;
 		if (mask & flags)
 			ext4_set_inode_flag(inode, i);
 		else
@@ -272,7 +280,7 @@ static int ext4_ioctl_setflags(struct inode *inode,
 	}
 
 	ext4_set_inode_flags(inode);
-	inode->i_ctime = ext4_current_time(inode);
+	inode->i_ctime = current_time(inode);
 
 	err = ext4_mark_iloc_dirty(handle, inode, &iloc);
 flags_err:
@@ -368,7 +376,7 @@ static int ext4_ioctl_setproject(struct file *filp, __u32 projid)
 	}
 
 	EXT4_I(inode)->i_projid = kprojid;
-	inode->i_ctime = ext4_current_time(inode);
+	inode->i_ctime = current_time(inode);
 out_dirty:
 	rc = ext4_mark_iloc_dirty(handle, inode, &iloc);
 	if (!err)
@@ -409,6 +417,10 @@ static inline __u32 ext4_iflags_to_xflags(unsigned long iflags)
 	return xflags;
 }
 
+#define EXT4_SUPPORTED_FS_XFLAGS (FS_XFLAG_SYNC | FS_XFLAG_IMMUTABLE | \
+				  FS_XFLAG_APPEND | FS_XFLAG_NODUMP | \
+				  FS_XFLAG_NOATIME | FS_XFLAG_PROJINHERIT)
+
 /* Transfer xflags flags to internal */
 static inline unsigned long ext4_xflags_to_iflags(__u32 xflags)
 {
@@ -453,12 +465,22 @@ long ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 		if (get_user(flags, (int __user *) arg))
 			return -EFAULT;
 
+		if (flags & ~EXT4_FL_USER_VISIBLE)
+			return -EOPNOTSUPP;
+		/*
+		 * chattr(1) grabs flags via GETFLAGS, modifies the result and
+		 * passes that to SETFLAGS. So we cannot easily make SETFLAGS
+		 * more restrictive than just silently masking off visible but
+		 * not settable flags as we always did.
+		 */
+		flags &= EXT4_FL_USER_MODIFIABLE;
+		if (ext4_mask_flags(inode->i_mode, flags) != flags)
+			return -EOPNOTSUPP;
+
 		err = mnt_want_write_file(filp);
 		if (err)
 			return err;
 
-		flags = ext4_mask_flags(inode->i_mode, flags);
-
 		inode_lock(inode);
 		err = ext4_ioctl_setflags(inode, flags);
 		inode_unlock(inode);
@@ -500,7 +522,7 @@ long ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 		}
 		err = ext4_reserve_inode_write(handle, inode, &iloc);
 		if (err == 0) {
-			inode->i_ctime = ext4_current_time(inode);
+			inode->i_ctime = current_time(inode);
 			inode->i_generation = generation;
 			err = ext4_mark_iloc_dirty(handle, inode, &iloc);
 		}
@@ -765,28 +787,19 @@ resizefs_out:
 	}
 	case EXT4_IOC_PRECACHE_EXTENTS:
 		return ext4_ext_precache(inode);
-	case EXT4_IOC_SET_ENCRYPTION_POLICY: {
-#ifdef CONFIG_EXT4_FS_ENCRYPTION
-		struct fscrypt_policy policy;
 
+	case EXT4_IOC_SET_ENCRYPTION_POLICY:
 		if (!ext4_has_feature_encrypt(sb))
 			return -EOPNOTSUPP;
+		return fscrypt_ioctl_set_policy(filp, (const void __user *)arg);
 
-		if (copy_from_user(&policy,
-				   (struct fscrypt_policy __user *)arg,
-				   sizeof(policy)))
-			return -EFAULT;
-		return fscrypt_process_policy(filp, &policy);
-#else
-		return -EOPNOTSUPP;
-#endif
-	}
 	case EXT4_IOC_GET_ENCRYPTION_PWSALT: {
+#ifdef CONFIG_EXT4_FS_ENCRYPTION
 		int err, err2;
 		struct ext4_sb_info *sbi = EXT4_SB(sb);
 		handle_t *handle;
 
-		if (!ext4_sb_has_crypto(sb))
+		if (!ext4_has_feature_encrypt(sb))
 			return -EOPNOTSUPP;
 		if (uuid_is_zero(sbi->s_es->s_encrypt_pw_salt)) {
 			err = mnt_want_write_file(filp);
@@ -816,24 +829,13 @@ resizefs_out:
 				 sbi->s_es->s_encrypt_pw_salt, 16))
 			return -EFAULT;
 		return 0;
-	}
-	case EXT4_IOC_GET_ENCRYPTION_POLICY: {
-#ifdef CONFIG_EXT4_FS_ENCRYPTION
-		struct fscrypt_policy policy;
-		int err = 0;
-
-		if (!ext4_encrypted_inode(inode))
-			return -ENOENT;
-		err = fscrypt_get_policy(inode, &policy);
-		if (err)
-			return err;
-		if (copy_to_user((void __user *)arg, &policy, sizeof(policy)))
-			return -EFAULT;
-		return 0;
 #else
 		return -EOPNOTSUPP;
 #endif
 	}
+	case EXT4_IOC_GET_ENCRYPTION_POLICY:
+		return fscrypt_ioctl_get_policy(filp, (void __user *)arg);
+
 	case EXT4_IOC_FSGETXATTR:
 	{
 		struct fsxattr fa;
@@ -865,13 +867,17 @@ resizefs_out:
 		if (!inode_owner_or_capable(inode))
 			return -EACCES;
 
+		if (fa.fsx_xflags & ~EXT4_SUPPORTED_FS_XFLAGS)
+			return -EOPNOTSUPP;
+
+		flags = ext4_xflags_to_iflags(fa.fsx_xflags);
+		if (ext4_mask_flags(inode->i_mode, flags) != flags)
+			return -EOPNOTSUPP;
+
 		err = mnt_want_write_file(filp);
 		if (err)
 			return err;
 
-		flags = ext4_xflags_to_iflags(fa.fsx_xflags);
-		flags = ext4_mask_flags(inode->i_mode, flags);
-
 		inode_lock(inode);
 		flags = (ei->i_flags & ~EXT4_FL_XFLAG_VISIBLE) |
 			 (flags & EXT4_FL_XFLAG_VISIBLE);
diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c
index f418f55c2bbe..7ae43c59bc79 100644
--- a/fs/ext4/mballoc.c
+++ b/fs/ext4/mballoc.c
@@ -669,7 +669,7 @@ static void ext4_mb_mark_free_simple(struct super_block *sb,
 	ext4_grpblk_t min;
 	ext4_grpblk_t max;
 	ext4_grpblk_t chunk;
-	unsigned short border;
+	unsigned int border;
 
 	BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb));
 
@@ -2287,7 +2287,7 @@ static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v)
 	struct ext4_group_info *grinfo;
 	struct sg {
 		struct ext4_group_info info;
-		ext4_grpblk_t counters[16];
+		ext4_grpblk_t counters[EXT4_MAX_BLOCK_LOG_SIZE + 2];
 	} sg;
 
 	group--;
diff --git a/fs/ext4/mmp.c b/fs/ext4/mmp.c
index d89754ef1aab..eb9835638680 100644
--- a/fs/ext4/mmp.c
+++ b/fs/ext4/mmp.c
@@ -35,7 +35,7 @@ static void ext4_mmp_csum_set(struct super_block *sb, struct mmp_struct *mmp)
 }
 
 /*
- * Write the MMP block using WRITE_SYNC to try to get the block on-disk
+ * Write the MMP block using REQ_SYNC to try to get the block on-disk
  * faster.
  */
 static int write_mmp_block(struct super_block *sb, struct buffer_head *bh)
@@ -52,7 +52,7 @@ static int write_mmp_block(struct super_block *sb, struct buffer_head *bh)
 	lock_buffer(bh);
 	bh->b_end_io = end_buffer_write_sync;
 	get_bh(bh);
-	submit_bh(REQ_OP_WRITE, WRITE_SYNC | REQ_META | REQ_PRIO, bh);
+	submit_bh(REQ_OP_WRITE, REQ_SYNC | REQ_META | REQ_PRIO, bh);
 	wait_on_buffer(bh);
 	sb_end_write(sb);
 	if (unlikely(!buffer_uptodate(bh)))
@@ -88,7 +88,7 @@ static int read_mmp_block(struct super_block *sb, struct buffer_head **bh,
 	get_bh(*bh);
 	lock_buffer(*bh);
 	(*bh)->b_end_io = end_buffer_read_sync;
-	submit_bh(REQ_OP_READ, READ_SYNC | REQ_META | REQ_PRIO, *bh);
+	submit_bh(REQ_OP_READ, REQ_META | REQ_PRIO, *bh);
 	wait_on_buffer(*bh);
 	if (!buffer_uptodate(*bh)) {
 		ret = -EIO;
diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c
index 104f8bfba718..eadba919f26b 100644
--- a/fs/ext4/namei.c
+++ b/fs/ext4/namei.c
@@ -1941,7 +1941,7 @@ static int add_dirent_to_buf(handle_t *handle, struct ext4_filename *fname,
 	 * happen is that the times are slightly out of date
 	 * and/or different from the directory change time.
 	 */
-	dir->i_mtime = dir->i_ctime = ext4_current_time(dir);
+	dir->i_mtime = dir->i_ctime = current_time(dir);
 	ext4_update_dx_flag(dir);
 	dir->i_version++;
 	ext4_mark_inode_dirty(handle, dir);
@@ -2987,7 +2987,7 @@ static int ext4_rmdir(struct inode *dir, struct dentry *dentry)
 	 * recovery. */
 	inode->i_size = 0;
 	ext4_orphan_add(handle, inode);
-	inode->i_ctime = dir->i_ctime = dir->i_mtime = ext4_current_time(inode);
+	inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
 	ext4_mark_inode_dirty(handle, inode);
 	ext4_dec_count(handle, dir);
 	ext4_update_dx_flag(dir);
@@ -3050,13 +3050,13 @@ static int ext4_unlink(struct inode *dir, struct dentry *dentry)
 	retval = ext4_delete_entry(handle, dir, de, bh);
 	if (retval)
 		goto end_unlink;
-	dir->i_ctime = dir->i_mtime = ext4_current_time(dir);
+	dir->i_ctime = dir->i_mtime = current_time(dir);
 	ext4_update_dx_flag(dir);
 	ext4_mark_inode_dirty(handle, dir);
 	drop_nlink(inode);
 	if (!inode->i_nlink)
 		ext4_orphan_add(handle, inode);
-	inode->i_ctime = ext4_current_time(inode);
+	inode->i_ctime = current_time(inode);
 	ext4_mark_inode_dirty(handle, inode);
 
 end_unlink:
@@ -3254,7 +3254,7 @@ retry:
 	if (IS_DIRSYNC(dir))
 		ext4_handle_sync(handle);
 
-	inode->i_ctime = ext4_current_time(inode);
+	inode->i_ctime = current_time(inode);
 	ext4_inc_count(handle, inode);
 	ihold(inode);
 
@@ -3381,7 +3381,7 @@ static int ext4_setent(handle_t *handle, struct ext4_renament *ent,
 		ent->de->file_type = file_type;
 	ent->dir->i_version++;
 	ent->dir->i_ctime = ent->dir->i_mtime =
-		ext4_current_time(ent->dir);
+		current_time(ent->dir);
 	ext4_mark_inode_dirty(handle, ent->dir);
 	BUFFER_TRACE(ent->bh, "call ext4_handle_dirty_metadata");
 	if (!ent->inlined) {
@@ -3651,7 +3651,7 @@ static int ext4_rename(struct inode *old_dir, struct dentry *old_dentry,
 	 * Like most other Unix systems, set the ctime for inodes on a
 	 * rename.
 	 */
-	old.inode->i_ctime = ext4_current_time(old.inode);
+	old.inode->i_ctime = current_time(old.inode);
 	ext4_mark_inode_dirty(handle, old.inode);
 
 	if (!whiteout) {
@@ -3663,9 +3663,9 @@ static int ext4_rename(struct inode *old_dir, struct dentry *old_dentry,
 
 	if (new.inode) {
 		ext4_dec_count(handle, new.inode);
-		new.inode->i_ctime = ext4_current_time(new.inode);
+		new.inode->i_ctime = current_time(new.inode);
 	}
-	old.dir->i_ctime = old.dir->i_mtime = ext4_current_time(old.dir);
+	old.dir->i_ctime = old.dir->i_mtime = current_time(old.dir);
 	ext4_update_dx_flag(old.dir);
 	if (old.dir_bh) {
 		retval = ext4_rename_dir_finish(handle, &old, new.dir->i_ino);
@@ -3723,6 +3723,7 @@ static int ext4_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
 	};
 	u8 new_file_type;
 	int retval;
+	struct timespec ctime;
 
 	if ((ext4_encrypted_inode(old_dir) ||
 	     ext4_encrypted_inode(new_dir)) &&
@@ -3823,8 +3824,9 @@ static int ext4_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
 	 * Like most other Unix systems, set the ctime for inodes on a
 	 * rename.
 	 */
-	old.inode->i_ctime = ext4_current_time(old.inode);
-	new.inode->i_ctime = ext4_current_time(new.inode);
+	ctime = current_time(old.inode);
+	old.inode->i_ctime = ctime;
+	new.inode->i_ctime = ctime;
 	ext4_mark_inode_dirty(handle, old.inode);
 	ext4_mark_inode_dirty(handle, new.inode);
 
diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c
index 0094923e5ebf..d83b0f3c5fe9 100644
--- a/fs/ext4/page-io.c
+++ b/fs/ext4/page-io.c
@@ -340,7 +340,7 @@ void ext4_io_submit(struct ext4_io_submit *io)
 
 	if (bio) {
 		int io_op_flags = io->io_wbc->sync_mode == WB_SYNC_ALL ?
-				  WRITE_SYNC : 0;
+				  REQ_SYNC : 0;
 		bio_set_op_attrs(io->io_bio, REQ_OP_WRITE, io_op_flags);
 		submit_bio(io->io_bio);
 	}
@@ -457,7 +457,7 @@ int ext4_bio_write_page(struct ext4_io_submit *io,
 		}
 		if (buffer_new(bh)) {
 			clear_buffer_new(bh);
-			unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
+			clean_bdev_bh_alias(bh);
 		}
 		set_buffer_async_write(bh);
 		nr_to_submit++;
@@ -470,7 +470,8 @@ int ext4_bio_write_page(struct ext4_io_submit *io,
 		gfp_t gfp_flags = GFP_NOFS;
 
 	retry_encrypt:
-		data_page = fscrypt_encrypt_page(inode, page, gfp_flags);
+		data_page = fscrypt_encrypt_page(inode, page, PAGE_SIZE, 0,
+						page->index, gfp_flags);
 		if (IS_ERR(data_page)) {
 			ret = PTR_ERR(data_page);
 			if (ret == -ENOMEM && wbc->sync_mode == WB_SYNC_ALL) {
diff --git a/fs/ext4/super.c b/fs/ext4/super.c
index 52b0530c5d65..dfc8309d7755 100644
--- a/fs/ext4/super.c
+++ b/fs/ext4/super.c
@@ -863,7 +863,6 @@ static void ext4_put_super(struct super_block *sb)
 	percpu_counter_destroy(&sbi->s_dirs_counter);
 	percpu_counter_destroy(&sbi->s_dirtyclusters_counter);
 	percpu_free_rwsem(&sbi->s_journal_flag_rwsem);
-	brelse(sbi->s_sbh);
 #ifdef CONFIG_QUOTA
 	for (i = 0; i < EXT4_MAXQUOTAS; i++)
 		kfree(sbi->s_qf_names[i]);
@@ -895,6 +894,7 @@ static void ext4_put_super(struct super_block *sb)
 	}
 	if (sbi->s_mmp_tsk)
 		kthread_stop(sbi->s_mmp_tsk);
+	brelse(sbi->s_sbh);
 	sb->s_fs_info = NULL;
 	/*
 	 * Now that we are completely done shutting down the
@@ -1114,37 +1114,55 @@ static int ext4_prepare_context(struct inode *inode)
 static int ext4_set_context(struct inode *inode, const void *ctx, size_t len,
 							void *fs_data)
 {
-	handle_t *handle;
-	int res, res2;
+	handle_t *handle = fs_data;
+	int res, res2, retries = 0;
+
+	/*
+	 * If a journal handle was specified, then the encryption context is
+	 * being set on a new inode via inheritance and is part of a larger
+	 * transaction to create the inode.  Otherwise the encryption context is
+	 * being set on an existing inode in its own transaction.  Only in the
+	 * latter case should the "retry on ENOSPC" logic be used.
+	 */
 
-	/* fs_data is null when internally used. */
-	if (fs_data) {
-		res  = ext4_xattr_set(inode, EXT4_XATTR_INDEX_ENCRYPTION,
-				EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, ctx,
-				len, 0);
+	if (handle) {
+		res = ext4_xattr_set_handle(handle, inode,
+					    EXT4_XATTR_INDEX_ENCRYPTION,
+					    EXT4_XATTR_NAME_ENCRYPTION_CONTEXT,
+					    ctx, len, 0);
 		if (!res) {
 			ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT);
 			ext4_clear_inode_state(inode,
 					EXT4_STATE_MAY_INLINE_DATA);
+			/*
+			 * Update inode->i_flags - e.g. S_DAX may get disabled
+			 */
+			ext4_set_inode_flags(inode);
 		}
 		return res;
 	}
 
+retry:
 	handle = ext4_journal_start(inode, EXT4_HT_MISC,
 			ext4_jbd2_credits_xattr(inode));
 	if (IS_ERR(handle))
 		return PTR_ERR(handle);
 
-	res = ext4_xattr_set(inode, EXT4_XATTR_INDEX_ENCRYPTION,
-			EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, ctx,
-			len, 0);
+	res = ext4_xattr_set_handle(handle, inode, EXT4_XATTR_INDEX_ENCRYPTION,
+				    EXT4_XATTR_NAME_ENCRYPTION_CONTEXT,
+				    ctx, len, 0);
 	if (!res) {
 		ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT);
+		/* Update inode->i_flags - e.g. S_DAX may get disabled */
+		ext4_set_inode_flags(inode);
 		res = ext4_mark_inode_dirty(handle, inode);
 		if (res)
 			EXT4_ERROR_INODE(inode, "Failed to mark inode dirty");
 	}
 	res2 = ext4_journal_stop(handle);
+
+	if (res == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
+		goto retry;
 	if (!res)
 		res = res2;
 	return res;
@@ -1883,12 +1901,6 @@ static int parse_options(char *options, struct super_block *sb,
 			return 0;
 		}
 	}
-	if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA &&
-	    test_opt(sb, JOURNAL_ASYNC_COMMIT)) {
-		ext4_msg(sb, KERN_ERR, "can't mount with journal_async_commit "
-			 "in data=ordered mode");
-		return 0;
-	}
 	return 1;
 }
 
@@ -2330,7 +2342,7 @@ static void ext4_orphan_cleanup(struct super_block *sb,
 				struct ext4_super_block *es)
 {
 	unsigned int s_flags = sb->s_flags;
-	int nr_orphans = 0, nr_truncates = 0;
+	int ret, nr_orphans = 0, nr_truncates = 0;
 #ifdef CONFIG_QUOTA
 	int i;
 #endif
@@ -2412,7 +2424,9 @@ static void ext4_orphan_cleanup(struct super_block *sb,
 				  inode->i_ino, inode->i_size);
 			inode_lock(inode);
 			truncate_inode_pages(inode->i_mapping, inode->i_size);
-			ext4_truncate(inode);
+			ret = ext4_truncate(inode);
+			if (ret)
+				ext4_std_error(inode->i_sb, ret);
 			inode_unlock(inode);
 			nr_truncates++;
 		} else {
@@ -3193,10 +3207,15 @@ static int count_overhead(struct super_block *sb, ext4_group_t grp,
 			ext4_set_bit(s++, buf);
 			count++;
 		}
-		for (j = ext4_bg_num_gdb(sb, grp); j > 0; j--) {
-			ext4_set_bit(EXT4_B2C(sbi, s++), buf);
-			count++;
+		j = ext4_bg_num_gdb(sb, grp);
+		if (s + j > EXT4_BLOCKS_PER_GROUP(sb)) {
+			ext4_error(sb, "Invalid number of block group "
+				   "descriptor blocks: %d", j);
+			j = EXT4_BLOCKS_PER_GROUP(sb) - s;
 		}
+		count += j;
+		for (; j > 0; j--)
+			ext4_set_bit(EXT4_B2C(sbi, s++), buf);
 	}
 	if (!count)
 		return 0;
@@ -3301,7 +3320,7 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent)
 	char *orig_data = kstrdup(data, GFP_KERNEL);
 	struct buffer_head *bh;
 	struct ext4_super_block *es = NULL;
-	struct ext4_sb_info *sbi;
+	struct ext4_sb_info *sbi = kzalloc(sizeof(*sbi), GFP_KERNEL);
 	ext4_fsblk_t block;
 	ext4_fsblk_t sb_block = get_sb_block(&data);
 	ext4_fsblk_t logical_sb_block;
@@ -3320,16 +3339,14 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent)
 	unsigned int journal_ioprio = DEFAULT_JOURNAL_IOPRIO;
 	ext4_group_t first_not_zeroed;
 
-	sbi = kzalloc(sizeof(*sbi), GFP_KERNEL);
-	if (!sbi)
-		goto out_free_orig;
+	if ((data && !orig_data) || !sbi)
+		goto out_free_base;
 
 	sbi->s_blockgroup_lock =
 		kzalloc(sizeof(struct blockgroup_lock), GFP_KERNEL);
-	if (!sbi->s_blockgroup_lock) {
-		kfree(sbi);
-		goto out_free_orig;
-	}
+	if (!sbi->s_blockgroup_lock)
+		goto out_free_base;
+
 	sb->s_fs_info = sbi;
 	sbi->s_sb = sb;
 	sbi->s_inode_readahead_blks = EXT4_DEF_INODE_READAHEAD_BLKS;
@@ -3475,11 +3492,19 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent)
 	 */
 	sbi->s_li_wait_mult = EXT4_DEF_LI_WAIT_MULT;
 
-	if (!parse_options((char *) sbi->s_es->s_mount_opts, sb,
-			   &journal_devnum, &journal_ioprio, 0)) {
-		ext4_msg(sb, KERN_WARNING,
-			 "failed to parse options in superblock: %s",
-			 sbi->s_es->s_mount_opts);
+	if (sbi->s_es->s_mount_opts[0]) {
+		char *s_mount_opts = kstrndup(sbi->s_es->s_mount_opts,
+					      sizeof(sbi->s_es->s_mount_opts),
+					      GFP_KERNEL);
+		if (!s_mount_opts)
+			goto failed_mount;
+		if (!parse_options(s_mount_opts, sb, &journal_devnum,
+				   &journal_ioprio, 0)) {
+			ext4_msg(sb, KERN_WARNING,
+				 "failed to parse options in superblock: %s",
+				 s_mount_opts);
+		}
+		kfree(s_mount_opts);
 	}
 	sbi->s_def_mount_opt = sbi->s_mount_opt;
 	if (!parse_options((char *) data, sb, &journal_devnum,
@@ -3505,6 +3530,11 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent)
 				 "both data=journal and dax");
 			goto failed_mount;
 		}
+		if (ext4_has_feature_encrypt(sb)) {
+			ext4_msg(sb, KERN_WARNING,
+				 "encrypted files will use data=ordered "
+				 "instead of data journaling mode");
+		}
 		if (test_opt(sb, DELALLOC))
 			clear_opt(sb, DELALLOC);
 	} else {
@@ -3660,12 +3690,16 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent)
 
 	sbi->s_blocks_per_group = le32_to_cpu(es->s_blocks_per_group);
 	sbi->s_inodes_per_group = le32_to_cpu(es->s_inodes_per_group);
-	if (EXT4_INODE_SIZE(sb) == 0 || EXT4_INODES_PER_GROUP(sb) == 0)
-		goto cantfind_ext4;
 
 	sbi->s_inodes_per_block = blocksize / EXT4_INODE_SIZE(sb);
 	if (sbi->s_inodes_per_block == 0)
 		goto cantfind_ext4;
+	if (sbi->s_inodes_per_group < sbi->s_inodes_per_block ||
+	    sbi->s_inodes_per_group > blocksize * 8) {
+		ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu\n",
+			 sbi->s_blocks_per_group);
+		goto failed_mount;
+	}
 	sbi->s_itb_per_group = sbi->s_inodes_per_group /
 					sbi->s_inodes_per_block;
 	sbi->s_desc_per_block = blocksize / EXT4_DESC_SIZE(sb);
@@ -3748,13 +3782,6 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent)
 	}
 	sbi->s_cluster_ratio = clustersize / blocksize;
 
-	if (sbi->s_inodes_per_group > blocksize * 8) {
-		ext4_msg(sb, KERN_ERR,
-		       "#inodes per group too big: %lu",
-		       sbi->s_inodes_per_group);
-		goto failed_mount;
-	}
-
 	/* Do we have standard group size of clustersize * 8 blocks ? */
 	if (sbi->s_blocks_per_group == clustersize << 3)
 		set_opt2(sb, STD_GROUP_SIZE);
@@ -3814,6 +3841,15 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent)
 			(EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb)));
 	db_count = (sbi->s_groups_count + EXT4_DESC_PER_BLOCK(sb) - 1) /
 		   EXT4_DESC_PER_BLOCK(sb);
+	if (ext4_has_feature_meta_bg(sb)) {
+		if (le32_to_cpu(es->s_first_meta_bg) >= db_count) {
+			ext4_msg(sb, KERN_WARNING,
+				 "first meta block group too large: %u "
+				 "(group descriptor block count %u)",
+				 le32_to_cpu(es->s_first_meta_bg), db_count);
+			goto failed_mount;
+		}
+	}
 	sbi->s_group_desc = ext4_kvmalloc(db_count *
 					  sizeof(struct buffer_head *),
 					  GFP_KERNEL);
@@ -3967,6 +4003,14 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent)
 	default:
 		break;
 	}
+
+	if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA &&
+	    test_opt(sb, JOURNAL_ASYNC_COMMIT)) {
+		ext4_msg(sb, KERN_ERR, "can't mount with "
+			"journal_async_commit in data=ordered mode");
+		goto failed_mount_wq;
+	}
+
 	set_task_ioprio(sbi->s_journal->j_task, journal_ioprio);
 
 	sbi->s_journal->j_commit_callback = ext4_journal_commit_callback;
@@ -4160,7 +4204,9 @@ no_journal:
 
 	if (___ratelimit(&ext4_mount_msg_ratelimit, "EXT4-fs mount"))
 		ext4_msg(sb, KERN_INFO, "mounted filesystem with%s. "
-			 "Opts: %s%s%s", descr, sbi->s_es->s_mount_opts,
+			 "Opts: %.*s%s%s", descr,
+			 (int) sizeof(sbi->s_es->s_mount_opts),
+			 sbi->s_es->s_mount_opts,
 			 *sbi->s_es->s_mount_opts ? "; " : "", orig_data);
 
 	if (es->s_error_count)
@@ -4239,8 +4285,8 @@ failed_mount:
 out_fail:
 	sb->s_fs_info = NULL;
 	kfree(sbi->s_blockgroup_lock);
+out_free_base:
 	kfree(sbi);
-out_free_orig:
 	kfree(orig_data);
 	return err ? err : ret;
 }
@@ -4550,7 +4596,8 @@ static int ext4_commit_super(struct super_block *sb, int sync)
 				&EXT4_SB(sb)->s_freeinodes_counter));
 	BUFFER_TRACE(sbh, "marking dirty");
 	ext4_superblock_csum_set(sb);
-	lock_buffer(sbh);
+	if (sync)
+		lock_buffer(sbh);
 	if (buffer_write_io_error(sbh)) {
 		/*
 		 * Oh, dear.  A previous attempt to write the
@@ -4566,10 +4613,10 @@ static int ext4_commit_super(struct super_block *sb, int sync)
 		set_buffer_uptodate(sbh);
 	}
 	mark_buffer_dirty(sbh);
-	unlock_buffer(sbh);
 	if (sync) {
+		unlock_buffer(sbh);
 		error = __sync_dirty_buffer(sbh,
-			test_opt(sb, BARRIER) ? WRITE_FUA : WRITE_SYNC);
+			test_opt(sb, BARRIER) ? REQ_FUA : REQ_SYNC);
 		if (error)
 			return error;
 
@@ -4857,6 +4904,13 @@ static int ext4_remount(struct super_block *sb, int *flags, char *data)
 			err = -EINVAL;
 			goto restore_opts;
 		}
+	} else if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA) {
+		if (test_opt(sb, JOURNAL_ASYNC_COMMIT)) {
+			ext4_msg(sb, KERN_ERR, "can't mount with "
+				"journal_async_commit in data=ordered mode");
+			err = -EINVAL;
+			goto restore_opts;
+		}
 	}
 
 	if ((sbi->s_mount_opt ^ old_opts.s_mount_opt) & EXT4_MOUNT_DAX) {
@@ -5366,7 +5420,7 @@ static int ext4_quota_off(struct super_block *sb, int type)
 	handle = ext4_journal_start(inode, EXT4_HT_QUOTA, 1);
 	if (IS_ERR(handle))
 		goto out;
-	inode->i_mtime = inode->i_ctime = CURRENT_TIME;
+	inode->i_mtime = inode->i_ctime = current_time(inode);
 	ext4_mark_inode_dirty(handle, inode);
 	ext4_journal_stop(handle);
 
diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c
index d77be9e9f535..5a94fa52b74f 100644
--- a/fs/ext4/xattr.c
+++ b/fs/ext4/xattr.c
@@ -185,6 +185,7 @@ ext4_xattr_check_names(struct ext4_xattr_entry *entry, void *end,
 {
 	struct ext4_xattr_entry *e = entry;
 
+	/* Find the end of the names list */
 	while (!IS_LAST_ENTRY(e)) {
 		struct ext4_xattr_entry *next = EXT4_XATTR_NEXT(e);
 		if ((void *)next >= end)
@@ -192,15 +193,29 @@ ext4_xattr_check_names(struct ext4_xattr_entry *entry, void *end,
 		e = next;
 	}
 
+	/* Check the values */
 	while (!IS_LAST_ENTRY(entry)) {
 		if (entry->e_value_block != 0)
 			return -EFSCORRUPTED;
-		if (entry->e_value_size != 0 &&
-		    (value_start + le16_to_cpu(entry->e_value_offs) <
-		     (void *)e + sizeof(__u32) ||
-		     value_start + le16_to_cpu(entry->e_value_offs) +
-		    le32_to_cpu(entry->e_value_size) > end))
-			return -EFSCORRUPTED;
+		if (entry->e_value_size != 0) {
+			u16 offs = le16_to_cpu(entry->e_value_offs);
+			u32 size = le32_to_cpu(entry->e_value_size);
+			void *value;
+
+			/*
+			 * The value cannot overlap the names, and the value
+			 * with padding cannot extend beyond 'end'.  Check both
+			 * the padded and unpadded sizes, since the size may
+			 * overflow to 0 when adding padding.
+			 */
+			if (offs > end - value_start)
+				return -EFSCORRUPTED;
+			value = value_start + offs;
+			if (value < (void *)e + sizeof(u32) ||
+			    size > end - value ||
+			    EXT4_XATTR_SIZE(size) > end - value)
+				return -EFSCORRUPTED;
+		}
 		entry = EXT4_XATTR_NEXT(entry);
 	}
 
@@ -231,13 +246,12 @@ static int
 __xattr_check_inode(struct inode *inode, struct ext4_xattr_ibody_header *header,
 			 void *end, const char *function, unsigned int line)
 {
-	struct ext4_xattr_entry *entry = IFIRST(header);
 	int error = -EFSCORRUPTED;
 
-	if (((void *) header >= end) ||
+	if (end - (void *)header < sizeof(*header) + sizeof(u32) ||
 	    (header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)))
 		goto errout;
-	error = ext4_xattr_check_names(entry, end, entry);
+	error = ext4_xattr_check_names(IFIRST(header), end, IFIRST(header));
 errout:
 	if (error)
 		__ext4_error_inode(inode, function, line, 0,
@@ -1109,7 +1123,7 @@ int ext4_xattr_ibody_inline_set(handle_t *handle, struct inode *inode,
 	return 0;
 }
 
-static int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode,
+static int ext4_xattr_ibody_set(struct inode *inode,
 				struct ext4_xattr_info *i,
 				struct ext4_xattr_ibody_find *is)
 {
@@ -1216,7 +1230,7 @@ ext4_xattr_set_handle(handle_t *handle, struct inode *inode, int name_index,
 	}
 	if (!value) {
 		if (!is.s.not_found)
-			error = ext4_xattr_ibody_set(handle, inode, &i, &is);
+			error = ext4_xattr_ibody_set(inode, &i, &is);
 		else if (!bs.s.not_found)
 			error = ext4_xattr_block_set(handle, inode, &i, &bs);
 	} else {
@@ -1227,7 +1241,7 @@ ext4_xattr_set_handle(handle_t *handle, struct inode *inode, int name_index,
 		if (!bs.s.not_found && ext4_xattr_value_same(&bs.s, &i))
 			goto cleanup;
 
-		error = ext4_xattr_ibody_set(handle, inode, &i, &is);
+		error = ext4_xattr_ibody_set(inode, &i, &is);
 		if (!error && !bs.s.not_found) {
 			i.value = NULL;
 			error = ext4_xattr_block_set(handle, inode, &i, &bs);
@@ -1242,14 +1256,13 @@ ext4_xattr_set_handle(handle_t *handle, struct inode *inode, int name_index,
 				goto cleanup;
 			if (!is.s.not_found) {
 				i.value = NULL;
-				error = ext4_xattr_ibody_set(handle, inode, &i,
-							     &is);
+				error = ext4_xattr_ibody_set(inode, &i, &is);
 			}
 		}
 	}
 	if (!error) {
 		ext4_xattr_update_super_block(handle, inode->i_sb);
-		inode->i_ctime = ext4_current_time(inode);
+		inode->i_ctime = current_time(inode);
 		if (!value)
 			ext4_clear_inode_state(inode, EXT4_STATE_NO_EXPAND);
 		error = ext4_mark_iloc_dirty(handle, inode, &is.iloc);
@@ -1384,7 +1397,7 @@ static int ext4_xattr_move_to_block(handle_t *handle, struct inode *inode,
 		goto out;
 
 	/* Remove the chosen entry from the inode */
-	error = ext4_xattr_ibody_set(handle, inode, &i, is);
+	error = ext4_xattr_ibody_set(inode, &i, is);
 	if (error)
 		goto out;
 
diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c
index 6fe23af509e1..8f487692c21f 100644
--- a/fs/f2fs/acl.c
+++ b/fs/f2fs/acl.c
@@ -384,7 +384,7 @@ int f2fs_init_acl(struct inode *inode, struct inode *dir, struct page *ipage,
 	if (error)
 		return error;
 
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, true);
 
 	if (default_acl) {
 		error = __f2fs_set_acl(inode, ACL_TYPE_DEFAULT, default_acl,
diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
index 7e9b504bd8b2..f73ee9534d83 100644
--- a/fs/f2fs/checkpoint.c
+++ b/fs/f2fs/checkpoint.c
@@ -65,7 +65,7 @@ static struct page *__get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index,
 		.sbi = sbi,
 		.type = META,
 		.op = REQ_OP_READ,
-		.op_flags = READ_SYNC | REQ_META | REQ_PRIO,
+		.op_flags = REQ_META | REQ_PRIO,
 		.old_blkaddr = index,
 		.new_blkaddr = index,
 		.encrypted_page = NULL,
@@ -160,7 +160,7 @@ int ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
 		.sbi = sbi,
 		.type = META,
 		.op = REQ_OP_READ,
-		.op_flags = sync ? (READ_SYNC | REQ_META | REQ_PRIO) : REQ_RAHEAD,
+		.op_flags = sync ? (REQ_META | REQ_PRIO) : REQ_RAHEAD,
 		.encrypted_page = NULL,
 	};
 	struct blk_plug plug;
@@ -228,7 +228,7 @@ void ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index)
 	f2fs_put_page(page, 0);
 
 	if (readahead)
-		ra_meta_pages(sbi, index, MAX_BIO_BLOCKS(sbi), META_POR, true);
+		ra_meta_pages(sbi, index, BIO_MAX_PAGES, META_POR, true);
 }
 
 static int f2fs_write_meta_page(struct page *page,
@@ -770,7 +770,12 @@ int get_valid_checkpoint(struct f2fs_sb_info *sbi)
 
 	/* Sanity checking of checkpoint */
 	if (sanity_check_ckpt(sbi))
-		goto fail_no_cp;
+		goto free_fail_no_cp;
+
+	if (cur_page == cp1)
+		sbi->cur_cp_pack = 1;
+	else
+		sbi->cur_cp_pack = 2;
 
 	if (cp_blks <= 1)
 		goto done;
@@ -793,6 +798,9 @@ done:
 	f2fs_put_page(cp2, 1);
 	return 0;
 
+free_fail_no_cp:
+	f2fs_put_page(cp1, 1);
+	f2fs_put_page(cp2, 1);
 fail_no_cp:
 	kfree(sbi->ckpt);
 	return -EINVAL;
@@ -921,7 +929,11 @@ int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi)
 		inode = igrab(&fi->vfs_inode);
 		spin_unlock(&sbi->inode_lock[DIRTY_META]);
 		if (inode) {
-			update_inode_page(inode);
+			sync_inode_metadata(inode, 0);
+
+			/* it's on eviction */
+			if (is_inode_flag_set(inode, FI_DIRTY_INODE))
+				update_inode_page(inode);
 			iput(inode);
 		}
 	};
@@ -987,7 +999,7 @@ static void unblock_operations(struct f2fs_sb_info *sbi)
 {
 	up_write(&sbi->node_write);
 
-	build_free_nids(sbi);
+	build_free_nids(sbi, false);
 	f2fs_unlock_all(sbi);
 }
 
@@ -998,7 +1010,7 @@ static void wait_on_all_pages_writeback(struct f2fs_sb_info *sbi)
 	for (;;) {
 		prepare_to_wait(&sbi->cp_wait, &wait, TASK_UNINTERRUPTIBLE);
 
-		if (!atomic_read(&sbi->nr_wb_bios))
+		if (!get_pages(sbi, F2FS_WB_CP_DATA))
 			break;
 
 		io_schedule_timeout(5*HZ);
@@ -1123,7 +1135,7 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 				le32_to_cpu(ckpt->checksum_offset)))
 				= cpu_to_le32(crc32);
 
-	start_blk = __start_cp_addr(sbi);
+	start_blk = __start_cp_next_addr(sbi);
 
 	/* need to wait for end_io results */
 	wait_on_all_pages_writeback(sbi);
@@ -1184,9 +1196,9 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 	if (unlikely(f2fs_cp_error(sbi)))
 		return -EIO;
 
-	clear_prefree_segments(sbi, cpc);
 	clear_sbi_flag(sbi, SBI_IS_DIRTY);
 	clear_sbi_flag(sbi, SBI_NEED_CP);
+	__set_cp_next_pack(sbi);
 
 	/*
 	 * redirty superblock if metadata like node page or inode cache is
@@ -1261,8 +1273,12 @@ int write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 
 	/* unlock all the fs_lock[] in do_checkpoint() */
 	err = do_checkpoint(sbi, cpc);
-
-	f2fs_wait_all_discard_bio(sbi);
+	if (err) {
+		release_discard_addrs(sbi);
+	} else {
+		clear_prefree_segments(sbi, cpc);
+		f2fs_wait_all_discard_bio(sbi);
+	}
 
 	unblock_operations(sbi);
 	stat_inc_cp_count(sbi->stat_info);
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index 9ae194fd2fdb..9ac262564fa6 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -29,6 +29,26 @@
 #include "trace.h"
 #include <trace/events/f2fs.h>
 
+static bool __is_cp_guaranteed(struct page *page)
+{
+	struct address_space *mapping = page->mapping;
+	struct inode *inode;
+	struct f2fs_sb_info *sbi;
+
+	if (!mapping)
+		return false;
+
+	inode = mapping->host;
+	sbi = F2FS_I_SB(inode);
+
+	if (inode->i_ino == F2FS_META_INO(sbi) ||
+			inode->i_ino ==  F2FS_NODE_INO(sbi) ||
+			S_ISDIR(inode->i_mode) ||
+			is_cold_data(page))
+		return true;
+	return false;
+}
+
 static void f2fs_read_end_io(struct bio *bio)
 {
 	struct bio_vec *bvec;
@@ -71,6 +91,7 @@ static void f2fs_write_end_io(struct bio *bio)
 
 	bio_for_each_segment_all(bvec, bio, i) {
 		struct page *page = bvec->bv_page;
+		enum count_type type = WB_DATA_TYPE(page);
 
 		fscrypt_pullback_bio_page(&page, true);
 
@@ -78,9 +99,11 @@ static void f2fs_write_end_io(struct bio *bio)
 			mapping_set_error(page->mapping, -EIO);
 			f2fs_stop_checkpoint(sbi, true);
 		}
+		dec_page_count(sbi, type);
+		clear_cold_data(page);
 		end_page_writeback(page);
 	}
-	if (atomic_dec_and_test(&sbi->nr_wb_bios) &&
+	if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
 				wq_has_sleeper(&sbi->cp_wait))
 		wake_up(&sbi->cp_wait);
 
@@ -88,6 +111,46 @@ static void f2fs_write_end_io(struct bio *bio)
 }
 
 /*
+ * Return true, if pre_bio's bdev is same as its target device.
+ */
+struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
+				block_t blk_addr, struct bio *bio)
+{
+	struct block_device *bdev = sbi->sb->s_bdev;
+	int i;
+
+	for (i = 0; i < sbi->s_ndevs; i++) {
+		if (FDEV(i).start_blk <= blk_addr &&
+					FDEV(i).end_blk >= blk_addr) {
+			blk_addr -= FDEV(i).start_blk;
+			bdev = FDEV(i).bdev;
+			break;
+		}
+	}
+	if (bio) {
+		bio->bi_bdev = bdev;
+		bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
+	}
+	return bdev;
+}
+
+int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
+{
+	int i;
+
+	for (i = 0; i < sbi->s_ndevs; i++)
+		if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
+			return i;
+	return 0;
+}
+
+static bool __same_bdev(struct f2fs_sb_info *sbi,
+				block_t blk_addr, struct bio *bio)
+{
+	return f2fs_target_device(sbi, blk_addr, NULL) == bio->bi_bdev;
+}
+
+/*
  * Low-level block read/write IO operations.
  */
 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
@@ -97,8 +160,7 @@ static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
 
 	bio = f2fs_bio_alloc(npages);
 
-	bio->bi_bdev = sbi->sb->s_bdev;
-	bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
+	f2fs_target_device(sbi, blk_addr, bio);
 	bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io;
 	bio->bi_private = is_read ? NULL : sbi;
 
@@ -109,8 +171,7 @@ static inline void __submit_bio(struct f2fs_sb_info *sbi,
 				struct bio *bio, enum page_type type)
 {
 	if (!is_read_io(bio_op(bio))) {
-		atomic_inc(&sbi->nr_wb_bios);
-		if (f2fs_sb_mounted_hmsmr(sbi->sb) &&
+		if (f2fs_sb_mounted_blkzoned(sbi->sb) &&
 			current->plug && (type == DATA || type == NODE))
 			blk_finish_plug(current->plug);
 	}
@@ -198,11 +259,9 @@ static void __f2fs_submit_merged_bio(struct f2fs_sb_info *sbi,
 	if (type >= META_FLUSH) {
 		io->fio.type = META_FLUSH;
 		io->fio.op = REQ_OP_WRITE;
-		if (test_opt(sbi, NOBARRIER))
-			io->fio.op_flags = WRITE_FLUSH | REQ_META | REQ_PRIO;
-		else
-			io->fio.op_flags = WRITE_FLUSH_FUA | REQ_META |
-								REQ_PRIO;
+		io->fio.op_flags = REQ_PREFLUSH | REQ_META | REQ_PRIO;
+		if (!test_opt(sbi, NOBARRIER))
+			io->fio.op_flags |= REQ_FUA;
 	}
 	__submit_merged_bio(io);
 out:
@@ -270,22 +329,24 @@ void f2fs_submit_page_mbio(struct f2fs_io_info *fio)
 		verify_block_addr(sbi, fio->old_blkaddr);
 	verify_block_addr(sbi, fio->new_blkaddr);
 
+	bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
+
+	if (!is_read)
+		inc_page_count(sbi, WB_DATA_TYPE(bio_page));
+
 	down_write(&io->io_rwsem);
 
 	if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
-	    (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags)))
+	    (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
+			!__same_bdev(sbi, fio->new_blkaddr, io->bio)))
 		__submit_merged_bio(io);
 alloc_new:
 	if (io->bio == NULL) {
-		int bio_blocks = MAX_BIO_BLOCKS(sbi);
-
 		io->bio = __bio_alloc(sbi, fio->new_blkaddr,
-						bio_blocks, is_read);
+						BIO_MAX_PAGES, is_read);
 		io->fio = *fio;
 	}
 
-	bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
-
 	if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) <
 							PAGE_SIZE) {
 		__submit_merged_bio(io);
@@ -483,7 +544,7 @@ struct page *find_data_page(struct inode *inode, pgoff_t index)
 		return page;
 	f2fs_put_page(page, 0);
 
-	page = get_read_data_page(inode, index, READ_SYNC, false);
+	page = get_read_data_page(inode, index, 0, false);
 	if (IS_ERR(page))
 		return page;
 
@@ -509,7 +570,7 @@ struct page *get_lock_data_page(struct inode *inode, pgoff_t index,
 	struct address_space *mapping = inode->i_mapping;
 	struct page *page;
 repeat:
-	page = get_read_data_page(inode, index, READ_SYNC, for_write);
+	page = get_read_data_page(inode, index, 0, for_write);
 	if (IS_ERR(page))
 		return page;
 
@@ -590,7 +651,6 @@ static int __allocate_data_block(struct dnode_of_data *dn)
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
 	struct f2fs_summary sum;
 	struct node_info ni;
-	int seg = CURSEG_WARM_DATA;
 	pgoff_t fofs;
 	blkcnt_t count = 1;
 
@@ -608,11 +668,8 @@ alloc:
 	get_node_info(sbi, dn->nid, &ni);
 	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
 
-	if (dn->ofs_in_node == 0 && dn->inode_page == dn->node_page)
-		seg = CURSEG_DIRECT_IO;
-
 	allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
-								&sum, seg);
+						&sum, CURSEG_WARM_DATA);
 	set_data_blkaddr(dn);
 
 	/* update i_size */
@@ -624,11 +681,18 @@ alloc:
 	return 0;
 }
 
-ssize_t f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
+static inline bool __force_buffered_io(struct inode *inode, int rw)
+{
+	return ((f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) ||
+			(rw == WRITE && test_opt(F2FS_I_SB(inode), LFS)) ||
+			F2FS_I_SB(inode)->s_ndevs);
+}
+
+int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
 {
 	struct inode *inode = file_inode(iocb->ki_filp);
 	struct f2fs_map_blocks map;
-	ssize_t ret = 0;
+	int err = 0;
 
 	map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
 	map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
@@ -640,19 +704,22 @@ ssize_t f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
 	map.m_next_pgofs = NULL;
 
 	if (iocb->ki_flags & IOCB_DIRECT) {
-		ret = f2fs_convert_inline_inode(inode);
-		if (ret)
-			return ret;
-		return f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_DIO);
+		err = f2fs_convert_inline_inode(inode);
+		if (err)
+			return err;
+		return f2fs_map_blocks(inode, &map, 1,
+			__force_buffered_io(inode, WRITE) ?
+				F2FS_GET_BLOCK_PRE_AIO :
+				F2FS_GET_BLOCK_PRE_DIO);
 	}
 	if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA) {
-		ret = f2fs_convert_inline_inode(inode);
-		if (ret)
-			return ret;
+		err = f2fs_convert_inline_inode(inode);
+		if (err)
+			return err;
 	}
 	if (!f2fs_has_inline_data(inode))
 		return f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO);
-	return ret;
+	return err;
 }
 
 /*
@@ -676,7 +743,6 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
 	unsigned int ofs_in_node, last_ofs_in_node;
 	blkcnt_t prealloc;
 	struct extent_info ei;
-	bool allocated = false;
 	block_t blkaddr;
 
 	if (!maxblocks)
@@ -716,7 +782,7 @@ next_dnode:
 	}
 
 	prealloc = 0;
-	ofs_in_node = dn.ofs_in_node;
+	last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
 	end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
 
 next_block:
@@ -735,10 +801,8 @@ next_block:
 				}
 			} else {
 				err = __allocate_data_block(&dn);
-				if (!err) {
+				if (!err)
 					set_inode_flag(inode, FI_APPEND_WRITE);
-					allocated = true;
-				}
 			}
 			if (err)
 				goto sync_out;
@@ -793,7 +857,6 @@ skip:
 		err = reserve_new_blocks(&dn, prealloc);
 		if (err)
 			goto sync_out;
-		allocated = dn.node_changed;
 
 		map->m_len += dn.ofs_in_node - ofs_in_node;
 		if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
@@ -812,9 +875,8 @@ skip:
 
 	if (create) {
 		f2fs_unlock_op(sbi);
-		f2fs_balance_fs(sbi, allocated);
+		f2fs_balance_fs(sbi, dn.node_changed);
 	}
-	allocated = false;
 	goto next_dnode;
 
 sync_out:
@@ -822,7 +884,7 @@ sync_out:
 unlock_out:
 	if (create) {
 		f2fs_unlock_op(sbi);
-		f2fs_balance_fs(sbi, allocated);
+		f2fs_balance_fs(sbi, dn.node_changed);
 	}
 out:
 	trace_f2fs_map_blocks(inode, map, err);
@@ -834,19 +896,19 @@ static int __get_data_block(struct inode *inode, sector_t iblock,
 			pgoff_t *next_pgofs)
 {
 	struct f2fs_map_blocks map;
-	int ret;
+	int err;
 
 	map.m_lblk = iblock;
 	map.m_len = bh->b_size >> inode->i_blkbits;
 	map.m_next_pgofs = next_pgofs;
 
-	ret = f2fs_map_blocks(inode, &map, create, flag);
-	if (!ret) {
+	err = f2fs_map_blocks(inode, &map, create, flag);
+	if (!err) {
 		map_bh(bh, inode->i_sb, map.m_pblk);
 		bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
 		bh->b_size = map.m_len << inode->i_blkbits;
 	}
-	return ret;
+	return err;
 }
 
 static int get_data_block(struct inode *inode, sector_t iblock,
@@ -891,7 +953,6 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 	struct buffer_head map_bh;
 	sector_t start_blk, last_blk;
 	pgoff_t next_pgofs;
-	loff_t isize;
 	u64 logical = 0, phys = 0, size = 0;
 	u32 flags = 0;
 	int ret = 0;
@@ -908,13 +969,6 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 
 	inode_lock(inode);
 
-	isize = i_size_read(inode);
-	if (start >= isize)
-		goto out;
-
-	if (start + len > isize)
-		len = isize - start;
-
 	if (logical_to_blk(inode, len) == 0)
 		len = blk_to_logical(inode, 1);
 
@@ -933,13 +987,11 @@ next:
 	/* HOLE */
 	if (!buffer_mapped(&map_bh)) {
 		start_blk = next_pgofs;
-		/* Go through holes util pass the EOF */
-		if (blk_to_logical(inode, start_blk) < isize)
+
+		if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
+					F2FS_I_SB(inode)->max_file_blocks))
 			goto prep_next;
-		/* Found a hole beyond isize means no more extents.
-		 * Note that the premise is that filesystems don't
-		 * punch holes beyond isize and keep size unchanged.
-		 */
+
 		flags |= FIEMAP_EXTENT_LAST;
 	}
 
@@ -982,7 +1034,6 @@ static struct bio *f2fs_grab_bio(struct inode *inode, block_t blkaddr,
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct fscrypt_ctx *ctx = NULL;
-	struct block_device *bdev = sbi->sb->s_bdev;
 	struct bio *bio;
 
 	if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) {
@@ -1000,8 +1051,7 @@ static struct bio *f2fs_grab_bio(struct inode *inode, block_t blkaddr,
 			fscrypt_release_ctx(ctx);
 		return ERR_PTR(-ENOMEM);
 	}
-	bio->bi_bdev = bdev;
-	bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blkaddr);
+	f2fs_target_device(sbi, blkaddr, bio);
 	bio->bi_end_io = f2fs_read_end_io;
 	bio->bi_private = ctx;
 
@@ -1096,7 +1146,8 @@ got_it:
 		 * This page will go to BIO.  Do we need to send this
 		 * BIO off first?
 		 */
-		if (bio && (last_block_in_bio != block_nr - 1)) {
+		if (bio && (last_block_in_bio != block_nr - 1 ||
+			!__same_bdev(F2FS_I_SB(inode), block_nr, bio))) {
 submit_and_realloc:
 			__submit_bio(F2FS_I_SB(inode), bio, DATA);
 			bio = NULL;
@@ -1195,7 +1246,9 @@ int do_write_data_page(struct f2fs_io_info *fio)
 							fio->old_blkaddr);
 retry_encrypt:
 		fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
-								gfp_flags);
+							PAGE_SIZE, 0,
+							fio->page->index,
+							gfp_flags);
 		if (IS_ERR(fio->encrypted_page)) {
 			err = PTR_ERR(fio->encrypted_page);
 			if (err == -ENOMEM) {
@@ -1251,7 +1304,7 @@ static int f2fs_write_data_page(struct page *page,
 		.sbi = sbi,
 		.type = DATA,
 		.op = REQ_OP_WRITE,
-		.op_flags = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : 0,
+		.op_flags = wbc_to_write_flags(wbc),
 		.page = page,
 		.encrypted_page = NULL,
 	};
@@ -1311,7 +1364,6 @@ done:
 	if (err && err != -ENOENT)
 		goto redirty_out;
 
-	clear_cold_data(page);
 out:
 	inode_dec_dirty_pages(inode);
 	if (err)
@@ -1332,6 +1384,8 @@ out:
 
 redirty_out:
 	redirty_page_for_writepage(wbc, page);
+	if (!err)
+		return AOP_WRITEPAGE_ACTIVATE;
 	unlock_page(page);
 	return err;
 }
@@ -1427,6 +1481,15 @@ continue_unlock:
 
 			ret = mapping->a_ops->writepage(page, wbc);
 			if (unlikely(ret)) {
+				/*
+				 * keep nr_to_write, since vfs uses this to
+				 * get # of written pages.
+				 */
+				if (ret == AOP_WRITEPAGE_ACTIVATE) {
+					unlock_page(page);
+					ret = 0;
+					continue;
+				}
 				done_index = page->index + 1;
 				done = 1;
 				break;
@@ -1663,7 +1726,7 @@ repeat:
 			err = PTR_ERR(bio);
 			goto fail;
 		}
-		bio_set_op_attrs(bio, REQ_OP_READ, READ_SYNC);
+		bio->bi_opf = REQ_OP_READ;
 		if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
 			bio_put(bio);
 			err = -EFAULT;
@@ -1714,7 +1777,6 @@ static int f2fs_write_end(struct file *file,
 		goto unlock_out;
 
 	set_page_dirty(page);
-	clear_cold_data(page);
 
 	if (pos + copied > i_size_read(inode))
 		f2fs_i_size_write(inode, pos + copied);
@@ -1751,9 +1813,7 @@ static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
 	if (err)
 		return err;
 
-	if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
-		return 0;
-	if (test_opt(F2FS_I_SB(inode), LFS))
+	if (__force_buffered_io(inode, rw))
 		return 0;
 
 	trace_f2fs_direct_IO_enter(inode, offset, count, rw);
@@ -1785,12 +1845,14 @@ void f2fs_invalidate_page(struct page *page, unsigned int offset,
 		return;
 
 	if (PageDirty(page)) {
-		if (inode->i_ino == F2FS_META_INO(sbi))
+		if (inode->i_ino == F2FS_META_INO(sbi)) {
 			dec_page_count(sbi, F2FS_DIRTY_META);
-		else if (inode->i_ino == F2FS_NODE_INO(sbi))
+		} else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
 			dec_page_count(sbi, F2FS_DIRTY_NODES);
-		else
+		} else {
 			inode_dec_dirty_pages(inode);
+			remove_dirty_inode(inode);
+		}
 	}
 
 	/* This is atomic written page, keep Private */
diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c
index fb245bd302e4..fbd5184140d0 100644
--- a/fs/f2fs/debug.c
+++ b/fs/f2fs/debug.c
@@ -50,7 +50,8 @@ static void update_general_status(struct f2fs_sb_info *sbi)
 	si->ndirty_files = sbi->ndirty_inode[FILE_INODE];
 	si->ndirty_all = sbi->ndirty_inode[DIRTY_META];
 	si->inmem_pages = get_pages(sbi, F2FS_INMEM_PAGES);
-	si->wb_bios = atomic_read(&sbi->nr_wb_bios);
+	si->nr_wb_cp_data = get_pages(sbi, F2FS_WB_CP_DATA);
+	si->nr_wb_data = get_pages(sbi, F2FS_WB_DATA);
 	si->total_count = (int)sbi->user_block_count / sbi->blocks_per_seg;
 	si->rsvd_segs = reserved_segments(sbi);
 	si->overp_segs = overprovision_segments(sbi);
@@ -74,7 +75,8 @@ static void update_general_status(struct f2fs_sb_info *sbi)
 	si->dirty_nats = NM_I(sbi)->dirty_nat_cnt;
 	si->sits = MAIN_SEGS(sbi);
 	si->dirty_sits = SIT_I(sbi)->dirty_sentries;
-	si->fnids = NM_I(sbi)->fcnt;
+	si->free_nids = NM_I(sbi)->nid_cnt[FREE_NID_LIST];
+	si->alloc_nids = NM_I(sbi)->nid_cnt[ALLOC_NID_LIST];
 	si->bg_gc = sbi->bg_gc;
 	si->util_free = (int)(free_user_blocks(sbi) >> sbi->log_blocks_per_seg)
 		* 100 / (int)(sbi->user_block_count >> sbi->log_blocks_per_seg)
@@ -194,7 +196,9 @@ get_cache:
 		si->cache_mem += sizeof(struct flush_cmd_control);
 
 	/* free nids */
-	si->cache_mem += NM_I(sbi)->fcnt * sizeof(struct free_nid);
+	si->cache_mem += (NM_I(sbi)->nid_cnt[FREE_NID_LIST] +
+				NM_I(sbi)->nid_cnt[ALLOC_NID_LIST]) *
+				sizeof(struct free_nid);
 	si->cache_mem += NM_I(sbi)->nat_cnt * sizeof(struct nat_entry);
 	si->cache_mem += NM_I(sbi)->dirty_nat_cnt *
 					sizeof(struct nat_entry_set);
@@ -310,22 +314,22 @@ static int stat_show(struct seq_file *s, void *v)
 		seq_printf(s, "  - Inner Struct Count: tree: %d(%d), node: %d\n",
 				si->ext_tree, si->zombie_tree, si->ext_node);
 		seq_puts(s, "\nBalancing F2FS Async:\n");
-		seq_printf(s, "  - inmem: %4lld, wb_bios: %4d\n",
-			   si->inmem_pages, si->wb_bios);
-		seq_printf(s, "  - nodes: %4lld in %4d\n",
+		seq_printf(s, "  - inmem: %4d, wb_cp_data: %4d, wb_data: %4d\n",
+			   si->inmem_pages, si->nr_wb_cp_data, si->nr_wb_data);
+		seq_printf(s, "  - nodes: %4d in %4d\n",
 			   si->ndirty_node, si->node_pages);
-		seq_printf(s, "  - dents: %4lld in dirs:%4d (%4d)\n",
+		seq_printf(s, "  - dents: %4d in dirs:%4d (%4d)\n",
 			   si->ndirty_dent, si->ndirty_dirs, si->ndirty_all);
-		seq_printf(s, "  - datas: %4lld in files:%4d\n",
+		seq_printf(s, "  - datas: %4d in files:%4d\n",
 			   si->ndirty_data, si->ndirty_files);
-		seq_printf(s, "  - meta: %4lld in %4d\n",
+		seq_printf(s, "  - meta: %4d in %4d\n",
 			   si->ndirty_meta, si->meta_pages);
-		seq_printf(s, "  - imeta: %4lld\n",
+		seq_printf(s, "  - imeta: %4d\n",
 			   si->ndirty_imeta);
 		seq_printf(s, "  - NATs: %9d/%9d\n  - SITs: %9d/%9d\n",
 			   si->dirty_nats, si->nats, si->dirty_sits, si->sits);
-		seq_printf(s, "  - free_nids: %9d\n",
-			   si->fnids);
+		seq_printf(s, "  - free_nids: %9d, alloc_nids: %9d\n",
+			   si->free_nids, si->alloc_nids);
 		seq_puts(s, "\nDistribution of User Blocks:");
 		seq_puts(s, " [ valid | invalid | free ]\n");
 		seq_puts(s, "  [");
@@ -373,6 +377,7 @@ static int stat_open(struct inode *inode, struct file *file)
 }
 
 static const struct file_operations stat_fops = {
+	.owner = THIS_MODULE,
 	.open = stat_open,
 	.read = seq_read,
 	.llseek = seq_lseek,
diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c
index 369f4513be37..827c5daef4fc 100644
--- a/fs/f2fs/dir.c
+++ b/fs/f2fs/dir.c
@@ -136,7 +136,7 @@ struct f2fs_dir_entry *find_target_dentry(struct fscrypt_name *fname,
 
 		/* show encrypted name */
 		if (fname->hash) {
-			if (de->hash_code == fname->hash)
+			if (de->hash_code == cpu_to_le32(fname->hash))
 				goto found;
 		} else if (de_name.len == name->len &&
 			de->hash_code == namehash &&
@@ -313,7 +313,7 @@ void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de,
 	set_page_dirty(page);
 
 	dir->i_mtime = dir->i_ctime = current_time(dir);
-	f2fs_mark_inode_dirty_sync(dir);
+	f2fs_mark_inode_dirty_sync(dir, false);
 	f2fs_put_page(page, 1);
 }
 
@@ -466,7 +466,7 @@ void update_parent_metadata(struct inode *dir, struct inode *inode,
 		clear_inode_flag(inode, FI_NEW_INODE);
 	}
 	dir->i_mtime = dir->i_ctime = current_time(dir);
-	f2fs_mark_inode_dirty_sync(dir);
+	f2fs_mark_inode_dirty_sync(dir, false);
 
 	if (F2FS_I(dir)->i_current_depth != current_depth)
 		f2fs_i_depth_write(dir, current_depth);
@@ -731,7 +731,7 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
 	set_page_dirty(page);
 
 	dir->i_ctime = dir->i_mtime = current_time(dir);
-	f2fs_mark_inode_dirty_sync(dir);
+	f2fs_mark_inode_dirty_sync(dir, false);
 
 	if (inode)
 		f2fs_drop_nlink(dir, inode);
@@ -742,6 +742,7 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
 		ClearPagePrivate(page);
 		ClearPageUptodate(page);
 		inode_dec_dirty_pages(dir);
+		remove_dirty_inode(dir);
 	}
 	f2fs_put_page(page, 1);
 }
@@ -784,7 +785,7 @@ bool f2fs_empty_dir(struct inode *dir)
 	return true;
 }
 
-bool f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
+int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
 			unsigned int start_pos, struct fscrypt_str *fstr)
 {
 	unsigned char d_type = DT_UNKNOWN;
@@ -819,7 +820,7 @@ bool f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
 						(u32)de->hash_code, 0,
 						&de_name, fstr);
 			if (err)
-				return true;
+				return err;
 
 			de_name = *fstr;
 			fstr->len = save_len;
@@ -827,12 +828,12 @@ bool f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
 
 		if (!dir_emit(ctx, de_name.name, de_name.len,
 					le32_to_cpu(de->ino), d_type))
-			return true;
+			return 1;
 
 		bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len));
 		ctx->pos = start_pos + bit_pos;
 	}
-	return false;
+	return 0;
 }
 
 static int f2fs_readdir(struct file *file, struct dir_context *ctx)
@@ -871,17 +872,21 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx)
 		dentry_page = get_lock_data_page(inode, n, false);
 		if (IS_ERR(dentry_page)) {
 			err = PTR_ERR(dentry_page);
-			if (err == -ENOENT)
+			if (err == -ENOENT) {
+				err = 0;
 				continue;
-			else
+			} else {
 				goto out;
+			}
 		}
 
 		dentry_blk = kmap(dentry_page);
 
 		make_dentry_ptr(inode, &d, (void *)dentry_blk, 1);
 
-		if (f2fs_fill_dentries(ctx, &d, n * NR_DENTRY_IN_BLOCK, &fstr)) {
+		err = f2fs_fill_dentries(ctx, &d,
+				n * NR_DENTRY_IN_BLOCK, &fstr);
+		if (err) {
 			kunmap(dentry_page);
 			f2fs_put_page(dentry_page, 1);
 			break;
@@ -891,10 +896,9 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx)
 		kunmap(dentry_page);
 		f2fs_put_page(dentry_page, 1);
 	}
-	err = 0;
 out:
 	fscrypt_fname_free_buffer(&fstr);
-	return err;
+	return err < 0 ? err : 0;
 }
 
 static int f2fs_dir_open(struct inode *inode, struct file *filp)
diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c
index 2b06d4fcd954..4db44da7ef69 100644
--- a/fs/f2fs/extent_cache.c
+++ b/fs/f2fs/extent_cache.c
@@ -172,7 +172,7 @@ static void __drop_largest_extent(struct inode *inode,
 
 	if (fofs < largest->fofs + largest->len && fofs + len > largest->fofs) {
 		largest->len = 0;
-		f2fs_mark_inode_dirty_sync(inode);
+		f2fs_mark_inode_dirty_sync(inode, true);
 	}
 }
 
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 9e8de18a168a..2da8c3aa0ce5 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -103,7 +103,7 @@ struct f2fs_mount_info {
 };
 
 #define F2FS_FEATURE_ENCRYPT	0x0001
-#define F2FS_FEATURE_HMSMR	0x0002
+#define F2FS_FEATURE_BLKZONED	0x0002
 
 #define F2FS_HAS_FEATURE(sb, mask)					\
 	((F2FS_SB(sb)->raw_super->feature & cpu_to_le32(mask)) != 0)
@@ -401,6 +401,7 @@ struct f2fs_map_blocks {
 #define FADVISE_LOST_PINO_BIT	0x02
 #define FADVISE_ENCRYPT_BIT	0x04
 #define FADVISE_ENC_NAME_BIT	0x08
+#define FADVISE_KEEP_SIZE_BIT	0x10
 
 #define file_is_cold(inode)	is_file(inode, FADVISE_COLD_BIT)
 #define file_wrong_pino(inode)	is_file(inode, FADVISE_LOST_PINO_BIT)
@@ -413,6 +414,8 @@ struct f2fs_map_blocks {
 #define file_clear_encrypt(inode) clear_file(inode, FADVISE_ENCRYPT_BIT)
 #define file_enc_name(inode)	is_file(inode, FADVISE_ENC_NAME_BIT)
 #define file_set_enc_name(inode) set_file(inode, FADVISE_ENC_NAME_BIT)
+#define file_keep_isize(inode)	is_file(inode, FADVISE_KEEP_SIZE_BIT)
+#define file_set_keep_isize(inode) set_file(inode, FADVISE_KEEP_SIZE_BIT)
 
 #define DEF_DIR_LEVEL		0
 
@@ -428,7 +431,7 @@ struct f2fs_inode_info {
 	/* Use below internally in f2fs*/
 	unsigned long flags;		/* use to pass per-file flags */
 	struct rw_semaphore i_sem;	/* protect fi info */
-	struct percpu_counter dirty_pages;	/* # of dirty pages */
+	atomic_t dirty_pages;		/* # of dirty pages */
 	f2fs_hash_t chash;		/* hash value of given file name */
 	unsigned int clevel;		/* maximum level of given file name */
 	nid_t i_xattr_nid;		/* node id that contains xattrs */
@@ -493,20 +496,26 @@ static inline bool __is_front_mergeable(struct extent_info *cur,
 	return __is_extent_mergeable(cur, front);
 }
 
-extern void f2fs_mark_inode_dirty_sync(struct inode *);
+extern void f2fs_mark_inode_dirty_sync(struct inode *, bool);
 static inline void __try_update_largest_extent(struct inode *inode,
 			struct extent_tree *et, struct extent_node *en)
 {
 	if (en->ei.len > et->largest.len) {
 		et->largest = en->ei;
-		f2fs_mark_inode_dirty_sync(inode);
+		f2fs_mark_inode_dirty_sync(inode, true);
 	}
 }
 
+enum nid_list {
+	FREE_NID_LIST,
+	ALLOC_NID_LIST,
+	MAX_NID_LIST,
+};
+
 struct f2fs_nm_info {
 	block_t nat_blkaddr;		/* base disk address of NAT */
 	nid_t max_nid;			/* maximum possible node ids */
-	nid_t available_nids;		/* maximum available node ids */
+	nid_t available_nids;		/* # of available node ids */
 	nid_t next_scan_nid;		/* the next nid to be scanned */
 	unsigned int ram_thresh;	/* control the memory footprint */
 	unsigned int ra_nid_pages;	/* # of nid pages to be readaheaded */
@@ -522,9 +531,9 @@ struct f2fs_nm_info {
 
 	/* free node ids management */
 	struct radix_tree_root free_nid_root;/* root of the free_nid cache */
-	struct list_head free_nid_list;	/* a list for free nids */
-	spinlock_t free_nid_list_lock;	/* protect free nid list */
-	unsigned int fcnt;		/* the number of free node id */
+	struct list_head nid_list[MAX_NID_LIST];/* lists for free nids */
+	unsigned int nid_cnt[MAX_NID_LIST];	/* the number of free node id */
+	spinlock_t nid_list_lock;	/* protect nid lists ops */
 	struct mutex build_lock;	/* lock for build free nids */
 
 	/* for checkpoint */
@@ -585,7 +594,6 @@ enum {
 	CURSEG_WARM_NODE,	/* direct node blocks of normal files */
 	CURSEG_COLD_NODE,	/* indirect node blocks */
 	NO_CHECK_TYPE,
-	CURSEG_DIRECT_IO,	/* to use for the direct IO path */
 };
 
 struct flush_cmd {
@@ -649,6 +657,7 @@ struct f2fs_sm_info {
  * f2fs monitors the number of several block types such as on-writeback,
  * dirty dentry blocks, dirty node blocks, and dirty meta blocks.
  */
+#define WB_DATA_TYPE(p)	(__is_cp_guaranteed(p) ? F2FS_WB_CP_DATA : F2FS_WB_DATA)
 enum count_type {
 	F2FS_DIRTY_DENTS,
 	F2FS_DIRTY_DATA,
@@ -656,6 +665,8 @@ enum count_type {
 	F2FS_DIRTY_META,
 	F2FS_INMEM_PAGES,
 	F2FS_DIRTY_IMETA,
+	F2FS_WB_CP_DATA,
+	F2FS_WB_DATA,
 	NR_COUNT_TYPE,
 };
 
@@ -688,7 +699,7 @@ struct f2fs_io_info {
 	struct f2fs_sb_info *sbi;	/* f2fs_sb_info pointer */
 	enum page_type type;	/* contains DATA/NODE/META/META_FLUSH */
 	int op;			/* contains REQ_OP_ */
-	int op_flags;		/* rq_flag_bits */
+	int op_flags;		/* req_flag_bits */
 	block_t new_blkaddr;	/* new block address to be written */
 	block_t old_blkaddr;	/* old block address before Cow */
 	struct page *page;	/* page to be written */
@@ -704,6 +715,20 @@ struct f2fs_bio_info {
 	struct rw_semaphore io_rwsem;	/* blocking op for bio */
 };
 
+#define FDEV(i)				(sbi->devs[i])
+#define RDEV(i)				(raw_super->devs[i])
+struct f2fs_dev_info {
+	struct block_device *bdev;
+	char path[MAX_PATH_LEN];
+	unsigned int total_segments;
+	block_t start_blk;
+	block_t end_blk;
+#ifdef CONFIG_BLK_DEV_ZONED
+	unsigned int nr_blkz;			/* Total number of zones */
+	u8 *blkz_type;				/* Array of zones type */
+#endif
+};
+
 enum inode_type {
 	DIR_INODE,			/* for dirty dir inode */
 	FILE_INODE,			/* for dirty regular/symlink inode */
@@ -750,6 +775,12 @@ struct f2fs_sb_info {
 	u8 key_prefix[F2FS_KEY_DESC_PREFIX_SIZE];
 	u8 key_prefix_size;
 #endif
+
+#ifdef CONFIG_BLK_DEV_ZONED
+	unsigned int blocks_per_blkz;		/* F2FS blocks per zone */
+	unsigned int log_blocks_per_blkz;	/* log2 F2FS blocks per zone */
+#endif
+
 	/* for node-related operations */
 	struct f2fs_nm_info *nm_info;		/* node manager */
 	struct inode *node_inode;		/* cache node blocks */
@@ -764,6 +795,7 @@ struct f2fs_sb_info {
 
 	/* for checkpoint */
 	struct f2fs_checkpoint *ckpt;		/* raw checkpoint pointer */
+	int cur_cp_pack;			/* remain current cp pack */
 	spinlock_t cp_lock;			/* for flag in ckpt */
 	struct inode *meta_inode;		/* cache meta blocks */
 	struct mutex cp_mutex;			/* checkpoint procedure lock */
@@ -815,10 +847,9 @@ struct f2fs_sb_info {
 	block_t discard_blks;			/* discard command candidats */
 	block_t last_valid_block_count;		/* for recovery */
 	u32 s_next_generation;			/* for NFS support */
-	atomic_t nr_wb_bios;			/* # of writeback bios */
 
 	/* # of pages, see count_type */
-	struct percpu_counter nr_pages[NR_COUNT_TYPE];
+	atomic_t nr_pages[NR_COUNT_TYPE];
 	/* # of allocated blocks */
 	struct percpu_counter alloc_valid_block_count;
 
@@ -863,6 +894,8 @@ struct f2fs_sb_info {
 
 	/* For shrinker support */
 	struct list_head s_list;
+	int s_ndevs;				/* number of devices */
+	struct f2fs_dev_info *devs;		/* for device list */
 	struct mutex umount_mutex;
 	unsigned int shrinker_run_no;
 
@@ -1105,13 +1138,6 @@ static inline void clear_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f)
 	spin_unlock(&sbi->cp_lock);
 }
 
-static inline bool f2fs_discard_en(struct f2fs_sb_info *sbi)
-{
-	struct request_queue *q = bdev_get_queue(sbi->sb->s_bdev);
-
-	return blk_queue_discard(q);
-}
-
 static inline void f2fs_lock_op(struct f2fs_sb_info *sbi)
 {
 	down_read(&sbi->cp_rwsem);
@@ -1232,9 +1258,10 @@ static inline void dec_valid_block_count(struct f2fs_sb_info *sbi,
 
 static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type)
 {
-	percpu_counter_inc(&sbi->nr_pages[count_type]);
+	atomic_inc(&sbi->nr_pages[count_type]);
 
-	if (count_type == F2FS_DIRTY_DATA || count_type == F2FS_INMEM_PAGES)
+	if (count_type == F2FS_DIRTY_DATA || count_type == F2FS_INMEM_PAGES ||
+		count_type == F2FS_WB_CP_DATA || count_type == F2FS_WB_DATA)
 		return;
 
 	set_sbi_flag(sbi, SBI_IS_DIRTY);
@@ -1242,14 +1269,14 @@ static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type)
 
 static inline void inode_inc_dirty_pages(struct inode *inode)
 {
-	percpu_counter_inc(&F2FS_I(inode)->dirty_pages);
+	atomic_inc(&F2FS_I(inode)->dirty_pages);
 	inc_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
 				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
 }
 
 static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type)
 {
-	percpu_counter_dec(&sbi->nr_pages[count_type]);
+	atomic_dec(&sbi->nr_pages[count_type]);
 }
 
 static inline void inode_dec_dirty_pages(struct inode *inode)
@@ -1258,19 +1285,19 @@ static inline void inode_dec_dirty_pages(struct inode *inode)
 			!S_ISLNK(inode->i_mode))
 		return;
 
-	percpu_counter_dec(&F2FS_I(inode)->dirty_pages);
+	atomic_dec(&F2FS_I(inode)->dirty_pages);
 	dec_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
 				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
 }
 
 static inline s64 get_pages(struct f2fs_sb_info *sbi, int count_type)
 {
-	return percpu_counter_sum_positive(&sbi->nr_pages[count_type]);
+	return atomic_read(&sbi->nr_pages[count_type]);
 }
 
-static inline s64 get_dirty_pages(struct inode *inode)
+static inline int get_dirty_pages(struct inode *inode)
 {
-	return percpu_counter_sum_positive(&F2FS_I(inode)->dirty_pages);
+	return atomic_read(&F2FS_I(inode)->dirty_pages);
 }
 
 static inline int get_blocktype_secs(struct f2fs_sb_info *sbi, int block_type)
@@ -1329,22 +1356,27 @@ static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag)
 
 static inline block_t __start_cp_addr(struct f2fs_sb_info *sbi)
 {
-	block_t start_addr;
-	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
-	unsigned long long ckpt_version = cur_cp_version(ckpt);
-
-	start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
+	block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
 
-	/*
-	 * odd numbered checkpoint should at cp segment 0
-	 * and even segment must be at cp segment 1
-	 */
-	if (!(ckpt_version & 1))
+	if (sbi->cur_cp_pack == 2)
 		start_addr += sbi->blocks_per_seg;
+	return start_addr;
+}
+
+static inline block_t __start_cp_next_addr(struct f2fs_sb_info *sbi)
+{
+	block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr);
 
+	if (sbi->cur_cp_pack == 1)
+		start_addr += sbi->blocks_per_seg;
 	return start_addr;
 }
 
+static inline void __set_cp_next_pack(struct f2fs_sb_info *sbi)
+{
+	sbi->cur_cp_pack = (sbi->cur_cp_pack == 1) ? 2 : 1;
+}
+
 static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi)
 {
 	return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum);
@@ -1621,7 +1653,7 @@ static inline void __mark_inode_dirty_flag(struct inode *inode,
 			return;
 	case FI_DATA_EXIST:
 	case FI_INLINE_DOTS:
-		f2fs_mark_inode_dirty_sync(inode);
+		f2fs_mark_inode_dirty_sync(inode, true);
 	}
 }
 
@@ -1648,7 +1680,7 @@ static inline void set_acl_inode(struct inode *inode, umode_t mode)
 {
 	F2FS_I(inode)->i_acl_mode = mode;
 	set_inode_flag(inode, FI_ACL_MODE);
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, false);
 }
 
 static inline void f2fs_i_links_write(struct inode *inode, bool inc)
@@ -1657,7 +1689,7 @@ static inline void f2fs_i_links_write(struct inode *inode, bool inc)
 		inc_nlink(inode);
 	else
 		drop_nlink(inode);
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, true);
 }
 
 static inline void f2fs_i_blocks_write(struct inode *inode,
@@ -1668,7 +1700,7 @@ static inline void f2fs_i_blocks_write(struct inode *inode,
 
 	inode->i_blocks = add ? inode->i_blocks + diff :
 				inode->i_blocks - diff;
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, true);
 	if (clean || recover)
 		set_inode_flag(inode, FI_AUTO_RECOVER);
 }
@@ -1682,34 +1714,27 @@ static inline void f2fs_i_size_write(struct inode *inode, loff_t i_size)
 		return;
 
 	i_size_write(inode, i_size);
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, true);
 	if (clean || recover)
 		set_inode_flag(inode, FI_AUTO_RECOVER);
 }
 
-static inline bool f2fs_skip_inode_update(struct inode *inode)
-{
-	if (!is_inode_flag_set(inode, FI_AUTO_RECOVER))
-		return false;
-	return F2FS_I(inode)->last_disk_size == i_size_read(inode);
-}
-
 static inline void f2fs_i_depth_write(struct inode *inode, unsigned int depth)
 {
 	F2FS_I(inode)->i_current_depth = depth;
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, true);
 }
 
 static inline void f2fs_i_xnid_write(struct inode *inode, nid_t xnid)
 {
 	F2FS_I(inode)->i_xattr_nid = xnid;
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, true);
 }
 
 static inline void f2fs_i_pino_write(struct inode *inode, nid_t pino)
 {
 	F2FS_I(inode)->i_pino = pino;
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, true);
 }
 
 static inline void get_inline_info(struct inode *inode, struct f2fs_inode *ri)
@@ -1837,13 +1862,31 @@ static inline int is_file(struct inode *inode, int type)
 static inline void set_file(struct inode *inode, int type)
 {
 	F2FS_I(inode)->i_advise |= type;
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, true);
 }
 
 static inline void clear_file(struct inode *inode, int type)
 {
 	F2FS_I(inode)->i_advise &= ~type;
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, true);
+}
+
+static inline bool f2fs_skip_inode_update(struct inode *inode, int dsync)
+{
+	if (dsync) {
+		struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
+		bool ret;
+
+		spin_lock(&sbi->inode_lock[DIRTY_META]);
+		ret = list_empty(&F2FS_I(inode)->gdirty_list);
+		spin_unlock(&sbi->inode_lock[DIRTY_META]);
+		return ret;
+	}
+	if (!is_inode_flag_set(inode, FI_AUTO_RECOVER) ||
+			file_keep_isize(inode) ||
+			i_size_read(inode) & PAGE_MASK)
+		return false;
+	return F2FS_I(inode)->last_disk_size == i_size_read(inode);
 }
 
 static inline int f2fs_readonly(struct super_block *sb)
@@ -1955,7 +1998,7 @@ void set_de_type(struct f2fs_dir_entry *, umode_t);
 unsigned char get_de_type(struct f2fs_dir_entry *);
 struct f2fs_dir_entry *find_target_dentry(struct fscrypt_name *,
 			f2fs_hash_t, int *, struct f2fs_dentry_ptr *);
-bool f2fs_fill_dentries(struct dir_context *, struct f2fs_dentry_ptr *,
+int f2fs_fill_dentries(struct dir_context *, struct f2fs_dentry_ptr *,
 			unsigned int, struct fscrypt_str *);
 void do_make_empty_dir(struct inode *, struct inode *,
 			struct f2fs_dentry_ptr *);
@@ -1995,7 +2038,7 @@ static inline int f2fs_add_link(struct dentry *dentry, struct inode *inode)
 /*
  * super.c
  */
-int f2fs_inode_dirtied(struct inode *);
+int f2fs_inode_dirtied(struct inode *, bool);
 void f2fs_inode_synced(struct inode *);
 int f2fs_commit_super(struct f2fs_sb_info *, bool);
 int f2fs_sync_fs(struct super_block *, int);
@@ -2034,7 +2077,7 @@ void move_node_page(struct page *, int);
 int fsync_node_pages(struct f2fs_sb_info *, struct inode *,
 			struct writeback_control *, bool);
 int sync_node_pages(struct f2fs_sb_info *, struct writeback_control *);
-void build_free_nids(struct f2fs_sb_info *);
+void build_free_nids(struct f2fs_sb_info *, bool);
 bool alloc_nid(struct f2fs_sb_info *, nid_t *);
 void alloc_nid_done(struct f2fs_sb_info *, nid_t);
 void alloc_nid_failed(struct f2fs_sb_info *, nid_t);
@@ -2060,7 +2103,7 @@ void f2fs_balance_fs(struct f2fs_sb_info *, bool);
 void f2fs_balance_fs_bg(struct f2fs_sb_info *);
 int f2fs_issue_flush(struct f2fs_sb_info *);
 int create_flush_cmd_control(struct f2fs_sb_info *);
-void destroy_flush_cmd_control(struct f2fs_sb_info *);
+void destroy_flush_cmd_control(struct f2fs_sb_info *, bool);
 void invalidate_blocks(struct f2fs_sb_info *, block_t);
 bool is_checkpointed_data(struct f2fs_sb_info *, block_t);
 void refresh_sit_entry(struct f2fs_sb_info *, block_t, block_t);
@@ -2132,12 +2175,15 @@ void f2fs_submit_merged_bio_cond(struct f2fs_sb_info *, struct inode *,
 void f2fs_flush_merged_bios(struct f2fs_sb_info *);
 int f2fs_submit_page_bio(struct f2fs_io_info *);
 void f2fs_submit_page_mbio(struct f2fs_io_info *);
+struct block_device *f2fs_target_device(struct f2fs_sb_info *,
+				block_t, struct bio *);
+int f2fs_target_device_index(struct f2fs_sb_info *, block_t);
 void set_data_blkaddr(struct dnode_of_data *);
 void f2fs_update_data_blkaddr(struct dnode_of_data *, block_t);
 int reserve_new_blocks(struct dnode_of_data *, blkcnt_t);
 int reserve_new_block(struct dnode_of_data *);
 int f2fs_get_block(struct dnode_of_data *, pgoff_t);
-ssize_t f2fs_preallocate_blocks(struct kiocb *, struct iov_iter *);
+int f2fs_preallocate_blocks(struct kiocb *, struct iov_iter *);
 int f2fs_reserve_block(struct dnode_of_data *, pgoff_t);
 struct page *get_read_data_page(struct inode *, pgoff_t, int, bool);
 struct page *find_data_page(struct inode *, pgoff_t);
@@ -2160,7 +2206,7 @@ int f2fs_migrate_page(struct address_space *, struct page *, struct page *,
 int start_gc_thread(struct f2fs_sb_info *);
 void stop_gc_thread(struct f2fs_sb_info *);
 block_t start_bidx_of_node(unsigned int, struct inode *);
-int f2fs_gc(struct f2fs_sb_info *, bool);
+int f2fs_gc(struct f2fs_sb_info *, bool, bool);
 void build_gc_manager(struct f2fs_sb_info *);
 
 /*
@@ -2181,12 +2227,12 @@ struct f2fs_stat_info {
 	unsigned long long hit_largest, hit_cached, hit_rbtree;
 	unsigned long long hit_total, total_ext;
 	int ext_tree, zombie_tree, ext_node;
-	s64 ndirty_node, ndirty_dent, ndirty_meta, ndirty_data, ndirty_imeta;
-	s64 inmem_pages;
+	int ndirty_node, ndirty_dent, ndirty_meta, ndirty_data, ndirty_imeta;
+	int inmem_pages;
 	unsigned int ndirty_dirs, ndirty_files, ndirty_all;
-	int nats, dirty_nats, sits, dirty_sits, fnids;
+	int nats, dirty_nats, sits, dirty_sits, free_nids, alloc_nids;
 	int total_count, utilization;
-	int bg_gc, wb_bios;
+	int bg_gc, nr_wb_cp_data, nr_wb_data;
 	int inline_xattr, inline_inode, inline_dir, orphans;
 	unsigned int valid_count, valid_node_count, valid_inode_count, discard_blks;
 	unsigned int bimodal, avg_vblocks;
@@ -2412,9 +2458,30 @@ static inline int f2fs_sb_has_crypto(struct super_block *sb)
 	return F2FS_HAS_FEATURE(sb, F2FS_FEATURE_ENCRYPT);
 }
 
-static inline int f2fs_sb_mounted_hmsmr(struct super_block *sb)
+static inline int f2fs_sb_mounted_blkzoned(struct super_block *sb)
+{
+	return F2FS_HAS_FEATURE(sb, F2FS_FEATURE_BLKZONED);
+}
+
+#ifdef CONFIG_BLK_DEV_ZONED
+static inline int get_blkz_type(struct f2fs_sb_info *sbi,
+			struct block_device *bdev, block_t blkaddr)
+{
+	unsigned int zno = blkaddr >> sbi->log_blocks_per_blkz;
+	int i;
+
+	for (i = 0; i < sbi->s_ndevs; i++)
+		if (FDEV(i).bdev == bdev)
+			return FDEV(i).blkz_type[zno];
+	return -EINVAL;
+}
+#endif
+
+static inline bool f2fs_discard_en(struct f2fs_sb_info *sbi)
 {
-	return F2FS_HAS_FEATURE(sb, F2FS_FEATURE_HMSMR);
+	struct request_queue *q = bdev_get_queue(sbi->sb->s_bdev);
+
+	return blk_queue_discard(q) || f2fs_sb_mounted_blkzoned(sbi->sb);
 }
 
 static inline void set_opt_mode(struct f2fs_sb_info *sbi, unsigned int mt)
@@ -2453,8 +2520,8 @@ static inline bool f2fs_may_encrypt(struct inode *inode)
 #define fscrypt_pullback_bio_page	fscrypt_notsupp_pullback_bio_page
 #define fscrypt_restore_control_page	fscrypt_notsupp_restore_control_page
 #define fscrypt_zeroout_range		fscrypt_notsupp_zeroout_range
-#define fscrypt_process_policy		fscrypt_notsupp_process_policy
-#define fscrypt_get_policy		fscrypt_notsupp_get_policy
+#define fscrypt_ioctl_set_policy	fscrypt_notsupp_ioctl_set_policy
+#define fscrypt_ioctl_get_policy	fscrypt_notsupp_ioctl_get_policy
 #define fscrypt_has_permitted_context	fscrypt_notsupp_has_permitted_context
 #define fscrypt_inherit_context		fscrypt_notsupp_inherit_context
 #define fscrypt_get_encryption_info	fscrypt_notsupp_get_encryption_info
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index c7865073cd26..49f10dce817d 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -94,8 +94,6 @@ mapped:
 	if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
 		f2fs_wait_on_encrypted_page_writeback(sbi, dn.data_blkaddr);
 
-	/* if gced page is attached, don't write to cold segment */
-	clear_cold_data(page);
 out:
 	sb_end_pagefault(inode->i_sb);
 	f2fs_update_time(sbi, REQ_TIME);
@@ -210,7 +208,7 @@ static int f2fs_do_sync_file(struct file *file, loff_t start, loff_t end,
 	}
 
 	/* if the inode is dirty, let's recover all the time */
-	if (!datasync && !f2fs_skip_inode_update(inode)) {
+	if (!f2fs_skip_inode_update(inode, datasync)) {
 		f2fs_write_inode(inode, NULL);
 		goto go_write;
 	}
@@ -264,7 +262,7 @@ sync_nodes:
 	}
 
 	if (need_inode_block_update(sbi, ino)) {
-		f2fs_mark_inode_dirty_sync(inode);
+		f2fs_mark_inode_dirty_sync(inode, true);
 		f2fs_write_inode(inode, NULL);
 		goto sync_nodes;
 	}
@@ -632,7 +630,7 @@ int f2fs_truncate(struct inode *inode)
 		return err;
 
 	inode->i_mtime = inode->i_ctime = current_time(inode);
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, false);
 	return 0;
 }
 
@@ -679,6 +677,7 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
 	int err;
+	bool size_changed = false;
 
 	err = setattr_prepare(dentry, attr);
 	if (err)
@@ -694,7 +693,6 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr)
 			err = f2fs_truncate(inode);
 			if (err)
 				return err;
-			f2fs_balance_fs(F2FS_I_SB(inode), true);
 		} else {
 			/*
 			 * do not trim all blocks after i_size if target size is
@@ -710,6 +708,8 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr)
 			}
 			inode->i_mtime = inode->i_ctime = current_time(inode);
 		}
+
+		size_changed = true;
 	}
 
 	__setattr_copy(inode, attr);
@@ -722,7 +722,12 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr)
 		}
 	}
 
-	f2fs_mark_inode_dirty_sync(inode);
+	/* file size may changed here */
+	f2fs_mark_inode_dirty_sync(inode, size_changed);
+
+	/* inode change will produce dirty node pages flushed by checkpoint */
+	f2fs_balance_fs(F2FS_I_SB(inode), true);
+
 	return err;
 }
 
@@ -967,7 +972,7 @@ static int __clone_blkaddrs(struct inode *src_inode, struct inode *dst_inode,
 				new_size = (dst + i) << PAGE_SHIFT;
 				if (dst_inode->i_size < new_size)
 					f2fs_i_size_write(dst_inode, new_size);
-			} while ((do_replace[i] || blkaddr[i] == NULL_ADDR) && --ilen);
+			} while (--ilen && (do_replace[i] || blkaddr[i] == NULL_ADDR));
 
 			f2fs_put_dnode(&dn);
 		} else {
@@ -1218,6 +1223,9 @@ static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len,
 			ret = f2fs_do_zero_range(&dn, index, end);
 			f2fs_put_dnode(&dn);
 			f2fs_unlock_op(sbi);
+
+			f2fs_balance_fs(sbi, dn.node_changed);
+
 			if (ret)
 				goto out;
 
@@ -1313,15 +1321,15 @@ static int expand_inode_data(struct inode *inode, loff_t offset,
 	pgoff_t pg_end;
 	loff_t new_size = i_size_read(inode);
 	loff_t off_end;
-	int ret;
+	int err;
 
-	ret = inode_newsize_ok(inode, (len + offset));
-	if (ret)
-		return ret;
+	err = inode_newsize_ok(inode, (len + offset));
+	if (err)
+		return err;
 
-	ret = f2fs_convert_inline_inode(inode);
-	if (ret)
-		return ret;
+	err = f2fs_convert_inline_inode(inode);
+	if (err)
+		return err;
 
 	f2fs_balance_fs(sbi, true);
 
@@ -1333,12 +1341,12 @@ static int expand_inode_data(struct inode *inode, loff_t offset,
 	if (off_end)
 		map.m_len++;
 
-	ret = f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO);
-	if (ret) {
+	err = f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO);
+	if (err) {
 		pgoff_t last_off;
 
 		if (!map.m_len)
-			return ret;
+			return err;
 
 		last_off = map.m_lblk + map.m_len - 1;
 
@@ -1352,7 +1360,7 @@ static int expand_inode_data(struct inode *inode, loff_t offset,
 	if (!(mode & FALLOC_FL_KEEP_SIZE) && i_size_read(inode) < new_size)
 		f2fs_i_size_write(inode, new_size);
 
-	return ret;
+	return err;
 }
 
 static long f2fs_fallocate(struct file *file, int mode,
@@ -1393,7 +1401,9 @@ static long f2fs_fallocate(struct file *file, int mode,
 
 	if (!ret) {
 		inode->i_mtime = inode->i_ctime = current_time(inode);
-		f2fs_mark_inode_dirty_sync(inode);
+		f2fs_mark_inode_dirty_sync(inode, false);
+		if (mode & FALLOC_FL_KEEP_SIZE)
+			file_set_keep_isize(inode);
 		f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
 	}
 
@@ -1526,7 +1536,7 @@ static int f2fs_ioc_start_atomic_write(struct file *filp)
 		goto out;
 
 	f2fs_msg(F2FS_I_SB(inode)->sb, KERN_WARNING,
-		"Unexpected flush for atomic writes: ino=%lu, npages=%lld",
+		"Unexpected flush for atomic writes: ino=%lu, npages=%u",
 					inode->i_ino, get_dirty_pages(inode));
 	ret = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX);
 	if (ret)
@@ -1752,31 +1762,16 @@ static bool uuid_is_nonzero(__u8 u[16])
 
 static int f2fs_ioc_set_encryption_policy(struct file *filp, unsigned long arg)
 {
-	struct fscrypt_policy policy;
 	struct inode *inode = file_inode(filp);
 
-	if (copy_from_user(&policy, (struct fscrypt_policy __user *)arg,
-							sizeof(policy)))
-		return -EFAULT;
-
 	f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
 
-	return fscrypt_process_policy(filp, &policy);
+	return fscrypt_ioctl_set_policy(filp, (const void __user *)arg);
 }
 
 static int f2fs_ioc_get_encryption_policy(struct file *filp, unsigned long arg)
 {
-	struct fscrypt_policy policy;
-	struct inode *inode = file_inode(filp);
-	int err;
-
-	err = fscrypt_get_policy(inode, &policy);
-	if (err)
-		return err;
-
-	if (copy_to_user((struct fscrypt_policy __user *)arg, &policy, sizeof(policy)))
-		return -EFAULT;
-	return 0;
+	return fscrypt_ioctl_get_policy(filp, (void __user *)arg);
 }
 
 static int f2fs_ioc_get_encryption_pwsalt(struct file *filp, unsigned long arg)
@@ -1842,7 +1837,7 @@ static int f2fs_ioc_gc(struct file *filp, unsigned long arg)
 		mutex_lock(&sbi->gc_mutex);
 	}
 
-	ret = f2fs_gc(sbi, sync);
+	ret = f2fs_gc(sbi, sync, true);
 out:
 	mnt_drop_write_file(filp);
 	return ret;
@@ -2256,12 +2251,15 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	inode_lock(inode);
 	ret = generic_write_checks(iocb, from);
 	if (ret > 0) {
-		ret = f2fs_preallocate_blocks(iocb, from);
-		if (!ret) {
-			blk_start_plug(&plug);
-			ret = __generic_file_write_iter(iocb, from);
-			blk_finish_plug(&plug);
+		int err = f2fs_preallocate_blocks(iocb, from);
+
+		if (err) {
+			inode_unlock(inode);
+			return err;
 		}
+		blk_start_plug(&plug);
+		ret = __generic_file_write_iter(iocb, from);
+		blk_finish_plug(&plug);
 	}
 	inode_unlock(inode);
 
diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
index 6f14ee923acd..88bfc3dff496 100644
--- a/fs/f2fs/gc.c
+++ b/fs/f2fs/gc.c
@@ -82,7 +82,7 @@ static int gc_thread_func(void *data)
 		stat_inc_bggc_count(sbi);
 
 		/* if return value is not zero, no victim was selected */
-		if (f2fs_gc(sbi, test_opt(sbi, FORCE_FG_GC)))
+		if (f2fs_gc(sbi, test_opt(sbi, FORCE_FG_GC), true))
 			wait_ms = gc_th->no_gc_sleep_time;
 
 		trace_f2fs_background_gc(sbi->sb, wait_ms,
@@ -544,13 +544,14 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
 	return true;
 }
 
-static void move_encrypted_block(struct inode *inode, block_t bidx)
+static void move_encrypted_block(struct inode *inode, block_t bidx,
+							unsigned int segno, int off)
 {
 	struct f2fs_io_info fio = {
 		.sbi = F2FS_I_SB(inode),
 		.type = DATA,
 		.op = REQ_OP_READ,
-		.op_flags = READ_SYNC,
+		.op_flags = 0,
 		.encrypted_page = NULL,
 	};
 	struct dnode_of_data dn;
@@ -565,6 +566,9 @@ static void move_encrypted_block(struct inode *inode, block_t bidx)
 	if (!page)
 		return;
 
+	if (!check_valid_map(F2FS_I_SB(inode), segno, off))
+		goto out;
+
 	set_new_dnode(&dn, inode, NULL, NULL, 0);
 	err = get_dnode_of_data(&dn, bidx, LOOKUP_NODE);
 	if (err)
@@ -625,7 +629,7 @@ static void move_encrypted_block(struct inode *inode, block_t bidx)
 	f2fs_wait_on_page_writeback(dn.node_page, NODE, true);
 
 	fio.op = REQ_OP_WRITE;
-	fio.op_flags = WRITE_SYNC;
+	fio.op_flags = REQ_SYNC;
 	fio.new_blkaddr = newaddr;
 	f2fs_submit_page_mbio(&fio);
 
@@ -645,7 +649,8 @@ out:
 	f2fs_put_page(page, 1);
 }
 
-static void move_data_page(struct inode *inode, block_t bidx, int gc_type)
+static void move_data_page(struct inode *inode, block_t bidx, int gc_type,
+							unsigned int segno, int off)
 {
 	struct page *page;
 
@@ -653,6 +658,9 @@ static void move_data_page(struct inode *inode, block_t bidx, int gc_type)
 	if (IS_ERR(page))
 		return;
 
+	if (!check_valid_map(F2FS_I_SB(inode), segno, off))
+		goto out;
+
 	if (gc_type == BG_GC) {
 		if (PageWriteback(page))
 			goto out;
@@ -663,7 +671,7 @@ static void move_data_page(struct inode *inode, block_t bidx, int gc_type)
 			.sbi = F2FS_I_SB(inode),
 			.type = DATA,
 			.op = REQ_OP_WRITE,
-			.op_flags = WRITE_SYNC,
+			.op_flags = REQ_SYNC,
 			.page = page,
 			.encrypted_page = NULL,
 		};
@@ -673,8 +681,10 @@ static void move_data_page(struct inode *inode, block_t bidx, int gc_type)
 retry:
 		set_page_dirty(page);
 		f2fs_wait_on_page_writeback(page, DATA, true);
-		if (clear_page_dirty_for_io(page))
+		if (clear_page_dirty_for_io(page)) {
 			inode_dec_dirty_pages(inode);
+			remove_dirty_inode(inode);
+		}
 
 		set_cold_data(page);
 
@@ -683,8 +693,6 @@ retry:
 			congestion_wait(BLK_RW_ASYNC, HZ/50);
 			goto retry;
 		}
-
-		clear_cold_data(page);
 	}
 out:
 	f2fs_put_page(page, 1);
@@ -794,9 +802,9 @@ next_step:
 			start_bidx = start_bidx_of_node(nofs, inode)
 								+ ofs_in_node;
 			if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
-				move_encrypted_block(inode, start_bidx);
+				move_encrypted_block(inode, start_bidx, segno, off);
 			else
-				move_data_page(inode, start_bidx, gc_type);
+				move_data_page(inode, start_bidx, gc_type, segno, off);
 
 			if (locked) {
 				up_write(&fi->dio_rwsem[WRITE]);
@@ -899,7 +907,7 @@ next:
 	return sec_freed;
 }
 
-int f2fs_gc(struct f2fs_sb_info *sbi, bool sync)
+int f2fs_gc(struct f2fs_sb_info *sbi, bool sync, bool background)
 {
 	unsigned int segno;
 	int gc_type = sync ? FG_GC : BG_GC;
@@ -940,6 +948,9 @@ gc_more:
 			if (ret)
 				goto stop;
 		}
+	} else if (gc_type == BG_GC && !background) {
+		/* f2fs_balance_fs doesn't need to do BG_GC in critical path. */
+		goto stop;
 	}
 
 	if (segno == NULL_SEGNO && !__get_victim(sbi, &segno, gc_type))
diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c
index 5f1a67f756af..e32a9e527968 100644
--- a/fs/f2fs/inline.c
+++ b/fs/f2fs/inline.c
@@ -111,7 +111,7 @@ int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page)
 		.sbi = F2FS_I_SB(dn->inode),
 		.type = DATA,
 		.op = REQ_OP_WRITE,
-		.op_flags = WRITE_SYNC | REQ_PRIO,
+		.op_flags = REQ_SYNC | REQ_PRIO,
 		.page = page,
 		.encrypted_page = NULL,
 	};
@@ -137,8 +137,10 @@ int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page)
 	fio.old_blkaddr = dn->data_blkaddr;
 	write_data_page(dn, &fio);
 	f2fs_wait_on_page_writeback(page, DATA, true);
-	if (dirty)
+	if (dirty) {
 		inode_dec_dirty_pages(dn->inode);
+		remove_dirty_inode(dn->inode);
+	}
 
 	/* this converted inline_data should be recovered. */
 	set_inode_flag(dn->inode, FI_APPEND_WRITE);
@@ -419,7 +421,7 @@ static int f2fs_add_inline_entries(struct inode *dir,
 		}
 
 		new_name.name = d.filename[bit_pos];
-		new_name.len = de->name_len;
+		new_name.len = le16_to_cpu(de->name_len);
 
 		ino = le32_to_cpu(de->ino);
 		fake_mode = get_de_type(de) << S_SHIFT;
@@ -573,7 +575,7 @@ void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, struct page *page,
 	f2fs_put_page(page, 1);
 
 	dir->i_ctime = dir->i_mtime = current_time(dir);
-	f2fs_mark_inode_dirty_sync(dir);
+	f2fs_mark_inode_dirty_sync(dir, false);
 
 	if (inode)
 		f2fs_drop_nlink(dir, inode);
@@ -610,6 +612,7 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
 	struct f2fs_inline_dentry *inline_dentry = NULL;
 	struct page *ipage = NULL;
 	struct f2fs_dentry_ptr d;
+	int err;
 
 	if (ctx->pos == NR_INLINE_DENTRY)
 		return 0;
@@ -622,11 +625,12 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
 
 	make_dentry_ptr(inode, &d, (void *)inline_dentry, 2);
 
-	if (!f2fs_fill_dentries(ctx, &d, 0, fstr))
+	err = f2fs_fill_dentries(ctx, &d, 0, fstr);
+	if (!err)
 		ctx->pos = NR_INLINE_DENTRY;
 
 	f2fs_put_page(ipage, 1);
-	return 0;
+	return err < 0 ? err : 0;
 }
 
 int f2fs_inline_data_fiemap(struct inode *inode,
diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index d7369895a78a..af06bda51a54 100644
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -19,10 +19,11 @@
 
 #include <trace/events/f2fs.h>
 
-void f2fs_mark_inode_dirty_sync(struct inode *inode)
+void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync)
 {
-	if (f2fs_inode_dirtied(inode))
+	if (f2fs_inode_dirtied(inode, sync))
 		return;
+
 	mark_inode_dirty_sync(inode);
 }
 
@@ -43,7 +44,7 @@ void f2fs_set_inode_flags(struct inode *inode)
 		new_fl |= S_DIRSYNC;
 	inode_set_flags(inode, new_fl,
 			S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC);
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, false);
 }
 
 static void __get_inode_rdev(struct inode *inode, struct f2fs_inode *ri)
@@ -252,6 +253,7 @@ retry:
 int update_inode(struct inode *inode, struct page *node_page)
 {
 	struct f2fs_inode *ri;
+	struct extent_tree *et = F2FS_I(inode)->extent_tree;
 
 	f2fs_inode_synced(inode);
 
@@ -267,11 +269,13 @@ int update_inode(struct inode *inode, struct page *node_page)
 	ri->i_size = cpu_to_le64(i_size_read(inode));
 	ri->i_blocks = cpu_to_le64(inode->i_blocks);
 
-	if (F2FS_I(inode)->extent_tree)
-		set_raw_extent(&F2FS_I(inode)->extent_tree->largest,
-							&ri->i_ext);
-	else
+	if (et) {
+		read_lock(&et->lock);
+		set_raw_extent(&et->largest, &ri->i_ext);
+		read_unlock(&et->lock);
+	} else {
 		memset(&ri->i_ext, 0, sizeof(ri->i_ext));
+	}
 	set_raw_inline(inode, ri);
 
 	ri->i_atime = cpu_to_le64(inode->i_atime.tv_sec);
@@ -335,7 +339,7 @@ int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc)
 	 * We need to balance fs here to prevent from producing dirty node pages
 	 * during the urgent cleaning time when runing out of free sections.
 	 */
-	if (update_inode_page(inode))
+	if (update_inode_page(inode) && wbc && wbc->nr_to_write)
 		f2fs_balance_fs(sbi, true);
 	return 0;
 }
@@ -373,6 +377,9 @@ void f2fs_evict_inode(struct inode *inode)
 		goto no_delete;
 #endif
 
+	remove_ino_entry(sbi, inode->i_ino, APPEND_INO);
+	remove_ino_entry(sbi, inode->i_ino, UPDATE_INO);
+
 	sb_start_intwrite(inode->i_sb);
 	set_inode_flag(inode, FI_NO_ALLOC);
 	i_size_write(inode, 0);
@@ -384,6 +391,8 @@ retry:
 		f2fs_lock_op(sbi);
 		err = remove_inode_page(inode);
 		f2fs_unlock_op(sbi);
+		if (err == -ENOENT)
+			err = 0;
 	}
 
 	/* give more chances, if ENOMEM case */
@@ -403,10 +412,12 @@ no_delete:
 	invalidate_mapping_pages(NODE_MAPPING(sbi), inode->i_ino, inode->i_ino);
 	if (xnid)
 		invalidate_mapping_pages(NODE_MAPPING(sbi), xnid, xnid);
-	if (is_inode_flag_set(inode, FI_APPEND_WRITE))
-		add_ino_entry(sbi, inode->i_ino, APPEND_INO);
-	if (is_inode_flag_set(inode, FI_UPDATE_WRITE))
-		add_ino_entry(sbi, inode->i_ino, UPDATE_INO);
+	if (inode->i_nlink) {
+		if (is_inode_flag_set(inode, FI_APPEND_WRITE))
+			add_ino_entry(sbi, inode->i_ino, APPEND_INO);
+		if (is_inode_flag_set(inode, FI_UPDATE_WRITE))
+			add_ino_entry(sbi, inode->i_ino, UPDATE_INO);
+	}
 	if (is_inode_flag_set(inode, FI_FREE_NID)) {
 		alloc_nid_failed(sbi, inode->i_ino);
 		clear_inode_flag(inode, FI_FREE_NID);
@@ -424,6 +435,18 @@ void handle_failed_inode(struct inode *inode)
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct node_info ni;
 
+	/*
+	 * clear nlink of inode in order to release resource of inode
+	 * immediately.
+	 */
+	clear_nlink(inode);
+
+	/*
+	 * we must call this to avoid inode being remained as dirty, resulting
+	 * in a panic when flushing dirty inodes in gdirty_list.
+	 */
+	update_inode_page(inode);
+
 	/* don't make bad inode, since it becomes a regular file. */
 	unlock_new_inode(inode);
 
diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
index 489fa0d5f914..db33b5631dc8 100644
--- a/fs/f2fs/namei.c
+++ b/fs/f2fs/namei.c
@@ -778,7 +778,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
 	up_write(&F2FS_I(old_inode)->i_sem);
 
 	old_inode->i_ctime = current_time(old_inode);
-	f2fs_mark_inode_dirty_sync(old_inode);
+	f2fs_mark_inode_dirty_sync(old_inode, false);
 
 	f2fs_delete_entry(old_entry, old_page, old_dir, NULL);
 
@@ -938,7 +938,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
 		f2fs_i_links_write(old_dir, old_nlink > 0);
 		up_write(&F2FS_I(old_dir)->i_sem);
 	}
-	f2fs_mark_inode_dirty_sync(old_dir);
+	f2fs_mark_inode_dirty_sync(old_dir, false);
 
 	/* update directory entry info of new dir inode */
 	f2fs_set_link(new_dir, new_entry, new_page, old_inode);
@@ -953,7 +953,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
 		f2fs_i_links_write(new_dir, new_nlink > 0);
 		up_write(&F2FS_I(new_dir)->i_sem);
 	}
-	f2fs_mark_inode_dirty_sync(new_dir);
+	f2fs_mark_inode_dirty_sync(new_dir, false);
 
 	f2fs_unlock_op(sbi);
 
diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c
index 01177ecdeab8..b9078fdb3743 100644
--- a/fs/f2fs/node.c
+++ b/fs/f2fs/node.c
@@ -45,8 +45,8 @@ bool available_free_memory(struct f2fs_sb_info *sbi, int type)
 	 * give 25%, 25%, 50%, 50%, 50% memory for each components respectively
 	 */
 	if (type == FREE_NIDS) {
-		mem_size = (nm_i->fcnt * sizeof(struct free_nid)) >>
-							PAGE_SHIFT;
+		mem_size = (nm_i->nid_cnt[FREE_NID_LIST] *
+				sizeof(struct free_nid)) >> PAGE_SHIFT;
 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
 	} else if (type == NAT_ENTRIES) {
 		mem_size = (nm_i->nat_cnt * sizeof(struct nat_entry)) >>
@@ -270,8 +270,9 @@ static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid,
 		e = grab_nat_entry(nm_i, nid);
 		node_info_from_raw_nat(&e->ni, ne);
 	} else {
-		f2fs_bug_on(sbi, nat_get_ino(e) != ne->ino ||
-				nat_get_blkaddr(e) != ne->block_addr ||
+		f2fs_bug_on(sbi, nat_get_ino(e) != le32_to_cpu(ne->ino) ||
+				nat_get_blkaddr(e) !=
+					le32_to_cpu(ne->block_addr) ||
 				nat_get_version(e) != ne->version);
 	}
 }
@@ -1134,7 +1135,7 @@ repeat:
 	if (!page)
 		return ERR_PTR(-ENOMEM);
 
-	err = read_node_page(page, READ_SYNC);
+	err = read_node_page(page, 0);
 	if (err < 0) {
 		f2fs_put_page(page, 1);
 		return ERR_PTR(err);
@@ -1204,6 +1205,7 @@ static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino)
 
 	ret = f2fs_write_inline_data(inode, page);
 	inode_dec_dirty_pages(inode);
+	remove_dirty_inode(inode);
 	if (ret)
 		set_page_dirty(page);
 page_out:
@@ -1338,7 +1340,8 @@ retry:
 			if (unlikely(f2fs_cp_error(sbi))) {
 				f2fs_put_page(last_page, 0);
 				pagevec_release(&pvec);
-				return -EIO;
+				ret = -EIO;
+				goto out;
 			}
 
 			if (!IS_DNODE(page) || !is_cold_node(page))
@@ -1407,11 +1410,12 @@ continue_unlock:
 			"Retry to write fsync mark: ino=%u, idx=%lx",
 					ino, last_page->index);
 		lock_page(last_page);
+		f2fs_wait_on_page_writeback(last_page, NODE, true);
 		set_page_dirty(last_page);
 		unlock_page(last_page);
 		goto retry;
 	}
-
+out:
 	if (nwritten)
 		f2fs_submit_merged_bio_cond(sbi, NULL, NULL, ino, NODE, WRITE);
 	return ret ? -EIO: 0;
@@ -1570,7 +1574,7 @@ static int f2fs_write_node_page(struct page *page,
 		.sbi = sbi,
 		.type = NODE,
 		.op = REQ_OP_WRITE,
-		.op_flags = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : 0,
+		.op_flags = wbc_to_write_flags(wbc),
 		.page = page,
 		.encrypted_page = NULL,
 	};
@@ -1692,11 +1696,35 @@ static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i,
 	return radix_tree_lookup(&nm_i->free_nid_root, n);
 }
 
-static void __del_from_free_nid_list(struct f2fs_nm_info *nm_i,
-						struct free_nid *i)
+static int __insert_nid_to_list(struct f2fs_sb_info *sbi,
+			struct free_nid *i, enum nid_list list, bool new)
 {
+	struct f2fs_nm_info *nm_i = NM_I(sbi);
+
+	if (new) {
+		int err = radix_tree_insert(&nm_i->free_nid_root, i->nid, i);
+		if (err)
+			return err;
+	}
+
+	f2fs_bug_on(sbi, list == FREE_NID_LIST ? i->state != NID_NEW :
+						i->state != NID_ALLOC);
+	nm_i->nid_cnt[list]++;
+	list_add_tail(&i->list, &nm_i->nid_list[list]);
+	return 0;
+}
+
+static void __remove_nid_from_list(struct f2fs_sb_info *sbi,
+			struct free_nid *i, enum nid_list list, bool reuse)
+{
+	struct f2fs_nm_info *nm_i = NM_I(sbi);
+
+	f2fs_bug_on(sbi, list == FREE_NID_LIST ? i->state != NID_NEW :
+						i->state != NID_ALLOC);
+	nm_i->nid_cnt[list]--;
 	list_del(&i->list);
-	radix_tree_delete(&nm_i->free_nid_root, i->nid);
+	if (!reuse)
+		radix_tree_delete(&nm_i->free_nid_root, i->nid);
 }
 
 static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build)
@@ -1704,9 +1732,7 @@ static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build)
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct free_nid *i;
 	struct nat_entry *ne;
-
-	if (!available_free_memory(sbi, FREE_NIDS))
-		return -1;
+	int err;
 
 	/* 0 nid should not be used */
 	if (unlikely(nid == 0))
@@ -1729,33 +1755,30 @@ static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build)
 		return 0;
 	}
 
-	spin_lock(&nm_i->free_nid_list_lock);
-	if (radix_tree_insert(&nm_i->free_nid_root, i->nid, i)) {
-		spin_unlock(&nm_i->free_nid_list_lock);
-		radix_tree_preload_end();
+	spin_lock(&nm_i->nid_list_lock);
+	err = __insert_nid_to_list(sbi, i, FREE_NID_LIST, true);
+	spin_unlock(&nm_i->nid_list_lock);
+	radix_tree_preload_end();
+	if (err) {
 		kmem_cache_free(free_nid_slab, i);
 		return 0;
 	}
-	list_add_tail(&i->list, &nm_i->free_nid_list);
-	nm_i->fcnt++;
-	spin_unlock(&nm_i->free_nid_list_lock);
-	radix_tree_preload_end();
 	return 1;
 }
 
-static void remove_free_nid(struct f2fs_nm_info *nm_i, nid_t nid)
+static void remove_free_nid(struct f2fs_sb_info *sbi, nid_t nid)
 {
+	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct free_nid *i;
 	bool need_free = false;
 
-	spin_lock(&nm_i->free_nid_list_lock);
+	spin_lock(&nm_i->nid_list_lock);
 	i = __lookup_free_nid_list(nm_i, nid);
 	if (i && i->state == NID_NEW) {
-		__del_from_free_nid_list(nm_i, i);
-		nm_i->fcnt--;
+		__remove_nid_from_list(sbi, i, FREE_NID_LIST, false);
 		need_free = true;
 	}
-	spin_unlock(&nm_i->free_nid_list_lock);
+	spin_unlock(&nm_i->nid_list_lock);
 
 	if (need_free)
 		kmem_cache_free(free_nid_slab, i);
@@ -1778,14 +1801,12 @@ static void scan_nat_page(struct f2fs_sb_info *sbi,
 
 		blk_addr = le32_to_cpu(nat_blk->entries[i].block_addr);
 		f2fs_bug_on(sbi, blk_addr == NEW_ADDR);
-		if (blk_addr == NULL_ADDR) {
-			if (add_free_nid(sbi, start_nid, true) < 0)
-				break;
-		}
+		if (blk_addr == NULL_ADDR)
+			add_free_nid(sbi, start_nid, true);
 	}
 }
 
-void build_free_nids(struct f2fs_sb_info *sbi)
+static void __build_free_nids(struct f2fs_sb_info *sbi, bool sync)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
@@ -1794,7 +1815,10 @@ void build_free_nids(struct f2fs_sb_info *sbi)
 	nid_t nid = nm_i->next_scan_nid;
 
 	/* Enough entries */
-	if (nm_i->fcnt >= NAT_ENTRY_PER_BLOCK)
+	if (nm_i->nid_cnt[FREE_NID_LIST] >= NAT_ENTRY_PER_BLOCK)
+		return;
+
+	if (!sync && !available_free_memory(sbi, FREE_NIDS))
 		return;
 
 	/* readahead nat pages to be scanned */
@@ -1830,7 +1854,7 @@ void build_free_nids(struct f2fs_sb_info *sbi)
 		if (addr == NULL_ADDR)
 			add_free_nid(sbi, nid, true);
 		else
-			remove_free_nid(nm_i, nid);
+			remove_free_nid(sbi, nid);
 	}
 	up_read(&curseg->journal_rwsem);
 	up_read(&nm_i->nat_tree_lock);
@@ -1839,6 +1863,13 @@ void build_free_nids(struct f2fs_sb_info *sbi)
 					nm_i->ra_nid_pages, META_NAT, false);
 }
 
+void build_free_nids(struct f2fs_sb_info *sbi, bool sync)
+{
+	mutex_lock(&NM_I(sbi)->build_lock);
+	__build_free_nids(sbi, sync);
+	mutex_unlock(&NM_I(sbi)->build_lock);
+}
+
 /*
  * If this function returns success, caller can obtain a new nid
  * from second parameter of this function.
@@ -1853,31 +1884,31 @@ retry:
 	if (time_to_inject(sbi, FAULT_ALLOC_NID))
 		return false;
 #endif
-	if (unlikely(sbi->total_valid_node_count + 1 > nm_i->available_nids))
-		return false;
+	spin_lock(&nm_i->nid_list_lock);
 
-	spin_lock(&nm_i->free_nid_list_lock);
+	if (unlikely(nm_i->available_nids == 0)) {
+		spin_unlock(&nm_i->nid_list_lock);
+		return false;
+	}
 
 	/* We should not use stale free nids created by build_free_nids */
-	if (nm_i->fcnt && !on_build_free_nids(nm_i)) {
-		f2fs_bug_on(sbi, list_empty(&nm_i->free_nid_list));
-		list_for_each_entry(i, &nm_i->free_nid_list, list)
-			if (i->state == NID_NEW)
-				break;
-
-		f2fs_bug_on(sbi, i->state != NID_NEW);
+	if (nm_i->nid_cnt[FREE_NID_LIST] && !on_build_free_nids(nm_i)) {
+		f2fs_bug_on(sbi, list_empty(&nm_i->nid_list[FREE_NID_LIST]));
+		i = list_first_entry(&nm_i->nid_list[FREE_NID_LIST],
+					struct free_nid, list);
 		*nid = i->nid;
+
+		__remove_nid_from_list(sbi, i, FREE_NID_LIST, true);
 		i->state = NID_ALLOC;
-		nm_i->fcnt--;
-		spin_unlock(&nm_i->free_nid_list_lock);
+		__insert_nid_to_list(sbi, i, ALLOC_NID_LIST, false);
+		nm_i->available_nids--;
+		spin_unlock(&nm_i->nid_list_lock);
 		return true;
 	}
-	spin_unlock(&nm_i->free_nid_list_lock);
+	spin_unlock(&nm_i->nid_list_lock);
 
 	/* Let's scan nat pages and its caches to get free nids */
-	mutex_lock(&nm_i->build_lock);
-	build_free_nids(sbi);
-	mutex_unlock(&nm_i->build_lock);
+	build_free_nids(sbi, true);
 	goto retry;
 }
 
@@ -1889,11 +1920,11 @@ void alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid)
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct free_nid *i;
 
-	spin_lock(&nm_i->free_nid_list_lock);
+	spin_lock(&nm_i->nid_list_lock);
 	i = __lookup_free_nid_list(nm_i, nid);
-	f2fs_bug_on(sbi, !i || i->state != NID_ALLOC);
-	__del_from_free_nid_list(nm_i, i);
-	spin_unlock(&nm_i->free_nid_list_lock);
+	f2fs_bug_on(sbi, !i);
+	__remove_nid_from_list(sbi, i, ALLOC_NID_LIST, false);
+	spin_unlock(&nm_i->nid_list_lock);
 
 	kmem_cache_free(free_nid_slab, i);
 }
@@ -1910,17 +1941,22 @@ void alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid)
 	if (!nid)
 		return;
 
-	spin_lock(&nm_i->free_nid_list_lock);
+	spin_lock(&nm_i->nid_list_lock);
 	i = __lookup_free_nid_list(nm_i, nid);
-	f2fs_bug_on(sbi, !i || i->state != NID_ALLOC);
+	f2fs_bug_on(sbi, !i);
+
 	if (!available_free_memory(sbi, FREE_NIDS)) {
-		__del_from_free_nid_list(nm_i, i);
+		__remove_nid_from_list(sbi, i, ALLOC_NID_LIST, false);
 		need_free = true;
 	} else {
+		__remove_nid_from_list(sbi, i, ALLOC_NID_LIST, true);
 		i->state = NID_NEW;
-		nm_i->fcnt++;
+		__insert_nid_to_list(sbi, i, FREE_NID_LIST, false);
 	}
-	spin_unlock(&nm_i->free_nid_list_lock);
+
+	nm_i->available_nids++;
+
+	spin_unlock(&nm_i->nid_list_lock);
 
 	if (need_free)
 		kmem_cache_free(free_nid_slab, i);
@@ -1932,24 +1968,24 @@ int try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink)
 	struct free_nid *i, *next;
 	int nr = nr_shrink;
 
-	if (nm_i->fcnt <= MAX_FREE_NIDS)
+	if (nm_i->nid_cnt[FREE_NID_LIST] <= MAX_FREE_NIDS)
 		return 0;
 
 	if (!mutex_trylock(&nm_i->build_lock))
 		return 0;
 
-	spin_lock(&nm_i->free_nid_list_lock);
-	list_for_each_entry_safe(i, next, &nm_i->free_nid_list, list) {
-		if (nr_shrink <= 0 || nm_i->fcnt <= MAX_FREE_NIDS)
+	spin_lock(&nm_i->nid_list_lock);
+	list_for_each_entry_safe(i, next, &nm_i->nid_list[FREE_NID_LIST],
+									list) {
+		if (nr_shrink <= 0 ||
+				nm_i->nid_cnt[FREE_NID_LIST] <= MAX_FREE_NIDS)
 			break;
-		if (i->state == NID_ALLOC)
-			continue;
-		__del_from_free_nid_list(nm_i, i);
+
+		__remove_nid_from_list(sbi, i, FREE_NID_LIST, false);
 		kmem_cache_free(free_nid_slab, i);
-		nm_i->fcnt--;
 		nr_shrink--;
 	}
-	spin_unlock(&nm_i->free_nid_list_lock);
+	spin_unlock(&nm_i->nid_list_lock);
 	mutex_unlock(&nm_i->build_lock);
 
 	return nr - nr_shrink;
@@ -2005,7 +2041,7 @@ recover_xnid:
 	if (unlikely(!inc_valid_node_count(sbi, inode)))
 		f2fs_bug_on(sbi, 1);
 
-	remove_free_nid(NM_I(sbi), new_xnid);
+	remove_free_nid(sbi, new_xnid);
 	get_node_info(sbi, new_xnid, &ni);
 	ni.ino = inode->i_ino;
 	set_node_addr(sbi, &ni, NEW_ADDR, false);
@@ -2035,7 +2071,7 @@ retry:
 	}
 
 	/* Should not use this inode from free nid list */
-	remove_free_nid(NM_I(sbi), ino);
+	remove_free_nid(sbi, ino);
 
 	if (!PageUptodate(ipage))
 		SetPageUptodate(ipage);
@@ -2069,7 +2105,6 @@ int restore_node_summary(struct f2fs_sb_info *sbi,
 	struct f2fs_node *rn;
 	struct f2fs_summary *sum_entry;
 	block_t addr;
-	int bio_blocks = MAX_BIO_BLOCKS(sbi);
 	int i, idx, last_offset, nrpages;
 
 	/* scan the node segment */
@@ -2078,7 +2113,7 @@ int restore_node_summary(struct f2fs_sb_info *sbi,
 	sum_entry = &sum->entries[0];
 
 	for (i = 0; i < last_offset; i += nrpages, addr += nrpages) {
-		nrpages = min(last_offset - i, bio_blocks);
+		nrpages = min(last_offset - i, BIO_MAX_PAGES);
 
 		/* readahead node pages */
 		ra_meta_pages(sbi, addr, nrpages, META_POR, true);
@@ -2120,6 +2155,19 @@ static void remove_nats_in_journal(struct f2fs_sb_info *sbi)
 			ne = grab_nat_entry(nm_i, nid);
 			node_info_from_raw_nat(&ne->ni, &raw_ne);
 		}
+
+		/*
+		 * if a free nat in journal has not been used after last
+		 * checkpoint, we should remove it from available nids,
+		 * since later we will add it again.
+		 */
+		if (!get_nat_flag(ne, IS_DIRTY) &&
+				le32_to_cpu(raw_ne.block_addr) == NULL_ADDR) {
+			spin_lock(&nm_i->nid_list_lock);
+			nm_i->available_nids--;
+			spin_unlock(&nm_i->nid_list_lock);
+		}
+
 		__set_nat_cache_dirty(nm_i, ne);
 	}
 	update_nats_in_cursum(journal, -i);
@@ -2192,8 +2240,12 @@ static void __flush_nat_entry_set(struct f2fs_sb_info *sbi,
 		raw_nat_from_node_info(raw_ne, &ne->ni);
 		nat_reset_flag(ne);
 		__clear_nat_cache_dirty(NM_I(sbi), ne);
-		if (nat_get_blkaddr(ne) == NULL_ADDR)
+		if (nat_get_blkaddr(ne) == NULL_ADDR) {
 			add_free_nid(sbi, nid, false);
+			spin_lock(&NM_I(sbi)->nid_list_lock);
+			NM_I(sbi)->available_nids++;
+			spin_unlock(&NM_I(sbi)->nid_list_lock);
+		}
 	}
 
 	if (to_journal)
@@ -2268,21 +2320,24 @@ static int init_node_manager(struct f2fs_sb_info *sbi)
 	nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nat_blocks;
 
 	/* not used nids: 0, node, meta, (and root counted as valid node) */
-	nm_i->available_nids = nm_i->max_nid - F2FS_RESERVED_NODE_NUM;
-	nm_i->fcnt = 0;
+	nm_i->available_nids = nm_i->max_nid - sbi->total_valid_node_count -
+							F2FS_RESERVED_NODE_NUM;
+	nm_i->nid_cnt[FREE_NID_LIST] = 0;
+	nm_i->nid_cnt[ALLOC_NID_LIST] = 0;
 	nm_i->nat_cnt = 0;
 	nm_i->ram_thresh = DEF_RAM_THRESHOLD;
 	nm_i->ra_nid_pages = DEF_RA_NID_PAGES;
 	nm_i->dirty_nats_ratio = DEF_DIRTY_NAT_RATIO_THRESHOLD;
 
 	INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC);
-	INIT_LIST_HEAD(&nm_i->free_nid_list);
+	INIT_LIST_HEAD(&nm_i->nid_list[FREE_NID_LIST]);
+	INIT_LIST_HEAD(&nm_i->nid_list[ALLOC_NID_LIST]);
 	INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO);
 	INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO);
 	INIT_LIST_HEAD(&nm_i->nat_entries);
 
 	mutex_init(&nm_i->build_lock);
-	spin_lock_init(&nm_i->free_nid_list_lock);
+	spin_lock_init(&nm_i->nid_list_lock);
 	init_rwsem(&nm_i->nat_tree_lock);
 
 	nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);
@@ -2310,7 +2365,7 @@ int build_node_manager(struct f2fs_sb_info *sbi)
 	if (err)
 		return err;
 
-	build_free_nids(sbi);
+	build_free_nids(sbi, true);
 	return 0;
 }
 
@@ -2327,17 +2382,18 @@ void destroy_node_manager(struct f2fs_sb_info *sbi)
 		return;
 
 	/* destroy free nid list */
-	spin_lock(&nm_i->free_nid_list_lock);
-	list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) {
-		f2fs_bug_on(sbi, i->state == NID_ALLOC);
-		__del_from_free_nid_list(nm_i, i);
-		nm_i->fcnt--;
-		spin_unlock(&nm_i->free_nid_list_lock);
+	spin_lock(&nm_i->nid_list_lock);
+	list_for_each_entry_safe(i, next_i, &nm_i->nid_list[FREE_NID_LIST],
+									list) {
+		__remove_nid_from_list(sbi, i, FREE_NID_LIST, false);
+		spin_unlock(&nm_i->nid_list_lock);
 		kmem_cache_free(free_nid_slab, i);
-		spin_lock(&nm_i->free_nid_list_lock);
+		spin_lock(&nm_i->nid_list_lock);
 	}
-	f2fs_bug_on(sbi, nm_i->fcnt);
-	spin_unlock(&nm_i->free_nid_list_lock);
+	f2fs_bug_on(sbi, nm_i->nid_cnt[FREE_NID_LIST]);
+	f2fs_bug_on(sbi, nm_i->nid_cnt[ALLOC_NID_LIST]);
+	f2fs_bug_on(sbi, !list_empty(&nm_i->nid_list[ALLOC_NID_LIST]));
+	spin_unlock(&nm_i->nid_list_lock);
 
 	/* destroy nat cache */
 	down_write(&nm_i->nat_tree_lock);
diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h
index 868bec65e51c..e7997e240366 100644
--- a/fs/f2fs/node.h
+++ b/fs/f2fs/node.h
@@ -169,14 +169,15 @@ static inline void next_free_nid(struct f2fs_sb_info *sbi, nid_t *nid)
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct free_nid *fnid;
 
-	spin_lock(&nm_i->free_nid_list_lock);
-	if (nm_i->fcnt <= 0) {
-		spin_unlock(&nm_i->free_nid_list_lock);
+	spin_lock(&nm_i->nid_list_lock);
+	if (nm_i->nid_cnt[FREE_NID_LIST] <= 0) {
+		spin_unlock(&nm_i->nid_list_lock);
 		return;
 	}
-	fnid = list_entry(nm_i->free_nid_list.next, struct free_nid, list);
+	fnid = list_entry(nm_i->nid_list[FREE_NID_LIST].next,
+						struct free_nid, list);
 	*nid = fnid->nid;
-	spin_unlock(&nm_i->free_nid_list_lock);
+	spin_unlock(&nm_i->nid_list_lock);
 }
 
 /*
@@ -313,7 +314,7 @@ static inline bool is_recoverable_dnode(struct page *page)
 				((unsigned char *)ckpt + crc_offset)));
 		cp_ver |= (crc << 32);
 	}
-	return cpu_to_le64(cp_ver) == cpver_of_node(page);
+	return cp_ver == cpver_of_node(page);
 }
 
 /*
diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c
index 2fc84a991325..981a9584b62f 100644
--- a/fs/f2fs/recovery.c
+++ b/fs/f2fs/recovery.c
@@ -180,13 +180,15 @@ static void recover_inode(struct inode *inode, struct page *page)
 
 	inode->i_mode = le16_to_cpu(raw->i_mode);
 	f2fs_i_size_write(inode, le64_to_cpu(raw->i_size));
-	inode->i_atime.tv_sec = le64_to_cpu(raw->i_mtime);
+	inode->i_atime.tv_sec = le64_to_cpu(raw->i_atime);
 	inode->i_ctime.tv_sec = le64_to_cpu(raw->i_ctime);
 	inode->i_mtime.tv_sec = le64_to_cpu(raw->i_mtime);
-	inode->i_atime.tv_nsec = le32_to_cpu(raw->i_mtime_nsec);
+	inode->i_atime.tv_nsec = le32_to_cpu(raw->i_atime_nsec);
 	inode->i_ctime.tv_nsec = le32_to_cpu(raw->i_ctime_nsec);
 	inode->i_mtime.tv_nsec = le32_to_cpu(raw->i_mtime_nsec);
 
+	F2FS_I(inode)->i_advise = raw->i_advise;
+
 	if (file_enc_name(inode))
 		name = "<encrypted>";
 	else
@@ -196,32 +198,6 @@ static void recover_inode(struct inode *inode, struct page *page)
 			ino_of_node(page), name);
 }
 
-static bool is_same_inode(struct inode *inode, struct page *ipage)
-{
-	struct f2fs_inode *ri = F2FS_INODE(ipage);
-	struct timespec disk;
-
-	if (!IS_INODE(ipage))
-		return true;
-
-	disk.tv_sec = le64_to_cpu(ri->i_ctime);
-	disk.tv_nsec = le32_to_cpu(ri->i_ctime_nsec);
-	if (timespec_compare(&inode->i_ctime, &disk) > 0)
-		return false;
-
-	disk.tv_sec = le64_to_cpu(ri->i_atime);
-	disk.tv_nsec = le32_to_cpu(ri->i_atime_nsec);
-	if (timespec_compare(&inode->i_atime, &disk) > 0)
-		return false;
-
-	disk.tv_sec = le64_to_cpu(ri->i_mtime);
-	disk.tv_nsec = le32_to_cpu(ri->i_mtime_nsec);
-	if (timespec_compare(&inode->i_mtime, &disk) > 0)
-		return false;
-
-	return true;
-}
-
 static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head)
 {
 	struct curseg_info *curseg;
@@ -248,10 +224,7 @@ static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head)
 			goto next;
 
 		entry = get_fsync_inode(head, ino_of_node(page));
-		if (entry) {
-			if (!is_same_inode(entry->inode, page))
-				goto next;
-		} else {
+		if (!entry) {
 			if (IS_INODE(page) && is_dent_dnode(page)) {
 				err = recover_inode_page(sbi, page);
 				if (err)
@@ -454,7 +427,8 @@ retry_dn:
 			continue;
 		}
 
-		if ((start + 1) << PAGE_SHIFT > i_size_read(inode))
+		if (!file_keep_isize(inode) &&
+				(i_size_read(inode) <= (start << PAGE_SHIFT)))
 			f2fs_i_size_write(inode, (start + 1) << PAGE_SHIFT);
 
 		/*
@@ -507,8 +481,10 @@ err:
 	f2fs_put_dnode(&dn);
 out:
 	f2fs_msg(sbi->sb, KERN_NOTICE,
-		"recover_data: ino = %lx, recovered = %d blocks, err = %d",
-		inode->i_ino, recovered, err);
+		"recover_data: ino = %lx (i_size: %s) recovered = %d, err = %d",
+		inode->i_ino,
+		file_keep_isize(inode) ? "keep" : "recover",
+		recovered, err);
 	return err;
 }
 
diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c
index fc886f008449..0738f48293cc 100644
--- a/fs/f2fs/segment.c
+++ b/fs/f2fs/segment.c
@@ -259,7 +259,7 @@ static int __commit_inmem_pages(struct inode *inode,
 		.sbi = sbi,
 		.type = DATA,
 		.op = REQ_OP_WRITE,
-		.op_flags = WRITE_SYNC | REQ_PRIO,
+		.op_flags = REQ_SYNC | REQ_PRIO,
 		.encrypted_page = NULL,
 	};
 	bool submit_bio = false;
@@ -274,8 +274,10 @@ static int __commit_inmem_pages(struct inode *inode,
 
 			set_page_dirty(page);
 			f2fs_wait_on_page_writeback(page, DATA, true);
-			if (clear_page_dirty_for_io(page))
+			if (clear_page_dirty_for_io(page)) {
 				inode_dec_dirty_pages(inode);
+				remove_dirty_inode(inode);
+			}
 
 			fio.page = page;
 			err = do_write_data_page(&fio);
@@ -287,7 +289,6 @@ static int __commit_inmem_pages(struct inode *inode,
 			/* record old blkaddr for revoking */
 			cur->old_addr = fio.old_blkaddr;
 
-			clear_cold_data(page);
 			submit_bio = true;
 		}
 		unlock_page(page);
@@ -363,7 +364,7 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
 	 */
 	if (has_not_enough_free_secs(sbi, 0, 0)) {
 		mutex_lock(&sbi->gc_mutex);
-		f2fs_gc(sbi, false);
+		f2fs_gc(sbi, false, false);
 	}
 }
 
@@ -380,14 +381,17 @@ void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
 	if (!available_free_memory(sbi, FREE_NIDS))
 		try_to_free_nids(sbi, MAX_FREE_NIDS);
 	else
-		build_free_nids(sbi);
+		build_free_nids(sbi, false);
+
+	if (!is_idle(sbi))
+		return;
 
 	/* checkpoint is the only way to shrink partial cached entries */
 	if (!available_free_memory(sbi, NAT_ENTRIES) ||
 			!available_free_memory(sbi, INO_ENTRIES) ||
 			excess_prefree_segs(sbi) ||
 			excess_dirty_nats(sbi) ||
-			(is_idle(sbi) && f2fs_time_over(sbi, CP_TIME))) {
+			f2fs_time_over(sbi, CP_TIME)) {
 		if (test_opt(sbi, DATA_FLUSH)) {
 			struct blk_plug plug;
 
@@ -400,6 +404,33 @@ void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
 	}
 }
 
+static int __submit_flush_wait(struct block_device *bdev)
+{
+	struct bio *bio = f2fs_bio_alloc(0);
+	int ret;
+
+	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
+	bio->bi_bdev = bdev;
+	ret = submit_bio_wait(bio);
+	bio_put(bio);
+	return ret;
+}
+
+static int submit_flush_wait(struct f2fs_sb_info *sbi)
+{
+	int ret = __submit_flush_wait(sbi->sb->s_bdev);
+	int i;
+
+	if (sbi->s_ndevs && !ret) {
+		for (i = 1; i < sbi->s_ndevs; i++) {
+			ret = __submit_flush_wait(FDEV(i).bdev);
+			if (ret)
+				break;
+		}
+	}
+	return ret;
+}
+
 static int issue_flush_thread(void *data)
 {
 	struct f2fs_sb_info *sbi = data;
@@ -410,25 +441,18 @@ repeat:
 		return 0;
 
 	if (!llist_empty(&fcc->issue_list)) {
-		struct bio *bio;
 		struct flush_cmd *cmd, *next;
 		int ret;
 
-		bio = f2fs_bio_alloc(0);
-
 		fcc->dispatch_list = llist_del_all(&fcc->issue_list);
 		fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
 
-		bio->bi_bdev = sbi->sb->s_bdev;
-		bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_FLUSH);
-		ret = submit_bio_wait(bio);
-
+		ret = submit_flush_wait(sbi);
 		llist_for_each_entry_safe(cmd, next,
 					  fcc->dispatch_list, llnode) {
 			cmd->ret = ret;
 			complete(&cmd->wait);
 		}
-		bio_put(bio);
 		fcc->dispatch_list = NULL;
 	}
 
@@ -449,15 +473,11 @@ int f2fs_issue_flush(struct f2fs_sb_info *sbi)
 		return 0;
 
 	if (!test_opt(sbi, FLUSH_MERGE) || !atomic_read(&fcc->submit_flush)) {
-		struct bio *bio = f2fs_bio_alloc(0);
 		int ret;
 
 		atomic_inc(&fcc->submit_flush);
-		bio->bi_bdev = sbi->sb->s_bdev;
-		bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_FLUSH);
-		ret = submit_bio_wait(bio);
+		ret = submit_flush_wait(sbi);
 		atomic_dec(&fcc->submit_flush);
-		bio_put(bio);
 		return ret;
 	}
 
@@ -469,8 +489,13 @@ int f2fs_issue_flush(struct f2fs_sb_info *sbi)
 	if (!fcc->dispatch_list)
 		wake_up(&fcc->flush_wait_queue);
 
-	wait_for_completion(&cmd.wait);
-	atomic_dec(&fcc->submit_flush);
+	if (fcc->f2fs_issue_flush) {
+		wait_for_completion(&cmd.wait);
+		atomic_dec(&fcc->submit_flush);
+	} else {
+		llist_del_all(&fcc->issue_list);
+		atomic_set(&fcc->submit_flush, 0);
+	}
 
 	return cmd.ret;
 }
@@ -481,6 +506,11 @@ int create_flush_cmd_control(struct f2fs_sb_info *sbi)
 	struct flush_cmd_control *fcc;
 	int err = 0;
 
+	if (SM_I(sbi)->cmd_control_info) {
+		fcc = SM_I(sbi)->cmd_control_info;
+		goto init_thread;
+	}
+
 	fcc = kzalloc(sizeof(struct flush_cmd_control), GFP_KERNEL);
 	if (!fcc)
 		return -ENOMEM;
@@ -488,6 +518,7 @@ int create_flush_cmd_control(struct f2fs_sb_info *sbi)
 	init_waitqueue_head(&fcc->flush_wait_queue);
 	init_llist_head(&fcc->issue_list);
 	SM_I(sbi)->cmd_control_info = fcc;
+init_thread:
 	fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
 				"f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
 	if (IS_ERR(fcc->f2fs_issue_flush)) {
@@ -500,14 +531,20 @@ int create_flush_cmd_control(struct f2fs_sb_info *sbi)
 	return err;
 }
 
-void destroy_flush_cmd_control(struct f2fs_sb_info *sbi)
+void destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
 {
 	struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
 
-	if (fcc && fcc->f2fs_issue_flush)
-		kthread_stop(fcc->f2fs_issue_flush);
-	kfree(fcc);
-	SM_I(sbi)->cmd_control_info = NULL;
+	if (fcc && fcc->f2fs_issue_flush) {
+		struct task_struct *flush_thread = fcc->f2fs_issue_flush;
+
+		fcc->f2fs_issue_flush = NULL;
+		kthread_stop(flush_thread);
+	}
+	if (free) {
+		kfree(fcc);
+		SM_I(sbi)->cmd_control_info = NULL;
+	}
 }
 
 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
@@ -633,15 +670,23 @@ static void f2fs_submit_bio_wait_endio(struct bio *bio)
 }
 
 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
-int __f2fs_issue_discard_async(struct f2fs_sb_info *sbi, sector_t sector,
-		sector_t nr_sects, gfp_t gfp_mask, unsigned long flags)
+static int __f2fs_issue_discard_async(struct f2fs_sb_info *sbi,
+		struct block_device *bdev, block_t blkstart, block_t blklen)
 {
-	struct block_device *bdev = sbi->sb->s_bdev;
 	struct bio *bio = NULL;
 	int err;
 
-	err = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, flags,
-			&bio);
+	trace_f2fs_issue_discard(sbi->sb, blkstart, blklen);
+
+	if (sbi->s_ndevs) {
+		int devi = f2fs_target_device_index(sbi, blkstart);
+
+		blkstart -= FDEV(devi).start_blk;
+	}
+	err = __blkdev_issue_discard(bdev,
+				SECTOR_FROM_BLOCK(blkstart),
+				SECTOR_FROM_BLOCK(blklen),
+				GFP_NOFS, 0, &bio);
 	if (!err && bio) {
 		struct bio_entry *be = __add_bio_entry(sbi, bio);
 
@@ -654,24 +699,101 @@ int __f2fs_issue_discard_async(struct f2fs_sb_info *sbi, sector_t sector,
 	return err;
 }
 
+#ifdef CONFIG_BLK_DEV_ZONED
+static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
+		struct block_device *bdev, block_t blkstart, block_t blklen)
+{
+	sector_t nr_sects = SECTOR_FROM_BLOCK(blklen);
+	sector_t sector;
+	int devi = 0;
+
+	if (sbi->s_ndevs) {
+		devi = f2fs_target_device_index(sbi, blkstart);
+		blkstart -= FDEV(devi).start_blk;
+	}
+	sector = SECTOR_FROM_BLOCK(blkstart);
+
+	if (sector & (bdev_zone_size(bdev) - 1) ||
+				nr_sects != bdev_zone_size(bdev)) {
+		f2fs_msg(sbi->sb, KERN_INFO,
+			"(%d) %s: Unaligned discard attempted (block %x + %x)",
+			devi, sbi->s_ndevs ? FDEV(devi).path: "",
+			blkstart, blklen);
+		return -EIO;
+	}
+
+	/*
+	 * We need to know the type of the zone: for conventional zones,
+	 * use regular discard if the drive supports it. For sequential
+	 * zones, reset the zone write pointer.
+	 */
+	switch (get_blkz_type(sbi, bdev, blkstart)) {
+
+	case BLK_ZONE_TYPE_CONVENTIONAL:
+		if (!blk_queue_discard(bdev_get_queue(bdev)))
+			return 0;
+		return __f2fs_issue_discard_async(sbi, bdev, blkstart, blklen);
+	case BLK_ZONE_TYPE_SEQWRITE_REQ:
+	case BLK_ZONE_TYPE_SEQWRITE_PREF:
+		trace_f2fs_issue_reset_zone(sbi->sb, blkstart);
+		return blkdev_reset_zones(bdev, sector,
+					  nr_sects, GFP_NOFS);
+	default:
+		/* Unknown zone type: broken device ? */
+		return -EIO;
+	}
+}
+#endif
+
+static int __issue_discard_async(struct f2fs_sb_info *sbi,
+		struct block_device *bdev, block_t blkstart, block_t blklen)
+{
+#ifdef CONFIG_BLK_DEV_ZONED
+	if (f2fs_sb_mounted_blkzoned(sbi->sb) &&
+				bdev_zoned_model(bdev) != BLK_ZONED_NONE)
+		return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
+#endif
+	return __f2fs_issue_discard_async(sbi, bdev, blkstart, blklen);
+}
+
 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
 				block_t blkstart, block_t blklen)
 {
-	sector_t start = SECTOR_FROM_BLOCK(blkstart);
-	sector_t len = SECTOR_FROM_BLOCK(blklen);
+	sector_t start = blkstart, len = 0;
+	struct block_device *bdev;
 	struct seg_entry *se;
 	unsigned int offset;
 	block_t i;
+	int err = 0;
+
+	bdev = f2fs_target_device(sbi, blkstart, NULL);
+
+	for (i = blkstart; i < blkstart + blklen; i++, len++) {
+		if (i != start) {
+			struct block_device *bdev2 =
+				f2fs_target_device(sbi, i, NULL);
+
+			if (bdev2 != bdev) {
+				err = __issue_discard_async(sbi, bdev,
+						start, len);
+				if (err)
+					return err;
+				bdev = bdev2;
+				start = i;
+				len = 0;
+			}
+		}
 
-	for (i = blkstart; i < blkstart + blklen; i++) {
 		se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
 		offset = GET_BLKOFF_FROM_SEG0(sbi, i);
 
 		if (!f2fs_test_and_set_bit(offset, se->discard_map))
 			sbi->discard_blks--;
 	}
-	trace_f2fs_issue_discard(sbi->sb, blkstart, blklen);
-	return __f2fs_issue_discard_async(sbi, start, len, GFP_NOFS, 0);
+
+	if (len)
+		err = __issue_discard_async(sbi, bdev, start, len);
+	return err;
 }
 
 static void __add_discard_entry(struct f2fs_sb_info *sbi,
@@ -1296,25 +1418,21 @@ static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
 	stat_inc_seg_type(sbi, curseg);
 }
 
-static void __allocate_new_segments(struct f2fs_sb_info *sbi, int type)
-{
-	struct curseg_info *curseg = CURSEG_I(sbi, type);
-	unsigned int old_segno;
-
-	old_segno = curseg->segno;
-	SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
-	locate_dirty_segment(sbi, old_segno);
-}
-
 void allocate_new_segments(struct f2fs_sb_info *sbi)
 {
+	struct curseg_info *curseg;
+	unsigned int old_segno;
 	int i;
 
 	if (test_opt(sbi, LFS))
 		return;
 
-	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
-		__allocate_new_segments(sbi, i);
+	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
+		curseg = CURSEG_I(sbi, i);
+		old_segno = curseg->segno;
+		SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true);
+		locate_dirty_segment(sbi, old_segno);
+	}
 }
 
 static const struct segment_allocation default_salloc_ops = {
@@ -1448,21 +1566,11 @@ void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
 		struct f2fs_summary *sum, int type)
 {
 	struct sit_info *sit_i = SIT_I(sbi);
-	struct curseg_info *curseg;
-	bool direct_io = (type == CURSEG_DIRECT_IO);
-
-	type = direct_io ? CURSEG_WARM_DATA : type;
-
-	curseg = CURSEG_I(sbi, type);
+	struct curseg_info *curseg = CURSEG_I(sbi, type);
 
 	mutex_lock(&curseg->curseg_mutex);
 	mutex_lock(&sit_i->sentry_lock);
 
-	/* direct_io'ed data is aligned to the segment for better performance */
-	if (direct_io && curseg->next_blkoff &&
-				!has_not_enough_free_secs(sbi, 0, 0))
-		__allocate_new_segments(sbi, type);
-
 	*new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
 
 	/*
@@ -1515,7 +1623,7 @@ void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
 		.sbi = sbi,
 		.type = META,
 		.op = REQ_OP_WRITE,
-		.op_flags = WRITE_SYNC | REQ_META | REQ_PRIO,
+		.op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
 		.old_blkaddr = page->index,
 		.new_blkaddr = page->index,
 		.page = page,
@@ -2166,7 +2274,6 @@ out:
 static int build_sit_info(struct f2fs_sb_info *sbi)
 {
 	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
-	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
 	struct sit_info *sit_i;
 	unsigned int sit_segs, start;
 	char *src_bitmap, *dst_bitmap;
@@ -2233,7 +2340,7 @@ static int build_sit_info(struct f2fs_sb_info *sbi)
 
 	sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
 	sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
-	sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count);
+	sit_i->written_valid_blocks = 0;
 	sit_i->sit_bitmap = dst_bitmap;
 	sit_i->bitmap_size = bitmap_size;
 	sit_i->dirty_sentries = 0;
@@ -2315,10 +2422,10 @@ static void build_sit_entries(struct f2fs_sb_info *sbi)
 	int sit_blk_cnt = SIT_BLK_CNT(sbi);
 	unsigned int i, start, end;
 	unsigned int readed, start_blk = 0;
-	int nrpages = MAX_BIO_BLOCKS(sbi) * 8;
 
 	do {
-		readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT, true);
+		readed = ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES,
+							META_SIT, true);
 
 		start = start_blk * sit_i->sents_per_block;
 		end = (start_blk + readed) * sit_i->sents_per_block;
@@ -2387,6 +2494,9 @@ static void init_free_segmap(struct f2fs_sb_info *sbi)
 		struct seg_entry *sentry = get_seg_entry(sbi, start);
 		if (!sentry->valid_blocks)
 			__set_free(sbi, start);
+		else
+			SIT_I(sbi)->written_valid_blocks +=
+						sentry->valid_blocks;
 	}
 
 	/* set use the current segments */
@@ -2645,7 +2755,7 @@ void destroy_segment_manager(struct f2fs_sb_info *sbi)
 
 	if (!sm_info)
 		return;
-	destroy_flush_cmd_control(sbi);
+	destroy_flush_cmd_control(sbi, true);
 	destroy_dirty_segmap(sbi);
 	destroy_curseg(sbi);
 	destroy_free_segmap(sbi);
diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h
index fecb856ad874..9d44ce83acb2 100644
--- a/fs/f2fs/segment.h
+++ b/fs/f2fs/segment.h
@@ -18,6 +18,8 @@
 #define DEF_RECLAIM_PREFREE_SEGMENTS	5	/* 5% over total segments */
 #define DEF_MAX_RECLAIM_PREFREE_SEGMENTS	4096	/* 8GB in maximum */
 
+#define F2FS_MIN_SEGMENTS	9 /* SB + 2 (CP + SIT + NAT) + SSA + MAIN */
+
 /* L: Logical segment # in volume, R: Relative segment # in main area */
 #define GET_L2R_SEGNO(free_i, segno)	(segno - free_i->start_segno)
 #define GET_R2L_SEGNO(free_i, segno)	(segno + free_i->start_segno)
@@ -102,8 +104,6 @@
 	(((sector_t)blk_addr) << F2FS_LOG_SECTORS_PER_BLOCK)
 #define SECTOR_TO_BLOCK(sectors)					\
 	(sectors >> F2FS_LOG_SECTORS_PER_BLOCK)
-#define MAX_BIO_BLOCKS(sbi)						\
-	((int)min((int)max_hw_blocks(sbi), BIO_MAX_PAGES))
 
 /*
  * indicate a block allocation direction: RIGHT and LEFT.
@@ -471,11 +471,12 @@ static inline bool need_SSR(struct f2fs_sb_info *sbi)
 {
 	int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
 	int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
+	int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
 
 	if (test_opt(sbi, LFS))
 		return false;
 
-	return free_sections(sbi) <= (node_secs + 2 * dent_secs +
+	return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
 						reserved_sections(sbi) + 1);
 }
 
@@ -484,14 +485,14 @@ static inline bool has_not_enough_free_secs(struct f2fs_sb_info *sbi,
 {
 	int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
 	int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
-
-	node_secs += get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
+	int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
 
 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 		return false;
 
 	return (free_sections(sbi) + freed) <=
-		(node_secs + 2 * dent_secs + reserved_sections(sbi) + needed);
+		(node_secs + 2 * dent_secs + imeta_secs +
+		reserved_sections(sbi) + needed);
 }
 
 static inline bool excess_prefree_segs(struct f2fs_sb_info *sbi)
@@ -695,13 +696,6 @@ static inline bool sec_usage_check(struct f2fs_sb_info *sbi, unsigned int secno)
 	return false;
 }
 
-static inline unsigned int max_hw_blocks(struct f2fs_sb_info *sbi)
-{
-	struct block_device *bdev = sbi->sb->s_bdev;
-	struct request_queue *q = bdev_get_queue(bdev);
-	return SECTOR_TO_BLOCK(queue_max_sectors(q));
-}
-
 /*
  * It is very important to gather dirty pages and write at once, so that we can
  * submit a big bio without interfering other data writes.
@@ -719,7 +713,7 @@ static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type)
 	else if (type == NODE)
 		return 8 * sbi->blocks_per_seg;
 	else if (type == META)
-		return 8 * MAX_BIO_BLOCKS(sbi);
+		return 8 * BIO_MAX_PAGES;
 	else
 		return 0;
 }
@@ -736,11 +730,9 @@ static inline long nr_pages_to_write(struct f2fs_sb_info *sbi, int type,
 		return 0;
 
 	nr_to_write = wbc->nr_to_write;
-
+	desired = BIO_MAX_PAGES;
 	if (type == NODE)
-		desired = 2 * max_hw_blocks(sbi);
-	else
-		desired = MAX_BIO_BLOCKS(sbi);
+		desired <<= 1;
 
 	wbc->nr_to_write = desired;
 	return desired - nr_to_write;
diff --git a/fs/f2fs/shrinker.c b/fs/f2fs/shrinker.c
index 46c915425923..5c60fc28ec75 100644
--- a/fs/f2fs/shrinker.c
+++ b/fs/f2fs/shrinker.c
@@ -21,14 +21,16 @@ static unsigned int shrinker_run_no;
 
 static unsigned long __count_nat_entries(struct f2fs_sb_info *sbi)
 {
-	return NM_I(sbi)->nat_cnt - NM_I(sbi)->dirty_nat_cnt;
+	long count = NM_I(sbi)->nat_cnt - NM_I(sbi)->dirty_nat_cnt;
+
+	return count > 0 ? count : 0;
 }
 
 static unsigned long __count_free_nids(struct f2fs_sb_info *sbi)
 {
-	if (NM_I(sbi)->fcnt > MAX_FREE_NIDS)
-		return NM_I(sbi)->fcnt - MAX_FREE_NIDS;
-	return 0;
+	long count = NM_I(sbi)->nid_cnt[FREE_NID_LIST] - MAX_FREE_NIDS;
+
+	return count > 0 ? count : 0;
 }
 
 static unsigned long __count_extent_cache(struct f2fs_sb_info *sbi)
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index 6132b4ce4e4c..702638e21c76 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -412,14 +412,20 @@ static int parse_options(struct super_block *sb, char *options)
 			q = bdev_get_queue(sb->s_bdev);
 			if (blk_queue_discard(q)) {
 				set_opt(sbi, DISCARD);
-			} else {
+			} else if (!f2fs_sb_mounted_blkzoned(sb)) {
 				f2fs_msg(sb, KERN_WARNING,
 					"mounting with \"discard\" option, but "
 					"the device does not support discard");
 			}
 			break;
 		case Opt_nodiscard:
+			if (f2fs_sb_mounted_blkzoned(sb)) {
+				f2fs_msg(sb, KERN_WARNING,
+					"discard is required for zoned block devices");
+				return -EINVAL;
+			}
 			clear_opt(sbi, DISCARD);
+			break;
 		case Opt_noheap:
 			set_opt(sbi, NOHEAP);
 			break;
@@ -512,6 +518,13 @@ static int parse_options(struct super_block *sb, char *options)
 				return -ENOMEM;
 			if (strlen(name) == 8 &&
 					!strncmp(name, "adaptive", 8)) {
+				if (f2fs_sb_mounted_blkzoned(sb)) {
+					f2fs_msg(sb, KERN_WARNING,
+						 "adaptive mode is not allowed with "
+						 "zoned block device feature");
+					kfree(name);
+					return -EINVAL;
+				}
 				set_opt_mode(sbi, F2FS_MOUNT_ADAPTIVE);
 			} else if (strlen(name) == 3 &&
 					!strncmp(name, "lfs", 3)) {
@@ -558,13 +571,9 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb)
 
 	init_once((void *) fi);
 
-	if (percpu_counter_init(&fi->dirty_pages, 0, GFP_NOFS)) {
-		kmem_cache_free(f2fs_inode_cachep, fi);
-		return NULL;
-	}
-
 	/* Initialize f2fs-specific inode info */
 	fi->vfs_inode.i_version = 1;
+	atomic_set(&fi->dirty_pages, 0);
 	fi->i_current_depth = 1;
 	fi->i_advise = 0;
 	init_rwsem(&fi->i_sem);
@@ -620,24 +629,25 @@ static int f2fs_drop_inode(struct inode *inode)
 	return generic_drop_inode(inode);
 }
 
-int f2fs_inode_dirtied(struct inode *inode)
+int f2fs_inode_dirtied(struct inode *inode, bool sync)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
+	int ret = 0;
 
 	spin_lock(&sbi->inode_lock[DIRTY_META]);
 	if (is_inode_flag_set(inode, FI_DIRTY_INODE)) {
-		spin_unlock(&sbi->inode_lock[DIRTY_META]);
-		return 1;
+		ret = 1;
+	} else {
+		set_inode_flag(inode, FI_DIRTY_INODE);
+		stat_inc_dirty_inode(sbi, DIRTY_META);
 	}
-
-	set_inode_flag(inode, FI_DIRTY_INODE);
-	list_add_tail(&F2FS_I(inode)->gdirty_list,
+	if (sync && list_empty(&F2FS_I(inode)->gdirty_list)) {
+		list_add_tail(&F2FS_I(inode)->gdirty_list,
 				&sbi->inode_list[DIRTY_META]);
-	inc_page_count(sbi, F2FS_DIRTY_IMETA);
-	stat_inc_dirty_inode(sbi, DIRTY_META);
+		inc_page_count(sbi, F2FS_DIRTY_IMETA);
+	}
 	spin_unlock(&sbi->inode_lock[DIRTY_META]);
-
-	return 0;
+	return ret;
 }
 
 void f2fs_inode_synced(struct inode *inode)
@@ -649,10 +659,12 @@ void f2fs_inode_synced(struct inode *inode)
 		spin_unlock(&sbi->inode_lock[DIRTY_META]);
 		return;
 	}
-	list_del_init(&F2FS_I(inode)->gdirty_list);
+	if (!list_empty(&F2FS_I(inode)->gdirty_list)) {
+		list_del_init(&F2FS_I(inode)->gdirty_list);
+		dec_page_count(sbi, F2FS_DIRTY_IMETA);
+	}
 	clear_inode_flag(inode, FI_DIRTY_INODE);
 	clear_inode_flag(inode, FI_AUTO_RECOVER);
-	dec_page_count(sbi, F2FS_DIRTY_IMETA);
 	stat_dec_dirty_inode(F2FS_I_SB(inode), DIRTY_META);
 	spin_unlock(&sbi->inode_lock[DIRTY_META]);
 }
@@ -676,7 +688,7 @@ static void f2fs_dirty_inode(struct inode *inode, int flags)
 	if (is_inode_flag_set(inode, FI_AUTO_RECOVER))
 		clear_inode_flag(inode, FI_AUTO_RECOVER);
 
-	f2fs_inode_dirtied(inode);
+	f2fs_inode_dirtied(inode, false);
 }
 
 static void f2fs_i_callback(struct rcu_head *head)
@@ -687,20 +699,28 @@ static void f2fs_i_callback(struct rcu_head *head)
 
 static void f2fs_destroy_inode(struct inode *inode)
 {
-	percpu_counter_destroy(&F2FS_I(inode)->dirty_pages);
 	call_rcu(&inode->i_rcu, f2fs_i_callback);
 }
 
 static void destroy_percpu_info(struct f2fs_sb_info *sbi)
 {
-	int i;
-
-	for (i = 0; i < NR_COUNT_TYPE; i++)
-		percpu_counter_destroy(&sbi->nr_pages[i]);
 	percpu_counter_destroy(&sbi->alloc_valid_block_count);
 	percpu_counter_destroy(&sbi->total_valid_inode_count);
 }
 
+static void destroy_device_list(struct f2fs_sb_info *sbi)
+{
+	int i;
+
+	for (i = 0; i < sbi->s_ndevs; i++) {
+		blkdev_put(FDEV(i).bdev, FMODE_EXCL);
+#ifdef CONFIG_BLK_DEV_ZONED
+		kfree(FDEV(i).blkz_type);
+#endif
+	}
+	kfree(sbi->devs);
+}
+
 static void f2fs_put_super(struct super_block *sb)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(sb);
@@ -738,7 +758,6 @@ static void f2fs_put_super(struct super_block *sb)
 	 * In addition, EIO will skip do checkpoint, we need this as well.
 	 */
 	release_ino_entry(sbi, true);
-	release_discard_addrs(sbi);
 
 	f2fs_leave_shrinker(sbi);
 	mutex_unlock(&sbi->umount_mutex);
@@ -762,6 +781,8 @@ static void f2fs_put_super(struct super_block *sb)
 		crypto_free_shash(sbi->s_chksum_driver);
 	kfree(sbi->raw_super);
 
+	destroy_device_list(sbi);
+
 	destroy_percpu_info(sbi);
 	kfree(sbi);
 }
@@ -789,13 +810,17 @@ int f2fs_sync_fs(struct super_block *sb, int sync)
 
 static int f2fs_freeze(struct super_block *sb)
 {
-	int err;
-
 	if (f2fs_readonly(sb))
 		return 0;
 
-	err = f2fs_sync_fs(sb, 1);
-	return err;
+	/* IO error happened before */
+	if (unlikely(f2fs_cp_error(F2FS_SB(sb))))
+		return -EIO;
+
+	/* must be clean, since sync_filesystem() was already called */
+	if (is_sbi_flag_set(F2FS_SB(sb), SBI_IS_DIRTY))
+		return -EINVAL;
+	return 0;
 }
 
 static int f2fs_unfreeze(struct super_block *sb)
@@ -822,7 +847,8 @@ static int f2fs_statfs(struct dentry *dentry, struct kstatfs *buf)
 	buf->f_bavail = user_block_count - valid_user_blocks(sbi);
 
 	buf->f_files = sbi->total_node_count - F2FS_RESERVED_NODE_NUM;
-	buf->f_ffree = buf->f_files - valid_inode_count(sbi);
+	buf->f_ffree = min(buf->f_files - valid_node_count(sbi),
+							buf->f_bavail);
 
 	buf->f_namelen = F2FS_NAME_LEN;
 	buf->f_fsid.val[0] = (u32)id;
@@ -974,7 +1000,7 @@ static void default_options(struct f2fs_sb_info *sbi)
 	set_opt(sbi, EXTENT_CACHE);
 	sbi->sb->s_flags |= MS_LAZYTIME;
 	set_opt(sbi, FLUSH_MERGE);
-	if (f2fs_sb_mounted_hmsmr(sbi->sb)) {
+	if (f2fs_sb_mounted_blkzoned(sbi->sb)) {
 		set_opt_mode(sbi, F2FS_MOUNT_LFS);
 		set_opt(sbi, DISCARD);
 	} else {
@@ -1076,8 +1102,9 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data)
 	 * or if flush_merge is not passed in mount option.
 	 */
 	if ((*flags & MS_RDONLY) || !test_opt(sbi, FLUSH_MERGE)) {
-		destroy_flush_cmd_control(sbi);
-	} else if (!SM_I(sbi)->cmd_control_info) {
+		clear_opt(sbi, FLUSH_MERGE);
+		destroy_flush_cmd_control(sbi, false);
+	} else {
 		err = create_flush_cmd_control(sbi);
 		if (err)
 			goto restore_gc;
@@ -1238,7 +1265,7 @@ static int __f2fs_commit_super(struct buffer_head *bh,
 	unlock_buffer(bh);
 
 	/* it's rare case, we can do fua all the time */
-	return __sync_dirty_buffer(bh, WRITE_FLUSH_FUA);
+	return __sync_dirty_buffer(bh, REQ_PREFLUSH | REQ_FUA);
 }
 
 static inline bool sanity_check_area_boundary(struct f2fs_sb_info *sbi,
@@ -1426,6 +1453,7 @@ int sanity_check_ckpt(struct f2fs_sb_info *sbi)
 	unsigned int total, fsmeta;
 	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
+	unsigned int ovp_segments, reserved_segments;
 
 	total = le32_to_cpu(raw_super->segment_count);
 	fsmeta = le32_to_cpu(raw_super->segment_count_ckpt);
@@ -1437,6 +1465,16 @@ int sanity_check_ckpt(struct f2fs_sb_info *sbi)
 	if (unlikely(fsmeta >= total))
 		return 1;
 
+	ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
+	reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
+
+	if (unlikely(fsmeta < F2FS_MIN_SEGMENTS ||
+			ovp_segments == 0 || reserved_segments == 0)) {
+		f2fs_msg(sbi->sb, KERN_ERR,
+			"Wrong layout: check mkfs.f2fs version");
+		return 1;
+	}
+
 	if (unlikely(f2fs_cp_error(sbi))) {
 		f2fs_msg(sbi->sb, KERN_ERR, "A bug case: need to run fsck");
 		return 1;
@@ -1447,6 +1485,7 @@ int sanity_check_ckpt(struct f2fs_sb_info *sbi)
 static void init_sb_info(struct f2fs_sb_info *sbi)
 {
 	struct f2fs_super_block *raw_super = sbi->raw_super;
+	int i;
 
 	sbi->log_sectors_per_block =
 		le32_to_cpu(raw_super->log_sectors_per_block);
@@ -1471,6 +1510,9 @@ static void init_sb_info(struct f2fs_sb_info *sbi)
 	sbi->interval_time[REQ_TIME] = DEF_IDLE_INTERVAL;
 	clear_sbi_flag(sbi, SBI_NEED_FSCK);
 
+	for (i = 0; i < NR_COUNT_TYPE; i++)
+		atomic_set(&sbi->nr_pages[i], 0);
+
 	INIT_LIST_HEAD(&sbi->s_list);
 	mutex_init(&sbi->umount_mutex);
 	mutex_init(&sbi->wio_mutex[NODE]);
@@ -1486,13 +1528,7 @@ static void init_sb_info(struct f2fs_sb_info *sbi)
 
 static int init_percpu_info(struct f2fs_sb_info *sbi)
 {
-	int i, err;
-
-	for (i = 0; i < NR_COUNT_TYPE; i++) {
-		err = percpu_counter_init(&sbi->nr_pages[i], 0, GFP_KERNEL);
-		if (err)
-			return err;
-	}
+	int err;
 
 	err = percpu_counter_init(&sbi->alloc_valid_block_count, 0, GFP_KERNEL);
 	if (err)
@@ -1502,6 +1538,71 @@ static int init_percpu_info(struct f2fs_sb_info *sbi)
 								GFP_KERNEL);
 }
 
+#ifdef CONFIG_BLK_DEV_ZONED
+static int init_blkz_info(struct f2fs_sb_info *sbi, int devi)
+{
+	struct block_device *bdev = FDEV(devi).bdev;
+	sector_t nr_sectors = bdev->bd_part->nr_sects;
+	sector_t sector = 0;
+	struct blk_zone *zones;
+	unsigned int i, nr_zones;
+	unsigned int n = 0;
+	int err = -EIO;
+
+	if (!f2fs_sb_mounted_blkzoned(sbi->sb))
+		return 0;
+
+	if (sbi->blocks_per_blkz && sbi->blocks_per_blkz !=
+				SECTOR_TO_BLOCK(bdev_zone_size(bdev)))
+		return -EINVAL;
+	sbi->blocks_per_blkz = SECTOR_TO_BLOCK(bdev_zone_size(bdev));
+	if (sbi->log_blocks_per_blkz && sbi->log_blocks_per_blkz !=
+				__ilog2_u32(sbi->blocks_per_blkz))
+		return -EINVAL;
+	sbi->log_blocks_per_blkz = __ilog2_u32(sbi->blocks_per_blkz);
+	FDEV(devi).nr_blkz = SECTOR_TO_BLOCK(nr_sectors) >>
+					sbi->log_blocks_per_blkz;
+	if (nr_sectors & (bdev_zone_size(bdev) - 1))
+		FDEV(devi).nr_blkz++;
+
+	FDEV(devi).blkz_type = kmalloc(FDEV(devi).nr_blkz, GFP_KERNEL);
+	if (!FDEV(devi).blkz_type)
+		return -ENOMEM;
+
+#define F2FS_REPORT_NR_ZONES   4096
+
+	zones = kcalloc(F2FS_REPORT_NR_ZONES, sizeof(struct blk_zone),
+			GFP_KERNEL);
+	if (!zones)
+		return -ENOMEM;
+
+	/* Get block zones type */
+	while (zones && sector < nr_sectors) {
+
+		nr_zones = F2FS_REPORT_NR_ZONES;
+		err = blkdev_report_zones(bdev, sector,
+					  zones, &nr_zones,
+					  GFP_KERNEL);
+		if (err)
+			break;
+		if (!nr_zones) {
+			err = -EIO;
+			break;
+		}
+
+		for (i = 0; i < nr_zones; i++) {
+			FDEV(devi).blkz_type[n] = zones[i].type;
+			sector += zones[i].len;
+			n++;
+		}
+	}
+
+	kfree(zones);
+
+	return err;
+}
+#endif
+
 /*
  * Read f2fs raw super block.
  * Because we have two copies of super block, so read both of them
@@ -1594,6 +1695,77 @@ int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover)
 	return err;
 }
 
+static int f2fs_scan_devices(struct f2fs_sb_info *sbi)
+{
+	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
+	int i;
+
+	for (i = 0; i < MAX_DEVICES; i++) {
+		if (!RDEV(i).path[0])
+			return 0;
+
+		if (i == 0) {
+			sbi->devs = kzalloc(sizeof(struct f2fs_dev_info) *
+						MAX_DEVICES, GFP_KERNEL);
+			if (!sbi->devs)
+				return -ENOMEM;
+		}
+
+		memcpy(FDEV(i).path, RDEV(i).path, MAX_PATH_LEN);
+		FDEV(i).total_segments = le32_to_cpu(RDEV(i).total_segments);
+		if (i == 0) {
+			FDEV(i).start_blk = 0;
+			FDEV(i).end_blk = FDEV(i).start_blk +
+				(FDEV(i).total_segments <<
+				sbi->log_blocks_per_seg) - 1 +
+				le32_to_cpu(raw_super->segment0_blkaddr);
+		} else {
+			FDEV(i).start_blk = FDEV(i - 1).end_blk + 1;
+			FDEV(i).end_blk = FDEV(i).start_blk +
+				(FDEV(i).total_segments <<
+				sbi->log_blocks_per_seg) - 1;
+		}
+
+		FDEV(i).bdev = blkdev_get_by_path(FDEV(i).path,
+					sbi->sb->s_mode, sbi->sb->s_type);
+		if (IS_ERR(FDEV(i).bdev))
+			return PTR_ERR(FDEV(i).bdev);
+
+		/* to release errored devices */
+		sbi->s_ndevs = i + 1;
+
+#ifdef CONFIG_BLK_DEV_ZONED
+		if (bdev_zoned_model(FDEV(i).bdev) == BLK_ZONED_HM &&
+				!f2fs_sb_mounted_blkzoned(sbi->sb)) {
+			f2fs_msg(sbi->sb, KERN_ERR,
+				"Zoned block device feature not enabled\n");
+			return -EINVAL;
+		}
+		if (bdev_zoned_model(FDEV(i).bdev) != BLK_ZONED_NONE) {
+			if (init_blkz_info(sbi, i)) {
+				f2fs_msg(sbi->sb, KERN_ERR,
+					"Failed to initialize F2FS blkzone information");
+				return -EINVAL;
+			}
+			f2fs_msg(sbi->sb, KERN_INFO,
+				"Mount Device [%2d]: %20s, %8u, %8x - %8x (zone: %s)",
+				i, FDEV(i).path,
+				FDEV(i).total_segments,
+				FDEV(i).start_blk, FDEV(i).end_blk,
+				bdev_zoned_model(FDEV(i).bdev) == BLK_ZONED_HA ?
+				"Host-aware" : "Host-managed");
+			continue;
+		}
+#endif
+		f2fs_msg(sbi->sb, KERN_INFO,
+			"Mount Device [%2d]: %20s, %8u, %8x - %8x",
+				i, FDEV(i).path,
+				FDEV(i).total_segments,
+				FDEV(i).start_blk, FDEV(i).end_blk);
+	}
+	return 0;
+}
+
 static int f2fs_fill_super(struct super_block *sb, void *data, int silent)
 {
 	struct f2fs_sb_info *sbi;
@@ -1641,6 +1813,18 @@ try_onemore:
 	sb->s_fs_info = sbi;
 	sbi->raw_super = raw_super;
 
+	/*
+	 * The BLKZONED feature indicates that the drive was formatted with
+	 * zone alignment optimization. This is optional for host-aware
+	 * devices, but mandatory for host-managed zoned block devices.
+	 */
+#ifndef CONFIG_BLK_DEV_ZONED
+	if (f2fs_sb_mounted_blkzoned(sb)) {
+		f2fs_msg(sb, KERN_ERR,
+			 "Zoned block device support is not enabled\n");
+		goto free_sb_buf;
+	}
+#endif
 	default_options(sbi);
 	/* parse mount options */
 	options = kstrdup((const char *)data, GFP_KERNEL);
@@ -1710,6 +1894,13 @@ try_onemore:
 		goto free_meta_inode;
 	}
 
+	/* Initialize device list */
+	err = f2fs_scan_devices(sbi);
+	if (err) {
+		f2fs_msg(sb, KERN_ERR, "Failed to find devices");
+		goto free_devices;
+	}
+
 	sbi->total_valid_node_count =
 				le32_to_cpu(sbi->ckpt->valid_node_count);
 	percpu_counter_set(&sbi->total_valid_inode_count,
@@ -1893,12 +2084,21 @@ free_node_inode:
 	mutex_lock(&sbi->umount_mutex);
 	release_ino_entry(sbi, true);
 	f2fs_leave_shrinker(sbi);
+	/*
+	 * Some dirty meta pages can be produced by recover_orphan_inodes()
+	 * failed by EIO. Then, iput(node_inode) can trigger balance_fs_bg()
+	 * followed by write_checkpoint() through f2fs_write_node_pages(), which
+	 * falls into an infinite loop in sync_meta_pages().
+	 */
+	truncate_inode_pages_final(META_MAPPING(sbi));
 	iput(sbi->node_inode);
 	mutex_unlock(&sbi->umount_mutex);
 free_nm:
 	destroy_node_manager(sbi);
 free_sm:
 	destroy_segment_manager(sbi);
+free_devices:
+	destroy_device_list(sbi);
 	kfree(sbi->ckpt);
 free_meta_inode:
 	make_bad_inode(sbi->meta_inode);
@@ -2044,3 +2244,4 @@ module_exit(exit_f2fs_fs)
 MODULE_AUTHOR("Samsung Electronics's Praesto Team");
 MODULE_DESCRIPTION("Flash Friendly File System");
 MODULE_LICENSE("GPL");
+
diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c
index 3e1c0280f866..c47ce2f330a1 100644
--- a/fs/f2fs/xattr.c
+++ b/fs/f2fs/xattr.c
@@ -106,7 +106,7 @@ static int f2fs_xattr_advise_set(const struct xattr_handler *handler,
 		return -EINVAL;
 
 	F2FS_I(inode)->i_advise |= *(char *)value;
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, true);
 	return 0;
 }
 
@@ -554,7 +554,7 @@ static int __f2fs_setxattr(struct inode *inode, int index,
 	if (index == F2FS_XATTR_INDEX_ENCRYPTION &&
 			!strcmp(name, F2FS_XATTR_NAME_ENCRYPTION_CONTEXT))
 		f2fs_set_encrypted_inode(inode);
-	f2fs_mark_inode_dirty_sync(inode);
+	f2fs_mark_inode_dirty_sync(inode, true);
 	if (!error && S_ISDIR(inode->i_mode))
 		set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_CP);
 exit:
diff --git a/fs/gfs2/dir.c b/fs/gfs2/dir.c
index 3cdde5f5d399..79113219be5f 100644
--- a/fs/gfs2/dir.c
+++ b/fs/gfs2/dir.c
@@ -62,6 +62,7 @@
 #include <linux/gfs2_ondisk.h>
 #include <linux/crc32.h>
 #include <linux/vmalloc.h>
+#include <linux/bio.h>
 
 #include "gfs2.h"
 #include "incore.h"
diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c
index e58ccef09c91..27c00a16def0 100644
--- a/fs/gfs2/log.c
+++ b/fs/gfs2/log.c
@@ -657,7 +657,7 @@ static void log_write_header(struct gfs2_sbd *sdp, u32 flags)
 	struct gfs2_log_header *lh;
 	unsigned int tail;
 	u32 hash;
-	int op_flags = WRITE_FLUSH_FUA | REQ_META;
+	int op_flags = REQ_PREFLUSH | REQ_FUA | REQ_META;
 	struct page *page = mempool_alloc(gfs2_page_pool, GFP_NOIO);
 	enum gfs2_freeze_state state = atomic_read(&sdp->sd_freeze_state);
 	lh = page_address(page);
@@ -682,7 +682,7 @@ static void log_write_header(struct gfs2_sbd *sdp, u32 flags)
 	if (test_bit(SDF_NOBARRIERS, &sdp->sd_flags)) {
 		gfs2_ordered_wait(sdp);
 		log_flush_wait(sdp);
-		op_flags = WRITE_SYNC | REQ_META | REQ_PRIO;
+		op_flags = REQ_SYNC | REQ_META | REQ_PRIO;
 	}
 
 	sdp->sd_log_idle = (tail == sdp->sd_log_flush_head);
diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c
index 49d5a1b61b06..b1f9144b42c7 100644
--- a/fs/gfs2/lops.c
+++ b/fs/gfs2/lops.c
@@ -231,7 +231,7 @@ static void gfs2_end_log_write(struct bio *bio)
  * gfs2_log_flush_bio - Submit any pending log bio
  * @sdp: The superblock
  * @op: REQ_OP
- * @op_flags: rq_flag_bits
+ * @op_flags: req_flag_bits
  *
  * Submit any pending part-built or full bio to the block device. If
  * there is no pending bio, then this is a no-op.
diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c
index 373639a59782..49db8ef13fdf 100644
--- a/fs/gfs2/meta_io.c
+++ b/fs/gfs2/meta_io.c
@@ -37,8 +37,7 @@ static int gfs2_aspace_writepage(struct page *page, struct writeback_control *wb
 {
 	struct buffer_head *bh, *head;
 	int nr_underway = 0;
-	int write_flags = REQ_META | REQ_PRIO |
-		(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : 0);
+	int write_flags = REQ_META | REQ_PRIO | wbc_to_write_flags(wbc);
 
 	BUG_ON(!PageLocked(page));
 	BUG_ON(!page_has_buffers(page));
@@ -285,7 +284,7 @@ int gfs2_meta_read(struct gfs2_glock *gl, u64 blkno, int flags,
 		}
 	}
 
-	gfs2_submit_bhs(REQ_OP_READ, READ_SYNC | REQ_META | REQ_PRIO, bhs, num);
+	gfs2_submit_bhs(REQ_OP_READ, REQ_META | REQ_PRIO, bhs, num);
 	if (!(flags & DIO_WAIT))
 		return 0;
 
@@ -453,7 +452,7 @@ struct buffer_head *gfs2_meta_ra(struct gfs2_glock *gl, u64 dblock, u32 extlen)
 	if (buffer_uptodate(first_bh))
 		goto out;
 	if (!buffer_locked(first_bh))
-		ll_rw_block(REQ_OP_READ, READ_SYNC | REQ_META, 1, &first_bh);
+		ll_rw_block(REQ_OP_READ, REQ_META, 1, &first_bh);
 
 	dblock++;
 	extlen--;
diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c
index ff72ac6439c8..a34308df927f 100644
--- a/fs/gfs2/ops_fstype.c
+++ b/fs/gfs2/ops_fstype.c
@@ -246,7 +246,7 @@ static int gfs2_read_super(struct gfs2_sbd *sdp, sector_t sector, int silent)
 
 	bio->bi_end_io = end_bio_io_page;
 	bio->bi_private = page;
-	bio_set_op_attrs(bio, REQ_OP_READ, READ_SYNC | REQ_META);
+	bio_set_op_attrs(bio, REQ_OP_READ, REQ_META);
 	submit_bio(bio);
 	wait_on_page_locked(page);
 	bio_put(bio);
diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c
index 11854dd84572..67aedf4c2e7c 100644
--- a/fs/hfsplus/super.c
+++ b/fs/hfsplus/super.c
@@ -221,7 +221,7 @@ static int hfsplus_sync_fs(struct super_block *sb, int wait)
 	error2 = hfsplus_submit_bio(sb,
 				   sbi->part_start + HFSPLUS_VOLHEAD_SECTOR,
 				   sbi->s_vhdr_buf, NULL, REQ_OP_WRITE,
-				   WRITE_SYNC);
+				   REQ_SYNC);
 	if (!error)
 		error = error2;
 	if (!write_backup)
@@ -230,7 +230,7 @@ static int hfsplus_sync_fs(struct super_block *sb, int wait)
 	error2 = hfsplus_submit_bio(sb,
 				  sbi->part_start + sbi->sect_count - 2,
 				  sbi->s_backup_vhdr_buf, NULL, REQ_OP_WRITE,
-				  WRITE_SYNC);
+				  REQ_SYNC);
 	if (!error)
 		error2 = error;
 out:
diff --git a/fs/internal.h b/fs/internal.h
index f4da3341b4a3..4fcf51766d4a 100644
--- a/fs/internal.h
+++ b/fs/internal.h
@@ -184,3 +184,6 @@ typedef loff_t (*iomap_actor_t)(struct inode *inode, loff_t pos, loff_t len,
 loff_t iomap_apply(struct inode *inode, loff_t pos, loff_t length,
 		unsigned flags, struct iomap_ops *ops, void *data,
 		iomap_actor_t actor);
+
+/* direct-io.c: */
+int sb_init_dio_done_wq(struct super_block *sb);
diff --git a/fs/iomap.c b/fs/iomap.c
index a8ee8c33ca78..354a123f170e 100644
--- a/fs/iomap.c
+++ b/fs/iomap.c
@@ -24,6 +24,7 @@
 #include <linux/uio.h>
 #include <linux/backing-dev.h>
 #include <linux/buffer_head.h>
+#include <linux/task_io_accounting_ops.h>
 #include <linux/dax.h>
 #include "internal.h"
 
@@ -467,8 +468,9 @@ int iomap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
 
 	offset = page_offset(page);
 	while (length > 0) {
-		ret = iomap_apply(inode, offset, length, IOMAP_WRITE,
-				ops, page, iomap_page_mkwrite_actor);
+		ret = iomap_apply(inode, offset, length,
+				IOMAP_WRITE | IOMAP_FAULT, ops, page,
+				iomap_page_mkwrite_actor);
 		if (unlikely(ret <= 0))
 			goto out_unlock;
 		offset += ret;
@@ -583,3 +585,375 @@ int iomap_fiemap(struct inode *inode, struct fiemap_extent_info *fi,
 	return 0;
 }
 EXPORT_SYMBOL_GPL(iomap_fiemap);
+
+/*
+ * Private flags for iomap_dio, must not overlap with the public ones in
+ * iomap.h:
+ */
+#define IOMAP_DIO_WRITE		(1 << 30)
+#define IOMAP_DIO_DIRTY		(1 << 31)
+
+struct iomap_dio {
+	struct kiocb		*iocb;
+	iomap_dio_end_io_t	*end_io;
+	loff_t			i_size;
+	loff_t			size;
+	atomic_t		ref;
+	unsigned		flags;
+	int			error;
+
+	union {
+		/* used during submission and for synchronous completion: */
+		struct {
+			struct iov_iter		*iter;
+			struct task_struct	*waiter;
+			struct request_queue	*last_queue;
+			blk_qc_t		cookie;
+		} submit;
+
+		/* used for aio completion: */
+		struct {
+			struct work_struct	work;
+		} aio;
+	};
+};
+
+static ssize_t iomap_dio_complete(struct iomap_dio *dio)
+{
+	struct kiocb *iocb = dio->iocb;
+	ssize_t ret;
+
+	if (dio->end_io) {
+		ret = dio->end_io(iocb,
+				dio->error ? dio->error : dio->size,
+				dio->flags);
+	} else {
+		ret = dio->error;
+	}
+
+	if (likely(!ret)) {
+		ret = dio->size;
+		/* check for short read */
+		if (iocb->ki_pos + ret > dio->i_size &&
+		    !(dio->flags & IOMAP_DIO_WRITE))
+			ret = dio->i_size - iocb->ki_pos;
+		iocb->ki_pos += ret;
+	}
+
+	inode_dio_end(file_inode(iocb->ki_filp));
+	kfree(dio);
+
+	return ret;
+}
+
+static void iomap_dio_complete_work(struct work_struct *work)
+{
+	struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);
+	struct kiocb *iocb = dio->iocb;
+	bool is_write = (dio->flags & IOMAP_DIO_WRITE);
+	ssize_t ret;
+
+	ret = iomap_dio_complete(dio);
+	if (is_write && ret > 0)
+		ret = generic_write_sync(iocb, ret);
+	iocb->ki_complete(iocb, ret, 0);
+}
+
+/*
+ * Set an error in the dio if none is set yet.  We have to use cmpxchg
+ * as the submission context and the completion context(s) can race to
+ * update the error.
+ */
+static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)
+{
+	cmpxchg(&dio->error, 0, ret);
+}
+
+static void iomap_dio_bio_end_io(struct bio *bio)
+{
+	struct iomap_dio *dio = bio->bi_private;
+	bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
+
+	if (bio->bi_error)
+		iomap_dio_set_error(dio, bio->bi_error);
+
+	if (atomic_dec_and_test(&dio->ref)) {
+		if (is_sync_kiocb(dio->iocb)) {
+			struct task_struct *waiter = dio->submit.waiter;
+
+			WRITE_ONCE(dio->submit.waiter, NULL);
+			wake_up_process(waiter);
+		} else if (dio->flags & IOMAP_DIO_WRITE) {
+			struct inode *inode = file_inode(dio->iocb->ki_filp);
+
+			INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
+			queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work);
+		} else {
+			iomap_dio_complete_work(&dio->aio.work);
+		}
+	}
+
+	if (should_dirty) {
+		bio_check_pages_dirty(bio);
+	} else {
+		struct bio_vec *bvec;
+		int i;
+
+		bio_for_each_segment_all(bvec, bio, i)
+			put_page(bvec->bv_page);
+		bio_put(bio);
+	}
+}
+
+static blk_qc_t
+iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos,
+		unsigned len)
+{
+	struct page *page = ZERO_PAGE(0);
+	struct bio *bio;
+
+	bio = bio_alloc(GFP_KERNEL, 1);
+	bio->bi_bdev = iomap->bdev;
+	bio->bi_iter.bi_sector =
+		iomap->blkno + ((pos - iomap->offset) >> 9);
+	bio->bi_private = dio;
+	bio->bi_end_io = iomap_dio_bio_end_io;
+
+	get_page(page);
+	if (bio_add_page(bio, page, len, 0) != len)
+		BUG();
+	bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC | REQ_IDLE);
+
+	atomic_inc(&dio->ref);
+	return submit_bio(bio);
+}
+
+static loff_t
+iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
+		void *data, struct iomap *iomap)
+{
+	struct iomap_dio *dio = data;
+	unsigned blkbits = blksize_bits(bdev_logical_block_size(iomap->bdev));
+	unsigned fs_block_size = (1 << inode->i_blkbits), pad;
+	unsigned align = iov_iter_alignment(dio->submit.iter);
+	struct iov_iter iter;
+	struct bio *bio;
+	bool need_zeroout = false;
+	int nr_pages, ret;
+
+	if ((pos | length | align) & ((1 << blkbits) - 1))
+		return -EINVAL;
+
+	switch (iomap->type) {
+	case IOMAP_HOLE:
+		if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))
+			return -EIO;
+		/*FALLTHRU*/
+	case IOMAP_UNWRITTEN:
+		if (!(dio->flags & IOMAP_DIO_WRITE)) {
+			iov_iter_zero(length, dio->submit.iter);
+			dio->size += length;
+			return length;
+		}
+		dio->flags |= IOMAP_DIO_UNWRITTEN;
+		need_zeroout = true;
+		break;
+	case IOMAP_MAPPED:
+		if (iomap->flags & IOMAP_F_SHARED)
+			dio->flags |= IOMAP_DIO_COW;
+		if (iomap->flags & IOMAP_F_NEW)
+			need_zeroout = true;
+		break;
+	default:
+		WARN_ON_ONCE(1);
+		return -EIO;
+	}
+
+	/*
+	 * Operate on a partial iter trimmed to the extent we were called for.
+	 * We'll update the iter in the dio once we're done with this extent.
+	 */
+	iter = *dio->submit.iter;
+	iov_iter_truncate(&iter, length);
+
+	nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
+	if (nr_pages <= 0)
+		return nr_pages;
+
+	if (need_zeroout) {
+		/* zero out from the start of the block to the write offset */
+		pad = pos & (fs_block_size - 1);
+		if (pad)
+			iomap_dio_zero(dio, iomap, pos - pad, pad);
+	}
+
+	do {
+		if (dio->error)
+			return 0;
+
+		bio = bio_alloc(GFP_KERNEL, nr_pages);
+		bio->bi_bdev = iomap->bdev;
+		bio->bi_iter.bi_sector =
+			iomap->blkno + ((pos - iomap->offset) >> 9);
+		bio->bi_private = dio;
+		bio->bi_end_io = iomap_dio_bio_end_io;
+
+		ret = bio_iov_iter_get_pages(bio, &iter);
+		if (unlikely(ret)) {
+			bio_put(bio);
+			return ret;
+		}
+
+		if (dio->flags & IOMAP_DIO_WRITE) {
+			bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC | REQ_IDLE);
+			task_io_account_write(bio->bi_iter.bi_size);
+		} else {
+			bio_set_op_attrs(bio, REQ_OP_READ, 0);
+			if (dio->flags & IOMAP_DIO_DIRTY)
+				bio_set_pages_dirty(bio);
+		}
+
+		dio->size += bio->bi_iter.bi_size;
+		pos += bio->bi_iter.bi_size;
+
+		nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
+
+		atomic_inc(&dio->ref);
+
+		dio->submit.last_queue = bdev_get_queue(iomap->bdev);
+		dio->submit.cookie = submit_bio(bio);
+	} while (nr_pages);
+
+	if (need_zeroout) {
+		/* zero out from the end of the write to the end of the block */
+		pad = pos & (fs_block_size - 1);
+		if (pad)
+			iomap_dio_zero(dio, iomap, pos, fs_block_size - pad);
+	}
+
+	iov_iter_advance(dio->submit.iter, length);
+	return length;
+}
+
+ssize_t
+iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, struct iomap_ops *ops,
+		iomap_dio_end_io_t end_io)
+{
+	struct address_space *mapping = iocb->ki_filp->f_mapping;
+	struct inode *inode = file_inode(iocb->ki_filp);
+	size_t count = iov_iter_count(iter);
+	loff_t pos = iocb->ki_pos, end = iocb->ki_pos + count - 1, ret = 0;
+	unsigned int flags = IOMAP_DIRECT;
+	struct blk_plug plug;
+	struct iomap_dio *dio;
+
+	lockdep_assert_held(&inode->i_rwsem);
+
+	if (!count)
+		return 0;
+
+	dio = kmalloc(sizeof(*dio), GFP_KERNEL);
+	if (!dio)
+		return -ENOMEM;
+
+	dio->iocb = iocb;
+	atomic_set(&dio->ref, 1);
+	dio->size = 0;
+	dio->i_size = i_size_read(inode);
+	dio->end_io = end_io;
+	dio->error = 0;
+	dio->flags = 0;
+
+	dio->submit.iter = iter;
+	if (is_sync_kiocb(iocb)) {
+		dio->submit.waiter = current;
+		dio->submit.cookie = BLK_QC_T_NONE;
+		dio->submit.last_queue = NULL;
+	}
+
+	if (iov_iter_rw(iter) == READ) {
+		if (pos >= dio->i_size)
+			goto out_free_dio;
+
+		if (iter->type == ITER_IOVEC)
+			dio->flags |= IOMAP_DIO_DIRTY;
+	} else {
+		dio->flags |= IOMAP_DIO_WRITE;
+		flags |= IOMAP_WRITE;
+	}
+
+	if (mapping->nrpages) {
+		ret = filemap_write_and_wait_range(mapping, iocb->ki_pos, end);
+		if (ret)
+			goto out_free_dio;
+
+		ret = invalidate_inode_pages2_range(mapping,
+				iocb->ki_pos >> PAGE_SHIFT, end >> PAGE_SHIFT);
+		WARN_ON_ONCE(ret);
+		ret = 0;
+	}
+
+	inode_dio_begin(inode);
+
+	blk_start_plug(&plug);
+	do {
+		ret = iomap_apply(inode, pos, count, flags, ops, dio,
+				iomap_dio_actor);
+		if (ret <= 0) {
+			/* magic error code to fall back to buffered I/O */
+			if (ret == -ENOTBLK)
+				ret = 0;
+			break;
+		}
+		pos += ret;
+	} while ((count = iov_iter_count(iter)) > 0);
+	blk_finish_plug(&plug);
+
+	if (ret < 0)
+		iomap_dio_set_error(dio, ret);
+
+	if (ret >= 0 && iov_iter_rw(iter) == WRITE && !is_sync_kiocb(iocb) &&
+			!inode->i_sb->s_dio_done_wq) {
+		ret = sb_init_dio_done_wq(inode->i_sb);
+		if (ret < 0)
+			iomap_dio_set_error(dio, ret);
+	}
+
+	if (!atomic_dec_and_test(&dio->ref)) {
+		if (!is_sync_kiocb(iocb))
+			return -EIOCBQUEUED;
+
+		for (;;) {
+			set_current_state(TASK_UNINTERRUPTIBLE);
+			if (!READ_ONCE(dio->submit.waiter))
+				break;
+
+			if (!(iocb->ki_flags & IOCB_HIPRI) ||
+			    !dio->submit.last_queue ||
+			    !blk_mq_poll(dio->submit.last_queue,
+					 dio->submit.cookie))
+				io_schedule();
+		}
+		__set_current_state(TASK_RUNNING);
+	}
+
+	/*
+	 * Try again to invalidate clean pages which might have been cached by
+	 * non-direct readahead, or faulted in by get_user_pages() if the source
+	 * of the write was an mmap'ed region of the file we're writing.  Either
+	 * one is a pretty crazy thing to do, so we don't support it 100%.  If
+	 * this invalidation fails, tough, the write still worked...
+	 */
+	if (iov_iter_rw(iter) == WRITE && mapping->nrpages) {
+		ret = invalidate_inode_pages2_range(mapping,
+				iocb->ki_pos >> PAGE_SHIFT, end >> PAGE_SHIFT);
+		WARN_ON_ONCE(ret);
+	}
+
+	return iomap_dio_complete(dio);
+
+out_free_dio:
+	kfree(dio);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(iomap_dio_rw);
diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c
index 44af14b2e916..9bb2fe35799d 100644
--- a/fs/isofs/compress.c
+++ b/fs/isofs/compress.c
@@ -18,6 +18,7 @@
 
 #include <linux/module.h>
 #include <linux/init.h>
+#include <linux/bio.h>
 
 #include <linux/vmalloc.h>
 #include <linux/zlib.h>
diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c
index 684996c8a3a4..4055f51617ef 100644
--- a/fs/jbd2/checkpoint.c
+++ b/fs/jbd2/checkpoint.c
@@ -186,7 +186,7 @@ __flush_batch(journal_t *journal, int *batch_count)
 
 	blk_start_plug(&plug);
 	for (i = 0; i < *batch_count; i++)
-		write_dirty_buffer(journal->j_chkpt_bhs[i], WRITE_SYNC);
+		write_dirty_buffer(journal->j_chkpt_bhs[i], REQ_SYNC);
 	blk_finish_plug(&plug);
 
 	for (i = 0; i < *batch_count; i++) {
diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c
index 31f8ca046639..8c514367ba5a 100644
--- a/fs/jbd2/commit.c
+++ b/fs/jbd2/commit.c
@@ -155,9 +155,10 @@ static int journal_submit_commit_record(journal_t *journal,
 
 	if (journal->j_flags & JBD2_BARRIER &&
 	    !jbd2_has_feature_async_commit(journal))
-		ret = submit_bh(REQ_OP_WRITE, WRITE_SYNC | WRITE_FLUSH_FUA, bh);
+		ret = submit_bh(REQ_OP_WRITE,
+			REQ_SYNC | REQ_PREFLUSH | REQ_FUA, bh);
 	else
-		ret = submit_bh(REQ_OP_WRITE, WRITE_SYNC, bh);
+		ret = submit_bh(REQ_OP_WRITE, REQ_SYNC, bh);
 
 	*cbh = bh;
 	return ret;
@@ -402,7 +403,7 @@ void jbd2_journal_commit_transaction(journal_t *journal)
 		jbd2_journal_update_sb_log_tail(journal,
 						journal->j_tail_sequence,
 						journal->j_tail,
-						WRITE_SYNC);
+						REQ_SYNC);
 		mutex_unlock(&journal->j_checkpoint_mutex);
 	} else {
 		jbd_debug(3, "superblock not updated\n");
@@ -717,7 +718,7 @@ start_journal_io:
 				clear_buffer_dirty(bh);
 				set_buffer_uptodate(bh);
 				bh->b_end_io = journal_end_buffer_io_sync;
-				submit_bh(REQ_OP_WRITE, WRITE_SYNC, bh);
+				submit_bh(REQ_OP_WRITE, REQ_SYNC, bh);
 			}
 			cond_resched();
 			stats.run.rs_blocks_logged += bufs;
diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c
index 927da4956a89..8ed971eeab44 100644
--- a/fs/jbd2/journal.c
+++ b/fs/jbd2/journal.c
@@ -913,7 +913,7 @@ int __jbd2_update_log_tail(journal_t *journal, tid_t tid, unsigned long block)
 	 * space and if we lose sb update during power failure we'd replay
 	 * old transaction with possibly newly overwritten data.
 	 */
-	ret = jbd2_journal_update_sb_log_tail(journal, tid, block, WRITE_FUA);
+	ret = jbd2_journal_update_sb_log_tail(journal, tid, block, REQ_FUA);
 	if (ret)
 		goto out;
 
@@ -1306,7 +1306,7 @@ static int journal_reset(journal_t *journal)
 		/* Lock here to make assertions happy... */
 		mutex_lock(&journal->j_checkpoint_mutex);
 		/*
-		 * Update log tail information. We use WRITE_FUA since new
+		 * Update log tail information. We use REQ_FUA since new
 		 * transaction will start reusing journal space and so we
 		 * must make sure information about current log tail is on
 		 * disk before that.
@@ -1314,7 +1314,7 @@ static int journal_reset(journal_t *journal)
 		jbd2_journal_update_sb_log_tail(journal,
 						journal->j_tail_sequence,
 						journal->j_tail,
-						WRITE_FUA);
+						REQ_FUA);
 		mutex_unlock(&journal->j_checkpoint_mutex);
 	}
 	return jbd2_journal_start_thread(journal);
@@ -1454,7 +1454,7 @@ void jbd2_journal_update_sb_errno(journal_t *journal)
 	sb->s_errno    = cpu_to_be32(journal->j_errno);
 	read_unlock(&journal->j_state_lock);
 
-	jbd2_write_superblock(journal, WRITE_FUA);
+	jbd2_write_superblock(journal, REQ_FUA);
 }
 EXPORT_SYMBOL(jbd2_journal_update_sb_errno);
 
@@ -1720,7 +1720,8 @@ int jbd2_journal_destroy(journal_t *journal)
 				++journal->j_transaction_sequence;
 			write_unlock(&journal->j_state_lock);
 
-			jbd2_mark_journal_empty(journal, WRITE_FLUSH_FUA);
+			jbd2_mark_journal_empty(journal,
+					REQ_PREFLUSH | REQ_FUA);
 			mutex_unlock(&journal->j_checkpoint_mutex);
 		} else
 			err = -EIO;
@@ -1979,7 +1980,7 @@ int jbd2_journal_flush(journal_t *journal)
 	 * the magic code for a fully-recovered superblock.  Any future
 	 * commits of data to the journal will restore the current
 	 * s_start value. */
-	jbd2_mark_journal_empty(journal, WRITE_FUA);
+	jbd2_mark_journal_empty(journal, REQ_FUA);
 	mutex_unlock(&journal->j_checkpoint_mutex);
 	write_lock(&journal->j_state_lock);
 	J_ASSERT(!journal->j_running_transaction);
@@ -2025,7 +2026,7 @@ int jbd2_journal_wipe(journal_t *journal, int write)
 	if (write) {
 		/* Lock to make assertions happy... */
 		mutex_lock(&journal->j_checkpoint_mutex);
-		jbd2_mark_journal_empty(journal, WRITE_FUA);
+		jbd2_mark_journal_empty(journal, REQ_FUA);
 		mutex_unlock(&journal->j_checkpoint_mutex);
 	}
 
diff --git a/fs/jbd2/revoke.c b/fs/jbd2/revoke.c
index 91171dc352cb..cfc38b552118 100644
--- a/fs/jbd2/revoke.c
+++ b/fs/jbd2/revoke.c
@@ -648,7 +648,7 @@ static void flush_descriptor(journal_t *journal,
 	set_buffer_jwrite(descriptor);
 	BUFFER_TRACE(descriptor, "write");
 	set_buffer_dirty(descriptor);
-	write_dirty_buffer(descriptor, WRITE_SYNC);
+	write_dirty_buffer(descriptor, REQ_SYNC);
 }
 #endif
 
diff --git a/fs/jfs/ioctl.c b/fs/jfs/ioctl.c
index 8653cac7e12e..b6fd1ff29ddf 100644
--- a/fs/jfs/ioctl.c
+++ b/fs/jfs/ioctl.c
@@ -121,7 +121,7 @@ long jfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 
 		jfs_set_inode_flags(inode);
 		inode_unlock(inode);
-		inode->i_ctime = CURRENT_TIME_SEC;
+		inode->i_ctime = current_time(inode);
 		mark_inode_dirty(inode);
 setflags_out:
 		mnt_drop_write_file(filp);
diff --git a/fs/jfs/jfs_logmgr.c b/fs/jfs/jfs_logmgr.c
index a21ea8b3e5fa..bb1da1feafeb 100644
--- a/fs/jfs/jfs_logmgr.c
+++ b/fs/jfs/jfs_logmgr.c
@@ -2002,7 +2002,7 @@ static int lbmRead(struct jfs_log * log, int pn, struct lbuf ** bpp)
 
 	bio->bi_end_io = lbmIODone;
 	bio->bi_private = bp;
-	bio_set_op_attrs(bio, REQ_OP_READ, READ_SYNC);
+	bio->bi_opf = REQ_OP_READ;
 	/*check if journaling to disk has been disabled*/
 	if (log->no_integrity) {
 		bio->bi_iter.bi_size = 0;
@@ -2146,7 +2146,7 @@ static void lbmStartIO(struct lbuf * bp)
 
 	bio->bi_end_io = lbmIODone;
 	bio->bi_private = bp;
-	bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_SYNC);
+	bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
 
 	/* check if journaling to disk has been disabled */
 	if (log->no_integrity) {
diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c
index a1982118f92f..ac9e108ce1ea 100644
--- a/fs/kernfs/inode.c
+++ b/fs/kernfs/inode.c
@@ -335,7 +335,7 @@ static int kernfs_xattr_set(const struct xattr_handler *handler,
 	return simple_xattr_set(&attrs->xattrs, name, value, size, flags);
 }
 
-const struct xattr_handler kernfs_trusted_xattr_handler = {
+static const struct xattr_handler kernfs_trusted_xattr_handler = {
 	.prefix = XATTR_TRUSTED_PREFIX,
 	.get = kernfs_xattr_get,
 	.set = kernfs_xattr_set,
@@ -372,7 +372,7 @@ static int kernfs_security_xattr_set(const struct xattr_handler *handler,
 	return error;
 }
 
-const struct xattr_handler kernfs_security_xattr_handler = {
+static const struct xattr_handler kernfs_security_xattr_handler = {
 	.prefix = XATTR_SECURITY_PREFIX,
 	.get = kernfs_xattr_get,
 	.set = kernfs_security_xattr_set,
diff --git a/fs/logfs/dev_bdev.c b/fs/logfs/dev_bdev.c
index a8329cc47dec..9bfa0151d7c9 100644
--- a/fs/logfs/dev_bdev.c
+++ b/fs/logfs/dev_bdev.c
@@ -19,16 +19,10 @@ static int sync_request(struct page *page, struct block_device *bdev, int op)
 	struct bio bio;
 	struct bio_vec bio_vec;
 
-	bio_init(&bio);
-	bio.bi_max_vecs = 1;
-	bio.bi_io_vec = &bio_vec;
-	bio_vec.bv_page = page;
-	bio_vec.bv_len = PAGE_SIZE;
-	bio_vec.bv_offset = 0;
-	bio.bi_vcnt = 1;
+	bio_init(&bio, &bio_vec, 1);
 	bio.bi_bdev = bdev;
+	bio_add_page(&bio, page, PAGE_SIZE, 0);
 	bio.bi_iter.bi_sector = page->index * (PAGE_SIZE >> 9);
-	bio.bi_iter.bi_size = PAGE_SIZE;
 	bio_set_op_attrs(&bio, op, 0);
 
 	return submit_bio_wait(&bio);
@@ -77,56 +71,45 @@ static int __bdev_writeseg(struct super_block *sb, u64 ofs, pgoff_t index,
 {
 	struct logfs_super *super = logfs_super(sb);
 	struct address_space *mapping = super->s_mapping_inode->i_mapping;
-	struct bio *bio;
+	struct bio *bio = NULL;
 	struct page *page;
 	unsigned int max_pages;
-	int i;
+	int i, ret;
 
 	max_pages = min_t(size_t, nr_pages, BIO_MAX_PAGES);
 
-	bio = bio_alloc(GFP_NOFS, max_pages);
-	BUG_ON(!bio);
-
 	for (i = 0; i < nr_pages; i++) {
-		if (i >= max_pages) {
-			/* Block layer cannot split bios :( */
-			bio->bi_vcnt = i;
-			bio->bi_iter.bi_size = i * PAGE_SIZE;
+		if (!bio) {
+			bio = bio_alloc(GFP_NOFS, max_pages);
+			BUG_ON(!bio);
+
 			bio->bi_bdev = super->s_bdev;
 			bio->bi_iter.bi_sector = ofs >> 9;
 			bio->bi_private = sb;
 			bio->bi_end_io = writeseg_end_io;
 			bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
-			atomic_inc(&super->s_pending_writes);
-			submit_bio(bio);
-
-			ofs += i * PAGE_SIZE;
-			index += i;
-			nr_pages -= i;
-			i = 0;
-
-			bio = bio_alloc(GFP_NOFS, max_pages);
-			BUG_ON(!bio);
 		}
 		page = find_lock_page(mapping, index + i);
 		BUG_ON(!page);
-		bio->bi_io_vec[i].bv_page = page;
-		bio->bi_io_vec[i].bv_len = PAGE_SIZE;
-		bio->bi_io_vec[i].bv_offset = 0;
+		ret = bio_add_page(bio, page, PAGE_SIZE, 0);
 
 		BUG_ON(PageWriteback(page));
 		set_page_writeback(page);
 		unlock_page(page);
+
+		if (!ret) {
+			/* Block layer cannot split bios :( */
+			ofs += bio->bi_iter.bi_size;
+			atomic_inc(&super->s_pending_writes);
+			submit_bio(bio);
+			bio = NULL;
+		}
+	}
+
+	if (bio) {
+		atomic_inc(&super->s_pending_writes);
+		submit_bio(bio);
 	}
-	bio->bi_vcnt = nr_pages;
-	bio->bi_iter.bi_size = nr_pages * PAGE_SIZE;
-	bio->bi_bdev = super->s_bdev;
-	bio->bi_iter.bi_sector = ofs >> 9;
-	bio->bi_private = sb;
-	bio->bi_end_io = writeseg_end_io;
-	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
-	atomic_inc(&super->s_pending_writes);
-	submit_bio(bio);
 	return 0;
 }
 
@@ -160,7 +143,6 @@ static void erase_end_io(struct bio *bio)
 	struct logfs_super *super = logfs_super(sb); 
 
 	BUG_ON(bio->bi_error); /* FIXME: Retry io or write elsewhere */ 
-	BUG_ON(bio->bi_vcnt == 0); 
 	bio_put(bio); 
 	if (atomic_dec_and_test(&super->s_pending_writes))
 		wake_up(&wq); 
@@ -170,49 +152,35 @@ static int do_erase(struct super_block *sb, u64 ofs, pgoff_t index,
 		size_t nr_pages)
 {
 	struct logfs_super *super = logfs_super(sb);
-	struct bio *bio;
+	struct bio *bio = NULL;
 	unsigned int max_pages;
-	int i;
+	int i, ret;
 
 	max_pages = min_t(size_t, nr_pages, BIO_MAX_PAGES);
 
-	bio = bio_alloc(GFP_NOFS, max_pages);
-	BUG_ON(!bio);
-
 	for (i = 0; i < nr_pages; i++) {
-		if (i >= max_pages) {
-			/* Block layer cannot split bios :( */
-			bio->bi_vcnt = i;
-			bio->bi_iter.bi_size = i * PAGE_SIZE;
+		if (!bio) {
+			bio = bio_alloc(GFP_NOFS, max_pages);
+			BUG_ON(!bio);
+
 			bio->bi_bdev = super->s_bdev;
 			bio->bi_iter.bi_sector = ofs >> 9;
 			bio->bi_private = sb;
 			bio->bi_end_io = erase_end_io;
 			bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
+		}
+		ret = bio_add_page(bio, super->s_erase_page, PAGE_SIZE, 0);
+		if (!ret) {
+			/* Block layer cannot split bios :( */
+			ofs += bio->bi_iter.bi_size;
 			atomic_inc(&super->s_pending_writes);
 			submit_bio(bio);
-
-			ofs += i * PAGE_SIZE;
-			index += i;
-			nr_pages -= i;
-			i = 0;
-
-			bio = bio_alloc(GFP_NOFS, max_pages);
-			BUG_ON(!bio);
 		}
-		bio->bi_io_vec[i].bv_page = super->s_erase_page;
-		bio->bi_io_vec[i].bv_len = PAGE_SIZE;
-		bio->bi_io_vec[i].bv_offset = 0;
 	}
-	bio->bi_vcnt = nr_pages;
-	bio->bi_iter.bi_size = nr_pages * PAGE_SIZE;
-	bio->bi_bdev = super->s_bdev;
-	bio->bi_iter.bi_sector = ofs >> 9;
-	bio->bi_private = sb;
-	bio->bi_end_io = erase_end_io;
-	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
-	atomic_inc(&super->s_pending_writes);
-	submit_bio(bio);
+	if (bio) {
+		atomic_inc(&super->s_pending_writes);
+		submit_bio(bio);
+	}
 	return 0;
 }
 
diff --git a/fs/logfs/logfs.h b/fs/logfs/logfs.h
index 27d040e35faa..11209eefbe59 100644
--- a/fs/logfs/logfs.h
+++ b/fs/logfs/logfs.h
@@ -10,9 +10,7 @@
 #ifndef FS_LOGFS_LOGFS_H
 #define FS_LOGFS_LOGFS_H
 
-#undef __CHECK_ENDIAN__
-#define __CHECK_ENDIAN__
-
+#include <linux/types.h>
 #include <linux/btree.h>
 #include <linux/crc32.h>
 #include <linux/fs.h>
diff --git a/fs/mbcache.c b/fs/mbcache.c
index c5bd19ffa326..b19be429d655 100644
--- a/fs/mbcache.c
+++ b/fs/mbcache.c
@@ -29,7 +29,7 @@ struct mb_cache {
 	/* log2 of hash table size */
 	int			c_bucket_bits;
 	/* Maximum entries in cache to avoid degrading hash too much */
-	int			c_max_entries;
+	unsigned long		c_max_entries;
 	/* Protects c_list, c_entry_count */
 	spinlock_t		c_list_lock;
 	struct list_head	c_list;
@@ -43,7 +43,7 @@ struct mb_cache {
 static struct kmem_cache *mb_entry_cache;
 
 static unsigned long mb_cache_shrink(struct mb_cache *cache,
-				     unsigned int nr_to_scan);
+				     unsigned long nr_to_scan);
 
 static inline struct hlist_bl_head *mb_cache_entry_head(struct mb_cache *cache,
 							u32 key)
@@ -155,12 +155,12 @@ out:
 }
 
 /*
- * mb_cache_entry_find_first - find the first entry in cache with given key
+ * mb_cache_entry_find_first - find the first reusable entry with the given key
  * @cache: cache where we should search
  * @key: key to look for
  *
- * Search in @cache for entry with key @key. Grabs reference to the first
- * entry found and returns the entry.
+ * Search in @cache for a reusable entry with key @key. Grabs reference to the
+ * first reusable entry found and returns the entry.
  */
 struct mb_cache_entry *mb_cache_entry_find_first(struct mb_cache *cache,
 						 u32 key)
@@ -170,14 +170,14 @@ struct mb_cache_entry *mb_cache_entry_find_first(struct mb_cache *cache,
 EXPORT_SYMBOL(mb_cache_entry_find_first);
 
 /*
- * mb_cache_entry_find_next - find next entry in cache with the same
+ * mb_cache_entry_find_next - find next reusable entry with the same key
  * @cache: cache where we should search
  * @entry: entry to start search from
  *
- * Finds next entry in the hash chain which has the same key as @entry.
- * If @entry is unhashed (which can happen when deletion of entry races
- * with the search), finds the first entry in the hash chain. The function
- * drops reference to @entry and returns with a reference to the found entry.
+ * Finds next reusable entry in the hash chain which has the same key as @entry.
+ * If @entry is unhashed (which can happen when deletion of entry races with the
+ * search), finds the first reusable entry in the hash chain. The function drops
+ * reference to @entry and returns with a reference to the found entry.
  */
 struct mb_cache_entry *mb_cache_entry_find_next(struct mb_cache *cache,
 						struct mb_cache_entry *entry)
@@ -274,11 +274,11 @@ static unsigned long mb_cache_count(struct shrinker *shrink,
 
 /* Shrink number of entries in cache */
 static unsigned long mb_cache_shrink(struct mb_cache *cache,
-				     unsigned int nr_to_scan)
+				     unsigned long nr_to_scan)
 {
 	struct mb_cache_entry *entry;
 	struct hlist_bl_head *head;
-	unsigned int shrunk = 0;
+	unsigned long shrunk = 0;
 
 	spin_lock(&cache->c_list_lock);
 	while (nr_to_scan-- && !list_empty(&cache->c_list)) {
@@ -286,7 +286,7 @@ static unsigned long mb_cache_shrink(struct mb_cache *cache,
 					 struct mb_cache_entry, e_list);
 		if (entry->e_referenced) {
 			entry->e_referenced = 0;
-			list_move_tail(&cache->c_list, &entry->e_list);
+			list_move_tail(&entry->e_list, &cache->c_list);
 			continue;
 		}
 		list_del_init(&entry->e_list);
@@ -316,10 +316,9 @@ static unsigned long mb_cache_shrink(struct mb_cache *cache,
 static unsigned long mb_cache_scan(struct shrinker *shrink,
 				   struct shrink_control *sc)
 {
-	int nr_to_scan = sc->nr_to_scan;
 	struct mb_cache *cache = container_of(shrink, struct mb_cache,
 					      c_shrink);
-	return mb_cache_shrink(cache, nr_to_scan);
+	return mb_cache_shrink(cache, sc->nr_to_scan);
 }
 
 /* We shrink 1/X of the cache when we have too many entries in it */
@@ -341,11 +340,8 @@ static void mb_cache_shrink_worker(struct work_struct *work)
 struct mb_cache *mb_cache_create(int bucket_bits)
 {
 	struct mb_cache *cache;
-	int bucket_count = 1 << bucket_bits;
-	int i;
-
-	if (!try_module_get(THIS_MODULE))
-		return NULL;
+	unsigned long bucket_count = 1UL << bucket_bits;
+	unsigned long i;
 
 	cache = kzalloc(sizeof(struct mb_cache), GFP_KERNEL);
 	if (!cache)
@@ -377,7 +373,6 @@ struct mb_cache *mb_cache_create(int bucket_bits)
 	return cache;
 
 err_out:
-	module_put(THIS_MODULE);
 	return NULL;
 }
 EXPORT_SYMBOL(mb_cache_create);
@@ -411,7 +406,6 @@ void mb_cache_destroy(struct mb_cache *cache)
 	}
 	kfree(cache->c_hash);
 	kfree(cache);
-	module_put(THIS_MODULE);
 }
 EXPORT_SYMBOL(mb_cache_destroy);
 
@@ -420,7 +414,8 @@ static int __init mbcache_init(void)
 	mb_entry_cache = kmem_cache_create("mbcache",
 				sizeof(struct mb_cache_entry), 0,
 				SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD, NULL);
-	BUG_ON(!mb_entry_cache);
+	if (!mb_entry_cache)
+		return -ENOMEM;
 	return 0;
 }
 
diff --git a/fs/mpage.c b/fs/mpage.c
index d2413af0823a..28af984a3d96 100644
--- a/fs/mpage.c
+++ b/fs/mpage.c
@@ -489,7 +489,7 @@ static int __mpage_writepage(struct page *page, struct writeback_control *wbc,
 	struct buffer_head map_bh;
 	loff_t i_size = i_size_read(inode);
 	int ret = 0;
-	int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?  WRITE_SYNC : 0);
+	int op_flags = wbc_to_write_flags(wbc);
 
 	if (page_has_buffers(page)) {
 		struct buffer_head *head = page_buffers(page);
@@ -555,8 +555,7 @@ static int __mpage_writepage(struct page *page, struct writeback_control *wbc,
 		if (mpd->get_block(inode, block_in_file, &map_bh, 1))
 			goto confused;
 		if (buffer_new(&map_bh))
-			unmap_underlying_metadata(map_bh.b_bdev,
-						map_bh.b_blocknr);
+			clean_bdev_bh_alias(&map_bh);
 		if (buffer_boundary(&map_bh)) {
 			boundary_block = map_bh.b_blocknr;
 			boundary_bdev = map_bh.b_bdev;
@@ -705,7 +704,7 @@ mpage_writepages(struct address_space *mapping,
 		ret = write_cache_pages(mapping, wbc, __mpage_writepage, &mpd);
 		if (mpd.bio) {
 			int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?
-				  WRITE_SYNC : 0);
+				  REQ_SYNC : 0);
 			mpage_bio_submit(REQ_OP_WRITE, op_flags, mpd.bio);
 		}
 	}
@@ -726,7 +725,7 @@ int mpage_writepage(struct page *page, get_block_t get_block,
 	int ret = __mpage_writepage(page, wbc, &mpd);
 	if (mpd.bio) {
 		int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?
-			  WRITE_SYNC : 0);
+			  REQ_SYNC : 0);
 		mpage_bio_submit(REQ_OP_WRITE, op_flags, mpd.bio);
 	}
 	return ret;
diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index e9aa235e9d10..f073a6d2c6a5 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -110,20 +110,52 @@ out:
 #if defined(CONFIG_NFS_V4_1)
 
 /*
- * Lookup a layout by filehandle.
+ * Lookup a layout inode by stateid
  *
- * Note: gets a refcount on the layout hdr and on its respective inode.
- * Caller must put the layout hdr and the inode.
+ * Note: returns a refcount on the inode and superblock
+ */
+static struct inode *nfs_layout_find_inode_by_stateid(struct nfs_client *clp,
+		const nfs4_stateid *stateid)
+{
+	struct nfs_server *server;
+	struct inode *inode;
+	struct pnfs_layout_hdr *lo;
+
+restart:
+	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
+		list_for_each_entry(lo, &server->layouts, plh_layouts) {
+			if (stateid != NULL &&
+			    !nfs4_stateid_match_other(stateid, &lo->plh_stateid))
+				continue;
+			inode = igrab(lo->plh_inode);
+			if (!inode)
+				continue;
+			if (!nfs_sb_active(inode->i_sb)) {
+				rcu_read_lock();
+				spin_unlock(&clp->cl_lock);
+				iput(inode);
+				spin_lock(&clp->cl_lock);
+				goto restart;
+			}
+			return inode;
+		}
+	}
+
+	return NULL;
+}
+
+/*
+ * Lookup a layout inode by filehandle.
+ *
+ * Note: returns a refcount on the inode and superblock
  *
- * TODO: keep track of all layouts (and delegations) in a hash table
- * hashed by filehandle.
  */
-static struct pnfs_layout_hdr * get_layout_by_fh_locked(struct nfs_client *clp,
-		struct nfs_fh *fh)
+static struct inode *nfs_layout_find_inode_by_fh(struct nfs_client *clp,
+		const struct nfs_fh *fh)
 {
 	struct nfs_server *server;
 	struct nfs_inode *nfsi;
-	struct inode *ino;
+	struct inode *inode;
 	struct pnfs_layout_hdr *lo;
 
 restart:
@@ -134,37 +166,38 @@ restart:
 				continue;
 			if (nfsi->layout != lo)
 				continue;
-			ino = igrab(lo->plh_inode);
-			if (!ino)
-				break;
-			spin_lock(&ino->i_lock);
-			/* Is this layout in the process of being freed? */
-			if (nfsi->layout != lo) {
-				spin_unlock(&ino->i_lock);
-				iput(ino);
+			inode = igrab(lo->plh_inode);
+			if (!inode)
+				continue;
+			if (!nfs_sb_active(inode->i_sb)) {
+				rcu_read_lock();
+				spin_unlock(&clp->cl_lock);
+				iput(inode);
+				spin_lock(&clp->cl_lock);
 				goto restart;
 			}
-			pnfs_get_layout_hdr(lo);
-			spin_unlock(&ino->i_lock);
-			return lo;
+			return inode;
 		}
 	}
 
 	return NULL;
 }
 
-static struct pnfs_layout_hdr * get_layout_by_fh(struct nfs_client *clp,
-		struct nfs_fh *fh)
+static struct inode *nfs_layout_find_inode(struct nfs_client *clp,
+		const struct nfs_fh *fh,
+		const nfs4_stateid *stateid)
 {
-	struct pnfs_layout_hdr *lo;
+	struct inode *inode;
 
 	spin_lock(&clp->cl_lock);
 	rcu_read_lock();
-	lo = get_layout_by_fh_locked(clp, fh);
+	inode = nfs_layout_find_inode_by_stateid(clp, stateid);
+	if (!inode)
+		inode = nfs_layout_find_inode_by_fh(clp, fh);
 	rcu_read_unlock();
 	spin_unlock(&clp->cl_lock);
 
-	return lo;
+	return inode;
 }
 
 /*
@@ -213,18 +246,20 @@ static u32 initiate_file_draining(struct nfs_client *clp,
 	u32 rv = NFS4ERR_NOMATCHING_LAYOUT;
 	LIST_HEAD(free_me_list);
 
-	lo = get_layout_by_fh(clp, &args->cbl_fh);
-	if (!lo) {
-		trace_nfs4_cb_layoutrecall_file(clp, &args->cbl_fh, NULL,
-				&args->cbl_stateid, -rv);
+	ino = nfs_layout_find_inode(clp, &args->cbl_fh, &args->cbl_stateid);
+	if (!ino)
 		goto out;
-	}
 
-	ino = lo->plh_inode;
 	pnfs_layoutcommit_inode(ino, false);
 
 
 	spin_lock(&ino->i_lock);
+	lo = NFS_I(ino)->layout;
+	if (!lo) {
+		spin_unlock(&ino->i_lock);
+		goto out;
+	}
+	pnfs_get_layout_hdr(lo);
 	rv = pnfs_check_callback_stateid(lo, &args->cbl_stateid);
 	if (rv != NFS_OK)
 		goto unlock;
@@ -258,10 +293,10 @@ unlock:
 	/* Free all lsegs that are attached to commit buckets */
 	nfs_commit_inode(ino, 0);
 	pnfs_put_layout_hdr(lo);
+out:
 	trace_nfs4_cb_layoutrecall_file(clp, &args->cbl_fh, ino,
 			&args->cbl_stateid, -rv);
-	iput(ino);
-out:
+	nfs_iput_and_deactive(ino);
 	return rv;
 }
 
diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index ebecfb8fba06..91a8d610ba0f 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -369,9 +369,7 @@ nfs_found_client(const struct nfs_client_initdata *cl_init,
  * Look up a client by IP address and protocol version
  * - creates a new record if one doesn't yet exist
  */
-struct nfs_client *
-nfs_get_client(const struct nfs_client_initdata *cl_init,
-	       rpc_authflavor_t authflavour)
+struct nfs_client *nfs_get_client(const struct nfs_client_initdata *cl_init)
 {
 	struct nfs_client *clp, *new = NULL;
 	struct nfs_net *nn = net_generic(cl_init->net, nfs_net_id);
@@ -655,7 +653,7 @@ static int nfs_init_server(struct nfs_server *server,
 		set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags);
 
 	/* Allocate or find a client reference we can use */
-	clp = nfs_get_client(&cl_init, RPC_AUTH_UNIX);
+	clp = nfs_get_client(&cl_init);
 	if (IS_ERR(clp)) {
 		dprintk("<-- nfs_init_server() = error %ld\n", PTR_ERR(clp));
 		return PTR_ERR(clp);
diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c
index dff600ae0d74..d7df5e67b0c1 100644
--- a/fs/nfs/delegation.c
+++ b/fs/nfs/delegation.c
@@ -391,10 +391,6 @@ int nfs_inode_set_delegation(struct inode *inode, struct rpc_cred *cred, struct
 	rcu_assign_pointer(nfsi->delegation, delegation);
 	delegation = NULL;
 
-	/* Ensure we revalidate the attributes and page cache! */
-	spin_lock(&inode->i_lock);
-	nfsi->cache_validity |= NFS_INO_REVAL_FORCED;
-	spin_unlock(&inode->i_lock);
 	trace_nfs4_set_delegation(inode, res->delegation_type);
 
 out:
diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
index 5f1af4cd1a33..cb22a9f9ae7e 100644
--- a/fs/nfs/dir.c
+++ b/fs/nfs/dir.c
@@ -455,14 +455,17 @@ bool nfs_use_readdirplus(struct inode *dir, struct dir_context *ctx)
 }
 
 /*
- * This function is called by the lookup code to request the use of
- * readdirplus to accelerate any future lookups in the same
+ * This function is called by the lookup and getattr code to request the
+ * use of readdirplus to accelerate any future lookups in the same
  * directory.
  */
-static
 void nfs_advise_use_readdirplus(struct inode *dir)
 {
-	set_bit(NFS_INO_ADVISE_RDPLUS, &NFS_I(dir)->flags);
+	struct nfs_inode *nfsi = NFS_I(dir);
+
+	if (nfs_server_capable(dir, NFS_CAP_READDIRPLUS) &&
+	    !list_empty(&nfsi->open_files))
+		set_bit(NFS_INO_ADVISE_RDPLUS, &nfsi->flags);
 }
 
 /*
@@ -475,9 +478,12 @@ void nfs_advise_use_readdirplus(struct inode *dir)
  */
 void nfs_force_use_readdirplus(struct inode *dir)
 {
-	if (!list_empty(&NFS_I(dir)->open_files)) {
-		nfs_advise_use_readdirplus(dir);
-		nfs_zap_mapping(dir, dir->i_mapping);
+	struct nfs_inode *nfsi = NFS_I(dir);
+
+	if (nfs_server_capable(dir, NFS_CAP_READDIRPLUS) &&
+	    !list_empty(&nfsi->open_files)) {
+		set_bit(NFS_INO_ADVISE_RDPLUS, &nfsi->flags);
+		invalidate_mapping_pages(dir->i_mapping, 0, -1);
 	}
 }
 
@@ -886,17 +892,6 @@ int uncached_readdir(nfs_readdir_descriptor_t *desc)
 	goto out;
 }
 
-static bool nfs_dir_mapping_need_revalidate(struct inode *dir)
-{
-	struct nfs_inode *nfsi = NFS_I(dir);
-
-	if (nfs_attribute_cache_expired(dir))
-		return true;
-	if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
-		return true;
-	return false;
-}
-
 /* The file offset position represents the dirent entry number.  A
    last cookie cache takes care of the common case of reading the
    whole directory.
@@ -928,7 +923,7 @@ static int nfs_readdir(struct file *file, struct dir_context *ctx)
 	desc->decode = NFS_PROTO(inode)->decode_dirent;
 	desc->plus = nfs_use_readdirplus(inode, ctx) ? 1 : 0;
 
-	if (ctx->pos == 0 || nfs_dir_mapping_need_revalidate(inode))
+	if (ctx->pos == 0 || nfs_attribute_cache_expired(inode))
 		res = nfs_revalidate_mapping(inode, file->f_mapping);
 	if (res < 0)
 		goto out;
@@ -1035,8 +1030,6 @@ EXPORT_SYMBOL_GPL(nfs_force_lookup_revalidate);
 static int nfs_check_verifier(struct inode *dir, struct dentry *dentry,
 			      int rcu_walk)
 {
-	int ret;
-
 	if (IS_ROOT(dentry))
 		return 1;
 	if (NFS_SERVER(dir)->flags & NFS_MOUNT_LOOKUP_CACHE_NONE)
@@ -1044,12 +1037,12 @@ static int nfs_check_verifier(struct inode *dir, struct dentry *dentry,
 	if (!nfs_verify_change_attribute(dir, dentry->d_time))
 		return 0;
 	/* Revalidate nfsi->cache_change_attribute before we declare a match */
-	if (rcu_walk)
-		ret = nfs_revalidate_inode_rcu(NFS_SERVER(dir), dir);
-	else
-		ret = nfs_revalidate_inode(NFS_SERVER(dir), dir);
-	if (ret < 0)
-		return 0;
+	if (nfs_mapping_need_revalidate_inode(dir)) {
+		if (rcu_walk)
+			return 0;
+		if (__nfs_revalidate_inode(NFS_SERVER(dir), dir) < 0)
+			return 0;
+	}
 	if (!nfs_verify_change_attribute(dir, dentry->d_time))
 		return 0;
 	return 1;
@@ -1161,7 +1154,7 @@ static int nfs_lookup_revalidate(struct dentry *dentry, unsigned int flags)
 				return -ECHILD;
 			goto out_bad;
 		}
-		goto out_valid_noent;
+		goto out_valid;
 	}
 
 	if (is_bad_inode(inode)) {
@@ -1184,6 +1177,7 @@ static int nfs_lookup_revalidate(struct dentry *dentry, unsigned int flags)
 				return -ECHILD;
 			goto out_zap_parent;
 		}
+		nfs_advise_use_readdirplus(dir);
 		goto out_valid;
 	}
 
@@ -1219,12 +1213,12 @@ static int nfs_lookup_revalidate(struct dentry *dentry, unsigned int flags)
 	nfs_free_fhandle(fhandle);
 	nfs4_label_free(label);
 
+	/* set a readdirplus hint that we had a cache miss */
+	nfs_force_use_readdirplus(dir);
+
 out_set_verifier:
 	nfs_set_verifier(dentry, nfs_save_change_attribute(dir));
  out_valid:
-	/* Success: notify readdir to use READDIRPLUS */
-	nfs_advise_use_readdirplus(dir);
- out_valid_noent:
 	if (flags & LOOKUP_RCU) {
 		if (parent != ACCESS_ONCE(dentry->d_parent))
 			return -ECHILD;
@@ -1424,8 +1418,8 @@ struct dentry *nfs_lookup(struct inode *dir, struct dentry * dentry, unsigned in
 	if (IS_ERR(res))
 		goto out_label;
 
-	/* Success: notify readdir to use READDIRPLUS */
-	nfs_advise_use_readdirplus(dir);
+	/* Notify readdir to use READDIRPLUS */
+	nfs_force_use_readdirplus(dir);
 
 no_entry:
 	res = d_splice_alias(inode, dentry);
@@ -1467,9 +1461,9 @@ static fmode_t flags_to_mode(int flags)
 	return res;
 }
 
-static struct nfs_open_context *create_nfs_open_context(struct dentry *dentry, int open_flags)
+static struct nfs_open_context *create_nfs_open_context(struct dentry *dentry, int open_flags, struct file *filp)
 {
-	return alloc_nfs_open_context(dentry, flags_to_mode(open_flags));
+	return alloc_nfs_open_context(dentry, flags_to_mode(open_flags), filp);
 }
 
 static int do_open(struct inode *inode, struct file *filp)
@@ -1535,8 +1529,13 @@ int nfs_atomic_open(struct inode *dir, struct dentry *dentry,
 		return -ENAMETOOLONG;
 
 	if (open_flags & O_CREAT) {
+		struct nfs_server *server = NFS_SERVER(dir);
+
+		if (!(server->attr_bitmask[2] & FATTR4_WORD2_MODE_UMASK))
+			mode &= ~current_umask();
+
 		attr.ia_valid |= ATTR_MODE;
-		attr.ia_mode = mode & ~current_umask();
+		attr.ia_mode = mode;
 	}
 	if (open_flags & O_TRUNC) {
 		attr.ia_valid |= ATTR_SIZE;
@@ -1554,7 +1553,7 @@ int nfs_atomic_open(struct inode *dir, struct dentry *dentry,
 			return finish_no_open(file, dentry);
 	}
 
-	ctx = create_nfs_open_context(dentry, open_flags);
+	ctx = create_nfs_open_context(dentry, open_flags, file);
 	err = PTR_ERR(ctx);
 	if (IS_ERR(ctx))
 		goto out;
diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c
index bd81bcf3ffcf..be88bcdca692 100644
--- a/fs/nfs/direct.c
+++ b/fs/nfs/direct.c
@@ -105,7 +105,7 @@ struct nfs_direct_req {
 
 static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
 static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;
-static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode);
+static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
 static void nfs_direct_write_schedule_work(struct work_struct *work);
 
 static inline void get_dreq(struct nfs_direct_req *dreq)
@@ -684,7 +684,7 @@ out_failed:
 	}
 
 	if (put_dreq(dreq))
-		nfs_direct_write_complete(dreq, dreq->inode);
+		nfs_direct_write_complete(dreq);
 }
 
 static void nfs_direct_commit_complete(struct nfs_commit_data *data)
@@ -717,7 +717,7 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data)
 	}
 
 	if (atomic_dec_and_test(&cinfo.mds->rpcs_out))
-		nfs_direct_write_complete(dreq, data->inode);
+		nfs_direct_write_complete(dreq);
 }
 
 static void nfs_direct_resched_write(struct nfs_commit_info *cinfo,
@@ -768,7 +768,7 @@ static void nfs_direct_write_schedule_work(struct work_struct *work)
 	}
 }
 
-static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
+static void nfs_direct_write_complete(struct nfs_direct_req *dreq)
 {
 	schedule_work(&dreq->work); /* Calls nfs_direct_write_schedule_work */
 }
@@ -824,7 +824,7 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
 
 out_put:
 	if (put_dreq(dreq))
-		nfs_direct_write_complete(dreq, hdr->inode);
+		nfs_direct_write_complete(dreq);
 	hdr->release(hdr);
 }
 
@@ -953,7 +953,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
 	}
 
 	if (put_dreq(dreq))
-		nfs_direct_write_complete(dreq, dreq->inode);
+		nfs_direct_write_complete(dreq);
 	return 0;
 }
 
diff --git a/fs/nfs/file.c b/fs/nfs/file.c
index 9ea85ae23c32..64c11f399b3d 100644
--- a/fs/nfs/file.c
+++ b/fs/nfs/file.c
@@ -102,8 +102,11 @@ static int nfs_revalidate_file_size(struct inode *inode, struct file *filp)
 {
 	struct nfs_server *server = NFS_SERVER(inode);
 	struct nfs_inode *nfsi = NFS_I(inode);
+	const unsigned long force_reval = NFS_INO_REVAL_PAGECACHE|NFS_INO_REVAL_FORCED;
+	unsigned long cache_validity = nfsi->cache_validity;
 
-	if (nfs_have_delegated_attributes(inode))
+	if (NFS_PROTO(inode)->have_delegation(inode, FMODE_READ) &&
+	    (cache_validity & force_reval) != force_reval)
 		goto out_noreval;
 
 	if (filp->f_flags & O_DIRECT)
diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c
index 4946ef40ba87..a5589b791439 100644
--- a/fs/nfs/filelayout/filelayoutdev.c
+++ b/fs/nfs/filelayout/filelayoutdev.c
@@ -279,8 +279,7 @@ nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
 
 	nfs4_pnfs_ds_connect(s, ds, devid, dataserver_timeo,
 			     dataserver_retrans, 4,
-			     s->nfs_client->cl_minorversion,
-			     s->nfs_client->cl_rpcclient->cl_auth->au_flavor);
+			     s->nfs_client->cl_minorversion);
 
 out_test_devid:
 	if (filelayout_test_devid_unavailable(devid))
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index 98ace127bf86..9e111d07f667 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -25,9 +25,20 @@
 #define NFSDBG_FACILITY         NFSDBG_PNFS_LD
 
 #define FF_LAYOUT_POLL_RETRY_MAX     (15*HZ)
+#define FF_LAYOUTRETURN_MAXERR 20
+
 
 static struct group_info	*ff_zero_group;
 
+static void ff_layout_read_record_layoutstats_done(struct rpc_task *task,
+		struct nfs_pgio_header *hdr);
+static int ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
+			       struct nfs42_layoutstat_devinfo *devinfo,
+			       int dev_limit);
+static void ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr,
+			      const struct nfs42_layoutstat_devinfo *devinfo,
+			      struct nfs4_ff_layout_mirror *mirror);
+
 static struct pnfs_layout_hdr *
 ff_layout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags)
 {
@@ -172,7 +183,7 @@ ff_layout_add_mirror(struct pnfs_layout_hdr *lo,
 
 	spin_lock(&inode->i_lock);
 	list_for_each_entry(pos, &ff_layout->mirrors, mirrors) {
-		if (mirror->mirror_ds != pos->mirror_ds)
+		if (memcmp(&mirror->devid, &pos->devid, sizeof(pos->devid)) != 0)
 			continue;
 		if (!ff_mirror_match_fh(mirror, pos))
 			continue;
@@ -349,19 +360,6 @@ static void ff_layout_sort_mirrors(struct nfs4_ff_layout_segment *fls)
 	}
 }
 
-static void ff_layout_mark_devices_valid(struct nfs4_ff_layout_segment *fls)
-{
-	struct nfs4_deviceid_node *node;
-	int i;
-
-	if (!(fls->flags & FF_FLAGS_NO_IO_THRU_MDS))
-		return;
-	for (i = 0; i < fls->mirror_array_cnt; i++) {
-		node = &fls->mirror_array[i]->mirror_ds->id_node;
-		clear_bit(NFS_DEVICEID_UNAVAILABLE, &node->flags);
-	}
-}
-
 static struct pnfs_layout_segment *
 ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
 		     struct nfs4_layoutget_res *lgr,
@@ -415,8 +413,6 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
 
 	for (i = 0; i < fls->mirror_array_cnt; i++) {
 		struct nfs4_ff_layout_mirror *mirror;
-		struct nfs4_deviceid devid;
-		struct nfs4_deviceid_node *idnode;
 		struct auth_cred acred = { .group_info = ff_zero_group };
 		struct rpc_cred	__rcu *cred;
 		u32 ds_count, fh_count, id;
@@ -441,24 +437,10 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
 		fls->mirror_array[i]->ds_count = ds_count;
 
 		/* deviceid */
-		rc = decode_deviceid(&stream, &devid);
+		rc = decode_deviceid(&stream, &fls->mirror_array[i]->devid);
 		if (rc)
 			goto out_err_free;
 
-		idnode = nfs4_find_get_deviceid(NFS_SERVER(lh->plh_inode),
-						&devid, lh->plh_lc_cred,
-						gfp_flags);
-		/*
-		 * upon success, mirror_ds is allocated by previous
-		 * getdeviceinfo, or newly by .alloc_deviceid_node
-		 * nfs4_find_get_deviceid failure is indeed getdeviceinfo falure
-		 */
-		if (idnode)
-			fls->mirror_array[i]->mirror_ds =
-				FF_LAYOUT_MIRROR_DS(idnode);
-		else
-			goto out_err_free;
-
 		/* efficiency */
 		rc = -EIO;
 		p = xdr_inline_decode(&stream, 4);
@@ -556,8 +538,6 @@ out_sort_mirrors:
 	rc = ff_layout_check_layout(lgr);
 	if (rc)
 		goto out_err_free;
-	ff_layout_mark_devices_valid(fls);
-
 	ret = &fls->generic_hdr;
 	dprintk("<-- %s (success)\n", __func__);
 out_free_page:
@@ -702,6 +682,7 @@ nfs4_ff_layout_stat_io_start_read(struct inode *inode,
 	spin_lock(&mirror->lock);
 	report = nfs4_ff_layoutstat_start_io(mirror, &mirror->read_stat, now);
 	nfs4_ff_layout_stat_io_update_requested(&mirror->read_stat, requested);
+	set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
 	spin_unlock(&mirror->lock);
 
 	if (report)
@@ -718,6 +699,7 @@ nfs4_ff_layout_stat_io_end_read(struct rpc_task *task,
 	nfs4_ff_layout_stat_io_update_completed(&mirror->read_stat,
 			requested, completed,
 			ktime_get(), task->tk_start);
+	set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
 	spin_unlock(&mirror->lock);
 }
 
@@ -731,6 +713,7 @@ nfs4_ff_layout_stat_io_start_write(struct inode *inode,
 	spin_lock(&mirror->lock);
 	report = nfs4_ff_layoutstat_start_io(mirror , &mirror->write_stat, now);
 	nfs4_ff_layout_stat_io_update_requested(&mirror->write_stat, requested);
+	set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
 	spin_unlock(&mirror->lock);
 
 	if (report)
@@ -750,6 +733,7 @@ nfs4_ff_layout_stat_io_end_write(struct rpc_task *task,
 	spin_lock(&mirror->lock);
 	nfs4_ff_layout_stat_io_update_completed(&mirror->write_stat,
 			requested, completed, ktime_get(), task->tk_start);
+	set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
 	spin_unlock(&mirror->lock);
 }
 
@@ -1293,6 +1277,7 @@ static int ff_layout_read_done_cb(struct rpc_task *task,
 					hdr->pgio_mirror_idx + 1,
 					&hdr->pgio_mirror_idx))
 			goto out_eagain;
+		ff_layout_read_record_layoutstats_done(task, hdr);
 		pnfs_read_resend_pnfs(hdr);
 		return task->tk_status;
 	case -NFS4ERR_RESET_TO_MDS:
@@ -1961,38 +1946,88 @@ ff_layout_free_deviceid_node(struct nfs4_deviceid_node *d)
 						  id_node));
 }
 
-static int ff_layout_encode_ioerr(struct nfs4_flexfile_layout *flo,
-				  struct xdr_stream *xdr,
-				  const struct nfs4_layoutreturn_args *args)
+static int ff_layout_encode_ioerr(struct xdr_stream *xdr,
+				  const struct nfs4_layoutreturn_args *args,
+				  const struct nfs4_flexfile_layoutreturn_args *ff_args)
 {
-	struct pnfs_layout_hdr *hdr = &flo->generic_hdr;
 	__be32 *start;
-	int count = 0, ret = 0;
 
 	start = xdr_reserve_space(xdr, 4);
 	if (unlikely(!start))
 		return -E2BIG;
 
+	*start = cpu_to_be32(ff_args->num_errors);
 	/* This assume we always return _ALL_ layouts */
-	spin_lock(&hdr->plh_inode->i_lock);
-	ret = ff_layout_encode_ds_ioerr(flo, xdr, &count, &args->range);
-	spin_unlock(&hdr->plh_inode->i_lock);
+	return ff_layout_encode_ds_ioerr(xdr, &ff_args->errors);
+}
 
-	*start = cpu_to_be32(count);
+static void
+encode_opaque_fixed(struct xdr_stream *xdr, const void *buf, size_t len)
+{
+	__be32 *p;
 
-	return ret;
+	p = xdr_reserve_space(xdr, len);
+	xdr_encode_opaque_fixed(p, buf, len);
+}
+
+static void
+ff_layout_encode_ff_iostat_head(struct xdr_stream *xdr,
+			    const nfs4_stateid *stateid,
+			    const struct nfs42_layoutstat_devinfo *devinfo)
+{
+	__be32 *p;
+
+	p = xdr_reserve_space(xdr, 8 + 8);
+	p = xdr_encode_hyper(p, devinfo->offset);
+	p = xdr_encode_hyper(p, devinfo->length);
+	encode_opaque_fixed(xdr, stateid->data, NFS4_STATEID_SIZE);
+	p = xdr_reserve_space(xdr, 4*8);
+	p = xdr_encode_hyper(p, devinfo->read_count);
+	p = xdr_encode_hyper(p, devinfo->read_bytes);
+	p = xdr_encode_hyper(p, devinfo->write_count);
+	p = xdr_encode_hyper(p, devinfo->write_bytes);
+	encode_opaque_fixed(xdr, devinfo->dev_id.data, NFS4_DEVICEID4_SIZE);
+}
+
+static void
+ff_layout_encode_ff_iostat(struct xdr_stream *xdr,
+			    const nfs4_stateid *stateid,
+			    const struct nfs42_layoutstat_devinfo *devinfo)
+{
+	ff_layout_encode_ff_iostat_head(xdr, stateid, devinfo);
+	ff_layout_encode_ff_layoutupdate(xdr, devinfo,
+			devinfo->ld_private.data);
 }
 
 /* report nothing for now */
-static void ff_layout_encode_iostats(struct nfs4_flexfile_layout *flo,
-				     struct xdr_stream *xdr,
-				     const struct nfs4_layoutreturn_args *args)
+static void ff_layout_encode_iostats_array(struct xdr_stream *xdr,
+		const struct nfs4_layoutreturn_args *args,
+		struct nfs4_flexfile_layoutreturn_args *ff_args)
 {
 	__be32 *p;
+	int i;
 
 	p = xdr_reserve_space(xdr, 4);
-	if (likely(p))
-		*p = cpu_to_be32(0);
+	*p = cpu_to_be32(ff_args->num_dev);
+	for (i = 0; i < ff_args->num_dev; i++)
+		ff_layout_encode_ff_iostat(xdr,
+				&args->layout->plh_stateid,
+				&ff_args->devinfo[i]);
+}
+
+static void
+ff_layout_free_iostats_array(struct nfs42_layoutstat_devinfo *devinfo,
+		unsigned int num_entries)
+{
+	unsigned int i;
+
+	for (i = 0; i < num_entries; i++) {
+		if (!devinfo[i].ld_private.ops)
+			continue;
+		if (!devinfo[i].ld_private.ops->free)
+			continue;
+		devinfo[i].ld_private.ops->free(&devinfo[i].ld_private);
+	}
 }
 
 static struct nfs4_deviceid_node *
@@ -2008,24 +2043,91 @@ ff_layout_alloc_deviceid_node(struct nfs_server *server,
 }
 
 static void
-ff_layout_encode_layoutreturn(struct pnfs_layout_hdr *lo,
-			      struct xdr_stream *xdr,
-			      const struct nfs4_layoutreturn_args *args)
-{
-	struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo);
+ff_layout_encode_layoutreturn(struct xdr_stream *xdr,
+		const void *voidargs,
+		const struct nfs4_xdr_opaque_data *ff_opaque)
+{
+	const struct nfs4_layoutreturn_args *args = voidargs;
+	struct nfs4_flexfile_layoutreturn_args *ff_args = ff_opaque->data;
+	struct xdr_buf tmp_buf = {
+		.head = {
+			[0] = {
+				.iov_base = page_address(ff_args->pages[0]),
+			},
+		},
+		.buflen = PAGE_SIZE,
+	};
+	struct xdr_stream tmp_xdr;
 	__be32 *start;
 
 	dprintk("%s: Begin\n", __func__);
-	start = xdr_reserve_space(xdr, 4);
-	BUG_ON(!start);
 
-	ff_layout_encode_ioerr(flo, xdr, args);
-	ff_layout_encode_iostats(flo, xdr, args);
+	xdr_init_encode(&tmp_xdr, &tmp_buf, NULL);
+
+	ff_layout_encode_ioerr(&tmp_xdr, args, ff_args);
+	ff_layout_encode_iostats_array(&tmp_xdr, args, ff_args);
+
+	start = xdr_reserve_space(xdr, 4);
+	*start = cpu_to_be32(tmp_buf.len);
+	xdr_write_pages(xdr, ff_args->pages, 0, tmp_buf.len);
 
-	*start = cpu_to_be32((xdr->p - start - 1) * 4);
 	dprintk("%s: Return\n", __func__);
 }
 
+static void
+ff_layout_free_layoutreturn(struct nfs4_xdr_opaque_data *args)
+{
+	struct nfs4_flexfile_layoutreturn_args *ff_args;
+
+	if (!args->data)
+		return;
+	ff_args = args->data;
+	args->data = NULL;
+
+	ff_layout_free_ds_ioerr(&ff_args->errors);
+	ff_layout_free_iostats_array(ff_args->devinfo, ff_args->num_dev);
+
+	put_page(ff_args->pages[0]);
+	kfree(ff_args);
+}
+
+const struct nfs4_xdr_opaque_ops layoutreturn_ops = {
+	.encode = ff_layout_encode_layoutreturn,
+	.free = ff_layout_free_layoutreturn,
+};
+
+static int
+ff_layout_prepare_layoutreturn(struct nfs4_layoutreturn_args *args)
+{
+	struct nfs4_flexfile_layoutreturn_args *ff_args;
+	struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(args->layout);
+
+	ff_args = kmalloc(sizeof(*ff_args), GFP_KERNEL);
+	if (!ff_args)
+		goto out_nomem;
+	ff_args->pages[0] = alloc_page(GFP_KERNEL);
+	if (!ff_args->pages[0])
+		goto out_nomem_free;
+
+	INIT_LIST_HEAD(&ff_args->errors);
+	ff_args->num_errors = ff_layout_fetch_ds_ioerr(args->layout,
+			&args->range, &ff_args->errors,
+			FF_LAYOUTRETURN_MAXERR);
+
+	spin_lock(&args->inode->i_lock);
+	ff_args->num_dev = ff_layout_mirror_prepare_stats(&ff_layout->generic_hdr,
+			&ff_args->devinfo[0], ARRAY_SIZE(ff_args->devinfo));
+	spin_unlock(&args->inode->i_lock);
+
+	args->ld_private->ops = &layoutreturn_ops;
+	args->ld_private->data = ff_args;
+	return 0;
+out_nomem_free:
+	kfree(ff_args);
+out_nomem:
+	return -ENOMEM;
+}
+
 static int
 ff_layout_ntop4(const struct sockaddr *sap, char *buf, const size_t buflen)
 {
@@ -2146,21 +2248,18 @@ ff_layout_encode_io_latency(struct xdr_stream *xdr,
 }
 
 static void
-ff_layout_encode_layoutstats(struct xdr_stream *xdr,
-			     struct nfs42_layoutstat_args *args,
-			     struct nfs42_layoutstat_devinfo *devinfo)
+ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr,
+			      const struct nfs42_layoutstat_devinfo *devinfo,
+			      struct nfs4_ff_layout_mirror *mirror)
 {
-	struct nfs4_ff_layout_mirror *mirror = devinfo->layout_private;
 	struct nfs4_pnfs_ds_addr *da;
 	struct nfs4_pnfs_ds *ds = mirror->mirror_ds->ds;
 	struct nfs_fh *fh = &mirror->fh_versions[0];
-	__be32 *p, *start;
+	__be32 *p;
 
 	da = list_first_entry(&ds->ds_addrs, struct nfs4_pnfs_ds_addr, da_node);
 	dprintk("%s: DS %s: encoding address %s\n",
 		__func__, ds->ds_remotestr, da->da_remotestr);
-	/* layoutupdate length */
-	start = xdr_reserve_space(xdr, 4);
 	/* netaddr4 */
 	ff_layout_encode_netaddr(xdr, da);
 	/* nfs_fh4 */
@@ -2177,42 +2276,71 @@ ff_layout_encode_layoutstats(struct xdr_stream *xdr,
 	/* bool */
 	p = xdr_reserve_space(xdr, 4);
 	*p = cpu_to_be32(false);
+}
+
+static void
+ff_layout_encode_layoutstats(struct xdr_stream *xdr, const void *args,
+			     const struct nfs4_xdr_opaque_data *opaque)
+{
+	struct nfs42_layoutstat_devinfo *devinfo = container_of(opaque,
+			struct nfs42_layoutstat_devinfo, ld_private);
+	__be32 *start;
+
+	/* layoutupdate length */
+	start = xdr_reserve_space(xdr, 4);
+	ff_layout_encode_ff_layoutupdate(xdr, devinfo, opaque->data);
 
 	*start = cpu_to_be32((xdr->p - start - 1) * 4);
 }
 
+static void
+ff_layout_free_layoutstats(struct nfs4_xdr_opaque_data *opaque)
+{
+	struct nfs4_ff_layout_mirror *mirror = opaque->data;
+
+	ff_layout_put_mirror(mirror);
+}
+
+static const struct nfs4_xdr_opaque_ops layoutstat_ops = {
+	.encode = ff_layout_encode_layoutstats,
+	.free	= ff_layout_free_layoutstats,
+};
+
 static int
-ff_layout_mirror_prepare_stats(struct nfs42_layoutstat_args *args,
-			       struct pnfs_layout_hdr *lo,
+ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
+			       struct nfs42_layoutstat_devinfo *devinfo,
 			       int dev_limit)
 {
 	struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(lo);
 	struct nfs4_ff_layout_mirror *mirror;
 	struct nfs4_deviceid_node *dev;
-	struct nfs42_layoutstat_devinfo *devinfo;
 	int i = 0;
 
 	list_for_each_entry(mirror, &ff_layout->mirrors, mirrors) {
 		if (i >= dev_limit)
 			break;
-		if (!mirror->mirror_ds)
+		if (IS_ERR_OR_NULL(mirror->mirror_ds))
+			continue;
+		if (!test_and_clear_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags))
 			continue;
 		/* mirror refcount put in cleanup_layoutstats */
 		if (!atomic_inc_not_zero(&mirror->ref))
 			continue;
 		dev = &mirror->mirror_ds->id_node; 
-		devinfo = &args->devinfo[i];
 		memcpy(&devinfo->dev_id, &dev->deviceid, NFS4_DEVICEID4_SIZE);
 		devinfo->offset = 0;
 		devinfo->length = NFS4_MAX_UINT64;
+		spin_lock(&mirror->lock);
 		devinfo->read_count = mirror->read_stat.io_stat.ops_completed;
 		devinfo->read_bytes = mirror->read_stat.io_stat.bytes_completed;
 		devinfo->write_count = mirror->write_stat.io_stat.ops_completed;
 		devinfo->write_bytes = mirror->write_stat.io_stat.bytes_completed;
+		spin_unlock(&mirror->lock);
 		devinfo->layout_type = LAYOUT_FLEX_FILES;
-		devinfo->layoutstats_encode = ff_layout_encode_layoutstats;
-		devinfo->layout_private = mirror;
+		devinfo->ld_private.ops = &layoutstat_ops;
+		devinfo->ld_private.data = mirror;
 
+		devinfo++;
 		i++;
 	}
 	return i;
@@ -2222,47 +2350,27 @@ static int
 ff_layout_prepare_layoutstats(struct nfs42_layoutstat_args *args)
 {
 	struct nfs4_flexfile_layout *ff_layout;
-	struct nfs4_ff_layout_mirror *mirror;
-	int dev_count = 0;
+	const int dev_count = PNFS_LAYOUTSTATS_MAXDEV;
 
-	spin_lock(&args->inode->i_lock);
-	ff_layout = FF_LAYOUT_FROM_HDR(NFS_I(args->inode)->layout);
-	list_for_each_entry(mirror, &ff_layout->mirrors, mirrors) {
-		if (atomic_read(&mirror->ref) != 0)
-			dev_count ++;
-	}
-	spin_unlock(&args->inode->i_lock);
 	/* For now, send at most PNFS_LAYOUTSTATS_MAXDEV statistics */
-	if (dev_count > PNFS_LAYOUTSTATS_MAXDEV) {
-		dprintk("%s: truncating devinfo to limit (%d:%d)\n",
-			__func__, dev_count, PNFS_LAYOUTSTATS_MAXDEV);
-		dev_count = PNFS_LAYOUTSTATS_MAXDEV;
-	}
 	args->devinfo = kmalloc_array(dev_count, sizeof(*args->devinfo), GFP_NOIO);
 	if (!args->devinfo)
 		return -ENOMEM;
 
 	spin_lock(&args->inode->i_lock);
-	args->num_dev = ff_layout_mirror_prepare_stats(args,
-			&ff_layout->generic_hdr, dev_count);
+	ff_layout = FF_LAYOUT_FROM_HDR(NFS_I(args->inode)->layout);
+	args->num_dev = ff_layout_mirror_prepare_stats(&ff_layout->generic_hdr,
+			&args->devinfo[0], dev_count);
 	spin_unlock(&args->inode->i_lock);
+	if (!args->num_dev) {
+		kfree(args->devinfo);
+		args->devinfo = NULL;
+		return -ENOENT;
+	}
 
 	return 0;
 }
 
-static void
-ff_layout_cleanup_layoutstats(struct nfs42_layoutstat_data *data)
-{
-	struct nfs4_ff_layout_mirror *mirror;
-	int i;
-
-	for (i = 0; i < data->args.num_dev; i++) {
-		mirror = data->args.devinfo[i].layout_private;
-		data->args.devinfo[i].layout_private = NULL;
-		ff_layout_put_mirror(mirror);
-	}
-}
-
 static struct pnfs_layoutdriver_type flexfilelayout_type = {
 	.id			= LAYOUT_FLEX_FILES,
 	.name			= "LAYOUT_FLEX_FILES",
@@ -2284,10 +2392,9 @@ static struct pnfs_layoutdriver_type flexfilelayout_type = {
 	.read_pagelist		= ff_layout_read_pagelist,
 	.write_pagelist		= ff_layout_write_pagelist,
 	.alloc_deviceid_node    = ff_layout_alloc_deviceid_node,
-	.encode_layoutreturn    = ff_layout_encode_layoutreturn,
+	.prepare_layoutreturn   = ff_layout_prepare_layoutreturn,
 	.sync			= pnfs_nfs_generic_sync,
 	.prepare_layoutstats	= ff_layout_prepare_layoutstats,
-	.cleanup_layoutstats	= ff_layout_cleanup_layoutstats,
 };
 
 static int __init nfs4flexfilelayout_init(void)
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h
index 3ee0c9fcea76..f4f39b0ab09b 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.h
+++ b/fs/nfs/flexfilelayout/flexfilelayout.h
@@ -21,6 +21,7 @@
 
 /* LAYOUTSTATS report interval in ms */
 #define FF_LAYOUTSTATS_REPORT_INTERVAL (60000L)
+#define FF_LAYOUTSTATS_MAXDEV 4
 
 struct nfs4_ff_ds_version {
 	u32				version;
@@ -73,6 +74,7 @@ struct nfs4_ff_layout_mirror {
 	struct list_head		mirrors;
 	u32				ds_count;
 	u32				efficiency;
+	struct nfs4_deviceid		devid;
 	struct nfs4_ff_layout_ds	*mirror_ds;
 	u32				fh_versions_cnt;
 	struct nfs_fh			*fh_versions;
@@ -81,12 +83,15 @@ struct nfs4_ff_layout_mirror {
 	struct rpc_cred	__rcu		*rw_cred;
 	atomic_t			ref;
 	spinlock_t			lock;
+	unsigned long			flags;
 	struct nfs4_ff_layoutstat	read_stat;
 	struct nfs4_ff_layoutstat	write_stat;
 	ktime_t				start_time;
 	u32				report_interval;
 };
 
+#define NFS4_FF_MIRROR_STAT_AVAIL	(0)
+
 struct nfs4_ff_layout_segment {
 	struct pnfs_layout_segment	generic_hdr;
 	u64				stripe_unit;
@@ -103,6 +108,14 @@ struct nfs4_flexfile_layout {
 	ktime_t			last_report_time; /* Layoutstat report times */
 };
 
+struct nfs4_flexfile_layoutreturn_args {
+	struct list_head errors;
+	struct nfs42_layoutstat_devinfo devinfo[FF_LAYOUTSTATS_MAXDEV];
+	unsigned int num_errors;
+	unsigned int num_dev;
+	struct page *pages[1];
+};
+
 static inline struct nfs4_flexfile_layout *
 FF_LAYOUT_FROM_HDR(struct pnfs_layout_hdr *lo)
 {
@@ -180,9 +193,12 @@ int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo,
 			     struct nfs4_ff_layout_mirror *mirror, u64 offset,
 			     u64 length, int status, enum nfs_opnum4 opnum,
 			     gfp_t gfp_flags);
-int ff_layout_encode_ds_ioerr(struct nfs4_flexfile_layout *flo,
-			      struct xdr_stream *xdr, int *count,
-			      const struct pnfs_layout_range *range);
+int ff_layout_encode_ds_ioerr(struct xdr_stream *xdr, const struct list_head *head);
+void ff_layout_free_ds_ioerr(struct list_head *head);
+unsigned int ff_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo,
+		const struct pnfs_layout_range *range,
+		struct list_head *head,
+		unsigned int maxnum);
 struct nfs_fh *
 nfs4_ff_layout_select_ds_fh(struct pnfs_layout_segment *lseg, u32 mirror_idx);
 
@@ -197,7 +213,6 @@ nfs4_ff_find_or_create_ds_client(struct pnfs_layout_segment *lseg,
 				 struct inode *inode);
 struct rpc_cred *ff_layout_get_ds_cred(struct pnfs_layout_segment *lseg,
 				       u32 ds_idx, struct rpc_cred *mdscred);
-bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg);
 bool ff_layout_avoid_mds_available_ds(struct pnfs_layout_segment *lseg);
 bool ff_layout_avoid_read_on_rw(struct pnfs_layout_segment *lseg);
 
diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
index f7a3f6b05369..3cc39d1c1206 100644
--- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c
+++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
@@ -20,9 +20,11 @@
 static unsigned int dataserver_timeo = NFS_DEF_TCP_RETRANS;
 static unsigned int dataserver_retrans;
 
+static bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg);
+
 void nfs4_ff_layout_put_deviceid(struct nfs4_ff_layout_ds *mirror_ds)
 {
-	if (mirror_ds)
+	if (!IS_ERR_OR_NULL(mirror_ds))
 		nfs4_put_deviceid_node(&mirror_ds->id_node);
 }
 
@@ -182,12 +184,29 @@ static void ff_layout_mark_devid_invalid(struct pnfs_layout_segment *lseg,
 }
 
 static bool ff_layout_mirror_valid(struct pnfs_layout_segment *lseg,
-		struct nfs4_ff_layout_mirror *mirror)
+				   struct nfs4_ff_layout_mirror *mirror,
+				   bool create)
 {
-	if (mirror == NULL || mirror->mirror_ds == NULL) {
-		pnfs_error_mark_layout_for_return(lseg->pls_layout->plh_inode,
-					lseg);
-		return false;
+	if (mirror == NULL || IS_ERR(mirror->mirror_ds))
+		goto outerr;
+	if (mirror->mirror_ds == NULL) {
+		if (create) {
+			struct nfs4_deviceid_node *node;
+			struct pnfs_layout_hdr *lh = lseg->pls_layout;
+			struct nfs4_ff_layout_ds *mirror_ds = ERR_PTR(-ENODEV);
+
+			node = nfs4_find_get_deviceid(NFS_SERVER(lh->plh_inode),
+					&mirror->devid, lh->plh_lc_cred,
+					GFP_KERNEL);
+			if (node)
+				mirror_ds = FF_LAYOUT_MIRROR_DS(node);
+
+			/* check for race with another call to this function */
+			if (cmpxchg(&mirror->mirror_ds, NULL, mirror_ds) &&
+			    mirror_ds != ERR_PTR(-ENODEV))
+				nfs4_put_deviceid_node(node);
+		} else
+			goto outerr;
 	}
 	if (mirror->mirror_ds->ds == NULL) {
 		struct nfs4_deviceid_node *devid;
@@ -196,15 +215,9 @@ static bool ff_layout_mirror_valid(struct pnfs_layout_segment *lseg,
 		return false;
 	}
 	return true;
-}
-
-static u64
-end_offset(u64 start, u64 len)
-{
-	u64 end;
-
-	end = start + len;
-	return end >= start ? end : NFS4_MAX_UINT64;
+outerr:
+	pnfs_error_mark_layout_for_return(lseg->pls_layout->plh_inode, lseg);
+	return false;
 }
 
 static void extend_ds_error(struct nfs4_ff_layout_ds_err *err,
@@ -212,8 +225,8 @@ static void extend_ds_error(struct nfs4_ff_layout_ds_err *err,
 {
 	u64 end;
 
-	end = max_t(u64, end_offset(err->offset, err->length),
-		    end_offset(offset, length));
+	end = max_t(u64, pnfs_end_offset(err->offset, err->length),
+		    pnfs_end_offset(offset, length));
 	err->offset = min_t(u64, err->offset, offset);
 	err->length = end - err->offset;
 }
@@ -235,9 +248,9 @@ ff_ds_error_match(const struct nfs4_ff_layout_ds_err *e1,
 	ret = memcmp(&e1->deviceid, &e2->deviceid, sizeof(e1->deviceid));
 	if (ret != 0)
 		return ret;
-	if (end_offset(e1->offset, e1->length) < e2->offset)
+	if (pnfs_end_offset(e1->offset, e1->length) < e2->offset)
 		return -1;
-	if (e1->offset > end_offset(e2->offset, e2->length))
+	if (e1->offset > pnfs_end_offset(e2->offset, e2->length))
 		return 1;
 	/* If ranges overlap or are contiguous, they are the same */
 	return 0;
@@ -263,8 +276,9 @@ ff_layout_add_ds_error_locked(struct nfs4_flexfile_layout *flo,
 		}
 		/* Entries match, so merge "err" into "dserr" */
 		extend_ds_error(dserr, err->offset, err->length);
-		list_del(&err->list);
+		list_replace(&err->list, &dserr->list);
 		kfree(err);
+		return;
 	}
 
 	list_add_tail(&dserr->list, head);
@@ -331,7 +345,7 @@ nfs4_ff_layout_select_ds_fh(struct pnfs_layout_segment *lseg, u32 mirror_idx)
 	struct nfs4_ff_layout_mirror *mirror = FF_LAYOUT_COMP(lseg, mirror_idx);
 	struct nfs_fh *fh = NULL;
 
-	if (!ff_layout_mirror_valid(lseg, mirror)) {
+	if (!ff_layout_mirror_valid(lseg, mirror, false)) {
 		pr_err_ratelimited("NFS: %s: No data server for mirror offset index %d\n",
 			__func__, mirror_idx);
 		goto out;
@@ -371,7 +385,7 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx,
 	struct nfs_server *s = NFS_SERVER(ino);
 	unsigned int max_payload;
 
-	if (!ff_layout_mirror_valid(lseg, mirror)) {
+	if (!ff_layout_mirror_valid(lseg, mirror, true)) {
 		pr_err_ratelimited("NFS: %s: No data server for offset index %d\n",
 			__func__, ds_idx);
 		goto out;
@@ -393,8 +407,7 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx,
 	nfs4_pnfs_ds_connect(s, ds, devid, dataserver_timeo,
 			     dataserver_retrans,
 			     mirror->mirror_ds->ds_versions[0].version,
-			     mirror->mirror_ds->ds_versions[0].minor_version,
-			     RPC_AUTH_UNIX);
+			     mirror->mirror_ds->ds_versions[0].minor_version);
 
 	/* connect success, check rsize/wsize limit */
 	if (ds->ds_clp) {
@@ -457,28 +470,26 @@ nfs4_ff_find_or_create_ds_client(struct pnfs_layout_segment *lseg, u32 ds_idx,
 	}
 }
 
-static bool is_range_intersecting(u64 offset1, u64 length1,
-				  u64 offset2, u64 length2)
+void ff_layout_free_ds_ioerr(struct list_head *head)
 {
-	u64 end1 = end_offset(offset1, length1);
-	u64 end2 = end_offset(offset2, length2);
+	struct nfs4_ff_layout_ds_err *err;
 
-	return (end1 == NFS4_MAX_UINT64 || end1 > offset2) &&
-	       (end2 == NFS4_MAX_UINT64 || end2 > offset1);
+	while (!list_empty(head)) {
+		err = list_first_entry(head,
+				struct nfs4_ff_layout_ds_err,
+				list);
+		list_del(&err->list);
+		kfree(err);
+	}
 }
 
 /* called with inode i_lock held */
-int ff_layout_encode_ds_ioerr(struct nfs4_flexfile_layout *flo,
-			      struct xdr_stream *xdr, int *count,
-			      const struct pnfs_layout_range *range)
+int ff_layout_encode_ds_ioerr(struct xdr_stream *xdr, const struct list_head *head)
 {
-	struct nfs4_ff_layout_ds_err *err, *n;
+	struct nfs4_ff_layout_ds_err *err;
 	__be32 *p;
 
-	list_for_each_entry_safe(err, n, &flo->error_list, list) {
-		if (!is_range_intersecting(err->offset, err->length,
-					   range->offset, range->length))
-			continue;
+	list_for_each_entry(err, head, list) {
 		/* offset(8) + length(8) + stateid(NFS4_STATEID_SIZE)
 		 * + array length + deviceid(NFS4_DEVICEID4_SIZE)
 		 * + status(4) + opnum(4)
@@ -497,17 +508,59 @@ int ff_layout_encode_ds_ioerr(struct nfs4_flexfile_layout *flo,
 					    NFS4_DEVICEID4_SIZE);
 		*p++ = cpu_to_be32(err->status);
 		*p++ = cpu_to_be32(err->opnum);
-		*count += 1;
-		list_del(&err->list);
-		dprintk("%s: offset %llu length %llu status %d op %d count %d\n",
+		dprintk("%s: offset %llu length %llu status %d op %d\n",
 			__func__, err->offset, err->length, err->status,
-			err->opnum, *count);
-		kfree(err);
+			err->opnum);
 	}
 
 	return 0;
 }
 
+static
+unsigned int do_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo,
+				      const struct pnfs_layout_range *range,
+				      struct list_head *head,
+				      unsigned int maxnum)
+{
+	struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo);
+	struct inode *inode = lo->plh_inode;
+	struct nfs4_ff_layout_ds_err *err, *n;
+	unsigned int ret = 0;
+
+	spin_lock(&inode->i_lock);
+	list_for_each_entry_safe(err, n, &flo->error_list, list) {
+		if (!pnfs_is_range_intersecting(err->offset,
+				pnfs_end_offset(err->offset, err->length),
+				range->offset,
+				pnfs_end_offset(range->offset, range->length)))
+			continue;
+		if (!maxnum)
+			break;
+		list_move(&err->list, head);
+		maxnum--;
+		ret++;
+	}
+	spin_unlock(&inode->i_lock);
+	return ret;
+}
+
+unsigned int ff_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo,
+				      const struct pnfs_layout_range *range,
+				      struct list_head *head,
+				      unsigned int maxnum)
+{
+	unsigned int ret;
+
+	ret = do_layout_fetch_ds_ioerr(lo, range, head, maxnum);
+	/* If we're over the max, discard all remaining entries */
+	if (ret == maxnum) {
+		LIST_HEAD(discard);
+		do_layout_fetch_ds_ioerr(lo, range, &discard, -1);
+		ff_layout_free_ds_ioerr(&discard);
+	}
+	return ret;
+}
+
 static bool ff_read_layout_has_available_ds(struct pnfs_layout_segment *lseg)
 {
 	struct nfs4_ff_layout_mirror *mirror;
@@ -516,7 +569,11 @@ static bool ff_read_layout_has_available_ds(struct pnfs_layout_segment *lseg)
 
 	for (idx = 0; idx < FF_LAYOUT_MIRROR_COUNT(lseg); idx++) {
 		mirror = FF_LAYOUT_COMP(lseg, idx);
-		if (mirror && mirror->mirror_ds) {
+		if (mirror) {
+			if (!mirror->mirror_ds)
+				return true;
+			if (IS_ERR(mirror->mirror_ds))
+				continue;
 			devid = &mirror->mirror_ds->id_node;
 			if (!ff_layout_test_devid_unavailable(devid))
 				return true;
@@ -534,8 +591,10 @@ static bool ff_rw_layout_has_available_ds(struct pnfs_layout_segment *lseg)
 
 	for (idx = 0; idx < FF_LAYOUT_MIRROR_COUNT(lseg); idx++) {
 		mirror = FF_LAYOUT_COMP(lseg, idx);
-		if (!mirror || !mirror->mirror_ds)
+		if (!mirror || IS_ERR(mirror->mirror_ds))
 			return false;
+		if (!mirror->mirror_ds)
+			continue;
 		devid = &mirror->mirror_ds->id_node;
 		if (ff_layout_test_devid_unavailable(devid))
 			return false;
@@ -544,7 +603,7 @@ static bool ff_rw_layout_has_available_ds(struct pnfs_layout_segment *lseg)
 	return FF_LAYOUT_MIRROR_COUNT(lseg) != 0;
 }
 
-bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg)
+static bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg)
 {
 	if (lseg->pls_range.iomode == IOMODE_READ)
 		return  ff_read_layout_has_available_ds(lseg);
diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index ce42dd00e4ee..5864146e05e6 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -634,15 +634,28 @@ void nfs_setattr_update_inode(struct inode *inode, struct iattr *attr,
 }
 EXPORT_SYMBOL_GPL(nfs_setattr_update_inode);
 
-static void nfs_request_parent_use_readdirplus(struct dentry *dentry)
+static void nfs_readdirplus_parent_cache_miss(struct dentry *dentry)
 {
 	struct dentry *parent;
 
+	if (!nfs_server_capable(d_inode(dentry), NFS_CAP_READDIRPLUS))
+		return;
 	parent = dget_parent(dentry);
 	nfs_force_use_readdirplus(d_inode(parent));
 	dput(parent);
 }
 
+static void nfs_readdirplus_parent_cache_hit(struct dentry *dentry)
+{
+	struct dentry *parent;
+
+	if (!nfs_server_capable(d_inode(dentry), NFS_CAP_READDIRPLUS))
+		return;
+	parent = dget_parent(dentry);
+	nfs_advise_use_readdirplus(d_inode(parent));
+	dput(parent);
+}
+
 static bool nfs_need_revalidate_inode(struct inode *inode)
 {
 	if (NFS_I(inode)->cache_validity &
@@ -683,10 +696,10 @@ int nfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
 	if (need_atime || nfs_need_revalidate_inode(inode)) {
 		struct nfs_server *server = NFS_SERVER(inode);
 
-		if (server->caps & NFS_CAP_READDIRPLUS)
-			nfs_request_parent_use_readdirplus(dentry);
+		nfs_readdirplus_parent_cache_miss(dentry);
 		err = __nfs_revalidate_inode(server, inode);
-	}
+	} else
+		nfs_readdirplus_parent_cache_hit(dentry);
 	if (!err) {
 		generic_fillattr(inode, stat);
 		stat->ino = nfs_compat_user_ino64(NFS_FILEID(inode));
@@ -702,8 +715,7 @@ EXPORT_SYMBOL_GPL(nfs_getattr);
 static void nfs_init_lock_context(struct nfs_lock_context *l_ctx)
 {
 	atomic_set(&l_ctx->count, 1);
-	l_ctx->lockowner.l_owner = current->files;
-	l_ctx->lockowner.l_pid = current->tgid;
+	l_ctx->lockowner = current->files;
 	INIT_LIST_HEAD(&l_ctx->list);
 	atomic_set(&l_ctx->io_count, 0);
 }
@@ -714,9 +726,7 @@ static struct nfs_lock_context *__nfs_find_lock_context(struct nfs_open_context
 	struct nfs_lock_context *pos = head;
 
 	do {
-		if (pos->lockowner.l_owner != current->files)
-			continue;
-		if (pos->lockowner.l_pid != current->tgid)
+		if (pos->lockowner != current->files)
 			continue;
 		atomic_inc(&pos->count);
 		return pos;
@@ -799,7 +809,9 @@ void nfs_close_context(struct nfs_open_context *ctx, int is_sync)
 }
 EXPORT_SYMBOL_GPL(nfs_close_context);
 
-struct nfs_open_context *alloc_nfs_open_context(struct dentry *dentry, fmode_t f_mode)
+struct nfs_open_context *alloc_nfs_open_context(struct dentry *dentry,
+						fmode_t f_mode,
+						struct file *filp)
 {
 	struct nfs_open_context *ctx;
 	struct rpc_cred *cred = rpc_lookup_cred();
@@ -818,6 +830,7 @@ struct nfs_open_context *alloc_nfs_open_context(struct dentry *dentry, fmode_t f
 	ctx->mode = f_mode;
 	ctx->flags = 0;
 	ctx->error = 0;
+	ctx->flock_owner = (fl_owner_t)filp;
 	nfs_init_lock_context(&ctx->lock_context);
 	ctx->lock_context.open_context = ctx;
 	INIT_LIST_HEAD(&ctx->list);
@@ -942,7 +955,7 @@ int nfs_open(struct inode *inode, struct file *filp)
 {
 	struct nfs_open_context *ctx;
 
-	ctx = alloc_nfs_open_context(file_dentry(filp), filp->f_mode);
+	ctx = alloc_nfs_open_context(file_dentry(filp), filp->f_mode, filp);
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
 	nfs_file_set_open_context(filp, ctx);
@@ -1099,11 +1112,17 @@ static int nfs_invalidate_mapping(struct inode *inode, struct address_space *map
 	return 0;
 }
 
-static bool nfs_mapping_need_revalidate_inode(struct inode *inode)
+bool nfs_mapping_need_revalidate_inode(struct inode *inode)
 {
-	if (nfs_have_delegated_attributes(inode))
-		return false;
-	return (NFS_I(inode)->cache_validity & NFS_INO_REVAL_PAGECACHE)
+	unsigned long cache_validity = NFS_I(inode)->cache_validity;
+
+	if (NFS_PROTO(inode)->have_delegation(inode, FMODE_READ)) {
+		const unsigned long force_reval =
+			NFS_INO_REVAL_PAGECACHE|NFS_INO_REVAL_FORCED;
+		return (cache_validity & force_reval) == force_reval;
+	}
+
+	return (cache_validity & NFS_INO_REVAL_PAGECACHE)
 		|| nfs_attribute_timeout(inode)
 		|| NFS_STALE(inode);
 }
@@ -1317,7 +1336,7 @@ static int nfs_check_inode_attributes(struct inode *inode, struct nfs_fattr *fat
 		invalid |= NFS_INO_INVALID_ATIME;
 
 	if (invalid != 0)
-		nfs_set_cache_invalid(inode, invalid);
+		nfs_set_cache_invalid(inode, invalid | NFS_INO_REVAL_FORCED);
 
 	nfsi->read_cache_jiffies = fattr->time_start;
 	return 0;
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 80bcc0befb07..6b79c2ca9b9a 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -154,8 +154,7 @@ extern const struct rpc_program nfs_program;
 extern void nfs_clients_init(struct net *net);
 extern struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *);
 int nfs_create_rpc_client(struct nfs_client *, const struct nfs_client_initdata *, rpc_authflavor_t);
-struct nfs_client *nfs_get_client(const struct nfs_client_initdata *,
-				  rpc_authflavor_t);
+struct nfs_client *nfs_get_client(const struct nfs_client_initdata *);
 int nfs_probe_fsinfo(struct nfs_server *server, struct nfs_fh *, struct nfs_fattr *);
 void nfs_server_insert_lists(struct nfs_server *);
 void nfs_server_remove_lists(struct nfs_server *);
@@ -194,14 +193,13 @@ extern struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv,
 					     int ds_addrlen, int ds_proto,
 					     unsigned int ds_timeo,
 					     unsigned int ds_retrans,
-					     u32 minor_version,
-					     rpc_authflavor_t au_flavor);
+					     u32 minor_version);
 extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *,
 						struct inode *);
 extern struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv,
 			const struct sockaddr *ds_addr, int ds_addrlen,
 			int ds_proto, unsigned int ds_timeo,
-			unsigned int ds_retrans, rpc_authflavor_t au_flavor);
+			unsigned int ds_retrans);
 #ifdef CONFIG_PROC_FS
 extern int __init nfs_fs_proc_init(void);
 extern void nfs_fs_proc_exit(void);
@@ -346,6 +344,7 @@ extern struct nfs_client *nfs_init_client(struct nfs_client *clp,
 			   const struct nfs_client_initdata *);
 
 /* dir.c */
+extern void nfs_advise_use_readdirplus(struct inode *dir);
 extern void nfs_force_use_readdirplus(struct inode *dir);
 extern unsigned long nfs_access_cache_count(struct shrinker *shrink,
 					    struct shrink_control *sc);
diff --git a/fs/nfs/nfs3client.c b/fs/nfs/nfs3client.c
index ee753547fb0a..7879f2a0fcfd 100644
--- a/fs/nfs/nfs3client.c
+++ b/fs/nfs/nfs3client.c
@@ -78,8 +78,7 @@ struct nfs_server *nfs3_clone_server(struct nfs_server *source,
  */
 struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv,
 		const struct sockaddr *ds_addr, int ds_addrlen,
-		int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans,
-		rpc_authflavor_t au_flavor)
+		int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans)
 {
 	struct rpc_timeout ds_timeout;
 	struct nfs_client *mds_clp = mds_srv->nfs_client;
@@ -106,7 +105,7 @@ struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv,
 
 	/* Use the MDS nfs_client cl_ipaddr. */
 	nfs_init_timeout_values(&ds_timeout, ds_proto, ds_timeo, ds_retrans);
-	clp = nfs_get_client(&cl_init, au_flavor);
+	clp = nfs_get_client(&cl_init);
 
 	return clp;
 }
diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c
index 608501971fe0..d12ff9385f49 100644
--- a/fs/nfs/nfs42proc.c
+++ b/fs/nfs/nfs42proc.c
@@ -397,10 +397,13 @@ static void
 nfs42_layoutstat_release(void *calldata)
 {
 	struct nfs42_layoutstat_data *data = calldata;
-	struct nfs_server *nfss = NFS_SERVER(data->args.inode);
+	struct nfs42_layoutstat_devinfo *devinfo = data->args.devinfo;
+	int i;
 
-	if (nfss->pnfs_curr_ld->cleanup_layoutstats)
-		nfss->pnfs_curr_ld->cleanup_layoutstats(data);
+	for (i = 0; i < data->args.num_dev; i++) {
+		if (devinfo[i].ld_private.ops && devinfo[i].ld_private.ops->free)
+			devinfo[i].ld_private.ops->free(&devinfo[i].ld_private);
+	}
 
 	pnfs_put_layout_hdr(NFS_I(data->args.inode)->layout);
 	smp_mb__before_atomic();
diff --git a/fs/nfs/nfs42xdr.c b/fs/nfs/nfs42xdr.c
index 8b2605882a20..6c7296454bbc 100644
--- a/fs/nfs/nfs42xdr.c
+++ b/fs/nfs/nfs42xdr.c
@@ -181,8 +181,9 @@ static void encode_layoutstats(struct xdr_stream *xdr,
 			NFS4_DEVICEID4_SIZE);
 	/* Encode layoutupdate4 */
 	*p++ = cpu_to_be32(devinfo->layout_type);
-	if (devinfo->layoutstats_encode != NULL)
-		devinfo->layoutstats_encode(xdr, args, devinfo);
+	if (devinfo->ld_private.ops)
+		devinfo->ld_private.ops->encode(xdr, args,
+				&devinfo->ld_private);
 	else
 		encode_uint32(xdr, 0);
 }
diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index 1452177c822d..665165833660 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -457,7 +457,7 @@ extern void nfs41_handle_server_scope(struct nfs_client *,
 extern void nfs4_put_lock_state(struct nfs4_lock_state *lsp);
 extern int nfs4_set_lock_state(struct nfs4_state *state, struct file_lock *fl);
 extern int nfs4_select_rw_stateid(struct nfs4_state *, fmode_t,
-		const struct nfs_lockowner *, nfs4_stateid *,
+		const struct nfs_lock_context *, nfs4_stateid *,
 		struct rpc_cred **);
 
 extern struct nfs_seqid *nfs_alloc_seqid(struct nfs_seqid_counter *counter, gfp_t gfp_mask);
diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c
index 074ac7131459..5ae9d64ea08b 100644
--- a/fs/nfs/nfs4client.c
+++ b/fs/nfs/nfs4client.c
@@ -464,6 +464,11 @@ static bool nfs4_match_client_owner_id(const struct nfs_client *clp1,
 	return strcmp(clp1->cl_owner_id, clp2->cl_owner_id) == 0;
 }
 
+static bool nfs4_same_verifier(nfs4_verifier *v1, nfs4_verifier *v2)
+{
+	return memcmp(v1->data, v2->data, sizeof(v1->data)) == 0;
+}
+
 /**
  * nfs40_walk_client_list - Find server that recognizes a client ID
  *
@@ -521,7 +526,21 @@ int nfs40_walk_client_list(struct nfs_client *new,
 
 		if (!nfs4_match_client_owner_id(pos, new))
 			continue;
-
+		/*
+		 * We just sent a new SETCLIENTID, which should have
+		 * caused the server to return a new cl_confirm.  So if
+		 * cl_confirm is the same, then this is a different
+		 * server that just returned the same cl_confirm by
+		 * coincidence:
+		 */
+		if ((new != pos) && nfs4_same_verifier(&pos->cl_confirm,
+						       &new->cl_confirm))
+			continue;
+		/*
+		 * But if the cl_confirm's are different, then the only
+		 * way that a SETCLIENTID_CONFIRM to pos can succeed is
+		 * if new and pos point to the same server:
+		 */
 		atomic_inc(&pos->cl_count);
 		spin_unlock(&nn->nfs_client_lock);
 
@@ -534,6 +553,7 @@ int nfs40_walk_client_list(struct nfs_client *new,
 			break;
 		case 0:
 			nfs4_swap_callback_idents(pos, new);
+			pos->cl_confirm = new->cl_confirm;
 
 			prev = NULL;
 			*result = pos;
@@ -881,7 +901,6 @@ static int nfs4_set_client(struct nfs_server *server,
 		const struct sockaddr *addr,
 		const size_t addrlen,
 		const char *ip_addr,
-		rpc_authflavor_t authflavour,
 		int proto, const struct rpc_timeout *timeparms,
 		u32 minorversion, struct net *net)
 {
@@ -907,7 +926,7 @@ static int nfs4_set_client(struct nfs_server *server,
 		set_bit(NFS_CS_MIGRATION, &cl_init.init_flags);
 
 	/* Allocate or find a client reference we can use */
-	clp = nfs_get_client(&cl_init, authflavour);
+	clp = nfs_get_client(&cl_init);
 	if (IS_ERR(clp)) {
 		error = PTR_ERR(clp);
 		goto error;
@@ -948,7 +967,7 @@ error:
 struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv,
 		const struct sockaddr *ds_addr, int ds_addrlen,
 		int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans,
-		u32 minor_version, rpc_authflavor_t au_flavor)
+		u32 minor_version)
 {
 	struct rpc_timeout ds_timeout;
 	struct nfs_client *mds_clp = mds_srv->nfs_client;
@@ -979,7 +998,7 @@ struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv,
 	 * (section 13.1 RFC 5661).
 	 */
 	nfs_init_timeout_values(&ds_timeout, ds_proto, ds_timeo, ds_retrans);
-	clp = nfs_get_client(&cl_init, au_flavor);
+	clp = nfs_get_client(&cl_init);
 
 	dprintk("<-- %s %p\n", __func__, clp);
 	return clp;
@@ -1103,7 +1122,6 @@ static int nfs4_init_server(struct nfs_server *server,
 			(const struct sockaddr *)&data->nfs_server.address,
 			data->nfs_server.addrlen,
 			data->client_address,
-			data->selected_flavor,
 			data->nfs_server.protocol,
 			&timeparms,
 			data->minorversion,
@@ -1200,7 +1218,6 @@ struct nfs_server *nfs4_create_referral_server(struct nfs_clone_mount *data,
 				data->addr,
 				data->addrlen,
 				parent_client->cl_ipaddr,
-				data->authflavor,
 				rpc_protocol(parent_server->client),
 				parent_server->client->cl_timeout,
 				parent_client->cl_mvops->minor_version,
@@ -1311,7 +1328,6 @@ int nfs4_update_server(struct nfs_server *server, const char *hostname,
 
 	nfs_server_remove_lists(server);
 	error = nfs4_set_client(server, hostname, sap, salen, buf,
-				clp->cl_rpcclient->cl_auth->au_flavor,
 				clp->cl_proto, clnt->cl_timeout,
 				clp->cl_minorversion, net);
 	nfs_put_client(clp);
diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c
index 89a77950e0b0..0efba77789b9 100644
--- a/fs/nfs/nfs4file.c
+++ b/fs/nfs/nfs4file.c
@@ -57,7 +57,7 @@ nfs4_file_open(struct inode *inode, struct file *filp)
 	parent = dget_parent(dentry);
 	dir = d_inode(parent);
 
-	ctx = alloc_nfs_open_context(file_dentry(filp), filp->f_mode);
+	ctx = alloc_nfs_open_context(file_dentry(filp), filp->f_mode, filp);
 	err = PTR_ERR(ctx);
 	if (IS_ERR(ctx))
 		goto out;
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 241da19b7da4..d33242c8d95d 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -94,7 +94,7 @@ static int nfs4_proc_getattr(struct nfs_server *, struct nfs_fh *, struct nfs_fa
 static int _nfs4_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle, struct nfs_fattr *fattr, struct nfs4_label *label);
 static int nfs4_do_setattr(struct inode *inode, struct rpc_cred *cred,
 			    struct nfs_fattr *fattr, struct iattr *sattr,
-			    struct nfs4_state *state, struct nfs4_label *ilabel,
+			    struct nfs_open_context *ctx, struct nfs4_label *ilabel,
 			    struct nfs4_label *olabel);
 #ifdef CONFIG_NFS_V4_1
 static int nfs41_test_stateid(struct nfs_server *, nfs4_stateid *,
@@ -226,7 +226,6 @@ static const u32 nfs4_pnfs_open_bitmap[3] = {
 
 static const u32 nfs4_open_noattr_bitmap[3] = {
 	FATTR4_WORD0_TYPE
-	| FATTR4_WORD0_CHANGE
 	| FATTR4_WORD0_FILEID,
 };
 
@@ -817,6 +816,10 @@ static int nfs41_sequence_process(struct rpc_task *task,
 	case -NFS4ERR_SEQ_FALSE_RETRY:
 		++slot->seq_nr;
 		goto retry_nowait;
+	case -NFS4ERR_DEADSESSION:
+	case -NFS4ERR_BADSESSION:
+		nfs4_schedule_session_recovery(session, res->sr_status);
+		goto retry_nowait;
 	default:
 		/* Just update the slot sequence no. */
 		slot->seq_done = 1;
@@ -1221,6 +1224,8 @@ static struct nfs4_opendata *nfs4_opendata_alloc(struct dentry *dentry,
 	atomic_inc(&sp->so_count);
 	p->o_arg.open_flags = flags;
 	p->o_arg.fmode = fmode & (FMODE_READ|FMODE_WRITE);
+	p->o_arg.umask = current_umask();
+	p->o_arg.claim = nfs4_map_atomic_open_claim(server, claim);
 	p->o_arg.share_access = nfs4_map_atomic_open_share(server,
 			fmode, flags);
 	/* don't put an ACCESS op in OPEN compound if O_EXCL, because ACCESS
@@ -1228,8 +1233,16 @@ static struct nfs4_opendata *nfs4_opendata_alloc(struct dentry *dentry,
 	if (!(flags & O_EXCL)) {
 		/* ask server to check for all possible rights as results
 		 * are cached */
-		p->o_arg.access = NFS4_ACCESS_READ | NFS4_ACCESS_MODIFY |
-				  NFS4_ACCESS_EXTEND | NFS4_ACCESS_EXECUTE;
+		switch (p->o_arg.claim) {
+		default:
+			break;
+		case NFS4_OPEN_CLAIM_NULL:
+		case NFS4_OPEN_CLAIM_FH:
+			p->o_arg.access = NFS4_ACCESS_READ |
+				NFS4_ACCESS_MODIFY |
+				NFS4_ACCESS_EXTEND |
+				NFS4_ACCESS_EXECUTE;
+		}
 	}
 	p->o_arg.clientid = server->nfs_client->cl_clientid;
 	p->o_arg.id.create_time = ktime_to_ns(sp->so_seqid.create_time);
@@ -1239,7 +1252,6 @@ static struct nfs4_opendata *nfs4_opendata_alloc(struct dentry *dentry,
 	p->o_arg.bitmask = nfs4_bitmask(server, label);
 	p->o_arg.open_bitmap = &nfs4_fattr_bitmap[0];
 	p->o_arg.label = nfs4_label_copy(p->a_label, label);
-	p->o_arg.claim = nfs4_map_atomic_open_claim(server, claim);
 	switch (p->o_arg.claim) {
 	case NFS4_OPEN_CLAIM_NULL:
 	case NFS4_OPEN_CLAIM_DELEGATE_CUR:
@@ -2819,7 +2831,7 @@ static int _nfs4_do_open(struct inode *dir,
 			nfs_fattr_init(opendata->o_res.f_attr);
 			status = nfs4_do_setattr(state->inode, cred,
 					opendata->o_res.f_attr, sattr,
-					state, label, olabel);
+					ctx, label, olabel);
 			if (status == 0) {
 				nfs_setattr_update_inode(state->inode, sattr,
 						opendata->o_res.f_attr);
@@ -2914,7 +2926,7 @@ static int _nfs4_do_setattr(struct inode *inode,
 			    struct nfs_setattrargs *arg,
 			    struct nfs_setattrres *res,
 			    struct rpc_cred *cred,
-			    struct nfs4_state *state)
+			    struct nfs_open_context *ctx)
 {
 	struct nfs_server *server = NFS_SERVER(inode);
         struct rpc_message msg = {
@@ -2937,15 +2949,17 @@ static int _nfs4_do_setattr(struct inode *inode,
 
 	if (nfs4_copy_delegation_stateid(inode, fmode, &arg->stateid, &delegation_cred)) {
 		/* Use that stateid */
-	} else if (truncate && state != NULL) {
-		struct nfs_lockowner lockowner = {
-			.l_owner = current->files,
-			.l_pid = current->tgid,
-		};
-		if (!nfs4_valid_open_stateid(state))
+	} else if (truncate && ctx != NULL) {
+		struct nfs_lock_context *l_ctx;
+		if (!nfs4_valid_open_stateid(ctx->state))
 			return -EBADF;
-		if (nfs4_select_rw_stateid(state, FMODE_WRITE, &lockowner,
-				&arg->stateid, &delegation_cred) == -EIO)
+		l_ctx = nfs_get_lock_context(ctx);
+		if (IS_ERR(l_ctx))
+			return PTR_ERR(l_ctx);
+		status = nfs4_select_rw_stateid(ctx->state, FMODE_WRITE, l_ctx,
+						&arg->stateid, &delegation_cred);
+		nfs_put_lock_context(l_ctx);
+		if (status == -EIO)
 			return -EBADF;
 	} else
 		nfs4_stateid_copy(&arg->stateid, &zero_stateid);
@@ -2955,7 +2969,7 @@ static int _nfs4_do_setattr(struct inode *inode,
 	status = nfs4_call_sync(server->client, server, &msg, &arg->seq_args, &res->seq_res, 1);
 
 	put_rpccred(delegation_cred);
-	if (status == 0 && state != NULL)
+	if (status == 0 && ctx != NULL)
 		renew_lease(server, timestamp);
 	trace_nfs4_setattr(inode, &arg->stateid, status);
 	return status;
@@ -2963,10 +2977,11 @@ static int _nfs4_do_setattr(struct inode *inode,
 
 static int nfs4_do_setattr(struct inode *inode, struct rpc_cred *cred,
 			   struct nfs_fattr *fattr, struct iattr *sattr,
-			   struct nfs4_state *state, struct nfs4_label *ilabel,
+			   struct nfs_open_context *ctx, struct nfs4_label *ilabel,
 			   struct nfs4_label *olabel)
 {
 	struct nfs_server *server = NFS_SERVER(inode);
+	struct nfs4_state *state = ctx ? ctx->state : NULL;
         struct nfs_setattrargs  arg = {
                 .fh             = NFS_FH(inode),
                 .iap            = sattr,
@@ -2991,7 +3006,7 @@ static int nfs4_do_setattr(struct inode *inode, struct rpc_cred *cred,
 		arg.bitmask = nfs4_bitmask(server, olabel);
 
 	do {
-		err = _nfs4_do_setattr(inode, &arg, &res, cred, state);
+		err = _nfs4_do_setattr(inode, &arg, &res, cred, ctx);
 		switch (err) {
 		case -NFS4ERR_OPENMODE:
 			if (!(sattr->ia_valid & ATTR_SIZE)) {
@@ -3028,10 +3043,15 @@ struct nfs4_closedata {
 	struct nfs4_state *state;
 	struct nfs_closeargs arg;
 	struct nfs_closeres res;
+	struct {
+		struct nfs4_layoutreturn_args arg;
+		struct nfs4_layoutreturn_res res;
+		struct nfs4_xdr_opaque_data ld_private;
+		u32 roc_barrier;
+		bool roc;
+	} lr;
 	struct nfs_fattr fattr;
 	unsigned long timestamp;
-	bool roc;
-	u32 roc_barrier;
 };
 
 static void nfs4_free_closedata(void *data)
@@ -3040,8 +3060,9 @@ static void nfs4_free_closedata(void *data)
 	struct nfs4_state_owner *sp = calldata->state->owner;
 	struct super_block *sb = calldata->state->inode->i_sb;
 
-	if (calldata->roc)
-		pnfs_roc_release(calldata->state->inode);
+	if (calldata->lr.roc)
+		pnfs_roc_release(&calldata->lr.arg, &calldata->lr.res,
+				calldata->res.lr_ret);
 	nfs4_put_open_state(calldata->state);
 	nfs_free_seqid(calldata->arg.seqid);
 	nfs4_put_state_owner(sp);
@@ -3060,15 +3081,38 @@ static void nfs4_close_done(struct rpc_task *task, void *data)
 	if (!nfs4_sequence_done(task, &calldata->res.seq_res))
 		return;
 	trace_nfs4_close(state, &calldata->arg, &calldata->res, task->tk_status);
+
+	/* Handle Layoutreturn errors */
+	if (calldata->arg.lr_args && task->tk_status != 0) {
+		switch (calldata->res.lr_ret) {
+		default:
+			calldata->res.lr_ret = -NFS4ERR_NOMATCHING_LAYOUT;
+			break;
+		case 0:
+			calldata->arg.lr_args = NULL;
+			calldata->res.lr_res = NULL;
+			break;
+		case -NFS4ERR_ADMIN_REVOKED:
+		case -NFS4ERR_DELEG_REVOKED:
+		case -NFS4ERR_EXPIRED:
+		case -NFS4ERR_BAD_STATEID:
+		case -NFS4ERR_OLD_STATEID:
+		case -NFS4ERR_UNKNOWN_LAYOUTTYPE:
+		case -NFS4ERR_WRONG_CRED:
+			calldata->arg.lr_args = NULL;
+			calldata->res.lr_res = NULL;
+			calldata->res.lr_ret = 0;
+			rpc_restart_call_prepare(task);
+			return;
+		}
+	}
+
         /* hmm. we are done with the inode, and in the process of freeing
 	 * the state_owner. we keep this around to process errors
 	 */
 	switch (task->tk_status) {
 		case 0:
 			res_stateid = &calldata->res.stateid;
-			if (calldata->roc)
-				pnfs_roc_set_barrier(state->inode,
-						     calldata->roc_barrier);
 			renew_lease(server, calldata->timestamp);
 			break;
 		case -NFS4ERR_ADMIN_REVOKED:
@@ -3144,15 +3188,20 @@ static void nfs4_close_prepare(struct rpc_task *task, void *data)
 		goto out_no_action;
 	}
 
-	if (nfs4_wait_on_layoutreturn(inode, task)) {
+	if (!calldata->lr.roc && nfs4_wait_on_layoutreturn(inode, task)) {
 		nfs_release_seqid(calldata->arg.seqid);
 		goto out_wait;
 	}
 
-	if (calldata->arg.fmode == 0)
+	if (calldata->arg.fmode == 0) {
 		task->tk_msg.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_CLOSE];
-	if (calldata->roc)
-		pnfs_roc_get_barrier(inode, &calldata->roc_barrier);
+
+		/* Close-to-open cache consistency revalidation */
+		if (!nfs4_have_delegation(inode, FMODE_READ))
+			calldata->arg.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
+		else
+			calldata->arg.bitmask = NULL;
+	}
 
 	calldata->arg.share_access =
 		nfs4_map_atomic_open_share(NFS_SERVER(inode),
@@ -3179,13 +3228,6 @@ static const struct rpc_call_ops nfs4_close_ops = {
 	.rpc_release = nfs4_free_closedata,
 };
 
-static bool nfs4_roc(struct inode *inode)
-{
-	if (!nfs_have_layout(inode))
-		return false;
-	return pnfs_roc(inode);
-}
-
 /* 
  * It is possible for data to be read/written from a mem-mapped file 
  * after the sys_close call (which hits the vfs layer as a flush).
@@ -3233,11 +3275,17 @@ int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait)
 	if (IS_ERR(calldata->arg.seqid))
 		goto out_free_calldata;
 	calldata->arg.fmode = 0;
-	calldata->arg.bitmask = server->cache_consistency_bitmask;
+	calldata->lr.arg.ld_private = &calldata->lr.ld_private;
 	calldata->res.fattr = &calldata->fattr;
 	calldata->res.seqid = calldata->arg.seqid;
 	calldata->res.server = server;
-	calldata->roc = nfs4_roc(state->inode);
+	calldata->res.lr_ret = -NFS4ERR_NOMATCHING_LAYOUT;
+	calldata->lr.roc = pnfs_roc(state->inode,
+			&calldata->lr.arg, &calldata->lr.res, msg.rpc_cred);
+	if (calldata->lr.roc) {
+		calldata->arg.lr_args = &calldata->lr.arg;
+		calldata->res.lr_res = &calldata->lr.res;
+	}
 	nfs_sb_active(calldata->inode->i_sb);
 
 	msg.rpc_argp = &calldata->arg;
@@ -3290,7 +3338,7 @@ static void nfs4_close_context(struct nfs_open_context *ctx, int is_sync)
 
 #define FATTR4_WORD1_NFS40_MASK (2*FATTR4_WORD1_MOUNTED_ON_FILEID - 1UL)
 #define FATTR4_WORD2_NFS41_MASK (2*FATTR4_WORD2_SUPPATTR_EXCLCREAT - 1UL)
-#define FATTR4_WORD2_NFS42_MASK (2*FATTR4_WORD2_SECURITY_LABEL - 1UL)
+#define FATTR4_WORD2_NFS42_MASK (2*FATTR4_WORD2_MODE_UMASK - 1UL)
 
 static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle)
 {
@@ -3687,7 +3735,7 @@ nfs4_proc_setattr(struct dentry *dentry, struct nfs_fattr *fattr,
 {
 	struct inode *inode = d_inode(dentry);
 	struct rpc_cred *cred = NULL;
-	struct nfs4_state *state = NULL;
+	struct nfs_open_context *ctx = NULL;
 	struct nfs4_label *label = NULL;
 	int status;
 
@@ -3708,20 +3756,17 @@ nfs4_proc_setattr(struct dentry *dentry, struct nfs_fattr *fattr,
 
 	/* Search for an existing open(O_WRITE) file */
 	if (sattr->ia_valid & ATTR_FILE) {
-		struct nfs_open_context *ctx;
 
 		ctx = nfs_file_open_context(sattr->ia_file);
-		if (ctx) {
+		if (ctx)
 			cred = ctx->cred;
-			state = ctx->state;
-		}
 	}
 
 	label = nfs4_label_alloc(NFS_SERVER(inode), GFP_KERNEL);
 	if (IS_ERR(label))
 		return PTR_ERR(label);
 
-	status = nfs4_do_setattr(inode, cred, fattr, sattr, state, NULL, label);
+	status = nfs4_do_setattr(inode, cred, fattr, sattr, ctx, NULL, label);
 	if (status == 0) {
 		nfs_setattr_update_inode(inode, sattr, fattr);
 		nfs_setsecurity(inode, fattr, label);
@@ -3966,18 +4011,20 @@ static int
 nfs4_proc_create(struct inode *dir, struct dentry *dentry, struct iattr *sattr,
 		 int flags)
 {
+	struct nfs_server *server = NFS_SERVER(dir);
 	struct nfs4_label l, *ilabel = NULL;
 	struct nfs_open_context *ctx;
 	struct nfs4_state *state;
 	int status = 0;
 
-	ctx = alloc_nfs_open_context(dentry, FMODE_READ);
+	ctx = alloc_nfs_open_context(dentry, FMODE_READ, NULL);
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
 
 	ilabel = nfs4_label_init_security(dir, dentry, sattr, &l);
 
-	sattr->ia_mode &= ~current_umask();
+	if (!(server->attr_bitmask[2] & FATTR4_WORD2_MODE_UMASK))
+		sattr->ia_mode &= ~current_umask();
 	state = nfs4_do_open(dir, ctx, flags, sattr, ilabel, NULL);
 	if (IS_ERR(state)) {
 		status = PTR_ERR(state);
@@ -4185,6 +4232,7 @@ static struct nfs4_createdata *nfs4_alloc_createdata(struct inode *dir,
 		data->arg.attrs = sattr;
 		data->arg.ftype = ftype;
 		data->arg.bitmask = nfs4_bitmask(server, data->label);
+		data->arg.umask = current_umask();
 		data->res.server = server;
 		data->res.fh = &data->fh;
 		data->res.fattr = &data->fattr;
@@ -4282,13 +4330,15 @@ out:
 static int nfs4_proc_mkdir(struct inode *dir, struct dentry *dentry,
 		struct iattr *sattr)
 {
+	struct nfs_server *server = NFS_SERVER(dir);
 	struct nfs4_exception exception = { };
 	struct nfs4_label l, *label = NULL;
 	int err;
 
 	label = nfs4_label_init_security(dir, dentry, sattr, &l);
 
-	sattr->ia_mode &= ~current_umask();
+	if (!(server->attr_bitmask[2] & FATTR4_WORD2_MODE_UMASK))
+		sattr->ia_mode &= ~current_umask();
 	do {
 		err = _nfs4_proc_mkdir(dir, dentry, sattr, label);
 		trace_nfs4_mkdir(dir, &dentry->d_name, err);
@@ -4391,13 +4441,15 @@ out:
 static int nfs4_proc_mknod(struct inode *dir, struct dentry *dentry,
 		struct iattr *sattr, dev_t rdev)
 {
+	struct nfs_server *server = NFS_SERVER(dir);
 	struct nfs4_exception exception = { };
 	struct nfs4_label l, *label = NULL;
 	int err;
 
 	label = nfs4_label_init_security(dir, dentry, sattr, &l);
 
-	sattr->ia_mode &= ~current_umask();
+	if (!(server->attr_bitmask[2] & FATTR4_WORD2_MODE_UMASK))
+		sattr->ia_mode &= ~current_umask();
 	do {
 		err = _nfs4_proc_mknod(dir, dentry, sattr, label, rdev);
 		trace_nfs4_mknod(dir, &dentry->d_name, err);
@@ -4541,11 +4593,7 @@ int nfs4_set_rw_stateid(nfs4_stateid *stateid,
 		const struct nfs_lock_context *l_ctx,
 		fmode_t fmode)
 {
-	const struct nfs_lockowner *lockowner = NULL;
-
-	if (l_ctx != NULL)
-		lockowner = &l_ctx->lockowner;
-	return nfs4_select_rw_stateid(ctx->state, fmode, lockowner, stateid, NULL);
+	return nfs4_select_rw_stateid(ctx->state, fmode, l_ctx, stateid, NULL);
 }
 EXPORT_SYMBOL_GPL(nfs4_set_rw_stateid);
 
@@ -5564,11 +5612,16 @@ struct nfs4_delegreturndata {
 	struct nfs_fh fh;
 	nfs4_stateid stateid;
 	unsigned long timestamp;
+	struct {
+		struct nfs4_layoutreturn_args arg;
+		struct nfs4_layoutreturn_res res;
+		struct nfs4_xdr_opaque_data ld_private;
+		u32 roc_barrier;
+		bool roc;
+	} lr;
 	struct nfs_fattr fattr;
 	int rpc_status;
 	struct inode *inode;
-	bool roc;
-	u32 roc_barrier;
 };
 
 static void nfs4_delegreturn_done(struct rpc_task *task, void *calldata)
@@ -5579,6 +5632,32 @@ static void nfs4_delegreturn_done(struct rpc_task *task, void *calldata)
 		return;
 
 	trace_nfs4_delegreturn_exit(&data->args, &data->res, task->tk_status);
+
+	/* Handle Layoutreturn errors */
+	if (data->args.lr_args && task->tk_status != 0) {
+		switch(data->res.lr_ret) {
+		default:
+			data->res.lr_ret = -NFS4ERR_NOMATCHING_LAYOUT;
+			break;
+		case 0:
+			data->args.lr_args = NULL;
+			data->res.lr_res = NULL;
+			break;
+		case -NFS4ERR_ADMIN_REVOKED:
+		case -NFS4ERR_DELEG_REVOKED:
+		case -NFS4ERR_EXPIRED:
+		case -NFS4ERR_BAD_STATEID:
+		case -NFS4ERR_OLD_STATEID:
+		case -NFS4ERR_UNKNOWN_LAYOUTTYPE:
+		case -NFS4ERR_WRONG_CRED:
+			data->args.lr_args = NULL;
+			data->res.lr_res = NULL;
+			data->res.lr_ret = 0;
+			rpc_restart_call_prepare(task);
+			return;
+		}
+	}
+
 	switch (task->tk_status) {
 	case 0:
 		renew_lease(data->res.server, data->timestamp);
@@ -5602,8 +5681,6 @@ static void nfs4_delegreturn_done(struct rpc_task *task, void *calldata)
 		}
 	}
 	data->rpc_status = task->tk_status;
-	if (data->roc && data->rpc_status == 0)
-		pnfs_roc_set_barrier(data->inode, data->roc_barrier);
 }
 
 static void nfs4_delegreturn_release(void *calldata)
@@ -5612,8 +5689,9 @@ static void nfs4_delegreturn_release(void *calldata)
 	struct inode *inode = data->inode;
 
 	if (inode) {
-		if (data->roc)
-			pnfs_roc_release(inode);
+		if (data->lr.roc)
+			pnfs_roc_release(&data->lr.arg, &data->lr.res,
+					data->res.lr_ret);
 		nfs_iput_and_deactive(inode);
 	}
 	kfree(calldata);
@@ -5625,12 +5703,9 @@ static void nfs4_delegreturn_prepare(struct rpc_task *task, void *data)
 
 	d_data = (struct nfs4_delegreturndata *)data;
 
-	if (nfs4_wait_on_layoutreturn(d_data->inode, task))
+	if (!d_data->lr.roc && nfs4_wait_on_layoutreturn(d_data->inode, task))
 		return;
 
-	if (d_data->roc)
-		pnfs_roc_get_barrier(d_data->inode, &d_data->roc_barrier);
-
 	nfs4_setup_sequence(d_data->res.server,
 			&d_data->args.seq_args,
 			&d_data->res.seq_res,
@@ -5676,12 +5751,22 @@ static int _nfs4_proc_delegreturn(struct inode *inode, struct rpc_cred *cred, co
 	nfs4_stateid_copy(&data->stateid, stateid);
 	data->res.fattr = &data->fattr;
 	data->res.server = server;
+	data->res.lr_ret = -NFS4ERR_NOMATCHING_LAYOUT;
+	data->lr.arg.ld_private = &data->lr.ld_private;
 	nfs_fattr_init(data->res.fattr);
 	data->timestamp = jiffies;
 	data->rpc_status = 0;
+	data->lr.roc = pnfs_roc(inode, &data->lr.arg, &data->lr.res, cred);
 	data->inode = nfs_igrab_and_active(inode);
-	if (data->inode)
-		data->roc = nfs4_roc(inode);
+	if (data->inode) {
+		if (data->lr.roc) {
+			data->args.lr_args = &data->lr.arg;
+			data->res.lr_res = &data->lr.res;
+		}
+	} else if (data->lr.roc) {
+		pnfs_roc_release(&data->lr.arg, &data->lr.res, 0);
+		data->lr.roc = false;
+	}
 
 	task_setup_data.callback_data = data;
 	msg.rpc_argp = &data->args;
@@ -8559,21 +8644,13 @@ static void nfs4_layoutreturn_release(void *calldata)
 {
 	struct nfs4_layoutreturn *lrp = calldata;
 	struct pnfs_layout_hdr *lo = lrp->args.layout;
-	LIST_HEAD(freeme);
 
 	dprintk("--> %s\n", __func__);
-	spin_lock(&lo->plh_inode->i_lock);
-	if (lrp->res.lrs_present) {
-		pnfs_mark_matching_lsegs_invalid(lo, &freeme,
-				&lrp->args.range,
-				be32_to_cpu(lrp->args.stateid.seqid));
-		pnfs_set_layout_stateid(lo, &lrp->res.stateid, true);
-	} else
-		pnfs_mark_layout_stateid_invalid(lo, &freeme);
-	pnfs_clear_layoutreturn_waitbit(lo);
-	spin_unlock(&lo->plh_inode->i_lock);
+	pnfs_layoutreturn_free_lsegs(lo, &lrp->args.stateid, &lrp->args.range,
+			lrp->res.lrs_present ? &lrp->res.stateid : NULL);
 	nfs4_sequence_free_slot(&lrp->res.seq_res);
-	pnfs_free_lseg_list(&freeme);
+	if (lrp->ld_private.ops && lrp->ld_private.ops->free)
+		lrp->ld_private.ops->free(&lrp->ld_private);
 	pnfs_put_layout_hdr(lrp->args.layout);
 	nfs_iput_and_deactive(lrp->inode);
 	kfree(calldata);
diff --git a/fs/nfs/nfs4session.c b/fs/nfs/nfs4session.c
index a61350f75c74..769b85655c4b 100644
--- a/fs/nfs/nfs4session.c
+++ b/fs/nfs/nfs4session.c
@@ -169,7 +169,7 @@ bool nfs4_try_to_lock_slot(struct nfs4_slot_table *tbl, struct nfs4_slot *slot)
 struct nfs4_slot *nfs4_lookup_slot(struct nfs4_slot_table *tbl, u32 slotid)
 {
 	if (slotid <= tbl->max_slotid)
-		return nfs4_find_or_create_slot(tbl, slotid, 1, GFP_NOWAIT);
+		return nfs4_find_or_create_slot(tbl, slotid, 0, GFP_NOWAIT);
 	return ERR_PTR(-E2BIG);
 }
 
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index 0959c9661662..95baf7d340f0 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -800,11 +800,13 @@ void nfs4_close_sync(struct nfs4_state *state, fmode_t fmode)
  * that is compatible with current->files
  */
 static struct nfs4_lock_state *
-__nfs4_find_lock_state(struct nfs4_state *state, fl_owner_t fl_owner)
+__nfs4_find_lock_state(struct nfs4_state *state,
+		       fl_owner_t fl_owner, fl_owner_t fl_owner2)
 {
 	struct nfs4_lock_state *pos;
 	list_for_each_entry(pos, &state->lock_states, ls_locks) {
-		if (pos->ls_owner != fl_owner)
+		if (pos->ls_owner != fl_owner &&
+		    pos->ls_owner != fl_owner2)
 			continue;
 		atomic_inc(&pos->ls_count);
 		return pos;
@@ -857,7 +859,7 @@ static struct nfs4_lock_state *nfs4_get_lock_state(struct nfs4_state *state, fl_
 	
 	for(;;) {
 		spin_lock(&state->state_lock);
-		lsp = __nfs4_find_lock_state(state, owner);
+		lsp = __nfs4_find_lock_state(state, owner, 0);
 		if (lsp != NULL)
 			break;
 		if (new != NULL) {
@@ -939,22 +941,23 @@ int nfs4_set_lock_state(struct nfs4_state *state, struct file_lock *fl)
 
 static int nfs4_copy_lock_stateid(nfs4_stateid *dst,
 		struct nfs4_state *state,
-		const struct nfs_lockowner *lockowner)
+		const struct nfs_lock_context *l_ctx)
 {
 	struct nfs4_lock_state *lsp;
-	fl_owner_t fl_owner;
+	fl_owner_t fl_owner, fl_flock_owner;
 	int ret = -ENOENT;
 
-
-	if (lockowner == NULL)
+	if (l_ctx == NULL)
 		goto out;
 
 	if (test_bit(LK_STATE_IN_USE, &state->flags) == 0)
 		goto out;
 
-	fl_owner = lockowner->l_owner;
+	fl_owner = l_ctx->lockowner;
+	fl_flock_owner = l_ctx->open_context->flock_owner;
+
 	spin_lock(&state->state_lock);
-	lsp = __nfs4_find_lock_state(state, fl_owner);
+	lsp = __nfs4_find_lock_state(state, fl_owner, fl_flock_owner);
 	if (lsp && test_bit(NFS_LOCK_LOST, &lsp->ls_flags))
 		ret = -EIO;
 	else if (lsp != NULL && test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) != 0) {
@@ -986,7 +989,7 @@ static void nfs4_copy_open_stateid(nfs4_stateid *dst, struct nfs4_state *state)
  * requests.
  */
 int nfs4_select_rw_stateid(struct nfs4_state *state,
-		fmode_t fmode, const struct nfs_lockowner *lockowner,
+		fmode_t fmode, const struct nfs_lock_context *l_ctx,
 		nfs4_stateid *dst, struct rpc_cred **cred)
 {
 	int ret;
@@ -995,7 +998,7 @@ int nfs4_select_rw_stateid(struct nfs4_state *state,
 		return -EIO;
 	if (cred != NULL)
 		*cred = NULL;
-	ret = nfs4_copy_lock_stateid(dst, state, lockowner);
+	ret = nfs4_copy_lock_stateid(dst, state, l_ctx);
 	if (ret == -EIO)
 		/* A lost lock - don't even consider delegations */
 		goto out;
@@ -2190,7 +2193,7 @@ void nfs4_schedule_session_recovery(struct nfs4_session *session, int err)
 	case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION:
 		set_bit(NFS4CLNT_BIND_CONN_TO_SESSION, &clp->cl_state);
 	}
-	nfs4_schedule_lease_recovery(clp);
+	nfs4_schedule_state_manager(clp);
 }
 EXPORT_SYMBOL_GPL(nfs4_schedule_session_recovery);
 
diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c
index fc89e5ed07ee..1af6268a7d8c 100644
--- a/fs/nfs/nfs4xdr.c
+++ b/fs/nfs/nfs4xdr.c
@@ -52,6 +52,7 @@
 #include <linux/nfs.h>
 #include <linux/nfs4.h>
 #include <linux/nfs_fs.h>
+#include <linux/fs_struct.h>
 
 #include "nfs4_fs.h"
 #include "internal.h"
@@ -415,6 +416,8 @@ static int nfs4_stat_to_errno(int);
 #else /* CONFIG_NFS_V4_1 */
 #define encode_sequence_maxsz	0
 #define decode_sequence_maxsz	0
+#define encode_layoutreturn_maxsz 0
+#define decode_layoutreturn_maxsz 0
 #endif /* CONFIG_NFS_V4_1 */
 
 #define NFS4_enc_compound_sz	(1024)  /* XXX: large enough? */
@@ -499,22 +502,22 @@ static int nfs4_stat_to_errno(int);
 				(compound_encode_hdr_maxsz + \
 				 encode_sequence_maxsz + \
 				 encode_putfh_maxsz + \
-				 encode_open_downgrade_maxsz + \
-				 encode_getattr_maxsz)
+				 encode_open_downgrade_maxsz)
 #define NFS4_dec_open_downgrade_sz \
 				(compound_decode_hdr_maxsz + \
 				 decode_sequence_maxsz + \
 				 decode_putfh_maxsz + \
-				 decode_open_downgrade_maxsz + \
-				 decode_getattr_maxsz)
+				 decode_open_downgrade_maxsz)
 #define NFS4_enc_close_sz	(compound_encode_hdr_maxsz + \
 				 encode_sequence_maxsz + \
 				 encode_putfh_maxsz + \
+				 encode_layoutreturn_maxsz + \
 				 encode_close_maxsz + \
 				 encode_getattr_maxsz)
 #define NFS4_dec_close_sz	(compound_decode_hdr_maxsz + \
 				 decode_sequence_maxsz + \
 				 decode_putfh_maxsz + \
+				 decode_layoutreturn_maxsz + \
 				 decode_close_maxsz + \
 				 decode_getattr_maxsz)
 #define NFS4_enc_setattr_sz	(compound_encode_hdr_maxsz + \
@@ -708,10 +711,13 @@ static int nfs4_stat_to_errno(int);
 #define NFS4_enc_delegreturn_sz	(compound_encode_hdr_maxsz + \
 				encode_sequence_maxsz + \
 				encode_putfh_maxsz + \
+				encode_layoutreturn_maxsz + \
 				encode_delegreturn_maxsz + \
 				encode_getattr_maxsz)
 #define NFS4_dec_delegreturn_sz (compound_decode_hdr_maxsz + \
 				decode_sequence_maxsz + \
+				decode_putfh_maxsz + \
+				decode_layoutreturn_maxsz + \
 				decode_delegreturn_maxsz + \
 				decode_getattr_maxsz)
 #define NFS4_enc_getacl_sz	(compound_encode_hdr_maxsz + \
@@ -1003,7 +1009,7 @@ static void encode_nfs4_verifier(struct xdr_stream *xdr, const nfs4_verifier *ve
 static void encode_attrs(struct xdr_stream *xdr, const struct iattr *iap,
 				const struct nfs4_label *label,
 				const struct nfs_server *server,
-				bool excl_check)
+				bool excl_check, const umode_t *umask)
 {
 	char owner_name[IDMAP_NAMESZ];
 	char owner_group[IDMAP_NAMESZ];
@@ -1017,18 +1023,21 @@ static void encode_attrs(struct xdr_stream *xdr, const struct iattr *iap,
 
 	/*
 	 * We reserve enough space to write the entire attribute buffer at once.
-	 * In the worst-case, this would be
-	 * 16(bitmap) + 4(attrlen) + 8(size) + 4(mode) + 4(atime) + 4(mtime)
-	 * = 40 bytes, plus any contribution from variable-length fields
-	 *            such as owner/group.
 	 */
 	if (iap->ia_valid & ATTR_SIZE) {
 		bmval[0] |= FATTR4_WORD0_SIZE;
 		len += 8;
 	}
+	if (!(server->attr_bitmask[2] & FATTR4_WORD2_MODE_UMASK))
+		umask = NULL;
 	if (iap->ia_valid & ATTR_MODE) {
-		bmval[1] |= FATTR4_WORD1_MODE;
-		len += 4;
+		if (umask) {
+			bmval[2] |= FATTR4_WORD2_MODE_UMASK;
+			len += 8;
+		} else {
+			bmval[1] |= FATTR4_WORD1_MODE;
+			len += 4;
+		}
 	}
 	if (iap->ia_valid & ATTR_UID) {
 		owner_namelen = nfs_map_uid_to_name(server, iap->ia_uid, owner_name, IDMAP_NAMESZ);
@@ -1129,6 +1138,10 @@ static void encode_attrs(struct xdr_stream *xdr, const struct iattr *iap,
 		*p++ = cpu_to_be32(label->len);
 		p = xdr_encode_opaque_fixed(p, label->label, label->len);
 	}
+	if (bmval[2] & FATTR4_WORD2_MODE_UMASK) {
+		*p++ = cpu_to_be32(iap->ia_mode & S_IALLUGO);
+		*p++ = cpu_to_be32(*umask);
+	}
 
 /* out: */
 }
@@ -1183,7 +1196,8 @@ static void encode_create(struct xdr_stream *xdr, const struct nfs4_create_arg *
 	}
 
 	encode_string(xdr, create->name->len, create->name->name);
-	encode_attrs(xdr, create->attrs, create->label, create->server, false);
+	encode_attrs(xdr, create->attrs, create->label, create->server, false,
+		     &create->umask);
 }
 
 static void encode_getattr_one(struct xdr_stream *xdr, uint32_t bitmap, struct compound_hdr *hdr)
@@ -1403,11 +1417,13 @@ static inline void encode_createmode(struct xdr_stream *xdr, const struct nfs_op
 	switch(arg->createmode) {
 	case NFS4_CREATE_UNCHECKED:
 		*p = cpu_to_be32(NFS4_CREATE_UNCHECKED);
-		encode_attrs(xdr, arg->u.attrs, arg->label, arg->server, false);
+		encode_attrs(xdr, arg->u.attrs, arg->label, arg->server, false,
+			     &arg->umask);
 		break;
 	case NFS4_CREATE_GUARDED:
 		*p = cpu_to_be32(NFS4_CREATE_GUARDED);
-		encode_attrs(xdr, arg->u.attrs, arg->label, arg->server, false);
+		encode_attrs(xdr, arg->u.attrs, arg->label, arg->server, false,
+			     &arg->umask);
 		break;
 	case NFS4_CREATE_EXCLUSIVE:
 		*p = cpu_to_be32(NFS4_CREATE_EXCLUSIVE);
@@ -1416,7 +1432,8 @@ static inline void encode_createmode(struct xdr_stream *xdr, const struct nfs_op
 	case NFS4_CREATE_EXCLUSIVE4_1:
 		*p = cpu_to_be32(NFS4_CREATE_EXCLUSIVE4_1);
 		encode_nfs4_verifier(xdr, &arg->u.verifier);
-		encode_attrs(xdr, arg->u.attrs, arg->label, arg->server, true);
+		encode_attrs(xdr, arg->u.attrs, arg->label, arg->server, true,
+			     &arg->umask);
 	}
 }
 
@@ -1672,7 +1689,7 @@ static void encode_setattr(struct xdr_stream *xdr, const struct nfs_setattrargs
 {
 	encode_op_hdr(xdr, OP_SETATTR, decode_setattr_maxsz, hdr);
 	encode_nfs4_stateid(xdr, &arg->stateid);
-	encode_attrs(xdr, arg->iap, arg->label, server, false);
+	encode_attrs(xdr, arg->iap, arg->label, server, false, NULL);
 }
 
 static void encode_setclientid(struct xdr_stream *xdr, const struct nfs4_setclientid *setclientid, struct compound_hdr *hdr)
@@ -2015,6 +2032,7 @@ encode_layoutreturn(struct xdr_stream *xdr,
 		    const struct nfs4_layoutreturn_args *args,
 		    struct compound_hdr *hdr)
 {
+	const struct pnfs_layoutdriver_type *lr_ops = NFS_SERVER(args->inode)->pnfs_curr_ld;
 	__be32 *p;
 
 	encode_op_hdr(xdr, OP_LAYOUTRETURN, decode_layoutreturn_maxsz, hdr);
@@ -2029,10 +2047,11 @@ encode_layoutreturn(struct xdr_stream *xdr,
 	spin_lock(&args->inode->i_lock);
 	encode_nfs4_stateid(xdr, &args->stateid);
 	spin_unlock(&args->inode->i_lock);
-	if (NFS_SERVER(args->inode)->pnfs_curr_ld->encode_layoutreturn) {
-		NFS_SERVER(args->inode)->pnfs_curr_ld->encode_layoutreturn(
-			NFS_I(args->inode)->layout, xdr, args);
-	} else
+	if (args->ld_private->ops && args->ld_private->ops->encode)
+		args->ld_private->ops->encode(xdr, args, args->ld_private);
+	else if (lr_ops->encode_layoutreturn)
+		lr_ops->encode_layoutreturn(xdr, args);
+	else
 		encode_uint32(xdr, 0);
 }
 
@@ -2062,6 +2081,13 @@ static void encode_free_stateid(struct xdr_stream *xdr,
 	encode_op_hdr(xdr, OP_FREE_STATEID, decode_free_stateid_maxsz, hdr);
 	encode_nfs4_stateid(xdr, &args->stateid);
 }
+#else
+static inline void
+encode_layoutreturn(struct xdr_stream *xdr,
+		    const struct nfs4_layoutreturn_args *args,
+		    struct compound_hdr *hdr)
+{
+}
 #endif /* CONFIG_NFS_V4_1 */
 
 /*
@@ -2249,8 +2275,11 @@ static void nfs4_xdr_enc_close(struct rpc_rqst *req, struct xdr_stream *xdr,
 	encode_compound_hdr(xdr, req, &hdr);
 	encode_sequence(xdr, &args->seq_args, &hdr);
 	encode_putfh(xdr, args->fh, &hdr);
+	if (args->lr_args)
+		encode_layoutreturn(xdr, args->lr_args, &hdr);
 	encode_close(xdr, args, &hdr);
-	encode_getfattr(xdr, args->bitmask, &hdr);
+	if (args->bitmask != NULL)
+		encode_getfattr(xdr, args->bitmask, &hdr);
 	encode_nops(&hdr);
 }
 
@@ -2328,7 +2357,6 @@ static void nfs4_xdr_enc_open_downgrade(struct rpc_rqst *req,
 	encode_sequence(xdr, &args->seq_args, &hdr);
 	encode_putfh(xdr, args->fh, &hdr);
 	encode_open_downgrade(xdr, args, &hdr);
-	encode_getfattr(xdr, args->bitmask, &hdr);
 	encode_nops(&hdr);
 }
 
@@ -2671,6 +2699,8 @@ static void nfs4_xdr_enc_delegreturn(struct rpc_rqst *req,
 	encode_compound_hdr(xdr, req, &hdr);
 	encode_sequence(xdr, &args->seq_args, &hdr);
 	encode_putfh(xdr, args->fhandle, &hdr);
+	if (args->lr_args)
+		encode_layoutreturn(xdr, args->lr_args, &hdr);
 	encode_getfattr(xdr, args->bitmask, &hdr);
 	encode_delegreturn(xdr, args->stateid, &hdr);
 	encode_nops(&hdr);
@@ -6089,6 +6119,13 @@ static int decode_free_stateid(struct xdr_stream *xdr,
 	res->status = decode_op_hdr(xdr, OP_FREE_STATEID);
 	return res->status;
 }
+#else
+static inline
+int decode_layoutreturn(struct xdr_stream *xdr,
+			       struct nfs4_layoutreturn_res *res)
+{
+	return 0;
+}
 #endif /* CONFIG_NFS_V4_1 */
 
 /*
@@ -6115,9 +6152,6 @@ static int nfs4_xdr_dec_open_downgrade(struct rpc_rqst *rqstp,
 	if (status)
 		goto out;
 	status = decode_open_downgrade(xdr, res);
-	if (status != 0)
-		goto out;
-	decode_getfattr(xdr, res->fattr, res->server);
 out:
 	return status;
 }
@@ -6444,6 +6478,12 @@ static int nfs4_xdr_dec_close(struct rpc_rqst *rqstp, struct xdr_stream *xdr,
 	status = decode_putfh(xdr);
 	if (status)
 		goto out;
+	if (res->lr_res) {
+		status = decode_layoutreturn(xdr, res->lr_res);
+		res->lr_ret = status;
+		if (status)
+			goto out;
+	}
 	status = decode_close(xdr, res);
 	if (status != 0)
 		goto out;
@@ -6920,6 +6960,12 @@ static int nfs4_xdr_dec_delegreturn(struct rpc_rqst *rqstp,
 	status = decode_putfh(xdr);
 	if (status != 0)
 		goto out;
+	if (res->lr_res) {
+		status = decode_layoutreturn(xdr, res->lr_res);
+		res->lr_ret = status;
+		if (status)
+			goto out;
+	}
 	status = decode_getfattr(xdr, res->fattr, res->server);
 	if (status != 0)
 		goto out;
diff --git a/fs/nfs/objlayout/objlayout.c b/fs/nfs/objlayout/objlayout.c
index 919efd4a1a23..2a4cdce939a0 100644
--- a/fs/nfs/objlayout/objlayout.c
+++ b/fs/nfs/objlayout/objlayout.c
@@ -504,10 +504,10 @@ encode_accumulated_error(struct objlayout *objlay, __be32 *p)
 }
 
 void
-objlayout_encode_layoutreturn(struct pnfs_layout_hdr *pnfslay,
-			      struct xdr_stream *xdr,
+objlayout_encode_layoutreturn(struct xdr_stream *xdr,
 			      const struct nfs4_layoutreturn_args *args)
 {
+	struct pnfs_layout_hdr *pnfslay = args->layout;
 	struct objlayout *objlay = OBJLAYOUT(pnfslay);
 	struct objlayout_io_res *oir, *tmp;
 	__be32 *start;
diff --git a/fs/nfs/objlayout/objlayout.h b/fs/nfs/objlayout/objlayout.h
index 2641dbad345c..fc94a5872ed4 100644
--- a/fs/nfs/objlayout/objlayout.h
+++ b/fs/nfs/objlayout/objlayout.h
@@ -175,7 +175,6 @@ extern void objlayout_encode_layoutcommit(
 	const struct nfs4_layoutcommit_args *);
 
 extern void objlayout_encode_layoutreturn(
-	struct pnfs_layout_hdr *,
 	struct xdr_stream *,
 	const struct nfs4_layoutreturn_args *);
 
diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c
index 965db474f4b0..6e629b856a00 100644
--- a/fs/nfs/pagelist.c
+++ b/fs/nfs/pagelist.c
@@ -867,8 +867,7 @@ static void nfs_pageio_cleanup_mirroring(struct nfs_pageio_descriptor *pgio)
 static bool nfs_match_lock_context(const struct nfs_lock_context *l1,
 		const struct nfs_lock_context *l2)
 {
-	return l1->lockowner.l_owner == l2->lockowner.l_owner
-		&& l1->lockowner.l_pid == l2->lockowner.l_pid;
+	return l1->lockowner == l2->lockowner;
 }
 
 /**
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 259ef85f435a..896df7bdf85f 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -54,6 +54,12 @@ static DEFINE_SPINLOCK(pnfs_spinlock);
 static LIST_HEAD(pnfs_modules_tbl);
 
 static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo);
+static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo,
+		struct list_head *free_me,
+		const struct pnfs_layout_range *range,
+		u32 seq);
+static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
+		                struct list_head *tmp_list);
 
 /* Return the registered pnfs layout driver module matching given id */
 static struct pnfs_layoutdriver_type *
@@ -299,6 +305,49 @@ pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
 	}
 }
 
+static void
+pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode,
+			 u32 seq)
+{
+	if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode)
+		iomode = IOMODE_ANY;
+	lo->plh_return_iomode = iomode;
+	set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
+	if (seq != 0) {
+		WARN_ON_ONCE(lo->plh_return_seq != 0 && lo->plh_return_seq != seq);
+		lo->plh_return_seq = seq;
+	}
+}
+
+static void
+pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo)
+{
+	lo->plh_return_iomode = 0;
+	lo->plh_return_seq = 0;
+	clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
+}
+
+static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
+{
+	clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
+	clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags);
+	smp_mb__after_atomic();
+	wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
+	rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
+}
+
+static void
+pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg,
+		struct list_head *free_me)
+{
+	clear_bit(NFS_LSEG_ROC, &lseg->pls_flags);
+	clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
+	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags))
+		pnfs_lseg_dec_and_remove_zero(lseg, free_me);
+	if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
+		pnfs_lseg_dec_and_remove_zero(lseg, free_me);
+}
+
 /*
  * Mark a pnfs_layout_hdr and all associated layout segments as invalid
  *
@@ -315,9 +364,17 @@ pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo,
 		.offset = 0,
 		.length = NFS4_MAX_UINT64,
 	};
+	struct pnfs_layout_segment *lseg, *next;
 
 	set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
-	return pnfs_mark_matching_lsegs_invalid(lo, lseg_list, &range, 0);
+	pnfs_clear_layoutreturn_info(lo);
+	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
+		pnfs_clear_lseg_state(lseg, lseg_list);
+	pnfs_free_returned_lsegs(lo, lseg_list, &range, 0);
+	if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) &&
+	    !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags))
+		pnfs_clear_layoutreturn_waitbit(lo);
+	return !list_empty(&lo->plh_segs);
 }
 
 static int
@@ -396,27 +453,42 @@ pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg,
 
 static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
 {
-	struct inode *ino = lseg->pls_layout->plh_inode;
-
-	NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
+	if (lseg != NULL) {
+		struct inode *inode = lseg->pls_layout->plh_inode;
+		NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg);
+	}
 }
 
 static void
 pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
 		struct pnfs_layout_segment *lseg)
 {
-	struct inode *inode = lo->plh_inode;
-
 	WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
 	list_del_init(&lseg->pls_list);
 	/* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
 	atomic_dec(&lo->plh_refcount);
-	if (list_empty(&lo->plh_segs)) {
+	if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
+		return;
+	if (list_empty(&lo->plh_segs) &&
+	    !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) &&
+	    !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
 		if (atomic_read(&lo->plh_outstanding) == 0)
 			set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
 		clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
 	}
-	rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
+}
+
+static bool
+pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo,
+		struct pnfs_layout_segment *lseg)
+{
+	if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
+	    pnfs_layout_is_valid(lo)) {
+		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);
+		list_move_tail(&lseg->pls_list, &lo->plh_return_segs);
+		return true;
+	}
+	return false;
 }
 
 void
@@ -442,6 +514,8 @@ pnfs_put_lseg(struct pnfs_layout_segment *lseg)
 		}
 		pnfs_get_layout_hdr(lo);
 		pnfs_layout_remove_lseg(lo, lseg);
+		if (pnfs_cache_lseg_for_layoutreturn(lo, lseg))
+			lseg = NULL;
 		spin_unlock(&inode->i_lock);
 		pnfs_free_lseg(lseg);
 		pnfs_put_layout_hdr(lo);
@@ -482,22 +556,15 @@ pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg)
 		struct pnfs_layout_hdr *lo = lseg->pls_layout;
 		if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags))
 			return;
-		pnfs_get_layout_hdr(lo);
 		pnfs_layout_remove_lseg(lo, lseg);
-		pnfs_free_lseg_async(lseg);
+		if (!pnfs_cache_lseg_for_layoutreturn(lo, lseg)) {
+			pnfs_get_layout_hdr(lo);
+			pnfs_free_lseg_async(lseg);
+		}
 	}
 }
 EXPORT_SYMBOL_GPL(pnfs_put_lseg_locked);
 
-static u64
-end_offset(u64 start, u64 len)
-{
-	u64 end;
-
-	end = start + len;
-	return end >= start ? end : NFS4_MAX_UINT64;
-}
-
 /*
  * is l2 fully contained in l1?
  *   start1                             end1
@@ -510,33 +577,13 @@ pnfs_lseg_range_contained(const struct pnfs_layout_range *l1,
 		 const struct pnfs_layout_range *l2)
 {
 	u64 start1 = l1->offset;
-	u64 end1 = end_offset(start1, l1->length);
+	u64 end1 = pnfs_end_offset(start1, l1->length);
 	u64 start2 = l2->offset;
-	u64 end2 = end_offset(start2, l2->length);
+	u64 end2 = pnfs_end_offset(start2, l2->length);
 
 	return (start1 <= start2) && (end1 >= end2);
 }
 
-/*
- * is l1 and l2 intersecting?
- *   start1                             end1
- *   [----------------------------------)
- *                              start2           end2
- *                              [----------------)
- */
-static bool
-pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1,
-		    const struct pnfs_layout_range *l2)
-{
-	u64 start1 = l1->offset;
-	u64 end1 = end_offset(start1, l1->length);
-	u64 start2 = l2->offset;
-	u64 end2 = end_offset(start2, l2->length);
-
-	return (end1 == NFS4_MAX_UINT64 || end1 > start2) &&
-	       (end2 == NFS4_MAX_UINT64 || end2 > start1);
-}
-
 static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
 		struct list_head *tmp_list)
 {
@@ -637,6 +684,20 @@ pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
 	return remaining;
 }
 
+static void
+pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo,
+		struct list_head *free_me,
+		const struct pnfs_layout_range *range,
+		u32 seq)
+{
+	struct pnfs_layout_segment *lseg, *next;
+
+	list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) {
+		if (pnfs_match_lseg_recall(lseg, range, seq))
+			list_move_tail(&lseg->pls_list, free_me);
+	}
+}
+
 /* note free_me must contain lsegs from a single layout_hdr */
 void
 pnfs_free_lseg_list(struct list_head *free_me)
@@ -701,6 +762,8 @@ pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
 	struct inode *inode;
 
 	list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
+		if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags))
+			continue;
 		inode = igrab(lo->plh_inode);
 		if (inode == NULL)
 			continue;
@@ -816,14 +879,6 @@ pnfs_destroy_all_layouts(struct nfs_client *clp)
 	pnfs_destroy_layouts_byclid(clp, false);
 }
 
-static void
-pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo)
-{
-	lo->plh_return_iomode = 0;
-	lo->plh_return_seq = 0;
-	clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
-}
-
 /* update lo->plh_stateid with new if is more recent */
 void
 pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
@@ -941,12 +996,31 @@ static void pnfs_clear_layoutcommit(struct inode *inode,
 	}
 }
 
-void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
+void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo,
+		const nfs4_stateid *arg_stateid,
+		const struct pnfs_layout_range *range,
+		const nfs4_stateid *stateid)
 {
-	clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
-	smp_mb__after_atomic();
-	wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
-	rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
+	struct inode *inode = lo->plh_inode;
+	LIST_HEAD(freeme);
+
+	spin_lock(&inode->i_lock);
+	if (!pnfs_layout_is_valid(lo) || !arg_stateid ||
+	    !nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid))
+		goto out_unlock;
+	if (stateid) {
+		u32 seq = be32_to_cpu(arg_stateid->seqid);
+
+		pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq);
+		pnfs_free_returned_lsegs(lo, &freeme, range, seq);
+		pnfs_set_layout_stateid(lo, stateid, true);
+	} else
+		pnfs_mark_layout_stateid_invalid(lo, &freeme);
+out_unlock:
+	pnfs_clear_layoutreturn_waitbit(lo);
+	spin_unlock(&inode->i_lock);
+	pnfs_free_lseg_list(&freeme);
+
 }
 
 static bool
@@ -957,8 +1031,9 @@ pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo,
 	/* Serialise LAYOUTGET/LAYOUTRETURN */
 	if (atomic_read(&lo->plh_outstanding) != 0)
 		return false;
-	if (test_and_set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
+	if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags))
 		return false;
+	set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
 	pnfs_get_layout_hdr(lo);
 	if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) {
 		if (stateid != NULL) {
@@ -978,11 +1053,29 @@ pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo,
 	return true;
 }
 
+static void
+pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args,
+		struct pnfs_layout_hdr *lo,
+		const nfs4_stateid *stateid,
+		enum pnfs_iomode iomode)
+{
+	struct inode *inode = lo->plh_inode;
+
+	args->layout_type = NFS_SERVER(inode)->pnfs_curr_ld->id;
+	args->inode = inode;
+	args->range.iomode = iomode;
+	args->range.offset = 0;
+	args->range.length = NFS4_MAX_UINT64;
+	args->layout = lo;
+	nfs4_stateid_copy(&args->stateid, stateid);
+}
+
 static int
 pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
 		       enum pnfs_iomode iomode, bool sync)
 {
 	struct inode *ino = lo->plh_inode;
+	struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
 	struct nfs4_layoutreturn *lrp;
 	int status = 0;
 
@@ -996,15 +1089,12 @@ pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
 		goto out;
 	}
 
-	nfs4_stateid_copy(&lrp->args.stateid, stateid);
-	lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
-	lrp->args.inode = ino;
-	lrp->args.range.iomode = iomode;
-	lrp->args.range.offset = 0;
-	lrp->args.range.length = NFS4_MAX_UINT64;
-	lrp->args.layout = lo;
+	pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode);
+	lrp->args.ld_private = &lrp->ld_private;
 	lrp->clp = NFS_SERVER(ino)->nfs_client;
 	lrp->cred = lo->plh_lc_cred;
+	if (ld->prepare_layoutreturn)
+		ld->prepare_layoutreturn(&lrp->args);
 
 	status = nfs4_proc_layoutreturn(lrp, sync);
 out:
@@ -1067,7 +1157,7 @@ _pnfs_return_layout(struct inode *ino)
 	struct nfs_inode *nfsi = NFS_I(ino);
 	LIST_HEAD(tmp_list);
 	nfs4_stateid stateid;
-	int status = 0, empty;
+	int status = 0;
 	bool send;
 
 	dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
@@ -1081,7 +1171,14 @@ _pnfs_return_layout(struct inode *ino)
 	}
 	/* Reference matched in nfs4_layoutreturn_release */
 	pnfs_get_layout_hdr(lo);
-	empty = list_empty(&lo->plh_segs);
+	/* Is there an outstanding layoutreturn ? */
+	if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) {
+		spin_unlock(&ino->i_lock);
+		if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN,
+					TASK_UNINTERRUPTIBLE))
+			goto out_put_layout_hdr;
+		spin_lock(&ino->i_lock);
+	}
 	pnfs_clear_layoutcommit(ino, &tmp_list);
 	pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL, 0);
 
@@ -1095,7 +1192,7 @@ _pnfs_return_layout(struct inode *ino)
 	}
 
 	/* Don't send a LAYOUTRETURN if list was initially empty */
-	if (empty) {
+	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) {
 		spin_unlock(&ino->i_lock);
 		dprintk("NFS: %s no layout segments to return\n", __func__);
 		goto out_put_layout_hdr;
@@ -1141,21 +1238,36 @@ pnfs_commit_and_return_layout(struct inode *inode)
 	return ret;
 }
 
-bool pnfs_roc(struct inode *ino)
+bool pnfs_roc(struct inode *ino,
+		struct nfs4_layoutreturn_args *args,
+		struct nfs4_layoutreturn_res *res,
+		const struct rpc_cred *cred)
 {
 	struct nfs_inode *nfsi = NFS_I(ino);
 	struct nfs_open_context *ctx;
 	struct nfs4_state *state;
 	struct pnfs_layout_hdr *lo;
-	struct pnfs_layout_segment *lseg, *tmp;
+	struct pnfs_layout_segment *lseg, *next;
 	nfs4_stateid stateid;
-	LIST_HEAD(tmp_list);
-	bool found = false, layoutreturn = false, roc = false;
+	enum pnfs_iomode iomode = 0;
+	bool layoutreturn = false, roc = false;
 
+	if (!nfs_have_layout(ino))
+		return false;
+retry:
 	spin_lock(&ino->i_lock);
 	lo = nfsi->layout;
-	if (!lo || test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
+	if (!lo || !pnfs_layout_is_valid(lo) ||
+	    test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
 		goto out_noroc;
+	if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) {
+		pnfs_get_layout_hdr(lo);
+		spin_unlock(&ino->i_lock);
+		wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN,
+				TASK_UNINTERRUPTIBLE);
+		pnfs_put_layout_hdr(lo);
+		goto retry;
+	}
 
 	/* no roc if we hold a delegation */
 	if (nfs4_check_delegation(ino, FMODE_READ))
@@ -1168,78 +1280,73 @@ bool pnfs_roc(struct inode *ino)
 			goto out_noroc;
 	}
 
-	/* always send layoutreturn if being marked so */
-	if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
-		layoutreturn = pnfs_prepare_layoutreturn(lo,
-				&stateid, NULL);
 
-	list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list)
+	list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) {
 		/* If we are sending layoutreturn, invalidate all valid lsegs */
-		if (layoutreturn || test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
-			mark_lseg_invalid(lseg, &tmp_list);
-			found = true;
-		}
+		if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags))
+			continue;
+		/*
+		 * Note: mark lseg for return so pnfs_layout_remove_lseg
+		 * doesn't invalidate the layout for us.
+		 */
+		set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
+		if (!mark_lseg_invalid(lseg, &lo->plh_return_segs))
+			continue;
+		pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0);
+	}
+
+	if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
+		goto out_noroc;
+
 	/* ROC in two conditions:
 	 * 1. there are ROC lsegs
 	 * 2. we don't send layoutreturn
 	 */
-	if (found && !layoutreturn) {
-		/* lo ref dropped in pnfs_roc_release() */
-		pnfs_get_layout_hdr(lo);
-		roc = true;
-	}
+	/* lo ref dropped in pnfs_roc_release() */
+	layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
+	/* If the creds don't match, we can't compound the layoutreturn */
+	if (!layoutreturn || cred != lo->plh_lc_cred)
+		goto out_noroc;
+
+	roc = layoutreturn;
+	pnfs_init_layoutreturn_args(args, lo, &stateid, iomode);
+	res->lrs_present = 0;
+	layoutreturn = false;
 
 out_noroc:
 	spin_unlock(&ino->i_lock);
-	pnfs_free_lseg_list(&tmp_list);
 	pnfs_layoutcommit_inode(ino, true);
+	if (roc) {
+		struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
+		if (ld->prepare_layoutreturn)
+			ld->prepare_layoutreturn(args);
+		return true;
+	}
 	if (layoutreturn)
-		pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
-	return roc;
-}
-
-void pnfs_roc_release(struct inode *ino)
-{
-	struct pnfs_layout_hdr *lo;
-
-	spin_lock(&ino->i_lock);
-	lo = NFS_I(ino)->layout;
-	pnfs_clear_layoutreturn_waitbit(lo);
-	if (atomic_dec_and_test(&lo->plh_refcount)) {
-		pnfs_detach_layout_hdr(lo);
-		spin_unlock(&ino->i_lock);
-		pnfs_free_layout_hdr(lo);
-	} else
-		spin_unlock(&ino->i_lock);
+		pnfs_send_layoutreturn(lo, &stateid, iomode, true);
+	return false;
 }
 
-void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
+void pnfs_roc_release(struct nfs4_layoutreturn_args *args,
+		struct nfs4_layoutreturn_res *res,
+		int ret)
 {
-	struct pnfs_layout_hdr *lo;
+	struct pnfs_layout_hdr *lo = args->layout;
+	const nfs4_stateid *arg_stateid = NULL;
+	const nfs4_stateid *res_stateid = NULL;
+	struct nfs4_xdr_opaque_data *ld_private = args->ld_private;
 
-	spin_lock(&ino->i_lock);
-	lo = NFS_I(ino)->layout;
-	if (pnfs_seqid_is_newer(barrier, lo->plh_barrier))
-		lo->plh_barrier = barrier;
-	spin_unlock(&ino->i_lock);
-	trace_nfs4_layoutreturn_on_close(ino, 0);
-}
-
-void pnfs_roc_get_barrier(struct inode *ino, u32 *barrier)
-{
-	struct nfs_inode *nfsi = NFS_I(ino);
-	struct pnfs_layout_hdr *lo;
-	u32 current_seqid;
-
-	spin_lock(&ino->i_lock);
-	lo = nfsi->layout;
-	current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
-
-	/* Since close does not return a layout stateid for use as
-	 * a barrier, we choose the worst-case barrier.
-	 */
-	*barrier = current_seqid + atomic_read(&lo->plh_outstanding);
-	spin_unlock(&ino->i_lock);
+	if (ret == 0) {
+		arg_stateid = &args->stateid;
+		if (res->lrs_present)
+			res_stateid = &res->stateid;
+	}
+	pnfs_layoutreturn_free_lsegs(lo, arg_stateid, &args->range,
+			res_stateid);
+	if (ld_private && ld_private->ops && ld_private->ops->free)
+		ld_private->ops->free(ld_private);
+	pnfs_put_layout_hdr(lo);
+	trace_nfs4_layoutreturn_on_close(args->inode, 0);
 }
 
 bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
@@ -1252,13 +1359,11 @@ bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
 	 * i_lock */
         spin_lock(&ino->i_lock);
         lo = nfsi->layout;
-        if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
+        if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
+                rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
                 sleep = true;
+	}
         spin_unlock(&ino->i_lock);
-
-        if (sleep)
-                rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
-
         return sleep;
 }
 
@@ -1375,6 +1480,7 @@ alloc_init_layout_hdr(struct inode *ino,
 	atomic_set(&lo->plh_refcount, 1);
 	INIT_LIST_HEAD(&lo->plh_layouts);
 	INIT_LIST_HEAD(&lo->plh_segs);
+	INIT_LIST_HEAD(&lo->plh_return_segs);
 	INIT_LIST_HEAD(&lo->plh_bulk_destroy);
 	lo->plh_inode = ino;
 	lo->plh_lc_cred = get_rpccred(ctx->cred);
@@ -1841,7 +1947,10 @@ pnfs_layout_process(struct nfs4_layoutget *lgp)
 		goto out_forget;
 	}
 
-	if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
+	if (!pnfs_layout_is_valid(lo)) {
+		/* We have a completely new layout */
+		pnfs_set_layout_stateid(lo, &res->stateid, true);
+	} else if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
 		/* existing state ID, make sure the sequence number matches. */
 		if (pnfs_layout_stateid_blocked(lo, &res->stateid)) {
 			dprintk("%s forget reply due to sequence\n", __func__);
@@ -1851,12 +1960,10 @@ pnfs_layout_process(struct nfs4_layoutget *lgp)
 	} else {
 		/*
 		 * We got an entirely new state ID.  Mark all segments for the
-		 * inode invalid, and don't bother validating the stateid
-		 * sequence number.
+		 * inode invalid, and retry the layoutget
 		 */
 		pnfs_mark_layout_stateid_invalid(lo, &free_me);
-
-		pnfs_set_layout_stateid(lo, &res->stateid, true);
+		goto out_forget;
 	}
 
 	pnfs_get_lseg(lseg);
@@ -1877,20 +1984,6 @@ out_forget:
 	return ERR_PTR(-EAGAIN);
 }
 
-static void
-pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode,
-			 u32 seq)
-{
-	if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode)
-		iomode = IOMODE_ANY;
-	lo->plh_return_iomode = iomode;
-	set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
-	if (seq != 0) {
-		WARN_ON_ONCE(lo->plh_return_seq != 0 && lo->plh_return_seq != seq);
-		lo->plh_return_seq = seq;
-	}
-}
-
 /**
  * pnfs_mark_matching_lsegs_return - Free or return matching layout segments
  * @lo: pointer to layout header
@@ -1945,17 +2038,18 @@ void pnfs_error_mark_layout_for_return(struct inode *inode,
 		.offset = 0,
 		.length = NFS4_MAX_UINT64,
 	};
-	LIST_HEAD(free_me);
 	bool return_now = false;
 
 	spin_lock(&inode->i_lock);
 	pnfs_set_plh_return_info(lo, range.iomode, 0);
+	/* Block LAYOUTGET */
+	set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
 	/*
 	 * mark all matching lsegs so that we are sure to have no live
 	 * segments at hand when sending layoutreturn. See pnfs_put_lseg()
 	 * for how it works.
 	 */
-	if (!pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0)) {
+	if (!pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, &range, 0)) {
 		nfs4_stateid stateid;
 		enum pnfs_iomode iomode;
 
@@ -1967,7 +2061,6 @@ void pnfs_error_mark_layout_for_return(struct inode *inode,
 		spin_unlock(&inode->i_lock);
 		nfs_commit_inode(inode, 0);
 	}
-	pnfs_free_lseg_list(&free_me);
 }
 EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);
 
@@ -2063,7 +2156,7 @@ pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
 	 *
 	 */
 	if (pgio->pg_lseg) {
-		seg_end = end_offset(pgio->pg_lseg->pls_range.offset,
+		seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset,
 				     pgio->pg_lseg->pls_range.length);
 		req_start = req_offset(req);
 		WARN_ON_ONCE(req_start >= seg_end);
@@ -2286,6 +2379,10 @@ void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr)
 	struct nfs_pageio_descriptor pgio;
 
 	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
+		/* Prevent deadlocks with layoutreturn! */
+		pnfs_put_lseg(hdr->lseg);
+		hdr->lseg = NULL;
+
 		nfs_pageio_init_read(&pgio, hdr->inode, false,
 					hdr->completion_ops);
 		hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr);
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 5c295512c967..63f77b49a586 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -96,6 +96,7 @@ enum {
 	NFS_LAYOUT_RW_FAILED,		/* get rw layout failed stop trying */
 	NFS_LAYOUT_BULK_RECALL,		/* bulk recall affecting layout */
 	NFS_LAYOUT_RETURN,		/* layoutreturn in progress */
+	NFS_LAYOUT_RETURN_LOCK,		/* Serialise layoutreturn */
 	NFS_LAYOUT_RETURN_REQUESTED,	/* Return this layout ASAP */
 	NFS_LAYOUT_INVALID_STID,	/* layout stateid id is invalid */
 	NFS_LAYOUT_FIRST_LAYOUTGET,	/* Serialize first layoutget */
@@ -171,8 +172,8 @@ struct pnfs_layoutdriver_type {
 			(struct nfs_server *server, struct pnfs_device *pdev,
 			gfp_t gfp_flags);
 
-	void (*encode_layoutreturn) (struct pnfs_layout_hdr *layoutid,
-				     struct xdr_stream *xdr,
+	int (*prepare_layoutreturn) (struct nfs4_layoutreturn_args *);
+	void (*encode_layoutreturn) (struct xdr_stream *xdr,
 				     const struct nfs4_layoutreturn_args *args);
 
 	void (*cleanup_layoutcommit) (struct nfs4_layoutcommit_data *data);
@@ -181,7 +182,6 @@ struct pnfs_layoutdriver_type {
 				     struct xdr_stream *xdr,
 				     const struct nfs4_layoutcommit_args *args);
 	int (*prepare_layoutstats) (struct nfs42_layoutstat_args *args);
-	void (*cleanup_layoutstats) (struct nfs42_layoutstat_data *data);
 };
 
 struct pnfs_layout_hdr {
@@ -190,6 +190,7 @@ struct pnfs_layout_hdr {
 	struct list_head	plh_layouts;   /* other client layouts */
 	struct list_head	plh_bulk_destroy;
 	struct list_head	plh_segs;      /* layout segments list */
+	struct list_head	plh_return_segs; /* invalid layout segments */
 	unsigned long		plh_block_lgets; /* block LAYOUTGET if >0 */
 	unsigned long		plh_retry_timestamp;
 	unsigned long		plh_flags;
@@ -270,10 +271,13 @@ int pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
 				u32 seq);
 int pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo,
 		struct list_head *lseg_list);
-bool pnfs_roc(struct inode *ino);
-void pnfs_roc_release(struct inode *ino);
-void pnfs_roc_set_barrier(struct inode *ino, u32 barrier);
-void pnfs_roc_get_barrier(struct inode *ino, u32 *barrier);
+bool pnfs_roc(struct inode *ino,
+		struct nfs4_layoutreturn_args *args,
+		struct nfs4_layoutreturn_res *res,
+		const struct rpc_cred *cred);
+void pnfs_roc_release(struct nfs4_layoutreturn_args *args,
+		struct nfs4_layoutreturn_res *res,
+		int ret);
 bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task);
 void pnfs_set_layoutcommit(struct inode *, struct pnfs_layout_segment *, loff_t);
 void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data);
@@ -292,7 +296,10 @@ struct pnfs_layout_segment *pnfs_update_layout(struct inode *ino,
 					       enum pnfs_iomode iomode,
 					       bool strict_iomode,
 					       gfp_t gfp_flags);
-void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo);
+void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo,
+		const nfs4_stateid *arg_stateid,
+		const struct pnfs_layout_range *range,
+		const nfs4_stateid *stateid);
 
 void pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo,
 		   struct pnfs_layout_segment *lseg,
@@ -362,8 +369,7 @@ struct nfs4_pnfs_ds *nfs4_pnfs_ds_add(struct list_head *dsaddrs,
 void nfs4_pnfs_v3_ds_connect_unload(void);
 void nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds,
 			  struct nfs4_deviceid_node *devid, unsigned int timeo,
-			  unsigned int retrans, u32 version, u32 minor_version,
-			  rpc_authflavor_t au_flavor);
+			  unsigned int retrans, u32 version, u32 minor_version);
 struct nfs4_pnfs_ds_addr *nfs4_decode_mp_ds_addr(struct net *net,
 						 struct xdr_stream *xdr,
 						 gfp_t gfp_flags);
@@ -559,6 +565,38 @@ pnfs_copy_range(struct pnfs_layout_range *dst,
 	memcpy(dst, src, sizeof(*dst));
 }
 
+static inline u64
+pnfs_end_offset(u64 start, u64 len)
+{
+	if (NFS4_MAX_UINT64 - start <= len)
+		return NFS4_MAX_UINT64;
+	return start + len;
+}
+
+/*
+ * Are 2 ranges intersecting?
+ *   start1                             end1
+ *   [----------------------------------)
+ *                                start2           end2
+ *                                [----------------)
+ */
+static inline bool
+pnfs_is_range_intersecting(u64 start1, u64 end1, u64 start2, u64 end2)
+{
+	return (end1 == NFS4_MAX_UINT64 || start2 < end1) &&
+		(end2 == NFS4_MAX_UINT64 || start1 < end2);
+}
+
+static inline bool
+pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1,
+		const struct pnfs_layout_range *l2)
+{
+	u64 end1 = pnfs_end_offset(l1->offset, l1->length);
+	u64 end2 = pnfs_end_offset(l2->offset, l2->length);
+
+	return pnfs_is_range_intersecting(l1->offset, end1, l2->offset, end2);
+}
+
 extern unsigned int layoutstats_timer;
 
 #ifdef NFS_DEBUG
@@ -630,23 +668,18 @@ pnfs_layoutcommit_outstanding(struct inode *inode)
 
 
 static inline bool
-pnfs_roc(struct inode *ino)
+pnfs_roc(struct inode *ino,
+		struct nfs4_layoutreturn_args *args,
+		struct nfs4_layoutreturn_res *res,
+		const struct rpc_cred *cred)
 {
 	return false;
 }
 
 static inline void
-pnfs_roc_release(struct inode *ino)
-{
-}
-
-static inline void
-pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
-{
-}
-
-static inline void
-pnfs_roc_get_barrier(struct inode *ino, u32 *barrier)
+pnfs_roc_release(struct nfs4_layoutreturn_args *args,
+		struct nfs4_layoutreturn_res *res,
+		int ret)
 {
 }
 
diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c
index 53b4705abcc7..9414b492439f 100644
--- a/fs/nfs/pnfs_nfs.c
+++ b/fs/nfs/pnfs_nfs.c
@@ -600,8 +600,7 @@ static struct nfs_client *(*get_v3_ds_connect)(
 			int ds_addrlen,
 			int ds_proto,
 			unsigned int ds_timeo,
-			unsigned int ds_retrans,
-			rpc_authflavor_t au_flavor);
+			unsigned int ds_retrans);
 
 static bool load_v3_ds_connect(void)
 {
@@ -625,15 +624,13 @@ EXPORT_SYMBOL_GPL(nfs4_pnfs_v3_ds_connect_unload);
 static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv,
 				 struct nfs4_pnfs_ds *ds,
 				 unsigned int timeo,
-				 unsigned int retrans,
-				 rpc_authflavor_t au_flavor)
+				 unsigned int retrans)
 {
 	struct nfs_client *clp = ERR_PTR(-EIO);
 	struct nfs4_pnfs_ds_addr *da;
 	int status = 0;
 
-	dprintk("--> %s DS %s au_flavor %d\n", __func__,
-		ds->ds_remotestr, au_flavor);
+	dprintk("--> %s DS %s\n", __func__, ds->ds_remotestr);
 
 	if (!load_v3_ds_connect())
 		goto out;
@@ -657,7 +654,7 @@ static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv,
 			clp = get_v3_ds_connect(mds_srv,
 					(struct sockaddr *)&da->da_addr,
 					da->da_addrlen, IPPROTO_TCP,
-					timeo, retrans, au_flavor);
+					timeo, retrans);
 	}
 
 	if (IS_ERR(clp)) {
@@ -676,15 +673,13 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
 				 struct nfs4_pnfs_ds *ds,
 				 unsigned int timeo,
 				 unsigned int retrans,
-				 u32 minor_version,
-				 rpc_authflavor_t au_flavor)
+				 u32 minor_version)
 {
 	struct nfs_client *clp = ERR_PTR(-EIO);
 	struct nfs4_pnfs_ds_addr *da;
 	int status = 0;
 
-	dprintk("--> %s DS %s au_flavor %d\n", __func__, ds->ds_remotestr,
-		au_flavor);
+	dprintk("--> %s DS %s\n", __func__, ds->ds_remotestr);
 
 	list_for_each_entry(da, &ds->ds_addrs, da_node) {
 		dprintk("%s: DS %s: trying address %s\n",
@@ -720,8 +715,7 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
 			clp = nfs4_set_ds_client(mds_srv,
 						(struct sockaddr *)&da->da_addr,
 						da->da_addrlen, IPPROTO_TCP,
-						timeo, retrans, minor_version,
-						au_flavor);
+						timeo, retrans, minor_version);
 			if (IS_ERR(clp))
 				continue;
 
@@ -755,19 +749,17 @@ out:
  */
 void nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds,
 			  struct nfs4_deviceid_node *devid, unsigned int timeo,
-			  unsigned int retrans, u32 version,
-			  u32 minor_version, rpc_authflavor_t au_flavor)
+			  unsigned int retrans, u32 version, u32 minor_version)
 {
 	if (test_and_set_bit(NFS4DS_CONNECTING, &ds->ds_state) == 0) {
 		int err = 0;
 
 		if (version == 3) {
 			err = _nfs4_pnfs_v3_ds_connect(mds_srv, ds, timeo,
-						       retrans, au_flavor);
+						       retrans);
 		} else if (version == 4) {
 			err = _nfs4_pnfs_v4_ds_connect(mds_srv, ds, timeo,
-						       retrans, minor_version,
-						       au_flavor);
+						       retrans, minor_version);
 		} else {
 			dprintk("%s: unsupported DS version %d\n", __func__,
 				version);
diff --git a/fs/nfs/super.c b/fs/nfs/super.c
index 001796bcd6c8..ddce94ce8142 100644
--- a/fs/nfs/super.c
+++ b/fs/nfs/super.c
@@ -2904,7 +2904,7 @@ module_param(max_session_slots, ushort, 0644);
 MODULE_PARM_DESC(max_session_slots, "Maximum number of outstanding NFSv4.1 "
 		"requests the client will negotiate");
 module_param(max_session_cb_slots, ushort, 0644);
-MODULE_PARM_DESC(max_session_slots, "Maximum number of parallel NFSv4.1 "
+MODULE_PARM_DESC(max_session_cb_slots, "Maximum number of parallel NFSv4.1 "
 		"callbacks the client will process for a given server");
 module_param(send_implementation_id, ushort, 0644);
 MODULE_PARM_DESC(send_implementation_id,
diff --git a/fs/nfs/write.c b/fs/nfs/write.c
index 53211838f72a..6e761f3f4cbf 100644
--- a/fs/nfs/write.c
+++ b/fs/nfs/write.c
@@ -1151,8 +1151,7 @@ int nfs_flush_incompatible(struct file *file, struct page *page)
 		if (l_ctx && flctx &&
 		    !(list_empty_careful(&flctx->flc_posix) &&
 		      list_empty_careful(&flctx->flc_flock))) {
-			do_flush |= l_ctx->lockowner.l_owner != current->files
-				|| l_ctx->lockowner.l_pid != current->tgid;
+			do_flush |= l_ctx->lockowner != current->files;
 		}
 		nfs_release_request(req);
 		if (!do_flush)
diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c
index c95d369e90aa..12eeae62a2b1 100644
--- a/fs/nilfs2/super.c
+++ b/fs/nilfs2/super.c
@@ -189,7 +189,7 @@ static int nilfs_sync_super(struct super_block *sb, int flag)
 	set_buffer_dirty(nilfs->ns_sbh[0]);
 	if (nilfs_test_opt(nilfs, BARRIER)) {
 		err = __sync_dirty_buffer(nilfs->ns_sbh[0],
-					  WRITE_SYNC | WRITE_FLUSH_FUA);
+					  REQ_SYNC | REQ_PREFLUSH | REQ_FUA);
 	} else {
 		err = sync_dirty_buffer(nilfs->ns_sbh[0]);
 	}
diff --git a/fs/ntfs/aops.c b/fs/ntfs/aops.c
index fe251f187ff8..cc91856b5e2d 100644
--- a/fs/ntfs/aops.c
+++ b/fs/ntfs/aops.c
@@ -29,6 +29,7 @@
 #include <linux/buffer_head.h>
 #include <linux/writeback.h>
 #include <linux/bit_spinlock.h>
+#include <linux/bio.h>
 
 #include "aops.h"
 #include "attrib.h"
@@ -764,7 +765,7 @@ lock_retry_remap:
 			}
 			// TODO: Instantiate the hole.
 			// clear_buffer_new(bh);
-			// unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
+			// clean_bdev_bh_alias(bh);
 			ntfs_error(vol->sb, "Writing into sparse regions is "
 					"not supported yet. Sorry.");
 			err = -EOPNOTSUPP;
diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c
index bf72a2c58b75..99510d811a8c 100644
--- a/fs/ntfs/file.c
+++ b/fs/ntfs/file.c
@@ -740,8 +740,7 @@ map_buffer_cached:
 					set_buffer_uptodate(bh);
 				if (unlikely(was_hole)) {
 					/* We allocated the buffer. */
-					unmap_underlying_metadata(bh->b_bdev,
-							bh->b_blocknr);
+					clean_bdev_bh_alias(bh);
 					if (bh_end <= pos || bh_pos >= end)
 						mark_buffer_dirty(bh);
 					else
@@ -784,7 +783,7 @@ map_buffer_cached:
 				continue;
 			}
 			/* We allocated the buffer. */
-			unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
+			clean_bdev_bh_alias(bh);
 			/*
 			 * If the buffer is fully outside the write, zero it,
 			 * set it uptodate, and mark it dirty so it gets
diff --git a/fs/ntfs/logfile.c b/fs/ntfs/logfile.c
index 761f12f7f3ef..353379ff6057 100644
--- a/fs/ntfs/logfile.c
+++ b/fs/ntfs/logfile.c
@@ -27,6 +27,7 @@
 #include <linux/buffer_head.h>
 #include <linux/bitops.h>
 #include <linux/log2.h>
+#include <linux/bio.h>
 
 #include "attrib.h"
 #include "aops.h"
diff --git a/fs/ntfs/mft.c b/fs/ntfs/mft.c
index d3c009626032..b6f402194f02 100644
--- a/fs/ntfs/mft.c
+++ b/fs/ntfs/mft.c
@@ -23,6 +23,7 @@
 #include <linux/buffer_head.h>
 #include <linux/slab.h>
 #include <linux/swap.h>
+#include <linux/bio.h>
 
 #include "attrib.h"
 #include "aops.h"
diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c
index 9a88984f9f6f..4d9c6f5ec28a 100644
--- a/fs/ocfs2/aops.c
+++ b/fs/ocfs2/aops.c
@@ -630,7 +630,7 @@ int ocfs2_map_page_blocks(struct page *page, u64 *p_blkno,
 
 		if (!buffer_mapped(bh)) {
 			map_bh(bh, inode->i_sb, *p_blkno);
-			unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
+			clean_bdev_bh_alias(bh);
 		}
 
 		if (PageUptodate(page)) {
diff --git a/fs/ocfs2/buffer_head_io.c b/fs/ocfs2/buffer_head_io.c
index 8f040f88ade4..d9ebe11c8990 100644
--- a/fs/ocfs2/buffer_head_io.c
+++ b/fs/ocfs2/buffer_head_io.c
@@ -26,6 +26,7 @@
 #include <linux/fs.h>
 #include <linux/types.h>
 #include <linux/highmem.h>
+#include <linux/bio.h>
 
 #include <cluster/masklog.h>
 
diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c
index 9158c9825094..96a155ab5059 100644
--- a/fs/ocfs2/cluster/heartbeat.c
+++ b/fs/ocfs2/cluster/heartbeat.c
@@ -627,7 +627,7 @@ static int o2hb_issue_node_write(struct o2hb_region *reg,
 	slot = o2nm_this_node();
 
 	bio = o2hb_setup_one_bio(reg, write_wc, &slot, slot+1, REQ_OP_WRITE,
-				 WRITE_SYNC);
+				 REQ_SYNC);
 	if (IS_ERR(bio)) {
 		status = PTR_ERR(bio);
 		mlog_errno(status);
diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c
index ef3b4eb54cf2..551bc74ed2b8 100644
--- a/fs/orangefs/inode.c
+++ b/fs/orangefs/inode.c
@@ -8,6 +8,7 @@
  *  Linux VFS inode operations.
  */
 
+#include <linux/bvec.h>
 #include "protocol.h"
 #include "orangefs-kernel.h"
 #include "orangefs-bufmap.h"
diff --git a/fs/orangefs/orangefs-debugfs.c b/fs/orangefs/orangefs-debugfs.c
index 38887cc5577f..b5dbc9c6530c 100644
--- a/fs/orangefs/orangefs-debugfs.c
+++ b/fs/orangefs/orangefs-debugfs.c
@@ -671,8 +671,10 @@ int orangefs_prepare_debugfs_help_string(int at_boot)
 		 */
 		cdm_element_count =
 			orangefs_prepare_cdm_array(client_debug_array_string);
-		if (cdm_element_count <= 0)
+		if (cdm_element_count <= 0) {
+			kfree(new);
 			goto out;
+		}
 
 		for (i = 0; i < cdm_element_count; i++) {
 			strlcat(new, "\t", string_size);
diff --git a/fs/orangefs/orangefs-sysfs.c b/fs/orangefs/orangefs-sysfs.c
index a799546a67f7..084954448f18 100644
--- a/fs/orangefs/orangefs-sysfs.c
+++ b/fs/orangefs/orangefs-sysfs.c
@@ -609,15 +609,6 @@ static ssize_t sysfs_service_op_store(struct kobject *kobj,
 			new_op->upcall.req.param.u.value32[0] = val1;
 			new_op->upcall.req.param.u.value32[1] = val2;
 			goto value_set;
-		} else if (!strcmp(attr->attr.name,
-				   "perf_counter_reset")) {
-			if ((val > 0)) {
-				new_op->upcall.req.param.op =
-				ORANGEFS_PARAM_REQUEST_OP_READAHEAD_COUNT_SIZE;
-			} else {
-				rc = 0;
-				goto out;
-			}
 		}
 
 	} else if (!strcmp(kobj->name, ACACHE_KOBJ_ID)) {
diff --git a/fs/proc/base.c b/fs/proc/base.c
index 9b99df4893a4..5ea836362870 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -1246,7 +1246,7 @@ static const struct file_operations proc_oom_score_adj_operations = {
 };
 
 #ifdef CONFIG_AUDITSYSCALL
-#define TMPBUFLEN 21
+#define TMPBUFLEN 11
 static ssize_t proc_loginuid_read(struct file * file, char __user * buf,
 				  size_t count, loff_t *ppos)
 {
@@ -1667,7 +1667,8 @@ const struct inode_operations proc_pid_link_inode_operations = {
 
 /* building an inode */
 
-struct inode *proc_pid_make_inode(struct super_block * sb, struct task_struct *task)
+struct inode *proc_pid_make_inode(struct super_block * sb,
+				  struct task_struct *task, umode_t mode)
 {
 	struct inode * inode;
 	struct proc_inode *ei;
@@ -1681,6 +1682,7 @@ struct inode *proc_pid_make_inode(struct super_block * sb, struct task_struct *t
 
 	/* Common stuff */
 	ei = PROC_I(inode);
+	inode->i_mode = mode;
 	inode->i_ino = get_next_ino();
 	inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode);
 	inode->i_op = &proc_def_inode_operations;
@@ -2007,7 +2009,9 @@ proc_map_files_instantiate(struct inode *dir, struct dentry *dentry,
 	struct proc_inode *ei;
 	struct inode *inode;
 
-	inode = proc_pid_make_inode(dir->i_sb, task);
+	inode = proc_pid_make_inode(dir->i_sb, task, S_IFLNK |
+				    ((mode & FMODE_READ ) ? S_IRUSR : 0) |
+				    ((mode & FMODE_WRITE) ? S_IWUSR : 0));
 	if (!inode)
 		return -ENOENT;
 
@@ -2016,12 +2020,6 @@ proc_map_files_instantiate(struct inode *dir, struct dentry *dentry,
 
 	inode->i_op = &proc_map_files_link_inode_operations;
 	inode->i_size = 64;
-	inode->i_mode = S_IFLNK;
-
-	if (mode & FMODE_READ)
-		inode->i_mode |= S_IRUSR;
-	if (mode & FMODE_WRITE)
-		inode->i_mode |= S_IWUSR;
 
 	d_set_d_op(dentry, &tid_map_files_dentry_operations);
 	d_add(dentry, inode);
@@ -2375,12 +2373,11 @@ static int proc_pident_instantiate(struct inode *dir,
 	struct inode *inode;
 	struct proc_inode *ei;
 
-	inode = proc_pid_make_inode(dir->i_sb, task);
+	inode = proc_pid_make_inode(dir->i_sb, task, p->mode);
 	if (!inode)
 		goto out;
 
 	ei = PROC_I(inode);
-	inode->i_mode = p->mode;
 	if (S_ISDIR(inode->i_mode))
 		set_nlink(inode, 2);	/* Use getattr to fix if necessary */
 	if (p->iop)
@@ -3062,11 +3059,10 @@ static int proc_pid_instantiate(struct inode *dir,
 {
 	struct inode *inode;
 
-	inode = proc_pid_make_inode(dir->i_sb, task);
+	inode = proc_pid_make_inode(dir->i_sb, task, S_IFDIR | S_IRUGO | S_IXUGO);
 	if (!inode)
 		goto out;
 
-	inode->i_mode = S_IFDIR|S_IRUGO|S_IXUGO;
 	inode->i_op = &proc_tgid_base_inode_operations;
 	inode->i_fop = &proc_tgid_base_operations;
 	inode->i_flags|=S_IMMUTABLE;
@@ -3354,11 +3350,10 @@ static int proc_task_instantiate(struct inode *dir,
 	struct dentry *dentry, struct task_struct *task, const void *ptr)
 {
 	struct inode *inode;
-	inode = proc_pid_make_inode(dir->i_sb, task);
+	inode = proc_pid_make_inode(dir->i_sb, task, S_IFDIR | S_IRUGO | S_IXUGO);
 
 	if (!inode)
 		goto out;
-	inode->i_mode = S_IFDIR|S_IRUGO|S_IXUGO;
 	inode->i_op = &proc_tid_base_inode_operations;
 	inode->i_fop = &proc_tid_base_operations;
 	inode->i_flags|=S_IMMUTABLE;
diff --git a/fs/proc/fd.c b/fs/proc/fd.c
index d21dafef3102..4274f83bf100 100644
--- a/fs/proc/fd.c
+++ b/fs/proc/fd.c
@@ -183,14 +183,13 @@ proc_fd_instantiate(struct inode *dir, struct dentry *dentry,
 	struct proc_inode *ei;
 	struct inode *inode;
 
-	inode = proc_pid_make_inode(dir->i_sb, task);
+	inode = proc_pid_make_inode(dir->i_sb, task, S_IFLNK);
 	if (!inode)
 		goto out;
 
 	ei = PROC_I(inode);
 	ei->fd = fd;
 
-	inode->i_mode = S_IFLNK;
 	inode->i_op = &proc_pid_link_inode_operations;
 	inode->i_size = 64;
 
@@ -322,14 +321,13 @@ proc_fdinfo_instantiate(struct inode *dir, struct dentry *dentry,
 	struct proc_inode *ei;
 	struct inode *inode;
 
-	inode = proc_pid_make_inode(dir->i_sb, task);
+	inode = proc_pid_make_inode(dir->i_sb, task, S_IFREG | S_IRUSR);
 	if (!inode)
 		goto out;
 
 	ei = PROC_I(inode);
 	ei->fd = fd;
 
-	inode->i_mode = S_IFREG | S_IRUSR;
 	inode->i_fop = &proc_fdinfo_file_operations;
 
 	d_set_d_op(dentry, &tid_fd_dentry_operations);
diff --git a/fs/proc/generic.c b/fs/proc/generic.c
index 5f2dc2032c79..7eb3cefcf2a3 100644
--- a/fs/proc/generic.c
+++ b/fs/proc/generic.c
@@ -479,6 +479,7 @@ struct proc_dir_entry *proc_create_mount_point(const char *name)
 	}
 	return ent;
 }
+EXPORT_SYMBOL(proc_create_mount_point);
 
 struct proc_dir_entry *proc_create_data(const char *name, umode_t mode,
 					struct proc_dir_entry *parent,
diff --git a/fs/proc/internal.h b/fs/proc/internal.h
index bbba5d22aada..2de5194ba378 100644
--- a/fs/proc/internal.h
+++ b/fs/proc/internal.h
@@ -162,7 +162,7 @@ extern int proc_pid_statm(struct seq_file *, struct pid_namespace *,
 extern const struct dentry_operations pid_dentry_operations;
 extern int pid_getattr(struct vfsmount *, struct dentry *, struct kstat *);
 extern int proc_setattr(struct dentry *, struct iattr *);
-extern struct inode *proc_pid_make_inode(struct super_block *, struct task_struct *);
+extern struct inode *proc_pid_make_inode(struct super_block *, struct task_struct *, umode_t);
 extern int pid_revalidate(struct dentry *, unsigned int);
 extern int pid_delete_dentry(const struct dentry *);
 extern int proc_pid_readdir(struct file *, struct dir_context *);
@@ -195,7 +195,6 @@ static inline bool is_empty_pde(const struct proc_dir_entry *pde)
 {
 	return S_ISDIR(pde->mode) && !pde->proc_iops;
 }
-struct proc_dir_entry *proc_create_mount_point(const char *name);
 
 /*
  * inode.c
diff --git a/fs/proc/namespaces.c b/fs/proc/namespaces.c
index 51b8b0a8ad91..766f0c637ad1 100644
--- a/fs/proc/namespaces.c
+++ b/fs/proc/namespaces.c
@@ -92,12 +92,11 @@ static int proc_ns_instantiate(struct inode *dir,
 	struct inode *inode;
 	struct proc_inode *ei;
 
-	inode = proc_pid_make_inode(dir->i_sb, task);
+	inode = proc_pid_make_inode(dir->i_sb, task, S_IFLNK | S_IRWXUGO);
 	if (!inode)
 		goto out;
 
 	ei = PROC_I(inode);
-	inode->i_mode = S_IFLNK|S_IRWXUGO;
 	inode->i_op = &proc_ns_link_inode_operations;
 	ei->ns_ops = ns_ops;
 
diff --git a/fs/pstore/ftrace.c b/fs/pstore/ftrace.c
index d4887705bb61..899d0ba0bd6c 100644
--- a/fs/pstore/ftrace.c
+++ b/fs/pstore/ftrace.c
@@ -27,6 +27,9 @@
 #include <asm/barrier.h>
 #include "internal.h"
 
+/* This doesn't need to be atomic: speed is chosen over correctness here. */
+static u64 pstore_ftrace_stamp;
+
 static void notrace pstore_ftrace_call(unsigned long ip,
 				       unsigned long parent_ip,
 				       struct ftrace_ops *op,
@@ -42,6 +45,7 @@ static void notrace pstore_ftrace_call(unsigned long ip,
 
 	rec.ip = ip;
 	rec.parent_ip = parent_ip;
+	pstore_ftrace_write_timestamp(&rec, pstore_ftrace_stamp++);
 	pstore_ftrace_encode_cpu(&rec, raw_smp_processor_id());
 	psinfo->write_buf(PSTORE_TYPE_FTRACE, 0, NULL, 0, (void *)&rec,
 			  0, sizeof(rec), psinfo);
@@ -71,10 +75,13 @@ static ssize_t pstore_ftrace_knob_write(struct file *f, const char __user *buf,
 	if (!on ^ pstore_ftrace_enabled)
 		goto out;
 
-	if (on)
+	if (on) {
+		ftrace_ops_set_global_filter(&pstore_ftrace_ops);
 		ret = register_ftrace_function(&pstore_ftrace_ops);
-	else
+	} else {
 		ret = unregister_ftrace_function(&pstore_ftrace_ops);
+	}
+
 	if (ret) {
 		pr_err("%s: unable to %sregister ftrace ops: %zd\n",
 		       __func__, on ? "" : "un", ret);
diff --git a/fs/pstore/inode.c b/fs/pstore/inode.c
index 1781dc50762e..57c0646479f5 100644
--- a/fs/pstore/inode.c
+++ b/fs/pstore/inode.c
@@ -107,9 +107,11 @@ static int pstore_ftrace_seq_show(struct seq_file *s, void *v)
 	struct pstore_ftrace_seq_data *data = v;
 	struct pstore_ftrace_record *rec = (void *)(ps->data + data->off);
 
-	seq_printf(s, "%d %08lx  %08lx  %pf <- %pF\n",
-		pstore_ftrace_decode_cpu(rec), rec->ip, rec->parent_ip,
-		(void *)rec->ip, (void *)rec->parent_ip);
+	seq_printf(s, "CPU:%d ts:%llu %08lx  %08lx  %pf <- %pF\n",
+		   pstore_ftrace_decode_cpu(rec),
+		   pstore_ftrace_read_timestamp(rec),
+		   rec->ip, rec->parent_ip, (void *)rec->ip,
+		   (void *)rec->parent_ip);
 
 	return 0;
 }
@@ -197,11 +199,14 @@ static int pstore_unlink(struct inode *dir, struct dentry *dentry)
 	if (err)
 		return err;
 
-	if (p->psi->erase)
+	if (p->psi->erase) {
+		mutex_lock(&p->psi->read_mutex);
 		p->psi->erase(p->type, p->id, p->count,
 			      d_inode(dentry)->i_ctime, p->psi);
-	else
+		mutex_unlock(&p->psi->read_mutex);
+	} else {
 		return -EPERM;
+	}
 
 	return simple_unlink(dir, dentry);
 }
diff --git a/fs/pstore/internal.h b/fs/pstore/internal.h
index e38a22b31282..da416e6591c9 100644
--- a/fs/pstore/internal.h
+++ b/fs/pstore/internal.h
@@ -5,40 +5,6 @@
 #include <linux/time.h>
 #include <linux/pstore.h>
 
-#if NR_CPUS <= 2 && defined(CONFIG_ARM_THUMB)
-#define PSTORE_CPU_IN_IP 0x1
-#elif NR_CPUS <= 4 && defined(CONFIG_ARM)
-#define PSTORE_CPU_IN_IP 0x3
-#endif
-
-struct pstore_ftrace_record {
-	unsigned long ip;
-	unsigned long parent_ip;
-#ifndef PSTORE_CPU_IN_IP
-	unsigned int cpu;
-#endif
-};
-
-static inline void
-pstore_ftrace_encode_cpu(struct pstore_ftrace_record *rec, unsigned int cpu)
-{
-#ifndef PSTORE_CPU_IN_IP
-	rec->cpu = cpu;
-#else
-	rec->ip |= cpu;
-#endif
-}
-
-static inline unsigned int
-pstore_ftrace_decode_cpu(struct pstore_ftrace_record *rec)
-{
-#ifndef PSTORE_CPU_IN_IP
-	return rec->cpu;
-#else
-	return rec->ip & PSTORE_CPU_IN_IP;
-#endif
-}
-
 #ifdef CONFIG_PSTORE_FTRACE
 extern void pstore_register_ftrace(void);
 extern void pstore_unregister_ftrace(void);
diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
index 14984d902a99..729677e18e36 100644
--- a/fs/pstore/platform.c
+++ b/fs/pstore/platform.c
@@ -493,6 +493,7 @@ static void pstore_dump(struct kmsg_dumper *dumper,
 		if (!is_locked) {
 			pr_err("pstore dump routine blocked in %s path, may corrupt error record\n"
 				       , in_nmi() ? "NMI" : why);
+			return;
 		}
 	} else {
 		spin_lock_irqsave(&psinfo->buf_lock, flags);
@@ -584,8 +585,8 @@ static void pstore_console_write(struct console *con, const char *s, unsigned c)
 		} else {
 			spin_lock_irqsave(&psinfo->buf_lock, flags);
 		}
-		memcpy(psinfo->buf, s, c);
-		psinfo->write(PSTORE_TYPE_CONSOLE, 0, &id, 0, 0, 0, c, psinfo);
+		psinfo->write_buf(PSTORE_TYPE_CONSOLE, 0, &id, 0,
+				  s, 0, c, psinfo);
 		spin_unlock_irqrestore(&psinfo->buf_lock, flags);
 		s += c;
 		c = e - s;
diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c
index 6ad831b9d1b8..27c059e1760a 100644
--- a/fs/pstore/ram.c
+++ b/fs/pstore/ram.c
@@ -85,10 +85,10 @@ MODULE_PARM_DESC(ramoops_ecc,
 		"bytes ECC)");
 
 struct ramoops_context {
-	struct persistent_ram_zone **przs;
-	struct persistent_ram_zone *cprz;
-	struct persistent_ram_zone *fprz;
-	struct persistent_ram_zone *mprz;
+	struct persistent_ram_zone **dprzs;	/* Oops dump zones */
+	struct persistent_ram_zone *cprz;	/* Console zone */
+	struct persistent_ram_zone **fprzs;	/* Ftrace zones */
+	struct persistent_ram_zone *mprz;	/* PMSG zone */
 	phys_addr_t phys_addr;
 	unsigned long size;
 	unsigned int memtype;
@@ -97,12 +97,14 @@ struct ramoops_context {
 	size_t ftrace_size;
 	size_t pmsg_size;
 	int dump_oops;
+	u32 flags;
 	struct persistent_ram_ecc_info ecc_info;
 	unsigned int max_dump_cnt;
 	unsigned int dump_write_cnt;
 	/* _read_cnt need clear on ramoops_pstore_open */
 	unsigned int dump_read_cnt;
 	unsigned int console_read_cnt;
+	unsigned int max_ftrace_cnt;
 	unsigned int ftrace_read_cnt;
 	unsigned int pmsg_read_cnt;
 	struct pstore_info pstore;
@@ -180,16 +182,69 @@ static bool prz_ok(struct persistent_ram_zone *prz)
 			   persistent_ram_ecc_string(prz, NULL, 0));
 }
 
+static ssize_t ftrace_log_combine(struct persistent_ram_zone *dest,
+				  struct persistent_ram_zone *src)
+{
+	size_t dest_size, src_size, total, dest_off, src_off;
+	size_t dest_idx = 0, src_idx = 0, merged_idx = 0;
+	void *merged_buf;
+	struct pstore_ftrace_record *drec, *srec, *mrec;
+	size_t record_size = sizeof(struct pstore_ftrace_record);
+
+	dest_off = dest->old_log_size % record_size;
+	dest_size = dest->old_log_size - dest_off;
+
+	src_off = src->old_log_size % record_size;
+	src_size = src->old_log_size - src_off;
+
+	total = dest_size + src_size;
+	merged_buf = kmalloc(total, GFP_KERNEL);
+	if (!merged_buf)
+		return -ENOMEM;
+
+	drec = (struct pstore_ftrace_record *)(dest->old_log + dest_off);
+	srec = (struct pstore_ftrace_record *)(src->old_log + src_off);
+	mrec = (struct pstore_ftrace_record *)(merged_buf);
+
+	while (dest_size > 0 && src_size > 0) {
+		if (pstore_ftrace_read_timestamp(&drec[dest_idx]) <
+		    pstore_ftrace_read_timestamp(&srec[src_idx])) {
+			mrec[merged_idx++] = drec[dest_idx++];
+			dest_size -= record_size;
+		} else {
+			mrec[merged_idx++] = srec[src_idx++];
+			src_size -= record_size;
+		}
+	}
+
+	while (dest_size > 0) {
+		mrec[merged_idx++] = drec[dest_idx++];
+		dest_size -= record_size;
+	}
+
+	while (src_size > 0) {
+		mrec[merged_idx++] = srec[src_idx++];
+		src_size -= record_size;
+	}
+
+	kfree(dest->old_log);
+	dest->old_log = merged_buf;
+	dest->old_log_size = total;
+
+	return 0;
+}
+
 static ssize_t ramoops_pstore_read(u64 *id, enum pstore_type_id *type,
 				   int *count, struct timespec *time,
 				   char **buf, bool *compressed,
 				   ssize_t *ecc_notice_size,
 				   struct pstore_info *psi)
 {
-	ssize_t size;
+	ssize_t size = 0;
 	struct ramoops_context *cxt = psi->data;
 	struct persistent_ram_zone *prz = NULL;
 	int header_length = 0;
+	bool free_prz = false;
 
 	/* Ramoops headers provide time stamps for PSTORE_TYPE_DMESG, but
 	 * PSTORE_TYPE_CONSOLE and PSTORE_TYPE_FTRACE don't currently have
@@ -201,7 +256,7 @@ static ssize_t ramoops_pstore_read(u64 *id, enum pstore_type_id *type,
 
 	/* Find the next valid persistent_ram_zone for DMESG */
 	while (cxt->dump_read_cnt < cxt->max_dump_cnt && !prz) {
-		prz = ramoops_get_next_prz(cxt->przs, &cxt->dump_read_cnt,
+		prz = ramoops_get_next_prz(cxt->dprzs, &cxt->dump_read_cnt,
 					   cxt->max_dump_cnt, id, type,
 					   PSTORE_TYPE_DMESG, 1);
 		if (!prz_ok(prz))
@@ -219,14 +274,56 @@ static ssize_t ramoops_pstore_read(u64 *id, enum pstore_type_id *type,
 	if (!prz_ok(prz))
 		prz = ramoops_get_next_prz(&cxt->cprz, &cxt->console_read_cnt,
 					   1, id, type, PSTORE_TYPE_CONSOLE, 0);
-	if (!prz_ok(prz))
-		prz = ramoops_get_next_prz(&cxt->fprz, &cxt->ftrace_read_cnt,
-					   1, id, type, PSTORE_TYPE_FTRACE, 0);
+
 	if (!prz_ok(prz))
 		prz = ramoops_get_next_prz(&cxt->mprz, &cxt->pmsg_read_cnt,
 					   1, id, type, PSTORE_TYPE_PMSG, 0);
-	if (!prz_ok(prz))
-		return 0;
+
+	/* ftrace is last since it may want to dynamically allocate memory. */
+	if (!prz_ok(prz)) {
+		if (!(cxt->flags & RAMOOPS_FLAG_FTRACE_PER_CPU)) {
+			prz = ramoops_get_next_prz(cxt->fprzs,
+					&cxt->ftrace_read_cnt, 1, id, type,
+					PSTORE_TYPE_FTRACE, 0);
+		} else {
+			/*
+			 * Build a new dummy record which combines all the
+			 * per-cpu records including metadata and ecc info.
+			 */
+			struct persistent_ram_zone *tmp_prz, *prz_next;
+
+			tmp_prz = kzalloc(sizeof(struct persistent_ram_zone),
+					  GFP_KERNEL);
+			if (!tmp_prz)
+				return -ENOMEM;
+			free_prz = true;
+
+			while (cxt->ftrace_read_cnt < cxt->max_ftrace_cnt) {
+				prz_next = ramoops_get_next_prz(cxt->fprzs,
+						&cxt->ftrace_read_cnt,
+						cxt->max_ftrace_cnt, id,
+						type, PSTORE_TYPE_FTRACE, 0);
+
+				if (!prz_ok(prz_next))
+					continue;
+
+				tmp_prz->ecc_info = prz_next->ecc_info;
+				tmp_prz->corrected_bytes +=
+						prz_next->corrected_bytes;
+				tmp_prz->bad_blocks += prz_next->bad_blocks;
+				size = ftrace_log_combine(tmp_prz, prz_next);
+				if (size)
+					goto out;
+			}
+			*id = 0;
+			prz = tmp_prz;
+		}
+	}
+
+	if (!prz_ok(prz)) {
+		size = 0;
+		goto out;
+	}
 
 	size = persistent_ram_old_size(prz) - header_length;
 
@@ -234,12 +331,21 @@ static ssize_t ramoops_pstore_read(u64 *id, enum pstore_type_id *type,
 	*ecc_notice_size = persistent_ram_ecc_string(prz, NULL, 0);
 
 	*buf = kmalloc(size + *ecc_notice_size + 1, GFP_KERNEL);
-	if (*buf == NULL)
-		return -ENOMEM;
+	if (*buf == NULL) {
+		size = -ENOMEM;
+		goto out;
+	}
 
 	memcpy(*buf, (char *)persistent_ram_old(prz) + header_length, size);
+
 	persistent_ram_ecc_string(prz, *buf + size, *ecc_notice_size + 1);
 
+out:
+	if (free_prz) {
+		kfree(prz->old_log);
+		kfree(prz);
+	}
+
 	return size;
 }
 
@@ -283,15 +389,23 @@ static int notrace ramoops_pstore_write_buf(enum pstore_type_id type,
 		persistent_ram_write(cxt->cprz, buf, size);
 		return 0;
 	} else if (type == PSTORE_TYPE_FTRACE) {
-		if (!cxt->fprz)
+		int zonenum;
+
+		if (!cxt->fprzs)
 			return -ENOMEM;
-		persistent_ram_write(cxt->fprz, buf, size);
+		/*
+		 * Choose zone by if we're using per-cpu buffers.
+		 */
+		if (cxt->flags & RAMOOPS_FLAG_FTRACE_PER_CPU)
+			zonenum = smp_processor_id();
+		else
+			zonenum = 0;
+
+		persistent_ram_write(cxt->fprzs[zonenum], buf, size);
 		return 0;
 	} else if (type == PSTORE_TYPE_PMSG) {
-		if (!cxt->mprz)
-			return -ENOMEM;
-		persistent_ram_write(cxt->mprz, buf, size);
-		return 0;
+		pr_warn_ratelimited("PMSG shouldn't call %s\n", __func__);
+		return -EINVAL;
 	}
 
 	if (type != PSTORE_TYPE_DMESG)
@@ -316,10 +430,10 @@ static int notrace ramoops_pstore_write_buf(enum pstore_type_id type,
 	if (part != 1)
 		return -ENOSPC;
 
-	if (!cxt->przs)
+	if (!cxt->dprzs)
 		return -ENOSPC;
 
-	prz = cxt->przs[cxt->dump_write_cnt];
+	prz = cxt->dprzs[cxt->dump_write_cnt];
 
 	hlen = ramoops_write_kmsg_hdr(prz, compressed);
 	if (size + hlen > prz->buffer_size)
@@ -359,13 +473,15 @@ static int ramoops_pstore_erase(enum pstore_type_id type, u64 id, int count,
 	case PSTORE_TYPE_DMESG:
 		if (id >= cxt->max_dump_cnt)
 			return -EINVAL;
-		prz = cxt->przs[id];
+		prz = cxt->dprzs[id];
 		break;
 	case PSTORE_TYPE_CONSOLE:
 		prz = cxt->cprz;
 		break;
 	case PSTORE_TYPE_FTRACE:
-		prz = cxt->fprz;
+		if (id >= cxt->max_ftrace_cnt)
+			return -EINVAL;
+		prz = cxt->fprzs[id];
 		break;
 	case PSTORE_TYPE_PMSG:
 		prz = cxt->mprz;
@@ -396,68 +512,113 @@ static void ramoops_free_przs(struct ramoops_context *cxt)
 {
 	int i;
 
-	if (!cxt->przs)
-		return;
+	/* Free dump PRZs */
+	if (cxt->dprzs) {
+		for (i = 0; i < cxt->max_dump_cnt; i++)
+			persistent_ram_free(cxt->dprzs[i]);
 
-	for (i = 0; i < cxt->max_dump_cnt; i++)
-		persistent_ram_free(cxt->przs[i]);
+		kfree(cxt->dprzs);
+		cxt->max_dump_cnt = 0;
+	}
 
-	kfree(cxt->przs);
-	cxt->max_dump_cnt = 0;
+	/* Free ftrace PRZs */
+	if (cxt->fprzs) {
+		for (i = 0; i < cxt->max_ftrace_cnt; i++)
+			persistent_ram_free(cxt->fprzs[i]);
+		kfree(cxt->fprzs);
+		cxt->max_ftrace_cnt = 0;
+	}
 }
 
-static int ramoops_init_przs(struct device *dev, struct ramoops_context *cxt,
-			     phys_addr_t *paddr, size_t dump_mem_sz)
+static int ramoops_init_przs(const char *name,
+			     struct device *dev, struct ramoops_context *cxt,
+			     struct persistent_ram_zone ***przs,
+			     phys_addr_t *paddr, size_t mem_sz,
+			     ssize_t record_size,
+			     unsigned int *cnt, u32 sig, u32 flags)
 {
 	int err = -ENOMEM;
 	int i;
+	size_t zone_sz;
+	struct persistent_ram_zone **prz_ar;
 
-	if (!cxt->record_size)
+	/* Allocate nothing for 0 mem_sz or 0 record_size. */
+	if (mem_sz == 0 || record_size == 0) {
+		*cnt = 0;
 		return 0;
+	}
 
-	if (*paddr + dump_mem_sz - cxt->phys_addr > cxt->size) {
-		dev_err(dev, "no room for dumps\n");
-		return -ENOMEM;
+	/*
+	 * If we have a negative record size, calculate it based on
+	 * mem_sz / *cnt. If we have a positive record size, calculate
+	 * cnt from mem_sz / record_size.
+	 */
+	if (record_size < 0) {
+		if (*cnt == 0)
+			return 0;
+		record_size = mem_sz / *cnt;
+		if (record_size == 0) {
+			dev_err(dev, "%s record size == 0 (%zu / %u)\n",
+				name, mem_sz, *cnt);
+			goto fail;
+		}
+	} else {
+		*cnt = mem_sz / record_size;
+		if (*cnt == 0) {
+			dev_err(dev, "%s record count == 0 (%zu / %zu)\n",
+				name, mem_sz, record_size);
+			goto fail;
+		}
 	}
 
-	cxt->max_dump_cnt = dump_mem_sz / cxt->record_size;
-	if (!cxt->max_dump_cnt)
-		return -ENOMEM;
+	if (*paddr + mem_sz - cxt->phys_addr > cxt->size) {
+		dev_err(dev, "no room for %s mem region (0x%zx@0x%llx) in (0x%lx@0x%llx)\n",
+			name,
+			mem_sz, (unsigned long long)*paddr,
+			cxt->size, (unsigned long long)cxt->phys_addr);
+		goto fail;
+	}
 
-	cxt->przs = kzalloc(sizeof(*cxt->przs) * cxt->max_dump_cnt,
-			     GFP_KERNEL);
-	if (!cxt->przs) {
-		dev_err(dev, "failed to initialize a prz array for dumps\n");
-		goto fail_mem;
+	zone_sz = mem_sz / *cnt;
+	if (!zone_sz) {
+		dev_err(dev, "%s zone size == 0\n", name);
+		goto fail;
 	}
 
-	for (i = 0; i < cxt->max_dump_cnt; i++) {
-		cxt->przs[i] = persistent_ram_new(*paddr, cxt->record_size, 0,
+	prz_ar = kcalloc(*cnt, sizeof(**przs), GFP_KERNEL);
+	if (!prz_ar)
+		goto fail;
+
+	for (i = 0; i < *cnt; i++) {
+		prz_ar[i] = persistent_ram_new(*paddr, zone_sz, sig,
 						  &cxt->ecc_info,
-						  cxt->memtype);
-		if (IS_ERR(cxt->przs[i])) {
-			err = PTR_ERR(cxt->przs[i]);
-			dev_err(dev, "failed to request mem region (0x%zx@0x%llx): %d\n",
-				cxt->record_size, (unsigned long long)*paddr, err);
+						  cxt->memtype, flags);
+		if (IS_ERR(prz_ar[i])) {
+			err = PTR_ERR(prz_ar[i]);
+			dev_err(dev, "failed to request %s mem region (0x%zx@0x%llx): %d\n",
+				name, record_size,
+				(unsigned long long)*paddr, err);
 
 			while (i > 0) {
 				i--;
-				persistent_ram_free(cxt->przs[i]);
+				persistent_ram_free(prz_ar[i]);
 			}
-			goto fail_prz;
+			kfree(prz_ar);
+			goto fail;
 		}
-		*paddr += cxt->record_size;
+		*paddr += zone_sz;
 	}
 
+	*przs = prz_ar;
 	return 0;
-fail_prz:
-	kfree(cxt->przs);
-fail_mem:
-	cxt->max_dump_cnt = 0;
+
+fail:
+	*cnt = 0;
 	return err;
 }
 
-static int ramoops_init_prz(struct device *dev, struct ramoops_context *cxt,
+static int ramoops_init_prz(const char *name,
+			    struct device *dev, struct ramoops_context *cxt,
 			    struct persistent_ram_zone **prz,
 			    phys_addr_t *paddr, size_t sz, u32 sig)
 {
@@ -465,18 +626,19 @@ static int ramoops_init_prz(struct device *dev, struct ramoops_context *cxt,
 		return 0;
 
 	if (*paddr + sz - cxt->phys_addr > cxt->size) {
-		dev_err(dev, "no room for mem region (0x%zx@0x%llx) in (0x%lx@0x%llx)\n",
-			sz, (unsigned long long)*paddr,
+		dev_err(dev, "no room for %s mem region (0x%zx@0x%llx) in (0x%lx@0x%llx)\n",
+			name, sz, (unsigned long long)*paddr,
 			cxt->size, (unsigned long long)cxt->phys_addr);
 		return -ENOMEM;
 	}
 
-	*prz = persistent_ram_new(*paddr, sz, sig, &cxt->ecc_info, cxt->memtype);
+	*prz = persistent_ram_new(*paddr, sz, sig, &cxt->ecc_info,
+				  cxt->memtype, 0);
 	if (IS_ERR(*prz)) {
 		int err = PTR_ERR(*prz);
 
-		dev_err(dev, "failed to request mem region (0x%zx@0x%llx): %d\n",
-			sz, (unsigned long long)*paddr, err);
+		dev_err(dev, "failed to request %s mem region (0x%zx@0x%llx): %d\n",
+			name, sz, (unsigned long long)*paddr, err);
 		return err;
 	}
 
@@ -543,6 +705,7 @@ static int ramoops_parse_dt(struct platform_device *pdev,
 	parse_size("ftrace-size", pdata->ftrace_size);
 	parse_size("pmsg-size", pdata->pmsg_size);
 	parse_size("ecc-size", pdata->ecc_info.ecc_size);
+	parse_size("flags", pdata->flags);
 
 #undef parse_size
 
@@ -561,6 +724,7 @@ static int ramoops_probe(struct platform_device *pdev)
 	if (dev_of_node(dev) && !pdata) {
 		pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
 		if (!pdata) {
+			pr_err("cannot allocate platform data buffer\n");
 			err = -ENOMEM;
 			goto fail_out;
 		}
@@ -570,11 +734,20 @@ static int ramoops_probe(struct platform_device *pdev)
 			goto fail_out;
 	}
 
-	/* Only a single ramoops area allowed at a time, so fail extra
+	/*
+	 * Only a single ramoops area allowed at a time, so fail extra
 	 * probes.
 	 */
-	if (cxt->max_dump_cnt)
+	if (cxt->max_dump_cnt) {
+		pr_err("already initialized\n");
 		goto fail_out;
+	}
+
+	/* Make sure we didn't get bogus platform data pointer. */
+	if (!pdata) {
+		pr_err("NULL platform data\n");
+		goto fail_out;
+	}
 
 	if (!pdata->mem_size || (!pdata->record_size && !pdata->console_size &&
 			!pdata->ftrace_size && !pdata->pmsg_size)) {
@@ -600,27 +773,37 @@ static int ramoops_probe(struct platform_device *pdev)
 	cxt->ftrace_size = pdata->ftrace_size;
 	cxt->pmsg_size = pdata->pmsg_size;
 	cxt->dump_oops = pdata->dump_oops;
+	cxt->flags = pdata->flags;
 	cxt->ecc_info = pdata->ecc_info;
 
 	paddr = cxt->phys_addr;
 
 	dump_mem_sz = cxt->size - cxt->console_size - cxt->ftrace_size
 			- cxt->pmsg_size;
-	err = ramoops_init_przs(dev, cxt, &paddr, dump_mem_sz);
+	err = ramoops_init_przs("dump", dev, cxt, &cxt->dprzs, &paddr,
+				dump_mem_sz, cxt->record_size,
+				&cxt->max_dump_cnt, 0, 0);
 	if (err)
 		goto fail_out;
 
-	err = ramoops_init_prz(dev, cxt, &cxt->cprz, &paddr,
+	err = ramoops_init_prz("console", dev, cxt, &cxt->cprz, &paddr,
 			       cxt->console_size, 0);
 	if (err)
 		goto fail_init_cprz;
 
-	err = ramoops_init_prz(dev, cxt, &cxt->fprz, &paddr, cxt->ftrace_size,
-			       LINUX_VERSION_CODE);
+	cxt->max_ftrace_cnt = (cxt->flags & RAMOOPS_FLAG_FTRACE_PER_CPU)
+				? nr_cpu_ids
+				: 1;
+	err = ramoops_init_przs("ftrace", dev, cxt, &cxt->fprzs, &paddr,
+				cxt->ftrace_size, -1,
+				&cxt->max_ftrace_cnt, LINUX_VERSION_CODE,
+				(cxt->flags & RAMOOPS_FLAG_FTRACE_PER_CPU)
+					? PRZ_FLAG_NO_LOCK : 0);
 	if (err)
 		goto fail_init_fprz;
 
-	err = ramoops_init_prz(dev, cxt, &cxt->mprz, &paddr, cxt->pmsg_size, 0);
+	err = ramoops_init_prz("pmsg", dev, cxt, &cxt->mprz, &paddr,
+				cxt->pmsg_size, 0);
 	if (err)
 		goto fail_init_mprz;
 
@@ -680,7 +863,6 @@ fail_clear:
 	cxt->pstore.bufsize = 0;
 	persistent_ram_free(cxt->mprz);
 fail_init_mprz:
-	persistent_ram_free(cxt->fprz);
 fail_init_fprz:
 	persistent_ram_free(cxt->cprz);
 fail_init_cprz:
@@ -699,7 +881,6 @@ static int ramoops_remove(struct platform_device *pdev)
 	cxt->pstore.bufsize = 0;
 
 	persistent_ram_free(cxt->mprz);
-	persistent_ram_free(cxt->fprz);
 	persistent_ram_free(cxt->cprz);
 	ramoops_free_przs(cxt);
 
@@ -741,6 +922,8 @@ static void ramoops_register_dummy(void)
 	dummy_data->ftrace_size = ramoops_ftrace_size;
 	dummy_data->pmsg_size = ramoops_pmsg_size;
 	dummy_data->dump_oops = dump_oops;
+	dummy_data->flags = RAMOOPS_FLAG_FTRACE_PER_CPU;
+
 	/*
 	 * For backwards compatibility ramoops.ecc=1 means 16 bytes ECC
 	 * (using 1 byte for ECC isn't much of use anyway).
diff --git a/fs/pstore/ram_core.c b/fs/pstore/ram_core.c
index 3975deec02f8..a857338b7dab 100644
--- a/fs/pstore/ram_core.c
+++ b/fs/pstore/ram_core.c
@@ -48,16 +48,15 @@ static inline size_t buffer_start(struct persistent_ram_zone *prz)
 	return atomic_read(&prz->buffer->start);
 }
 
-static DEFINE_RAW_SPINLOCK(buffer_lock);
-
 /* increase and wrap the start pointer, returning the old value */
 static size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a)
 {
 	int old;
 	int new;
-	unsigned long flags;
+	unsigned long flags = 0;
 
-	raw_spin_lock_irqsave(&buffer_lock, flags);
+	if (!(prz->flags & PRZ_FLAG_NO_LOCK))
+		raw_spin_lock_irqsave(&prz->buffer_lock, flags);
 
 	old = atomic_read(&prz->buffer->start);
 	new = old + a;
@@ -65,7 +64,8 @@ static size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a)
 		new -= prz->buffer_size;
 	atomic_set(&prz->buffer->start, new);
 
-	raw_spin_unlock_irqrestore(&buffer_lock, flags);
+	if (!(prz->flags & PRZ_FLAG_NO_LOCK))
+		raw_spin_unlock_irqrestore(&prz->buffer_lock, flags);
 
 	return old;
 }
@@ -75,9 +75,10 @@ static void buffer_size_add(struct persistent_ram_zone *prz, size_t a)
 {
 	size_t old;
 	size_t new;
-	unsigned long flags;
+	unsigned long flags = 0;
 
-	raw_spin_lock_irqsave(&buffer_lock, flags);
+	if (!(prz->flags & PRZ_FLAG_NO_LOCK))
+		raw_spin_lock_irqsave(&prz->buffer_lock, flags);
 
 	old = atomic_read(&prz->buffer->size);
 	if (old == prz->buffer_size)
@@ -89,7 +90,8 @@ static void buffer_size_add(struct persistent_ram_zone *prz, size_t a)
 	atomic_set(&prz->buffer->size, new);
 
 exit:
-	raw_spin_unlock_irqrestore(&buffer_lock, flags);
+	if (!(prz->flags & PRZ_FLAG_NO_LOCK))
+		raw_spin_unlock_irqrestore(&prz->buffer_lock, flags);
 }
 
 static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz,
@@ -465,7 +467,8 @@ static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size,
 }
 
 static int persistent_ram_post_init(struct persistent_ram_zone *prz, u32 sig,
-				    struct persistent_ram_ecc_info *ecc_info)
+				    struct persistent_ram_ecc_info *ecc_info,
+				    unsigned long flags)
 {
 	int ret;
 
@@ -493,6 +496,8 @@ static int persistent_ram_post_init(struct persistent_ram_zone *prz, u32 sig,
 
 	prz->buffer->sig = sig;
 	persistent_ram_zap(prz);
+	prz->buffer_lock = __RAW_SPIN_LOCK_UNLOCKED(buffer_lock);
+	prz->flags = flags;
 
 	return 0;
 }
@@ -517,7 +522,7 @@ void persistent_ram_free(struct persistent_ram_zone *prz)
 
 struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size,
 			u32 sig, struct persistent_ram_ecc_info *ecc_info,
-			unsigned int memtype)
+			unsigned int memtype, u32 flags)
 {
 	struct persistent_ram_zone *prz;
 	int ret = -ENOMEM;
@@ -532,7 +537,7 @@ struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size,
 	if (ret)
 		goto err;
 
-	ret = persistent_ram_post_init(prz, sig, ecc_info);
+	ret = persistent_ram_post_init(prz, sig, ecc_info, flags);
 	if (ret)
 		goto err;
 
diff --git a/fs/reiserfs/inode.c b/fs/reiserfs/inode.c
index 58b2dedb2a3a..cfeae9b0a2b7 100644
--- a/fs/reiserfs/inode.c
+++ b/fs/reiserfs/inode.c
@@ -19,6 +19,7 @@
 #include <linux/quotaops.h>
 #include <linux/swap.h>
 #include <linux/uio.h>
+#include <linux/bio.h>
 
 int reiserfs_commit_write(struct file *f, struct page *page,
 			  unsigned from, unsigned to);
diff --git a/fs/reiserfs/journal.c b/fs/reiserfs/journal.c
index bc2dde2423c2..aa40c242f1db 100644
--- a/fs/reiserfs/journal.c
+++ b/fs/reiserfs/journal.c
@@ -1111,7 +1111,8 @@ static int flush_commit_list(struct super_block *s,
 		mark_buffer_dirty(jl->j_commit_bh) ;
 		depth = reiserfs_write_unlock_nested(s);
 		if (reiserfs_barrier_flush(s))
-			__sync_dirty_buffer(jl->j_commit_bh, WRITE_FLUSH_FUA);
+			__sync_dirty_buffer(jl->j_commit_bh,
+					REQ_PREFLUSH | REQ_FUA);
 		else
 			sync_dirty_buffer(jl->j_commit_bh);
 		reiserfs_write_lock_nested(s, depth);
@@ -1269,7 +1270,8 @@ static int _update_journal_header_block(struct super_block *sb,
 		depth = reiserfs_write_unlock_nested(sb);
 
 		if (reiserfs_barrier_flush(sb))
-			__sync_dirty_buffer(journal->j_header_bh, WRITE_FLUSH_FUA);
+			__sync_dirty_buffer(journal->j_header_bh,
+					REQ_PREFLUSH | REQ_FUA);
 		else
 			sync_dirty_buffer(journal->j_header_bh);
 
diff --git a/fs/reiserfs/stree.c b/fs/reiserfs/stree.c
index a97e352d05d3..0037aea97d39 100644
--- a/fs/reiserfs/stree.c
+++ b/fs/reiserfs/stree.c
@@ -11,6 +11,7 @@
 #include <linux/time.h>
 #include <linux/string.h>
 #include <linux/pagemap.h>
+#include <linux/bio.h>
 #include "reiserfs.h"
 #include <linux/buffer_head.h>
 #include <linux/quotaops.h>
diff --git a/fs/splice.c b/fs/splice.c
index 5a7750bd2eea..8ed7c9d8c0fb 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -17,6 +17,7 @@
  * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
  *
  */
+#include <linux/bvec.h>
 #include <linux/fs.h>
 #include <linux/file.h>
 #include <linux/pagemap.h>
diff --git a/fs/squashfs/block.c b/fs/squashfs/block.c
index ce62a380314f..2751476e6b6e 100644
--- a/fs/squashfs/block.c
+++ b/fs/squashfs/block.c
@@ -31,6 +31,7 @@
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/buffer_head.h>
+#include <linux/bio.h>
 
 #include "squashfs_fs.h"
 #include "squashfs_fs_sb.h"
diff --git a/fs/ubifs/Kconfig b/fs/ubifs/Kconfig
index 7ff7712f284e..0a908ae7af13 100644
--- a/fs/ubifs/Kconfig
+++ b/fs/ubifs/Kconfig
@@ -50,3 +50,14 @@ config UBIFS_ATIME_SUPPORT
 	  strictatime is the "heavy", relatime is "lighter", etc.
 
 	  If unsure, say 'N'
+
+config UBIFS_FS_ENCRYPTION
+	bool "UBIFS Encryption"
+	depends on UBIFS_FS
+	select FS_ENCRYPTION
+	default n
+	help
+	  Enable encryption of UBIFS files and directories. This
+	  feature is similar to ecryptfs, but it is more memory
+	  efficient since it avoids caching the encrypted and
+	  decrypted pages in the page cache.
diff --git a/fs/ubifs/Makefile b/fs/ubifs/Makefile
index c54a24360f85..6f3251c2bf08 100644
--- a/fs/ubifs/Makefile
+++ b/fs/ubifs/Makefile
@@ -5,3 +5,4 @@ ubifs-y += tnc.o master.o scan.o replay.o log.o commit.o gc.o orphan.o
 ubifs-y += budget.o find.o tnc_commit.o compress.o lpt.o lprops.o
 ubifs-y += recovery.o ioctl.o lpt_commit.o tnc_misc.o xattr.o debug.o
 ubifs-y += misc.o
+ubifs-$(CONFIG_UBIFS_FS_ENCRYPTION) += crypto.o
diff --git a/fs/ubifs/crypto.c b/fs/ubifs/crypto.c
new file mode 100644
index 000000000000..3402720f2b28
--- /dev/null
+++ b/fs/ubifs/crypto.c
@@ -0,0 +1,97 @@
+#include "ubifs.h"
+
+static int ubifs_crypt_get_context(struct inode *inode, void *ctx, size_t len)
+{
+	return ubifs_xattr_get(inode, UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT,
+			       ctx, len);
+}
+
+static int ubifs_crypt_set_context(struct inode *inode, const void *ctx,
+				   size_t len, void *fs_data)
+{
+	return ubifs_xattr_set(inode, UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT,
+			       ctx, len, 0);
+}
+
+static bool ubifs_crypt_empty_dir(struct inode *inode)
+{
+	return ubifs_check_dir_empty(inode) == 0;
+}
+
+static unsigned int ubifs_crypt_max_namelen(struct inode *inode)
+{
+	if (S_ISLNK(inode->i_mode))
+		return UBIFS_MAX_INO_DATA;
+	else
+		return UBIFS_MAX_NLEN;
+}
+
+static int ubifs_key_prefix(struct inode *inode, u8 **key)
+{
+	static char prefix[] = "ubifs:";
+
+	*key = prefix;
+
+	return sizeof(prefix) - 1;
+}
+
+int ubifs_encrypt(const struct inode *inode, struct ubifs_data_node *dn,
+		  unsigned int in_len, unsigned int *out_len, int block)
+{
+	struct ubifs_info *c = inode->i_sb->s_fs_info;
+	void *p = &dn->data;
+	struct page *ret;
+	unsigned int pad_len = round_up(in_len, UBIFS_CIPHER_BLOCK_SIZE);
+
+	ubifs_assert(pad_len <= *out_len);
+	dn->compr_size = cpu_to_le16(in_len);
+
+	/* pad to full block cipher length */
+	if (pad_len != in_len)
+		memset(p + in_len, 0, pad_len - in_len);
+
+	ret = fscrypt_encrypt_page(inode, virt_to_page(&dn->data), pad_len,
+			offset_in_page(&dn->data), block, GFP_NOFS);
+	if (IS_ERR(ret)) {
+		ubifs_err(c, "fscrypt_encrypt_page failed: %ld", PTR_ERR(ret));
+		return PTR_ERR(ret);
+	}
+	*out_len = pad_len;
+
+	return 0;
+}
+
+int ubifs_decrypt(const struct inode *inode, struct ubifs_data_node *dn,
+		  unsigned int *out_len, int block)
+{
+	struct ubifs_info *c = inode->i_sb->s_fs_info;
+	int err;
+	unsigned int clen = le16_to_cpu(dn->compr_size);
+	unsigned int dlen = *out_len;
+
+	if (clen <= 0 || clen > UBIFS_BLOCK_SIZE || clen > dlen) {
+		ubifs_err(c, "bad compr_size: %i", clen);
+		return -EINVAL;
+	}
+
+	ubifs_assert(dlen <= UBIFS_BLOCK_SIZE);
+	err = fscrypt_decrypt_page(inode, virt_to_page(&dn->data), dlen,
+			offset_in_page(&dn->data), block);
+	if (err) {
+		ubifs_err(c, "fscrypt_decrypt_page failed: %i", err);
+		return err;
+	}
+	*out_len = clen;
+
+	return 0;
+}
+
+struct fscrypt_operations ubifs_crypt_operations = {
+	.flags			= FS_CFLG_OWN_PAGES,
+	.get_context		= ubifs_crypt_get_context,
+	.set_context		= ubifs_crypt_set_context,
+	.is_encrypted		= __ubifs_crypt_is_encrypted,
+	.empty_dir		= ubifs_crypt_empty_dir,
+	.max_namelen		= ubifs_crypt_max_namelen,
+	.key_prefix		= ubifs_key_prefix,
+};
diff --git a/fs/ubifs/debug.c b/fs/ubifs/debug.c
index 69e287e20732..1e712a364680 100644
--- a/fs/ubifs/debug.c
+++ b/fs/ubifs/debug.c
@@ -233,7 +233,7 @@ static void dump_ch(const struct ubifs_ch *ch)
 void ubifs_dump_inode(struct ubifs_info *c, const struct inode *inode)
 {
 	const struct ubifs_inode *ui = ubifs_inode(inode);
-	struct qstr nm = { .name = NULL };
+	struct fscrypt_name nm = {0};
 	union ubifs_key key;
 	struct ubifs_dent_node *dent, *pdent = NULL;
 	int count = 2;
@@ -289,8 +289,8 @@ void ubifs_dump_inode(struct ubifs_info *c, const struct inode *inode)
 		pr_err("\t%d: %s (%s)\n",
 		       count++, dent->name, get_dent_type(dent->type));
 
-		nm.name = dent->name;
-		nm.len = le16_to_cpu(dent->nlen);
+		fname_name(&nm) = dent->name;
+		fname_len(&nm) = le16_to_cpu(dent->nlen);
 		kfree(pdent);
 		pdent = dent;
 		key_read(c, &dent->key, &key);
@@ -1107,7 +1107,7 @@ int dbg_check_dir(struct ubifs_info *c, const struct inode *dir)
 	unsigned int nlink = 2;
 	union ubifs_key key;
 	struct ubifs_dent_node *dent, *pdent = NULL;
-	struct qstr nm = { .name = NULL };
+	struct fscrypt_name nm = {0};
 	loff_t size = UBIFS_INO_NODE_SZ;
 
 	if (!dbg_is_chk_gen(c))
@@ -1128,9 +1128,9 @@ int dbg_check_dir(struct ubifs_info *c, const struct inode *dir)
 			return err;
 		}
 
-		nm.name = dent->name;
-		nm.len = le16_to_cpu(dent->nlen);
-		size += CALC_DENT_SIZE(nm.len);
+		fname_name(&nm) = dent->name;
+		fname_len(&nm) = le16_to_cpu(dent->nlen);
+		size += CALC_DENT_SIZE(fname_len(&nm));
 		if (dent->type == UBIFS_ITYPE_DIR)
 			nlink += 1;
 		kfree(pdent);
diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c
index ca16c5d7bab1..1c5331ac9614 100644
--- a/fs/ubifs/dir.c
+++ b/fs/ubifs/dir.c
@@ -85,11 +85,26 @@ static int inherit_flags(const struct inode *dir, umode_t mode)
  * initializes it. Returns new inode in case of success and an error code in
  * case of failure.
  */
-struct inode *ubifs_new_inode(struct ubifs_info *c, const struct inode *dir,
+struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir,
 			      umode_t mode)
 {
+	int err;
 	struct inode *inode;
 	struct ubifs_inode *ui;
+	bool encrypted = false;
+
+	if (ubifs_crypt_is_encrypted(dir)) {
+		err = fscrypt_get_encryption_info(dir);
+		if (err) {
+			ubifs_err(c, "fscrypt_get_encryption_info failed: %i", err);
+			return ERR_PTR(err);
+		}
+
+		if (!fscrypt_has_encryption_key(dir))
+			return ERR_PTR(-EPERM);
+
+		encrypted = true;
+	}
 
 	inode = new_inode(c->vfs_sb);
 	ui = ubifs_inode(inode);
@@ -165,18 +180,29 @@ struct inode *ubifs_new_inode(struct ubifs_info *c, const struct inode *dir,
 	 */
 	ui->creat_sqnum = ++c->max_sqnum;
 	spin_unlock(&c->cnt_lock);
+
+	if (encrypted) {
+		err = fscrypt_inherit_context(dir, inode, &encrypted, true);
+		if (err) {
+			ubifs_err(c, "fscrypt_inherit_context failed: %i", err);
+			make_bad_inode(inode);
+			iput(inode);
+			return ERR_PTR(err);
+		}
+	}
+
 	return inode;
 }
 
 static int dbg_check_name(const struct ubifs_info *c,
 			  const struct ubifs_dent_node *dent,
-			  const struct qstr *nm)
+			  const struct fscrypt_name *nm)
 {
 	if (!dbg_is_chk_gen(c))
 		return 0;
-	if (le16_to_cpu(dent->nlen) != nm->len)
+	if (le16_to_cpu(dent->nlen) != fname_len(nm))
 		return -EINVAL;
-	if (memcmp(dent->name, nm->name, nm->len))
+	if (memcmp(dent->name, fname_name(nm), fname_len(nm)))
 		return -EINVAL;
 	return 0;
 }
@@ -189,30 +215,61 @@ static struct dentry *ubifs_lookup(struct inode *dir, struct dentry *dentry,
 	struct inode *inode = NULL;
 	struct ubifs_dent_node *dent;
 	struct ubifs_info *c = dir->i_sb->s_fs_info;
+	struct fscrypt_name nm;
 
 	dbg_gen("'%pd' in dir ino %lu", dentry, dir->i_ino);
 
-	if (dentry->d_name.len > UBIFS_MAX_NLEN)
-		return ERR_PTR(-ENAMETOOLONG);
+	if (ubifs_crypt_is_encrypted(dir)) {
+		err = fscrypt_get_encryption_info(dir);
+
+		/*
+		 * DCACHE_ENCRYPTED_WITH_KEY is set if the dentry is
+		 * created while the directory was encrypted and we
+		 * have access to the key.
+		 */
+		if (fscrypt_has_encryption_key(dir))
+			fscrypt_set_encrypted_dentry(dentry);
+		fscrypt_set_d_op(dentry);
+		if (err && err != -ENOKEY)
+			return ERR_PTR(err);
+	}
+
+	err = fscrypt_setup_filename(dir, &dentry->d_name, 1, &nm);
+	if (err)
+		return ERR_PTR(err);
+
+	if (fname_len(&nm) > UBIFS_MAX_NLEN) {
+		err = -ENAMETOOLONG;
+		goto out_fname;
+	}
 
 	dent = kmalloc(UBIFS_MAX_DENT_NODE_SZ, GFP_NOFS);
-	if (!dent)
-		return ERR_PTR(-ENOMEM);
+	if (!dent) {
+		err = -ENOMEM;
+		goto out_fname;
+	}
 
-	dent_key_init(c, &key, dir->i_ino, &dentry->d_name);
+	if (nm.hash) {
+		ubifs_assert(fname_len(&nm) == 0);
+		ubifs_assert(fname_name(&nm) == NULL);
+		dent_key_init_hash(c, &key, dir->i_ino, nm.hash);
+		err = ubifs_tnc_lookup_dh(c, &key, dent, nm.minor_hash);
+	} else {
+		dent_key_init(c, &key, dir->i_ino, &nm);
+		err = ubifs_tnc_lookup_nm(c, &key, dent, &nm);
+	}
 
-	err = ubifs_tnc_lookup_nm(c, &key, dent, &dentry->d_name);
 	if (err) {
 		if (err == -ENOENT) {
 			dbg_gen("not found");
 			goto done;
 		}
-		goto out;
+		goto out_dent;
 	}
 
-	if (dbg_check_name(c, dent, &dentry->d_name)) {
+	if (dbg_check_name(c, dent, &nm)) {
 		err = -EINVAL;
-		goto out;
+		goto out_dent;
 	}
 
 	inode = ubifs_iget(dir->i_sb, le64_to_cpu(dent->inum));
@@ -225,11 +282,12 @@ static struct dentry *ubifs_lookup(struct inode *dir, struct dentry *dentry,
 		ubifs_err(c, "dead directory entry '%pd', error %d",
 			  dentry, err);
 		ubifs_ro_mode(c, err);
-		goto out;
+		goto out_dent;
 	}
 
 done:
 	kfree(dent);
+	fscrypt_free_filename(&nm);
 	/*
 	 * Note, d_splice_alias() would be required instead if we supported
 	 * NFS.
@@ -237,8 +295,10 @@ done:
 	d_add(dentry, inode);
 	return NULL;
 
-out:
+out_dent:
 	kfree(dent);
+out_fname:
+	fscrypt_free_filename(&nm);
 	return ERR_PTR(err);
 }
 
@@ -247,10 +307,11 @@ static int ubifs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
 {
 	struct inode *inode;
 	struct ubifs_info *c = dir->i_sb->s_fs_info;
-	int err, sz_change = CALC_DENT_SIZE(dentry->d_name.len);
 	struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1,
 					.dirtied_ino = 1 };
 	struct ubifs_inode *dir_ui = ubifs_inode(dir);
+	struct fscrypt_name nm;
+	int err, sz_change;
 
 	/*
 	 * Budget request settings: new inode, new direntry, changing the
@@ -264,10 +325,16 @@ static int ubifs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
 	if (err)
 		return err;
 
+	err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm);
+	if (err)
+		goto out_budg;
+
+	sz_change = CALC_DENT_SIZE(fname_len(&nm));
+
 	inode = ubifs_new_inode(c, dir, mode);
 	if (IS_ERR(inode)) {
 		err = PTR_ERR(inode);
-		goto out_budg;
+		goto out_fname;
 	}
 
 	err = ubifs_init_security(dir, inode, &dentry->d_name);
@@ -278,12 +345,13 @@ static int ubifs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
 	dir->i_size += sz_change;
 	dir_ui->ui_size = dir->i_size;
 	dir->i_mtime = dir->i_ctime = inode->i_ctime;
-	err = ubifs_jnl_update(c, dir, &dentry->d_name, inode, 0, 0);
+	err = ubifs_jnl_update(c, dir, &nm, inode, 0, 0);
 	if (err)
 		goto out_cancel;
 	mutex_unlock(&dir_ui->ui_mutex);
 
 	ubifs_release_budget(c, &req);
+	fscrypt_free_filename(&nm);
 	insert_inode_hash(inode);
 	d_instantiate(dentry, inode);
 	return 0;
@@ -295,6 +363,8 @@ out_cancel:
 out_inode:
 	make_bad_inode(inode);
 	iput(inode);
+out_fname:
+	fscrypt_free_filename(&nm);
 out_budg:
 	ubifs_release_budget(c, &req);
 	ubifs_err(c, "cannot create regular file, error %d", err);
@@ -310,6 +380,7 @@ static int do_tmpfile(struct inode *dir, struct dentry *dentry,
 	struct ubifs_budget_req ino_req = { .dirtied_ino = 1 };
 	struct ubifs_inode *ui, *dir_ui = ubifs_inode(dir);
 	int err, instantiated = 0;
+	struct fscrypt_name nm;
 
 	/*
 	 * Budget request settings: new dirty inode, new direntry,
@@ -319,13 +390,30 @@ static int do_tmpfile(struct inode *dir, struct dentry *dentry,
 	dbg_gen("dent '%pd', mode %#hx in dir ino %lu",
 		dentry, mode, dir->i_ino);
 
-	err = ubifs_budget_space(c, &req);
+	if (ubifs_crypt_is_encrypted(dir)) {
+		err = fscrypt_get_encryption_info(dir);
+		if (err)
+			return err;
+
+		if (!fscrypt_has_encryption_key(dir)) {
+			return -EPERM;
+		}
+	}
+
+	err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm);
 	if (err)
 		return err;
 
+	err = ubifs_budget_space(c, &req);
+	if (err) {
+		fscrypt_free_filename(&nm);
+		return err;
+	}
+
 	err = ubifs_budget_space(c, &ino_req);
 	if (err) {
 		ubifs_release_budget(c, &req);
+		fscrypt_free_filename(&nm);
 		return err;
 	}
 
@@ -361,7 +449,7 @@ static int do_tmpfile(struct inode *dir, struct dentry *dentry,
 	mutex_unlock(&ui->ui_mutex);
 
 	mutex_lock(&dir_ui->ui_mutex);
-	err = ubifs_jnl_update(c, dir, &dentry->d_name, inode, 1, 0);
+	err = ubifs_jnl_update(c, dir, &nm, inode, 1, 0);
 	if (err)
 		goto out_cancel;
 	mutex_unlock(&dir_ui->ui_mutex);
@@ -380,6 +468,7 @@ out_budg:
 	ubifs_release_budget(c, &req);
 	if (!instantiated)
 		ubifs_release_budget(c, &ino_req);
+	fscrypt_free_filename(&nm);
 	ubifs_err(c, "cannot create temporary file, error %d", err);
 	return err;
 }
@@ -439,12 +528,14 @@ static unsigned int vfs_dent_type(uint8_t type)
  */
 static int ubifs_readdir(struct file *file, struct dir_context *ctx)
 {
-	int err = 0;
-	struct qstr nm;
+	int fstr_real_len = 0, err = 0;
+	struct fscrypt_name nm;
+	struct fscrypt_str fstr = {0};
 	union ubifs_key key;
 	struct ubifs_dent_node *dent;
 	struct inode *dir = file_inode(file);
 	struct ubifs_info *c = dir->i_sb->s_fs_info;
+	bool encrypted = ubifs_crypt_is_encrypted(dir);
 
 	dbg_gen("dir ino %lu, f_pos %#llx", dir->i_ino, ctx->pos);
 
@@ -455,6 +546,18 @@ static int ubifs_readdir(struct file *file, struct dir_context *ctx)
 		 */
 		return 0;
 
+	if (encrypted) {
+		err = fscrypt_get_encryption_info(dir);
+		if (err && err != -ENOKEY)
+			return err;
+
+		err = fscrypt_fname_alloc_buffer(dir, UBIFS_MAX_NLEN, &fstr);
+		if (err)
+			return err;
+
+		fstr_real_len = fstr.len;
+	}
+
 	if (file->f_version == 0) {
 		/*
 		 * The file was seek'ed, which means that @file->private_data
@@ -476,12 +579,15 @@ static int ubifs_readdir(struct file *file, struct dir_context *ctx)
 	/* File positions 0 and 1 correspond to "." and ".." */
 	if (ctx->pos < 2) {
 		ubifs_assert(!file->private_data);
-		if (!dir_emit_dots(file, ctx))
+		if (!dir_emit_dots(file, ctx)) {
+			if (encrypted)
+				fscrypt_fname_free_buffer(&fstr);
 			return 0;
+		}
 
 		/* Find the first entry in TNC and save it */
 		lowest_dent_key(c, &key, dir->i_ino);
-		nm.name = NULL;
+		fname_len(&nm) = 0;
 		dent = ubifs_tnc_next_ent(c, &key, &nm);
 		if (IS_ERR(dent)) {
 			err = PTR_ERR(dent);
@@ -499,7 +605,7 @@ static int ubifs_readdir(struct file *file, struct dir_context *ctx)
 		 * Find the entry corresponding to @ctx->pos or the closest one.
 		 */
 		dent_key_init_hash(c, &key, dir->i_ino, ctx->pos);
-		nm.name = NULL;
+		fname_len(&nm) = 0;
 		dent = ubifs_tnc_next_ent(c, &key, &nm);
 		if (IS_ERR(dent)) {
 			err = PTR_ERR(dent);
@@ -516,15 +622,33 @@ static int ubifs_readdir(struct file *file, struct dir_context *ctx)
 		ubifs_assert(le64_to_cpu(dent->ch.sqnum) >
 			     ubifs_inode(dir)->creat_sqnum);
 
-		nm.len = le16_to_cpu(dent->nlen);
-		if (!dir_emit(ctx, dent->name, nm.len,
+		fname_len(&nm) = le16_to_cpu(dent->nlen);
+		fname_name(&nm) = dent->name;
+
+		if (encrypted) {
+			fstr.len = fstr_real_len;
+
+			err = fscrypt_fname_disk_to_usr(dir, key_hash_flash(c,
+							&dent->key),
+							le32_to_cpu(dent->cookie),
+							&nm.disk_name, &fstr);
+			if (err)
+				goto out;
+		} else {
+			fstr.len = fname_len(&nm);
+			fstr.name = fname_name(&nm);
+		}
+
+		if (!dir_emit(ctx, fstr.name, fstr.len,
 			       le64_to_cpu(dent->inum),
-			       vfs_dent_type(dent->type)))
+			       vfs_dent_type(dent->type))) {
+			if (encrypted)
+				fscrypt_fname_free_buffer(&fstr);
 			return 0;
+		}
 
 		/* Switch to the next entry */
 		key_read(c, &dent->key, &key);
-		nm.name = dent->name;
 		dent = ubifs_tnc_next_ent(c, &key, &nm);
 		if (IS_ERR(dent)) {
 			err = PTR_ERR(dent);
@@ -541,6 +665,9 @@ out:
 	kfree(file->private_data);
 	file->private_data = NULL;
 
+	if (encrypted)
+		fscrypt_fname_free_buffer(&fstr);
+
 	if (err != -ENOENT)
 		ubifs_err(c, "cannot find next direntry, error %d", err);
 	else
@@ -601,6 +728,7 @@ static int ubifs_link(struct dentry *old_dentry, struct inode *dir,
 	int err, sz_change = CALC_DENT_SIZE(dentry->d_name.len);
 	struct ubifs_budget_req req = { .new_dent = 1, .dirtied_ino = 2,
 				.dirtied_ino_d = ALIGN(ui->data_len, 8) };
+	struct fscrypt_name nm;
 
 	/*
 	 * Budget request settings: new direntry, changing the target inode,
@@ -613,13 +741,29 @@ static int ubifs_link(struct dentry *old_dentry, struct inode *dir,
 	ubifs_assert(inode_is_locked(dir));
 	ubifs_assert(inode_is_locked(inode));
 
-	err = dbg_check_synced_i_size(c, inode);
+	if (ubifs_crypt_is_encrypted(dir)) {
+		if (!fscrypt_has_permitted_context(dir, inode))
+			return -EPERM;
+
+		err = fscrypt_get_encryption_info(inode);
+		if (err)
+			return err;
+
+		if (!fscrypt_has_encryption_key(inode))
+			return -EPERM;
+	}
+
+	err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm);
 	if (err)
 		return err;
 
+	err = dbg_check_synced_i_size(c, inode);
+	if (err)
+		goto out_fname;
+
 	err = ubifs_budget_space(c, &req);
 	if (err)
-		return err;
+		goto out_fname;
 
 	lock_2_inodes(dir, inode);
 	inc_nlink(inode);
@@ -628,13 +772,14 @@ static int ubifs_link(struct dentry *old_dentry, struct inode *dir,
 	dir->i_size += sz_change;
 	dir_ui->ui_size = dir->i_size;
 	dir->i_mtime = dir->i_ctime = inode->i_ctime;
-	err = ubifs_jnl_update(c, dir, &dentry->d_name, inode, 0, 0);
+	err = ubifs_jnl_update(c, dir, &nm, inode, 0, 0);
 	if (err)
 		goto out_cancel;
 	unlock_2_inodes(dir, inode);
 
 	ubifs_release_budget(c, &req);
 	d_instantiate(dentry, inode);
+	fscrypt_free_filename(&nm);
 	return 0;
 
 out_cancel:
@@ -644,6 +789,8 @@ out_cancel:
 	unlock_2_inodes(dir, inode);
 	ubifs_release_budget(c, &req);
 	iput(inode);
+out_fname:
+	fscrypt_free_filename(&nm);
 	return err;
 }
 
@@ -652,10 +799,10 @@ static int ubifs_unlink(struct inode *dir, struct dentry *dentry)
 	struct ubifs_info *c = dir->i_sb->s_fs_info;
 	struct inode *inode = d_inode(dentry);
 	struct ubifs_inode *dir_ui = ubifs_inode(dir);
-	int sz_change = CALC_DENT_SIZE(dentry->d_name.len);
-	int err, budgeted = 1;
+	int err, sz_change, budgeted = 1;
 	struct ubifs_budget_req req = { .mod_dent = 1, .dirtied_ino = 2 };
 	unsigned int saved_nlink = inode->i_nlink;
+	struct fscrypt_name nm;
 
 	/*
 	 * Budget request settings: deletion direntry, deletion inode (+1 for
@@ -667,16 +814,29 @@ static int ubifs_unlink(struct inode *dir, struct dentry *dentry)
 	dbg_gen("dent '%pd' from ino %lu (nlink %d) in dir ino %lu",
 		dentry, inode->i_ino,
 		inode->i_nlink, dir->i_ino);
+
+	if (ubifs_crypt_is_encrypted(dir)) {
+		err = fscrypt_get_encryption_info(dir);
+		if (err && err != -ENOKEY)
+			return err;
+	}
+
+	err = fscrypt_setup_filename(dir, &dentry->d_name, 1, &nm);
+	if (err)
+		return err;
+
+	sz_change = CALC_DENT_SIZE(fname_len(&nm));
+
 	ubifs_assert(inode_is_locked(dir));
 	ubifs_assert(inode_is_locked(inode));
 	err = dbg_check_synced_i_size(c, inode);
 	if (err)
-		return err;
+		goto out_fname;
 
 	err = ubifs_budget_space(c, &req);
 	if (err) {
 		if (err != -ENOSPC)
-			return err;
+			goto out_fname;
 		budgeted = 0;
 	}
 
@@ -686,7 +846,7 @@ static int ubifs_unlink(struct inode *dir, struct dentry *dentry)
 	dir->i_size -= sz_change;
 	dir_ui->ui_size = dir->i_size;
 	dir->i_mtime = dir->i_ctime = inode->i_ctime;
-	err = ubifs_jnl_update(c, dir, &dentry->d_name, inode, 1, 0);
+	err = ubifs_jnl_update(c, dir, &nm, inode, 1, 0);
 	if (err)
 		goto out_cancel;
 	unlock_2_inodes(dir, inode);
@@ -698,6 +858,7 @@ static int ubifs_unlink(struct inode *dir, struct dentry *dentry)
 		c->bi.nospace = c->bi.nospace_rp = 0;
 		smp_wmb();
 	}
+	fscrypt_free_filename(&nm);
 	return 0;
 
 out_cancel:
@@ -707,21 +868,23 @@ out_cancel:
 	unlock_2_inodes(dir, inode);
 	if (budgeted)
 		ubifs_release_budget(c, &req);
+out_fname:
+	fscrypt_free_filename(&nm);
 	return err;
 }
 
 /**
  * check_dir_empty - check if a directory is empty or not.
- * @c: UBIFS file-system description object
  * @dir: VFS inode object of the directory to check
  *
  * This function checks if directory @dir is empty. Returns zero if the
  * directory is empty, %-ENOTEMPTY if it is not, and other negative error codes
  * in case of of errors.
  */
-static int check_dir_empty(struct ubifs_info *c, struct inode *dir)
+int ubifs_check_dir_empty(struct inode *dir)
 {
-	struct qstr nm = { .name = NULL };
+	struct ubifs_info *c = dir->i_sb->s_fs_info;
+	struct fscrypt_name nm = { 0 };
 	struct ubifs_dent_node *dent;
 	union ubifs_key key;
 	int err;
@@ -743,10 +906,10 @@ static int ubifs_rmdir(struct inode *dir, struct dentry *dentry)
 {
 	struct ubifs_info *c = dir->i_sb->s_fs_info;
 	struct inode *inode = d_inode(dentry);
-	int sz_change = CALC_DENT_SIZE(dentry->d_name.len);
-	int err, budgeted = 1;
+	int err, sz_change, budgeted = 1;
 	struct ubifs_inode *dir_ui = ubifs_inode(dir);
 	struct ubifs_budget_req req = { .mod_dent = 1, .dirtied_ino = 2 };
+	struct fscrypt_name nm;
 
 	/*
 	 * Budget request settings: deletion direntry, deletion inode and
@@ -758,14 +921,26 @@ static int ubifs_rmdir(struct inode *dir, struct dentry *dentry)
 		inode->i_ino, dir->i_ino);
 	ubifs_assert(inode_is_locked(dir));
 	ubifs_assert(inode_is_locked(inode));
-	err = check_dir_empty(c, d_inode(dentry));
+	err = ubifs_check_dir_empty(d_inode(dentry));
 	if (err)
 		return err;
 
+	if (ubifs_crypt_is_encrypted(dir)) {
+		err = fscrypt_get_encryption_info(dir);
+		if (err && err != -ENOKEY)
+			return err;
+	}
+
+	err = fscrypt_setup_filename(dir, &dentry->d_name, 1, &nm);
+	if (err)
+		return err;
+
+	sz_change = CALC_DENT_SIZE(fname_len(&nm));
+
 	err = ubifs_budget_space(c, &req);
 	if (err) {
 		if (err != -ENOSPC)
-			return err;
+			goto out_fname;
 		budgeted = 0;
 	}
 
@@ -776,7 +951,7 @@ static int ubifs_rmdir(struct inode *dir, struct dentry *dentry)
 	dir->i_size -= sz_change;
 	dir_ui->ui_size = dir->i_size;
 	dir->i_mtime = dir->i_ctime = inode->i_ctime;
-	err = ubifs_jnl_update(c, dir, &dentry->d_name, inode, 1, 0);
+	err = ubifs_jnl_update(c, dir, &nm, inode, 1, 0);
 	if (err)
 		goto out_cancel;
 	unlock_2_inodes(dir, inode);
@@ -788,6 +963,7 @@ static int ubifs_rmdir(struct inode *dir, struct dentry *dentry)
 		c->bi.nospace = c->bi.nospace_rp = 0;
 		smp_wmb();
 	}
+	fscrypt_free_filename(&nm);
 	return 0;
 
 out_cancel:
@@ -798,6 +974,8 @@ out_cancel:
 	unlock_2_inodes(dir, inode);
 	if (budgeted)
 		ubifs_release_budget(c, &req);
+out_fname:
+	fscrypt_free_filename(&nm);
 	return err;
 }
 
@@ -806,8 +984,9 @@ static int ubifs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
 	struct inode *inode;
 	struct ubifs_inode *dir_ui = ubifs_inode(dir);
 	struct ubifs_info *c = dir->i_sb->s_fs_info;
-	int err, sz_change = CALC_DENT_SIZE(dentry->d_name.len);
+	int err, sz_change;
 	struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1 };
+	struct fscrypt_name nm;
 
 	/*
 	 * Budget request settings: new inode, new direntry and changing parent
@@ -821,10 +1000,27 @@ static int ubifs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
 	if (err)
 		return err;
 
+	if (ubifs_crypt_is_encrypted(dir)) {
+		err = fscrypt_get_encryption_info(dir);
+		if (err)
+			goto out_budg;
+
+		if (!fscrypt_has_encryption_key(dir)) {
+			err = -EPERM;
+			goto out_budg;
+		}
+	}
+
+	err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm);
+	if (err)
+		goto out_budg;
+
+	sz_change = CALC_DENT_SIZE(fname_len(&nm));
+
 	inode = ubifs_new_inode(c, dir, S_IFDIR | mode);
 	if (IS_ERR(inode)) {
 		err = PTR_ERR(inode);
-		goto out_budg;
+		goto out_fname;
 	}
 
 	err = ubifs_init_security(dir, inode, &dentry->d_name);
@@ -838,7 +1034,7 @@ static int ubifs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
 	dir->i_size += sz_change;
 	dir_ui->ui_size = dir->i_size;
 	dir->i_mtime = dir->i_ctime = inode->i_ctime;
-	err = ubifs_jnl_update(c, dir, &dentry->d_name, inode, 0, 0);
+	err = ubifs_jnl_update(c, dir, &nm, inode, 0, 0);
 	if (err) {
 		ubifs_err(c, "cannot create directory, error %d", err);
 		goto out_cancel;
@@ -847,6 +1043,7 @@ static int ubifs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
 
 	ubifs_release_budget(c, &req);
 	d_instantiate(dentry, inode);
+	fscrypt_free_filename(&nm);
 	return 0;
 
 out_cancel:
@@ -857,6 +1054,8 @@ out_cancel:
 out_inode:
 	make_bad_inode(inode);
 	iput(inode);
+out_fname:
+	fscrypt_free_filename(&nm);
 out_budg:
 	ubifs_release_budget(c, &req);
 	return err;
@@ -870,11 +1069,12 @@ static int ubifs_mknod(struct inode *dir, struct dentry *dentry,
 	struct ubifs_inode *dir_ui = ubifs_inode(dir);
 	struct ubifs_info *c = dir->i_sb->s_fs_info;
 	union ubifs_dev_desc *dev = NULL;
-	int sz_change = CALC_DENT_SIZE(dentry->d_name.len);
+	int sz_change;
 	int err, devlen = 0;
 	struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1,
 					.new_ino_d = ALIGN(devlen, 8),
 					.dirtied_ino = 1 };
+	struct fscrypt_name nm;
 
 	/*
 	 * Budget request settings: new inode, new direntry and changing parent
@@ -896,11 +1096,28 @@ static int ubifs_mknod(struct inode *dir, struct dentry *dentry,
 		return err;
 	}
 
+	if (ubifs_crypt_is_encrypted(dir)) {
+		err = fscrypt_get_encryption_info(dir);
+		if (err)
+			goto out_budg;
+
+		if (!fscrypt_has_encryption_key(dir)) {
+			err = -EPERM;
+			goto out_budg;
+		}
+	}
+
+	err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm);
+	if (err)
+		goto out_budg;
+
+	sz_change = CALC_DENT_SIZE(fname_len(&nm));
+
 	inode = ubifs_new_inode(c, dir, mode);
 	if (IS_ERR(inode)) {
 		kfree(dev);
 		err = PTR_ERR(inode);
-		goto out_budg;
+		goto out_fname;
 	}
 
 	init_special_inode(inode, inode->i_mode, rdev);
@@ -917,7 +1134,7 @@ static int ubifs_mknod(struct inode *dir, struct dentry *dentry,
 	dir->i_size += sz_change;
 	dir_ui->ui_size = dir->i_size;
 	dir->i_mtime = dir->i_ctime = inode->i_ctime;
-	err = ubifs_jnl_update(c, dir, &dentry->d_name, inode, 0, 0);
+	err = ubifs_jnl_update(c, dir, &nm, inode, 0, 0);
 	if (err)
 		goto out_cancel;
 	mutex_unlock(&dir_ui->ui_mutex);
@@ -925,6 +1142,7 @@ static int ubifs_mknod(struct inode *dir, struct dentry *dentry,
 	ubifs_release_budget(c, &req);
 	insert_inode_hash(inode);
 	d_instantiate(dentry, inode);
+	fscrypt_free_filename(&nm);
 	return 0;
 
 out_cancel:
@@ -934,6 +1152,8 @@ out_cancel:
 out_inode:
 	make_bad_inode(inode);
 	iput(inode);
+out_fname:
+	fscrypt_free_filename(&nm);
 out_budg:
 	ubifs_release_budget(c, &req);
 	return err;
@@ -947,10 +1167,27 @@ static int ubifs_symlink(struct inode *dir, struct dentry *dentry,
 	struct ubifs_inode *dir_ui = ubifs_inode(dir);
 	struct ubifs_info *c = dir->i_sb->s_fs_info;
 	int err, len = strlen(symname);
-	int sz_change = CALC_DENT_SIZE(dentry->d_name.len);
+	int sz_change = CALC_DENT_SIZE(len);
+	struct fscrypt_str disk_link = FSTR_INIT((char *)symname, len + 1);
+	struct fscrypt_symlink_data *sd = NULL;
 	struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1,
 					.new_ino_d = ALIGN(len, 8),
 					.dirtied_ino = 1 };
+	struct fscrypt_name nm;
+
+	if (ubifs_crypt_is_encrypted(dir)) {
+		err = fscrypt_get_encryption_info(dir);
+		if (err)
+			goto out_budg;
+
+		if (!fscrypt_has_encryption_key(dir)) {
+			err = -EPERM;
+			goto out_budg;
+		}
+
+		disk_link.len = (fscrypt_fname_encrypted_size(dir, len) +
+				sizeof(struct fscrypt_symlink_data));
+	}
 
 	/*
 	 * Budget request settings: new inode, new direntry and changing parent
@@ -960,36 +1197,77 @@ static int ubifs_symlink(struct inode *dir, struct dentry *dentry,
 	dbg_gen("dent '%pd', target '%s' in dir ino %lu", dentry,
 		symname, dir->i_ino);
 
-	if (len > UBIFS_MAX_INO_DATA)
+	if (disk_link.len > UBIFS_MAX_INO_DATA)
 		return -ENAMETOOLONG;
 
 	err = ubifs_budget_space(c, &req);
 	if (err)
 		return err;
 
+	err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm);
+	if (err)
+		goto out_budg;
+
 	inode = ubifs_new_inode(c, dir, S_IFLNK | S_IRWXUGO);
 	if (IS_ERR(inode)) {
 		err = PTR_ERR(inode);
-		goto out_budg;
+		goto out_fname;
 	}
 
 	ui = ubifs_inode(inode);
-	ui->data = kmalloc(len + 1, GFP_NOFS);
+	ui->data = kmalloc(disk_link.len, GFP_NOFS);
 	if (!ui->data) {
 		err = -ENOMEM;
 		goto out_inode;
 	}
 
-	memcpy(ui->data, symname, len);
-	((char *)ui->data)[len] = '\0';
-	inode->i_link = ui->data;
+	if (ubifs_crypt_is_encrypted(dir)) {
+		struct qstr istr = QSTR_INIT(symname, len);
+		struct fscrypt_str ostr;
+
+		sd = kzalloc(disk_link.len, GFP_NOFS);
+		if (!sd) {
+			err = -ENOMEM;
+			goto out_inode;
+		}
+
+		err = fscrypt_get_encryption_info(inode);
+		if (err) {
+			kfree(sd);
+			goto out_inode;
+		}
+
+		if (!fscrypt_has_encryption_key(inode)) {
+			kfree(sd);
+			err = -EPERM;
+			goto out_inode;
+		}
+
+		ostr.name = sd->encrypted_path;
+		ostr.len = disk_link.len;
+
+		err = fscrypt_fname_usr_to_disk(inode, &istr, &ostr);
+		if (err) {
+			kfree(sd);
+			goto out_inode;
+		}
+
+		sd->len = cpu_to_le16(ostr.len);
+		disk_link.name = (char *)sd;
+	} else {
+		inode->i_link = ui->data;
+	}
+
+	memcpy(ui->data, disk_link.name, disk_link.len);
+	((char *)ui->data)[disk_link.len - 1] = '\0';
+
 	/*
 	 * The terminating zero byte is not written to the flash media and it
 	 * is put just to make later in-memory string processing simpler. Thus,
 	 * data length is @len, not @len + %1.
 	 */
-	ui->data_len = len;
-	inode->i_size = ubifs_inode(inode)->ui_size = len;
+	ui->data_len = disk_link.len - 1;
+	inode->i_size = ubifs_inode(inode)->ui_size = disk_link.len - 1;
 
 	err = ubifs_init_security(dir, inode, &dentry->d_name);
 	if (err)
@@ -999,7 +1277,7 @@ static int ubifs_symlink(struct inode *dir, struct dentry *dentry,
 	dir->i_size += sz_change;
 	dir_ui->ui_size = dir->i_size;
 	dir->i_mtime = dir->i_ctime = inode->i_ctime;
-	err = ubifs_jnl_update(c, dir, &dentry->d_name, inode, 0, 0);
+	err = ubifs_jnl_update(c, dir, &nm, inode, 0, 0);
 	if (err)
 		goto out_cancel;
 	mutex_unlock(&dir_ui->ui_mutex);
@@ -1007,6 +1285,7 @@ static int ubifs_symlink(struct inode *dir, struct dentry *dentry,
 	ubifs_release_budget(c, &req);
 	insert_inode_hash(inode);
 	d_instantiate(dentry, inode);
+	fscrypt_free_filename(&nm);
 	return 0;
 
 out_cancel:
@@ -1016,6 +1295,8 @@ out_cancel:
 out_inode:
 	make_bad_inode(inode);
 	iput(inode);
+out_fname:
+	fscrypt_free_filename(&nm);
 out_budg:
 	ubifs_release_budget(c, &req);
 	return err;
@@ -1078,15 +1359,14 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry,
 	struct ubifs_inode *whiteout_ui = NULL;
 	int err, release, sync = 0, move = (new_dir != old_dir);
 	int is_dir = S_ISDIR(old_inode->i_mode);
-	int unlink = !!new_inode;
-	int new_sz = CALC_DENT_SIZE(new_dentry->d_name.len);
-	int old_sz = CALC_DENT_SIZE(old_dentry->d_name.len);
+	int unlink = !!new_inode, new_sz, old_sz;
 	struct ubifs_budget_req req = { .new_dent = 1, .mod_dent = 1,
 					.dirtied_ino = 3 };
 	struct ubifs_budget_req ino_req = { .dirtied_ino = 1,
 			.dirtied_ino_d = ALIGN(old_inode_ui->data_len, 8) };
 	struct timespec time;
 	unsigned int uninitialized_var(saved_nlink);
+	struct fscrypt_name old_nm, new_nm;
 
 	if (flags & ~RENAME_NOREPLACE)
 		return -EINVAL;
@@ -1107,17 +1387,41 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry,
 	if (unlink)
 		ubifs_assert(inode_is_locked(new_inode));
 
+	if (old_dir != new_dir) {
+		if (ubifs_crypt_is_encrypted(new_dir) &&
+		    !fscrypt_has_permitted_context(new_dir, old_inode))
+			return -EPERM;
+	}
+
 	if (unlink && is_dir) {
-		err = check_dir_empty(c, new_inode);
+		err = ubifs_check_dir_empty(new_inode);
 		if (err)
 			return err;
 	}
 
-	err = ubifs_budget_space(c, &req);
+	err = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_nm);
 	if (err)
 		return err;
+
+	err = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_nm);
+	if (err) {
+		fscrypt_free_filename(&old_nm);
+		return err;
+	}
+
+	new_sz = CALC_DENT_SIZE(fname_len(&new_nm));
+	old_sz = CALC_DENT_SIZE(fname_len(&old_nm));
+
+	err = ubifs_budget_space(c, &req);
+	if (err) {
+		fscrypt_free_filename(&old_nm);
+		fscrypt_free_filename(&new_nm);
+		return err;
+	}
 	err = ubifs_budget_space(c, &ino_req);
 	if (err) {
+		fscrypt_free_filename(&old_nm);
+		fscrypt_free_filename(&new_nm);
 		ubifs_release_budget(c, &req);
 		return err;
 	}
@@ -1239,8 +1543,8 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry,
 		iput(whiteout);
 	}
 
-	err = ubifs_jnl_rename(c, old_dir, old_dentry, new_dir, new_dentry, whiteout,
-			       sync);
+	err = ubifs_jnl_rename(c, old_dir, old_inode, &old_nm, new_dir,
+			       new_inode, &new_nm, whiteout, sync);
 	if (err)
 		goto out_cancel;
 
@@ -1256,6 +1560,9 @@ static int do_rename(struct inode *old_dir, struct dentry *old_dentry,
 		ubifs_release_budget(c, &ino_req);
 	if (IS_SYNC(old_inode))
 		err = old_inode->i_sb->s_op->write_inode(old_inode, NULL);
+
+	fscrypt_free_filename(&old_nm);
+	fscrypt_free_filename(&new_nm);
 	return err;
 
 out_cancel:
@@ -1284,6 +1591,8 @@ out_cancel:
 	unlock_4_inodes(old_dir, new_dir, new_inode, whiteout);
 	ubifs_release_budget(c, &ino_req);
 	ubifs_release_budget(c, &req);
+	fscrypt_free_filename(&old_nm);
+	fscrypt_free_filename(&new_nm);
 	return err;
 }
 
@@ -1298,9 +1607,27 @@ static int ubifs_xrename(struct inode *old_dir, struct dentry *old_dentry,
 	struct inode *snd_inode = d_inode(new_dentry);
 	struct timespec time;
 	int err;
+	struct fscrypt_name fst_nm, snd_nm;
 
 	ubifs_assert(fst_inode && snd_inode);
 
+	if ((ubifs_crypt_is_encrypted(old_dir) ||
+	    ubifs_crypt_is_encrypted(new_dir)) &&
+	    (old_dir != new_dir) &&
+	    (!fscrypt_has_permitted_context(new_dir, fst_inode) ||
+	     !fscrypt_has_permitted_context(old_dir, snd_inode)))
+		return -EPERM;
+
+	err = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &fst_nm);
+	if (err)
+		return err;
+
+	err = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &snd_nm);
+	if (err) {
+		fscrypt_free_filename(&fst_nm);
+		return err;
+	}
+
 	lock_4_inodes(old_dir, new_dir, NULL, NULL);
 
 	time = ubifs_current_time(old_dir);
@@ -1320,12 +1647,14 @@ static int ubifs_xrename(struct inode *old_dir, struct dentry *old_dentry,
 		}
 	}
 
-	err = ubifs_jnl_xrename(c, old_dir, old_dentry, new_dir, new_dentry,
-				sync);
+	err = ubifs_jnl_xrename(c, old_dir, fst_inode, &fst_nm, new_dir,
+				snd_inode, &snd_nm, sync);
 
 	unlock_4_inodes(old_dir, new_dir, NULL, NULL);
 	ubifs_release_budget(c, &req);
 
+	fscrypt_free_filename(&fst_nm);
+	fscrypt_free_filename(&snd_nm);
 	return err;
 }
 
@@ -1384,6 +1713,14 @@ int ubifs_getattr(struct vfsmount *mnt, struct dentry *dentry,
 	return 0;
 }
 
+static int ubifs_dir_open(struct inode *dir, struct file *file)
+{
+	if (ubifs_crypt_is_encrypted(dir))
+		return fscrypt_get_encryption_info(dir) ? -EACCES : 0;
+
+	return 0;
+}
+
 const struct inode_operations ubifs_dir_inode_operations = {
 	.lookup      = ubifs_lookup,
 	.create      = ubifs_create,
@@ -1410,6 +1747,7 @@ const struct file_operations ubifs_dir_operations = {
 	.iterate_shared = ubifs_readdir,
 	.fsync          = ubifs_fsync,
 	.unlocked_ioctl = ubifs_ioctl,
+	.open		= ubifs_dir_open,
 #ifdef CONFIG_COMPAT
 	.compat_ioctl   = ubifs_compat_ioctl,
 #endif
diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c
index b4fbeefba246..aa0625f4f642 100644
--- a/fs/ubifs/file.c
+++ b/fs/ubifs/file.c
@@ -78,6 +78,13 @@ static int read_block(struct inode *inode, void *addr, unsigned int block,
 		goto dump;
 
 	dlen = le32_to_cpu(dn->ch.len) - UBIFS_DATA_NODE_SZ;
+
+	if (ubifs_crypt_is_encrypted(inode)) {
+		err = ubifs_decrypt(inode, dn, &dlen, block);
+		if (err)
+			goto dump;
+	}
+
 	out_len = UBIFS_BLOCK_SIZE;
 	err = ubifs_decompress(c, &dn->data, dlen, addr, &out_len,
 			       le16_to_cpu(dn->compr_type));
@@ -650,6 +657,13 @@ static int populate_page(struct ubifs_info *c, struct page *page,
 
 			dlen = le32_to_cpu(dn->ch.len) - UBIFS_DATA_NODE_SZ;
 			out_len = UBIFS_BLOCK_SIZE;
+
+			if (ubifs_crypt_is_encrypted(inode)) {
+				err = ubifs_decrypt(inode, dn, &dlen, page_block);
+				if (err)
+					goto out_err;
+			}
+
 			err = ubifs_decompress(c, &dn->data, dlen, addr, &out_len,
 					       le16_to_cpu(dn->compr_type));
 			if (err || len != out_len)
@@ -1594,6 +1608,15 @@ static const struct vm_operations_struct ubifs_file_vm_ops = {
 static int ubifs_file_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	int err;
+	struct inode *inode = file->f_mapping->host;
+
+	if (ubifs_crypt_is_encrypted(inode)) {
+		err = fscrypt_get_encryption_info(inode);
+		if (err)
+			return -EACCES;
+		if (!fscrypt_has_encryption_key(inode))
+			return -ENOKEY;
+	}
 
 	err = generic_file_mmap(file, vma);
 	if (err)
@@ -1605,6 +1628,88 @@ static int ubifs_file_mmap(struct file *file, struct vm_area_struct *vma)
 	return 0;
 }
 
+static int ubifs_file_open(struct inode *inode, struct file *filp)
+{
+	int ret;
+	struct dentry *dir;
+	struct ubifs_info *c = inode->i_sb->s_fs_info;
+
+	if (ubifs_crypt_is_encrypted(inode)) {
+		ret = fscrypt_get_encryption_info(inode);
+		if (ret)
+			return -EACCES;
+		if (!fscrypt_has_encryption_key(inode))
+			return -ENOKEY;
+	}
+
+	dir = dget_parent(file_dentry(filp));
+	if (ubifs_crypt_is_encrypted(d_inode(dir)) &&
+			!fscrypt_has_permitted_context(d_inode(dir), inode)) {
+		ubifs_err(c, "Inconsistent encryption contexts: %lu/%lu",
+			  (unsigned long) d_inode(dir)->i_ino,
+			  (unsigned long) inode->i_ino);
+		dput(dir);
+		ubifs_ro_mode(c, -EPERM);
+		return -EPERM;
+	}
+	dput(dir);
+
+	return 0;
+}
+
+static const char *ubifs_get_link(struct dentry *dentry,
+					    struct inode *inode,
+					    struct delayed_call *done)
+{
+	int err;
+	struct fscrypt_symlink_data *sd;
+	struct ubifs_inode *ui = ubifs_inode(inode);
+	struct fscrypt_str cstr;
+	struct fscrypt_str pstr;
+
+	if (!ubifs_crypt_is_encrypted(inode))
+		return ui->data;
+
+	if (!dentry)
+		return ERR_PTR(-ECHILD);
+
+	err = fscrypt_get_encryption_info(inode);
+	if (err)
+		return ERR_PTR(err);
+
+	sd = (struct fscrypt_symlink_data *)ui->data;
+	cstr.name = sd->encrypted_path;
+	cstr.len = le16_to_cpu(sd->len);
+
+	if (cstr.len == 0)
+		return ERR_PTR(-ENOENT);
+
+	if ((cstr.len + sizeof(struct fscrypt_symlink_data) - 1) > ui->data_len)
+		return ERR_PTR(-EIO);
+
+	err = fscrypt_fname_alloc_buffer(inode, cstr.len, &pstr);
+	if (err)
+		return ERR_PTR(err);
+
+	err = fscrypt_fname_disk_to_usr(inode, 0, 0, &cstr, &pstr);
+	if (err) {
+		fscrypt_fname_free_buffer(&pstr);
+		return ERR_PTR(err);
+	}
+
+	pstr.name[pstr.len] = '\0';
+
+	// XXX this probably won't happen anymore...
+	if (pstr.name[0] == '\0') {
+		fscrypt_fname_free_buffer(&pstr);
+		return ERR_PTR(-ENOENT);
+	}
+
+	set_delayed_call(done, kfree_link, pstr.name);
+	return pstr.name;
+}
+
+
 const struct address_space_operations ubifs_file_address_operations = {
 	.readpage       = ubifs_readpage,
 	.writepage      = ubifs_writepage,
@@ -1629,7 +1734,7 @@ const struct inode_operations ubifs_file_inode_operations = {
 
 const struct inode_operations ubifs_symlink_inode_operations = {
 	.readlink    = generic_readlink,
-	.get_link    = simple_get_link,
+	.get_link    = ubifs_get_link,
 	.setattr     = ubifs_setattr,
 	.getattr     = ubifs_getattr,
 	.listxattr   = ubifs_listxattr,
@@ -1647,6 +1752,7 @@ const struct file_operations ubifs_file_operations = {
 	.unlocked_ioctl = ubifs_ioctl,
 	.splice_read	= generic_file_splice_read,
 	.splice_write	= iter_file_splice_write,
+	.open		= ubifs_file_open,
 #ifdef CONFIG_COMPAT
 	.compat_ioctl   = ubifs_compat_ioctl,
 #endif
diff --git a/fs/ubifs/gc.c b/fs/ubifs/gc.c
index e845c64b6ce1..7b35e3d6cde7 100644
--- a/fs/ubifs/gc.c
+++ b/fs/ubifs/gc.c
@@ -846,10 +846,6 @@ int ubifs_gc_start_commit(struct ubifs_info *c)
 	 */
 	while (1) {
 		lp = ubifs_fast_find_freeable(c);
-		if (IS_ERR(lp)) {
-			err = PTR_ERR(lp);
-			goto out;
-		}
 		if (!lp)
 			break;
 		ubifs_assert(!(lp->flags & LPROPS_TAKEN));
diff --git a/fs/ubifs/io.c b/fs/ubifs/io.c
index 97be41215332..3be28900bf37 100644
--- a/fs/ubifs/io.c
+++ b/fs/ubifs/io.c
@@ -452,16 +452,22 @@ static enum hrtimer_restart wbuf_timer_callback_nolock(struct hrtimer *timer)
  */
 static void new_wbuf_timer_nolock(struct ubifs_wbuf *wbuf)
 {
+	ktime_t softlimit = ms_to_ktime(dirty_writeback_interval * 10);
+	unsigned long long delta = dirty_writeback_interval;
+
+	/* centi to milli, milli to nano, then 10% */
+	delta *= 10ULL * NSEC_PER_MSEC / 10ULL;
+
 	ubifs_assert(!hrtimer_active(&wbuf->timer));
+	ubifs_assert(delta <= ULONG_MAX);
 
 	if (wbuf->no_timer)
 		return;
 	dbg_io("set timer for jhead %s, %llu-%llu millisecs",
 	       dbg_jhead(wbuf->jhead),
-	       div_u64(ktime_to_ns(wbuf->softlimit), USEC_PER_SEC),
-	       div_u64(ktime_to_ns(wbuf->softlimit) + wbuf->delta,
-		       USEC_PER_SEC));
-	hrtimer_start_range_ns(&wbuf->timer, wbuf->softlimit, wbuf->delta,
+	       div_u64(ktime_to_ns(softlimit), USEC_PER_SEC),
+	       div_u64(ktime_to_ns(softlimit) + delta, USEC_PER_SEC));
+	hrtimer_start_range_ns(&wbuf->timer, softlimit, delta,
 			       HRTIMER_MODE_REL);
 }
 
@@ -1059,10 +1065,6 @@ int ubifs_wbuf_init(struct ubifs_info *c, struct ubifs_wbuf *wbuf)
 
 	hrtimer_init(&wbuf->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	wbuf->timer.function = wbuf_timer_callback_nolock;
-	wbuf->softlimit = ktime_set(WBUF_TIMEOUT_SOFTLIMIT, 0);
-	wbuf->delta = WBUF_TIMEOUT_HARDLIMIT - WBUF_TIMEOUT_SOFTLIMIT;
-	wbuf->delta *= 1000000000ULL;
-	ubifs_assert(wbuf->delta <= ULONG_MAX);
 	return 0;
 }
 
diff --git a/fs/ubifs/ioctl.c b/fs/ubifs/ioctl.c
index 3c7b29de0ca7..78d713644df3 100644
--- a/fs/ubifs/ioctl.c
+++ b/fs/ubifs/ioctl.c
@@ -181,6 +181,26 @@ long ubifs_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 		mnt_drop_write_file(file);
 		return err;
 	}
+	case FS_IOC_SET_ENCRYPTION_POLICY: {
+#ifdef CONFIG_UBIFS_FS_ENCRYPTION
+		struct ubifs_info *c = inode->i_sb->s_fs_info;
+
+		err = ubifs_enable_encryption(c);
+		if (err)
+			return err;
+
+		return fscrypt_ioctl_set_policy(file, (const void __user *)arg);
+#else
+		return -EOPNOTSUPP;
+#endif
+	}
+	case FS_IOC_GET_ENCRYPTION_POLICY: {
+#ifdef CONFIG_UBIFS_FS_ENCRYPTION
+		return fscrypt_ioctl_get_policy(file, (void __user *)arg);
+#else
+		return -EOPNOTSUPP;
+#endif
+	}
 
 	default:
 		return -ENOTTY;
diff --git a/fs/ubifs/journal.c b/fs/ubifs/journal.c
index 91bc76dc559e..a459211a1c21 100644
--- a/fs/ubifs/journal.c
+++ b/fs/ubifs/journal.c
@@ -78,16 +78,6 @@ static inline void zero_ino_node_unused(struct ubifs_ino_node *ino)
 static inline void zero_dent_node_unused(struct ubifs_dent_node *dent)
 {
 	dent->padding1 = 0;
-	memset(dent->padding2, 0, 4);
-}
-
-/**
- * zero_data_node_unused - zero out unused fields of an on-flash data node.
- * @data: the data node to zero out
- */
-static inline void zero_data_node_unused(struct ubifs_data_node *data)
-{
-	memset(data->padding, 0, 2);
 }
 
 /**
@@ -511,6 +501,14 @@ static void mark_inode_clean(struct ubifs_info *c, struct ubifs_inode *ui)
 	ui->dirty = 0;
 }
 
+static void set_dent_cookie(struct ubifs_info *c, struct ubifs_dent_node *dent)
+{
+	if (c->double_hash)
+		dent->cookie = prandom_u32();
+	else
+		dent->cookie = 0;
+}
+
 /**
  * ubifs_jnl_update - update inode.
  * @c: UBIFS file-system description object
@@ -539,7 +537,7 @@ static void mark_inode_clean(struct ubifs_info *c, struct ubifs_inode *ui)
  * success. In case of failure, a negative error code is returned.
  */
 int ubifs_jnl_update(struct ubifs_info *c, const struct inode *dir,
-		     const struct qstr *nm, const struct inode *inode,
+		     const struct fscrypt_name *nm, const struct inode *inode,
 		     int deletion, int xent)
 {
 	int err, dlen, ilen, len, lnum, ino_offs, dent_offs;
@@ -551,11 +549,11 @@ int ubifs_jnl_update(struct ubifs_info *c, const struct inode *dir,
 	struct ubifs_ino_node *ino;
 	union ubifs_key dent_key, ino_key;
 
-	dbg_jnl("ino %lu, dent '%.*s', data len %d in dir ino %lu",
-		inode->i_ino, nm->len, nm->name, ui->data_len, dir->i_ino);
+	//dbg_jnl("ino %lu, dent '%.*s', data len %d in dir ino %lu",
+	//	inode->i_ino, nm->len, nm->name, ui->data_len, dir->i_ino);
 	ubifs_assert(mutex_is_locked(&host_ui->ui_mutex));
 
-	dlen = UBIFS_DENT_NODE_SZ + nm->len + 1;
+	dlen = UBIFS_DENT_NODE_SZ + fname_len(nm) + 1;
 	ilen = UBIFS_INO_NODE_SZ;
 
 	/*
@@ -596,9 +594,11 @@ int ubifs_jnl_update(struct ubifs_info *c, const struct inode *dir,
 	key_write(c, &dent_key, dent->key);
 	dent->inum = deletion ? 0 : cpu_to_le64(inode->i_ino);
 	dent->type = get_dent_type(inode->i_mode);
-	dent->nlen = cpu_to_le16(nm->len);
-	memcpy(dent->name, nm->name, nm->len);
-	dent->name[nm->len] = '\0';
+	dent->nlen = cpu_to_le16(fname_len(nm));
+	memcpy(dent->name, fname_name(nm), fname_len(nm));
+	dent->name[fname_len(nm)] = '\0';
+	set_dent_cookie(c, dent);
+
 	zero_dent_node_unused(dent);
 	ubifs_prep_grp_node(c, dent, dlen, 0);
 
@@ -697,14 +697,18 @@ int ubifs_jnl_write_data(struct ubifs_info *c, const struct inode *inode,
 			 const union ubifs_key *key, const void *buf, int len)
 {
 	struct ubifs_data_node *data;
-	int err, lnum, offs, compr_type, out_len;
+	int err, lnum, offs, compr_type, out_len, compr_len;
 	int dlen = COMPRESSED_DATA_NODE_BUF_SZ, allocated = 1;
 	struct ubifs_inode *ui = ubifs_inode(inode);
+	bool encrypted = ubifs_crypt_is_encrypted(inode);
 
 	dbg_jnlk(key, "ino %lu, blk %u, len %d, key ",
 		(unsigned long)key_inum(c, key), key_block(c, key), len);
 	ubifs_assert(len <= UBIFS_BLOCK_SIZE);
 
+	if (encrypted)
+		dlen += UBIFS_CIPHER_BLOCK_SIZE;
+
 	data = kmalloc(dlen, GFP_NOFS | __GFP_NOWARN);
 	if (!data) {
 		/*
@@ -722,7 +726,6 @@ int ubifs_jnl_write_data(struct ubifs_info *c, const struct inode *inode,
 	data->ch.node_type = UBIFS_DATA_NODE;
 	key_write(c, key, &data->key);
 	data->size = cpu_to_le32(len);
-	zero_data_node_unused(data);
 
 	if (!(ui->flags & UBIFS_COMPR_FL))
 		/* Compression is disabled for this inode */
@@ -730,9 +733,18 @@ int ubifs_jnl_write_data(struct ubifs_info *c, const struct inode *inode,
 	else
 		compr_type = ui->compr_type;
 
-	out_len = dlen - UBIFS_DATA_NODE_SZ;
-	ubifs_compress(c, buf, len, &data->data, &out_len, &compr_type);
-	ubifs_assert(out_len <= UBIFS_BLOCK_SIZE);
+	out_len = compr_len = dlen - UBIFS_DATA_NODE_SZ;
+	ubifs_compress(c, buf, len, &data->data, &compr_len, &compr_type);
+	ubifs_assert(compr_len <= UBIFS_BLOCK_SIZE);
+
+	if (encrypted) {
+		err = ubifs_encrypt(inode, data, compr_len, &out_len, key_block(c, key));
+		if (err)
+			goto out_free;
+
+	} else {
+		data->compr_size = 0;
+	}
 
 	dlen = UBIFS_DATA_NODE_SZ + out_len;
 	data->compr_type = cpu_to_le16(compr_type);
@@ -911,9 +923,11 @@ int ubifs_jnl_delete_inode(struct ubifs_info *c, const struct inode *inode)
  * ubifs_jnl_xrename - cross rename two directory entries.
  * @c: UBIFS file-system description object
  * @fst_dir: parent inode of 1st directory entry to exchange
- * @fst_dentry: 1st directory entry to exchange
+ * @fst_inode: 1st inode to exchange
+ * @fst_nm: name of 1st inode to exchange
  * @snd_dir: parent inode of 2nd directory entry to exchange
- * @snd_dentry: 2nd directory entry to exchange
+ * @snd_inode: 2nd inode to exchange
+ * @snd_nm: name of 2nd inode to exchange
  * @sync: non-zero if the write-buffer has to be synchronized
  *
  * This function implements the cross rename operation which may involve
@@ -922,29 +936,29 @@ int ubifs_jnl_delete_inode(struct ubifs_info *c, const struct inode *inode)
  * returned.
  */
 int ubifs_jnl_xrename(struct ubifs_info *c, const struct inode *fst_dir,
-		      const struct dentry *fst_dentry,
+		      const struct inode *fst_inode,
+		      const struct fscrypt_name *fst_nm,
 		      const struct inode *snd_dir,
-		      const struct dentry *snd_dentry, int sync)
+		      const struct inode *snd_inode,
+		      const struct fscrypt_name *snd_nm, int sync)
 {
 	union ubifs_key key;
 	struct ubifs_dent_node *dent1, *dent2;
 	int err, dlen1, dlen2, lnum, offs, len, plen = UBIFS_INO_NODE_SZ;
 	int aligned_dlen1, aligned_dlen2;
 	int twoparents = (fst_dir != snd_dir);
-	const struct inode *fst_inode = d_inode(fst_dentry);
-	const struct inode *snd_inode = d_inode(snd_dentry);
 	void *p;
 
-	dbg_jnl("dent '%pd' in dir ino %lu between dent '%pd' in dir ino %lu",
-		fst_dentry, fst_dir->i_ino, snd_dentry, snd_dir->i_ino);
+	//dbg_jnl("dent '%pd' in dir ino %lu between dent '%pd' in dir ino %lu",
+	//	fst_dentry, fst_dir->i_ino, snd_dentry, snd_dir->i_ino);
 
 	ubifs_assert(ubifs_inode(fst_dir)->data_len == 0);
 	ubifs_assert(ubifs_inode(snd_dir)->data_len == 0);
 	ubifs_assert(mutex_is_locked(&ubifs_inode(fst_dir)->ui_mutex));
 	ubifs_assert(mutex_is_locked(&ubifs_inode(snd_dir)->ui_mutex));
 
-	dlen1 = UBIFS_DENT_NODE_SZ + snd_dentry->d_name.len + 1;
-	dlen2 = UBIFS_DENT_NODE_SZ + fst_dentry->d_name.len + 1;
+	dlen1 = UBIFS_DENT_NODE_SZ + fname_len(snd_nm) + 1;
+	dlen2 = UBIFS_DENT_NODE_SZ + fname_len(fst_nm) + 1;
 	aligned_dlen1 = ALIGN(dlen1, 8);
 	aligned_dlen2 = ALIGN(dlen2, 8);
 
@@ -963,24 +977,24 @@ int ubifs_jnl_xrename(struct ubifs_info *c, const struct inode *fst_dir,
 
 	/* Make new dent for 1st entry */
 	dent1->ch.node_type = UBIFS_DENT_NODE;
-	dent_key_init_flash(c, &dent1->key, snd_dir->i_ino, &snd_dentry->d_name);
+	dent_key_init_flash(c, &dent1->key, snd_dir->i_ino, snd_nm);
 	dent1->inum = cpu_to_le64(fst_inode->i_ino);
 	dent1->type = get_dent_type(fst_inode->i_mode);
-	dent1->nlen = cpu_to_le16(snd_dentry->d_name.len);
-	memcpy(dent1->name, snd_dentry->d_name.name, snd_dentry->d_name.len);
-	dent1->name[snd_dentry->d_name.len] = '\0';
+	dent1->nlen = cpu_to_le16(fname_len(snd_nm));
+	memcpy(dent1->name, fname_name(snd_nm), fname_len(snd_nm));
+	dent1->name[fname_len(snd_nm)] = '\0';
 	zero_dent_node_unused(dent1);
 	ubifs_prep_grp_node(c, dent1, dlen1, 0);
 
 	/* Make new dent for 2nd entry */
 	dent2 = (void *)dent1 + aligned_dlen1;
 	dent2->ch.node_type = UBIFS_DENT_NODE;
-	dent_key_init_flash(c, &dent2->key, fst_dir->i_ino, &fst_dentry->d_name);
+	dent_key_init_flash(c, &dent2->key, fst_dir->i_ino, fst_nm);
 	dent2->inum = cpu_to_le64(snd_inode->i_ino);
 	dent2->type = get_dent_type(snd_inode->i_mode);
-	dent2->nlen = cpu_to_le16(fst_dentry->d_name.len);
-	memcpy(dent2->name, fst_dentry->d_name.name, fst_dentry->d_name.len);
-	dent2->name[fst_dentry->d_name.len] = '\0';
+	dent2->nlen = cpu_to_le16(fname_len(fst_nm));
+	memcpy(dent2->name, fname_name(fst_nm), fname_len(fst_nm));
+	dent2->name[fname_len(fst_nm)] = '\0';
 	zero_dent_node_unused(dent2);
 	ubifs_prep_grp_node(c, dent2, dlen2, 0);
 
@@ -1004,14 +1018,14 @@ int ubifs_jnl_xrename(struct ubifs_info *c, const struct inode *fst_dir,
 	}
 	release_head(c, BASEHD);
 
-	dent_key_init(c, &key, snd_dir->i_ino, &snd_dentry->d_name);
-	err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen1, &snd_dentry->d_name);
+	dent_key_init(c, &key, snd_dir->i_ino, snd_nm);
+	err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen1, snd_nm);
 	if (err)
 		goto out_ro;
 
 	offs += aligned_dlen1;
-	dent_key_init(c, &key, fst_dir->i_ino, &fst_dentry->d_name);
-	err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen2, &fst_dentry->d_name);
+	dent_key_init(c, &key, fst_dir->i_ino, fst_nm);
+	err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen2, fst_nm);
 	if (err)
 		goto out_ro;
 
@@ -1063,31 +1077,31 @@ out_free:
  * returned.
  */
 int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
-		     const struct dentry *old_dentry,
+		     const struct inode *old_inode,
+		     const struct fscrypt_name *old_nm,
 		     const struct inode *new_dir,
-		     const struct dentry *new_dentry,
+		     const struct inode *new_inode,
+		     const struct fscrypt_name *new_nm,
 		     const struct inode *whiteout, int sync)
 {
 	void *p;
 	union ubifs_key key;
 	struct ubifs_dent_node *dent, *dent2;
 	int err, dlen1, dlen2, ilen, lnum, offs, len;
-	const struct inode *old_inode = d_inode(old_dentry);
-	const struct inode *new_inode = d_inode(new_dentry);
 	int aligned_dlen1, aligned_dlen2, plen = UBIFS_INO_NODE_SZ;
 	int last_reference = !!(new_inode && new_inode->i_nlink == 0);
 	int move = (old_dir != new_dir);
 	struct ubifs_inode *uninitialized_var(new_ui);
 
-	dbg_jnl("dent '%pd' in dir ino %lu to dent '%pd' in dir ino %lu",
-		old_dentry, old_dir->i_ino, new_dentry, new_dir->i_ino);
+	//dbg_jnl("dent '%pd' in dir ino %lu to dent '%pd' in dir ino %lu",
+	//	old_dentry, old_dir->i_ino, new_dentry, new_dir->i_ino);
 	ubifs_assert(ubifs_inode(old_dir)->data_len == 0);
 	ubifs_assert(ubifs_inode(new_dir)->data_len == 0);
 	ubifs_assert(mutex_is_locked(&ubifs_inode(old_dir)->ui_mutex));
 	ubifs_assert(mutex_is_locked(&ubifs_inode(new_dir)->ui_mutex));
 
-	dlen1 = UBIFS_DENT_NODE_SZ + new_dentry->d_name.len + 1;
-	dlen2 = UBIFS_DENT_NODE_SZ + old_dentry->d_name.len + 1;
+	dlen1 = UBIFS_DENT_NODE_SZ + fname_len(new_nm) + 1;
+	dlen2 = UBIFS_DENT_NODE_SZ + fname_len(old_nm) + 1;
 	if (new_inode) {
 		new_ui = ubifs_inode(new_inode);
 		ubifs_assert(mutex_is_locked(&new_ui->ui_mutex));
@@ -1113,19 +1127,19 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
 
 	/* Make new dent */
 	dent->ch.node_type = UBIFS_DENT_NODE;
-	dent_key_init_flash(c, &dent->key, new_dir->i_ino, &new_dentry->d_name);
+	dent_key_init_flash(c, &dent->key, new_dir->i_ino, new_nm);
 	dent->inum = cpu_to_le64(old_inode->i_ino);
 	dent->type = get_dent_type(old_inode->i_mode);
-	dent->nlen = cpu_to_le16(new_dentry->d_name.len);
-	memcpy(dent->name, new_dentry->d_name.name, new_dentry->d_name.len);
-	dent->name[new_dentry->d_name.len] = '\0';
+	dent->nlen = cpu_to_le16(fname_len(new_nm));
+	memcpy(dent->name, fname_name(new_nm), fname_len(new_nm));
+	dent->name[fname_len(new_nm)] = '\0';
+	set_dent_cookie(c, dent);
 	zero_dent_node_unused(dent);
 	ubifs_prep_grp_node(c, dent, dlen1, 0);
 
 	dent2 = (void *)dent + aligned_dlen1;
 	dent2->ch.node_type = UBIFS_DENT_NODE;
-	dent_key_init_flash(c, &dent2->key, old_dir->i_ino,
-			    &old_dentry->d_name);
+	dent_key_init_flash(c, &dent2->key, old_dir->i_ino, old_nm);
 
 	if (whiteout) {
 		dent2->inum = cpu_to_le64(whiteout->i_ino);
@@ -1135,9 +1149,10 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
 		dent2->inum = 0;
 		dent2->type = DT_UNKNOWN;
 	}
-	dent2->nlen = cpu_to_le16(old_dentry->d_name.len);
-	memcpy(dent2->name, old_dentry->d_name.name, old_dentry->d_name.len);
-	dent2->name[old_dentry->d_name.len] = '\0';
+	dent2->nlen = cpu_to_le16(fname_len(old_nm));
+	memcpy(dent2->name, fname_name(old_nm), fname_len(old_nm));
+	dent2->name[fname_len(old_nm)] = '\0';
+	set_dent_cookie(c, dent2);
 	zero_dent_node_unused(dent2);
 	ubifs_prep_grp_node(c, dent2, dlen2, 0);
 
@@ -1178,15 +1193,15 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
 	}
 	release_head(c, BASEHD);
 
-	dent_key_init(c, &key, new_dir->i_ino, &new_dentry->d_name);
-	err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen1, &new_dentry->d_name);
+	dent_key_init(c, &key, new_dir->i_ino, new_nm);
+	err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen1, new_nm);
 	if (err)
 		goto out_ro;
 
 	offs += aligned_dlen1;
 	if (whiteout) {
-		dent_key_init(c, &key, old_dir->i_ino, &old_dentry->d_name);
-		err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen2, &old_dentry->d_name);
+		dent_key_init(c, &key, old_dir->i_ino, old_nm);
+		err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen2, old_nm);
 		if (err)
 			goto out_ro;
 
@@ -1196,8 +1211,8 @@ int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
 		if (err)
 			goto out_ro;
 
-		dent_key_init(c, &key, old_dir->i_ino, &old_dentry->d_name);
-		err = ubifs_tnc_remove_nm(c, &key, &old_dentry->d_name);
+		dent_key_init(c, &key, old_dir->i_ino, old_nm);
+		err = ubifs_tnc_remove_nm(c, &key, old_nm);
 		if (err)
 			goto out_ro;
 	}
@@ -1251,31 +1266,55 @@ out_free:
 }
 
 /**
- * recomp_data_node - re-compress a truncated data node.
+ * truncate_data_node - re-compress/encrypt a truncated data node.
+ * @c: UBIFS file-system description object
+ * @inode: inode which referes to the data node
+ * @block: data block number
  * @dn: data node to re-compress
  * @new_len: new length
  *
  * This function is used when an inode is truncated and the last data node of
- * the inode has to be re-compressed and re-written.
+ * the inode has to be re-compressed/encrypted and re-written.
  */
-static int recomp_data_node(const struct ubifs_info *c,
-			    struct ubifs_data_node *dn, int *new_len)
+static int truncate_data_node(const struct ubifs_info *c, const struct inode *inode,
+			      unsigned int block, struct ubifs_data_node *dn,
+			      int *new_len)
 {
 	void *buf;
-	int err, len, compr_type, out_len;
+	int err, dlen, compr_type, out_len, old_dlen;
 
 	out_len = le32_to_cpu(dn->size);
 	buf = kmalloc(out_len * WORST_COMPR_FACTOR, GFP_NOFS);
 	if (!buf)
 		return -ENOMEM;
 
-	len = le32_to_cpu(dn->ch.len) - UBIFS_DATA_NODE_SZ;
+	dlen = old_dlen = le32_to_cpu(dn->ch.len) - UBIFS_DATA_NODE_SZ;
 	compr_type = le16_to_cpu(dn->compr_type);
-	err = ubifs_decompress(c, &dn->data, len, buf, &out_len, compr_type);
-	if (err)
-		goto out;
 
-	ubifs_compress(c, buf, *new_len, &dn->data, &out_len, &compr_type);
+	if (ubifs_crypt_is_encrypted(inode)) {
+		err = ubifs_decrypt(inode, dn, &dlen, block);
+		if (err)
+			goto out;
+	}
+
+	if (compr_type != UBIFS_COMPR_NONE) {
+		err = ubifs_decompress(c, &dn->data, dlen, buf, &out_len, compr_type);
+		if (err)
+			goto out;
+
+		ubifs_compress(c, buf, *new_len, &dn->data, &out_len, &compr_type);
+	}
+
+	if (ubifs_crypt_is_encrypted(inode)) {
+		err = ubifs_encrypt(inode, dn, out_len, &old_dlen, block);
+		if (err)
+			goto out;
+
+		out_len = old_dlen;
+	} else {
+		dn->compr_size = 0;
+	}
+
 	ubifs_assert(out_len <= UBIFS_BLOCK_SIZE);
 	dn->compr_type = cpu_to_le16(compr_type);
 	dn->size = cpu_to_le32(*new_len);
@@ -1347,17 +1386,9 @@ int ubifs_jnl_truncate(struct ubifs_info *c, const struct inode *inode,
 			if (le32_to_cpu(dn->size) <= dlen)
 				dlen = 0; /* Nothing to do */
 			else {
-				int compr_type = le16_to_cpu(dn->compr_type);
-
-				if (compr_type != UBIFS_COMPR_NONE) {
-					err = recomp_data_node(c, dn, &dlen);
-					if (err)
-						goto out_free;
-				} else {
-					dn->size = cpu_to_le32(dlen);
-					dlen += UBIFS_DATA_NODE_SZ;
-				}
-				zero_data_node_unused(dn);
+				err = truncate_data_node(c, inode, blk, dn, &dlen);
+				if (err)
+					goto out_free;
 			}
 		}
 	}
@@ -1442,7 +1473,8 @@ out_free:
  * error code in case of failure.
  */
 int ubifs_jnl_delete_xattr(struct ubifs_info *c, const struct inode *host,
-			   const struct inode *inode, const struct qstr *nm)
+			   const struct inode *inode,
+			   const struct fscrypt_name *nm)
 {
 	int err, xlen, hlen, len, lnum, xent_offs, aligned_xlen;
 	struct ubifs_dent_node *xent;
@@ -1451,9 +1483,9 @@ int ubifs_jnl_delete_xattr(struct ubifs_info *c, const struct inode *host,
 	int sync = IS_DIRSYNC(host);
 	struct ubifs_inode *host_ui = ubifs_inode(host);
 
-	dbg_jnl("host %lu, xattr ino %lu, name '%s', data len %d",
-		host->i_ino, inode->i_ino, nm->name,
-		ubifs_inode(inode)->data_len);
+	//dbg_jnl("host %lu, xattr ino %lu, name '%s', data len %d",
+	//	host->i_ino, inode->i_ino, nm->name,
+	//	ubifs_inode(inode)->data_len);
 	ubifs_assert(inode->i_nlink == 0);
 	ubifs_assert(mutex_is_locked(&host_ui->ui_mutex));
 
@@ -1461,7 +1493,7 @@ int ubifs_jnl_delete_xattr(struct ubifs_info *c, const struct inode *host,
 	 * Since we are deleting the inode, we do not bother to attach any data
 	 * to it and assume its length is %UBIFS_INO_NODE_SZ.
 	 */
-	xlen = UBIFS_DENT_NODE_SZ + nm->len + 1;
+	xlen = UBIFS_DENT_NODE_SZ + fname_len(nm) + 1;
 	aligned_xlen = ALIGN(xlen, 8);
 	hlen = host_ui->data_len + UBIFS_INO_NODE_SZ;
 	len = aligned_xlen + UBIFS_INO_NODE_SZ + ALIGN(hlen, 8);
@@ -1482,9 +1514,9 @@ int ubifs_jnl_delete_xattr(struct ubifs_info *c, const struct inode *host,
 	key_write(c, &xent_key, xent->key);
 	xent->inum = 0;
 	xent->type = get_dent_type(inode->i_mode);
-	xent->nlen = cpu_to_le16(nm->len);
-	memcpy(xent->name, nm->name, nm->len);
-	xent->name[nm->len] = '\0';
+	xent->nlen = cpu_to_le16(fname_len(nm));
+	memcpy(xent->name, fname_name(nm), fname_len(nm));
+	xent->name[fname_len(nm)] = '\0';
 	zero_dent_node_unused(xent);
 	ubifs_prep_grp_node(c, xent, xlen, 0);
 
diff --git a/fs/ubifs/key.h b/fs/ubifs/key.h
index c0a95e393347..7547be512db2 100644
--- a/fs/ubifs/key.h
+++ b/fs/ubifs/key.h
@@ -69,7 +69,7 @@ static inline uint32_t key_r5_hash(const char *s, int len)
 	uint32_t a = 0;
 	const signed char *str = (const signed char *)s;
 
-	while (*str) {
+	while (len--) {
 		a += *str << 4;
 		a += *str >> 4;
 		a *= 11;
@@ -153,13 +153,13 @@ static inline void highest_ino_key(const struct ubifs_info *c,
  * @c: UBIFS file-system description object
  * @key: key to initialize
  * @inum: parent inode number
- * @nm: direntry name and length
+ * @nm: direntry name and length. Not a string when encrypted!
  */
 static inline void dent_key_init(const struct ubifs_info *c,
 				 union ubifs_key *key, ino_t inum,
-				 const struct qstr *nm)
+				 const struct fscrypt_name *nm)
 {
-	uint32_t hash = c->key_hash(nm->name, nm->len);
+	uint32_t hash = c->key_hash(fname_name(nm), fname_len(nm));
 
 	ubifs_assert(!(hash & ~UBIFS_S_KEY_HASH_MASK));
 	key->u32[0] = inum;
@@ -191,10 +191,11 @@ static inline void dent_key_init_hash(const struct ubifs_info *c,
  * @nm: direntry name and length
  */
 static inline void dent_key_init_flash(const struct ubifs_info *c, void *k,
-				       ino_t inum, const struct qstr *nm)
+				       ino_t inum,
+				       const struct fscrypt_name *nm)
 {
 	union ubifs_key *key = k;
-	uint32_t hash = c->key_hash(nm->name, nm->len);
+	uint32_t hash = c->key_hash(fname_name(nm), fname_len(nm));
 
 	ubifs_assert(!(hash & ~UBIFS_S_KEY_HASH_MASK));
 	key->j32[0] = cpu_to_le32(inum);
@@ -225,9 +226,9 @@ static inline void lowest_dent_key(const struct ubifs_info *c,
  */
 static inline void xent_key_init(const struct ubifs_info *c,
 				 union ubifs_key *key, ino_t inum,
-				 const struct qstr *nm)
+				 const struct fscrypt_name *nm)
 {
-	uint32_t hash = c->key_hash(nm->name, nm->len);
+	uint32_t hash = c->key_hash(fname_name(nm), fname_len(nm));
 
 	ubifs_assert(!(hash & ~UBIFS_S_KEY_HASH_MASK));
 	key->u32[0] = inum;
@@ -242,10 +243,10 @@ static inline void xent_key_init(const struct ubifs_info *c,
  * @nm: extended attribute entry name and length
  */
 static inline void xent_key_init_flash(const struct ubifs_info *c, void *k,
-				       ino_t inum, const struct qstr *nm)
+				       ino_t inum, const struct fscrypt_name *nm)
 {
 	union ubifs_key *key = k;
-	uint32_t hash = c->key_hash(nm->name, nm->len);
+	uint32_t hash = c->key_hash(fname_name(nm), fname_len(nm));
 
 	ubifs_assert(!(hash & ~UBIFS_S_KEY_HASH_MASK));
 	key->j32[0] = cpu_to_le32(inum);
diff --git a/fs/ubifs/replay.c b/fs/ubifs/replay.c
index fb0f44cd1e28..ae5c02f22f3e 100644
--- a/fs/ubifs/replay.c
+++ b/fs/ubifs/replay.c
@@ -61,7 +61,7 @@ struct replay_entry {
 	struct list_head list;
 	union ubifs_key key;
 	union {
-		struct qstr nm;
+		struct fscrypt_name nm;
 		struct {
 			loff_t old_size;
 			loff_t new_size;
@@ -327,7 +327,7 @@ static void destroy_replay_list(struct ubifs_info *c)
 
 	list_for_each_entry_safe(r, tmp, &c->replay_list, list) {
 		if (is_hash_key(c, &r->key))
-			kfree(r->nm.name);
+			kfree(fname_name(&r->nm));
 		list_del(&r->list);
 		kfree(r);
 	}
@@ -430,10 +430,10 @@ static int insert_dent(struct ubifs_info *c, int lnum, int offs, int len,
 	r->deletion = !!deletion;
 	r->sqnum = sqnum;
 	key_copy(c, key, &r->key);
-	r->nm.len = nlen;
+	fname_len(&r->nm) = nlen;
 	memcpy(nbuf, name, nlen);
 	nbuf[nlen] = '\0';
-	r->nm.name = nbuf;
+	fname_name(&r->nm) = nbuf;
 
 	list_add_tail(&r->list, &c->replay_list);
 	return 0;
@@ -456,7 +456,7 @@ int ubifs_validate_entry(struct ubifs_info *c,
 	if (le32_to_cpu(dent->ch.len) != nlen + UBIFS_DENT_NODE_SZ + 1 ||
 	    dent->type >= UBIFS_ITYPES_CNT ||
 	    nlen > UBIFS_MAX_NLEN || dent->name[nlen] != 0 ||
-	    strnlen(dent->name, nlen) != nlen ||
+	    (key_type == UBIFS_XENT_KEY && strnlen(dent->name, nlen) != nlen) ||
 	    le64_to_cpu(dent->inum) > MAX_INUM) {
 		ubifs_err(c, "bad %s node", key_type == UBIFS_DENT_KEY ?
 			  "directory entry" : "extended attribute entry");
diff --git a/fs/ubifs/sb.c b/fs/ubifs/sb.c
index 3cbb904a6d7d..7f1ead29e727 100644
--- a/fs/ubifs/sb.c
+++ b/fs/ubifs/sb.c
@@ -163,6 +163,7 @@ static int create_default_filesystem(struct ubifs_info *c)
 	tmp64 = (long long)max_buds * c->leb_size;
 	if (big_lpt)
 		sup_flags |= UBIFS_FLG_BIGLPT;
+	sup_flags |= UBIFS_FLG_DOUBLE_HASH;
 
 	sup->ch.node_type  = UBIFS_SB_NODE;
 	sup->key_hash      = UBIFS_KEY_HASH_R5;
@@ -465,6 +466,16 @@ static int validate_sb(struct ubifs_info *c, struct ubifs_sb_node *sup)
 		goto failed;
 	}
 
+	if (!c->double_hash && c->fmt_version >= 5) {
+		err = 16;
+		goto failed;
+	}
+
+	if (c->encrypted && c->fmt_version < 5) {
+		err = 17;
+		goto failed;
+	}
+
 	return 0;
 
 failed:
@@ -620,6 +631,24 @@ int ubifs_read_superblock(struct ubifs_info *c)
 	memcpy(&c->uuid, &sup->uuid, 16);
 	c->big_lpt = !!(sup_flags & UBIFS_FLG_BIGLPT);
 	c->space_fixup = !!(sup_flags & UBIFS_FLG_SPACE_FIXUP);
+	c->double_hash = !!(sup_flags & UBIFS_FLG_DOUBLE_HASH);
+	c->encrypted = !!(sup_flags & UBIFS_FLG_ENCRYPTION);
+
+	if ((sup_flags & ~UBIFS_FLG_MASK) != 0) {
+		ubifs_err(c, "Unknown feature flags found: %#x",
+			  sup_flags & ~UBIFS_FLG_MASK);
+		err = -EINVAL;
+		goto out;
+	}
+
+#ifndef CONFIG_UBIFS_FS_ENCRYPTION
+	if (c->encrypted) {
+		ubifs_err(c, "file system contains encrypted files but UBIFS"
+			     " was built without crypto support.");
+		err = -EINVAL;
+		goto out;
+	}
+#endif
 
 	/* Automatically increase file system size to the maximum size */
 	c->old_leb_cnt = c->leb_cnt;
@@ -807,3 +836,33 @@ int ubifs_fixup_free_space(struct ubifs_info *c)
 	ubifs_msg(c, "free space fixup complete");
 	return err;
 }
+
+int ubifs_enable_encryption(struct ubifs_info *c)
+{
+	int err;
+	struct ubifs_sb_node *sup;
+
+	if (c->encrypted)
+		return 0;
+
+	if (c->ro_mount || c->ro_media)
+		return -EROFS;
+
+	if (c->fmt_version < 5) {
+		ubifs_err(c, "on-flash format version 5 is needed for encryption");
+		return -EINVAL;
+	}
+
+	sup = ubifs_read_sb_node(c);
+	if (IS_ERR(sup))
+		return PTR_ERR(sup);
+
+	sup->flags |= cpu_to_le32(UBIFS_FLG_ENCRYPTION);
+
+	err = ubifs_write_sb_node(c, sup);
+	if (!err)
+		c->encrypted = 1;
+	kfree(sup);
+
+	return err;
+}
diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c
index 4ec051089186..e08aa04fc835 100644
--- a/fs/ubifs/super.c
+++ b/fs/ubifs/super.c
@@ -198,7 +198,6 @@ struct inode *ubifs_iget(struct super_block *sb, unsigned long inum)
 		}
 		memcpy(ui->data, ino->data, ui->data_len);
 		((char *)ui->data)[ui->data_len] = '\0';
-		inode->i_link = ui->data;
 		break;
 	case S_IFBLK:
 	case S_IFCHR:
@@ -380,6 +379,9 @@ out:
 	}
 done:
 	clear_inode(inode);
+#ifdef CONFIG_UBIFS_FS_ENCRYPTION
+	fscrypt_put_encryption_info(inode, NULL);
+#endif
 }
 
 static void ubifs_dirty_inode(struct inode *inode, int flags)
@@ -1207,7 +1209,8 @@ static int mount_ubifs(struct ubifs_info *c)
 		bu_init(c);
 
 	if (!c->ro_mount) {
-		c->write_reserve_buf = kmalloc(COMPRESSED_DATA_NODE_BUF_SZ,
+		c->write_reserve_buf = kmalloc(COMPRESSED_DATA_NODE_BUF_SZ + \
+					       UBIFS_CIPHER_BLOCK_SIZE,
 					       GFP_KERNEL);
 		if (!c->write_reserve_buf)
 			goto out_free;
@@ -1620,7 +1623,8 @@ static int ubifs_remount_rw(struct ubifs_info *c)
 		goto out;
 	}
 
-	c->write_reserve_buf = kmalloc(COMPRESSED_DATA_NODE_BUF_SZ, GFP_KERNEL);
+	c->write_reserve_buf = kmalloc(COMPRESSED_DATA_NODE_BUF_SZ + \
+				       UBIFS_CIPHER_BLOCK_SIZE, GFP_KERNEL);
 	if (!c->write_reserve_buf) {
 		err = -ENOMEM;
 		goto out;
@@ -1995,6 +1999,12 @@ static struct ubifs_info *alloc_ubifs_info(struct ubi_volume_desc *ubi)
 	return c;
 }
 
+#ifndef CONFIG_UBIFS_FS_ENCRYPTION
+struct fscrypt_operations ubifs_crypt_operations = {
+	.is_encrypted		= __ubifs_crypt_is_encrypted,
+};
+#endif
+
 static int ubifs_fill_super(struct super_block *sb, void *data, int silent)
 {
 	struct ubifs_info *c = sb->s_fs_info;
@@ -2041,6 +2051,7 @@ static int ubifs_fill_super(struct super_block *sb, void *data, int silent)
 		sb->s_maxbytes = c->max_inode_sz = MAX_LFS_FILESIZE;
 	sb->s_op = &ubifs_super_operations;
 	sb->s_xattr = ubifs_xattr_handlers;
+	sb->s_cop = &ubifs_crypt_operations;
 
 	mutex_lock(&c->umount_mutex);
 	err = mount_ubifs(c);
diff --git a/fs/ubifs/tnc.c b/fs/ubifs/tnc.c
index fa9a20cc60d6..74ae2de949df 100644
--- a/fs/ubifs/tnc.c
+++ b/fs/ubifs/tnc.c
@@ -378,7 +378,7 @@ static void lnc_free(struct ubifs_zbranch *zbr)
 }
 
 /**
- * tnc_read_node_nm - read a "hashed" leaf node.
+ * tnc_read_hashed_node - read a "hashed" leaf node.
  * @c: UBIFS file-system description object
  * @zbr: key and position of the node
  * @node: node is returned here
@@ -388,8 +388,8 @@ static void lnc_free(struct ubifs_zbranch *zbr)
  * added to LNC. Returns zero in case of success or a negative negative error
  * code in case of failure.
  */
-static int tnc_read_node_nm(struct ubifs_info *c, struct ubifs_zbranch *zbr,
-			    void *node)
+static int tnc_read_hashed_node(struct ubifs_info *c, struct ubifs_zbranch *zbr,
+				void *node)
 {
 	int err;
 
@@ -519,7 +519,7 @@ static int fallible_read_node(struct ubifs_info *c, const union ubifs_key *key,
  * of failure, a negative error code is returned.
  */
 static int matches_name(struct ubifs_info *c, struct ubifs_zbranch *zbr,
-			const struct qstr *nm)
+			const struct fscrypt_name *nm)
 {
 	struct ubifs_dent_node *dent;
 	int nlen, err;
@@ -542,11 +542,11 @@ static int matches_name(struct ubifs_info *c, struct ubifs_zbranch *zbr,
 		dent = zbr->leaf;
 
 	nlen = le16_to_cpu(dent->nlen);
-	err = memcmp(dent->name, nm->name, min_t(int, nlen, nm->len));
+	err = memcmp(dent->name, fname_name(nm), min_t(int, nlen, fname_len(nm)));
 	if (err == 0) {
-		if (nlen == nm->len)
+		if (nlen == fname_len(nm))
 			return NAME_MATCHES;
-		else if (nlen < nm->len)
+		else if (nlen < fname_len(nm))
 			return NAME_LESS;
 		else
 			return NAME_GREATER;
@@ -689,7 +689,7 @@ static int tnc_prev(struct ubifs_info *c, struct ubifs_znode **zn, int *n)
  */
 static int resolve_collision(struct ubifs_info *c, const union ubifs_key *key,
 			     struct ubifs_znode **zn, int *n,
-			     const struct qstr *nm)
+			     const struct fscrypt_name *nm)
 {
 	int err;
 
@@ -807,7 +807,7 @@ static int resolve_collision(struct ubifs_info *c, const union ubifs_key *key,
  */
 static int fallible_matches_name(struct ubifs_info *c,
 				 struct ubifs_zbranch *zbr,
-				 const struct qstr *nm)
+				 const struct fscrypt_name *nm)
 {
 	struct ubifs_dent_node *dent;
 	int nlen, err;
@@ -835,11 +835,11 @@ static int fallible_matches_name(struct ubifs_info *c,
 		dent = zbr->leaf;
 
 	nlen = le16_to_cpu(dent->nlen);
-	err = memcmp(dent->name, nm->name, min_t(int, nlen, nm->len));
+	err = memcmp(dent->name, fname_name(nm), min_t(int, nlen, fname_len(nm)));
 	if (err == 0) {
-		if (nlen == nm->len)
+		if (nlen == fname_len(nm))
 			return NAME_MATCHES;
-		else if (nlen < nm->len)
+		else if (nlen < fname_len(nm))
 			return NAME_LESS;
 		else
 			return NAME_GREATER;
@@ -878,7 +878,8 @@ out_free:
 static int fallible_resolve_collision(struct ubifs_info *c,
 				      const union ubifs_key *key,
 				      struct ubifs_znode **zn, int *n,
-				      const struct qstr *nm, int adding)
+				      const struct fscrypt_name *nm,
+				      int adding)
 {
 	struct ubifs_znode *o_znode = NULL, *znode = *zn;
 	int uninitialized_var(o_n), err, cmp, unsure = 0, nn = *n;
@@ -1453,7 +1454,7 @@ again:
 		 * In this case the leaf node cache gets used, so we pass the
 		 * address of the zbranch and keep the mutex locked
 		 */
-		err = tnc_read_node_nm(c, zt, node);
+		err = tnc_read_hashed_node(c, zt, node);
 		goto out;
 	}
 	if (safely) {
@@ -1782,19 +1783,19 @@ int ubifs_tnc_bulk_read(struct ubifs_info *c, struct bu_info *bu)
  * @node: the node is returned here
  * @nm: node name
  *
- * This function look up and reads a node which contains name hash in the key.
+ * This function looks up and reads a node which contains name hash in the key.
  * Since the hash may have collisions, there may be many nodes with the same
  * key, so we have to sequentially look to all of them until the needed one is
  * found. This function returns zero in case of success, %-ENOENT if the node
  * was not found, and a negative error code in case of failure.
  */
 static int do_lookup_nm(struct ubifs_info *c, const union ubifs_key *key,
-			void *node, const struct qstr *nm)
+			void *node, const struct fscrypt_name *nm)
 {
 	int found, n, err;
 	struct ubifs_znode *znode;
 
-	dbg_tnck(key, "name '%.*s' key ", nm->len, nm->name);
+	//dbg_tnck(key, "name '%.*s' key ", nm->len, nm->name);
 	mutex_lock(&c->tnc_mutex);
 	found = ubifs_lookup_level0(c, key, &znode, &n);
 	if (!found) {
@@ -1816,7 +1817,7 @@ static int do_lookup_nm(struct ubifs_info *c, const union ubifs_key *key,
 		goto out_unlock;
 	}
 
-	err = tnc_read_node_nm(c, &znode->zbranch[n], node);
+	err = tnc_read_hashed_node(c, &znode->zbranch[n], node);
 
 out_unlock:
 	mutex_unlock(&c->tnc_mutex);
@@ -1830,14 +1831,14 @@ out_unlock:
  * @node: the node is returned here
  * @nm: node name
  *
- * This function look up and reads a node which contains name hash in the key.
+ * This function looks up and reads a node which contains name hash in the key.
  * Since the hash may have collisions, there may be many nodes with the same
  * key, so we have to sequentially look to all of them until the needed one is
  * found. This function returns zero in case of success, %-ENOENT if the node
  * was not found, and a negative error code in case of failure.
  */
 int ubifs_tnc_lookup_nm(struct ubifs_info *c, const union ubifs_key *key,
-			void *node, const struct qstr *nm)
+			void *node, const struct fscrypt_name *nm)
 {
 	int err, len;
 	const struct ubifs_dent_node *dent = node;
@@ -1851,16 +1852,105 @@ int ubifs_tnc_lookup_nm(struct ubifs_info *c, const union ubifs_key *key,
 		return err;
 
 	len = le16_to_cpu(dent->nlen);
-	if (nm->len == len && !memcmp(dent->name, nm->name, len))
+	if (fname_len(nm) == len && !memcmp(dent->name, fname_name(nm), len))
 		return 0;
 
 	/*
 	 * Unluckily, there are hash collisions and we have to iterate over
 	 * them look at each direntry with colliding name hash sequentially.
 	 */
+
 	return do_lookup_nm(c, key, node, nm);
 }
 
+static int do_lookup_dh(struct ubifs_info *c, const union ubifs_key *key,
+			struct ubifs_dent_node *dent, uint32_t cookie)
+{
+	int n, err, type = key_type(c, key);
+	struct ubifs_znode *znode;
+	struct ubifs_zbranch *zbr;
+	union ubifs_key *dkey, start_key;
+
+	ubifs_assert(is_hash_key(c, key));
+
+	lowest_dent_key(c, &start_key, key_inum(c, key));
+
+	mutex_lock(&c->tnc_mutex);
+	err = ubifs_lookup_level0(c, &start_key, &znode, &n);
+	if (unlikely(err < 0))
+		goto out_unlock;
+
+	for (;;) {
+		if (!err) {
+			err = tnc_next(c, &znode, &n);
+			if (err)
+				goto out_unlock;
+		}
+
+		zbr = &znode->zbranch[n];
+		dkey = &zbr->key;
+
+		if (key_inum(c, dkey) != key_inum(c, key) ||
+		    key_type(c, dkey) != type) {
+			err = -ENOENT;
+			goto out_unlock;
+		}
+
+		err = tnc_read_hashed_node(c, zbr, dent);
+		if (err)
+			goto out_unlock;
+
+		if (key_hash(c, key) == key_hash(c, dkey) &&
+		    le32_to_cpu(dent->cookie) == cookie)
+			goto out_unlock;
+	}
+
+out_unlock:
+	mutex_unlock(&c->tnc_mutex);
+	return err;
+}
+
+/**
+ * ubifs_tnc_lookup_dh - look up a "double hashed" node.
+ * @c: UBIFS file-system description object
+ * @key: node key to lookup
+ * @node: the node is returned here
+ * @cookie: node cookie for collision resolution
+ *
+ * This function looks up and reads a node which contains name hash in the key.
+ * Since the hash may have collisions, there may be many nodes with the same
+ * key, so we have to sequentially look to all of them until the needed one
+ * with the same cookie value is found.
+ * This function returns zero in case of success, %-ENOENT if the node
+ * was not found, and a negative error code in case of failure.
+ */
+int ubifs_tnc_lookup_dh(struct ubifs_info *c, const union ubifs_key *key,
+			void *node, uint32_t cookie)
+{
+	int err;
+	const struct ubifs_dent_node *dent = node;
+
+	if (!c->double_hash)
+		return -EOPNOTSUPP;
+
+	/*
+	 * We assume that in most of the cases there are no name collisions and
+	 * 'ubifs_tnc_lookup()' returns us the right direntry.
+	 */
+	err = ubifs_tnc_lookup(c, key, node);
+	if (err)
+		return err;
+
+	if (le32_to_cpu(dent->cookie) == cookie)
+		return 0;
+
+	/*
+	 * Unluckily, there are hash collisions and we have to iterate over
+	 * them look at each direntry with colliding name hash sequentially.
+	 */
+	return do_lookup_dh(c, key, node, cookie);
+}
+
 /**
  * correct_parent_keys - correct parent znodes' keys.
  * @c: UBIFS file-system description object
@@ -2279,14 +2369,15 @@ out_unlock:
  * may have collisions, like directory entry keys.
  */
 int ubifs_tnc_add_nm(struct ubifs_info *c, const union ubifs_key *key,
-		     int lnum, int offs, int len, const struct qstr *nm)
+		     int lnum, int offs, int len,
+		     const struct fscrypt_name *nm)
 {
 	int found, n, err = 0;
 	struct ubifs_znode *znode;
 
 	mutex_lock(&c->tnc_mutex);
-	dbg_tnck(key, "LEB %d:%d, name '%.*s', key ",
-		 lnum, offs, nm->len, nm->name);
+	//dbg_tnck(key, "LEB %d:%d, name '%.*s', key ",
+	//	 lnum, offs, nm->len, nm->name);
 	found = lookup_level0_dirty(c, key, &znode, &n);
 	if (found < 0) {
 		err = found;
@@ -2344,7 +2435,7 @@ int ubifs_tnc_add_nm(struct ubifs_info *c, const union ubifs_key *key,
 			 * by passing 'ubifs_tnc_remove_nm()' the same key but
 			 * an unmatchable name.
 			 */
-			struct qstr noname = { .name = "" };
+			struct fscrypt_name noname = { .disk_name = { .name = "", .len = 1 } };
 
 			err = dbg_check_tnc(c, 0);
 			mutex_unlock(&c->tnc_mutex);
@@ -2514,13 +2605,13 @@ out_unlock:
  * Returns %0 on success or negative error code on failure.
  */
 int ubifs_tnc_remove_nm(struct ubifs_info *c, const union ubifs_key *key,
-			const struct qstr *nm)
+			const struct fscrypt_name *nm)
 {
 	int n, err;
 	struct ubifs_znode *znode;
 
 	mutex_lock(&c->tnc_mutex);
-	dbg_tnck(key, "%.*s, key ", nm->len, nm->name);
+	//dbg_tnck(key, "%.*s, key ", nm->len, nm->name);
 	err = lookup_level0_dirty(c, key, &znode, &n);
 	if (err < 0)
 		goto out_unlock;
@@ -2669,7 +2760,7 @@ int ubifs_tnc_remove_ino(struct ubifs_info *c, ino_t inum)
 {
 	union ubifs_key key1, key2;
 	struct ubifs_dent_node *xent, *pxent = NULL;
-	struct qstr nm = { .name = NULL };
+	struct fscrypt_name nm = {0};
 
 	dbg_tnc("ino %lu", (unsigned long)inum);
 
@@ -2694,8 +2785,8 @@ int ubifs_tnc_remove_ino(struct ubifs_info *c, ino_t inum)
 		dbg_tnc("xent '%s', ino %lu", xent->name,
 			(unsigned long)xattr_inum);
 
-		nm.name = xent->name;
-		nm.len = le16_to_cpu(xent->nlen);
+		fname_name(&nm) = xent->name;
+		fname_len(&nm) = le16_to_cpu(xent->nlen);
 		err = ubifs_tnc_remove_nm(c, &key1, &nm);
 		if (err) {
 			kfree(xent);
@@ -2747,7 +2838,7 @@ int ubifs_tnc_remove_ino(struct ubifs_info *c, ino_t inum)
  */
 struct ubifs_dent_node *ubifs_tnc_next_ent(struct ubifs_info *c,
 					   union ubifs_key *key,
-					   const struct qstr *nm)
+					   const struct fscrypt_name *nm)
 {
 	int n, err, type = key_type(c, key);
 	struct ubifs_znode *znode;
@@ -2755,7 +2846,7 @@ struct ubifs_dent_node *ubifs_tnc_next_ent(struct ubifs_info *c,
 	struct ubifs_zbranch *zbr;
 	union ubifs_key *dkey;
 
-	dbg_tnck(key, "%s ", nm->name ? (char *)nm->name : "(lowest)");
+	//dbg_tnck(key, "%s ", nm->name ? (char *)nm->name : "(lowest)");
 	ubifs_assert(is_hash_key(c, key));
 
 	mutex_lock(&c->tnc_mutex);
@@ -2763,7 +2854,7 @@ struct ubifs_dent_node *ubifs_tnc_next_ent(struct ubifs_info *c,
 	if (unlikely(err < 0))
 		goto out_unlock;
 
-	if (nm->name) {
+	if (fname_len(nm) > 0) {
 		if (err) {
 			/* Handle collisions */
 			err = resolve_collision(c, key, &znode, &n, nm);
@@ -2813,7 +2904,7 @@ struct ubifs_dent_node *ubifs_tnc_next_ent(struct ubifs_info *c,
 		goto out_free;
 	}
 
-	err = tnc_read_node_nm(c, zbr, dent);
+	err = tnc_read_hashed_node(c, zbr, dent);
 	if (unlikely(err))
 		goto out_free;
 
diff --git a/fs/ubifs/ubifs-media.h b/fs/ubifs/ubifs-media.h
index e24380cf46ed..e8c23c9d4f4a 100644
--- a/fs/ubifs/ubifs-media.h
+++ b/fs/ubifs/ubifs-media.h
@@ -46,7 +46,7 @@
  * UBIFS went into mainline kernel with format version 4. The older formats
  * were development formats.
  */
-#define UBIFS_FORMAT_VERSION 4
+#define UBIFS_FORMAT_VERSION 5
 
 /*
  * Read-only compatibility version. If the UBIFS format is changed, older UBIFS
@@ -301,6 +301,13 @@ enum {
 #define UBIFS_MAX_NODE_SZ UBIFS_MAX_INO_NODE_SZ
 
 /*
+ * xattr name of UBIFS encryption context, we don't use a prefix
+ * nor a long name to not waste space on the flash.
+ */
+#define UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT "c"
+
+
+/*
  * On-flash inode flags.
  *
  * UBIFS_COMPR_FL: use compression for this inode
@@ -309,6 +316,7 @@ enum {
  * UBIFS_APPEND_FL: writes to the inode may only append data
  * UBIFS_DIRSYNC_FL: I/O on this directory inode has to be synchronous
  * UBIFS_XATTR_FL: this inode is the inode for an extended attribute value
+ * UBIFS_CRYPT_FL: use encryption for this inode
  *
  * Note, these are on-flash flags which correspond to ioctl flags
  * (@FS_COMPR_FL, etc). They have the same values now, but generally, do not
@@ -321,6 +329,7 @@ enum {
 	UBIFS_APPEND_FL    = 0x08,
 	UBIFS_DIRSYNC_FL   = 0x10,
 	UBIFS_XATTR_FL     = 0x20,
+	UBIFS_CRYPT_FL     = 0x40,
 };
 
 /* Inode flag bits used by UBIFS */
@@ -409,12 +418,19 @@ enum {
  *
  * UBIFS_FLG_BIGLPT: if "big" LPT model is used if set
  * UBIFS_FLG_SPACE_FIXUP: first-mount "fixup" of free space within LEBs needed
+ * UBIFS_FLG_DOUBLE_HASH: store a 32bit cookie in directory entry nodes to
+ *			  support 64bit cookies for lookups by hash
+ * UBIFS_FLG_ENCRYPTION: this filesystem contains encrypted files
  */
 enum {
 	UBIFS_FLG_BIGLPT = 0x02,
 	UBIFS_FLG_SPACE_FIXUP = 0x04,
+	UBIFS_FLG_DOUBLE_HASH = 0x08,
+	UBIFS_FLG_ENCRYPTION = 0x10,
 };
 
+#define UBIFS_FLG_MASK (UBIFS_FLG_BIGLPT|UBIFS_FLG_SPACE_FIXUP|UBIFS_FLG_DOUBLE_HASH|UBIFS_FLG_ENCRYPTION)
+
 /**
  * struct ubifs_ch - common header node.
  * @magic: UBIFS node magic number (%UBIFS_NODE_MAGIC)
@@ -521,7 +537,8 @@ struct ubifs_ino_node {
  * @padding1: reserved for future, zeroes
  * @type: type of the target inode (%UBIFS_ITYPE_REG, %UBIFS_ITYPE_DIR, etc)
  * @nlen: name length
- * @padding2: reserved for future, zeroes
+ * @cookie: A 32bits random number, used to construct a 64bits
+ *          identifier.
  * @name: zero-terminated name
  *
  * Note, do not forget to amend 'zero_dent_node_unused()' function when
@@ -534,7 +551,7 @@ struct ubifs_dent_node {
 	__u8 padding1;
 	__u8 type;
 	__le16 nlen;
-	__u8 padding2[4]; /* Watch 'zero_dent_node_unused()' if changing! */
+	__le32 cookie;
 	__u8 name[];
 } __packed;
 
@@ -544,18 +561,16 @@ struct ubifs_dent_node {
  * @key: node key
  * @size: uncompressed data size in bytes
  * @compr_type: compression type (%UBIFS_COMPR_NONE, %UBIFS_COMPR_LZO, etc)
- * @padding: reserved for future, zeroes
+ * @compr_size: compressed data size in bytes, only valid when data is encrypted
  * @data: data
  *
- * Note, do not forget to amend 'zero_data_node_unused()' function when
- * changing the padding fields.
  */
 struct ubifs_data_node {
 	struct ubifs_ch ch;
 	__u8 key[UBIFS_MAX_KEY_LEN];
 	__le32 size;
 	__le16 compr_type;
-	__u8 padding[2]; /* Watch 'zero_data_node_unused()' if changing! */
+	__le16 compr_size;
 	__u8 data[];
 } __packed;
 
diff --git a/fs/ubifs/ubifs.h b/fs/ubifs/ubifs.h
index 096035eb29d0..ca72382ce6cc 100644
--- a/fs/ubifs/ubifs.h
+++ b/fs/ubifs/ubifs.h
@@ -38,6 +38,8 @@
 #include <linux/backing-dev.h>
 #include <linux/security.h>
 #include <linux/xattr.h>
+#include <linux/fscrypto.h>
+#include <linux/random.h>
 #include "ubifs-media.h"
 
 /* Version of this UBIFS implementation */
@@ -83,10 +85,6 @@
  */
 #define BGT_NAME_PATTERN "ubifs_bgt%d_%d"
 
-/* Write-buffer synchronization timeout interval in seconds */
-#define WBUF_TIMEOUT_SOFTLIMIT 3
-#define WBUF_TIMEOUT_HARDLIMIT 5
-
 /* Maximum possible inode number (only 32-bit inodes are supported now) */
 #define MAX_INUM 0xFFFFFFFF
 
@@ -138,6 +136,12 @@
  */
 #define WORST_COMPR_FACTOR 2
 
+#ifdef CONFIG_UBIFS_FS_ENCRYPTION
+#define UBIFS_CIPHER_BLOCK_SIZE FS_CRYPTO_BLOCK_SIZE
+#else
+#define UBIFS_CIPHER_BLOCK_SIZE 0
+#endif
+
 /*
  * How much memory is needed for a buffer where we compress a data node.
  */
@@ -645,9 +649,6 @@ typedef int (*ubifs_lpt_scan_callback)(struct ubifs_info *c,
  * @io_mutex: serializes write-buffer I/O
  * @lock: serializes @buf, @lnum, @offs, @avail, @used, @next_ino and @inodes
  *        fields
- * @softlimit: soft write-buffer timeout interval
- * @delta: hard and soft timeouts delta (the timer expire interval is @softlimit
- *         and @softlimit + @delta)
  * @timer: write-buffer timer
  * @no_timer: non-zero if this write-buffer does not have a timer
  * @need_sync: non-zero if the timer expired and the wbuf needs sync'ing
@@ -676,8 +677,6 @@ struct ubifs_wbuf {
 	int (*sync_callback)(struct ubifs_info *c, int lnum, int free, int pad);
 	struct mutex io_mutex;
 	spinlock_t lock;
-	ktime_t softlimit;
-	unsigned long long delta;
 	struct hrtimer timer;
 	unsigned int no_timer:1;
 	unsigned int need_sync:1;
@@ -1007,6 +1006,8 @@ struct ubifs_debug_info;
  *
  * @big_lpt: flag that LPT is too big to write whole during commit
  * @space_fixup: flag indicating that free space in LEBs needs to be cleaned up
+ * @double_hash: flag indicating that we can do lookups by hash
+ * @encrypted: flag indicating that this file system contains encrypted files
  * @no_chk_data_crc: do not check CRCs when reading data nodes (except during
  *                   recovery)
  * @bulk_read: enable bulk-reads
@@ -1249,6 +1250,8 @@ struct ubifs_info {
 
 	unsigned int big_lpt:1;
 	unsigned int space_fixup:1;
+	unsigned int double_hash:1;
+	unsigned int encrypted:1;
 	unsigned int no_chk_data_crc:1;
 	unsigned int bulk_read:1;
 	unsigned int default_compr:2;
@@ -1515,25 +1518,29 @@ int ubifs_consolidate_log(struct ubifs_info *c);
 
 /* journal.c */
 int ubifs_jnl_update(struct ubifs_info *c, const struct inode *dir,
-		     const struct qstr *nm, const struct inode *inode,
+		     const struct fscrypt_name *nm, const struct inode *inode,
 		     int deletion, int xent);
 int ubifs_jnl_write_data(struct ubifs_info *c, const struct inode *inode,
 			 const union ubifs_key *key, const void *buf, int len);
 int ubifs_jnl_write_inode(struct ubifs_info *c, const struct inode *inode);
 int ubifs_jnl_delete_inode(struct ubifs_info *c, const struct inode *inode);
 int ubifs_jnl_xrename(struct ubifs_info *c, const struct inode *fst_dir,
-		      const struct dentry *fst_dentry,
+		      const struct inode *fst_inode,
+		      const struct fscrypt_name *fst_nm,
 		      const struct inode *snd_dir,
-		      const struct dentry *snd_dentry, int sync);
+		      const struct inode *snd_inode,
+		      const struct fscrypt_name *snd_nm, int sync);
 int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir,
-		     const struct dentry *old_dentry,
+		     const struct inode *old_inode,
+		     const struct fscrypt_name *old_nm,
 		     const struct inode *new_dir,
-		     const struct dentry *new_dentry,
+		     const struct inode *new_inode,
+		     const struct fscrypt_name *new_nm,
 		     const struct inode *whiteout, int sync);
 int ubifs_jnl_truncate(struct ubifs_info *c, const struct inode *inode,
 		       loff_t old_size, loff_t new_size);
 int ubifs_jnl_delete_xattr(struct ubifs_info *c, const struct inode *host,
-			   const struct inode *inode, const struct qstr *nm);
+			   const struct inode *inode, const struct fscrypt_name *nm);
 int ubifs_jnl_change_xattr(struct ubifs_info *c, const struct inode *inode1,
 			   const struct inode *inode2);
 
@@ -1568,7 +1575,9 @@ int ubifs_save_dirty_idx_lnums(struct ubifs_info *c);
 int ubifs_lookup_level0(struct ubifs_info *c, const union ubifs_key *key,
 			struct ubifs_znode **zn, int *n);
 int ubifs_tnc_lookup_nm(struct ubifs_info *c, const union ubifs_key *key,
-			void *node, const struct qstr *nm);
+			void *node, const struct fscrypt_name *nm);
+int ubifs_tnc_lookup_dh(struct ubifs_info *c, const union ubifs_key *key,
+			void *node, uint32_t secondary_hash);
 int ubifs_tnc_locate(struct ubifs_info *c, const union ubifs_key *key,
 		     void *node, int *lnum, int *offs);
 int ubifs_tnc_add(struct ubifs_info *c, const union ubifs_key *key, int lnum,
@@ -1576,16 +1585,16 @@ int ubifs_tnc_add(struct ubifs_info *c, const union ubifs_key *key, int lnum,
 int ubifs_tnc_replace(struct ubifs_info *c, const union ubifs_key *key,
 		      int old_lnum, int old_offs, int lnum, int offs, int len);
 int ubifs_tnc_add_nm(struct ubifs_info *c, const union ubifs_key *key,
-		     int lnum, int offs, int len, const struct qstr *nm);
+		     int lnum, int offs, int len, const struct fscrypt_name *nm);
 int ubifs_tnc_remove(struct ubifs_info *c, const union ubifs_key *key);
 int ubifs_tnc_remove_nm(struct ubifs_info *c, const union ubifs_key *key,
-			const struct qstr *nm);
+			const struct fscrypt_name *nm);
 int ubifs_tnc_remove_range(struct ubifs_info *c, union ubifs_key *from_key,
 			   union ubifs_key *to_key);
 int ubifs_tnc_remove_ino(struct ubifs_info *c, ino_t inum);
 struct ubifs_dent_node *ubifs_tnc_next_ent(struct ubifs_info *c,
 					   union ubifs_key *key,
-					   const struct qstr *nm);
+					   const struct fscrypt_name *nm);
 void ubifs_tnc_close(struct ubifs_info *c);
 int ubifs_tnc_has_node(struct ubifs_info *c, union ubifs_key *key, int level,
 		       int lnum, int offs, int is_idx);
@@ -1642,6 +1651,7 @@ int ubifs_read_superblock(struct ubifs_info *c);
 struct ubifs_sb_node *ubifs_read_sb_node(struct ubifs_info *c);
 int ubifs_write_sb_node(struct ubifs_info *c, struct ubifs_sb_node *sup);
 int ubifs_fixup_free_space(struct ubifs_info *c);
+int ubifs_enable_encryption(struct ubifs_info *c);
 
 /* replay.c */
 int ubifs_validate_entry(struct ubifs_info *c,
@@ -1733,16 +1743,21 @@ int ubifs_update_time(struct inode *inode, struct timespec *time, int flags);
 #endif
 
 /* dir.c */
-struct inode *ubifs_new_inode(struct ubifs_info *c, const struct inode *dir,
+struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir,
 			      umode_t mode);
 int ubifs_getattr(struct vfsmount *mnt, struct dentry *dentry,
 		  struct kstat *stat);
+int ubifs_check_dir_empty(struct inode *dir);
 
 /* xattr.c */
 extern const struct xattr_handler *ubifs_xattr_handlers[];
 ssize_t ubifs_listxattr(struct dentry *dentry, char *buffer, size_t size);
 int ubifs_init_security(struct inode *dentry, struct inode *inode,
 			const struct qstr *qstr);
+int ubifs_xattr_set(struct inode *host, const char *name, const void *value,
+		    size_t size, int flags);
+ssize_t ubifs_xattr_get(struct inode *host, const char *name, void *buf,
+			size_t size);
 
 /* super.c */
 struct inode *ubifs_iget(struct super_block *sb, unsigned long inum);
@@ -1781,6 +1796,66 @@ int ubifs_decompress(const struct ubifs_info *c, const void *buf, int len,
 #include "misc.h"
 #include "key.h"
 
+#ifndef CONFIG_UBIFS_FS_ENCRYPTION
+#define fscrypt_set_d_op(i)
+#define fscrypt_get_ctx                 fscrypt_notsupp_get_ctx
+#define fscrypt_release_ctx             fscrypt_notsupp_release_ctx
+#define fscrypt_encrypt_page            fscrypt_notsupp_encrypt_page
+#define fscrypt_decrypt_page            fscrypt_notsupp_decrypt_page
+#define fscrypt_decrypt_bio_pages       fscrypt_notsupp_decrypt_bio_pages
+#define fscrypt_pullback_bio_page       fscrypt_notsupp_pullback_bio_page
+#define fscrypt_restore_control_page    fscrypt_notsupp_restore_control_page
+#define fscrypt_zeroout_range           fscrypt_notsupp_zeroout_range
+#define fscrypt_ioctl_set_policy	fscrypt_notsupp_ioctl_set_policy
+#define fscrypt_ioctl_get_policy	fscrypt_notsupp_ioctl_get_policy
+#define fscrypt_has_permitted_context   fscrypt_notsupp_has_permitted_context
+#define fscrypt_inherit_context         fscrypt_notsupp_inherit_context
+#define fscrypt_get_encryption_info     fscrypt_notsupp_get_encryption_info
+#define fscrypt_put_encryption_info     fscrypt_notsupp_put_encryption_info
+#define fscrypt_setup_filename          fscrypt_notsupp_setup_filename
+#define fscrypt_free_filename           fscrypt_notsupp_free_filename
+#define fscrypt_fname_encrypted_size    fscrypt_notsupp_fname_encrypted_size
+#define fscrypt_fname_alloc_buffer      fscrypt_notsupp_fname_alloc_buffer
+#define fscrypt_fname_free_buffer       fscrypt_notsupp_fname_free_buffer
+#define fscrypt_fname_disk_to_usr       fscrypt_notsupp_fname_disk_to_usr
+#define fscrypt_fname_usr_to_disk       fscrypt_notsupp_fname_usr_to_disk
+static inline int ubifs_encrypt(const struct inode *inode,
+				struct ubifs_data_node *dn,
+				unsigned int in_len, unsigned int *out_len,
+				int block)
+{
+	ubifs_assert(0);
+	return -EOPNOTSUPP;
+}
+static inline int ubifs_decrypt(const struct inode *inode,
+				struct ubifs_data_node *dn,
+				unsigned int *out_len, int block)
+{
+	ubifs_assert(0);
+	return -EOPNOTSUPP;
+}
+#else
+/* crypto.c */
+int ubifs_encrypt(const struct inode *inode, struct ubifs_data_node *dn,
+		  unsigned int in_len, unsigned int *out_len, int block);
+int ubifs_decrypt(const struct inode *inode, struct ubifs_data_node *dn,
+		  unsigned int *out_len, int block);
+#endif
+
+extern struct fscrypt_operations ubifs_crypt_operations;
+
+static inline bool __ubifs_crypt_is_encrypted(struct inode *inode)
+{
+	struct ubifs_inode *ui = ubifs_inode(inode);
+
+	return ui->flags & UBIFS_CRYPT_FL;
+}
+
+static inline bool ubifs_crypt_is_encrypted(const struct inode *inode)
+{
+	return __ubifs_crypt_is_encrypted((struct inode *)inode);
+}
+
 /* Normal UBIFS messages */
 __printf(2, 3)
 void ubifs_msg(const struct ubifs_info *c, const char *fmt, ...);
diff --git a/fs/ubifs/xattr.c b/fs/ubifs/xattr.c
index d9f9615bfd71..efe00fcb8b75 100644
--- a/fs/ubifs/xattr.c
+++ b/fs/ubifs/xattr.c
@@ -97,7 +97,7 @@ static const struct file_operations empty_fops;
  * of failure.
  */
 static int create_xattr(struct ubifs_info *c, struct inode *host,
-			const struct qstr *nm, const void *value, int size)
+			const struct fscrypt_name *nm, const void *value, int size)
 {
 	int err, names_len;
 	struct inode *inode;
@@ -117,7 +117,7 @@ static int create_xattr(struct ubifs_info *c, struct inode *host,
 	 * extended attributes if the name list becomes larger. This limitation
 	 * is artificial for UBIFS, though.
 	 */
-	names_len = host_ui->xattr_names + host_ui->xattr_cnt + nm->len + 1;
+	names_len = host_ui->xattr_names + host_ui->xattr_cnt + fname_len(nm) + 1;
 	if (names_len > XATTR_LIST_MAX) {
 		ubifs_err(c, "cannot add one more xattr name to inode %lu, total names length would become %d, max. is %d",
 			  host->i_ino, names_len, XATTR_LIST_MAX);
@@ -154,9 +154,18 @@ static int create_xattr(struct ubifs_info *c, struct inode *host,
 	mutex_lock(&host_ui->ui_mutex);
 	host->i_ctime = ubifs_current_time(host);
 	host_ui->xattr_cnt += 1;
-	host_ui->xattr_size += CALC_DENT_SIZE(nm->len);
+	host_ui->xattr_size += CALC_DENT_SIZE(fname_len(nm));
 	host_ui->xattr_size += CALC_XATTR_BYTES(size);
-	host_ui->xattr_names += nm->len;
+	host_ui->xattr_names += fname_len(nm);
+
+	/*
+	 * We handle UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT here because we
+	 * have to set the UBIFS_CRYPT_FL flag on the host inode.
+	 * To avoid multiple updates of the same inode in the same operation,
+	 * let's do it here.
+	 */
+	if (strcmp(fname_name(nm), UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT) == 0)
+		host_ui->flags |= UBIFS_CRYPT_FL;
 
 	err = ubifs_jnl_update(c, host, nm, inode, 0, 1);
 	if (err)
@@ -170,9 +179,10 @@ static int create_xattr(struct ubifs_info *c, struct inode *host,
 
 out_cancel:
 	host_ui->xattr_cnt -= 1;
-	host_ui->xattr_size -= CALC_DENT_SIZE(nm->len);
+	host_ui->xattr_size -= CALC_DENT_SIZE(fname_len(nm));
 	host_ui->xattr_size -= CALC_XATTR_BYTES(size);
-	host_ui->xattr_names -= nm->len;
+	host_ui->xattr_names -= fname_len(nm);
+	host_ui->flags &= ~UBIFS_CRYPT_FL;
 	mutex_unlock(&host_ui->ui_mutex);
 out_free:
 	make_bad_inode(inode);
@@ -269,22 +279,28 @@ static struct inode *iget_xattr(struct ubifs_info *c, ino_t inum)
 	return ERR_PTR(-EINVAL);
 }
 
-static int __ubifs_setxattr(struct inode *host, const char *name,
-			    const void *value, size_t size, int flags)
+int ubifs_xattr_set(struct inode *host, const char *name, const void *value,
+		    size_t size, int flags)
 {
 	struct inode *inode;
 	struct ubifs_info *c = host->i_sb->s_fs_info;
-	struct qstr nm = QSTR_INIT(name, strlen(name));
+	struct fscrypt_name nm = { .disk_name = FSTR_INIT((char *)name, strlen(name))};
 	struct ubifs_dent_node *xent;
 	union ubifs_key key;
 	int err;
 
-	ubifs_assert(inode_is_locked(host));
+	/*
+	 * Creating an encryption context is done unlocked since we
+	 * operate on a new inode which is not visible to other users
+	 * at this point.
+	 */
+	if (strcmp(name, UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT) != 0)
+		ubifs_assert(inode_is_locked(host));
 
 	if (size > UBIFS_MAX_INO_DATA)
 		return -ERANGE;
 
-	if (nm.len > UBIFS_MAX_NLEN)
+	if (fname_len(&nm) > UBIFS_MAX_NLEN)
 		return -ENAMETOOLONG;
 
 	xent = kmalloc(UBIFS_MAX_XENT_NODE_SZ, GFP_NOFS);
@@ -329,18 +345,18 @@ out_free:
 	return err;
 }
 
-static ssize_t __ubifs_getxattr(struct inode *host, const char *name,
-				void *buf, size_t size)
+ssize_t ubifs_xattr_get(struct inode *host, const char *name, void *buf,
+			size_t size)
 {
 	struct inode *inode;
 	struct ubifs_info *c = host->i_sb->s_fs_info;
-	struct qstr nm = QSTR_INIT(name, strlen(name));
+	struct fscrypt_name nm = { .disk_name = FSTR_INIT((char *)name, strlen(name))};
 	struct ubifs_inode *ui;
 	struct ubifs_dent_node *xent;
 	union ubifs_key key;
 	int err;
 
-	if (nm.len > UBIFS_MAX_NLEN)
+	if (fname_len(&nm) > UBIFS_MAX_NLEN)
 		return -ENAMETOOLONG;
 
 	xent = kmalloc(UBIFS_MAX_XENT_NODE_SZ, GFP_NOFS);
@@ -387,6 +403,20 @@ out_unlock:
 	return err;
 }
 
+static bool xattr_visible(const char *name)
+{
+	/* File encryption related xattrs are for internal use only */
+	if (strcmp(name, UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT) == 0)
+		return false;
+
+	/* Show trusted namespace only for "power" users */
+	if (strncmp(name, XATTR_TRUSTED_PREFIX,
+		    XATTR_TRUSTED_PREFIX_LEN) == 0 && !capable(CAP_SYS_ADMIN))
+		return false;
+
+	return true;
+}
+
 ssize_t ubifs_listxattr(struct dentry *dentry, char *buffer, size_t size)
 {
 	union ubifs_key key;
@@ -395,7 +425,7 @@ ssize_t ubifs_listxattr(struct dentry *dentry, char *buffer, size_t size)
 	struct ubifs_inode *host_ui = ubifs_inode(host);
 	struct ubifs_dent_node *xent, *pxent = NULL;
 	int err, len, written = 0;
-	struct qstr nm = { .name = NULL };
+	struct fscrypt_name nm = {0};
 
 	dbg_gen("ino %lu ('%pd'), buffer size %zd", host->i_ino,
 		dentry, size);
@@ -419,15 +449,12 @@ ssize_t ubifs_listxattr(struct dentry *dentry, char *buffer, size_t size)
 			break;
 		}
 
-		nm.name = xent->name;
-		nm.len = le16_to_cpu(xent->nlen);
+		fname_name(&nm) = xent->name;
+		fname_len(&nm) = le16_to_cpu(xent->nlen);
 
-		/* Show trusted namespace only for "power" users */
-		if (strncmp(xent->name, XATTR_TRUSTED_PREFIX,
-			    XATTR_TRUSTED_PREFIX_LEN) ||
-		    capable(CAP_SYS_ADMIN)) {
-			memcpy(buffer + written, nm.name, nm.len + 1);
-			written += nm.len + 1;
+		if (xattr_visible(xent->name)) {
+			memcpy(buffer + written, fname_name(&nm), fname_len(&nm) + 1);
+			written += fname_len(&nm) + 1;
 		}
 
 		kfree(pxent);
@@ -446,7 +473,7 @@ ssize_t ubifs_listxattr(struct dentry *dentry, char *buffer, size_t size)
 }
 
 static int remove_xattr(struct ubifs_info *c, struct inode *host,
-			struct inode *inode, const struct qstr *nm)
+			struct inode *inode, const struct fscrypt_name *nm)
 {
 	int err;
 	struct ubifs_inode *host_ui = ubifs_inode(host);
@@ -463,9 +490,9 @@ static int remove_xattr(struct ubifs_info *c, struct inode *host,
 	mutex_lock(&host_ui->ui_mutex);
 	host->i_ctime = ubifs_current_time(host);
 	host_ui->xattr_cnt -= 1;
-	host_ui->xattr_size -= CALC_DENT_SIZE(nm->len);
+	host_ui->xattr_size -= CALC_DENT_SIZE(fname_len(nm));
 	host_ui->xattr_size -= CALC_XATTR_BYTES(ui->data_len);
-	host_ui->xattr_names -= nm->len;
+	host_ui->xattr_names -= fname_len(nm);
 
 	err = ubifs_jnl_delete_xattr(c, host, inode, nm);
 	if (err)
@@ -477,27 +504,27 @@ static int remove_xattr(struct ubifs_info *c, struct inode *host,
 
 out_cancel:
 	host_ui->xattr_cnt += 1;
-	host_ui->xattr_size += CALC_DENT_SIZE(nm->len);
+	host_ui->xattr_size += CALC_DENT_SIZE(fname_len(nm));
 	host_ui->xattr_size += CALC_XATTR_BYTES(ui->data_len);
-	host_ui->xattr_names += nm->len;
+	host_ui->xattr_names += fname_len(nm);
 	mutex_unlock(&host_ui->ui_mutex);
 	ubifs_release_budget(c, &req);
 	make_bad_inode(inode);
 	return err;
 }
 
-static int __ubifs_removexattr(struct inode *host, const char *name)
+static int ubifs_xattr_remove(struct inode *host, const char *name)
 {
 	struct inode *inode;
 	struct ubifs_info *c = host->i_sb->s_fs_info;
-	struct qstr nm = QSTR_INIT(name, strlen(name));
+	struct fscrypt_name nm = { .disk_name = FSTR_INIT((char *)name, strlen(name))};
 	struct ubifs_dent_node *xent;
 	union ubifs_key key;
 	int err;
 
 	ubifs_assert(inode_is_locked(host));
 
-	if (nm.len > UBIFS_MAX_NLEN)
+	if (fname_len(&nm) > UBIFS_MAX_NLEN)
 		return -ENAMETOOLONG;
 
 	xent = kmalloc(UBIFS_MAX_XENT_NODE_SZ, GFP_NOFS);
@@ -548,7 +575,8 @@ static int init_xattrs(struct inode *inode, const struct xattr *xattr_array,
 		}
 		strcpy(name, XATTR_SECURITY_PREFIX);
 		strcpy(name + XATTR_SECURITY_PREFIX_LEN, xattr->name);
-		err = __ubifs_setxattr(inode, name, xattr->value, xattr->value_len, 0);
+		err = ubifs_xattr_set(inode, name, xattr->value,
+				      xattr->value_len, 0);
 		kfree(name);
 		if (err < 0)
 			break;
@@ -572,7 +600,7 @@ int ubifs_init_security(struct inode *dentry, struct inode *inode,
 	return err;
 }
 
-static int ubifs_xattr_get(const struct xattr_handler *handler,
+static int xattr_get(const struct xattr_handler *handler,
 			   struct dentry *dentry, struct inode *inode,
 			   const char *name, void *buffer, size_t size)
 {
@@ -580,10 +608,10 @@ static int ubifs_xattr_get(const struct xattr_handler *handler,
 		inode->i_ino, dentry, size);
 
 	name = xattr_full_name(handler, name);
-	return __ubifs_getxattr(inode, name, buffer, size);
+	return ubifs_xattr_get(inode, name, buffer, size);
 }
 
-static int ubifs_xattr_set(const struct xattr_handler *handler,
+static int xattr_set(const struct xattr_handler *handler,
 			   struct dentry *dentry, struct inode *inode,
 			   const char *name, const void *value,
 			   size_t size, int flags)
@@ -594,27 +622,27 @@ static int ubifs_xattr_set(const struct xattr_handler *handler,
 	name = xattr_full_name(handler, name);
 
 	if (value)
-		return __ubifs_setxattr(inode, name, value, size, flags);
+		return ubifs_xattr_set(inode, name, value, size, flags);
 	else
-		return __ubifs_removexattr(inode, name);
+		return ubifs_xattr_remove(inode, name);
 }
 
 static const struct xattr_handler ubifs_user_xattr_handler = {
 	.prefix = XATTR_USER_PREFIX,
-	.get = ubifs_xattr_get,
-	.set = ubifs_xattr_set,
+	.get = xattr_get,
+	.set = xattr_set,
 };
 
 static const struct xattr_handler ubifs_trusted_xattr_handler = {
 	.prefix = XATTR_TRUSTED_PREFIX,
-	.get = ubifs_xattr_get,
-	.set = ubifs_xattr_set,
+	.get = xattr_get,
+	.set = xattr_set,
 };
 
 static const struct xattr_handler ubifs_security_xattr_handler = {
 	.prefix = XATTR_SECURITY_PREFIX,
-	.get = ubifs_xattr_get,
-	.set = ubifs_xattr_set,
+	.get = xattr_get,
+	.set = xattr_set,
 };
 
 const struct xattr_handler *ubifs_xattr_handlers[] = {
diff --git a/fs/udf/dir.c b/fs/udf/dir.c
index aaec13c95253..2d0e028067eb 100644
--- a/fs/udf/dir.c
+++ b/fs/udf/dir.c
@@ -30,6 +30,7 @@
 #include <linux/errno.h>
 #include <linux/mm.h>
 #include <linux/slab.h>
+#include <linux/bio.h>
 
 #include "udf_i.h"
 #include "udf_sb.h"
diff --git a/fs/udf/directory.c b/fs/udf/directory.c
index 988d5352bdb8..7aa48bd7cbaf 100644
--- a/fs/udf/directory.c
+++ b/fs/udf/directory.c
@@ -16,6 +16,7 @@
 
 #include <linux/fs.h>
 #include <linux/string.h>
+#include <linux/bio.h>
 
 struct fileIdentDesc *udf_fileident_read(struct inode *dir, loff_t *nf_pos,
 					 struct udf_fileident_bh *fibh,
diff --git a/fs/udf/inode.c b/fs/udf/inode.c
index aad46401ede5..0f3db71753aa 100644
--- a/fs/udf/inode.c
+++ b/fs/udf/inode.c
@@ -38,6 +38,7 @@
 #include <linux/crc-itu-t.h>
 #include <linux/mpage.h>
 #include <linux/uio.h>
+#include <linux/bio.h>
 
 #include "udf_i.h"
 #include "udf_sb.h"
diff --git a/fs/ufs/balloc.c b/fs/ufs/balloc.c
index 67e085d591d8..a0376a2c1c29 100644
--- a/fs/ufs/balloc.c
+++ b/fs/ufs/balloc.c
@@ -15,6 +15,7 @@
 #include <linux/buffer_head.h>
 #include <linux/capability.h>
 #include <linux/bitops.h>
+#include <linux/bio.h>
 #include <asm/byteorder.h>
 
 #include "ufs_fs.h"
@@ -306,8 +307,7 @@ static void ufs_change_blocknr(struct inode *inode, sector_t beg,
 			     (unsigned long long)(pos + newb), pos);
 
 			bh->b_blocknr = newb + pos;
-			unmap_underlying_metadata(bh->b_bdev,
-						  bh->b_blocknr);
+			clean_bdev_bh_alias(bh);
 			mark_buffer_dirty(bh);
 			++j;
 			bh = bh->b_this_page;
diff --git a/fs/ufs/inode.c b/fs/ufs/inode.c
index 190d64be22ed..45ceb94e89e4 100644
--- a/fs/ufs/inode.c
+++ b/fs/ufs/inode.c
@@ -1070,8 +1070,7 @@ static int ufs_alloc_lastblock(struct inode *inode, loff_t size)
 
        if (buffer_new(bh)) {
 	       clear_buffer_new(bh);
-	       unmap_underlying_metadata(bh->b_bdev,
-					 bh->b_blocknr);
+	       clean_bdev_bh_alias(bh);
 	       /*
 		* we do not zeroize fragment, because of
 		* if it maped to hole, it already contains zeroes
diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index 85959d8324df..d96e2f30084b 100644
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -257,9 +257,9 @@ out:
  * fatal_signal_pending()s, and the mmap_sem must be released before
  * returning it.
  */
-int handle_userfault(struct fault_env *fe, unsigned long reason)
+int handle_userfault(struct vm_fault *vmf, unsigned long reason)
 {
-	struct mm_struct *mm = fe->vma->vm_mm;
+	struct mm_struct *mm = vmf->vma->vm_mm;
 	struct userfaultfd_ctx *ctx;
 	struct userfaultfd_wait_queue uwq;
 	int ret;
@@ -268,7 +268,7 @@ int handle_userfault(struct fault_env *fe, unsigned long reason)
 	BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
 
 	ret = VM_FAULT_SIGBUS;
-	ctx = fe->vma->vm_userfaultfd_ctx.ctx;
+	ctx = vmf->vma->vm_userfaultfd_ctx.ctx;
 	if (!ctx)
 		goto out;
 
@@ -301,17 +301,18 @@ int handle_userfault(struct fault_env *fe, unsigned long reason)
 	 * without first stopping userland access to the memory. For
 	 * VM_UFFD_MISSING userfaults this is enough for now.
 	 */
-	if (unlikely(!(fe->flags & FAULT_FLAG_ALLOW_RETRY))) {
+	if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {
 		/*
 		 * Validate the invariant that nowait must allow retry
 		 * to be sure not to return SIGBUS erroneously on
 		 * nowait invocations.
 		 */
-		BUG_ON(fe->flags & FAULT_FLAG_RETRY_NOWAIT);
+		BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);
 #ifdef CONFIG_DEBUG_VM
 		if (printk_ratelimit()) {
 			printk(KERN_WARNING
-			       "FAULT_FLAG_ALLOW_RETRY missing %x\n", fe->flags);
+			       "FAULT_FLAG_ALLOW_RETRY missing %x\n",
+			       vmf->flags);
 			dump_stack();
 		}
 #endif
@@ -323,7 +324,7 @@ int handle_userfault(struct fault_env *fe, unsigned long reason)
 	 * and wait.
 	 */
 	ret = VM_FAULT_RETRY;
-	if (fe->flags & FAULT_FLAG_RETRY_NOWAIT)
+	if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
 		goto out;
 
 	/* take the reference before dropping the mmap_sem */
@@ -331,11 +332,11 @@ int handle_userfault(struct fault_env *fe, unsigned long reason)
 
 	init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
 	uwq.wq.private = current;
-	uwq.msg = userfault_msg(fe->address, fe->flags, reason);
+	uwq.msg = userfault_msg(vmf->address, vmf->flags, reason);
 	uwq.ctx = ctx;
 
 	return_to_userland =
-		(fe->flags & (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE)) ==
+		(vmf->flags & (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE)) ==
 		(FAULT_FLAG_USER|FAULT_FLAG_KILLABLE);
 
 	spin_lock(&ctx->fault_pending_wqh.lock);
@@ -353,7 +354,8 @@ int handle_userfault(struct fault_env *fe, unsigned long reason)
 			  TASK_KILLABLE);
 	spin_unlock(&ctx->fault_pending_wqh.lock);
 
-	must_wait = userfaultfd_must_wait(ctx, fe->address, fe->flags, reason);
+	must_wait = userfaultfd_must_wait(ctx, vmf->address, vmf->flags,
+					  reason);
 	up_read(&mm->mmap_sem);
 
 	if (likely(must_wait && !ACCESS_ONCE(ctx->released) &&
diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c
index effb64cf714f..5050056a0b06 100644
--- a/fs/xfs/libxfs/xfs_alloc.c
+++ b/fs/xfs/libxfs/xfs_alloc.c
@@ -2455,12 +2455,15 @@ xfs_agf_verify(
 	      be32_to_cpu(agf->agf_flcount) <= XFS_AGFL_SIZE(mp)))
 		return false;
 
-	if (be32_to_cpu(agf->agf_levels[XFS_BTNUM_BNO]) > XFS_BTREE_MAXLEVELS ||
+	if (be32_to_cpu(agf->agf_levels[XFS_BTNUM_BNO]) < 1 ||
+	    be32_to_cpu(agf->agf_levels[XFS_BTNUM_CNT]) < 1 ||
+	    be32_to_cpu(agf->agf_levels[XFS_BTNUM_BNO]) > XFS_BTREE_MAXLEVELS ||
 	    be32_to_cpu(agf->agf_levels[XFS_BTNUM_CNT]) > XFS_BTREE_MAXLEVELS)
 		return false;
 
 	if (xfs_sb_version_hasrmapbt(&mp->m_sb) &&
-	    be32_to_cpu(agf->agf_levels[XFS_BTNUM_RMAP]) > XFS_BTREE_MAXLEVELS)
+	    (be32_to_cpu(agf->agf_levels[XFS_BTNUM_RMAP]) < 1 ||
+	     be32_to_cpu(agf->agf_levels[XFS_BTNUM_RMAP]) > XFS_BTREE_MAXLEVELS))
 		return false;
 
 	/*
@@ -2477,7 +2480,8 @@ xfs_agf_verify(
 		return false;
 
 	if (xfs_sb_version_hasreflink(&mp->m_sb) &&
-	    be32_to_cpu(agf->agf_refcount_level) > XFS_BTREE_MAXLEVELS)
+	    (be32_to_cpu(agf->agf_refcount_level) < 1 ||
+	     be32_to_cpu(agf->agf_refcount_level) > XFS_BTREE_MAXLEVELS))
 		return false;
 
 	return true;;
diff --git a/fs/xfs/libxfs/xfs_alloc_btree.c b/fs/xfs/libxfs/xfs_alloc_btree.c
index 5ba2dac5e67c..efb467b10a71 100644
--- a/fs/xfs/libxfs/xfs_alloc_btree.c
+++ b/fs/xfs/libxfs/xfs_alloc_btree.c
@@ -421,13 +421,17 @@ xfs_allocbt_init_cursor(
 
 	ASSERT(btnum == XFS_BTNUM_BNO || btnum == XFS_BTNUM_CNT);
 
-	cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_SLEEP);
+	cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS);
 
 	cur->bc_tp = tp;
 	cur->bc_mp = mp;
 	cur->bc_btnum = btnum;
 	cur->bc_blocklog = mp->m_sb.sb_blocklog;
 	cur->bc_ops = &xfs_allocbt_ops;
+	if (btnum == XFS_BTNUM_BNO)
+		cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_abtb_2);
+	else
+		cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_abtc_2);
 
 	if (btnum == XFS_BTNUM_CNT) {
 		cur->bc_nlevels = be32_to_cpu(agf->agf_levels[XFS_BTNUM_CNT]);
diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c
index 8ea91f363093..2852521fc8ec 100644
--- a/fs/xfs/libxfs/xfs_attr_leaf.c
+++ b/fs/xfs/libxfs/xfs_attr_leaf.c
@@ -253,6 +253,7 @@ xfs_attr3_leaf_verify(
 {
 	struct xfs_mount	*mp = bp->b_target->bt_mount;
 	struct xfs_attr_leafblock *leaf = bp->b_addr;
+	struct xfs_perag *pag = bp->b_pag;
 	struct xfs_attr3_icleaf_hdr ichdr;
 
 	xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &ichdr, leaf);
@@ -273,7 +274,12 @@ xfs_attr3_leaf_verify(
 		if (ichdr.magic != XFS_ATTR_LEAF_MAGIC)
 			return false;
 	}
-	if (ichdr.count == 0)
+	/*
+	 * In recovery there is a transient state where count == 0 is valid
+	 * because we may have transitioned an empty shortform attr to a leaf
+	 * if the attr didn't fit in shortform.
+	 */
+	if (pag && pag->pagf_init && ichdr.count == 0)
 		return false;
 
 	/* XXX: need to range check rest of attr header values */
diff --git a/fs/xfs/libxfs/xfs_attr_leaf.h b/fs/xfs/libxfs/xfs_attr_leaf.h
index 4f2aed04f827..f7dda0c237b0 100644
--- a/fs/xfs/libxfs/xfs_attr_leaf.h
+++ b/fs/xfs/libxfs/xfs_attr_leaf.h
@@ -51,7 +51,7 @@ int	xfs_attr_shortform_getvalue(struct xfs_da_args *args);
 int	xfs_attr_shortform_to_leaf(struct xfs_da_args *args);
 int	xfs_attr_shortform_remove(struct xfs_da_args *args);
 int	xfs_attr_shortform_allfit(struct xfs_buf *bp, struct xfs_inode *dp);
-int	xfs_attr_shortform_bytesfit(xfs_inode_t *dp, int bytes);
+int	xfs_attr_shortform_bytesfit(struct xfs_inode *dp, int bytes);
 void	xfs_attr_fork_remove(struct xfs_inode *ip, struct xfs_trans *tp);
 
 /*
@@ -77,7 +77,7 @@ int	xfs_attr3_leaf_add(struct xfs_buf *leaf_buffer,
 				 struct xfs_da_args *args);
 int	xfs_attr3_leaf_remove(struct xfs_buf *leaf_buffer,
 				    struct xfs_da_args *args);
-int	xfs_attr3_leaf_list_int(struct xfs_buf *bp,
+void	xfs_attr3_leaf_list_int(struct xfs_buf *bp,
 				      struct xfs_attr_list_context *context);
 
 /*
diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c
index c6eb21940783..2760bc3b2536 100644
--- a/fs/xfs/libxfs/xfs_bmap.c
+++ b/fs/xfs/libxfs/xfs_bmap.c
@@ -49,6 +49,8 @@
 #include "xfs_rmap.h"
 #include "xfs_ag_resv.h"
 #include "xfs_refcount.h"
+#include "xfs_rmap_btree.h"
+#include "xfs_icache.h"
 
 
 kmem_zone_t		*xfs_bmap_free_item_zone;
@@ -190,8 +192,12 @@ xfs_bmap_worst_indlen(
 	int		maxrecs;	/* maximum record count at this level */
 	xfs_mount_t	*mp;		/* mount structure */
 	xfs_filblks_t	rval;		/* return value */
+	xfs_filblks_t   orig_len;
 
 	mp = ip->i_mount;
+
+	/* Calculate the worst-case size of the bmbt. */
+	orig_len = len;
 	maxrecs = mp->m_bmap_dmxr[0];
 	for (level = 0, rval = 0;
 	     level < XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK);
@@ -199,12 +205,20 @@ xfs_bmap_worst_indlen(
 		len += maxrecs - 1;
 		do_div(len, maxrecs);
 		rval += len;
-		if (len == 1)
-			return rval + XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK) -
+		if (len == 1) {
+			rval += XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK) -
 				level - 1;
+			break;
+		}
 		if (level == 0)
 			maxrecs = mp->m_bmap_dmxr[1];
 	}
+
+	/* Calculate the worst-case size of the rmapbt. */
+	if (xfs_sb_version_hasrmapbt(&mp->m_sb))
+		rval += 1 + xfs_rmapbt_calc_size(mp, orig_len) +
+				mp->m_rmap_maxlevels;
+
 	return rval;
 }
 
@@ -504,7 +518,7 @@ void
 xfs_bmap_trace_exlist(
 	xfs_inode_t	*ip,		/* incore inode pointer */
 	xfs_extnum_t	cnt,		/* count of entries in the list */
-	int		whichfork,	/* data or attr fork */
+	int		whichfork,	/* data or attr or cow fork */
 	unsigned long	caller_ip)
 {
 	xfs_extnum_t	idx;		/* extent record index */
@@ -513,11 +527,13 @@ xfs_bmap_trace_exlist(
 
 	if (whichfork == XFS_ATTR_FORK)
 		state |= BMAP_ATTRFORK;
+	else if (whichfork == XFS_COW_FORK)
+		state |= BMAP_COWFORK;
 
 	ifp = XFS_IFORK_PTR(ip, whichfork);
-	ASSERT(cnt == (ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t)));
+	ASSERT(cnt == xfs_iext_count(ifp));
 	for (idx = 0; idx < cnt; idx++)
-		trace_xfs_extlist(ip, idx, whichfork, caller_ip);
+		trace_xfs_extlist(ip, idx, state, caller_ip);
 }
 
 /*
@@ -811,7 +827,7 @@ try_another_ag:
 				XFS_BTREE_LONG_PTRS);
 
 	arp = XFS_BMBT_REC_ADDR(mp, ablock, 1);
-	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	nextents =  xfs_iext_count(ifp);
 	for (cnt = i = 0; i < nextents; i++) {
 		ep = xfs_iext_get_ext(ifp, i);
 		if (!isnullstartblock(xfs_bmbt_get_startblock(ep))) {
@@ -1137,6 +1153,10 @@ xfs_bmap_add_attrfork(
 		goto trans_cancel;
 	if (XFS_IFORK_Q(ip))
 		goto trans_cancel;
+	if (ip->i_d.di_anextents != 0) {
+		error = -EFSCORRUPTED;
+		goto trans_cancel;
+	}
 	if (ip->i_d.di_aformat != XFS_DINODE_FMT_EXTENTS) {
 		/*
 		 * For inodes coming from pre-6.2 filesystems.
@@ -1144,7 +1164,6 @@ xfs_bmap_add_attrfork(
 		ASSERT(ip->i_d.di_aformat == 0);
 		ip->i_d.di_aformat = XFS_DINODE_FMT_EXTENTS;
 	}
-	ASSERT(ip->i_d.di_anextents == 0);
 
 	xfs_trans_ijoin(tp, ip, 0);
 	xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
@@ -1296,7 +1315,7 @@ xfs_bmap_read_extents(
 	/*
 	 * Here with bp and block set to the leftmost leaf node in the tree.
 	 */
-	room = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	room = xfs_iext_count(ifp);
 	i = 0;
 	/*
 	 * Loop over all leaf nodes.  Copy information to the extent records.
@@ -1361,8 +1380,9 @@ xfs_bmap_read_extents(
 			return error;
 		block = XFS_BUF_TO_BLOCK(bp);
 	}
-	ASSERT(i == (ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t)));
-	ASSERT(i == XFS_IFORK_NEXTENTS(ip, whichfork));
+	if (i != XFS_IFORK_NEXTENTS(ip, whichfork))
+		return -EFSCORRUPTED;
+	ASSERT(i == xfs_iext_count(ifp));
 	XFS_BMAP_TRACE_EXLIST(ip, i, whichfork);
 	return 0;
 error0:
@@ -1370,97 +1390,6 @@ error0:
 	return -EFSCORRUPTED;
 }
 
-
-/*
- * Search the extent records for the entry containing block bno.
- * If bno lies in a hole, point to the next entry.  If bno lies
- * past eof, *eofp will be set, and *prevp will contain the last
- * entry (null if none).  Else, *lastxp will be set to the index
- * of the found entry; *gotp will contain the entry.
- */
-STATIC xfs_bmbt_rec_host_t *		/* pointer to found extent entry */
-xfs_bmap_search_multi_extents(
-	xfs_ifork_t	*ifp,		/* inode fork pointer */
-	xfs_fileoff_t	bno,		/* block number searched for */
-	int		*eofp,		/* out: end of file found */
-	xfs_extnum_t	*lastxp,	/* out: last extent index */
-	xfs_bmbt_irec_t	*gotp,		/* out: extent entry found */
-	xfs_bmbt_irec_t	*prevp)		/* out: previous extent entry found */
-{
-	xfs_bmbt_rec_host_t *ep;		/* extent record pointer */
-	xfs_extnum_t	lastx;		/* last extent index */
-
-	/*
-	 * Initialize the extent entry structure to catch access to
-	 * uninitialized br_startblock field.
-	 */
-	gotp->br_startoff = 0xffa5a5a5a5a5a5a5LL;
-	gotp->br_blockcount = 0xa55a5a5a5a5a5a5aLL;
-	gotp->br_state = XFS_EXT_INVALID;
-	gotp->br_startblock = 0xffffa5a5a5a5a5a5LL;
-	prevp->br_startoff = NULLFILEOFF;
-
-	ep = xfs_iext_bno_to_ext(ifp, bno, &lastx);
-	if (lastx > 0) {
-		xfs_bmbt_get_all(xfs_iext_get_ext(ifp, lastx - 1), prevp);
-	}
-	if (lastx < (ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t))) {
-		xfs_bmbt_get_all(ep, gotp);
-		*eofp = 0;
-	} else {
-		if (lastx > 0) {
-			*gotp = *prevp;
-		}
-		*eofp = 1;
-		ep = NULL;
-	}
-	*lastxp = lastx;
-	return ep;
-}
-
-/*
- * Search the extents list for the inode, for the extent containing bno.
- * If bno lies in a hole, point to the next entry.  If bno lies past eof,
- * *eofp will be set, and *prevp will contain the last entry (null if none).
- * Else, *lastxp will be set to the index of the found
- * entry; *gotp will contain the entry.
- */
-xfs_bmbt_rec_host_t *                 /* pointer to found extent entry */
-xfs_bmap_search_extents(
-	xfs_inode_t     *ip,            /* incore inode pointer */
-	xfs_fileoff_t   bno,            /* block number searched for */
-	int             fork,      	/* data or attr fork */
-	int             *eofp,          /* out: end of file found */
-	xfs_extnum_t    *lastxp,        /* out: last extent index */
-	xfs_bmbt_irec_t *gotp,          /* out: extent entry found */
-	xfs_bmbt_irec_t *prevp)         /* out: previous extent entry found */
-{
-	xfs_ifork_t	*ifp;		/* inode fork pointer */
-	xfs_bmbt_rec_host_t  *ep;            /* extent record pointer */
-
-	XFS_STATS_INC(ip->i_mount, xs_look_exlist);
-	ifp = XFS_IFORK_PTR(ip, fork);
-
-	ep = xfs_bmap_search_multi_extents(ifp, bno, eofp, lastxp, gotp, prevp);
-
-	if (unlikely(!(gotp->br_startblock) && (*lastxp != NULLEXTNUM) &&
-		     !(XFS_IS_REALTIME_INODE(ip) && fork == XFS_DATA_FORK))) {
-		xfs_alert_tag(ip->i_mount, XFS_PTAG_FSBLOCK_ZERO,
-				"Access to block zero in inode %llu "
-				"start_block: %llx start_off: %llx "
-				"blkcnt: %llx extent-state: %x lastx: %x",
-			(unsigned long long)ip->i_ino,
-			(unsigned long long)gotp->br_startblock,
-			(unsigned long long)gotp->br_startoff,
-			(unsigned long long)gotp->br_blockcount,
-			gotp->br_state, *lastxp);
-		*lastxp = NULLEXTNUM;
-		*eofp = 1;
-		return NULL;
-	}
-	return ep;
-}
-
 /*
  * Returns the file-relative block number of the first unused block(s)
  * in the file with at least "len" logically contiguous blocks free.
@@ -1497,7 +1426,7 @@ xfs_bmap_first_unused(
 	    (error = xfs_iread_extents(tp, ip, whichfork)))
 		return error;
 	lowest = *first_unused;
-	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	nextents = xfs_iext_count(ifp);
 	for (idx = 0, lastaddr = 0, max = lowest; idx < nextents; idx++) {
 		xfs_bmbt_rec_host_t *ep = xfs_iext_get_ext(ifp, idx);
 		off = xfs_bmbt_get_startoff(ep);
@@ -1523,44 +1452,44 @@ xfs_bmap_first_unused(
  */
 int						/* error */
 xfs_bmap_last_before(
-	xfs_trans_t	*tp,			/* transaction pointer */
-	xfs_inode_t	*ip,			/* incore inode */
-	xfs_fileoff_t	*last_block,		/* last block */
-	int		whichfork)		/* data or attr fork */
+	struct xfs_trans	*tp,		/* transaction pointer */
+	struct xfs_inode	*ip,		/* incore inode */
+	xfs_fileoff_t		*last_block,	/* last block */
+	int			whichfork)	/* data or attr fork */
 {
-	xfs_fileoff_t	bno;			/* input file offset */
-	int		eof;			/* hit end of file */
-	xfs_bmbt_rec_host_t *ep;		/* pointer to last extent */
-	int		error;			/* error return value */
-	xfs_bmbt_irec_t	got;			/* current extent value */
-	xfs_ifork_t	*ifp;			/* inode fork pointer */
-	xfs_extnum_t	lastx;			/* last extent used */
-	xfs_bmbt_irec_t	prev;			/* previous extent value */
+	struct xfs_ifork	*ifp = XFS_IFORK_PTR(ip, whichfork);
+	struct xfs_bmbt_irec	got;
+	xfs_extnum_t		idx;
+	int			error;
 
-	if (XFS_IFORK_FORMAT(ip, whichfork) != XFS_DINODE_FMT_BTREE &&
-	    XFS_IFORK_FORMAT(ip, whichfork) != XFS_DINODE_FMT_EXTENTS &&
-	    XFS_IFORK_FORMAT(ip, whichfork) != XFS_DINODE_FMT_LOCAL)
-	       return -EIO;
-	if (XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_LOCAL) {
+	switch (XFS_IFORK_FORMAT(ip, whichfork)) {
+	case XFS_DINODE_FMT_LOCAL:
 		*last_block = 0;
 		return 0;
+	case XFS_DINODE_FMT_BTREE:
+	case XFS_DINODE_FMT_EXTENTS:
+		break;
+	default:
+		return -EIO;
 	}
-	ifp = XFS_IFORK_PTR(ip, whichfork);
-	if (!(ifp->if_flags & XFS_IFEXTENTS) &&
-	    (error = xfs_iread_extents(tp, ip, whichfork)))
-		return error;
-	bno = *last_block - 1;
-	ep = xfs_bmap_search_extents(ip, bno, whichfork, &eof, &lastx, &got,
-		&prev);
-	if (eof || xfs_bmbt_get_startoff(ep) > bno) {
-		if (prev.br_startoff == NULLFILEOFF)
-			*last_block = 0;
-		else
-			*last_block = prev.br_startoff + prev.br_blockcount;
+
+	if (!(ifp->if_flags & XFS_IFEXTENTS)) {
+		error = xfs_iread_extents(tp, ip, whichfork);
+		if (error)
+			return error;
 	}
-	/*
-	 * Otherwise *last_block is already the right answer.
-	 */
+
+	if (xfs_iext_lookup_extent(ip, ifp, *last_block - 1, &idx, &got)) {
+		if (got.br_startoff <= *last_block - 1)
+			return 0;
+	}
+
+	if (xfs_iext_get_extent(ifp, idx - 1, &got)) {
+		*last_block = got.br_startoff + got.br_blockcount;
+		return 0;
+	}
+
+	*last_block = 0;
 	return 0;
 }
 
@@ -1582,7 +1511,7 @@ xfs_bmap_last_extent(
 			return error;
 	}
 
-	nextents = ifp->if_bytes / sizeof(xfs_bmbt_rec_t);
+	nextents = xfs_iext_count(ifp);
 	if (nextents == 0) {
 		*is_empty = 1;
 		return 0;
@@ -1735,7 +1664,7 @@ xfs_bmap_add_extent_delay_real(
 						&bma->ip->i_d.di_nextents);
 
 	ASSERT(bma->idx >= 0);
-	ASSERT(bma->idx <= ifp->if_bytes / sizeof(struct xfs_bmbt_rec));
+	ASSERT(bma->idx <= xfs_iext_count(ifp));
 	ASSERT(!isnullstartblock(new->br_startblock));
 	ASSERT(!bma->cur ||
 	       (bma->cur->bc_private.b.flags & XFS_BTCUR_BPRV_WASDEL));
@@ -1794,7 +1723,7 @@ xfs_bmap_add_extent_delay_real(
 	 * Don't set contiguous if the combined extent would be too large.
 	 * Also check for all-three-contiguous being too large.
 	 */
-	if (bma->idx < ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t) - 1) {
+	if (bma->idx < xfs_iext_count(ifp) - 1) {
 		state |= BMAP_RIGHT_VALID;
 		xfs_bmbt_get_all(xfs_iext_get_ext(ifp, bma->idx + 1), &RIGHT);
 
@@ -2300,7 +2229,7 @@ xfs_bmap_add_extent_unwritten_real(
 	ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK);
 
 	ASSERT(*idx >= 0);
-	ASSERT(*idx <= ifp->if_bytes / sizeof(struct xfs_bmbt_rec));
+	ASSERT(*idx <= xfs_iext_count(ifp));
 	ASSERT(!isnullstartblock(new->br_startblock));
 
 	XFS_STATS_INC(mp, xs_add_exlist);
@@ -2356,7 +2285,7 @@ xfs_bmap_add_extent_unwritten_real(
 	 * Don't set contiguous if the combined extent would be too large.
 	 * Also check for all-three-contiguous being too large.
 	 */
-	if (*idx < ip->i_df.if_bytes / (uint)sizeof(xfs_bmbt_rec_t) - 1) {
+	if (*idx < xfs_iext_count(&ip->i_df) - 1) {
 		state |= BMAP_RIGHT_VALID;
 		xfs_bmbt_get_all(xfs_iext_get_ext(ifp, *idx + 1), &RIGHT);
 		if (isnullstartblock(RIGHT.br_startblock))
@@ -2836,7 +2765,7 @@ xfs_bmap_add_extent_hole_delay(
 	 * Check and set flags if the current (right) segment exists.
 	 * If it doesn't exist, we're converting the hole at end-of-file.
 	 */
-	if (*idx < ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t)) {
+	if (*idx < xfs_iext_count(ifp)) {
 		state |= BMAP_RIGHT_VALID;
 		xfs_bmbt_get_all(xfs_iext_get_ext(ifp, *idx), &right);
 
@@ -2966,7 +2895,7 @@ xfs_bmap_add_extent_hole_real(
 	ifp = XFS_IFORK_PTR(bma->ip, whichfork);
 
 	ASSERT(bma->idx >= 0);
-	ASSERT(bma->idx <= ifp->if_bytes / sizeof(struct xfs_bmbt_rec));
+	ASSERT(bma->idx <= xfs_iext_count(ifp));
 	ASSERT(!isnullstartblock(new->br_startblock));
 	ASSERT(!bma->cur ||
 	       !(bma->cur->bc_private.b.flags & XFS_BTCUR_BPRV_WASDEL));
@@ -2992,7 +2921,7 @@ xfs_bmap_add_extent_hole_real(
 	 * Check and set flags if this segment has a current value.
 	 * Not true if we're inserting into the "hole" at eof.
 	 */
-	if (bma->idx < ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t)) {
+	if (bma->idx < xfs_iext_count(ifp)) {
 		state |= BMAP_RIGHT_VALID;
 		xfs_bmbt_get_all(xfs_iext_get_ext(ifp, bma->idx), &right);
 		if (isnullstartblock(right.br_startblock))
@@ -4145,12 +4074,11 @@ xfs_bmapi_read(
 	struct xfs_mount	*mp = ip->i_mount;
 	struct xfs_ifork	*ifp;
 	struct xfs_bmbt_irec	got;
-	struct xfs_bmbt_irec	prev;
 	xfs_fileoff_t		obno;
 	xfs_fileoff_t		end;
-	xfs_extnum_t		lastx;
+	xfs_extnum_t		idx;
 	int			error;
-	int			eof;
+	bool			eof = false;
 	int			n = 0;
 	int			whichfork = xfs_bmapi_whichfork(flags);
 
@@ -4190,7 +4118,8 @@ xfs_bmapi_read(
 			return error;
 	}
 
-	xfs_bmap_search_extents(ip, bno, whichfork, &eof, &lastx, &got, &prev);
+	if (!xfs_iext_lookup_extent(ip, ifp, bno, &idx, &got))
+		eof = true;
 	end = bno + len;
 	obno = bno;
 
@@ -4221,10 +4150,8 @@ xfs_bmapi_read(
 			break;
 
 		/* Else go on to the next record. */
-		if (++lastx < ifp->if_bytes / sizeof(xfs_bmbt_rec_t))
-			xfs_bmbt_get_all(xfs_iext_get_ext(ifp, lastx), &got);
-		else
-			eof = 1;
+		if (!xfs_iext_get_extent(ifp, ++idx, &got))
+			eof = true;
 	}
 	*nmap = n;
 	return 0;
@@ -4234,10 +4161,10 @@ int
 xfs_bmapi_reserve_delalloc(
 	struct xfs_inode	*ip,
 	int			whichfork,
-	xfs_fileoff_t		aoff,
+	xfs_fileoff_t		off,
 	xfs_filblks_t		len,
+	xfs_filblks_t		prealloc,
 	struct xfs_bmbt_irec	*got,
-	struct xfs_bmbt_irec	*prev,
 	xfs_extnum_t		*lastx,
 	int			eof)
 {
@@ -4248,10 +4175,17 @@ xfs_bmapi_reserve_delalloc(
 	char			rt = XFS_IS_REALTIME_INODE(ip);
 	xfs_extlen_t		extsz;
 	int			error;
+	xfs_fileoff_t		aoff = off;
 
-	alen = XFS_FILBLKS_MIN(len, MAXEXTLEN);
+	/*
+	 * Cap the alloc length. Keep track of prealloc so we know whether to
+	 * tag the inode before we return.
+	 */
+	alen = XFS_FILBLKS_MIN(len + prealloc, MAXEXTLEN);
 	if (!eof)
 		alen = XFS_FILBLKS_MIN(alen, got->br_startoff - aoff);
+	if (prealloc && alen >= len)
+		prealloc = alen - len;
 
 	/* Figure out the extent size, adjust alen */
 	if (whichfork == XFS_COW_FORK)
@@ -4259,7 +4193,12 @@ xfs_bmapi_reserve_delalloc(
 	else
 		extsz = xfs_get_extsz_hint(ip);
 	if (extsz) {
-		error = xfs_bmap_extsize_align(mp, got, prev, extsz, rt, eof,
+		struct xfs_bmbt_irec	prev;
+
+		if (!xfs_iext_get_extent(ifp, *lastx - 1, &prev))
+			prev.br_startoff = NULLFILEOFF;
+
+		error = xfs_bmap_extsize_align(mp, got, &prev, extsz, rt, eof,
 					       1, 0, &aoff, &alen);
 		ASSERT(!error);
 	}
@@ -4312,6 +4251,16 @@ xfs_bmapi_reserve_delalloc(
 	 */
 	xfs_bmbt_get_all(xfs_iext_get_ext(ifp, *lastx), got);
 
+	/*
+	 * Tag the inode if blocks were preallocated. Note that COW fork
+	 * preallocation can occur at the start or end of the extent, even when
+	 * prealloc == 0, so we must also check the aligned offset and length.
+	 */
+	if (whichfork == XFS_DATA_FORK && prealloc)
+		xfs_inode_set_eofblocks_tag(ip);
+	if (whichfork == XFS_COW_FORK && (prealloc || aoff < off || alen > len))
+		xfs_inode_set_cowblocks_tag(ip);
+
 	ASSERT(got->br_startoff <= aoff);
 	ASSERT(got->br_startoff + got->br_blockcount >= aoff + alen);
 	ASSERT(isnullstartblock(got->br_startblock));
@@ -4349,7 +4298,7 @@ xfs_bmapi_allocate(
 	if (bma->wasdel) {
 		bma->length = (xfs_extlen_t)bma->got.br_blockcount;
 		bma->offset = bma->got.br_startoff;
-		if (bma->idx != NULLEXTNUM && bma->idx) {
+		if (bma->idx) {
 			xfs_bmbt_get_all(xfs_iext_get_ext(ifp, bma->idx - 1),
 					 &bma->prev);
 		}
@@ -4563,7 +4512,7 @@ xfs_bmapi_write(
 	struct xfs_ifork	*ifp;
 	struct xfs_bmalloca	bma = { NULL };	/* args for xfs_bmap_alloc */
 	xfs_fileoff_t		end;		/* end of mapped file region */
-	int			eof;		/* after the end of extents */
+	bool			eof = false;	/* after the end of extents */
 	int			error;		/* error return */
 	int			n;		/* current extent index */
 	xfs_fileoff_t		obno;		/* old block number (offset) */
@@ -4641,12 +4590,14 @@ xfs_bmapi_write(
 			goto error0;
 	}
 
-	xfs_bmap_search_extents(ip, bno, whichfork, &eof, &bma.idx, &bma.got,
-				&bma.prev);
 	n = 0;
 	end = bno + len;
 	obno = bno;
 
+	if (!xfs_iext_lookup_extent(ip, ifp, bno, &bma.idx, &bma.got))
+		eof = true;
+	if (!xfs_iext_get_extent(ifp, bma.idx - 1, &bma.prev))
+		bma.prev.br_startoff = NULLFILEOFF;
 	bma.tp = tp;
 	bma.ip = ip;
 	bma.total = total;
@@ -4733,11 +4684,8 @@ xfs_bmapi_write(
 
 		/* Else go on to the next record. */
 		bma.prev = bma.got;
-		if (++bma.idx < ifp->if_bytes / sizeof(xfs_bmbt_rec_t)) {
-			xfs_bmbt_get_all(xfs_iext_get_ext(ifp, bma.idx),
-					 &bma.got);
-		} else
-			eof = 1;
+		if (!xfs_iext_get_extent(ifp, ++bma.idx, &bma.got))
+			eof = true;
 	}
 	*nmap = n;
 
@@ -4885,7 +4833,7 @@ xfs_bmap_del_extent_delay(
 	da_new = 0;
 
 	ASSERT(*idx >= 0);
-	ASSERT(*idx < ifp->if_bytes / sizeof(struct xfs_bmbt_rec));
+	ASSERT(*idx <= xfs_iext_count(ifp));
 	ASSERT(del->br_blockcount > 0);
 	ASSERT(got->br_startoff <= del->br_startoff);
 	ASSERT(got_endoff >= del_endoff);
@@ -4902,8 +4850,11 @@ xfs_bmap_del_extent_delay(
 	 * sb counters as we might have to borrow some blocks for the
 	 * indirect block accounting.
 	 */
-	xfs_trans_reserve_quota_nblks(NULL, ip, -((long)del->br_blockcount), 0,
+	error = xfs_trans_reserve_quota_nblks(NULL, ip,
+			-((long)del->br_blockcount), 0,
 			isrt ? XFS_QMOPT_RES_RTBLKS : XFS_QMOPT_RES_REGBLKS);
+	if (error)
+		return error;
 	ip->i_delayed_blks -= del->br_blockcount;
 
 	if (whichfork == XFS_COW_FORK)
@@ -5013,7 +4964,7 @@ xfs_bmap_del_extent_cow(
 	got_endoff = got->br_startoff + got->br_blockcount;
 
 	ASSERT(*idx >= 0);
-	ASSERT(*idx < ifp->if_bytes / sizeof(struct xfs_bmbt_rec));
+	ASSERT(*idx <= xfs_iext_count(ifp));
 	ASSERT(del->br_blockcount > 0);
 	ASSERT(got->br_startoff <= del->br_startoff);
 	ASSERT(got_endoff >= del_endoff);
@@ -5119,8 +5070,7 @@ xfs_bmap_del_extent(
 		state |= BMAP_COWFORK;
 
 	ifp = XFS_IFORK_PTR(ip, whichfork);
-	ASSERT((*idx >= 0) && (*idx < ifp->if_bytes /
-		(uint)sizeof(xfs_bmbt_rec_t)));
+	ASSERT((*idx >= 0) && (*idx < xfs_iext_count(ifp)));
 	ASSERT(del->br_blockcount > 0);
 	ep = xfs_iext_get_ext(ifp, *idx);
 	xfs_bmbt_get_all(ep, &got);
@@ -5434,8 +5384,6 @@ __xfs_bunmapi(
 {
 	xfs_btree_cur_t		*cur;		/* bmap btree cursor */
 	xfs_bmbt_irec_t		del;		/* extent being deleted */
-	int			eof;		/* is deleting at eof */
-	xfs_bmbt_rec_host_t	*ep;		/* extent record pointer */
 	int			error;		/* error return value */
 	xfs_extnum_t		extno;		/* extent number in list */
 	xfs_bmbt_irec_t		got;		/* current extent record */
@@ -5445,8 +5393,6 @@ __xfs_bunmapi(
 	int			logflags;	/* transaction logging flags */
 	xfs_extlen_t		mod;		/* rt extent offset */
 	xfs_mount_t		*mp;		/* mount structure */
-	xfs_extnum_t		nextents;	/* number of file extents */
-	xfs_bmbt_irec_t		prev;		/* previous extent record */
 	xfs_fileoff_t		start;		/* first file offset deleted */
 	int			tmp_logflags;	/* partial logging flags */
 	int			wasdel;		/* was a delayed alloc extent */
@@ -5477,8 +5423,7 @@ __xfs_bunmapi(
 	if (!(ifp->if_flags & XFS_IFEXTENTS) &&
 	    (error = xfs_iread_extents(tp, ip, whichfork)))
 		return error;
-	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
-	if (nextents == 0) {
+	if (xfs_iext_count(ifp) == 0) {
 		*rlen = 0;
 		return 0;
 	}
@@ -5486,18 +5431,17 @@ __xfs_bunmapi(
 	isrt = (whichfork == XFS_DATA_FORK) && XFS_IS_REALTIME_INODE(ip);
 	start = bno;
 	bno = start + len - 1;
-	ep = xfs_bmap_search_extents(ip, bno, whichfork, &eof, &lastx, &got,
-		&prev);
 
 	/*
 	 * Check to see if the given block number is past the end of the
 	 * file, back up to the last block if so...
 	 */
-	if (eof) {
-		ep = xfs_iext_get_ext(ifp, --lastx);
-		xfs_bmbt_get_all(ep, &got);
+	if (!xfs_iext_lookup_extent(ip, ifp, bno, &lastx, &got)) {
+		ASSERT(lastx > 0);
+		xfs_iext_get_extent(ifp, --lastx, &got);
 		bno = got.br_startoff + got.br_blockcount - 1;
 	}
+
 	logflags = 0;
 	if (ifp->if_flags & XFS_IFBROOT) {
 		ASSERT(XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_BTREE);
@@ -5528,8 +5472,7 @@ __xfs_bunmapi(
 		if (got.br_startoff > bno) {
 			if (--lastx < 0)
 				break;
-			ep = xfs_iext_get_ext(ifp, lastx);
-			xfs_bmbt_get_all(ep, &got);
+			xfs_iext_get_extent(ifp, lastx, &got);
 		}
 		/*
 		 * Is the last block of this extent before the range
@@ -5543,7 +5486,6 @@ __xfs_bunmapi(
 		 * Then deal with the (possibly delayed) allocated space
 		 * we found.
 		 */
-		ASSERT(ep != NULL);
 		del = got;
 		wasdel = isnullstartblock(del.br_startblock);
 		if (got.br_startoff < start) {
@@ -5624,15 +5566,12 @@ __xfs_bunmapi(
 				 */
 				ASSERT(bno >= del.br_blockcount);
 				bno -= del.br_blockcount;
-				if (got.br_startoff > bno) {
-					if (--lastx >= 0) {
-						ep = xfs_iext_get_ext(ifp,
-								      lastx);
-						xfs_bmbt_get_all(ep, &got);
-					}
-				}
+				if (got.br_startoff > bno && --lastx >= 0)
+					xfs_iext_get_extent(ifp, lastx, &got);
 				continue;
 			} else if (del.br_state == XFS_EXT_UNWRITTEN) {
+				struct xfs_bmbt_irec	prev;
+
 				/*
 				 * This one is already unwritten.
 				 * It must have a written left neighbor.
@@ -5640,8 +5579,7 @@ __xfs_bunmapi(
 				 * try again.
 				 */
 				ASSERT(lastx > 0);
-				xfs_bmbt_get_all(xfs_iext_get_ext(ifp,
-						lastx - 1), &prev);
+				xfs_iext_get_extent(ifp, lastx - 1, &prev);
 				ASSERT(prev.br_state == XFS_EXT_NORM);
 				ASSERT(!isnullstartblock(prev.br_startblock));
 				ASSERT(del.br_startblock ==
@@ -5739,13 +5677,9 @@ nodelete:
 		 */
 		if (bno != (xfs_fileoff_t)-1 && bno >= start) {
 			if (lastx >= 0) {
-				ep = xfs_iext_get_ext(ifp, lastx);
-				if (xfs_bmbt_get_startoff(ep) > bno) {
-					if (--lastx >= 0)
-						ep = xfs_iext_get_ext(ifp,
-								      lastx);
-				}
-				xfs_bmbt_get_all(ep, &got);
+				xfs_iext_get_extent(ifp, lastx, &got);
+				if (got.br_startoff > bno && --lastx >= 0)
+					xfs_iext_get_extent(ifp, lastx, &got);
 			}
 			extno++;
 		}
@@ -5963,7 +5897,7 @@ xfs_bmse_shift_one(
 
 	mp = ip->i_mount;
 	ifp = XFS_IFORK_PTR(ip, whichfork);
-	total_extents = ifp->if_bytes / sizeof(xfs_bmbt_rec_t);
+	total_extents = xfs_iext_count(ifp);
 
 	xfs_bmbt_get_all(gotp, &got);
 
@@ -6140,7 +6074,7 @@ xfs_bmap_shift_extents(
 	 * are collapsing out, so we cannot use the count of real extents here.
 	 * Instead we have to calculate it from the incore fork.
 	 */
-	total_extents = ifp->if_bytes / sizeof(xfs_bmbt_rec_t);
+	total_extents = xfs_iext_count(ifp);
 	if (total_extents == 0) {
 		*done = 1;
 		goto del_cursor;
@@ -6200,7 +6134,7 @@ xfs_bmap_shift_extents(
 		 * count can change. Update the total and grade the next record.
 		 */
 		if (direction == SHIFT_LEFT) {
-			total_extents = ifp->if_bytes / sizeof(xfs_bmbt_rec_t);
+			total_extents = xfs_iext_count(ifp);
 			stop_extent = total_extents;
 		}
 
diff --git a/fs/xfs/libxfs/xfs_bmap.h b/fs/xfs/libxfs/xfs_bmap.h
index 7cae6ec27fa6..cecd094404cc 100644
--- a/fs/xfs/libxfs/xfs_bmap.h
+++ b/fs/xfs/libxfs/xfs_bmap.h
@@ -237,14 +237,9 @@ int	xfs_bmap_shift_extents(struct xfs_trans *tp, struct xfs_inode *ip,
 		struct xfs_defer_ops *dfops, enum shift_direction direction,
 		int num_exts);
 int	xfs_bmap_split_extent(struct xfs_inode *ip, xfs_fileoff_t split_offset);
-struct xfs_bmbt_rec_host *
-	xfs_bmap_search_extents(struct xfs_inode *ip, xfs_fileoff_t bno,
-		int fork, int *eofp, xfs_extnum_t *lastxp,
-		struct xfs_bmbt_irec *gotp, struct xfs_bmbt_irec *prevp);
 int	xfs_bmapi_reserve_delalloc(struct xfs_inode *ip, int whichfork,
-		xfs_fileoff_t aoff, xfs_filblks_t len,
-		struct xfs_bmbt_irec *got, struct xfs_bmbt_irec *prev,
-		xfs_extnum_t *lastx, int eof);
+		xfs_fileoff_t off, xfs_filblks_t len, xfs_filblks_t prealloc,
+		struct xfs_bmbt_irec *got, xfs_extnum_t *lastx, int eof);
 
 enum xfs_bmap_intent_type {
 	XFS_BMAP_MAP = 1,
diff --git a/fs/xfs/libxfs/xfs_bmap_btree.c b/fs/xfs/libxfs/xfs_bmap_btree.c
index 8007d2ba9aef..d6330c297ca0 100644
--- a/fs/xfs/libxfs/xfs_bmap_btree.c
+++ b/fs/xfs/libxfs/xfs_bmap_btree.c
@@ -796,13 +796,14 @@ xfs_bmbt_init_cursor(
 	struct xfs_btree_cur	*cur;
 	ASSERT(whichfork != XFS_COW_FORK);
 
-	cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_SLEEP);
+	cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS);
 
 	cur->bc_tp = tp;
 	cur->bc_mp = mp;
 	cur->bc_nlevels = be16_to_cpu(ifp->if_broot->bb_level) + 1;
 	cur->bc_btnum = XFS_BTNUM_BMAP;
 	cur->bc_blocklog = mp->m_sb.sb_blocklog;
+	cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_bmbt_2);
 
 	cur->bc_ops = &xfs_bmbt_ops;
 	cur->bc_flags = XFS_BTREE_LONG_PTRS | XFS_BTREE_ROOT_IN_INODE;
diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c
index 0e80993c8a59..21e6a6ab6b9a 100644
--- a/fs/xfs/libxfs/xfs_btree.c
+++ b/fs/xfs/libxfs/xfs_btree.c
@@ -1769,8 +1769,28 @@ xfs_btree_lookup_get_block(
 	if (error)
 		return error;
 
+	/* Check the inode owner since the verifiers don't. */
+	if (xfs_sb_version_hascrc(&cur->bc_mp->m_sb) &&
+	    (cur->bc_flags & XFS_BTREE_LONG_PTRS) &&
+	    be64_to_cpu((*blkp)->bb_u.l.bb_owner) !=
+			cur->bc_private.b.ip->i_ino)
+		goto out_bad;
+
+	/* Did we get the level we were looking for? */
+	if (be16_to_cpu((*blkp)->bb_level) != level)
+		goto out_bad;
+
+	/* Check that internal nodes have at least one record. */
+	if (level != 0 && be16_to_cpu((*blkp)->bb_numrecs) == 0)
+		goto out_bad;
+
 	xfs_btree_setbuf(cur, level, bp);
 	return 0;
+
+out_bad:
+	*blkp = NULL;
+	xfs_trans_brelse(cur->bc_tp, bp);
+	return -EFSCORRUPTED;
 }
 
 /*
diff --git a/fs/xfs/libxfs/xfs_btree.h b/fs/xfs/libxfs/xfs_btree.h
index c2b01d1c79ee..b69b947c4c1b 100644
--- a/fs/xfs/libxfs/xfs_btree.h
+++ b/fs/xfs/libxfs/xfs_btree.h
@@ -96,46 +96,10 @@ union xfs_btree_rec {
 /*
  * Generic stats interface
  */
-#define __XFS_BTREE_STATS_INC(mp, type, stat) \
-	XFS_STATS_INC(mp, xs_ ## type ## _2_ ## stat)
 #define XFS_BTREE_STATS_INC(cur, stat)	\
-do {    \
-	struct xfs_mount *__mp = cur->bc_mp; \
-	switch (cur->bc_btnum) {  \
-	case XFS_BTNUM_BNO: __XFS_BTREE_STATS_INC(__mp, abtb, stat); break; \
-	case XFS_BTNUM_CNT: __XFS_BTREE_STATS_INC(__mp, abtc, stat); break; \
-	case XFS_BTNUM_BMAP: __XFS_BTREE_STATS_INC(__mp, bmbt, stat); break; \
-	case XFS_BTNUM_INO: __XFS_BTREE_STATS_INC(__mp, ibt, stat); break; \
-	case XFS_BTNUM_FINO: __XFS_BTREE_STATS_INC(__mp, fibt, stat); break; \
-	case XFS_BTNUM_RMAP: __XFS_BTREE_STATS_INC(__mp, rmap, stat); break; \
-	case XFS_BTNUM_REFC: __XFS_BTREE_STATS_INC(__mp, refcbt, stat); break; \
-	case XFS_BTNUM_MAX: ASSERT(0); /* fucking gcc */ ; break;	\
-	}       \
-} while (0)
-
-#define __XFS_BTREE_STATS_ADD(mp, type, stat, val) \
-	XFS_STATS_ADD(mp, xs_ ## type ## _2_ ## stat, val)
-#define XFS_BTREE_STATS_ADD(cur, stat, val)  \
-do {    \
-	struct xfs_mount *__mp = cur->bc_mp; \
-	switch (cur->bc_btnum) {  \
-	case XFS_BTNUM_BNO:	\
-		__XFS_BTREE_STATS_ADD(__mp, abtb, stat, val); break; \
-	case XFS_BTNUM_CNT:	\
-		__XFS_BTREE_STATS_ADD(__mp, abtc, stat, val); break; \
-	case XFS_BTNUM_BMAP:	\
-		__XFS_BTREE_STATS_ADD(__mp, bmbt, stat, val); break; \
-	case XFS_BTNUM_INO:	\
-		__XFS_BTREE_STATS_ADD(__mp, ibt, stat, val); break; \
-	case XFS_BTNUM_FINO:	\
-		__XFS_BTREE_STATS_ADD(__mp, fibt, stat, val); break; \
-	case XFS_BTNUM_RMAP:	\
-		__XFS_BTREE_STATS_ADD(__mp, rmap, stat, val); break; \
-	case XFS_BTNUM_REFC:	\
-		__XFS_BTREE_STATS_ADD(__mp, refcbt, stat, val); break; \
-	case XFS_BTNUM_MAX: ASSERT(0); /* fucking gcc */ ; break; \
-	}       \
-} while (0)
+	XFS_STATS_INC_OFF((cur)->bc_mp, (cur)->bc_statoff + __XBTS_ ## stat)
+#define XFS_BTREE_STATS_ADD(cur, stat, val)	\
+	XFS_STATS_ADD_OFF((cur)->bc_mp, (cur)->bc_statoff + __XBTS_ ## stat, val)
 
 #define	XFS_BTREE_MAXLEVELS	9	/* max of all btrees */
 
@@ -253,6 +217,7 @@ typedef struct xfs_btree_cur
 	__uint8_t	bc_nlevels;	/* number of levels in the tree */
 	__uint8_t	bc_blocklog;	/* log2(blocksize) of btree blocks */
 	xfs_btnum_t	bc_btnum;	/* identifies which btree type */
+	int		bc_statoff;	/* offset of btre stats array */
 	union {
 		struct {			/* needed for BNO, CNT, INO */
 			struct xfs_buf	*agbp;	/* agf/agi buffer pointer */
diff --git a/fs/xfs/libxfs/xfs_cksum.h b/fs/xfs/libxfs/xfs_cksum.h
index fad1676ad8cd..a416c7cb23ea 100644
--- a/fs/xfs/libxfs/xfs_cksum.h
+++ b/fs/xfs/libxfs/xfs_cksum.h
@@ -6,10 +6,11 @@
 /*
  * Calculate the intermediate checksum for a buffer that has the CRC field
  * inside it.  The offset of the 32bit crc fields is passed as the
- * cksum_offset parameter.
+ * cksum_offset parameter. We do not modify the buffer during verification,
+ * hence we have to split the CRC calculation across the cksum_offset.
  */
 static inline __uint32_t
-xfs_start_cksum(char *buffer, size_t length, unsigned long cksum_offset)
+xfs_start_cksum_safe(char *buffer, size_t length, unsigned long cksum_offset)
 {
 	__uint32_t zero = 0;
 	__uint32_t crc;
@@ -26,6 +27,20 @@ xfs_start_cksum(char *buffer, size_t length, unsigned long cksum_offset)
 }
 
 /*
+ * Fast CRC method where the buffer is modified. Callers must have exclusive
+ * access to the buffer while the calculation takes place.
+ */
+static inline __uint32_t
+xfs_start_cksum_update(char *buffer, size_t length, unsigned long cksum_offset)
+{
+	/* zero the CRC field */
+	*(__le32 *)(buffer + cksum_offset) = 0;
+
+	/* single pass CRC calculation for the entire buffer */
+	return crc32c(XFS_CRC_SEED, buffer, length);
+}
+
+/*
  * Convert the intermediate checksum to the final ondisk format.
  *
  * The CRC32c calculation uses LE format even on BE machines, but returns the
@@ -40,11 +55,14 @@ xfs_end_cksum(__uint32_t crc)
 
 /*
  * Helper to generate the checksum for a buffer.
+ *
+ * This modifies the buffer temporarily - callers must have exclusive
+ * access to the buffer while the calculation takes place.
  */
 static inline void
 xfs_update_cksum(char *buffer, size_t length, unsigned long cksum_offset)
 {
-	__uint32_t crc = xfs_start_cksum(buffer, length, cksum_offset);
+	__uint32_t crc = xfs_start_cksum_update(buffer, length, cksum_offset);
 
 	*(__le32 *)(buffer + cksum_offset) = xfs_end_cksum(crc);
 }
@@ -55,7 +73,7 @@ xfs_update_cksum(char *buffer, size_t length, unsigned long cksum_offset)
 static inline int
 xfs_verify_cksum(char *buffer, size_t length, unsigned long cksum_offset)
 {
-	__uint32_t crc = xfs_start_cksum(buffer, length, cksum_offset);
+	__uint32_t crc = xfs_start_cksum_safe(buffer, length, cksum_offset);
 
 	return *(__le32 *)(buffer + cksum_offset) == xfs_end_cksum(crc);
 }
diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c
index 20a96dd5af7e..c58d72c220f5 100644
--- a/fs/xfs/libxfs/xfs_dir2.c
+++ b/fs/xfs/libxfs/xfs_dir2.c
@@ -93,7 +93,7 @@ xfs_ascii_ci_compname(
 	return result;
 }
 
-static struct xfs_nameops xfs_ascii_ci_nameops = {
+static const struct xfs_nameops xfs_ascii_ci_nameops = {
 	.hashname	= xfs_ascii_ci_hashname,
 	.compname	= xfs_ascii_ci_compname,
 };
diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h
index becc926c3e3d..0197590fa7d7 100644
--- a/fs/xfs/libxfs/xfs_dir2.h
+++ b/fs/xfs/libxfs/xfs_dir2.h
@@ -157,6 +157,9 @@ extern int xfs_dir2_isleaf(struct xfs_da_args *args, int *r);
 extern int xfs_dir2_shrink_inode(struct xfs_da_args *args, xfs_dir2_db_t db,
 				struct xfs_buf *bp);
 
+extern void xfs_dir2_data_freescan_int(struct xfs_da_geometry *geo,
+		const struct xfs_dir_ops *ops,
+		struct xfs_dir2_data_hdr *hdr, int *loghead);
 extern void xfs_dir2_data_freescan(struct xfs_inode *dp,
 		struct xfs_dir2_data_hdr *hdr, int *loghead);
 extern void xfs_dir2_data_log_entry(struct xfs_da_args *args,
@@ -177,6 +180,8 @@ extern struct xfs_dir2_data_free *xfs_dir2_data_freefind(
 		struct xfs_dir2_data_hdr *hdr, struct xfs_dir2_data_free *bf,
 		struct xfs_dir2_data_unused *dup);
 
+extern int xfs_dir_ino_validate(struct xfs_mount *mp, xfs_ino_t ino);
+
 extern const struct xfs_buf_ops xfs_dir3_block_buf_ops;
 extern const struct xfs_buf_ops xfs_dir3_leafn_buf_ops;
 extern const struct xfs_buf_ops xfs_dir3_leaf1_buf_ops;
diff --git a/fs/xfs/libxfs/xfs_dir2_data.c b/fs/xfs/libxfs/xfs_dir2_data.c
index 725fc7841fde..d478065b9544 100644
--- a/fs/xfs/libxfs/xfs_dir2_data.c
+++ b/fs/xfs/libxfs/xfs_dir2_data.c
@@ -329,7 +329,7 @@ xfs_dir3_data_read(
 
 	err = xfs_da_read_buf(tp, dp, bno, mapped_bno, bpp,
 				XFS_DATA_FORK, &xfs_dir3_data_buf_ops);
-	if (!err && tp)
+	if (!err && tp && *bpp)
 		xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_DATA_BUF);
 	return err;
 }
@@ -505,8 +505,9 @@ xfs_dir2_data_freeremove(
  * Given a data block, reconstruct its bestfree map.
  */
 void
-xfs_dir2_data_freescan(
-	struct xfs_inode	*dp,
+xfs_dir2_data_freescan_int(
+	struct xfs_da_geometry	*geo,
+	const struct xfs_dir_ops *ops,
 	struct xfs_dir2_data_hdr *hdr,
 	int			*loghead)
 {
@@ -516,7 +517,6 @@ xfs_dir2_data_freescan(
 	struct xfs_dir2_data_free *bf;
 	char			*endp;		/* end of block's data */
 	char			*p;		/* current entry pointer */
-	struct xfs_da_geometry	*geo = dp->i_mount->m_dir_geo;
 
 	ASSERT(hdr->magic == cpu_to_be32(XFS_DIR2_DATA_MAGIC) ||
 	       hdr->magic == cpu_to_be32(XFS_DIR3_DATA_MAGIC) ||
@@ -526,13 +526,13 @@ xfs_dir2_data_freescan(
 	/*
 	 * Start by clearing the table.
 	 */
-	bf = dp->d_ops->data_bestfree_p(hdr);
+	bf = ops->data_bestfree_p(hdr);
 	memset(bf, 0, sizeof(*bf) * XFS_DIR2_DATA_FD_COUNT);
 	*loghead = 1;
 	/*
 	 * Set up pointers.
 	 */
-	p = (char *)dp->d_ops->data_entry_p(hdr);
+	p = (char *)ops->data_entry_p(hdr);
 	if (hdr->magic == cpu_to_be32(XFS_DIR2_BLOCK_MAGIC) ||
 	    hdr->magic == cpu_to_be32(XFS_DIR3_BLOCK_MAGIC)) {
 		btp = xfs_dir2_block_tail_p(geo, hdr);
@@ -559,12 +559,22 @@ xfs_dir2_data_freescan(
 		else {
 			dep = (xfs_dir2_data_entry_t *)p;
 			ASSERT((char *)dep - (char *)hdr ==
-			       be16_to_cpu(*dp->d_ops->data_entry_tag_p(dep)));
-			p += dp->d_ops->data_entsize(dep->namelen);
+			       be16_to_cpu(*ops->data_entry_tag_p(dep)));
+			p += ops->data_entsize(dep->namelen);
 		}
 	}
 }
 
+void
+xfs_dir2_data_freescan(
+	struct xfs_inode	*dp,
+	struct xfs_dir2_data_hdr *hdr,
+	int			*loghead)
+{
+	return xfs_dir2_data_freescan_int(dp->i_mount->m_dir_geo, dp->d_ops,
+			hdr, loghead);
+}
+
 /*
  * Initialize a data block at the given block number in the directory.
  * Give back the buffer for the created block.
diff --git a/fs/xfs/libxfs/xfs_dir2_priv.h b/fs/xfs/libxfs/xfs_dir2_priv.h
index ef9f6ead96a4..d04547fcf274 100644
--- a/fs/xfs/libxfs/xfs_dir2_priv.h
+++ b/fs/xfs/libxfs/xfs_dir2_priv.h
@@ -21,7 +21,6 @@
 struct dir_context;
 
 /* xfs_dir2.c */
-extern int xfs_dir_ino_validate(struct xfs_mount *mp, xfs_ino_t ino);
 extern int xfs_dir2_grow_inode(struct xfs_da_args *args, int space,
 				xfs_dir2_db_t *dbp);
 extern int xfs_dir_cilookup_result(struct xfs_da_args *args,
diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c
index 51b4e0de1fdc..f272abff11e1 100644
--- a/fs/xfs/libxfs/xfs_ialloc.c
+++ b/fs/xfs/libxfs/xfs_ialloc.c
@@ -2344,7 +2344,8 @@ xfs_imap(
 
 		imap->im_blkno = XFS_AGB_TO_DADDR(mp, agno, agbno);
 		imap->im_len = XFS_FSB_TO_BB(mp, 1);
-		imap->im_boffset = (ushort)(offset << mp->m_sb.sb_inodelog);
+		imap->im_boffset = (unsigned short)(offset <<
+							mp->m_sb.sb_inodelog);
 		return 0;
 	}
 
@@ -2372,7 +2373,7 @@ out_map:
 
 	imap->im_blkno = XFS_AGB_TO_DADDR(mp, agno, cluster_agbno);
 	imap->im_len = XFS_FSB_TO_BB(mp, blks_per_cluster);
-	imap->im_boffset = (ushort)(offset << mp->m_sb.sb_inodelog);
+	imap->im_boffset = (unsigned short)(offset << mp->m_sb.sb_inodelog);
 
 	/*
 	 * If the inode number maps to a block outside the bounds
@@ -2450,8 +2451,6 @@ xfs_ialloc_log_agi(
 	ASSERT(agi->agi_magicnum == cpu_to_be32(XFS_AGI_MAGIC));
 #endif
 
-	xfs_trans_buf_set_type(tp, bp, XFS_BLFT_AGI_BUF);
-
 	/*
 	 * Compute byte offsets for the first and last fields in the first
 	 * region and log the agi buffer. This only logs up through
@@ -2512,8 +2511,15 @@ xfs_agi_verify(
 	if (!XFS_AGI_GOOD_VERSION(be32_to_cpu(agi->agi_versionnum)))
 		return false;
 
-	if (be32_to_cpu(agi->agi_level) > XFS_BTREE_MAXLEVELS)
+	if (be32_to_cpu(agi->agi_level) < 1 ||
+	    be32_to_cpu(agi->agi_level) > XFS_BTREE_MAXLEVELS)
+		return false;
+
+	if (xfs_sb_version_hasfinobt(&mp->m_sb) &&
+	    (be32_to_cpu(agi->agi_free_level) < 1 ||
+	     be32_to_cpu(agi->agi_free_level) > XFS_BTREE_MAXLEVELS))
 		return false;
+
 	/*
 	 * during growfs operations, the perag is not fully initialised,
 	 * so we can't use it for any useful checking. growfs ensures we can't
@@ -2592,6 +2598,8 @@ xfs_read_agi(
 			XFS_FSS_TO_BB(mp, 1), 0, bpp, &xfs_agi_buf_ops);
 	if (error)
 		return error;
+	if (tp)
+		xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_AGI_BUF);
 
 	xfs_buf_set_ref(*bpp, XFS_AGI_REF);
 	return 0;
diff --git a/fs/xfs/libxfs/xfs_ialloc_btree.c b/fs/xfs/libxfs/xfs_ialloc_btree.c
index eab68ae2e011..0fd086d03d41 100644
--- a/fs/xfs/libxfs/xfs_ialloc_btree.c
+++ b/fs/xfs/libxfs/xfs_ialloc_btree.c
@@ -357,7 +357,7 @@ xfs_inobt_init_cursor(
 	struct xfs_agi		*agi = XFS_BUF_TO_AGI(agbp);
 	struct xfs_btree_cur	*cur;
 
-	cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_SLEEP);
+	cur = kmem_zone_zalloc(xfs_btree_cur_zone, KM_NOFS);
 
 	cur->bc_tp = tp;
 	cur->bc_mp = mp;
@@ -365,9 +365,11 @@ xfs_inobt_init_cursor(
 	if (btnum == XFS_BTNUM_INO) {
 		cur->bc_nlevels = be32_to_cpu(agi->agi_level);
 		cur->bc_ops = &xfs_inobt_ops;
+		cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_ibt_2);
 	} else {
 		cur->bc_nlevels = be32_to_cpu(agi->agi_free_level);
 		cur->bc_ops = &xfs_finobt_ops;
+		cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_fibt_2);
 	}
 
 	cur->bc_blocklog = mp->m_sb.sb_blocklog;
diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c
index 134424fac434..dd483e2767f7 100644
--- a/fs/xfs/libxfs/xfs_inode_buf.c
+++ b/fs/xfs/libxfs/xfs_inode_buf.c
@@ -383,7 +383,7 @@ xfs_log_dinode_to_disk(
 static bool
 xfs_dinode_verify(
 	struct xfs_mount	*mp,
-	struct xfs_inode	*ip,
+	xfs_ino_t		ino,
 	struct xfs_dinode	*dip)
 {
 	uint16_t		flags;
@@ -392,6 +392,14 @@ xfs_dinode_verify(
 	if (dip->di_magic != cpu_to_be16(XFS_DINODE_MAGIC))
 		return false;
 
+	/* don't allow invalid i_size */
+	if (be64_to_cpu(dip->di_size) & (1ULL << 63))
+		return false;
+
+	/* No zero-length symlinks. */
+	if (S_ISLNK(be16_to_cpu(dip->di_mode)) && dip->di_size == 0)
+		return false;
+
 	/* only version 3 or greater inodes are extensively verified here */
 	if (dip->di_version < 3)
 		return true;
@@ -401,7 +409,7 @@ xfs_dinode_verify(
 	if (!xfs_verify_cksum((char *)dip, mp->m_sb.sb_inodesize,
 			      XFS_DINODE_CRC_OFF))
 		return false;
-	if (be64_to_cpu(dip->di_ino) != ip->i_ino)
+	if (be64_to_cpu(dip->di_ino) != ino)
 		return false;
 	if (!uuid_equal(&dip->di_uuid, &mp->m_sb.sb_meta_uuid))
 		return false;
@@ -436,7 +444,7 @@ xfs_dinode_calc_crc(
 		return;
 
 	ASSERT(xfs_sb_version_hascrc(&mp->m_sb));
-	crc = xfs_start_cksum((char *)dip, mp->m_sb.sb_inodesize,
+	crc = xfs_start_cksum_update((char *)dip, mp->m_sb.sb_inodesize,
 			      XFS_DINODE_CRC_OFF);
 	dip->di_crc = xfs_end_cksum(crc);
 }
@@ -493,7 +501,7 @@ xfs_iread(
 		return error;
 
 	/* even unallocated inodes are verified */
-	if (!xfs_dinode_verify(mp, ip, dip)) {
+	if (!xfs_dinode_verify(mp, ip->i_ino, dip)) {
 		xfs_alert(mp, "%s: validation failed for inode %lld failed",
 				__func__, ip->i_ino);
 
diff --git a/fs/xfs/libxfs/xfs_inode_buf.h b/fs/xfs/libxfs/xfs_inode_buf.h
index 3cfe12a4f58a..6848a0afbce7 100644
--- a/fs/xfs/libxfs/xfs_inode_buf.h
+++ b/fs/xfs/libxfs/xfs_inode_buf.h
@@ -58,8 +58,8 @@ struct xfs_icdinode {
  */
 struct xfs_imap {
 	xfs_daddr_t	im_blkno;	/* starting BB of inode chunk */
-	ushort		im_len;		/* length in BBs of inode chunk */
-	ushort		im_boffset;	/* inode offset in block in bytes */
+	unsigned short	im_len;		/* length in BBs of inode chunk */
+	unsigned short	im_boffset;	/* inode offset in block in bytes */
 };
 
 int	xfs_imap_to_bp(struct xfs_mount *, struct xfs_trans *,
diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c
index 5dd56d3dbb3a..222e103356c6 100644
--- a/fs/xfs/libxfs/xfs_inode_fork.c
+++ b/fs/xfs/libxfs/xfs_inode_fork.c
@@ -775,6 +775,13 @@ xfs_idestroy_fork(
 	}
 }
 
+/* Count number of incore extents based on if_bytes */
+xfs_extnum_t
+xfs_iext_count(struct xfs_ifork *ifp)
+{
+	return ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+}
+
 /*
  * Convert in-core extents to on-disk form
  *
@@ -803,7 +810,7 @@ xfs_iextents_copy(
 	ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL|XFS_ILOCK_SHARED));
 	ASSERT(ifp->if_bytes > 0);
 
-	nrecs = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	nrecs = xfs_iext_count(ifp);
 	XFS_BMAP_TRACE_EXLIST(ip, nrecs, whichfork);
 	ASSERT(nrecs > 0);
 
@@ -941,7 +948,7 @@ xfs_iext_get_ext(
 	xfs_extnum_t	idx)		/* index of target extent */
 {
 	ASSERT(idx >= 0);
-	ASSERT(idx < ifp->if_bytes / sizeof(xfs_bmbt_rec_t));
+	ASSERT(idx < xfs_iext_count(ifp));
 
 	if ((ifp->if_flags & XFS_IFEXTIREC) && (idx == 0)) {
 		return ifp->if_u1.if_ext_irec->er_extbuf;
@@ -1017,7 +1024,7 @@ xfs_iext_add(
 	int		new_size;	/* size of extents after adding */
 	xfs_extnum_t	nextents;	/* number of extents in file */
 
-	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	nextents = xfs_iext_count(ifp);
 	ASSERT((idx >= 0) && (idx <= nextents));
 	byte_diff = ext_diff * sizeof(xfs_bmbt_rec_t);
 	new_size = ifp->if_bytes + byte_diff;
@@ -1241,7 +1248,7 @@ xfs_iext_remove(
 	trace_xfs_iext_remove(ip, idx, state, _RET_IP_);
 
 	ASSERT(ext_diff > 0);
-	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	nextents = xfs_iext_count(ifp);
 	new_size = (nextents - ext_diff) * sizeof(xfs_bmbt_rec_t);
 
 	if (new_size == 0) {
@@ -1270,7 +1277,7 @@ xfs_iext_remove_inline(
 
 	ASSERT(!(ifp->if_flags & XFS_IFEXTIREC));
 	ASSERT(idx < XFS_INLINE_EXTS);
-	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	nextents = xfs_iext_count(ifp);
 	ASSERT(((nextents - ext_diff) > 0) &&
 		(nextents - ext_diff) < XFS_INLINE_EXTS);
 
@@ -1309,7 +1316,7 @@ xfs_iext_remove_direct(
 	ASSERT(!(ifp->if_flags & XFS_IFEXTIREC));
 	new_size = ifp->if_bytes -
 		(ext_diff * sizeof(xfs_bmbt_rec_t));
-	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	nextents = xfs_iext_count(ifp);
 
 	if (new_size == 0) {
 		xfs_iext_destroy(ifp);
@@ -1546,7 +1553,7 @@ xfs_iext_indirect_to_direct(
 	int		size;		/* size of file extents */
 
 	ASSERT(ifp->if_flags & XFS_IFEXTIREC);
-	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	nextents = xfs_iext_count(ifp);
 	ASSERT(nextents <= XFS_LINEAR_EXTS);
 	size = nextents * sizeof(xfs_bmbt_rec_t);
 
@@ -1620,7 +1627,7 @@ xfs_iext_bno_to_ext(
 	xfs_extnum_t	nextents;	/* number of file extents */
 	xfs_fileoff_t	startoff = 0;	/* start offset of extent */
 
-	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	nextents = xfs_iext_count(ifp);
 	if (nextents == 0) {
 		*idxp = 0;
 		return NULL;
@@ -1733,8 +1740,8 @@ xfs_iext_idx_to_irec(
 
 	ASSERT(ifp->if_flags & XFS_IFEXTIREC);
 	ASSERT(page_idx >= 0);
-	ASSERT(page_idx <= ifp->if_bytes / sizeof(xfs_bmbt_rec_t));
-	ASSERT(page_idx < ifp->if_bytes / sizeof(xfs_bmbt_rec_t) || realloc);
+	ASSERT(page_idx <= xfs_iext_count(ifp));
+	ASSERT(page_idx < xfs_iext_count(ifp) || realloc);
 
 	nlists = ifp->if_real_bytes / XFS_IEXT_BUFSZ;
 	erp_idx = 0;
@@ -1782,7 +1789,7 @@ xfs_iext_irec_init(
 	xfs_extnum_t	nextents;	/* number of extents in file */
 
 	ASSERT(!(ifp->if_flags & XFS_IFEXTIREC));
-	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	nextents = xfs_iext_count(ifp);
 	ASSERT(nextents <= XFS_LINEAR_EXTS);
 
 	erp = kmem_alloc(sizeof(xfs_ext_irec_t), KM_NOFS);
@@ -1906,7 +1913,7 @@ xfs_iext_irec_compact(
 
 	ASSERT(ifp->if_flags & XFS_IFEXTIREC);
 	nlists = ifp->if_real_bytes / XFS_IEXT_BUFSZ;
-	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	nextents = xfs_iext_count(ifp);
 
 	if (nextents == 0) {
 		xfs_iext_destroy(ifp);
@@ -1996,3 +2003,49 @@ xfs_ifork_init_cow(
 	ip->i_cformat = XFS_DINODE_FMT_EXTENTS;
 	ip->i_cnextents = 0;
 }
+
+/*
+ * Lookup the extent covering bno.
+ *
+ * If there is an extent covering bno return the extent index, and store the
+ * expanded extent structure in *gotp, and the extent index in *idx.
+ * If there is no extent covering bno, but there is an extent after it (e.g.
+ * it lies in a hole) return that extent in *gotp and its index in *idx
+ * instead.
+ * If bno is beyond the last extent return false, and return the index after
+ * the last valid index in *idxp.
+ */
+bool
+xfs_iext_lookup_extent(
+	struct xfs_inode	*ip,
+	struct xfs_ifork	*ifp,
+	xfs_fileoff_t		bno,
+	xfs_extnum_t		*idxp,
+	struct xfs_bmbt_irec	*gotp)
+{
+	struct xfs_bmbt_rec_host *ep;
+
+	XFS_STATS_INC(ip->i_mount, xs_look_exlist);
+
+	ep = xfs_iext_bno_to_ext(ifp, bno, idxp);
+	if (!ep)
+		return false;
+	xfs_bmbt_get_all(ep, gotp);
+	return true;
+}
+
+/*
+ * Return true if there is an extent at index idx, and return the expanded
+ * extent structure at idx in that case.  Else return false.
+ */
+bool
+xfs_iext_get_extent(
+	struct xfs_ifork	*ifp,
+	xfs_extnum_t		idx,
+	struct xfs_bmbt_irec	*gotp)
+{
+	if (idx < 0 || idx >= xfs_iext_count(ifp))
+		return false;
+	xfs_bmbt_get_all(xfs_iext_get_ext(ifp, idx), gotp);
+	return true;
+}
diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h
index c9476f50e32d..7fb8365326d1 100644
--- a/fs/xfs/libxfs/xfs_inode_fork.h
+++ b/fs/xfs/libxfs/xfs_inode_fork.h
@@ -152,6 +152,7 @@ void		xfs_init_local_fork(struct xfs_inode *, int, const void *, int);
 
 struct xfs_bmbt_rec_host *
 		xfs_iext_get_ext(struct xfs_ifork *, xfs_extnum_t);
+xfs_extnum_t	xfs_iext_count(struct xfs_ifork *);
 void		xfs_iext_insert(struct xfs_inode *, xfs_extnum_t, xfs_extnum_t,
 				struct xfs_bmbt_irec *, int);
 void		xfs_iext_add(struct xfs_ifork *, xfs_extnum_t, int);
@@ -181,6 +182,12 @@ void		xfs_iext_irec_compact_pages(struct xfs_ifork *);
 void		xfs_iext_irec_compact_full(struct xfs_ifork *);
 void		xfs_iext_irec_update_extoffs(struct xfs_ifork *, int, int);
 
+bool		xfs_iext_lookup_extent(struct xfs_inode *ip,
+			struct xfs_ifork *ifp, xfs_fileoff_t bno,
+			xfs_extnum_t *idxp, struct xfs_bmbt_irec *gotp);
+bool		xfs_iext_get_extent(struct xfs_ifork *ifp, xfs_extnum_t idx,
+			struct xfs_bmbt_irec *gotp);
+
 extern struct kmem_zone	*xfs_ifork_zone;
 
 extern void xfs_ifork_init_cow(struct xfs_inode *ip);
diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h
index 083cdd6d6c28..7ae571f8e34a 100644
--- a/fs/xfs/libxfs/xfs_log_format.h
+++ b/fs/xfs/libxfs/xfs_log_format.h
@@ -481,8 +481,8 @@ static inline uint xfs_log_dinode_size(int version)
 typedef struct xfs_buf_log_format {
 	unsigned short	blf_type;	/* buf log item type indicator */
 	unsigned short	blf_size;	/* size of this item */
-	ushort		blf_flags;	/* misc state */
-	ushort		blf_len;	/* number of blocks in this buf */
+	unsigned short	blf_flags;	/* misc state */
+	unsigned short	blf_len;	/* number of blocks in this buf */
 	__int64_t	blf_blkno;	/* starting blkno of this buf */
 	unsigned int	blf_map_size;	/* used size of data bitmap in words */
 	unsigned int	blf_data_map[XFS_BLF_DATAMAP_SIZE]; /* dirty bitmap */
diff --git a/fs/xfs/libxfs/xfs_log_recover.h b/fs/xfs/libxfs/xfs_log_recover.h
index 8e385f91d660..d9f65e2d5cc8 100644
--- a/fs/xfs/libxfs/xfs_log_recover.h
+++ b/fs/xfs/libxfs/xfs_log_recover.h
@@ -52,7 +52,7 @@ typedef struct xlog_recover {
 	struct list_head	r_itemq;	/* q for items */
 } xlog_recover_t;
 
-#define ITEM_TYPE(i)	(*(ushort *)(i)->ri_buf[0].i_addr)
+#define ITEM_TYPE(i)	(*(unsigned short *)(i)->ri_buf[0].i_addr)
 
 /*
  * This is the number of entries in the l_buf_cancel_table used during
diff --git a/fs/xfs/libxfs/xfs_refcount_btree.c b/fs/xfs/libxfs/xfs_refcount_btree.c
index 453bb2757ec2..6fb2215f8ff7 100644
--- a/fs/xfs/libxfs/xfs_refcount_btree.c
+++ b/fs/xfs/libxfs/xfs_refcount_btree.c
@@ -354,6 +354,7 @@ xfs_refcountbt_init_cursor(
 	cur->bc_btnum = XFS_BTNUM_REFC;
 	cur->bc_blocklog = mp->m_sb.sb_blocklog;
 	cur->bc_ops = &xfs_refcountbt_ops;
+	cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_refcbt_2);
 
 	cur->bc_nlevels = be32_to_cpu(agf->agf_refcount_level);
 
diff --git a/fs/xfs/libxfs/xfs_rmap_btree.c b/fs/xfs/libxfs/xfs_rmap_btree.c
index 83e672ff7577..de25771764ba 100644
--- a/fs/xfs/libxfs/xfs_rmap_btree.c
+++ b/fs/xfs/libxfs/xfs_rmap_btree.c
@@ -484,6 +484,7 @@ xfs_rmapbt_init_cursor(
 	cur->bc_blocklog = mp->m_sb.sb_blocklog;
 	cur->bc_ops = &xfs_rmapbt_ops;
 	cur->bc_nlevels = be32_to_cpu(agf->agf_levels[XFS_BTNUM_RMAP]);
+	cur->bc_statoff = XFS_STATS_CALC_INDEX(xs_rmap_2);
 
 	cur->bc_private.a.agbp = agbp;
 	cur->bc_private.a.agno = agno;
diff --git a/fs/xfs/libxfs/xfs_rtbitmap.c b/fs/xfs/libxfs/xfs_rtbitmap.c
index e2e1106c9fad..ea45584a9913 100644
--- a/fs/xfs/libxfs/xfs_rtbitmap.c
+++ b/fs/xfs/libxfs/xfs_rtbitmap.c
@@ -1016,4 +1016,3 @@ xfs_rtfree_extent(
 	}
 	return 0;
 }
-
diff --git a/fs/xfs/libxfs/xfs_sb.c b/fs/xfs/libxfs/xfs_sb.c
index a70aec910626..2580262e4ea0 100644
--- a/fs/xfs/libxfs/xfs_sb.c
+++ b/fs/xfs/libxfs/xfs_sb.c
@@ -262,6 +262,12 @@ xfs_mount_validate_sb(
 		return -EFSCORRUPTED;
 	}
 
+	if (xfs_sb_version_hascrc(&mp->m_sb) &&
+	    sbp->sb_blocksize < XFS_MIN_CRC_BLOCKSIZE) {
+		xfs_notice(mp, "v5 SB sanity check failed");
+		return -EFSCORRUPTED;
+	}
+
 	/*
 	 * Until this is fixed only page-sized or smaller data blocks work.
 	 */
@@ -338,13 +344,16 @@ xfs_sb_quota_from_disk(struct xfs_sb *sbp)
 					XFS_PQUOTA_CHKD : XFS_GQUOTA_CHKD;
 	sbp->sb_qflags &= ~(XFS_OQUOTA_ENFD | XFS_OQUOTA_CHKD);
 
-	if (sbp->sb_qflags & XFS_PQUOTA_ACCT)  {
+	if (sbp->sb_qflags & XFS_PQUOTA_ACCT &&
+	    sbp->sb_gquotino != NULLFSINO)  {
 		/*
 		 * In older version of superblock, on-disk superblock only
 		 * has sb_gquotino, and in-core superblock has both sb_gquotino
 		 * and sb_pquotino. But, only one of them is supported at any
 		 * point of time. So, if PQUOTA is set in disk superblock,
-		 * copy over sb_gquotino to sb_pquotino.
+		 * copy over sb_gquotino to sb_pquotino.  The NULLFSINO test
+		 * above is to make sure we don't do this twice and wipe them
+		 * both out!
 		 */
 		sbp->sb_pquotino = sbp->sb_gquotino;
 		sbp->sb_gquotino = NULLFSINO;
diff --git a/fs/xfs/libxfs/xfs_types.h b/fs/xfs/libxfs/xfs_types.h
index 8d74870468c2..717909f2f7b7 100644
--- a/fs/xfs/libxfs/xfs_types.h
+++ b/fs/xfs/libxfs/xfs_types.h
@@ -57,7 +57,6 @@ typedef __int64_t	xfs_sfiloff_t;	/* signed block number in a file */
 
 #define	NULLAGBLOCK	((xfs_agblock_t)-1)
 #define	NULLAGNUMBER	((xfs_agnumber_t)-1)
-#define	NULLEXTNUM	((xfs_extnum_t)-1)
 
 #define NULLCOMMITLSN	((xfs_lsn_t)-1)
 
@@ -75,11 +74,14 @@ typedef __int64_t	xfs_sfiloff_t;	/* signed block number in a file */
  * Minimum and maximum blocksize and sectorsize.
  * The blocksize upper limit is pretty much arbitrary.
  * The sectorsize upper limit is due to sizeof(sb_sectsize).
+ * CRC enable filesystems use 512 byte inodes, meaning 512 byte block sizes
+ * cannot be used.
  */
 #define XFS_MIN_BLOCKSIZE_LOG	9	/* i.e. 512 bytes */
 #define XFS_MAX_BLOCKSIZE_LOG	16	/* i.e. 65536 bytes */
 #define XFS_MIN_BLOCKSIZE	(1 << XFS_MIN_BLOCKSIZE_LOG)
 #define XFS_MAX_BLOCKSIZE	(1 << XFS_MAX_BLOCKSIZE_LOG)
+#define XFS_MIN_CRC_BLOCKSIZE	(1 << (XFS_MIN_BLOCKSIZE_LOG + 1))
 #define XFS_MIN_SECTORSIZE_LOG	9	/* i.e. 512 bytes */
 #define XFS_MAX_SECTORSIZE_LOG	15	/* i.e. 32768 bytes */
 #define XFS_MIN_SECTORSIZE	(1 << XFS_MIN_SECTORSIZE_LOG)
diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c
index 3e57a56cf829..0f56fcd3a5d5 100644
--- a/fs/xfs/xfs_aops.c
+++ b/fs/xfs/xfs_aops.c
@@ -37,11 +37,6 @@
 #include <linux/pagevec.h>
 #include <linux/writeback.h>
 
-/* flags for direct write completions */
-#define XFS_DIO_FLAG_UNWRITTEN	(1 << 0)
-#define XFS_DIO_FLAG_APPEND	(1 << 1)
-#define XFS_DIO_FLAG_COW	(1 << 2)
-
 /*
  * structure owned by writepages passed to individual writepage calls
  */
@@ -495,8 +490,8 @@ xfs_submit_ioend(
 
 	ioend->io_bio->bi_private = ioend;
 	ioend->io_bio->bi_end_io = xfs_end_bio;
-	bio_set_op_attrs(ioend->io_bio, REQ_OP_WRITE,
-			 (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : 0);
+	ioend->io_bio->bi_opf = REQ_OP_WRITE | wbc_to_write_flags(wbc);
+
 	/*
 	 * If we are failing the IO now, just mark the ioend with an
 	 * error and finish it. This will run IO completion immediately
@@ -567,8 +562,7 @@ xfs_chain_bio(
 
 	bio_chain(ioend->io_bio, new);
 	bio_get(ioend->io_bio);		/* for xfs_destroy_ioend */
-	bio_set_op_attrs(ioend->io_bio, REQ_OP_WRITE,
-			  (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : 0);
+	ioend->io_bio->bi_opf = REQ_OP_WRITE | wbc_to_write_flags(wbc);
 	submit_bio(ioend->io_bio);
 	ioend->io_bio = new;
 }
@@ -777,7 +771,7 @@ xfs_map_cow(
 {
 	struct xfs_inode	*ip = XFS_I(inode);
 	struct xfs_bmbt_irec	imap;
-	bool			is_cow = false, need_alloc = false;
+	bool			is_cow = false;
 	int			error;
 
 	/*
@@ -795,7 +789,7 @@ xfs_map_cow(
 	 * Else we need to check if there is a COW mapping at this offset.
 	 */
 	xfs_ilock(ip, XFS_ILOCK_SHARED);
-	is_cow = xfs_reflink_find_cow_mapping(ip, offset, &imap, &need_alloc);
+	is_cow = xfs_reflink_find_cow_mapping(ip, offset, &imap);
 	xfs_iunlock(ip, XFS_ILOCK_SHARED);
 
 	if (!is_cow)
@@ -805,7 +799,7 @@ xfs_map_cow(
 	 * And if the COW mapping has a delayed extent here we need to
 	 * allocate real space for it now.
 	 */
-	if (need_alloc) {
+	if (isnullstartblock(imap.br_startblock)) {
 		error = xfs_iomap_write_allocate(ip, XFS_COW_FORK, offset,
 				&imap);
 		if (error)
@@ -1176,45 +1170,6 @@ xfs_vm_releasepage(
 }
 
 /*
- * When we map a DIO buffer, we may need to pass flags to
- * xfs_end_io_direct_write to tell it what kind of write IO we are doing.
- *
- * Note that for DIO, an IO to the highest supported file block offset (i.e.
- * 2^63 - 1FSB bytes) will result in the offset + count overflowing a signed 64
- * bit variable. Hence if we see this overflow, we have to assume that the IO is
- * extending the file size. We won't know for sure until IO completion is run
- * and the actual max write offset is communicated to the IO completion
- * routine.
- */
-static void
-xfs_map_direct(
-	struct inode		*inode,
-	struct buffer_head	*bh_result,
-	struct xfs_bmbt_irec	*imap,
-	xfs_off_t		offset,
-	bool			is_cow)
-{
-	uintptr_t		*flags = (uintptr_t *)&bh_result->b_private;
-	xfs_off_t		size = bh_result->b_size;
-
-	trace_xfs_get_blocks_map_direct(XFS_I(inode), offset, size,
-		ISUNWRITTEN(imap) ? XFS_IO_UNWRITTEN : is_cow ? XFS_IO_COW :
-		XFS_IO_OVERWRITE, imap);
-
-	if (ISUNWRITTEN(imap)) {
-		*flags |= XFS_DIO_FLAG_UNWRITTEN;
-		set_buffer_defer_completion(bh_result);
-	} else if (is_cow) {
-		*flags |= XFS_DIO_FLAG_COW;
-		set_buffer_defer_completion(bh_result);
-	}
-	if (offset + size > i_size_read(inode) || offset + size < 0) {
-		*flags |= XFS_DIO_FLAG_APPEND;
-		set_buffer_defer_completion(bh_result);
-	}
-}
-
-/*
  * If this is O_DIRECT or the mpage code calling tell them how large the mapping
  * is, so that we can avoid repeated get_blocks calls.
  *
@@ -1254,52 +1209,12 @@ xfs_map_trim_size(
 	bh_result->b_size = mapping_size;
 }
 
-/* Bounce unaligned directio writes to the page cache. */
 static int
-xfs_bounce_unaligned_dio_write(
-	struct xfs_inode	*ip,
-	xfs_fileoff_t		offset_fsb,
-	struct xfs_bmbt_irec	*imap)
-{
-	struct xfs_bmbt_irec	irec;
-	xfs_fileoff_t		delta;
-	bool			shared;
-	bool			x;
-	int			error;
-
-	irec = *imap;
-	if (offset_fsb > irec.br_startoff) {
-		delta = offset_fsb - irec.br_startoff;
-		irec.br_blockcount -= delta;
-		irec.br_startblock += delta;
-		irec.br_startoff = offset_fsb;
-	}
-	error = xfs_reflink_trim_around_shared(ip, &irec, &shared, &x);
-	if (error)
-		return error;
-
-	/*
-	 * We're here because we're trying to do a directio write to a
-	 * region that isn't aligned to a filesystem block.  If any part
-	 * of the extent is shared, fall back to buffered mode to handle
-	 * the RMW.  This is done by returning -EREMCHG ("remote addr
-	 * changed"), which is caught further up the call stack.
-	 */
-	if (shared) {
-		trace_xfs_reflink_bounce_dio_write(ip, imap);
-		return -EREMCHG;
-	}
-	return 0;
-}
-
-STATIC int
-__xfs_get_blocks(
+xfs_get_blocks(
 	struct inode		*inode,
 	sector_t		iblock,
 	struct buffer_head	*bh_result,
-	int			create,
-	bool			direct,
-	bool			dax_fault)
+	int			create)
 {
 	struct xfs_inode	*ip = XFS_I(inode);
 	struct xfs_mount	*mp = ip->i_mount;
@@ -1310,11 +1225,8 @@ __xfs_get_blocks(
 	int			nimaps = 1;
 	xfs_off_t		offset;
 	ssize_t			size;
-	int			new = 0;
-	bool			is_cow = false;
-	bool			need_alloc = false;
 
-	BUG_ON(create && !direct);
+	BUG_ON(create);
 
 	if (XFS_FORCED_SHUTDOWN(mp))
 		return -EIO;
@@ -1323,7 +1235,7 @@ __xfs_get_blocks(
 	ASSERT(bh_result->b_size >= (1 << inode->i_blkbits));
 	size = bh_result->b_size;
 
-	if (!create && offset >= i_size_read(inode))
+	if (offset >= i_size_read(inode))
 		return 0;
 
 	/*
@@ -1338,52 +1250,12 @@ __xfs_get_blocks(
 	end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + size);
 	offset_fsb = XFS_B_TO_FSBT(mp, offset);
 
-	if (create && direct && xfs_is_reflink_inode(ip))
-		is_cow = xfs_reflink_find_cow_mapping(ip, offset, &imap,
-					&need_alloc);
-	if (!is_cow) {
-		error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb,
-					&imap, &nimaps, XFS_BMAPI_ENTIRE);
-		/*
-		 * Truncate an overwrite extent if there's a pending CoW
-		 * reservation before the end of this extent.  This
-		 * forces us to come back to get_blocks to take care of
-		 * the CoW.
-		 */
-		if (create && direct && nimaps &&
-		    imap.br_startblock != HOLESTARTBLOCK &&
-		    imap.br_startblock != DELAYSTARTBLOCK &&
-		    !ISUNWRITTEN(&imap))
-			xfs_reflink_trim_irec_to_next_cow(ip, offset_fsb,
-					&imap);
-	}
-	ASSERT(!need_alloc);
+	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb,
+				&imap, &nimaps, XFS_BMAPI_ENTIRE);
 	if (error)
 		goto out_unlock;
 
-	/* for DAX, we convert unwritten extents directly */
-	if (create &&
-	    (!nimaps ||
-	     (imap.br_startblock == HOLESTARTBLOCK ||
-	      imap.br_startblock == DELAYSTARTBLOCK) ||
-	     (IS_DAX(inode) && ISUNWRITTEN(&imap)))) {
-		/*
-		 * xfs_iomap_write_direct() expects the shared lock. It
-		 * is unlocked on return.
-		 */
-		if (lockmode == XFS_ILOCK_EXCL)
-			xfs_ilock_demote(ip, lockmode);
-
-		error = xfs_iomap_write_direct(ip, offset, size,
-					       &imap, nimaps);
-		if (error)
-			return error;
-		new = 1;
-
-		trace_xfs_get_blocks_alloc(ip, offset, size,
-				ISUNWRITTEN(&imap) ? XFS_IO_UNWRITTEN
-						   : XFS_IO_DELALLOC, &imap);
-	} else if (nimaps) {
+	if (nimaps) {
 		trace_xfs_get_blocks_found(ip, offset, size,
 				ISUNWRITTEN(&imap) ? XFS_IO_UNWRITTEN
 						   : XFS_IO_OVERWRITE, &imap);
@@ -1393,12 +1265,6 @@ __xfs_get_blocks(
 		goto out_unlock;
 	}
 
-	if (IS_DAX(inode) && create) {
-		ASSERT(!ISUNWRITTEN(&imap));
-		/* zeroing is not needed at a higher layer */
-		new = 0;
-	}
-
 	/* trim mapping down to size requested */
 	xfs_map_trim_size(inode, iblock, bh_result, &imap, offset, size);
 
@@ -1408,50 +1274,14 @@ __xfs_get_blocks(
 	 */
 	if (imap.br_startblock != HOLESTARTBLOCK &&
 	    imap.br_startblock != DELAYSTARTBLOCK &&
-	    (create || !ISUNWRITTEN(&imap))) {
-		if (create && direct && !is_cow) {
-			error = xfs_bounce_unaligned_dio_write(ip, offset_fsb,
-					&imap);
-			if (error)
-				return error;
-		}
-
+	    !ISUNWRITTEN(&imap))
 		xfs_map_buffer(inode, bh_result, &imap, offset);
-		if (ISUNWRITTEN(&imap))
-			set_buffer_unwritten(bh_result);
-		/* direct IO needs special help */
-		if (create) {
-			if (dax_fault)
-				ASSERT(!ISUNWRITTEN(&imap));
-			else
-				xfs_map_direct(inode, bh_result, &imap, offset,
-						is_cow);
-		}
-	}
 
 	/*
 	 * If this is a realtime file, data may be on a different device.
 	 * to that pointed to from the buffer_head b_bdev currently.
 	 */
 	bh_result->b_bdev = xfs_find_bdev_for_inode(inode);
-
-	/*
-	 * If we previously allocated a block out beyond eof and we are now
-	 * coming back to use it then we will need to flag it as new even if it
-	 * has a disk address.
-	 *
-	 * With sub-block writes into unwritten extents we also need to mark
-	 * the buffer as new so that the unwritten parts of the buffer gets
-	 * correctly zeroed.
-	 */
-	if (create &&
-	    ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) ||
-	     (offset >= i_size_read(inode)) ||
-	     (new || ISUNWRITTEN(&imap))))
-		set_buffer_new(bh_result);
-
-	BUG_ON(direct && imap.br_startblock == DELAYSTARTBLOCK);
-
 	return 0;
 
 out_unlock:
@@ -1459,110 +1289,6 @@ out_unlock:
 	return error;
 }
 
-int
-xfs_get_blocks(
-	struct inode		*inode,
-	sector_t		iblock,
-	struct buffer_head	*bh_result,
-	int			create)
-{
-	return __xfs_get_blocks(inode, iblock, bh_result, create, false, false);
-}
-
-int
-xfs_get_blocks_direct(
-	struct inode		*inode,
-	sector_t		iblock,
-	struct buffer_head	*bh_result,
-	int			create)
-{
-	return __xfs_get_blocks(inode, iblock, bh_result, create, true, false);
-}
-
-int
-xfs_get_blocks_dax_fault(
-	struct inode		*inode,
-	sector_t		iblock,
-	struct buffer_head	*bh_result,
-	int			create)
-{
-	return __xfs_get_blocks(inode, iblock, bh_result, create, true, true);
-}
-
-/*
- * Complete a direct I/O write request.
- *
- * xfs_map_direct passes us some flags in the private data to tell us what to
- * do.  If no flags are set, then the write IO is an overwrite wholly within
- * the existing allocated file size and so there is nothing for us to do.
- *
- * Note that in this case the completion can be called in interrupt context,
- * whereas if we have flags set we will always be called in task context
- * (i.e. from a workqueue).
- */
-int
-xfs_end_io_direct_write(
-	struct kiocb		*iocb,
-	loff_t			offset,
-	ssize_t			size,
-	void			*private)
-{
-	struct inode		*inode = file_inode(iocb->ki_filp);
-	struct xfs_inode	*ip = XFS_I(inode);
-	uintptr_t		flags = (uintptr_t)private;
-	int			error = 0;
-
-	trace_xfs_end_io_direct_write(ip, offset, size);
-
-	if (XFS_FORCED_SHUTDOWN(ip->i_mount))
-		return -EIO;
-
-	if (size <= 0)
-		return size;
-
-	/*
-	 * The flags tell us whether we are doing unwritten extent conversions
-	 * or an append transaction that updates the on-disk file size. These
-	 * cases are the only cases where we should *potentially* be needing
-	 * to update the VFS inode size.
-	 */
-	if (flags == 0) {
-		ASSERT(offset + size <= i_size_read(inode));
-		return 0;
-	}
-
-	/*
-	 * We need to update the in-core inode size here so that we don't end up
-	 * with the on-disk inode size being outside the in-core inode size. We
-	 * have no other method of updating EOF for AIO, so always do it here
-	 * if necessary.
-	 *
-	 * We need to lock the test/set EOF update as we can be racing with
-	 * other IO completions here to update the EOF. Failing to serialise
-	 * here can result in EOF moving backwards and Bad Things Happen when
-	 * that occurs.
-	 */
-	spin_lock(&ip->i_flags_lock);
-	if (offset + size > i_size_read(inode))
-		i_size_write(inode, offset + size);
-	spin_unlock(&ip->i_flags_lock);
-
-	if (flags & XFS_DIO_FLAG_COW)
-		error = xfs_reflink_end_cow(ip, offset, size);
-	if (flags & XFS_DIO_FLAG_UNWRITTEN) {
-		trace_xfs_end_io_direct_write_unwritten(ip, offset, size);
-
-		error = xfs_iomap_write_unwritten(ip, offset, size);
-	}
-	if (flags & XFS_DIO_FLAG_APPEND) {
-		trace_xfs_end_io_direct_write_append(ip, offset, size);
-
-		error = xfs_setfilesize(ip, offset, size);
-	}
-
-	return error;
-}
-
 STATIC ssize_t
 xfs_vm_direct_IO(
 	struct kiocb		*iocb,
@@ -1583,7 +1309,6 @@ xfs_vm_bmap(
 	struct xfs_inode	*ip = XFS_I(inode);
 
 	trace_xfs_vm_bmap(XFS_I(inode));
-	xfs_ilock(ip, XFS_IOLOCK_SHARED);
 
 	/*
 	 * The swap code (ab-)uses ->bmap to get a block mapping and then
@@ -1591,12 +1316,10 @@ xfs_vm_bmap(
 	 * that on reflinks inodes, so we have to skip out here.  And yes,
 	 * 0 is the magic code for a bmap error..
 	 */
-	if (xfs_is_reflink_inode(ip)) {
-		xfs_iunlock(ip, XFS_IOLOCK_SHARED);
+	if (xfs_is_reflink_inode(ip))
 		return 0;
-	}
+
 	filemap_write_and_wait(mapping);
-	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
 	return generic_block_bmap(mapping, block, xfs_get_blocks);
 }
 
diff --git a/fs/xfs/xfs_aops.h b/fs/xfs/xfs_aops.h
index b3c6634f9518..cc174ec6c2fd 100644
--- a/fs/xfs/xfs_aops.h
+++ b/fs/xfs/xfs_aops.h
@@ -55,15 +55,6 @@ struct xfs_ioend {
 
 extern const struct address_space_operations xfs_address_space_operations;
 
-int	xfs_get_blocks(struct inode *inode, sector_t offset,
-		       struct buffer_head *map_bh, int create);
-int	xfs_get_blocks_direct(struct inode *inode, sector_t offset,
-			      struct buffer_head *map_bh, int create);
-int	xfs_get_blocks_dax_fault(struct inode *inode, sector_t offset,
-			         struct buffer_head *map_bh, int create);
-
-int	xfs_end_io_direct_write(struct kiocb *iocb, loff_t offset,
-		ssize_t size, void *private);
 int	xfs_setfilesize(struct xfs_inode *ip, xfs_off_t offset, size_t size);
 
 extern void xfs_count_page_state(struct page *, int *, int *);
diff --git a/fs/xfs/xfs_attr.h b/fs/xfs/xfs_attr.h
index e3da5d448bcf..d14691aa02b4 100644
--- a/fs/xfs/xfs_attr.h
+++ b/fs/xfs/xfs_attr.h
@@ -112,8 +112,8 @@ typedef struct attrlist_cursor_kern {
  *========================================================================*/
 
 
-/* Return 0 on success, or -errno; other state communicated via *context */
-typedef int (*put_listent_func_t)(struct xfs_attr_list_context *, int,
+/* void; state communicated via *context */
+typedef void (*put_listent_func_t)(struct xfs_attr_list_context *, int,
 			      unsigned char *, int, int);
 
 typedef struct xfs_attr_list_context {
diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c
index 25e76cd6c053..97c45b6eb91e 100644
--- a/fs/xfs/xfs_attr_list.c
+++ b/fs/xfs/xfs_attr_list.c
@@ -74,7 +74,6 @@ xfs_attr_shortform_list(xfs_attr_list_context_t *context)
 	xfs_attr_sf_entry_t *sfe;
 	xfs_inode_t *dp;
 	int sbsize, nsbuf, count, i;
-	int error;
 
 	ASSERT(context != NULL);
 	dp = context->dp;
@@ -102,13 +101,11 @@ xfs_attr_shortform_list(xfs_attr_list_context_t *context)
 	    (XFS_ISRESET_CURSOR(cursor) &&
              (dp->i_afp->if_bytes + sf->hdr.count * 16) < context->bufsize)) {
 		for (i = 0, sfe = &sf->list[0]; i < sf->hdr.count; i++) {
-			error = context->put_listent(context,
-					   sfe->flags,
-					   sfe->nameval,
-					   (int)sfe->namelen,
-					   (int)sfe->valuelen);
-			if (error)
-				return error;
+			context->put_listent(context,
+					     sfe->flags,
+					     sfe->nameval,
+					     (int)sfe->namelen,
+					     (int)sfe->valuelen);
 			/*
 			 * Either search callback finished early or
 			 * didn't fit it all in the buffer after all.
@@ -193,15 +190,11 @@ xfs_attr_shortform_list(xfs_attr_list_context_t *context)
 			cursor->hashval = sbp->hash;
 			cursor->offset = 0;
 		}
-		error = context->put_listent(context,
-					sbp->flags,
-					sbp->name,
-					sbp->namelen,
-					sbp->valuelen);
-		if (error) {
-			kmem_free(sbuf);
-			return error;
-		}
+		context->put_listent(context,
+				     sbp->flags,
+				     sbp->name,
+				     sbp->namelen,
+				     sbp->valuelen);
 		if (context->seen_enough)
 			break;
 		cursor->offset++;
@@ -335,11 +328,7 @@ xfs_attr_node_list(xfs_attr_list_context_t *context)
 	 */
 	for (;;) {
 		leaf = bp->b_addr;
-		error = xfs_attr3_leaf_list_int(bp, context);
-		if (error) {
-			xfs_trans_brelse(NULL, bp);
-			return error;
-		}
+		xfs_attr3_leaf_list_int(bp, context);
 		xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &leafhdr, leaf);
 		if (context->seen_enough || leafhdr.forw == 0)
 			break;
@@ -356,7 +345,7 @@ xfs_attr_node_list(xfs_attr_list_context_t *context)
 /*
  * Copy out attribute list entries for attr_list(), for leaf attribute lists.
  */
-int
+void
 xfs_attr3_leaf_list_int(
 	struct xfs_buf			*bp,
 	struct xfs_attr_list_context	*context)
@@ -366,7 +355,6 @@ xfs_attr3_leaf_list_int(
 	struct xfs_attr3_icleaf_hdr	ichdr;
 	struct xfs_attr_leaf_entry	*entries;
 	struct xfs_attr_leaf_entry	*entry;
-	int				retval;
 	int				i;
 	struct xfs_mount		*mp = context->dp->i_mount;
 
@@ -399,7 +387,7 @@ xfs_attr3_leaf_list_int(
 		}
 		if (i == ichdr.count) {
 			trace_xfs_attr_list_notfound(context);
-			return 0;
+			return;
 		}
 	} else {
 		entry = &entries[0];
@@ -410,7 +398,6 @@ xfs_attr3_leaf_list_int(
 	/*
 	 * We have found our place, start copying out the new attributes.
 	 */
-	retval = 0;
 	for (; i < ichdr.count; entry++, i++) {
 		char *name;
 		int namelen, valuelen;
@@ -439,16 +426,14 @@ xfs_attr3_leaf_list_int(
 			valuelen = be32_to_cpu(name_rmt->valuelen);
 		}
 
-		retval = context->put_listent(context, entry->flags,
+		context->put_listent(context, entry->flags,
 					      name, namelen, valuelen);
-		if (retval)
-			break;
 		if (context->seen_enough)
 			break;
 		cursor->offset++;
 	}
 	trace_xfs_attr_list_leaf_end(context);
-	return retval;
+	return;
 }
 
 /*
@@ -467,9 +452,9 @@ xfs_attr_leaf_list(xfs_attr_list_context_t *context)
 	if (error)
 		return error;
 
-	error = xfs_attr3_leaf_list_int(bp, context);
+	xfs_attr3_leaf_list_int(bp, context);
 	xfs_trans_brelse(NULL, bp);
-	return error;
+	return 0;
 }
 
 int
@@ -513,7 +498,7 @@ xfs_attr_list_int(
  * Take care to check values and protect against them changing later,
  * we may be reading them directly out of a user buffer.
  */
-STATIC int
+STATIC void
 xfs_attr_put_listent(
 	xfs_attr_list_context_t *context,
 	int		flags,
@@ -536,10 +521,10 @@ xfs_attr_put_listent(
 	 */
 	if (((context->flags & ATTR_SECURE) == 0) !=
 	    ((flags & XFS_ATTR_SECURE) == 0))
-		return 0;
+		return;
 	if (((context->flags & ATTR_ROOT) == 0) !=
 	    ((flags & XFS_ATTR_ROOT) == 0))
-		return 0;
+		return;
 
 	arraytop = sizeof(*alist) +
 			context->count * sizeof(alist->al_offset[0]);
@@ -548,7 +533,7 @@ xfs_attr_put_listent(
 		trace_xfs_attr_list_full(context);
 		alist->al_more = 1;
 		context->seen_enough = 1;
-		return 0;
+		return;
 	}
 
 	aep = (attrlist_ent_t *)&context->alist[context->firstu];
@@ -558,7 +543,7 @@ xfs_attr_put_listent(
 	alist->al_offset[context->count++] = context->firstu;
 	alist->al_count = context->count;
 	trace_xfs_attr_list_add(context);
-	return 0;
+	return;
 }
 
 /*
diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c
index 552465e011ec..b9abce524c33 100644
--- a/fs/xfs/xfs_bmap_util.c
+++ b/fs/xfs/xfs_bmap_util.c
@@ -359,9 +359,7 @@ xfs_bmap_count_blocks(
 	mp = ip->i_mount;
 	ifp = XFS_IFORK_PTR(ip, whichfork);
 	if ( XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_EXTENTS ) {
-		xfs_bmap_count_leaves(ifp, 0,
-			ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t),
-			count);
+		xfs_bmap_count_leaves(ifp, 0, xfs_iext_count(ifp), count);
 		return 0;
 	}
 
@@ -426,7 +424,7 @@ xfs_getbmapx_fix_eof_hole(
 		ifp = XFS_IFORK_PTR(ip, whichfork);
 		if (!moretocome &&
 		    xfs_iext_bno_to_ext(ifp, fileblock, &lastx) &&
-		   (lastx == (ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t))-1))
+		   (lastx == xfs_iext_count(ifp) - 1))
 			out->bmv_oflags |= BMV_OF_LAST;
 	}
 
@@ -1792,6 +1790,7 @@ xfs_swap_extent_forks(
 	struct xfs_ifork	tempifp, *ifp, *tifp;
 	int			aforkblks = 0;
 	int			taforkblks = 0;
+	xfs_extnum_t		nextents;
 	__uint64_t		tmp;
 	int			error;
 
@@ -1877,14 +1876,13 @@ xfs_swap_extent_forks(
 
 	switch (ip->i_d.di_format) {
 	case XFS_DINODE_FMT_EXTENTS:
-		/* If the extents fit in the inode, fix the
-		 * pointer.  Otherwise it's already NULL or
-		 * pointing to the extent.
+		/*
+		 * If the extents fit in the inode, fix the pointer.  Otherwise
+		 * it's already NULL or pointing to the extent.
 		 */
-		if (ip->i_d.di_nextents <= XFS_INLINE_EXTS) {
-			ifp->if_u1.if_extents =
-				ifp->if_u2.if_inline_ext;
-		}
+		nextents = xfs_iext_count(&ip->i_df);
+		if (nextents <= XFS_INLINE_EXTS)
+			ifp->if_u1.if_extents = ifp->if_u2.if_inline_ext;
 		(*src_log_flags) |= XFS_ILOG_DEXT;
 		break;
 	case XFS_DINODE_FMT_BTREE:
@@ -1896,14 +1894,13 @@ xfs_swap_extent_forks(
 
 	switch (tip->i_d.di_format) {
 	case XFS_DINODE_FMT_EXTENTS:
-		/* If the extents fit in the inode, fix the
-		 * pointer.  Otherwise it's already NULL or
-		 * pointing to the extent.
+		/*
+		 * If the extents fit in the inode, fix the pointer.  Otherwise
+		 * it's already NULL or pointing to the extent.
 		 */
-		if (tip->i_d.di_nextents <= XFS_INLINE_EXTS) {
-			tifp->if_u1.if_extents =
-				tifp->if_u2.if_inline_ext;
-		}
+		nextents = xfs_iext_count(&tip->i_df);
+		if (nextents <= XFS_INLINE_EXTS)
+			tifp->if_u1.if_extents = tifp->if_u2.if_inline_ext;
 		(*target_log_flags) |= XFS_ILOG_DEXT;
 		break;
 	case XFS_DINODE_FMT_BTREE:
@@ -1938,8 +1935,8 @@ xfs_swap_extents(
 	 * page cache safely. Once we have done this we can take the ilocks and
 	 * do the rest of the checks.
 	 */
-	lock_flags = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL;
-	xfs_lock_two_inodes(ip, tip, XFS_IOLOCK_EXCL);
+	lock_two_nondirectories(VFS_I(ip), VFS_I(tip));
+	lock_flags = XFS_MMAPLOCK_EXCL;
 	xfs_lock_two_inodes(ip, tip, XFS_MMAPLOCK_EXCL);
 
 	/* Verify that both files have the same format */
@@ -2079,15 +2076,13 @@ xfs_swap_extents(
 	trace_xfs_swap_extent_after(ip, 0);
 	trace_xfs_swap_extent_after(tip, 1);
 
+out_unlock:
 	xfs_iunlock(ip, lock_flags);
 	xfs_iunlock(tip, lock_flags);
+	unlock_two_nondirectories(VFS_I(ip), VFS_I(tip));
 	return error;
 
 out_trans_cancel:
 	xfs_trans_cancel(tp);
-
-out_unlock:
-	xfs_iunlock(ip, lock_flags);
-	xfs_iunlock(tip, lock_flags);
-	return error;
+	goto out_unlock;
 }
diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c
index b5b9bffe3520..7f0a01f7b592 100644
--- a/fs/xfs/xfs_buf.c
+++ b/fs/xfs/xfs_buf.c
@@ -219,7 +219,6 @@ _xfs_buf_alloc(
 	init_completion(&bp->b_iowait);
 	INIT_LIST_HEAD(&bp->b_lru);
 	INIT_LIST_HEAD(&bp->b_list);
-	RB_CLEAR_NODE(&bp->b_rbnode);
 	sema_init(&bp->b_sema, 0); /* held, no waiters */
 	spin_lock_init(&bp->b_lock);
 	XB_SET_OWNER(bp);
@@ -473,6 +472,62 @@ _xfs_buf_map_pages(
 /*
  *	Finding and Reading Buffers
  */
+static int
+_xfs_buf_obj_cmp(
+	struct rhashtable_compare_arg	*arg,
+	const void			*obj)
+{
+	const struct xfs_buf_map	*map = arg->key;
+	const struct xfs_buf		*bp = obj;
+
+	/*
+	 * The key hashing in the lookup path depends on the key being the
+	 * first element of the compare_arg, make sure to assert this.
+	 */
+	BUILD_BUG_ON(offsetof(struct xfs_buf_map, bm_bn) != 0);
+
+	if (bp->b_bn != map->bm_bn)
+		return 1;
+
+	if (unlikely(bp->b_length != map->bm_len)) {
+		/*
+		 * found a block number match. If the range doesn't
+		 * match, the only way this is allowed is if the buffer
+		 * in the cache is stale and the transaction that made
+		 * it stale has not yet committed. i.e. we are
+		 * reallocating a busy extent. Skip this buffer and
+		 * continue searching for an exact match.
+		 */
+		ASSERT(bp->b_flags & XBF_STALE);
+		return 1;
+	}
+	return 0;
+}
+
+static const struct rhashtable_params xfs_buf_hash_params = {
+	.min_size		= 32,	/* empty AGs have minimal footprint */
+	.nelem_hint		= 16,
+	.key_len		= sizeof(xfs_daddr_t),
+	.key_offset		= offsetof(struct xfs_buf, b_bn),
+	.head_offset		= offsetof(struct xfs_buf, b_rhash_head),
+	.automatic_shrinking	= true,
+	.obj_cmpfn		= _xfs_buf_obj_cmp,
+};
+
+int
+xfs_buf_hash_init(
+	struct xfs_perag	*pag)
+{
+	spin_lock_init(&pag->pag_buf_lock);
+	return rhashtable_init(&pag->pag_buf_hash, &xfs_buf_hash_params);
+}
+
+void
+xfs_buf_hash_destroy(
+	struct xfs_perag	*pag)
+{
+	rhashtable_destroy(&pag->pag_buf_hash);
+}
 
 /*
  *	Look up, and creates if absent, a lockable buffer for
@@ -488,27 +543,24 @@ _xfs_buf_find(
 	xfs_buf_t		*new_bp)
 {
 	struct xfs_perag	*pag;
-	struct rb_node		**rbp;
-	struct rb_node		*parent;
 	xfs_buf_t		*bp;
-	xfs_daddr_t		blkno = map[0].bm_bn;
+	struct xfs_buf_map	cmap = { .bm_bn = map[0].bm_bn };
 	xfs_daddr_t		eofs;
-	int			numblks = 0;
 	int			i;
 
 	for (i = 0; i < nmaps; i++)
-		numblks += map[i].bm_len;
+		cmap.bm_len += map[i].bm_len;
 
 	/* Check for IOs smaller than the sector size / not sector aligned */
-	ASSERT(!(BBTOB(numblks) < btp->bt_meta_sectorsize));
-	ASSERT(!(BBTOB(blkno) & (xfs_off_t)btp->bt_meta_sectormask));
+	ASSERT(!(BBTOB(cmap.bm_len) < btp->bt_meta_sectorsize));
+	ASSERT(!(BBTOB(cmap.bm_bn) & (xfs_off_t)btp->bt_meta_sectormask));
 
 	/*
 	 * Corrupted block numbers can get through to here, unfortunately, so we
 	 * have to check that the buffer falls within the filesystem bounds.
 	 */
 	eofs = XFS_FSB_TO_BB(btp->bt_mount, btp->bt_mount->m_sb.sb_dblocks);
-	if (blkno < 0 || blkno >= eofs) {
+	if (cmap.bm_bn < 0 || cmap.bm_bn >= eofs) {
 		/*
 		 * XXX (dgc): we should really be returning -EFSCORRUPTED here,
 		 * but none of the higher level infrastructure supports
@@ -516,53 +568,29 @@ _xfs_buf_find(
 		 */
 		xfs_alert(btp->bt_mount,
 			  "%s: Block out of range: block 0x%llx, EOFS 0x%llx ",
-			  __func__, blkno, eofs);
+			  __func__, cmap.bm_bn, eofs);
 		WARN_ON(1);
 		return NULL;
 	}
 
-	/* get tree root */
 	pag = xfs_perag_get(btp->bt_mount,
-				xfs_daddr_to_agno(btp->bt_mount, blkno));
+			    xfs_daddr_to_agno(btp->bt_mount, cmap.bm_bn));
 
-	/* walk tree */
 	spin_lock(&pag->pag_buf_lock);
-	rbp = &pag->pag_buf_tree.rb_node;
-	parent = NULL;
-	bp = NULL;
-	while (*rbp) {
-		parent = *rbp;
-		bp = rb_entry(parent, struct xfs_buf, b_rbnode);
-
-		if (blkno < bp->b_bn)
-			rbp = &(*rbp)->rb_left;
-		else if (blkno > bp->b_bn)
-			rbp = &(*rbp)->rb_right;
-		else {
-			/*
-			 * found a block number match. If the range doesn't
-			 * match, the only way this is allowed is if the buffer
-			 * in the cache is stale and the transaction that made
-			 * it stale has not yet committed. i.e. we are
-			 * reallocating a busy extent. Skip this buffer and
-			 * continue searching to the right for an exact match.
-			 */
-			if (bp->b_length != numblks) {
-				ASSERT(bp->b_flags & XBF_STALE);
-				rbp = &(*rbp)->rb_right;
-				continue;
-			}
-			atomic_inc(&bp->b_hold);
-			goto found;
-		}
+	bp = rhashtable_lookup_fast(&pag->pag_buf_hash, &cmap,
+				    xfs_buf_hash_params);
+	if (bp) {
+		atomic_inc(&bp->b_hold);
+		goto found;
 	}
 
 	/* No match found */
 	if (new_bp) {
-		rb_link_node(&new_bp->b_rbnode, parent, rbp);
-		rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
 		/* the buffer keeps the perag reference until it is freed */
 		new_bp->b_pag = pag;
+		rhashtable_insert_fast(&pag->pag_buf_hash,
+				       &new_bp->b_rhash_head,
+				       xfs_buf_hash_params);
 		spin_unlock(&pag->pag_buf_lock);
 	} else {
 		XFS_STATS_INC(btp->bt_mount, xb_miss_locked);
@@ -930,7 +958,6 @@ xfs_buf_rele(
 
 	if (!pag) {
 		ASSERT(list_empty(&bp->b_lru));
-		ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
 		if (atomic_dec_and_test(&bp->b_hold)) {
 			xfs_buf_ioacct_dec(bp);
 			xfs_buf_free(bp);
@@ -938,8 +965,6 @@ xfs_buf_rele(
 		return;
 	}
 
-	ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
-
 	ASSERT(atomic_read(&bp->b_hold) > 0);
 
 	release = atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock);
@@ -983,7 +1008,8 @@ xfs_buf_rele(
 		}
 
 		ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
-		rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
+		rhashtable_remove_fast(&pag->pag_buf_hash, &bp->b_rhash_head,
+				       xfs_buf_hash_params);
 		spin_unlock(&pag->pag_buf_lock);
 		xfs_perag_put(pag);
 		freebuf = true;
@@ -1304,7 +1330,7 @@ _xfs_buf_ioapply(
 	if (bp->b_flags & XBF_WRITE) {
 		op = REQ_OP_WRITE;
 		if (bp->b_flags & XBF_SYNCIO)
-			op_flags = WRITE_SYNC;
+			op_flags = REQ_SYNC;
 		if (bp->b_flags & XBF_FUA)
 			op_flags |= REQ_FUA;
 		if (bp->b_flags & XBF_FLUSH)
@@ -1711,8 +1737,7 @@ xfs_free_buftarg(
 	percpu_counter_destroy(&btp->bt_io_count);
 	list_lru_destroy(&btp->bt_lru);
 
-	if (mp->m_flags & XFS_MOUNT_BARRIER)
-		xfs_blkdev_issue_flush(btp);
+	xfs_blkdev_issue_flush(btp);
 
 	kmem_free(btp);
 }
diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h
index 1c2e52b2d926..8a9d3a9599f0 100644
--- a/fs/xfs/xfs_buf.h
+++ b/fs/xfs/xfs_buf.h
@@ -71,6 +71,7 @@ typedef unsigned int xfs_buf_flags_t;
 	{ XBF_READ,		"READ" }, \
 	{ XBF_WRITE,		"WRITE" }, \
 	{ XBF_READ_AHEAD,	"READ_AHEAD" }, \
+	{ XBF_NO_IOACCT,	"NO_IOACCT" }, \
 	{ XBF_ASYNC,		"ASYNC" }, \
 	{ XBF_DONE,		"DONE" }, \
 	{ XBF_STALE,		"STALE" }, \
@@ -150,7 +151,7 @@ typedef struct xfs_buf {
 	 * which is the only bit that is touched if we hit the semaphore
 	 * fast-path on locking.
 	 */
-	struct rb_node		b_rbnode;	/* rbtree node */
+	struct rhash_head	b_rhash_head;	/* pag buffer hash node */
 	xfs_daddr_t		b_bn;		/* block number of buffer */
 	int			b_length;	/* size of buffer in BBs */
 	atomic_t		b_hold;		/* reference count */
diff --git a/fs/xfs/xfs_dir2_readdir.c b/fs/xfs/xfs_dir2_readdir.c
index 29816981b50a..003a99b83bd8 100644
--- a/fs/xfs/xfs_dir2_readdir.c
+++ b/fs/xfs/xfs_dir2_readdir.c
@@ -677,7 +677,6 @@ xfs_readdir(
 	args.dp = dp;
 	args.geo = dp->i_mount->m_dir_geo;
 
-	xfs_ilock(dp, XFS_IOLOCK_SHARED);
 	if (dp->i_d.di_format == XFS_DINODE_FMT_LOCAL)
 		rval = xfs_dir2_sf_getdents(&args, ctx);
 	else if ((rval = xfs_dir2_isblock(&args, &v)))
@@ -686,7 +685,6 @@ xfs_readdir(
 		rval = xfs_dir2_block_getdents(&args, ctx);
 	else
 		rval = xfs_dir2_leaf_getdents(&args, ctx, bufsize);
-	xfs_iunlock(dp, XFS_IOLOCK_SHARED);
 
 	return rval;
 }
diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c
index 6e4f7f900fea..65d27a502909 100644
--- a/fs/xfs/xfs_file.c
+++ b/fs/xfs/xfs_file.c
@@ -48,40 +48,6 @@
 static const struct vm_operations_struct xfs_file_vm_ops;
 
 /*
- * Locking primitives for read and write IO paths to ensure we consistently use
- * and order the inode->i_mutex, ip->i_lock and ip->i_iolock.
- */
-static inline void
-xfs_rw_ilock(
-	struct xfs_inode	*ip,
-	int			type)
-{
-	if (type & XFS_IOLOCK_EXCL)
-		inode_lock(VFS_I(ip));
-	xfs_ilock(ip, type);
-}
-
-static inline void
-xfs_rw_iunlock(
-	struct xfs_inode	*ip,
-	int			type)
-{
-	xfs_iunlock(ip, type);
-	if (type & XFS_IOLOCK_EXCL)
-		inode_unlock(VFS_I(ip));
-}
-
-static inline void
-xfs_rw_ilock_demote(
-	struct xfs_inode	*ip,
-	int			type)
-{
-	xfs_ilock_demote(ip, type);
-	if (type & XFS_IOLOCK_EXCL)
-		inode_unlock(VFS_I(ip));
-}
-
-/*
  * Clear the specified ranges to zero through either the pagecache or DAX.
  * Holes and unwritten extents will be left as-is as they already are zeroed.
  */
@@ -183,19 +149,16 @@ xfs_file_fsync(
 
 	xfs_iflags_clear(ip, XFS_ITRUNCATED);
 
-	if (mp->m_flags & XFS_MOUNT_BARRIER) {
-		/*
-		 * If we have an RT and/or log subvolume we need to make sure
-		 * to flush the write cache the device used for file data
-		 * first.  This is to ensure newly written file data make
-		 * it to disk before logging the new inode size in case of
-		 * an extending write.
-		 */
-		if (XFS_IS_REALTIME_INODE(ip))
-			xfs_blkdev_issue_flush(mp->m_rtdev_targp);
-		else if (mp->m_logdev_targp != mp->m_ddev_targp)
-			xfs_blkdev_issue_flush(mp->m_ddev_targp);
-	}
+	/*
+	 * If we have an RT and/or log subvolume we need to make sure to flush
+	 * the write cache the device used for file data first.  This is to
+	 * ensure newly written file data make it to disk before logging the new
+	 * inode size in case of an extending write.
+	 */
+	if (XFS_IS_REALTIME_INODE(ip))
+		xfs_blkdev_issue_flush(mp->m_rtdev_targp);
+	else if (mp->m_logdev_targp != mp->m_ddev_targp)
+		xfs_blkdev_issue_flush(mp->m_ddev_targp);
 
 	/*
 	 * All metadata updates are logged, which means that we just have to
@@ -230,10 +193,8 @@ xfs_file_fsync(
 	 * an already allocated file and thus do not have any metadata to
 	 * commit.
 	 */
-	if ((mp->m_flags & XFS_MOUNT_BARRIER) &&
-	    mp->m_logdev_targp == mp->m_ddev_targp &&
-	    !XFS_IS_REALTIME_INODE(ip) &&
-	    !log_flushed)
+	if (!log_flushed && !XFS_IS_REALTIME_INODE(ip) &&
+	    mp->m_logdev_targp == mp->m_ddev_targp)
 		xfs_blkdev_issue_flush(mp->m_ddev_targp);
 
 	return error;
@@ -244,62 +205,21 @@ xfs_file_dio_aio_read(
 	struct kiocb		*iocb,
 	struct iov_iter		*to)
 {
-	struct address_space	*mapping = iocb->ki_filp->f_mapping;
-	struct inode		*inode = mapping->host;
-	struct xfs_inode	*ip = XFS_I(inode);
-	loff_t			isize = i_size_read(inode);
+	struct xfs_inode	*ip = XFS_I(file_inode(iocb->ki_filp));
 	size_t			count = iov_iter_count(to);
-	loff_t			end = iocb->ki_pos + count - 1;
-	struct iov_iter		data;
-	struct xfs_buftarg	*target;
-	ssize_t			ret = 0;
+	ssize_t			ret;
 
 	trace_xfs_file_direct_read(ip, count, iocb->ki_pos);
 
 	if (!count)
 		return 0; /* skip atime */
 
-	if (XFS_IS_REALTIME_INODE(ip))
-		target = ip->i_mount->m_rtdev_targp;
-	else
-		target = ip->i_mount->m_ddev_targp;
-
-	/* DIO must be aligned to device logical sector size */
-	if ((iocb->ki_pos | count) & target->bt_logical_sectormask) {
-		if (iocb->ki_pos == isize)
-			return 0;
-		return -EINVAL;
-	}
-
 	file_accessed(iocb->ki_filp);
 
-	xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
-	if (mapping->nrpages) {
-		ret = filemap_write_and_wait_range(mapping, iocb->ki_pos, end);
-		if (ret)
-			goto out_unlock;
-
-		/*
-		 * Invalidate whole pages. This can return an error if we fail
-		 * to invalidate a page, but this should never happen on XFS.
-		 * Warn if it does fail.
-		 */
-		ret = invalidate_inode_pages2_range(mapping,
-				iocb->ki_pos >> PAGE_SHIFT, end >> PAGE_SHIFT);
-		WARN_ON_ONCE(ret);
-		ret = 0;
-	}
+	xfs_ilock(ip, XFS_IOLOCK_SHARED);
+	ret = iomap_dio_rw(iocb, to, &xfs_iomap_ops, NULL);
+	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
 
-	data = *to;
-	ret = __blockdev_direct_IO(iocb, inode, target->bt_bdev, &data,
-			xfs_get_blocks_direct, NULL, NULL, 0);
-	if (ret >= 0) {
-		iocb->ki_pos += ret;
-		iov_iter_advance(to, ret);
-	}
-
-out_unlock:
-	xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
 	return ret;
 }
 
@@ -317,9 +237,9 @@ xfs_file_dax_read(
 	if (!count)
 		return 0; /* skip atime */
 
-	xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
-	ret = iomap_dax_rw(iocb, to, &xfs_iomap_ops);
-	xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
+	xfs_ilock(ip, XFS_IOLOCK_SHARED);
+	ret = dax_iomap_rw(iocb, to, &xfs_iomap_ops);
+	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
 
 	file_accessed(iocb->ki_filp);
 	return ret;
@@ -335,9 +255,9 @@ xfs_file_buffered_aio_read(
 
 	trace_xfs_file_buffered_read(ip, iov_iter_count(to), iocb->ki_pos);
 
-	xfs_rw_ilock(ip, XFS_IOLOCK_SHARED);
+	xfs_ilock(ip, XFS_IOLOCK_SHARED);
 	ret = generic_file_read_iter(iocb, to);
-	xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED);
+	xfs_iunlock(ip, XFS_IOLOCK_SHARED);
 
 	return ret;
 }
@@ -418,15 +338,18 @@ restart:
 	if (error <= 0)
 		return error;
 
-	error = xfs_break_layouts(inode, iolock, true);
+	error = xfs_break_layouts(inode, iolock);
 	if (error)
 		return error;
 
-	/* For changing security info in file_remove_privs() we need i_mutex */
+	/*
+	 * For changing security info in file_remove_privs() we need i_rwsem
+	 * exclusively.
+	 */
 	if (*iolock == XFS_IOLOCK_SHARED && !IS_NOSEC(inode)) {
-		xfs_rw_iunlock(ip, *iolock);
+		xfs_iunlock(ip, *iolock);
 		*iolock = XFS_IOLOCK_EXCL;
-		xfs_rw_ilock(ip, *iolock);
+		xfs_ilock(ip, *iolock);
 		goto restart;
 	}
 	/*
@@ -451,9 +374,9 @@ restart:
 		spin_unlock(&ip->i_flags_lock);
 		if (!drained_dio) {
 			if (*iolock == XFS_IOLOCK_SHARED) {
-				xfs_rw_iunlock(ip, *iolock);
+				xfs_iunlock(ip, *iolock);
 				*iolock = XFS_IOLOCK_EXCL;
-				xfs_rw_ilock(ip, *iolock);
+				xfs_ilock(ip, *iolock);
 				iov_iter_reexpand(from, count);
 			}
 			/*
@@ -496,6 +419,58 @@ restart:
 	return 0;
 }
 
+static int
+xfs_dio_write_end_io(
+	struct kiocb		*iocb,
+	ssize_t			size,
+	unsigned		flags)
+{
+	struct inode		*inode = file_inode(iocb->ki_filp);
+	struct xfs_inode	*ip = XFS_I(inode);
+	loff_t			offset = iocb->ki_pos;
+	bool			update_size = false;
+	int			error = 0;
+
+	trace_xfs_end_io_direct_write(ip, offset, size);
+
+	if (XFS_FORCED_SHUTDOWN(ip->i_mount))
+		return -EIO;
+
+	if (size <= 0)
+		return size;
+
+	/*
+	 * We need to update the in-core inode size here so that we don't end up
+	 * with the on-disk inode size being outside the in-core inode size. We
+	 * have no other method of updating EOF for AIO, so always do it here
+	 * if necessary.
+	 *
+	 * We need to lock the test/set EOF update as we can be racing with
+	 * other IO completions here to update the EOF. Failing to serialise
+	 * here can result in EOF moving backwards and Bad Things Happen when
+	 * that occurs.
+	 */
+	spin_lock(&ip->i_flags_lock);
+	if (offset + size > i_size_read(inode)) {
+		i_size_write(inode, offset + size);
+		update_size = true;
+	}
+	spin_unlock(&ip->i_flags_lock);
+
+	if (flags & IOMAP_DIO_COW) {
+		error = xfs_reflink_end_cow(ip, offset, size);
+		if (error)
+			return error;
+	}
+
+	if (flags & IOMAP_DIO_UNWRITTEN)
+		error = xfs_iomap_write_unwritten(ip, offset, size);
+	else if (update_size)
+		error = xfs_setfilesize(ip, offset, size);
+
+	return error;
+}
+
 /*
  * xfs_file_dio_aio_write - handle direct IO writes
  *
@@ -535,9 +510,7 @@ xfs_file_dio_aio_write(
 	int			unaligned_io = 0;
 	int			iolock;
 	size_t			count = iov_iter_count(from);
-	loff_t			end;
-	struct iov_iter		data;
-	struct xfs_buftarg	*target = XFS_IS_REALTIME_INODE(ip) ?
+	struct xfs_buftarg      *target = XFS_IS_REALTIME_INODE(ip) ?
 					mp->m_rtdev_targp : mp->m_ddev_targp;
 
 	/* DIO must be aligned to device logical sector size */
@@ -559,29 +532,12 @@ xfs_file_dio_aio_write(
 		iolock = XFS_IOLOCK_SHARED;
 	}
 
-	xfs_rw_ilock(ip, iolock);
+	xfs_ilock(ip, iolock);
 
 	ret = xfs_file_aio_write_checks(iocb, from, &iolock);
 	if (ret)
 		goto out;
 	count = iov_iter_count(from);
-	end = iocb->ki_pos + count - 1;
-
-	if (mapping->nrpages) {
-		ret = filemap_write_and_wait_range(mapping, iocb->ki_pos, end);
-		if (ret)
-			goto out;
-
-		/*
-		 * Invalidate whole pages. This can return an error if we fail
-		 * to invalidate a page, but this should never happen on XFS.
-		 * Warn if it does fail.
-		 */
-		ret = invalidate_inode_pages2_range(mapping,
-				iocb->ki_pos >> PAGE_SHIFT, end >> PAGE_SHIFT);
-		WARN_ON_ONCE(ret);
-		ret = 0;
-	}
 
 	/*
 	 * If we are doing unaligned IO, wait for all other IO to drain,
@@ -591,7 +547,7 @@ xfs_file_dio_aio_write(
 	if (unaligned_io)
 		inode_dio_wait(inode);
 	else if (iolock == XFS_IOLOCK_EXCL) {
-		xfs_rw_ilock_demote(ip, XFS_IOLOCK_EXCL);
+		xfs_ilock_demote(ip, XFS_IOLOCK_EXCL);
 		iolock = XFS_IOLOCK_SHARED;
 	}
 
@@ -604,24 +560,9 @@ xfs_file_dio_aio_write(
 			goto out;
 	}
 
-	data = *from;
-	ret = __blockdev_direct_IO(iocb, inode, target->bt_bdev, &data,
-			xfs_get_blocks_direct, xfs_end_io_direct_write,
-			NULL, DIO_ASYNC_EXTEND);
-
-	/* see generic_file_direct_write() for why this is necessary */
-	if (mapping->nrpages) {
-		invalidate_inode_pages2_range(mapping,
-					      iocb->ki_pos >> PAGE_SHIFT,
-					      end >> PAGE_SHIFT);
-	}
-
-	if (ret > 0) {
-		iocb->ki_pos += ret;
-		iov_iter_advance(from, ret);
-	}
+	ret = iomap_dio_rw(iocb, from, &xfs_iomap_ops, xfs_dio_write_end_io);
 out:
-	xfs_rw_iunlock(ip, iolock);
+	xfs_iunlock(ip, iolock);
 
 	/*
 	 * No fallback to buffered IO on errors for XFS, direct IO will either
@@ -643,7 +584,7 @@ xfs_file_dax_write(
 	size_t			count;
 	loff_t			pos;
 
-	xfs_rw_ilock(ip, iolock);
+	xfs_ilock(ip, iolock);
 	ret = xfs_file_aio_write_checks(iocb, from, &iolock);
 	if (ret)
 		goto out;
@@ -652,15 +593,13 @@ xfs_file_dax_write(
 	count = iov_iter_count(from);
 
 	trace_xfs_file_dax_write(ip, count, pos);
-
-	ret = iomap_dax_rw(iocb, from, &xfs_iomap_ops);
+	ret = dax_iomap_rw(iocb, from, &xfs_iomap_ops);
 	if (ret > 0 && iocb->ki_pos > i_size_read(inode)) {
 		i_size_write(inode, iocb->ki_pos);
 		error = xfs_setfilesize(ip, pos, ret);
 	}
-
 out:
-	xfs_rw_iunlock(ip, iolock);
+	xfs_iunlock(ip, iolock);
 	return error ? error : ret;
 }
 
@@ -677,7 +616,7 @@ xfs_file_buffered_aio_write(
 	int			enospc = 0;
 	int			iolock = XFS_IOLOCK_EXCL;
 
-	xfs_rw_ilock(ip, iolock);
+	xfs_ilock(ip, iolock);
 
 	ret = xfs_file_aio_write_checks(iocb, from, &iolock);
 	if (ret)
@@ -721,7 +660,7 @@ write_retry:
 
 	current->backing_dev_info = NULL;
 out:
-	xfs_rw_iunlock(ip, iolock);
+	xfs_iunlock(ip, iolock);
 	return ret;
 }
 
@@ -797,7 +736,7 @@ xfs_file_fallocate(
 		return -EOPNOTSUPP;
 
 	xfs_ilock(ip, iolock);
-	error = xfs_break_layouts(inode, &iolock, false);
+	error = xfs_break_layouts(inode, &iolock);
 	if (error)
 		goto out_unlock;
 
@@ -939,7 +878,6 @@ xfs_file_clone_range(
 				     len, false);
 }
 
-#define XFS_MAX_DEDUPE_LEN	(16 * 1024 * 1024)
 STATIC ssize_t
 xfs_file_dedupe_range(
 	struct file	*src_file,
@@ -950,14 +888,6 @@ xfs_file_dedupe_range(
 {
 	int		error;
 
-	/*
-	 * Limit the total length we will dedupe for each operation.
-	 * This is intended to bound the total time spent in this
-	 * ioctl to something sane.
-	 */
-	if (len > XFS_MAX_DEDUPE_LEN)
-		len = XFS_MAX_DEDUPE_LEN;
-
 	error = xfs_reflink_remap_range(src_file, loff, dst_file, dst_loff,
 				     len, true);
 	if (error)
@@ -1474,7 +1404,7 @@ xfs_filemap_page_mkwrite(
 	xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
 
 	if (IS_DAX(inode)) {
-		ret = iomap_dax_fault(vma, vmf, &xfs_iomap_ops);
+		ret = dax_iomap_fault(vma, vmf, &xfs_iomap_ops);
 	} else {
 		ret = iomap_page_mkwrite(vma, vmf, &xfs_iomap_ops);
 		ret = block_page_mkwrite_return(ret);
@@ -1501,15 +1431,9 @@ xfs_filemap_fault(
 		return xfs_filemap_page_mkwrite(vma, vmf);
 
 	xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
-	if (IS_DAX(inode)) {
-		/*
-		 * we do not want to trigger unwritten extent conversion on read
-		 * faults - that is unnecessary overhead and would also require
-		 * changes to xfs_get_blocks_direct() to map unwritten extent
-		 * ioend for conversion on read-only mappings.
-		 */
-		ret = iomap_dax_fault(vma, vmf, &xfs_iomap_ops);
-	} else
+	if (IS_DAX(inode))
+		ret = dax_iomap_fault(vma, vmf, &xfs_iomap_ops);
+	else
 		ret = filemap_fault(vma, vmf);
 	xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
 
@@ -1545,7 +1469,7 @@ xfs_filemap_pmd_fault(
 	}
 
 	xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
-	ret = dax_pmd_fault(vma, addr, pmd, flags, xfs_get_blocks_dax_fault);
+	ret = dax_iomap_pmd_fault(vma, addr, pmd, flags, &xfs_iomap_ops);
 	xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED);
 
 	if (flags & FAULT_FLAG_WRITE)
diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c
index f295049db681..ff4d6311c7f4 100644
--- a/fs/xfs/xfs_icache.c
+++ b/fs/xfs/xfs_icache.c
@@ -70,8 +70,6 @@ xfs_inode_alloc(
 	ASSERT(!xfs_isiflocked(ip));
 	ASSERT(ip->i_ino == 0);
 
-	mrlock_init(&ip->i_iolock, MRLOCK_BARRIER, "xfsio", ip->i_ino);
-
 	/* initialise the xfs inode */
 	ip->i_ino = ino;
 	ip->i_mount = mp;
@@ -123,7 +121,6 @@ __xfs_inode_free(
 {
 	/* asserts to verify all state is correct here */
 	ASSERT(atomic_read(&ip->i_pincount) == 0);
-	ASSERT(!xfs_isiflocked(ip));
 	XFS_STATS_DEC(ip->i_mount, vn_active);
 
 	call_rcu(&VFS_I(ip)->i_rcu, xfs_inode_free_callback);
@@ -133,6 +130,8 @@ void
 xfs_inode_free(
 	struct xfs_inode	*ip)
 {
+	ASSERT(!xfs_isiflocked(ip));
+
 	/*
 	 * Because we use RCU freeing we need to ensure the inode always
 	 * appears to be reclaimed with an invalid inode number when in the
@@ -393,8 +392,8 @@ xfs_iget_cache_hit(
 		xfs_inode_clear_reclaim_tag(pag, ip->i_ino);
 		inode->i_state = I_NEW;
 
-		ASSERT(!rwsem_is_locked(&ip->i_iolock.mr_lock));
-		mrlock_init(&ip->i_iolock, MRLOCK_BARRIER, "xfsio", ip->i_ino);
+		ASSERT(!rwsem_is_locked(&inode->i_rwsem));
+		init_rwsem(&inode->i_rwsem);
 
 		spin_unlock(&ip->i_flags_lock);
 		spin_unlock(&pag->pag_ici_lock);
@@ -981,6 +980,7 @@ restart:
 
 	if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
 		xfs_iunpin_wait(ip);
+		/* xfs_iflush_abort() drops the flush lock */
 		xfs_iflush_abort(ip, false);
 		goto reclaim;
 	}
@@ -989,10 +989,10 @@ restart:
 			goto out_ifunlock;
 		xfs_iunpin_wait(ip);
 	}
-	if (xfs_iflags_test(ip, XFS_ISTALE))
-		goto reclaim;
-	if (xfs_inode_clean(ip))
+	if (xfs_iflags_test(ip, XFS_ISTALE) || xfs_inode_clean(ip)) {
+		xfs_ifunlock(ip);
 		goto reclaim;
+	}
 
 	/*
 	 * Never flush out dirty data during non-blocking reclaim, as it would
@@ -1030,25 +1030,24 @@ restart:
 		xfs_buf_relse(bp);
 	}
 
-	xfs_iflock(ip);
 reclaim:
+	ASSERT(!xfs_isiflocked(ip));
+
 	/*
 	 * Because we use RCU freeing we need to ensure the inode always appears
 	 * to be reclaimed with an invalid inode number when in the free state.
-	 * We do this as early as possible under the ILOCK and flush lock so
-	 * that xfs_iflush_cluster() can be guaranteed to detect races with us
-	 * here. By doing this, we guarantee that once xfs_iflush_cluster has
-	 * locked both the XFS_ILOCK and the flush lock that it will see either
-	 * a valid, flushable inode that will serialise correctly against the
-	 * locks below, or it will see a clean (and invalid) inode that it can
-	 * skip.
+	 * We do this as early as possible under the ILOCK so that
+	 * xfs_iflush_cluster() can be guaranteed to detect races with us here.
+	 * By doing this, we guarantee that once xfs_iflush_cluster has locked
+	 * XFS_ILOCK that it will see either a valid, flushable inode that will
+	 * serialise correctly, or it will see a clean (and invalid) inode that
+	 * it can skip.
 	 */
 	spin_lock(&ip->i_flags_lock);
 	ip->i_flags = XFS_IRECLAIM;
 	ip->i_ino = 0;
 	spin_unlock(&ip->i_flags_lock);
 
-	xfs_ifunlock(ip);
 	xfs_iunlock(ip, XFS_ILOCK_EXCL);
 
 	XFS_STATS_INC(ip->i_mount, xs_ig_reclaims);
@@ -1580,10 +1579,15 @@ xfs_inode_free_cowblocks(
 	struct xfs_eofblocks *eofb = args;
 	bool need_iolock = true;
 	int match;
+	struct xfs_ifork	*ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
 
 	ASSERT(!eofb || (eofb && eofb->eof_scan_owner != 0));
 
-	if (!xfs_reflink_has_real_cow_blocks(ip)) {
+	/*
+	 * Just clear the tag if we have an empty cow fork or none at all. It's
+	 * possible the inode was fully unshared since it was originally tagged.
+	 */
+	if (!xfs_is_reflink_inode(ip) || !ifp->if_bytes) {
 		trace_xfs_inode_free_cowblocks_invalid(ip);
 		xfs_inode_clear_cowblocks_tag(ip);
 		return 0;
diff --git a/fs/xfs/xfs_icreate_item.c b/fs/xfs/xfs_icreate_item.c
index d45ca72af6fb..865ad1373e5e 100644
--- a/fs/xfs/xfs_icreate_item.c
+++ b/fs/xfs/xfs_icreate_item.c
@@ -133,7 +133,7 @@ xfs_icreate_item_committing(
 /*
  * This is the ops vector shared by all buf log items.
  */
-static struct xfs_item_ops xfs_icreate_item_ops = {
+static const struct xfs_item_ops xfs_icreate_item_ops = {
 	.iop_size	= xfs_icreate_item_size,
 	.iop_format	= xfs_icreate_item_format,
 	.iop_pin	= xfs_icreate_item_pin,
diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c
index 4e560e6a12c1..b9557795eb74 100644
--- a/fs/xfs/xfs_inode.c
+++ b/fs/xfs/xfs_inode.c
@@ -142,31 +142,31 @@ xfs_ilock_attr_map_shared(
 }
 
 /*
- * The xfs inode contains 3 multi-reader locks: the i_iolock the i_mmap_lock and
- * the i_lock.  This routine allows various combinations of the locks to be
- * obtained.
+ * In addition to i_rwsem in the VFS inode, the xfs inode contains 2
+ * multi-reader locks: i_mmap_lock and the i_lock.  This routine allows
+ * various combinations of the locks to be obtained.
  *
  * The 3 locks should always be ordered so that the IO lock is obtained first,
  * the mmap lock second and the ilock last in order to prevent deadlock.
  *
  * Basic locking order:
  *
- * i_iolock -> i_mmap_lock -> page_lock -> i_ilock
+ * i_rwsem -> i_mmap_lock -> page_lock -> i_ilock
  *
  * mmap_sem locking order:
  *
- * i_iolock -> page lock -> mmap_sem
+ * i_rwsem -> page lock -> mmap_sem
  * mmap_sem -> i_mmap_lock -> page_lock
  *
  * The difference in mmap_sem locking order mean that we cannot hold the
  * i_mmap_lock over syscall based read(2)/write(2) based IO. These IO paths can
  * fault in pages during copy in/out (for buffered IO) or require the mmap_sem
  * in get_user_pages() to map the user pages into the kernel address space for
- * direct IO. Similarly the i_iolock cannot be taken inside a page fault because
+ * direct IO. Similarly the i_rwsem cannot be taken inside a page fault because
  * page faults already hold the mmap_sem.
  *
  * Hence to serialise fully against both syscall and mmap based IO, we need to
- * take both the i_iolock and the i_mmap_lock. These locks should *only* be both
+ * take both the i_rwsem and the i_mmap_lock. These locks should *only* be both
  * taken in places where we need to invalidate the page cache in a race
  * free manner (e.g. truncate, hole punch and other extent manipulation
  * functions).
@@ -191,10 +191,13 @@ xfs_ilock(
 	       (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL));
 	ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0);
 
-	if (lock_flags & XFS_IOLOCK_EXCL)
-		mrupdate_nested(&ip->i_iolock, XFS_IOLOCK_DEP(lock_flags));
-	else if (lock_flags & XFS_IOLOCK_SHARED)
-		mraccess_nested(&ip->i_iolock, XFS_IOLOCK_DEP(lock_flags));
+	if (lock_flags & XFS_IOLOCK_EXCL) {
+		down_write_nested(&VFS_I(ip)->i_rwsem,
+				  XFS_IOLOCK_DEP(lock_flags));
+	} else if (lock_flags & XFS_IOLOCK_SHARED) {
+		down_read_nested(&VFS_I(ip)->i_rwsem,
+				 XFS_IOLOCK_DEP(lock_flags));
+	}
 
 	if (lock_flags & XFS_MMAPLOCK_EXCL)
 		mrupdate_nested(&ip->i_mmaplock, XFS_MMAPLOCK_DEP(lock_flags));
@@ -240,10 +243,10 @@ xfs_ilock_nowait(
 	ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0);
 
 	if (lock_flags & XFS_IOLOCK_EXCL) {
-		if (!mrtryupdate(&ip->i_iolock))
+		if (!down_write_trylock(&VFS_I(ip)->i_rwsem))
 			goto out;
 	} else if (lock_flags & XFS_IOLOCK_SHARED) {
-		if (!mrtryaccess(&ip->i_iolock))
+		if (!down_read_trylock(&VFS_I(ip)->i_rwsem))
 			goto out;
 	}
 
@@ -271,9 +274,9 @@ out_undo_mmaplock:
 		mrunlock_shared(&ip->i_mmaplock);
 out_undo_iolock:
 	if (lock_flags & XFS_IOLOCK_EXCL)
-		mrunlock_excl(&ip->i_iolock);
+		up_write(&VFS_I(ip)->i_rwsem);
 	else if (lock_flags & XFS_IOLOCK_SHARED)
-		mrunlock_shared(&ip->i_iolock);
+		up_read(&VFS_I(ip)->i_rwsem);
 out:
 	return 0;
 }
@@ -310,9 +313,9 @@ xfs_iunlock(
 	ASSERT(lock_flags != 0);
 
 	if (lock_flags & XFS_IOLOCK_EXCL)
-		mrunlock_excl(&ip->i_iolock);
+		up_write(&VFS_I(ip)->i_rwsem);
 	else if (lock_flags & XFS_IOLOCK_SHARED)
-		mrunlock_shared(&ip->i_iolock);
+		up_read(&VFS_I(ip)->i_rwsem);
 
 	if (lock_flags & XFS_MMAPLOCK_EXCL)
 		mrunlock_excl(&ip->i_mmaplock);
@@ -345,7 +348,7 @@ xfs_ilock_demote(
 	if (lock_flags & XFS_MMAPLOCK_EXCL)
 		mrdemote(&ip->i_mmaplock);
 	if (lock_flags & XFS_IOLOCK_EXCL)
-		mrdemote(&ip->i_iolock);
+		downgrade_write(&VFS_I(ip)->i_rwsem);
 
 	trace_xfs_ilock_demote(ip, lock_flags, _RET_IP_);
 }
@@ -370,8 +373,9 @@ xfs_isilocked(
 
 	if (lock_flags & (XFS_IOLOCK_EXCL|XFS_IOLOCK_SHARED)) {
 		if (!(lock_flags & XFS_IOLOCK_SHARED))
-			return !!ip->i_iolock.mr_writer;
-		return rwsem_is_locked(&ip->i_iolock.mr_lock);
+			return !debug_locks ||
+				lockdep_is_held_type(&VFS_I(ip)->i_rwsem, 0);
+		return rwsem_is_locked(&VFS_I(ip)->i_rwsem);
 	}
 
 	ASSERT(0);
@@ -421,11 +425,7 @@ xfs_lock_inumorder(int lock_mode, int subclass)
 
 	if (lock_mode & (XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL)) {
 		ASSERT(subclass <= XFS_IOLOCK_MAX_SUBCLASS);
-		ASSERT(xfs_lockdep_subclass_ok(subclass +
-						XFS_IOLOCK_PARENT_VAL));
 		class += subclass << XFS_IOLOCK_SHIFT;
-		if (lock_mode & XFS_IOLOCK_PARENT)
-			class += XFS_IOLOCK_PARENT_VAL << XFS_IOLOCK_SHIFT;
 	}
 
 	if (lock_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL)) {
@@ -477,8 +477,6 @@ xfs_lock_inodes(
 			    XFS_ILOCK_EXCL));
 	ASSERT(!(lock_mode & (XFS_IOLOCK_SHARED | XFS_MMAPLOCK_SHARED |
 			      XFS_ILOCK_SHARED)));
-	ASSERT(!(lock_mode & XFS_IOLOCK_EXCL) ||
-		inodes <= XFS_IOLOCK_MAX_SUBCLASS + 1);
 	ASSERT(!(lock_mode & XFS_MMAPLOCK_EXCL) ||
 		inodes <= XFS_MMAPLOCK_MAX_SUBCLASS + 1);
 	ASSERT(!(lock_mode & XFS_ILOCK_EXCL) ||
@@ -581,10 +579,8 @@ xfs_lock_two_inodes(
 	int			attempts = 0;
 	xfs_log_item_t		*lp;
 
-	if (lock_mode & (XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL)) {
-		ASSERT(!(lock_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL)));
-		ASSERT(!(lock_mode & (XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)));
-	} else if (lock_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL))
+	ASSERT(!(lock_mode & (XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL)));
+	if (lock_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL))
 		ASSERT(!(lock_mode & (XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)));
 
 	ASSERT(ip0->i_ino != ip1->i_ino);
@@ -715,7 +711,6 @@ xfs_lookup(
 	if (XFS_FORCED_SHUTDOWN(dp->i_mount))
 		return -EIO;
 
-	xfs_ilock(dp, XFS_IOLOCK_SHARED);
 	error = xfs_dir_lookup(NULL, dp, name, &inum, ci_name);
 	if (error)
 		goto out_unlock;
@@ -724,14 +719,12 @@ xfs_lookup(
 	if (error)
 		goto out_free_name;
 
-	xfs_iunlock(dp, XFS_IOLOCK_SHARED);
 	return 0;
 
 out_free_name:
 	if (ci_name)
 		kmem_free(ci_name->name);
 out_unlock:
-	xfs_iunlock(dp, XFS_IOLOCK_SHARED);
 	*ipp = NULL;
 	return error;
 }
@@ -1215,8 +1208,7 @@ xfs_create(
 	if (error)
 		goto out_release_inode;
 
-	xfs_ilock(dp, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL |
-		      XFS_IOLOCK_PARENT | XFS_ILOCK_PARENT);
+	xfs_ilock(dp, XFS_ILOCK_EXCL | XFS_ILOCK_PARENT);
 	unlock_dp_on_error = true;
 
 	xfs_defer_init(&dfops, &first_block);
@@ -1252,7 +1244,7 @@ xfs_create(
 	 * the transaction cancel unlocking dp so don't do it explicitly in the
 	 * error path.
 	 */
-	xfs_trans_ijoin(tp, dp, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL);
+	xfs_trans_ijoin(tp, dp, XFS_ILOCK_EXCL);
 	unlock_dp_on_error = false;
 
 	error = xfs_dir_createname(tp, dp, name, ip->i_ino,
@@ -1325,7 +1317,7 @@ xfs_create(
 	xfs_qm_dqrele(pdqp);
 
 	if (unlock_dp_on_error)
-		xfs_iunlock(dp, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL);
+		xfs_iunlock(dp, XFS_ILOCK_EXCL);
 	return error;
 }
 
@@ -1466,11 +1458,10 @@ xfs_link(
 	if (error)
 		goto std_return;
 
-	xfs_ilock(tdp, XFS_IOLOCK_EXCL | XFS_IOLOCK_PARENT);
 	xfs_lock_two_inodes(sip, tdp, XFS_ILOCK_EXCL);
 
 	xfs_trans_ijoin(tp, sip, XFS_ILOCK_EXCL);
-	xfs_trans_ijoin(tp, tdp, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL);
+	xfs_trans_ijoin(tp, tdp, XFS_ILOCK_EXCL);
 
 	/*
 	 * If we are using project inheritance, we only allow hard link
@@ -2041,7 +2032,6 @@ xfs_iunlink(
 	agi->agi_unlinked[bucket_index] = cpu_to_be32(agino);
 	offset = offsetof(xfs_agi_t, agi_unlinked) +
 		(sizeof(xfs_agino_t) * bucket_index);
-	xfs_trans_buf_set_type(tp, agibp, XFS_BLFT_AGI_BUF);
 	xfs_trans_log_buf(tp, agibp, offset,
 			  (offset + sizeof(xfs_agino_t) - 1));
 	return 0;
@@ -2133,7 +2123,6 @@ xfs_iunlink_remove(
 		agi->agi_unlinked[bucket_index] = cpu_to_be32(next_agino);
 		offset = offsetof(xfs_agi_t, agi_unlinked) +
 			(sizeof(xfs_agino_t) * bucket_index);
-		xfs_trans_buf_set_type(tp, agibp, XFS_BLFT_AGI_BUF);
 		xfs_trans_log_buf(tp, agibp, offset,
 				  (offset + sizeof(xfs_agino_t) - 1));
 	} else {
@@ -2579,10 +2568,9 @@ xfs_remove(
 		goto std_return;
 	}
 
-	xfs_ilock(dp, XFS_IOLOCK_EXCL | XFS_IOLOCK_PARENT);
 	xfs_lock_two_inodes(dp, ip, XFS_ILOCK_EXCL);
 
-	xfs_trans_ijoin(tp, dp, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL);
+	xfs_trans_ijoin(tp, dp, XFS_ILOCK_EXCL);
 	xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
 
 	/*
@@ -2963,12 +2951,6 @@ xfs_rename(
 	 * whether the target directory is the same as the source
 	 * directory, we can lock from 2 to 4 inodes.
 	 */
-	if (!new_parent)
-		xfs_ilock(src_dp, XFS_IOLOCK_EXCL | XFS_IOLOCK_PARENT);
-	else
-		xfs_lock_two_inodes(src_dp, target_dp,
-				    XFS_IOLOCK_EXCL | XFS_IOLOCK_PARENT);
-
 	xfs_lock_inodes(inodes, num_inodes, XFS_ILOCK_EXCL);
 
 	/*
@@ -2976,9 +2958,9 @@ xfs_rename(
 	 * we can rely on either trans_commit or trans_cancel to unlock
 	 * them.
 	 */
-	xfs_trans_ijoin(tp, src_dp, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL);
+	xfs_trans_ijoin(tp, src_dp, XFS_ILOCK_EXCL);
 	if (new_parent)
-		xfs_trans_ijoin(tp, target_dp, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL);
+		xfs_trans_ijoin(tp, target_dp, XFS_ILOCK_EXCL);
 	xfs_trans_ijoin(tp, src_ip, XFS_ILOCK_EXCL);
 	if (target_ip)
 		xfs_trans_ijoin(tp, target_ip, XFS_ILOCK_EXCL);
diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h
index f14c1de2549d..10dcf27b4c85 100644
--- a/fs/xfs/xfs_inode.h
+++ b/fs/xfs/xfs_inode.h
@@ -56,7 +56,6 @@ typedef struct xfs_inode {
 	/* Transaction and locking information. */
 	struct xfs_inode_log_item *i_itemp;	/* logging information */
 	mrlock_t		i_lock;		/* inode lock */
-	mrlock_t		i_iolock;	/* inode IO lock */
 	mrlock_t		i_mmaplock;	/* inode mmap IO lock */
 	atomic_t		i_pincount;	/* inode pin count */
 	spinlock_t		i_flags_lock;	/* inode i_flags lock */
@@ -246,6 +245,11 @@ static inline bool xfs_is_reflink_inode(struct xfs_inode *ip)
  * Synchronize processes attempting to flush the in-core inode back to disk.
  */
 
+static inline int xfs_isiflocked(struct xfs_inode *ip)
+{
+	return xfs_iflags_test(ip, XFS_IFLOCK);
+}
+
 extern void __xfs_iflock(struct xfs_inode *ip);
 
 static inline int xfs_iflock_nowait(struct xfs_inode *ip)
@@ -261,16 +265,12 @@ static inline void xfs_iflock(struct xfs_inode *ip)
 
 static inline void xfs_ifunlock(struct xfs_inode *ip)
 {
+	ASSERT(xfs_isiflocked(ip));
 	xfs_iflags_clear(ip, XFS_IFLOCK);
 	smp_mb();
 	wake_up_bit(&ip->i_flags, __XFS_IFLOCK_BIT);
 }
 
-static inline int xfs_isiflocked(struct xfs_inode *ip)
-{
-	return xfs_iflags_test(ip, XFS_IFLOCK);
-}
-
 /*
  * Flags for inode locking.
  * Bit ranges:	1<<1  - 1<<16-1 -- iolock/ilock modes (bitfield)
@@ -332,7 +332,7 @@ static inline int xfs_isiflocked(struct xfs_inode *ip)
  * IOLOCK values
  *
  * 0-3		subclass value
- * 4-7		PARENT subclass values
+ * 4-7		unused
  *
  * MMAPLOCK values
  *
@@ -347,10 +347,8 @@ static inline int xfs_isiflocked(struct xfs_inode *ip)
  * 
  */
 #define XFS_IOLOCK_SHIFT		16
-#define XFS_IOLOCK_PARENT_VAL		4
-#define XFS_IOLOCK_MAX_SUBCLASS		(XFS_IOLOCK_PARENT_VAL - 1)
+#define XFS_IOLOCK_MAX_SUBCLASS		3
 #define XFS_IOLOCK_DEP_MASK		0x000f0000
-#define	XFS_IOLOCK_PARENT		(XFS_IOLOCK_PARENT_VAL << XFS_IOLOCK_SHIFT)
 
 #define XFS_MMAPLOCK_SHIFT		20
 #define XFS_MMAPLOCK_NUMORDER		0
diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c
index 9610e9c00952..d90e7811ccdd 100644
--- a/fs/xfs/xfs_inode_item.c
+++ b/fs/xfs/xfs_inode_item.c
@@ -164,7 +164,7 @@ xfs_inode_item_format_data_fork(
 			struct xfs_bmbt_rec *p;
 
 			ASSERT(ip->i_df.if_u1.if_extents != NULL);
-			ASSERT(ip->i_df.if_bytes / sizeof(xfs_bmbt_rec_t) > 0);
+			ASSERT(xfs_iext_count(&ip->i_df) > 0);
 
 			p = xlog_prepare_iovec(lv, vecp, XLOG_REG_TYPE_IEXT);
 			data_bytes = xfs_iextents_copy(ip, p, XFS_DATA_FORK);
@@ -261,7 +261,7 @@ xfs_inode_item_format_attr_fork(
 		    ip->i_afp->if_bytes > 0) {
 			struct xfs_bmbt_rec *p;
 
-			ASSERT(ip->i_afp->if_bytes / sizeof(xfs_bmbt_rec_t) ==
+			ASSERT(xfs_iext_count(ip->i_afp) ==
 				ip->i_d.di_anextents);
 			ASSERT(ip->i_afp->if_u1.if_extents != NULL);
 
diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c
index c245bed3249b..fc563b82aea6 100644
--- a/fs/xfs/xfs_ioctl.c
+++ b/fs/xfs/xfs_ioctl.c
@@ -639,7 +639,7 @@ xfs_ioc_space(
 		return error;
 
 	xfs_ilock(ip, iolock);
-	error = xfs_break_layouts(inode, &iolock, false);
+	error = xfs_break_layouts(inode, &iolock);
 	if (error)
 		goto out_unlock;
 
@@ -910,16 +910,14 @@ xfs_ioc_fsgetxattr(
 	if (attr) {
 		if (ip->i_afp) {
 			if (ip->i_afp->if_flags & XFS_IFEXTENTS)
-				fa.fsx_nextents = ip->i_afp->if_bytes /
-							sizeof(xfs_bmbt_rec_t);
+				fa.fsx_nextents = xfs_iext_count(ip->i_afp);
 			else
 				fa.fsx_nextents = ip->i_d.di_anextents;
 		} else
 			fa.fsx_nextents = 0;
 	} else {
 		if (ip->i_df.if_flags & XFS_IFEXTENTS)
-			fa.fsx_nextents = ip->i_df.if_bytes /
-						sizeof(xfs_bmbt_rec_t);
+			fa.fsx_nextents = xfs_iext_count(&ip->i_df);
 		else
 			fa.fsx_nextents = ip->i_d.di_nextents;
 	}
diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c
index 436e109bb01e..0d147428971e 100644
--- a/fs/xfs/xfs_iomap.c
+++ b/fs/xfs/xfs_iomap.c
@@ -395,11 +395,12 @@ xfs_iomap_prealloc_size(
 	struct xfs_inode	*ip,
 	loff_t			offset,
 	loff_t			count,
-	xfs_extnum_t		idx,
-	struct xfs_bmbt_irec	*prev)
+	xfs_extnum_t		idx)
 {
 	struct xfs_mount	*mp = ip->i_mount;
+	struct xfs_ifork	*ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK);
 	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
+	struct xfs_bmbt_irec	prev;
 	int			shift = 0;
 	int64_t			freesp;
 	xfs_fsblock_t		qblocks;
@@ -419,8 +420,8 @@ xfs_iomap_prealloc_size(
 	 */
 	if ((mp->m_flags & XFS_MOUNT_DFLT_IOSIZE) ||
 	    XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign) ||
-	    idx == 0 ||
-	    prev->br_startoff + prev->br_blockcount < offset_fsb)
+	    !xfs_iext_get_extent(ifp, idx - 1, &prev) ||
+	    prev.br_startoff + prev.br_blockcount < offset_fsb)
 		return mp->m_writeio_blocks;
 
 	/*
@@ -439,8 +440,8 @@ xfs_iomap_prealloc_size(
 	 * always extends to MAXEXTLEN rather than falling short due to things
 	 * like stripe unit/width alignment of real extents.
 	 */
-	if (prev->br_blockcount <= (MAXEXTLEN >> 1))
-		alloc_blocks = prev->br_blockcount << 1;
+	if (prev.br_blockcount <= (MAXEXTLEN >> 1))
+		alloc_blocks = prev.br_blockcount << 1;
 	else
 		alloc_blocks = XFS_B_TO_FSB(mp, offset);
 	if (!alloc_blocks)
@@ -535,11 +536,11 @@ xfs_file_iomap_begin_delay(
 	xfs_fileoff_t		offset_fsb = XFS_B_TO_FSBT(mp, offset);
 	xfs_fileoff_t		maxbytes_fsb =
 		XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes);
-	xfs_fileoff_t		end_fsb, orig_end_fsb;
+	xfs_fileoff_t		end_fsb;
 	int			error = 0, eof = 0;
 	struct xfs_bmbt_irec	got;
-	struct xfs_bmbt_irec	prev;
 	xfs_extnum_t		idx;
+	xfs_fsblock_t		prealloc_blocks = 0;
 
 	ASSERT(!XFS_IS_REALTIME_INODE(ip));
 	ASSERT(!xfs_get_extsz_hint(ip));
@@ -563,8 +564,7 @@ xfs_file_iomap_begin_delay(
 			goto out_unlock;
 	}
 
-	xfs_bmap_search_extents(ip, offset_fsb, XFS_DATA_FORK, &eof, &idx,
-			&got, &prev);
+	eof = !xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got);
 	if (!eof && got.br_startoff <= offset_fsb) {
 		if (xfs_is_reflink_inode(ip)) {
 			bool		shared;
@@ -595,35 +595,32 @@ xfs_file_iomap_begin_delay(
 	 * the lower level functions are updated.
 	 */
 	count = min_t(loff_t, count, 1024 * PAGE_SIZE);
-	end_fsb = orig_end_fsb =
-		min(XFS_B_TO_FSB(mp, offset + count), maxbytes_fsb);
+	end_fsb = min(XFS_B_TO_FSB(mp, offset + count), maxbytes_fsb);
 
 	if (eof) {
-		xfs_fsblock_t	prealloc_blocks;
-
-		prealloc_blocks =
-			xfs_iomap_prealloc_size(ip, offset, count, idx, &prev);
+		prealloc_blocks = xfs_iomap_prealloc_size(ip, offset, count, idx);
 		if (prealloc_blocks) {
 			xfs_extlen_t	align;
 			xfs_off_t	end_offset;
+			xfs_fileoff_t	p_end_fsb;
 
 			end_offset = XFS_WRITEIO_ALIGN(mp, offset + count - 1);
-			end_fsb = XFS_B_TO_FSBT(mp, end_offset) +
-				prealloc_blocks;
+			p_end_fsb = XFS_B_TO_FSBT(mp, end_offset) +
+					prealloc_blocks;
 
 			align = xfs_eof_alignment(ip, 0);
 			if (align)
-				end_fsb = roundup_64(end_fsb, align);
+				p_end_fsb = roundup_64(p_end_fsb, align);
 
-			end_fsb = min(end_fsb, maxbytes_fsb);
-			ASSERT(end_fsb > offset_fsb);
+			p_end_fsb = min(p_end_fsb, maxbytes_fsb);
+			ASSERT(p_end_fsb > offset_fsb);
+			prealloc_blocks = p_end_fsb - end_fsb;
 		}
 	}
 
 retry:
 	error = xfs_bmapi_reserve_delalloc(ip, XFS_DATA_FORK, offset_fsb,
-			end_fsb - offset_fsb, &got,
-			&prev, &idx, eof);
+			end_fsb - offset_fsb, prealloc_blocks, &got, &idx, eof);
 	switch (error) {
 	case 0:
 		break;
@@ -631,8 +628,8 @@ retry:
 	case -EDQUOT:
 		/* retry without any preallocation */
 		trace_xfs_delalloc_enospc(ip, offset, count);
-		if (end_fsb != orig_end_fsb) {
-			end_fsb = orig_end_fsb;
+		if (prealloc_blocks) {
+			prealloc_blocks = 0;
 			goto retry;
 		}
 		/*FALLTHRU*/
@@ -640,13 +637,6 @@ retry:
 		goto out_unlock;
 	}
 
-	/*
-	 * Tag the inode as speculatively preallocated so we can reclaim this
-	 * space on demand, if necessary.
-	 */
-	if (end_fsb != orig_end_fsb)
-		xfs_inode_set_eofblocks_tag(ip);
-
 	trace_xfs_iomap_alloc(ip, offset, count, 0, &got);
 done:
 	if (isnullstartblock(got.br_startblock))
@@ -960,6 +950,19 @@ static inline bool imap_needs_alloc(struct inode *inode,
 		(IS_DAX(inode) && ISUNWRITTEN(imap));
 }
 
+static inline bool need_excl_ilock(struct xfs_inode *ip, unsigned flags)
+{
+	/*
+	 * COW writes will allocate delalloc space, so we need to make sure
+	 * to take the lock exclusively here.
+	 */
+	if (xfs_is_reflink_inode(ip) && (flags & (IOMAP_WRITE | IOMAP_ZERO)))
+		return true;
+	if ((flags & IOMAP_DIRECT) && (flags & IOMAP_WRITE))
+		return true;
+	return false;
+}
+
 static int
 xfs_file_iomap_begin(
 	struct inode		*inode,
@@ -979,18 +982,14 @@ xfs_file_iomap_begin(
 	if (XFS_FORCED_SHUTDOWN(mp))
 		return -EIO;
 
-	if ((flags & IOMAP_WRITE) && !IS_DAX(inode) &&
-		   !xfs_get_extsz_hint(ip)) {
+	if (((flags & (IOMAP_WRITE | IOMAP_DIRECT)) == IOMAP_WRITE) &&
+			!IS_DAX(inode) && !xfs_get_extsz_hint(ip)) {
 		/* Reserve delalloc blocks for regular writeback. */
 		return xfs_file_iomap_begin_delay(inode, offset, length, flags,
 				iomap);
 	}
 
-	/*
-	 * COW writes will allocate delalloc space, so we need to make sure
-	 * to take the lock exclusively here.
-	 */
-	if ((flags & (IOMAP_WRITE | IOMAP_ZERO)) && xfs_is_reflink_inode(ip)) {
+	if (need_excl_ilock(ip, flags)) {
 		lockmode = XFS_ILOCK_EXCL;
 		xfs_ilock(ip, XFS_ILOCK_EXCL);
 	} else {
@@ -1003,17 +1002,41 @@ xfs_file_iomap_begin(
 	offset_fsb = XFS_B_TO_FSBT(mp, offset);
 	end_fsb = XFS_B_TO_FSB(mp, offset + length);
 
+	if (xfs_is_reflink_inode(ip) &&
+	    (flags & IOMAP_WRITE) && (flags & IOMAP_DIRECT)) {
+		shared = xfs_reflink_find_cow_mapping(ip, offset, &imap);
+		if (shared) {
+			xfs_iunlock(ip, lockmode);
+			goto alloc_done;
+		}
+		ASSERT(!isnullstartblock(imap.br_startblock));
+	}
+
 	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
 			       &nimaps, 0);
 	if (error)
 		goto out_unlock;
 
-	if (flags & IOMAP_REPORT) {
+	if ((flags & IOMAP_REPORT) ||
+	    (xfs_is_reflink_inode(ip) &&
+	     (flags & IOMAP_WRITE) && (flags & IOMAP_DIRECT))) {
 		/* Trim the mapping to the nearest shared extent boundary. */
 		error = xfs_reflink_trim_around_shared(ip, &imap, &shared,
 				&trimmed);
 		if (error)
 			goto out_unlock;
+
+		/*
+		 * We're here because we're trying to do a directio write to a
+		 * region that isn't aligned to a filesystem block.  If the
+		 * extent is shared, fall back to buffered mode to handle the
+		 * RMW.
+		 */
+		if (!(flags & IOMAP_REPORT) && shared) {
+			trace_xfs_reflink_bounce_dio_write(ip, &imap);
+			error = -EREMCHG;
+			goto out_unlock;
+		}
 	}
 
 	if ((flags & (IOMAP_WRITE | IOMAP_ZERO)) && xfs_is_reflink_inode(ip)) {
@@ -1048,6 +1071,7 @@ xfs_file_iomap_begin(
 		if (error)
 			return error;
 
+alloc_done:
 		iomap->flags = IOMAP_F_NEW;
 		trace_xfs_iomap_alloc(ip, offset, length, 0, &imap);
 	} else {
diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c
index 405a65cd9d6b..b930be0b1596 100644
--- a/fs/xfs/xfs_iops.c
+++ b/fs/xfs/xfs_iops.c
@@ -983,15 +983,13 @@ xfs_vn_setattr(
 		struct xfs_inode	*ip = XFS_I(d_inode(dentry));
 		uint			iolock = XFS_IOLOCK_EXCL;
 
-		xfs_ilock(ip, iolock);
-		error = xfs_break_layouts(d_inode(dentry), &iolock, true);
-		if (!error) {
-			xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
-			iolock |= XFS_MMAPLOCK_EXCL;
+		error = xfs_break_layouts(d_inode(dentry), &iolock);
+		if (error)
+			return error;
 
-			error = xfs_vn_setattr_size(dentry, iattr);
-		}
-		xfs_iunlock(ip, iolock);
+		xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
+		error = xfs_vn_setattr_size(dentry, iattr);
+		xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
 	} else {
 		error = xfs_vn_setattr_nonsize(dentry, iattr);
 	}
diff --git a/fs/xfs/xfs_linux.h b/fs/xfs/xfs_linux.h
index 68640fb63a54..a415f822f2c1 100644
--- a/fs/xfs/xfs_linux.h
+++ b/fs/xfs/xfs_linux.h
@@ -78,6 +78,7 @@ typedef __u32			xfs_nlink_t;
 #include <linux/freezer.h>
 #include <linux/list_sort.h>
 #include <linux/ratelimit.h>
+#include <linux/rhashtable.h>
 
 #include <asm/page.h>
 #include <asm/div64.h>
diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c
index 3b74fa011bb1..c39ac14ff540 100644
--- a/fs/xfs/xfs_log.c
+++ b/fs/xfs/xfs_log.c
@@ -1668,7 +1668,7 @@ xlog_cksum(
 	__uint32_t		crc;
 
 	/* first generate the crc for the record header ... */
-	crc = xfs_start_cksum((char *)rhead,
+	crc = xfs_start_cksum_update((char *)rhead,
 			      sizeof(struct xlog_rec_header),
 			      offsetof(struct xlog_rec_header, h_crc));
 
@@ -1862,26 +1862,21 @@ xlog_sync(
 
 	bp->b_io_length = BTOBB(count);
 	bp->b_fspriv = iclog;
-	bp->b_flags &= ~(XBF_FUA | XBF_FLUSH);
-	bp->b_flags |= (XBF_ASYNC | XBF_SYNCIO | XBF_WRITE);
+	bp->b_flags &= ~XBF_FLUSH;
+	bp->b_flags |= (XBF_ASYNC | XBF_SYNCIO | XBF_WRITE | XBF_FUA);
 
-	if (log->l_mp->m_flags & XFS_MOUNT_BARRIER) {
-		bp->b_flags |= XBF_FUA;
-
-		/*
-		 * Flush the data device before flushing the log to make
-		 * sure all meta data written back from the AIL actually made
-		 * it to disk before stamping the new log tail LSN into the
-		 * log buffer.  For an external log we need to issue the
-		 * flush explicitly, and unfortunately synchronously here;
-		 * for an internal log we can simply use the block layer
-		 * state machine for preflushes.
-		 */
-		if (log->l_mp->m_logdev_targp != log->l_mp->m_ddev_targp)
-			xfs_blkdev_issue_flush(log->l_mp->m_ddev_targp);
-		else
-			bp->b_flags |= XBF_FLUSH;
-	}
+	/*
+	 * Flush the data device before flushing the log to make sure all meta
+	 * data written back from the AIL actually made it to disk before
+	 * stamping the new log tail LSN into the log buffer.  For an external
+	 * log we need to issue the flush explicitly, and unfortunately
+	 * synchronously here; for an internal log we can simply use the block
+	 * layer state machine for preflushes.
+	 */
+	if (log->l_mp->m_logdev_targp != log->l_mp->m_ddev_targp)
+		xfs_blkdev_issue_flush(log->l_mp->m_ddev_targp);
+	else
+		bp->b_flags |= XBF_FLUSH;
 
 	ASSERT(XFS_BUF_ADDR(bp) <= log->l_logBBsize-1);
 	ASSERT(XFS_BUF_ADDR(bp) + BTOBB(count) <= log->l_logBBsize);
@@ -1906,10 +1901,8 @@ xlog_sync(
 		xfs_buf_associate_memory(bp,
 				(char *)&iclog->ic_header + count, split);
 		bp->b_fspriv = iclog;
-		bp->b_flags &= ~(XBF_FUA | XBF_FLUSH);
-		bp->b_flags |= (XBF_ASYNC | XBF_SYNCIO | XBF_WRITE);
-		if (log->l_mp->m_flags & XFS_MOUNT_BARRIER)
-			bp->b_flags |= XBF_FUA;
+		bp->b_flags &= ~XBF_FLUSH;
+		bp->b_flags |= (XBF_ASYNC | XBF_SYNCIO | XBF_WRITE | XBF_FUA);
 
 		ASSERT(XFS_BUF_ADDR(bp) <= log->l_logBBsize-1);
 		ASSERT(XFS_BUF_ADDR(bp) + BTOBB(count) <= log->l_logBBsize);
diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c
index 9b3d7c76915d..4a98762ec8b4 100644
--- a/fs/xfs/xfs_log_recover.c
+++ b/fs/xfs/xfs_log_recover.c
@@ -2025,7 +2025,7 @@ xlog_peek_buffer_cancelled(
 	struct xlog		*log,
 	xfs_daddr_t		blkno,
 	uint			len,
-	ushort			flags)
+	unsigned short			flags)
 {
 	struct list_head	*bucket;
 	struct xfs_buf_cancel	*bcp;
@@ -2065,7 +2065,7 @@ xlog_check_buffer_cancelled(
 	struct xlog		*log,
 	xfs_daddr_t		blkno,
 	uint			len,
-	ushort			flags)
+	unsigned short			flags)
 {
 	struct xfs_buf_cancel	*bcp;
 
@@ -5113,19 +5113,21 @@ xlog_recover_process(
 	struct list_head	*buffer_list)
 {
 	int			error;
+	__le32			old_crc = rhead->h_crc;
 	__le32			crc;
 
+
 	crc = xlog_cksum(log, rhead, dp, be32_to_cpu(rhead->h_len));
 
 	/*
 	 * Nothing else to do if this is a CRC verification pass. Just return
 	 * if this a record with a non-zero crc. Unfortunately, mkfs always
-	 * sets h_crc to 0 so we must consider this valid even on v5 supers.
+	 * sets old_crc to 0 so we must consider this valid even on v5 supers.
 	 * Otherwise, return EFSBADCRC on failure so the callers up the stack
 	 * know precisely what failed.
 	 */
 	if (pass == XLOG_RECOVER_CRCPASS) {
-		if (rhead->h_crc && crc != rhead->h_crc)
+		if (old_crc && crc != old_crc)
 			return -EFSBADCRC;
 		return 0;
 	}
@@ -5136,11 +5138,11 @@ xlog_recover_process(
 	 * zero CRC check prevents warnings from being emitted when upgrading
 	 * the kernel from one that does not add CRCs by default.
 	 */
-	if (crc != rhead->h_crc) {
-		if (rhead->h_crc || xfs_sb_version_hascrc(&log->l_mp->m_sb)) {
+	if (crc != old_crc) {
+		if (old_crc || xfs_sb_version_hascrc(&log->l_mp->m_sb)) {
 			xfs_alert(log->l_mp,
 		"log record CRC mismatch: found 0x%x, expected 0x%x.",
-					le32_to_cpu(rhead->h_crc),
+					le32_to_cpu(old_crc),
 					le32_to_cpu(crc));
 			xfs_hex_dump(dp, 32);
 		}
diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c
index b341f10cf481..9b9540db17a6 100644
--- a/fs/xfs/xfs_mount.c
+++ b/fs/xfs/xfs_mount.c
@@ -157,6 +157,7 @@ xfs_free_perag(
 		spin_unlock(&mp->m_perag_lock);
 		ASSERT(pag);
 		ASSERT(atomic_read(&pag->pag_ref) == 0);
+		xfs_buf_hash_destroy(pag);
 		call_rcu(&pag->rcu_head, __xfs_free_perag);
 	}
 }
@@ -212,8 +213,8 @@ xfs_initialize_perag(
 		spin_lock_init(&pag->pag_ici_lock);
 		mutex_init(&pag->pag_ici_reclaim_lock);
 		INIT_RADIX_TREE(&pag->pag_ici_root, GFP_ATOMIC);
-		spin_lock_init(&pag->pag_buf_lock);
-		pag->pag_buf_tree = RB_ROOT;
+		if (xfs_buf_hash_init(pag))
+			goto out_unwind;
 
 		if (radix_tree_preload(GFP_NOFS))
 			goto out_unwind;
@@ -239,9 +240,11 @@ xfs_initialize_perag(
 	return 0;
 
 out_unwind:
+	xfs_buf_hash_destroy(pag);
 	kmem_free(pag);
 	for (; index > first_initialised; index--) {
 		pag = radix_tree_delete(&mp->m_perag_tree, index);
+		xfs_buf_hash_destroy(pag);
 		kmem_free(pag);
 	}
 	return error;
diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h
index 819b80b15bfb..84f785218907 100644
--- a/fs/xfs/xfs_mount.h
+++ b/fs/xfs/xfs_mount.h
@@ -393,8 +393,8 @@ typedef struct xfs_perag {
 	unsigned long	pag_ici_reclaim_cursor;	/* reclaim restart point */
 
 	/* buffer cache index */
-	spinlock_t	pag_buf_lock;	/* lock for pag_buf_tree */
-	struct rb_root	pag_buf_tree;	/* ordered tree of active buffers */
+	spinlock_t	pag_buf_lock;	/* lock for pag_buf_hash */
+	struct rhashtable pag_buf_hash;
 
 	/* for rcu-safe freeing */
 	struct rcu_head	rcu_head;
@@ -424,6 +424,9 @@ xfs_perag_resv(
 	}
 }
 
+int xfs_buf_hash_init(xfs_perag_t *pag);
+void xfs_buf_hash_destroy(xfs_perag_t *pag);
+
 extern void	xfs_uuid_table_free(void);
 extern int	xfs_log_sbcount(xfs_mount_t *);
 extern __uint64_t xfs_default_resblks(xfs_mount_t *mp);
diff --git a/fs/xfs/xfs_pnfs.c b/fs/xfs/xfs_pnfs.c
index 93a7aafa56d6..2f2dc3c09ad0 100644
--- a/fs/xfs/xfs_pnfs.c
+++ b/fs/xfs/xfs_pnfs.c
@@ -32,8 +32,7 @@
 int
 xfs_break_layouts(
 	struct inode		*inode,
-	uint			*iolock,
-	bool			with_imutex)
+	uint			*iolock)
 {
 	struct xfs_inode	*ip = XFS_I(inode);
 	int			error;
@@ -42,12 +41,8 @@ xfs_break_layouts(
 
 	while ((error = break_layout(inode, false) == -EWOULDBLOCK)) {
 		xfs_iunlock(ip, *iolock);
-		if (with_imutex && (*iolock & XFS_IOLOCK_EXCL))
-			inode_unlock(inode);
 		error = break_layout(inode, true);
 		*iolock = XFS_IOLOCK_EXCL;
-		if (with_imutex)
-			inode_lock(inode);
 		xfs_ilock(ip, *iolock);
 	}
 
diff --git a/fs/xfs/xfs_pnfs.h b/fs/xfs/xfs_pnfs.h
index e8339f74966b..b587cb99b2b7 100644
--- a/fs/xfs/xfs_pnfs.h
+++ b/fs/xfs/xfs_pnfs.h
@@ -8,10 +8,10 @@ int xfs_fs_map_blocks(struct inode *inode, loff_t offset, u64 length,
 int xfs_fs_commit_blocks(struct inode *inode, struct iomap *maps, int nr_maps,
 		struct iattr *iattr);
 
-int xfs_break_layouts(struct inode *inode, uint *iolock, bool with_imutex);
+int xfs_break_layouts(struct inode *inode, uint *iolock);
 #else
 static inline int
-xfs_break_layouts(struct inode *inode, uint *iolock, bool with_imutex)
+xfs_break_layouts(struct inode *inode, uint *iolock)
 {
 	return 0;
 }
diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c
index a60d9e2739d1..45e50ea90769 100644
--- a/fs/xfs/xfs_qm.c
+++ b/fs/xfs/xfs_qm.c
@@ -1135,7 +1135,7 @@ xfs_qm_get_rtblks(
 			return error;
 	}
 	rtblks = 0;
-	nextents = ifp->if_bytes / (uint)sizeof(xfs_bmbt_rec_t);
+	nextents = xfs_iext_count(ifp);
 	for (idx = 0; idx < nextents; idx++)
 		rtblks += xfs_bmbt_get_blockcount(xfs_iext_get_ext(ifp, idx));
 	*O_rtblks = (xfs_qcnt_t)rtblks;
diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c
index a279b4e7f5fe..88fd03c66e99 100644
--- a/fs/xfs/xfs_reflink.c
+++ b/fs/xfs/xfs_reflink.c
@@ -243,12 +243,11 @@ xfs_reflink_reserve_cow(
 	struct xfs_bmbt_irec	*imap,
 	bool			*shared)
 {
-	struct xfs_bmbt_irec	got, prev;
-	xfs_fileoff_t		end_fsb, orig_end_fsb;
-	int			eof = 0, error = 0;
-	bool			trimmed;
+	struct xfs_ifork	*ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
+	struct xfs_bmbt_irec	got;
+	int			error = 0;
+	bool			eof = false, trimmed;
 	xfs_extnum_t		idx;
-	xfs_extlen_t		align;
 
 	/*
 	 * Search the COW fork extent list first.  This serves two purposes:
@@ -258,8 +257,9 @@ xfs_reflink_reserve_cow(
 	 * extent list is generally faster than going out to the shared extent
 	 * tree.
 	 */
-	xfs_bmap_search_extents(ip, imap->br_startoff, XFS_COW_FORK, &eof, &idx,
-			&got, &prev);
+
+	if (!xfs_iext_lookup_extent(ip, ifp, imap->br_startoff, &idx, &got))
+		eof = true;
 	if (!eof && got.br_startoff <= imap->br_startoff) {
 		trace_xfs_reflink_cow_found(ip, imap);
 		xfs_trim_extent(imap, got.br_startoff, got.br_blockcount);
@@ -285,33 +285,12 @@ xfs_reflink_reserve_cow(
 	if (error)
 		return error;
 
-	end_fsb = orig_end_fsb = imap->br_startoff + imap->br_blockcount;
-
-	align = xfs_eof_alignment(ip, xfs_get_cowextsz_hint(ip));
-	if (align)
-		end_fsb = roundup_64(end_fsb, align);
-
-retry:
 	error = xfs_bmapi_reserve_delalloc(ip, XFS_COW_FORK, imap->br_startoff,
-			end_fsb - imap->br_startoff, &got, &prev, &idx, eof);
-	switch (error) {
-	case 0:
-		break;
-	case -ENOSPC:
-	case -EDQUOT:
-		/* retry without any preallocation */
+			imap->br_blockcount, 0, &got, &idx, eof);
+	if (error == -ENOSPC || error == -EDQUOT)
 		trace_xfs_reflink_cow_enospc(ip, imap);
-		if (end_fsb != orig_end_fsb) {
-			end_fsb = orig_end_fsb;
-			goto retry;
-		}
-		/*FALLTHRU*/
-	default:
+	if (error)
 		return error;
-	}
-
-	if (end_fsb != orig_end_fsb)
-		xfs_inode_set_cowblocks_tag(ip);
 
 	trace_xfs_reflink_cow_alloc(ip, &got);
 	return 0;
@@ -418,87 +397,65 @@ xfs_reflink_allocate_cow_range(
 }
 
 /*
- * Find the CoW reservation (and whether or not it needs block allocation)
- * for a given byte offset of a file.
+ * Find the CoW reservation for a given byte offset of a file.
  */
 bool
 xfs_reflink_find_cow_mapping(
 	struct xfs_inode		*ip,
 	xfs_off_t			offset,
-	struct xfs_bmbt_irec		*imap,
-	bool				*need_alloc)
+	struct xfs_bmbt_irec		*imap)
 {
-	struct xfs_bmbt_irec		irec;
-	struct xfs_ifork		*ifp;
-	struct xfs_bmbt_rec_host	*gotp;
-	xfs_fileoff_t			bno;
+	struct xfs_ifork		*ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
+	xfs_fileoff_t			offset_fsb;
+	struct xfs_bmbt_irec		got;
 	xfs_extnum_t			idx;
 
 	ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL | XFS_ILOCK_SHARED));
 	ASSERT(xfs_is_reflink_inode(ip));
 
-	/* Find the extent in the CoW fork. */
-	ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
-	bno = XFS_B_TO_FSBT(ip->i_mount, offset);
-	gotp = xfs_iext_bno_to_ext(ifp, bno, &idx);
-	if (!gotp)
+	offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
+	if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got))
 		return false;
-
-	xfs_bmbt_get_all(gotp, &irec);
-	if (bno >= irec.br_startoff + irec.br_blockcount ||
-	    bno < irec.br_startoff)
+	if (got.br_startoff > offset_fsb)
 		return false;
 
 	trace_xfs_reflink_find_cow_mapping(ip, offset, 1, XFS_IO_OVERWRITE,
-			&irec);
-
-	/* If it's still delalloc, we must allocate later. */
-	*imap = irec;
-	*need_alloc = !!(isnullstartblock(irec.br_startblock));
-
+			&got);
+	*imap = got;
 	return true;
 }
 
 /*
  * Trim an extent to end at the next CoW reservation past offset_fsb.
  */
-int
+void
 xfs_reflink_trim_irec_to_next_cow(
 	struct xfs_inode		*ip,
 	xfs_fileoff_t			offset_fsb,
 	struct xfs_bmbt_irec		*imap)
 {
-	struct xfs_bmbt_irec		irec;
-	struct xfs_ifork		*ifp;
-	struct xfs_bmbt_rec_host	*gotp;
+	struct xfs_ifork		*ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
+	struct xfs_bmbt_irec		got;
 	xfs_extnum_t			idx;
 
 	if (!xfs_is_reflink_inode(ip))
-		return 0;
+		return;
 
 	/* Find the extent in the CoW fork. */
-	ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
-	gotp = xfs_iext_bno_to_ext(ifp, offset_fsb, &idx);
-	if (!gotp)
-		return 0;
-	xfs_bmbt_get_all(gotp, &irec);
+	if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got))
+		return;
 
 	/* This is the extent before; try sliding up one. */
-	if (irec.br_startoff < offset_fsb) {
-		idx++;
-		if (idx >= ifp->if_bytes / sizeof(xfs_bmbt_rec_t))
-			return 0;
-		gotp = xfs_iext_get_ext(ifp, idx);
-		xfs_bmbt_get_all(gotp, &irec);
+	if (got.br_startoff < offset_fsb) {
+		if (!xfs_iext_get_extent(ifp, idx + 1, &got))
+			return;
 	}
 
-	if (irec.br_startoff >= imap->br_startoff + imap->br_blockcount)
-		return 0;
+	if (got.br_startoff >= imap->br_startoff + imap->br_blockcount)
+		return;
 
-	imap->br_blockcount = irec.br_startoff - imap->br_startoff;
+	imap->br_blockcount = got.br_startoff - imap->br_startoff;
 	trace_xfs_reflink_trim_irec(ip, imap);
-
-	return 0;
 }
 
 /*
@@ -512,18 +469,15 @@ xfs_reflink_cancel_cow_blocks(
 	xfs_fileoff_t			end_fsb)
 {
 	struct xfs_ifork		*ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
-	struct xfs_bmbt_irec		got, prev, del;
+	struct xfs_bmbt_irec		got, del;
 	xfs_extnum_t			idx;
 	xfs_fsblock_t			firstfsb;
 	struct xfs_defer_ops		dfops;
-	int				error = 0, eof = 0;
+	int				error = 0;
 
 	if (!xfs_is_reflink_inode(ip))
 		return 0;
-
-	xfs_bmap_search_extents(ip, offset_fsb, XFS_COW_FORK, &eof, &idx,
-			&got, &prev);
-	if (eof)
+	if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got))
 		return 0;
 
 	while (got.br_startoff < end_fsb) {
@@ -566,9 +520,8 @@ xfs_reflink_cancel_cow_blocks(
 			xfs_bmap_del_extent_cow(ip, &idx, &got, &del);
 		}
 
-		if (++idx >= ifp->if_bytes / sizeof(struct xfs_bmbt_rec))
+		if (!xfs_iext_get_extent(ifp, ++idx, &got))
 			break;
-		xfs_bmbt_get_all(xfs_iext_get_ext(ifp, idx), &got);
 	}
 
 	/* clear tag if cow fork is emptied */
@@ -638,13 +591,13 @@ xfs_reflink_end_cow(
 	xfs_off_t			count)
 {
 	struct xfs_ifork		*ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
-	struct xfs_bmbt_irec		got, prev, del;
+	struct xfs_bmbt_irec		got, del;
 	struct xfs_trans		*tp;
 	xfs_fileoff_t			offset_fsb;
 	xfs_fileoff_t			end_fsb;
 	xfs_fsblock_t			firstfsb;
 	struct xfs_defer_ops		dfops;
-	int				error, eof = 0;
+	int				error;
 	unsigned int			resblks;
 	xfs_filblks_t			rlen;
 	xfs_extnum_t			idx;
@@ -668,13 +621,11 @@ xfs_reflink_end_cow(
 	xfs_ilock(ip, XFS_ILOCK_EXCL);
 	xfs_trans_ijoin(tp, ip, 0);
 
-	xfs_bmap_search_extents(ip, end_fsb - 1, XFS_COW_FORK, &eof, &idx,
-			&got, &prev);
-
 	/* If there is a hole at end_fsb - 1 go to the previous extent */
-	if (eof || got.br_startoff > end_fsb) {
+	if (!xfs_iext_lookup_extent(ip, ifp, end_fsb - 1, &idx, &got) ||
+	    got.br_startoff > end_fsb) {
 		ASSERT(idx > 0);
-		xfs_bmbt_get_all(xfs_iext_get_ext(ifp, --idx), &got);
+		xfs_iext_get_extent(ifp, --idx, &got);
 	}
 
 	/* Walk backwards until we're out of the I/O range... */
@@ -722,11 +673,9 @@ xfs_reflink_end_cow(
 		error = xfs_defer_finish(&tp, &dfops, ip);
 		if (error)
 			goto out_defer;
-
 next_extent:
-		if (idx < 0)
+		if (!xfs_iext_get_extent(ifp, idx, &got))
 			break;
-		xfs_bmbt_get_all(xfs_iext_get_ext(ifp, idx), &got);
 	}
 
 	error = xfs_trans_commit(tp);
@@ -1302,13 +1251,11 @@ xfs_reflink_remap_range(
 		return -EIO;
 
 	/* Lock both files against IO */
-	if (same_inode) {
-		xfs_ilock(src, XFS_IOLOCK_EXCL);
+	lock_two_nondirectories(inode_in, inode_out);
+	if (same_inode)
 		xfs_ilock(src, XFS_MMAPLOCK_EXCL);
-	} else {
-		xfs_lock_two_inodes(src, dest, XFS_IOLOCK_EXCL);
+	else
 		xfs_lock_two_inodes(src, dest, XFS_MMAPLOCK_EXCL);
-	}
 
 	/* Don't touch certain kinds of inodes */
 	ret = -EPERM;
@@ -1345,8 +1292,14 @@ xfs_reflink_remap_range(
 		goto out_unlock;
 	}
 
-	if (len == 0)
+	/* Zero length dedupe exits immediately; reflink goes to EOF. */
+	if (len == 0) {
+		if (is_dedupe) {
+			ret = 0;
+			goto out_unlock;
+		}
 		len = isize - pos_in;
+	}
 
 	/* Ensure offsets don't wrap and the input is inside i_size */
 	if (pos_in + len < pos_in || pos_out + len < pos_out ||
@@ -1447,11 +1400,9 @@ xfs_reflink_remap_range(
 
 out_unlock:
 	xfs_iunlock(src, XFS_MMAPLOCK_EXCL);
-	xfs_iunlock(src, XFS_IOLOCK_EXCL);
-	if (src->i_ino != dest->i_ino) {
+	if (!same_inode)
 		xfs_iunlock(dest, XFS_MMAPLOCK_EXCL);
-		xfs_iunlock(dest, XFS_IOLOCK_EXCL);
-	}
+	unlock_two_nondirectories(inode_in, inode_out);
 	if (ret)
 		trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_);
 	return ret;
@@ -1697,37 +1648,3 @@ out:
 	trace_xfs_reflink_unshare_error(ip, error, _RET_IP_);
 	return error;
 }
-
-/*
- * Does this inode have any real CoW reservations?
- */
-bool
-xfs_reflink_has_real_cow_blocks(
-	struct xfs_inode		*ip)
-{
-	struct xfs_bmbt_irec		irec;
-	struct xfs_ifork		*ifp;
-	struct xfs_bmbt_rec_host	*gotp;
-	xfs_extnum_t			idx;
-
-	if (!xfs_is_reflink_inode(ip))
-		return false;
-
-	/* Go find the old extent in the CoW fork. */
-	ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
-	gotp = xfs_iext_bno_to_ext(ifp, 0, &idx);
-	while (gotp) {
-		xfs_bmbt_get_all(gotp, &irec);
-
-		if (!isnullstartblock(irec.br_startblock))
-			return true;
-
-		/* Roll on... */
-		idx++;
-		if (idx >= ifp->if_bytes / sizeof(xfs_bmbt_rec_t))
-			break;
-		gotp = xfs_iext_get_ext(ifp, idx);
-	}
-
-	return false;
-}
diff --git a/fs/xfs/xfs_reflink.h b/fs/xfs/xfs_reflink.h
index fad11607c9ad..aa6a4d64bd35 100644
--- a/fs/xfs/xfs_reflink.h
+++ b/fs/xfs/xfs_reflink.h
@@ -31,8 +31,8 @@ extern int xfs_reflink_reserve_cow(struct xfs_inode *ip,
 extern int xfs_reflink_allocate_cow_range(struct xfs_inode *ip,
 		xfs_off_t offset, xfs_off_t count);
 extern bool xfs_reflink_find_cow_mapping(struct xfs_inode *ip, xfs_off_t offset,
-		struct xfs_bmbt_irec *imap, bool *need_alloc);
-extern int xfs_reflink_trim_irec_to_next_cow(struct xfs_inode *ip,
+		struct xfs_bmbt_irec *imap);
+extern void xfs_reflink_trim_irec_to_next_cow(struct xfs_inode *ip,
 		xfs_fileoff_t offset_fsb, struct xfs_bmbt_irec *imap);
 
 extern int xfs_reflink_cancel_cow_blocks(struct xfs_inode *ip,
@@ -50,6 +50,4 @@ extern int xfs_reflink_clear_inode_flag(struct xfs_inode *ip,
 extern int xfs_reflink_unshare(struct xfs_inode *ip, xfs_off_t offset,
 		xfs_off_t len);
 
-extern bool xfs_reflink_has_real_cow_blocks(struct xfs_inode *ip);
-
 #endif /* __XFS_REFLINK_H */
diff --git a/fs/xfs/xfs_stats.c b/fs/xfs/xfs_stats.c
index 12d48cd8f8a4..f11282c96887 100644
--- a/fs/xfs/xfs_stats.c
+++ b/fs/xfs/xfs_stats.c
@@ -80,9 +80,9 @@ int xfs_stats_format(struct xfsstats __percpu *stats, char *buf)
 	}
 	/* extra precision counters */
 	for_each_possible_cpu(i) {
-		xs_xstrat_bytes += per_cpu_ptr(stats, i)->xs_xstrat_bytes;
-		xs_write_bytes += per_cpu_ptr(stats, i)->xs_write_bytes;
-		xs_read_bytes += per_cpu_ptr(stats, i)->xs_read_bytes;
+		xs_xstrat_bytes += per_cpu_ptr(stats, i)->s.xs_xstrat_bytes;
+		xs_write_bytes += per_cpu_ptr(stats, i)->s.xs_write_bytes;
+		xs_read_bytes += per_cpu_ptr(stats, i)->s.xs_read_bytes;
 	}
 
 	len += snprintf(buf + len, PATH_MAX-len, "xpc %Lu %Lu %Lu\n",
@@ -106,9 +106,9 @@ void xfs_stats_clearall(struct xfsstats __percpu *stats)
 	for_each_possible_cpu(c) {
 		preempt_disable();
 		/* save vn_active, it's a universal truth! */
-		vn_active = per_cpu_ptr(stats, c)->vn_active;
+		vn_active = per_cpu_ptr(stats, c)->s.vn_active;
 		memset(per_cpu_ptr(stats, c), 0, sizeof(*stats));
-		per_cpu_ptr(stats, c)->vn_active = vn_active;
+		per_cpu_ptr(stats, c)->s.vn_active = vn_active;
 		preempt_enable();
 	}
 }
diff --git a/fs/xfs/xfs_stats.h b/fs/xfs/xfs_stats.h
index 79ad2e69fc33..375840f5a99a 100644
--- a/fs/xfs/xfs_stats.h
+++ b/fs/xfs/xfs_stats.h
@@ -22,9 +22,37 @@
 #include <linux/percpu.h>
 
 /*
+ * The btree stats arrays have fixed offsets for the different stats. We
+ * store the base index in the btree cursor via XFS_STATS_CALC_INDEX() and
+ * that allows us to use fixed offsets into the stats array for each btree
+ * stat. These index offsets are defined in the order they will be emitted
+ * in the stats files, so it is possible to add new btree stat types by
+ * appending to the enum list below.
+ */
+enum {
+	__XBTS_lookup = 0,
+	__XBTS_compare = 1,
+	__XBTS_insrec = 2,
+	__XBTS_delrec = 3,
+	__XBTS_newroot = 4,
+	__XBTS_killroot = 5,
+	__XBTS_increment = 6,
+	__XBTS_decrement = 7,
+	__XBTS_lshift = 8,
+	__XBTS_rshift = 9,
+	__XBTS_split = 10,
+	__XBTS_join = 11,
+	__XBTS_alloc = 12,
+	__XBTS_free = 13,
+	__XBTS_moves = 14,
+
+	__XBTS_MAX = 15,
+};
+
+/*
  * XFS global statistics
  */
-struct xfsstats {
+struct __xfsstats {
 # define XFSSTAT_END_EXTENT_ALLOC	4
 	__uint32_t		xs_allocx;
 	__uint32_t		xs_allocb;
@@ -117,118 +145,20 @@ struct xfsstats {
 	__uint32_t		xb_page_found;
 	__uint32_t		xb_get_read;
 /* Version 2 btree counters */
-#define XFSSTAT_END_ABTB_V2		(XFSSTAT_END_BUF+15)
-	__uint32_t		xs_abtb_2_lookup;
-	__uint32_t		xs_abtb_2_compare;
-	__uint32_t		xs_abtb_2_insrec;
-	__uint32_t		xs_abtb_2_delrec;
-	__uint32_t		xs_abtb_2_newroot;
-	__uint32_t		xs_abtb_2_killroot;
-	__uint32_t		xs_abtb_2_increment;
-	__uint32_t		xs_abtb_2_decrement;
-	__uint32_t		xs_abtb_2_lshift;
-	__uint32_t		xs_abtb_2_rshift;
-	__uint32_t		xs_abtb_2_split;
-	__uint32_t		xs_abtb_2_join;
-	__uint32_t		xs_abtb_2_alloc;
-	__uint32_t		xs_abtb_2_free;
-	__uint32_t		xs_abtb_2_moves;
-#define XFSSTAT_END_ABTC_V2		(XFSSTAT_END_ABTB_V2+15)
-	__uint32_t		xs_abtc_2_lookup;
-	__uint32_t		xs_abtc_2_compare;
-	__uint32_t		xs_abtc_2_insrec;
-	__uint32_t		xs_abtc_2_delrec;
-	__uint32_t		xs_abtc_2_newroot;
-	__uint32_t		xs_abtc_2_killroot;
-	__uint32_t		xs_abtc_2_increment;
-	__uint32_t		xs_abtc_2_decrement;
-	__uint32_t		xs_abtc_2_lshift;
-	__uint32_t		xs_abtc_2_rshift;
-	__uint32_t		xs_abtc_2_split;
-	__uint32_t		xs_abtc_2_join;
-	__uint32_t		xs_abtc_2_alloc;
-	__uint32_t		xs_abtc_2_free;
-	__uint32_t		xs_abtc_2_moves;
-#define XFSSTAT_END_BMBT_V2		(XFSSTAT_END_ABTC_V2+15)
-	__uint32_t		xs_bmbt_2_lookup;
-	__uint32_t		xs_bmbt_2_compare;
-	__uint32_t		xs_bmbt_2_insrec;
-	__uint32_t		xs_bmbt_2_delrec;
-	__uint32_t		xs_bmbt_2_newroot;
-	__uint32_t		xs_bmbt_2_killroot;
-	__uint32_t		xs_bmbt_2_increment;
-	__uint32_t		xs_bmbt_2_decrement;
-	__uint32_t		xs_bmbt_2_lshift;
-	__uint32_t		xs_bmbt_2_rshift;
-	__uint32_t		xs_bmbt_2_split;
-	__uint32_t		xs_bmbt_2_join;
-	__uint32_t		xs_bmbt_2_alloc;
-	__uint32_t		xs_bmbt_2_free;
-	__uint32_t		xs_bmbt_2_moves;
-#define XFSSTAT_END_IBT_V2		(XFSSTAT_END_BMBT_V2+15)
-	__uint32_t		xs_ibt_2_lookup;
-	__uint32_t		xs_ibt_2_compare;
-	__uint32_t		xs_ibt_2_insrec;
-	__uint32_t		xs_ibt_2_delrec;
-	__uint32_t		xs_ibt_2_newroot;
-	__uint32_t		xs_ibt_2_killroot;
-	__uint32_t		xs_ibt_2_increment;
-	__uint32_t		xs_ibt_2_decrement;
-	__uint32_t		xs_ibt_2_lshift;
-	__uint32_t		xs_ibt_2_rshift;
-	__uint32_t		xs_ibt_2_split;
-	__uint32_t		xs_ibt_2_join;
-	__uint32_t		xs_ibt_2_alloc;
-	__uint32_t		xs_ibt_2_free;
-	__uint32_t		xs_ibt_2_moves;
-#define XFSSTAT_END_FIBT_V2		(XFSSTAT_END_IBT_V2+15)
-	__uint32_t		xs_fibt_2_lookup;
-	__uint32_t		xs_fibt_2_compare;
-	__uint32_t		xs_fibt_2_insrec;
-	__uint32_t		xs_fibt_2_delrec;
-	__uint32_t		xs_fibt_2_newroot;
-	__uint32_t		xs_fibt_2_killroot;
-	__uint32_t		xs_fibt_2_increment;
-	__uint32_t		xs_fibt_2_decrement;
-	__uint32_t		xs_fibt_2_lshift;
-	__uint32_t		xs_fibt_2_rshift;
-	__uint32_t		xs_fibt_2_split;
-	__uint32_t		xs_fibt_2_join;
-	__uint32_t		xs_fibt_2_alloc;
-	__uint32_t		xs_fibt_2_free;
-	__uint32_t		xs_fibt_2_moves;
-#define XFSSTAT_END_RMAP_V2		(XFSSTAT_END_FIBT_V2+15)
-	__uint32_t		xs_rmap_2_lookup;
-	__uint32_t		xs_rmap_2_compare;
-	__uint32_t		xs_rmap_2_insrec;
-	__uint32_t		xs_rmap_2_delrec;
-	__uint32_t		xs_rmap_2_newroot;
-	__uint32_t		xs_rmap_2_killroot;
-	__uint32_t		xs_rmap_2_increment;
-	__uint32_t		xs_rmap_2_decrement;
-	__uint32_t		xs_rmap_2_lshift;
-	__uint32_t		xs_rmap_2_rshift;
-	__uint32_t		xs_rmap_2_split;
-	__uint32_t		xs_rmap_2_join;
-	__uint32_t		xs_rmap_2_alloc;
-	__uint32_t		xs_rmap_2_free;
-	__uint32_t		xs_rmap_2_moves;
-#define XFSSTAT_END_REFCOUNT		(XFSSTAT_END_RMAP_V2 + 15)
-	__uint32_t		xs_refcbt_2_lookup;
-	__uint32_t		xs_refcbt_2_compare;
-	__uint32_t		xs_refcbt_2_insrec;
-	__uint32_t		xs_refcbt_2_delrec;
-	__uint32_t		xs_refcbt_2_newroot;
-	__uint32_t		xs_refcbt_2_killroot;
-	__uint32_t		xs_refcbt_2_increment;
-	__uint32_t		xs_refcbt_2_decrement;
-	__uint32_t		xs_refcbt_2_lshift;
-	__uint32_t		xs_refcbt_2_rshift;
-	__uint32_t		xs_refcbt_2_split;
-	__uint32_t		xs_refcbt_2_join;
-	__uint32_t		xs_refcbt_2_alloc;
-	__uint32_t		xs_refcbt_2_free;
-	__uint32_t		xs_refcbt_2_moves;
+#define XFSSTAT_END_ABTB_V2		(XFSSTAT_END_BUF + __XBTS_MAX)
+	__uint32_t		xs_abtb_2[__XBTS_MAX];
+#define XFSSTAT_END_ABTC_V2		(XFSSTAT_END_ABTB_V2 + __XBTS_MAX)
+	__uint32_t		xs_abtc_2[__XBTS_MAX];
+#define XFSSTAT_END_BMBT_V2		(XFSSTAT_END_ABTC_V2 + __XBTS_MAX)
+	__uint32_t		xs_bmbt_2[__XBTS_MAX];
+#define XFSSTAT_END_IBT_V2		(XFSSTAT_END_BMBT_V2 + __XBTS_MAX)
+	__uint32_t		xs_ibt_2[__XBTS_MAX];
+#define XFSSTAT_END_FIBT_V2		(XFSSTAT_END_IBT_V2 + __XBTS_MAX)
+	__uint32_t		xs_fibt_2[__XBTS_MAX];
+#define XFSSTAT_END_RMAP_V2		(XFSSTAT_END_FIBT_V2 + __XBTS_MAX)
+	__uint32_t		xs_rmap_2[__XBTS_MAX];
+#define XFSSTAT_END_REFCOUNT		(XFSSTAT_END_RMAP_V2 + __XBTS_MAX)
+	__uint32_t		xs_refcbt_2[__XBTS_MAX];
 #define XFSSTAT_END_XQMSTAT		(XFSSTAT_END_REFCOUNT + 6)
 	__uint32_t		xs_qm_dqreclaims;
 	__uint32_t		xs_qm_dqreclaim_misses;
@@ -245,26 +175,58 @@ struct xfsstats {
 	__uint64_t		xs_read_bytes;
 };
 
+struct xfsstats {
+	union {
+		struct __xfsstats	s;
+		uint32_t		a[XFSSTAT_END_XQMSTAT];
+	};
+};
+
+/*
+ * simple wrapper for getting the array index of s struct member offset
+ */
+#define XFS_STATS_CALC_INDEX(member)	\
+	(offsetof(struct __xfsstats, member) / (int)sizeof(__uint32_t))
+
+
 int xfs_stats_format(struct xfsstats __percpu *stats, char *buf);
 void xfs_stats_clearall(struct xfsstats __percpu *stats);
 extern struct xstats xfsstats;
 
 #define XFS_STATS_INC(mp, v)					\
 do {								\
-	per_cpu_ptr(xfsstats.xs_stats, current_cpu())->v++;	\
-	per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->v++;	\
+	per_cpu_ptr(xfsstats.xs_stats, current_cpu())->s.v++;	\
+	per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->s.v++;	\
 } while (0)
 
 #define XFS_STATS_DEC(mp, v)					\
 do {								\
-	per_cpu_ptr(xfsstats.xs_stats, current_cpu())->v--;	\
-	per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->v--;	\
+	per_cpu_ptr(xfsstats.xs_stats, current_cpu())->s.v--;	\
+	per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->s.v--;	\
 } while (0)
 
 #define XFS_STATS_ADD(mp, v, inc)					\
 do {									\
-	per_cpu_ptr(xfsstats.xs_stats, current_cpu())->v += (inc);	\
-	per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->v += (inc);	\
+	per_cpu_ptr(xfsstats.xs_stats, current_cpu())->s.v += (inc);	\
+	per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->s.v += (inc);	\
+} while (0)
+
+#define XFS_STATS_INC_OFF(mp, off)				\
+do {								\
+	per_cpu_ptr(xfsstats.xs_stats, current_cpu())->a[off]++;	\
+	per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->a[off]++;	\
+} while (0)
+
+#define XFS_STATS_DEC_OFF(mp, off)					\
+do {								\
+	per_cpu_ptr(xfsstats.xs_stats, current_cpu())->a[off];	\
+	per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->a[off];	\
+} while (0)
+
+#define XFS_STATS_ADD_OFF(mp, off, inc)					\
+do {									\
+	per_cpu_ptr(xfsstats.xs_stats, current_cpu())->a[off] += (inc);	\
+	per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->a[off] += (inc);	\
 } while (0)
 
 #if defined(CONFIG_PROC_FS)
diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c
index ade4691e3f74..eecbaac08eba 100644
--- a/fs/xfs/xfs_super.c
+++ b/fs/xfs/xfs_super.c
@@ -104,9 +104,6 @@ static const match_table_t tokens = {
 	{Opt_sysvgroups,"sysvgroups"},	/* group-ID from current process */
 	{Opt_allocsize,	"allocsize=%s"},/* preferred allocation size */
 	{Opt_norecovery,"norecovery"},	/* don't run XFS recovery */
-	{Opt_barrier,	"barrier"},	/* use writer barriers for log write and
-					 * unwritten extent conversion */
-	{Opt_nobarrier,	"nobarrier"},	/* .. disable */
 	{Opt_inode64,	"inode64"},	/* inodes can be allocated anywhere */
 	{Opt_inode32,   "inode32"},	/* inode allocation limited to
 					 * XFS_MAXINUMBER_32 */
@@ -134,6 +131,12 @@ static const match_table_t tokens = {
 	{Opt_nodiscard,	"nodiscard"},	/* Do not discard unused blocks */
 
 	{Opt_dax,	"dax"},		/* Enable direct access to bdev pages */
+
+	/* Deprecated mount options scheduled for removal */
+	{Opt_barrier,	"barrier"},	/* use writer barriers for log write and
+					 * unwritten extent conversion */
+	{Opt_nobarrier,	"nobarrier"},	/* .. disable */
+
 	{Opt_err,	NULL},
 };
 
@@ -301,12 +304,6 @@ xfs_parseargs(
 		case Opt_nouuid:
 			mp->m_flags |= XFS_MOUNT_NOUUID;
 			break;
-		case Opt_barrier:
-			mp->m_flags |= XFS_MOUNT_BARRIER;
-			break;
-		case Opt_nobarrier:
-			mp->m_flags &= ~XFS_MOUNT_BARRIER;
-			break;
 		case Opt_ikeep:
 			mp->m_flags |= XFS_MOUNT_IKEEP;
 			break;
@@ -374,6 +371,14 @@ xfs_parseargs(
 			mp->m_flags |= XFS_MOUNT_DAX;
 			break;
 #endif
+		case Opt_barrier:
+			xfs_warn(mp, "%s option is deprecated, ignoring.", p);
+			mp->m_flags |= XFS_MOUNT_BARRIER;
+			break;
+		case Opt_nobarrier:
+			xfs_warn(mp, "%s option is deprecated, ignoring.", p);
+			mp->m_flags &= ~XFS_MOUNT_BARRIER;
+			break;
 		default:
 			xfs_warn(mp, "unknown mount option [%s].", p);
 			return -EINVAL;
@@ -943,7 +948,7 @@ xfs_fs_destroy_inode(
 
 	trace_xfs_destroy_inode(ip);
 
-	ASSERT(!rwsem_is_locked(&ip->i_iolock.mr_lock));
+	ASSERT(!rwsem_is_locked(&inode->i_rwsem));
 	XFS_STATS_INC(ip->i_mount, vn_rele);
 	XFS_STATS_INC(ip->i_mount, vn_remove);
 
@@ -1238,9 +1243,11 @@ xfs_fs_remount(
 		token = match_token(p, tokens, args);
 		switch (token) {
 		case Opt_barrier:
+			xfs_warn(mp, "%s option is deprecated, ignoring.", p);
 			mp->m_flags |= XFS_MOUNT_BARRIER;
 			break;
 		case Opt_nobarrier:
+			xfs_warn(mp, "%s option is deprecated, ignoring.", p);
 			mp->m_flags &= ~XFS_MOUNT_BARRIER;
 			break;
 		case Opt_inode64:
diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c
index 58142aeeeea6..f2cb45ed1d54 100644
--- a/fs/xfs/xfs_symlink.c
+++ b/fs/xfs/xfs_symlink.c
@@ -238,8 +238,7 @@ xfs_symlink(
 	if (error)
 		goto out_release_inode;
 
-	xfs_ilock(dp, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL |
-		      XFS_IOLOCK_PARENT | XFS_ILOCK_PARENT);
+	xfs_ilock(dp, XFS_ILOCK_EXCL | XFS_ILOCK_PARENT);
 	unlock_dp_on_error = true;
 
 	/*
@@ -287,7 +286,7 @@ xfs_symlink(
 	 * the transaction cancel unlocking dp so don't do it explicitly in the
 	 * error path.
 	 */
-	xfs_trans_ijoin(tp, dp, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL);
+	xfs_trans_ijoin(tp, dp, XFS_ILOCK_EXCL);
 	unlock_dp_on_error = false;
 
 	/*
@@ -412,7 +411,7 @@ out_release_inode:
 	xfs_qm_dqrele(pdqp);
 
 	if (unlock_dp_on_error)
-		xfs_iunlock(dp, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL);
+		xfs_iunlock(dp, XFS_ILOCK_EXCL);
 	return error;
 }
 
diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h
index 0907752be62d..69c5bcd9a51b 100644
--- a/fs/xfs/xfs_trace.h
+++ b/fs/xfs/xfs_trace.h
@@ -355,7 +355,6 @@ DEFINE_BUF_EVENT(xfs_buf_rele);
 DEFINE_BUF_EVENT(xfs_buf_iodone);
 DEFINE_BUF_EVENT(xfs_buf_submit);
 DEFINE_BUF_EVENT(xfs_buf_submit_wait);
-DEFINE_BUF_EVENT(xfs_buf_bawrite);
 DEFINE_BUF_EVENT(xfs_buf_lock);
 DEFINE_BUF_EVENT(xfs_buf_lock_done);
 DEFINE_BUF_EVENT(xfs_buf_trylock_fail);
@@ -367,19 +366,15 @@ DEFINE_BUF_EVENT(xfs_buf_delwri_queue);
 DEFINE_BUF_EVENT(xfs_buf_delwri_queued);
 DEFINE_BUF_EVENT(xfs_buf_delwri_split);
 DEFINE_BUF_EVENT(xfs_buf_get_uncached);
-DEFINE_BUF_EVENT(xfs_bdstrat_shut);
 DEFINE_BUF_EVENT(xfs_buf_item_relse);
 DEFINE_BUF_EVENT(xfs_buf_item_iodone_async);
 DEFINE_BUF_EVENT(xfs_buf_error_relse);
 DEFINE_BUF_EVENT(xfs_buf_wait_buftarg);
-DEFINE_BUF_EVENT(xfs_trans_read_buf_io);
 DEFINE_BUF_EVENT(xfs_trans_read_buf_shut);
 
 /* not really buffer traces, but the buf provides useful information */
 DEFINE_BUF_EVENT(xfs_btree_corrupt);
-DEFINE_BUF_EVENT(xfs_da_btree_corrupt);
 DEFINE_BUF_EVENT(xfs_reset_dqcounts);
-DEFINE_BUF_EVENT(xfs_inode_item_push);
 
 /* pass flags explicitly */
 DECLARE_EVENT_CLASS(xfs_buf_flags_class,
@@ -541,7 +536,6 @@ DEFINE_BUF_ITEM_EVENT(xfs_trans_bjoin);
 DEFINE_BUF_ITEM_EVENT(xfs_trans_bhold);
 DEFINE_BUF_ITEM_EVENT(xfs_trans_bhold_release);
 DEFINE_BUF_ITEM_EVENT(xfs_trans_binval);
-DEFINE_BUF_ITEM_EVENT(xfs_trans_buf_ordered);
 
 DECLARE_EVENT_CLASS(xfs_filestream_class,
 	TP_PROTO(struct xfs_inode *ip, xfs_agnumber_t agno),
@@ -680,7 +674,6 @@ DEFINE_INODE_EVENT(xfs_ioctl_setattr);
 DEFINE_INODE_EVENT(xfs_dir_fsync);
 DEFINE_INODE_EVENT(xfs_file_fsync);
 DEFINE_INODE_EVENT(xfs_destroy_inode);
-DEFINE_INODE_EVENT(xfs_evict_inode);
 DEFINE_INODE_EVENT(xfs_update_time);
 
 DEFINE_INODE_EVENT(xfs_dquot_dqalloc);
@@ -798,7 +791,6 @@ TRACE_EVENT(xfs_irec_merge_post,
 DEFINE_EVENT(xfs_iref_class, name, \
 	TP_PROTO(struct xfs_inode *ip, unsigned long caller_ip), \
 	TP_ARGS(ip, caller_ip))
-DEFINE_IREF_EVENT(xfs_ihold);
 DEFINE_IREF_EVENT(xfs_irele);
 DEFINE_IREF_EVENT(xfs_inode_pin);
 DEFINE_IREF_EVENT(xfs_inode_unpin);
@@ -939,7 +931,6 @@ DEFINE_DQUOT_EVENT(xfs_dqget_miss);
 DEFINE_DQUOT_EVENT(xfs_dqget_freeing);
 DEFINE_DQUOT_EVENT(xfs_dqget_dup);
 DEFINE_DQUOT_EVENT(xfs_dqput);
-DEFINE_DQUOT_EVENT(xfs_dqput_wait);
 DEFINE_DQUOT_EVENT(xfs_dqput_free);
 DEFINE_DQUOT_EVENT(xfs_dqrele);
 DEFINE_DQUOT_EVENT(xfs_dqflush);
@@ -1815,7 +1806,6 @@ DEFINE_ATTR_EVENT(xfs_attr_sf_addname);
 DEFINE_ATTR_EVENT(xfs_attr_sf_create);
 DEFINE_ATTR_EVENT(xfs_attr_sf_lookup);
 DEFINE_ATTR_EVENT(xfs_attr_sf_remove);
-DEFINE_ATTR_EVENT(xfs_attr_sf_removename);
 DEFINE_ATTR_EVENT(xfs_attr_sf_to_leaf);
 
 DEFINE_ATTR_EVENT(xfs_attr_leaf_add);
@@ -1844,7 +1834,6 @@ DEFINE_ATTR_EVENT(xfs_attr_leaf_toosmall);
 
 DEFINE_ATTR_EVENT(xfs_attr_node_addname);
 DEFINE_ATTR_EVENT(xfs_attr_node_get);
-DEFINE_ATTR_EVENT(xfs_attr_node_lookup);
 DEFINE_ATTR_EVENT(xfs_attr_node_replace);
 DEFINE_ATTR_EVENT(xfs_attr_node_removename);
 
@@ -2440,11 +2429,9 @@ DEFINE_DEFER_EVENT(xfs_defer_finish_done);
 
 DEFINE_DEFER_ERROR_EVENT(xfs_defer_trans_roll_error);
 DEFINE_DEFER_ERROR_EVENT(xfs_defer_finish_error);
-DEFINE_DEFER_ERROR_EVENT(xfs_defer_op_finish_error);
 
 DEFINE_DEFER_PENDING_EVENT(xfs_defer_intake_work);
 DEFINE_DEFER_PENDING_EVENT(xfs_defer_intake_cancel);
-DEFINE_DEFER_PENDING_EVENT(xfs_defer_pending_commit);
 DEFINE_DEFER_PENDING_EVENT(xfs_defer_pending_cancel);
 DEFINE_DEFER_PENDING_EVENT(xfs_defer_pending_finish);
 DEFINE_DEFER_PENDING_EVENT(xfs_defer_pending_abort);
@@ -3092,87 +3079,6 @@ DEFINE_EVENT(xfs_double_io_class, name,	\
 		 struct xfs_inode *dest, xfs_off_t doffset), \
 	TP_ARGS(src, soffset, len, dest, doffset))
 
-/* two-file vfs io tracepoint class */
-DECLARE_EVENT_CLASS(xfs_double_vfs_io_class,
-	TP_PROTO(struct inode *src, u64 soffset, u64 len,
-		 struct inode *dest, u64 doffset),
-	TP_ARGS(src, soffset, len, dest, doffset),
-	TP_STRUCT__entry(
-		__field(dev_t, dev)
-		__field(unsigned long, src_ino)
-		__field(loff_t, src_isize)
-		__field(loff_t, src_offset)
-		__field(size_t, len)
-		__field(unsigned long, dest_ino)
-		__field(loff_t, dest_isize)
-		__field(loff_t, dest_offset)
-	),
-	TP_fast_assign(
-		__entry->dev = src->i_sb->s_dev;
-		__entry->src_ino = src->i_ino;
-		__entry->src_isize = i_size_read(src);
-		__entry->src_offset = soffset;
-		__entry->len = len;
-		__entry->dest_ino = dest->i_ino;
-		__entry->dest_isize = i_size_read(dest);
-		__entry->dest_offset = doffset;
-	),
-	TP_printk("dev %d:%d count %zd "
-		  "ino 0x%lx isize 0x%llx offset 0x%llx -> "
-		  "ino 0x%lx isize 0x%llx offset 0x%llx",
-		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  __entry->len,
-		  __entry->src_ino,
-		  __entry->src_isize,
-		  __entry->src_offset,
-		  __entry->dest_ino,
-		  __entry->dest_isize,
-		  __entry->dest_offset)
-)
-
-#define DEFINE_DOUBLE_VFS_IO_EVENT(name)	\
-DEFINE_EVENT(xfs_double_vfs_io_class, name,	\
-	TP_PROTO(struct inode *src, u64 soffset, u64 len, \
-		 struct inode *dest, u64 doffset), \
-	TP_ARGS(src, soffset, len, dest, doffset))
-
-/* CoW write tracepoint */
-DECLARE_EVENT_CLASS(xfs_copy_on_write_class,
-	TP_PROTO(struct xfs_inode *ip, xfs_fileoff_t lblk, xfs_fsblock_t pblk,
-		 xfs_extlen_t len, xfs_fsblock_t new_pblk),
-	TP_ARGS(ip, lblk, pblk, len, new_pblk),
-	TP_STRUCT__entry(
-		__field(dev_t, dev)
-		__field(xfs_ino_t, ino)
-		__field(xfs_fileoff_t, lblk)
-		__field(xfs_fsblock_t, pblk)
-		__field(xfs_extlen_t, len)
-		__field(xfs_fsblock_t, new_pblk)
-	),
-	TP_fast_assign(
-		__entry->dev = VFS_I(ip)->i_sb->s_dev;
-		__entry->ino = ip->i_ino;
-		__entry->lblk = lblk;
-		__entry->pblk = pblk;
-		__entry->len = len;
-		__entry->new_pblk = new_pblk;
-	),
-	TP_printk("dev %d:%d ino 0x%llx lblk 0x%llx pblk 0x%llx "
-		  "len 0x%x new_pblk %llu",
-		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  __entry->ino,
-		  __entry->lblk,
-		  __entry->pblk,
-		  __entry->len,
-		  __entry->new_pblk)
-)
-
-#define DEFINE_COW_EVENT(name)	\
-DEFINE_EVENT(xfs_copy_on_write_class, name,	\
-	TP_PROTO(struct xfs_inode *ip, xfs_fileoff_t lblk, xfs_fsblock_t pblk, \
-		 xfs_extlen_t len, xfs_fsblock_t new_pblk), \
-	TP_ARGS(ip, lblk, pblk, len, new_pblk))
-
 /* inode/irec events */
 DECLARE_EVENT_CLASS(xfs_inode_irec_class,
 	TP_PROTO(struct xfs_inode *ip, struct xfs_bmbt_irec *irec),
@@ -3292,8 +3198,6 @@ DEFINE_DOUBLE_IO_EVENT(xfs_reflink_remap_range);
 DEFINE_INODE_ERROR_EVENT(xfs_reflink_remap_range_error);
 DEFINE_INODE_ERROR_EVENT(xfs_reflink_set_inode_flag_error);
 DEFINE_INODE_ERROR_EVENT(xfs_reflink_update_inode_size_error);
-DEFINE_INODE_ERROR_EVENT(xfs_reflink_reflink_main_loop_error);
-DEFINE_INODE_ERROR_EVENT(xfs_reflink_read_iomap_error);
 DEFINE_INODE_ERROR_EVENT(xfs_reflink_remap_blocks_error);
 DEFINE_INODE_ERROR_EVENT(xfs_reflink_remap_extent_error);
 
@@ -3302,9 +3206,6 @@ DEFINE_DOUBLE_IO_EVENT(xfs_reflink_compare_extents);
 DEFINE_INODE_ERROR_EVENT(xfs_reflink_compare_extents_error);
 
 /* ioctl tracepoints */
-DEFINE_DOUBLE_VFS_IO_EVENT(xfs_ioctl_reflink);
-DEFINE_DOUBLE_VFS_IO_EVENT(xfs_ioctl_clone_range);
-DEFINE_DOUBLE_VFS_IO_EVENT(xfs_ioctl_file_extent_same);
 TRACE_EVENT(xfs_ioctl_clone,
 	TP_PROTO(struct inode *src, struct inode *dest),
 	TP_ARGS(src, dest),
@@ -3334,11 +3235,7 @@ TRACE_EVENT(xfs_ioctl_clone,
 
 /* unshare tracepoints */
 DEFINE_SIMPLE_IO_EVENT(xfs_reflink_unshare);
-DEFINE_SIMPLE_IO_EVENT(xfs_reflink_cow_eof_block);
-DEFINE_PAGE_EVENT(xfs_reflink_unshare_page);
 DEFINE_INODE_ERROR_EVENT(xfs_reflink_unshare_error);
-DEFINE_INODE_ERROR_EVENT(xfs_reflink_cow_eof_block_error);
-DEFINE_INODE_ERROR_EVENT(xfs_reflink_dirty_page_error);
 
 /* copy on write */
 DEFINE_INODE_IREC_EVENT(xfs_reflink_trim_around_shared);
@@ -3361,14 +3258,8 @@ DEFINE_INODE_ERROR_EVENT(xfs_reflink_allocate_cow_range_error);
 DEFINE_INODE_ERROR_EVENT(xfs_reflink_cancel_cow_range_error);
 DEFINE_INODE_ERROR_EVENT(xfs_reflink_end_cow_error);
 
-DEFINE_COW_EVENT(xfs_reflink_fork_buf);
-DEFINE_COW_EVENT(xfs_reflink_finish_fork_buf);
-DEFINE_INODE_ERROR_EVENT(xfs_reflink_fork_buf_error);
-DEFINE_INODE_ERROR_EVENT(xfs_reflink_finish_fork_buf_error);
 
-DEFINE_INODE_EVENT(xfs_reflink_cancel_pending_cow);
 DEFINE_INODE_IREC_EVENT(xfs_reflink_cancel_cow);
-DEFINE_INODE_ERROR_EVENT(xfs_reflink_cancel_pending_cow_error);
 
 /* rmap swapext tracepoints */
 DEFINE_INODE_IREC_EVENT(xfs_swap_extent_rmap_remap);
diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c
index 62900938f26d..0594db435972 100644
--- a/fs/xfs/xfs_xattr.c
+++ b/fs/xfs/xfs_xattr.c
@@ -130,7 +130,7 @@ const struct xattr_handler *xfs_xattr_handlers[] = {
 	NULL
 };
 
-static int
+static void
 __xfs_xattr_put_listent(
 	struct xfs_attr_list_context *context,
 	char *prefix,
@@ -148,7 +148,7 @@ __xfs_xattr_put_listent(
 	if (arraytop > context->firstu) {
 		context->count = -1;	/* insufficient space */
 		context->seen_enough = 1;
-		return 0;
+		return;
 	}
 	offset = (char *)context->alist + context->count;
 	strncpy(offset, prefix, prefix_len);
@@ -159,10 +159,10 @@ __xfs_xattr_put_listent(
 
 compute_size:
 	context->count += prefix_len + namelen + 1;
-	return 0;
+	return;
 }
 
-static int
+static void
 xfs_xattr_put_listent(
 	struct xfs_attr_list_context *context,
 	int		flags,
@@ -180,23 +180,19 @@ xfs_xattr_put_listent(
 		if (namelen == SGI_ACL_FILE_SIZE &&
 		    strncmp(name, SGI_ACL_FILE,
 			    SGI_ACL_FILE_SIZE) == 0) {
-			int ret = __xfs_xattr_put_listent(
+			__xfs_xattr_put_listent(
 					context, XATTR_SYSTEM_PREFIX,
 					XATTR_SYSTEM_PREFIX_LEN,
 					XATTR_POSIX_ACL_ACCESS,
 					strlen(XATTR_POSIX_ACL_ACCESS));
-			if (ret)
-				return ret;
 		} else if (namelen == SGI_ACL_DEFAULT_SIZE &&
 			 strncmp(name, SGI_ACL_DEFAULT,
 				 SGI_ACL_DEFAULT_SIZE) == 0) {
-			int ret = __xfs_xattr_put_listent(
+			__xfs_xattr_put_listent(
 					context, XATTR_SYSTEM_PREFIX,
 					XATTR_SYSTEM_PREFIX_LEN,
 					XATTR_POSIX_ACL_DEFAULT,
 					strlen(XATTR_POSIX_ACL_DEFAULT));
-			if (ret)
-				return ret;
 		}
 #endif
 
@@ -205,7 +201,7 @@ xfs_xattr_put_listent(
 		 * see them.
 		 */
 		if (!capable(CAP_SYS_ADMIN))
-			return 0;
+			return;
 
 		prefix = XATTR_TRUSTED_PREFIX;
 		prefix_len = XATTR_TRUSTED_PREFIX_LEN;
@@ -217,8 +213,9 @@ xfs_xattr_put_listent(
 		prefix_len = XATTR_USER_PREFIX_LEN;
 	}
 
-	return __xfs_xattr_put_listent(context, prefix, prefix_len, name,
-				       namelen);
+	__xfs_xattr_put_listent(context, prefix, prefix_len, name,
+				namelen);
+	return;
 }
 
 ssize_t
diff --git a/include/acpi/acconfig.h b/include/acpi/acconfig.h
index 12c2882bf647..d25da936750e 100644
--- a/include/acpi/acconfig.h
+++ b/include/acpi/acconfig.h
@@ -146,7 +146,7 @@
 
 /* Maximum number of While() loops before abort */
 
-#define ACPI_MAX_LOOP_COUNT             0xFFFF
+#define ACPI_MAX_LOOP_COUNT             0x000FFFFF
 
 /******************************************************************************
  *
diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h
index c1a524de67c5..4242c31ffaee 100644
--- a/include/acpi/acpi_bus.h
+++ b/include/acpi/acpi_bus.h
@@ -573,6 +573,8 @@ struct acpi_pci_root {
 
 bool acpi_dma_supported(struct acpi_device *adev);
 enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev);
+void acpi_dma_configure(struct device *dev, enum dev_dma_attr attr);
+void acpi_dma_deconfigure(struct device *dev);
 
 struct acpi_device *acpi_find_child_device(struct acpi_device *parent,
 					   u64 address, bool check_children);
diff --git a/include/acpi/acpixf.h b/include/acpi/acpixf.h
index c7b3a132dbe7..5c7356adc10b 100644
--- a/include/acpi/acpixf.h
+++ b/include/acpi/acpixf.h
@@ -46,7 +46,7 @@
 
 /* Current ACPICA subsystem version in YYYYMMDD format */
 
-#define ACPI_CA_VERSION                 0x20160831
+#define ACPI_CA_VERSION                 0x20160930
 
 #include <acpi/acconfig.h>
 #include <acpi/actypes.h>
@@ -259,6 +259,13 @@ ACPI_INIT_GLOBAL(u8, acpi_gbl_osi_data, 0);
 ACPI_INIT_GLOBAL(u8, acpi_gbl_reduced_hardware, FALSE);
 
 /*
+ * Maximum number of While() loop iterations before forced method abort.
+ * This mechanism is intended to prevent infinite loops during interpreter
+ * execution within a host kernel.
+ */
+ACPI_INIT_GLOBAL(u32, acpi_gbl_max_loop_iterations, ACPI_MAX_LOOP_COUNT);
+
+/*
  * This mechanism is used to trace a specified AML method. The method is
  * traced each time it is executed.
  */
diff --git a/include/acpi/processor.h b/include/acpi/processor.h
index f3db11c24654..c1ba00fc4888 100644
--- a/include/acpi/processor.h
+++ b/include/acpi/processor.h
@@ -249,6 +249,7 @@ extern int acpi_processor_register_performance(struct acpi_processor_performance
 					       *performance, unsigned int cpu);
 extern void acpi_processor_unregister_performance(unsigned int cpu);
 
+int acpi_processor_pstate_control(void);
 /* note: this locks both the calling module and the processor module
          if a _PPC object exists, rmmod is disallowed then */
 int acpi_processor_notify_smm(struct module *calling_module);
@@ -294,7 +295,7 @@ static inline void acpi_processor_ffh_cstate_enter(struct acpi_processor_cx
 #ifdef CONFIG_CPU_FREQ
 void acpi_processor_ppc_init(void);
 void acpi_processor_ppc_exit(void);
-int acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag);
+void acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag);
 extern int acpi_processor_get_bios_limit(int cpu, unsigned int *limit);
 #else
 static inline void acpi_processor_ppc_init(void)
diff --git a/include/acpi/video.h b/include/acpi/video.h
index 4536bd345ab4..bfe484da55d2 100644
--- a/include/acpi/video.h
+++ b/include/acpi/video.h
@@ -30,6 +30,17 @@ struct acpi_device;
 #define ACPI_VIDEO_DISPLAY_LEGACY_PANEL   0x0110
 #define ACPI_VIDEO_DISPLAY_LEGACY_TV      0x0200
 
+#define ACPI_VIDEO_NOTIFY_SWITCH		0x80
+#define ACPI_VIDEO_NOTIFY_PROBE			0x81
+#define ACPI_VIDEO_NOTIFY_CYCLE			0x82
+#define ACPI_VIDEO_NOTIFY_NEXT_OUTPUT		0x83
+#define ACPI_VIDEO_NOTIFY_PREV_OUTPUT		0x84
+#define ACPI_VIDEO_NOTIFY_CYCLE_BRIGHTNESS	0x85
+#define ACPI_VIDEO_NOTIFY_INC_BRIGHTNESS	0x86
+#define ACPI_VIDEO_NOTIFY_DEC_BRIGHTNESS	0x87
+#define ACPI_VIDEO_NOTIFY_ZERO_BRIGHTNESS	0x88
+#define ACPI_VIDEO_NOTIFY_DISPLAY_OFF		0x89
+
 enum acpi_backlight_type {
 	acpi_backlight_undef = -1,
 	acpi_backlight_none = 0,
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 31e1d639abed..9e3aa34341f4 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -566,6 +566,7 @@
 	IRQCHIP_OF_MATCH_TABLE()					\
 	ACPI_PROBE_TABLE(irqchip)					\
 	ACPI_PROBE_TABLE(clksrc)					\
+	ACPI_PROBE_TABLE(iort)						\
 	EARLYCON_TABLE()
 
 #define INIT_TEXT							\
diff --git a/include/clocksource/pxa.h b/include/clocksource/pxa.h
index 1efbe5a66958..a9a0f03024a4 100644
--- a/include/clocksource/pxa.h
+++ b/include/clocksource/pxa.h
@@ -12,7 +12,6 @@
 #ifndef _CLOCKSOURCE_PXA_H
 #define _CLOCKSOURCE_PXA_H
 
-extern void pxa_timer_nodt_init(int irq, void __iomem *base,
-			   unsigned long clock_tick_rate);
+extern void pxa_timer_nodt_init(int irq, void __iomem *base);
 
 #endif
diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h
new file mode 100644
index 000000000000..e328b52425a8
--- /dev/null
+++ b/include/crypto/acompress.h
@@ -0,0 +1,269 @@
+/*
+ * Asynchronous Compression operations
+ *
+ * Copyright (c) 2016, Intel Corporation
+ * Authors: Weigang Li <weigang.li@intel.com>
+ *          Giovanni Cabiddu <giovanni.cabiddu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+#ifndef _CRYPTO_ACOMP_H
+#define _CRYPTO_ACOMP_H
+#include <linux/crypto.h>
+
+#define CRYPTO_ACOMP_ALLOC_OUTPUT	0x00000001
+
+/**
+ * struct acomp_req - asynchronous (de)compression request
+ *
+ * @base:	Common attributes for asynchronous crypto requests
+ * @src:	Source Data
+ * @dst:	Destination data
+ * @slen:	Size of the input buffer
+ * @dlen:	Size of the output buffer and number of bytes produced
+ * @flags:	Internal flags
+ * @__ctx:	Start of private context data
+ */
+struct acomp_req {
+	struct crypto_async_request base;
+	struct scatterlist *src;
+	struct scatterlist *dst;
+	unsigned int slen;
+	unsigned int dlen;
+	u32 flags;
+	void *__ctx[] CRYPTO_MINALIGN_ATTR;
+};
+
+/**
+ * struct crypto_acomp - user-instantiated objects which encapsulate
+ * algorithms and core processing logic
+ *
+ * @compress:		Function performs a compress operation
+ * @decompress:		Function performs a de-compress operation
+ * @dst_free:		Frees destination buffer if allocated inside the
+ *			algorithm
+ * @reqsize:		Context size for (de)compression requests
+ * @base:		Common crypto API algorithm data structure
+ */
+struct crypto_acomp {
+	int (*compress)(struct acomp_req *req);
+	int (*decompress)(struct acomp_req *req);
+	void (*dst_free)(struct scatterlist *dst);
+	unsigned int reqsize;
+	struct crypto_tfm base;
+};
+
+/**
+ * struct acomp_alg - asynchronous compression algorithm
+ *
+ * @compress:	Function performs a compress operation
+ * @decompress:	Function performs a de-compress operation
+ * @dst_free:	Frees destination buffer if allocated inside the algorithm
+ * @init:	Initialize the cryptographic transformation object.
+ *		This function is used to initialize the cryptographic
+ *		transformation object. This function is called only once at
+ *		the instantiation time, right after the transformation context
+ *		was allocated. In case the cryptographic hardware has some
+ *		special requirements which need to be handled by software, this
+ *		function shall check for the precise requirement of the
+ *		transformation and put any software fallbacks in place.
+ * @exit:	Deinitialize the cryptographic transformation object. This is a
+ *		counterpart to @init, used to remove various changes set in
+ *		@init.
+ *
+ * @reqsize:	Context size for (de)compression requests
+ * @base:	Common crypto API algorithm data structure
+ */
+struct acomp_alg {
+	int (*compress)(struct acomp_req *req);
+	int (*decompress)(struct acomp_req *req);
+	void (*dst_free)(struct scatterlist *dst);
+	int (*init)(struct crypto_acomp *tfm);
+	void (*exit)(struct crypto_acomp *tfm);
+	unsigned int reqsize;
+	struct crypto_alg base;
+};
+
+/**
+ * DOC: Asynchronous Compression API
+ *
+ * The Asynchronous Compression API is used with the algorithms of type
+ * CRYPTO_ALG_TYPE_ACOMPRESS (listed as type "acomp" in /proc/crypto)
+ */
+
+/**
+ * crypto_alloc_acomp() -- allocate ACOMPRESS tfm handle
+ * @alg_name:	is the cra_name / name or cra_driver_name / driver name of the
+ *		compression algorithm e.g. "deflate"
+ * @type:	specifies the type of the algorithm
+ * @mask:	specifies the mask for the algorithm
+ *
+ * Allocate a handle for a compression algorithm. The returned struct
+ * crypto_acomp is the handle that is required for any subsequent
+ * API invocation for the compression operations.
+ *
+ * Return:	allocated handle in case of success; IS_ERR() is true in case
+ *		of an error, PTR_ERR() returns the error code.
+ */
+struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type,
+					u32 mask);
+
+static inline struct crypto_tfm *crypto_acomp_tfm(struct crypto_acomp *tfm)
+{
+	return &tfm->base;
+}
+
+static inline struct acomp_alg *__crypto_acomp_alg(struct crypto_alg *alg)
+{
+	return container_of(alg, struct acomp_alg, base);
+}
+
+static inline struct crypto_acomp *__crypto_acomp_tfm(struct crypto_tfm *tfm)
+{
+	return container_of(tfm, struct crypto_acomp, base);
+}
+
+static inline struct acomp_alg *crypto_acomp_alg(struct crypto_acomp *tfm)
+{
+	return __crypto_acomp_alg(crypto_acomp_tfm(tfm)->__crt_alg);
+}
+
+static inline unsigned int crypto_acomp_reqsize(struct crypto_acomp *tfm)
+{
+	return tfm->reqsize;
+}
+
+static inline void acomp_request_set_tfm(struct acomp_req *req,
+					 struct crypto_acomp *tfm)
+{
+	req->base.tfm = crypto_acomp_tfm(tfm);
+}
+
+static inline struct crypto_acomp *crypto_acomp_reqtfm(struct acomp_req *req)
+{
+	return __crypto_acomp_tfm(req->base.tfm);
+}
+
+/**
+ * crypto_free_acomp() -- free ACOMPRESS tfm handle
+ *
+ * @tfm:	ACOMPRESS tfm handle allocated with crypto_alloc_acomp()
+ */
+static inline void crypto_free_acomp(struct crypto_acomp *tfm)
+{
+	crypto_destroy_tfm(tfm, crypto_acomp_tfm(tfm));
+}
+
+static inline int crypto_has_acomp(const char *alg_name, u32 type, u32 mask)
+{
+	type &= ~CRYPTO_ALG_TYPE_MASK;
+	type |= CRYPTO_ALG_TYPE_ACOMPRESS;
+	mask |= CRYPTO_ALG_TYPE_MASK;
+
+	return crypto_has_alg(alg_name, type, mask);
+}
+
+/**
+ * acomp_request_alloc() -- allocates asynchronous (de)compression request
+ *
+ * @tfm:	ACOMPRESS tfm handle allocated with crypto_alloc_acomp()
+ *
+ * Return:	allocated handle in case of success or NULL in case of an error
+ */
+struct acomp_req *acomp_request_alloc(struct crypto_acomp *tfm);
+
+/**
+ * acomp_request_free() -- zeroize and free asynchronous (de)compression
+ *			   request as well as the output buffer if allocated
+ *			   inside the algorithm
+ *
+ * @req:	request to free
+ */
+void acomp_request_free(struct acomp_req *req);
+
+/**
+ * acomp_request_set_callback() -- Sets an asynchronous callback
+ *
+ * Callback will be called when an asynchronous operation on a given
+ * request is finished.
+ *
+ * @req:	request that the callback will be set for
+ * @flgs:	specify for instance if the operation may backlog
+ * @cmlp:	callback which will be called
+ * @data:	private data used by the caller
+ */
+static inline void acomp_request_set_callback(struct acomp_req *req,
+					      u32 flgs,
+					      crypto_completion_t cmpl,
+					      void *data)
+{
+	req->base.complete = cmpl;
+	req->base.data = data;
+	req->base.flags = flgs;
+}
+
+/**
+ * acomp_request_set_params() -- Sets request parameters
+ *
+ * Sets parameters required by an acomp operation
+ *
+ * @req:	asynchronous compress request
+ * @src:	pointer to input buffer scatterlist
+ * @dst:	pointer to output buffer scatterlist. If this is NULL, the
+ *		acomp layer will allocate the output memory
+ * @slen:	size of the input buffer
+ * @dlen:	size of the output buffer. If dst is NULL, this can be used by
+ *		the user to specify the maximum amount of memory to allocate
+ */
+static inline void acomp_request_set_params(struct acomp_req *req,
+					    struct scatterlist *src,
+					    struct scatterlist *dst,
+					    unsigned int slen,
+					    unsigned int dlen)
+{
+	req->src = src;
+	req->dst = dst;
+	req->slen = slen;
+	req->dlen = dlen;
+
+	if (!req->dst)
+		req->flags |= CRYPTO_ACOMP_ALLOC_OUTPUT;
+}
+
+/**
+ * crypto_acomp_compress() -- Invoke asynchronous compress operation
+ *
+ * Function invokes the asynchronous compress operation
+ *
+ * @req:	asynchronous compress request
+ *
+ * Return:	zero on success; error code in case of error
+ */
+static inline int crypto_acomp_compress(struct acomp_req *req)
+{
+	struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
+
+	return tfm->compress(req);
+}
+
+/**
+ * crypto_acomp_decompress() -- Invoke asynchronous decompress operation
+ *
+ * Function invokes the asynchronous decompress operation
+ *
+ * @req:	asynchronous compress request
+ *
+ * Return:	zero on success; error code in case of error
+ */
+static inline int crypto_acomp_decompress(struct acomp_req *req)
+{
+	struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
+
+	return tfm->decompress(req);
+}
+
+#endif
diff --git a/include/crypto/cbc.h b/include/crypto/cbc.h
new file mode 100644
index 000000000000..f5b8bfc22e6d
--- /dev/null
+++ b/include/crypto/cbc.h
@@ -0,0 +1,146 @@
+/*
+ * CBC: Cipher Block Chaining mode
+ *
+ * Copyright (c) 2016 Herbert Xu <herbert@gondor.apana.org.au>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#ifndef _CRYPTO_CBC_H
+#define _CRYPTO_CBC_H
+
+#include <crypto/internal/skcipher.h>
+#include <linux/string.h>
+#include <linux/types.h>
+
+static inline int crypto_cbc_encrypt_segment(
+	struct skcipher_walk *walk, struct crypto_skcipher *tfm,
+	void (*fn)(struct crypto_skcipher *, const u8 *, u8 *))
+{
+	unsigned int bsize = crypto_skcipher_blocksize(tfm);
+	unsigned int nbytes = walk->nbytes;
+	u8 *src = walk->src.virt.addr;
+	u8 *dst = walk->dst.virt.addr;
+	u8 *iv = walk->iv;
+
+	do {
+		crypto_xor(iv, src, bsize);
+		fn(tfm, iv, dst);
+		memcpy(iv, dst, bsize);
+
+		src += bsize;
+		dst += bsize;
+	} while ((nbytes -= bsize) >= bsize);
+
+	return nbytes;
+}
+
+static inline int crypto_cbc_encrypt_inplace(
+	struct skcipher_walk *walk, struct crypto_skcipher *tfm,
+	void (*fn)(struct crypto_skcipher *, const u8 *, u8 *))
+{
+	unsigned int bsize = crypto_skcipher_blocksize(tfm);
+	unsigned int nbytes = walk->nbytes;
+	u8 *src = walk->src.virt.addr;
+	u8 *iv = walk->iv;
+
+	do {
+		crypto_xor(src, iv, bsize);
+		fn(tfm, src, src);
+		iv = src;
+
+		src += bsize;
+	} while ((nbytes -= bsize) >= bsize);
+
+	memcpy(walk->iv, iv, bsize);
+
+	return nbytes;
+}
+
+static inline int crypto_cbc_encrypt_walk(struct skcipher_request *req,
+					  void (*fn)(struct crypto_skcipher *,
+						     const u8 *, u8 *))
+{
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	struct skcipher_walk walk;
+	int err;
+
+	err = skcipher_walk_virt(&walk, req, false);
+
+	while (walk.nbytes) {
+		if (walk.src.virt.addr == walk.dst.virt.addr)
+			err = crypto_cbc_encrypt_inplace(&walk, tfm, fn);
+		else
+			err = crypto_cbc_encrypt_segment(&walk, tfm, fn);
+		err = skcipher_walk_done(&walk, err);
+	}
+
+	return err;
+}
+
+static inline int crypto_cbc_decrypt_segment(
+	struct skcipher_walk *walk, struct crypto_skcipher *tfm,
+	void (*fn)(struct crypto_skcipher *, const u8 *, u8 *))
+{
+	unsigned int bsize = crypto_skcipher_blocksize(tfm);
+	unsigned int nbytes = walk->nbytes;
+	u8 *src = walk->src.virt.addr;
+	u8 *dst = walk->dst.virt.addr;
+	u8 *iv = walk->iv;
+
+	do {
+		fn(tfm, src, dst);
+		crypto_xor(dst, iv, bsize);
+		iv = src;
+
+		src += bsize;
+		dst += bsize;
+	} while ((nbytes -= bsize) >= bsize);
+
+	memcpy(walk->iv, iv, bsize);
+
+	return nbytes;
+}
+
+static inline int crypto_cbc_decrypt_inplace(
+	struct skcipher_walk *walk, struct crypto_skcipher *tfm,
+	void (*fn)(struct crypto_skcipher *, const u8 *, u8 *))
+{
+	unsigned int bsize = crypto_skcipher_blocksize(tfm);
+	unsigned int nbytes = walk->nbytes;
+	u8 *src = walk->src.virt.addr;
+	u8 last_iv[bsize];
+
+	/* Start of the last block. */
+	src += nbytes - (nbytes & (bsize - 1)) - bsize;
+	memcpy(last_iv, src, bsize);
+
+	for (;;) {
+		fn(tfm, src, src);
+		if ((nbytes -= bsize) < bsize)
+			break;
+		crypto_xor(src, src - bsize, bsize);
+		src -= bsize;
+	}
+
+	crypto_xor(src, walk->iv, bsize);
+	memcpy(walk->iv, last_iv, bsize);
+
+	return nbytes;
+}
+
+static inline int crypto_cbc_decrypt_blocks(
+	struct skcipher_walk *walk, struct crypto_skcipher *tfm,
+	void (*fn)(struct crypto_skcipher *, const u8 *, u8 *))
+{
+	if (walk->src.virt.addr == walk->dst.virt.addr)
+		return crypto_cbc_decrypt_inplace(walk, tfm, fn);
+	else
+		return crypto_cbc_decrypt_segment(walk, tfm, fn);
+}
+
+#endif	/* _CRYPTO_CBC_H */
diff --git a/include/crypto/cryptd.h b/include/crypto/cryptd.h
index bc792d5a9e88..94418cbf9013 100644
--- a/include/crypto/cryptd.h
+++ b/include/crypto/cryptd.h
@@ -12,10 +12,10 @@
 #ifndef _CRYPTO_CRYPT_H
 #define _CRYPTO_CRYPT_H
 
-#include <linux/crypto.h>
 #include <linux/kernel.h>
 #include <crypto/aead.h>
 #include <crypto/hash.h>
+#include <crypto/skcipher.h>
 
 struct cryptd_ablkcipher {
 	struct crypto_ablkcipher base;
@@ -34,6 +34,17 @@ struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm);
 bool cryptd_ablkcipher_queued(struct cryptd_ablkcipher *tfm);
 void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm);
 
+struct cryptd_skcipher {
+	struct crypto_skcipher base;
+};
+
+struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name,
+					      u32 type, u32 mask);
+struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm);
+/* Must be called without moving CPUs. */
+bool cryptd_skcipher_queued(struct cryptd_skcipher *tfm);
+void cryptd_free_skcipher(struct cryptd_skcipher *tfm);
+
 struct cryptd_ahash {
 	struct crypto_ahash base;
 };
diff --git a/include/crypto/engine.h b/include/crypto/engine.h
index 04eb5c77addd..1bf600fc99f7 100644
--- a/include/crypto/engine.h
+++ b/include/crypto/engine.h
@@ -43,8 +43,7 @@
  * @prepare_hash_request: do some prepare if need before handle the current request
  * @unprepare_hash_request: undo any work done by prepare_hash_request()
  * @hash_one_request: do hash for current request
- * @kworker: thread struct for request pump
- * @kworker_task: pointer to task for request pump kworker thread
+ * @kworker: kthread worker struct for request pump
  * @pump_requests: work struct for scheduling work to the request pump
  * @priv_data: the engine private data
  * @cur_req: the current request which is on processing
@@ -78,8 +77,7 @@ struct crypto_engine {
 	int (*hash_one_request)(struct crypto_engine *engine,
 				struct ahash_request *req);
 
-	struct kthread_worker           kworker;
-	struct task_struct              *kworker_task;
+	struct kthread_worker           *kworker;
 	struct kthread_work             pump_requests;
 
 	void				*priv_data;
diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h
index da2530e34b26..592d47e565a8 100644
--- a/include/crypto/gf128mul.h
+++ b/include/crypto/gf128mul.h
@@ -177,24 +177,23 @@ void gf128mul_4k_bbe(be128 *a, struct gf128mul_4k *t);
 
 static inline void gf128mul_free_4k(struct gf128mul_4k *t)
 {
-	kfree(t);
+	kzfree(t);
 }
 
 
-/* 64k table optimization, implemented for lle and bbe */
+/* 64k table optimization, implemented for bbe */
 
 struct gf128mul_64k {
 	struct gf128mul_4k *t[16];
 };
 
-/* first initialize with the constant factor with which you
- * want to multiply and then call gf128_64k_lle with the other
- * factor in the first argument, the table in the second and a
- * scratch register in the third. Afterwards *a = *r. */
-struct gf128mul_64k *gf128mul_init_64k_lle(const be128 *g);
+/* First initialize with the constant factor with which you
+ * want to multiply and then call gf128mul_64k_bbe with the other
+ * factor in the first argument, and the table in the second.
+ * Afterwards, the result is stored in *a.
+ */
 struct gf128mul_64k *gf128mul_init_64k_bbe(const be128 *g);
 void gf128mul_free_64k(struct gf128mul_64k *t);
-void gf128mul_64k_lle(be128 *a, struct gf128mul_64k *t);
 void gf128mul_64k_bbe(be128 *a, struct gf128mul_64k *t);
 
 #endif /* _CRYPTO_GF128MUL_H */
diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h
new file mode 100644
index 000000000000..1de2b5af12d7
--- /dev/null
+++ b/include/crypto/internal/acompress.h
@@ -0,0 +1,81 @@
+/*
+ * Asynchronous Compression operations
+ *
+ * Copyright (c) 2016, Intel Corporation
+ * Authors: Weigang Li <weigang.li@intel.com>
+ *          Giovanni Cabiddu <giovanni.cabiddu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+#ifndef _CRYPTO_ACOMP_INT_H
+#define _CRYPTO_ACOMP_INT_H
+#include <crypto/acompress.h>
+
+/*
+ * Transform internal helpers.
+ */
+static inline void *acomp_request_ctx(struct acomp_req *req)
+{
+	return req->__ctx;
+}
+
+static inline void *acomp_tfm_ctx(struct crypto_acomp *tfm)
+{
+	return tfm->base.__crt_ctx;
+}
+
+static inline void acomp_request_complete(struct acomp_req *req,
+					  int err)
+{
+	req->base.complete(&req->base, err);
+}
+
+static inline const char *acomp_alg_name(struct crypto_acomp *tfm)
+{
+	return crypto_acomp_tfm(tfm)->__crt_alg->cra_name;
+}
+
+static inline struct acomp_req *__acomp_request_alloc(struct crypto_acomp *tfm)
+{
+	struct acomp_req *req;
+
+	req = kzalloc(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL);
+	if (likely(req))
+		acomp_request_set_tfm(req, tfm);
+	return req;
+}
+
+static inline void __acomp_request_free(struct acomp_req *req)
+{
+	kzfree(req);
+}
+
+/**
+ * crypto_register_acomp() -- Register asynchronous compression algorithm
+ *
+ * Function registers an implementation of an asynchronous
+ * compression algorithm
+ *
+ * @alg:	algorithm definition
+ *
+ * Return:	zero on success; error code in case of error
+ */
+int crypto_register_acomp(struct acomp_alg *alg);
+
+/**
+ * crypto_unregister_acomp() -- Unregister asynchronous compression algorithm
+ *
+ * Function unregisters an implementation of an asynchronous
+ * compression algorithm
+ *
+ * @alg:	algorithm definition
+ *
+ * Return:	zero on success; error code in case of error
+ */
+int crypto_unregister_acomp(struct acomp_alg *alg);
+
+#endif
diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h
new file mode 100644
index 000000000000..3fda3c5655a0
--- /dev/null
+++ b/include/crypto/internal/scompress.h
@@ -0,0 +1,136 @@
+/*
+ * Synchronous Compression operations
+ *
+ * Copyright 2015 LG Electronics Inc.
+ * Copyright (c) 2016, Intel Corporation
+ * Author: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+#ifndef _CRYPTO_SCOMP_INT_H
+#define _CRYPTO_SCOMP_INT_H
+#include <linux/crypto.h>
+
+#define SCOMP_SCRATCH_SIZE	131072
+
+struct crypto_scomp {
+	struct crypto_tfm base;
+};
+
+/**
+ * struct scomp_alg - synchronous compression algorithm
+ *
+ * @alloc_ctx:	Function allocates algorithm specific context
+ * @free_ctx:	Function frees context allocated with alloc_ctx
+ * @compress:	Function performs a compress operation
+ * @decompress:	Function performs a de-compress operation
+ * @init:	Initialize the cryptographic transformation object.
+ *		This function is used to initialize the cryptographic
+ *		transformation object. This function is called only once at
+ *		the instantiation time, right after the transformation context
+ *		was allocated. In case the cryptographic hardware has some
+ *		special requirements which need to be handled by software, this
+ *		function shall check for the precise requirement of the
+ *		transformation and put any software fallbacks in place.
+ * @exit:	Deinitialize the cryptographic transformation object. This is a
+ *		counterpart to @init, used to remove various changes set in
+ *		@init.
+ * @base:	Common crypto API algorithm data structure
+ */
+struct scomp_alg {
+	void *(*alloc_ctx)(struct crypto_scomp *tfm);
+	void (*free_ctx)(struct crypto_scomp *tfm, void *ctx);
+	int (*compress)(struct crypto_scomp *tfm, const u8 *src,
+			unsigned int slen, u8 *dst, unsigned int *dlen,
+			void *ctx);
+	int (*decompress)(struct crypto_scomp *tfm, const u8 *src,
+			  unsigned int slen, u8 *dst, unsigned int *dlen,
+			  void *ctx);
+	struct crypto_alg base;
+};
+
+static inline struct scomp_alg *__crypto_scomp_alg(struct crypto_alg *alg)
+{
+	return container_of(alg, struct scomp_alg, base);
+}
+
+static inline struct crypto_scomp *__crypto_scomp_tfm(struct crypto_tfm *tfm)
+{
+	return container_of(tfm, struct crypto_scomp, base);
+}
+
+static inline struct crypto_tfm *crypto_scomp_tfm(struct crypto_scomp *tfm)
+{
+	return &tfm->base;
+}
+
+static inline void crypto_free_scomp(struct crypto_scomp *tfm)
+{
+	crypto_destroy_tfm(tfm, crypto_scomp_tfm(tfm));
+}
+
+static inline struct scomp_alg *crypto_scomp_alg(struct crypto_scomp *tfm)
+{
+	return __crypto_scomp_alg(crypto_scomp_tfm(tfm)->__crt_alg);
+}
+
+static inline void *crypto_scomp_alloc_ctx(struct crypto_scomp *tfm)
+{
+	return crypto_scomp_alg(tfm)->alloc_ctx(tfm);
+}
+
+static inline void crypto_scomp_free_ctx(struct crypto_scomp *tfm,
+					 void *ctx)
+{
+	return crypto_scomp_alg(tfm)->free_ctx(tfm, ctx);
+}
+
+static inline int crypto_scomp_compress(struct crypto_scomp *tfm,
+					const u8 *src, unsigned int slen,
+					u8 *dst, unsigned int *dlen, void *ctx)
+{
+	return crypto_scomp_alg(tfm)->compress(tfm, src, slen, dst, dlen, ctx);
+}
+
+static inline int crypto_scomp_decompress(struct crypto_scomp *tfm,
+					  const u8 *src, unsigned int slen,
+					  u8 *dst, unsigned int *dlen,
+					  void *ctx)
+{
+	return crypto_scomp_alg(tfm)->decompress(tfm, src, slen, dst, dlen,
+						 ctx);
+}
+
+int crypto_init_scomp_ops_async(struct crypto_tfm *tfm);
+struct acomp_req *crypto_acomp_scomp_alloc_ctx(struct acomp_req *req);
+void crypto_acomp_scomp_free_ctx(struct acomp_req *req);
+
+/**
+ * crypto_register_scomp() -- Register synchronous compression algorithm
+ *
+ * Function registers an implementation of a synchronous
+ * compression algorithm
+ *
+ * @alg:	algorithm definition
+ *
+ * Return: zero on success; error code in case of error
+ */
+int crypto_register_scomp(struct scomp_alg *alg);
+
+/**
+ * crypto_unregister_scomp() -- Unregister synchronous compression algorithm
+ *
+ * Function unregisters an implementation of a synchronous
+ * compression algorithm
+ *
+ * @alg:	algorithm definition
+ *
+ * Return: zero on success; error code in case of error
+ */
+int crypto_unregister_scomp(struct scomp_alg *alg);
+
+#endif
diff --git a/include/crypto/internal/simd.h b/include/crypto/internal/simd.h
new file mode 100644
index 000000000000..429509968f68
--- /dev/null
+++ b/include/crypto/internal/simd.h
@@ -0,0 +1,17 @@
+/*
+ * Shared crypto simd helpers
+ */
+
+#ifndef _CRYPTO_INTERNAL_SIMD_H
+#define _CRYPTO_INTERNAL_SIMD_H
+
+struct simd_skcipher_alg;
+
+struct simd_skcipher_alg *simd_skcipher_create_compat(const char *algname,
+						      const char *drvname,
+						      const char *basename);
+struct simd_skcipher_alg *simd_skcipher_create(const char *algname,
+					       const char *basename);
+void simd_skcipher_free(struct simd_skcipher_alg *alg);
+
+#endif /* _CRYPTO_INTERNAL_SIMD_H */
diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h
index a21a95e1a375..8735979ed341 100644
--- a/include/crypto/internal/skcipher.h
+++ b/include/crypto/internal/skcipher.h
@@ -15,8 +15,10 @@
 
 #include <crypto/algapi.h>
 #include <crypto/skcipher.h>
+#include <linux/list.h>
 #include <linux/types.h>
 
+struct aead_request;
 struct rtattr;
 
 struct skcipher_instance {
@@ -34,6 +36,40 @@ struct crypto_skcipher_spawn {
 	struct crypto_spawn base;
 };
 
+struct skcipher_walk {
+	union {
+		struct {
+			struct page *page;
+			unsigned long offset;
+		} phys;
+
+		struct {
+			u8 *page;
+			void *addr;
+		} virt;
+	} src, dst;
+
+	struct scatter_walk in;
+	unsigned int nbytes;
+
+	struct scatter_walk out;
+	unsigned int total;
+
+	struct list_head buffers;
+
+	u8 *page;
+	u8 *buffer;
+	u8 *oiv;
+	void *iv;
+
+	unsigned int ivsize;
+
+	int flags;
+	unsigned int blocksize;
+	unsigned int chunksize;
+	unsigned int alignmask;
+};
+
 extern const struct crypto_type crypto_givcipher_type;
 
 static inline struct crypto_instance *skcipher_crypto_instance(
@@ -68,14 +104,6 @@ static inline void crypto_set_skcipher_spawn(
 int crypto_grab_skcipher(struct crypto_skcipher_spawn *spawn, const char *name,
 			 u32 type, u32 mask);
 
-static inline int crypto_grab_skcipher2(struct crypto_skcipher_spawn *spawn,
-					const char *name, u32 type, u32 mask)
-{
-	return crypto_grab_skcipher(spawn, name, type, mask);
-}
-
-struct crypto_alg *crypto_lookup_skcipher(const char *name, u32 type, u32 mask);
-
 static inline void crypto_drop_skcipher(struct crypto_skcipher_spawn *spawn)
 {
 	crypto_drop_spawn(&spawn->base);
@@ -99,12 +127,6 @@ static inline struct crypto_skcipher *crypto_spawn_skcipher(
 	return crypto_spawn_tfm2(&spawn->base);
 }
 
-static inline struct crypto_skcipher *crypto_spawn_skcipher2(
-	struct crypto_skcipher_spawn *spawn)
-{
-	return crypto_spawn_skcipher(spawn);
-}
-
 static inline void crypto_skcipher_set_reqsize(
 	struct crypto_skcipher *skcipher, unsigned int reqsize)
 {
@@ -118,6 +140,21 @@ void crypto_unregister_skciphers(struct skcipher_alg *algs, int count);
 int skcipher_register_instance(struct crypto_template *tmpl,
 			       struct skcipher_instance *inst);
 
+int skcipher_walk_done(struct skcipher_walk *walk, int err);
+int skcipher_walk_virt(struct skcipher_walk *walk,
+		       struct skcipher_request *req,
+		       bool atomic);
+void skcipher_walk_atomise(struct skcipher_walk *walk);
+int skcipher_walk_async(struct skcipher_walk *walk,
+			struct skcipher_request *req);
+int skcipher_walk_aead(struct skcipher_walk *walk, struct aead_request *req,
+		       bool atomic);
+int skcipher_walk_aead_encrypt(struct skcipher_walk *walk,
+			       struct aead_request *req, bool atomic);
+int skcipher_walk_aead_decrypt(struct skcipher_walk *walk,
+			       struct aead_request *req, bool atomic);
+void skcipher_walk_complete(struct skcipher_walk *walk, int err);
+
 static inline void ablkcipher_request_complete(struct ablkcipher_request *req,
 					       int err)
 {
diff --git a/include/crypto/xts.h b/include/crypto/xts.h
index ede6b97b24cc..77b630672b2c 100644
--- a/include/crypto/xts.h
+++ b/include/crypto/xts.h
@@ -2,8 +2,7 @@
 #define _CRYPTO_XTS_H
 
 #include <crypto/b128ops.h>
-#include <linux/crypto.h>
-#include <crypto/algapi.h>
+#include <crypto/internal/skcipher.h>
 #include <linux/fips.h>
 
 struct scatterlist;
@@ -51,4 +50,27 @@ static inline int xts_check_key(struct crypto_tfm *tfm,
 	return 0;
 }
 
+static inline int xts_verify_key(struct crypto_skcipher *tfm,
+				 const u8 *key, unsigned int keylen)
+{
+	/*
+	 * key consists of keys of equal size concatenated, therefore
+	 * the length must be even.
+	 */
+	if (keylen % 2) {
+		crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	/* ensure that the AES and tweak key are not identical */
+	if ((fips_enabled || crypto_skcipher_get_flags(tfm) &
+			     CRYPTO_TFM_REQ_WEAK_KEY) &&
+	    !crypto_memneq(key, key + (keylen / 2), keylen / 2)) {
+		crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 #endif  /* _CRYPTO_XTS_H */
diff --git a/include/drm/bridge/mhl.h b/include/drm/bridge/mhl.h
new file mode 100644
index 000000000000..3629b2734db6
--- /dev/null
+++ b/include/drm/bridge/mhl.h
@@ -0,0 +1,291 @@
+/*
+ * Defines for Mobile High-Definition Link (MHL) interface
+ *
+ * Copyright (C) 2015, Samsung Electronics, Co., Ltd.
+ * Andrzej Hajda <a.hajda@samsung.com>
+ *
+ * Based on MHL driver for Android devices.
+ * Copyright (C) 2013-2014 Silicon Image, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __MHL_H__
+#define __MHL_H__
+
+/* Device Capabilities Registers */
+enum {
+	MHL_DCAP_DEV_STATE,
+	MHL_DCAP_MHL_VERSION,
+	MHL_DCAP_CAT,
+	MHL_DCAP_ADOPTER_ID_H,
+	MHL_DCAP_ADOPTER_ID_L,
+	MHL_DCAP_VID_LINK_MODE,
+	MHL_DCAP_AUD_LINK_MODE,
+	MHL_DCAP_VIDEO_TYPE,
+	MHL_DCAP_LOG_DEV_MAP,
+	MHL_DCAP_BANDWIDTH,
+	MHL_DCAP_FEATURE_FLAG,
+	MHL_DCAP_DEVICE_ID_H,
+	MHL_DCAP_DEVICE_ID_L,
+	MHL_DCAP_SCRATCHPAD_SIZE,
+	MHL_DCAP_INT_STAT_SIZE,
+	MHL_DCAP_RESERVED,
+	MHL_DCAP_SIZE
+};
+
+#define MHL_DCAP_CAT_SINK			0x01
+#define MHL_DCAP_CAT_SOURCE			0x02
+#define MHL_DCAP_CAT_POWER			0x10
+#define MHL_DCAP_CAT_PLIM(x)			((x) << 5)
+
+#define MHL_DCAP_VID_LINK_RGB444		0x01
+#define MHL_DCAP_VID_LINK_YCBCR444		0x02
+#define MHL_DCAP_VID_LINK_YCBCR422		0x04
+#define MHL_DCAP_VID_LINK_PPIXEL		0x08
+#define MHL_DCAP_VID_LINK_ISLANDS		0x10
+#define MHL_DCAP_VID_LINK_VGA			0x20
+#define MHL_DCAP_VID_LINK_16BPP			0x40
+
+#define MHL_DCAP_AUD_LINK_2CH			0x01
+#define MHL_DCAP_AUD_LINK_8CH			0x02
+
+#define MHL_DCAP_VT_GRAPHICS			0x00
+#define MHL_DCAP_VT_PHOTO			0x02
+#define MHL_DCAP_VT_CINEMA			0x04
+#define MHL_DCAP_VT_GAMES			0x08
+#define MHL_DCAP_SUPP_VT			0x80
+
+#define MHL_DCAP_LD_DISPLAY			0x01
+#define MHL_DCAP_LD_VIDEO			0x02
+#define MHL_DCAP_LD_AUDIO			0x04
+#define MHL_DCAP_LD_MEDIA			0x08
+#define MHL_DCAP_LD_TUNER			0x10
+#define MHL_DCAP_LD_RECORD			0x20
+#define MHL_DCAP_LD_SPEAKER			0x40
+#define MHL_DCAP_LD_GUI				0x80
+#define MHL_DCAP_LD_ALL				0xFF
+
+#define MHL_DCAP_FEATURE_RCP_SUPPORT		0x01
+#define MHL_DCAP_FEATURE_RAP_SUPPORT		0x02
+#define MHL_DCAP_FEATURE_SP_SUPPORT		0x04
+#define MHL_DCAP_FEATURE_UCP_SEND_SUPPOR	0x08
+#define MHL_DCAP_FEATURE_UCP_RECV_SUPPORT	0x10
+#define MHL_DCAP_FEATURE_RBP_SUPPORT		0x40
+
+/* Extended Device Capabilities Registers */
+enum {
+	MHL_XDC_ECBUS_SPEEDS,
+	MHL_XDC_TMDS_SPEEDS,
+	MHL_XDC_ECBUS_ROLES,
+	MHL_XDC_LOG_DEV_MAPX,
+	MHL_XDC_SIZE
+};
+
+#define MHL_XDC_ECBUS_S_075			0x01
+#define MHL_XDC_ECBUS_S_8BIT			0x02
+#define MHL_XDC_ECBUS_S_12BIT			0x04
+#define MHL_XDC_ECBUS_D_150			0x10
+#define MHL_XDC_ECBUS_D_8BIT			0x20
+
+#define MHL_XDC_TMDS_000			0x00
+#define MHL_XDC_TMDS_150			0x01
+#define MHL_XDC_TMDS_300			0x02
+#define MHL_XDC_TMDS_600			0x04
+
+/* MHL_XDC_ECBUS_ROLES flags */
+#define MHL_XDC_DEV_HOST			0x01
+#define MHL_XDC_DEV_DEVICE			0x02
+#define MHL_XDC_DEV_CHARGER			0x04
+#define MHL_XDC_HID_HOST			0x08
+#define MHL_XDC_HID_DEVICE			0x10
+
+/* MHL_XDC_LOG_DEV_MAPX flags */
+#define MHL_XDC_LD_PHONE			0x01
+
+/* Device Status Registers */
+enum {
+	MHL_DST_CONNECTED_RDY,
+	MHL_DST_LINK_MODE,
+	MHL_DST_VERSION,
+	MHL_DST_SIZE
+};
+
+/* Offset of DEVSTAT registers */
+#define MHL_DST_OFFSET				0x30
+#define MHL_DST_REG(name) (MHL_DST_OFFSET + MHL_DST_##name)
+
+#define MHL_DST_CONN_DCAP_RDY			0x01
+#define MHL_DST_CONN_XDEVCAPP_SUPP		0x02
+#define MHL_DST_CONN_POW_STAT			0x04
+#define MHL_DST_CONN_PLIM_STAT_MASK		0x38
+
+#define MHL_DST_LM_CLK_MODE_MASK		0x07
+#define MHL_DST_LM_CLK_MODE_PACKED_PIXEL	0x02
+#define MHL_DST_LM_CLK_MODE_NORMAL		0x03
+#define MHL_DST_LM_PATH_EN_MASK			0x08
+#define MHL_DST_LM_PATH_ENABLED			0x08
+#define MHL_DST_LM_PATH_DISABLED		0x00
+#define MHL_DST_LM_MUTED_MASK			0x10
+
+/* Extended Device Status Registers */
+enum {
+	MHL_XDS_CURR_ECBUS_MODE,
+	MHL_XDS_AVLINK_MODE_STATUS,
+	MHL_XDS_AVLINK_MODE_CONTROL,
+	MHL_XDS_MULTI_SINK_STATUS,
+	MHL_XDS_SIZE
+};
+
+/* Offset of XDEVSTAT registers */
+#define MHL_XDS_OFFSET				0x90
+#define MHL_XDS_REG(name) (MHL_XDS_OFFSET + MHL_XDS_##name)
+
+/* MHL_XDS_REG_CURR_ECBUS_MODE flags */
+#define MHL_XDS_SLOT_MODE_8BIT			0x00
+#define MHL_XDS_SLOT_MODE_6BIT			0x01
+#define MHL_XDS_ECBUS_S				0x04
+#define MHL_XDS_ECBUS_D				0x08
+
+#define MHL_XDS_LINK_CLOCK_75MHZ		0x00
+#define MHL_XDS_LINK_CLOCK_150MHZ		0x10
+#define MHL_XDS_LINK_CLOCK_300MHZ		0x20
+#define MHL_XDS_LINK_CLOCK_600MHZ		0x30
+
+#define MHL_XDS_LINK_STATUS_NO_SIGNAL		0x00
+#define MHL_XDS_LINK_STATUS_CRU_LOCKED		0x01
+#define MHL_XDS_LINK_STATUS_TMDS_NORMAL		0x02
+#define MHL_XDS_LINK_STATUS_TMDS_RESERVED	0x03
+
+#define MHL_XDS_LINK_RATE_1_5_GBPS		0x00
+#define MHL_XDS_LINK_RATE_3_0_GBPS		0x01
+#define MHL_XDS_LINK_RATE_6_0_GBPS		0x02
+#define MHL_XDS_ATT_CAPABLE			0x08
+
+#define MHL_XDS_SINK_STATUS_1_HPD_LOW		0x00
+#define MHL_XDS_SINK_STATUS_1_HPD_HIGH		0x01
+#define MHL_XDS_SINK_STATUS_2_HPD_LOW		0x00
+#define MHL_XDS_SINK_STATUS_2_HPD_HIGH		0x04
+#define MHL_XDS_SINK_STATUS_3_HPD_LOW		0x00
+#define MHL_XDS_SINK_STATUS_3_HPD_HIGH		0x10
+#define MHL_XDS_SINK_STATUS_4_HPD_LOW		0x00
+#define MHL_XDS_SINK_STATUS_4_HPD_HIGH		0x40
+
+/* Interrupt Registers */
+enum {
+	MHL_INT_RCHANGE,
+	MHL_INT_DCHANGE,
+	MHL_INT_SIZE
+};
+
+/* Offset of DEVSTAT registers */
+#define MHL_INT_OFFSET				0x20
+#define MHL_INT_REG(name) (MHL_INT_OFFSET + MHL_INT_##name)
+
+#define	MHL_INT_RC_DCAP_CHG			0x01
+#define MHL_INT_RC_DSCR_CHG			0x02
+#define MHL_INT_RC_REQ_WRT			0x04
+#define MHL_INT_RC_GRT_WRT			0x08
+#define MHL_INT_RC_3D_REQ			0x10
+#define MHL_INT_RC_FEAT_REQ			0x20
+#define MHL_INT_RC_FEAT_COMPLETE		0x40
+
+#define MHL_INT_DC_EDID_CHG			0x02
+
+enum {
+	MHL_ACK = 0x33, /* Command or Data byte acknowledge */
+	MHL_NACK = 0x34, /* Command or Data byte not acknowledge */
+	MHL_ABORT = 0x35, /* Transaction abort */
+	MHL_WRITE_STAT = 0xe0, /* Write one status register */
+	MHL_SET_INT = 0x60, /* Write one interrupt register */
+	MHL_READ_DEVCAP_REG = 0x61, /* Read one register */
+	MHL_GET_STATE = 0x62, /* Read CBUS revision level from follower */
+	MHL_GET_VENDOR_ID = 0x63, /* Read vendor ID value from follower */
+	MHL_SET_HPD = 0x64, /* Set Hot Plug Detect in follower */
+	MHL_CLR_HPD = 0x65, /* Clear Hot Plug Detect in follower */
+	MHL_SET_CAP_ID = 0x66, /* Set Capture ID for downstream device */
+	MHL_GET_CAP_ID = 0x67, /* Get Capture ID from downstream device */
+	MHL_MSC_MSG = 0x68, /* VS command to send RCP sub-commands */
+	MHL_GET_SC1_ERRORCODE = 0x69, /* Get Vendor-Specific error code */
+	MHL_GET_DDC_ERRORCODE = 0x6A, /* Get DDC channel command error code */
+	MHL_GET_MSC_ERRORCODE = 0x6B, /* Get MSC command error code */
+	MHL_WRITE_BURST = 0x6C, /* Write 1-16 bytes to responder's scratchpad */
+	MHL_GET_SC3_ERRORCODE = 0x6D, /* Get channel 3 command error code */
+	MHL_WRITE_XSTAT = 0x70, /* Write one extended status register */
+	MHL_READ_XDEVCAP_REG = 0x71, /* Read one extended devcap register */
+	/* let the rest of these float, they are software specific */
+	MHL_READ_EDID_BLOCK,
+	MHL_SEND_3D_REQ_OR_FEAT_REQ,
+	MHL_READ_DEVCAP,
+	MHL_READ_XDEVCAP
+};
+
+/* MSC message types */
+enum {
+	MHL_MSC_MSG_RCP = 0x10, /* RCP sub-command */
+	MHL_MSC_MSG_RCPK = 0x11, /* RCP Acknowledge sub-command */
+	MHL_MSC_MSG_RCPE = 0x12, /* RCP Error sub-command */
+	MHL_MSC_MSG_RAP = 0x20, /* Mode Change Warning sub-command */
+	MHL_MSC_MSG_RAPK = 0x21, /* MCW Acknowledge sub-command */
+	MHL_MSC_MSG_RBP = 0x22, /* Remote Button Protocol sub-command */
+	MHL_MSC_MSG_RBPK = 0x23, /* RBP Acknowledge sub-command */
+	MHL_MSC_MSG_RBPE = 0x24, /* RBP Error sub-command */
+	MHL_MSC_MSG_UCP = 0x30, /* UCP sub-command */
+	MHL_MSC_MSG_UCPK = 0x31, /* UCP Acknowledge sub-command */
+	MHL_MSC_MSG_UCPE = 0x32, /* UCP Error sub-command */
+	MHL_MSC_MSG_RUSB = 0x40, /* Request USB host role */
+	MHL_MSC_MSG_RUSBK = 0x41, /* Acknowledge request for USB host role */
+	MHL_MSC_MSG_RHID = 0x42, /* Request HID host role */
+	MHL_MSC_MSG_RHIDK = 0x43, /* Acknowledge request for HID host role */
+	MHL_MSC_MSG_ATT = 0x50, /* Request attention sub-command */
+	MHL_MSC_MSG_ATTK = 0x51, /* ATT Acknowledge sub-command */
+	MHL_MSC_MSG_BIST_TRIGGER = 0x60,
+	MHL_MSC_MSG_BIST_REQUEST_STAT = 0x61,
+	MHL_MSC_MSG_BIST_READY = 0x62,
+	MHL_MSC_MSG_BIST_STOP = 0x63,
+};
+
+/* RAP action codes */
+#define MHL_RAP_POLL		0x00	/* Just do an ack */
+#define MHL_RAP_CONTENT_ON	0x10	/* Turn content stream ON */
+#define MHL_RAP_CONTENT_OFF	0x11	/* Turn content stream OFF */
+#define MHL_RAP_CBUS_MODE_DOWN	0x20
+#define MHL_RAP_CBUS_MODE_UP	0x21
+
+/* RAPK status codes */
+#define MHL_RAPK_NO_ERR		0x00	/* RAP action recognized & supported */
+#define MHL_RAPK_UNRECOGNIZED	0x01	/* Unknown RAP action code received */
+#define MHL_RAPK_UNSUPPORTED	0x02	/* Rcvd RAP action code not supported */
+#define MHL_RAPK_BUSY		0x03	/* Responder too busy to respond */
+
+/*
+ * Error status codes for RCPE messages
+ */
+/* No error. (Not allowed in RCPE messages) */
+#define MHL_RCPE_STATUS_NO_ERROR		0x00
+/* Unsupported/unrecognized key code */
+#define MHL_RCPE_STATUS_INEFFECTIVE_KEY_CODE	0x01
+/* Responder busy. Initiator may retry message */
+#define MHL_RCPE_STATUS_BUSY			0x02
+
+/*
+ * Error status codes for RBPE messages
+ */
+/* No error. (Not allowed in RBPE messages) */
+#define MHL_RBPE_STATUS_NO_ERROR		0x00
+/* Unsupported/unrecognized button code */
+#define MHL_RBPE_STATUS_INEFFECTIVE_BUTTON_CODE	0x01
+/* Responder busy. Initiator may retry message */
+#define MHL_RBPE_STATUS_BUSY			0x02
+
+/*
+ * Error status codes for UCPE messages
+ */
+/* No error. (Not allowed in UCPE messages) */
+#define MHL_UCPE_STATUS_NO_ERROR		0x00
+/* Unsupported/unrecognized key code */
+#define MHL_UCPE_STATUS_INEFFECTIVE_KEY_CODE	0x01
+
+#endif /* __MHL_H__ */
diff --git a/include/drm/drmP.h b/include/drm/drmP.h
index 672644031bd5..a9cfd33c7b1a 100644
--- a/include/drm/drmP.h
+++ b/include/drm/drmP.h
@@ -57,7 +57,7 @@
 #include <linux/types.h>
 #include <linux/vmalloc.h>
 #include <linux/workqueue.h>
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 
 #include <asm/mman.h>
 #include <asm/pgalloc.h>
@@ -76,6 +76,7 @@
 #include <drm/drm_os_linux.h>
 #include <drm/drm_sarea.h>
 #include <drm/drm_vma_manager.h>
+#include <drm/drm_drv.h>
 
 struct module;
 
@@ -135,35 +136,12 @@ struct dma_buf_attachment;
 #define DRM_UT_PRIME		0x08
 #define DRM_UT_ATOMIC		0x10
 #define DRM_UT_VBL		0x20
-
-extern __printf(6, 7)
-void drm_dev_printk(const struct device *dev, const char *level,
-		    unsigned int category, const char *function_name,
-		    const char *prefix, const char *format, ...);
-
-extern __printf(3, 4)
-void drm_printk(const char *level, unsigned int category,
-		const char *format, ...);
+#define DRM_UT_STATE		0x40
 
 /***********************************************************************/
 /** \name DRM template customization defaults */
 /*@{*/
 
-/* driver capabilities and requirements mask */
-#define DRIVER_USE_AGP			0x1
-#define DRIVER_LEGACY			0x2
-#define DRIVER_PCI_DMA			0x8
-#define DRIVER_SG			0x10
-#define DRIVER_HAVE_DMA			0x20
-#define DRIVER_HAVE_IRQ			0x40
-#define DRIVER_IRQ_SHARED		0x80
-#define DRIVER_GEM			0x1000
-#define DRIVER_MODESET			0x2000
-#define DRIVER_PRIME			0x4000
-#define DRIVER_RENDER			0x8000
-#define DRIVER_ATOMIC			0x10000
-#define DRIVER_KMS_LEGACY_CONTEXT	0x20000
-
 /***********************************************************************/
 /** \name Macros to make printk easier */
 /*@{*/
@@ -306,6 +284,27 @@ void drm_printk(const char *level, unsigned int category,
 #define DRM_DEBUG_PRIME_RATELIMITED(fmt, args...)			\
 	DRM_DEV_DEBUG_PRIME_RATELIMITED(NULL, fmt, ##args)
 
+/* Format strings and argument splitters to simplify printing
+ * various "complex" objects
+ */
+#define DRM_MODE_FMT    "%d:\"%s\" %d %d %d %d %d %d %d %d %d %d 0x%x 0x%x"
+#define DRM_MODE_ARG(m) \
+	(m)->base.id, (m)->name, (m)->vrefresh, (m)->clock, \
+	(m)->hdisplay, (m)->hsync_start, (m)->hsync_end, (m)->htotal, \
+	(m)->vdisplay, (m)->vsync_start, (m)->vsync_end, (m)->vtotal, \
+	(m)->type, (m)->flags
+
+#define DRM_RECT_FMT    "%dx%d%+d%+d"
+#define DRM_RECT_ARG(r) drm_rect_width(r), drm_rect_height(r), (r)->x1, (r)->y1
+
+/* for rect's in fixed-point format: */
+#define DRM_RECT_FP_FMT "%d.%06ux%d.%06u%+d.%06u%+d.%06u"
+#define DRM_RECT_FP_ARG(r) \
+		drm_rect_width(r) >> 16, ((drm_rect_width(r) & 0xffff) * 15625) >> 10, \
+		drm_rect_height(r) >> 16, ((drm_rect_height(r) & 0xffff) * 15625) >> 10, \
+		(r)->x1 >> 16, (((r)->x1 & 0xffff) * 15625) >> 10, \
+		(r)->y1 >> 16, (((r)->y1 & 0xffff) * 15625) >> 10
+
 /*@}*/
 
 /***********************************************************************/
@@ -362,7 +361,7 @@ struct drm_ioctl_desc {
 struct drm_pending_event {
 	struct completion *completion;
 	struct drm_event *event;
-	struct fence *fence;
+	struct dma_fence *fence;
 	struct list_head link;
 	struct list_head pending_link;
 	struct drm_file *file_priv;
@@ -458,263 +457,6 @@ struct drm_lock_data {
 #define DRM_SCANOUTPOS_IN_VBLANK    (1 << 1)
 #define DRM_SCANOUTPOS_ACCURATE     (1 << 2)
 
-/**
- * DRM driver structure. This structure represent the common code for
- * a family of cards. There will one drm_device for each card present
- * in this family
- */
-struct drm_driver {
-	int (*load) (struct drm_device *, unsigned long flags);
-	int (*firstopen) (struct drm_device *);
-	int (*open) (struct drm_device *, struct drm_file *);
-	void (*preclose) (struct drm_device *, struct drm_file *file_priv);
-	void (*postclose) (struct drm_device *, struct drm_file *);
-	void (*lastclose) (struct drm_device *);
-	int (*unload) (struct drm_device *);
-	int (*dma_ioctl) (struct drm_device *dev, void *data, struct drm_file *file_priv);
-	int (*dma_quiescent) (struct drm_device *);
-	int (*context_dtor) (struct drm_device *dev, int context);
-	int (*set_busid)(struct drm_device *dev, struct drm_master *master);
-
-	/**
-	 * get_vblank_counter - get raw hardware vblank counter
-	 * @dev: DRM device
-	 * @pipe: counter to fetch
-	 *
-	 * Driver callback for fetching a raw hardware vblank counter for @crtc.
-	 * If a device doesn't have a hardware counter, the driver can simply
-	 * use drm_vblank_no_hw_counter() function. The DRM core will account for
-	 * missed vblank events while interrupts where disabled based on system
-	 * timestamps.
-	 *
-	 * Wraparound handling and loss of events due to modesetting is dealt
-	 * with in the DRM core code.
-	 *
-	 * RETURNS
-	 * Raw vblank counter value.
-	 */
-	u32 (*get_vblank_counter) (struct drm_device *dev, unsigned int pipe);
-
-	/**
-	 * enable_vblank - enable vblank interrupt events
-	 * @dev: DRM device
-	 * @pipe: which irq to enable
-	 *
-	 * Enable vblank interrupts for @crtc.  If the device doesn't have
-	 * a hardware vblank counter, the driver should use the
-	 * drm_vblank_no_hw_counter() function that keeps a virtual counter.
-	 *
-	 * RETURNS
-	 * Zero on success, appropriate errno if the given @crtc's vblank
-	 * interrupt cannot be enabled.
-	 */
-	int (*enable_vblank) (struct drm_device *dev, unsigned int pipe);
-
-	/**
-	 * disable_vblank - disable vblank interrupt events
-	 * @dev: DRM device
-	 * @pipe: which irq to enable
-	 *
-	 * Disable vblank interrupts for @crtc.  If the device doesn't have
-	 * a hardware vblank counter, the driver should use the
-	 * drm_vblank_no_hw_counter() function that keeps a virtual counter.
-	 */
-	void (*disable_vblank) (struct drm_device *dev, unsigned int pipe);
-
-	/**
-	 * Called by \c drm_device_is_agp.  Typically used to determine if a
-	 * card is really attached to AGP or not.
-	 *
-	 * \param dev  DRM device handle
-	 *
-	 * \returns
-	 * One of three values is returned depending on whether or not the
-	 * card is absolutely \b not AGP (return of 0), absolutely \b is AGP
-	 * (return of 1), or may or may not be AGP (return of 2).
-	 */
-	int (*device_is_agp) (struct drm_device *dev);
-
-	/**
-	 * Called by vblank timestamping code.
-	 *
-	 * Return the current display scanout position from a crtc, and an
-	 * optional accurate ktime_get timestamp of when position was measured.
-	 *
-	 * \param dev  DRM device.
-	 * \param pipe Id of the crtc to query.
-	 * \param flags Flags from the caller (DRM_CALLED_FROM_VBLIRQ or 0).
-	 * \param *vpos Target location for current vertical scanout position.
-	 * \param *hpos Target location for current horizontal scanout position.
-	 * \param *stime Target location for timestamp taken immediately before
-	 *               scanout position query. Can be NULL to skip timestamp.
-	 * \param *etime Target location for timestamp taken immediately after
-	 *               scanout position query. Can be NULL to skip timestamp.
-	 * \param mode Current display timings.
-	 *
-	 * Returns vpos as a positive number while in active scanout area.
-	 * Returns vpos as a negative number inside vblank, counting the number
-	 * of scanlines to go until end of vblank, e.g., -1 means "one scanline
-	 * until start of active scanout / end of vblank."
-	 *
-	 * \return Flags, or'ed together as follows:
-	 *
-	 * DRM_SCANOUTPOS_VALID = Query successful.
-	 * DRM_SCANOUTPOS_INVBL = Inside vblank.
-	 * DRM_SCANOUTPOS_ACCURATE = Returned position is accurate. A lack of
-	 * this flag means that returned position may be offset by a constant
-	 * but unknown small number of scanlines wrt. real scanout position.
-	 *
-	 */
-	int (*get_scanout_position) (struct drm_device *dev, unsigned int pipe,
-				     unsigned int flags, int *vpos, int *hpos,
-				     ktime_t *stime, ktime_t *etime,
-				     const struct drm_display_mode *mode);
-
-	/**
-	 * Called by \c drm_get_last_vbltimestamp. Should return a precise
-	 * timestamp when the most recent VBLANK interval ended or will end.
-	 *
-	 * Specifically, the timestamp in @vblank_time should correspond as
-	 * closely as possible to the time when the first video scanline of
-	 * the video frame after the end of VBLANK will start scanning out,
-	 * the time immediately after end of the VBLANK interval. If the
-	 * @crtc is currently inside VBLANK, this will be a time in the future.
-	 * If the @crtc is currently scanning out a frame, this will be the
-	 * past start time of the current scanout. This is meant to adhere
-	 * to the OpenML OML_sync_control extension specification.
-	 *
-	 * \param dev dev DRM device handle.
-	 * \param pipe crtc for which timestamp should be returned.
-	 * \param *max_error Maximum allowable timestamp error in nanoseconds.
-	 *                   Implementation should strive to provide timestamp
-	 *                   with an error of at most *max_error nanoseconds.
-	 *                   Returns true upper bound on error for timestamp.
-	 * \param *vblank_time Target location for returned vblank timestamp.
-	 * \param flags 0 = Defaults, no special treatment needed.
-	 * \param       DRM_CALLED_FROM_VBLIRQ = Function is called from vblank
-	 *	        irq handler. Some drivers need to apply some workarounds
-	 *              for gpu-specific vblank irq quirks if flag is set.
-	 *
-	 * \returns
-	 * Zero if timestamping isn't supported in current display mode or a
-	 * negative number on failure. A positive status code on success,
-	 * which describes how the vblank_time timestamp was computed.
-	 */
-	int (*get_vblank_timestamp) (struct drm_device *dev, unsigned int pipe,
-				     int *max_error,
-				     struct timeval *vblank_time,
-				     unsigned flags);
-
-	/* these have to be filled in */
-
-	irqreturn_t(*irq_handler) (int irq, void *arg);
-	void (*irq_preinstall) (struct drm_device *dev);
-	int (*irq_postinstall) (struct drm_device *dev);
-	void (*irq_uninstall) (struct drm_device *dev);
-
-	/* Master routines */
-	int (*master_create)(struct drm_device *dev, struct drm_master *master);
-	void (*master_destroy)(struct drm_device *dev, struct drm_master *master);
-	/**
-	 * master_set is called whenever the minor master is set.
-	 * master_drop is called whenever the minor master is dropped.
-	 */
-
-	int (*master_set)(struct drm_device *dev, struct drm_file *file_priv,
-			  bool from_open);
-	void (*master_drop)(struct drm_device *dev, struct drm_file *file_priv);
-
-	int (*debugfs_init)(struct drm_minor *minor);
-	void (*debugfs_cleanup)(struct drm_minor *minor);
-
-	/**
-	 * @gem_free_object: deconstructor for drm_gem_objects
-	 *
-	 * This is deprecated and should not be used by new drivers. Use
-	 * @gem_free_object_unlocked instead.
-	 */
-	void (*gem_free_object) (struct drm_gem_object *obj);
-
-	/**
-	 * @gem_free_object_unlocked: deconstructor for drm_gem_objects
-	 *
-	 * This is for drivers which are not encumbered with dev->struct_mutex
-	 * legacy locking schemes. Use this hook instead of @gem_free_object.
-	 */
-	void (*gem_free_object_unlocked) (struct drm_gem_object *obj);
-
-	int (*gem_open_object) (struct drm_gem_object *, struct drm_file *);
-	void (*gem_close_object) (struct drm_gem_object *, struct drm_file *);
-
-	/**
-	 * Hook for allocating the GEM object struct, for use by core
-	 * helpers.
-	 */
-	struct drm_gem_object *(*gem_create_object)(struct drm_device *dev,
-						    size_t size);
-
-	/* prime: */
-	/* export handle -> fd (see drm_gem_prime_handle_to_fd() helper) */
-	int (*prime_handle_to_fd)(struct drm_device *dev, struct drm_file *file_priv,
-				uint32_t handle, uint32_t flags, int *prime_fd);
-	/* import fd -> handle (see drm_gem_prime_fd_to_handle() helper) */
-	int (*prime_fd_to_handle)(struct drm_device *dev, struct drm_file *file_priv,
-				int prime_fd, uint32_t *handle);
-	/* export GEM -> dmabuf */
-	struct dma_buf * (*gem_prime_export)(struct drm_device *dev,
-				struct drm_gem_object *obj, int flags);
-	/* import dmabuf -> GEM */
-	struct drm_gem_object * (*gem_prime_import)(struct drm_device *dev,
-				struct dma_buf *dma_buf);
-	/* low-level interface used by drm_gem_prime_{import,export} */
-	int (*gem_prime_pin)(struct drm_gem_object *obj);
-	void (*gem_prime_unpin)(struct drm_gem_object *obj);
-	struct reservation_object * (*gem_prime_res_obj)(
-				struct drm_gem_object *obj);
-	struct sg_table *(*gem_prime_get_sg_table)(struct drm_gem_object *obj);
-	struct drm_gem_object *(*gem_prime_import_sg_table)(
-				struct drm_device *dev,
-				struct dma_buf_attachment *attach,
-				struct sg_table *sgt);
-	void *(*gem_prime_vmap)(struct drm_gem_object *obj);
-	void (*gem_prime_vunmap)(struct drm_gem_object *obj, void *vaddr);
-	int (*gem_prime_mmap)(struct drm_gem_object *obj,
-				struct vm_area_struct *vma);
-
-	/* vga arb irq handler */
-	void (*vgaarb_irq)(struct drm_device *dev, bool state);
-
-	/* dumb alloc support */
-	int (*dumb_create)(struct drm_file *file_priv,
-			   struct drm_device *dev,
-			   struct drm_mode_create_dumb *args);
-	int (*dumb_map_offset)(struct drm_file *file_priv,
-			       struct drm_device *dev, uint32_t handle,
-			       uint64_t *offset);
-	int (*dumb_destroy)(struct drm_file *file_priv,
-			    struct drm_device *dev,
-			    uint32_t handle);
-
-	/* Driver private ops for this object */
-	const struct vm_operations_struct *gem_vm_ops;
-
-	int major;
-	int minor;
-	int patchlevel;
-	char *name;
-	char *desc;
-	char *date;
-
-	u32 driver_features;
-	int dev_priv_size;
-	const struct drm_ioctl_desc *ioctls;
-	int num_ioctls;
-	const struct file_operations *fops;
-
-	/* List of devices hanging off this driver with stealth attach. */
-	struct list_head legacy_dev_list;
-};
-
 enum drm_minor_type {
 	DRM_MINOR_PRIMARY,
 	DRM_MINOR_CONTROL,
@@ -941,8 +683,13 @@ static inline bool drm_is_primary_client(const struct drm_file *file_priv)
 extern int drm_ioctl_permit(u32 flags, struct drm_file *file_priv);
 extern long drm_ioctl(struct file *filp,
 		      unsigned int cmd, unsigned long arg);
+#ifdef CONFIG_COMPAT
 extern long drm_compat_ioctl(struct file *filp,
 			     unsigned int cmd, unsigned long arg);
+#else
+/* Let drm_compat_ioctl be assigned to .compat_ioctl unconditionally */
+#define drm_compat_ioctl NULL
+#endif
 extern bool drm_ioctl_flags(unsigned int nr, unsigned int *flags);
 
 /* File Operations (drm_fops.c) */
@@ -980,15 +727,6 @@ void drm_clflush_virt_range(void *addr, unsigned long length);
  * DMA quiscent + idle. DMA quiescent usually requires the hardware lock.
  */
 
-/* Modesetting support */
-extern void drm_vblank_pre_modeset(struct drm_device *dev, unsigned int pipe);
-extern void drm_vblank_post_modeset(struct drm_device *dev, unsigned int pipe);
-
-/* drm_drv.c */
-void drm_put_dev(struct drm_device *dev);
-void drm_unplug_dev(struct drm_device *dev);
-extern unsigned int drm_debug;
-
 				/* Debugfs support */
 #if defined(CONFIG_DEBUG_FS)
 extern int drm_debugfs_create_files(const struct drm_info_list *files,
@@ -1041,19 +779,6 @@ extern void drm_pci_free(struct drm_device *dev, struct drm_dma_handle * dmah);
 extern void drm_sysfs_hotplug_event(struct drm_device *dev);
 
 
-struct drm_device *drm_dev_alloc(struct drm_driver *driver,
-				 struct device *parent);
-int drm_dev_init(struct drm_device *dev,
-		 struct drm_driver *driver,
-		 struct device *parent);
-void drm_dev_ref(struct drm_device *dev);
-void drm_dev_unref(struct drm_device *dev);
-int drm_dev_register(struct drm_device *dev, unsigned long flags);
-void drm_dev_unregister(struct drm_device *dev);
-
-struct drm_minor *drm_minor_acquire(unsigned int minor_id);
-void drm_minor_release(struct drm_minor *minor);
-
 /*@}*/
 
 /* PCI section */
diff --git a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h
index 9701f2dfb784..d6d241f63b9f 100644
--- a/include/drm/drm_atomic.h
+++ b/include/drm/drm_atomic.h
@@ -144,6 +144,7 @@ struct __drm_crtcs_state {
 	struct drm_crtc *ptr;
 	struct drm_crtc_state *state;
 	struct drm_crtc_commit *commit;
+	s64 __user *out_fence_ptr;
 };
 
 struct __drm_connnectors_state {
@@ -153,6 +154,7 @@ struct __drm_connnectors_state {
 
 /**
  * struct drm_atomic_state - the global state object for atomic updates
+ * @ref: count of all references to this state (will not be freed until zero)
  * @dev: parent DRM device
  * @allow_modeset: allow full modeset
  * @legacy_cursor_update: hint to enforce legacy cursor IOCTL semantics
@@ -164,6 +166,8 @@ struct __drm_connnectors_state {
  * @acquire_ctx: acquire context for this atomic modeset state update
  */
 struct drm_atomic_state {
+	struct kref ref;
+
 	struct drm_device *dev;
 	bool allow_modeset : 1;
 	bool legacy_cursor_update : 1;
@@ -193,7 +197,33 @@ static inline void drm_crtc_commit_get(struct drm_crtc_commit *commit)
 struct drm_atomic_state * __must_check
 drm_atomic_state_alloc(struct drm_device *dev);
 void drm_atomic_state_clear(struct drm_atomic_state *state);
-void drm_atomic_state_free(struct drm_atomic_state *state);
+
+/**
+ * drm_atomic_state_get - acquire a reference to the atomic state
+ * @state: The atomic state
+ *
+ * Returns a new reference to the @state
+ */
+static inline struct drm_atomic_state *
+drm_atomic_state_get(struct drm_atomic_state *state)
+{
+	kref_get(&state->ref);
+	return state;
+}
+
+void __drm_atomic_state_free(struct kref *ref);
+
+/**
+ * drm_atomic_state_put - release a reference to the atomic state
+ * @state: The atomic state
+ *
+ * This releases a reference to @state which is freed after removing the
+ * final reference. No locking required and callable from any context.
+ */
+static inline void drm_atomic_state_put(struct drm_atomic_state *state)
+{
+	kref_put(&state->ref, __drm_atomic_state_free);
+}
 
 int  __must_check
 drm_atomic_state_init(struct drm_device *dev, struct drm_atomic_state *state);
@@ -316,6 +346,8 @@ drm_atomic_set_crtc_for_plane(struct drm_plane_state *plane_state,
 			      struct drm_crtc *crtc);
 void drm_atomic_set_fb_for_plane(struct drm_plane_state *plane_state,
 				 struct drm_framebuffer *fb);
+void drm_atomic_set_fence_for_plane(struct drm_plane_state *plane_state,
+				    struct dma_fence *fence);
 int __must_check
 drm_atomic_set_crtc_for_connector(struct drm_connector_state *conn_state,
 				  struct drm_crtc *crtc);
@@ -335,6 +367,14 @@ int __must_check drm_atomic_check_only(struct drm_atomic_state *state);
 int __must_check drm_atomic_commit(struct drm_atomic_state *state);
 int __must_check drm_atomic_nonblocking_commit(struct drm_atomic_state *state);
 
+void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
+
+#ifdef CONFIG_DEBUG_FS
+struct drm_minor;
+int drm_atomic_debugfs_init(struct drm_minor *minor);
+int drm_atomic_debugfs_cleanup(struct drm_minor *minor);
+#endif
+
 #define for_each_connector_in_state(__state, connector, connector_state, __i) \
 	for ((__i) = 0;							\
 	     (__i) < (__state)->num_connector &&				\
@@ -365,11 +405,20 @@ int __must_check drm_atomic_nonblocking_commit(struct drm_atomic_state *state);
  *
  * To give drivers flexibility struct &drm_crtc_state has 3 booleans to track
  * whether the state CRTC changed enough to need a full modeset cycle:
- * connectors_changed, mode_changed and active_change. This helper simply
+ * connectors_changed, mode_changed and active_changed. This helper simply
  * combines these three to compute the overall need for a modeset for @state.
+ *
+ * The atomic helper code sets these booleans, but drivers can and should
+ * change them appropriately to accurately represent whether a modeset is
+ * really needed. In general, drivers should avoid full modesets whenever
+ * possible.
+ *
+ * For example if the CRTC mode has changed, and the hardware is able to enact
+ * the requested mode change without going through a full modeset, the driver
+ * should clear mode_changed during its ->atomic_check.
  */
 static inline bool
-drm_atomic_crtc_needs_modeset(struct drm_crtc_state *state)
+drm_atomic_crtc_needs_modeset(const struct drm_crtc_state *state)
 {
 	return state->mode_changed || state->active_changed ||
 	       state->connectors_changed;
diff --git a/include/drm/drm_blend.h b/include/drm/drm_blend.h
index 36baa175de99..13221cf9b3eb 100644
--- a/include/drm/drm_blend.h
+++ b/include/drm/drm_blend.h
@@ -47,8 +47,14 @@ struct drm_atomic_state;
 #define DRM_REFLECT_Y	BIT(5)
 #define DRM_REFLECT_MASK (DRM_REFLECT_X | DRM_REFLECT_Y)
 
-struct drm_property *drm_mode_create_rotation_property(struct drm_device *dev,
-						       unsigned int supported_rotations);
+static inline bool drm_rotation_90_or_270(unsigned int rotation)
+{
+	return rotation & (DRM_ROTATE_90 | DRM_ROTATE_270);
+}
+
+int drm_plane_create_rotation_property(struct drm_plane *plane,
+				       unsigned int rotation,
+				       unsigned int supported_rotations);
 unsigned int drm_rotation_simplify(unsigned int rotation,
 				   unsigned int supported_rotations);
 
diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h
index ac9d7d8e0e43..a9b95246e26e 100644
--- a/include/drm/drm_connector.h
+++ b/include/drm/drm_connector.h
@@ -37,6 +37,7 @@ struct drm_crtc;
 struct drm_encoder;
 struct drm_property;
 struct drm_property_blob;
+struct drm_printer;
 struct edid;
 
 enum drm_connector_force {
@@ -194,10 +195,40 @@ int drm_display_info_set_bus_formats(struct drm_display_info *info,
 				     unsigned int num_formats);
 
 /**
+ * struct drm_tv_connector_state - TV connector related states
+ * @subconnector: selected subconnector
+ * @margins: left/right/top/bottom margins
+ * @mode: TV mode
+ * @brightness: brightness in percent
+ * @contrast: contrast in percent
+ * @flicker_reduction: flicker reduction in percent
+ * @overscan: overscan in percent
+ * @saturation: saturation in percent
+ * @hue: hue in percent
+ */
+struct drm_tv_connector_state {
+	enum drm_mode_subconnector subconnector;
+	struct {
+		unsigned int left;
+		unsigned int right;
+		unsigned int top;
+		unsigned int bottom;
+	} margins;
+	unsigned int mode;
+	unsigned int brightness;
+	unsigned int contrast;
+	unsigned int flicker_reduction;
+	unsigned int overscan;
+	unsigned int saturation;
+	unsigned int hue;
+};
+
+/**
  * struct drm_connector_state - mutable connector state
  * @connector: backpointer to the connector
  * @best_encoder: can be used by helpers and drivers to select the encoder
  * @state: backpointer to global drm_atomic_state
+ * @tv: TV connector state
  */
 struct drm_connector_state {
 	struct drm_connector *connector;
@@ -213,6 +244,8 @@ struct drm_connector_state {
 	struct drm_encoder *best_encoder;
 
 	struct drm_atomic_state *state;
+
+	struct drm_tv_connector_state tv;
 };
 
 /**
@@ -261,6 +294,9 @@ struct drm_connector_funcs {
 	 * connector due to a user request. force can be used by the driver to
 	 * avoid expensive, destructive operations during automated probing.
 	 *
+	 * This callback is optional, if not implemented the connector will be
+	 * considered as always being attached.
+	 *
 	 * FIXME:
 	 *
 	 * Note that this hook is only called by the probe helper. It's not in
@@ -481,6 +517,18 @@ struct drm_connector_funcs {
 				   const struct drm_connector_state *state,
 				   struct drm_property *property,
 				   uint64_t *val);
+
+	/**
+	 * @atomic_print_state:
+	 *
+	 * If driver subclasses struct &drm_connector_state, it should implement
+	 * this optional hook for printing additional driver specific state.
+	 *
+	 * Do not call this directly, use drm_atomic_connector_print_state()
+	 * instead.
+	 */
+	void (*atomic_print_state)(struct drm_printer *p,
+				   const struct drm_connector_state *state);
 };
 
 /* mode specified on the command line */
@@ -762,6 +810,30 @@ int drm_mode_connector_update_edid_property(struct drm_connector *connector,
 					    const struct edid *edid);
 
 /**
+ * struct drm_tile_group - Tile group metadata
+ * @refcount: reference count
+ * @dev: DRM device
+ * @id: tile group id exposed to userspace
+ * @group_data: Sink-private data identifying this group
+ *
+ * @group_data corresponds to displayid vend/prod/serial for external screens
+ * with an EDID.
+ */
+struct drm_tile_group {
+	struct kref refcount;
+	struct drm_device *dev;
+	int id;
+	u8 group_data[8];
+};
+
+struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev,
+						  char topology[8]);
+struct drm_tile_group *drm_mode_get_tile_group(struct drm_device *dev,
+					       char topology[8]);
+void drm_mode_put_tile_group(struct drm_device *dev,
+			     struct drm_tile_group *tg);
+
+/**
  * drm_for_each_connector - iterate over all connectors
  * @connector: the loop cursor
  * @dev: the DRM device
diff --git a/include/drm/drm_crtc.h b/include/drm/drm_crtc.h
index 0aa292526567..946672f97e1e 100644
--- a/include/drm/drm_crtc.h
+++ b/include/drm/drm_crtc.h
@@ -28,7 +28,6 @@
 #include <linux/i2c.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
-#include <linux/idr.h>
 #include <linux/fb.h>
 #include <linux/hdmi.h>
 #include <linux/media-bus-format.h>
@@ -47,13 +46,16 @@
 #include <drm/drm_plane.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_color_mgmt.h>
+#include <drm/drm_debugfs_crc.h>
+#include <drm/drm_mode_config.h>
 
 struct drm_device;
 struct drm_mode_set;
 struct drm_file;
 struct drm_clip_rect;
+struct drm_printer;
 struct device_node;
-struct fence;
+struct dma_fence;
 struct edid;
 
 static inline int64_t U642I64(uint64_t val)
@@ -65,14 +67,6 @@ static inline uint64_t I642U64(int64_t val)
 	return (uint64_t)*((uint64_t *)&val);
 }
 
-/* data corresponds to displayid vend/prod/serial */
-struct drm_tile_group {
-	struct kref refcount;
-	struct drm_device *dev;
-	int id;
-	u8 group_data[8];
-};
-
 struct drm_crtc;
 struct drm_encoder;
 struct drm_pending_vblank_event;
@@ -116,6 +110,11 @@ struct drm_plane_helper_funcs;
  * never return in a failure from the ->atomic_check callback. Userspace assumes
  * that a DPMS On will always succeed. In other words: @enable controls resource
  * assignment, @active controls the actual hardware state.
+ *
+ * The three booleans active_changed, connectors_changed and mode_changed are
+ * intended to indicate whether a full modeset is needed, rather than strictly
+ * describing what has changed in a commit.
+ * See also: drm_atomic_crtc_needs_modeset()
  */
 struct drm_crtc_state {
 	struct drm_crtc *crtc;
@@ -564,6 +563,42 @@ struct drm_crtc_funcs {
 	 * before data structures are torndown.
 	 */
 	void (*early_unregister)(struct drm_crtc *crtc);
+
+	/**
+	 * @set_crc_source:
+	 *
+	 * Changes the source of CRC checksums of frames at the request of
+	 * userspace, typically for testing purposes. The sources available are
+	 * specific of each driver and a %NULL value indicates that CRC
+	 * generation is to be switched off.
+	 *
+	 * When CRC generation is enabled, the driver should call
+	 * drm_crtc_add_crc_entry() at each frame, providing any information
+	 * that characterizes the frame contents in the crcN arguments, as
+	 * provided from the configured source. Drivers must accept a "auto"
+	 * source name that will select a default source for this CRTC.
+	 *
+	 * This callback is optional if the driver does not support any CRC
+	 * generation functionality.
+	 *
+	 * RETURNS:
+	 *
+	 * 0 on success or a negative error code on failure.
+	 */
+	int (*set_crc_source)(struct drm_crtc *crtc, const char *source,
+			      size_t *values_cnt);
+
+	/**
+	 * @atomic_print_state:
+	 *
+	 * If driver subclasses struct &drm_crtc_state, it should implement
+	 * this optional hook for printing additional driver specific state.
+	 *
+	 * Do not call this directly, use drm_atomic_crtc_print_state()
+	 * instead.
+	 */
+	void (*atomic_print_state)(struct drm_printer *p,
+				   const struct drm_crtc_state *state);
 };
 
 /**
@@ -680,660 +715,90 @@ struct drm_crtc {
 	 * context.
 	 */
 	struct drm_modeset_acquire_ctx *acquire_ctx;
-};
 
-/**
- * struct drm_mode_set - new values for a CRTC config change
- * @fb: framebuffer to use for new config
- * @crtc: CRTC whose configuration we're about to change
- * @mode: mode timings to use
- * @x: position of this CRTC relative to @fb
- * @y: position of this CRTC relative to @fb
- * @connectors: array of connectors to drive with this CRTC if possible
- * @num_connectors: size of @connectors array
- *
- * Represents a single crtc the connectors that it drives with what mode
- * and from which framebuffer it scans out from.
- *
- * This is used to set modes.
- */
-struct drm_mode_set {
-	struct drm_framebuffer *fb;
-	struct drm_crtc *crtc;
-	struct drm_display_mode *mode;
-
-	uint32_t x;
-	uint32_t y;
-
-	struct drm_connector **connectors;
-	size_t num_connectors;
-};
-
-/**
- * struct drm_mode_config_funcs - basic driver provided mode setting functions
- *
- * Some global (i.e. not per-CRTC, connector, etc) mode setting functions that
- * involve drivers.
- */
-struct drm_mode_config_funcs {
+#ifdef CONFIG_DEBUG_FS
 	/**
-	 * @fb_create:
-	 *
-	 * Create a new framebuffer object. The core does basic checks on the
-	 * requested metadata, but most of that is left to the driver. See
-	 * struct &drm_mode_fb_cmd2 for details.
-	 *
-	 * If the parameters are deemed valid and the backing storage objects in
-	 * the underlying memory manager all exist, then the driver allocates
-	 * a new &drm_framebuffer structure, subclassed to contain
-	 * driver-specific information (like the internal native buffer object
-	 * references). It also needs to fill out all relevant metadata, which
-	 * should be done by calling drm_helper_mode_fill_fb_struct().
+	 * @debugfs_entry:
 	 *
-	 * The initialization is finalized by calling drm_framebuffer_init(),
-	 * which registers the framebuffer and makes it accessible to other
-	 * threads.
-	 *
-	 * RETURNS:
-	 *
-	 * A new framebuffer with an initial reference count of 1 or a negative
-	 * error code encoded with ERR_PTR().
+	 * Debugfs directory for this CRTC.
 	 */
-	struct drm_framebuffer *(*fb_create)(struct drm_device *dev,
-					     struct drm_file *file_priv,
-					     const struct drm_mode_fb_cmd2 *mode_cmd);
+	struct dentry *debugfs_entry;
 
 	/**
-	 * @output_poll_changed:
-	 *
-	 * Callback used by helpers to inform the driver of output configuration
-	 * changes.
+	 * @crc:
 	 *
-	 * Drivers implementing fbdev emulation with the helpers can call
-	 * drm_fb_helper_hotplug_changed from this hook to inform the fbdev
-	 * helper of output changes.
-	 *
-	 * FIXME:
-	 *
-	 * Except that there's no vtable for device-level helper callbacks
-	 * there's no reason this is a core function.
+	 * Configuration settings of CRC capture.
 	 */
-	void (*output_poll_changed)(struct drm_device *dev);
+	struct drm_crtc_crc crc;
+#endif
 
 	/**
-	 * @atomic_check:
-	 *
-	 * This is the only hook to validate an atomic modeset update. This
-	 * function must reject any modeset and state changes which the hardware
-	 * or driver doesn't support. This includes but is of course not limited
-	 * to:
-	 *
-	 *  - Checking that the modes, framebuffers, scaling and placement
-	 *    requirements and so on are within the limits of the hardware.
-	 *
-	 *  - Checking that any hidden shared resources are not oversubscribed.
-	 *    This can be shared PLLs, shared lanes, overall memory bandwidth,
-	 *    display fifo space (where shared between planes or maybe even
-	 *    CRTCs).
-	 *
-	 *  - Checking that virtualized resources exported to userspace are not
-	 *    oversubscribed. For various reasons it can make sense to expose
-	 *    more planes, crtcs or encoders than which are physically there. One
-	 *    example is dual-pipe operations (which generally should be hidden
-	 *    from userspace if when lockstepped in hardware, exposed otherwise),
-	 *    where a plane might need 1 hardware plane (if it's just on one
-	 *    pipe), 2 hardware planes (when it spans both pipes) or maybe even
-	 *    shared a hardware plane with a 2nd plane (if there's a compatible
-	 *    plane requested on the area handled by the other pipe).
-	 *
-	 *  - Check that any transitional state is possible and that if
-	 *    requested, the update can indeed be done in the vblank period
-	 *    without temporarily disabling some functions.
-	 *
-	 *  - Check any other constraints the driver or hardware might have.
-	 *
-	 *  - This callback also needs to correctly fill out the &drm_crtc_state
-	 *    in this update to make sure that drm_atomic_crtc_needs_modeset()
-	 *    reflects the nature of the possible update and returns true if and
-	 *    only if the update cannot be applied without tearing within one
-	 *    vblank on that CRTC. The core uses that information to reject
-	 *    updates which require a full modeset (i.e. blanking the screen, or
-	 *    at least pausing updates for a substantial amount of time) if
-	 *    userspace has disallowed that in its request.
-	 *
-	 *  - The driver also does not need to repeat basic input validation
-	 *    like done for the corresponding legacy entry points. The core does
-	 *    that before calling this hook.
-	 *
-	 * See the documentation of @atomic_commit for an exhaustive list of
-	 * error conditions which don't have to be checked at the
-	 * ->atomic_check() stage?
-	 *
-	 * See the documentation for struct &drm_atomic_state for how exactly
-	 * an atomic modeset update is described.
-	 *
-	 * Drivers using the atomic helpers can implement this hook using
-	 * drm_atomic_helper_check(), or one of the exported sub-functions of
-	 * it.
+	 * @fence_context:
 	 *
-	 * RETURNS:
-	 *
-	 * 0 on success or one of the below negative error codes:
-	 *
-	 *  - -EINVAL, if any of the above constraints are violated.
-	 *
-	 *  - -EDEADLK, when returned from an attempt to acquire an additional
-	 *    &drm_modeset_lock through drm_modeset_lock().
-	 *
-	 *  - -ENOMEM, if allocating additional state sub-structures failed due
-	 *    to lack of memory.
-	 *
-	 *  - -EINTR, -EAGAIN or -ERESTARTSYS, if the IOCTL should be restarted.
-	 *    This can either be due to a pending signal, or because the driver
-	 *    needs to completely bail out to recover from an exceptional
-	 *    situation like a GPU hang. From a userspace point all errors are
-	 *    treated equally.
+	 * timeline context used for fence operations.
 	 */
-	int (*atomic_check)(struct drm_device *dev,
-			    struct drm_atomic_state *state);
+	unsigned int fence_context;
 
 	/**
-	 * @atomic_commit:
-	 *
-	 * This is the only hook to commit an atomic modeset update. The core
-	 * guarantees that @atomic_check has been called successfully before
-	 * calling this function, and that nothing has been changed in the
-	 * interim.
-	 *
-	 * See the documentation for struct &drm_atomic_state for how exactly
-	 * an atomic modeset update is described.
-	 *
-	 * Drivers using the atomic helpers can implement this hook using
-	 * drm_atomic_helper_commit(), or one of the exported sub-functions of
-	 * it.
-	 *
-	 * Nonblocking commits (as indicated with the nonblock parameter) must
-	 * do any preparatory work which might result in an unsuccessful commit
-	 * in the context of this callback. The only exceptions are hardware
-	 * errors resulting in -EIO. But even in that case the driver must
-	 * ensure that the display pipe is at least running, to avoid
-	 * compositors crashing when pageflips don't work. Anything else,
-	 * specifically committing the update to the hardware, should be done
-	 * without blocking the caller. For updates which do not require a
-	 * modeset this must be guaranteed.
-	 *
-	 * The driver must wait for any pending rendering to the new
-	 * framebuffers to complete before executing the flip. It should also
-	 * wait for any pending rendering from other drivers if the underlying
-	 * buffer is a shared dma-buf. Nonblocking commits must not wait for
-	 * rendering in the context of this callback.
-	 *
-	 * An application can request to be notified when the atomic commit has
-	 * completed. These events are per-CRTC and can be distinguished by the
-	 * CRTC index supplied in &drm_event to userspace.
-	 *
-	 * The drm core will supply a struct &drm_event in the event
-	 * member of each CRTC's &drm_crtc_state structure. See the
-	 * documentation for &drm_crtc_state for more details about the precise
-	 * semantics of this event.
-	 *
-	 * NOTE:
-	 *
-	 * Drivers are not allowed to shut down any display pipe successfully
-	 * enabled through an atomic commit on their own. Doing so can result in
-	 * compositors crashing if a page flip is suddenly rejected because the
-	 * pipe is off.
-	 *
-	 * RETURNS:
-	 *
-	 * 0 on success or one of the below negative error codes:
-	 *
-	 *  - -EBUSY, if a nonblocking updated is requested and there is
-	 *    an earlier updated pending. Drivers are allowed to support a queue
-	 *    of outstanding updates, but currently no driver supports that.
-	 *    Note that drivers must wait for preceding updates to complete if a
-	 *    synchronous update is requested, they are not allowed to fail the
-	 *    commit in that case.
-	 *
-	 *  - -ENOMEM, if the driver failed to allocate memory. Specifically
-	 *    this can happen when trying to pin framebuffers, which must only
-	 *    be done when committing the state.
-	 *
-	 *  - -ENOSPC, as a refinement of the more generic -ENOMEM to indicate
-	 *    that the driver has run out of vram, iommu space or similar GPU
-	 *    address space needed for framebuffer.
-	 *
-	 *  - -EIO, if the hardware completely died.
+	 * @fence_lock:
 	 *
-	 *  - -EINTR, -EAGAIN or -ERESTARTSYS, if the IOCTL should be restarted.
-	 *    This can either be due to a pending signal, or because the driver
-	 *    needs to completely bail out to recover from an exceptional
-	 *    situation like a GPU hang. From a userspace point of view all errors are
-	 *    treated equally.
-	 *
-	 * This list is exhaustive. Specifically this hook is not allowed to
-	 * return -EINVAL (any invalid requests should be caught in
-	 * @atomic_check) or -EDEADLK (this function must not acquire
-	 * additional modeset locks).
-	 */
-	int (*atomic_commit)(struct drm_device *dev,
-			     struct drm_atomic_state *state,
-			     bool nonblock);
-
-	/**
-	 * @atomic_state_alloc:
-	 *
-	 * This optional hook can be used by drivers that want to subclass struct
-	 * &drm_atomic_state to be able to track their own driver-private global
-	 * state easily. If this hook is implemented, drivers must also
-	 * implement @atomic_state_clear and @atomic_state_free.
-	 *
-	 * RETURNS:
-	 *
-	 * A new &drm_atomic_state on success or NULL on failure.
+	 * spinlock to protect the fences in the fence_context.
 	 */
-	struct drm_atomic_state *(*atomic_state_alloc)(struct drm_device *dev);
 
+	spinlock_t fence_lock;
 	/**
-	 * @atomic_state_clear:
-	 *
-	 * This hook must clear any driver private state duplicated into the
-	 * passed-in &drm_atomic_state. This hook is called when the caller
-	 * encountered a &drm_modeset_lock deadlock and needs to drop all
-	 * already acquired locks as part of the deadlock avoidance dance
-	 * implemented in drm_modeset_lock_backoff().
+	 * @fence_seqno:
 	 *
-	 * Any duplicated state must be invalidated since a concurrent atomic
-	 * update might change it, and the drm atomic interfaces always apply
-	 * updates as relative changes to the current state.
-	 *
-	 * Drivers that implement this must call drm_atomic_state_default_clear()
-	 * to clear common state.
+	 * Seqno variable used as monotonic counter for the fences
+	 * created on the CRTC's timeline.
 	 */
-	void (*atomic_state_clear)(struct drm_atomic_state *state);
+	unsigned long fence_seqno;
 
 	/**
-	 * @atomic_state_free:
-	 *
-	 * This hook needs driver private resources and the &drm_atomic_state
-	 * itself. Note that the core first calls drm_atomic_state_clear() to
-	 * avoid code duplicate between the clear and free hooks.
+	 * @timeline_name:
 	 *
-	 * Drivers that implement this must call drm_atomic_state_default_free()
-	 * to release common resources.
+	 * The name of the CRTC's fence timeline.
 	 */
-	void (*atomic_state_free)(struct drm_atomic_state *state);
+	char timeline_name[32];
 };
 
 /**
- * struct drm_mode_config - Mode configuration control structure
- * @mutex: mutex protecting KMS related lists and structures
- * @connection_mutex: ww mutex protecting connector state and routing
- * @acquire_ctx: global implicit acquire context used by atomic drivers for
- * 	legacy IOCTLs
- * @fb_lock: mutex to protect fb state and lists
- * @num_fb: number of fbs available
- * @fb_list: list of framebuffers available
- * @num_encoder: number of encoders on this device
- * @encoder_list: list of encoder objects
- * @num_overlay_plane: number of overlay planes on this device
- * @num_total_plane: number of universal (i.e. with primary/curso) planes on this device
- * @plane_list: list of plane objects
- * @num_crtc: number of CRTCs on this device
- * @crtc_list: list of CRTC objects
- * @property_list: list of property objects
- * @min_width: minimum pixel width on this device
- * @min_height: minimum pixel height on this device
- * @max_width: maximum pixel width on this device
- * @max_height: maximum pixel height on this device
- * @funcs: core driver provided mode setting functions
- * @fb_base: base address of the framebuffer
- * @poll_enabled: track polling support for this device
- * @poll_running: track polling status for this device
- * @delayed_event: track delayed poll uevent deliver for this device
- * @output_poll_work: delayed work for polling in process context
- * @property_blob_list: list of all the blob property objects
- * @blob_lock: mutex for blob property allocation and management
- * @*_property: core property tracking
- * @preferred_depth: preferred RBG pixel depth, used by fb helpers
- * @prefer_shadow: hint to userspace to prefer shadow-fb rendering
- * @cursor_width: hint to userspace for max cursor width
- * @cursor_height: hint to userspace for max cursor height
- * @helper_private: mid-layer private data
+ * struct drm_mode_set - new values for a CRTC config change
+ * @fb: framebuffer to use for new config
+ * @crtc: CRTC whose configuration we're about to change
+ * @mode: mode timings to use
+ * @x: position of this CRTC relative to @fb
+ * @y: position of this CRTC relative to @fb
+ * @connectors: array of connectors to drive with this CRTC if possible
+ * @num_connectors: size of @connectors array
  *
- * Core mode resource tracking structure.  All CRTC, encoders, and connectors
- * enumerated by the driver are added here, as are global properties.  Some
- * global restrictions are also here, e.g. dimension restrictions.
+ * Represents a single crtc the connectors that it drives with what mode
+ * and from which framebuffer it scans out from.
+ *
+ * This is used to set modes.
  */
-struct drm_mode_config {
-	struct mutex mutex; /* protects configuration (mode lists etc.) */
-	struct drm_modeset_lock connection_mutex; /* protects connector->encoder and encoder->crtc links */
-	struct drm_modeset_acquire_ctx *acquire_ctx; /* for legacy _lock_all() / _unlock_all() */
-
-	/**
-	 * @idr_mutex:
-	 *
-	 * Mutex for KMS ID allocation and management. Protects both @crtc_idr
-	 * and @tile_idr.
-	 */
-	struct mutex idr_mutex;
-
-	/**
-	 * @crtc_idr:
-	 *
-	 * Main KMS ID tracking object. Use this idr for all IDs, fb, crtc,
-	 * connector, modes - just makes life easier to have only one.
-	 */
-	struct idr crtc_idr;
-
-	/**
-	 * @tile_idr:
-	 *
-	 * Use this idr for allocating new IDs for tiled sinks like use in some
-	 * high-res DP MST screens.
-	 */
-	struct idr tile_idr;
-
-	struct mutex fb_lock; /* proctects global and per-file fb lists */
-	int num_fb;
-	struct list_head fb_list;
-
-	/**
-	 * @num_connector: Number of connectors on this device.
-	 */
-	int num_connector;
-	/**
-	 * @connector_ida: ID allocator for connector indices.
-	 */
-	struct ida connector_ida;
-	/**
-	 * @connector_list: List of connector objects.
-	 */
-	struct list_head connector_list;
-	int num_encoder;
-	struct list_head encoder_list;
-
-	/*
-	 * Track # of overlay planes separately from # of total planes.  By
-	 * default we only advertise overlay planes to userspace; if userspace
-	 * sets the "universal plane" capability bit, we'll go ahead and
-	 * expose all planes.
-	 */
-	int num_overlay_plane;
-	int num_total_plane;
-	struct list_head plane_list;
-
-	int num_crtc;
-	struct list_head crtc_list;
-
-	struct list_head property_list;
-
-	int min_width, min_height;
-	int max_width, max_height;
-	const struct drm_mode_config_funcs *funcs;
-	resource_size_t fb_base;
-
-	/* output poll support */
-	bool poll_enabled;
-	bool poll_running;
-	bool delayed_event;
-	struct delayed_work output_poll_work;
-
-	struct mutex blob_lock;
-
-	/* pointers to standard properties */
-	struct list_head property_blob_list;
-	/**
-	 * @edid_property: Default connector property to hold the EDID of the
-	 * currently connected sink, if any.
-	 */
-	struct drm_property *edid_property;
-	/**
-	 * @dpms_property: Default connector property to control the
-	 * connector's DPMS state.
-	 */
-	struct drm_property *dpms_property;
-	/**
-	 * @path_property: Default connector property to hold the DP MST path
-	 * for the port.
-	 */
-	struct drm_property *path_property;
-	/**
-	 * @tile_property: Default connector property to store the tile
-	 * position of a tiled screen, for sinks which need to be driven with
-	 * multiple CRTCs.
-	 */
-	struct drm_property *tile_property;
-	/**
-	 * @plane_type_property: Default plane property to differentiate
-	 * CURSOR, PRIMARY and OVERLAY legacy uses of planes.
-	 */
-	struct drm_property *plane_type_property;
-	/**
-	 * @rotation_property: Optional property for planes or CRTCs to specifiy
-	 * rotation.
-	 */
-	struct drm_property *rotation_property;
-	/**
-	 * @prop_src_x: Default atomic plane property for the plane source
-	 * position in the connected &drm_framebuffer.
-	 */
-	struct drm_property *prop_src_x;
-	/**
-	 * @prop_src_y: Default atomic plane property for the plane source
-	 * position in the connected &drm_framebuffer.
-	 */
-	struct drm_property *prop_src_y;
-	/**
-	 * @prop_src_w: Default atomic plane property for the plane source
-	 * position in the connected &drm_framebuffer.
-	 */
-	struct drm_property *prop_src_w;
-	/**
-	 * @prop_src_h: Default atomic plane property for the plane source
-	 * position in the connected &drm_framebuffer.
-	 */
-	struct drm_property *prop_src_h;
-	/**
-	 * @prop_crtc_x: Default atomic plane property for the plane destination
-	 * position in the &drm_crtc is is being shown on.
-	 */
-	struct drm_property *prop_crtc_x;
-	/**
-	 * @prop_crtc_y: Default atomic plane property for the plane destination
-	 * position in the &drm_crtc is is being shown on.
-	 */
-	struct drm_property *prop_crtc_y;
-	/**
-	 * @prop_crtc_w: Default atomic plane property for the plane destination
-	 * position in the &drm_crtc is is being shown on.
-	 */
-	struct drm_property *prop_crtc_w;
-	/**
-	 * @prop_crtc_h: Default atomic plane property for the plane destination
-	 * position in the &drm_crtc is is being shown on.
-	 */
-	struct drm_property *prop_crtc_h;
-	/**
-	 * @prop_fb_id: Default atomic plane property to specify the
-	 * &drm_framebuffer.
-	 */
-	struct drm_property *prop_fb_id;
-	/**
-	 * @prop_crtc_id: Default atomic plane property to specify the
-	 * &drm_crtc.
-	 */
-	struct drm_property *prop_crtc_id;
-	/**
-	 * @prop_active: Default atomic CRTC property to control the active
-	 * state, which is the simplified implementation for DPMS in atomic
-	 * drivers.
-	 */
-	struct drm_property *prop_active;
-	/**
-	 * @prop_mode_id: Default atomic CRTC property to set the mode for a
-	 * CRTC. A 0 mode implies that the CRTC is entirely disabled - all
-	 * connectors must be of and active must be set to disabled, too.
-	 */
-	struct drm_property *prop_mode_id;
-
-	/**
-	 * @dvi_i_subconnector_property: Optional DVI-I property to
-	 * differentiate between analog or digital mode.
-	 */
-	struct drm_property *dvi_i_subconnector_property;
-	/**
-	 * @dvi_i_select_subconnector_property: Optional DVI-I property to
-	 * select between analog or digital mode.
-	 */
-	struct drm_property *dvi_i_select_subconnector_property;
-
-	/**
-	 * @tv_subconnector_property: Optional TV property to differentiate
-	 * between different TV connector types.
-	 */
-	struct drm_property *tv_subconnector_property;
-	/**
-	 * @tv_select_subconnector_property: Optional TV property to select
-	 * between different TV connector types.
-	 */
-	struct drm_property *tv_select_subconnector_property;
-	/**
-	 * @tv_mode_property: Optional TV property to select
-	 * the output TV mode.
-	 */
-	struct drm_property *tv_mode_property;
-	/**
-	 * @tv_left_margin_property: Optional TV property to set the left
-	 * margin.
-	 */
-	struct drm_property *tv_left_margin_property;
-	/**
-	 * @tv_right_margin_property: Optional TV property to set the right
-	 * margin.
-	 */
-	struct drm_property *tv_right_margin_property;
-	/**
-	 * @tv_top_margin_property: Optional TV property to set the right
-	 * margin.
-	 */
-	struct drm_property *tv_top_margin_property;
-	/**
-	 * @tv_bottom_margin_property: Optional TV property to set the right
-	 * margin.
-	 */
-	struct drm_property *tv_bottom_margin_property;
-	/**
-	 * @tv_brightness_property: Optional TV property to set the
-	 * brightness.
-	 */
-	struct drm_property *tv_brightness_property;
-	/**
-	 * @tv_contrast_property: Optional TV property to set the
-	 * contrast.
-	 */
-	struct drm_property *tv_contrast_property;
-	/**
-	 * @tv_flicker_reduction_property: Optional TV property to control the
-	 * flicker reduction mode.
-	 */
-	struct drm_property *tv_flicker_reduction_property;
-	/**
-	 * @tv_overscan_property: Optional TV property to control the overscan
-	 * setting.
-	 */
-	struct drm_property *tv_overscan_property;
-	/**
-	 * @tv_saturation_property: Optional TV property to set the
-	 * saturation.
-	 */
-	struct drm_property *tv_saturation_property;
-	/**
-	 * @tv_hue_property: Optional TV property to set the hue.
-	 */
-	struct drm_property *tv_hue_property;
-
-	/**
-	 * @scaling_mode_property: Optional connector property to control the
-	 * upscaling, mostly used for built-in panels.
-	 */
-	struct drm_property *scaling_mode_property;
-	/**
-	 * @aspect_ratio_property: Optional connector property to control the
-	 * HDMI infoframe aspect ratio setting.
-	 */
-	struct drm_property *aspect_ratio_property;
-	/**
-	 * @degamma_lut_property: Optional CRTC property to set the LUT used to
-	 * convert the framebuffer's colors to linear gamma.
-	 */
-	struct drm_property *degamma_lut_property;
-	/**
-	 * @degamma_lut_size_property: Optional CRTC property for the size of
-	 * the degamma LUT as supported by the driver (read-only).
-	 */
-	struct drm_property *degamma_lut_size_property;
-	/**
-	 * @ctm_property: Optional CRTC property to set the
-	 * matrix used to convert colors after the lookup in the
-	 * degamma LUT.
-	 */
-	struct drm_property *ctm_property;
-	/**
-	 * @gamma_lut_property: Optional CRTC property to set the LUT used to
-	 * convert the colors, after the CTM matrix, to the gamma space of the
-	 * connected screen.
-	 */
-	struct drm_property *gamma_lut_property;
-	/**
-	 * @gamma_lut_size_property: Optional CRTC property for the size of the
-	 * gamma LUT as supported by the driver (read-only).
-	 */
-	struct drm_property *gamma_lut_size_property;
-
-	/**
-	 * @suggested_x_property: Optional connector property with a hint for
-	 * the position of the output on the host's screen.
-	 */
-	struct drm_property *suggested_x_property;
-	/**
-	 * @suggested_y_property: Optional connector property with a hint for
-	 * the position of the output on the host's screen.
-	 */
-	struct drm_property *suggested_y_property;
-
-	/* dumb ioctl parameters */
-	uint32_t preferred_depth, prefer_shadow;
-
-	/**
-	 * @async_page_flip: Does this device support async flips on the primary
-	 * plane?
-	 */
-	bool async_page_flip;
-
-	/**
-	 * @allow_fb_modifiers:
-	 *
-	 * Whether the driver supports fb modifiers in the ADDFB2.1 ioctl call.
-	 */
-	bool allow_fb_modifiers;
+struct drm_mode_set {
+	struct drm_framebuffer *fb;
+	struct drm_crtc *crtc;
+	struct drm_display_mode *mode;
 
-	/* cursor size */
-	uint32_t cursor_width, cursor_height;
+	uint32_t x;
+	uint32_t y;
 
-	struct drm_mode_config_helper_funcs *helper_private;
+	struct drm_connector **connectors;
+	size_t num_connectors;
 };
 
 #define obj_to_crtc(x) container_of(x, struct drm_crtc, base)
 
-extern __printf(6, 7)
+__printf(6, 7)
 int drm_crtc_init_with_planes(struct drm_device *dev,
 			      struct drm_crtc *crtc,
 			      struct drm_plane *primary,
 			      struct drm_plane *cursor,
 			      const struct drm_crtc_funcs *funcs,
 			      const char *name, ...);
-extern void drm_crtc_cleanup(struct drm_crtc *crtc);
+void drm_crtc_cleanup(struct drm_crtc *crtc);
 
 /**
  * drm_crtc_index - find the index of a registered CRTC
@@ -1354,28 +819,17 @@ static inline unsigned int drm_crtc_index(const struct drm_crtc *crtc)
  * Given a registered CRTC, return the mask bit of that CRTC for an
  * encoder's possible_crtcs field.
  */
-static inline uint32_t drm_crtc_mask(struct drm_crtc *crtc)
+static inline uint32_t drm_crtc_mask(const struct drm_crtc *crtc)
 {
 	return 1 << drm_crtc_index(crtc);
 }
 
-extern void drm_crtc_get_hv_timing(const struct drm_display_mode *mode,
-				   int *hdisplay, int *vdisplay);
-extern int drm_crtc_force_disable(struct drm_crtc *crtc);
-extern int drm_crtc_force_disable_all(struct drm_device *dev);
-
-extern void drm_mode_config_init(struct drm_device *dev);
-extern void drm_mode_config_reset(struct drm_device *dev);
-extern void drm_mode_config_cleanup(struct drm_device *dev);
-
-extern int drm_mode_set_config_internal(struct drm_mode_set *set);
+void drm_crtc_get_hv_timing(const struct drm_display_mode *mode,
+			    int *hdisplay, int *vdisplay);
+int drm_crtc_force_disable(struct drm_crtc *crtc);
+int drm_crtc_force_disable_all(struct drm_device *dev);
 
-extern struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev,
-							 char topology[8]);
-extern struct drm_tile_group *drm_mode_get_tile_group(struct drm_device *dev,
-					       char topology[8]);
-extern void drm_mode_put_tile_group(struct drm_device *dev,
-				   struct drm_tile_group *tg);
+int drm_mode_set_config_internal(struct drm_mode_set *set);
 
 /* Helpers */
 static inline struct drm_crtc *drm_crtc_find(struct drm_device *dev,
diff --git a/include/drm/drm_debugfs_crc.h b/include/drm/drm_debugfs_crc.h
new file mode 100644
index 000000000000..7d63b1d4adb9
--- /dev/null
+++ b/include/drm/drm_debugfs_crc.h
@@ -0,0 +1,73 @@
+/*
+ * Copyright © 2016 Collabora Ltd.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef __DRM_DEBUGFS_CRC_H__
+#define __DRM_DEBUGFS_CRC_H__
+
+#define DRM_MAX_CRC_NR		10
+
+/**
+ * struct drm_crtc_crc_entry - entry describing a frame's content
+ * @has_frame_counter: whether the source was able to provide a frame number
+ * @frame: number of the frame this CRC is about, if @has_frame_counter is true
+ * @crc: array of values that characterize the frame
+ */
+struct drm_crtc_crc_entry {
+	bool has_frame_counter;
+	uint32_t frame;
+	uint32_t crcs[DRM_MAX_CRC_NR];
+};
+
+#define DRM_CRC_ENTRIES_NR	128
+
+/**
+ * struct drm_crtc_crc - data supporting CRC capture on a given CRTC
+ * @lock: protects the fields in this struct
+ * @source: name of the currently configured source of CRCs
+ * @opened: whether userspace has opened the data file for reading
+ * @entries: array of entries, with size of %DRM_CRC_ENTRIES_NR
+ * @head: head of circular queue
+ * @tail: tail of circular queue
+ * @values_cnt: number of CRC values per entry, up to %DRM_MAX_CRC_NR
+ * @wq: workqueue used to synchronize reading and writing
+ */
+struct drm_crtc_crc {
+	spinlock_t lock;
+	const char *source;
+	bool opened;
+	struct drm_crtc_crc_entry *entries;
+	int head, tail;
+	size_t values_cnt;
+	wait_queue_head_t wq;
+};
+
+#if defined(CONFIG_DEBUG_FS)
+int drm_crtc_add_crc_entry(struct drm_crtc *crtc, bool has_frame,
+			   uint32_t frame, uint32_t *crcs);
+#else
+static inline int drm_crtc_add_crc_entry(struct drm_crtc *crtc, bool has_frame,
+					 uint32_t frame, uint32_t *crcs)
+{
+	return -EINVAL;
+}
+#endif /* defined(CONFIG_DEBUG_FS) */
+
+#endif /* __DRM_DEBUGFS_CRC_H__ */
diff --git a/include/drm/drm_dp_dual_mode_helper.h b/include/drm/drm_dp_dual_mode_helper.h
index e8a9dfd0e055..4c42db81fcb4 100644
--- a/include/drm/drm_dp_dual_mode_helper.h
+++ b/include/drm/drm_dp_dual_mode_helper.h
@@ -40,6 +40,8 @@
 #define  DP_DUAL_MODE_REV_TYPE2 0x00
 #define  DP_DUAL_MODE_TYPE_MASK 0xf0
 #define  DP_DUAL_MODE_TYPE_TYPE2 0xa0
+/* This field is marked reserved in dual mode spec, used in LSPCON */
+#define  DP_DUAL_MODE_TYPE_HAS_DPCD 0x08
 #define DP_DUAL_MODE_IEEE_OUI 0x11 /* 11-13*/
 #define  DP_DUAL_IEEE_OUI_LEN 3
 #define DP_DUAL_DEVICE_ID 0x14 /* 14-19 */
@@ -55,6 +57,11 @@
 #define  DP_DUAL_MODE_CEC_ENABLE 0x01
 #define DP_DUAL_MODE_I2C_SPEED_CTRL 0x22
 
+/* LSPCON specific registers, defined by MCA */
+#define DP_DUAL_MODE_LSPCON_MODE_CHANGE		0x40
+#define DP_DUAL_MODE_LSPCON_CURRENT_MODE		0x41
+#define  DP_DUAL_MODE_LSPCON_MODE_PCON			0x1
+
 struct i2c_adapter;
 
 ssize_t drm_dp_dual_mode_read(struct i2c_adapter *adapter,
@@ -63,6 +70,20 @@ ssize_t drm_dp_dual_mode_write(struct i2c_adapter *adapter,
 			       u8 offset, const void *buffer, size_t size);
 
 /**
+ * enum drm_lspcon_mode
+ * @DRM_LSPCON_MODE_INVALID: No LSPCON.
+ * @DRM_LSPCON_MODE_LS: Level shifter mode of LSPCON
+ *	which drives DP++ to HDMI 1.4 conversion.
+ * @DRM_LSPCON_MODE_PCON: Protocol converter mode of LSPCON
+ *	which drives DP++ to HDMI 2.0 active conversion.
+ */
+enum drm_lspcon_mode {
+	DRM_LSPCON_MODE_INVALID,
+	DRM_LSPCON_MODE_LS,
+	DRM_LSPCON_MODE_PCON,
+};
+
+/**
  * enum drm_dp_dual_mode_type - Type of the DP dual mode adaptor
  * @DRM_DP_DUAL_MODE_NONE: No DP dual mode adaptor
  * @DRM_DP_DUAL_MODE_UNKNOWN: Could be either none or type 1 DVI adaptor
@@ -70,6 +91,7 @@ ssize_t drm_dp_dual_mode_write(struct i2c_adapter *adapter,
  * @DRM_DP_DUAL_MODE_TYPE1_HDMI: Type 1 HDMI adaptor
  * @DRM_DP_DUAL_MODE_TYPE2_DVI: Type 2 DVI adaptor
  * @DRM_DP_DUAL_MODE_TYPE2_HDMI: Type 2 HDMI adaptor
+ * @DRM_DP_DUAL_MODE_LSPCON: Level shifter / protocol converter
  */
 enum drm_dp_dual_mode_type {
 	DRM_DP_DUAL_MODE_NONE,
@@ -78,6 +100,7 @@ enum drm_dp_dual_mode_type {
 	DRM_DP_DUAL_MODE_TYPE1_HDMI,
 	DRM_DP_DUAL_MODE_TYPE2_DVI,
 	DRM_DP_DUAL_MODE_TYPE2_HDMI,
+	DRM_DP_DUAL_MODE_LSPCON,
 };
 
 enum drm_dp_dual_mode_type drm_dp_dual_mode_detect(struct i2c_adapter *adapter);
@@ -89,4 +112,8 @@ int drm_dp_dual_mode_set_tmds_output(enum drm_dp_dual_mode_type type,
 				     struct i2c_adapter *adapter, bool enable);
 const char *drm_dp_get_dual_mode_type_name(enum drm_dp_dual_mode_type type);
 
+int drm_lspcon_get_mode(struct i2c_adapter *adapter,
+			enum drm_lspcon_mode *current_mode);
+int drm_lspcon_set_mode(struct i2c_adapter *adapter,
+			enum drm_lspcon_mode reqd_mode);
 #endif
diff --git a/include/drm/drm_dp_helper.h b/include/drm/drm_dp_helper.h
index 2a79882cb68e..55bbeb0ff594 100644
--- a/include/drm/drm_dp_helper.h
+++ b/include/drm/drm_dp_helper.h
@@ -690,6 +690,12 @@ drm_dp_tps3_supported(const u8 dpcd[DP_RECEIVER_CAP_SIZE])
 		dpcd[DP_MAX_LANE_COUNT] & DP_TPS3_SUPPORTED;
 }
 
+static inline bool
+drm_dp_is_branch(const u8 dpcd[DP_RECEIVER_CAP_SIZE])
+{
+	return dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT;
+}
+
 /*
  * DisplayPort AUX channel
  */
diff --git a/include/drm/drm_drv.h b/include/drm/drm_drv.h
new file mode 100644
index 000000000000..c4fc49583dc0
--- /dev/null
+++ b/include/drm/drm_drv.h
@@ -0,0 +1,435 @@
+/*
+ * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
+ * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
+ * Copyright (c) 2009-2010, Code Aurora Forum.
+ * Copyright 2016 Intel Corp.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _DRM_DRV_H_
+#define _DRM_DRV_H_
+
+#include <linux/list.h>
+#include <linux/irqreturn.h>
+
+struct drm_device;
+struct drm_file;
+struct drm_gem_object;
+struct drm_master;
+struct drm_minor;
+struct dma_buf_attachment;
+struct drm_display_mode;
+struct drm_mode_create_dumb;
+
+/* driver capabilities and requirements mask */
+#define DRIVER_USE_AGP			0x1
+#define DRIVER_LEGACY			0x2
+#define DRIVER_PCI_DMA			0x8
+#define DRIVER_SG			0x10
+#define DRIVER_HAVE_DMA			0x20
+#define DRIVER_HAVE_IRQ			0x40
+#define DRIVER_IRQ_SHARED		0x80
+#define DRIVER_GEM			0x1000
+#define DRIVER_MODESET			0x2000
+#define DRIVER_PRIME			0x4000
+#define DRIVER_RENDER			0x8000
+#define DRIVER_ATOMIC			0x10000
+#define DRIVER_KMS_LEGACY_CONTEXT	0x20000
+
+/**
+ * struct drm_driver - DRM driver structure
+ *
+ * This structure represent the common code for a family of cards. There will
+ * one drm_device for each card present in this family. It contains lots of
+ * vfunc entries, and a pile of those probably should be moved to more
+ * appropriate places like &drm_mode_config_funcs or into a new operations
+ * structure for GEM drivers.
+ */
+struct drm_driver {
+	int (*load) (struct drm_device *, unsigned long flags);
+	int (*firstopen) (struct drm_device *);
+	int (*open) (struct drm_device *, struct drm_file *);
+	void (*preclose) (struct drm_device *, struct drm_file *file_priv);
+	void (*postclose) (struct drm_device *, struct drm_file *);
+	void (*lastclose) (struct drm_device *);
+	int (*unload) (struct drm_device *);
+	int (*dma_ioctl) (struct drm_device *dev, void *data, struct drm_file *file_priv);
+	int (*dma_quiescent) (struct drm_device *);
+	int (*context_dtor) (struct drm_device *dev, int context);
+	int (*set_busid)(struct drm_device *dev, struct drm_master *master);
+
+	/**
+	 * @get_vblank_counter:
+	 *
+	 * Driver callback for fetching a raw hardware vblank counter for the
+	 * CRTC specified with the pipe argument.  If a device doesn't have a
+	 * hardware counter, the driver can simply use
+	 * drm_vblank_no_hw_counter() function. The DRM core will account for
+	 * missed vblank events while interrupts where disabled based on system
+	 * timestamps.
+	 *
+	 * Wraparound handling and loss of events due to modesetting is dealt
+	 * with in the DRM core code, as long as drivers call
+	 * drm_crtc_vblank_off() and drm_crtc_vblank_on() when disabling or
+	 * enabling a CRTC.
+	 *
+	 * Returns:
+	 *
+	 * Raw vblank counter value.
+	 */
+	u32 (*get_vblank_counter) (struct drm_device *dev, unsigned int pipe);
+
+	/**
+	 * @enable_vblank:
+	 *
+	 * Enable vblank interrupts for the CRTC specified with the pipe
+	 * argument.
+	 *
+	 * Returns:
+	 *
+	 * Zero on success, appropriate errno if the given @crtc's vblank
+	 * interrupt cannot be enabled.
+	 */
+	int (*enable_vblank) (struct drm_device *dev, unsigned int pipe);
+
+	/**
+	 * @disable_vblank:
+	 *
+	 * Disable vblank interrupts for the CRTC specified with the pipe
+	 * argument.
+	 */
+	void (*disable_vblank) (struct drm_device *dev, unsigned int pipe);
+
+	/**
+	 * @device_is_agp:
+	 *
+	 * Called by drm_device_is_agp().  Typically used to determine if a card
+	 * is really attached to AGP or not.
+	 *
+	 * Returns:
+	 *
+	 * One of three values is returned depending on whether or not the
+	 * card is absolutely not AGP (return of 0), absolutely is AGP
+	 * (return of 1), or may or may not be AGP (return of 2).
+	 */
+	int (*device_is_agp) (struct drm_device *dev);
+
+	/**
+	 * @get_scanout_position:
+	 *
+	 * Called by vblank timestamping code.
+	 *
+	 * Returns the current display scanout position from a crtc, and an
+	 * optional accurate ktime_get() timestamp of when position was
+	 * measured. Note that this is a helper callback which is only used if a
+	 * driver uses drm_calc_vbltimestamp_from_scanoutpos() for the
+	 * @get_vblank_timestamp callback.
+	 *
+	 * Parameters:
+	 *
+	 * dev:
+	 *     DRM device.
+	 * pipe:
+	 *     Id of the crtc to query.
+	 * flags:
+	 *     Flags from the caller (DRM_CALLED_FROM_VBLIRQ or 0).
+	 * vpos:
+	 *     Target location for current vertical scanout position.
+	 * hpos:
+	 *     Target location for current horizontal scanout position.
+	 * stime:
+	 *     Target location for timestamp taken immediately before
+	 *     scanout position query. Can be NULL to skip timestamp.
+	 * etime:
+	 *     Target location for timestamp taken immediately after
+	 *     scanout position query. Can be NULL to skip timestamp.
+	 * mode:
+	 *     Current display timings.
+	 *
+	 * Returns vpos as a positive number while in active scanout area.
+	 * Returns vpos as a negative number inside vblank, counting the number
+	 * of scanlines to go until end of vblank, e.g., -1 means "one scanline
+	 * until start of active scanout / end of vblank."
+	 *
+	 * Returns:
+	 *
+	 * Flags, or'ed together as follows:
+	 *
+	 * DRM_SCANOUTPOS_VALID:
+	 *     Query successful.
+	 * DRM_SCANOUTPOS_INVBL:
+	 *     Inside vblank.
+	 * DRM_SCANOUTPOS_ACCURATE: Returned position is accurate. A lack of
+	 *     this flag means that returned position may be offset by a
+	 *     constant but unknown small number of scanlines wrt. real scanout
+	 *     position.
+	 *
+	 */
+	int (*get_scanout_position) (struct drm_device *dev, unsigned int pipe,
+				     unsigned int flags, int *vpos, int *hpos,
+				     ktime_t *stime, ktime_t *etime,
+				     const struct drm_display_mode *mode);
+
+	/**
+	 * @get_vblank_timestamp:
+	 *
+	 * Called by drm_get_last_vbltimestamp(). Should return a precise
+	 * timestamp when the most recent VBLANK interval ended or will end.
+	 *
+	 * Specifically, the timestamp in @vblank_time should correspond as
+	 * closely as possible to the time when the first video scanline of
+	 * the video frame after the end of VBLANK will start scanning out,
+	 * the time immediately after end of the VBLANK interval. If the
+	 * @crtc is currently inside VBLANK, this will be a time in the future.
+	 * If the @crtc is currently scanning out a frame, this will be the
+	 * past start time of the current scanout. This is meant to adhere
+	 * to the OpenML OML_sync_control extension specification.
+	 *
+	 * Paramters:
+	 *
+	 * dev:
+	 *     dev DRM device handle.
+	 * pipe:
+	 *     crtc for which timestamp should be returned.
+	 * max_error:
+	 *     Maximum allowable timestamp error in nanoseconds.
+	 *     Implementation should strive to provide timestamp
+	 *     with an error of at most max_error nanoseconds.
+	 *     Returns true upper bound on error for timestamp.
+	 * vblank_time:
+	 *     Target location for returned vblank timestamp.
+	 * flags:
+	 *     0 = Defaults, no special treatment needed.
+	 *     DRM_CALLED_FROM_VBLIRQ = Function is called from vblank
+	 *     irq handler. Some drivers need to apply some workarounds
+	 *     for gpu-specific vblank irq quirks if flag is set.
+	 *
+	 * Returns:
+	 *
+	 * Zero if timestamping isn't supported in current display mode or a
+	 * negative number on failure. A positive status code on success,
+	 * which describes how the vblank_time timestamp was computed.
+	 */
+	int (*get_vblank_timestamp) (struct drm_device *dev, unsigned int pipe,
+				     int *max_error,
+				     struct timeval *vblank_time,
+				     unsigned flags);
+
+	/* these have to be filled in */
+
+	irqreturn_t(*irq_handler) (int irq, void *arg);
+	void (*irq_preinstall) (struct drm_device *dev);
+	int (*irq_postinstall) (struct drm_device *dev);
+	void (*irq_uninstall) (struct drm_device *dev);
+
+	/**
+	 * @master_create:
+	 *
+	 * Called whenever a new master is created. Only used by vmwgfx.
+	 */
+	int (*master_create)(struct drm_device *dev, struct drm_master *master);
+
+	/**
+	 * @master_destroy:
+	 *
+	 * Called whenever a master is destroyed. Only used by vmwgfx.
+	 */
+	void (*master_destroy)(struct drm_device *dev, struct drm_master *master);
+
+	/**
+	 * @master_set:
+	 *
+	 * Called whenever the minor master is set. Only used by vmwgfx.
+	 */
+	int (*master_set)(struct drm_device *dev, struct drm_file *file_priv,
+			  bool from_open);
+	/**
+	 * @master_drop:
+	 *
+	 * Called whenever the minor master is dropped. Only used by vmwgfx.
+	 */
+	void (*master_drop)(struct drm_device *dev, struct drm_file *file_priv);
+
+	int (*debugfs_init)(struct drm_minor *minor);
+	void (*debugfs_cleanup)(struct drm_minor *minor);
+
+	/**
+	 * @gem_free_object: deconstructor for drm_gem_objects
+	 *
+	 * This is deprecated and should not be used by new drivers. Use
+	 * @gem_free_object_unlocked instead.
+	 */
+	void (*gem_free_object) (struct drm_gem_object *obj);
+
+	/**
+	 * @gem_free_object_unlocked: deconstructor for drm_gem_objects
+	 *
+	 * This is for drivers which are not encumbered with dev->struct_mutex
+	 * legacy locking schemes. Use this hook instead of @gem_free_object.
+	 */
+	void (*gem_free_object_unlocked) (struct drm_gem_object *obj);
+
+	int (*gem_open_object) (struct drm_gem_object *, struct drm_file *);
+	void (*gem_close_object) (struct drm_gem_object *, struct drm_file *);
+
+	/**
+	 * @gem_create_object: constructor for gem objects
+	 *
+	 * Hook for allocating the GEM object struct, for use by core
+	 * helpers.
+	 */
+	struct drm_gem_object *(*gem_create_object)(struct drm_device *dev,
+						    size_t size);
+
+	/* prime: */
+	/* export handle -> fd (see drm_gem_prime_handle_to_fd() helper) */
+	int (*prime_handle_to_fd)(struct drm_device *dev, struct drm_file *file_priv,
+				uint32_t handle, uint32_t flags, int *prime_fd);
+	/* import fd -> handle (see drm_gem_prime_fd_to_handle() helper) */
+	int (*prime_fd_to_handle)(struct drm_device *dev, struct drm_file *file_priv,
+				int prime_fd, uint32_t *handle);
+	/* export GEM -> dmabuf */
+	struct dma_buf * (*gem_prime_export)(struct drm_device *dev,
+				struct drm_gem_object *obj, int flags);
+	/* import dmabuf -> GEM */
+	struct drm_gem_object * (*gem_prime_import)(struct drm_device *dev,
+				struct dma_buf *dma_buf);
+	/* low-level interface used by drm_gem_prime_{import,export} */
+	int (*gem_prime_pin)(struct drm_gem_object *obj);
+	void (*gem_prime_unpin)(struct drm_gem_object *obj);
+	struct reservation_object * (*gem_prime_res_obj)(
+				struct drm_gem_object *obj);
+	struct sg_table *(*gem_prime_get_sg_table)(struct drm_gem_object *obj);
+	struct drm_gem_object *(*gem_prime_import_sg_table)(
+				struct drm_device *dev,
+				struct dma_buf_attachment *attach,
+				struct sg_table *sgt);
+	void *(*gem_prime_vmap)(struct drm_gem_object *obj);
+	void (*gem_prime_vunmap)(struct drm_gem_object *obj, void *vaddr);
+	int (*gem_prime_mmap)(struct drm_gem_object *obj,
+				struct vm_area_struct *vma);
+
+	/* vga arb irq handler */
+	void (*vgaarb_irq)(struct drm_device *dev, bool state);
+
+	/**
+	 * @dumb_create:
+	 *
+	 * This creates a new dumb buffer in the driver's backing storage manager (GEM,
+	 * TTM or something else entirely) and returns the resulting buffer handle. This
+	 * handle can then be wrapped up into a framebuffer modeset object.
+	 *
+	 * Note that userspace is not allowed to use such objects for render
+	 * acceleration - drivers must create their own private ioctls for such a use
+	 * case.
+	 *
+	 * Width, height and depth are specified in the &drm_mode_create_dumb
+	 * argument. The callback needs to fill the handle, pitch and size for
+	 * the created buffer.
+	 *
+	 * Called by the user via ioctl.
+	 *
+	 * Returns:
+	 *
+	 * Zero on success, negative errno on failure.
+	 */
+	int (*dumb_create)(struct drm_file *file_priv,
+			   struct drm_device *dev,
+			   struct drm_mode_create_dumb *args);
+	/**
+	 * @dumb_map_offset:
+	 *
+	 * Allocate an offset in the drm device node's address space to be able to
+	 * memory map a dumb buffer. GEM-based drivers must use
+	 * drm_gem_create_mmap_offset() to implement this.
+	 *
+	 * Called by the user via ioctl.
+	 *
+	 * Returns:
+	 *
+	 * Zero on success, negative errno on failure.
+	 */
+	int (*dumb_map_offset)(struct drm_file *file_priv,
+			       struct drm_device *dev, uint32_t handle,
+			       uint64_t *offset);
+	/**
+	 * @dumb_destroy:
+	 *
+	 * This destroys the userspace handle for the given dumb backing storage buffer.
+	 * Since buffer objects must be reference counted in the kernel a buffer object
+	 * won't be immediately freed if a framebuffer modeset object still uses it.
+	 *
+	 * Called by the user via ioctl.
+	 *
+	 * Returns:
+	 *
+	 * Zero on success, negative errno on failure.
+	 */
+	int (*dumb_destroy)(struct drm_file *file_priv,
+			    struct drm_device *dev,
+			    uint32_t handle);
+
+	/* Driver private ops for this object */
+	const struct vm_operations_struct *gem_vm_ops;
+
+	int major;
+	int minor;
+	int patchlevel;
+	char *name;
+	char *desc;
+	char *date;
+
+	u32 driver_features;
+	int dev_priv_size;
+	const struct drm_ioctl_desc *ioctls;
+	int num_ioctls;
+	const struct file_operations *fops;
+
+	/* List of devices hanging off this driver with stealth attach. */
+	struct list_head legacy_dev_list;
+};
+
+extern __printf(6, 7)
+void drm_dev_printk(const struct device *dev, const char *level,
+		    unsigned int category, const char *function_name,
+		    const char *prefix, const char *format, ...);
+extern __printf(3, 4)
+void drm_printk(const char *level, unsigned int category,
+		const char *format, ...);
+extern unsigned int drm_debug;
+
+int drm_dev_init(struct drm_device *dev,
+		 struct drm_driver *driver,
+		 struct device *parent);
+struct drm_device *drm_dev_alloc(struct drm_driver *driver,
+				 struct device *parent);
+int drm_dev_register(struct drm_device *dev, unsigned long flags);
+void drm_dev_unregister(struct drm_device *dev);
+
+void drm_dev_ref(struct drm_device *dev);
+void drm_dev_unref(struct drm_device *dev);
+void drm_put_dev(struct drm_device *dev);
+void drm_unplug_dev(struct drm_device *dev);
+
+int drm_dev_set_unique(struct drm_device *dev, const char *name);
+
+
+#endif
diff --git a/include/drm/drm_edid.h b/include/drm/drm_edid.h
index c3a7d440bc11..38eabf65f19d 100644
--- a/include/drm/drm_edid.h
+++ b/include/drm/drm_edid.h
@@ -330,7 +330,6 @@ int drm_edid_to_sad(struct edid *edid, struct cea_sad **sads);
 int drm_edid_to_speaker_allocation(struct edid *edid, u8 **sadb);
 int drm_av_sync_delay(struct drm_connector *connector,
 		      const struct drm_display_mode *mode);
-struct drm_connector *drm_select_eld(struct drm_encoder *encoder);
 
 #ifdef CONFIG_DRM_LOAD_EDID_FIRMWARE
 int drm_load_edid_firmware(struct drm_connector *connector);
diff --git a/include/drm/drm_encoder.h b/include/drm/drm_encoder.h
index 387e33a4d6ee..c7438ff0d609 100644
--- a/include/drm/drm_encoder.h
+++ b/include/drm/drm_encoder.h
@@ -189,7 +189,7 @@ static inline unsigned int drm_encoder_index(struct drm_encoder *encoder)
 }
 
 /* FIXME: We have an include file mess still, drm_crtc.h needs untangling. */
-static inline uint32_t drm_crtc_mask(struct drm_crtc *crtc);
+static inline uint32_t drm_crtc_mask(const struct drm_crtc *crtc);
 
 /**
  * drm_encoder_crtc_ok - can a given crtc drive a given encoder?
diff --git a/include/drm/drm_fb_cma_helper.h b/include/drm/drm_fb_cma_helper.h
index f313211f8ed5..3b00f6480b83 100644
--- a/include/drm/drm_fb_cma_helper.h
+++ b/include/drm/drm_fb_cma_helper.h
@@ -12,6 +12,8 @@ struct drm_fb_helper;
 struct drm_device;
 struct drm_file;
 struct drm_mode_fb_cmd2;
+struct drm_plane;
+struct drm_plane_state;
 
 struct drm_fbdev_cma *drm_fbdev_cma_init_with_funcs(struct drm_device *dev,
 	unsigned int preferred_bpp, unsigned int num_crtc,
@@ -41,6 +43,9 @@ struct drm_framebuffer *drm_fb_cma_create(struct drm_device *dev,
 struct drm_gem_cma_object *drm_fb_cma_get_gem_obj(struct drm_framebuffer *fb,
 	unsigned int plane);
 
+int drm_fb_cma_prepare_fb(struct drm_plane *plane,
+			  struct drm_plane_state *state);
+
 #ifdef CONFIG_DEBUG_FS
 struct seq_file;
 
diff --git a/include/drm/drm_fb_helper.h b/include/drm/drm_fb_helper.h
index ed8edfef75b2..975deedd593e 100644
--- a/include/drm/drm_fb_helper.h
+++ b/include/drm/drm_fb_helper.h
@@ -228,7 +228,9 @@ struct drm_fb_helper {
 	.fb_set_par	= drm_fb_helper_set_par, \
 	.fb_setcmap	= drm_fb_helper_setcmap, \
 	.fb_blank	= drm_fb_helper_blank, \
-	.fb_pan_display	= drm_fb_helper_pan_display
+	.fb_pan_display	= drm_fb_helper_pan_display, \
+	.fb_debug_enter = drm_fb_helper_debug_enter, \
+	.fb_debug_leave = drm_fb_helper_debug_leave
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
 void drm_fb_helper_prepare(struct drm_device *dev, struct drm_fb_helper *helper,
diff --git a/include/drm/drm_fourcc.h b/include/drm/drm_fourcc.h
index 30c30fa87ee8..fcc08da850c8 100644
--- a/include/drm/drm_fourcc.h
+++ b/include/drm/drm_fourcc.h
@@ -25,14 +25,43 @@
 #include <linux/types.h>
 #include <uapi/drm/drm_fourcc.h>
 
+/**
+ * struct drm_format_info - information about a DRM format
+ * @format: 4CC format identifier (DRM_FORMAT_*)
+ * @depth: Color depth (number of bits per pixel excluding padding bits),
+ *	valid for a subset of RGB formats only. This is a legacy field, do not
+ *	use in new code and set to 0 for new formats.
+ * @num_planes: Number of color planes (1 to 3)
+ * @cpp: Number of bytes per pixel (per plane)
+ * @hsub: Horizontal chroma subsampling factor
+ * @vsub: Vertical chroma subsampling factor
+ */
+struct drm_format_info {
+	u32 format;
+	u8 depth;
+	u8 num_planes;
+	u8 cpp[3];
+	u8 hsub;
+	u8 vsub;
+};
+
+/**
+ * struct drm_format_name_buf - name of a DRM format
+ * @str: string buffer containing the format name
+ */
+struct drm_format_name_buf {
+	char str[32];
+};
+
+const struct drm_format_info *__drm_format_info(u32 format);
+const struct drm_format_info *drm_format_info(u32 format);
 uint32_t drm_mode_legacy_fb_format(uint32_t bpp, uint32_t depth);
-void drm_fb_get_bpp_depth(uint32_t format, unsigned int *depth, int *bpp);
 int drm_format_num_planes(uint32_t format);
 int drm_format_plane_cpp(uint32_t format, int plane);
 int drm_format_horz_chroma_subsampling(uint32_t format);
 int drm_format_vert_chroma_subsampling(uint32_t format);
 int drm_format_plane_width(int width, uint32_t format, int plane);
 int drm_format_plane_height(int height, uint32_t format, int plane);
-char *drm_get_format_name(uint32_t format) __malloc;
+const char *drm_get_format_name(uint32_t format, struct drm_format_name_buf *buf);
 
 #endif /* __DRM_FOURCC_H__ */
diff --git a/include/drm/drm_framebuffer.h b/include/drm/drm_framebuffer.h
index f5ae1f436a4b..1ddfa2928802 100644
--- a/include/drm/drm_framebuffer.h
+++ b/include/drm/drm_framebuffer.h
@@ -149,12 +149,12 @@ struct drm_framebuffer {
 	 */
 	unsigned int offsets[4];
 	/**
-	 * @modifier: Data layout modifier, per buffer. This is used to describe
+	 * @modifier: Data layout modifier. This is used to describe
 	 * tiling, or also special layouts (like compression) of auxiliary
 	 * buffers. For userspace created object this is copied from
 	 * drm_mode_fb_cmd2.
 	 */
-	uint64_t modifier[4];
+	uint64_t modifier;
 	/**
 	 * @width: Logical width of the visible area of the framebuffer, in
 	 * pixels.
@@ -251,6 +251,24 @@ static inline uint32_t drm_framebuffer_read_refcount(struct drm_framebuffer *fb)
 }
 
 /**
+ * drm_framebuffer_assign - store a reference to the fb
+ * @p: location to store framebuffer
+ * @fb: new framebuffer (maybe NULL)
+ *
+ * This functions sets the location to store a reference to the framebuffer,
+ * unreferencing the framebuffer that was previously stored in that location.
+ */
+static inline void drm_framebuffer_assign(struct drm_framebuffer **p,
+					  struct drm_framebuffer *fb)
+{
+	if (fb)
+		drm_framebuffer_reference(fb);
+	if (*p)
+		drm_framebuffer_unreference(*p);
+	*p = fb;
+}
+
+/*
  * drm_for_each_fb - iterate over all framebuffers
  * @fb: the loop cursor
  * @dev: the DRM device
diff --git a/include/drm/drm_irq.h b/include/drm/drm_irq.h
index 2401b14d301f..293d08caab60 100644
--- a/include/drm/drm_irq.h
+++ b/include/drm/drm_irq.h
@@ -130,42 +130,37 @@ struct drm_vblank_crtc {
 	bool enabled;
 };
 
-extern int drm_irq_install(struct drm_device *dev, int irq);
-extern int drm_irq_uninstall(struct drm_device *dev);
+int drm_irq_install(struct drm_device *dev, int irq);
+int drm_irq_uninstall(struct drm_device *dev);
 
-extern int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs);
-extern int drm_wait_vblank(struct drm_device *dev, void *data,
-			   struct drm_file *filp);
-extern u32 drm_vblank_count(struct drm_device *dev, unsigned int pipe);
-extern u32 drm_crtc_vblank_count(struct drm_crtc *crtc);
-extern u32 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
-					  struct timeval *vblanktime);
-extern void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
-				       struct drm_pending_vblank_event *e);
-extern void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
-				      struct drm_pending_vblank_event *e);
-extern bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe);
-extern bool drm_crtc_handle_vblank(struct drm_crtc *crtc);
-extern int drm_crtc_vblank_get(struct drm_crtc *crtc);
-extern void drm_crtc_vblank_put(struct drm_crtc *crtc);
-extern void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe);
-extern void drm_crtc_wait_one_vblank(struct drm_crtc *crtc);
-extern void drm_vblank_off(struct drm_device *dev, unsigned int pipe);
-extern void drm_vblank_on(struct drm_device *dev, unsigned int pipe);
-extern void drm_crtc_vblank_off(struct drm_crtc *crtc);
-extern void drm_crtc_vblank_reset(struct drm_crtc *crtc);
-extern void drm_crtc_vblank_on(struct drm_crtc *crtc);
-extern void drm_vblank_cleanup(struct drm_device *dev);
-extern u32 drm_accurate_vblank_count(struct drm_crtc *crtc);
-extern u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe);
+int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs);
+u32 drm_crtc_vblank_count(struct drm_crtc *crtc);
+u32 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
+				   struct timeval *vblanktime);
+void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
+			       struct drm_pending_vblank_event *e);
+void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
+			      struct drm_pending_vblank_event *e);
+bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe);
+bool drm_crtc_handle_vblank(struct drm_crtc *crtc);
+int drm_crtc_vblank_get(struct drm_crtc *crtc);
+void drm_crtc_vblank_put(struct drm_crtc *crtc);
+void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe);
+void drm_crtc_wait_one_vblank(struct drm_crtc *crtc);
+void drm_crtc_vblank_off(struct drm_crtc *crtc);
+void drm_crtc_vblank_reset(struct drm_crtc *crtc);
+void drm_crtc_vblank_on(struct drm_crtc *crtc);
+void drm_vblank_cleanup(struct drm_device *dev);
+u32 drm_accurate_vblank_count(struct drm_crtc *crtc);
+u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe);
 
-extern int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
-						 unsigned int pipe, int *max_error,
-						 struct timeval *vblank_time,
-						 unsigned flags,
-						 const struct drm_display_mode *mode);
-extern void drm_calc_timestamping_constants(struct drm_crtc *crtc,
-					    const struct drm_display_mode *mode);
+int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
+					  unsigned int pipe, int *max_error,
+					  struct timeval *vblank_time,
+					  unsigned flags,
+					  const struct drm_display_mode *mode);
+void drm_calc_timestamping_constants(struct drm_crtc *crtc,
+				     const struct drm_display_mode *mode);
 
 /**
  * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
diff --git a/include/drm/drm_mm.h b/include/drm/drm_mm.h
index 205ddcf6d55d..0b8371795aeb 100644
--- a/include/drm/drm_mm.h
+++ b/include/drm/drm_mm.h
@@ -44,6 +44,9 @@
 #ifdef CONFIG_DEBUG_FS
 #include <linux/seq_file.h>
 #endif
+#ifdef CONFIG_DRM_DEBUG_MM
+#include <linux/stackdepot.h>
+#endif
 
 enum drm_mm_search_flags {
 	DRM_MM_SEARCH_DEFAULT =		0,
@@ -74,6 +77,9 @@ struct drm_mm_node {
 	u64 size;
 	u64 __subtree_last;
 	struct drm_mm *mm;
+#ifdef CONFIG_DRM_DEBUG_MM
+	depot_stack_handle_t stack;
+#endif
 };
 
 struct drm_mm {
@@ -302,10 +308,26 @@ void drm_mm_takedown(struct drm_mm *mm);
 bool drm_mm_clean(struct drm_mm *mm);
 
 struct drm_mm_node *
-drm_mm_interval_first(struct drm_mm *mm, u64 start, u64 last);
+__drm_mm_interval_first(struct drm_mm *mm, u64 start, u64 last);
 
-struct drm_mm_node *
-drm_mm_interval_next(struct drm_mm_node *node, u64 start, u64 last);
+/**
+ * drm_mm_for_each_node_in_range - iterator to walk over a range of
+ * allocated nodes
+ * @node__: drm_mm_node structure to assign to in each iteration step
+ * @mm__: drm_mm allocator to walk
+ * @start__: starting offset, the first node will overlap this
+ * @end__: ending offset, the last node will start before this (but may overlap)
+ *
+ * This iterator walks over all nodes in the range allocator that lie
+ * between @start and @end. It is implemented similarly to list_for_each(),
+ * but using the internal interval tree to accelerate the search for the
+ * starting node, and so not safe against removal of elements. It assumes
+ * that @end is within (or is the upper limit of) the drm_mm allocator.
+ */
+#define drm_mm_for_each_node_in_range(node__, mm__, start__, end__)	\
+	for (node__ = __drm_mm_interval_first((mm__), (start__), (end__)-1); \
+	     node__ && node__->start < (end__);				\
+	     node__ = list_next_entry(node__, node_list))
 
 void drm_mm_init_scan(struct drm_mm *mm,
 		      u64 size,
diff --git a/include/drm/drm_mode_config.h b/include/drm/drm_mode_config.h
new file mode 100644
index 000000000000..bf9991b20611
--- /dev/null
+++ b/include/drm/drm_mode_config.h
@@ -0,0 +1,663 @@
+/*
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#ifndef __DRM_MODE_CONFIG_H__
+#define __DRM_MODE_CONFIG_H__
+
+#include <linux/mutex.h>
+#include <linux/types.h>
+#include <linux/idr.h>
+#include <linux/workqueue.h>
+
+#include <drm/drm_modeset_lock.h>
+
+struct drm_file;
+struct drm_device;
+struct drm_atomic_state;
+struct drm_mode_fb_cmd2;
+
+/**
+ * struct drm_mode_config_funcs - basic driver provided mode setting functions
+ *
+ * Some global (i.e. not per-CRTC, connector, etc) mode setting functions that
+ * involve drivers.
+ */
+struct drm_mode_config_funcs {
+	/**
+	 * @fb_create:
+	 *
+	 * Create a new framebuffer object. The core does basic checks on the
+	 * requested metadata, but most of that is left to the driver. See
+	 * struct &drm_mode_fb_cmd2 for details.
+	 *
+	 * If the parameters are deemed valid and the backing storage objects in
+	 * the underlying memory manager all exist, then the driver allocates
+	 * a new &drm_framebuffer structure, subclassed to contain
+	 * driver-specific information (like the internal native buffer object
+	 * references). It also needs to fill out all relevant metadata, which
+	 * should be done by calling drm_helper_mode_fill_fb_struct().
+	 *
+	 * The initialization is finalized by calling drm_framebuffer_init(),
+	 * which registers the framebuffer and makes it accessible to other
+	 * threads.
+	 *
+	 * RETURNS:
+	 *
+	 * A new framebuffer with an initial reference count of 1 or a negative
+	 * error code encoded with ERR_PTR().
+	 */
+	struct drm_framebuffer *(*fb_create)(struct drm_device *dev,
+					     struct drm_file *file_priv,
+					     const struct drm_mode_fb_cmd2 *mode_cmd);
+
+	/**
+	 * @output_poll_changed:
+	 *
+	 * Callback used by helpers to inform the driver of output configuration
+	 * changes.
+	 *
+	 * Drivers implementing fbdev emulation with the helpers can call
+	 * drm_fb_helper_hotplug_changed from this hook to inform the fbdev
+	 * helper of output changes.
+	 *
+	 * FIXME:
+	 *
+	 * Except that there's no vtable for device-level helper callbacks
+	 * there's no reason this is a core function.
+	 */
+	void (*output_poll_changed)(struct drm_device *dev);
+
+	/**
+	 * @atomic_check:
+	 *
+	 * This is the only hook to validate an atomic modeset update. This
+	 * function must reject any modeset and state changes which the hardware
+	 * or driver doesn't support. This includes but is of course not limited
+	 * to:
+	 *
+	 *  - Checking that the modes, framebuffers, scaling and placement
+	 *    requirements and so on are within the limits of the hardware.
+	 *
+	 *  - Checking that any hidden shared resources are not oversubscribed.
+	 *    This can be shared PLLs, shared lanes, overall memory bandwidth,
+	 *    display fifo space (where shared between planes or maybe even
+	 *    CRTCs).
+	 *
+	 *  - Checking that virtualized resources exported to userspace are not
+	 *    oversubscribed. For various reasons it can make sense to expose
+	 *    more planes, crtcs or encoders than which are physically there. One
+	 *    example is dual-pipe operations (which generally should be hidden
+	 *    from userspace if when lockstepped in hardware, exposed otherwise),
+	 *    where a plane might need 1 hardware plane (if it's just on one
+	 *    pipe), 2 hardware planes (when it spans both pipes) or maybe even
+	 *    shared a hardware plane with a 2nd plane (if there's a compatible
+	 *    plane requested on the area handled by the other pipe).
+	 *
+	 *  - Check that any transitional state is possible and that if
+	 *    requested, the update can indeed be done in the vblank period
+	 *    without temporarily disabling some functions.
+	 *
+	 *  - Check any other constraints the driver or hardware might have.
+	 *
+	 *  - This callback also needs to correctly fill out the &drm_crtc_state
+	 *    in this update to make sure that drm_atomic_crtc_needs_modeset()
+	 *    reflects the nature of the possible update and returns true if and
+	 *    only if the update cannot be applied without tearing within one
+	 *    vblank on that CRTC. The core uses that information to reject
+	 *    updates which require a full modeset (i.e. blanking the screen, or
+	 *    at least pausing updates for a substantial amount of time) if
+	 *    userspace has disallowed that in its request.
+	 *
+	 *  - The driver also does not need to repeat basic input validation
+	 *    like done for the corresponding legacy entry points. The core does
+	 *    that before calling this hook.
+	 *
+	 * See the documentation of @atomic_commit for an exhaustive list of
+	 * error conditions which don't have to be checked at the
+	 * ->atomic_check() stage?
+	 *
+	 * See the documentation for struct &drm_atomic_state for how exactly
+	 * an atomic modeset update is described.
+	 *
+	 * Drivers using the atomic helpers can implement this hook using
+	 * drm_atomic_helper_check(), or one of the exported sub-functions of
+	 * it.
+	 *
+	 * RETURNS:
+	 *
+	 * 0 on success or one of the below negative error codes:
+	 *
+	 *  - -EINVAL, if any of the above constraints are violated.
+	 *
+	 *  - -EDEADLK, when returned from an attempt to acquire an additional
+	 *    &drm_modeset_lock through drm_modeset_lock().
+	 *
+	 *  - -ENOMEM, if allocating additional state sub-structures failed due
+	 *    to lack of memory.
+	 *
+	 *  - -EINTR, -EAGAIN or -ERESTARTSYS, if the IOCTL should be restarted.
+	 *    This can either be due to a pending signal, or because the driver
+	 *    needs to completely bail out to recover from an exceptional
+	 *    situation like a GPU hang. From a userspace point all errors are
+	 *    treated equally.
+	 */
+	int (*atomic_check)(struct drm_device *dev,
+			    struct drm_atomic_state *state);
+
+	/**
+	 * @atomic_commit:
+	 *
+	 * This is the only hook to commit an atomic modeset update. The core
+	 * guarantees that @atomic_check has been called successfully before
+	 * calling this function, and that nothing has been changed in the
+	 * interim.
+	 *
+	 * See the documentation for struct &drm_atomic_state for how exactly
+	 * an atomic modeset update is described.
+	 *
+	 * Drivers using the atomic helpers can implement this hook using
+	 * drm_atomic_helper_commit(), or one of the exported sub-functions of
+	 * it.
+	 *
+	 * Nonblocking commits (as indicated with the nonblock parameter) must
+	 * do any preparatory work which might result in an unsuccessful commit
+	 * in the context of this callback. The only exceptions are hardware
+	 * errors resulting in -EIO. But even in that case the driver must
+	 * ensure that the display pipe is at least running, to avoid
+	 * compositors crashing when pageflips don't work. Anything else,
+	 * specifically committing the update to the hardware, should be done
+	 * without blocking the caller. For updates which do not require a
+	 * modeset this must be guaranteed.
+	 *
+	 * The driver must wait for any pending rendering to the new
+	 * framebuffers to complete before executing the flip. It should also
+	 * wait for any pending rendering from other drivers if the underlying
+	 * buffer is a shared dma-buf. Nonblocking commits must not wait for
+	 * rendering in the context of this callback.
+	 *
+	 * An application can request to be notified when the atomic commit has
+	 * completed. These events are per-CRTC and can be distinguished by the
+	 * CRTC index supplied in &drm_event to userspace.
+	 *
+	 * The drm core will supply a struct &drm_event in the event
+	 * member of each CRTC's &drm_crtc_state structure. See the
+	 * documentation for &drm_crtc_state for more details about the precise
+	 * semantics of this event.
+	 *
+	 * NOTE:
+	 *
+	 * Drivers are not allowed to shut down any display pipe successfully
+	 * enabled through an atomic commit on their own. Doing so can result in
+	 * compositors crashing if a page flip is suddenly rejected because the
+	 * pipe is off.
+	 *
+	 * RETURNS:
+	 *
+	 * 0 on success or one of the below negative error codes:
+	 *
+	 *  - -EBUSY, if a nonblocking updated is requested and there is
+	 *    an earlier updated pending. Drivers are allowed to support a queue
+	 *    of outstanding updates, but currently no driver supports that.
+	 *    Note that drivers must wait for preceding updates to complete if a
+	 *    synchronous update is requested, they are not allowed to fail the
+	 *    commit in that case.
+	 *
+	 *  - -ENOMEM, if the driver failed to allocate memory. Specifically
+	 *    this can happen when trying to pin framebuffers, which must only
+	 *    be done when committing the state.
+	 *
+	 *  - -ENOSPC, as a refinement of the more generic -ENOMEM to indicate
+	 *    that the driver has run out of vram, iommu space or similar GPU
+	 *    address space needed for framebuffer.
+	 *
+	 *  - -EIO, if the hardware completely died.
+	 *
+	 *  - -EINTR, -EAGAIN or -ERESTARTSYS, if the IOCTL should be restarted.
+	 *    This can either be due to a pending signal, or because the driver
+	 *    needs to completely bail out to recover from an exceptional
+	 *    situation like a GPU hang. From a userspace point of view all errors are
+	 *    treated equally.
+	 *
+	 * This list is exhaustive. Specifically this hook is not allowed to
+	 * return -EINVAL (any invalid requests should be caught in
+	 * @atomic_check) or -EDEADLK (this function must not acquire
+	 * additional modeset locks).
+	 */
+	int (*atomic_commit)(struct drm_device *dev,
+			     struct drm_atomic_state *state,
+			     bool nonblock);
+
+	/**
+	 * @atomic_state_alloc:
+	 *
+	 * This optional hook can be used by drivers that want to subclass struct
+	 * &drm_atomic_state to be able to track their own driver-private global
+	 * state easily. If this hook is implemented, drivers must also
+	 * implement @atomic_state_clear and @atomic_state_free.
+	 *
+	 * RETURNS:
+	 *
+	 * A new &drm_atomic_state on success or NULL on failure.
+	 */
+	struct drm_atomic_state *(*atomic_state_alloc)(struct drm_device *dev);
+
+	/**
+	 * @atomic_state_clear:
+	 *
+	 * This hook must clear any driver private state duplicated into the
+	 * passed-in &drm_atomic_state. This hook is called when the caller
+	 * encountered a &drm_modeset_lock deadlock and needs to drop all
+	 * already acquired locks as part of the deadlock avoidance dance
+	 * implemented in drm_modeset_lock_backoff().
+	 *
+	 * Any duplicated state must be invalidated since a concurrent atomic
+	 * update might change it, and the drm atomic interfaces always apply
+	 * updates as relative changes to the current state.
+	 *
+	 * Drivers that implement this must call drm_atomic_state_default_clear()
+	 * to clear common state.
+	 */
+	void (*atomic_state_clear)(struct drm_atomic_state *state);
+
+	/**
+	 * @atomic_state_free:
+	 *
+	 * This hook needs driver private resources and the &drm_atomic_state
+	 * itself. Note that the core first calls drm_atomic_state_clear() to
+	 * avoid code duplicate between the clear and free hooks.
+	 *
+	 * Drivers that implement this must call drm_atomic_state_default_free()
+	 * to release common resources.
+	 */
+	void (*atomic_state_free)(struct drm_atomic_state *state);
+};
+
+/**
+ * struct drm_mode_config - Mode configuration control structure
+ * @mutex: mutex protecting KMS related lists and structures
+ * @connection_mutex: ww mutex protecting connector state and routing
+ * @acquire_ctx: global implicit acquire context used by atomic drivers for
+ * 	legacy IOCTLs
+ * @fb_lock: mutex to protect fb state and lists
+ * @num_fb: number of fbs available
+ * @fb_list: list of framebuffers available
+ * @num_encoder: number of encoders on this device
+ * @encoder_list: list of encoder objects
+ * @num_overlay_plane: number of overlay planes on this device
+ * @num_total_plane: number of universal (i.e. with primary/curso) planes on this device
+ * @plane_list: list of plane objects
+ * @num_crtc: number of CRTCs on this device
+ * @crtc_list: list of CRTC objects
+ * @property_list: list of property objects
+ * @min_width: minimum pixel width on this device
+ * @min_height: minimum pixel height on this device
+ * @max_width: maximum pixel width on this device
+ * @max_height: maximum pixel height on this device
+ * @funcs: core driver provided mode setting functions
+ * @fb_base: base address of the framebuffer
+ * @poll_enabled: track polling support for this device
+ * @poll_running: track polling status for this device
+ * @delayed_event: track delayed poll uevent deliver for this device
+ * @output_poll_work: delayed work for polling in process context
+ * @property_blob_list: list of all the blob property objects
+ * @blob_lock: mutex for blob property allocation and management
+ * @*_property: core property tracking
+ * @preferred_depth: preferred RBG pixel depth, used by fb helpers
+ * @prefer_shadow: hint to userspace to prefer shadow-fb rendering
+ * @cursor_width: hint to userspace for max cursor width
+ * @cursor_height: hint to userspace for max cursor height
+ * @helper_private: mid-layer private data
+ *
+ * Core mode resource tracking structure.  All CRTC, encoders, and connectors
+ * enumerated by the driver are added here, as are global properties.  Some
+ * global restrictions are also here, e.g. dimension restrictions.
+ */
+struct drm_mode_config {
+	struct mutex mutex; /* protects configuration (mode lists etc.) */
+	struct drm_modeset_lock connection_mutex; /* protects connector->encoder and encoder->crtc links */
+	struct drm_modeset_acquire_ctx *acquire_ctx; /* for legacy _lock_all() / _unlock_all() */
+
+	/**
+	 * @idr_mutex:
+	 *
+	 * Mutex for KMS ID allocation and management. Protects both @crtc_idr
+	 * and @tile_idr.
+	 */
+	struct mutex idr_mutex;
+
+	/**
+	 * @crtc_idr:
+	 *
+	 * Main KMS ID tracking object. Use this idr for all IDs, fb, crtc,
+	 * connector, modes - just makes life easier to have only one.
+	 */
+	struct idr crtc_idr;
+
+	/**
+	 * @tile_idr:
+	 *
+	 * Use this idr for allocating new IDs for tiled sinks like use in some
+	 * high-res DP MST screens.
+	 */
+	struct idr tile_idr;
+
+	struct mutex fb_lock; /* proctects global and per-file fb lists */
+	int num_fb;
+	struct list_head fb_list;
+
+	/**
+	 * @num_connector: Number of connectors on this device.
+	 */
+	int num_connector;
+	/**
+	 * @connector_ida: ID allocator for connector indices.
+	 */
+	struct ida connector_ida;
+	/**
+	 * @connector_list: List of connector objects.
+	 */
+	struct list_head connector_list;
+	int num_encoder;
+	struct list_head encoder_list;
+
+	/*
+	 * Track # of overlay planes separately from # of total planes.  By
+	 * default we only advertise overlay planes to userspace; if userspace
+	 * sets the "universal plane" capability bit, we'll go ahead and
+	 * expose all planes.
+	 */
+	int num_overlay_plane;
+	int num_total_plane;
+	struct list_head plane_list;
+
+	int num_crtc;
+	struct list_head crtc_list;
+
+	struct list_head property_list;
+
+	int min_width, min_height;
+	int max_width, max_height;
+	const struct drm_mode_config_funcs *funcs;
+	resource_size_t fb_base;
+
+	/* output poll support */
+	bool poll_enabled;
+	bool poll_running;
+	bool delayed_event;
+	struct delayed_work output_poll_work;
+
+	struct mutex blob_lock;
+
+	/* pointers to standard properties */
+	struct list_head property_blob_list;
+	/**
+	 * @edid_property: Default connector property to hold the EDID of the
+	 * currently connected sink, if any.
+	 */
+	struct drm_property *edid_property;
+	/**
+	 * @dpms_property: Default connector property to control the
+	 * connector's DPMS state.
+	 */
+	struct drm_property *dpms_property;
+	/**
+	 * @path_property: Default connector property to hold the DP MST path
+	 * for the port.
+	 */
+	struct drm_property *path_property;
+	/**
+	 * @tile_property: Default connector property to store the tile
+	 * position of a tiled screen, for sinks which need to be driven with
+	 * multiple CRTCs.
+	 */
+	struct drm_property *tile_property;
+	/**
+	 * @plane_type_property: Default plane property to differentiate
+	 * CURSOR, PRIMARY and OVERLAY legacy uses of planes.
+	 */
+	struct drm_property *plane_type_property;
+	/**
+	 * @prop_src_x: Default atomic plane property for the plane source
+	 * position in the connected &drm_framebuffer.
+	 */
+	struct drm_property *prop_src_x;
+	/**
+	 * @prop_src_y: Default atomic plane property for the plane source
+	 * position in the connected &drm_framebuffer.
+	 */
+	struct drm_property *prop_src_y;
+	/**
+	 * @prop_src_w: Default atomic plane property for the plane source
+	 * position in the connected &drm_framebuffer.
+	 */
+	struct drm_property *prop_src_w;
+	/**
+	 * @prop_src_h: Default atomic plane property for the plane source
+	 * position in the connected &drm_framebuffer.
+	 */
+	struct drm_property *prop_src_h;
+	/**
+	 * @prop_crtc_x: Default atomic plane property for the plane destination
+	 * position in the &drm_crtc is is being shown on.
+	 */
+	struct drm_property *prop_crtc_x;
+	/**
+	 * @prop_crtc_y: Default atomic plane property for the plane destination
+	 * position in the &drm_crtc is is being shown on.
+	 */
+	struct drm_property *prop_crtc_y;
+	/**
+	 * @prop_crtc_w: Default atomic plane property for the plane destination
+	 * position in the &drm_crtc is is being shown on.
+	 */
+	struct drm_property *prop_crtc_w;
+	/**
+	 * @prop_crtc_h: Default atomic plane property for the plane destination
+	 * position in the &drm_crtc is is being shown on.
+	 */
+	struct drm_property *prop_crtc_h;
+	/**
+	 * @prop_fb_id: Default atomic plane property to specify the
+	 * &drm_framebuffer.
+	 */
+	struct drm_property *prop_fb_id;
+	/**
+	 * @prop_in_fence_fd: Sync File fd representing the incoming fences
+	 * for a Plane.
+	 */
+	struct drm_property *prop_in_fence_fd;
+	/**
+	 * @prop_out_fence_ptr: Sync File fd pointer representing the
+	 * outgoing fences for a CRTC. Userspace should provide a pointer to a
+	 * value of type s64, and then cast that pointer to u64.
+	 */
+	struct drm_property *prop_out_fence_ptr;
+	/**
+	 * @prop_crtc_id: Default atomic plane property to specify the
+	 * &drm_crtc.
+	 */
+	struct drm_property *prop_crtc_id;
+	/**
+	 * @prop_active: Default atomic CRTC property to control the active
+	 * state, which is the simplified implementation for DPMS in atomic
+	 * drivers.
+	 */
+	struct drm_property *prop_active;
+	/**
+	 * @prop_mode_id: Default atomic CRTC property to set the mode for a
+	 * CRTC. A 0 mode implies that the CRTC is entirely disabled - all
+	 * connectors must be of and active must be set to disabled, too.
+	 */
+	struct drm_property *prop_mode_id;
+
+	/**
+	 * @dvi_i_subconnector_property: Optional DVI-I property to
+	 * differentiate between analog or digital mode.
+	 */
+	struct drm_property *dvi_i_subconnector_property;
+	/**
+	 * @dvi_i_select_subconnector_property: Optional DVI-I property to
+	 * select between analog or digital mode.
+	 */
+	struct drm_property *dvi_i_select_subconnector_property;
+
+	/**
+	 * @tv_subconnector_property: Optional TV property to differentiate
+	 * between different TV connector types.
+	 */
+	struct drm_property *tv_subconnector_property;
+	/**
+	 * @tv_select_subconnector_property: Optional TV property to select
+	 * between different TV connector types.
+	 */
+	struct drm_property *tv_select_subconnector_property;
+	/**
+	 * @tv_mode_property: Optional TV property to select
+	 * the output TV mode.
+	 */
+	struct drm_property *tv_mode_property;
+	/**
+	 * @tv_left_margin_property: Optional TV property to set the left
+	 * margin.
+	 */
+	struct drm_property *tv_left_margin_property;
+	/**
+	 * @tv_right_margin_property: Optional TV property to set the right
+	 * margin.
+	 */
+	struct drm_property *tv_right_margin_property;
+	/**
+	 * @tv_top_margin_property: Optional TV property to set the right
+	 * margin.
+	 */
+	struct drm_property *tv_top_margin_property;
+	/**
+	 * @tv_bottom_margin_property: Optional TV property to set the right
+	 * margin.
+	 */
+	struct drm_property *tv_bottom_margin_property;
+	/**
+	 * @tv_brightness_property: Optional TV property to set the
+	 * brightness.
+	 */
+	struct drm_property *tv_brightness_property;
+	/**
+	 * @tv_contrast_property: Optional TV property to set the
+	 * contrast.
+	 */
+	struct drm_property *tv_contrast_property;
+	/**
+	 * @tv_flicker_reduction_property: Optional TV property to control the
+	 * flicker reduction mode.
+	 */
+	struct drm_property *tv_flicker_reduction_property;
+	/**
+	 * @tv_overscan_property: Optional TV property to control the overscan
+	 * setting.
+	 */
+	struct drm_property *tv_overscan_property;
+	/**
+	 * @tv_saturation_property: Optional TV property to set the
+	 * saturation.
+	 */
+	struct drm_property *tv_saturation_property;
+	/**
+	 * @tv_hue_property: Optional TV property to set the hue.
+	 */
+	struct drm_property *tv_hue_property;
+
+	/**
+	 * @scaling_mode_property: Optional connector property to control the
+	 * upscaling, mostly used for built-in panels.
+	 */
+	struct drm_property *scaling_mode_property;
+	/**
+	 * @aspect_ratio_property: Optional connector property to control the
+	 * HDMI infoframe aspect ratio setting.
+	 */
+	struct drm_property *aspect_ratio_property;
+	/**
+	 * @degamma_lut_property: Optional CRTC property to set the LUT used to
+	 * convert the framebuffer's colors to linear gamma.
+	 */
+	struct drm_property *degamma_lut_property;
+	/**
+	 * @degamma_lut_size_property: Optional CRTC property for the size of
+	 * the degamma LUT as supported by the driver (read-only).
+	 */
+	struct drm_property *degamma_lut_size_property;
+	/**
+	 * @ctm_property: Optional CRTC property to set the
+	 * matrix used to convert colors after the lookup in the
+	 * degamma LUT.
+	 */
+	struct drm_property *ctm_property;
+	/**
+	 * @gamma_lut_property: Optional CRTC property to set the LUT used to
+	 * convert the colors, after the CTM matrix, to the gamma space of the
+	 * connected screen.
+	 */
+	struct drm_property *gamma_lut_property;
+	/**
+	 * @gamma_lut_size_property: Optional CRTC property for the size of the
+	 * gamma LUT as supported by the driver (read-only).
+	 */
+	struct drm_property *gamma_lut_size_property;
+
+	/**
+	 * @suggested_x_property: Optional connector property with a hint for
+	 * the position of the output on the host's screen.
+	 */
+	struct drm_property *suggested_x_property;
+	/**
+	 * @suggested_y_property: Optional connector property with a hint for
+	 * the position of the output on the host's screen.
+	 */
+	struct drm_property *suggested_y_property;
+
+	/* dumb ioctl parameters */
+	uint32_t preferred_depth, prefer_shadow;
+
+	/**
+	 * @async_page_flip: Does this device support async flips on the primary
+	 * plane?
+	 */
+	bool async_page_flip;
+
+	/**
+	 * @allow_fb_modifiers:
+	 *
+	 * Whether the driver supports fb modifiers in the ADDFB2.1 ioctl call.
+	 */
+	bool allow_fb_modifiers;
+
+	/* cursor size */
+	uint32_t cursor_width, cursor_height;
+
+	struct drm_mode_config_helper_funcs *helper_private;
+};
+
+void drm_mode_config_init(struct drm_device *dev);
+void drm_mode_config_reset(struct drm_device *dev);
+void drm_mode_config_cleanup(struct drm_device *dev);
+
+#endif
diff --git a/include/drm/drm_modeset_helper_vtables.h b/include/drm/drm_modeset_helper_vtables.h
index 10e449c86dbd..69c3974bf133 100644
--- a/include/drm/drm_modeset_helper_vtables.h
+++ b/include/drm/drm_modeset_helper_vtables.h
@@ -361,8 +361,8 @@ struct drm_crtc_helper_funcs {
 	 *
 	 * Note that the power state of the display pipe when this function is
 	 * called depends upon the exact helpers and calling sequence the driver
-	 * has picked. See drm_atomic_commit_planes() for a discussion of the
-	 * tradeoffs and variants of plane commit helpers.
+	 * has picked. See drm_atomic_helper_commit_planes() for a discussion of
+	 * the tradeoffs and variants of plane commit helpers.
 	 *
 	 * This callback is used by the atomic modeset helpers and by the
 	 * transitional plane helpers, but it is optional.
@@ -385,8 +385,8 @@ struct drm_crtc_helper_funcs {
 	 *
 	 * Note that the power state of the display pipe when this function is
 	 * called depends upon the exact helpers and calling sequence the driver
-	 * has picked. See drm_atomic_commit_planes() for a discussion of the
-	 * tradeoffs and variants of plane commit helpers.
+	 * has picked. See drm_atomic_helper_commit_planes() for a discussion of
+	 * the tradeoffs and variants of plane commit helpers.
 	 *
 	 * This callback is used by the atomic modeset helpers and by the
 	 * transitional plane helpers, but it is optional.
@@ -940,8 +940,8 @@ struct drm_plane_helper_funcs {
 	 *
 	 * Note that the power state of the display pipe when this function is
 	 * called depends upon the exact helpers and calling sequence the driver
-	 * has picked. See drm_atomic_commit_planes() for a discussion of the
-	 * tradeoffs and variants of plane commit helpers.
+	 * has picked. See drm_atomic_helper_commit_planes() for a discussion of
+	 * the tradeoffs and variants of plane commit helpers.
 	 *
 	 * This callback is used by the atomic modeset helpers and by the
 	 * transitional plane helpers, but it is optional.
@@ -963,8 +963,8 @@ struct drm_plane_helper_funcs {
 	 *
 	 * Note that the power state of the display pipe when this function is
 	 * called depends upon the exact helpers and calling sequence the driver
-	 * has picked. See drm_atomic_commit_planes() for a discussion of the
-	 * tradeoffs and variants of plane commit helpers.
+	 * has picked. See drm_atomic_helper_commit_planes() for a discussion of
+	 * the tradeoffs and variants of plane commit helpers.
 	 *
 	 * This callback is used by the atomic modeset helpers and by the
 	 * transitional plane helpers, but it is optional.
@@ -999,10 +999,14 @@ struct drm_mode_config_helper_funcs {
 	 * to implement blocking and nonblocking commits easily. It is not used
 	 * by the atomic helpers
 	 *
-	 * This hook should first commit the given atomic state to the hardware.
-	 * But drivers can add more waiting calls at the start of their
-	 * implementation, e.g. to wait for driver-internal request for implicit
-	 * syncing, before starting to commit the update to the hardware.
+	 * This function is called when the new atomic state has already been
+	 * swapped into the various state pointers. The passed in state
+	 * therefore contains copies of the old/previous state. This hook should
+	 * commit the new state into hardware. Note that the helpers have
+	 * already waited for preceeding atomic commits and fences, but drivers
+	 * can add more waiting calls at the start of their implementation, e.g.
+	 * to wait for driver-internal request for implicit syncing, before
+	 * starting to commit the update to the hardware.
 	 *
 	 * After the atomic update is committed to the hardware this hook needs
 	 * to call drm_atomic_helper_commit_hw_done(). Then wait for the upate
diff --git a/include/drm/drm_modeset_lock.h b/include/drm/drm_modeset_lock.h
index c5576fbcb909..d918ce45ec2c 100644
--- a/include/drm/drm_modeset_lock.h
+++ b/include/drm/drm_modeset_lock.h
@@ -82,8 +82,6 @@ struct drm_modeset_lock {
 	struct list_head head;
 };
 
-extern struct ww_class crtc_ww_class;
-
 void drm_modeset_acquire_init(struct drm_modeset_acquire_ctx *ctx,
 		uint32_t flags);
 void drm_modeset_acquire_fini(struct drm_modeset_acquire_ctx *ctx);
@@ -91,15 +89,7 @@ void drm_modeset_drop_locks(struct drm_modeset_acquire_ctx *ctx);
 void drm_modeset_backoff(struct drm_modeset_acquire_ctx *ctx);
 int drm_modeset_backoff_interruptible(struct drm_modeset_acquire_ctx *ctx);
 
-/**
- * drm_modeset_lock_init - initialize lock
- * @lock: lock to init
- */
-static inline void drm_modeset_lock_init(struct drm_modeset_lock *lock)
-{
-	ww_mutex_init(&lock->mutex, &crtc_ww_class);
-	INIT_LIST_HEAD(&lock->head);
-}
+void drm_modeset_lock_init(struct drm_modeset_lock *lock);
 
 /**
  * drm_modeset_lock_fini - cleanup lock
diff --git a/include/drm/drm_of.h b/include/drm/drm_of.h
index 3fd87b386ed7..26a64805cc15 100644
--- a/include/drm/drm_of.h
+++ b/include/drm/drm_of.h
@@ -4,6 +4,7 @@
 #include <linux/of_graph.h>
 
 struct component_master_ops;
+struct component_match;
 struct device;
 struct drm_device;
 struct drm_encoder;
@@ -12,6 +13,10 @@ struct device_node;
 #ifdef CONFIG_OF
 extern uint32_t drm_of_find_possible_crtcs(struct drm_device *dev,
 					   struct device_node *port);
+extern void drm_of_component_match_add(struct device *master,
+				       struct component_match **matchptr,
+				       int (*compare)(struct device *, void *),
+				       struct device_node *node);
 extern int drm_of_component_probe(struct device *dev,
 				  int (*compare_of)(struct device *, void *),
 				  const struct component_master_ops *m_ops);
@@ -25,6 +30,14 @@ static inline uint32_t drm_of_find_possible_crtcs(struct drm_device *dev,
 	return 0;
 }
 
+static inline void
+drm_of_component_match_add(struct device *master,
+			   struct component_match **matchptr,
+			   int (*compare)(struct device *, void *),
+			   struct device_node *node)
+{
+}
+
 static inline int
 drm_of_component_probe(struct device *dev,
 		       int (*compare_of)(struct device *, void *),
diff --git a/include/drm/drm_plane.h b/include/drm/drm_plane.h
index 8b4dc62470ff..db3bbdeb36d5 100644
--- a/include/drm/drm_plane.h
+++ b/include/drm/drm_plane.h
@@ -28,15 +28,11 @@
 #include <drm/drm_mode_object.h>
 
 struct drm_crtc;
+struct drm_printer;
 
 /**
  * struct drm_plane_state - mutable plane state
  * @plane: backpointer to the plane
- * @crtc: currently bound CRTC, NULL if disabled
- * @fb: currently bound framebuffer
- * @fence: optional fence to wait for before scanning out @fb
- * @crtc_x: left position of visible portion of plane on crtc
- * @crtc_y: upper position of visible portion of plane on crtc
  * @crtc_w: width of visible portion of plane on crtc
  * @crtc_h: height of visible portion of plane on crtc
  * @src_x: left position of visible portion of plane within
@@ -57,18 +53,51 @@ struct drm_crtc;
  *	it can be trusted.
  * @src: clipped source coordinates of the plane (in 16.16)
  * @dst: clipped destination coordinates of the plane
- * @visible: visibility of the plane
  * @state: backpointer to global drm_atomic_state
  */
 struct drm_plane_state {
 	struct drm_plane *plane;
 
-	struct drm_crtc *crtc;   /* do not write directly, use drm_atomic_set_crtc_for_plane() */
-	struct drm_framebuffer *fb;  /* do not write directly, use drm_atomic_set_fb_for_plane() */
-	struct fence *fence;
+	/**
+	 * @crtc:
+	 *
+	 * Currently bound CRTC, NULL if disabled. Do not this write directly,
+	 * use drm_atomic_set_crtc_for_plane()
+	 */
+	struct drm_crtc *crtc;
+
+	/**
+	 * @fb:
+	 *
+	 * Currently bound framebuffer. Do not write this directly, use
+	 * drm_atomic_set_fb_for_plane()
+	 */
+	struct drm_framebuffer *fb;
+
+	/**
+	 * @fence:
+	 *
+	 * Optional fence to wait for before scanning out @fb. Do not write this
+	 * directly, use drm_atomic_set_fence_for_plane()
+	 */
+	struct dma_fence *fence;
+
+	/**
+	 * @crtc_x:
+	 *
+	 * Left position of visible portion of plane on crtc, signed dest
+	 * location allows it to be partially off screen.
+	 */
+
+	int32_t crtc_x;
+	/**
+	 * @crtc_y:
+	 *
+	 * Upper position of visible portion of plane on crtc, signed dest
+	 * location allows it to be partially off screen.
+	 */
+	int32_t crtc_y;
 
-	/* Signed dest location allows it to be partially off screen */
-	int32_t crtc_x, crtc_y;
 	uint32_t crtc_w, crtc_h;
 
 	/* Source values are 16.16 fixed point */
@@ -85,15 +114,40 @@ struct drm_plane_state {
 	/* Clipped coordinates */
 	struct drm_rect src, dst;
 
-	/*
-	 * Is the plane actually visible? Can be false even
-	 * if fb!=NULL and crtc!=NULL, due to clipping.
+	/**
+	 * @visible:
+	 *
+	 * Visibility of the plane. This can be false even if fb!=NULL and
+	 * crtc!=NULL, due to clipping.
 	 */
 	bool visible;
 
 	struct drm_atomic_state *state;
 };
 
+static inline struct drm_rect
+drm_plane_state_src(const struct drm_plane_state *state)
+{
+	struct drm_rect src = {
+		.x1 = state->src_x,
+		.y1 = state->src_y,
+		.x2 = state->src_x + state->src_w,
+		.y2 = state->src_y + state->src_h,
+	};
+	return src;
+}
+
+static inline struct drm_rect
+drm_plane_state_dest(const struct drm_plane_state *state)
+{
+	struct drm_rect dest = {
+		.x1 = state->crtc_x,
+		.y1 = state->crtc_y,
+		.x2 = state->crtc_x + state->crtc_w,
+		.y2 = state->crtc_y + state->crtc_h,
+	};
+	return dest;
+}
 
 /**
  * struct drm_plane_funcs - driver plane control functions
@@ -323,6 +377,18 @@ struct drm_plane_funcs {
 	 * before data structures are torndown.
 	 */
 	void (*early_unregister)(struct drm_plane *plane);
+
+	/**
+	 * @atomic_print_state:
+	 *
+	 * If driver subclasses struct &drm_plane_state, it should implement
+	 * this optional hook for printing additional driver specific state.
+	 *
+	 * Do not call this directly, use drm_atomic_plane_print_state()
+	 * instead.
+	 */
+	void (*atomic_print_state)(struct drm_printer *p,
+				   const struct drm_plane_state *state);
 };
 
 /**
@@ -392,6 +458,7 @@ enum drm_plane_type {
  * @type: type of plane (overlay, primary, cursor)
  * @state: current atomic state for this plane
  * @zpos_property: zpos property for this plane
+ * @rotation_property: rotation property for this plane
  * @helper_private: mid-layer private data
  */
 struct drm_plane {
@@ -438,6 +505,7 @@ struct drm_plane {
 	struct drm_plane_state *state;
 
 	struct drm_property *zpos_property;
+	struct drm_property *rotation_property;
 };
 
 #define obj_to_plane(x) container_of(x, struct drm_plane, base)
@@ -445,7 +513,7 @@ struct drm_plane {
 extern __printf(8, 9)
 int drm_universal_plane_init(struct drm_device *dev,
 			     struct drm_plane *plane,
-			     unsigned long possible_crtcs,
+			     uint32_t possible_crtcs,
 			     const struct drm_plane_funcs *funcs,
 			     const uint32_t *formats,
 			     unsigned int format_count,
@@ -453,7 +521,7 @@ int drm_universal_plane_init(struct drm_device *dev,
 			     const char *name, ...);
 extern int drm_plane_init(struct drm_device *dev,
 			  struct drm_plane *plane,
-			  unsigned long possible_crtcs,
+			  uint32_t possible_crtcs,
 			  const struct drm_plane_funcs *funcs,
 			  const uint32_t *formats, unsigned int format_count,
 			  bool is_primary);
diff --git a/include/drm/drm_print.h b/include/drm/drm_print.h
new file mode 100644
index 000000000000..1adf84aea622
--- /dev/null
+++ b/include/drm/drm_print.h
@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2016 Red Hat
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ * Rob Clark <robdclark@gmail.com>
+ */
+
+#ifndef DRM_PRINT_H_
+#define DRM_PRINT_H_
+
+#include <linux/seq_file.h>
+#include <linux/device.h>
+
+/**
+ * DOC: print
+ *
+ * A simple wrapper for dev_printk(), seq_printf(), etc.  Allows same
+ * debug code to be used for both debugfs and printk logging.
+ *
+ * For example::
+ *
+ *     void log_some_info(struct drm_printer *p)
+ *     {
+ *             drm_printf(p, "foo=%d\n", foo);
+ *             drm_printf(p, "bar=%d\n", bar);
+ *     }
+ *
+ *     #ifdef CONFIG_DEBUG_FS
+ *     void debugfs_show(struct seq_file *f)
+ *     {
+ *             struct drm_printer p = drm_seq_file_printer(f);
+ *             log_some_info(&p);
+ *     }
+ *     #endif
+ *
+ *     void some_other_function(...)
+ *     {
+ *             struct drm_printer p = drm_info_printer(drm->dev);
+ *             log_some_info(&p);
+ *     }
+ */
+
+/**
+ * struct drm_printer - drm output "stream"
+ * @printfn: actual output fxn
+ * @arg: output fxn specific data
+ *
+ * Do not use struct members directly.  Use drm_printer_seq_file(),
+ * drm_printer_info(), etc to initialize.  And drm_printf() for output.
+ */
+struct drm_printer {
+	void (*printfn)(struct drm_printer *p, struct va_format *vaf);
+	void *arg;
+};
+
+void __drm_printfn_seq_file(struct drm_printer *p, struct va_format *vaf);
+void __drm_printfn_info(struct drm_printer *p, struct va_format *vaf);
+
+void drm_printf(struct drm_printer *p, const char *f, ...);
+
+
+/**
+ * drm_seq_file_printer - construct a &drm_printer that outputs to &seq_file
+ * @f:  the struct &seq_file to output to
+ *
+ * RETURNS:
+ * The &drm_printer object
+ */
+static inline struct drm_printer drm_seq_file_printer(struct seq_file *f)
+{
+	struct drm_printer p = {
+		.printfn = __drm_printfn_seq_file,
+		.arg = f,
+	};
+	return p;
+}
+
+/**
+ * drm_info_printer - construct a &drm_printer that outputs to dev_printk()
+ * @dev: the struct &device pointer
+ *
+ * RETURNS:
+ * The &drm_printer object
+ */
+static inline struct drm_printer drm_info_printer(struct device *dev)
+{
+	struct drm_printer p = {
+		.printfn = __drm_printfn_info,
+		.arg = dev,
+	};
+	return p;
+}
+
+#endif /* DRM_PRINT_H_ */
diff --git a/include/drm/i915_component.h b/include/drm/i915_component.h
index b46fa0ef3005..545c6e0fea7d 100644
--- a/include/drm/i915_component.h
+++ b/include/drm/i915_component.h
@@ -64,7 +64,7 @@ struct i915_audio_component_ops {
 	 * Called from audio driver. After audio driver sets the
 	 * sample rate, it will call this function to set n/cts
 	 */
-	int (*sync_audio_rate)(struct device *, int port, int rate);
+	int (*sync_audio_rate)(struct device *, int port, int pipe, int rate);
 	/**
 	 * @get_eld: fill the audio state and ELD bytes for the given port
 	 *
@@ -77,7 +77,7 @@ struct i915_audio_component_ops {
 	 * Note that the returned size may be over @max_bytes.  Then it
 	 * implies that only a part of ELD has been copied to the buffer.
 	 */
-	int (*get_eld)(struct device *, int port, bool *enabled,
+	int (*get_eld)(struct device *, int port, int pipe, bool *enabled,
 		       unsigned char *buf, int max_bytes);
 };
 
@@ -97,7 +97,7 @@ struct i915_audio_component_audio_ops {
 	 * status accordingly (even when the HDA controller is in power save
 	 * mode).
 	 */
-	void (*pin_eld_notify)(void *audio_ptr, int port);
+	void (*pin_eld_notify)(void *audio_ptr, int port, int pipe);
 };
 
 /**
diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h
index 9eb940d6755f..652e45be97c8 100644
--- a/include/drm/ttm/ttm_bo_api.h
+++ b/include/drm/ttm/ttm_bo_api.h
@@ -47,6 +47,8 @@ struct drm_mm_node;
 
 struct ttm_placement;
 
+struct ttm_place;
+
 /**
  * struct ttm_bus_placement
  *
@@ -209,7 +211,7 @@ struct ttm_buffer_object {
 	 * Members protected by a bo reservation.
 	 */
 
-	struct fence *moving;
+	struct dma_fence *moving;
 
 	struct drm_vma_offset_node vma_node;
 
@@ -396,6 +398,17 @@ extern void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev,
 					    int resched);
 
 /**
+ * ttm_bo_eviction_valuable
+ *
+ * @bo: The buffer object to evict
+ * @place: the placement we need to make room for
+ *
+ * Check if it is valuable to evict the BO to make room for the given placement.
+ */
+bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
+			      const struct ttm_place *place);
+
+/**
  * ttm_bo_synccpu_write_grab
  *
  * @bo: The buffer object:
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index 4f0a92185995..cdbdb40eb5bd 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -303,7 +303,7 @@ struct ttm_mem_type_manager {
 	/*
 	 * Protected by @move_lock.
 	 */
-	struct fence *move;
+	struct dma_fence *move;
 };
 
 /**
@@ -371,9 +371,21 @@ struct ttm_bo_driver {
 	 * submission as a consequence.
 	 */
 
-	int (*invalidate_caches) (struct ttm_bo_device *bdev, uint32_t flags);
-	int (*init_mem_type) (struct ttm_bo_device *bdev, uint32_t type,
-			      struct ttm_mem_type_manager *man);
+	int (*invalidate_caches)(struct ttm_bo_device *bdev, uint32_t flags);
+	int (*init_mem_type)(struct ttm_bo_device *bdev, uint32_t type,
+			     struct ttm_mem_type_manager *man);
+
+	/**
+	 * struct ttm_bo_driver member eviction_valuable
+	 *
+	 * @bo: the buffer object to be evicted
+	 * @place: placement we need room for
+	 *
+	 * Check with the driver if it is valuable to evict a BO to make room
+	 * for a certain placement.
+	 */
+	bool (*eviction_valuable)(struct ttm_buffer_object *bo,
+				  const struct ttm_place *place);
 	/**
 	 * struct ttm_bo_driver member evict_flags:
 	 *
@@ -384,8 +396,9 @@ struct ttm_bo_driver {
 	 * finished, they'll end up in bo->mem.flags
 	 */
 
-	 void(*evict_flags) (struct ttm_buffer_object *bo,
-				struct ttm_placement *placement);
+	void (*evict_flags)(struct ttm_buffer_object *bo,
+			    struct ttm_placement *placement);
+
 	/**
 	 * struct ttm_bo_driver member move:
 	 *
@@ -399,10 +412,9 @@ struct ttm_bo_driver {
 	 *
 	 * Move a buffer between two memory regions.
 	 */
-	int (*move) (struct ttm_buffer_object *bo,
-		     bool evict, bool interruptible,
-		     bool no_wait_gpu,
-		     struct ttm_mem_reg *new_mem);
+	int (*move)(struct ttm_buffer_object *bo, bool evict,
+		    bool interruptible, bool no_wait_gpu,
+		    struct ttm_mem_reg *new_mem);
 
 	/**
 	 * struct ttm_bo_driver_member verify_access
@@ -416,8 +428,8 @@ struct ttm_bo_driver {
 	 * access for all buffer objects.
 	 * This function should return 0 if access is granted, -EPERM otherwise.
 	 */
-	int (*verify_access) (struct ttm_buffer_object *bo,
-			      struct file *filp);
+	int (*verify_access)(struct ttm_buffer_object *bo,
+			     struct file *filp);
 
 	/* hook to notify driver about a driver move so it
 	 * can do tiling things */
@@ -430,7 +442,7 @@ struct ttm_bo_driver {
 	/**
 	 * notify the driver that we're about to swap out this bo
 	 */
-	void (*swap_notify) (struct ttm_buffer_object *bo);
+	void (*swap_notify)(struct ttm_buffer_object *bo);
 
 	/**
 	 * Driver callback on when mapping io memory (for bo_move_memcpy
@@ -438,8 +450,10 @@ struct ttm_bo_driver {
 	 * the mapping is not use anymore. io_mem_reserve & io_mem_free
 	 * are balanced.
 	 */
-	int (*io_mem_reserve)(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem);
-	void (*io_mem_free)(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem);
+	int (*io_mem_reserve)(struct ttm_bo_device *bdev,
+			      struct ttm_mem_reg *mem);
+	void (*io_mem_free)(struct ttm_bo_device *bdev,
+			    struct ttm_mem_reg *mem);
 
 	/**
 	 * Optional driver callback for when BO is removed from the LRU.
@@ -1025,7 +1039,7 @@ extern void ttm_bo_free_old_node(struct ttm_buffer_object *bo);
  */
 
 extern int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
-				     struct fence *fence, bool evict,
+				     struct dma_fence *fence, bool evict,
 				     struct ttm_mem_reg *new_mem);
 
 /**
@@ -1040,7 +1054,7 @@ extern int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
  * immediately or hang it on a temporary buffer object.
  */
 int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
-			 struct fence *fence, bool evict,
+			 struct dma_fence *fence, bool evict,
 			 struct ttm_mem_reg *new_mem);
 
 /**
diff --git a/include/drm/ttm/ttm_execbuf_util.h b/include/drm/ttm/ttm_execbuf_util.h
index b620c317c772..47f35b8e6d09 100644
--- a/include/drm/ttm/ttm_execbuf_util.h
+++ b/include/drm/ttm/ttm_execbuf_util.h
@@ -114,6 +114,6 @@ extern int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket,
 
 extern void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket,
 					struct list_head *list,
-					struct fence *fence);
+					struct dma_fence *fence);
 
 #endif
diff --git a/include/dt-bindings/clock/hi3516cv300-clock.h b/include/dt-bindings/clock/hi3516cv300-clock.h
new file mode 100644
index 000000000000..5ba51b8388fc
--- /dev/null
+++ b/include/dt-bindings/clock/hi3516cv300-clock.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2016 HiSilicon Technologies Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __DTS_HI3516CV300_CLOCK_H
+#define __DTS_HI3516CV300_CLOCK_H
+
+/* hi3516CV300 core CRG */
+#define HI3516CV300_APB_CLK		0
+#define HI3516CV300_UART0_CLK		1
+#define HI3516CV300_UART1_CLK		2
+#define HI3516CV300_UART2_CLK		3
+#define HI3516CV300_SPI0_CLK		4
+#define HI3516CV300_SPI1_CLK		5
+#define HI3516CV300_FMC_CLK		6
+#define HI3516CV300_MMC0_CLK		7
+#define HI3516CV300_MMC1_CLK		8
+#define HI3516CV300_MMC2_CLK		9
+#define HI3516CV300_MMC3_CLK		10
+#define HI3516CV300_ETH_CLK		11
+#define HI3516CV300_ETH_MACIF_CLK	12
+#define HI3516CV300_DMAC_CLK		13
+#define HI3516CV300_PWM_CLK		14
+#define HI3516CV300_USB2_BUS_CLK	15
+#define HI3516CV300_USB2_OHCI48M_CLK	16
+#define HI3516CV300_USB2_OHCI12M_CLK	17
+#define HI3516CV300_USB2_OTG_UTMI_CLK	18
+#define HI3516CV300_USB2_HST_PHY_CLK	19
+#define HI3516CV300_USB2_UTMI0_CLK	20
+#define HI3516CV300_USB2_PHY_CLK	21
+
+/* hi3516CV300 sysctrl CRG */
+#define HI3516CV300_WDT_CLK		1
+
+#endif	/* __DTS_HI3516CV300_CLOCK_H */
diff --git a/include/dt-bindings/clock/histb-clock.h b/include/dt-bindings/clock/histb-clock.h
new file mode 100644
index 000000000000..181c0f070f7c
--- /dev/null
+++ b/include/dt-bindings/clock/histb-clock.h
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2016 HiSilicon Technologies Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __DTS_HISTB_CLOCK_H
+#define __DTS_HISTB_CLOCK_H
+
+/* clocks provided by core CRG */
+#define HISTB_OSC_CLK			0
+#define HISTB_APB_CLK			1
+#define HISTB_AHB_CLK			2
+#define HISTB_UART1_CLK		3
+#define HISTB_UART2_CLK		4
+#define HISTB_UART3_CLK		5
+#define HISTB_I2C0_CLK		6
+#define HISTB_I2C1_CLK		7
+#define HISTB_I2C2_CLK		8
+#define HISTB_I2C3_CLK		9
+#define HISTB_I2C4_CLK		10
+#define HISTB_I2C5_CLK		11
+#define HISTB_SPI0_CLK		12
+#define HISTB_SPI1_CLK		13
+#define HISTB_SPI2_CLK		14
+#define HISTB_SCI_CLK			15
+#define HISTB_FMC_CLK			16
+#define HISTB_MMC_BIU_CLK		17
+#define HISTB_MMC_CIU_CLK		18
+#define HISTB_MMC_DRV_CLK		19
+#define HISTB_MMC_SAMPLE_CLK		20
+#define HISTB_SDIO0_BIU_CLK		21
+#define HISTB_SDIO0_CIU_CLK		22
+#define HISTB_SDIO0_DRV_CLK		23
+#define HISTB_SDIO0_SAMPLE_CLK	24
+#define HISTB_PCIE_AUX_CLK		25
+#define HISTB_PCIE_PIPE_CLK		26
+#define HISTB_PCIE_SYS_CLK		27
+#define HISTB_PCIE_BUS_CLK		28
+#define HISTB_ETH0_MAC_CLK		29
+#define HISTB_ETH0_MACIF_CLK		30
+#define HISTB_ETH1_MAC_CLK		31
+#define HISTB_ETH1_MACIF_CLK		32
+#define HISTB_COMBPHY1_CLK		33
+
+
+/* clocks provided by mcu CRG */
+#define HISTB_MCE_CLK	1
+#define HISTB_IR_CLK	2
+#define HISTB_TIMER01_CLK	3
+#define HISTB_LEDC_CLK	4
+#define HISTB_UART0_CLK	5
+#define HISTB_LSADC_CLK	6
+
+#endif	/* __DTS_HISTB_CLOCK_H */
diff --git a/include/dt-bindings/clock/imx6ul-clock.h b/include/dt-bindings/clock/imx6ul-clock.h
index fd8aee8f64ae..ee9f1a508d2f 100644
--- a/include/dt-bindings/clock/imx6ul-clock.h
+++ b/include/dt-bindings/clock/imx6ul-clock.h
@@ -236,6 +236,19 @@
 #define IMX6UL_CLK_PLL3_120M		223
 #define IMX6UL_CLK_KPP			224
 
-#define IMX6UL_CLK_END			225
+/* For i.MX6ULL */
+#define IMX6ULL_CLK_ESAI_PRED		225
+#define IMX6ULL_CLK_ESAI_PODF		226
+#define IMX6ULL_CLK_ESAI_EXTAL		227
+#define IMX6ULL_CLK_ESAI_MEM		228
+#define IMX6ULL_CLK_ESAI_IPG		229
+#define IMX6ULL_CLK_DCP_CLK		230
+#define IMX6ULL_CLK_EPDC_PRE_SEL	231
+#define IMX6ULL_CLK_EPDC_SEL		232
+#define IMX6ULL_CLK_EPDC_PODF		233
+#define IMX6ULL_CLK_EPDC_ACLK		234
+#define IMX6ULL_CLK_EPDC_PIX		235
+#define IMX6ULL_CLK_ESAI_SEL		236
+#define IMX6UL_CLK_END			237
 
 #endif /* __DT_BINDINGS_CLOCK_IMX6UL_H */
diff --git a/include/dt-bindings/clock/oxsemi,ox810se.h b/include/dt-bindings/clock/oxsemi,ox810se.h
new file mode 100644
index 000000000000..d5facb5e83f1
--- /dev/null
+++ b/include/dt-bindings/clock/oxsemi,ox810se.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2016 Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef DT_CLOCK_OXSEMI_OX810SE_H
+#define DT_CLOCK_OXSEMI_OX810SE_H
+
+#define CLK_810_LEON		0
+#define CLK_810_DMA_SGDMA	1
+#define CLK_810_CIPHER		2
+#define CLK_810_SATA		3
+#define CLK_810_AUDIO		4
+#define CLK_810_USBMPH		5
+#define CLK_810_ETHA		6
+#define CLK_810_PCIEA		7
+#define CLK_810_NAND		8
+
+#endif /* DT_CLOCK_OXSEMI_OX810SE_H */
diff --git a/include/dt-bindings/clock/oxsemi,ox820.h b/include/dt-bindings/clock/oxsemi,ox820.h
new file mode 100644
index 000000000000..f661ecc8d760
--- /dev/null
+++ b/include/dt-bindings/clock/oxsemi,ox820.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2016 Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef DT_CLOCK_OXSEMI_OX820_H
+#define DT_CLOCK_OXSEMI_OX820_H
+
+/* PLLs */
+#define CLK_820_PLLA		0
+#define CLK_820_PLLB		1
+
+/* Gate Clocks */
+#define CLK_820_LEON		2
+#define CLK_820_DMA_SGDMA	3
+#define CLK_820_CIPHER		4
+#define CLK_820_SD		5
+#define CLK_820_SATA		6
+#define CLK_820_AUDIO		7
+#define CLK_820_USBMPH		8
+#define CLK_820_ETHA		9
+#define CLK_820_PCIEA		10
+#define CLK_820_NAND		11
+#define CLK_820_PCIEB		12
+#define CLK_820_ETHB		13
+#define CLK_820_REF600		14
+#define CLK_820_USBDEV		15
+
+#endif /* DT_CLOCK_OXSEMI_OX820_H */
diff --git a/include/dt-bindings/clock/qcom,gcc-msm8994.h b/include/dt-bindings/clock/qcom,gcc-msm8994.h
new file mode 100644
index 000000000000..8fa535be2ebc
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,gcc-msm8994.h
@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+
+#ifndef _DT_BINDINGS_CLK_MSM_GCC_8994_H
+#define _DT_BINDINGS_CLK_MSM_GCC_8994_H
+
+#define GPLL0_EARLY				0
+#define GPLL0					1
+#define GPLL4_EARLY				2
+#define GPLL4					3
+#define UFS_AXI_CLK_SRC				4
+#define USB30_MASTER_CLK_SRC			5
+#define BLSP1_QUP1_I2C_APPS_CLK_SRC		6
+#define BLSP1_QUP1_SPI_APPS_CLK_SRC		7
+#define BLSP1_QUP2_I2C_APPS_CLK_SRC		8
+#define BLSP1_QUP2_SPI_APPS_CLK_SRC		9
+#define BLSP1_QUP3_I2C_APPS_CLK_SRC		10
+#define BLSP1_QUP3_SPI_APPS_CLK_SRC		11
+#define BLSP1_QUP4_I2C_APPS_CLK_SRC		12
+#define BLSP1_QUP4_SPI_APPS_CLK_SRC		13
+#define BLSP1_QUP5_I2C_APPS_CLK_SRC		14
+#define BLSP1_QUP5_SPI_APPS_CLK_SRC		15
+#define BLSP1_QUP6_I2C_APPS_CLK_SRC		16
+#define BLSP1_QUP6_SPI_APPS_CLK_SRC		17
+#define BLSP1_UART1_APPS_CLK_SRC		18
+#define BLSP1_UART2_APPS_CLK_SRC		19
+#define BLSP1_UART3_APPS_CLK_SRC		20
+#define BLSP1_UART4_APPS_CLK_SRC		21
+#define BLSP1_UART5_APPS_CLK_SRC		22
+#define BLSP1_UART6_APPS_CLK_SRC		23
+#define BLSP2_QUP1_I2C_APPS_CLK_SRC		24
+#define BLSP2_QUP1_SPI_APPS_CLK_SRC		25
+#define BLSP2_QUP2_I2C_APPS_CLK_SRC		26
+#define BLSP2_QUP2_SPI_APPS_CLK_SRC		27
+#define BLSP2_QUP3_I2C_APPS_CLK_SRC		28
+#define BLSP2_QUP3_SPI_APPS_CLK_SRC		29
+#define BLSP2_QUP4_I2C_APPS_CLK_SRC		30
+#define BLSP2_QUP4_SPI_APPS_CLK_SRC		31
+#define BLSP2_QUP5_I2C_APPS_CLK_SRC		32
+#define BLSP2_QUP5_SPI_APPS_CLK_SRC		33
+#define BLSP2_QUP6_I2C_APPS_CLK_SRC		34
+#define BLSP2_QUP6_SPI_APPS_CLK_SRC		35
+#define BLSP2_UART1_APPS_CLK_SRC		36
+#define BLSP2_UART2_APPS_CLK_SRC		37
+#define BLSP2_UART3_APPS_CLK_SRC		38
+#define BLSP2_UART4_APPS_CLK_SRC		39
+#define BLSP2_UART5_APPS_CLK_SRC		40
+#define BLSP2_UART6_APPS_CLK_SRC		41
+#define GP1_CLK_SRC				42
+#define GP2_CLK_SRC				43
+#define GP3_CLK_SRC				44
+#define PCIE_0_AUX_CLK_SRC			45
+#define PCIE_0_PIPE_CLK_SRC			46
+#define PCIE_1_AUX_CLK_SRC			47
+#define PCIE_1_PIPE_CLK_SRC			48
+#define PDM2_CLK_SRC				49
+#define SDCC1_APPS_CLK_SRC			50
+#define SDCC2_APPS_CLK_SRC			51
+#define SDCC3_APPS_CLK_SRC			52
+#define SDCC4_APPS_CLK_SRC			53
+#define TSIF_REF_CLK_SRC			54
+#define USB30_MOCK_UTMI_CLK_SRC			55
+#define USB3_PHY_AUX_CLK_SRC			56
+#define USB_HS_SYSTEM_CLK_SRC			57
+#define GCC_BLSP1_AHB_CLK			58
+#define GCC_BLSP1_QUP1_I2C_APPS_CLK		59
+#define GCC_BLSP1_QUP1_SPI_APPS_CLK		60
+#define GCC_BLSP1_QUP2_I2C_APPS_CLK		61
+#define GCC_BLSP1_QUP2_SPI_APPS_CLK		62
+#define GCC_BLSP1_QUP3_I2C_APPS_CLK		63
+#define GCC_BLSP1_QUP3_SPI_APPS_CLK		64
+#define GCC_BLSP1_QUP4_I2C_APPS_CLK		65
+#define GCC_BLSP1_QUP4_SPI_APPS_CLK		66
+#define GCC_BLSP1_QUP5_I2C_APPS_CLK		67
+#define GCC_BLSP1_QUP5_SPI_APPS_CLK		68
+#define GCC_BLSP1_QUP6_I2C_APPS_CLK		69
+#define GCC_BLSP1_QUP6_SPI_APPS_CLK		70
+#define GCC_BLSP1_UART1_APPS_CLK		71
+#define GCC_BLSP1_UART2_APPS_CLK		72
+#define GCC_BLSP1_UART3_APPS_CLK		73
+#define GCC_BLSP1_UART4_APPS_CLK		74
+#define GCC_BLSP1_UART5_APPS_CLK		75
+#define GCC_BLSP1_UART6_APPS_CLK		76
+#define GCC_BLSP2_AHB_CLK			77
+#define GCC_BLSP2_QUP1_I2C_APPS_CLK		78
+#define GCC_BLSP2_QUP1_SPI_APPS_CLK		79
+#define GCC_BLSP2_QUP2_I2C_APPS_CLK		80
+#define GCC_BLSP2_QUP2_SPI_APPS_CLK		81
+#define GCC_BLSP2_QUP3_I2C_APPS_CLK		82
+#define GCC_BLSP2_QUP3_SPI_APPS_CLK		83
+#define GCC_BLSP2_QUP4_I2C_APPS_CLK		84
+#define GCC_BLSP2_QUP4_SPI_APPS_CLK		85
+#define GCC_BLSP2_QUP5_I2C_APPS_CLK		86
+#define GCC_BLSP2_QUP5_SPI_APPS_CLK		87
+#define GCC_BLSP2_QUP6_I2C_APPS_CLK		88
+#define GCC_BLSP2_QUP6_SPI_APPS_CLK		89
+#define GCC_BLSP2_UART1_APPS_CLK		90
+#define GCC_BLSP2_UART2_APPS_CLK		91
+#define GCC_BLSP2_UART3_APPS_CLK		92
+#define GCC_BLSP2_UART4_APPS_CLK		93
+#define GCC_BLSP2_UART5_APPS_CLK		94
+#define GCC_BLSP2_UART6_APPS_CLK		95
+#define GCC_GP1_CLK				96
+#define GCC_GP2_CLK				97
+#define GCC_GP3_CLK				98
+#define GCC_PCIE_0_AUX_CLK			99
+#define GCC_PCIE_0_PIPE_CLK			100
+#define GCC_PCIE_1_AUX_CLK			101
+#define GCC_PCIE_1_PIPE_CLK			102
+#define GCC_PDM2_CLK				103
+#define GCC_SDCC1_APPS_CLK			104
+#define GCC_SDCC2_APPS_CLK			105
+#define GCC_SDCC3_APPS_CLK			106
+#define GCC_SDCC4_APPS_CLK			107
+#define GCC_SYS_NOC_UFS_AXI_CLK			108
+#define GCC_SYS_NOC_USB3_AXI_CLK		109
+#define GCC_TSIF_REF_CLK			110
+#define GCC_UFS_AXI_CLK				111
+#define GCC_UFS_RX_CFG_CLK			112
+#define GCC_UFS_TX_CFG_CLK			113
+#define GCC_USB30_MASTER_CLK			114
+#define GCC_USB30_MOCK_UTMI_CLK			115
+#define GCC_USB3_PHY_AUX_CLK			116
+#define GCC_USB_HS_SYSTEM_CLK			117
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,rpmcc.h b/include/dt-bindings/clock/qcom,rpmcc.h
new file mode 100644
index 000000000000..5924cdb71336
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,rpmcc.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2015 Linaro Limited
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DT_BINDINGS_CLK_MSM_RPMCC_H
+#define _DT_BINDINGS_CLK_MSM_RPMCC_H
+
+/* apq8064 */
+#define RPM_PXO_CLK				0
+#define RPM_PXO_A_CLK				1
+#define RPM_CXO_CLK				2
+#define RPM_CXO_A_CLK				3
+#define RPM_APPS_FABRIC_CLK			4
+#define RPM_APPS_FABRIC_A_CLK			5
+#define RPM_CFPB_CLK				6
+#define RPM_CFPB_A_CLK				7
+#define RPM_QDSS_CLK				8
+#define RPM_QDSS_A_CLK				9
+#define RPM_DAYTONA_FABRIC_CLK			10
+#define RPM_DAYTONA_FABRIC_A_CLK		11
+#define RPM_EBI1_CLK				12
+#define RPM_EBI1_A_CLK				13
+#define RPM_MM_FABRIC_CLK			14
+#define RPM_MM_FABRIC_A_CLK			15
+#define RPM_MMFPB_CLK				16
+#define RPM_MMFPB_A_CLK				17
+#define RPM_SYS_FABRIC_CLK			18
+#define RPM_SYS_FABRIC_A_CLK			19
+#define RPM_SFPB_CLK				20
+#define RPM_SFPB_A_CLK				21
+
+/* msm8916 */
+#define RPM_SMD_XO_CLK_SRC				0
+#define RPM_SMD_XO_A_CLK_SRC			1
+#define RPM_SMD_PCNOC_CLK				2
+#define RPM_SMD_PCNOC_A_CLK				3
+#define RPM_SMD_SNOC_CLK				4
+#define RPM_SMD_SNOC_A_CLK				5
+#define RPM_SMD_BIMC_CLK				6
+#define RPM_SMD_BIMC_A_CLK				7
+#define RPM_SMD_QDSS_CLK				8
+#define RPM_SMD_QDSS_A_CLK				9
+#define RPM_SMD_BB_CLK1				10
+#define RPM_SMD_BB_CLK1_A				11
+#define RPM_SMD_BB_CLK2				12
+#define RPM_SMD_BB_CLK2_A				13
+#define RPM_SMD_RF_CLK1				14
+#define RPM_SMD_RF_CLK1_A				15
+#define RPM_SMD_RF_CLK2				16
+#define RPM_SMD_RF_CLK2_A				17
+#define RPM_SMD_BB_CLK1_PIN				18
+#define RPM_SMD_BB_CLK1_A_PIN			19
+#define RPM_SMD_BB_CLK2_PIN				20
+#define RPM_SMD_BB_CLK2_A_PIN			21
+#define RPM_SMD_RF_CLK1_PIN				22
+#define RPM_SMD_RF_CLK1_A_PIN			23
+#define RPM_SMD_RF_CLK2_PIN				24
+#define RPM_SMD_RF_CLK2_A_PIN			25
+
+#endif
diff --git a/include/dt-bindings/clock/r7s72100-clock.h b/include/dt-bindings/clock/r7s72100-clock.h
index 3cd813896d08..29e01ed10e74 100644
--- a/include/dt-bindings/clock/r7s72100-clock.h
+++ b/include/dt-bindings/clock/r7s72100-clock.h
@@ -28,6 +28,9 @@
 /* MSTP7 */
 #define R7S72100_CLK_ETHER	4
 
+/* MSTP8 */
+#define R7S72100_CLK_MMCIF	4
+
 /* MSTP9 */
 #define R7S72100_CLK_I2C0	7
 #define R7S72100_CLK_I2C1	6
@@ -41,4 +44,8 @@
 #define R7S72100_CLK_SPI3	4
 #define R7S72100_CLK_SPI4	3
 
+/* MSTP12 */
+#define R7S72100_CLK_SDHI0	3
+#define R7S72100_CLK_SDHI1	2
+
 #endif /* __DT_BINDINGS_CLOCK_R7S72100_H__ */
diff --git a/include/dt-bindings/clock/r8a7743-cpg-mssr.h b/include/dt-bindings/clock/r8a7743-cpg-mssr.h
new file mode 100644
index 000000000000..e1d1f3c6a99e
--- /dev/null
+++ b/include/dt-bindings/clock/r8a7743-cpg-mssr.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2016 Cogent Embedded Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+#ifndef __DT_BINDINGS_CLOCK_R8A7743_CPG_MSSR_H__
+#define __DT_BINDINGS_CLOCK_R8A7743_CPG_MSSR_H__
+
+#include <dt-bindings/clock/renesas-cpg-mssr.h>
+
+/* r8a7743 CPG Core Clocks */
+#define R8A7743_CLK_Z		0
+#define R8A7743_CLK_ZG		1
+#define R8A7743_CLK_ZTR		2
+#define R8A7743_CLK_ZTRD2	3
+#define R8A7743_CLK_ZT		4
+#define R8A7743_CLK_ZX		5
+#define R8A7743_CLK_ZS		6
+#define R8A7743_CLK_HP		7
+#define R8A7743_CLK_B		9
+#define R8A7743_CLK_LB		10
+#define R8A7743_CLK_P		11
+#define R8A7743_CLK_CL		12
+#define R8A7743_CLK_M2		13
+#define R8A7743_CLK_ZB3		15
+#define R8A7743_CLK_ZB3D2	16
+#define R8A7743_CLK_DDR		17
+#define R8A7743_CLK_SDH		18
+#define R8A7743_CLK_SD0		19
+#define R8A7743_CLK_SD2		20
+#define R8A7743_CLK_SD3		21
+#define R8A7743_CLK_MMC0	22
+#define R8A7743_CLK_MP		23
+#define R8A7743_CLK_QSPI	26
+#define R8A7743_CLK_CP		27
+#define R8A7743_CLK_RCAN	28
+#define R8A7743_CLK_R		29
+#define R8A7743_CLK_OSC		30
+
+#endif /* __DT_BINDINGS_CLOCK_R8A7743_CPG_MSSR_H__ */
diff --git a/include/dt-bindings/clock/r8a7745-cpg-mssr.h b/include/dt-bindings/clock/r8a7745-cpg-mssr.h
new file mode 100644
index 000000000000..56ad6f0c6760
--- /dev/null
+++ b/include/dt-bindings/clock/r8a7745-cpg-mssr.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2016 Cogent Embedded Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+#ifndef __DT_BINDINGS_CLOCK_R8A7745_CPG_MSSR_H__
+#define __DT_BINDINGS_CLOCK_R8A7745_CPG_MSSR_H__
+
+#include <dt-bindings/clock/renesas-cpg-mssr.h>
+
+/* r8a7745 CPG Core Clocks */
+#define R8A7745_CLK_Z2		0
+#define R8A7745_CLK_ZG		1
+#define R8A7745_CLK_ZTR		2
+#define R8A7745_CLK_ZTRD2	3
+#define R8A7745_CLK_ZT		4
+#define R8A7745_CLK_ZX		5
+#define R8A7745_CLK_ZS		6
+#define R8A7745_CLK_HP		7
+#define R8A7745_CLK_B		9
+#define R8A7745_CLK_LB		10
+#define R8A7745_CLK_P		11
+#define R8A7745_CLK_CL		12
+#define R8A7745_CLK_CP		13
+#define R8A7745_CLK_M2		14
+#define R8A7745_CLK_ZB3		16
+#define R8A7745_CLK_ZB3D2	17
+#define R8A7745_CLK_DDR		18
+#define R8A7745_CLK_SDH		19
+#define R8A7745_CLK_SD0		20
+#define R8A7745_CLK_SD2		21
+#define R8A7745_CLK_SD3		22
+#define R8A7745_CLK_MMC0	23
+#define R8A7745_CLK_MP		24
+#define R8A7745_CLK_QSPI	25
+#define R8A7745_CLK_CPEX	26
+#define R8A7745_CLK_RCAN	27
+#define R8A7745_CLK_R		28
+#define R8A7745_CLK_OSC		29
+
+#endif /* __DT_BINDINGS_CLOCK_R8A7745_CPG_MSSR_H__ */
diff --git a/include/dt-bindings/clock/r8a7794-clock.h b/include/dt-bindings/clock/r8a7794-clock.h
index 9d02f5317c7c..88e64846cf37 100644
--- a/include/dt-bindings/clock/r8a7794-clock.h
+++ b/include/dt-bindings/clock/r8a7794-clock.h
@@ -20,8 +20,7 @@
 #define R8A7794_CLK_QSPI		5
 #define R8A7794_CLK_SDH			6
 #define R8A7794_CLK_SD0			7
-#define R8A7794_CLK_Z			8
-#define R8A7794_CLK_RCAN		9
+#define R8A7794_CLK_RCAN		8
 
 /* MSTP0 */
 #define R8A7794_CLK_MSIOF0		0
diff --git a/include/dt-bindings/clock/rk1108-cru.h b/include/dt-bindings/clock/rk1108-cru.h
new file mode 100644
index 000000000000..9350a5527a36
--- /dev/null
+++ b/include/dt-bindings/clock/rk1108-cru.h
@@ -0,0 +1,269 @@
+/*
+ * Copyright (c) 2016 Rockchip Electronics Co. Ltd.
+ * Author: Shawn Lin <shawn.lin@rock-chips.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DT_BINDINGS_CLK_ROCKCHIP_RK1108_H
+#define _DT_BINDINGS_CLK_ROCKCHIP_RK1108_H
+
+/* pll id */
+#define PLL_APLL			0
+#define PLL_DPLL			1
+#define PLL_GPLL			2
+#define ARMCLK				3
+
+/* sclk gates (special clocks) */
+#define SCLK_SPI0			65
+#define SCLK_NANDC			67
+#define SCLK_SDMMC			68
+#define SCLK_SDIO			69
+#define SCLK_EMMC			71
+#define SCLK_UART0			72
+#define SCLK_UART1			73
+#define SCLK_UART2			74
+#define SCLK_I2S0			75
+#define SCLK_I2S1			76
+#define SCLK_I2S2			77
+#define SCLK_TIMER0			78
+#define SCLK_TIMER1			79
+#define SCLK_SFC			80
+#define SCLK_SDMMC_DRV			81
+#define SCLK_SDIO_DRV			82
+#define SCLK_EMMC_DRV			83
+#define SCLK_SDMMC_SAMPLE		84
+#define SCLK_SDIO_SAMPLE		85
+#define SCLK_EMMC_SAMPLE		86
+
+/* aclk gates */
+#define ACLK_DMAC			192
+#define ACLK_PRE			193
+#define ACLK_CORE			194
+#define ACLK_ENMCORE			195
+
+/* pclk gates */
+#define PCLK_GPIO1			256
+#define PCLK_GPIO2			257
+#define PCLK_GPIO3			258
+#define PCLK_GRF			259
+#define PCLK_I2C1			260
+#define PCLK_I2C2			261
+#define PCLK_I2C3			262
+#define PCLK_SPI			263
+#define PCLK_SFC			264
+#define PCLK_UART0			265
+#define PCLK_UART1			266
+#define PCLK_UART2			267
+#define PCLK_TSADC			268
+#define PCLK_PWM			269
+#define PCLK_TIMER			270
+#define PCLK_PERI			271
+
+/* hclk gates */
+#define HCLK_I2S0_8CH			320
+#define HCLK_I2S1_8CH			321
+#define HCLK_I2S2_2CH			322
+#define HCLK_NANDC			323
+#define HCLK_SDMMC			324
+#define HCLK_SDIO			325
+#define HCLK_EMMC			326
+#define HCLK_PERI			327
+#define HCLK_SFC			328
+
+#define CLK_NR_CLKS			(HCLK_SFC + 1)
+
+/* reset id */
+#define SRST_CORE_PO_AD		0
+#define SRST_CORE_AD			1
+#define SRST_L2_AD			2
+#define SRST_CPU_NIU_AD		3
+#define SRST_CORE_PO			4
+#define SRST_CORE			5
+#define SRST_L2			6
+#define SRST_CORE_DBG			8
+#define PRST_DBG			9
+#define RST_DAP			10
+#define PRST_DBG_NIU			11
+#define ARST_STRC_SYS_AD		15
+
+#define SRST_DDRPHY_CLKDIV		16
+#define SRST_DDRPHY			17
+#define PRST_DDRPHY			18
+#define PRST_HDMIPHY			19
+#define PRST_VDACPHY			20
+#define PRST_VADCPHY			21
+#define PRST_MIPI_CSI_PHY		22
+#define PRST_MIPI_DSI_PHY		23
+#define PRST_ACODEC			24
+#define ARST_BUS_NIU			25
+#define PRST_TOP_NIU			26
+#define ARST_INTMEM			27
+#define HRST_ROM			28
+#define ARST_DMAC			29
+#define SRST_MSCH_NIU			30
+#define PRST_MSCH_NIU			31
+
+#define PRST_DDRUPCTL			32
+#define NRST_DDRUPCTL			33
+#define PRST_DDRMON			34
+#define HRST_I2S0_8CH			35
+#define MRST_I2S0_8CH			36
+#define HRST_I2S1_2CH			37
+#define MRST_IS21_2CH			38
+#define HRST_I2S2_2CH			39
+#define MRST_I2S2_2CH			40
+#define HRST_CRYPTO			41
+#define SRST_CRYPTO			42
+#define PRST_SPI			43
+#define SRST_SPI			44
+#define PRST_UART0			45
+#define PRST_UART1			46
+#define PRST_UART2			47
+
+#define SRST_UART0			48
+#define SRST_UART1			49
+#define SRST_UART2			50
+#define PRST_I2C1			51
+#define PRST_I2C2			52
+#define PRST_I2C3			53
+#define SRST_I2C1			54
+#define SRST_I2C2			55
+#define SRST_I2C3			56
+#define PRST_PWM1			58
+#define SRST_PWM1			60
+#define PRST_WDT			61
+#define PRST_GPIO1			62
+#define PRST_GPIO2			63
+
+#define PRST_GPIO3			64
+#define PRST_GRF			65
+#define PRST_EFUSE			66
+#define PRST_EFUSE512			67
+#define PRST_TIMER0			68
+#define SRST_TIMER0			69
+#define SRST_TIMER1			70
+#define PRST_TSADC			71
+#define SRST_TSADC			72
+#define PRST_SARADC			73
+#define SRST_SARADC			74
+#define HRST_SYSBUS			75
+#define PRST_USBGRF			76
+
+#define ARST_PERIPH_NIU		80
+#define HRST_PERIPH_NIU		81
+#define PRST_PERIPH_NIU		82
+#define HRST_PERIPH			83
+#define HRST_SDMMC			84
+#define HRST_SDIO			85
+#define HRST_EMMC			86
+#define HRST_NANDC			87
+#define NRST_NANDC			88
+#define HRST_SFC			89
+#define SRST_SFC			90
+#define ARST_GMAC			91
+#define HRST_OTG			92
+#define SRST_OTG			93
+#define SRST_OTG_ADP			94
+#define HRST_HOST0			95
+
+#define HRST_HOST0_AUX			96
+#define HRST_HOST0_ARB			97
+#define SRST_HOST0_EHCIPHY		98
+#define SRST_HOST0_UTMI		99
+#define SRST_USBPOR			100
+#define SRST_UTMI0			101
+#define SRST_UTMI1			102
+
+#define ARST_VIO0_NIU			102
+#define ARST_VIO1_NIU			103
+#define HRST_VIO_NIU			104
+#define PRST_VIO_NIU			105
+#define ARST_VOP			106
+#define HRST_VOP			107
+#define DRST_VOP			108
+#define ARST_IEP			109
+#define HRST_IEP			110
+#define ARST_RGA			111
+#define HRST_RGA			112
+#define SRST_RGA			113
+#define PRST_CVBS			114
+#define PRST_HDMI			115
+#define SRST_HDMI			116
+#define PRST_MIPI_DSI			117
+
+#define ARST_ISP_NIU			118
+#define HRST_ISP_NIU			119
+#define HRST_ISP			120
+#define SRST_ISP			121
+#define ARST_VIP0			122
+#define HRST_VIP0			123
+#define PRST_VIP0			124
+#define ARST_VIP1			125
+#define HRST_VIP1			126
+#define PRST_VIP1			127
+#define ARST_VIP2			128
+#define HRST_VIP2			129
+#define PRST_VIP2			120
+#define ARST_VIP3			121
+#define HRST_VIP3			122
+#define PRST_VIP4			123
+
+#define PRST_CIF1TO4			124
+#define SRST_CVBS_CLK			125
+#define HRST_CVBS			126
+
+#define ARST_VPU_NIU			140
+#define HRST_VPU_NIU			141
+#define ARST_VPU			142
+#define HRST_VPU			143
+#define ARST_RKVDEC_NIU		144
+#define HRST_RKVDEC_NIU		145
+#define ARST_RKVDEC			146
+#define HRST_RKVDEC			147
+#define SRST_RKVDEC_CABAC		148
+#define SRST_RKVDEC_CORE		149
+#define ARST_RKVENC_NIU		150
+#define HRST_RKVENC_NIU		151
+#define ARST_RKVENC			152
+#define HRST_RKVENC			153
+#define SRST_RKVENC_CORE		154
+
+#define SRST_DSP_CORE			156
+#define SRST_DSP_SYS			157
+#define SRST_DSP_GLOBAL		158
+#define SRST_DSP_OECM			159
+#define PRST_DSP_IOP_NIU		160
+#define ARST_DSP_EPP_NIU		161
+#define ARST_DSP_EDP_NIU		162
+#define PRST_DSP_DBG_NIU		163
+#define PRST_DSP_CFG_NIU		164
+#define PRST_DSP_GRF			165
+#define PRST_DSP_MAILBOX		166
+#define PRST_DSP_INTC			167
+#define PRST_DSP_PFM_MON		169
+#define SRST_DSP_PFM_MON		170
+#define ARST_DSP_EDAP_NIU		171
+
+#define SRST_PMU			172
+#define SRST_PMU_I2C0			173
+#define PRST_PMU_I2C0			174
+#define PRST_PMU_GPIO0			175
+#define PRST_PMU_INTMEM		176
+#define PRST_PMU_PWM0			177
+#define SRST_PMU_PWM0			178
+#define PRST_PMU_GRF			179
+#define SRST_PMU_NIU			180
+#define SRST_PMU_PVTM			181
+#define ARST_DSP_EDP_PERF		184
+#define ARST_DSP_EPP_PERF		185
+
+#endif /* _DT_BINDINGS_CLK_ROCKCHIP_RK1108_H */
diff --git a/include/dt-bindings/clock/rk3188-cru-common.h b/include/dt-bindings/clock/rk3188-cru-common.h
index 4f53e70f68ee..d141c1f0c778 100644
--- a/include/dt-bindings/clock/rk3188-cru-common.h
+++ b/include/dt-bindings/clock/rk3188-cru-common.h
@@ -72,6 +72,8 @@
 #define ACLK_IPP		200
 #define ACLK_RGA		201
 #define ACLK_CIF0		202
+#define ACLK_CPU		203
+#define ACLK_PERI		204
 
 /* pclk gates */
 #define PCLK_GRF		320
@@ -104,6 +106,8 @@
 #define PCLK_EFUSE		347
 #define PCLK_TZPC		348
 #define PCLK_TSADC		349
+#define PCLK_CPU		350
+#define PCLK_PERI		351
 
 /* hclk gates */
 #define HCLK_SDMMC		448
@@ -126,8 +130,10 @@
 #define HCLK_IPP		465
 #define HCLK_RGA		466
 #define HCLK_NANDC0		467
+#define HCLK_CPU		468
+#define HCLK_PERI		469
 
-#define CLK_NR_CLKS		(HCLK_NANDC0 + 1)
+#define CLK_NR_CLKS		(HCLK_PERI + 1)
 
 /* soft-reset indices */
 #define SRST_MCORE		2
diff --git a/include/dt-bindings/clock/stih415-clks.h b/include/dt-bindings/clock/stih415-clks.h
deleted file mode 100644
index d80caa68aebd..000000000000
--- a/include/dt-bindings/clock/stih415-clks.h
+++ /dev/null
@@ -1,16 +0,0 @@
-/*
- * This header provides constants clk index STMicroelectronics
- * STiH415 SoC.
- */
-#ifndef _CLK_STIH415
-#define _CLK_STIH415
-
-/* CLOCKGEN A0 */
-#define CLK_ICN_REG		0
-#define CLK_ETH1_PHY		4
-
-/* CLOCKGEN A1 */
-#define CLK_ICN_IF_2		0
-#define CLK_GMAC0_PHY		3
-
-#endif
diff --git a/include/dt-bindings/clock/sun50i-a64-ccu.h b/include/dt-bindings/clock/sun50i-a64-ccu.h
new file mode 100644
index 000000000000..370c0a0473fc
--- /dev/null
+++ b/include/dt-bindings/clock/sun50i-a64-ccu.h
@@ -0,0 +1,134 @@
+/*
+ * Copyright (C) 2016 Maxime Ripard <maxime.ripard@free-electrons.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _DT_BINDINGS_CLK_SUN50I_A64_H_
+#define _DT_BINDINGS_CLK_SUN50I_A64_H_
+
+#define CLK_BUS_MIPI_DSI	28
+#define CLK_BUS_CE		29
+#define CLK_BUS_DMA		30
+#define CLK_BUS_MMC0		31
+#define CLK_BUS_MMC1		32
+#define CLK_BUS_MMC2		33
+#define CLK_BUS_NAND		34
+#define CLK_BUS_DRAM		35
+#define CLK_BUS_EMAC		36
+#define CLK_BUS_TS		37
+#define CLK_BUS_HSTIMER		38
+#define CLK_BUS_SPI0		39
+#define CLK_BUS_SPI1		40
+#define CLK_BUS_OTG		41
+#define CLK_BUS_EHCI0		42
+#define CLK_BUS_EHCI1		43
+#define CLK_BUS_OHCI0		44
+#define CLK_BUS_OHCI1		45
+#define CLK_BUS_VE		46
+#define CLK_BUS_TCON0		47
+#define CLK_BUS_TCON1		48
+#define CLK_BUS_DEINTERLACE	49
+#define CLK_BUS_CSI		50
+#define CLK_BUS_HDMI		51
+#define CLK_BUS_DE		52
+#define CLK_BUS_GPU		53
+#define CLK_BUS_MSGBOX		54
+#define CLK_BUS_SPINLOCK	55
+#define CLK_BUS_CODEC		56
+#define CLK_BUS_SPDIF		57
+#define CLK_BUS_PIO		58
+#define CLK_BUS_THS		59
+#define CLK_BUS_I2S0		60
+#define CLK_BUS_I2S1		61
+#define CLK_BUS_I2S2		62
+#define CLK_BUS_I2C0		63
+#define CLK_BUS_I2C1		64
+#define CLK_BUS_I2C2		65
+#define CLK_BUS_SCR		66
+#define CLK_BUS_UART0		67
+#define CLK_BUS_UART1		68
+#define CLK_BUS_UART2		69
+#define CLK_BUS_UART3		70
+#define CLK_BUS_UART4		71
+#define CLK_BUS_DBG		72
+#define CLK_THS			73
+#define CLK_NAND		74
+#define CLK_MMC0		75
+#define CLK_MMC1		76
+#define CLK_MMC2		77
+#define CLK_TS			78
+#define CLK_CE			79
+#define CLK_SPI0		80
+#define CLK_SPI1		81
+#define CLK_I2S0		82
+#define CLK_I2S1		83
+#define CLK_I2S2		84
+#define CLK_SPDIF		85
+#define CLK_USB_PHY0		86
+#define CLK_USB_PHY1		87
+#define CLK_USB_HSIC		88
+#define CLK_USB_HSIC_12M	89
+
+#define CLK_USB_OHCI0		91
+
+#define CLK_USB_OHCI1		93
+
+#define CLK_DRAM_VE		95
+#define CLK_DRAM_CSI		96
+#define CLK_DRAM_DEINTERLACE	97
+#define CLK_DRAM_TS		98
+#define CLK_DE			99
+#define CLK_TCON0		100
+#define CLK_TCON1		101
+#define CLK_DEINTERLACE		102
+#define CLK_CSI_MISC		103
+#define CLK_CSI_SCLK		104
+#define CLK_CSI_MCLK		105
+#define CLK_VE			106
+#define CLK_AC_DIG		107
+#define CLK_AC_DIG_4X		108
+#define CLK_AVS			109
+#define CLK_HDMI		110
+#define CLK_HDMI_DDC		111
+
+#define CLK_DSI_DPHY		113
+#define CLK_GPU			114
+
+#endif /* _DT_BINDINGS_CLK_SUN50I_H_ */
diff --git a/include/dt-bindings/clock/tegra186-clock.h b/include/dt-bindings/clock/tegra186-clock.h
new file mode 100644
index 000000000000..f73d32098f99
--- /dev/null
+++ b/include/dt-bindings/clock/tegra186-clock.h
@@ -0,0 +1,940 @@
+/** @file */
+
+#ifndef _MACH_T186_CLK_T186_H
+#define _MACH_T186_CLK_T186_H
+
+/**
+ * @defgroup clock_ids Clock Identifiers
+ * @{
+ *   @defgroup extern_input external input clocks
+ *   @{
+ *     @def TEGRA186_CLK_OSC
+ *     @def TEGRA186_CLK_CLK_32K
+ *     @def TEGRA186_CLK_DTV_INPUT
+ *     @def TEGRA186_CLK_SOR0_PAD_CLKOUT
+ *     @def TEGRA186_CLK_SOR1_PAD_CLKOUT
+ *     @def TEGRA186_CLK_I2S1_SYNC_INPUT
+ *     @def TEGRA186_CLK_I2S2_SYNC_INPUT
+ *     @def TEGRA186_CLK_I2S3_SYNC_INPUT
+ *     @def TEGRA186_CLK_I2S4_SYNC_INPUT
+ *     @def TEGRA186_CLK_I2S5_SYNC_INPUT
+ *     @def TEGRA186_CLK_I2S6_SYNC_INPUT
+ *     @def TEGRA186_CLK_SPDIFIN_SYNC_INPUT
+ *   @}
+ *
+ *   @defgroup extern_output external output clocks
+ *   @{
+ *     @def TEGRA186_CLK_EXTPERIPH1
+ *     @def TEGRA186_CLK_EXTPERIPH2
+ *     @def TEGRA186_CLK_EXTPERIPH3
+ *     @def TEGRA186_CLK_EXTPERIPH4
+ *   @}
+ *
+ *   @defgroup display_clks display related clocks
+ *   @{
+ *     @def TEGRA186_CLK_CEC
+ *     @def TEGRA186_CLK_DSIC
+ *     @def TEGRA186_CLK_DSIC_LP
+ *     @def TEGRA186_CLK_DSID
+ *     @def TEGRA186_CLK_DSID_LP
+ *     @def TEGRA186_CLK_DPAUX1
+ *     @def TEGRA186_CLK_DPAUX
+ *     @def TEGRA186_CLK_HDA2HDMICODEC
+ *     @def TEGRA186_CLK_NVDISPLAY_DISP
+ *     @def TEGRA186_CLK_NVDISPLAY_DSC
+ *     @def TEGRA186_CLK_NVDISPLAY_P0
+ *     @def TEGRA186_CLK_NVDISPLAY_P1
+ *     @def TEGRA186_CLK_NVDISPLAY_P2
+ *     @def TEGRA186_CLK_NVDISPLAYHUB
+ *     @def TEGRA186_CLK_SOR_SAFE
+ *     @def TEGRA186_CLK_SOR0
+ *     @def TEGRA186_CLK_SOR0_OUT
+ *     @def TEGRA186_CLK_SOR1
+ *     @def TEGRA186_CLK_SOR1_OUT
+ *     @def TEGRA186_CLK_DSI
+ *     @def TEGRA186_CLK_MIPI_CAL
+ *     @def TEGRA186_CLK_DSIA_LP
+ *     @def TEGRA186_CLK_DSIB
+ *     @def TEGRA186_CLK_DSIB_LP
+ *   @}
+ *
+ *   @defgroup camera_clks camera related clocks
+ *   @{
+ *     @def TEGRA186_CLK_NVCSI
+ *     @def TEGRA186_CLK_NVCSILP
+ *     @def TEGRA186_CLK_VI
+ *   @}
+ *
+ *   @defgroup audio_clks audio related clocks
+ *   @{
+ *     @def TEGRA186_CLK_ACLK
+ *     @def TEGRA186_CLK_ADSP
+ *     @def TEGRA186_CLK_ADSPNEON
+ *     @def TEGRA186_CLK_AHUB
+ *     @def TEGRA186_CLK_APE
+ *     @def TEGRA186_CLK_APB2APE
+ *     @def TEGRA186_CLK_AUD_MCLK
+ *     @def TEGRA186_CLK_DMIC1
+ *     @def TEGRA186_CLK_DMIC2
+ *     @def TEGRA186_CLK_DMIC3
+ *     @def TEGRA186_CLK_DMIC4
+ *     @def TEGRA186_CLK_DSPK1
+ *     @def TEGRA186_CLK_DSPK2
+ *     @def TEGRA186_CLK_HDA
+ *     @def TEGRA186_CLK_HDA2CODEC_2X
+ *     @def TEGRA186_CLK_I2S1
+ *     @def TEGRA186_CLK_I2S2
+ *     @def TEGRA186_CLK_I2S3
+ *     @def TEGRA186_CLK_I2S4
+ *     @def TEGRA186_CLK_I2S5
+ *     @def TEGRA186_CLK_I2S6
+ *     @def TEGRA186_CLK_MAUD
+ *     @def TEGRA186_CLK_PLL_A_OUT0
+ *     @def TEGRA186_CLK_SPDIF_DOUBLER
+ *     @def TEGRA186_CLK_SPDIF_IN
+ *     @def TEGRA186_CLK_SPDIF_OUT
+ *     @def TEGRA186_CLK_SYNC_DMIC1
+ *     @def TEGRA186_CLK_SYNC_DMIC2
+ *     @def TEGRA186_CLK_SYNC_DMIC3
+ *     @def TEGRA186_CLK_SYNC_DMIC4
+ *     @def TEGRA186_CLK_SYNC_DMIC5
+ *     @def TEGRA186_CLK_SYNC_DSPK1
+ *     @def TEGRA186_CLK_SYNC_DSPK2
+ *     @def TEGRA186_CLK_SYNC_I2S1
+ *     @def TEGRA186_CLK_SYNC_I2S2
+ *     @def TEGRA186_CLK_SYNC_I2S3
+ *     @def TEGRA186_CLK_SYNC_I2S4
+ *     @def TEGRA186_CLK_SYNC_I2S5
+ *     @def TEGRA186_CLK_SYNC_I2S6
+ *     @def TEGRA186_CLK_SYNC_SPDIF
+ *   @}
+ *
+ *   @defgroup uart_clks UART clocks
+ *   @{
+ *     @def TEGRA186_CLK_AON_UART_FST_MIPI_CAL
+ *     @def TEGRA186_CLK_UARTA
+ *     @def TEGRA186_CLK_UARTB
+ *     @def TEGRA186_CLK_UARTC
+ *     @def TEGRA186_CLK_UARTD
+ *     @def TEGRA186_CLK_UARTE
+ *     @def TEGRA186_CLK_UARTF
+ *     @def TEGRA186_CLK_UARTG
+ *     @def TEGRA186_CLK_UART_FST_MIPI_CAL
+ *   @}
+ *
+ *   @defgroup i2c_clks I2C clocks
+ *   @{
+ *     @def TEGRA186_CLK_AON_I2C_SLOW
+ *     @def TEGRA186_CLK_I2C1
+ *     @def TEGRA186_CLK_I2C2
+ *     @def TEGRA186_CLK_I2C3
+ *     @def TEGRA186_CLK_I2C4
+ *     @def TEGRA186_CLK_I2C5
+ *     @def TEGRA186_CLK_I2C6
+ *     @def TEGRA186_CLK_I2C8
+ *     @def TEGRA186_CLK_I2C9
+ *     @def TEGRA186_CLK_I2C1
+ *     @def TEGRA186_CLK_I2C12
+ *     @def TEGRA186_CLK_I2C13
+ *     @def TEGRA186_CLK_I2C14
+ *     @def TEGRA186_CLK_I2C_SLOW
+ *     @def TEGRA186_CLK_VI_I2C
+ *   @}
+ *
+ *   @defgroup spi_clks SPI clocks
+ *   @{
+ *     @def TEGRA186_CLK_SPI1
+ *     @def TEGRA186_CLK_SPI2
+ *     @def TEGRA186_CLK_SPI3
+ *     @def TEGRA186_CLK_SPI4
+ *   @}
+ *
+ *   @defgroup storage storage related clocks
+ *   @{
+ *     @def TEGRA186_CLK_SATA
+ *     @def TEGRA186_CLK_SATA_OOB
+ *     @def TEGRA186_CLK_SATA_IOBIST
+ *     @def TEGRA186_CLK_SDMMC_LEGACY_TM
+ *     @def TEGRA186_CLK_SDMMC1
+ *     @def TEGRA186_CLK_SDMMC2
+ *     @def TEGRA186_CLK_SDMMC3
+ *     @def TEGRA186_CLK_SDMMC4
+ *     @def TEGRA186_CLK_QSPI
+ *     @def TEGRA186_CLK_QSPI_OUT
+ *     @def TEGRA186_CLK_UFSDEV_REF
+ *     @def TEGRA186_CLK_UFSHC
+ *   @}
+ *
+ *   @defgroup pwm_clks PWM clocks
+ *   @{
+ *     @def TEGRA186_CLK_PWM1
+ *     @def TEGRA186_CLK_PWM2
+ *     @def TEGRA186_CLK_PWM3
+ *     @def TEGRA186_CLK_PWM4
+ *     @def TEGRA186_CLK_PWM5
+ *     @def TEGRA186_CLK_PWM6
+ *     @def TEGRA186_CLK_PWM7
+ *     @def TEGRA186_CLK_PWM8
+ *   @}
+ *
+ *   @defgroup plls PLLs and related clocks
+ *   @{
+ *     @def TEGRA186_CLK_PLLREFE_OUT_GATED
+ *     @def TEGRA186_CLK_PLLREFE_OUT1
+ *     @def TEGRA186_CLK_PLLD_OUT1
+ *     @def TEGRA186_CLK_PLLP_OUT0
+ *     @def TEGRA186_CLK_PLLP_OUT5
+ *     @def TEGRA186_CLK_PLLA
+ *     @def TEGRA186_CLK_PLLE_PWRSEQ
+ *     @def TEGRA186_CLK_PLLA_OUT1
+ *     @def TEGRA186_CLK_PLLREFE_REF
+ *     @def TEGRA186_CLK_UPHY_PLL0_PWRSEQ
+ *     @def TEGRA186_CLK_UPHY_PLL1_PWRSEQ
+ *     @def TEGRA186_CLK_PLLREFE_PLLE_PASSTHROUGH
+ *     @def TEGRA186_CLK_PLLREFE_PEX
+ *     @def TEGRA186_CLK_PLLREFE_IDDQ
+ *     @def TEGRA186_CLK_PLLC_OUT_AON
+ *     @def TEGRA186_CLK_PLLC_OUT_ISP
+ *     @def TEGRA186_CLK_PLLC_OUT_VE
+ *     @def TEGRA186_CLK_PLLC4_OUT
+ *     @def TEGRA186_CLK_PLLREFE_OUT
+ *     @def TEGRA186_CLK_PLLREFE_PLL_REF
+ *     @def TEGRA186_CLK_PLLE
+ *     @def TEGRA186_CLK_PLLC
+ *     @def TEGRA186_CLK_PLLP
+ *     @def TEGRA186_CLK_PLLD
+ *     @def TEGRA186_CLK_PLLD2
+ *     @def TEGRA186_CLK_PLLREFE_VCO
+ *     @def TEGRA186_CLK_PLLC2
+ *     @def TEGRA186_CLK_PLLC3
+ *     @def TEGRA186_CLK_PLLDP
+ *     @def TEGRA186_CLK_PLLC4_VCO
+ *     @def TEGRA186_CLK_PLLA1
+ *     @def TEGRA186_CLK_PLLNVCSI
+ *     @def TEGRA186_CLK_PLLDISPHUB
+ *     @def TEGRA186_CLK_PLLD3
+ *     @def TEGRA186_CLK_PLLBPMPCAM
+ *     @def TEGRA186_CLK_PLLAON
+ *     @def TEGRA186_CLK_PLLU
+ *     @def TEGRA186_CLK_PLLC4_VCO_DIV2
+ *     @def TEGRA186_CLK_PLL_REF
+ *     @def TEGRA186_CLK_PLLREFE_OUT1_DIV5
+ *     @def TEGRA186_CLK_UTMIP_PLL_PWRSEQ
+ *     @def TEGRA186_CLK_PLL_U_48M
+ *     @def TEGRA186_CLK_PLL_U_480M
+ *     @def TEGRA186_CLK_PLLC4_OUT0
+ *     @def TEGRA186_CLK_PLLC4_OUT1
+ *     @def TEGRA186_CLK_PLLC4_OUT2
+ *     @def TEGRA186_CLK_PLLC4_OUT_MUX
+ *     @def TEGRA186_CLK_DFLLDISP_DIV
+ *     @def TEGRA186_CLK_PLLDISPHUB_DIV
+ *     @def TEGRA186_CLK_PLLP_DIV8
+ *   @}
+ *
+ *   @defgroup nafll_clks NAFLL clock sources
+ *   @{
+ *     @def TEGRA186_CLK_NAFLL_AXI_CBB
+ *     @def TEGRA186_CLK_NAFLL_BCPU
+ *     @def TEGRA186_CLK_NAFLL_BPMP
+ *     @def TEGRA186_CLK_NAFLL_DISP
+ *     @def TEGRA186_CLK_NAFLL_GPU
+ *     @def TEGRA186_CLK_NAFLL_ISP
+ *     @def TEGRA186_CLK_NAFLL_MCPU
+ *     @def TEGRA186_CLK_NAFLL_NVDEC
+ *     @def TEGRA186_CLK_NAFLL_NVENC
+ *     @def TEGRA186_CLK_NAFLL_NVJPG
+ *     @def TEGRA186_CLK_NAFLL_SCE
+ *     @def TEGRA186_CLK_NAFLL_SE
+ *     @def TEGRA186_CLK_NAFLL_TSEC
+ *     @def TEGRA186_CLK_NAFLL_TSECB
+ *     @def TEGRA186_CLK_NAFLL_VI
+ *     @def TEGRA186_CLK_NAFLL_VIC
+ *   @}
+ *
+ *   @defgroup mphy MPHY related clocks
+ *   @{
+ *     @def TEGRA186_CLK_MPHY_L0_RX_SYMB
+ *     @def TEGRA186_CLK_MPHY_L0_RX_LS_BIT
+ *     @def TEGRA186_CLK_MPHY_L0_TX_SYMB
+ *     @def TEGRA186_CLK_MPHY_L0_TX_LS_3XBIT
+ *     @def TEGRA186_CLK_MPHY_L0_RX_ANA
+ *     @def TEGRA186_CLK_MPHY_L1_RX_ANA
+ *     @def TEGRA186_CLK_MPHY_IOBIST
+ *     @def TEGRA186_CLK_MPHY_TX_1MHZ_REF
+ *     @def TEGRA186_CLK_MPHY_CORE_PLL_FIXED
+ *   @}
+ *
+ *   @defgroup eavb EAVB related clocks
+ *   @{
+ *     @def TEGRA186_CLK_EQOS_AXI
+ *     @def TEGRA186_CLK_EQOS_PTP_REF
+ *     @def TEGRA186_CLK_EQOS_RX
+ *     @def TEGRA186_CLK_EQOS_RX_INPUT
+ *     @def TEGRA186_CLK_EQOS_TX
+ *   @}
+ *
+ *   @defgroup usb USB related clocks
+ *   @{
+ *     @def TEGRA186_CLK_PEX_USB_PAD0_MGMT
+ *     @def TEGRA186_CLK_PEX_USB_PAD1_MGMT
+ *     @def TEGRA186_CLK_HSIC_TRK
+ *     @def TEGRA186_CLK_USB2_TRK
+ *     @def TEGRA186_CLK_USB2_HSIC_TRK
+ *     @def TEGRA186_CLK_XUSB_CORE_SS
+ *     @def TEGRA186_CLK_XUSB_CORE_DEV
+ *     @def TEGRA186_CLK_XUSB_FALCON
+ *     @def TEGRA186_CLK_XUSB_FS
+ *     @def TEGRA186_CLK_XUSB
+ *     @def TEGRA186_CLK_XUSB_DEV
+ *     @def TEGRA186_CLK_XUSB_HOST
+ *     @def TEGRA186_CLK_XUSB_SS
+ *   @}
+ *
+ *   @defgroup bigblock compute block related clocks
+ *   @{
+ *     @def TEGRA186_CLK_GPCCLK
+ *     @def TEGRA186_CLK_GPC2CLK
+ *     @def TEGRA186_CLK_GPU
+ *     @def TEGRA186_CLK_HOST1X
+ *     @def TEGRA186_CLK_ISP
+ *     @def TEGRA186_CLK_NVDEC
+ *     @def TEGRA186_CLK_NVENC
+ *     @def TEGRA186_CLK_NVJPG
+ *     @def TEGRA186_CLK_SE
+ *     @def TEGRA186_CLK_TSEC
+ *     @def TEGRA186_CLK_TSECB
+ *     @def TEGRA186_CLK_VIC
+ *   @}
+ *
+ *   @defgroup can CAN bus related clocks
+ *   @{
+ *     @def TEGRA186_CLK_CAN1
+ *     @def TEGRA186_CLK_CAN1_HOST
+ *     @def TEGRA186_CLK_CAN2
+ *     @def TEGRA186_CLK_CAN2_HOST
+ *   @}
+ *
+ *   @defgroup system basic system clocks
+ *   @{
+ *     @def TEGRA186_CLK_ACTMON
+ *     @def TEGRA186_CLK_AON_APB
+ *     @def TEGRA186_CLK_AON_CPU_NIC
+ *     @def TEGRA186_CLK_AON_NIC
+ *     @def TEGRA186_CLK_AXI_CBB
+ *     @def TEGRA186_CLK_BPMP_APB
+ *     @def TEGRA186_CLK_BPMP_CPU_NIC
+ *     @def TEGRA186_CLK_BPMP_NIC_RATE
+ *     @def TEGRA186_CLK_CLK_M
+ *     @def TEGRA186_CLK_EMC
+ *     @def TEGRA186_CLK_MSS_ENCRYPT
+ *     @def TEGRA186_CLK_SCE_APB
+ *     @def TEGRA186_CLK_SCE_CPU_NIC
+ *     @def TEGRA186_CLK_SCE_NIC
+ *     @def TEGRA186_CLK_TSC
+ *   @}
+ *
+ *   @defgroup pcie_clks PCIe related clocks
+ *   @{
+ *     @def TEGRA186_CLK_AFI
+ *     @def TEGRA186_CLK_PCIE
+ *     @def TEGRA186_CLK_PCIE2_IOBIST
+ *     @def TEGRA186_CLK_PCIERX0
+ *     @def TEGRA186_CLK_PCIERX1
+ *     @def TEGRA186_CLK_PCIERX2
+ *     @def TEGRA186_CLK_PCIERX3
+ *     @def TEGRA186_CLK_PCIERX4
+ *   @}
+ */
+
+/** @brief output of gate CLK_ENB_FUSE */
+#define TEGRA186_CLK_FUSE 0
+/**
+ * @brief It's not what you think
+ * @details output of gate CLK_ENB_GPU. This output connects to the GPU
+ * pwrclk. @warning: This is almost certainly not the clock you think
+ * it is. If you're looking for the clock of the graphics engine, see
+ * TEGRA186_GPCCLK
+ */
+#define TEGRA186_CLK_GPU 1
+/** @brief output of gate CLK_ENB_PCIE */
+#define TEGRA186_CLK_PCIE 3
+/** @brief output of the divider IPFS_CLK_DIVISOR */
+#define TEGRA186_CLK_AFI 4
+/** @brief output of gate CLK_ENB_PCIE2_IOBIST */
+#define TEGRA186_CLK_PCIE2_IOBIST 5
+/** @brief output of gate CLK_ENB_PCIERX0*/
+#define TEGRA186_CLK_PCIERX0 6
+/** @brief output of gate CLK_ENB_PCIERX1*/
+#define TEGRA186_CLK_PCIERX1 7
+/** @brief output of gate CLK_ENB_PCIERX2*/
+#define TEGRA186_CLK_PCIERX2 8
+/** @brief output of gate CLK_ENB_PCIERX3*/
+#define TEGRA186_CLK_PCIERX3 9
+/** @brief output of gate CLK_ENB_PCIERX4*/
+#define TEGRA186_CLK_PCIERX4 10
+/** @brief output branch of PLL_C for ISP, controlled by gate CLK_ENB_PLLC_OUT_ISP */
+#define TEGRA186_CLK_PLLC_OUT_ISP 11
+/** @brief output branch of PLL_C for VI, controlled by gate CLK_ENB_PLLC_OUT_VE */
+#define TEGRA186_CLK_PLLC_OUT_VE 12
+/** @brief output branch of PLL_C for AON domain, controlled by gate CLK_ENB_PLLC_OUT_AON */
+#define TEGRA186_CLK_PLLC_OUT_AON 13
+/** @brief output of gate CLK_ENB_SOR_SAFE */
+#define TEGRA186_CLK_SOR_SAFE 39
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2S2 */
+#define TEGRA186_CLK_I2S2 42
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2S3 */
+#define TEGRA186_CLK_I2S3 43
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SPDF_IN */
+#define TEGRA186_CLK_SPDIF_IN 44
+/** @brief output of gate CLK_ENB_SPDIF_DOUBLER */
+#define TEGRA186_CLK_SPDIF_DOUBLER 45
+/**  @clkdesc{spi_clks, out, mux, CLK_RST_CONTROLLER_CLK_SOURCE_SPI3} */
+#define TEGRA186_CLK_SPI3 46
+/** @clkdesc{i2c_clks, out, mux, CLK_RST_CONTROLLER_CLK_SOURCE_I2C1} */
+#define TEGRA186_CLK_I2C1 47
+/** @clkdesc{i2c_clks, out, mux, CLK_RST_CONTROLLER_CLK_SOURCE_I2C5} */
+#define TEGRA186_CLK_I2C5 48
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SPI1 */
+#define TEGRA186_CLK_SPI1 49
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_ISP */
+#define TEGRA186_CLK_ISP 50
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_VI */
+#define TEGRA186_CLK_VI 51
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SDMMC1 */
+#define TEGRA186_CLK_SDMMC1 52
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SDMMC2 */
+#define TEGRA186_CLK_SDMMC2 53
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SDMMC4 */
+#define TEGRA186_CLK_SDMMC4 54
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTA */
+#define TEGRA186_CLK_UARTA 55
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTB */
+#define TEGRA186_CLK_UARTB 56
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_HOST1X */
+#define TEGRA186_CLK_HOST1X 57
+/**
+ * @brief controls the EMC clock frequency.
+ * @details Doing a clk_set_rate on this clock will select the
+ * appropriate clock source, program the source rate and execute a
+ * specific sequence to switch to the new clock source for both memory
+ * controllers. This can be used to control the balance between memory
+ * throughput and memory controller power.
+ */
+#define TEGRA186_CLK_EMC 58
+/* @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_EXTPERIPH4 */
+#define TEGRA186_CLK_EXTPERIPH4 73
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SPI4 */
+#define TEGRA186_CLK_SPI4 74
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C3 */
+#define TEGRA186_CLK_I2C3 75
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SDMMC3 */
+#define TEGRA186_CLK_SDMMC3 76
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTD */
+#define TEGRA186_CLK_UARTD 77
+/** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2S1 */
+#define TEGRA186_CLK_I2S1 79
+/** output of gate CLK_ENB_DTV */
+#define TEGRA186_CLK_DTV 80
+/** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_TSEC */
+#define TEGRA186_CLK_TSEC 81
+/** @brief output of gate CLK_ENB_DP2 */
+#define TEGRA186_CLK_DP2 82
+/** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2S4 */
+#define TEGRA186_CLK_I2S4 84
+/** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2S5 */
+#define TEGRA186_CLK_I2S5 85
+/** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C4 */
+#define TEGRA186_CLK_I2C4 86
+/** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_AHUB */
+#define TEGRA186_CLK_AHUB 87
+/** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_HDA2CODEC_2X */
+#define TEGRA186_CLK_HDA2CODEC_2X 88
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_EXTPERIPH1 */
+#define TEGRA186_CLK_EXTPERIPH1 89
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_EXTPERIPH2 */
+#define TEGRA186_CLK_EXTPERIPH2 90
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_EXTPERIPH3 */
+#define TEGRA186_CLK_EXTPERIPH3 91
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C_SLOW */
+#define TEGRA186_CLK_I2C_SLOW 92
+/** @brief output of the SOR1_CLK_SRC mux in CLK_RST_CONTROLLER_CLK_SOURCE_SOR1 */
+#define TEGRA186_CLK_SOR1 93
+/** @brief output of gate CLK_ENB_CEC */
+#define TEGRA186_CLK_CEC 94
+/** @brief output of gate CLK_ENB_DPAUX1 */
+#define TEGRA186_CLK_DPAUX1 95
+/** @brief output of gate CLK_ENB_DPAUX */
+#define TEGRA186_CLK_DPAUX 96
+/** @brief output of the SOR0_CLK_SRC mux in CLK_RST_CONTROLLER_CLK_SOURCE_SOR0 */
+#define TEGRA186_CLK_SOR0 97
+/** @brief output of gate CLK_ENB_HDA2HDMICODEC */
+#define TEGRA186_CLK_HDA2HDMICODEC 98
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SATA */
+#define TEGRA186_CLK_SATA 99
+/** @brief output of gate CLK_ENB_SATA_OOB */
+#define TEGRA186_CLK_SATA_OOB 100
+/** @brief output of gate CLK_ENB_SATA_IOBIST */
+#define TEGRA186_CLK_SATA_IOBIST 101
+/** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_HDA */
+#define TEGRA186_CLK_HDA 102
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SE */
+#define TEGRA186_CLK_SE 103
+/** @brief output of gate CLK_ENB_APB2APE */
+#define TEGRA186_CLK_APB2APE 104
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_APE */
+#define TEGRA186_CLK_APE 105
+/** @brief output of gate CLK_ENB_IQC1 */
+#define TEGRA186_CLK_IQC1 106
+/** @brief output of gate CLK_ENB_IQC2 */
+#define TEGRA186_CLK_IQC2 107
+/** divide by 2 version of TEGRA186_CLK_PLLREFE_VCO */
+#define TEGRA186_CLK_PLLREFE_OUT 108
+/** @brief output of gate CLK_ENB_PLLREFE_PLL_REF */
+#define TEGRA186_CLK_PLLREFE_PLL_REF 109
+/** @brief output of gate CLK_ENB_PLLC4_OUT */
+#define TEGRA186_CLK_PLLC4_OUT 110
+/** @brief output of mux xusb_core_clk_switch on page 67 of T186_Clocks_IAS.doc */
+#define TEGRA186_CLK_XUSB 111
+/** controls xusb_dev_ce signal on page 66 and 67 of T186_Clocks_IAS.doc */
+#define TEGRA186_CLK_XUSB_DEV 112
+/** controls xusb_host_ce signal on page 67 of T186_Clocks_IAS.doc */
+#define TEGRA186_CLK_XUSB_HOST 113
+/** controls xusb_ss_ce signal on page 67 of T186_Clocks_IAS.doc */
+#define TEGRA186_CLK_XUSB_SS 114
+/** @brief output of gate CLK_ENB_DSI */
+#define TEGRA186_CLK_DSI 115
+/** @brief output of gate CLK_ENB_MIPI_CAL */
+#define TEGRA186_CLK_MIPI_CAL 116
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DSIA_LP */
+#define TEGRA186_CLK_DSIA_LP 117
+/** @brief output of gate CLK_ENB_DSIB */
+#define TEGRA186_CLK_DSIB 118
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DSIB_LP */
+#define TEGRA186_CLK_DSIB_LP 119
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DMIC1 */
+#define TEGRA186_CLK_DMIC1 122
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DMIC2 */
+#define TEGRA186_CLK_DMIC2 123
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_AUD_MCLK */
+#define TEGRA186_CLK_AUD_MCLK 124
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C6 */
+#define TEGRA186_CLK_I2C6 125
+/**output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UART_FST_MIPI_CAL */
+#define TEGRA186_CLK_UART_FST_MIPI_CAL 126
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_VIC */
+#define TEGRA186_CLK_VIC 127
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SDMMC_LEGACY_TM */
+#define TEGRA186_CLK_SDMMC_LEGACY_TM 128
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVDEC */
+#define TEGRA186_CLK_NVDEC 129
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVJPG */
+#define TEGRA186_CLK_NVJPG 130
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVENC */
+#define TEGRA186_CLK_NVENC 131
+/** @brief output of the QSPI_CLK_SRC mux in CLK_RST_CONTROLLER_CLK_SOURCE_QSPI */
+#define TEGRA186_CLK_QSPI 132
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_VI_I2C */
+#define TEGRA186_CLK_VI_I2C 133
+/** @brief output of gate CLK_ENB_HSIC_TRK */
+#define TEGRA186_CLK_HSIC_TRK 134
+/** @brief output of gate CLK_ENB_USB2_TRK */
+#define TEGRA186_CLK_USB2_TRK 135
+/** output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_MAUD */
+#define TEGRA186_CLK_MAUD 136
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_TSECB */
+#define TEGRA186_CLK_TSECB 137
+/** @brief output of gate CLK_ENB_ADSP */
+#define TEGRA186_CLK_ADSP 138
+/** @brief output of gate CLK_ENB_ADSPNEON */
+#define TEGRA186_CLK_ADSPNEON 139
+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_MPHY_L0_RX_LS_SYMB */
+#define TEGRA186_CLK_MPHY_L0_RX_SYMB 140
+/** @brief output of gate CLK_ENB_MPHY_L0_RX_LS_BIT */
+#define TEGRA186_CLK_MPHY_L0_RX_LS_BIT 141
+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_MPHY_L0_TX_LS_SYMB */
+#define TEGRA186_CLK_MPHY_L0_TX_SYMB 142
+/** @brief output of gate CLK_ENB_MPHY_L0_TX_LS_3XBIT */
+#define TEGRA186_CLK_MPHY_L0_TX_LS_3XBIT 143
+/** @brief output of gate CLK_ENB_MPHY_L0_RX_ANA */
+#define TEGRA186_CLK_MPHY_L0_RX_ANA 144
+/** @brief output of gate CLK_ENB_MPHY_L1_RX_ANA */
+#define TEGRA186_CLK_MPHY_L1_RX_ANA 145
+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_MPHY_IOBIST */
+#define TEGRA186_CLK_MPHY_IOBIST 146
+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_MPHY_TX_1MHZ_REF */
+#define TEGRA186_CLK_MPHY_TX_1MHZ_REF 147
+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_MPHY_CORE_PLL_FIXED */
+#define TEGRA186_CLK_MPHY_CORE_PLL_FIXED 148
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_AXI_CBB */
+#define TEGRA186_CLK_AXI_CBB 149
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DMIC3 */
+#define TEGRA186_CLK_DMIC3 150
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DMIC4 */
+#define TEGRA186_CLK_DMIC4 151
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DSPK1 */
+#define TEGRA186_CLK_DSPK1 152
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DSPK2 */
+#define TEGRA186_CLK_DSPK2 153
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C6 */
+#define TEGRA186_CLK_I2S6 154
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVDISPLAY_P0 */
+#define TEGRA186_CLK_NVDISPLAY_P0 155
+/** @brief output of the NVDISPLAY_DISP_CLK_SRC mux in CLK_RST_CONTROLLER_CLK_SOURCE_NVDISPLAY_DISP */
+#define TEGRA186_CLK_NVDISPLAY_DISP 156
+/** @brief output of gate CLK_ENB_NVDISPLAY_DSC */
+#define TEGRA186_CLK_NVDISPLAY_DSC 157
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVDISPLAYHUB */
+#define TEGRA186_CLK_NVDISPLAYHUB 158
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVDISPLAY_P1 */
+#define TEGRA186_CLK_NVDISPLAY_P1 159
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVDISPLAY_P2 */
+#define TEGRA186_CLK_NVDISPLAY_P2 160
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_TACH */
+#define TEGRA186_CLK_TACH 166
+/** @brief output of gate CLK_ENB_EQOS */
+#define TEGRA186_CLK_EQOS_AXI 167
+/** @brief output of gate CLK_ENB_EQOS_RX */
+#define TEGRA186_CLK_EQOS_RX 168
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UFSHC_CG_SYS */
+#define TEGRA186_CLK_UFSHC 178
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UFSDEV_REF */
+#define TEGRA186_CLK_UFSDEV_REF 179
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVCSI */
+#define TEGRA186_CLK_NVCSI 180
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_NVCSILP */
+#define TEGRA186_CLK_NVCSILP 181
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C7 */
+#define TEGRA186_CLK_I2C7 182
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C9 */
+#define TEGRA186_CLK_I2C9 183
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C12 */
+#define TEGRA186_CLK_I2C12 184
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C13 */
+#define TEGRA186_CLK_I2C13 185
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C14 */
+#define TEGRA186_CLK_I2C14 186
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_PWM1 */
+#define TEGRA186_CLK_PWM1 187
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_PWM2 */
+#define TEGRA186_CLK_PWM2 188
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_PWM3 */
+#define TEGRA186_CLK_PWM3 189
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_PWM5 */
+#define TEGRA186_CLK_PWM5 190
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_PWM6 */
+#define TEGRA186_CLK_PWM6 191
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_PWM7 */
+#define TEGRA186_CLK_PWM7 192
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_PWM8 */
+#define TEGRA186_CLK_PWM8 193
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTE */
+#define TEGRA186_CLK_UARTE 194
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTF */
+#define TEGRA186_CLK_UARTF 195
+/** @deprecated */
+#define TEGRA186_CLK_DBGAPB 196
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_BPMP_CPU_NIC */
+#define TEGRA186_CLK_BPMP_CPU_NIC 197
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_BPMP_APB */
+#define TEGRA186_CLK_BPMP_APB 199
+/** @brief output of mux controlled by TEGRA186_CLK_SOC_ACTMON */
+#define TEGRA186_CLK_ACTMON 201
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_AON_CPU_NIC */
+#define TEGRA186_CLK_AON_CPU_NIC 208
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_CAN1 */
+#define TEGRA186_CLK_CAN1 210
+/** @brief output of gate CLK_ENB_CAN1_HOST */
+#define TEGRA186_CLK_CAN1_HOST 211
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_CAN2 */
+#define TEGRA186_CLK_CAN2 212
+/** @brief output of gate CLK_ENB_CAN2_HOST */
+#define TEGRA186_CLK_CAN2_HOST 213
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_AON_APB */
+#define TEGRA186_CLK_AON_APB 214
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTC */
+#define TEGRA186_CLK_UARTC 215
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_UARTG */
+#define TEGRA186_CLK_UARTG 216
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_AON_UART_FST_MIPI_CAL */
+#define TEGRA186_CLK_AON_UART_FST_MIPI_CAL 217
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C2 */
+#define TEGRA186_CLK_I2C2 218
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C8 */
+#define TEGRA186_CLK_I2C8 219
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_I2C10 */
+#define TEGRA186_CLK_I2C10 220
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_AON_I2C_SLOW */
+#define TEGRA186_CLK_AON_I2C_SLOW 221
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SPI2 */
+#define TEGRA186_CLK_SPI2 222
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DMIC5 */
+#define TEGRA186_CLK_DMIC5 223
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_AON_TOUCH */
+#define TEGRA186_CLK_AON_TOUCH 224
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_PWM4 */
+#define TEGRA186_CLK_PWM4 225
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_TSC. This clock object is read only and is used for all timers in the system. */
+#define TEGRA186_CLK_TSC 226
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_MSS_ENCRYPT */
+#define TEGRA186_CLK_MSS_ENCRYPT 227
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SCE_CPU_NIC */
+#define TEGRA186_CLK_SCE_CPU_NIC 228
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SCE_APB */
+#define TEGRA186_CLK_SCE_APB 230
+/** @brief output of gate CLK_ENB_DSIC */
+#define TEGRA186_CLK_DSIC 231
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DSIC_LP */
+#define TEGRA186_CLK_DSIC_LP 232
+/** @brief output of gate CLK_ENB_DSID */
+#define TEGRA186_CLK_DSID 233
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_DSID_LP */
+#define TEGRA186_CLK_DSID_LP 234
+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_PEX_SATA_USB_RX_BYP */
+#define TEGRA186_CLK_PEX_SATA_USB_RX_BYP 236
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_CLK_SOURCE_SPDIF_OUT */
+#define TEGRA186_CLK_SPDIF_OUT 238
+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_EQOS_PTP_REF_CLK_0 */
+#define TEGRA186_CLK_EQOS_PTP_REF 239
+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_EQOS_TX_CLK */
+#define TEGRA186_CLK_EQOS_TX 240
+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_USB2_HSIC_TRK */
+#define TEGRA186_CLK_USB2_HSIC_TRK 241
+/** @brief output of mux xusb_ss_clk_switch on page 66 of T186_Clocks_IAS.doc */
+#define TEGRA186_CLK_XUSB_CORE_SS 242
+/** @brief output of mux xusb_core_dev_clk_switch on page 67 of T186_Clocks_IAS.doc */
+#define TEGRA186_CLK_XUSB_CORE_DEV 243
+/** @brief output of mux xusb_core_falcon_clk_switch on page 67 of T186_Clocks_IAS.doc */
+#define TEGRA186_CLK_XUSB_FALCON 244
+/** @brief output of mux xusb_fs_clk_switch on page 66 of T186_Clocks_IAS.doc */
+#define TEGRA186_CLK_XUSB_FS 245
+/** @brief output of the divider CLK_RST_CONTROLLER_PLLA_OUT */
+#define TEGRA186_CLK_PLL_A_OUT0 246
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_I2S1 */
+#define TEGRA186_CLK_SYNC_I2S1 247
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_I2S2 */
+#define TEGRA186_CLK_SYNC_I2S2 248
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_I2S3 */
+#define TEGRA186_CLK_SYNC_I2S3 249
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_I2S4 */
+#define TEGRA186_CLK_SYNC_I2S4 250
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_I2S5 */
+#define TEGRA186_CLK_SYNC_I2S5 251
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_I2S6 */
+#define TEGRA186_CLK_SYNC_I2S6 252
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_DSPK1 */
+#define TEGRA186_CLK_SYNC_DSPK1 253
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_DSPK2 */
+#define TEGRA186_CLK_SYNC_DSPK2 254
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_DMIC1 */
+#define TEGRA186_CLK_SYNC_DMIC1 255
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_DMIC2 */
+#define TEGRA186_CLK_SYNC_DMIC2 256
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_DMIC3 */
+#define TEGRA186_CLK_SYNC_DMIC3 257
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_DMIC4 */
+#define TEGRA186_CLK_SYNC_DMIC4 259
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_AUDIO_SYNC_CLK_SPDIF */
+#define TEGRA186_CLK_SYNC_SPDIF 260
+/** @brief output of gate CLK_ENB_PLLREFE_OUT */
+#define TEGRA186_CLK_PLLREFE_OUT_GATED 261
+/** @brief output of the divider PLLREFE_DIVP in CLK_RST_CONTROLLER_PLLREFE_BASE. PLLREFE has 2 outputs:
+  *      * VCO/pdiv defined by this clock object
+  *      * VCO/2 defined by TEGRA186_CLK_PLLREFE_OUT
+  */
+#define TEGRA186_CLK_PLLREFE_OUT1 262
+#define TEGRA186_CLK_PLLD_OUT1 267
+/** @brief output of the divider PLLP_DIVP in CLK_RST_CONTROLLER_PLLP_BASE */
+#define TEGRA186_CLK_PLLP_OUT0 269
+/** @brief output of the divider CLK_RST_CONTROLLER_PLLP_OUTC */
+#define TEGRA186_CLK_PLLP_OUT5 270
+/** PLL controlled by CLK_RST_CONTROLLER_PLLA_BASE for use by audio clocks */
+#define TEGRA186_CLK_PLLA 271
+/** @brief output of mux controlled by CLK_RST_CONTROLLER_ACLK_BURST_POLICY divided by the divider controlled by ACLK_CLK_DIVISOR in CLK_RST_CONTROLLER_SUPER_ACLK_DIVIDER */
+#define TEGRA186_CLK_ACLK 273
+/** fixed 48MHz clock divided down from TEGRA186_CLK_PLL_U */
+#define TEGRA186_CLK_PLL_U_48M 274
+/** fixed 480MHz clock divided down from TEGRA186_CLK_PLL_U */
+#define TEGRA186_CLK_PLL_U_480M 275
+/** @brief output of the divider PLLC4_DIVP in CLK_RST_CONTROLLER_PLLC4_BASE. Output frequency is TEGRA186_CLK_PLLC4_VCO/PLLC4_DIVP */
+#define TEGRA186_CLK_PLLC4_OUT0 276
+/** fixed /3 divider. Output frequency of this clock is TEGRA186_CLK_PLLC4_VCO/3 */
+#define TEGRA186_CLK_PLLC4_OUT1 277
+/** fixed /5 divider. Output frequency of this clock is TEGRA186_CLK_PLLC4_VCO/5 */
+#define TEGRA186_CLK_PLLC4_OUT2 278
+/** @brief output of mux controlled by PLLC4_CLK_SEL in CLK_RST_CONTROLLER_PLLC4_MISC1 */
+#define TEGRA186_CLK_PLLC4_OUT_MUX 279
+/** @brief output of divider NVDISPLAY_DISP_CLK_DIVISOR in CLK_RST_CONTROLLER_CLK_SOURCE_NVDISPLAY_DISP when DFLLDISP_DIV is selected in NVDISPLAY_DISP_CLK_SRC */
+#define TEGRA186_CLK_DFLLDISP_DIV 284
+/** @brief output of divider NVDISPLAY_DISP_CLK_DIVISOR in CLK_RST_CONTROLLER_CLK_SOURCE_NVDISPLAY_DISP when PLLDISPHUB_DIV is selected in NVDISPLAY_DISP_CLK_SRC */
+#define TEGRA186_CLK_PLLDISPHUB_DIV 285
+/** fixed /8 divider which is used as the input for TEGRA186_CLK_SOR_SAFE */
+#define TEGRA186_CLK_PLLP_DIV8 286
+/** @brief output of divider CLK_RST_CONTROLLER_BPMP_NIC_RATE */
+#define TEGRA186_CLK_BPMP_NIC 287
+/** @brief output of the divider CLK_RST_CONTROLLER_PLLA1_OUT1 */
+#define TEGRA186_CLK_PLL_A_OUT1 288
+/** @deprecated */
+#define TEGRA186_CLK_GPC2CLK 289
+/** A fake clock which must be enabled during KFUSE read operations to ensure adequate VDD_CORE voltage. */
+#define TEGRA186_CLK_KFUSE 293
+/**
+ * @brief controls the PLLE hardware sequencer.
+ * @details This clock only has enable and disable methods. When the
+ * PLLE hw sequencer is enabled, PLLE, will be enabled or disabled by
+ * hw based on the control signals from the PCIe, SATA and XUSB
+ * clocks. When the PLLE hw sequencer is disabled, the state of PLLE
+ * is controlled by sw using clk_enable/clk_disable on
+ * TEGRA186_CLK_PLLE.
+ */
+#define TEGRA186_CLK_PLLE_PWRSEQ 294
+/** fixed 60MHz clock divided down from, TEGRA186_CLK_PLL_U */
+#define TEGRA186_CLK_PLLREFE_REF 295
+/** @brief output of mux controlled by SOR0_CLK_SEL0 and SOR0_CLK_SEL1 in CLK_RST_CONTROLLER_CLK_SOURCE_SOR0 */
+#define TEGRA186_CLK_SOR0_OUT 296
+/** @brief output of mux controlled by SOR1_CLK_SEL0 and SOR1_CLK_SEL1 in CLK_RST_CONTROLLER_CLK_SOURCE_SOR1 */
+#define TEGRA186_CLK_SOR1_OUT 297
+/** @brief fixed /5 divider.  Output frequency of this clock is TEGRA186_CLK_PLLREFE_OUT1/5. Used as input for TEGRA186_CLK_EQOS_AXI */
+#define TEGRA186_CLK_PLLREFE_OUT1_DIV5 298
+/** @brief controls the UTMIP_PLL (aka PLLU) hardware sqeuencer */
+#define TEGRA186_CLK_UTMIP_PLL_PWRSEQ 301
+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_PEX_USB_PAD_PLL0_MGMT */
+#define TEGRA186_CLK_PEX_USB_PAD0_MGMT 302
+/** @brief output of the divider CLK_RST_CONTROLLER_CLK_SOURCE_PEX_USB_PAD_PLL1_MGMT */
+#define TEGRA186_CLK_PEX_USB_PAD1_MGMT 303
+/** @brief controls the UPHY_PLL0 hardware sqeuencer */
+#define TEGRA186_CLK_UPHY_PLL0_PWRSEQ 304
+/** @brief controls the UPHY_PLL1 hardware sqeuencer */
+#define TEGRA186_CLK_UPHY_PLL1_PWRSEQ 305
+/** @brief control for PLLREFE_IDDQ in CLK_RST_CONTROLLER_PLLREFE_MISC so the bypass output even be used when the PLL is disabled */
+#define TEGRA186_CLK_PLLREFE_PLLE_PASSTHROUGH 306
+/** @brief output of the mux controlled by PLLREFE_SEL_CLKIN_PEX in CLK_RST_CONTROLLER_PLLREFE_MISC */
+#define TEGRA186_CLK_PLLREFE_PEX 307
+/** @brief control for PLLREFE_IDDQ in CLK_RST_CONTROLLER_PLLREFE_MISC to turn on the PLL when enabled */
+#define TEGRA186_CLK_PLLREFE_IDDQ 308
+/** @brief output of the divider QSPI_CLK_DIV2_SEL in CLK_RST_CONTROLLER_CLK_SOURCE_QSPI */
+#define TEGRA186_CLK_QSPI_OUT 309
+/**
+ * @brief GPC2CLK-div-2
+ * @details fixed /2 divider. Output frequency is
+ * TEGRA186_CLK_GPC2CLK/2. The frequency of this clock is the
+ * frequency at which the GPU graphics engine runs. */
+#define TEGRA186_CLK_GPCCLK 310
+/** @brief output of divider CLK_RST_CONTROLLER_AON_NIC_RATE */
+#define TEGRA186_CLK_AON_NIC 450
+/** @brief output of divider CLK_RST_CONTROLLER_SCE_NIC_RATE */
+#define TEGRA186_CLK_SCE_NIC 451
+/** Fixed 100MHz PLL for PCIe, SATA and superspeed USB */
+#define TEGRA186_CLK_PLLE 512
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLC_BASE */
+#define TEGRA186_CLK_PLLC 513
+/** Fixed 408MHz PLL for use by peripheral clocks */
+#define TEGRA186_CLK_PLLP 516
+/** @deprecated */
+#define TEGRA186_CLK_PLL_P TEGRA186_CLK_PLLP
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLD_BASE for use by DSI */
+#define TEGRA186_CLK_PLLD 518
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLD2_BASE for use by HDMI or DP */
+#define TEGRA186_CLK_PLLD2 519
+/**
+ * @brief PLL controlled by CLK_RST_CONTROLLER_PLLREFE_BASE.
+ * @details Note that this clock only controls the VCO output, before
+ * the post-divider. See TEGRA186_CLK_PLLREFE_OUT1 for more
+ * information.
+ */
+#define TEGRA186_CLK_PLLREFE_VCO 520
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLC2_BASE */
+#define TEGRA186_CLK_PLLC2 521
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLC3_BASE */
+#define TEGRA186_CLK_PLLC3 522
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLDP_BASE for use as the DP link clock */
+#define TEGRA186_CLK_PLLDP 523
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLC4_BASE */
+#define TEGRA186_CLK_PLLC4_VCO 524
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLA1_BASE for use by audio clocks */
+#define TEGRA186_CLK_PLLA1 525
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLNVCSI_BASE */
+#define TEGRA186_CLK_PLLNVCSI 526
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLDISPHUB_BASE */
+#define TEGRA186_CLK_PLLDISPHUB 527
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLD3_BASE for use by HDMI or DP */
+#define TEGRA186_CLK_PLLD3 528
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLBPMPCAM_BASE */
+#define TEGRA186_CLK_PLLBPMPCAM 531
+/** @brief PLL controlled by CLK_RST_CONTROLLER_PLLAON_BASE for use by IP blocks in the AON domain */
+#define TEGRA186_CLK_PLLAON 532
+/** Fixed frequency 960MHz PLL for USB and EAVB */
+#define TEGRA186_CLK_PLLU 533
+/** fixed /2 divider. Output frequency is TEGRA186_CLK_PLLC4_VCO/2 */
+#define TEGRA186_CLK_PLLC4_VCO_DIV2 535
+/** @brief NAFLL clock source for AXI_CBB */
+#define TEGRA186_CLK_NAFLL_AXI_CBB 564
+/** @brief NAFLL clock source for BPMP */
+#define TEGRA186_CLK_NAFLL_BPMP 565
+/** @brief NAFLL clock source for ISP */
+#define TEGRA186_CLK_NAFLL_ISP 566
+/** @brief NAFLL clock source for NVDEC */
+#define TEGRA186_CLK_NAFLL_NVDEC 567
+/** @brief NAFLL clock source for NVENC */
+#define TEGRA186_CLK_NAFLL_NVENC 568
+/** @brief NAFLL clock source for NVJPG */
+#define TEGRA186_CLK_NAFLL_NVJPG 569
+/** @brief NAFLL clock source for SCE */
+#define TEGRA186_CLK_NAFLL_SCE 570
+/** @brief NAFLL clock source for SE */
+#define TEGRA186_CLK_NAFLL_SE 571
+/** @brief NAFLL clock source for TSEC */
+#define TEGRA186_CLK_NAFLL_TSEC 572
+/** @brief NAFLL clock source for TSECB */
+#define TEGRA186_CLK_NAFLL_TSECB 573
+/** @brief NAFLL clock source for VI */
+#define TEGRA186_CLK_NAFLL_VI 574
+/** @brief NAFLL clock source for VIC */
+#define TEGRA186_CLK_NAFLL_VIC 575
+/** @brief NAFLL clock source for DISP */
+#define TEGRA186_CLK_NAFLL_DISP 576
+/** @brief NAFLL clock source for GPU */
+#define TEGRA186_CLK_NAFLL_GPU 577
+/** @brief NAFLL clock source for M-CPU cluster */
+#define TEGRA186_CLK_NAFLL_MCPU 578
+/** @brief NAFLL clock source for B-CPU cluster */
+#define TEGRA186_CLK_NAFLL_BCPU 579
+/** @brief input from Tegra's CLK_32K_IN pad */
+#define TEGRA186_CLK_CLK_32K 608
+/** @brief output of divider CLK_RST_CONTROLLER_CLK_M_DIVIDE */
+#define TEGRA186_CLK_CLK_M 609
+/** @brief output of divider PLL_REF_DIV in CLK_RST_CONTROLLER_OSC_CTRL */
+#define TEGRA186_CLK_PLL_REF 610
+/** @brief input from Tegra's XTAL_IN */
+#define TEGRA186_CLK_OSC 612
+/** @brief clock recovered from EAVB input */
+#define TEGRA186_CLK_EQOS_RX_INPUT 613
+/** @brief clock recovered from DTV input */
+#define TEGRA186_CLK_DTV_INPUT 614
+/** @brief SOR0 brick output which feeds into SOR0_CLK_SEL mux in CLK_RST_CONTROLLER_CLK_SOURCE_SOR0*/
+#define TEGRA186_CLK_SOR0_PAD_CLKOUT 615
+/** @brief SOR1 brick output which feeds into SOR1_CLK_SEL mux in CLK_RST_CONTROLLER_CLK_SOURCE_SOR1*/
+#define TEGRA186_CLK_SOR1_PAD_CLKOUT 616
+/** @brief clock recovered from I2S1 input */
+#define TEGRA186_CLK_I2S1_SYNC_INPUT 617
+/** @brief clock recovered from I2S2 input */
+#define TEGRA186_CLK_I2S2_SYNC_INPUT 618
+/** @brief clock recovered from I2S3 input */
+#define TEGRA186_CLK_I2S3_SYNC_INPUT 619
+/** @brief clock recovered from I2S4 input */
+#define TEGRA186_CLK_I2S4_SYNC_INPUT 620
+/** @brief clock recovered from I2S5 input */
+#define TEGRA186_CLK_I2S5_SYNC_INPUT 621
+/** @brief clock recovered from I2S6 input */
+#define TEGRA186_CLK_I2S6_SYNC_INPUT 622
+/** @brief clock recovered from SPDIFIN input */
+#define TEGRA186_CLK_SPDIFIN_SYNC_INPUT 623
+
+/**
+ * @brief subject to change
+ * @details maximum clock identifier value plus one.
+ */
+#define TEGRA186_CLK_CLK_MAX 624
+
+/** @} */
+
+#endif
diff --git a/include/dt-bindings/gpio/meson-gxl-gpio.h b/include/dt-bindings/gpio/meson-gxl-gpio.h
new file mode 100644
index 000000000000..684d0d7add1c
--- /dev/null
+++ b/include/dt-bindings/gpio/meson-gxl-gpio.h
@@ -0,0 +1,131 @@
+/*
+ * GPIO definitions for Amlogic Meson GXL SoCs
+ *
+ * Copyright (C) 2016 Endless Mobile, Inc.
+ * Author: Carlo Caione <carlo@endlessm.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _DT_BINDINGS_MESON_GXL_GPIO_H
+#define _DT_BINDINGS_MESON_GXL_GPIO_H
+
+#define	GPIOAO_0	0
+#define	GPIOAO_1	1
+#define	GPIOAO_2	2
+#define	GPIOAO_3	3
+#define	GPIOAO_4	4
+#define	GPIOAO_5	5
+#define	GPIOAO_6	6
+#define	GPIOAO_7	7
+#define	GPIOAO_8	8
+#define	GPIOAO_9	9
+
+#define	GPIOZ_0		0
+#define	GPIOZ_1		1
+#define	GPIOZ_2		2
+#define	GPIOZ_3		3
+#define	GPIOZ_4		4
+#define	GPIOZ_5		5
+#define	GPIOZ_6		6
+#define	GPIOZ_7		7
+#define	GPIOZ_8		8
+#define	GPIOZ_9		9
+#define	GPIOZ_10	10
+#define	GPIOZ_11	11
+#define	GPIOZ_12	12
+#define	GPIOZ_13	13
+#define	GPIOZ_14	14
+#define	GPIOZ_15	15
+#define	GPIOH_0		16
+#define	GPIOH_1		17
+#define	GPIOH_2		18
+#define	GPIOH_3		19
+#define	GPIOH_4		20
+#define	GPIOH_5		21
+#define	GPIOH_6		22
+#define	GPIOH_7		23
+#define	GPIOH_8		24
+#define	GPIOH_9		25
+#define	BOOT_0		26
+#define	BOOT_1		27
+#define	BOOT_2		28
+#define	BOOT_3		29
+#define	BOOT_4		30
+#define	BOOT_5		31
+#define	BOOT_6		32
+#define	BOOT_7		33
+#define	BOOT_8		34
+#define	BOOT_9		35
+#define	BOOT_10		36
+#define	BOOT_11		37
+#define	BOOT_12		38
+#define	BOOT_13		39
+#define	BOOT_14		40
+#define	BOOT_15		41
+#define	CARD_0		42
+#define	CARD_1		43
+#define	CARD_2		44
+#define	CARD_3		45
+#define	CARD_4		46
+#define	CARD_5		47
+#define	CARD_6		48
+#define	GPIODV_0	49
+#define	GPIODV_1	50
+#define	GPIODV_2	51
+#define	GPIODV_3	52
+#define	GPIODV_4	53
+#define	GPIODV_5	54
+#define	GPIODV_6	55
+#define	GPIODV_7	56
+#define	GPIODV_8	57
+#define	GPIODV_9	58
+#define	GPIODV_10	59
+#define	GPIODV_11	60
+#define	GPIODV_12	61
+#define	GPIODV_13	62
+#define	GPIODV_14	63
+#define	GPIODV_15	64
+#define	GPIODV_16	65
+#define	GPIODV_17	66
+#define	GPIODV_18	67
+#define	GPIODV_19	68
+#define	GPIODV_20	69
+#define	GPIODV_21	70
+#define	GPIODV_22	71
+#define	GPIODV_23	72
+#define	GPIODV_24	73
+#define	GPIODV_25	74
+#define	GPIODV_26	75
+#define	GPIODV_27	76
+#define	GPIODV_28	77
+#define	GPIODV_29	78
+#define	GPIOX_0		79
+#define	GPIOX_1		80
+#define	GPIOX_2		81
+#define	GPIOX_3		82
+#define	GPIOX_4		83
+#define	GPIOX_5		84
+#define	GPIOX_6		85
+#define	GPIOX_7		86
+#define	GPIOX_8		87
+#define	GPIOX_9		88
+#define	GPIOX_10	89
+#define	GPIOX_11	90
+#define	GPIOX_12	91
+#define	GPIOX_13	92
+#define	GPIOX_14	93
+#define	GPIOX_15	94
+#define	GPIOX_16	95
+#define	GPIOX_17	96
+#define	GPIOX_18	97
+#define	GPIOCLK_0	98
+#define	GPIOCLK_1	99
+#define	GPIO_TEST_N	100
+
+#endif
diff --git a/include/dt-bindings/mailbox/tegra186-hsp.h b/include/dt-bindings/mailbox/tegra186-hsp.h
new file mode 100644
index 000000000000..f5d66e5f5f10
--- /dev/null
+++ b/include/dt-bindings/mailbox/tegra186-hsp.h
@@ -0,0 +1,24 @@
+/*
+ * This header provides constants for binding nvidia,tegra186-hsp.
+ */
+
+#ifndef _DT_BINDINGS_MAILBOX_TEGRA186_HSP_H
+#define _DT_BINDINGS_MAILBOX_TEGRA186_HSP_H
+
+/*
+ * These define the type of mailbox that is to be used (doorbell, shared
+ * mailbox, shared semaphore or arbitrated semaphore).
+ */
+#define TEGRA_HSP_MBOX_TYPE_DB 0x0
+#define TEGRA_HSP_MBOX_TYPE_SM 0x1
+#define TEGRA_HSP_MBOX_TYPE_SS 0x2
+#define TEGRA_HSP_MBOX_TYPE_AS 0x3
+
+/*
+ * These defines represent the bit associated with the given master ID in the
+ * doorbell registers.
+ */
+#define TEGRA_HSP_DB_MASTER_CCPLEX 17
+#define TEGRA_HSP_DB_MASTER_BPMP 19
+
+#endif
diff --git a/include/dt-bindings/mfd/tps65217.h b/include/dt-bindings/mfd/tps65217.h
new file mode 100644
index 000000000000..cafb9e60cf12
--- /dev/null
+++ b/include/dt-bindings/mfd/tps65217.h
@@ -0,0 +1,26 @@
+/*
+ * This header provides macros for TI TPS65217 DT bindings.
+ *
+ * Copyright (C) 2016 Texas Instruments
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __DT_BINDINGS_TPS65217_H__
+#define __DT_BINDINGS_TPS65217_H__
+
+#define TPS65217_IRQ_USB	0
+#define TPS65217_IRQ_AC		1
+#define TPS65217_IRQ_PB		2
+
+#endif
diff --git a/include/dt-bindings/pinctrl/at91.h b/include/dt-bindings/pinctrl/at91.h
index bbca3d038900..2732d6c0fb39 100644
--- a/include/dt-bindings/pinctrl/at91.h
+++ b/include/dt-bindings/pinctrl/at91.h
@@ -15,6 +15,8 @@
 #define AT91_PINCTRL_DEGLITCH		(1 << 2)
 #define AT91_PINCTRL_PULL_DOWN		(1 << 3)
 #define AT91_PINCTRL_DIS_SCHMIT		(1 << 4)
+#define AT91_PINCTRL_OUTPUT		(1 << 7)
+#define AT91_PINCTRL_OUTPUT_VAL(x)	((x & 0x1) << 8)
 #define AT91_PINCTRL_DEBOUNCE		(1 << 16)
 #define AT91_PINCTRL_DEBOUNCE_VAL(x)	(x << 17)
 
diff --git a/include/dt-bindings/pinctrl/bcm2835.h b/include/dt-bindings/pinctrl/bcm2835.h
index 6f0bc37af39c..e4e4fdf5d38f 100644
--- a/include/dt-bindings/pinctrl/bcm2835.h
+++ b/include/dt-bindings/pinctrl/bcm2835.h
@@ -24,4 +24,9 @@
 #define BCM2835_FSEL_ALT2	6
 #define BCM2835_FSEL_ALT3	7
 
+/* brcm,pull property */
+#define BCM2835_PUD_OFF		0
+#define BCM2835_PUD_DOWN	1
+#define BCM2835_PUD_UP		2
+
 #endif /* __DT_BINDINGS_PINCTRL_BCM2835_H__ */
diff --git a/include/dt-bindings/pinctrl/qcom,pmic-gpio.h b/include/dt-bindings/pinctrl/qcom,pmic-gpio.h
index aafa76cb569d..d33f17c8a515 100644
--- a/include/dt-bindings/pinctrl/qcom,pmic-gpio.h
+++ b/include/dt-bindings/pinctrl/qcom,pmic-gpio.h
@@ -89,6 +89,10 @@
 #define PMA8084_GPIO_S4			2
 #define PMA8084_GPIO_L6			3
 
+#define PM8994_GPIO_VPH			0
+#define PM8994_GPIO_S4			2
+#define PM8994_GPIO_L12			3
+
 /* To be used with "function" */
 #define PMIC_GPIO_FUNC_NORMAL		"normal"
 #define PMIC_GPIO_FUNC_PAIRED		"paired"
diff --git a/include/dt-bindings/pinctrl/qcom,pmic-mpp.h b/include/dt-bindings/pinctrl/qcom,pmic-mpp.h
index a15c1704d0ec..2e360d8f7801 100644
--- a/include/dt-bindings/pinctrl/qcom,pmic-mpp.h
+++ b/include/dt-bindings/pinctrl/qcom,pmic-mpp.h
@@ -65,6 +65,12 @@
 #define PMA8084_MPP_S4			2
 #define PMA8084_MPP_L6			3
 
+#define PM8994_MPP_VPH			0
+/* Only supported for MPP_05-MPP_08 */
+#define PM8994_MPP_L19			1
+#define PM8994_MPP_S4			2
+#define PM8994_MPP_L12			3
+
 /*
  * Analog Input - Set the source for analog input.
  * To be used with "qcom,amux-route" property
diff --git a/include/dt-bindings/pinctrl/rockchip.h b/include/dt-bindings/pinctrl/rockchip.h
index 743e66a95e13..aaec8baaa354 100644
--- a/include/dt-bindings/pinctrl/rockchip.h
+++ b/include/dt-bindings/pinctrl/rockchip.h
@@ -25,6 +25,39 @@
 #define RK_GPIO4	4
 #define RK_GPIO6	6
 
+#define RK_PA0		0
+#define RK_PA1		1
+#define RK_PA2		2
+#define RK_PA3		3
+#define RK_PA4		4
+#define RK_PA5		5
+#define RK_PA6		6
+#define RK_PA7		7
+#define RK_PB0		8
+#define RK_PB1		9
+#define RK_PB2		10
+#define RK_PB3		11
+#define RK_PB4		12
+#define RK_PB5		13
+#define RK_PB6		14
+#define RK_PB7		15
+#define RK_PC0		16
+#define RK_PC1		17
+#define RK_PC2		18
+#define RK_PC3		19
+#define RK_PC4		20
+#define RK_PC5		21
+#define RK_PC6		22
+#define RK_PC7		23
+#define RK_PD0		24
+#define RK_PD1		25
+#define RK_PD2		26
+#define RK_PD3		27
+#define RK_PD4		28
+#define RK_PD5		29
+#define RK_PD6		30
+#define RK_PD7		31
+
 #define RK_FUNC_GPIO	0
 #define RK_FUNC_1	1
 #define RK_FUNC_2	2
diff --git a/include/dt-bindings/power/mt2701-power.h b/include/dt-bindings/power/mt2701-power.h
new file mode 100644
index 000000000000..64cc826d642c
--- /dev/null
+++ b/include/dt-bindings/power/mt2701-power.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2015 MediaTek Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _DT_BINDINGS_POWER_MT2701_POWER_H
+#define _DT_BINDINGS_POWER_MT2701_POWER_H
+
+#define MT2701_POWER_DOMAIN_CONN	0
+#define MT2701_POWER_DOMAIN_DISP	1
+#define MT2701_POWER_DOMAIN_MFG		2
+#define MT2701_POWER_DOMAIN_VDEC	3
+#define MT2701_POWER_DOMAIN_ISP		4
+#define MT2701_POWER_DOMAIN_BDP		5
+#define MT2701_POWER_DOMAIN_ETH		6
+#define MT2701_POWER_DOMAIN_HIF		7
+#define MT2701_POWER_DOMAIN_IFR_MSC	8
+
+#endif /* _DT_BINDINGS_POWER_MT2701_POWER_H */
diff --git a/include/dt-bindings/power/r8a7743-sysc.h b/include/dt-bindings/power/r8a7743-sysc.h
new file mode 100644
index 000000000000..61cfbb2907ea
--- /dev/null
+++ b/include/dt-bindings/power/r8a7743-sysc.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2016 Cogent Embedded Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef __DT_BINDINGS_POWER_R8A7743_SYSC_H__
+#define __DT_BINDINGS_POWER_R8A7743_SYSC_H__
+
+/*
+ * These power domain indices match the numbers of the interrupt bits
+ * representing the power areas in the various Interrupt Registers
+ * (e.g. SYSCISR, Interrupt Status Register)
+ */
+
+#define R8A7743_PD_CA15_CPU0		 0
+#define R8A7743_PD_CA15_CPU1		 1
+#define R8A7743_PD_CA15_SCU		12
+#define R8A7743_PD_SGX			20
+
+/* Always-on power area */
+#define R8A7743_PD_ALWAYS_ON		32
+
+#endif /* __DT_BINDINGS_POWER_R8A7743_SYSC_H__ */
diff --git a/include/dt-bindings/power/r8a7745-sysc.h b/include/dt-bindings/power/r8a7745-sysc.h
new file mode 100644
index 000000000000..1844c1171c04
--- /dev/null
+++ b/include/dt-bindings/power/r8a7745-sysc.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2016 Cogent Embedded Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef __DT_BINDINGS_POWER_R8A7745_SYSC_H__
+#define __DT_BINDINGS_POWER_R8A7745_SYSC_H__
+
+/*
+ * These power domain indices match the numbers of the interrupt bits
+ * representing the power areas in the various Interrupt Registers
+ * (e.g. SYSCISR, Interrupt Status Register)
+ */
+
+#define R8A7745_PD_CA7_CPU0		 5
+#define R8A7745_PD_CA7_CPU1		 6
+#define R8A7745_PD_SGX			20
+#define R8A7745_PD_CA7_SCU		21
+
+/* Always-on power area */
+#define R8A7745_PD_ALWAYS_ON		32
+
+#endif /* __DT_BINDINGS_POWER_R8A7745_SYSC_H__ */
diff --git a/include/dt-bindings/power/tegra186-powergate.h b/include/dt-bindings/power/tegra186-powergate.h
new file mode 100644
index 000000000000..388d6e228dc8
--- /dev/null
+++ b/include/dt-bindings/power/tegra186-powergate.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2015-2016, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _DT_BINDINGS_POWER_TEGRA186_POWERGATE_H
+#define _DT_BINDINGS_POWER_TEGRA186_POWERGATE_H
+
+#define TEGRA186_POWER_DOMAIN_AUD	0
+#define TEGRA186_POWER_DOMAIN_DFD	1
+#define TEGRA186_POWER_DOMAIN_DISP	2
+#define TEGRA186_POWER_DOMAIN_DISPB	3
+#define TEGRA186_POWER_DOMAIN_DISPC	4
+#define TEGRA186_POWER_DOMAIN_ISPA	5
+#define TEGRA186_POWER_DOMAIN_NVDEC	6
+#define TEGRA186_POWER_DOMAIN_NVJPG	7
+#define TEGRA186_POWER_DOMAIN_MPE	8
+#define TEGRA186_POWER_DOMAIN_PCX	9
+#define TEGRA186_POWER_DOMAIN_SAX	10
+#define TEGRA186_POWER_DOMAIN_VE	11
+#define TEGRA186_POWER_DOMAIN_VIC	12
+#define TEGRA186_POWER_DOMAIN_XUSBA	13
+#define TEGRA186_POWER_DOMAIN_XUSBB	14
+#define TEGRA186_POWER_DOMAIN_XUSBC	15
+#define TEGRA186_POWER_DOMAIN_GPU	43
+#define TEGRA186_POWER_DOMAIN_MAX	44
+
+#endif
diff --git a/include/dt-bindings/reset/oxsemi,ox810se.h b/include/dt-bindings/reset/oxsemi,ox810se.h
new file mode 100644
index 000000000000..960c26e4504a
--- /dev/null
+++ b/include/dt-bindings/reset/oxsemi,ox810se.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2016 Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef DT_RESET_OXSEMI_OX810SE_H
+#define DT_RESET_OXSEMI_OX810SE_H
+
+#define RESET_ARM	0
+#define RESET_COPRO	1
+/* Reserved		2 */
+/* Reserved		3 */
+#define RESET_USBHS	4
+#define RESET_USBHSPHY	5
+#define RESET_MAC	6
+#define RESET_PCI	7
+#define RESET_DMA	8
+#define RESET_DPE	9
+#define RESET_DDR	10
+#define RESET_SATA	11
+#define RESET_SATA_LINK	12
+#define RESET_SATA_PHY	13
+ /* Reserved		14 */
+#define RESET_NAND	15
+#define RESET_GPIO	16
+#define RESET_UART1	17
+#define RESET_UART2	18
+#define RESET_MISC	19
+#define RESET_I2S	20
+#define RESET_AHB_MON	21
+#define RESET_UART3	22
+#define RESET_UART4	23
+#define RESET_SGDMA	24
+/* Reserved		25 */
+/* Reserved		26 */
+/* Reserved		27 */
+/* Reserved		28 */
+/* Reserved		29 */
+/* Reserved		30 */
+#define RESET_BUS	31
+
+#endif /* DT_RESET_OXSEMI_OX810SE_H */
diff --git a/include/dt-bindings/reset/oxsemi,ox820.h b/include/dt-bindings/reset/oxsemi,ox820.h
new file mode 100644
index 000000000000..cc6797bf01d8
--- /dev/null
+++ b/include/dt-bindings/reset/oxsemi,ox820.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2016 Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef DT_RESET_OXSEMI_OX820_H
+#define DT_RESET_OXSEMI_OX820_H
+
+#define RESET_SCU	0
+#define RESET_LEON	1
+#define RESET_ARM0	2
+#define RESET_ARM1	3
+#define RESET_USBHS	4
+#define RESET_USBPHYA	5
+#define RESET_MAC	6
+#define RESET_PCIEA	7
+#define RESET_SGDMA	8
+#define RESET_CIPHER	9
+#define RESET_DDR	10
+#define RESET_SATA	11
+#define RESET_SATA_LINK	12
+#define RESET_SATA_PHY	13
+#define RESET_PCIEPHY	14
+#define RESET_NAND	15
+#define RESET_GPIO	16
+#define RESET_UART1	17
+#define RESET_UART2	18
+#define RESET_MISC	19
+#define RESET_I2S	20
+#define RESET_SD	21
+#define RESET_MAC_2	22
+#define RESET_PCIEB	23
+#define RESET_VIDEO	24
+#define RESET_DDR_PHY	25
+#define RESET_USBPHYB	26
+#define RESET_USBDEV	27
+/* Reserved		29 */
+#define RESET_ARMDBG	29
+#define RESET_PLLA	30
+#define RESET_PLLB	31
+
+#endif /* DT_RESET_OXSEMI_OX820_H */
diff --git a/include/dt-bindings/reset/sun50i-a64-ccu.h b/include/dt-bindings/reset/sun50i-a64-ccu.h
new file mode 100644
index 000000000000..db60b29ddb11
--- /dev/null
+++ b/include/dt-bindings/reset/sun50i-a64-ccu.h
@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2016 Maxime Ripard <maxime.ripard@free-electrons.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _DT_BINDINGS_RST_SUN50I_A64_H_
+#define _DT_BINDINGS_RST_SUN50I_A64_H_
+
+#define RST_USB_PHY0		0
+#define RST_USB_PHY1		1
+#define RST_USB_HSIC		2
+#define RST_DRAM		3
+#define RST_MBUS		4
+#define RST_BUS_MIPI_DSI	5
+#define RST_BUS_CE		6
+#define RST_BUS_DMA		7
+#define RST_BUS_MMC0		8
+#define RST_BUS_MMC1		9
+#define RST_BUS_MMC2		10
+#define RST_BUS_NAND		11
+#define RST_BUS_DRAM		12
+#define RST_BUS_EMAC		13
+#define RST_BUS_TS		14
+#define RST_BUS_HSTIMER		15
+#define RST_BUS_SPI0		16
+#define RST_BUS_SPI1		17
+#define RST_BUS_OTG		18
+#define RST_BUS_EHCI0		19
+#define RST_BUS_EHCI1		20
+#define RST_BUS_OHCI0		21
+#define RST_BUS_OHCI1		22
+#define RST_BUS_VE		23
+#define RST_BUS_TCON0		24
+#define RST_BUS_TCON1		25
+#define RST_BUS_DEINTERLACE	26
+#define RST_BUS_CSI		27
+#define RST_BUS_HDMI0		28
+#define RST_BUS_HDMI1		29
+#define RST_BUS_DE		30
+#define RST_BUS_GPU		31
+#define RST_BUS_MSGBOX		32
+#define RST_BUS_SPINLOCK	33
+#define RST_BUS_DBG		34
+#define RST_BUS_LVDS		35
+#define RST_BUS_CODEC		36
+#define RST_BUS_SPDIF		37
+#define RST_BUS_THS		38
+#define RST_BUS_I2S0		39
+#define RST_BUS_I2S1		40
+#define RST_BUS_I2S2		41
+#define RST_BUS_I2C0		42
+#define RST_BUS_I2C1		43
+#define RST_BUS_I2C2		44
+#define RST_BUS_SCR		45
+#define RST_BUS_UART0		46
+#define RST_BUS_UART1		47
+#define RST_BUS_UART2		48
+#define RST_BUS_UART3		49
+#define RST_BUS_UART4		50
+
+#endif /* _DT_BINDINGS_RST_SUN50I_A64_H_ */
diff --git a/include/dt-bindings/reset/tegra186-reset.h b/include/dt-bindings/reset/tegra186-reset.h
new file mode 100644
index 000000000000..8a184e357955
--- /dev/null
+++ b/include/dt-bindings/reset/tegra186-reset.h
@@ -0,0 +1,217 @@
+/*
+ * Copyright (c) 2015, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _ABI_MACH_T186_RESET_T186_H_
+#define _ABI_MACH_T186_RESET_T186_H_
+
+
+#define TEGRA186_RESET_ACTMON			0
+#define TEGRA186_RESET_AFI			1
+#define TEGRA186_RESET_CEC			2
+#define TEGRA186_RESET_CSITE			3
+#define TEGRA186_RESET_DP2			4
+#define TEGRA186_RESET_DPAUX			5
+#define TEGRA186_RESET_DSI			6
+#define TEGRA186_RESET_DSIB			7
+#define TEGRA186_RESET_DTV			8
+#define TEGRA186_RESET_DVFS			9
+#define TEGRA186_RESET_ENTROPY			10
+#define TEGRA186_RESET_EXTPERIPH1		11
+#define TEGRA186_RESET_EXTPERIPH2		12
+#define TEGRA186_RESET_EXTPERIPH3		13
+#define TEGRA186_RESET_GPU			14
+#define TEGRA186_RESET_HDA			15
+#define TEGRA186_RESET_HDA2CODEC_2X		16
+#define TEGRA186_RESET_HDA2HDMICODEC		17
+#define TEGRA186_RESET_HOST1X			18
+#define TEGRA186_RESET_I2C1			19
+#define TEGRA186_RESET_I2C2			20
+#define TEGRA186_RESET_I2C3			21
+#define TEGRA186_RESET_I2C4			22
+#define TEGRA186_RESET_I2C5			23
+#define TEGRA186_RESET_I2C6			24
+#define TEGRA186_RESET_ISP			25
+#define TEGRA186_RESET_KFUSE			26
+#define TEGRA186_RESET_LA			27
+#define TEGRA186_RESET_MIPI_CAL			28
+#define TEGRA186_RESET_PCIE			29
+#define TEGRA186_RESET_PCIEXCLK			30
+#define TEGRA186_RESET_SATA			31
+#define TEGRA186_RESET_SATACOLD			32
+#define TEGRA186_RESET_SDMMC1			33
+#define TEGRA186_RESET_SDMMC2			34
+#define TEGRA186_RESET_SDMMC3			35
+#define TEGRA186_RESET_SDMMC4			36
+#define TEGRA186_RESET_SE			37
+#define TEGRA186_RESET_SOC_THERM		38
+#define TEGRA186_RESET_SOR0			39
+#define TEGRA186_RESET_SPI1			40
+#define TEGRA186_RESET_SPI2			41
+#define TEGRA186_RESET_SPI3			42
+#define TEGRA186_RESET_SPI4			43
+#define TEGRA186_RESET_TMR			44
+#define TEGRA186_RESET_TRIG_SYS			45
+#define TEGRA186_RESET_TSEC			46
+#define TEGRA186_RESET_UARTA			47
+#define TEGRA186_RESET_UARTB			48
+#define TEGRA186_RESET_UARTC			49
+#define TEGRA186_RESET_UARTD			50
+#define TEGRA186_RESET_VI			51
+#define TEGRA186_RESET_VIC			52
+#define TEGRA186_RESET_XUSB_DEV			53
+#define TEGRA186_RESET_XUSB_HOST		54
+#define TEGRA186_RESET_XUSB_PADCTL		55
+#define TEGRA186_RESET_XUSB_SS			56
+#define TEGRA186_RESET_AON_APB			57
+#define TEGRA186_RESET_AXI_CBB			58
+#define TEGRA186_RESET_BPMP_APB			59
+#define TEGRA186_RESET_CAN1			60
+#define TEGRA186_RESET_CAN2			61
+#define TEGRA186_RESET_DMIC5			62
+#define TEGRA186_RESET_DSIC			63
+#define TEGRA186_RESET_DSID			64
+#define TEGRA186_RESET_EMC_EMC			65
+#define TEGRA186_RESET_EMC_MEM			66
+#define TEGRA186_RESET_EMCSB_EMC		67
+#define TEGRA186_RESET_EMCSB_MEM		68
+#define TEGRA186_RESET_EQOS			69
+#define TEGRA186_RESET_GPCDMA			70
+#define TEGRA186_RESET_GPIO_CTL0		71
+#define TEGRA186_RESET_GPIO_CTL1		72
+#define TEGRA186_RESET_GPIO_CTL2		73
+#define TEGRA186_RESET_GPIO_CTL3		74
+#define TEGRA186_RESET_GPIO_CTL4		75
+#define TEGRA186_RESET_GPIO_CTL5		76
+#define TEGRA186_RESET_I2C10			77
+#define TEGRA186_RESET_I2C12			78
+#define TEGRA186_RESET_I2C13			79
+#define TEGRA186_RESET_I2C14			80
+#define TEGRA186_RESET_I2C7			81
+#define TEGRA186_RESET_I2C8			82
+#define TEGRA186_RESET_I2C9			83
+#define TEGRA186_RESET_JTAG2AXI			84
+#define TEGRA186_RESET_MPHY_IOBIST		85
+#define TEGRA186_RESET_MPHY_L0_RX		86
+#define TEGRA186_RESET_MPHY_L0_TX		87
+#define TEGRA186_RESET_NVCSI			88
+#define TEGRA186_RESET_NVDISPLAY0_HEAD0		89
+#define TEGRA186_RESET_NVDISPLAY0_HEAD1		90
+#define TEGRA186_RESET_NVDISPLAY0_HEAD2		91
+#define TEGRA186_RESET_NVDISPLAY0_MISC		92
+#define TEGRA186_RESET_NVDISPLAY0_WGRP0		93
+#define TEGRA186_RESET_NVDISPLAY0_WGRP1		94
+#define TEGRA186_RESET_NVDISPLAY0_WGRP2		95
+#define TEGRA186_RESET_NVDISPLAY0_WGRP3		96
+#define TEGRA186_RESET_NVDISPLAY0_WGRP4		97
+#define TEGRA186_RESET_NVDISPLAY0_WGRP5		98
+#define TEGRA186_RESET_PWM1			99
+#define TEGRA186_RESET_PWM2			100
+#define TEGRA186_RESET_PWM3			101
+#define TEGRA186_RESET_PWM4			102
+#define TEGRA186_RESET_PWM5			103
+#define TEGRA186_RESET_PWM6			104
+#define TEGRA186_RESET_PWM7			105
+#define TEGRA186_RESET_PWM8			106
+#define TEGRA186_RESET_SCE_APB			107
+#define TEGRA186_RESET_SOR1			108
+#define TEGRA186_RESET_TACH			109
+#define TEGRA186_RESET_TSC			110
+#define TEGRA186_RESET_UARTF			111
+#define TEGRA186_RESET_UARTG			112
+#define TEGRA186_RESET_UFSHC			113
+#define TEGRA186_RESET_UFSHC_AXI_M		114
+#define TEGRA186_RESET_UPHY			115
+#define TEGRA186_RESET_ADSP			116
+#define TEGRA186_RESET_ADSPDBG			117
+#define TEGRA186_RESET_ADSPINTF			118
+#define TEGRA186_RESET_ADSPNEON			119
+#define TEGRA186_RESET_ADSPPERIPH		120
+#define TEGRA186_RESET_ADSPSCU			121
+#define TEGRA186_RESET_ADSPWDT			122
+#define TEGRA186_RESET_APE			123
+#define TEGRA186_RESET_DPAUX1			124
+#define TEGRA186_RESET_NVDEC			125
+#define TEGRA186_RESET_NVENC			126
+#define TEGRA186_RESET_NVJPG			127
+#define TEGRA186_RESET_PEX_USB_UPHY		128
+#define TEGRA186_RESET_QSPI			129
+#define TEGRA186_RESET_TSECB			130
+#define TEGRA186_RESET_VI_I2C			131
+#define TEGRA186_RESET_UARTE			132
+#define TEGRA186_RESET_TOP_GTE			133
+#define TEGRA186_RESET_SHSP			134
+#define TEGRA186_RESET_PEX_USB_UPHY_L5		135
+#define TEGRA186_RESET_PEX_USB_UPHY_L4		136
+#define TEGRA186_RESET_PEX_USB_UPHY_L3		137
+#define TEGRA186_RESET_PEX_USB_UPHY_L2		138
+#define TEGRA186_RESET_PEX_USB_UPHY_L1		139
+#define TEGRA186_RESET_PEX_USB_UPHY_L0		140
+#define TEGRA186_RESET_PEX_USB_UPHY_PLL1	141
+#define TEGRA186_RESET_PEX_USB_UPHY_PLL0	142
+#define TEGRA186_RESET_TSCTNVI			143
+#define TEGRA186_RESET_EXTPERIPH4		144
+#define TEGRA186_RESET_DSIPADCTL		145
+#define TEGRA186_RESET_AUD_MCLK			146
+#define TEGRA186_RESET_MPHY_CLK_CTL		147
+#define TEGRA186_RESET_MPHY_L1_RX		148
+#define TEGRA186_RESET_MPHY_L1_TX		149
+#define TEGRA186_RESET_UFSHC_LP			150
+#define TEGRA186_RESET_BPMP_NIC			151
+#define TEGRA186_RESET_BPMP_NSYSPORESET		152
+#define TEGRA186_RESET_BPMP_NRESET		153
+#define TEGRA186_RESET_BPMP_DBGRESETN		154
+#define TEGRA186_RESET_BPMP_PRESETDBGN		155
+#define TEGRA186_RESET_BPMP_PM			156
+#define TEGRA186_RESET_BPMP_CVC			157
+#define TEGRA186_RESET_BPMP_DMA			158
+#define TEGRA186_RESET_BPMP_HSP			159
+#define TEGRA186_RESET_TSCTNBPMP		160
+#define TEGRA186_RESET_BPMP_TKE			161
+#define TEGRA186_RESET_BPMP_GTE			162
+#define TEGRA186_RESET_BPMP_PM_ACTMON		163
+#define TEGRA186_RESET_AON_NIC			164
+#define TEGRA186_RESET_AON_NSYSPORESET		165
+#define TEGRA186_RESET_AON_NRESET		166
+#define TEGRA186_RESET_AON_DBGRESETN		167
+#define TEGRA186_RESET_AON_PRESETDBGN		168
+#define TEGRA186_RESET_AON_ACTMON		169
+#define TEGRA186_RESET_AOPM			170
+#define TEGRA186_RESET_AOVC			171
+#define TEGRA186_RESET_AON_DMA			172
+#define TEGRA186_RESET_AON_GPIO			173
+#define TEGRA186_RESET_AON_HSP			174
+#define TEGRA186_RESET_TSCTNAON			175
+#define TEGRA186_RESET_AON_TKE			176
+#define TEGRA186_RESET_AON_GTE			177
+#define TEGRA186_RESET_SCE_NIC			178
+#define TEGRA186_RESET_SCE_NSYSPORESET		179
+#define TEGRA186_RESET_SCE_NRESET		180
+#define TEGRA186_RESET_SCE_DBGRESETN		181
+#define TEGRA186_RESET_SCE_PRESETDBGN		182
+#define TEGRA186_RESET_SCE_ACTMON		183
+#define TEGRA186_RESET_SCE_PM			184
+#define TEGRA186_RESET_SCE_DMA			185
+#define TEGRA186_RESET_SCE_HSP			186
+#define TEGRA186_RESET_TSCTNSCE			187
+#define TEGRA186_RESET_SCE_TKE			188
+#define TEGRA186_RESET_SCE_GTE			189
+#define TEGRA186_RESET_SCE_CFG			190
+#define TEGRA186_RESET_ADSP_ALL			191
+/** @brief controls the power up/down sequence of UFSHC PSW partition. Controls LP_PWR_READY, LP_ISOL_EN, and LP_RESET_N signals */
+#define TEGRA186_RESET_UFSHC_LP_SEQ		192
+#define TEGRA186_RESET_SIZE			193
+
+#endif
diff --git a/include/dt-bindings/sound/cs42l42.h b/include/dt-bindings/sound/cs42l42.h
new file mode 100644
index 000000000000..399a123aed58
--- /dev/null
+++ b/include/dt-bindings/sound/cs42l42.h
@@ -0,0 +1,73 @@
+/*
+ * cs42l42.h -- CS42L42 ALSA SoC audio driver DT bindings header
+ *
+ * Copyright 2016 Cirrus Logic, Inc.
+ *
+ * Author: James Schulman <james.schulman@cirrus.com>
+ * Author: Brian Austin <brian.austin@cirrus.com>
+ * Author: Michael White <michael.white@cirrus.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __DT_CS42L42_H
+#define __DT_CS42L42_H
+
+/* HPOUT Load Capacity */
+#define CS42L42_HPOUT_LOAD_1NF		0
+#define CS42L42_HPOUT_LOAD_10NF		1
+
+/* HPOUT Clamp to GND Overide */
+#define CS42L42_HPOUT_CLAMP_EN		0
+#define CS42L42_HPOUT_CLAMP_DIS		1
+
+/* Tip Sense Inversion */
+#define CS42L42_TS_INV_DIS			0
+#define CS42L42_TS_INV_EN			1
+
+/* Tip Sense Debounce */
+#define CS42L42_TS_DBNCE_0			0
+#define CS42L42_TS_DBNCE_125			1
+#define CS42L42_TS_DBNCE_250			2
+#define CS42L42_TS_DBNCE_500			3
+#define CS42L42_TS_DBNCE_750			4
+#define CS42L42_TS_DBNCE_1000			5
+#define CS42L42_TS_DBNCE_1250			6
+#define CS42L42_TS_DBNCE_1500			7
+
+/* Button Press Software Debounce Times */
+#define CS42L42_BTN_DET_INIT_DBNCE_MIN		0
+#define CS42L42_BTN_DET_INIT_DBNCE_DEFAULT	100
+#define CS42L42_BTN_DET_INIT_DBNCE_MAX		200
+
+#define CS42L42_BTN_DET_EVENT_DBNCE_MIN		0
+#define CS42L42_BTN_DET_EVENT_DBNCE_DEFAULT	10
+#define CS42L42_BTN_DET_EVENT_DBNCE_MAX		20
+
+/* Button Detect Level Sensitivities */
+#define CS42L42_NUM_BIASES		4
+
+#define CS42L42_HS_DET_LEVEL_15		0x0F
+#define CS42L42_HS_DET_LEVEL_8		0x08
+#define CS42L42_HS_DET_LEVEL_4		0x04
+#define CS42L42_HS_DET_LEVEL_1		0x01
+
+#define CS42L42_HS_DET_LEVEL_MIN	0
+#define CS42L42_HS_DET_LEVEL_MAX	0x3F
+
+/* HS Bias Ramp Rate */
+
+#define CS42L42_HSBIAS_RAMP_FAST_RISE_SLOW_FALL		0
+#define CS42L42_HSBIAS_RAMP_FAST			1
+#define CS42L42_HSBIAS_RAMP_SLOW			2
+#define CS42L42_HSBIAS_RAMP_SLOWEST			3
+
+#define CS42L42_HSBIAS_RAMP_TIME0			10
+#define CS42L42_HSBIAS_RAMP_TIME1			40
+#define CS42L42_HSBIAS_RAMP_TIME2			90
+#define CS42L42_HSBIAS_RAMP_TIME3			170
+
+#endif /* __DT_CS42L42_H */
diff --git a/include/linux/acpi.h b/include/linux/acpi.h
index 051023756520..5b36974ed60a 100644
--- a/include/linux/acpi.h
+++ b/include/linux/acpi.h
@@ -56,6 +56,27 @@ static inline acpi_handle acpi_device_handle(struct acpi_device *adev)
 	acpi_fwnode_handle(adev) : NULL)
 #define ACPI_HANDLE(dev)		acpi_device_handle(ACPI_COMPANION(dev))
 
+static inline struct fwnode_handle *acpi_alloc_fwnode_static(void)
+{
+	struct fwnode_handle *fwnode;
+
+	fwnode = kzalloc(sizeof(struct fwnode_handle), GFP_KERNEL);
+	if (!fwnode)
+		return NULL;
+
+	fwnode->type = FWNODE_ACPI_STATIC;
+
+	return fwnode;
+}
+
+static inline void acpi_free_fwnode_static(struct fwnode_handle *fwnode)
+{
+	if (WARN_ON(!fwnode || fwnode->type != FWNODE_ACPI_STATIC))
+		return;
+
+	kfree(fwnode);
+}
+
 /**
  * ACPI_DEVICE_CLASS - macro used to describe an ACPI device with
  * the PCI-defined class-code information
@@ -220,10 +241,6 @@ int __init acpi_table_parse_entries(char *id, unsigned long table_size,
 			      int entry_id,
 			      acpi_tbl_entry_handler handler,
 			      unsigned int max_entries);
-int __init acpi_table_parse_entries(char *id, unsigned long table_size,
-			      int entry_id,
-			      acpi_tbl_entry_handler handler,
-			      unsigned int max_entries);
 int __init acpi_table_parse_entries_array(char *id, unsigned long table_size,
 			      struct acpi_subtable_proc *proc, int proc_num,
 			      unsigned int max_entries);
@@ -420,6 +437,8 @@ static inline int acpi_dev_filter_resource_type_cb(struct acpi_resource *ares,
 	return acpi_dev_filter_resource_type(ares, (unsigned long)arg);
 }
 
+struct acpi_device *acpi_resource_consumer(struct resource *res);
+
 int acpi_check_resource_conflict(const struct resource *res);
 
 int acpi_check_region(resource_size_t start, resource_size_t n,
@@ -745,6 +764,11 @@ static inline enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev)
 	return DEV_DMA_NOT_SUPPORTED;
 }
 
+static inline void acpi_dma_configure(struct device *dev,
+				      enum dev_dma_attr attr) { }
+
+static inline void acpi_dma_deconfigure(struct device *dev) { }
+
 #define ACPI_PTR(_ptr)	(NULL)
 
 static inline void acpi_device_set_enumerated(struct acpi_device *adev)
@@ -765,6 +789,11 @@ static inline int acpi_reconfig_notifier_unregister(struct notifier_block *nb)
 	return -EINVAL;
 }
 
+static inline struct acpi_device *acpi_resource_consumer(struct resource *res)
+{
+	return NULL;
+}
+
 #endif	/* !CONFIG_ACPI */
 
 #ifdef CONFIG_ACPI_HOTPLUG_IOAPIC
diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h
index 0e32dac8fd03..77e08099e554 100644
--- a/include/linux/acpi_iort.h
+++ b/include/linux/acpi_iort.h
@@ -23,20 +23,36 @@
 #include <linux/fwnode.h>
 #include <linux/irqdomain.h>
 
+#define IORT_IRQ_MASK(irq)		(irq & 0xffffffffULL)
+#define IORT_IRQ_TRIGGER_MASK(irq)	((irq >> 32) & 0xffffffffULL)
+
 int iort_register_domain_token(int trans_id, struct fwnode_handle *fw_node);
 void iort_deregister_domain_token(int trans_id);
 struct fwnode_handle *iort_find_domain_token(int trans_id);
 #ifdef CONFIG_ACPI_IORT
 void acpi_iort_init(void);
+bool iort_node_match(u8 type);
 u32 iort_msi_map_rid(struct device *dev, u32 req_id);
 struct irq_domain *iort_get_device_domain(struct device *dev, u32 req_id);
+/* IOMMU interface */
+void iort_set_dma_mask(struct device *dev);
+const struct iommu_ops *iort_iommu_configure(struct device *dev);
 #else
 static inline void acpi_iort_init(void) { }
+static inline bool iort_node_match(u8 type) { return false; }
 static inline u32 iort_msi_map_rid(struct device *dev, u32 req_id)
 { return req_id; }
 static inline struct irq_domain *iort_get_device_domain(struct device *dev,
 							u32 req_id)
 { return NULL; }
+/* IOMMU interface */
+static inline void iort_set_dma_mask(struct device *dev) { }
+static inline
+const struct iommu_ops *iort_iommu_configure(struct device *dev)
+{ return NULL; }
 #endif
 
+#define IORT_ACPI_DECLARE(name, table_id, fn)		\
+	ACPI_DECLARE_PROBE_ENTRY(iort, name, table_id, 0, NULL, 0, fn)
+
 #endif /* __ACPI_IORT_H__ */
diff --git a/include/linux/ahci-remap.h b/include/linux/ahci-remap.h
new file mode 100644
index 000000000000..62be3a40239d
--- /dev/null
+++ b/include/linux/ahci-remap.h
@@ -0,0 +1,28 @@
+#ifndef _LINUX_AHCI_REMAP_H
+#define _LINUX_AHCI_REMAP_H
+
+#include <linux/sizes.h>
+
+#define AHCI_VSCAP		0xa4
+#define AHCI_REMAP_CAP		0x800
+
+/* device class code */
+#define AHCI_REMAP_N_DCC	0x880
+
+/* remap-device base relative to ahci-bar */
+#define AHCI_REMAP_N_OFFSET	SZ_16K
+#define AHCI_REMAP_N_SIZE	SZ_16K
+
+#define AHCI_MAX_REMAP		3
+
+static inline unsigned int ahci_remap_dcc(int i)
+{
+	return AHCI_REMAP_N_DCC + i * 0x80;
+}
+
+static inline unsigned int ahci_remap_base(int i)
+{
+	return AHCI_REMAP_N_OFFSET + i * AHCI_REMAP_N_SIZE;
+}
+
+#endif /* _LINUX_AHCI_REMAP_H */
diff --git a/include/linux/amba/pl061.h b/include/linux/amba/pl061.h
deleted file mode 100644
index fb83c0453489..000000000000
--- a/include/linux/amba/pl061.h
+++ /dev/null
@@ -1,16 +0,0 @@
-#include <linux/types.h>
-
-/* platform data for the PL061 GPIO driver */
-
-struct pl061_platform_data {
-	/* number of the first GPIO */
-	unsigned	gpio_base;
-
-	/* number of the first IRQ.
-	 * If the IRQ functionality in not desired this must be set to 0.
-	 */
-	unsigned	irq_base;
-
-	u8		directions;	/* startup directions, 1: out, 0: in */
-	u8		values;		/* startup values */
-};
diff --git a/include/linux/amba/pl08x.h b/include/linux/amba/pl08x.h
index 27e9ec8778eb..5308eae9ce35 100644
--- a/include/linux/amba/pl08x.h
+++ b/include/linux/amba/pl08x.h
@@ -84,6 +84,8 @@ struct pl08x_channel_data {
  * running any DMA transfer and multiplexing can be recycled
  * @lli_buses: buses which LLIs can be fetched from: PL08X_AHB1 | PL08X_AHB2
  * @mem_buses: buses which memory can be accessed from: PL08X_AHB1 | PL08X_AHB2
+ * @slave_map: DMA slave matching table
+ * @slave_map_len: number of elements in @slave_map
  */
 struct pl08x_platform_data {
 	struct pl08x_channel_data *slave_channels;
@@ -93,6 +95,8 @@ struct pl08x_platform_data {
 	void (*put_xfer_signal)(const struct pl08x_channel_data *, int);
 	u8 lli_buses;
 	u8 mem_buses;
+	const struct dma_slave_map *slave_map;
+	int slave_map_len;
 };
 
 #ifdef CONFIG_AMBA_PL08X
diff --git a/include/linux/ata.h b/include/linux/ata.h
index fdb180367ba1..af6859b3a93d 100644
--- a/include/linux/ata.h
+++ b/include/linux/ata.h
@@ -348,6 +348,7 @@ enum {
 	ATA_LOG_DEVSLP_DETO	  = 0x01,
 	ATA_LOG_DEVSLP_VALID	  = 0x07,
 	ATA_LOG_DEVSLP_VALID_MASK = 0x80,
+	ATA_LOG_NCQ_PRIO_OFFSET   = 0x09,
 
 	/* NCQ send and receive log */
 	ATA_LOG_NCQ_SEND_RECV_SUBCMDS_OFFSET	= 0x00,
@@ -940,6 +941,11 @@ static inline bool ata_id_has_ncq_non_data(const u16 *id)
 	return id[ATA_ID_SATA_CAPABILITY_2] & BIT(5);
 }
 
+static inline bool ata_id_has_ncq_prio(const u16 *id)
+{
+	return id[ATA_ID_SATA_CAPABILITY] & BIT(12);
+}
+
 static inline bool ata_id_has_trim(const u16 *id)
 {
 	if (ata_id_major_version(id) >= 7 &&
diff --git a/include/linux/backing-dev-defs.h b/include/linux/backing-dev-defs.h
index 0b5b1af35e5e..e850e76acaaf 100644
--- a/include/linux/backing-dev-defs.h
+++ b/include/linux/backing-dev-defs.h
@@ -116,6 +116,8 @@ struct bdi_writeback {
 	struct list_head work_list;
 	struct delayed_work dwork;	/* work item used for writeback */
 
+	unsigned long dirty_sleep;	/* last wait */
+
 	struct list_head bdi_node;	/* anchored at bdi->wb_list */
 
 #ifdef CONFIG_CGROUP_WRITEBACK
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 97cb48f03dc7..7cf8a6c70a3f 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -63,6 +63,12 @@
 #define bio_end_sector(bio)	((bio)->bi_iter.bi_sector + bio_sectors((bio)))
 
 /*
+ * Return the data direction, READ or WRITE.
+ */
+#define bio_data_dir(bio) \
+	(op_is_write(bio_op(bio)) ? WRITE : READ)
+
+/*
  * Check whether this bio carries any data or not. A NULL bio is allowed.
  */
 static inline bool bio_has_data(struct bio *bio)
@@ -70,7 +76,8 @@ static inline bool bio_has_data(struct bio *bio)
 	if (bio &&
 	    bio->bi_iter.bi_size &&
 	    bio_op(bio) != REQ_OP_DISCARD &&
-	    bio_op(bio) != REQ_OP_SECURE_ERASE)
+	    bio_op(bio) != REQ_OP_SECURE_ERASE &&
+	    bio_op(bio) != REQ_OP_WRITE_ZEROES)
 		return true;
 
 	return false;
@@ -80,18 +87,8 @@ static inline bool bio_no_advance_iter(struct bio *bio)
 {
 	return bio_op(bio) == REQ_OP_DISCARD ||
 	       bio_op(bio) == REQ_OP_SECURE_ERASE ||
-	       bio_op(bio) == REQ_OP_WRITE_SAME;
-}
-
-static inline bool bio_is_rw(struct bio *bio)
-{
-	if (!bio_has_data(bio))
-		return false;
-
-	if (bio_no_advance_iter(bio))
-		return false;
-
-	return true;
+	       bio_op(bio) == REQ_OP_WRITE_SAME ||
+	       bio_op(bio) == REQ_OP_WRITE_ZEROES;
 }
 
 static inline bool bio_mergeable(struct bio *bio)
@@ -193,18 +190,20 @@ static inline unsigned bio_segments(struct bio *bio)
 	struct bvec_iter iter;
 
 	/*
-	 * We special case discard/write same, because they interpret bi_size
-	 * differently:
+	 * We special case discard/write same/write zeroes, because they
+	 * interpret bi_size differently:
 	 */
 
-	if (bio_op(bio) == REQ_OP_DISCARD)
-		return 1;
-
-	if (bio_op(bio) == REQ_OP_SECURE_ERASE)
-		return 1;
-
-	if (bio_op(bio) == REQ_OP_WRITE_SAME)
+	switch (bio_op(bio)) {
+	case REQ_OP_DISCARD:
+	case REQ_OP_SECURE_ERASE:
+	case REQ_OP_WRITE_ZEROES:
+		return 0;
+	case REQ_OP_WRITE_SAME:
 		return 1;
+	default:
+		break;
+	}
 
 	bio_for_each_segment(bv, bio, iter)
 		segs++;
@@ -409,6 +408,8 @@ static inline struct bio *bio_clone_kmalloc(struct bio *bio, gfp_t gfp_mask)
 
 }
 
+extern blk_qc_t submit_bio(struct bio *);
+
 extern void bio_endio(struct bio *);
 
 static inline void bio_io_error(struct bio *bio)
@@ -423,13 +424,15 @@ extern int bio_phys_segments(struct request_queue *, struct bio *);
 extern int submit_bio_wait(struct bio *bio);
 extern void bio_advance(struct bio *, unsigned);
 
-extern void bio_init(struct bio *);
+extern void bio_init(struct bio *bio, struct bio_vec *table,
+		     unsigned short max_vecs);
 extern void bio_reset(struct bio *);
 void bio_chain(struct bio *, struct bio *);
 
 extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
 extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
 			   unsigned int, unsigned int);
+int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);
 struct rq_map_data;
 extern struct bio *bio_map_user_iov(struct request_queue *,
 				    const struct iov_iter *, gfp_t);
diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h
index 3bf5d33800ab..01b62e7bac74 100644
--- a/include/linux/blk-cgroup.h
+++ b/include/linux/blk-cgroup.h
@@ -581,15 +581,14 @@ static inline void blkg_rwstat_exit(struct blkg_rwstat *rwstat)
 /**
  * blkg_rwstat_add - add a value to a blkg_rwstat
  * @rwstat: target blkg_rwstat
- * @op: REQ_OP
- * @op_flags: rq_flag_bits
+ * @op: REQ_OP and flags
  * @val: value to add
  *
  * Add @val to @rwstat.  The counters are chosen according to @rw.  The
  * caller is responsible for synchronizing calls to this function.
  */
 static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
-				   int op, int op_flags, uint64_t val)
+				   unsigned int op, uint64_t val)
 {
 	struct percpu_counter *cnt;
 
@@ -600,7 +599,7 @@ static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
 
 	__percpu_counter_add(cnt, val, BLKG_STAT_CPU_BATCH);
 
-	if (op_flags & REQ_SYNC)
+	if (op_is_sync(op))
 		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
 	else
 		cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
@@ -705,9 +704,9 @@ static inline bool blkcg_bio_issue_check(struct request_queue *q,
 
 	if (!throtl) {
 		blkg = blkg ?: q->root_blkg;
-		blkg_rwstat_add(&blkg->stat_bytes, bio_op(bio), bio->bi_opf,
+		blkg_rwstat_add(&blkg->stat_bytes, bio->bi_opf,
 				bio->bi_iter.bi_size);
-		blkg_rwstat_add(&blkg->stat_ios, bio_op(bio), bio->bi_opf, 1);
+		blkg_rwstat_add(&blkg->stat_ios, bio->bi_opf, 1);
 	}
 
 	rcu_read_unlock();
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 535ab2e13d2e..4a2ab5d99ff7 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -3,6 +3,7 @@
 
 #include <linux/blkdev.h>
 #include <linux/sbitmap.h>
+#include <linux/srcu.h>
 
 struct blk_mq_tags;
 struct blk_flush_queue;
@@ -35,6 +36,8 @@ struct blk_mq_hw_ctx {
 
 	struct blk_mq_tags	*tags;
 
+	struct srcu_struct	queue_rq_srcu;
+
 	unsigned long		queued;
 	unsigned long		run;
 #define BLK_MQ_MAX_DISPATCH_ORDER	7
@@ -215,18 +218,20 @@ void blk_mq_start_request(struct request *rq);
 void blk_mq_end_request(struct request *rq, int error);
 void __blk_mq_end_request(struct request *rq, int error);
 
-void blk_mq_requeue_request(struct request *rq);
-void blk_mq_add_to_requeue_list(struct request *rq, bool at_head);
-void blk_mq_cancel_requeue_work(struct request_queue *q);
+void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list);
+void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
+				bool kick_requeue_list);
 void blk_mq_kick_requeue_list(struct request_queue *q);
 void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs);
 void blk_mq_abort_requeue_list(struct request_queue *q);
 void blk_mq_complete_request(struct request *rq, int error);
 
+bool blk_mq_queue_stopped(struct request_queue *q);
 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
 void blk_mq_stop_hw_queues(struct request_queue *q);
 void blk_mq_start_hw_queues(struct request_queue *q);
+void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async);
 void blk_mq_run_hw_queues(struct request_queue *q, bool async);
 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
@@ -237,6 +242,7 @@ void blk_mq_unfreeze_queue(struct request_queue *q);
 void blk_mq_freeze_queue_start(struct request_queue *q);
 int blk_mq_reinit_tagset(struct blk_mq_tag_set *set);
 
+int blk_mq_map_queues(struct blk_mq_tag_set *set);
 void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues);
 
 /*
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index cd395ecec99d..519ea2c9df61 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -17,7 +17,6 @@ struct io_context;
 struct cgroup_subsys_state;
 typedef void (bio_end_io_t) (struct bio *);
 
-#ifdef CONFIG_BLOCK
 /*
  * main unit of I/O for the block layer and lower layers (ie drivers and
  * stacking drivers)
@@ -88,24 +87,6 @@ struct bio {
 	struct bio_vec		bi_inline_vecs[0];
 };
 
-#define BIO_OP_SHIFT	(8 * FIELD_SIZEOF(struct bio, bi_opf) - REQ_OP_BITS)
-#define bio_flags(bio)	((bio)->bi_opf & ((1 << BIO_OP_SHIFT) - 1))
-#define bio_op(bio)	((bio)->bi_opf >> BIO_OP_SHIFT)
-
-#define bio_set_op_attrs(bio, op, op_flags) do {			\
-	if (__builtin_constant_p(op))					\
-		BUILD_BUG_ON((op) + 0U >= (1U << REQ_OP_BITS));		\
-	else								\
-		WARN_ON_ONCE((op) + 0U >= (1U << REQ_OP_BITS));		\
-	if (__builtin_constant_p(op_flags))				\
-		BUILD_BUG_ON((op_flags) + 0U >= (1U << BIO_OP_SHIFT));	\
-	else								\
-		WARN_ON_ONCE((op_flags) + 0U >= (1U << BIO_OP_SHIFT));	\
-	(bio)->bi_opf = bio_flags(bio);					\
-	(bio)->bi_opf |= (((op) + 0U) << BIO_OP_SHIFT);			\
-	(bio)->bi_opf |= (op_flags);					\
-} while (0)
-
 #define BIO_RESET_BYTES		offsetof(struct bio, bi_max_vecs)
 
 /*
@@ -119,6 +100,8 @@ struct bio {
 #define BIO_QUIET	6	/* Make BIO Quiet */
 #define BIO_CHAIN	7	/* chained bio, ->bi_remaining in effect */
 #define BIO_REFFED	8	/* bio has elevated ->bi_cnt */
+#define BIO_THROTTLED	9	/* This bio has already been subjected to
+				 * throttling rules. Don't do it again. */
 
 /*
  * Flags starting here get preserved by bio_reset() - this includes
@@ -142,53 +125,61 @@ struct bio {
 #define BVEC_POOL_OFFSET	(16 - BVEC_POOL_BITS)
 #define BVEC_POOL_IDX(bio)	((bio)->bi_flags >> BVEC_POOL_OFFSET)
 
-#endif /* CONFIG_BLOCK */
-
 /*
- * Request flags.  For use in the cmd_flags field of struct request, and in
- * bi_opf of struct bio.  Note that some flags are only valid in either one.
+ * Operations and flags common to the bio and request structures.
+ * We use 8 bits for encoding the operation, and the remaining 24 for flags.
+ *
+ * The least significant bit of the operation number indicates the data
+ * transfer direction:
+ *
+ *   - if the least significant bit is set transfers are TO the device
+ *   - if the least significant bit is not set transfers are FROM the device
+ *
+ * If a operation does not transfer data the least significant bit has no
+ * meaning.
  */
-enum rq_flag_bits {
-	/* common flags */
-	__REQ_FAILFAST_DEV,	/* no driver retries of device errors */
+#define REQ_OP_BITS	8
+#define REQ_OP_MASK	((1 << REQ_OP_BITS) - 1)
+#define REQ_FLAG_BITS	24
+
+enum req_opf {
+	/* read sectors from the device */
+	REQ_OP_READ		= 0,
+	/* write sectors to the device */
+	REQ_OP_WRITE		= 1,
+	/* flush the volatile write cache */
+	REQ_OP_FLUSH		= 2,
+	/* discard sectors */
+	REQ_OP_DISCARD		= 3,
+	/* get zone information */
+	REQ_OP_ZONE_REPORT	= 4,
+	/* securely erase sectors */
+	REQ_OP_SECURE_ERASE	= 5,
+	/* seset a zone write pointer */
+	REQ_OP_ZONE_RESET	= 6,
+	/* write the same sector many times */
+	REQ_OP_WRITE_SAME	= 7,
+	/* write the zero filled sector many times */
+	REQ_OP_WRITE_ZEROES	= 8,
+
+	REQ_OP_LAST,
+};
+
+enum req_flag_bits {
+	__REQ_FAILFAST_DEV =	/* no driver retries of device errors */
+		REQ_OP_BITS,
 	__REQ_FAILFAST_TRANSPORT, /* no driver retries of transport errors */
 	__REQ_FAILFAST_DRIVER,	/* no driver retries of driver errors */
-
 	__REQ_SYNC,		/* request is sync (sync write or read) */
 	__REQ_META,		/* metadata io request */
 	__REQ_PRIO,		/* boost priority in cfq */
-
-	__REQ_NOIDLE,		/* don't anticipate more IO after this one */
+	__REQ_NOMERGE,		/* don't touch this for merging */
+	__REQ_IDLE,		/* anticipate more IO after this one */
 	__REQ_INTEGRITY,	/* I/O includes block integrity payload */
 	__REQ_FUA,		/* forced unit access */
 	__REQ_PREFLUSH,		/* request for cache flush */
-
-	/* bio only flags */
 	__REQ_RAHEAD,		/* read ahead, can fail anytime */
-	__REQ_THROTTLED,	/* This bio has already been subjected to
-				 * throttling rules. Don't do it again. */
-
-	/* request only flags */
-	__REQ_SORTED,		/* elevator knows about this request */
-	__REQ_SOFTBARRIER,	/* may not be passed by ioscheduler */
-	__REQ_NOMERGE,		/* don't touch this for merging */
-	__REQ_STARTED,		/* drive already may have started this one */
-	__REQ_DONTPREP,		/* don't call prep for this one */
-	__REQ_QUEUED,		/* uses queueing */
-	__REQ_ELVPRIV,		/* elevator private data attached */
-	__REQ_FAILED,		/* set if the request failed */
-	__REQ_QUIET,		/* don't worry about errors */
-	__REQ_PREEMPT,		/* set for "ide_preempt" requests and also
-				   for requests for which the SCSI "quiesce"
-				   state must be ignored. */
-	__REQ_ALLOCED,		/* request came from our alloc pool */
-	__REQ_COPY_USER,	/* contains copies of user pages */
-	__REQ_FLUSH_SEQ,	/* request for flush sequence */
-	__REQ_IO_STAT,		/* account I/O stat */
-	__REQ_MIXED_MERGE,	/* merge of different types, fail separately */
-	__REQ_PM,		/* runtime pm request */
-	__REQ_HASHED,		/* on IO scheduler merge hash */
-	__REQ_MQ_INFLIGHT,	/* track inflight for MQ */
+	__REQ_BACKGROUND,	/* background IO */
 	__REQ_NR_BITS,		/* stops here */
 };
 
@@ -198,54 +189,47 @@ enum rq_flag_bits {
 #define REQ_SYNC		(1ULL << __REQ_SYNC)
 #define REQ_META		(1ULL << __REQ_META)
 #define REQ_PRIO		(1ULL << __REQ_PRIO)
-#define REQ_NOIDLE		(1ULL << __REQ_NOIDLE)
+#define REQ_NOMERGE		(1ULL << __REQ_NOMERGE)
+#define REQ_IDLE		(1ULL << __REQ_IDLE)
 #define REQ_INTEGRITY		(1ULL << __REQ_INTEGRITY)
+#define REQ_FUA			(1ULL << __REQ_FUA)
+#define REQ_PREFLUSH		(1ULL << __REQ_PREFLUSH)
+#define REQ_RAHEAD		(1ULL << __REQ_RAHEAD)
+#define REQ_BACKGROUND		(1ULL << __REQ_BACKGROUND)
 
 #define REQ_FAILFAST_MASK \
 	(REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER)
-#define REQ_COMMON_MASK \
-	(REQ_FAILFAST_MASK | REQ_SYNC | REQ_META | REQ_PRIO | REQ_NOIDLE | \
-	 REQ_PREFLUSH | REQ_FUA | REQ_INTEGRITY | REQ_NOMERGE)
-#define REQ_CLONE_MASK		REQ_COMMON_MASK
 
-/* This mask is used for both bio and request merge checking */
 #define REQ_NOMERGE_FLAGS \
-	(REQ_NOMERGE | REQ_STARTED | REQ_SOFTBARRIER | REQ_PREFLUSH | REQ_FUA | REQ_FLUSH_SEQ)
+	(REQ_NOMERGE | REQ_PREFLUSH | REQ_FUA)
 
-#define REQ_RAHEAD		(1ULL << __REQ_RAHEAD)
-#define REQ_THROTTLED		(1ULL << __REQ_THROTTLED)
+#define bio_op(bio) \
+	((bio)->bi_opf & REQ_OP_MASK)
+#define req_op(req) \
+	((req)->cmd_flags & REQ_OP_MASK)
 
-#define REQ_SORTED		(1ULL << __REQ_SORTED)
-#define REQ_SOFTBARRIER		(1ULL << __REQ_SOFTBARRIER)
-#define REQ_FUA			(1ULL << __REQ_FUA)
-#define REQ_NOMERGE		(1ULL << __REQ_NOMERGE)
-#define REQ_STARTED		(1ULL << __REQ_STARTED)
-#define REQ_DONTPREP		(1ULL << __REQ_DONTPREP)
-#define REQ_QUEUED		(1ULL << __REQ_QUEUED)
-#define REQ_ELVPRIV		(1ULL << __REQ_ELVPRIV)
-#define REQ_FAILED		(1ULL << __REQ_FAILED)
-#define REQ_QUIET		(1ULL << __REQ_QUIET)
-#define REQ_PREEMPT		(1ULL << __REQ_PREEMPT)
-#define REQ_ALLOCED		(1ULL << __REQ_ALLOCED)
-#define REQ_COPY_USER		(1ULL << __REQ_COPY_USER)
-#define REQ_PREFLUSH		(1ULL << __REQ_PREFLUSH)
-#define REQ_FLUSH_SEQ		(1ULL << __REQ_FLUSH_SEQ)
-#define REQ_IO_STAT		(1ULL << __REQ_IO_STAT)
-#define REQ_MIXED_MERGE		(1ULL << __REQ_MIXED_MERGE)
-#define REQ_PM			(1ULL << __REQ_PM)
-#define REQ_HASHED		(1ULL << __REQ_HASHED)
-#define REQ_MQ_INFLIGHT		(1ULL << __REQ_MQ_INFLIGHT)
-
-enum req_op {
-	REQ_OP_READ,
-	REQ_OP_WRITE,
-	REQ_OP_DISCARD,		/* request to discard sectors */
-	REQ_OP_SECURE_ERASE,	/* request to securely erase sectors */
-	REQ_OP_WRITE_SAME,	/* write same block many times */
-	REQ_OP_FLUSH,		/* request for cache flush */
-};
+/* obsolete, don't use in new code */
+static inline void bio_set_op_attrs(struct bio *bio, unsigned op,
+		unsigned op_flags)
+{
+	bio->bi_opf = op | op_flags;
+}
 
-#define REQ_OP_BITS 3
+static inline bool op_is_write(unsigned int op)
+{
+	return (op & 1);
+}
+
+/*
+ * Reads are always treated as synchronous, as are requests with the FUA or
+ * PREFLUSH flag.  Other operations may be marked as synchronous using the
+ * REQ_SYNC flag.
+ */
+static inline bool op_is_sync(unsigned int op)
+{
+	return (op & REQ_OP_MASK) == REQ_OP_READ ||
+		(op & (REQ_SYNC | REQ_FUA | REQ_PREFLUSH));
+}
 
 typedef unsigned int blk_qc_t;
 #define BLK_QC_T_NONE	-1U
@@ -271,4 +255,20 @@ static inline unsigned int blk_qc_t_to_tag(blk_qc_t cookie)
 	return cookie & ((1u << BLK_QC_T_SHIFT) - 1);
 }
 
+struct blk_issue_stat {
+	u64 time;
+};
+
+#define BLK_RQ_STAT_BATCH	64
+
+struct blk_rq_stat {
+	s64 mean;
+	u64 min;
+	u64 max;
+	s32 nr_samples;
+	s32 nr_batch;
+	u64 batch;
+	s64 time;
+};
+
 #endif /* __LINUX_BLK_TYPES_H */
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index c47c358ba052..286b2a264383 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -24,6 +24,7 @@
 #include <linux/rcupdate.h>
 #include <linux/percpu-refcount.h>
 #include <linux/scatterlist.h>
+#include <linux/blkzoned.h>
 
 struct module;
 struct scsi_ioctl_command;
@@ -37,6 +38,7 @@ struct bsg_job;
 struct blkcg_gq;
 struct blk_flush_queue;
 struct pr_ops;
+struct rq_wb;
 
 #define BLKDEV_MIN_RQ	4
 #define BLKDEV_MAX_RQ	128	/* Default maximum */
@@ -77,6 +79,55 @@ enum rq_cmd_type_bits {
 	REQ_TYPE_DRV_PRIV,		/* driver defined types from here */
 };
 
+/*
+ * request flags */
+typedef __u32 __bitwise req_flags_t;
+
+/* elevator knows about this request */
+#define RQF_SORTED		((__force req_flags_t)(1 << 0))
+/* drive already may have started this one */
+#define RQF_STARTED		((__force req_flags_t)(1 << 1))
+/* uses tagged queueing */
+#define RQF_QUEUED		((__force req_flags_t)(1 << 2))
+/* may not be passed by ioscheduler */
+#define RQF_SOFTBARRIER		((__force req_flags_t)(1 << 3))
+/* request for flush sequence */
+#define RQF_FLUSH_SEQ		((__force req_flags_t)(1 << 4))
+/* merge of different types, fail separately */
+#define RQF_MIXED_MERGE		((__force req_flags_t)(1 << 5))
+/* track inflight for MQ */
+#define RQF_MQ_INFLIGHT		((__force req_flags_t)(1 << 6))
+/* don't call prep for this one */
+#define RQF_DONTPREP		((__force req_flags_t)(1 << 7))
+/* set for "ide_preempt" requests and also for requests for which the SCSI
+   "quiesce" state must be ignored. */
+#define RQF_PREEMPT		((__force req_flags_t)(1 << 8))
+/* contains copies of user pages */
+#define RQF_COPY_USER		((__force req_flags_t)(1 << 9))
+/* vaguely specified driver internal error.  Ignored by the block layer */
+#define RQF_FAILED		((__force req_flags_t)(1 << 10))
+/* don't warn about errors */
+#define RQF_QUIET		((__force req_flags_t)(1 << 11))
+/* elevator private data attached */
+#define RQF_ELVPRIV		((__force req_flags_t)(1 << 12))
+/* account I/O stat */
+#define RQF_IO_STAT		((__force req_flags_t)(1 << 13))
+/* request came from our alloc pool */
+#define RQF_ALLOCED		((__force req_flags_t)(1 << 14))
+/* runtime pm request */
+#define RQF_PM			((__force req_flags_t)(1 << 15))
+/* on IO scheduler merge hash */
+#define RQF_HASHED		((__force req_flags_t)(1 << 16))
+/* IO stats tracking on */
+#define RQF_STATS		((__force req_flags_t)(1 << 17))
+/* Look at ->special_vec for the actual data payload instead of the
+   bio chain. */
+#define RQF_SPECIAL_PAYLOAD	((__force req_flags_t)(1 << 18))
+
+/* flags that prevent us from merging requests: */
+#define RQF_NOMERGE_FLAGS \
+	(RQF_STARTED | RQF_SOFTBARRIER | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)
+
 #define BLK_MAX_CDB	16
 
 /*
@@ -97,7 +148,8 @@ struct request {
 
 	int cpu;
 	unsigned cmd_type;
-	u64 cmd_flags;
+	unsigned int cmd_flags;		/* op and common flags */
+	req_flags_t rq_flags;
 	unsigned long atomic_flags;
 
 	/* the following two fields are internal, NEVER access directly */
@@ -126,6 +178,7 @@ struct request {
 	 */
 	union {
 		struct rb_node rb_node;	/* sort/lookup */
+		struct bio_vec special_vec;
 		void *completion_data;
 	};
 
@@ -151,6 +204,7 @@ struct request {
 	struct gendisk *rq_disk;
 	struct hd_struct *part;
 	unsigned long start_time;
+	struct blk_issue_stat issue_stat;
 #ifdef CONFIG_BLK_CGROUP
 	struct request_list *rl;		/* rl this rq is alloced from */
 	unsigned long long start_time_ns;
@@ -198,20 +252,6 @@ struct request {
 	struct request *next_rq;
 };
 
-#define REQ_OP_SHIFT (8 * sizeof(u64) - REQ_OP_BITS)
-#define req_op(req)  ((req)->cmd_flags >> REQ_OP_SHIFT)
-
-#define req_set_op(req, op) do {				\
-	WARN_ON(op >= (1 << REQ_OP_BITS));			\
-	(req)->cmd_flags &= ((1ULL << REQ_OP_SHIFT) - 1);	\
-	(req)->cmd_flags |= ((u64) (op) << REQ_OP_SHIFT);	\
-} while (0)
-
-#define req_set_op_attrs(req, op, flags) do {	\
-	req_set_op(req, op);			\
-	(req)->cmd_flags |= flags;		\
-} while (0)
-
 static inline unsigned short req_get_ioprio(struct request *req)
 {
 	return req->ioprio;
@@ -261,6 +301,15 @@ struct blk_queue_tag {
 #define BLK_SCSI_MAX_CMDS	(256)
 #define BLK_SCSI_CMD_PER_LONG	(BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))
 
+/*
+ * Zoned block device models (zoned limit).
+ */
+enum blk_zoned_model {
+	BLK_ZONED_NONE,	/* Regular block device */
+	BLK_ZONED_HA,	/* Host-aware zoned block device */
+	BLK_ZONED_HM,	/* Host-managed zoned block device */
+};
+
 struct queue_limits {
 	unsigned long		bounce_pfn;
 	unsigned long		seg_boundary_mask;
@@ -278,6 +327,7 @@ struct queue_limits {
 	unsigned int		max_discard_sectors;
 	unsigned int		max_hw_discard_sectors;
 	unsigned int		max_write_same_sectors;
+	unsigned int		max_write_zeroes_sectors;
 	unsigned int		discard_granularity;
 	unsigned int		discard_alignment;
 
@@ -290,8 +340,45 @@ struct queue_limits {
 	unsigned char		cluster;
 	unsigned char		discard_zeroes_data;
 	unsigned char		raid_partial_stripes_expensive;
+	enum blk_zoned_model	zoned;
 };
 
+#ifdef CONFIG_BLK_DEV_ZONED
+
+struct blk_zone_report_hdr {
+	unsigned int	nr_zones;
+	u8		padding[60];
+};
+
+extern int blkdev_report_zones(struct block_device *bdev,
+			       sector_t sector, struct blk_zone *zones,
+			       unsigned int *nr_zones, gfp_t gfp_mask);
+extern int blkdev_reset_zones(struct block_device *bdev, sector_t sectors,
+			      sector_t nr_sectors, gfp_t gfp_mask);
+
+extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
+				     unsigned int cmd, unsigned long arg);
+extern int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode,
+				    unsigned int cmd, unsigned long arg);
+
+#else /* CONFIG_BLK_DEV_ZONED */
+
+static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
+					    fmode_t mode, unsigned int cmd,
+					    unsigned long arg)
+{
+	return -ENOTTY;
+}
+
+static inline int blkdev_reset_zones_ioctl(struct block_device *bdev,
+					   fmode_t mode, unsigned int cmd,
+					   unsigned long arg)
+{
+	return -ENOTTY;
+}
+
+#endif /* CONFIG_BLK_DEV_ZONED */
+
 struct request_queue {
 	/*
 	 * Together with queue_head for cacheline sharing
@@ -302,6 +389,8 @@ struct request_queue {
 	int			nr_rqs[2];	/* # allocated [a]sync rqs */
 	int			nr_rqs_elvpriv;	/* # allocated rqs w/ elvpriv */
 
+	struct rq_wb		*rq_wb;
+
 	/*
 	 * If blkcg is not used, @q->root_rl serves all requests.  If blkcg
 	 * is used, root blkg allocates from @q->root_rl and all other
@@ -327,6 +416,8 @@ struct request_queue {
 	struct blk_mq_ctx __percpu	*queue_ctx;
 	unsigned int		nr_queues;
 
+	unsigned int		queue_depth;
+
 	/* hw dispatch queues */
 	struct blk_mq_hw_ctx	**queue_hw_ctx;
 	unsigned int		nr_hw_queues;
@@ -412,6 +503,9 @@ struct request_queue {
 
 	unsigned int		nr_sorted;
 	unsigned int		in_flight[2];
+
+	struct blk_rq_stat	rq_stats[2];
+
 	/*
 	 * Number of active block driver functions for which blk_drain_queue()
 	 * must wait. Must be incremented around functions that unlock the
@@ -420,6 +514,7 @@ struct request_queue {
 	unsigned int		request_fn_active;
 
 	unsigned int		rq_timeout;
+	int			poll_nsec;
 	struct timer_list	timeout;
 	struct work_struct	timeout_work;
 	struct list_head	timeout_list;
@@ -505,6 +600,7 @@ struct request_queue {
 #define QUEUE_FLAG_FUA	       24	/* device supports FUA writes */
 #define QUEUE_FLAG_FLUSH_NQ    25	/* flush not queueuable */
 #define QUEUE_FLAG_DAX         26	/* device supports DAX */
+#define QUEUE_FLAG_STATS       27	/* track rq completion times */
 
 #define QUEUE_FLAG_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
 				 (1 << QUEUE_FLAG_STACKABLE)	|	\
@@ -601,7 +697,7 @@ static inline void queue_flag_clear(unsigned int flag, struct request_queue *q)
 			     REQ_FAILFAST_DRIVER))
 
 #define blk_account_rq(rq) \
-	(((rq)->cmd_flags & REQ_STARTED) && \
+	(((rq)->rq_flags & RQF_STARTED) && \
 	 ((rq)->cmd_type == REQ_TYPE_FS))
 
 #define blk_rq_cpu_valid(rq)	((rq)->cpu != -1)
@@ -627,17 +723,31 @@ static inline unsigned int blk_queue_cluster(struct request_queue *q)
 	return q->limits.cluster;
 }
 
-/*
- * We regard a request as sync, if either a read or a sync write
- */
-static inline bool rw_is_sync(int op, unsigned int rw_flags)
+static inline enum blk_zoned_model
+blk_queue_zoned_model(struct request_queue *q)
+{
+	return q->limits.zoned;
+}
+
+static inline bool blk_queue_is_zoned(struct request_queue *q)
+{
+	switch (blk_queue_zoned_model(q)) {
+	case BLK_ZONED_HA:
+	case BLK_ZONED_HM:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static inline unsigned int blk_queue_zone_size(struct request_queue *q)
 {
-	return op == REQ_OP_READ || (rw_flags & REQ_SYNC);
+	return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0;
 }
 
 static inline bool rq_is_sync(struct request *rq)
 {
-	return rw_is_sync(req_op(rq), rq->cmd_flags);
+	return op_is_sync(rq->cmd_flags);
 }
 
 static inline bool blk_rl_full(struct request_list *rl, bool sync)
@@ -669,8 +779,13 @@ static inline bool rq_mergeable(struct request *rq)
 	if (req_op(rq) == REQ_OP_FLUSH)
 		return false;
 
+	if (req_op(rq) == REQ_OP_WRITE_ZEROES)
+		return false;
+
 	if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
 		return false;
+	if (rq->rq_flags & RQF_NOMERGE_FLAGS)
+		return false;
 
 	return true;
 }
@@ -683,6 +798,14 @@ static inline bool blk_write_same_mergeable(struct bio *a, struct bio *b)
 	return false;
 }
 
+static inline unsigned int blk_queue_depth(struct request_queue *q)
+{
+	if (q->queue_depth)
+		return q->queue_depth;
+
+	return q->nr_requests;
+}
+
 /*
  * q->prep_rq_fn return values
  */
@@ -790,8 +913,6 @@ extern void __blk_put_request(struct request_queue *, struct request *);
 extern struct request *blk_get_request(struct request_queue *, int, gfp_t);
 extern void blk_rq_set_block_pc(struct request *);
 extern void blk_requeue_request(struct request_queue *, struct request *);
-extern void blk_add_request_payload(struct request *rq, struct page *page,
-		int offset, unsigned int len);
 extern int blk_lld_busy(struct request_queue *q);
 extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
 			     struct bio_set *bs, gfp_t gfp_mask,
@@ -824,6 +945,7 @@ extern void __blk_run_queue(struct request_queue *q);
 extern void __blk_run_queue_uncond(struct request_queue *q);
 extern void blk_run_queue(struct request_queue *);
 extern void blk_run_queue_async(struct request_queue *q);
+extern void blk_mq_quiesce_queue(struct request_queue *q);
 extern int blk_rq_map_user(struct request_queue *, struct request *,
 			   struct rq_map_data *, void __user *, unsigned long,
 			   gfp_t);
@@ -837,7 +959,7 @@ extern int blk_execute_rq(struct request_queue *, struct gendisk *,
 extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
 				  struct request *, int, rq_end_io_fn *);
 
-bool blk_poll(struct request_queue *q, blk_qc_t cookie);
+bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie);
 
 static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
 {
@@ -888,6 +1010,9 @@ static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
 	if (unlikely(op == REQ_OP_WRITE_SAME))
 		return q->limits.max_write_same_sectors;
 
+	if (unlikely(op == REQ_OP_WRITE_ZEROES))
+		return q->limits.max_write_zeroes_sectors;
+
 	return q->limits.max_sectors;
 }
 
@@ -934,6 +1059,20 @@ static inline unsigned int blk_rq_count_bios(struct request *rq)
 }
 
 /*
+ * blk_rq_set_prio - associate a request with prio from ioc
+ * @rq: request of interest
+ * @ioc: target iocontext
+ *
+ * Assocate request prio with ioc prio so request based drivers
+ * can leverage priority information.
+ */
+static inline void blk_rq_set_prio(struct request *rq, struct io_context *ioc)
+{
+	if (ioc)
+		rq->ioprio = ioc->ioprio;
+}
+
+/*
  * Request issue related functions.
  */
 extern struct request *blk_peek_request(struct request_queue *q);
@@ -991,6 +1130,8 @@ extern void blk_queue_max_discard_sectors(struct request_queue *q,
 		unsigned int max_discard_sectors);
 extern void blk_queue_max_write_same_sectors(struct request_queue *q,
 		unsigned int max_write_same_sectors);
+extern void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
+		unsigned int max_write_same_sectors);
 extern void blk_queue_logical_block_size(struct request_queue *, unsigned short);
 extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
 extern void blk_queue_alignment_offset(struct request_queue *q,
@@ -999,6 +1140,7 @@ extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
 extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
 extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
 extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
+extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
 extern void blk_set_default_limits(struct queue_limits *lim);
 extern void blk_set_stacking_limits(struct queue_limits *lim);
 extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
@@ -1027,6 +1169,13 @@ extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
 extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
 extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev);
 
+static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)
+{
+	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
+		return 1;
+	return rq->nr_phys_segments;
+}
+
 extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
 extern void blk_dump_rq_flags(struct request *, char *);
 extern long nr_blockdev_pages(void);
@@ -1057,7 +1206,7 @@ static inline int blk_pre_runtime_suspend(struct request_queue *q)
 static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {}
 static inline void blk_pre_runtime_resume(struct request_queue *q) {}
 static inline void blk_post_runtime_resume(struct request_queue *q, int err) {}
-extern inline void blk_set_runtime_active(struct request_queue *q) {}
+static inline void blk_set_runtime_active(struct request_queue *q) {}
 #endif
 
 /*
@@ -1078,6 +1227,7 @@ struct blk_plug {
 	struct list_head cb_list; /* md requires an unplug callback */
 };
 #define BLK_MAX_REQUEST_COUNT 16
+#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
 
 struct blk_plug_cb;
 typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
@@ -1151,6 +1301,9 @@ extern int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
 		struct bio **biop);
 extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
 		sector_t nr_sects, gfp_t gfp_mask, struct page *page);
+extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
+		sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
+		bool discard);
 extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
 		sector_t nr_sects, gfp_t gfp_mask, bool discard);
 static inline int sb_issue_discard(struct super_block *sb, sector_t block,
@@ -1354,6 +1507,46 @@ static inline unsigned int bdev_write_same(struct block_device *bdev)
 	return 0;
 }
 
+static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
+{
+	struct request_queue *q = bdev_get_queue(bdev);
+
+	if (q)
+		return q->limits.max_write_zeroes_sectors;
+
+	return 0;
+}
+
+static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev)
+{
+	struct request_queue *q = bdev_get_queue(bdev);
+
+	if (q)
+		return blk_queue_zoned_model(q);
+
+	return BLK_ZONED_NONE;
+}
+
+static inline bool bdev_is_zoned(struct block_device *bdev)
+{
+	struct request_queue *q = bdev_get_queue(bdev);
+
+	if (q)
+		return blk_queue_is_zoned(q);
+
+	return false;
+}
+
+static inline unsigned int bdev_zone_size(struct block_device *bdev)
+{
+	struct request_queue *q = bdev_get_queue(bdev);
+
+	if (q)
+		return blk_queue_zone_size(q);
+
+	return 0;
+}
+
 static inline int queue_dma_alignment(struct request_queue *q)
 {
 	return q ? q->dma_alignment : 511;
diff --git a/include/linux/blktrace_api.h b/include/linux/blktrace_api.h
index cceb72f9e29f..e417f080219a 100644
--- a/include/linux/blktrace_api.h
+++ b/include/linux/blktrace_api.h
@@ -118,7 +118,7 @@ static inline int blk_cmd_buf_len(struct request *rq)
 }
 
 extern void blk_dump_cmd(char *buf, struct request *rq);
-extern void blk_fill_rwbs(char *rwbs, int op, u32 rw, int bytes);
+extern void blk_fill_rwbs(char *rwbs, unsigned int op, int bytes);
 
 #endif /* CONFIG_EVENT_TRACING && CONFIG_BLOCK */
 
diff --git a/include/linux/bsg-lib.h b/include/linux/bsg-lib.h
index a226652a5a6c..657a718c27d2 100644
--- a/include/linux/bsg-lib.h
+++ b/include/linux/bsg-lib.h
@@ -40,6 +40,8 @@ struct bsg_job {
 	struct device *dev;
 	struct request *req;
 
+	struct kref kref;
+
 	/* Transport/driver specific request/reply structs */
 	void *request;
 	void *reply;
@@ -67,5 +69,7 @@ void bsg_job_done(struct bsg_job *job, int result,
 int bsg_setup_queue(struct device *dev, struct request_queue *q, char *name,
 		    bsg_job_fn *job_fn, int dd_job_size);
 void bsg_request_fn(struct request_queue *q);
+void bsg_job_put(struct bsg_job *job);
+int __must_check bsg_job_get(struct bsg_job *job);
 
 #endif
diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h
index ebbacd14d450..d67ab83823ad 100644
--- a/include/linux/buffer_head.h
+++ b/include/linux/buffer_head.h
@@ -168,7 +168,12 @@ int inode_has_buffers(struct inode *);
 void invalidate_inode_buffers(struct inode *);
 int remove_inode_buffers(struct inode *inode);
 int sync_mapping_buffers(struct address_space *mapping);
-void unmap_underlying_metadata(struct block_device *bdev, sector_t block);
+void clean_bdev_aliases(struct block_device *bdev, sector_t block,
+			sector_t len);
+static inline void clean_bdev_bh_alias(struct buffer_head *bh)
+{
+	clean_bdev_aliases(bh->b_bdev, bh->b_blocknr, 1);
+}
 
 void mark_buffer_async_write(struct buffer_head *bh);
 void __wait_on_buffer(struct buffer_head *);
diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h
index 2189935075b4..a951fd10aaaa 100644
--- a/include/linux/cacheinfo.h
+++ b/include/linux/cacheinfo.h
@@ -71,6 +71,7 @@ struct cpu_cacheinfo {
 	struct cacheinfo *info_list;
 	unsigned int num_levels;
 	unsigned int num_leaves;
+	bool cpu_map_populated;
 };
 
 /*
diff --git a/include/linux/capability.h b/include/linux/capability.h
index dbc21c719ce6..6ffb67e10c06 100644
--- a/include/linux/capability.h
+++ b/include/linux/capability.h
@@ -240,8 +240,10 @@ static inline bool ns_capable_noaudit(struct user_namespace *ns, int cap)
 	return true;
 }
 #endif /* CONFIG_MULTIUSER */
+extern bool privileged_wrt_inode_uidgid(struct user_namespace *ns, const struct inode *inode);
 extern bool capable_wrt_inode_uidgid(const struct inode *inode, int cap);
 extern bool file_ns_capable(const struct file *file, struct user_namespace *ns, int cap);
+extern bool ptracer_capable(struct task_struct *tsk, struct user_namespace *ns);
 
 /* audit system wants to get cap info from files as well */
 extern int get_vfs_caps_from_disk(const struct dentry *dentry, struct cpu_vfs_cap_data *cpu_caps);
diff --git a/include/linux/ccp.h b/include/linux/ccp.h
index a7653339fedb..c71dd8fa5764 100644
--- a/include/linux/ccp.h
+++ b/include/linux/ccp.h
@@ -11,8 +11,8 @@
  * published by the Free Software Foundation.
  */
 
-#ifndef __CPP_H__
-#define __CPP_H__
+#ifndef __CCP_H__
+#define __CCP_H__
 
 #include <linux/scatterlist.h>
 #include <linux/workqueue.h>
@@ -553,7 +553,7 @@ enum ccp_engine {
 #define CCP_CMD_PASSTHRU_NO_DMA_MAP	0x00000002
 
 /**
- * struct ccp_cmd - CPP operation request
+ * struct ccp_cmd - CCP operation request
  * @entry: list element (ccp driver use only)
  * @work: work element used for callbacks (ccp driver use only)
  * @ccp: CCP device to be run on (ccp driver use only)
diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h
index 8dbd7879fdc6..67bcef2ecddb 100644
--- a/include/linux/ceph/messenger.h
+++ b/include/linux/ceph/messenger.h
@@ -1,7 +1,7 @@
 #ifndef __FS_CEPH_MESSENGER_H
 #define __FS_CEPH_MESSENGER_H
 
-#include <linux/blk_types.h>
+#include <linux/bvec.h>
 #include <linux/kref.h>
 #include <linux/mutex.h>
 #include <linux/net.h>
diff --git a/include/linux/clk.h b/include/linux/clk.h
index 123c02788807..e9d36b3e49de 100644
--- a/include/linux/clk.h
+++ b/include/linux/clk.h
@@ -17,8 +17,9 @@
 #include <linux/notifier.h>
 
 struct device;
-
 struct clk;
+struct device_node;
+struct of_phandle_args;
 
 /**
  * DOC: clk notifier callback types
@@ -249,6 +250,23 @@ struct clk *clk_get(struct device *dev, const char *id);
 struct clk *devm_clk_get(struct device *dev, const char *id);
 
 /**
+ * devm_get_clk_from_child - lookup and obtain a managed reference to a
+ *			     clock producer from child node.
+ * @dev: device for clock "consumer"
+ * @np: pointer to clock consumer node
+ * @con_id: clock consumer ID
+ *
+ * This function parses the clocks, and uses them to look up the
+ * struct clk from the registered list of clock providers by using
+ * @np and @con_id
+ *
+ * The clock will automatically be freed when the device is unbound
+ * from the bus.
+ */
+struct clk *devm_get_clk_from_child(struct device *dev,
+				    struct device_node *np, const char *con_id);
+
+/**
  * clk_enable - inform the system when the clock source should be running.
  * @clk: clock source
  *
@@ -432,6 +450,12 @@ static inline struct clk *devm_clk_get(struct device *dev, const char *id)
 	return NULL;
 }
 
+static inline struct clk *devm_get_clk_from_child(struct device *dev,
+				struct device_node *np, const char *con_id)
+{
+	return NULL;
+}
+
 static inline void clk_put(struct clk *clk) {}
 
 static inline void devm_clk_put(struct device *dev, struct clk *clk) {}
@@ -501,9 +525,6 @@ static inline void clk_disable_unprepare(struct clk *clk)
 	clk_unprepare(clk);
 }
 
-struct device_node;
-struct of_phandle_args;
-
 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
 struct clk *of_clk_get(struct device_node *np, int index);
 struct clk *of_clk_get_by_name(struct device_node *np, const char *name);
diff --git a/include/linux/clk/renesas.h b/include/linux/clk/renesas.h
index ba6fa4148515..9ebf1f8243bb 100644
--- a/include/linux/clk/renesas.h
+++ b/include/linux/clk/renesas.h
@@ -20,10 +20,6 @@ struct device;
 struct device_node;
 struct generic_pm_domain;
 
-void r8a7778_clocks_init(u32 mode);
-void r8a7779_clocks_init(u32 mode);
-void rcar_gen2_clocks_init(u32 mode);
-
 void cpg_mstp_add_clk_domain(struct device_node *np);
 #ifdef CONFIG_CLK_RENESAS_CPG_MSTP
 int cpg_mstp_attach_dev(struct generic_pm_domain *unused, struct device *dev);
diff --git a/include/linux/configfs.h b/include/linux/configfs.h
index d9d6a9d77489..9a30b921f740 100644
--- a/include/linux/configfs.h
+++ b/include/linux/configfs.h
@@ -228,7 +228,7 @@ static struct configfs_bin_attribute _pfx##attr_##_name = {	\
 struct configfs_item_operations {
 	void (*release)(struct config_item *);
 	int (*allow_link)(struct config_item *src, struct config_item *target);
-	int (*drop_link)(struct config_item *src, struct config_item *target);
+	void (*drop_link)(struct config_item *src, struct config_item *target);
 };
 
 struct configfs_group_operations {
diff --git a/include/linux/console.h b/include/linux/console.h
index d530c4627e54..9c26c6685587 100644
--- a/include/linux/console.h
+++ b/include/linux/console.h
@@ -28,9 +28,17 @@ struct tty_struct;
 #define VT100ID "\033[?1;2c"
 #define VT102ID "\033[?6c"
 
+enum con_scroll {
+	SM_UP,
+	SM_DOWN,
+};
+
 /**
  * struct consw - callbacks for consoles
  *
+ * @con_scroll: move lines from @top to @bottom in direction @dir by @lines.
+ *		Return true if no generic handling should be done.
+ *		Invoked by csi_M and printing to the console.
  * @con_set_palette: sets the palette of the console to @table (optional)
  * @con_scrolldelta: the contents of the console should be scrolled by @lines.
  *		     Invoked by user. (optional)
@@ -44,7 +52,9 @@ struct consw {
 	void	(*con_putc)(struct vc_data *, int, int, int);
 	void	(*con_putcs)(struct vc_data *, const unsigned short *, int, int, int);
 	void	(*con_cursor)(struct vc_data *, int);
-	int	(*con_scroll)(struct vc_data *, int, int, int, int);
+	bool	(*con_scroll)(struct vc_data *, unsigned int top,
+			unsigned int bottom, enum con_scroll dir,
+			unsigned int lines);
 	int	(*con_switch)(struct vc_data *);
 	int	(*con_blank)(struct vc_data *, int, int);
 	int	(*con_font_set)(struct vc_data *, struct console_font *, unsigned);
@@ -99,10 +109,6 @@ static inline int con_debug_leave(void)
 }
 #endif
 
-/* scroll */
-#define SM_UP       (1)
-#define SM_DOWN     (2)
-
 /* cursor */
 #define CM_DRAW     (1)
 #define CM_ERASE    (2)
diff --git a/include/linux/cpu.h b/include/linux/cpu.h
index e571128ad99a..09807c2ce328 100644
--- a/include/linux/cpu.h
+++ b/include/linux/cpu.h
@@ -238,6 +238,8 @@ void arch_cpu_idle_dead(void);
 int cpu_report_state(int cpu);
 int cpu_check_up_prepare(int cpu);
 void cpu_set_state_online(int cpu);
+void play_idle(unsigned long duration_ms);
+
 #ifdef CONFIG_HOTPLUG_CPU
 bool cpu_wait_death(unsigned int cpu, int seconds);
 bool cpu_report_death(void);
diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
index 32dc0cbd51ca..7e05c5e4e45c 100644
--- a/include/linux/cpufreq.h
+++ b/include/linux/cpufreq.h
@@ -175,7 +175,7 @@ void disable_cpufreq(void);
 
 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy);
 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu);
-int cpufreq_update_policy(unsigned int cpu);
+void cpufreq_update_policy(unsigned int cpu);
 bool have_governor_per_policy(void);
 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy);
 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy);
@@ -234,6 +234,10 @@ __ATTR(_name, _perm, show_##_name, NULL)
 static struct freq_attr _name =			\
 __ATTR(_name, 0644, show_##_name, store_##_name)
 
+#define cpufreq_freq_attr_wo(_name)		\
+static struct freq_attr _name =			\
+__ATTR(_name, 0200, NULL, store_##_name)
+
 struct global_attr {
 	struct attribute attr;
 	ssize_t (*show)(struct kobject *kobj,
diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h
index bb31373c3478..da346f2817a8 100644
--- a/include/linux/cpuidle.h
+++ b/include/linux/cpuidle.h
@@ -74,6 +74,7 @@ struct cpuidle_driver_kobj;
 struct cpuidle_device {
 	unsigned int		registered:1;
 	unsigned int		enabled:1;
+	unsigned int		use_deepest_state:1;
 	unsigned int		cpu;
 
 	int			last_residency;
@@ -192,11 +193,12 @@ static inline struct cpuidle_driver *cpuidle_get_cpu_driver(
 static inline struct cpuidle_device *cpuidle_get_device(void) {return NULL; }
 #endif
 
-#if defined(CONFIG_CPU_IDLE) && defined(CONFIG_SUSPEND)
+#ifdef CONFIG_CPU_IDLE
 extern int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
 				      struct cpuidle_device *dev);
 extern int cpuidle_enter_freeze(struct cpuidle_driver *drv,
 				struct cpuidle_device *dev);
+extern void cpuidle_use_deepest_state(bool enable);
 #else
 static inline int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
 					     struct cpuidle_device *dev)
@@ -204,6 +206,9 @@ static inline int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
 static inline int cpuidle_enter_freeze(struct cpuidle_driver *drv,
 				       struct cpuidle_device *dev)
 {return -ENODEV; }
+static inline void cpuidle_use_deepest_state(bool enable)
+{
+}
 #endif
 
 /* kernel/sched/idle.c */
@@ -235,8 +240,6 @@ struct cpuidle_governor {
 	int  (*select)		(struct cpuidle_driver *drv,
 					struct cpuidle_device *dev);
 	void (*reflect)		(struct cpuidle_device *dev, int index);
-
-	struct module 		*owner;
 };
 
 #ifdef CONFIG_CPU_IDLE
diff --git a/include/linux/crypto.h b/include/linux/crypto.h
index 7cee5551625b..167aea29d41e 100644
--- a/include/linux/crypto.h
+++ b/include/linux/crypto.h
@@ -50,6 +50,8 @@
 #define CRYPTO_ALG_TYPE_SKCIPHER	0x00000005
 #define CRYPTO_ALG_TYPE_GIVCIPHER	0x00000006
 #define CRYPTO_ALG_TYPE_KPP		0x00000008
+#define CRYPTO_ALG_TYPE_ACOMPRESS	0x0000000a
+#define CRYPTO_ALG_TYPE_SCOMPRESS	0x0000000b
 #define CRYPTO_ALG_TYPE_RNG		0x0000000c
 #define CRYPTO_ALG_TYPE_AKCIPHER	0x0000000d
 #define CRYPTO_ALG_TYPE_DIGEST		0x0000000e
@@ -60,6 +62,7 @@
 #define CRYPTO_ALG_TYPE_HASH_MASK	0x0000000e
 #define CRYPTO_ALG_TYPE_AHASH_MASK	0x0000000e
 #define CRYPTO_ALG_TYPE_BLKCIPHER_MASK	0x0000000c
+#define CRYPTO_ALG_TYPE_ACOMPRESS_MASK	0x0000000e
 
 #define CRYPTO_ALG_LARVAL		0x00000010
 #define CRYPTO_ALG_DEAD			0x00000020
@@ -87,7 +90,7 @@
 #define CRYPTO_ALG_TESTED		0x00000400
 
 /*
- * Set if the algorithm is an instance that is build from templates.
+ * Set if the algorithm is an instance that is built from templates.
  */
 #define CRYPTO_ALG_INSTANCE		0x00000800
 
diff --git a/include/linux/dax.h b/include/linux/dax.h
index add6c4bc568f..f97bcfe79472 100644
--- a/include/linux/dax.h
+++ b/include/linux/dax.h
@@ -8,25 +8,44 @@
 
 struct iomap_ops;
 
-/* We use lowest available exceptional entry bit for locking */
+/*
+ * We use lowest available bit in exceptional entry for locking, one bit for
+ * the entry size (PMD) and two more to tell us if the entry is a huge zero
+ * page (HZP) or an empty entry that is just used for locking.  In total four
+ * special bits.
+ *
+ * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the HZP and
+ * EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
+ * block allocation.
+ */
+#define RADIX_DAX_SHIFT	(RADIX_TREE_EXCEPTIONAL_SHIFT + 4)
 #define RADIX_DAX_ENTRY_LOCK (1 << RADIX_TREE_EXCEPTIONAL_SHIFT)
+#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1))
+#define RADIX_DAX_HZP (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2))
+#define RADIX_DAX_EMPTY (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3))
 
-ssize_t iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter,
+static inline unsigned long dax_radix_sector(void *entry)
+{
+	return (unsigned long)entry >> RADIX_DAX_SHIFT;
+}
+
+static inline void *dax_radix_locked_entry(sector_t sector, unsigned long flags)
+{
+	return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags |
+			((unsigned long)sector << RADIX_DAX_SHIFT) |
+			RADIX_DAX_ENTRY_LOCK);
+}
+
+ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
 		struct iomap_ops *ops);
-ssize_t dax_do_io(struct kiocb *, struct inode *, struct iov_iter *,
-		  get_block_t, dio_iodone_t, int flags);
-int dax_zero_page_range(struct inode *, loff_t from, unsigned len, get_block_t);
-int dax_truncate_page(struct inode *, loff_t from, get_block_t);
-int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
+int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
 			struct iomap_ops *ops);
-int dax_fault(struct vm_area_struct *, struct vm_fault *, get_block_t);
 int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index);
 void dax_wake_mapping_entry_waiter(struct address_space *mapping,
-				   pgoff_t index, bool wake_all);
+		pgoff_t index, void *entry, bool wake_all);
 
 #ifdef CONFIG_FS_DAX
 struct page *read_dax_sector(struct block_device *bdev, sector_t n);
-void dax_unlock_mapping_entry(struct address_space *mapping, pgoff_t index);
 int __dax_zero_page_range(struct block_device *bdev, sector_t sector,
 		unsigned int offset, unsigned int length);
 #else
@@ -35,12 +54,6 @@ static inline struct page *read_dax_sector(struct block_device *bdev,
 {
 	return ERR_PTR(-ENXIO);
 }
-/* Shouldn't ever be called when dax is disabled. */
-static inline void dax_unlock_mapping_entry(struct address_space *mapping,
-					    pgoff_t index)
-{
-	BUG();
-}
 static inline int __dax_zero_page_range(struct block_device *bdev,
 		sector_t sector, unsigned int offset, unsigned int length)
 {
@@ -48,18 +61,28 @@ static inline int __dax_zero_page_range(struct block_device *bdev,
 }
 #endif
 
-#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
-int dax_pmd_fault(struct vm_area_struct *, unsigned long addr, pmd_t *,
-				unsigned int flags, get_block_t);
+#ifdef CONFIG_FS_DAX_PMD
+static inline unsigned int dax_radix_order(void *entry)
+{
+	if ((unsigned long)entry & RADIX_DAX_PMD)
+		return PMD_SHIFT - PAGE_SHIFT;
+	return 0;
+}
+int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address,
+		pmd_t *pmd, unsigned int flags, struct iomap_ops *ops);
 #else
-static inline int dax_pmd_fault(struct vm_area_struct *vma, unsigned long addr,
-				pmd_t *pmd, unsigned int flags, get_block_t gb)
+static inline unsigned int dax_radix_order(void *entry)
+{
+	return 0;
+}
+static inline int dax_iomap_pmd_fault(struct vm_area_struct *vma,
+		unsigned long address, pmd_t *pmd, unsigned int flags,
+		struct iomap_ops *ops)
 {
 	return VM_FAULT_FALLBACK;
 }
 #endif
 int dax_pfn_mkwrite(struct vm_area_struct *, struct vm_fault *);
-#define dax_mkwrite(vma, vmf, gb)	dax_fault(vma, vmf, gb)
 
 static inline bool vma_is_dax(struct vm_area_struct *vma)
 {
diff --git a/include/linux/debugfs.h b/include/linux/debugfs.h
index bf1907d96097..014cc564d1c4 100644
--- a/include/linux/debugfs.h
+++ b/include/linux/debugfs.h
@@ -63,6 +63,21 @@ debugfs_real_fops(const struct file *filp)
 	return filp->f_path.dentry->d_fsdata;
 }
 
+#define DEFINE_DEBUGFS_ATTRIBUTE(__fops, __get, __set, __fmt)		\
+static int __fops ## _open(struct inode *inode, struct file *file)	\
+{									\
+	__simple_attr_check_format(__fmt, 0ull);			\
+	return simple_attr_open(inode, file, __get, __set, __fmt);	\
+}									\
+static const struct file_operations __fops = {				\
+	.owner	 = THIS_MODULE,						\
+	.open	 = __fops ## _open,					\
+	.release = simple_attr_release,					\
+	.read	 = debugfs_attr_read,					\
+	.write	 = debugfs_attr_write,					\
+	.llseek  = generic_file_llseek,					\
+}
+
 #if defined(CONFIG_DEBUG_FS)
 
 struct dentry *debugfs_create_file(const char *name, umode_t mode,
@@ -100,21 +115,6 @@ ssize_t debugfs_attr_read(struct file *file, char __user *buf,
 ssize_t debugfs_attr_write(struct file *file, const char __user *buf,
 			size_t len, loff_t *ppos);
 
-#define DEFINE_DEBUGFS_ATTRIBUTE(__fops, __get, __set, __fmt)		\
-static int __fops ## _open(struct inode *inode, struct file *file)	\
-{									\
-	__simple_attr_check_format(__fmt, 0ull);			\
-	return simple_attr_open(inode, file, __get, __set, __fmt);	\
-}									\
-static const struct file_operations __fops = {				\
-	.owner	 = THIS_MODULE,					\
-	.open	 = __fops ## _open,					\
-	.release = simple_attr_release,				\
-	.read	 = debugfs_attr_read,					\
-	.write	 = debugfs_attr_write,					\
-	.llseek  = generic_file_llseek,				\
-}
-
 struct dentry *debugfs_rename(struct dentry *old_dir, struct dentry *old_dentry,
                 struct dentry *new_dir, const char *new_name);
 
@@ -234,8 +234,18 @@ static inline void debugfs_use_file_finish(int srcu_idx)
 	__releases(&debugfs_srcu)
 { }
 
-#define DEFINE_DEBUGFS_ATTRIBUTE(__fops, __get, __set, __fmt)	\
-	static const struct file_operations __fops = { 0 }
+static inline ssize_t debugfs_attr_read(struct file *file, char __user *buf,
+					size_t len, loff_t *ppos)
+{
+	return -ENODEV;
+}
+
+static inline ssize_t debugfs_attr_write(struct file *file,
+					const char __user *buf,
+					size_t len, loff_t *ppos)
+{
+	return -ENODEV;
+}
 
 static inline struct dentry *debugfs_rename(struct dentry *old_dir, struct dentry *old_dentry,
                 struct dentry *new_dir, char *new_name)
diff --git a/include/linux/devfreq_cooling.h b/include/linux/devfreq_cooling.h
index 7adf6cc4b305..c35d0c0e0ada 100644
--- a/include/linux/devfreq_cooling.h
+++ b/include/linux/devfreq_cooling.h
@@ -20,7 +20,6 @@
 #include <linux/devfreq.h>
 #include <linux/thermal.h>
 
-#ifdef CONFIG_DEVFREQ_THERMAL
 
 /**
  * struct devfreq_cooling_power - Devfreq cooling power ops
@@ -37,12 +36,16 @@
  *			@dyn_power_coeff * frequency * voltage^2
  */
 struct devfreq_cooling_power {
-	unsigned long (*get_static_power)(unsigned long voltage);
-	unsigned long (*get_dynamic_power)(unsigned long freq,
+	unsigned long (*get_static_power)(struct devfreq *devfreq,
+					  unsigned long voltage);
+	unsigned long (*get_dynamic_power)(struct devfreq *devfreq,
+					   unsigned long freq,
 					   unsigned long voltage);
 	unsigned long dyn_power_coeff;
 };
 
+#ifdef CONFIG_DEVFREQ_THERMAL
+
 struct thermal_cooling_device *
 of_devfreq_cooling_register_power(struct device_node *np, struct devfreq *df,
 				  struct devfreq_cooling_power *dfc_power);
diff --git a/include/linux/device.h b/include/linux/device.h
index 94926d3ad6c6..491b4c0ca633 100644
--- a/include/linux/device.h
+++ b/include/linux/device.h
@@ -362,6 +362,7 @@ int subsys_virtual_register(struct bus_type *subsys,
  * @name:	Name of the class.
  * @owner:	The module owner.
  * @class_attrs: Default attributes of this class.
+ * @class_groups: Default attributes of this class.
  * @dev_groups:	Default attributes of the devices that belong to the class.
  * @dev_kobj:	The kobject that represents this class and links it into the hierarchy.
  * @dev_uevent:	Called when a device is added, removed from this class, or a
@@ -390,6 +391,7 @@ struct class {
 	struct module		*owner;
 
 	struct class_attribute		*class_attrs;
+	const struct attribute_group	**class_groups;
 	const struct attribute_group	**dev_groups;
 	struct kobject			*dev_kobj;
 
@@ -465,6 +467,8 @@ struct class_attribute {
 	struct class_attribute class_attr_##_name = __ATTR_RW(_name)
 #define CLASS_ATTR_RO(_name) \
 	struct class_attribute class_attr_##_name = __ATTR_RO(_name)
+#define CLASS_ATTR_WO(_name) \
+	struct class_attribute class_attr_##_name = __ATTR_WO(_name)
 
 extern int __must_check class_create_file_ns(struct class *class,
 					     const struct class_attribute *attr,
@@ -727,6 +731,87 @@ struct device_dma_parameters {
 };
 
 /**
+ * enum device_link_state - Device link states.
+ * @DL_STATE_NONE: The presence of the drivers is not being tracked.
+ * @DL_STATE_DORMANT: None of the supplier/consumer drivers is present.
+ * @DL_STATE_AVAILABLE: The supplier driver is present, but the consumer is not.
+ * @DL_STATE_CONSUMER_PROBE: The consumer is probing (supplier driver present).
+ * @DL_STATE_ACTIVE: Both the supplier and consumer drivers are present.
+ * @DL_STATE_SUPPLIER_UNBIND: The supplier driver is unbinding.
+ */
+enum device_link_state {
+	DL_STATE_NONE = -1,
+	DL_STATE_DORMANT = 0,
+	DL_STATE_AVAILABLE,
+	DL_STATE_CONSUMER_PROBE,
+	DL_STATE_ACTIVE,
+	DL_STATE_SUPPLIER_UNBIND,
+};
+
+/*
+ * Device link flags.
+ *
+ * STATELESS: The core won't track the presence of supplier/consumer drivers.
+ * AUTOREMOVE: Remove this link automatically on consumer driver unbind.
+ * PM_RUNTIME: If set, the runtime PM framework will use this link.
+ * RPM_ACTIVE: Run pm_runtime_get_sync() on the supplier during link creation.
+ */
+#define DL_FLAG_STATELESS	BIT(0)
+#define DL_FLAG_AUTOREMOVE	BIT(1)
+#define DL_FLAG_PM_RUNTIME	BIT(2)
+#define DL_FLAG_RPM_ACTIVE	BIT(3)
+
+/**
+ * struct device_link - Device link representation.
+ * @supplier: The device on the supplier end of the link.
+ * @s_node: Hook to the supplier device's list of links to consumers.
+ * @consumer: The device on the consumer end of the link.
+ * @c_node: Hook to the consumer device's list of links to suppliers.
+ * @status: The state of the link (with respect to the presence of drivers).
+ * @flags: Link flags.
+ * @rpm_active: Whether or not the consumer device is runtime-PM-active.
+ * @rcu_head: An RCU head to use for deferred execution of SRCU callbacks.
+ */
+struct device_link {
+	struct device *supplier;
+	struct list_head s_node;
+	struct device *consumer;
+	struct list_head c_node;
+	enum device_link_state status;
+	u32 flags;
+	bool rpm_active;
+#ifdef CONFIG_SRCU
+	struct rcu_head rcu_head;
+#endif
+};
+
+/**
+ * enum dl_dev_state - Device driver presence tracking information.
+ * @DL_DEV_NO_DRIVER: There is no driver attached to the device.
+ * @DL_DEV_PROBING: A driver is probing.
+ * @DL_DEV_DRIVER_BOUND: The driver has been bound to the device.
+ * @DL_DEV_UNBINDING: The driver is unbinding from the device.
+ */
+enum dl_dev_state {
+	DL_DEV_NO_DRIVER = 0,
+	DL_DEV_PROBING,
+	DL_DEV_DRIVER_BOUND,
+	DL_DEV_UNBINDING,
+};
+
+/**
+ * struct dev_links_info - Device data related to device links.
+ * @suppliers: List of links to supplier devices.
+ * @consumers: List of links to consumer devices.
+ * @status: Driver status information.
+ */
+struct dev_links_info {
+	struct list_head suppliers;
+	struct list_head consumers;
+	enum dl_dev_state status;
+};
+
+/**
  * struct device - The basic device structure
  * @parent:	The device's "parent" device, the device to which it is attached.
  * 		In most cases, a parent device is some sort of bus or host
@@ -751,6 +836,7 @@ struct device_dma_parameters {
  * 		on.  This shrinks the "Board Support Packages" (BSPs) and
  * 		minimizes board-specific #ifdefs in drivers.
  * @driver_data: Private pointer for driver specific info.
+ * @links:	Links to suppliers and consumers of this device.
  * @power:	For device power management.
  * 		See Documentation/power/admin-guide/devices.rst for details.
  * @pm_domain:	Provide callbacks that are executed during system suspend,
@@ -818,6 +904,7 @@ struct device {
 					   core doesn't touch it */
 	void		*driver_data;	/* Driver data, set and get with
 					   dev_set/get_drvdata */
+	struct dev_links_info	links;
 	struct dev_pm_info	power;
 	struct dev_pm_domain	*pm_domain;
 
@@ -1135,6 +1222,10 @@ extern void device_shutdown(void);
 /* debugging and troubleshooting/diagnostic helpers. */
 extern const char *dev_driver_string(const struct device *dev);
 
+/* Device links interface. */
+struct device_link *device_link_add(struct device *consumer,
+				    struct device *supplier, u32 flags);
+void device_link_del(struct device_link *link);
 
 #ifdef CONFIG_PRINTK
 
diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h
index b91b023deffb..a52c6580cc9a 100644
--- a/include/linux/dm-io.h
+++ b/include/linux/dm-io.h
@@ -58,7 +58,7 @@ struct dm_io_notify {
 struct dm_io_client;
 struct dm_io_request {
 	int bi_op;			/* REQ_OP */
-	int bi_op_flags;		/* rq_flag_bits */
+	int bi_op_flags;		/* req_flag_bits */
 	struct dm_io_memory mem;	/* Memory to use for io */
 	struct dm_io_notify notify;	/* Synchronous if notify.fn is NULL */
 	struct dm_io_client *client;	/* Client memory handler */
diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
index e0b0741ae671..8daeb3ce0016 100644
--- a/include/linux/dma-buf.h
+++ b/include/linux/dma-buf.h
@@ -30,7 +30,7 @@
 #include <linux/list.h>
 #include <linux/dma-mapping.h>
 #include <linux/fs.h>
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 #include <linux/wait.h>
 
 struct device;
@@ -143,7 +143,7 @@ struct dma_buf {
 	wait_queue_head_t poll;
 
 	struct dma_buf_poll_cb_t {
-		struct fence_cb cb;
+		struct dma_fence_cb cb;
 		wait_queue_head_t *poll;
 
 		unsigned long active;
diff --git a/include/linux/dma-fence-array.h b/include/linux/dma-fence-array.h
new file mode 100644
index 000000000000..5900945f962d
--- /dev/null
+++ b/include/linux/dma-fence-array.h
@@ -0,0 +1,86 @@
+/*
+ * fence-array: aggregates fence to be waited together
+ *
+ * Copyright (C) 2016 Collabora Ltd
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ * Authors:
+ *	Gustavo Padovan <gustavo@padovan.org>
+ *	Christian König <christian.koenig@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __LINUX_DMA_FENCE_ARRAY_H
+#define __LINUX_DMA_FENCE_ARRAY_H
+
+#include <linux/dma-fence.h>
+
+/**
+ * struct dma_fence_array_cb - callback helper for fence array
+ * @cb: fence callback structure for signaling
+ * @array: reference to the parent fence array object
+ */
+struct dma_fence_array_cb {
+	struct dma_fence_cb cb;
+	struct dma_fence_array *array;
+};
+
+/**
+ * struct dma_fence_array - fence to represent an array of fences
+ * @base: fence base class
+ * @lock: spinlock for fence handling
+ * @num_fences: number of fences in the array
+ * @num_pending: fences in the array still pending
+ * @fences: array of the fences
+ */
+struct dma_fence_array {
+	struct dma_fence base;
+
+	spinlock_t lock;
+	unsigned num_fences;
+	atomic_t num_pending;
+	struct dma_fence **fences;
+};
+
+extern const struct dma_fence_ops dma_fence_array_ops;
+
+/**
+ * dma_fence_is_array - check if a fence is from the array subsclass
+ * @fence: fence to test
+ *
+ * Return true if it is a dma_fence_array and false otherwise.
+ */
+static inline bool dma_fence_is_array(struct dma_fence *fence)
+{
+	return fence->ops == &dma_fence_array_ops;
+}
+
+/**
+ * to_dma_fence_array - cast a fence to a dma_fence_array
+ * @fence: fence to cast to a dma_fence_array
+ *
+ * Returns NULL if the fence is not a dma_fence_array,
+ * or the dma_fence_array otherwise.
+ */
+static inline struct dma_fence_array *
+to_dma_fence_array(struct dma_fence *fence)
+{
+	if (fence->ops != &dma_fence_array_ops)
+		return NULL;
+
+	return container_of(fence, struct dma_fence_array, base);
+}
+
+struct dma_fence_array *dma_fence_array_create(int num_fences,
+					       struct dma_fence **fences,
+					       u64 context, unsigned seqno,
+					       bool signal_on_any);
+
+#endif /* __LINUX_DMA_FENCE_ARRAY_H */
diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h
new file mode 100644
index 000000000000..d51a7d23c358
--- /dev/null
+++ b/include/linux/dma-fence.h
@@ -0,0 +1,438 @@
+/*
+ * Fence mechanism for dma-buf to allow for asynchronous dma access
+ *
+ * Copyright (C) 2012 Canonical Ltd
+ * Copyright (C) 2012 Texas Instruments
+ *
+ * Authors:
+ * Rob Clark <robdclark@gmail.com>
+ * Maarten Lankhorst <maarten.lankhorst@canonical.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __LINUX_DMA_FENCE_H
+#define __LINUX_DMA_FENCE_H
+
+#include <linux/err.h>
+#include <linux/wait.h>
+#include <linux/list.h>
+#include <linux/bitops.h>
+#include <linux/kref.h>
+#include <linux/sched.h>
+#include <linux/printk.h>
+#include <linux/rcupdate.h>
+
+struct dma_fence;
+struct dma_fence_ops;
+struct dma_fence_cb;
+
+/**
+ * struct dma_fence - software synchronization primitive
+ * @refcount: refcount for this fence
+ * @ops: dma_fence_ops associated with this fence
+ * @rcu: used for releasing fence with kfree_rcu
+ * @cb_list: list of all callbacks to call
+ * @lock: spin_lock_irqsave used for locking
+ * @context: execution context this fence belongs to, returned by
+ *           dma_fence_context_alloc()
+ * @seqno: the sequence number of this fence inside the execution context,
+ * can be compared to decide which fence would be signaled later.
+ * @flags: A mask of DMA_FENCE_FLAG_* defined below
+ * @timestamp: Timestamp when the fence was signaled.
+ * @status: Optional, only valid if < 0, must be set before calling
+ * dma_fence_signal, indicates that the fence has completed with an error.
+ *
+ * the flags member must be manipulated and read using the appropriate
+ * atomic ops (bit_*), so taking the spinlock will not be needed most
+ * of the time.
+ *
+ * DMA_FENCE_FLAG_SIGNALED_BIT - fence is already signaled
+ * DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT - enable_signaling might have been called
+ * DMA_FENCE_FLAG_USER_BITS - start of the unused bits, can be used by the
+ * implementer of the fence for its own purposes. Can be used in different
+ * ways by different fence implementers, so do not rely on this.
+ *
+ * Since atomic bitops are used, this is not guaranteed to be the case.
+ * Particularly, if the bit was set, but dma_fence_signal was called right
+ * before this bit was set, it would have been able to set the
+ * DMA_FENCE_FLAG_SIGNALED_BIT, before enable_signaling was called.
+ * Adding a check for DMA_FENCE_FLAG_SIGNALED_BIT after setting
+ * DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT closes this race, and makes sure that
+ * after dma_fence_signal was called, any enable_signaling call will have either
+ * been completed, or never called at all.
+ */
+struct dma_fence {
+	struct kref refcount;
+	const struct dma_fence_ops *ops;
+	struct rcu_head rcu;
+	struct list_head cb_list;
+	spinlock_t *lock;
+	u64 context;
+	unsigned seqno;
+	unsigned long flags;
+	ktime_t timestamp;
+	int status;
+};
+
+enum dma_fence_flag_bits {
+	DMA_FENCE_FLAG_SIGNALED_BIT,
+	DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
+	DMA_FENCE_FLAG_USER_BITS, /* must always be last member */
+};
+
+typedef void (*dma_fence_func_t)(struct dma_fence *fence,
+				 struct dma_fence_cb *cb);
+
+/**
+ * struct dma_fence_cb - callback for dma_fence_add_callback
+ * @node: used by dma_fence_add_callback to append this struct to fence::cb_list
+ * @func: dma_fence_func_t to call
+ *
+ * This struct will be initialized by dma_fence_add_callback, additional
+ * data can be passed along by embedding dma_fence_cb in another struct.
+ */
+struct dma_fence_cb {
+	struct list_head node;
+	dma_fence_func_t func;
+};
+
+/**
+ * struct dma_fence_ops - operations implemented for fence
+ * @get_driver_name: returns the driver name.
+ * @get_timeline_name: return the name of the context this fence belongs to.
+ * @enable_signaling: enable software signaling of fence.
+ * @signaled: [optional] peek whether the fence is signaled, can be null.
+ * @wait: custom wait implementation, or dma_fence_default_wait.
+ * @release: [optional] called on destruction of fence, can be null
+ * @fill_driver_data: [optional] callback to fill in free-form debug info
+ * Returns amount of bytes filled, or -errno.
+ * @fence_value_str: [optional] fills in the value of the fence as a string
+ * @timeline_value_str: [optional] fills in the current value of the timeline
+ * as a string
+ *
+ * Notes on enable_signaling:
+ * For fence implementations that have the capability for hw->hw
+ * signaling, they can implement this op to enable the necessary
+ * irqs, or insert commands into cmdstream, etc.  This is called
+ * in the first wait() or add_callback() path to let the fence
+ * implementation know that there is another driver waiting on
+ * the signal (ie. hw->sw case).
+ *
+ * This function can be called called from atomic context, but not
+ * from irq context, so normal spinlocks can be used.
+ *
+ * A return value of false indicates the fence already passed,
+ * or some failure occurred that made it impossible to enable
+ * signaling. True indicates successful enabling.
+ *
+ * fence->status may be set in enable_signaling, but only when false is
+ * returned.
+ *
+ * Calling dma_fence_signal before enable_signaling is called allows
+ * for a tiny race window in which enable_signaling is called during,
+ * before, or after dma_fence_signal. To fight this, it is recommended
+ * that before enable_signaling returns true an extra reference is
+ * taken on the fence, to be released when the fence is signaled.
+ * This will mean dma_fence_signal will still be called twice, but
+ * the second time will be a noop since it was already signaled.
+ *
+ * Notes on signaled:
+ * May set fence->status if returning true.
+ *
+ * Notes on wait:
+ * Must not be NULL, set to dma_fence_default_wait for default implementation.
+ * the dma_fence_default_wait implementation should work for any fence, as long
+ * as enable_signaling works correctly.
+ *
+ * Must return -ERESTARTSYS if the wait is intr = true and the wait was
+ * interrupted, and remaining jiffies if fence has signaled, or 0 if wait
+ * timed out. Can also return other error values on custom implementations,
+ * which should be treated as if the fence is signaled. For example a hardware
+ * lockup could be reported like that.
+ *
+ * Notes on release:
+ * Can be NULL, this function allows additional commands to run on
+ * destruction of the fence. Can be called from irq context.
+ * If pointer is set to NULL, kfree will get called instead.
+ */
+
+struct dma_fence_ops {
+	const char * (*get_driver_name)(struct dma_fence *fence);
+	const char * (*get_timeline_name)(struct dma_fence *fence);
+	bool (*enable_signaling)(struct dma_fence *fence);
+	bool (*signaled)(struct dma_fence *fence);
+	signed long (*wait)(struct dma_fence *fence,
+			    bool intr, signed long timeout);
+	void (*release)(struct dma_fence *fence);
+
+	int (*fill_driver_data)(struct dma_fence *fence, void *data, int size);
+	void (*fence_value_str)(struct dma_fence *fence, char *str, int size);
+	void (*timeline_value_str)(struct dma_fence *fence,
+				   char *str, int size);
+};
+
+void dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
+		    spinlock_t *lock, u64 context, unsigned seqno);
+
+void dma_fence_release(struct kref *kref);
+void dma_fence_free(struct dma_fence *fence);
+
+/**
+ * dma_fence_put - decreases refcount of the fence
+ * @fence:	[in]	fence to reduce refcount of
+ */
+static inline void dma_fence_put(struct dma_fence *fence)
+{
+	if (fence)
+		kref_put(&fence->refcount, dma_fence_release);
+}
+
+/**
+ * dma_fence_get - increases refcount of the fence
+ * @fence:	[in]	fence to increase refcount of
+ *
+ * Returns the same fence, with refcount increased by 1.
+ */
+static inline struct dma_fence *dma_fence_get(struct dma_fence *fence)
+{
+	if (fence)
+		kref_get(&fence->refcount);
+	return fence;
+}
+
+/**
+ * dma_fence_get_rcu - get a fence from a reservation_object_list with
+ *                     rcu read lock
+ * @fence:	[in]	fence to increase refcount of
+ *
+ * Function returns NULL if no refcount could be obtained, or the fence.
+ */
+static inline struct dma_fence *dma_fence_get_rcu(struct dma_fence *fence)
+{
+	if (kref_get_unless_zero(&fence->refcount))
+		return fence;
+	else
+		return NULL;
+}
+
+/**
+ * dma_fence_get_rcu_safe  - acquire a reference to an RCU tracked fence
+ * @fencep:	[in]	pointer to fence to increase refcount of
+ *
+ * Function returns NULL if no refcount could be obtained, or the fence.
+ * This function handles acquiring a reference to a fence that may be
+ * reallocated within the RCU grace period (such as with SLAB_DESTROY_BY_RCU),
+ * so long as the caller is using RCU on the pointer to the fence.
+ *
+ * An alternative mechanism is to employ a seqlock to protect a bunch of
+ * fences, such as used by struct reservation_object. When using a seqlock,
+ * the seqlock must be taken before and checked after a reference to the
+ * fence is acquired (as shown here).
+ *
+ * The caller is required to hold the RCU read lock.
+ */
+static inline struct dma_fence *
+dma_fence_get_rcu_safe(struct dma_fence * __rcu *fencep)
+{
+	do {
+		struct dma_fence *fence;
+
+		fence = rcu_dereference(*fencep);
+		if (!fence || !dma_fence_get_rcu(fence))
+			return NULL;
+
+		/* The atomic_inc_not_zero() inside dma_fence_get_rcu()
+		 * provides a full memory barrier upon success (such as now).
+		 * This is paired with the write barrier from assigning
+		 * to the __rcu protected fence pointer so that if that
+		 * pointer still matches the current fence, we know we
+		 * have successfully acquire a reference to it. If it no
+		 * longer matches, we are holding a reference to some other
+		 * reallocated pointer. This is possible if the allocator
+		 * is using a freelist like SLAB_DESTROY_BY_RCU where the
+		 * fence remains valid for the RCU grace period, but it
+		 * may be reallocated. When using such allocators, we are
+		 * responsible for ensuring the reference we get is to
+		 * the right fence, as below.
+		 */
+		if (fence == rcu_access_pointer(*fencep))
+			return rcu_pointer_handoff(fence);
+
+		dma_fence_put(fence);
+	} while (1);
+}
+
+int dma_fence_signal(struct dma_fence *fence);
+int dma_fence_signal_locked(struct dma_fence *fence);
+signed long dma_fence_default_wait(struct dma_fence *fence,
+				   bool intr, signed long timeout);
+int dma_fence_add_callback(struct dma_fence *fence,
+			   struct dma_fence_cb *cb,
+			   dma_fence_func_t func);
+bool dma_fence_remove_callback(struct dma_fence *fence,
+			       struct dma_fence_cb *cb);
+void dma_fence_enable_sw_signaling(struct dma_fence *fence);
+
+/**
+ * dma_fence_is_signaled_locked - Return an indication if the fence
+ *                                is signaled yet.
+ * @fence:	[in]	the fence to check
+ *
+ * Returns true if the fence was already signaled, false if not. Since this
+ * function doesn't enable signaling, it is not guaranteed to ever return
+ * true if dma_fence_add_callback, dma_fence_wait or
+ * dma_fence_enable_sw_signaling haven't been called before.
+ *
+ * This function requires fence->lock to be held.
+ */
+static inline bool
+dma_fence_is_signaled_locked(struct dma_fence *fence)
+{
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+		return true;
+
+	if (fence->ops->signaled && fence->ops->signaled(fence)) {
+		dma_fence_signal_locked(fence);
+		return true;
+	}
+
+	return false;
+}
+
+/**
+ * dma_fence_is_signaled - Return an indication if the fence is signaled yet.
+ * @fence:	[in]	the fence to check
+ *
+ * Returns true if the fence was already signaled, false if not. Since this
+ * function doesn't enable signaling, it is not guaranteed to ever return
+ * true if dma_fence_add_callback, dma_fence_wait or
+ * dma_fence_enable_sw_signaling haven't been called before.
+ *
+ * It's recommended for seqno fences to call dma_fence_signal when the
+ * operation is complete, it makes it possible to prevent issues from
+ * wraparound between time of issue and time of use by checking the return
+ * value of this function before calling hardware-specific wait instructions.
+ */
+static inline bool
+dma_fence_is_signaled(struct dma_fence *fence)
+{
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+		return true;
+
+	if (fence->ops->signaled && fence->ops->signaled(fence)) {
+		dma_fence_signal(fence);
+		return true;
+	}
+
+	return false;
+}
+
+/**
+ * dma_fence_is_later - return if f1 is chronologically later than f2
+ * @f1:	[in]	the first fence from the same context
+ * @f2:	[in]	the second fence from the same context
+ *
+ * Returns true if f1 is chronologically later than f2. Both fences must be
+ * from the same context, since a seqno is not re-used across contexts.
+ */
+static inline bool dma_fence_is_later(struct dma_fence *f1,
+				      struct dma_fence *f2)
+{
+	if (WARN_ON(f1->context != f2->context))
+		return false;
+
+	return (int)(f1->seqno - f2->seqno) > 0;
+}
+
+/**
+ * dma_fence_later - return the chronologically later fence
+ * @f1:	[in]	the first fence from the same context
+ * @f2:	[in]	the second fence from the same context
+ *
+ * Returns NULL if both fences are signaled, otherwise the fence that would be
+ * signaled last. Both fences must be from the same context, since a seqno is
+ * not re-used across contexts.
+ */
+static inline struct dma_fence *dma_fence_later(struct dma_fence *f1,
+						struct dma_fence *f2)
+{
+	if (WARN_ON(f1->context != f2->context))
+		return NULL;
+
+	/*
+	 * Can't check just DMA_FENCE_FLAG_SIGNALED_BIT here, it may never
+	 * have been set if enable_signaling wasn't called, and enabling that
+	 * here is overkill.
+	 */
+	if (dma_fence_is_later(f1, f2))
+		return dma_fence_is_signaled(f1) ? NULL : f1;
+	else
+		return dma_fence_is_signaled(f2) ? NULL : f2;
+}
+
+signed long dma_fence_wait_timeout(struct dma_fence *,
+				   bool intr, signed long timeout);
+signed long dma_fence_wait_any_timeout(struct dma_fence **fences,
+				       uint32_t count,
+				       bool intr, signed long timeout,
+				       uint32_t *idx);
+
+/**
+ * dma_fence_wait - sleep until the fence gets signaled
+ * @fence:	[in]	the fence to wait on
+ * @intr:	[in]	if true, do an interruptible wait
+ *
+ * This function will return -ERESTARTSYS if interrupted by a signal,
+ * or 0 if the fence was signaled. Other error values may be
+ * returned on custom implementations.
+ *
+ * Performs a synchronous wait on this fence. It is assumed the caller
+ * directly or indirectly holds a reference to the fence, otherwise the
+ * fence might be freed before return, resulting in undefined behavior.
+ */
+static inline signed long dma_fence_wait(struct dma_fence *fence, bool intr)
+{
+	signed long ret;
+
+	/* Since dma_fence_wait_timeout cannot timeout with
+	 * MAX_SCHEDULE_TIMEOUT, only valid return values are
+	 * -ERESTARTSYS and MAX_SCHEDULE_TIMEOUT.
+	 */
+	ret = dma_fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT);
+
+	return ret < 0 ? ret : 0;
+}
+
+u64 dma_fence_context_alloc(unsigned num);
+
+#define DMA_FENCE_TRACE(f, fmt, args...) \
+	do {								\
+		struct dma_fence *__ff = (f);				\
+		if (IS_ENABLED(CONFIG_DMA_FENCE_TRACE))			\
+			pr_info("f %llu#%u: " fmt,			\
+				__ff->context, __ff->seqno, ##args);	\
+	} while (0)
+
+#define DMA_FENCE_WARN(f, fmt, args...) \
+	do {								\
+		struct dma_fence *__ff = (f);				\
+		pr_warn("f %llu#%u: " fmt, __ff->context, __ff->seqno,	\
+			 ##args);					\
+	} while (0)
+
+#define DMA_FENCE_ERR(f, fmt, args...) \
+	do {								\
+		struct dma_fence *__ff = (f);				\
+		pr_err("f %llu#%u: " fmt, __ff->context, __ff->seqno,	\
+			##args);					\
+	} while (0)
+
+#endif /* __LINUX_DMA_FENCE_H */
diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h
index 32c589062bd9..7f7e9a7e3839 100644
--- a/include/linux/dma-iommu.h
+++ b/include/linux/dma-iommu.h
@@ -61,6 +61,10 @@ void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size,
 		enum dma_data_direction dir, unsigned long attrs);
 void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
 		enum dma_data_direction dir, unsigned long attrs);
+dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
+		size_t size, enum dma_data_direction dir, unsigned long attrs);
+void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,
+		size_t size, enum dma_data_direction dir, unsigned long attrs);
 int iommu_dma_supported(struct device *dev, u64 mask);
 int iommu_dma_mapping_error(struct device *dev, dma_addr_t dma_addr);
 
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index 08528afdf58b..10c5a17b1f51 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -243,29 +243,33 @@ static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg
 		ops->unmap_sg(dev, sg, nents, dir, attrs);
 }
 
-static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
-				      size_t offset, size_t size,
-				      enum dma_data_direction dir)
+static inline dma_addr_t dma_map_page_attrs(struct device *dev,
+					    struct page *page,
+					    size_t offset, size_t size,
+					    enum dma_data_direction dir,
+					    unsigned long attrs)
 {
 	struct dma_map_ops *ops = get_dma_ops(dev);
 	dma_addr_t addr;
 
 	kmemcheck_mark_initialized(page_address(page) + offset, size);
 	BUG_ON(!valid_dma_direction(dir));
-	addr = ops->map_page(dev, page, offset, size, dir, 0);
+	addr = ops->map_page(dev, page, offset, size, dir, attrs);
 	debug_dma_map_page(dev, page, offset, size, dir, addr, false);
 
 	return addr;
 }
 
-static inline void dma_unmap_page(struct device *dev, dma_addr_t addr,
-				  size_t size, enum dma_data_direction dir)
+static inline void dma_unmap_page_attrs(struct device *dev,
+					dma_addr_t addr, size_t size,
+					enum dma_data_direction dir,
+					unsigned long attrs)
 {
 	struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
 	if (ops->unmap_page)
-		ops->unmap_page(dev, addr, size, dir, 0);
+		ops->unmap_page(dev, addr, size, dir, attrs);
 	debug_dma_unmap_page(dev, addr, size, dir, false);
 }
 
@@ -385,6 +389,8 @@ dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 #define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, 0)
 #define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, 0)
 #define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, 0)
+#define dma_map_page(d, p, o, s, r) dma_map_page_attrs(d, p, o, s, r, 0)
+#define dma_unmap_page(d, a, s, r) dma_unmap_page_attrs(d, a, s, r, 0)
 
 extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
 			   void *cpu_addr, dma_addr_t dma_addr, size_t size);
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index cc535a478bae..feee6ec6a13b 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -336,6 +336,12 @@ enum dma_slave_buswidth {
  * may or may not be applicable on memory sources.
  * @dst_maxburst: same as src_maxburst but for destination target
  * mutatis mutandis.
+ * @src_port_window_size: The length of the register area in words the data need
+ * to be accessed on the device side. It is only used for devices which is using
+ * an area instead of a single register to receive the data. Typically the DMA
+ * loops in this area in order to transfer the data.
+ * @dst_port_window_size: same as src_port_window_size but for the destination
+ * port.
  * @device_fc: Flow Controller Settings. Only valid for slave channels. Fill
  * with 'true' if peripheral should be flow controller. Direction will be
  * selected at Runtime.
@@ -363,6 +369,8 @@ struct dma_slave_config {
 	enum dma_slave_buswidth dst_addr_width;
 	u32 src_maxburst;
 	u32 dst_maxburst;
+	u32 src_port_window_size;
+	u32 dst_port_window_size;
 	bool device_fc;
 	unsigned int slave_id;
 };
diff --git a/include/linux/edac.h b/include/linux/edac.h
index 9e0d78966552..07c52c0af62d 100644
--- a/include/linux/edac.h
+++ b/include/linux/edac.h
@@ -18,6 +18,8 @@
 #include <linux/workqueue.h>
 #include <linux/debugfs.h>
 
+#define EDAC_DEVICE_NAME_LEN	31
+
 struct device;
 
 #define EDAC_OPSTATE_INVAL	-1
@@ -128,12 +130,21 @@ enum dev_type {
  *				fatal (maybe it is on an unused memory area,
  *				or the memory controller could recover from
  *				it for example, by re-trying the operation).
+ * @HW_EVENT_ERR_DEFERRED:	Deferred Error - Indicates an uncorrectable
+ *				error whose handling is not urgent. This could
+ *				be due to hardware data poisoning where the
+ *				system can continue operation until the poisoned
+ *				data is consumed. Preemptive measures may also
+ *				be taken, e.g. offlining pages, etc.
  * @HW_EVENT_ERR_FATAL:		Fatal Error - Uncorrected error that could not
  *				be recovered.
+ * @HW_EVENT_ERR_INFO:		Informational - The CPER spec defines a forth
+ *				type of error: informational logs.
  */
 enum hw_event_mc_err_type {
 	HW_EVENT_ERR_CORRECTED,
 	HW_EVENT_ERR_UNCORRECTED,
+	HW_EVENT_ERR_DEFERRED,
 	HW_EVENT_ERR_FATAL,
 	HW_EVENT_ERR_INFO,
 };
@@ -145,6 +156,8 @@ static inline char *mc_event_error_type(const unsigned int err_type)
 		return "Corrected";
 	case HW_EVENT_ERR_UNCORRECTED:
 		return "Uncorrected";
+	case HW_EVENT_ERR_DEFERRED:
+		return "Deferred";
 	case HW_EVENT_ERR_FATAL:
 		return "Fatal";
 	default:
@@ -157,7 +170,7 @@ static inline char *mc_event_error_type(const unsigned int err_type)
  * enum mem_type - memory types. For a more detailed reference, please see
  *			http://en.wikipedia.org/wiki/DRAM
  *
- * @MEM_EMPTY		Empty csrow
+ * @MEM_EMPTY:		Empty csrow
  * @MEM_RESERVED:	Reserved csrow type
  * @MEM_UNKNOWN:	Unknown csrow type
  * @MEM_FPM:		FPM - Fast Page Mode, used on systems up to 1995.
@@ -192,10 +205,11 @@ static inline char *mc_event_error_type(const unsigned int err_type)
  * @MEM_DDR3:		DDR3 RAM
  * @MEM_RDDR3:		Registered DDR3 RAM
  *			This is a variant of the DDR3 memories.
- * @MEM_LRDDR3		Load-Reduced DDR3 memory.
+ * @MEM_LRDDR3:		Load-Reduced DDR3 memory.
  * @MEM_DDR4:		Unbuffered DDR4 RAM
  * @MEM_RDDR4:		Registered DDR4 RAM
  *			This is a variant of the DDR4 memories.
+ * @MEM_LRDDR4:		Load-Reduced DDR4 memory.
  */
 enum mem_type {
 	MEM_EMPTY = 0,
@@ -218,6 +232,7 @@ enum mem_type {
 	MEM_LRDDR3,
 	MEM_DDR4,
 	MEM_RDDR4,
+	MEM_LRDDR4,
 };
 
 #define MEM_FLAG_EMPTY		BIT(MEM_EMPTY)
@@ -239,6 +254,7 @@ enum mem_type {
 #define MEM_FLAG_RDDR3          BIT(MEM_RDDR3)
 #define MEM_FLAG_DDR4           BIT(MEM_DDR4)
 #define MEM_FLAG_RDDR4          BIT(MEM_RDDR4)
+#define MEM_FLAG_LRDDR4         BIT(MEM_LRDDR4)
 
 /**
  * enum edac-type - Error Detection and Correction capabilities and mode
@@ -278,7 +294,7 @@ enum edac_type {
 
 /**
  * enum scrub_type - scrubbing capabilities
- * @SCRUB_UNKNOWN		Unknown if scrubber is available
+ * @SCRUB_UNKNOWN:		Unknown if scrubber is available
  * @SCRUB_NONE:			No scrubber
  * @SCRUB_SW_PROG:		SW progressive (sequential) scrubbing
  * @SCRUB_SW_SRC:		Software scrub only errors
@@ -287,7 +303,7 @@ enum edac_type {
  * @SCRUB_HW_PROG:		HW progressive (sequential) scrubbing
  * @SCRUB_HW_SRC:		Hardware scrub only errors
  * @SCRUB_HW_PROG_SRC:		Progressive hardware scrub from an error
- * SCRUB_HW_TUNABLE:		Hardware scrub frequency is tunable
+ * @SCRUB_HW_TUNABLE:		Hardware scrub frequency is tunable
  */
 enum scrub_type {
 	SCRUB_UNKNOWN =	0,
@@ -320,114 +336,6 @@ enum scrub_type {
 #define OP_RUNNING_POLL_INTR	0x203
 #define OP_OFFLINE		0x300
 
-/*
- * Concepts used at the EDAC subsystem
- *
- * There are several things to be aware of that aren't at all obvious:
- *
- * SOCKETS, SOCKET SETS, BANKS, ROWS, CHIP-SELECT ROWS, CHANNELS, etc..
- *
- * These are some of the many terms that are thrown about that don't always
- * mean what people think they mean (Inconceivable!).  In the interest of
- * creating a common ground for discussion, terms and their definitions
- * will be established.
- *
- * Memory devices:	The individual DRAM chips on a memory stick.  These
- *			devices commonly output 4 and 8 bits each (x4, x8).
- *			Grouping several of these in parallel provides the
- *			number of bits that the memory controller expects:
- *			typically 72 bits, in order to provide 64 bits +
- *			8 bits of ECC data.
- *
- * Memory Stick:	A printed circuit board that aggregates multiple
- *			memory devices in parallel.  In general, this is the
- *			Field Replaceable Unit (FRU) which gets replaced, in
- *			the case of excessive errors. Most often it is also
- *			called DIMM (Dual Inline Memory Module).
- *
- * Memory Socket:	A physical connector on the motherboard that accepts
- *			a single memory stick. Also called as "slot" on several
- *			datasheets.
- *
- * Channel:		A memory controller channel, responsible to communicate
- *			with a group of DIMMs. Each channel has its own
- *			independent control (command) and data bus, and can
- *			be used independently or grouped with other channels.
- *
- * Branch:		It is typically the highest hierarchy on a
- *			Fully-Buffered DIMM memory controller.
- *			Typically, it contains two channels.
- *			Two channels at the same branch can be used in single
- *			mode or in lockstep mode.
- *			When lockstep is enabled, the cacheline is doubled,
- *			but it generally brings some performance penalty.
- *			Also, it is generally not possible to point to just one
- *			memory stick when an error occurs, as the error
- *			correction code is calculated using two DIMMs instead
- *			of one. Due to that, it is capable of correcting more
- *			errors than on single mode.
- *
- * Single-channel:	The data accessed by the memory controller is contained
- *			into one dimm only. E. g. if the data is 64 bits-wide,
- *			the data flows to the CPU using one 64 bits parallel
- *			access.
- *			Typically used with SDR, DDR, DDR2 and DDR3 memories.
- *			FB-DIMM and RAMBUS use a different concept for channel,
- *			so this concept doesn't apply there.
- *
- * Double-channel:	The data size accessed by the memory controller is
- *			interlaced into two dimms, accessed at the same time.
- *			E. g. if the DIMM is 64 bits-wide (72 bits with ECC),
- *			the data flows to the CPU using a 128 bits parallel
- *			access.
- *
- * Chip-select row:	This is the name of the DRAM signal used to select the
- *			DRAM ranks to be accessed. Common chip-select rows for
- *			single channel are 64 bits, for dual channel 128 bits.
- *			It may not be visible by the memory controller, as some
- *			DIMM types have a memory buffer that can hide direct
- *			access to it from the Memory Controller.
- *
- * Single-Ranked stick:	A Single-ranked stick has 1 chip-select row of memory.
- *			Motherboards commonly drive two chip-select pins to
- *			a memory stick. A single-ranked stick, will occupy
- *			only one of those rows. The other will be unused.
- *
- * Double-Ranked stick:	A double-ranked stick has two chip-select rows which
- *			access different sets of memory devices.  The two
- *			rows cannot be accessed concurrently.
- *
- * Double-sided stick:	DEPRECATED TERM, see Double-Ranked stick.
- *			A double-sided stick has two chip-select rows which
- *			access different sets of memory devices. The two
- *			rows cannot be accessed concurrently. "Double-sided"
- *			is irrespective of the memory devices being mounted
- *			on both sides of the memory stick.
- *
- * Socket set:		All of the memory sticks that are required for
- *			a single memory access or all of the memory sticks
- *			spanned by a chip-select row.  A single socket set
- *			has two chip-select rows and if double-sided sticks
- *			are used these will occupy those chip-select rows.
- *
- * Bank:		This term is avoided because it is unclear when
- *			needing to distinguish between chip-select rows and
- *			socket sets.
- *
- * Controller pages:
- *
- * Physical pages:
- *
- * Virtual pages:
- *
- *
- * STRUCTURE ORGANIZATION AND CHOICES
- *
- *
- *
- * PS - I enjoyed writing all that about as much as you enjoyed reading it.
- */
-
 /**
  * enum edac_mc_layer - memory controller hierarchy layer
  *
@@ -452,7 +360,7 @@ enum edac_mc_layer_type {
 
 /**
  * struct edac_mc_layer - describes the memory controller hierarchy
- * @layer:		layer type
+ * @type:		layer type
  * @size:		number of components per layer. For example,
  *			if the channel layer has two channels, size = 2
  * @is_virt_csrow:	This layer is part of the "csrow" when old API
@@ -475,24 +383,28 @@ struct edac_mc_layer {
 #define EDAC_MAX_LAYERS		3
 
 /**
- * EDAC_DIMM_OFF - Macro responsible to get a pointer offset inside a pointer array
- *		   for the element given by [layer0,layer1,layer2] position
+ * EDAC_DIMM_OFF - Macro responsible to get a pointer offset inside a pointer
+ *		   array for the element given by [layer0,layer1,layer2]
+ *		   position
  *
  * @layers:	a struct edac_mc_layer array, describing how many elements
  *		were allocated for each layer
- * @n_layers:	Number of layers at the @layers array
+ * @nlayers:	Number of layers at the @layers array
  * @layer0:	layer0 position
  * @layer1:	layer1 position. Unused if n_layers < 2
  * @layer2:	layer2 position. Unused if n_layers < 3
  *
- * For 1 layer, this macro returns &var[layer0] - &var
+ * For 1 layer, this macro returns "var[layer0] - var";
+ *
  * For 2 layers, this macro is similar to allocate a bi-dimensional array
- *		and to return "&var[layer0][layer1] - &var"
+ * and to return "var[layer0][layer1] - var";
+ *
  * For 3 layers, this macro is similar to allocate a tri-dimensional array
- *		and to return "&var[layer0][layer1][layer2] - &var"
+ * and to return "var[layer0][layer1][layer2] - var".
  *
  * A loop could be used here to make it more generic, but, as we only have
  * 3 layers, this is a little faster.
+ *
  * By design, layers can never be 0 or more than 3. If that ever happens,
  * a NULL is returned, causing an OOPS during the memory allocation routine,
  * with would point to the developer that he's doing something wrong.
@@ -519,16 +431,18 @@ struct edac_mc_layer {
  *		were allocated for each layer
  * @var:	name of the var where we want to get the pointer
  *		(like mci->dimms)
- * @n_layers:	Number of layers at the @layers array
+ * @nlayers:	Number of layers at the @layers array
  * @layer0:	layer0 position
  * @layer1:	layer1 position. Unused if n_layers < 2
  * @layer2:	layer2 position. Unused if n_layers < 3
  *
- * For 1 layer, this macro returns &var[layer0]
+ * For 1 layer, this macro returns "var[layer0]";
+ *
  * For 2 layers, this macro is similar to allocate a bi-dimensional array
- *		and to return "&var[layer0][layer1]"
+ * and to return "var[layer0][layer1]";
+ *
  * For 3 layers, this macro is similar to allocate a tri-dimensional array
- *		and to return "&var[layer0][layer1][layer2]"
+ * and to return "var[layer0][layer1][layer2]";
  */
 #define EDAC_DIMM_PTR(layers, var, nlayers, layer0, layer1, layer2) ({	\
 	typeof(*var) __p;						\
@@ -614,7 +528,7 @@ struct errcount_attribute_data {
 };
 
 /**
- * edac_raw_error_desc - Raw error report structure
+ * struct edac_raw_error_desc - Raw error report structure
  * @grain:			minimum granularity for an error report, in bytes
  * @error_count:		number of errors of the same type
  * @top_layer:			top layer of the error (layer[0])
diff --git a/include/linux/elevator.h b/include/linux/elevator.h
index e7f358d2e5fc..b276e9ef0e0b 100644
--- a/include/linux/elevator.h
+++ b/include/linux/elevator.h
@@ -30,7 +30,7 @@ typedef int (elevator_dispatch_fn) (struct request_queue *, int);
 typedef void (elevator_add_req_fn) (struct request_queue *, struct request *);
 typedef struct request *(elevator_request_list_fn) (struct request_queue *, struct request *);
 typedef void (elevator_completed_req_fn) (struct request_queue *, struct request *);
-typedef int (elevator_may_queue_fn) (struct request_queue *, int, int);
+typedef int (elevator_may_queue_fn) (struct request_queue *, unsigned int);
 
 typedef void (elevator_init_icq_fn) (struct io_cq *);
 typedef void (elevator_exit_icq_fn) (struct io_cq *);
@@ -108,6 +108,11 @@ struct elevator_type
 
 #define ELV_HASH_BITS 6
 
+void elv_rqhash_del(struct request_queue *q, struct request *rq);
+void elv_rqhash_add(struct request_queue *q, struct request *rq);
+void elv_rqhash_reposition(struct request_queue *q, struct request *rq);
+struct request *elv_rqhash_find(struct request_queue *q, sector_t offset);
+
 /*
  * each queue has an elevator_queue associated with it
  */
@@ -139,7 +144,7 @@ extern struct request *elv_former_request(struct request_queue *, struct request
 extern struct request *elv_latter_request(struct request_queue *, struct request *);
 extern int elv_register_queue(struct request_queue *q);
 extern void elv_unregister_queue(struct request_queue *q);
-extern int elv_may_queue(struct request_queue *, int, int);
+extern int elv_may_queue(struct request_queue *, unsigned int);
 extern void elv_completed_request(struct request_queue *, struct request *);
 extern int elv_set_request(struct request_queue *q, struct request *rq,
 			   struct bio *bio, gfp_t gfp_mask);
diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h
index 422630b8e588..cea41a124a80 100644
--- a/include/linux/f2fs_fs.h
+++ b/include/linux/f2fs_fs.h
@@ -52,10 +52,17 @@
 
 #define VERSION_LEN	256
 #define MAX_VOLUME_NAME		512
+#define MAX_PATH_LEN		64
+#define MAX_DEVICES		8
 
 /*
  * For superblock
  */
+struct f2fs_device {
+	__u8 path[MAX_PATH_LEN];
+	__le32 total_segments;
+} __packed;
+
 struct f2fs_super_block {
 	__le32 magic;			/* Magic Number */
 	__le16 major_ver;		/* Major Version */
@@ -94,7 +101,8 @@ struct f2fs_super_block {
 	__le32 feature;			/* defined features */
 	__u8 encryption_level;		/* versioning level for encryption */
 	__u8 encrypt_pw_salt[16];	/* Salt used for string2key algorithm */
-	__u8 reserved[871];		/* valid reserved region */
+	struct f2fs_device devs[MAX_DEVICES];	/* device list */
+	__u8 reserved[327];		/* valid reserved region */
 } __packed;
 
 /*
diff --git a/include/linux/fence-array.h b/include/linux/fence-array.h
deleted file mode 100644
index a44794e508df..000000000000
--- a/include/linux/fence-array.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * fence-array: aggregates fence to be waited together
- *
- * Copyright (C) 2016 Collabora Ltd
- * Copyright (C) 2016 Advanced Micro Devices, Inc.
- * Authors:
- *	Gustavo Padovan <gustavo@padovan.org>
- *	Christian König <christian.koenig@amd.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __LINUX_FENCE_ARRAY_H
-#define __LINUX_FENCE_ARRAY_H
-
-#include <linux/fence.h>
-
-/**
- * struct fence_array_cb - callback helper for fence array
- * @cb: fence callback structure for signaling
- * @array: reference to the parent fence array object
- */
-struct fence_array_cb {
-	struct fence_cb cb;
-	struct fence_array *array;
-};
-
-/**
- * struct fence_array - fence to represent an array of fences
- * @base: fence base class
- * @lock: spinlock for fence handling
- * @num_fences: number of fences in the array
- * @num_pending: fences in the array still pending
- * @fences: array of the fences
- */
-struct fence_array {
-	struct fence base;
-
-	spinlock_t lock;
-	unsigned num_fences;
-	atomic_t num_pending;
-	struct fence **fences;
-};
-
-extern const struct fence_ops fence_array_ops;
-
-/**
- * fence_is_array - check if a fence is from the array subsclass
- *
- * Return true if it is a fence_array and false otherwise.
- */
-static inline bool fence_is_array(struct fence *fence)
-{
-	return fence->ops == &fence_array_ops;
-}
-
-/**
- * to_fence_array - cast a fence to a fence_array
- * @fence: fence to cast to a fence_array
- *
- * Returns NULL if the fence is not a fence_array,
- * or the fence_array otherwise.
- */
-static inline struct fence_array *to_fence_array(struct fence *fence)
-{
-	if (fence->ops != &fence_array_ops)
-		return NULL;
-
-	return container_of(fence, struct fence_array, base);
-}
-
-struct fence_array *fence_array_create(int num_fences, struct fence **fences,
-				       u64 context, unsigned seqno,
-				       bool signal_on_any);
-
-#endif /* __LINUX_FENCE_ARRAY_H */
diff --git a/include/linux/fence.h b/include/linux/fence.h
deleted file mode 100644
index 0d763053f97a..000000000000
--- a/include/linux/fence.h
+++ /dev/null
@@ -1,378 +0,0 @@
-/*
- * Fence mechanism for dma-buf to allow for asynchronous dma access
- *
- * Copyright (C) 2012 Canonical Ltd
- * Copyright (C) 2012 Texas Instruments
- *
- * Authors:
- * Rob Clark <robdclark@gmail.com>
- * Maarten Lankhorst <maarten.lankhorst@canonical.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __LINUX_FENCE_H
-#define __LINUX_FENCE_H
-
-#include <linux/err.h>
-#include <linux/wait.h>
-#include <linux/list.h>
-#include <linux/bitops.h>
-#include <linux/kref.h>
-#include <linux/sched.h>
-#include <linux/printk.h>
-#include <linux/rcupdate.h>
-
-struct fence;
-struct fence_ops;
-struct fence_cb;
-
-/**
- * struct fence - software synchronization primitive
- * @refcount: refcount for this fence
- * @ops: fence_ops associated with this fence
- * @rcu: used for releasing fence with kfree_rcu
- * @cb_list: list of all callbacks to call
- * @lock: spin_lock_irqsave used for locking
- * @context: execution context this fence belongs to, returned by
- *           fence_context_alloc()
- * @seqno: the sequence number of this fence inside the execution context,
- * can be compared to decide which fence would be signaled later.
- * @flags: A mask of FENCE_FLAG_* defined below
- * @timestamp: Timestamp when the fence was signaled.
- * @status: Optional, only valid if < 0, must be set before calling
- * fence_signal, indicates that the fence has completed with an error.
- *
- * the flags member must be manipulated and read using the appropriate
- * atomic ops (bit_*), so taking the spinlock will not be needed most
- * of the time.
- *
- * FENCE_FLAG_SIGNALED_BIT - fence is already signaled
- * FENCE_FLAG_ENABLE_SIGNAL_BIT - enable_signaling might have been called*
- * FENCE_FLAG_USER_BITS - start of the unused bits, can be used by the
- * implementer of the fence for its own purposes. Can be used in different
- * ways by different fence implementers, so do not rely on this.
- *
- * Since atomic bitops are used, this is not guaranteed to be the case.
- * Particularly, if the bit was set, but fence_signal was called right
- * before this bit was set, it would have been able to set the
- * FENCE_FLAG_SIGNALED_BIT, before enable_signaling was called.
- * Adding a check for FENCE_FLAG_SIGNALED_BIT after setting
- * FENCE_FLAG_ENABLE_SIGNAL_BIT closes this race, and makes sure that
- * after fence_signal was called, any enable_signaling call will have either
- * been completed, or never called at all.
- */
-struct fence {
-	struct kref refcount;
-	const struct fence_ops *ops;
-	struct rcu_head rcu;
-	struct list_head cb_list;
-	spinlock_t *lock;
-	u64 context;
-	unsigned seqno;
-	unsigned long flags;
-	ktime_t timestamp;
-	int status;
-};
-
-enum fence_flag_bits {
-	FENCE_FLAG_SIGNALED_BIT,
-	FENCE_FLAG_ENABLE_SIGNAL_BIT,
-	FENCE_FLAG_USER_BITS, /* must always be last member */
-};
-
-typedef void (*fence_func_t)(struct fence *fence, struct fence_cb *cb);
-
-/**
- * struct fence_cb - callback for fence_add_callback
- * @node: used by fence_add_callback to append this struct to fence::cb_list
- * @func: fence_func_t to call
- *
- * This struct will be initialized by fence_add_callback, additional
- * data can be passed along by embedding fence_cb in another struct.
- */
-struct fence_cb {
-	struct list_head node;
-	fence_func_t func;
-};
-
-/**
- * struct fence_ops - operations implemented for fence
- * @get_driver_name: returns the driver name.
- * @get_timeline_name: return the name of the context this fence belongs to.
- * @enable_signaling: enable software signaling of fence.
- * @signaled: [optional] peek whether the fence is signaled, can be null.
- * @wait: custom wait implementation, or fence_default_wait.
- * @release: [optional] called on destruction of fence, can be null
- * @fill_driver_data: [optional] callback to fill in free-form debug info
- * Returns amount of bytes filled, or -errno.
- * @fence_value_str: [optional] fills in the value of the fence as a string
- * @timeline_value_str: [optional] fills in the current value of the timeline
- * as a string
- *
- * Notes on enable_signaling:
- * For fence implementations that have the capability for hw->hw
- * signaling, they can implement this op to enable the necessary
- * irqs, or insert commands into cmdstream, etc.  This is called
- * in the first wait() or add_callback() path to let the fence
- * implementation know that there is another driver waiting on
- * the signal (ie. hw->sw case).
- *
- * This function can be called called from atomic context, but not
- * from irq context, so normal spinlocks can be used.
- *
- * A return value of false indicates the fence already passed,
- * or some failure occurred that made it impossible to enable
- * signaling. True indicates successful enabling.
- *
- * fence->status may be set in enable_signaling, but only when false is
- * returned.
- *
- * Calling fence_signal before enable_signaling is called allows
- * for a tiny race window in which enable_signaling is called during,
- * before, or after fence_signal. To fight this, it is recommended
- * that before enable_signaling returns true an extra reference is
- * taken on the fence, to be released when the fence is signaled.
- * This will mean fence_signal will still be called twice, but
- * the second time will be a noop since it was already signaled.
- *
- * Notes on signaled:
- * May set fence->status if returning true.
- *
- * Notes on wait:
- * Must not be NULL, set to fence_default_wait for default implementation.
- * the fence_default_wait implementation should work for any fence, as long
- * as enable_signaling works correctly.
- *
- * Must return -ERESTARTSYS if the wait is intr = true and the wait was
- * interrupted, and remaining jiffies if fence has signaled, or 0 if wait
- * timed out. Can also return other error values on custom implementations,
- * which should be treated as if the fence is signaled. For example a hardware
- * lockup could be reported like that.
- *
- * Notes on release:
- * Can be NULL, this function allows additional commands to run on
- * destruction of the fence. Can be called from irq context.
- * If pointer is set to NULL, kfree will get called instead.
- */
-
-struct fence_ops {
-	const char * (*get_driver_name)(struct fence *fence);
-	const char * (*get_timeline_name)(struct fence *fence);
-	bool (*enable_signaling)(struct fence *fence);
-	bool (*signaled)(struct fence *fence);
-	signed long (*wait)(struct fence *fence, bool intr, signed long timeout);
-	void (*release)(struct fence *fence);
-
-	int (*fill_driver_data)(struct fence *fence, void *data, int size);
-	void (*fence_value_str)(struct fence *fence, char *str, int size);
-	void (*timeline_value_str)(struct fence *fence, char *str, int size);
-};
-
-void fence_init(struct fence *fence, const struct fence_ops *ops,
-		spinlock_t *lock, u64 context, unsigned seqno);
-
-void fence_release(struct kref *kref);
-void fence_free(struct fence *fence);
-
-/**
- * fence_get - increases refcount of the fence
- * @fence:	[in]	fence to increase refcount of
- *
- * Returns the same fence, with refcount increased by 1.
- */
-static inline struct fence *fence_get(struct fence *fence)
-{
-	if (fence)
-		kref_get(&fence->refcount);
-	return fence;
-}
-
-/**
- * fence_get_rcu - get a fence from a reservation_object_list with rcu read lock
- * @fence:	[in]	fence to increase refcount of
- *
- * Function returns NULL if no refcount could be obtained, or the fence.
- */
-static inline struct fence *fence_get_rcu(struct fence *fence)
-{
-	if (kref_get_unless_zero(&fence->refcount))
-		return fence;
-	else
-		return NULL;
-}
-
-/**
- * fence_put - decreases refcount of the fence
- * @fence:	[in]	fence to reduce refcount of
- */
-static inline void fence_put(struct fence *fence)
-{
-	if (fence)
-		kref_put(&fence->refcount, fence_release);
-}
-
-int fence_signal(struct fence *fence);
-int fence_signal_locked(struct fence *fence);
-signed long fence_default_wait(struct fence *fence, bool intr, signed long timeout);
-int fence_add_callback(struct fence *fence, struct fence_cb *cb,
-		       fence_func_t func);
-bool fence_remove_callback(struct fence *fence, struct fence_cb *cb);
-void fence_enable_sw_signaling(struct fence *fence);
-
-/**
- * fence_is_signaled_locked - Return an indication if the fence is signaled yet.
- * @fence:	[in]	the fence to check
- *
- * Returns true if the fence was already signaled, false if not. Since this
- * function doesn't enable signaling, it is not guaranteed to ever return
- * true if fence_add_callback, fence_wait or fence_enable_sw_signaling
- * haven't been called before.
- *
- * This function requires fence->lock to be held.
- */
-static inline bool
-fence_is_signaled_locked(struct fence *fence)
-{
-	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
-		return true;
-
-	if (fence->ops->signaled && fence->ops->signaled(fence)) {
-		fence_signal_locked(fence);
-		return true;
-	}
-
-	return false;
-}
-
-/**
- * fence_is_signaled - Return an indication if the fence is signaled yet.
- * @fence:	[in]	the fence to check
- *
- * Returns true if the fence was already signaled, false if not. Since this
- * function doesn't enable signaling, it is not guaranteed to ever return
- * true if fence_add_callback, fence_wait or fence_enable_sw_signaling
- * haven't been called before.
- *
- * It's recommended for seqno fences to call fence_signal when the
- * operation is complete, it makes it possible to prevent issues from
- * wraparound between time of issue and time of use by checking the return
- * value of this function before calling hardware-specific wait instructions.
- */
-static inline bool
-fence_is_signaled(struct fence *fence)
-{
-	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
-		return true;
-
-	if (fence->ops->signaled && fence->ops->signaled(fence)) {
-		fence_signal(fence);
-		return true;
-	}
-
-	return false;
-}
-
-/**
- * fence_is_later - return if f1 is chronologically later than f2
- * @f1:	[in]	the first fence from the same context
- * @f2:	[in]	the second fence from the same context
- *
- * Returns true if f1 is chronologically later than f2. Both fences must be
- * from the same context, since a seqno is not re-used across contexts.
- */
-static inline bool fence_is_later(struct fence *f1, struct fence *f2)
-{
-	if (WARN_ON(f1->context != f2->context))
-		return false;
-
-	return (int)(f1->seqno - f2->seqno) > 0;
-}
-
-/**
- * fence_later - return the chronologically later fence
- * @f1:	[in]	the first fence from the same context
- * @f2:	[in]	the second fence from the same context
- *
- * Returns NULL if both fences are signaled, otherwise the fence that would be
- * signaled last. Both fences must be from the same context, since a seqno is
- * not re-used across contexts.
- */
-static inline struct fence *fence_later(struct fence *f1, struct fence *f2)
-{
-	if (WARN_ON(f1->context != f2->context))
-		return NULL;
-
-	/*
-	 * can't check just FENCE_FLAG_SIGNALED_BIT here, it may never have been
-	 * set if enable_signaling wasn't called, and enabling that here is
-	 * overkill.
-	 */
-	if (fence_is_later(f1, f2))
-		return fence_is_signaled(f1) ? NULL : f1;
-	else
-		return fence_is_signaled(f2) ? NULL : f2;
-}
-
-signed long fence_wait_timeout(struct fence *, bool intr, signed long timeout);
-signed long fence_wait_any_timeout(struct fence **fences, uint32_t count,
-				   bool intr, signed long timeout);
-
-/**
- * fence_wait - sleep until the fence gets signaled
- * @fence:	[in]	the fence to wait on
- * @intr:	[in]	if true, do an interruptible wait
- *
- * This function will return -ERESTARTSYS if interrupted by a signal,
- * or 0 if the fence was signaled. Other error values may be
- * returned on custom implementations.
- *
- * Performs a synchronous wait on this fence. It is assumed the caller
- * directly or indirectly holds a reference to the fence, otherwise the
- * fence might be freed before return, resulting in undefined behavior.
- */
-static inline signed long fence_wait(struct fence *fence, bool intr)
-{
-	signed long ret;
-
-	/* Since fence_wait_timeout cannot timeout with
-	 * MAX_SCHEDULE_TIMEOUT, only valid return values are
-	 * -ERESTARTSYS and MAX_SCHEDULE_TIMEOUT.
-	 */
-	ret = fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT);
-
-	return ret < 0 ? ret : 0;
-}
-
-u64 fence_context_alloc(unsigned num);
-
-#define FENCE_TRACE(f, fmt, args...) \
-	do {								\
-		struct fence *__ff = (f);				\
-		if (IS_ENABLED(CONFIG_FENCE_TRACE))			\
-			pr_info("f %llu#%u: " fmt,			\
-				__ff->context, __ff->seqno, ##args);	\
-	} while (0)
-
-#define FENCE_WARN(f, fmt, args...) \
-	do {								\
-		struct fence *__ff = (f);				\
-		pr_warn("f %llu#%u: " fmt, __ff->context, __ff->seqno,	\
-			 ##args);					\
-	} while (0)
-
-#define FENCE_ERR(f, fmt, args...) \
-	do {								\
-		struct fence *__ff = (f);				\
-		pr_err("f %llu#%u: " fmt, __ff->context, __ff->seqno,	\
-			##args);					\
-	} while (0)
-
-#endif /* __LINUX_FENCE_H */
diff --git a/include/linux/fpga/fpga-bridge.h b/include/linux/fpga/fpga-bridge.h
new file mode 100644
index 000000000000..dba6e3c697c7
--- /dev/null
+++ b/include/linux/fpga/fpga-bridge.h
@@ -0,0 +1,60 @@
+#include <linux/device.h>
+#include <linux/fpga/fpga-mgr.h>
+
+#ifndef _LINUX_FPGA_BRIDGE_H
+#define _LINUX_FPGA_BRIDGE_H
+
+struct fpga_bridge;
+
+/**
+ * struct fpga_bridge_ops - ops for low level FPGA bridge drivers
+ * @enable_show: returns the FPGA bridge's status
+ * @enable_set: set a FPGA bridge as enabled or disabled
+ * @fpga_bridge_remove: set FPGA into a specific state during driver remove
+ */
+struct fpga_bridge_ops {
+	int (*enable_show)(struct fpga_bridge *bridge);
+	int (*enable_set)(struct fpga_bridge *bridge, bool enable);
+	void (*fpga_bridge_remove)(struct fpga_bridge *bridge);
+};
+
+/**
+ * struct fpga_bridge - FPGA bridge structure
+ * @name: name of low level FPGA bridge
+ * @dev: FPGA bridge device
+ * @mutex: enforces exclusive reference to bridge
+ * @br_ops: pointer to struct of FPGA bridge ops
+ * @info: fpga image specific information
+ * @node: FPGA bridge list node
+ * @priv: low level driver private date
+ */
+struct fpga_bridge {
+	const char *name;
+	struct device dev;
+	struct mutex mutex; /* for exclusive reference to bridge */
+	const struct fpga_bridge_ops *br_ops;
+	struct fpga_image_info *info;
+	struct list_head node;
+	void *priv;
+};
+
+#define to_fpga_bridge(d) container_of(d, struct fpga_bridge, dev)
+
+struct fpga_bridge *of_fpga_bridge_get(struct device_node *node,
+				       struct fpga_image_info *info);
+void fpga_bridge_put(struct fpga_bridge *bridge);
+int fpga_bridge_enable(struct fpga_bridge *bridge);
+int fpga_bridge_disable(struct fpga_bridge *bridge);
+
+int fpga_bridges_enable(struct list_head *bridge_list);
+int fpga_bridges_disable(struct list_head *bridge_list);
+void fpga_bridges_put(struct list_head *bridge_list);
+int fpga_bridge_get_to_list(struct device_node *np,
+			    struct fpga_image_info *info,
+			    struct list_head *bridge_list);
+
+int fpga_bridge_register(struct device *dev, const char *name,
+			 const struct fpga_bridge_ops *br_ops, void *priv);
+void fpga_bridge_unregister(struct device *dev);
+
+#endif /* _LINUX_FPGA_BRIDGE_H */
diff --git a/include/linux/fpga/fpga-mgr.h b/include/linux/fpga/fpga-mgr.h
index 0940bf45e2f2..16551d5eac36 100644
--- a/include/linux/fpga/fpga-mgr.h
+++ b/include/linux/fpga/fpga-mgr.h
@@ -65,11 +65,26 @@ enum fpga_mgr_states {
 /*
  * FPGA Manager flags
  * FPGA_MGR_PARTIAL_RECONFIG: do partial reconfiguration if supported
+ * FPGA_MGR_EXTERNAL_CONFIG: FPGA has been configured prior to Linux booting
  */
 #define FPGA_MGR_PARTIAL_RECONFIG	BIT(0)
+#define FPGA_MGR_EXTERNAL_CONFIG	BIT(1)
+
+/**
+ * struct fpga_image_info - information specific to a FPGA image
+ * @flags: boolean flags as defined above
+ * @enable_timeout_us: maximum time to enable traffic through bridge (uSec)
+ * @disable_timeout_us: maximum time to disable traffic through bridge (uSec)
+ */
+struct fpga_image_info {
+	u32 flags;
+	u32 enable_timeout_us;
+	u32 disable_timeout_us;
+};
 
 /**
  * struct fpga_manager_ops - ops for low level fpga manager drivers
+ * @initial_header_size: Maximum number of bytes that should be passed into write_init
  * @state: returns an enum value of the FPGA's state
  * @write_init: prepare the FPGA to receive confuration data
  * @write: write count bytes of configuration data to the FPGA
@@ -81,11 +96,14 @@ enum fpga_mgr_states {
  * called, so leaving them out is fine.
  */
 struct fpga_manager_ops {
+	size_t initial_header_size;
 	enum fpga_mgr_states (*state)(struct fpga_manager *mgr);
-	int (*write_init)(struct fpga_manager *mgr, u32 flags,
+	int (*write_init)(struct fpga_manager *mgr,
+			  struct fpga_image_info *info,
 			  const char *buf, size_t count);
 	int (*write)(struct fpga_manager *mgr, const char *buf, size_t count);
-	int (*write_complete)(struct fpga_manager *mgr, u32 flags);
+	int (*write_complete)(struct fpga_manager *mgr,
+			      struct fpga_image_info *info);
 	void (*fpga_remove)(struct fpga_manager *mgr);
 };
 
@@ -109,14 +127,17 @@ struct fpga_manager {
 
 #define to_fpga_manager(d) container_of(d, struct fpga_manager, dev)
 
-int fpga_mgr_buf_load(struct fpga_manager *mgr, u32 flags,
+int fpga_mgr_buf_load(struct fpga_manager *mgr, struct fpga_image_info *info,
 		      const char *buf, size_t count);
 
-int fpga_mgr_firmware_load(struct fpga_manager *mgr, u32 flags,
+int fpga_mgr_firmware_load(struct fpga_manager *mgr,
+			   struct fpga_image_info *info,
 			   const char *image_name);
 
 struct fpga_manager *of_fpga_mgr_get(struct device_node *node);
 
+struct fpga_manager *fpga_mgr_get(struct device *dev);
+
 void fpga_mgr_put(struct fpga_manager *mgr);
 
 int fpga_mgr_register(struct device *dev, const char *name,
diff --git a/include/linux/fs.h b/include/linux/fs.h
index dc0478c07b2a..b84230e070be 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -28,7 +28,6 @@
 #include <linux/uidgid.h>
 #include <linux/lockdep.h>
 #include <linux/percpu-rwsem.h>
-#include <linux/blk_types.h>
 #include <linux/workqueue.h>
 #include <linux/percpu-rwsem.h>
 #include <linux/delayed_call.h>
@@ -38,6 +37,7 @@
 
 struct backing_dev_info;
 struct bdi_writeback;
+struct bio;
 struct export_operations;
 struct hd_geometry;
 struct iovec;
@@ -152,58 +152,6 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset,
 #define CHECK_IOVEC_ONLY -1
 
 /*
- * The below are the various read and write flags that we support. Some of
- * them include behavioral modifiers that send information down to the
- * block layer and IO scheduler. They should be used along with a req_op.
- * Terminology:
- *
- *	The block layer uses device plugging to defer IO a little bit, in
- *	the hope that we will see more IO very shortly. This increases
- *	coalescing of adjacent IO and thus reduces the number of IOs we
- *	have to send to the device. It also allows for better queuing,
- *	if the IO isn't mergeable. If the caller is going to be waiting
- *	for the IO, then he must ensure that the device is unplugged so
- *	that the IO is dispatched to the driver.
- *
- *	All IO is handled async in Linux. This is fine for background
- *	writes, but for reads or writes that someone waits for completion
- *	on, we want to notify the block layer and IO scheduler so that they
- *	know about it. That allows them to make better scheduling
- *	decisions. So when the below references 'sync' and 'async', it
- *	is referencing this priority hint.
- *
- * With that in mind, the available types are:
- *
- * READ			A normal read operation. Device will be plugged.
- * READ_SYNC		A synchronous read. Device is not plugged, caller can
- *			immediately wait on this read without caring about
- *			unplugging.
- * WRITE		A normal async write. Device will be plugged.
- * WRITE_SYNC		Synchronous write. Identical to WRITE, but passes down
- *			the hint that someone will be waiting on this IO
- *			shortly. The write equivalent of READ_SYNC.
- * WRITE_ODIRECT	Special case write for O_DIRECT only.
- * WRITE_FLUSH		Like WRITE_SYNC but with preceding cache flush.
- * WRITE_FUA		Like WRITE_SYNC but data is guaranteed to be on
- *			non-volatile media on completion.
- * WRITE_FLUSH_FUA	Combination of WRITE_FLUSH and FUA. The IO is preceded
- *			by a cache flush and data is guaranteed to be on
- *			non-volatile media on completion.
- *
- */
-#define RW_MASK			REQ_OP_WRITE
-
-#define READ			REQ_OP_READ
-#define WRITE			REQ_OP_WRITE
-
-#define READ_SYNC		REQ_SYNC
-#define WRITE_SYNC		(REQ_SYNC | REQ_NOIDLE)
-#define WRITE_ODIRECT		REQ_SYNC
-#define WRITE_FLUSH		(REQ_SYNC | REQ_NOIDLE | REQ_PREFLUSH)
-#define WRITE_FUA		(REQ_SYNC | REQ_NOIDLE | REQ_FUA)
-#define WRITE_FLUSH_FUA		(REQ_SYNC | REQ_NOIDLE | REQ_PREFLUSH | REQ_FUA)
-
-/*
  * Attribute flags.  These should be or-ed together to figure out what
  * has been changed!
  */
@@ -2499,19 +2447,6 @@ extern void make_bad_inode(struct inode *);
 extern bool is_bad_inode(struct inode *);
 
 #ifdef CONFIG_BLOCK
-static inline bool op_is_write(unsigned int op)
-{
-	return op == REQ_OP_READ ? false : true;
-}
-
-/*
- * return data direction, READ or WRITE
- */
-static inline int bio_data_dir(struct bio *bio)
-{
-	return op_is_write(bio_op(bio)) ? WRITE : READ;
-}
-
 extern void check_disk_size_change(struct gendisk *disk,
 				   struct block_device *bdev);
 extern int revalidate_disk(struct gendisk *);
@@ -2782,7 +2717,6 @@ static inline void remove_inode_hash(struct inode *inode)
 extern void inode_sb_list_add(struct inode *inode);
 
 #ifdef CONFIG_BLOCK
-extern blk_qc_t submit_bio(struct bio *);
 extern int bdev_read_only(struct block_device *);
 #endif
 extern int set_blocksize(struct block_device *, int);
diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h
index ff8b11b26f31..c074b670aa99 100644
--- a/include/linux/fscrypto.h
+++ b/include/linux/fscrypto.h
@@ -18,73 +18,9 @@
 #include <crypto/skcipher.h>
 #include <uapi/linux/fs.h>
 
-#define FS_KEY_DERIVATION_NONCE_SIZE		16
-#define FS_ENCRYPTION_CONTEXT_FORMAT_V1		1
-
-#define FS_POLICY_FLAGS_PAD_4		0x00
-#define FS_POLICY_FLAGS_PAD_8		0x01
-#define FS_POLICY_FLAGS_PAD_16		0x02
-#define FS_POLICY_FLAGS_PAD_32		0x03
-#define FS_POLICY_FLAGS_PAD_MASK	0x03
-#define FS_POLICY_FLAGS_VALID		0x03
-
-/* Encryption algorithms */
-#define FS_ENCRYPTION_MODE_INVALID		0
-#define FS_ENCRYPTION_MODE_AES_256_XTS		1
-#define FS_ENCRYPTION_MODE_AES_256_GCM		2
-#define FS_ENCRYPTION_MODE_AES_256_CBC		3
-#define FS_ENCRYPTION_MODE_AES_256_CTS		4
-
-/**
- * Encryption context for inode
- *
- * Protector format:
- *  1 byte: Protector format (1 = this version)
- *  1 byte: File contents encryption mode
- *  1 byte: File names encryption mode
- *  1 byte: Flags
- *  8 bytes: Master Key descriptor
- *  16 bytes: Encryption Key derivation nonce
- */
-struct fscrypt_context {
-	u8 format;
-	u8 contents_encryption_mode;
-	u8 filenames_encryption_mode;
-	u8 flags;
-	u8 master_key_descriptor[FS_KEY_DESCRIPTOR_SIZE];
-	u8 nonce[FS_KEY_DERIVATION_NONCE_SIZE];
-} __packed;
-
-/* Encryption parameters */
-#define FS_XTS_TWEAK_SIZE		16
-#define FS_AES_128_ECB_KEY_SIZE		16
-#define FS_AES_256_GCM_KEY_SIZE		32
-#define FS_AES_256_CBC_KEY_SIZE		32
-#define FS_AES_256_CTS_KEY_SIZE		32
-#define FS_AES_256_XTS_KEY_SIZE		64
-#define FS_MAX_KEY_SIZE			64
-
-#define FS_KEY_DESC_PREFIX		"fscrypt:"
-#define FS_KEY_DESC_PREFIX_SIZE		8
-
-/* This is passed in from userspace into the kernel keyring */
-struct fscrypt_key {
-	u32 mode;
-	u8 raw[FS_MAX_KEY_SIZE];
-	u32 size;
-} __packed;
-
-struct fscrypt_info {
-	u8 ci_data_mode;
-	u8 ci_filename_mode;
-	u8 ci_flags;
-	struct crypto_skcipher *ci_ctfm;
-	struct key *ci_keyring_key;
-	u8 ci_master_key[FS_KEY_DESCRIPTOR_SIZE];
-};
+#define FS_CRYPTO_BLOCK_SIZE		16
 
-#define FS_CTX_REQUIRES_FREE_ENCRYPT_FL		0x00000001
-#define FS_WRITE_PATH_FL			0x00000002
+struct fscrypt_info;
 
 struct fscrypt_ctx {
 	union {
@@ -102,19 +38,6 @@ struct fscrypt_ctx {
 	u8 mode;				/* Encryption mode for tfm */
 };
 
-struct fscrypt_completion_result {
-	struct completion completion;
-	int res;
-};
-
-#define DECLARE_FS_COMPLETION_RESULT(ecr) \
-	struct fscrypt_completion_result ecr = { \
-		COMPLETION_INITIALIZER((ecr).completion), 0 }
-
-#define FS_FNAME_NUM_SCATTER_ENTRIES	4
-#define FS_CRYPTO_BLOCK_SIZE		16
-#define FS_FNAME_CRYPTO_DIGEST_SIZE	32
-
 /**
  * For encrypted symlinks, the ciphertext length is stored at the beginning
  * of the string in little-endian format.
@@ -154,9 +77,15 @@ struct fscrypt_name {
 #define fname_len(p)		((p)->disk_name.len)
 
 /*
+ * fscrypt superblock flags
+ */
+#define FS_CFLG_OWN_PAGES (1U << 1)
+
+/*
  * crypto opertions for filesystems
  */
 struct fscrypt_operations {
+	unsigned int flags;
 	int (*get_context)(struct inode *, void *, size_t);
 	int (*key_prefix)(struct inode *, u8 **);
 	int (*prepare_context)(struct inode *);
@@ -206,7 +135,7 @@ static inline struct page *fscrypt_control_page(struct page *page)
 #endif
 }
 
-static inline int fscrypt_has_encryption_key(struct inode *inode)
+static inline int fscrypt_has_encryption_key(const struct inode *inode)
 {
 #if IS_ENABLED(CONFIG_FS_ENCRYPTION)
 	return (inode->i_crypt_info != NULL);
@@ -238,25 +167,25 @@ static inline void fscrypt_set_d_op(struct dentry *dentry)
 #if IS_ENABLED(CONFIG_FS_ENCRYPTION)
 /* crypto.c */
 extern struct kmem_cache *fscrypt_info_cachep;
-int fscrypt_initialize(void);
-
-extern struct fscrypt_ctx *fscrypt_get_ctx(struct inode *, gfp_t);
+extern struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *, gfp_t);
 extern void fscrypt_release_ctx(struct fscrypt_ctx *);
-extern struct page *fscrypt_encrypt_page(struct inode *, struct page *, gfp_t);
-extern int fscrypt_decrypt_page(struct page *);
+extern struct page *fscrypt_encrypt_page(const struct inode *, struct page *,
+						unsigned int, unsigned int,
+						u64, gfp_t);
+extern int fscrypt_decrypt_page(const struct inode *, struct page *, unsigned int,
+				unsigned int, u64);
 extern void fscrypt_decrypt_bio_pages(struct fscrypt_ctx *, struct bio *);
 extern void fscrypt_pullback_bio_page(struct page **, bool);
 extern void fscrypt_restore_control_page(struct page *);
-extern int fscrypt_zeroout_range(struct inode *, pgoff_t, sector_t,
+extern int fscrypt_zeroout_range(const struct inode *, pgoff_t, sector_t,
 						unsigned int);
 /* policy.c */
-extern int fscrypt_process_policy(struct file *, const struct fscrypt_policy *);
-extern int fscrypt_get_policy(struct inode *, struct fscrypt_policy *);
+extern int fscrypt_ioctl_set_policy(struct file *, const void __user *);
+extern int fscrypt_ioctl_get_policy(struct file *, void __user *);
 extern int fscrypt_has_permitted_context(struct inode *, struct inode *);
 extern int fscrypt_inherit_context(struct inode *, struct inode *,
 					void *, bool);
 /* keyinfo.c */
-extern int get_crypt_info(struct inode *);
 extern int fscrypt_get_encryption_info(struct inode *);
 extern void fscrypt_put_encryption_info(struct inode *, struct fscrypt_info *);
 
@@ -264,8 +193,8 @@ extern void fscrypt_put_encryption_info(struct inode *, struct fscrypt_info *);
 extern int fscrypt_setup_filename(struct inode *, const struct qstr *,
 				int lookup, struct fscrypt_name *);
 extern void fscrypt_free_filename(struct fscrypt_name *);
-extern u32 fscrypt_fname_encrypted_size(struct inode *, u32);
-extern int fscrypt_fname_alloc_buffer(struct inode *, u32,
+extern u32 fscrypt_fname_encrypted_size(const struct inode *, u32);
+extern int fscrypt_fname_alloc_buffer(const struct inode *, u32,
 				struct fscrypt_str *);
 extern void fscrypt_fname_free_buffer(struct fscrypt_str *);
 extern int fscrypt_fname_disk_to_usr(struct inode *, u32, u32,
@@ -275,7 +204,7 @@ extern int fscrypt_fname_usr_to_disk(struct inode *, const struct qstr *,
 #endif
 
 /* crypto.c */
-static inline struct fscrypt_ctx *fscrypt_notsupp_get_ctx(struct inode *i,
+static inline struct fscrypt_ctx *fscrypt_notsupp_get_ctx(const struct inode *i,
 							gfp_t f)
 {
 	return ERR_PTR(-EOPNOTSUPP);
@@ -286,13 +215,18 @@ static inline void fscrypt_notsupp_release_ctx(struct fscrypt_ctx *c)
 	return;
 }
 
-static inline struct page *fscrypt_notsupp_encrypt_page(struct inode *i,
-						struct page *p, gfp_t f)
+static inline struct page *fscrypt_notsupp_encrypt_page(const struct inode *i,
+						struct page *p,
+						unsigned int len,
+						unsigned int offs,
+						u64 lblk_num, gfp_t f)
 {
 	return ERR_PTR(-EOPNOTSUPP);
 }
 
-static inline int fscrypt_notsupp_decrypt_page(struct page *p)
+static inline int fscrypt_notsupp_decrypt_page(const struct inode *i, struct page *p,
+						unsigned int len, unsigned int offs,
+						u64 lblk_num)
 {
 	return -EOPNOTSUPP;
 }
@@ -313,21 +247,21 @@ static inline void fscrypt_notsupp_restore_control_page(struct page *p)
 	return;
 }
 
-static inline int fscrypt_notsupp_zeroout_range(struct inode *i, pgoff_t p,
+static inline int fscrypt_notsupp_zeroout_range(const struct inode *i, pgoff_t p,
 					sector_t s, unsigned int f)
 {
 	return -EOPNOTSUPP;
 }
 
 /* policy.c */
-static inline int fscrypt_notsupp_process_policy(struct file *f,
-				const struct fscrypt_policy *p)
+static inline int fscrypt_notsupp_ioctl_set_policy(struct file *f,
+				const void __user *arg)
 {
 	return -EOPNOTSUPP;
 }
 
-static inline int fscrypt_notsupp_get_policy(struct inode *i,
-				struct fscrypt_policy *p)
+static inline int fscrypt_notsupp_ioctl_get_policy(struct file *f,
+				void __user *arg)
 {
 	return -EOPNOTSUPP;
 }
diff --git a/include/linux/fsl/guts.h b/include/linux/fsl/guts.h
index 649e9171a9b3..3efa3b861d44 100644
--- a/include/linux/fsl/guts.h
+++ b/include/linux/fsl/guts.h
@@ -29,83 +29,112 @@
  * #ifdefs.
  */
 struct ccsr_guts {
-	__be32	porpllsr;	/* 0x.0000 - POR PLL Ratio Status Register */
-	__be32	porbmsr;	/* 0x.0004 - POR Boot Mode Status Register */
-	__be32	porimpscr;	/* 0x.0008 - POR I/O Impedance Status and Control Register */
-	__be32	pordevsr;	/* 0x.000c - POR I/O Device Status Register */
-	__be32	pordbgmsr;	/* 0x.0010 - POR Debug Mode Status Register */
-	__be32	pordevsr2;	/* 0x.0014 - POR device status register 2 */
+	u32	porpllsr;	/* 0x.0000 - POR PLL Ratio Status Register */
+	u32	porbmsr;	/* 0x.0004 - POR Boot Mode Status Register */
+	u32	porimpscr;	/* 0x.0008 - POR I/O Impedance Status and
+				 *           Control Register
+				 */
+	u32	pordevsr;	/* 0x.000c - POR I/O Device Status Register */
+	u32	pordbgmsr;	/* 0x.0010 - POR Debug Mode Status Register */
+	u32	pordevsr2;	/* 0x.0014 - POR device status register 2 */
 	u8	res018[0x20 - 0x18];
-	__be32	porcir;		/* 0x.0020 - POR Configuration Information Register */
+	u32	porcir;		/* 0x.0020 - POR Configuration Information
+				 *           Register
+				 */
 	u8	res024[0x30 - 0x24];
-	__be32	gpiocr;		/* 0x.0030 - GPIO Control Register */
+	u32	gpiocr;		/* 0x.0030 - GPIO Control Register */
 	u8	res034[0x40 - 0x34];
-	__be32	gpoutdr;	/* 0x.0040 - General-Purpose Output Data Register */
+	u32	gpoutdr;	/* 0x.0040 - General-Purpose Output Data
+				 *           Register
+				 */
 	u8	res044[0x50 - 0x44];
-	__be32	gpindr;		/* 0x.0050 - General-Purpose Input Data Register */
+	u32	gpindr;		/* 0x.0050 - General-Purpose Input Data
+				 *           Register
+				 */
 	u8	res054[0x60 - 0x54];
-	__be32	pmuxcr;		/* 0x.0060 - Alternate Function Signal Multiplex Control */
-        __be32  pmuxcr2;	/* 0x.0064 - Alternate function signal multiplex control 2 */
-        __be32  dmuxcr;		/* 0x.0068 - DMA Mux Control Register */
+	u32	pmuxcr;		/* 0x.0060 - Alternate Function Signal
+				 *           Multiplex Control
+				 */
+	u32	pmuxcr2;	/* 0x.0064 - Alternate function signal
+				 *           multiplex control 2
+				 */
+	u32	dmuxcr;		/* 0x.0068 - DMA Mux Control Register */
         u8	res06c[0x70 - 0x6c];
-	__be32	devdisr;	/* 0x.0070 - Device Disable Control */
+	u32	devdisr;	/* 0x.0070 - Device Disable Control */
 #define CCSR_GUTS_DEVDISR_TB1	0x00001000
 #define CCSR_GUTS_DEVDISR_TB0	0x00004000
-	__be32	devdisr2;	/* 0x.0074 - Device Disable Control 2 */
+	u32	devdisr2;	/* 0x.0074 - Device Disable Control 2 */
 	u8	res078[0x7c - 0x78];
-	__be32  pmjcr;		/* 0x.007c - 4 Power Management Jog Control Register */
-	__be32	powmgtcsr;	/* 0x.0080 - Power Management Status and Control Register */
-	__be32  pmrccr;		/* 0x.0084 - Power Management Reset Counter Configuration Register */
-	__be32  pmpdccr;	/* 0x.0088 - Power Management Power Down Counter Configuration Register */
-	__be32  pmcdr;		/* 0x.008c - 4Power management clock disable register */
-	__be32	mcpsumr;	/* 0x.0090 - Machine Check Summary Register */
-	__be32	rstrscr;	/* 0x.0094 - Reset Request Status and Control Register */
-	__be32  ectrstcr;	/* 0x.0098 - Exception reset control register */
-	__be32  autorstsr;	/* 0x.009c - Automatic reset status register */
-	__be32	pvr;		/* 0x.00a0 - Processor Version Register */
-	__be32	svr;		/* 0x.00a4 - System Version Register */
+	u32	pmjcr;		/* 0x.007c - 4 Power Management Jog Control
+				 *           Register
+				 */
+	u32	powmgtcsr;	/* 0x.0080 - Power Management Status and
+				 *           Control Register
+				 */
+	u32	pmrccr;		/* 0x.0084 - Power Management Reset Counter
+				 *           Configuration Register
+				 */
+	u32	pmpdccr;	/* 0x.0088 - Power Management Power Down Counter
+				 *           Configuration Register
+				 */
+	u32	pmcdr;		/* 0x.008c - 4Power management clock disable
+				 *           register
+				 */
+	u32	mcpsumr;	/* 0x.0090 - Machine Check Summary Register */
+	u32	rstrscr;	/* 0x.0094 - Reset Request Status and
+				 *           Control Register
+				 */
+	u32	ectrstcr;	/* 0x.0098 - Exception reset control register */
+	u32	autorstsr;	/* 0x.009c - Automatic reset status register */
+	u32	pvr;		/* 0x.00a0 - Processor Version Register */
+	u32	svr;		/* 0x.00a4 - System Version Register */
 	u8	res0a8[0xb0 - 0xa8];
-	__be32	rstcr;		/* 0x.00b0 - Reset Control Register */
+	u32	rstcr;		/* 0x.00b0 - Reset Control Register */
 	u8	res0b4[0xc0 - 0xb4];
-	__be32  iovselsr;	/* 0x.00c0 - I/O voltage select status register
+	u32	iovselsr;	/* 0x.00c0 - I/O voltage select status register
 					     Called 'elbcvselcr' on 86xx SOCs */
 	u8	res0c4[0x100 - 0xc4];
-	__be32	rcwsr[16];	/* 0x.0100 - Reset Control Word Status registers
+	u32	rcwsr[16];	/* 0x.0100 - Reset Control Word Status registers
 					     There are 16 registers */
 	u8	res140[0x224 - 0x140];
-	__be32  iodelay1;	/* 0x.0224 - IO delay control register 1 */
-	__be32  iodelay2;	/* 0x.0228 - IO delay control register 2 */
+	u32	iodelay1;	/* 0x.0224 - IO delay control register 1 */
+	u32	iodelay2;	/* 0x.0228 - IO delay control register 2 */
 	u8	res22c[0x604 - 0x22c];
-	__be32	pamubypenr; 	/* 0x.604 - PAMU bypass enable register */
+	u32	pamubypenr;	/* 0x.604 - PAMU bypass enable register */
 	u8	res608[0x800 - 0x608];
-	__be32	clkdvdr;	/* 0x.0800 - Clock Divide Register */
+	u32	clkdvdr;	/* 0x.0800 - Clock Divide Register */
 	u8	res804[0x900 - 0x804];
-	__be32	ircr;		/* 0x.0900 - Infrared Control Register */
+	u32	ircr;		/* 0x.0900 - Infrared Control Register */
 	u8	res904[0x908 - 0x904];
-	__be32	dmacr;		/* 0x.0908 - DMA Control Register */
+	u32	dmacr;		/* 0x.0908 - DMA Control Register */
 	u8	res90c[0x914 - 0x90c];
-	__be32	elbccr;		/* 0x.0914 - eLBC Control Register */
+	u32	elbccr;		/* 0x.0914 - eLBC Control Register */
 	u8	res918[0xb20 - 0x918];
-	__be32	ddr1clkdr;	/* 0x.0b20 - DDR1 Clock Disable Register */
-	__be32	ddr2clkdr;	/* 0x.0b24 - DDR2 Clock Disable Register */
-	__be32	ddrclkdr;	/* 0x.0b28 - DDR Clock Disable Register */
+	u32	ddr1clkdr;	/* 0x.0b20 - DDR1 Clock Disable Register */
+	u32	ddr2clkdr;	/* 0x.0b24 - DDR2 Clock Disable Register */
+	u32	ddrclkdr;	/* 0x.0b28 - DDR Clock Disable Register */
 	u8	resb2c[0xe00 - 0xb2c];
-	__be32	clkocr;		/* 0x.0e00 - Clock Out Select Register */
+	u32	clkocr;		/* 0x.0e00 - Clock Out Select Register */
 	u8	rese04[0xe10 - 0xe04];
-	__be32	ddrdllcr;	/* 0x.0e10 - DDR DLL Control Register */
+	u32	ddrdllcr;	/* 0x.0e10 - DDR DLL Control Register */
 	u8	rese14[0xe20 - 0xe14];
-	__be32	lbcdllcr;	/* 0x.0e20 - LBC DLL Control Register */
-	__be32  cpfor;		/* 0x.0e24 - L2 charge pump fuse override register */
+	u32	lbcdllcr;	/* 0x.0e20 - LBC DLL Control Register */
+	u32	cpfor;		/* 0x.0e24 - L2 charge pump fuse override
+				 *           register
+				 */
 	u8	rese28[0xf04 - 0xe28];
-	__be32	srds1cr0;	/* 0x.0f04 - SerDes1 Control Register 0 */
-	__be32	srds1cr1;	/* 0x.0f08 - SerDes1 Control Register 0 */
+	u32	srds1cr0;	/* 0x.0f04 - SerDes1 Control Register 0 */
+	u32	srds1cr1;	/* 0x.0f08 - SerDes1 Control Register 0 */
 	u8	resf0c[0xf2c - 0xf0c];
-	__be32  itcr;		/* 0x.0f2c - Internal transaction control register */
+	u32	itcr;		/* 0x.0f2c - Internal transaction control
+				 *           register
+				 */
 	u8	resf30[0xf40 - 0xf30];
-	__be32	srds2cr0;	/* 0x.0f40 - SerDes2 Control Register 0 */
-	__be32	srds2cr1;	/* 0x.0f44 - SerDes2 Control Register 0 */
+	u32	srds2cr0;	/* 0x.0f40 - SerDes2 Control Register 0 */
+	u32	srds2cr1;	/* 0x.0f44 - SerDes2 Control Register 0 */
 } __attribute__ ((packed));
 
+u32 fsl_guts_get_svr(void);
 
 /* Alternate function signal multiplex control */
 #define MPC85xx_PMUXCR_QE(x) (0x8000 >> (x))
diff --git a/include/linux/fsl_devices.h b/include/linux/fsl_devices.h
index f2912914141a..60cef8227534 100644
--- a/include/linux/fsl_devices.h
+++ b/include/linux/fsl_devices.h
@@ -100,6 +100,7 @@ struct fsl_usb2_platform_data {
 	unsigned	already_suspended:1;
 	unsigned        has_fsl_erratum_a007792:1;
 	unsigned        has_fsl_erratum_a005275:1;
+	unsigned	has_fsl_erratum_a005697:1;
 	unsigned        check_phy_clk_valid:1;
 
 	/* register save area for suspend/resume */
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index b3d34d3e0e7e..3633e8beff39 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -398,6 +398,7 @@ int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
 void ftrace_set_global_filter(unsigned char *buf, int len, int reset);
 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset);
 void ftrace_free_filter(struct ftrace_ops *ops);
+void ftrace_ops_set_global_filter(struct ftrace_ops *ops);
 
 int register_ftrace_command(struct ftrace_func_command *cmd);
 int unregister_ftrace_command(struct ftrace_func_command *cmd);
@@ -645,6 +646,7 @@ static inline unsigned long ftrace_location(unsigned long ip)
 #define ftrace_set_filter(ops, buf, len, reset) ({ -ENODEV; })
 #define ftrace_set_notrace(ops, buf, len, reset) ({ -ENODEV; })
 #define ftrace_free_filter(ops) do { } while (0)
+#define ftrace_ops_set_global_filter(ops) do { } while (0)
 
 static inline ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf,
 			    size_t cnt, loff_t *ppos) { return -ENODEV; }
@@ -945,6 +947,10 @@ extern int __disable_trace_on_warning;
 #define INIT_TRACE_RECURSION		.trace_recursion = 0,
 #endif
 
+int tracepoint_printk_sysctl(struct ctl_table *table, int write,
+			     void __user *buffer, size_t *lenp,
+			     loff_t *ppos);
+
 #else /* CONFIG_TRACING */
 static inline void  disable_trace_on_warning(void) { }
 #endif /* CONFIG_TRACING */
diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h
index 851671742790..8bd28ce6d76e 100644
--- a/include/linux/fwnode.h
+++ b/include/linux/fwnode.h
@@ -17,8 +17,9 @@ enum fwnode_type {
 	FWNODE_OF,
 	FWNODE_ACPI,
 	FWNODE_ACPI_DATA,
+	FWNODE_ACPI_STATIC,
 	FWNODE_PDATA,
-	FWNODE_IRQCHIP,
+	FWNODE_IRQCHIP
 };
 
 struct fwnode_handle {
diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index f8041f9de31e..4175dca4ac39 100644
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -506,6 +506,8 @@ extern void free_hot_cold_page(struct page *page, bool cold);
 extern void free_hot_cold_page_list(struct list_head *list, bool cold);
 
 struct page_frag_cache;
+extern void __page_frag_drain(struct page *page, unsigned int order,
+			      unsigned int count);
 extern void *__alloc_page_frag(struct page_frag_cache *nc,
 			       unsigned int fragsz, gfp_t gfp_mask);
 extern void __free_page_frag(void *addr);
diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h
index 24e2cc56beb1..c2748accea71 100644
--- a/include/linux/gpio/driver.h
+++ b/include/linux/gpio/driver.h
@@ -82,8 +82,6 @@ enum single_ended_mode {
  *	implies that if the chip supports IRQs, these IRQs need to be threaded
  *	as the chip access may sleep when e.g. reading out the IRQ status
  *	registers.
- * @irq_not_threaded: flag must be set if @can_sleep is set but the
- *	IRQs don't need to be threaded
  * @read_reg: reader function for generic GPIO
  * @write_reg: writer function for generic GPIO
  * @pin2mask: some generic GPIO controllers work with the big-endian bits
@@ -91,7 +89,7 @@ enum single_ended_mode {
  *	bit. This callback assigns the right bit mask.
  * @reg_dat: data (in) register for generic GPIO
  * @reg_set: output set register (out=high) for generic GPIO
- * @reg_clk: output clear register (out=low) for generic GPIO
+ * @reg_clr: output clear register (out=low) for generic GPIO
  * @reg_dir: direction setting register for generic GPIO
  * @bgpio_bits: number of register bits used for a generic GPIO i.e.
  *	<register width> * 8
@@ -109,8 +107,10 @@ enum single_ended_mode {
  *	for GPIO IRQs, provided by GPIO driver
  * @irq_default_type: default IRQ triggering type applied during GPIO driver
  *	initialization, provided by GPIO driver
- * @irq_parent: GPIO IRQ chip parent/bank linux irq number,
- *	provided by GPIO driver
+ * @irq_chained_parent: GPIO IRQ chip parent/bank linux irq number,
+ *	provided by GPIO driver for chained interrupt (not for nested
+ *	interrupts).
+ * @irq_nested: True if set the interrupt handling is nested.
  * @irq_need_valid_mask: If set core allocates @irq_valid_mask with all
  *	bits set to one
  * @irq_valid_mask: If not %NULL holds bitmask of GPIOs which are valid to
@@ -166,7 +166,6 @@ struct gpio_chip {
 	u16			ngpio;
 	const char		*const *names;
 	bool			can_sleep;
-	bool			irq_not_threaded;
 
 #if IS_ENABLED(CONFIG_GPIO_GENERIC)
 	unsigned long (*read_reg)(void __iomem *reg);
@@ -192,7 +191,8 @@ struct gpio_chip {
 	unsigned int		irq_base;
 	irq_flow_handler_t	irq_handler;
 	unsigned int		irq_default_type;
-	int			irq_parent;
+	int			irq_chained_parent;
+	bool			irq_nested;
 	bool			irq_need_valid_mask;
 	unsigned long		*irq_valid_mask;
 	struct lock_class_key	*lock_key;
@@ -270,24 +270,40 @@ void gpiochip_set_chained_irqchip(struct gpio_chip *gpiochip,
 		int parent_irq,
 		irq_flow_handler_t parent_handler);
 
+void gpiochip_set_nested_irqchip(struct gpio_chip *gpiochip,
+		struct irq_chip *irqchip,
+		int parent_irq);
+
 int _gpiochip_irqchip_add(struct gpio_chip *gpiochip,
 			  struct irq_chip *irqchip,
 			  unsigned int first_irq,
 			  irq_flow_handler_t handler,
 			  unsigned int type,
+			  bool nested,
 			  struct lock_class_key *lock_key);
 
+/* FIXME: I assume threaded IRQchips do not have the lockdep problem */
+static inline int gpiochip_irqchip_add_nested(struct gpio_chip *gpiochip,
+			  struct irq_chip *irqchip,
+			  unsigned int first_irq,
+			  irq_flow_handler_t handler,
+			  unsigned int type)
+{
+	return _gpiochip_irqchip_add(gpiochip, irqchip, first_irq,
+				     handler, type, true, NULL);
+}
+
 #ifdef CONFIG_LOCKDEP
 #define gpiochip_irqchip_add(...)				\
 (								\
 	({							\
 		static struct lock_class_key _key;		\
-		_gpiochip_irqchip_add(__VA_ARGS__, &_key);	\
+		_gpiochip_irqchip_add(__VA_ARGS__, false, &_key); \
 	})							\
 )
 #else
 #define gpiochip_irqchip_add(...)				\
-	_gpiochip_irqchip_add(__VA_ARGS__, NULL)
+	_gpiochip_irqchip_add(__VA_ARGS__, false, NULL)
 #endif
 
 #endif /* CONFIG_GPIOLIB_IRQCHIP */
diff --git a/include/linux/hdmi.h b/include/linux/hdmi.h
index e9744202fa29..edbb4fc674ed 100644
--- a/include/linux/hdmi.h
+++ b/include/linux/hdmi.h
@@ -78,6 +78,8 @@ enum hdmi_picture_aspect {
 	HDMI_PICTURE_ASPECT_NONE,
 	HDMI_PICTURE_ASPECT_4_3,
 	HDMI_PICTURE_ASPECT_16_9,
+	HDMI_PICTURE_ASPECT_64_27,
+	HDMI_PICTURE_ASPECT_256_135,
 	HDMI_PICTURE_ASPECT_RESERVED,
 };
 
diff --git a/include/linux/hid.h b/include/linux/hid.h
index b2ec82712baa..28f38e2b8f30 100644
--- a/include/linux/hid.h
+++ b/include/linux/hid.h
@@ -231,7 +231,11 @@ struct hid_item {
 #define HID_DG_TAP		0x000d0035
 #define HID_DG_TABLETFUNCTIONKEY	0x000d0039
 #define HID_DG_PROGRAMCHANGEKEY	0x000d003a
+#define HID_DG_BATTERYSTRENGTH	0x000d003b
 #define HID_DG_INVERT		0x000d003c
+#define HID_DG_TILT_X		0x000d003d
+#define HID_DG_TILT_Y		0x000d003e
+#define HID_DG_TWIST		0x000d0041
 #define HID_DG_TIPSWITCH	0x000d0042
 #define HID_DG_TIPSWITCH2	0x000d0043
 #define HID_DG_BARRELSWITCH	0x000d0044
@@ -479,6 +483,7 @@ struct hid_input {
 	struct list_head list;
 	struct hid_report *report;
 	struct input_dev *input;
+	bool registered;
 };
 
 enum hid_type {
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 1f782aa1d8e6..97e478d6b690 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -1,12 +1,12 @@
 #ifndef _LINUX_HUGE_MM_H
 #define _LINUX_HUGE_MM_H
 
-extern int do_huge_pmd_anonymous_page(struct fault_env *fe);
+extern int do_huge_pmd_anonymous_page(struct vm_fault *vmf);
 extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 			 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
 			 struct vm_area_struct *vma);
-extern void huge_pmd_set_accessed(struct fault_env *fe, pmd_t orig_pmd);
-extern int do_huge_pmd_wp_page(struct fault_env *fe, pmd_t orig_pmd);
+extern void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd);
+extern int do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd);
 extern struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
 					  unsigned long addr,
 					  pmd_t *pmd,
@@ -142,7 +142,7 @@ static inline int hpage_nr_pages(struct page *page)
 	return 1;
 }
 
-extern int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t orig_pmd);
+extern int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t orig_pmd);
 
 extern struct page *huge_zero_page;
 
@@ -212,7 +212,7 @@ static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
 	return NULL;
 }
 
-static inline int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t orig_pmd)
+static inline int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t orig_pmd)
 {
 	return 0;
 }
diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h
index 34a0dc18f327..bee0827766a3 100644
--- a/include/linux/hw_random.h
+++ b/include/linux/hw_random.h
@@ -30,8 +30,7 @@
  *			Must not be NULL.    *OBSOLETE*
  * @read:		New API. drivers can fill up to max bytes of data
  *			into the buffer. The buffer is aligned for any type
- *			and max is guaranteed to be >= to that alignment
- *			(either 4 or 8 depending on architecture).
+ *			and max is a multiple of 4 and >= 32 bytes.
  * @priv:		Private data, for use by the RNG driver.
  * @quality:		Estimation of true entropy in RNG's bitstream
  *			(per mill).
diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h
index 9d2f8bde7d12..78d59dba563e 100644
--- a/include/linux/hwmon.h
+++ b/include/linux/hwmon.h
@@ -298,8 +298,8 @@ enum hwmon_pwm_attributes {
  *			Channel number
  *		The function returns the file permissions.
  *		If the return value is 0, no attribute will be created.
- * @read:       Read callback. Optional. If not provided, attributes
- *		will not be readable.
+ * @read:	Read callback for data attributes. Mandatory if readable
+ *		data attributes are present.
  *		Parameters are:
  *		@dev:	Pointer to hardware monitoring device
  *		@type:	Sensor type
@@ -308,8 +308,19 @@ enum hwmon_pwm_attributes {
  *			Channel number
  *		@val:	Pointer to returned value
  *		The function returns 0 on success or a negative error number.
- * @write:	Write callback. Optional. If not provided, attributes
- *		will not be writable.
+ * @read_string:
+ *		Read callback for string attributes. Mandatory if string
+ *		attributes are present.
+ *		Parameters are:
+ *		@dev:	Pointer to hardware monitoring device
+ *		@type:	Sensor type
+ *		@attr:	Sensor attribute
+ *		@channel:
+ *			Channel number
+ *		@str:	Pointer to returned string
+ *		The function returns 0 on success or a negative error number.
+ * @write:	Write callback for data attributes. Mandatory if writeable
+ *		data attributes are present.
  *		Parameters are:
  *		@dev:	Pointer to hardware monitoring device
  *		@type:	Sensor type
@@ -324,6 +335,8 @@ struct hwmon_ops {
 			      u32 attr, int channel);
 	int (*read)(struct device *dev, enum hwmon_sensor_types type,
 		    u32 attr, int channel, long *val);
+	int (*read_string)(struct device *dev, enum hwmon_sensor_types type,
+		    u32 attr, int channel, char **str);
 	int (*write)(struct device *dev, enum hwmon_sensor_types type,
 		     u32 attr, int channel, long val);
 };
@@ -349,7 +362,9 @@ struct hwmon_chip_info {
 	const struct hwmon_channel_info **info;
 };
 
+/* hwmon_device_register() is deprecated */
 struct device *hwmon_device_register(struct device *dev);
+
 struct device *
 hwmon_device_register_with_groups(struct device *dev, const char *name,
 				  void *drvdata,
@@ -362,12 +377,12 @@ struct device *
 hwmon_device_register_with_info(struct device *dev,
 				const char *name, void *drvdata,
 				const struct hwmon_chip_info *info,
-				const struct attribute_group **groups);
+				const struct attribute_group **extra_groups);
 struct device *
 devm_hwmon_device_register_with_info(struct device *dev,
-				     const char *name, void *drvdata,
-				     const struct hwmon_chip_info *info,
-				     const struct attribute_group **groups);
+				const char *name, void *drvdata,
+				const struct hwmon_chip_info *info,
+				const struct attribute_group **extra_groups);
 
 void hwmon_device_unregister(struct device *dev);
 void devm_hwmon_device_unregister(struct device *dev);
diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h
index cd184bdca58f..42fe43fb0c80 100644
--- a/include/linux/hyperv.h
+++ b/include/linux/hyperv.h
@@ -696,7 +696,7 @@ enum vmbus_device_type {
 	HV_FCOPY,
 	HV_BACKUP,
 	HV_DM,
-	HV_UNKOWN,
+	HV_UNKNOWN,
 };
 
 struct vmbus_device {
@@ -1119,6 +1119,12 @@ struct hv_driver {
 
 	struct device_driver driver;
 
+	/* dynamic device GUID's */
+	struct  {
+		spinlock_t lock;
+		struct list_head list;
+	} dynids;
+
 	int (*probe)(struct hv_device *, const struct hv_vmbus_device_id *);
 	int (*remove)(struct hv_device *);
 	void (*shutdown)(struct hv_device *);
@@ -1447,6 +1453,7 @@ void hv_event_tasklet_enable(struct vmbus_channel *channel);
 
 void hv_process_channel_removal(struct vmbus_channel *channel, u32 relid);
 
+void vmbus_setevent(struct vmbus_channel *channel);
 /*
  * Negotiated version with the Host.
  */
@@ -1479,10 +1486,11 @@ hv_get_ring_buffer(struct hv_ring_buffer_info *ring_info)
  *    there is room for the producer to send the pending packet.
  */
 
-static inline  bool hv_need_to_signal_on_read(struct hv_ring_buffer_info *rbi)
+static inline  void hv_signal_on_read(struct vmbus_channel *channel)
 {
 	u32 cur_write_sz;
 	u32 pending_sz;
+	struct hv_ring_buffer_info *rbi = &channel->inbound;
 
 	/*
 	 * Issue a full memory barrier before making the signaling decision.
@@ -1500,14 +1508,14 @@ static inline  bool hv_need_to_signal_on_read(struct hv_ring_buffer_info *rbi)
 	pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz);
 	/* If the other end is not blocked on write don't bother. */
 	if (pending_sz == 0)
-		return false;
+		return;
 
 	cur_write_sz = hv_get_bytes_to_write(rbi);
 
 	if (cur_write_sz >= pending_sz)
-		return true;
+		vmbus_setevent(channel);
 
-	return false;
+	return;
 }
 
 /*
@@ -1519,31 +1527,23 @@ static inline struct vmpacket_descriptor *
 get_next_pkt_raw(struct vmbus_channel *channel)
 {
 	struct hv_ring_buffer_info *ring_info = &channel->inbound;
-	u32 read_loc = ring_info->priv_read_index;
+	u32 priv_read_loc = ring_info->priv_read_index;
 	void *ring_buffer = hv_get_ring_buffer(ring_info);
-	struct vmpacket_descriptor *cur_desc;
-	u32 packetlen;
 	u32 dsize = ring_info->ring_datasize;
-	u32 delta = read_loc - ring_info->ring_buffer->read_index;
+	/*
+	 * delta is the difference between what is available to read and
+	 * what was already consumed in place. We commit read index after
+	 * the whole batch is processed.
+	 */
+	u32 delta = priv_read_loc >= ring_info->ring_buffer->read_index ?
+		priv_read_loc - ring_info->ring_buffer->read_index :
+		(dsize - ring_info->ring_buffer->read_index) + priv_read_loc;
 	u32 bytes_avail_toread = (hv_get_bytes_to_read(ring_info) - delta);
 
 	if (bytes_avail_toread < sizeof(struct vmpacket_descriptor))
 		return NULL;
 
-	if ((read_loc + sizeof(*cur_desc)) > dsize)
-		return NULL;
-
-	cur_desc = ring_buffer + read_loc;
-	packetlen = cur_desc->len8 << 3;
-
-	/*
-	 * If the packet under consideration is wrapping around,
-	 * return failure.
-	 */
-	if ((read_loc + packetlen + VMBUS_PKT_TRAILER) > (dsize - 1))
-		return NULL;
-
-	return cur_desc;
+	return ring_buffer + priv_read_loc;
 }
 
 /*
@@ -1555,16 +1555,14 @@ static inline void put_pkt_raw(struct vmbus_channel *channel,
 				struct vmpacket_descriptor *desc)
 {
 	struct hv_ring_buffer_info *ring_info = &channel->inbound;
-	u32 read_loc = ring_info->priv_read_index;
 	u32 packetlen = desc->len8 << 3;
 	u32 dsize = ring_info->ring_datasize;
 
-	if ((read_loc + packetlen + VMBUS_PKT_TRAILER) > dsize)
-		BUG();
 	/*
 	 * Include the packet trailer.
 	 */
 	ring_info->priv_read_index += packetlen + VMBUS_PKT_TRAILER;
+	ring_info->priv_read_index %= dsize;
 }
 
 /*
@@ -1589,8 +1587,7 @@ static inline void commit_rd_index(struct vmbus_channel *channel)
 	virt_rmb();
 	ring_info->ring_buffer->read_index = ring_info->priv_read_index;
 
-	if (hv_need_to_signal_on_read(ring_info))
-		vmbus_set_event(channel);
+	hv_signal_on_read(channel);
 }
 
 
diff --git a/include/linux/i2c-smbus.h b/include/linux/i2c-smbus.h
index c2e3324f9468..a1385023a29b 100644
--- a/include/linux/i2c-smbus.h
+++ b/include/linux/i2c-smbus.h
@@ -50,31 +50,4 @@ struct i2c_client *i2c_setup_smbus_alert(struct i2c_adapter *adapter,
 					 struct i2c_smbus_alert_setup *setup);
 int i2c_handle_smbus_alert(struct i2c_client *ara);
 
-/**
- * smbus_host_notify - internal structure used by the Host Notify mechanism.
- * @adapter: the I2C adapter associated with this struct
- * @work: worker used to schedule the IRQ in the slave device
- * @lock: spinlock to check if a notification is already pending
- * @pending: flag set when a notification is pending (any new notification will
- *		be rejected if pending is true)
- * @payload: the actual payload of the Host Notify event
- * @addr: the address of the slave device which raised the notification
- *
- * This struct needs to be allocated by i2c_setup_smbus_host_notify() and does
- * not need to be freed. Internally, i2c_setup_smbus_host_notify() uses a
- * managed resource to clean this up when the adapter get released.
- */
-struct smbus_host_notify {
-	struct i2c_adapter	*adapter;
-	struct work_struct	work;
-	spinlock_t		lock;
-	bool			pending;
-	u16			payload;
-	u8			addr;
-};
-
-struct smbus_host_notify *i2c_setup_smbus_host_notify(struct i2c_adapter *adap);
-int i2c_handle_smbus_host_notify(struct smbus_host_notify *host_notify,
-				 unsigned short addr, unsigned int data);
-
 #endif /* _LINUX_I2C_SMBUS_H */
diff --git a/include/linux/i2c.h b/include/linux/i2c.h
index 6422eef428c4..b2109c522dec 100644
--- a/include/linux/i2c.h
+++ b/include/linux/i2c.h
@@ -30,6 +30,7 @@
 #include <linux/device.h>	/* for struct device */
 #include <linux/sched.h>	/* for completion */
 #include <linux/mutex.h>
+#include <linux/irqdomain.h>		/* for Host Notify IRQ */
 #include <linux/of.h>		/* for struct device_node */
 #include <linux/swab.h>		/* for swab16 */
 #include <uapi/linux/i2c.h>
@@ -135,7 +136,8 @@ enum i2c_alert_protocol {
  * struct i2c_driver - represent an I2C device driver
  * @class: What kind of i2c device we instantiate (for detect)
  * @attach_adapter: Callback for bus addition (deprecated)
- * @probe: Callback for device binding
+ * @probe: Callback for device binding - soon to be deprecated
+ * @probe_new: New callback for device binding
  * @remove: Callback for device unbinding
  * @shutdown: Callback for device shutdown
  * @alert: Alert callback, for example for the SMBus alert protocol
@@ -178,6 +180,11 @@ struct i2c_driver {
 	int (*probe)(struct i2c_client *, const struct i2c_device_id *);
 	int (*remove)(struct i2c_client *);
 
+	/* New driver model interface to aid the seamless removal of the
+	 * current probe()'s, more commonly unused than used second parameter.
+	 */
+	int (*probe_new)(struct i2c_client *);
+
 	/* driver model interfaces that don't relate to enumeration  */
 	void (*shutdown)(struct i2c_client *);
 
@@ -243,6 +250,8 @@ struct i2c_client {
 
 extern struct i2c_client *i2c_verify_client(struct device *dev);
 extern struct i2c_adapter *i2c_verify_adapter(struct device *dev);
+extern const struct i2c_device_id *i2c_match_id(const struct i2c_device_id *id,
+					const struct i2c_client *client);
 
 static inline struct i2c_client *kobj_to_i2c_client(struct kobject *kobj)
 {
@@ -567,6 +576,8 @@ struct i2c_adapter {
 
 	struct i2c_bus_recovery_info *bus_recovery_info;
 	const struct i2c_adapter_quirks *quirks;
+
+	struct irq_domain *host_notify_domain;
 };
 #define to_i2c_adapter(d) container_of(d, struct i2c_adapter, dev)
 
@@ -739,6 +750,7 @@ static inline u8 i2c_8bit_addr_from_msg(const struct i2c_msg *msg)
 	return (msg->addr << 1) | (msg->flags & I2C_M_RD ? 1 : 0);
 }
 
+int i2c_handle_smbus_host_notify(struct i2c_adapter *adap, unsigned short addr);
 /**
  * module_i2c_driver() - Helper macro for registering a modular I2C driver
  * @__i2c_driver: i2c_driver struct
@@ -774,6 +786,10 @@ extern struct i2c_adapter *of_find_i2c_adapter_by_node(struct device_node *node)
 /* must call i2c_put_adapter() when done with returned i2c_adapter device */
 struct i2c_adapter *of_get_i2c_adapter_by_node(struct device_node *node);
 
+extern const struct of_device_id
+*i2c_of_match_device(const struct of_device_id *matches,
+		     struct i2c_client *client);
+
 #else
 
 static inline struct i2c_client *of_find_i2c_device_by_node(struct device_node *node)
@@ -790,6 +806,14 @@ static inline struct i2c_adapter *of_get_i2c_adapter_by_node(struct device_node
 {
 	return NULL;
 }
+
+static inline const struct of_device_id
+*i2c_of_match_device(const struct of_device_id *matches,
+		     struct i2c_client *client)
+{
+	return NULL;
+}
+
 #endif /* CONFIG_OF */
 
 #if IS_ENABLED(CONFIG_ACPI)
diff --git a/include/linux/i2c/mlxcpld.h b/include/linux/i2c/mlxcpld.h
new file mode 100644
index 000000000000..b08dcb183fca
--- /dev/null
+++ b/include/linux/i2c/mlxcpld.h
@@ -0,0 +1,52 @@
+/*
+ * mlxcpld.h - Mellanox I2C multiplexer support in CPLD
+ *
+ * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2016 Michael Shych <michaels@mellanox.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the names of the copyright holders nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _LINUX_I2C_MLXCPLD_H
+#define _LINUX_I2C_MLXCPLD_H
+
+/* Platform data for the CPLD I2C multiplexers */
+
+/* mlxcpld_mux_plat_data - per mux data, used with i2c_register_board_info
+ * @adap_ids - adapter array
+ * @num_adaps - number of adapters
+ * @sel_reg_addr - mux select register offset in CPLD space
+ */
+struct mlxcpld_mux_plat_data {
+	int *adap_ids;
+	int num_adaps;
+	int sel_reg_addr;
+};
+
+#endif /* _LINUX_I2C_MLXCPLD_H */
diff --git a/include/linux/idr.h b/include/linux/idr.h
index 083d61e92706..3c01b89aed67 100644
--- a/include/linux/idr.h
+++ b/include/linux/idr.h
@@ -18,12 +18,11 @@
 #include <linux/rcupdate.h>
 
 /*
- * We want shallower trees and thus more bits covered at each layer.  8
- * bits gives us large enough first layer for most use cases and maximum
- * tree depth of 4.  Each idr_layer is slightly larger than 2k on 64bit and
- * 1k on 32bit.
+ * Using 6 bits at each layer allows us to allocate 7 layers out of each page.
+ * 8 bits only gave us 3 layers out of every pair of pages, which is less
+ * efficient except for trees with a largest element between 192-255 inclusive.
  */
-#define IDR_BITS 8
+#define IDR_BITS 6
 #define IDR_SIZE (1 << IDR_BITS)
 #define IDR_MASK ((1 << IDR_BITS)-1)
 
@@ -56,6 +55,32 @@ struct idr {
 #define DEFINE_IDR(name)	struct idr name = IDR_INIT(name)
 
 /**
+ * idr_get_cursor - Return the current position of the cyclic allocator
+ * @idr: idr handle
+ *
+ * The value returned is the value that will be next returned from
+ * idr_alloc_cyclic() if it is free (otherwise the search will start from
+ * this position).
+ */
+static inline unsigned int idr_get_cursor(struct idr *idr)
+{
+	return READ_ONCE(idr->cur);
+}
+
+/**
+ * idr_set_cursor - Set the current position of the cyclic allocator
+ * @idr: idr handle
+ * @val: new position
+ *
+ * The next call to idr_alloc_cyclic() will return @val if it is free
+ * (otherwise the search will start from this position).
+ */
+static inline void idr_set_cursor(struct idr *idr, unsigned int val)
+{
+	WRITE_ONCE(idr->cur, val);
+}
+
+/**
  * DOC: idr sync
  * idr synchronization (stolen from radix-tree.h)
  *
@@ -195,6 +220,11 @@ static inline int ida_get_new(struct ida *ida, int *p_id)
 	return ida_get_new_above(ida, 0, p_id);
 }
 
+static inline bool ida_is_empty(struct ida *ida)
+{
+	return idr_is_empty(&ida->idr);
+}
+
 void __init idr_init_cache(void);
 
 #endif /* __IDR_H__ */
diff --git a/include/linux/iio/adc/ad_sigma_delta.h b/include/linux/iio/adc/ad_sigma_delta.h
index e7fdec4db9da..5ba430cc9a87 100644
--- a/include/linux/iio/adc/ad_sigma_delta.h
+++ b/include/linux/iio/adc/ad_sigma_delta.h
@@ -136,6 +136,7 @@ int ad_sd_validate_trigger(struct iio_dev *indio_dev, struct iio_trigger *trig);
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \
 			BIT(IIO_CHAN_INFO_OFFSET), \
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \
 		.scan_index = (_si), \
 		.scan_type = { \
 			.sign = 'u', \
diff --git a/include/linux/iio/consumer.h b/include/linux/iio/consumer.h
index 9edccfba1ffb..47eeec3218b5 100644
--- a/include/linux/iio/consumer.h
+++ b/include/linux/iio/consumer.h
@@ -226,6 +226,34 @@ int iio_read_channel_processed(struct iio_channel *chan, int *val);
 int iio_write_channel_raw(struct iio_channel *chan, int val);
 
 /**
+ * iio_read_max_channel_raw() - read maximum available raw value from a given
+ *				channel, i.e. the maximum possible value.
+ * @chan:		The channel being queried.
+ * @val:		Value read back.
+ *
+ * Note raw reads from iio channels are in adc counts and hence
+ * scale will need to be applied if standard units are required.
+ */
+int iio_read_max_channel_raw(struct iio_channel *chan, int *val);
+
+/**
+ * iio_read_avail_channel_raw() - read available raw values from a given channel
+ * @chan:		The channel being queried.
+ * @vals:		Available values read back.
+ * @length:		Number of entries in vals.
+ *
+ * Returns an error code, IIO_AVAIL_RANGE or IIO_AVAIL_LIST.
+ *
+ * For ranges, three vals are always returned; min, step and max.
+ * For lists, all the possible values are enumerated.
+ *
+ * Note raw available values from iio channels are in adc counts and
+ * hence scale will need to be applied if standard units are required.
+ */
+int iio_read_avail_channel_raw(struct iio_channel *chan,
+			       const int **vals, int *length);
+
+/**
  * iio_get_channel_type() - get the type of a channel
  * @channel:		The channel being queried.
  * @type:		The type of the channel.
@@ -236,6 +264,19 @@ int iio_get_channel_type(struct iio_channel *channel,
 			 enum iio_chan_type *type);
 
 /**
+ * iio_read_channel_offset() - read the offset value for a channel
+ * @chan:		The channel being queried.
+ * @val:		First part of value read back.
+ * @val2:		Second part of value read back.
+ *
+ * Note returns a description of what is in val and val2, such
+ * as IIO_VAL_INT_PLUS_MICRO telling us we have a value of val
+ * + val2/1e6
+ */
+int iio_read_channel_offset(struct iio_channel *chan, int *val,
+			   int *val2);
+
+/**
  * iio_read_channel_scale() - read the scale value for a channel
  * @chan:		The channel being queried.
  * @val:		First part of value read back.
diff --git a/include/linux/iio/dac/mcp4725.h b/include/linux/iio/dac/mcp4725.h
index 91530e6611e9..628b2cf54c50 100644
--- a/include/linux/iio/dac/mcp4725.h
+++ b/include/linux/iio/dac/mcp4725.h
@@ -9,8 +9,18 @@
 #ifndef IIO_DAC_MCP4725_H_
 #define IIO_DAC_MCP4725_H_
 
+/**
+ * struct mcp4725_platform_data - MCP4725/6 DAC specific data.
+ * @use_vref: Whether an external reference voltage on Vref pin should be used.
+ *            Additional vref-supply must be specified when used.
+ * @vref_buffered: Controls buffering of the external reference voltage.
+ *
+ * Vref related settings are available only on MCP4756. See
+ * Documentation/devicetree/bindings/iio/dac/mcp4725.txt for more information.
+ */
 struct mcp4725_platform_data {
-	u16 vref_mv;
+	bool use_vref;
+	bool vref_buffered;
 };
 
 #endif /* IIO_DAC_MCP4725_H_ */
diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h
index b4a0679e4a49..3f5ea2e9a39e 100644
--- a/include/linux/iio/iio.h
+++ b/include/linux/iio/iio.h
@@ -225,12 +225,22 @@ struct iio_event_spec {
  *			endianness:	little or big endian
  * @info_mask_separate: What information is to be exported that is specific to
  *			this channel.
+ * @info_mask_separate_available: What availability information is to be
+ *			exported that is specific to this channel.
  * @info_mask_shared_by_type: What information is to be exported that is shared
  *			by all channels of the same type.
+ * @info_mask_shared_by_type_available: What availability information is to be
+ *			exported that is shared by all channels of the same
+ *			type.
  * @info_mask_shared_by_dir: What information is to be exported that is shared
  *			by all channels of the same direction.
+ * @info_mask_shared_by_dir_available: What availability information is to be
+ *			exported that is shared by all channels of the same
+ *			direction.
  * @info_mask_shared_by_all: What information is to be exported that is shared
  *			by all channels.
+ * @info_mask_shared_by_all_available: What availability information is to be
+ *			exported that is shared by all channels.
  * @event_spec:		Array of events which should be registered for this
  *			channel.
  * @num_event_specs:	Size of the event_spec array.
@@ -269,9 +279,13 @@ struct iio_chan_spec {
 		enum iio_endian endianness;
 	} scan_type;
 	long			info_mask_separate;
+	long			info_mask_separate_available;
 	long			info_mask_shared_by_type;
+	long			info_mask_shared_by_type_available;
 	long			info_mask_shared_by_dir;
+	long			info_mask_shared_by_dir_available;
 	long			info_mask_shared_by_all;
+	long			info_mask_shared_by_all_available;
 	const struct iio_event_spec *event_spec;
 	unsigned int		num_event_specs;
 	const struct iio_chan_spec_ext_info *ext_info;
@@ -301,6 +315,23 @@ static inline bool iio_channel_has_info(const struct iio_chan_spec *chan,
 		(chan->info_mask_shared_by_all & BIT(type));
 }
 
+/**
+ * iio_channel_has_available() - Checks if a channel has an available attribute
+ * @chan: The channel to be queried
+ * @type: Type of the available attribute to be checked
+ *
+ * Returns true if the channel supports reporting available values for the
+ * given attribute type, false otherwise.
+ */
+static inline bool iio_channel_has_available(const struct iio_chan_spec *chan,
+					     enum iio_chan_info_enum type)
+{
+	return (chan->info_mask_separate_available & BIT(type)) |
+		(chan->info_mask_shared_by_type_available & BIT(type)) |
+		(chan->info_mask_shared_by_dir_available & BIT(type)) |
+		(chan->info_mask_shared_by_all_available & BIT(type));
+}
+
 #define IIO_CHAN_SOFT_TIMESTAMP(_si) {					\
 	.type = IIO_TIMESTAMP,						\
 	.channel = -1,							\
@@ -349,6 +380,14 @@ struct iio_dev;
  *			max_len specifies maximum number of elements
  *			vals pointer can contain. val_len is used to return
  *			length of valid elements in vals.
+ * @read_avail:		function to return the available values from the device.
+ *			mask specifies which value. Note 0 means the available
+ *			values for the channel in question.  Return value
+ *			specifies if a IIO_AVAIL_LIST or a IIO_AVAIL_RANGE is
+ *			returned in vals. The type of the vals are returned in
+ *			type and the number of vals is returned in length. For
+ *			ranges, there are always three vals returned; min, step
+ *			and max. For lists, all possible values are enumerated.
  * @write_raw:		function to write a value to the device.
  *			Parameters are the same as for read_raw.
  * @write_raw_get_fmt:	callback function to query the expected
@@ -381,7 +420,7 @@ struct iio_dev;
  **/
 struct iio_info {
 	struct module			*driver_module;
-	struct attribute_group		*event_attrs;
+	const struct attribute_group	*event_attrs;
 	const struct attribute_group	*attrs;
 
 	int (*read_raw)(struct iio_dev *indio_dev,
@@ -397,6 +436,13 @@ struct iio_info {
 			int *val_len,
 			long mask);
 
+	int (*read_avail)(struct iio_dev *indio_dev,
+			  struct iio_chan_spec const *chan,
+			  const int **vals,
+			  int *type,
+			  int *length,
+			  long mask);
+
 	int (*write_raw)(struct iio_dev *indio_dev,
 			 struct iio_chan_spec const *chan,
 			 int val,
diff --git a/include/linux/iio/sysfs.h b/include/linux/iio/sysfs.h
index 9cd8f747212f..ce9426c507fd 100644
--- a/include/linux/iio/sysfs.h
+++ b/include/linux/iio/sysfs.h
@@ -55,10 +55,34 @@ struct iio_const_attr {
 	{ .dev_attr = __ATTR(_name, _mode, _show, _store),	\
 	  .address = _addr }
 
+#define IIO_ATTR_RO(_name, _addr)       \
+	{ .dev_attr = __ATTR_RO(_name), \
+	  .address = _addr }
+
+#define IIO_ATTR_WO(_name, _addr)       \
+	{ .dev_attr = __ATTR_WO(_name), \
+	  .address = _addr }
+
+#define IIO_ATTR_RW(_name, _addr)       \
+	{ .dev_attr = __ATTR_RW(_name), \
+	  .address = _addr }
+
 #define IIO_DEVICE_ATTR(_name, _mode, _show, _store, _addr)	\
 	struct iio_dev_attr iio_dev_attr_##_name		\
 	= IIO_ATTR(_name, _mode, _show, _store, _addr)
 
+#define IIO_DEVICE_ATTR_RO(_name, _addr)                       \
+	struct iio_dev_attr iio_dev_attr_##_name                \
+	= IIO_ATTR_RO(_name, _addr)
+
+#define IIO_DEVICE_ATTR_WO(_name, _addr)                       \
+	struct iio_dev_attr iio_dev_attr_##_name                \
+	= IIO_ATTR_WO(_name, _addr)
+
+#define IIO_DEVICE_ATTR_RW(_name, _addr)                                   \
+	struct iio_dev_attr iio_dev_attr_##_name                            \
+	= IIO_ATTR_RW(_name, _addr)
+
 #define IIO_DEVICE_ATTR_NAMED(_vname, _name, _mode, _show, _store, _addr) \
 	struct iio_dev_attr iio_dev_attr_##_vname			\
 	= IIO_ATTR(_name, _mode, _show, _store, _addr)
diff --git a/include/linux/iio/trigger.h b/include/linux/iio/trigger.h
index 4f1154f7a33c..ea08302f2d7b 100644
--- a/include/linux/iio/trigger.h
+++ b/include/linux/iio/trigger.h
@@ -170,6 +170,8 @@ void iio_trigger_free(struct iio_trigger *trig);
  */
 bool iio_trigger_using_own(struct iio_dev *indio_dev);
 
+int iio_trigger_validate_own_device(struct iio_trigger *trig,
+				     struct iio_dev *indio_dev);
 
 #else
 struct iio_trigger;
diff --git a/include/linux/iio/types.h b/include/linux/iio/types.h
index 32b579525004..2aa7b6384d64 100644
--- a/include/linux/iio/types.h
+++ b/include/linux/iio/types.h
@@ -29,4 +29,9 @@ enum iio_event_info {
 #define IIO_VAL_FRACTIONAL 10
 #define IIO_VAL_FRACTIONAL_LOG2 11
 
+enum iio_available_type {
+	IIO_AVAIL_LIST,
+	IIO_AVAIL_RANGE,
+};
+
 #endif /* _IIO_TYPES_H_ */
diff --git a/include/linux/init.h b/include/linux/init.h
index e30104ceb86d..885c3e6d0f9d 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -126,6 +126,9 @@ void prepare_namespace(void);
 void __init load_default_modules(void);
 int __init init_rootfs(void);
 
+#if defined(CONFIG_DEBUG_RODATA) || defined(CONFIG_DEBUG_SET_MODULE_RONX)
+extern bool rodata_enabled;
+#endif
 #ifdef CONFIG_DEBUG_RODATA
 void mark_rodata_ro(void);
 #endif
diff --git a/include/linux/iomap.h b/include/linux/iomap.h
index 7892f55a1866..a4c94b86401e 100644
--- a/include/linux/iomap.h
+++ b/include/linux/iomap.h
@@ -49,6 +49,8 @@ struct iomap {
 #define IOMAP_WRITE		(1 << 0) /* writing, must allocate blocks */
 #define IOMAP_ZERO		(1 << 1) /* zeroing operation, may skip holes */
 #define IOMAP_REPORT		(1 << 2) /* report extent status, e.g. FIEMAP */
+#define IOMAP_FAULT		(1 << 3) /* mapping for page fault */
+#define IOMAP_DIRECT		(1 << 4) /* direct I/O */
 
 struct iomap_ops {
 	/*
@@ -82,4 +84,14 @@ int iomap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
 int iomap_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 		loff_t start, loff_t len, struct iomap_ops *ops);
 
+/*
+ * Flags for direct I/O ->end_io:
+ */
+#define IOMAP_DIO_UNWRITTEN	(1 << 0)	/* covers unwritten extent(s) */
+#define IOMAP_DIO_COW		(1 << 1)	/* covers COW extent(s) */
+typedef int (iomap_dio_end_io_t)(struct kiocb *iocb, ssize_t ret,
+		unsigned flags);
+ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
+		struct iomap_ops *ops, iomap_dio_end_io_t end_io);
+
 #endif /* LINUX_IOMAP_H */
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 436dc21318af..0ff5111f6959 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -253,6 +253,7 @@ extern void iommu_group_remove_device(struct device *dev);
 extern int iommu_group_for_each_dev(struct iommu_group *group, void *data,
 				    int (*fn)(struct device *, void *));
 extern struct iommu_group *iommu_group_get(struct device *dev);
+extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group);
 extern void iommu_group_put(struct iommu_group *group);
 extern int iommu_group_register_notifier(struct iommu_group *group,
 					 struct notifier_block *nb);
@@ -351,6 +352,9 @@ int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
 		      const struct iommu_ops *ops);
 void iommu_fwspec_free(struct device *dev);
 int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids);
+void iommu_register_instance(struct fwnode_handle *fwnode,
+			     const struct iommu_ops *ops);
+const struct iommu_ops *iommu_get_instance(struct fwnode_handle *fwnode);
 
 #else /* CONFIG_IOMMU_API */
 
@@ -580,6 +584,17 @@ static inline int iommu_fwspec_add_ids(struct device *dev, u32 *ids,
 	return -ENODEV;
 }
 
+static inline void iommu_register_instance(struct fwnode_handle *fwnode,
+					   const struct iommu_ops *ops)
+{
+}
+
+static inline
+const struct iommu_ops *iommu_get_instance(struct fwnode_handle *fwnode)
+{
+	return NULL;
+}
+
 #endif /* CONFIG_IOMMU_API */
 
 #endif /* __LINUX_IOMMU_H */
diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h
index 5118d3a0c9ca..e808f8ae6f14 100644
--- a/include/linux/irqchip/arm-gic-v3.h
+++ b/include/linux/irqchip/arm-gic-v3.h
@@ -295,10 +295,10 @@
 #define GITS_BASER_InnerShareable					\
 	GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable)
 #define GITS_BASER_PAGE_SIZE_SHIFT	(8)
-#define GITS_BASER_PAGE_SIZE_4K		(0UL << GITS_BASER_PAGE_SIZE_SHIFT)
-#define GITS_BASER_PAGE_SIZE_16K	(1UL << GITS_BASER_PAGE_SIZE_SHIFT)
-#define GITS_BASER_PAGE_SIZE_64K	(2UL << GITS_BASER_PAGE_SIZE_SHIFT)
-#define GITS_BASER_PAGE_SIZE_MASK	(3UL << GITS_BASER_PAGE_SIZE_SHIFT)
+#define GITS_BASER_PAGE_SIZE_4K		(0ULL << GITS_BASER_PAGE_SIZE_SHIFT)
+#define GITS_BASER_PAGE_SIZE_16K	(1ULL << GITS_BASER_PAGE_SIZE_SHIFT)
+#define GITS_BASER_PAGE_SIZE_64K	(2ULL << GITS_BASER_PAGE_SIZE_SHIFT)
+#define GITS_BASER_PAGE_SIZE_MASK	(3ULL << GITS_BASER_PAGE_SIZE_SHIFT)
 #define GITS_BASER_PAGES_MAX		256
 #define GITS_BASER_PAGES_SHIFT		(0)
 #define GITS_BASER_NR_PAGES(r)		(((r) & 0xff) + 1)
diff --git a/include/linux/kdb.h b/include/linux/kdb.h
index 410decacff8f..68bd88223417 100644
--- a/include/linux/kdb.h
+++ b/include/linux/kdb.h
@@ -77,7 +77,6 @@ extern int kdb_poll_idx;
  * number whenever the kernel debugger is entered.
  */
 extern int kdb_initial_cpu;
-extern atomic_t kdb_event;
 
 /* Types and messages used for dynamically added kdb shell commands */
 
@@ -162,6 +161,7 @@ enum kdb_msgsrc {
 };
 
 extern int kdb_trap_printk;
+extern int kdb_printf_cpu;
 extern __printf(2, 0) int vkdb_printf(enum kdb_msgsrc src, const char *fmt,
 				      va_list args);
 extern __printf(1, 2) int kdb_printf(const char *, ...);
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index bc6ed52a39b9..56aec84237ad 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -45,11 +45,16 @@
 
 #define REPEAT_BYTE(x)	((~0ul / 0xff) * (x))
 
+/* @a is a power of 2 value */
 #define ALIGN(x, a)		__ALIGN_KERNEL((x), (a))
 #define __ALIGN_MASK(x, mask)	__ALIGN_KERNEL_MASK((x), (mask))
 #define PTR_ALIGN(p, a)		((typeof(p))ALIGN((unsigned long)(p), (a)))
 #define IS_ALIGNED(x, a)		(((x) & ((typeof(x))(a) - 1)) == 0)
 
+/* generic data direction definitions */
+#define READ			0
+#define WRITE			1
+
 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]) + __must_be_array(arr))
 
 #define u64_to_user_ptr(x) (		\
@@ -506,6 +511,15 @@ extern enum system_states {
 #define TAINT_UNSIGNED_MODULE		13
 #define TAINT_SOFTLOCKUP		14
 #define TAINT_LIVEPATCH			15
+#define TAINT_FLAGS_COUNT		16
+
+struct taint_flag {
+	char true;	/* character printed when tainted */
+	char false;	/* character printed when not tainted */
+	bool module;	/* also show as a per-module taint flag */
+};
+
+extern const struct taint_flag taint_flags[TAINT_FLAGS_COUNT];
 
 extern const char hex_asc[];
 #define hex_asc_lo(x)	hex_asc[((x) & 0x0f)]
diff --git a/include/linux/kexec.h b/include/linux/kexec.h
index 406c33dcae13..d419d0e51fe5 100644
--- a/include/linux/kexec.h
+++ b/include/linux/kexec.h
@@ -148,7 +148,36 @@ struct kexec_file_ops {
 	kexec_verify_sig_t *verify_sig;
 #endif
 };
-#endif
+
+/**
+ * struct kexec_buf - parameters for finding a place for a buffer in memory
+ * @image:	kexec image in which memory to search.
+ * @buffer:	Contents which will be copied to the allocated memory.
+ * @bufsz:	Size of @buffer.
+ * @mem:	On return will have address of the buffer in memory.
+ * @memsz:	Size for the buffer in memory.
+ * @buf_align:	Minimum alignment needed.
+ * @buf_min:	The buffer can't be placed below this address.
+ * @buf_max:	The buffer can't be placed above this address.
+ * @top_down:	Allocate from top of memory.
+ */
+struct kexec_buf {
+	struct kimage *image;
+	void *buffer;
+	unsigned long bufsz;
+	unsigned long mem;
+	unsigned long memsz;
+	unsigned long buf_align;
+	unsigned long buf_min;
+	unsigned long buf_max;
+	bool top_down;
+};
+
+int __weak arch_kexec_walk_mem(struct kexec_buf *kbuf,
+			       int (*func)(u64, u64, void *));
+extern int kexec_add_buffer(struct kexec_buf *kbuf);
+int kexec_locate_mem_hole(struct kexec_buf *kbuf);
+#endif /* CONFIG_KEXEC_FILE */
 
 struct kimage {
 	kimage_entry_t head;
@@ -212,11 +241,6 @@ extern asmlinkage long sys_kexec_load(unsigned long entry,
 					struct kexec_segment __user *segments,
 					unsigned long flags);
 extern int kernel_kexec(void);
-extern int kexec_add_buffer(struct kimage *image, char *buffer,
-			    unsigned long bufsz, unsigned long memsz,
-			    unsigned long buf_align, unsigned long buf_min,
-			    unsigned long buf_max, bool top_down,
-			    unsigned long *load_addr);
 extern struct page *kimage_alloc_control_pages(struct kimage *image,
 						unsigned int order);
 extern int kexec_load_purgatory(struct kimage *image, unsigned long min,
@@ -259,12 +283,6 @@ phys_addr_t paddr_vmcoreinfo_note(void);
 	vmcoreinfo_append_str("NUMBER(%s)=%ld\n", #name, (long)name)
 #define VMCOREINFO_CONFIG(name) \
 	vmcoreinfo_append_str("CONFIG_%s=y\n", #name)
-#define VMCOREINFO_PAGE_OFFSET(value) \
-	vmcoreinfo_append_str("PAGE_OFFSET=%lx\n", (unsigned long)value)
-#define VMCOREINFO_VMALLOC_START(value) \
-	vmcoreinfo_append_str("VMALLOC_START=%lx\n", (unsigned long)value)
-#define VMCOREINFO_VMEMMAP_START(value) \
-	vmcoreinfo_append_str("VMEMMAP_START=%lx\n", (unsigned long)value)
 
 extern struct kimage *kexec_image;
 extern struct kimage *kexec_crash_image;
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 81ba3ba641ba..1c5190dab2c1 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -438,6 +438,9 @@ struct kvm {
 	pr_info("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
 #define kvm_debug(fmt, ...) \
 	pr_debug("kvm [%i]: " fmt, task_pid_nr(current), ## __VA_ARGS__)
+#define kvm_debug_ratelimited(fmt, ...) \
+	pr_debug_ratelimited("kvm [%i]: " fmt, task_pid_nr(current), \
+			     ## __VA_ARGS__)
 #define kvm_pr_unimpl(fmt, ...) \
 	pr_err_ratelimited("kvm [%i]: " fmt, \
 			   task_tgid_nr(current), ## __VA_ARGS__)
@@ -449,6 +452,9 @@ struct kvm {
 
 #define vcpu_debug(vcpu, fmt, ...)					\
 	kvm_debug("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
+#define vcpu_debug_ratelimited(vcpu, fmt, ...)				\
+	kvm_debug_ratelimited("vcpu%i " fmt, (vcpu)->vcpu_id,           \
+			      ## __VA_ARGS__)
 #define vcpu_err(vcpu, fmt, ...)					\
 	kvm_err("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
 
@@ -1108,6 +1114,10 @@ static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu)
 
 extern bool kvm_rebooting;
 
+extern unsigned int halt_poll_ns;
+extern unsigned int halt_poll_ns_grow;
+extern unsigned int halt_poll_ns_shrink;
+
 struct kvm_device {
 	struct kvm_device_ops *ops;
 	struct kvm *kvm;
diff --git a/include/linux/leds.h b/include/linux/leds.h
index ddfcb2df3656..569cb531094c 100644
--- a/include/linux/leds.h
+++ b/include/linux/leds.h
@@ -42,16 +42,20 @@ struct led_classdev {
 #define LED_UNREGISTERING	(1 << 1)
 	/* Upper 16 bits reflect control information */
 #define LED_CORE_SUSPENDRESUME	(1 << 16)
-#define LED_BLINK_SW		(1 << 17)
-#define LED_BLINK_ONESHOT	(1 << 18)
-#define LED_BLINK_ONESHOT_STOP	(1 << 19)
-#define LED_BLINK_INVERT	(1 << 20)
-#define LED_BLINK_BRIGHTNESS_CHANGE (1 << 21)
-#define LED_BLINK_DISABLE	(1 << 22)
-#define LED_SYSFS_DISABLE	(1 << 23)
-#define LED_DEV_CAP_FLASH	(1 << 24)
-#define LED_HW_PLUGGABLE	(1 << 25)
-#define LED_PANIC_INDICATOR	(1 << 26)
+#define LED_SYSFS_DISABLE	(1 << 17)
+#define LED_DEV_CAP_FLASH	(1 << 18)
+#define LED_HW_PLUGGABLE	(1 << 19)
+#define LED_PANIC_INDICATOR	(1 << 20)
+
+	/* set_brightness_work / blink_timer flags, atomic, private. */
+	unsigned long		work_flags;
+
+#define LED_BLINK_SW			0
+#define LED_BLINK_ONESHOT		1
+#define LED_BLINK_ONESHOT_STOP		2
+#define LED_BLINK_INVERT		3
+#define LED_BLINK_BRIGHTNESS_CHANGE 	4
+#define LED_BLINK_DISABLE		5
 
 	/* Set LED brightness level
 	 * Must not sleep. Use brightness_set_blocking for drivers
@@ -89,6 +93,7 @@ struct led_classdev {
 	unsigned long		 blink_delay_on, blink_delay_off;
 	struct timer_list	 blink_timer;
 	int			 blink_brightness;
+	int			 new_blink_brightness;
 	void			(*flash_resume)(struct led_classdev *led_cdev);
 
 	struct work_struct	set_brightness_work;
diff --git a/include/linux/libata.h b/include/linux/libata.h
index 616eef4d81ea..c170be548b7f 100644
--- a/include/linux/libata.h
+++ b/include/linux/libata.h
@@ -166,6 +166,8 @@ enum {
 	ATA_DFLAG_NO_UNLOAD	= (1 << 17), /* device doesn't support unload */
 	ATA_DFLAG_UNLOCK_HPA	= (1 << 18), /* unlock HPA */
 	ATA_DFLAG_NCQ_SEND_RECV = (1 << 19), /* device supports NCQ SEND and RECV */
+	ATA_DFLAG_NCQ_PRIO	= (1 << 20), /* device supports NCQ priority */
+	ATA_DFLAG_NCQ_PRIO_ENABLE = (1 << 21), /* Priority cmds sent to dev */
 	ATA_DFLAG_INIT_MASK	= (1 << 24) - 1,
 
 	ATA_DFLAG_DETACH	= (1 << 24),
@@ -342,7 +344,9 @@ enum {
 	ATA_SHIFT_PIO		= 0,
 	ATA_SHIFT_MWDMA		= ATA_SHIFT_PIO + ATA_NR_PIO_MODES,
 	ATA_SHIFT_UDMA		= ATA_SHIFT_MWDMA + ATA_NR_MWDMA_MODES,
+	ATA_SHIFT_PRIO		= 6,
 
+	ATA_PRIO_HIGH		= 2,
 	/* size of buffer to pad xfers ending on unaligned boundaries */
 	ATA_DMA_PAD_SZ		= 4,
 
@@ -542,6 +546,7 @@ typedef void (*ata_postreset_fn_t)(struct ata_link *link, unsigned int *classes)
 
 extern struct device_attribute dev_attr_link_power_management_policy;
 extern struct device_attribute dev_attr_unload_heads;
+extern struct device_attribute dev_attr_ncq_prio_enable;
 extern struct device_attribute dev_attr_em_message_type;
 extern struct device_attribute dev_attr_em_message;
 extern struct device_attribute dev_attr_sw_activity;
diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h
index d190786e4ad8..7c273bbc5351 100644
--- a/include/linux/lightnvm.h
+++ b/include/linux/lightnvm.h
@@ -47,6 +47,7 @@ struct ppa_addr {
 struct nvm_rq;
 struct nvm_id;
 struct nvm_dev;
+struct nvm_tgt_dev;
 
 typedef int (nvm_l2p_update_fn)(u64, u32, __le64 *, void *);
 typedef int (nvm_id_fn)(struct nvm_dev *, struct nvm_id *);
@@ -107,6 +108,8 @@ enum {
 	NVM_RSP_NOT_CHANGEABLE	= 0x1,
 	NVM_RSP_ERR_FAILWRITE	= 0x40ff,
 	NVM_RSP_ERR_EMPTYPAGE	= 0x42ff,
+	NVM_RSP_ERR_FAILECC	= 0x4281,
+	NVM_RSP_WARN_HIGHECC	= 0x4700,
 
 	/* Device opcodes */
 	NVM_OP_HBREAD		= 0x02,
@@ -208,7 +211,7 @@ struct nvm_id {
 
 struct nvm_target {
 	struct list_head list;
-	struct nvm_dev *dev;
+	struct nvm_tgt_dev *dev;
 	struct nvm_tgt_type *type;
 	struct gendisk *disk;
 };
@@ -228,7 +231,7 @@ typedef void (nvm_end_io_fn)(struct nvm_rq *);
 
 struct nvm_rq {
 	struct nvm_tgt_instance *ins;
-	struct nvm_dev *dev;
+	struct nvm_tgt_dev *dev;
 
 	struct bio *bio;
 
@@ -263,35 +266,12 @@ static inline void *nvm_rq_to_pdu(struct nvm_rq *rqdata)
 	return rqdata + 1;
 }
 
-struct nvm_block;
-
-struct nvm_lun {
-	int id;
-
-	int lun_id;
-	int chnl_id;
-
-	spinlock_t lock;
-
-	unsigned int nr_free_blocks;	/* Number of unused blocks */
-	struct nvm_block *blocks;
-};
-
 enum {
 	NVM_BLK_ST_FREE =	0x1,	/* Free block */
 	NVM_BLK_ST_TGT =	0x2,	/* Block in use by target */
 	NVM_BLK_ST_BAD =	0x8,	/* Bad block */
 };
 
-struct nvm_block {
-	struct list_head list;
-	struct nvm_lun *lun;
-	unsigned long id;
-
-	void *priv;
-	int state;
-};
-
 /* system block cpu representation */
 struct nvm_sb_info {
 	unsigned long		seqnr;
@@ -301,22 +281,12 @@ struct nvm_sb_info {
 	struct ppa_addr		fs_ppa;
 };
 
-struct nvm_dev {
-	struct nvm_dev_ops *ops;
-
-	struct list_head devices;
-
-	/* Media manager */
-	struct nvmm_type *mt;
-	void *mp;
-
-	/* System blocks */
-	struct nvm_sb_info sb;
-
-	/* Device information */
+/* Device generic information */
+struct nvm_geo {
 	int nr_chnls;
+	int nr_luns;
+	int luns_per_chnl; /* -1 if channels are not symmetric */
 	int nr_planes;
-	int luns_per_chnl;
 	int sec_per_pg; /* only sectors for a single page */
 	int pgs_per_blk;
 	int blks_per_lun;
@@ -336,14 +306,44 @@ struct nvm_dev {
 	int sec_per_pl; /* all sectors across planes */
 	int sec_per_blk;
 	int sec_per_lun;
+};
+
+struct nvm_tgt_dev {
+	/* Device information */
+	struct nvm_geo geo;
+
+	/* Base ppas for target LUNs */
+	struct ppa_addr *luns;
+
+	sector_t total_secs;
+
+	struct nvm_id identity;
+	struct request_queue *q;
+
+	struct nvm_dev *parent;
+	void *map;
+};
+
+struct nvm_dev {
+	struct nvm_dev_ops *ops;
+
+	struct list_head devices;
+
+	/* Media manager */
+	struct nvmm_type *mt;
+	void *mp;
+
+	/* System blocks */
+	struct nvm_sb_info sb;
+
+	/* Device information */
+	struct nvm_geo geo;
 
 	/* lower page table */
 	int lps_per_blk;
 	int *lptbl;
 
-	unsigned long total_blocks;
 	unsigned long total_secs;
-	int nr_luns;
 
 	unsigned long *lun_map;
 	void *dma_pool;
@@ -352,26 +352,57 @@ struct nvm_dev {
 
 	/* Backend device */
 	struct request_queue *q;
-	struct device dev;
-	struct device *parent_dev;
 	char name[DISK_NAME_LEN];
 	void *private_data;
 
+	void *rmap;
+
 	struct mutex mlock;
 	spinlock_t lock;
 };
 
+static inline struct ppa_addr linear_to_generic_addr(struct nvm_geo *geo,
+						     u64 pba)
+{
+	struct ppa_addr l;
+	int secs, pgs, blks, luns;
+	sector_t ppa = pba;
+
+	l.ppa = 0;
+
+	div_u64_rem(ppa, geo->sec_per_pg, &secs);
+	l.g.sec = secs;
+
+	sector_div(ppa, geo->sec_per_pg);
+	div_u64_rem(ppa, geo->pgs_per_blk, &pgs);
+	l.g.pg = pgs;
+
+	sector_div(ppa, geo->pgs_per_blk);
+	div_u64_rem(ppa, geo->blks_per_lun, &blks);
+	l.g.blk = blks;
+
+	sector_div(ppa, geo->blks_per_lun);
+	div_u64_rem(ppa, geo->luns_per_chnl, &luns);
+	l.g.lun = luns;
+
+	sector_div(ppa, geo->luns_per_chnl);
+	l.g.ch = ppa;
+
+	return l;
+}
+
 static inline struct ppa_addr generic_to_dev_addr(struct nvm_dev *dev,
 						struct ppa_addr r)
 {
+	struct nvm_geo *geo = &dev->geo;
 	struct ppa_addr l;
 
-	l.ppa = ((u64)r.g.blk) << dev->ppaf.blk_offset;
-	l.ppa |= ((u64)r.g.pg) << dev->ppaf.pg_offset;
-	l.ppa |= ((u64)r.g.sec) << dev->ppaf.sect_offset;
-	l.ppa |= ((u64)r.g.pl) << dev->ppaf.pln_offset;
-	l.ppa |= ((u64)r.g.lun) << dev->ppaf.lun_offset;
-	l.ppa |= ((u64)r.g.ch) << dev->ppaf.ch_offset;
+	l.ppa = ((u64)r.g.blk) << geo->ppaf.blk_offset;
+	l.ppa |= ((u64)r.g.pg) << geo->ppaf.pg_offset;
+	l.ppa |= ((u64)r.g.sec) << geo->ppaf.sect_offset;
+	l.ppa |= ((u64)r.g.pl) << geo->ppaf.pln_offset;
+	l.ppa |= ((u64)r.g.lun) << geo->ppaf.lun_offset;
+	l.ppa |= ((u64)r.g.ch) << geo->ppaf.ch_offset;
 
 	return l;
 }
@@ -379,24 +410,25 @@ static inline struct ppa_addr generic_to_dev_addr(struct nvm_dev *dev,
 static inline struct ppa_addr dev_to_generic_addr(struct nvm_dev *dev,
 						struct ppa_addr r)
 {
+	struct nvm_geo *geo = &dev->geo;
 	struct ppa_addr l;
 
 	l.ppa = 0;
 	/*
 	 * (r.ppa << X offset) & X len bitmask. X eq. blk, pg, etc.
 	 */
-	l.g.blk = (r.ppa >> dev->ppaf.blk_offset) &
-					(((1 << dev->ppaf.blk_len) - 1));
-	l.g.pg |= (r.ppa >> dev->ppaf.pg_offset) &
-					(((1 << dev->ppaf.pg_len) - 1));
-	l.g.sec |= (r.ppa >> dev->ppaf.sect_offset) &
-					(((1 << dev->ppaf.sect_len) - 1));
-	l.g.pl |= (r.ppa >> dev->ppaf.pln_offset) &
-					(((1 << dev->ppaf.pln_len) - 1));
-	l.g.lun |= (r.ppa >> dev->ppaf.lun_offset) &
-					(((1 << dev->ppaf.lun_len) - 1));
-	l.g.ch |= (r.ppa >> dev->ppaf.ch_offset) &
-					(((1 << dev->ppaf.ch_len) - 1));
+	l.g.blk = (r.ppa >> geo->ppaf.blk_offset) &
+					(((1 << geo->ppaf.blk_len) - 1));
+	l.g.pg |= (r.ppa >> geo->ppaf.pg_offset) &
+					(((1 << geo->ppaf.pg_len) - 1));
+	l.g.sec |= (r.ppa >> geo->ppaf.sect_offset) &
+					(((1 << geo->ppaf.sect_len) - 1));
+	l.g.pl |= (r.ppa >> geo->ppaf.pln_offset) &
+					(((1 << geo->ppaf.pln_len) - 1));
+	l.g.lun |= (r.ppa >> geo->ppaf.lun_offset) &
+					(((1 << geo->ppaf.lun_len) - 1));
+	l.g.ch |= (r.ppa >> geo->ppaf.ch_offset) &
+					(((1 << geo->ppaf.ch_len) - 1));
 
 	return l;
 }
@@ -411,18 +443,13 @@ static inline void ppa_set_empty(struct ppa_addr *ppa_addr)
 	ppa_addr->ppa = ADDR_EMPTY;
 }
 
-static inline struct ppa_addr block_to_ppa(struct nvm_dev *dev,
-							struct nvm_block *blk)
+static inline int ppa_cmp_blk(struct ppa_addr ppa1, struct ppa_addr ppa2)
 {
-	struct ppa_addr ppa;
-	struct nvm_lun *lun = blk->lun;
-
-	ppa.ppa = 0;
-	ppa.g.blk = blk->id % dev->blks_per_lun;
-	ppa.g.lun = lun->lun_id;
-	ppa.g.ch = lun->chnl_id;
+	if (ppa_empty(ppa1) || ppa_empty(ppa2))
+		return 0;
 
-	return ppa;
+	return ((ppa1.g.ch == ppa2.g.ch) && (ppa1.g.lun == ppa2.g.lun) &&
+					(ppa1.g.blk == ppa2.g.blk));
 }
 
 static inline int ppa_to_slc(struct nvm_dev *dev, int slc_pg)
@@ -432,7 +459,7 @@ static inline int ppa_to_slc(struct nvm_dev *dev, int slc_pg)
 
 typedef blk_qc_t (nvm_tgt_make_rq_fn)(struct request_queue *, struct bio *);
 typedef sector_t (nvm_tgt_capacity_fn)(void *);
-typedef void *(nvm_tgt_init_fn)(struct nvm_dev *, struct gendisk *, int, int);
+typedef void *(nvm_tgt_init_fn)(struct nvm_tgt_dev *, struct gendisk *);
 typedef void (nvm_tgt_exit_fn)(void *);
 
 struct nvm_tgt_type {
@@ -465,23 +492,18 @@ typedef void (nvmm_unregister_fn)(struct nvm_dev *);
 
 typedef int (nvmm_create_tgt_fn)(struct nvm_dev *, struct nvm_ioctl_create *);
 typedef int (nvmm_remove_tgt_fn)(struct nvm_dev *, struct nvm_ioctl_remove *);
-typedef struct nvm_block *(nvmm_get_blk_fn)(struct nvm_dev *,
-					      struct nvm_lun *, unsigned long);
-typedef void (nvmm_put_blk_fn)(struct nvm_dev *, struct nvm_block *);
-typedef int (nvmm_open_blk_fn)(struct nvm_dev *, struct nvm_block *);
-typedef int (nvmm_close_blk_fn)(struct nvm_dev *, struct nvm_block *);
-typedef void (nvmm_flush_blk_fn)(struct nvm_dev *, struct nvm_block *);
-typedef int (nvmm_submit_io_fn)(struct nvm_dev *, struct nvm_rq *);
-typedef int (nvmm_erase_blk_fn)(struct nvm_dev *, struct nvm_block *,
-								unsigned long);
-typedef void (nvmm_mark_blk_fn)(struct nvm_dev *, struct ppa_addr, int);
-typedef struct nvm_lun *(nvmm_get_lun_fn)(struct nvm_dev *, int);
-typedef int (nvmm_reserve_lun)(struct nvm_dev *, int);
-typedef void (nvmm_release_lun)(struct nvm_dev *, int);
-typedef void (nvmm_lun_info_print_fn)(struct nvm_dev *);
-
+typedef int (nvmm_submit_io_fn)(struct nvm_tgt_dev *, struct nvm_rq *);
+typedef int (nvmm_erase_blk_fn)(struct nvm_tgt_dev *, struct ppa_addr *, int);
 typedef int (nvmm_get_area_fn)(struct nvm_dev *, sector_t *, sector_t);
 typedef void (nvmm_put_area_fn)(struct nvm_dev *, sector_t);
+typedef struct ppa_addr (nvmm_trans_ppa_fn)(struct nvm_tgt_dev *,
+					    struct ppa_addr, int);
+typedef void (nvmm_part_to_tgt_fn)(struct nvm_dev *, sector_t*, int);
+
+enum {
+	TRANS_TGT_TO_DEV =	0x0,
+	TRANS_DEV_TO_TGT =	0x1,
+};
 
 struct nvmm_type {
 	const char *name;
@@ -493,54 +515,41 @@ struct nvmm_type {
 	nvmm_create_tgt_fn *create_tgt;
 	nvmm_remove_tgt_fn *remove_tgt;
 
-	/* Block administration callbacks */
-	nvmm_get_blk_fn *get_blk;
-	nvmm_put_blk_fn *put_blk;
-	nvmm_open_blk_fn *open_blk;
-	nvmm_close_blk_fn *close_blk;
-	nvmm_flush_blk_fn *flush_blk;
-
 	nvmm_submit_io_fn *submit_io;
 	nvmm_erase_blk_fn *erase_blk;
 
-	/* Bad block mgmt */
-	nvmm_mark_blk_fn *mark_blk;
-
-	/* Configuration management */
-	nvmm_get_lun_fn *get_lun;
-	nvmm_reserve_lun *reserve_lun;
-	nvmm_release_lun *release_lun;
-
-	/* Statistics */
-	nvmm_lun_info_print_fn *lun_info_print;
-
 	nvmm_get_area_fn *get_area;
 	nvmm_put_area_fn *put_area;
 
+	nvmm_trans_ppa_fn *trans_ppa;
+	nvmm_part_to_tgt_fn *part_to_tgt;
+
 	struct list_head list;
 };
 
 extern int nvm_register_mgr(struct nvmm_type *);
 extern void nvm_unregister_mgr(struct nvmm_type *);
 
-extern struct nvm_block *nvm_get_blk(struct nvm_dev *, struct nvm_lun *,
-								unsigned long);
-extern void nvm_put_blk(struct nvm_dev *, struct nvm_block *);
-
 extern struct nvm_dev *nvm_alloc_dev(int);
 extern int nvm_register(struct nvm_dev *);
 extern void nvm_unregister(struct nvm_dev *);
 
-void nvm_mark_blk(struct nvm_dev *dev, struct ppa_addr ppa, int type);
-
-extern int nvm_submit_io(struct nvm_dev *, struct nvm_rq *);
+extern int nvm_set_bb_tbl(struct nvm_dev *, struct ppa_addr *, int, int);
+extern int nvm_set_tgt_bb_tbl(struct nvm_tgt_dev *, struct ppa_addr *,
+			      int, int);
+extern int nvm_max_phys_sects(struct nvm_tgt_dev *);
+extern int nvm_submit_io(struct nvm_tgt_dev *, struct nvm_rq *);
 extern void nvm_generic_to_addr_mode(struct nvm_dev *, struct nvm_rq *);
 extern void nvm_addr_to_generic_mode(struct nvm_dev *, struct nvm_rq *);
 extern int nvm_set_rqd_ppalist(struct nvm_dev *, struct nvm_rq *,
 					const struct ppa_addr *, int, int);
 extern void nvm_free_rqd_ppalist(struct nvm_dev *, struct nvm_rq *);
-extern int nvm_erase_ppa(struct nvm_dev *, struct ppa_addr *, int);
-extern int nvm_erase_blk(struct nvm_dev *, struct nvm_block *);
+extern int nvm_erase_ppa(struct nvm_dev *, struct ppa_addr *, int, int);
+extern int nvm_erase_blk(struct nvm_tgt_dev *, struct ppa_addr *, int);
+extern int nvm_get_l2p_tbl(struct nvm_tgt_dev *, u64, u32, nvm_l2p_update_fn *,
+			   void *);
+extern int nvm_get_area(struct nvm_tgt_dev *, sector_t *, sector_t);
+extern void nvm_put_area(struct nvm_tgt_dev *, sector_t);
 extern void nvm_end_io(struct nvm_rq *, int);
 extern int nvm_submit_ppa(struct nvm_dev *, struct ppa_addr *, int, int, int,
 								void *, int);
@@ -548,6 +557,7 @@ extern int nvm_submit_ppa_list(struct nvm_dev *, struct ppa_addr *, int, int,
 							int, void *, int);
 extern int nvm_bb_tbl_fold(struct nvm_dev *, u8 *, int);
 extern int nvm_get_bb_tbl(struct nvm_dev *, struct ppa_addr, u8 *);
+extern int nvm_get_tgt_bb_tbl(struct nvm_tgt_dev *, struct ppa_addr, u8 *);
 
 /* sysblk.c */
 #define NVM_SYSBLK_MAGIC 0x4E564D53 /* "NVMS" */
@@ -569,10 +579,10 @@ extern int nvm_init_sysblock(struct nvm_dev *, struct nvm_sb_info *);
 
 extern int nvm_dev_factory(struct nvm_dev *, int flags);
 
-#define nvm_for_each_lun_ppa(dev, ppa, chid, lunid)			\
-	for ((chid) = 0, (ppa).ppa = 0; (chid) < (dev)->nr_chnls;	\
+#define nvm_for_each_lun_ppa(geo, ppa, chid, lunid)			\
+	for ((chid) = 0, (ppa).ppa = 0; (chid) < (geo)->nr_chnls;	\
 					(chid)++, (ppa).g.ch = (chid))	\
-		for ((lunid) = 0; (lunid) < (dev)->luns_per_chnl;	\
+		for ((lunid) = 0; (lunid) < (geo)->luns_per_chnl;	\
 					(lunid)++, (ppa).g.lun = (lunid))
 
 #else /* CONFIG_NVM */
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
index c1458fede1f9..1e327bb80838 100644
--- a/include/linux/lockdep.h
+++ b/include/linux/lockdep.h
@@ -338,9 +338,18 @@ extern void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 extern void lock_release(struct lockdep_map *lock, int nested,
 			 unsigned long ip);
 
-#define lockdep_is_held(lock)	lock_is_held(&(lock)->dep_map)
+/*
+ * Same "read" as for lock_acquire(), except -1 means any.
+ */
+extern int lock_is_held_type(struct lockdep_map *lock, int read);
+
+static inline int lock_is_held(struct lockdep_map *lock)
+{
+	return lock_is_held_type(lock, -1);
+}
 
-extern int lock_is_held(struct lockdep_map *lock);
+#define lockdep_is_held(lock)		lock_is_held(&(lock)->dep_map)
+#define lockdep_is_held_type(lock, r)	lock_is_held_type(&(lock)->dep_map, (r))
 
 extern void lock_set_class(struct lockdep_map *lock, const char *name,
 			   struct lock_class_key *key, unsigned int subclass,
@@ -372,6 +381,14 @@ extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie);
 		WARN_ON(debug_locks && !lockdep_is_held(l));	\
 	} while (0)
 
+#define lockdep_assert_held_exclusive(l)	do {			\
+		WARN_ON(debug_locks && !lockdep_is_held_type(l, 0));	\
+	} while (0)
+
+#define lockdep_assert_held_read(l)	do {				\
+		WARN_ON(debug_locks && !lockdep_is_held_type(l, 1));	\
+	} while (0)
+
 #define lockdep_assert_held_once(l)	do {				\
 		WARN_ON_ONCE(debug_locks && !lockdep_is_held(l));	\
 	} while (0)
@@ -428,7 +445,11 @@ struct lock_class_key { };
 
 #define lockdep_depth(tsk)	(0)
 
+#define lockdep_is_held_type(l, r)		(1)
+
 #define lockdep_assert_held(l)			do { (void)(l); } while (0)
+#define lockdep_assert_held_exclusive(l)	do { (void)(l); } while (0)
+#define lockdep_assert_held_read(l)		do { (void)(l); } while (0)
 #define lockdep_assert_held_once(l)		do { (void)(l); } while (0)
 
 #define lockdep_recursing(tsk)			(0)
diff --git a/include/linux/mdev.h b/include/linux/mdev.h
new file mode 100644
index 000000000000..ec819e9a115a
--- /dev/null
+++ b/include/linux/mdev.h
@@ -0,0 +1,168 @@
+/*
+ * Mediated device definition
+ *
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ *     Author: Neo Jia <cjia@nvidia.com>
+ *             Kirti Wankhede <kwankhede@nvidia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef MDEV_H
+#define MDEV_H
+
+/* Parent device */
+struct parent_device {
+	struct device		*dev;
+	const struct parent_ops	*ops;
+
+	/* internal */
+	struct kref		ref;
+	struct mutex		lock;
+	struct list_head	next;
+	struct kset		*mdev_types_kset;
+	struct list_head	type_list;
+};
+
+/* Mediated device */
+struct mdev_device {
+	struct device		dev;
+	struct parent_device	*parent;
+	uuid_le			uuid;
+	void			*driver_data;
+
+	/* internal */
+	struct kref		ref;
+	struct list_head	next;
+	struct kobject		*type_kobj;
+};
+
+/**
+ * struct parent_ops - Structure to be registered for each parent device to
+ * register the device to mdev module.
+ *
+ * @owner:		The module owner.
+ * @dev_attr_groups:	Attributes of the parent device.
+ * @mdev_attr_groups:	Attributes of the mediated device.
+ * @supported_type_groups: Attributes to define supported types. It is mandatory
+ *			to provide supported types.
+ * @create:		Called to allocate basic resources in parent device's
+ *			driver for a particular mediated device. It is
+ *			mandatory to provide create ops.
+ *			@kobj: kobject of type for which 'create' is called.
+ *			@mdev: mdev_device structure on of mediated device
+ *			      that is being created
+ *			Returns integer: success (0) or error (< 0)
+ * @remove:		Called to free resources in parent device's driver for a
+ *			a mediated device. It is mandatory to provide 'remove'
+ *			ops.
+ *			@mdev: mdev_device device structure which is being
+ *			       destroyed
+ *			Returns integer: success (0) or error (< 0)
+ * @open:		Open mediated device.
+ *			@mdev: mediated device.
+ *			Returns integer: success (0) or error (< 0)
+ * @release:		release mediated device
+ *			@mdev: mediated device.
+ * @read:		Read emulation callback
+ *			@mdev: mediated device structure
+ *			@buf: read buffer
+ *			@count: number of bytes to read
+ *			@ppos: address.
+ *			Retuns number on bytes read on success or error.
+ * @write:		Write emulation callback
+ *			@mdev: mediated device structure
+ *			@buf: write buffer
+ *			@count: number of bytes to be written
+ *			@ppos: address.
+ *			Retuns number on bytes written on success or error.
+ * @ioctl:		IOCTL callback
+ *			@mdev: mediated device structure
+ *			@cmd: ioctl command
+ *			@arg: arguments to ioctl
+ * @mmap:		mmap callback
+ *			@mdev: mediated device structure
+ *			@vma: vma structure
+ * Parent device that support mediated device should be registered with mdev
+ * module with parent_ops structure.
+ **/
+
+struct parent_ops {
+	struct module   *owner;
+	const struct attribute_group **dev_attr_groups;
+	const struct attribute_group **mdev_attr_groups;
+	struct attribute_group **supported_type_groups;
+
+	int     (*create)(struct kobject *kobj, struct mdev_device *mdev);
+	int     (*remove)(struct mdev_device *mdev);
+	int     (*open)(struct mdev_device *mdev);
+	void    (*release)(struct mdev_device *mdev);
+	ssize_t (*read)(struct mdev_device *mdev, char __user *buf,
+			size_t count, loff_t *ppos);
+	ssize_t (*write)(struct mdev_device *mdev, const char __user *buf,
+			 size_t count, loff_t *ppos);
+	ssize_t (*ioctl)(struct mdev_device *mdev, unsigned int cmd,
+			 unsigned long arg);
+	int	(*mmap)(struct mdev_device *mdev, struct vm_area_struct *vma);
+};
+
+/* interface for exporting mdev supported type attributes */
+struct mdev_type_attribute {
+	struct attribute attr;
+	ssize_t (*show)(struct kobject *kobj, struct device *dev, char *buf);
+	ssize_t (*store)(struct kobject *kobj, struct device *dev,
+			 const char *buf, size_t count);
+};
+
+#define MDEV_TYPE_ATTR(_name, _mode, _show, _store)		\
+struct mdev_type_attribute mdev_type_attr_##_name =		\
+	__ATTR(_name, _mode, _show, _store)
+#define MDEV_TYPE_ATTR_RW(_name) \
+	struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_RW(_name)
+#define MDEV_TYPE_ATTR_RO(_name) \
+	struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_RO(_name)
+#define MDEV_TYPE_ATTR_WO(_name) \
+	struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_WO(_name)
+
+/**
+ * struct mdev_driver - Mediated device driver
+ * @name: driver name
+ * @probe: called when new device created
+ * @remove: called when device removed
+ * @driver: device driver structure
+ *
+ **/
+struct mdev_driver {
+	const char *name;
+	int  (*probe)(struct device *dev);
+	void (*remove)(struct device *dev);
+	struct device_driver driver;
+};
+
+#define to_mdev_driver(drv)	container_of(drv, struct mdev_driver, driver)
+#define to_mdev_device(dev)	container_of(dev, struct mdev_device, dev)
+
+static inline void *mdev_get_drvdata(struct mdev_device *mdev)
+{
+	return mdev->driver_data;
+}
+
+static inline void mdev_set_drvdata(struct mdev_device *mdev, void *data)
+{
+	mdev->driver_data = data;
+}
+
+extern struct bus_type mdev_bus_type;
+
+#define dev_is_mdev(d) ((d)->bus == &mdev_bus_type)
+
+extern int  mdev_register_device(struct device *dev,
+				 const struct parent_ops *ops);
+extern void mdev_unregister_device(struct device *dev);
+
+extern int  mdev_register_driver(struct mdev_driver *drv, struct module *owner);
+extern void mdev_unregister_driver(struct mdev_driver *drv);
+
+#endif /* MDEV_H */
diff --git a/include/linux/mei_cl_bus.h b/include/linux/mei_cl_bus.h
index e746919530f5..a0d274fe08f1 100644
--- a/include/linux/mei_cl_bus.h
+++ b/include/linux/mei_cl_bus.h
@@ -8,8 +8,7 @@
 struct mei_cl_device;
 struct mei_device;
 
-typedef void (*mei_cldev_event_cb_t)(struct mei_cl_device *cldev,
-				     u32 events, void *context);
+typedef void (*mei_cldev_cb_t)(struct mei_cl_device *cldev);
 
 /**
  * struct mei_cl_device - MEI device handle
@@ -24,12 +23,12 @@ typedef void (*mei_cldev_event_cb_t)(struct mei_cl_device *cldev,
  * @me_cl: me client
  * @cl: mei client
  * @name: device name
- * @event_work: async work to execute event callback
- * @event_cb: Drivers register this callback to get asynchronous ME
- *	events (e.g. Rx buffer pending) notifications.
- * @event_context: event callback run context
- * @events_mask: Events bit mask requested by driver.
- * @events: Events bitmask sent to the driver.
+ * @rx_work: async work to execute Rx event callback
+ * @rx_cb: Drivers register this callback to get asynchronous ME
+ *	Rx buffer pending notifications.
+ * @notif_work: async work to execute FW notif event callback
+ * @notif_cb: Drivers register this callback to get asynchronous ME
+ *	FW notification pending notifications.
  *
  * @do_match: wheather device can be matched with a driver
  * @is_added: device is already scanned
@@ -44,11 +43,10 @@ struct mei_cl_device {
 	struct mei_cl *cl;
 	char name[MEI_CL_NAME_SIZE];
 
-	struct work_struct event_work;
-	mei_cldev_event_cb_t event_cb;
-	void *event_context;
-	unsigned long events_mask;
-	unsigned long events;
+	struct work_struct rx_work;
+	mei_cldev_cb_t rx_cb;
+	struct work_struct notif_work;
+	mei_cldev_cb_t notif_cb;
 
 	unsigned int do_match:1;
 	unsigned int is_added:1;
@@ -74,16 +72,27 @@ int __mei_cldev_driver_register(struct mei_cl_driver *cldrv,
 
 void mei_cldev_driver_unregister(struct mei_cl_driver *cldrv);
 
-ssize_t mei_cldev_send(struct mei_cl_device *cldev, u8 *buf, size_t length);
-ssize_t  mei_cldev_recv(struct mei_cl_device *cldev, u8 *buf, size_t length);
+/**
+ * module_mei_cl_driver - Helper macro for registering mei cl driver
+ *
+ * @__mei_cldrv: mei_cl_driver structure
+ *
+ *  Helper macro for mei cl drivers which do not do anything special in module
+ *  init/exit, for eliminating a boilerplate code.
+ */
+#define module_mei_cl_driver(__mei_cldrv) \
+	module_driver(__mei_cldrv, \
+		      mei_cldev_driver_register,\
+		      mei_cldev_driver_unregister)
 
-int mei_cldev_register_event_cb(struct mei_cl_device *cldev,
-				unsigned long event_mask,
-				mei_cldev_event_cb_t read_cb, void *context);
+ssize_t mei_cldev_send(struct mei_cl_device *cldev, u8 *buf, size_t length);
+ssize_t mei_cldev_recv(struct mei_cl_device *cldev, u8 *buf, size_t length);
+ssize_t mei_cldev_recv_nonblock(struct mei_cl_device *cldev, u8 *buf,
+				size_t length);
 
-#define MEI_CL_EVENT_RX 0
-#define MEI_CL_EVENT_TX 1
-#define MEI_CL_EVENT_NOTIF 2
+int mei_cldev_register_rx_cb(struct mei_cl_device *cldev, mei_cldev_cb_t rx_cb);
+int mei_cldev_register_notif_cb(struct mei_cl_device *cldev,
+				mei_cldev_cb_t notif_cb);
 
 const uuid_le *mei_cldev_uuid(const struct mei_cl_device *cldev);
 u8 mei_cldev_ver(const struct mei_cl_device *cldev);
diff --git a/include/linux/mfd/cros_ec.h b/include/linux/mfd/cros_ec.h
index 76f7ef4d3a0d..f62043a75f43 100644
--- a/include/linux/mfd/cros_ec.h
+++ b/include/linux/mfd/cros_ec.h
@@ -148,6 +148,15 @@ struct cros_ec_device {
 	int event_size;
 };
 
+/**
+ * struct cros_ec_sensor_platform - ChromeOS EC sensor platform information
+ *
+ * @sensor_num: Id of the sensor, as reported by the EC.
+ */
+struct cros_ec_sensor_platform {
+	u8 sensor_num;
+};
+
 /* struct cros_ec_platform - ChromeOS EC platform information
  *
  * @ec_name: name of EC device (e.g. 'cros-ec', 'cros-pd', ...)
@@ -175,6 +184,7 @@ struct cros_ec_dev {
 	struct cros_ec_device *ec_dev;
 	struct device *dev;
 	u16 cmd_offset;
+	u32 features[2];
 };
 
 /**
diff --git a/include/linux/mfd/cros_ec_commands.h b/include/linux/mfd/cros_ec_commands.h
index 76728ff37d01..1683003603f3 100644
--- a/include/linux/mfd/cros_ec_commands.h
+++ b/include/linux/mfd/cros_ec_commands.h
@@ -713,6 +713,90 @@ struct ec_response_get_set_value {
 /* More than one command can use these structs to get/set paramters. */
 #define EC_CMD_GSV_PAUSE_IN_S5	0x0c
 
+/*****************************************************************************/
+/* List the features supported by the firmware */
+#define EC_CMD_GET_FEATURES  0x0d
+
+/* Supported features */
+enum ec_feature_code {
+	/*
+	 * This image contains a limited set of features. Another image
+	 * in RW partition may support more features.
+	 */
+	EC_FEATURE_LIMITED = 0,
+	/*
+	 * Commands for probing/reading/writing/erasing the flash in the
+	 * EC are present.
+	 */
+	EC_FEATURE_FLASH = 1,
+	/*
+	 * Can control the fan speed directly.
+	 */
+	EC_FEATURE_PWM_FAN = 2,
+	/*
+	 * Can control the intensity of the keyboard backlight.
+	 */
+	EC_FEATURE_PWM_KEYB = 3,
+	/*
+	 * Support Google lightbar, introduced on Pixel.
+	 */
+	EC_FEATURE_LIGHTBAR = 4,
+	/* Control of LEDs  */
+	EC_FEATURE_LED = 5,
+	/* Exposes an interface to control gyro and sensors.
+	 * The host goes through the EC to access these sensors.
+	 * In addition, the EC may provide composite sensors, like lid angle.
+	 */
+	EC_FEATURE_MOTION_SENSE = 6,
+	/* The keyboard is controlled by the EC */
+	EC_FEATURE_KEYB = 7,
+	/* The AP can use part of the EC flash as persistent storage. */
+	EC_FEATURE_PSTORE = 8,
+	/* The EC monitors BIOS port 80h, and can return POST codes. */
+	EC_FEATURE_PORT80 = 9,
+	/*
+	 * Thermal management: include TMP specific commands.
+	 * Higher level than direct fan control.
+	 */
+	EC_FEATURE_THERMAL = 10,
+	/* Can switch the screen backlight on/off */
+	EC_FEATURE_BKLIGHT_SWITCH = 11,
+	/* Can switch the wifi module on/off */
+	EC_FEATURE_WIFI_SWITCH = 12,
+	/* Monitor host events, through for example SMI or SCI */
+	EC_FEATURE_HOST_EVENTS = 13,
+	/* The EC exposes GPIO commands to control/monitor connected devices. */
+	EC_FEATURE_GPIO = 14,
+	/* The EC can send i2c messages to downstream devices. */
+	EC_FEATURE_I2C = 15,
+	/* Command to control charger are included */
+	EC_FEATURE_CHARGER = 16,
+	/* Simple battery support. */
+	EC_FEATURE_BATTERY = 17,
+	/*
+	 * Support Smart battery protocol
+	 * (Common Smart Battery System Interface Specification)
+	 */
+	EC_FEATURE_SMART_BATTERY = 18,
+	/* EC can dectect when the host hangs. */
+	EC_FEATURE_HANG_DETECT = 19,
+	/* Report power information, for pit only */
+	EC_FEATURE_PMU = 20,
+	/* Another Cros EC device is present downstream of this one */
+	EC_FEATURE_SUB_MCU = 21,
+	/* Support USB Power delivery (PD) commands */
+	EC_FEATURE_USB_PD = 22,
+	/* Control USB multiplexer, for audio through USB port for instance. */
+	EC_FEATURE_USB_MUX = 23,
+	/* Motion Sensor code has an internal software FIFO */
+	EC_FEATURE_MOTION_SENSE_FIFO = 24,
+};
+
+#define EC_FEATURE_MASK_0(event_code) (1UL << (event_code % 32))
+#define EC_FEATURE_MASK_1(event_code) (1UL << (event_code - 32))
+struct ec_response_get_features {
+	uint32_t flags[2];
+} __packed;
 
 /*****************************************************************************/
 /* Flash commands */
@@ -1315,6 +1399,24 @@ enum motionsense_command {
 	 */
 	MOTIONSENSE_CMD_KB_WAKE_ANGLE = 5,
 
+	/*
+	 * Returns a single sensor data.
+	 */
+	MOTIONSENSE_CMD_DATA = 6,
+
+	/*
+	 * Perform low level calibration.. On sensors that support it, ask to
+	 * do offset calibration.
+	 */
+	MOTIONSENSE_CMD_PERFORM_CALIB = 10,
+
+	/*
+	 * Sensor Offset command is a setter/getter command for the offset used
+	 * for calibration. The offsets can be calculated by the host, or via
+	 * PERFORM_CALIB command.
+	 */
+	MOTIONSENSE_CMD_SENSOR_OFFSET = 11,
+
 	/* Number of motionsense sub-commands. */
 	MOTIONSENSE_NUM_CMDS
 };
@@ -1335,12 +1437,18 @@ enum motionsensor_id {
 enum motionsensor_type {
 	MOTIONSENSE_TYPE_ACCEL = 0,
 	MOTIONSENSE_TYPE_GYRO = 1,
+	MOTIONSENSE_TYPE_MAG = 2,
+	MOTIONSENSE_TYPE_PROX = 3,
+	MOTIONSENSE_TYPE_LIGHT = 4,
+	MOTIONSENSE_TYPE_ACTIVITY = 5,
+	MOTIONSENSE_TYPE_MAX
 };
 
 /* List of motion sensor locations. */
 enum motionsensor_location {
 	MOTIONSENSE_LOC_BASE = 0,
 	MOTIONSENSE_LOC_LID = 1,
+	MOTIONSENSE_LOC_MAX,
 };
 
 /* List of motion sensor chips. */
@@ -1361,6 +1469,31 @@ enum motionsensor_chip {
  */
 #define EC_MOTION_SENSE_NO_VALUE -1
 
+#define EC_MOTION_SENSE_INVALID_CALIB_TEMP 0x8000
+
+/* Set Calibration information */
+#define MOTION_SENSE_SET_OFFSET	1
+
+struct ec_response_motion_sensor_data {
+	/* Flags for each sensor. */
+	uint8_t flags;
+	/* Sensor number the data comes from */
+	uint8_t sensor_num;
+	/* Each sensor is up to 3-axis. */
+	union {
+		int16_t             data[3];
+		struct {
+			uint16_t    rsvd;
+			uint32_t    timestamp;
+		} __packed;
+		struct {
+			uint8_t     activity; /* motionsensor_activity */
+			uint8_t     state;
+			int16_t     add_info[2];
+		};
+	};
+} __packed;
+
 struct ec_params_motion_sense {
 	uint8_t cmd;
 	union {
@@ -1378,9 +1511,37 @@ struct ec_params_motion_sense {
 			int16_t data;
 		} ec_rate, kb_wake_angle;
 
+		/* Used for MOTIONSENSE_CMD_SENSOR_OFFSET */
+		struct {
+			uint8_t sensor_num;
+
+			/*
+			 * bit 0: If set (MOTION_SENSE_SET_OFFSET), set
+			 * the calibration information in the EC.
+			 * If unset, just retrieve calibration information.
+			 */
+			uint16_t flags;
+
+			/*
+			 * Temperature at calibration, in units of 0.01 C
+			 * 0x8000: invalid / unknown.
+			 * 0x0: 0C
+			 * 0x7fff: +327.67C
+			 */
+			int16_t temp;
+
+			/*
+			 * Offset for calibration.
+			 * Unit:
+			 * Accelerometer: 1/1024 g
+			 * Gyro:          1/1024 deg/s
+			 * Compass:       1/16 uT
+			 */
+			int16_t offset[3];
+		} __packed sensor_offset;
+
 		/* Used for MOTIONSENSE_CMD_INFO. */
 		struct {
-			/* Should be element of enum motionsensor_id. */
 			uint8_t sensor_num;
 		} info;
 
@@ -1410,11 +1571,14 @@ struct ec_response_motion_sense {
 			/* Flags representing the motion sensor module. */
 			uint8_t module_flags;
 
-			/* Flags for each sensor in enum motionsensor_id. */
-			uint8_t sensor_flags[EC_MOTION_SENSOR_COUNT];
+			/* Number of sensors managed directly by the EC. */
+			uint8_t sensor_count;
 
-			/* Array of all sensor data. Each sensor is 3-axis. */
-			int16_t data[3*EC_MOTION_SENSOR_COUNT];
+			/*
+			 * Sensor data is truncated if response_max is too small
+			 * for holding all the data.
+			 */
+			struct ec_response_motion_sensor_data sensor[0];
 		} dump;
 
 		/* Used for MOTIONSENSE_CMD_INFO. */
@@ -1429,6 +1593,9 @@ struct ec_response_motion_sense {
 			uint8_t chip;
 		} info;
 
+		/* Used for MOTIONSENSE_CMD_DATA */
+		struct ec_response_motion_sensor_data data;
+
 		/*
 		 * Used for MOTIONSENSE_CMD_EC_RATE, MOTIONSENSE_CMD_SENSOR_ODR,
 		 * MOTIONSENSE_CMD_SENSOR_RANGE, and
@@ -1438,6 +1605,12 @@ struct ec_response_motion_sense {
 			/* Current value of the parameter queried. */
 			int32_t ret;
 		} ec_rate, sensor_odr, sensor_range, kb_wake_angle;
+
+		/* Used for MOTIONSENSE_CMD_SENSOR_OFFSET */
+		struct {
+			int16_t temp;
+			int16_t offset[3];
+		} sensor_offset, perform_calib;
 	};
 } __packed;
 
diff --git a/include/linux/mfd/max77620.h b/include/linux/mfd/max77620.h
index 3ca0af07fc78..ad2a9a852aea 100644
--- a/include/linux/mfd/max77620.h
+++ b/include/linux/mfd/max77620.h
@@ -180,6 +180,7 @@
 #define MAX77620_SD_CFG1_FPWM_SD_MASK		BIT(2)
 #define MAX77620_SD_CFG1_FPWM_SD_SKIP		0
 #define MAX77620_SD_CFG1_FPWM_SD_FPWM		BIT(2)
+#define MAX20024_SD_CFG1_MPOK_MASK		BIT(1)
 #define MAX77620_SD_CFG1_FSRADE_SD_MASK		BIT(0)
 #define MAX77620_SD_CFG1_FSRADE_SD_DISABLE	0
 #define MAX77620_SD_CFG1_FSRADE_SD_ENABLE	BIT(0)
@@ -187,6 +188,7 @@
 /* LDO_CNFG2 */
 #define MAX77620_LDO_POWER_MODE_MASK		0xC0
 #define MAX77620_LDO_POWER_MODE_SHIFT		6
+#define MAX20024_LDO_CFG2_MPOK_MASK		BIT(2)
 #define MAX77620_LDO_CFG2_ADE_MASK		BIT(1)
 #define MAX77620_LDO_CFG2_ADE_DISABLE		0
 #define MAX77620_LDO_CFG2_ADE_ENABLE		BIT(1)
diff --git a/include/linux/mfd/ti_am335x_tscadc.h b/include/linux/mfd/ti_am335x_tscadc.h
index 7f55b8b41032..b9a53e013bff 100644
--- a/include/linux/mfd/ti_am335x_tscadc.h
+++ b/include/linux/mfd/ti_am335x_tscadc.h
@@ -23,6 +23,8 @@
 #define REG_IRQENABLE		0x02C
 #define REG_IRQCLR		0x030
 #define REG_IRQWAKEUP		0x034
+#define REG_DMAENABLE_SET	0x038
+#define REG_DMAENABLE_CLEAR	0x03c
 #define REG_CTRL		0x040
 #define REG_ADCFSM		0x044
 #define REG_CLKDIV		0x04C
@@ -36,6 +38,7 @@
 #define REG_FIFO0THR		0xE8
 #define REG_FIFO1CNT		0xF0
 #define REG_FIFO1THR		0xF4
+#define REG_DMA1REQ		0xF8
 #define REG_FIFO0		0x100
 #define REG_FIFO1		0x200
 
@@ -126,6 +129,10 @@
 #define FIFOREAD_DATA_MASK (0xfff << 0)
 #define FIFOREAD_CHNLID_MASK (0xf << 16)
 
+/* DMA ENABLE/CLEAR Register */
+#define DMA_FIFO0		BIT(0)
+#define DMA_FIFO1		BIT(1)
+
 /* Sequencer Status */
 #define SEQ_STATUS BIT(5)
 #define CHARGE_STEP		0x11
@@ -155,6 +162,7 @@ struct ti_tscadc_dev {
 	struct device *dev;
 	struct regmap *regmap;
 	void __iomem *tscadc_base;
+	phys_addr_t tscadc_phys_base;
 	int irq;
 	int used_cells;	/* 1-2 */
 	int tsc_wires;
diff --git a/include/linux/mfd/tmio.h b/include/linux/mfd/tmio.h
index 7a26286db895..fba44abd05ba 100644
--- a/include/linux/mfd/tmio.h
+++ b/include/linux/mfd/tmio.h
@@ -100,6 +100,11 @@
 #define TMIO_MMC_SDIO_STATUS_QUIRK	(1 << 8)
 
 /*
+ * Some controllers have a 32-bit wide data port register
+ */
+#define TMIO_MMC_32BIT_DATA_PORT	(1 << 9)
+
+/*
  * Some controllers allows to set SDx actual clock
  */
 #define TMIO_MMC_CLK_ACTUAL		(1 << 10)
diff --git a/include/linux/mfd/tps65217.h b/include/linux/mfd/tps65217.h
index 4ccda8969639..3cbec4b2496a 100644
--- a/include/linux/mfd/tps65217.h
+++ b/include/linux/mfd/tps65217.h
@@ -234,12 +234,11 @@ struct tps65217_bl_pdata {
 	int dft_brightness;
 };
 
-enum tps65217_irq_type {
-	TPS65217_IRQ_PB,
-	TPS65217_IRQ_AC,
-	TPS65217_IRQ_USB,
-	TPS65217_NUM_IRQ
-};
+/* Interrupt numbers */
+#define TPS65217_IRQ_USB		0
+#define TPS65217_IRQ_AC			1
+#define TPS65217_IRQ_PB			2
+#define TPS65217_NUM_IRQ		3
 
 /**
  * struct tps65217_board - packages regulator init data
diff --git a/include/linux/miscdevice.h b/include/linux/miscdevice.h
index 722698a43d79..a426cb55dc43 100644
--- a/include/linux/miscdevice.h
+++ b/include/linux/miscdevice.h
@@ -72,6 +72,13 @@ extern int misc_register(struct miscdevice *misc);
 extern void misc_deregister(struct miscdevice *misc);
 
 /*
+ * Helper macro for drivers that don't do anything special in the initcall.
+ * This helps in eleminating of boilerplate code.
+ */
+#define builtin_misc_device(__misc_device) \
+	builtin_driver(__misc_device, misc_register)
+
+/*
  * Helper macro for drivers that don't do anything special in module init / exit
  * call. This helps in eleminating of boilerplate code.
  */
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index a5f0fbedf1e7..57bec544e20a 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -577,7 +577,7 @@ struct mlx5_ifc_per_protocol_networking_offload_caps_bits {
 	u8         self_lb_en_modifiable[0x1];
 	u8         reserved_at_9[0x2];
 	u8         max_lso_cap[0x5];
-	u8         reserved_at_10[0x2];
+	u8         multi_pkt_send_wqe[0x2];
 	u8	   wqe_inline_mode[0x2];
 	u8         rss_ind_tbl_cap[0x4];
 	u8         reg_umr_sq[0x1];
diff --git a/include/linux/mm.h b/include/linux/mm.h
index a92c8d73aeaf..4424784ac374 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -292,36 +292,23 @@ extern pgprot_t protection_map[16];
  * pgoff should be used in favour of virtual_address, if possible.
  */
 struct vm_fault {
+	struct vm_area_struct *vma;	/* Target VMA */
 	unsigned int flags;		/* FAULT_FLAG_xxx flags */
 	gfp_t gfp_mask;			/* gfp mask to be used for allocations */
 	pgoff_t pgoff;			/* Logical page offset based on vma */
-	void __user *virtual_address;	/* Faulting virtual address */
+	unsigned long address;		/* Faulting virtual address */
+	pmd_t *pmd;			/* Pointer to pmd entry matching
+					 * the 'address' */
+	pte_t orig_pte;			/* Value of PTE at the time of fault */
 
-	struct page *cow_page;		/* Handler may choose to COW */
+	struct page *cow_page;		/* Page handler may use for COW fault */
+	struct mem_cgroup *memcg;	/* Cgroup cow_page belongs to */
 	struct page *page;		/* ->fault handlers should return a
 					 * page here, unless VM_FAULT_NOPAGE
 					 * is set (which is also implied by
 					 * VM_FAULT_ERROR).
 					 */
-	void *entry;			/* ->fault handler can alternatively
-					 * return locked DAX entry. In that
-					 * case handler should return
-					 * VM_FAULT_DAX_LOCKED and fill in
-					 * entry here.
-					 */
-};
-
-/*
- * Page fault context: passes though page fault handler instead of endless list
- * of function arguments.
- */
-struct fault_env {
-	struct vm_area_struct *vma;	/* Target VMA */
-	unsigned long address;		/* Faulting virtual address */
-	unsigned int flags;		/* FAULT_FLAG_xxx flags */
-	pmd_t *pmd;			/* Pointer to pmd entry matching
-					 * the 'address'
-					 */
+	/* These three entries are valid only while holding ptl lock */
 	pte_t *pte;			/* Pointer to pte entry matching
 					 * the 'address'. NULL if the page
 					 * table hasn't been allocated.
@@ -351,7 +338,7 @@ struct vm_operations_struct {
 	int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
 	int (*pmd_fault)(struct vm_area_struct *, unsigned long address,
 						pmd_t *, unsigned int flags);
-	void (*map_pages)(struct fault_env *fe,
+	void (*map_pages)(struct vm_fault *vmf,
 			pgoff_t start_pgoff, pgoff_t end_pgoff);
 
 	/* notification that a previously read-only page is about to become
@@ -625,8 +612,10 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
 	return pte;
 }
 
-int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg,
+int alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,
 		struct page *page);
+int finish_fault(struct vm_fault *vmf);
+int finish_mkwrite_fault(struct vm_fault *vmf);
 #endif
 
 /*
@@ -1110,7 +1099,7 @@ static inline void clear_page_pfmemalloc(struct page *page)
 #define VM_FAULT_LOCKED	0x0200	/* ->fault locked the returned page */
 #define VM_FAULT_RETRY	0x0400	/* ->fault blocked, must retry */
 #define VM_FAULT_FALLBACK 0x0800	/* huge page fault failed, fall back to small */
-#define VM_FAULT_DAX_LOCKED 0x1000	/* ->fault has locked DAX entry */
+#define VM_FAULT_DONE_COW   0x1000	/* ->fault has fully handled COW */
 
 #define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */
 
@@ -1221,6 +1210,8 @@ int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
 			struct vm_area_struct *vma);
 void unmap_mapping_range(struct address_space *mapping,
 		loff_t const holebegin, loff_t const holelen, int even_cows);
+int follow_pte(struct mm_struct *mm, unsigned long address, pte_t **ptepp,
+	       spinlock_t **ptlp);
 int follow_pfn(struct vm_area_struct *vma, unsigned long address,
 	unsigned long *pfn);
 int follow_phys(struct vm_area_struct *vma, unsigned long address,
@@ -1270,19 +1261,18 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *
 		unsigned int gup_flags);
 extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
 		void *buf, int len, unsigned int gup_flags);
+extern int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
+		unsigned long addr, void *buf, int len, unsigned int gup_flags);
 
 long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
 			    unsigned long start, unsigned long nr_pages,
 			    unsigned int gup_flags, struct page **pages,
-			    struct vm_area_struct **vmas);
+			    struct vm_area_struct **vmas, int *locked);
 long get_user_pages(unsigned long start, unsigned long nr_pages,
 			    unsigned int gup_flags, struct page **pages,
 			    struct vm_area_struct **vmas);
 long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
 		    unsigned int gup_flags, struct page **pages, int *locked);
-long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
-			       unsigned long start, unsigned long nr_pages,
-			       struct page **pages, unsigned int gup_flags);
 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
 		    struct page **pages, unsigned int gup_flags);
 int get_user_pages_fast(unsigned long start, int nr_pages, int write,
@@ -2097,7 +2087,7 @@ extern void truncate_inode_pages_final(struct address_space *);
 
 /* generic vm_area_ops exported for stackable file systems */
 extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
-extern void filemap_map_pages(struct fault_env *fe,
+extern void filemap_map_pages(struct vm_fault *vmf,
 		pgoff_t start_pgoff, pgoff_t end_pgoff);
 extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf);
 
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 4a8acedf4b7d..08d947fc4c59 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -473,6 +473,7 @@ struct mm_struct {
 	 */
 	struct task_struct __rcu *owner;
 #endif
+	struct user_namespace *user_ns;
 
 	/* store ref to file /proc/<pid>/exe symlink points to */
 	struct file __rcu *exe_file;
diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h
index 73fad83acbcb..95d69d498296 100644
--- a/include/linux/mmc/card.h
+++ b/include/linux/mmc/card.h
@@ -89,6 +89,8 @@ struct mmc_ext_csd {
 	unsigned int		boot_ro_lock;		/* ro lock support */
 	bool			boot_ro_lockable;
 	bool			ffu_capable;	/* Firmware upgrade support */
+	bool			cmdq_support;	/* Command Queue supported */
+	unsigned int		cmdq_depth;	/* Command Queue depth */
 #define MMC_FIRMWARE_LEN 8
 	u8			fwrev[MMC_FIRMWARE_LEN];  /* FW version */
 	u8			raw_exception_status;	/* 54 */
@@ -207,18 +209,6 @@ struct sdio_func_tuple;
 
 #define SDIO_MAX_FUNCS		7
 
-enum mmc_blk_status {
-	MMC_BLK_SUCCESS = 0,
-	MMC_BLK_PARTIAL,
-	MMC_BLK_CMD_ERR,
-	MMC_BLK_RETRY,
-	MMC_BLK_ABORT,
-	MMC_BLK_DATA_ERR,
-	MMC_BLK_ECC_ERR,
-	MMC_BLK_NOMEDIUM,
-	MMC_BLK_NEW_REQUEST,
-};
-
 /* The number of MMC physical partitions.  These consist of:
  * boot partitions (2), general purpose partitions (4) and
  * RPMB partition (1) in MMC v4.4.
diff --git a/include/linux/mmc/core.h b/include/linux/mmc/core.h
index 2b953eb8ceae..e33cc748dcfe 100644
--- a/include/linux/mmc/core.h
+++ b/include/linux/mmc/core.h
@@ -15,6 +15,18 @@ struct request;
 struct mmc_data;
 struct mmc_request;
 
+enum mmc_blk_status {
+	MMC_BLK_SUCCESS = 0,
+	MMC_BLK_PARTIAL,
+	MMC_BLK_CMD_ERR,
+	MMC_BLK_RETRY,
+	MMC_BLK_ABORT,
+	MMC_BLK_DATA_ERR,
+	MMC_BLK_ECC_ERR,
+	MMC_BLK_NOMEDIUM,
+	MMC_BLK_NEW_REQUEST,
+};
+
 struct mmc_command {
 	u32			opcode;
 	u32			arg;
@@ -150,7 +162,8 @@ struct mmc_async_req;
 extern int mmc_stop_bkops(struct mmc_card *);
 extern int mmc_read_bkops_status(struct mmc_card *);
 extern struct mmc_async_req *mmc_start_req(struct mmc_host *,
-					   struct mmc_async_req *, int *);
+					   struct mmc_async_req *,
+					   enum mmc_blk_status *);
 extern int mmc_interrupt_hpi(struct mmc_card *);
 extern void mmc_wait_for_req(struct mmc_host *, struct mmc_request *);
 extern void mmc_wait_for_req_done(struct mmc_host *host,
@@ -163,6 +176,7 @@ extern int mmc_wait_for_app_cmd(struct mmc_host *, struct mmc_card *,
 extern void mmc_start_bkops(struct mmc_card *card, bool from_exception);
 extern int mmc_switch(struct mmc_card *, u8, u8, u8, unsigned int);
 extern int mmc_send_tuning(struct mmc_host *host, u32 opcode, int *cmd_error);
+extern int mmc_abort_tuning(struct mmc_host *host, u32 opcode);
 extern int mmc_get_ext_csd(struct mmc_card *card, u8 **new_ext_csd);
 
 #define MMC_ERASE_ARG		0x00000000
diff --git a/include/linux/mmc/dw_mmc.h b/include/linux/mmc/dw_mmc.h
index f5af2bd35e7f..15db6f83f53f 100644
--- a/include/linux/mmc/dw_mmc.h
+++ b/include/linux/mmc/dw_mmc.h
@@ -39,6 +39,12 @@ enum {
 	EVENT_DATA_ERROR,
 };
 
+enum dw_mci_cookie {
+	COOKIE_UNMAPPED,
+	COOKIE_PRE_MAPPED,	/* mapped by pre_req() of dwmmc */
+	COOKIE_MAPPED,		/* mapped by prepare_data() of dwmmc */
+};
+
 struct mmc_data;
 
 enum {
diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h
index 0b2439441cc8..8bc884121465 100644
--- a/include/linux/mmc/host.h
+++ b/include/linux/mmc/host.h
@@ -93,8 +93,7 @@ struct mmc_host_ops {
 	 */
 	void	(*post_req)(struct mmc_host *host, struct mmc_request *req,
 			    int err);
-	void	(*pre_req)(struct mmc_host *host, struct mmc_request *req,
-			   bool is_first_req);
+	void	(*pre_req)(struct mmc_host *host, struct mmc_request *req);
 	void	(*request)(struct mmc_host *host, struct mmc_request *req);
 
 	/*
@@ -173,7 +172,7 @@ struct mmc_async_req {
 	 * Check error status of completed mmc request.
 	 * Returns 0 if success otherwise non zero.
 	 */
-	int (*err_check) (struct mmc_card *, struct mmc_async_req *);
+	enum mmc_blk_status (*err_check)(struct mmc_card *, struct mmc_async_req *);
 };
 
 /**
@@ -198,14 +197,12 @@ struct mmc_slot {
  * @is_new_req		wake up reason was new request
  * @is_waiting_last_req	mmc context waiting for single running request
  * @wait		wait queue
- * @lock		lock to protect data fields
  */
 struct mmc_context_info {
 	bool			is_done_rcv;
 	bool			is_new_req;
 	bool			is_waiting_last_req;
 	wait_queue_head_t	wait;
-	spinlock_t		lock;
 };
 
 struct regulator;
@@ -495,11 +492,6 @@ static inline int mmc_host_uhs(struct mmc_host *host)
 		 MMC_CAP_UHS_DDR50);
 }
 
-static inline int mmc_host_packed_wr(struct mmc_host *host)
-{
-	return host->caps2 & MMC_CAP2_PACKED_WR;
-}
-
 static inline int mmc_card_hs(struct mmc_card *card)
 {
 	return card->host->ios.timing == MMC_TIMING_SD_HS ||
@@ -546,6 +538,11 @@ static inline void mmc_retune_recheck(struct mmc_host *host)
 		host->retune_now = 1;
 }
 
+static inline bool mmc_can_retune(struct mmc_host *host)
+{
+	return host->can_retune == 1;
+}
+
 void mmc_retune_pause(struct mmc_host *host);
 void mmc_retune_unpause(struct mmc_host *host);
 
diff --git a/include/linux/mmc/mmc.h b/include/linux/mmc/mmc.h
index c376209c70ef..672730acc705 100644
--- a/include/linux/mmc/mmc.h
+++ b/include/linux/mmc/mmc.h
@@ -84,6 +84,13 @@
 #define MMC_APP_CMD              55   /* ac   [31:16] RCA        R1  */
 #define MMC_GEN_CMD              56   /* adtc [0] RD/WR          R1  */
 
+  /* class 11 */
+#define MMC_QUE_TASK_PARAMS      44   /* ac   [20:16] task id    R1  */
+#define MMC_QUE_TASK_ADDR        45   /* ac   [31:0] data addr   R1  */
+#define MMC_EXECUTE_READ_TASK    46   /* adtc [20:16] task id    R1  */
+#define MMC_EXECUTE_WRITE_TASK   47   /* adtc [20:16] task id    R1  */
+#define MMC_CMDQ_TASK_MGMT       48   /* ac   [20:16] task id    R1b */
+
 static inline bool mmc_op_multi(u32 opcode)
 {
 	return opcode == MMC_WRITE_MULTIPLE_BLOCK ||
@@ -272,6 +279,7 @@ struct _mmc_csd {
  * EXT_CSD fields
  */
 
+#define EXT_CSD_CMDQ_MODE_EN		15	/* R/W */
 #define EXT_CSD_FLUSH_CACHE		32      /* W */
 #define EXT_CSD_CACHE_CTRL		33      /* R/W */
 #define EXT_CSD_POWER_OFF_NOTIFICATION	34	/* R/W */
@@ -331,6 +339,8 @@ struct _mmc_csd {
 #define EXT_CSD_CACHE_SIZE		249	/* RO, 4 bytes */
 #define EXT_CSD_PWR_CL_DDR_200_360	253	/* RO */
 #define EXT_CSD_FIRMWARE_VERSION	254	/* RO, 8 bytes */
+#define EXT_CSD_CMDQ_DEPTH		307	/* RO */
+#define EXT_CSD_CMDQ_SUPPORT		308	/* RO */
 #define EXT_CSD_SUPPORTED_MODE		493	/* RO */
 #define EXT_CSD_TAG_UNIT_SIZE		498	/* RO */
 #define EXT_CSD_DATA_TAG_SUPPORT	499	/* RO */
@@ -438,6 +448,13 @@ struct _mmc_csd {
 #define EXT_CSD_MANUAL_BKOPS_MASK	0x01
 
 /*
+ * Command Queue
+ */
+#define EXT_CSD_CMDQ_MODE_ENABLED	BIT(0)
+#define EXT_CSD_CMDQ_DEPTH_MASK		GENMASK(4, 0)
+#define EXT_CSD_CMDQ_SUPPORTED		BIT(0)
+
+/*
  * MMC_SWITCH access modes
  */
 
diff --git a/include/linux/mmc/slot-gpio.h b/include/linux/mmc/slot-gpio.h
index 3945a8c9d3cb..a7972cd3bc14 100644
--- a/include/linux/mmc/slot-gpio.h
+++ b/include/linux/mmc/slot-gpio.h
@@ -29,5 +29,6 @@ int mmc_gpiod_request_ro(struct mmc_host *host, const char *con_id,
 void mmc_gpio_set_cd_isr(struct mmc_host *host,
 			 irqreturn_t (*isr)(int irq, void *dev_id));
 void mmc_gpiod_request_cd_irq(struct mmc_host *host);
+bool mmc_can_gpio_cd(struct mmc_host *host);
 
 #endif
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 0f088f3a2fed..36d9896fbc1e 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -246,7 +246,7 @@ struct lruvec {
 #define ISOLATE_UNEVICTABLE	((__force isolate_mode_t)0x8)
 
 /* LRU Isolation modes. */
-typedef unsigned __bitwise__ isolate_mode_t;
+typedef unsigned __bitwise isolate_mode_t;
 
 enum zone_watermarks {
 	WMARK_MIN,
diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h
index ed84c07f6a51..8a57f0b1242d 100644
--- a/include/linux/mod_devicetable.h
+++ b/include/linux/mod_devicetable.h
@@ -175,7 +175,8 @@ struct ap_device_id {
 	kernel_ulong_t driver_info;
 };
 
-#define AP_DEVICE_ID_MATCH_DEVICE_TYPE		0x01
+#define AP_DEVICE_ID_MATCH_CARD_TYPE		0x01
+#define AP_DEVICE_ID_MATCH_QUEUE_TYPE		0x02
 
 /* s390 css bus devices (subchannels) */
 struct css_device_id {
diff --git a/include/linux/module.h b/include/linux/module.h
index 0c3207d26ac0..7c84273d60b9 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -399,7 +399,7 @@ struct module {
 	/* Arch-specific module values */
 	struct mod_arch_specific arch;
 
-	unsigned int taints;	/* same bits as kernel:tainted */
+	unsigned long taints;	/* same bits as kernel:taint_flags */
 
 #ifdef CONFIG_GENERIC_BUG
 	/* Support for BUG */
@@ -412,7 +412,7 @@ struct module {
 	/* Protected by RCU and/or module_mutex: use rcu_dereference() */
 	struct mod_kallsyms *kallsyms;
 	struct mod_kallsyms core_kallsyms;
-	
+
 	/* Section attributes */
 	struct module_sect_attrs *sect_attrs;
 
diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h
index 9094faf0699d..bca536341d1a 100644
--- a/include/linux/nfs4.h
+++ b/include/linux/nfs4.h
@@ -440,6 +440,7 @@ enum lock_type4 {
 #define FATTR4_WORD2_MDSTHRESHOLD       (1UL << 4)
 #define FATTR4_WORD2_CLONE_BLKSIZE	(1UL << 13)
 #define FATTR4_WORD2_SECURITY_LABEL     (1UL << 16)
+#define FATTR4_WORD2_MODE_UMASK		(1UL << 17)
 
 /* MDS threshold bitmap bits */
 #define THRESHOLD_RD                    (1UL << 0)
diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index 810124b33327..cb631973839a 100644
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -55,22 +55,18 @@ struct nfs_access_entry {
 	struct rcu_head		rcu_head;
 };
 
-struct nfs_lockowner {
-	fl_owner_t l_owner;
-	pid_t l_pid;
-};
-
 struct nfs_lock_context {
 	atomic_t count;
 	struct list_head list;
 	struct nfs_open_context *open_context;
-	struct nfs_lockowner lockowner;
+	fl_owner_t lockowner;
 	atomic_t io_count;
 };
 
 struct nfs4_state;
 struct nfs_open_context {
 	struct nfs_lock_context lock_context;
+	fl_owner_t flock_owner;
 	struct dentry *dentry;
 	struct rpc_cred *cred;
 	struct nfs4_state *state;
@@ -349,6 +345,7 @@ extern int nfs_attribute_cache_expired(struct inode *inode);
 extern int nfs_revalidate_inode(struct nfs_server *server, struct inode *inode);
 extern int nfs_revalidate_inode_rcu(struct nfs_server *server, struct inode *inode);
 extern int __nfs_revalidate_inode(struct nfs_server *, struct inode *);
+extern bool nfs_mapping_need_revalidate_inode(struct inode *inode);
 extern int nfs_revalidate_mapping(struct inode *inode, struct address_space *mapping);
 extern int nfs_revalidate_mapping_rcu(struct inode *inode);
 extern int nfs_setattr(struct dentry *, struct iattr *);
@@ -358,7 +355,7 @@ extern void nfs_setsecurity(struct inode *inode, struct nfs_fattr *fattr,
 extern struct nfs_open_context *get_nfs_open_context(struct nfs_open_context *ctx);
 extern void put_nfs_open_context(struct nfs_open_context *ctx);
 extern struct nfs_open_context *nfs_find_open_context(struct inode *inode, struct rpc_cred *cred, fmode_t mode);
-extern struct nfs_open_context *alloc_nfs_open_context(struct dentry *dentry, fmode_t f_mode);
+extern struct nfs_open_context *alloc_nfs_open_context(struct dentry *dentry, fmode_t f_mode, struct file *filp);
 extern void nfs_inode_attach_open_context(struct nfs_open_context *ctx);
 extern void nfs_file_set_open_context(struct file *filp, struct nfs_open_context *ctx);
 extern void nfs_file_clear_open_context(struct file *flip);
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index beb1e10f446e..348f7c158084 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -216,6 +216,20 @@ struct nfs4_get_lease_time_res {
 	struct nfs_fsinfo	       *lr_fsinfo;
 };
 
+struct xdr_stream;
+struct nfs4_xdr_opaque_data;
+
+struct nfs4_xdr_opaque_ops {
+	void (*encode)(struct xdr_stream *, const void *args,
+			const struct nfs4_xdr_opaque_data *);
+	void (*free)(struct nfs4_xdr_opaque_data *);
+};
+
+struct nfs4_xdr_opaque_data {
+	const struct nfs4_xdr_opaque_ops *ops;
+	void *data;
+};
+
 #define PNFS_LAYOUT_MAXSIZE 4096
 
 struct nfs4_layoutdriver_data {
@@ -306,6 +320,7 @@ struct nfs4_layoutreturn_args {
 	struct pnfs_layout_range range;
 	nfs4_stateid stateid;
 	__u32   layout_type;
+	struct nfs4_xdr_opaque_data *ld_private;
 };
 
 struct nfs4_layoutreturn_res {
@@ -321,6 +336,7 @@ struct nfs4_layoutreturn {
 	struct nfs_client *clp;
 	struct inode *inode;
 	int rpc_status;
+	struct nfs4_xdr_opaque_data ld_private;
 };
 
 #define PNFS_LAYOUTSTATS_MAXSIZE 256
@@ -341,8 +357,7 @@ struct nfs42_layoutstat_devinfo {
 	__u64 write_count;
 	__u64 write_bytes;
 	__u32 layout_type;
-	layoutstats_encode_t layoutstats_encode;
-	void *layout_private;
+	struct nfs4_xdr_opaque_data ld_private;
 };
 
 struct nfs42_layoutstat_args {
@@ -418,6 +433,7 @@ struct nfs_openargs {
 	enum open_claim_type4	claim;
 	enum createmode4	createmode;
 	const struct nfs4_label *label;
+	umode_t			umask;
 };
 
 struct nfs_openres {
@@ -469,6 +485,7 @@ struct nfs_closeargs {
 	fmode_t			fmode;
 	u32			share_access;
 	const u32 *		bitmask;
+	struct nfs4_layoutreturn_args *lr_args;
 };
 
 struct nfs_closeres {
@@ -477,6 +494,8 @@ struct nfs_closeres {
 	struct nfs_fattr *	fattr;
 	struct nfs_seqid *	seqid;
 	const struct nfs_server *server;
+	struct nfs4_layoutreturn_res *lr_res;
+	int lr_ret;
 };
 /*
  *  * Arguments to the lock,lockt, and locku call.
@@ -549,12 +568,15 @@ struct nfs4_delegreturnargs {
 	const struct nfs_fh *fhandle;
 	const nfs4_stateid *stateid;
 	const u32 * bitmask;
+	struct nfs4_layoutreturn_args *lr_args;
 };
 
 struct nfs4_delegreturnres {
 	struct nfs4_sequence_res	seq_res;
 	struct nfs_fattr * fattr;
 	struct nfs_server *server;
+	struct nfs4_layoutreturn_res *lr_res;
+	int lr_ret;
 };
 
 /*
@@ -937,6 +959,7 @@ struct nfs4_create_arg {
 	const struct nfs_fh *		dir_fh;
 	const u32 *			bitmask;
 	const struct nfs4_label		*label;
+	umode_t				umask;
 };
 
 struct nfs4_create_res {
diff --git a/include/linux/nmi.h b/include/linux/nmi.h
index a78c35cff1ae..aacca824a6ae 100644
--- a/include/linux/nmi.h
+++ b/include/linux/nmi.h
@@ -7,6 +7,23 @@
 #include <linux/sched.h>
 #include <asm/irq.h>
 
+/*
+ * The run state of the lockup detectors is controlled by the content of the
+ * 'watchdog_enabled' variable. Each lockup detector has its dedicated bit -
+ * bit 0 for the hard lockup detector and bit 1 for the soft lockup detector.
+ *
+ * 'watchdog_user_enabled', 'nmi_watchdog_enabled' and 'soft_watchdog_enabled'
+ * are variables that are only used as an 'interface' between the parameters
+ * in /proc/sys/kernel and the internal state bits in 'watchdog_enabled'. The
+ * 'watchdog_thresh' variable is handled differently because its value is not
+ * boolean, and the lockup detectors are 'suspended' while 'watchdog_thresh'
+ * is equal zero.
+ */
+#define NMI_WATCHDOG_ENABLED_BIT   0
+#define SOFT_WATCHDOG_ENABLED_BIT  1
+#define NMI_WATCHDOG_ENABLED      (1 << NMI_WATCHDOG_ENABLED_BIT)
+#define SOFT_WATCHDOG_ENABLED     (1 << SOFT_WATCHDOG_ENABLED_BIT)
+
 /**
  * touch_nmi_watchdog - restart NMI watchdog timeout.
  * 
@@ -91,9 +108,16 @@ extern int nmi_watchdog_enabled;
 extern int soft_watchdog_enabled;
 extern int watchdog_user_enabled;
 extern int watchdog_thresh;
+extern unsigned long watchdog_enabled;
 extern unsigned long *watchdog_cpumask_bits;
+#ifdef CONFIG_SMP
 extern int sysctl_softlockup_all_cpu_backtrace;
 extern int sysctl_hardlockup_all_cpu_backtrace;
+#else
+#define sysctl_softlockup_all_cpu_backtrace 0
+#define sysctl_hardlockup_all_cpu_backtrace 0
+#endif
+extern bool is_hardlockup(void);
 struct ctl_table;
 extern int proc_watchdog(struct ctl_table *, int ,
 			 void __user *, size_t *, loff_t *);
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index 6f47562d477b..50a7dbe88cf3 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -896,7 +896,7 @@ static inline int ntb_spad_is_unsafe(struct ntb_dev *ntb)
 }
 
 /**
- * ntb_mw_count() - get the number of scratchpads
+ * ntb_spad_count() - get the number of scratchpads
  * @ntb:	NTB device context.
  *
  * Hardware and topology may support a different number of scratchpads.
diff --git a/include/linux/nvme-fc-driver.h b/include/linux/nvme-fc-driver.h
new file mode 100644
index 000000000000..f21471f7ee40
--- /dev/null
+++ b/include/linux/nvme-fc-driver.h
@@ -0,0 +1,851 @@
+/*
+ * Copyright (c) 2016, Avago Technologies
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef _NVME_FC_DRIVER_H
+#define _NVME_FC_DRIVER_H 1
+
+
+/*
+ * **********************  LLDD FC-NVME Host API ********************
+ *
+ *  For FC LLDD's that are the NVME Host role.
+ *
+ * ******************************************************************
+ */
+
+
+
+/* FC Port role bitmask - can merge with FC Port Roles in fc transport */
+#define FC_PORT_ROLE_NVME_INITIATOR	0x10
+#define FC_PORT_ROLE_NVME_TARGET	0x11
+#define FC_PORT_ROLE_NVME_DISCOVERY	0x12
+
+
+/**
+ * struct nvme_fc_port_info - port-specific ids and FC connection-specific
+ *                            data element used during NVME Host role
+ *                            registrations
+ *
+ * Static fields describing the port being registered:
+ * @node_name: FC WWNN for the port
+ * @port_name: FC WWPN for the port
+ * @port_role: What NVME roles are supported (see FC_PORT_ROLE_xxx)
+ *
+ * Initialization values for dynamic port fields:
+ * @port_id:      FC N_Port_ID currently assigned the port. Upper 8 bits must
+ *                be set to 0.
+ */
+struct nvme_fc_port_info {
+	u64			node_name;
+	u64			port_name;
+	u32			port_role;
+	u32			port_id;
+};
+
+
+/**
+ * struct nvmefc_ls_req - Request structure passed from NVME-FC transport
+ *                        to LLDD in order to perform a NVME FC-4 LS
+ *                        request and obtain a response.
+ *
+ * Values set by the NVME-FC layer prior to calling the LLDD ls_req
+ * entrypoint.
+ * @rqstaddr: pointer to request buffer
+ * @rqstdma:  PCI DMA address of request buffer
+ * @rqstlen:  Length, in bytes, of request buffer
+ * @rspaddr:  pointer to response buffer
+ * @rspdma:   PCI DMA address of response buffer
+ * @rsplen:   Length, in bytes, of response buffer
+ * @timeout:  Maximum amount of time, in seconds, to wait for the LS response.
+ *            If timeout exceeded, LLDD to abort LS exchange and complete
+ *            LS request with error status.
+ * @private:  pointer to memory allocated alongside the ls request structure
+ *            that is specifically for the LLDD to use while processing the
+ *            request. The length of the buffer corresponds to the
+ *            lsrqst_priv_sz value specified in the nvme_fc_port_template
+ *            supplied by the LLDD.
+ * @done:     The callback routine the LLDD is to invoke upon completion of
+ *            the LS request. req argument is the pointer to the original LS
+ *            request structure. Status argument must be 0 upon success, a
+ *            negative errno on failure (example: -ENXIO).
+ */
+struct nvmefc_ls_req {
+	void			*rqstaddr;
+	dma_addr_t		rqstdma;
+	u32			rqstlen;
+	void			*rspaddr;
+	dma_addr_t		rspdma;
+	u32			rsplen;
+	u32			timeout;
+
+	void			*private;
+
+	void (*done)(struct nvmefc_ls_req *req, int status);
+
+} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
+
+
+enum nvmefc_fcp_datadir {
+	NVMEFC_FCP_NODATA,	/* payload_length and sg_cnt will be zero */
+	NVMEFC_FCP_WRITE,
+	NVMEFC_FCP_READ,
+};
+
+
+#define NVME_FC_MAX_SEGMENTS		256
+
+/**
+ * struct nvmefc_fcp_req - Request structure passed from NVME-FC transport
+ *                         to LLDD in order to perform a NVME FCP IO operation.
+ *
+ * Values set by the NVME-FC layer prior to calling the LLDD fcp_io
+ * entrypoint.
+ * @cmdaddr:   pointer to the FCP CMD IU buffer
+ * @rspaddr:   pointer to the FCP RSP IU buffer
+ * @cmddma:    PCI DMA address of the FCP CMD IU buffer
+ * @rspdma:    PCI DMA address of the FCP RSP IU buffer
+ * @cmdlen:    Length, in bytes, of the FCP CMD IU buffer
+ * @rsplen:    Length, in bytes, of the FCP RSP IU buffer
+ * @payload_length: Length of DATA_IN or DATA_OUT payload data to transfer
+ * @sg_table:  scatter/gather structure for payload data
+ * @first_sgl: memory for 1st scatter/gather list segment for payload data
+ * @sg_cnt:    number of elements in the scatter/gather list
+ * @io_dir:    direction of the FCP request (see NVMEFC_FCP_xxx)
+ * @sqid:      The nvme SQID the command is being issued on
+ * @done:      The callback routine the LLDD is to invoke upon completion of
+ *             the FCP operation. req argument is the pointer to the original
+ *             FCP IO operation.
+ * @private:   pointer to memory allocated alongside the FCP operation
+ *             request structure that is specifically for the LLDD to use
+ *             while processing the operation. The length of the buffer
+ *             corresponds to the fcprqst_priv_sz value specified in the
+ *             nvme_fc_port_template supplied by the LLDD.
+ *
+ * Values set by the LLDD indicating completion status of the FCP operation.
+ * Must be set prior to calling the done() callback.
+ * @transferred_length: amount of payload data, in bytes, that were
+ *             transferred. Should equal payload_length on success.
+ * @rcv_rsplen: length, in bytes, of the FCP RSP IU received.
+ * @status:    Completion status of the FCP operation. must be 0 upon success,
+ *             NVME_SC_FC_xxx value upon failure. Note: this is NOT a
+ *             reflection of the NVME CQE completion status. Only the status
+ *             of the FCP operation at the NVME-FC level.
+ */
+struct nvmefc_fcp_req {
+	void			*cmdaddr;
+	void			*rspaddr;
+	dma_addr_t		cmddma;
+	dma_addr_t		rspdma;
+	u16			cmdlen;
+	u16			rsplen;
+
+	u32			payload_length;
+	struct sg_table		sg_table;
+	struct scatterlist	*first_sgl;
+	int			sg_cnt;
+	enum nvmefc_fcp_datadir	io_dir;
+
+	__le16			sqid;
+
+	void (*done)(struct nvmefc_fcp_req *req);
+
+	void			*private;
+
+	u32			transferred_length;
+	u16			rcv_rsplen;
+	u32			status;
+} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
+
+
+/*
+ * Direct copy of fc_port_state enum. For later merging
+ */
+enum nvme_fc_obj_state {
+	FC_OBJSTATE_UNKNOWN,
+	FC_OBJSTATE_NOTPRESENT,
+	FC_OBJSTATE_ONLINE,
+	FC_OBJSTATE_OFFLINE,		/* User has taken Port Offline */
+	FC_OBJSTATE_BLOCKED,
+	FC_OBJSTATE_BYPASSED,
+	FC_OBJSTATE_DIAGNOSTICS,
+	FC_OBJSTATE_LINKDOWN,
+	FC_OBJSTATE_ERROR,
+	FC_OBJSTATE_LOOPBACK,
+	FC_OBJSTATE_DELETED,
+};
+
+
+/**
+ * struct nvme_fc_local_port - structure used between NVME-FC transport and
+ *                 a LLDD to reference a local NVME host port.
+ *                 Allocated/created by the nvme_fc_register_localport()
+ *                 transport interface.
+ *
+ * Fields with static values for the port. Initialized by the
+ * port_info struct supplied to the registration call.
+ * @port_num:  NVME-FC transport host port number
+ * @port_role: NVME roles are supported on the port (see FC_PORT_ROLE_xxx)
+ * @node_name: FC WWNN for the port
+ * @port_name: FC WWPN for the port
+ * @private:   pointer to memory allocated alongside the local port
+ *             structure that is specifically for the LLDD to use.
+ *             The length of the buffer corresponds to the local_priv_sz
+ *             value specified in the nvme_fc_port_template supplied by
+ *             the LLDD.
+ *
+ * Fields with dynamic values. Values may change base on link state. LLDD
+ * may reference fields directly to change them. Initialized by the
+ * port_info struct supplied to the registration call.
+ * @port_id:      FC N_Port_ID currently assigned the port. Upper 8 bits must
+ *                be set to 0.
+ * @port_state:   Operational state of the port.
+ */
+struct nvme_fc_local_port {
+	/* static/read-only fields */
+	u32 port_num;
+	u32 port_role;
+	u64 node_name;
+	u64 port_name;
+
+	void *private;
+
+	/* dynamic fields */
+	u32 port_id;
+	enum nvme_fc_obj_state port_state;
+} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
+
+
+/**
+ * struct nvme_fc_remote_port - structure used between NVME-FC transport and
+ *                 a LLDD to reference a remote NVME subsystem port.
+ *                 Allocated/created by the nvme_fc_register_remoteport()
+ *                 transport interface.
+ *
+ * Fields with static values for the port. Initialized by the
+ * port_info struct supplied to the registration call.
+ * @port_num:  NVME-FC transport remote subsystem port number
+ * @port_role: NVME roles are supported on the port (see FC_PORT_ROLE_xxx)
+ * @node_name: FC WWNN for the port
+ * @port_name: FC WWPN for the port
+ * @localport: pointer to the NVME-FC local host port the subsystem is
+ *             connected to.
+ * @private:   pointer to memory allocated alongside the remote port
+ *             structure that is specifically for the LLDD to use.
+ *             The length of the buffer corresponds to the remote_priv_sz
+ *             value specified in the nvme_fc_port_template supplied by
+ *             the LLDD.
+ *
+ * Fields with dynamic values. Values may change base on link or login
+ * state. LLDD may reference fields directly to change them. Initialized by
+ * the port_info struct supplied to the registration call.
+ * @port_id:      FC N_Port_ID currently assigned the port. Upper 8 bits must
+ *                be set to 0.
+ * @port_state:   Operational state of the remote port. Valid values are
+ *                ONLINE or UNKNOWN.
+ */
+struct nvme_fc_remote_port {
+	/* static fields */
+	u32 port_num;
+	u32 port_role;
+	u64 node_name;
+	u64 port_name;
+
+	struct nvme_fc_local_port *localport;
+
+	void *private;
+
+	/* dynamic fields */
+	u32 port_id;
+	enum nvme_fc_obj_state port_state;
+} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
+
+
+/**
+ * struct nvme_fc_port_template - structure containing static entrypoints and
+ *                 operational parameters for an LLDD that supports NVME host
+ *                 behavior. Passed by reference in port registrations.
+ *                 NVME-FC transport remembers template reference and may
+ *                 access it during runtime operation.
+ *
+ * Host/Initiator Transport Entrypoints/Parameters:
+ *
+ * @localport_delete:  The LLDD initiates deletion of a localport via
+ *       nvme_fc_deregister_localport(). However, the teardown is
+ *       asynchronous. This routine is called upon the completion of the
+ *       teardown to inform the LLDD that the localport has been deleted.
+ *       Entrypoint is Mandatory.
+ *
+ * @remoteport_delete:  The LLDD initiates deletion of a remoteport via
+ *       nvme_fc_deregister_remoteport(). However, the teardown is
+ *       asynchronous. This routine is called upon the completion of the
+ *       teardown to inform the LLDD that the remoteport has been deleted.
+ *       Entrypoint is Mandatory.
+ *
+ * @create_queue:  Upon creating a host<->controller association, queues are
+ *       created such that they can be affinitized to cpus/cores. This
+ *       callback into the LLDD to notify that a controller queue is being
+ *       created.  The LLDD may choose to allocate an associated hw queue
+ *       or map it onto a shared hw queue. Upon return from the call, the
+ *       LLDD specifies a handle that will be given back to it for any
+ *       command that is posted to the controller queue.  The handle can
+ *       be used by the LLDD to map quickly to the proper hw queue for
+ *       command execution.  The mask of cpu's that will map to this queue
+ *       at the block-level is also passed in. The LLDD should use the
+ *       queue id and/or cpu masks to ensure proper affinitization of the
+ *       controller queue to the hw queue.
+ *       Entrypoint is Optional.
+ *
+ * @delete_queue:  This is the inverse of the crete_queue. During
+ *       host<->controller association teardown, this routine is called
+ *       when a controller queue is being terminated. Any association with
+ *       a hw queue should be termined. If there is a unique hw queue, the
+ *       hw queue should be torn down.
+ *       Entrypoint is Optional.
+ *
+ * @poll_queue:  Called to poll for the completion of an io on a blk queue.
+ *       Entrypoint is Optional.
+ *
+ * @ls_req:  Called to issue a FC-NVME FC-4 LS service request.
+ *       The nvme_fc_ls_req structure will fully describe the buffers for
+ *       the request payload and where to place the response payload. The
+ *       LLDD is to allocate an exchange, issue the LS request, obtain the
+ *       LS response, and call the "done" routine specified in the request
+ *       structure (argument to done is the ls request structure itself).
+ *       Entrypoint is Mandatory.
+ *
+ * @fcp_io:  called to issue a FC-NVME I/O request.  The I/O may be for
+ *       an admin queue or an i/o queue.  The nvmefc_fcp_req structure will
+ *       fully describe the io: the buffer containing the FC-NVME CMD IU
+ *       (which contains the SQE), the sg list for the payload if applicable,
+ *       and the buffer to place the FC-NVME RSP IU into.  The LLDD will
+ *       complete the i/o, indicating the amount of data transferred or
+ *       any transport error, and call the "done" routine specified in the
+ *       request structure (argument to done is the fcp request structure
+ *       itself).
+ *       Entrypoint is Mandatory.
+ *
+ * @ls_abort: called to request the LLDD to abort the indicated ls request.
+ *       The call may return before the abort has completed. After aborting
+ *       the request, the LLDD must still call the ls request done routine
+ *       indicating an FC transport Aborted status.
+ *       Entrypoint is Mandatory.
+ *
+ * @fcp_abort: called to request the LLDD to abort the indicated fcp request.
+ *       The call may return before the abort has completed. After aborting
+ *       the request, the LLDD must still call the fcp request done routine
+ *       indicating an FC transport Aborted status.
+ *       Entrypoint is Mandatory.
+ *
+ * @max_hw_queues:  indicates the maximum number of hw queues the LLDD
+ *       supports for cpu affinitization.
+ *       Value is Mandatory. Must be at least 1.
+ *
+ * @max_sgl_segments:  indicates the maximum number of sgl segments supported
+ *       by the LLDD
+ *       Value is Mandatory. Must be at least 1. Recommend at least 256.
+ *
+ * @max_dif_sgl_segments:  indicates the maximum number of sgl segments
+ *       supported by the LLDD for DIF operations.
+ *       Value is Mandatory. Must be at least 1. Recommend at least 256.
+ *
+ * @dma_boundary:  indicates the dma address boundary where dma mappings
+ *       will be split across.
+ *       Value is Mandatory. Typical value is 0xFFFFFFFF to split across
+ *       4Gig address boundarys
+ *
+ * @local_priv_sz: The LLDD sets this field to the amount of additional
+ *       memory that it would like fc nvme layer to allocate on the LLDD's
+ *       behalf whenever a localport is allocated.  The additional memory
+ *       area solely for the of the LLDD and its location is specified by
+ *       the localport->private pointer.
+ *       Value is Mandatory. Allowed to be zero.
+ *
+ * @remote_priv_sz: The LLDD sets this field to the amount of additional
+ *       memory that it would like fc nvme layer to allocate on the LLDD's
+ *       behalf whenever a remoteport is allocated.  The additional memory
+ *       area solely for the of the LLDD and its location is specified by
+ *       the remoteport->private pointer.
+ *       Value is Mandatory. Allowed to be zero.
+ *
+ * @lsrqst_priv_sz: The LLDD sets this field to the amount of additional
+ *       memory that it would like fc nvme layer to allocate on the LLDD's
+ *       behalf whenever a ls request structure is allocated. The additional
+ *       memory area solely for the of the LLDD and its location is
+ *       specified by the ls_request->private pointer.
+ *       Value is Mandatory. Allowed to be zero.
+ *
+ * @fcprqst_priv_sz: The LLDD sets this field to the amount of additional
+ *       memory that it would like fc nvme layer to allocate on the LLDD's
+ *       behalf whenever a fcp request structure is allocated. The additional
+ *       memory area solely for the of the LLDD and its location is
+ *       specified by the fcp_request->private pointer.
+ *       Value is Mandatory. Allowed to be zero.
+ */
+struct nvme_fc_port_template {
+	/* initiator-based functions */
+	void	(*localport_delete)(struct nvme_fc_local_port *);
+	void	(*remoteport_delete)(struct nvme_fc_remote_port *);
+	int	(*create_queue)(struct nvme_fc_local_port *,
+				unsigned int qidx, u16 qsize,
+				void **handle);
+	void	(*delete_queue)(struct nvme_fc_local_port *,
+				unsigned int qidx, void *handle);
+	void	(*poll_queue)(struct nvme_fc_local_port *, void *handle);
+	int	(*ls_req)(struct nvme_fc_local_port *,
+				struct nvme_fc_remote_port *,
+				struct nvmefc_ls_req *);
+	int	(*fcp_io)(struct nvme_fc_local_port *,
+				struct nvme_fc_remote_port *,
+				void *hw_queue_handle,
+				struct nvmefc_fcp_req *);
+	void	(*ls_abort)(struct nvme_fc_local_port *,
+				struct nvme_fc_remote_port *,
+				struct nvmefc_ls_req *);
+	void	(*fcp_abort)(struct nvme_fc_local_port *,
+				struct nvme_fc_remote_port *,
+				void *hw_queue_handle,
+				struct nvmefc_fcp_req *);
+
+	u32	max_hw_queues;
+	u16	max_sgl_segments;
+	u16	max_dif_sgl_segments;
+	u64	dma_boundary;
+
+	/* sizes of additional private data for data structures */
+	u32	local_priv_sz;
+	u32	remote_priv_sz;
+	u32	lsrqst_priv_sz;
+	u32	fcprqst_priv_sz;
+};
+
+
+/*
+ * Initiator/Host functions
+ */
+
+int nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
+			struct nvme_fc_port_template *template,
+			struct device *dev,
+			struct nvme_fc_local_port **lport_p);
+
+int nvme_fc_unregister_localport(struct nvme_fc_local_port *localport);
+
+int nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
+			struct nvme_fc_port_info *pinfo,
+			struct nvme_fc_remote_port **rport_p);
+
+int nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *remoteport);
+
+
+
+/*
+ * ***************  LLDD FC-NVME Target/Subsystem API ***************
+ *
+ *  For FC LLDD's that are the NVME Subsystem role
+ *
+ * ******************************************************************
+ */
+
+/**
+ * struct nvmet_fc_port_info - port-specific ids and FC connection-specific
+ *                             data element used during NVME Subsystem role
+ *                             registrations
+ *
+ * Static fields describing the port being registered:
+ * @node_name: FC WWNN for the port
+ * @port_name: FC WWPN for the port
+ *
+ * Initialization values for dynamic port fields:
+ * @port_id:      FC N_Port_ID currently assigned the port. Upper 8 bits must
+ *                be set to 0.
+ */
+struct nvmet_fc_port_info {
+	u64			node_name;
+	u64			port_name;
+	u32			port_id;
+};
+
+
+/**
+ * struct nvmefc_tgt_ls_req - Structure used between LLDD and NVMET-FC
+ *                            layer to represent the exchange context for
+ *                            a FC-NVME Link Service (LS).
+ *
+ * The structure is allocated by the LLDD whenever a LS Request is received
+ * from the FC link. The address of the structure is passed to the nvmet-fc
+ * layer via the nvmet_fc_rcv_ls_req() call. The address of the structure
+ * will be passed back to the LLDD when the response is to be transmit.
+ * The LLDD is to use the address to map back to the LLDD exchange structure
+ * which maintains information such as the targetport the LS was received
+ * on, the remote FC NVME initiator that sent the LS, and any FC exchange
+ * context.  Upon completion of the LS response transmit, the address of the
+ * structure will be passed back to the LS rsp done() routine, allowing the
+ * nvmet-fc layer to release dma resources. Upon completion of the done()
+ * routine, no further access will be made by the nvmet-fc layer and the
+ * LLDD can de-allocate the structure.
+ *
+ * Field initialization:
+ *   At the time of the nvmet_fc_rcv_ls_req() call, there is no content that
+ *     is valid in the structure.
+ *
+ *   When the structure is used for the LLDD->xmt_ls_rsp() call, the nvmet-fc
+ *     layer will fully set the fields in order to specify the response
+ *     payload buffer and its length as well as the done routine to be called
+ *     upon compeletion of the transmit.  The nvmet-fc layer will also set a
+ *     private pointer for its own use in the done routine.
+ *
+ * Values set by the NVMET-FC layer prior to calling the LLDD xmt_ls_rsp
+ * entrypoint.
+ * @rspbuf:   pointer to the LS response buffer
+ * @rspdma:   PCI DMA address of the LS response buffer
+ * @rsplen:   Length, in bytes, of the LS response buffer
+ * @done:     The callback routine the LLDD is to invoke upon completion of
+ *            transmitting the LS response. req argument is the pointer to
+ *            the original ls request.
+ * @nvmet_fc_private:  pointer to an internal NVMET-FC layer structure used
+ *            as part of the NVMET-FC processing. The LLDD is not to access
+ *            this pointer.
+ */
+struct nvmefc_tgt_ls_req {
+	void		*rspbuf;
+	dma_addr_t	rspdma;
+	u16		rsplen;
+
+	void (*done)(struct nvmefc_tgt_ls_req *req);
+	void *nvmet_fc_private;		/* LLDD is not to access !! */
+};
+
+/* Operations that NVME-FC layer may request the LLDD to perform for FCP */
+enum {
+	NVMET_FCOP_READDATA	= 1,	/* xmt data to initiator */
+	NVMET_FCOP_WRITEDATA	= 2,	/* xmt data from initiator */
+	NVMET_FCOP_READDATA_RSP	= 3,	/* xmt data to initiator and send
+					 * rsp as well
+					 */
+	NVMET_FCOP_RSP		= 4,	/* send rsp frame */
+	NVMET_FCOP_ABORT	= 5,	/* abort exchange via ABTS */
+	NVMET_FCOP_BA_ACC	= 6,	/* send BA_ACC */
+	NVMET_FCOP_BA_RJT	= 7,	/* send BA_RJT */
+};
+
+/**
+ * struct nvmefc_tgt_fcp_req - Structure used between LLDD and NVMET-FC
+ *                            layer to represent the exchange context and
+ *                            the specific FC-NVME IU operation(s) to perform
+ *                            for a FC-NVME FCP IO.
+ *
+ * Structure used between LLDD and nvmet-fc layer to represent the exchange
+ * context for a FC-NVME FCP I/O operation (e.g. a nvme sqe, the sqe-related
+ * memory transfers, and its assocated cqe transfer).
+ *
+ * The structure is allocated by the LLDD whenever a FCP CMD IU is received
+ * from the FC link. The address of the structure is passed to the nvmet-fc
+ * layer via the nvmet_fc_rcv_fcp_req() call. The address of the structure
+ * will be passed back to the LLDD for the data operations and transmit of
+ * the response. The LLDD is to use the address to map back to the LLDD
+ * exchange structure which maintains information such as the targetport
+ * the FCP I/O was received on, the remote FC NVME initiator that sent the
+ * FCP I/O, and any FC exchange context.  Upon completion of the FCP target
+ * operation, the address of the structure will be passed back to the FCP
+ * op done() routine, allowing the nvmet-fc layer to release dma resources.
+ * Upon completion of the done() routine for either RSP or ABORT ops, no
+ * further access will be made by the nvmet-fc layer and the LLDD can
+ * de-allocate the structure.
+ *
+ * Field initialization:
+ *   At the time of the nvmet_fc_rcv_fcp_req() call, there is no content that
+ *     is valid in the structure.
+ *
+ *   When the structure is used for an FCP target operation, the nvmet-fc
+ *     layer will fully set the fields in order to specify the scattergather
+ *     list, the transfer length, as well as the done routine to be called
+ *     upon compeletion of the operation.  The nvmet-fc layer will also set a
+ *     private pointer for its own use in the done routine.
+ *
+ * Note: the LLDD must never fail a NVMET_FCOP_ABORT request !!
+ *
+ * Values set by the NVMET-FC layer prior to calling the LLDD fcp_op
+ * entrypoint.
+ * @op:       Indicates the FCP IU operation to perform (see NVMET_FCOP_xxx)
+ * @hwqid:    Specifies the hw queue index (0..N-1, where N is the
+ *            max_hw_queues value from the LLD's nvmet_fc_target_template)
+ *            that the operation is to use.
+ * @offset:   Indicates the DATA_OUT/DATA_IN payload offset to be tranferred.
+ *            Field is only valid on WRITEDATA, READDATA, or READDATA_RSP ops.
+ * @timeout:  amount of time, in seconds, to wait for a response from the NVME
+ *            host. A value of 0 is an infinite wait.
+ *            Valid only for the following ops:
+ *              WRITEDATA: caps the wait for data reception
+ *              READDATA_RSP & RSP: caps wait for FCP_CONF reception (if used)
+ * @transfer_length: the length, in bytes, of the DATA_OUT or DATA_IN payload
+ *            that is to be transferred.
+ *            Valid only for the WRITEDATA, READDATA, or READDATA_RSP ops.
+ * @ba_rjt:   Contains the BA_RJT payload that is to be transferred.
+ *            Valid only for the NVMET_FCOP_BA_RJT op.
+ * @sg:       Scatter/gather list for the DATA_OUT/DATA_IN payload data.
+ *            Valid only for the WRITEDATA, READDATA, or READDATA_RSP ops.
+ * @sg_cnt:   Number of valid entries in the scatter/gather list.
+ *            Valid only for the WRITEDATA, READDATA, or READDATA_RSP ops.
+ * @rspaddr:  pointer to the FCP RSP IU buffer to be transmit
+ *            Used by RSP and READDATA_RSP ops
+ * @rspdma:   PCI DMA address of the FCP RSP IU buffer
+ *            Used by RSP and READDATA_RSP ops
+ * @rsplen:   Length, in bytes, of the FCP RSP IU buffer
+ *            Used by RSP and READDATA_RSP ops
+ * @done:     The callback routine the LLDD is to invoke upon completion of
+ *            the operation. req argument is the pointer to the original
+ *            FCP subsystem op request.
+ * @nvmet_fc_private:  pointer to an internal NVMET-FC layer structure used
+ *            as part of the NVMET-FC processing. The LLDD is not to
+ *            reference this field.
+ *
+ * Values set by the LLDD indicating completion status of the FCP operation.
+ * Must be set prior to calling the done() callback.
+ * @transferred_length: amount of DATA_OUT payload data received by a
+ *            a WRITEDATA operation. If not a WRITEDATA operation, value must
+ *            be set to 0. Should equal transfer_length on success.
+ * @fcp_error: status of the FCP operation. Must be 0 on success; on failure
+ *            must be a NVME_SC_FC_xxxx value.
+ */
+struct nvmefc_tgt_fcp_req {
+	u8			op;
+	u16			hwqid;
+	u32			offset;
+	u32			timeout;
+	u32			transfer_length;
+	struct fc_ba_rjt	ba_rjt;
+	struct scatterlist	sg[NVME_FC_MAX_SEGMENTS];
+	int			sg_cnt;
+	void			*rspaddr;
+	dma_addr_t		rspdma;
+	u16			rsplen;
+
+	void (*done)(struct nvmefc_tgt_fcp_req *);
+
+	void *nvmet_fc_private;		/* LLDD is not to access !! */
+
+	u32			transferred_length;
+	int			fcp_error;
+};
+
+
+/* Target Features (Bit fields) LLDD supports */
+enum {
+	NVMET_FCTGTFEAT_READDATA_RSP = (1 << 0),
+		/* Bit 0: supports the NVMET_FCPOP_READDATA_RSP op, which
+		 * sends (the last) Read Data sequence followed by the RSP
+		 * sequence in one LLDD operation. Errors during Data
+		 * sequence transmit must not allow RSP sequence to be sent.
+		 */
+	NVMET_FCTGTFEAT_NEEDS_CMD_CPUSCHED = (1 << 1),
+		/* Bit 1: When 0, the LLDD will deliver FCP CMD
+		 * on the CPU it should be affinitized to. Thus work will
+		 * be scheduled on the cpu received on. When 1, the LLDD
+		 * may not deliver the CMD on the CPU it should be worked
+		 * on. The transport should pick a cpu to schedule the work
+		 * on.
+		 */
+};
+
+
+/**
+ * struct nvmet_fc_target_port - structure used between NVME-FC transport and
+ *                 a LLDD to reference a local NVME subsystem port.
+ *                 Allocated/created by the nvme_fc_register_targetport()
+ *                 transport interface.
+ *
+ * Fields with static values for the port. Initialized by the
+ * port_info struct supplied to the registration call.
+ * @port_num:  NVME-FC transport subsytem port number
+ * @node_name: FC WWNN for the port
+ * @port_name: FC WWPN for the port
+ * @private:   pointer to memory allocated alongside the local port
+ *             structure that is specifically for the LLDD to use.
+ *             The length of the buffer corresponds to the target_priv_sz
+ *             value specified in the nvme_fc_target_template supplied by
+ *             the LLDD.
+ *
+ * Fields with dynamic values. Values may change base on link state. LLDD
+ * may reference fields directly to change them. Initialized by the
+ * port_info struct supplied to the registration call.
+ * @port_id:      FC N_Port_ID currently assigned the port. Upper 8 bits must
+ *                be set to 0.
+ * @port_state:   Operational state of the port.
+ */
+struct nvmet_fc_target_port {
+	/* static/read-only fields */
+	u32 port_num;
+	u64 node_name;
+	u64 port_name;
+
+	void *private;
+
+	/* dynamic fields */
+	u32 port_id;
+	enum nvme_fc_obj_state port_state;
+} __aligned(sizeof(u64));	/* alignment for other things alloc'd with */
+
+
+/**
+ * struct nvmet_fc_target_template - structure containing static entrypoints
+ *                 and operational parameters for an LLDD that supports NVME
+ *                 subsystem behavior. Passed by reference in port
+ *                 registrations. NVME-FC transport remembers template
+ *                 reference and may access it during runtime operation.
+ *
+ * Subsystem/Target Transport Entrypoints/Parameters:
+ *
+ * @targetport_delete:  The LLDD initiates deletion of a targetport via
+ *       nvmet_fc_unregister_targetport(). However, the teardown is
+ *       asynchronous. This routine is called upon the completion of the
+ *       teardown to inform the LLDD that the targetport has been deleted.
+ *       Entrypoint is Mandatory.
+ *
+ * @xmt_ls_rsp:  Called to transmit the response to a FC-NVME FC-4 LS service.
+ *       The nvmefc_tgt_ls_req structure is the same LLDD-supplied exchange
+ *       structure specified in the nvmet_fc_rcv_ls_req() call made when
+ *       the LS request was received.  The structure will fully describe
+ *       the buffers for the response payload and the dma address of the
+ *       payload. The LLDD is to transmit the response (or return a non-zero
+ *       errno status), and upon completion of the transmit, call the
+ *       "done" routine specified in the nvmefc_tgt_ls_req structure
+ *       (argument to done is the ls reqwuest structure itself).
+ *       After calling the done routine, the LLDD shall consider the
+ *       LS handling complete and the nvmefc_tgt_ls_req structure may
+ *       be freed/released.
+ *       Entrypoint is Mandatory.
+ *
+ * @fcp_op:  Called to perform a data transfer, transmit a response, or
+ *       abort an FCP opertion. The nvmefc_tgt_fcp_req structure is the same
+ *       LLDD-supplied exchange structure specified in the
+ *       nvmet_fc_rcv_fcp_req() call made when the FCP CMD IU was received.
+ *       The op field in the structure shall indicate the operation for
+ *       the LLDD to perform relative to the io.
+ *         NVMET_FCOP_READDATA operation: the LLDD is to send the
+ *           payload data (described by sglist) to the host in 1 or
+ *           more FC sequences (preferrably 1).  Note: the fc-nvme layer
+ *           may call the READDATA operation multiple times for longer
+ *           payloads.
+ *         NVMET_FCOP_WRITEDATA operation: the LLDD is to receive the
+ *           payload data (described by sglist) from the host via 1 or
+ *           more FC sequences (preferrably 1). The LLDD is to generate
+ *           the XFER_RDY IU(s) corresponding to the data being requested.
+ *           Note: the FC-NVME layer may call the WRITEDATA operation
+ *           multiple times for longer payloads.
+ *         NVMET_FCOP_READDATA_RSP operation: the LLDD is to send the
+ *           payload data (described by sglist) to the host in 1 or
+ *           more FC sequences (preferrably 1). If an error occurs during
+ *           payload data transmission, the LLDD is to set the
+ *           nvmefc_tgt_fcp_req fcp_error and transferred_length field, then
+ *           consider the operation complete. On error, the LLDD is to not
+ *           transmit the FCP_RSP iu. If all payload data is transferred
+ *           successfully, the LLDD is to update the nvmefc_tgt_fcp_req
+ *           transferred_length field and may subsequently transmit the
+ *           FCP_RSP iu payload (described by rspbuf, rspdma, rsplen).
+ *           The LLDD is to await FCP_CONF reception to confirm the RSP
+ *           reception by the host. The LLDD may retramsit the FCP_RSP iu
+ *           if necessary per FC-NVME. Upon reception of FCP_CONF, or upon
+ *           FCP_CONF failure, the LLDD is to set the nvmefc_tgt_fcp_req
+ *           fcp_error field and consider the operation complete..
+ *         NVMET_FCOP_RSP: the LLDD is to transmit the FCP_RSP iu payload
+ *           (described by rspbuf, rspdma, rsplen).  The LLDD is to await
+ *           FCP_CONF reception to confirm the RSP reception by the host.
+ *           The LLDD may retramsit the FCP_RSP iu if necessary per FC-NVME.
+ *           Upon reception of FCP_CONF, or upon FCP_CONF failure, the
+ *           LLDD is to set the nvmefc_tgt_fcp_req fcp_error field and
+ *           consider the operation complete..
+ *         NVMET_FCOP_ABORT: the LLDD is to terminate the exchange
+ *           corresponding to the fcp operation. The LLDD shall send
+ *           ABTS and follow FC exchange abort-multi rules, including
+ *           ABTS retries and possible logout.
+ *       Upon completing the indicated operation, the LLDD is to set the
+ *       status fields for the operation (tranferred_length and fcp_error
+ *       status) in the request, then all the "done" routine
+ *       indicated in the fcp request.  Upon return from the "done"
+ *       routine for either a NVMET_FCOP_RSP or NVMET_FCOP_ABORT operation
+ *       the fc-nvme layer will not longer reference the fcp request,
+ *       allowing the LLDD to free/release the fcp request.
+ *       Note: when calling the done routine for READDATA or WRITEDATA
+ *       operations, the fc-nvme layer may immediate convert, in the same
+ *       thread and before returning to the LLDD, the fcp operation to
+ *       the next operation for the fcp io and call the LLDDs fcp_op
+ *       call again. If fields in the fcp request are to be accessed post
+ *       the done call, the LLDD should save their values prior to calling
+ *       the done routine, and inspect the save values after the done
+ *       routine.
+ *       Returns 0 on success, -<errno> on failure (Ex: -EIO)
+ *       Entrypoint is Mandatory.
+ *
+ * @max_hw_queues:  indicates the maximum number of hw queues the LLDD
+ *       supports for cpu affinitization.
+ *       Value is Mandatory. Must be at least 1.
+ *
+ * @max_sgl_segments:  indicates the maximum number of sgl segments supported
+ *       by the LLDD
+ *       Value is Mandatory. Must be at least 1. Recommend at least 256.
+ *
+ * @max_dif_sgl_segments:  indicates the maximum number of sgl segments
+ *       supported by the LLDD for DIF operations.
+ *       Value is Mandatory. Must be at least 1. Recommend at least 256.
+ *
+ * @dma_boundary:  indicates the dma address boundary where dma mappings
+ *       will be split across.
+ *       Value is Mandatory. Typical value is 0xFFFFFFFF to split across
+ *       4Gig address boundarys
+ *
+ * @target_features: The LLDD sets bits in this field to correspond to
+ *       optional features that are supported by the LLDD.
+ *       Refer to the NVMET_FCTGTFEAT_xxx values.
+ *       Value is Mandatory. Allowed to be zero.
+ *
+ * @target_priv_sz: The LLDD sets this field to the amount of additional
+ *       memory that it would like fc nvme layer to allocate on the LLDD's
+ *       behalf whenever a targetport is allocated.  The additional memory
+ *       area solely for the of the LLDD and its location is specified by
+ *       the targetport->private pointer.
+ *       Value is Mandatory. Allowed to be zero.
+ */
+struct nvmet_fc_target_template {
+	void (*targetport_delete)(struct nvmet_fc_target_port *tgtport);
+	int (*xmt_ls_rsp)(struct nvmet_fc_target_port *tgtport,
+				struct nvmefc_tgt_ls_req *tls_req);
+	int (*fcp_op)(struct nvmet_fc_target_port *tgtport,
+				struct nvmefc_tgt_fcp_req *);
+
+	u32	max_hw_queues;
+	u16	max_sgl_segments;
+	u16	max_dif_sgl_segments;
+	u64	dma_boundary;
+
+	u32	target_features;
+
+	u32	target_priv_sz;
+};
+
+
+int nvmet_fc_register_targetport(struct nvmet_fc_port_info *portinfo,
+			struct nvmet_fc_target_template *template,
+			struct device *dev,
+			struct nvmet_fc_target_port **tgtport_p);
+
+int nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *tgtport);
+
+int nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *tgtport,
+			struct nvmefc_tgt_ls_req *lsreq,
+			void *lsreqbuf, u32 lsreqbuf_len);
+
+int nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *tgtport,
+			struct nvmefc_tgt_fcp_req *fcpreq,
+			void *cmdiubuf, u32 cmdiubuf_len);
+
+#endif /* _NVME_FC_DRIVER_H */
diff --git a/include/linux/nvme-fc.h b/include/linux/nvme-fc.h
new file mode 100644
index 000000000000..4b45226bd604
--- /dev/null
+++ b/include/linux/nvme-fc.h
@@ -0,0 +1,268 @@
+/*
+ * Copyright (c) 2016 Avago Technologies.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful.
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
+ * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
+ * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO
+ * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
+ * See the GNU General Public License for more details, a copy of which
+ * can be found in the file COPYING included with this package
+ *
+ */
+
+/*
+ * This file contains definitions relative to FC-NVME r1.11 and a few
+ * newer items
+ */
+
+#ifndef _NVME_FC_H
+#define _NVME_FC_H 1
+
+
+#define NVME_CMD_SCSI_ID		0xFD
+#define NVME_CMD_FC_ID			FC_TYPE_NVME
+
+/* FC-NVME Cmd IU Flags */
+#define FCNVME_CMD_FLAGS_DIRMASK	0x03
+#define FCNVME_CMD_FLAGS_WRITE		0x01
+#define FCNVME_CMD_FLAGS_READ		0x02
+
+struct nvme_fc_cmd_iu {
+	__u8			scsi_id;
+	__u8			fc_id;
+	__be16			iu_len;
+	__u8			rsvd4[3];
+	__u8			flags;
+	__be64			connection_id;
+	__be32			csn;
+	__be32			data_len;
+	struct nvme_command	sqe;
+	__be32			rsvd88[2];
+};
+
+#define NVME_FC_SIZEOF_ZEROS_RSP	12
+
+struct nvme_fc_ersp_iu {
+	__u8			rsvd0[2];
+	__be16			iu_len;
+	__be32			rsn;
+	__be32			xfrd_len;
+	__be32			rsvd12;
+	struct nvme_completion	cqe;
+	/* for now - no additional payload */
+};
+
+
+/* FC-NVME r1.03/16-119v0 NVME Link Services */
+enum {
+	FCNVME_LS_RSVD			= 0,
+	FCNVME_LS_RJT			= 1,
+	FCNVME_LS_ACC			= 2,
+	FCNVME_LS_CREATE_ASSOCIATION	= 3,
+	FCNVME_LS_CREATE_CONNECTION	= 4,
+	FCNVME_LS_DISCONNECT		= 5,
+};
+
+/* FC-NVME r1.03/16-119v0 NVME Link Service Descriptors */
+enum {
+	FCNVME_LSDESC_RSVD		= 0x0,
+	FCNVME_LSDESC_RQST		= 0x1,
+	FCNVME_LSDESC_RJT		= 0x2,
+	FCNVME_LSDESC_CREATE_ASSOC_CMD	= 0x3,
+	FCNVME_LSDESC_CREATE_CONN_CMD	= 0x4,
+	FCNVME_LSDESC_DISCONN_CMD	= 0x5,
+	FCNVME_LSDESC_CONN_ID		= 0x6,
+	FCNVME_LSDESC_ASSOC_ID		= 0x7,
+};
+
+
+/* ********** start of Link Service Descriptors ********** */
+
+
+/*
+ * fills in length of a descriptor. Struture minus descriptor header
+ */
+static inline __be32 fcnvme_lsdesc_len(size_t sz)
+{
+	return cpu_to_be32(sz - (2 * sizeof(u32)));
+}
+
+
+struct fcnvme_ls_rqst_w0 {
+	u8	ls_cmd;			/* FCNVME_LS_xxx */
+	u8	zeros[3];
+};
+
+/* FCNVME_LSDESC_RQST */
+struct fcnvme_lsdesc_rqst {
+	__be32	desc_tag;		/* FCNVME_LSDESC_xxx */
+	__be32	desc_len;
+	struct fcnvme_ls_rqst_w0	w0;
+	__be32	rsvd12;
+};
+
+
+
+
+/* FCNVME_LSDESC_RJT */
+struct fcnvme_lsdesc_rjt {
+	__be32	desc_tag;		/* FCNVME_LSDESC_xxx */
+	__be32	desc_len;
+	u8	rsvd8;
+
+	/*
+	 * Reject reason and explanaction codes are generic
+	 * to ELs's from LS-3.
+	 */
+	u8	reason_code;
+	u8	reason_explanation;
+
+	u8	vendor;
+	__be32	rsvd12;
+};
+
+
+#define FCNVME_ASSOC_HOSTID_LEN		64
+#define FCNVME_ASSOC_HOSTNQN_LEN	256
+#define FCNVME_ASSOC_SUBNQN_LEN		256
+
+/* FCNVME_LSDESC_CREATE_ASSOC_CMD */
+struct fcnvme_lsdesc_cr_assoc_cmd {
+	__be32	desc_tag;		/* FCNVME_LSDESC_xxx */
+	__be32	desc_len;
+	__be16	ersp_ratio;
+	__be16	rsvd10;
+	__be32	rsvd12[9];
+	__be16	cntlid;
+	__be16	sqsize;
+	__be32	rsvd52;
+	u8	hostid[FCNVME_ASSOC_HOSTID_LEN];
+	u8	hostnqn[FCNVME_ASSOC_HOSTNQN_LEN];
+	u8	subnqn[FCNVME_ASSOC_SUBNQN_LEN];
+	u8	rsvd632[384];
+};
+
+/* FCNVME_LSDESC_CREATE_CONN_CMD */
+struct fcnvme_lsdesc_cr_conn_cmd {
+	__be32	desc_tag;		/* FCNVME_LSDESC_xxx */
+	__be32	desc_len;
+	__be16	ersp_ratio;
+	__be16	rsvd10;
+	__be32	rsvd12[9];
+	__be16	qid;
+	__be16	sqsize;
+	__be32  rsvd52;
+};
+
+/* Disconnect Scope Values */
+enum {
+	FCNVME_DISCONN_ASSOCIATION	= 0,
+	FCNVME_DISCONN_CONNECTION	= 1,
+};
+
+/* FCNVME_LSDESC_DISCONN_CMD */
+struct fcnvme_lsdesc_disconn_cmd {
+	__be32	desc_tag;		/* FCNVME_LSDESC_xxx */
+	__be32	desc_len;
+	u8	rsvd8[3];
+	/* note: scope is really a 1 bit field */
+	u8	scope;			/* FCNVME_DISCONN_xxx */
+	__be32	rsvd12;
+	__be64	id;
+};
+
+/* FCNVME_LSDESC_CONN_ID */
+struct fcnvme_lsdesc_conn_id {
+	__be32	desc_tag;		/* FCNVME_LSDESC_xxx */
+	__be32	desc_len;
+	__be64	connection_id;
+};
+
+/* FCNVME_LSDESC_ASSOC_ID */
+struct fcnvme_lsdesc_assoc_id {
+	__be32	desc_tag;		/* FCNVME_LSDESC_xxx */
+	__be32	desc_len;
+	__be64	association_id;
+};
+
+/* r_ctl values */
+enum {
+	FCNVME_RS_RCTL_DATA		= 1,
+	FCNVME_RS_RCTL_XFER_RDY		= 5,
+	FCNVME_RS_RCTL_RSP		= 8,
+};
+
+
+/* ********** start of Link Services ********** */
+
+
+/* FCNVME_LS_RJT */
+struct fcnvme_ls_rjt {
+	struct fcnvme_ls_rqst_w0		w0;
+	__be32					desc_list_len;
+	struct fcnvme_lsdesc_rqst		rqst;
+	struct fcnvme_lsdesc_rjt		rjt;
+};
+
+/* FCNVME_LS_ACC */
+struct fcnvme_ls_acc_hdr {
+	struct fcnvme_ls_rqst_w0		w0;
+	__be32					desc_list_len;
+	struct fcnvme_lsdesc_rqst		rqst;
+	/* Followed by cmd-specific ACC descriptors, see next definitions */
+};
+
+/* FCNVME_LS_CREATE_ASSOCIATION */
+struct fcnvme_ls_cr_assoc_rqst {
+	struct fcnvme_ls_rqst_w0		w0;
+	__be32					desc_list_len;
+	struct fcnvme_lsdesc_cr_assoc_cmd	assoc_cmd;
+};
+
+struct fcnvme_ls_cr_assoc_acc {
+	struct fcnvme_ls_acc_hdr		hdr;
+	struct fcnvme_lsdesc_assoc_id		associd;
+	struct fcnvme_lsdesc_conn_id		connectid;
+};
+
+
+/* FCNVME_LS_CREATE_CONNECTION */
+struct fcnvme_ls_cr_conn_rqst {
+	struct fcnvme_ls_rqst_w0		w0;
+	__be32					desc_list_len;
+	struct fcnvme_lsdesc_assoc_id		associd;
+	struct fcnvme_lsdesc_cr_conn_cmd	connect_cmd;
+};
+
+struct fcnvme_ls_cr_conn_acc {
+	struct fcnvme_ls_acc_hdr		hdr;
+	struct fcnvme_lsdesc_conn_id		connectid;
+};
+
+/* FCNVME_LS_DISCONNECT */
+struct fcnvme_ls_disconnect_rqst {
+	struct fcnvme_ls_rqst_w0		w0;
+	__be32					desc_list_len;
+	struct fcnvme_lsdesc_assoc_id		associd;
+	struct fcnvme_lsdesc_disconn_cmd	discon_cmd;
+};
+
+struct fcnvme_ls_disconnect_acc {
+	struct fcnvme_ls_acc_hdr		hdr;
+};
+
+
+/*
+ * Yet to be defined in FC-NVME:
+ */
+#define NVME_FC_CONNECT_TIMEOUT_SEC	2		/* 2 seconds */
+#define NVME_FC_LS_TIMEOUT_SEC		2		/* 2 seconds */
+#define NVME_FC_TGTOP_TIMEOUT_SEC	2		/* 2 seconds */
+
+
+#endif /* _NVME_FC_H */
diff --git a/include/linux/nvme.h b/include/linux/nvme.h
index fc3c24206593..3d1c6f1b15c9 100644
--- a/include/linux/nvme.h
+++ b/include/linux/nvme.h
@@ -242,6 +242,7 @@ enum {
 	NVME_CTRL_ONCS_COMPARE			= 1 << 0,
 	NVME_CTRL_ONCS_WRITE_UNCORRECTABLE	= 1 << 1,
 	NVME_CTRL_ONCS_DSM			= 1 << 2,
+	NVME_CTRL_ONCS_WRITE_ZEROES		= 1 << 3,
 	NVME_CTRL_VWC_PRESENT			= 1 << 0,
 };
 
@@ -558,6 +559,23 @@ struct nvme_dsm_range {
 	__le64			slba;
 };
 
+struct nvme_write_zeroes_cmd {
+	__u8			opcode;
+	__u8			flags;
+	__u16			command_id;
+	__le32			nsid;
+	__u64			rsvd2;
+	__le64			metadata;
+	union nvme_data_ptr	dptr;
+	__le64			slba;
+	__le16			length;
+	__le16			control;
+	__le32			dsmgmt;
+	__le32			reftag;
+	__le16			apptag;
+	__le16			appmask;
+};
+
 /* Admin commands */
 
 enum nvme_admin_opcode {
@@ -857,6 +875,7 @@ struct nvme_command {
 		struct nvme_download_firmware dlfw;
 		struct nvme_format_cmd format;
 		struct nvme_dsm_cmd dsm;
+		struct nvme_write_zeroes_cmd write_zeroes;
 		struct nvme_abort_cmd abort;
 		struct nvme_get_log_page_command get_log_page;
 		struct nvmf_common_command fabrics;
@@ -947,6 +966,7 @@ enum {
 	NVME_SC_BAD_ATTRIBUTES		= 0x180,
 	NVME_SC_INVALID_PI		= 0x181,
 	NVME_SC_READ_ONLY		= 0x182,
+	NVME_SC_ONCS_NOT_SUPPORTED	= 0x183,
 
 	/*
 	 * I/O Command Set Specific - Fabrics commands:
@@ -973,17 +993,30 @@ enum {
 	NVME_SC_UNWRITTEN_BLOCK		= 0x287,
 
 	NVME_SC_DNR			= 0x4000,
+
+
+	/*
+	 * FC Transport-specific error status values for NVME commands
+	 *
+	 * Transport-specific status code values must be in the range 0xB0..0xBF
+	 */
+
+	/* Generic FC failure - catchall */
+	NVME_SC_FC_TRANSPORT_ERROR	= 0x00B0,
+
+	/* I/O failure due to FC ABTS'd */
+	NVME_SC_FC_TRANSPORT_ABORTED	= 0x00B1,
 };
 
 struct nvme_completion {
 	/*
 	 * Used by Admin and Fabrics commands to return data:
 	 */
-	union {
-		__le16	result16;
-		__le32	result;
-		__le64	result64;
-	};
+	union nvme_result {
+		__le16	u16;
+		__le32	u32;
+		__le64	u64;
+	} result;
 	__le16	sq_head;	/* how much of this queue may be reclaimed */
 	__le16	sq_id;		/* submission queue that generated this entry */
 	__u16	command_id;	/* of the command which completed */
diff --git a/include/linux/of.h b/include/linux/of.h
index 299aeb192727..d72f01009297 100644
--- a/include/linux/of.h
+++ b/include/linux/of.h
@@ -1266,6 +1266,18 @@ static inline bool of_device_is_system_power_controller(const struct device_node
  * Overlay support
  */
 
+enum of_overlay_notify_action {
+	OF_OVERLAY_PRE_APPLY,
+	OF_OVERLAY_POST_APPLY,
+	OF_OVERLAY_PRE_REMOVE,
+	OF_OVERLAY_POST_REMOVE,
+};
+
+struct of_overlay_notify_data {
+	struct device_node *overlay;
+	struct device_node *target;
+};
+
 #ifdef CONFIG_OF_OVERLAY
 
 /* ID based overlays; the API for external users */
@@ -1273,6 +1285,9 @@ int of_overlay_create(struct device_node *tree);
 int of_overlay_destroy(int id);
 int of_overlay_destroy_all(void);
 
+int of_overlay_notifier_register(struct notifier_block *nb);
+int of_overlay_notifier_unregister(struct notifier_block *nb);
+
 #else
 
 static inline int of_overlay_create(struct device_node *tree)
@@ -1290,6 +1305,16 @@ static inline int of_overlay_destroy_all(void)
 	return -ENOTSUPP;
 }
 
+static inline int of_overlay_notifier_register(struct notifier_block *nb)
+{
+	return 0;
+}
+
+static inline int of_overlay_notifier_unregister(struct notifier_block *nb)
+{
+	return 0;
+}
+
 #endif
 
 #endif /* _LINUX_OF_H */
diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
index e80b9c762a03..6a7fc5051099 100644
--- a/include/linux/of_iommu.h
+++ b/include/linux/of_iommu.h
@@ -31,8 +31,16 @@ static inline const struct iommu_ops *of_iommu_configure(struct device *dev,
 
 #endif	/* CONFIG_OF_IOMMU */
 
-void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops);
-const struct iommu_ops *of_iommu_get_ops(struct device_node *np);
+static inline void of_iommu_set_ops(struct device_node *np,
+				    const struct iommu_ops *ops)
+{
+	iommu_register_instance(&np->fwnode, ops);
+}
+
+static inline const struct iommu_ops *of_iommu_get_ops(struct device_node *np)
+{
+	return iommu_get_instance(&np->fwnode);
+}
 
 extern struct of_device_id __iommu_of_table;
 
diff --git a/include/linux/of_pci.h b/include/linux/of_pci.h
index 7fd5cfce9140..0e0974eceb80 100644
--- a/include/linux/of_pci.h
+++ b/include/linux/of_pci.h
@@ -16,6 +16,7 @@ int of_pci_get_devfn(struct device_node *np);
 int of_irq_parse_and_map_pci(const struct pci_dev *dev, u8 slot, u8 pin);
 int of_pci_parse_bus_range(struct device_node *node, struct resource *res);
 int of_get_pci_domain_nr(struct device_node *node);
+int of_pci_get_max_link_speed(struct device_node *node);
 void of_pci_check_probe_only(void);
 int of_pci_map_rid(struct device_node *np, u32 rid,
 		   const char *map_name, const char *map_mask_name,
@@ -62,6 +63,12 @@ static inline int of_pci_map_rid(struct device_node *np, u32 rid,
 	return -EINVAL;
 }
 
+static inline int
+of_pci_get_max_link_speed(struct device_node *node)
+{
+	return -EINVAL;
+}
+
 static inline void of_pci_check_probe_only(void) { }
 #endif
 
diff --git a/include/linux/parser.h b/include/linux/parser.h
index 39d5b7955b23..884c1e6eb3fe 100644
--- a/include/linux/parser.h
+++ b/include/linux/parser.h
@@ -27,6 +27,7 @@ typedef struct {
 
 int match_token(char *, const match_table_t table, substring_t args[]);
 int match_int(substring_t *, int *result);
+int match_u64(substring_t *, u64 *result);
 int match_octal(substring_t *, int *result);
 int match_hex(substring_t *, int *result);
 bool match_wildcard(const char *pattern, const char *str);
diff --git a/include/linux/pci-acpi.h b/include/linux/pci-acpi.h
index 7d63a66e8ed4..7a4e83a8c89c 100644
--- a/include/linux/pci-acpi.h
+++ b/include/linux/pci-acpi.h
@@ -24,7 +24,9 @@ static inline acpi_status pci_acpi_remove_pm_notifier(struct acpi_device *dev)
 }
 extern phys_addr_t acpi_pci_root_get_mcfg_addr(acpi_handle handle);
 
-extern phys_addr_t pci_mcfg_lookup(u16 domain, struct resource *bus_res);
+struct pci_ecam_ops;
+extern int pci_mcfg_lookup(struct acpi_pci_root *root, struct resource *cfgres,
+			   struct pci_ecam_ops **ecam_ops);
 
 static inline acpi_handle acpi_find_root_bridge_handle(struct pci_dev *pdev)
 {
diff --git a/include/linux/pci-ecam.h b/include/linux/pci-ecam.h
index 7adad206b1f4..f0d2b9451270 100644
--- a/include/linux/pci-ecam.h
+++ b/include/linux/pci-ecam.h
@@ -59,6 +59,15 @@ void __iomem *pci_ecam_map_bus(struct pci_bus *bus, unsigned int devfn,
 /* default ECAM ops */
 extern struct pci_ecam_ops pci_generic_ecam_ops;
 
+#if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)
+extern struct pci_ecam_ops pci_32b_ops;		/* 32-bit accesses only */
+extern struct pci_ecam_ops hisi_pcie_ops;	/* HiSilicon */
+extern struct pci_ecam_ops thunder_pem_ecam_ops; /* Cavium ThunderX 1.x & 2.x */
+extern struct pci_ecam_ops pci_thunder_ecam_ops; /* Cavium ThunderX 1.x */
+extern struct pci_ecam_ops xgene_v1_pcie_ecam_ops; /* APM X-Gene PCIe v1 */
+extern struct pci_ecam_ops xgene_v2_pcie_ecam_ops; /* APM X-Gene PCIe v2.x */
+#endif
+
 #ifdef CONFIG_PCI_HOST_GENERIC
 /* for DT-based PCI controllers that support ECAM */
 int pci_host_common_probe(struct platform_device *pdev,
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 30d6c162e053..e2d1a124216a 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -420,9 +420,13 @@ static inline int pci_channel_offline(struct pci_dev *pdev)
 struct pci_host_bridge {
 	struct device dev;
 	struct pci_bus *bus;		/* root bus */
+	struct pci_ops *ops;
+	void *sysdata;
+	int busnr;
 	struct list_head windows;	/* resource_entry */
 	void (*release_fn)(struct pci_host_bridge *);
 	void *release_data;
+	struct msi_controller *msi;
 	unsigned int ignore_reset_delay:1;	/* for entire hierarchy */
 	/* Resource alignment requirements */
 	resource_size_t (*align_resource)(struct pci_dev *dev,
@@ -430,10 +434,23 @@ struct pci_host_bridge {
 			resource_size_t start,
 			resource_size_t size,
 			resource_size_t align);
+	unsigned long private[0] ____cacheline_aligned;
 };
 
 #define	to_pci_host_bridge(n) container_of(n, struct pci_host_bridge, dev)
 
+static inline void *pci_host_bridge_priv(struct pci_host_bridge *bridge)
+{
+	return (void *)bridge->private;
+}
+
+static inline struct pci_host_bridge *pci_host_bridge_from_priv(void *priv)
+{
+	return container_of(priv, struct pci_host_bridge, private);
+}
+
+struct pci_host_bridge *pci_alloc_host_bridge(size_t priv);
+int pci_register_host_bridge(struct pci_host_bridge *bridge);
 struct pci_host_bridge *pci_find_host_bridge(struct pci_bus *bus);
 
 void pci_set_host_bridge_release(struct pci_host_bridge *bridge,
diff --git a/include/linux/pci_hotplug.h b/include/linux/pci_hotplug.h
index 8c7895061121..2e855afa0212 100644
--- a/include/linux/pci_hotplug.h
+++ b/include/linux/pci_hotplug.h
@@ -176,6 +176,7 @@ struct hotplug_params {
 #ifdef CONFIG_ACPI
 #include <linux/acpi.h>
 int pci_get_hp_params(struct pci_dev *dev, struct hotplug_params *hpp);
+bool pciehp_is_native(struct pci_dev *pdev);
 int acpi_get_hp_hw_control_from_firmware(struct pci_dev *dev, u32 flags);
 int acpi_pci_check_ejectable(struct pci_bus *pbus, acpi_handle handle);
 int acpi_pci_detect_ejectable(acpi_handle handle);
@@ -185,5 +186,6 @@ static inline int pci_get_hp_params(struct pci_dev *dev,
 {
 	return -ENODEV;
 }
+static inline bool pciehp_is_native(struct pci_dev *pdev) { return true; }
 #endif
 #endif
diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h
index c58752fe16c4..73dda0edcb97 100644
--- a/include/linux/pci_ids.h
+++ b/include/linux/pci_ids.h
@@ -23,8 +23,10 @@
 #define PCI_CLASS_STORAGE_SATA		0x0106
 #define PCI_CLASS_STORAGE_SATA_AHCI	0x010601
 #define PCI_CLASS_STORAGE_SAS		0x0107
+#define PCI_CLASS_STORAGE_EXPRESS	0x010802
 #define PCI_CLASS_STORAGE_OTHER		0x0180
 
+
 #define PCI_BASE_CLASS_NETWORK		0x02
 #define PCI_CLASS_NETWORK_ETHERNET	0x0200
 #define PCI_CLASS_NETWORK_TOKEN_RING	0x0201
@@ -2251,17 +2253,35 @@
 #define PCI_DEVICE_ID_RASTEL_2PORT	0x2000
 
 #define PCI_VENDOR_ID_VMWARE		0x15ad
+#define PCI_DEVICE_ID_VMWARE_VMXNET3	0x07b0
 
 #define PCI_VENDOR_ID_ZOLTRIX		0x15b0
 #define PCI_DEVICE_ID_ZOLTRIX_2BD0	0x2bd0
 
 #define PCI_VENDOR_ID_MELLANOX		0x15b3
-#define PCI_DEVICE_ID_MELLANOX_TAVOR	0x5a44
+#define PCI_DEVICE_ID_MELLANOX_CONNECTX3	0x1003
+#define PCI_DEVICE_ID_MELLANOX_CONNECTX3_PRO	0x1007
+#define PCI_DEVICE_ID_MELLANOX_CONNECTIB	0x1011
+#define PCI_DEVICE_ID_MELLANOX_CONNECTX4	0x1013
+#define PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX	0x1015
+#define PCI_DEVICE_ID_MELLANOX_TAVOR		0x5a44
 #define PCI_DEVICE_ID_MELLANOX_TAVOR_BRIDGE	0x5a46
-#define PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT 0x6278
-#define PCI_DEVICE_ID_MELLANOX_ARBEL	0x6282
-#define PCI_DEVICE_ID_MELLANOX_SINAI_OLD 0x5e8c
-#define PCI_DEVICE_ID_MELLANOX_SINAI	0x6274
+#define PCI_DEVICE_ID_MELLANOX_SINAI_OLD	0x5e8c
+#define PCI_DEVICE_ID_MELLANOX_SINAI		0x6274
+#define PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT	0x6278
+#define PCI_DEVICE_ID_MELLANOX_ARBEL		0x6282
+#define PCI_DEVICE_ID_MELLANOX_HERMON_SDR	0x6340
+#define PCI_DEVICE_ID_MELLANOX_HERMON_DDR	0x634a
+#define PCI_DEVICE_ID_MELLANOX_HERMON_QDR	0x6354
+#define PCI_DEVICE_ID_MELLANOX_HERMON_EN	0x6368
+#define PCI_DEVICE_ID_MELLANOX_CONNECTX_EN	0x6372
+#define PCI_DEVICE_ID_MELLANOX_HERMON_DDR_GEN2	0x6732
+#define PCI_DEVICE_ID_MELLANOX_HERMON_QDR_GEN2	0x673c
+#define PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_5_GEN2 0x6746
+#define PCI_DEVICE_ID_MELLANOX_HERMON_EN_GEN2	0x6750
+#define PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_T_GEN2 0x675a
+#define PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_GEN2	0x6764
+#define PCI_DEVICE_ID_MELLANOX_CONNECTX2	0x676e
 
 #define PCI_VENDOR_ID_DFI		0x15bd
 
diff --git a/include/linux/phy/phy-qcom-ufs.h b/include/linux/phy/phy-qcom-ufs.h
index 9d18e9f948e9..35c070ea6ea3 100644
--- a/include/linux/phy/phy-qcom-ufs.h
+++ b/include/linux/phy/phy-qcom-ufs.h
@@ -18,22 +18,6 @@
 #include "phy.h"
 
 /**
- * ufs_qcom_phy_enable_ref_clk() - Enable the phy
- * ref clock.
- * @phy: reference to a generic phy
- *
- * returns 0 for success, and non-zero for error.
- */
-int ufs_qcom_phy_enable_ref_clk(struct phy *phy);
-
-/**
- * ufs_qcom_phy_disable_ref_clk() - Disable the phy
- * ref clock.
- * @phy: reference to a generic phy.
- */
-void ufs_qcom_phy_disable_ref_clk(struct phy *phy);
-
-/**
  * ufs_qcom_phy_enable_dev_ref_clk() - Enable the device
  * ref clock.
  * @phy: reference to a generic phy.
@@ -47,8 +31,6 @@ void ufs_qcom_phy_enable_dev_ref_clk(struct phy *phy);
  */
 void ufs_qcom_phy_disable_dev_ref_clk(struct phy *phy);
 
-int ufs_qcom_phy_enable_iface_clk(struct phy *phy);
-void ufs_qcom_phy_disable_iface_clk(struct phy *phy);
 int ufs_qcom_phy_start_serdes(struct phy *phy);
 int ufs_qcom_phy_set_tx_lane_enable(struct phy *phy, u32 tx_lanes);
 int ufs_qcom_phy_calibrate_phy(struct phy *phy, bool is_rate_B);
diff --git a/include/linux/platform_data/dma-dw.h b/include/linux/platform_data/dma-dw.h
index 5f0e11e7354c..e69e415d0d98 100644
--- a/include/linux/platform_data/dma-dw.h
+++ b/include/linux/platform_data/dma-dw.h
@@ -14,6 +14,7 @@
 #include <linux/device.h>
 
 #define DW_DMA_MAX_NR_MASTERS	4
+#define DW_DMA_MAX_NR_CHANNELS	8
 
 /**
  * struct dw_dma_slave - Controller-specific information about a slave
@@ -40,19 +41,18 @@ struct dw_dma_slave {
  * @is_private: The device channels should be marked as private and not for
  *	by the general purpose DMA channel allocator.
  * @is_memcpy: The device channels do support memory-to-memory transfers.
- * @is_nollp: The device channels does not support multi block transfers.
  * @chan_allocation_order: Allocate channels starting from 0 or 7
  * @chan_priority: Set channel priority increasing from 0 to 7 or 7 to 0.
  * @block_size: Maximum block size supported by the controller
  * @nr_masters: Number of AHB masters supported by the controller
  * @data_width: Maximum data width supported by hardware per AHB master
  *		(in bytes, power of 2)
+ * @multi_block: Multi block transfers supported by hardware per channel.
  */
 struct dw_dma_platform_data {
 	unsigned int	nr_channels;
 	bool		is_private;
 	bool		is_memcpy;
-	bool		is_nollp;
 #define CHAN_ALLOCATION_ASCENDING	0	/* zero to seven */
 #define CHAN_ALLOCATION_DESCENDING	1	/* seven to zero */
 	unsigned char	chan_allocation_order;
@@ -62,6 +62,7 @@ struct dw_dma_platform_data {
 	unsigned int	block_size;
 	unsigned char	nr_masters;
 	unsigned char	data_width[DW_DMA_MAX_NR_MASTERS];
+	unsigned char	multi_block[DW_DMA_MAX_NR_CHANNELS];
 };
 
 #endif /* _PLATFORM_DATA_DMA_DW_H */
diff --git a/include/linux/platform_data/mlxcpld-hotplug.h b/include/linux/platform_data/mlxcpld-hotplug.h
new file mode 100644
index 000000000000..e4cfcffaa6f4
--- /dev/null
+++ b/include/linux/platform_data/mlxcpld-hotplug.h
@@ -0,0 +1,99 @@
+/*
+ * include/linux/platform_data/mlxcpld-hotplug.h
+ * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2016 Vadim Pasternak <vadimp@mellanox.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the names of the copyright holders nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __LINUX_PLATFORM_DATA_MLXCPLD_HOTPLUG_H
+#define __LINUX_PLATFORM_DATA_MLXCPLD_HOTPLUG_H
+
+/**
+ * struct mlxcpld_hotplug_device - I2C device data:
+ * @adapter: I2C device adapter;
+ * @client: I2C device client;
+ * @brdinfo: device board information;
+ * @bus: I2C bus, where device is attached;
+ *
+ * Structure represents I2C hotplug device static data (board topology) and
+ * dynamic data (related kernel objects handles).
+ */
+struct mlxcpld_hotplug_device {
+	struct i2c_adapter *adapter;
+	struct i2c_client *client;
+	struct i2c_board_info brdinfo;
+	u16 bus;
+};
+
+/**
+ * struct mlxcpld_hotplug_platform_data - device platform data:
+ * @top_aggr_offset: offset of top aggregation interrupt register;
+ * @top_aggr_mask: top aggregation interrupt common mask;
+ * @top_aggr_psu_mask: top aggregation interrupt PSU mask;
+ * @psu_reg_offset: offset of PSU interrupt register;
+ * @psu_mask: PSU interrupt mask;
+ * @psu_count: number of equipped replaceable PSUs;
+ * @psu: pointer to PSU devices data array;
+ * @top_aggr_pwr_mask: top aggregation interrupt power mask;
+ * @pwr_reg_offset: offset of power interrupt register
+ * @pwr_mask: power interrupt mask;
+ * @pwr_count: number of power sources;
+ * @pwr: pointer to power devices data array;
+ * @top_aggr_fan_mask: top aggregation interrupt FAN mask;
+ * @fan_reg_offset: offset of FAN interrupt register;
+ * @fan_mask: FAN interrupt mask;
+ * @fan_count: number of equipped replaceable FANs;
+ * @fan: pointer to FAN devices data array;
+ *
+ * Structure represents board platform data, related to system hotplug events,
+ * like FAN, PSU, power cable insertion and removing. This data provides the
+ * number of hot-pluggable devices and hardware description for event handling.
+ */
+struct mlxcpld_hotplug_platform_data {
+	u16 top_aggr_offset;
+	u8 top_aggr_mask;
+	u8 top_aggr_psu_mask;
+	u16 psu_reg_offset;
+	u8 psu_mask;
+	u8 psu_count;
+	struct mlxcpld_hotplug_device *psu;
+	u8 top_aggr_pwr_mask;
+	u16 pwr_reg_offset;
+	u8 pwr_mask;
+	u8 pwr_count;
+	struct mlxcpld_hotplug_device *pwr;
+	u8 top_aggr_fan_mask;
+	u16 fan_reg_offset;
+	u8 fan_mask;
+	u8 fan_count;
+	struct mlxcpld_hotplug_device *fan;
+};
+
+#endif /* __LINUX_PLATFORM_DATA_MLXCPLD_HOTPLUG_H */
diff --git a/include/linux/platform_data/spi-s3c64xx.h b/include/linux/platform_data/spi-s3c64xx.h
index 5c1e21c87270..da79774078a7 100644
--- a/include/linux/platform_data/spi-s3c64xx.h
+++ b/include/linux/platform_data/spi-s3c64xx.h
@@ -40,9 +40,6 @@ struct s3c64xx_spi_info {
 	int num_cs;
 	bool no_cs;
 	int (*cfg_gpio)(void);
-	dma_filter_fn filter;
-	void *dma_tx;
-	void *dma_rx;
 };
 
 /**
diff --git a/include/linux/platform_data/usb-davinci.h b/include/linux/platform_data/usb-davinci.h
index e0bc4abe69c2..0926e99f2e8f 100644
--- a/include/linux/platform_data/usb-davinci.h
+++ b/include/linux/platform_data/usb-davinci.h
@@ -11,29 +11,6 @@
 #ifndef __ASM_ARCH_USB_H
 #define __ASM_ARCH_USB_H
 
-/* DA8xx CFGCHIP2 (USB 2.0 PHY Control) register bits */
-#define CFGCHIP2_PHYCLKGD	(1 << 17)
-#define CFGCHIP2_VBUSSENSE	(1 << 16)
-#define CFGCHIP2_RESET		(1 << 15)
-#define CFGCHIP2_OTGMODE	(3 << 13)
-#define CFGCHIP2_NO_OVERRIDE	(0 << 13)
-#define CFGCHIP2_FORCE_HOST	(1 << 13)
-#define CFGCHIP2_FORCE_DEVICE 	(2 << 13)
-#define CFGCHIP2_FORCE_HOST_VBUS_LOW (3 << 13)
-#define CFGCHIP2_USB1PHYCLKMUX	(1 << 12)
-#define CFGCHIP2_USB2PHYCLKMUX	(1 << 11)
-#define CFGCHIP2_PHYPWRDN	(1 << 10)
-#define CFGCHIP2_OTGPWRDN	(1 << 9)
-#define CFGCHIP2_DATPOL 	(1 << 8)
-#define CFGCHIP2_USB1SUSPENDM	(1 << 7)
-#define CFGCHIP2_PHY_PLLON	(1 << 6)	/* override PLL suspend */
-#define CFGCHIP2_SESENDEN	(1 << 5)	/* Vsess_end comparator */
-#define CFGCHIP2_VBDTCTEN	(1 << 4)	/* Vbus comparator */
-#define CFGCHIP2_REFFREQ	(0xf << 0)
-#define CFGCHIP2_REFFREQ_12MHZ	(1 << 0)
-#define CFGCHIP2_REFFREQ_24MHZ	(2 << 0)
-#define CFGCHIP2_REFFREQ_48MHZ	(3 << 0)
-
 struct	da8xx_ohci_root_hub;
 
 typedef void (*da8xx_ocic_handler_t)(struct da8xx_ohci_root_hub *hub,
diff --git a/include/linux/pm.h b/include/linux/pm.h
index efa67b2dfee9..f926af41e122 100644
--- a/include/linux/pm.h
+++ b/include/linux/pm.h
@@ -559,6 +559,7 @@ struct dev_pm_info {
 	pm_message_t		power_state;
 	unsigned int		can_wakeup:1;
 	unsigned int		async_suspend:1;
+	bool			in_dpm_list:1;	/* Owned by the PM core */
 	bool			is_prepared:1;	/* Owned by the PM core */
 	bool			is_suspended:1;	/* Ditto */
 	bool			is_noirq_suspended:1;
@@ -596,6 +597,7 @@ struct dev_pm_info {
 	unsigned int		use_autosuspend:1;
 	unsigned int		timer_autosuspends:1;
 	unsigned int		memalloc_noio:1;
+	unsigned int		links_count;
 	enum rpm_request	request;
 	enum rpm_status		runtime_status;
 	int			runtime_error;
diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h
index a09fe5c009c8..81ece61075df 100644
--- a/include/linux/pm_domain.h
+++ b/include/linux/pm_domain.h
@@ -15,11 +15,11 @@
 #include <linux/err.h>
 #include <linux/of.h>
 #include <linux/notifier.h>
+#include <linux/spinlock.h>
 
 /* Defines used for the flags field in the struct generic_pm_domain */
 #define GENPD_FLAG_PM_CLK	(1U << 0) /* PM domain uses PM clk */
-
-#define GENPD_MAX_NUM_STATES	8 /* Number of possible low power states */
+#define GENPD_FLAG_IRQ_SAFE	(1U << 1) /* PM domain operates in atomic */
 
 enum gpd_status {
 	GPD_STATE_ACTIVE = 0,	/* PM domain is active */
@@ -40,15 +40,18 @@ struct gpd_dev_ops {
 struct genpd_power_state {
 	s64 power_off_latency_ns;
 	s64 power_on_latency_ns;
+	s64 residency_ns;
+	struct fwnode_handle *fwnode;
 };
 
+struct genpd_lock_ops;
+
 struct generic_pm_domain {
 	struct dev_pm_domain domain;	/* PM domain operations */
 	struct list_head gpd_list_node;	/* Node in the global PM domains list */
 	struct list_head master_links;	/* Links with PM domain as a master */
 	struct list_head slave_links;	/* Links with PM domain as a slave */
 	struct list_head dev_list;	/* List of devices */
-	struct mutex lock;
 	struct dev_power_governor *gov;
 	struct work_struct power_off_work;
 	struct fwnode_handle *provider;	/* Identity of the domain provider */
@@ -70,9 +73,18 @@ struct generic_pm_domain {
 	void (*detach_dev)(struct generic_pm_domain *domain,
 			   struct device *dev);
 	unsigned int flags;		/* Bit field of configs for genpd */
-	struct genpd_power_state states[GENPD_MAX_NUM_STATES];
+	struct genpd_power_state *states;
 	unsigned int state_count; /* number of states */
 	unsigned int state_idx; /* state that genpd will go to when off */
+	void *free; /* Free the state that was allocated for default */
+	const struct genpd_lock_ops *lock_ops;
+	union {
+		struct mutex mlock;
+		struct {
+			spinlock_t slock;
+			unsigned long lock_flags;
+		};
+	};
 
 };
 
@@ -205,6 +217,8 @@ extern int of_genpd_add_device(struct of_phandle_args *args,
 extern int of_genpd_add_subdomain(struct of_phandle_args *parent,
 				  struct of_phandle_args *new_subdomain);
 extern struct generic_pm_domain *of_genpd_remove_last(struct device_node *np);
+extern int of_genpd_parse_idle_states(struct device_node *dn,
+			struct genpd_power_state **states, int *n);
 
 int genpd_dev_pm_attach(struct device *dev);
 #else /* !CONFIG_PM_GENERIC_DOMAINS_OF */
@@ -234,6 +248,12 @@ static inline int of_genpd_add_subdomain(struct of_phandle_args *parent,
 	return -ENODEV;
 }
 
+static inline int of_genpd_parse_idle_states(struct device_node *dn,
+			struct genpd_power_state **states, int *n)
+{
+	return -ENODEV;
+}
+
 static inline int genpd_dev_pm_attach(struct device *dev)
 {
 	return -ENODEV;
diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h
index bca26157f5b6..0edd88f93904 100644
--- a/include/linux/pm_opp.h
+++ b/include/linux/pm_opp.h
@@ -17,13 +17,65 @@
 #include <linux/err.h>
 #include <linux/notifier.h>
 
+struct clk;
+struct regulator;
 struct dev_pm_opp;
 struct device;
+struct opp_table;
 
 enum dev_pm_opp_event {
 	OPP_EVENT_ADD, OPP_EVENT_REMOVE, OPP_EVENT_ENABLE, OPP_EVENT_DISABLE,
 };
 
+/**
+ * struct dev_pm_opp_supply - Power supply voltage/current values
+ * @u_volt:	Target voltage in microvolts corresponding to this OPP
+ * @u_volt_min:	Minimum voltage in microvolts corresponding to this OPP
+ * @u_volt_max:	Maximum voltage in microvolts corresponding to this OPP
+ * @u_amp:	Maximum current drawn by the device in microamperes
+ *
+ * This structure stores the voltage/current values for a single power supply.
+ */
+struct dev_pm_opp_supply {
+	unsigned long u_volt;
+	unsigned long u_volt_min;
+	unsigned long u_volt_max;
+	unsigned long u_amp;
+};
+
+/**
+ * struct dev_pm_opp_info - OPP freq/voltage/current values
+ * @rate:	Target clk rate in hz
+ * @supplies:	Array of voltage/current values for all power supplies
+ *
+ * This structure stores the freq/voltage/current values for a single OPP.
+ */
+struct dev_pm_opp_info {
+	unsigned long rate;
+	struct dev_pm_opp_supply *supplies;
+};
+
+/**
+ * struct dev_pm_set_opp_data - Set OPP data
+ * @old_opp:	Old OPP info
+ * @new_opp:	New OPP info
+ * @regulators:	Array of regulator pointers
+ * @regulator_count: Number of regulators
+ * @clk:	Pointer to clk
+ * @dev:	Pointer to the struct device
+ *
+ * This structure contains all information required for setting an OPP.
+ */
+struct dev_pm_set_opp_data {
+	struct dev_pm_opp_info old_opp;
+	struct dev_pm_opp_info new_opp;
+
+	struct regulator **regulators;
+	unsigned int regulator_count;
+	struct clk *clk;
+	struct device *dev;
+};
+
 #if defined(CONFIG_PM_OPP)
 
 unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp);
@@ -62,8 +114,10 @@ int dev_pm_opp_set_supported_hw(struct device *dev, const u32 *versions,
 void dev_pm_opp_put_supported_hw(struct device *dev);
 int dev_pm_opp_set_prop_name(struct device *dev, const char *name);
 void dev_pm_opp_put_prop_name(struct device *dev);
-int dev_pm_opp_set_regulator(struct device *dev, const char *name);
-void dev_pm_opp_put_regulator(struct device *dev);
+struct opp_table *dev_pm_opp_set_regulators(struct device *dev, const char * const names[], unsigned int count);
+void dev_pm_opp_put_regulators(struct opp_table *opp_table);
+int dev_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data));
+void dev_pm_opp_register_put_opp_helper(struct device *dev);
 int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq);
 int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, const struct cpumask *cpumask);
 int dev_pm_opp_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask);
@@ -163,6 +217,14 @@ static inline int dev_pm_opp_set_supported_hw(struct device *dev,
 
 static inline void dev_pm_opp_put_supported_hw(struct device *dev) {}
 
+static inline int dev_pm_opp_register_set_opp_helper(struct device *dev,
+			int (*set_opp)(struct dev_pm_set_opp_data *data))
+{
+	return -ENOTSUPP;
+}
+
+static inline void dev_pm_opp_register_put_opp_helper(struct device *dev) {}
+
 static inline int dev_pm_opp_set_prop_name(struct device *dev, const char *name)
 {
 	return -ENOTSUPP;
@@ -170,12 +232,12 @@ static inline int dev_pm_opp_set_prop_name(struct device *dev, const char *name)
 
 static inline void dev_pm_opp_put_prop_name(struct device *dev) {}
 
-static inline int dev_pm_opp_set_regulator(struct device *dev, const char *name)
+static inline struct opp_table *dev_pm_opp_set_regulators(struct device *dev, const char * const names[], unsigned int count)
 {
-	return -ENOTSUPP;
+	return ERR_PTR(-ENOTSUPP);
 }
 
-static inline void dev_pm_opp_put_regulator(struct device *dev) {}
+static inline void dev_pm_opp_put_regulators(struct opp_table *opp_table) {}
 
 static inline int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq)
 {
diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h
index 2e14d2667b6c..ca4823e675e2 100644
--- a/include/linux/pm_runtime.h
+++ b/include/linux/pm_runtime.h
@@ -55,18 +55,17 @@ extern unsigned long pm_runtime_autosuspend_expiration(struct device *dev);
 extern void pm_runtime_update_max_time_suspended(struct device *dev,
 						 s64 delta_ns);
 extern void pm_runtime_set_memalloc_noio(struct device *dev, bool enable);
+extern void pm_runtime_clean_up_links(struct device *dev);
+extern void pm_runtime_get_suppliers(struct device *dev);
+extern void pm_runtime_put_suppliers(struct device *dev);
+extern void pm_runtime_new_link(struct device *dev);
+extern void pm_runtime_drop_link(struct device *dev);
 
 static inline void pm_suspend_ignore_children(struct device *dev, bool enable)
 {
 	dev->power.ignore_children = enable;
 }
 
-static inline bool pm_children_suspended(struct device *dev)
-{
-	return dev->power.ignore_children
-		|| !atomic_read(&dev->power.child_count);
-}
-
 static inline void pm_runtime_get_noresume(struct device *dev)
 {
 	atomic_inc(&dev->power.usage_count);
@@ -162,7 +161,6 @@ static inline void pm_runtime_allow(struct device *dev) {}
 static inline void pm_runtime_forbid(struct device *dev) {}
 
 static inline void pm_suspend_ignore_children(struct device *dev, bool enable) {}
-static inline bool pm_children_suspended(struct device *dev) { return false; }
 static inline void pm_runtime_get_noresume(struct device *dev) {}
 static inline void pm_runtime_put_noidle(struct device *dev) {}
 static inline bool device_run_wake(struct device *dev) { return false; }
@@ -186,6 +184,11 @@ static inline unsigned long pm_runtime_autosuspend_expiration(
 				struct device *dev) { return 0; }
 static inline void pm_runtime_set_memalloc_noio(struct device *dev,
 						bool enable){}
+static inline void pm_runtime_clean_up_links(struct device *dev) {}
+static inline void pm_runtime_get_suppliers(struct device *dev) {}
+static inline void pm_runtime_put_suppliers(struct device *dev) {}
+static inline void pm_runtime_new_link(struct device *dev) {}
+static inline void pm_runtime_drop_link(struct device *dev) {}
 
 #endif /* !CONFIG_PM */
 
@@ -265,9 +268,9 @@ static inline int pm_runtime_set_active(struct device *dev)
 	return __pm_runtime_set_status(dev, RPM_ACTIVE);
 }
 
-static inline void pm_runtime_set_suspended(struct device *dev)
+static inline int pm_runtime_set_suspended(struct device *dev)
 {
-	__pm_runtime_set_status(dev, RPM_SUSPENDED);
+	return __pm_runtime_set_status(dev, RPM_SUSPENDED);
 }
 
 static inline void pm_runtime_disable(struct device *dev)
diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h
index e30deb046156..bed9557b69e7 100644
--- a/include/linux/power/bq27xxx_battery.h
+++ b/include/linux/power/bq27xxx_battery.h
@@ -4,7 +4,8 @@
 enum bq27xxx_chip {
 	BQ27000 = 1, /* bq27000, bq27200 */
 	BQ27010, /* bq27010, bq27210 */
-	BQ27500, /* bq27500, bq27510, bq27520 */
+	BQ27500, /* bq27500 */
+	BQ27510, /* bq27510, bq27520 */
 	BQ27530, /* bq27530, bq27531 */
 	BQ27541, /* bq27541, bq27542, bq27546, bq27742 */
 	BQ27545, /* bq27545 */
diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h
index 368c7ad06ae5..2d2bf592d9db 100644
--- a/include/linux/proc_fs.h
+++ b/include/linux/proc_fs.h
@@ -21,6 +21,7 @@ extern struct proc_dir_entry *proc_mkdir_data(const char *, umode_t,
 					      struct proc_dir_entry *, void *);
 extern struct proc_dir_entry *proc_mkdir_mode(const char *, umode_t,
 					      struct proc_dir_entry *);
+struct proc_dir_entry *proc_create_mount_point(const char *name);
  
 extern struct proc_dir_entry *proc_create_data(const char *, umode_t,
 					       struct proc_dir_entry *,
@@ -56,6 +57,7 @@ static inline struct proc_dir_entry *proc_symlink(const char *name,
 		struct proc_dir_entry *parent,const char *dest) { return NULL;}
 static inline struct proc_dir_entry *proc_mkdir(const char *name,
 	struct proc_dir_entry *parent) {return NULL;}
+static inline struct proc_dir_entry *proc_create_mount_point(const char *name) { return NULL; }
 static inline struct proc_dir_entry *proc_mkdir_data(const char *name,
 	umode_t mode, struct proc_dir_entry *parent, void *data) { return NULL; }
 static inline struct proc_dir_entry *proc_mkdir_mode(const char *name,
diff --git a/include/linux/pstore.h b/include/linux/pstore.h
index 92013cc9cc8c..0da29cae009b 100644
--- a/include/linux/pstore.h
+++ b/include/linux/pstore.h
@@ -89,4 +89,80 @@ extern int pstore_register(struct pstore_info *);
 extern void pstore_unregister(struct pstore_info *);
 extern bool pstore_cannot_block_path(enum kmsg_dump_reason reason);
 
+struct pstore_ftrace_record {
+	unsigned long ip;
+	unsigned long parent_ip;
+	u64 ts;
+};
+
+/*
+ * ftrace related stuff: Both backends and frontends need these so expose
+ * them here.
+ */
+
+#if NR_CPUS <= 2 && defined(CONFIG_ARM_THUMB)
+#define PSTORE_CPU_IN_IP 0x1
+#elif NR_CPUS <= 4 && defined(CONFIG_ARM)
+#define PSTORE_CPU_IN_IP 0x3
+#endif
+
+#define TS_CPU_SHIFT 8
+#define TS_CPU_MASK (BIT(TS_CPU_SHIFT) - 1)
+
+/*
+ * If CPU number can be stored in IP, store it there, otherwise store it in
+ * the time stamp. This means more timestamp resolution is available when
+ * the CPU can be stored in the IP.
+ */
+#ifdef PSTORE_CPU_IN_IP
+static inline void
+pstore_ftrace_encode_cpu(struct pstore_ftrace_record *rec, unsigned int cpu)
+{
+	rec->ip |= cpu;
+}
+
+static inline unsigned int
+pstore_ftrace_decode_cpu(struct pstore_ftrace_record *rec)
+{
+	return rec->ip & PSTORE_CPU_IN_IP;
+}
+
+static inline u64
+pstore_ftrace_read_timestamp(struct pstore_ftrace_record *rec)
+{
+	return rec->ts;
+}
+
+static inline void
+pstore_ftrace_write_timestamp(struct pstore_ftrace_record *rec, u64 val)
+{
+	rec->ts = val;
+}
+#else
+static inline void
+pstore_ftrace_encode_cpu(struct pstore_ftrace_record *rec, unsigned int cpu)
+{
+	rec->ts &= ~(TS_CPU_MASK);
+	rec->ts |= cpu;
+}
+
+static inline unsigned int
+pstore_ftrace_decode_cpu(struct pstore_ftrace_record *rec)
+{
+	return rec->ts & TS_CPU_MASK;
+}
+
+static inline u64
+pstore_ftrace_read_timestamp(struct pstore_ftrace_record *rec)
+{
+	return rec->ts >> TS_CPU_SHIFT;
+}
+
+static inline void
+pstore_ftrace_write_timestamp(struct pstore_ftrace_record *rec, u64 val)
+{
+	rec->ts = (rec->ts & TS_CPU_MASK) | (val << TS_CPU_SHIFT);
+}
+#endif
+
 #endif /*_LINUX_PSTORE_H*/
diff --git a/include/linux/pstore_ram.h b/include/linux/pstore_ram.h
index c668c861c96c..9395f06e8372 100644
--- a/include/linux/pstore_ram.h
+++ b/include/linux/pstore_ram.h
@@ -24,6 +24,13 @@
 #include <linux/list.h>
 #include <linux/types.h>
 
+/*
+ * Choose whether access to the RAM zone requires locking or not.  If a zone
+ * can be written to from different CPUs like with ftrace for example, then
+ * PRZ_FLAG_NO_LOCK is used. For all other cases, locking is required.
+ */
+#define PRZ_FLAG_NO_LOCK	BIT(0)
+
 struct persistent_ram_buffer;
 struct rs_control;
 
@@ -40,6 +47,8 @@ struct persistent_ram_zone {
 	void *vaddr;
 	struct persistent_ram_buffer *buffer;
 	size_t buffer_size;
+	u32 flags;
+	raw_spinlock_t buffer_lock;
 
 	/* ECC correction */
 	char *par_buffer;
@@ -55,7 +64,7 @@ struct persistent_ram_zone {
 
 struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size,
 			u32 sig, struct persistent_ram_ecc_info *ecc_info,
-			unsigned int memtype);
+			unsigned int memtype, u32 flags);
 void persistent_ram_free(struct persistent_ram_zone *prz);
 void persistent_ram_zap(struct persistent_ram_zone *prz);
 
@@ -77,6 +86,8 @@ ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz,
  * @mem_address	physical memory address to contain ramoops
  */
 
+#define RAMOOPS_FLAG_FTRACE_PER_CPU	BIT(0)
+
 struct ramoops_platform_data {
 	unsigned long	mem_size;
 	phys_addr_t	mem_address;
@@ -86,6 +97,7 @@ struct ramoops_platform_data {
 	unsigned long	ftrace_size;
 	unsigned long	pmsg_size;
 	int		dump_oops;
+	u32		flags;
 	struct persistent_ram_ecc_info ecc_info;
 };
 
diff --git a/include/linux/ptrace.h b/include/linux/ptrace.h
index 504c98a278d4..e0e539321ab9 100644
--- a/include/linux/ptrace.h
+++ b/include/linux/ptrace.h
@@ -8,6 +8,9 @@
 #include <linux/pid_namespace.h>	/* For task_active_pid_ns.  */
 #include <uapi/linux/ptrace.h>
 
+extern int ptrace_access_vm(struct task_struct *tsk, unsigned long addr,
+			    void *buf, int len, unsigned int gup_flags);
+
 /*
  * Ptrace flags
  *
@@ -19,7 +22,6 @@
 #define PT_SEIZED	0x00010000	/* SEIZE used, enable new behavior */
 #define PT_PTRACED	0x00000001
 #define PT_DTRACE	0x00000002	/* delayed trace (used on m68k, i386) */
-#define PT_PTRACE_CAP	0x00000004	/* ptracer can follow suid-exec */
 
 #define PT_OPT_FLAG_SHIFT	3
 /* PT_TRACE_* event enable flags */
diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h
index 744486057e9e..5dea8f6440e4 100644
--- a/include/linux/radix-tree.h
+++ b/include/linux/radix-tree.h
@@ -80,23 +80,25 @@ static inline bool radix_tree_is_internal_node(void *ptr)
 #define RADIX_TREE_MAX_PATH (DIV_ROUND_UP(RADIX_TREE_INDEX_BITS, \
 					  RADIX_TREE_MAP_SHIFT))
 
+/*
+ * @count is the count of every non-NULL element in the ->slots array
+ * whether that is an exceptional entry, a retry entry, a user pointer,
+ * a sibling entry or a pointer to the next level of the tree.
+ * @exceptional is the count of every element in ->slots which is
+ * either radix_tree_exceptional_entry() or is a sibling entry for an
+ * exceptional entry.
+ */
 struct radix_tree_node {
 	unsigned char	shift;		/* Bits remaining in each slot */
 	unsigned char	offset;		/* Slot offset in parent */
 	unsigned char	count;		/* Total entry count */
 	unsigned char	exceptional;	/* Exceptional entry count */
+	struct radix_tree_node *parent;		/* Used when ascending tree */
+	void *private_data;			/* For tree user */
 	union {
-		struct {
-			/* Used when ascending tree */
-			struct radix_tree_node *parent;
-			/* For tree user */
-			void *private_data;
-		};
-		/* Used when freeing node */
-		struct rcu_head	rcu_head;
+		struct list_head private_list;	/* For tree user */
+		struct rcu_head	rcu_head;	/* Used when freeing node */
 	};
-	/* For tree user */
-	struct list_head private_list;
 	void __rcu	*slots[RADIX_TREE_MAP_SIZE];
 	unsigned long	tags[RADIX_TREE_MAX_TAGS][RADIX_TREE_TAG_LONGS];
 };
@@ -127,6 +129,41 @@ static inline bool radix_tree_empty(struct radix_tree_root *root)
 }
 
 /**
+ * struct radix_tree_iter - radix tree iterator state
+ *
+ * @index:	index of current slot
+ * @next_index:	one beyond the last index for this chunk
+ * @tags:	bit-mask for tag-iterating
+ * @node:	node that contains current slot
+ * @shift:	shift for the node that holds our slots
+ *
+ * This radix tree iterator works in terms of "chunks" of slots.  A chunk is a
+ * subinterval of slots contained within one radix tree leaf node.  It is
+ * described by a pointer to its first slot and a struct radix_tree_iter
+ * which holds the chunk's position in the tree and its size.  For tagged
+ * iteration radix_tree_iter also holds the slots' bit-mask for one chosen
+ * radix tree tag.
+ */
+struct radix_tree_iter {
+	unsigned long	index;
+	unsigned long	next_index;
+	unsigned long	tags;
+	struct radix_tree_node *node;
+#ifdef CONFIG_RADIX_TREE_MULTIORDER
+	unsigned int	shift;
+#endif
+};
+
+static inline unsigned int iter_shift(const struct radix_tree_iter *iter)
+{
+#ifdef CONFIG_RADIX_TREE_MULTIORDER
+	return iter->shift;
+#else
+	return 0;
+#endif
+}
+
+/**
  * Radix-tree synchronization
  *
  * The radix-tree API requires that users provide all synchronisation (with
@@ -264,6 +301,8 @@ void __radix_tree_replace(struct radix_tree_root *root,
 			  struct radix_tree_node *node,
 			  void **slot, void *item,
 			  radix_tree_update_node_t update_node, void *private);
+void radix_tree_iter_replace(struct radix_tree_root *,
+		const struct radix_tree_iter *, void **slot, void *item);
 void radix_tree_replace_slot(struct radix_tree_root *root,
 			     void **slot, void *item);
 void __radix_tree_delete_node(struct radix_tree_root *root,
@@ -289,6 +328,8 @@ void *radix_tree_tag_clear(struct radix_tree_root *root,
 			unsigned long index, unsigned int tag);
 int radix_tree_tag_get(struct radix_tree_root *root,
 			unsigned long index, unsigned int tag);
+void radix_tree_iter_tag_set(struct radix_tree_root *root,
+		const struct radix_tree_iter *iter, unsigned int tag);
 unsigned int
 radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results,
 		unsigned long first_index, unsigned int max_items,
@@ -297,50 +338,18 @@ unsigned int
 radix_tree_gang_lookup_tag_slot(struct radix_tree_root *root, void ***results,
 		unsigned long first_index, unsigned int max_items,
 		unsigned int tag);
-unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
-		unsigned long *first_indexp, unsigned long last_index,
-		unsigned long nr_to_tag,
-		unsigned int fromtag, unsigned int totag);
 int radix_tree_tagged(struct radix_tree_root *root, unsigned int tag);
-unsigned long radix_tree_locate_item(struct radix_tree_root *root, void *item);
 
 static inline void radix_tree_preload_end(void)
 {
 	preempt_enable();
 }
 
-/**
- * struct radix_tree_iter - radix tree iterator state
- *
- * @index:	index of current slot
- * @next_index:	one beyond the last index for this chunk
- * @tags:	bit-mask for tag-iterating
- * @shift:	shift for the node that holds our slots
- *
- * This radix tree iterator works in terms of "chunks" of slots.  A chunk is a
- * subinterval of slots contained within one radix tree leaf node.  It is
- * described by a pointer to its first slot and a struct radix_tree_iter
- * which holds the chunk's position in the tree and its size.  For tagged
- * iteration radix_tree_iter also holds the slots' bit-mask for one chosen
- * radix tree tag.
- */
-struct radix_tree_iter {
-	unsigned long	index;
-	unsigned long	next_index;
-	unsigned long	tags;
-#ifdef CONFIG_RADIX_TREE_MULTIORDER
-	unsigned int	shift;
-#endif
-};
-
-static inline unsigned int iter_shift(struct radix_tree_iter *iter)
-{
-#ifdef CONFIG_RADIX_TREE_MULTIORDER
-	return iter->shift;
-#else
-	return 0;
-#endif
-}
+int radix_tree_split_preload(unsigned old_order, unsigned new_order, gfp_t);
+int radix_tree_split(struct radix_tree_root *, unsigned long index,
+			unsigned new_order);
+int radix_tree_join(struct radix_tree_root *, unsigned long index,
+			unsigned new_order, void *);
 
 #define RADIX_TREE_ITER_TAG_MASK	0x00FF	/* tag index in lower byte */
 #define RADIX_TREE_ITER_TAGGED		0x0100	/* lookup tagged slots */
@@ -409,20 +418,17 @@ __radix_tree_iter_add(struct radix_tree_iter *iter, unsigned long slots)
 }
 
 /**
- * radix_tree_iter_next - resume iterating when the chunk may be invalid
- * @iter:	iterator state
+ * radix_tree_iter_resume - resume iterating when the chunk may be invalid
+ * @slot: pointer to current slot
+ * @iter: iterator state
+ * Returns: New slot pointer
  *
  * If the iterator needs to release then reacquire a lock, the chunk may
  * have been invalidated by an insertion or deletion.  Call this function
- * to continue the iteration from the next index.
+ * before releasing the lock to continue the iteration from the next index.
  */
-static inline __must_check
-void **radix_tree_iter_next(struct radix_tree_iter *iter)
-{
-	iter->next_index = __radix_tree_iter_add(iter, 1);
-	iter->tags = 0;
-	return NULL;
-}
+void **__must_check radix_tree_iter_resume(void **slot,
+					struct radix_tree_iter *iter);
 
 /**
  * radix_tree_chunk_size - get current chunk size
@@ -436,10 +442,17 @@ radix_tree_chunk_size(struct radix_tree_iter *iter)
 	return (iter->next_index - iter->index) >> iter_shift(iter);
 }
 
-static inline struct radix_tree_node *entry_to_node(void *ptr)
+#ifdef CONFIG_RADIX_TREE_MULTIORDER
+void ** __radix_tree_next_slot(void **slot, struct radix_tree_iter *iter,
+				unsigned flags);
+#else
+/* Can't happen without sibling entries, but the compiler can't tell that */
+static inline void ** __radix_tree_next_slot(void **slot,
+				struct radix_tree_iter *iter, unsigned flags)
 {
-	return (void *)((unsigned long)ptr & ~RADIX_TREE_INTERNAL_NODE);
+	return slot;
 }
+#endif
 
 /**
  * radix_tree_next_slot - find next slot in chunk
@@ -453,7 +466,7 @@ static inline struct radix_tree_node *entry_to_node(void *ptr)
  * For tagged lookup it also eats @iter->tags.
  *
  * There are several cases where 'slot' can be passed in as NULL to this
- * function.  These cases result from the use of radix_tree_iter_next() or
+ * function.  These cases result from the use of radix_tree_iter_resume() or
  * radix_tree_iter_retry().  In these cases we don't end up dereferencing
  * 'slot' because either:
  * a) we are doing tagged iteration and iter->tags has been set to 0, or
@@ -464,51 +477,31 @@ static __always_inline void **
 radix_tree_next_slot(void **slot, struct radix_tree_iter *iter, unsigned flags)
 {
 	if (flags & RADIX_TREE_ITER_TAGGED) {
-		void *canon = slot;
-
 		iter->tags >>= 1;
 		if (unlikely(!iter->tags))
 			return NULL;
-		while (IS_ENABLED(CONFIG_RADIX_TREE_MULTIORDER) &&
-					radix_tree_is_internal_node(slot[1])) {
-			if (entry_to_node(slot[1]) == canon) {
-				iter->tags >>= 1;
-				iter->index = __radix_tree_iter_add(iter, 1);
-				slot++;
-				continue;
-			}
-			iter->next_index = __radix_tree_iter_add(iter, 1);
-			return NULL;
-		}
 		if (likely(iter->tags & 1ul)) {
 			iter->index = __radix_tree_iter_add(iter, 1);
-			return slot + 1;
+			slot++;
+			goto found;
 		}
 		if (!(flags & RADIX_TREE_ITER_CONTIG)) {
 			unsigned offset = __ffs(iter->tags);
 
-			iter->tags >>= offset;
-			iter->index = __radix_tree_iter_add(iter, offset + 1);
-			return slot + offset + 1;
+			iter->tags >>= offset++;
+			iter->index = __radix_tree_iter_add(iter, offset);
+			slot += offset;
+			goto found;
 		}
 	} else {
 		long count = radix_tree_chunk_size(iter);
-		void *canon = slot;
 
 		while (--count > 0) {
 			slot++;
 			iter->index = __radix_tree_iter_add(iter, 1);
 
-			if (IS_ENABLED(CONFIG_RADIX_TREE_MULTIORDER) &&
-			    radix_tree_is_internal_node(*slot)) {
-				if (entry_to_node(*slot) == canon)
-					continue;
-				iter->next_index = iter->index;
-				break;
-			}
-
 			if (likely(*slot))
-				return slot;
+				goto found;
 			if (flags & RADIX_TREE_ITER_CONTIG) {
 				/* forbid switching to the next chunk */
 				iter->next_index = 0;
@@ -517,6 +510,11 @@ radix_tree_next_slot(void **slot, struct radix_tree_iter *iter, unsigned flags)
 		}
 	}
 	return NULL;
+
+ found:
+	if (unlikely(radix_tree_is_internal_node(*slot)))
+		return __radix_tree_next_slot(slot, iter, flags);
+	return slot;
 }
 
 /**
@@ -567,6 +565,6 @@ radix_tree_next_slot(void **slot, struct radix_tree_iter *iter, unsigned flags)
 	     slot || (slot = radix_tree_next_chunk(root, iter,		\
 			      RADIX_TREE_ITER_TAGGED | tag)) ;		\
 	     slot = radix_tree_next_slot(slot, iter,			\
-				RADIX_TREE_ITER_TAGGED))
+				RADIX_TREE_ITER_TAGGED | tag))
 
 #endif /* _LINUX_RADIX_TREE_H */
diff --git a/include/linux/regulator/consumer.h b/include/linux/regulator/consumer.h
index 692108222271..ea0fffa5faeb 100644
--- a/include/linux/regulator/consumer.h
+++ b/include/linux/regulator/consumer.h
@@ -120,6 +120,25 @@ struct regmap;
 #define REGULATOR_EVENT_PRE_DISABLE		0x400
 #define REGULATOR_EVENT_ABORT_DISABLE		0x800
 
+/*
+ * Regulator errors that can be queried using regulator_get_error_flags
+ *
+ * UNDER_VOLTAGE  Regulator output is under voltage.
+ * OVER_CURRENT   Regulator output current is too high.
+ * REGULATION_OUT Regulator output is out of regulation.
+ * FAIL           Regulator output has failed.
+ * OVER_TEMP      Regulator over temp.
+ *
+ * NOTE: These errors can be OR'ed together.
+ */
+
+#define REGULATOR_ERROR_UNDER_VOLTAGE		BIT(1)
+#define REGULATOR_ERROR_OVER_CURRENT		BIT(2)
+#define REGULATOR_ERROR_REGULATION_OUT		BIT(3)
+#define REGULATOR_ERROR_FAIL			BIT(4)
+#define REGULATOR_ERROR_OVER_TEMP		BIT(5)
+
+
 /**
  * struct pre_voltage_change_data - Data sent with PRE_VOLTAGE_CHANGE event
  *
@@ -237,6 +256,8 @@ int regulator_get_current_limit(struct regulator *regulator);
 
 int regulator_set_mode(struct regulator *regulator, unsigned int mode);
 unsigned int regulator_get_mode(struct regulator *regulator);
+int regulator_get_error_flags(struct regulator *regulator,
+				unsigned int *flags);
 int regulator_set_load(struct regulator *regulator, int load_uA);
 
 int regulator_allow_bypass(struct regulator *regulator, bool allow);
@@ -477,6 +498,12 @@ static inline unsigned int regulator_get_mode(struct regulator *regulator)
 	return REGULATOR_MODE_NORMAL;
 }
 
+static inline int regulator_get_error_flags(struct regulator *regulator,
+					    unsigned int *flags)
+{
+	return -EINVAL;
+}
+
 static inline int regulator_set_load(struct regulator *regulator, int load_uA)
 {
 	return REGULATOR_MODE_NORMAL;
diff --git a/include/linux/regulator/driver.h b/include/linux/regulator/driver.h
index 37b532410528..dac8e7b16bc6 100644
--- a/include/linux/regulator/driver.h
+++ b/include/linux/regulator/driver.h
@@ -100,6 +100,7 @@ struct regulator_linear_range {
  *
  * @set_mode: Set the configured operating mode for the regulator.
  * @get_mode: Get the configured operating mode for the regulator.
+ * @get_error_flags: Get the current error(s) for the regulator.
  * @get_status: Return actual (not as-configured) status of regulator, as a
  *	REGULATOR_STATUS value (or negative errno)
  * @get_optimum_mode: Get the most efficient operating mode for the regulator
@@ -169,6 +170,9 @@ struct regulator_ops {
 	int (*set_mode) (struct regulator_dev *, unsigned int mode);
 	unsigned int (*get_mode) (struct regulator_dev *);
 
+	/* retrieve current error flags on the regulator */
+	int (*get_error_flags)(struct regulator_dev *, unsigned int *flags);
+
 	/* Time taken to enable or set voltage on the regulator */
 	int (*enable_time) (struct regulator_dev *);
 	int (*set_ramp_delay) (struct regulator_dev *, int ramp_delay);
diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h
index 930023b7c825..e2f3a3281d8f 100644
--- a/include/linux/remoteproc.h
+++ b/include/linux/remoteproc.h
@@ -400,6 +400,7 @@ enum rproc_crash_type {
  * @firmware_loading_complete: marks e/o asynchronous firmware loading
  * @bootaddr: address of first instruction to boot rproc with (optional)
  * @rvdevs: list of remote virtio devices
+ * @subdevs: list of subdevices, to following the running state
  * @notifyids: idr for dynamically assigning rproc-wide unique notify ids
  * @index: index of this rproc device
  * @crash_handler: workqueue for handling a crash
@@ -407,15 +408,14 @@ enum rproc_crash_type {
  * @crash_comp: completion used to sync crash handler and the rproc reload
  * @recovery_disabled: flag that state if recovery was disabled
  * @max_notifyid: largest allocated notify id.
- * @table_ptr: pointer to the resource table in effect
- * @cached_table: copy of the resource table
+ * @table_ptr: our copy of the resource table
  * @has_iommu: flag to indicate if remote processor is behind an MMU
  */
 struct rproc {
 	struct list_head node;
 	struct iommu_domain *domain;
 	const char *name;
-	const char *firmware;
+	char *firmware;
 	void *priv;
 	const struct rproc_ops *ops;
 	struct device dev;
@@ -431,6 +431,7 @@ struct rproc {
 	struct completion firmware_loading_complete;
 	u32 bootaddr;
 	struct list_head rvdevs;
+	struct list_head subdevs;
 	struct idr notifyids;
 	int index;
 	struct work_struct crash_handler;
@@ -439,11 +440,23 @@ struct rproc {
 	bool recovery_disabled;
 	int max_notifyid;
 	struct resource_table *table_ptr;
-	struct resource_table *cached_table;
 	bool has_iommu;
 	bool auto_boot;
 };
 
+/**
+ * struct rproc_subdev - subdevice tied to a remoteproc
+ * @node: list node related to the rproc subdevs list
+ * @probe: probe function, called as the rproc is started
+ * @remove: remove function, called as the rproc is stopped
+ */
+struct rproc_subdev {
+	struct list_head node;
+
+	int (*probe)(struct rproc_subdev *subdev);
+	void (*remove)(struct rproc_subdev *subdev);
+};
+
 /* we currently support only two vrings per rvdev */
 
 #define RVDEV_NUM_VRINGS 2
@@ -472,6 +485,9 @@ struct rproc_vring {
 
 /**
  * struct rproc_vdev - remoteproc state for a supported virtio device
+ * @refcount: reference counter for the vdev and vring allocations
+ * @subdev: handle for registering the vdev as a rproc subdevice
+ * @id: virtio device id (as in virtio_ids.h)
  * @node: list node
  * @rproc: the rproc handle
  * @vdev: the virio device
@@ -479,6 +495,11 @@ struct rproc_vring {
  * @rsc_offset: offset of the vdev's resource entry
  */
 struct rproc_vdev {
+	struct kref refcount;
+
+	struct rproc_subdev subdev;
+
+	unsigned int id;
 	struct list_head node;
 	struct rproc *rproc;
 	struct virtio_device vdev;
@@ -511,4 +532,11 @@ static inline struct rproc *vdev_to_rproc(struct virtio_device *vdev)
 	return rvdev->rproc;
 }
 
+void rproc_add_subdev(struct rproc *rproc,
+		      struct rproc_subdev *subdev,
+		      int (*probe)(struct rproc_subdev *subdev),
+		      void (*remove)(struct rproc_subdev *subdev));
+
+void rproc_remove_subdev(struct rproc *rproc, struct rproc_subdev *subdev);
+
 #endif /* REMOTEPROC_H */
diff --git a/include/linux/remoteproc/st_slim_rproc.h b/include/linux/remoteproc/st_slim_rproc.h
new file mode 100644
index 000000000000..4155556fa4b2
--- /dev/null
+++ b/include/linux/remoteproc/st_slim_rproc.h
@@ -0,0 +1,58 @@
+/*
+ * SLIM core rproc driver header
+ *
+ * Copyright (C) 2016 STMicroelectronics
+ *
+ * Author: Peter Griffin <peter.griffin@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+#ifndef _ST_REMOTEPROC_SLIM_H
+#define _ST_REMOTEPROC_SLIM_H
+
+#define ST_SLIM_MEM_MAX 2
+#define ST_SLIM_MAX_CLK 4
+
+enum {
+	ST_SLIM_DMEM,
+	ST_SLIM_IMEM,
+};
+
+/**
+ * struct st_slim_mem - slim internal memory structure
+ * @cpu_addr: MPU virtual address of the memory region
+ * @bus_addr: Bus address used to access the memory region
+ * @size: Size of the memory region
+ */
+struct st_slim_mem {
+	void __iomem *cpu_addr;
+	phys_addr_t bus_addr;
+	size_t size;
+};
+
+/**
+ * struct st_slim_rproc - SLIM slim core
+ * @rproc: rproc handle
+ * @mem: slim memory information
+ * @slimcore: slim slimcore regs
+ * @peri: slim peripheral regs
+ * @clks: slim clocks
+ */
+struct st_slim_rproc {
+	struct rproc *rproc;
+	struct st_slim_mem mem[ST_SLIM_MEM_MAX];
+	void __iomem *slimcore;
+	void __iomem *peri;
+
+	/* st_slim_rproc private */
+	struct clk *clks[ST_SLIM_MAX_CLK];
+};
+
+struct st_slim_rproc *st_slim_rproc_alloc(struct platform_device *pdev,
+					char *fw_name);
+void st_slim_rproc_put(struct st_slim_rproc *slim_rproc);
+
+#endif
diff --git a/include/linux/reservation.h b/include/linux/reservation.h
index b0f305e77b7f..d9706a6f5ae2 100644
--- a/include/linux/reservation.h
+++ b/include/linux/reservation.h
@@ -40,7 +40,7 @@
 #define _LINUX_RESERVATION_H
 
 #include <linux/ww_mutex.h>
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 #include <linux/slab.h>
 #include <linux/seqlock.h>
 #include <linux/rcupdate.h>
@@ -59,7 +59,7 @@ extern const char reservation_seqcount_string[];
 struct reservation_object_list {
 	struct rcu_head rcu;
 	u32 shared_count, shared_max;
-	struct fence __rcu *shared[];
+	struct dma_fence __rcu *shared[];
 };
 
 /**
@@ -74,7 +74,7 @@ struct reservation_object {
 	struct ww_mutex lock;
 	seqcount_t seq;
 
-	struct fence __rcu *fence_excl;
+	struct dma_fence __rcu *fence_excl;
 	struct reservation_object_list __rcu *fence;
 	struct reservation_object_list *staged;
 };
@@ -107,7 +107,7 @@ reservation_object_fini(struct reservation_object *obj)
 {
 	int i;
 	struct reservation_object_list *fobj;
-	struct fence *excl;
+	struct dma_fence *excl;
 
 	/*
 	 * This object should be dead and all references must have
@@ -115,12 +115,12 @@ reservation_object_fini(struct reservation_object *obj)
 	 */
 	excl = rcu_dereference_protected(obj->fence_excl, 1);
 	if (excl)
-		fence_put(excl);
+		dma_fence_put(excl);
 
 	fobj = rcu_dereference_protected(obj->fence, 1);
 	if (fobj) {
 		for (i = 0; i < fobj->shared_count; ++i)
-			fence_put(rcu_dereference_protected(fobj->shared[i], 1));
+			dma_fence_put(rcu_dereference_protected(fobj->shared[i], 1));
 
 		kfree(fobj);
 	}
@@ -155,7 +155,7 @@ reservation_object_get_list(struct reservation_object *obj)
  * RETURNS
  * The exclusive fence or NULL
  */
-static inline struct fence *
+static inline struct dma_fence *
 reservation_object_get_excl(struct reservation_object *obj)
 {
 	return rcu_dereference_protected(obj->fence_excl,
@@ -173,35 +173,32 @@ reservation_object_get_excl(struct reservation_object *obj)
  * RETURNS
  * The exclusive fence or NULL if none
  */
-static inline struct fence *
+static inline struct dma_fence *
 reservation_object_get_excl_rcu(struct reservation_object *obj)
 {
-	struct fence *fence;
-	unsigned seq;
-retry:
-	seq = read_seqcount_begin(&obj->seq);
+	struct dma_fence *fence;
+
+	if (!rcu_access_pointer(obj->fence_excl))
+		return NULL;
+
 	rcu_read_lock();
-	fence = rcu_dereference(obj->fence_excl);
-	if (read_seqcount_retry(&obj->seq, seq)) {
-		rcu_read_unlock();
-		goto retry;
-	}
-	fence = fence_get(fence);
+	fence = dma_fence_get_rcu_safe(&obj->fence_excl);
 	rcu_read_unlock();
+
 	return fence;
 }
 
 int reservation_object_reserve_shared(struct reservation_object *obj);
 void reservation_object_add_shared_fence(struct reservation_object *obj,
-					 struct fence *fence);
+					 struct dma_fence *fence);
 
 void reservation_object_add_excl_fence(struct reservation_object *obj,
-				       struct fence *fence);
+				       struct dma_fence *fence);
 
 int reservation_object_get_fences_rcu(struct reservation_object *obj,
-				      struct fence **pfence_excl,
+				      struct dma_fence **pfence_excl,
 				      unsigned *pshared_count,
-				      struct fence ***pshared);
+				      struct dma_fence ***pshared);
 
 long reservation_object_wait_timeout_rcu(struct reservation_object *obj,
 					 bool wait_all, bool intr,
diff --git a/include/linux/restart_block.h b/include/linux/restart_block.h
new file mode 100644
index 000000000000..0d905d8ec553
--- /dev/null
+++ b/include/linux/restart_block.h
@@ -0,0 +1,51 @@
+/*
+ * Common syscall restarting data
+ */
+#ifndef __LINUX_RESTART_BLOCK_H
+#define __LINUX_RESTART_BLOCK_H
+
+#include <linux/compiler.h>
+#include <linux/types.h>
+
+struct timespec;
+struct compat_timespec;
+struct pollfd;
+
+/*
+ * System call restart block.
+ */
+struct restart_block {
+	long (*fn)(struct restart_block *);
+	union {
+		/* For futex_wait and futex_wait_requeue_pi */
+		struct {
+			u32 __user *uaddr;
+			u32 val;
+			u32 flags;
+			u32 bitset;
+			u64 time;
+			u32 __user *uaddr2;
+		} futex;
+		/* For nanosleep */
+		struct {
+			clockid_t clockid;
+			struct timespec __user *rmtp;
+#ifdef CONFIG_COMPAT
+			struct compat_timespec __user *compat_rmtp;
+#endif
+			u64 expires;
+		} nanosleep;
+		/* For poll */
+		struct {
+			struct pollfd __user *ufds;
+			int nfds;
+			int has_timeout;
+			unsigned long tv_sec;
+			unsigned long tv_nsec;
+		} poll;
+	};
+};
+
+extern long do_no_restart_syscall(struct restart_block *parm);
+
+#endif /* __LINUX_RESTART_BLOCK_H */
diff --git a/include/linux/rpmsg.h b/include/linux/rpmsg.h
index 452d393cc8dd..18f9e1ae4b7e 100644
--- a/include/linux/rpmsg.h
+++ b/include/linux/rpmsg.h
@@ -37,6 +37,7 @@
 
 #include <linux/types.h>
 #include <linux/device.h>
+#include <linux/err.h>
 #include <linux/mod_devicetable.h>
 #include <linux/kref.h>
 #include <linux/mutex.h>
@@ -64,6 +65,7 @@ struct rpmsg_channel_info {
  * rpmsg_device - device that belong to the rpmsg bus
  * @dev: the device struct
  * @id: device id (used to match between rpmsg drivers and devices)
+ * @driver_override: driver name to force a match
  * @src: local address
  * @dst: destination address
  * @ept: the rpmsg endpoint of this channel
@@ -72,6 +74,7 @@ struct rpmsg_channel_info {
 struct rpmsg_device {
 	struct device dev;
 	struct rpmsg_device_id id;
+	char *driver_override;
 	u32 src;
 	u32 dst;
 	struct rpmsg_endpoint *ept;
@@ -132,6 +135,8 @@ struct rpmsg_driver {
 	int (*callback)(struct rpmsg_device *, void *, int, void *, u32);
 };
 
+#if IS_ENABLED(CONFIG_RPMSG)
+
 int register_rpmsg_device(struct rpmsg_device *dev);
 void unregister_rpmsg_device(struct rpmsg_device *dev);
 int __register_rpmsg_driver(struct rpmsg_driver *drv, struct module *owner);
@@ -141,6 +146,116 @@ struct rpmsg_endpoint *rpmsg_create_ept(struct rpmsg_device *,
 					rpmsg_rx_cb_t cb, void *priv,
 					struct rpmsg_channel_info chinfo);
 
+int rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len);
+int rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst);
+int rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst,
+			  void *data, int len);
+
+int rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len);
+int rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst);
+int rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst,
+			     void *data, int len);
+
+#else
+
+static inline int register_rpmsg_device(struct rpmsg_device *dev)
+{
+	return -ENXIO;
+}
+
+static inline void unregister_rpmsg_device(struct rpmsg_device *dev)
+{
+	/* This shouldn't be possible */
+	WARN_ON(1);
+}
+
+static inline int __register_rpmsg_driver(struct rpmsg_driver *drv,
+					  struct module *owner)
+{
+	/* This shouldn't be possible */
+	WARN_ON(1);
+
+	return -ENXIO;
+}
+
+static inline void unregister_rpmsg_driver(struct rpmsg_driver *drv)
+{
+	/* This shouldn't be possible */
+	WARN_ON(1);
+}
+
+static inline void rpmsg_destroy_ept(struct rpmsg_endpoint *ept)
+{
+	/* This shouldn't be possible */
+	WARN_ON(1);
+}
+
+static inline struct rpmsg_endpoint *rpmsg_create_ept(struct rpmsg_device *rpdev,
+						      rpmsg_rx_cb_t cb,
+						      void *priv,
+						      struct rpmsg_channel_info chinfo)
+{
+	/* This shouldn't be possible */
+	WARN_ON(1);
+
+	return ERR_PTR(-ENXIO);
+}
+
+static inline int rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len)
+{
+	/* This shouldn't be possible */
+	WARN_ON(1);
+
+	return -ENXIO;
+}
+
+static inline int rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len,
+			       u32 dst)
+{
+	/* This shouldn't be possible */
+	WARN_ON(1);
+
+	return -ENXIO;
+
+}
+
+static inline int rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src,
+					u32 dst, void *data, int len)
+{
+	/* This shouldn't be possible */
+	WARN_ON(1);
+
+	return -ENXIO;
+}
+
+static inline int rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len)
+{
+	/* This shouldn't be possible */
+	WARN_ON(1);
+
+	return -ENXIO;
+}
+
+static inline int rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data,
+				  int len, u32 dst)
+{
+	/* This shouldn't be possible */
+	WARN_ON(1);
+
+	return -ENXIO;
+}
+
+static inline int rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src,
+					   u32 dst, void *data, int len)
+{
+	/* This shouldn't be possible */
+	WARN_ON(1);
+
+	return -ENXIO;
+}
+
+#endif /* IS_ENABLED(CONFIG_RPMSG) */
+
 /* use a macro to avoid include chaining to get THIS_MODULE */
 #define register_rpmsg_driver(drv) \
 	__register_rpmsg_driver(drv, THIS_MODULE)
@@ -157,14 +272,4 @@ struct rpmsg_endpoint *rpmsg_create_ept(struct rpmsg_device *,
 	module_driver(__rpmsg_driver, register_rpmsg_driver, \
 			unregister_rpmsg_driver)
 
-int rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len);
-int rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst);
-int rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst,
-			  void *data, int len);
-
-int rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len);
-int rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst);
-int rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst,
-			     void *data, int len);
-
 #endif /* _LINUX_RPMSG_H */
diff --git a/include/linux/rpmsg/qcom_smd.h b/include/linux/rpmsg/qcom_smd.h
new file mode 100644
index 000000000000..e674b2e3074b
--- /dev/null
+++ b/include/linux/rpmsg/qcom_smd.h
@@ -0,0 +1,33 @@
+
+#ifndef _LINUX_RPMSG_QCOM_SMD_H
+#define _LINUX_RPMSG_QCOM_SMD_H
+
+#include <linux/device.h>
+
+struct qcom_smd_edge;
+
+#if IS_ENABLED(CONFIG_RPMSG_QCOM_SMD) || IS_ENABLED(CONFIG_QCOM_SMD)
+
+struct qcom_smd_edge *qcom_smd_register_edge(struct device *parent,
+					     struct device_node *node);
+int qcom_smd_unregister_edge(struct qcom_smd_edge *edge);
+
+#else
+
+static inline struct qcom_smd_edge *
+qcom_smd_register_edge(struct device *parent,
+		       struct device_node *node)
+{
+	return ERR_PTR(-ENXIO);
+}
+
+static inline int qcom_smd_unregister_edge(struct qcom_smd_edge *edge)
+{
+	/* This shouldn't be possible */
+	WARN_ON(1);
+	return -ENXIO;
+}
+
+#endif
+
+#endif
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 0e90f2973719..a440cf178191 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1685,6 +1685,7 @@ struct task_struct {
 	struct list_head cpu_timers[3];
 
 /* process credentials */
+	const struct cred __rcu *ptracer_cred; /* Tracer's credentials at attach */
 	const struct cred __rcu *real_cred; /* objective and real subjective task
 					 * credentials (COW) */
 	const struct cred __rcu *cred;	/* effective (overridable) subjective task
@@ -2287,6 +2288,7 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut,
 /*
  * Per process flags
  */
+#define PF_IDLE		0x00000002	/* I am an IDLE thread */
 #define PF_EXITING	0x00000004	/* getting shut down */
 #define PF_EXITPIDONE	0x00000008	/* pi exit done on shut down */
 #define PF_VCPU		0x00000010	/* I'm a virtual CPU */
@@ -2648,7 +2650,7 @@ extern struct task_struct *idle_task(int cpu);
  */
 static inline bool is_idle_task(const struct task_struct *p)
 {
-	return p->pid == 0;
+	return !!(p->flags & PF_IDLE);
 }
 extern struct task_struct *curr_task(int cpu);
 extern void ia64_set_curr_task(int cpu, struct task_struct *p);
diff --git a/include/linux/seqno-fence.h b/include/linux/seqno-fence.h
index a1ba6a5ccdd6..c58c535d12a8 100644
--- a/include/linux/seqno-fence.h
+++ b/include/linux/seqno-fence.h
@@ -20,7 +20,7 @@
 #ifndef __LINUX_SEQNO_FENCE_H
 #define __LINUX_SEQNO_FENCE_H
 
-#include <linux/fence.h>
+#include <linux/dma-fence.h>
 #include <linux/dma-buf.h>
 
 enum seqno_fence_condition {
@@ -29,15 +29,15 @@ enum seqno_fence_condition {
 };
 
 struct seqno_fence {
-	struct fence base;
+	struct dma_fence base;
 
-	const struct fence_ops *ops;
+	const struct dma_fence_ops *ops;
 	struct dma_buf *sync_buf;
 	uint32_t seqno_ofs;
 	enum seqno_fence_condition condition;
 };
 
-extern const struct fence_ops seqno_fence_ops;
+extern const struct dma_fence_ops seqno_fence_ops;
 
 /**
  * to_seqno_fence - cast a fence to a seqno_fence
@@ -47,7 +47,7 @@ extern const struct fence_ops seqno_fence_ops;
  * or the seqno_fence otherwise.
  */
 static inline struct seqno_fence *
-to_seqno_fence(struct fence *fence)
+to_seqno_fence(struct dma_fence *fence)
 {
 	if (fence->ops != &seqno_fence_ops)
 		return NULL;
@@ -83,9 +83,9 @@ to_seqno_fence(struct fence *fence)
  * dma-buf for sync_buf, since mapping or unmapping the sync_buf to the
  * device's vm can be expensive.
  *
- * It is recommended for creators of seqno_fence to call fence_signal
+ * It is recommended for creators of seqno_fence to call dma_fence_signal()
  * before destruction. This will prevent possible issues from wraparound at
- * time of issue vs time of check, since users can check fence_is_signaled
+ * time of issue vs time of check, since users can check dma_fence_is_signaled()
  * before submitting instructions for the hardware to wait on the fence.
  * However, when ops.enable_signaling is not called, it doesn't have to be
  * done as soon as possible, just before there's any real danger of seqno
@@ -96,18 +96,18 @@ seqno_fence_init(struct seqno_fence *fence, spinlock_t *lock,
 		 struct dma_buf *sync_buf,  uint32_t context,
 		 uint32_t seqno_ofs, uint32_t seqno,
 		 enum seqno_fence_condition cond,
-		 const struct fence_ops *ops)
+		 const struct dma_fence_ops *ops)
 {
 	BUG_ON(!fence || !sync_buf || !ops);
 	BUG_ON(!ops->wait || !ops->enable_signaling ||
 	       !ops->get_driver_name || !ops->get_timeline_name);
 
 	/*
-	 * ops is used in fence_init for get_driver_name, so needs to be
+	 * ops is used in dma_fence_init for get_driver_name, so needs to be
 	 * initialized first
 	 */
 	fence->ops = ops;
-	fence_init(&fence->base, &seqno_fence_ops, lock, context, seqno);
+	dma_fence_init(&fence->base, &seqno_fence_ops, lock, context, seqno);
 	get_dma_buf(sync_buf);
 	fence->sync_buf = sync_buf;
 	fence->seqno_ofs = seqno_ofs;
diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h
index 48ec7651989b..61fbb440449c 100644
--- a/include/linux/serial_8250.h
+++ b/include/linux/serial_8250.h
@@ -36,6 +36,8 @@ struct plat_serial8250_port {
 	void		(*set_termios)(struct uart_port *,
 			               struct ktermios *new,
 			               struct ktermios *old);
+	void		(*set_ldisc)(struct uart_port *,
+				     struct ktermios *);
 	unsigned int	(*get_mctrl)(struct uart_port *);
 	int		(*handle_irq)(struct uart_port *);
 	void		(*pm)(struct uart_port *, unsigned int state,
@@ -94,7 +96,7 @@ struct uart_8250_port {
 	struct uart_port	port;
 	struct timer_list	timer;		/* "no irq" timer */
 	struct list_head	list;		/* ports on this IRQ */
-	unsigned short		capabilities;	/* port capabilities */
+	u32			capabilities;	/* port capabilities */
 	unsigned short		bugs;		/* port bugs */
 	bool			fifo_bug;	/* min RX trigger if enabled */
 	unsigned int		tx_loadsz;	/* transmit fifo load size */
@@ -149,6 +151,8 @@ extern int early_serial8250_setup(struct earlycon_device *device,
 					 const char *options);
 extern void serial8250_do_set_termios(struct uart_port *port,
 		struct ktermios *termios, struct ktermios *old);
+extern void serial8250_do_set_ldisc(struct uart_port *port,
+				    struct ktermios *termios);
 extern unsigned int serial8250_do_get_mctrl(struct uart_port *port);
 extern int serial8250_do_startup(struct uart_port *port);
 extern void serial8250_do_shutdown(struct uart_port *port);
@@ -168,6 +172,6 @@ int serial8250_console_setup(struct uart_port *port, char *options, bool probe);
 
 extern void serial8250_set_isa_configurator(void (*v)
 					(int port, struct uart_port *up,
-						unsigned short *capabilities));
+						u32 *capabilities));
 
 #endif
diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h
index 344201437017..5def8e830fb0 100644
--- a/include/linux/serial_core.h
+++ b/include/linux/serial_core.h
@@ -111,8 +111,8 @@ struct uart_icount {
 	__u32	buf_overrun;
 };
 
-typedef unsigned int __bitwise__ upf_t;
-typedef unsigned int __bitwise__ upstat_t;
+typedef unsigned int __bitwise upf_t;
+typedef unsigned int __bitwise upstat_t;
 
 struct uart_port {
 	spinlock_t		lock;			/* port lock */
@@ -123,6 +123,8 @@ struct uart_port {
 	void			(*set_termios)(struct uart_port *,
 				               struct ktermios *new,
 				               struct ktermios *old);
+	void			(*set_ldisc)(struct uart_port *,
+					     struct ktermios *);
 	unsigned int		(*get_mctrl)(struct uart_port *);
 	void			(*set_mctrl)(struct uart_port *, unsigned int);
 	int			(*startup)(struct uart_port *port);
diff --git a/include/linux/signal.h b/include/linux/signal.h
index b63f63eaa39c..5308304993be 100644
--- a/include/linux/signal.h
+++ b/include/linux/signal.h
@@ -97,6 +97,23 @@ static inline int sigisemptyset(sigset_t *set)
 	}
 }
 
+static inline int sigequalsets(const sigset_t *set1, const sigset_t *set2)
+{
+	switch (_NSIG_WORDS) {
+	case 4:
+		return	(set1->sig[3] == set2->sig[3]) &&
+			(set1->sig[2] == set2->sig[2]) &&
+			(set1->sig[1] == set2->sig[1]) &&
+			(set1->sig[0] == set2->sig[0]);
+	case 2:
+		return	(set1->sig[1] == set2->sig[1]) &&
+			(set1->sig[0] == set2->sig[0]);
+	case 1:
+		return	set1->sig[0] == set2->sig[0];
+	}
+	return 0;
+}
+
 #define sigmask(sig)	(1UL << ((sig) - 1))
 
 #ifndef __HAVE_ARCH_SIG_SETOPS
diff --git a/include/linux/soc/qcom/wcnss_ctrl.h b/include/linux/soc/qcom/wcnss_ctrl.h
index a37bc5538f19..eab64976a73b 100644
--- a/include/linux/soc/qcom/wcnss_ctrl.h
+++ b/include/linux/soc/qcom/wcnss_ctrl.h
@@ -3,6 +3,19 @@
 
 #include <linux/soc/qcom/smd.h>
 
+#if IS_ENABLED(CONFIG_QCOM_WCNSS_CTRL)
+
 struct qcom_smd_channel *qcom_wcnss_open_channel(void *wcnss, const char *name, qcom_smd_cb_t cb);
 
+#else
+
+static inline struct qcom_smd_channel*
+qcom_wcnss_open_channel(void *wcnss, const char *name, qcom_smd_cb_t cb)
+{
+	WARN_ON(1);
+	return ERR_PTR(-ENXIO);
+}
+
+#endif
+
 #endif
diff --git a/include/linux/soc/renesas/rcar-rst.h b/include/linux/soc/renesas/rcar-rst.h
new file mode 100644
index 000000000000..a18e0783946b
--- /dev/null
+++ b/include/linux/soc/renesas/rcar-rst.h
@@ -0,0 +1,6 @@
+#ifndef __LINUX_SOC_RENESAS_RCAR_RST_H__
+#define __LINUX_SOC_RENESAS_RCAR_RST_H__
+
+int rcar_rst_read_mode_pins(u32 *mode);
+
+#endif /* __LINUX_SOC_RENESAS_RCAR_RST_H__ */
diff --git a/include/linux/soc/ti/ti_sci_protocol.h b/include/linux/soc/ti/ti_sci_protocol.h
new file mode 100644
index 000000000000..0ccbc138c26a
--- /dev/null
+++ b/include/linux/soc/ti/ti_sci_protocol.h
@@ -0,0 +1,249 @@
+/*
+ * Texas Instruments System Control Interface Protocol
+ *
+ * Copyright (C) 2015-2016 Texas Instruments Incorporated - http://www.ti.com/
+ *	Nishanth Menon
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __TISCI_PROTOCOL_H
+#define __TISCI_PROTOCOL_H
+
+/**
+ * struct ti_sci_version_info - version information structure
+ * @abi_major:	Major ABI version. Change here implies risk of backward
+ *		compatibility break.
+ * @abi_minor:	Minor ABI version. Change here implies new feature addition,
+ *		or compatible change in ABI.
+ * @firmware_revision:	Firmware revision (not usually used).
+ * @firmware_description: Firmware description (not usually used).
+ */
+struct ti_sci_version_info {
+	u8 abi_major;
+	u8 abi_minor;
+	u16 firmware_revision;
+	char firmware_description[32];
+};
+
+struct ti_sci_handle;
+
+/**
+ * struct ti_sci_core_ops - SoC Core Operations
+ * @reboot_device: Reboot the SoC
+ *		Returns 0 for successful request(ideally should never return),
+ *		else returns corresponding error value.
+ */
+struct ti_sci_core_ops {
+	int (*reboot_device)(const struct ti_sci_handle *handle);
+};
+
+/**
+ * struct ti_sci_dev_ops - Device control operations
+ * @get_device: Command to request for device managed by TISCI
+ *		Returns 0 for successful exclusive request, else returns
+ *		corresponding error message.
+ * @idle_device: Command to idle a device managed by TISCI
+ *		Returns 0 for successful exclusive request, else returns
+ *		corresponding error message.
+ * @put_device:	Command to release a device managed by TISCI
+ *		Returns 0 for successful release, else returns corresponding
+ *		error message.
+ * @is_valid:	Check if the device ID is a valid ID.
+ *		Returns 0 if the ID is valid, else returns corresponding error.
+ * @get_context_loss_count: Command to retrieve context loss counter - this
+ *		increments every time the device looses context. Overflow
+ *		is possible.
+ *		- count: pointer to u32 which will retrieve counter
+ *		Returns 0 for successful information request and count has
+ *		proper data, else returns corresponding error message.
+ * @is_idle:	Reports back about device idle state
+ *		- req_state: Returns requested idle state
+ *		Returns 0 for successful information request and req_state and
+ *		current_state has proper data, else returns corresponding error
+ *		message.
+ * @is_stop:	Reports back about device stop state
+ *		- req_state: Returns requested stop state
+ *		- current_state: Returns current stop state
+ *		Returns 0 for successful information request and req_state and
+ *		current_state has proper data, else returns corresponding error
+ *		message.
+ * @is_on:	Reports back about device ON(or active) state
+ *		- req_state: Returns requested ON state
+ *		- current_state: Returns current ON state
+ *		Returns 0 for successful information request and req_state and
+ *		current_state has proper data, else returns corresponding error
+ *		message.
+ * @is_transitioning: Reports back if the device is in the middle of transition
+ *		of state.
+ *		-current_state: Returns 'true' if currently transitioning.
+ * @set_device_resets: Command to configure resets for device managed by TISCI.
+ *		-reset_state: Device specific reset bit field
+ *		Returns 0 for successful request, else returns
+ *		corresponding error message.
+ * @get_device_resets: Command to read state of resets for device managed
+ *		by TISCI.
+ *		-reset_state: pointer to u32 which will retrieve resets
+ *		Returns 0 for successful request, else returns
+ *		corresponding error message.
+ *
+ * NOTE: for all these functions, the following parameters are generic in
+ * nature:
+ * -handle:	Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * -id:		Device Identifier
+ *
+ * Request for the device - NOTE: the client MUST maintain integrity of
+ * usage count by balancing get_device with put_device. No refcounting is
+ * managed by driver for that purpose.
+ */
+struct ti_sci_dev_ops {
+	int (*get_device)(const struct ti_sci_handle *handle, u32 id);
+	int (*idle_device)(const struct ti_sci_handle *handle, u32 id);
+	int (*put_device)(const struct ti_sci_handle *handle, u32 id);
+	int (*is_valid)(const struct ti_sci_handle *handle, u32 id);
+	int (*get_context_loss_count)(const struct ti_sci_handle *handle,
+				      u32 id, u32 *count);
+	int (*is_idle)(const struct ti_sci_handle *handle, u32 id,
+		       bool *requested_state);
+	int (*is_stop)(const struct ti_sci_handle *handle, u32 id,
+		       bool *req_state, bool *current_state);
+	int (*is_on)(const struct ti_sci_handle *handle, u32 id,
+		     bool *req_state, bool *current_state);
+	int (*is_transitioning)(const struct ti_sci_handle *handle, u32 id,
+				bool *current_state);
+	int (*set_device_resets)(const struct ti_sci_handle *handle, u32 id,
+				 u32 reset_state);
+	int (*get_device_resets)(const struct ti_sci_handle *handle, u32 id,
+				 u32 *reset_state);
+};
+
+/**
+ * struct ti_sci_clk_ops - Clock control operations
+ * @get_clock:	Request for activation of clock and manage by processor
+ *		- needs_ssc: 'true' if Spread Spectrum clock is desired.
+ *		- can_change_freq: 'true' if frequency change is desired.
+ *		- enable_input_term: 'true' if input termination is desired.
+ * @idle_clock:	Request for Idling a clock managed by processor
+ * @put_clock:	Release the clock to be auto managed by TISCI
+ * @is_auto:	Is the clock being auto managed
+ *		- req_state: state indicating if the clock is auto managed
+ * @is_on:	Is the clock ON
+ *		- req_state: if the clock is requested to be forced ON
+ *		- current_state: if the clock is currently ON
+ * @is_off:	Is the clock OFF
+ *		- req_state: if the clock is requested to be forced OFF
+ *		- current_state: if the clock is currently Gated
+ * @set_parent:	Set the clock source of a specific device clock
+ *		- parent_id: Parent clock identifier to set.
+ * @get_parent:	Get the current clock source of a specific device clock
+ *		- parent_id: Parent clock identifier which is the parent.
+ * @get_num_parents: Get the number of parents of the current clock source
+ *		- num_parents: returns the number of parent clocks.
+ * @get_best_match_freq: Find a best matching frequency for a frequency
+ *		range.
+ *		- match_freq: Best matching frequency in Hz.
+ * @set_freq:	Set the Clock frequency
+ * @get_freq:	Get the Clock frequency
+ *		- current_freq: Frequency in Hz that the clock is at.
+ *
+ * NOTE: for all these functions, the following parameters are generic in
+ * nature:
+ * -handle:	Pointer to TISCI handle as retrieved by *ti_sci_get_handle
+ * -did:	Device identifier this request is for
+ * -cid:	Clock identifier for the device for this request.
+ *		Each device has it's own set of clock inputs. This indexes
+ *		which clock input to modify.
+ * -min_freq:	The minimum allowable frequency in Hz. This is the minimum
+ *		allowable programmed frequency and does not account for clock
+ *		tolerances and jitter.
+ * -target_freq: The target clock frequency in Hz. A frequency will be
+ *		processed as close to this target frequency as possible.
+ * -max_freq:	The maximum allowable frequency in Hz. This is the maximum
+ *		allowable programmed frequency and does not account for clock
+ *		tolerances and jitter.
+ *
+ * Request for the clock - NOTE: the client MUST maintain integrity of
+ * usage count by balancing get_clock with put_clock. No refcounting is
+ * managed by driver for that purpose.
+ */
+struct ti_sci_clk_ops {
+	int (*get_clock)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+			 bool needs_ssc, bool can_change_freq,
+			 bool enable_input_term);
+	int (*idle_clock)(const struct ti_sci_handle *handle, u32 did, u8 cid);
+	int (*put_clock)(const struct ti_sci_handle *handle, u32 did, u8 cid);
+	int (*is_auto)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+		       bool *req_state);
+	int (*is_on)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+		     bool *req_state, bool *current_state);
+	int (*is_off)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+		      bool *req_state, bool *current_state);
+	int (*set_parent)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+			  u8 parent_id);
+	int (*get_parent)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+			  u8 *parent_id);
+	int (*get_num_parents)(const struct ti_sci_handle *handle, u32 did,
+			       u8 cid, u8 *num_parents);
+	int (*get_best_match_freq)(const struct ti_sci_handle *handle, u32 did,
+				   u8 cid, u64 min_freq, u64 target_freq,
+				   u64 max_freq, u64 *match_freq);
+	int (*set_freq)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+			u64 min_freq, u64 target_freq, u64 max_freq);
+	int (*get_freq)(const struct ti_sci_handle *handle, u32 did, u8 cid,
+			u64 *current_freq);
+};
+
+/**
+ * struct ti_sci_ops - Function support for TI SCI
+ * @dev_ops:	Device specific operations
+ * @clk_ops:	Clock specific operations
+ */
+struct ti_sci_ops {
+	struct ti_sci_core_ops core_ops;
+	struct ti_sci_dev_ops dev_ops;
+	struct ti_sci_clk_ops clk_ops;
+};
+
+/**
+ * struct ti_sci_handle - Handle returned to TI SCI clients for usage.
+ * @version:	structure containing version information
+ * @ops:	operations that are made available to TI SCI clients
+ */
+struct ti_sci_handle {
+	struct ti_sci_version_info version;
+	struct ti_sci_ops ops;
+};
+
+#if IS_ENABLED(CONFIG_TI_SCI_PROTOCOL)
+const struct ti_sci_handle *ti_sci_get_handle(struct device *dev);
+int ti_sci_put_handle(const struct ti_sci_handle *handle);
+const struct ti_sci_handle *devm_ti_sci_get_handle(struct device *dev);
+
+#else	/* CONFIG_TI_SCI_PROTOCOL */
+
+static inline const struct ti_sci_handle *ti_sci_get_handle(struct device *dev)
+{
+	return ERR_PTR(-EINVAL);
+}
+
+static inline int ti_sci_put_handle(const struct ti_sci_handle *handle)
+{
+	return -EINVAL;
+}
+
+static inline
+const struct ti_sci_handle *devm_ti_sci_get_handle(struct device *dev)
+{
+	return ERR_PTR(-EINVAL);
+}
+
+#endif	/* CONFIG_TI_SCI_PROTOCOL */
+
+#endif	/* __TISCI_PROTOCOL_H */
diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h
index 4b743ac35396..75c6bd0ac605 100644
--- a/include/linux/spi/spi.h
+++ b/include/linux/spi/spi.h
@@ -442,6 +442,7 @@ struct spi_master {
 #define SPI_MASTER_NO_TX	BIT(2)		/* can't do buffer write */
 #define SPI_MASTER_MUST_RX      BIT(3)		/* requires rx */
 #define SPI_MASTER_MUST_TX      BIT(4)		/* requires tx */
+#define SPI_MASTER_GPIO_SS      BIT(5)		/* GPIO CS must select slave */
 
 	/*
 	 * on some hardware transfer / message size may be constrained
diff --git a/include/linux/stm.h b/include/linux/stm.h
index 8369d8a8cabd..210ff2292361 100644
--- a/include/linux/stm.h
+++ b/include/linux/stm.h
@@ -133,7 +133,7 @@ int stm_source_register_device(struct device *parent,
 			       struct stm_source_data *data);
 void stm_source_unregister_device(struct stm_source_data *data);
 
-int stm_source_write(struct stm_source_data *data, unsigned int chan,
-		     const char *buf, size_t count);
+int notrace stm_source_write(struct stm_source_data *data, unsigned int chan,
+			     const char *buf, size_t count);
 
 #endif /* _STM_H_ */
diff --git a/include/linux/suspend.h b/include/linux/suspend.h
index d9718378a8be..0c729c3c8549 100644
--- a/include/linux/suspend.h
+++ b/include/linux/suspend.h
@@ -194,6 +194,8 @@ struct platform_freeze_ops {
 };
 
 #ifdef CONFIG_SUSPEND
+extern suspend_state_t mem_sleep_default;
+
 /**
  * suspend_set_ops - set platform dependent suspend operations
  * @ops: The new suspend operations to set.
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 09b212d37f1d..09f4be179ff3 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -319,6 +319,9 @@ extern int kswapd_run(int nid);
 extern void kswapd_stop(int nid);
 
 #ifdef CONFIG_SWAP
+
+#include <linux/blk_types.h> /* for bio_end_io_t */
+
 /* linux/mm/page_io.c */
 extern int swap_readpage(struct page *);
 extern int swap_writepage(struct page *page, struct writeback_control *wbc);
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index 5f81f8a187f2..183f37c8a5e1 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -44,11 +44,13 @@ enum dma_sync_target {
 extern phys_addr_t swiotlb_tbl_map_single(struct device *hwdev,
 					  dma_addr_t tbl_dma_addr,
 					  phys_addr_t phys, size_t size,
-					  enum dma_data_direction dir);
+					  enum dma_data_direction dir,
+					  unsigned long attrs);
 
 extern void swiotlb_tbl_unmap_single(struct device *hwdev,
 				     phys_addr_t tlb_addr,
-				     size_t size, enum dma_data_direction dir);
+				     size_t size, enum dma_data_direction dir,
+				     unsigned long attrs);
 
 extern void swiotlb_tbl_sync_single(struct device *hwdev,
 				    phys_addr_t tlb_addr,
@@ -73,14 +75,6 @@ extern void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
 			       unsigned long attrs);
 
 extern int
-swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nents,
-	       enum dma_data_direction dir);
-
-extern void
-swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents,
-		 enum dma_data_direction dir);
-
-extern int
 swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
 		     enum dma_data_direction dir,
 		     unsigned long attrs);
diff --git a/include/linux/sync_file.h b/include/linux/sync_file.h
index aa17ccfc2f57..3e3ab84fc4cd 100644
--- a/include/linux/sync_file.h
+++ b/include/linux/sync_file.h
@@ -18,8 +18,8 @@
 #include <linux/ktime.h>
 #include <linux/list.h>
 #include <linux/spinlock.h>
-#include <linux/fence.h>
-#include <linux/fence-array.h>
+#include <linux/dma-fence.h>
+#include <linux/dma-fence-array.h>
 
 /**
  * struct sync_file - sync file to export to the userspace
@@ -41,13 +41,13 @@ struct sync_file {
 
 	wait_queue_head_t	wq;
 
-	struct fence		*fence;
-	struct fence_cb cb;
+	struct dma_fence	*fence;
+	struct dma_fence_cb cb;
 };
 
-#define POLL_ENABLED FENCE_FLAG_USER_BITS
+#define POLL_ENABLED DMA_FENCE_FLAG_USER_BITS
 
-struct sync_file *sync_file_create(struct fence *fence);
-struct fence *sync_file_get_fence(int fd);
+struct sync_file *sync_file_create(struct dma_fence *fence);
+struct dma_fence *sync_file_get_fence(int fd);
 
 #endif /* _LINUX_SYNC_H */
diff --git a/include/linux/sys_soc.h b/include/linux/sys_soc.h
index 2739ccb69571..bed223b70217 100644
--- a/include/linux/sys_soc.h
+++ b/include/linux/sys_soc.h
@@ -13,6 +13,7 @@ struct soc_device_attribute {
 	const char *family;
 	const char *revision;
 	const char *soc_id;
+	const void *data;
 };
 
 /**
@@ -34,4 +35,12 @@ void soc_device_unregister(struct soc_device *soc_dev);
  */
 struct device *soc_device_to_device(struct soc_device *soc);
 
+#ifdef CONFIG_SOC_BUS
+const struct soc_device_attribute *soc_device_match(
+	const struct soc_device_attribute *matches);
+#else
+static inline const struct soc_device_attribute *soc_device_match(
+	const struct soc_device_attribute *matches) { return NULL; }
+#endif
+
 #endif /* __SOC_BUS_H */
diff --git a/include/linux/thermal.h b/include/linux/thermal.h
index 511182a88e76..e275e98bdceb 100644
--- a/include/linux/thermal.h
+++ b/include/linux/thermal.h
@@ -28,6 +28,7 @@
 #include <linux/of.h>
 #include <linux/idr.h>
 #include <linux/device.h>
+#include <linux/sysfs.h>
 #include <linux/workqueue.h>
 #include <uapi/linux/thermal.h>
 
@@ -204,6 +205,7 @@ struct thermal_zone_device {
 	int id;
 	char type[THERMAL_NAME_LENGTH];
 	struct device device;
+	struct attribute_group trips_attribute_group;
 	struct thermal_attr *trip_temp_attrs;
 	struct thermal_attr *trip_type_attrs;
 	struct thermal_attr *trip_hyst_attrs;
diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h
index 2873baf5372a..58373875e8ee 100644
--- a/include/linux/thread_info.h
+++ b/include/linux/thread_info.h
@@ -9,50 +9,17 @@
 
 #include <linux/types.h>
 #include <linux/bug.h>
-
-struct timespec;
-struct compat_timespec;
+#include <linux/restart_block.h>
 
 #ifdef CONFIG_THREAD_INFO_IN_TASK
-#define current_thread_info() ((struct thread_info *)current)
-#endif
-
 /*
- * System call restart block.
+ * For CONFIG_THREAD_INFO_IN_TASK kernels we need <asm/current.h> for the
+ * definition of current, but for !CONFIG_THREAD_INFO_IN_TASK kernels,
+ * including <asm/current.h> can cause a circular dependency on some platforms.
  */
-struct restart_block {
-	long (*fn)(struct restart_block *);
-	union {
-		/* For futex_wait and futex_wait_requeue_pi */
-		struct {
-			u32 __user *uaddr;
-			u32 val;
-			u32 flags;
-			u32 bitset;
-			u64 time;
-			u32 __user *uaddr2;
-		} futex;
-		/* For nanosleep */
-		struct {
-			clockid_t clockid;
-			struct timespec __user *rmtp;
-#ifdef CONFIG_COMPAT
-			struct compat_timespec __user *compat_rmtp;
+#include <asm/current.h>
+#define current_thread_info() ((struct thread_info *)current)
 #endif
-			u64 expires;
-		} nanosleep;
-		/* For poll */
-		struct {
-			struct pollfd __user *ufds;
-			int nfds;
-			int has_timeout;
-			unsigned long tv_sec;
-			unsigned long tv_nsec;
-		} poll;
-	};
-};
-
-extern long do_no_restart_syscall(struct restart_block *parm);
 
 #include <linux/bitops.h>
 #include <asm/thread_info.h>
diff --git a/include/linux/trace.h b/include/linux/trace.h
new file mode 100644
index 000000000000..9330a58e2651
--- /dev/null
+++ b/include/linux/trace.h
@@ -0,0 +1,28 @@
+#ifndef _LINUX_TRACE_H
+#define _LINUX_TRACE_H
+
+#ifdef CONFIG_TRACING
+/*
+ * The trace export - an export of Ftrace output. The trace_export
+ * can process traces and export them to a registered destination as
+ * an addition to the current only output of Ftrace - i.e. ring buffer.
+ *
+ * If you want traces to be sent to some other place rather than ring
+ * buffer only, just need to register a new trace_export and implement
+ * its own .write() function for writing traces to the storage.
+ *
+ * next		- pointer to the next trace_export
+ * write	- copy traces which have been delt with ->commit() to
+ *		  the destination
+ */
+struct trace_export {
+	struct trace_export __rcu	*next;
+	void (*write)(const void *, unsigned int);
+};
+
+int register_ftrace_export(struct trace_export *export);
+int unregister_ftrace_export(struct trace_export *export);
+
+#endif	/* CONFIG_TRACING */
+
+#endif	/* _LINUX_TRACE_H */
diff --git a/include/linux/tracepoint-defs.h b/include/linux/tracepoint-defs.h
index 4ac89acb6136..a03192052066 100644
--- a/include/linux/tracepoint-defs.h
+++ b/include/linux/tracepoint-defs.h
@@ -29,7 +29,7 @@ struct tracepoint_func {
 struct tracepoint {
 	const char *name;		/* Tracepoint name */
 	struct static_key key;
-	void (*regfunc)(void);
+	int (*regfunc)(void);
 	void (*unregfunc)(void);
 	struct tracepoint_func __rcu *funcs;
 };
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index be586c632a0c..f72fcfe0e66a 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -81,7 +81,7 @@ static inline void tracepoint_synchronize_unregister(void)
 }
 
 #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
-extern void syscall_regfunc(void);
+extern int syscall_regfunc(void);
 extern void syscall_unregfunc(void);
 #endif /* CONFIG_HAVE_SYSCALL_TRACEPOINTS */
 
diff --git a/include/linux/types.h b/include/linux/types.h
index baf718324f4a..d501ad3ba247 100644
--- a/include/linux/types.h
+++ b/include/linux/types.h
@@ -154,8 +154,8 @@ typedef u64 dma_addr_t;
 typedef u32 dma_addr_t;
 #endif
 
-typedef unsigned __bitwise__ gfp_t;
-typedef unsigned __bitwise__ fmode_t;
+typedef unsigned __bitwise gfp_t;
+typedef unsigned __bitwise fmode_t;
 
 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 typedef u64 phys_addr_t;
diff --git a/include/linux/uio.h b/include/linux/uio.h
index 6e22b544d039..d5aba1512b8b 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -125,7 +125,7 @@ static inline bool iter_is_iovec(const struct iov_iter *i)
  *
  * The ?: is just for type safety.
  */
-#define iov_iter_rw(i) ((0 ? (struct iov_iter *)0 : (i))->type & RW_MASK)
+#define iov_iter_rw(i) ((0 ? (struct iov_iter *)0 : (i))->type & (READ | WRITE))
 
 /*
  * Cap the iov_iter by given limit; note that the second argument is
diff --git a/include/linux/usb.h b/include/linux/usb.h
index eba1f10e8cfd..7e68259360de 100644
--- a/include/linux/usb.h
+++ b/include/linux/usb.h
@@ -1160,7 +1160,7 @@ extern struct bus_type usb_bus_type;
  * @minor_base: the start of the minor range for this driver.
  *
  * This structure is used for the usb_register_dev() and
- * usb_unregister_dev() functions, to consolidate a number of the
+ * usb_deregister_dev() functions, to consolidate a number of the
  * parameters used for them.
  */
 struct usb_class_driver {
diff --git a/include/linux/usb/gadget.h b/include/linux/usb/gadget.h
index 8e81f9eb95e4..e4516e9ded0f 100644
--- a/include/linux/usb/gadget.h
+++ b/include/linux/usb/gadget.h
@@ -429,7 +429,9 @@ static inline struct usb_gadget *dev_to_usb_gadget(struct device *dev)
  */
 static inline size_t usb_ep_align(struct usb_ep *ep, size_t len)
 {
-	return round_up(len, (size_t)le16_to_cpu(ep->desc->wMaxPacketSize));
+	int max_packet_size = (size_t)usb_endpoint_maxp(ep->desc) & 0x7ff;
+
+	return round_up(len, max_packet_size);
 }
 
 /**
diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h
index 66fc13705ab7..40edf6a8533e 100644
--- a/include/linux/usb/hcd.h
+++ b/include/linux/usb/hcd.h
@@ -566,21 +566,22 @@ extern void usb_ep0_reinit(struct usb_device *);
 	((USB_DIR_OUT|USB_TYPE_STANDARD|USB_RECIP_INTERFACE)<<8)
 
 /* class requests from the USB 2.0 hub spec, table 11-15 */
+#define HUB_CLASS_REQ(dir, type, request) ((((dir) | (type)) << 8) | (request))
 /* GetBusState and SetHubDescriptor are optional, omitted */
-#define ClearHubFeature		(0x2000 | USB_REQ_CLEAR_FEATURE)
-#define ClearPortFeature	(0x2300 | USB_REQ_CLEAR_FEATURE)
-#define GetHubDescriptor	(0xa000 | USB_REQ_GET_DESCRIPTOR)
-#define GetHubStatus		(0xa000 | USB_REQ_GET_STATUS)
-#define GetPortStatus		(0xa300 | USB_REQ_GET_STATUS)
-#define SetHubFeature		(0x2000 | USB_REQ_SET_FEATURE)
-#define SetPortFeature		(0x2300 | USB_REQ_SET_FEATURE)
+#define ClearHubFeature		HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_HUB, USB_REQ_CLEAR_FEATURE)
+#define ClearPortFeature	HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_PORT, USB_REQ_CLEAR_FEATURE)
+#define GetHubDescriptor	HUB_CLASS_REQ(USB_DIR_IN, USB_RT_HUB, USB_REQ_GET_DESCRIPTOR)
+#define GetHubStatus		HUB_CLASS_REQ(USB_DIR_IN, USB_RT_HUB, USB_REQ_GET_STATUS)
+#define GetPortStatus		HUB_CLASS_REQ(USB_DIR_IN, USB_RT_PORT, USB_REQ_GET_STATUS)
+#define SetHubFeature		HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_HUB, USB_REQ_SET_FEATURE)
+#define SetPortFeature		HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_PORT, USB_REQ_SET_FEATURE)
 
 
 /*-------------------------------------------------------------------------*/
 
 /* class requests from USB 3.1 hub spec, table 10-7 */
-#define SetHubDepth		(0x2000 | HUB_SET_DEPTH)
-#define GetPortErrorCount	(0xa300 | HUB_GET_PORT_ERR_COUNT)
+#define SetHubDepth		HUB_CLASS_REQ(USB_DIR_OUT, USB_RT_HUB, HUB_SET_DEPTH)
+#define GetPortErrorCount	HUB_CLASS_REQ(USB_DIR_IN, USB_RT_PORT, HUB_GET_PORT_ERR_COUNT)
 
 /*
  * Generic bandwidth allocation constants/support
diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h
index dd66a952e8cd..11b92b047a1e 100644
--- a/include/linux/userfaultfd_k.h
+++ b/include/linux/userfaultfd_k.h
@@ -27,7 +27,7 @@
 #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK)
 #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS)
 
-extern int handle_userfault(struct fault_env *fe, unsigned long reason);
+extern int handle_userfault(struct vm_fault *vmf, unsigned long reason);
 
 extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,
 			    unsigned long src_start, unsigned long len);
@@ -55,7 +55,7 @@ static inline bool userfaultfd_armed(struct vm_area_struct *vma)
 #else /* CONFIG_USERFAULTFD */
 
 /* mm helpers */
-static inline int handle_userfault(struct fault_env *fe, unsigned long reason)
+static inline int handle_userfault(struct vm_fault *vmf, unsigned long reason)
 {
 	return VM_FAULT_SIGBUS;
 }
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index 0ecae0b1cd34..edf9b2cad277 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -75,7 +75,16 @@ struct vfio_iommu_driver_ops {
 					struct iommu_group *group);
 	void		(*detach_group)(void *iommu_data,
 					struct iommu_group *group);
-
+	int		(*pin_pages)(void *iommu_data, unsigned long *user_pfn,
+				     int npage, int prot,
+				     unsigned long *phys_pfn);
+	int		(*unpin_pages)(void *iommu_data,
+				       unsigned long *user_pfn, int npage);
+	int		(*register_notifier)(void *iommu_data,
+					     unsigned long *events,
+					     struct notifier_block *nb);
+	int		(*unregister_notifier)(void *iommu_data,
+					       struct notifier_block *nb);
 };
 
 extern int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops);
@@ -92,6 +101,36 @@ extern int vfio_external_user_iommu_id(struct vfio_group *group);
 extern long vfio_external_check_extension(struct vfio_group *group,
 					  unsigned long arg);
 
+#define VFIO_PIN_PAGES_MAX_ENTRIES	(PAGE_SIZE/sizeof(unsigned long))
+
+extern int vfio_pin_pages(struct device *dev, unsigned long *user_pfn,
+			  int npage, int prot, unsigned long *phys_pfn);
+extern int vfio_unpin_pages(struct device *dev, unsigned long *user_pfn,
+			    int npage);
+
+/* each type has independent events */
+enum vfio_notify_type {
+	VFIO_IOMMU_NOTIFY = 0,
+	VFIO_GROUP_NOTIFY = 1,
+};
+
+/* events for VFIO_IOMMU_NOTIFY */
+#define VFIO_IOMMU_NOTIFY_DMA_UNMAP	BIT(0)
+
+/* events for VFIO_GROUP_NOTIFY */
+#define VFIO_GROUP_NOTIFY_SET_KVM	BIT(0)
+
+extern int vfio_register_notifier(struct device *dev,
+				  enum vfio_notify_type type,
+				  unsigned long *required_events,
+				  struct notifier_block *nb);
+extern int vfio_unregister_notifier(struct device *dev,
+				    enum vfio_notify_type type,
+				    struct notifier_block *nb);
+
+struct kvm;
+extern void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm);
+
 /*
  * Sub-module helpers
  */
@@ -103,6 +142,13 @@ extern struct vfio_info_cap_header *vfio_info_cap_add(
 		struct vfio_info_cap *caps, size_t size, u16 id, u16 version);
 extern void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset);
 
+extern int vfio_info_add_capability(struct vfio_info_cap *caps,
+				    int cap_type_id, void *cap_type);
+
+extern int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr,
+					      int num_irqs, int max_irq_type,
+					      size_t *data_size);
+
 struct pci_dev;
 #ifdef CONFIG_EEH
 extern void vfio_spapr_pci_eeh_open(struct pci_dev *pdev);
diff --git a/include/linux/vme.h b/include/linux/vme.h
index ea6095deba20..8c589176c2f8 100644
--- a/include/linux/vme.h
+++ b/include/linux/vme.h
@@ -113,7 +113,6 @@ struct vme_driver {
 	int (*match)(struct vme_dev *);
 	int (*probe)(struct vme_dev *);
 	int (*remove)(struct vme_dev *);
-	void (*shutdown)(void);
 	struct device_driver driver;
 	struct list_head devices;
 };
diff --git a/include/linux/vt_kern.h b/include/linux/vt_kern.h
index 6abd24f258bc..833fdd4794a0 100644
--- a/include/linux/vt_kern.h
+++ b/include/linux/vt_kern.h
@@ -191,5 +191,7 @@ extern void vt_set_led_state(int console, int leds);
 extern void vt_kbd_con_start(int console);
 extern void vt_kbd_con_stop(int console);
 
+void vc_scrolldelta_helper(struct vc_data *c, int lines,
+		unsigned int rolled_over, void *_base, unsigned int size);
 
 #endif /* _VT_KERN_H */
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index d4f16cf6281c..a26cc437293c 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -603,14 +603,6 @@ static inline bool schedule_delayed_work(struct delayed_work *dwork,
 	return queue_delayed_work(system_wq, dwork, delay);
 }
 
-/**
- * keventd_up - is workqueue initialized yet?
- */
-static inline bool keventd_up(void)
-{
-	return system_wq != NULL;
-}
-
 #ifndef CONFIG_SMP
 static inline long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
 {
@@ -645,4 +637,7 @@ int workqueue_online_cpu(unsigned int cpu);
 int workqueue_offline_cpu(unsigned int cpu);
 #endif
 
+int __init workqueue_init_early(void);
+int __init workqueue_init(void);
+
 #endif
diff --git a/include/linux/writeback.h b/include/linux/writeback.h
index 797100e10010..c78f9f0920b5 100644
--- a/include/linux/writeback.h
+++ b/include/linux/writeback.h
@@ -9,6 +9,9 @@
 #include <linux/fs.h>
 #include <linux/flex_proportions.h>
 #include <linux/backing-dev-defs.h>
+#include <linux/blk_types.h>
+
+struct bio;
 
 DECLARE_PER_CPU(int, dirty_throttle_leaks);
 
@@ -100,6 +103,16 @@ struct writeback_control {
 #endif
 };
 
+static inline int wbc_to_write_flags(struct writeback_control *wbc)
+{
+	if (wbc->sync_mode == WB_SYNC_ALL)
+		return REQ_SYNC;
+	else if (wbc->for_kupdate || wbc->for_background)
+		return REQ_BACKGROUND;
+
+	return 0;
+}
+
 /*
  * A wb_domain represents a domain that wb's (bdi_writeback's) belong to
  * and are measured against each other in.  There always is one global
diff --git a/include/rdma/ib_cm.h b/include/rdma/ib_cm.h
index 92a7d85917b4..b49258b16f4e 100644
--- a/include/rdma/ib_cm.h
+++ b/include/rdma/ib_cm.h
@@ -603,4 +603,10 @@ struct ib_cm_sidr_rep_param {
 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
 			struct ib_cm_sidr_rep_param *param);
 
+/**
+ * ibcm_reject_msg - return a pointer to a reject message string.
+ * @reason: Value returned in the REJECT event status field.
+ */
+const char *__attribute_const__ ibcm_reject_msg(int reason);
+
 #endif /* IB_CM_H */
diff --git a/include/rdma/ib_mad.h b/include/rdma/ib_mad.h
index c8a773ffe23b..981214b3790c 100644
--- a/include/rdma/ib_mad.h
+++ b/include/rdma/ib_mad.h
@@ -46,7 +46,7 @@
 #define IB_MGMT_BASE_VERSION			1
 #define OPA_MGMT_BASE_VERSION			0x80
 
-#define OPA_SMP_CLASS_VERSION			0x80
+#define OPA_SM_CLASS_VERSION			0x80
 
 /* Management classes */
 #define IB_MGMT_CLASS_SUBN_LID_ROUTED		0x01
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 5ad43a487745..8029d2a51f14 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -1102,6 +1102,7 @@ enum ib_qp_attr_mask {
 	IB_QP_RESERVED2			= (1<<22),
 	IB_QP_RESERVED3			= (1<<23),
 	IB_QP_RESERVED4			= (1<<24),
+	IB_QP_RATE_LIMIT		= (1<<25),
 };
 
 enum ib_qp_state {
@@ -1151,6 +1152,7 @@ struct ib_qp_attr {
 	u8			rnr_retry;
 	u8			alt_port_num;
 	u8			alt_timeout;
+	u32			rate_limit;
 };
 
 enum ib_wr_opcode {
@@ -1592,17 +1594,19 @@ enum ib_flow_attr_type {
 /* Supported steering header types */
 enum ib_flow_spec_type {
 	/* L2 headers*/
-	IB_FLOW_SPEC_ETH	= 0x20,
-	IB_FLOW_SPEC_IB		= 0x22,
+	IB_FLOW_SPEC_ETH		= 0x20,
+	IB_FLOW_SPEC_IB			= 0x22,
 	/* L3 header*/
-	IB_FLOW_SPEC_IPV4	= 0x30,
-	IB_FLOW_SPEC_IPV6	= 0x31,
+	IB_FLOW_SPEC_IPV4		= 0x30,
+	IB_FLOW_SPEC_IPV6		= 0x31,
 	/* L4 headers*/
-	IB_FLOW_SPEC_TCP	= 0x40,
-	IB_FLOW_SPEC_UDP	= 0x41
+	IB_FLOW_SPEC_TCP		= 0x40,
+	IB_FLOW_SPEC_UDP		= 0x41,
+	IB_FLOW_SPEC_VXLAN_TUNNEL	= 0x50,
+	IB_FLOW_SPEC_INNER		= 0x100,
 };
 #define IB_FLOW_SPEC_LAYER_MASK	0xF0
-#define IB_FLOW_SPEC_SUPPORT_LAYERS 4
+#define IB_FLOW_SPEC_SUPPORT_LAYERS 8
 
 /* Flow steering rule priority is set according to it's domain.
  * Lower domain value means higher priority.
@@ -1630,7 +1634,7 @@ struct ib_flow_eth_filter {
 };
 
 struct ib_flow_spec_eth {
-	enum ib_flow_spec_type	  type;
+	u32			  type;
 	u16			  size;
 	struct ib_flow_eth_filter val;
 	struct ib_flow_eth_filter mask;
@@ -1644,7 +1648,7 @@ struct ib_flow_ib_filter {
 };
 
 struct ib_flow_spec_ib {
-	enum ib_flow_spec_type	 type;
+	u32			 type;
 	u16			 size;
 	struct ib_flow_ib_filter val;
 	struct ib_flow_ib_filter mask;
@@ -1669,7 +1673,7 @@ struct ib_flow_ipv4_filter {
 };
 
 struct ib_flow_spec_ipv4 {
-	enum ib_flow_spec_type	   type;
+	u32			   type;
 	u16			   size;
 	struct ib_flow_ipv4_filter val;
 	struct ib_flow_ipv4_filter mask;
@@ -1687,7 +1691,7 @@ struct ib_flow_ipv6_filter {
 };
 
 struct ib_flow_spec_ipv6 {
-	enum ib_flow_spec_type	   type;
+	u32			   type;
 	u16			   size;
 	struct ib_flow_ipv6_filter val;
 	struct ib_flow_ipv6_filter mask;
@@ -1701,15 +1705,30 @@ struct ib_flow_tcp_udp_filter {
 };
 
 struct ib_flow_spec_tcp_udp {
-	enum ib_flow_spec_type	      type;
+	u32			      type;
 	u16			      size;
 	struct ib_flow_tcp_udp_filter val;
 	struct ib_flow_tcp_udp_filter mask;
 };
 
+struct ib_flow_tunnel_filter {
+	__be32	tunnel_id;
+	u8	real_sz[0];
+};
+
+/* ib_flow_spec_tunnel describes the Vxlan tunnel
+ * the tunnel_id from val has the vni value
+ */
+struct ib_flow_spec_tunnel {
+	u32			      type;
+	u16			      size;
+	struct ib_flow_tunnel_filter  val;
+	struct ib_flow_tunnel_filter  mask;
+};
+
 union ib_flow_spec {
 	struct {
-		enum ib_flow_spec_type	type;
+		u32			type;
 		u16			size;
 	};
 	struct ib_flow_spec_eth		eth;
@@ -1717,6 +1736,7 @@ union ib_flow_spec {
 	struct ib_flow_spec_ipv4        ipv4;
 	struct ib_flow_spec_tcp_udp	tcp_udp;
 	struct ib_flow_spec_ipv6        ipv6;
+	struct ib_flow_spec_tunnel      tunnel;
 };
 
 struct ib_flow_attr {
@@ -1933,7 +1953,8 @@ struct ib_device {
 					       struct ib_udata *udata);
 	int                        (*dealloc_pd)(struct ib_pd *pd);
 	struct ib_ah *             (*create_ah)(struct ib_pd *pd,
-						struct ib_ah_attr *ah_attr);
+						struct ib_ah_attr *ah_attr,
+						struct ib_udata *udata);
 	int                        (*modify_ah)(struct ib_ah *ah,
 						struct ib_ah_attr *ah_attr);
 	int                        (*query_ah)(struct ib_ah *ah,
@@ -2581,6 +2602,24 @@ void ib_dealloc_pd(struct ib_pd *pd);
 struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
 
 /**
+ * ib_get_gids_from_rdma_hdr - Get sgid and dgid from GRH or IPv4 header
+ *   work completion.
+ * @hdr: the L3 header to parse
+ * @net_type: type of header to parse
+ * @sgid: place to store source gid
+ * @dgid: place to store destination gid
+ */
+int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
+			      enum rdma_network_type net_type,
+			      union ib_gid *sgid, union ib_gid *dgid);
+
+/**
+ * ib_get_rdma_header_version - Get the header version
+ * @hdr: the L3 header to parse
+ */
+int ib_get_rdma_header_version(const union rdma_network_hdr *hdr);
+
+/**
  * ib_init_ah_from_wc - Initializes address handle attributes from a
  *   work completion.
  * @device: Device on which the received message arrived.
@@ -3357,4 +3396,7 @@ int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
 void ib_drain_rq(struct ib_qp *qp);
 void ib_drain_sq(struct ib_qp *qp);
 void ib_drain_qp(struct ib_qp *qp);
+
+int ib_resolve_eth_dmac(struct ib_device *device,
+			struct ib_ah_attr *ah_attr);
 #endif /* IB_VERBS_H */
diff --git a/include/rdma/iw_cm.h b/include/rdma/iw_cm.h
index 6d0065c322b7..5cd7701db148 100644
--- a/include/rdma/iw_cm.h
+++ b/include/rdma/iw_cm.h
@@ -253,4 +253,10 @@ int iw_cm_disconnect(struct iw_cm_id *cm_id, int abrupt);
 int iw_cm_init_qp_attr(struct iw_cm_id *cm_id, struct ib_qp_attr *qp_attr,
 		       int *qp_attr_mask);
 
+/**
+ * iwcm_reject_msg - return a pointer to a reject message string.
+ * @reason: Value returned in the REJECT event status field.
+ */
+const char *__attribute_const__ iwcm_reject_msg(int reason);
+
 #endif /* IW_CM_H */
diff --git a/include/rdma/opa_smi.h b/include/rdma/opa_smi.h
index 4a529ef47995..f7896117936e 100644
--- a/include/rdma/opa_smi.h
+++ b/include/rdma/opa_smi.h
@@ -44,8 +44,6 @@
 #define OPA_MAX_SLS				32
 #define OPA_MAX_SCS				32
 
-#define OPA_SMI_CLASS_VERSION			0x80
-
 #define OPA_LID_PERMISSIVE			cpu_to_be32(0xFFFFFFFF)
 
 struct opa_smp {
diff --git a/include/rdma/rdma_cm.h b/include/rdma/rdma_cm.h
index 81fb1d15e8bb..d3968b561f86 100644
--- a/include/rdma/rdma_cm.h
+++ b/include/rdma/rdma_cm.h
@@ -388,4 +388,29 @@ int rdma_set_afonly(struct rdma_cm_id *id, int afonly);
  */
 __be64 rdma_get_service_id(struct rdma_cm_id *id, struct sockaddr *addr);
 
+/**
+ * rdma_reject_msg - return a pointer to a reject message string.
+ * @id: Communication identifier that received the REJECT event.
+ * @reason: Value returned in the REJECT event status field.
+ */
+const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id,
+						int reason);
+/**
+ * rdma_is_consumer_reject - return true if the consumer rejected the connect
+ *                           request.
+ * @id: Communication identifier that received the REJECT event.
+ * @reason: Value returned in the REJECT event status field.
+ */
+bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason);
+
+/**
+ * rdma_consumer_reject_data - return the consumer reject private data and
+ *			       length, if any.
+ * @id: Communication identifier that received the REJECT event.
+ * @ev: RDMA CM reject event.
+ * @data_len: Pointer to the resulting length of the consumer data.
+ */
+const void *rdma_consumer_reject_data(struct rdma_cm_id *id,
+				      struct rdma_cm_event *ev, u8 *data_len);
+
 #endif /* RDMA_CM_H */
diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h
index e31502107a58..861e23eaebda 100644
--- a/include/rdma/rdma_vt.h
+++ b/include/rdma/rdma_vt.h
@@ -185,6 +185,27 @@ struct rvt_driver_provided {
 	 * check_support() for details.
 	 */
 
+	/* hot path calldowns in a single cacheline */
+
+	/*
+	 * Give the driver a notice that there is send work to do. It is up to
+	 * the driver to generally push the packets out, this just queues the
+	 * work with the driver. There are two variants here. The no_lock
+	 * version requires the s_lock not to be held. The other assumes the
+	 * s_lock is held.
+	 */
+	void (*schedule_send)(struct rvt_qp *qp);
+	void (*schedule_send_no_lock)(struct rvt_qp *qp);
+
+	/* Driver specific work request checking */
+	int (*check_send_wqe)(struct rvt_qp *qp, struct rvt_swqe *wqe);
+
+	/*
+	 * Sometimes rdmavt needs to kick the driver's send progress. That is
+	 * done by this call back.
+	 */
+	void (*do_send)(struct rvt_qp *qp);
+
 	/* Passed to ib core registration. Callback to create syfs files */
 	int (*port_callback)(struct ib_device *, u8, struct kobject *);
 
@@ -223,22 +244,6 @@ struct rvt_driver_provided {
 	void (*notify_qp_reset)(struct rvt_qp *qp);
 
 	/*
-	 * Give the driver a notice that there is send work to do. It is up to
-	 * the driver to generally push the packets out, this just queues the
-	 * work with the driver. There are two variants here. The no_lock
-	 * version requires the s_lock not to be held. The other assumes the
-	 * s_lock is held.
-	 */
-	void (*schedule_send)(struct rvt_qp *qp);
-	void (*schedule_send_no_lock)(struct rvt_qp *qp);
-
-	/*
-	 * Sometimes rdmavt needs to kick the driver's send progress. That is
-	 * done by this call back.
-	 */
-	void (*do_send)(struct rvt_qp *qp);
-
-	/*
 	 * Get a path mtu from the driver based on qp attributes.
 	 */
 	int (*get_pmtu_from_attr)(struct rvt_dev_info *rdi, struct rvt_qp *qp,
@@ -324,9 +329,6 @@ struct rvt_driver_provided {
 	void (*modify_qp)(struct rvt_qp *qp, struct ib_qp_attr *attr,
 			  int attr_mask, struct ib_udata *udata);
 
-	/* Driver specific work request checking */
-	int (*check_send_wqe)(struct rvt_qp *qp, struct rvt_swqe *wqe);
-
 	/* Notify driver a mad agent has been created */
 	void (*notify_create_mad_agent)(struct rvt_dev_info *rdi, int port_idx);
 
@@ -355,12 +357,12 @@ struct rvt_dev_info {
 	/* post send table */
 	const struct rvt_operation_params *post_parms;
 
-	struct rvt_mregion __rcu *dma_mr;
-	struct rvt_lkey_table lkey_table;
-
 	/* Driver specific helper functions */
 	struct rvt_driver_provided driver_f;
 
+	struct rvt_mregion __rcu *dma_mr;
+	struct rvt_lkey_table lkey_table;
+
 	/* Internal use */
 	int n_pds_allocated;
 	spinlock_t n_pds_lock; /* Protect pd allocated count */
diff --git a/include/rdma/rdmavt_mr.h b/include/rdma/rdmavt_mr.h
index 6b3c6c8b6b77..de59de28b6a2 100644
--- a/include/rdma/rdmavt_mr.h
+++ b/include/rdma/rdmavt_mr.h
@@ -90,11 +90,15 @@ struct rvt_mregion {
 #define RVT_MAX_LKEY_TABLE_BITS 23
 
 struct rvt_lkey_table {
-	spinlock_t lock; /* protect changes in this struct */
-	u32 next;               /* next unused index (speeds search) */
-	u32 gen;                /* generation count */
+	/* read mostly fields */
 	u32 max;                /* size of the table */
+	u32 shift;              /* lkey/rkey shift */
 	struct rvt_mregion __rcu **table;
+	/* writeable fields */
+	/* protect changes in this struct */
+	spinlock_t lock ____cacheline_aligned_in_smp;
+	u32 next;               /* next unused index (speeds search) */
+	u32 gen;                /* generation count */
 };
 
 /*
diff --git a/include/rdma/rdmavt_qp.h b/include/rdma/rdmavt_qp.h
index 2c5183ef0243..f3dbd157ae5c 100644
--- a/include/rdma/rdmavt_qp.h
+++ b/include/rdma/rdmavt_qp.h
@@ -51,6 +51,7 @@
 #include <rdma/rdma_vt.h>
 #include <rdma/ib_pack.h>
 #include <rdma/ib_verbs.h>
+#include <rdma/rdmavt_cq.h>
 /*
  * Atomic bit definitions for r_aflags.
  */
@@ -485,6 +486,23 @@ static inline void rvt_put_qp(struct rvt_qp *qp)
 }
 
 /**
+ * rvt_put_swqe - drop mr refs held by swqe
+ * @wqe - the send wqe
+ *
+ * This drops any mr references held by the swqe
+ */
+static inline void rvt_put_swqe(struct rvt_swqe *wqe)
+{
+	int i;
+
+	for (i = 0; i < wqe->wr.num_sge; i++) {
+		struct rvt_sge *sge = &wqe->sg_list[i];
+
+		rvt_put_mr(sge->mr);
+	}
+}
+
+/**
  * rvt_qp_wqe_reserve - reserve operation
  * @qp - the rvt qp
  * @wqe - the send wqe
@@ -527,6 +545,65 @@ static inline void rvt_qp_wqe_unreserve(
 	}
 }
 
+extern const enum ib_wc_opcode ib_rvt_wc_opcode[];
+
+/**
+ * rvt_qp_swqe_complete() - insert send completion
+ * @qp - the qp
+ * @wqe - the send wqe
+ * @status - completion status
+ *
+ * Insert a send completion into the completion
+ * queue if the qp indicates it should be done.
+ *
+ * See IBTA 10.7.3.1 for info on completion
+ * control.
+ */
+static inline void rvt_qp_swqe_complete(
+	struct rvt_qp *qp,
+	struct rvt_swqe *wqe,
+	enum ib_wc_status status)
+{
+	if (unlikely(wqe->wr.send_flags & RVT_SEND_RESERVE_USED))
+		return;
+	if (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
+	    (wqe->wr.send_flags & IB_SEND_SIGNALED) ||
+	     status != IB_WC_SUCCESS) {
+		struct ib_wc wc;
+
+		memset(&wc, 0, sizeof(wc));
+		wc.wr_id = wqe->wr.wr_id;
+		wc.status = status;
+		wc.opcode = ib_rvt_wc_opcode[wqe->wr.opcode];
+		wc.qp = &qp->ibqp;
+		wc.byte_len = wqe->length;
+		rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.send_cq), &wc,
+			     status != IB_WC_SUCCESS);
+	}
+}
+
+/**
+ * @qp - the qp pair
+ * @len - the length
+ *
+ * Perform a shift based mtu round up divide
+ */
+static inline u32 rvt_div_round_up_mtu(struct rvt_qp *qp, u32 len)
+{
+	return (len + qp->pmtu - 1) >> qp->log_pmtu;
+}
+
+/**
+ * @qp - the qp pair
+ * @len - the length
+ *
+ * Perform a shift based mtu divide
+ */
+static inline u32 rvt_div_mtu(struct rvt_qp *qp, u32 len)
+{
+	return len >> qp->log_pmtu;
+}
+
 extern const int  ib_rvt_state_ops[];
 
 struct rvt_dev_info;
diff --git a/include/scsi/iscsi_proto.h b/include/scsi/iscsi_proto.h
index c1260d80ef30..df156f1d50b2 100644
--- a/include/scsi/iscsi_proto.h
+++ b/include/scsi/iscsi_proto.h
@@ -74,7 +74,7 @@ static inline int iscsi_sna_gte(u32 n1, u32 n2)
 #define zero_data(p) {p[0]=0;p[1]=0;p[2]=0;}
 
 /* initiator tags; opaque for target */
-typedef uint32_t __bitwise__ itt_t;
+typedef uint32_t __bitwise itt_t;
 /* below makes sense only for initiator that created this tag */
 #define build_itt(itt, age) ((__force itt_t)\
 	((itt) | ((age) << ISCSI_AGE_SHIFT)))
diff --git a/include/scsi/libfc.h b/include/scsi/libfc.h
index 7428a53257ca..96dd0b3f70d7 100644
--- a/include/scsi/libfc.h
+++ b/include/scsi/libfc.h
@@ -44,6 +44,11 @@
 #define	FC_NO_ERR	0	/* no error */
 #define	FC_EX_TIMEOUT	1	/* Exchange timeout */
 #define	FC_EX_CLOSED	2	/* Exchange closed */
+#define FC_EX_ALLOC_ERR	3	/* Exchange allocation failed */
+#define FC_EX_XMIT_ERR	4	/* Exchange transmit failed */
+#define FC_EX_ELS_RJT	5	/* ELS rejected */
+#define FC_EX_INV_LOGIN	6	/* Login not completed */
+#define FC_EX_SEQ_ERR	6	/* Exchange sequence error */
 
 /**
  * enum fc_lport_state - Local port states
@@ -350,7 +355,8 @@ struct fc_fcp_pkt {
 
 	/* Timeout/error related information */
 	struct timer_list timer;
-	int	          wait_for_comp;
+	int		  wait_for_comp;
+	int		  timer_delay;
 	u32		  recov_retry;
 	struct fc_seq	  *recov_seq;
 	struct completion tm_done;
@@ -385,6 +391,7 @@ struct fc_seq {
 
 #define FC_EX_DONE		(1 << 0) /* ep is completed */
 #define FC_EX_RST_CLEANUP	(1 << 1) /* reset is forcing completion */
+#define FC_EX_QUARANTINE	(1 << 2) /* exch is quarantined */
 
 /**
  * struct fc_exch - Fibre Channel Exchange
@@ -478,37 +485,6 @@ struct libfc_function_template {
 				     void *arg, u32 timer_msec);
 
 	/*
-	 * Send the FC frame payload using a new exchange and sequence.
-	 *
-	 * The exchange response handler is set in this routine to resp()
-	 * function pointer. It can be called in two scenarios: if a timeout
-	 * occurs or if a response frame is received for the exchange. The
-	 * fc_frame pointer in response handler will also indicate timeout
-	 * as error using IS_ERR related macros.
-	 *
-	 * The exchange destructor handler is also set in this routine.
-	 * The destructor handler is invoked by EM layer when exchange
-	 * is about to free, this can be used by caller to free its
-	 * resources along with exchange free.
-	 *
-	 * The arg is passed back to resp and destructor handler.
-	 *
-	 * The timeout value (in msec) for an exchange is set if non zero
-	 * timer_msec argument is specified. The timer is canceled when
-	 * it fires or when the exchange is done. The exchange timeout handler
-	 * is registered by EM layer.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	struct fc_seq *(*exch_seq_send)(struct fc_lport *, struct fc_frame *,
-					void (*resp)(struct fc_seq *,
-						     struct fc_frame *,
-						     void *),
-					void (*destructor)(struct fc_seq *,
-							   void *),
-					void *, unsigned int timer_msec);
-
-	/*
 	 * Sets up the DDP context for a given exchange id on the given
 	 * scatterlist if LLD supports DDP for large receive.
 	 *
@@ -537,73 +513,6 @@ struct libfc_function_template {
 	 * STATUS: OPTIONAL
 	 */
 	void (*get_lesb)(struct fc_lport *, struct fc_els_lesb *lesb);
-	/*
-	 * Send a frame using an existing sequence and exchange.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	int (*seq_send)(struct fc_lport *, struct fc_seq *,
-			struct fc_frame *);
-
-	/*
-	 * Send an ELS response using information from the received frame.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	void (*seq_els_rsp_send)(struct fc_frame *, enum fc_els_cmd,
-				 struct fc_seq_els_data *);
-
-	/*
-	 * Abort an exchange and sequence. Generally called because of a
-	 * exchange timeout or an abort from the upper layer.
-	 *
-	 * A timer_msec can be specified for abort timeout, if non-zero
-	 * timer_msec value is specified then exchange resp handler
-	 * will be called with timeout error if no response to abort.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	int (*seq_exch_abort)(const struct fc_seq *,
-			      unsigned int timer_msec);
-
-	/*
-	 * Indicate that an exchange/sequence tuple is complete and the memory
-	 * allocated for the related objects may be freed.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	void (*exch_done)(struct fc_seq *);
-
-	/*
-	 * Start a new sequence on the same exchange/sequence tuple.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	struct fc_seq *(*seq_start_next)(struct fc_seq *);
-
-	/*
-	 * Set a response handler for the exchange of the sequence.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	void (*seq_set_resp)(struct fc_seq *sp,
-			     void (*resp)(struct fc_seq *, struct fc_frame *,
-					  void *),
-			     void *arg);
-
-	/*
-	 * Assign a sequence for an incoming request frame.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	struct fc_seq *(*seq_assign)(struct fc_lport *, struct fc_frame *);
-
-	/*
-	 * Release the reference on the sequence returned by seq_assign().
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	void (*seq_release)(struct fc_seq *);
 
 	/*
 	 * Reset an exchange manager, completing all sequences and exchanges.
@@ -615,27 +524,6 @@ struct libfc_function_template {
 	void (*exch_mgr_reset)(struct fc_lport *, u32 s_id, u32 d_id);
 
 	/*
-	 * Flush the rport work queue. Generally used before shutdown.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	void (*rport_flush_queue)(void);
-
-	/*
-	 * Receive a frame for a local port.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	void (*lport_recv)(struct fc_lport *, struct fc_frame *);
-
-	/*
-	 * Reset the local port.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	int (*lport_reset)(struct fc_lport *);
-
-	/*
 	 * Set the local port FC_ID.
 	 *
 	 * This may be provided by the LLD to allow it to be
@@ -656,54 +544,6 @@ struct libfc_function_template {
 				  struct fc_frame *);
 
 	/*
-	 * Create a remote port with a given port ID
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	struct fc_rport_priv *(*rport_create)(struct fc_lport *, u32);
-
-	/*
-	 * Initiates the RP state machine. It is called from the LP module.
-	 * This function will issue the following commands to the N_Port
-	 * identified by the FC ID provided.
-	 *
-	 * - PLOGI
-	 * - PRLI
-	 * - RTV
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	int (*rport_login)(struct fc_rport_priv *);
-
-	/*
-	 * Logoff, and remove the rport from the transport if
-	 * it had been added. This will send a LOGO to the target.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	int (*rport_logoff)(struct fc_rport_priv *);
-
-	/*
-	 * Receive a request from a remote port.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	void (*rport_recv_req)(struct fc_lport *, struct fc_frame *);
-
-	/*
-	 * lookup an rport by it's port ID.
-	 *
-	 * STATUS: OPTIONAL
-	 */
-	struct fc_rport_priv *(*rport_lookup)(const struct fc_lport *, u32);
-
-	/*
-	 * Destroy an rport after final kref_put().
-	 * The argument is a pointer to the kref inside the fc_rport_priv.
-	 */
-	void (*rport_destroy)(struct kref *);
-
-	/*
 	 * Callback routine after the remote port is logged in
 	 *
 	 * STATUS: OPTIONAL
@@ -1068,18 +908,26 @@ void fc_vport_setlink(struct fc_lport *);
 void fc_vports_linkchange(struct fc_lport *);
 int fc_lport_config(struct fc_lport *);
 int fc_lport_reset(struct fc_lport *);
+void fc_lport_recv(struct fc_lport *lport, struct fc_frame *fp);
 int fc_set_mfs(struct fc_lport *, u32 mfs);
 struct fc_lport *libfc_vport_create(struct fc_vport *, int privsize);
 struct fc_lport *fc_vport_id_lookup(struct fc_lport *, u32 port_id);
-int fc_lport_bsg_request(struct fc_bsg_job *);
+int fc_lport_bsg_request(struct bsg_job *);
 void fc_lport_set_local_id(struct fc_lport *, u32 port_id);
 void fc_lport_iterate(void (*func)(struct fc_lport *, void *), void *);
 
 /*
  * REMOTE PORT LAYER
  *****************************/
-int fc_rport_init(struct fc_lport *);
 void fc_rport_terminate_io(struct fc_rport *);
+struct fc_rport_priv *fc_rport_lookup(const struct fc_lport *lport,
+				      u32 port_id);
+struct fc_rport_priv *fc_rport_create(struct fc_lport *, u32);
+void fc_rport_destroy(struct kref *kref);
+int fc_rport_login(struct fc_rport_priv *rdata);
+int fc_rport_logoff(struct fc_rport_priv *rdata);
+void fc_rport_recv_req(struct fc_lport *lport, struct fc_frame *fp);
+void fc_rport_flush_queue(void);
 
 /*
  * DISCOVERY LAYER
@@ -1131,6 +979,21 @@ void fc_fill_hdr(struct fc_frame *, const struct fc_frame *,
  *****************************/
 int fc_exch_init(struct fc_lport *);
 void fc_exch_update_stats(struct fc_lport *lport);
+struct fc_seq *fc_exch_seq_send(struct fc_lport *lport,
+				struct fc_frame *fp,
+				void (*resp)(struct fc_seq *,
+					     struct fc_frame *fp,
+					     void *arg),
+				void (*destructor)(struct fc_seq *, void *),
+				void *arg, u32 timer_msec);
+void fc_seq_els_rsp_send(struct fc_frame *, enum fc_els_cmd,
+			 struct fc_seq_els_data *);
+struct fc_seq *fc_seq_start_next(struct fc_seq *sp);
+void fc_seq_set_resp(struct fc_seq *sp,
+		     void (*resp)(struct fc_seq *, struct fc_frame *, void *),
+		     void *arg);
+struct fc_seq *fc_seq_assign(struct fc_lport *lport, struct fc_frame *fp);
+void fc_seq_release(struct fc_seq *sp);
 struct fc_exch_mgr_anchor *fc_exch_mgr_add(struct fc_lport *,
 					   struct fc_exch_mgr *,
 					   bool (*match)(struct fc_frame *));
@@ -1142,6 +1005,9 @@ struct fc_exch_mgr *fc_exch_mgr_alloc(struct fc_lport *, enum fc_class class,
 void fc_exch_mgr_free(struct fc_lport *);
 void fc_exch_recv(struct fc_lport *, struct fc_frame *);
 void fc_exch_mgr_reset(struct fc_lport *, u32 s_id, u32 d_id);
+int fc_seq_send(struct fc_lport *lport, struct fc_seq *sp, struct fc_frame *fp);
+int fc_seq_exch_abort(const struct fc_seq *, unsigned int timer_msec);
+void fc_exch_done(struct fc_seq *sp);
 
 /*
  * Functions for fc_functions_template
diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h
index 8a9563144890..8990e580b278 100644
--- a/include/scsi/scsi_device.h
+++ b/include/scsi/scsi_device.h
@@ -414,14 +414,14 @@ extern int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
 extern int scsi_execute_req_flags(struct scsi_device *sdev,
 	const unsigned char *cmd, int data_direction, void *buffer,
 	unsigned bufflen, struct scsi_sense_hdr *sshdr, int timeout,
-	int retries, int *resid, u64 flags);
+	int retries, int *resid, u64 flags, req_flags_t rq_flags);
 static inline int scsi_execute_req(struct scsi_device *sdev,
 	const unsigned char *cmd, int data_direction, void *buffer,
 	unsigned bufflen, struct scsi_sense_hdr *sshdr, int timeout,
 	int retries, int *resid)
 {
 	return scsi_execute_req_flags(sdev, cmd, data_direction, buffer,
-		bufflen, sshdr, timeout, retries, resid, 0);
+		bufflen, sshdr, timeout, retries, resid, 0, 0);
 }
 extern void sdev_disable_disk_events(struct scsi_device *sdev);
 extern void sdev_enable_disk_events(struct scsi_device *sdev);
diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h
index 7e4cd53139ed..36680f13270d 100644
--- a/include/scsi/scsi_host.h
+++ b/include/scsi/scsi_host.h
@@ -278,6 +278,14 @@ struct scsi_host_template {
 	int (* change_queue_depth)(struct scsi_device *, int);
 
 	/*
+	 * This functions lets the driver expose the queue mapping
+	 * to the block layer.
+	 *
+	 * Status: OPTIONAL
+	 */
+	int (* map_queues)(struct Scsi_Host *shost);
+
+	/*
 	 * This function determines the BIOS parameters for a given
 	 * harddisk.  These tend to be numbers that are made up by
 	 * the host adapter.  Parameters:
diff --git a/include/scsi/scsi_proto.h b/include/scsi/scsi_proto.h
index d1defd1ebd95..6ba66e01f6df 100644
--- a/include/scsi/scsi_proto.h
+++ b/include/scsi/scsi_proto.h
@@ -299,4 +299,21 @@ struct scsi_lun {
 #define SCSI_ACCESS_STATE_MASK        0x0f
 #define SCSI_ACCESS_STATE_PREFERRED   0x80
 
+/* Reporting options for REPORT ZONES */
+enum zbc_zone_reporting_options {
+	ZBC_ZONE_REPORTING_OPTION_ALL = 0,
+	ZBC_ZONE_REPORTING_OPTION_EMPTY,
+	ZBC_ZONE_REPORTING_OPTION_IMPLICIT_OPEN,
+	ZBC_ZONE_REPORTING_OPTION_EXPLICIT_OPEN,
+	ZBC_ZONE_REPORTING_OPTION_CLOSED,
+	ZBC_ZONE_REPORTING_OPTION_FULL,
+	ZBC_ZONE_REPORTING_OPTION_READONLY,
+	ZBC_ZONE_REPORTING_OPTION_OFFLINE,
+	ZBC_ZONE_REPORTING_OPTION_NEED_RESET_WP = 0x10,
+	ZBC_ZONE_REPORTING_OPTION_NON_SEQWRITE,
+	ZBC_ZONE_REPORTING_OPTION_NON_WP = 0x3f,
+};
+
+#define ZBC_REPORT_ZONE_PARTIAL 0x80
+
 #endif /* _SCSI_PROTO_H_ */
diff --git a/include/scsi/scsi_transport_fc.h b/include/scsi/scsi_transport_fc.h
index bf66ea6bed2b..924c8e614b45 100644
--- a/include/scsi/scsi_transport_fc.h
+++ b/include/scsi/scsi_transport_fc.h
@@ -28,9 +28,11 @@
 #define SCSI_TRANSPORT_FC_H
 
 #include <linux/sched.h>
+#include <linux/bsg-lib.h>
 #include <asm/unaligned.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_netlink.h>
+#include <scsi/scsi_host.h>
 
 struct scsi_transport_template;
 
@@ -624,48 +626,6 @@ struct fc_host_attrs {
 #define fc_host_dev_loss_tmo(x) \
 	(((struct fc_host_attrs *)(x)->shost_data)->dev_loss_tmo)
 
-
-struct fc_bsg_buffer {
-	unsigned int payload_len;
-	int sg_cnt;
-	struct scatterlist *sg_list;
-};
-
-/* Values for fc_bsg_job->state_flags (bitflags) */
-#define FC_RQST_STATE_INPROGRESS	0
-#define FC_RQST_STATE_DONE		1
-
-struct fc_bsg_job {
-	struct Scsi_Host *shost;
-	struct fc_rport *rport;
-	struct device *dev;
-	struct request *req;
-	spinlock_t job_lock;
-	unsigned int state_flags;
-	unsigned int ref_cnt;
-	void (*job_done)(struct fc_bsg_job *);
-
-	struct fc_bsg_request *request;
-	struct fc_bsg_reply *reply;
-	unsigned int request_len;
-	unsigned int reply_len;
-	/*
-	 * On entry : reply_len indicates the buffer size allocated for
-	 * the reply.
-	 *
-	 * Upon completion : the message handler must set reply_len
-	 *  to indicates the size of the reply to be returned to the
-	 *  caller.
-	 */
-
-	/* DMA payloads for the request/response */
-	struct fc_bsg_buffer request_payload;
-	struct fc_bsg_buffer reply_payload;
-
-	void *dd_data;			/* Used for driver-specific storage */
-};
-
-
 /* The functions by which the transport class and the driver communicate */
 struct fc_function_template {
 	void    (*get_rport_dev_loss_tmo)(struct fc_rport *);
@@ -702,8 +662,8 @@ struct fc_function_template {
 	int     (* it_nexus_response)(struct Scsi_Host *, u64, int);
 
 	/* bsg support */
-	int	(*bsg_request)(struct fc_bsg_job *);
-	int	(*bsg_timeout)(struct fc_bsg_job *);
+	int	(*bsg_request)(struct bsg_job *);
+	int	(*bsg_timeout)(struct bsg_job *);
 
 	/* allocation lengths for host-specific data */
 	u32	 			dd_fcrport_size;
@@ -849,4 +809,18 @@ struct fc_vport *fc_vport_create(struct Scsi_Host *shost, int channel,
 int fc_vport_terminate(struct fc_vport *vport);
 int fc_block_scsi_eh(struct scsi_cmnd *cmnd);
 
+static inline struct Scsi_Host *fc_bsg_to_shost(struct bsg_job *job)
+{
+	if (scsi_is_host_device(job->dev))
+		return dev_to_shost(job->dev);
+	return rport_to_shost(dev_to_rport(job->dev));
+}
+
+static inline struct fc_rport *fc_bsg_to_rport(struct bsg_job *job)
+{
+	if (scsi_is_fc_rport(job->dev))
+		return dev_to_rport(job->dev);
+	return NULL;
+}
+
 #endif /* SCSI_TRANSPORT_FC_H */
diff --git a/include/soc/arc/aux.h b/include/soc/arc/aux.h
new file mode 100644
index 000000000000..8c3fb13e0452
--- /dev/null
+++ b/include/soc/arc/aux.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2016-2017 Synopsys, Inc. (www.synopsys.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __SOC_ARC_AUX_H__
+#define __SOC_ARC_AUX_H__
+
+#ifdef CONFIG_ARC
+
+#define read_aux_reg(r)		__builtin_arc_lr(r)
+
+/* gcc builtin sr needs reg param to be long immediate */
+#define write_aux_reg(r, v)	__builtin_arc_sr((unsigned int)(v), r)
+
+#else	/* !CONFIG_ARC */
+
+static inline int read_aux_reg(u32 r)
+{
+	return 0;
+}
+
+/*
+ * function helps elide unused variable warning
+ * see: http://lists.infradead.org/pipermail/linux-snps-arc/2016-November/001748.html
+ */
+static inline void write_aux_reg(u32 r, u32 v)
+{
+	;
+}
+
+#endif
+
+#define READ_BCR(reg, into)				\
+{							\
+	unsigned int tmp;				\
+	tmp = read_aux_reg(reg);			\
+	if (sizeof(tmp) == sizeof(into)) {		\
+		into = *((typeof(into) *)&tmp);		\
+	} else {					\
+		extern void bogus_undefined(void);	\
+		bogus_undefined();			\
+	}						\
+}
+
+#define WRITE_AUX(reg, into)				\
+{							\
+	unsigned int tmp;				\
+	if (sizeof(tmp) == sizeof(into)) {		\
+		tmp = (*(unsigned int *)&(into));	\
+		write_aux_reg(reg, tmp);		\
+	} else  {					\
+		extern void bogus_undefined(void);	\
+		bogus_undefined();			\
+	}						\
+}
+
+
+#endif
diff --git a/arch/arc/include/asm/mcip.h b/include/soc/arc/mcip.h
index c8fbe4114bad..6902c2a8bd23 100644
--- a/arch/arc/include/asm/mcip.h
+++ b/include/soc/arc/mcip.h
@@ -8,12 +8,10 @@
  * published by the Free Software Foundation.
  */
 
-#ifndef __ASM_MCIP_H
-#define __ASM_MCIP_H
+#ifndef __SOC_ARC_MCIP_H
+#define __SOC_ARC_MCIP_H
 
-#ifdef CONFIG_ISA_ARCV2
-
-#include <asm/arcregs.h>
+#include <soc/arc/aux.h>
 
 #define ARC_REG_MCIP_BCR	0x0d0
 #define ARC_REG_MCIP_CMD	0x600
@@ -103,5 +101,3 @@ static inline void __mcip_cmd_data(unsigned int cmd, unsigned int param,
 }
 
 #endif
-
-#endif
diff --git a/include/soc/arc/timers.h b/include/soc/arc/timers.h
new file mode 100644
index 000000000000..a20ed2fbc432
--- /dev/null
+++ b/include/soc/arc/timers.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2016-17 Synopsys, Inc. (www.synopsys.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __SOC_ARC_TIMERS_H
+#define __SOC_ARC_TIMERS_H
+
+#include <soc/arc/aux.h>
+
+/* Timer related Aux registers */
+#define ARC_REG_TIMER0_LIMIT	0x23	/* timer 0 limit */
+#define ARC_REG_TIMER0_CTRL	0x22	/* timer 0 control */
+#define ARC_REG_TIMER0_CNT	0x21	/* timer 0 count */
+#define ARC_REG_TIMER1_LIMIT	0x102	/* timer 1 limit */
+#define ARC_REG_TIMER1_CTRL	0x101	/* timer 1 control */
+#define ARC_REG_TIMER1_CNT	0x100	/* timer 1 count */
+
+/* CTRL reg bits */
+#define TIMER_CTRL_IE	        (1 << 0) /* Interrupt when Count reaches limit */
+#define TIMER_CTRL_NH	        (1 << 1) /* Count only when CPU NOT halted */
+
+#define ARC_TIMERN_MAX		0xFFFFFFFF
+
+#define ARC_REG_TIMERS_BCR	0x75
+
+struct bcr_timer {
+#ifdef CONFIG_CPU_BIG_ENDIAN
+	unsigned int pad2:15, rtsc:1, pad1:5, rtc:1, t1:1, t0:1, ver:8;
+#else
+	unsigned int ver:8, t0:1, t1:1, rtc:1, pad1:5, rtsc:1, pad2:15;
+#endif
+};
+
+#endif
diff --git a/include/soc/at91/atmel-secumod.h b/include/soc/at91/atmel-secumod.h
new file mode 100644
index 000000000000..22cd5d506926
--- /dev/null
+++ b/include/soc/at91/atmel-secumod.h
@@ -0,0 +1,19 @@
+/*
+ * Atmel Security Module register offsets and bit definitions.
+ *
+ * Copyright (C) 2016 Atmel
+ *
+ * Author: Alexandre Belloni <alexandre.belloni@free-electrons.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef _LINUX_SOC_AT91_ATMEL_SECUMOD_H
+#define _LINUX_SOC_AT91_ATMEL_SECUMOD_H
+
+#define AT91_SECUMOD_RAMRDY	0x14
+#define AT91_SECUMOD_RAMRDY_READY	BIT(0)
+
+#endif /* _LINUX_SOC_AT91_ATMEL_SECUMOD_H */
diff --git a/include/soc/bcm2835/raspberrypi-firmware.h b/include/soc/bcm2835/raspberrypi-firmware.h
index 3fb357193f09..cb979ad90401 100644
--- a/include/soc/bcm2835/raspberrypi-firmware.h
+++ b/include/soc/bcm2835/raspberrypi-firmware.h
@@ -109,14 +109,35 @@ enum rpi_firmware_property_tag {
 	RPI_FIRMWARE_FRAMEBUFFER_SET_OVERSCAN =               0x0004800a,
 	RPI_FIRMWARE_FRAMEBUFFER_SET_PALETTE =                0x0004800b,
 
+	RPI_FIRMWARE_VCHIQ_INIT =                             0x00048010,
+
 	RPI_FIRMWARE_GET_COMMAND_LINE =                       0x00050001,
 	RPI_FIRMWARE_GET_DMA_CHANNELS =                       0x00060001,
 };
 
+#if IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE)
 int rpi_firmware_property(struct rpi_firmware *fw,
 			  u32 tag, void *data, size_t len);
 int rpi_firmware_property_list(struct rpi_firmware *fw,
 			       void *data, size_t tag_size);
 struct rpi_firmware *rpi_firmware_get(struct device_node *firmware_node);
+#else
+static inline int rpi_firmware_property(struct rpi_firmware *fw, u32 tag,
+					void *data, size_t len)
+{
+	return 0;
+}
+
+static inline int rpi_firmware_property_list(struct rpi_firmware *fw,
+					     void *data, size_t tag_size)
+{
+	return 0;
+}
+
+static inline struct rpi_firmware *rpi_firmware_get(struct device_node *firmware_node)
+{
+	return NULL;
+}
+#endif
 
 #endif /* __SOC_RASPBERRY_FIRMWARE_H__ */
diff --git a/include/soc/fsl/qman.h b/include/soc/fsl/qman.h
index 37f3eb001a16..3d4df74a96de 100644
--- a/include/soc/fsl/qman.h
+++ b/include/soc/fsl/qman.h
@@ -244,11 +244,11 @@ static inline int qm_sg_entry_get_off(const struct qm_sg_entry *sg)
 struct qm_dqrr_entry {
 	u8 verb;
 	u8 stat;
-	u16 seqnum;	/* 15-bit */
+	__be16 seqnum;	/* 15-bit */
 	u8 tok;
 	u8 __reserved2[3];
-	u32 fqid;	/* 24-bit */
-	u32 contextB;
+	__be32 fqid;	/* 24-bit */
+	__be32 context_b;
 	struct qm_fd fd;
 	u8 __reserved4[32];
 } __packed;
@@ -262,6 +262,11 @@ struct qm_dqrr_entry {
 #define QM_DQRR_STAT_UNSCHEDULED	0x02	/* Unscheduled dequeue */
 #define QM_DQRR_STAT_DQCR_EXPIRED	0x01	/* VDQCR or PDQCR expired*/
 
+/* 'fqid' is a 24-bit field in every h/w descriptor */
+#define QM_FQID_MASK	GENMASK(23, 0)
+#define qm_fqid_set(p, v) ((p)->fqid = cpu_to_be32((v) & QM_FQID_MASK))
+#define qm_fqid_get(p)    (be32_to_cpu((p)->fqid) & QM_FQID_MASK)
+
 /* "ERN Message Response" */
 /* "FQ State Change Notification" */
 union qm_mr_entry {
@@ -272,12 +277,11 @@ union qm_mr_entry {
 	struct {
 		u8 verb;
 		u8 dca;
-		u16 seqnum;
+		__be16 seqnum;
 		u8 rc;		/* Rej Code: 8-bit */
-		u8 orp_hi;	/* ORP: 24-bit */
-		u16 orp_lo;
-		u32 fqid;	/* 24-bit */
-		u32 tag;
+		u8 __reserved[3];
+		__be32 fqid;	/* 24-bit */
+		__be32 tag;
 		struct qm_fd fd;
 		u8 __reserved1[32];
 	} __packed ern;
@@ -285,8 +289,8 @@ union qm_mr_entry {
 		u8 verb;
 		u8 fqs;		/* Frame Queue Status */
 		u8 __reserved1[6];
-		u32 fqid;	/* 24-bit */
-		u32 contextB;
+		__be32 fqid;	/* 24-bit */
+		__be32 context_b;
 		u8 __reserved2[48];
 	} __packed fq;		/* FQRN/FQRNI/FQRL/FQPN */
 };
@@ -405,13 +409,13 @@ static inline u64 qm_fqd_context_a_get64(const struct qm_fqd *fqd)
 
 static inline void qm_fqd_stashing_set64(struct qm_fqd *fqd, u64 addr)
 {
-	fqd->context_a.context_hi = upper_32_bits(addr);
-	fqd->context_a.context_lo = lower_32_bits(addr);
+	fqd->context_a.context_hi = cpu_to_be16(upper_32_bits(addr));
+	fqd->context_a.context_lo = cpu_to_be32(lower_32_bits(addr));
 }
 
 static inline void qm_fqd_context_a_set64(struct qm_fqd *fqd, u64 addr)
 {
-	fqd->context_a.hi = cpu_to_be16(upper_32_bits(addr));
+	fqd->context_a.hi = cpu_to_be32(upper_32_bits(addr));
 	fqd->context_a.lo = cpu_to_be32(lower_32_bits(addr));
 }
 
@@ -521,7 +525,7 @@ static inline int qm_fqd_get_wq(const struct qm_fqd *fqd)
  */
 struct qm_cgr_wr_parm {
 	/* MA[24-31], Mn[19-23], SA[12-18], Sn[6-11], Pn[0-5] */
-	u32 word;
+	__be32 word;
 };
 /*
  * This struct represents the 13-bit "CS_THRES" CGR field. In the corresponding
@@ -532,7 +536,7 @@ struct qm_cgr_wr_parm {
  */
 struct qm_cgr_cs_thres {
 	/* _res[13-15], TA[5-12], Tn[0-4] */
-	u16 word;
+	__be16 word;
 };
 /*
  * This identical structure of CGR fields is present in the "Init/Modify CGR"
@@ -549,10 +553,10 @@ struct __qm_mc_cgr {
 	u8 cscn_en;	/* boolean, use QM_CGR_EN */
 	union {
 		struct {
-			u16 cscn_targ_upd_ctrl; /* use QM_CSCN_TARG_UDP_ */
-			u16 cscn_targ_dcp_low;	/* CSCN_TARG_DCP low-16bits */
+			__be16 cscn_targ_upd_ctrl; /* use QM_CGR_TARG_UDP_* */
+			__be16 cscn_targ_dcp_low;
 		};
-		u32 cscn_targ;	/* use QM_CGR_TARG_* */
+		__be32 cscn_targ;	/* use QM_CGR_TARG_* */
 	};
 	u8 cstd_en;	/* boolean, use QM_CGR_EN */
 	u8 cs;		/* boolean, only used in query response */
@@ -568,7 +572,9 @@ struct __qm_mc_cgr {
 /* Convert CGR thresholds to/from "cs_thres" format */
 static inline u64 qm_cgr_cs_thres_get64(const struct qm_cgr_cs_thres *th)
 {
-	return ((th->word >> 5) & 0xff) << (th->word & 0x1f);
+	int thres = be16_to_cpu(th->word);
+
+	return ((thres >> 5) & 0xff) << (thres & 0x1f);
 }
 
 static inline int qm_cgr_cs_thres_set64(struct qm_cgr_cs_thres *th, u64 val,
@@ -584,23 +590,23 @@ static inline int qm_cgr_cs_thres_set64(struct qm_cgr_cs_thres *th, u64 val,
 		if (roundup && oddbit)
 			val++;
 	}
-	th->word = ((val & 0xff) << 5) | (e & 0x1f);
+	th->word = cpu_to_be16(((val & 0xff) << 5) | (e & 0x1f));
 	return 0;
 }
 
 /* "Initialize FQ" */
 struct qm_mcc_initfq {
 	u8 __reserved1[2];
-	u16 we_mask;	/* Write Enable Mask */
-	u32 fqid;	/* 24-bit */
-	u16 count;	/* Initialises 'count+1' FQDs */
+	__be16 we_mask;	/* Write Enable Mask */
+	__be32 fqid;	/* 24-bit */
+	__be16 count;	/* Initialises 'count+1' FQDs */
 	struct qm_fqd fqd; /* the FQD fields go here */
 	u8 __reserved2[30];
 } __packed;
 /* "Initialize/Modify CGR" */
 struct qm_mcc_initcgr {
 	u8 __reserve1[2];
-	u16 we_mask;	/* Write Enable Mask */
+	__be16 we_mask;	/* Write Enable Mask */
 	struct __qm_mc_cgr cgr;	/* CGR fields */
 	u8 __reserved2[2];
 	u8 cgid;
@@ -654,7 +660,7 @@ struct qman_cgr;
 /*
  * This enum, and the callback type that returns it, are used when handling
  * dequeued frames via DQRR. Note that for "null" callbacks registered with the
- * portal object (for handling dequeues that do not demux because contextB is
+ * portal object (for handling dequeues that do not demux because context_b is
  * NULL), the return value *MUST* be qman_cb_dqrr_consume.
  */
 enum qman_cb_dqrr_result {
@@ -859,11 +865,11 @@ void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools);
  * qman_fq" for more info). NO_MODIFY is only intended for enqueuing to
  * pre-existing frame-queues that aren't to be otherwise interfered with, it
  * prevents all other modifications to the frame queue. The TO_DCPORTAL flag
- * causes the driver to honour any contextB modifications requested in the
+ * causes the driver to honour any context_b modifications requested in the
  * qm_init_fq() API, as this indicates the frame queue will be consumed by a
  * direct-connect portal (PME, CAAM, or Fman). When frame queues are consumed by
- * software portals, the contextB field is controlled by the driver and can't be
- * modified by the caller.
+ * software portals, the context_b field is controlled by the driver and can't
+ * be modified by the caller.
  */
 int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq);
 
diff --git a/include/soc/nps/mtm.h b/include/soc/nps/mtm.h
new file mode 100644
index 000000000000..d2f5e7e3703e
--- /dev/null
+++ b/include/soc/nps/mtm.h
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef SOC_NPS_MTM_H
+#define SOC_NPS_MTM_H
+
+#define CTOP_INST_HWSCHD_OFF_R3                 0x3B6F00BF
+#define CTOP_INST_HWSCHD_RESTORE_R3             0x3E6F70C3
+
+static inline void hw_schd_save(unsigned int *flags)
+{
+	__asm__ __volatile__(
+	"       .word %1\n"
+	"       st r3,[%0]\n"
+	:
+	: "r"(flags), "i"(CTOP_INST_HWSCHD_OFF_R3)
+	: "r3", "memory");
+}
+
+static inline void hw_schd_restore(unsigned int flags)
+{
+	__asm__ __volatile__(
+	"       mov r3, %0\n"
+	"       .word %1\n"
+	:
+	: "r"(flags), "i"(CTOP_INST_HWSCHD_RESTORE_R3)
+	: "r3");
+}
+
+#endif /* SOC_NPS_MTM_H */
diff --git a/include/soc/tegra/bpmp-abi.h b/include/soc/tegra/bpmp-abi.h
new file mode 100644
index 000000000000..0aaef5960e29
--- /dev/null
+++ b/include/soc/tegra/bpmp-abi.h
@@ -0,0 +1,1601 @@
+/*
+ * Copyright (c) 2014-2016, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _ABI_BPMP_ABI_H_
+#define _ABI_BPMP_ABI_H_
+
+#ifdef LK
+#include <stdint.h>
+#endif
+
+#ifndef __ABI_PACKED
+#define __ABI_PACKED __attribute__((packed))
+#endif
+
+#ifdef NO_GCC_EXTENSIONS
+#define EMPTY char empty;
+#define EMPTY_ARRAY 1
+#else
+#define EMPTY
+#define EMPTY_ARRAY 0
+#endif
+
+#ifndef __UNION_ANON
+#define __UNION_ANON
+#endif
+/**
+ * @file
+ */
+
+
+/**
+ * @defgroup MRQ MRQ Messages
+ * @brief Messages sent to/from BPMP via IPC
+ * @{
+ *   @defgroup MRQ_Format Message Format
+ *   @defgroup MRQ_Codes Message Request (MRQ) Codes
+ *   @defgroup MRQ_Payloads Message Payloads
+ *   @defgroup Error_Codes Error Codes
+ * @}
+ */
+
+/**
+ * @addtogroup MRQ_Format Message Format
+ * @{
+ * The CPU requests the BPMP to perform a particular service by
+ * sending it an IVC frame containing a single MRQ message. An MRQ
+ * message consists of a @ref mrq_request followed by a payload whose
+ * format depends on mrq_request::mrq.
+ *
+ * The BPMP processes the data and replies with an IVC frame (on the
+ * same IVC channel) containing and MRQ response. An MRQ response
+ * consists of a @ref mrq_response followed by a payload whose format
+ * depends on the associated mrq_request::mrq.
+ *
+ * A well-defined subset of the MRQ messages that the CPU sends to the
+ * BPMP can lead to BPMP eventually sending an MRQ message to the
+ * CPU. For example, when the CPU uses an #MRQ_THERMAL message to set
+ * a thermal trip point, the BPMP may eventually send a single
+ * #MRQ_THERMAL message of its own to the CPU indicating that the trip
+ * point has been crossed.
+ * @}
+ */
+
+/**
+ * @ingroup MRQ_Format
+ * @brief header for an MRQ message
+ *
+ * Provides the MRQ number for the MRQ message: #mrq. The remainder of
+ * the MRQ message is a payload (immediately following the
+ * mrq_request) whose format depends on mrq.
+ *
+ * @todo document the flags
+ */
+struct mrq_request {
+	/** @brief MRQ number of the request */
+	uint32_t mrq;
+	/** @brief flags for the request */
+	uint32_t flags;
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Format
+ * @brief header for an MRQ response
+ *
+ *  Provides an error code for the associated MRQ message. The
+ *  remainder of the MRQ response is a payload (immediately following
+ *  the mrq_response) whose format depends on the associated
+ *  mrq_request::mrq
+ *
+ * @todo document the flags
+ */
+struct mrq_response {
+	/** @brief error code for the MRQ request itself */
+	int32_t err;
+	/** @brief flags for the response */
+	uint32_t flags;
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Format
+ * Minimum needed size for an IPC message buffer
+ */
+#define MSG_MIN_SZ	128
+/**
+ * @ingroup MRQ_Format
+ *  Minimum size guaranteed for data in an IPC message buffer
+ */
+#define MSG_DATA_MIN_SZ	120
+
+/**
+ * @ingroup MRQ_Codes
+ * @name Legal MRQ codes
+ * These are the legal values for mrq_request::mrq
+ * @{
+ */
+
+#define MRQ_PING		0
+#define MRQ_QUERY_TAG		1
+#define MRQ_MODULE_LOAD		4
+#define MRQ_MODULE_UNLOAD	5
+#define MRQ_TRACE_MODIFY	7
+#define MRQ_WRITE_TRACE		8
+#define MRQ_THREADED_PING	9
+#define MRQ_MODULE_MAIL		11
+#define MRQ_DEBUGFS		19
+#define MRQ_RESET		20
+#define MRQ_I2C			21
+#define MRQ_CLK			22
+#define MRQ_QUERY_ABI		23
+#define MRQ_PG_READ_STATE	25
+#define MRQ_PG_UPDATE_STATE	26
+#define MRQ_THERMAL		27
+#define MRQ_CPU_VHINT		28
+#define MRQ_ABI_RATCHET		29
+#define MRQ_EMC_DVFS_LATENCY	31
+#define MRQ_TRACE_ITER		64
+
+/** @} */
+
+/**
+ * @ingroup MRQ_Codes
+ * @brief Maximum MRQ code to be sent by CPU software to
+ * BPMP. Subject to change in future
+ */
+#define MAX_CPU_MRQ_ID		64
+
+/**
+ * @addtogroup MRQ_Payloads Message Payloads
+ * @{
+ *   @defgroup Ping
+ *   @defgroup Query_Tag Query Tag
+ *   @defgroup Module Loadable Modules
+ *   @defgroup Trace
+ *   @defgroup Debugfs
+ *   @defgroup Reset
+ *   @defgroup I2C
+ *   @defgroup Clocks
+ *   @defgroup ABI_info ABI Info
+ *   @defgroup MC_Flush MC Flush
+ *   @defgroup Powergating
+ *   @defgroup Thermal
+ *   @defgroup Vhint CPU Voltage hint
+ *   @defgroup MRQ_Deprecated Deprecated MRQ messages
+ *   @defgroup EMC
+ * @}
+ */
+
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_PING
+ * @brief A simple ping
+ *
+ * * Platforms: All
+ * * Initiators: Any
+ * * Targets: Any
+ * * Request Payload: @ref mrq_ping_request
+ * * Response Payload: @ref mrq_ping_response
+ *
+ * @ingroup MRQ_Codes
+ * @def MRQ_THREADED_PING
+ * @brief A deeper ping
+ *
+ * * Platforms: All
+ * * Initiators: Any
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_ping_request
+ * * Response Payload: @ref mrq_ping_response
+ *
+ * Behavior is equivalent to a simple #MRQ_PING except that BPMP
+ * responds from a thread context (providing a slightly more robust
+ * sign of life).
+ *
+ */
+
+/**
+ * @ingroup Ping
+ * @brief request with #MRQ_PING
+ *
+ * Used by the sender of an #MRQ_PING message to request a pong from
+ * recipient. The response from the recipient is computed based on
+ * #challenge.
+ */
+struct mrq_ping_request {
+/** @brief arbitrarily chosen value */
+	uint32_t challenge;
+} __ABI_PACKED;
+
+/**
+ * @ingroup Ping
+ * @brief response to #MRQ_PING
+ *
+ * Sent in response to an #MRQ_PING message. #reply should be the
+ * mrq_ping_request challenge left shifted by 1 with the carry-bit
+ * dropped.
+ *
+ */
+struct mrq_ping_response {
+	/** @brief response to the MRQ_PING challege */
+	uint32_t reply;
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_QUERY_TAG
+ * @brief Query BPMP firmware's tag (i.e. version information)
+ *
+ * * Platforms: All
+ * * Initiators: CCPLEX
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_query_tag_request
+ * * Response Payload: N/A
+ *
+ */
+
+/**
+ * @ingroup Query_Tag
+ * @brief request with #MRQ_QUERY_TAG
+ *
+ * Used by #MRQ_QUERY_TAG call to ask BPMP to fill in the memory
+ * pointed by #addr with BPMP firmware header.
+ *
+ * The sender is reponsible for ensuring that #addr is mapped in to
+ * the recipient's address map.
+ */
+struct mrq_query_tag_request {
+  /** @brief base address to store the firmware header */
+	uint32_t addr;
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_MODULE_LOAD
+ * @brief dynamically load a BPMP code module
+ *
+ * * Platforms: All
+ * * Initiators: CCPLEX
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_module_load_request
+ * * Response Payload: @ref mrq_module_load_response
+ *
+ * @note This MRQ is disabled on production systems
+ *
+ */
+
+/**
+ * @ingroup Module
+ * @brief request with #MRQ_MODULE_LOAD
+ *
+ * Used by #MRQ_MODULE_LOAD calls to ask the recipient to dynamically
+ * load the code located at #phys_addr and having size #size
+ * bytes. #phys_addr is treated as a void pointer.
+ *
+ * The recipient copies the code from #phys_addr to locally allocated
+ * memory prior to responding to this message.
+ *
+ * @todo document the module header format
+ *
+ * The sender is responsible for ensuring that the code is mapped in
+ * the recipient's address map.
+ *
+ */
+struct mrq_module_load_request {
+	/** @brief base address of the code to load. Treated as (void *) */
+	uint32_t phys_addr; /* (void *) */
+	/** @brief size in bytes of code to load */
+	uint32_t size;
+} __ABI_PACKED;
+
+/**
+ * @ingroup Module
+ * @brief response to #MRQ_MODULE_LOAD
+ *
+ * @todo document mrq_response::err
+ */
+struct mrq_module_load_response {
+	/** @brief handle to the loaded module */
+	uint32_t base;
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_MODULE_UNLOAD
+ * @brief unload a previously loaded code module
+ *
+ * * Platforms: All
+ * * Initiators: CCPLEX
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_module_unload_request
+ * * Response Payload: N/A
+ *
+ * @note This MRQ is disabled on production systems
+ */
+
+/**
+ * @ingroup Module
+ * @brief request with #MRQ_MODULE_UNLOAD
+ *
+ * Used by #MRQ_MODULE_UNLOAD calls to request that a previously loaded
+ * module be unloaded.
+ */
+struct mrq_module_unload_request {
+	/** @brief handle of the module to unload */
+	uint32_t base;
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_TRACE_MODIFY
+ * @brief modify the set of enabled trace events
+ *
+ * * Platforms: All
+ * * Initiators: CCPLEX
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_trace_modify_request
+ * * Response Payload: @ref mrq_trace_modify_response
+ *
+ * @note This MRQ is disabled on production systems
+ */
+
+/**
+ * @ingroup Trace
+ * @brief request with #MRQ_TRACE_MODIFY
+ *
+ * Used by %MRQ_TRACE_MODIFY calls to enable or disable specify trace
+ * events.  #set takes precedence for any bit set in both #set and
+ * #clr.
+ */
+struct mrq_trace_modify_request {
+	/** @brief bit mask of trace events to disable */
+	uint32_t clr;
+	/** @brief bit mask of trace events to enable */
+	uint32_t set;
+} __ABI_PACKED;
+
+/**
+ * @ingroup Trace
+ * @brief response to #MRQ_TRACE_MODIFY
+ *
+ * Sent in repsonse to an #MRQ_TRACE_MODIFY message. #mask reflects the
+ * state of which events are enabled after the recipient acted on the
+ * message.
+ *
+ */
+struct mrq_trace_modify_response {
+	/** @brief bit mask of trace event enable states */
+	uint32_t mask;
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_WRITE_TRACE
+ * @brief Write trace data to a buffer
+ *
+ * * Platforms: All
+ * * Initiators: CCPLEX
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_write_trace_request
+ * * Response Payload: @ref mrq_write_trace_response
+ *
+ * mrq_response::err depends on the @ref mrq_write_trace_request field
+ * values. err is -#BPMP_EINVAL if size is zero or area is NULL or
+ * area is in an illegal range. A positive value for err indicates the
+ * number of bytes written to area.
+ *
+ * @note This MRQ is disabled on production systems
+ */
+
+/**
+ * @ingroup Trace
+ * @brief request with #MRQ_WRITE_TRACE
+ *
+ * Used by MRQ_WRITE_TRACE calls to ask the recipient to copy trace
+ * data from the recipient's local buffer to the output buffer. #area
+ * is treated as a byte-aligned pointer in the recipient's address
+ * space.
+ *
+ * The sender is responsible for ensuring that the output
+ * buffer is mapped in the recipient's address map. The recipient is
+ * responsible for protecting its own code and data from accidental
+ * overwrites.
+ */
+struct mrq_write_trace_request {
+	/** @brief base address of output buffer */
+	uint32_t area;
+	/** @brief size in bytes of the output buffer */
+	uint32_t size;
+} __ABI_PACKED;
+
+/**
+ * @ingroup Trace
+ * @brief response to #MRQ_WRITE_TRACE
+ *
+ * Once this response is sent, the respondent will not access the
+ * output buffer further.
+ */
+struct mrq_write_trace_response {
+	/**
+	 * @brief flag whether more data remains in local buffer
+	 *
+	 * Value is 1 if the entire local trace buffer has been
+	 * drained to the outputbuffer. Value is 0 otherwise.
+	 */
+	uint32_t eof;
+} __ABI_PACKED;
+
+/** @private */
+struct mrq_threaded_ping_request {
+	uint32_t challenge;
+} __ABI_PACKED;
+
+/** @private */
+struct mrq_threaded_ping_response {
+	uint32_t reply;
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_MODULE_MAIL
+ * @brief send a message to a loadable module
+ *
+ * * Platforms: All
+ * * Initiators: Any
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_module_mail_request
+ * * Response Payload: @ref mrq_module_mail_response
+ *
+ * @note This MRQ is disabled on production systems
+ */
+
+/**
+ * @ingroup Module
+ * @brief request with #MRQ_MODULE_MAIL
+ */
+struct mrq_module_mail_request {
+	/** @brief handle to the previously loaded module */
+	uint32_t base;
+	/** @brief module-specific mail payload
+	 *
+	 * The length of data[ ] is unknown to the BPMP core firmware
+	 * but it is limited to the size of an IPC message.
+	 */
+	uint8_t data[EMPTY_ARRAY];
+} __ABI_PACKED;
+
+/**
+ * @ingroup Module
+ * @brief response to #MRQ_MODULE_MAIL
+ */
+struct mrq_module_mail_response {
+	/** @brief module-specific mail payload
+	 *
+	 * The length of data[ ] is unknown to the BPMP core firmware
+	 * but it is limited to the size of an IPC message.
+	 */
+	uint8_t data[EMPTY_ARRAY];
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_DEBUGFS
+ * @brief Interact with BPMP's debugfs file nodes
+ *
+ * * Platforms: T186
+ * * Initiators: Any
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_debugfs_request
+ * * Response Payload: @ref mrq_debugfs_response
+ */
+
+/**
+ * @addtogroup Debugfs
+ * @{
+ *
+ * The BPMP firmware implements a pseudo-filesystem called
+ * debugfs. Any driver within the firmware may register with debugfs
+ * to expose an arbitrary set of "files" in the filesystem. When
+ * software on the CPU writes to a debugfs file, debugfs passes the
+ * written data to a callback provided by the driver. When software on
+ * the CPU reads a debugfs file, debugfs queries the driver for the
+ * data to return to the CPU. The intention of the debugfs filesystem
+ * is to provide information useful for debugging the system at
+ * runtime.
+ *
+ * @note The files exposed via debugfs are not part of the
+ * BPMP firmware's ABI. debugfs files may be added or removed in any
+ * given version of the firmware. Typically the semantics of a debugfs
+ * file are consistent from version to version but even that is not
+ * guaranteed.
+ *
+ * @}
+ */
+/** @ingroup Debugfs */
+enum mrq_debugfs_commands {
+	CMD_DEBUGFS_READ = 1,
+	CMD_DEBUGFS_WRITE = 2,
+	CMD_DEBUGFS_DUMPDIR = 3,
+	CMD_DEBUGFS_MAX
+};
+
+/**
+ * @ingroup Debugfs
+ * @brief parameters for CMD_DEBUGFS_READ/WRITE command
+ */
+struct cmd_debugfs_fileop_request {
+	/** @brief physical address pointing at filename */
+	uint32_t fnameaddr;
+	/** @brief length in bytes of filename buffer */
+	uint32_t fnamelen;
+	/** @brief physical address pointing to data buffer */
+	uint32_t dataaddr;
+	/** @brief length in bytes of data buffer */
+	uint32_t datalen;
+} __ABI_PACKED;
+
+/**
+ * @ingroup Debugfs
+ * @brief parameters for CMD_DEBUGFS_READ/WRITE command
+ */
+struct cmd_debugfs_dumpdir_request {
+	/** @brief physical address pointing to data buffer */
+	uint32_t dataaddr;
+	/** @brief length in bytes of data buffer */
+	uint32_t datalen;
+} __ABI_PACKED;
+
+/**
+ * @ingroup Debugfs
+ * @brief response data for CMD_DEBUGFS_READ/WRITE command
+ */
+struct cmd_debugfs_fileop_response {
+	/** @brief always 0 */
+	uint32_t reserved;
+	/** @brief number of bytes read from or written to data buffer */
+	uint32_t nbytes;
+} __ABI_PACKED;
+
+/**
+ * @ingroup Debugfs
+ * @brief response data for CMD_DEBUGFS_DUMPDIR command
+ */
+struct cmd_debugfs_dumpdir_response {
+	/** @brief always 0 */
+	uint32_t reserved;
+	/** @brief number of bytes read from or written to data buffer */
+	uint32_t nbytes;
+} __ABI_PACKED;
+
+/**
+ * @ingroup Debugfs
+ * @brief request with #MRQ_DEBUGFS.
+ *
+ * The sender of an MRQ_DEBUGFS message uses #cmd to specify a debugfs
+ * command to execute. Legal commands are the values of @ref
+ * mrq_debugfs_commands. Each command requires a specific additional
+ * payload of data.
+ *
+ * |command            |payload|
+ * |-------------------|-------|
+ * |CMD_DEBUGFS_READ   |fop    |
+ * |CMD_DEBUGFS_WRITE  |fop    |
+ * |CMD_DEBUGFS_DUMPDIR|dumpdir|
+ */
+struct mrq_debugfs_request {
+	uint32_t cmd;
+	union {
+		struct cmd_debugfs_fileop_request fop;
+		struct cmd_debugfs_dumpdir_request dumpdir;
+	} __UNION_ANON;
+} __ABI_PACKED;
+
+/**
+ * @ingroup Debugfs
+ */
+struct mrq_debugfs_response {
+	/** @brief always 0 */
+	int32_t reserved;
+	union {
+		/** @brief response data for CMD_DEBUGFS_READ OR
+		 * CMD_DEBUGFS_WRITE command
+		 */
+		struct cmd_debugfs_fileop_response fop;
+		/** @brief response data for CMD_DEBUGFS_DUMPDIR command */
+		struct cmd_debugfs_dumpdir_response dumpdir;
+	} __UNION_ANON;
+} __ABI_PACKED;
+
+/**
+ * @addtogroup Debugfs
+ * @{
+ */
+#define DEBUGFS_S_ISDIR	(1 << 9)
+#define DEBUGFS_S_IRUSR	(1 << 8)
+#define DEBUGFS_S_IWUSR	(1 << 7)
+/** @} */
+
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_RESET
+ * @brief reset an IP block
+ *
+ * * Platforms: T186
+ * * Initiators: Any
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_reset_request
+ * * Response Payload: N/A
+ */
+
+/**
+ * @ingroup Reset
+ */
+enum mrq_reset_commands {
+	CMD_RESET_ASSERT = 1,
+	CMD_RESET_DEASSERT = 2,
+	CMD_RESET_MODULE = 3,
+	CMD_RESET_MAX, /* not part of ABI and subject to change */
+};
+
+/**
+ * @ingroup Reset
+ * @brief request with MRQ_RESET
+ *
+ * Used by the sender of an #MRQ_RESET message to request BPMP to
+ * assert or or deassert a given reset line.
+ */
+struct mrq_reset_request {
+	/** @brief reset action to perform (@enum mrq_reset_commands) */
+	uint32_t cmd;
+	/** @brief id of the reset to affected */
+	uint32_t reset_id;
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_I2C
+ * @brief issue an i2c transaction
+ *
+ * * Platforms: T186
+ * * Initiators: Any
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_i2c_request
+ * * Response Payload: @ref mrq_i2c_response
+ */
+
+/**
+ * @addtogroup I2C
+ * @{
+ */
+#define TEGRA_I2C_IPC_MAX_IN_BUF_SIZE	(MSG_DATA_MIN_SZ - 12)
+#define TEGRA_I2C_IPC_MAX_OUT_BUF_SIZE	(MSG_DATA_MIN_SZ - 4)
+/** @} */
+
+/**
+ * @ingroup I2C
+ * @name Serial I2C flags
+ * Use these flags with serial_i2c_request::flags
+ * @{
+ */
+#define SERIALI2C_TEN           0x0010
+#define SERIALI2C_RD            0x0001
+#define SERIALI2C_STOP          0x8000
+#define SERIALI2C_NOSTART       0x4000
+#define SERIALI2C_REV_DIR_ADDR  0x2000
+#define SERIALI2C_IGNORE_NAK    0x1000
+#define SERIALI2C_NO_RD_ACK     0x0800
+#define SERIALI2C_RECV_LEN      0x0400
+/** @} */
+/** @ingroup I2C */
+enum {
+	CMD_I2C_XFER = 1
+};
+
+/**
+ * @ingroup I2C
+ * @brief serializable i2c request
+ *
+ * Instances of this structure are packed (little-endian) into
+ * cmd_i2c_xfer_request::data_buf. Each instance represents a single
+ * transaction (or a portion of a transaction with repeated starts) on
+ * an i2c bus.
+ *
+ * Because these structures are packed, some instances are likely to
+ * be misaligned. Additionally because #data is variable length, it is
+ * not possible to iterate through a serialized list of these
+ * structures without inspecting #len in each instance.  It may be
+ * easier to serialize or deserialize cmd_i2c_xfer_request::data_buf
+ * manually rather than using this structure definition.
+*/
+struct serial_i2c_request {
+	/** @brief I2C slave address */
+	uint16_t addr;
+	/** @brief bitmask of SERIALI2C_ flags */
+	uint16_t flags;
+	/** @brief length of I2C transaction in bytes */
+	uint16_t len;
+	/** @brief for write transactions only, #len bytes of data */
+	uint8_t data[];
+} __ABI_PACKED;
+
+/**
+ * @ingroup I2C
+ * @brief trigger one or more i2c transactions
+ */
+struct cmd_i2c_xfer_request {
+	/** @brief valid bus number from mach-t186/i2c-t186.h*/
+	uint32_t bus_id;
+
+	/** @brief count of valid bytes in #data_buf*/
+	uint32_t data_size;
+
+	/** @brief serialized packed instances of @ref serial_i2c_request*/
+	uint8_t data_buf[TEGRA_I2C_IPC_MAX_IN_BUF_SIZE];
+} __ABI_PACKED;
+
+/**
+ * @ingroup I2C
+ * @brief container for data read from the i2c bus
+ *
+ * Processing an cmd_i2c_xfer_request::data_buf causes BPMP to execute
+ * zero or more I2C reads. The data read from the bus is serialized
+ * into #data_buf.
+ */
+struct cmd_i2c_xfer_response {
+	/** @brief count of valid bytes in #data_buf*/
+	uint32_t data_size;
+	/** @brief i2c read data */
+	uint8_t data_buf[TEGRA_I2C_IPC_MAX_OUT_BUF_SIZE];
+} __ABI_PACKED;
+
+/**
+ * @ingroup I2C
+ * @brief request with #MRQ_I2C
+ */
+struct mrq_i2c_request {
+	/** @brief always CMD_I2C_XFER (i.e. 1) */
+	uint32_t cmd;
+	/** @brief parameters of the transfer request */
+	struct cmd_i2c_xfer_request xfer;
+} __ABI_PACKED;
+
+/**
+ * @ingroup I2C
+ * @brief response to #MRQ_I2C
+ */
+struct mrq_i2c_response {
+	struct cmd_i2c_xfer_response xfer;
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_CLK
+ *
+ * * Platforms: T186
+ * * Initiators: Any
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_clk_request
+ * * Response Payload: @ref mrq_clk_response
+ * @addtogroup Clocks
+ * @{
+ */
+
+/**
+ * @name MRQ_CLK sub-commands
+ * @{
+ */
+enum {
+	CMD_CLK_GET_RATE = 1,
+	CMD_CLK_SET_RATE = 2,
+	CMD_CLK_ROUND_RATE = 3,
+	CMD_CLK_GET_PARENT = 4,
+	CMD_CLK_SET_PARENT = 5,
+	CMD_CLK_IS_ENABLED = 6,
+	CMD_CLK_ENABLE = 7,
+	CMD_CLK_DISABLE = 8,
+	CMD_CLK_GET_ALL_INFO = 14,
+	CMD_CLK_GET_MAX_CLK_ID = 15,
+	CMD_CLK_MAX,
+};
+/** @} */
+
+#define MRQ_CLK_NAME_MAXLEN	40
+#define MRQ_CLK_MAX_PARENTS	16
+
+/** @private */
+struct cmd_clk_get_rate_request {
+	EMPTY
+} __ABI_PACKED;
+
+struct cmd_clk_get_rate_response {
+	int64_t rate;
+} __ABI_PACKED;
+
+struct cmd_clk_set_rate_request {
+	int32_t unused;
+	int64_t rate;
+} __ABI_PACKED;
+
+struct cmd_clk_set_rate_response {
+	int64_t rate;
+} __ABI_PACKED;
+
+struct cmd_clk_round_rate_request {
+	int32_t unused;
+	int64_t rate;
+} __ABI_PACKED;
+
+struct cmd_clk_round_rate_response {
+	int64_t rate;
+} __ABI_PACKED;
+
+/** @private */
+struct cmd_clk_get_parent_request {
+	EMPTY
+} __ABI_PACKED;
+
+struct cmd_clk_get_parent_response {
+	uint32_t parent_id;
+} __ABI_PACKED;
+
+struct cmd_clk_set_parent_request {
+	uint32_t parent_id;
+} __ABI_PACKED;
+
+struct cmd_clk_set_parent_response {
+	uint32_t parent_id;
+} __ABI_PACKED;
+
+/** @private */
+struct cmd_clk_is_enabled_request {
+	EMPTY
+} __ABI_PACKED;
+
+struct cmd_clk_is_enabled_response {
+	int32_t state;
+} __ABI_PACKED;
+
+/** @private */
+struct cmd_clk_enable_request {
+	EMPTY
+} __ABI_PACKED;
+
+/** @private */
+struct cmd_clk_enable_response {
+	EMPTY
+} __ABI_PACKED;
+
+/** @private */
+struct cmd_clk_disable_request {
+	EMPTY
+} __ABI_PACKED;
+
+/** @private */
+struct cmd_clk_disable_response {
+	EMPTY
+} __ABI_PACKED;
+
+/** @private */
+struct cmd_clk_get_all_info_request {
+	EMPTY
+} __ABI_PACKED;
+
+struct cmd_clk_get_all_info_response {
+	uint32_t flags;
+	uint32_t parent;
+	uint32_t parents[MRQ_CLK_MAX_PARENTS];
+	uint8_t num_parents;
+	uint8_t name[MRQ_CLK_NAME_MAXLEN];
+} __ABI_PACKED;
+
+/** @private */
+struct cmd_clk_get_max_clk_id_request {
+	EMPTY
+} __ABI_PACKED;
+
+struct cmd_clk_get_max_clk_id_response {
+	uint32_t max_id;
+} __ABI_PACKED;
+/** @} */
+
+/**
+ * @ingroup Clocks
+ * @brief request with #MRQ_CLK
+ *
+ * Used by the sender of an #MRQ_CLK message to control clocks. The
+ * clk_request is split into several sub-commands. Some sub-commands
+ * require no additional data. Others have a sub-command specific
+ * payload
+ *
+ * |sub-command                 |payload                |
+ * |----------------------------|-----------------------|
+ * |CMD_CLK_GET_RATE            |-                      |
+ * |CMD_CLK_SET_RATE            |clk_set_rate           |
+ * |CMD_CLK_ROUND_RATE          |clk_round_rate         |
+ * |CMD_CLK_GET_PARENT          |-                      |
+ * |CMD_CLK_SET_PARENT          |clk_set_parent         |
+ * |CMD_CLK_IS_ENABLED          |-                      |
+ * |CMD_CLK_ENABLE              |-                      |
+ * |CMD_CLK_DISABLE             |-                      |
+ * |CMD_CLK_GET_ALL_INFO        |-                      |
+ * |CMD_CLK_GET_MAX_CLK_ID      |-                      |
+ *
+ */
+
+struct mrq_clk_request {
+	/** @brief sub-command and clock id concatenated to 32-bit word.
+	 * - bits[31..24] is the sub-cmd.
+	 * - bits[23..0] is the clock id
+	 */
+	uint32_t cmd_and_id;
+
+	union {
+		/** @private */
+		struct cmd_clk_get_rate_request clk_get_rate;
+		struct cmd_clk_set_rate_request clk_set_rate;
+		struct cmd_clk_round_rate_request clk_round_rate;
+		/** @private */
+		struct cmd_clk_get_parent_request clk_get_parent;
+		struct cmd_clk_set_parent_request clk_set_parent;
+		/** @private */
+		struct cmd_clk_enable_request clk_enable;
+		/** @private */
+		struct cmd_clk_disable_request clk_disable;
+		/** @private */
+		struct cmd_clk_is_enabled_request clk_is_enabled;
+		/** @private */
+		struct cmd_clk_get_all_info_request clk_get_all_info;
+		/** @private */
+		struct cmd_clk_get_max_clk_id_request clk_get_max_clk_id;
+	} __UNION_ANON;
+} __ABI_PACKED;
+
+/**
+ * @ingroup Clocks
+ * @brief response to MRQ_CLK
+ *
+ * Each sub-command supported by @ref mrq_clk_request may return
+ * sub-command-specific data. Some do and some do not as indicated in
+ * the following table
+ *
+ * |sub-command                 |payload                 |
+ * |----------------------------|------------------------|
+ * |CMD_CLK_GET_RATE            |clk_get_rate            |
+ * |CMD_CLK_SET_RATE            |clk_set_rate            |
+ * |CMD_CLK_ROUND_RATE          |clk_round_rate          |
+ * |CMD_CLK_GET_PARENT          |clk_get_parent          |
+ * |CMD_CLK_SET_PARENT          |clk_set_parent          |
+ * |CMD_CLK_IS_ENABLED          |clk_is_enabled          |
+ * |CMD_CLK_ENABLE              |-                       |
+ * |CMD_CLK_DISABLE             |-                       |
+ * |CMD_CLK_GET_ALL_INFO        |clk_get_all_info        |
+ * |CMD_CLK_GET_MAX_CLK_ID      |clk_get_max_id          |
+ *
+ */
+
+struct mrq_clk_response {
+	union {
+		struct cmd_clk_get_rate_response clk_get_rate;
+		struct cmd_clk_set_rate_response clk_set_rate;
+		struct cmd_clk_round_rate_response clk_round_rate;
+		struct cmd_clk_get_parent_response clk_get_parent;
+		struct cmd_clk_set_parent_response clk_set_parent;
+		/** @private */
+		struct cmd_clk_enable_response clk_enable;
+		/** @private */
+		struct cmd_clk_disable_response clk_disable;
+		struct cmd_clk_is_enabled_response clk_is_enabled;
+		struct cmd_clk_get_all_info_response clk_get_all_info;
+		struct cmd_clk_get_max_clk_id_response clk_get_max_clk_id;
+	} __UNION_ANON;
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_QUERY_ABI
+ * @brief check if an MRQ is implemented
+ *
+ * * Platforms: All
+ * * Initiators: Any
+ * * Targets: Any
+ * * Request Payload: @ref mrq_query_abi_request
+ * * Response Payload: @ref mrq_query_abi_response
+ */
+
+/**
+ * @ingroup ABI_info
+ * @brief request with MRQ_QUERY_ABI
+ *
+ * Used by #MRQ_QUERY_ABI call to check if MRQ code #mrq is supported
+ * by the recipient.
+ */
+struct mrq_query_abi_request {
+	/** @brief MRQ code to query */
+	uint32_t mrq;
+} __ABI_PACKED;
+
+/**
+ * @ingroup ABI_info
+ * @brief response to MRQ_QUERY_ABI
+ */
+struct mrq_query_abi_response {
+	/** @brief 0 if queried MRQ is supported. Else, -#BPMP_ENODEV */
+	int32_t status;
+} __ABI_PACKED;
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_PG_READ_STATE
+ * @brief read the power-gating state of a partition
+ *
+ * * Platforms: T186
+ * * Initiators: Any
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_pg_read_state_request
+ * * Response Payload: @ref mrq_pg_read_state_response
+ * @addtogroup Powergating
+ * @{
+ */
+
+/**
+ * @brief request with #MRQ_PG_READ_STATE
+ *
+ * Used by MRQ_PG_READ_STATE call to read the current state of a
+ * partition.
+ */
+struct mrq_pg_read_state_request {
+	/** @brief ID of partition */
+	uint32_t partition_id;
+} __ABI_PACKED;
+
+/**
+ * @brief response to MRQ_PG_READ_STATE
+ * @todo define possible errors.
+ */
+struct mrq_pg_read_state_response {
+	/** @brief read as don't care */
+	uint32_t sram_state;
+	/** @brief state of power partition
+	 * * 0 : off
+	 * * 1 : on
+	 */
+	uint32_t logic_state;
+} __ABI_PACKED;
+
+/** @} */
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_PG_UPDATE_STATE
+ * @brief modify the power-gating state of a partition
+ *
+ * * Platforms: T186
+ * * Initiators: Any
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_pg_update_state_request
+ * * Response Payload: N/A
+ * @addtogroup Powergating
+ * @{
+ */
+
+/**
+ * @brief request with mrq_pg_update_state_request
+ *
+ * Used by #MRQ_PG_UPDATE_STATE call to request BPMP to change the
+ * state of a power partition #partition_id.
+ */
+struct mrq_pg_update_state_request {
+	/** @brief ID of partition */
+	uint32_t partition_id;
+	/** @brief secondary control of power partition
+	 *  @details Ignored by many versions of the BPMP
+	 *  firmware. For maximum compatibility, set the value
+	 *  according to @logic_state
+	 * *  0x1: power ON partition (@ref logic_state == 0x3)
+	 * *  0x3: power OFF partition (@ref logic_state == 0x1)
+	 */
+	uint32_t sram_state;
+	/** @brief controls state of power partition, legal values are
+	 * *  0x1 : power OFF partition
+	 * *  0x3 : power ON partition
+	 */
+	uint32_t logic_state;
+	/** @brief change state of clocks of the power partition, legal values
+	 * *  0x0 : do not change clock state
+	 * *  0x1 : disable partition clocks (only applicable when
+	 *          @ref logic_state == 0x1)
+	 * *  0x3 : enable partition clocks (only applicable when
+	 *          @ref logic_state == 0x3)
+	 */
+	uint32_t clock_state;
+} __ABI_PACKED;
+/** @} */
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_THERMAL
+ * @brief interact with BPMP thermal framework
+ *
+ * * Platforms: T186
+ * * Initiators: Any
+ * * Targets: Any
+ * * Request Payload: TODO
+ * * Response Payload: TODO
+ *
+ * @addtogroup Thermal
+ *
+ * The BPMP firmware includes a thermal framework. Drivers within the
+ * bpmp firmware register with the framework to provide thermal
+ * zones. Each thermal zone corresponds to an entity whose temperature
+ * can be measured. The framework also has a notion of trip points. A
+ * trip point consists of a thermal zone id, a temperature, and a
+ * callback routine. The framework invokes the callback when the zone
+ * hits the indicated temperature. The BPMP firmware uses this thermal
+ * framework interally to implement various temperature-dependent
+ * functions.
+ *
+ * Software on the CPU can use #MRQ_THERMAL (with payload @ref
+ * mrq_thermal_host_to_bpmp_request) to interact with the BPMP thermal
+ * framework. The CPU must It can query the number of supported zones,
+ * query zone temperatures, and set trip points.
+ *
+ * When a trip point set by the CPU gets crossed, BPMP firmware issues
+ * an IPC to the CPU having mrq_request::mrq = #MRQ_THERMAL and a
+ * payload of @ref mrq_thermal_bpmp_to_host_request.
+ * @{
+ */
+enum mrq_thermal_host_to_bpmp_cmd {
+	/**
+	 * @brief Check whether the BPMP driver supports the specified
+	 * request type.
+	 *
+	 * Host needs to supply request parameters.
+	 *
+	 * mrq_response::err is 0 if the specified request is
+	 * supported and -#BPMP_ENODEV otherwise.
+	 */
+	CMD_THERMAL_QUERY_ABI = 0,
+
+	/**
+	 * @brief Get the current temperature of the specified zone.
+	 *
+	 * Host needs to supply request parameters.
+	 *
+	 * mrq_response::err is
+	 * *  0: Temperature query succeeded.
+	 * *  -#BPMP_EINVAL: Invalid request parameters.
+	 * *  -#BPMP_ENOENT: No driver registered for thermal zone..
+	 * *  -#BPMP_EFAULT: Problem reading temperature measurement.
+	 */
+	CMD_THERMAL_GET_TEMP = 1,
+
+	/**
+	 * @brief Enable or disable and set the lower and upper
+	 *   thermal limits for a thermal trip point. Each zone has
+	 *   one trip point.
+	 *
+	 * Host needs to supply request parameters. Once the
+	 * temperature hits a trip point, the BPMP will send a message
+	 * to the CPU having MRQ=MRQ_THERMAL and
+	 * type=CMD_THERMAL_HOST_TRIP_REACHED
+	 *
+	 * mrq_response::err is
+	 * *  0: Trip successfully set.
+	 * *  -#BPMP_EINVAL: Invalid request parameters.
+	 * *  -#BPMP_ENOENT: No driver registered for thermal zone.
+	 * *  -#BPMP_EFAULT: Problem setting trip point.
+	 */
+	CMD_THERMAL_SET_TRIP = 2,
+
+	/**
+	 * @brief Get the number of supported thermal zones.
+	 *
+	 * No request parameters required.
+	 *
+	 * mrq_response::err is always 0, indicating success.
+	 */
+	CMD_THERMAL_GET_NUM_ZONES = 3,
+
+	/** @brief: number of supported host-to-bpmp commands. May
+	 * increase in future
+	 */
+	CMD_THERMAL_HOST_TO_BPMP_NUM
+};
+
+enum mrq_thermal_bpmp_to_host_cmd {
+	/**
+	 * @brief Indication that the temperature for a zone has
+	 *   exceeded the range indicated in the thermal trip point
+	 *   for the zone.
+	 *
+	 * BPMP needs to supply request parameters. Host only needs to
+	 * acknowledge.
+	 */
+	CMD_THERMAL_HOST_TRIP_REACHED = 100,
+
+	/** @brief: number of supported bpmp-to-host commands. May
+	 * increase in future
+	 */
+	CMD_THERMAL_BPMP_TO_HOST_NUM
+};
+
+/*
+ * Host->BPMP request data for request type CMD_THERMAL_QUERY_ABI
+ *
+ * zone: Request type for which to check existence.
+ */
+struct cmd_thermal_query_abi_request {
+	uint32_t type;
+} __ABI_PACKED;
+
+/*
+ * Host->BPMP request data for request type CMD_THERMAL_GET_TEMP
+ *
+ * zone: Number of thermal zone.
+ */
+struct cmd_thermal_get_temp_request {
+	uint32_t zone;
+} __ABI_PACKED;
+
+/*
+ * BPMP->Host reply data for request CMD_THERMAL_GET_TEMP
+ *
+ * error: 0 if request succeeded.
+ *	-BPMP_EINVAL if request parameters were invalid.
+ *      -BPMP_ENOENT if no driver was registered for the specified thermal zone.
+ *      -BPMP_EFAULT for other thermal zone driver errors.
+ * temp: Current temperature in millicelsius.
+ */
+struct cmd_thermal_get_temp_response {
+	int32_t temp;
+} __ABI_PACKED;
+
+/*
+ * Host->BPMP request data for request type CMD_THERMAL_SET_TRIP
+ *
+ * zone: Number of thermal zone.
+ * low: Temperature of lower trip point in millicelsius
+ * high: Temperature of upper trip point in millicelsius
+ * enabled: 1 to enable trip point, 0 to disable trip point
+ */
+struct cmd_thermal_set_trip_request {
+	uint32_t zone;
+	int32_t low;
+	int32_t high;
+	uint32_t enabled;
+} __ABI_PACKED;
+
+/*
+ * BPMP->Host request data for request type CMD_THERMAL_HOST_TRIP_REACHED
+ *
+ * zone: Number of thermal zone where trip point was reached.
+ */
+struct cmd_thermal_host_trip_reached_request {
+	uint32_t zone;
+} __ABI_PACKED;
+
+/*
+ * BPMP->Host reply data for request type CMD_THERMAL_GET_NUM_ZONES
+ *
+ * num: Number of supported thermal zones. The thermal zones are indexed
+ *      starting from zero.
+ */
+struct cmd_thermal_get_num_zones_response {
+	uint32_t num;
+} __ABI_PACKED;
+
+/*
+ * Host->BPMP request data.
+ *
+ * Reply type is union mrq_thermal_bpmp_to_host_response.
+ *
+ * type: Type of request. Values listed in enum mrq_thermal_type.
+ * data: Request type specific parameters.
+ */
+struct mrq_thermal_host_to_bpmp_request {
+	uint32_t type;
+	union {
+		struct cmd_thermal_query_abi_request query_abi;
+		struct cmd_thermal_get_temp_request get_temp;
+		struct cmd_thermal_set_trip_request set_trip;
+	} __UNION_ANON;
+} __ABI_PACKED;
+
+/*
+ * BPMP->Host request data.
+ *
+ * type: Type of request. Values listed in enum mrq_thermal_type.
+ * data: Request type specific parameters.
+ */
+struct mrq_thermal_bpmp_to_host_request {
+	uint32_t type;
+	union {
+		struct cmd_thermal_host_trip_reached_request host_trip_reached;
+	} __UNION_ANON;
+} __ABI_PACKED;
+
+/*
+ * Data in reply to a Host->BPMP request.
+ */
+union mrq_thermal_bpmp_to_host_response {
+	struct cmd_thermal_get_temp_response get_temp;
+	struct cmd_thermal_get_num_zones_response get_num_zones;
+} __ABI_PACKED;
+/** @} */
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_CPU_VHINT
+ * @brief Query CPU voltage hint data
+ *
+ * * Platforms: T186
+ * * Initiators: CCPLEX
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_cpu_vhint_request
+ * * Response Payload: N/A
+ *
+ * @addtogroup Vhint CPU Voltage hint
+ * @{
+ */
+
+/**
+ * @brief request with #MRQ_CPU_VHINT
+ *
+ * Used by #MRQ_CPU_VHINT call by CCPLEX to retrieve voltage hint data
+ * from BPMP to memory space pointed by #addr. CCPLEX is responsible
+ * to allocate sizeof(cpu_vhint_data) sized block of memory and
+ * appropriately map it for BPMP before sending the request.
+ */
+struct mrq_cpu_vhint_request {
+	/** @brief IOVA address for the #cpu_vhint_data */
+	uint32_t addr; /* struct cpu_vhint_data * */
+	/** @brief ID of the cluster whose data is requested */
+	uint32_t cluster_id; /* enum cluster_id */
+} __ABI_PACKED;
+
+/**
+ * @brief description of the CPU v/f relation
+ *
+ * Used by #MRQ_CPU_VHINT call to carry data pointed by #addr of
+ * struct mrq_cpu_vhint_request
+ */
+struct cpu_vhint_data {
+	uint32_t ref_clk_hz; /**< reference frequency in Hz */
+	uint16_t pdiv; /**< post divider value */
+	uint16_t mdiv; /**< input divider value */
+	uint16_t ndiv_max; /**< fMAX expressed with max NDIV value */
+	/** table of ndiv values as a function of vINDEX (voltage index) */
+	uint16_t ndiv[80];
+	/** minimum allowed NDIV value */
+	uint16_t ndiv_min;
+	/** minimum allowed voltage hint value (as in vINDEX) */
+	uint16_t vfloor;
+	/** maximum allowed voltage hint value (as in vINDEX) */
+	uint16_t vceil;
+	/** post-multiplier for vindex value */
+	uint16_t vindex_mult;
+	/** post-divider for vindex value */
+	uint16_t vindex_div;
+	/** reserved for future use */
+	uint16_t reserved[328];
+} __ABI_PACKED;
+
+/** @} */
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_ABI_RATCHET
+ * @brief ABI ratchet value query
+ *
+ * * Platforms: T186
+ * * Initiators: Any
+ * * Targets: BPMP
+ * * Request Payload: @ref mrq_abi_ratchet_request
+ * * Response Payload: @ref mrq_abi_ratchet_response
+ * @addtogroup ABI_info
+ * @{
+ */
+
+/**
+ * @brief an ABI compatibility mechanism
+ *
+ * BPMP_ABI_RATCHET_VALUE may increase for various reasons in a future
+ * revision of this header file.
+ * 1. That future revision deprecates some MRQ
+ * 2. That future revision introduces a breaking change to an existing
+ *    MRQ or
+ * 3. A bug is discovered in an existing implementation of the BPMP-FW
+ *    (or possibly one of its clients) which warrants deprecating that
+ *    implementation.
+ */
+#define BPMP_ABI_RATCHET_VALUE 3
+
+/**
+ * @brief request with #MRQ_ABI_RATCHET.
+ *
+ * #ratchet should be #BPMP_ABI_RATCHET_VALUE from the ABI header
+ * against which the requester was compiled.
+ *
+ * If ratchet is less than BPMP's #BPMP_ABI_RATCHET_VALUE, BPMP may
+ * reply with mrq_response::err = -#BPMP_ERANGE to indicate that
+ * BPMP-FW cannot interoperate correctly with the requester. Requester
+ * should cease further communication with BPMP.
+ *
+ * Otherwise, err shall be 0.
+ */
+struct mrq_abi_ratchet_request {
+	/** @brief requester's ratchet value */
+	uint16_t ratchet;
+};
+
+/**
+ * @brief response to #MRQ_ABI_RATCHET
+ *
+ * #ratchet shall be #BPMP_ABI_RATCHET_VALUE from the ABI header
+ * against which BPMP firwmare was compiled.
+ *
+ * If #ratchet is less than the requester's #BPMP_ABI_RATCHET_VALUE,
+ * the requster must either interoperate with BPMP according to an ABI
+ * header version with BPMP_ABI_RATCHET_VALUE = ratchet or cease
+ * communication with BPMP.
+ *
+ * If mrq_response::err is 0 and ratchet is greater than or equal to the
+ * requester's BPMP_ABI_RATCHET_VALUE, the requester should continue
+ * normal operation.
+ */
+struct mrq_abi_ratchet_response {
+	/** @brief BPMP's ratchet value */
+	uint16_t ratchet;
+};
+/** @} */
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_EMC_DVFS_LATENCY
+ * @brief query frequency dependent EMC DVFS latency
+ *
+ * * Platforms: T186
+ * * Initiators: CCPLEX
+ * * Targets: BPMP
+ * * Request Payload: N/A
+ * * Response Payload: @ref mrq_emc_dvfs_latency_response
+ * @addtogroup EMC
+ * @{
+ */
+
+/**
+ * @brief used by @ref mrq_emc_dvfs_latency_response
+ */
+struct emc_dvfs_latency {
+	/** @brief EMC frequency in kHz */
+	uint32_t freq;
+	/** @brief EMC DVFS latency in nanoseconds */
+	uint32_t latency;
+} __ABI_PACKED;
+
+#define EMC_DVFS_LATENCY_MAX_SIZE	14
+/**
+ * @brief response to #MRQ_EMC_DVFS_LATENCY
+ */
+struct mrq_emc_dvfs_latency_response {
+	/** @brief the number valid entries in #pairs */
+	uint32_t num_pairs;
+	/** @brief EMC <frequency, latency> information */
+	struct emc_dvfs_latency pairs[EMC_DVFS_LATENCY_MAX_SIZE];
+} __ABI_PACKED;
+
+/** @} */
+
+/**
+ * @ingroup MRQ_Codes
+ * @def MRQ_TRACE_ITER
+ * @brief manage the trace iterator
+ *
+ * * Platforms: All
+ * * Initiators: CCPLEX
+ * * Targets: BPMP
+ * * Request Payload: N/A
+ * * Response Payload: @ref mrq_trace_iter_request
+ * @addtogroup Trace
+ * @{
+ */
+enum {
+	/** @brief (re)start the tracing now. Ignore older events */
+	TRACE_ITER_INIT = 0,
+	/** @brief clobber all events in the trace buffer */
+	TRACE_ITER_CLEAN = 1
+};
+
+/**
+ * @brief request with #MRQ_TRACE_ITER
+ */
+struct mrq_trace_iter_request {
+	/** @brief TRACE_ITER_INIT or TRACE_ITER_CLEAN */
+	uint32_t cmd;
+} __ABI_PACKED;
+
+/** @} */
+
+/*
+ *  4. Enumerations
+ */
+
+/*
+ *   4.1 CPU enumerations
+ *
+ * See <mach-t186/system-t186.h>
+ *
+ *   4.2 CPU Cluster enumerations
+ *
+ * See <mach-t186/system-t186.h>
+ *
+ *   4.3 System low power state enumerations
+ *
+ * See <mach-t186/system-t186.h>
+ */
+
+/*
+ *   4.4 Clock enumerations
+ *
+ * For clock enumerations, see <mach-t186/clk-t186.h>
+ */
+
+/*
+ *   4.5 Reset enumerations
+ *
+ * For reset enumerations, see <mach-t186/reset-t186.h>
+ */
+
+/*
+ *   4.6 Thermal sensor enumerations
+ *
+ * For thermal sensor enumerations, see <mach-t186/thermal-t186.h>
+ */
+
+/**
+ * @defgroup Error_Codes
+ * Negative values for mrq_response::err generally indicate some
+ * error. The ABI defines the following error codes. Negating these
+ * defines is an exercise left to the user.
+ * @{
+ */
+/** @brief No such file or directory */
+#define BPMP_ENOENT	2
+/** @brief No MRQ handler */
+#define BPMP_ENOHANDLER	3
+/** @brief I/O error */
+#define BPMP_EIO	5
+/** @brief Bad sub-MRQ command */
+#define BPMP_EBADCMD	6
+/** @brief Not enough memory */
+#define BPMP_ENOMEM	12
+/** @brief Permission denied */
+#define BPMP_EACCES	13
+/** @brief Bad address */
+#define BPMP_EFAULT	14
+/** @brief No such device */
+#define BPMP_ENODEV	19
+/** @brief Argument is a directory */
+#define BPMP_EISDIR	21
+/** @brief Invalid argument */
+#define BPMP_EINVAL	22
+/** @brief Timeout during operation */
+#define BPMP_ETIMEDOUT  23
+/** @brief Out of range */
+#define BPMP_ERANGE	34
+/** @} */
+/** @} */
+#endif
diff --git a/include/soc/tegra/bpmp.h b/include/soc/tegra/bpmp.h
new file mode 100644
index 000000000000..13dcd44e91bb
--- /dev/null
+++ b/include/soc/tegra/bpmp.h
@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __SOC_TEGRA_BPMP_H
+#define __SOC_TEGRA_BPMP_H
+
+#include <linux/mailbox_client.h>
+#include <linux/reset-controller.h>
+#include <linux/semaphore.h>
+#include <linux/types.h>
+
+#include <soc/tegra/bpmp-abi.h>
+
+struct tegra_bpmp_clk;
+
+struct tegra_bpmp_soc {
+	struct {
+		struct {
+			unsigned int offset;
+			unsigned int count;
+			unsigned int timeout;
+		} cpu_tx, thread, cpu_rx;
+	} channels;
+	unsigned int num_resets;
+};
+
+struct tegra_bpmp_mb_data {
+	u32 code;
+	u32 flags;
+	u8 data[MSG_DATA_MIN_SZ];
+} __packed;
+
+struct tegra_bpmp_channel {
+	struct tegra_bpmp *bpmp;
+	struct tegra_bpmp_mb_data *ib;
+	struct tegra_bpmp_mb_data *ob;
+	struct completion completion;
+	struct tegra_ivc *ivc;
+};
+
+typedef void (*tegra_bpmp_mrq_handler_t)(unsigned int mrq,
+					 struct tegra_bpmp_channel *channel,
+					 void *data);
+
+struct tegra_bpmp_mrq {
+	struct list_head list;
+	unsigned int mrq;
+	tegra_bpmp_mrq_handler_t handler;
+	void *data;
+};
+
+struct tegra_bpmp {
+	const struct tegra_bpmp_soc *soc;
+	struct device *dev;
+
+	struct {
+		struct gen_pool *pool;
+		dma_addr_t phys;
+		void *virt;
+	} tx, rx;
+
+	struct {
+		struct mbox_client client;
+		struct mbox_chan *channel;
+	} mbox;
+
+	struct tegra_bpmp_channel *channels;
+	unsigned int num_channels;
+
+	struct {
+		unsigned long *allocated;
+		unsigned long *busy;
+		unsigned int count;
+		struct semaphore lock;
+	} threaded;
+
+	struct list_head mrqs;
+	spinlock_t lock;
+
+	struct tegra_bpmp_clk **clocks;
+	unsigned int num_clocks;
+
+	struct reset_controller_dev rstc;
+};
+
+struct tegra_bpmp *tegra_bpmp_get(struct device *dev);
+void tegra_bpmp_put(struct tegra_bpmp *bpmp);
+
+struct tegra_bpmp_message {
+	unsigned int mrq;
+
+	struct {
+		const void *data;
+		size_t size;
+	} tx;
+
+	struct {
+		void *data;
+		size_t size;
+	} rx;
+};
+
+int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
+			       struct tegra_bpmp_message *msg);
+int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
+			struct tegra_bpmp_message *msg);
+
+int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
+			   tegra_bpmp_mrq_handler_t handler, void *data);
+void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
+			 void *data);
+
+#if IS_ENABLED(CONFIG_CLK_TEGRA_BPMP)
+int tegra_bpmp_init_clocks(struct tegra_bpmp *bpmp);
+#else
+static inline int tegra_bpmp_init_clocks(struct tegra_bpmp *bpmp)
+{
+	return 0;
+}
+#endif
+
+#if IS_ENABLED(CONFIG_RESET_TEGRA_BPMP)
+int tegra_bpmp_init_resets(struct tegra_bpmp *bpmp);
+#else
+static inline int tegra_bpmp_init_resets(struct tegra_bpmp *bpmp)
+{
+	return 0;
+}
+#endif
+
+#endif /* __SOC_TEGRA_BPMP_H */
diff --git a/include/soc/tegra/ivc.h b/include/soc/tegra/ivc.h
new file mode 100644
index 000000000000..b13cc43ad9d8
--- /dev/null
+++ b/include/soc/tegra/ivc.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __TEGRA_IVC_H
+
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/types.h>
+
+struct tegra_ivc_header;
+
+struct tegra_ivc {
+	struct device *peer;
+
+	struct {
+		struct tegra_ivc_header *channel;
+		unsigned int position;
+		dma_addr_t phys;
+	} rx, tx;
+
+	void (*notify)(struct tegra_ivc *ivc, void *data);
+	void *notify_data;
+
+	unsigned int num_frames;
+	size_t frame_size;
+};
+
+/**
+ * tegra_ivc_read_get_next_frame - Peek at the next frame to receive
+ * @ivc		pointer of the IVC channel
+ *
+ * Peek at the next frame to be received, without removing it from
+ * the queue.
+ *
+ * Returns a pointer to the frame, or an error encoded pointer.
+ */
+void *tegra_ivc_read_get_next_frame(struct tegra_ivc *ivc);
+
+/**
+ * tegra_ivc_read_advance - Advance the read queue
+ * @ivc		pointer of the IVC channel
+ *
+ * Advance the read queue
+ *
+ * Returns 0, or a negative error value if failed.
+ */
+int tegra_ivc_read_advance(struct tegra_ivc *ivc);
+
+/**
+ * tegra_ivc_write_get_next_frame - Poke at the next frame to transmit
+ * @ivc		pointer of the IVC channel
+ *
+ * Get access to the next frame.
+ *
+ * Returns a pointer to the frame, or an error encoded pointer.
+ */
+void *tegra_ivc_write_get_next_frame(struct tegra_ivc *ivc);
+
+/**
+ * tegra_ivc_write_advance - Advance the write queue
+ * @ivc		pointer of the IVC channel
+ *
+ * Advance the write queue
+ *
+ * Returns 0, or a negative error value if failed.
+ */
+int tegra_ivc_write_advance(struct tegra_ivc *ivc);
+
+/**
+ * tegra_ivc_notified - handle internal messages
+ * @ivc		pointer of the IVC channel
+ *
+ * This function must be called following every notification.
+ *
+ * Returns 0 if the channel is ready for communication, or -EAGAIN if a channel
+ * reset is in progress.
+ */
+int tegra_ivc_notified(struct tegra_ivc *ivc);
+
+/**
+ * tegra_ivc_reset - initiates a reset of the shared memory state
+ * @ivc		pointer of the IVC channel
+ *
+ * This function must be called after a channel is reserved before it is used
+ * for communication. The channel will be ready for use when a subsequent call
+ * to notify the remote of the channel reset.
+ */
+void tegra_ivc_reset(struct tegra_ivc *ivc);
+
+size_t tegra_ivc_align(size_t size);
+unsigned tegra_ivc_total_queue_size(unsigned queue_size);
+int tegra_ivc_init(struct tegra_ivc *ivc, struct device *peer, void *rx,
+		   dma_addr_t rx_phys, void *tx, dma_addr_t tx_phys,
+		   unsigned int num_frames, size_t frame_size,
+		   void (*notify)(struct tegra_ivc *ivc, void *data),
+		   void *data);
+void tegra_ivc_cleanup(struct tegra_ivc *ivc);
+
+#endif /* __TEGRA_IVC_H */
diff --git a/include/soc/tegra/pmc.h b/include/soc/tegra/pmc.h
index e9e53473a63e..2f271d1b9cea 100644
--- a/include/soc/tegra/pmc.h
+++ b/include/soc/tegra/pmc.h
@@ -76,37 +76,73 @@ int tegra_pmc_cpu_remove_clamping(unsigned int cpuid);
 
 #define TEGRA_POWERGATE_3D0	TEGRA_POWERGATE_3D
 
-#define TEGRA_IO_RAIL_CSIA	0
-#define TEGRA_IO_RAIL_CSIB	1
-#define TEGRA_IO_RAIL_DSI	2
-#define TEGRA_IO_RAIL_MIPI_BIAS	3
-#define TEGRA_IO_RAIL_PEX_BIAS	4
-#define TEGRA_IO_RAIL_PEX_CLK1	5
-#define TEGRA_IO_RAIL_PEX_CLK2	6
-#define TEGRA_IO_RAIL_USB0	9
-#define TEGRA_IO_RAIL_USB1	10
-#define TEGRA_IO_RAIL_USB2	11
-#define TEGRA_IO_RAIL_USB_BIAS	12
-#define TEGRA_IO_RAIL_NAND	13
-#define TEGRA_IO_RAIL_UART	14
-#define TEGRA_IO_RAIL_BB	15
-#define TEGRA_IO_RAIL_AUDIO	17
-#define TEGRA_IO_RAIL_HSIC	19
-#define TEGRA_IO_RAIL_COMP	22
-#define TEGRA_IO_RAIL_HDMI	28
-#define TEGRA_IO_RAIL_PEX_CNTRL	32
-#define TEGRA_IO_RAIL_SDMMC1	33
-#define TEGRA_IO_RAIL_SDMMC3	34
-#define TEGRA_IO_RAIL_SDMMC4	35
-#define TEGRA_IO_RAIL_CAM	36
-#define TEGRA_IO_RAIL_RES	37
-#define TEGRA_IO_RAIL_HV	38
-#define TEGRA_IO_RAIL_DSIB	39
-#define TEGRA_IO_RAIL_DSIC	40
-#define TEGRA_IO_RAIL_DSID	41
-#define TEGRA_IO_RAIL_CSIE	44
-#define TEGRA_IO_RAIL_LVDS	57
-#define TEGRA_IO_RAIL_SYS_DDC	58
+/**
+ * enum tegra_io_pad - I/O pad group identifier
+ *
+ * I/O pins on Tegra SoCs are grouped into so-called I/O pads. Each such pad
+ * can be used to control the common voltage signal level and power state of
+ * the pins of the given pad.
+ */
+enum tegra_io_pad {
+	TEGRA_IO_PAD_AUDIO,
+	TEGRA_IO_PAD_AUDIO_HV,
+	TEGRA_IO_PAD_BB,
+	TEGRA_IO_PAD_CAM,
+	TEGRA_IO_PAD_COMP,
+	TEGRA_IO_PAD_CSIA,
+	TEGRA_IO_PAD_CSIB,
+	TEGRA_IO_PAD_CSIC,
+	TEGRA_IO_PAD_CSID,
+	TEGRA_IO_PAD_CSIE,
+	TEGRA_IO_PAD_CSIF,
+	TEGRA_IO_PAD_DBG,
+	TEGRA_IO_PAD_DEBUG_NONAO,
+	TEGRA_IO_PAD_DMIC,
+	TEGRA_IO_PAD_DP,
+	TEGRA_IO_PAD_DSI,
+	TEGRA_IO_PAD_DSIB,
+	TEGRA_IO_PAD_DSIC,
+	TEGRA_IO_PAD_DSID,
+	TEGRA_IO_PAD_EMMC,
+	TEGRA_IO_PAD_EMMC2,
+	TEGRA_IO_PAD_GPIO,
+	TEGRA_IO_PAD_HDMI,
+	TEGRA_IO_PAD_HSIC,
+	TEGRA_IO_PAD_HV,
+	TEGRA_IO_PAD_LVDS,
+	TEGRA_IO_PAD_MIPI_BIAS,
+	TEGRA_IO_PAD_NAND,
+	TEGRA_IO_PAD_PEX_BIAS,
+	TEGRA_IO_PAD_PEX_CLK1,
+	TEGRA_IO_PAD_PEX_CLK2,
+	TEGRA_IO_PAD_PEX_CNTRL,
+	TEGRA_IO_PAD_SDMMC1,
+	TEGRA_IO_PAD_SDMMC3,
+	TEGRA_IO_PAD_SDMMC4,
+	TEGRA_IO_PAD_SPI,
+	TEGRA_IO_PAD_SPI_HV,
+	TEGRA_IO_PAD_SYS_DDC,
+	TEGRA_IO_PAD_UART,
+	TEGRA_IO_PAD_USB0,
+	TEGRA_IO_PAD_USB1,
+	TEGRA_IO_PAD_USB2,
+	TEGRA_IO_PAD_USB3,
+	TEGRA_IO_PAD_USB_BIAS,
+};
+
+/* deprecated, use TEGRA_IO_PAD_{HDMI,LVDS} instead */
+#define TEGRA_IO_RAIL_HDMI	TEGRA_IO_PAD_HDMI
+#define TEGRA_IO_RAIL_LVDS	TEGRA_IO_PAD_LVDS
+
+/**
+ * enum tegra_io_pad_voltage - voltage level of the I/O pad's source rail
+ * @TEGRA_IO_PAD_1800000UV: 1.8 V
+ * @TEGRA_IO_PAD_3300000UV: 3.3 V
+ */
+enum tegra_io_pad_voltage {
+	TEGRA_IO_PAD_1800000UV,
+	TEGRA_IO_PAD_3300000UV,
+};
 
 #ifdef CONFIG_ARCH_TEGRA
 int tegra_powergate_is_powered(unsigned int id);
@@ -118,6 +154,13 @@ int tegra_powergate_remove_clamping(unsigned int id);
 int tegra_powergate_sequence_power_up(unsigned int id, struct clk *clk,
 				      struct reset_control *rst);
 
+int tegra_io_pad_power_enable(enum tegra_io_pad id);
+int tegra_io_pad_power_disable(enum tegra_io_pad id);
+int tegra_io_pad_set_voltage(enum tegra_io_pad id,
+			     enum tegra_io_pad_voltage voltage);
+int tegra_io_pad_get_voltage(enum tegra_io_pad id);
+
+/* deprecated, use tegra_io_pad_power_{enable,disable}() instead */
 int tegra_io_rail_power_on(unsigned int id);
 int tegra_io_rail_power_off(unsigned int id);
 #else
@@ -148,6 +191,27 @@ static inline int tegra_powergate_sequence_power_up(unsigned int id,
 	return -ENOSYS;
 }
 
+static inline int tegra_io_pad_power_enable(enum tegra_io_pad id)
+{
+	return -ENOSYS;
+}
+
+static inline int tegra_io_pad_power_disable(enum tegra_io_pad id)
+{
+	return -ENOSYS;
+}
+
+static inline int tegra_io_pad_set_voltage(enum tegra_io_pad id,
+					   enum tegra_io_pad_voltage voltage)
+{
+	return -ENOSYS;
+}
+
+static inline int tegra_io_pad_get_voltage(enum tegra_io_pad id)
+{
+	return -ENOSYS;
+}
+
 static inline int tegra_io_rail_power_on(unsigned int id)
 {
 	return -ENOSYS;
diff --git a/include/sound/compress_driver.h b/include/sound/compress_driver.h
index cee8c00f3d3e..9924bc9cbc7c 100644
--- a/include/sound/compress_driver.h
+++ b/include/sound/compress_driver.h
@@ -155,6 +155,7 @@ struct snd_compr {
 	struct mutex lock;
 	int device;
 #ifdef CONFIG_SND_VERBOSE_PROCFS
+	/* private: */
 	char id[64];
 	struct snd_info_entry *proc_root;
 	struct snd_info_entry *proc_info_entry;
diff --git a/include/sound/core.h b/include/sound/core.h
index 31079ea5e484..f7d8c10c4c45 100644
--- a/include/sound/core.h
+++ b/include/sound/core.h
@@ -308,8 +308,8 @@ __printf(4, 5)
 void __snd_printk(unsigned int level, const char *file, int line,
 		  const char *format, ...);
 #else
-#define __snd_printk(level, file, line, format, args...) \
-	printk(format, ##args)
+#define __snd_printk(level, file, line, format, ...) \
+	printk(format, ##__VA_ARGS__)
 #endif
 
 /**
@@ -319,8 +319,8 @@ void __snd_printk(unsigned int level, const char *file, int line,
  * Works like printk() but prints the file and the line of the caller
  * when configured with CONFIG_SND_VERBOSE_PRINTK.
  */
-#define snd_printk(fmt, args...) \
-	__snd_printk(0, __FILE__, __LINE__, fmt, ##args)
+#define snd_printk(fmt, ...) \
+	__snd_printk(0, __FILE__, __LINE__, fmt, ##__VA_ARGS__)
 
 #ifdef CONFIG_SND_DEBUG
 /**
@@ -330,10 +330,10 @@ void __snd_printk(unsigned int level, const char *file, int line,
  * Works like snd_printk() for debugging purposes.
  * Ignored when CONFIG_SND_DEBUG is not set.
  */
-#define snd_printd(fmt, args...) \
-	__snd_printk(1, __FILE__, __LINE__, fmt, ##args)
-#define _snd_printd(level, fmt, args...) \
-	__snd_printk(level, __FILE__, __LINE__, fmt, ##args)
+#define snd_printd(fmt, ...) \
+	__snd_printk(1, __FILE__, __LINE__, fmt, ##__VA_ARGS__)
+#define _snd_printd(level, fmt, ...) \
+	__snd_printk(level, __FILE__, __LINE__, fmt, ##__VA_ARGS__)
 
 /**
  * snd_BUG - give a BUG warning message and stack trace
@@ -383,8 +383,8 @@ static inline bool snd_printd_ratelimit(void) { return false; }
  * Works like snd_printk() for debugging purposes.
  * Ignored when CONFIG_SND_DEBUG_VERBOSE is not set.
  */
-#define snd_printdd(format, args...) \
-	__snd_printk(2, __FILE__, __LINE__, format, ##args)
+#define snd_printdd(format, ...) \
+	__snd_printk(2, __FILE__, __LINE__, format, ##__VA_ARGS__)
 #else
 __printf(1, 2)
 static inline void snd_printdd(const char *format, ...) {}
diff --git a/include/sound/cs35l34.h b/include/sound/cs35l34.h
new file mode 100644
index 000000000000..9c927cffbe46
--- /dev/null
+++ b/include/sound/cs35l34.h
@@ -0,0 +1,35 @@
+/*
+ * linux/sound/cs35l34.h -- Platform data for CS35l34
+ *
+ * Copyright (c) 2016 Cirrus Logic Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __CS35L34_H
+#define __CS35L34_H
+
+struct cs35l34_platform_data {
+	/* Set AIF to half drive strength */
+	bool aif_half_drv;
+	/* Digital Soft Ramp Disable */
+	bool digsft_disable;
+	/* Amplifier Invert */
+	bool amp_inv;
+	/* Peak current (mA) */
+	unsigned int boost_peak;
+	/* Boost inductor value (nH) */
+	unsigned int boost_ind;
+	/* Boost Controller Voltage Setting (mV) */
+	unsigned int boost_vtge;
+	/* Gain Change Zero Cross */
+	bool gain_zc_disable;
+	/* SDIN Left/Right Selection */
+	unsigned int i2s_sdinloc;
+	/* TDM Rising Edge */
+	bool tdm_rising_edge;
+};
+
+#endif /* __CS35L34_H */
diff --git a/include/sound/dmaengine_pcm.h b/include/sound/dmaengine_pcm.h
index 67be2445941a..1c8f9e1ef2a5 100644
--- a/include/sound/dmaengine_pcm.h
+++ b/include/sound/dmaengine_pcm.h
@@ -71,7 +71,6 @@ struct dma_chan *snd_dmaengine_pcm_get_chan(struct snd_pcm_substream *substream)
  * @slave_id: Slave requester id for the DMA channel.
  * @filter_data: Custom DMA channel filter data, this will usually be used when
  * requesting the DMA channel.
- * @chan_name: Custom channel name to use when requesting DMA channel.
  * @fifo_size: FIFO size of the DAI controller in bytes
  * @flags: PCM_DAI flags, only SND_DMAENGINE_PCM_DAI_FLAG_PACK for now
  */
@@ -81,7 +80,6 @@ struct snd_dmaengine_dai_dma_data {
 	u32 maxburst;
 	unsigned int slave_id;
 	void *filter_data;
-	const char *chan_name;
 	unsigned int fifo_size;
 	unsigned int flags;
 };
@@ -107,10 +105,6 @@ void snd_dmaengine_pcm_set_config_from_dai_data(
  * playback.
  */
 #define SND_DMAENGINE_PCM_FLAG_HALF_DUPLEX BIT(3)
-/*
- * The PCM streams have custom channel names specified.
- */
-#define SND_DMAENGINE_PCM_FLAG_CUSTOM_CHANNEL_NAME BIT(4)
 
 /**
  * struct snd_dmaengine_pcm_config - Configuration data for dmaengine based PCM
diff --git a/include/sound/emu10k1.h b/include/sound/emu10k1.h
index 5bd134651f5e..4f42affe777c 100644
--- a/include/sound/emu10k1.h
+++ b/include/sound/emu10k1.h
@@ -1688,7 +1688,8 @@ struct snd_emu1010 {
 	unsigned int internal_clock; /* 44100 or 48000 */
 	unsigned int optical_in; /* 0:SPDIF, 1:ADAT */
 	unsigned int optical_out; /* 0:SPDIF, 1:ADAT */
-	struct task_struct *firmware_thread;
+	struct delayed_work firmware_work;
+	u32 last_reg;
 };
 
 struct snd_emu10k1 {
diff --git a/include/sound/hda_i915.h b/include/sound/hda_i915.h
index 796cabf6be5e..5ab972e116ec 100644
--- a/include/sound/hda_i915.h
+++ b/include/sound/hda_i915.h
@@ -10,8 +10,9 @@
 int snd_hdac_set_codec_wakeup(struct hdac_bus *bus, bool enable);
 int snd_hdac_display_power(struct hdac_bus *bus, bool enable);
 void snd_hdac_i915_set_bclk(struct hdac_bus *bus);
-int snd_hdac_sync_audio_rate(struct hdac_device *codec, hda_nid_t nid, int rate);
-int snd_hdac_acomp_get_eld(struct hdac_device *codec, hda_nid_t nid,
+int snd_hdac_sync_audio_rate(struct hdac_device *codec, hda_nid_t nid,
+			     int dev_id, int rate);
+int snd_hdac_acomp_get_eld(struct hdac_device *codec, hda_nid_t nid, int dev_id,
 			   bool *audio_enabled, char *buffer, int max_bytes);
 int snd_hdac_i915_init(struct hdac_bus *bus);
 int snd_hdac_i915_exit(struct hdac_bus *bus);
@@ -29,13 +30,13 @@ static inline void snd_hdac_i915_set_bclk(struct hdac_bus *bus)
 {
 }
 static inline int snd_hdac_sync_audio_rate(struct hdac_device *codec,
-					   hda_nid_t nid, int rate)
+					   hda_nid_t nid, int dev_id, int rate)
 {
 	return 0;
 }
 static inline int snd_hdac_acomp_get_eld(struct hdac_device *codec, hda_nid_t nid,
-					 bool *audio_enabled, char *buffer,
-					 int max_bytes)
+					 int dev_id, bool *audio_enabled,
+					 char *buffer, int max_bytes)
 {
 	return -ENODEV;
 }
diff --git a/include/sound/rt5514.h b/include/sound/rt5514.h
new file mode 100644
index 000000000000..ef18494769ee
--- /dev/null
+++ b/include/sound/rt5514.h
@@ -0,0 +1,20 @@
+/*
+ * linux/sound/rt5514.h -- Platform data for RT5514
+ *
+ * Copyright 2016 Realtek Semiconductor Corp.
+ * Author: Oder Chiou <oder_chiou@realtek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __LINUX_SND_RT5514_H
+#define __LINUX_SND_RT5514_H
+
+struct rt5514_platform_data {
+	unsigned int dmic_init_delay;
+};
+
+#endif
+
diff --git a/include/sound/rt5665.h b/include/sound/rt5665.h
new file mode 100755
index 000000000000..963229e71dc7
--- /dev/null
+++ b/include/sound/rt5665.h
@@ -0,0 +1,47 @@
+/*
+ * linux/sound/rt5665.h -- Platform data for RT5665
+ *
+ * Copyright 2016 Realtek Microelectronics
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __LINUX_SND_RT5665_H
+#define __LINUX_SND_RT5665_H
+
+enum rt5665_dmic1_data_pin {
+	RT5665_DMIC1_NULL,
+	RT5665_DMIC1_DATA_GPIO4,
+	RT5665_DMIC1_DATA_IN2N,
+};
+
+enum rt5665_dmic2_data_pin {
+	RT5665_DMIC2_NULL,
+	RT5665_DMIC2_DATA_GPIO5,
+	RT5665_DMIC2_DATA_IN2P,
+};
+
+enum rt5665_jd_src {
+	RT5665_JD_NULL,
+	RT5665_JD1,
+};
+
+struct rt5665_platform_data {
+	bool in1_diff;
+	bool in2_diff;
+	bool in3_diff;
+	bool in4_diff;
+
+	int ldo1_en; /* GPIO for LDO1_EN */
+
+	enum rt5665_dmic1_data_pin dmic1_data_pin;
+	enum rt5665_dmic2_data_pin dmic2_data_pin;
+	enum rt5665_jd_src jd_src;
+
+	unsigned int sar_hs_type;
+};
+
+#endif
+
diff --git a/include/sound/simple_card_utils.h b/include/sound/simple_card_utils.h
index fd6412551145..64e90ca9ad32 100644
--- a/include/sound/simple_card_utils.h
+++ b/include/sound/simple_card_utils.h
@@ -1,5 +1,5 @@
 /*
- * simple_card_core.h
+ * simple_card_utils.h
  *
  * Copyright (c) 2016 Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
  *
@@ -7,8 +7,8 @@
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
-#ifndef __SIMPLE_CARD_CORE_H
-#define __SIMPLE_CARD_CORE_H
+#ifndef __SIMPLE_CARD_UTILS_H
+#define __SIMPLE_CARD_UTILS_H
 
 #include <sound/soc.h>
 
@@ -68,4 +68,4 @@ void asoc_simple_card_canonicalize_cpu(struct snd_soc_dai_link *dai_link,
 
 int asoc_simple_card_clean_reference(struct snd_soc_card *card);
 
-#endif /* __SIMPLE_CARD_CORE_H */
+#endif /* __SIMPLE_CARD_UTILS_H */
diff --git a/include/sound/soc-dai.h b/include/sound/soc-dai.h
index 964b7de1a1cc..200e1f04c166 100644
--- a/include/sound/soc-dai.h
+++ b/include/sound/soc-dai.h
@@ -15,6 +15,7 @@
 
 
 #include <linux/list.h>
+#include <sound/asoc.h>
 
 struct snd_pcm_substream;
 struct snd_soc_dapm_widget;
@@ -26,13 +27,13 @@ struct snd_compr_stream;
  * Describes the physical PCM data formating and clocking. Add new formats
  * to the end.
  */
-#define SND_SOC_DAIFMT_I2S		1 /* I2S mode */
-#define SND_SOC_DAIFMT_RIGHT_J		2 /* Right Justified mode */
-#define SND_SOC_DAIFMT_LEFT_J		3 /* Left Justified mode */
-#define SND_SOC_DAIFMT_DSP_A		4 /* L data MSB after FRM LRC */
-#define SND_SOC_DAIFMT_DSP_B		5 /* L data MSB during FRM LRC */
-#define SND_SOC_DAIFMT_AC97		6 /* AC97 */
-#define SND_SOC_DAIFMT_PDM		7 /* Pulse density modulation */
+#define SND_SOC_DAIFMT_I2S		SND_SOC_DAI_FORMAT_I2S
+#define SND_SOC_DAIFMT_RIGHT_J		SND_SOC_DAI_FORMAT_RIGHT_J
+#define SND_SOC_DAIFMT_LEFT_J		SND_SOC_DAI_FORMAT_LEFT_J
+#define SND_SOC_DAIFMT_DSP_A		SND_SOC_DAI_FORMAT_DSP_A
+#define SND_SOC_DAIFMT_DSP_B		SND_SOC_DAI_FORMAT_DSP_B
+#define SND_SOC_DAIFMT_AC97		SND_SOC_DAI_FORMAT_AC97
+#define SND_SOC_DAIFMT_PDM		SND_SOC_DAI_FORMAT_PDM
 
 /* left and right justified also known as MSB and LSB respectively */
 #define SND_SOC_DAIFMT_MSB		SND_SOC_DAIFMT_LEFT_J
@@ -207,6 +208,30 @@ struct snd_soc_dai_ops {
 		struct snd_soc_dai *);
 };
 
+struct snd_soc_cdai_ops {
+	/*
+	 * for compress ops
+	 */
+	int (*startup)(struct snd_compr_stream *,
+			struct snd_soc_dai *);
+	int (*shutdown)(struct snd_compr_stream *,
+			struct snd_soc_dai *);
+	int (*set_params)(struct snd_compr_stream *,
+			struct snd_compr_params *, struct snd_soc_dai *);
+	int (*get_params)(struct snd_compr_stream *,
+			struct snd_codec *, struct snd_soc_dai *);
+	int (*set_metadata)(struct snd_compr_stream *,
+			struct snd_compr_metadata *, struct snd_soc_dai *);
+	int (*get_metadata)(struct snd_compr_stream *,
+			struct snd_compr_metadata *, struct snd_soc_dai *);
+	int (*trigger)(struct snd_compr_stream *, int,
+			struct snd_soc_dai *);
+	int (*pointer)(struct snd_compr_stream *,
+			struct snd_compr_tstamp *, struct snd_soc_dai *);
+	int (*ack)(struct snd_compr_stream *, size_t,
+			struct snd_soc_dai *);
+};
+
 /*
  * Digital Audio Interface Driver.
  *
@@ -236,6 +261,7 @@ struct snd_soc_dai_driver {
 
 	/* ops */
 	const struct snd_soc_dai_ops *ops;
+	const struct snd_soc_cdai_ops *cops;
 
 	/* DAI capabilities */
 	struct snd_soc_pcm_stream capture;
@@ -268,8 +294,9 @@ struct snd_soc_dai {
 	unsigned int symmetric_rates:1;
 	unsigned int symmetric_channels:1;
 	unsigned int symmetric_samplebits:1;
+	unsigned int probed:1;
+
 	unsigned int active;
-	unsigned char probed:1;
 
 	struct snd_soc_dapm_widget *playback_widget;
 	struct snd_soc_dapm_widget *capture_widget;
diff --git a/include/sound/soc-dapm.h b/include/sound/soc-dapm.h
index f60d755f7ac6..a466f4bdc835 100644
--- a/include/sound/soc-dapm.h
+++ b/include/sound/soc-dapm.h
@@ -272,6 +272,16 @@ struct device;
 
 
 /* dapm kcontrol types */
+#define SOC_DAPM_DOUBLE(xname, reg, lshift, rshift, max, invert) \
+{	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \
+	.info = snd_soc_info_volsw, \
+	.get = snd_soc_dapm_get_volsw, .put = snd_soc_dapm_put_volsw, \
+	.private_value = SOC_DOUBLE_VALUE(reg, lshift, rshift, max, invert, 0) }
+#define SOC_DAPM_DOUBLE_R(xname, lreg, rreg, shift, max, invert) \
+{	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \
+	.info = snd_soc_info_volsw, \
+	.get = snd_soc_dapm_get_volsw, .put = snd_soc_dapm_put_volsw, \
+	.private_value = SOC_DOUBLE_R_VALUE(lreg, rreg, shift, max, invert) }
 #define SOC_DAPM_SINGLE(xname, reg, shift, max, invert) \
 {	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \
 	.info = snd_soc_info_volsw, \
@@ -615,6 +625,10 @@ struct snd_soc_dapm_update {
 	int reg;
 	int mask;
 	int val;
+	int reg2;
+	int mask2;
+	int val2;
+	bool has_second_set;
 };
 
 struct snd_soc_dapm_wcache {
diff --git a/include/sound/soc-topology.h b/include/sound/soc-topology.h
index b897b9d63161..f9cc7b9271ac 100644
--- a/include/sound/soc-topology.h
+++ b/include/sound/soc-topology.h
@@ -53,7 +53,7 @@ struct snd_soc_dobj_control {
 
 /* dynamic widget object */
 struct snd_soc_dobj_widget {
-	unsigned int kcontrol_enum:1;	/* this widget is an enum kcontrol */
+	unsigned int kcontrol_type;	/* kcontrol type: mixer, enum, bytes */
 };
 
 /* generic dynamic object - all dynamic objects belong to this struct */
diff --git a/include/sound/soc.h b/include/sound/soc.h
index 4f1c784e44f6..2b502f6cc6d0 100644
--- a/include/sound/soc.h
+++ b/include/sound/soc.h
@@ -782,6 +782,8 @@ struct snd_soc_component_driver {
 
 	int (*probe)(struct snd_soc_component *);
 	void (*remove)(struct snd_soc_component *);
+	int (*suspend)(struct snd_soc_component *);
+	int (*resume)(struct snd_soc_component *);
 
 	/* DT */
 	int (*of_xlate_dai_name)(struct snd_soc_component *component,
@@ -807,9 +809,11 @@ struct snd_soc_component {
 
 	unsigned int ignore_pmdown_time:1; /* pmdown_time is ignored at stop */
 	unsigned int registered_as_component:1;
+	unsigned int auxiliary:1; /* for auxiliary component of the card */
+	unsigned int suspended:1; /* is in suspend PM state */
 
 	struct list_head list;
-	struct list_head list_aux; /* for auxiliary component of the card */
+	struct list_head card_list;
 
 	struct snd_soc_dai_driver *dai_drv;
 	int num_dai;
@@ -852,6 +856,8 @@ struct snd_soc_component {
 
 	int (*probe)(struct snd_soc_component *);
 	void (*remove)(struct snd_soc_component *);
+	int (*suspend)(struct snd_soc_component *);
+	int (*resume)(struct snd_soc_component *);
 
 	/* machine specific init */
 	int (*init)(struct snd_soc_component *component);
@@ -868,11 +874,9 @@ struct snd_soc_codec {
 	const struct snd_soc_codec_driver *driver;
 
 	struct list_head list;
-	struct list_head card_list;
 
 	/* runtime */
 	unsigned int cache_bypass:1; /* Suppress access to the cache */
-	unsigned int suspended:1; /* Codec is in suspend PM state */
 	unsigned int cache_init:1; /* codec cache has been initialized */
 
 	/* codec IO */
@@ -1025,13 +1029,13 @@ struct snd_soc_dai_link {
 	const struct snd_soc_ops *ops;
 	const struct snd_soc_compr_ops *compr_ops;
 
-	/* For unidirectional dai links */
-	bool playback_only;
-	bool capture_only;
-
 	/* Mark this pcm with non atomic ops */
 	bool nonatomic;
 
+	/* For unidirectional dai links */
+	unsigned int playback_only:1;
+	unsigned int capture_only:1;
+
 	/* Keep DAI active over suspend */
 	unsigned int ignore_suspend:1;
 
@@ -1148,7 +1152,6 @@ struct snd_soc_card {
 	 */
 	struct snd_soc_aux_dev *aux_dev;
 	int num_aux_devs;
-	struct list_head aux_comp_list;
 
 	const struct snd_kcontrol_new *controls;
 	int num_controls;
@@ -1170,7 +1173,7 @@ struct snd_soc_card {
 	struct work_struct deferred_resume_work;
 
 	/* lists of probed devices belonging to this card */
-	struct list_head codec_dev_list;
+	struct list_head component_dev_list;
 
 	struct list_head widgets;
 	struct list_head paths;
@@ -1203,14 +1206,11 @@ struct snd_soc_pcm_runtime {
 	enum snd_soc_pcm_subclass pcm_subclass;
 	struct snd_pcm_ops ops;
 
-	unsigned int dev_registered:1;
-
 	/* Dynamic PCM BE runtime data */
 	struct snd_soc_dpcm_runtime dpcm[2];
 	int fe_compr;
 
 	long pmdown_time;
-	unsigned char pop_wait:1;
 
 	/* runtime devices */
 	struct snd_pcm *pcm;
@@ -1219,7 +1219,6 @@ struct snd_soc_pcm_runtime {
 	struct snd_soc_platform *platform;
 	struct snd_soc_dai *codec_dai;
 	struct snd_soc_dai *cpu_dai;
-	struct snd_soc_component *component; /* Only valid for AUX dev rtds */
 
 	struct snd_soc_dai **codec_dais;
 	unsigned int num_codecs;
@@ -1232,6 +1231,10 @@ struct snd_soc_pcm_runtime {
 
 	unsigned int num; /* 0-based and monotonic increasing */
 	struct list_head list; /* rtd list of the soc card */
+
+	/* bit field */
+	unsigned int dev_registered:1;
+	unsigned int pop_wait:1;
 };
 
 /* mixer control */
@@ -1541,11 +1544,10 @@ static inline void *snd_soc_platform_get_drvdata(struct snd_soc_platform *platfo
 
 static inline void snd_soc_initialize_card_lists(struct snd_soc_card *card)
 {
-	INIT_LIST_HEAD(&card->codec_dev_list);
 	INIT_LIST_HEAD(&card->widgets);
 	INIT_LIST_HEAD(&card->paths);
 	INIT_LIST_HEAD(&card->dapm_list);
-	INIT_LIST_HEAD(&card->aux_comp_list);
+	INIT_LIST_HEAD(&card->component_dev_list);
 }
 
 static inline bool snd_soc_volsw_is_stereo(struct soc_mixer_control *mc)
@@ -1642,25 +1644,43 @@ static inline struct snd_soc_platform *snd_soc_kcontrol_platform(
 int snd_soc_util_init(void);
 void snd_soc_util_exit(void);
 
-int snd_soc_of_parse_card_name(struct snd_soc_card *card,
-			       const char *propname);
-int snd_soc_of_parse_audio_simple_widgets(struct snd_soc_card *card,
-					  const char *propname);
+#define snd_soc_of_parse_card_name(card, propname) \
+	snd_soc_of_parse_card_name_from_node(card, NULL, propname)
+int snd_soc_of_parse_card_name_from_node(struct snd_soc_card *card,
+					 struct device_node *np,
+					 const char *propname);
+#define snd_soc_of_parse_audio_simple_widgets(card, propname)\
+	snd_soc_of_parse_audio_simple_widgets_from_node(card, NULL, propname)
+int snd_soc_of_parse_audio_simple_widgets_from_node(struct snd_soc_card *card,
+						    struct device_node *np,
+						    const char *propname);
+
 int snd_soc_of_parse_tdm_slot(struct device_node *np,
 			      unsigned int *tx_mask,
 			      unsigned int *rx_mask,
 			      unsigned int *slots,
 			      unsigned int *slot_width);
-void snd_soc_of_parse_audio_prefix(struct snd_soc_card *card,
+#define snd_soc_of_parse_audio_prefix(card, codec_conf, of_node, propname) \
+	snd_soc_of_parse_audio_prefix_from_node(card, NULL, codec_conf, \
+						of_node, propname)
+void snd_soc_of_parse_audio_prefix_from_node(struct snd_soc_card *card,
+				   struct device_node *np,
 				   struct snd_soc_codec_conf *codec_conf,
 				   struct device_node *of_node,
 				   const char *propname);
-int snd_soc_of_parse_audio_routing(struct snd_soc_card *card,
-				   const char *propname);
+
+#define snd_soc_of_parse_audio_routing(card, propname) \
+	snd_soc_of_parse_audio_routing_from_node(card, NULL, propname)
+int snd_soc_of_parse_audio_routing_from_node(struct snd_soc_card *card,
+					     struct device_node *np,
+					     const char *propname);
+
 unsigned int snd_soc_of_parse_daifmt(struct device_node *np,
 				     const char *prefix,
 				     struct device_node **bitclkmaster,
 				     struct device_node **framemaster);
+int snd_soc_get_dai_name(struct of_phandle_args *args,
+			 const char **dai_name);
 int snd_soc_of_get_dai_name(struct device_node *of_node,
 			    const char **dai_name);
 int snd_soc_of_get_dai_link_codecs(struct device *dev,
@@ -1671,6 +1691,9 @@ int snd_soc_add_dai_link(struct snd_soc_card *card,
 				struct snd_soc_dai_link *dai_link);
 void snd_soc_remove_dai_link(struct snd_soc_card *card,
 			     struct snd_soc_dai_link *dai_link);
+struct snd_soc_dai_link *snd_soc_find_dai_link(struct snd_soc_card *card,
+					       int id, const char *name,
+					       const char *stream_name);
 
 int snd_soc_register_dai(struct snd_soc_component *component,
 	struct snd_soc_dai_driver *dai_drv);
@@ -1697,4 +1720,24 @@ static inline void snd_soc_dapm_mutex_unlock(struct snd_soc_dapm_context *dapm)
 	mutex_unlock(&dapm->card->dapm_mutex);
 }
 
+int snd_soc_component_enable_pin(struct snd_soc_component *component,
+				 const char *pin);
+int snd_soc_component_enable_pin_unlocked(struct snd_soc_component *component,
+					  const char *pin);
+int snd_soc_component_disable_pin(struct snd_soc_component *component,
+				  const char *pin);
+int snd_soc_component_disable_pin_unlocked(struct snd_soc_component *component,
+					   const char *pin);
+int snd_soc_component_nc_pin(struct snd_soc_component *component,
+			     const char *pin);
+int snd_soc_component_nc_pin_unlocked(struct snd_soc_component *component,
+				      const char *pin);
+int snd_soc_component_get_pin_status(struct snd_soc_component *component,
+				     const char *pin);
+int snd_soc_component_force_enable_pin(struct snd_soc_component *component,
+				       const char *pin);
+int snd_soc_component_force_enable_pin_unlocked(
+					struct snd_soc_component *component,
+					const char *pin);
+
 #endif
diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index c2119008990a..00558287936d 100644
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -149,7 +149,7 @@ enum se_cmd_flags_table {
  * Used by transport_send_check_condition_and_sense()
  * to signal which ASC/ASCQ sense payload should be built.
  */
-typedef unsigned __bitwise__ sense_reason_t;
+typedef unsigned __bitwise sense_reason_t;
 
 enum tcm_sense_reason_table {
 #define R(x)	(__force sense_reason_t )(x)
diff --git a/include/trace/events/bcache.h b/include/trace/events/bcache.h
index d336b890e31f..df3e9ae5ad8d 100644
--- a/include/trace/events/bcache.h
+++ b/include/trace/events/bcache.h
@@ -27,8 +27,7 @@ DECLARE_EVENT_CLASS(bcache_request,
 		__entry->sector		= bio->bi_iter.bi_sector;
 		__entry->orig_sector	= bio->bi_iter.bi_sector - 16;
 		__entry->nr_sector	= bio->bi_iter.bi_size >> 9;
-		blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf,
-			      bio->bi_iter.bi_size);
+		blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size);
 	),
 
 	TP_printk("%d,%d %s %llu + %u (from %d,%d @ %llu)",
@@ -102,8 +101,7 @@ DECLARE_EVENT_CLASS(bcache_bio,
 		__entry->dev		= bio->bi_bdev->bd_dev;
 		__entry->sector		= bio->bi_iter.bi_sector;
 		__entry->nr_sector	= bio->bi_iter.bi_size >> 9;
-		blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf,
-			      bio->bi_iter.bi_size);
+		blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size);
 	),
 
 	TP_printk("%d,%d  %s %llu + %u",
@@ -138,8 +136,7 @@ TRACE_EVENT(bcache_read,
 		__entry->dev		= bio->bi_bdev->bd_dev;
 		__entry->sector		= bio->bi_iter.bi_sector;
 		__entry->nr_sector	= bio->bi_iter.bi_size >> 9;
-		blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf,
-			      bio->bi_iter.bi_size);
+		blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size);
 		__entry->cache_hit = hit;
 		__entry->bypass = bypass;
 	),
@@ -170,8 +167,7 @@ TRACE_EVENT(bcache_write,
 		__entry->inode		= inode;
 		__entry->sector		= bio->bi_iter.bi_sector;
 		__entry->nr_sector	= bio->bi_iter.bi_size >> 9;
-		blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf,
-			      bio->bi_iter.bi_size);
+		blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size);
 		__entry->writeback = writeback;
 		__entry->bypass = bypass;
 	),
diff --git a/include/trace/events/block.h b/include/trace/events/block.h
index 8f3a163b8166..3e02e3a25413 100644
--- a/include/trace/events/block.h
+++ b/include/trace/events/block.h
@@ -84,8 +84,7 @@ DECLARE_EVENT_CLASS(block_rq_with_error,
 					0 : blk_rq_sectors(rq);
 		__entry->errors    = rq->errors;
 
-		blk_fill_rwbs(__entry->rwbs, req_op(rq), rq->cmd_flags,
-			      blk_rq_bytes(rq));
+		blk_fill_rwbs(__entry->rwbs, rq->cmd_flags, blk_rq_bytes(rq));
 		blk_dump_cmd(__get_str(cmd), rq);
 	),
 
@@ -163,7 +162,7 @@ TRACE_EVENT(block_rq_complete,
 		__entry->nr_sector = nr_bytes >> 9;
 		__entry->errors    = rq->errors;
 
-		blk_fill_rwbs(__entry->rwbs, req_op(rq), rq->cmd_flags, nr_bytes);
+		blk_fill_rwbs(__entry->rwbs, rq->cmd_flags, nr_bytes);
 		blk_dump_cmd(__get_str(cmd), rq);
 	),
 
@@ -199,8 +198,7 @@ DECLARE_EVENT_CLASS(block_rq,
 		__entry->bytes     = (rq->cmd_type == REQ_TYPE_BLOCK_PC) ?
 					blk_rq_bytes(rq) : 0;
 
-		blk_fill_rwbs(__entry->rwbs, req_op(rq), rq->cmd_flags,
-			      blk_rq_bytes(rq));
+		blk_fill_rwbs(__entry->rwbs, rq->cmd_flags, blk_rq_bytes(rq));
 		blk_dump_cmd(__get_str(cmd), rq);
 		memcpy(__entry->comm, current->comm, TASK_COMM_LEN);
 	),
@@ -274,8 +272,7 @@ TRACE_EVENT(block_bio_bounce,
 					  bio->bi_bdev->bd_dev : 0;
 		__entry->sector		= bio->bi_iter.bi_sector;
 		__entry->nr_sector	= bio_sectors(bio);
-		blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf,
-			      bio->bi_iter.bi_size);
+		blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size);
 		memcpy(__entry->comm, current->comm, TASK_COMM_LEN);
 	),
 
@@ -313,8 +310,7 @@ TRACE_EVENT(block_bio_complete,
 		__entry->sector		= bio->bi_iter.bi_sector;
 		__entry->nr_sector	= bio_sectors(bio);
 		__entry->error		= error;
-		blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf,
-			      bio->bi_iter.bi_size);
+		blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size);
 	),
 
 	TP_printk("%d,%d %s %llu + %u [%d]",
@@ -341,8 +337,7 @@ DECLARE_EVENT_CLASS(block_bio_merge,
 		__entry->dev		= bio->bi_bdev->bd_dev;
 		__entry->sector		= bio->bi_iter.bi_sector;
 		__entry->nr_sector	= bio_sectors(bio);
-		blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf,
-			      bio->bi_iter.bi_size);
+		blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size);
 		memcpy(__entry->comm, current->comm, TASK_COMM_LEN);
 	),
 
@@ -409,8 +404,7 @@ TRACE_EVENT(block_bio_queue,
 		__entry->dev		= bio->bi_bdev->bd_dev;
 		__entry->sector		= bio->bi_iter.bi_sector;
 		__entry->nr_sector	= bio_sectors(bio);
-		blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf,
-			      bio->bi_iter.bi_size);
+		blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size);
 		memcpy(__entry->comm, current->comm, TASK_COMM_LEN);
 	),
 
@@ -438,7 +432,7 @@ DECLARE_EVENT_CLASS(block_get_rq,
 		__entry->dev		= bio ? bio->bi_bdev->bd_dev : 0;
 		__entry->sector		= bio ? bio->bi_iter.bi_sector : 0;
 		__entry->nr_sector	= bio ? bio_sectors(bio) : 0;
-		blk_fill_rwbs(__entry->rwbs, bio ? bio_op(bio) : 0,
+		blk_fill_rwbs(__entry->rwbs,
 			      bio ? bio->bi_opf : 0, __entry->nr_sector);
 		memcpy(__entry->comm, current->comm, TASK_COMM_LEN);
         ),
@@ -573,8 +567,7 @@ TRACE_EVENT(block_split,
 		__entry->dev		= bio->bi_bdev->bd_dev;
 		__entry->sector		= bio->bi_iter.bi_sector;
 		__entry->new_sector	= new_sector;
-		blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf,
-			      bio->bi_iter.bi_size);
+		blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size);
 		memcpy(__entry->comm, current->comm, TASK_COMM_LEN);
 	),
 
@@ -617,8 +610,7 @@ TRACE_EVENT(block_bio_remap,
 		__entry->nr_sector	= bio_sectors(bio);
 		__entry->old_dev	= dev;
 		__entry->old_sector	= from;
-		blk_fill_rwbs(__entry->rwbs, bio_op(bio), bio->bi_opf,
-			      bio->bi_iter.bi_size);
+		blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size);
 	),
 
 	TP_printk("%d,%d %s %llu + %u <- (%d,%d) %llu",
@@ -664,8 +656,7 @@ TRACE_EVENT(block_rq_remap,
 		__entry->old_dev	= dev;
 		__entry->old_sector	= from;
 		__entry->nr_bios	= blk_rq_count_bios(rq);
-		blk_fill_rwbs(__entry->rwbs, req_op(rq), rq->cmd_flags,
-			      blk_rq_bytes(rq));
+		blk_fill_rwbs(__entry->rwbs, rq->cmd_flags, blk_rq_bytes(rq));
 	),
 
 	TP_printk("%d,%d %s %llu + %u <- (%d,%d) %llu %u",
diff --git a/include/trace/events/fence.h b/include/trace/events/dma_fence.h
index d6dfa05ba322..1157cb4c3c6f 100644
--- a/include/trace/events/fence.h
+++ b/include/trace/events/dma_fence.h
@@ -1,17 +1,17 @@
 #undef TRACE_SYSTEM
-#define TRACE_SYSTEM fence
+#define TRACE_SYSTEM dma_fence
 
 #if !defined(_TRACE_FENCE_H) || defined(TRACE_HEADER_MULTI_READ)
-#define _TRACE_FENCE_H
+#define _TRACE_DMA_FENCE_H
 
 #include <linux/tracepoint.h>
 
-struct fence;
+struct dma_fence;
 
-TRACE_EVENT(fence_annotate_wait_on,
+TRACE_EVENT(dma_fence_annotate_wait_on,
 
 	/* fence: the fence waiting on f1, f1: the fence to be waited on. */
-	TP_PROTO(struct fence *fence, struct fence *f1),
+	TP_PROTO(struct dma_fence *fence, struct dma_fence *f1),
 
 	TP_ARGS(fence, f1),
 
@@ -48,9 +48,9 @@ TRACE_EVENT(fence_annotate_wait_on,
 		  __entry->waiting_context, __entry->waiting_seqno)
 );
 
-DECLARE_EVENT_CLASS(fence,
+DECLARE_EVENT_CLASS(dma_fence,
 
-	TP_PROTO(struct fence *fence),
+	TP_PROTO(struct dma_fence *fence),
 
 	TP_ARGS(fence),
 
@@ -73,56 +73,56 @@ DECLARE_EVENT_CLASS(fence,
 		  __entry->seqno)
 );
 
-DEFINE_EVENT(fence, fence_emit,
+DEFINE_EVENT(dma_fence, dma_fence_emit,
 
-	TP_PROTO(struct fence *fence),
+	TP_PROTO(struct dma_fence *fence),
 
 	TP_ARGS(fence)
 );
 
-DEFINE_EVENT(fence, fence_init,
+DEFINE_EVENT(dma_fence, dma_fence_init,
 
-	TP_PROTO(struct fence *fence),
+	TP_PROTO(struct dma_fence *fence),
 
 	TP_ARGS(fence)
 );
 
-DEFINE_EVENT(fence, fence_destroy,
+DEFINE_EVENT(dma_fence, dma_fence_destroy,
 
-	TP_PROTO(struct fence *fence),
+	TP_PROTO(struct dma_fence *fence),
 
 	TP_ARGS(fence)
 );
 
-DEFINE_EVENT(fence, fence_enable_signal,
+DEFINE_EVENT(dma_fence, dma_fence_enable_signal,
 
-	TP_PROTO(struct fence *fence),
+	TP_PROTO(struct dma_fence *fence),
 
 	TP_ARGS(fence)
 );
 
-DEFINE_EVENT(fence, fence_signaled,
+DEFINE_EVENT(dma_fence, dma_fence_signaled,
 
-	TP_PROTO(struct fence *fence),
+	TP_PROTO(struct dma_fence *fence),
 
 	TP_ARGS(fence)
 );
 
-DEFINE_EVENT(fence, fence_wait_start,
+DEFINE_EVENT(dma_fence, dma_fence_wait_start,
 
-	TP_PROTO(struct fence *fence),
+	TP_PROTO(struct dma_fence *fence),
 
 	TP_ARGS(fence)
 );
 
-DEFINE_EVENT(fence, fence_wait_end,
+DEFINE_EVENT(dma_fence, dma_fence_wait_end,
 
-	TP_PROTO(struct fence *fence),
+	TP_PROTO(struct dma_fence *fence),
 
 	TP_ARGS(fence)
 );
 
-#endif /*  _TRACE_FENCE_H */
+#endif /*  _TRACE_DMA_FENCE_H */
 
 /* This part must be outside protection */
 #include <trace/define_trace.h>
diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h
index 903a09165bb1..01b3c9869a0d 100644
--- a/include/trace/events/f2fs.h
+++ b/include/trace/events/f2fs.h
@@ -32,7 +32,7 @@ TRACE_DEFINE_ENUM(LFS);
 TRACE_DEFINE_ENUM(SSR);
 TRACE_DEFINE_ENUM(__REQ_RAHEAD);
 TRACE_DEFINE_ENUM(__REQ_SYNC);
-TRACE_DEFINE_ENUM(__REQ_NOIDLE);
+TRACE_DEFINE_ENUM(__REQ_IDLE);
 TRACE_DEFINE_ENUM(__REQ_PREFLUSH);
 TRACE_DEFINE_ENUM(__REQ_FUA);
 TRACE_DEFINE_ENUM(__REQ_PRIO);
@@ -55,7 +55,7 @@ TRACE_DEFINE_ENUM(CP_DISCARD);
 		{ IPU,		"IN-PLACE" },				\
 		{ OPU,		"OUT-OF-PLACE" })
 
-#define F2FS_BIO_FLAG_MASK(t)	(t & (REQ_RAHEAD | WRITE_FLUSH_FUA))
+#define F2FS_BIO_FLAG_MASK(t)	(t & (REQ_RAHEAD | REQ_PREFLUSH | REQ_FUA))
 #define F2FS_BIO_EXTRA_MASK(t)	(t & (REQ_META | REQ_PRIO))
 
 #define show_bio_type(op_flags)	show_bio_op_flags(op_flags), 		\
@@ -65,11 +65,9 @@ TRACE_DEFINE_ENUM(CP_DISCARD);
 	__print_symbolic(F2FS_BIO_FLAG_MASK(flags),			\
 		{ 0,			"WRITE" },			\
 		{ REQ_RAHEAD, 		"READAHEAD" },			\
-		{ READ_SYNC, 		"READ_SYNC" },			\
-		{ WRITE_SYNC, 		"WRITE_SYNC" },			\
-		{ WRITE_FLUSH,		"WRITE_FLUSH" },		\
-		{ WRITE_FUA, 		"WRITE_FUA" },			\
-		{ WRITE_FLUSH_FUA,	"WRITE_FLUSH_FUA" })
+		{ REQ_SYNC, 		"REQ_SYNC" },			\
+		{ REQ_PREFLUSH,		"REQ_PREFLUSH" },		\
+		{ REQ_FUA,		"REQ_FUA" })
 
 #define show_bio_extra(type)						\
 	__print_symbolic(F2FS_BIO_EXTRA_MASK(type),			\
@@ -1113,6 +1111,27 @@ TRACE_EVENT(f2fs_issue_discard,
 		(unsigned long long)__entry->blklen)
 );
 
+TRACE_EVENT(f2fs_issue_reset_zone,
+
+	TP_PROTO(struct super_block *sb, block_t blkstart),
+
+	TP_ARGS(sb, blkstart),
+
+	TP_STRUCT__entry(
+		__field(dev_t,	dev)
+		__field(block_t, blkstart)
+	),
+
+	TP_fast_assign(
+		__entry->dev	= sb->s_dev;
+		__entry->blkstart = blkstart;
+	),
+
+	TP_printk("dev = (%d,%d), reset zone at block = 0x%llx",
+		show_dev(__entry),
+		(unsigned long long)__entry->blkstart)
+);
+
 TRACE_EVENT(f2fs_issue_flush,
 
 	TP_PROTO(struct super_block *sb, unsigned int nobarrier,
diff --git a/include/trace/events/i2c.h b/include/trace/events/i2c.h
index fe17187df65d..4abb8eab34d3 100644
--- a/include/trace/events/i2c.h
+++ b/include/trace/events/i2c.h
@@ -20,7 +20,7 @@
 /*
  * drivers/i2c/i2c-core.c
  */
-extern void i2c_transfer_trace_reg(void);
+extern int i2c_transfer_trace_reg(void);
 extern void i2c_transfer_trace_unreg(void);
 
 /*
diff --git a/include/trace/events/wbt.h b/include/trace/events/wbt.h
new file mode 100644
index 000000000000..3c518e455680
--- /dev/null
+++ b/include/trace/events/wbt.h
@@ -0,0 +1,153 @@
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM wbt
+
+#if !defined(_TRACE_WBT_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_WBT_H
+
+#include <linux/tracepoint.h>
+#include "../../../block/blk-wbt.h"
+
+/**
+ * wbt_stat - trace stats for blk_wb
+ * @stat: array of read/write stats
+ */
+TRACE_EVENT(wbt_stat,
+
+	TP_PROTO(struct backing_dev_info *bdi, struct blk_rq_stat *stat),
+
+	TP_ARGS(bdi, stat),
+
+	TP_STRUCT__entry(
+		__array(char, name, 32)
+		__field(s64, rmean)
+		__field(u64, rmin)
+		__field(u64, rmax)
+		__field(s64, rnr_samples)
+		__field(s64, rtime)
+		__field(s64, wmean)
+		__field(u64, wmin)
+		__field(u64, wmax)
+		__field(s64, wnr_samples)
+		__field(s64, wtime)
+	),
+
+	TP_fast_assign(
+		strncpy(__entry->name, dev_name(bdi->dev), 32);
+		__entry->rmean		= stat[0].mean;
+		__entry->rmin		= stat[0].min;
+		__entry->rmax		= stat[0].max;
+		__entry->rnr_samples	= stat[0].nr_samples;
+		__entry->wmean		= stat[1].mean;
+		__entry->wmin		= stat[1].min;
+		__entry->wmax		= stat[1].max;
+		__entry->wnr_samples	= stat[1].nr_samples;
+	),
+
+	TP_printk("%s: rmean=%llu, rmin=%llu, rmax=%llu, rsamples=%llu, "
+		  "wmean=%llu, wmin=%llu, wmax=%llu, wsamples=%llu\n",
+		  __entry->name, __entry->rmean, __entry->rmin, __entry->rmax,
+		  __entry->rnr_samples, __entry->wmean, __entry->wmin,
+		  __entry->wmax, __entry->wnr_samples)
+);
+
+/**
+ * wbt_lat - trace latency event
+ * @lat: latency trigger
+ */
+TRACE_EVENT(wbt_lat,
+
+	TP_PROTO(struct backing_dev_info *bdi, unsigned long lat),
+
+	TP_ARGS(bdi, lat),
+
+	TP_STRUCT__entry(
+		__array(char, name, 32)
+		__field(unsigned long, lat)
+	),
+
+	TP_fast_assign(
+		strncpy(__entry->name, dev_name(bdi->dev), 32);
+		__entry->lat = div_u64(lat, 1000);
+	),
+
+	TP_printk("%s: latency %lluus\n", __entry->name,
+			(unsigned long long) __entry->lat)
+);
+
+/**
+ * wbt_step - trace wb event step
+ * @msg: context message
+ * @step: the current scale step count
+ * @window: the current monitoring window
+ * @bg: the current background queue limit
+ * @normal: the current normal writeback limit
+ * @max: the current max throughput writeback limit
+ */
+TRACE_EVENT(wbt_step,
+
+	TP_PROTO(struct backing_dev_info *bdi, const char *msg,
+		 int step, unsigned long window, unsigned int bg,
+		 unsigned int normal, unsigned int max),
+
+	TP_ARGS(bdi, msg, step, window, bg, normal, max),
+
+	TP_STRUCT__entry(
+		__array(char, name, 32)
+		__field(const char *, msg)
+		__field(int, step)
+		__field(unsigned long, window)
+		__field(unsigned int, bg)
+		__field(unsigned int, normal)
+		__field(unsigned int, max)
+	),
+
+	TP_fast_assign(
+		strncpy(__entry->name, dev_name(bdi->dev), 32);
+		__entry->msg	= msg;
+		__entry->step	= step;
+		__entry->window	= div_u64(window, 1000);
+		__entry->bg	= bg;
+		__entry->normal	= normal;
+		__entry->max	= max;
+	),
+
+	TP_printk("%s: %s: step=%d, window=%luus, background=%u, normal=%u, max=%u\n",
+		  __entry->name, __entry->msg, __entry->step, __entry->window,
+		  __entry->bg, __entry->normal, __entry->max)
+);
+
+/**
+ * wbt_timer - trace wb timer event
+ * @status: timer state status
+ * @step: the current scale step count
+ * @inflight: tracked writes inflight
+ */
+TRACE_EVENT(wbt_timer,
+
+	TP_PROTO(struct backing_dev_info *bdi, unsigned int status,
+		 int step, unsigned int inflight),
+
+	TP_ARGS(bdi, status, step, inflight),
+
+	TP_STRUCT__entry(
+		__array(char, name, 32)
+		__field(unsigned int, status)
+		__field(int, step)
+		__field(unsigned int, inflight)
+	),
+
+	TP_fast_assign(
+		strncpy(__entry->name, dev_name(bdi->dev), 32);
+		__entry->status		= status;
+		__entry->step		= step;
+		__entry->inflight	= inflight;
+	),
+
+	TP_printk("%s: status=%u, step=%d, inflight=%u\n", __entry->name,
+		  __entry->status, __entry->step, __entry->inflight)
+);
+
+#endif /* _TRACE_WBT_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index d6b5a21f3d3c..396183628f3c 100644
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -50,6 +50,7 @@ extern "C" {
 #define DRM_AMDGPU_WAIT_CS		0x09
 #define DRM_AMDGPU_GEM_OP		0x10
 #define DRM_AMDGPU_GEM_USERPTR		0x11
+#define DRM_AMDGPU_WAIT_FENCES		0x12
 
 #define DRM_IOCTL_AMDGPU_GEM_CREATE	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_CREATE, union drm_amdgpu_gem_create)
 #define DRM_IOCTL_AMDGPU_GEM_MMAP	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_MMAP, union drm_amdgpu_gem_mmap)
@@ -63,6 +64,7 @@ extern "C" {
 #define DRM_IOCTL_AMDGPU_WAIT_CS	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_WAIT_CS, union drm_amdgpu_wait_cs)
 #define DRM_IOCTL_AMDGPU_GEM_OP		DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_OP, struct drm_amdgpu_gem_op)
 #define DRM_IOCTL_AMDGPU_GEM_USERPTR	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_USERPTR, struct drm_amdgpu_gem_userptr)
+#define DRM_IOCTL_AMDGPU_WAIT_FENCES	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_WAIT_FENCES, union drm_amdgpu_wait_fences)
 
 #define AMDGPU_GEM_DOMAIN_CPU		0x1
 #define AMDGPU_GEM_DOMAIN_GTT		0x2
@@ -81,6 +83,8 @@ extern "C" {
 #define AMDGPU_GEM_CREATE_VRAM_CLEARED		(1 << 3)
 /* Flag that create shadow bo(GTT) while allocating vram bo */
 #define AMDGPU_GEM_CREATE_SHADOW		(1 << 4)
+/* Flag that allocating the BO should use linear VRAM */
+#define AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS	(1 << 5)
 
 struct drm_amdgpu_gem_create_in  {
 	/** the requested memory size */
@@ -305,6 +309,32 @@ union drm_amdgpu_wait_cs {
 	struct drm_amdgpu_wait_cs_out out;
 };
 
+struct drm_amdgpu_fence {
+	__u32 ctx_id;
+	__u32 ip_type;
+	__u32 ip_instance;
+	__u32 ring;
+	__u64 seq_no;
+};
+
+struct drm_amdgpu_wait_fences_in {
+	/** This points to uint64_t * which points to fences */
+	__u64 fences;
+	__u32 fence_count;
+	__u32 wait_all;
+	__u64 timeout_ns;
+};
+
+struct drm_amdgpu_wait_fences_out {
+	__u32 status;
+	__u32 first_signaled;
+};
+
+union drm_amdgpu_wait_fences {
+	struct drm_amdgpu_wait_fences_in in;
+	struct drm_amdgpu_wait_fences_out out;
+};
+
 #define AMDGPU_GEM_OP_GET_GEM_CREATE_INFO	0
 #define AMDGPU_GEM_OP_SET_PLACEMENT		1
 
@@ -436,6 +466,7 @@ struct drm_amdgpu_cs_chunk_data {
  *
  */
 #define AMDGPU_IDS_FLAGS_FUSION         0x1
+#define AMDGPU_IDS_FLAGS_PREEMPTION     0x2
 
 /* indicate if acceleration can be working */
 #define AMDGPU_INFO_ACCEL_WORKING		0x00
@@ -487,6 +518,16 @@ struct drm_amdgpu_cs_chunk_data {
 #define AMDGPU_INFO_VIS_VRAM_USAGE		0x17
 /* number of TTM buffer evictions */
 #define AMDGPU_INFO_NUM_EVICTIONS		0x18
+/* Query memory about VRAM and GTT domains */
+#define AMDGPU_INFO_MEMORY			0x19
+/* Query vce clock table */
+#define AMDGPU_INFO_VCE_CLOCK_TABLE		0x1A
+/* Query vbios related information */
+#define AMDGPU_INFO_VBIOS			0x1B
+	/* Subquery id: Query vbios size */
+	#define AMDGPU_INFO_VBIOS_SIZE		0x1
+	/* Subquery id: Query vbios image */
+	#define AMDGPU_INFO_VBIOS_IMAGE		0x2
 
 #define AMDGPU_INFO_MMR_SE_INDEX_SHIFT	0
 #define AMDGPU_INFO_MMR_SE_INDEX_MASK	0xff
@@ -545,6 +586,11 @@ struct drm_amdgpu_info {
 		} read_mmr_reg;
 
 		struct drm_amdgpu_query_fw query_fw;
+
+		struct {
+			__u32 type;
+			__u32 offset;
+		} vbios_info;
 	};
 };
 
@@ -572,6 +618,34 @@ struct drm_amdgpu_info_vram_gtt {
 	__u64 gtt_size;
 };
 
+struct drm_amdgpu_heap_info {
+	/** max. physical memory */
+	__u64 total_heap_size;
+
+	/** Theoretical max. available memory in the given heap */
+	__u64 usable_heap_size;
+
+	/**
+	 * Number of bytes allocated in the heap. This includes all processes
+	 * and private allocations in the kernel. It changes when new buffers
+	 * are allocated, freed, and moved. It cannot be larger than
+	 * heap_size.
+	 */
+	__u64 heap_usage;
+
+	/**
+	 * Theoretical possible max. size of buffer which
+	 * could be allocated in the given heap
+	 */
+	__u64 max_allocation;
+};
+
+struct drm_amdgpu_memory_info {
+	struct drm_amdgpu_heap_info vram;
+	struct drm_amdgpu_heap_info cpu_accessible_vram;
+	struct drm_amdgpu_heap_info gtt;
+};
+
 struct drm_amdgpu_info_firmware {
 	__u32 ver;
 	__u32 feature;
@@ -645,6 +719,24 @@ struct drm_amdgpu_info_hw_ip {
 	__u32  _pad;
 };
 
+#define AMDGPU_VCE_CLOCK_TABLE_ENTRIES		6
+
+struct drm_amdgpu_info_vce_clock_table_entry {
+	/** System clock */
+	__u32 sclk;
+	/** Memory clock */
+	__u32 mclk;
+	/** VCE clock */
+	__u32 eclk;
+	__u32 pad;
+};
+
+struct drm_amdgpu_info_vce_clock_table {
+	struct drm_amdgpu_info_vce_clock_table_entry entries[AMDGPU_VCE_CLOCK_TABLE_ENTRIES];
+	__u32 num_valid_entries;
+	__u32 pad;
+};
+
 /*
  * Supported GPU families
  */
diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h
index df0e3504c349..ce7efe2e8a5e 100644
--- a/include/uapi/drm/drm_mode.h
+++ b/include/uapi/drm/drm_mode.h
@@ -47,7 +47,15 @@ extern "C" {
 #define DRM_MODE_TYPE_DRIVER	(1<<6)
 
 /* Video mode flags */
-/* bit compatible with the xorg definitions. */
+/* bit compatible with the xrandr RR_ definitions (bits 0-13)
+ *
+ * ABI warning: Existing userspace really expects
+ * the mode flags to match the xrandr definitions. Any
+ * changes that don't match the xrandr definitions will
+ * likely need a new client cap or some other mechanism
+ * to avoid breaking existing userspace. This includes
+ * allocating new flags in the previously unused bits!
+ */
 #define DRM_MODE_FLAG_PHSYNC			(1<<0)
 #define DRM_MODE_FLAG_NHSYNC			(1<<1)
 #define DRM_MODE_FLAG_PVSYNC			(1<<2)
@@ -77,6 +85,19 @@ extern "C" {
 #define  DRM_MODE_FLAG_3D_TOP_AND_BOTTOM	(7<<14)
 #define  DRM_MODE_FLAG_3D_SIDE_BY_SIDE_HALF	(8<<14)
 
+/* Picture aspect ratio options */
+#define DRM_MODE_PICTURE_ASPECT_NONE		0
+#define DRM_MODE_PICTURE_ASPECT_4_3		1
+#define DRM_MODE_PICTURE_ASPECT_16_9		2
+
+/* Aspect ratio flag bitmask (4 bits 22:19) */
+#define DRM_MODE_FLAG_PIC_AR_MASK		(0x0F<<19)
+#define  DRM_MODE_FLAG_PIC_AR_NONE \
+			(DRM_MODE_PICTURE_ASPECT_NONE<<19)
+#define  DRM_MODE_FLAG_PIC_AR_4_3 \
+			(DRM_MODE_PICTURE_ASPECT_4_3<<19)
+#define  DRM_MODE_FLAG_PIC_AR_16_9 \
+			(DRM_MODE_PICTURE_ASPECT_16_9<<19)
 
 /* DPMS flags */
 /* bit compatible with the xorg definitions. */
@@ -92,11 +113,6 @@ extern "C" {
 #define DRM_MODE_SCALE_CENTER		2 /* Centered, no scaling */
 #define DRM_MODE_SCALE_ASPECT		3 /* Full screen, preserve aspect */
 
-/* Picture aspect ratio options */
-#define DRM_MODE_PICTURE_ASPECT_NONE	0
-#define DRM_MODE_PICTURE_ASPECT_4_3	1
-#define DRM_MODE_PICTURE_ASPECT_16_9	2
-
 /* Dithering mode options */
 #define DRM_MODE_DITHERING_OFF	0
 #define DRM_MODE_DITHERING_ON	1
@@ -220,14 +236,16 @@ struct drm_mode_get_encoder {
 
 /* This is for connectors with multiple signal types. */
 /* Try to match DRM_MODE_CONNECTOR_X as closely as possible. */
-#define DRM_MODE_SUBCONNECTOR_Automatic	0
-#define DRM_MODE_SUBCONNECTOR_Unknown	0
-#define DRM_MODE_SUBCONNECTOR_DVID	3
-#define DRM_MODE_SUBCONNECTOR_DVIA	4
-#define DRM_MODE_SUBCONNECTOR_Composite	5
-#define DRM_MODE_SUBCONNECTOR_SVIDEO	6
-#define DRM_MODE_SUBCONNECTOR_Component	8
-#define DRM_MODE_SUBCONNECTOR_SCART	9
+enum drm_mode_subconnector {
+	DRM_MODE_SUBCONNECTOR_Automatic = 0,
+	DRM_MODE_SUBCONNECTOR_Unknown = 0,
+	DRM_MODE_SUBCONNECTOR_DVID = 3,
+	DRM_MODE_SUBCONNECTOR_DVIA = 4,
+	DRM_MODE_SUBCONNECTOR_Composite = 5,
+	DRM_MODE_SUBCONNECTOR_SVIDEO = 6,
+	DRM_MODE_SUBCONNECTOR_Component = 8,
+	DRM_MODE_SUBCONNECTOR_SCART = 9,
+};
 
 #define DRM_MODE_CONNECTOR_Unknown	0
 #define DRM_MODE_CONNECTOR_VGA		1
@@ -392,17 +410,20 @@ struct drm_mode_fb_cmd2 {
 	 * offsets[1].  Note that offsets[0] will generally
 	 * be 0 (but this is not required).
 	 *
-	 * To accommodate tiled, compressed, etc formats, a per-plane
+	 * To accommodate tiled, compressed, etc formats, a
 	 * modifier can be specified.  The default value of zero
 	 * indicates "native" format as specified by the fourcc.
-	 * Vendor specific modifier token.  This allows, for example,
-	 * different tiling/swizzling pattern on different planes.
-	 * See discussion above of DRM_FORMAT_MOD_xxx.
+	 * Vendor specific modifier token.  Note that even though
+	 * it looks like we have a modifier per-plane, we in fact
+	 * do not. The modifier for each plane must be identical.
+	 * Thus all combinations of different data layouts for
+	 * multi plane formats must be enumerated as separate
+	 * modifiers.
 	 */
 	__u32 handles[4];
 	__u32 pitches[4]; /* pitch for each plane */
 	__u32 offsets[4]; /* offset of each plane */
-	__u64 modifier[4]; /* ie, tiling, compressed (per plane) */
+	__u64 modifier[4]; /* ie, tiling, compress */
 };
 
 #define DRM_MODE_FB_DIRTY_ANNOTATE_COPY 0x01
diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h
index 03725fe89859..1c12a350eca3 100644
--- a/include/uapi/drm/i915_drm.h
+++ b/include/uapi/drm/i915_drm.h
@@ -389,6 +389,11 @@ typedef struct drm_i915_irq_wait {
 #define I915_PARAM_MIN_EU_IN_POOL	 39
 #define I915_PARAM_MMAP_GTT_VERSION	 40
 
+/* Query whether DRM_I915_GEM_EXECBUFFER2 supports user defined execution
+ * priorities and the driver will attempt to execute batches in priority order.
+ */
+#define I915_PARAM_HAS_SCHEDULER	 41
+
 typedef struct drm_i915_getparam {
 	__s32 param;
 	/*
diff --git a/include/uapi/drm/msm_drm.h b/include/uapi/drm/msm_drm.h
index 8c51e8a0df89..4d5d6a2bc59e 100644
--- a/include/uapi/drm/msm_drm.h
+++ b/include/uapi/drm/msm_drm.h
@@ -2,17 +2,24 @@
  * Copyright (C) 2013 Red Hat
  * Author: Rob Clark <robdclark@gmail.com>
  *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
  *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
  *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  */
 
 #ifndef __MSM_DRM_H__
diff --git a/include/uapi/drm/vc4_drm.h b/include/uapi/drm/vc4_drm.h
index ad7edc3edf7c..f07a09016726 100644
--- a/include/uapi/drm/vc4_drm.h
+++ b/include/uapi/drm/vc4_drm.h
@@ -286,6 +286,8 @@ struct drm_vc4_get_hang_state {
 #define DRM_VC4_PARAM_V3D_IDENT1		1
 #define DRM_VC4_PARAM_V3D_IDENT2		2
 #define DRM_VC4_PARAM_SUPPORTS_BRANCHES		3
+#define DRM_VC4_PARAM_SUPPORTS_ETC1		4
+#define DRM_VC4_PARAM_SUPPORTS_THREADED_FS	5
 
 struct drm_vc4_get_param {
 	__u32 param;
diff --git a/include/uapi/linux/Kbuild b/include/uapi/linux/Kbuild
index 855908e70610..a8b93e685239 100644
--- a/include/uapi/linux/Kbuild
+++ b/include/uapi/linux/Kbuild
@@ -70,6 +70,7 @@ header-y += bfs_fs.h
 header-y += binfmts.h
 header-y += blkpg.h
 header-y += blktrace_api.h
+header-y += blkzoned.h
 header-y += bpf_common.h
 header-y += bpf_perf_event.h
 header-y += bpf.h
@@ -428,6 +429,7 @@ header-y += udp.h
 header-y += uhid.h
 header-y += uinput.h
 header-y += uio.h
+header-y += uleds.h
 header-y += ultrasound.h
 header-y += un.h
 header-y += unistd.h
@@ -462,6 +464,7 @@ header-y += virtio_rng.h
 header-y += virtio_scsi.h
 header-y += virtio_types.h
 header-y += virtio_vsock.h
+header-y += virtio_crypto.h
 header-y += vm_sockets.h
 header-y += vt.h
 header-y += vtpm_proxy.h
diff --git a/include/uapi/linux/audit.h b/include/uapi/linux/audit.h
index 208df7b44e90..1c107cb1c83f 100644
--- a/include/uapi/linux/audit.h
+++ b/include/uapi/linux/audit.h
@@ -254,6 +254,7 @@
 #define AUDIT_OBJ_LEV_LOW	22
 #define AUDIT_OBJ_LEV_HIGH	23
 #define AUDIT_LOGINUID_SET	24
+#define AUDIT_SESSIONID	25	/* Session ID */
 
 				/* These are ONLY useful when checking
 				 * at syscall exit time (AUDIT_AT_EXIT). */
@@ -330,10 +331,12 @@ enum {
 #define AUDIT_FEATURE_BITMAP_BACKLOG_WAIT_TIME	0x00000002
 #define AUDIT_FEATURE_BITMAP_EXECUTABLE_PATH	0x00000004
 #define AUDIT_FEATURE_BITMAP_EXCLUDE_EXTEND	0x00000008
+#define AUDIT_FEATURE_BITMAP_SESSIONID_FILTER	0x00000010
 #define AUDIT_FEATURE_BITMAP_ALL (AUDIT_FEATURE_BITMAP_BACKLOG_LIMIT | \
 				  AUDIT_FEATURE_BITMAP_BACKLOG_WAIT_TIME | \
 				  AUDIT_FEATURE_BITMAP_EXECUTABLE_PATH | \
-				  AUDIT_FEATURE_BITMAP_EXCLUDE_EXTEND)
+				  AUDIT_FEATURE_BITMAP_EXCLUDE_EXTEND | \
+				  AUDIT_FEATURE_BITMAP_SESSIONID_FILTER)
 
 /* deprecated: AUDIT_VERSION_* */
 #define AUDIT_VERSION_LATEST 		AUDIT_FEATURE_BITMAP_ALL
diff --git a/include/uapi/linux/blkzoned.h b/include/uapi/linux/blkzoned.h
new file mode 100644
index 000000000000..40d1d7bff537
--- /dev/null
+++ b/include/uapi/linux/blkzoned.h
@@ -0,0 +1,143 @@
+/*
+ * Zoned block devices handling.
+ *
+ * Copyright (C) 2015 Seagate Technology PLC
+ *
+ * Written by: Shaun Tancheff <shaun.tancheff@seagate.com>
+ *
+ * Modified by: Damien Le Moal <damien.lemoal@hgst.com>
+ * Copyright (C) 2016 Western Digital
+ *
+ * This file is licensed under  the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+#ifndef _UAPI_BLKZONED_H
+#define _UAPI_BLKZONED_H
+
+#include <linux/types.h>
+#include <linux/ioctl.h>
+
+/**
+ * enum blk_zone_type - Types of zones allowed in a zoned device.
+ *
+ * @BLK_ZONE_TYPE_CONVENTIONAL: The zone has no write pointer and can be writen
+ *                              randomly. Zone reset has no effect on the zone.
+ * @BLK_ZONE_TYPE_SEQWRITE_REQ: The zone must be written sequentially
+ * @BLK_ZONE_TYPE_SEQWRITE_PREF: The zone can be written non-sequentially
+ *
+ * Any other value not defined is reserved and must be considered as invalid.
+ */
+enum blk_zone_type {
+	BLK_ZONE_TYPE_CONVENTIONAL	= 0x1,
+	BLK_ZONE_TYPE_SEQWRITE_REQ	= 0x2,
+	BLK_ZONE_TYPE_SEQWRITE_PREF	= 0x3,
+};
+
+/**
+ * enum blk_zone_cond - Condition [state] of a zone in a zoned device.
+ *
+ * @BLK_ZONE_COND_NOT_WP: The zone has no write pointer, it is conventional.
+ * @BLK_ZONE_COND_EMPTY: The zone is empty.
+ * @BLK_ZONE_COND_IMP_OPEN: The zone is open, but not explicitly opened.
+ * @BLK_ZONE_COND_EXP_OPEN: The zones was explicitly opened by an
+ *                          OPEN ZONE command.
+ * @BLK_ZONE_COND_CLOSED: The zone was [explicitly] closed after writing.
+ * @BLK_ZONE_COND_FULL: The zone is marked as full, possibly by a zone
+ *                      FINISH ZONE command.
+ * @BLK_ZONE_COND_READONLY: The zone is read-only.
+ * @BLK_ZONE_COND_OFFLINE: The zone is offline (sectors cannot be read/written).
+ *
+ * The Zone Condition state machine in the ZBC/ZAC standards maps the above
+ * deinitions as:
+ *   - ZC1: Empty         | BLK_ZONE_EMPTY
+ *   - ZC2: Implicit Open | BLK_ZONE_COND_IMP_OPEN
+ *   - ZC3: Explicit Open | BLK_ZONE_COND_EXP_OPEN
+ *   - ZC4: Closed        | BLK_ZONE_CLOSED
+ *   - ZC5: Full          | BLK_ZONE_FULL
+ *   - ZC6: Read Only     | BLK_ZONE_READONLY
+ *   - ZC7: Offline       | BLK_ZONE_OFFLINE
+ *
+ * Conditions 0x5 to 0xC are reserved by the current ZBC/ZAC spec and should
+ * be considered invalid.
+ */
+enum blk_zone_cond {
+	BLK_ZONE_COND_NOT_WP	= 0x0,
+	BLK_ZONE_COND_EMPTY	= 0x1,
+	BLK_ZONE_COND_IMP_OPEN	= 0x2,
+	BLK_ZONE_COND_EXP_OPEN	= 0x3,
+	BLK_ZONE_COND_CLOSED	= 0x4,
+	BLK_ZONE_COND_READONLY	= 0xD,
+	BLK_ZONE_COND_FULL	= 0xE,
+	BLK_ZONE_COND_OFFLINE	= 0xF,
+};
+
+/**
+ * struct blk_zone - Zone descriptor for BLKREPORTZONE ioctl.
+ *
+ * @start: Zone start in 512 B sector units
+ * @len: Zone length in 512 B sector units
+ * @wp: Zone write pointer location in 512 B sector units
+ * @type: see enum blk_zone_type for possible values
+ * @cond: see enum blk_zone_cond for possible values
+ * @non_seq: Flag indicating that the zone is using non-sequential resources
+ *           (for host-aware zoned block devices only).
+ * @reset: Flag indicating that a zone reset is recommended.
+ * @reserved: Padding to 64 B to match the ZBC/ZAC defined zone descriptor size.
+ *
+ * start, len and wp use the regular 512 B sector unit, regardless of the
+ * device logical block size. The overall structure size is 64 B to match the
+ * ZBC/ZAC defined zone descriptor and allow support for future additional
+ * zone information.
+ */
+struct blk_zone {
+	__u64	start;		/* Zone start sector */
+	__u64	len;		/* Zone length in number of sectors */
+	__u64	wp;		/* Zone write pointer position */
+	__u8	type;		/* Zone type */
+	__u8	cond;		/* Zone condition */
+	__u8	non_seq;	/* Non-sequential write resources active */
+	__u8	reset;		/* Reset write pointer recommended */
+	__u8	reserved[36];
+};
+
+/**
+ * struct blk_zone_report - BLKREPORTZONE ioctl request/reply
+ *
+ * @sector: starting sector of report
+ * @nr_zones: IN maximum / OUT actual
+ * @reserved: padding to 16 byte alignment
+ * @zones: Space to hold @nr_zones @zones entries on reply.
+ *
+ * The array of at most @nr_zones must follow this structure in memory.
+ */
+struct blk_zone_report {
+	__u64		sector;
+	__u32		nr_zones;
+	__u8		reserved[4];
+	struct blk_zone zones[0];
+} __packed;
+
+/**
+ * struct blk_zone_range - BLKRESETZONE ioctl request
+ * @sector: starting sector of the first zone to issue reset write pointer
+ * @nr_sectors: Total number of sectors of 1 or more zones to reset
+ */
+struct blk_zone_range {
+	__u64		sector;
+	__u64		nr_sectors;
+};
+
+/**
+ * Zoned block device ioctl's:
+ *
+ * @BLKREPORTZONE: Get zone information. Takes a zone report as argument.
+ *                 The zone report will start from the zone containing the
+ *                 sector specified in the report request structure.
+ * @BLKRESETZONE: Reset the write pointer of the zones in the specified
+ *                sector range. The sector range must be zone aligned.
+ */
+#define BLKREPORTZONE	_IOWR(0x12, 130, struct blk_zone_report)
+#define BLKRESETZONE	_IOW(0x12, 131, struct blk_zone_range)
+
+#endif /* _UAPI_BLKZONED_H */
diff --git a/include/uapi/linux/cryptouser.h b/include/uapi/linux/cryptouser.h
index 79b5ded2001a..11d21fce14d6 100644
--- a/include/uapi/linux/cryptouser.h
+++ b/include/uapi/linux/cryptouser.h
@@ -46,6 +46,7 @@ enum crypto_attr_type_t {
 	CRYPTOCFGA_REPORT_CIPHER,	/* struct crypto_report_cipher */
 	CRYPTOCFGA_REPORT_AKCIPHER,	/* struct crypto_report_akcipher */
 	CRYPTOCFGA_REPORT_KPP,		/* struct crypto_report_kpp */
+	CRYPTOCFGA_REPORT_ACOMP,	/* struct crypto_report_acomp */
 	__CRYPTOCFGA_MAX
 
 #define CRYPTOCFGA_MAX (__CRYPTOCFGA_MAX - 1)
@@ -112,5 +113,9 @@ struct crypto_report_kpp {
 	char type[CRYPTO_MAX_NAME];
 };
 
+struct crypto_report_acomp {
+	char type[CRYPTO_MAX_NAME];
+};
+
 #define CRYPTO_REPORT_MAXSIZE (sizeof(struct crypto_user_alg) + \
 			       sizeof(struct crypto_report_blkcipher))
diff --git a/include/uapi/linux/dm-log-userspace.h b/include/uapi/linux/dm-log-userspace.h
index 0fa0d9ef06a5..05e91e14c501 100644
--- a/include/uapi/linux/dm-log-userspace.h
+++ b/include/uapi/linux/dm-log-userspace.h
@@ -7,6 +7,7 @@
 #ifndef __DM_LOG_USERSPACE_H__
 #define __DM_LOG_USERSPACE_H__
 
+#include <linux/types.h>
 #include <linux/dm-ioctl.h> /* For DM_UUID_LEN */
 
 /*
@@ -147,12 +148,12 @@
 
 /*
  * DM_ULOG_GET_REGION_SIZE corresponds to (found in dm-dirty-log.h):
- * uint32_t (*get_region_size)(struct dm_dirty_log *log);
+ * __u32 (*get_region_size)(struct dm_dirty_log *log);
  *
  * Payload-to-userspace:
  *	None.
  * Payload-to-kernel:
- *	uint64_t - contains the region size
+ *	__u64 - contains the region size
  *
  * The region size is something that was determined at constructor time.
  * It is returned in the payload area and 'data_size' is set to
@@ -168,11 +169,11 @@
  * int (*is_clean)(struct dm_dirty_log *log, region_t region);
  *
  * Payload-to-userspace:
- *	uint64_t - the region to get clean status on
+ *	__u64 - the region to get clean status on
  * Payload-to-kernel:
- *	int64_t  - 1 if clean, 0 otherwise
+ *	__s64  - 1 if clean, 0 otherwise
  *
- * Payload is sizeof(uint64_t) and contains the region for which the clean
+ * Payload is sizeof(__u64) and contains the region for which the clean
  * status is being made.
  *
  * When the request has been processed, user-space must return the
@@ -187,9 +188,9 @@
  *		  int can_block);
  *
  * Payload-to-userspace:
- *	uint64_t - the region to get sync status on
+ *	__u64 - the region to get sync status on
  * Payload-to-kernel:
- *	int64_t - 1 if in-sync, 0 otherwise
+ *	__s64 - 1 if in-sync, 0 otherwise
  *
  * Exactly the same as 'is_clean' above, except this time asking "has the
  * region been recovered?" vs. "is the region not being modified?"
@@ -203,7 +204,7 @@
  * Payload-to-userspace:
  *	If the 'integrated_flush' directive is present in the constructor
  *	table, the payload is as same as DM_ULOG_MARK_REGION:
- *		uint64_t [] - region(s) to mark
+ *		__u64 [] - region(s) to mark
  *	else
  *		None
  * Payload-to-kernel:
@@ -225,13 +226,13 @@
  * void (*mark_region)(struct dm_dirty_log *log, region_t region);
  *
  * Payload-to-userspace:
- *	uint64_t [] - region(s) to mark
+ *	__u64 [] - region(s) to mark
  * Payload-to-kernel:
  *	None.
  *
  * Incoming payload contains the one or more regions to mark dirty.
  * The number of regions contained in the payload can be determined from
- * 'data_size/sizeof(uint64_t)'.
+ * 'data_size/sizeof(__u64)'.
  *
  * When the request has been processed, user-space must return the
  * dm_ulog_request to the kernel - setting the 'error' field and clearing
@@ -244,13 +245,13 @@
  * void (*clear_region)(struct dm_dirty_log *log, region_t region);
  *
  * Payload-to-userspace:
- *	uint64_t [] - region(s) to clear
+ *	__u64 [] - region(s) to clear
  * Payload-to-kernel:
  *	None.
  *
  * Incoming payload contains the one or more regions to mark clean.
  * The number of regions contained in the payload can be determined from
- * 'data_size/sizeof(uint64_t)'.
+ * 'data_size/sizeof(__u64)'.
  *
  * When the request has been processed, user-space must return the
  * dm_ulog_request to the kernel - setting the 'error' field and clearing
@@ -266,8 +267,8 @@
  *	None.
  * Payload-to-kernel:
  *	{
- *		int64_t i; -- 1 if recovery necessary, 0 otherwise
- *		uint64_t r; -- The region to recover if i=1
+ *		__s64 i; -- 1 if recovery necessary, 0 otherwise
+ *		__u64 r; -- The region to recover if i=1
  *	}
  * 'data_size' should be set appropriately.
  *
@@ -283,8 +284,8 @@
  *
  * Payload-to-userspace:
  *	{
- *		uint64_t - region to set sync state on
- *		int64_t  - 0 if not-in-sync, 1 if in-sync
+ *		__u64 - region to set sync state on
+ *		__s64  - 0 if not-in-sync, 1 if in-sync
  *	}
  * Payload-to-kernel:
  *	None.
@@ -302,7 +303,7 @@
  * Payload-to-userspace:
  *	None.
  * Payload-to-kernel:
- *	uint64_t - the number of in-sync regions
+ *	__u64 - the number of in-sync regions
  *
  * No incoming payload.  Kernel-bound payload contains the number of
  * regions that are in-sync (in a size_t).
@@ -350,11 +351,11 @@
  * int (*is_remote_recovering)(struct dm_dirty_log *log, region_t region);
  *
  * Payload-to-userspace:
- *	uint64_t - region to determine recovery status on
+ *	__u64 - region to determine recovery status on
  * Payload-to-kernel:
  *	{
- *		int64_t is_recovering;  -- 0 if no, 1 if yes
- *		uint64_t in_sync_hint;  -- lowest region still needing resync
+ *		__s64 is_recovering;  -- 0 if no, 1 if yes
+ *		__u64 in_sync_hint;  -- lowest region still needing resync
  *	}
  *
  * When the request has been processed, user-space must return the
@@ -413,16 +414,16 @@ struct dm_ulog_request {
 	 * differentiate between logs that are being swapped and have the
 	 * same 'uuid'.  (Think "live" and "inactive" device-mapper tables.)
 	 */
-	uint64_t luid;
+	__u64 luid;
 	char uuid[DM_UUID_LEN];
 	char padding[3];        /* Padding because DM_UUID_LEN = 129 */
 
-	uint32_t version;       /* See DM_ULOG_REQUEST_VERSION */
-	int32_t error;          /* Used to report back processing errors */
+	__u32 version;       /* See DM_ULOG_REQUEST_VERSION */
+	__s32 error;          /* Used to report back processing errors */
 
-	uint32_t seq;           /* Sequence number for request */
-	uint32_t request_type;  /* DM_ULOG_* defined above */
-	uint32_t data_size;     /* How much data (not including this struct) */
+	__u32 seq;           /* Sequence number for request */
+	__u32 request_type;  /* DM_ULOG_* defined above */
+	__u32 data_size;     /* How much data (not including this struct) */
 
 	char data[0];
 };
diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h
index acb2b6152ba0..36da93fbf188 100644
--- a/include/uapi/linux/fs.h
+++ b/include/uapi/linux/fs.h
@@ -225,6 +225,10 @@ struct fsxattr {
 #define BLKSECDISCARD _IO(0x12,125)
 #define BLKROTATIONAL _IO(0x12,126)
 #define BLKZEROOUT _IO(0x12,127)
+/*
+ * A jump here: 130-131 are reserved for zoned block devices
+ * (see uapi/linux/blkzoned.h)
+ */
 
 #define BMAP_IOCTL 1		/* obsolete - kept for compatibility */
 #define FIBMAP	   _IO(0x00,1)	/* bmap access */
@@ -254,6 +258,20 @@ struct fsxattr {
 /* Policy provided via an ioctl on the topmost directory */
 #define FS_KEY_DESCRIPTOR_SIZE	8
 
+#define FS_POLICY_FLAGS_PAD_4		0x00
+#define FS_POLICY_FLAGS_PAD_8		0x01
+#define FS_POLICY_FLAGS_PAD_16		0x02
+#define FS_POLICY_FLAGS_PAD_32		0x03
+#define FS_POLICY_FLAGS_PAD_MASK	0x03
+#define FS_POLICY_FLAGS_VALID		0x03
+
+/* Encryption algorithms */
+#define FS_ENCRYPTION_MODE_INVALID		0
+#define FS_ENCRYPTION_MODE_AES_256_XTS		1
+#define FS_ENCRYPTION_MODE_AES_256_GCM		2
+#define FS_ENCRYPTION_MODE_AES_256_CBC		3
+#define FS_ENCRYPTION_MODE_AES_256_CTS		4
+
 struct fscrypt_policy {
 	__u8 version;
 	__u8 contents_encryption_mode;
diff --git a/include/uapi/linux/hw_breakpoint.h b/include/uapi/linux/hw_breakpoint.h
index b04000a2296a..2b65efd19a46 100644
--- a/include/uapi/linux/hw_breakpoint.h
+++ b/include/uapi/linux/hw_breakpoint.h
@@ -4,7 +4,11 @@
 enum {
 	HW_BREAKPOINT_LEN_1 = 1,
 	HW_BREAKPOINT_LEN_2 = 2,
+	HW_BREAKPOINT_LEN_3 = 3,
 	HW_BREAKPOINT_LEN_4 = 4,
+	HW_BREAKPOINT_LEN_5 = 5,
+	HW_BREAKPOINT_LEN_6 = 6,
+	HW_BREAKPOINT_LEN_7 = 7,
 	HW_BREAKPOINT_LEN_8 = 8,
 };
 
diff --git a/include/uapi/linux/iio/types.h b/include/uapi/linux/iio/types.h
index 22e5e589a274..e54d14a7f876 100644
--- a/include/uapi/linux/iio/types.h
+++ b/include/uapi/linux/iio/types.h
@@ -40,6 +40,8 @@ enum iio_chan_type {
 	IIO_PH,
 	IIO_UVINDEX,
 	IIO_ELECTRICALCONDUCTIVITY,
+	IIO_COUNT,
+	IIO_INDEX,
 };
 
 enum iio_modifier {
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 4ee67cb99143..cac48eda1075 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -651,6 +651,9 @@ struct kvm_enable_cap {
 };
 
 /* for KVM_PPC_GET_PVINFO */
+
+#define KVM_PPC_PVINFO_FLAGS_EV_IDLE   (1<<0)
+
 struct kvm_ppc_pvinfo {
 	/* out */
 	__u32 flags;
@@ -682,8 +685,6 @@ struct kvm_ppc_smmu_info {
 	struct kvm_ppc_one_seg_page_size sps[KVM_PPC_PAGE_SIZES_MAX_SZ];
 };
 
-#define KVM_PPC_PVINFO_FLAGS_EV_IDLE   (1<<0)
-
 #define KVMIO 0xAE
 
 /* machine type bits, to be used as argument to KVM_CREATE_VM */
diff --git a/include/uapi/linux/mmc/ioctl.h b/include/uapi/linux/mmc/ioctl.h
index 7e385b83b9d8..700a55156eee 100644
--- a/include/uapi/linux/mmc/ioctl.h
+++ b/include/uapi/linux/mmc/ioctl.h
@@ -69,6 +69,6 @@ struct mmc_ioc_multi_cmd {
  * is enforced per ioctl call.  For larger data transfers, use the normal
  * block device operations.
  */
-#define MMC_IOC_MAX_BYTES  (512L * 256)
+#define MMC_IOC_MAX_BYTES  (512L * 1024)
 #define MMC_IOC_MAX_CMDS    255
 #endif /* LINUX_MMC_IOCTL_H */
diff --git a/include/uapi/linux/nbd.h b/include/uapi/linux/nbd.h
index e08e413d5f71..c91c642ea900 100644
--- a/include/uapi/linux/nbd.h
+++ b/include/uapi/linux/nbd.h
@@ -38,11 +38,12 @@ enum {
 };
 
 /* values for flags field */
-#define NBD_FLAG_HAS_FLAGS    (1 << 0) /* nbd-server supports flags */
-#define NBD_FLAG_READ_ONLY    (1 << 1) /* device is read-only */
-#define NBD_FLAG_SEND_FLUSH   (1 << 2) /* can flush writeback cache */
+#define NBD_FLAG_HAS_FLAGS	(1 << 0) /* nbd-server supports flags */
+#define NBD_FLAG_READ_ONLY	(1 << 1) /* device is read-only */
+#define NBD_FLAG_SEND_FLUSH	(1 << 2) /* can flush writeback cache */
 /* there is a gap here to match userspace */
-#define NBD_FLAG_SEND_TRIM    (1 << 5) /* send trim/discard */
+#define NBD_FLAG_SEND_TRIM	(1 << 5) /* send trim/discard */
+#define NBD_FLAG_CAN_MULTI_CONN	(1 << 8)	/* Server supports multiple connections per export. */
 
 /* userspace doesn't need the nbd_device structure */
 
diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h
index e5a2e68b2236..174d1147081b 100644
--- a/include/uapi/linux/pci_regs.h
+++ b/include/uapi/linux/pci_regs.h
@@ -23,6 +23,14 @@
 #define LINUX_PCI_REGS_H
 
 /*
+ * Conventional PCI and PCI-X Mode 1 devices have 256 bytes of
+ * configuration space.  PCI-X Mode 2 and PCIe devices have 4096 bytes of
+ * configuration space.
+ */
+#define PCI_CFG_SPACE_SIZE	256
+#define PCI_CFG_SPACE_EXP_SIZE	4096
+
+/*
  * Under PCI, each device has 256 bytes of configuration address space,
  * of which the first 64 bytes are standardized as follows:
  */
diff --git a/include/uapi/linux/raid/md_p.h b/include/uapi/linux/raid/md_p.h
index c3e654c6d518..9930f3e9040f 100644
--- a/include/uapi/linux/raid/md_p.h
+++ b/include/uapi/linux/raid/md_p.h
@@ -84,6 +84,10 @@
 #define MD_DISK_CANDIDATE	5 /* disk is added as spare (local) until confirmed
 				   * For clustered enviroments only.
 				   */
+#define MD_DISK_FAILFAST	10 /* Send REQ_FAILFAST if there are multiple
+				    * devices available - and don't try to
+				    * correct read errors.
+				    */
 
 #define	MD_DISK_WRITEMOSTLY	9 /* disk is "write-mostly" is RAID1 config.
 				   * read requests will only be sent here in
@@ -265,8 +269,9 @@ struct mdp_superblock_1 {
 	__le32	dev_number;	/* permanent identifier of this  device - not role in raid */
 	__le32	cnt_corrected_read; /* number of read errors that were corrected by re-writing */
 	__u8	device_uuid[16]; /* user-space setable, ignored by kernel */
-	__u8	devflags;	/* per-device flags.  Only one defined...*/
+	__u8	devflags;	/* per-device flags.  Only two defined...*/
 #define	WriteMostly1	1	/* mask for writemostly flag in above */
+#define	FailFast1	2	/* Should avoid retries and fixups and just fail */
 	/* Bad block log.  If there are any bad blocks the feature flag is set.
 	 * If offset and size are non-zero, that space is reserved and available
 	 */
diff --git a/include/uapi/linux/types.h b/include/uapi/linux/types.h
index acf0979b790a..41e5914f0a8e 100644
--- a/include/uapi/linux/types.h
+++ b/include/uapi/linux/types.h
@@ -23,11 +23,7 @@
 #else
 #define __bitwise__
 #endif
-#ifdef __CHECK_ENDIAN__
 #define __bitwise __bitwise__
-#else
-#define __bitwise
-#endif
 
 typedef __u16 __bitwise __le16;
 typedef __u16 __bitwise __be16;
diff --git a/include/uapi/linux/uleds.h b/include/uapi/linux/uleds.h
new file mode 100644
index 000000000000..95186578c46e
--- /dev/null
+++ b/include/uapi/linux/uleds.h
@@ -0,0 +1,24 @@
+/*
+ * Userspace driver support for the LED subsystem
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef _UAPI__ULEDS_H_
+#define _UAPI__ULEDS_H_
+
+#define LED_MAX_NAME_SIZE	64
+
+struct uleds_user_dev {
+	char name[LED_MAX_NAME_SIZE];
+	int max_brightness;
+};
+
+#endif /* _UAPI__ULEDS_H_ */
diff --git a/include/uapi/linux/usb/ch9.h b/include/uapi/linux/usb/ch9.h
index a8acc24765fe..2c5d7c4a69e3 100644
--- a/include/uapi/linux/usb/ch9.h
+++ b/include/uapi/linux/usb/ch9.h
@@ -423,6 +423,12 @@ struct usb_endpoint_descriptor {
 #define USB_ENDPOINT_XFER_INT		3
 #define USB_ENDPOINT_MAX_ADJUSTABLE	0x80
 
+#define USB_ENDPOINT_MAXP_MASK	0x07ff
+#define USB_EP_MAXP_MULT_SHIFT	11
+#define USB_EP_MAXP_MULT_MASK	(3 << USB_EP_MAXP_MULT_SHIFT)
+#define USB_EP_MAXP_MULT(m) \
+	(((m) & USB_EP_MAXP_MULT_MASK) >> USB_EP_MAXP_MULT_SHIFT)
+
 /* The USB 3.0 spec redefines bits 5:4 of bmAttributes as interrupt ep type. */
 #define USB_ENDPOINT_INTRTYPE		0x30
 #define USB_ENDPOINT_INTR_PERIODIC	(0 << 4)
@@ -623,11 +629,25 @@ static inline int usb_endpoint_is_isoc_out(
  * usb_endpoint_maxp - get endpoint's max packet size
  * @epd: endpoint to be checked
  *
- * Returns @epd's max packet
+ * Returns @epd's max packet bits [10:0]
  */
 static inline int usb_endpoint_maxp(const struct usb_endpoint_descriptor *epd)
 {
-	return __le16_to_cpu(epd->wMaxPacketSize);
+	return __le16_to_cpu(epd->wMaxPacketSize) & USB_ENDPOINT_MAXP_MASK;
+}
+
+/**
+ * usb_endpoint_maxp_mult - get endpoint's transactional opportunities
+ * @epd: endpoint to be checked
+ *
+ * Return @epd's wMaxPacketSize[12:11] + 1
+ */
+static inline int
+usb_endpoint_maxp_mult(const struct usb_endpoint_descriptor *epd)
+{
+	int maxp = __le16_to_cpu(epd->wMaxPacketSize);
+
+	return USB_EP_MAXP_MULT(maxp) + 1;
 }
 
 static inline int usb_endpoint_interrupt_type(
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index 255a2113f53c..519eff362c1c 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -203,6 +203,16 @@ struct vfio_device_info {
 };
 #define VFIO_DEVICE_GET_INFO		_IO(VFIO_TYPE, VFIO_BASE + 7)
 
+/*
+ * Vendor driver using Mediated device framework should provide device_api
+ * attribute in supported type attribute groups. Device API string should be one
+ * of the following corresponding to device flags in vfio_device_info structure.
+ */
+
+#define VFIO_DEVICE_API_PCI_STRING		"vfio-pci"
+#define VFIO_DEVICE_API_PLATFORM_STRING		"vfio-platform"
+#define VFIO_DEVICE_API_AMBA_STRING		"vfio-amba"
+
 /**
  * VFIO_DEVICE_GET_REGION_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 8,
  *				       struct vfio_region_info)
diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
index 56b7ab584cc0..60180c0b5dc6 100644
--- a/include/uapi/linux/vhost.h
+++ b/include/uapi/linux/vhost.h
@@ -172,8 +172,6 @@ struct vhost_memory {
 #define VHOST_F_LOG_ALL 26
 /* vhost-net should add virtio_net_hdr for RX, and strip for TX packets. */
 #define VHOST_NET_F_VIRTIO_NET_HDR 27
-/* Vhost have device IOTLB */
-#define VHOST_F_DEVICE_IOTLB 63
 
 /* VHOST_SCSI specific definitions */
 
diff --git a/include/uapi/linux/virtio_crypto.h b/include/uapi/linux/virtio_crypto.h
new file mode 100644
index 000000000000..50cdc8aebfcf
--- /dev/null
+++ b/include/uapi/linux/virtio_crypto.h
@@ -0,0 +1,450 @@
+#ifndef _VIRTIO_CRYPTO_H
+#define _VIRTIO_CRYPTO_H
+/* This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include <linux/types.h>
+#include <linux/virtio_types.h>
+#include <linux/virtio_ids.h>
+#include <linux/virtio_config.h>
+
+
+#define VIRTIO_CRYPTO_SERVICE_CIPHER 0
+#define VIRTIO_CRYPTO_SERVICE_HASH   1
+#define VIRTIO_CRYPTO_SERVICE_MAC    2
+#define VIRTIO_CRYPTO_SERVICE_AEAD   3
+
+#define VIRTIO_CRYPTO_OPCODE(service, op)   (((service) << 8) | (op))
+
+struct virtio_crypto_ctrl_header {
+#define VIRTIO_CRYPTO_CIPHER_CREATE_SESSION \
+	   VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_CIPHER, 0x02)
+#define VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION \
+	   VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_CIPHER, 0x03)
+#define VIRTIO_CRYPTO_HASH_CREATE_SESSION \
+	   VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_HASH, 0x02)
+#define VIRTIO_CRYPTO_HASH_DESTROY_SESSION \
+	   VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_HASH, 0x03)
+#define VIRTIO_CRYPTO_MAC_CREATE_SESSION \
+	   VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_MAC, 0x02)
+#define VIRTIO_CRYPTO_MAC_DESTROY_SESSION \
+	   VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_MAC, 0x03)
+#define VIRTIO_CRYPTO_AEAD_CREATE_SESSION \
+	   VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x02)
+#define VIRTIO_CRYPTO_AEAD_DESTROY_SESSION \
+	   VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x03)
+	__le32 opcode;
+	__le32 algo;
+	__le32 flag;
+	/* data virtqueue id */
+	__le32 queue_id;
+};
+
+struct virtio_crypto_cipher_session_para {
+#define VIRTIO_CRYPTO_NO_CIPHER                 0
+#define VIRTIO_CRYPTO_CIPHER_ARC4               1
+#define VIRTIO_CRYPTO_CIPHER_AES_ECB            2
+#define VIRTIO_CRYPTO_CIPHER_AES_CBC            3
+#define VIRTIO_CRYPTO_CIPHER_AES_CTR            4
+#define VIRTIO_CRYPTO_CIPHER_DES_ECB            5
+#define VIRTIO_CRYPTO_CIPHER_DES_CBC            6
+#define VIRTIO_CRYPTO_CIPHER_3DES_ECB           7
+#define VIRTIO_CRYPTO_CIPHER_3DES_CBC           8
+#define VIRTIO_CRYPTO_CIPHER_3DES_CTR           9
+#define VIRTIO_CRYPTO_CIPHER_KASUMI_F8          10
+#define VIRTIO_CRYPTO_CIPHER_SNOW3G_UEA2        11
+#define VIRTIO_CRYPTO_CIPHER_AES_F8             12
+#define VIRTIO_CRYPTO_CIPHER_AES_XTS            13
+#define VIRTIO_CRYPTO_CIPHER_ZUC_EEA3           14
+	__le32 algo;
+	/* length of key */
+	__le32 keylen;
+
+#define VIRTIO_CRYPTO_OP_ENCRYPT  1
+#define VIRTIO_CRYPTO_OP_DECRYPT  2
+	/* encrypt or decrypt */
+	__le32 op;
+	__le32 padding;
+};
+
+struct virtio_crypto_session_input {
+	/* Device-writable part */
+	__le64 session_id;
+	__le32 status;
+	__le32 padding;
+};
+
+struct virtio_crypto_cipher_session_req {
+	struct virtio_crypto_cipher_session_para para;
+	__u8 padding[32];
+};
+
+struct virtio_crypto_hash_session_para {
+#define VIRTIO_CRYPTO_NO_HASH            0
+#define VIRTIO_CRYPTO_HASH_MD5           1
+#define VIRTIO_CRYPTO_HASH_SHA1          2
+#define VIRTIO_CRYPTO_HASH_SHA_224       3
+#define VIRTIO_CRYPTO_HASH_SHA_256       4
+#define VIRTIO_CRYPTO_HASH_SHA_384       5
+#define VIRTIO_CRYPTO_HASH_SHA_512       6
+#define VIRTIO_CRYPTO_HASH_SHA3_224      7
+#define VIRTIO_CRYPTO_HASH_SHA3_256      8
+#define VIRTIO_CRYPTO_HASH_SHA3_384      9
+#define VIRTIO_CRYPTO_HASH_SHA3_512      10
+#define VIRTIO_CRYPTO_HASH_SHA3_SHAKE128      11
+#define VIRTIO_CRYPTO_HASH_SHA3_SHAKE256      12
+	__le32 algo;
+	/* hash result length */
+	__le32 hash_result_len;
+	__u8 padding[8];
+};
+
+struct virtio_crypto_hash_create_session_req {
+	struct virtio_crypto_hash_session_para para;
+	__u8 padding[40];
+};
+
+struct virtio_crypto_mac_session_para {
+#define VIRTIO_CRYPTO_NO_MAC                       0
+#define VIRTIO_CRYPTO_MAC_HMAC_MD5                 1
+#define VIRTIO_CRYPTO_MAC_HMAC_SHA1                2
+#define VIRTIO_CRYPTO_MAC_HMAC_SHA_224             3
+#define VIRTIO_CRYPTO_MAC_HMAC_SHA_256             4
+#define VIRTIO_CRYPTO_MAC_HMAC_SHA_384             5
+#define VIRTIO_CRYPTO_MAC_HMAC_SHA_512             6
+#define VIRTIO_CRYPTO_MAC_CMAC_3DES                25
+#define VIRTIO_CRYPTO_MAC_CMAC_AES                 26
+#define VIRTIO_CRYPTO_MAC_KASUMI_F9                27
+#define VIRTIO_CRYPTO_MAC_SNOW3G_UIA2              28
+#define VIRTIO_CRYPTO_MAC_GMAC_AES                 41
+#define VIRTIO_CRYPTO_MAC_GMAC_TWOFISH             42
+#define VIRTIO_CRYPTO_MAC_CBCMAC_AES               49
+#define VIRTIO_CRYPTO_MAC_CBCMAC_KASUMI_F9         50
+#define VIRTIO_CRYPTO_MAC_XCBC_AES                 53
+	__le32 algo;
+	/* hash result length */
+	__le32 hash_result_len;
+	/* length of authenticated key */
+	__le32 auth_key_len;
+	__le32 padding;
+};
+
+struct virtio_crypto_mac_create_session_req {
+	struct virtio_crypto_mac_session_para para;
+	__u8 padding[40];
+};
+
+struct virtio_crypto_aead_session_para {
+#define VIRTIO_CRYPTO_NO_AEAD     0
+#define VIRTIO_CRYPTO_AEAD_GCM    1
+#define VIRTIO_CRYPTO_AEAD_CCM    2
+#define VIRTIO_CRYPTO_AEAD_CHACHA20_POLY1305  3
+	__le32 algo;
+	/* length of key */
+	__le32 key_len;
+	/* hash result length */
+	__le32 hash_result_len;
+	/* length of the additional authenticated data (AAD) in bytes */
+	__le32 aad_len;
+	/* encrypt or decrypt, See above VIRTIO_CRYPTO_OP_* */
+	__le32 op;
+	__le32 padding;
+};
+
+struct virtio_crypto_aead_create_session_req {
+	struct virtio_crypto_aead_session_para para;
+	__u8 padding[32];
+};
+
+struct virtio_crypto_alg_chain_session_para {
+#define VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER  1
+#define VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH  2
+	__le32 alg_chain_order;
+/* Plain hash */
+#define VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN    1
+/* Authenticated hash (mac) */
+#define VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH     2
+/* Nested hash */
+#define VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED   3
+	__le32 hash_mode;
+	struct virtio_crypto_cipher_session_para cipher_param;
+	union {
+		struct virtio_crypto_hash_session_para hash_param;
+		struct virtio_crypto_mac_session_para mac_param;
+		__u8 padding[16];
+	} u;
+	/* length of the additional authenticated data (AAD) in bytes */
+	__le32 aad_len;
+	__le32 padding;
+};
+
+struct virtio_crypto_alg_chain_session_req {
+	struct virtio_crypto_alg_chain_session_para para;
+};
+
+struct virtio_crypto_sym_create_session_req {
+	union {
+		struct virtio_crypto_cipher_session_req cipher;
+		struct virtio_crypto_alg_chain_session_req chain;
+		__u8 padding[48];
+	} u;
+
+	/* Device-readable part */
+
+/* No operation */
+#define VIRTIO_CRYPTO_SYM_OP_NONE  0
+/* Cipher only operation on the data */
+#define VIRTIO_CRYPTO_SYM_OP_CIPHER  1
+/*
+ * Chain any cipher with any hash or mac operation. The order
+ * depends on the value of alg_chain_order param
+ */
+#define VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING  2
+	__le32 op_type;
+	__le32 padding;
+};
+
+struct virtio_crypto_destroy_session_req {
+	/* Device-readable part */
+	__le64  session_id;
+	__u8 padding[48];
+};
+
+/* The request of the control virtqueue's packet */
+struct virtio_crypto_op_ctrl_req {
+	struct virtio_crypto_ctrl_header header;
+
+	union {
+		struct virtio_crypto_sym_create_session_req
+			sym_create_session;
+		struct virtio_crypto_hash_create_session_req
+			hash_create_session;
+		struct virtio_crypto_mac_create_session_req
+			mac_create_session;
+		struct virtio_crypto_aead_create_session_req
+			aead_create_session;
+		struct virtio_crypto_destroy_session_req
+			destroy_session;
+		__u8 padding[56];
+	} u;
+};
+
+struct virtio_crypto_op_header {
+#define VIRTIO_CRYPTO_CIPHER_ENCRYPT \
+	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_CIPHER, 0x00)
+#define VIRTIO_CRYPTO_CIPHER_DECRYPT \
+	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_CIPHER, 0x01)
+#define VIRTIO_CRYPTO_HASH \
+	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_HASH, 0x00)
+#define VIRTIO_CRYPTO_MAC \
+	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_MAC, 0x00)
+#define VIRTIO_CRYPTO_AEAD_ENCRYPT \
+	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x00)
+#define VIRTIO_CRYPTO_AEAD_DECRYPT \
+	VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x01)
+	__le32 opcode;
+	/* algo should be service-specific algorithms */
+	__le32 algo;
+	/* session_id should be service-specific algorithms */
+	__le64 session_id;
+	/* control flag to control the request */
+	__le32 flag;
+	__le32 padding;
+};
+
+struct virtio_crypto_cipher_para {
+	/*
+	 * Byte Length of valid IV/Counter
+	 *
+	 * For block ciphers in CBC or F8 mode, or for Kasumi in F8 mode, or for
+	 *   SNOW3G in UEA2 mode, this is the length of the IV (which
+	 *   must be the same as the block length of the cipher).
+	 * For block ciphers in CTR mode, this is the length of the counter
+	 *   (which must be the same as the block length of the cipher).
+	 * For AES-XTS, this is the 128bit tweak, i, from IEEE Std 1619-2007.
+	 *
+	 * The IV/Counter will be updated after every partial cryptographic
+	 * operation.
+	 */
+	__le32 iv_len;
+	/* length of source data */
+	__le32 src_data_len;
+	/* length of dst data */
+	__le32 dst_data_len;
+	__le32 padding;
+};
+
+struct virtio_crypto_hash_para {
+	/* length of source data */
+	__le32 src_data_len;
+	/* hash result length */
+	__le32 hash_result_len;
+};
+
+struct virtio_crypto_mac_para {
+	struct virtio_crypto_hash_para hash;
+};
+
+struct virtio_crypto_aead_para {
+	/*
+	 * Byte Length of valid IV data pointed to by the below iv_addr
+	 * parameter.
+	 *
+	 * For GCM mode, this is either 12 (for 96-bit IVs) or 16, in which
+	 *   case iv_addr points to J0.
+	 * For CCM mode, this is the length of the nonce, which can be in the
+	 *   range 7 to 13 inclusive.
+	 */
+	__le32 iv_len;
+	/* length of additional auth data */
+	__le32 aad_len;
+	/* length of source data */
+	__le32 src_data_len;
+	/* length of dst data */
+	__le32 dst_data_len;
+};
+
+struct virtio_crypto_cipher_data_req {
+	/* Device-readable part */
+	struct virtio_crypto_cipher_para para;
+	__u8 padding[24];
+};
+
+struct virtio_crypto_hash_data_req {
+	/* Device-readable part */
+	struct virtio_crypto_hash_para para;
+	__u8 padding[40];
+};
+
+struct virtio_crypto_mac_data_req {
+	/* Device-readable part */
+	struct virtio_crypto_mac_para para;
+	__u8 padding[40];
+};
+
+struct virtio_crypto_alg_chain_data_para {
+	__le32 iv_len;
+	/* Length of source data */
+	__le32 src_data_len;
+	/* Length of destination data */
+	__le32 dst_data_len;
+	/* Starting point for cipher processing in source data */
+	__le32 cipher_start_src_offset;
+	/* Length of the source data that the cipher will be computed on */
+	__le32 len_to_cipher;
+	/* Starting point for hash processing in source data */
+	__le32 hash_start_src_offset;
+	/* Length of the source data that the hash will be computed on */
+	__le32 len_to_hash;
+	/* Length of the additional auth data */
+	__le32 aad_len;
+	/* Length of the hash result */
+	__le32 hash_result_len;
+	__le32 reserved;
+};
+
+struct virtio_crypto_alg_chain_data_req {
+	/* Device-readable part */
+	struct virtio_crypto_alg_chain_data_para para;
+};
+
+struct virtio_crypto_sym_data_req {
+	union {
+		struct virtio_crypto_cipher_data_req cipher;
+		struct virtio_crypto_alg_chain_data_req chain;
+		__u8 padding[40];
+	} u;
+
+	/* See above VIRTIO_CRYPTO_SYM_OP_* */
+	__le32 op_type;
+	__le32 padding;
+};
+
+struct virtio_crypto_aead_data_req {
+	/* Device-readable part */
+	struct virtio_crypto_aead_para para;
+	__u8 padding[32];
+};
+
+/* The request of the data virtqueue's packet */
+struct virtio_crypto_op_data_req {
+	struct virtio_crypto_op_header header;
+
+	union {
+		struct virtio_crypto_sym_data_req  sym_req;
+		struct virtio_crypto_hash_data_req hash_req;
+		struct virtio_crypto_mac_data_req mac_req;
+		struct virtio_crypto_aead_data_req aead_req;
+		__u8 padding[48];
+	} u;
+};
+
+#define VIRTIO_CRYPTO_OK        0
+#define VIRTIO_CRYPTO_ERR       1
+#define VIRTIO_CRYPTO_BADMSG    2
+#define VIRTIO_CRYPTO_NOTSUPP   3
+#define VIRTIO_CRYPTO_INVSESS   4 /* Invalid session id */
+
+/* The accelerator hardware is ready */
+#define VIRTIO_CRYPTO_S_HW_READY  (1 << 0)
+
+struct virtio_crypto_config {
+	/* See VIRTIO_CRYPTO_OP_* above */
+	__u32  status;
+
+	/*
+	 * Maximum number of data queue
+	 */
+	__u32  max_dataqueues;
+
+	/*
+	 * Specifies the services mask which the device support,
+	 * see VIRTIO_CRYPTO_SERVICE_* above
+	 */
+	__u32 crypto_services;
+
+	/* Detailed algorithms mask */
+	__u32 cipher_algo_l;
+	__u32 cipher_algo_h;
+	__u32 hash_algo;
+	__u32 mac_algo_l;
+	__u32 mac_algo_h;
+	__u32 aead_algo;
+	/* Maximum length of cipher key */
+	__u32 max_cipher_key_len;
+	/* Maximum length of authenticated key */
+	__u32 max_auth_key_len;
+	__u32 reserve;
+	/* Maximum size of each crypto request's content */
+	__u64 max_size;
+};
+
+struct virtio_crypto_inhdr {
+	/* See VIRTIO_CRYPTO_* above */
+	__u8 status;
+};
+#endif
diff --git a/include/uapi/linux/virtio_ids.h b/include/uapi/linux/virtio_ids.h
index 3228d582234a..6d5c3b2d4f4d 100644
--- a/include/uapi/linux/virtio_ids.h
+++ b/include/uapi/linux/virtio_ids.h
@@ -42,5 +42,6 @@
 #define VIRTIO_ID_GPU          16 /* virtio GPU */
 #define VIRTIO_ID_INPUT        18 /* virtio input */
 #define VIRTIO_ID_VSOCK        19 /* virtio vsock transport */
+#define VIRTIO_ID_CRYPTO       20 /* virtio crypto */
 
 #endif /* _LINUX_VIRTIO_IDS_H */
diff --git a/include/uapi/linux/virtio_types.h b/include/uapi/linux/virtio_types.h
index e845e8c4cbee..55c3b738722c 100644
--- a/include/uapi/linux/virtio_types.h
+++ b/include/uapi/linux/virtio_types.h
@@ -39,8 +39,8 @@
  * - __le{16,32,64} for standard-compliant virtio devices
  */
 
-typedef __u16 __bitwise__ __virtio16;
-typedef __u32 __bitwise__ __virtio32;
-typedef __u64 __bitwise__ __virtio64;
+typedef __u16 __bitwise __virtio16;
+typedef __u32 __bitwise __virtio32;
+typedef __u64 __bitwise __virtio64;
 
 #endif /* _UAPI_LINUX_VIRTIO_TYPES_H */
diff --git a/include/uapi/linux/vtpm_proxy.h b/include/uapi/linux/vtpm_proxy.h
index 41e8e2252a30..a69e991eb080 100644
--- a/include/uapi/linux/vtpm_proxy.h
+++ b/include/uapi/linux/vtpm_proxy.h
@@ -1,6 +1,7 @@
 /*
  * Definitions for the VTPM proxy driver
  * Copyright (c) 2015, 2016, IBM Corporation
+ * Copyright (C) 2016 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -18,8 +19,23 @@
 #include <linux/types.h>
 #include <linux/ioctl.h>
 
-/* ioctls */
+/**
+ * enum vtpm_proxy_flags - flags for the proxy TPM
+ * @VTPM_PROXY_FLAG_TPM2:	the proxy TPM uses TPM 2.0 protocol
+ */
+enum vtpm_proxy_flags {
+	VTPM_PROXY_FLAG_TPM2	= 1,
+};
 
+/**
+ * struct vtpm_proxy_new_dev - parameter structure for the
+ *                             %VTPM_PROXY_IOC_NEW_DEV ioctl
+ * @flags:	flags for the proxy TPM
+ * @tpm_num:	index of the TPM device
+ * @fd:		the file descriptor used by the proxy TPM
+ * @major:	the major number of the TPM device
+ * @minor:	the minor number of the TPM device
+ */
 struct vtpm_proxy_new_dev {
 	__u32 flags;         /* input */
 	__u32 tpm_num;       /* output */
@@ -28,9 +44,6 @@ struct vtpm_proxy_new_dev {
 	__u32 minor;         /* output */
 };
 
-/* above flags */
-#define VTPM_PROXY_FLAG_TPM2  1  /* emulator is TPM 2 */
-
-#define VTPM_PROXY_IOC_NEW_DEV   _IOWR(0xa1, 0x00, struct vtpm_proxy_new_dev)
+#define VTPM_PROXY_IOC_NEW_DEV	_IOWR(0xa1, 0x00, struct vtpm_proxy_new_dev)
 
 #endif /* _UAPI_LINUX_VTPM_PROXY_H */
diff --git a/include/uapi/rdma/Kbuild b/include/uapi/rdma/Kbuild
index f14ab7ff5fee..82bdf5626859 100644
--- a/include/uapi/rdma/Kbuild
+++ b/include/uapi/rdma/Kbuild
@@ -14,3 +14,5 @@ header-y += mlx5-abi.h
 header-y += mthca-abi.h
 header-y += nes-abi.h
 header-y += ocrdma-abi.h
+header-y += hns-abi.h
+header-y += vmw_pvrdma-abi.h
diff --git a/include/uapi/rdma/hfi/hfi1_user.h b/include/uapi/rdma/hfi/hfi1_user.h
index d15e7289d835..587b7360e820 100644
--- a/include/uapi/rdma/hfi/hfi1_user.h
+++ b/include/uapi/rdma/hfi/hfi1_user.h
@@ -75,7 +75,7 @@
  * may not be implemented; the user code must deal with this if it
  * cares, or it must abort after initialization reports the difference.
  */
-#define HFI1_USER_SWMINOR 2
+#define HFI1_USER_SWMINOR 3
 
 /*
  * We will encode the major/minor inside a single 32bit version number.
diff --git a/drivers/infiniband/hw/hns/hns_roce_user.h b/include/uapi/rdma/hns-abi.h
index a28f761a9f65..5d7401963e35 100644
--- a/drivers/infiniband/hw/hns/hns_roce_user.h
+++ b/include/uapi/rdma/hns-abi.h
@@ -30,8 +30,10 @@
  * SOFTWARE.
  */
 
-#ifndef _HNS_ROCE_USER_H
-#define _HNS_ROCE_USER_H
+#ifndef HNS_ABI_USER_H
+#define HNS_ABI_USER_H
+
+#include <linux/types.h>
 
 struct hns_roce_ib_create_cq {
 	__u64   buf_addr;
@@ -49,5 +51,4 @@ struct hns_roce_ib_create_qp {
 struct hns_roce_ib_alloc_ucontext_resp {
 	__u32	qp_tab_size;
 };
-
-#endif /*_HNS_ROCE_USER_H */
+#endif /* HNS_ABI_USER_H */
diff --git a/include/uapi/rdma/ib_user_verbs.h b/include/uapi/rdma/ib_user_verbs.h
index 25225ebbc7d5..dfdfe4e92d31 100644
--- a/include/uapi/rdma/ib_user_verbs.h
+++ b/include/uapi/rdma/ib_user_verbs.h
@@ -37,6 +37,7 @@
 #define IB_USER_VERBS_H
 
 #include <linux/types.h>
+#include <rdma/ib_verbs.h>
 
 /*
  * Increment this value if any changes that break userspace ABI
@@ -93,6 +94,7 @@ enum {
 	IB_USER_VERBS_EX_CMD_QUERY_DEVICE = IB_USER_VERBS_CMD_QUERY_DEVICE,
 	IB_USER_VERBS_EX_CMD_CREATE_CQ = IB_USER_VERBS_CMD_CREATE_CQ,
 	IB_USER_VERBS_EX_CMD_CREATE_QP = IB_USER_VERBS_CMD_CREATE_QP,
+	IB_USER_VERBS_EX_CMD_MODIFY_QP = IB_USER_VERBS_CMD_MODIFY_QP,
 	IB_USER_VERBS_EX_CMD_CREATE_FLOW = IB_USER_VERBS_CMD_THRESHOLD,
 	IB_USER_VERBS_EX_CMD_DESTROY_FLOW,
 	IB_USER_VERBS_EX_CMD_CREATE_WQ,
@@ -545,6 +547,14 @@ enum {
 	IB_UVERBS_CREATE_QP_SUP_COMP_MASK = IB_UVERBS_CREATE_QP_MASK_IND_TABLE,
 };
 
+enum {
+	IB_USER_LEGACY_LAST_QP_ATTR_MASK = IB_QP_DEST_QPN
+};
+
+enum {
+	IB_USER_LAST_QP_ATTR_MASK = IB_QP_RATE_LIMIT
+};
+
 struct ib_uverbs_ex_create_qp {
 	__u64 user_handle;
 	__u32 pd_handle;
@@ -684,9 +694,20 @@ struct ib_uverbs_modify_qp {
 	__u64 driver_data[0];
 };
 
+struct ib_uverbs_ex_modify_qp {
+	struct ib_uverbs_modify_qp base;
+	__u32	rate_limit;
+	__u32	reserved;
+};
+
 struct ib_uverbs_modify_qp_resp {
 };
 
+struct ib_uverbs_ex_modify_qp_resp {
+	__u32  comp_mask;
+	__u32  response_length;
+};
+
 struct ib_uverbs_destroy_qp {
 	__u64 response;
 	__u32 qp_handle;
@@ -908,6 +929,23 @@ struct ib_uverbs_flow_spec_ipv6 {
 	struct ib_uverbs_flow_ipv6_filter mask;
 };
 
+struct ib_uverbs_flow_tunnel_filter {
+	__be32 tunnel_id;
+};
+
+struct ib_uverbs_flow_spec_tunnel {
+	union {
+		struct ib_uverbs_flow_spec_hdr hdr;
+		struct {
+			__u32 type;
+			__u16 size;
+			__u16 reserved;
+		};
+	};
+	struct ib_uverbs_flow_tunnel_filter val;
+	struct ib_uverbs_flow_tunnel_filter mask;
+};
+
 struct ib_uverbs_flow_attr {
 	__u32 type;
 	__u16 size;
diff --git a/include/uapi/rdma/mlx5-abi.h b/include/uapi/rdma/mlx5-abi.h
index f5d0f4e83b59..fae6cdaeb56d 100644
--- a/include/uapi/rdma/mlx5-abi.h
+++ b/include/uapi/rdma/mlx5-abi.h
@@ -82,6 +82,7 @@ enum mlx5_ib_alloc_ucontext_resp_mask {
 
 enum mlx5_user_cmds_supp_uhw {
 	MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE = 1 << 0,
+	MLX5_USER_CMDS_SUPP_UHW_CREATE_AH    = 1 << 1,
 };
 
 struct mlx5_ib_alloc_ucontext_resp {
@@ -124,18 +125,47 @@ struct mlx5_ib_rss_caps {
 	__u8 reserved[7];
 };
 
+enum mlx5_ib_cqe_comp_res_format {
+	MLX5_IB_CQE_RES_FORMAT_HASH	= 1 << 0,
+	MLX5_IB_CQE_RES_FORMAT_CSUM	= 1 << 1,
+	MLX5_IB_CQE_RES_RESERVED	= 1 << 2,
+};
+
+struct mlx5_ib_cqe_comp_caps {
+	__u32 max_num;
+	__u32 supported_format; /* enum mlx5_ib_cqe_comp_res_format */
+};
+
+struct mlx5_packet_pacing_caps {
+	__u32 qp_rate_limit_min;
+	__u32 qp_rate_limit_max; /* In kpbs */
+
+	/* Corresponding bit will be set if qp type from
+	 * 'enum ib_qp_type' is supported, e.g.
+	 * supported_qpts |= 1 << IB_QPT_RAW_PACKET
+	 */
+	__u32 supported_qpts;
+	__u32 reserved;
+};
+
 struct mlx5_ib_query_device_resp {
 	__u32	comp_mask;
 	__u32	response_length;
 	struct	mlx5_ib_tso_caps tso_caps;
 	struct	mlx5_ib_rss_caps rss_caps;
+	struct	mlx5_ib_cqe_comp_caps cqe_comp_caps;
+	struct	mlx5_packet_pacing_caps packet_pacing_caps;
+	__u32	mlx5_ib_support_multi_pkt_send_wqes;
+	__u32	reserved;
 };
 
 struct mlx5_ib_create_cq {
 	__u64	buf_addr;
 	__u64	db_addr;
 	__u32	cqe_size;
-	__u32	reserved; /* explicit padding (optional on i386) */
+	__u8    cqe_comp_en;
+	__u8    cqe_comp_res_format;
+	__u16	reserved; /* explicit padding (optional on i386) */
 };
 
 struct mlx5_ib_create_cq_resp {
@@ -232,6 +262,12 @@ struct mlx5_ib_create_wq {
 	__u32   reserved;
 };
 
+struct mlx5_ib_create_ah_resp {
+	__u32	response_length;
+	__u8	dmac[ETH_ALEN];
+	__u8	reserved[6];
+};
+
 struct mlx5_ib_create_wq_resp {
 	__u32	response_length;
 	__u32	reserved;
diff --git a/include/uapi/rdma/rdma_user_cm.h b/include/uapi/rdma/rdma_user_cm.h
index 01923d463673..d71da36e3cd6 100644
--- a/include/uapi/rdma/rdma_user_cm.h
+++ b/include/uapi/rdma/rdma_user_cm.h
@@ -110,7 +110,7 @@ struct rdma_ucm_bind {
 	__u32 id;
 	__u16 addr_size;
 	__u16 reserved;
-	struct sockaddr_storage addr;
+	struct __kernel_sockaddr_storage addr;
 };
 
 struct rdma_ucm_resolve_ip {
@@ -126,8 +126,8 @@ struct rdma_ucm_resolve_addr {
 	__u16 src_size;
 	__u16 dst_size;
 	__u32 reserved;
-	struct sockaddr_storage src_addr;
-	struct sockaddr_storage dst_addr;
+	struct __kernel_sockaddr_storage src_addr;
+	struct __kernel_sockaddr_storage dst_addr;
 };
 
 struct rdma_ucm_resolve_route {
@@ -164,8 +164,8 @@ struct rdma_ucm_query_addr_resp {
 	__u16 pkey;
 	__u16 src_size;
 	__u16 dst_size;
-	struct sockaddr_storage src_addr;
-	struct sockaddr_storage dst_addr;
+	struct __kernel_sockaddr_storage src_addr;
+	struct __kernel_sockaddr_storage dst_addr;
 };
 
 struct rdma_ucm_query_path_resp {
@@ -257,7 +257,7 @@ struct rdma_ucm_join_mcast {
 	__u32 id;
 	__u16 addr_size;
 	__u16 join_flags;
-	struct sockaddr_storage addr;
+	struct __kernel_sockaddr_storage addr;
 };
 
 struct rdma_ucm_get_event {
diff --git a/include/uapi/rdma/vmw_pvrdma-abi.h b/include/uapi/rdma/vmw_pvrdma-abi.h
new file mode 100644
index 000000000000..5016abc9ee97
--- /dev/null
+++ b/include/uapi/rdma/vmw_pvrdma-abi.h
@@ -0,0 +1,289 @@
+/*
+ * Copyright (c) 2012-2016 VMware, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of EITHER the GNU General Public License
+ * version 2 as published by the Free Software Foundation or the BSD
+ * 2-Clause License. This program is distributed in the hope that it
+ * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License version 2 for more details at
+ * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program available in the file COPYING in the main
+ * directory of this source tree.
+ *
+ * The BSD 2-Clause License
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __VMW_PVRDMA_ABI_H__
+#define __VMW_PVRDMA_ABI_H__
+
+#include <linux/types.h>
+
+#define PVRDMA_UVERBS_ABI_VERSION	3		/* ABI Version. */
+#define PVRDMA_UAR_HANDLE_MASK		0x00FFFFFF	/* Bottom 24 bits. */
+#define PVRDMA_UAR_QP_OFFSET		0		/* QP doorbell. */
+#define PVRDMA_UAR_QP_SEND		BIT(30)		/* Send bit. */
+#define PVRDMA_UAR_QP_RECV		BIT(31)		/* Recv bit. */
+#define PVRDMA_UAR_CQ_OFFSET		4		/* CQ doorbell. */
+#define PVRDMA_UAR_CQ_ARM_SOL		BIT(29)		/* Arm solicited bit. */
+#define PVRDMA_UAR_CQ_ARM		BIT(30)		/* Arm bit. */
+#define PVRDMA_UAR_CQ_POLL		BIT(31)		/* Poll bit. */
+
+enum pvrdma_wr_opcode {
+	PVRDMA_WR_RDMA_WRITE,
+	PVRDMA_WR_RDMA_WRITE_WITH_IMM,
+	PVRDMA_WR_SEND,
+	PVRDMA_WR_SEND_WITH_IMM,
+	PVRDMA_WR_RDMA_READ,
+	PVRDMA_WR_ATOMIC_CMP_AND_SWP,
+	PVRDMA_WR_ATOMIC_FETCH_AND_ADD,
+	PVRDMA_WR_LSO,
+	PVRDMA_WR_SEND_WITH_INV,
+	PVRDMA_WR_RDMA_READ_WITH_INV,
+	PVRDMA_WR_LOCAL_INV,
+	PVRDMA_WR_FAST_REG_MR,
+	PVRDMA_WR_MASKED_ATOMIC_CMP_AND_SWP,
+	PVRDMA_WR_MASKED_ATOMIC_FETCH_AND_ADD,
+	PVRDMA_WR_BIND_MW,
+	PVRDMA_WR_REG_SIG_MR,
+};
+
+enum pvrdma_wc_status {
+	PVRDMA_WC_SUCCESS,
+	PVRDMA_WC_LOC_LEN_ERR,
+	PVRDMA_WC_LOC_QP_OP_ERR,
+	PVRDMA_WC_LOC_EEC_OP_ERR,
+	PVRDMA_WC_LOC_PROT_ERR,
+	PVRDMA_WC_WR_FLUSH_ERR,
+	PVRDMA_WC_MW_BIND_ERR,
+	PVRDMA_WC_BAD_RESP_ERR,
+	PVRDMA_WC_LOC_ACCESS_ERR,
+	PVRDMA_WC_REM_INV_REQ_ERR,
+	PVRDMA_WC_REM_ACCESS_ERR,
+	PVRDMA_WC_REM_OP_ERR,
+	PVRDMA_WC_RETRY_EXC_ERR,
+	PVRDMA_WC_RNR_RETRY_EXC_ERR,
+	PVRDMA_WC_LOC_RDD_VIOL_ERR,
+	PVRDMA_WC_REM_INV_RD_REQ_ERR,
+	PVRDMA_WC_REM_ABORT_ERR,
+	PVRDMA_WC_INV_EECN_ERR,
+	PVRDMA_WC_INV_EEC_STATE_ERR,
+	PVRDMA_WC_FATAL_ERR,
+	PVRDMA_WC_RESP_TIMEOUT_ERR,
+	PVRDMA_WC_GENERAL_ERR,
+};
+
+enum pvrdma_wc_opcode {
+	PVRDMA_WC_SEND,
+	PVRDMA_WC_RDMA_WRITE,
+	PVRDMA_WC_RDMA_READ,
+	PVRDMA_WC_COMP_SWAP,
+	PVRDMA_WC_FETCH_ADD,
+	PVRDMA_WC_BIND_MW,
+	PVRDMA_WC_LSO,
+	PVRDMA_WC_LOCAL_INV,
+	PVRDMA_WC_FAST_REG_MR,
+	PVRDMA_WC_MASKED_COMP_SWAP,
+	PVRDMA_WC_MASKED_FETCH_ADD,
+	PVRDMA_WC_RECV = 1 << 7,
+	PVRDMA_WC_RECV_RDMA_WITH_IMM,
+};
+
+enum pvrdma_wc_flags {
+	PVRDMA_WC_GRH			= 1 << 0,
+	PVRDMA_WC_WITH_IMM		= 1 << 1,
+	PVRDMA_WC_WITH_INVALIDATE	= 1 << 2,
+	PVRDMA_WC_IP_CSUM_OK		= 1 << 3,
+	PVRDMA_WC_WITH_SMAC		= 1 << 4,
+	PVRDMA_WC_WITH_VLAN		= 1 << 5,
+	PVRDMA_WC_FLAGS_MAX		= PVRDMA_WC_WITH_VLAN,
+};
+
+struct pvrdma_alloc_ucontext_resp {
+	__u32 qp_tab_size;
+	__u32 reserved;
+};
+
+struct pvrdma_alloc_pd_resp {
+	__u32 pdn;
+	__u32 reserved;
+};
+
+struct pvrdma_create_cq {
+	__u64 buf_addr;
+	__u32 buf_size;
+	__u32 reserved;
+};
+
+struct pvrdma_create_cq_resp {
+	__u32 cqn;
+	__u32 reserved;
+};
+
+struct pvrdma_resize_cq {
+	__u64 buf_addr;
+	__u32 buf_size;
+	__u32 reserved;
+};
+
+struct pvrdma_create_srq {
+	__u64 buf_addr;
+};
+
+struct pvrdma_create_srq_resp {
+	__u32 srqn;
+	__u32 reserved;
+};
+
+struct pvrdma_create_qp {
+	__u64 rbuf_addr;
+	__u64 sbuf_addr;
+	__u32 rbuf_size;
+	__u32 sbuf_size;
+	__u64 qp_addr;
+};
+
+/* PVRDMA masked atomic compare and swap */
+struct pvrdma_ex_cmp_swap {
+	__u64 swap_val;
+	__u64 compare_val;
+	__u64 swap_mask;
+	__u64 compare_mask;
+};
+
+/* PVRDMA masked atomic fetch and add */
+struct pvrdma_ex_fetch_add {
+	__u64 add_val;
+	__u64 field_boundary;
+};
+
+/* PVRDMA address vector. */
+struct pvrdma_av {
+	__u32 port_pd;
+	__u32 sl_tclass_flowlabel;
+	__u8 dgid[16];
+	__u8 src_path_bits;
+	__u8 gid_index;
+	__u8 stat_rate;
+	__u8 hop_limit;
+	__u8 dmac[6];
+	__u8 reserved[6];
+};
+
+/* PVRDMA scatter/gather entry */
+struct pvrdma_sge {
+	__u64   addr;
+	__u32   length;
+	__u32   lkey;
+};
+
+/* PVRDMA receive queue work request */
+struct pvrdma_rq_wqe_hdr {
+	__u64 wr_id;		/* wr id */
+	__u32 num_sge;		/* size of s/g array */
+	__u32 total_len;	/* reserved */
+};
+/* Use pvrdma_sge (ib_sge) for receive queue s/g array elements. */
+
+/* PVRDMA send queue work request */
+struct pvrdma_sq_wqe_hdr {
+	__u64 wr_id;		/* wr id */
+	__u32 num_sge;		/* size of s/g array */
+	__u32 total_len;	/* reserved */
+	__u32 opcode;		/* operation type */
+	__u32 send_flags;	/* wr flags */
+	union {
+		__u32 imm_data;
+		__u32 invalidate_rkey;
+	} ex;
+	__u32 reserved;
+	union {
+		struct {
+			__u64 remote_addr;
+			__u32 rkey;
+			__u8 reserved[4];
+		} rdma;
+		struct {
+			__u64 remote_addr;
+			__u64 compare_add;
+			__u64 swap;
+			__u32 rkey;
+			__u32 reserved;
+		} atomic;
+		struct {
+			__u64 remote_addr;
+			__u32 log_arg_sz;
+			__u32 rkey;
+			union {
+				struct pvrdma_ex_cmp_swap  cmp_swap;
+				struct pvrdma_ex_fetch_add fetch_add;
+			} wr_data;
+		} masked_atomics;
+		struct {
+			__u64 iova_start;
+			__u64 pl_pdir_dma;
+			__u32 page_shift;
+			__u32 page_list_len;
+			__u32 length;
+			__u32 access_flags;
+			__u32 rkey;
+		} fast_reg;
+		struct {
+			__u32 remote_qpn;
+			__u32 remote_qkey;
+			struct pvrdma_av av;
+		} ud;
+	} wr;
+};
+/* Use pvrdma_sge (ib_sge) for send queue s/g array elements. */
+
+/* Completion queue element. */
+struct pvrdma_cqe {
+	__u64 wr_id;
+	__u64 qp;
+	__u32 opcode;
+	__u32 status;
+	__u32 byte_len;
+	__u32 imm_data;
+	__u32 src_qp;
+	__u32 wc_flags;
+	__u32 vendor_err;
+	__u16 pkey_index;
+	__u16 slid;
+	__u8 sl;
+	__u8 dlid_path_bits;
+	__u8 port_num;
+	__u8 smac[6];
+	__u8 reserved2[7]; /* Pad to next power of 2 (64). */
+};
+
+#endif /* __VMW_PVRDMA_ABI_H__ */
diff --git a/include/uapi/scsi/fc/fc_fs.h b/include/uapi/scsi/fc/fc_fs.h
index 50f28b143451..dcf314dc2a27 100644
--- a/include/uapi/scsi/fc/fc_fs.h
+++ b/include/uapi/scsi/fc/fc_fs.h
@@ -190,6 +190,7 @@ enum fc_fh_type {
 	FC_TYPE_FCP =	0x08,	/* SCSI FCP */
 	FC_TYPE_CT =	0x20,	/* Fibre Channel Services (FC-CT) */
 	FC_TYPE_ILS =	0x22,	/* internal link service */
+	FC_TYPE_NVME =	0x28,	/* FC-NVME */
 };
 
 /*
@@ -203,6 +204,7 @@ enum fc_fh_type {
 	[FC_TYPE_FCP] =		"FCP",			\
 	[FC_TYPE_CT] =		"CT",			\
 	[FC_TYPE_ILS] =		"ILS",			\
+	[FC_TYPE_NVME] =	"NVME",			\
 }
 
 /*
diff --git a/include/uapi/sound/asoc.h b/include/uapi/sound/asoc.h
index 819d895edfdc..6702533c8bd8 100644
--- a/include/uapi/sound/asoc.h
+++ b/include/uapi/sound/asoc.h
@@ -33,6 +33,11 @@
  */
 #define SND_SOC_TPLG_STREAM_CONFIG_MAX  8
 
+/*
+ * Maximum number of physical link's hardware configs
+ */
+#define SND_SOC_TPLG_HW_CONFIG_MAX	8
+
 /* individual kcontrol info types - can be mixed with other types */
 #define SND_SOC_TPLG_CTL_VOLSW		1
 #define SND_SOC_TPLG_CTL_VOLSW_SX	2
@@ -77,7 +82,8 @@
 #define SND_SOC_TPLG_NUM_TEXTS		16
 
 /* ABI version */
-#define SND_SOC_TPLG_ABI_VERSION	0x5
+#define SND_SOC_TPLG_ABI_VERSION	0x5	/* current version */
+#define SND_SOC_TPLG_ABI_VERSION_MIN	0x4	/* oldest version supported */
 
 /* Max size of TLV data */
 #define SND_SOC_TPLG_TLV_SIZE		32
@@ -99,8 +105,8 @@
 #define SND_SOC_TPLG_TYPE_CODEC_LINK	9
 #define SND_SOC_TPLG_TYPE_BACKEND_LINK	10
 #define SND_SOC_TPLG_TYPE_PDATA		11
-#define SND_SOC_TPLG_TYPE_BE_DAI	12
-#define SND_SOC_TPLG_TYPE_MAX		SND_SOC_TPLG_TYPE_BE_DAI
+#define SND_SOC_TPLG_TYPE_DAI		12
+#define SND_SOC_TPLG_TYPE_MAX		SND_SOC_TPLG_TYPE_DAI
 
 /* vendor block IDs - please add new vendor types to end */
 #define SND_SOC_TPLG_TYPE_VENDOR_FW	1000
@@ -119,11 +125,32 @@
 #define SND_SOC_TPLG_TUPLE_TYPE_WORD	4
 #define SND_SOC_TPLG_TUPLE_TYPE_SHORT	5
 
-/* BE DAI flags */
+/* DAI flags */
 #define SND_SOC_TPLG_DAI_FLGBIT_SYMMETRIC_RATES         (1 << 0)
 #define SND_SOC_TPLG_DAI_FLGBIT_SYMMETRIC_CHANNELS      (1 << 1)
 #define SND_SOC_TPLG_DAI_FLGBIT_SYMMETRIC_SAMPLEBITS    (1 << 2)
 
+/* DAI physical PCM data formats.
+ * Add new formats to the end of the list.
+ */
+#define SND_SOC_DAI_FORMAT_I2S          1 /* I2S mode */
+#define SND_SOC_DAI_FORMAT_RIGHT_J      2 /* Right Justified mode */
+#define SND_SOC_DAI_FORMAT_LEFT_J       3 /* Left Justified mode */
+#define SND_SOC_DAI_FORMAT_DSP_A        4 /* L data MSB after FRM LRC */
+#define SND_SOC_DAI_FORMAT_DSP_B        5 /* L data MSB during FRM LRC */
+#define SND_SOC_DAI_FORMAT_AC97         6 /* AC97 */
+#define SND_SOC_DAI_FORMAT_PDM          7 /* Pulse density modulation */
+
+/* left and right justified also known as MSB and LSB respectively */
+#define SND_SOC_DAI_FORMAT_MSB          SND_SOC_DAI_FORMAT_LEFT_J
+#define SND_SOC_DAI_FORMAT_LSB          SND_SOC_DAI_FORMAT_RIGHT_J
+
+/* DAI link flags */
+#define SND_SOC_TPLG_LNK_FLGBIT_SYMMETRIC_RATES         (1 << 0)
+#define SND_SOC_TPLG_LNK_FLGBIT_SYMMETRIC_CHANNELS      (1 << 1)
+#define SND_SOC_TPLG_LNK_FLGBIT_SYMMETRIC_SAMPLEBITS    (1 << 2)
+#define SND_SOC_TPLG_LNK_FLGBIT_VOICE_WAKEUP            (1 << 3)
+
 /*
  * Block Header.
  * This header precedes all object and object arrays below.
@@ -267,6 +294,35 @@ struct snd_soc_tplg_stream {
 	__le32 channels;	/* channels */
 } __attribute__((packed));
 
+
+/*
+ * Describes a physical link's runtime supported hardware config,
+ * i.e. hardware audio formats.
+ */
+struct snd_soc_tplg_hw_config {
+	__le32 size;            /* in bytes of this structure */
+	__le32 id;		/* unique ID - - used to match */
+	__le32 fmt;		/* SND_SOC_DAI_FORMAT_ format value */
+	__u8 clock_gated;	/* 1 if clock can be gated to save power */
+	__u8 invert_bclk;	/* 1 for inverted BCLK, 0 for normal */
+	__u8 invert_fsync;	/* 1 for inverted frame clock, 0 for normal */
+	__u8 bclk_master;	/* 1 for master of BCLK, 0 for slave */
+	__u8 fsync_master;	/* 1 for master of FSYNC, 0 for slave */
+	__u8 mclk_direction;    /* 0 for input, 1 for output */
+	__le16 reserved;	/* for 32bit alignment */
+	__le32 mclk_rate;	/* MCLK or SYSCLK freqency in Hz */
+	__le32 bclk_rate;	/* BCLK freqency in Hz */
+	__le32 fsync_rate;	/* frame clock in Hz */
+	__le32 tdm_slots;	/* number of TDM slots in use */
+	__le32 tdm_slot_width;	/* width in bits for each slot */
+	__le32 tx_slots;	/* bit mask for active Tx slots */
+	__le32 rx_slots;	/* bit mask for active Rx slots */
+	__le32 tx_channels;	/* number of Tx channels */
+	__le32 tx_chanmap[SND_SOC_TPLG_MAX_CHAN]; /* array of slot number */
+	__le32 rx_channels;	/* number of Rx channels */
+	__le32 rx_chanmap[SND_SOC_TPLG_MAX_CHAN]; /* array of slot number */
+} __attribute__((packed));
+
 /*
  * Manifest. List totals for each payload type. Not used in parsing, but will
  * be passed to the component driver before any other objects in order for any
@@ -286,7 +342,7 @@ struct snd_soc_tplg_manifest {
 	__le32 graph_elems;	/* number of graph elements */
 	__le32 pcm_elems;	/* number of PCM elements */
 	__le32 dai_link_elems;	/* number of DAI link elements */
-	__le32 be_dai_elems;	/* number of BE DAI elements */
+	__le32 dai_elems;	/* number of physical DAI elements */
 	__le32 reserved[20];	/* reserved for new ABI element types */
 	struct snd_soc_tplg_private priv;
 } __attribute__((packed));
@@ -434,13 +490,16 @@ struct snd_soc_tplg_pcm {
 	struct snd_soc_tplg_stream stream[SND_SOC_TPLG_STREAM_CONFIG_MAX]; /* for DAI link */
 	__le32 num_streams;	/* number of streams */
 	struct snd_soc_tplg_stream_caps caps[2]; /* playback and capture for DAI */
+	__le32 flag_mask;       /* bitmask of flags to configure */
+	__le32 flags;           /* SND_SOC_TPLG_LNK_FLGBIT_* flag value */
+	struct snd_soc_tplg_private priv;
 } __attribute__((packed));
 
 
 /*
- * Describes the BE or CC link runtime supported configs or params
+ * Describes the physical link runtime supported configs or params
  *
- * File block representation for BE/CC link config :-
+ * File block representation for physical link config :-
  * +-----------------------------------+-----+
  * | struct snd_soc_tplg_hdr           |  1  |
  * +-----------------------------------+-----+
@@ -450,21 +509,30 @@ struct snd_soc_tplg_pcm {
 struct snd_soc_tplg_link_config {
 	__le32 size;            /* in bytes of this structure */
 	__le32 id;              /* unique ID - used to match */
+	char name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN]; /* name - used to match */
+	char stream_name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN]; /* stream name - used to match */
 	struct snd_soc_tplg_stream stream[SND_SOC_TPLG_STREAM_CONFIG_MAX]; /* supported configs playback and captrure */
 	__le32 num_streams;     /* number of streams */
+	struct snd_soc_tplg_hw_config hw_config[SND_SOC_TPLG_HW_CONFIG_MAX]; /* hw configs */
+	__le32 num_hw_configs;         /* number of hw configs */
+	__le32 default_hw_config_id;   /* default hw config ID for init */
+	__le32 flag_mask;       /* bitmask of flags to configure */
+	__le32 flags;           /* SND_SOC_TPLG_LNK_FLGBIT_* flag value */
+	struct snd_soc_tplg_private priv;
 } __attribute__((packed));
 
 /*
- * Describes SW/FW specific features of BE DAI.
+ * Describes SW/FW specific features of physical DAI.
+ * It can be used to configure backend DAIs for DPCM.
  *
- * File block representation for BE DAI :-
+ * File block representation for physical DAI :-
  * +-----------------------------------+-----+
  * | struct snd_soc_tplg_hdr           |  1  |
  * +-----------------------------------+-----+
- * | struct snd_soc_tplg_be_dai        |  N  |
+ * | struct snd_soc_tplg_dai           |  N  |
  * +-----------------------------------+-----+
  */
-struct snd_soc_tplg_be_dai {
+struct snd_soc_tplg_dai {
 	__le32 size;            /* in bytes of this structure */
 	char dai_name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN]; /* name - used to match */
 	__le32 dai_id;          /* unique ID - used to match */
diff --git a/include/uapi/sound/snd_sst_tokens.h b/include/uapi/sound/snd_sst_tokens.h
index 1ee2e943d66a..93392bedcc58 100644
--- a/include/uapi/sound/snd_sst_tokens.h
+++ b/include/uapi/sound/snd_sst_tokens.h
@@ -157,6 +157,10 @@
  *
  * %SKL_TKN_STR_LIB_NAME:       Specifies the library name
  *
+ * %SKL_TKN_U32_PMODE:		Specifies the power mode for pipe
+ *
+ * %SKL_TKL_U32_D0I3_CAPS:	Specifies the D0i3 capability for module
+ *
  * module_id and loadable flags dont have tokens as these values will be
  * read from the DSP FW manifest
  */
@@ -208,7 +212,9 @@ enum SKL_TKNS {
 	SKL_TKN_U32_PROC_DOMAIN,
 	SKL_TKN_U32_LIB_COUNT,
 	SKL_TKN_STR_LIB_NAME,
-	SKL_TKN_MAX = SKL_TKN_STR_LIB_NAME,
+	SKL_TKN_U32_PMODE,
+	SKL_TKL_U32_D0I3_CAPS,
+	SKL_TKN_MAX = SKL_TKL_U32_D0I3_CAPS,
 };
 
 #endif
diff --git a/include/video/display_timing.h b/include/video/display_timing.h
index 28d9d0d566ca..3d289e990aca 100644
--- a/include/video/display_timing.h
+++ b/include/video/display_timing.h
@@ -28,6 +28,10 @@ enum display_flags {
 	DISPLAY_FLAGS_INTERLACED	= BIT(8),
 	DISPLAY_FLAGS_DOUBLESCAN	= BIT(9),
 	DISPLAY_FLAGS_DOUBLECLK		= BIT(10),
+	/* drive sync on pos. edge */
+	DISPLAY_FLAGS_SYNC_POSEDGE	= BIT(11),
+	/* drive sync on neg. edge */
+	DISPLAY_FLAGS_SYNC_NEGEDGE	= BIT(12),
 };
 
 /*
diff --git a/include/video/imx-ipu-v3.h b/include/video/imx-ipu-v3.h
index 173073eb6aaf..53cd07ccaa4c 100644
--- a/include/video/imx-ipu-v3.h
+++ b/include/video/imx-ipu-v3.h
@@ -247,8 +247,6 @@ void ipu_cpmem_set_yuv_planar_full(struct ipuv3_channel *ch,
 				   unsigned int uv_stride,
 				   unsigned int u_offset,
 				   unsigned int v_offset);
-void ipu_cpmem_set_yuv_planar(struct ipuv3_channel *ch,
-			      u32 pixel_format, int stride, int height);
 int ipu_cpmem_set_fmt(struct ipuv3_channel *ch, u32 drm_fourcc);
 int ipu_cpmem_set_image(struct ipuv3_channel *ch, struct ipu_image *image);
 void ipu_cpmem_dump(struct ipuv3_channel *ch);
@@ -320,6 +318,7 @@ int ipu_csi_init_interface(struct ipu_csi *csi,
 bool ipu_csi_is_interlaced(struct ipu_csi *csi);
 void ipu_csi_get_window(struct ipu_csi *csi, struct v4l2_rect *w);
 void ipu_csi_set_window(struct ipu_csi *csi, struct v4l2_rect *w);
+void ipu_csi_set_downsize(struct ipu_csi *csi, bool horiz, bool vert);
 void ipu_csi_set_test_generator(struct ipu_csi *csi, bool active,
 				u32 r_value, u32 g_value, u32 b_value,
 				u32 pix_clk);
diff --git a/include/video/of_display_timing.h b/include/video/of_display_timing.h
index ea755b5616d8..956455fc9f9a 100644
--- a/include/video/of_display_timing.h
+++ b/include/video/of_display_timing.h
@@ -16,21 +16,22 @@ struct display_timings;
 #define OF_USE_NATIVE_MODE -1
 
 #ifdef CONFIG_OF
-int of_get_display_timing(struct device_node *np, const char *name,
+int of_get_display_timing(const struct device_node *np, const char *name,
 		struct display_timing *dt);
-struct display_timings *of_get_display_timings(struct device_node *np);
-int of_display_timings_exist(struct device_node *np);
+struct display_timings *of_get_display_timings(const struct device_node *np);
+int of_display_timings_exist(const struct device_node *np);
 #else
-static inline int of_get_display_timing(struct device_node *np, const char *name,
-		struct display_timing *dt)
+static inline int of_get_display_timing(const struct device_node *np,
+		const char *name, struct display_timing *dt)
 {
 	return -ENOSYS;
 }
-static inline struct display_timings *of_get_display_timings(struct device_node *np)
+static inline struct display_timings *
+of_get_display_timings(const struct device_node *np)
 {
 	return NULL;
 }
-static inline int of_display_timings_exist(struct device_node *np)
+static inline int of_display_timings_exist(const struct device_node *np)
 {
 	return -ENOSYS;
 }
diff --git a/include/xen/arm/hypercall.h b/include/xen/arm/hypercall.h
new file mode 100644
index 000000000000..9d874db13c0e
--- /dev/null
+++ b/include/xen/arm/hypercall.h
@@ -0,0 +1,87 @@
+/******************************************************************************
+ * hypercall.h
+ *
+ * Linux-specific hypervisor handling.
+ *
+ * Stefano Stabellini <stefano.stabellini@eu.citrix.com>, Citrix, 2012
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation; or, when distributed
+ * separately from the Linux kernel or incorporated into other
+ * software packages, subject to the following license:
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this source file (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use, copy, modify,
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef _ASM_ARM_XEN_HYPERCALL_H
+#define _ASM_ARM_XEN_HYPERCALL_H
+
+#include <linux/bug.h>
+
+#include <xen/interface/xen.h>
+#include <xen/interface/sched.h>
+#include <xen/interface/platform.h>
+
+long privcmd_call(unsigned call, unsigned long a1,
+		unsigned long a2, unsigned long a3,
+		unsigned long a4, unsigned long a5);
+int HYPERVISOR_xen_version(int cmd, void *arg);
+int HYPERVISOR_console_io(int cmd, int count, char *str);
+int HYPERVISOR_grant_table_op(unsigned int cmd, void *uop, unsigned int count);
+int HYPERVISOR_sched_op(int cmd, void *arg);
+int HYPERVISOR_event_channel_op(int cmd, void *arg);
+unsigned long HYPERVISOR_hvm_op(int op, void *arg);
+int HYPERVISOR_memory_op(unsigned int cmd, void *arg);
+int HYPERVISOR_physdev_op(int cmd, void *arg);
+int HYPERVISOR_vcpu_op(int cmd, int vcpuid, void *extra_args);
+int HYPERVISOR_tmem_op(void *arg);
+int HYPERVISOR_vm_assist(unsigned int cmd, unsigned int type);
+int HYPERVISOR_platform_op_raw(void *arg);
+static inline int HYPERVISOR_platform_op(struct xen_platform_op *op)
+{
+	op->interface_version = XENPF_INTERFACE_VERSION;
+	return HYPERVISOR_platform_op_raw(op);
+}
+int HYPERVISOR_multicall(struct multicall_entry *calls, uint32_t nr);
+
+static inline int
+HYPERVISOR_suspend(unsigned long start_info_mfn)
+{
+	struct sched_shutdown r = { .reason = SHUTDOWN_suspend };
+
+	/* start_info_mfn is unused on ARM */
+	return HYPERVISOR_sched_op(SCHEDOP_shutdown, &r);
+}
+
+static inline void
+MULTI_update_va_mapping(struct multicall_entry *mcl, unsigned long va,
+			unsigned int new_val, unsigned long flags)
+{
+	BUG();
+}
+
+static inline void
+MULTI_mmu_update(struct multicall_entry *mcl, struct mmu_update *req,
+		 int count, int *success_count, domid_t domid)
+{
+	BUG();
+}
+
+#endif /* _ASM_ARM_XEN_HYPERCALL_H */
diff --git a/include/xen/arm/hypervisor.h b/include/xen/arm/hypervisor.h
new file mode 100644
index 000000000000..95251512e2c4
--- /dev/null
+++ b/include/xen/arm/hypervisor.h
@@ -0,0 +1,39 @@
+#ifndef _ASM_ARM_XEN_HYPERVISOR_H
+#define _ASM_ARM_XEN_HYPERVISOR_H
+
+#include <linux/init.h>
+
+extern struct shared_info *HYPERVISOR_shared_info;
+extern struct start_info *xen_start_info;
+
+/* Lazy mode for batching updates / context switch */
+enum paravirt_lazy_mode {
+	PARAVIRT_LAZY_NONE,
+	PARAVIRT_LAZY_MMU,
+	PARAVIRT_LAZY_CPU,
+};
+
+static inline enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
+{
+	return PARAVIRT_LAZY_NONE;
+}
+
+extern struct dma_map_ops *xen_dma_ops;
+
+#ifdef CONFIG_XEN
+void __init xen_early_init(void);
+#else
+static inline void xen_early_init(void) { return; }
+#endif
+
+#ifdef CONFIG_HOTPLUG_CPU
+static inline void xen_arch_register_cpu(int num)
+{
+}
+
+static inline void xen_arch_unregister_cpu(int num)
+{
+}
+#endif
+
+#endif /* _ASM_ARM_XEN_HYPERVISOR_H */
diff --git a/include/xen/arm/interface.h b/include/xen/arm/interface.h
new file mode 100644
index 000000000000..75d596862892
--- /dev/null
+++ b/include/xen/arm/interface.h
@@ -0,0 +1,85 @@
+/******************************************************************************
+ * Guest OS interface to ARM Xen.
+ *
+ * Stefano Stabellini <stefano.stabellini@eu.citrix.com>, Citrix, 2012
+ */
+
+#ifndef _ASM_ARM_XEN_INTERFACE_H
+#define _ASM_ARM_XEN_INTERFACE_H
+
+#include <linux/types.h>
+
+#define uint64_aligned_t uint64_t __attribute__((aligned(8)))
+
+#define __DEFINE_GUEST_HANDLE(name, type) \
+	typedef struct { union { type *p; uint64_aligned_t q; }; }  \
+        __guest_handle_ ## name
+
+#define DEFINE_GUEST_HANDLE_STRUCT(name) \
+	__DEFINE_GUEST_HANDLE(name, struct name)
+#define DEFINE_GUEST_HANDLE(name) __DEFINE_GUEST_HANDLE(name, name)
+#define GUEST_HANDLE(name)        __guest_handle_ ## name
+
+#define set_xen_guest_handle(hnd, val)			\
+	do {						\
+		if (sizeof(hnd) == 8)			\
+			*(uint64_t *)&(hnd) = 0;	\
+		(hnd).p = val;				\
+	} while (0)
+
+#define __HYPERVISOR_platform_op_raw __HYPERVISOR_platform_op
+
+#ifndef __ASSEMBLY__
+/* Explicitly size integers that represent pfns in the interface with
+ * Xen so that we can have one ABI that works for 32 and 64 bit guests.
+ * Note that this means that the xen_pfn_t type may be capable of
+ * representing pfn's which the guest cannot represent in its own pfn
+ * type. However since pfn space is controlled by the guest this is
+ * fine since it simply wouldn't be able to create any sure pfns in
+ * the first place.
+ */
+typedef uint64_t xen_pfn_t;
+#define PRI_xen_pfn "llx"
+typedef uint64_t xen_ulong_t;
+#define PRI_xen_ulong "llx"
+typedef int64_t xen_long_t;
+#define PRI_xen_long "llx"
+/* Guest handles for primitive C types. */
+__DEFINE_GUEST_HANDLE(uchar, unsigned char);
+__DEFINE_GUEST_HANDLE(uint,  unsigned int);
+DEFINE_GUEST_HANDLE(char);
+DEFINE_GUEST_HANDLE(int);
+DEFINE_GUEST_HANDLE(void);
+DEFINE_GUEST_HANDLE(uint64_t);
+DEFINE_GUEST_HANDLE(uint32_t);
+DEFINE_GUEST_HANDLE(xen_pfn_t);
+DEFINE_GUEST_HANDLE(xen_ulong_t);
+
+/* Maximum number of virtual CPUs in multi-processor guests. */
+#define MAX_VIRT_CPUS 1
+
+struct arch_vcpu_info { };
+struct arch_shared_info { };
+
+/* TODO: Move pvclock definitions some place arch independent */
+struct pvclock_vcpu_time_info {
+	u32   version;
+	u32   pad0;
+	u64   tsc_timestamp;
+	u64   system_time;
+	u32   tsc_to_system_mul;
+	s8    tsc_shift;
+	u8    flags;
+	u8    pad[2];
+} __attribute__((__packed__)); /* 32 bytes */
+
+/* It is OK to have a 12 bytes struct with no padding because it is packed */
+struct pvclock_wall_clock {
+	u32   version;
+	u32   sec;
+	u32   nsec;
+	u32   sec_hi;
+} __attribute__((__packed__));
+#endif
+
+#endif /* _ASM_ARM_XEN_INTERFACE_H */
diff --git a/include/xen/arm/page-coherent.h b/include/xen/arm/page-coherent.h
new file mode 100644
index 000000000000..95ce6ac3a971
--- /dev/null
+++ b/include/xen/arm/page-coherent.h
@@ -0,0 +1,98 @@
+#ifndef _ASM_ARM_XEN_PAGE_COHERENT_H
+#define _ASM_ARM_XEN_PAGE_COHERENT_H
+
+#include <asm/page.h>
+#include <linux/dma-mapping.h>
+
+void __xen_dma_map_page(struct device *hwdev, struct page *page,
+	     dma_addr_t dev_addr, unsigned long offset, size_t size,
+	     enum dma_data_direction dir, unsigned long attrs);
+void __xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle,
+		size_t size, enum dma_data_direction dir,
+		unsigned long attrs);
+void __xen_dma_sync_single_for_cpu(struct device *hwdev,
+		dma_addr_t handle, size_t size, enum dma_data_direction dir);
+
+void __xen_dma_sync_single_for_device(struct device *hwdev,
+		dma_addr_t handle, size_t size, enum dma_data_direction dir);
+
+static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size,
+		dma_addr_t *dma_handle, gfp_t flags, unsigned long attrs)
+{
+	return __generic_dma_ops(hwdev)->alloc(hwdev, size, dma_handle, flags, attrs);
+}
+
+static inline void xen_free_coherent_pages(struct device *hwdev, size_t size,
+		void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs)
+{
+	__generic_dma_ops(hwdev)->free(hwdev, size, cpu_addr, dma_handle, attrs);
+}
+
+static inline void xen_dma_map_page(struct device *hwdev, struct page *page,
+	     dma_addr_t dev_addr, unsigned long offset, size_t size,
+	     enum dma_data_direction dir, unsigned long attrs)
+{
+	unsigned long page_pfn = page_to_xen_pfn(page);
+	unsigned long dev_pfn = XEN_PFN_DOWN(dev_addr);
+	unsigned long compound_pages =
+		(1<<compound_order(page)) * XEN_PFN_PER_PAGE;
+	bool local = (page_pfn <= dev_pfn) &&
+		(dev_pfn - page_pfn < compound_pages);
+
+	/*
+	 * Dom0 is mapped 1:1, while the Linux page can span across
+	 * multiple Xen pages, it's not possible for it to contain a
+	 * mix of local and foreign Xen pages. So if the first xen_pfn
+	 * == mfn the page is local otherwise it's a foreign page
+	 * grant-mapped in dom0. If the page is local we can safely
+	 * call the native dma_ops function, otherwise we call the xen
+	 * specific function.
+	 */
+	if (local)
+		__generic_dma_ops(hwdev)->map_page(hwdev, page, offset, size, dir, attrs);
+	else
+		__xen_dma_map_page(hwdev, page, dev_addr, offset, size, dir, attrs);
+}
+
+static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle,
+		size_t size, enum dma_data_direction dir, unsigned long attrs)
+{
+	unsigned long pfn = PFN_DOWN(handle);
+	/*
+	 * Dom0 is mapped 1:1, while the Linux page can be spanned accross
+	 * multiple Xen page, it's not possible to have a mix of local and
+	 * foreign Xen page. Dom0 is mapped 1:1, so calling pfn_valid on a
+	 * foreign mfn will always return false. If the page is local we can
+	 * safely call the native dma_ops function, otherwise we call the xen
+	 * specific function.
+	 */
+	if (pfn_valid(pfn)) {
+		if (__generic_dma_ops(hwdev)->unmap_page)
+			__generic_dma_ops(hwdev)->unmap_page(hwdev, handle, size, dir, attrs);
+	} else
+		__xen_dma_unmap_page(hwdev, handle, size, dir, attrs);
+}
+
+static inline void xen_dma_sync_single_for_cpu(struct device *hwdev,
+		dma_addr_t handle, size_t size, enum dma_data_direction dir)
+{
+	unsigned long pfn = PFN_DOWN(handle);
+	if (pfn_valid(pfn)) {
+		if (__generic_dma_ops(hwdev)->sync_single_for_cpu)
+			__generic_dma_ops(hwdev)->sync_single_for_cpu(hwdev, handle, size, dir);
+	} else
+		__xen_dma_sync_single_for_cpu(hwdev, handle, size, dir);
+}
+
+static inline void xen_dma_sync_single_for_device(struct device *hwdev,
+		dma_addr_t handle, size_t size, enum dma_data_direction dir)
+{
+	unsigned long pfn = PFN_DOWN(handle);
+	if (pfn_valid(pfn)) {
+		if (__generic_dma_ops(hwdev)->sync_single_for_device)
+			__generic_dma_ops(hwdev)->sync_single_for_device(hwdev, handle, size, dir);
+	} else
+		__xen_dma_sync_single_for_device(hwdev, handle, size, dir);
+}
+
+#endif /* _ASM_ARM_XEN_PAGE_COHERENT_H */
diff --git a/include/xen/arm/page.h b/include/xen/arm/page.h
new file mode 100644
index 000000000000..415dbc6e43fd
--- /dev/null
+++ b/include/xen/arm/page.h
@@ -0,0 +1,122 @@
+#ifndef _ASM_ARM_XEN_PAGE_H
+#define _ASM_ARM_XEN_PAGE_H
+
+#include <asm/page.h>
+#include <asm/pgtable.h>
+
+#include <linux/pfn.h>
+#include <linux/types.h>
+#include <linux/dma-mapping.h>
+
+#include <xen/xen.h>
+#include <xen/interface/grant_table.h>
+
+#define phys_to_machine_mapping_valid(pfn) (1)
+
+/* Xen machine address */
+typedef struct xmaddr {
+	phys_addr_t maddr;
+} xmaddr_t;
+
+/* Xen pseudo-physical address */
+typedef struct xpaddr {
+	phys_addr_t paddr;
+} xpaddr_t;
+
+#define XMADDR(x)	((xmaddr_t) { .maddr = (x) })
+#define XPADDR(x)	((xpaddr_t) { .paddr = (x) })
+
+#define INVALID_P2M_ENTRY      (~0UL)
+
+/*
+ * The pseudo-physical frame (pfn) used in all the helpers is always based
+ * on Xen page granularity (i.e 4KB).
+ *
+ * A Linux page may be split across multiple non-contiguous Xen page so we
+ * have to keep track with frame based on 4KB page granularity.
+ *
+ * PV drivers should never make a direct usage of those helpers (particularly
+ * pfn_to_gfn and gfn_to_pfn).
+ */
+
+unsigned long __pfn_to_mfn(unsigned long pfn);
+extern struct rb_root phys_to_mach;
+
+/* Pseudo-physical <-> Guest conversion */
+static inline unsigned long pfn_to_gfn(unsigned long pfn)
+{
+	return pfn;
+}
+
+static inline unsigned long gfn_to_pfn(unsigned long gfn)
+{
+	return gfn;
+}
+
+/* Pseudo-physical <-> BUS conversion */
+static inline unsigned long pfn_to_bfn(unsigned long pfn)
+{
+	unsigned long mfn;
+
+	if (phys_to_mach.rb_node != NULL) {
+		mfn = __pfn_to_mfn(pfn);
+		if (mfn != INVALID_P2M_ENTRY)
+			return mfn;
+	}
+
+	return pfn;
+}
+
+static inline unsigned long bfn_to_pfn(unsigned long bfn)
+{
+	return bfn;
+}
+
+#define bfn_to_local_pfn(bfn)	bfn_to_pfn(bfn)
+
+/* VIRT <-> GUEST conversion */
+#define virt_to_gfn(v)		(pfn_to_gfn(virt_to_phys(v) >> XEN_PAGE_SHIFT))
+#define gfn_to_virt(m)		(__va(gfn_to_pfn(m) << XEN_PAGE_SHIFT))
+
+/* Only used in PV code. But ARM guests are always HVM. */
+static inline xmaddr_t arbitrary_virt_to_machine(void *vaddr)
+{
+	BUG();
+}
+
+/* TODO: this shouldn't be here but it is because the frontend drivers
+ * are using it (its rolled in headers) even though we won't hit the code path.
+ * So for right now just punt with this.
+ */
+static inline pte_t *lookup_address(unsigned long address, unsigned int *level)
+{
+	BUG();
+	return NULL;
+}
+
+extern int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops,
+				   struct gnttab_map_grant_ref *kmap_ops,
+				   struct page **pages, unsigned int count);
+
+extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
+				     struct gnttab_unmap_grant_ref *kunmap_ops,
+				     struct page **pages, unsigned int count);
+
+bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn);
+bool __set_phys_to_machine_multi(unsigned long pfn, unsigned long mfn,
+		unsigned long nr_pages);
+
+static inline bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)
+{
+	return __set_phys_to_machine(pfn, mfn);
+}
+
+#define xen_remap(cookie, size) ioremap_cache((cookie), (size))
+#define xen_unmap(cookie) iounmap((cookie))
+
+bool xen_arch_need_swiotlb(struct device *dev,
+			   phys_addr_t phys,
+			   dma_addr_t dev_addr);
+unsigned long xen_get_swiotlb_free_pages(unsigned int order);
+
+#endif /* _ASM_ARM_XEN_PAGE_H */
diff --git a/include/xen/swiotlb-xen.h b/include/xen/swiotlb-xen.h
index 7c35e279d1e3..a0083be5d529 100644
--- a/include/xen/swiotlb-xen.h
+++ b/include/xen/swiotlb-xen.h
@@ -51,9 +51,6 @@ xen_swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
 			       int nelems, enum dma_data_direction dir);
 
 extern int
-xen_swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr);
-
-extern int
 xen_swiotlb_dma_supported(struct device *hwdev, u64 mask);
 
 extern int
diff --git a/include/xen/xenbus.h b/include/xen/xenbus.h
index 32b944b7cebd..271ba62503c7 100644
--- a/include/xen/xenbus.h
+++ b/include/xen/xenbus.h
@@ -151,6 +151,10 @@ __scanf(4, 5)
 int xenbus_scanf(struct xenbus_transaction t,
 		 const char *dir, const char *node, const char *fmt, ...);
 
+/* Read an (optional) unsigned value. */
+unsigned int xenbus_read_unsigned(const char *dir, const char *node,
+				  unsigned int default_val);
+
 /* Single printf and write: returns -errno or 0. */
 __printf(4, 5)
 int xenbus_printf(struct xenbus_transaction t,
diff --git a/init/main.c b/init/main.c
index fa201166cba7..c81c9fa21bc7 100644
--- a/init/main.c
+++ b/init/main.c
@@ -81,6 +81,7 @@
 #include <linux/integrity.h>
 #include <linux/proc_ns.h>
 #include <linux/io.h>
+#include <linux/cache.h>
 
 #include <asm/io.h>
 #include <asm/bugs.h>
@@ -553,6 +554,14 @@ asmlinkage __visible void __init start_kernel(void)
 		 "Interrupts were enabled *very* early, fixing it\n"))
 		local_irq_disable();
 	idr_init_cache();
+
+	/*
+	 * Allow workqueue creation and work item queueing/cancelling
+	 * early.  Work item execution depends on kthreads and starts after
+	 * workqueue_init().
+	 */
+	workqueue_init_early();
+
 	rcu_init();
 
 	/* trace_printk() and trace points may be used after this */
@@ -917,14 +926,16 @@ static int try_to_run_init_process(const char *init_filename)
 
 static noinline void __init kernel_init_freeable(void);
 
-#ifdef CONFIG_DEBUG_RODATA
-static bool rodata_enabled = true;
+#if defined(CONFIG_DEBUG_RODATA) || defined(CONFIG_DEBUG_SET_MODULE_RONX)
+bool rodata_enabled __ro_after_init = true;
 static int __init set_debug_rodata(char *str)
 {
 	return strtobool(str, &rodata_enabled);
 }
 __setup("rodata=", set_debug_rodata);
+#endif
 
+#ifdef CONFIG_DEBUG_RODATA
 static void mark_readonly(void)
 {
 	if (rodata_enabled)
@@ -1009,6 +1020,8 @@ static noinline void __init kernel_init_freeable(void)
 
 	smp_prepare_cpus(setup_max_cpus);
 
+	workqueue_init();
+
 	do_pre_smp_initcalls();
 	lockup_detector_init();
 
diff --git a/ipc/msg.c b/ipc/msg.c
index 32e9bd837cde..e3e52ce01123 100644
--- a/ipc/msg.c
+++ b/ipc/msg.c
@@ -763,7 +763,10 @@ static inline int convert_mode(long *msgtyp, int msgflg)
 	if (*msgtyp == 0)
 		return SEARCH_ANY;
 	if (*msgtyp < 0) {
-		*msgtyp = -*msgtyp;
+		if (*msgtyp == LONG_MIN) /* -LONG_MIN is undefined */
+			*msgtyp = LONG_MAX;
+		else
+			*msgtyp = -*msgtyp;
 		return SEARCH_LESSEQUAL;
 	}
 	if (msgflg & MSG_EXCEPT)
diff --git a/ipc/sem.c b/ipc/sem.c
index 10b94bc59d4a..e08b94851922 100644
--- a/ipc/sem.c
+++ b/ipc/sem.c
@@ -11,6 +11,7 @@
  * (c) 2001 Red Hat Inc
  * Lockless wakeup
  * (c) 2003 Manfred Spraul <manfred@colorfullife.com>
+ * (c) 2016 Davidlohr Bueso <dave@stgolabs.net>
  * Further wakeup optimizations, documentation
  * (c) 2010 Manfred Spraul <manfred@colorfullife.com>
  *
@@ -53,15 +54,11 @@
  *   Semaphores are actively given to waiting tasks (necessary for FIFO).
  *   (see update_queue())
  * - To improve the scalability, the actual wake-up calls are performed after
- *   dropping all locks. (see wake_up_sem_queue_prepare(),
- *   wake_up_sem_queue_do())
+ *   dropping all locks. (see wake_up_sem_queue_prepare())
  * - All work is done by the waker, the woken up task does not have to do
  *   anything - not even acquiring a lock or dropping a refcount.
  * - A woken up task may not even touch the semaphore array anymore, it may
  *   have been destroyed already by a semctl(RMID).
- * - The synchronizations between wake-ups due to a timeout/signal and a
- *   wake-up due to a completed semaphore operation is achieved by using an
- *   intermediate state (IN_WAKEUP).
  * - UNDO values are stored in an array (one per process and per
  *   semaphore array, lazily allocated). For backwards compatibility, multiple
  *   modes for the UNDO variables are supported (per process, per thread)
@@ -118,7 +115,8 @@ struct sem_queue {
 	struct sembuf		*sops;	 /* array of pending operations */
 	struct sembuf		*blocking; /* the operation that blocked */
 	int			nsops;	 /* number of operations */
-	int			alter;	 /* does *sops alter the array? */
+	bool			alter;	 /* does *sops alter the array? */
+	bool                    dupsop;	 /* sops on more than one sem_num */
 };
 
 /* Each task has a list of undo requests. They are executed automatically
@@ -416,29 +414,6 @@ static inline void sem_unlock(struct sem_array *sma, int locknum)
  *
  * The caller holds the RCU read lock.
  */
-static inline struct sem_array *sem_obtain_lock(struct ipc_namespace *ns,
-			int id, struct sembuf *sops, int nsops, int *locknum)
-{
-	struct kern_ipc_perm *ipcp;
-	struct sem_array *sma;
-
-	ipcp = ipc_obtain_object_idr(&sem_ids(ns), id);
-	if (IS_ERR(ipcp))
-		return ERR_CAST(ipcp);
-
-	sma = container_of(ipcp, struct sem_array, sem_perm);
-	*locknum = sem_lock(sma, sops, nsops);
-
-	/* ipc_rmid() may have already freed the ID while sem_lock
-	 * was spinning: verify that the structure is still valid
-	 */
-	if (ipc_valid_object(ipcp))
-		return container_of(ipcp, struct sem_array, sem_perm);
-
-	sem_unlock(sma, *locknum);
-	return ERR_PTR(-EINVAL);
-}
-
 static inline struct sem_array *sem_obtain_object(struct ipc_namespace *ns, int id)
 {
 	struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&sem_ids(ns), id);
@@ -471,40 +446,6 @@ static inline void sem_rmid(struct ipc_namespace *ns, struct sem_array *s)
 	ipc_rmid(&sem_ids(ns), &s->sem_perm);
 }
 
-/*
- * Lockless wakeup algorithm:
- * Without the check/retry algorithm a lockless wakeup is possible:
- * - queue.status is initialized to -EINTR before blocking.
- * - wakeup is performed by
- *	* unlinking the queue entry from the pending list
- *	* setting queue.status to IN_WAKEUP
- *	  This is the notification for the blocked thread that a
- *	  result value is imminent.
- *	* call wake_up_process
- *	* set queue.status to the final value.
- * - the previously blocked thread checks queue.status:
- *	* if it's IN_WAKEUP, then it must wait until the value changes
- *	* if it's not -EINTR, then the operation was completed by
- *	  update_queue. semtimedop can return queue.status without
- *	  performing any operation on the sem array.
- *	* otherwise it must acquire the spinlock and check what's up.
- *
- * The two-stage algorithm is necessary to protect against the following
- * races:
- * - if queue.status is set after wake_up_process, then the woken up idle
- *   thread could race forward and try (and fail) to acquire sma->lock
- *   before update_queue had a chance to set queue.status
- * - if queue.status is written before wake_up_process and if the
- *   blocked process is woken up by a signal between writing
- *   queue.status and the wake_up_process, then the woken up
- *   process could return from semtimedop and die by calling
- *   sys_exit before wake_up_process is called. Then wake_up_process
- *   will oops, because the task structure is already invalid.
- *   (yes, this happened on s390 with sysv msg).
- *
- */
-#define IN_WAKEUP	1
-
 /**
  * newary - Create a new semaphore set
  * @ns: namespace
@@ -624,15 +565,23 @@ SYSCALL_DEFINE3(semget, key_t, key, int, nsems, int, semflg)
 }
 
 /**
- * perform_atomic_semop - Perform (if possible) a semaphore operation
+ * perform_atomic_semop[_slow] - Attempt to perform semaphore
+ *                               operations on a given array.
  * @sma: semaphore array
  * @q: struct sem_queue that describes the operation
  *
+ * Caller blocking are as follows, based the value
+ * indicated by the semaphore operation (sem_op):
+ *
+ *  (1) >0 never blocks.
+ *  (2)  0 (wait-for-zero operation): semval is non-zero.
+ *  (3) <0 attempting to decrement semval to a value smaller than zero.
+ *
  * Returns 0 if the operation was possible.
  * Returns 1 if the operation is impossible, the caller must sleep.
- * Negative values are error codes.
+ * Returns <0 for error codes.
  */
-static int perform_atomic_semop(struct sem_array *sma, struct sem_queue *q)
+static int perform_atomic_semop_slow(struct sem_array *sma, struct sem_queue *q)
 {
 	int result, sem_op, nsops, pid;
 	struct sembuf *sop;
@@ -703,51 +652,84 @@ undo:
 	return result;
 }
 
-/** wake_up_sem_queue_prepare(q, error): Prepare wake-up
- * @q: queue entry that must be signaled
- * @error: Error value for the signal
- *
- * Prepare the wake-up of the queue entry q.
- */
-static void wake_up_sem_queue_prepare(struct list_head *pt,
-				struct sem_queue *q, int error)
+static int perform_atomic_semop(struct sem_array *sma, struct sem_queue *q)
 {
-	if (list_empty(pt)) {
-		/*
-		 * Hold preempt off so that we don't get preempted and have the
-		 * wakee busy-wait until we're scheduled back on.
-		 */
-		preempt_disable();
+	int result, sem_op, nsops;
+	struct sembuf *sop;
+	struct sem *curr;
+	struct sembuf *sops;
+	struct sem_undo *un;
+
+	sops = q->sops;
+	nsops = q->nsops;
+	un = q->undo;
+
+	if (unlikely(q->dupsop))
+		return perform_atomic_semop_slow(sma, q);
+
+	/*
+	 * We scan the semaphore set twice, first to ensure that the entire
+	 * operation can succeed, therefore avoiding any pointless writes
+	 * to shared memory and having to undo such changes in order to block
+	 * until the operations can go through.
+	 */
+	for (sop = sops; sop < sops + nsops; sop++) {
+		curr = sma->sem_base + sop->sem_num;
+		sem_op = sop->sem_op;
+		result = curr->semval;
+
+		if (!sem_op && result)
+			goto would_block; /* wait-for-zero */
+
+		result += sem_op;
+		if (result < 0)
+			goto would_block;
+
+		if (result > SEMVMX)
+			return -ERANGE;
+
+		if (sop->sem_flg & SEM_UNDO) {
+			int undo = un->semadj[sop->sem_num] - sem_op;
+
+			/* Exceeding the undo range is an error. */
+			if (undo < (-SEMAEM - 1) || undo > SEMAEM)
+				return -ERANGE;
+		}
+	}
+
+	for (sop = sops; sop < sops + nsops; sop++) {
+		curr = sma->sem_base + sop->sem_num;
+		sem_op = sop->sem_op;
+		result = curr->semval;
+
+		if (sop->sem_flg & SEM_UNDO) {
+			int undo = un->semadj[sop->sem_num] - sem_op;
+
+			un->semadj[sop->sem_num] = undo;
+		}
+		curr->semval += sem_op;
+		curr->sempid = q->pid;
 	}
-	q->status = IN_WAKEUP;
-	q->pid = error;
 
-	list_add_tail(&q->list, pt);
+	return 0;
+
+would_block:
+	q->blocking = sop;
+	return sop->sem_flg & IPC_NOWAIT ? -EAGAIN : 1;
 }
 
-/**
- * wake_up_sem_queue_do - do the actual wake-up
- * @pt: list of tasks to be woken up
- *
- * Do the actual wake-up.
- * The function is called without any locks held, thus the semaphore array
- * could be destroyed already and the tasks can disappear as soon as the
- * status is set to the actual return code.
- */
-static void wake_up_sem_queue_do(struct list_head *pt)
+static inline void wake_up_sem_queue_prepare(struct sem_queue *q, int error,
+					     struct wake_q_head *wake_q)
 {
-	struct sem_queue *q, *t;
-	int did_something;
-
-	did_something = !list_empty(pt);
-	list_for_each_entry_safe(q, t, pt, list) {
-		wake_up_process(q->sleeper);
-		/* q can disappear immediately after writing q->status. */
-		smp_wmb();
-		q->status = q->pid;
-	}
-	if (did_something)
-		preempt_enable();
+	wake_q_add(wake_q, q->sleeper);
+	/*
+	 * Rely on the above implicit barrier, such that we can
+	 * ensure that we hold reference to the task before setting
+	 * q->status. Otherwise we could race with do_exit if the
+	 * task is awoken by an external event before calling
+	 * wake_up_process().
+	 */
+	WRITE_ONCE(q->status, error);
 }
 
 static void unlink_queue(struct sem_array *sma, struct sem_queue *q)
@@ -767,7 +749,7 @@ static void unlink_queue(struct sem_array *sma, struct sem_queue *q)
  * modified the array.
  * Note that wait-for-zero operations are handled without restart.
  */
-static int check_restart(struct sem_array *sma, struct sem_queue *q)
+static inline int check_restart(struct sem_array *sma, struct sem_queue *q)
 {
 	/* pending complex alter operations are too difficult to analyse */
 	if (!list_empty(&sma->pending_alter))
@@ -795,21 +777,20 @@ static int check_restart(struct sem_array *sma, struct sem_queue *q)
  * wake_const_ops - wake up non-alter tasks
  * @sma: semaphore array.
  * @semnum: semaphore that was modified.
- * @pt: list head for the tasks that must be woken up.
+ * @wake_q: lockless wake-queue head.
  *
  * wake_const_ops must be called after a semaphore in a semaphore array
  * was set to 0. If complex const operations are pending, wake_const_ops must
  * be called with semnum = -1, as well as with the number of each modified
  * semaphore.
- * The tasks that must be woken up are added to @pt. The return code
+ * The tasks that must be woken up are added to @wake_q. The return code
  * is stored in q->pid.
  * The function returns 1 if at least one operation was completed successfully.
  */
 static int wake_const_ops(struct sem_array *sma, int semnum,
-				struct list_head *pt)
+			  struct wake_q_head *wake_q)
 {
-	struct sem_queue *q;
-	struct list_head *walk;
+	struct sem_queue *q, *tmp;
 	struct list_head *pending_list;
 	int semop_completed = 0;
 
@@ -818,25 +799,19 @@ static int wake_const_ops(struct sem_array *sma, int semnum,
 	else
 		pending_list = &sma->sem_base[semnum].pending_const;
 
-	walk = pending_list->next;
-	while (walk != pending_list) {
-		int error;
-
-		q = container_of(walk, struct sem_queue, list);
-		walk = walk->next;
-
-		error = perform_atomic_semop(sma, q);
-
-		if (error <= 0) {
-			/* operation completed, remove from queue & wakeup */
+	list_for_each_entry_safe(q, tmp, pending_list, list) {
+		int error = perform_atomic_semop(sma, q);
 
-			unlink_queue(sma, q);
+		if (error > 0)
+			continue;
+		/* operation completed, remove from queue & wakeup */
+		unlink_queue(sma, q);
 
-			wake_up_sem_queue_prepare(pt, q, error);
-			if (error == 0)
-				semop_completed = 1;
-		}
+		wake_up_sem_queue_prepare(q, error, wake_q);
+		if (error == 0)
+			semop_completed = 1;
 	}
+
 	return semop_completed;
 }
 
@@ -845,14 +820,14 @@ static int wake_const_ops(struct sem_array *sma, int semnum,
  * @sma: semaphore array
  * @sops: operations that were performed
  * @nsops: number of operations
- * @pt: list head of the tasks that must be woken up.
+ * @wake_q: lockless wake-queue head
  *
  * Checks all required queue for wait-for-zero operations, based
  * on the actual changes that were performed on the semaphore array.
  * The function returns 1 if at least one operation was completed successfully.
  */
 static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops,
-					int nsops, struct list_head *pt)
+				int nsops, struct wake_q_head *wake_q)
 {
 	int i;
 	int semop_completed = 0;
@@ -865,7 +840,7 @@ static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops,
 
 			if (sma->sem_base[num].semval == 0) {
 				got_zero = 1;
-				semop_completed |= wake_const_ops(sma, num, pt);
+				semop_completed |= wake_const_ops(sma, num, wake_q);
 			}
 		}
 	} else {
@@ -876,7 +851,7 @@ static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops,
 		for (i = 0; i < sma->sem_nsems; i++) {
 			if (sma->sem_base[i].semval == 0) {
 				got_zero = 1;
-				semop_completed |= wake_const_ops(sma, i, pt);
+				semop_completed |= wake_const_ops(sma, i, wake_q);
 			}
 		}
 	}
@@ -885,7 +860,7 @@ static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops,
 	 * then check the global queue, too.
 	 */
 	if (got_zero)
-		semop_completed |= wake_const_ops(sma, -1, pt);
+		semop_completed |= wake_const_ops(sma, -1, wake_q);
 
 	return semop_completed;
 }
@@ -895,22 +870,21 @@ static int do_smart_wakeup_zero(struct sem_array *sma, struct sembuf *sops,
  * update_queue - look for tasks that can be completed.
  * @sma: semaphore array.
  * @semnum: semaphore that was modified.
- * @pt: list head for the tasks that must be woken up.
+ * @wake_q: lockless wake-queue head.
  *
  * update_queue must be called after a semaphore in a semaphore array
  * was modified. If multiple semaphores were modified, update_queue must
  * be called with semnum = -1, as well as with the number of each modified
  * semaphore.
- * The tasks that must be woken up are added to @pt. The return code
+ * The tasks that must be woken up are added to @wake_q. The return code
  * is stored in q->pid.
  * The function internally checks if const operations can now succeed.
  *
  * The function return 1 if at least one semop was completed successfully.
  */
-static int update_queue(struct sem_array *sma, int semnum, struct list_head *pt)
+static int update_queue(struct sem_array *sma, int semnum, struct wake_q_head *wake_q)
 {
-	struct sem_queue *q;
-	struct list_head *walk;
+	struct sem_queue *q, *tmp;
 	struct list_head *pending_list;
 	int semop_completed = 0;
 
@@ -920,13 +894,9 @@ static int update_queue(struct sem_array *sma, int semnum, struct list_head *pt)
 		pending_list = &sma->sem_base[semnum].pending_alter;
 
 again:
-	walk = pending_list->next;
-	while (walk != pending_list) {
+	list_for_each_entry_safe(q, tmp, pending_list, list) {
 		int error, restart;
 
-		q = container_of(walk, struct sem_queue, list);
-		walk = walk->next;
-
 		/* If we are scanning the single sop, per-semaphore list of
 		 * one semaphore and that semaphore is 0, then it is not
 		 * necessary to scan further: simple increments
@@ -949,11 +919,11 @@ again:
 			restart = 0;
 		} else {
 			semop_completed = 1;
-			do_smart_wakeup_zero(sma, q->sops, q->nsops, pt);
+			do_smart_wakeup_zero(sma, q->sops, q->nsops, wake_q);
 			restart = check_restart(sma, q);
 		}
 
-		wake_up_sem_queue_prepare(pt, q, error);
+		wake_up_sem_queue_prepare(q, error, wake_q);
 		if (restart)
 			goto again;
 	}
@@ -984,24 +954,24 @@ static void set_semotime(struct sem_array *sma, struct sembuf *sops)
  * @sops: operations that were performed
  * @nsops: number of operations
  * @otime: force setting otime
- * @pt: list head of the tasks that must be woken up.
+ * @wake_q: lockless wake-queue head
  *
  * do_smart_update() does the required calls to update_queue and wakeup_zero,
  * based on the actual changes that were performed on the semaphore array.
  * Note that the function does not do the actual wake-up: the caller is
- * responsible for calling wake_up_sem_queue_do(@pt).
+ * responsible for calling wake_up_q().
  * It is safe to perform this call after dropping all locks.
  */
 static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops,
-			int otime, struct list_head *pt)
+			    int otime, struct wake_q_head *wake_q)
 {
 	int i;
 
-	otime |= do_smart_wakeup_zero(sma, sops, nsops, pt);
+	otime |= do_smart_wakeup_zero(sma, sops, nsops, wake_q);
 
 	if (!list_empty(&sma->pending_alter)) {
 		/* semaphore array uses the global queue - just process it. */
-		otime |= update_queue(sma, -1, pt);
+		otime |= update_queue(sma, -1, wake_q);
 	} else {
 		if (!sops) {
 			/*
@@ -1009,7 +979,7 @@ static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsop
 			 * known. Check all.
 			 */
 			for (i = 0; i < sma->sem_nsems; i++)
-				otime |= update_queue(sma, i, pt);
+				otime |= update_queue(sma, i, wake_q);
 		} else {
 			/*
 			 * Check the semaphores that were increased:
@@ -1023,7 +993,7 @@ static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsop
 			for (i = 0; i < nsops; i++) {
 				if (sops[i].sem_op > 0) {
 					otime |= update_queue(sma,
-							sops[i].sem_num, pt);
+							      sops[i].sem_num, wake_q);
 				}
 			}
 		}
@@ -1111,8 +1081,8 @@ static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
 	struct sem_undo *un, *tu;
 	struct sem_queue *q, *tq;
 	struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm);
-	struct list_head tasks;
 	int i;
+	DEFINE_WAKE_Q(wake_q);
 
 	/* Free the existing undo structures for this semaphore set.  */
 	ipc_assert_locked_object(&sma->sem_perm);
@@ -1126,25 +1096,24 @@ static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
 	}
 
 	/* Wake up all pending processes and let them fail with EIDRM. */
-	INIT_LIST_HEAD(&tasks);
 	list_for_each_entry_safe(q, tq, &sma->pending_const, list) {
 		unlink_queue(sma, q);
-		wake_up_sem_queue_prepare(&tasks, q, -EIDRM);
+		wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
 	}
 
 	list_for_each_entry_safe(q, tq, &sma->pending_alter, list) {
 		unlink_queue(sma, q);
-		wake_up_sem_queue_prepare(&tasks, q, -EIDRM);
+		wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
 	}
 	for (i = 0; i < sma->sem_nsems; i++) {
 		struct sem *sem = sma->sem_base + i;
 		list_for_each_entry_safe(q, tq, &sem->pending_const, list) {
 			unlink_queue(sma, q);
-			wake_up_sem_queue_prepare(&tasks, q, -EIDRM);
+			wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
 		}
 		list_for_each_entry_safe(q, tq, &sem->pending_alter, list) {
 			unlink_queue(sma, q);
-			wake_up_sem_queue_prepare(&tasks, q, -EIDRM);
+			wake_up_sem_queue_prepare(q, -EIDRM, &wake_q);
 		}
 	}
 
@@ -1153,7 +1122,7 @@ static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
 	sem_unlock(sma, -1);
 	rcu_read_unlock();
 
-	wake_up_sem_queue_do(&tasks);
+	wake_up_q(&wake_q);
 	ns->used_sems -= sma->sem_nsems;
 	ipc_rcu_putref(sma, sem_rcu_free);
 }
@@ -1292,9 +1261,9 @@ static int semctl_setval(struct ipc_namespace *ns, int semid, int semnum,
 	struct sem_undo *un;
 	struct sem_array *sma;
 	struct sem *curr;
-	int err;
-	struct list_head tasks;
-	int val;
+	int err, val;
+	DEFINE_WAKE_Q(wake_q);
+
 #if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN)
 	/* big-endian 64bit */
 	val = arg >> 32;
@@ -1306,8 +1275,6 @@ static int semctl_setval(struct ipc_namespace *ns, int semid, int semnum,
 	if (val > SEMVMX || val < 0)
 		return -ERANGE;
 
-	INIT_LIST_HEAD(&tasks);
-
 	rcu_read_lock();
 	sma = sem_obtain_object_check(ns, semid);
 	if (IS_ERR(sma)) {
@@ -1350,10 +1317,10 @@ static int semctl_setval(struct ipc_namespace *ns, int semid, int semnum,
 	curr->sempid = task_tgid_vnr(current);
 	sma->sem_ctime = get_seconds();
 	/* maybe some queued-up processes were waiting for this */
-	do_smart_update(sma, NULL, 0, 0, &tasks);
+	do_smart_update(sma, NULL, 0, 0, &wake_q);
 	sem_unlock(sma, -1);
 	rcu_read_unlock();
-	wake_up_sem_queue_do(&tasks);
+	wake_up_q(&wake_q);
 	return 0;
 }
 
@@ -1365,9 +1332,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
 	int err, nsems;
 	ushort fast_sem_io[SEMMSL_FAST];
 	ushort *sem_io = fast_sem_io;
-	struct list_head tasks;
-
-	INIT_LIST_HEAD(&tasks);
+	DEFINE_WAKE_Q(wake_q);
 
 	rcu_read_lock();
 	sma = sem_obtain_object_check(ns, semid);
@@ -1478,7 +1443,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
 		}
 		sma->sem_ctime = get_seconds();
 		/* maybe some queued-up processes were waiting for this */
-		do_smart_update(sma, NULL, 0, 0, &tasks);
+		do_smart_update(sma, NULL, 0, 0, &wake_q);
 		err = 0;
 		goto out_unlock;
 	}
@@ -1514,7 +1479,7 @@ out_unlock:
 	sem_unlock(sma, -1);
 out_rcu_wakeup:
 	rcu_read_unlock();
-	wake_up_sem_queue_do(&tasks);
+	wake_up_q(&wake_q);
 out_free:
 	if (sem_io != fast_sem_io)
 		ipc_free(sem_io);
@@ -1787,32 +1752,6 @@ out:
 	return un;
 }
 
-
-/**
- * get_queue_result - retrieve the result code from sem_queue
- * @q: Pointer to queue structure
- *
- * Retrieve the return code from the pending queue. If IN_WAKEUP is found in
- * q->status, then we must loop until the value is replaced with the final
- * value: This may happen if a task is woken up by an unrelated event (e.g.
- * signal) and in parallel the task is woken up by another task because it got
- * the requested semaphores.
- *
- * The function can be called with or without holding the semaphore spinlock.
- */
-static int get_queue_result(struct sem_queue *q)
-{
-	int error;
-
-	error = q->status;
-	while (unlikely(error == IN_WAKEUP)) {
-		cpu_relax();
-		error = q->status;
-	}
-
-	return error;
-}
-
 SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
 		unsigned, nsops, const struct timespec __user *, timeout)
 {
@@ -1821,11 +1760,11 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
 	struct sembuf fast_sops[SEMOPM_FAST];
 	struct sembuf *sops = fast_sops, *sop;
 	struct sem_undo *un;
-	int undos = 0, alter = 0, max, locknum;
+	int max, locknum;
+	bool undos = false, alter = false, dupsop = false;
 	struct sem_queue queue;
-	unsigned long jiffies_left = 0;
+	unsigned long dup = 0, jiffies_left = 0;
 	struct ipc_namespace *ns;
-	struct list_head tasks;
 
 	ns = current->nsproxy->ipc_ns;
 
@@ -1838,10 +1777,12 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
 		if (sops == NULL)
 			return -ENOMEM;
 	}
+
 	if (copy_from_user(sops, tsops, nsops * sizeof(*tsops))) {
 		error =  -EFAULT;
 		goto out_free;
 	}
+
 	if (timeout) {
 		struct timespec _timeout;
 		if (copy_from_user(&_timeout, timeout, sizeof(*timeout))) {
@@ -1855,18 +1796,30 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
 		}
 		jiffies_left = timespec_to_jiffies(&_timeout);
 	}
+
 	max = 0;
 	for (sop = sops; sop < sops + nsops; sop++) {
+		unsigned long mask = 1ULL << ((sop->sem_num) % BITS_PER_LONG);
+
 		if (sop->sem_num >= max)
 			max = sop->sem_num;
 		if (sop->sem_flg & SEM_UNDO)
-			undos = 1;
-		if (sop->sem_op != 0)
-			alter = 1;
+			undos = true;
+		if (dup & mask) {
+			/*
+			 * There was a previous alter access that appears
+			 * to have accessed the same semaphore, thus use
+			 * the dupsop logic. "appears", because the detection
+			 * can only check % BITS_PER_LONG.
+			 */
+			dupsop = true;
+		}
+		if (sop->sem_op != 0) {
+			alter = true;
+			dup |= mask;
+		}
 	}
 
-	INIT_LIST_HEAD(&tasks);
-
 	if (undos) {
 		/* On success, find_alloc_undo takes the rcu_read_lock */
 		un = find_alloc_undo(ns, semid);
@@ -1887,16 +1840,22 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
 	}
 
 	error = -EFBIG;
-	if (max >= sma->sem_nsems)
-		goto out_rcu_wakeup;
+	if (max >= sma->sem_nsems) {
+		rcu_read_unlock();
+		goto out_free;
+	}
 
 	error = -EACCES;
-	if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO))
-		goto out_rcu_wakeup;
+	if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO)) {
+		rcu_read_unlock();
+		goto out_free;
+	}
 
 	error = security_sem_semop(sma, sops, nsops, alter);
-	if (error)
-		goto out_rcu_wakeup;
+	if (error) {
+		rcu_read_unlock();
+		goto out_free;
+	}
 
 	error = -EIDRM;
 	locknum = sem_lock(sma, sops, nsops);
@@ -1925,24 +1884,34 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
 	queue.undo = un;
 	queue.pid = task_tgid_vnr(current);
 	queue.alter = alter;
+	queue.dupsop = dupsop;
 
 	error = perform_atomic_semop(sma, &queue);
-	if (error == 0) {
-		/* If the operation was successful, then do
+	if (error == 0) { /* non-blocking succesfull path */
+		DEFINE_WAKE_Q(wake_q);
+
+		/*
+		 * If the operation was successful, then do
 		 * the required updates.
 		 */
 		if (alter)
-			do_smart_update(sma, sops, nsops, 1, &tasks);
+			do_smart_update(sma, sops, nsops, 1, &wake_q);
 		else
 			set_semotime(sma, sops);
+
+		sem_unlock(sma, locknum);
+		rcu_read_unlock();
+		wake_up_q(&wake_q);
+
+		goto out_free;
 	}
-	if (error <= 0)
+	if (error < 0) /* non-blocking error path */
 		goto out_unlock_free;
 
-	/* We need to sleep on this operation, so we put the current
+	/*
+	 * We need to sleep on this operation, so we put the current
 	 * task into the pending queue and go to sleep.
 	 */
-
 	if (nsops == 1) {
 		struct sem *curr;
 		curr = &sma->sem_base[sops->sem_num];
@@ -1971,77 +1940,69 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
 		sma->complex_count++;
 	}
 
-	queue.status = -EINTR;
-	queue.sleeper = current;
+	do {
+		queue.status = -EINTR;
+		queue.sleeper = current;
 
-sleep_again:
-	__set_current_state(TASK_INTERRUPTIBLE);
-	sem_unlock(sma, locknum);
-	rcu_read_unlock();
-
-	if (timeout)
-		jiffies_left = schedule_timeout(jiffies_left);
-	else
-		schedule();
+		__set_current_state(TASK_INTERRUPTIBLE);
+		sem_unlock(sma, locknum);
+		rcu_read_unlock();
 
-	error = get_queue_result(&queue);
+		if (timeout)
+			jiffies_left = schedule_timeout(jiffies_left);
+		else
+			schedule();
 
-	if (error != -EINTR) {
-		/* fast path: update_queue already obtained all requested
-		 * resources.
-		 * Perform a smp_mb(): User space could assume that semop()
-		 * is a memory barrier: Without the mb(), the cpu could
-		 * speculatively read in user space stale data that was
-		 * overwritten by the previous owner of the semaphore.
+		/*
+		 * fastpath: the semop has completed, either successfully or
+		 * not, from the syscall pov, is quite irrelevant to us at this
+		 * point; we're done.
+		 *
+		 * We _do_ care, nonetheless, about being awoken by a signal or
+		 * spuriously.  The queue.status is checked again in the
+		 * slowpath (aka after taking sem_lock), such that we can detect
+		 * scenarios where we were awakened externally, during the
+		 * window between wake_q_add() and wake_up_q().
 		 */
-		smp_mb();
-
-		goto out_free;
-	}
-
-	rcu_read_lock();
-	sma = sem_obtain_lock(ns, semid, sops, nsops, &locknum);
-
-	/*
-	 * Wait until it's guaranteed that no wakeup_sem_queue_do() is ongoing.
-	 */
-	error = get_queue_result(&queue);
+		error = READ_ONCE(queue.status);
+		if (error != -EINTR) {
+			/*
+			 * User space could assume that semop() is a memory
+			 * barrier: Without the mb(), the cpu could
+			 * speculatively read in userspace stale data that was
+			 * overwritten by the previous owner of the semaphore.
+			 */
+			smp_mb();
+			goto out_free;
+		}
 
-	/*
-	 * Array removed? If yes, leave without sem_unlock().
-	 */
-	if (IS_ERR(sma)) {
-		rcu_read_unlock();
-		goto out_free;
-	}
+		rcu_read_lock();
+		sem_lock(sma, sops, nsops);
 
+		if (!ipc_valid_object(&sma->sem_perm))
+			goto out_unlock_free;
 
-	/*
-	 * If queue.status != -EINTR we are woken up by another process.
-	 * Leave without unlink_queue(), but with sem_unlock().
-	 */
-	if (error != -EINTR)
-		goto out_unlock_free;
+		error = READ_ONCE(queue.status);
 
-	/*
-	 * If an interrupt occurred we have to clean up the queue
-	 */
-	if (timeout && jiffies_left == 0)
-		error = -EAGAIN;
+		/*
+		 * If queue.status != -EINTR we are woken up by another process.
+		 * Leave without unlink_queue(), but with sem_unlock().
+		 */
+		if (error != -EINTR)
+			goto out_unlock_free;
 
-	/*
-	 * If the wakeup was spurious, just retry
-	 */
-	if (error == -EINTR && !signal_pending(current))
-		goto sleep_again;
+		/*
+		 * If an interrupt occurred we have to clean up the queue.
+		 */
+		if (timeout && jiffies_left == 0)
+			error = -EAGAIN;
+	} while (error == -EINTR && !signal_pending(current)); /* spurious */
 
 	unlink_queue(sma, &queue);
 
 out_unlock_free:
 	sem_unlock(sma, locknum);
-out_rcu_wakeup:
 	rcu_read_unlock();
-	wake_up_sem_queue_do(&tasks);
 out_free:
 	if (sops != fast_sops)
 		kfree(sops);
@@ -2102,8 +2063,8 @@ void exit_sem(struct task_struct *tsk)
 	for (;;) {
 		struct sem_array *sma;
 		struct sem_undo *un;
-		struct list_head tasks;
 		int semid, i;
+		DEFINE_WAKE_Q(wake_q);
 
 		cond_resched();
 
@@ -2191,11 +2152,10 @@ void exit_sem(struct task_struct *tsk)
 			}
 		}
 		/* maybe some queued-up processes were waiting for this */
-		INIT_LIST_HEAD(&tasks);
-		do_smart_update(sma, NULL, 0, 1, &tasks);
+		do_smart_update(sma, NULL, 0, 1, &wake_q);
 		sem_unlock(sma, -1);
 		rcu_read_unlock();
-		wake_up_sem_queue_do(&tasks);
+		wake_up_q(&wake_q);
 
 		kfree_rcu(un, rcu);
 	}
diff --git a/ipc/shm.c b/ipc/shm.c
index dbac8860c721..81203e8ba013 100644
--- a/ipc/shm.c
+++ b/ipc/shm.c
@@ -89,6 +89,7 @@ void shm_init_ns(struct ipc_namespace *ns)
 static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
 {
 	struct shmid_kernel *shp;
+
 	shp = container_of(ipcp, struct shmid_kernel, shm_perm);
 
 	if (shp->shm_nattch) {
@@ -387,6 +388,7 @@ static int shm_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
 	struct file *file = vma->vm_file;
 	struct shm_file_data *sfd = shm_file_data(file);
 	int err = 0;
+
 	if (sfd->vm_ops->set_policy)
 		err = sfd->vm_ops->set_policy(vma, new);
 	return err;
@@ -417,7 +419,7 @@ static int shm_mmap(struct file *file, struct vm_area_struct *vma)
 	 * In case of remap_file_pages() emulation, the file can represent
 	 * removed IPC ID: propogate shm_lock() error to caller.
 	 */
-	ret =__shm_open(vma);
+	ret = __shm_open(vma);
 	if (ret)
 		return ret;
 
@@ -468,6 +470,7 @@ static unsigned long shm_get_unmapped_area(struct file *file,
 	unsigned long flags)
 {
 	struct shm_file_data *sfd = shm_file_data(file);
+
 	return sfd->file->f_op->get_unmapped_area(sfd->file, addr, len,
 						pgoff, flags);
 }
@@ -766,6 +769,7 @@ static void shm_add_rss_swap(struct shmid_kernel *shp,
 	} else {
 #ifdef CONFIG_SHMEM
 		struct shmem_inode_info *info = SHMEM_I(inode);
+
 		spin_lock_irq(&info->lock);
 		*rss_add += inode->i_mapping->nrpages;
 		*swp_add += info->swapped;
@@ -1028,6 +1032,7 @@ SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
 
 		if (!ns_capable(ns->user_ns, CAP_IPC_LOCK)) {
 			kuid_t euid = current_euid();
+
 			if (!uid_eq(euid, shp->shm_perm.uid) &&
 			    !uid_eq(euid, shp->shm_perm.cuid)) {
 				err = -EPERM;
@@ -1045,6 +1050,7 @@ SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
 
 		if (cmd == SHM_LOCK) {
 			struct user_struct *user = current_user();
+
 			err = shmem_lock(shm_file, 1, user);
 			if (!err && !(shp->shm_perm.mode & SHM_LOCKED)) {
 				shp->shm_perm.mode |= SHM_LOCKED;
@@ -1354,9 +1360,10 @@ SYSCALL_DEFINE1(shmdt, char __user *, shmaddr)
 		vma = next;
 	}
 
-#else /* CONFIG_MMU */
+#else	/* CONFIG_MMU */
 	/* under NOMMU conditions, the exact address to be destroyed must be
-	 * given */
+	 * given
+	 */
 	if (vma && vma->vm_start == addr && vma->vm_ops == &shm_vm_ops) {
 		do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start);
 		retval = 0;
diff --git a/kernel/Makefile b/kernel/Makefile
index eb26e12c6c2a..12c679f769c6 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -84,6 +84,7 @@ obj-$(CONFIG_KPROBES) += kprobes.o
 obj-$(CONFIG_KGDB) += debug/
 obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o
 obj-$(CONFIG_LOCKUP_DETECTOR) += watchdog.o
+obj-$(CONFIG_HARDLOCKUP_DETECTOR) += watchdog_hld.o
 obj-$(CONFIG_SECCOMP) += seccomp.o
 obj-$(CONFIG_RELAY) += relay.o
 obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
@@ -115,8 +116,6 @@ obj-$(CONFIG_HAS_IOMEM) += memremap.o
 
 $(obj)/configs.o: $(obj)/config_data.h
 
-# config_data.h contains the same information as ikconfig.h but gzipped.
-# Info from config_data can be extracted from /proc/config*
 targets += config_data.gz
 $(obj)/config_data.gz: $(KCONFIG_CONFIG) FORCE
 	$(call if_changed,gzip)
diff --git a/kernel/audit.c b/kernel/audit.c
index 67b9fbd871be..91bff3c0b368 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -107,7 +107,6 @@ static u32	audit_rate_limit;
  * When set to zero, this means unlimited. */
 static u32	audit_backlog_limit = 64;
 #define AUDIT_BACKLOG_WAIT_TIME (60 * HZ)
-static u32	audit_backlog_wait_time_master = AUDIT_BACKLOG_WAIT_TIME;
 static u32	audit_backlog_wait_time = AUDIT_BACKLOG_WAIT_TIME;
 
 /* The identity of the user shutting down the audit system. */
@@ -138,11 +137,18 @@ static DEFINE_SPINLOCK(audit_freelist_lock);
 static int	   audit_freelist_count;
 static LIST_HEAD(audit_freelist);
 
-static struct sk_buff_head audit_skb_queue;
-/* queue of skbs to send to auditd when/if it comes back */
-static struct sk_buff_head audit_skb_hold_queue;
+/* queue msgs to send via kauditd_task */
+static struct sk_buff_head audit_queue;
+/* queue msgs due to temporary unicast send problems */
+static struct sk_buff_head audit_retry_queue;
+/* queue msgs waiting for new auditd connection */
+static struct sk_buff_head audit_hold_queue;
+
+/* queue servicing thread */
 static struct task_struct *kauditd_task;
 static DECLARE_WAIT_QUEUE_HEAD(kauditd_wait);
+
+/* waitqueue for callers who are blocked on the audit backlog */
 static DECLARE_WAIT_QUEUE_HEAD(audit_backlog_wait);
 
 static struct audit_features af = {.vers = AUDIT_FEATURE_VERSION,
@@ -338,7 +344,7 @@ static int audit_set_backlog_limit(u32 limit)
 static int audit_set_backlog_wait_time(u32 timeout)
 {
 	return audit_do_config_change("audit_backlog_wait_time",
-				      &audit_backlog_wait_time_master, timeout);
+				      &audit_backlog_wait_time, timeout);
 }
 
 static int audit_set_enabled(u32 state)
@@ -365,29 +371,10 @@ static int audit_set_failure(u32 state)
 }
 
 /*
- * Queue skbs to be sent to auditd when/if it comes back.  These skbs should
- * already have been sent via prink/syslog and so if these messages are dropped
- * it is not a huge concern since we already passed the audit_log_lost()
- * notification and stuff.  This is just nice to get audit messages during
- * boot before auditd is running or messages generated while auditd is stopped.
- * This only holds messages is audit_default is set, aka booting with audit=1
- * or building your kernel that way.
- */
-static void audit_hold_skb(struct sk_buff *skb)
-{
-	if (audit_default &&
-	    (!audit_backlog_limit ||
-	     skb_queue_len(&audit_skb_hold_queue) < audit_backlog_limit))
-		skb_queue_tail(&audit_skb_hold_queue, skb);
-	else
-		kfree_skb(skb);
-}
-
-/*
  * For one reason or another this nlh isn't getting delivered to the userspace
  * audit daemon, just send it to printk.
  */
-static void audit_printk_skb(struct sk_buff *skb)
+static void kauditd_printk_skb(struct sk_buff *skb)
 {
 	struct nlmsghdr *nlh = nlmsg_hdr(skb);
 	char *data = nlmsg_data(nlh);
@@ -398,58 +385,123 @@ static void audit_printk_skb(struct sk_buff *skb)
 		else
 			audit_log_lost("printk limit exceeded");
 	}
+}
+
+/**
+ * kauditd_hold_skb - Queue an audit record, waiting for auditd
+ * @skb: audit record
+ *
+ * Description:
+ * Queue the audit record, waiting for an instance of auditd.  When this
+ * function is called we haven't given up yet on sending the record, but things
+ * are not looking good.  The first thing we want to do is try to write the
+ * record via printk and then see if we want to try and hold on to the record
+ * and queue it, if we have room.  If we want to hold on to the record, but we
+ * don't have room, record a record lost message.
+ */
+static void kauditd_hold_skb(struct sk_buff *skb)
+{
+	/* at this point it is uncertain if we will ever send this to auditd so
+	 * try to send the message via printk before we go any further */
+	kauditd_printk_skb(skb);
+
+	/* can we just silently drop the message? */
+	if (!audit_default) {
+		kfree_skb(skb);
+		return;
+	}
+
+	/* if we have room, queue the message */
+	if (!audit_backlog_limit ||
+	    skb_queue_len(&audit_hold_queue) < audit_backlog_limit) {
+		skb_queue_tail(&audit_hold_queue, skb);
+		return;
+	}
 
-	audit_hold_skb(skb);
+	/* we have no other options - drop the message */
+	audit_log_lost("kauditd hold queue overflow");
+	kfree_skb(skb);
 }
 
-static void kauditd_send_skb(struct sk_buff *skb)
+/**
+ * kauditd_retry_skb - Queue an audit record, attempt to send again to auditd
+ * @skb: audit record
+ *
+ * Description:
+ * Not as serious as kauditd_hold_skb() as we still have a connected auditd,
+ * but for some reason we are having problems sending it audit records so
+ * queue the given record and attempt to resend.
+ */
+static void kauditd_retry_skb(struct sk_buff *skb)
 {
-	int err;
-	int attempts = 0;
-#define AUDITD_RETRIES 5
+	/* NOTE: because records should only live in the retry queue for a
+	 * short period of time, before either being sent or moved to the hold
+	 * queue, we don't currently enforce a limit on this queue */
+	skb_queue_tail(&audit_retry_queue, skb);
+}
+
+/**
+ * auditd_reset - Disconnect the auditd connection
+ *
+ * Description:
+ * Break the auditd/kauditd connection and move all the records in the retry
+ * queue into the hold queue in case auditd reconnects.  The audit_cmd_mutex
+ * must be held when calling this function.
+ */
+static void auditd_reset(void)
+{
+	struct sk_buff *skb;
+
+	/* break the connection */
+	if (audit_sock) {
+		sock_put(audit_sock);
+		audit_sock = NULL;
+	}
+	audit_pid = 0;
+	audit_nlk_portid = 0;
+
+	/* flush all of the retry queue to the hold queue */
+	while ((skb = skb_dequeue(&audit_retry_queue)))
+		kauditd_hold_skb(skb);
+}
+
+/**
+ * kauditd_send_unicast_skb - Send a record via unicast to auditd
+ * @skb: audit record
+ */
+static int kauditd_send_unicast_skb(struct sk_buff *skb)
+{
+	int rc;
 
-restart:
-	/* take a reference in case we can't send it and we want to hold it */
+	/* if we know nothing is connected, don't even try the netlink call */
+	if (!audit_pid)
+		return -ECONNREFUSED;
+
+	/* get an extra skb reference in case we fail to send */
 	skb_get(skb);
-	err = netlink_unicast(audit_sock, skb, audit_nlk_portid, 0);
-	if (err < 0) {
-		pr_err("netlink_unicast sending to audit_pid=%d returned error: %d\n",
-		       audit_pid, err);
-		if (audit_pid) {
-			if (err == -ECONNREFUSED || err == -EPERM
-			    || ++attempts >= AUDITD_RETRIES) {
-				char s[32];
-
-				snprintf(s, sizeof(s), "audit_pid=%d reset", audit_pid);
-				audit_log_lost(s);
-				audit_pid = 0;
-				audit_sock = NULL;
-			} else {
-				pr_warn("re-scheduling(#%d) write to audit_pid=%d\n",
-					attempts, audit_pid);
-				set_current_state(TASK_INTERRUPTIBLE);
-				schedule();
-				goto restart;
-			}
-		}
-		/* we might get lucky and get this in the next auditd */
-		audit_hold_skb(skb);
-	} else
-		/* drop the extra reference if sent ok */
+	rc = netlink_unicast(audit_sock, skb, audit_nlk_portid, 0);
+	if (rc >= 0) {
 		consume_skb(skb);
+		rc = 0;
+	}
+
+	return rc;
 }
 
 /*
- * kauditd_send_multicast_skb - send the skb to multicast userspace listeners
+ * kauditd_send_multicast_skb - Send a record to any multicast listeners
+ * @skb: audit record
  *
+ * Description:
  * This function doesn't consume an skb as might be expected since it has to
  * copy it anyways.
  */
-static void kauditd_send_multicast_skb(struct sk_buff *skb, gfp_t gfp_mask)
+static void kauditd_send_multicast_skb(struct sk_buff *skb)
 {
-	struct sk_buff		*copy;
-	struct audit_net	*aunet = net_generic(&init_net, audit_net_id);
-	struct sock		*sock = aunet->nlsk;
+	struct sk_buff *copy;
+	struct audit_net *aunet = net_generic(&init_net, audit_net_id);
+	struct sock *sock = aunet->nlsk;
+	struct nlmsghdr *nlh;
 
 	if (!netlink_has_listeners(sock, AUDIT_NLGRP_READLOG))
 		return;
@@ -464,74 +516,161 @@ static void kauditd_send_multicast_skb(struct sk_buff *skb, gfp_t gfp_mask)
 	 * no reason for new multicast clients to continue with this
 	 * non-compliance.
 	 */
-	copy = skb_copy(skb, gfp_mask);
+	copy = skb_copy(skb, GFP_KERNEL);
 	if (!copy)
 		return;
+	nlh = nlmsg_hdr(copy);
+	nlh->nlmsg_len = skb->len;
 
-	nlmsg_multicast(sock, copy, 0, AUDIT_NLGRP_READLOG, gfp_mask);
+	nlmsg_multicast(sock, copy, 0, AUDIT_NLGRP_READLOG, GFP_KERNEL);
 }
 
-/*
- * flush_hold_queue - empty the hold queue if auditd appears
- *
- * If auditd just started, drain the queue of messages already
- * sent to syslog/printk.  Remember loss here is ok.  We already
- * called audit_log_lost() if it didn't go out normally.  so the
- * race between the skb_dequeue and the next check for audit_pid
- * doesn't matter.
+/**
+ * kauditd_wake_condition - Return true when it is time to wake kauditd_thread
  *
- * If you ever find kauditd to be too slow we can get a perf win
- * by doing our own locking and keeping better track if there
- * are messages in this queue.  I don't see the need now, but
- * in 5 years when I want to play with this again I'll see this
- * note and still have no friggin idea what i'm thinking today.
+ * Description:
+ * This function is for use by the wait_event_freezable() call in
+ * kauditd_thread().
  */
-static void flush_hold_queue(void)
+static int kauditd_wake_condition(void)
 {
-	struct sk_buff *skb;
-
-	if (!audit_default || !audit_pid)
-		return;
-
-	skb = skb_dequeue(&audit_skb_hold_queue);
-	if (likely(!skb))
-		return;
+	static int pid_last = 0;
+	int rc;
+	int pid = audit_pid;
 
-	while (skb && audit_pid) {
-		kauditd_send_skb(skb);
-		skb = skb_dequeue(&audit_skb_hold_queue);
-	}
+	/* wake on new messages or a change in the connected auditd */
+	rc = skb_queue_len(&audit_queue) || (pid && pid != pid_last);
+	if (rc)
+		pid_last = pid;
 
-	/*
-	 * if auditd just disappeared but we
-	 * dequeued an skb we need to drop ref
-	 */
-	consume_skb(skb);
+	return rc;
 }
 
 static int kauditd_thread(void *dummy)
 {
+	int rc;
+	int auditd = 0;
+	int reschedule = 0;
+	struct sk_buff *skb;
+	struct nlmsghdr *nlh;
+
+#define UNICAST_RETRIES 5
+#define AUDITD_BAD(x,y) \
+	((x) == -ECONNREFUSED || (x) == -EPERM || ++(y) >= UNICAST_RETRIES)
+
+	/* NOTE: we do invalidate the auditd connection flag on any sending
+	 * errors, but we only "restore" the connection flag at specific places
+	 * in the loop in order to help ensure proper ordering of audit
+	 * records */
+
 	set_freezable();
 	while (!kthread_should_stop()) {
-		struct sk_buff *skb;
-
-		flush_hold_queue();
+		/* NOTE: possible area for future improvement is to look at
+		 *       the hold and retry queues, since only this thread
+		 *       has access to these queues we might be able to do
+		 *       our own queuing and skip some/all of the locking */
+
+		/* NOTE: it might be a fun experiment to split the hold and
+		 *       retry queue handling to another thread, but the
+		 *       synchronization issues and other overhead might kill
+		 *       any performance gains */
+
+		/* attempt to flush the hold queue */
+		while (auditd && (skb = skb_dequeue(&audit_hold_queue))) {
+			rc = kauditd_send_unicast_skb(skb);
+			if (rc) {
+				/* requeue to the same spot */
+				skb_queue_head(&audit_hold_queue, skb);
+
+				auditd = 0;
+				if (AUDITD_BAD(rc, reschedule)) {
+					mutex_lock(&audit_cmd_mutex);
+					auditd_reset();
+					mutex_unlock(&audit_cmd_mutex);
+					reschedule = 0;
+				}
+			} else
+				/* we were able to send successfully */
+				reschedule = 0;
+		}
 
-		skb = skb_dequeue(&audit_skb_queue);
+		/* attempt to flush the retry queue */
+		while (auditd && (skb = skb_dequeue(&audit_retry_queue))) {
+			rc = kauditd_send_unicast_skb(skb);
+			if (rc) {
+				auditd = 0;
+				if (AUDITD_BAD(rc, reschedule)) {
+					kauditd_hold_skb(skb);
+					mutex_lock(&audit_cmd_mutex);
+					auditd_reset();
+					mutex_unlock(&audit_cmd_mutex);
+					reschedule = 0;
+				} else
+					/* temporary problem (we hope), queue
+					 * to the same spot and retry */
+					skb_queue_head(&audit_retry_queue, skb);
+			} else
+				/* we were able to send successfully */
+				reschedule = 0;
+		}
 
+		/* standard queue processing, try to be as quick as possible */
+quick_loop:
+		skb = skb_dequeue(&audit_queue);
 		if (skb) {
-			if (!audit_backlog_limit ||
-			    (skb_queue_len(&audit_skb_queue) <= audit_backlog_limit))
-				wake_up(&audit_backlog_wait);
-			if (audit_pid)
-				kauditd_send_skb(skb);
+			/* setup the netlink header, see the comments in
+			 * kauditd_send_multicast_skb() for length quirks */
+			nlh = nlmsg_hdr(skb);
+			nlh->nlmsg_len = skb->len - NLMSG_HDRLEN;
+
+			/* attempt to send to any multicast listeners */
+			kauditd_send_multicast_skb(skb);
+
+			/* attempt to send to auditd, queue on failure */
+			if (auditd) {
+				rc = kauditd_send_unicast_skb(skb);
+				if (rc) {
+					auditd = 0;
+					if (AUDITD_BAD(rc, reschedule)) {
+						mutex_lock(&audit_cmd_mutex);
+						auditd_reset();
+						mutex_unlock(&audit_cmd_mutex);
+						reschedule = 0;
+					}
+
+					/* move to the retry queue */
+					kauditd_retry_skb(skb);
+				} else
+					/* everything is working so go fast! */
+					goto quick_loop;
+			} else if (reschedule)
+				/* we are currently having problems, move to
+				 * the retry queue */
+				kauditd_retry_skb(skb);
 			else
-				audit_printk_skb(skb);
-			continue;
-		}
+				/* dump the message via printk and hold it */
+				kauditd_hold_skb(skb);
+		} else {
+			/* we have flushed the backlog so wake everyone */
+			wake_up(&audit_backlog_wait);
+
+			/* if everything is okay with auditd (if present), go
+			 * to sleep until there is something new in the queue
+			 * or we have a change in the connected auditd;
+			 * otherwise simply reschedule to give things a chance
+			 * to recover */
+			if (reschedule) {
+				set_current_state(TASK_INTERRUPTIBLE);
+				schedule();
+			} else
+				wait_event_freezable(kauditd_wait,
+						     kauditd_wake_condition());
 
-		wait_event_freezable(kauditd_wait, skb_queue_len(&audit_skb_queue));
+			/* update the auditd connection status */
+			auditd = (audit_pid ? 1 : 0);
+		}
 	}
+
 	return 0;
 }
 
@@ -596,6 +735,7 @@ static int audit_send_reply_thread(void *arg)
 	kfree(reply);
 	return 0;
 }
+
 /**
  * audit_send_reply - send an audit reply message via netlink
  * @request_skb: skb of request we are replying to (used to target the reply)
@@ -832,16 +972,6 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
 	if (err)
 		return err;
 
-	/* As soon as there's any sign of userspace auditd,
-	 * start kauditd to talk to it */
-	if (!kauditd_task) {
-		kauditd_task = kthread_run(kauditd_thread, NULL, "kauditd");
-		if (IS_ERR(kauditd_task)) {
-			err = PTR_ERR(kauditd_task);
-			kauditd_task = NULL;
-			return err;
-		}
-	}
 	seq  = nlh->nlmsg_seq;
 	data = nlmsg_data(nlh);
 
@@ -855,9 +985,9 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
 		s.rate_limit		= audit_rate_limit;
 		s.backlog_limit		= audit_backlog_limit;
 		s.lost			= atomic_read(&audit_lost);
-		s.backlog		= skb_queue_len(&audit_skb_queue);
+		s.backlog		= skb_queue_len(&audit_queue);
 		s.feature_bitmap	= AUDIT_FEATURE_BITMAP_ALL;
-		s.backlog_wait_time	= audit_backlog_wait_time_master;
+		s.backlog_wait_time	= audit_backlog_wait_time;
 		audit_send_reply(skb, seq, AUDIT_GET, 0, 0, &s, sizeof(s));
 		break;
 	}
@@ -897,9 +1027,17 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
 			}
 			if (audit_enabled != AUDIT_OFF)
 				audit_log_config_change("audit_pid", new_pid, audit_pid, 1);
-			audit_pid = new_pid;
-			audit_nlk_portid = NETLINK_CB(skb).portid;
-			audit_sock = skb->sk;
+			if (new_pid) {
+				if (audit_sock)
+					sock_put(audit_sock);
+				audit_pid = new_pid;
+				audit_nlk_portid = NETLINK_CB(skb).portid;
+				sock_hold(skb->sk);
+				audit_sock = skb->sk;
+			} else {
+				auditd_reset();
+			}
+			wake_up_interruptible(&kauditd_wait);
 		}
 		if (s.mask & AUDIT_STATUS_RATE_LIMIT) {
 			err = audit_set_rate_limit(s.rate_limit);
@@ -1167,10 +1305,10 @@ static void __net_exit audit_net_exit(struct net *net)
 {
 	struct audit_net *aunet = net_generic(net, audit_net_id);
 	struct sock *sock = aunet->nlsk;
-	if (sock == audit_sock) {
-		audit_pid = 0;
-		audit_sock = NULL;
-	}
+	mutex_lock(&audit_cmd_mutex);
+	if (sock == audit_sock)
+		auditd_reset();
+	mutex_unlock(&audit_cmd_mutex);
 
 	netlink_kernel_release(sock);
 	aunet->nlsk = NULL;
@@ -1195,17 +1333,24 @@ static int __init audit_init(void)
 		audit_default ? "enabled" : "disabled");
 	register_pernet_subsys(&audit_net_ops);
 
-	skb_queue_head_init(&audit_skb_queue);
-	skb_queue_head_init(&audit_skb_hold_queue);
+	skb_queue_head_init(&audit_queue);
+	skb_queue_head_init(&audit_retry_queue);
+	skb_queue_head_init(&audit_hold_queue);
 	audit_initialized = AUDIT_INITIALIZED;
 	audit_enabled = audit_default;
 	audit_ever_enabled |= !!audit_default;
 
-	audit_log(NULL, GFP_KERNEL, AUDIT_KERNEL, "initialized");
-
 	for (i = 0; i < AUDIT_INODE_BUCKETS; i++)
 		INIT_LIST_HEAD(&audit_inode_hash[i]);
 
+	kauditd_task = kthread_run(kauditd_thread, NULL, "kauditd");
+	if (IS_ERR(kauditd_task)) {
+		int err = PTR_ERR(kauditd_task);
+		panic("audit: failed to start the kauditd thread (%d)\n", err);
+	}
+
+	audit_log(NULL, GFP_KERNEL, AUDIT_KERNEL, "initialized");
+
 	return 0;
 }
 __initcall(audit_init);
@@ -1338,24 +1483,6 @@ static inline void audit_get_stamp(struct audit_context *ctx,
 	}
 }
 
-/*
- * Wait for auditd to drain the queue a little
- */
-static long wait_for_auditd(long sleep_time)
-{
-	DECLARE_WAITQUEUE(wait, current);
-
-	if (audit_backlog_limit &&
-	    skb_queue_len(&audit_skb_queue) > audit_backlog_limit) {
-		add_wait_queue_exclusive(&audit_backlog_wait, &wait);
-		set_current_state(TASK_UNINTERRUPTIBLE);
-		sleep_time = schedule_timeout(sleep_time);
-		remove_wait_queue(&audit_backlog_wait, &wait);
-	}
-
-	return sleep_time;
-}
-
 /**
  * audit_log_start - obtain an audit buffer
  * @ctx: audit_context (may be NULL)
@@ -1374,12 +1501,9 @@ static long wait_for_auditd(long sleep_time)
 struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask,
 				     int type)
 {
-	struct audit_buffer	*ab	= NULL;
-	struct timespec		t;
-	unsigned int		uninitialized_var(serial);
-	int reserve = 5; /* Allow atomic callers to go up to five
-			    entries over the normal backlog limit */
-	unsigned long timeout_start = jiffies;
+	struct audit_buffer *ab;
+	struct timespec t;
+	unsigned int uninitialized_var(serial);
 
 	if (audit_initialized != AUDIT_INITIALIZED)
 		return NULL;
@@ -1387,38 +1511,48 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask,
 	if (unlikely(!audit_filter(type, AUDIT_FILTER_TYPE)))
 		return NULL;
 
-	if (gfp_mask & __GFP_DIRECT_RECLAIM) {
-		if (audit_pid && audit_pid == current->tgid)
-			gfp_mask &= ~__GFP_DIRECT_RECLAIM;
-		else
-			reserve = 0;
-	}
-
-	while (audit_backlog_limit
-	       && skb_queue_len(&audit_skb_queue) > audit_backlog_limit + reserve) {
-		if (gfp_mask & __GFP_DIRECT_RECLAIM && audit_backlog_wait_time) {
-			long sleep_time;
+	/* don't ever fail/sleep on these two conditions:
+	 * 1. auditd generated record - since we need auditd to drain the
+	 *    queue; also, when we are checking for auditd, compare PIDs using
+	 *    task_tgid_vnr() since auditd_pid is set in audit_receive_msg()
+	 *    using a PID anchored in the caller's namespace
+	 * 2. audit command message - record types 1000 through 1099 inclusive
+	 *    are command messages/records used to manage the kernel subsystem
+	 *    and the audit userspace, blocking on these messages could cause
+	 *    problems under load so don't do it (note: not all of these
+	 *    command types are valid as record types, but it is quicker to
+	 *    just check two ints than a series of ints in a if/switch stmt) */
+	if (!((audit_pid && audit_pid == task_tgid_vnr(current)) ||
+	      (type >= 1000 && type <= 1099))) {
+		long sleep_time = audit_backlog_wait_time;
+
+		while (audit_backlog_limit &&
+		       (skb_queue_len(&audit_queue) > audit_backlog_limit)) {
+			/* wake kauditd to try and flush the queue */
+			wake_up_interruptible(&kauditd_wait);
 
-			sleep_time = timeout_start + audit_backlog_wait_time - jiffies;
-			if (sleep_time > 0) {
-				sleep_time = wait_for_auditd(sleep_time);
-				if (sleep_time > 0)
-					continue;
+			/* sleep if we are allowed and we haven't exhausted our
+			 * backlog wait limit */
+			if ((gfp_mask & __GFP_DIRECT_RECLAIM) &&
+			    (sleep_time > 0)) {
+				DECLARE_WAITQUEUE(wait, current);
+
+				add_wait_queue_exclusive(&audit_backlog_wait,
+							 &wait);
+				set_current_state(TASK_UNINTERRUPTIBLE);
+				sleep_time = schedule_timeout(sleep_time);
+				remove_wait_queue(&audit_backlog_wait, &wait);
+			} else {
+				if (audit_rate_check() && printk_ratelimit())
+					pr_warn("audit_backlog=%d > audit_backlog_limit=%d\n",
+						skb_queue_len(&audit_queue),
+						audit_backlog_limit);
+				audit_log_lost("backlog limit exceeded");
+				return NULL;
 			}
 		}
-		if (audit_rate_check() && printk_ratelimit())
-			pr_warn("audit_backlog=%d > audit_backlog_limit=%d\n",
-				skb_queue_len(&audit_skb_queue),
-				audit_backlog_limit);
-		audit_log_lost("backlog limit exceeded");
-		audit_backlog_wait_time = 0;
-		wake_up(&audit_backlog_wait);
-		return NULL;
 	}
 
-	if (!reserve && !audit_backlog_wait_time)
-		audit_backlog_wait_time = audit_backlog_wait_time_master;
-
 	ab = audit_buffer_alloc(ctx, gfp_mask, type);
 	if (!ab) {
 		audit_log_lost("out of memory in audit_log_start");
@@ -1426,9 +1560,9 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask,
 	}
 
 	audit_get_stamp(ab->ctx, &t, &serial);
-
 	audit_log_format(ab, "audit(%lu.%03lu:%u): ",
 			 t.tv_sec, t.tv_nsec/1000000, serial);
+
 	return ab;
 }
 
@@ -1978,10 +2112,10 @@ out:
  * audit_log_end - end one audit record
  * @ab: the audit_buffer
  *
- * netlink_unicast() cannot be called inside an irq context because it blocks
- * (last arg, flags, is not set to MSG_DONTWAIT), so the audit buffer is placed
- * on a queue and a tasklet is scheduled to remove them from the queue outside
- * the irq context.  May be called in any context.
+ * We can not do a netlink send inside an irq context because it blocks (last
+ * arg, flags, is not set to MSG_DONTWAIT), so the audit buffer is placed on a
+ * queue and a tasklet is scheduled to remove them from the queue outside the
+ * irq context.  May be called in any context.
  */
 void audit_log_end(struct audit_buffer *ab)
 {
@@ -1990,28 +2124,8 @@ void audit_log_end(struct audit_buffer *ab)
 	if (!audit_rate_check()) {
 		audit_log_lost("rate limit exceeded");
 	} else {
-		struct nlmsghdr *nlh = nlmsg_hdr(ab->skb);
-
-		nlh->nlmsg_len = ab->skb->len;
-		kauditd_send_multicast_skb(ab->skb, ab->gfp_mask);
-
-		/*
-		 * The original kaudit unicast socket sends up messages with
-		 * nlmsg_len set to the payload length rather than the entire
-		 * message length.  This breaks the standard set by netlink.
-		 * The existing auditd daemon assumes this breakage.  Fixing
-		 * this would require co-ordinating a change in the established
-		 * protocol between the kaudit kernel subsystem and the auditd
-		 * userspace code.
-		 */
-		nlh->nlmsg_len -= NLMSG_HDRLEN;
-
-		if (audit_pid) {
-			skb_queue_tail(&audit_skb_queue, ab->skb);
-			wake_up_interruptible(&kauditd_wait);
-		} else {
-			audit_printk_skb(ab->skb);
-		}
+		skb_queue_tail(&audit_queue, ab->skb);
+		wake_up_interruptible(&kauditd_wait);
 		ab->skb = NULL;
 	}
 	audit_buffer_free(ab);
diff --git a/kernel/audit_fsnotify.c b/kernel/audit_fsnotify.c
index f84f8d06e1f6..f75154889aa9 100644
--- a/kernel/audit_fsnotify.c
+++ b/kernel/audit_fsnotify.c
@@ -130,10 +130,9 @@ static void audit_mark_log_rule_change(struct audit_fsnotify_mark *audit_mark, c
 	ab = audit_log_start(NULL, GFP_NOFS, AUDIT_CONFIG_CHANGE);
 	if (unlikely(!ab))
 		return;
-	audit_log_format(ab, "auid=%u ses=%u op=",
+	audit_log_format(ab, "auid=%u ses=%u op=%s",
 			 from_kuid(&init_user_ns, audit_get_loginuid(current)),
-			 audit_get_sessionid(current));
-	audit_log_string(ab, op);
+			 audit_get_sessionid(current), op);
 	audit_log_format(ab, " path=");
 	audit_log_untrustedstring(ab, audit_mark->path);
 	audit_log_key(ab, rule->filterkey);
diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c
index 25772476fa4a..055f11b0a50f 100644
--- a/kernel/audit_tree.c
+++ b/kernel/audit_tree.c
@@ -458,8 +458,7 @@ static void audit_tree_log_remove_rule(struct audit_krule *rule)
 	ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE);
 	if (unlikely(!ab))
 		return;
-	audit_log_format(ab, "op=");
-	audit_log_string(ab, "remove_rule");
+	audit_log_format(ab, "op=remove_rule");
 	audit_log_format(ab, " dir=");
 	audit_log_untrustedstring(ab, rule->tree->pathname);
 	audit_log_key(ab, rule->filterkey);
diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c
index 0d302a87f21b..686e068ec3da 100644
--- a/kernel/audit_watch.c
+++ b/kernel/audit_watch.c
@@ -242,10 +242,9 @@ static void audit_watch_log_rule_change(struct audit_krule *r, struct audit_watc
 		ab = audit_log_start(NULL, GFP_NOFS, AUDIT_CONFIG_CHANGE);
 		if (unlikely(!ab))
 			return;
-		audit_log_format(ab, "auid=%u ses=%u op=",
+		audit_log_format(ab, "auid=%u ses=%u op=%s",
 				 from_kuid(&init_user_ns, audit_get_loginuid(current)),
-				 audit_get_sessionid(current));
-		audit_log_string(ab, op);
+				 audit_get_sessionid(current), op);
 		audit_log_format(ab, " path=");
 		audit_log_untrustedstring(ab, w->path);
 		audit_log_key(ab, r->filterkey);
diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c
index 85d9cac497e4..880519d6cf2a 100644
--- a/kernel/auditfilter.c
+++ b/kernel/auditfilter.c
@@ -363,6 +363,7 @@ static int audit_field_valid(struct audit_entry *entry, struct audit_field *f)
 	case AUDIT_EXIT:
 	case AUDIT_SUCCESS:
 	case AUDIT_INODE:
+	case AUDIT_SESSIONID:
 		/* bit ops are only useful on syscall args */
 		if (f->op == Audit_bitmask || f->op == Audit_bittest)
 			return -EINVAL;
@@ -476,6 +477,7 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data,
 			if (!gid_valid(f->gid))
 				goto exit_free;
 			break;
+		case AUDIT_SESSIONID:
 		case AUDIT_ARCH:
 			entry->rule.arch_f = f;
 			break;
@@ -1074,8 +1076,7 @@ static void audit_log_rule_change(char *action, struct audit_krule *rule, int re
 		return;
 	audit_log_format(ab, "auid=%u ses=%u" ,loginuid, sessionid);
 	audit_log_task_context(ab);
-	audit_log_format(ab, " op=");
-	audit_log_string(ab, action);
+	audit_log_format(ab, " op=%s", action);
 	audit_log_key(ab, rule->filterkey);
 	audit_log_format(ab, " list=%d res=%d", rule->listnr, res);
 	audit_log_end(ab);
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index 2cd5256dbff7..cf1fa43512c1 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -446,6 +446,7 @@ static int audit_filter_rules(struct task_struct *tsk,
 	const struct cred *cred;
 	int i, need_sid = 1;
 	u32 sid;
+	unsigned int sessionid;
 
 	cred = rcu_dereference_check(tsk->cred, tsk == current || task_creation);
 
@@ -508,6 +509,10 @@ static int audit_filter_rules(struct task_struct *tsk,
 		case AUDIT_FSGID:
 			result = audit_gid_comparator(cred->fsgid, f->op, f->gid);
 			break;
+		case AUDIT_SESSIONID:
+			sessionid = audit_get_sessionid(current);
+			result = audit_comparator(sessionid, f->op, f->val);
+			break;
 		case AUDIT_PERS:
 			result = audit_comparator(tsk->personality, f->op, f->val);
 			break;
@@ -1000,7 +1005,7 @@ static void audit_log_execve_info(struct audit_context *context,
 	long len_rem;
 	long len_full;
 	long len_buf;
-	long len_abuf;
+	long len_abuf = 0;
 	long len_tmp;
 	bool require_data;
 	bool encode;
@@ -2025,8 +2030,11 @@ int audit_set_loginuid(kuid_t loginuid)
 		goto out;
 
 	/* are we setting or clearing? */
-	if (uid_valid(loginuid))
+	if (uid_valid(loginuid)) {
 		sessionid = (unsigned int)atomic_inc_return(&session_id);
+		if (unlikely(sessionid == (unsigned int)-1))
+			sessionid = (unsigned int)atomic_inc_return(&session_id);
+	}
 
 	task->sessionid = sessionid;
 	task->loginuid = loginuid;
diff --git a/kernel/capability.c b/kernel/capability.c
index 00411c82dac5..4984e1f552eb 100644
--- a/kernel/capability.c
+++ b/kernel/capability.c
@@ -457,6 +457,19 @@ bool file_ns_capable(const struct file *file, struct user_namespace *ns,
 EXPORT_SYMBOL(file_ns_capable);
 
 /**
+ * privileged_wrt_inode_uidgid - Do capabilities in the namespace work over the inode?
+ * @ns: The user namespace in question
+ * @inode: The inode in question
+ *
+ * Return true if the inode uid and gid are within the namespace.
+ */
+bool privileged_wrt_inode_uidgid(struct user_namespace *ns, const struct inode *inode)
+{
+	return kuid_has_mapping(ns, inode->i_uid) &&
+		kgid_has_mapping(ns, inode->i_gid);
+}
+
+/**
  * capable_wrt_inode_uidgid - Check nsown_capable and uid and gid mapped
  * @inode: The inode in question
  * @cap: The capability in question
@@ -469,7 +482,26 @@ bool capable_wrt_inode_uidgid(const struct inode *inode, int cap)
 {
 	struct user_namespace *ns = current_user_ns();
 
-	return ns_capable(ns, cap) && kuid_has_mapping(ns, inode->i_uid) &&
-		kgid_has_mapping(ns, inode->i_gid);
+	return ns_capable(ns, cap) && privileged_wrt_inode_uidgid(ns, inode);
 }
 EXPORT_SYMBOL(capable_wrt_inode_uidgid);
+
+/**
+ * ptracer_capable - Determine if the ptracer holds CAP_SYS_PTRACE in the namespace
+ * @tsk: The task that may be ptraced
+ * @ns: The user namespace to search for CAP_SYS_PTRACE in
+ *
+ * Return true if the task that is ptracing the current task had CAP_SYS_PTRACE
+ * in the specified user namespace.
+ */
+bool ptracer_capable(struct task_struct *tsk, struct user_namespace *ns)
+{
+	int ret = 0;  /* An absent tracer adds no restrictions */
+	const struct cred *cred;
+	rcu_read_lock();
+	cred = rcu_dereference(tsk->ptracer_cred);
+	if (cred)
+		ret = security_capable_noaudit(cred, ns, CAP_SYS_PTRACE);
+	rcu_read_unlock();
+	return (ret == 0);
+}
diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c
index 0874e2edd275..79517e5549f1 100644
--- a/kernel/debug/debug_core.c
+++ b/kernel/debug/debug_core.c
@@ -598,11 +598,11 @@ return_normal:
 	/*
 	 * Wait for the other CPUs to be notified and be waiting for us:
 	 */
-	time_left = loops_per_jiffy * HZ;
+	time_left = MSEC_PER_SEC;
 	while (kgdb_do_roundup && --time_left &&
 	       (atomic_read(&masters_in_kgdb) + atomic_read(&slaves_in_kgdb)) !=
 		   online_cpus)
-		cpu_relax();
+		udelay(1000);
 	if (!time_left)
 		pr_crit("Timed out waiting for secondary CPUs.\n");
 
diff --git a/kernel/debug/kdb/kdb_io.c b/kernel/debug/kdb/kdb_io.c
index 98c9011eac78..e74be38245ad 100644
--- a/kernel/debug/kdb/kdb_io.c
+++ b/kernel/debug/kdb/kdb_io.c
@@ -30,6 +30,7 @@
 char kdb_prompt_str[CMD_BUFLEN];
 
 int kdb_trap_printk;
+int kdb_printf_cpu = -1;
 
 static int kgdb_transition_check(char *buffer)
 {
@@ -554,31 +555,26 @@ int vkdb_printf(enum kdb_msgsrc src, const char *fmt, va_list ap)
 	int linecount;
 	int colcount;
 	int logging, saved_loglevel = 0;
-	int saved_trap_printk;
-	int got_printf_lock = 0;
 	int retlen = 0;
 	int fnd, len;
+	int this_cpu, old_cpu;
 	char *cp, *cp2, *cphold = NULL, replaced_byte = ' ';
 	char *moreprompt = "more> ";
 	struct console *c = console_drivers;
-	static DEFINE_SPINLOCK(kdb_printf_lock);
 	unsigned long uninitialized_var(flags);
 
-	preempt_disable();
-	saved_trap_printk = kdb_trap_printk;
-	kdb_trap_printk = 0;
-
 	/* Serialize kdb_printf if multiple cpus try to write at once.
 	 * But if any cpu goes recursive in kdb, just print the output,
 	 * even if it is interleaved with any other text.
 	 */
-	if (!KDB_STATE(PRINTF_LOCK)) {
-		KDB_STATE_SET(PRINTF_LOCK);
-		spin_lock_irqsave(&kdb_printf_lock, flags);
-		got_printf_lock = 1;
-		atomic_inc(&kdb_event);
-	} else {
-		__acquire(kdb_printf_lock);
+	local_irq_save(flags);
+	this_cpu = smp_processor_id();
+	for (;;) {
+		old_cpu = cmpxchg(&kdb_printf_cpu, -1, this_cpu);
+		if (old_cpu == -1 || old_cpu == this_cpu)
+			break;
+
+		cpu_relax();
 	}
 
 	diag = kdbgetintenv("LINES", &linecount);
@@ -847,16 +843,9 @@ kdb_print_out:
 	suspend_grep = 0; /* end of what may have been a recursive call */
 	if (logging)
 		console_loglevel = saved_loglevel;
-	if (KDB_STATE(PRINTF_LOCK) && got_printf_lock) {
-		got_printf_lock = 0;
-		spin_unlock_irqrestore(&kdb_printf_lock, flags);
-		KDB_STATE_CLEAR(PRINTF_LOCK);
-		atomic_dec(&kdb_event);
-	} else {
-		__release(kdb_printf_lock);
-	}
-	kdb_trap_printk = saved_trap_printk;
-	preempt_enable();
+	/* kdb_printf_cpu locked the code above. */
+	smp_store_release(&kdb_printf_cpu, old_cpu);
+	local_irq_restore(flags);
 	return retlen;
 }
 
diff --git a/kernel/debug/kdb/kdb_main.c b/kernel/debug/kdb/kdb_main.c
index 2a20c0dfdafc..ca183919d302 100644
--- a/kernel/debug/kdb/kdb_main.c
+++ b/kernel/debug/kdb/kdb_main.c
@@ -60,7 +60,6 @@ int kdb_grep_trailing;
  * Kernel debugger state flags
  */
 int kdb_flags;
-atomic_t kdb_event;
 
 /*
  * kdb_lock protects updates to kdb_initial_cpu.  Used to
diff --git a/kernel/debug/kdb/kdb_private.h b/kernel/debug/kdb/kdb_private.h
index 75014d7f4568..fc224fbcf954 100644
--- a/kernel/debug/kdb/kdb_private.h
+++ b/kernel/debug/kdb/kdb_private.h
@@ -132,7 +132,6 @@ extern int kdb_state;
 #define KDB_STATE_PAGER		0x00000400	/* pager is available */
 #define KDB_STATE_GO_SWITCH	0x00000800	/* go is switching
 						 * back to initial cpu */
-#define KDB_STATE_PRINTF_LOCK	0x00001000	/* Holds kdb_printf lock */
 #define KDB_STATE_WAIT_IPI	0x00002000	/* Waiting for kdb_ipi() NMI */
 #define KDB_STATE_RECURSE	0x00004000	/* Recursive entry to kdb */
 #define KDB_STATE_IP_ADJUSTED	0x00008000	/* Restart IP has been
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index f9ec9add2164..215871bda3a2 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -301,7 +301,7 @@ int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr,
 retry:
 	/* Read the page with vaddr into memory */
 	ret = get_user_pages_remote(NULL, mm, vaddr, 1, FOLL_FORCE, &old_page,
-			&vma);
+			&vma, NULL);
 	if (ret <= 0)
 		return ret;
 
@@ -1712,7 +1712,7 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
 	 * essentially a kernel access to the memory.
 	 */
 	result = get_user_pages_remote(NULL, mm, vaddr, 1, FOLL_FORCE, &page,
-			NULL);
+			NULL, NULL);
 	if (result < 0)
 		return result;
 
diff --git a/kernel/fork.c b/kernel/fork.c
index 7377f414f3ce..869b8ccc00bf 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -747,7 +747,8 @@ static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
 #endif
 }
 
-static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
+static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
+	struct user_namespace *user_ns)
 {
 	mm->mmap = NULL;
 	mm->mm_rb = RB_ROOT;
@@ -787,6 +788,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
 	if (init_new_context(p, mm))
 		goto fail_nocontext;
 
+	mm->user_ns = get_user_ns(user_ns);
 	return mm;
 
 fail_nocontext:
@@ -832,7 +834,7 @@ struct mm_struct *mm_alloc(void)
 		return NULL;
 
 	memset(mm, 0, sizeof(*mm));
-	return mm_init(mm, current);
+	return mm_init(mm, current, current_user_ns());
 }
 
 /*
@@ -847,6 +849,7 @@ void __mmdrop(struct mm_struct *mm)
 	destroy_context(mm);
 	mmu_notifier_mm_destroy(mm);
 	check_mm(mm);
+	put_user_ns(mm->user_ns);
 	free_mm(mm);
 }
 EXPORT_SYMBOL_GPL(__mmdrop);
@@ -1128,7 +1131,7 @@ static struct mm_struct *dup_mm(struct task_struct *tsk)
 
 	memcpy(mm, oldmm, sizeof(*mm));
 
-	if (!mm_init(mm, tsk))
+	if (!mm_init(mm, tsk, mm->user_ns))
 		goto fail_nomem;
 
 	err = dup_mmap(mm, oldmm);
@@ -1544,7 +1547,7 @@ static __latent_entropy struct task_struct *copy_process(
 		goto bad_fork_cleanup_count;
 
 	delayacct_tsk_init(p);	/* Must remain after dup_task_struct() */
-	p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER);
+	p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE);
 	p->flags |= PF_FORKNOEXEC;
 	INIT_LIST_HEAD(&p->children);
 	INIT_LIST_HEAD(&p->sibling);
diff --git a/kernel/kcov.c b/kernel/kcov.c
index 3cbb0c879705..cc2fa35ca480 100644
--- a/kernel/kcov.c
+++ b/kernel/kcov.c
@@ -1,11 +1,16 @@
 #define pr_fmt(fmt) "kcov: " fmt
 
 #define DISABLE_BRANCH_PROFILING
+#include <linux/atomic.h>
 #include <linux/compiler.h>
+#include <linux/errno.h>
+#include <linux/export.h>
 #include <linux/types.h>
 #include <linux/file.h>
 #include <linux/fs.h>
+#include <linux/init.h>
 #include <linux/mm.h>
+#include <linux/preempt.h>
 #include <linux/printk.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c
index 561675589511..5617cc412444 100644
--- a/kernel/kexec_core.c
+++ b/kernel/kexec_core.c
@@ -441,6 +441,8 @@ static struct page *kimage_alloc_crash_control_pages(struct kimage *image,
 	while (hole_end <= crashk_res.end) {
 		unsigned long i;
 
+		cond_resched();
+
 		if (hole_end > KEXEC_CRASH_CONTROL_MEMORY_LIMIT)
 			break;
 		/* See if I overlap any of the segments */
@@ -1467,9 +1469,6 @@ static int __init crash_save_vmcoreinfo_init(void)
 #endif
 	VMCOREINFO_NUMBER(PG_head_mask);
 	VMCOREINFO_NUMBER(PAGE_BUDDY_MAPCOUNT_VALUE);
-#ifdef CONFIG_X86
-	VMCOREINFO_NUMBER(KERNEL_IMAGE_SIZE);
-#endif
 #ifdef CONFIG_HUGETLB_PAGE
 	VMCOREINFO_NUMBER(HUGETLB_PAGE_DTOR);
 #endif
diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c
index 037c321c5618..0c2df7f73792 100644
--- a/kernel/kexec_file.c
+++ b/kernel/kexec_file.c
@@ -428,25 +428,65 @@ static int locate_mem_hole_callback(u64 start, u64 end, void *arg)
 	return locate_mem_hole_bottom_up(start, end, kbuf);
 }
 
-/*
- * Helper function for placing a buffer in a kexec segment. This assumes
- * that kexec_mutex is held.
+/**
+ * arch_kexec_walk_mem - call func(data) on free memory regions
+ * @kbuf:	Context info for the search. Also passed to @func.
+ * @func:	Function to call for each memory region.
+ *
+ * Return: The memory walk will stop when func returns a non-zero value
+ * and that value will be returned. If all free regions are visited without
+ * func returning non-zero, then zero will be returned.
+ */
+int __weak arch_kexec_walk_mem(struct kexec_buf *kbuf,
+			       int (*func)(u64, u64, void *))
+{
+	if (kbuf->image->type == KEXEC_TYPE_CRASH)
+		return walk_iomem_res_desc(crashk_res.desc,
+					   IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
+					   crashk_res.start, crashk_res.end,
+					   kbuf, func);
+	else
+		return walk_system_ram_res(0, ULONG_MAX, kbuf, func);
+}
+
+/**
+ * kexec_locate_mem_hole - find free memory for the purgatory or the next kernel
+ * @kbuf:	Parameters for the memory search.
+ *
+ * On success, kbuf->mem will have the start address of the memory region found.
+ *
+ * Return: 0 on success, negative errno on error.
  */
-int kexec_add_buffer(struct kimage *image, char *buffer, unsigned long bufsz,
-		     unsigned long memsz, unsigned long buf_align,
-		     unsigned long buf_min, unsigned long buf_max,
-		     bool top_down, unsigned long *load_addr)
+int kexec_locate_mem_hole(struct kexec_buf *kbuf)
+{
+	int ret;
+
+	ret = arch_kexec_walk_mem(kbuf, locate_mem_hole_callback);
+
+	return ret == 1 ? 0 : -EADDRNOTAVAIL;
+}
+
+/**
+ * kexec_add_buffer - place a buffer in a kexec segment
+ * @kbuf:	Buffer contents and memory parameters.
+ *
+ * This function assumes that kexec_mutex is held.
+ * On successful return, @kbuf->mem will have the physical address of
+ * the buffer in memory.
+ *
+ * Return: 0 on success, negative errno on error.
+ */
+int kexec_add_buffer(struct kexec_buf *kbuf)
 {
 
 	struct kexec_segment *ksegment;
-	struct kexec_buf buf, *kbuf;
 	int ret;
 
 	/* Currently adding segment this way is allowed only in file mode */
-	if (!image->file_mode)
+	if (!kbuf->image->file_mode)
 		return -EINVAL;
 
-	if (image->nr_segments >= KEXEC_SEGMENT_MAX)
+	if (kbuf->image->nr_segments >= KEXEC_SEGMENT_MAX)
 		return -EINVAL;
 
 	/*
@@ -456,45 +496,27 @@ int kexec_add_buffer(struct kimage *image, char *buffer, unsigned long bufsz,
 	 * logic goes through list of segments to make sure there are
 	 * no destination overlaps.
 	 */
-	if (!list_empty(&image->control_pages)) {
+	if (!list_empty(&kbuf->image->control_pages)) {
 		WARN_ON(1);
 		return -EINVAL;
 	}
 
-	memset(&buf, 0, sizeof(struct kexec_buf));
-	kbuf = &buf;
-	kbuf->image = image;
-	kbuf->buffer = buffer;
-	kbuf->bufsz = bufsz;
-
-	kbuf->memsz = ALIGN(memsz, PAGE_SIZE);
-	kbuf->buf_align = max(buf_align, PAGE_SIZE);
-	kbuf->buf_min = buf_min;
-	kbuf->buf_max = buf_max;
-	kbuf->top_down = top_down;
+	/* Ensure minimum alignment needed for segments. */
+	kbuf->memsz = ALIGN(kbuf->memsz, PAGE_SIZE);
+	kbuf->buf_align = max(kbuf->buf_align, PAGE_SIZE);
 
 	/* Walk the RAM ranges and allocate a suitable range for the buffer */
-	if (image->type == KEXEC_TYPE_CRASH)
-		ret = walk_iomem_res_desc(crashk_res.desc,
-				IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
-				crashk_res.start, crashk_res.end, kbuf,
-				locate_mem_hole_callback);
-	else
-		ret = walk_system_ram_res(0, -1, kbuf,
-					  locate_mem_hole_callback);
-	if (ret != 1) {
-		/* A suitable memory range could not be found for buffer */
-		return -EADDRNOTAVAIL;
-	}
+	ret = kexec_locate_mem_hole(kbuf);
+	if (ret)
+		return ret;
 
 	/* Found a suitable memory range */
-	ksegment = &image->segment[image->nr_segments];
+	ksegment = &kbuf->image->segment[kbuf->image->nr_segments];
 	ksegment->kbuf = kbuf->buffer;
 	ksegment->bufsz = kbuf->bufsz;
 	ksegment->mem = kbuf->mem;
 	ksegment->memsz = kbuf->memsz;
-	image->nr_segments++;
-	*load_addr = ksegment->mem;
+	kbuf->image->nr_segments++;
 	return 0;
 }
 
@@ -616,13 +638,15 @@ static int __kexec_load_purgatory(struct kimage *image, unsigned long min,
 				  unsigned long max, int top_down)
 {
 	struct purgatory_info *pi = &image->purgatory_info;
-	unsigned long align, buf_align, bss_align, buf_sz, bss_sz, bss_pad;
-	unsigned long memsz, entry, load_addr, curr_load_addr, bss_addr, offset;
+	unsigned long align, bss_align, bss_sz, bss_pad;
+	unsigned long entry, load_addr, curr_load_addr, bss_addr, offset;
 	unsigned char *buf_addr, *src;
 	int i, ret = 0, entry_sidx = -1;
 	const Elf_Shdr *sechdrs_c;
 	Elf_Shdr *sechdrs = NULL;
-	void *purgatory_buf = NULL;
+	struct kexec_buf kbuf = { .image = image, .bufsz = 0, .buf_align = 1,
+				  .buf_min = min, .buf_max = max,
+				  .top_down = top_down };
 
 	/*
 	 * sechdrs_c points to section headers in purgatory and are read
@@ -688,9 +712,7 @@ static int __kexec_load_purgatory(struct kimage *image, unsigned long min,
 	}
 
 	/* Determine how much memory is needed to load relocatable object. */
-	buf_align = 1;
 	bss_align = 1;
-	buf_sz = 0;
 	bss_sz = 0;
 
 	for (i = 0; i < pi->ehdr->e_shnum; i++) {
@@ -699,10 +721,10 @@ static int __kexec_load_purgatory(struct kimage *image, unsigned long min,
 
 		align = sechdrs[i].sh_addralign;
 		if (sechdrs[i].sh_type != SHT_NOBITS) {
-			if (buf_align < align)
-				buf_align = align;
-			buf_sz = ALIGN(buf_sz, align);
-			buf_sz += sechdrs[i].sh_size;
+			if (kbuf.buf_align < align)
+				kbuf.buf_align = align;
+			kbuf.bufsz = ALIGN(kbuf.bufsz, align);
+			kbuf.bufsz += sechdrs[i].sh_size;
 		} else {
 			/* bss section */
 			if (bss_align < align)
@@ -714,32 +736,31 @@ static int __kexec_load_purgatory(struct kimage *image, unsigned long min,
 
 	/* Determine the bss padding required to align bss properly */
 	bss_pad = 0;
-	if (buf_sz & (bss_align - 1))
-		bss_pad = bss_align - (buf_sz & (bss_align - 1));
+	if (kbuf.bufsz & (bss_align - 1))
+		bss_pad = bss_align - (kbuf.bufsz & (bss_align - 1));
 
-	memsz = buf_sz + bss_pad + bss_sz;
+	kbuf.memsz = kbuf.bufsz + bss_pad + bss_sz;
 
 	/* Allocate buffer for purgatory */
-	purgatory_buf = vzalloc(buf_sz);
-	if (!purgatory_buf) {
+	kbuf.buffer = vzalloc(kbuf.bufsz);
+	if (!kbuf.buffer) {
 		ret = -ENOMEM;
 		goto out;
 	}
 
-	if (buf_align < bss_align)
-		buf_align = bss_align;
+	if (kbuf.buf_align < bss_align)
+		kbuf.buf_align = bss_align;
 
 	/* Add buffer to segment list */
-	ret = kexec_add_buffer(image, purgatory_buf, buf_sz, memsz,
-				buf_align, min, max, top_down,
-				&pi->purgatory_load_addr);
+	ret = kexec_add_buffer(&kbuf);
 	if (ret)
 		goto out;
+	pi->purgatory_load_addr = kbuf.mem;
 
 	/* Load SHF_ALLOC sections */
-	buf_addr = purgatory_buf;
+	buf_addr = kbuf.buffer;
 	load_addr = curr_load_addr = pi->purgatory_load_addr;
-	bss_addr = load_addr + buf_sz + bss_pad;
+	bss_addr = load_addr + kbuf.bufsz + bss_pad;
 
 	for (i = 0; i < pi->ehdr->e_shnum; i++) {
 		if (!(sechdrs[i].sh_flags & SHF_ALLOC))
@@ -785,11 +806,11 @@ static int __kexec_load_purgatory(struct kimage *image, unsigned long min,
 	 * Used later to identify which section is purgatory and skip it
 	 * from checksumming.
 	 */
-	pi->purgatory_buf = purgatory_buf;
+	pi->purgatory_buf = kbuf.buffer;
 	return ret;
 out:
 	vfree(sechdrs);
-	vfree(purgatory_buf);
+	vfree(kbuf.buffer);
 	return ret;
 }
 
diff --git a/kernel/kexec_internal.h b/kernel/kexec_internal.h
index 0a52315d9c62..4cef7e4706b0 100644
--- a/kernel/kexec_internal.h
+++ b/kernel/kexec_internal.h
@@ -20,22 +20,6 @@ struct kexec_sha_region {
 	unsigned long len;
 };
 
-/*
- * Keeps track of buffer parameters as provided by caller for requesting
- * memory placement of buffer.
- */
-struct kexec_buf {
-	struct kimage *image;
-	char *buffer;
-	unsigned long bufsz;
-	unsigned long mem;
-	unsigned long memsz;
-	unsigned long buf_align;
-	unsigned long buf_min;
-	unsigned long buf_max;
-	bool top_down;		/* allocate from top of memory hole */
-};
-
 void kimage_file_post_load_cleanup(struct kimage *image);
 #else /* CONFIG_KEXEC_FILE */
 static inline void kimage_file_post_load_cleanup(struct kimage *image) { }
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 7bd265f6b098..7c38f8f3d97b 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -3191,7 +3191,7 @@ print_lock_nested_lock_not_held(struct task_struct *curr,
 	return 0;
 }
 
-static int __lock_is_held(struct lockdep_map *lock);
+static int __lock_is_held(struct lockdep_map *lock, int read);
 
 /*
  * This gets called for every mutex_lock*()/spin_lock*() operation.
@@ -3332,7 +3332,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 	}
 	chain_key = iterate_chain_key(chain_key, class_idx);
 
-	if (nest_lock && !__lock_is_held(nest_lock))
+	if (nest_lock && !__lock_is_held(nest_lock, -1))
 		return print_lock_nested_lock_not_held(curr, hlock, ip);
 
 	if (!validate_chain(curr, lock, hlock, chain_head, chain_key))
@@ -3579,7 +3579,7 @@ found_it:
 	return 1;
 }
 
-static int __lock_is_held(struct lockdep_map *lock)
+static int __lock_is_held(struct lockdep_map *lock, int read)
 {
 	struct task_struct *curr = current;
 	int i;
@@ -3587,8 +3587,12 @@ static int __lock_is_held(struct lockdep_map *lock)
 	for (i = 0; i < curr->lockdep_depth; i++) {
 		struct held_lock *hlock = curr->held_locks + i;
 
-		if (match_held_lock(hlock, lock))
-			return 1;
+		if (match_held_lock(hlock, lock)) {
+			if (read == -1 || hlock->read == read)
+				return 1;
+
+			return 0;
+		}
 	}
 
 	return 0;
@@ -3772,7 +3776,7 @@ void lock_release(struct lockdep_map *lock, int nested,
 }
 EXPORT_SYMBOL_GPL(lock_release);
 
-int lock_is_held(struct lockdep_map *lock)
+int lock_is_held_type(struct lockdep_map *lock, int read)
 {
 	unsigned long flags;
 	int ret = 0;
@@ -3784,13 +3788,13 @@ int lock_is_held(struct lockdep_map *lock)
 	check_flags(flags);
 
 	current->lockdep_recursion = 1;
-	ret = __lock_is_held(lock);
+	ret = __lock_is_held(lock, read);
 	current->lockdep_recursion = 0;
 	raw_local_irq_restore(flags);
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(lock_is_held);
+EXPORT_SYMBOL_GPL(lock_is_held_type);
 
 struct pin_cookie lock_pin_lock(struct lockdep_map *lock)
 {
diff --git a/kernel/module.c b/kernel/module.c
index 0e54d5bf0097..f7482db0f843 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -313,8 +313,11 @@ struct load_info {
 	} index;
 };
 
-/* We require a truly strong try_module_get(): 0 means failure due to
-   ongoing or failed initialization etc. */
+/*
+ * We require a truly strong try_module_get(): 0 means success.
+ * Otherwise an error is returned due to ongoing or failed
+ * initialization etc.
+ */
 static inline int strong_try_module_get(struct module *mod)
 {
 	BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
@@ -330,7 +333,7 @@ static inline void add_taint_module(struct module *mod, unsigned flag,
 				    enum lockdep_ok lockdep_ok)
 {
 	add_taint(flag, lockdep_ok);
-	mod->taints |= (1U << flag);
+	set_bit(flag, &mod->taints);
 }
 
 /*
@@ -1138,24 +1141,13 @@ static inline int module_unload_init(struct module *mod)
 static size_t module_flags_taint(struct module *mod, char *buf)
 {
 	size_t l = 0;
+	int i;
+
+	for (i = 0; i < TAINT_FLAGS_COUNT; i++) {
+		if (taint_flags[i].module && test_bit(i, &mod->taints))
+			buf[l++] = taint_flags[i].true;
+	}
 
-	if (mod->taints & (1 << TAINT_PROPRIETARY_MODULE))
-		buf[l++] = 'P';
-	if (mod->taints & (1 << TAINT_OOT_MODULE))
-		buf[l++] = 'O';
-	if (mod->taints & (1 << TAINT_FORCED_MODULE))
-		buf[l++] = 'F';
-	if (mod->taints & (1 << TAINT_CRAP))
-		buf[l++] = 'C';
-	if (mod->taints & (1 << TAINT_UNSIGNED_MODULE))
-		buf[l++] = 'E';
-	if (mod->taints & (1 << TAINT_LIVEPATCH))
-		buf[l++] = 'K';
-	/*
-	 * TAINT_FORCED_RMMOD: could be added.
-	 * TAINT_CPU_OUT_OF_SPEC, TAINT_MACHINE_CHECK, TAINT_BAD_PAGE don't
-	 * apply to modules.
-	 */
 	return l;
 }
 
@@ -1911,6 +1903,9 @@ static void frob_writable_data(const struct module_layout *layout,
 /* livepatching wants to disable read-only so it can frob module. */
 void module_disable_ro(const struct module *mod)
 {
+	if (!rodata_enabled)
+		return;
+
 	frob_text(&mod->core_layout, set_memory_rw);
 	frob_rodata(&mod->core_layout, set_memory_rw);
 	frob_ro_after_init(&mod->core_layout, set_memory_rw);
@@ -1920,6 +1915,9 @@ void module_disable_ro(const struct module *mod)
 
 void module_enable_ro(const struct module *mod, bool after_init)
 {
+	if (!rodata_enabled)
+		return;
+
 	frob_text(&mod->core_layout, set_memory_ro);
 	frob_rodata(&mod->core_layout, set_memory_ro);
 	frob_text(&mod->init_layout, set_memory_ro);
@@ -1952,6 +1950,9 @@ void set_all_modules_text_rw(void)
 {
 	struct module *mod;
 
+	if (!rodata_enabled)
+		return;
+
 	mutex_lock(&module_mutex);
 	list_for_each_entry_rcu(mod, &modules, list) {
 		if (mod->state == MODULE_STATE_UNFORMED)
@@ -1968,9 +1969,18 @@ void set_all_modules_text_ro(void)
 {
 	struct module *mod;
 
+	if (!rodata_enabled)
+		return;
+
 	mutex_lock(&module_mutex);
 	list_for_each_entry_rcu(mod, &modules, list) {
-		if (mod->state == MODULE_STATE_UNFORMED)
+		/*
+		 * Ignore going modules since it's possible that ro
+		 * protection has already been disabled, otherwise we'll
+		 * run into protection faults at module deallocation.
+		 */
+		if (mod->state == MODULE_STATE_UNFORMED ||
+			mod->state == MODULE_STATE_GOING)
 			continue;
 
 		frob_text(&mod->core_layout, set_memory_ro);
@@ -1981,10 +1991,12 @@ void set_all_modules_text_ro(void)
 
 static void disable_ro_nx(const struct module_layout *layout)
 {
-	frob_text(layout, set_memory_rw);
-	frob_rodata(layout, set_memory_rw);
+	if (rodata_enabled) {
+		frob_text(layout, set_memory_rw);
+		frob_rodata(layout, set_memory_rw);
+		frob_ro_after_init(layout, set_memory_rw);
+	}
 	frob_rodata(layout, set_memory_x);
-	frob_ro_after_init(layout, set_memory_rw);
 	frob_ro_after_init(layout, set_memory_x);
 	frob_writable_data(layout, set_memory_x);
 }
@@ -3709,6 +3721,7 @@ static int load_module(struct load_info *info, const char __user *uargs,
  sysfs_cleanup:
 	mod_sysfs_teardown(mod);
  coming_cleanup:
+	mod->state = MODULE_STATE_GOING;
 	blocking_notifier_call_chain(&module_notify_list,
 				     MODULE_STATE_GOING, mod);
 	klp_module_going(mod);
@@ -4042,6 +4055,10 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *,
 }
 #endif /* CONFIG_KALLSYMS */
 
+/* Maximum number of characters written by module_flags() */
+#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)
+
+/* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */
 static char *module_flags(struct module *mod, char *buf)
 {
 	int bx = 0;
@@ -4086,7 +4103,7 @@ static void m_stop(struct seq_file *m, void *p)
 static int m_show(struct seq_file *m, void *p)
 {
 	struct module *mod = list_entry(p, struct module, list);
-	char buf[8];
+	char buf[MODULE_FLAGS_BUF_SIZE];
 
 	/* We always ignore unformed modules. */
 	if (mod->state == MODULE_STATE_UNFORMED)
@@ -4257,7 +4274,7 @@ EXPORT_SYMBOL_GPL(__module_text_address);
 void print_modules(void)
 {
 	struct module *mod;
-	char buf[8];
+	char buf[MODULE_FLAGS_BUF_SIZE];
 
 	printk(KERN_DEFAULT "Modules linked in:");
 	/* Most callers should already have preempt disabled, but make sure */
diff --git a/kernel/padata.c b/kernel/padata.c
index 7848f0566403..05316c9f32da 100644
--- a/kernel/padata.c
+++ b/kernel/padata.c
@@ -64,15 +64,11 @@ static int padata_cpu_hash(struct parallel_data *pd)
 static void padata_parallel_worker(struct work_struct *parallel_work)
 {
 	struct padata_parallel_queue *pqueue;
-	struct parallel_data *pd;
-	struct padata_instance *pinst;
 	LIST_HEAD(local_list);
 
 	local_bh_disable();
 	pqueue = container_of(parallel_work,
 			      struct padata_parallel_queue, work);
-	pd = pqueue->pd;
-	pinst = pd->pinst;
 
 	spin_lock(&pqueue->parallel.lock);
 	list_replace_init(&pqueue->parallel.list, &local_list);
diff --git a/kernel/panic.c b/kernel/panic.c
index e6480e20379e..c51edaa04fce 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -298,30 +298,27 @@ void panic(const char *fmt, ...)
 
 EXPORT_SYMBOL(panic);
 
-
-struct tnt {
-	u8	bit;
-	char	true;
-	char	false;
-};
-
-static const struct tnt tnts[] = {
-	{ TAINT_PROPRIETARY_MODULE,	'P', 'G' },
-	{ TAINT_FORCED_MODULE,		'F', ' ' },
-	{ TAINT_CPU_OUT_OF_SPEC,	'S', ' ' },
-	{ TAINT_FORCED_RMMOD,		'R', ' ' },
-	{ TAINT_MACHINE_CHECK,		'M', ' ' },
-	{ TAINT_BAD_PAGE,		'B', ' ' },
-	{ TAINT_USER,			'U', ' ' },
-	{ TAINT_DIE,			'D', ' ' },
-	{ TAINT_OVERRIDDEN_ACPI_TABLE,	'A', ' ' },
-	{ TAINT_WARN,			'W', ' ' },
-	{ TAINT_CRAP,			'C', ' ' },
-	{ TAINT_FIRMWARE_WORKAROUND,	'I', ' ' },
-	{ TAINT_OOT_MODULE,		'O', ' ' },
-	{ TAINT_UNSIGNED_MODULE,	'E', ' ' },
-	{ TAINT_SOFTLOCKUP,		'L', ' ' },
-	{ TAINT_LIVEPATCH,		'K', ' ' },
+/*
+ * TAINT_FORCED_RMMOD could be a per-module flag but the module
+ * is being removed anyway.
+ */
+const struct taint_flag taint_flags[TAINT_FLAGS_COUNT] = {
+	{ 'P', 'G', true },	/* TAINT_PROPRIETARY_MODULE */
+	{ 'F', ' ', true },	/* TAINT_FORCED_MODULE */
+	{ 'S', ' ', false },	/* TAINT_CPU_OUT_OF_SPEC */
+	{ 'R', ' ', false },	/* TAINT_FORCED_RMMOD */
+	{ 'M', ' ', false },	/* TAINT_MACHINE_CHECK */
+	{ 'B', ' ', false },	/* TAINT_BAD_PAGE */
+	{ 'U', ' ', false },	/* TAINT_USER */
+	{ 'D', ' ', false },	/* TAINT_DIE */
+	{ 'A', ' ', false },	/* TAINT_OVERRIDDEN_ACPI_TABLE */
+	{ 'W', ' ', false },	/* TAINT_WARN */
+	{ 'C', ' ', true },	/* TAINT_CRAP */
+	{ 'I', ' ', false },	/* TAINT_FIRMWARE_WORKAROUND */
+	{ 'O', ' ', true },	/* TAINT_OOT_MODULE */
+	{ 'E', ' ', true },	/* TAINT_UNSIGNED_MODULE */
+	{ 'L', ' ', false },	/* TAINT_SOFTLOCKUP */
+	{ 'K', ' ', true },	/* TAINT_LIVEPATCH */
 };
 
 /**
@@ -348,16 +345,16 @@ static const struct tnt tnts[] = {
  */
 const char *print_tainted(void)
 {
-	static char buf[ARRAY_SIZE(tnts) + sizeof("Tainted: ")];
+	static char buf[TAINT_FLAGS_COUNT + sizeof("Tainted: ")];
 
 	if (tainted_mask) {
 		char *s;
 		int i;
 
 		s = buf + sprintf(buf, "Tainted: ");
-		for (i = 0; i < ARRAY_SIZE(tnts); i++) {
-			const struct tnt *t = &tnts[i];
-			*s++ = test_bit(t->bit, &tainted_mask) ?
+		for (i = 0; i < TAINT_FLAGS_COUNT; i++) {
+			const struct taint_flag *t = &taint_flags[i];
+			*s++ = test_bit(i, &tainted_mask) ?
 					t->true : t->false;
 		}
 		*s = 0;
diff --git a/kernel/power/main.c b/kernel/power/main.c
index 281a697fd458..d401c21136d1 100644
--- a/kernel/power/main.c
+++ b/kernel/power/main.c
@@ -78,6 +78,78 @@ static ssize_t pm_async_store(struct kobject *kobj, struct kobj_attribute *attr,
 
 power_attr(pm_async);
 
+#ifdef CONFIG_SUSPEND
+static ssize_t mem_sleep_show(struct kobject *kobj, struct kobj_attribute *attr,
+			      char *buf)
+{
+	char *s = buf;
+	suspend_state_t i;
+
+	for (i = PM_SUSPEND_MIN; i < PM_SUSPEND_MAX; i++)
+		if (mem_sleep_states[i]) {
+			const char *label = mem_sleep_states[i];
+
+			if (mem_sleep_current == i)
+				s += sprintf(s, "[%s] ", label);
+			else
+				s += sprintf(s, "%s ", label);
+		}
+
+	/* Convert the last space to a newline if needed. */
+	if (s != buf)
+		*(s-1) = '\n';
+
+	return (s - buf);
+}
+
+static suspend_state_t decode_suspend_state(const char *buf, size_t n)
+{
+	suspend_state_t state;
+	char *p;
+	int len;
+
+	p = memchr(buf, '\n', n);
+	len = p ? p - buf : n;
+
+	for (state = PM_SUSPEND_MIN; state < PM_SUSPEND_MAX; state++) {
+		const char *label = mem_sleep_states[state];
+
+		if (label && len == strlen(label) && !strncmp(buf, label, len))
+			return state;
+	}
+
+	return PM_SUSPEND_ON;
+}
+
+static ssize_t mem_sleep_store(struct kobject *kobj, struct kobj_attribute *attr,
+			       const char *buf, size_t n)
+{
+	suspend_state_t state;
+	int error;
+
+	error = pm_autosleep_lock();
+	if (error)
+		return error;
+
+	if (pm_autosleep_state() > PM_SUSPEND_ON) {
+		error = -EBUSY;
+		goto out;
+	}
+
+	state = decode_suspend_state(buf, n);
+	if (state < PM_SUSPEND_MAX && state > PM_SUSPEND_ON)
+		mem_sleep_current = state;
+	else
+		error = -EINVAL;
+
+ out:
+	pm_autosleep_unlock();
+	return error ? error : n;
+}
+
+power_attr(mem_sleep);
+#endif /* CONFIG_SUSPEND */
+
 #ifdef CONFIG_PM_DEBUG
 int pm_test_level = TEST_NONE;
 
@@ -368,12 +440,16 @@ static ssize_t state_store(struct kobject *kobj, struct kobj_attribute *attr,
 	}
 
 	state = decode_state(buf, n);
-	if (state < PM_SUSPEND_MAX)
+	if (state < PM_SUSPEND_MAX) {
+		if (state == PM_SUSPEND_MEM)
+			state = mem_sleep_current;
+
 		error = pm_suspend(state);
-	else if (state == PM_SUSPEND_MAX)
+	} else if (state == PM_SUSPEND_MAX) {
 		error = hibernate();
-	else
+	} else {
 		error = -EINVAL;
+	}
 
  out:
 	pm_autosleep_unlock();
@@ -485,6 +561,9 @@ static ssize_t autosleep_store(struct kobject *kobj,
 	    && strcmp(buf, "off") && strcmp(buf, "off\n"))
 		return -EINVAL;
 
+	if (state == PM_SUSPEND_MEM)
+		state = mem_sleep_current;
+
 	error = pm_autosleep_set_state(state);
 	return error ? error : n;
 }
@@ -602,6 +681,9 @@ static struct attribute * g[] = {
 #ifdef CONFIG_PM_SLEEP
 	&pm_async_attr.attr,
 	&wakeup_count_attr.attr,
+#ifdef CONFIG_SUSPEND
+	&mem_sleep_attr.attr,
+#endif
 #ifdef CONFIG_PM_AUTOSLEEP
 	&autosleep_attr.attr,
 #endif
diff --git a/kernel/power/power.h b/kernel/power/power.h
index 56d1d0dedf76..1dfa0da827d3 100644
--- a/kernel/power/power.h
+++ b/kernel/power/power.h
@@ -189,11 +189,15 @@ extern void swsusp_show_speed(ktime_t, ktime_t, unsigned int, char *);
 
 #ifdef CONFIG_SUSPEND
 /* kernel/power/suspend.c */
-extern const char *pm_labels[];
+extern const char * const pm_labels[];
 extern const char *pm_states[];
+extern const char *mem_sleep_states[];
+extern suspend_state_t mem_sleep_current;
 
 extern int suspend_devices_and_enter(suspend_state_t state);
 #else /* !CONFIG_SUSPEND */
+#define mem_sleep_current	PM_SUSPEND_ON
+
 static inline int suspend_devices_and_enter(suspend_state_t state)
 {
 	return -ENOSYS;
diff --git a/kernel/power/qos.c b/kernel/power/qos.c
index 168ff442ebde..97b0df71303e 100644
--- a/kernel/power/qos.c
+++ b/kernel/power/qos.c
@@ -482,16 +482,7 @@ void pm_qos_update_request(struct pm_qos_request *req,
 		return;
 	}
 
-	/*
-	 * This function may be called very early during boot, for example,
-	 * from of_clk_init(), where irq needs to stay disabled.
-	 * cancel_delayed_work_sync() assumes that irq is enabled on
-	 * invocation and re-enables it on return.  Avoid calling it until
-	 * workqueue is initialized.
-	 */
-	if (keventd_up())
-		cancel_delayed_work_sync(&req->work);
-
+	cancel_delayed_work_sync(&req->work);
 	__pm_qos_update_request(req, new_value);
 }
 EXPORT_SYMBOL_GPL(pm_qos_update_request);
diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c
index 6ccb08f57fcb..f67ceb7768b8 100644
--- a/kernel/power/suspend.c
+++ b/kernel/power/suspend.c
@@ -32,8 +32,21 @@
 
 #include "power.h"
 
-const char *pm_labels[] = { "mem", "standby", "freeze", NULL };
+const char * const pm_labels[] = {
+	[PM_SUSPEND_FREEZE] = "freeze",
+	[PM_SUSPEND_STANDBY] = "standby",
+	[PM_SUSPEND_MEM] = "mem",
+};
 const char *pm_states[PM_SUSPEND_MAX];
+static const char * const mem_sleep_labels[] = {
+	[PM_SUSPEND_FREEZE] = "s2idle",
+	[PM_SUSPEND_STANDBY] = "shallow",
+	[PM_SUSPEND_MEM] = "deep",
+};
+const char *mem_sleep_states[PM_SUSPEND_MAX];
+
+suspend_state_t mem_sleep_current = PM_SUSPEND_FREEZE;
+suspend_state_t mem_sleep_default = PM_SUSPEND_MAX;
 
 unsigned int pm_suspend_global_flags;
 EXPORT_SYMBOL_GPL(pm_suspend_global_flags);
@@ -110,30 +123,32 @@ static bool valid_state(suspend_state_t state)
 	return suspend_ops && suspend_ops->valid && suspend_ops->valid(state);
 }
 
-/*
- * If this is set, the "mem" label always corresponds to the deepest sleep state
- * available, the "standby" label corresponds to the second deepest sleep state
- * available (if any), and the "freeze" label corresponds to the remaining
- * available sleep state (if there is one).
- */
-static bool relative_states;
-
 void __init pm_states_init(void)
 {
+	/* "mem" and "freeze" are always present in /sys/power/state. */
+	pm_states[PM_SUSPEND_MEM] = pm_labels[PM_SUSPEND_MEM];
+	pm_states[PM_SUSPEND_FREEZE] = pm_labels[PM_SUSPEND_FREEZE];
 	/*
-	 * freeze state should be supported even without any suspend_ops,
-	 * initialize pm_states accordingly here
+	 * Suspend-to-idle should be supported even without any suspend_ops,
+	 * initialize mem_sleep_states[] accordingly here.
 	 */
-	pm_states[PM_SUSPEND_FREEZE] = pm_labels[relative_states ? 0 : 2];
+	mem_sleep_states[PM_SUSPEND_FREEZE] = mem_sleep_labels[PM_SUSPEND_FREEZE];
 }
 
-static int __init sleep_states_setup(char *str)
+static int __init mem_sleep_default_setup(char *str)
 {
-	relative_states = !strncmp(str, "1", 1);
+	suspend_state_t state;
+
+	for (state = PM_SUSPEND_FREEZE; state <= PM_SUSPEND_MEM; state++)
+		if (mem_sleep_labels[state] &&
+		    !strcmp(str, mem_sleep_labels[state])) {
+			mem_sleep_default = state;
+			break;
+		}
+
 	return 1;
 }
-
-__setup("relative_sleep_states=", sleep_states_setup);
+__setup("mem_sleep_default=", mem_sleep_default_setup);
 
 /**
  * suspend_set_ops - Set the global suspend method table.
@@ -141,21 +156,21 @@ __setup("relative_sleep_states=", sleep_states_setup);
  */
 void suspend_set_ops(const struct platform_suspend_ops *ops)
 {
-	suspend_state_t i;
-	int j = 0;
-
 	lock_system_sleep();
 
 	suspend_ops = ops;
-	for (i = PM_SUSPEND_MEM; i >= PM_SUSPEND_STANDBY; i--)
-		if (valid_state(i)) {
-			pm_states[i] = pm_labels[j++];
-		} else if (!relative_states) {
-			pm_states[i] = NULL;
-			j++;
-		}
 
-	pm_states[PM_SUSPEND_FREEZE] = pm_labels[j];
+	if (valid_state(PM_SUSPEND_STANDBY)) {
+		mem_sleep_states[PM_SUSPEND_STANDBY] = mem_sleep_labels[PM_SUSPEND_STANDBY];
+		pm_states[PM_SUSPEND_STANDBY] = pm_labels[PM_SUSPEND_STANDBY];
+		if (mem_sleep_default == PM_SUSPEND_STANDBY)
+			mem_sleep_current = PM_SUSPEND_STANDBY;
+	}
+	if (valid_state(PM_SUSPEND_MEM)) {
+		mem_sleep_states[PM_SUSPEND_MEM] = mem_sleep_labels[PM_SUSPEND_MEM];
+		if (mem_sleep_default >= PM_SUSPEND_MEM)
+			mem_sleep_current = PM_SUSPEND_MEM;
+	}
 
 	unlock_system_sleep();
 }
diff --git a/kernel/power/swap.c b/kernel/power/swap.c
index a3b1e617bcdc..32e0c232efba 100644
--- a/kernel/power/swap.c
+++ b/kernel/power/swap.c
@@ -307,7 +307,7 @@ static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags)
 {
 	int error;
 
-	hib_submit_io(REQ_OP_READ, READ_SYNC, swsusp_resume_block,
+	hib_submit_io(REQ_OP_READ, 0, swsusp_resume_block,
 		      swsusp_header, NULL);
 	if (!memcmp("SWAP-SPACE",swsusp_header->sig, 10) ||
 	    !memcmp("SWAPSPACE2",swsusp_header->sig, 10)) {
@@ -317,7 +317,7 @@ static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags)
 		swsusp_header->flags = flags;
 		if (flags & SF_CRC32_MODE)
 			swsusp_header->crc32 = handle->crc32;
-		error = hib_submit_io(REQ_OP_WRITE, WRITE_SYNC,
+		error = hib_submit_io(REQ_OP_WRITE, REQ_SYNC,
 				      swsusp_resume_block, swsusp_header, NULL);
 	} else {
 		printk(KERN_ERR "PM: Swap header not found!\n");
@@ -397,7 +397,7 @@ static int write_page(void *buf, sector_t offset, struct hib_bio_batch *hb)
 	} else {
 		src = buf;
 	}
-	return hib_submit_io(REQ_OP_WRITE, WRITE_SYNC, offset, src, hb);
+	return hib_submit_io(REQ_OP_WRITE, REQ_SYNC, offset, src, hb);
 }
 
 static void release_swap_writer(struct swap_map_handle *handle)
@@ -1000,8 +1000,7 @@ static int get_swap_reader(struct swap_map_handle *handle,
 			return -ENOMEM;
 		}
 
-		error = hib_submit_io(REQ_OP_READ, READ_SYNC, offset,
-				      tmp->map, NULL);
+		error = hib_submit_io(REQ_OP_READ, 0, offset, tmp->map, NULL);
 		if (error) {
 			release_swap_reader(handle);
 			return error;
@@ -1025,7 +1024,7 @@ static int swap_read_page(struct swap_map_handle *handle, void *buf,
 	offset = handle->cur->entries[handle->k];
 	if (!offset)
 		return -EFAULT;
-	error = hib_submit_io(REQ_OP_READ, READ_SYNC, offset, buf, hb);
+	error = hib_submit_io(REQ_OP_READ, 0, offset, buf, hb);
 	if (error)
 		return error;
 	if (++handle->k >= MAP_PAGE_ENTRIES) {
@@ -1534,7 +1533,7 @@ int swsusp_check(void)
 	if (!IS_ERR(hib_resume_bdev)) {
 		set_blocksize(hib_resume_bdev, PAGE_SIZE);
 		clear_page(swsusp_header);
-		error = hib_submit_io(REQ_OP_READ, READ_SYNC,
+		error = hib_submit_io(REQ_OP_READ, 0,
 					swsusp_resume_block,
 					swsusp_header, NULL);
 		if (error)
@@ -1543,7 +1542,7 @@ int swsusp_check(void)
 		if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) {
 			memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10);
 			/* Reset swap signature now */
-			error = hib_submit_io(REQ_OP_WRITE, WRITE_SYNC,
+			error = hib_submit_io(REQ_OP_WRITE, REQ_SYNC,
 						swsusp_resume_block,
 						swsusp_header, NULL);
 		} else {
@@ -1588,11 +1587,11 @@ int swsusp_unmark(void)
 {
 	int error;
 
-	hib_submit_io(REQ_OP_READ, READ_SYNC, swsusp_resume_block,
+	hib_submit_io(REQ_OP_READ, 0, swsusp_resume_block,
 		      swsusp_header, NULL);
 	if (!memcmp(HIBERNATE_SIG,swsusp_header->sig, 10)) {
 		memcpy(swsusp_header->sig,swsusp_header->orig_sig, 10);
-		error = hib_submit_io(REQ_OP_WRITE, WRITE_SYNC,
+		error = hib_submit_io(REQ_OP_WRITE, REQ_SYNC,
 					swsusp_resume_block,
 					swsusp_header, NULL);
 	} else {
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index 577f2288d19f..bc2e220ed2b0 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -356,7 +356,6 @@ DECLARE_WAIT_QUEUE_HEAD(log_wait);
 /* the next printk record to read by syslog(READ) or /proc/kmsg */
 static u64 syslog_seq;
 static u32 syslog_idx;
-static enum log_flags syslog_prev;
 static size_t syslog_partial;
 
 /* index and sequence number of the first record stored in the buffer */
@@ -370,7 +369,6 @@ static u32 log_next_idx;
 /* the next printk record to write to the console */
 static u64 console_seq;
 static u32 console_idx;
-static enum log_flags console_prev;
 
 /* the next printk record to read after the last 'clear' command */
 static u64 clear_seq;
@@ -639,27 +637,15 @@ static void append_char(char **pp, char *e, char c)
 }
 
 static ssize_t msg_print_ext_header(char *buf, size_t size,
-				    struct printk_log *msg, u64 seq,
-				    enum log_flags prev_flags)
+				    struct printk_log *msg, u64 seq)
 {
 	u64 ts_usec = msg->ts_nsec;
-	char cont = '-';
 
 	do_div(ts_usec, 1000);
 
-	/*
-	 * If we couldn't merge continuation line fragments during the print,
-	 * export the stored flags to allow an optional external merge of the
-	 * records. Merging the records isn't always neccessarily correct, like
-	 * when we hit a race during printing. In most cases though, it produces
-	 * better readable output. 'c' in the record flags mark the first
-	 * fragment of a line, '+' the following.
-	 */
-	if (msg->flags & LOG_CONT)
-		cont = (prev_flags & LOG_CONT) ? '+' : 'c';
-
 	return scnprintf(buf, size, "%u,%llu,%llu,%c;",
-		       (msg->facility << 3) | msg->level, seq, ts_usec, cont);
+		       (msg->facility << 3) | msg->level, seq, ts_usec,
+		       msg->flags & LOG_CONT ? 'c' : '-');
 }
 
 static ssize_t msg_print_ext_body(char *buf, size_t size,
@@ -714,7 +700,6 @@ static ssize_t msg_print_ext_body(char *buf, size_t size,
 struct devkmsg_user {
 	u64 seq;
 	u32 idx;
-	enum log_flags prev;
 	struct ratelimit_state rs;
 	struct mutex lock;
 	char buf[CONSOLE_EXT_LOG_MAX];
@@ -824,12 +809,11 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf,
 
 	msg = log_from_idx(user->idx);
 	len = msg_print_ext_header(user->buf, sizeof(user->buf),
-				   msg, user->seq, user->prev);
+				   msg, user->seq);
 	len += msg_print_ext_body(user->buf + len, sizeof(user->buf) - len,
 				  log_dict(msg), msg->dict_len,
 				  log_text(msg), msg->text_len);
 
-	user->prev = msg->flags;
 	user->idx = log_next(user->idx);
 	user->seq++;
 	raw_spin_unlock_irq(&logbuf_lock);
@@ -1210,26 +1194,12 @@ static size_t print_prefix(const struct printk_log *msg, bool syslog, char *buf)
 	return len;
 }
 
-static size_t msg_print_text(const struct printk_log *msg, enum log_flags prev,
-			     bool syslog, char *buf, size_t size)
+static size_t msg_print_text(const struct printk_log *msg, bool syslog, char *buf, size_t size)
 {
 	const char *text = log_text(msg);
 	size_t text_size = msg->text_len;
-	bool prefix = true;
-	bool newline = true;
 	size_t len = 0;
 
-	if ((prev & LOG_CONT) && !(msg->flags & LOG_PREFIX))
-		prefix = false;
-
-	if (msg->flags & LOG_CONT) {
-		if ((prev & LOG_CONT) && !(prev & LOG_NEWLINE))
-			prefix = false;
-
-		if (!(msg->flags & LOG_NEWLINE))
-			newline = false;
-	}
-
 	do {
 		const char *next = memchr(text, '\n', text_size);
 		size_t text_len;
@@ -1247,22 +1217,17 @@ static size_t msg_print_text(const struct printk_log *msg, enum log_flags prev,
 			    text_len + 1 >= size - len)
 				break;
 
-			if (prefix)
-				len += print_prefix(msg, syslog, buf + len);
+			len += print_prefix(msg, syslog, buf + len);
 			memcpy(buf + len, text, text_len);
 			len += text_len;
-			if (next || newline)
-				buf[len++] = '\n';
+			buf[len++] = '\n';
 		} else {
 			/* SYSLOG_ACTION_* buffer size only calculation */
-			if (prefix)
-				len += print_prefix(msg, syslog, NULL);
+			len += print_prefix(msg, syslog, NULL);
 			len += text_len;
-			if (next || newline)
-				len++;
+			len++;
 		}
 
-		prefix = true;
 		text = next;
 	} while (text);
 
@@ -1288,7 +1253,6 @@ static int syslog_print(char __user *buf, int size)
 			/* messages are gone, move to first one */
 			syslog_seq = log_first_seq;
 			syslog_idx = log_first_idx;
-			syslog_prev = 0;
 			syslog_partial = 0;
 		}
 		if (syslog_seq == log_next_seq) {
@@ -1298,13 +1262,11 @@ static int syslog_print(char __user *buf, int size)
 
 		skip = syslog_partial;
 		msg = log_from_idx(syslog_idx);
-		n = msg_print_text(msg, syslog_prev, true, text,
-				   LOG_LINE_MAX + PREFIX_MAX);
+		n = msg_print_text(msg, true, text, LOG_LINE_MAX + PREFIX_MAX);
 		if (n - syslog_partial <= size) {
 			/* message fits into buffer, move forward */
 			syslog_idx = log_next(syslog_idx);
 			syslog_seq++;
-			syslog_prev = msg->flags;
 			n -= syslog_partial;
 			syslog_partial = 0;
 		} else if (!len){
@@ -1347,7 +1309,6 @@ static int syslog_print_all(char __user *buf, int size, bool clear)
 		u64 next_seq;
 		u64 seq;
 		u32 idx;
-		enum log_flags prev;
 
 		/*
 		 * Find first record that fits, including all following records,
@@ -1355,12 +1316,10 @@ static int syslog_print_all(char __user *buf, int size, bool clear)
 		 */
 		seq = clear_seq;
 		idx = clear_idx;
-		prev = 0;
 		while (seq < log_next_seq) {
 			struct printk_log *msg = log_from_idx(idx);
 
-			len += msg_print_text(msg, prev, true, NULL, 0);
-			prev = msg->flags;
+			len += msg_print_text(msg, true, NULL, 0);
 			idx = log_next(idx);
 			seq++;
 		}
@@ -1368,12 +1327,10 @@ static int syslog_print_all(char __user *buf, int size, bool clear)
 		/* move first record forward until length fits into the buffer */
 		seq = clear_seq;
 		idx = clear_idx;
-		prev = 0;
 		while (len > size && seq < log_next_seq) {
 			struct printk_log *msg = log_from_idx(idx);
 
-			len -= msg_print_text(msg, prev, true, NULL, 0);
-			prev = msg->flags;
+			len -= msg_print_text(msg, true, NULL, 0);
 			idx = log_next(idx);
 			seq++;
 		}
@@ -1386,7 +1343,7 @@ static int syslog_print_all(char __user *buf, int size, bool clear)
 			struct printk_log *msg = log_from_idx(idx);
 			int textlen;
 
-			textlen = msg_print_text(msg, prev, true, text,
+			textlen = msg_print_text(msg, true, text,
 						 LOG_LINE_MAX + PREFIX_MAX);
 			if (textlen < 0) {
 				len = textlen;
@@ -1394,7 +1351,6 @@ static int syslog_print_all(char __user *buf, int size, bool clear)
 			}
 			idx = log_next(idx);
 			seq++;
-			prev = msg->flags;
 
 			raw_spin_unlock_irq(&logbuf_lock);
 			if (copy_to_user(buf + len, text, textlen))
@@ -1407,7 +1363,6 @@ static int syslog_print_all(char __user *buf, int size, bool clear)
 				/* messages are gone, move to next one */
 				seq = log_first_seq;
 				idx = log_first_idx;
-				prev = 0;
 			}
 		}
 	}
@@ -1508,7 +1463,6 @@ int do_syslog(int type, char __user *buf, int len, int source)
 			/* messages are gone, move to first one */
 			syslog_seq = log_first_seq;
 			syslog_idx = log_first_idx;
-			syslog_prev = 0;
 			syslog_partial = 0;
 		}
 		if (source == SYSLOG_FROM_PROC) {
@@ -1521,16 +1475,14 @@ int do_syslog(int type, char __user *buf, int len, int source)
 		} else {
 			u64 seq = syslog_seq;
 			u32 idx = syslog_idx;
-			enum log_flags prev = syslog_prev;
 
 			error = 0;
 			while (seq < log_next_seq) {
 				struct printk_log *msg = log_from_idx(idx);
 
-				error += msg_print_text(msg, prev, true, NULL, 0);
+				error += msg_print_text(msg, true, NULL, 0);
 				idx = log_next(idx);
 				seq++;
-				prev = msg->flags;
 			}
 			error -= syslog_partial;
 		}
@@ -1631,46 +1583,25 @@ static inline void printk_delay(void)
 static struct cont {
 	char buf[LOG_LINE_MAX];
 	size_t len;			/* length == 0 means unused buffer */
-	size_t cons;			/* bytes written to console */
 	struct task_struct *owner;	/* task of first print*/
 	u64 ts_nsec;			/* time of first print */
 	u8 level;			/* log level of first message */
 	u8 facility;			/* log facility of first message */
 	enum log_flags flags;		/* prefix, newline flags */
-	bool flushed:1;			/* buffer sealed and committed */
 } cont;
 
 static void cont_flush(void)
 {
-	if (cont.flushed)
-		return;
 	if (cont.len == 0)
 		return;
-	if (cont.cons) {
-		/*
-		 * If a fragment of this line was directly flushed to the
-		 * console; wait for the console to pick up the rest of the
-		 * line. LOG_NOCONS suppresses a duplicated output.
-		 */
-		log_store(cont.facility, cont.level, cont.flags | LOG_NOCONS,
-			  cont.ts_nsec, NULL, 0, cont.buf, cont.len);
-		cont.flushed = true;
-	} else {
-		/*
-		 * If no fragment of this line ever reached the console,
-		 * just submit it to the store and free the buffer.
-		 */
-		log_store(cont.facility, cont.level, cont.flags, 0,
-			  NULL, 0, cont.buf, cont.len);
-		cont.len = 0;
-	}
+
+	log_store(cont.facility, cont.level, cont.flags, cont.ts_nsec,
+		  NULL, 0, cont.buf, cont.len);
+	cont.len = 0;
 }
 
 static bool cont_add(int facility, int level, enum log_flags flags, const char *text, size_t len)
 {
-	if (cont.len && cont.flushed)
-		return false;
-
 	/*
 	 * If ext consoles are present, flush and skip in-kernel
 	 * continuation.  See nr_ext_console_drivers definition.  Also, if
@@ -1687,8 +1618,6 @@ static bool cont_add(int facility, int level, enum log_flags flags, const char *
 		cont.owner = current;
 		cont.ts_nsec = local_clock();
 		cont.flags = flags;
-		cont.cons = 0;
-		cont.flushed = false;
 	}
 
 	memcpy(cont.buf + cont.len, text, len);
@@ -1707,34 +1636,6 @@ static bool cont_add(int facility, int level, enum log_flags flags, const char *
 	return true;
 }
 
-static size_t cont_print_text(char *text, size_t size)
-{
-	size_t textlen = 0;
-	size_t len;
-
-	if (cont.cons == 0 && (console_prev & LOG_NEWLINE)) {
-		textlen += print_time(cont.ts_nsec, text);
-		size -= textlen;
-	}
-
-	len = cont.len - cont.cons;
-	if (len > 0) {
-		if (len+1 > size)
-			len = size-1;
-		memcpy(text + textlen, cont.buf + cont.cons, len);
-		textlen += len;
-		cont.cons = cont.len;
-	}
-
-	if (cont.flushed) {
-		if (cont.flags & LOG_NEWLINE)
-			text[textlen++] = '\n';
-		/* got everything, release buffer */
-		cont.len = 0;
-	}
-	return textlen;
-}
-
 static size_t log_output(int facility, int level, enum log_flags lflags, const char *dict, size_t dictlen, char *text, size_t text_len)
 {
 	/*
@@ -1926,7 +1827,8 @@ int vprintk_default(const char *fmt, va_list args)
 	int r;
 
 #ifdef CONFIG_KGDB_KDB
-	if (unlikely(kdb_trap_printk)) {
+	/* Allow to pass printk() to kdb but avoid a recursion. */
+	if (unlikely(kdb_trap_printk && kdb_printf_cpu < 0)) {
 		r = vkdb_printf(KDB_MSGSRC_PRINTK, fmt, args);
 		return r;
 	}
@@ -1980,33 +1882,24 @@ static u64 syslog_seq;
 static u32 syslog_idx;
 static u64 console_seq;
 static u32 console_idx;
-static enum log_flags syslog_prev;
 static u64 log_first_seq;
 static u32 log_first_idx;
 static u64 log_next_seq;
-static enum log_flags console_prev;
-static struct cont {
-	size_t len;
-	size_t cons;
-	u8 level;
-	bool flushed:1;
-} cont;
 static char *log_text(const struct printk_log *msg) { return NULL; }
 static char *log_dict(const struct printk_log *msg) { return NULL; }
 static struct printk_log *log_from_idx(u32 idx) { return NULL; }
 static u32 log_next(u32 idx) { return 0; }
 static ssize_t msg_print_ext_header(char *buf, size_t size,
-				    struct printk_log *msg, u64 seq,
-				    enum log_flags prev_flags) { return 0; }
+				    struct printk_log *msg,
+				    u64 seq) { return 0; }
 static ssize_t msg_print_ext_body(char *buf, size_t size,
 				  char *dict, size_t dict_len,
 				  char *text, size_t text_len) { return 0; }
 static void call_console_drivers(int level,
 				 const char *ext_text, size_t ext_len,
 				 const char *text, size_t len) {}
-static size_t msg_print_text(const struct printk_log *msg, enum log_flags prev,
+static size_t msg_print_text(const struct printk_log *msg,
 			     bool syslog, char *buf, size_t size) { return 0; }
-static size_t cont_print_text(char *text, size_t size) { return 0; }
 static bool suppress_message_printing(int level) { return false; }
 
 /* Still needs to be defined for users */
@@ -2270,42 +2163,6 @@ static inline int can_use_console(void)
 	return cpu_online(raw_smp_processor_id()) || have_callable_console();
 }
 
-static void console_cont_flush(char *text, size_t size)
-{
-	unsigned long flags;
-	size_t len;
-
-	raw_spin_lock_irqsave(&logbuf_lock, flags);
-
-	if (!cont.len)
-		goto out;
-
-	if (suppress_message_printing(cont.level)) {
-		cont.cons = cont.len;
-		if (cont.flushed)
-			cont.len = 0;
-		goto out;
-	}
-
-	/*
-	 * We still queue earlier records, likely because the console was
-	 * busy. The earlier ones need to be printed before this one, we
-	 * did not flush any fragment so far, so just let it queue up.
-	 */
-	if (console_seq < log_next_seq && !cont.cons)
-		goto out;
-
-	len = cont_print_text(text, size);
-	raw_spin_unlock(&logbuf_lock);
-	stop_critical_timings();
-	call_console_drivers(cont.level, NULL, 0, text, len);
-	start_critical_timings();
-	local_irq_restore(flags);
-	return;
-out:
-	raw_spin_unlock_irqrestore(&logbuf_lock, flags);
-}
-
 /**
  * console_unlock - unlock the console system
  *
@@ -2359,9 +2216,6 @@ again:
 		return;
 	}
 
-	/* flush buffered message fragment immediately to console */
-	console_cont_flush(text, sizeof(text));
-
 	for (;;) {
 		struct printk_log *msg;
 		size_t ext_len = 0;
@@ -2381,7 +2235,6 @@ again:
 			/* messages are gone, move to first one */
 			console_seq = log_first_seq;
 			console_idx = log_first_idx;
-			console_prev = 0;
 		} else {
 			len = 0;
 		}
@@ -2391,8 +2244,7 @@ skip:
 
 		msg = log_from_idx(console_idx);
 		level = msg->level;
-		if ((msg->flags & LOG_NOCONS) ||
-				suppress_message_printing(level)) {
+		if (suppress_message_printing(level)) {
 			/*
 			 * Skip record we have buffered and already printed
 			 * directly to the console when we received it, and
@@ -2400,22 +2252,14 @@ skip:
 			 */
 			console_idx = log_next(console_idx);
 			console_seq++;
-			/*
-			 * We will get here again when we register a new
-			 * CON_PRINTBUFFER console. Clear the flag so we
-			 * will properly dump everything later.
-			 */
-			msg->flags &= ~LOG_NOCONS;
-			console_prev = msg->flags;
 			goto skip;
 		}
 
-		len += msg_print_text(msg, console_prev, false,
-				      text + len, sizeof(text) - len);
+		len += msg_print_text(msg, false, text + len, sizeof(text) - len);
 		if (nr_ext_console_drivers) {
 			ext_len = msg_print_ext_header(ext_text,
 						sizeof(ext_text),
-						msg, console_seq, console_prev);
+						msg, console_seq);
 			ext_len += msg_print_ext_body(ext_text + ext_len,
 						sizeof(ext_text) - ext_len,
 						log_dict(msg), msg->dict_len,
@@ -2423,7 +2267,6 @@ skip:
 		}
 		console_idx = log_next(console_idx);
 		console_seq++;
-		console_prev = msg->flags;
 		raw_spin_unlock(&logbuf_lock);
 
 		stop_critical_timings();	/* don't trace print latency */
@@ -2718,7 +2561,6 @@ void register_console(struct console *newcon)
 		raw_spin_lock_irqsave(&logbuf_lock, flags);
 		console_seq = syslog_seq;
 		console_idx = syslog_idx;
-		console_prev = syslog_prev;
 		raw_spin_unlock_irqrestore(&logbuf_lock, flags);
 		/*
 		 * We're about to replay the log buffer.  Only do this to the
@@ -3074,7 +2916,7 @@ bool kmsg_dump_get_line_nolock(struct kmsg_dumper *dumper, bool syslog,
 		goto out;
 
 	msg = log_from_idx(dumper->cur_idx);
-	l = msg_print_text(msg, 0, syslog, line, size);
+	l = msg_print_text(msg, syslog, line, size);
 
 	dumper->cur_idx = log_next(dumper->cur_idx);
 	dumper->cur_seq++;
@@ -3143,7 +2985,6 @@ bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog,
 	u32 idx;
 	u64 next_seq;
 	u32 next_idx;
-	enum log_flags prev;
 	size_t l = 0;
 	bool ret = false;
 
@@ -3166,27 +3007,23 @@ bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog,
 	/* calculate length of entire buffer */
 	seq = dumper->cur_seq;
 	idx = dumper->cur_idx;
-	prev = 0;
 	while (seq < dumper->next_seq) {
 		struct printk_log *msg = log_from_idx(idx);
 
-		l += msg_print_text(msg, prev, true, NULL, 0);
+		l += msg_print_text(msg, true, NULL, 0);
 		idx = log_next(idx);
 		seq++;
-		prev = msg->flags;
 	}
 
 	/* move first record forward until length fits into the buffer */
 	seq = dumper->cur_seq;
 	idx = dumper->cur_idx;
-	prev = 0;
 	while (l > size && seq < dumper->next_seq) {
 		struct printk_log *msg = log_from_idx(idx);
 
-		l -= msg_print_text(msg, prev, true, NULL, 0);
+		l -= msg_print_text(msg, true, NULL, 0);
 		idx = log_next(idx);
 		seq++;
-		prev = msg->flags;
 	}
 
 	/* last message in next interation */
@@ -3197,10 +3034,9 @@ bool kmsg_dump_get_buffer(struct kmsg_dumper *dumper, bool syslog,
 	while (seq < dumper->next_seq) {
 		struct printk_log *msg = log_from_idx(idx);
 
-		l += msg_print_text(msg, prev, syslog, buf + l, size - l);
+		l += msg_print_text(msg, syslog, buf + l, size - l);
 		idx = log_next(idx);
 		seq++;
-		prev = msg->flags;
 	}
 
 	dumper->next_seq = next_seq;
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index e6474f7272ec..49ba7c1ade9d 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -27,6 +27,35 @@
 #include <linux/cn_proc.h>
 #include <linux/compat.h>
 
+/*
+ * Access another process' address space via ptrace.
+ * Source/target buffer must be kernel space,
+ * Do not walk the page table directly, use get_user_pages
+ */
+int ptrace_access_vm(struct task_struct *tsk, unsigned long addr,
+		     void *buf, int len, unsigned int gup_flags)
+{
+	struct mm_struct *mm;
+	int ret;
+
+	mm = get_task_mm(tsk);
+	if (!mm)
+		return 0;
+
+	if (!tsk->ptrace ||
+	    (current != tsk->parent) ||
+	    ((get_dumpable(mm) != SUID_DUMP_USER) &&
+	     !ptracer_capable(tsk, mm->user_ns))) {
+		mmput(mm);
+		return 0;
+	}
+
+	ret = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags);
+	mmput(mm);
+
+	return ret;
+}
+
 
 /*
  * ptrace a task: make the debugger its new parent and
@@ -39,6 +68,9 @@ void __ptrace_link(struct task_struct *child, struct task_struct *new_parent)
 	BUG_ON(!list_empty(&child->ptrace_entry));
 	list_add(&child->ptrace_entry, &new_parent->ptraced);
 	child->parent = new_parent;
+	rcu_read_lock();
+	child->ptracer_cred = get_cred(__task_cred(new_parent));
+	rcu_read_unlock();
 }
 
 /**
@@ -71,12 +103,16 @@ void __ptrace_link(struct task_struct *child, struct task_struct *new_parent)
  */
 void __ptrace_unlink(struct task_struct *child)
 {
+	const struct cred *old_cred;
 	BUG_ON(!child->ptrace);
 
 	clear_tsk_thread_flag(child, TIF_SYSCALL_TRACE);
 
 	child->parent = child->real_parent;
 	list_del_init(&child->ptrace_entry);
+	old_cred = child->ptracer_cred;
+	child->ptracer_cred = NULL;
+	put_cred(old_cred);
 
 	spin_lock(&child->sighand->siglock);
 	child->ptrace = 0;
@@ -220,7 +256,7 @@ static int ptrace_has_cap(struct user_namespace *ns, unsigned int mode)
 static int __ptrace_may_access(struct task_struct *task, unsigned int mode)
 {
 	const struct cred *cred = current_cred(), *tcred;
-	int dumpable = 0;
+	struct mm_struct *mm;
 	kuid_t caller_uid;
 	kgid_t caller_gid;
 
@@ -271,16 +307,11 @@ static int __ptrace_may_access(struct task_struct *task, unsigned int mode)
 	return -EPERM;
 ok:
 	rcu_read_unlock();
-	smp_rmb();
-	if (task->mm)
-		dumpable = get_dumpable(task->mm);
-	rcu_read_lock();
-	if (dumpable != SUID_DUMP_USER &&
-	    !ptrace_has_cap(__task_cred(task)->user_ns, mode)) {
-		rcu_read_unlock();
-		return -EPERM;
-	}
-	rcu_read_unlock();
+	mm = task->mm;
+	if (mm &&
+	    ((get_dumpable(mm) != SUID_DUMP_USER) &&
+	     !ptrace_has_cap(mm->user_ns, mode)))
+	    return -EPERM;
 
 	return security_ptrace_access_check(task, mode);
 }
@@ -344,10 +375,6 @@ static int ptrace_attach(struct task_struct *task, long request,
 
 	if (seize)
 		flags |= PT_SEIZED;
-	rcu_read_lock();
-	if (ns_capable(__task_cred(task)->user_ns, CAP_SYS_PTRACE))
-		flags |= PT_PTRACE_CAP;
-	rcu_read_unlock();
 	task->ptrace = flags;
 
 	__ptrace_link(task, current);
@@ -537,7 +564,8 @@ int ptrace_readdata(struct task_struct *tsk, unsigned long src, char __user *dst
 		int this_len, retval;
 
 		this_len = (len > sizeof(buf)) ? sizeof(buf) : len;
-		retval = access_process_vm(tsk, src, buf, this_len, FOLL_FORCE);
+		retval = ptrace_access_vm(tsk, src, buf, this_len, FOLL_FORCE);
+
 		if (!retval) {
 			if (copied)
 				break;
@@ -564,7 +592,7 @@ int ptrace_writedata(struct task_struct *tsk, char __user *src, unsigned long ds
 		this_len = (len > sizeof(buf)) ? sizeof(buf) : len;
 		if (copy_from_user(buf, src, this_len))
 			return -EFAULT;
-		retval = access_process_vm(tsk, dst, buf, this_len,
+		retval = ptrace_access_vm(tsk, dst, buf, this_len,
 				FOLL_FORCE | FOLL_WRITE);
 		if (!retval) {
 			if (copied)
@@ -1128,7 +1156,7 @@ int generic_ptrace_peekdata(struct task_struct *tsk, unsigned long addr,
 	unsigned long tmp;
 	int copied;
 
-	copied = access_process_vm(tsk, addr, &tmp, sizeof(tmp), FOLL_FORCE);
+	copied = ptrace_access_vm(tsk, addr, &tmp, sizeof(tmp), FOLL_FORCE);
 	if (copied != sizeof(tmp))
 		return -EIO;
 	return put_user(tmp, (unsigned long __user *)data);
@@ -1139,7 +1167,7 @@ int generic_ptrace_pokedata(struct task_struct *tsk, unsigned long addr,
 {
 	int copied;
 
-	copied = access_process_vm(tsk, addr, &data, sizeof(data),
+	copied = ptrace_access_vm(tsk, addr, &data, sizeof(data),
 			FOLL_FORCE | FOLL_WRITE);
 	return (copied == sizeof(data)) ? 0 : -EIO;
 }
@@ -1157,7 +1185,7 @@ int compat_ptrace_request(struct task_struct *child, compat_long_t request,
 	switch (request) {
 	case PTRACE_PEEKTEXT:
 	case PTRACE_PEEKDATA:
-		ret = access_process_vm(child, addr, &word, sizeof(word),
+		ret = ptrace_access_vm(child, addr, &word, sizeof(word),
 				FOLL_FORCE);
 		if (ret != sizeof(word))
 			ret = -EIO;
@@ -1167,7 +1195,7 @@ int compat_ptrace_request(struct task_struct *child, compat_long_t request,
 
 	case PTRACE_POKETEXT:
 	case PTRACE_POKEDATA:
-		ret = access_process_vm(child, addr, &data, sizeof(data),
+		ret = ptrace_access_vm(child, addr, &data, sizeof(data),
 				FOLL_FORCE | FOLL_WRITE);
 		ret = (ret != sizeof(data) ? -EIO : 0);
 		break;
diff --git a/kernel/relay.c b/kernel/relay.c
index da79a109dbeb..8f18d314a96a 100644
--- a/kernel/relay.c
+++ b/kernel/relay.c
@@ -809,11 +809,11 @@ void relay_subbufs_consumed(struct rchan *chan,
 {
 	struct rchan_buf *buf;
 
-	if (!chan)
+	if (!chan || cpu >= NR_CPUS)
 		return;
 
 	buf = *per_cpu_ptr(chan->buf, cpu);
-	if (cpu >= NR_CPUS || !buf || subbufs_consumed > chan->n_subbufs)
+	if (!buf || subbufs_consumed > chan->n_subbufs)
 		return;
 
 	if (subbufs_consumed > buf->subbufs_produced - buf->subbufs_consumed)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index d18804491d9f..966556ebdbb3 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5280,6 +5280,7 @@ void init_idle(struct task_struct *idle, int cpu)
 	__sched_fork(0, idle);
 	idle->state = TASK_RUNNING;
 	idle->se.exec_start = sched_clock();
+	idle->flags |= PF_IDLE;
 
 	kasan_unpoison_task_stack(idle);
 
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index 69e06898997d..fd4659313640 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -12,11 +12,14 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/cpufreq.h>
+#include <linux/kthread.h>
 #include <linux/slab.h>
 #include <trace/events/power.h>
 
 #include "sched.h"
 
+#define SUGOV_KTHREAD_PRIORITY	50
+
 struct sugov_tunables {
 	struct gov_attr_set attr_set;
 	unsigned int rate_limit_us;
@@ -35,8 +38,10 @@ struct sugov_policy {
 
 	/* The next fields are only needed if fast switch cannot be used. */
 	struct irq_work irq_work;
-	struct work_struct work;
+	struct kthread_work work;
 	struct mutex work_lock;
+	struct kthread_worker worker;
+	struct task_struct *thread;
 	bool work_in_progress;
 
 	bool need_freq_update;
@@ -291,7 +296,7 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time,
 	raw_spin_unlock(&sg_policy->update_lock);
 }
 
-static void sugov_work(struct work_struct *work)
+static void sugov_work(struct kthread_work *work)
 {
 	struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work);
 
@@ -308,7 +313,21 @@ static void sugov_irq_work(struct irq_work *irq_work)
 	struct sugov_policy *sg_policy;
 
 	sg_policy = container_of(irq_work, struct sugov_policy, irq_work);
-	schedule_work_on(smp_processor_id(), &sg_policy->work);
+
+	/*
+	 * For RT and deadline tasks, the schedutil governor shoots the
+	 * frequency to maximum. Special care must be taken to ensure that this
+	 * kthread doesn't result in the same behavior.
+	 *
+	 * This is (mostly) guaranteed by the work_in_progress flag. The flag is
+	 * updated only at the end of the sugov_work() function and before that
+	 * the schedutil governor rejects all other frequency scaling requests.
+	 *
+	 * There is a very rare case though, where the RT thread yields right
+	 * after the work_in_progress flag is cleared. The effects of that are
+	 * neglected for now.
+	 */
+	kthread_queue_work(&sg_policy->worker, &sg_policy->work);
 }
 
 /************************** sysfs interface ************************/
@@ -371,19 +390,64 @@ static struct sugov_policy *sugov_policy_alloc(struct cpufreq_policy *policy)
 		return NULL;
 
 	sg_policy->policy = policy;
-	init_irq_work(&sg_policy->irq_work, sugov_irq_work);
-	INIT_WORK(&sg_policy->work, sugov_work);
-	mutex_init(&sg_policy->work_lock);
 	raw_spin_lock_init(&sg_policy->update_lock);
 	return sg_policy;
 }
 
 static void sugov_policy_free(struct sugov_policy *sg_policy)
 {
-	mutex_destroy(&sg_policy->work_lock);
 	kfree(sg_policy);
 }
 
+static int sugov_kthread_create(struct sugov_policy *sg_policy)
+{
+	struct task_struct *thread;
+	struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO / 2 };
+	struct cpufreq_policy *policy = sg_policy->policy;
+	int ret;
+
+	/* kthread only required for slow path */
+	if (policy->fast_switch_enabled)
+		return 0;
+
+	kthread_init_work(&sg_policy->work, sugov_work);
+	kthread_init_worker(&sg_policy->worker);
+	thread = kthread_create(kthread_worker_fn, &sg_policy->worker,
+				"sugov:%d",
+				cpumask_first(policy->related_cpus));
+	if (IS_ERR(thread)) {
+		pr_err("failed to create sugov thread: %ld\n", PTR_ERR(thread));
+		return PTR_ERR(thread);
+	}
+
+	ret = sched_setscheduler_nocheck(thread, SCHED_FIFO, &param);
+	if (ret) {
+		kthread_stop(thread);
+		pr_warn("%s: failed to set SCHED_FIFO\n", __func__);
+		return ret;
+	}
+
+	sg_policy->thread = thread;
+	kthread_bind_mask(thread, policy->related_cpus);
+	init_irq_work(&sg_policy->irq_work, sugov_irq_work);
+	mutex_init(&sg_policy->work_lock);
+
+	wake_up_process(thread);
+
+	return 0;
+}
+
+static void sugov_kthread_stop(struct sugov_policy *sg_policy)
+{
+	/* kthread only required for slow path */
+	if (sg_policy->policy->fast_switch_enabled)
+		return;
+
+	kthread_flush_worker(&sg_policy->worker);
+	kthread_stop(sg_policy->thread);
+	mutex_destroy(&sg_policy->work_lock);
+}
+
 static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy)
 {
 	struct sugov_tunables *tunables;
@@ -416,16 +480,24 @@ static int sugov_init(struct cpufreq_policy *policy)
 	if (policy->governor_data)
 		return -EBUSY;
 
+	cpufreq_enable_fast_switch(policy);
+
 	sg_policy = sugov_policy_alloc(policy);
-	if (!sg_policy)
-		return -ENOMEM;
+	if (!sg_policy) {
+		ret = -ENOMEM;
+		goto disable_fast_switch;
+	}
+
+	ret = sugov_kthread_create(sg_policy);
+	if (ret)
+		goto free_sg_policy;
 
 	mutex_lock(&global_tunables_lock);
 
 	if (global_tunables) {
 		if (WARN_ON(have_governor_per_policy())) {
 			ret = -EINVAL;
-			goto free_sg_policy;
+			goto stop_kthread;
 		}
 		policy->governor_data = sg_policy;
 		sg_policy->tunables = global_tunables;
@@ -437,7 +509,7 @@ static int sugov_init(struct cpufreq_policy *policy)
 	tunables = sugov_tunables_alloc(sg_policy);
 	if (!tunables) {
 		ret = -ENOMEM;
-		goto free_sg_policy;
+		goto stop_kthread;
 	}
 
 	tunables->rate_limit_us = LATENCY_MULTIPLIER;
@@ -454,20 +526,25 @@ static int sugov_init(struct cpufreq_policy *policy)
 	if (ret)
 		goto fail;
 
- out:
+out:
 	mutex_unlock(&global_tunables_lock);
-
-	cpufreq_enable_fast_switch(policy);
 	return 0;
 
- fail:
+fail:
 	policy->governor_data = NULL;
 	sugov_tunables_free(tunables);
 
- free_sg_policy:
+stop_kthread:
+	sugov_kthread_stop(sg_policy);
+
+free_sg_policy:
 	mutex_unlock(&global_tunables_lock);
 
 	sugov_policy_free(sg_policy);
+
+disable_fast_switch:
+	cpufreq_disable_fast_switch(policy);
+
 	pr_err("initialization failed (error %d)\n", ret);
 	return ret;
 }
@@ -478,8 +555,6 @@ static void sugov_exit(struct cpufreq_policy *policy)
 	struct sugov_tunables *tunables = sg_policy->tunables;
 	unsigned int count;
 
-	cpufreq_disable_fast_switch(policy);
-
 	mutex_lock(&global_tunables_lock);
 
 	count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);
@@ -489,7 +564,9 @@ static void sugov_exit(struct cpufreq_policy *policy)
 
 	mutex_unlock(&global_tunables_lock);
 
+	sugov_kthread_stop(sg_policy);
 	sugov_policy_free(sg_policy);
+	cpufreq_disable_fast_switch(policy);
 }
 
 static int sugov_start(struct cpufreq_policy *policy)
@@ -535,8 +612,10 @@ static void sugov_stop(struct cpufreq_policy *policy)
 
 	synchronize_sched();
 
-	irq_work_sync(&sg_policy->irq_work);
-	cancel_work_sync(&sg_policy->work);
+	if (!policy->fast_switch_enabled) {
+		irq_work_sync(&sg_policy->irq_work);
+		kthread_cancel_work_sync(&sg_policy->work);
+	}
 }
 
 static void sugov_limits(struct cpufreq_policy *policy)
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 1d8718d5300d..6a4bae0a649d 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -164,11 +164,14 @@ static void cpuidle_idle_call(void)
 	 * timekeeping to prevent timer interrupts from kicking us out of idle
 	 * until a proper wakeup interrupt happens.
 	 */
-	if (idle_should_freeze()) {
-		entered_state = cpuidle_enter_freeze(drv, dev);
-		if (entered_state > 0) {
-			local_irq_enable();
-			goto exit_idle;
+
+	if (idle_should_freeze() || dev->use_deepest_state) {
+		if (idle_should_freeze()) {
+			entered_state = cpuidle_enter_freeze(drv, dev);
+			if (entered_state > 0) {
+				local_irq_enable();
+				goto exit_idle;
+			}
 		}
 
 		next_state = cpuidle_find_deepest_state(drv, dev);
@@ -202,76 +205,65 @@ exit_idle:
  *
  * Called with polling cleared.
  */
-static void cpu_idle_loop(void)
+static void do_idle(void)
 {
-	int cpu = smp_processor_id();
-
-	while (1) {
-		/*
-		 * If the arch has a polling bit, we maintain an invariant:
-		 *
-		 * Our polling bit is clear if we're not scheduled (i.e. if
-		 * rq->curr != rq->idle).  This means that, if rq->idle has
-		 * the polling bit set, then setting need_resched is
-		 * guaranteed to cause the cpu to reschedule.
-		 */
-
-		__current_set_polling();
-		quiet_vmstat();
-		tick_nohz_idle_enter();
+	/*
+	 * If the arch has a polling bit, we maintain an invariant:
+	 *
+	 * Our polling bit is clear if we're not scheduled (i.e. if rq->curr !=
+	 * rq->idle). This means that, if rq->idle has the polling bit set,
+	 * then setting need_resched is guaranteed to cause the CPU to
+	 * reschedule.
+	 */
 
-		while (!need_resched()) {
-			check_pgt_cache();
-			rmb();
+	__current_set_polling();
+	tick_nohz_idle_enter();
 
-			if (cpu_is_offline(cpu)) {
-				cpuhp_report_idle_dead();
-				arch_cpu_idle_dead();
-			}
+	while (!need_resched()) {
+		check_pgt_cache();
+		rmb();
 
-			local_irq_disable();
-			arch_cpu_idle_enter();
-
-			/*
-			 * In poll mode we reenable interrupts and spin.
-			 *
-			 * Also if we detected in the wakeup from idle
-			 * path that the tick broadcast device expired
-			 * for us, we don't want to go deep idle as we
-			 * know that the IPI is going to arrive right
-			 * away
-			 */
-			if (cpu_idle_force_poll || tick_check_broadcast_expired())
-				cpu_idle_poll();
-			else
-				cpuidle_idle_call();
-
-			arch_cpu_idle_exit();
+		if (cpu_is_offline(smp_processor_id())) {
+			cpuhp_report_idle_dead();
+			arch_cpu_idle_dead();
 		}
 
-		/*
-		 * Since we fell out of the loop above, we know
-		 * TIF_NEED_RESCHED must be set, propagate it into
-		 * PREEMPT_NEED_RESCHED.
-		 *
-		 * This is required because for polling idle loops we will
-		 * not have had an IPI to fold the state for us.
-		 */
-		preempt_set_need_resched();
-		tick_nohz_idle_exit();
-		__current_clr_polling();
+		local_irq_disable();
+		arch_cpu_idle_enter();
 
 		/*
-		 * We promise to call sched_ttwu_pending and reschedule
-		 * if need_resched is set while polling is set.  That
-		 * means that clearing polling needs to be visible
-		 * before doing these things.
+		 * In poll mode we reenable interrupts and spin. Also if we
+		 * detected in the wakeup from idle path that the tick
+		 * broadcast device expired for us, we don't want to go deep
+		 * idle as we know that the IPI is going to arrive right away.
 		 */
-		smp_mb__after_atomic();
-
-		sched_ttwu_pending();
-		schedule_preempt_disabled();
+		if (cpu_idle_force_poll || tick_check_broadcast_expired())
+			cpu_idle_poll();
+		else
+			cpuidle_idle_call();
+		arch_cpu_idle_exit();
 	}
+
+	/*
+	 * Since we fell out of the loop above, we know TIF_NEED_RESCHED must
+	 * be set, propagate it into PREEMPT_NEED_RESCHED.
+	 *
+	 * This is required because for polling idle loops we will not have had
+	 * an IPI to fold the state for us.
+	 */
+	preempt_set_need_resched();
+	tick_nohz_idle_exit();
+	__current_clr_polling();
+
+	/*
+	 * We promise to call sched_ttwu_pending() and reschedule if
+	 * need_resched() is set while polling is set. That means that clearing
+	 * polling needs to be visible before doing these things.
+	 */
+	smp_mb__after_atomic();
+
+	sched_ttwu_pending();
+	schedule_preempt_disabled();
 }
 
 bool cpu_in_idle(unsigned long pc)
@@ -280,6 +272,56 @@ bool cpu_in_idle(unsigned long pc)
 		pc < (unsigned long)__cpuidle_text_end;
 }
 
+struct idle_timer {
+	struct hrtimer timer;
+	int done;
+};
+
+static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
+{
+	struct idle_timer *it = container_of(timer, struct idle_timer, timer);
+
+	WRITE_ONCE(it->done, 1);
+	set_tsk_need_resched(current);
+
+	return HRTIMER_NORESTART;
+}
+
+void play_idle(unsigned long duration_ms)
+{
+	struct idle_timer it;
+
+	/*
+	 * Only FIFO tasks can disable the tick since they don't need the forced
+	 * preemption.
+	 */
+	WARN_ON_ONCE(current->policy != SCHED_FIFO);
+	WARN_ON_ONCE(current->nr_cpus_allowed != 1);
+	WARN_ON_ONCE(!(current->flags & PF_KTHREAD));
+	WARN_ON_ONCE(!(current->flags & PF_NO_SETAFFINITY));
+	WARN_ON_ONCE(!duration_ms);
+
+	rcu_sleep_check();
+	preempt_disable();
+	current->flags |= PF_IDLE;
+	cpuidle_use_deepest_state(true);
+
+	it.done = 0;
+	hrtimer_init_on_stack(&it.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	it.timer.function = idle_inject_timer_fn;
+	hrtimer_start(&it.timer, ms_to_ktime(duration_ms), HRTIMER_MODE_REL_PINNED);
+
+	while (!READ_ONCE(it.done))
+		do_idle();
+
+	cpuidle_use_deepest_state(false);
+	current->flags &= ~PF_IDLE;
+
+	preempt_fold_need_resched();
+	preempt_enable();
+}
+EXPORT_SYMBOL_GPL(play_idle);
+
 void cpu_startup_entry(enum cpuhp_state state)
 {
 	/*
@@ -299,5 +341,6 @@ void cpu_startup_entry(enum cpuhp_state state)
 #endif
 	arch_cpu_idle_prepare();
 	cpuhp_online_idle(state);
-	cpu_idle_loop();
+	while (1)
+		do_idle();
 }
diff --git a/kernel/seccomp.c b/kernel/seccomp.c
index bff9c774987a..f7ce79a46050 100644
--- a/kernel/seccomp.c
+++ b/kernel/seccomp.c
@@ -41,8 +41,7 @@
  *         outside of a lifetime-guarded section.  In general, this
  *         is only needed for handling filters shared across tasks.
  * @prev: points to a previously installed, or inherited, filter
- * @len: the number of instructions in the program
- * @insnsi: the BPF program instructions to evaluate
+ * @prog: the BPF program to evaluate
  *
  * seccomp_filter objects are organized in a tree linked via the @prev
  * pointer.  For any task, it appears to be a singly-linked list starting
@@ -168,8 +167,8 @@ static int seccomp_check_filter(struct sock_filter *filter, unsigned int flen)
 }
 
 /**
- * seccomp_run_filters - evaluates all seccomp filters against @syscall
- * @syscall: number of the current system call
+ * seccomp_run_filters - evaluates all seccomp filters against @sd
+ * @sd: optional seccomp data to be passed to filters
  *
  * Returns valid seccomp BPF response codes.
  */
diff --git a/kernel/signal.c b/kernel/signal.c
index 29a410780aa9..ae60996fedff 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -2491,6 +2491,13 @@ void __set_current_blocked(const sigset_t *newset)
 {
 	struct task_struct *tsk = current;
 
+	/*
+	 * In case the signal mask hasn't changed, there is nothing we need
+	 * to do. The current->blocked shouldn't be modified by other task.
+	 */
+	if (sigequalsets(&tsk->blocked, newset))
+		return;
+
 	spin_lock_irq(&tsk->sighand->siglock);
 	__set_task_blocked(tsk, newset);
 	spin_unlock_irq(&tsk->sighand->siglock);
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 39b3368f6de6..1a292ebcbbb6 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -627,7 +627,7 @@ static struct ctl_table kern_table[] = {
 		.data		= &tracepoint_printk,
 		.maxlen		= sizeof(tracepoint_printk),
 		.mode		= 0644,
-		.proc_handler	= proc_dointvec,
+		.proc_handler	= tracepoint_printk_sysctl,
 	},
 #endif
 #ifdef CONFIG_KEXEC_CORE
@@ -2389,9 +2389,11 @@ static void validate_coredump_safety(void)
 #ifdef CONFIG_COREDUMP
 	if (suid_dumpable == SUID_DUMP_ROOT &&
 	    core_pattern[0] != '/' && core_pattern[0] != '|') {
-		printk(KERN_WARNING "Unsafe core_pattern used with "\
-			"suid_dumpable=2. Pipe handler or fully qualified "\
-			"core dump path required.\n");
+		printk(KERN_WARNING
+"Unsafe core_pattern used with fs.suid_dumpable=2.\n"
+"Pipe handler or fully qualified core dump path required.\n"
+"Set kernel.core_pattern before fs.suid_dumpable.\n"
+		);
 	}
 #endif
 }
diff --git a/kernel/sysctl_binary.c b/kernel/sysctl_binary.c
index 6eb99c17dbd8..ece4b177052b 100644
--- a/kernel/sysctl_binary.c
+++ b/kernel/sysctl_binary.c
@@ -1354,8 +1354,8 @@ static void deprecated_sysctl_warning(const int *name, int nlen)
 			"warning: process `%s' used the deprecated sysctl "
 			"system call with ", current->comm);
 		for (i = 0; i < nlen; i++)
-			printk("%d.", name[i]);
-		printk("\n");
+			printk(KERN_CONT "%d.", name[i]);
+		printk(KERN_CONT "\n");
 	}
 	return;
 }
diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c
index 9b08ca391aed..3921cf7fea8e 100644
--- a/kernel/time/alarmtimer.c
+++ b/kernel/time/alarmtimer.c
@@ -516,7 +516,8 @@ static enum alarmtimer_restart alarm_handle_timer(struct alarm *alarm,
 
 	spin_lock_irqsave(&ptr->it_lock, flags);
 	if ((ptr->it_sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE) {
-		if (posix_timer_event(ptr, 0) != 0)
+		if (IS_ENABLED(CONFIG_POSIX_TIMERS) &&
+		    posix_timer_event(ptr, 0) != 0)
 			ptr->it_overrun++;
 	}
 
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 2a96b063d659..d5038005eb5d 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -70,6 +70,7 @@ config FTRACE_NMI_ENTER
 
 config EVENT_TRACING
 	select CONTEXT_SWITCH_TRACER
+        select GLOB
 	bool
 
 config CONTEXT_SWITCH_TRACER
@@ -133,6 +134,7 @@ config FUNCTION_TRACER
 	select KALLSYMS
 	select GENERIC_TRACER
 	select CONTEXT_SWITCH_TRACER
+        select GLOB
 	help
 	  Enable the kernel to trace every kernel function. This is done
 	  by using a compiler feature to insert a small, 5-byte No-Operation
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c
index dbafc5df03f3..95cecbf67f5c 100644
--- a/kernel/trace/blktrace.c
+++ b/kernel/trace/blktrace.c
@@ -1777,14 +1777,14 @@ void blk_dump_cmd(char *buf, struct request *rq)
 	}
 }
 
-void blk_fill_rwbs(char *rwbs, int op, u32 rw, int bytes)
+void blk_fill_rwbs(char *rwbs, unsigned int op, int bytes)
 {
 	int i = 0;
 
-	if (rw & REQ_PREFLUSH)
+	if (op & REQ_PREFLUSH)
 		rwbs[i++] = 'F';
 
-	switch (op) {
+	switch (op & REQ_OP_MASK) {
 	case REQ_OP_WRITE:
 	case REQ_OP_WRITE_SAME:
 		rwbs[i++] = 'W';
@@ -1806,13 +1806,13 @@ void blk_fill_rwbs(char *rwbs, int op, u32 rw, int bytes)
 		rwbs[i++] = 'N';
 	}
 
-	if (rw & REQ_FUA)
+	if (op & REQ_FUA)
 		rwbs[i++] = 'F';
-	if (rw & REQ_RAHEAD)
+	if (op & REQ_RAHEAD)
 		rwbs[i++] = 'A';
-	if (rw & REQ_SYNC)
+	if (op & REQ_SYNC)
 		rwbs[i++] = 'S';
-	if (rw & REQ_META)
+	if (op & REQ_META)
 		rwbs[i++] = 'M';
 
 	rwbs[i] = '\0';
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index da87b3cba5b3..1f0f547c54da 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -3511,6 +3511,10 @@ static int ftrace_match(char *str, struct ftrace_glob *g)
 		    memcmp(str + slen - g->len, g->search, g->len) == 0)
 			matched = 1;
 		break;
+	case MATCH_GLOB:
+		if (glob_match(g->search, str))
+			matched = 1;
+		break;
 	}
 
 	return matched;
@@ -4258,6 +4262,23 @@ int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
 }
 EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
 
+/**
+ * ftrace_ops_set_global_filter - setup ops to use global filters
+ * @ops - the ops which will use the global filters
+ *
+ * ftrace users who need global function trace filtering should call this.
+ * It can set the global filter only if ops were not initialized before.
+ */
+void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
+{
+	if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
+		return;
+
+	ftrace_ops_init(ops);
+	ops->func_hash = &global_ops.local_hash;
+}
+EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);
+
 static int
 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
 		 int reset, int enable)
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 89a2611a1635..a85739efcc30 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -245,7 +245,7 @@ unsigned ring_buffer_event_length(struct ring_buffer_event *event)
 EXPORT_SYMBOL_GPL(ring_buffer_event_length);
 
 /* inline for ring buffer fast paths */
-static void *
+static __always_inline void *
 rb_event_data(struct ring_buffer_event *event)
 {
 	if (event->type_len == RINGBUF_TYPE_TIME_EXTEND)
@@ -1798,48 +1798,48 @@ void ring_buffer_change_overwrite(struct ring_buffer *buffer, int val)
 }
 EXPORT_SYMBOL_GPL(ring_buffer_change_overwrite);
 
-static inline void *
+static __always_inline void *
 __rb_data_page_index(struct buffer_data_page *bpage, unsigned index)
 {
 	return bpage->data + index;
 }
 
-static inline void *__rb_page_index(struct buffer_page *bpage, unsigned index)
+static __always_inline void *__rb_page_index(struct buffer_page *bpage, unsigned index)
 {
 	return bpage->page->data + index;
 }
 
-static inline struct ring_buffer_event *
+static __always_inline struct ring_buffer_event *
 rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer)
 {
 	return __rb_page_index(cpu_buffer->reader_page,
 			       cpu_buffer->reader_page->read);
 }
 
-static inline struct ring_buffer_event *
+static __always_inline struct ring_buffer_event *
 rb_iter_head_event(struct ring_buffer_iter *iter)
 {
 	return __rb_page_index(iter->head_page, iter->head);
 }
 
-static inline unsigned rb_page_commit(struct buffer_page *bpage)
+static __always_inline unsigned rb_page_commit(struct buffer_page *bpage)
 {
 	return local_read(&bpage->page->commit);
 }
 
 /* Size is determined by what has been committed */
-static inline unsigned rb_page_size(struct buffer_page *bpage)
+static __always_inline unsigned rb_page_size(struct buffer_page *bpage)
 {
 	return rb_page_commit(bpage);
 }
 
-static inline unsigned
+static __always_inline unsigned
 rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer)
 {
 	return rb_page_commit(cpu_buffer->commit_page);
 }
 
-static inline unsigned
+static __always_inline unsigned
 rb_event_index(struct ring_buffer_event *event)
 {
 	unsigned long addr = (unsigned long)event;
@@ -2355,7 +2355,7 @@ static void rb_start_commit(struct ring_buffer_per_cpu *cpu_buffer)
 	local_inc(&cpu_buffer->commits);
 }
 
-static void
+static __always_inline void
 rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
 {
 	unsigned long max_count;
@@ -2410,7 +2410,7 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
 		goto again;
 }
 
-static inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer)
+static __always_inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer)
 {
 	unsigned long commits;
 
@@ -2455,7 +2455,7 @@ static inline void rb_event_discard(struct ring_buffer_event *event)
 		event->time_delta = 1;
 }
 
-static inline bool
+static __always_inline bool
 rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
 		   struct ring_buffer_event *event)
 {
@@ -2469,7 +2469,7 @@ rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
 		rb_commit_index(cpu_buffer) == index;
 }
 
-static void
+static __always_inline void
 rb_update_write_stamp(struct ring_buffer_per_cpu *cpu_buffer,
 		      struct ring_buffer_event *event)
 {
@@ -2702,7 +2702,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
 	return event;
 }
 
-static struct ring_buffer_event *
+static __always_inline struct ring_buffer_event *
 rb_reserve_next_event(struct ring_buffer *buffer,
 		      struct ring_buffer_per_cpu *cpu_buffer,
 		      unsigned long length)
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 54d5270a5042..66f829c47bec 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -40,6 +40,7 @@
 #include <linux/poll.h>
 #include <linux/nmi.h>
 #include <linux/fs.h>
+#include <linux/trace.h>
 #include <linux/sched/rt.h>
 
 #include "trace.h"
@@ -68,6 +69,7 @@ bool __read_mostly tracing_selftest_disabled;
 /* Pipe tracepoints to printk */
 struct trace_iterator *tracepoint_print_iter;
 int tracepoint_printk;
+static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
 
 /* For tracers that don't implement custom flags */
 static struct tracer_opt dummy_tracer_opt[] = {
@@ -738,6 +740,31 @@ static inline void ftrace_trace_stack(struct trace_array *tr,
 
 #endif
 
+static __always_inline void
+trace_event_setup(struct ring_buffer_event *event,
+		  int type, unsigned long flags, int pc)
+{
+	struct trace_entry *ent = ring_buffer_event_data(event);
+
+	tracing_generic_entry_update(ent, flags, pc);
+	ent->type = type;
+}
+
+static __always_inline struct ring_buffer_event *
+__trace_buffer_lock_reserve(struct ring_buffer *buffer,
+			  int type,
+			  unsigned long len,
+			  unsigned long flags, int pc)
+{
+	struct ring_buffer_event *event;
+
+	event = ring_buffer_lock_reserve(buffer, len);
+	if (event != NULL)
+		trace_event_setup(event, type, flags, pc);
+
+	return event;
+}
+
 static void tracer_tracing_on(struct trace_array *tr)
 {
 	if (tr->trace_buffer.buffer)
@@ -767,6 +794,22 @@ void tracing_on(void)
 }
 EXPORT_SYMBOL_GPL(tracing_on);
 
+
+static __always_inline void
+__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
+{
+	__this_cpu_write(trace_cmdline_save, true);
+
+	/* If this is the temp buffer, we need to commit fully */
+	if (this_cpu_read(trace_buffered_event) == event) {
+		/* Length is in event->array[0] */
+		ring_buffer_write(buffer, event->array[0], &event->array[1]);
+		/* Release the temp buffer */
+		this_cpu_dec(trace_buffered_event_cnt);
+	} else
+		ring_buffer_unlock_commit(buffer, event);
+}
+
 /**
  * __trace_puts - write a constant string into the trace buffer.
  * @ip:	   The address of the caller
@@ -794,8 +837,8 @@ int __trace_puts(unsigned long ip, const char *str, int size)
 
 	local_save_flags(irq_flags);
 	buffer = global_trace.trace_buffer.buffer;
-	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 
-					  irq_flags, pc);
+	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 
+					    irq_flags, pc);
 	if (!event)
 		return 0;
 
@@ -842,8 +885,8 @@ int __trace_bputs(unsigned long ip, const char *str)
 
 	local_save_flags(irq_flags);
 	buffer = global_trace.trace_buffer.buffer;
-	event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
-					  irq_flags, pc);
+	event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
+					    irq_flags, pc);
 	if (!event)
 		return 0;
 
@@ -1907,35 +1950,19 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
 #endif
 		((pc & NMI_MASK    ) ? TRACE_FLAG_NMI     : 0) |
 		((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
-		((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
+		((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
 		(tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
 		(test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
 }
 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
 
-static __always_inline void
-trace_event_setup(struct ring_buffer_event *event,
-		  int type, unsigned long flags, int pc)
-{
-	struct trace_entry *ent = ring_buffer_event_data(event);
-
-	tracing_generic_entry_update(ent, flags, pc);
-	ent->type = type;
-}
-
 struct ring_buffer_event *
 trace_buffer_lock_reserve(struct ring_buffer *buffer,
 			  int type,
 			  unsigned long len,
 			  unsigned long flags, int pc)
 {
-	struct ring_buffer_event *event;
-
-	event = ring_buffer_lock_reserve(buffer, len);
-	if (event != NULL)
-		trace_event_setup(event, type, flags, pc);
-
-	return event;
+	return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
 }
 
 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
@@ -2049,21 +2076,6 @@ void trace_buffered_event_disable(void)
 	preempt_enable();
 }
 
-void
-__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
-{
-	__this_cpu_write(trace_cmdline_save, true);
-
-	/* If this is the temp buffer, we need to commit fully */
-	if (this_cpu_read(trace_buffered_event) == event) {
-		/* Length is in event->array[0] */
-		ring_buffer_write(buffer, event->array[0], &event->array[1]);
-		/* Release the temp buffer */
-		this_cpu_dec(trace_buffered_event_cnt);
-	} else
-		ring_buffer_unlock_commit(buffer, event);
-}
-
 static struct ring_buffer *temp_buffer;
 
 struct ring_buffer_event *
@@ -2090,8 +2102,8 @@ trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
 		this_cpu_dec(trace_buffered_event_cnt);
 	}
 
-	entry = trace_buffer_lock_reserve(*current_rb,
-					 type, len, flags, pc);
+	entry = __trace_buffer_lock_reserve(*current_rb,
+					    type, len, flags, pc);
 	/*
 	 * If tracing is off, but we have triggers enabled
 	 * we still need to look at the event data. Use the temp_buffer
@@ -2100,13 +2112,88 @@ trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
 	 */
 	if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
 		*current_rb = temp_buffer;
-		entry = trace_buffer_lock_reserve(*current_rb,
-						  type, len, flags, pc);
+		entry = __trace_buffer_lock_reserve(*current_rb,
+						    type, len, flags, pc);
 	}
 	return entry;
 }
 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
 
+static DEFINE_SPINLOCK(tracepoint_iter_lock);
+static DEFINE_MUTEX(tracepoint_printk_mutex);
+
+static void output_printk(struct trace_event_buffer *fbuffer)
+{
+	struct trace_event_call *event_call;
+	struct trace_event *event;
+	unsigned long flags;
+	struct trace_iterator *iter = tracepoint_print_iter;
+
+	/* We should never get here if iter is NULL */
+	if (WARN_ON_ONCE(!iter))
+		return;
+
+	event_call = fbuffer->trace_file->event_call;
+	if (!event_call || !event_call->event.funcs ||
+	    !event_call->event.funcs->trace)
+		return;
+
+	event = &fbuffer->trace_file->event_call->event;
+
+	spin_lock_irqsave(&tracepoint_iter_lock, flags);
+	trace_seq_init(&iter->seq);
+	iter->ent = fbuffer->entry;
+	event_call->event.funcs->trace(iter, 0, event);
+	trace_seq_putc(&iter->seq, 0);
+	printk("%s", iter->seq.buffer);
+
+	spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
+}
+
+int tracepoint_printk_sysctl(struct ctl_table *table, int write,
+			     void __user *buffer, size_t *lenp,
+			     loff_t *ppos)
+{
+	int save_tracepoint_printk;
+	int ret;
+
+	mutex_lock(&tracepoint_printk_mutex);
+	save_tracepoint_printk = tracepoint_printk;
+
+	ret = proc_dointvec(table, write, buffer, lenp, ppos);
+
+	/*
+	 * This will force exiting early, as tracepoint_printk
+	 * is always zero when tracepoint_printk_iter is not allocated
+	 */
+	if (!tracepoint_print_iter)
+		tracepoint_printk = 0;
+
+	if (save_tracepoint_printk == tracepoint_printk)
+		goto out;
+
+	if (tracepoint_printk)
+		static_key_enable(&tracepoint_printk_key.key);
+	else
+		static_key_disable(&tracepoint_printk_key.key);
+
+ out:
+	mutex_unlock(&tracepoint_printk_mutex);
+
+	return ret;
+}
+
+void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
+{
+	if (static_key_false(&tracepoint_printk_key.key))
+		output_printk(fbuffer);
+
+	event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer,
+				    fbuffer->event, fbuffer->entry,
+				    fbuffer->flags, fbuffer->pc);
+}
+EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
+
 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
 				     struct ring_buffer *buffer,
 				     struct ring_buffer_event *event,
@@ -2129,6 +2216,139 @@ void trace_buffer_unlock_commit_regs(struct trace_array *tr,
 	ftrace_trace_userstack(buffer, flags, pc);
 }
 
+/*
+ * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
+ */
+void
+trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer,
+				   struct ring_buffer_event *event)
+{
+	__buffer_unlock_commit(buffer, event);
+}
+
+static void
+trace_process_export(struct trace_export *export,
+	       struct ring_buffer_event *event)
+{
+	struct trace_entry *entry;
+	unsigned int size = 0;
+
+	entry = ring_buffer_event_data(event);
+	size = ring_buffer_event_length(event);
+	export->write(entry, size);
+}
+
+static DEFINE_MUTEX(ftrace_export_lock);
+
+static struct trace_export __rcu *ftrace_exports_list __read_mostly;
+
+static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
+
+static inline void ftrace_exports_enable(void)
+{
+	static_branch_enable(&ftrace_exports_enabled);
+}
+
+static inline void ftrace_exports_disable(void)
+{
+	static_branch_disable(&ftrace_exports_enabled);
+}
+
+void ftrace_exports(struct ring_buffer_event *event)
+{
+	struct trace_export *export;
+
+	preempt_disable_notrace();
+
+	export = rcu_dereference_raw_notrace(ftrace_exports_list);
+	while (export) {
+		trace_process_export(export, event);
+		export = rcu_dereference_raw_notrace(export->next);
+	}
+
+	preempt_enable_notrace();
+}
+
+static inline void
+add_trace_export(struct trace_export **list, struct trace_export *export)
+{
+	rcu_assign_pointer(export->next, *list);
+	/*
+	 * We are entering export into the list but another
+	 * CPU might be walking that list. We need to make sure
+	 * the export->next pointer is valid before another CPU sees
+	 * the export pointer included into the list.
+	 */
+	rcu_assign_pointer(*list, export);
+}
+
+static inline int
+rm_trace_export(struct trace_export **list, struct trace_export *export)
+{
+	struct trace_export **p;
+
+	for (p = list; *p != NULL; p = &(*p)->next)
+		if (*p == export)
+			break;
+
+	if (*p != export)
+		return -1;
+
+	rcu_assign_pointer(*p, (*p)->next);
+
+	return 0;
+}
+
+static inline void
+add_ftrace_export(struct trace_export **list, struct trace_export *export)
+{
+	if (*list == NULL)
+		ftrace_exports_enable();
+
+	add_trace_export(list, export);
+}
+
+static inline int
+rm_ftrace_export(struct trace_export **list, struct trace_export *export)
+{
+	int ret;
+
+	ret = rm_trace_export(list, export);
+	if (*list == NULL)
+		ftrace_exports_disable();
+
+	return ret;
+}
+
+int register_ftrace_export(struct trace_export *export)
+{
+	if (WARN_ON_ONCE(!export->write))
+		return -1;
+
+	mutex_lock(&ftrace_export_lock);
+
+	add_ftrace_export(&ftrace_exports_list, export);
+
+	mutex_unlock(&ftrace_export_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(register_ftrace_export);
+
+int unregister_ftrace_export(struct trace_export *export)
+{
+	int ret;
+
+	mutex_lock(&ftrace_export_lock);
+
+	ret = rm_ftrace_export(&ftrace_exports_list, export);
+
+	mutex_unlock(&ftrace_export_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(unregister_ftrace_export);
+
 void
 trace_function(struct trace_array *tr,
 	       unsigned long ip, unsigned long parent_ip, unsigned long flags,
@@ -2139,16 +2359,19 @@ trace_function(struct trace_array *tr,
 	struct ring_buffer_event *event;
 	struct ftrace_entry *entry;
 
-	event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
-					  flags, pc);
+	event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
+					    flags, pc);
 	if (!event)
 		return;
 	entry	= ring_buffer_event_data(event);
 	entry->ip			= ip;
 	entry->parent_ip		= parent_ip;
 
-	if (!call_filter_check_discard(call, entry, buffer, event))
+	if (!call_filter_check_discard(call, entry, buffer, event)) {
+		if (static_branch_unlikely(&ftrace_exports_enabled))
+			ftrace_exports(event);
 		__buffer_unlock_commit(buffer, event);
+	}
 }
 
 #ifdef CONFIG_STACKTRACE
@@ -2216,8 +2439,8 @@ static void __ftrace_trace_stack(struct ring_buffer *buffer,
 
 	size *= sizeof(unsigned long);
 
-	event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
-					  sizeof(*entry) + size, flags, pc);
+	event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
+					    sizeof(*entry) + size, flags, pc);
 	if (!event)
 		goto out;
 	entry = ring_buffer_event_data(event);
@@ -2318,8 +2541,8 @@ ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
 
 	__this_cpu_inc(user_stack_count);
 
-	event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
-					  sizeof(*entry), flags, pc);
+	event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
+					    sizeof(*entry), flags, pc);
 	if (!event)
 		goto out_drop_count;
 	entry	= ring_buffer_event_data(event);
@@ -2489,8 +2712,8 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
 	local_save_flags(flags);
 	size = sizeof(*entry) + sizeof(u32) * len;
 	buffer = tr->trace_buffer.buffer;
-	event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
-					  flags, pc);
+	event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
+					    flags, pc);
 	if (!event)
 		goto out;
 	entry = ring_buffer_event_data(event);
@@ -2545,8 +2768,8 @@ __trace_array_vprintk(struct ring_buffer *buffer,
 
 	local_save_flags(flags);
 	size = sizeof(*entry) + len + 1;
-	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
-					  flags, pc);
+	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
+					    flags, pc);
 	if (!event)
 		goto out;
 	entry = ring_buffer_event_data(event);
@@ -4055,6 +4278,7 @@ static const char readme_msg[] =
 	"     x86-tsc:   TSC cycle counter\n"
 #endif
 	"\n  trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
+	"\n  trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
 	"  tracing_cpumask\t- Limit which CPUs to trace\n"
 	"  instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
 	"\t\t\t  Remove sub-buffer with rmdir\n"
@@ -4066,7 +4290,7 @@ static const char readme_msg[] =
 	"\n  available_filter_functions - list of functions that can be filtered on\n"
 	"  set_ftrace_filter\t- echo function name in here to only trace these\n"
 	"\t\t\t  functions\n"
-	"\t     accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
+	"\t     accepts: func_full_name or glob-matching-pattern\n"
 	"\t     modules: Can select a group via module\n"
 	"\t      Format: :mod:<module-name>\n"
 	"\t     example: echo :mod:ext3 > set_ftrace_filter\n"
@@ -5519,21 +5743,18 @@ static ssize_t
 tracing_mark_write(struct file *filp, const char __user *ubuf,
 					size_t cnt, loff_t *fpos)
 {
-	unsigned long addr = (unsigned long)ubuf;
 	struct trace_array *tr = filp->private_data;
 	struct ring_buffer_event *event;
 	struct ring_buffer *buffer;
 	struct print_entry *entry;
 	unsigned long irq_flags;
-	struct page *pages[2];
-	void *map_page[2];
-	int nr_pages = 1;
+	const char faulted[] = "<faulted>";
 	ssize_t written;
-	int offset;
 	int size;
 	int len;
-	int ret;
-	int i;
+
+/* Used in tracing_mark_raw_write() as well */
+#define FAULTED_SIZE (sizeof(faulted) - 1) /* '\0' is already accounted for */
 
 	if (tracing_disabled)
 		return -EINVAL;
@@ -5544,60 +5765,33 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,
 	if (cnt > TRACE_BUF_SIZE)
 		cnt = TRACE_BUF_SIZE;
 
-	/*
-	 * Userspace is injecting traces into the kernel trace buffer.
-	 * We want to be as non intrusive as possible.
-	 * To do so, we do not want to allocate any special buffers
-	 * or take any locks, but instead write the userspace data
-	 * straight into the ring buffer.
-	 *
-	 * First we need to pin the userspace buffer into memory,
-	 * which, most likely it is, because it just referenced it.
-	 * But there's no guarantee that it is. By using get_user_pages_fast()
-	 * and kmap_atomic/kunmap_atomic() we can get access to the
-	 * pages directly. We then write the data directly into the
-	 * ring buffer.
-	 */
 	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
 
-	/* check if we cross pages */
-	if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
-		nr_pages = 2;
-
-	offset = addr & (PAGE_SIZE - 1);
-	addr &= PAGE_MASK;
-
-	ret = get_user_pages_fast(addr, nr_pages, 0, pages);
-	if (ret < nr_pages) {
-		while (--ret >= 0)
-			put_page(pages[ret]);
-		written = -EFAULT;
-		goto out;
-	}
+	local_save_flags(irq_flags);
+	size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
 
-	for (i = 0; i < nr_pages; i++)
-		map_page[i] = kmap_atomic(pages[i]);
+	/* If less than "<faulted>", then make sure we can still add that */
+	if (cnt < FAULTED_SIZE)
+		size += FAULTED_SIZE - cnt;
 
-	local_save_flags(irq_flags);
-	size = sizeof(*entry) + cnt + 2; /* possible \n added */
 	buffer = tr->trace_buffer.buffer;
-	event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
-					  irq_flags, preempt_count());
-	if (!event) {
+	event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
+					    irq_flags, preempt_count());
+	if (unlikely(!event))
 		/* Ring buffer disabled, return as if not open for write */
-		written = -EBADF;
-		goto out_unlock;
-	}
+		return -EBADF;
 
 	entry = ring_buffer_event_data(event);
 	entry->ip = _THIS_IP_;
 
-	if (nr_pages == 2) {
-		len = PAGE_SIZE - offset;
-		memcpy(&entry->buf, map_page[0] + offset, len);
-		memcpy(&entry->buf[len], map_page[1], cnt - len);
+	len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
+	if (len) {
+		memcpy(&entry->buf, faulted, FAULTED_SIZE);
+		cnt = FAULTED_SIZE;
+		written = -EFAULT;
 	} else
-		memcpy(&entry->buf, map_page[0] + offset, cnt);
+		written = cnt;
+	len = cnt;
 
 	if (entry->buf[cnt - 1] != '\n') {
 		entry->buf[cnt] = '\n';
@@ -5607,16 +5801,73 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,
 
 	__buffer_unlock_commit(buffer, event);
 
-	written = cnt;
+	if (written > 0)
+		*fpos += written;
 
-	*fpos += written;
+	return written;
+}
+
+/* Limit it for now to 3K (including tag) */
+#define RAW_DATA_MAX_SIZE (1024*3)
+
+static ssize_t
+tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
+					size_t cnt, loff_t *fpos)
+{
+	struct trace_array *tr = filp->private_data;
+	struct ring_buffer_event *event;
+	struct ring_buffer *buffer;
+	struct raw_data_entry *entry;
+	const char faulted[] = "<faulted>";
+	unsigned long irq_flags;
+	ssize_t written;
+	int size;
+	int len;
+
+#define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
+
+	if (tracing_disabled)
+		return -EINVAL;
+
+	if (!(tr->trace_flags & TRACE_ITER_MARKERS))
+		return -EINVAL;
+
+	/* The marker must at least have a tag id */
+	if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
+		return -EINVAL;
+
+	if (cnt > TRACE_BUF_SIZE)
+		cnt = TRACE_BUF_SIZE;
+
+	BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
+
+	local_save_flags(irq_flags);
+	size = sizeof(*entry) + cnt;
+	if (cnt < FAULT_SIZE_ID)
+		size += FAULT_SIZE_ID - cnt;
+
+	buffer = tr->trace_buffer.buffer;
+	event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
+					    irq_flags, preempt_count());
+	if (!event)
+		/* Ring buffer disabled, return as if not open for write */
+		return -EBADF;
+
+	entry = ring_buffer_event_data(event);
+
+	len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
+	if (len) {
+		entry->id = -1;
+		memcpy(&entry->buf, faulted, FAULTED_SIZE);
+		written = -EFAULT;
+	} else
+		written = cnt;
+
+	__buffer_unlock_commit(buffer, event);
+
+	if (written > 0)
+		*fpos += written;
 
- out_unlock:
-	for (i = nr_pages - 1; i >= 0; i--) {
-		kunmap_atomic(map_page[i]);
-		put_page(pages[i]);
-	}
- out:
 	return written;
 }
 
@@ -5946,6 +6197,13 @@ static const struct file_operations tracing_mark_fops = {
 	.release	= tracing_release_generic_tr,
 };
 
+static const struct file_operations tracing_mark_raw_fops = {
+	.open		= tracing_open_generic_tr,
+	.write		= tracing_mark_raw_write,
+	.llseek		= generic_file_llseek,
+	.release	= tracing_release_generic_tr,
+};
+
 static const struct file_operations trace_clock_fops = {
 	.open		= tracing_clock_open,
 	.read		= seq_read,
@@ -7215,6 +7473,9 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
 	trace_create_file("trace_marker", 0220, d_tracer,
 			  tr, &tracing_mark_fops);
 
+	trace_create_file("trace_marker_raw", 0220, d_tracer,
+			  tr, &tracing_mark_raw_fops);
+
 	trace_create_file("trace_clock", 0644, d_tracer, tr,
 			  &trace_clock_fops);
 
@@ -7752,6 +8013,8 @@ void __init trace_init(void)
 			kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
 		if (WARN_ON(!tracepoint_print_iter))
 			tracepoint_printk = 0;
+		else
+			static_key_enable(&tracepoint_printk_key.key);
 	}
 	tracer_alloc_buffers();
 	trace_event_init();
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index fd24b1f9ac43..c2234494f40c 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -15,6 +15,7 @@
 #include <linux/trace_events.h>
 #include <linux/compiler.h>
 #include <linux/trace_seq.h>
+#include <linux/glob.h>
 
 #ifdef CONFIG_FTRACE_SYSCALLS
 #include <asm/unistd.h>		/* For NR_SYSCALLS	     */
@@ -39,6 +40,7 @@ enum trace_type {
 	TRACE_BLK,
 	TRACE_BPUTS,
 	TRACE_HWLAT,
+	TRACE_RAW_DATA,
 
 	__TRACE_LAST_TYPE,
 };
@@ -330,6 +332,7 @@ extern void __ftrace_bad_type(void);
 		IF_ASSIGN(var, ent, struct bprint_entry, TRACE_BPRINT);	\
 		IF_ASSIGN(var, ent, struct bputs_entry, TRACE_BPUTS);	\
 		IF_ASSIGN(var, ent, struct hwlat_entry, TRACE_HWLAT);	\
+		IF_ASSIGN(var, ent, struct raw_data_entry, TRACE_RAW_DATA);\
 		IF_ASSIGN(var, ent, struct trace_mmiotrace_rw,		\
 			  TRACE_MMIO_RW);				\
 		IF_ASSIGN(var, ent, struct trace_mmiotrace_map,		\
@@ -599,8 +602,8 @@ struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
 					  int *ent_cpu, u64 *ent_ts);
 
-void __buffer_unlock_commit(struct ring_buffer *buffer,
-			    struct ring_buffer_event *event);
+void trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer,
+					struct ring_buffer_event *event);
 
 int trace_empty(struct trace_iterator *iter);
 
@@ -843,6 +846,17 @@ static inline int ftrace_graph_notrace_addr(unsigned long addr)
 	return 0;
 }
 #endif /* CONFIG_DYNAMIC_FTRACE */
+
+extern unsigned int fgraph_max_depth;
+
+static inline bool ftrace_graph_ignore_func(struct ftrace_graph_ent *trace)
+{
+	/* trace it when it is-nested-in or is a function enabled. */
+	return !(trace->depth || ftrace_graph_addr(trace->func)) ||
+		(trace->depth < 0) ||
+		(fgraph_max_depth && trace->depth >= fgraph_max_depth);
+}
+
 #else /* CONFIG_FUNCTION_GRAPH_TRACER */
 static inline enum print_line_t
 print_graph_function_flags(struct trace_iterator *iter, u32 flags)
@@ -1257,6 +1271,7 @@ enum regex_type {
 	MATCH_FRONT_ONLY,
 	MATCH_MIDDLE_ONLY,
 	MATCH_END_ONLY,
+	MATCH_GLOB,
 };
 
 struct regex {
diff --git a/kernel/trace/trace_benchmark.c b/kernel/trace/trace_benchmark.c
index 0f109c4130d3..e3b488825ae3 100644
--- a/kernel/trace/trace_benchmark.c
+++ b/kernel/trace/trace_benchmark.c
@@ -21,6 +21,8 @@ static u64 bm_stddev;
 static unsigned int bm_avg;
 static unsigned int bm_std;
 
+static bool ok_to_run;
+
 /*
  * This gets called in a loop recording the time it took to write
  * the tracepoint. What it writes is the time statistics of the last
@@ -164,11 +166,21 @@ static int benchmark_event_kthread(void *arg)
  * When the benchmark tracepoint is enabled, it calls this
  * function and the thread that calls the tracepoint is created.
  */
-void trace_benchmark_reg(void)
+int trace_benchmark_reg(void)
 {
+	if (!ok_to_run) {
+		pr_warning("trace benchmark cannot be started via kernel command line\n");
+		return -EBUSY;
+	}
+
 	bm_event_thread = kthread_run(benchmark_event_kthread,
 				      NULL, "event_benchmark");
-	WARN_ON(!bm_event_thread);
+	if (!bm_event_thread) {
+		pr_warning("trace benchmark failed to create kernel thread\n");
+		return -ENOMEM;
+	}
+
+	return 0;
 }
 
 /*
@@ -182,6 +194,7 @@ void trace_benchmark_unreg(void)
 		return;
 
 	kthread_stop(bm_event_thread);
+	bm_event_thread = NULL;
 
 	strcpy(bm_str, "START");
 	bm_total = 0;
@@ -196,3 +209,12 @@ void trace_benchmark_unreg(void)
 	bm_avg = 0;
 	bm_stddev = 0;
 }
+
+static __init int ok_to_run_trace_benchmark(void)
+{
+	ok_to_run = true;
+
+	return 0;
+}
+
+early_initcall(ok_to_run_trace_benchmark);
diff --git a/kernel/trace/trace_benchmark.h b/kernel/trace/trace_benchmark.h
index 3c1df1df4e29..ebdbfc2f2a64 100644
--- a/kernel/trace/trace_benchmark.h
+++ b/kernel/trace/trace_benchmark.h
@@ -6,7 +6,7 @@
 
 #include <linux/tracepoint.h>
 
-extern void trace_benchmark_reg(void);
+extern int trace_benchmark_reg(void);
 extern void trace_benchmark_unreg(void);
 
 #define BENCHMARK_EVENT_STRLEN		128
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c
index 3a2a73716a5b..75489de546b6 100644
--- a/kernel/trace/trace_branch.c
+++ b/kernel/trace/trace_branch.c
@@ -81,7 +81,7 @@ probe_likely_condition(struct ftrace_branch_data *f, int val, int expect)
 	entry->correct = val == expect;
 
 	if (!call_filter_check_discard(call, entry, buffer, event))
-		__buffer_unlock_commit(buffer, event);
+		trace_buffer_unlock_commit_nostack(buffer, event);
 
  out:
 	current->trace_recursion &= ~TRACE_BRANCH_BIT;
diff --git a/kernel/trace/trace_entries.h b/kernel/trace/trace_entries.h
index d1cc37e78f99..eb7396b7e7c3 100644
--- a/kernel/trace/trace_entries.h
+++ b/kernel/trace/trace_entries.h
@@ -244,6 +244,21 @@ FTRACE_ENTRY(print, print_entry,
 	FILTER_OTHER
 );
 
+FTRACE_ENTRY(raw_data, raw_data_entry,
+
+	TRACE_RAW_DATA,
+
+	F_STRUCT(
+		__field(	unsigned int,	id	)
+		__dynamic_array(	char,	buf	)
+	),
+
+	F_printk("id:%04x %08x",
+		 __entry->id, (int)__entry->buf[0]),
+
+	FILTER_OTHER
+);
+
 FTRACE_ENTRY(bputs, bputs_entry,
 
 	TRACE_BPUTS,
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index 03c0a48c3ac4..93116549a284 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -283,46 +283,6 @@ void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
 }
 EXPORT_SYMBOL_GPL(trace_event_buffer_reserve);
 
-static DEFINE_SPINLOCK(tracepoint_iter_lock);
-
-static void output_printk(struct trace_event_buffer *fbuffer)
-{
-	struct trace_event_call *event_call;
-	struct trace_event *event;
-	unsigned long flags;
-	struct trace_iterator *iter = tracepoint_print_iter;
-
-	if (!iter)
-		return;
-
-	event_call = fbuffer->trace_file->event_call;
-	if (!event_call || !event_call->event.funcs ||
-	    !event_call->event.funcs->trace)
-		return;
-
-	event = &fbuffer->trace_file->event_call->event;
-
-	spin_lock_irqsave(&tracepoint_iter_lock, flags);
-	trace_seq_init(&iter->seq);
-	iter->ent = fbuffer->entry;
-	event_call->event.funcs->trace(iter, 0, event);
-	trace_seq_putc(&iter->seq, 0);
-	printk("%s", iter->seq.buffer);
-
-	spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
-}
-
-void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
-{
-	if (tracepoint_printk)
-		output_printk(fbuffer);
-
-	event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer,
-				    fbuffer->event, fbuffer->entry,
-				    fbuffer->flags, fbuffer->pc);
-}
-EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
-
 int trace_event_reg(struct trace_event_call *call,
 		    enum trace_reg type, void *data)
 {
@@ -742,6 +702,7 @@ __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
 	struct trace_event_call *call;
 	const char *name;
 	int ret = -EINVAL;
+	int eret = 0;
 
 	list_for_each_entry(file, &tr->events, list) {
 
@@ -765,9 +726,17 @@ __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
 		if (event && strcmp(event, name) != 0)
 			continue;
 
-		ftrace_event_enable_disable(file, set);
+		ret = ftrace_event_enable_disable(file, set);
 
-		ret = 0;
+		/*
+		 * Save the first error and return that. Some events
+		 * may still have been enabled, but let the user
+		 * know that something went wrong.
+		 */
+		if (ret && !eret)
+			eret = ret;
+
+		ret = eret;
 	}
 
 	return ret;
@@ -2843,20 +2812,32 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
 		return -ENOMEM;
 	}
 
+	entry = trace_create_file("enable", 0644, d_events,
+				  tr, &ftrace_tr_enable_fops);
+	if (!entry) {
+		pr_warn("Could not create tracefs 'enable' entry\n");
+		return -ENOMEM;
+	}
+
+	/* There are not as crucial, just warn if they are not created */
+
 	entry = tracefs_create_file("set_event_pid", 0644, parent,
 				    tr, &ftrace_set_event_pid_fops);
+	if (!entry)
+		pr_warn("Could not create tracefs 'set_event_pid' entry\n");
 
 	/* ring buffer internal formats */
-	trace_create_file("header_page", 0444, d_events,
-			  ring_buffer_print_page_header,
-			  &ftrace_show_header_fops);
-
-	trace_create_file("header_event", 0444, d_events,
-			  ring_buffer_print_entry_header,
-			  &ftrace_show_header_fops);
+	entry = trace_create_file("header_page", 0444, d_events,
+				  ring_buffer_print_page_header,
+				  &ftrace_show_header_fops);
+	if (!entry)
+		pr_warn("Could not create tracefs 'header_page' entry\n");
 
-	trace_create_file("enable", 0644, d_events,
-			  tr, &ftrace_tr_enable_fops);
+	entry = trace_create_file("header_event", 0444, d_events,
+				  ring_buffer_print_entry_header,
+				  &ftrace_show_header_fops);
+	if (!entry)
+		pr_warn("Could not create tracefs 'header_event' entry\n");
 
 	tr->event_dir = d_events;
 
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
index 9daa9b3bc6d9..59a411ff60c7 100644
--- a/kernel/trace/trace_events_filter.c
+++ b/kernel/trace/trace_events_filter.c
@@ -108,12 +108,12 @@ static char *err_text[] = {
 };
 
 struct opstack_op {
-	int op;
+	enum filter_op_ids op;
 	struct list_head list;
 };
 
 struct postfix_elt {
-	int op;
+	enum filter_op_ids op;
 	char *operand;
 	struct list_head list;
 };
@@ -145,34 +145,50 @@ struct pred_stack {
 
 /* If not of not match is equal to not of not, then it is a match */
 #define DEFINE_COMPARISON_PRED(type)					\
-static int filter_pred_##type(struct filter_pred *pred, void *event)	\
+static int filter_pred_LT_##type(struct filter_pred *pred, void *event)	\
 {									\
 	type *addr = (type *)(event + pred->offset);			\
 	type val = (type)pred->val;					\
-	int match = 0;							\
-									\
-	switch (pred->op) {						\
-	case OP_LT:							\
-		match = (*addr < val);					\
-		break;							\
-	case OP_LE:							\
-		match = (*addr <= val);					\
-		break;							\
-	case OP_GT:							\
-		match = (*addr > val);					\
-		break;							\
-	case OP_GE:							\
-		match = (*addr >= val);					\
-		break;							\
-	case OP_BAND:							\
-		match = (*addr & val);					\
-		break;							\
-	default:							\
-		break;							\
-	}								\
-									\
+	int match = (*addr < val);					\
 	return !!match == !pred->not;					\
-}
+}									\
+static int filter_pred_LE_##type(struct filter_pred *pred, void *event)	\
+{									\
+	type *addr = (type *)(event + pred->offset);			\
+	type val = (type)pred->val;					\
+	int match = (*addr <= val);					\
+	return !!match == !pred->not;					\
+}									\
+static int filter_pred_GT_##type(struct filter_pred *pred, void *event)	\
+{									\
+	type *addr = (type *)(event + pred->offset);			\
+	type val = (type)pred->val;					\
+	int match = (*addr > val);					\
+	return !!match == !pred->not;					\
+}									\
+static int filter_pred_GE_##type(struct filter_pred *pred, void *event)	\
+{									\
+	type *addr = (type *)(event + pred->offset);			\
+	type val = (type)pred->val;					\
+	int match = (*addr >= val);					\
+	return !!match == !pred->not;					\
+}									\
+static int filter_pred_BAND_##type(struct filter_pred *pred, void *event) \
+{									\
+	type *addr = (type *)(event + pred->offset);			\
+	type val = (type)pred->val;					\
+	int match = !!(*addr & val);					\
+	return match == !pred->not;					\
+}									\
+static const filter_pred_fn_t pred_funcs_##type[] = {			\
+	filter_pred_LT_##type,						\
+	filter_pred_LE_##type,						\
+	filter_pred_GT_##type,						\
+	filter_pred_GE_##type,						\
+	filter_pred_BAND_##type,					\
+};
+
+#define PRED_FUNC_START			OP_LT
 
 #define DEFINE_EQUALITY_PRED(size)					\
 static int filter_pred_##size(struct filter_pred *pred, void *event)	\
@@ -344,6 +360,12 @@ static int regex_match_end(char *str, struct regex *r, int len)
 	return 0;
 }
 
+static int regex_match_glob(char *str, struct regex *r, int len __maybe_unused)
+{
+	if (glob_match(r->pattern, str))
+		return 1;
+	return 0;
+}
 /**
  * filter_parse_regex - parse a basic regex
  * @buff:   the raw regex
@@ -380,14 +402,20 @@ enum regex_type filter_parse_regex(char *buff, int len, char **search, int *not)
 			if (!i) {
 				*search = buff + 1;
 				type = MATCH_END_ONLY;
-			} else {
+			} else if (i == len - 1) {
 				if (type == MATCH_END_ONLY)
 					type = MATCH_MIDDLE_ONLY;
 				else
 					type = MATCH_FRONT_ONLY;
 				buff[i] = 0;
 				break;
+			} else {	/* pattern continues, use full glob */
+				type = MATCH_GLOB;
+				break;
 			}
+		} else if (strchr("[?\\", buff[i])) {
+			type = MATCH_GLOB;
+			break;
 		}
 	}
 
@@ -420,6 +448,9 @@ static void filter_build_regex(struct filter_pred *pred)
 	case MATCH_END_ONLY:
 		r->match = regex_match_end;
 		break;
+	case MATCH_GLOB:
+		r->match = regex_match_glob;
+		break;
 	}
 
 	pred->not ^= not;
@@ -946,7 +977,7 @@ int filter_assign_type(const char *type)
 	return FILTER_OTHER;
 }
 
-static bool is_legal_op(struct ftrace_event_field *field, int op)
+static bool is_legal_op(struct ftrace_event_field *field, enum filter_op_ids op)
 {
 	if (is_string_field(field) &&
 	    (op != OP_EQ && op != OP_NE && op != OP_GLOB))
@@ -957,8 +988,8 @@ static bool is_legal_op(struct ftrace_event_field *field, int op)
 	return true;
 }
 
-static filter_pred_fn_t select_comparison_fn(int op, int field_size,
-					     int field_is_signed)
+static filter_pred_fn_t select_comparison_fn(enum filter_op_ids op,
+					    int field_size, int field_is_signed)
 {
 	filter_pred_fn_t fn = NULL;
 
@@ -967,33 +998,33 @@ static filter_pred_fn_t select_comparison_fn(int op, int field_size,
 		if (op == OP_EQ || op == OP_NE)
 			fn = filter_pred_64;
 		else if (field_is_signed)
-			fn = filter_pred_s64;
+			fn = pred_funcs_s64[op - PRED_FUNC_START];
 		else
-			fn = filter_pred_u64;
+			fn = pred_funcs_u64[op - PRED_FUNC_START];
 		break;
 	case 4:
 		if (op == OP_EQ || op == OP_NE)
 			fn = filter_pred_32;
 		else if (field_is_signed)
-			fn = filter_pred_s32;
+			fn = pred_funcs_s32[op - PRED_FUNC_START];
 		else
-			fn = filter_pred_u32;
+			fn = pred_funcs_u32[op - PRED_FUNC_START];
 		break;
 	case 2:
 		if (op == OP_EQ || op == OP_NE)
 			fn = filter_pred_16;
 		else if (field_is_signed)
-			fn = filter_pred_s16;
+			fn = pred_funcs_s16[op - PRED_FUNC_START];
 		else
-			fn = filter_pred_u16;
+			fn = pred_funcs_u16[op - PRED_FUNC_START];
 		break;
 	case 1:
 		if (op == OP_EQ || op == OP_NE)
 			fn = filter_pred_8;
 		else if (field_is_signed)
-			fn = filter_pred_s8;
+			fn = pred_funcs_s8[op - PRED_FUNC_START];
 		else
-			fn = filter_pred_u8;
+			fn = pred_funcs_u8[op - PRED_FUNC_START];
 		break;
 	}
 
@@ -1166,7 +1197,8 @@ static inline int append_operand_char(struct filter_parse_state *ps, char c)
 	return 0;
 }
 
-static int filter_opstack_push(struct filter_parse_state *ps, int op)
+static int filter_opstack_push(struct filter_parse_state *ps,
+			       enum filter_op_ids op)
 {
 	struct opstack_op *opstack_op;
 
@@ -1200,7 +1232,7 @@ static int filter_opstack_top(struct filter_parse_state *ps)
 static int filter_opstack_pop(struct filter_parse_state *ps)
 {
 	struct opstack_op *opstack_op;
-	int op;
+	enum filter_op_ids op;
 
 	if (filter_opstack_empty(ps))
 		return OP_NONE;
@@ -1245,7 +1277,7 @@ static int postfix_append_operand(struct filter_parse_state *ps, char *operand)
 	return 0;
 }
 
-static int postfix_append_op(struct filter_parse_state *ps, int op)
+static int postfix_append_op(struct filter_parse_state *ps, enum filter_op_ids op)
 {
 	struct postfix_elt *elt;
 
@@ -1275,8 +1307,8 @@ static void postfix_clear(struct filter_parse_state *ps)
 
 static int filter_parse(struct filter_parse_state *ps)
 {
+	enum filter_op_ids op, top_op;
 	int in_string = 0;
-	int op, top_op;
 	char ch;
 
 	while ((ch = infix_next(ps))) {
@@ -1367,7 +1399,8 @@ parse_operand:
 
 static struct filter_pred *create_pred(struct filter_parse_state *ps,
 				       struct trace_event_call *call,
-				       int op, char *operand1, char *operand2)
+				       enum filter_op_ids op,
+				       char *operand1, char *operand2)
 {
 	struct ftrace_event_field *field;
 	static struct filter_pred pred;
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 4e480e870474..d56123cdcc89 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -65,7 +65,7 @@ struct fgraph_data {
 
 #define TRACE_GRAPH_INDENT	2
 
-static unsigned int max_depth;
+unsigned int fgraph_max_depth;
 
 static struct tracer_opt trace_opts[] = {
 	/* Display overruns? (for self-debug purpose) */
@@ -358,7 +358,7 @@ int __trace_graph_entry(struct trace_array *tr,
 	entry	= ring_buffer_event_data(event);
 	entry->graph_ent			= *trace;
 	if (!call_filter_check_discard(call, entry, buffer, event))
-		__buffer_unlock_commit(buffer, event);
+		trace_buffer_unlock_commit_nostack(buffer, event);
 
 	return 1;
 }
@@ -384,10 +384,10 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
 	if (!ftrace_trace_task(tr))
 		return 0;
 
-	/* trace it when it is-nested-in or is a function enabled. */
-	if ((!(trace->depth || ftrace_graph_addr(trace->func)) ||
-	     ftrace_graph_ignore_irqs()) || (trace->depth < 0) ||
-	    (max_depth && trace->depth >= max_depth))
+	if (ftrace_graph_ignore_func(trace))
+		return 0;
+
+	if (ftrace_graph_ignore_irqs())
 		return 0;
 
 	/*
@@ -469,7 +469,7 @@ void __trace_graph_return(struct trace_array *tr,
 	entry	= ring_buffer_event_data(event);
 	entry->ret				= *trace;
 	if (!call_filter_check_discard(call, entry, buffer, event))
-		__buffer_unlock_commit(buffer, event);
+		trace_buffer_unlock_commit_nostack(buffer, event);
 }
 
 void trace_graph_return(struct ftrace_graph_ret *trace)
@@ -842,6 +842,10 @@ print_graph_entry_leaf(struct trace_iterator *iter,
 
 		cpu_data = per_cpu_ptr(data->cpu_data, cpu);
 
+		/* If a graph tracer ignored set_graph_notrace */
+		if (call->depth < -1)
+			call->depth += FTRACE_NOTRACE_DEPTH;
+
 		/*
 		 * Comments display at + 1 to depth. Since
 		 * this is a leaf function, keep the comments
@@ -850,7 +854,8 @@ print_graph_entry_leaf(struct trace_iterator *iter,
 		cpu_data->depth = call->depth - 1;
 
 		/* No need to keep this function around for this depth */
-		if (call->depth < FTRACE_RETFUNC_DEPTH)
+		if (call->depth < FTRACE_RETFUNC_DEPTH &&
+		    !WARN_ON_ONCE(call->depth < 0))
 			cpu_data->enter_funcs[call->depth] = 0;
 	}
 
@@ -880,11 +885,16 @@ print_graph_entry_nested(struct trace_iterator *iter,
 		struct fgraph_cpu_data *cpu_data;
 		int cpu = iter->cpu;
 
+		/* If a graph tracer ignored set_graph_notrace */
+		if (call->depth < -1)
+			call->depth += FTRACE_NOTRACE_DEPTH;
+
 		cpu_data = per_cpu_ptr(data->cpu_data, cpu);
 		cpu_data->depth = call->depth;
 
 		/* Save this function pointer to see if the exit matches */
-		if (call->depth < FTRACE_RETFUNC_DEPTH)
+		if (call->depth < FTRACE_RETFUNC_DEPTH &&
+		    !WARN_ON_ONCE(call->depth < 0))
 			cpu_data->enter_funcs[call->depth] = call->func;
 	}
 
@@ -1114,7 +1124,8 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
 		 */
 		cpu_data->depth = trace->depth - 1;
 
-		if (trace->depth < FTRACE_RETFUNC_DEPTH) {
+		if (trace->depth < FTRACE_RETFUNC_DEPTH &&
+		    !WARN_ON_ONCE(trace->depth < 0)) {
 			if (cpu_data->enter_funcs[trace->depth] != trace->func)
 				func_match = 0;
 			cpu_data->enter_funcs[trace->depth] = 0;
@@ -1489,7 +1500,7 @@ graph_depth_write(struct file *filp, const char __user *ubuf, size_t cnt,
 	if (ret)
 		return ret;
 
-	max_depth = val;
+	fgraph_max_depth = val;
 
 	*ppos += cnt;
 
@@ -1503,7 +1514,7 @@ graph_depth_read(struct file *filp, char __user *ubuf, size_t cnt,
 	char buf[15]; /* More than enough to hold UINT_MAX + "\n"*/
 	int n;
 
-	n = sprintf(buf, "%d\n", max_depth);
+	n = sprintf(buf, "%d\n", fgraph_max_depth);
 
 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, n);
 }
diff --git a/kernel/trace/trace_hwlat.c b/kernel/trace/trace_hwlat.c
index b97286c48735..775569ec50d0 100644
--- a/kernel/trace/trace_hwlat.c
+++ b/kernel/trace/trace_hwlat.c
@@ -127,7 +127,7 @@ static void trace_hwlat_sample(struct hwlat_sample *sample)
 	entry->nmi_count		= sample->nmi_count;
 
 	if (!call_filter_check_discard(call, entry, buffer, event))
-		__buffer_unlock_commit(buffer, event);
+		trace_buffer_unlock_commit_nostack(buffer, event);
 }
 
 /* Macros to encapsulate the time capturing infrastructure */
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index 03cdff84d026..86654d7e1afe 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -175,6 +175,18 @@ static int irqsoff_graph_entry(struct ftrace_graph_ent *trace)
 	int ret;
 	int pc;
 
+	if (ftrace_graph_ignore_func(trace))
+		return 0;
+	/*
+	 * Do not trace a function if it's filtered by set_graph_notrace.
+	 * Make the index of ret stack negative to indicate that it should
+	 * ignore further functions.  But it needs its own ret stack entry
+	 * to recover the original index in order to continue tracing after
+	 * returning from the function.
+	 */
+	if (ftrace_graph_notrace_addr(trace->func))
+		return 1;
+
 	if (!func_prolog_dec(tr, &data, &flags))
 		return 0;
 
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index eb6c9f1d3a93..a133ecd741e4 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -73,6 +73,17 @@ static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk)
 	return !!strchr(trace_kprobe_symbol(tk), ':');
 }
 
+static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
+{
+	unsigned long nhit = 0;
+	int cpu;
+
+	for_each_possible_cpu(cpu)
+		nhit += *per_cpu_ptr(tk->nhit, cpu);
+
+	return nhit;
+}
+
 static int register_kprobe_event(struct trace_kprobe *tk);
 static int unregister_kprobe_event(struct trace_kprobe *tk);
 
@@ -882,14 +893,10 @@ static const struct file_operations kprobe_events_ops = {
 static int probes_profile_seq_show(struct seq_file *m, void *v)
 {
 	struct trace_kprobe *tk = v;
-	unsigned long nhit = 0;
-	int cpu;
-
-	for_each_possible_cpu(cpu)
-		nhit += *per_cpu_ptr(tk->nhit, cpu);
 
 	seq_printf(m, "  %-44s %15lu %15lu\n",
-		   trace_event_name(&tk->tp.call), nhit,
+		   trace_event_name(&tk->tp.call),
+		   trace_kprobe_nhit(tk),
 		   tk->rp.kp.nmissed);
 
 	return 0;
@@ -1354,18 +1361,18 @@ fs_initcall(init_kprobe_trace);
 
 
 #ifdef CONFIG_FTRACE_STARTUP_TEST
-
 /*
  * The "__used" keeps gcc from removing the function symbol
- * from the kallsyms table.
+ * from the kallsyms table. 'noinline' makes sure that there
+ * isn't an inlined version used by the test method below
  */
-static __used int kprobe_trace_selftest_target(int a1, int a2, int a3,
-					       int a4, int a5, int a6)
+static __used __init noinline int
+kprobe_trace_selftest_target(int a1, int a2, int a3, int a4, int a5, int a6)
 {
 	return a1 + a2 + a3 + a4 + a5 + a6;
 }
 
-static struct trace_event_file *
+static struct __init trace_event_file *
 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
 {
 	struct trace_event_file *file;
@@ -1443,12 +1450,25 @@ static __init int kprobe_trace_self_tests_init(void)
 
 	ret = target(1, 2, 3, 4, 5, 6);
 
+	/*
+	 * Not expecting an error here, the check is only to prevent the
+	 * optimizer from removing the call to target() as otherwise there
+	 * are no side-effects and the call is never performed.
+	 */
+	if (ret != 21)
+		warn++;
+
 	/* Disable trace points before removing it */
 	tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
 	if (WARN_ON_ONCE(tk == NULL)) {
 		pr_warn("error on getting test probe.\n");
 		warn++;
 	} else {
+		if (trace_kprobe_nhit(tk) != 1) {
+			pr_warn("incorrect number of testprobe hits\n");
+			warn++;
+		}
+
 		file = find_trace_probe_file(tk, top_trace_array());
 		if (WARN_ON_ONCE(file == NULL)) {
 			pr_warn("error on getting probe file.\n");
@@ -1462,6 +1482,11 @@ static __init int kprobe_trace_self_tests_init(void)
 		pr_warn("error on getting 2nd test probe.\n");
 		warn++;
 	} else {
+		if (trace_kprobe_nhit(tk) != 1) {
+			pr_warn("incorrect number of testprobe2 hits\n");
+			warn++;
+		}
+
 		file = find_trace_probe_file(tk, top_trace_array());
 		if (WARN_ON_ONCE(file == NULL)) {
 			pr_warn("error on getting probe file.\n");
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
index 3fc20422c166..5d33a7352919 100644
--- a/kernel/trace/trace_output.c
+++ b/kernel/trace/trace_output.c
@@ -1288,6 +1288,35 @@ static struct trace_event trace_print_event = {
 	.funcs		= &trace_print_funcs,
 };
 
+static enum print_line_t trace_raw_data(struct trace_iterator *iter, int flags,
+					 struct trace_event *event)
+{
+	struct raw_data_entry *field;
+	int i;
+
+	trace_assign_type(field, iter->ent);
+
+	trace_seq_printf(&iter->seq, "# %x buf:", field->id);
+
+	for (i = 0; i < iter->ent_size - offsetof(struct raw_data_entry, buf); i++)
+		trace_seq_printf(&iter->seq, " %02x",
+				 (unsigned char)field->buf[i]);
+
+	trace_seq_putc(&iter->seq, '\n');
+
+	return trace_handle_return(&iter->seq);
+}
+
+static struct trace_event_functions trace_raw_data_funcs = {
+	.trace		= trace_raw_data,
+	.raw		= trace_raw_data,
+};
+
+static struct trace_event trace_raw_data_event = {
+	.type	 	= TRACE_RAW_DATA,
+	.funcs		= &trace_raw_data_funcs,
+};
+
 
 static struct trace_event *events[] __initdata = {
 	&trace_fn_event,
@@ -1299,6 +1328,7 @@ static struct trace_event *events[] __initdata = {
 	&trace_bprint_event,
 	&trace_print_event,
 	&trace_hwlat_event,
+	&trace_raw_data_event,
 	NULL
 };
 
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index 9d4399b553a3..5d0bb025bb21 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -239,6 +239,18 @@ static int wakeup_graph_entry(struct ftrace_graph_ent *trace)
 	unsigned long flags;
 	int pc, ret = 0;
 
+	if (ftrace_graph_ignore_func(trace))
+		return 0;
+	/*
+	 * Do not trace a function if it's filtered by set_graph_notrace.
+	 * Make the index of ret stack negative to indicate that it should
+	 * ignore further functions.  But it needs its own ret stack entry
+	 * to recover the original index in order to continue tracing after
+	 * returning from the function.
+	 */
+	if (ftrace_graph_notrace_addr(trace->func))
+		return 1;
+
 	if (!func_prolog_preempt_disable(tr, &data, &pc))
 		return 0;
 
@@ -790,6 +802,7 @@ static struct tracer wakeup_dl_tracer __read_mostly =
 #endif
 	.open		= wakeup_trace_open,
 	.close		= wakeup_trace_close,
+	.allow_instances = true,
 	.use_max_tr	= true,
 };
 
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index d0639d917899..1f9a31f934a4 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -194,9 +194,13 @@ static int tracepoint_add_func(struct tracepoint *tp,
 			       struct tracepoint_func *func, int prio)
 {
 	struct tracepoint_func *old, *tp_funcs;
+	int ret;
 
-	if (tp->regfunc && !static_key_enabled(&tp->key))
-		tp->regfunc();
+	if (tp->regfunc && !static_key_enabled(&tp->key)) {
+		ret = tp->regfunc();
+		if (ret < 0)
+			return ret;
+	}
 
 	tp_funcs = rcu_dereference_protected(tp->funcs,
 			lockdep_is_held(&tracepoints_mutex));
@@ -529,7 +533,7 @@ EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint);
 /* NB: reg/unreg are called while guarded with the tracepoints_mutex */
 static int sys_tracepoint_refcount;
 
-void syscall_regfunc(void)
+int syscall_regfunc(void)
 {
 	struct task_struct *p, *t;
 
@@ -541,6 +545,8 @@ void syscall_regfunc(void)
 		read_unlock(&tasklist_lock);
 	}
 	sys_tracepoint_refcount++;
+
+	return 0;
 }
 
 void syscall_unregfunc(void)
diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index 9acb29f280ec..d4b0fa01cae3 100644
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -24,32 +24,14 @@
 
 #include <asm/irq_regs.h>
 #include <linux/kvm_para.h>
-#include <linux/perf_event.h>
 #include <linux/kthread.h>
 
-/*
- * The run state of the lockup detectors is controlled by the content of the
- * 'watchdog_enabled' variable. Each lockup detector has its dedicated bit -
- * bit 0 for the hard lockup detector and bit 1 for the soft lockup detector.
- *
- * 'watchdog_user_enabled', 'nmi_watchdog_enabled' and 'soft_watchdog_enabled'
- * are variables that are only used as an 'interface' between the parameters
- * in /proc/sys/kernel and the internal state bits in 'watchdog_enabled'. The
- * 'watchdog_thresh' variable is handled differently because its value is not
- * boolean, and the lockup detectors are 'suspended' while 'watchdog_thresh'
- * is equal zero.
- */
-#define NMI_WATCHDOG_ENABLED_BIT   0
-#define SOFT_WATCHDOG_ENABLED_BIT  1
-#define NMI_WATCHDOG_ENABLED      (1 << NMI_WATCHDOG_ENABLED_BIT)
-#define SOFT_WATCHDOG_ENABLED     (1 << SOFT_WATCHDOG_ENABLED_BIT)
-
 static DEFINE_MUTEX(watchdog_proc_mutex);
 
-#ifdef CONFIG_HARDLOCKUP_DETECTOR
-static unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED|NMI_WATCHDOG_ENABLED;
+#if defined(CONFIG_HAVE_NMI_WATCHDOG) || defined(CONFIG_HARDLOCKUP_DETECTOR)
+unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED|NMI_WATCHDOG_ENABLED;
 #else
-static unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED;
+unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED;
 #endif
 int __read_mostly nmi_watchdog_enabled;
 int __read_mostly soft_watchdog_enabled;
@@ -59,9 +41,6 @@ int __read_mostly watchdog_thresh = 10;
 #ifdef CONFIG_SMP
 int __read_mostly sysctl_softlockup_all_cpu_backtrace;
 int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
-#else
-#define sysctl_softlockup_all_cpu_backtrace 0
-#define sysctl_hardlockup_all_cpu_backtrace 0
 #endif
 static struct cpumask watchdog_cpumask __read_mostly;
 unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
@@ -100,50 +79,9 @@ static DEFINE_PER_CPU(bool, soft_watchdog_warn);
 static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts);
 static DEFINE_PER_CPU(unsigned long, soft_lockup_hrtimer_cnt);
 static DEFINE_PER_CPU(struct task_struct *, softlockup_task_ptr_saved);
-#ifdef CONFIG_HARDLOCKUP_DETECTOR
-static DEFINE_PER_CPU(bool, hard_watchdog_warn);
-static DEFINE_PER_CPU(bool, watchdog_nmi_touch);
 static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
-static DEFINE_PER_CPU(struct perf_event *, watchdog_ev);
-#endif
 static unsigned long soft_lockup_nmi_warn;
 
-/* boot commands */
-/*
- * Should we panic when a soft-lockup or hard-lockup occurs:
- */
-#ifdef CONFIG_HARDLOCKUP_DETECTOR
-unsigned int __read_mostly hardlockup_panic =
-			CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE;
-static unsigned long hardlockup_allcpu_dumped;
-/*
- * We may not want to enable hard lockup detection by default in all cases,
- * for example when running the kernel as a guest on a hypervisor. In these
- * cases this function can be called to disable hard lockup detection. This
- * function should only be executed once by the boot processor before the
- * kernel command line parameters are parsed, because otherwise it is not
- * possible to override this in hardlockup_panic_setup().
- */
-void hardlockup_detector_disable(void)
-{
-	watchdog_enabled &= ~NMI_WATCHDOG_ENABLED;
-}
-
-static int __init hardlockup_panic_setup(char *str)
-{
-	if (!strncmp(str, "panic", 5))
-		hardlockup_panic = 1;
-	else if (!strncmp(str, "nopanic", 7))
-		hardlockup_panic = 0;
-	else if (!strncmp(str, "0", 1))
-		watchdog_enabled &= ~NMI_WATCHDOG_ENABLED;
-	else if (!strncmp(str, "1", 1))
-		watchdog_enabled |= NMI_WATCHDOG_ENABLED;
-	return 1;
-}
-__setup("nmi_watchdog=", hardlockup_panic_setup);
-#endif
-
 unsigned int __read_mostly softlockup_panic =
 			CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE;
 
@@ -264,32 +202,14 @@ void touch_all_softlockup_watchdogs(void)
 	wq_watchdog_touch(-1);
 }
 
-#ifdef CONFIG_HARDLOCKUP_DETECTOR
-void touch_nmi_watchdog(void)
-{
-	/*
-	 * Using __raw here because some code paths have
-	 * preemption enabled.  If preemption is enabled
-	 * then interrupts should be enabled too, in which
-	 * case we shouldn't have to worry about the watchdog
-	 * going off.
-	 */
-	raw_cpu_write(watchdog_nmi_touch, true);
-	touch_softlockup_watchdog();
-}
-EXPORT_SYMBOL(touch_nmi_watchdog);
-
-#endif
-
 void touch_softlockup_watchdog_sync(void)
 {
 	__this_cpu_write(softlockup_touch_sync, true);
 	__this_cpu_write(watchdog_touch_ts, 0);
 }
 
-#ifdef CONFIG_HARDLOCKUP_DETECTOR
 /* watchdog detector functions */
-static bool is_hardlockup(void)
+bool is_hardlockup(void)
 {
 	unsigned long hrint = __this_cpu_read(hrtimer_interrupts);
 
@@ -299,7 +219,6 @@ static bool is_hardlockup(void)
 	__this_cpu_write(hrtimer_interrupts_saved, hrint);
 	return false;
 }
-#endif
 
 static int is_softlockup(unsigned long touch_ts)
 {
@@ -313,78 +232,22 @@ static int is_softlockup(unsigned long touch_ts)
 	return 0;
 }
 
-#ifdef CONFIG_HARDLOCKUP_DETECTOR
-
-static struct perf_event_attr wd_hw_attr = {
-	.type		= PERF_TYPE_HARDWARE,
-	.config		= PERF_COUNT_HW_CPU_CYCLES,
-	.size		= sizeof(struct perf_event_attr),
-	.pinned		= 1,
-	.disabled	= 1,
-};
-
-/* Callback function for perf event subsystem */
-static void watchdog_overflow_callback(struct perf_event *event,
-		 struct perf_sample_data *data,
-		 struct pt_regs *regs)
-{
-	/* Ensure the watchdog never gets throttled */
-	event->hw.interrupts = 0;
-
-	if (__this_cpu_read(watchdog_nmi_touch) == true) {
-		__this_cpu_write(watchdog_nmi_touch, false);
-		return;
-	}
-
-	/* check for a hardlockup
-	 * This is done by making sure our timer interrupt
-	 * is incrementing.  The timer interrupt should have
-	 * fired multiple times before we overflow'd.  If it hasn't
-	 * then this is a good indication the cpu is stuck
-	 */
-	if (is_hardlockup()) {
-		int this_cpu = smp_processor_id();
-		struct pt_regs *regs = get_irq_regs();
-
-		/* only print hardlockups once */
-		if (__this_cpu_read(hard_watchdog_warn) == true)
-			return;
-
-		pr_emerg("Watchdog detected hard LOCKUP on cpu %d", this_cpu);
-		print_modules();
-		print_irqtrace_events(current);
-		if (regs)
-			show_regs(regs);
-		else
-			dump_stack();
-
-		/*
-		 * Perform all-CPU dump only once to avoid multiple hardlockups
-		 * generating interleaving traces
-		 */
-		if (sysctl_hardlockup_all_cpu_backtrace &&
-				!test_and_set_bit(0, &hardlockup_allcpu_dumped))
-			trigger_allbutself_cpu_backtrace();
-
-		if (hardlockup_panic)
-			nmi_panic(regs, "Hard LOCKUP");
-
-		__this_cpu_write(hard_watchdog_warn, true);
-		return;
-	}
-
-	__this_cpu_write(hard_watchdog_warn, false);
-	return;
-}
-#endif /* CONFIG_HARDLOCKUP_DETECTOR */
-
 static void watchdog_interrupt_count(void)
 {
 	__this_cpu_inc(hrtimer_interrupts);
 }
 
-static int watchdog_nmi_enable(unsigned int cpu);
-static void watchdog_nmi_disable(unsigned int cpu);
+/*
+ * These two functions are mostly architecture specific
+ * defining them as weak here.
+ */
+int __weak watchdog_nmi_enable(unsigned int cpu)
+{
+	return 0;
+}
+void __weak watchdog_nmi_disable(unsigned int cpu)
+{
+}
 
 static int watchdog_enable_all_cpus(void);
 static void watchdog_disable_all_cpus(void);
@@ -577,109 +440,6 @@ static void watchdog(unsigned int cpu)
 		watchdog_nmi_disable(cpu);
 }
 
-#ifdef CONFIG_HARDLOCKUP_DETECTOR
-/*
- * People like the simple clean cpu node info on boot.
- * Reduce the watchdog noise by only printing messages
- * that are different from what cpu0 displayed.
- */
-static unsigned long cpu0_err;
-
-static int watchdog_nmi_enable(unsigned int cpu)
-{
-	struct perf_event_attr *wd_attr;
-	struct perf_event *event = per_cpu(watchdog_ev, cpu);
-
-	/* nothing to do if the hard lockup detector is disabled */
-	if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED))
-		goto out;
-
-	/* is it already setup and enabled? */
-	if (event && event->state > PERF_EVENT_STATE_OFF)
-		goto out;
-
-	/* it is setup but not enabled */
-	if (event != NULL)
-		goto out_enable;
-
-	wd_attr = &wd_hw_attr;
-	wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh);
-
-	/* Try to register using hardware perf events */
-	event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL);
-
-	/* save cpu0 error for future comparision */
-	if (cpu == 0 && IS_ERR(event))
-		cpu0_err = PTR_ERR(event);
-
-	if (!IS_ERR(event)) {
-		/* only print for cpu0 or different than cpu0 */
-		if (cpu == 0 || cpu0_err)
-			pr_info("enabled on all CPUs, permanently consumes one hw-PMU counter.\n");
-		goto out_save;
-	}
-
-	/*
-	 * Disable the hard lockup detector if _any_ CPU fails to set up
-	 * set up the hardware perf event. The watchdog() function checks
-	 * the NMI_WATCHDOG_ENABLED bit periodically.
-	 *
-	 * The barriers are for syncing up watchdog_enabled across all the
-	 * cpus, as clear_bit() does not use barriers.
-	 */
-	smp_mb__before_atomic();
-	clear_bit(NMI_WATCHDOG_ENABLED_BIT, &watchdog_enabled);
-	smp_mb__after_atomic();
-
-	/* skip displaying the same error again */
-	if (cpu > 0 && (PTR_ERR(event) == cpu0_err))
-		return PTR_ERR(event);
-
-	/* vary the KERN level based on the returned errno */
-	if (PTR_ERR(event) == -EOPNOTSUPP)
-		pr_info("disabled (cpu%i): not supported (no LAPIC?)\n", cpu);
-	else if (PTR_ERR(event) == -ENOENT)
-		pr_warn("disabled (cpu%i): hardware events not enabled\n",
-			 cpu);
-	else
-		pr_err("disabled (cpu%i): unable to create perf event: %ld\n",
-			cpu, PTR_ERR(event));
-
-	pr_info("Shutting down hard lockup detector on all cpus\n");
-
-	return PTR_ERR(event);
-
-	/* success path */
-out_save:
-	per_cpu(watchdog_ev, cpu) = event;
-out_enable:
-	perf_event_enable(per_cpu(watchdog_ev, cpu));
-out:
-	return 0;
-}
-
-static void watchdog_nmi_disable(unsigned int cpu)
-{
-	struct perf_event *event = per_cpu(watchdog_ev, cpu);
-
-	if (event) {
-		perf_event_disable(event);
-		per_cpu(watchdog_ev, cpu) = NULL;
-
-		/* should be in cleanup, but blocks oprofile */
-		perf_event_release_kernel(event);
-	}
-	if (cpu == 0) {
-		/* watchdog_nmi_enable() expects this to be zero initially. */
-		cpu0_err = 0;
-	}
-}
-
-#else
-static int watchdog_nmi_enable(unsigned int cpu) { return 0; }
-static void watchdog_nmi_disable(unsigned int cpu) { return; }
-#endif /* CONFIG_HARDLOCKUP_DETECTOR */
-
 static struct smp_hotplug_thread watchdog_threads = {
 	.store			= &softlockup_watchdog,
 	.thread_should_run	= watchdog_should_run,
diff --git a/kernel/watchdog_hld.c b/kernel/watchdog_hld.c
new file mode 100644
index 000000000000..84016c8aee6b
--- /dev/null
+++ b/kernel/watchdog_hld.c
@@ -0,0 +1,227 @@
+/*
+ * Detect hard lockups on a system
+ *
+ * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
+ *
+ * Note: Most of this code is borrowed heavily from the original softlockup
+ * detector, so thanks to Ingo for the initial implementation.
+ * Some chunks also taken from the old x86-specific nmi watchdog code, thanks
+ * to those contributors as well.
+ */
+
+#define pr_fmt(fmt) "NMI watchdog: " fmt
+
+#include <linux/nmi.h>
+#include <linux/module.h>
+#include <asm/irq_regs.h>
+#include <linux/perf_event.h>
+
+static DEFINE_PER_CPU(bool, hard_watchdog_warn);
+static DEFINE_PER_CPU(bool, watchdog_nmi_touch);
+static DEFINE_PER_CPU(struct perf_event *, watchdog_ev);
+
+/* boot commands */
+/*
+ * Should we panic when a soft-lockup or hard-lockup occurs:
+ */
+unsigned int __read_mostly hardlockup_panic =
+			CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE;
+static unsigned long hardlockup_allcpu_dumped;
+/*
+ * We may not want to enable hard lockup detection by default in all cases,
+ * for example when running the kernel as a guest on a hypervisor. In these
+ * cases this function can be called to disable hard lockup detection. This
+ * function should only be executed once by the boot processor before the
+ * kernel command line parameters are parsed, because otherwise it is not
+ * possible to override this in hardlockup_panic_setup().
+ */
+void hardlockup_detector_disable(void)
+{
+	watchdog_enabled &= ~NMI_WATCHDOG_ENABLED;
+}
+
+static int __init hardlockup_panic_setup(char *str)
+{
+	if (!strncmp(str, "panic", 5))
+		hardlockup_panic = 1;
+	else if (!strncmp(str, "nopanic", 7))
+		hardlockup_panic = 0;
+	else if (!strncmp(str, "0", 1))
+		watchdog_enabled &= ~NMI_WATCHDOG_ENABLED;
+	else if (!strncmp(str, "1", 1))
+		watchdog_enabled |= NMI_WATCHDOG_ENABLED;
+	return 1;
+}
+__setup("nmi_watchdog=", hardlockup_panic_setup);
+
+void touch_nmi_watchdog(void)
+{
+	/*
+	 * Using __raw here because some code paths have
+	 * preemption enabled.  If preemption is enabled
+	 * then interrupts should be enabled too, in which
+	 * case we shouldn't have to worry about the watchdog
+	 * going off.
+	 */
+	raw_cpu_write(watchdog_nmi_touch, true);
+	touch_softlockup_watchdog();
+}
+EXPORT_SYMBOL(touch_nmi_watchdog);
+
+static struct perf_event_attr wd_hw_attr = {
+	.type		= PERF_TYPE_HARDWARE,
+	.config		= PERF_COUNT_HW_CPU_CYCLES,
+	.size		= sizeof(struct perf_event_attr),
+	.pinned		= 1,
+	.disabled	= 1,
+};
+
+/* Callback function for perf event subsystem */
+static void watchdog_overflow_callback(struct perf_event *event,
+		 struct perf_sample_data *data,
+		 struct pt_regs *regs)
+{
+	/* Ensure the watchdog never gets throttled */
+	event->hw.interrupts = 0;
+
+	if (__this_cpu_read(watchdog_nmi_touch) == true) {
+		__this_cpu_write(watchdog_nmi_touch, false);
+		return;
+	}
+
+	/* check for a hardlockup
+	 * This is done by making sure our timer interrupt
+	 * is incrementing.  The timer interrupt should have
+	 * fired multiple times before we overflow'd.  If it hasn't
+	 * then this is a good indication the cpu is stuck
+	 */
+	if (is_hardlockup()) {
+		int this_cpu = smp_processor_id();
+
+		/* only print hardlockups once */
+		if (__this_cpu_read(hard_watchdog_warn) == true)
+			return;
+
+		pr_emerg("Watchdog detected hard LOCKUP on cpu %d", this_cpu);
+		print_modules();
+		print_irqtrace_events(current);
+		if (regs)
+			show_regs(regs);
+		else
+			dump_stack();
+
+		/*
+		 * Perform all-CPU dump only once to avoid multiple hardlockups
+		 * generating interleaving traces
+		 */
+		if (sysctl_hardlockup_all_cpu_backtrace &&
+				!test_and_set_bit(0, &hardlockup_allcpu_dumped))
+			trigger_allbutself_cpu_backtrace();
+
+		if (hardlockup_panic)
+			nmi_panic(regs, "Hard LOCKUP");
+
+		__this_cpu_write(hard_watchdog_warn, true);
+		return;
+	}
+
+	__this_cpu_write(hard_watchdog_warn, false);
+	return;
+}
+
+/*
+ * People like the simple clean cpu node info on boot.
+ * Reduce the watchdog noise by only printing messages
+ * that are different from what cpu0 displayed.
+ */
+static unsigned long cpu0_err;
+
+int watchdog_nmi_enable(unsigned int cpu)
+{
+	struct perf_event_attr *wd_attr;
+	struct perf_event *event = per_cpu(watchdog_ev, cpu);
+
+	/* nothing to do if the hard lockup detector is disabled */
+	if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED))
+		goto out;
+
+	/* is it already setup and enabled? */
+	if (event && event->state > PERF_EVENT_STATE_OFF)
+		goto out;
+
+	/* it is setup but not enabled */
+	if (event != NULL)
+		goto out_enable;
+
+	wd_attr = &wd_hw_attr;
+	wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh);
+
+	/* Try to register using hardware perf events */
+	event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL);
+
+	/* save cpu0 error for future comparision */
+	if (cpu == 0 && IS_ERR(event))
+		cpu0_err = PTR_ERR(event);
+
+	if (!IS_ERR(event)) {
+		/* only print for cpu0 or different than cpu0 */
+		if (cpu == 0 || cpu0_err)
+			pr_info("enabled on all CPUs, permanently consumes one hw-PMU counter.\n");
+		goto out_save;
+	}
+
+	/*
+	 * Disable the hard lockup detector if _any_ CPU fails to set up
+	 * set up the hardware perf event. The watchdog() function checks
+	 * the NMI_WATCHDOG_ENABLED bit periodically.
+	 *
+	 * The barriers are for syncing up watchdog_enabled across all the
+	 * cpus, as clear_bit() does not use barriers.
+	 */
+	smp_mb__before_atomic();
+	clear_bit(NMI_WATCHDOG_ENABLED_BIT, &watchdog_enabled);
+	smp_mb__after_atomic();
+
+	/* skip displaying the same error again */
+	if (cpu > 0 && (PTR_ERR(event) == cpu0_err))
+		return PTR_ERR(event);
+
+	/* vary the KERN level based on the returned errno */
+	if (PTR_ERR(event) == -EOPNOTSUPP)
+		pr_info("disabled (cpu%i): not supported (no LAPIC?)\n", cpu);
+	else if (PTR_ERR(event) == -ENOENT)
+		pr_warn("disabled (cpu%i): hardware events not enabled\n",
+			 cpu);
+	else
+		pr_err("disabled (cpu%i): unable to create perf event: %ld\n",
+			cpu, PTR_ERR(event));
+
+	pr_info("Shutting down hard lockup detector on all cpus\n");
+
+	return PTR_ERR(event);
+
+	/* success path */
+out_save:
+	per_cpu(watchdog_ev, cpu) = event;
+out_enable:
+	perf_event_enable(per_cpu(watchdog_ev, cpu));
+out:
+	return 0;
+}
+
+void watchdog_nmi_disable(unsigned int cpu)
+{
+	struct perf_event *event = per_cpu(watchdog_ev, cpu);
+
+	if (event) {
+		perf_event_disable(event);
+		per_cpu(watchdog_ev, cpu) = NULL;
+
+		/* should be in cleanup, but blocks oprofile */
+		perf_event_release_kernel(event);
+	}
+	if (cpu == 0) {
+		/* watchdog_nmi_enable() expects this to be zero initially. */
+		cpu0_err = 0;
+	}
+}
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 479d840db286..1d9fb6543a66 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -290,6 +290,8 @@ module_param_named(disable_numa, wq_disable_numa, bool, 0444);
 static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
 module_param_named(power_efficient, wq_power_efficient, bool, 0444);
 
+static bool wq_online;			/* can kworkers be created yet? */
+
 static bool wq_numa_enabled;		/* unbound NUMA affinity enabled */
 
 /* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
@@ -2583,6 +2585,9 @@ void flush_workqueue(struct workqueue_struct *wq)
 	};
 	int next_color;
 
+	if (WARN_ON(!wq_online))
+		return;
+
 	lock_map_acquire(&wq->lockdep_map);
 	lock_map_release(&wq->lockdep_map);
 
@@ -2843,6 +2848,9 @@ bool flush_work(struct work_struct *work)
 {
 	struct wq_barrier barr;
 
+	if (WARN_ON(!wq_online))
+		return false;
+
 	lock_map_acquire(&work->lockdep_map);
 	lock_map_release(&work->lockdep_map);
 
@@ -2913,7 +2921,13 @@ static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
 	mark_work_canceling(work);
 	local_irq_restore(flags);
 
-	flush_work(work);
+	/*
+	 * This allows canceling during early boot.  We know that @work
+	 * isn't executing.
+	 */
+	if (wq_online)
+		flush_work(work);
+
 	clear_work_data(work);
 
 	/*
@@ -3364,7 +3378,7 @@ static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
 		goto fail;
 
 	/* create and start the initial worker */
-	if (!create_worker(pool))
+	if (wq_online && !create_worker(pool))
 		goto fail;
 
 	/* install */
@@ -3429,6 +3443,7 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq)
 {
 	struct workqueue_struct *wq = pwq->wq;
 	bool freezable = wq->flags & WQ_FREEZABLE;
+	unsigned long flags;
 
 	/* for @wq->saved_max_active */
 	lockdep_assert_held(&wq->mutex);
@@ -3437,7 +3452,8 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq)
 	if (!freezable && pwq->max_active == wq->saved_max_active)
 		return;
 
-	spin_lock_irq(&pwq->pool->lock);
+	/* this function can be called during early boot w/ irq disabled */
+	spin_lock_irqsave(&pwq->pool->lock, flags);
 
 	/*
 	 * During [un]freezing, the caller is responsible for ensuring that
@@ -3460,7 +3476,7 @@ static void pwq_adjust_max_active(struct pool_workqueue *pwq)
 		pwq->max_active = 0;
 	}
 
-	spin_unlock_irq(&pwq->pool->lock);
+	spin_unlock_irqrestore(&pwq->pool->lock, flags);
 }
 
 /* initialize newly alloced @pwq which is associated with @wq and @pool */
@@ -4033,6 +4049,7 @@ void destroy_workqueue(struct workqueue_struct *wq)
 		for (i = 0; i < WORK_NR_COLORS; i++) {
 			if (WARN_ON(pwq->nr_in_flight[i])) {
 				mutex_unlock(&wq->mutex);
+				show_workqueue_state();
 				return;
 			}
 		}
@@ -4041,6 +4058,7 @@ void destroy_workqueue(struct workqueue_struct *wq)
 		    WARN_ON(pwq->nr_active) ||
 		    WARN_ON(!list_empty(&pwq->delayed_works))) {
 			mutex_unlock(&wq->mutex);
+			show_workqueue_state();
 			return;
 		}
 	}
@@ -5467,7 +5485,17 @@ static void __init wq_numa_init(void)
 	wq_numa_enabled = true;
 }
 
-static int __init init_workqueues(void)
+/**
+ * workqueue_init_early - early init for workqueue subsystem
+ *
+ * This is the first half of two-staged workqueue subsystem initialization
+ * and invoked as soon as the bare basics - memory allocation, cpumasks and
+ * idr are up.  It sets up all the data structures and system workqueues
+ * and allows early boot code to create workqueues and queue/cancel work
+ * items.  Actual work item execution starts only after kthreads can be
+ * created and scheduled right before early initcalls.
+ */
+int __init workqueue_init_early(void)
 {
 	int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
 	int i, cpu;
@@ -5479,8 +5507,6 @@ static int __init init_workqueues(void)
 
 	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
 
-	wq_numa_init();
-
 	/* initialize CPU pools */
 	for_each_possible_cpu(cpu) {
 		struct worker_pool *pool;
@@ -5500,16 +5526,6 @@ static int __init init_workqueues(void)
 		}
 	}
 
-	/* create the initial worker */
-	for_each_online_cpu(cpu) {
-		struct worker_pool *pool;
-
-		for_each_cpu_worker_pool(pool, cpu) {
-			pool->flags &= ~POOL_DISASSOCIATED;
-			BUG_ON(!create_worker(pool));
-		}
-	}
-
 	/* create default unbound and ordered wq attrs */
 	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
 		struct workqueue_attrs *attrs;
@@ -5546,8 +5562,59 @@ static int __init init_workqueues(void)
 	       !system_power_efficient_wq ||
 	       !system_freezable_power_efficient_wq);
 
+	return 0;
+}
+
+/**
+ * workqueue_init - bring workqueue subsystem fully online
+ *
+ * This is the latter half of two-staged workqueue subsystem initialization
+ * and invoked as soon as kthreads can be created and scheduled.
+ * Workqueues have been created and work items queued on them, but there
+ * are no kworkers executing the work items yet.  Populate the worker pools
+ * with the initial workers and enable future kworker creations.
+ */
+int __init workqueue_init(void)
+{
+	struct workqueue_struct *wq;
+	struct worker_pool *pool;
+	int cpu, bkt;
+
+	/*
+	 * It'd be simpler to initialize NUMA in workqueue_init_early() but
+	 * CPU to node mapping may not be available that early on some
+	 * archs such as power and arm64.  As per-cpu pools created
+	 * previously could be missing node hint and unbound pools NUMA
+	 * affinity, fix them up.
+	 */
+	wq_numa_init();
+
+	mutex_lock(&wq_pool_mutex);
+
+	for_each_possible_cpu(cpu) {
+		for_each_cpu_worker_pool(pool, cpu) {
+			pool->node = cpu_to_node(cpu);
+		}
+	}
+
+	list_for_each_entry(wq, &workqueues, list)
+		wq_update_unbound_numa(wq, smp_processor_id(), true);
+
+	mutex_unlock(&wq_pool_mutex);
+
+	/* create the initial workers */
+	for_each_online_cpu(cpu) {
+		for_each_cpu_worker_pool(pool, cpu) {
+			pool->flags &= ~POOL_DISASSOCIATED;
+			BUG_ON(!create_worker(pool));
+		}
+	}
+
+	hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
+		BUG_ON(!create_worker(pool));
+
+	wq_online = true;
 	wq_watchdog_init();
 
 	return 0;
 }
-early_initcall(init_workqueues);
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index e6327d102184..7446097f72bd 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -194,8 +194,8 @@ config GDB_SCRIPTS
 	  build directory. If you load vmlinux into gdb, the helper
 	  scripts will be automatically imported by gdb as well, and
 	  additional functions are available to analyze a Linux kernel
-	  instance. See Documentation/gdb-kernel-debugging.txt for further
-	  details.
+	  instance. See Documentation/dev-tools/gdb-kernel-debugging.rst
+	  for further details.
 
 config ENABLE_WARN_DEPRECATED
 	bool "Enable __deprecated logic"
@@ -542,7 +542,7 @@ config DEBUG_KMEMLEAK
 	  difference being that the orphan objects are not freed but
 	  only shown in /sys/kernel/debug/kmemleak. Enabling this
 	  feature will introduce an overhead to memory
-	  allocations. See Documentation/kmemleak.txt for more
+	  allocations. See Documentation/dev-tools/kmemleak.rst for more
 	  details.
 
 	  Enabling DEBUG_SLAB or SLUB_DEBUG may increase the chances
@@ -739,7 +739,7 @@ config KCOV
 	  different machines and across reboots. If you need stable PC values,
 	  disable RANDOMIZE_BASE.
 
-	  For more details, see Documentation/kcov.txt.
+	  For more details, see Documentation/dev-tools/kcov.rst.
 
 config KCOV_INSTRUMENT_ALL
 	bool "Instrument all code by default"
diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan
index bc6e651df68c..a669c193b878 100644
--- a/lib/Kconfig.ubsan
+++ b/lib/Kconfig.ubsan
@@ -10,7 +10,8 @@ config UBSAN
 	  This option enables undefined behaviour sanity checker
 	  Compile-time instrumentation is used to detect various undefined
 	  behaviours in runtime. Various types of checks may be enabled
-	  via boot parameter ubsan_handle (see: Documentation/ubsan.txt).
+	  via boot parameter ubsan_handle
+	  (see: Documentation/dev-tools/ubsan.rst).
 
 config UBSAN_SANITIZE_ALL
 	bool "Enable instrumentation for the entire kernel"
diff --git a/lib/debugobjects.c b/lib/debugobjects.c
index 056052dc8e91..04c1ef717fe0 100644
--- a/lib/debugobjects.c
+++ b/lib/debugobjects.c
@@ -199,7 +199,7 @@ static void free_object(struct debug_obj *obj)
 	 * initialized:
 	 */
 	if (obj_pool_free > ODEBUG_POOL_SIZE && obj_cache)
-		sched = keventd_up();
+		sched = 1;
 	hlist_add_head(&obj->node, &obj_pool);
 	obj_pool_free++;
 	obj_pool_used--;
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index f2bd21b93dfc..691a52b634fe 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -1,4 +1,5 @@
 #include <linux/export.h>
+#include <linux/bvec.h>
 #include <linux/uio.h>
 #include <linux/pagemap.h>
 #include <linux/slab.h>
diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c
index f6c2c1e7779c..9a2b811966eb 100644
--- a/lib/kobject_uevent.c
+++ b/lib/kobject_uevent.c
@@ -56,7 +56,7 @@ static const char *kobject_actions[] = {
  * kobject_action_type - translate action string to numeric type
  *
  * @buf: buffer containing the action string, newline is ignored
- * @len: length of buffer
+ * @count: length of buffer
  * @type: pointer to the location to store the action type
  *
  * Returns 0 if the action string was recognized.
@@ -154,8 +154,8 @@ static void cleanup_uevent_env(struct subprocess_info *info)
 /**
  * kobject_uevent_env - send an uevent with environmental data
  *
- * @action: action that is happening
  * @kobj: struct kobject that the action is happening to
+ * @action: action that is happening
  * @envp_ext: pointer to environmental data
  *
  * Returns 0 if kobject_uevent_env() is completed with success or the
@@ -363,8 +363,8 @@ EXPORT_SYMBOL_GPL(kobject_uevent_env);
 /**
  * kobject_uevent - notify userspace by sending an uevent
  *
- * @action: action that is happening
  * @kobj: struct kobject that the action is happening to
+ * @action: action that is happening
  *
  * Returns 0 if kobject_uevent() is completed with success or the
  * corresponding error when it fails.
diff --git a/lib/parser.c b/lib/parser.c
index b6d11631231b..3278958b472a 100644
--- a/lib/parser.c
+++ b/lib/parser.c
@@ -152,6 +152,36 @@ static int match_number(substring_t *s, int *result, int base)
 }
 
 /**
+ * match_u64int: scan a number in the given base from a substring_t
+ * @s: substring to be scanned
+ * @result: resulting u64 on success
+ * @base: base to use when converting string
+ *
+ * Description: Given a &substring_t and a base, attempts to parse the substring
+ * as a number in that base. On success, sets @result to the integer represented
+ * by the string and returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure.
+ */
+static int match_u64int(substring_t *s, u64 *result, int base)
+{
+	char *buf;
+	int ret;
+	u64 val;
+	size_t len = s->to - s->from;
+
+	buf = kmalloc(len + 1, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+	memcpy(buf, s->from, len);
+	buf[len] = '\0';
+
+	ret = kstrtoull(buf, base, &val);
+	if (!ret)
+		*result = val;
+	kfree(buf);
+	return ret;
+}
+
+/**
  * match_int: - scan a decimal representation of an integer from a substring_t
  * @s: substring_t to be scanned
  * @result: resulting integer on success
@@ -167,6 +197,23 @@ int match_int(substring_t *s, int *result)
 EXPORT_SYMBOL(match_int);
 
 /**
+ * match_u64: - scan a decimal representation of a u64 from
+ *                  a substring_t
+ * @s: substring_t to be scanned
+ * @result: resulting unsigned long long on success
+ *
+ * Description: Attempts to parse the &substring_t @s as a long decimal
+ * integer. On success, sets @result to the integer represented by the
+ * string and returns 0.
+ * Returns -ENOMEM, -EINVAL, or -ERANGE on failure.
+ */
+int match_u64(substring_t *s, u64 *result)
+{
+	return match_u64int(s, result, 0);
+}
+EXPORT_SYMBOL(match_u64);
+
+/**
  * match_octal: - scan an octal representation of an integer from a substring_t
  * @s: substring_t to be scanned
  * @result: resulting integer on success
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index 2e8c6f7aa56e..6f382e07de77 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -22,6 +22,7 @@
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
+#include <linux/cpu.h>
 #include <linux/errno.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
@@ -30,7 +31,6 @@
 #include <linux/percpu.h>
 #include <linux/slab.h>
 #include <linux/kmemleak.h>
-#include <linux/notifier.h>
 #include <linux/cpu.h>
 #include <linux/string.h>
 #include <linux/bitops.h>
@@ -69,6 +69,11 @@ struct radix_tree_preload {
 };
 static DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { 0, };
 
+static inline struct radix_tree_node *entry_to_node(void *ptr)
+{
+	return (void *)((unsigned long)ptr & ~RADIX_TREE_INTERNAL_NODE);
+}
+
 static inline void *node_to_entry(void *ptr)
 {
 	return (void *)((unsigned long)ptr | RADIX_TREE_INTERNAL_NODE);
@@ -191,13 +196,12 @@ static inline int any_tag_set(struct radix_tree_node *node, unsigned int tag)
  * Returns next bit offset, or size if nothing found.
  */
 static __always_inline unsigned long
-radix_tree_find_next_bit(const unsigned long *addr,
-			 unsigned long size, unsigned long offset)
+radix_tree_find_next_bit(struct radix_tree_node *node, unsigned int tag,
+			 unsigned long offset)
 {
-	if (!__builtin_constant_p(size))
-		return find_next_bit(addr, size, offset);
+	const unsigned long *addr = node->tags[tag];
 
-	if (offset < size) {
+	if (offset < RADIX_TREE_MAP_SIZE) {
 		unsigned long tmp;
 
 		addr += offset / BITS_PER_LONG;
@@ -205,14 +209,32 @@ radix_tree_find_next_bit(const unsigned long *addr,
 		if (tmp)
 			return __ffs(tmp) + offset;
 		offset = (offset + BITS_PER_LONG) & ~(BITS_PER_LONG - 1);
-		while (offset < size) {
+		while (offset < RADIX_TREE_MAP_SIZE) {
 			tmp = *++addr;
 			if (tmp)
 				return __ffs(tmp) + offset;
 			offset += BITS_PER_LONG;
 		}
 	}
-	return size;
+	return RADIX_TREE_MAP_SIZE;
+}
+
+static unsigned int iter_offset(const struct radix_tree_iter *iter)
+{
+	return (iter->index >> iter_shift(iter)) & RADIX_TREE_MAP_MASK;
+}
+
+/*
+ * The maximum index which can be stored in a radix tree
+ */
+static inline unsigned long shift_maxindex(unsigned int shift)
+{
+	return (RADIX_TREE_MAP_SIZE << shift) - 1;
+}
+
+static inline unsigned long node_maxindex(struct radix_tree_node *node)
+{
+	return shift_maxindex(node->shift);
 }
 
 #ifndef __KERNEL__
@@ -220,10 +242,11 @@ static void dump_node(struct radix_tree_node *node, unsigned long index)
 {
 	unsigned long i;
 
-	pr_debug("radix node: %p offset %d tags %lx %lx %lx shift %d count %d exceptional %d parent %p\n",
-		node, node->offset,
+	pr_debug("radix node: %p offset %d indices %lu-%lu parent %p tags %lx %lx %lx shift %d count %d exceptional %d\n",
+		node, node->offset, index, index | node_maxindex(node),
+		node->parent,
 		node->tags[0][0], node->tags[1][0], node->tags[2][0],
-		node->shift, node->count, node->exceptional, node->parent);
+		node->shift, node->count, node->exceptional);
 
 	for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) {
 		unsigned long first = index | (i << node->shift);
@@ -231,14 +254,16 @@ static void dump_node(struct radix_tree_node *node, unsigned long index)
 		void *entry = node->slots[i];
 		if (!entry)
 			continue;
-		if (is_sibling_entry(node, entry)) {
-			pr_debug("radix sblng %p offset %ld val %p indices %ld-%ld\n",
-					entry, i,
-					*(void **)entry_to_node(entry),
-					first, last);
+		if (entry == RADIX_TREE_RETRY) {
+			pr_debug("radix retry offset %ld indices %lu-%lu parent %p\n",
+					i, first, last, node);
 		} else if (!radix_tree_is_internal_node(entry)) {
-			pr_debug("radix entry %p offset %ld indices %ld-%ld\n",
-					entry, i, first, last);
+			pr_debug("radix entry %p offset %ld indices %lu-%lu parent %p\n",
+					entry, i, first, last, node);
+		} else if (is_sibling_entry(node, entry)) {
+			pr_debug("radix sblng %p offset %ld indices %lu-%lu parent %p val %p\n",
+					entry, i, first, last, node,
+					*(void **)entry_to_node(entry));
 		} else {
 			dump_node(entry_to_node(entry), first);
 		}
@@ -262,7 +287,10 @@ static void radix_tree_dump(struct radix_tree_root *root)
  * that the caller has pinned this thread of control to the current CPU.
  */
 static struct radix_tree_node *
-radix_tree_node_alloc(struct radix_tree_root *root)
+radix_tree_node_alloc(struct radix_tree_root *root,
+			struct radix_tree_node *parent,
+			unsigned int shift, unsigned int offset,
+			unsigned int count, unsigned int exceptional)
 {
 	struct radix_tree_node *ret = NULL;
 	gfp_t gfp_mask = root_gfp_mask(root);
@@ -307,6 +335,13 @@ radix_tree_node_alloc(struct radix_tree_root *root)
 	ret = kmem_cache_alloc(radix_tree_node_cachep, gfp_mask);
 out:
 	BUG_ON(radix_tree_is_internal_node(ret));
+	if (ret) {
+		ret->parent = parent;
+		ret->shift = shift;
+		ret->offset = offset;
+		ret->count = count;
+		ret->exceptional = exceptional;
+	}
 	return ret;
 }
 
@@ -314,17 +349,15 @@ static void radix_tree_node_rcu_free(struct rcu_head *head)
 {
 	struct radix_tree_node *node =
 			container_of(head, struct radix_tree_node, rcu_head);
-	int i;
 
 	/*
-	 * must only free zeroed nodes into the slab. radix_tree_shrink
-	 * can leave us with a non-NULL entry in the first slot, so clear
-	 * that here to make sure.
+	 * Must only free zeroed nodes into the slab.  We can be left with
+	 * non-NULL entries by radix_tree_free_nodes, so clear the entries
+	 * and tags here.
 	 */
-	for (i = 0; i < RADIX_TREE_MAX_TAGS; i++)
-		tag_clear(node, i, 0);
-
-	node->slots[0] = NULL;
+	memset(node->slots, 0, sizeof(node->slots));
+	memset(node->tags, 0, sizeof(node->tags));
+	INIT_LIST_HEAD(&node->private_list);
 
 	kmem_cache_free(radix_tree_node_cachep, node);
 }
@@ -344,7 +377,7 @@ radix_tree_node_free(struct radix_tree_node *node)
  * To make use of this facility, the radix tree must be initialised without
  * __GFP_DIRECT_RECLAIM being passed to INIT_RADIX_TREE().
  */
-static int __radix_tree_preload(gfp_t gfp_mask, int nr)
+static int __radix_tree_preload(gfp_t gfp_mask, unsigned nr)
 {
 	struct radix_tree_preload *rtp;
 	struct radix_tree_node *node;
@@ -410,6 +443,28 @@ int radix_tree_maybe_preload(gfp_t gfp_mask)
 }
 EXPORT_SYMBOL(radix_tree_maybe_preload);
 
+#ifdef CONFIG_RADIX_TREE_MULTIORDER
+/*
+ * Preload with enough objects to ensure that we can split a single entry
+ * of order @old_order into many entries of size @new_order
+ */
+int radix_tree_split_preload(unsigned int old_order, unsigned int new_order,
+							gfp_t gfp_mask)
+{
+	unsigned top = 1 << (old_order % RADIX_TREE_MAP_SHIFT);
+	unsigned layers = (old_order / RADIX_TREE_MAP_SHIFT) -
+				(new_order / RADIX_TREE_MAP_SHIFT);
+	unsigned nr = 0;
+
+	WARN_ON_ONCE(!gfpflags_allow_blocking(gfp_mask));
+	BUG_ON(new_order >= old_order);
+
+	while (layers--)
+		nr = nr * RADIX_TREE_MAP_SIZE + 1;
+	return __radix_tree_preload(gfp_mask, top * nr);
+}
+#endif
+
 /*
  * The same as function above, but preload number of nodes required to insert
  * (1 << order) continuous naturally-aligned elements.
@@ -455,19 +510,6 @@ int radix_tree_maybe_preload_order(gfp_t gfp_mask, int order)
 	return __radix_tree_preload(gfp_mask, nr_nodes);
 }
 
-/*
- * The maximum index which can be stored in a radix tree
- */
-static inline unsigned long shift_maxindex(unsigned int shift)
-{
-	return (RADIX_TREE_MAP_SIZE << shift) - 1;
-}
-
-static inline unsigned long node_maxindex(struct radix_tree_node *node)
-{
-	return shift_maxindex(node->shift);
-}
-
 static unsigned radix_tree_load_root(struct radix_tree_root *root,
 		struct radix_tree_node **nodep, unsigned long *maxindex)
 {
@@ -505,8 +547,8 @@ static int radix_tree_extend(struct radix_tree_root *root,
 		goto out;
 
 	do {
-		struct radix_tree_node *node = radix_tree_node_alloc(root);
-
+		struct radix_tree_node *node = radix_tree_node_alloc(root,
+							NULL, shift, 0, 1, 0);
 		if (!node)
 			return -ENOMEM;
 
@@ -517,16 +559,11 @@ static int radix_tree_extend(struct radix_tree_root *root,
 		}
 
 		BUG_ON(shift > BITS_PER_LONG);
-		node->shift = shift;
-		node->offset = 0;
-		node->count = 1;
-		node->parent = NULL;
 		if (radix_tree_is_internal_node(slot)) {
 			entry_to_node(slot)->parent = node;
-		} else {
+		} else if (radix_tree_exceptional_entry(slot)) {
 			/* Moving an exceptional root->rnode to a node */
-			if (radix_tree_exceptional_entry(slot))
-				node->exceptional = 1;
+			node->exceptional = 1;
 		}
 		node->slots[0] = slot;
 		slot = node_to_entry(node);
@@ -665,26 +702,24 @@ int __radix_tree_create(struct radix_tree_root *root, unsigned long index,
 	shift = radix_tree_load_root(root, &child, &maxindex);
 
 	/* Make sure the tree is high enough.  */
+	if (order > 0 && max == ((1UL << order) - 1))
+		max++;
 	if (max > maxindex) {
 		int error = radix_tree_extend(root, max, shift);
 		if (error < 0)
 			return error;
 		shift = error;
 		child = root->rnode;
-		if (order == shift)
-			shift += RADIX_TREE_MAP_SHIFT;
 	}
 
 	while (shift > order) {
 		shift -= RADIX_TREE_MAP_SHIFT;
 		if (child == NULL) {
 			/* Have to add a child node.  */
-			child = radix_tree_node_alloc(root);
+			child = radix_tree_node_alloc(root, node, shift,
+							offset, 0, 0);
 			if (!child)
 				return -ENOMEM;
-			child->shift = shift;
-			child->offset = offset;
-			child->parent = node;
 			rcu_assign_pointer(*slot, node_to_entry(child));
 			if (node)
 				node->count++;
@@ -697,31 +732,125 @@ int __radix_tree_create(struct radix_tree_root *root, unsigned long index,
 		slot = &node->slots[offset];
 	}
 
+	if (nodep)
+		*nodep = node;
+	if (slotp)
+		*slotp = slot;
+	return 0;
+}
+
 #ifdef CONFIG_RADIX_TREE_MULTIORDER
-	/* Insert pointers to the canonical entry */
-	if (order > shift) {
-		unsigned i, n = 1 << (order - shift);
+/*
+ * Free any nodes below this node.  The tree is presumed to not need
+ * shrinking, and any user data in the tree is presumed to not need a
+ * destructor called on it.  If we need to add a destructor, we can
+ * add that functionality later.  Note that we may not clear tags or
+ * slots from the tree as an RCU walker may still have a pointer into
+ * this subtree.  We could replace the entries with RADIX_TREE_RETRY,
+ * but we'll still have to clear those in rcu_free.
+ */
+static void radix_tree_free_nodes(struct radix_tree_node *node)
+{
+	unsigned offset = 0;
+	struct radix_tree_node *child = entry_to_node(node);
+
+	for (;;) {
+		void *entry = child->slots[offset];
+		if (radix_tree_is_internal_node(entry) &&
+					!is_sibling_entry(child, entry)) {
+			child = entry_to_node(entry);
+			offset = 0;
+			continue;
+		}
+		offset++;
+		while (offset == RADIX_TREE_MAP_SIZE) {
+			struct radix_tree_node *old = child;
+			offset = child->offset + 1;
+			child = child->parent;
+			radix_tree_node_free(old);
+			if (old == entry_to_node(node))
+				return;
+		}
+	}
+}
+
+static inline int insert_entries(struct radix_tree_node *node, void **slot,
+				void *item, unsigned order, bool replace)
+{
+	struct radix_tree_node *child;
+	unsigned i, n, tag, offset, tags = 0;
+
+	if (node) {
+		if (order > node->shift)
+			n = 1 << (order - node->shift);
+		else
+			n = 1;
+		offset = get_slot_offset(node, slot);
+	} else {
+		n = 1;
+		offset = 0;
+	}
+
+	if (n > 1) {
 		offset = offset & ~(n - 1);
 		slot = &node->slots[offset];
-		child = node_to_entry(slot);
-		for (i = 0; i < n; i++) {
-			if (slot[i])
+	}
+	child = node_to_entry(slot);
+
+	for (i = 0; i < n; i++) {
+		if (slot[i]) {
+			if (replace) {
+				node->count--;
+				for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
+					if (tag_get(node, tag, offset + i))
+						tags |= 1 << tag;
+			} else
 				return -EEXIST;
 		}
+	}
 
-		for (i = 1; i < n; i++) {
+	for (i = 0; i < n; i++) {
+		struct radix_tree_node *old = slot[i];
+		if (i) {
 			rcu_assign_pointer(slot[i], child);
-			node->count++;
+			for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
+				if (tags & (1 << tag))
+					tag_clear(node, tag, offset + i);
+		} else {
+			rcu_assign_pointer(slot[i], item);
+			for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
+				if (tags & (1 << tag))
+					tag_set(node, tag, offset);
 		}
+		if (radix_tree_is_internal_node(old) &&
+					!is_sibling_entry(node, old) &&
+					(old != RADIX_TREE_RETRY))
+			radix_tree_free_nodes(old);
+		if (radix_tree_exceptional_entry(old))
+			node->exceptional--;
 	}
-#endif
-
-	if (nodep)
-		*nodep = node;
-	if (slotp)
-		*slotp = slot;
-	return 0;
+	if (node) {
+		node->count += n;
+		if (radix_tree_exceptional_entry(item))
+			node->exceptional += n;
+	}
+	return n;
+}
+#else
+static inline int insert_entries(struct radix_tree_node *node, void **slot,
+				void *item, unsigned order, bool replace)
+{
+	if (*slot)
+		return -EEXIST;
+	rcu_assign_pointer(*slot, item);
+	if (node) {
+		node->count++;
+		if (radix_tree_exceptional_entry(item))
+			node->exceptional++;
+	}
+	return 1;
 }
+#endif
 
 /**
  *	__radix_tree_insert    -    insert into a radix tree
@@ -744,15 +873,13 @@ int __radix_tree_insert(struct radix_tree_root *root, unsigned long index,
 	error = __radix_tree_create(root, index, order, &node, &slot);
 	if (error)
 		return error;
-	if (*slot != NULL)
-		return -EEXIST;
-	rcu_assign_pointer(*slot, item);
+
+	error = insert_entries(node, slot, item, order, false);
+	if (error < 0)
+		return error;
 
 	if (node) {
 		unsigned offset = get_slot_offset(node, slot);
-		node->count++;
-		if (radix_tree_exceptional_entry(item))
-			node->exceptional++;
 		BUG_ON(tag_get(node, 0, offset));
 		BUG_ON(tag_get(node, 1, offset));
 		BUG_ON(tag_get(node, 2, offset));
@@ -850,6 +977,24 @@ void *radix_tree_lookup(struct radix_tree_root *root, unsigned long index)
 }
 EXPORT_SYMBOL(radix_tree_lookup);
 
+static inline int slot_count(struct radix_tree_node *node,
+						void **slot)
+{
+	int n = 1;
+#ifdef CONFIG_RADIX_TREE_MULTIORDER
+	void *ptr = node_to_entry(slot);
+	unsigned offset = get_slot_offset(node, slot);
+	int i;
+
+	for (i = 1; offset + i < RADIX_TREE_MAP_SIZE; i++) {
+		if (node->slots[offset + i] != ptr)
+			break;
+		n++;
+	}
+#endif
+	return n;
+}
+
 static void replace_slot(struct radix_tree_root *root,
 			 struct radix_tree_node *node,
 			 void **slot, void *item,
@@ -868,12 +1013,35 @@ static void replace_slot(struct radix_tree_root *root,
 
 	if (node) {
 		node->count += count;
-		node->exceptional += exceptional;
+		if (exceptional) {
+			exceptional *= slot_count(node, slot);
+			node->exceptional += exceptional;
+		}
 	}
 
 	rcu_assign_pointer(*slot, item);
 }
 
+static inline void delete_sibling_entries(struct radix_tree_node *node,
+						void **slot)
+{
+#ifdef CONFIG_RADIX_TREE_MULTIORDER
+	bool exceptional = radix_tree_exceptional_entry(*slot);
+	void *ptr = node_to_entry(slot);
+	unsigned offset = get_slot_offset(node, slot);
+	int i;
+
+	for (i = 1; offset + i < RADIX_TREE_MAP_SIZE; i++) {
+		if (node->slots[offset + i] != ptr)
+			break;
+		node->slots[offset + i] = NULL;
+		node->count--;
+		if (exceptional)
+			node->exceptional--;
+	}
+#endif
+}
+
 /**
  * __radix_tree_replace		- replace item in a slot
  * @root:		radix tree root
@@ -891,6 +1059,8 @@ void __radix_tree_replace(struct radix_tree_root *root,
 			  void **slot, void *item,
 			  radix_tree_update_node_t update_node, void *private)
 {
+	if (!item)
+		delete_sibling_entries(node, slot);
 	/*
 	 * This function supports replacing exceptional entries and
 	 * deleting entries, but that needs accounting against the
@@ -921,7 +1091,8 @@ void __radix_tree_replace(struct radix_tree_root *root,
  * NOTE: This cannot be used to switch between non-entries (empty slots),
  * regular entries, and exceptional entries, as that requires accounting
  * inside the radix tree node. When switching from one type of entry or
- * deleting, use __radix_tree_lookup() and __radix_tree_replace().
+ * deleting, use __radix_tree_lookup() and __radix_tree_replace() or
+ * radix_tree_iter_replace().
  */
 void radix_tree_replace_slot(struct radix_tree_root *root,
 			     void **slot, void *item)
@@ -930,6 +1101,164 @@ void radix_tree_replace_slot(struct radix_tree_root *root,
 }
 
 /**
+ * radix_tree_iter_replace - replace item in a slot
+ * @root:	radix tree root
+ * @slot:	pointer to slot
+ * @item:	new item to store in the slot.
+ *
+ * For use with radix_tree_split() and radix_tree_for_each_slot().
+ * Caller must hold tree write locked across split and replacement.
+ */
+void radix_tree_iter_replace(struct radix_tree_root *root,
+		const struct radix_tree_iter *iter, void **slot, void *item)
+{
+	__radix_tree_replace(root, iter->node, slot, item, NULL, NULL);
+}
+
+#ifdef CONFIG_RADIX_TREE_MULTIORDER
+/**
+ * radix_tree_join - replace multiple entries with one multiorder entry
+ * @root: radix tree root
+ * @index: an index inside the new entry
+ * @order: order of the new entry
+ * @item: new entry
+ *
+ * Call this function to replace several entries with one larger entry.
+ * The existing entries are presumed to not need freeing as a result of
+ * this call.
+ *
+ * The replacement entry will have all the tags set on it that were set
+ * on any of the entries it is replacing.
+ */
+int radix_tree_join(struct radix_tree_root *root, unsigned long index,
+			unsigned order, void *item)
+{
+	struct radix_tree_node *node;
+	void **slot;
+	int error;
+
+	BUG_ON(radix_tree_is_internal_node(item));
+
+	error = __radix_tree_create(root, index, order, &node, &slot);
+	if (!error)
+		error = insert_entries(node, slot, item, order, true);
+	if (error > 0)
+		error = 0;
+
+	return error;
+}
+
+/**
+ * radix_tree_split - Split an entry into smaller entries
+ * @root: radix tree root
+ * @index: An index within the large entry
+ * @order: Order of new entries
+ *
+ * Call this function as the first step in replacing a multiorder entry
+ * with several entries of lower order.  After this function returns,
+ * loop over the relevant portion of the tree using radix_tree_for_each_slot()
+ * and call radix_tree_iter_replace() to set up each new entry.
+ *
+ * The tags from this entry are replicated to all the new entries.
+ *
+ * The radix tree should be locked against modification during the entire
+ * replacement operation.  Lock-free lookups will see RADIX_TREE_RETRY which
+ * should prompt RCU walkers to restart the lookup from the root.
+ */
+int radix_tree_split(struct radix_tree_root *root, unsigned long index,
+				unsigned order)
+{
+	struct radix_tree_node *parent, *node, *child;
+	void **slot;
+	unsigned int offset, end;
+	unsigned n, tag, tags = 0;
+
+	if (!__radix_tree_lookup(root, index, &parent, &slot))
+		return -ENOENT;
+	if (!parent)
+		return -ENOENT;
+
+	offset = get_slot_offset(parent, slot);
+
+	for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
+		if (tag_get(parent, tag, offset))
+			tags |= 1 << tag;
+
+	for (end = offset + 1; end < RADIX_TREE_MAP_SIZE; end++) {
+		if (!is_sibling_entry(parent, parent->slots[end]))
+			break;
+		for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
+			if (tags & (1 << tag))
+				tag_set(parent, tag, end);
+		/* rcu_assign_pointer ensures tags are set before RETRY */
+		rcu_assign_pointer(parent->slots[end], RADIX_TREE_RETRY);
+	}
+	rcu_assign_pointer(parent->slots[offset], RADIX_TREE_RETRY);
+	parent->exceptional -= (end - offset);
+
+	if (order == parent->shift)
+		return 0;
+	if (order > parent->shift) {
+		while (offset < end)
+			offset += insert_entries(parent, &parent->slots[offset],
+					RADIX_TREE_RETRY, order, true);
+		return 0;
+	}
+
+	node = parent;
+
+	for (;;) {
+		if (node->shift > order) {
+			child = radix_tree_node_alloc(root, node,
+					node->shift - RADIX_TREE_MAP_SHIFT,
+					offset, 0, 0);
+			if (!child)
+				goto nomem;
+			if (node != parent) {
+				node->count++;
+				node->slots[offset] = node_to_entry(child);
+				for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
+					if (tags & (1 << tag))
+						tag_set(node, tag, offset);
+			}
+
+			node = child;
+			offset = 0;
+			continue;
+		}
+
+		n = insert_entries(node, &node->slots[offset],
+					RADIX_TREE_RETRY, order, false);
+		BUG_ON(n > RADIX_TREE_MAP_SIZE);
+
+		for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
+			if (tags & (1 << tag))
+				tag_set(node, tag, offset);
+		offset += n;
+
+		while (offset == RADIX_TREE_MAP_SIZE) {
+			if (node == parent)
+				break;
+			offset = node->offset;
+			child = node;
+			node = node->parent;
+			rcu_assign_pointer(node->slots[offset],
+						node_to_entry(child));
+			offset++;
+		}
+		if ((node == parent) && (offset == end))
+			return 0;
+	}
+
+ nomem:
+	/* Shouldn't happen; did user forget to preload? */
+	/* TODO: free all the allocated nodes */
+	WARN_ON(1);
+	return -ENOMEM;
+}
+#endif
+
+/**
  *	radix_tree_tag_set - set a tag on a radix tree node
  *	@root:		radix tree root
  *	@index:		index key
@@ -990,6 +1319,34 @@ static void node_tag_clear(struct radix_tree_root *root,
 		root_tag_clear(root, tag);
 }
 
+static void node_tag_set(struct radix_tree_root *root,
+				struct radix_tree_node *node,
+				unsigned int tag, unsigned int offset)
+{
+	while (node) {
+		if (tag_get(node, tag, offset))
+			return;
+		tag_set(node, tag, offset);
+		offset = node->offset;
+		node = node->parent;
+	}
+
+	if (!root_tag_get(root, tag))
+		root_tag_set(root, tag);
+}
+
+/**
+ * radix_tree_iter_tag_set - set a tag on the current iterator entry
+ * @root:	radix tree root
+ * @iter:	iterator state
+ * @tag:	tag to set
+ */
+void radix_tree_iter_tag_set(struct radix_tree_root *root,
+			const struct radix_tree_iter *iter, unsigned int tag)
+{
+	node_tag_set(root, iter->node, tag, iter_offset(iter));
+}
+
 /**
  *	radix_tree_tag_clear - clear a tag on a radix tree node
  *	@root:		radix tree root
@@ -1085,6 +1442,121 @@ static inline void __set_iter_shift(struct radix_tree_iter *iter,
 #endif
 }
 
+/* Construct iter->tags bit-mask from node->tags[tag] array */
+static void set_iter_tags(struct radix_tree_iter *iter,
+				struct radix_tree_node *node, unsigned offset,
+				unsigned tag)
+{
+	unsigned tag_long = offset / BITS_PER_LONG;
+	unsigned tag_bit  = offset % BITS_PER_LONG;
+
+	iter->tags = node->tags[tag][tag_long] >> tag_bit;
+
+	/* This never happens if RADIX_TREE_TAG_LONGS == 1 */
+	if (tag_long < RADIX_TREE_TAG_LONGS - 1) {
+		/* Pick tags from next element */
+		if (tag_bit)
+			iter->tags |= node->tags[tag][tag_long + 1] <<
+						(BITS_PER_LONG - tag_bit);
+		/* Clip chunk size, here only BITS_PER_LONG tags */
+		iter->next_index = __radix_tree_iter_add(iter, BITS_PER_LONG);
+	}
+}
+
+#ifdef CONFIG_RADIX_TREE_MULTIORDER
+static void **skip_siblings(struct radix_tree_node **nodep,
+			void **slot, struct radix_tree_iter *iter)
+{
+	void *sib = node_to_entry(slot - 1);
+
+	while (iter->index < iter->next_index) {
+		*nodep = rcu_dereference_raw(*slot);
+		if (*nodep && *nodep != sib)
+			return slot;
+		slot++;
+		iter->index = __radix_tree_iter_add(iter, 1);
+		iter->tags >>= 1;
+	}
+
+	*nodep = NULL;
+	return NULL;
+}
+
+void ** __radix_tree_next_slot(void **slot, struct radix_tree_iter *iter,
+					unsigned flags)
+{
+	unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK;
+	struct radix_tree_node *node = rcu_dereference_raw(*slot);
+
+	slot = skip_siblings(&node, slot, iter);
+
+	while (radix_tree_is_internal_node(node)) {
+		unsigned offset;
+		unsigned long next_index;
+
+		if (node == RADIX_TREE_RETRY)
+			return slot;
+		node = entry_to_node(node);
+		iter->node = node;
+		iter->shift = node->shift;
+
+		if (flags & RADIX_TREE_ITER_TAGGED) {
+			offset = radix_tree_find_next_bit(node, tag, 0);
+			if (offset == RADIX_TREE_MAP_SIZE)
+				return NULL;
+			slot = &node->slots[offset];
+			iter->index = __radix_tree_iter_add(iter, offset);
+			set_iter_tags(iter, node, offset, tag);
+			node = rcu_dereference_raw(*slot);
+		} else {
+			offset = 0;
+			slot = &node->slots[0];
+			for (;;) {
+				node = rcu_dereference_raw(*slot);
+				if (node)
+					break;
+				slot++;
+				offset++;
+				if (offset == RADIX_TREE_MAP_SIZE)
+					return NULL;
+			}
+			iter->index = __radix_tree_iter_add(iter, offset);
+		}
+		if ((flags & RADIX_TREE_ITER_CONTIG) && (offset > 0))
+			goto none;
+		next_index = (iter->index | shift_maxindex(iter->shift)) + 1;
+		if (next_index < iter->next_index)
+			iter->next_index = next_index;
+	}
+
+	return slot;
+ none:
+	iter->next_index = 0;
+	return NULL;
+}
+EXPORT_SYMBOL(__radix_tree_next_slot);
+#else
+static void **skip_siblings(struct radix_tree_node **nodep,
+			void **slot, struct radix_tree_iter *iter)
+{
+	return slot;
+}
+#endif
+
+void **radix_tree_iter_resume(void **slot, struct radix_tree_iter *iter)
+{
+	struct radix_tree_node *node;
+
+	slot++;
+	iter->index = __radix_tree_iter_add(iter, 1);
+	node = rcu_dereference_raw(*slot);
+	skip_siblings(&node, slot, iter);
+	iter->next_index = iter->index;
+	iter->tags = 0;
+	return NULL;
+}
+EXPORT_SYMBOL(radix_tree_iter_resume);
+
 /**
  * radix_tree_next_chunk - find next chunk of slots for iteration
  *
@@ -1110,7 +1582,7 @@ void **radix_tree_next_chunk(struct radix_tree_root *root,
 	 * because RADIX_TREE_MAP_SHIFT < BITS_PER_LONG.
 	 *
 	 * This condition also used by radix_tree_next_slot() to stop
-	 * contiguous iterating, and forbid swithing to the next chunk.
+	 * contiguous iterating, and forbid switching to the next chunk.
 	 */
 	index = iter->next_index;
 	if (!index && iter->index)
@@ -1128,6 +1600,7 @@ void **radix_tree_next_chunk(struct radix_tree_root *root,
 		iter->index = index;
 		iter->next_index = maxindex + 1;
 		iter->tags = 1;
+		iter->node = NULL;
 		__set_iter_shift(iter, 0);
 		return (void **)&root->rnode;
 	}
@@ -1143,9 +1616,7 @@ void **radix_tree_next_chunk(struct radix_tree_root *root,
 				return NULL;
 
 			if (flags & RADIX_TREE_ITER_TAGGED)
-				offset = radix_tree_find_next_bit(
-						node->tags[tag],
-						RADIX_TREE_MAP_SIZE,
+				offset = radix_tree_find_next_bit(node, tag,
 						offset + 1);
 			else
 				while (++offset	< RADIX_TREE_MAP_SIZE) {
@@ -1165,154 +1636,26 @@ void **radix_tree_next_chunk(struct radix_tree_root *root,
 			child = rcu_dereference_raw(node->slots[offset]);
 		}
 
-		if ((child == NULL) || (child == RADIX_TREE_RETRY))
+		if (!child)
 			goto restart;
+		if (child == RADIX_TREE_RETRY)
+			break;
 	} while (radix_tree_is_internal_node(child));
 
 	/* Update the iterator state */
 	iter->index = (index &~ node_maxindex(node)) | (offset << node->shift);
 	iter->next_index = (index | node_maxindex(node)) + 1;
+	iter->node = node;
 	__set_iter_shift(iter, node->shift);
 
-	/* Construct iter->tags bit-mask from node->tags[tag] array */
-	if (flags & RADIX_TREE_ITER_TAGGED) {
-		unsigned tag_long, tag_bit;
-
-		tag_long = offset / BITS_PER_LONG;
-		tag_bit  = offset % BITS_PER_LONG;
-		iter->tags = node->tags[tag][tag_long] >> tag_bit;
-		/* This never happens if RADIX_TREE_TAG_LONGS == 1 */
-		if (tag_long < RADIX_TREE_TAG_LONGS - 1) {
-			/* Pick tags from next element */
-			if (tag_bit)
-				iter->tags |= node->tags[tag][tag_long + 1] <<
-						(BITS_PER_LONG - tag_bit);
-			/* Clip chunk size, here only BITS_PER_LONG tags */
-			iter->next_index = index + BITS_PER_LONG;
-		}
-	}
+	if (flags & RADIX_TREE_ITER_TAGGED)
+		set_iter_tags(iter, node, offset, tag);
 
 	return node->slots + offset;
 }
 EXPORT_SYMBOL(radix_tree_next_chunk);
 
 /**
- * radix_tree_range_tag_if_tagged - for each item in given range set given
- *				   tag if item has another tag set
- * @root:		radix tree root
- * @first_indexp:	pointer to a starting index of a range to scan
- * @last_index:		last index of a range to scan
- * @nr_to_tag:		maximum number items to tag
- * @iftag:		tag index to test
- * @settag:		tag index to set if tested tag is set
- *
- * This function scans range of radix tree from first_index to last_index
- * (inclusive).  For each item in the range if iftag is set, the function sets
- * also settag. The function stops either after tagging nr_to_tag items or
- * after reaching last_index.
- *
- * The tags must be set from the leaf level only and propagated back up the
- * path to the root. We must do this so that we resolve the full path before
- * setting any tags on intermediate nodes. If we set tags as we descend, then
- * we can get to the leaf node and find that the index that has the iftag
- * set is outside the range we are scanning. This reults in dangling tags and
- * can lead to problems with later tag operations (e.g. livelocks on lookups).
- *
- * The function returns the number of leaves where the tag was set and sets
- * *first_indexp to the first unscanned index.
- * WARNING! *first_indexp can wrap if last_index is ULONG_MAX. Caller must
- * be prepared to handle that.
- */
-unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
-		unsigned long *first_indexp, unsigned long last_index,
-		unsigned long nr_to_tag,
-		unsigned int iftag, unsigned int settag)
-{
-	struct radix_tree_node *parent, *node, *child;
-	unsigned long maxindex;
-	unsigned long tagged = 0;
-	unsigned long index = *first_indexp;
-
-	radix_tree_load_root(root, &child, &maxindex);
-	last_index = min(last_index, maxindex);
-	if (index > last_index)
-		return 0;
-	if (!nr_to_tag)
-		return 0;
-	if (!root_tag_get(root, iftag)) {
-		*first_indexp = last_index + 1;
-		return 0;
-	}
-	if (!radix_tree_is_internal_node(child)) {
-		*first_indexp = last_index + 1;
-		root_tag_set(root, settag);
-		return 1;
-	}
-
-	node = entry_to_node(child);
-
-	for (;;) {
-		unsigned offset = radix_tree_descend(node, &child, index);
-		if (!child)
-			goto next;
-		if (!tag_get(node, iftag, offset))
-			goto next;
-		/* Sibling slots never have tags set on them */
-		if (radix_tree_is_internal_node(child)) {
-			node = entry_to_node(child);
-			continue;
-		}
-
-		/* tag the leaf */
-		tagged++;
-		tag_set(node, settag, offset);
-
-		/* walk back up the path tagging interior nodes */
-		parent = node;
-		for (;;) {
-			offset = parent->offset;
-			parent = parent->parent;
-			if (!parent)
-				break;
-			/* stop if we find a node with the tag already set */
-			if (tag_get(parent, settag, offset))
-				break;
-			tag_set(parent, settag, offset);
-		}
- next:
-		/* Go to next entry in node */
-		index = ((index >> node->shift) + 1) << node->shift;
-		/* Overflow can happen when last_index is ~0UL... */
-		if (index > last_index || !index)
-			break;
-		offset = (index >> node->shift) & RADIX_TREE_MAP_MASK;
-		while (offset == 0) {
-			/*
-			 * We've fully scanned this node. Go up. Because
-			 * last_index is guaranteed to be in the tree, what
-			 * we do below cannot wander astray.
-			 */
-			node = node->parent;
-			offset = (index >> node->shift) & RADIX_TREE_MAP_MASK;
-		}
-		if (is_sibling_entry(node, node->slots[offset]))
-			goto next;
-		if (tagged >= nr_to_tag)
-			break;
-	}
-	/*
-	 * We need not to tag the root tag if there is no tag which is set with
-	 * settag within the range from *first_indexp to last_index.
-	 */
-	if (tagged > 0)
-		root_tag_set(root, settag);
-	*first_indexp = index;
-
-	return tagged;
-}
-EXPORT_SYMBOL(radix_tree_range_tag_if_tagged);
-
-/**
  *	radix_tree_gang_lookup - perform multiple lookup on a radix tree
  *	@root:		radix tree root
  *	@results:	where the results of the lookup are placed
@@ -1477,105 +1820,6 @@ radix_tree_gang_lookup_tag_slot(struct radix_tree_root *root, void ***results,
 }
 EXPORT_SYMBOL(radix_tree_gang_lookup_tag_slot);
 
-#if defined(CONFIG_SHMEM) && defined(CONFIG_SWAP)
-#include <linux/sched.h> /* for cond_resched() */
-
-struct locate_info {
-	unsigned long found_index;
-	bool stop;
-};
-
-/*
- * This linear search is at present only useful to shmem_unuse_inode().
- */
-static unsigned long __locate(struct radix_tree_node *slot, void *item,
-			      unsigned long index, struct locate_info *info)
-{
-	unsigned long i;
-
-	do {
-		unsigned int shift = slot->shift;
-
-		for (i = (index >> shift) & RADIX_TREE_MAP_MASK;
-		     i < RADIX_TREE_MAP_SIZE;
-		     i++, index += (1UL << shift)) {
-			struct radix_tree_node *node =
-					rcu_dereference_raw(slot->slots[i]);
-			if (node == RADIX_TREE_RETRY)
-				goto out;
-			if (!radix_tree_is_internal_node(node)) {
-				if (node == item) {
-					info->found_index = index;
-					info->stop = true;
-					goto out;
-				}
-				continue;
-			}
-			node = entry_to_node(node);
-			if (is_sibling_entry(slot, node))
-				continue;
-			slot = node;
-			break;
-		}
-	} while (i < RADIX_TREE_MAP_SIZE);
-
-out:
-	if ((index == 0) && (i == RADIX_TREE_MAP_SIZE))
-		info->stop = true;
-	return index;
-}
-
-/**
- *	radix_tree_locate_item - search through radix tree for item
- *	@root:		radix tree root
- *	@item:		item to be found
- *
- *	Returns index where item was found, or -1 if not found.
- *	Caller must hold no lock (since this time-consuming function needs
- *	to be preemptible), and must check afterwards if item is still there.
- */
-unsigned long radix_tree_locate_item(struct radix_tree_root *root, void *item)
-{
-	struct radix_tree_node *node;
-	unsigned long max_index;
-	unsigned long cur_index = 0;
-	struct locate_info info = {
-		.found_index = -1,
-		.stop = false,
-	};
-
-	do {
-		rcu_read_lock();
-		node = rcu_dereference_raw(root->rnode);
-		if (!radix_tree_is_internal_node(node)) {
-			rcu_read_unlock();
-			if (node == item)
-				info.found_index = 0;
-			break;
-		}
-
-		node = entry_to_node(node);
-
-		max_index = node_maxindex(node);
-		if (cur_index > max_index) {
-			rcu_read_unlock();
-			break;
-		}
-
-		cur_index = __locate(node, item, cur_index, &info);
-		rcu_read_unlock();
-		cond_resched();
-	} while (!info.stop && cur_index <= max_index);
-
-	return info.found_index;
-}
-#else
-unsigned long radix_tree_locate_item(struct radix_tree_root *root, void *item)
-{
-	return -1;
-}
-#endif /* CONFIG_SHMEM && CONFIG_SWAP */
-
 /**
  *	__radix_tree_delete_node    -    try to free node after clearing a slot
  *	@root:		radix tree root
@@ -1591,20 +1835,6 @@ void __radix_tree_delete_node(struct radix_tree_root *root,
 	delete_node(root, node, NULL, NULL);
 }
 
-static inline void delete_sibling_entries(struct radix_tree_node *node,
-					void *ptr, unsigned offset)
-{
-#ifdef CONFIG_RADIX_TREE_MULTIORDER
-	int i;
-	for (i = 1; offset + i < RADIX_TREE_MAP_SIZE; i++) {
-		if (node->slots[offset + i] != ptr)
-			break;
-		node->slots[offset + i] = NULL;
-		node->count--;
-	}
-#endif
-}
-
 /**
  *	radix_tree_delete_item    -    delete an item from a radix tree
  *	@root:		radix tree root
@@ -1644,7 +1874,6 @@ void *radix_tree_delete_item(struct radix_tree_root *root,
 	for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
 		node_tag_clear(root, node, tag, offset);
 
-	delete_sibling_entries(node, node_to_entry(slot), offset);
 	__radix_tree_replace(root, node, slot, NULL, NULL, NULL);
 
 	return entry;
diff --git a/lib/raid6/avx2.c b/lib/raid6/avx2.c
index 76734004358d..20bca3d44f67 100644
--- a/lib/raid6/avx2.c
+++ b/lib/raid6/avx2.c
@@ -87,9 +87,57 @@ static void raid6_avx21_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	kernel_fpu_end();
 }
 
+static void raid6_avx21_xor_syndrome(int disks, int start, int stop,
+				     size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
+
+	for (d = 0 ; d < bytes ; d += 32) {
+		asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));
+		asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));
+		asm volatile("vpxor %ymm4,%ymm2,%ymm2");
+		/* P/Q data pages */
+		for (z = z0-1 ; z >= start ; z--) {
+			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
+			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));
+			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+		}
+		/* P/Q left side optimization */
+		for (z = start-1 ; z >= 0 ; z--) {
+			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
+			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+		}
+		asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));
+		/* Don't use movntdq for r/w memory area < cache line */
+		asm volatile("vmovdqa %%ymm4,%0" : "=m" (q[d]));
+		asm volatile("vmovdqa %%ymm2,%0" : "=m" (p[d]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
 const struct raid6_calls raid6_avx2x1 = {
 	raid6_avx21_gen_syndrome,
-	NULL,			/* XOR not yet implemented */
+	raid6_avx21_xor_syndrome,
 	raid6_have_avx2,
 	"avx2x1",
 	1			/* Has cache hints */
@@ -149,9 +197,77 @@ static void raid6_avx22_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	kernel_fpu_end();
 }
 
+static void raid6_avx22_xor_syndrome(int disks, int start, int stop,
+				     size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
+
+	for (d = 0 ; d < bytes ; d += 64) {
+		asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));
+		asm volatile("vmovdqa %0,%%ymm6" :: "m" (dptr[z0][d+32]));
+		asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));
+		asm volatile("vmovdqa %0,%%ymm3" : : "m" (p[d+32]));
+		asm volatile("vpxor %ymm4,%ymm2,%ymm2");
+		asm volatile("vpxor %ymm6,%ymm3,%ymm3");
+		/* P/Q data pages */
+		for (z = z0-1 ; z >= start ; z--) {
+			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm7,%ymm7,%ymm7");
+			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
+			asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpand %ymm0,%ymm7,%ymm7");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+			asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));
+			asm volatile("vmovdqa %0,%%ymm7"
+				     :: "m" (dptr[z][d+32]));
+			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
+			asm volatile("vpxor %ymm7,%ymm3,%ymm3");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+		}
+		/* P/Q left side optimization */
+		for (z = start-1 ; z >= 0 ; z--) {
+			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm7,%ymm7,%ymm7");
+			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
+			asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpand %ymm0,%ymm7,%ymm7");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+		}
+		asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));
+		asm volatile("vpxor %0,%%ymm6,%%ymm6" : : "m" (q[d+32]));
+		/* Don't use movntdq for r/w memory area < cache line */
+		asm volatile("vmovdqa %%ymm4,%0" : "=m" (q[d]));
+		asm volatile("vmovdqa %%ymm6,%0" : "=m" (q[d+32]));
+		asm volatile("vmovdqa %%ymm2,%0" : "=m" (p[d]));
+		asm volatile("vmovdqa %%ymm3,%0" : "=m" (p[d+32]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
 const struct raid6_calls raid6_avx2x2 = {
 	raid6_avx22_gen_syndrome,
-	NULL,			/* XOR not yet implemented */
+	raid6_avx22_xor_syndrome,
 	raid6_have_avx2,
 	"avx2x2",
 	1			/* Has cache hints */
@@ -242,9 +358,119 @@ static void raid6_avx24_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	kernel_fpu_end();
 }
 
+static void raid6_avx24_xor_syndrome(int disks, int start, int stop,
+				     size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa %0,%%ymm0" :: "m" (raid6_avx2_constants.x1d[0]));
+
+	for (d = 0 ; d < bytes ; d += 128) {
+		asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));
+		asm volatile("vmovdqa %0,%%ymm6" :: "m" (dptr[z0][d+32]));
+		asm volatile("vmovdqa %0,%%ymm12" :: "m" (dptr[z0][d+64]));
+		asm volatile("vmovdqa %0,%%ymm14" :: "m" (dptr[z0][d+96]));
+		asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));
+		asm volatile("vmovdqa %0,%%ymm3" : : "m" (p[d+32]));
+		asm volatile("vmovdqa %0,%%ymm10" : : "m" (p[d+64]));
+		asm volatile("vmovdqa %0,%%ymm11" : : "m" (p[d+96]));
+		asm volatile("vpxor %ymm4,%ymm2,%ymm2");
+		asm volatile("vpxor %ymm6,%ymm3,%ymm3");
+		asm volatile("vpxor %ymm12,%ymm10,%ymm10");
+		asm volatile("vpxor %ymm14,%ymm11,%ymm11");
+		/* P/Q data pages */
+		for (z = z0-1 ; z >= start ; z--) {
+			asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
+			asm volatile("prefetchnta %0" :: "m" (dptr[z][d+64]));
+			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm7,%ymm7,%ymm7");
+			asm volatile("vpxor %ymm13,%ymm13,%ymm13");
+			asm volatile("vpxor %ymm15,%ymm15,%ymm15");
+			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
+			asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
+			asm volatile("vpcmpgtb %ymm12,%ymm13,%ymm13");
+			asm volatile("vpcmpgtb %ymm14,%ymm15,%ymm15");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
+			asm volatile("vpaddb %ymm12,%ymm12,%ymm12");
+			asm volatile("vpaddb %ymm14,%ymm14,%ymm14");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpand %ymm0,%ymm7,%ymm7");
+			asm volatile("vpand %ymm0,%ymm13,%ymm13");
+			asm volatile("vpand %ymm0,%ymm15,%ymm15");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
+			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
+			asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));
+			asm volatile("vmovdqa %0,%%ymm7"
+				     :: "m" (dptr[z][d+32]));
+			asm volatile("vmovdqa %0,%%ymm13"
+				     :: "m" (dptr[z][d+64]));
+			asm volatile("vmovdqa %0,%%ymm15"
+				     :: "m" (dptr[z][d+96]));
+			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
+			asm volatile("vpxor %ymm7,%ymm3,%ymm3");
+			asm volatile("vpxor %ymm13,%ymm10,%ymm10");
+			asm volatile("vpxor %ymm15,%ymm11,%ymm11");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
+			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
+		}
+		asm volatile("prefetchnta %0" :: "m" (q[d]));
+		asm volatile("prefetchnta %0" :: "m" (q[d+64]));
+		/* P/Q left side optimization */
+		for (z = start-1 ; z >= 0 ; z--) {
+			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm7,%ymm7,%ymm7");
+			asm volatile("vpxor %ymm13,%ymm13,%ymm13");
+			asm volatile("vpxor %ymm15,%ymm15,%ymm15");
+			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
+			asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
+			asm volatile("vpcmpgtb %ymm12,%ymm13,%ymm13");
+			asm volatile("vpcmpgtb %ymm14,%ymm15,%ymm15");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
+			asm volatile("vpaddb %ymm12,%ymm12,%ymm12");
+			asm volatile("vpaddb %ymm14,%ymm14,%ymm14");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpand %ymm0,%ymm7,%ymm7");
+			asm volatile("vpand %ymm0,%ymm13,%ymm13");
+			asm volatile("vpand %ymm0,%ymm15,%ymm15");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
+			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
+		}
+		asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
+		asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));
+		asm volatile("vmovntdq %%ymm10,%0" : "=m" (p[d+64]));
+		asm volatile("vmovntdq %%ymm11,%0" : "=m" (p[d+96]));
+		asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));
+		asm volatile("vpxor %0,%%ymm6,%%ymm6" : : "m" (q[d+32]));
+		asm volatile("vpxor %0,%%ymm12,%%ymm12" : : "m" (q[d+64]));
+		asm volatile("vpxor %0,%%ymm14,%%ymm14" : : "m" (q[d+96]));
+		asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
+		asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));
+		asm volatile("vmovntdq %%ymm12,%0" : "=m" (q[d+64]));
+		asm volatile("vmovntdq %%ymm14,%0" : "=m" (q[d+96]));
+	}
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
 const struct raid6_calls raid6_avx2x4 = {
 	raid6_avx24_gen_syndrome,
-	NULL,			/* XOR not yet implemented */
+	raid6_avx24_xor_syndrome,
 	raid6_have_avx2,
 	"avx2x4",
 	1			/* Has cache hints */
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 22e13a0e19d7..cb1b54ee8527 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -425,7 +425,8 @@ static void swiotlb_bounce(phys_addr_t orig_addr, phys_addr_t tlb_addr,
 phys_addr_t swiotlb_tbl_map_single(struct device *hwdev,
 				   dma_addr_t tbl_dma_addr,
 				   phys_addr_t orig_addr, size_t size,
-				   enum dma_data_direction dir)
+				   enum dma_data_direction dir,
+				   unsigned long attrs)
 {
 	unsigned long flags;
 	phys_addr_t tlb_addr;
@@ -526,7 +527,8 @@ found:
 	 */
 	for (i = 0; i < nslots; i++)
 		io_tlb_orig_addr[index+i] = orig_addr + (i << IO_TLB_SHIFT);
-	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
+	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
+	    (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
 		swiotlb_bounce(orig_addr, tlb_addr, size, DMA_TO_DEVICE);
 
 	return tlb_addr;
@@ -539,18 +541,20 @@ EXPORT_SYMBOL_GPL(swiotlb_tbl_map_single);
 
 static phys_addr_t
 map_single(struct device *hwdev, phys_addr_t phys, size_t size,
-	   enum dma_data_direction dir)
+	   enum dma_data_direction dir, unsigned long attrs)
 {
 	dma_addr_t start_dma_addr = phys_to_dma(hwdev, io_tlb_start);
 
-	return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, dir);
+	return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size,
+				      dir, attrs);
 }
 
 /*
  * dma_addr is the kernel virtual address of the bounce buffer to unmap.
  */
 void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
-			      size_t size, enum dma_data_direction dir)
+			      size_t size, enum dma_data_direction dir,
+			      unsigned long attrs)
 {
 	unsigned long flags;
 	int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
@@ -561,6 +565,7 @@ void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
 	 * First, sync the memory before unmapping the entry
 	 */
 	if (orig_addr != INVALID_PHYS_ADDR &&
+	    !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
 	    ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
 		swiotlb_bounce(orig_addr, tlb_addr, size, DMA_FROM_DEVICE);
 
@@ -654,7 +659,8 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
 		 * GFP_DMA memory; fall back on map_single(), which
 		 * will grab memory from the lowest available address range.
 		 */
-		phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE);
+		phys_addr_t paddr = map_single(hwdev, 0, size,
+					       DMA_FROM_DEVICE, 0);
 		if (paddr == SWIOTLB_MAP_ERROR)
 			goto err_warn;
 
@@ -667,9 +673,13 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
 			       (unsigned long long)dma_mask,
 			       (unsigned long long)dev_addr);
 
-			/* DMA_TO_DEVICE to avoid memcpy in unmap_single */
+			/*
+			 * DMA_TO_DEVICE to avoid memcpy in unmap_single.
+			 * The DMA_ATTR_SKIP_CPU_SYNC is optional.
+			 */
 			swiotlb_tbl_unmap_single(hwdev, paddr,
-						 size, DMA_TO_DEVICE);
+						 size, DMA_TO_DEVICE,
+						 DMA_ATTR_SKIP_CPU_SYNC);
 			goto err_warn;
 		}
 	}
@@ -698,8 +708,12 @@ swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
 	if (!is_swiotlb_buffer(paddr))
 		free_pages((unsigned long)vaddr, get_order(size));
 	else
-		/* DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single */
-		swiotlb_tbl_unmap_single(hwdev, paddr, size, DMA_TO_DEVICE);
+		/*
+		 * DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single.
+		 * DMA_ATTR_SKIP_CPU_SYNC is optional.
+		 */
+		swiotlb_tbl_unmap_single(hwdev, paddr, size, DMA_TO_DEVICE,
+					 DMA_ATTR_SKIP_CPU_SYNC);
 }
 EXPORT_SYMBOL(swiotlb_free_coherent);
 
@@ -714,8 +728,8 @@ swiotlb_full(struct device *dev, size_t size, enum dma_data_direction dir,
 	 * When the mapping is small enough return a static buffer to limit
 	 * the damage, or panic when the transfer is too big.
 	 */
-	printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at "
-	       "device %s\n", size, dev ? dev_name(dev) : "?");
+	dev_err_ratelimited(dev, "DMA: Out of SW-IOMMU space for %zu bytes\n",
+			    size);
 
 	if (size <= io_tlb_overflow || !do_panic)
 		return;
@@ -755,7 +769,7 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
 	trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force);
 
 	/* Oh well, have to allocate and map a bounce buffer. */
-	map = map_single(dev, phys, size, dir);
+	map = map_single(dev, phys, size, dir, attrs);
 	if (map == SWIOTLB_MAP_ERROR) {
 		swiotlb_full(dev, size, dir, 1);
 		return phys_to_dma(dev, io_tlb_overflow_buffer);
@@ -764,12 +778,13 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
 	dev_addr = phys_to_dma(dev, map);
 
 	/* Ensure that the address returned is DMA'ble */
-	if (!dma_capable(dev, dev_addr, size)) {
-		swiotlb_tbl_unmap_single(dev, map, size, dir);
-		return phys_to_dma(dev, io_tlb_overflow_buffer);
-	}
+	if (dma_capable(dev, dev_addr, size))
+		return dev_addr;
 
-	return dev_addr;
+	attrs |= DMA_ATTR_SKIP_CPU_SYNC;
+	swiotlb_tbl_unmap_single(dev, map, size, dir, attrs);
+
+	return phys_to_dma(dev, io_tlb_overflow_buffer);
 }
 EXPORT_SYMBOL_GPL(swiotlb_map_page);
 
@@ -782,14 +797,15 @@ EXPORT_SYMBOL_GPL(swiotlb_map_page);
  * whatever the device wrote there.
  */
 static void unmap_single(struct device *hwdev, dma_addr_t dev_addr,
-			 size_t size, enum dma_data_direction dir)
+			 size_t size, enum dma_data_direction dir,
+			 unsigned long attrs)
 {
 	phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
 
 	BUG_ON(dir == DMA_NONE);
 
 	if (is_swiotlb_buffer(paddr)) {
-		swiotlb_tbl_unmap_single(hwdev, paddr, size, dir);
+		swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs);
 		return;
 	}
 
@@ -809,7 +825,7 @@ void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
 			size_t size, enum dma_data_direction dir,
 			unsigned long attrs)
 {
-	unmap_single(hwdev, dev_addr, size, dir);
+	unmap_single(hwdev, dev_addr, size, dir, attrs);
 }
 EXPORT_SYMBOL_GPL(swiotlb_unmap_page);
 
@@ -891,11 +907,12 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
 		if (swiotlb_force ||
 		    !dma_capable(hwdev, dev_addr, sg->length)) {
 			phys_addr_t map = map_single(hwdev, sg_phys(sg),
-						     sg->length, dir);
+						     sg->length, dir, attrs);
 			if (map == SWIOTLB_MAP_ERROR) {
 				/* Don't panic here, we expect map_sg users
 				   to do proper error handling. */
 				swiotlb_full(hwdev, sg->length, dir, 0);
+				attrs |= DMA_ATTR_SKIP_CPU_SYNC;
 				swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
 						       attrs);
 				sg_dma_len(sgl) = 0;
@@ -910,14 +927,6 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
 }
 EXPORT_SYMBOL(swiotlb_map_sg_attrs);
 
-int
-swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
-	       enum dma_data_direction dir)
-{
-	return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, 0);
-}
-EXPORT_SYMBOL(swiotlb_map_sg);
-
 /*
  * Unmap a set of streaming mode DMA translations.  Again, cpu read rules
  * concerning calls here are the same as for swiotlb_unmap_page() above.
@@ -933,19 +942,11 @@ swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
 	BUG_ON(dir == DMA_NONE);
 
 	for_each_sg(sgl, sg, nelems, i)
-		unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir);
-
+		unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir,
+			     attrs);
 }
 EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);
 
-void
-swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
-		 enum dma_data_direction dir)
-{
-	return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, 0);
-}
-EXPORT_SYMBOL(swiotlb_unmap_sg);
-
 /*
  * Make physical memory consistent for a set of streaming mode DMA translations
  * after a transfer.
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index 8fde443f36d7..3bfed5ab2475 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -310,6 +310,7 @@ static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi,
 	spin_lock_init(&wb->work_lock);
 	INIT_LIST_HEAD(&wb->work_list);
 	INIT_DELAYED_WORK(&wb->dwork, wb_workfn);
+	wb->dirty_sleep = jiffies;
 
 	wb->congested = wb_congested_get_create(bdi, blkcg_id, gfp);
 	if (!wb->congested)
diff --git a/mm/compaction.c b/mm/compaction.c
index 223464227299..949198d01260 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -818,6 +818,13 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 		    page_count(page) > page_mapcount(page))
 			goto isolate_fail;
 
+		/*
+		 * Only allow to migrate anonymous pages in GFP_NOFS context
+		 * because those do not depend on fs locks.
+		 */
+		if (!(cc->gfp_mask & __GFP_FS) && page_mapping(page))
+			goto isolate_fail;
+
 		/* If we already hold the lock, we can skip some rechecking */
 		if (!locked) {
 			locked = compact_trylock_irqsave(zone_lru_lock(zone),
@@ -1677,14 +1684,16 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
 		unsigned int alloc_flags, const struct alloc_context *ac,
 		enum compact_priority prio)
 {
-	int may_enter_fs = gfp_mask & __GFP_FS;
 	int may_perform_io = gfp_mask & __GFP_IO;
 	struct zoneref *z;
 	struct zone *zone;
 	enum compact_result rc = COMPACT_SKIPPED;
 
-	/* Check if the GFP flags allow compaction */
-	if (!may_enter_fs || !may_perform_io)
+	/*
+	 * Check if the GFP flags allow compaction - GFP_NOIO is really
+	 * tricky context because the migration might require IO
+	 */
+	if (!may_perform_io)
 		return COMPACT_SKIPPED;
 
 	trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);
@@ -1751,6 +1760,7 @@ static void compact_node(int nid)
 		.mode = MIGRATE_SYNC,
 		.ignore_skip_hint = true,
 		.whole_zone = true,
+		.gfp_mask = GFP_KERNEL,
 	};
 
 
@@ -1876,6 +1886,7 @@ static void kcompactd_do_work(pg_data_t *pgdat)
 		.classzone_idx = pgdat->kcompactd_classzone_idx,
 		.mode = MIGRATE_SYNC_LIGHT,
 		.ignore_skip_hint = true,
+		.gfp_mask = GFP_KERNEL,
 
 	};
 	trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
diff --git a/mm/filemap.c b/mm/filemap.c
index 5b4dd03130da..32be3c8f3a11 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -135,10 +135,9 @@ static int page_cache_tree_insert(struct address_space *mapping,
 		} else {
 			/* DAX can replace empty locked entry with a hole */
 			WARN_ON_ONCE(p !=
-				(void *)(RADIX_TREE_EXCEPTIONAL_ENTRY |
-					 RADIX_DAX_ENTRY_LOCK));
+				dax_radix_locked_entry(0, RADIX_DAX_EMPTY));
 			/* Wakeup waiters for exceptional entry lock */
-			dax_wake_mapping_entry_waiter(mapping, page->index,
+			dax_wake_mapping_entry_waiter(mapping, page->index, p,
 						      false);
 		}
 	}
@@ -1638,7 +1637,7 @@ static ssize_t do_generic_file_read(struct file *filp, loff_t *ppos,
 	int error = 0;
 
 	if (unlikely(*ppos >= inode->i_sb->s_maxbytes))
-		return -EINVAL;
+		return 0;
 	iov_iter_truncate(iter, inode->i_sb->s_maxbytes);
 
 	index = *ppos >> PAGE_SHIFT;
@@ -2165,12 +2164,12 @@ page_not_uptodate:
 }
 EXPORT_SYMBOL(filemap_fault);
 
-void filemap_map_pages(struct fault_env *fe,
+void filemap_map_pages(struct vm_fault *vmf,
 		pgoff_t start_pgoff, pgoff_t end_pgoff)
 {
 	struct radix_tree_iter iter;
 	void **slot;
-	struct file *file = fe->vma->vm_file;
+	struct file *file = vmf->vma->vm_file;
 	struct address_space *mapping = file->f_mapping;
 	pgoff_t last_pgoff = start_pgoff;
 	loff_t size;
@@ -2226,11 +2225,11 @@ repeat:
 		if (file->f_ra.mmap_miss > 0)
 			file->f_ra.mmap_miss--;
 
-		fe->address += (iter.index - last_pgoff) << PAGE_SHIFT;
-		if (fe->pte)
-			fe->pte += iter.index - last_pgoff;
+		vmf->address += (iter.index - last_pgoff) << PAGE_SHIFT;
+		if (vmf->pte)
+			vmf->pte += iter.index - last_pgoff;
 		last_pgoff = iter.index;
-		if (alloc_set_pte(fe, NULL, page))
+		if (alloc_set_pte(vmf, NULL, page))
 			goto unlock;
 		unlock_page(page);
 		goto next;
@@ -2240,7 +2239,7 @@ skip:
 		put_page(page);
 next:
 		/* Huge page is mapped? No need to proceed. */
-		if (pmd_trans_huge(*fe->pmd))
+		if (pmd_trans_huge(*vmf->pmd))
 			break;
 		if (iter.index == end_pgoff)
 			break;
diff --git a/mm/gup.c b/mm/gup.c
index e50178c58b97..55315555489d 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -865,9 +865,10 @@ EXPORT_SYMBOL(get_user_pages_locked);
  * caller if required (just like with __get_user_pages). "FOLL_GET"
  * is set implicitly if "pages" is non-NULL.
  */
-__always_inline long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
-					       unsigned long start, unsigned long nr_pages,
-					       struct page **pages, unsigned int gup_flags)
+static __always_inline long __get_user_pages_unlocked(struct task_struct *tsk,
+		struct mm_struct *mm, unsigned long start,
+		unsigned long nr_pages, struct page **pages,
+		unsigned int gup_flags)
 {
 	long ret;
 	int locked = 1;
@@ -879,7 +880,6 @@ __always_inline long __get_user_pages_unlocked(struct task_struct *tsk, struct m
 		up_read(&mm->mmap_sem);
 	return ret;
 }
-EXPORT_SYMBOL(__get_user_pages_unlocked);
 
 /*
  * get_user_pages_unlocked() is suitable to replace the form:
@@ -917,6 +917,9 @@ EXPORT_SYMBOL(get_user_pages_unlocked);
  *		only intends to ensure the pages are faulted in.
  * @vmas:	array of pointers to vmas corresponding to each page.
  *		Or NULL if the caller does not require them.
+ * @locked:	pointer to lock flag indicating whether lock is held and
+ *		subsequently whether VM_FAULT_RETRY functionality can be
+ *		utilised. Lock must initially be held.
  *
  * Returns number of pages pinned. This may be fewer than the number
  * requested. If nr_pages is 0 or negative, returns 0. If no pages
@@ -960,10 +963,10 @@ EXPORT_SYMBOL(get_user_pages_unlocked);
 long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
 		unsigned long start, unsigned long nr_pages,
 		unsigned int gup_flags, struct page **pages,
-		struct vm_area_struct **vmas)
+		struct vm_area_struct **vmas, int *locked)
 {
 	return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas,
-				       NULL, false,
+				       locked, true,
 				       gup_flags | FOLL_TOUCH | FOLL_REMOTE);
 }
 EXPORT_SYMBOL(get_user_pages_remote);
@@ -971,8 +974,9 @@ EXPORT_SYMBOL(get_user_pages_remote);
 /*
  * This is the same as get_user_pages_remote(), just with a
  * less-flexible calling convention where we assume that the task
- * and mm being operated on are the current task's.  We also
- * obviously don't pass FOLL_REMOTE in here.
+ * and mm being operated on are the current task's and don't allow
+ * passing of a locked parameter.  We also obviously don't pass
+ * FOLL_REMOTE in here.
  */
 long get_user_pages(unsigned long start, unsigned long nr_pages,
 		unsigned int gup_flags, struct page **pages,
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index cee42cf05477..10eedbf14421 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -542,13 +542,13 @@ unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
 }
 EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
 
-static int __do_huge_pmd_anonymous_page(struct fault_env *fe, struct page *page,
+static int __do_huge_pmd_anonymous_page(struct vm_fault *vmf, struct page *page,
 		gfp_t gfp)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
 	struct mem_cgroup *memcg;
 	pgtable_t pgtable;
-	unsigned long haddr = fe->address & HPAGE_PMD_MASK;
+	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
 
 	VM_BUG_ON_PAGE(!PageCompound(page), page);
 
@@ -573,9 +573,9 @@ static int __do_huge_pmd_anonymous_page(struct fault_env *fe, struct page *page,
 	 */
 	__SetPageUptodate(page);
 
-	fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
-	if (unlikely(!pmd_none(*fe->pmd))) {
-		spin_unlock(fe->ptl);
+	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
+	if (unlikely(!pmd_none(*vmf->pmd))) {
+		spin_unlock(vmf->ptl);
 		mem_cgroup_cancel_charge(page, memcg, true);
 		put_page(page);
 		pte_free(vma->vm_mm, pgtable);
@@ -586,11 +586,11 @@ static int __do_huge_pmd_anonymous_page(struct fault_env *fe, struct page *page,
 		if (userfaultfd_missing(vma)) {
 			int ret;
 
-			spin_unlock(fe->ptl);
+			spin_unlock(vmf->ptl);
 			mem_cgroup_cancel_charge(page, memcg, true);
 			put_page(page);
 			pte_free(vma->vm_mm, pgtable);
-			ret = handle_userfault(fe, VM_UFFD_MISSING);
+			ret = handle_userfault(vmf, VM_UFFD_MISSING);
 			VM_BUG_ON(ret & VM_FAULT_FALLBACK);
 			return ret;
 		}
@@ -600,11 +600,11 @@ static int __do_huge_pmd_anonymous_page(struct fault_env *fe, struct page *page,
 		page_add_new_anon_rmap(page, vma, haddr, true);
 		mem_cgroup_commit_charge(page, memcg, false, true);
 		lru_cache_add_active_or_unevictable(page, vma);
-		pgtable_trans_huge_deposit(vma->vm_mm, fe->pmd, pgtable);
-		set_pmd_at(vma->vm_mm, haddr, fe->pmd, entry);
+		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
+		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
 		add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
 		atomic_long_inc(&vma->vm_mm->nr_ptes);
-		spin_unlock(fe->ptl);
+		spin_unlock(vmf->ptl);
 		count_vm_event(THP_FAULT_ALLOC);
 	}
 
@@ -651,12 +651,12 @@ static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
 	return true;
 }
 
-int do_huge_pmd_anonymous_page(struct fault_env *fe)
+int do_huge_pmd_anonymous_page(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
 	gfp_t gfp;
 	struct page *page;
-	unsigned long haddr = fe->address & HPAGE_PMD_MASK;
+	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
 
 	if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
 		return VM_FAULT_FALLBACK;
@@ -664,7 +664,7 @@ int do_huge_pmd_anonymous_page(struct fault_env *fe)
 		return VM_FAULT_OOM;
 	if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
 		return VM_FAULT_OOM;
-	if (!(fe->flags & FAULT_FLAG_WRITE) &&
+	if (!(vmf->flags & FAULT_FLAG_WRITE) &&
 			!mm_forbids_zeropage(vma->vm_mm) &&
 			transparent_hugepage_use_zero_page()) {
 		pgtable_t pgtable;
@@ -680,22 +680,22 @@ int do_huge_pmd_anonymous_page(struct fault_env *fe)
 			count_vm_event(THP_FAULT_FALLBACK);
 			return VM_FAULT_FALLBACK;
 		}
-		fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
+		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
 		ret = 0;
 		set = false;
-		if (pmd_none(*fe->pmd)) {
+		if (pmd_none(*vmf->pmd)) {
 			if (userfaultfd_missing(vma)) {
-				spin_unlock(fe->ptl);
-				ret = handle_userfault(fe, VM_UFFD_MISSING);
+				spin_unlock(vmf->ptl);
+				ret = handle_userfault(vmf, VM_UFFD_MISSING);
 				VM_BUG_ON(ret & VM_FAULT_FALLBACK);
 			} else {
 				set_huge_zero_page(pgtable, vma->vm_mm, vma,
-						   haddr, fe->pmd, zero_page);
-				spin_unlock(fe->ptl);
+						   haddr, vmf->pmd, zero_page);
+				spin_unlock(vmf->ptl);
 				set = true;
 			}
 		} else
-			spin_unlock(fe->ptl);
+			spin_unlock(vmf->ptl);
 		if (!set)
 			pte_free(vma->vm_mm, pgtable);
 		return ret;
@@ -707,7 +707,7 @@ int do_huge_pmd_anonymous_page(struct fault_env *fe)
 		return VM_FAULT_FALLBACK;
 	}
 	prep_transhuge_page(page);
-	return __do_huge_pmd_anonymous_page(fe, page, gfp);
+	return __do_huge_pmd_anonymous_page(vmf, page, gfp);
 }
 
 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
@@ -879,30 +879,30 @@ out:
 	return ret;
 }
 
-void huge_pmd_set_accessed(struct fault_env *fe, pmd_t orig_pmd)
+void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd)
 {
 	pmd_t entry;
 	unsigned long haddr;
 
-	fe->ptl = pmd_lock(fe->vma->vm_mm, fe->pmd);
-	if (unlikely(!pmd_same(*fe->pmd, orig_pmd)))
+	vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
+	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
 		goto unlock;
 
 	entry = pmd_mkyoung(orig_pmd);
-	haddr = fe->address & HPAGE_PMD_MASK;
-	if (pmdp_set_access_flags(fe->vma, haddr, fe->pmd, entry,
-				fe->flags & FAULT_FLAG_WRITE))
-		update_mmu_cache_pmd(fe->vma, fe->address, fe->pmd);
+	haddr = vmf->address & HPAGE_PMD_MASK;
+	if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry,
+				vmf->flags & FAULT_FLAG_WRITE))
+		update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);
 
 unlock:
-	spin_unlock(fe->ptl);
+	spin_unlock(vmf->ptl);
 }
 
-static int do_huge_pmd_wp_page_fallback(struct fault_env *fe, pmd_t orig_pmd,
+static int do_huge_pmd_wp_page_fallback(struct vm_fault *vmf, pmd_t orig_pmd,
 		struct page *page)
 {
-	struct vm_area_struct *vma = fe->vma;
-	unsigned long haddr = fe->address & HPAGE_PMD_MASK;
+	struct vm_area_struct *vma = vmf->vma;
+	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
 	struct mem_cgroup *memcg;
 	pgtable_t pgtable;
 	pmd_t _pmd;
@@ -921,7 +921,7 @@ static int do_huge_pmd_wp_page_fallback(struct fault_env *fe, pmd_t orig_pmd,
 	for (i = 0; i < HPAGE_PMD_NR; i++) {
 		pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE |
 					       __GFP_OTHER_NODE, vma,
-					       fe->address, page_to_nid(page));
+					       vmf->address, page_to_nid(page));
 		if (unlikely(!pages[i] ||
 			     mem_cgroup_try_charge(pages[i], vma->vm_mm,
 				     GFP_KERNEL, &memcg, false))) {
@@ -952,15 +952,15 @@ static int do_huge_pmd_wp_page_fallback(struct fault_env *fe, pmd_t orig_pmd,
 	mmun_end   = haddr + HPAGE_PMD_SIZE;
 	mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
 
-	fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
-	if (unlikely(!pmd_same(*fe->pmd, orig_pmd)))
+	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
+	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
 		goto out_free_pages;
 	VM_BUG_ON_PAGE(!PageHead(page), page);
 
-	pmdp_huge_clear_flush_notify(vma, haddr, fe->pmd);
+	pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);
 	/* leave pmd empty until pte is filled */
 
-	pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, fe->pmd);
+	pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, vmf->pmd);
 	pmd_populate(vma->vm_mm, &_pmd, pgtable);
 
 	for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
@@ -969,20 +969,20 @@ static int do_huge_pmd_wp_page_fallback(struct fault_env *fe, pmd_t orig_pmd,
 		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
 		memcg = (void *)page_private(pages[i]);
 		set_page_private(pages[i], 0);
-		page_add_new_anon_rmap(pages[i], fe->vma, haddr, false);
+		page_add_new_anon_rmap(pages[i], vmf->vma, haddr, false);
 		mem_cgroup_commit_charge(pages[i], memcg, false, false);
 		lru_cache_add_active_or_unevictable(pages[i], vma);
-		fe->pte = pte_offset_map(&_pmd, haddr);
-		VM_BUG_ON(!pte_none(*fe->pte));
-		set_pte_at(vma->vm_mm, haddr, fe->pte, entry);
-		pte_unmap(fe->pte);
+		vmf->pte = pte_offset_map(&_pmd, haddr);
+		VM_BUG_ON(!pte_none(*vmf->pte));
+		set_pte_at(vma->vm_mm, haddr, vmf->pte, entry);
+		pte_unmap(vmf->pte);
 	}
 	kfree(pages);
 
 	smp_wmb(); /* make pte visible before pmd */
-	pmd_populate(vma->vm_mm, fe->pmd, pgtable);
+	pmd_populate(vma->vm_mm, vmf->pmd, pgtable);
 	page_remove_rmap(page, true);
-	spin_unlock(fe->ptl);
+	spin_unlock(vmf->ptl);
 
 	mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
 
@@ -993,7 +993,7 @@ out:
 	return ret;
 
 out_free_pages:
-	spin_unlock(fe->ptl);
+	spin_unlock(vmf->ptl);
 	mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
 	for (i = 0; i < HPAGE_PMD_NR; i++) {
 		memcg = (void *)page_private(pages[i]);
@@ -1005,23 +1005,23 @@ out_free_pages:
 	goto out;
 }
 
-int do_huge_pmd_wp_page(struct fault_env *fe, pmd_t orig_pmd)
+int do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
 	struct page *page = NULL, *new_page;
 	struct mem_cgroup *memcg;
-	unsigned long haddr = fe->address & HPAGE_PMD_MASK;
+	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
 	unsigned long mmun_start;	/* For mmu_notifiers */
 	unsigned long mmun_end;		/* For mmu_notifiers */
 	gfp_t huge_gfp;			/* for allocation and charge */
 	int ret = 0;
 
-	fe->ptl = pmd_lockptr(vma->vm_mm, fe->pmd);
+	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
 	VM_BUG_ON_VMA(!vma->anon_vma, vma);
 	if (is_huge_zero_pmd(orig_pmd))
 		goto alloc;
-	spin_lock(fe->ptl);
-	if (unlikely(!pmd_same(*fe->pmd, orig_pmd)))
+	spin_lock(vmf->ptl);
+	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
 		goto out_unlock;
 
 	page = pmd_page(orig_pmd);
@@ -1034,13 +1034,13 @@ int do_huge_pmd_wp_page(struct fault_env *fe, pmd_t orig_pmd)
 		pmd_t entry;
 		entry = pmd_mkyoung(orig_pmd);
 		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
-		if (pmdp_set_access_flags(vma, haddr, fe->pmd, entry,  1))
-			update_mmu_cache_pmd(vma, fe->address, fe->pmd);
+		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry,  1))
+			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
 		ret |= VM_FAULT_WRITE;
 		goto out_unlock;
 	}
 	get_page(page);
-	spin_unlock(fe->ptl);
+	spin_unlock(vmf->ptl);
 alloc:
 	if (transparent_hugepage_enabled(vma) &&
 	    !transparent_hugepage_debug_cow()) {
@@ -1053,12 +1053,12 @@ alloc:
 		prep_transhuge_page(new_page);
 	} else {
 		if (!page) {
-			split_huge_pmd(vma, fe->pmd, fe->address);
+			split_huge_pmd(vma, vmf->pmd, vmf->address);
 			ret |= VM_FAULT_FALLBACK;
 		} else {
-			ret = do_huge_pmd_wp_page_fallback(fe, orig_pmd, page);
+			ret = do_huge_pmd_wp_page_fallback(vmf, orig_pmd, page);
 			if (ret & VM_FAULT_OOM) {
-				split_huge_pmd(vma, fe->pmd, fe->address);
+				split_huge_pmd(vma, vmf->pmd, vmf->address);
 				ret |= VM_FAULT_FALLBACK;
 			}
 			put_page(page);
@@ -1070,7 +1070,7 @@ alloc:
 	if (unlikely(mem_cgroup_try_charge(new_page, vma->vm_mm,
 					huge_gfp, &memcg, true))) {
 		put_page(new_page);
-		split_huge_pmd(vma, fe->pmd, fe->address);
+		split_huge_pmd(vma, vmf->pmd, vmf->address);
 		if (page)
 			put_page(page);
 		ret |= VM_FAULT_FALLBACK;
@@ -1090,11 +1090,11 @@ alloc:
 	mmun_end   = haddr + HPAGE_PMD_SIZE;
 	mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
 
-	spin_lock(fe->ptl);
+	spin_lock(vmf->ptl);
 	if (page)
 		put_page(page);
-	if (unlikely(!pmd_same(*fe->pmd, orig_pmd))) {
-		spin_unlock(fe->ptl);
+	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
+		spin_unlock(vmf->ptl);
 		mem_cgroup_cancel_charge(new_page, memcg, true);
 		put_page(new_page);
 		goto out_mn;
@@ -1102,12 +1102,12 @@ alloc:
 		pmd_t entry;
 		entry = mk_huge_pmd(new_page, vma->vm_page_prot);
 		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
-		pmdp_huge_clear_flush_notify(vma, haddr, fe->pmd);
+		pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);
 		page_add_new_anon_rmap(new_page, vma, haddr, true);
 		mem_cgroup_commit_charge(new_page, memcg, false, true);
 		lru_cache_add_active_or_unevictable(new_page, vma);
-		set_pmd_at(vma->vm_mm, haddr, fe->pmd, entry);
-		update_mmu_cache_pmd(vma, fe->address, fe->pmd);
+		set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
+		update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
 		if (!page) {
 			add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
 		} else {
@@ -1117,13 +1117,13 @@ alloc:
 		}
 		ret |= VM_FAULT_WRITE;
 	}
-	spin_unlock(fe->ptl);
+	spin_unlock(vmf->ptl);
 out_mn:
 	mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
 out:
 	return ret;
 out_unlock:
-	spin_unlock(fe->ptl);
+	spin_unlock(vmf->ptl);
 	return ret;
 }
 
@@ -1196,12 +1196,12 @@ out:
 }
 
 /* NUMA hinting page fault entry point for trans huge pmds */
-int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t pmd)
+int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
 	struct anon_vma *anon_vma = NULL;
 	struct page *page;
-	unsigned long haddr = fe->address & HPAGE_PMD_MASK;
+	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
 	int page_nid = -1, this_nid = numa_node_id();
 	int target_nid, last_cpupid = -1;
 	bool page_locked;
@@ -1209,8 +1209,8 @@ int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t pmd)
 	bool was_writable;
 	int flags = 0;
 
-	fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
-	if (unlikely(!pmd_same(pmd, *fe->pmd)))
+	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
+	if (unlikely(!pmd_same(pmd, *vmf->pmd)))
 		goto out_unlock;
 
 	/*
@@ -1218,9 +1218,9 @@ int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t pmd)
 	 * without disrupting NUMA hinting information. Do not relock and
 	 * check_same as the page may no longer be mapped.
 	 */
-	if (unlikely(pmd_trans_migrating(*fe->pmd))) {
-		page = pmd_page(*fe->pmd);
-		spin_unlock(fe->ptl);
+	if (unlikely(pmd_trans_migrating(*vmf->pmd))) {
+		page = pmd_page(*vmf->pmd);
+		spin_unlock(vmf->ptl);
 		wait_on_page_locked(page);
 		goto out;
 	}
@@ -1253,7 +1253,7 @@ int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t pmd)
 
 	/* Migration could have started since the pmd_trans_migrating check */
 	if (!page_locked) {
-		spin_unlock(fe->ptl);
+		spin_unlock(vmf->ptl);
 		wait_on_page_locked(page);
 		page_nid = -1;
 		goto out;
@@ -1264,12 +1264,12 @@ int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t pmd)
 	 * to serialises splits
 	 */
 	get_page(page);
-	spin_unlock(fe->ptl);
+	spin_unlock(vmf->ptl);
 	anon_vma = page_lock_anon_vma_read(page);
 
 	/* Confirm the PMD did not change while page_table_lock was released */
-	spin_lock(fe->ptl);
-	if (unlikely(!pmd_same(pmd, *fe->pmd))) {
+	spin_lock(vmf->ptl);
+	if (unlikely(!pmd_same(pmd, *vmf->pmd))) {
 		unlock_page(page);
 		put_page(page);
 		page_nid = -1;
@@ -1287,9 +1287,9 @@ int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t pmd)
 	 * Migrate the THP to the requested node, returns with page unlocked
 	 * and access rights restored.
 	 */
-	spin_unlock(fe->ptl);
+	spin_unlock(vmf->ptl);
 	migrated = migrate_misplaced_transhuge_page(vma->vm_mm, vma,
-				fe->pmd, pmd, fe->address, page, target_nid);
+				vmf->pmd, pmd, vmf->address, page, target_nid);
 	if (migrated) {
 		flags |= TNF_MIGRATED;
 		page_nid = target_nid;
@@ -1304,18 +1304,19 @@ clear_pmdnuma:
 	pmd = pmd_mkyoung(pmd);
 	if (was_writable)
 		pmd = pmd_mkwrite(pmd);
-	set_pmd_at(vma->vm_mm, haddr, fe->pmd, pmd);
-	update_mmu_cache_pmd(vma, fe->address, fe->pmd);
+	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);
+	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
 	unlock_page(page);
 out_unlock:
-	spin_unlock(fe->ptl);
+	spin_unlock(vmf->ptl);
 
 out:
 	if (anon_vma)
 		page_unlock_anon_vma_read(anon_vma);
 
 	if (page_nid != -1)
-		task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, fe->flags);
+		task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR,
+				vmf->flags);
 
 	return 0;
 }
diff --git a/mm/init-mm.c b/mm/init-mm.c
index a56a851908d2..975e49f00f34 100644
--- a/mm/init-mm.c
+++ b/mm/init-mm.c
@@ -6,6 +6,7 @@
 #include <linux/cpumask.h>
 
 #include <linux/atomic.h>
+#include <linux/user_namespace.h>
 #include <asm/pgtable.h>
 #include <asm/mmu.h>
 
@@ -21,5 +22,6 @@ struct mm_struct init_mm = {
 	.mmap_sem	= __RWSEM_INITIALIZER(init_mm.mmap_sem),
 	.page_table_lock =  __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
 	.mmlist		= LIST_HEAD_INIT(init_mm.mmlist),
+	.user_ns	= &init_user_ns,
 	INIT_MM_CONTEXT(init_mm)
 };
diff --git a/mm/internal.h b/mm/internal.h
index 537ac9951f5f..44d68895a9b9 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -36,7 +36,7 @@
 /* Do not use these with a slab allocator */
 #define GFP_SLAB_BUG_MASK (__GFP_DMA32|__GFP_HIGHMEM|~__GFP_BITS_MASK)
 
-int do_swap_page(struct fault_env *fe, pte_t orig_pte);
+int do_swap_page(struct vm_fault *vmf);
 
 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
 		unsigned long floor, unsigned long ceiling);
diff --git a/mm/kasan/kasan.c b/mm/kasan/kasan.c
index 0e9505f66ec1..b2a0cff2bb35 100644
--- a/mm/kasan/kasan.c
+++ b/mm/kasan/kasan.c
@@ -80,7 +80,14 @@ void kasan_unpoison_task_stack(struct task_struct *task)
 /* Unpoison the stack for the current task beyond a watermark sp value. */
 asmlinkage void kasan_unpoison_task_stack_below(const void *watermark)
 {
-	__kasan_unpoison_stack(current, watermark);
+	/*
+	 * Calculate the task stack base address.  Avoid using 'current'
+	 * because this function is called by early resume code which hasn't
+	 * yet set up the percpu register (%gs).
+	 */
+	void *base = (void *)((unsigned long)watermark & ~(THREAD_SIZE - 1));
+
+	kasan_unpoison_shadow(base, watermark - base);
 }
 
 /*
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 09460955e818..e32389a97030 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -875,13 +875,13 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm,
 					unsigned long address, pmd_t *pmd,
 					int referenced)
 {
-	pte_t pteval;
 	int swapped_in = 0, ret = 0;
-	struct fault_env fe = {
+	struct vm_fault vmf = {
 		.vma = vma,
 		.address = address,
 		.flags = FAULT_FLAG_ALLOW_RETRY,
 		.pmd = pmd,
+		.pgoff = linear_page_index(vma, address),
 	};
 
 	/* we only decide to swapin, if there is enough young ptes */
@@ -889,19 +889,19 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm,
 		trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
 		return false;
 	}
-	fe.pte = pte_offset_map(pmd, address);
-	for (; fe.address < address + HPAGE_PMD_NR*PAGE_SIZE;
-			fe.pte++, fe.address += PAGE_SIZE) {
-		pteval = *fe.pte;
-		if (!is_swap_pte(pteval))
+	vmf.pte = pte_offset_map(pmd, address);
+	for (; vmf.address < address + HPAGE_PMD_NR*PAGE_SIZE;
+			vmf.pte++, vmf.address += PAGE_SIZE) {
+		vmf.orig_pte = *vmf.pte;
+		if (!is_swap_pte(vmf.orig_pte))
 			continue;
 		swapped_in++;
-		ret = do_swap_page(&fe, pteval);
+		ret = do_swap_page(&vmf);
 
 		/* do_swap_page returns VM_FAULT_RETRY with released mmap_sem */
 		if (ret & VM_FAULT_RETRY) {
 			down_read(&mm->mmap_sem);
-			if (hugepage_vma_revalidate(mm, address, &fe.vma)) {
+			if (hugepage_vma_revalidate(mm, address, &vmf.vma)) {
 				/* vma is no longer available, don't continue to swapin */
 				trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0);
 				return false;
@@ -915,10 +915,10 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm,
 			return false;
 		}
 		/* pte is unmapped now, we need to map it */
-		fe.pte = pte_offset_map(pmd, fe.address);
+		vmf.pte = pte_offset_map(pmd, vmf.address);
 	}
-	fe.pte--;
-	pte_unmap(fe.pte);
+	vmf.pte--;
+	pte_unmap(vmf.pte);
 	trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1);
 	return true;
 }
@@ -1446,7 +1446,7 @@ static void collapse_shmem(struct mm_struct *mm,
 		radix_tree_replace_slot(&mapping->page_tree, slot,
 				new_page + (index % HPAGE_PMD_NR));
 
-		slot = radix_tree_iter_next(&iter);
+		slot = radix_tree_iter_resume(slot, &iter);
 		index++;
 		continue;
 out_lru:
@@ -1546,7 +1546,6 @@ tree_unlocked:
 				/* Put holes back where they were */
 				radix_tree_delete(&mapping->page_tree,
 						  iter.index);
-				slot = radix_tree_iter_next(&iter);
 				continue;
 			}
 
@@ -1557,11 +1556,11 @@ tree_unlocked:
 			page_ref_unfreeze(page, 2);
 			radix_tree_replace_slot(&mapping->page_tree,
 						slot, page);
+			slot = radix_tree_iter_resume(slot, &iter);
 			spin_unlock_irq(&mapping->tree_lock);
 			putback_lru_page(page);
 			unlock_page(page);
 			spin_lock_irq(&mapping->tree_lock);
-			slot = radix_tree_iter_next(&iter);
 		}
 		VM_BUG_ON(nr_none);
 		spin_unlock_irq(&mapping->tree_lock);
@@ -1641,8 +1640,8 @@ static void khugepaged_scan_shmem(struct mm_struct *mm,
 		present++;
 
 		if (need_resched()) {
+			slot = radix_tree_iter_resume(slot, &iter);
 			cond_resched_rcu();
-			slot = radix_tree_iter_next(&iter);
 		}
 	}
 	rcu_read_unlock();
diff --git a/mm/memory.c b/mm/memory.c
index 32e9b7aec366..455c3e628d52 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2034,20 +2034,17 @@ static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
  *
  * We do this without the lock held, so that it can sleep if it needs to.
  */
-static int do_page_mkwrite(struct vm_area_struct *vma, struct page *page,
-	       unsigned long address)
+static int do_page_mkwrite(struct vm_fault *vmf)
 {
-	struct vm_fault vmf;
 	int ret;
+	struct page *page = vmf->page;
+	unsigned int old_flags = vmf->flags;
 
-	vmf.virtual_address = (void __user *)(address & PAGE_MASK);
-	vmf.pgoff = page->index;
-	vmf.flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
-	vmf.gfp_mask = __get_fault_gfp_mask(vma);
-	vmf.page = page;
-	vmf.cow_page = NULL;
+	vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
 
-	ret = vma->vm_ops->page_mkwrite(vma, &vmf);
+	ret = vmf->vma->vm_ops->page_mkwrite(vmf->vma, vmf);
+	/* Restore original flags so that caller is not surprised */
+	vmf->flags = old_flags;
 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
 		return ret;
 	if (unlikely(!(ret & VM_FAULT_LOCKED))) {
@@ -2063,6 +2060,41 @@ static int do_page_mkwrite(struct vm_area_struct *vma, struct page *page,
 }
 
 /*
+ * Handle dirtying of a page in shared file mapping on a write fault.
+ *
+ * The function expects the page to be locked and unlocks it.
+ */
+static void fault_dirty_shared_page(struct vm_area_struct *vma,
+				    struct page *page)
+{
+	struct address_space *mapping;
+	bool dirtied;
+	bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
+
+	dirtied = set_page_dirty(page);
+	VM_BUG_ON_PAGE(PageAnon(page), page);
+	/*
+	 * Take a local copy of the address_space - page.mapping may be zeroed
+	 * by truncate after unlock_page().   The address_space itself remains
+	 * pinned by vma->vm_file's reference.  We rely on unlock_page()'s
+	 * release semantics to prevent the compiler from undoing this copying.
+	 */
+	mapping = page_rmapping(page);
+	unlock_page(page);
+
+	if ((dirtied || page_mkwrite) && mapping) {
+		/*
+		 * Some device drivers do not set page.mapping
+		 * but still dirty their pages
+		 */
+		balance_dirty_pages_ratelimited(mapping);
+	}
+
+	if (!page_mkwrite)
+		file_update_time(vma->vm_file);
+}
+
+/*
  * Handle write page faults for pages that can be reused in the current vma
  *
  * This can happen either due to the mapping being with the VM_SHARED flag,
@@ -2070,11 +2102,11 @@ static int do_page_mkwrite(struct vm_area_struct *vma, struct page *page,
  * case, all we need to do here is to mark the page as writable and update
  * any related book-keeping.
  */
-static inline int wp_page_reuse(struct fault_env *fe, pte_t orig_pte,
-			struct page *page, int page_mkwrite, int dirty_shared)
-	__releases(fe->ptl)
+static inline void wp_page_reuse(struct vm_fault *vmf)
+	__releases(vmf->ptl)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
+	struct page *page = vmf->page;
 	pte_t entry;
 	/*
 	 * Clear the pages cpupid information as the existing
@@ -2084,39 +2116,12 @@ static inline int wp_page_reuse(struct fault_env *fe, pte_t orig_pte,
 	if (page)
 		page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
 
-	flush_cache_page(vma, fe->address, pte_pfn(orig_pte));
-	entry = pte_mkyoung(orig_pte);
+	flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
+	entry = pte_mkyoung(vmf->orig_pte);
 	entry = maybe_mkwrite(pte_mkdirty(entry), vma);
-	if (ptep_set_access_flags(vma, fe->address, fe->pte, entry, 1))
-		update_mmu_cache(vma, fe->address, fe->pte);
-	pte_unmap_unlock(fe->pte, fe->ptl);
-
-	if (dirty_shared) {
-		struct address_space *mapping;
-		int dirtied;
-
-		if (!page_mkwrite)
-			lock_page(page);
-
-		dirtied = set_page_dirty(page);
-		VM_BUG_ON_PAGE(PageAnon(page), page);
-		mapping = page->mapping;
-		unlock_page(page);
-		put_page(page);
-
-		if ((dirtied || page_mkwrite) && mapping) {
-			/*
-			 * Some device drivers do not set page.mapping
-			 * but still dirty their pages
-			 */
-			balance_dirty_pages_ratelimited(mapping);
-		}
-
-		if (!page_mkwrite)
-			file_update_time(vma->vm_file);
-	}
-
-	return VM_FAULT_WRITE;
+	if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
+		update_mmu_cache(vma, vmf->address, vmf->pte);
+	pte_unmap_unlock(vmf->pte, vmf->ptl);
 }
 
 /*
@@ -2135,31 +2140,32 @@ static inline int wp_page_reuse(struct fault_env *fe, pte_t orig_pte,
  *   held to the old page, as well as updating the rmap.
  * - In any case, unlock the PTL and drop the reference we took to the old page.
  */
-static int wp_page_copy(struct fault_env *fe, pte_t orig_pte,
-		struct page *old_page)
+static int wp_page_copy(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
 	struct mm_struct *mm = vma->vm_mm;
+	struct page *old_page = vmf->page;
 	struct page *new_page = NULL;
 	pte_t entry;
 	int page_copied = 0;
-	const unsigned long mmun_start = fe->address & PAGE_MASK;
+	const unsigned long mmun_start = vmf->address & PAGE_MASK;
 	const unsigned long mmun_end = mmun_start + PAGE_SIZE;
 	struct mem_cgroup *memcg;
 
 	if (unlikely(anon_vma_prepare(vma)))
 		goto oom;
 
-	if (is_zero_pfn(pte_pfn(orig_pte))) {
-		new_page = alloc_zeroed_user_highpage_movable(vma, fe->address);
+	if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
+		new_page = alloc_zeroed_user_highpage_movable(vma,
+							      vmf->address);
 		if (!new_page)
 			goto oom;
 	} else {
 		new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
-				fe->address);
+				vmf->address);
 		if (!new_page)
 			goto oom;
-		cow_user_page(new_page, old_page, fe->address, vma);
+		cow_user_page(new_page, old_page, vmf->address, vma);
 	}
 
 	if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg, false))
@@ -2172,8 +2178,8 @@ static int wp_page_copy(struct fault_env *fe, pte_t orig_pte,
 	/*
 	 * Re-check the pte - we dropped the lock
 	 */
-	fe->pte = pte_offset_map_lock(mm, fe->pmd, fe->address, &fe->ptl);
-	if (likely(pte_same(*fe->pte, orig_pte))) {
+	vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
+	if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
 		if (old_page) {
 			if (!PageAnon(old_page)) {
 				dec_mm_counter_fast(mm,
@@ -2183,7 +2189,7 @@ static int wp_page_copy(struct fault_env *fe, pte_t orig_pte,
 		} else {
 			inc_mm_counter_fast(mm, MM_ANONPAGES);
 		}
-		flush_cache_page(vma, fe->address, pte_pfn(orig_pte));
+		flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
 		entry = mk_pte(new_page, vma->vm_page_prot);
 		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
 		/*
@@ -2192,8 +2198,8 @@ static int wp_page_copy(struct fault_env *fe, pte_t orig_pte,
 		 * seen in the presence of one thread doing SMC and another
 		 * thread doing COW.
 		 */
-		ptep_clear_flush_notify(vma, fe->address, fe->pte);
-		page_add_new_anon_rmap(new_page, vma, fe->address, false);
+		ptep_clear_flush_notify(vma, vmf->address, vmf->pte);
+		page_add_new_anon_rmap(new_page, vma, vmf->address, false);
 		mem_cgroup_commit_charge(new_page, memcg, false, false);
 		lru_cache_add_active_or_unevictable(new_page, vma);
 		/*
@@ -2201,8 +2207,8 @@ static int wp_page_copy(struct fault_env *fe, pte_t orig_pte,
 		 * mmu page tables (such as kvm shadow page tables), we want the
 		 * new page to be mapped directly into the secondary page table.
 		 */
-		set_pte_at_notify(mm, fe->address, fe->pte, entry);
-		update_mmu_cache(vma, fe->address, fe->pte);
+		set_pte_at_notify(mm, vmf->address, vmf->pte, entry);
+		update_mmu_cache(vma, vmf->address, vmf->pte);
 		if (old_page) {
 			/*
 			 * Only after switching the pte to the new page may
@@ -2239,7 +2245,7 @@ static int wp_page_copy(struct fault_env *fe, pte_t orig_pte,
 	if (new_page)
 		put_page(new_page);
 
-	pte_unmap_unlock(fe->pte, fe->ptl);
+	pte_unmap_unlock(vmf->pte, vmf->ptl);
 	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
 	if (old_page) {
 		/*
@@ -2263,79 +2269,91 @@ oom:
 	return VM_FAULT_OOM;
 }
 
+/**
+ * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
+ *			  writeable once the page is prepared
+ *
+ * @vmf: structure describing the fault
+ *
+ * This function handles all that is needed to finish a write page fault in a
+ * shared mapping due to PTE being read-only once the mapped page is prepared.
+ * It handles locking of PTE and modifying it. The function returns
+ * VM_FAULT_WRITE on success, 0 when PTE got changed before we acquired PTE
+ * lock.
+ *
+ * The function expects the page to be locked or other protection against
+ * concurrent faults / writeback (such as DAX radix tree locks).
+ */
+int finish_mkwrite_fault(struct vm_fault *vmf)
+{
+	WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
+	vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
+				       &vmf->ptl);
+	/*
+	 * We might have raced with another page fault while we released the
+	 * pte_offset_map_lock.
+	 */
+	if (!pte_same(*vmf->pte, vmf->orig_pte)) {
+		pte_unmap_unlock(vmf->pte, vmf->ptl);
+		return VM_FAULT_NOPAGE;
+	}
+	wp_page_reuse(vmf);
+	return 0;
+}
+
 /*
  * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
  * mapping
  */
-static int wp_pfn_shared(struct fault_env *fe,  pte_t orig_pte)
+static int wp_pfn_shared(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
 
 	if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
-		struct vm_fault vmf = {
-			.page = NULL,
-			.pgoff = linear_page_index(vma, fe->address),
-			.virtual_address =
-				(void __user *)(fe->address & PAGE_MASK),
-			.flags = FAULT_FLAG_WRITE | FAULT_FLAG_MKWRITE,
-		};
 		int ret;
 
-		pte_unmap_unlock(fe->pte, fe->ptl);
-		ret = vma->vm_ops->pfn_mkwrite(vma, &vmf);
-		if (ret & VM_FAULT_ERROR)
+		pte_unmap_unlock(vmf->pte, vmf->ptl);
+		vmf->flags |= FAULT_FLAG_MKWRITE;
+		ret = vma->vm_ops->pfn_mkwrite(vma, vmf);
+		if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
 			return ret;
-		fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,
-				&fe->ptl);
-		/*
-		 * We might have raced with another page fault while we
-		 * released the pte_offset_map_lock.
-		 */
-		if (!pte_same(*fe->pte, orig_pte)) {
-			pte_unmap_unlock(fe->pte, fe->ptl);
-			return 0;
-		}
+		return finish_mkwrite_fault(vmf);
 	}
-	return wp_page_reuse(fe, orig_pte, NULL, 0, 0);
+	wp_page_reuse(vmf);
+	return VM_FAULT_WRITE;
 }
 
-static int wp_page_shared(struct fault_env *fe, pte_t orig_pte,
-		struct page *old_page)
-	__releases(fe->ptl)
+static int wp_page_shared(struct vm_fault *vmf)
+	__releases(vmf->ptl)
 {
-	struct vm_area_struct *vma = fe->vma;
-	int page_mkwrite = 0;
+	struct vm_area_struct *vma = vmf->vma;
 
-	get_page(old_page);
+	get_page(vmf->page);
 
 	if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
 		int tmp;
 
-		pte_unmap_unlock(fe->pte, fe->ptl);
-		tmp = do_page_mkwrite(vma, old_page, fe->address);
+		pte_unmap_unlock(vmf->pte, vmf->ptl);
+		tmp = do_page_mkwrite(vmf);
 		if (unlikely(!tmp || (tmp &
 				      (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
-			put_page(old_page);
+			put_page(vmf->page);
 			return tmp;
 		}
-		/*
-		 * Since we dropped the lock we need to revalidate
-		 * the PTE as someone else may have changed it.  If
-		 * they did, we just return, as we can count on the
-		 * MMU to tell us if they didn't also make it writable.
-		 */
-		fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,
-						 &fe->ptl);
-		if (!pte_same(*fe->pte, orig_pte)) {
-			unlock_page(old_page);
-			pte_unmap_unlock(fe->pte, fe->ptl);
-			put_page(old_page);
-			return 0;
+		tmp = finish_mkwrite_fault(vmf);
+		if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
+			unlock_page(vmf->page);
+			put_page(vmf->page);
+			return tmp;
 		}
-		page_mkwrite = 1;
+	} else {
+		wp_page_reuse(vmf);
+		lock_page(vmf->page);
 	}
+	fault_dirty_shared_page(vma, vmf->page);
+	put_page(vmf->page);
 
-	return wp_page_reuse(fe, orig_pte, old_page, page_mkwrite, 1);
+	return VM_FAULT_WRITE;
 }
 
 /*
@@ -2356,14 +2374,13 @@ static int wp_page_shared(struct fault_env *fe, pte_t orig_pte,
  * but allow concurrent faults), with pte both mapped and locked.
  * We return with mmap_sem still held, but pte unmapped and unlocked.
  */
-static int do_wp_page(struct fault_env *fe, pte_t orig_pte)
-	__releases(fe->ptl)
+static int do_wp_page(struct vm_fault *vmf)
+	__releases(vmf->ptl)
 {
-	struct vm_area_struct *vma = fe->vma;
-	struct page *old_page;
+	struct vm_area_struct *vma = vmf->vma;
 
-	old_page = vm_normal_page(vma, fe->address, orig_pte);
-	if (!old_page) {
+	vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
+	if (!vmf->page) {
 		/*
 		 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
 		 * VM_PFNMAP VMA.
@@ -2373,33 +2390,33 @@ static int do_wp_page(struct fault_env *fe, pte_t orig_pte)
 		 */
 		if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
 				     (VM_WRITE|VM_SHARED))
-			return wp_pfn_shared(fe, orig_pte);
+			return wp_pfn_shared(vmf);
 
-		pte_unmap_unlock(fe->pte, fe->ptl);
-		return wp_page_copy(fe, orig_pte, old_page);
+		pte_unmap_unlock(vmf->pte, vmf->ptl);
+		return wp_page_copy(vmf);
 	}
 
 	/*
 	 * Take out anonymous pages first, anonymous shared vmas are
 	 * not dirty accountable.
 	 */
-	if (PageAnon(old_page) && !PageKsm(old_page)) {
+	if (PageAnon(vmf->page) && !PageKsm(vmf->page)) {
 		int total_mapcount;
-		if (!trylock_page(old_page)) {
-			get_page(old_page);
-			pte_unmap_unlock(fe->pte, fe->ptl);
-			lock_page(old_page);
-			fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd,
-					fe->address, &fe->ptl);
-			if (!pte_same(*fe->pte, orig_pte)) {
-				unlock_page(old_page);
-				pte_unmap_unlock(fe->pte, fe->ptl);
-				put_page(old_page);
+		if (!trylock_page(vmf->page)) {
+			get_page(vmf->page);
+			pte_unmap_unlock(vmf->pte, vmf->ptl);
+			lock_page(vmf->page);
+			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
+					vmf->address, &vmf->ptl);
+			if (!pte_same(*vmf->pte, vmf->orig_pte)) {
+				unlock_page(vmf->page);
+				pte_unmap_unlock(vmf->pte, vmf->ptl);
+				put_page(vmf->page);
 				return 0;
 			}
-			put_page(old_page);
+			put_page(vmf->page);
 		}
-		if (reuse_swap_page(old_page, &total_mapcount)) {
+		if (reuse_swap_page(vmf->page, &total_mapcount)) {
 			if (total_mapcount == 1) {
 				/*
 				 * The page is all ours. Move it to
@@ -2408,24 +2425,25 @@ static int do_wp_page(struct fault_env *fe, pte_t orig_pte)
 				 * Protected against the rmap code by
 				 * the page lock.
 				 */
-				page_move_anon_rmap(old_page, vma);
+				page_move_anon_rmap(vmf->page, vma);
 			}
-			unlock_page(old_page);
-			return wp_page_reuse(fe, orig_pte, old_page, 0, 0);
+			unlock_page(vmf->page);
+			wp_page_reuse(vmf);
+			return VM_FAULT_WRITE;
 		}
-		unlock_page(old_page);
+		unlock_page(vmf->page);
 	} else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
 					(VM_WRITE|VM_SHARED))) {
-		return wp_page_shared(fe, orig_pte, old_page);
+		return wp_page_shared(vmf);
 	}
 
 	/*
 	 * Ok, we need to copy. Oh, well..
 	 */
-	get_page(old_page);
+	get_page(vmf->page);
 
-	pte_unmap_unlock(fe->pte, fe->ptl);
-	return wp_page_copy(fe, orig_pte, old_page);
+	pte_unmap_unlock(vmf->pte, vmf->ptl);
+	return wp_page_copy(vmf);
 }
 
 static void unmap_mapping_range_vma(struct vm_area_struct *vma,
@@ -2513,9 +2531,9 @@ EXPORT_SYMBOL(unmap_mapping_range);
  * We return with the mmap_sem locked or unlocked in the same cases
  * as does filemap_fault().
  */
-int do_swap_page(struct fault_env *fe, pte_t orig_pte)
+int do_swap_page(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
 	struct page *page, *swapcache;
 	struct mem_cgroup *memcg;
 	swp_entry_t entry;
@@ -2524,17 +2542,18 @@ int do_swap_page(struct fault_env *fe, pte_t orig_pte)
 	int exclusive = 0;
 	int ret = 0;
 
-	if (!pte_unmap_same(vma->vm_mm, fe->pmd, fe->pte, orig_pte))
+	if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, vmf->orig_pte))
 		goto out;
 
-	entry = pte_to_swp_entry(orig_pte);
+	entry = pte_to_swp_entry(vmf->orig_pte);
 	if (unlikely(non_swap_entry(entry))) {
 		if (is_migration_entry(entry)) {
-			migration_entry_wait(vma->vm_mm, fe->pmd, fe->address);
+			migration_entry_wait(vma->vm_mm, vmf->pmd,
+					     vmf->address);
 		} else if (is_hwpoison_entry(entry)) {
 			ret = VM_FAULT_HWPOISON;
 		} else {
-			print_bad_pte(vma, fe->address, orig_pte, NULL);
+			print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
 			ret = VM_FAULT_SIGBUS;
 		}
 		goto out;
@@ -2542,16 +2561,16 @@ int do_swap_page(struct fault_env *fe, pte_t orig_pte)
 	delayacct_set_flag(DELAYACCT_PF_SWAPIN);
 	page = lookup_swap_cache(entry);
 	if (!page) {
-		page = swapin_readahead(entry,
-					GFP_HIGHUSER_MOVABLE, vma, fe->address);
+		page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, vma,
+					vmf->address);
 		if (!page) {
 			/*
 			 * Back out if somebody else faulted in this pte
 			 * while we released the pte lock.
 			 */
-			fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd,
-					fe->address, &fe->ptl);
-			if (likely(pte_same(*fe->pte, orig_pte)))
+			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
+					vmf->address, &vmf->ptl);
+			if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
 				ret = VM_FAULT_OOM;
 			delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
 			goto unlock;
@@ -2573,7 +2592,7 @@ int do_swap_page(struct fault_env *fe, pte_t orig_pte)
 	}
 
 	swapcache = page;
-	locked = lock_page_or_retry(page, vma->vm_mm, fe->flags);
+	locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags);
 
 	delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
 	if (!locked) {
@@ -2590,7 +2609,7 @@ int do_swap_page(struct fault_env *fe, pte_t orig_pte)
 	if (unlikely(!PageSwapCache(page) || page_private(page) != entry.val))
 		goto out_page;
 
-	page = ksm_might_need_to_copy(page, vma, fe->address);
+	page = ksm_might_need_to_copy(page, vma, vmf->address);
 	if (unlikely(!page)) {
 		ret = VM_FAULT_OOM;
 		page = swapcache;
@@ -2606,9 +2625,9 @@ int do_swap_page(struct fault_env *fe, pte_t orig_pte)
 	/*
 	 * Back out if somebody else already faulted in this pte.
 	 */
-	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,
-			&fe->ptl);
-	if (unlikely(!pte_same(*fe->pte, orig_pte)))
+	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
+			&vmf->ptl);
+	if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
 		goto out_nomap;
 
 	if (unlikely(!PageUptodate(page))) {
@@ -2629,22 +2648,23 @@ int do_swap_page(struct fault_env *fe, pte_t orig_pte)
 	inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
 	dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS);
 	pte = mk_pte(page, vma->vm_page_prot);
-	if ((fe->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) {
+	if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) {
 		pte = maybe_mkwrite(pte_mkdirty(pte), vma);
-		fe->flags &= ~FAULT_FLAG_WRITE;
+		vmf->flags &= ~FAULT_FLAG_WRITE;
 		ret |= VM_FAULT_WRITE;
 		exclusive = RMAP_EXCLUSIVE;
 	}
 	flush_icache_page(vma, page);
-	if (pte_swp_soft_dirty(orig_pte))
+	if (pte_swp_soft_dirty(vmf->orig_pte))
 		pte = pte_mksoft_dirty(pte);
-	set_pte_at(vma->vm_mm, fe->address, fe->pte, pte);
+	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
+	vmf->orig_pte = pte;
 	if (page == swapcache) {
-		do_page_add_anon_rmap(page, vma, fe->address, exclusive);
+		do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
 		mem_cgroup_commit_charge(page, memcg, true, false);
 		activate_page(page);
 	} else { /* ksm created a completely new copy */
-		page_add_new_anon_rmap(page, vma, fe->address, false);
+		page_add_new_anon_rmap(page, vma, vmf->address, false);
 		mem_cgroup_commit_charge(page, memcg, false, false);
 		lru_cache_add_active_or_unevictable(page, vma);
 	}
@@ -2667,22 +2687,22 @@ int do_swap_page(struct fault_env *fe, pte_t orig_pte)
 		put_page(swapcache);
 	}
 
-	if (fe->flags & FAULT_FLAG_WRITE) {
-		ret |= do_wp_page(fe, pte);
+	if (vmf->flags & FAULT_FLAG_WRITE) {
+		ret |= do_wp_page(vmf);
 		if (ret & VM_FAULT_ERROR)
 			ret &= VM_FAULT_ERROR;
 		goto out;
 	}
 
 	/* No need to invalidate - it was non-present before */
-	update_mmu_cache(vma, fe->address, fe->pte);
+	update_mmu_cache(vma, vmf->address, vmf->pte);
 unlock:
-	pte_unmap_unlock(fe->pte, fe->ptl);
+	pte_unmap_unlock(vmf->pte, vmf->ptl);
 out:
 	return ret;
 out_nomap:
 	mem_cgroup_cancel_charge(page, memcg, false);
-	pte_unmap_unlock(fe->pte, fe->ptl);
+	pte_unmap_unlock(vmf->pte, vmf->ptl);
 out_page:
 	unlock_page(page);
 out_release:
@@ -2733,9 +2753,9 @@ static inline int check_stack_guard_page(struct vm_area_struct *vma, unsigned lo
  * but allow concurrent faults), and pte mapped but not yet locked.
  * We return with mmap_sem still held, but pte unmapped and unlocked.
  */
-static int do_anonymous_page(struct fault_env *fe)
+static int do_anonymous_page(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
 	struct mem_cgroup *memcg;
 	struct page *page;
 	pte_t entry;
@@ -2745,7 +2765,7 @@ static int do_anonymous_page(struct fault_env *fe)
 		return VM_FAULT_SIGBUS;
 
 	/* Check if we need to add a guard page to the stack */
-	if (check_stack_guard_page(vma, fe->address) < 0)
+	if (check_stack_guard_page(vma, vmf->address) < 0)
 		return VM_FAULT_SIGSEGV;
 
 	/*
@@ -2758,26 +2778,26 @@ static int do_anonymous_page(struct fault_env *fe)
 	 *
 	 * Here we only have down_read(mmap_sem).
 	 */
-	if (pte_alloc(vma->vm_mm, fe->pmd, fe->address))
+	if (pte_alloc(vma->vm_mm, vmf->pmd, vmf->address))
 		return VM_FAULT_OOM;
 
 	/* See the comment in pte_alloc_one_map() */
-	if (unlikely(pmd_trans_unstable(fe->pmd)))
+	if (unlikely(pmd_trans_unstable(vmf->pmd)))
 		return 0;
 
 	/* Use the zero-page for reads */
-	if (!(fe->flags & FAULT_FLAG_WRITE) &&
+	if (!(vmf->flags & FAULT_FLAG_WRITE) &&
 			!mm_forbids_zeropage(vma->vm_mm)) {
-		entry = pte_mkspecial(pfn_pte(my_zero_pfn(fe->address),
+		entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
 						vma->vm_page_prot));
-		fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,
-				&fe->ptl);
-		if (!pte_none(*fe->pte))
+		vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
+				vmf->address, &vmf->ptl);
+		if (!pte_none(*vmf->pte))
 			goto unlock;
 		/* Deliver the page fault to userland, check inside PT lock */
 		if (userfaultfd_missing(vma)) {
-			pte_unmap_unlock(fe->pte, fe->ptl);
-			return handle_userfault(fe, VM_UFFD_MISSING);
+			pte_unmap_unlock(vmf->pte, vmf->ptl);
+			return handle_userfault(vmf, VM_UFFD_MISSING);
 		}
 		goto setpte;
 	}
@@ -2785,7 +2805,7 @@ static int do_anonymous_page(struct fault_env *fe)
 	/* Allocate our own private page. */
 	if (unlikely(anon_vma_prepare(vma)))
 		goto oom;
-	page = alloc_zeroed_user_highpage_movable(vma, fe->address);
+	page = alloc_zeroed_user_highpage_movable(vma, vmf->address);
 	if (!page)
 		goto oom;
 
@@ -2803,30 +2823,30 @@ static int do_anonymous_page(struct fault_env *fe)
 	if (vma->vm_flags & VM_WRITE)
 		entry = pte_mkwrite(pte_mkdirty(entry));
 
-	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,
-			&fe->ptl);
-	if (!pte_none(*fe->pte))
+	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
+			&vmf->ptl);
+	if (!pte_none(*vmf->pte))
 		goto release;
 
 	/* Deliver the page fault to userland, check inside PT lock */
 	if (userfaultfd_missing(vma)) {
-		pte_unmap_unlock(fe->pte, fe->ptl);
+		pte_unmap_unlock(vmf->pte, vmf->ptl);
 		mem_cgroup_cancel_charge(page, memcg, false);
 		put_page(page);
-		return handle_userfault(fe, VM_UFFD_MISSING);
+		return handle_userfault(vmf, VM_UFFD_MISSING);
 	}
 
 	inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
-	page_add_new_anon_rmap(page, vma, fe->address, false);
+	page_add_new_anon_rmap(page, vma, vmf->address, false);
 	mem_cgroup_commit_charge(page, memcg, false, false);
 	lru_cache_add_active_or_unevictable(page, vma);
 setpte:
-	set_pte_at(vma->vm_mm, fe->address, fe->pte, entry);
+	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
 
 	/* No need to invalidate - it was non-present before */
-	update_mmu_cache(vma, fe->address, fe->pte);
+	update_mmu_cache(vma, vmf->address, vmf->pte);
 unlock:
-	pte_unmap_unlock(fe->pte, fe->ptl);
+	pte_unmap_unlock(vmf->pte, vmf->ptl);
 	return 0;
 release:
 	mem_cgroup_cancel_charge(page, memcg, false);
@@ -2843,62 +2863,50 @@ oom:
  * released depending on flags and vma->vm_ops->fault() return value.
  * See filemap_fault() and __lock_page_retry().
  */
-static int __do_fault(struct fault_env *fe, pgoff_t pgoff,
-		struct page *cow_page, struct page **page, void **entry)
+static int __do_fault(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
-	struct vm_fault vmf;
+	struct vm_area_struct *vma = vmf->vma;
 	int ret;
 
-	vmf.virtual_address = (void __user *)(fe->address & PAGE_MASK);
-	vmf.pgoff = pgoff;
-	vmf.flags = fe->flags;
-	vmf.page = NULL;
-	vmf.gfp_mask = __get_fault_gfp_mask(vma);
-	vmf.cow_page = cow_page;
-
-	ret = vma->vm_ops->fault(vma, &vmf);
-	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
-		return ret;
-	if (ret & VM_FAULT_DAX_LOCKED) {
-		*entry = vmf.entry;
+	ret = vma->vm_ops->fault(vma, vmf);
+	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
+			    VM_FAULT_DONE_COW)))
 		return ret;
-	}
 
-	if (unlikely(PageHWPoison(vmf.page))) {
+	if (unlikely(PageHWPoison(vmf->page))) {
 		if (ret & VM_FAULT_LOCKED)
-			unlock_page(vmf.page);
-		put_page(vmf.page);
+			unlock_page(vmf->page);
+		put_page(vmf->page);
+		vmf->page = NULL;
 		return VM_FAULT_HWPOISON;
 	}
 
 	if (unlikely(!(ret & VM_FAULT_LOCKED)))
-		lock_page(vmf.page);
+		lock_page(vmf->page);
 	else
-		VM_BUG_ON_PAGE(!PageLocked(vmf.page), vmf.page);
+		VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
 
-	*page = vmf.page;
 	return ret;
 }
 
-static int pte_alloc_one_map(struct fault_env *fe)
+static int pte_alloc_one_map(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
 
-	if (!pmd_none(*fe->pmd))
+	if (!pmd_none(*vmf->pmd))
 		goto map_pte;
-	if (fe->prealloc_pte) {
-		fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
-		if (unlikely(!pmd_none(*fe->pmd))) {
-			spin_unlock(fe->ptl);
+	if (vmf->prealloc_pte) {
+		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
+		if (unlikely(!pmd_none(*vmf->pmd))) {
+			spin_unlock(vmf->ptl);
 			goto map_pte;
 		}
 
 		atomic_long_inc(&vma->vm_mm->nr_ptes);
-		pmd_populate(vma->vm_mm, fe->pmd, fe->prealloc_pte);
-		spin_unlock(fe->ptl);
-		fe->prealloc_pte = 0;
-	} else if (unlikely(pte_alloc(vma->vm_mm, fe->pmd, fe->address))) {
+		pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
+		spin_unlock(vmf->ptl);
+		vmf->prealloc_pte = 0;
+	} else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd, vmf->address))) {
 		return VM_FAULT_OOM;
 	}
 map_pte:
@@ -2913,11 +2921,11 @@ map_pte:
 	 * through an atomic read in C, which is what pmd_trans_unstable()
 	 * provides.
 	 */
-	if (pmd_trans_unstable(fe->pmd) || pmd_devmap(*fe->pmd))
+	if (pmd_trans_unstable(vmf->pmd) || pmd_devmap(*vmf->pmd))
 		return VM_FAULT_NOPAGE;
 
-	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,
-			&fe->ptl);
+	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
+			&vmf->ptl);
 	return 0;
 }
 
@@ -2935,24 +2943,24 @@ static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
 	return true;
 }
 
-static void deposit_prealloc_pte(struct fault_env *fe)
+static void deposit_prealloc_pte(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
 
-	pgtable_trans_huge_deposit(vma->vm_mm, fe->pmd, fe->prealloc_pte);
+	pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
 	/*
 	 * We are going to consume the prealloc table,
 	 * count that as nr_ptes.
 	 */
 	atomic_long_inc(&vma->vm_mm->nr_ptes);
-	fe->prealloc_pte = 0;
+	vmf->prealloc_pte = 0;
 }
 
-static int do_set_pmd(struct fault_env *fe, struct page *page)
+static int do_set_pmd(struct vm_fault *vmf, struct page *page)
 {
-	struct vm_area_struct *vma = fe->vma;
-	bool write = fe->flags & FAULT_FLAG_WRITE;
-	unsigned long haddr = fe->address & HPAGE_PMD_MASK;
+	struct vm_area_struct *vma = vmf->vma;
+	bool write = vmf->flags & FAULT_FLAG_WRITE;
+	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
 	pmd_t entry;
 	int i, ret;
 
@@ -2966,15 +2974,15 @@ static int do_set_pmd(struct fault_env *fe, struct page *page)
 	 * Archs like ppc64 need additonal space to store information
 	 * related to pte entry. Use the preallocated table for that.
 	 */
-	if (arch_needs_pgtable_deposit() && !fe->prealloc_pte) {
-		fe->prealloc_pte = pte_alloc_one(vma->vm_mm, fe->address);
-		if (!fe->prealloc_pte)
+	if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
+		vmf->prealloc_pte = pte_alloc_one(vma->vm_mm, vmf->address);
+		if (!vmf->prealloc_pte)
 			return VM_FAULT_OOM;
 		smp_wmb(); /* See comment in __pte_alloc() */
 	}
 
-	fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
-	if (unlikely(!pmd_none(*fe->pmd)))
+	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
+	if (unlikely(!pmd_none(*vmf->pmd)))
 		goto out;
 
 	for (i = 0; i < HPAGE_PMD_NR; i++)
@@ -2990,11 +2998,11 @@ static int do_set_pmd(struct fault_env *fe, struct page *page)
 	 * deposit and withdraw with pmd lock held
 	 */
 	if (arch_needs_pgtable_deposit())
-		deposit_prealloc_pte(fe);
+		deposit_prealloc_pte(vmf);
 
-	set_pmd_at(vma->vm_mm, haddr, fe->pmd, entry);
+	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
 
-	update_mmu_cache_pmd(vma, haddr, fe->pmd);
+	update_mmu_cache_pmd(vma, haddr, vmf->pmd);
 
 	/* fault is handled */
 	ret = 0;
@@ -3005,13 +3013,13 @@ out:
 	 * withdraw with pmd lock held.
 	 */
 	if (arch_needs_pgtable_deposit() && ret == VM_FAULT_FALLBACK)
-		fe->prealloc_pte = pgtable_trans_huge_withdraw(vma->vm_mm,
-							       fe->pmd);
-	spin_unlock(fe->ptl);
+		vmf->prealloc_pte = pgtable_trans_huge_withdraw(vma->vm_mm,
+								vmf->pmd);
+	spin_unlock(vmf->ptl);
 	return ret;
 }
 #else
-static int do_set_pmd(struct fault_env *fe, struct page *page)
+static int do_set_pmd(struct vm_fault *vmf, struct page *page)
 {
 	BUILD_BUG();
 	return 0;
@@ -3022,41 +3030,42 @@ static int do_set_pmd(struct fault_env *fe, struct page *page)
  * alloc_set_pte - setup new PTE entry for given page and add reverse page
  * mapping. If needed, the fucntion allocates page table or use pre-allocated.
  *
- * @fe: fault environment
+ * @vmf: fault environment
  * @memcg: memcg to charge page (only for private mappings)
  * @page: page to map
  *
- * Caller must take care of unlocking fe->ptl, if fe->pte is non-NULL on return.
+ * Caller must take care of unlocking vmf->ptl, if vmf->pte is non-NULL on
+ * return.
  *
  * Target users are page handler itself and implementations of
  * vm_ops->map_pages.
  */
-int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg,
+int alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,
 		struct page *page)
 {
-	struct vm_area_struct *vma = fe->vma;
-	bool write = fe->flags & FAULT_FLAG_WRITE;
+	struct vm_area_struct *vma = vmf->vma;
+	bool write = vmf->flags & FAULT_FLAG_WRITE;
 	pte_t entry;
 	int ret;
 
-	if (pmd_none(*fe->pmd) && PageTransCompound(page) &&
+	if (pmd_none(*vmf->pmd) && PageTransCompound(page) &&
 			IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
 		/* THP on COW? */
 		VM_BUG_ON_PAGE(memcg, page);
 
-		ret = do_set_pmd(fe, page);
+		ret = do_set_pmd(vmf, page);
 		if (ret != VM_FAULT_FALLBACK)
 			goto fault_handled;
 	}
 
-	if (!fe->pte) {
-		ret = pte_alloc_one_map(fe);
+	if (!vmf->pte) {
+		ret = pte_alloc_one_map(vmf);
 		if (ret)
 			goto fault_handled;
 	}
 
 	/* Re-check under ptl */
-	if (unlikely(!pte_none(*fe->pte))) {
+	if (unlikely(!pte_none(*vmf->pte))) {
 		ret = VM_FAULT_NOPAGE;
 		goto fault_handled;
 	}
@@ -3068,28 +3077,60 @@ int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg,
 	/* copy-on-write page */
 	if (write && !(vma->vm_flags & VM_SHARED)) {
 		inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
-		page_add_new_anon_rmap(page, vma, fe->address, false);
+		page_add_new_anon_rmap(page, vma, vmf->address, false);
 		mem_cgroup_commit_charge(page, memcg, false, false);
 		lru_cache_add_active_or_unevictable(page, vma);
 	} else {
 		inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
 		page_add_file_rmap(page, false);
 	}
-	set_pte_at(vma->vm_mm, fe->address, fe->pte, entry);
+	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
 
 	/* no need to invalidate: a not-present page won't be cached */
-	update_mmu_cache(vma, fe->address, fe->pte);
+	update_mmu_cache(vma, vmf->address, vmf->pte);
 	ret = 0;
 
 fault_handled:
 	/* preallocated pagetable is unused: free it */
-	if (fe->prealloc_pte) {
-		pte_free(fe->vma->vm_mm, fe->prealloc_pte);
-		fe->prealloc_pte = 0;
+	if (vmf->prealloc_pte) {
+		pte_free(vmf->vma->vm_mm, vmf->prealloc_pte);
+		vmf->prealloc_pte = 0;
 	}
 	return ret;
 }
 
+
+/**
+ * finish_fault - finish page fault once we have prepared the page to fault
+ *
+ * @vmf: structure describing the fault
+ *
+ * This function handles all that is needed to finish a page fault once the
+ * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
+ * given page, adds reverse page mapping, handles memcg charges and LRU
+ * addition. The function returns 0 on success, VM_FAULT_ code in case of
+ * error.
+ *
+ * The function expects the page to be locked and on success it consumes a
+ * reference of a page being mapped (for the PTE which maps it).
+ */
+int finish_fault(struct vm_fault *vmf)
+{
+	struct page *page;
+	int ret;
+
+	/* Did we COW the page? */
+	if ((vmf->flags & FAULT_FLAG_WRITE) &&
+	    !(vmf->vma->vm_flags & VM_SHARED))
+		page = vmf->cow_page;
+	else
+		page = vmf->page;
+	ret = alloc_set_pte(vmf, vmf->memcg, page);
+	if (vmf->pte)
+		pte_unmap_unlock(vmf->pte, vmf->ptl);
+	return ret;
+}
+
 static unsigned long fault_around_bytes __read_mostly =
 	rounddown_pow_of_two(65536);
 
@@ -3154,17 +3195,18 @@ late_initcall(fault_around_debugfs);
  * fault_around_pages() value (and therefore to page order).  This way it's
  * easier to guarantee that we don't cross page table boundaries.
  */
-static int do_fault_around(struct fault_env *fe, pgoff_t start_pgoff)
+static int do_fault_around(struct vm_fault *vmf)
 {
-	unsigned long address = fe->address, nr_pages, mask;
+	unsigned long address = vmf->address, nr_pages, mask;
+	pgoff_t start_pgoff = vmf->pgoff;
 	pgoff_t end_pgoff;
 	int off, ret = 0;
 
 	nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT;
 	mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK;
 
-	fe->address = max(address & mask, fe->vma->vm_start);
-	off = ((address - fe->address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
+	vmf->address = max(address & mask, vmf->vma->vm_start);
+	off = ((address - vmf->address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
 	start_pgoff -= off;
 
 	/*
@@ -3172,45 +3214,45 @@ static int do_fault_around(struct fault_env *fe, pgoff_t start_pgoff)
 	 *  or fault_around_pages() from start_pgoff, depending what is nearest.
 	 */
 	end_pgoff = start_pgoff -
-		((fe->address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) +
+		((vmf->address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) +
 		PTRS_PER_PTE - 1;
-	end_pgoff = min3(end_pgoff, vma_pages(fe->vma) + fe->vma->vm_pgoff - 1,
+	end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1,
 			start_pgoff + nr_pages - 1);
 
-	if (pmd_none(*fe->pmd)) {
-		fe->prealloc_pte = pte_alloc_one(fe->vma->vm_mm, fe->address);
-		if (!fe->prealloc_pte)
+	if (pmd_none(*vmf->pmd)) {
+		vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm,
+						  vmf->address);
+		if (!vmf->prealloc_pte)
 			goto out;
 		smp_wmb(); /* See comment in __pte_alloc() */
 	}
 
-	fe->vma->vm_ops->map_pages(fe, start_pgoff, end_pgoff);
+	vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff);
 
 	/* Huge page is mapped? Page fault is solved */
-	if (pmd_trans_huge(*fe->pmd)) {
+	if (pmd_trans_huge(*vmf->pmd)) {
 		ret = VM_FAULT_NOPAGE;
 		goto out;
 	}
 
 	/* ->map_pages() haven't done anything useful. Cold page cache? */
-	if (!fe->pte)
+	if (!vmf->pte)
 		goto out;
 
 	/* check if the page fault is solved */
-	fe->pte -= (fe->address >> PAGE_SHIFT) - (address >> PAGE_SHIFT);
-	if (!pte_none(*fe->pte))
+	vmf->pte -= (vmf->address >> PAGE_SHIFT) - (address >> PAGE_SHIFT);
+	if (!pte_none(*vmf->pte))
 		ret = VM_FAULT_NOPAGE;
-	pte_unmap_unlock(fe->pte, fe->ptl);
+	pte_unmap_unlock(vmf->pte, vmf->ptl);
 out:
-	fe->address = address;
-	fe->pte = NULL;
+	vmf->address = address;
+	vmf->pte = NULL;
 	return ret;
 }
 
-static int do_read_fault(struct fault_env *fe, pgoff_t pgoff)
+static int do_read_fault(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
-	struct page *fault_page;
+	struct vm_area_struct *vma = vmf->vma;
 	int ret = 0;
 
 	/*
@@ -3219,80 +3261,67 @@ static int do_read_fault(struct fault_env *fe, pgoff_t pgoff)
 	 * something).
 	 */
 	if (vma->vm_ops->map_pages && fault_around_bytes >> PAGE_SHIFT > 1) {
-		ret = do_fault_around(fe, pgoff);
+		ret = do_fault_around(vmf);
 		if (ret)
 			return ret;
 	}
 
-	ret = __do_fault(fe, pgoff, NULL, &fault_page, NULL);
+	ret = __do_fault(vmf);
 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
 		return ret;
 
-	ret |= alloc_set_pte(fe, NULL, fault_page);
-	if (fe->pte)
-		pte_unmap_unlock(fe->pte, fe->ptl);
-	unlock_page(fault_page);
+	ret |= finish_fault(vmf);
+	unlock_page(vmf->page);
 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
-		put_page(fault_page);
+		put_page(vmf->page);
 	return ret;
 }
 
-static int do_cow_fault(struct fault_env *fe, pgoff_t pgoff)
+static int do_cow_fault(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
-	struct page *fault_page, *new_page;
-	void *fault_entry;
-	struct mem_cgroup *memcg;
+	struct vm_area_struct *vma = vmf->vma;
 	int ret;
 
 	if (unlikely(anon_vma_prepare(vma)))
 		return VM_FAULT_OOM;
 
-	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, fe->address);
-	if (!new_page)
+	vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
+	if (!vmf->cow_page)
 		return VM_FAULT_OOM;
 
-	if (mem_cgroup_try_charge(new_page, vma->vm_mm, GFP_KERNEL,
-				&memcg, false)) {
-		put_page(new_page);
+	if (mem_cgroup_try_charge(vmf->cow_page, vma->vm_mm, GFP_KERNEL,
+				&vmf->memcg, false)) {
+		put_page(vmf->cow_page);
 		return VM_FAULT_OOM;
 	}
 
-	ret = __do_fault(fe, pgoff, new_page, &fault_page, &fault_entry);
+	ret = __do_fault(vmf);
 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
 		goto uncharge_out;
+	if (ret & VM_FAULT_DONE_COW)
+		return ret;
 
-	if (!(ret & VM_FAULT_DAX_LOCKED))
-		copy_user_highpage(new_page, fault_page, fe->address, vma);
-	__SetPageUptodate(new_page);
+	copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
+	__SetPageUptodate(vmf->cow_page);
 
-	ret |= alloc_set_pte(fe, memcg, new_page);
-	if (fe->pte)
-		pte_unmap_unlock(fe->pte, fe->ptl);
-	if (!(ret & VM_FAULT_DAX_LOCKED)) {
-		unlock_page(fault_page);
-		put_page(fault_page);
-	} else {
-		dax_unlock_mapping_entry(vma->vm_file->f_mapping, pgoff);
-	}
+	ret |= finish_fault(vmf);
+	unlock_page(vmf->page);
+	put_page(vmf->page);
 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
 		goto uncharge_out;
 	return ret;
 uncharge_out:
-	mem_cgroup_cancel_charge(new_page, memcg, false);
-	put_page(new_page);
+	mem_cgroup_cancel_charge(vmf->cow_page, vmf->memcg, false);
+	put_page(vmf->cow_page);
 	return ret;
 }
 
-static int do_shared_fault(struct fault_env *fe, pgoff_t pgoff)
+static int do_shared_fault(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
-	struct page *fault_page;
-	struct address_space *mapping;
-	int dirtied = 0;
+	struct vm_area_struct *vma = vmf->vma;
 	int ret, tmp;
 
-	ret = __do_fault(fe, pgoff, NULL, &fault_page, NULL);
+	ret = __do_fault(vmf);
 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
 		return ret;
 
@@ -3301,46 +3330,24 @@ static int do_shared_fault(struct fault_env *fe, pgoff_t pgoff)
 	 * about to become writable
 	 */
 	if (vma->vm_ops->page_mkwrite) {
-		unlock_page(fault_page);
-		tmp = do_page_mkwrite(vma, fault_page, fe->address);
+		unlock_page(vmf->page);
+		tmp = do_page_mkwrite(vmf);
 		if (unlikely(!tmp ||
 				(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
-			put_page(fault_page);
+			put_page(vmf->page);
 			return tmp;
 		}
 	}
 
-	ret |= alloc_set_pte(fe, NULL, fault_page);
-	if (fe->pte)
-		pte_unmap_unlock(fe->pte, fe->ptl);
+	ret |= finish_fault(vmf);
 	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
 					VM_FAULT_RETRY))) {
-		unlock_page(fault_page);
-		put_page(fault_page);
+		unlock_page(vmf->page);
+		put_page(vmf->page);
 		return ret;
 	}
 
-	if (set_page_dirty(fault_page))
-		dirtied = 1;
-	/*
-	 * Take a local copy of the address_space - page.mapping may be zeroed
-	 * by truncate after unlock_page().   The address_space itself remains
-	 * pinned by vma->vm_file's reference.  We rely on unlock_page()'s
-	 * release semantics to prevent the compiler from undoing this copying.
-	 */
-	mapping = page_rmapping(fault_page);
-	unlock_page(fault_page);
-	if ((dirtied || vma->vm_ops->page_mkwrite) && mapping) {
-		/*
-		 * Some device drivers do not set page.mapping but still
-		 * dirty their pages
-		 */
-		balance_dirty_pages_ratelimited(mapping);
-	}
-
-	if (!vma->vm_ops->page_mkwrite)
-		file_update_time(vma->vm_file);
-
+	fault_dirty_shared_page(vma, vmf->page);
 	return ret;
 }
 
@@ -3350,19 +3357,18 @@ static int do_shared_fault(struct fault_env *fe, pgoff_t pgoff)
  * The mmap_sem may have been released depending on flags and our
  * return value.  See filemap_fault() and __lock_page_or_retry().
  */
-static int do_fault(struct fault_env *fe)
+static int do_fault(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
-	pgoff_t pgoff = linear_page_index(vma, fe->address);
+	struct vm_area_struct *vma = vmf->vma;
 
 	/* The VMA was not fully populated on mmap() or missing VM_DONTEXPAND */
 	if (!vma->vm_ops->fault)
 		return VM_FAULT_SIGBUS;
-	if (!(fe->flags & FAULT_FLAG_WRITE))
-		return do_read_fault(fe, pgoff);
+	if (!(vmf->flags & FAULT_FLAG_WRITE))
+		return do_read_fault(vmf);
 	if (!(vma->vm_flags & VM_SHARED))
-		return do_cow_fault(fe, pgoff);
-	return do_shared_fault(fe, pgoff);
+		return do_cow_fault(vmf);
+	return do_shared_fault(vmf);
 }
 
 static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
@@ -3380,14 +3386,15 @@ static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
 	return mpol_misplaced(page, vma, addr);
 }
 
-static int do_numa_page(struct fault_env *fe, pte_t pte)
+static int do_numa_page(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
 	struct page *page = NULL;
 	int page_nid = -1;
 	int last_cpupid;
 	int target_nid;
 	bool migrated = false;
+	pte_t pte = vmf->orig_pte;
 	bool was_writable = pte_write(pte);
 	int flags = 0;
 
@@ -3400,10 +3407,10 @@ static int do_numa_page(struct fault_env *fe, pte_t pte)
 	* page table entry is not accessible, so there would be no
 	* concurrent hardware modifications to the PTE.
 	*/
-	fe->ptl = pte_lockptr(vma->vm_mm, fe->pmd);
-	spin_lock(fe->ptl);
-	if (unlikely(!pte_same(*fe->pte, pte))) {
-		pte_unmap_unlock(fe->pte, fe->ptl);
+	vmf->ptl = pte_lockptr(vma->vm_mm, vmf->pmd);
+	spin_lock(vmf->ptl);
+	if (unlikely(!pte_same(*vmf->pte, pte))) {
+		pte_unmap_unlock(vmf->pte, vmf->ptl);
 		goto out;
 	}
 
@@ -3412,18 +3419,18 @@ static int do_numa_page(struct fault_env *fe, pte_t pte)
 	pte = pte_mkyoung(pte);
 	if (was_writable)
 		pte = pte_mkwrite(pte);
-	set_pte_at(vma->vm_mm, fe->address, fe->pte, pte);
-	update_mmu_cache(vma, fe->address, fe->pte);
+	set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
+	update_mmu_cache(vma, vmf->address, vmf->pte);
 
-	page = vm_normal_page(vma, fe->address, pte);
+	page = vm_normal_page(vma, vmf->address, pte);
 	if (!page) {
-		pte_unmap_unlock(fe->pte, fe->ptl);
+		pte_unmap_unlock(vmf->pte, vmf->ptl);
 		return 0;
 	}
 
 	/* TODO: handle PTE-mapped THP */
 	if (PageCompound(page)) {
-		pte_unmap_unlock(fe->pte, fe->ptl);
+		pte_unmap_unlock(vmf->pte, vmf->ptl);
 		return 0;
 	}
 
@@ -3447,9 +3454,9 @@ static int do_numa_page(struct fault_env *fe, pte_t pte)
 
 	last_cpupid = page_cpupid_last(page);
 	page_nid = page_to_nid(page);
-	target_nid = numa_migrate_prep(page, vma, fe->address, page_nid,
+	target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
 			&flags);
-	pte_unmap_unlock(fe->pte, fe->ptl);
+	pte_unmap_unlock(vmf->pte, vmf->ptl);
 	if (target_nid == -1) {
 		put_page(page);
 		goto out;
@@ -3469,28 +3476,28 @@ out:
 	return 0;
 }
 
-static int create_huge_pmd(struct fault_env *fe)
+static int create_huge_pmd(struct vm_fault *vmf)
 {
-	struct vm_area_struct *vma = fe->vma;
+	struct vm_area_struct *vma = vmf->vma;
 	if (vma_is_anonymous(vma))
-		return do_huge_pmd_anonymous_page(fe);
+		return do_huge_pmd_anonymous_page(vmf);
 	if (vma->vm_ops->pmd_fault)
-		return vma->vm_ops->pmd_fault(vma, fe->address, fe->pmd,
-				fe->flags);
+		return vma->vm_ops->pmd_fault(vma, vmf->address, vmf->pmd,
+				vmf->flags);
 	return VM_FAULT_FALLBACK;
 }
 
-static int wp_huge_pmd(struct fault_env *fe, pmd_t orig_pmd)
+static int wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd)
 {
-	if (vma_is_anonymous(fe->vma))
-		return do_huge_pmd_wp_page(fe, orig_pmd);
-	if (fe->vma->vm_ops->pmd_fault)
-		return fe->vma->vm_ops->pmd_fault(fe->vma, fe->address, fe->pmd,
-				fe->flags);
+	if (vma_is_anonymous(vmf->vma))
+		return do_huge_pmd_wp_page(vmf, orig_pmd);
+	if (vmf->vma->vm_ops->pmd_fault)
+		return vmf->vma->vm_ops->pmd_fault(vmf->vma, vmf->address,
+						   vmf->pmd, vmf->flags);
 
 	/* COW handled on pte level: split pmd */
-	VM_BUG_ON_VMA(fe->vma->vm_flags & VM_SHARED, fe->vma);
-	__split_huge_pmd(fe->vma, fe->pmd, fe->address, false, NULL);
+	VM_BUG_ON_VMA(vmf->vma->vm_flags & VM_SHARED, vmf->vma);
+	__split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);
 
 	return VM_FAULT_FALLBACK;
 }
@@ -3515,21 +3522,21 @@ static inline bool vma_is_accessible(struct vm_area_struct *vma)
  * The mmap_sem may have been released depending on flags and our return value.
  * See filemap_fault() and __lock_page_or_retry().
  */
-static int handle_pte_fault(struct fault_env *fe)
+static int handle_pte_fault(struct vm_fault *vmf)
 {
 	pte_t entry;
 
-	if (unlikely(pmd_none(*fe->pmd))) {
+	if (unlikely(pmd_none(*vmf->pmd))) {
 		/*
 		 * Leave __pte_alloc() until later: because vm_ops->fault may
 		 * want to allocate huge page, and if we expose page table
 		 * for an instant, it will be difficult to retract from
 		 * concurrent faults and from rmap lookups.
 		 */
-		fe->pte = NULL;
+		vmf->pte = NULL;
 	} else {
 		/* See comment in pte_alloc_one_map() */
-		if (pmd_trans_unstable(fe->pmd) || pmd_devmap(*fe->pmd))
+		if (pmd_trans_unstable(vmf->pmd) || pmd_devmap(*vmf->pmd))
 			return 0;
 		/*
 		 * A regular pmd is established and it can't morph into a huge
@@ -3537,9 +3544,8 @@ static int handle_pte_fault(struct fault_env *fe)
 		 * mmap_sem read mode and khugepaged takes it in write mode.
 		 * So now it's safe to run pte_offset_map().
 		 */
-		fe->pte = pte_offset_map(fe->pmd, fe->address);
-
-		entry = *fe->pte;
+		vmf->pte = pte_offset_map(vmf->pmd, vmf->address);
+		vmf->orig_pte = *vmf->pte;
 
 		/*
 		 * some architectures can have larger ptes than wordsize,
@@ -3550,38 +3556,39 @@ static int handle_pte_fault(struct fault_env *fe)
 		 * ptl lock held. So here a barrier will do.
 		 */
 		barrier();
-		if (pte_none(entry)) {
-			pte_unmap(fe->pte);
-			fe->pte = NULL;
+		if (pte_none(vmf->orig_pte)) {
+			pte_unmap(vmf->pte);
+			vmf->pte = NULL;
 		}
 	}
 
-	if (!fe->pte) {
-		if (vma_is_anonymous(fe->vma))
-			return do_anonymous_page(fe);
+	if (!vmf->pte) {
+		if (vma_is_anonymous(vmf->vma))
+			return do_anonymous_page(vmf);
 		else
-			return do_fault(fe);
+			return do_fault(vmf);
 	}
 
-	if (!pte_present(entry))
-		return do_swap_page(fe, entry);
+	if (!pte_present(vmf->orig_pte))
+		return do_swap_page(vmf);
 
-	if (pte_protnone(entry) && vma_is_accessible(fe->vma))
-		return do_numa_page(fe, entry);
+	if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
+		return do_numa_page(vmf);
 
-	fe->ptl = pte_lockptr(fe->vma->vm_mm, fe->pmd);
-	spin_lock(fe->ptl);
-	if (unlikely(!pte_same(*fe->pte, entry)))
+	vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
+	spin_lock(vmf->ptl);
+	entry = vmf->orig_pte;
+	if (unlikely(!pte_same(*vmf->pte, entry)))
 		goto unlock;
-	if (fe->flags & FAULT_FLAG_WRITE) {
+	if (vmf->flags & FAULT_FLAG_WRITE) {
 		if (!pte_write(entry))
-			return do_wp_page(fe, entry);
+			return do_wp_page(vmf);
 		entry = pte_mkdirty(entry);
 	}
 	entry = pte_mkyoung(entry);
-	if (ptep_set_access_flags(fe->vma, fe->address, fe->pte, entry,
-				fe->flags & FAULT_FLAG_WRITE)) {
-		update_mmu_cache(fe->vma, fe->address, fe->pte);
+	if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
+				vmf->flags & FAULT_FLAG_WRITE)) {
+		update_mmu_cache(vmf->vma, vmf->address, vmf->pte);
 	} else {
 		/*
 		 * This is needed only for protection faults but the arch code
@@ -3589,11 +3596,11 @@ static int handle_pte_fault(struct fault_env *fe)
 		 * This still avoids useless tlb flushes for .text page faults
 		 * with threads.
 		 */
-		if (fe->flags & FAULT_FLAG_WRITE)
-			flush_tlb_fix_spurious_fault(fe->vma, fe->address);
+		if (vmf->flags & FAULT_FLAG_WRITE)
+			flush_tlb_fix_spurious_fault(vmf->vma, vmf->address);
 	}
 unlock:
-	pte_unmap_unlock(fe->pte, fe->ptl);
+	pte_unmap_unlock(vmf->pte, vmf->ptl);
 	return 0;
 }
 
@@ -3606,10 +3613,12 @@ unlock:
 static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
 		unsigned int flags)
 {
-	struct fault_env fe = {
+	struct vm_fault vmf = {
 		.vma = vma,
-		.address = address,
+		.address = address & PAGE_MASK,
 		.flags = flags,
+		.pgoff = linear_page_index(vma, address),
+		.gfp_mask = __get_fault_gfp_mask(vma),
 	};
 	struct mm_struct *mm = vma->vm_mm;
 	pgd_t *pgd;
@@ -3619,35 +3628,35 @@ static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
 	pud = pud_alloc(mm, pgd, address);
 	if (!pud)
 		return VM_FAULT_OOM;
-	fe.pmd = pmd_alloc(mm, pud, address);
-	if (!fe.pmd)
+	vmf.pmd = pmd_alloc(mm, pud, address);
+	if (!vmf.pmd)
 		return VM_FAULT_OOM;
-	if (pmd_none(*fe.pmd) && transparent_hugepage_enabled(vma)) {
-		int ret = create_huge_pmd(&fe);
+	if (pmd_none(*vmf.pmd) && transparent_hugepage_enabled(vma)) {
+		int ret = create_huge_pmd(&vmf);
 		if (!(ret & VM_FAULT_FALLBACK))
 			return ret;
 	} else {
-		pmd_t orig_pmd = *fe.pmd;
+		pmd_t orig_pmd = *vmf.pmd;
 		int ret;
 
 		barrier();
 		if (pmd_trans_huge(orig_pmd) || pmd_devmap(orig_pmd)) {
 			if (pmd_protnone(orig_pmd) && vma_is_accessible(vma))
-				return do_huge_pmd_numa_page(&fe, orig_pmd);
+				return do_huge_pmd_numa_page(&vmf, orig_pmd);
 
-			if ((fe.flags & FAULT_FLAG_WRITE) &&
+			if ((vmf.flags & FAULT_FLAG_WRITE) &&
 					!pmd_write(orig_pmd)) {
-				ret = wp_huge_pmd(&fe, orig_pmd);
+				ret = wp_huge_pmd(&vmf, orig_pmd);
 				if (!(ret & VM_FAULT_FALLBACK))
 					return ret;
 			} else {
-				huge_pmd_set_accessed(&fe, orig_pmd);
+				huge_pmd_set_accessed(&vmf, orig_pmd);
 				return 0;
 			}
 		}
 	}
 
-	return handle_pte_fault(&fe);
+	return handle_pte_fault(&vmf);
 }
 
 /*
@@ -3808,8 +3817,8 @@ out:
 	return -EINVAL;
 }
 
-static inline int follow_pte(struct mm_struct *mm, unsigned long address,
-			     pte_t **ptepp, spinlock_t **ptlp)
+int follow_pte(struct mm_struct *mm, unsigned long address, pte_t **ptepp,
+	       spinlock_t **ptlp)
 {
 	int res;
 
@@ -3904,7 +3913,7 @@ EXPORT_SYMBOL_GPL(generic_access_phys);
  * Access another process' address space as given in mm.  If non-NULL, use the
  * given task for page fault accounting.
  */
-static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
+int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
 		unsigned long addr, void *buf, int len, unsigned int gup_flags)
 {
 	struct vm_area_struct *vma;
@@ -3919,7 +3928,7 @@ static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
 		struct page *page = NULL;
 
 		ret = get_user_pages_remote(tsk, mm, addr, 1,
-				gup_flags, &page, &vma);
+				gup_flags, &page, &vma, NULL);
 		if (ret <= 0) {
 #ifndef CONFIG_HAVE_IOREMAP_PROT
 			break;
@@ -4002,6 +4011,7 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr,
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(access_process_vm);
 
 /*
  * Print the name of a VMA.
diff --git a/mm/nommu.c b/mm/nommu.c
index 8b8faaf2a9e9..210d7ec2843c 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -176,9 +176,10 @@ long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
 }
 EXPORT_SYMBOL(get_user_pages_locked);
 
-long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
-			       unsigned long start, unsigned long nr_pages,
-			       struct page **pages, unsigned int gup_flags)
+static long __get_user_pages_unlocked(struct task_struct *tsk,
+			struct mm_struct *mm, unsigned long start,
+			unsigned long nr_pages, struct page **pages,
+			unsigned int gup_flags)
 {
 	long ret;
 	down_read(&mm->mmap_sem);
@@ -187,7 +188,6 @@ long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
 	up_read(&mm->mmap_sem);
 	return ret;
 }
-EXPORT_SYMBOL(__get_user_pages_unlocked);
 
 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
 			     struct page **pages, unsigned int gup_flags)
@@ -1801,14 +1801,14 @@ int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 }
 EXPORT_SYMBOL(filemap_fault);
 
-void filemap_map_pages(struct fault_env *fe,
+void filemap_map_pages(struct vm_fault *vmf,
 		pgoff_t start_pgoff, pgoff_t end_pgoff)
 {
 	BUG();
 }
 EXPORT_SYMBOL(filemap_map_pages);
 
-static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
+int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
 		unsigned long addr, void *buf, int len, unsigned int gup_flags)
 {
 	struct vm_area_struct *vma;
@@ -1878,6 +1878,7 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, in
 	mmput(mm);
 	return len;
 }
+EXPORT_SYMBOL_GPL(access_process_vm);
 
 /**
  * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 439cc63ad903..290e8b7d3181 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -1778,6 +1778,7 @@ pause:
 					  pause,
 					  start_time);
 		__set_current_state(TASK_KILLABLE);
+		wb->dirty_sleep = now;
 		io_schedule_timeout(pause);
 
 		current->dirty_paused_when = now + pause;
@@ -2105,18 +2106,26 @@ void tag_pages_for_writeback(struct address_space *mapping,
 			     pgoff_t start, pgoff_t end)
 {
 #define WRITEBACK_TAG_BATCH 4096
-	unsigned long tagged;
-
-	do {
-		spin_lock_irq(&mapping->tree_lock);
-		tagged = radix_tree_range_tag_if_tagged(&mapping->page_tree,
-				&start, end, WRITEBACK_TAG_BATCH,
-				PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE);
+	unsigned long tagged = 0;
+	struct radix_tree_iter iter;
+	void **slot;
+
+	spin_lock_irq(&mapping->tree_lock);
+	radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, start,
+							PAGECACHE_TAG_DIRTY) {
+		if (iter.index > end)
+			break;
+		radix_tree_iter_tag_set(&mapping->page_tree, &iter,
+							PAGECACHE_TAG_TOWRITE);
+		tagged++;
+		if ((tagged % WRITEBACK_TAG_BATCH) != 0)
+			continue;
+		slot = radix_tree_iter_resume(slot, &iter);
 		spin_unlock_irq(&mapping->tree_lock);
-		WARN_ON_ONCE(tagged > WRITEBACK_TAG_BATCH);
 		cond_resched();
-		/* We check 'start' to handle wrapping when end == ~0UL */
-	} while (tagged >= WRITEBACK_TAG_BATCH && start);
+		spin_lock_irq(&mapping->tree_lock);
+	}
+	spin_unlock_irq(&mapping->tree_lock);
 }
 EXPORT_SYMBOL(tag_pages_for_writeback);
 
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index f64e7bcb43b7..2c6d5f64feca 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -3925,6 +3925,20 @@ static struct page *__page_frag_refill(struct page_frag_cache *nc,
 	return page;
 }
 
+void __page_frag_drain(struct page *page, unsigned int order,
+		       unsigned int count)
+{
+	VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
+
+	if (page_ref_sub_and_test(page, count)) {
+		if (order == 0)
+			free_hot_cold_page(page, false);
+		else
+			__free_pages_ok(page, order);
+	}
+}
+EXPORT_SYMBOL(__page_frag_drain);
+
 void *__alloc_page_frag(struct page_frag_cache *nc,
 			unsigned int fragsz, gfp_t gfp_mask)
 {
diff --git a/mm/page_io.c b/mm/page_io.c
index a2651f58c86a..23f6d0d3470f 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -320,10 +320,7 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc,
 		ret = -ENOMEM;
 		goto out;
 	}
-	if (wbc->sync_mode == WB_SYNC_ALL)
-		bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC);
-	else
-		bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
+	bio->bi_opf = REQ_OP_WRITE | wbc_to_write_flags(wbc);
 	count_vm_event(PSWPOUT);
 	set_page_writeback(page);
 	unlock_page(page);
diff --git a/mm/percpu.c b/mm/percpu.c
index f696385bcc44..0686f566d347 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -886,7 +886,8 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved,
 
 	size = ALIGN(size, 2);
 
-	if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) {
+	if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE ||
+		     !is_power_of_2(align))) {
 		WARN(true, "illegal size (%zu) or align (%zu) for percpu allocation\n",
 		     size, align);
 		return NULL;
diff --git a/mm/process_vm_access.c b/mm/process_vm_access.c
index be8dc8d1edb9..84d0c7eada2b 100644
--- a/mm/process_vm_access.c
+++ b/mm/process_vm_access.c
@@ -88,7 +88,7 @@ static int process_vm_rw_single_vec(unsigned long addr,
 	ssize_t rc = 0;
 	unsigned long max_pages_per_loop = PVM_MAX_KMALLOC_PAGES
 		/ sizeof(struct pages *);
-	unsigned int flags = FOLL_REMOTE;
+	unsigned int flags = 0;
 
 	/* Work out address and page range required */
 	if (len == 0)
@@ -100,15 +100,19 @@ static int process_vm_rw_single_vec(unsigned long addr,
 
 	while (!rc && nr_pages && iov_iter_count(iter)) {
 		int pages = min(nr_pages, max_pages_per_loop);
+		int locked = 1;
 		size_t bytes;
 
 		/*
 		 * Get the pages we're interested in.  We must
-		 * add FOLL_REMOTE because task/mm might not
+		 * access remotely because task/mm might not
 		 * current/current->mm
 		 */
-		pages = __get_user_pages_unlocked(task, mm, pa, pages,
-						  process_pages, flags);
+		down_read(&mm->mmap_sem);
+		pages = get_user_pages_remote(task, mm, pa, pages, flags,
+					      process_pages, NULL, &locked);
+		if (locked)
+			up_read(&mm->mmap_sem);
 		if (pages <= 0)
 			return -EFAULT;
 
diff --git a/mm/shmem.c b/mm/shmem.c
index abd7403aba41..54287d443806 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -661,8 +661,8 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping,
 			swapped++;
 
 		if (need_resched()) {
+			slot = radix_tree_iter_resume(slot, &iter);
 			cond_resched_rcu();
-			slot = radix_tree_iter_next(&iter);
 		}
 	}
 
@@ -1049,6 +1049,30 @@ static void shmem_evict_inode(struct inode *inode)
 	clear_inode(inode);
 }
 
+static unsigned long find_swap_entry(struct radix_tree_root *root, void *item)
+{
+	struct radix_tree_iter iter;
+	void **slot;
+	unsigned long found = -1;
+	unsigned int checked = 0;
+
+	rcu_read_lock();
+	radix_tree_for_each_slot(slot, root, &iter, 0) {
+		if (*slot == item) {
+			found = iter.index;
+			break;
+		}
+		checked++;
+		if ((checked % 4096) != 0)
+			continue;
+		slot = radix_tree_iter_resume(slot, &iter);
+		cond_resched_rcu();
+	}
+
+	rcu_read_unlock();
+	return found;
+}
+
 /*
  * If swap found in inode, free it and move page from swapcache to filecache.
  */
@@ -1062,7 +1086,7 @@ static int shmem_unuse_inode(struct shmem_inode_info *info,
 	int error = 0;
 
 	radswap = swp_to_radix_entry(swap);
-	index = radix_tree_locate_item(&mapping->page_tree, radswap);
+	index = find_swap_entry(&mapping->page_tree, radswap);
 	if (index == -1)
 		return -EAGAIN;	/* tell shmem_unuse we found nothing */
 
@@ -2447,8 +2471,8 @@ static void shmem_tag_pins(struct address_space *mapping)
 		}
 
 		if (need_resched()) {
+			slot = radix_tree_iter_resume(slot, &iter);
 			cond_resched_rcu();
-			slot = radix_tree_iter_next(&iter);
 		}
 	}
 	rcu_read_unlock();
@@ -2517,8 +2541,8 @@ static int shmem_wait_for_pins(struct address_space *mapping)
 			spin_unlock_irq(&mapping->tree_lock);
 continue_resched:
 			if (need_resched()) {
+				slot = radix_tree_iter_resume(slot, &iter);
 				cond_resched_rcu();
-				slot = radix_tree_iter_next(&iter);
 			}
 		}
 		rcu_read_unlock();
diff --git a/mm/slab.c b/mm/slab.c
index 87b29e76cafd..29bc6c0dedd0 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -552,12 +552,7 @@ static void start_cpu_timer(int cpu)
 {
 	struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
 
-	/*
-	 * When this gets called from do_initcalls via cpucache_init(),
-	 * init_workqueues() has already run, so keventd will be setup
-	 * at that time.
-	 */
-	if (keventd_up() && reap_work->work.func == NULL) {
+	if (reap_work->work.func == NULL) {
 		init_reap_node(cpu);
 		INIT_DEFERRABLE_WORK(reap_work, cache_reap);
 		schedule_delayed_work_on(cpu, reap_work,
diff --git a/net/bluetooth/Makefile b/net/bluetooth/Makefile
index b3ff12eb9b6d..4bfaa19a5573 100644
--- a/net/bluetooth/Makefile
+++ b/net/bluetooth/Makefile
@@ -20,5 +20,3 @@ bluetooth-$(CONFIG_BT_HS) += a2mp.o amp.o
 bluetooth-$(CONFIG_BT_LEDS) += leds.o
 bluetooth-$(CONFIG_BT_DEBUGFS) += hci_debugfs.o
 bluetooth-$(CONFIG_BT_SELFTEST) += selftest.o
-
-subdir-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c
index 50fdc1b59777..3c4bbec39713 100644
--- a/net/core/net_namespace.c
+++ b/net/core/net_namespace.c
@@ -218,16 +218,15 @@ static void rtnl_net_notifyid(struct net *net, int cmd, int id);
  */
 int peernet2id_alloc(struct net *net, struct net *peer)
 {
-	unsigned long flags;
 	bool alloc;
 	int id;
 
 	if (atomic_read(&net->count) == 0)
 		return NETNSA_NSID_NOT_ASSIGNED;
-	spin_lock_irqsave(&net->nsid_lock, flags);
+	spin_lock_bh(&net->nsid_lock);
 	alloc = atomic_read(&peer->count) == 0 ? false : true;
 	id = __peernet2id_alloc(net, peer, &alloc);
-	spin_unlock_irqrestore(&net->nsid_lock, flags);
+	spin_unlock_bh(&net->nsid_lock);
 	if (alloc && id >= 0)
 		rtnl_net_notifyid(net, RTM_NEWNSID, id);
 	return id;
@@ -236,12 +235,11 @@ int peernet2id_alloc(struct net *net, struct net *peer)
 /* This function returns, if assigned, the id of a peer netns. */
 int peernet2id(struct net *net, struct net *peer)
 {
-	unsigned long flags;
 	int id;
 
-	spin_lock_irqsave(&net->nsid_lock, flags);
+	spin_lock_bh(&net->nsid_lock);
 	id = __peernet2id(net, peer);
-	spin_unlock_irqrestore(&net->nsid_lock, flags);
+	spin_unlock_bh(&net->nsid_lock);
 	return id;
 }
 EXPORT_SYMBOL(peernet2id);
@@ -256,18 +254,17 @@ bool peernet_has_id(struct net *net, struct net *peer)
 
 struct net *get_net_ns_by_id(struct net *net, int id)
 {
-	unsigned long flags;
 	struct net *peer;
 
 	if (id < 0)
 		return NULL;
 
 	rcu_read_lock();
-	spin_lock_irqsave(&net->nsid_lock, flags);
+	spin_lock_bh(&net->nsid_lock);
 	peer = idr_find(&net->netns_ids, id);
 	if (peer)
 		get_net(peer);
-	spin_unlock_irqrestore(&net->nsid_lock, flags);
+	spin_unlock_bh(&net->nsid_lock);
 	rcu_read_unlock();
 
 	return peer;
@@ -437,17 +434,17 @@ static void cleanup_net(struct work_struct *work)
 		for_each_net(tmp) {
 			int id;
 
-			spin_lock_irq(&tmp->nsid_lock);
+			spin_lock_bh(&tmp->nsid_lock);
 			id = __peernet2id(tmp, net);
 			if (id >= 0)
 				idr_remove(&tmp->netns_ids, id);
-			spin_unlock_irq(&tmp->nsid_lock);
+			spin_unlock_bh(&tmp->nsid_lock);
 			if (id >= 0)
 				rtnl_net_notifyid(tmp, RTM_DELNSID, id);
 		}
-		spin_lock_irq(&net->nsid_lock);
+		spin_lock_bh(&net->nsid_lock);
 		idr_destroy(&net->netns_ids);
-		spin_unlock_irq(&net->nsid_lock);
+		spin_unlock_bh(&net->nsid_lock);
 
 	}
 	rtnl_unlock();
@@ -576,7 +573,6 @@ static int rtnl_net_newid(struct sk_buff *skb, struct nlmsghdr *nlh)
 {
 	struct net *net = sock_net(skb->sk);
 	struct nlattr *tb[NETNSA_MAX + 1];
-	unsigned long flags;
 	struct net *peer;
 	int nsid, err;
 
@@ -597,15 +593,15 @@ static int rtnl_net_newid(struct sk_buff *skb, struct nlmsghdr *nlh)
 	if (IS_ERR(peer))
 		return PTR_ERR(peer);
 
-	spin_lock_irqsave(&net->nsid_lock, flags);
+	spin_lock_bh(&net->nsid_lock);
 	if (__peernet2id(net, peer) >= 0) {
-		spin_unlock_irqrestore(&net->nsid_lock, flags);
+		spin_unlock_bh(&net->nsid_lock);
 		err = -EEXIST;
 		goto out;
 	}
 
 	err = alloc_netid(net, peer, nsid);
-	spin_unlock_irqrestore(&net->nsid_lock, flags);
+	spin_unlock_bh(&net->nsid_lock);
 	if (err >= 0) {
 		rtnl_net_notifyid(net, RTM_NEWNSID, err);
 		err = 0;
@@ -727,11 +723,10 @@ static int rtnl_net_dumpid(struct sk_buff *skb, struct netlink_callback *cb)
 		.idx = 0,
 		.s_idx = cb->args[0],
 	};
-	unsigned long flags;
 
-	spin_lock_irqsave(&net->nsid_lock, flags);
+	spin_lock_bh(&net->nsid_lock);
 	idr_for_each(&net->netns_ids, rtnl_net_dumpid_one, &net_cb);
-	spin_unlock_irqrestore(&net->nsid_lock, flags);
+	spin_unlock_bh(&net->nsid_lock);
 
 	cb->args[0] = net_cb.idx;
 	return skb->len;
diff --git a/net/ieee802154/6lowpan/6lowpan_i.h b/net/ieee802154/6lowpan/6lowpan_i.h
index 5ac778962e4e..ac7c96b73ad5 100644
--- a/net/ieee802154/6lowpan/6lowpan_i.h
+++ b/net/ieee802154/6lowpan/6lowpan_i.h
@@ -7,7 +7,7 @@
 #include <net/inet_frag.h>
 #include <net/6lowpan.h>
 
-typedef unsigned __bitwise__ lowpan_rx_result;
+typedef unsigned __bitwise lowpan_rx_result;
 #define RX_CONTINUE		((__force lowpan_rx_result) 0u)
 #define RX_DROP_UNUSABLE	((__force lowpan_rx_result) 1u)
 #define RX_DROP			((__force lowpan_rx_result) 2u)
diff --git a/net/ieee802154/Makefile b/net/ieee802154/Makefile
index 4adfd4d5471b..9b92ade687a3 100644
--- a/net/ieee802154/Makefile
+++ b/net/ieee802154/Makefile
@@ -7,5 +7,3 @@ ieee802154-y := netlink.o nl-mac.o nl-phy.o nl_policy.o core.o \
 ieee802154_socket-y := socket.o
 
 CFLAGS_trace.o := -I$(src)
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/net/mac80211/Makefile b/net/mac80211/Makefile
index 0b202b343fd4..282912245938 100644
--- a/net/mac80211/Makefile
+++ b/net/mac80211/Makefile
@@ -61,4 +61,4 @@ rc80211_minstrel_ht-$(CONFIG_MAC80211_DEBUGFS) += rc80211_minstrel_ht_debugfs.o
 mac80211-$(CONFIG_MAC80211_RC_MINSTREL) += $(rc80211_minstrel-y)
 mac80211-$(CONFIG_MAC80211_RC_MINSTREL_HT) += $(rc80211_minstrel_ht-y)
 
-ccflags-y += -D__CHECK_ENDIAN__ -DDEBUG
+ccflags-y += -DDEBUG
diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h
index d37a577f63a1..b2069fbd60f9 100644
--- a/net/mac80211/ieee80211_i.h
+++ b/net/mac80211/ieee80211_i.h
@@ -159,7 +159,7 @@ enum ieee80211_bss_valid_data_flags {
 	IEEE80211_BSS_VALID_ERP			= BIT(3)
 };
 
-typedef unsigned __bitwise__ ieee80211_tx_result;
+typedef unsigned __bitwise ieee80211_tx_result;
 #define TX_CONTINUE	((__force ieee80211_tx_result) 0u)
 #define TX_DROP		((__force ieee80211_tx_result) 1u)
 #define TX_QUEUED	((__force ieee80211_tx_result) 2u)
@@ -180,7 +180,7 @@ struct ieee80211_tx_data {
 };
 
 
-typedef unsigned __bitwise__ ieee80211_rx_result;
+typedef unsigned __bitwise ieee80211_rx_result;
 #define RX_CONTINUE		((__force ieee80211_rx_result) 0u)
 #define RX_DROP_UNUSABLE	((__force ieee80211_rx_result) 1u)
 #define RX_DROP_MONITOR		((__force ieee80211_rx_result) 2u)
diff --git a/net/mac802154/Makefile b/net/mac802154/Makefile
index 17a51e8389e2..5857bb1e1695 100644
--- a/net/mac802154/Makefile
+++ b/net/mac802154/Makefile
@@ -3,5 +3,3 @@ mac802154-objs		:= main.o rx.o tx.o mac_cmd.o mib.o \
 			   iface.o llsec.o util.o cfg.o trace.o
 
 CFLAGS_trace.o := -I$(src)
-
-ccflags-y += -D__CHECK_ENDIAN__
diff --git a/net/rds/rdma_transport.c b/net/rds/rdma_transport.c
index 345f09059e9f..d5f311767157 100644
--- a/net/rds/rdma_transport.c
+++ b/net/rds/rdma_transport.c
@@ -100,11 +100,14 @@ int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id,
 		trans->cm_connect_complete(conn, event);
 		break;
 
+	case RDMA_CM_EVENT_REJECTED:
+		rdsdebug("Connection rejected: %s\n",
+			 rdma_reject_msg(cm_id, event->status));
+		/* FALLTHROUGH */
 	case RDMA_CM_EVENT_ADDR_ERROR:
 	case RDMA_CM_EVENT_ROUTE_ERROR:
 	case RDMA_CM_EVENT_CONNECT_ERROR:
 	case RDMA_CM_EVENT_UNREACHABLE:
-	case RDMA_CM_EVENT_REJECTED:
 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
 	case RDMA_CM_EVENT_ADDR_CHANGE:
 		if (conn)
diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c
index 2d59c9be40e1..5f63f6dcaabb 100644
--- a/net/rxrpc/af_rxrpc.c
+++ b/net/rxrpc/af_rxrpc.c
@@ -762,16 +762,17 @@ static const struct net_proto_family rxrpc_family_ops = {
 static int __init af_rxrpc_init(void)
 {
 	int ret = -1;
+	unsigned int tmp;
 
 	BUILD_BUG_ON(sizeof(struct rxrpc_skb_priv) > FIELD_SIZEOF(struct sk_buff, cb));
 
 	get_random_bytes(&rxrpc_epoch, sizeof(rxrpc_epoch));
 	rxrpc_epoch |= RXRPC_RANDOM_EPOCH;
-	get_random_bytes(&rxrpc_client_conn_ids.cur,
-			 sizeof(rxrpc_client_conn_ids.cur));
-	rxrpc_client_conn_ids.cur &= 0x3fffffff;
-	if (rxrpc_client_conn_ids.cur == 0)
-		rxrpc_client_conn_ids.cur = 1;
+	get_random_bytes(&tmp, sizeof(tmp));
+	tmp &= 0x3fffffff;
+	if (tmp == 0)
+		tmp = 1;
+	idr_set_cursor(&rxrpc_client_conn_ids, tmp);
 
 	ret = -ENOMEM;
 	rxrpc_call_jar = kmem_cache_create(
diff --git a/net/rxrpc/conn_client.c b/net/rxrpc/conn_client.c
index 60ef9605167e..6cbcdcc29853 100644
--- a/net/rxrpc/conn_client.c
+++ b/net/rxrpc/conn_client.c
@@ -263,12 +263,12 @@ static bool rxrpc_may_reuse_conn(struct rxrpc_connection *conn)
 	 * times the maximum number of client conns away from the current
 	 * allocation point to try and keep the IDs concentrated.
 	 */
-	id_cursor = READ_ONCE(rxrpc_client_conn_ids.cur);
+	id_cursor = idr_get_cursor(&rxrpc_client_conn_ids);
 	id = conn->proto.cid >> RXRPC_CIDSHIFT;
 	distance = id - id_cursor;
 	if (distance < 0)
 		distance = -distance;
-	limit = round_up(rxrpc_max_client_connections, IDR_SIZE) * 4;
+	limit = max(rxrpc_max_client_connections * 4, 1024U);
 	if (distance > limit)
 		goto mark_dont_reuse;
 
diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c
index 3dfd769dc5b5..16cea00c959b 100644
--- a/net/sunrpc/auth_gss/auth_gss.c
+++ b/net/sunrpc/auth_gss/auth_gss.c
@@ -541,9 +541,13 @@ gss_setup_upcall(struct gss_auth *gss_auth, struct rpc_cred *cred)
 		return gss_new;
 	gss_msg = gss_add_msg(gss_new);
 	if (gss_msg == gss_new) {
-		int res = rpc_queue_upcall(gss_new->pipe, &gss_new->msg);
+		int res;
+		atomic_inc(&gss_msg->count);
+		res = rpc_queue_upcall(gss_new->pipe, &gss_new->msg);
 		if (res) {
 			gss_unhash_msg(gss_new);
+			atomic_dec(&gss_msg->count);
+			gss_release_msg(gss_new);
 			gss_msg = ERR_PTR(res);
 		}
 	} else
@@ -836,6 +840,7 @@ gss_pipe_destroy_msg(struct rpc_pipe_msg *msg)
 			warn_gssd();
 		gss_release_msg(gss_msg);
 	}
+	gss_release_msg(gss_msg);
 }
 
 static void gss_pipe_dentry_destroy(struct dentry *dir,
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 62a482790937..1efbe48e794f 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -1926,6 +1926,8 @@ call_connect_status(struct rpc_task *task)
 	case -EADDRINUSE:
 	case -ENOBUFS:
 	case -EPIPE:
+		xprt_conditional_disconnect(task->tk_rqstp->rq_xprt,
+					    task->tk_rqstp->rq_connect_cookie);
 		if (RPC_IS_SOFTCONN(task))
 			break;
 		/* retry with existing socket, after a delay */
diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
index 2ecb994314c1..caeb01ad2b5a 100644
--- a/net/sunrpc/stats.c
+++ b/net/sunrpc/stats.c
@@ -157,15 +157,17 @@ void rpc_count_iostats_metrics(const struct rpc_task *task,
 	spin_lock(&op_metrics->om_lock);
 
 	op_metrics->om_ops++;
-	op_metrics->om_ntrans += req->rq_ntrans;
+	/* kernel API: om_ops must never become larger than om_ntrans */
+	op_metrics->om_ntrans += max(req->rq_ntrans, 1);
 	op_metrics->om_timeouts += task->tk_timeouts;
 
 	op_metrics->om_bytes_sent += req->rq_xmit_bytes_sent;
 	op_metrics->om_bytes_recv += req->rq_reply_bytes_recvd;
 
-	delta = ktime_sub(req->rq_xtime, task->tk_start);
-	op_metrics->om_queue = ktime_add(op_metrics->om_queue, delta);
-
+	if (ktime_to_ns(req->rq_xtime)) {
+		delta = ktime_sub(req->rq_xtime, task->tk_start);
+		op_metrics->om_queue = ktime_add(op_metrics->om_queue, delta);
+	}
 	op_metrics->om_rtt = ktime_add(op_metrics->om_rtt, req->rq_rtt);
 
 	delta = ktime_sub(now, task->tk_start);
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index 685e6d225414..9a6be030ca7d 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -669,7 +669,7 @@ void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie)
 	spin_lock_bh(&xprt->transport_lock);
 	if (cookie != xprt->connect_cookie)
 		goto out;
-	if (test_bit(XPRT_CLOSING, &xprt->state) || !xprt_connected(xprt))
+	if (test_bit(XPRT_CLOSING, &xprt->state))
 		goto out;
 	set_bit(XPRT_CLOSE_WAIT, &xprt->state);
 	/* Try to schedule an autoclose RPC call */
@@ -772,6 +772,7 @@ void xprt_connect(struct rpc_task *task)
 	if (!xprt_connected(xprt)) {
 		task->tk_rqstp->rq_bytes_sent = 0;
 		task->tk_timeout = task->tk_rqstp->rq_timeout;
+		task->tk_rqstp->rq_connect_cookie = xprt->connect_cookie;
 		rpc_sleep_on(&xprt->pending, task, xprt_connect_status);
 
 		if (test_bit(XPRT_CLOSING, &xprt->state))
diff --git a/net/sunrpc/xprtrdma/backchannel.c b/net/sunrpc/xprtrdma/backchannel.c
index 2c472e1b4827..24fedd4b117e 100644
--- a/net/sunrpc/xprtrdma/backchannel.c
+++ b/net/sunrpc/xprtrdma/backchannel.c
@@ -55,7 +55,8 @@ static int rpcrdma_bc_setup_rqst(struct rpcrdma_xprt *r_xprt,
 	if (IS_ERR(rb))
 		goto out_fail;
 	req->rl_sendbuf = rb;
-	xdr_buf_init(&rqst->rq_snd_buf, rb->rg_base, size);
+	xdr_buf_init(&rqst->rq_snd_buf, rb->rg_base,
+		     min_t(size_t, size, PAGE_SIZE));
 	rpcrdma_set_xprtdata(rqst, req);
 	return 0;
 
@@ -191,6 +192,7 @@ size_t xprt_rdma_bc_maxpayload(struct rpc_xprt *xprt)
 	size_t maxmsg;
 
 	maxmsg = min_t(unsigned int, cdata->inline_rsize, cdata->inline_wsize);
+	maxmsg = min_t(unsigned int, maxmsg, PAGE_SIZE);
 	return maxmsg - RPCRDMA_HDRLEN_MIN;
 }
 
diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c
index 26b26beef2d4..47bed5333c7f 100644
--- a/net/sunrpc/xprtrdma/frwr_ops.c
+++ b/net/sunrpc/xprtrdma/frwr_ops.c
@@ -101,7 +101,7 @@ frwr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r)
 	struct rpcrdma_frmr *f = &r->frmr;
 	int rc;
 
-	f->fr_mr = ib_alloc_mr(ia->ri_pd, IB_MR_TYPE_MEM_REG, depth);
+	f->fr_mr = ib_alloc_mr(ia->ri_pd, ia->ri_mrtype, depth);
 	if (IS_ERR(f->fr_mr))
 		goto out_mr_err;
 
@@ -157,7 +157,7 @@ __frwr_reset_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r)
 		return rc;
 	}
 
-	f->fr_mr = ib_alloc_mr(ia->ri_pd, IB_MR_TYPE_MEM_REG,
+	f->fr_mr = ib_alloc_mr(ia->ri_pd, ia->ri_mrtype,
 			       ia->ri_max_frmr_depth);
 	if (IS_ERR(f->fr_mr)) {
 		pr_warn("rpcrdma: ib_alloc_mr status %ld, frwr %p orphaned\n",
@@ -171,10 +171,6 @@ __frwr_reset_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r)
 }
 
 /* Reset of a single FRMR. Generate a fresh rkey by replacing the MR.
- *
- * There's no recovery if this fails. The FRMR is abandoned, but
- * remains in rb_all. It will be cleaned up when the transport is
- * destroyed.
  */
 static void
 frwr_op_recover_mr(struct rpcrdma_mw *mw)
@@ -210,11 +206,16 @@ static int
 frwr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
 	     struct rpcrdma_create_data_internal *cdata)
 {
+	struct ib_device_attr *attrs = &ia->ri_device->attrs;
 	int depth, delta;
 
+	ia->ri_mrtype = IB_MR_TYPE_MEM_REG;
+	if (attrs->device_cap_flags & IB_DEVICE_SG_GAPS_REG)
+		ia->ri_mrtype = IB_MR_TYPE_SG_GAPS;
+
 	ia->ri_max_frmr_depth =
 			min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS,
-			      ia->ri_device->attrs.max_fast_reg_page_list_len);
+			      attrs->max_fast_reg_page_list_len);
 	dprintk("RPC:       %s: device's max FR page list len = %u\n",
 		__func__, ia->ri_max_frmr_depth);
 
@@ -241,8 +242,8 @@ frwr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
 	}
 
 	ep->rep_attr.cap.max_send_wr *= depth;
-	if (ep->rep_attr.cap.max_send_wr > ia->ri_device->attrs.max_qp_wr) {
-		cdata->max_requests = ia->ri_device->attrs.max_qp_wr / depth;
+	if (ep->rep_attr.cap.max_send_wr > attrs->max_qp_wr) {
+		cdata->max_requests = attrs->max_qp_wr / depth;
 		if (!cdata->max_requests)
 			return -EINVAL;
 		ep->rep_attr.cap.max_send_wr = cdata->max_requests *
@@ -348,6 +349,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
 	    int nsegs, bool writing, struct rpcrdma_mw **out)
 {
 	struct rpcrdma_ia *ia = &r_xprt->rx_ia;
+	bool holes_ok = ia->ri_mrtype == IB_MR_TYPE_SG_GAPS;
 	struct rpcrdma_mw *mw;
 	struct rpcrdma_frmr *frmr;
 	struct ib_mr *mr;
@@ -383,8 +385,8 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
 
 		++seg;
 		++i;
-
-		/* Check for holes */
+		if (holes_ok)
+			continue;
 		if ((i < nsegs && offset_in_page(seg->mr_offset)) ||
 		    offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))
 			break;
@@ -421,7 +423,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
 			 IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE :
 			 IB_ACCESS_REMOTE_READ;
 
-	DECR_CQCOUNT(&r_xprt->rx_ep);
+	rpcrdma_set_signaled(&r_xprt->rx_ep, &reg_wr->wr);
 	rc = ib_post_send(ia->ri_id->qp, &reg_wr->wr, &bad_wr);
 	if (rc)
 		goto out_senderr;
@@ -451,26 +453,6 @@ out_senderr:
 	return -ENOTCONN;
 }
 
-static struct ib_send_wr *
-__frwr_prepare_linv_wr(struct rpcrdma_mw *mw)
-{
-	struct rpcrdma_frmr *f = &mw->frmr;
-	struct ib_send_wr *invalidate_wr;
-
-	dprintk("RPC:       %s: invalidating frmr %p\n", __func__, f);
-
-	f->fr_state = FRMR_IS_INVALID;
-	invalidate_wr = &f->fr_invwr;
-
-	memset(invalidate_wr, 0, sizeof(*invalidate_wr));
-	f->fr_cqe.done = frwr_wc_localinv;
-	invalidate_wr->wr_cqe = &f->fr_cqe;
-	invalidate_wr->opcode = IB_WR_LOCAL_INV;
-	invalidate_wr->ex.invalidate_rkey = f->fr_mr->rkey;
-
-	return invalidate_wr;
-}
-
 /* Invalidate all memory regions that were registered for "req".
  *
  * Sleeps until it is safe for the host CPU to access the
@@ -481,12 +463,12 @@ __frwr_prepare_linv_wr(struct rpcrdma_mw *mw)
 static void
 frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
 {
-	struct ib_send_wr *invalidate_wrs, *pos, *prev, *bad_wr;
+	struct ib_send_wr *first, **prev, *last, *bad_wr;
 	struct rpcrdma_rep *rep = req->rl_reply;
 	struct rpcrdma_ia *ia = &r_xprt->rx_ia;
 	struct rpcrdma_mw *mw, *tmp;
 	struct rpcrdma_frmr *f;
-	int rc;
+	int count, rc;
 
 	dprintk("RPC:       %s: req %p\n", __func__, req);
 
@@ -496,22 +478,29 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
 	 * a single ib_post_send() call.
 	 */
 	f = NULL;
-	invalidate_wrs = pos = prev = NULL;
+	count = 0;
+	prev = &first;
 	list_for_each_entry(mw, &req->rl_registered, mw_list) {
+		mw->frmr.fr_state = FRMR_IS_INVALID;
+
 		if ((rep->rr_wc_flags & IB_WC_WITH_INVALIDATE) &&
-		    (mw->mw_handle == rep->rr_inv_rkey)) {
-			mw->frmr.fr_state = FRMR_IS_INVALID;
+		    (mw->mw_handle == rep->rr_inv_rkey))
 			continue;
-		}
-
-		pos = __frwr_prepare_linv_wr(mw);
 
-		if (!invalidate_wrs)
-			invalidate_wrs = pos;
-		else
-			prev->next = pos;
-		prev = pos;
 		f = &mw->frmr;
+		dprintk("RPC:       %s: invalidating frmr %p\n",
+			__func__, f);
+
+		f->fr_cqe.done = frwr_wc_localinv;
+		last = &f->fr_invwr;
+		memset(last, 0, sizeof(*last));
+		last->wr_cqe = &f->fr_cqe;
+		last->opcode = IB_WR_LOCAL_INV;
+		last->ex.invalidate_rkey = mw->mw_handle;
+		count++;
+
+		*prev = last;
+		prev = &last->next;
 	}
 	if (!f)
 		goto unmap;
@@ -520,17 +509,22 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
 	 * last WR in the chain completes, all WRs in the chain
 	 * are complete.
 	 */
-	f->fr_invwr.send_flags = IB_SEND_SIGNALED;
+	last->send_flags = IB_SEND_SIGNALED;
 	f->fr_cqe.done = frwr_wc_localinv_wake;
 	reinit_completion(&f->fr_linv_done);
-	INIT_CQCOUNT(&r_xprt->rx_ep);
+
+	/* Initialize CQ count, since there is always a signaled
+	 * WR being posted here.  The new cqcount depends on how
+	 * many SQEs are about to be consumed.
+	 */
+	rpcrdma_init_cqcount(&r_xprt->rx_ep, count);
 
 	/* Transport disconnect drains the receive CQ before it
 	 * replaces the QP. The RPC reply handler won't call us
 	 * unless ri_id->qp is a valid pointer.
 	 */
 	r_xprt->rx_stats.local_inv_needed++;
-	rc = ib_post_send(ia->ri_id->qp, invalidate_wrs, &bad_wr);
+	rc = ib_post_send(ia->ri_id->qp, first, &bad_wr);
 	if (rc)
 		goto reset_mrs;
 
@@ -541,7 +535,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
 	 */
 unmap:
 	list_for_each_entry_safe(mw, tmp, &req->rl_registered, mw_list) {
-		dprintk("RPC:       %s: unmapping frmr %p\n",
+		dprintk("RPC:       %s: DMA unmapping frmr %p\n",
 			__func__, &mw->frmr);
 		list_del_init(&mw->mw_list);
 		ib_dma_unmap_sg(ia->ri_device,
@@ -559,7 +553,7 @@ reset_mrs:
 	 */
 	list_for_each_entry(mw, &req->rl_registered, mw_list) {
 		f = &mw->frmr;
-		if (mw->frmr.fr_mr->rkey == bad_wr->ex.invalidate_rkey) {
+		if (mw->mw_handle == bad_wr->ex.invalidate_rkey) {
 			__frwr_reset_mr(ia, mw);
 			bad_wr = bad_wr->next;
 		}
diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
index d987c2d3dd6e..c52e0f2ffe52 100644
--- a/net/sunrpc/xprtrdma/rpc_rdma.c
+++ b/net/sunrpc/xprtrdma/rpc_rdma.c
@@ -786,7 +786,7 @@ rpcrdma_count_chunks(struct rpcrdma_rep *rep, int wrchunk, __be32 **iptrp)
 		ifdebug(FACILITY) {
 			u64 off;
 			xdr_decode_hyper((__be32 *)&seg->rs_offset, &off);
-			dprintk("RPC:       %s: chunk %d@0x%llx:0x%x\n",
+			dprintk("RPC:       %s: chunk %d@0x%016llx:0x%08x\n",
 				__func__,
 				be32_to_cpu(seg->rs_length),
 				(unsigned long long)off,
@@ -906,28 +906,6 @@ rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, int pad)
 	return fixup_copy_count;
 }
 
-void
-rpcrdma_connect_worker(struct work_struct *work)
-{
-	struct rpcrdma_ep *ep =
-		container_of(work, struct rpcrdma_ep, rep_connect_worker.work);
-	struct rpcrdma_xprt *r_xprt =
-		container_of(ep, struct rpcrdma_xprt, rx_ep);
-	struct rpc_xprt *xprt = &r_xprt->rx_xprt;
-
-	spin_lock_bh(&xprt->transport_lock);
-	if (++xprt->connect_cookie == 0)	/* maintain a reserved value */
-		++xprt->connect_cookie;
-	if (ep->rep_connected > 0) {
-		if (!xprt_test_and_set_connected(xprt))
-			xprt_wake_pending_tasks(xprt, 0);
-	} else {
-		if (xprt_test_and_clear_connected(xprt))
-			xprt_wake_pending_tasks(xprt, -ENOTCONN);
-	}
-	spin_unlock_bh(&xprt->transport_lock);
-}
-
 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
 /* By convention, backchannel calls arrive via rdma_msg type
  * messages, and never populate the chunk lists. This makes
@@ -959,18 +937,6 @@ rpcrdma_is_bcall(struct rpcrdma_msg *headerp)
 }
 #endif	/* CONFIG_SUNRPC_BACKCHANNEL */
 
-/*
- * This function is called when an async event is posted to
- * the connection which changes the connection state. All it
- * does at this point is mark the connection up/down, the rpc
- * timers do the rest.
- */
-void
-rpcrdma_conn_func(struct rpcrdma_ep *ep)
-{
-	schedule_delayed_work(&ep->rep_connect_worker, 0);
-}
-
 /* Process received RPC/RDMA messages.
  *
  * Errors must result in the RPC task either being awakened, or
diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c
index ed5e285fd2ea..534c178d2a7e 100644
--- a/net/sunrpc/xprtrdma/transport.c
+++ b/net/sunrpc/xprtrdma/transport.c
@@ -219,6 +219,34 @@ xprt_rdma_free_addresses(struct rpc_xprt *xprt)
 		}
 }
 
+void
+rpcrdma_conn_func(struct rpcrdma_ep *ep)
+{
+	schedule_delayed_work(&ep->rep_connect_worker, 0);
+}
+
+void
+rpcrdma_connect_worker(struct work_struct *work)
+{
+	struct rpcrdma_ep *ep =
+		container_of(work, struct rpcrdma_ep, rep_connect_worker.work);
+	struct rpcrdma_xprt *r_xprt =
+		container_of(ep, struct rpcrdma_xprt, rx_ep);
+	struct rpc_xprt *xprt = &r_xprt->rx_xprt;
+
+	spin_lock_bh(&xprt->transport_lock);
+	if (++xprt->connect_cookie == 0)	/* maintain a reserved value */
+		++xprt->connect_cookie;
+	if (ep->rep_connected > 0) {
+		if (!xprt_test_and_set_connected(xprt))
+			xprt_wake_pending_tasks(xprt, 0);
+	} else {
+		if (xprt_test_and_clear_connected(xprt))
+			xprt_wake_pending_tasks(xprt, -ENOTCONN);
+	}
+	spin_unlock_bh(&xprt->transport_lock);
+}
+
 static void
 xprt_rdma_connect_worker(struct work_struct *work)
 {
@@ -621,7 +649,8 @@ xprt_rdma_free(struct rpc_task *task)
 
 	dprintk("RPC:       %s: called on 0x%p\n", __func__, req->rl_reply);
 
-	ia->ri_ops->ro_unmap_safe(r_xprt, req, !RPC_IS_ASYNC(task));
+	if (unlikely(!list_empty(&req->rl_registered)))
+		ia->ri_ops->ro_unmap_safe(r_xprt, req, !RPC_IS_ASYNC(task));
 	rpcrdma_unmap_sges(ia, req);
 	rpcrdma_buffer_put(req);
 }
@@ -657,7 +686,8 @@ xprt_rdma_send_request(struct rpc_task *task)
 	int rc = 0;
 
 	/* On retransmit, remove any previously registered chunks */
-	r_xprt->rx_ia.ri_ops->ro_unmap_safe(r_xprt, req, false);
+	if (unlikely(!list_empty(&req->rl_registered)))
+		r_xprt->rx_ia.ri_ops->ro_unmap_safe(r_xprt, req, false);
 
 	rc = rpcrdma_marshal_req(rqst);
 	if (rc < 0)
diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index ec74289af7ec..11d07748f699 100644
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -103,9 +103,9 @@ rpcrdma_qp_async_error_upcall(struct ib_event *event, void *context)
 {
 	struct rpcrdma_ep *ep = context;
 
-	pr_err("RPC:       %s: %s on device %s ep %p\n",
-	       __func__, ib_event_msg(event->event),
-		event->device->name, context);
+	pr_err("rpcrdma: %s on device %s ep %p\n",
+	       ib_event_msg(event->event), event->device->name, context);
+
 	if (ep->rep_connected == 1) {
 		ep->rep_connected = -EIO;
 		rpcrdma_conn_func(ep);
@@ -223,8 +223,8 @@ rpcrdma_update_connect_private(struct rpcrdma_xprt *r_xprt,
 		cdata->inline_rsize = rsize;
 	if (wsize < cdata->inline_wsize)
 		cdata->inline_wsize = wsize;
-	pr_info("rpcrdma: max send %u, max recv %u\n",
-		cdata->inline_wsize, cdata->inline_rsize);
+	dprintk("RPC:       %s: max send %u, max recv %u\n",
+		__func__, cdata->inline_wsize, cdata->inline_rsize);
 	rpcrdma_set_max_header_sizes(r_xprt);
 }
 
@@ -331,6 +331,7 @@ static struct rdma_cm_id *
 rpcrdma_create_id(struct rpcrdma_xprt *xprt,
 			struct rpcrdma_ia *ia, struct sockaddr *addr)
 {
+	unsigned long wtimeout = msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1;
 	struct rdma_cm_id *id;
 	int rc;
 
@@ -352,8 +353,12 @@ rpcrdma_create_id(struct rpcrdma_xprt *xprt,
 			__func__, rc);
 		goto out;
 	}
-	wait_for_completion_interruptible_timeout(&ia->ri_done,
-				msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
+	rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout);
+	if (rc < 0) {
+		dprintk("RPC:       %s: wait() exited: %i\n",
+			__func__, rc);
+		goto out;
+	}
 
 	/* FIXME:
 	 * Until xprtrdma supports DEVICE_REMOVAL, the provider must
@@ -376,8 +381,12 @@ rpcrdma_create_id(struct rpcrdma_xprt *xprt,
 			__func__, rc);
 		goto put;
 	}
-	wait_for_completion_interruptible_timeout(&ia->ri_done,
-				msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
+	rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout);
+	if (rc < 0) {
+		dprintk("RPC:       %s: wait() exited: %i\n",
+			__func__, rc);
+		goto put;
+	}
 	rc = ia->ri_async_rc;
 	if (rc)
 		goto put;
@@ -532,7 +541,7 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
 	ep->rep_cqinit = ep->rep_attr.cap.max_send_wr/2 - 1;
 	if (ep->rep_cqinit <= 2)
 		ep->rep_cqinit = 0;	/* always signal? */
-	INIT_CQCOUNT(ep);
+	rpcrdma_init_cqcount(ep, 0);
 	init_waitqueue_head(&ep->rep_connect_wait);
 	INIT_DELAYED_WORK(&ep->rep_connect_worker, rpcrdma_connect_worker);
 
@@ -1311,13 +1320,7 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia,
 	dprintk("RPC:       %s: posting %d s/g entries\n",
 		__func__, send_wr->num_sge);
 
-	if (DECR_CQCOUNT(ep) > 0)
-		send_wr->send_flags = 0;
-	else { /* Provider must take a send completion every now and then */
-		INIT_CQCOUNT(ep);
-		send_wr->send_flags = IB_SEND_SIGNALED;
-	}
-
+	rpcrdma_set_signaled(ep, send_wr);
 	rc = ib_post_send(ia->ri_id->qp, send_wr, &send_wr_fail);
 	if (rc)
 		goto out_postsend_err;
diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index 6e1bba358203..e35efd4ac1e4 100644
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -75,6 +75,7 @@ struct rpcrdma_ia {
 	unsigned int		ri_max_inline_write;
 	unsigned int		ri_max_inline_read;
 	bool			ri_reminv_expected;
+	enum ib_mr_type		ri_mrtype;
 	struct ib_qp_attr	ri_qp_attr;
 	struct ib_qp_init_attr	ri_qp_init_attr;
 };
@@ -95,8 +96,24 @@ struct rpcrdma_ep {
 	struct delayed_work	rep_connect_worker;
 };
 
-#define INIT_CQCOUNT(ep) atomic_set(&(ep)->rep_cqcount, (ep)->rep_cqinit)
-#define DECR_CQCOUNT(ep) atomic_sub_return(1, &(ep)->rep_cqcount)
+static inline void
+rpcrdma_init_cqcount(struct rpcrdma_ep *ep, int count)
+{
+	atomic_set(&ep->rep_cqcount, ep->rep_cqinit - count);
+}
+
+/* To update send queue accounting, provider must take a
+ * send completion every now and then.
+ */
+static inline void
+rpcrdma_set_signaled(struct rpcrdma_ep *ep, struct ib_send_wr *send_wr)
+{
+	send_wr->send_flags = 0;
+	if (unlikely(atomic_sub_return(1, &ep->rep_cqcount) <= 0)) {
+		rpcrdma_init_cqcount(ep, 0);
+		send_wr->send_flags = IB_SEND_SIGNALED;
+	}
+}
 
 /* Pre-allocate extra Work Requests for handling backward receives
  * and sends. This is a fixed value because the Work Queues are
@@ -473,6 +490,7 @@ int rpcrdma_ep_create(struct rpcrdma_ep *, struct rpcrdma_ia *,
 				struct rpcrdma_create_data_internal *);
 void rpcrdma_ep_destroy(struct rpcrdma_ep *, struct rpcrdma_ia *);
 int rpcrdma_ep_connect(struct rpcrdma_ep *, struct rpcrdma_ia *);
+void rpcrdma_conn_func(struct rpcrdma_ep *ep);
 void rpcrdma_ep_disconnect(struct rpcrdma_ep *, struct rpcrdma_ia *);
 
 int rpcrdma_ep_post(struct rpcrdma_ia *, struct rpcrdma_ep *,
@@ -532,13 +550,6 @@ rpcrdma_data_dir(bool writing)
 }
 
 /*
- * RPC/RDMA connection management calls - xprtrdma/rpc_rdma.c
- */
-void rpcrdma_connect_worker(struct work_struct *);
-void rpcrdma_conn_func(struct rpcrdma_ep *);
-void rpcrdma_reply_handler(struct work_struct *);
-
-/*
  * RPC/RDMA protocol calls - xprtrdma/rpc_rdma.c
  */
 
@@ -555,12 +566,14 @@ bool rpcrdma_prepare_send_sges(struct rpcrdma_ia *, struct rpcrdma_req *,
 void rpcrdma_unmap_sges(struct rpcrdma_ia *, struct rpcrdma_req *);
 int rpcrdma_marshal_req(struct rpc_rqst *);
 void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *);
+void rpcrdma_reply_handler(struct work_struct *work);
 
 /* RPC/RDMA module init - xprtrdma/transport.c
  */
 extern unsigned int xprt_rdma_max_inline_read;
 void xprt_rdma_format_addresses(struct rpc_xprt *xprt, struct sockaddr *sap);
 void xprt_rdma_free_addresses(struct rpc_xprt *xprt);
+void rpcrdma_connect_worker(struct work_struct *work);
 void xprt_rdma_print_stats(struct rpc_xprt *xprt, struct seq_file *seq);
 int xprt_rdma_init(void);
 void xprt_rdma_cleanup(void);
diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c
index 2e47f9f06b96..6788264acc63 100644
--- a/net/vmw_vsock/virtio_transport.c
+++ b/net/vmw_vsock/virtio_transport.c
@@ -379,7 +379,7 @@ static void virtio_vsock_reset_sock(struct sock *sk)
 static void virtio_vsock_update_guest_cid(struct virtio_vsock *vsock)
 {
 	struct virtio_device *vdev = vsock->vdev;
-	u64 guest_cid;
+	__le64 guest_cid;
 
 	vdev->config->get(vdev, offsetof(struct virtio_vsock_config, guest_cid),
 			  &guest_cid, sizeof(guest_cid));
diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
index 687e9fdb3d67..849c4ad0411e 100644
--- a/net/vmw_vsock/virtio_transport_common.c
+++ b/net/vmw_vsock/virtio_transport_common.c
@@ -32,7 +32,7 @@ static const struct virtio_transport *virtio_transport_get_ops(void)
 	return container_of(t, struct virtio_transport, transport);
 }
 
-struct virtio_vsock_pkt *
+static struct virtio_vsock_pkt *
 virtio_transport_alloc_pkt(struct virtio_vsock_pkt_info *info,
 			   size_t len,
 			   u32 src_cid,
@@ -82,7 +82,6 @@ out_pkt:
 	kfree(pkt);
 	return NULL;
 }
-EXPORT_SYMBOL_GPL(virtio_transport_alloc_pkt);
 
 static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
 					  struct virtio_vsock_pkt_info *info)
@@ -606,9 +605,9 @@ static int virtio_transport_reset_no_sock(struct virtio_vsock_pkt *pkt)
 		return 0;
 
 	pkt = virtio_transport_alloc_pkt(&info, 0,
-					 le32_to_cpu(pkt->hdr.dst_cid),
+					 le64_to_cpu(pkt->hdr.dst_cid),
 					 le32_to_cpu(pkt->hdr.dst_port),
-					 le32_to_cpu(pkt->hdr.src_cid),
+					 le64_to_cpu(pkt->hdr.src_cid),
 					 le32_to_cpu(pkt->hdr.src_port));
 	if (!pkt)
 		return -ENOMEM;
@@ -823,7 +822,7 @@ virtio_transport_send_response(struct vsock_sock *vsk,
 	struct virtio_vsock_pkt_info info = {
 		.op = VIRTIO_VSOCK_OP_RESPONSE,
 		.type = VIRTIO_VSOCK_TYPE_STREAM,
-		.remote_cid = le32_to_cpu(pkt->hdr.src_cid),
+		.remote_cid = le64_to_cpu(pkt->hdr.src_cid),
 		.remote_port = le32_to_cpu(pkt->hdr.src_port),
 		.reply = true,
 	};
@@ -863,9 +862,9 @@ virtio_transport_recv_listen(struct sock *sk, struct virtio_vsock_pkt *pkt)
 	child->sk_state = SS_CONNECTED;
 
 	vchild = vsock_sk(child);
-	vsock_addr_init(&vchild->local_addr, le32_to_cpu(pkt->hdr.dst_cid),
+	vsock_addr_init(&vchild->local_addr, le64_to_cpu(pkt->hdr.dst_cid),
 			le32_to_cpu(pkt->hdr.dst_port));
-	vsock_addr_init(&vchild->remote_addr, le32_to_cpu(pkt->hdr.src_cid),
+	vsock_addr_init(&vchild->remote_addr, le64_to_cpu(pkt->hdr.src_cid),
 			le32_to_cpu(pkt->hdr.src_port));
 
 	vsock_insert_connected(vchild);
@@ -904,9 +903,9 @@ void virtio_transport_recv_pkt(struct virtio_vsock_pkt *pkt)
 	struct sock *sk;
 	bool space_available;
 
-	vsock_addr_init(&src, le32_to_cpu(pkt->hdr.src_cid),
+	vsock_addr_init(&src, le64_to_cpu(pkt->hdr.src_cid),
 			le32_to_cpu(pkt->hdr.src_port));
-	vsock_addr_init(&dst, le32_to_cpu(pkt->hdr.dst_cid),
+	vsock_addr_init(&dst, le64_to_cpu(pkt->hdr.dst_cid),
 			le32_to_cpu(pkt->hdr.dst_port));
 
 	trace_virtio_transport_recv_pkt(src.svm_cid, src.svm_port,
diff --git a/net/wireless/Makefile b/net/wireless/Makefile
index 4c9e39f04ef8..816c9331c8d2 100644
--- a/net/wireless/Makefile
+++ b/net/wireless/Makefile
@@ -17,8 +17,6 @@ cfg80211-$(CONFIG_CFG80211_INTERNAL_REGDB) += regdb.o
 
 CFLAGS_trace.o := -I$(src)
 
-ccflags-y += -D__CHECK_ENDIAN__
-
 $(obj)/regdb.c: $(src)/db.txt $(src)/genregdb.awk
 	@$(AWK) -f $(srctree)/$(src)/genregdb.awk < $< > $@
 
diff --git a/samples/seccomp/Makefile b/samples/seccomp/Makefile
index ae7ff6f24f36..bf7cc6b0dc19 100644
--- a/samples/seccomp/Makefile
+++ b/samples/seccomp/Makefile
@@ -36,13 +36,13 @@ HOSTLOADLIBES_bpf-direct += $(MFLAG)
 HOSTLOADLIBES_bpf-fancy += $(MFLAG)
 HOSTLOADLIBES_dropper += $(MFLAG)
 endif
-always := $(hostprogs-y)
+always := $(hostprogs-m)
 else
 # MIPS system calls are defined based on the -mabi that is passed
 # to the toolchain which may or may not be a valid option
 # for the host toolchain. So disable tests if target architecture
 # is MIPS but the host isn't.
 ifndef CONFIG_MIPS
-always := $(hostprogs-y)
+always := $(hostprogs-m)
 endif
 endif
diff --git a/samples/seccomp/bpf-helper.c b/samples/seccomp/bpf-helper.c
index 05cb4d5ff9f5..1ef0f4d72898 100644
--- a/samples/seccomp/bpf-helper.c
+++ b/samples/seccomp/bpf-helper.c
@@ -18,41 +18,41 @@
 int bpf_resolve_jumps(struct bpf_labels *labels,
 		      struct sock_filter *filter, size_t count)
 {
-	struct sock_filter *begin = filter;
-	__u8 insn = count - 1;
+	size_t i;
 
-	if (count < 1)
+	if (count < 1 || count > BPF_MAXINSNS)
 		return -1;
 	/*
 	* Walk it once, backwards, to build the label table and do fixups.
 	* Since backward jumps are disallowed by BPF, this is easy.
 	*/
-	filter += insn;
-	for (; filter >= begin; --insn, --filter) {
-		if (filter->code != (BPF_JMP+BPF_JA))
+	for (i = 0; i < count; ++i) {
+		size_t offset = count - i - 1;
+		struct sock_filter *instr = &filter[offset];
+		if (instr->code != (BPF_JMP+BPF_JA))
 			continue;
-		switch ((filter->jt<<8)|filter->jf) {
+		switch ((instr->jt<<8)|instr->jf) {
 		case (JUMP_JT<<8)|JUMP_JF:
-			if (labels->labels[filter->k].location == 0xffffffff) {
+			if (labels->labels[instr->k].location == 0xffffffff) {
 				fprintf(stderr, "Unresolved label: '%s'\n",
-					labels->labels[filter->k].label);
+					labels->labels[instr->k].label);
 				return 1;
 			}
-			filter->k = labels->labels[filter->k].location -
-				    (insn + 1);
-			filter->jt = 0;
-			filter->jf = 0;
+			instr->k = labels->labels[instr->k].location -
+				    (offset + 1);
+			instr->jt = 0;
+			instr->jf = 0;
 			continue;
 		case (LABEL_JT<<8)|LABEL_JF:
-			if (labels->labels[filter->k].location != 0xffffffff) {
+			if (labels->labels[instr->k].location != 0xffffffff) {
 				fprintf(stderr, "Duplicate label use: '%s'\n",
-					labels->labels[filter->k].label);
+					labels->labels[instr->k].label);
 				return 1;
 			}
-			labels->labels[filter->k].location = insn;
-			filter->k = 0; /* fall through */
-			filter->jt = 0;
-			filter->jf = 0;
+			labels->labels[instr->k].location = offset;
+			instr->k = 0; /* fall through */
+			instr->jt = 0;
+			instr->jf = 0;
 			continue;
 		}
 	}
diff --git a/samples/seccomp/dropper.c b/samples/seccomp/dropper.c
index c69c347c7011..68325ca5e71c 100644
--- a/samples/seccomp/dropper.c
+++ b/samples/seccomp/dropper.c
@@ -11,7 +11,6 @@
  * When run, returns the specified errno for the specified
  * system call number against the given architecture.
  *
- * Run this one as root as PR_SET_NO_NEW_PRIVS is not called.
  */
 
 #include <errno.h>
@@ -42,8 +41,12 @@ static int install_filter(int nr, int arch, int error)
 		.len = (unsigned short)(sizeof(filter)/sizeof(filter[0])),
 		.filter = filter,
 	};
+	if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) {
+		perror("prctl(NO_NEW_PRIVS)");
+		return 1;
+	}
 	if (prctl(PR_SET_SECCOMP, 2, &prog)) {
-		perror("prctl");
+		perror("prctl(PR_SET_SECCOMP)");
 		return 1;
 	}
 	return 0;
diff --git a/samples/trace_events/trace-events-sample.c b/samples/trace_events/trace-events-sample.c
index 880a7d1d27d2..30e282d33d4d 100644
--- a/samples/trace_events/trace-events-sample.c
+++ b/samples/trace_events/trace-events-sample.c
@@ -79,7 +79,7 @@ static int simple_thread_fn(void *arg)
 
 static DEFINE_MUTEX(thread_mutex);
 
-void foo_bar_reg(void)
+int foo_bar_reg(void)
 {
 	pr_info("Starting thread for foo_bar_fn\n");
 	/*
@@ -90,6 +90,7 @@ void foo_bar_reg(void)
 	mutex_lock(&thread_mutex);
 	simple_tsk_fn = kthread_run(simple_thread_fn, NULL, "event-sample-fn");
 	mutex_unlock(&thread_mutex);
+	return 0;
 }
 
 void foo_bar_unreg(void)
diff --git a/samples/trace_events/trace-events-sample.h b/samples/trace_events/trace-events-sample.h
index d6b75bb495b3..76a75ab7a608 100644
--- a/samples/trace_events/trace-events-sample.h
+++ b/samples/trace_events/trace-events-sample.h
@@ -354,7 +354,7 @@ TRACE_EVENT_CONDITION(foo_bar_with_cond,
 	TP_printk("foo %s %d", __get_str(foo), __entry->bar)
 );
 
-void foo_bar_reg(void);
+int foo_bar_reg(void);
 void foo_bar_unreg(void);
 
 /*
diff --git a/samples/vfio-mdev/Makefile b/samples/vfio-mdev/Makefile
new file mode 100644
index 000000000000..a932edbe38eb
--- /dev/null
+++ b/samples/vfio-mdev/Makefile
@@ -0,0 +1,13 @@
+#
+# Makefile for mtty.c file
+#
+KERNEL_DIR:=/lib/modules/$(shell uname -r)/build
+
+obj-m:=mtty.o
+
+modules clean modules_install:
+	$(MAKE) -C $(KERNEL_DIR) SUBDIRS=$(PWD) $@
+
+default: modules
+
+module: modules
diff --git a/samples/vfio-mdev/mtty.c b/samples/vfio-mdev/mtty.c
new file mode 100644
index 000000000000..6b633a4ea333
--- /dev/null
+++ b/samples/vfio-mdev/mtty.c
@@ -0,0 +1,1503 @@
+/*
+ * Mediated virtual PCI serial host device driver
+ *
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ *     Author: Neo Jia <cjia@nvidia.com>
+ *             Kirti Wankhede <kwankhede@nvidia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Sample driver that creates mdev device that simulates serial port over PCI
+ * card.
+ *
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/cdev.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/uuid.h>
+#include <linux/vfio.h>
+#include <linux/iommu.h>
+#include <linux/sysfs.h>
+#include <linux/ctype.h>
+#include <linux/file.h>
+#include <linux/mdev.h>
+#include <linux/pci.h>
+#include <linux/serial.h>
+#include <uapi/linux/serial_reg.h>
+#include <linux/eventfd.h>
+/*
+ * #defines
+ */
+
+#define VERSION_STRING  "0.1"
+#define DRIVER_AUTHOR   "NVIDIA Corporation"
+
+#define MTTY_CLASS_NAME "mtty"
+
+#define MTTY_NAME       "mtty"
+
+#define MTTY_STRING_LEN		16
+
+#define MTTY_CONFIG_SPACE_SIZE  0xff
+#define MTTY_IO_BAR_SIZE        0x8
+#define MTTY_MMIO_BAR_SIZE      0x100000
+
+#define STORE_LE16(addr, val)   (*(u16 *)addr = val)
+#define STORE_LE32(addr, val)   (*(u32 *)addr = val)
+
+#define MAX_FIFO_SIZE   16
+
+#define CIRCULAR_BUF_INC_IDX(idx)    (idx = (idx + 1) & (MAX_FIFO_SIZE - 1))
+
+#define MTTY_VFIO_PCI_OFFSET_SHIFT   40
+
+#define MTTY_VFIO_PCI_OFFSET_TO_INDEX(off)   (off >> MTTY_VFIO_PCI_OFFSET_SHIFT)
+#define MTTY_VFIO_PCI_INDEX_TO_OFFSET(index) \
+				((u64)(index) << MTTY_VFIO_PCI_OFFSET_SHIFT)
+#define MTTY_VFIO_PCI_OFFSET_MASK    \
+				(((u64)(1) << MTTY_VFIO_PCI_OFFSET_SHIFT) - 1)
+#define MAX_MTTYS	24
+
+/*
+ * Global Structures
+ */
+
+struct mtty_dev {
+	dev_t		vd_devt;
+	struct class	*vd_class;
+	struct cdev	vd_cdev;
+	struct idr	vd_idr;
+	struct device	dev;
+} mtty_dev;
+
+struct mdev_region_info {
+	u64 start;
+	u64 phys_start;
+	u32 size;
+	u64 vfio_offset;
+};
+
+#if defined(DEBUG_REGS)
+const char *wr_reg[] = {
+	"TX",
+	"IER",
+	"FCR",
+	"LCR",
+	"MCR",
+	"LSR",
+	"MSR",
+	"SCR"
+};
+
+const char *rd_reg[] = {
+	"RX",
+	"IER",
+	"IIR",
+	"LCR",
+	"MCR",
+	"LSR",
+	"MSR",
+	"SCR"
+};
+#endif
+
+/* loop back buffer */
+struct rxtx {
+	u8 fifo[MAX_FIFO_SIZE];
+	u8 head, tail;
+	u8 count;
+};
+
+struct serial_port {
+	u8 uart_reg[8];         /* 8 registers */
+	struct rxtx rxtx;       /* loop back buffer */
+	bool dlab;
+	bool overrun;
+	u16 divisor;
+	u8 fcr;                 /* FIFO control register */
+	u8 max_fifo_size;
+	u8 intr_trigger_level;  /* interrupt trigger level */
+};
+
+/* State of each mdev device */
+struct mdev_state {
+	int irq_fd;
+	struct eventfd_ctx *intx_evtfd;
+	struct eventfd_ctx *msi_evtfd;
+	int irq_index;
+	u8 *vconfig;
+	struct mutex ops_lock;
+	struct mdev_device *mdev;
+	struct mdev_region_info region_info[VFIO_PCI_NUM_REGIONS];
+	u32 bar_mask[VFIO_PCI_NUM_REGIONS];
+	struct list_head next;
+	struct serial_port s[2];
+	struct mutex rxtx_lock;
+	struct vfio_device_info dev_info;
+	int nr_ports;
+};
+
+struct mutex mdev_list_lock;
+struct list_head mdev_devices_list;
+
+static const struct file_operations vd_fops = {
+	.owner          = THIS_MODULE,
+};
+
+/* function prototypes */
+
+static int mtty_trigger_interrupt(uuid_le uuid);
+
+/* Helper functions */
+static struct mdev_state *find_mdev_state_by_uuid(uuid_le uuid)
+{
+	struct mdev_state *mds;
+
+	list_for_each_entry(mds, &mdev_devices_list, next) {
+		if (uuid_le_cmp(mds->mdev->uuid, uuid) == 0)
+			return mds;
+	}
+
+	return NULL;
+}
+
+void dump_buffer(char *buf, uint32_t count)
+{
+#if defined(DEBUG)
+	int i;
+
+	pr_info("Buffer:\n");
+	for (i = 0; i < count; i++) {
+		pr_info("%2x ", *(buf + i));
+		if ((i + 1) % 16 == 0)
+			pr_info("\n");
+	}
+#endif
+}
+
+static void mtty_create_config_space(struct mdev_state *mdev_state)
+{
+	/* PCI dev ID */
+	STORE_LE32((u32 *) &mdev_state->vconfig[0x0], 0x32534348);
+
+	/* Control: I/O+, Mem-, BusMaster- */
+	STORE_LE16((u16 *) &mdev_state->vconfig[0x4], 0x0001);
+
+	/* Status: capabilities list absent */
+	STORE_LE16((u16 *) &mdev_state->vconfig[0x6], 0x0200);
+
+	/* Rev ID */
+	mdev_state->vconfig[0x8] =  0x10;
+
+	/* programming interface class : 16550-compatible serial controller */
+	mdev_state->vconfig[0x9] =  0x02;
+
+	/* Sub class : 00 */
+	mdev_state->vconfig[0xa] =  0x00;
+
+	/* Base class : Simple Communication controllers */
+	mdev_state->vconfig[0xb] =  0x07;
+
+	/* base address registers */
+	/* BAR0: IO space */
+	STORE_LE32((u32 *) &mdev_state->vconfig[0x10], 0x000001);
+	mdev_state->bar_mask[0] = ~(MTTY_IO_BAR_SIZE) + 1;
+
+	if (mdev_state->nr_ports == 2) {
+		/* BAR1: IO space */
+		STORE_LE32((u32 *) &mdev_state->vconfig[0x14], 0x000001);
+		mdev_state->bar_mask[1] = ~(MTTY_IO_BAR_SIZE) + 1;
+	}
+
+	/* Subsystem ID */
+	STORE_LE32((u32 *) &mdev_state->vconfig[0x2c], 0x32534348);
+
+	mdev_state->vconfig[0x34] =  0x00;   /* Cap Ptr */
+	mdev_state->vconfig[0x3d] =  0x01;   /* interrupt pin (INTA#) */
+
+	/* Vendor specific data */
+	mdev_state->vconfig[0x40] =  0x23;
+	mdev_state->vconfig[0x43] =  0x80;
+	mdev_state->vconfig[0x44] =  0x23;
+	mdev_state->vconfig[0x48] =  0x23;
+	mdev_state->vconfig[0x4c] =  0x23;
+
+	mdev_state->vconfig[0x60] =  0x50;
+	mdev_state->vconfig[0x61] =  0x43;
+	mdev_state->vconfig[0x62] =  0x49;
+	mdev_state->vconfig[0x63] =  0x20;
+	mdev_state->vconfig[0x64] =  0x53;
+	mdev_state->vconfig[0x65] =  0x65;
+	mdev_state->vconfig[0x66] =  0x72;
+	mdev_state->vconfig[0x67] =  0x69;
+	mdev_state->vconfig[0x68] =  0x61;
+	mdev_state->vconfig[0x69] =  0x6c;
+	mdev_state->vconfig[0x6a] =  0x2f;
+	mdev_state->vconfig[0x6b] =  0x55;
+	mdev_state->vconfig[0x6c] =  0x41;
+	mdev_state->vconfig[0x6d] =  0x52;
+	mdev_state->vconfig[0x6e] =  0x54;
+}
+
+static void handle_pci_cfg_write(struct mdev_state *mdev_state, u16 offset,
+				 char *buf, u32 count)
+{
+	u32 cfg_addr, bar_mask, bar_index = 0;
+
+	switch (offset) {
+	case 0x04: /* device control */
+	case 0x06: /* device status */
+		/* do nothing */
+		break;
+	case 0x3c:  /* interrupt line */
+		mdev_state->vconfig[0x3c] = buf[0];
+		break;
+	case 0x3d:
+		/*
+		 * Interrupt Pin is hardwired to INTA.
+		 * This field is write protected by hardware
+		 */
+		break;
+	case 0x10:  /* BAR0 */
+	case 0x14:  /* BAR1 */
+		if (offset == 0x10)
+			bar_index = 0;
+		else if (offset == 0x14)
+			bar_index = 1;
+
+		if ((mdev_state->nr_ports == 1) && (bar_index == 1)) {
+			STORE_LE32(&mdev_state->vconfig[offset], 0);
+			break;
+		}
+
+		cfg_addr = *(u32 *)buf;
+		pr_info("BAR%d addr 0x%x\n", bar_index, cfg_addr);
+
+		if (cfg_addr == 0xffffffff) {
+			bar_mask = mdev_state->bar_mask[bar_index];
+			cfg_addr = (cfg_addr & bar_mask);
+		}
+
+		cfg_addr |= (mdev_state->vconfig[offset] & 0x3ul);
+		STORE_LE32(&mdev_state->vconfig[offset], cfg_addr);
+		break;
+	case 0x18:  /* BAR2 */
+	case 0x1c:  /* BAR3 */
+	case 0x20:  /* BAR4 */
+		STORE_LE32(&mdev_state->vconfig[offset], 0);
+		break;
+	default:
+		pr_info("PCI config write @0x%x of %d bytes not handled\n",
+			offset, count);
+		break;
+	}
+}
+
+static void handle_bar_write(unsigned int index, struct mdev_state *mdev_state,
+				u16 offset, char *buf, u32 count)
+{
+	u8 data = *buf;
+
+	/* Handle data written by guest */
+	switch (offset) {
+	case UART_TX:
+		/* if DLAB set, data is LSB of divisor */
+		if (mdev_state->s[index].dlab) {
+			mdev_state->s[index].divisor |= data;
+			break;
+		}
+
+		mutex_lock(&mdev_state->rxtx_lock);
+
+		/* save in TX buffer */
+		if (mdev_state->s[index].rxtx.count <
+				mdev_state->s[index].max_fifo_size) {
+			mdev_state->s[index].rxtx.fifo[
+					mdev_state->s[index].rxtx.head] = data;
+			mdev_state->s[index].rxtx.count++;
+			CIRCULAR_BUF_INC_IDX(mdev_state->s[index].rxtx.head);
+			mdev_state->s[index].overrun = false;
+
+			/*
+			 * Trigger interrupt if receive data interrupt is
+			 * enabled and fifo reached trigger level
+			 */
+			if ((mdev_state->s[index].uart_reg[UART_IER] &
+						UART_IER_RDI) &&
+			   (mdev_state->s[index].rxtx.count ==
+				    mdev_state->s[index].intr_trigger_level)) {
+				/* trigger interrupt */
+#if defined(DEBUG_INTR)
+				pr_err("Serial port %d: Fifo level trigger\n",
+					index);
+#endif
+				mtty_trigger_interrupt(mdev_state->mdev->uuid);
+			}
+		} else {
+#if defined(DEBUG_INTR)
+			pr_err("Serial port %d: Buffer Overflow\n", index);
+#endif
+			mdev_state->s[index].overrun = true;
+
+			/*
+			 * Trigger interrupt if receiver line status interrupt
+			 * is enabled
+			 */
+			if (mdev_state->s[index].uart_reg[UART_IER] &
+								UART_IER_RLSI)
+				mtty_trigger_interrupt(mdev_state->mdev->uuid);
+		}
+		mutex_unlock(&mdev_state->rxtx_lock);
+		break;
+
+	case UART_IER:
+		/* if DLAB set, data is MSB of divisor */
+		if (mdev_state->s[index].dlab)
+			mdev_state->s[index].divisor |= (u16)data << 8;
+		else {
+			mdev_state->s[index].uart_reg[offset] = data;
+			mutex_lock(&mdev_state->rxtx_lock);
+			if ((data & UART_IER_THRI) &&
+			    (mdev_state->s[index].rxtx.head ==
+					mdev_state->s[index].rxtx.tail)) {
+#if defined(DEBUG_INTR)
+				pr_err("Serial port %d: IER_THRI write\n",
+					index);
+#endif
+				mtty_trigger_interrupt(mdev_state->mdev->uuid);
+			}
+
+			mutex_unlock(&mdev_state->rxtx_lock);
+		}
+
+		break;
+
+	case UART_FCR:
+		mdev_state->s[index].fcr = data;
+
+		mutex_lock(&mdev_state->rxtx_lock);
+		if (data & (UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT)) {
+			/* clear loop back FIFO */
+			mdev_state->s[index].rxtx.count = 0;
+			mdev_state->s[index].rxtx.head = 0;
+			mdev_state->s[index].rxtx.tail = 0;
+		}
+		mutex_unlock(&mdev_state->rxtx_lock);
+
+		switch (data & UART_FCR_TRIGGER_MASK) {
+		case UART_FCR_TRIGGER_1:
+			mdev_state->s[index].intr_trigger_level = 1;
+			break;
+
+		case UART_FCR_TRIGGER_4:
+			mdev_state->s[index].intr_trigger_level = 4;
+			break;
+
+		case UART_FCR_TRIGGER_8:
+			mdev_state->s[index].intr_trigger_level = 8;
+			break;
+
+		case UART_FCR_TRIGGER_14:
+			mdev_state->s[index].intr_trigger_level = 14;
+			break;
+		}
+
+		/*
+		 * Set trigger level to 1 otherwise or  implement timer with
+		 * timeout of 4 characters and on expiring that timer set
+		 * Recevice data timeout in IIR register
+		 */
+		mdev_state->s[index].intr_trigger_level = 1;
+		if (data & UART_FCR_ENABLE_FIFO)
+			mdev_state->s[index].max_fifo_size = MAX_FIFO_SIZE;
+		else {
+			mdev_state->s[index].max_fifo_size = 1;
+			mdev_state->s[index].intr_trigger_level = 1;
+		}
+
+		break;
+
+	case UART_LCR:
+		if (data & UART_LCR_DLAB) {
+			mdev_state->s[index].dlab = true;
+			mdev_state->s[index].divisor = 0;
+		} else
+			mdev_state->s[index].dlab = false;
+
+		mdev_state->s[index].uart_reg[offset] = data;
+		break;
+
+	case UART_MCR:
+		mdev_state->s[index].uart_reg[offset] = data;
+
+		if ((mdev_state->s[index].uart_reg[UART_IER] & UART_IER_MSI) &&
+				(data & UART_MCR_OUT2)) {
+#if defined(DEBUG_INTR)
+			pr_err("Serial port %d: MCR_OUT2 write\n", index);
+#endif
+			mtty_trigger_interrupt(mdev_state->mdev->uuid);
+		}
+
+		if ((mdev_state->s[index].uart_reg[UART_IER] & UART_IER_MSI) &&
+				(data & (UART_MCR_RTS | UART_MCR_DTR))) {
+#if defined(DEBUG_INTR)
+			pr_err("Serial port %d: MCR RTS/DTR write\n", index);
+#endif
+			mtty_trigger_interrupt(mdev_state->mdev->uuid);
+		}
+		break;
+
+	case UART_LSR:
+	case UART_MSR:
+		/* do nothing */
+		break;
+
+	case UART_SCR:
+		mdev_state->s[index].uart_reg[offset] = data;
+		break;
+
+	default:
+		break;
+	}
+}
+
+static void handle_bar_read(unsigned int index, struct mdev_state *mdev_state,
+			    u16 offset, char *buf, u32 count)
+{
+	/* Handle read requests by guest */
+	switch (offset) {
+	case UART_RX:
+		/* if DLAB set, data is LSB of divisor */
+		if (mdev_state->s[index].dlab) {
+			*buf  = (u8)mdev_state->s[index].divisor;
+			break;
+		}
+
+		mutex_lock(&mdev_state->rxtx_lock);
+		/* return data in tx buffer */
+		if (mdev_state->s[index].rxtx.head !=
+				 mdev_state->s[index].rxtx.tail) {
+			*buf = mdev_state->s[index].rxtx.fifo[
+						mdev_state->s[index].rxtx.tail];
+			mdev_state->s[index].rxtx.count--;
+			CIRCULAR_BUF_INC_IDX(mdev_state->s[index].rxtx.tail);
+		}
+
+		if (mdev_state->s[index].rxtx.head ==
+				mdev_state->s[index].rxtx.tail) {
+		/*
+		 *  Trigger interrupt if tx buffer empty interrupt is
+		 *  enabled and fifo is empty
+		 */
+#if defined(DEBUG_INTR)
+			pr_err("Serial port %d: Buffer Empty\n", index);
+#endif
+			if (mdev_state->s[index].uart_reg[UART_IER] &
+							 UART_IER_THRI)
+				mtty_trigger_interrupt(mdev_state->mdev->uuid);
+		}
+		mutex_unlock(&mdev_state->rxtx_lock);
+
+		break;
+
+	case UART_IER:
+		if (mdev_state->s[index].dlab) {
+			*buf = (u8)(mdev_state->s[index].divisor >> 8);
+			break;
+		}
+		*buf = mdev_state->s[index].uart_reg[offset] & 0x0f;
+		break;
+
+	case UART_IIR:
+	{
+		u8 ier = mdev_state->s[index].uart_reg[UART_IER];
+		*buf = 0;
+
+		mutex_lock(&mdev_state->rxtx_lock);
+		/* Interrupt priority 1: Parity, overrun, framing or break */
+		if ((ier & UART_IER_RLSI) && mdev_state->s[index].overrun)
+			*buf |= UART_IIR_RLSI;
+
+		/* Interrupt priority 2: Fifo trigger level reached */
+		if ((ier & UART_IER_RDI) &&
+		    (mdev_state->s[index].rxtx.count ==
+		      mdev_state->s[index].intr_trigger_level))
+			*buf |= UART_IIR_RDI;
+
+		/* Interrupt priotiry 3: transmitter holding register empty */
+		if ((ier & UART_IER_THRI) &&
+		    (mdev_state->s[index].rxtx.head ==
+				mdev_state->s[index].rxtx.tail))
+			*buf |= UART_IIR_THRI;
+
+		/* Interrupt priotiry 4: Modem status: CTS, DSR, RI or DCD  */
+		if ((ier & UART_IER_MSI) &&
+		    (mdev_state->s[index].uart_reg[UART_MCR] &
+				 (UART_MCR_RTS | UART_MCR_DTR)))
+			*buf |= UART_IIR_MSI;
+
+		/* bit0: 0=> interrupt pending, 1=> no interrupt is pending */
+		if (*buf == 0)
+			*buf = UART_IIR_NO_INT;
+
+		/* set bit 6 & 7 to be 16550 compatible */
+		*buf |= 0xC0;
+		mutex_unlock(&mdev_state->rxtx_lock);
+	}
+	break;
+
+	case UART_LCR:
+	case UART_MCR:
+		*buf = mdev_state->s[index].uart_reg[offset];
+		break;
+
+	case UART_LSR:
+	{
+		u8 lsr = 0;
+
+		mutex_lock(&mdev_state->rxtx_lock);
+		/* atleast one char in FIFO */
+		if (mdev_state->s[index].rxtx.head !=
+				 mdev_state->s[index].rxtx.tail)
+			lsr |= UART_LSR_DR;
+
+		/* if FIFO overrun */
+		if (mdev_state->s[index].overrun)
+			lsr |= UART_LSR_OE;
+
+		/* transmit FIFO empty and tramsitter empty */
+		if (mdev_state->s[index].rxtx.head ==
+				 mdev_state->s[index].rxtx.tail)
+			lsr |= UART_LSR_TEMT | UART_LSR_THRE;
+
+		mutex_unlock(&mdev_state->rxtx_lock);
+		*buf = lsr;
+		break;
+	}
+	case UART_MSR:
+		*buf = UART_MSR_DSR | UART_MSR_DDSR | UART_MSR_DCD;
+
+		mutex_lock(&mdev_state->rxtx_lock);
+		/* if AFE is 1 and FIFO have space, set CTS bit */
+		if (mdev_state->s[index].uart_reg[UART_MCR] &
+						 UART_MCR_AFE) {
+			if (mdev_state->s[index].rxtx.count <
+					mdev_state->s[index].max_fifo_size)
+				*buf |= UART_MSR_CTS | UART_MSR_DCTS;
+		} else
+			*buf |= UART_MSR_CTS | UART_MSR_DCTS;
+		mutex_unlock(&mdev_state->rxtx_lock);
+
+		break;
+
+	case UART_SCR:
+		*buf = mdev_state->s[index].uart_reg[offset];
+		break;
+
+	default:
+		break;
+	}
+}
+
+static void mdev_read_base(struct mdev_state *mdev_state)
+{
+	int index, pos;
+	u32 start_lo, start_hi;
+	u32 mem_type;
+
+	pos = PCI_BASE_ADDRESS_0;
+
+	for (index = 0; index <= VFIO_PCI_BAR5_REGION_INDEX; index++) {
+
+		if (!mdev_state->region_info[index].size)
+			continue;
+
+		start_lo = (*(u32 *)(mdev_state->vconfig + pos)) &
+			PCI_BASE_ADDRESS_MEM_MASK;
+		mem_type = (*(u32 *)(mdev_state->vconfig + pos)) &
+			PCI_BASE_ADDRESS_MEM_TYPE_MASK;
+
+		switch (mem_type) {
+		case PCI_BASE_ADDRESS_MEM_TYPE_64:
+			start_hi = (*(u32 *)(mdev_state->vconfig + pos + 4));
+			pos += 4;
+			break;
+		case PCI_BASE_ADDRESS_MEM_TYPE_32:
+		case PCI_BASE_ADDRESS_MEM_TYPE_1M:
+			/* 1M mem BAR treated as 32-bit BAR */
+		default:
+			/* mem unknown type treated as 32-bit BAR */
+			start_hi = 0;
+			break;
+		}
+		pos += 4;
+		mdev_state->region_info[index].start = ((u64)start_hi << 32) |
+							start_lo;
+	}
+}
+
+static ssize_t mdev_access(struct mdev_device *mdev, char *buf, size_t count,
+			   loff_t pos, bool is_write)
+{
+	struct mdev_state *mdev_state;
+	unsigned int index;
+	loff_t offset;
+	int ret = 0;
+
+	if (!mdev || !buf)
+		return -EINVAL;
+
+	mdev_state = mdev_get_drvdata(mdev);
+	if (!mdev_state) {
+		pr_err("%s mdev_state not found\n", __func__);
+		return -EINVAL;
+	}
+
+	mutex_lock(&mdev_state->ops_lock);
+
+	index = MTTY_VFIO_PCI_OFFSET_TO_INDEX(pos);
+	offset = pos & MTTY_VFIO_PCI_OFFSET_MASK;
+	switch (index) {
+	case VFIO_PCI_CONFIG_REGION_INDEX:
+
+#if defined(DEBUG)
+		pr_info("%s: PCI config space %s at offset 0x%llx\n",
+			 __func__, is_write ? "write" : "read", offset);
+#endif
+		if (is_write) {
+			dump_buffer(buf, count);
+			handle_pci_cfg_write(mdev_state, offset, buf, count);
+		} else {
+			memcpy(buf, (mdev_state->vconfig + offset), count);
+			dump_buffer(buf, count);
+		}
+
+		break;
+
+	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
+		if (!mdev_state->region_info[index].start)
+			mdev_read_base(mdev_state);
+
+		if (is_write) {
+			dump_buffer(buf, count);
+
+#if defined(DEBUG_REGS)
+			pr_info("%s: BAR%d  WR @0x%llx %s val:0x%02x dlab:%d\n",
+				__func__, index, offset, wr_reg[offset],
+				(u8)*buf, mdev_state->s[index].dlab);
+#endif
+			handle_bar_write(index, mdev_state, offset, buf, count);
+		} else {
+			handle_bar_read(index, mdev_state, offset, buf, count);
+			dump_buffer(buf, count);
+
+#if defined(DEBUG_REGS)
+			pr_info("%s: BAR%d  RD @0x%llx %s val:0x%02x dlab:%d\n",
+				__func__, index, offset, rd_reg[offset],
+				(u8)*buf, mdev_state->s[index].dlab);
+#endif
+		}
+		break;
+
+	default:
+		ret = -1;
+		goto accessfailed;
+	}
+
+	ret = count;
+
+
+accessfailed:
+	mutex_unlock(&mdev_state->ops_lock);
+
+	return ret;
+}
+
+int mtty_create(struct kobject *kobj, struct mdev_device *mdev)
+{
+	struct mdev_state *mdev_state;
+	char name[MTTY_STRING_LEN];
+	int nr_ports = 0, i;
+
+	if (!mdev)
+		return -EINVAL;
+
+	for (i = 0; i < 2; i++) {
+		snprintf(name, MTTY_STRING_LEN, "%s-%d",
+			dev_driver_string(mdev->parent->dev), i + 1);
+		if (!strcmp(kobj->name, name)) {
+			nr_ports = i + 1;
+			break;
+		}
+	}
+
+	if (!nr_ports)
+		return -EINVAL;
+
+	mdev_state = kzalloc(sizeof(struct mdev_state), GFP_KERNEL);
+	if (mdev_state == NULL)
+		return -ENOMEM;
+
+	mdev_state->nr_ports = nr_ports;
+	mdev_state->irq_index = -1;
+	mdev_state->s[0].max_fifo_size = MAX_FIFO_SIZE;
+	mdev_state->s[1].max_fifo_size = MAX_FIFO_SIZE;
+	mutex_init(&mdev_state->rxtx_lock);
+	mdev_state->vconfig = kzalloc(MTTY_CONFIG_SPACE_SIZE, GFP_KERNEL);
+
+	if (mdev_state->vconfig == NULL) {
+		kfree(mdev_state);
+		return -ENOMEM;
+	}
+
+	mutex_init(&mdev_state->ops_lock);
+	mdev_state->mdev = mdev;
+	mdev_set_drvdata(mdev, mdev_state);
+
+	mtty_create_config_space(mdev_state);
+
+	mutex_lock(&mdev_list_lock);
+	list_add(&mdev_state->next, &mdev_devices_list);
+	mutex_unlock(&mdev_list_lock);
+
+	return 0;
+}
+
+int mtty_remove(struct mdev_device *mdev)
+{
+	struct mdev_state *mds, *tmp_mds;
+	struct mdev_state *mdev_state = mdev_get_drvdata(mdev);
+	int ret = -EINVAL;
+
+	mutex_lock(&mdev_list_lock);
+	list_for_each_entry_safe(mds, tmp_mds, &mdev_devices_list, next) {
+		if (mdev_state == mds) {
+			list_del(&mdev_state->next);
+			mdev_set_drvdata(mdev, NULL);
+			kfree(mdev_state->vconfig);
+			kfree(mdev_state);
+			ret = 0;
+			break;
+		}
+	}
+	mutex_unlock(&mdev_list_lock);
+
+	return ret;
+}
+
+int mtty_reset(struct mdev_device *mdev)
+{
+	struct mdev_state *mdev_state;
+
+	if (!mdev)
+		return -EINVAL;
+
+	mdev_state = mdev_get_drvdata(mdev);
+	if (!mdev_state)
+		return -EINVAL;
+
+	pr_info("%s: called\n", __func__);
+
+	return 0;
+}
+
+ssize_t mtty_read(struct mdev_device *mdev, char __user *buf, size_t count,
+		  loff_t *ppos)
+{
+	unsigned int done = 0;
+	int ret;
+
+	while (count) {
+		size_t filled;
+
+		if (count >= 4 && !(*ppos % 4)) {
+			u32 val;
+
+			ret =  mdev_access(mdev, (char *)&val, sizeof(val),
+					   *ppos, false);
+			if (ret <= 0)
+				goto read_err;
+
+			if (copy_to_user(buf, &val, sizeof(val)))
+				goto read_err;
+
+			filled = 4;
+		} else if (count >= 2 && !(*ppos % 2)) {
+			u16 val;
+
+			ret = mdev_access(mdev, (char *)&val, sizeof(val),
+					  *ppos, false);
+			if (ret <= 0)
+				goto read_err;
+
+			if (copy_to_user(buf, &val, sizeof(val)))
+				goto read_err;
+
+			filled = 2;
+		} else {
+			u8 val;
+
+			ret = mdev_access(mdev, (char *)&val, sizeof(val),
+					  *ppos, false);
+			if (ret <= 0)
+				goto read_err;
+
+			if (copy_to_user(buf, &val, sizeof(val)))
+				goto read_err;
+
+			filled = 1;
+		}
+
+		count -= filled;
+		done += filled;
+		*ppos += filled;
+		buf += filled;
+	}
+
+	return done;
+
+read_err:
+	return -EFAULT;
+}
+
+ssize_t mtty_write(struct mdev_device *mdev, const char __user *buf,
+		   size_t count, loff_t *ppos)
+{
+	unsigned int done = 0;
+	int ret;
+
+	while (count) {
+		size_t filled;
+
+		if (count >= 4 && !(*ppos % 4)) {
+			u32 val;
+
+			if (copy_from_user(&val, buf, sizeof(val)))
+				goto write_err;
+
+			ret = mdev_access(mdev, (char *)&val, sizeof(val),
+					  *ppos, true);
+			if (ret <= 0)
+				goto write_err;
+
+			filled = 4;
+		} else if (count >= 2 && !(*ppos % 2)) {
+			u16 val;
+
+			if (copy_from_user(&val, buf, sizeof(val)))
+				goto write_err;
+
+			ret = mdev_access(mdev, (char *)&val, sizeof(val),
+					  *ppos, true);
+			if (ret <= 0)
+				goto write_err;
+
+			filled = 2;
+		} else {
+			u8 val;
+
+			if (copy_from_user(&val, buf, sizeof(val)))
+				goto write_err;
+
+			ret = mdev_access(mdev, (char *)&val, sizeof(val),
+					  *ppos, true);
+			if (ret <= 0)
+				goto write_err;
+
+			filled = 1;
+		}
+		count -= filled;
+		done += filled;
+		*ppos += filled;
+		buf += filled;
+	}
+
+	return done;
+write_err:
+	return -EFAULT;
+}
+
+static int mtty_set_irqs(struct mdev_device *mdev, uint32_t flags,
+			 unsigned int index, unsigned int start,
+			 unsigned int count, void *data)
+{
+	int ret = 0;
+	struct mdev_state *mdev_state;
+
+	if (!mdev)
+		return -EINVAL;
+
+	mdev_state = mdev_get_drvdata(mdev);
+	if (!mdev_state)
+		return -EINVAL;
+
+	mutex_lock(&mdev_state->ops_lock);
+	switch (index) {
+	case VFIO_PCI_INTX_IRQ_INDEX:
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+		{
+			if (flags & VFIO_IRQ_SET_DATA_NONE) {
+				pr_info("%s: disable INTx\n", __func__);
+				if (mdev_state->intx_evtfd)
+					eventfd_ctx_put(mdev_state->intx_evtfd);
+				break;
+			}
+
+			if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+				int fd = *(int *)data;
+
+				if (fd > 0) {
+					struct eventfd_ctx *evt;
+
+					evt = eventfd_ctx_fdget(fd);
+					if (IS_ERR(evt)) {
+						ret = PTR_ERR(evt);
+						break;
+					}
+					mdev_state->intx_evtfd = evt;
+					mdev_state->irq_fd = fd;
+					mdev_state->irq_index = index;
+					break;
+				}
+			}
+			break;
+		}
+		}
+		break;
+	case VFIO_PCI_MSI_IRQ_INDEX:
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			if (flags & VFIO_IRQ_SET_DATA_NONE) {
+				if (mdev_state->msi_evtfd)
+					eventfd_ctx_put(mdev_state->msi_evtfd);
+				pr_info("%s: disable MSI\n", __func__);
+				mdev_state->irq_index = VFIO_PCI_INTX_IRQ_INDEX;
+				break;
+			}
+			if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+				int fd = *(int *)data;
+				struct eventfd_ctx *evt;
+
+				if (fd <= 0)
+					break;
+
+				if (mdev_state->msi_evtfd)
+					break;
+
+				evt = eventfd_ctx_fdget(fd);
+				if (IS_ERR(evt)) {
+					ret = PTR_ERR(evt);
+					break;
+				}
+				mdev_state->msi_evtfd = evt;
+				mdev_state->irq_fd = fd;
+				mdev_state->irq_index = index;
+			}
+			break;
+	}
+	break;
+	case VFIO_PCI_MSIX_IRQ_INDEX:
+		pr_info("%s: MSIX_IRQ\n", __func__);
+		break;
+	case VFIO_PCI_ERR_IRQ_INDEX:
+		pr_info("%s: ERR_IRQ\n", __func__);
+		break;
+	case VFIO_PCI_REQ_IRQ_INDEX:
+		pr_info("%s: REQ_IRQ\n", __func__);
+		break;
+	}
+
+	mutex_unlock(&mdev_state->ops_lock);
+	return ret;
+}
+
+static int mtty_trigger_interrupt(uuid_le uuid)
+{
+	int ret = -1;
+	struct mdev_state *mdev_state;
+
+	mdev_state = find_mdev_state_by_uuid(uuid);
+
+	if (!mdev_state) {
+		pr_info("%s: mdev not found\n", __func__);
+		return -EINVAL;
+	}
+
+	if ((mdev_state->irq_index == VFIO_PCI_MSI_IRQ_INDEX) &&
+	    (!mdev_state->msi_evtfd))
+		return -EINVAL;
+	else if ((mdev_state->irq_index == VFIO_PCI_INTX_IRQ_INDEX) &&
+		 (!mdev_state->intx_evtfd)) {
+		pr_info("%s: Intr eventfd not found\n", __func__);
+		return -EINVAL;
+	}
+
+	if (mdev_state->irq_index == VFIO_PCI_MSI_IRQ_INDEX)
+		ret = eventfd_signal(mdev_state->msi_evtfd, 1);
+	else
+		ret = eventfd_signal(mdev_state->intx_evtfd, 1);
+
+#if defined(DEBUG_INTR)
+	pr_info("Intx triggered\n");
+#endif
+	if (ret != 1)
+		pr_err("%s: eventfd signal failed (%d)\n", __func__, ret);
+
+	return ret;
+}
+
+int mtty_get_region_info(struct mdev_device *mdev,
+			 struct vfio_region_info *region_info,
+			 u16 *cap_type_id, void **cap_type)
+{
+	unsigned int size = 0;
+	struct mdev_state *mdev_state;
+	int bar_index;
+
+	if (!mdev)
+		return -EINVAL;
+
+	mdev_state = mdev_get_drvdata(mdev);
+	if (!mdev_state)
+		return -EINVAL;
+
+	mutex_lock(&mdev_state->ops_lock);
+	bar_index = region_info->index;
+
+	switch (bar_index) {
+	case VFIO_PCI_CONFIG_REGION_INDEX:
+		size = MTTY_CONFIG_SPACE_SIZE;
+		break;
+	case VFIO_PCI_BAR0_REGION_INDEX:
+		size = MTTY_IO_BAR_SIZE;
+		break;
+	case VFIO_PCI_BAR1_REGION_INDEX:
+		if (mdev_state->nr_ports == 2)
+			size = MTTY_IO_BAR_SIZE;
+		break;
+	default:
+		size = 0;
+		break;
+	}
+
+	mdev_state->region_info[bar_index].size = size;
+	mdev_state->region_info[bar_index].vfio_offset =
+		MTTY_VFIO_PCI_INDEX_TO_OFFSET(bar_index);
+
+	region_info->size = size;
+	region_info->offset = MTTY_VFIO_PCI_INDEX_TO_OFFSET(bar_index);
+	region_info->flags = VFIO_REGION_INFO_FLAG_READ |
+		VFIO_REGION_INFO_FLAG_WRITE;
+	mutex_unlock(&mdev_state->ops_lock);
+	return 0;
+}
+
+int mtty_get_irq_info(struct mdev_device *mdev, struct vfio_irq_info *irq_info)
+{
+	switch (irq_info->index) {
+	case VFIO_PCI_INTX_IRQ_INDEX:
+	case VFIO_PCI_MSI_IRQ_INDEX:
+	case VFIO_PCI_REQ_IRQ_INDEX:
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	irq_info->flags = VFIO_IRQ_INFO_EVENTFD;
+	irq_info->count = 1;
+
+	if (irq_info->index == VFIO_PCI_INTX_IRQ_INDEX)
+		irq_info->flags |= (VFIO_IRQ_INFO_MASKABLE |
+				VFIO_IRQ_INFO_AUTOMASKED);
+	else
+		irq_info->flags |= VFIO_IRQ_INFO_NORESIZE;
+
+	return 0;
+}
+
+int mtty_get_device_info(struct mdev_device *mdev,
+			 struct vfio_device_info *dev_info)
+{
+	dev_info->flags = VFIO_DEVICE_FLAGS_PCI;
+	dev_info->num_regions = VFIO_PCI_NUM_REGIONS;
+	dev_info->num_irqs = VFIO_PCI_NUM_IRQS;
+
+	return 0;
+}
+
+static long mtty_ioctl(struct mdev_device *mdev, unsigned int cmd,
+			unsigned long arg)
+{
+	int ret = 0;
+	unsigned long minsz;
+	struct mdev_state *mdev_state;
+
+	if (!mdev)
+		return -EINVAL;
+
+	mdev_state = mdev_get_drvdata(mdev);
+	if (!mdev_state)
+		return -ENODEV;
+
+	switch (cmd) {
+	case VFIO_DEVICE_GET_INFO:
+	{
+		struct vfio_device_info info;
+
+		minsz = offsetofend(struct vfio_device_info, num_irqs);
+
+		if (copy_from_user(&info, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		if (info.argsz < minsz)
+			return -EINVAL;
+
+		ret = mtty_get_device_info(mdev, &info);
+		if (ret)
+			return ret;
+
+		memcpy(&mdev_state->dev_info, &info, sizeof(info));
+
+		return copy_to_user((void __user *)arg, &info, minsz);
+	}
+	case VFIO_DEVICE_GET_REGION_INFO:
+	{
+		struct vfio_region_info info;
+		u16 cap_type_id = 0;
+		void *cap_type = NULL;
+
+		minsz = offsetofend(struct vfio_region_info, offset);
+
+		if (copy_from_user(&info, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		if (info.argsz < minsz)
+			return -EINVAL;
+
+		ret = mtty_get_region_info(mdev, &info, &cap_type_id,
+					   &cap_type);
+		if (ret)
+			return ret;
+
+		return copy_to_user((void __user *)arg, &info, minsz);
+	}
+
+	case VFIO_DEVICE_GET_IRQ_INFO:
+	{
+		struct vfio_irq_info info;
+
+		minsz = offsetofend(struct vfio_irq_info, count);
+
+		if (copy_from_user(&info, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		if ((info.argsz < minsz) ||
+		    (info.index >= mdev_state->dev_info.num_irqs))
+			return -EINVAL;
+
+		ret = mtty_get_irq_info(mdev, &info);
+		if (ret)
+			return ret;
+
+		if (info.count == -1)
+			return -EINVAL;
+
+		return copy_to_user((void __user *)arg, &info, minsz);
+	}
+	case VFIO_DEVICE_SET_IRQS:
+	{
+		struct vfio_irq_set hdr;
+		u8 *data = NULL, *ptr = NULL;
+		size_t data_size = 0;
+
+		minsz = offsetofend(struct vfio_irq_set, count);
+
+		if (copy_from_user(&hdr, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		ret = vfio_set_irqs_validate_and_prepare(&hdr,
+						mdev_state->dev_info.num_irqs,
+						VFIO_PCI_NUM_IRQS,
+						&data_size);
+		if (ret)
+			return ret;
+
+		if (data_size) {
+			ptr = data = memdup_user((void __user *)(arg + minsz),
+						 data_size);
+			if (IS_ERR(data))
+				return PTR_ERR(data);
+		}
+
+		ret = mtty_set_irqs(mdev, hdr.flags, hdr.index, hdr.start,
+				    hdr.count, data);
+
+		kfree(ptr);
+		return ret;
+	}
+	case VFIO_DEVICE_RESET:
+		return mtty_reset(mdev);
+	}
+	return -ENOTTY;
+}
+
+int mtty_open(struct mdev_device *mdev)
+{
+	pr_info("%s\n", __func__);
+	return 0;
+}
+
+void mtty_close(struct mdev_device *mdev)
+{
+	pr_info("%s\n", __func__);
+}
+
+static ssize_t
+sample_mtty_dev_show(struct device *dev, struct device_attribute *attr,
+		     char *buf)
+{
+	return sprintf(buf, "This is phy device\n");
+}
+
+static DEVICE_ATTR_RO(sample_mtty_dev);
+
+static struct attribute *mtty_dev_attrs[] = {
+	&dev_attr_sample_mtty_dev.attr,
+	NULL,
+};
+
+static const struct attribute_group mtty_dev_group = {
+	.name  = "mtty_dev",
+	.attrs = mtty_dev_attrs,
+};
+
+const struct attribute_group *mtty_dev_groups[] = {
+	&mtty_dev_group,
+	NULL,
+};
+
+static ssize_t
+sample_mdev_dev_show(struct device *dev, struct device_attribute *attr,
+		     char *buf)
+{
+	struct mdev_device *mdev = to_mdev_device(dev);
+
+	if (mdev)
+		return sprintf(buf, "This is MDEV %s\n", dev_name(&mdev->dev));
+
+	return sprintf(buf, "\n");
+}
+
+static DEVICE_ATTR_RO(sample_mdev_dev);
+
+static struct attribute *mdev_dev_attrs[] = {
+	&dev_attr_sample_mdev_dev.attr,
+	NULL,
+};
+
+static const struct attribute_group mdev_dev_group = {
+	.name  = "vendor",
+	.attrs = mdev_dev_attrs,
+};
+
+const struct attribute_group *mdev_dev_groups[] = {
+	&mdev_dev_group,
+	NULL,
+};
+
+static ssize_t
+name_show(struct kobject *kobj, struct device *dev, char *buf)
+{
+	char name[MTTY_STRING_LEN];
+	int i;
+	const char *name_str[2] = {"Single port serial", "Dual port serial"};
+
+	for (i = 0; i < 2; i++) {
+		snprintf(name, MTTY_STRING_LEN, "%s-%d",
+			 dev_driver_string(dev), i + 1);
+		if (!strcmp(kobj->name, name))
+			return sprintf(buf, "%s\n", name_str[i]);
+	}
+
+	return -EINVAL;
+}
+
+MDEV_TYPE_ATTR_RO(name);
+
+static ssize_t
+available_instances_show(struct kobject *kobj, struct device *dev, char *buf)
+{
+	char name[MTTY_STRING_LEN];
+	int i;
+	struct mdev_state *mds;
+	int ports = 0, used = 0;
+
+	for (i = 0; i < 2; i++) {
+		snprintf(name, MTTY_STRING_LEN, "%s-%d",
+			 dev_driver_string(dev), i + 1);
+		if (!strcmp(kobj->name, name)) {
+			ports = i + 1;
+			break;
+		}
+	}
+
+	if (!ports)
+		return -EINVAL;
+
+	list_for_each_entry(mds, &mdev_devices_list, next)
+		used += mds->nr_ports;
+
+	return sprintf(buf, "%d\n", (MAX_MTTYS - used)/ports);
+}
+
+MDEV_TYPE_ATTR_RO(available_instances);
+
+
+static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
+			       char *buf)
+{
+	return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING);
+}
+
+MDEV_TYPE_ATTR_RO(device_api);
+
+static struct attribute *mdev_types_attrs[] = {
+	&mdev_type_attr_name.attr,
+	&mdev_type_attr_device_api.attr,
+	&mdev_type_attr_available_instances.attr,
+	NULL,
+};
+
+static struct attribute_group mdev_type_group1 = {
+	.name  = "1",
+	.attrs = mdev_types_attrs,
+};
+
+static struct attribute_group mdev_type_group2 = {
+	.name  = "2",
+	.attrs = mdev_types_attrs,
+};
+
+struct attribute_group *mdev_type_groups[] = {
+	&mdev_type_group1,
+	&mdev_type_group2,
+	NULL,
+};
+
+struct parent_ops mdev_fops = {
+	.owner                  = THIS_MODULE,
+	.dev_attr_groups        = mtty_dev_groups,
+	.mdev_attr_groups       = mdev_dev_groups,
+	.supported_type_groups  = mdev_type_groups,
+	.create                 = mtty_create,
+	.remove			= mtty_remove,
+	.open                   = mtty_open,
+	.release                = mtty_close,
+	.read                   = mtty_read,
+	.write                  = mtty_write,
+	.ioctl		        = mtty_ioctl,
+};
+
+static void mtty_device_release(struct device *dev)
+{
+	dev_dbg(dev, "mtty: released\n");
+}
+
+static int __init mtty_dev_init(void)
+{
+	int ret = 0;
+
+	pr_info("mtty_dev: %s\n", __func__);
+
+	memset(&mtty_dev, 0, sizeof(mtty_dev));
+
+	idr_init(&mtty_dev.vd_idr);
+
+	ret = alloc_chrdev_region(&mtty_dev.vd_devt, 0, MINORMASK, MTTY_NAME);
+
+	if (ret < 0) {
+		pr_err("Error: failed to register mtty_dev, err:%d\n", ret);
+		return ret;
+	}
+
+	cdev_init(&mtty_dev.vd_cdev, &vd_fops);
+	cdev_add(&mtty_dev.vd_cdev, mtty_dev.vd_devt, MINORMASK);
+
+	pr_info("major_number:%d\n", MAJOR(mtty_dev.vd_devt));
+
+	mtty_dev.vd_class = class_create(THIS_MODULE, MTTY_CLASS_NAME);
+
+	if (IS_ERR(mtty_dev.vd_class)) {
+		pr_err("Error: failed to register mtty_dev class\n");
+		goto failed1;
+	}
+
+	mtty_dev.dev.class = mtty_dev.vd_class;
+	mtty_dev.dev.release = mtty_device_release;
+	dev_set_name(&mtty_dev.dev, "%s", MTTY_NAME);
+
+	ret = device_register(&mtty_dev.dev);
+	if (ret)
+		goto failed2;
+
+	if (mdev_register_device(&mtty_dev.dev, &mdev_fops) != 0)
+		goto failed3;
+
+	mutex_init(&mdev_list_lock);
+	INIT_LIST_HEAD(&mdev_devices_list);
+
+	goto all_done;
+
+failed3:
+
+	device_unregister(&mtty_dev.dev);
+failed2:
+	class_destroy(mtty_dev.vd_class);
+
+failed1:
+	cdev_del(&mtty_dev.vd_cdev);
+	unregister_chrdev_region(mtty_dev.vd_devt, MINORMASK);
+
+all_done:
+	return ret;
+}
+
+static void __exit mtty_dev_exit(void)
+{
+	mtty_dev.dev.bus = NULL;
+	mdev_unregister_device(&mtty_dev.dev);
+
+	device_unregister(&mtty_dev.dev);
+	idr_destroy(&mtty_dev.vd_idr);
+	cdev_del(&mtty_dev.vd_cdev);
+	unregister_chrdev_region(mtty_dev.vd_devt, MINORMASK);
+	class_destroy(mtty_dev.vd_class);
+	mtty_dev.vd_class = NULL;
+	pr_info("mtty_dev: Unloaded!\n");
+}
+
+module_init(mtty_dev_init)
+module_exit(mtty_dev_exit)
+
+MODULE_LICENSE("GPL v2");
+MODULE_INFO(supported, "Test driver that simulate serial port over PCI");
+MODULE_VERSION(VERSION_STRING);
+MODULE_AUTHOR(DRIVER_AUTHOR);
diff --git a/scripts/checkkconfigsymbols.py b/scripts/checkkconfigsymbols.py
index a32e4da4c117..3820f00b066a 100755
--- a/scripts/checkkconfigsymbols.py
+++ b/scripts/checkkconfigsymbols.py
@@ -88,7 +88,7 @@ def parse_options():
     if args.commit and args.diff:
         sys.exit("Please specify only one option at once.")
 
-    if args.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", args.diff):
+    if args.diff and not re.match(r"^[\w\-\.\^]+\.\.[\w\-\.\^]+$", args.diff):
         sys.exit("Please specify valid input in the following format: "
                  "\'commit1..commit2\'")
 
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index fd3556b2a5d1..982c52ca6473 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -335,7 +335,7 @@ our $Attribute	= qr{
 			__percpu|
 			__nocast|
 			__safe|
-			__bitwise__|
+			__bitwise|
 			__packed__|
 			__packed2__|
 			__naked|
@@ -3681,7 +3681,7 @@ sub process {
 		    $line !~ /\btypedef\s+$Type\s*\(\s*\*?$Ident\s*\)\s*\(/ &&
 		    $line !~ /\btypedef\s+$Type\s+$Ident\s*\(/ &&
 		    $line !~ /\b$typeTypedefs\b/ &&
-		    $line !~ /\b__bitwise(?:__|)\b/) {
+		    $line !~ /\b__bitwise\b/) {
 			WARN("NEW_TYPEDEFS",
 			     "do not add new typedefs\n" . $herecurr);
 		}
diff --git a/scripts/gcc-plugins/latent_entropy_plugin.c b/scripts/gcc-plugins/latent_entropy_plugin.c
index 8160f1c1b56e..12541126575b 100644
--- a/scripts/gcc-plugins/latent_entropy_plugin.c
+++ b/scripts/gcc-plugins/latent_entropy_plugin.c
@@ -619,7 +619,7 @@ __visible int plugin_init(struct plugin_name_args *plugin_info,
 			enabled = false;
 			continue;
 		}
-		error(G_("unkown option '-fplugin-arg-%s-%s'"), plugin_name, argv[i].key);
+		error(G_("unknown option '-fplugin-arg-%s-%s'"), plugin_name, argv[i].key);
 	}
 
 	register_callback(plugin_name, PLUGIN_INFO, NULL,
diff --git a/scripts/gcc-plugins/sancov_plugin.c b/scripts/gcc-plugins/sancov_plugin.c
index 7ea0b3f50739..70f5fe0d590a 100644
--- a/scripts/gcc-plugins/sancov_plugin.c
+++ b/scripts/gcc-plugins/sancov_plugin.c
@@ -126,7 +126,7 @@ __visible int plugin_init(struct plugin_name_args *plugin_info, struct plugin_gc
 			enable = false;
 			continue;
 		}
-		error(G_("unkown option '-fplugin-arg-%s-%s'"), plugin_name, argv[i].key);
+		error(G_("unknown option '-fplugin-arg-%s-%s'"), plugin_name, argv[i].key);
 	}
 
 	register_callback(plugin_name, PLUGIN_INFO, NULL, &sancov_plugin_info);
diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c
index bd8349759095..5a6b39a29b7a 100644
--- a/scripts/mod/modpost.c
+++ b/scripts/mod/modpost.c
@@ -2371,6 +2371,7 @@ static void write_dump(const char *fname)
 		}
 	}
 	write_if_changed(&buf, fname);
+	free(buf.p);
 }
 
 struct ext_sym_list {
@@ -2496,6 +2497,7 @@ int main(int argc, char **argv)
 			      "Set CONFIG_SECTION_MISMATCH_WARN_ONLY=y to allow them.\n");
 		}
 	}
+	free(buf.p);
 
 	return err;
 }
diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c
index 5423a58d1b06..aeb34223167c 100644
--- a/scripts/recordmcount.c
+++ b/scripts/recordmcount.c
@@ -213,6 +213,59 @@ static int make_nop_x86(void *map, size_t const offset)
 	return 0;
 }
 
+static unsigned char ideal_nop4_arm_le[4] = { 0x00, 0x00, 0xa0, 0xe1 }; /* mov r0, r0 */
+static unsigned char ideal_nop4_arm_be[4] = { 0xe1, 0xa0, 0x00, 0x00 }; /* mov r0, r0 */
+static unsigned char *ideal_nop4_arm;
+
+static unsigned char bl_mcount_arm_le[4] = { 0xfe, 0xff, 0xff, 0xeb }; /* bl */
+static unsigned char bl_mcount_arm_be[4] = { 0xeb, 0xff, 0xff, 0xfe }; /* bl */
+static unsigned char *bl_mcount_arm;
+
+static unsigned char push_arm_le[4] = { 0x04, 0xe0, 0x2d, 0xe5 }; /* push {lr} */
+static unsigned char push_arm_be[4] = { 0xe5, 0x2d, 0xe0, 0x04 }; /* push {lr} */
+static unsigned char *push_arm;
+
+static unsigned char ideal_nop2_thumb_le[2] = { 0x00, 0xbf }; /* nop */
+static unsigned char ideal_nop2_thumb_be[2] = { 0xbf, 0x00 }; /* nop */
+static unsigned char *ideal_nop2_thumb;
+
+static unsigned char push_bl_mcount_thumb_le[6] = { 0x00, 0xb5, 0xff, 0xf7, 0xfe, 0xff }; /* push {lr}, bl */
+static unsigned char push_bl_mcount_thumb_be[6] = { 0xb5, 0x00, 0xf7, 0xff, 0xff, 0xfe }; /* push {lr}, bl */
+static unsigned char *push_bl_mcount_thumb;
+
+static int make_nop_arm(void *map, size_t const offset)
+{
+	char *ptr;
+	int cnt = 1;
+	int nop_size;
+	size_t off = offset;
+
+	ptr = map + offset;
+	if (memcmp(ptr, bl_mcount_arm, 4) == 0) {
+		if (memcmp(ptr - 4, push_arm, 4) == 0) {
+			off -= 4;
+			cnt = 2;
+		}
+		ideal_nop = ideal_nop4_arm;
+		nop_size = 4;
+	} else if (memcmp(ptr - 2, push_bl_mcount_thumb, 6) == 0) {
+		cnt = 3;
+		nop_size = 2;
+		off -= 2;
+		ideal_nop = ideal_nop2_thumb;
+	} else
+		return -1;
+
+	/* Convert to nop */
+	ulseek(fd_map, off, SEEK_SET);
+
+	do {
+		uwrite(fd_map, ideal_nop, nop_size);
+	} while (--cnt > 0);
+
+	return 0;
+}
+
 static unsigned char ideal_nop4_arm64[4] = {0x1f, 0x20, 0x03, 0xd5};
 static int make_nop_arm64(void *map, size_t const offset)
 {
@@ -430,6 +483,11 @@ do_file(char const *const fname)
 			w2 = w2rev;
 			w8 = w8rev;
 		}
+		ideal_nop4_arm = ideal_nop4_arm_le;
+		bl_mcount_arm = bl_mcount_arm_le;
+		push_arm = push_arm_le;
+		ideal_nop2_thumb = ideal_nop2_thumb_le;
+		push_bl_mcount_thumb = push_bl_mcount_thumb_le;
 		break;
 	case ELFDATA2MSB:
 		if (*(unsigned char const *)&endian != 0) {
@@ -438,6 +496,11 @@ do_file(char const *const fname)
 			w2 = w2rev;
 			w8 = w8rev;
 		}
+		ideal_nop4_arm = ideal_nop4_arm_be;
+		bl_mcount_arm = bl_mcount_arm_be;
+		push_arm = push_arm_be;
+		ideal_nop2_thumb = ideal_nop2_thumb_be;
+		push_bl_mcount_thumb = push_bl_mcount_thumb_be;
 		break;
 	}  /* end switch */
 	if (memcmp(ELFMAG, ehdr->e_ident, SELFMAG) != 0
@@ -463,6 +526,8 @@ do_file(char const *const fname)
 		break;
 	case EM_ARM:	 reltype = R_ARM_ABS32;
 			 altmcount = "__gnu_mcount_nc";
+			 make_nop = make_nop_arm;
+			 rel_type_nop = R_ARM_NONE;
 			 break;
 	case EM_AARCH64:
 			reltype = R_AARCH64_ABS64;
diff --git a/scripts/sign-file.c b/scripts/sign-file.c
index 53af6dc3e6c1..19ec468b1168 100644
--- a/scripts/sign-file.c
+++ b/scripts/sign-file.c
@@ -267,7 +267,7 @@ int main(int argc, char **argv)
 	}
 	x509_name = argv[2];
 	module_name = argv[3];
-	if (argc == 5) {
+	if (argc == 5 && strcmp(argv[3], argv[4]) != 0) {
 		dest_name = argv[4];
 		replace_orig = false;
 	} else {
diff --git a/scripts/sortextable.c b/scripts/sortextable.c
index f453b7ce99d6..365a907f98b3 100644
--- a/scripts/sortextable.c
+++ b/scripts/sortextable.c
@@ -316,6 +316,8 @@ do_file(char const *const fname)
 	case EM_S390:
 	case EM_AARCH64:
 	case EM_PARISC:
+	case EM_PPC:
+	case EM_PPC64:
 		custom_sort = sort_relative_table;
 		break;
 	case EM_ARCOMPACT:
diff --git a/security/integrity/digsig.c b/security/integrity/digsig.c
index 4304372b323f..106e855e2d9d 100644
--- a/security/integrity/digsig.c
+++ b/security/integrity/digsig.c
@@ -51,7 +51,7 @@ static bool init_keyring __initdata;
 int integrity_digsig_verify(const unsigned int id, const char *sig, int siglen,
 			    const char *digest, int digestlen)
 {
-	if (id >= INTEGRITY_KEYRING_MAX)
+	if (id >= INTEGRITY_KEYRING_MAX || siglen < 2)
 		return -EINVAL;
 
 	if (!keyring[id]) {
diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c
index bf663915412e..d7f282d75cc1 100644
--- a/security/integrity/evm/evm_crypto.c
+++ b/security/integrity/evm/evm_crypto.c
@@ -151,8 +151,16 @@ static void hmac_add_misc(struct shash_desc *desc, struct inode *inode,
 	memset(&hmac_misc, 0, sizeof(hmac_misc));
 	hmac_misc.ino = inode->i_ino;
 	hmac_misc.generation = inode->i_generation;
-	hmac_misc.uid = from_kuid(inode->i_sb->s_user_ns, inode->i_uid);
-	hmac_misc.gid = from_kgid(inode->i_sb->s_user_ns, inode->i_gid);
+	/* The hmac uid and gid must be encoded in the initial user
+	 * namespace (not the filesystems user namespace) as encoding
+	 * them in the filesystems user namespace allows an attack
+	 * where first they are written in an unprivileged fuse mount
+	 * of a filesystem and then the system is tricked to mount the
+	 * filesystem for real on next boot and trust it because
+	 * everything is signed.
+	 */
+	hmac_misc.uid = from_kuid(&init_user_ns, inode->i_uid);
+	hmac_misc.gid = from_kgid(&init_user_ns, inode->i_gid);
 	hmac_misc.mode = inode->i_mode;
 	crypto_shash_update(desc, (const u8 *)&hmac_misc, sizeof(hmac_misc));
 	if (evm_hmac_attrs & EVM_ATTR_FSUUID)
diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c
index ba8615576d4d..e2ed498c0f5f 100644
--- a/security/integrity/evm/evm_main.c
+++ b/security/integrity/evm/evm_main.c
@@ -145,6 +145,10 @@ static enum integrity_status evm_verify_hmac(struct dentry *dentry,
 	/* check value type */
 	switch (xattr_data->type) {
 	case EVM_XATTR_HMAC:
+		if (xattr_len != sizeof(struct evm_ima_xattr_data)) {
+			evm_status = INTEGRITY_FAIL;
+			goto out;
+		}
 		rc = evm_calc_hmac(dentry, xattr_name, xattr_value,
 				   xattr_value_len, calc.digest);
 		if (rc)
diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c
index 389325ac6067..1fd9539a969d 100644
--- a/security/integrity/ima/ima_appraise.c
+++ b/security/integrity/ima/ima_appraise.c
@@ -130,6 +130,7 @@ enum hash_algo ima_get_hash_algo(struct evm_ima_xattr_data *xattr_value,
 				 int xattr_len)
 {
 	struct signature_v2_hdr *sig;
+	enum hash_algo ret;
 
 	if (!xattr_value || xattr_len < 2)
 		/* return default hash algo */
@@ -143,7 +144,9 @@ enum hash_algo ima_get_hash_algo(struct evm_ima_xattr_data *xattr_value,
 		return sig->hash_algo;
 		break;
 	case IMA_XATTR_DIGEST_NG:
-		return xattr_value->digest[0];
+		ret = xattr_value->digest[0];
+		if (ret < HASH_ALGO__LAST)
+			return ret;
 		break;
 	case IMA_XATTR_DIGEST:
 		/* this is for backward compatibility */
@@ -384,14 +387,10 @@ int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name,
 	result = ima_protect_xattr(dentry, xattr_name, xattr_value,
 				   xattr_value_len);
 	if (result == 1) {
-		bool digsig;
-
 		if (!xattr_value_len || (xvalue->type >= IMA_XATTR_LAST))
 			return -EINVAL;
-		digsig = (xvalue->type == EVM_IMA_XATTR_DIGSIG);
-		if (!digsig && (ima_appraise & IMA_APPRAISE_ENFORCE))
-			return -EPERM;
-		ima_reset_appraise_flags(d_backing_inode(dentry), digsig);
+		ima_reset_appraise_flags(d_backing_inode(dentry),
+			 (xvalue->type == EVM_IMA_XATTR_DIGSIG) ? 1 : 0);
 		result = 0;
 	}
 	return result;
diff --git a/security/integrity/ima/ima_fs.c b/security/integrity/ima/ima_fs.c
index c07a3844ea0a..3df46906492d 100644
--- a/security/integrity/ima/ima_fs.c
+++ b/security/integrity/ima/ima_fs.c
@@ -401,7 +401,7 @@ static int ima_release_policy(struct inode *inode, struct file *file)
 	const char *cause = valid_policy ? "completed" : "failed";
 
 	if ((file->f_flags & O_ACCMODE) == O_RDONLY)
-		return 0;
+		return seq_release(inode, file);
 
 	if (valid_policy && ima_check_policy() < 0) {
 		cause = "failed";
diff --git a/security/integrity/ima/ima_init.c b/security/integrity/ima/ima_init.c
index 32912bd54ead..2ac1f41db5c0 100644
--- a/security/integrity/ima/ima_init.c
+++ b/security/integrity/ima/ima_init.c
@@ -115,7 +115,8 @@ int __init ima_init(void)
 		ima_used_chip = 1;
 
 	if (!ima_used_chip)
-		pr_info("No TPM chip found, activating TPM-bypass!\n");
+		pr_info("No TPM chip found, activating TPM-bypass! (rc=%d)\n",
+			rc);
 
 	rc = integrity_init_keyring(INTEGRITY_KEYRING_IMA);
 	if (rc)
diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index 38b79d797aaf..c7c6619431d5 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -231,12 +231,13 @@ static int inode_alloc_security(struct inode *inode)
 	if (!isec)
 		return -ENOMEM;
 
-	mutex_init(&isec->lock);
+	spin_lock_init(&isec->lock);
 	INIT_LIST_HEAD(&isec->list);
 	isec->inode = inode;
 	isec->sid = SECINITSID_UNLABELED;
 	isec->sclass = SECCLASS_FILE;
 	isec->task_sid = sid;
+	isec->initialized = LABEL_INVALID;
 	inode->i_security = isec;
 
 	return 0;
@@ -247,7 +248,7 @@ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dent
 /*
  * Try reloading inode security labels that have been marked as invalid.  The
  * @may_sleep parameter indicates when sleeping and thus reloading labels is
- * allowed; when set to false, returns ERR_PTR(-ECHILD) when the label is
+ * allowed; when set to false, returns -ECHILD when the label is
  * invalid.  The @opt_dentry parameter should be set to a dentry of the inode;
  * when no dentry is available, set it to NULL instead.
  */
@@ -1100,11 +1101,12 @@ static int selinux_parse_opts_str(char *options,
 	}
 
 	rc = -ENOMEM;
-	opts->mnt_opts = kcalloc(NUM_SEL_MNT_OPTS, sizeof(char *), GFP_ATOMIC);
+	opts->mnt_opts = kcalloc(NUM_SEL_MNT_OPTS, sizeof(char *), GFP_KERNEL);
 	if (!opts->mnt_opts)
 		goto out_err;
 
-	opts->mnt_opts_flags = kcalloc(NUM_SEL_MNT_OPTS, sizeof(int), GFP_ATOMIC);
+	opts->mnt_opts_flags = kcalloc(NUM_SEL_MNT_OPTS, sizeof(int),
+				       GFP_KERNEL);
 	if (!opts->mnt_opts_flags) {
 		kfree(opts->mnt_opts);
 		goto out_err;
@@ -1380,7 +1382,8 @@ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dent
 {
 	struct superblock_security_struct *sbsec = NULL;
 	struct inode_security_struct *isec = inode->i_security;
-	u32 sid;
+	u32 task_sid, sid = 0;
+	u16 sclass;
 	struct dentry *dentry;
 #define INITCONTEXTLEN 255
 	char *context = NULL;
@@ -1388,12 +1391,15 @@ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dent
 	int rc = 0;
 
 	if (isec->initialized == LABEL_INITIALIZED)
-		goto out;
+		return 0;
 
-	mutex_lock(&isec->lock);
+	spin_lock(&isec->lock);
 	if (isec->initialized == LABEL_INITIALIZED)
 		goto out_unlock;
 
+	if (isec->sclass == SECCLASS_FILE)
+		isec->sclass = inode_mode_to_security_class(inode->i_mode);
+
 	sbsec = inode->i_sb->s_security;
 	if (!(sbsec->flags & SE_SBINITIALIZED)) {
 		/* Defer initialization until selinux_complete_init,
@@ -1406,12 +1412,18 @@ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dent
 		goto out_unlock;
 	}
 
+	sclass = isec->sclass;
+	task_sid = isec->task_sid;
+	sid = isec->sid;
+	isec->initialized = LABEL_PENDING;
+	spin_unlock(&isec->lock);
+
 	switch (sbsec->behavior) {
 	case SECURITY_FS_USE_NATIVE:
 		break;
 	case SECURITY_FS_USE_XATTR:
 		if (!(inode->i_opflags & IOP_XATTR)) {
-			isec->sid = sbsec->def_sid;
+			sid = sbsec->def_sid;
 			break;
 		}
 		/* Need a dentry, since the xattr API requires one.
@@ -1433,7 +1445,7 @@ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dent
 			 * inode_doinit with a dentry, before these inodes could
 			 * be used again by userspace.
 			 */
-			goto out_unlock;
+			goto out;
 		}
 
 		len = INITCONTEXTLEN;
@@ -1441,7 +1453,7 @@ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dent
 		if (!context) {
 			rc = -ENOMEM;
 			dput(dentry);
-			goto out_unlock;
+			goto out;
 		}
 		context[len] = '\0';
 		rc = __vfs_getxattr(dentry, inode, XATTR_NAME_SELINUX, context, len);
@@ -1452,14 +1464,14 @@ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dent
 			rc = __vfs_getxattr(dentry, inode, XATTR_NAME_SELINUX, NULL, 0);
 			if (rc < 0) {
 				dput(dentry);
-				goto out_unlock;
+				goto out;
 			}
 			len = rc;
 			context = kmalloc(len+1, GFP_NOFS);
 			if (!context) {
 				rc = -ENOMEM;
 				dput(dentry);
-				goto out_unlock;
+				goto out;
 			}
 			context[len] = '\0';
 			rc = __vfs_getxattr(dentry, inode, XATTR_NAME_SELINUX, context, len);
@@ -1471,7 +1483,7 @@ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dent
 				       "%d for dev=%s ino=%ld\n", __func__,
 				       -rc, inode->i_sb->s_id, inode->i_ino);
 				kfree(context);
-				goto out_unlock;
+				goto out;
 			}
 			/* Map ENODATA to the default file SID */
 			sid = sbsec->def_sid;
@@ -1501,29 +1513,25 @@ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dent
 			}
 		}
 		kfree(context);
-		isec->sid = sid;
 		break;
 	case SECURITY_FS_USE_TASK:
-		isec->sid = isec->task_sid;
+		sid = task_sid;
 		break;
 	case SECURITY_FS_USE_TRANS:
 		/* Default to the fs SID. */
-		isec->sid = sbsec->sid;
+		sid = sbsec->sid;
 
 		/* Try to obtain a transition SID. */
-		isec->sclass = inode_mode_to_security_class(inode->i_mode);
-		rc = security_transition_sid(isec->task_sid, sbsec->sid,
-					     isec->sclass, NULL, &sid);
+		rc = security_transition_sid(task_sid, sid, sclass, NULL, &sid);
 		if (rc)
-			goto out_unlock;
-		isec->sid = sid;
+			goto out;
 		break;
 	case SECURITY_FS_USE_MNTPOINT:
-		isec->sid = sbsec->mntpoint_sid;
+		sid = sbsec->mntpoint_sid;
 		break;
 	default:
 		/* Default to the fs superblock SID. */
-		isec->sid = sbsec->sid;
+		sid = sbsec->sid;
 
 		if ((sbsec->flags & SE_SBGENFS) && !S_ISLNK(inode->i_mode)) {
 			/* We must have a dentry to determine the label on
@@ -1546,25 +1554,30 @@ static int inode_doinit_with_dentry(struct inode *inode, struct dentry *opt_dent
 			 * could be used again by userspace.
 			 */
 			if (!dentry)
-				goto out_unlock;
-			isec->sclass = inode_mode_to_security_class(inode->i_mode);
-			rc = selinux_genfs_get_sid(dentry, isec->sclass,
+				goto out;
+			rc = selinux_genfs_get_sid(dentry, sclass,
 						   sbsec->flags, &sid);
 			dput(dentry);
 			if (rc)
-				goto out_unlock;
-			isec->sid = sid;
+				goto out;
 		}
 		break;
 	}
 
-	isec->initialized = LABEL_INITIALIZED;
+out:
+	spin_lock(&isec->lock);
+	if (isec->initialized == LABEL_PENDING) {
+		if (!sid || rc) {
+			isec->initialized = LABEL_INVALID;
+			goto out_unlock;
+		}
+
+		isec->initialized = LABEL_INITIALIZED;
+		isec->sid = sid;
+	}
 
 out_unlock:
-	mutex_unlock(&isec->lock);
-out:
-	if (isec->sclass == SECCLASS_FILE)
-		isec->sclass = inode_mode_to_security_class(inode->i_mode);
+	spin_unlock(&isec->lock);
 	return rc;
 }
 
@@ -3198,9 +3211,11 @@ static void selinux_inode_post_setxattr(struct dentry *dentry, const char *name,
 	}
 
 	isec = backing_inode_security(dentry);
+	spin_lock(&isec->lock);
 	isec->sclass = inode_mode_to_security_class(inode->i_mode);
 	isec->sid = newsid;
 	isec->initialized = LABEL_INITIALIZED;
+	spin_unlock(&isec->lock);
 
 	return;
 }
@@ -3293,9 +3308,11 @@ static int selinux_inode_setsecurity(struct inode *inode, const char *name,
 	if (rc)
 		return rc;
 
+	spin_lock(&isec->lock);
 	isec->sclass = inode_mode_to_security_class(inode->i_mode);
 	isec->sid = newsid;
 	isec->initialized = LABEL_INITIALIZED;
+	spin_unlock(&isec->lock);
 	return 0;
 }
 
@@ -3956,8 +3973,11 @@ static void selinux_task_to_inode(struct task_struct *p,
 	struct inode_security_struct *isec = inode->i_security;
 	u32 sid = task_sid(p);
 
+	spin_lock(&isec->lock);
+	isec->sclass = inode_mode_to_security_class(inode->i_mode);
 	isec->sid = sid;
 	isec->initialized = LABEL_INITIALIZED;
+	spin_unlock(&isec->lock);
 }
 
 /* Returns error only if unable to parse addresses */
@@ -4276,24 +4296,24 @@ static int selinux_socket_post_create(struct socket *sock, int family,
 	const struct task_security_struct *tsec = current_security();
 	struct inode_security_struct *isec = inode_security_novalidate(SOCK_INODE(sock));
 	struct sk_security_struct *sksec;
+	u16 sclass = socket_type_to_security_class(family, type, protocol);
+	u32 sid = SECINITSID_KERNEL;
 	int err = 0;
 
-	isec->sclass = socket_type_to_security_class(family, type, protocol);
-
-	if (kern)
-		isec->sid = SECINITSID_KERNEL;
-	else {
-		err = socket_sockcreate_sid(tsec, isec->sclass, &(isec->sid));
+	if (!kern) {
+		err = socket_sockcreate_sid(tsec, sclass, &sid);
 		if (err)
 			return err;
 	}
 
+	isec->sclass = sclass;
+	isec->sid = sid;
 	isec->initialized = LABEL_INITIALIZED;
 
 	if (sock->sk) {
 		sksec = sock->sk->sk_security;
-		sksec->sid = isec->sid;
-		sksec->sclass = isec->sclass;
+		sksec->sclass = sclass;
+		sksec->sid = sid;
 		err = selinux_netlbl_socket_post_create(sock->sk, family);
 	}
 
@@ -4469,16 +4489,22 @@ static int selinux_socket_accept(struct socket *sock, struct socket *newsock)
 	int err;
 	struct inode_security_struct *isec;
 	struct inode_security_struct *newisec;
+	u16 sclass;
+	u32 sid;
 
 	err = sock_has_perm(current, sock->sk, SOCKET__ACCEPT);
 	if (err)
 		return err;
 
-	newisec = inode_security_novalidate(SOCK_INODE(newsock));
-
 	isec = inode_security_novalidate(SOCK_INODE(sock));
-	newisec->sclass = isec->sclass;
-	newisec->sid = isec->sid;
+	spin_lock(&isec->lock);
+	sclass = isec->sclass;
+	sid = isec->sid;
+	spin_unlock(&isec->lock);
+
+	newisec = inode_security_novalidate(SOCK_INODE(newsock));
+	newisec->sclass = sclass;
+	newisec->sid = sid;
 	newisec->initialized = LABEL_INITIALIZED;
 
 	return 0;
@@ -5981,9 +6007,9 @@ static void selinux_inode_invalidate_secctx(struct inode *inode)
 {
 	struct inode_security_struct *isec = inode->i_security;
 
-	mutex_lock(&isec->lock);
+	spin_lock(&isec->lock);
 	isec->initialized = LABEL_INVALID;
-	mutex_unlock(&isec->lock);
+	spin_unlock(&isec->lock);
 }
 
 /*
diff --git a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h
index 1f1f4b2f6018..e2d4ad3a4b4c 100644
--- a/security/selinux/include/classmap.h
+++ b/security/selinux/include/classmap.h
@@ -24,6 +24,10 @@
 #define COMMON_CAP2_PERMS  "mac_override", "mac_admin", "syslog", \
 		"wake_alarm", "block_suspend", "audit_read"
 
+#if CAP_LAST_CAP > CAP_AUDIT_READ
+#error New capability defined, please update COMMON_CAP2_PERMS.
+#endif
+
 /*
  * Note: The name for any socket class should be suffixed by "socket",
  *	 and doesn't contain more than one substr of "socket".
diff --git a/security/selinux/include/objsec.h b/security/selinux/include/objsec.h
index c21e135460a5..e8dab0f02c72 100644
--- a/security/selinux/include/objsec.h
+++ b/security/selinux/include/objsec.h
@@ -39,7 +39,8 @@ struct task_security_struct {
 
 enum label_initialized {
 	LABEL_INVALID,		/* invalid or not initialized */
-	LABEL_INITIALIZED	/* initialized */
+	LABEL_INITIALIZED,	/* initialized */
+	LABEL_PENDING
 };
 
 struct inode_security_struct {
@@ -52,7 +53,7 @@ struct inode_security_struct {
 	u32 sid;		/* SID of this object */
 	u16 sclass;		/* security class of this object */
 	unsigned char initialized;	/* initialization flag */
-	struct mutex lock;
+	spinlock_t lock;
 };
 
 struct file_security_struct {
diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c
index 72c145dd799f..cf9293e01fc1 100644
--- a/security/selinux/selinuxfs.c
+++ b/security/selinux/selinuxfs.c
@@ -163,6 +163,8 @@ static ssize_t sel_write_enforce(struct file *file, const char __user *buf,
 	if (sscanf(page, "%d", &new_value) != 1)
 		goto out;
 
+	new_value = !!new_value;
+
 	if (new_value != selinux_enforcing) {
 		length = task_has_security(current, SECURITY__SETENFORCE);
 		if (length)
@@ -1301,7 +1303,7 @@ static int sel_make_bools(void)
 			goto out;
 
 		isec->sid = sid;
-		isec->initialized = 1;
+		isec->initialized = LABEL_INITIALIZED;
 		inode->i_fop = &sel_bool_ops;
 		inode->i_ino = i|SEL_BOOL_INO_OFFSET;
 		d_add(dentry, inode);
@@ -1834,7 +1836,7 @@ static int sel_fill_super(struct super_block *sb, void *data, int silent)
 	isec = (struct inode_security_struct *)inode->i_security;
 	isec->sid = SECINITSID_DEVNULL;
 	isec->sclass = SECCLASS_CHR_FILE;
-	isec->initialized = 1;
+	isec->initialized = LABEL_INITIALIZED;
 
 	init_special_inode(inode, S_IFCHR | S_IRUGO | S_IWUGO, MKDEV(MEM_MAJOR, 3));
 	d_add(dentry, inode);
diff --git a/security/smack/smack.h b/security/smack/smack.h
index 51fd30192c08..77abe2efacae 100644
--- a/security/smack/smack.h
+++ b/security/smack/smack.h
@@ -336,7 +336,6 @@ extern int smack_ptrace_rule;
 extern struct smack_known smack_known_floor;
 extern struct smack_known smack_known_hat;
 extern struct smack_known smack_known_huh;
-extern struct smack_known smack_known_invalid;
 extern struct smack_known smack_known_star;
 extern struct smack_known smack_known_web;
 
diff --git a/security/smack/smack_access.c b/security/smack/smack_access.c
index 23e5808a0970..356e3764cad9 100644
--- a/security/smack/smack_access.c
+++ b/security/smack/smack_access.c
@@ -36,11 +36,6 @@ struct smack_known smack_known_floor = {
 	.smk_secid	= 5,
 };
 
-struct smack_known smack_known_invalid = {
-	.smk_known	= "",
-	.smk_secid	= 6,
-};
-
 struct smack_known smack_known_web = {
 	.smk_known	= "@",
 	.smk_secid	= 7,
@@ -615,7 +610,7 @@ struct smack_known *smack_from_secid(const u32 secid)
 	 * of a secid that is not on the list.
 	 */
 	rcu_read_unlock();
-	return &smack_known_invalid;
+	return &smack_known_huh;
 }
 
 /*
diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c
index 1cb060293505..4d90257d03ad 100644
--- a/security/smack/smack_lsm.c
+++ b/security/smack/smack_lsm.c
@@ -692,12 +692,12 @@ static int smack_parse_opts_str(char *options,
 		}
 	}
 
-	opts->mnt_opts = kcalloc(NUM_SMK_MNT_OPTS, sizeof(char *), GFP_ATOMIC);
+	opts->mnt_opts = kcalloc(NUM_SMK_MNT_OPTS, sizeof(char *), GFP_KERNEL);
 	if (!opts->mnt_opts)
 		goto out_err;
 
 	opts->mnt_opts_flags = kcalloc(NUM_SMK_MNT_OPTS, sizeof(int),
-			GFP_ATOMIC);
+			GFP_KERNEL);
 	if (!opts->mnt_opts_flags) {
 		kfree(opts->mnt_opts);
 		goto out_err;
@@ -769,6 +769,31 @@ static int smack_set_mnt_opts(struct super_block *sb,
 	if (sp->smk_flags & SMK_SB_INITIALIZED)
 		return 0;
 
+	if (!smack_privileged(CAP_MAC_ADMIN)) {
+		/*
+		 * Unprivileged mounts don't get to specify Smack values.
+		 */
+		if (num_opts)
+			return -EPERM;
+		/*
+		 * Unprivileged mounts get root and default from the caller.
+		 */
+		skp = smk_of_current();
+		sp->smk_root = skp;
+		sp->smk_default = skp;
+		/*
+		 * For a handful of fs types with no user-controlled
+		 * backing store it's okay to trust security labels
+		 * in the filesystem. The rest are untrusted.
+		 */
+		if (sb->s_user_ns != &init_user_ns &&
+		    sb->s_magic != SYSFS_MAGIC && sb->s_magic != TMPFS_MAGIC &&
+		    sb->s_magic != RAMFS_MAGIC) {
+			transmute = 1;
+			sp->smk_flags |= SMK_SB_UNTRUSTED;
+		}
+	}
+
 	sp->smk_flags |= SMK_SB_INITIALIZED;
 
 	for (i = 0; i < num_opts; i++) {
@@ -809,31 +834,6 @@ static int smack_set_mnt_opts(struct super_block *sb,
 		}
 	}
 
-	if (!smack_privileged(CAP_MAC_ADMIN)) {
-		/*
-		 * Unprivileged mounts don't get to specify Smack values.
-		 */
-		if (num_opts)
-			return -EPERM;
-		/*
-		 * Unprivileged mounts get root and default from the caller.
-		 */
-		skp = smk_of_current();
-		sp->smk_root = skp;
-		sp->smk_default = skp;
-		/*
-		 * For a handful of fs types with no user-controlled
-		 * backing store it's okay to trust security labels
-		 * in the filesystem. The rest are untrusted.
-		 */
-		if (sb->s_user_ns != &init_user_ns &&
-		    sb->s_magic != SYSFS_MAGIC && sb->s_magic != TMPFS_MAGIC &&
-		    sb->s_magic != RAMFS_MAGIC) {
-			transmute = 1;
-			sp->smk_flags |= SMK_SB_UNTRUSTED;
-		}
-	}
-
 	/*
 	 * Initialize the root inode.
 	 */
@@ -1384,20 +1384,14 @@ static void smack_inode_post_setxattr(struct dentry *dentry, const char *name,
 		skp = smk_import_entry(value, size);
 		if (!IS_ERR(skp))
 			isp->smk_inode = skp;
-		else
-			isp->smk_inode = &smack_known_invalid;
 	} else if (strcmp(name, XATTR_NAME_SMACKEXEC) == 0) {
 		skp = smk_import_entry(value, size);
 		if (!IS_ERR(skp))
 			isp->smk_task = skp;
-		else
-			isp->smk_task = &smack_known_invalid;
 	} else if (strcmp(name, XATTR_NAME_SMACKMMAP) == 0) {
 		skp = smk_import_entry(value, size);
 		if (!IS_ERR(skp))
 			isp->smk_mmap = skp;
-		else
-			isp->smk_mmap = &smack_known_invalid;
 	}
 
 	return;
@@ -2023,6 +2017,8 @@ static int smack_cred_prepare(struct cred *new, const struct cred *old,
 	if (new_tsp == NULL)
 		return -ENOMEM;
 
+	new->security = new_tsp;
+
 	rc = smk_copy_rules(&new_tsp->smk_rules, &old_tsp->smk_rules, gfp);
 	if (rc != 0)
 		return rc;
@@ -2032,7 +2028,6 @@ static int smack_cred_prepare(struct cred *new, const struct cred *old,
 	if (rc != 0)
 		return rc;
 
-	new->security = new_tsp;
 	return 0;
 }
 
@@ -2067,12 +2062,8 @@ static void smack_cred_transfer(struct cred *new, const struct cred *old)
 static int smack_kernel_act_as(struct cred *new, u32 secid)
 {
 	struct task_smack *new_tsp = new->security;
-	struct smack_known *skp = smack_from_secid(secid);
-
-	if (skp == NULL)
-		return -EINVAL;
 
-	new_tsp->smk_task = skp;
+	new_tsp->smk_task = smack_from_secid(secid);
 	return 0;
 }
 
@@ -2337,8 +2328,16 @@ static int smack_sk_alloc_security(struct sock *sk, int family, gfp_t gfp_flags)
 	if (ssp == NULL)
 		return -ENOMEM;
 
-	ssp->smk_in = skp;
-	ssp->smk_out = skp;
+	/*
+	 * Sockets created by kernel threads receive web label.
+	 */
+	if (unlikely(current->flags & PF_KTHREAD)) {
+		ssp->smk_in = &smack_known_web;
+		ssp->smk_out = &smack_known_web;
+	} else {
+		ssp->smk_in = skp;
+		ssp->smk_out = skp;
+	}
 	ssp->smk_packet = NULL;
 
 	sk->sk_security = ssp;
@@ -2435,17 +2434,17 @@ static struct smack_known *smack_ipv6host_label(struct sockaddr_in6 *sip)
 
 	list_for_each_entry_rcu(snp, &smk_net6addr_list, list) {
 		/*
+		 * If the label is NULL the entry has
+		 * been renounced. Ignore it.
+		 */
+		if (snp->smk_label == NULL)
+			continue;
+		/*
 		* we break after finding the first match because
 		* the list is sorted from longest to shortest mask
 		* so we have found the most specific match
 		*/
 		for (found = 1, i = 0; i < 8; i++) {
-			/*
-			 * If the label is NULL the entry has
-			 * been renounced. Ignore it.
-			 */
-			if (snp->smk_label == NULL)
-				continue;
 			if ((sap->s6_addr16[i] & snp->smk_mask.s6_addr16[i]) !=
 			    snp->smk_host.s6_addr16[i]) {
 				found = 0;
@@ -3661,10 +3660,11 @@ static int smack_setprocattr(struct task_struct *p, char *name,
 		return PTR_ERR(skp);
 
 	/*
-	 * No process is ever allowed the web ("@") label.
+	 * No process is ever allowed the web ("@") label
+	 * and the star ("*") label.
 	 */
-	if (skp == &smack_known_web)
-		return -EPERM;
+	if (skp == &smack_known_web || skp == &smack_known_star)
+		return -EINVAL;
 
 	if (!smack_privileged(CAP_MAC_ADMIN)) {
 		rc = -EPERM;
@@ -3884,21 +3884,11 @@ static struct smack_known *smack_from_secattr(struct netlbl_lsm_secattr *sap,
 			return &smack_known_web;
 		return &smack_known_star;
 	}
-	if ((sap->flags & NETLBL_SECATTR_SECID) != 0) {
+	if ((sap->flags & NETLBL_SECATTR_SECID) != 0)
 		/*
 		 * Looks like a fallback, which gives us a secid.
 		 */
-		skp = smack_from_secid(sap->attr.secid);
-		/*
-		 * This has got to be a bug because it is
-		 * impossible to specify a fallback without
-		 * specifying the label, which will ensure
-		 * it has a secid, and the only way to get a
-		 * secid is from a fallback.
-		 */
-		BUG_ON(skp == NULL);
-		return skp;
-	}
+		return smack_from_secid(sap->attr.secid);
 	/*
 	 * Without guidance regarding the smack value
 	 * for the packet fall back on the network
@@ -4761,7 +4751,6 @@ static __init void init_smack_known_list(void)
 	mutex_init(&smack_known_hat.smk_rules_lock);
 	mutex_init(&smack_known_floor.smk_rules_lock);
 	mutex_init(&smack_known_star.smk_rules_lock);
-	mutex_init(&smack_known_invalid.smk_rules_lock);
 	mutex_init(&smack_known_web.smk_rules_lock);
 	/*
 	 * Initialize rule lists
@@ -4770,7 +4759,6 @@ static __init void init_smack_known_list(void)
 	INIT_LIST_HEAD(&smack_known_hat.smk_rules);
 	INIT_LIST_HEAD(&smack_known_star.smk_rules);
 	INIT_LIST_HEAD(&smack_known_floor.smk_rules);
-	INIT_LIST_HEAD(&smack_known_invalid.smk_rules);
 	INIT_LIST_HEAD(&smack_known_web.smk_rules);
 	/*
 	 * Create the known labels list
@@ -4779,7 +4767,6 @@ static __init void init_smack_known_list(void)
 	smk_insert_entry(&smack_known_hat);
 	smk_insert_entry(&smack_known_star);
 	smk_insert_entry(&smack_known_floor);
-	smk_insert_entry(&smack_known_invalid);
 	smk_insert_entry(&smack_known_web);
 }
 
diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c
index 6492fe96cae4..13743a01b35b 100644
--- a/security/smack/smackfs.c
+++ b/security/smack/smackfs.c
@@ -2998,9 +2998,6 @@ static int __init init_smk_fs(void)
 	rc = smk_preset_netlabel(&smack_known_huh);
 	if (err == 0 && rc < 0)
 		err = rc;
-	rc = smk_preset_netlabel(&smack_known_invalid);
-	if (err == 0 && rc < 0)
-		err = rc;
 	rc = smk_preset_netlabel(&smack_known_star);
 	if (err == 0 && rc < 0)
 		err = rc;
diff --git a/security/tomoyo/domain.c b/security/tomoyo/domain.c
index 682b73af7766..838ffa78cfda 100644
--- a/security/tomoyo/domain.c
+++ b/security/tomoyo/domain.c
@@ -881,7 +881,7 @@ bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos,
 	 * the execve().
 	 */
 	if (get_user_pages_remote(current, bprm->mm, pos, 1,
-				FOLL_FORCE, &page, NULL) <= 0)
+				FOLL_FORCE, &page, NULL, NULL) <= 0)
 		return false;
 #else
 	page = bprm->page[pos / PAGE_SIZE];
diff --git a/security/yama/yama_lsm.c b/security/yama/yama_lsm.c
index 0309f2111c70..968e5e0a3f81 100644
--- a/security/yama/yama_lsm.c
+++ b/security/yama/yama_lsm.c
@@ -309,7 +309,7 @@ static int task_is_descendant(struct task_struct *parent,
  * @tracer: the task_struct of the process attempting ptrace
  * @tracee: the task_struct of the process to be ptraced
  *
- * Returns 1 if tracer has is ptracer exception ancestor for tracee.
+ * Returns 1 if tracer has a ptracer exception ancestor for tracee.
  */
 static int ptracer_exception_found(struct task_struct *tracer,
 				   struct task_struct *tracee)
@@ -320,6 +320,18 @@ static int ptracer_exception_found(struct task_struct *tracer,
 	bool found = false;
 
 	rcu_read_lock();
+
+	/*
+	 * If there's already an active tracing relationship, then make an
+	 * exception for the sake of other accesses, like process_vm_rw().
+	 */
+	parent = ptrace_parent(tracee);
+	if (parent != NULL && same_thread_group(parent, tracer)) {
+		rc = 1;
+		goto unlock;
+	}
+
+	/* Look for a PR_SET_PTRACER relationship. */
 	if (!thread_group_leader(tracee))
 		tracee = rcu_dereference(tracee->group_leader);
 	list_for_each_entry_rcu(relation, &ptracer_relations, node) {
@@ -334,6 +346,8 @@ static int ptracer_exception_found(struct task_struct *tracer,
 
 	if (found && (parent == NULL || task_is_descendant(parent, tracer)))
 		rc = 1;
+
+unlock:
 	rcu_read_unlock();
 
 	return rc;
diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c
index ebc9fdfe64df..698a01419515 100644
--- a/sound/core/oss/pcm_oss.c
+++ b/sound/core/oss/pcm_oss.c
@@ -2501,7 +2501,7 @@ static long snd_pcm_oss_ioctl(struct file *file, unsigned int cmd, unsigned long
 		return put_user(SNDRV_OSS_VERSION, p);
 	if (cmd == OSS_ALSAEMULVER)
 		return put_user(1, p);
-#if defined(CONFIG_SND_MIXER_OSS) || (defined(MODULE) && defined(CONFIG_SND_MIXER_OSS_MODULE))
+#if IS_REACHABLE(CONFIG_SND_MIXER_OSS)
 	if (((cmd >> 8) & 0xff) == 'M')	{	/* mixer ioctl - for OSS compatibility */
 		struct snd_pcm_substream *substream;
 		int idx;
diff --git a/sound/core/rawmidi.c b/sound/core/rawmidi.c
index b450a27588c8..2096bb0835c8 100644
--- a/sound/core/rawmidi.c
+++ b/sound/core/rawmidi.c
@@ -1610,7 +1610,7 @@ static int snd_rawmidi_dev_free(struct snd_device *device)
 	return snd_rawmidi_free(rmidi);
 }
 
-#if defined(CONFIG_SND_SEQUENCER) || (defined(MODULE) && defined(CONFIG_SND_SEQUENCER_MODULE))
+#if IS_REACHABLE(CONFIG_SND_SEQUENCER)
 static void snd_rawmidi_dev_seq_free(struct snd_seq_device *device)
 {
 	struct snd_rawmidi *rmidi = device->private_data;
@@ -1691,7 +1691,7 @@ static int snd_rawmidi_dev_register(struct snd_device *device)
 		}
 	}
 	rmidi->proc_entry = entry;
-#if defined(CONFIG_SND_SEQUENCER) || (defined(MODULE) && defined(CONFIG_SND_SEQUENCER_MODULE))
+#if IS_REACHABLE(CONFIG_SND_SEQUENCER)
 	if (!rmidi->ops || !rmidi->ops->dev_register) { /* own registration mechanism */
 		if (snd_seq_device_new(rmidi->card, rmidi->device, SNDRV_SEQ_DEV_ID_MIDISYNTH, 0, &rmidi->seq_dev) >= 0) {
 			rmidi->seq_dev->private_data = rmidi;
diff --git a/sound/drivers/opl3/opl3_lib.c b/sound/drivers/opl3/opl3_lib.c
index 369cef212ea9..cd9e9f31720f 100644
--- a/sound/drivers/opl3/opl3_lib.c
+++ b/sound/drivers/opl3/opl3_lib.c
@@ -528,7 +528,7 @@ int snd_opl3_hwdep_new(struct snd_opl3 * opl3,
 
 	opl3->hwdep = hw;
 	opl3->seq_dev_num = seq_device;
-#if defined(CONFIG_SND_SEQUENCER) || (defined(MODULE) && defined(CONFIG_SND_SEQUENCER_MODULE))
+#if IS_REACHABLE(CONFIG_SND_SEQUENCER)
 	if (snd_seq_device_new(card, seq_device, SNDRV_SEQ_DEV_ID_OPL3,
 			       sizeof(struct snd_opl3 *), &opl3->seq_dev) >= 0) {
 		strcpy(opl3->seq_dev->name, hw->name);
diff --git a/sound/firewire/bebob/bebob.c b/sound/firewire/bebob/bebob.c
index 3469ac14c89c..730ea91d9be8 100644
--- a/sound/firewire/bebob/bebob.c
+++ b/sound/firewire/bebob/bebob.c
@@ -172,12 +172,12 @@ get_saffire_spec(struct fw_unit *unit)
 static bool
 check_audiophile_booted(struct fw_unit *unit)
 {
-	char name[24] = {0};
+	char name[28] = {0};
 
 	if (fw_csr_string(unit->directory, CSR_MODEL, name, sizeof(name)) < 0)
 		return false;
 
-	return strncmp(name, "FW Audiophile Bootloader", 15) != 0;
+	return strncmp(name, "FW Audiophile Bootloader", 24) != 0;
 }
 
 static void
diff --git a/sound/hda/hdac_i915.c b/sound/hda/hdac_i915.c
index c9af022676c2..0659bf389489 100644
--- a/sound/hda/hdac_i915.c
+++ b/sound/hda/hdac_i915.c
@@ -193,6 +193,7 @@ static int pin2port(struct hdac_device *codec, hda_nid_t pin_nid)
  * snd_hdac_sync_audio_rate - Set N/CTS based on the sample rate
  * @codec: HDA codec
  * @nid: the pin widget NID
+ * @dev_id: device identifier
  * @rate: the sample rate to set
  *
  * This function is supposed to be used only by a HD-audio controller
@@ -201,18 +202,20 @@ static int pin2port(struct hdac_device *codec, hda_nid_t pin_nid)
  * This function sets N/CTS value based on the given sample rate.
  * Returns zero for success, or a negative error code.
  */
-int snd_hdac_sync_audio_rate(struct hdac_device *codec, hda_nid_t nid, int rate)
+int snd_hdac_sync_audio_rate(struct hdac_device *codec, hda_nid_t nid,
+			     int dev_id, int rate)
 {
 	struct hdac_bus *bus = codec->bus;
 	struct i915_audio_component *acomp = bus->audio_component;
-	int port;
+	int port, pipe;
 
 	if (!acomp || !acomp->ops || !acomp->ops->sync_audio_rate)
 		return -ENODEV;
 	port = pin2port(codec, nid);
 	if (port < 0)
 		return -EINVAL;
-	return acomp->ops->sync_audio_rate(acomp->dev, port, rate);
+	pipe = dev_id;
+	return acomp->ops->sync_audio_rate(acomp->dev, port, pipe, rate);
 }
 EXPORT_SYMBOL_GPL(snd_hdac_sync_audio_rate);
 
@@ -220,6 +223,7 @@ EXPORT_SYMBOL_GPL(snd_hdac_sync_audio_rate);
  * snd_hdac_acomp_get_eld - Get the audio state and ELD via component
  * @codec: HDA codec
  * @nid: the pin widget NID
+ * @dev_id: device identifier
  * @audio_enabled: the pointer to store the current audio state
  * @buffer: the buffer pointer to store ELD bytes
  * @max_bytes: the max bytes to be stored on @buffer
@@ -236,12 +240,12 @@ EXPORT_SYMBOL_GPL(snd_hdac_sync_audio_rate);
  * thus it may be over @max_bytes.  If it's over @max_bytes, it implies
  * that only a part of ELD bytes have been fetched.
  */
-int snd_hdac_acomp_get_eld(struct hdac_device *codec, hda_nid_t nid,
+int snd_hdac_acomp_get_eld(struct hdac_device *codec, hda_nid_t nid, int dev_id,
 			   bool *audio_enabled, char *buffer, int max_bytes)
 {
 	struct hdac_bus *bus = codec->bus;
 	struct i915_audio_component *acomp = bus->audio_component;
-	int port;
+	int port, pipe;
 
 	if (!acomp || !acomp->ops || !acomp->ops->get_eld)
 		return -ENODEV;
@@ -249,7 +253,9 @@ int snd_hdac_acomp_get_eld(struct hdac_device *codec, hda_nid_t nid,
 	port = pin2port(codec, nid);
 	if (port < 0)
 		return -EINVAL;
-	return acomp->ops->get_eld(acomp->dev, port, audio_enabled,
+
+	pipe = dev_id;
+	return acomp->ops->get_eld(acomp->dev, port, pipe, audio_enabled,
 				   buffer, max_bytes);
 }
 EXPORT_SYMBOL_GPL(snd_hdac_acomp_get_eld);
diff --git a/sound/pci/ac97/ac97_codec.c b/sound/pci/ac97/ac97_codec.c
index 82259ca61e64..1ef7cdf1d3e8 100644
--- a/sound/pci/ac97/ac97_codec.c
+++ b/sound/pci/ac97/ac97_codec.c
@@ -1907,7 +1907,7 @@ static int ac97_reset_wait(struct snd_ac97 *ac97, int timeout, int with_modem)
  * write).  The other callbacks, wait and reset, are not mandatory.
  * 
  * The clock is set to 48000.  If another clock is needed, set
- * (*rbus)->clock manually.
+ * ``(*rbus)->clock`` manually.
  *
  * The AC97 bus instance is registered as a low-level device, so you don't
  * have to release it manually.
diff --git a/sound/pci/als4000.c b/sound/pci/als4000.c
index edabe1371660..92bc06d01288 100644
--- a/sound/pci/als4000.c
+++ b/sound/pci/als4000.c
@@ -84,7 +84,7 @@ MODULE_DESCRIPTION("Avance Logic ALS4000");
 MODULE_LICENSE("GPL");
 MODULE_SUPPORTED_DEVICE("{{Avance Logic,ALS4000}}");
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 #define SUPPORT_JOYSTICK 1
 #endif
 
diff --git a/sound/pci/au88x0/au88x0_game.c b/sound/pci/au88x0/au88x0_game.c
index 151815b857a0..53abcd3eccbd 100644
--- a/sound/pci/au88x0/au88x0_game.c
+++ b/sound/pci/au88x0/au88x0_game.c
@@ -36,7 +36,7 @@
 #include <linux/gameport.h>
 #include <linux/export.h>
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 
 #define VORTEX_GAME_DWAIT	20	/* 20 ms */
 
diff --git a/sound/pci/azt3328.c b/sound/pci/azt3328.c
index 80c4a4456197..79b2e6b7d88b 100644
--- a/sound/pci/azt3328.c
+++ b/sound/pci/azt3328.c
@@ -212,7 +212,7 @@ MODULE_DESCRIPTION("Aztech AZF3328 (PCI168)");
 MODULE_LICENSE("GPL");
 MODULE_SUPPORTED_DEVICE("{{Aztech,AZF3328}}");
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 #define SUPPORT_GAMEPORT 1
 #endif
 
diff --git a/sound/pci/cmipci.c b/sound/pci/cmipci.c
index 73f593526b2d..aeedc270ed9b 100644
--- a/sound/pci/cmipci.c
+++ b/sound/pci/cmipci.c
@@ -48,7 +48,7 @@ MODULE_SUPPORTED_DEVICE("{{C-Media,CMI8738},"
 		"{C-Media,CMI8338A},"
 		"{C-Media,CMI8338B}}");
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 #define SUPPORT_JOYSTICK 1
 #endif
 
diff --git a/sound/pci/cs4281.c b/sound/pci/cs4281.c
index 615d8a99d8c8..8f0f5f24e40e 100644
--- a/sound/pci/cs4281.c
+++ b/sound/pci/cs4281.c
@@ -1194,7 +1194,7 @@ static void snd_cs4281_proc_init(struct cs4281 *chip)
  * joystick support
  */
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 
 static void snd_cs4281_gameport_trigger(struct gameport *gameport)
 {
diff --git a/sound/pci/cs46xx/cs46xx_lib.c b/sound/pci/cs46xx/cs46xx_lib.c
index 528102cc2d5d..fde3cd48258c 100644
--- a/sound/pci/cs46xx/cs46xx_lib.c
+++ b/sound/pci/cs46xx/cs46xx_lib.c
@@ -2718,7 +2718,7 @@ int snd_cs46xx_midi(struct snd_cs46xx *chip, int device)
  * gameport interface
  */
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 
 static void snd_cs46xx_gameport_trigger(struct gameport *gameport)
 {
diff --git a/sound/pci/cs46xx/dsp_spos.c b/sound/pci/cs46xx/dsp_spos.c
index 4a0cbd2241d8..aa61615288ff 100644
--- a/sound/pci/cs46xx/dsp_spos.c
+++ b/sound/pci/cs46xx/dsp_spos.c
@@ -107,7 +107,8 @@ static int shadow_and_reallocate_code (struct snd_cs46xx * chip, u32 * data, u32
             
 						dev_dbg(chip->card->dev,
 							"handle_wideop:[2] %05x:%05x addr %04x\n",
-							hival, loval, address);						nreallocated++;
+							hival, loval, address);
+						nreallocated++;
 					} /* wide_opcodes[j] == wide_op */
 				} /* for */
 			} /* mod_type == 0 ... */
diff --git a/sound/pci/echoaudio/layla24_dsp.c b/sound/pci/echoaudio/layla24_dsp.c
index df28e5117359..c02bc1dcc170 100644
--- a/sound/pci/echoaudio/layla24_dsp.c
+++ b/sound/pci/echoaudio/layla24_dsp.c
@@ -135,7 +135,7 @@ static int load_asic(struct echoaudio *chip)
 	err = load_asic_generic(chip, DSP_FNC_LOAD_LAYLA24_EXTERNAL_ASIC,
 				FW_LAYLA24_2S_ASIC);
 	if (err < 0)
-		return false;
+		return err;
 
 	/* Now give the external ASIC a little time to set up */
 	mdelay(10);
diff --git a/sound/pci/emu10k1/emu10k1.c b/sound/pci/emu10k1/emu10k1.c
index db7a2e5e4a14..6a0e49ac5273 100644
--- a/sound/pci/emu10k1/emu10k1.c
+++ b/sound/pci/emu10k1/emu10k1.c
@@ -37,7 +37,7 @@ MODULE_LICENSE("GPL");
 MODULE_SUPPORTED_DEVICE("{{Creative Labs,SB Live!/PCI512/E-mu APS},"
 	       "{Creative Labs,SB Audigy}}");
 
-#if defined(CONFIG_SND_SEQUENCER) || (defined(MODULE) && defined(CONFIG_SND_SEQUENCER_MODULE))
+#if IS_REACHABLE(CONFIG_SND_SEQUENCER)
 #define ENABLE_SYNTH
 #include <sound/emu10k1_synth.h>
 #endif
@@ -194,6 +194,9 @@ static int snd_card_emu10k1_probe(struct pci_dev *pci,
 	if ((err = snd_card_register(card)) < 0)
 		goto error;
 
+	if (emu->card_capabilities->emu_model)
+		schedule_delayed_work(&emu->emu1010.firmware_work, 0);
+
 	pci_set_drvdata(pci, card);
 	dev++;
 	return 0;
@@ -219,6 +222,8 @@ static int snd_emu10k1_suspend(struct device *dev)
 
 	emu->suspend = 1;
 
+	cancel_delayed_work_sync(&emu->emu1010.firmware_work);
+
 	snd_pcm_suspend_all(emu->pcm);
 	snd_pcm_suspend_all(emu->pcm_mic);
 	snd_pcm_suspend_all(emu->pcm_efx);
@@ -252,6 +257,10 @@ static int snd_emu10k1_resume(struct device *dev)
 	emu->suspend = 0;
 
 	snd_power_change_state(card, SNDRV_CTL_POWER_D0);
+
+	if (emu->card_capabilities->emu_model)
+		schedule_delayed_work(&emu->emu1010.firmware_work, 0);
+
 	return 0;
 }
 
diff --git a/sound/pci/emu10k1/emu10k1_main.c b/sound/pci/emu10k1/emu10k1_main.c
index 891453451543..ccf4415a1c7b 100644
--- a/sound/pci/emu10k1/emu10k1_main.c
+++ b/sound/pci/emu10k1/emu10k1_main.c
@@ -32,7 +32,6 @@
  */
 
 #include <linux/sched.h>
-#include <linux/kthread.h>
 #include <linux/delay.h>
 #include <linux/init.h>
 #include <linux/module.h>
@@ -662,7 +661,7 @@ static int snd_emu10k1_cardbus_init(struct snd_emu10k1 *emu)
 	return 0;
 }
 
-static int snd_emu1010_load_firmware(struct snd_emu10k1 *emu,
+static int snd_emu1010_load_firmware_entry(struct snd_emu10k1 *emu,
 				     const struct firmware *fw_entry)
 {
 	int n, i;
@@ -708,98 +707,104 @@ static int snd_emu1010_load_firmware(struct snd_emu10k1 *emu,
 	return 0;
 }
 
-static int emu1010_firmware_thread(void *data)
+/* firmware file names, per model, init-fw and dock-fw (optional) */
+static const char * const firmware_names[5][2] = {
+	[EMU_MODEL_EMU1010] = {
+		HANA_FILENAME, DOCK_FILENAME
+	},
+	[EMU_MODEL_EMU1010B] = {
+		EMU1010B_FILENAME, MICRO_DOCK_FILENAME
+	},
+	[EMU_MODEL_EMU1616] = {
+		EMU1010_NOTEBOOK_FILENAME, MICRO_DOCK_FILENAME
+	},
+	[EMU_MODEL_EMU0404] = {
+		EMU0404_FILENAME, NULL
+	},
+};
+
+static int snd_emu1010_load_firmware(struct snd_emu10k1 *emu, int dock,
+				     const struct firmware **fw)
 {
-	struct snd_emu10k1 *emu = data;
+	const char *filename;
+	int err;
+
+	if (!*fw) {
+		filename = firmware_names[emu->card_capabilities->emu_model][dock];
+		if (!filename)
+			return 0;
+		err = request_firmware(fw, filename, &emu->pci->dev);
+		if (err)
+			return err;
+	}
+
+	return snd_emu1010_load_firmware_entry(emu, *fw);
+}
+
+static void emu1010_firmware_work(struct work_struct *work)
+{
+	struct snd_emu10k1 *emu;
 	u32 tmp, tmp2, reg;
-	u32 last_reg = 0;
 	int err;
 
-	for (;;) {
-		/* Delay to allow Audio Dock to settle */
-		msleep_interruptible(1000);
-		if (kthread_should_stop())
-			break;
+	emu = container_of(work, struct snd_emu10k1,
+			   emu1010.firmware_work.work);
+	if (emu->card->shutdown)
+		return;
 #ifdef CONFIG_PM_SLEEP
-		if (emu->suspend)
-			continue;
+	if (emu->suspend)
+		return;
 #endif
-		snd_emu1010_fpga_read(emu, EMU_HANA_IRQ_STATUS, &tmp); /* IRQ Status */
-		snd_emu1010_fpga_read(emu, EMU_HANA_OPTION_CARDS, &reg); /* OPTIONS: Which cards are attached to the EMU */
-		if (reg & EMU_HANA_OPTION_DOCK_OFFLINE) {
-			/* Audio Dock attached */
-			/* Return to Audio Dock programming mode */
-			dev_info(emu->card->dev,
-				 "emu1010: Loading Audio Dock Firmware\n");
-			snd_emu1010_fpga_write(emu, EMU_HANA_FPGA_CONFIG, EMU_HANA_FPGA_CONFIG_AUDIODOCK);
-
-			if (!emu->dock_fw) {
-				const char *filename = NULL;
-				switch (emu->card_capabilities->emu_model) {
-				case EMU_MODEL_EMU1010:
-					filename = DOCK_FILENAME;
-					break;
-				case EMU_MODEL_EMU1010B:
-					filename = MICRO_DOCK_FILENAME;
-					break;
-				case EMU_MODEL_EMU1616:
-					filename = MICRO_DOCK_FILENAME;
-					break;
-				}
-				if (filename) {
-					err = request_firmware(&emu->dock_fw,
-							       filename,
-							       &emu->pci->dev);
-					if (err)
-						continue;
-				}
-			}
-
-			if (emu->dock_fw) {
-				err = snd_emu1010_load_firmware(emu, emu->dock_fw);
-				if (err)
-					continue;
-			}
+	snd_emu1010_fpga_read(emu, EMU_HANA_IRQ_STATUS, &tmp); /* IRQ Status */
+	snd_emu1010_fpga_read(emu, EMU_HANA_OPTION_CARDS, &reg); /* OPTIONS: Which cards are attached to the EMU */
+	if (reg & EMU_HANA_OPTION_DOCK_OFFLINE) {
+		/* Audio Dock attached */
+		/* Return to Audio Dock programming mode */
+		dev_info(emu->card->dev,
+			 "emu1010: Loading Audio Dock Firmware\n");
+		snd_emu1010_fpga_write(emu, EMU_HANA_FPGA_CONFIG,
+				       EMU_HANA_FPGA_CONFIG_AUDIODOCK);
+		err = snd_emu1010_load_firmware(emu, 1, &emu->dock_fw);
+		if (err < 0)
+			goto next;
 
-			snd_emu1010_fpga_write(emu, EMU_HANA_FPGA_CONFIG, 0);
-			snd_emu1010_fpga_read(emu, EMU_HANA_IRQ_STATUS, &reg);
-			dev_info(emu->card->dev,
-				 "emu1010: EMU_HANA+DOCK_IRQ_STATUS = 0x%x\n",
-				 reg);
-			/* ID, should read & 0x7f = 0x55 when FPGA programmed. */
-			snd_emu1010_fpga_read(emu, EMU_HANA_ID, &reg);
-			dev_info(emu->card->dev,
-				 "emu1010: EMU_HANA+DOCK_ID = 0x%x\n", reg);
-			if ((reg & 0x1f) != 0x15) {
-				/* FPGA failed to be programmed */
-				dev_info(emu->card->dev,
-					 "emu1010: Loading Audio Dock Firmware file failed, reg = 0x%x\n",
-					 reg);
-				continue;
-			}
-			dev_info(emu->card->dev,
-				 "emu1010: Audio Dock Firmware loaded\n");
-			snd_emu1010_fpga_read(emu, EMU_DOCK_MAJOR_REV, &tmp);
-			snd_emu1010_fpga_read(emu, EMU_DOCK_MINOR_REV, &tmp2);
-			dev_info(emu->card->dev, "Audio Dock ver: %u.%u\n",
-				   tmp, tmp2);
-			/* Sync clocking between 1010 and Dock */
-			/* Allow DLL to settle */
-			msleep(10);
-			/* Unmute all. Default is muted after a firmware load */
-			snd_emu1010_fpga_write(emu, EMU_HANA_UNMUTE, EMU_UNMUTE);
-		} else if (!reg && last_reg) {
-			/* Audio Dock removed */
+		snd_emu1010_fpga_write(emu, EMU_HANA_FPGA_CONFIG, 0);
+		snd_emu1010_fpga_read(emu, EMU_HANA_IRQ_STATUS, &tmp);
+		dev_info(emu->card->dev,
+			 "emu1010: EMU_HANA+DOCK_IRQ_STATUS = 0x%x\n", tmp);
+		/* ID, should read & 0x7f = 0x55 when FPGA programmed. */
+		snd_emu1010_fpga_read(emu, EMU_HANA_ID, &tmp);
+		dev_info(emu->card->dev,
+			 "emu1010: EMU_HANA+DOCK_ID = 0x%x\n", tmp);
+		if ((tmp & 0x1f) != 0x15) {
+			/* FPGA failed to be programmed */
 			dev_info(emu->card->dev,
-				 "emu1010: Audio Dock detached\n");
-			/* Unmute all */
-			snd_emu1010_fpga_write(emu, EMU_HANA_UNMUTE, EMU_UNMUTE);
+				 "emu1010: Loading Audio Dock Firmware file failed, reg = 0x%x\n",
+				 tmp);
+			goto next;
 		}
-
-		last_reg = reg;
+		dev_info(emu->card->dev,
+			 "emu1010: Audio Dock Firmware loaded\n");
+		snd_emu1010_fpga_read(emu, EMU_DOCK_MAJOR_REV, &tmp);
+		snd_emu1010_fpga_read(emu, EMU_DOCK_MINOR_REV, &tmp2);
+		dev_info(emu->card->dev, "Audio Dock ver: %u.%u\n", tmp, tmp2);
+		/* Sync clocking between 1010 and Dock */
+		/* Allow DLL to settle */
+		msleep(10);
+		/* Unmute all. Default is muted after a firmware load */
+		snd_emu1010_fpga_write(emu, EMU_HANA_UNMUTE, EMU_UNMUTE);
+	} else if (!reg && emu->emu1010.last_reg) {
+		/* Audio Dock removed */
+		dev_info(emu->card->dev, "emu1010: Audio Dock detached\n");
+		/* Unmute all */
+		snd_emu1010_fpga_write(emu, EMU_HANA_UNMUTE, EMU_UNMUTE);
 	}
-	dev_info(emu->card->dev, "emu1010: firmware thread stopping\n");
-	return 0;
+
+ next:
+	emu->emu1010.last_reg = reg;
+	if (!emu->card->shutdown)
+		schedule_delayed_work(&emu->emu1010.firmware_work,
+				      msecs_to_jiffies(1000));
 }
 
 /*
@@ -881,39 +886,8 @@ static int snd_emu10k1_emu1010_init(struct snd_emu10k1 *emu)
 	}
 	dev_info(emu->card->dev, "emu1010: EMU_HANA_ID = 0x%x\n", reg);
 
-	if (!emu->firmware) {
-		const char *filename;
-		switch (emu->card_capabilities->emu_model) {
-		case EMU_MODEL_EMU1010:
-			filename = HANA_FILENAME;
-			break;
-		case EMU_MODEL_EMU1010B:
-			filename = EMU1010B_FILENAME;
-			break;
-		case EMU_MODEL_EMU1616:
-			filename = EMU1010_NOTEBOOK_FILENAME;
-			break;
-		case EMU_MODEL_EMU0404:
-			filename = EMU0404_FILENAME;
-			break;
-		default:
-			return -ENODEV;
-		}
-
-		err = request_firmware(&emu->firmware, filename, &emu->pci->dev);
-		if (err != 0) {
-			dev_info(emu->card->dev,
-				 "emu1010: firmware: %s not found. Err = %d\n",
-				 filename, err);
-			return err;
-		}
-		dev_info(emu->card->dev,
-			 "emu1010: firmware file = %s, size = 0x%zx\n",
-			   filename, emu->firmware->size);
-	}
-
-	err = snd_emu1010_load_firmware(emu, emu->firmware);
-	if (err != 0) {
+	err = snd_emu1010_load_firmware(emu, 0, &emu->firmware);
+	if (err < 0) {
 		dev_info(emu->card->dev, "emu1010: Loading Firmware failed\n");
 		return err;
 	}
@@ -1136,22 +1110,6 @@ static int snd_emu10k1_emu1010_init(struct snd_emu10k1 *emu)
 	snd_emu1010_fpga_read(emu, EMU_HANA_SPDIF_MODE, &tmp);
 	snd_emu1010_fpga_write(emu, EMU_HANA_SPDIF_MODE, 0x10); /* SPDIF Format spdif  (or 0x11 for aes/ebu) */
 
-	/* Start Micro/Audio Dock firmware loader thread */
-	if (!emu->emu1010.firmware_thread) {
-		emu->emu1010.firmware_thread =
-			kthread_create(emu1010_firmware_thread, emu,
-				       "emu1010_firmware");
-		if (IS_ERR(emu->emu1010.firmware_thread)) {
-			err = PTR_ERR(emu->emu1010.firmware_thread);
-			emu->emu1010.firmware_thread = NULL;
-			dev_info(emu->card->dev,
-					"emu1010: Creating thread failed\n");
-			return err;
-		}
-
-		wake_up_process(emu->emu1010.firmware_thread);
-	}
-
 #if 0
 	snd_emu1010_fpga_link_dst_src_write(emu,
 		EMU_DST_HAMOA_DAC_LEFT1, EMU_SRC_ALICE_EMU32B + 2); /* ALICE2 bus 0xa2 */
@@ -1309,8 +1267,7 @@ static int snd_emu10k1_free(struct snd_emu10k1 *emu)
 		/* Disable 48Volt power to Audio Dock */
 		snd_emu1010_fpga_write(emu, EMU_HANA_DOCK_PWR, 0);
 	}
-	if (emu->emu1010.firmware_thread)
-		kthread_stop(emu->emu1010.firmware_thread);
+	cancel_delayed_work_sync(&emu->emu1010.firmware_work);
 	release_firmware(emu->firmware);
 	release_firmware(emu->dock_fw);
 	if (emu->irq >= 0)
@@ -1852,6 +1809,7 @@ int snd_emu10k1_create(struct snd_card *card,
 	emu->irq = -1;
 	emu->synth = NULL;
 	emu->get_synth_voice = NULL;
+	INIT_DELAYED_WORK(&emu->emu1010.firmware_work, emu1010_firmware_work);
 	/* read revision & serial */
 	emu->revision = pci->revision;
 	pci_read_config_dword(pci, PCI_SUBSYSTEM_VENDOR_ID, &emu->serial);
diff --git a/sound/pci/ens1370.c b/sound/pci/ens1370.c
index 7e760fed0728..51736c2b5a00 100644
--- a/sound/pci/ens1370.c
+++ b/sound/pci/ens1370.c
@@ -79,7 +79,7 @@ MODULE_SUPPORTED_DEVICE("{{Ensoniq,AudioPCI ES1371/73},"
 		"{Ectiva,EV1938}}");
 #endif
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 #define SUPPORT_JOYSTICK
 #endif
 
diff --git a/sound/pci/es1938.c b/sound/pci/es1938.c
index 681355829484..e8d943071a8c 100644
--- a/sound/pci/es1938.c
+++ b/sound/pci/es1938.c
@@ -72,7 +72,7 @@ MODULE_SUPPORTED_DEVICE("{{ESS,ES1938},"
                 "{ESS,ES1969},"
 		"{TerraTec,128i PCI}}");
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 #define SUPPORT_JOYSTICK 1
 #endif
 
diff --git a/sound/pci/es1968.c b/sound/pci/es1968.c
index 8146fb76a4ad..2ec2b1ce0af6 100644
--- a/sound/pci/es1968.c
+++ b/sound/pci/es1968.c
@@ -126,7 +126,7 @@ MODULE_SUPPORTED_DEVICE("{{ESS,Maestro 2e},"
 		"{ESS,Maestro 1},"
 		"{TerraTec,DMX}}");
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 #define SUPPORT_JOYSTICK 1
 #endif
 
diff --git a/sound/pci/hda/hda_auto_parser.c b/sound/pci/hda/hda_auto_parser.c
index 7f57a145a47e..a03cf68d0bcd 100644
--- a/sound/pci/hda/hda_auto_parser.c
+++ b/sound/pci/hda/hda_auto_parser.c
@@ -884,6 +884,8 @@ void snd_hda_apply_fixup(struct hda_codec *codec, int action)
 }
 EXPORT_SYMBOL_GPL(snd_hda_apply_fixup);
 
+#define IGNORE_SEQ_ASSOC (~(AC_DEFCFG_SEQUENCE | AC_DEFCFG_DEF_ASSOC))
+
 static bool pin_config_match(struct hda_codec *codec,
 			     const struct hda_pintbl *pins)
 {
@@ -901,7 +903,7 @@ static bool pin_config_match(struct hda_codec *codec,
 		for (; t_pins->nid; t_pins++) {
 			if (t_pins->nid == nid) {
 				found = 1;
-				if (t_pins->val == cfg)
+				if ((t_pins->val & IGNORE_SEQ_ASSOC) == (cfg & IGNORE_SEQ_ASSOC))
 					break;
 				else if ((cfg & 0xf0000000) == 0x40000000 && (t_pins->val & 0xf0000000) == 0x40000000)
 					break;
diff --git a/sound/pci/hda/patch_ca0132.c b/sound/pci/hda/patch_ca0132.c
index ad06866d7c69..11b9b2f17a2e 100644
--- a/sound/pci/hda/patch_ca0132.c
+++ b/sound/pci/hda/patch_ca0132.c
@@ -780,6 +780,7 @@ static const struct hda_pintbl alienware_pincfgs[] = {
 static const struct snd_pci_quirk ca0132_quirks[] = {
 	SND_PCI_QUIRK(0x1028, 0x0685, "Alienware 15 2015", QUIRK_ALIENWARE),
 	SND_PCI_QUIRK(0x1028, 0x0688, "Alienware 17 2015", QUIRK_ALIENWARE),
+	SND_PCI_QUIRK(0x1028, 0x0708, "Alienware 15 R2 2016", QUIRK_ALIENWARE),
 	{}
 };
 
diff --git a/sound/pci/hda/patch_conexant.c b/sound/pci/hda/patch_conexant.c
index ed62748a6d55..c15c51bea26d 100644
--- a/sound/pci/hda/patch_conexant.c
+++ b/sound/pci/hda/patch_conexant.c
@@ -262,6 +262,7 @@ enum {
 	CXT_FIXUP_CAP_MIX_AMP_5047,
 	CXT_FIXUP_MUTE_LED_EAPD,
 	CXT_FIXUP_HP_SPECTRE,
+	CXT_FIXUP_HP_GATE_MIC,
 };
 
 /* for hda_fixup_thinkpad_acpi() */
@@ -633,6 +634,17 @@ static void cxt_fixup_cap_mix_amp_5047(struct hda_codec *codec,
 				  (1 << AC_AMPCAP_MUTE_SHIFT));
 }
 
+static void cxt_fixup_hp_gate_mic_jack(struct hda_codec *codec,
+				       const struct hda_fixup *fix,
+				       int action)
+{
+	/* the mic pin (0x19) doesn't give an unsolicited event;
+	 * probe the mic pin together with the headphone pin (0x16)
+	 */
+	if (action == HDA_FIXUP_ACT_PROBE)
+		snd_hda_jack_set_gating_jack(codec, 0x19, 0x16);
+}
+
 /* ThinkPad X200 & co with cxt5051 */
 static const struct hda_pintbl cxt_pincfg_lenovo_x200[] = {
 	{ 0x16, 0x042140ff }, /* HP (seq# overridden) */
@@ -774,6 +786,10 @@ static const struct hda_fixup cxt_fixups[] = {
 			{ }
 		}
 	},
+	[CXT_FIXUP_HP_GATE_MIC] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = cxt_fixup_hp_gate_mic_jack,
+	},
 };
 
 static const struct snd_pci_quirk cxt5045_fixups[] = {
@@ -824,6 +840,7 @@ static const struct snd_pci_quirk cxt5066_fixups[] = {
 	SND_PCI_QUIRK(0x1025, 0x054c, "Acer Aspire 3830TG", CXT_FIXUP_ASPIRE_DMIC),
 	SND_PCI_QUIRK(0x1025, 0x054f, "Acer Aspire 4830T", CXT_FIXUP_ASPIRE_DMIC),
 	SND_PCI_QUIRK(0x103c, 0x8174, "HP Spectre x360", CXT_FIXUP_HP_SPECTRE),
+	SND_PCI_QUIRK(0x103c, 0x8115, "HP Z1 Gen3", CXT_FIXUP_HP_GATE_MIC),
 	SND_PCI_QUIRK(0x1043, 0x138d, "Asus", CXT_FIXUP_HEADPHONE_MIC_PIN),
 	SND_PCI_QUIRK(0x152d, 0x0833, "OLPC XO-1.5", CXT_FIXUP_OLPC_XO),
 	SND_PCI_QUIRK(0x17aa, 0x20f2, "Lenovo T400", CXT_PINCFG_LENOVO_TP410),
diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c
index 56e5204ac9c1..cf9bc042fe96 100644
--- a/sound/pci/hda/patch_hdmi.c
+++ b/sound/pci/hda/patch_hdmi.c
@@ -1485,7 +1485,7 @@ static void sync_eld_via_acomp(struct hda_codec *codec,
 
 	mutex_lock(&per_pin->lock);
 	eld->monitor_present = false;
-	size = snd_hdac_acomp_get_eld(&codec->core, per_pin->pin_nid,
+	size = snd_hdac_acomp_get_eld(&codec->core, per_pin->pin_nid, -1,
 				      &eld->monitor_present, eld->eld_buffer,
 				      ELD_MAX_SIZE);
 	if (size > 0) {
@@ -1744,7 +1744,8 @@ static int generic_hdmi_playback_pcm_prepare(struct hda_pcm_stream *hinfo,
 	/* Call sync_audio_rate to set the N/CTS/M manually if necessary */
 	/* Todo: add DP1.2 MST audio support later */
 	if (codec_has_acomp(codec))
-		snd_hdac_sync_audio_rate(&codec->core, pin_nid, runtime->rate);
+		snd_hdac_sync_audio_rate(&codec->core, pin_nid, -1,
+					 runtime->rate);
 
 	non_pcm = check_non_pcm_per_cvt(codec, cvt_nid);
 	mutex_lock(&per_pin->lock);
@@ -2290,7 +2291,7 @@ static void haswell_set_power_state(struct hda_codec *codec, hda_nid_t fg,
 	snd_hda_codec_set_power_to_all(codec, fg, power_state);
 }
 
-static void intel_pin_eld_notify(void *audio_ptr, int port)
+static void intel_pin_eld_notify(void *audio_ptr, int port, int pipe)
 {
 	struct hda_codec *codec = audio_ptr;
 	int pin_nid;
diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c
index ea81c08ddc7a..9448daff9d8b 100644
--- a/sound/pci/hda/patch_realtek.c
+++ b/sound/pci/hda/patch_realtek.c
@@ -420,7 +420,7 @@ static void alc_auto_setup_eapd(struct hda_codec *codec, bool on)
 }
 
 /* generic shutup callback;
- * just turning off EPAD and a little pause for avoiding pop-noise
+ * just turning off EAPD and a little pause for avoiding pop-noise
  */
 static void alc_eapd_shutup(struct hda_codec *codec)
 {
@@ -5917,6 +5917,9 @@ static const struct snd_hda_pin_quirk alc269_pin_fixup_tbl[] = {
 		{0x12, 0x90a60180},
 		{0x14, 0x90170120},
 		{0x21, 0x02211030}),
+	SND_HDA_PIN_QUIRK(0x10ec0255, 0x1028, "Dell", ALC255_FIXUP_DELL1_MIC_NO_PRESENCE,
+		{0x1b, 0x01011020},
+		{0x21, 0x02211010}),
 	SND_HDA_PIN_QUIRK(0x10ec0256, 0x1028, "Dell", ALC255_FIXUP_DELL1_MIC_NO_PRESENCE,
 		{0x12, 0x90a60160},
 		{0x14, 0x90170120},
@@ -6561,6 +6564,30 @@ static void alc662_fixup_led_gpio1(struct hda_codec *codec,
 	}
 }
 
+static void alc662_usi_automute_hook(struct hda_codec *codec,
+					 struct hda_jack_callback *jack)
+{
+	struct alc_spec *spec = codec->spec;
+	int vref;
+	msleep(200);
+	snd_hda_gen_hp_automute(codec, jack);
+
+	vref = spec->gen.hp_jack_present ? PIN_VREF80 : 0;
+	msleep(100);
+	snd_hda_codec_write(codec, 0x19, 0, AC_VERB_SET_PIN_WIDGET_CONTROL,
+			    vref);
+}
+
+static void alc662_fixup_usi_headset_mic(struct hda_codec *codec,
+				     const struct hda_fixup *fix, int action)
+{
+	struct alc_spec *spec = codec->spec;
+	if (action == HDA_FIXUP_ACT_PRE_PROBE) {
+		spec->parse_flags |= HDA_PINCFG_HEADSET_MIC;
+		spec->gen.hp_automute_hook = alc662_usi_automute_hook;
+	}
+}
+
 static struct coef_fw alc668_coefs[] = {
 	WRITE_COEF(0x01, 0xbebe), WRITE_COEF(0x02, 0xaaaa), WRITE_COEF(0x03,    0x0),
 	WRITE_COEF(0x04, 0x0180), WRITE_COEF(0x06,    0x0), WRITE_COEF(0x07, 0x0f80),
@@ -6626,6 +6653,8 @@ enum {
 	ALC891_FIXUP_DELL_MIC_NO_PRESENCE,
 	ALC662_FIXUP_ACER_VERITON,
 	ALC892_FIXUP_ASROCK_MOBO,
+	ALC662_FIXUP_USI_FUNC,
+	ALC662_FIXUP_USI_HEADSET_MODE,
 };
 
 static const struct hda_fixup alc662_fixups[] = {
@@ -6910,6 +6939,20 @@ static const struct hda_fixup alc662_fixups[] = {
 			{ }
 		}
 	},
+	[ALC662_FIXUP_USI_FUNC] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = alc662_fixup_usi_headset_mic,
+	},
+	[ALC662_FIXUP_USI_HEADSET_MODE] = {
+		.type = HDA_FIXUP_PINS,
+		.v.pins = (const struct hda_pintbl[]) {
+			{ 0x19, 0x02a1913c }, /* use as headset mic, without its own jack detect */
+			{ 0x18, 0x01a1903d },
+			{ }
+		},
+		.chained = true,
+		.chain_id = ALC662_FIXUP_USI_FUNC
+	},
 };
 
 static const struct snd_pci_quirk alc662_fixup_tbl[] = {
@@ -6945,6 +6988,7 @@ static const struct snd_pci_quirk alc662_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1043, 0x8469, "ASUS mobo", ALC662_FIXUP_NO_JACK_DETECT),
 	SND_PCI_QUIRK(0x105b, 0x0cd6, "Foxconn", ALC662_FIXUP_ASUS_MODE2),
 	SND_PCI_QUIRK(0x144d, 0xc051, "Samsung R720", ALC662_FIXUP_IDEAPAD),
+	SND_PCI_QUIRK(0x14cd, 0x5003, "USI", ALC662_FIXUP_USI_HEADSET_MODE),
 	SND_PCI_QUIRK(0x17aa, 0x38af, "Lenovo Ideapad Y550P", ALC662_FIXUP_IDEAPAD),
 	SND_PCI_QUIRK(0x17aa, 0x3a0d, "Lenovo Ideapad Y550", ALC662_FIXUP_IDEAPAD),
 	SND_PCI_QUIRK(0x1849, 0x5892, "ASRock B150M", ALC892_FIXUP_ASROCK_MOBO),
diff --git a/sound/pci/riptide/riptide.c b/sound/pci/riptide/riptide.c
index ada5f01d479c..19c9df6b0f3d 100644
--- a/sound/pci/riptide/riptide.c
+++ b/sound/pci/riptide/riptide.c
@@ -110,7 +110,7 @@
 #include <sound/opl3.h>
 #include <sound/initval.h>
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 #define SUPPORT_JOYSTICK 1
 #endif
 
diff --git a/sound/pci/sonicvibes.c b/sound/pci/sonicvibes.c
index e1a13870bb80..a6aa48c5b969 100644
--- a/sound/pci/sonicvibes.c
+++ b/sound/pci/sonicvibes.c
@@ -45,7 +45,7 @@ MODULE_DESCRIPTION("S3 SonicVibes PCI");
 MODULE_LICENSE("GPL");
 MODULE_SUPPORTED_DEVICE("{{S3,SonicVibes PCI}}");
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 #define SUPPORT_JOYSTICK 1
 #endif
 
diff --git a/sound/pci/trident/trident_main.c b/sound/pci/trident/trident_main.c
index 27f0ed840979..92ad2d7a6bf8 100644
--- a/sound/pci/trident/trident_main.c
+++ b/sound/pci/trident/trident_main.c
@@ -3120,7 +3120,7 @@ static int snd_trident_mixer(struct snd_trident *trident, int pcm_spdif_device)
  * gameport interface
  */
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 
 static unsigned char snd_trident_gameport_read(struct gameport *gameport)
 {
diff --git a/sound/pci/via82xx.c b/sound/pci/via82xx.c
index 38a17b4342a6..2d8c14e3f8d2 100644
--- a/sound/pci/via82xx.c
+++ b/sound/pci/via82xx.c
@@ -72,7 +72,7 @@ MODULE_DESCRIPTION("VIA VT82xx audio");
 MODULE_LICENSE("GPL");
 MODULE_SUPPORTED_DEVICE("{{VIA,VT82C686A/B/C,pci},{VIA,VT8233A/C,8235}}");
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 #define SUPPORT_JOYSTICK 1
 #endif
 
diff --git a/sound/pci/ymfpci/ymfpci.h b/sound/pci/ymfpci/ymfpci.h
index 149d4cb46998..aa9bb065f385 100644
--- a/sound/pci/ymfpci/ymfpci.h
+++ b/sound/pci/ymfpci/ymfpci.h
@@ -176,7 +176,7 @@
 #define YMFPCI_LEGACY2_IMOD	(1 << 15)	/* legacy IRQ mode */
 /* SIEN:IMOD 0:0 = legacy irq, 0:1 = INTA, 1:0 = serialized IRQ */
 
-#if defined(CONFIG_GAMEPORT) || (defined(MODULE) && defined(CONFIG_GAMEPORT_MODULE))
+#if IS_REACHABLE(CONFIG_GAMEPORT)
 #define SUPPORT_JOYSTICK
 #endif
 
diff --git a/sound/soc/atmel/Kconfig b/sound/soc/atmel/Kconfig
index 22aec9a1e9a4..4a56f3dfba51 100644
--- a/sound/soc/atmel/Kconfig
+++ b/sound/soc/atmel/Kconfig
@@ -78,4 +78,14 @@ config SND_ATMEL_SOC_PDMIC
 	help
 	  Say Y if you want to add support for Atmel ASoC driver for boards using
 	  PDMIC.
+
+config SND_ATMEL_SOC_TSE850_PCM5142
+	tristate "ASoC driver for the Axentia TSE-850"
+	depends on ARCH_AT91 && OF
+	depends on ATMEL_SSC && I2C
+	select SND_ATMEL_SOC_SSC_DMA
+	select SND_SOC_PCM512x_I2C
+	help
+	  Say Y if you want to add support for the ASoC driver for the
+	  Axentia TSE-850 with a PCM5142 codec.
 endif
diff --git a/sound/soc/atmel/Makefile b/sound/soc/atmel/Makefile
index a2b127bd9c87..67e10cbd4ed7 100644
--- a/sound/soc/atmel/Makefile
+++ b/sound/soc/atmel/Makefile
@@ -13,9 +13,11 @@ snd-atmel-soc-wm8904-objs := atmel_wm8904.o
 snd-soc-sam9x5-wm8731-objs := sam9x5_wm8731.o
 snd-atmel-soc-classd-objs := atmel-classd.o
 snd-atmel-soc-pdmic-objs := atmel-pdmic.o
+snd-atmel-soc-tse850-pcm5142-objs := tse850-pcm5142.o
 
 obj-$(CONFIG_SND_AT91_SOC_SAM9G20_WM8731) += snd-soc-sam9g20-wm8731.o
 obj-$(CONFIG_SND_ATMEL_SOC_WM8904) += snd-atmel-soc-wm8904.o
 obj-$(CONFIG_SND_AT91_SOC_SAM9X5_WM8731) += snd-soc-sam9x5-wm8731.o
 obj-$(CONFIG_SND_ATMEL_SOC_CLASSD) += snd-atmel-soc-classd.o
 obj-$(CONFIG_SND_ATMEL_SOC_PDMIC) += snd-atmel-soc-pdmic.o
+obj-$(CONFIG_SND_ATMEL_SOC_TSE850_PCM5142) += snd-atmel-soc-tse850-pcm5142.o
diff --git a/sound/soc/atmel/atmel_ssc_dai.c b/sound/soc/atmel/atmel_ssc_dai.c
index 16e459aedffe..a1e2c5682dcd 100644
--- a/sound/soc/atmel/atmel_ssc_dai.c
+++ b/sound/soc/atmel/atmel_ssc_dai.c
@@ -380,6 +380,7 @@ static void atmel_ssc_shutdown(struct snd_pcm_substream *substream,
 		ssc_writel(ssc_p->ssc->regs, CR, SSC_BIT(CR_SWRST));
 		/* Clear the SSC dividers */
 		ssc_p->cmr_div = ssc_p->tcmr_period = ssc_p->rcmr_period = 0;
+		ssc_p->forced_divider = 0;
 	}
 	spin_unlock_irq(&ssc_p->lock);
 
@@ -426,14 +427,17 @@ static int atmel_ssc_set_dai_clkdiv(struct snd_soc_dai *cpu_dai,
 		else
 			if (div != ssc_p->cmr_div)
 				return -EBUSY;
+		ssc_p->forced_divider |= BIT(ATMEL_SSC_CMR_DIV);
 		break;
 
 	case ATMEL_SSC_TCMR_PERIOD:
 		ssc_p->tcmr_period = div;
+		ssc_p->forced_divider |= BIT(ATMEL_SSC_TCMR_PERIOD);
 		break;
 
 	case ATMEL_SSC_RCMR_PERIOD:
 		ssc_p->rcmr_period = div;
+		ssc_p->forced_divider |= BIT(ATMEL_SSC_RCMR_PERIOD);
 		break;
 
 	default:
@@ -443,6 +447,28 @@ static int atmel_ssc_set_dai_clkdiv(struct snd_soc_dai *cpu_dai,
 	return 0;
 }
 
+/* Is the cpu-dai master of the frame clock? */
+static int atmel_ssc_cfs(struct atmel_ssc_info *ssc_p)
+{
+	switch (ssc_p->daifmt & SND_SOC_DAIFMT_MASTER_MASK) {
+	case SND_SOC_DAIFMT_CBM_CFS:
+	case SND_SOC_DAIFMT_CBS_CFS:
+		return 1;
+	}
+	return 0;
+}
+
+/* Is the cpu-dai master of the bit clock? */
+static int atmel_ssc_cbs(struct atmel_ssc_info *ssc_p)
+{
+	switch (ssc_p->daifmt & SND_SOC_DAIFMT_MASTER_MASK) {
+	case SND_SOC_DAIFMT_CBS_CFM:
+	case SND_SOC_DAIFMT_CBS_CFS:
+		return 1;
+	}
+	return 0;
+}
+
 /*
  * Configure the SSC.
  */
@@ -459,6 +485,9 @@ static int atmel_ssc_hw_params(struct snd_pcm_substream *substream,
 	u32 tfmr, rfmr, tcmr, rcmr;
 	int ret;
 	int fslen, fslen_ext;
+	u32 cmr_div;
+	u32 tcmr_period;
+	u32 rcmr_period;
 
 	/*
 	 * Currently, there is only one set of dma params for
@@ -470,6 +499,46 @@ static int atmel_ssc_hw_params(struct snd_pcm_substream *substream,
 	else
 		dir = 1;
 
+	/*
+	 * If the cpu dai should provide BCLK, but noone has provided the
+	 * divider needed for that to work, fall back to something sensible.
+	 */
+	cmr_div = ssc_p->cmr_div;
+	if (!(ssc_p->forced_divider & BIT(ATMEL_SSC_CMR_DIV)) &&
+	    atmel_ssc_cbs(ssc_p)) {
+		int bclk_rate = snd_soc_params_to_bclk(params);
+
+		if (bclk_rate < 0) {
+			dev_err(dai->dev, "unable to calculate cmr_div: %d\n",
+				bclk_rate);
+			return bclk_rate;
+		}
+
+		cmr_div = DIV_ROUND_CLOSEST(ssc_p->mck_rate, 2 * bclk_rate);
+	}
+
+	/*
+	 * If the cpu dai should provide LRCLK, but noone has provided the
+	 * dividers needed for that to work, fall back to something sensible.
+	 */
+	tcmr_period = ssc_p->tcmr_period;
+	rcmr_period = ssc_p->rcmr_period;
+	if (atmel_ssc_cfs(ssc_p)) {
+		int frame_size = snd_soc_params_to_frame_size(params);
+
+		if (frame_size < 0) {
+			dev_err(dai->dev,
+				"unable to calculate tx/rx cmr_period: %d\n",
+				frame_size);
+			return frame_size;
+		}
+
+		if (!(ssc_p->forced_divider & BIT(ATMEL_SSC_TCMR_PERIOD)))
+			tcmr_period = frame_size / 2 - 1;
+		if (!(ssc_p->forced_divider & BIT(ATMEL_SSC_RCMR_PERIOD)))
+			rcmr_period = frame_size / 2 - 1;
+	}
+
 	dma_params = ssc_p->dma_params[dir];
 
 	channels = params_channels(params);
@@ -524,7 +593,7 @@ static int atmel_ssc_hw_params(struct snd_pcm_substream *substream,
 		fslen_ext = (bits - 1) / 16;
 		fslen = (bits - 1) % 16;
 
-		rcmr =	  SSC_BF(RCMR_PERIOD, ssc_p->rcmr_period)
+		rcmr =	  SSC_BF(RCMR_PERIOD, rcmr_period)
 			| SSC_BF(RCMR_STTDLY, START_DELAY)
 			| SSC_BF(RCMR_START, SSC_START_FALLING_RF)
 			| SSC_BF(RCMR_CKI, SSC_CKI_RISING)
@@ -540,7 +609,7 @@ static int atmel_ssc_hw_params(struct snd_pcm_substream *substream,
 			| SSC_BF(RFMR_LOOP, 0)
 			| SSC_BF(RFMR_DATLEN, (bits - 1));
 
-		tcmr =	  SSC_BF(TCMR_PERIOD, ssc_p->tcmr_period)
+		tcmr =	  SSC_BF(TCMR_PERIOD, tcmr_period)
 			| SSC_BF(TCMR_STTDLY, START_DELAY)
 			| SSC_BF(TCMR_START, SSC_START_FALLING_RF)
 			| SSC_BF(TCMR_CKI, SSC_CKI_FALLING)
@@ -606,7 +675,7 @@ static int atmel_ssc_hw_params(struct snd_pcm_substream *substream,
 		fslen_ext = (bits - 1) / 16;
 		fslen = (bits - 1) % 16;
 
-		rcmr =	  SSC_BF(RCMR_PERIOD, ssc_p->rcmr_period)
+		rcmr =	  SSC_BF(RCMR_PERIOD, rcmr_period)
 			| SSC_BF(RCMR_STTDLY, START_DELAY)
 			| SSC_BF(RCMR_START, SSC_START_FALLING_RF)
 			| SSC_BF(RCMR_CKI, SSC_CKI_RISING)
@@ -623,7 +692,7 @@ static int atmel_ssc_hw_params(struct snd_pcm_substream *substream,
 			| SSC_BF(RFMR_LOOP, 0)
 			| SSC_BF(RFMR_DATLEN, (bits - 1));
 
-		tcmr =	  SSC_BF(TCMR_PERIOD, ssc_p->tcmr_period)
+		tcmr =	  SSC_BF(TCMR_PERIOD, tcmr_period)
 			| SSC_BF(TCMR_STTDLY, START_DELAY)
 			| SSC_BF(TCMR_START, SSC_START_FALLING_RF)
 			| SSC_BF(TCMR_CKI, SSC_CKI_FALLING)
@@ -650,7 +719,7 @@ static int atmel_ssc_hw_params(struct snd_pcm_substream *substream,
 		 * MCK divider, and the BCLK signal is output
 		 * on the SSC TK line.
 		 */
-		rcmr =	  SSC_BF(RCMR_PERIOD, ssc_p->rcmr_period)
+		rcmr =	  SSC_BF(RCMR_PERIOD, rcmr_period)
 			| SSC_BF(RCMR_STTDLY, 1)
 			| SSC_BF(RCMR_START, SSC_START_RISING_RF)
 			| SSC_BF(RCMR_CKI, SSC_CKI_RISING)
@@ -665,7 +734,7 @@ static int atmel_ssc_hw_params(struct snd_pcm_substream *substream,
 			| SSC_BF(RFMR_LOOP, 0)
 			| SSC_BF(RFMR_DATLEN, (bits - 1));
 
-		tcmr =	  SSC_BF(TCMR_PERIOD, ssc_p->tcmr_period)
+		tcmr =	  SSC_BF(TCMR_PERIOD, tcmr_period)
 			| SSC_BF(TCMR_STTDLY, 1)
 			| SSC_BF(TCMR_START, SSC_START_RISING_RF)
 			| SSC_BF(TCMR_CKI, SSC_CKI_FALLING)
@@ -760,7 +829,7 @@ static int atmel_ssc_hw_params(struct snd_pcm_substream *substream,
 	}
 
 	/* set SSC clock mode register */
-	ssc_writel(ssc_p->ssc->regs, CMR, ssc_p->cmr_div);
+	ssc_writel(ssc_p->ssc->regs, CMR, cmr_div);
 
 	/* set receive clock mode and format */
 	ssc_writel(ssc_p->ssc->regs, RCMR, rcmr);
diff --git a/sound/soc/atmel/atmel_ssc_dai.h b/sound/soc/atmel/atmel_ssc_dai.h
index 80b153857a88..75194f582131 100644
--- a/sound/soc/atmel/atmel_ssc_dai.h
+++ b/sound/soc/atmel/atmel_ssc_dai.h
@@ -113,6 +113,7 @@ struct atmel_ssc_info {
 	unsigned short cmr_div;
 	unsigned short tcmr_period;
 	unsigned short rcmr_period;
+	unsigned int forced_divider;
 	struct atmel_pcm_dma_params *dma_params[2];
 	struct atmel_ssc_state ssc_state;
 	unsigned long mck_rate;
diff --git a/sound/soc/atmel/atmel_wm8904.c b/sound/soc/atmel/atmel_wm8904.c
index fdd28ed3e0b9..fbc10f61eb55 100644
--- a/sound/soc/atmel/atmel_wm8904.c
+++ b/sound/soc/atmel/atmel_wm8904.c
@@ -53,7 +53,7 @@ static int atmel_asoc_wm8904_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static struct snd_soc_ops atmel_asoc_wm8904_ops = {
+static const struct snd_soc_ops atmel_asoc_wm8904_ops = {
 	.hw_params = atmel_asoc_wm8904_hw_params,
 };
 
diff --git a/sound/soc/atmel/tse850-pcm5142.c b/sound/soc/atmel/tse850-pcm5142.c
new file mode 100644
index 000000000000..ac6a814c8ecf
--- /dev/null
+++ b/sound/soc/atmel/tse850-pcm5142.c
@@ -0,0 +1,472 @@
+/*
+ * TSE-850 audio - ASoC driver for the Axentia TSE-850 with a PCM5142 codec
+ *
+ * Copyright (C) 2016 Axentia Technologies AB
+ *
+ * Author: Peter Rosin <peda@axentia.se>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/*
+ *               loop1 relays
+ *   IN1 +---o  +------------+  o---+ OUT1
+ *            \                /
+ *             +              +
+ *             |   /          |
+ *             +--o  +--.     |
+ *             |  add   |     |
+ *             |        V     |
+ *             |      .---.   |
+ *   DAC +----------->|Sum|---+
+ *             |      '---'   |
+ *             |              |
+ *             +              +
+ *
+ *   IN2 +---o--+------------+--o---+ OUT2
+ *               loop2 relays
+ *
+ * The 'loop1' gpio pin controlls two relays, which are either in loop
+ * position, meaning that input and output are directly connected, or
+ * they are in mixer position, meaning that the signal is passed through
+ * the 'Sum' mixer. Similarly for 'loop2'.
+ *
+ * In the above, the 'loop1' relays are inactive, thus feeding IN1 to the
+ * mixer (if 'add' is active) and feeding the mixer output to OUT1. The
+ * 'loop2' relays are active, short-cutting the TSE-850 from channel 2.
+ * IN1, IN2, OUT1 and OUT2 are TSE-850 connectors and DAC is the PCB name
+ * of the (filtered) output from the PCM5142 codec.
+ */
+
+#include <linux/clk.h>
+#include <linux/gpio.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_gpio.h>
+#include <linux/regulator/consumer.h>
+
+#include <sound/soc.h>
+#include <sound/pcm_params.h>
+
+#include "atmel_ssc_dai.h"
+
+struct tse850_priv {
+	int ssc_id;
+
+	struct gpio_desc *add;
+	struct gpio_desc *loop1;
+	struct gpio_desc *loop2;
+
+	struct regulator *ana;
+
+	int add_cache;
+	int loop1_cache;
+	int loop2_cache;
+};
+
+static int tse850_get_mux1(struct snd_kcontrol *kctrl,
+			   struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kctrl);
+	struct snd_soc_card *card = dapm->card;
+	struct tse850_priv *tse850 = snd_soc_card_get_drvdata(card);
+
+	ucontrol->value.enumerated.item[0] = tse850->loop1_cache;
+
+	return 0;
+}
+
+static int tse850_put_mux1(struct snd_kcontrol *kctrl,
+			   struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kctrl);
+	struct snd_soc_card *card = dapm->card;
+	struct tse850_priv *tse850 = snd_soc_card_get_drvdata(card);
+	struct soc_enum *e = (struct soc_enum *)kctrl->private_value;
+	unsigned int val = ucontrol->value.enumerated.item[0];
+
+	if (val >= e->items)
+		return -EINVAL;
+
+	gpiod_set_value_cansleep(tse850->loop1, val);
+	tse850->loop1_cache = val;
+
+	return snd_soc_dapm_put_enum_double(kctrl, ucontrol);
+}
+
+static int tse850_get_mux2(struct snd_kcontrol *kctrl,
+			   struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kctrl);
+	struct snd_soc_card *card = dapm->card;
+	struct tse850_priv *tse850 = snd_soc_card_get_drvdata(card);
+
+	ucontrol->value.enumerated.item[0] = tse850->loop2_cache;
+
+	return 0;
+}
+
+static int tse850_put_mux2(struct snd_kcontrol *kctrl,
+			   struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kctrl);
+	struct snd_soc_card *card = dapm->card;
+	struct tse850_priv *tse850 = snd_soc_card_get_drvdata(card);
+	struct soc_enum *e = (struct soc_enum *)kctrl->private_value;
+	unsigned int val = ucontrol->value.enumerated.item[0];
+
+	if (val >= e->items)
+		return -EINVAL;
+
+	gpiod_set_value_cansleep(tse850->loop2, val);
+	tse850->loop2_cache = val;
+
+	return snd_soc_dapm_put_enum_double(kctrl, ucontrol);
+}
+
+int tse850_get_mix(struct snd_kcontrol *kctrl,
+		   struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kctrl);
+	struct snd_soc_card *card = dapm->card;
+	struct tse850_priv *tse850 = snd_soc_card_get_drvdata(card);
+
+	ucontrol->value.enumerated.item[0] = tse850->add_cache;
+
+	return 0;
+}
+
+int tse850_put_mix(struct snd_kcontrol *kctrl,
+		   struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kctrl);
+	struct snd_soc_card *card = dapm->card;
+	struct tse850_priv *tse850 = snd_soc_card_get_drvdata(card);
+	int connect = !!ucontrol->value.integer.value[0];
+
+	if (tse850->add_cache == connect)
+		return 0;
+
+	/*
+	 * Hmmm, this gpiod_set_value_cansleep call should probably happen
+	 * inside snd_soc_dapm_mixer_update_power in the loop.
+	 */
+	gpiod_set_value_cansleep(tse850->add, connect);
+	tse850->add_cache = connect;
+
+	snd_soc_dapm_mixer_update_power(dapm, kctrl, connect, NULL);
+	return 1;
+}
+
+int tse850_get_ana(struct snd_kcontrol *kctrl,
+		   struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kctrl);
+	struct snd_soc_card *card = dapm->card;
+	struct tse850_priv *tse850 = snd_soc_card_get_drvdata(card);
+	int ret;
+
+	ret = regulator_get_voltage(tse850->ana);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Map regulator output values like so:
+	 *      -11.5V to "Low" (enum 0)
+	 * 11.5V-12.5V to "12V" (enum 1)
+	 * 12.5V-13.5V to "13V" (enum 2)
+	 *     ...
+	 * 18.5V-19.5V to "19V" (enum 8)
+	 * 19.5V-      to "20V" (enum 9)
+	 */
+	if (ret < 11000000)
+		ret = 11000000;
+	else if (ret > 20000000)
+		ret = 20000000;
+	ret -= 11000000;
+	ret = (ret + 500000) / 1000000;
+
+	ucontrol->value.enumerated.item[0] = ret;
+
+	return 0;
+}
+
+int tse850_put_ana(struct snd_kcontrol *kctrl,
+		   struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kctrl);
+	struct snd_soc_card *card = dapm->card;
+	struct tse850_priv *tse850 = snd_soc_card_get_drvdata(card);
+	struct soc_enum *e = (struct soc_enum *)kctrl->private_value;
+	unsigned int uV = ucontrol->value.enumerated.item[0];
+	int ret;
+
+	if (uV >= e->items)
+		return -EINVAL;
+
+	/*
+	 * Map enum zero (Low) to 2 volts on the regulator, do this since
+	 * the ana regulator is supplied by the system 12V voltage and
+	 * requesting anything below the system voltage causes the system
+	 * voltage to be passed through the regulator. Also, the ana
+	 * regulator induces noise when requesting voltages near the
+	 * system voltage. So, by mapping Low to 2V, that noise is
+	 * eliminated when all that is needed is 12V (the system voltage).
+	 */
+	if (uV)
+		uV = 11000000 + (1000000 * uV);
+	else
+		uV = 2000000;
+
+	ret = regulator_set_voltage(tse850->ana, uV, uV);
+	if (ret < 0)
+		return ret;
+
+	return snd_soc_dapm_put_enum_double(kctrl, ucontrol);
+}
+
+static const char * const mux_text[] = { "Mixer", "Loop" };
+
+static const struct soc_enum mux_enum =
+	SOC_ENUM_SINGLE(SND_SOC_NOPM, 0, 2, mux_text);
+
+static const struct snd_kcontrol_new mux1 =
+	SOC_DAPM_ENUM_EXT("MUX1", mux_enum, tse850_get_mux1, tse850_put_mux1);
+
+static const struct snd_kcontrol_new mux2 =
+	SOC_DAPM_ENUM_EXT("MUX2", mux_enum, tse850_get_mux2, tse850_put_mux2);
+
+#define TSE850_DAPM_SINGLE_EXT(xname, reg, shift, max, invert, xget, xput) \
+{	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \
+	.info = snd_soc_info_volsw, \
+	.get = xget, \
+	.put = xput, \
+	.private_value = SOC_SINGLE_VALUE(reg, shift, max, invert, 0) }
+
+static const struct snd_kcontrol_new mix[] = {
+	TSE850_DAPM_SINGLE_EXT("IN Switch", SND_SOC_NOPM, 0, 1, 0,
+			       tse850_get_mix, tse850_put_mix),
+};
+
+static const char * const ana_text[] = {
+	"Low", "12V", "13V", "14V", "15V", "16V", "17V", "18V", "19V", "20V"
+};
+
+static const struct soc_enum ana_enum =
+	SOC_ENUM_SINGLE(SND_SOC_NOPM, 0, 9, ana_text);
+
+static const struct snd_kcontrol_new out =
+	SOC_DAPM_ENUM_EXT("ANA", ana_enum, tse850_get_ana, tse850_put_ana);
+
+static const struct snd_soc_dapm_widget tse850_dapm_widgets[] = {
+	SND_SOC_DAPM_LINE("OUT1", NULL),
+	SND_SOC_DAPM_LINE("OUT2", NULL),
+	SND_SOC_DAPM_LINE("IN1", NULL),
+	SND_SOC_DAPM_LINE("IN2", NULL),
+	SND_SOC_DAPM_INPUT("DAC"),
+	SND_SOC_DAPM_AIF_IN("AIFINL", "Playback", 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_IN("AIFINR", "Playback", 1, SND_SOC_NOPM, 0, 0),
+	SOC_MIXER_ARRAY("MIX", SND_SOC_NOPM, 0, 0, mix),
+	SND_SOC_DAPM_MUX("MUX1", SND_SOC_NOPM, 0, 0, &mux1),
+	SND_SOC_DAPM_MUX("MUX2", SND_SOC_NOPM, 0, 0, &mux2),
+	SND_SOC_DAPM_OUT_DRV("OUT", SND_SOC_NOPM, 0, 0, &out, 1),
+};
+
+/*
+ * These connections are not entirely correct, since both IN1 and IN2
+ * are always fed to MIX (if the "IN switch" is set so), i.e. without
+ * regard to the loop1 and loop2 relays that according to this only
+ * control MUX1 and MUX2 but in fact also control how the input signals
+ * are routed.
+ * But, 1) I don't know how to do it right, and 2) it doesn't seem to
+ * matter in practice since nothing is powered in those sections anyway.
+ */
+static const struct snd_soc_dapm_route tse850_intercon[] = {
+	{ "OUT1", NULL, "MUX1" },
+	{ "OUT2", NULL, "MUX2" },
+
+	{ "MUX1", "Loop",  "IN1" },
+	{ "MUX1", "Mixer", "OUT" },
+
+	{ "MUX2", "Loop",  "IN2" },
+	{ "MUX2", "Mixer", "OUT" },
+
+	{ "OUT", NULL, "MIX" },
+
+	{ "MIX", NULL, "DAC" },
+	{ "MIX", "IN Switch", "IN1" },
+	{ "MIX", "IN Switch", "IN2" },
+
+	/* connect board input to the codec left channel output pin */
+	{ "DAC", NULL, "OUTL" },
+};
+
+static struct snd_soc_dai_link tse850_dailink = {
+	.name = "TSE-850",
+	.stream_name = "TSE-850-PCM",
+	.codec_dai_name = "pcm512x-hifi",
+	.dai_fmt = SND_SOC_DAIFMT_I2S
+		 | SND_SOC_DAIFMT_NB_NF
+		 | SND_SOC_DAIFMT_CBM_CFS,
+};
+
+static struct snd_soc_card tse850_card = {
+	.name = "TSE-850-ASoC",
+	.owner = THIS_MODULE,
+	.dai_link = &tse850_dailink,
+	.num_links = 1,
+	.dapm_widgets = tse850_dapm_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(tse850_dapm_widgets),
+	.dapm_routes = tse850_intercon,
+	.num_dapm_routes = ARRAY_SIZE(tse850_intercon),
+	.fully_routed = true,
+};
+
+static int tse850_dt_init(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct device_node *codec_np, *cpu_np;
+	struct snd_soc_card *card = &tse850_card;
+	struct snd_soc_dai_link *dailink = &tse850_dailink;
+	struct tse850_priv *tse850 = snd_soc_card_get_drvdata(card);
+
+	if (!np) {
+		dev_err(&pdev->dev, "only device tree supported\n");
+		return -EINVAL;
+	}
+
+	cpu_np = of_parse_phandle(np, "axentia,ssc-controller", 0);
+	if (!cpu_np) {
+		dev_err(&pdev->dev, "failed to get dai and pcm info\n");
+		return -EINVAL;
+	}
+	dailink->cpu_of_node = cpu_np;
+	dailink->platform_of_node = cpu_np;
+	tse850->ssc_id = of_alias_get_id(cpu_np, "ssc");
+	of_node_put(cpu_np);
+
+	codec_np = of_parse_phandle(np, "axentia,audio-codec", 0);
+	if (!codec_np) {
+		dev_err(&pdev->dev, "failed to get codec info\n");
+		return -EINVAL;
+	}
+	dailink->codec_of_node = codec_np;
+	of_node_put(codec_np);
+
+	return 0;
+}
+
+static int tse850_probe(struct platform_device *pdev)
+{
+	struct snd_soc_card *card = &tse850_card;
+	struct device *dev = card->dev = &pdev->dev;
+	struct tse850_priv *tse850;
+	int ret;
+
+	tse850 = devm_kzalloc(dev, sizeof(*tse850), GFP_KERNEL);
+	if (!tse850)
+		return -ENOMEM;
+
+	snd_soc_card_set_drvdata(card, tse850);
+
+	ret = tse850_dt_init(pdev);
+	if (ret) {
+		dev_err(dev, "failed to init dt info\n");
+		return ret;
+	}
+
+	tse850->add = devm_gpiod_get(dev, "axentia,add", GPIOD_OUT_HIGH);
+	if (IS_ERR(tse850->add)) {
+		if (PTR_ERR(tse850->add) != -EPROBE_DEFER)
+			dev_err(dev, "failed to get 'add' gpio\n");
+		return PTR_ERR(tse850->add);
+	}
+	tse850->add_cache = 1;
+
+	tse850->loop1 = devm_gpiod_get(dev, "axentia,loop1", GPIOD_OUT_HIGH);
+	if (IS_ERR(tse850->loop1)) {
+		if (PTR_ERR(tse850->loop1) != -EPROBE_DEFER)
+			dev_err(dev, "failed to get 'loop1' gpio\n");
+		return PTR_ERR(tse850->loop1);
+	}
+	tse850->loop1_cache = 1;
+
+	tse850->loop2 = devm_gpiod_get(dev, "axentia,loop2", GPIOD_OUT_HIGH);
+	if (IS_ERR(tse850->loop2)) {
+		if (PTR_ERR(tse850->loop2) != -EPROBE_DEFER)
+			dev_err(dev, "failed to get 'loop2' gpio\n");
+		return PTR_ERR(tse850->loop2);
+	}
+	tse850->loop2_cache = 1;
+
+	tse850->ana = devm_regulator_get(dev, "axentia,ana");
+	if (IS_ERR(tse850->ana)) {
+		if (PTR_ERR(tse850->ana) != -EPROBE_DEFER)
+			dev_err(dev, "failed to get 'ana' regulator\n");
+		return PTR_ERR(tse850->ana);
+	}
+
+	ret = regulator_enable(tse850->ana);
+	if (ret < 0) {
+		dev_err(dev, "failed to enable the 'ana' regulator\n");
+		return ret;
+	}
+
+	ret = atmel_ssc_set_audio(tse850->ssc_id);
+	if (ret != 0) {
+		dev_err(dev,
+			"failed to set SSC %d for audio\n", tse850->ssc_id);
+		goto err_disable_ana;
+	}
+
+	ret = snd_soc_register_card(card);
+	if (ret) {
+		dev_err(dev, "snd_soc_register_card failed\n");
+		goto err_put_audio;
+	}
+
+	return 0;
+
+err_put_audio:
+	atmel_ssc_put_audio(tse850->ssc_id);
+err_disable_ana:
+	regulator_disable(tse850->ana);
+	return ret;
+}
+
+static int tse850_remove(struct platform_device *pdev)
+{
+	struct snd_soc_card *card = platform_get_drvdata(pdev);
+	struct tse850_priv *tse850 = snd_soc_card_get_drvdata(card);
+
+	snd_soc_unregister_card(card);
+	atmel_ssc_put_audio(tse850->ssc_id);
+	regulator_disable(tse850->ana);
+
+	return 0;
+}
+
+static const struct of_device_id tse850_dt_ids[] = {
+	{ .compatible = "axentia,tse850-pcm5142", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, tse850_dt_ids);
+
+static struct platform_driver tse850_driver = {
+	.driver = {
+		.name = "axentia-tse850-pcm5142",
+		.of_match_table = of_match_ptr(tse850_dt_ids),
+	},
+	.probe = tse850_probe,
+	.remove = tse850_remove,
+};
+
+module_platform_driver(tse850_driver);
+
+/* Module information */
+MODULE_AUTHOR("Peter Rosin <peda@axentia.se>");
+MODULE_DESCRIPTION("ALSA SoC driver for TSE-850 with PCM5142 codec");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/bcm/Kconfig b/sound/soc/bcm/Kconfig
index d528aaceaad9..edf367100ebd 100644
--- a/sound/soc/bcm/Kconfig
+++ b/sound/soc/bcm/Kconfig
@@ -11,6 +11,7 @@ config SND_BCM2835_SOC_I2S
 config SND_SOC_CYGNUS
 	tristate "SoC platform audio for Broadcom Cygnus chips"
 	depends on ARCH_BCM_CYGNUS || COMPILE_TEST
+	depends on HAS_DMA
 	help
 	  Say Y if you want to add support for ASoC audio on Broadcom
 	  Cygnus chips (bcm958300, bcm958305, bcm911360)
diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig
index c67667bb970f..9e1718a8cb1c 100644
--- a/sound/soc/codecs/Kconfig
+++ b/sound/soc/codecs/Kconfig
@@ -48,6 +48,8 @@ config SND_SOC_ALL_CODECS
 	select SND_SOC_CQ0093VC if MFD_DAVINCI_VOICECODEC
 	select SND_SOC_CS35L32 if I2C
 	select SND_SOC_CS35L33 if I2C
+	select SND_SOC_CS35L34 if I2C
+	select SND_SOC_CS42L42 if I2C
 	select SND_SOC_CS42L51_I2C if I2C
 	select SND_SOC_CS42L52 if I2C && INPUT
 	select SND_SOC_CS42L56 if I2C && INPUT
@@ -83,6 +85,7 @@ config SND_SOC_ALL_CODECS
 	select SND_SOC_MAX98095 if I2C
 	select SND_SOC_MAX98357A if GPIOLIB
 	select SND_SOC_MAX98371 if I2C
+	select SND_SOC_MAX98504 if I2C
 	select SND_SOC_MAX9867 if I2C
 	select SND_SOC_MAX98925 if I2C
 	select SND_SOC_MAX98926 if I2C
@@ -114,6 +117,7 @@ config SND_SOC_ALL_CODECS
 	select SND_SOC_RT5651 if I2C
 	select SND_SOC_RT5659 if I2C
 	select SND_SOC_RT5660 if I2C
+	select SND_SOC_RT5665 if I2C
 	select SND_SOC_RT5663 if I2C
 	select SND_SOC_RT5670 if I2C
 	select SND_SOC_RT5677 if I2C && SPI_MASTER
@@ -399,6 +403,14 @@ config SND_SOC_CS35L33
 	tristate "Cirrus Logic CS35L33 CODEC"
 	depends on I2C
 
+config SND_SOC_CS35L34
+	tristate "Cirrus Logic CS35L34 CODEC"
+	depends on I2C
+
+config SND_SOC_CS42L42
+	tristate "Cirrus Logic CS42L42 CODEC"
+	depends on I2C
+
 config SND_SOC_CS42L51
 	tristate
 
@@ -581,6 +593,13 @@ config SND_SOC_MAX9860
 	depends on I2C
 	select REGMAP_I2C
 
+config SND_SOC_MSM8916_WCD_ANALOG
+	tristate "Qualcomm MSM8916 WCD Analog Codec"
+	depends on SPMI || COMPILE_TEST
+
+config SND_SOC_MSM8916_WCD_DIGITAL
+	tristate "Qualcomm MSM8916 WCD DIGITAL Codec"
+
 config SND_SOC_PCM1681
 	tristate "Texas Instruments PCM1681 CODEC"
 	depends on I2C
@@ -649,6 +668,7 @@ config SND_SOC_RL6231
 	default y if SND_SOC_RT5651=y
 	default y if SND_SOC_RT5659=y
 	default y if SND_SOC_RT5660=y
+	default y if SND_SOC_RT5665=y
 	default y if SND_SOC_RT5663=y
 	default y if SND_SOC_RT5670=y
 	default y if SND_SOC_RT5677=y
@@ -659,6 +679,7 @@ config SND_SOC_RL6231
 	default m if SND_SOC_RT5651=m
 	default m if SND_SOC_RT5659=m
 	default m if SND_SOC_RT5660=m
+	default m if SND_SOC_RT5665=m
 	default m if SND_SOC_RT5663=m
 	default m if SND_SOC_RT5670=m
 	default m if SND_SOC_RT5677=m
@@ -672,7 +693,6 @@ config SND_SOC_RL6347A
 
 config SND_SOC_RT286
 	tristate
-	select SND_SOC_RT5663
 	depends on I2C
 
 config SND_SOC_RT298
@@ -708,6 +728,9 @@ config SND_SOC_RT5659
 config SND_SOC_RT5660
 	tristate
 
+config SND_SOC_RT5665
+	tristate
+
 config SND_SOC_RT5663
 	tristate
 
@@ -874,6 +897,7 @@ config SND_SOC_UDA134X
 
 config SND_SOC_UDA1380
         tristate
+	depends on I2C
 
 config SND_SOC_WL1273
 	tristate
@@ -914,7 +938,7 @@ config SND_SOC_WM8523
 	depends on I2C
 
 config SND_SOC_WM8580
-	tristate "Wolfson Microelectronics WM8523 CODEC"
+	tristate "Wolfson Microelectronics WM8580 and WM8581 CODECs"
 	depends on I2C
 
 config SND_SOC_WM8711
@@ -1048,15 +1072,18 @@ config SND_SOC_WM8998
 
 config SND_SOC_WM9081
 	tristate
+	depends on I2C
 
 config SND_SOC_WM9090
 	tristate
 
 config SND_SOC_WM9705
 	tristate
+	select REGMAP_AC97
 
 config SND_SOC_WM9712
 	tristate
+	select REGMAP_AC97
 
 config SND_SOC_WM9713
 	tristate
diff --git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile
index 958cd4912fbc..7e1dad79610b 100644
--- a/sound/soc/codecs/Makefile
+++ b/sound/soc/codecs/Makefile
@@ -38,6 +38,8 @@ snd-soc-bt-sco-objs := bt-sco.o
 snd-soc-cq93vc-objs := cq93vc.o
 snd-soc-cs35l32-objs := cs35l32.o
 snd-soc-cs35l33-objs := cs35l33.o
+snd-soc-cs35l34-objs := cs35l34.o
+snd-soc-cs42l42-objs := cs42l42.o
 snd-soc-cs42l51-objs := cs42l51.o
 snd-soc-cs42l51-i2c-objs := cs42l51-i2c.o
 snd-soc-cs42l52-objs := cs42l52.o
@@ -86,6 +88,8 @@ snd-soc-max9850-objs := max9850.o
 snd-soc-max9860-objs := max9860.o
 snd-soc-mc13783-objs := mc13783.o
 snd-soc-ml26124-objs := ml26124.o
+snd-soc-msm8916-analog-objs := msm8916-wcd-analog.o
+snd-soc-msm8916-digital-objs := msm8916-wcd-digital.o
 snd-soc-nau8810-objs := nau8810.o
 snd-soc-nau8825-objs := nau8825.o
 snd-soc-hdmi-codec-objs := hdmi-codec.o
@@ -114,6 +118,7 @@ snd-soc-rt5645-objs := rt5645.o
 snd-soc-rt5651-objs := rt5651.o
 snd-soc-rt5659-objs := rt5659.o
 snd-soc-rt5660-objs := rt5660.o
+snd-soc-rt5665-objs := rt5665.o
 snd-soc-rt5663-objs := rt5663.o
 snd-soc-rt5670-objs := rt5670.o
 snd-soc-rt5677-objs := rt5677.o
@@ -214,7 +219,6 @@ snd-soc-wm9705-objs := wm9705.o
 snd-soc-wm9712-objs := wm9712.o
 snd-soc-wm9713-objs := wm9713.o
 snd-soc-wm-hubs-objs := wm_hubs.o
-
 # Amp
 snd-soc-max9877-objs := max9877.o
 snd-soc-max98504-objs := max98504.o
@@ -263,6 +267,8 @@ obj-$(CONFIG_SND_SOC_BT_SCO)	+= snd-soc-bt-sco.o
 obj-$(CONFIG_SND_SOC_CQ0093VC) += snd-soc-cq93vc.o
 obj-$(CONFIG_SND_SOC_CS35L32)	+= snd-soc-cs35l32.o
 obj-$(CONFIG_SND_SOC_CS35L33)	+= snd-soc-cs35l33.o
+obj-$(CONFIG_SND_SOC_CS35L34)	+= snd-soc-cs35l34.o
+obj-$(CONFIG_SND_SOC_CS42L42)	+= snd-soc-cs42l42.o
 obj-$(CONFIG_SND_SOC_CS42L51)	+= snd-soc-cs42l51.o
 obj-$(CONFIG_SND_SOC_CS42L51_I2C)	+= snd-soc-cs42l51-i2c.o
 obj-$(CONFIG_SND_SOC_CS42L52)	+= snd-soc-cs42l52.o
@@ -310,6 +316,8 @@ obj-$(CONFIG_SND_SOC_MAX9850)	+= snd-soc-max9850.o
 obj-$(CONFIG_SND_SOC_MAX9860)	+= snd-soc-max9860.o
 obj-$(CONFIG_SND_SOC_MC13783)	+= snd-soc-mc13783.o
 obj-$(CONFIG_SND_SOC_ML26124)	+= snd-soc-ml26124.o
+obj-$(CONFIG_SND_SOC_MSM8916_WCD_ANALOG) +=snd-soc-msm8916-analog.o
+obj-$(CONFIG_SND_SOC_MSM8916_WCD_DIGITAL) +=snd-soc-msm8916-digital.o
 obj-$(CONFIG_SND_SOC_NAU8810)   += snd-soc-nau8810.o
 obj-$(CONFIG_SND_SOC_NAU8825)   += snd-soc-nau8825.o
 obj-$(CONFIG_SND_SOC_HDMI_CODEC)	+= snd-soc-hdmi-codec.o
@@ -338,6 +346,7 @@ obj-$(CONFIG_SND_SOC_RT5645)	+= snd-soc-rt5645.o
 obj-$(CONFIG_SND_SOC_RT5651)	+= snd-soc-rt5651.o
 obj-$(CONFIG_SND_SOC_RT5659)	+= snd-soc-rt5659.o
 obj-$(CONFIG_SND_SOC_RT5660)	+= snd-soc-rt5660.o
+obj-$(CONFIG_SND_SOC_RT5665)	+= snd-soc-rt5665.o
 obj-$(CONFIG_SND_SOC_RT5663)	+= snd-soc-rt5663.o
 obj-$(CONFIG_SND_SOC_RT5670)	+= snd-soc-rt5670.o
 obj-$(CONFIG_SND_SOC_RT5677)	+= snd-soc-rt5677.o
diff --git a/sound/soc/codecs/ab8500-codec.c b/sound/soc/codecs/ab8500-codec.c
index 935ff7cb71c5..312b2a11abb6 100644
--- a/sound/soc/codecs/ab8500-codec.c
+++ b/sound/soc/codecs/ab8500-codec.c
@@ -2587,8 +2587,6 @@ static struct platform_driver ab8500_codec_platform_driver = {
 	},
 	.probe		= ab8500_codec_driver_probe,
 	.remove		= ab8500_codec_driver_remove,
-	.suspend	= NULL,
-	.resume		= NULL,
 };
 module_platform_driver(ab8500_codec_platform_driver);
 
diff --git a/sound/soc/codecs/adau17x1.c b/sound/soc/codecs/adau17x1.c
index 439aa3ff1f99..b36511d965c8 100644
--- a/sound/soc/codecs/adau17x1.c
+++ b/sound/soc/codecs/adau17x1.c
@@ -160,7 +160,7 @@ static int adau17x1_dsp_mux_enum_put(struct snd_kcontrol *kcontrol,
 	struct snd_soc_dapm_context *dapm = snd_soc_codec_get_dapm(codec);
 	struct adau *adau = snd_soc_codec_get_drvdata(codec);
 	struct soc_enum *e = (struct soc_enum *)kcontrol->private_value;
-	struct snd_soc_dapm_update update;
+	struct snd_soc_dapm_update update = { 0 };
 	unsigned int stream = e->shift_l;
 	unsigned int val, change;
 	int reg;
diff --git a/sound/soc/codecs/ak4641.c b/sound/soc/codecs/ak4641.c
index c91717d08513..ebdaf56c1d61 100644
--- a/sound/soc/codecs/ak4641.c
+++ b/sound/soc/codecs/ak4641.c
@@ -27,7 +27,27 @@
 #include <sound/tlv.h>
 #include <sound/ak4641.h>
 
-#include "ak4641.h"
+/* AK4641 register space */
+#define AK4641_PM1		0x00
+#define AK4641_PM2		0x01
+#define AK4641_SIG1		0x02
+#define AK4641_SIG2		0x03
+#define AK4641_MODE1		0x04
+#define AK4641_MODE2		0x05
+#define AK4641_DAC		0x06
+#define AK4641_MIC		0x07
+#define AK4641_TIMER		0x08
+#define AK4641_ALC1		0x09
+#define AK4641_ALC2		0x0a
+#define AK4641_PGA		0x0b
+#define AK4641_LATT		0x0c
+#define AK4641_RATT		0x0d
+#define AK4641_VOL		0x0e
+#define AK4641_STATUS		0x0f
+#define AK4641_EQLO		0x10
+#define AK4641_EQMID		0x11
+#define AK4641_EQHI		0x12
+#define AK4641_BTIF		0x13
 
 /* codec private data */
 struct ak4641_priv {
diff --git a/sound/soc/codecs/ak4641.h b/sound/soc/codecs/ak4641.h
deleted file mode 100644
index 4a263248efea..000000000000
--- a/sound/soc/codecs/ak4641.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * ak4641.h  --  AK4641 SoC Audio driver
- *
- * Copyright 2008 Harald Welte <laforge@gnufiish.org>
- *
- * Based on ak4535.h
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifndef _AK4641_H
-#define _AK4641_H
-
-/* AK4641 register space */
-
-#define AK4641_PM1		0x00
-#define AK4641_PM2		0x01
-#define AK4641_SIG1		0x02
-#define AK4641_SIG2		0x03
-#define AK4641_MODE1		0x04
-#define AK4641_MODE2		0x05
-#define AK4641_DAC		0x06
-#define AK4641_MIC		0x07
-#define AK4641_TIMER		0x08
-#define AK4641_ALC1		0x09
-#define AK4641_ALC2		0x0a
-#define AK4641_PGA		0x0b
-#define AK4641_LATT		0x0c
-#define AK4641_RATT		0x0d
-#define AK4641_VOL		0x0e
-#define AK4641_STATUS		0x0f
-#define AK4641_EQLO		0x10
-#define AK4641_EQMID		0x11
-#define AK4641_EQHI		0x12
-#define AK4641_BTIF		0x13
-
-#define AK4641_CACHEREGNUM	0x14
-
-
-
-#define AK4641_DAI_HIFI		0
-#define AK4641_DAI_VOICE	1
-
-
-#endif
diff --git a/sound/soc/codecs/arizona.c b/sound/soc/codecs/arizona.c
index 846ca079845f..0a734d910850 100644
--- a/sound/soc/codecs/arizona.c
+++ b/sound/soc/codecs/arizona.c
@@ -191,6 +191,14 @@ int arizona_init_spk(struct snd_soc_codec *codec)
 		break;
 	}
 
+	return 0;
+}
+EXPORT_SYMBOL_GPL(arizona_init_spk);
+
+int arizona_init_spk_irqs(struct arizona *arizona)
+{
+	int ret;
+
 	ret = arizona_request_irq(arizona, ARIZONA_IRQ_SPK_OVERHEAT_WARN,
 				  "Thermal warning", arizona_thermal_warn,
 				  arizona);
@@ -209,19 +217,16 @@ int arizona_init_spk(struct snd_soc_codec *codec)
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(arizona_init_spk);
+EXPORT_SYMBOL_GPL(arizona_init_spk_irqs);
 
-int arizona_free_spk(struct snd_soc_codec *codec)
+int arizona_free_spk_irqs(struct arizona *arizona)
 {
-	struct arizona_priv *priv = snd_soc_codec_get_drvdata(codec);
-	struct arizona *arizona = priv->arizona;
-
 	arizona_free_irq(arizona, ARIZONA_IRQ_SPK_OVERHEAT_WARN, arizona);
 	arizona_free_irq(arizona, ARIZONA_IRQ_SPK_OVERHEAT, arizona);
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(arizona_free_spk);
+EXPORT_SYMBOL_GPL(arizona_free_spk_irqs);
 
 static const struct snd_soc_dapm_route arizona_mono_routes[] = {
 	{ "OUT1R", NULL, "OUT1L" },
@@ -252,6 +257,7 @@ EXPORT_SYMBOL_GPL(arizona_init_mono);
 int arizona_init_gpio(struct snd_soc_codec *codec)
 {
 	struct snd_soc_dapm_context *dapm = snd_soc_codec_get_dapm(codec);
+	struct snd_soc_component *component = snd_soc_dapm_to_component(dapm);
 	struct arizona_priv *priv = snd_soc_codec_get_drvdata(codec);
 	struct arizona *arizona = priv->arizona;
 	int i;
@@ -259,21 +265,24 @@ int arizona_init_gpio(struct snd_soc_codec *codec)
 	switch (arizona->type) {
 	case WM5110:
 	case WM8280:
-		snd_soc_dapm_disable_pin(dapm, "DRC2 Signal Activity");
+		snd_soc_component_disable_pin(component,
+					      "DRC2 Signal Activity");
 		break;
 	default:
 		break;
 	}
 
-	snd_soc_dapm_disable_pin(dapm, "DRC1 Signal Activity");
+	snd_soc_component_disable_pin(component, "DRC1 Signal Activity");
 
 	for (i = 0; i < ARRAY_SIZE(arizona->pdata.gpio_defaults); i++) {
 		switch (arizona->pdata.gpio_defaults[i] & ARIZONA_GPN_FN_MASK) {
 		case ARIZONA_GP_FN_DRC1_SIGNAL_DETECT:
-			snd_soc_dapm_enable_pin(dapm, "DRC1 Signal Activity");
+			snd_soc_component_enable_pin(component,
+						     "DRC1 Signal Activity");
 			break;
 		case ARIZONA_GP_FN_DRC2_SIGNAL_DETECT:
-			snd_soc_dapm_enable_pin(dapm, "DRC2 Signal Activity");
+			snd_soc_component_enable_pin(component,
+						     "DRC2 Signal Activity");
 			break;
 		default:
 			break;
@@ -1233,6 +1242,46 @@ static int arizona_set_opclk(struct snd_soc_codec *codec, unsigned int clk,
 	return -EINVAL;
 }
 
+int arizona_clk_ev(struct snd_soc_dapm_widget *w,
+		   struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	struct arizona *arizona = dev_get_drvdata(codec->dev->parent);
+	unsigned int val;
+	int clk_idx;
+	int ret;
+
+	ret = regmap_read(arizona->regmap, w->reg, &val);
+	if (ret) {
+		dev_err(codec->dev, "Failed to check clock source: %d\n", ret);
+		return ret;
+	}
+
+	val = (val & ARIZONA_SYSCLK_SRC_MASK) >> ARIZONA_SYSCLK_SRC_SHIFT;
+
+	switch (val) {
+	case ARIZONA_CLK_SRC_MCLK1:
+		clk_idx = ARIZONA_MCLK1;
+		break;
+	case ARIZONA_CLK_SRC_MCLK2:
+		clk_idx = ARIZONA_MCLK2;
+		break;
+	default:
+		return 0;
+	}
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		return clk_prepare_enable(arizona->mclk[clk_idx]);
+	case SND_SOC_DAPM_POST_PMD:
+		clk_disable_unprepare(arizona->mclk[clk_idx]);
+		return 0;
+	default:
+		return 0;
+	}
+}
+EXPORT_SYMBOL_GPL(arizona_clk_ev);
+
 int arizona_set_sysclk(struct snd_soc_codec *codec, int clk_id,
 		       int source, unsigned int freq, int dir)
 {
@@ -2242,6 +2291,42 @@ static int arizona_is_enabled_fll(struct arizona_fll *fll, int base)
 	return reg & ARIZONA_FLL1_ENA;
 }
 
+static int arizona_set_fll_clks(struct arizona_fll *fll, int base, bool ena)
+{
+	struct arizona *arizona = fll->arizona;
+	unsigned int val;
+	struct clk *clk;
+	int ret;
+
+	ret = regmap_read(arizona->regmap, base + 6, &val);
+	if (ret != 0) {
+		arizona_fll_err(fll, "Failed to read current source: %d\n",
+				ret);
+		return ret;
+	}
+
+	val &= ARIZONA_FLL1_CLK_REF_SRC_MASK;
+	val >>= ARIZONA_FLL1_CLK_REF_SRC_SHIFT;
+
+	switch (val) {
+	case ARIZONA_FLL_SRC_MCLK1:
+		clk = arizona->mclk[ARIZONA_MCLK1];
+		break;
+	case ARIZONA_FLL_SRC_MCLK2:
+		clk = arizona->mclk[ARIZONA_MCLK2];
+		break;
+	default:
+		return 0;
+	}
+
+	if (ena) {
+		return clk_prepare_enable(clk);
+	} else {
+		clk_disable_unprepare(clk);
+		return 0;
+	}
+}
+
 static int arizona_enable_fll(struct arizona_fll *fll)
 {
 	struct arizona *arizona = fll->arizona;
@@ -2264,6 +2349,10 @@ static int arizona_enable_fll(struct arizona_fll *fll)
 		udelay(32);
 		regmap_update_bits_async(fll->arizona->regmap, fll->base + 0x9,
 					 ARIZONA_FLL1_GAIN_MASK, 0);
+
+		if (arizona_is_enabled_fll(fll, fll->base + 0x10) > 0)
+			arizona_set_fll_clks(fll, fll->base + 0x10, false);
+		arizona_set_fll_clks(fll, fll->base, false);
 	}
 
 	/*
@@ -2318,10 +2407,13 @@ static int arizona_enable_fll(struct arizona_fll *fll)
 	if (!already_enabled)
 		pm_runtime_get_sync(arizona->dev);
 
-	if (use_sync)
+	if (use_sync) {
+		arizona_set_fll_clks(fll, fll->base + 0x10, true);
 		regmap_update_bits_async(arizona->regmap, fll->base + 0x11,
 					 ARIZONA_FLL1_SYNC_ENA,
 					 ARIZONA_FLL1_SYNC_ENA);
+	}
+	arizona_set_fll_clks(fll, fll->base, true);
 	regmap_update_bits_async(arizona->regmap, fll->base + 1,
 				 ARIZONA_FLL1_ENA, ARIZONA_FLL1_ENA);
 
@@ -2354,19 +2446,24 @@ static int arizona_enable_fll(struct arizona_fll *fll)
 static void arizona_disable_fll(struct arizona_fll *fll)
 {
 	struct arizona *arizona = fll->arizona;
-	bool change;
+	bool ref_change, sync_change;
 
 	regmap_update_bits_async(arizona->regmap, fll->base + 1,
 				 ARIZONA_FLL1_FREERUN, ARIZONA_FLL1_FREERUN);
 	regmap_update_bits_check(arizona->regmap, fll->base + 1,
-				 ARIZONA_FLL1_ENA, 0, &change);
-	regmap_update_bits(arizona->regmap, fll->base + 0x11,
-			   ARIZONA_FLL1_SYNC_ENA, 0);
+				 ARIZONA_FLL1_ENA, 0, &ref_change);
+	regmap_update_bits_check(arizona->regmap, fll->base + 0x11,
+				 ARIZONA_FLL1_SYNC_ENA, 0, &sync_change);
 	regmap_update_bits_async(arizona->regmap, fll->base + 1,
 				 ARIZONA_FLL1_FREERUN, 0);
 
-	if (change)
+	if (sync_change)
+		arizona_set_fll_clks(fll, fll->base + 0x10, false);
+
+	if (ref_change) {
+		arizona_set_fll_clks(fll, fll->base, false);
 		pm_runtime_put_autosuspend(arizona->dev);
+	}
 }
 
 int arizona_set_fll_refclk(struct arizona_fll *fll, int source,
@@ -2598,30 +2695,6 @@ int arizona_lhpf_coeff_put(struct snd_kcontrol *kcontrol,
 }
 EXPORT_SYMBOL_GPL(arizona_lhpf_coeff_put);
 
-int arizona_register_notifier(struct snd_soc_codec *codec,
-			      struct notifier_block *nb,
-			      int (*notify)(struct notifier_block *nb,
-					    unsigned long action, void *data))
-{
-	struct arizona_priv *priv = snd_soc_codec_get_drvdata(codec);
-	struct arizona *arizona = priv->arizona;
-
-	nb->notifier_call = notify;
-
-	return blocking_notifier_chain_register(&arizona->notifier, nb);
-}
-EXPORT_SYMBOL_GPL(arizona_register_notifier);
-
-int arizona_unregister_notifier(struct snd_soc_codec *codec,
-				struct notifier_block *nb)
-{
-	struct arizona_priv *priv = snd_soc_codec_get_drvdata(codec);
-	struct arizona *arizona = priv->arizona;
-
-	return blocking_notifier_chain_unregister(&arizona->notifier, nb);
-}
-EXPORT_SYMBOL_GPL(arizona_unregister_notifier);
-
 MODULE_DESCRIPTION("ASoC Wolfson Arizona class device support");
 MODULE_AUTHOR("Mark Brown <broonie@opensource.wolfsonmicro.com>");
 MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/arizona.h b/sound/soc/codecs/arizona.h
index 850aa338ba29..56707860657c 100644
--- a/sound/soc/codecs/arizona.h
+++ b/sound/soc/codecs/arizona.h
@@ -14,6 +14,8 @@
 #define _ASOC_ARIZONA_H
 
 #include <linux/completion.h>
+#include <linux/notifier.h>
+#include <linux/mfd/arizona/core.h>
 
 #include <sound/soc.h>
 
@@ -66,7 +68,6 @@
 /* Notifier events */
 #define ARIZONA_NOTIFY_VOICE_TRIGGER   0x1
 
-struct arizona;
 struct wm_adsp;
 
 struct arizona_dai_priv {
@@ -255,26 +256,24 @@ extern const struct soc_enum arizona_output_anc_src[];
 
 extern const struct snd_kcontrol_new arizona_voice_trigger_switch[];
 
-extern int arizona_in_ev(struct snd_soc_dapm_widget *w,
-			 struct snd_kcontrol *kcontrol,
-			 int event);
-extern int arizona_out_ev(struct snd_soc_dapm_widget *w,
-			  struct snd_kcontrol *kcontrol,
-			  int event);
-extern int arizona_hp_ev(struct snd_soc_dapm_widget *w,
-			 struct snd_kcontrol *kcontrol,
-			 int event);
-extern int arizona_anc_ev(struct snd_soc_dapm_widget *w,
-			  struct snd_kcontrol *kcontrol,
-			  int event);
-
-extern int arizona_eq_coeff_put(struct snd_kcontrol *kcontrol,
-				struct snd_ctl_elem_value *ucontrol);
-extern int arizona_lhpf_coeff_put(struct snd_kcontrol *kcontrol,
-				  struct snd_ctl_elem_value *ucontrol);
-
-extern int arizona_set_sysclk(struct snd_soc_codec *codec, int clk_id,
-			      int source, unsigned int freq, int dir);
+int arizona_in_ev(struct snd_soc_dapm_widget *w, struct snd_kcontrol *kcontrol,
+		  int event);
+int arizona_out_ev(struct snd_soc_dapm_widget *w, struct snd_kcontrol *kcontrol,
+		   int event);
+int arizona_hp_ev(struct snd_soc_dapm_widget *w, struct snd_kcontrol *kcontrol,
+		  int event);
+int arizona_anc_ev(struct snd_soc_dapm_widget *w, struct snd_kcontrol *kcontrol,
+		   int event);
+
+int arizona_eq_coeff_put(struct snd_kcontrol *kcontrol,
+			 struct snd_ctl_elem_value *ucontrol);
+int arizona_lhpf_coeff_put(struct snd_kcontrol *kcontrol,
+			   struct snd_ctl_elem_value *ucontrol);
+
+int arizona_clk_ev(struct snd_soc_dapm_widget *w, struct snd_kcontrol *kcontrol,
+		   int event);
+int arizona_set_sysclk(struct snd_soc_codec *codec, int clk_id, int source,
+		       unsigned int freq, int dir);
 
 extern const struct snd_soc_dai_ops arizona_dai_ops;
 extern const struct snd_soc_dai_ops arizona_simple_dai_ops;
@@ -297,41 +296,57 @@ struct arizona_fll {
 	char clock_ok_name[ARIZONA_FLL_NAME_LEN];
 };
 
-extern int arizona_dvfs_up(struct snd_soc_codec *codec, unsigned int flags);
-extern int arizona_dvfs_down(struct snd_soc_codec *codec, unsigned int flags);
-extern int arizona_dvfs_sysclk_ev(struct snd_soc_dapm_widget *w,
-				  struct snd_kcontrol *kcontrol, int event);
-extern void arizona_init_dvfs(struct arizona_priv *priv);
-
-extern int arizona_init_fll(struct arizona *arizona, int id, int base,
-			    int lock_irq, int ok_irq, struct arizona_fll *fll);
-extern int arizona_set_fll_refclk(struct arizona_fll *fll, int source,
-				  unsigned int Fref, unsigned int Fout);
-extern int arizona_set_fll(struct arizona_fll *fll, int source,
+int arizona_dvfs_up(struct snd_soc_codec *codec, unsigned int flags);
+int arizona_dvfs_down(struct snd_soc_codec *codec, unsigned int flags);
+int arizona_dvfs_sysclk_ev(struct snd_soc_dapm_widget *w,
+			   struct snd_kcontrol *kcontrol, int event);
+void arizona_init_dvfs(struct arizona_priv *priv);
+
+int arizona_init_fll(struct arizona *arizona, int id, int base,
+		     int lock_irq, int ok_irq, struct arizona_fll *fll);
+int arizona_set_fll_refclk(struct arizona_fll *fll, int source,
 			   unsigned int Fref, unsigned int Fout);
+int arizona_set_fll(struct arizona_fll *fll, int source,
+		    unsigned int Fref, unsigned int Fout);
 
-extern int arizona_init_spk(struct snd_soc_codec *codec);
-extern int arizona_init_gpio(struct snd_soc_codec *codec);
-extern int arizona_init_mono(struct snd_soc_codec *codec);
-extern int arizona_init_notifiers(struct snd_soc_codec *codec);
+int arizona_init_spk(struct snd_soc_codec *codec);
+int arizona_init_gpio(struct snd_soc_codec *codec);
+int arizona_init_mono(struct snd_soc_codec *codec);
+int arizona_init_notifiers(struct snd_soc_codec *codec);
 
-extern int arizona_free_spk(struct snd_soc_codec *codec);
+int arizona_init_spk_irqs(struct arizona *arizona);
+int arizona_free_spk_irqs(struct arizona *arizona);
 
-extern int arizona_init_dai(struct arizona_priv *priv, int dai);
+int arizona_init_dai(struct arizona_priv *priv, int dai);
 
 int arizona_set_output_mode(struct snd_soc_codec *codec, int output,
 			    bool diff);
 
-extern bool arizona_input_analog(struct snd_soc_codec *codec, int shift);
+bool arizona_input_analog(struct snd_soc_codec *codec, int shift);
+
+const char *arizona_sample_rate_val_to_name(unsigned int rate_val);
+
+static inline int arizona_register_notifier(struct snd_soc_codec *codec,
+					    struct notifier_block *nb,
+					    int (*notify)
+					    (struct notifier_block *nb,
+					    unsigned long action, void *data))
+{
+	struct arizona_priv *priv = snd_soc_codec_get_drvdata(codec);
+	struct arizona *arizona = priv->arizona;
+
+	nb->notifier_call = notify;
+
+	return blocking_notifier_chain_register(&arizona->notifier, nb);
+}
 
-extern const char *arizona_sample_rate_val_to_name(unsigned int rate_val);
+static inline int arizona_unregister_notifier(struct snd_soc_codec *codec,
+					      struct notifier_block *nb)
+{
+	struct arizona_priv *priv = snd_soc_codec_get_drvdata(codec);
+	struct arizona *arizona = priv->arizona;
 
-extern int arizona_register_notifier(struct snd_soc_codec *codec,
-				     struct notifier_block *nb,
-				     int (*notify)(struct notifier_block *nb,
-						   unsigned long action,
-						   void *data));
-extern int arizona_unregister_notifier(struct snd_soc_codec *codec,
-				       struct notifier_block *nb);
+	return blocking_notifier_chain_unregister(&arizona->notifier, nb);
+}
 
 #endif
diff --git a/sound/soc/codecs/cs35l34.c b/sound/soc/codecs/cs35l34.c
new file mode 100644
index 000000000000..7c5d1510cf2c
--- /dev/null
+++ b/sound/soc/codecs/cs35l34.c
@@ -0,0 +1,1251 @@
+/*
+ * cs35l34.c -- CS35l34 ALSA SoC audio driver
+ *
+ * Copyright 2016 Cirrus Logic, Inc.
+ *
+ * Author: Paul Handrigan <Paul.Handrigan@cirrus.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/regulator/machine.h>
+#include <linux/pm_runtime.h>
+#include <linux/of_device.h>
+#include <linux/of_gpio.h>
+#include <linux/of_irq.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/soc-dapm.h>
+#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
+#include <sound/initval.h>
+#include <sound/tlv.h>
+#include <sound/cs35l34.h>
+
+#include "cs35l34.h"
+
+#define PDN_DONE_ATTEMPTS 10
+#define CS35L34_START_DELAY 50
+
+struct  cs35l34_private {
+	struct snd_soc_codec *codec;
+	struct cs35l34_platform_data pdata;
+	struct regmap *regmap;
+	struct regulator_bulk_data core_supplies[2];
+	int num_core_supplies;
+	int mclk_int;
+	bool tdm_mode;
+	struct gpio_desc *reset_gpio;	/* Active-low reset GPIO */
+};
+
+static const struct reg_default cs35l34_reg[] = {
+	{CS35L34_PWRCTL1, 0x01},
+	{CS35L34_PWRCTL2, 0x19},
+	{CS35L34_PWRCTL3, 0x01},
+	{CS35L34_ADSP_CLK_CTL, 0x08},
+	{CS35L34_MCLK_CTL, 0x11},
+	{CS35L34_AMP_INP_DRV_CTL, 0x01},
+	{CS35L34_AMP_DIG_VOL_CTL, 0x12},
+	{CS35L34_AMP_DIG_VOL, 0x00},
+	{CS35L34_AMP_ANLG_GAIN_CTL, 0x0F},
+	{CS35L34_PROTECT_CTL, 0x06},
+	{CS35L34_AMP_KEEP_ALIVE_CTL, 0x04},
+	{CS35L34_BST_CVTR_V_CTL, 0x00},
+	{CS35L34_BST_PEAK_I, 0x10},
+	{CS35L34_BST_RAMP_CTL, 0x87},
+	{CS35L34_BST_CONV_COEF_1, 0x24},
+	{CS35L34_BST_CONV_COEF_2, 0x24},
+	{CS35L34_BST_CONV_SLOPE_COMP, 0x4E},
+	{CS35L34_BST_CONV_SW_FREQ, 0x08},
+	{CS35L34_CLASS_H_CTL, 0x0D},
+	{CS35L34_CLASS_H_HEADRM_CTL, 0x0D},
+	{CS35L34_CLASS_H_RELEASE_RATE, 0x08},
+	{CS35L34_CLASS_H_FET_DRIVE_CTL, 0x41},
+	{CS35L34_CLASS_H_STATUS, 0x05},
+	{CS35L34_VPBR_CTL, 0x0A},
+	{CS35L34_VPBR_VOL_CTL, 0x90},
+	{CS35L34_VPBR_TIMING_CTL, 0x6A},
+	{CS35L34_PRED_MAX_ATTEN_SPK_LOAD, 0x95},
+	{CS35L34_PRED_BROWNOUT_THRESH, 0x1C},
+	{CS35L34_PRED_BROWNOUT_VOL_CTL, 0x00},
+	{CS35L34_PRED_BROWNOUT_RATE_CTL, 0x10},
+	{CS35L34_PRED_WAIT_CTL, 0x10},
+	{CS35L34_PRED_ZVP_INIT_IMP_CTL, 0x08},
+	{CS35L34_PRED_MAN_SAFE_VPI_CTL, 0x80},
+	{CS35L34_VPBR_ATTEN_STATUS, 0x00},
+	{CS35L34_PRED_BRWNOUT_ATT_STATUS, 0x00},
+	{CS35L34_SPKR_MON_CTL, 0xC6},
+	{CS35L34_ADSP_I2S_CTL, 0x00},
+	{CS35L34_ADSP_TDM_CTL, 0x00},
+	{CS35L34_TDM_TX_CTL_1_VMON, 0x00},
+	{CS35L34_TDM_TX_CTL_2_IMON, 0x04},
+	{CS35L34_TDM_TX_CTL_3_VPMON, 0x03},
+	{CS35L34_TDM_TX_CTL_4_VBSTMON, 0x07},
+	{CS35L34_TDM_TX_CTL_5_FLAG1, 0x08},
+	{CS35L34_TDM_TX_CTL_6_FLAG2, 0x09},
+	{CS35L34_TDM_TX_SLOT_EN_1, 0x00},
+	{CS35L34_TDM_TX_SLOT_EN_2, 0x00},
+	{CS35L34_TDM_TX_SLOT_EN_3, 0x00},
+	{CS35L34_TDM_TX_SLOT_EN_4, 0x00},
+	{CS35L34_TDM_RX_CTL_1_AUDIN, 0x40},
+	{CS35L34_TDM_RX_CTL_3_ALIVE, 0x04},
+	{CS35L34_MULT_DEV_SYNCH1, 0x00},
+	{CS35L34_MULT_DEV_SYNCH2, 0x80},
+	{CS35L34_PROT_RELEASE_CTL, 0x00},
+	{CS35L34_DIAG_MODE_REG_LOCK, 0x00},
+	{CS35L34_DIAG_MODE_CTL_1, 0x00},
+	{CS35L34_DIAG_MODE_CTL_2, 0x00},
+	{CS35L34_INT_MASK_1, 0xFF},
+	{CS35L34_INT_MASK_2, 0xFF},
+	{CS35L34_INT_MASK_3, 0xFF},
+	{CS35L34_INT_MASK_4, 0xFF},
+	{CS35L34_INT_STATUS_1, 0x30},
+	{CS35L34_INT_STATUS_2, 0x05},
+	{CS35L34_INT_STATUS_3, 0x00},
+	{CS35L34_INT_STATUS_4, 0x00},
+	{CS35L34_OTP_TRIM_STATUS, 0x00},
+};
+
+static bool cs35l34_volatile_register(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case CS35L34_DEVID_AB:
+	case CS35L34_DEVID_CD:
+	case CS35L34_DEVID_E:
+	case CS35L34_FAB_ID:
+	case CS35L34_REV_ID:
+	case CS35L34_INT_STATUS_1:
+	case CS35L34_INT_STATUS_2:
+	case CS35L34_INT_STATUS_3:
+	case CS35L34_INT_STATUS_4:
+	case CS35L34_CLASS_H_STATUS:
+	case CS35L34_VPBR_ATTEN_STATUS:
+	case CS35L34_OTP_TRIM_STATUS:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool cs35l34_readable_register(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case	CS35L34_DEVID_AB:
+	case	CS35L34_DEVID_CD:
+	case	CS35L34_DEVID_E:
+	case	CS35L34_FAB_ID:
+	case	CS35L34_REV_ID:
+	case	CS35L34_PWRCTL1:
+	case	CS35L34_PWRCTL2:
+	case	CS35L34_PWRCTL3:
+	case	CS35L34_ADSP_CLK_CTL:
+	case	CS35L34_MCLK_CTL:
+	case	CS35L34_AMP_INP_DRV_CTL:
+	case	CS35L34_AMP_DIG_VOL_CTL:
+	case	CS35L34_AMP_DIG_VOL:
+	case	CS35L34_AMP_ANLG_GAIN_CTL:
+	case	CS35L34_PROTECT_CTL:
+	case	CS35L34_AMP_KEEP_ALIVE_CTL:
+	case	CS35L34_BST_CVTR_V_CTL:
+	case	CS35L34_BST_PEAK_I:
+	case	CS35L34_BST_RAMP_CTL:
+	case	CS35L34_BST_CONV_COEF_1:
+	case	CS35L34_BST_CONV_COEF_2:
+	case	CS35L34_BST_CONV_SLOPE_COMP:
+	case	CS35L34_BST_CONV_SW_FREQ:
+	case	CS35L34_CLASS_H_CTL:
+	case	CS35L34_CLASS_H_HEADRM_CTL:
+	case	CS35L34_CLASS_H_RELEASE_RATE:
+	case	CS35L34_CLASS_H_FET_DRIVE_CTL:
+	case	CS35L34_CLASS_H_STATUS:
+	case	CS35L34_VPBR_CTL:
+	case	CS35L34_VPBR_VOL_CTL:
+	case	CS35L34_VPBR_TIMING_CTL:
+	case	CS35L34_PRED_MAX_ATTEN_SPK_LOAD:
+	case	CS35L34_PRED_BROWNOUT_THRESH:
+	case	CS35L34_PRED_BROWNOUT_VOL_CTL:
+	case	CS35L34_PRED_BROWNOUT_RATE_CTL:
+	case	CS35L34_PRED_WAIT_CTL:
+	case	CS35L34_PRED_ZVP_INIT_IMP_CTL:
+	case	CS35L34_PRED_MAN_SAFE_VPI_CTL:
+	case	CS35L34_VPBR_ATTEN_STATUS:
+	case	CS35L34_PRED_BRWNOUT_ATT_STATUS:
+	case	CS35L34_SPKR_MON_CTL:
+	case	CS35L34_ADSP_I2S_CTL:
+	case	CS35L34_ADSP_TDM_CTL:
+	case	CS35L34_TDM_TX_CTL_1_VMON:
+	case	CS35L34_TDM_TX_CTL_2_IMON:
+	case	CS35L34_TDM_TX_CTL_3_VPMON:
+	case	CS35L34_TDM_TX_CTL_4_VBSTMON:
+	case	CS35L34_TDM_TX_CTL_5_FLAG1:
+	case	CS35L34_TDM_TX_CTL_6_FLAG2:
+	case	CS35L34_TDM_TX_SLOT_EN_1:
+	case	CS35L34_TDM_TX_SLOT_EN_2:
+	case	CS35L34_TDM_TX_SLOT_EN_3:
+	case	CS35L34_TDM_TX_SLOT_EN_4:
+	case	CS35L34_TDM_RX_CTL_1_AUDIN:
+	case	CS35L34_TDM_RX_CTL_3_ALIVE:
+	case	CS35L34_MULT_DEV_SYNCH1:
+	case	CS35L34_MULT_DEV_SYNCH2:
+	case	CS35L34_PROT_RELEASE_CTL:
+	case	CS35L34_DIAG_MODE_REG_LOCK:
+	case	CS35L34_DIAG_MODE_CTL_1:
+	case	CS35L34_DIAG_MODE_CTL_2:
+	case	CS35L34_INT_MASK_1:
+	case	CS35L34_INT_MASK_2:
+	case	CS35L34_INT_MASK_3:
+	case	CS35L34_INT_MASK_4:
+	case	CS35L34_INT_STATUS_1:
+	case	CS35L34_INT_STATUS_2:
+	case	CS35L34_INT_STATUS_3:
+	case	CS35L34_INT_STATUS_4:
+	case	CS35L34_OTP_TRIM_STATUS:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool cs35l34_precious_register(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case CS35L34_INT_STATUS_1:
+	case CS35L34_INT_STATUS_2:
+	case CS35L34_INT_STATUS_3:
+	case CS35L34_INT_STATUS_4:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static int cs35l34_sdin_event(struct snd_soc_dapm_widget *w,
+		struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	struct cs35l34_private *priv = snd_soc_codec_get_drvdata(codec);
+	int ret;
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		if (priv->tdm_mode)
+			regmap_update_bits(priv->regmap, CS35L34_PWRCTL3,
+						CS35L34_PDN_TDM, 0x00);
+
+		ret = regmap_update_bits(priv->regmap, CS35L34_PWRCTL1,
+						CS35L34_PDN_ALL, 0);
+		if (ret < 0) {
+			dev_err(codec->dev, "Cannot set Power bits %d\n", ret);
+			return ret;
+		}
+		usleep_range(5000, 5100);
+	break;
+	case SND_SOC_DAPM_POST_PMD:
+		if (priv->tdm_mode) {
+			regmap_update_bits(priv->regmap, CS35L34_PWRCTL3,
+					CS35L34_PDN_TDM, CS35L34_PDN_TDM);
+		}
+		ret = regmap_update_bits(priv->regmap, CS35L34_PWRCTL1,
+					CS35L34_PDN_ALL, CS35L34_PDN_ALL);
+	break;
+	default:
+		pr_err("Invalid event = 0x%x\n", event);
+	}
+	return 0;
+}
+
+static int cs35l34_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
+				unsigned int rx_mask, int slots, int slot_width)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	struct cs35l34_private *priv = snd_soc_codec_get_drvdata(codec);
+	unsigned int reg, bit_pos;
+	int slot, slot_num;
+
+	if (slot_width != 8)
+		return -EINVAL;
+
+	priv->tdm_mode = true;
+	/* scan rx_mask for aud slot */
+	slot = ffs(rx_mask) - 1;
+	if (slot >= 0)
+		snd_soc_update_bits(codec, CS35L34_TDM_RX_CTL_1_AUDIN,
+					CS35L34_X_LOC, slot);
+
+	/* scan tx_mask: vmon(2 slots); imon (2 slots); vpmon (1 slot)
+	 * vbstmon (1 slot)
+	 */
+	slot = ffs(tx_mask) - 1;
+	slot_num = 0;
+
+	/* disable vpmon/vbstmon: enable later if set in tx_mask */
+	snd_soc_update_bits(codec, CS35L34_TDM_TX_CTL_3_VPMON,
+				CS35L34_X_STATE | CS35L34_X_LOC,
+				CS35L34_X_STATE | CS35L34_X_LOC);
+	snd_soc_update_bits(codec, CS35L34_TDM_TX_CTL_4_VBSTMON,
+				CS35L34_X_STATE | CS35L34_X_LOC,
+				CS35L34_X_STATE | CS35L34_X_LOC);
+
+	/* disconnect {vp,vbst}_mon routes: eanble later if set in tx_mask*/
+	while (slot >= 0) {
+		/* configure VMON_TX_LOC */
+		if (slot_num == 0)
+			snd_soc_update_bits(codec, CS35L34_TDM_TX_CTL_1_VMON,
+					CS35L34_X_STATE | CS35L34_X_LOC, slot);
+
+		/* configure IMON_TX_LOC */
+		if (slot_num == 4) {
+			snd_soc_update_bits(codec, CS35L34_TDM_TX_CTL_2_IMON,
+					CS35L34_X_STATE | CS35L34_X_LOC, slot);
+		}
+		/* configure VPMON_TX_LOC */
+		if (slot_num == 3) {
+			snd_soc_update_bits(codec, CS35L34_TDM_TX_CTL_3_VPMON,
+					CS35L34_X_STATE | CS35L34_X_LOC, slot);
+		}
+		/* configure VBSTMON_TX_LOC */
+		if (slot_num == 7) {
+			snd_soc_update_bits(codec,
+				CS35L34_TDM_TX_CTL_4_VBSTMON,
+				CS35L34_X_STATE | CS35L34_X_LOC, slot);
+		}
+
+		/* Enable the relevant tx slot */
+		reg = CS35L34_TDM_TX_SLOT_EN_4 - (slot/8);
+		bit_pos = slot - ((slot / 8) * (8));
+		snd_soc_update_bits(codec, reg,
+			1 << bit_pos, 1 << bit_pos);
+
+		tx_mask &= ~(1 << slot);
+		slot = ffs(tx_mask) - 1;
+		slot_num++;
+	}
+
+	return 0;
+}
+
+static int cs35l34_main_amp_event(struct snd_soc_dapm_widget *w,
+		struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	struct cs35l34_private *priv = snd_soc_codec_get_drvdata(codec);
+
+	switch (event) {
+	case SND_SOC_DAPM_POST_PMU:
+		regmap_update_bits(priv->regmap, CS35L34_BST_CVTR_V_CTL,
+				CS35L34_BST_CVTL_MASK, priv->pdata.boost_vtge);
+		usleep_range(5000, 5100);
+		regmap_update_bits(priv->regmap, CS35L34_PROTECT_CTL,
+						CS35L34_MUTE, 0);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		regmap_update_bits(priv->regmap, CS35L34_BST_CVTR_V_CTL,
+			CS35L34_BST_CVTL_MASK, 0);
+		regmap_update_bits(priv->regmap, CS35L34_PROTECT_CTL,
+			CS35L34_MUTE, CS35L34_MUTE);
+		usleep_range(5000, 5100);
+		break;
+	default:
+		pr_err("Invalid event = 0x%x\n", event);
+	}
+	return 0;
+}
+
+static DECLARE_TLV_DB_SCALE(dig_vol_tlv, -10200, 50, 0);
+
+static DECLARE_TLV_DB_SCALE(amp_gain_tlv, 300, 100, 0);
+
+
+static const struct snd_kcontrol_new cs35l34_snd_controls[] = {
+	SOC_SINGLE_SX_TLV("Digital Volume", CS35L34_AMP_DIG_VOL,
+		      0, 0x34, 0xE4, dig_vol_tlv),
+	SOC_SINGLE_TLV("Amp Gain Volume", CS35L34_AMP_ANLG_GAIN_CTL,
+		      0, 0xF, 0, amp_gain_tlv),
+};
+
+
+static int cs35l34_mclk_event(struct snd_soc_dapm_widget *w,
+		struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	struct cs35l34_private *priv = snd_soc_codec_get_drvdata(codec);
+	int ret, i;
+	unsigned int reg;
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMD:
+		ret = regmap_read(priv->regmap, CS35L34_AMP_DIG_VOL_CTL,
+			&reg);
+		if (ret != 0) {
+			pr_err("%s regmap read failure %d\n", __func__, ret);
+			return ret;
+		}
+		if (reg & CS35L34_AMP_DIGSFT)
+			msleep(40);
+		else
+			usleep_range(2000, 2100);
+
+		for (i = 0; i < PDN_DONE_ATTEMPTS; i++) {
+			ret = regmap_read(priv->regmap, CS35L34_INT_STATUS_2,
+				&reg);
+			if (ret != 0) {
+				pr_err("%s regmap read failure %d\n",
+					__func__, ret);
+				return ret;
+			}
+			if (reg & CS35L34_PDN_DONE)
+				break;
+
+			usleep_range(5000, 5100);
+		}
+		if (i == PDN_DONE_ATTEMPTS)
+			pr_err("%s Device did not power down properly\n",
+				__func__);
+		break;
+	default:
+		pr_err("Invalid event = 0x%x\n", event);
+		break;
+	}
+	return 0;
+}
+
+static const struct snd_soc_dapm_widget cs35l34_dapm_widgets[] = {
+	SND_SOC_DAPM_AIF_IN_E("SDIN", NULL, 0, CS35L34_PWRCTL3,
+					1, 1, cs35l34_sdin_event,
+					SND_SOC_DAPM_PRE_PMU |
+					SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_AIF_OUT("SDOUT", NULL, 0, CS35L34_PWRCTL3, 2, 1),
+
+	SND_SOC_DAPM_SUPPLY("EXTCLK", CS35L34_PWRCTL3, 7, 1,
+		cs35l34_mclk_event, SND_SOC_DAPM_PRE_PMD),
+
+	SND_SOC_DAPM_OUTPUT("SPK"),
+
+	SND_SOC_DAPM_INPUT("VP"),
+	SND_SOC_DAPM_INPUT("VPST"),
+	SND_SOC_DAPM_INPUT("ISENSE"),
+	SND_SOC_DAPM_INPUT("VSENSE"),
+
+	SND_SOC_DAPM_ADC("VMON ADC", NULL, CS35L34_PWRCTL2, 7, 1),
+	SND_SOC_DAPM_ADC("IMON ADC", NULL, CS35L34_PWRCTL2, 6, 1),
+	SND_SOC_DAPM_ADC("VPMON ADC", NULL, CS35L34_PWRCTL3, 3, 1),
+	SND_SOC_DAPM_ADC("VBSTMON ADC", NULL, CS35L34_PWRCTL3, 4, 1),
+	SND_SOC_DAPM_ADC("CLASS H", NULL, CS35L34_PWRCTL2, 5, 1),
+	SND_SOC_DAPM_ADC("BOOST", NULL, CS35L34_PWRCTL2, 2, 1),
+
+	SND_SOC_DAPM_OUT_DRV_E("Main AMP", CS35L34_PWRCTL2, 0, 1, NULL, 0,
+		cs35l34_main_amp_event, SND_SOC_DAPM_POST_PMU |
+			SND_SOC_DAPM_POST_PMD),
+};
+
+static const struct snd_soc_dapm_route cs35l34_audio_map[] = {
+	{"SDIN", NULL, "AMP Playback"},
+	{"BOOST", NULL, "SDIN"},
+	{"CLASS H", NULL, "BOOST"},
+	{"Main AMP", NULL, "CLASS H"},
+	{"SPK", NULL, "Main AMP"},
+
+	{"VPMON ADC", NULL, "CLASS H"},
+	{"VBSTMON ADC", NULL, "CLASS H"},
+	{"SPK", NULL, "VPMON ADC"},
+	{"SPK", NULL, "VBSTMON ADC"},
+
+	{"IMON ADC", NULL, "ISENSE"},
+	{"VMON ADC", NULL, "VSENSE"},
+	{"SDOUT", NULL, "IMON ADC"},
+	{"SDOUT", NULL, "VMON ADC"},
+	{"AMP Capture", NULL, "SDOUT"},
+
+	{"SDIN", NULL, "EXTCLK"},
+	{"SDOUT", NULL, "EXTCLK"},
+};
+
+struct cs35l34_mclk_div {
+	int mclk;
+	int srate;
+	u8 adsp_rate;
+};
+
+static struct cs35l34_mclk_div cs35l34_mclk_coeffs[] = {
+
+	/* MCLK, Sample Rate, adsp_rate */
+
+	{5644800, 11025, 0x1},
+	{5644800, 22050, 0x4},
+	{5644800, 44100, 0x7},
+
+	{6000000,  8000, 0x0},
+	{6000000, 11025, 0x1},
+	{6000000, 12000, 0x2},
+	{6000000, 16000, 0x3},
+	{6000000, 22050, 0x4},
+	{6000000, 24000, 0x5},
+	{6000000, 32000, 0x6},
+	{6000000, 44100, 0x7},
+	{6000000, 48000, 0x8},
+
+	{6144000,  8000, 0x0},
+	{6144000, 11025, 0x1},
+	{6144000, 12000, 0x2},
+	{6144000, 16000, 0x3},
+	{6144000, 22050, 0x4},
+	{6144000, 24000, 0x5},
+	{6144000, 32000, 0x6},
+	{6144000, 44100, 0x7},
+	{6144000, 48000, 0x8},
+};
+
+static int cs35l34_get_mclk_coeff(int mclk, int srate)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(cs35l34_mclk_coeffs); i++) {
+		if (cs35l34_mclk_coeffs[i].mclk == mclk &&
+			cs35l34_mclk_coeffs[i].srate == srate)
+			return i;
+	}
+	return -EINVAL;
+}
+
+static int cs35l34_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
+{
+	struct snd_soc_codec *codec = codec_dai->codec;
+	struct cs35l34_private *priv = snd_soc_codec_get_drvdata(codec);
+
+	switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) {
+	case SND_SOC_DAIFMT_CBM_CFM:
+		regmap_update_bits(priv->regmap, CS35L34_ADSP_CLK_CTL,
+				    0x80, 0x80);
+		break;
+	case SND_SOC_DAIFMT_CBS_CFS:
+		regmap_update_bits(priv->regmap, CS35L34_ADSP_CLK_CTL,
+				    0x80, 0x00);
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int cs35l34_pcm_hw_params(struct snd_pcm_substream *substream,
+				 struct snd_pcm_hw_params *params,
+				 struct snd_soc_dai *dai)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	struct cs35l34_private *priv = snd_soc_codec_get_drvdata(codec);
+	int srate = params_rate(params);
+	int ret;
+
+	int coeff = cs35l34_get_mclk_coeff(priv->mclk_int, srate);
+
+	if (coeff < 0) {
+		dev_err(codec->dev, "ERROR: Invalid mclk %d and/or srate %d\n",
+			priv->mclk_int, srate);
+		return coeff;
+	}
+
+	ret = regmap_update_bits(priv->regmap, CS35L34_ADSP_CLK_CTL,
+		CS35L34_ADSP_RATE, cs35l34_mclk_coeffs[coeff].adsp_rate);
+	if (ret != 0)
+		dev_err(codec->dev, "Failed to set clock state %d\n", ret);
+
+	return ret;
+}
+
+static unsigned int cs35l34_src_rates[] = {
+	8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000
+};
+
+
+static struct snd_pcm_hw_constraint_list cs35l34_constraints = {
+	.count  = ARRAY_SIZE(cs35l34_src_rates),
+	.list   = cs35l34_src_rates,
+};
+
+static int cs35l34_pcm_startup(struct snd_pcm_substream *substream,
+			       struct snd_soc_dai *dai)
+{
+
+	snd_pcm_hw_constraint_list(substream->runtime, 0,
+				SNDRV_PCM_HW_PARAM_RATE, &cs35l34_constraints);
+	return 0;
+}
+
+
+static int cs35l34_set_tristate(struct snd_soc_dai *dai, int tristate)
+{
+
+	struct snd_soc_codec *codec = dai->codec;
+
+	if (tristate)
+		snd_soc_update_bits(codec, CS35L34_PWRCTL3,
+					CS35L34_PDN_SDOUT, CS35L34_PDN_SDOUT);
+	else
+		snd_soc_update_bits(codec, CS35L34_PWRCTL3,
+					CS35L34_PDN_SDOUT, 0);
+	return 0;
+}
+
+static int cs35l34_dai_set_sysclk(struct snd_soc_dai *dai,
+				int clk_id, unsigned int freq, int dir)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	struct cs35l34_private *cs35l34 = snd_soc_codec_get_drvdata(codec);
+	unsigned int value;
+
+	switch (freq) {
+	case CS35L34_MCLK_5644:
+		value = CS35L34_MCLK_RATE_5P6448;
+		cs35l34->mclk_int = freq;
+	break;
+	case CS35L34_MCLK_6:
+		value = CS35L34_MCLK_RATE_6P0000;
+		cs35l34->mclk_int = freq;
+	break;
+	case CS35L34_MCLK_6144:
+		value = CS35L34_MCLK_RATE_6P1440;
+		cs35l34->mclk_int = freq;
+	break;
+	case CS35L34_MCLK_11289:
+		value = CS35L34_MCLK_DIV | CS35L34_MCLK_RATE_5P6448;
+		cs35l34->mclk_int = freq / 2;
+	break;
+	case CS35L34_MCLK_12:
+		value = CS35L34_MCLK_DIV | CS35L34_MCLK_RATE_6P0000;
+		cs35l34->mclk_int = freq / 2;
+	break;
+	case CS35L34_MCLK_12288:
+		value = CS35L34_MCLK_DIV | CS35L34_MCLK_RATE_6P1440;
+		cs35l34->mclk_int = freq / 2;
+	break;
+	default:
+		dev_err(codec->dev, "ERROR: Invalid Frequency %d\n", freq);
+		cs35l34->mclk_int = 0;
+		return -EINVAL;
+	}
+	regmap_update_bits(cs35l34->regmap, CS35L34_MCLK_CTL,
+			CS35L34_MCLK_DIV | CS35L34_MCLK_RATE_MASK, value);
+	return 0;
+}
+
+static const struct snd_soc_dai_ops cs35l34_ops = {
+	.startup = cs35l34_pcm_startup,
+	.set_tristate = cs35l34_set_tristate,
+	.set_fmt = cs35l34_set_dai_fmt,
+	.hw_params = cs35l34_pcm_hw_params,
+	.set_sysclk = cs35l34_dai_set_sysclk,
+	.set_tdm_slot = cs35l34_set_tdm_slot,
+};
+
+static struct snd_soc_dai_driver cs35l34_dai = {
+		.name = "cs35l34",
+		.id = 0,
+		.playback = {
+			.stream_name = "AMP Playback",
+			.channels_min = 1,
+			.channels_max = 8,
+			.rates = CS35L34_RATES,
+			.formats = CS35L34_FORMATS,
+		},
+		.capture = {
+			.stream_name = "AMP Capture",
+			.channels_min = 1,
+			.channels_max = 8,
+			.rates = CS35L34_RATES,
+			.formats = CS35L34_FORMATS,
+		},
+		.ops = &cs35l34_ops,
+		.symmetric_rates = 1,
+};
+
+static int cs35l34_boost_inductor(struct cs35l34_private *cs35l34,
+	unsigned int inductor)
+{
+	struct snd_soc_codec *codec = cs35l34->codec;
+
+	switch (inductor) {
+	case 1000: /* 1 uH */
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_COEF_1, 0x24);
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_COEF_2, 0x24);
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_SLOPE_COMP,
+			0x4E);
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_SW_FREQ, 0);
+		break;
+	case 1200: /* 1.2 uH */
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_COEF_1, 0x20);
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_COEF_2, 0x20);
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_SLOPE_COMP,
+			0x47);
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_SW_FREQ, 1);
+		break;
+	case 1500: /* 1.5uH */
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_COEF_1, 0x20);
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_COEF_2, 0x20);
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_SLOPE_COMP,
+			0x3C);
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_SW_FREQ, 2);
+		break;
+	case 2200: /* 2.2uH */
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_COEF_1, 0x19);
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_COEF_2, 0x25);
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_SLOPE_COMP,
+			0x23);
+		regmap_write(cs35l34->regmap, CS35L34_BST_CONV_SW_FREQ, 3);
+		break;
+	default:
+		dev_err(codec->dev, "%s Invalid Inductor Value %d uH\n",
+			__func__, inductor);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int cs35l34_probe(struct snd_soc_codec *codec)
+{
+	int ret = 0;
+	struct cs35l34_private *cs35l34 = snd_soc_codec_get_drvdata(codec);
+
+	pm_runtime_get_sync(codec->dev);
+
+	/* Set over temperature warning attenuation to 6 dB */
+	regmap_update_bits(cs35l34->regmap, CS35L34_PROTECT_CTL,
+		 CS35L34_OTW_ATTN_MASK, 0x8);
+
+	/* Set Power control registers 2 and 3 to have everything
+	 * powered down at initialization
+	 */
+	regmap_write(cs35l34->regmap, CS35L34_PWRCTL2, 0xFD);
+	regmap_write(cs35l34->regmap, CS35L34_PWRCTL3, 0x1F);
+
+	/* Set mute bit at startup */
+	regmap_update_bits(cs35l34->regmap, CS35L34_PROTECT_CTL,
+				CS35L34_MUTE, CS35L34_MUTE);
+
+	/* Set Platform Data */
+	if (cs35l34->pdata.boost_peak)
+		regmap_update_bits(cs35l34->regmap, CS35L34_BST_PEAK_I,
+				CS35L34_BST_PEAK_MASK,
+				cs35l34->pdata.boost_peak);
+
+	if (cs35l34->pdata.gain_zc_disable)
+		regmap_update_bits(cs35l34->regmap, CS35L34_PROTECT_CTL,
+			CS35L34_GAIN_ZC_MASK, 0);
+	else
+		regmap_update_bits(cs35l34->regmap, CS35L34_PROTECT_CTL,
+			CS35L34_GAIN_ZC_MASK, CS35L34_GAIN_ZC_MASK);
+
+	if (cs35l34->pdata.aif_half_drv)
+		regmap_update_bits(cs35l34->regmap, CS35L34_ADSP_CLK_CTL,
+			CS35L34_ADSP_DRIVE, 0);
+
+	if (cs35l34->pdata.digsft_disable)
+		regmap_update_bits(cs35l34->regmap, CS35L34_AMP_DIG_VOL_CTL,
+			CS35L34_AMP_DIGSFT, 0);
+
+	if (cs35l34->pdata.amp_inv)
+		regmap_update_bits(cs35l34->regmap, CS35L34_AMP_DIG_VOL_CTL,
+			CS35L34_INV, CS35L34_INV);
+
+	if (cs35l34->pdata.boost_ind)
+		ret = cs35l34_boost_inductor(cs35l34, cs35l34->pdata.boost_ind);
+
+	if (cs35l34->pdata.i2s_sdinloc)
+		regmap_update_bits(cs35l34->regmap, CS35L34_ADSP_I2S_CTL,
+			CS35L34_I2S_LOC_MASK,
+			cs35l34->pdata.i2s_sdinloc << CS35L34_I2S_LOC_SHIFT);
+
+	if (cs35l34->pdata.tdm_rising_edge)
+		regmap_update_bits(cs35l34->regmap, CS35L34_ADSP_TDM_CTL,
+			1, 1);
+
+	pm_runtime_put_sync(codec->dev);
+
+	return ret;
+}
+
+
+static struct snd_soc_codec_driver soc_codec_dev_cs35l34 = {
+	.probe = cs35l34_probe,
+
+	.component_driver = {
+		.dapm_widgets = cs35l34_dapm_widgets,
+		.num_dapm_widgets = ARRAY_SIZE(cs35l34_dapm_widgets),
+		.dapm_routes = cs35l34_audio_map,
+		.num_dapm_routes = ARRAY_SIZE(cs35l34_audio_map),
+		.controls = cs35l34_snd_controls,
+		.num_controls = ARRAY_SIZE(cs35l34_snd_controls),
+	},
+};
+
+static struct regmap_config cs35l34_regmap = {
+	.reg_bits = 8,
+	.val_bits = 8,
+
+	.max_register = CS35L34_MAX_REGISTER,
+	.reg_defaults = cs35l34_reg,
+	.num_reg_defaults = ARRAY_SIZE(cs35l34_reg),
+	.volatile_reg = cs35l34_volatile_register,
+	.readable_reg = cs35l34_readable_register,
+	.precious_reg = cs35l34_precious_register,
+	.cache_type = REGCACHE_RBTREE,
+};
+
+static int cs35l34_handle_of_data(struct i2c_client *i2c_client,
+				struct cs35l34_platform_data *pdata)
+{
+	struct device_node *np = i2c_client->dev.of_node;
+	unsigned int val;
+
+	if (of_property_read_u32(np, "cirrus,boost-vtge-millivolt",
+		&val) >= 0) {
+		/* Boost Voltage has a maximum of 8V */
+		if (val > 8000 || (val < 3300 && val > 0)) {
+			dev_err(&i2c_client->dev,
+				"Invalid Boost Voltage %d mV\n", val);
+			return -EINVAL;
+		}
+		if (val == 0)
+			pdata->boost_vtge = 0; /* Use VP */
+		else
+			pdata->boost_vtge = ((val - 3300)/100) + 1;
+	} else {
+		dev_warn(&i2c_client->dev,
+			"Boost Voltage not specified. Using VP\n");
+	}
+
+	if (of_property_read_u32(np, "cirrus,boost-ind-nanohenry", &val) >= 0) {
+		pdata->boost_ind = val;
+	} else {
+		dev_err(&i2c_client->dev, "Inductor not specified.\n");
+		return -EINVAL;
+	}
+
+	if (of_property_read_u32(np, "cirrus,boost-peak-milliamp", &val) >= 0) {
+		if (val > 3840 || val < 1200) {
+			dev_err(&i2c_client->dev,
+				"Invalid Boost Peak Current %d mA\n", val);
+			return -EINVAL;
+		}
+		pdata->boost_peak = ((val - 1200)/80) + 1;
+	}
+
+	pdata->aif_half_drv = of_property_read_bool(np,
+		"cirrus,aif-half-drv");
+	pdata->digsft_disable = of_property_read_bool(np,
+		"cirrus,digsft-disable");
+
+	pdata->gain_zc_disable = of_property_read_bool(np,
+		"cirrus,gain-zc-disable");
+	pdata->amp_inv = of_property_read_bool(np, "cirrus,amp-inv");
+
+	if (of_property_read_u32(np, "cirrus,i2s-sdinloc", &val) >= 0)
+		pdata->i2s_sdinloc = val;
+	if (of_property_read_u32(np, "cirrus,tdm-rising-edge", &val) >= 0)
+		pdata->tdm_rising_edge = val;
+
+	return 0;
+}
+
+static irqreturn_t cs35l34_irq_thread(int irq, void *data)
+{
+	struct cs35l34_private *cs35l34 = data;
+	struct snd_soc_codec *codec = cs35l34->codec;
+	unsigned int sticky1, sticky2, sticky3, sticky4;
+	unsigned int mask1, mask2, mask3, mask4, current1;
+
+
+	/* ack the irq by reading all status registers */
+	regmap_read(cs35l34->regmap, CS35L34_INT_STATUS_4, &sticky4);
+	regmap_read(cs35l34->regmap, CS35L34_INT_STATUS_3, &sticky3);
+	regmap_read(cs35l34->regmap, CS35L34_INT_STATUS_2, &sticky2);
+	regmap_read(cs35l34->regmap, CS35L34_INT_STATUS_1, &sticky1);
+
+	regmap_read(cs35l34->regmap, CS35L34_INT_MASK_4, &mask4);
+	regmap_read(cs35l34->regmap, CS35L34_INT_MASK_3, &mask3);
+	regmap_read(cs35l34->regmap, CS35L34_INT_MASK_2, &mask2);
+	regmap_read(cs35l34->regmap, CS35L34_INT_MASK_1, &mask1);
+
+	if (!(sticky1 & ~mask1) && !(sticky2 & ~mask2) && !(sticky3 & ~mask3)
+		&& !(sticky4 & ~mask4))
+		return IRQ_NONE;
+
+	regmap_read(cs35l34->regmap, CS35L34_INT_STATUS_1, &current1);
+
+	if (sticky1 & CS35L34_CAL_ERR) {
+		dev_err(codec->dev, "Cal error\n");
+
+		/* error is no longer asserted; safe to reset */
+		if (!(current1 & CS35L34_CAL_ERR)) {
+			dev_dbg(codec->dev, "Cal error release\n");
+			regmap_update_bits(cs35l34->regmap,
+					CS35L34_PROT_RELEASE_CTL,
+					CS35L34_CAL_ERR_RLS, 0);
+			regmap_update_bits(cs35l34->regmap,
+					CS35L34_PROT_RELEASE_CTL,
+					CS35L34_CAL_ERR_RLS,
+					CS35L34_CAL_ERR_RLS);
+			regmap_update_bits(cs35l34->regmap,
+					CS35L34_PROT_RELEASE_CTL,
+					CS35L34_CAL_ERR_RLS, 0);
+			/* note: amp will re-calibrate on next resume */
+		}
+	}
+
+	if (sticky1 & CS35L34_ALIVE_ERR)
+		dev_err(codec->dev, "Alive error\n");
+
+	if (sticky1 & CS35L34_AMP_SHORT) {
+		dev_crit(codec->dev, "Amp short error\n");
+
+		/* error is no longer asserted; safe to reset */
+		if (!(current1 & CS35L34_AMP_SHORT)) {
+			dev_dbg(codec->dev,
+				"Amp short error release\n");
+			regmap_update_bits(cs35l34->regmap,
+					CS35L34_PROT_RELEASE_CTL,
+					CS35L34_SHORT_RLS, 0);
+			regmap_update_bits(cs35l34->regmap,
+					CS35L34_PROT_RELEASE_CTL,
+					CS35L34_SHORT_RLS,
+					CS35L34_SHORT_RLS);
+			regmap_update_bits(cs35l34->regmap,
+					CS35L34_PROT_RELEASE_CTL,
+					CS35L34_SHORT_RLS, 0);
+		}
+	}
+
+	if (sticky1 & CS35L34_OTW) {
+		dev_crit(codec->dev, "Over temperature warning\n");
+
+		/* error is no longer asserted; safe to reset */
+		if (!(current1 & CS35L34_OTW)) {
+			dev_dbg(codec->dev,
+				"Over temperature warning release\n");
+			regmap_update_bits(cs35l34->regmap,
+					CS35L34_PROT_RELEASE_CTL,
+					CS35L34_OTW_RLS, 0);
+			regmap_update_bits(cs35l34->regmap,
+					CS35L34_PROT_RELEASE_CTL,
+					CS35L34_OTW_RLS,
+					CS35L34_OTW_RLS);
+			regmap_update_bits(cs35l34->regmap,
+					CS35L34_PROT_RELEASE_CTL,
+					CS35L34_OTW_RLS, 0);
+		}
+	}
+
+	if (sticky1 & CS35L34_OTE) {
+		dev_crit(codec->dev, "Over temperature error\n");
+
+		/* error is no longer asserted; safe to reset */
+		if (!(current1 & CS35L34_OTE)) {
+			dev_dbg(codec->dev,
+				"Over temperature error release\n");
+			regmap_update_bits(cs35l34->regmap,
+					CS35L34_PROT_RELEASE_CTL,
+					CS35L34_OTE_RLS, 0);
+			regmap_update_bits(cs35l34->regmap,
+					CS35L34_PROT_RELEASE_CTL,
+					CS35L34_OTE_RLS,
+					CS35L34_OTE_RLS);
+			regmap_update_bits(cs35l34->regmap,
+					CS35L34_PROT_RELEASE_CTL,
+					CS35L34_OTE_RLS, 0);
+		}
+	}
+
+	if (sticky3 & CS35L34_BST_HIGH) {
+		dev_crit(codec->dev, "VBST too high error; powering off!\n");
+		regmap_update_bits(cs35l34->regmap, CS35L34_PWRCTL2,
+				CS35L34_PDN_AMP, CS35L34_PDN_AMP);
+		regmap_update_bits(cs35l34->regmap, CS35L34_PWRCTL1,
+				CS35L34_PDN_ALL, CS35L34_PDN_ALL);
+	}
+
+	if (sticky3 & CS35L34_LBST_SHORT) {
+		dev_crit(codec->dev, "LBST short error; powering off!\n");
+		regmap_update_bits(cs35l34->regmap, CS35L34_PWRCTL2,
+				CS35L34_PDN_AMP, CS35L34_PDN_AMP);
+		regmap_update_bits(cs35l34->regmap, CS35L34_PWRCTL1,
+				CS35L34_PDN_ALL, CS35L34_PDN_ALL);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static const char * const cs35l34_core_supplies[] = {
+	"VA",
+	"VP",
+};
+
+static int cs35l34_i2c_probe(struct i2c_client *i2c_client,
+			      const struct i2c_device_id *id)
+{
+	struct cs35l34_private *cs35l34;
+	struct cs35l34_platform_data *pdata =
+		dev_get_platdata(&i2c_client->dev);
+	int i;
+	int ret;
+	unsigned int devid = 0;
+	unsigned int reg;
+
+	cs35l34 = devm_kzalloc(&i2c_client->dev,
+			       sizeof(struct cs35l34_private),
+			       GFP_KERNEL);
+	if (!cs35l34) {
+		dev_err(&i2c_client->dev, "could not allocate codec\n");
+		return -ENOMEM;
+	}
+
+	i2c_set_clientdata(i2c_client, cs35l34);
+	cs35l34->regmap = devm_regmap_init_i2c(i2c_client, &cs35l34_regmap);
+	if (IS_ERR(cs35l34->regmap)) {
+		ret = PTR_ERR(cs35l34->regmap);
+		dev_err(&i2c_client->dev, "regmap_init() failed: %d\n", ret);
+		return ret;
+	}
+
+	cs35l34->num_core_supplies = ARRAY_SIZE(cs35l34_core_supplies);
+	for (i = 0; i < ARRAY_SIZE(cs35l34_core_supplies); i++)
+		cs35l34->core_supplies[i].supply = cs35l34_core_supplies[i];
+
+	ret = devm_regulator_bulk_get(&i2c_client->dev,
+		cs35l34->num_core_supplies,
+		cs35l34->core_supplies);
+	if (ret != 0) {
+		dev_err(&i2c_client->dev,
+			"Failed to request core supplies %d\n", ret);
+		return ret;
+	}
+
+	ret = regulator_bulk_enable(cs35l34->num_core_supplies,
+					cs35l34->core_supplies);
+	if (ret != 0) {
+		dev_err(&i2c_client->dev,
+			"Failed to enable core supplies: %d\n", ret);
+		return ret;
+	}
+
+	if (pdata) {
+		cs35l34->pdata = *pdata;
+	} else {
+		pdata = devm_kzalloc(&i2c_client->dev,
+				sizeof(struct cs35l34_platform_data),
+				GFP_KERNEL);
+		if (!pdata) {
+			dev_err(&i2c_client->dev,
+				"could not allocate pdata\n");
+			return -ENOMEM;
+		}
+		if (i2c_client->dev.of_node) {
+			ret = cs35l34_handle_of_data(i2c_client, pdata);
+			if (ret != 0)
+				return ret;
+
+		}
+		cs35l34->pdata = *pdata;
+	}
+
+	ret = devm_request_threaded_irq(&i2c_client->dev, i2c_client->irq, NULL,
+			cs35l34_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW,
+			"cs35l34", cs35l34);
+	if (ret != 0)
+		dev_err(&i2c_client->dev, "Failed to request IRQ: %d\n", ret);
+
+	cs35l34->reset_gpio = devm_gpiod_get_optional(&i2c_client->dev,
+				"reset-gpios", GPIOD_OUT_LOW);
+	if (IS_ERR(cs35l34->reset_gpio))
+		return PTR_ERR(cs35l34->reset_gpio);
+
+	gpiod_set_value_cansleep(cs35l34->reset_gpio, 1);
+
+	msleep(CS35L34_START_DELAY);
+
+	ret = regmap_read(cs35l34->regmap, CS35L34_DEVID_AB, &reg);
+
+	devid = (reg & 0xFF) << 12;
+	ret = regmap_read(cs35l34->regmap, CS35L34_DEVID_CD, &reg);
+	devid |= (reg & 0xFF) << 4;
+	ret = regmap_read(cs35l34->regmap, CS35L34_DEVID_E, &reg);
+	devid |= (reg & 0xF0) >> 4;
+
+	if (devid != CS35L34_CHIP_ID) {
+		dev_err(&i2c_client->dev,
+			"CS35l34 Device ID (%X). Expected ID %X\n",
+			devid, CS35L34_CHIP_ID);
+		ret = -ENODEV;
+		goto err_regulator;
+	}
+
+	ret = regmap_read(cs35l34->regmap, CS35L34_REV_ID, &reg);
+	if (ret < 0) {
+		dev_err(&i2c_client->dev, "Get Revision ID failed\n");
+		goto err_regulator;
+	}
+
+	dev_info(&i2c_client->dev,
+		 "Cirrus Logic CS35l34 (%x), Revision: %02X\n", devid,
+		reg & 0xFF);
+
+	/* Unmask critical interrupts */
+	regmap_update_bits(cs35l34->regmap, CS35L34_INT_MASK_1,
+				CS35L34_M_CAL_ERR | CS35L34_M_ALIVE_ERR |
+				CS35L34_M_AMP_SHORT | CS35L34_M_OTW |
+				CS35L34_M_OTE, 0);
+	regmap_update_bits(cs35l34->regmap, CS35L34_INT_MASK_3,
+				CS35L34_M_BST_HIGH | CS35L34_M_LBST_SHORT, 0);
+
+	pm_runtime_set_autosuspend_delay(&i2c_client->dev, 100);
+	pm_runtime_use_autosuspend(&i2c_client->dev);
+	pm_runtime_set_active(&i2c_client->dev);
+	pm_runtime_enable(&i2c_client->dev);
+
+	ret =  snd_soc_register_codec(&i2c_client->dev,
+			&soc_codec_dev_cs35l34, &cs35l34_dai, 1);
+	if (ret < 0) {
+		dev_err(&i2c_client->dev,
+			"%s: Register codec failed\n", __func__);
+		goto err_regulator;
+	}
+
+	return 0;
+
+err_regulator:
+	regulator_bulk_disable(cs35l34->num_core_supplies,
+		cs35l34->core_supplies);
+
+	return ret;
+}
+
+static int cs35l34_i2c_remove(struct i2c_client *client)
+{
+	struct cs35l34_private *cs35l34 = i2c_get_clientdata(client);
+
+	snd_soc_unregister_codec(&client->dev);
+
+	if (cs35l34->reset_gpio)
+		gpiod_set_value_cansleep(cs35l34->reset_gpio, 0);
+
+	pm_runtime_disable(&client->dev);
+	regulator_bulk_disable(cs35l34->num_core_supplies,
+		cs35l34->core_supplies);
+
+	return 0;
+}
+
+static int __maybe_unused cs35l34_runtime_resume(struct device *dev)
+{
+	struct cs35l34_private *cs35l34 = dev_get_drvdata(dev);
+	int ret;
+
+	ret = regulator_bulk_enable(cs35l34->num_core_supplies,
+		cs35l34->core_supplies);
+
+	if (ret != 0) {
+		dev_err(dev, "Failed to enable core supplies: %d\n",
+			ret);
+		return ret;
+	}
+
+	regcache_cache_only(cs35l34->regmap, false);
+
+	gpiod_set_value_cansleep(cs35l34->reset_gpio, 1);
+	msleep(CS35L34_START_DELAY);
+
+	ret = regcache_sync(cs35l34->regmap);
+	if (ret != 0) {
+		dev_err(dev, "Failed to restore register cache\n");
+		goto err;
+	}
+	return 0;
+err:
+	regcache_cache_only(cs35l34->regmap, true);
+	regulator_bulk_disable(cs35l34->num_core_supplies,
+		cs35l34->core_supplies);
+
+	return ret;
+}
+
+static int __maybe_unused cs35l34_runtime_suspend(struct device *dev)
+{
+	struct cs35l34_private *cs35l34 = dev_get_drvdata(dev);
+
+	regcache_cache_only(cs35l34->regmap, true);
+	regcache_mark_dirty(cs35l34->regmap);
+
+	gpiod_set_value_cansleep(cs35l34->reset_gpio, 0);
+
+	regulator_bulk_disable(cs35l34->num_core_supplies,
+			cs35l34->core_supplies);
+
+	return 0;
+}
+
+static const struct dev_pm_ops cs35l34_pm_ops = {
+	SET_RUNTIME_PM_OPS(cs35l34_runtime_suspend,
+			   cs35l34_runtime_resume,
+			   NULL)
+};
+
+static const struct of_device_id cs35l34_of_match[] = {
+	{.compatible = "cirrus,cs35l34"},
+	{},
+};
+MODULE_DEVICE_TABLE(of, cs35l34_of_match);
+
+static const struct i2c_device_id cs35l34_id[] = {
+	{"cs35l34", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, cs35l34_id);
+
+static struct i2c_driver cs35l34_i2c_driver = {
+	.driver = {
+		.name = "cs35l34",
+		.pm = &cs35l34_pm_ops,
+		.of_match_table = cs35l34_of_match,
+
+		},
+	.id_table = cs35l34_id,
+	.probe = cs35l34_i2c_probe,
+	.remove = cs35l34_i2c_remove,
+
+};
+
+static int __init cs35l34_modinit(void)
+{
+	int ret;
+
+	ret = i2c_add_driver(&cs35l34_i2c_driver);
+	if (ret != 0) {
+		pr_err("Failed to register CS35l34 I2C driver: %d\n", ret);
+		return ret;
+	}
+	return 0;
+}
+module_init(cs35l34_modinit);
+
+static void __exit cs35l34_exit(void)
+{
+	i2c_del_driver(&cs35l34_i2c_driver);
+}
+module_exit(cs35l34_exit);
+
+MODULE_DESCRIPTION("ASoC CS35l34 driver");
+MODULE_AUTHOR("Paul Handrigan, Cirrus Logic Inc, <Paul.Handrigan@cirrus.com>");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/cs35l34.h b/sound/soc/codecs/cs35l34.h
new file mode 100644
index 000000000000..bcd54f127559
--- /dev/null
+++ b/sound/soc/codecs/cs35l34.h
@@ -0,0 +1,269 @@
+/*
+ * cs35l34.h -- CS35L34 ALSA SoC audio driver
+ *
+ * Copyright 2016 Cirrus Logic, Inc.
+ *
+ * Author: Paul Handrigan <Paul.Handrigan@cirrus.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __CS35L34_H__
+#define __CS35L34_H__
+
+#define CS35L34_CHIP_ID			0x00035A34
+#define CS35L34_DEVID_AB		0x01	/* Device ID A & B [RO] */
+#define CS35L34_DEVID_CD		0x02    /* Device ID C & D [RO] */
+#define CS35L34_DEVID_E			0x03    /* Device ID E [RO] */
+#define CS35L34_FAB_ID			0x04	/* Fab ID [RO] */
+#define CS35L34_REV_ID			0x05	/* Revision ID [RO] */
+#define CS35L34_PWRCTL1			0x06    /* Power Ctl 1 */
+#define CS35L34_PWRCTL2			0x07    /* Power Ctl 2 */
+#define CS35L34_PWRCTL3			0x08	/* Power Ctl 3 */
+#define CS35L34_ADSP_CLK_CTL		0x0A	/* (ADSP) Clock Ctl */
+#define CS35L34_MCLK_CTL		0x0B	/* Master Clocking Ctl */
+#define CS35L34_AMP_INP_DRV_CTL		0x14	/* Amp Input Drive Ctl */
+#define CS35L34_AMP_DIG_VOL_CTL		0x15	/* Amplifier Dig Volume Ctl */
+#define CS35L34_AMP_DIG_VOL		0x16	/* Amplifier Dig Volume */
+#define CS35L34_AMP_ANLG_GAIN_CTL	0x17	/* Amplifier Analog Gain Ctl */
+#define CS35L34_PROTECT_CTL		0x18	/* Amp Gain - Prot Ctl Param */
+#define CS35L34_AMP_KEEP_ALIVE_CTL	0x1A	/* Amplifier Keep Alive Ctl */
+#define CS35L34_BST_CVTR_V_CTL		0x1D	/* Boost Conv Voltage Ctl */
+#define CS35L34_BST_PEAK_I		0x1E	/* Boost Conv Peak Current */
+#define CS35L34_BST_RAMP_CTL		0x20	/* Boost Conv Soft Ramp Ctl */
+#define CS35L34_BST_CONV_COEF_1		0x21	/* Boost Conv Coefficients 1 */
+#define CS35L34_BST_CONV_COEF_2		0x22	/* Boost Conv Coefficients 2 */
+#define CS35L34_BST_CONV_SLOPE_COMP	0x23	/* Boost Conv Slope Comp */
+#define CS35L34_BST_CONV_SW_FREQ	0x24	/* Boost Conv L BST SW Freq */
+#define CS35L34_CLASS_H_CTL		0x30	/* CLS H Control */
+#define CS35L34_CLASS_H_HEADRM_CTL	0x31	/* CLS H Headroom Ctl */
+#define CS35L34_CLASS_H_RELEASE_RATE	0x32	/* CLS H Release Rate */
+#define CS35L34_CLASS_H_FET_DRIVE_CTL	0x33	/* CLS H Weak FET Drive Ctl */
+#define CS35L34_CLASS_H_STATUS		0x38	/* CLS H Status */
+#define CS35L34_VPBR_CTL		0x3A	/* VPBR Ctl */
+#define CS35L34_VPBR_VOL_CTL		0x3B	/* VPBR Volume Ctl */
+#define CS35L34_VPBR_TIMING_CTL		0x3C	/* VPBR Timing Ctl */
+#define CS35L34_PRED_MAX_ATTEN_SPK_LOAD	0x40	/* PRD Max Atten / Spkr Load */
+#define CS35L34_PRED_BROWNOUT_THRESH	0x41	/* PRD Brownout Threshold */
+#define CS35L34_PRED_BROWNOUT_VOL_CTL	0x42	/* PRD Brownout Volume Ctl */
+#define CS35L34_PRED_BROWNOUT_RATE_CTL	0x43	/* PRD Brownout Rate Ctl */
+#define CS35L34_PRED_WAIT_CTL		0x44	/* PRD Wait Ctl */
+#define CS35L34_PRED_ZVP_INIT_IMP_CTL	0x46	/* PRD ZVP Initial Imp Ctl */
+#define CS35L34_PRED_MAN_SAFE_VPI_CTL	0x47	/* PRD Manual Safe VPI Ctl */
+#define CS35L34_VPBR_ATTEN_STATUS	0x4B	/* VPBR Attenuation Status */
+#define CS35L34_PRED_BRWNOUT_ATT_STATUS	0x4C	/* PRD Brownout Atten Status */
+#define CS35L34_SPKR_MON_CTL		0x4E	/* Speaker Monitoring Ctl */
+#define CS35L34_ADSP_I2S_CTL		0x50	/* ADSP I2S Ctl */
+#define CS35L34_ADSP_TDM_CTL		0x51	/* ADSP TDM Ctl */
+#define CS35L34_TDM_TX_CTL_1_VMON	0x52	/* TDM TX Ctl 1 (VMON) */
+#define CS35L34_TDM_TX_CTL_2_IMON	0x53	/* TDM TX Ctl 2 (IMON) */
+#define CS35L34_TDM_TX_CTL_3_VPMON	0x54	/* TDM TX Ctl 3 (VPMON) */
+#define CS35L34_TDM_TX_CTL_4_VBSTMON	0x55	/* TDM TX Ctl 4 (VBSTMON) */
+#define CS35L34_TDM_TX_CTL_5_FLAG1	0x56	/* TDM TX Ctl 5 (FLAG1) */
+#define CS35L34_TDM_TX_CTL_6_FLAG2	0x57	/* TDM TX Ctl 6 (FLAG2) */
+#define CS35L34_TDM_TX_SLOT_EN_1	0x5A	/* TDM TX Slot Enable */
+#define CS35L34_TDM_TX_SLOT_EN_2	0x5B	/* TDM TX Slot Enable */
+#define CS35L34_TDM_TX_SLOT_EN_3	0x5C	/* TDM TX Slot Enable */
+#define CS35L34_TDM_TX_SLOT_EN_4	0x5D	/* TDM TX Slot Enable */
+#define CS35L34_TDM_RX_CTL_1_AUDIN	0x5E	/* TDM RX Ctl 1 */
+#define CS35L34_TDM_RX_CTL_3_ALIVE	0x60	/* TDM RX Ctl 3 (ALIVE) */
+#define CS35L34_MULT_DEV_SYNCH1		0x62	/* Multidevice Synch */
+#define CS35L34_MULT_DEV_SYNCH2		0x63	/* Multidevice Synch 2 */
+#define CS35L34_PROT_RELEASE_CTL	0x64	/* Protection Release Ctl */
+#define CS35L34_DIAG_MODE_REG_LOCK	0x68	/* Diagnostic Mode Reg Lock */
+#define CS35L34_DIAG_MODE_CTL_1		0x69	/* Diagnostic Mode Ctl 1 */
+#define CS35L34_DIAG_MODE_CTL_2		0x6A	/* Diagnostic Mode Ctl 2 */
+#define CS35L34_INT_MASK_1		0x70	/* Interrupt Mask 1 */
+#define CS35L34_INT_MASK_2		0x71	/* Interrupt Mask 2 */
+#define CS35L34_INT_MASK_3		0x72	/* Interrupt Mask 3 */
+#define CS35L34_INT_MASK_4		0x73	/* Interrupt Mask 4 */
+#define CS35L34_INT_STATUS_1		0x74	/* Interrupt Status 1 */
+#define CS35L34_INT_STATUS_2		0x75	/* Interrupt Status 2 */
+#define CS35L34_INT_STATUS_3		0x76	/* Interrupt Status 3 */
+#define CS35L34_INT_STATUS_4		0x77	/* Interrupt Status 4 */
+#define CS35L34_OTP_TRIM_STATUS		0x7E	/* OTP Trim Status */
+
+#define CS35L34_MAX_REGISTER		0x7F
+#define CS35L34_REGISTER_COUNT		0x4E
+
+#define CS35L34_MCLK_5644		5644800
+#define CS35L34_MCLK_6144		6144000
+#define CS35L34_MCLK_6			6000000
+#define CS35L34_MCLK_11289		11289600
+#define CS35L34_MCLK_12			12000000
+#define CS35L34_MCLK_12288		12288000
+
+/* CS35L34_PWRCTL1 */
+#define CS35L34_SFT_RST			(1 << 7)
+#define CS35L34_DISCHG_FLT		(1 << 1)
+#define CS35L34_PDN_ALL			1
+
+/* CS35L34_PWRCTL2 */
+#define CS35L34_PDN_VMON		(1 << 7)
+#define CS35L34_PDN_IMON		(1 << 6)
+#define CS35L34_PDN_CLASSH		(1 << 5)
+#define CS35L34_PDN_VPBR		(1 << 4)
+#define CS35L34_PDN_PRED		(1 << 3)
+#define CS35L34_PDN_BST			(1 << 2)
+#define CS35L34_PDN_AMP			1
+
+/* CS35L34_PWRCTL3 */
+#define CS35L34_MCLK_DIS		(1 << 7)
+#define CS35L34_PDN_VBSTMON_OUT		(1 << 4)
+#define CS35L34_PDN_VMON_OUT		(1 << 3)
+/* Tristate the ADSP SDOUT when in I2C mode */
+#define CS35L34_PDN_SDOUT		(1 << 2)
+#define CS35L34_PDN_SDIN		(1 << 1)
+#define CS35L34_PDN_TDM			1
+
+/* CS35L34_ADSP_CLK_CTL */
+#define CS35L34_ADSP_RATE		0xF
+#define CS35L34_ADSP_DRIVE		(1 << 4)
+#define CS35L34_ADSP_M_S		(1 << 7)
+
+/* CS35L34_MCLK_CTL */
+#define CS35L34_MCLK_DIV		(1 << 4)
+#define CS35L34_MCLK_RATE_MASK		0x7
+#define CS35L34_MCLK_RATE_6P1440	0x2
+#define CS35L34_MCLK_RATE_6P0000	0x1
+#define CS35L34_MCLK_RATE_5P6448	0x0
+#define CS35L34_MCLKDIS			(1 << 7)
+#define CS35L34_MCLKDIV2		(1 << 6)
+#define CS35L34_SDOUT_3ST_TDM		(1 << 5)
+#define CS35L34_INT_FS_RATE		(1 << 4)
+#define CS35L34_ADSP_FS			0xF
+
+/* CS35L34_AMP_INP_DRV_CTL */
+#define CS35L34_DRV_STR_SRC		(1 << 1)
+#define CS35L34_DRV_STR			1
+
+/* CS35L34_AMP_DIG_VOL_CTL */
+#define CS35L34_AMP_DSR_RATE_MASK	0xF0
+#define CS35L34_AMP_DSR_RATE_SHIFT	(1 << 4)
+#define CS35L34_NOTCH_DIS		(1 << 3)
+#define CS35L34_AMP_DIGSFT		(1 << 1)
+#define CS35L34_INV			1
+
+/* CS35L34_PROTECT_CTL */
+#define CS35L34_OTW_ATTN_MASK		0xC
+#define CS35L34_OTW_THRD_MASK		0x3
+#define CS35L34_MUTE			(1 << 5)
+#define CS35L34_GAIN_ZC			(1 << 4)
+#define CS35L34_GAIN_ZC_MASK		0x10
+#define CS35L34_GAIN_ZC_SHIFT		4
+
+/* CS35L34_AMP_KEEP_ALIVE_CTL */
+#define CS35L34_ALIVE_WD_DIS		(1 << 2)
+
+/* CS35L34_BST_CVTR_V_CTL */
+#define CS35L34_BST_CVTL_MASK		0x3F
+
+/* CS35L34_BST_PEAK_I */
+#define CS35L34_BST_PEAK_MASK		0x3F
+
+/* CS35L34_ADSP_I2S_CTL */
+#define CS35L34_I2S_LOC_MASK		0xC
+#define CS35L34_I2S_LOC_SHIFT		2
+
+/* CS35L34_MULT_DEV_SYNCH2 */
+#define CS35L34_SYNC2_MASK		0xF
+
+/* CS35L34_PROT_RELEASE_CTL */
+#define CS35L34_CAL_ERR_RLS		(1 << 7)
+#define CS35L34_SHORT_RLS		(1 << 2)
+#define CS35L34_OTW_RLS			(1 << 1)
+#define CS35L34_OTE_RLS			1
+
+/* CS35L34_INT_MASK_1 */
+#define CS35L34_M_CAL_ERR_SHIFT		7
+#define CS35L34_M_CAL_ERR		(1 << CS35L34_M_CAL_ERR_SHIFT)
+#define CS35L34_M_ALIVE_ERR_SHIFT	5
+#define CS35L34_M_ALIVE_ERR		(1 << CS35L34_M_ALIVE_ERR_SHIFT)
+#define CS35L34_M_ADSP_CLK_SHIFT	4
+#define CS35L34_M_ADSP_CLK_ERR		(1 << CS35L34_M_ADSP_CLK_SHIFT)
+#define CS35L34_M_MCLK_SHIFT		3
+#define CS35L34_M_MCLK_ERR		(1 << CS35L34_M_MCLK_SHIFT)
+#define CS35L34_M_AMP_SHORT_SHIFT	2
+#define CS35L34_M_AMP_SHORT		(1 << CS35L34_M_AMP_SHORT_SHIFT)
+#define CS35L34_M_OTW_SHIFT		1
+#define CS35L34_M_OTW			(1 << CS35L34_M_OTW_SHIFT)
+#define CS35L34_M_OTE_SHIFT		0
+#define CS35L34_M_OTE			(1 << CS35L34_M_OTE_SHIFT)
+
+/* CS35L34_INT_MASK_2 */
+#define CS35L34_M_PDN_DONE_SHIFT	4
+#define CS35L34_M_PDN_DONE		(1 << CS35L34_M_PDN_DONE_SHIFT)
+#define CS35L34_M_PRED_SHIFT		3
+#define CS35L34_M_PRED_ERR		(1 << CS35L34_M_PRED_SHIFT)
+#define CS35L34_M_PRED_CLR_SHIFT	2
+#define CS35L34_M_PRED_CLR		(1 << CS35L34_M_PRED_CLR_SHIFT)
+#define CS35L34_M_VPBR_SHIFT		1
+#define CS35L34_M_VPBR_ERR		(1 << CS35L34_M_VPBR_SHIFT)
+#define CS35L34_M_VPBR_CLR_SHIFT	0
+#define CS35L34_M_VPBR_CLR		(1 << CS35L34_M_VPBR_CLR_SHIFT)
+
+/* CS35L34_INT_MASK_3 */
+#define CS35L34_M_BST_HIGH_SHIFT	4
+#define CS35L34_M_BST_HIGH		(1 << CS35L34_M_BST_HIGH_SHIFT)
+#define CS35L34_M_BST_HIGH_FLAG_SHIFT	3
+#define CS35L34_M_BST_HIGH_FLAG		(1 << CS35L34_M_BST_HIGH_FLAG_SHIFT)
+#define CS35L34_M_BST_IPK_FLAG_SHIFT	2
+#define CS35L34_M_BST_IPK_FLAG		(1 << CS35L34_M_BST_IPK_FLAG_SHIFT)
+#define CS35L34_M_LBST_SHORT_SHIFT	0
+#define CS35L34_M_LBST_SHORT		(1 << CS35L34_M_LBST_SHORT_SHIFT)
+
+/* CS35L34_INT_MASK_4 */
+#define CS35L34_M_VMON_OVFL_SHIFT	3
+#define CS35L34_M_VMON_OVFL		(1 << CS35L34_M_VMON_OVFL_SHIFT)
+#define CS35L34_M_IMON_OVFL_SHIFT	2
+#define CS35L34_M_IMON_OVFL		(1 << CS35L34_M_IMON_OVFL_SHIFT)
+#define CS35L34_M_VPMON_OVFL_SHIFT	1
+#define CS35L34_M_VPMON_OVFL		(1 << CS35L34_M_VPMON_OVFL_SHIFT)
+#define CS35L34_M_VBSTMON_OVFL_SHIFT	1
+#define CS35L34_M_VBSTMON_OVFL		(1 << CS35L34_M_VBSTMON_OVFL_SHIFT)
+
+/* CS35L34_INT_1 */
+#define CS35L34_CAL_ERR			(1 << CS35L34_M_CAL_ERR_SHIFT)
+#define CS35L34_ALIVE_ERR		(1 << CS35L34_M_ALIVE_ERR_SHIFT)
+#define CS35L34_M_ADSP_CLK_ERR		(1 << CS35L34_M_ADSP_CLK_SHIFT)
+#define CS35L34_MCLK_ERR		(1 << CS35L34_M_MCLK_SHIFT)
+#define CS35L34_AMP_SHORT		(1 << CS35L34_M_AMP_SHORT_SHIFT)
+#define CS35L34_OTW			(1 << CS35L34_M_OTW_SHIFT)
+#define CS35L34_OTE			(1 << CS35L34_M_OTE_SHIFT)
+
+/* CS35L34_INT_2 */
+#define CS35L34_PDN_DONE		(1 << CS35L34_M_PDN_DONE_SHIFT)
+#define CS35L34_PRED_ERR		(1 << CS35L34_M_PRED_SHIFT)
+#define CS35L34_PRED_CLR		(1 << CS35L34_M_PRED_CLR_SHIFT)
+#define CS35L34_VPBR_ERR		(1 << CS35L34_M_VPBR_SHIFT)
+#define CS35L34_VPBR_CLR		(1 << CS35L34_M_VPBR_CLR_SHIFT)
+
+/* CS35L34_INT_3 */
+#define CS35L34_BST_HIGH		(1 << CS35L34_M_BST_HIGH_SHIFT)
+#define CS35L34_BST_HIGH_FLAG		(1 << CS35L34_M_BST_HIGH_FLAG_SHIFT)
+#define CS35L34_BST_IPK_FLAG		(1 << CS35L34_M_BST_IPK_FLAG_SHIFT)
+#define CS35L34_LBST_SHORT		(1 << CS35L34_M_LBST_SHORT_SHIFT)
+
+/* CS35L34_INT_4 */
+#define CS35L34_VMON_OVFL		(1 << CS35L34_M_VMON_OVFL_SHIFT)
+#define CS35L34_IMON_OVFL		(1 << CS35L34_M_IMON_OVFL_SHIFT)
+#define CS35L34_VPMON_OVFL		(1 << CS35L34_M_VPMON_OVFL_SHIFT)
+#define CS35L34_VBSTMON_OVFL		(1 << CS35L34_M_VBSTMON_OVFL_SHIFT)
+
+/* CS35L34_{RX,TX}_X */
+#define CS35L34_X_STATE_SHIFT		7
+#define CS35L34_X_STATE			(1 << CS35L34_X_STATE_SHIFT)
+#define CS35L34_X_LOC_SHIFT		0
+#define CS35L34_X_LOC			(0x1F << CS35L34_X_LOC_SHIFT)
+
+#define CS35L34_RATES (SNDRV_PCM_RATE_48000 | \
+			SNDRV_PCM_RATE_44100 | \
+			SNDRV_PCM_RATE_32000)
+#define CS35L34_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | \
+			SNDRV_PCM_FMTBIT_S24_LE | \
+			SNDRV_PCM_FMTBIT_S32_LE)
+
+#endif
diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c
new file mode 100644
index 000000000000..55e4520cdcaf
--- /dev/null
+++ b/sound/soc/codecs/cs42l42.c
@@ -0,0 +1,1986 @@
+/*
+ * cs42l42.c -- CS42L42 ALSA SoC audio driver
+ *
+ * Copyright 2016 Cirrus Logic, Inc.
+ *
+ * Author: James Schulman <james.schulman@cirrus.com>
+ * Author: Brian Austin <brian.austin@cirrus.com>
+ * Author: Michael White <michael.white@cirrus.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/version.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/gpio.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/gpio/consumer.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/of_device.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/soc-dapm.h>
+#include <sound/initval.h>
+#include <sound/tlv.h>
+#include <dt-bindings/sound/cs42l42.h>
+
+#include "cs42l42.h"
+
+static const struct reg_default cs42l42_reg_defaults[] = {
+	{ CS42L42_FRZ_CTL,			0x00 },
+	{ CS42L42_SRC_CTL,			0x10 },
+	{ CS42L42_MCLK_STATUS,			0x02 },
+	{ CS42L42_MCLK_CTL,			0x02 },
+	{ CS42L42_SFTRAMP_RATE,			0xA4 },
+	{ CS42L42_I2C_DEBOUNCE,			0x88 },
+	{ CS42L42_I2C_STRETCH,			0x03 },
+	{ CS42L42_I2C_TIMEOUT,			0xB7 },
+	{ CS42L42_PWR_CTL1,			0xFF },
+	{ CS42L42_PWR_CTL2,			0x84 },
+	{ CS42L42_PWR_CTL3,			0x20 },
+	{ CS42L42_RSENSE_CTL1,			0x40 },
+	{ CS42L42_RSENSE_CTL2,			0x00 },
+	{ CS42L42_OSC_SWITCH,			0x00 },
+	{ CS42L42_OSC_SWITCH_STATUS,		0x05 },
+	{ CS42L42_RSENSE_CTL3,			0x1B },
+	{ CS42L42_TSENSE_CTL,			0x1B },
+	{ CS42L42_TSRS_INT_DISABLE,		0x00 },
+	{ CS42L42_TRSENSE_STATUS,		0x00 },
+	{ CS42L42_HSDET_CTL1,			0x77 },
+	{ CS42L42_HSDET_CTL2,			0x00 },
+	{ CS42L42_HS_SWITCH_CTL,		0xF3 },
+	{ CS42L42_HS_DET_STATUS,		0x00 },
+	{ CS42L42_HS_CLAMP_DISABLE,		0x00 },
+	{ CS42L42_MCLK_SRC_SEL,			0x00 },
+	{ CS42L42_SPDIF_CLK_CFG,		0x00 },
+	{ CS42L42_FSYNC_PW_LOWER,		0x00 },
+	{ CS42L42_FSYNC_PW_UPPER,		0x00 },
+	{ CS42L42_FSYNC_P_LOWER,		0xF9 },
+	{ CS42L42_FSYNC_P_UPPER,		0x00 },
+	{ CS42L42_ASP_CLK_CFG,			0x00 },
+	{ CS42L42_ASP_FRM_CFG,			0x10 },
+	{ CS42L42_FS_RATE_EN,			0x00 },
+	{ CS42L42_IN_ASRC_CLK,			0x00 },
+	{ CS42L42_OUT_ASRC_CLK,			0x00 },
+	{ CS42L42_PLL_DIV_CFG1,			0x00 },
+	{ CS42L42_ADC_OVFL_STATUS,		0x00 },
+	{ CS42L42_MIXER_STATUS,			0x00 },
+	{ CS42L42_SRC_STATUS,			0x00 },
+	{ CS42L42_ASP_RX_STATUS,		0x00 },
+	{ CS42L42_ASP_TX_STATUS,		0x00 },
+	{ CS42L42_CODEC_STATUS,			0x00 },
+	{ CS42L42_DET_INT_STATUS1,		0x00 },
+	{ CS42L42_DET_INT_STATUS2,		0x00 },
+	{ CS42L42_SRCPL_INT_STATUS,		0x00 },
+	{ CS42L42_VPMON_STATUS,			0x00 },
+	{ CS42L42_PLL_LOCK_STATUS,		0x00 },
+	{ CS42L42_TSRS_PLUG_STATUS,		0x00 },
+	{ CS42L42_ADC_OVFL_INT_MASK,		0x01 },
+	{ CS42L42_MIXER_INT_MASK,		0x0F },
+	{ CS42L42_SRC_INT_MASK,			0x0F },
+	{ CS42L42_ASP_RX_INT_MASK,		0x1F },
+	{ CS42L42_ASP_TX_INT_MASK,		0x0F },
+	{ CS42L42_CODEC_INT_MASK,		0x03 },
+	{ CS42L42_SRCPL_INT_MASK,		0xFF },
+	{ CS42L42_VPMON_INT_MASK,		0x01 },
+	{ CS42L42_PLL_LOCK_INT_MASK,		0x01 },
+	{ CS42L42_TSRS_PLUG_INT_MASK,		0x0F },
+	{ CS42L42_PLL_CTL1,			0x00 },
+	{ CS42L42_PLL_DIV_FRAC0,		0x00 },
+	{ CS42L42_PLL_DIV_FRAC1,		0x00 },
+	{ CS42L42_PLL_DIV_FRAC2,		0x00 },
+	{ CS42L42_PLL_DIV_INT,			0x40 },
+	{ CS42L42_PLL_CTL3,			0x10 },
+	{ CS42L42_PLL_CAL_RATIO,		0x80 },
+	{ CS42L42_PLL_CTL4,			0x03 },
+	{ CS42L42_LOAD_DET_RCSTAT,		0x00 },
+	{ CS42L42_LOAD_DET_DONE,		0x00 },
+	{ CS42L42_LOAD_DET_EN,			0x00 },
+	{ CS42L42_HSBIAS_SC_AUTOCTL,		0x03 },
+	{ CS42L42_WAKE_CTL,			0xC0 },
+	{ CS42L42_ADC_DISABLE_MUTE,		0x00 },
+	{ CS42L42_TIPSENSE_CTL,			0x02 },
+	{ CS42L42_MISC_DET_CTL,			0x03 },
+	{ CS42L42_MIC_DET_CTL1,			0x1F },
+	{ CS42L42_MIC_DET_CTL2,			0x2F },
+	{ CS42L42_DET_STATUS1,			0x00 },
+	{ CS42L42_DET_STATUS2,			0x00 },
+	{ CS42L42_DET_INT1_MASK,		0xE0 },
+	{ CS42L42_DET_INT2_MASK,		0xFF },
+	{ CS42L42_HS_BIAS_CTL,			0xC2 },
+	{ CS42L42_ADC_CTL,			0x00 },
+	{ CS42L42_ADC_VOLUME,			0x00 },
+	{ CS42L42_ADC_WNF_HPF_CTL,		0x71 },
+	{ CS42L42_DAC_CTL1,			0x00 },
+	{ CS42L42_DAC_CTL2,			0x02 },
+	{ CS42L42_HP_CTL,			0x0D },
+	{ CS42L42_CLASSH_CTL,			0x07 },
+	{ CS42L42_MIXER_CHA_VOL,		0x3F },
+	{ CS42L42_MIXER_ADC_VOL,		0x3F },
+	{ CS42L42_MIXER_CHB_VOL,		0x3F },
+	{ CS42L42_EQ_COEF_IN0,			0x22 },
+	{ CS42L42_EQ_COEF_IN1,			0x00 },
+	{ CS42L42_EQ_COEF_IN2,			0x00 },
+	{ CS42L42_EQ_COEF_IN3,			0x00 },
+	{ CS42L42_EQ_COEF_RW,			0x00 },
+	{ CS42L42_EQ_COEF_OUT0,			0x00 },
+	{ CS42L42_EQ_COEF_OUT1,			0x00 },
+	{ CS42L42_EQ_COEF_OUT2,			0x00 },
+	{ CS42L42_EQ_COEF_OUT3,			0x00 },
+	{ CS42L42_EQ_INIT_STAT,			0x00 },
+	{ CS42L42_EQ_START_FILT,		0x00 },
+	{ CS42L42_EQ_MUTE_CTL,			0x00 },
+	{ CS42L42_SP_RX_CH_SEL,			0x04 },
+	{ CS42L42_SP_RX_ISOC_CTL,		0x04 },
+	{ CS42L42_SP_RX_FS,			0x8C },
+	{ CS42l42_SPDIF_CH_SEL,			0x0E },
+	{ CS42L42_SP_TX_ISOC_CTL,		0x04 },
+	{ CS42L42_SP_TX_FS,			0xCC },
+	{ CS42L42_SPDIF_SW_CTL1,		0x3F },
+	{ CS42L42_SRC_SDIN_FS,			0x40 },
+	{ CS42L42_SRC_SDOUT_FS,			0x40 },
+	{ CS42L42_SPDIF_CTL1,			0x01 },
+	{ CS42L42_SPDIF_CTL2,			0x00 },
+	{ CS42L42_SPDIF_CTL3,			0x00 },
+	{ CS42L42_SPDIF_CTL4,			0x42 },
+	{ CS42L42_ASP_TX_SZ_EN,			0x00 },
+	{ CS42L42_ASP_TX_CH_EN,			0x00 },
+	{ CS42L42_ASP_TX_CH_AP_RES,		0x0F },
+	{ CS42L42_ASP_TX_CH1_BIT_MSB,		0x00 },
+	{ CS42L42_ASP_TX_CH1_BIT_LSB,		0x00 },
+	{ CS42L42_ASP_TX_HIZ_DLY_CFG,		0x00 },
+	{ CS42L42_ASP_TX_CH2_BIT_MSB,		0x00 },
+	{ CS42L42_ASP_TX_CH2_BIT_LSB,		0x00 },
+	{ CS42L42_ASP_RX_DAI0_EN,		0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH1_AP_RES,	0x03 },
+	{ CS42L42_ASP_RX_DAI0_CH1_BIT_MSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH1_BIT_LSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH2_AP_RES,	0x03 },
+	{ CS42L42_ASP_RX_DAI0_CH2_BIT_MSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH2_BIT_LSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH3_AP_RES,	0x03 },
+	{ CS42L42_ASP_RX_DAI0_CH3_BIT_MSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH3_BIT_LSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH4_AP_RES,	0x03 },
+	{ CS42L42_ASP_RX_DAI0_CH4_BIT_MSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH4_BIT_LSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI1_CH1_AP_RES,	0x03 },
+	{ CS42L42_ASP_RX_DAI1_CH1_BIT_MSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI1_CH1_BIT_LSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI1_CH2_AP_RES,	0x03 },
+	{ CS42L42_ASP_RX_DAI1_CH2_BIT_MSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI1_CH2_BIT_LSB,	0x00 },
+	{ CS42L42_SUB_REVID,			0x03 },
+};
+
+static bool cs42l42_readable_register(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case CS42L42_PAGE_REGISTER:
+	case CS42L42_DEVID_AB:
+	case CS42L42_DEVID_CD:
+	case CS42L42_DEVID_E:
+	case CS42L42_FABID:
+	case CS42L42_REVID:
+	case CS42L42_FRZ_CTL:
+	case CS42L42_SRC_CTL:
+	case CS42L42_MCLK_STATUS:
+	case CS42L42_MCLK_CTL:
+	case CS42L42_SFTRAMP_RATE:
+	case CS42L42_I2C_DEBOUNCE:
+	case CS42L42_I2C_STRETCH:
+	case CS42L42_I2C_TIMEOUT:
+	case CS42L42_PWR_CTL1:
+	case CS42L42_PWR_CTL2:
+	case CS42L42_PWR_CTL3:
+	case CS42L42_RSENSE_CTL1:
+	case CS42L42_RSENSE_CTL2:
+	case CS42L42_OSC_SWITCH:
+	case CS42L42_OSC_SWITCH_STATUS:
+	case CS42L42_RSENSE_CTL3:
+	case CS42L42_TSENSE_CTL:
+	case CS42L42_TSRS_INT_DISABLE:
+	case CS42L42_TRSENSE_STATUS:
+	case CS42L42_HSDET_CTL1:
+	case CS42L42_HSDET_CTL2:
+	case CS42L42_HS_SWITCH_CTL:
+	case CS42L42_HS_DET_STATUS:
+	case CS42L42_HS_CLAMP_DISABLE:
+	case CS42L42_MCLK_SRC_SEL:
+	case CS42L42_SPDIF_CLK_CFG:
+	case CS42L42_FSYNC_PW_LOWER:
+	case CS42L42_FSYNC_PW_UPPER:
+	case CS42L42_FSYNC_P_LOWER:
+	case CS42L42_FSYNC_P_UPPER:
+	case CS42L42_ASP_CLK_CFG:
+	case CS42L42_ASP_FRM_CFG:
+	case CS42L42_FS_RATE_EN:
+	case CS42L42_IN_ASRC_CLK:
+	case CS42L42_OUT_ASRC_CLK:
+	case CS42L42_PLL_DIV_CFG1:
+	case CS42L42_ADC_OVFL_STATUS:
+	case CS42L42_MIXER_STATUS:
+	case CS42L42_SRC_STATUS:
+	case CS42L42_ASP_RX_STATUS:
+	case CS42L42_ASP_TX_STATUS:
+	case CS42L42_CODEC_STATUS:
+	case CS42L42_DET_INT_STATUS1:
+	case CS42L42_DET_INT_STATUS2:
+	case CS42L42_SRCPL_INT_STATUS:
+	case CS42L42_VPMON_STATUS:
+	case CS42L42_PLL_LOCK_STATUS:
+	case CS42L42_TSRS_PLUG_STATUS:
+	case CS42L42_ADC_OVFL_INT_MASK:
+	case CS42L42_MIXER_INT_MASK:
+	case CS42L42_SRC_INT_MASK:
+	case CS42L42_ASP_RX_INT_MASK:
+	case CS42L42_ASP_TX_INT_MASK:
+	case CS42L42_CODEC_INT_MASK:
+	case CS42L42_SRCPL_INT_MASK:
+	case CS42L42_VPMON_INT_MASK:
+	case CS42L42_PLL_LOCK_INT_MASK:
+	case CS42L42_TSRS_PLUG_INT_MASK:
+	case CS42L42_PLL_CTL1:
+	case CS42L42_PLL_DIV_FRAC0:
+	case CS42L42_PLL_DIV_FRAC1:
+	case CS42L42_PLL_DIV_FRAC2:
+	case CS42L42_PLL_DIV_INT:
+	case CS42L42_PLL_CTL3:
+	case CS42L42_PLL_CAL_RATIO:
+	case CS42L42_PLL_CTL4:
+	case CS42L42_LOAD_DET_RCSTAT:
+	case CS42L42_LOAD_DET_DONE:
+	case CS42L42_LOAD_DET_EN:
+	case CS42L42_HSBIAS_SC_AUTOCTL:
+	case CS42L42_WAKE_CTL:
+	case CS42L42_ADC_DISABLE_MUTE:
+	case CS42L42_TIPSENSE_CTL:
+	case CS42L42_MISC_DET_CTL:
+	case CS42L42_MIC_DET_CTL1:
+	case CS42L42_MIC_DET_CTL2:
+	case CS42L42_DET_STATUS1:
+	case CS42L42_DET_STATUS2:
+	case CS42L42_DET_INT1_MASK:
+	case CS42L42_DET_INT2_MASK:
+	case CS42L42_HS_BIAS_CTL:
+	case CS42L42_ADC_CTL:
+	case CS42L42_ADC_VOLUME:
+	case CS42L42_ADC_WNF_HPF_CTL:
+	case CS42L42_DAC_CTL1:
+	case CS42L42_DAC_CTL2:
+	case CS42L42_HP_CTL:
+	case CS42L42_CLASSH_CTL:
+	case CS42L42_MIXER_CHA_VOL:
+	case CS42L42_MIXER_ADC_VOL:
+	case CS42L42_MIXER_CHB_VOL:
+	case CS42L42_EQ_COEF_IN0:
+	case CS42L42_EQ_COEF_IN1:
+	case CS42L42_EQ_COEF_IN2:
+	case CS42L42_EQ_COEF_IN3:
+	case CS42L42_EQ_COEF_RW:
+	case CS42L42_EQ_COEF_OUT0:
+	case CS42L42_EQ_COEF_OUT1:
+	case CS42L42_EQ_COEF_OUT2:
+	case CS42L42_EQ_COEF_OUT3:
+	case CS42L42_EQ_INIT_STAT:
+	case CS42L42_EQ_START_FILT:
+	case CS42L42_EQ_MUTE_CTL:
+	case CS42L42_SP_RX_CH_SEL:
+	case CS42L42_SP_RX_ISOC_CTL:
+	case CS42L42_SP_RX_FS:
+	case CS42l42_SPDIF_CH_SEL:
+	case CS42L42_SP_TX_ISOC_CTL:
+	case CS42L42_SP_TX_FS:
+	case CS42L42_SPDIF_SW_CTL1:
+	case CS42L42_SRC_SDIN_FS:
+	case CS42L42_SRC_SDOUT_FS:
+	case CS42L42_SPDIF_CTL1:
+	case CS42L42_SPDIF_CTL2:
+	case CS42L42_SPDIF_CTL3:
+	case CS42L42_SPDIF_CTL4:
+	case CS42L42_ASP_TX_SZ_EN:
+	case CS42L42_ASP_TX_CH_EN:
+	case CS42L42_ASP_TX_CH_AP_RES:
+	case CS42L42_ASP_TX_CH1_BIT_MSB:
+	case CS42L42_ASP_TX_CH1_BIT_LSB:
+	case CS42L42_ASP_TX_HIZ_DLY_CFG:
+	case CS42L42_ASP_TX_CH2_BIT_MSB:
+	case CS42L42_ASP_TX_CH2_BIT_LSB:
+	case CS42L42_ASP_RX_DAI0_EN:
+	case CS42L42_ASP_RX_DAI0_CH1_AP_RES:
+	case CS42L42_ASP_RX_DAI0_CH1_BIT_MSB:
+	case CS42L42_ASP_RX_DAI0_CH1_BIT_LSB:
+	case CS42L42_ASP_RX_DAI0_CH2_AP_RES:
+	case CS42L42_ASP_RX_DAI0_CH2_BIT_MSB:
+	case CS42L42_ASP_RX_DAI0_CH2_BIT_LSB:
+	case CS42L42_ASP_RX_DAI0_CH3_AP_RES:
+	case CS42L42_ASP_RX_DAI0_CH3_BIT_MSB:
+	case CS42L42_ASP_RX_DAI0_CH3_BIT_LSB:
+	case CS42L42_ASP_RX_DAI0_CH4_AP_RES:
+	case CS42L42_ASP_RX_DAI0_CH4_BIT_MSB:
+	case CS42L42_ASP_RX_DAI0_CH4_BIT_LSB:
+	case CS42L42_ASP_RX_DAI1_CH1_AP_RES:
+	case CS42L42_ASP_RX_DAI1_CH1_BIT_MSB:
+	case CS42L42_ASP_RX_DAI1_CH1_BIT_LSB:
+	case CS42L42_ASP_RX_DAI1_CH2_AP_RES:
+	case CS42L42_ASP_RX_DAI1_CH2_BIT_MSB:
+	case CS42L42_ASP_RX_DAI1_CH2_BIT_LSB:
+	case CS42L42_SUB_REVID:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool cs42l42_volatile_register(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case CS42L42_DEVID_AB:
+	case CS42L42_DEVID_CD:
+	case CS42L42_DEVID_E:
+	case CS42L42_MCLK_STATUS:
+	case CS42L42_TRSENSE_STATUS:
+	case CS42L42_HS_DET_STATUS:
+	case CS42L42_ADC_OVFL_STATUS:
+	case CS42L42_MIXER_STATUS:
+	case CS42L42_SRC_STATUS:
+	case CS42L42_ASP_RX_STATUS:
+	case CS42L42_ASP_TX_STATUS:
+	case CS42L42_CODEC_STATUS:
+	case CS42L42_DET_INT_STATUS1:
+	case CS42L42_DET_INT_STATUS2:
+	case CS42L42_SRCPL_INT_STATUS:
+	case CS42L42_VPMON_STATUS:
+	case CS42L42_PLL_LOCK_STATUS:
+	case CS42L42_TSRS_PLUG_STATUS:
+	case CS42L42_LOAD_DET_RCSTAT:
+	case CS42L42_LOAD_DET_DONE:
+	case CS42L42_DET_STATUS1:
+	case CS42L42_DET_STATUS2:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct regmap_range_cfg cs42l42_page_range = {
+	.name = "Pages",
+	.range_min = 0,
+	.range_max = CS42L42_MAX_REGISTER,
+	.selector_reg = CS42L42_PAGE_REGISTER,
+	.selector_mask = 0xff,
+	.selector_shift = 0,
+	.window_start = 0,
+	.window_len = 256,
+};
+
+static const struct regmap_config cs42l42_regmap = {
+	.reg_bits = 8,
+	.val_bits = 8,
+
+	.readable_reg = cs42l42_readable_register,
+	.volatile_reg = cs42l42_volatile_register,
+
+	.ranges = &cs42l42_page_range,
+	.num_ranges = 1,
+
+	.max_register = CS42L42_MAX_REGISTER,
+	.reg_defaults = cs42l42_reg_defaults,
+	.num_reg_defaults = ARRAY_SIZE(cs42l42_reg_defaults),
+	.cache_type = REGCACHE_RBTREE,
+};
+
+static DECLARE_TLV_DB_SCALE(adc_tlv, -9600, 100, false);
+static DECLARE_TLV_DB_SCALE(mixer_tlv, -6200, 100, false);
+
+static const char * const cs42l42_hpf_freq_text[] = {
+	"1.86Hz", "120Hz", "235Hz", "466Hz"
+};
+
+static SOC_ENUM_SINGLE_DECL(cs42l42_hpf_freq_enum, CS42L42_ADC_WNF_HPF_CTL,
+			    CS42L42_ADC_HPF_CF_SHIFT,
+			    cs42l42_hpf_freq_text);
+
+static const char * const cs42l42_wnf3_freq_text[] = {
+	"160Hz", "180Hz", "200Hz", "220Hz",
+	"240Hz", "260Hz", "280Hz", "300Hz"
+};
+
+static SOC_ENUM_SINGLE_DECL(cs42l42_wnf3_freq_enum, CS42L42_ADC_WNF_HPF_CTL,
+			    CS42L42_ADC_WNF_CF_SHIFT,
+			    cs42l42_wnf3_freq_text);
+
+static const char * const cs42l42_wnf05_freq_text[] = {
+	"280Hz", "315Hz", "350Hz", "385Hz",
+	"420Hz", "455Hz", "490Hz", "525Hz"
+};
+
+static SOC_ENUM_SINGLE_DECL(cs42l42_wnf05_freq_enum, CS42L42_ADC_WNF_HPF_CTL,
+			    CS42L42_ADC_WNF_CF_SHIFT,
+			    cs42l42_wnf05_freq_text);
+
+static const struct snd_kcontrol_new cs42l42_snd_controls[] = {
+	/* ADC Volume and Filter Controls */
+	SOC_SINGLE("ADC Notch Switch", CS42L42_ADC_CTL,
+				CS42L42_ADC_NOTCH_DIS_SHIFT, true, false),
+	SOC_SINGLE("ADC Weak Force Switch", CS42L42_ADC_CTL,
+				CS42L42_ADC_FORCE_WEAK_VCM_SHIFT, true, false),
+	SOC_SINGLE("ADC Invert Switch", CS42L42_ADC_CTL,
+				CS42L42_ADC_INV_SHIFT, true, false),
+	SOC_SINGLE("ADC Boost Switch", CS42L42_ADC_CTL,
+				CS42L42_ADC_DIG_BOOST_SHIFT, true, false),
+	SOC_SINGLE_SX_TLV("ADC Volume", CS42L42_ADC_VOLUME,
+				CS42L42_ADC_VOL_SHIFT, 0xA0, 0x6C, adc_tlv),
+	SOC_SINGLE("ADC WNF Switch", CS42L42_ADC_WNF_HPF_CTL,
+				CS42L42_ADC_WNF_EN_SHIFT, true, false),
+	SOC_SINGLE("ADC HPF Switch", CS42L42_ADC_WNF_HPF_CTL,
+				CS42L42_ADC_HPF_EN_SHIFT, true, false),
+	SOC_ENUM("HPF Corner Freq", cs42l42_hpf_freq_enum),
+	SOC_ENUM("WNF 3dB Freq", cs42l42_wnf3_freq_enum),
+	SOC_ENUM("WNF 05dB Freq", cs42l42_wnf05_freq_enum),
+
+	/* DAC Volume and Filter Controls */
+	SOC_SINGLE("DACA Invert Switch", CS42L42_DAC_CTL1,
+				CS42L42_DACA_INV_SHIFT, true, false),
+	SOC_SINGLE("DACB Invert Switch", CS42L42_DAC_CTL1,
+				CS42L42_DACB_INV_SHIFT, true, false),
+	SOC_SINGLE("DAC HPF Switch", CS42L42_DAC_CTL2,
+				CS42L42_DAC_HPF_EN_SHIFT, true, false),
+	SOC_DOUBLE_R_TLV("Mixer Volume", CS42L42_MIXER_CHA_VOL,
+			 CS42L42_MIXER_CHB_VOL, CS42L42_MIXER_CH_VOL_SHIFT,
+				0x3e, 1, mixer_tlv)
+};
+
+static int cs42l42_hpdrv_evt(struct snd_soc_dapm_widget *w,
+				struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+
+	if (event & SND_SOC_DAPM_POST_PMU) {
+		/* Enable the channels */
+		snd_soc_update_bits(codec, CS42L42_ASP_RX_DAI0_EN,
+				CS42L42_ASP_RX0_CH_EN_MASK,
+				(CS42L42_ASP_RX0_CH1_EN |
+				CS42L42_ASP_RX0_CH2_EN) <<
+				CS42L42_ASP_RX0_CH_EN_SHIFT);
+
+		/* Power up */
+		snd_soc_update_bits(codec, CS42L42_PWR_CTL1,
+			CS42L42_ASP_DAI_PDN_MASK | CS42L42_MIXER_PDN_MASK |
+				CS42L42_HP_PDN_MASK, 0);
+	} else if (event & SND_SOC_DAPM_PRE_PMD) {
+		/* Disable the channels */
+		snd_soc_update_bits(codec, CS42L42_ASP_RX_DAI0_EN,
+				CS42L42_ASP_RX0_CH_EN_MASK, 0);
+
+		/* Power down */
+		snd_soc_update_bits(codec, CS42L42_PWR_CTL1,
+			CS42L42_ASP_DAI_PDN_MASK | CS42L42_MIXER_PDN_MASK |
+				CS42L42_HP_PDN_MASK,
+			CS42L42_ASP_DAI_PDN_MASK | CS42L42_MIXER_PDN_MASK |
+				CS42L42_HP_PDN_MASK);
+	} else {
+		dev_err(codec->dev, "Invalid event 0x%x\n", event);
+	}
+	return 0;
+}
+
+static const struct snd_soc_dapm_widget cs42l42_dapm_widgets[] = {
+	SND_SOC_DAPM_OUTPUT("HP"),
+	SND_SOC_DAPM_AIF_IN("SDIN", NULL, 0, CS42L42_ASP_CLK_CFG,
+					CS42L42_ASP_SCLK_EN_SHIFT, false),
+	SND_SOC_DAPM_OUT_DRV_E("HPDRV", SND_SOC_NOPM, 0,
+					0, NULL, 0, cs42l42_hpdrv_evt,
+					SND_SOC_DAPM_POST_PMU |
+					SND_SOC_DAPM_PRE_PMD)
+};
+
+static const struct snd_soc_dapm_route cs42l42_audio_map[] = {
+	{"SDIN", NULL, "Playback"},
+	{"HPDRV", NULL, "SDIN"},
+	{"HP", NULL, "HPDRV"}
+};
+
+static int cs42l42_set_bias_level(struct snd_soc_codec *codec,
+					enum snd_soc_bias_level level)
+{
+	struct cs42l42_private *cs42l42 = snd_soc_codec_get_drvdata(codec);
+	int ret;
+
+	switch (level) {
+	case SND_SOC_BIAS_ON:
+		break;
+	case SND_SOC_BIAS_PREPARE:
+		break;
+	case SND_SOC_BIAS_STANDBY:
+		if (snd_soc_codec_get_bias_level(codec) == SND_SOC_BIAS_OFF) {
+			regcache_cache_only(cs42l42->regmap, false);
+			regcache_sync(cs42l42->regmap);
+			ret = regulator_bulk_enable(
+						ARRAY_SIZE(cs42l42->supplies),
+						cs42l42->supplies);
+			if (ret != 0) {
+				dev_err(codec->dev,
+					"Failed to enable regulators: %d\n",
+					ret);
+				return ret;
+			}
+		}
+		break;
+	case SND_SOC_BIAS_OFF:
+
+		regcache_cache_only(cs42l42->regmap, true);
+		regulator_bulk_disable(ARRAY_SIZE(cs42l42->supplies),
+						    cs42l42->supplies);
+		break;
+	}
+
+	return 0;
+}
+
+static int cs42l42_codec_probe(struct snd_soc_codec *codec)
+{
+	struct cs42l42_private *cs42l42 =
+		(struct cs42l42_private *)snd_soc_codec_get_drvdata(codec);
+
+	cs42l42->codec = codec;
+
+	return 0;
+}
+
+static const struct snd_soc_codec_driver soc_codec_dev_cs42l42 = {
+	.probe = cs42l42_codec_probe,
+	.set_bias_level = cs42l42_set_bias_level,
+	.ignore_pmdown_time = true,
+
+	.component_driver = {
+		.dapm_widgets = cs42l42_dapm_widgets,
+		.num_dapm_widgets = ARRAY_SIZE(cs42l42_dapm_widgets),
+		.dapm_routes = cs42l42_audio_map,
+		.num_dapm_routes = ARRAY_SIZE(cs42l42_audio_map),
+
+		.controls = cs42l42_snd_controls,
+		.num_controls = ARRAY_SIZE(cs42l42_snd_controls),
+	},
+};
+
+struct cs42l42_pll_params {
+	u32 sclk;
+	u8 mclk_div;
+	u8 mclk_src_sel;
+	u8 sclk_prediv;
+	u8 pll_div_int;
+	u32 pll_div_frac;
+	u8 pll_mode;
+	u8 pll_divout;
+	u32 mclk_int;
+	u8 pll_cal_ratio;
+};
+
+/*
+ * Common PLL Settings for given SCLK
+ * Table 4-5 from the Datasheet
+ */
+static const struct cs42l42_pll_params pll_ratio_table[] = {
+	{ 1536000, 0, 1, 0x00, 0x7D, 0x000000, 0x03, 0x10, 12000000, 125 },
+	{ 2822400, 0, 1, 0x00, 0x40, 0x000000, 0x03, 0x10, 11289600, 128 },
+	{ 3000000, 0, 1, 0x00, 0x40, 0x000000, 0x03, 0x10, 12000000, 128 },
+	{ 3072000, 0, 1, 0x00, 0x3E, 0x800000, 0x03, 0x10, 12000000, 125 },
+	{ 4000000, 0, 1, 0x00, 0x30, 0x800000, 0x03, 0x10, 12000000, 96 },
+	{ 4096000, 0, 1, 0x00, 0x2E, 0xE00000, 0x03, 0x10, 12000000, 94 },
+	{ 5644800, 0, 1, 0x01, 0x40, 0x000000, 0x03, 0x10, 11289600, 128 },
+	{ 6000000, 0, 1, 0x01, 0x40, 0x000000, 0x03, 0x10, 12000000, 128 },
+	{ 6144000, 0, 1, 0x01, 0x3E, 0x800000, 0x03, 0x10, 12000000, 125 },
+	{ 11289600, 0, 0, 0, 0, 0, 0, 0, 11289600, 0 },
+	{ 12000000, 0, 0, 0, 0, 0, 0, 0, 12000000, 0 },
+	{ 12288000, 0, 0, 0, 0, 0, 0, 0, 12288000, 0 },
+	{ 22579200, 1, 0, 0, 0, 0, 0, 0, 22579200, 0 },
+	{ 24000000, 1, 0, 0, 0, 0, 0, 0, 24000000, 0 },
+	{ 24576000, 1, 0, 0, 0, 0, 0, 0, 24576000, 0 }
+};
+
+static int cs42l42_pll_config(struct snd_soc_codec *codec)
+{
+	struct cs42l42_private *cs42l42 = snd_soc_codec_get_drvdata(codec);
+	int i;
+	u32 fsync;
+
+	for (i = 0; i < ARRAY_SIZE(pll_ratio_table); i++) {
+		if (pll_ratio_table[i].sclk == cs42l42->sclk) {
+			/* Configure the internal sample rate */
+			snd_soc_update_bits(codec, CS42L42_MCLK_CTL,
+					CS42L42_INTERNAL_FS_MASK,
+					((pll_ratio_table[i].mclk_int !=
+					12000000) &&
+					(pll_ratio_table[i].mclk_int !=
+					24000000)) <<
+					CS42L42_INTERNAL_FS_SHIFT);
+			/* Set the MCLK src (PLL or SCLK) and the divide
+			 * ratio
+			 */
+			snd_soc_update_bits(codec, CS42L42_MCLK_SRC_SEL,
+					CS42L42_MCLK_SRC_SEL_MASK |
+					CS42L42_MCLKDIV_MASK,
+					(pll_ratio_table[i].mclk_src_sel
+					<< CS42L42_MCLK_SRC_SEL_SHIFT) |
+					(pll_ratio_table[i].mclk_div <<
+					CS42L42_MCLKDIV_SHIFT));
+			/* Set up the LRCLK */
+			fsync = cs42l42->sclk / cs42l42->srate;
+			if (((fsync * cs42l42->srate) != cs42l42->sclk)
+				|| ((fsync % 2) != 0)) {
+				dev_err(codec->dev,
+					"Unsupported sclk %d/sample rate %d\n",
+					cs42l42->sclk,
+					cs42l42->srate);
+				return -EINVAL;
+			}
+			/* Set the LRCLK period */
+			snd_soc_update_bits(codec,
+					CS42L42_FSYNC_P_LOWER,
+					CS42L42_FSYNC_PERIOD_MASK,
+					CS42L42_FRAC0_VAL(fsync - 1) <<
+					CS42L42_FSYNC_PERIOD_SHIFT);
+			snd_soc_update_bits(codec,
+					CS42L42_FSYNC_P_UPPER,
+					CS42L42_FSYNC_PERIOD_MASK,
+					CS42L42_FRAC1_VAL(fsync - 1) <<
+					CS42L42_FSYNC_PERIOD_SHIFT);
+			/* Set the LRCLK to 50% duty cycle */
+			fsync = fsync / 2;
+			snd_soc_update_bits(codec,
+					CS42L42_FSYNC_PW_LOWER,
+					CS42L42_FSYNC_PULSE_WIDTH_MASK,
+					CS42L42_FRAC0_VAL(fsync - 1) <<
+					CS42L42_FSYNC_PULSE_WIDTH_SHIFT);
+			snd_soc_update_bits(codec,
+					CS42L42_FSYNC_PW_UPPER,
+					CS42L42_FSYNC_PULSE_WIDTH_MASK,
+					CS42L42_FRAC1_VAL(fsync - 1) <<
+					CS42L42_FSYNC_PULSE_WIDTH_SHIFT);
+			snd_soc_update_bits(codec,
+					CS42L42_ASP_FRM_CFG,
+					CS42L42_ASP_5050_MASK,
+					CS42L42_ASP_5050_MASK);
+			/* Set the frame delay to 1.0 SCLK clocks */
+			snd_soc_update_bits(codec, CS42L42_ASP_FRM_CFG,
+					CS42L42_ASP_FSD_MASK,
+					CS42L42_ASP_FSD_1_0 <<
+					CS42L42_ASP_FSD_SHIFT);
+			/* Set the sample rates (96k or lower) */
+			snd_soc_update_bits(codec, CS42L42_FS_RATE_EN,
+					CS42L42_FS_EN_MASK,
+					(CS42L42_FS_EN_IASRC_96K |
+					CS42L42_FS_EN_OASRC_96K) <<
+					CS42L42_FS_EN_SHIFT);
+			/* Set the input/output internal MCLK clock ~12 MHz */
+			snd_soc_update_bits(codec, CS42L42_IN_ASRC_CLK,
+					CS42L42_CLK_IASRC_SEL_MASK,
+					CS42L42_CLK_IASRC_SEL_12 <<
+					CS42L42_CLK_IASRC_SEL_SHIFT);
+			snd_soc_update_bits(codec,
+					CS42L42_OUT_ASRC_CLK,
+					CS42L42_CLK_OASRC_SEL_MASK,
+					CS42L42_CLK_OASRC_SEL_12 <<
+					CS42L42_CLK_OASRC_SEL_SHIFT);
+			/* channel 1 on low LRCLK, 32 bit */
+			snd_soc_update_bits(codec,
+					CS42L42_ASP_RX_DAI0_CH1_AP_RES,
+					CS42L42_ASP_RX_CH_AP_MASK |
+					CS42L42_ASP_RX_CH_RES_MASK,
+					(CS42L42_ASP_RX_CH_AP_LOW <<
+					CS42L42_ASP_RX_CH_AP_SHIFT) |
+					(CS42L42_ASP_RX_CH_RES_32 <<
+					CS42L42_ASP_RX_CH_RES_SHIFT));
+			/* Channel 2 on high LRCLK, 32 bit */
+			snd_soc_update_bits(codec,
+					CS42L42_ASP_RX_DAI0_CH2_AP_RES,
+					CS42L42_ASP_RX_CH_AP_MASK |
+					CS42L42_ASP_RX_CH_RES_MASK,
+					(CS42L42_ASP_RX_CH_AP_HI <<
+					CS42L42_ASP_RX_CH_AP_SHIFT) |
+					(CS42L42_ASP_RX_CH_RES_32 <<
+					CS42L42_ASP_RX_CH_RES_SHIFT));
+			if (pll_ratio_table[i].mclk_src_sel == 0) {
+				/* Pass the clock straight through */
+				snd_soc_update_bits(codec,
+					CS42L42_PLL_CTL1,
+					CS42L42_PLL_START_MASK,	0);
+			} else {
+				/* Configure PLL per table 4-5 */
+				snd_soc_update_bits(codec,
+					CS42L42_PLL_DIV_CFG1,
+					CS42L42_SCLK_PREDIV_MASK,
+					pll_ratio_table[i].sclk_prediv
+					<< CS42L42_SCLK_PREDIV_SHIFT);
+				snd_soc_update_bits(codec,
+					CS42L42_PLL_DIV_INT,
+					CS42L42_PLL_DIV_INT_MASK,
+					pll_ratio_table[i].pll_div_int
+					<< CS42L42_PLL_DIV_INT_SHIFT);
+				snd_soc_update_bits(codec,
+					CS42L42_PLL_DIV_FRAC0,
+					CS42L42_PLL_DIV_FRAC_MASK,
+					CS42L42_FRAC0_VAL(
+					pll_ratio_table[i].pll_div_frac)
+					<< CS42L42_PLL_DIV_FRAC_SHIFT);
+				snd_soc_update_bits(codec,
+					CS42L42_PLL_DIV_FRAC1,
+					CS42L42_PLL_DIV_FRAC_MASK,
+					CS42L42_FRAC1_VAL(
+					pll_ratio_table[i].pll_div_frac)
+					<< CS42L42_PLL_DIV_FRAC_SHIFT);
+				snd_soc_update_bits(codec,
+					CS42L42_PLL_DIV_FRAC2,
+					CS42L42_PLL_DIV_FRAC_MASK,
+					CS42L42_FRAC2_VAL(
+					pll_ratio_table[i].pll_div_frac)
+					<< CS42L42_PLL_DIV_FRAC_SHIFT);
+				snd_soc_update_bits(codec,
+					CS42L42_PLL_CTL4,
+					CS42L42_PLL_MODE_MASK,
+					pll_ratio_table[i].pll_mode
+					<< CS42L42_PLL_MODE_SHIFT);
+				snd_soc_update_bits(codec,
+					CS42L42_PLL_CTL3,
+					CS42L42_PLL_DIVOUT_MASK,
+					pll_ratio_table[i].pll_divout
+					<< CS42L42_PLL_DIVOUT_SHIFT);
+				snd_soc_update_bits(codec,
+					CS42L42_PLL_CAL_RATIO,
+					CS42L42_PLL_CAL_RATIO_MASK,
+					pll_ratio_table[i].pll_cal_ratio
+					<< CS42L42_PLL_CAL_RATIO_SHIFT);
+			}
+			return 0;
+		}
+	}
+
+	return -EINVAL;
+}
+
+static int cs42l42_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
+{
+	struct snd_soc_codec *codec = codec_dai->codec;
+	u32 asp_cfg_val = 0;
+
+	switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) {
+	case SND_SOC_DAIFMT_CBS_CFM:
+		asp_cfg_val |= CS42L42_ASP_MASTER_MODE <<
+				CS42L42_ASP_MODE_SHIFT;
+		break;
+	case SND_SOC_DAIFMT_CBS_CFS:
+		asp_cfg_val |= CS42L42_ASP_SLAVE_MODE <<
+				CS42L42_ASP_MODE_SHIFT;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* interface format */
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+	case SND_SOC_DAIFMT_LEFT_J:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Bitclock/frame inversion */
+	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
+	case SND_SOC_DAIFMT_NB_NF:
+		break;
+	case SND_SOC_DAIFMT_NB_IF:
+		asp_cfg_val |= CS42L42_ASP_POL_INV <<
+				CS42L42_ASP_LCPOL_IN_SHIFT;
+		break;
+	case SND_SOC_DAIFMT_IB_NF:
+		asp_cfg_val |= CS42L42_ASP_POL_INV <<
+				CS42L42_ASP_SCPOL_IN_DAC_SHIFT;
+		break;
+	case SND_SOC_DAIFMT_IB_IF:
+		asp_cfg_val |= CS42L42_ASP_POL_INV <<
+				CS42L42_ASP_LCPOL_IN_SHIFT;
+		asp_cfg_val |= CS42L42_ASP_POL_INV <<
+				CS42L42_ASP_SCPOL_IN_DAC_SHIFT;
+		break;
+	}
+
+	snd_soc_update_bits(codec, CS42L42_ASP_CLK_CFG,
+				CS42L42_ASP_MODE_MASK |
+				CS42L42_ASP_SCPOL_IN_DAC_MASK |
+				CS42L42_ASP_LCPOL_IN_MASK, asp_cfg_val);
+
+	return 0;
+}
+
+static int cs42l42_pcm_hw_params(struct snd_pcm_substream *substream,
+				struct snd_pcm_hw_params *params,
+				struct snd_soc_dai *dai)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	struct cs42l42_private *cs42l42 = snd_soc_codec_get_drvdata(codec);
+	int retval;
+
+	cs42l42->srate = params_rate(params);
+	cs42l42->swidth = params_width(params);
+
+	retval = cs42l42_pll_config(codec);
+
+	return retval;
+}
+
+static int cs42l42_set_sysclk(struct snd_soc_dai *dai,
+				int clk_id, unsigned int freq, int dir)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	struct cs42l42_private *cs42l42 = snd_soc_codec_get_drvdata(codec);
+
+	cs42l42->sclk = freq;
+
+	return 0;
+}
+
+static int cs42l42_digital_mute(struct snd_soc_dai *dai, int mute)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	unsigned int regval;
+	u8 fullScaleVol;
+
+	if (mute) {
+		/* Mark SCLK as not present to turn on the internal
+		 * oscillator.
+		 */
+		snd_soc_update_bits(codec, CS42L42_OSC_SWITCH,
+						CS42L42_SCLK_PRESENT_MASK, 0);
+
+		snd_soc_update_bits(codec, CS42L42_PLL_CTL1,
+				CS42L42_PLL_START_MASK,
+				0 << CS42L42_PLL_START_SHIFT);
+
+		/* Mute the headphone */
+		snd_soc_update_bits(codec, CS42L42_HP_CTL,
+				CS42L42_HP_ANA_AMUTE_MASK |
+				CS42L42_HP_ANA_BMUTE_MASK,
+				CS42L42_HP_ANA_AMUTE_MASK |
+				CS42L42_HP_ANA_BMUTE_MASK);
+	} else {
+		snd_soc_update_bits(codec, CS42L42_PLL_CTL1,
+				CS42L42_PLL_START_MASK,
+				1 << CS42L42_PLL_START_SHIFT);
+		/* Read the headphone load */
+		regval = snd_soc_read(codec, CS42L42_LOAD_DET_RCSTAT);
+		if (((regval & CS42L42_RLA_STAT_MASK) >>
+			CS42L42_RLA_STAT_SHIFT) == CS42L42_RLA_STAT_15_OHM) {
+			fullScaleVol = CS42L42_HP_FULL_SCALE_VOL_MASK;
+		} else {
+			fullScaleVol = 0;
+		}
+
+		/* Un-mute the headphone, set the full scale volume flag */
+		snd_soc_update_bits(codec, CS42L42_HP_CTL,
+				CS42L42_HP_ANA_AMUTE_MASK |
+				CS42L42_HP_ANA_BMUTE_MASK |
+				CS42L42_HP_FULL_SCALE_VOL_MASK, fullScaleVol);
+
+		/* Mark SCLK as present, turn off internal oscillator */
+		snd_soc_update_bits(codec, CS42L42_OSC_SWITCH,
+				CS42L42_SCLK_PRESENT_MASK,
+				CS42L42_SCLK_PRESENT_MASK);
+	}
+
+	return 0;
+}
+
+#define CS42L42_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S18_3LE | \
+			SNDRV_PCM_FMTBIT_S20_3LE | SNDRV_PCM_FMTBIT_S24_LE | \
+			SNDRV_PCM_FMTBIT_S32_LE)
+
+
+static struct snd_soc_dai_ops cs42l42_ops = {
+	.hw_params	= cs42l42_pcm_hw_params,
+	.set_fmt	= cs42l42_set_dai_fmt,
+	.set_sysclk	= cs42l42_set_sysclk,
+	.digital_mute = cs42l42_digital_mute
+};
+
+static struct snd_soc_dai_driver cs42l42_dai = {
+		.name = "cs42l42",
+		.playback = {
+			.stream_name = "Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = SNDRV_PCM_RATE_8000_192000,
+			.formats = CS42L42_FORMATS,
+		},
+		.capture = {
+			.stream_name = "Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = SNDRV_PCM_RATE_8000_192000,
+			.formats = CS42L42_FORMATS,
+		},
+		.ops = &cs42l42_ops,
+};
+
+static void cs42l42_process_hs_type_detect(struct cs42l42_private *cs42l42)
+{
+	unsigned int hs_det_status;
+	unsigned int int_status;
+
+	/* Mask the auto detect interrupt */
+	regmap_update_bits(cs42l42->regmap,
+		CS42L42_CODEC_INT_MASK,
+		CS42L42_PDN_DONE_MASK |
+		CS42L42_HSDET_AUTO_DONE_MASK,
+		(1 << CS42L42_PDN_DONE_SHIFT) |
+		(1 << CS42L42_HSDET_AUTO_DONE_SHIFT));
+
+	/* Set hs detect to automatic, disabled mode */
+	regmap_update_bits(cs42l42->regmap,
+		CS42L42_HSDET_CTL2,
+		CS42L42_HSDET_CTRL_MASK |
+		CS42L42_HSDET_SET_MASK |
+		CS42L42_HSBIAS_REF_MASK |
+		CS42L42_HSDET_AUTO_TIME_MASK,
+		(2 << CS42L42_HSDET_CTRL_SHIFT) |
+		(2 << CS42L42_HSDET_SET_SHIFT) |
+		(0 << CS42L42_HSBIAS_REF_SHIFT) |
+		(3 << CS42L42_HSDET_AUTO_TIME_SHIFT));
+
+	/* Read and save the hs detection result */
+	regmap_read(cs42l42->regmap, CS42L42_HS_DET_STATUS, &hs_det_status);
+
+	cs42l42->hs_type = (hs_det_status & CS42L42_HSDET_TYPE_MASK) >>
+				CS42L42_HSDET_TYPE_SHIFT;
+
+	/* Set up button detection */
+	if ((cs42l42->hs_type == CS42L42_PLUG_CTIA) ||
+	      (cs42l42->hs_type == CS42L42_PLUG_OMTP)) {
+		/* Set auto HS bias settings to default */
+		regmap_update_bits(cs42l42->regmap,
+			CS42L42_HSBIAS_SC_AUTOCTL,
+			CS42L42_HSBIAS_SENSE_EN_MASK |
+			CS42L42_AUTO_HSBIAS_HIZ_MASK |
+			CS42L42_TIP_SENSE_EN_MASK |
+			CS42L42_HSBIAS_SENSE_TRIP_MASK,
+			(0 << CS42L42_HSBIAS_SENSE_EN_SHIFT) |
+			(0 << CS42L42_AUTO_HSBIAS_HIZ_SHIFT) |
+			(0 << CS42L42_TIP_SENSE_EN_SHIFT) |
+			(3 << CS42L42_HSBIAS_SENSE_TRIP_SHIFT));
+
+		/* Set up hs detect level sensitivity */
+		regmap_update_bits(cs42l42->regmap,
+			CS42L42_MIC_DET_CTL1,
+			CS42L42_LATCH_TO_VP_MASK |
+			CS42L42_EVENT_STAT_SEL_MASK |
+			CS42L42_HS_DET_LEVEL_MASK,
+			(1 << CS42L42_LATCH_TO_VP_SHIFT) |
+			(0 << CS42L42_EVENT_STAT_SEL_SHIFT) |
+			(cs42l42->bias_thresholds[0] <<
+			CS42L42_HS_DET_LEVEL_SHIFT));
+
+		/* Set auto HS bias settings to default */
+		regmap_update_bits(cs42l42->regmap,
+			CS42L42_HSBIAS_SC_AUTOCTL,
+			CS42L42_HSBIAS_SENSE_EN_MASK |
+			CS42L42_AUTO_HSBIAS_HIZ_MASK |
+			CS42L42_TIP_SENSE_EN_MASK |
+			CS42L42_HSBIAS_SENSE_TRIP_MASK,
+			(1 << CS42L42_HSBIAS_SENSE_EN_SHIFT) |
+			(1 << CS42L42_AUTO_HSBIAS_HIZ_SHIFT) |
+			(0 << CS42L42_TIP_SENSE_EN_SHIFT) |
+			(3 << CS42L42_HSBIAS_SENSE_TRIP_SHIFT));
+
+		/* Turn on level detect circuitry */
+		regmap_update_bits(cs42l42->regmap,
+			CS42L42_MISC_DET_CTL,
+			CS42L42_DETECT_MODE_MASK |
+			CS42L42_HSBIAS_CTL_MASK |
+			CS42L42_PDN_MIC_LVL_DET_MASK,
+			(0 << CS42L42_DETECT_MODE_SHIFT) |
+			(3 << CS42L42_HSBIAS_CTL_SHIFT) |
+			(0 << CS42L42_PDN_MIC_LVL_DET_SHIFT));
+
+		msleep(cs42l42->btn_det_init_dbnce);
+
+		/* Clear any button interrupts before unmasking them */
+		regmap_read(cs42l42->regmap, CS42L42_DET_INT_STATUS2,
+			    &int_status);
+
+		/* Unmask button detect interrupts */
+		regmap_update_bits(cs42l42->regmap,
+			CS42L42_DET_INT2_MASK,
+			CS42L42_M_DETECT_TF_MASK |
+			CS42L42_M_DETECT_FT_MASK |
+			CS42L42_M_HSBIAS_HIZ_MASK |
+			CS42L42_M_SHORT_RLS_MASK |
+			CS42L42_M_SHORT_DET_MASK,
+			(0 << CS42L42_M_DETECT_TF_SHIFT) |
+			(0 << CS42L42_M_DETECT_FT_SHIFT) |
+			(0 << CS42L42_M_HSBIAS_HIZ_SHIFT) |
+			(1 << CS42L42_M_SHORT_RLS_SHIFT) |
+			(1 << CS42L42_M_SHORT_DET_SHIFT));
+	} else {
+		/* Make sure button detect and HS bias circuits are off */
+		regmap_update_bits(cs42l42->regmap,
+			CS42L42_MISC_DET_CTL,
+			CS42L42_DETECT_MODE_MASK |
+			CS42L42_HSBIAS_CTL_MASK |
+			CS42L42_PDN_MIC_LVL_DET_MASK,
+			(0 << CS42L42_DETECT_MODE_SHIFT) |
+			(1 << CS42L42_HSBIAS_CTL_SHIFT) |
+			(1 << CS42L42_PDN_MIC_LVL_DET_SHIFT));
+	}
+
+	regmap_update_bits(cs42l42->regmap,
+				CS42L42_DAC_CTL2,
+				CS42L42_HPOUT_PULLDOWN_MASK |
+				CS42L42_HPOUT_LOAD_MASK |
+				CS42L42_HPOUT_CLAMP_MASK |
+				CS42L42_DAC_HPF_EN_MASK |
+				CS42L42_DAC_MON_EN_MASK,
+				(0 << CS42L42_HPOUT_PULLDOWN_SHIFT) |
+				(0 << CS42L42_HPOUT_LOAD_SHIFT) |
+				(0 << CS42L42_HPOUT_CLAMP_SHIFT) |
+				(1 << CS42L42_DAC_HPF_EN_SHIFT) |
+				(0 << CS42L42_DAC_MON_EN_SHIFT));
+
+	/* Unmask tip sense interrupts */
+	regmap_update_bits(cs42l42->regmap,
+		CS42L42_TSRS_PLUG_INT_MASK,
+		CS42L42_RS_PLUG_MASK |
+		CS42L42_RS_UNPLUG_MASK |
+		CS42L42_TS_PLUG_MASK |
+		CS42L42_TS_UNPLUG_MASK,
+		(1 << CS42L42_RS_PLUG_SHIFT) |
+		(1 << CS42L42_RS_UNPLUG_SHIFT) |
+		(0 << CS42L42_TS_PLUG_SHIFT) |
+		(0 << CS42L42_TS_UNPLUG_SHIFT));
+}
+
+static void cs42l42_init_hs_type_detect(struct cs42l42_private *cs42l42)
+{
+	/* Mask tip sense interrupts */
+	regmap_update_bits(cs42l42->regmap,
+				CS42L42_TSRS_PLUG_INT_MASK,
+				CS42L42_RS_PLUG_MASK |
+				CS42L42_RS_UNPLUG_MASK |
+				CS42L42_TS_PLUG_MASK |
+				CS42L42_TS_UNPLUG_MASK,
+				(1 << CS42L42_RS_PLUG_SHIFT) |
+				(1 << CS42L42_RS_UNPLUG_SHIFT) |
+				(1 << CS42L42_TS_PLUG_SHIFT) |
+				(1 << CS42L42_TS_UNPLUG_SHIFT));
+
+	/* Make sure button detect and HS bias circuits are off */
+	regmap_update_bits(cs42l42->regmap,
+				CS42L42_MISC_DET_CTL,
+				CS42L42_DETECT_MODE_MASK |
+				CS42L42_HSBIAS_CTL_MASK |
+				CS42L42_PDN_MIC_LVL_DET_MASK,
+				(0 << CS42L42_DETECT_MODE_SHIFT) |
+				(1 << CS42L42_HSBIAS_CTL_SHIFT) |
+				(1 << CS42L42_PDN_MIC_LVL_DET_SHIFT));
+
+	/* Set auto HS bias settings to default */
+	regmap_update_bits(cs42l42->regmap,
+				CS42L42_HSBIAS_SC_AUTOCTL,
+				CS42L42_HSBIAS_SENSE_EN_MASK |
+				CS42L42_AUTO_HSBIAS_HIZ_MASK |
+				CS42L42_TIP_SENSE_EN_MASK |
+				CS42L42_HSBIAS_SENSE_TRIP_MASK,
+				(0 << CS42L42_HSBIAS_SENSE_EN_SHIFT) |
+				(0 << CS42L42_AUTO_HSBIAS_HIZ_SHIFT) |
+				(0 << CS42L42_TIP_SENSE_EN_SHIFT) |
+				(3 << CS42L42_HSBIAS_SENSE_TRIP_SHIFT));
+
+	/* Set hs detect to manual, disabled mode */
+	regmap_update_bits(cs42l42->regmap,
+				CS42L42_HSDET_CTL2,
+				CS42L42_HSDET_CTRL_MASK |
+				CS42L42_HSDET_SET_MASK |
+				CS42L42_HSBIAS_REF_MASK |
+				CS42L42_HSDET_AUTO_TIME_MASK,
+				(0 << CS42L42_HSDET_CTRL_SHIFT) |
+				(2 << CS42L42_HSDET_SET_SHIFT) |
+				(0 << CS42L42_HSBIAS_REF_SHIFT) |
+				(3 << CS42L42_HSDET_AUTO_TIME_SHIFT));
+
+	regmap_update_bits(cs42l42->regmap,
+				CS42L42_DAC_CTL2,
+				CS42L42_HPOUT_PULLDOWN_MASK |
+				CS42L42_HPOUT_LOAD_MASK |
+				CS42L42_HPOUT_CLAMP_MASK |
+				CS42L42_DAC_HPF_EN_MASK |
+				CS42L42_DAC_MON_EN_MASK,
+				(8 << CS42L42_HPOUT_PULLDOWN_SHIFT) |
+				(0 << CS42L42_HPOUT_LOAD_SHIFT) |
+				(1 << CS42L42_HPOUT_CLAMP_SHIFT) |
+				(1 << CS42L42_DAC_HPF_EN_SHIFT) |
+				(1 << CS42L42_DAC_MON_EN_SHIFT));
+
+	/* Power up HS bias to 2.7V */
+	regmap_update_bits(cs42l42->regmap,
+				CS42L42_MISC_DET_CTL,
+				CS42L42_DETECT_MODE_MASK |
+				CS42L42_HSBIAS_CTL_MASK |
+				CS42L42_PDN_MIC_LVL_DET_MASK,
+				(0 << CS42L42_DETECT_MODE_SHIFT) |
+				(3 << CS42L42_HSBIAS_CTL_SHIFT) |
+				(1 << CS42L42_PDN_MIC_LVL_DET_SHIFT));
+
+	/* Wait for HS bias to ramp up */
+	msleep(cs42l42->hs_bias_ramp_time);
+
+	/* Unmask auto detect interrupt */
+	regmap_update_bits(cs42l42->regmap,
+				CS42L42_CODEC_INT_MASK,
+				CS42L42_PDN_DONE_MASK |
+				CS42L42_HSDET_AUTO_DONE_MASK,
+				(1 << CS42L42_PDN_DONE_SHIFT) |
+				(0 << CS42L42_HSDET_AUTO_DONE_SHIFT));
+
+	/* Set hs detect to automatic, enabled mode */
+	regmap_update_bits(cs42l42->regmap,
+				CS42L42_HSDET_CTL2,
+				CS42L42_HSDET_CTRL_MASK |
+				CS42L42_HSDET_SET_MASK |
+				CS42L42_HSBIAS_REF_MASK |
+				CS42L42_HSDET_AUTO_TIME_MASK,
+				(3 << CS42L42_HSDET_CTRL_SHIFT) |
+				(2 << CS42L42_HSDET_SET_SHIFT) |
+				(0 << CS42L42_HSBIAS_REF_SHIFT) |
+				(3 << CS42L42_HSDET_AUTO_TIME_SHIFT));
+}
+
+static void cs42l42_cancel_hs_type_detect(struct cs42l42_private *cs42l42)
+{
+	/* Mask button detect interrupts */
+	regmap_update_bits(cs42l42->regmap,
+		CS42L42_DET_INT2_MASK,
+		CS42L42_M_DETECT_TF_MASK |
+		CS42L42_M_DETECT_FT_MASK |
+		CS42L42_M_HSBIAS_HIZ_MASK |
+		CS42L42_M_SHORT_RLS_MASK |
+		CS42L42_M_SHORT_DET_MASK,
+		(1 << CS42L42_M_DETECT_TF_SHIFT) |
+		(1 << CS42L42_M_DETECT_FT_SHIFT) |
+		(1 << CS42L42_M_HSBIAS_HIZ_SHIFT) |
+		(1 << CS42L42_M_SHORT_RLS_SHIFT) |
+		(1 << CS42L42_M_SHORT_DET_SHIFT));
+
+	/* Ground HS bias */
+	regmap_update_bits(cs42l42->regmap,
+				CS42L42_MISC_DET_CTL,
+				CS42L42_DETECT_MODE_MASK |
+				CS42L42_HSBIAS_CTL_MASK |
+				CS42L42_PDN_MIC_LVL_DET_MASK,
+				(0 << CS42L42_DETECT_MODE_SHIFT) |
+				(1 << CS42L42_HSBIAS_CTL_SHIFT) |
+				(1 << CS42L42_PDN_MIC_LVL_DET_SHIFT));
+
+	/* Set auto HS bias settings to default */
+	regmap_update_bits(cs42l42->regmap,
+				CS42L42_HSBIAS_SC_AUTOCTL,
+				CS42L42_HSBIAS_SENSE_EN_MASK |
+				CS42L42_AUTO_HSBIAS_HIZ_MASK |
+				CS42L42_TIP_SENSE_EN_MASK |
+				CS42L42_HSBIAS_SENSE_TRIP_MASK,
+				(0 << CS42L42_HSBIAS_SENSE_EN_SHIFT) |
+				(0 << CS42L42_AUTO_HSBIAS_HIZ_SHIFT) |
+				(0 << CS42L42_TIP_SENSE_EN_SHIFT) |
+				(3 << CS42L42_HSBIAS_SENSE_TRIP_SHIFT));
+
+	/* Set hs detect to manual, disabled mode */
+	regmap_update_bits(cs42l42->regmap,
+				CS42L42_HSDET_CTL2,
+				CS42L42_HSDET_CTRL_MASK |
+				CS42L42_HSDET_SET_MASK |
+				CS42L42_HSBIAS_REF_MASK |
+				CS42L42_HSDET_AUTO_TIME_MASK,
+				(0 << CS42L42_HSDET_CTRL_SHIFT) |
+				(2 << CS42L42_HSDET_SET_SHIFT) |
+				(0 << CS42L42_HSBIAS_REF_SHIFT) |
+				(3 << CS42L42_HSDET_AUTO_TIME_SHIFT));
+}
+
+static void cs42l42_handle_button_press(struct cs42l42_private *cs42l42)
+{
+	int bias_level;
+	unsigned int detect_status;
+
+	/* Mask button detect interrupts */
+	regmap_update_bits(cs42l42->regmap,
+		CS42L42_DET_INT2_MASK,
+		CS42L42_M_DETECT_TF_MASK |
+		CS42L42_M_DETECT_FT_MASK |
+		CS42L42_M_HSBIAS_HIZ_MASK |
+		CS42L42_M_SHORT_RLS_MASK |
+		CS42L42_M_SHORT_DET_MASK,
+		(1 << CS42L42_M_DETECT_TF_SHIFT) |
+		(1 << CS42L42_M_DETECT_FT_SHIFT) |
+		(1 << CS42L42_M_HSBIAS_HIZ_SHIFT) |
+		(1 << CS42L42_M_SHORT_RLS_SHIFT) |
+		(1 << CS42L42_M_SHORT_DET_SHIFT));
+
+	usleep_range(cs42l42->btn_det_event_dbnce * 1000,
+		     cs42l42->btn_det_event_dbnce * 2000);
+
+	/* Test all 4 level detect biases */
+	bias_level = 1;
+	do {
+		/* Adjust button detect level sensitivity */
+		regmap_update_bits(cs42l42->regmap,
+			CS42L42_MIC_DET_CTL1,
+			CS42L42_LATCH_TO_VP_MASK |
+			CS42L42_EVENT_STAT_SEL_MASK |
+			CS42L42_HS_DET_LEVEL_MASK,
+			(1 << CS42L42_LATCH_TO_VP_SHIFT) |
+			(0 << CS42L42_EVENT_STAT_SEL_SHIFT) |
+			(cs42l42->bias_thresholds[bias_level] <<
+			CS42L42_HS_DET_LEVEL_SHIFT));
+
+		regmap_read(cs42l42->regmap, CS42L42_DET_STATUS2,
+				&detect_status);
+	} while ((detect_status & CS42L42_HS_TRUE_MASK) &&
+		(++bias_level < CS42L42_NUM_BIASES));
+
+	switch (bias_level) {
+	case 1: /* Function C button press */
+		dev_dbg(cs42l42->codec->dev, "Function C button press\n");
+		break;
+	case 2: /* Function B button press */
+		dev_dbg(cs42l42->codec->dev, "Function B button press\n");
+		break;
+	case 3: /* Function D button press */
+		dev_dbg(cs42l42->codec->dev, "Function D button press\n");
+		break;
+	case 4: /* Function A button press */
+		dev_dbg(cs42l42->codec->dev, "Function A button press\n");
+		break;
+	}
+
+	/* Set button detect level sensitivity back to default */
+	regmap_update_bits(cs42l42->regmap,
+		CS42L42_MIC_DET_CTL1,
+		CS42L42_LATCH_TO_VP_MASK |
+		CS42L42_EVENT_STAT_SEL_MASK |
+		CS42L42_HS_DET_LEVEL_MASK,
+		(1 << CS42L42_LATCH_TO_VP_SHIFT) |
+		(0 << CS42L42_EVENT_STAT_SEL_SHIFT) |
+		(cs42l42->bias_thresholds[0] << CS42L42_HS_DET_LEVEL_SHIFT));
+
+	/* Clear any button interrupts before unmasking them */
+	regmap_read(cs42l42->regmap, CS42L42_DET_INT_STATUS2,
+		    &detect_status);
+
+	/* Unmask button detect interrupts */
+	regmap_update_bits(cs42l42->regmap,
+		CS42L42_DET_INT2_MASK,
+		CS42L42_M_DETECT_TF_MASK |
+		CS42L42_M_DETECT_FT_MASK |
+		CS42L42_M_HSBIAS_HIZ_MASK |
+		CS42L42_M_SHORT_RLS_MASK |
+		CS42L42_M_SHORT_DET_MASK,
+		(0 << CS42L42_M_DETECT_TF_SHIFT) |
+		(0 << CS42L42_M_DETECT_FT_SHIFT) |
+		(0 << CS42L42_M_HSBIAS_HIZ_SHIFT) |
+		(1 << CS42L42_M_SHORT_RLS_SHIFT) |
+		(1 << CS42L42_M_SHORT_DET_SHIFT));
+}
+
+struct cs42l42_irq_params {
+	u16 status_addr;
+	u16 mask_addr;
+	u8 mask;
+};
+
+static const struct cs42l42_irq_params irq_params_table[] = {
+	{CS42L42_ADC_OVFL_STATUS, CS42L42_ADC_OVFL_INT_MASK,
+		CS42L42_ADC_OVFL_VAL_MASK},
+	{CS42L42_MIXER_STATUS, CS42L42_MIXER_INT_MASK,
+		CS42L42_MIXER_VAL_MASK},
+	{CS42L42_SRC_STATUS, CS42L42_SRC_INT_MASK,
+		CS42L42_SRC_VAL_MASK},
+	{CS42L42_ASP_RX_STATUS, CS42L42_ASP_RX_INT_MASK,
+		CS42L42_ASP_RX_VAL_MASK},
+	{CS42L42_ASP_TX_STATUS, CS42L42_ASP_TX_INT_MASK,
+		CS42L42_ASP_TX_VAL_MASK},
+	{CS42L42_CODEC_STATUS, CS42L42_CODEC_INT_MASK,
+		CS42L42_CODEC_VAL_MASK},
+	{CS42L42_DET_INT_STATUS1, CS42L42_DET_INT1_MASK,
+		CS42L42_DET_INT_VAL1_MASK},
+	{CS42L42_DET_INT_STATUS2, CS42L42_DET_INT2_MASK,
+		CS42L42_DET_INT_VAL2_MASK},
+	{CS42L42_SRCPL_INT_STATUS, CS42L42_SRCPL_INT_MASK,
+		CS42L42_SRCPL_VAL_MASK},
+	{CS42L42_VPMON_STATUS, CS42L42_VPMON_INT_MASK,
+		CS42L42_VPMON_VAL_MASK},
+	{CS42L42_PLL_LOCK_STATUS, CS42L42_PLL_LOCK_INT_MASK,
+		CS42L42_PLL_LOCK_VAL_MASK},
+	{CS42L42_TSRS_PLUG_STATUS, CS42L42_TSRS_PLUG_INT_MASK,
+		CS42L42_TSRS_PLUG_VAL_MASK}
+};
+
+static irqreturn_t cs42l42_irq_thread(int irq, void *data)
+{
+	struct cs42l42_private *cs42l42 = (struct cs42l42_private *)data;
+	struct snd_soc_codec *codec = cs42l42->codec;
+	unsigned int stickies[12];
+	unsigned int masks[12];
+	unsigned int current_plug_status;
+	unsigned int current_button_status;
+	unsigned int i;
+
+	/* Read sticky registers to clear interurpt */
+	for (i = 0; i < ARRAY_SIZE(stickies); i++) {
+		regmap_read(cs42l42->regmap, irq_params_table[i].status_addr,
+				&(stickies[i]));
+		regmap_read(cs42l42->regmap, irq_params_table[i].mask_addr,
+				&(masks[i]));
+		stickies[i] = stickies[i] & (~masks[i]) &
+				irq_params_table[i].mask;
+	}
+
+	/* Read tip sense status before handling type detect */
+	current_plug_status = (stickies[11] &
+		(CS42L42_TS_PLUG_MASK | CS42L42_TS_UNPLUG_MASK)) >>
+		CS42L42_TS_PLUG_SHIFT;
+
+	/* Read button sense status */
+	current_button_status = stickies[7] &
+		(CS42L42_M_DETECT_TF_MASK |
+		CS42L42_M_DETECT_FT_MASK |
+		CS42L42_M_HSBIAS_HIZ_MASK);
+
+	/* Check auto-detect status */
+	if ((~masks[5]) & irq_params_table[5].mask) {
+		if (stickies[5] & CS42L42_HSDET_AUTO_DONE_MASK) {
+			cs42l42_process_hs_type_detect(cs42l42);
+			dev_dbg(codec->dev,
+				"Auto detect done (%d)\n",
+				cs42l42->hs_type);
+		}
+	}
+
+	/* Check tip sense status */
+	if ((~masks[11]) & irq_params_table[11].mask) {
+		switch (current_plug_status) {
+		case CS42L42_TS_PLUG:
+			if (cs42l42->plug_state != CS42L42_TS_PLUG) {
+				cs42l42->plug_state = CS42L42_TS_PLUG;
+				cs42l42_init_hs_type_detect(cs42l42);
+			}
+			break;
+
+		case CS42L42_TS_UNPLUG:
+			if (cs42l42->plug_state != CS42L42_TS_UNPLUG) {
+				cs42l42->plug_state = CS42L42_TS_UNPLUG;
+				cs42l42_cancel_hs_type_detect(cs42l42);
+				dev_dbg(codec->dev,
+					"Unplug event\n");
+			}
+			break;
+
+		default:
+			if (cs42l42->plug_state != CS42L42_TS_TRANS)
+				cs42l42->plug_state = CS42L42_TS_TRANS;
+		}
+	}
+
+	/* Check button detect status */
+	if ((~masks[7]) & irq_params_table[7].mask) {
+		if (!(current_button_status &
+			CS42L42_M_HSBIAS_HIZ_MASK)) {
+
+			if (current_button_status &
+				CS42L42_M_DETECT_TF_MASK) {
+				dev_dbg(codec->dev,
+					"Button released\n");
+			} else if (current_button_status &
+				CS42L42_M_DETECT_FT_MASK) {
+				cs42l42_handle_button_press(cs42l42);
+			}
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void cs42l42_set_interrupt_masks(struct cs42l42_private *cs42l42)
+{
+	regmap_update_bits(cs42l42->regmap, CS42L42_ADC_OVFL_INT_MASK,
+			CS42L42_ADC_OVFL_MASK,
+			(1 << CS42L42_ADC_OVFL_SHIFT));
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_MIXER_INT_MASK,
+			CS42L42_MIX_CHB_OVFL_MASK |
+			CS42L42_MIX_CHA_OVFL_MASK |
+			CS42L42_EQ_OVFL_MASK |
+			CS42L42_EQ_BIQUAD_OVFL_MASK,
+			(1 << CS42L42_MIX_CHB_OVFL_SHIFT) |
+			(1 << CS42L42_MIX_CHA_OVFL_SHIFT) |
+			(1 << CS42L42_EQ_OVFL_SHIFT) |
+			(1 << CS42L42_EQ_BIQUAD_OVFL_SHIFT));
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_SRC_INT_MASK,
+			CS42L42_SRC_ILK_MASK |
+			CS42L42_SRC_OLK_MASK |
+			CS42L42_SRC_IUNLK_MASK |
+			CS42L42_SRC_OUNLK_MASK,
+			(1 << CS42L42_SRC_ILK_SHIFT) |
+			(1 << CS42L42_SRC_OLK_SHIFT) |
+			(1 << CS42L42_SRC_IUNLK_SHIFT) |
+			(1 << CS42L42_SRC_OUNLK_SHIFT));
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_ASP_RX_INT_MASK,
+			CS42L42_ASPRX_NOLRCK_MASK |
+			CS42L42_ASPRX_EARLY_MASK |
+			CS42L42_ASPRX_LATE_MASK |
+			CS42L42_ASPRX_ERROR_MASK |
+			CS42L42_ASPRX_OVLD_MASK,
+			(1 << CS42L42_ASPRX_NOLRCK_SHIFT) |
+			(1 << CS42L42_ASPRX_EARLY_SHIFT) |
+			(1 << CS42L42_ASPRX_LATE_SHIFT) |
+			(1 << CS42L42_ASPRX_ERROR_SHIFT) |
+			(1 << CS42L42_ASPRX_OVLD_SHIFT));
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_ASP_TX_INT_MASK,
+			CS42L42_ASPTX_NOLRCK_MASK |
+			CS42L42_ASPTX_EARLY_MASK |
+			CS42L42_ASPTX_LATE_MASK |
+			CS42L42_ASPTX_SMERROR_MASK,
+			(1 << CS42L42_ASPTX_NOLRCK_SHIFT) |
+			(1 << CS42L42_ASPTX_EARLY_SHIFT) |
+			(1 << CS42L42_ASPTX_LATE_SHIFT) |
+			(1 << CS42L42_ASPTX_SMERROR_SHIFT));
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_CODEC_INT_MASK,
+			CS42L42_PDN_DONE_MASK |
+			CS42L42_HSDET_AUTO_DONE_MASK,
+			(1 << CS42L42_PDN_DONE_SHIFT) |
+			(1 << CS42L42_HSDET_AUTO_DONE_SHIFT));
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_SRCPL_INT_MASK,
+			CS42L42_SRCPL_ADC_LK_MASK |
+			CS42L42_SRCPL_DAC_LK_MASK |
+			CS42L42_SRCPL_ADC_UNLK_MASK |
+			CS42L42_SRCPL_DAC_UNLK_MASK,
+			(1 << CS42L42_SRCPL_ADC_LK_SHIFT) |
+			(1 << CS42L42_SRCPL_DAC_LK_SHIFT) |
+			(1 << CS42L42_SRCPL_ADC_UNLK_SHIFT) |
+			(1 << CS42L42_SRCPL_DAC_UNLK_SHIFT));
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_DET_INT1_MASK,
+			CS42L42_TIP_SENSE_UNPLUG_MASK |
+			CS42L42_TIP_SENSE_PLUG_MASK |
+			CS42L42_HSBIAS_SENSE_MASK,
+			(1 << CS42L42_TIP_SENSE_UNPLUG_SHIFT) |
+			(1 << CS42L42_TIP_SENSE_PLUG_SHIFT) |
+			(1 << CS42L42_HSBIAS_SENSE_SHIFT));
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_DET_INT2_MASK,
+			CS42L42_M_DETECT_TF_MASK |
+			CS42L42_M_DETECT_FT_MASK |
+			CS42L42_M_HSBIAS_HIZ_MASK |
+			CS42L42_M_SHORT_RLS_MASK |
+			CS42L42_M_SHORT_DET_MASK,
+			(1 << CS42L42_M_DETECT_TF_SHIFT) |
+			(1 << CS42L42_M_DETECT_FT_SHIFT) |
+			(1 << CS42L42_M_HSBIAS_HIZ_SHIFT) |
+			(1 << CS42L42_M_SHORT_RLS_SHIFT) |
+			(1 << CS42L42_M_SHORT_DET_SHIFT));
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_VPMON_INT_MASK,
+			CS42L42_VPMON_MASK,
+			(1 << CS42L42_VPMON_SHIFT));
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_PLL_LOCK_INT_MASK,
+			CS42L42_PLL_LOCK_MASK,
+			(1 << CS42L42_PLL_LOCK_SHIFT));
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_TSRS_PLUG_INT_MASK,
+			CS42L42_RS_PLUG_MASK |
+			CS42L42_RS_UNPLUG_MASK |
+			CS42L42_TS_PLUG_MASK |
+			CS42L42_TS_UNPLUG_MASK,
+			(1 << CS42L42_RS_PLUG_SHIFT) |
+			(1 << CS42L42_RS_UNPLUG_SHIFT) |
+			(0 << CS42L42_TS_PLUG_SHIFT) |
+			(0 << CS42L42_TS_UNPLUG_SHIFT));
+}
+
+static void cs42l42_setup_hs_type_detect(struct cs42l42_private *cs42l42)
+{
+	unsigned int reg;
+
+	cs42l42->hs_type = CS42L42_PLUG_INVALID;
+
+	/* Latch analog controls to VP power domain */
+	regmap_update_bits(cs42l42->regmap, CS42L42_MIC_DET_CTL1,
+			CS42L42_LATCH_TO_VP_MASK |
+			CS42L42_EVENT_STAT_SEL_MASK |
+			CS42L42_HS_DET_LEVEL_MASK,
+			(1 << CS42L42_LATCH_TO_VP_SHIFT) |
+			(0 << CS42L42_EVENT_STAT_SEL_SHIFT) |
+			(cs42l42->bias_thresholds[0] <<
+			CS42L42_HS_DET_LEVEL_SHIFT));
+
+	/* Remove ground noise-suppression clamps */
+	regmap_update_bits(cs42l42->regmap,
+			CS42L42_HS_CLAMP_DISABLE,
+			CS42L42_HS_CLAMP_DISABLE_MASK,
+			(1 << CS42L42_HS_CLAMP_DISABLE_SHIFT));
+
+	/* Enable the tip sense circuit */
+	regmap_update_bits(cs42l42->regmap, CS42L42_TIPSENSE_CTL,
+			CS42L42_TIP_SENSE_CTRL_MASK |
+			CS42L42_TIP_SENSE_INV_MASK |
+			CS42L42_TIP_SENSE_DEBOUNCE_MASK,
+			(3 << CS42L42_TIP_SENSE_CTRL_SHIFT) |
+			(0 << CS42L42_TIP_SENSE_INV_SHIFT) |
+			(2 << CS42L42_TIP_SENSE_DEBOUNCE_SHIFT));
+
+	/* Save the initial status of the tip sense */
+	regmap_read(cs42l42->regmap,
+			  CS42L42_TSRS_PLUG_STATUS,
+			  &reg);
+	cs42l42->plug_state = (((char) reg) &
+		      (CS42L42_TS_PLUG_MASK | CS42L42_TS_UNPLUG_MASK)) >>
+		      CS42L42_TS_PLUG_SHIFT;
+}
+
+static const unsigned int threshold_defaults[] = {
+	CS42L42_HS_DET_LEVEL_15,
+	CS42L42_HS_DET_LEVEL_8,
+	CS42L42_HS_DET_LEVEL_4,
+	CS42L42_HS_DET_LEVEL_1
+};
+
+static int cs42l42_handle_device_data(struct i2c_client *i2c_client,
+					struct cs42l42_private *cs42l42)
+{
+	struct device_node *np = i2c_client->dev.of_node;
+	unsigned int val;
+	unsigned int thresholds[CS42L42_NUM_BIASES];
+	int ret;
+	int i;
+
+	ret = of_property_read_u32(np, "cirrus,ts-inv", &val);
+
+	if (!ret) {
+		switch (val) {
+		case CS42L42_TS_INV_EN:
+		case CS42L42_TS_INV_DIS:
+			cs42l42->ts_inv = val;
+			break;
+		default:
+			dev_err(&i2c_client->dev,
+				"Wrong cirrus,ts-inv DT value %d\n",
+				val);
+			cs42l42->ts_inv = CS42L42_TS_INV_DIS;
+		}
+	} else {
+		cs42l42->ts_inv = CS42L42_TS_INV_DIS;
+	}
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_TSENSE_CTL,
+			CS42L42_TS_INV_MASK,
+			(cs42l42->ts_inv << CS42L42_TS_INV_SHIFT));
+
+	ret = of_property_read_u32(np, "cirrus,ts-dbnc-rise", &val);
+
+	if (!ret) {
+		switch (val) {
+		case CS42L42_TS_DBNCE_0:
+		case CS42L42_TS_DBNCE_125:
+		case CS42L42_TS_DBNCE_250:
+		case CS42L42_TS_DBNCE_500:
+		case CS42L42_TS_DBNCE_750:
+		case CS42L42_TS_DBNCE_1000:
+		case CS42L42_TS_DBNCE_1250:
+		case CS42L42_TS_DBNCE_1500:
+			cs42l42->ts_dbnc_rise = val;
+			break;
+		default:
+			dev_err(&i2c_client->dev,
+				"Wrong cirrus,ts-dbnc-rise DT value %d\n",
+				val);
+			cs42l42->ts_dbnc_rise = CS42L42_TS_DBNCE_1000;
+		}
+	} else {
+		cs42l42->ts_dbnc_rise = CS42L42_TS_DBNCE_1000;
+	}
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_TSENSE_CTL,
+			CS42L42_TS_RISE_DBNCE_TIME_MASK,
+			(cs42l42->ts_dbnc_rise <<
+			CS42L42_TS_RISE_DBNCE_TIME_SHIFT));
+
+	ret = of_property_read_u32(np, "cirrus,ts-dbnc-fall", &val);
+
+	if (!ret) {
+		switch (val) {
+		case CS42L42_TS_DBNCE_0:
+		case CS42L42_TS_DBNCE_125:
+		case CS42L42_TS_DBNCE_250:
+		case CS42L42_TS_DBNCE_500:
+		case CS42L42_TS_DBNCE_750:
+		case CS42L42_TS_DBNCE_1000:
+		case CS42L42_TS_DBNCE_1250:
+		case CS42L42_TS_DBNCE_1500:
+			cs42l42->ts_dbnc_fall = val;
+			break;
+		default:
+			dev_err(&i2c_client->dev,
+				"Wrong cirrus,ts-dbnc-fall DT value %d\n",
+				val);
+			cs42l42->ts_dbnc_fall = CS42L42_TS_DBNCE_0;
+		}
+	} else {
+		cs42l42->ts_dbnc_fall = CS42L42_TS_DBNCE_0;
+	}
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_TSENSE_CTL,
+			CS42L42_TS_FALL_DBNCE_TIME_MASK,
+			(cs42l42->ts_dbnc_fall <<
+			CS42L42_TS_FALL_DBNCE_TIME_SHIFT));
+
+	ret = of_property_read_u32(np, "cirrus,btn-det-init-dbnce", &val);
+
+	if (!ret) {
+		if ((val >= CS42L42_BTN_DET_INIT_DBNCE_MIN) &&
+			(val <= CS42L42_BTN_DET_INIT_DBNCE_MAX))
+			cs42l42->btn_det_init_dbnce = val;
+		else {
+			dev_err(&i2c_client->dev,
+				"Wrong cirrus,btn-det-init-dbnce DT value %d\n",
+				val);
+			cs42l42->btn_det_init_dbnce =
+				CS42L42_BTN_DET_INIT_DBNCE_DEFAULT;
+		}
+	} else {
+		cs42l42->btn_det_init_dbnce =
+			CS42L42_BTN_DET_INIT_DBNCE_DEFAULT;
+	}
+
+	ret = of_property_read_u32(np, "cirrus,btn-det-event-dbnce", &val);
+
+	if (!ret) {
+		if ((val >= CS42L42_BTN_DET_EVENT_DBNCE_MIN) &&
+			(val <= CS42L42_BTN_DET_EVENT_DBNCE_MAX))
+			cs42l42->btn_det_event_dbnce = val;
+		else {
+			dev_err(&i2c_client->dev,
+			"Wrong cirrus,btn-det-event-dbnce DT value %d\n", val);
+			cs42l42->btn_det_event_dbnce =
+				CS42L42_BTN_DET_EVENT_DBNCE_DEFAULT;
+		}
+	} else {
+		cs42l42->btn_det_event_dbnce =
+			CS42L42_BTN_DET_EVENT_DBNCE_DEFAULT;
+	}
+
+	ret = of_property_read_u32_array(np, "cirrus,bias-lvls",
+				   (u32 *)thresholds, CS42L42_NUM_BIASES);
+
+	if (!ret) {
+		for (i = 0; i < CS42L42_NUM_BIASES; i++) {
+			if ((thresholds[i] >= CS42L42_HS_DET_LEVEL_MIN) &&
+				(thresholds[i] <= CS42L42_HS_DET_LEVEL_MAX))
+				cs42l42->bias_thresholds[i] = thresholds[i];
+			else {
+				dev_err(&i2c_client->dev,
+				"Wrong cirrus,bias-lvls[%d] DT value %d\n", i,
+					thresholds[i]);
+				cs42l42->bias_thresholds[i] =
+					threshold_defaults[i];
+			}
+		}
+	} else {
+		for (i = 0; i < CS42L42_NUM_BIASES; i++)
+			cs42l42->bias_thresholds[i] = threshold_defaults[i];
+	}
+
+	ret = of_property_read_u32(np, "cirrus,hs-bias-ramp-rate", &val);
+
+	if (!ret) {
+		switch (val) {
+		case CS42L42_HSBIAS_RAMP_FAST_RISE_SLOW_FALL:
+			cs42l42->hs_bias_ramp_rate = val;
+			cs42l42->hs_bias_ramp_time = CS42L42_HSBIAS_RAMP_TIME0;
+			break;
+		case CS42L42_HSBIAS_RAMP_FAST:
+			cs42l42->hs_bias_ramp_rate = val;
+			cs42l42->hs_bias_ramp_time = CS42L42_HSBIAS_RAMP_TIME1;
+			break;
+		case CS42L42_HSBIAS_RAMP_SLOW:
+			cs42l42->hs_bias_ramp_rate = val;
+			cs42l42->hs_bias_ramp_time = CS42L42_HSBIAS_RAMP_TIME2;
+			break;
+		case CS42L42_HSBIAS_RAMP_SLOWEST:
+			cs42l42->hs_bias_ramp_rate = val;
+			cs42l42->hs_bias_ramp_time = CS42L42_HSBIAS_RAMP_TIME3;
+			break;
+		default:
+			dev_err(&i2c_client->dev,
+				"Wrong cirrus,hs-bias-ramp-rate DT value %d\n",
+				val);
+			cs42l42->hs_bias_ramp_rate = CS42L42_HSBIAS_RAMP_SLOW;
+			cs42l42->hs_bias_ramp_time = CS42L42_HSBIAS_RAMP_TIME2;
+		}
+	} else {
+		cs42l42->hs_bias_ramp_rate = CS42L42_HSBIAS_RAMP_SLOW;
+		cs42l42->hs_bias_ramp_time = CS42L42_HSBIAS_RAMP_TIME2;
+	}
+
+	regmap_update_bits(cs42l42->regmap, CS42L42_HS_BIAS_CTL,
+			CS42L42_HSBIAS_RAMP_MASK,
+			(cs42l42->hs_bias_ramp_rate <<
+			CS42L42_HSBIAS_RAMP_SHIFT));
+
+	return 0;
+}
+
+static int cs42l42_i2c_probe(struct i2c_client *i2c_client,
+				       const struct i2c_device_id *id)
+{
+	struct cs42l42_private *cs42l42;
+	int ret, i;
+	unsigned int devid = 0;
+	unsigned int reg;
+
+	cs42l42 = devm_kzalloc(&i2c_client->dev, sizeof(struct cs42l42_private),
+			       GFP_KERNEL);
+	if (!cs42l42)
+		return -ENOMEM;
+
+	i2c_set_clientdata(i2c_client, cs42l42);
+
+	cs42l42->regmap = devm_regmap_init_i2c(i2c_client, &cs42l42_regmap);
+	if (IS_ERR(cs42l42->regmap)) {
+		ret = PTR_ERR(cs42l42->regmap);
+		dev_err(&i2c_client->dev, "regmap_init() failed: %d\n", ret);
+		return ret;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(cs42l42->supplies); i++)
+		cs42l42->supplies[i].supply = cs42l42_supply_names[i];
+
+	ret = devm_regulator_bulk_get(&i2c_client->dev,
+				      ARRAY_SIZE(cs42l42->supplies),
+				      cs42l42->supplies);
+	if (ret != 0) {
+		dev_err(&i2c_client->dev,
+			"Failed to request supplies: %d\n", ret);
+		return ret;
+	}
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(cs42l42->supplies),
+				    cs42l42->supplies);
+	if (ret != 0) {
+		dev_err(&i2c_client->dev,
+			"Failed to enable supplies: %d\n", ret);
+		return ret;
+	}
+
+	/* Reset the Device */
+	cs42l42->reset_gpio = devm_gpiod_get_optional(&i2c_client->dev,
+		"reset", GPIOD_OUT_LOW);
+	if (IS_ERR(cs42l42->reset_gpio))
+		return PTR_ERR(cs42l42->reset_gpio);
+
+	if (cs42l42->reset_gpio) {
+		dev_dbg(&i2c_client->dev, "Found reset GPIO\n");
+		gpiod_set_value_cansleep(cs42l42->reset_gpio, 1);
+	}
+	mdelay(3);
+
+	/* Request IRQ */
+	ret = devm_request_threaded_irq(&i2c_client->dev,
+			i2c_client->irq,
+			NULL, cs42l42_irq_thread,
+			IRQF_ONESHOT | IRQF_TRIGGER_LOW,
+			"cs42l42", cs42l42);
+
+	if (ret != 0)
+		dev_err(&i2c_client->dev,
+			"Failed to request IRQ: %d\n", ret);
+
+	/* initialize codec */
+	ret = regmap_read(cs42l42->regmap, CS42L42_DEVID_AB, &reg);
+	devid = (reg & 0xFF) << 12;
+
+	ret = regmap_read(cs42l42->regmap, CS42L42_DEVID_CD, &reg);
+	devid |= (reg & 0xFF) << 4;
+
+	ret = regmap_read(cs42l42->regmap, CS42L42_DEVID_E, &reg);
+	devid |= (reg & 0xF0) >> 4;
+
+	if (devid != CS42L42_CHIP_ID) {
+		ret = -ENODEV;
+		dev_err(&i2c_client->dev,
+			"CS42L42 Device ID (%X). Expected %X\n",
+			devid, CS42L42_CHIP_ID);
+		return ret;
+	}
+
+	ret = regmap_read(cs42l42->regmap, CS42L42_REVID, &reg);
+	if (ret < 0) {
+		dev_err(&i2c_client->dev, "Get Revision ID failed\n");
+		return ret;
+	}
+
+	dev_info(&i2c_client->dev,
+		 "Cirrus Logic CS42L42, Revision: %02X\n", reg & 0xFF);
+
+	/* Power up the codec */
+	regmap_update_bits(cs42l42->regmap, CS42L42_PWR_CTL1,
+			CS42L42_ASP_DAO_PDN_MASK |
+			CS42L42_ASP_DAI_PDN_MASK |
+			CS42L42_MIXER_PDN_MASK |
+			CS42L42_EQ_PDN_MASK |
+			CS42L42_HP_PDN_MASK |
+			CS42L42_ADC_PDN_MASK |
+			CS42L42_PDN_ALL_MASK,
+			(1 << CS42L42_ASP_DAO_PDN_SHIFT) |
+			(1 << CS42L42_ASP_DAI_PDN_SHIFT) |
+			(1 << CS42L42_MIXER_PDN_SHIFT) |
+			(1 << CS42L42_EQ_PDN_SHIFT) |
+			(1 << CS42L42_HP_PDN_SHIFT) |
+			(1 << CS42L42_ADC_PDN_SHIFT) |
+			(0 << CS42L42_PDN_ALL_SHIFT));
+
+	if (i2c_client->dev.of_node) {
+		ret = cs42l42_handle_device_data(i2c_client, cs42l42);
+		if (ret != 0)
+			return ret;
+	}
+
+	/* Setup headset detection */
+	cs42l42_setup_hs_type_detect(cs42l42);
+
+	/* Mask/Unmask Interrupts */
+	cs42l42_set_interrupt_masks(cs42l42);
+
+	/* Register codec for machine driver */
+	ret =  snd_soc_register_codec(&i2c_client->dev,
+			&soc_codec_dev_cs42l42, &cs42l42_dai, 1);
+	if (ret < 0)
+		goto err_disable;
+	return 0;
+
+err_disable:
+	regulator_bulk_disable(ARRAY_SIZE(cs42l42->supplies),
+				cs42l42->supplies);
+	return ret;
+}
+
+static int cs42l42_i2c_remove(struct i2c_client *i2c_client)
+{
+	struct cs42l42_private *cs42l42 = i2c_get_clientdata(i2c_client);
+
+	snd_soc_unregister_codec(&i2c_client->dev);
+
+	/* Hold down reset */
+	if (cs42l42->reset_gpio)
+		gpiod_set_value_cansleep(cs42l42->reset_gpio, 0);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int cs42l42_runtime_suspend(struct device *dev)
+{
+	struct cs42l42_private *cs42l42 = dev_get_drvdata(dev);
+
+	regcache_cache_only(cs42l42->regmap, true);
+	regcache_mark_dirty(cs42l42->regmap);
+
+	/* Hold down reset */
+	if (cs42l42->reset_gpio)
+		gpiod_set_value_cansleep(cs42l42->reset_gpio, 0);
+
+	/* remove power */
+	regulator_bulk_disable(ARRAY_SIZE(cs42l42->supplies),
+				cs42l42->supplies);
+
+	return 0;
+}
+
+static int cs42l42_runtime_resume(struct device *dev)
+{
+	struct cs42l42_private *cs42l42 = dev_get_drvdata(dev);
+	int ret;
+
+	/* Enable power */
+	ret = regulator_bulk_enable(ARRAY_SIZE(cs42l42->supplies),
+					cs42l42->supplies);
+	if (ret != 0) {
+		dev_err(dev, "Failed to enable supplies: %d\n",
+			ret);
+		return ret;
+	}
+
+	if (cs42l42->reset_gpio)
+		gpiod_set_value_cansleep(cs42l42->reset_gpio, 1);
+
+	regcache_cache_only(cs42l42->regmap, false);
+	regcache_sync(cs42l42->regmap);
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops cs42l42_runtime_pm = {
+	SET_RUNTIME_PM_OPS(cs42l42_runtime_suspend, cs42l42_runtime_resume,
+			   NULL)
+};
+
+static const struct of_device_id cs42l42_of_match[] = {
+	{ .compatible = "cirrus,cs42l42", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, cs42l42_of_match);
+
+
+static const struct i2c_device_id cs42l42_id[] = {
+	{"cs42l42", 0},
+	{}
+};
+
+MODULE_DEVICE_TABLE(i2c, cs42l42_id);
+
+static struct i2c_driver cs42l42_i2c_driver = {
+	.driver = {
+		.name = "cs42l42",
+		.pm = &cs42l42_runtime_pm,
+		.of_match_table = cs42l42_of_match,
+		},
+	.id_table = cs42l42_id,
+	.probe = cs42l42_i2c_probe,
+	.remove = cs42l42_i2c_remove,
+};
+
+module_i2c_driver(cs42l42_i2c_driver);
+
+MODULE_DESCRIPTION("ASoC CS42L42 driver");
+MODULE_AUTHOR("James Schulman, Cirrus Logic Inc, <james.schulman@cirrus.com>");
+MODULE_AUTHOR("Brian Austin, Cirrus Logic Inc, <brian.austin@cirrus.com>");
+MODULE_AUTHOR("Michael White, Cirrus Logic Inc, <michael.white@cirrus.com>");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/cs42l42.h b/sound/soc/codecs/cs42l42.h
new file mode 100644
index 000000000000..d87a0a5322d5
--- /dev/null
+++ b/sound/soc/codecs/cs42l42.h
@@ -0,0 +1,776 @@
+/*
+ * cs42l42.h -- CS42L42 ALSA SoC audio driver header
+ *
+ * Copyright 2016 Cirrus Logic, Inc.
+ *
+ * Author: James Schulman <james.schulman@cirrus.com>
+ * Author: Brian Austin <brian.austin@cirrus.com>
+ * Author: Michael White <michael.white@cirrus.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __CS42L42_H__
+#define __CS42L42_H__
+
+#define CS42L42_PAGE_REGISTER	0x00	/* Page Select Register */
+#define CS42L42_WIN_START	0x00
+#define CS42L42_WIN_LEN		0x100
+#define CS42L42_RANGE_MIN	0x00
+#define CS42L42_RANGE_MAX	0x7F
+
+#define CS42L42_PAGE_10		0x1000
+#define CS42L42_PAGE_11		0x1100
+#define CS42L42_PAGE_12		0x1200
+#define CS42L42_PAGE_13		0x1300
+#define CS42L42_PAGE_15		0x1500
+#define CS42L42_PAGE_19		0x1900
+#define CS42L42_PAGE_1B		0x1B00
+#define CS42L42_PAGE_1C		0x1C00
+#define CS42L42_PAGE_1D		0x1D00
+#define CS42L42_PAGE_1F		0x1F00
+#define CS42L42_PAGE_20		0x2000
+#define CS42L42_PAGE_21		0x2100
+#define CS42L42_PAGE_23		0x2300
+#define CS42L42_PAGE_24		0x2400
+#define CS42L42_PAGE_25		0x2500
+#define CS42L42_PAGE_26		0x2600
+#define CS42L42_PAGE_28		0x2800
+#define CS42L42_PAGE_29		0x2900
+#define CS42L42_PAGE_2A		0x2A00
+#define CS42L42_PAGE_30		0x3000
+
+#define CS42L42_CHIP_ID		0x42A42
+
+/* Page 0x10 Global Registers */
+#define CS42L42_DEVID_AB		(CS42L42_PAGE_10 + 0x01)
+#define CS42L42_DEVID_CD		(CS42L42_PAGE_10 + 0x02)
+#define CS42L42_DEVID_E			(CS42L42_PAGE_10 + 0x03)
+#define CS42L42_FABID			(CS42L42_PAGE_10 + 0x04)
+#define CS42L42_REVID			(CS42L42_PAGE_10 + 0x05)
+#define CS42L42_FRZ_CTL			(CS42L42_PAGE_10 + 0x06)
+
+#define CS42L42_SRC_CTL			(CS42L42_PAGE_10 + 0x07)
+#define CS42L42_SRC_BYPASS_DAC_SHIFT	1
+#define CS42L42_SRC_BYPASS_DAC_MASK	(1 << CS42L42_SRC_BYPASS_DAC_SHIFT)
+
+#define CS42L42_MCLK_STATUS		(CS42L42_PAGE_10 + 0x08)
+
+#define CS42L42_MCLK_CTL		(CS42L42_PAGE_10 + 0x09)
+#define CS42L42_INTERNAL_FS_SHIFT	1
+#define CS42L42_INTERNAL_FS_MASK	(1 << CS42L42_INTERNAL_FS_SHIFT)
+
+#define CS42L42_SFTRAMP_RATE		(CS42L42_PAGE_10 + 0x0A)
+#define CS42L42_I2C_DEBOUNCE		(CS42L42_PAGE_10 + 0x0E)
+#define CS42L42_I2C_STRETCH		(CS42L42_PAGE_10 + 0x0F)
+#define CS42L42_I2C_TIMEOUT		(CS42L42_PAGE_10 + 0x10)
+
+/* Page 0x11 Power and Headset Detect Registers */
+#define CS42L42_PWR_CTL1		(CS42L42_PAGE_11 + 0x01)
+#define CS42L42_ASP_DAO_PDN_SHIFT	7
+#define CS42L42_ASP_DAO_PDN_MASK	(1 << CS42L42_ASP_DAO_PDN_SHIFT)
+#define CS42L42_ASP_DAI_PDN_SHIFT	6
+#define CS42L42_ASP_DAI_PDN_MASK	(1 << CS42L42_ASP_DAI_PDN_SHIFT)
+#define CS42L42_MIXER_PDN_SHIFT		5
+#define CS42L42_MIXER_PDN_MASK		(1 << CS42L42_MIXER_PDN_SHIFT)
+#define CS42L42_EQ_PDN_SHIFT		4
+#define CS42L42_EQ_PDN_MASK		(1 << CS42L42_EQ_PDN_SHIFT)
+#define CS42L42_HP_PDN_SHIFT		3
+#define CS42L42_HP_PDN_MASK		(1 << CS42L42_HP_PDN_SHIFT)
+#define CS42L42_ADC_PDN_SHIFT		2
+#define CS42L42_ADC_PDN_MASK		(1 << CS42L42_HP_PDN_SHIFT)
+#define CS42L42_PDN_ALL_SHIFT		0
+#define CS42L42_PDN_ALL_MASK		(1 << CS42L42_PDN_ALL_SHIFT)
+
+#define CS42L42_PWR_CTL2		(CS42L42_PAGE_11 + 0x02)
+#define CS42L42_ADC_SRC_PDNB_SHIFT	0
+#define CS42L42_ADC_SRC_PDNB_MASK	(1 << CS42L42_ADC_SRC_PDNB_SHIFT)
+#define CS42L42_DAC_SRC_PDNB_SHIFT	1
+#define CS42L42_DAC_SRC_PDNB_MASK	(1 << CS42L42_DAC_SRC_PDNB_SHIFT)
+#define CS42L42_ASP_DAI1_PDN_SHIFT	2
+#define CS42L42_ASP_DAI1_PDN_MASK	(1 << CS42L42_ASP_DAI1_PDN_SHIFT)
+#define CS42L42_SRC_PDN_OVERRIDE_SHIFT	3
+#define CS42L42_SRC_PDN_OVERRIDE_MASK	(1 << CS42L42_SRC_PDN_OVERRIDE_SHIFT)
+#define CS42L42_DISCHARGE_FILT_SHIFT	4
+#define CS42L42_DISCHARGE_FILT_MASK	(1 << CS42L42_DISCHARGE_FILT_SHIFT)
+
+#define CS42L42_PWR_CTL3			(CS42L42_PAGE_11 + 0x03)
+#define CS42L42_RING_SENSE_PDNB_SHIFT		1
+#define CS42L42_RING_SENSE_PDNB_MASK		(1 << \
+					CS42L42_RING_SENSE_PDNB_SHIFT)
+#define CS42L42_VPMON_PDNB_SHIFT		2
+#define CS42L42_VPMON_PDNB_MASK			(1 << \
+					CS42L42_VPMON_PDNB_SHIFT)
+#define CS42L42_SW_CLK_STP_STAT_SEL_SHIFT	5
+#define CS42L42_SW_CLK_STP_STAT_SEL_MASK	(3 << \
+					CS42L42_SW_CLK_STP_STAT_SEL_SHIFT)
+
+#define CS42L42_RSENSE_CTL1			(CS42L42_PAGE_11 + 0x04)
+#define CS42L42_RS_TRIM_R_SHIFT			0
+#define CS42L42_RS_TRIM_R_MASK			(1 << \
+					CS42L42_RS_TRIM_R_SHIFT)
+#define CS42L42_RS_TRIM_T_SHIFT			1
+#define CS42L42_RS_TRIM_T_MASK			(1 << \
+					CS42L42_RS_TRIM_T_SHIFT)
+#define CS42L42_HPREF_RS_SHIFT			2
+#define CS42L42_HPREF_RS_MASK			(1 << \
+					CS42L42_HPREF_RS_SHIFT)
+#define CS42L42_HSBIAS_FILT_REF_RS_SHIFT	3
+#define CS42L42_HSBIAS_FILT_REF_RS_MASK		(1 << \
+					CS42L42_HSBIAS_FILT_REF_RS_SHIFT)
+#define CS42L42_RING_SENSE_PU_HIZ_SHIFT		6
+#define CS42L42_RING_SENSE_PU_HIZ_MASK		(1 << \
+					CS42L42_RING_SENSE_PU_HIZ_SHIFT)
+
+#define CS42L42_RSENSE_CTL2		(CS42L42_PAGE_11 + 0x05)
+#define CS42L42_TS_RS_GATE_SHIFT	7
+#define CS42L42_TS_RS_GATE_MAS		(1 << CS42L42_TS_RS_GATE_SHIFT)
+
+#define CS42L42_OSC_SWITCH		(CS42L42_PAGE_11 + 0x07)
+#define CS42L42_SCLK_PRESENT_SHIFT	0
+#define CS42L42_SCLK_PRESENT_MASK	(1 << CS42L42_SCLK_PRESENT_SHIFT)
+
+#define CS42L42_OSC_SWITCH_STATUS	(CS42L42_PAGE_11 + 0x09)
+#define CS42L42_OSC_SW_SEL_STAT_SHIFT	0
+#define CS42L42_OSC_SW_SEL_STAT_MASK	(3 << CS42L42_OSC_SW_SEL_STAT_SHIFT)
+#define CS42L42_OSC_PDNB_STAT_SHIFT	2
+#define CS42L42_OSC_PDNB_STAT_MASK	(1 << CS42L42_OSC_SW_SEL_STAT_SHIFT)
+
+#define CS42L42_RSENSE_CTL3			(CS42L42_PAGE_11 + 0x12)
+#define CS42L42_RS_RISE_DBNCE_TIME_SHIFT	0
+#define CS42L42_RS_RISE_DBNCE_TIME_MASK		(7 << \
+					CS42L42_RS_RISE_DBNCE_TIME_SHIFT)
+#define CS42L42_RS_FALL_DBNCE_TIME_SHIFT	3
+#define CS42L42_RS_FALL_DBNCE_TIME_MASK		(7 << \
+					CS42L42_RS_FALL_DBNCE_TIME_SHIFT)
+#define CS42L42_RS_PU_EN_SHIFT			6
+#define CS42L42_RS_PU_EN_MASK			(1 << \
+					CS42L42_RS_PU_EN_SHIFT)
+#define CS42L42_RS_INV_SHIFT			7
+#define CS42L42_RS_INV_MASK			(1 << \
+					CS42L42_RS_INV_SHIFT)
+
+#define CS42L42_TSENSE_CTL			(CS42L42_PAGE_11 + 0x13)
+#define CS42L42_TS_RISE_DBNCE_TIME_SHIFT	0
+#define CS42L42_TS_RISE_DBNCE_TIME_MASK		(7 << \
+					CS42L42_TS_RISE_DBNCE_TIME_SHIFT)
+#define CS42L42_TS_FALL_DBNCE_TIME_SHIFT	3
+#define CS42L42_TS_FALL_DBNCE_TIME_MASK		(7 << \
+					CS42L42_TS_FALL_DBNCE_TIME_SHIFT)
+#define CS42L42_TS_INV_SHIFT			7
+#define CS42L42_TS_INV_MASK			(1 << \
+					CS42L42_TS_INV_SHIFT)
+
+#define CS42L42_TSRS_INT_DISABLE	(CS42L42_PAGE_11 + 0x14)
+#define CS42L42_D_RS_PLUG_DBNC_SHIFT	0
+#define CS42L42_D_RS_PLUG_DBNC_MASK	(1 << CS42L42_D_RS_PLUG_DBNC_SHIFT)
+#define CS42L42_D_RS_UNPLUG_DBNC_SHIFT	1
+#define CS42L42_D_RS_UNPLUG_DBNC_MASK	(1 << CS42L42_D_RS_UNPLUG_DBNC_SHIFT)
+#define CS42L42_D_TS_PLUG_DBNC_SHIFT	2
+#define CS42L42_D_TS_PLUG_DBNC_MASK	(1 << CS42L42_D_TS_PLUG_DBNC_SHIFT)
+#define CS42L42_D_TS_UNPLUG_DBNC_SHIFT	3
+#define CS42L42_D_TS_UNPLUG_DBNC_MASK	(1 << CS42L42_D_TS_UNPLUG_DBNC_SHIFT)
+
+#define CS42L42_TRSENSE_STATUS		(CS42L42_PAGE_11 + 0x15)
+#define CS42L42_RS_PLUG_DBNC_SHIFT	0
+#define CS42L42_RS_PLUG_DBNC_MASK	(1 << CS42L42_RS_PLUG_DBNC_SHIFT)
+#define CS42L42_RS_UNPLUG_DBNC_SHIFT	1
+#define CS42L42_RS_UNPLUG_DBNC_MASK	(1 << CS42L42_RS_UNPLUG_DBNC_SHIFT)
+#define CS42L42_TS_PLUG_DBNC_SHIFT	2
+#define CS42L42_TS_PLUG_DBNC_MASK	(1 << CS42L42_TS_PLUG_DBNC_SHIFT)
+#define CS42L42_TS_UNPLUG_DBNC_SHIFT	3
+#define CS42L42_TS_UNPLUG_DBNC_MASK	(1 << CS42L42_TS_UNPLUG_DBNC_SHIFT)
+
+#define CS42L42_HSDET_CTL1		(CS42L42_PAGE_11 + 0x1F)
+#define CS42L42_HSDET_COMP1_LVL_SHIFT	0
+#define CS42L42_HSDET_COMP1_LVL_MASK	(15 << CS42L42_HSDET_COMP1_LVL_SHIFT)
+#define CS42L42_HSDET_COMP2_LVL_SHIFT	4
+#define CS42L42_HSDET_COMP2_LVL_MASK	(15 << CS42L42_HSDET_COMP2_LVL_SHIFT)
+
+#define CS42L42_HSDET_CTL2		(CS42L42_PAGE_11 + 0x20)
+#define CS42L42_HSDET_AUTO_TIME_SHIFT	0
+#define CS42L42_HSDET_AUTO_TIME_MASK	(3 << CS42L42_HSDET_AUTO_TIME_SHIFT)
+#define CS42L42_HSBIAS_REF_SHIFT	3
+#define CS42L42_HSBIAS_REF_MASK		(1 << CS42L42_HSBIAS_REF_SHIFT)
+#define CS42L42_HSDET_SET_SHIFT		4
+#define CS42L42_HSDET_SET_MASK		(3 << CS42L42_HSDET_SET_SHIFT)
+#define CS42L42_HSDET_CTRL_SHIFT	6
+#define CS42L42_HSDET_CTRL_MASK		(3 << CS42L42_HSDET_CTRL_SHIFT)
+
+#define CS42L42_HS_SWITCH_CTL		(CS42L42_PAGE_11 + 0x21)
+#define CS42L42_SW_GNDHS_HS4_SHIFT	0
+#define CS42L42_SW_GNDHS_HS4_MASK	(1 << CS42L42_SW_GNDHS_HS4_SHIFT)
+#define CS42L42_SW_GNDHS_HS3_SHIFT	1
+#define CS42L42_SW_GNDHS_HS3_MASK	(1 << CS42L42_SW_GNDHS_HS3_SHIFT)
+#define CS42L42_SW_HSB_HS4_SHIFT	2
+#define CS42L42_SW_HSB_HS4_MASK		(1 << CS42L42_SW_HSB_HS4_SHIFT)
+#define CS42L42_SW_HSB_HS3_SHIFT	3
+#define CS42L42_SW_HSB_HS3_MASK		(1 << CS42L42_SW_HSB_HS3_SHIFT)
+#define CS42L42_SW_HSB_FILT_HS4_SHIFT	4
+#define CS42L42_SW_HSB_FILT_HS4_MASK	(1 << CS42L42_SW_HSB_FILT_HS4_SHIFT)
+#define CS42L42_SW_HSB_FILT_HS3_SHIFT	5
+#define CS42L42_SW_HSB_FILT_HS3_MASK	(1 << CS42L42_SW_HSB_FILT_HS3_SHIFT)
+#define CS42L42_SW_REF_HS4_SHIFT	6
+#define CS42L42_SW_REF_HS4_MASK		(1 << CS42L42_SW_REF_HS4_SHIFT)
+#define CS42L42_SW_REF_HS3_SHIFT	7
+#define CS42L42_SW_REF_HS3_MASK		(1 << CS42L42_SW_REF_HS3_SHIFT)
+
+#define CS42L42_HS_DET_STATUS		(CS42L42_PAGE_11 + 0x24)
+#define CS42L42_HSDET_TYPE_SHIFT	0
+#define CS42L42_HSDET_TYPE_MASK		(3 << CS42L42_HSDET_TYPE_SHIFT)
+#define CS42L42_HSDET_COMP1_OUT_SHIFT	6
+#define CS42L42_HSDET_COMP1_OUT_MASK	(1 << CS42L42_HSDET_COMP1_OUT_SHIFT)
+#define CS42L42_HSDET_COMP2_OUT_SHIFT	7
+#define CS42L42_HSDET_COMP2_OUT_MASK	(1 << CS42L42_HSDET_COMP2_OUT_SHIFT)
+#define CS42L42_PLUG_CTIA		0
+#define CS42L42_PLUG_OMTP		1
+#define CS42L42_PLUG_HEADPHONE		2
+#define CS42L42_PLUG_INVALID		3
+
+#define CS42L42_HS_CLAMP_DISABLE	(CS42L42_PAGE_11 + 0x29)
+#define CS42L42_HS_CLAMP_DISABLE_SHIFT	0
+#define CS42L42_HS_CLAMP_DISABLE_MASK	(1 << CS42L42_HS_CLAMP_DISABLE_SHIFT)
+
+/* Page 0x12 Clocking Registers */
+#define CS42L42_MCLK_SRC_SEL		(CS42L42_PAGE_12 + 0x01)
+#define CS42L42_MCLKDIV_SHIFT		1
+#define CS42L42_MCLKDIV_MASK		(1 << CS42L42_MCLKDIV_SHIFT)
+#define CS42L42_MCLK_SRC_SEL_SHIFT	0
+#define CS42L42_MCLK_SRC_SEL_MASK	(1 << CS42L42_MCLK_SRC_SEL_SHIFT)
+
+#define CS42L42_SPDIF_CLK_CFG		(CS42L42_PAGE_12 + 0x02)
+#define CS42L42_FSYNC_PW_LOWER		(CS42L42_PAGE_12 + 0x03)
+
+#define CS42L42_FSYNC_PW_UPPER			(CS42L42_PAGE_12 + 0x04)
+#define CS42L42_FSYNC_PULSE_WIDTH_SHIFT		0
+#define CS42L42_FSYNC_PULSE_WIDTH_MASK		(0xff << \
+					CS42L42_FSYNC_PULSE_WIDTH_SHIFT)
+
+#define CS42L42_FSYNC_P_LOWER		(CS42L42_PAGE_12 + 0x05)
+
+#define CS42L42_FSYNC_P_UPPER		(CS42L42_PAGE_12 + 0x06)
+#define CS42L42_FSYNC_PERIOD_SHIFT	0
+#define CS42L42_FSYNC_PERIOD_MASK	(0xff << CS42L42_FSYNC_PERIOD_SHIFT)
+
+#define CS42L42_ASP_CLK_CFG		(CS42L42_PAGE_12 + 0x07)
+#define CS42L42_ASP_SCLK_EN_SHIFT	5
+#define CS42L42_ASP_SCLK_EN_MASK	(1 << CS42L42_ASP_SCLK_EN_SHIFT)
+#define CS42L42_ASP_MASTER_MODE		0x01
+#define CS42L42_ASP_SLAVE_MODE		0x00
+#define CS42L42_ASP_MODE_SHIFT		4
+#define CS42L42_ASP_MODE_MASK		(1 << CS42L42_ASP_MODE_SHIFT)
+#define CS42L42_ASP_SCPOL_IN_DAC_SHIFT	2
+#define CS42L42_ASP_SCPOL_IN_DAC_MASK	(1 << CS42L42_ASP_SCPOL_IN_DAC_SHIFT)
+#define CS42L42_ASP_LCPOL_IN_SHIFT	0
+#define CS42L42_ASP_LCPOL_IN_MASK	(1 << CS42L42_ASP_LCPOL_IN_SHIFT)
+#define CS42L42_ASP_POL_INV		1
+
+#define CS42L42_ASP_FRM_CFG		(CS42L42_PAGE_12 + 0x08)
+#define CS42L42_ASP_STP_SHIFT		4
+#define CS42L42_ASP_STP_MASK		(1 << CS42L42_ASP_STP_SHIFT)
+#define CS42L42_ASP_5050_SHIFT		3
+#define CS42L42_ASP_5050_MASK		(1 << CS42L42_ASP_5050_SHIFT)
+#define CS42L42_ASP_FSD_SHIFT		0
+#define CS42L42_ASP_FSD_MASK		(7 << CS42L42_ASP_FSD_SHIFT)
+#define CS42L42_ASP_FSD_0_5		1
+#define CS42L42_ASP_FSD_1_0		2
+#define CS42L42_ASP_FSD_1_5		3
+#define CS42L42_ASP_FSD_2_0		4
+
+#define CS42L42_FS_RATE_EN		(CS42L42_PAGE_12 + 0x09)
+#define CS42L42_FS_EN_SHIFT		0
+#define CS42L42_FS_EN_MASK		(0xf << CS42L42_FS_EN_SHIFT)
+#define CS42L42_FS_EN_IASRC_96K		0x1
+#define CS42L42_FS_EN_OASRC_96K		0x2
+
+#define CS42L42_IN_ASRC_CLK		(CS42L42_PAGE_12 + 0x0A)
+#define CS42L42_CLK_IASRC_SEL_SHIFT	0
+#define CS42L42_CLK_IASRC_SEL_MASK	(1 << CS42L42_CLK_IASRC_SEL_SHIFT)
+#define CS42L42_CLK_IASRC_SEL_12	1
+
+#define CS42L42_OUT_ASRC_CLK		(CS42L42_PAGE_12 + 0x0B)
+#define CS42L42_CLK_OASRC_SEL_SHIFT	0
+#define CS42L42_CLK_OASRC_SEL_MASK	(1 << CS42L42_CLK_OASRC_SEL_SHIFT)
+#define CS42L42_CLK_OASRC_SEL_12	1
+
+#define CS42L42_PLL_DIV_CFG1		(CS42L42_PAGE_12 + 0x0C)
+#define CS42L42_SCLK_PREDIV_SHIFT	0
+#define CS42L42_SCLK_PREDIV_MASK	(3 << CS42L42_SCLK_PREDIV_SHIFT)
+
+/* Page 0x13 Interrupt Registers */
+/* Interrupts */
+#define CS42L42_ADC_OVFL_STATUS		(CS42L42_PAGE_13 + 0x01)
+#define CS42L42_MIXER_STATUS		(CS42L42_PAGE_13 + 0x02)
+#define CS42L42_SRC_STATUS		(CS42L42_PAGE_13 + 0x03)
+#define CS42L42_ASP_RX_STATUS		(CS42L42_PAGE_13 + 0x04)
+#define CS42L42_ASP_TX_STATUS		(CS42L42_PAGE_13 + 0x05)
+#define CS42L42_CODEC_STATUS		(CS42L42_PAGE_13 + 0x08)
+#define CS42L42_DET_INT_STATUS1		(CS42L42_PAGE_13 + 0x09)
+#define CS42L42_DET_INT_STATUS2		(CS42L42_PAGE_13 + 0x0A)
+#define CS42L42_SRCPL_INT_STATUS	(CS42L42_PAGE_13 + 0x0B)
+#define CS42L42_VPMON_STATUS		(CS42L42_PAGE_13 + 0x0D)
+#define CS42L42_PLL_LOCK_STATUS		(CS42L42_PAGE_13 + 0x0E)
+#define CS42L42_TSRS_PLUG_STATUS	(CS42L42_PAGE_13 + 0x0F)
+/* Masks */
+#define CS42L42_ADC_OVFL_INT_MASK	(CS42L42_PAGE_13 + 0x16)
+#define CS42L42_ADC_OVFL_SHIFT		0
+#define CS42L42_ADC_OVFL_MASK		(1 << CS42L42_ADC_OVFL_SHIFT)
+#define CS42L42_ADC_OVFL_VAL_MASK	CS42L42_ADC_OVFL_MASK
+
+#define CS42L42_MIXER_INT_MASK		(CS42L42_PAGE_13 + 0x17)
+#define CS42L42_MIX_CHB_OVFL_SHIFT	0
+#define CS42L42_MIX_CHB_OVFL_MASK	(1 << CS42L42_MIX_CHB_OVFL_SHIFT)
+#define CS42L42_MIX_CHA_OVFL_SHIFT	1
+#define CS42L42_MIX_CHA_OVFL_MASK	(1 << CS42L42_MIX_CHA_OVFL_SHIFT)
+#define CS42L42_EQ_OVFL_SHIFT		2
+#define CS42L42_EQ_OVFL_MASK		(1 << CS42L42_EQ_OVFL_SHIFT)
+#define CS42L42_EQ_BIQUAD_OVFL_SHIFT	3
+#define CS42L42_EQ_BIQUAD_OVFL_MASK	(1 << CS42L42_EQ_BIQUAD_OVFL_SHIFT)
+#define CS42L42_MIXER_VAL_MASK		(CS42L42_MIX_CHB_OVFL_MASK | \
+					CS42L42_MIX_CHA_OVFL_MASK | \
+					CS42L42_EQ_OVFL_MASK | \
+					CS42L42_EQ_BIQUAD_OVFL_MASK)
+
+#define CS42L42_SRC_INT_MASK		(CS42L42_PAGE_13 + 0x18)
+#define CS42L42_SRC_ILK_SHIFT		0
+#define CS42L42_SRC_ILK_MASK		(1 << CS42L42_SRC_ILK_SHIFT)
+#define CS42L42_SRC_OLK_SHIFT		1
+#define CS42L42_SRC_OLK_MASK		(1 << CS42L42_SRC_OLK_SHIFT)
+#define CS42L42_SRC_IUNLK_SHIFT		2
+#define CS42L42_SRC_IUNLK_MASK		(1 << CS42L42_SRC_IUNLK_SHIFT)
+#define CS42L42_SRC_OUNLK_SHIFT		3
+#define CS42L42_SRC_OUNLK_MASK		(1 << CS42L42_SRC_OUNLK_SHIFT)
+#define CS42L42_SRC_VAL_MASK		(CS42L42_SRC_ILK_MASK | \
+					CS42L42_SRC_OLK_MASK | \
+					CS42L42_SRC_IUNLK_MASK | \
+					CS42L42_SRC_OUNLK_MASK)
+
+#define CS42L42_ASP_RX_INT_MASK		(CS42L42_PAGE_13 + 0x19)
+#define CS42L42_ASPRX_NOLRCK_SHIFT	0
+#define CS42L42_ASPRX_NOLRCK_MASK	(1 << CS42L42_ASPRX_NOLRCK_SHIFT)
+#define CS42L42_ASPRX_EARLY_SHIFT	1
+#define CS42L42_ASPRX_EARLY_MASK	(1 << CS42L42_ASPRX_EARLY_SHIFT)
+#define CS42L42_ASPRX_LATE_SHIFT	2
+#define CS42L42_ASPRX_LATE_MASK		(1 << CS42L42_ASPRX_LATE_SHIFT)
+#define CS42L42_ASPRX_ERROR_SHIFT	3
+#define CS42L42_ASPRX_ERROR_MASK	(1 << CS42L42_ASPRX_ERROR_SHIFT)
+#define CS42L42_ASPRX_OVLD_SHIFT	4
+#define CS42L42_ASPRX_OVLD_MASK		(1 << CS42L42_ASPRX_OVLD_SHIFT)
+#define CS42L42_ASP_RX_VAL_MASK		(CS42L42_ASPRX_NOLRCK_MASK | \
+					CS42L42_ASPRX_EARLY_MASK | \
+					CS42L42_ASPRX_LATE_MASK | \
+					CS42L42_ASPRX_ERROR_MASK | \
+					CS42L42_ASPRX_OVLD_MASK)
+
+#define CS42L42_ASP_TX_INT_MASK		(CS42L42_PAGE_13 + 0x1A)
+#define CS42L42_ASPTX_NOLRCK_SHIFT	0
+#define CS42L42_ASPTX_NOLRCK_MASK	(1 << CS42L42_ASPTX_NOLRCK_SHIFT)
+#define CS42L42_ASPTX_EARLY_SHIFT	1
+#define CS42L42_ASPTX_EARLY_MASK	(1 << CS42L42_ASPTX_EARLY_SHIFT)
+#define CS42L42_ASPTX_LATE_SHIFT	2
+#define CS42L42_ASPTX_LATE_MASK		(1 << CS42L42_ASPTX_LATE_SHIFT)
+#define CS42L42_ASPTX_SMERROR_SHIFT	3
+#define CS42L42_ASPTX_SMERROR_MASK	(1 << CS42L42_ASPTX_SMERROR_SHIFT)
+#define CS42L42_ASP_TX_VAL_MASK		(CS42L42_ASPTX_NOLRCK_MASK | \
+					CS42L42_ASPTX_EARLY_MASK | \
+					CS42L42_ASPTX_LATE_MASK | \
+					CS42L42_ASPTX_SMERROR_MASK)
+
+#define CS42L42_CODEC_INT_MASK		(CS42L42_PAGE_13 + 0x1B)
+#define CS42L42_PDN_DONE_SHIFT		0
+#define CS42L42_PDN_DONE_MASK		(1 << CS42L42_PDN_DONE_SHIFT)
+#define CS42L42_HSDET_AUTO_DONE_SHIFT	1
+#define CS42L42_HSDET_AUTO_DONE_MASK	(1 << CS42L42_HSDET_AUTO_DONE_SHIFT)
+#define CS42L42_CODEC_VAL_MASK		(CS42L42_PDN_DONE_MASK | \
+					CS42L42_HSDET_AUTO_DONE_MASK)
+
+#define CS42L42_SRCPL_INT_MASK		(CS42L42_PAGE_13 + 0x1C)
+#define CS42L42_SRCPL_ADC_LK_SHIFT	0
+#define CS42L42_SRCPL_ADC_LK_MASK	(1 << CS42L42_SRCPL_ADC_LK_SHIFT)
+#define CS42L42_SRCPL_DAC_LK_SHIFT	2
+#define CS42L42_SRCPL_DAC_LK_MASK	(1 << CS42L42_SRCPL_DAC_LK_SHIFT)
+#define CS42L42_SRCPL_ADC_UNLK_SHIFT	5
+#define CS42L42_SRCPL_ADC_UNLK_MASK	(1 << CS42L42_SRCPL_ADC_UNLK_SHIFT)
+#define CS42L42_SRCPL_DAC_UNLK_SHIFT	6
+#define CS42L42_SRCPL_DAC_UNLK_MASK	(1 << CS42L42_SRCPL_DAC_UNLK_SHIFT)
+#define CS42L42_SRCPL_VAL_MASK		(CS42L42_SRCPL_ADC_LK_MASK | \
+					CS42L42_SRCPL_DAC_LK_MASK | \
+					CS42L42_SRCPL_ADC_UNLK_MASK | \
+					CS42L42_SRCPL_DAC_UNLK_MASK)
+
+#define CS42L42_VPMON_INT_MASK		(CS42L42_PAGE_13 + 0x1E)
+#define CS42L42_VPMON_SHIFT		0
+#define CS42L42_VPMON_MASK		(1 << CS42L42_VPMON_SHIFT)
+#define CS42L42_VPMON_VAL_MASK		CS42L42_VPMON_MASK
+
+#define CS42L42_PLL_LOCK_INT_MASK	(CS42L42_PAGE_13 + 0x1F)
+#define CS42L42_PLL_LOCK_SHIFT		0
+#define CS42L42_PLL_LOCK_MASK		(1 << CS42L42_PLL_LOCK_SHIFT)
+#define CS42L42_PLL_LOCK_VAL_MASK	CS42L42_PLL_LOCK_MASK
+
+#define CS42L42_TSRS_PLUG_INT_MASK	(CS42L42_PAGE_13 + 0x20)
+#define CS42L42_RS_PLUG_SHIFT		0
+#define CS42L42_RS_PLUG_MASK		(1 << CS42L42_RS_PLUG_SHIFT)
+#define CS42L42_RS_UNPLUG_SHIFT		1
+#define CS42L42_RS_UNPLUG_MASK		(1 << CS42L42_RS_UNPLUG_SHIFT)
+#define CS42L42_TS_PLUG_SHIFT		2
+#define CS42L42_TS_PLUG_MASK		(1 << CS42L42_TS_PLUG_SHIFT)
+#define CS42L42_TS_UNPLUG_SHIFT		3
+#define CS42L42_TS_UNPLUG_MASK		(1 << CS42L42_TS_UNPLUG_SHIFT)
+#define CS42L42_TSRS_PLUG_VAL_MASK	(CS42L42_RS_PLUG_MASK | \
+					CS42L42_RS_UNPLUG_MASK | \
+					CS42L42_TS_PLUG_MASK | \
+					CS42L42_TS_UNPLUG_MASK)
+#define CS42L42_TS_PLUG			3
+#define CS42L42_TS_UNPLUG		0
+#define CS42L42_TS_TRANS		1
+
+/* Page 0x15 Fractional-N PLL Registers */
+#define CS42L42_PLL_CTL1		(CS42L42_PAGE_15 + 0x01)
+#define CS42L42_PLL_START_SHIFT		0
+#define CS42L42_PLL_START_MASK		(1 << CS42L42_PLL_START_SHIFT)
+
+#define CS42L42_PLL_DIV_FRAC0		(CS42L42_PAGE_15 + 0x02)
+#define CS42L42_PLL_DIV_FRAC_SHIFT	0
+#define CS42L42_PLL_DIV_FRAC_MASK	(0xff << CS42L42_PLL_DIV_FRAC_SHIFT)
+
+#define CS42L42_PLL_DIV_FRAC1		(CS42L42_PAGE_15 + 0x03)
+#define CS42L42_PLL_DIV_FRAC2		(CS42L42_PAGE_15 + 0x04)
+
+#define CS42L42_PLL_DIV_INT		(CS42L42_PAGE_15 + 0x05)
+#define CS42L42_PLL_DIV_INT_SHIFT	0
+#define CS42L42_PLL_DIV_INT_MASK	(0xff << CS42L42_PLL_DIV_INT_SHIFT)
+
+#define CS42L42_PLL_CTL3		(CS42L42_PAGE_15 + 0x08)
+#define CS42L42_PLL_DIVOUT_SHIFT	0
+#define CS42L42_PLL_DIVOUT_MASK		(0xff << CS42L42_PLL_DIVOUT_SHIFT)
+
+#define CS42L42_PLL_CAL_RATIO		(CS42L42_PAGE_15 + 0x0A)
+#define CS42L42_PLL_CAL_RATIO_SHIFT	0
+#define CS42L42_PLL_CAL_RATIO_MASK	(0xff << CS42L42_PLL_CAL_RATIO_SHIFT)
+
+#define CS42L42_PLL_CTL4		(CS42L42_PAGE_15 + 0x1B)
+#define CS42L42_PLL_MODE_SHIFT		0
+#define CS42L42_PLL_MODE_MASK		(3 << CS42L42_PLL_MODE_SHIFT)
+
+/* Page 0x19 HP Load Detect Registers */
+#define CS42L42_LOAD_DET_RCSTAT		(CS42L42_PAGE_19 + 0x25)
+#define CS42L42_RLA_STAT_SHIFT		0
+#define CS42L42_RLA_STAT_MASK		(3 << CS42L42_RLA_STAT_SHIFT)
+#define CS42L42_RLA_STAT_15_OHM		0
+
+#define CS42L42_LOAD_DET_DONE		(CS42L42_PAGE_19 + 0x26)
+#define CS42L42_HPLOAD_DET_DONE_SHIFT	0
+#define CS42L42_HPLOAD_DET_DONE_MASK	(1 << CS42L42_HPLOAD_DET_DONE_SHIFT)
+
+#define CS42L42_LOAD_DET_EN		(CS42L42_PAGE_19 + 0x27)
+#define CS42L42_HP_LD_EN_SHIFT		0
+#define CS42L42_HP_LD_EN_MASK		(1 << CS42L42_HP_LD_EN_SHIFT)
+
+/* Page 0x1B Headset Interface Registers */
+#define CS42L42_HSBIAS_SC_AUTOCTL		(CS42L42_PAGE_1B + 0x70)
+#define CS42L42_HSBIAS_SENSE_TRIP_SHIFT		0
+#define CS42L42_HSBIAS_SENSE_TRIP_MASK		(7 << \
+					CS42L42_HSBIAS_SENSE_TRIP_SHIFT)
+#define CS42L42_TIP_SENSE_EN_SHIFT		5
+#define CS42L42_TIP_SENSE_EN_MASK		(1 << \
+					CS42L42_TIP_SENSE_EN_SHIFT)
+#define CS42L42_AUTO_HSBIAS_HIZ_SHIFT		6
+#define CS42L42_AUTO_HSBIAS_HIZ_MASK		(1 << \
+					CS42L42_AUTO_HSBIAS_HIZ_SHIFT)
+#define CS42L42_HSBIAS_SENSE_EN_SHIFT		7
+#define CS42L42_HSBIAS_SENSE_EN_MASK		(1 << \
+					CS42L42_HSBIAS_SENSE_EN_SHIFT)
+
+#define CS42L42_WAKE_CTL		(CS42L42_PAGE_1B + 0x71)
+#define CS42L42_WAKEB_CLEAR_SHIFT	0
+#define CS42L42_WAKEB_CLEAR_MASK	(1 << CS42L42_WAKEB_CLEAR_SHIFT)
+#define CS42L42_WAKEB_MODE_SHIFT	5
+#define CS42L42_WAKEB_MODE_MASK		(1 << CS42L42_WAKEB_MODE_SHIFT)
+#define CS42L42_M_HP_WAKE_SHIFT		6
+#define CS42L42_M_HP_WAKE_MASK		(1 << CS42L42_M_HP_WAKE_SHIFT)
+#define CS42L42_M_MIC_WAKE_SHIFT	7
+#define CS42L42_M_MIC_WAKE_MASK		(1 << CS42L42_M_MIC_WAKE_SHIFT)
+
+#define CS42L42_ADC_DISABLE_MUTE		(CS42L42_PAGE_1B + 0x72)
+#define CS42L42_ADC_DISABLE_S0_MUTE_SHIFT	7
+#define CS42L42_ADC_DISABLE_S0_MUTE_MASK	(1 << \
+					CS42L42_ADC_DISABLE_S0_MUTE_SHIFT)
+
+#define CS42L42_TIPSENSE_CTL			(CS42L42_PAGE_1B + 0x73)
+#define CS42L42_TIP_SENSE_DEBOUNCE_SHIFT	0
+#define CS42L42_TIP_SENSE_DEBOUNCE_MASK		(3 << \
+					CS42L42_TIP_SENSE_DEBOUNCE_SHIFT)
+#define CS42L42_TIP_SENSE_INV_SHIFT		5
+#define CS42L42_TIP_SENSE_INV_MASK		(1 << \
+					CS42L42_TIP_SENSE_INV_SHIFT)
+#define CS42L42_TIP_SENSE_CTRL_SHIFT		6
+#define CS42L42_TIP_SENSE_CTRL_MASK		(3 << \
+					CS42L42_TIP_SENSE_CTRL_SHIFT)
+
+#define CS42L42_MISC_DET_CTL		(CS42L42_PAGE_1B + 0x74)
+#define CS42L42_PDN_MIC_LVL_DET_SHIFT	0
+#define CS42L42_PDN_MIC_LVL_DET_MASK	(1 << CS42L42_PDN_MIC_LVL_DET_SHIFT)
+#define CS42L42_HSBIAS_CTL_SHIFT	1
+#define CS42L42_HSBIAS_CTL_MASK		(3 << CS42L42_HSBIAS_CTL_SHIFT)
+#define CS42L42_DETECT_MODE_SHIFT	3
+#define CS42L42_DETECT_MODE_MASK	(3 << CS42L42_DETECT_MODE_SHIFT)
+
+#define CS42L42_MIC_DET_CTL1		(CS42L42_PAGE_1B + 0x75)
+#define CS42L42_HS_DET_LEVEL_SHIFT	0
+#define CS42L42_HS_DET_LEVEL_MASK	(0x3F << CS42L42_HS_DET_LEVEL_SHIFT)
+#define CS42L42_EVENT_STAT_SEL_SHIFT	6
+#define CS42L42_EVENT_STAT_SEL_MASK	(1 << CS42L42_EVENT_STAT_SEL_SHIFT)
+#define CS42L42_LATCH_TO_VP_SHIFT	7
+#define CS42L42_LATCH_TO_VP_MASK	(1 << CS42L42_LATCH_TO_VP_SHIFT)
+
+#define CS42L42_MIC_DET_CTL2		(CS42L42_PAGE_1B + 0x76)
+#define CS42L42_DEBOUNCE_TIME_SHIFT	5
+#define CS42L42_DEBOUNCE_TIME_MASK	(0x07 << CS42L42_DEBOUNCE_TIME_SHIFT)
+
+#define CS42L42_DET_STATUS1		(CS42L42_PAGE_1B + 0x77)
+#define CS42L42_HSBIAS_HIZ_MODE_SHIFT	6
+#define CS42L42_HSBIAS_HIZ_MODE_MASK	(1 << CS42L42_HSBIAS_HIZ_MODE_SHIFT)
+#define CS42L42_TIP_SENSE_SHIFT		7
+#define CS42L42_TIP_SENSE_MASK		(1 << CS42L42_TIP_SENSE_SHIFT)
+
+#define CS42L42_DET_STATUS2		(CS42L42_PAGE_1B + 0x78)
+#define CS42L42_SHORT_TRUE_SHIFT	0
+#define CS42L42_SHORT_TRUE_MASK		(1 << CS42L42_SHORT_TRUE_SHIFT)
+#define CS42L42_HS_TRUE_SHIFT	1
+#define CS42L42_HS_TRUE_MASK		(1 << CS42L42_HS_TRUE_SHIFT)
+
+#define CS42L42_DET_INT1_MASK		(CS42L42_PAGE_1B + 0x79)
+#define CS42L42_TIP_SENSE_UNPLUG_SHIFT	5
+#define CS42L42_TIP_SENSE_UNPLUG_MASK	(1 << CS42L42_TIP_SENSE_UNPLUG_SHIFT)
+#define CS42L42_TIP_SENSE_PLUG_SHIFT	6
+#define CS42L42_TIP_SENSE_PLUG_MASK	(1 << CS42L42_TIP_SENSE_PLUG_SHIFT)
+#define CS42L42_HSBIAS_SENSE_SHIFT	7
+#define CS42L42_HSBIAS_SENSE_MASK	(1 << CS42L42_HSBIAS_SENSE_SHIFT)
+#define CS42L42_DET_INT_VAL1_MASK	(CS42L42_TIP_SENSE_UNPLUG_MASK | \
+					CS42L42_TIP_SENSE_PLUG_MASK | \
+					CS42L42_HSBIAS_SENSE_MASK)
+
+#define CS42L42_DET_INT2_MASK		(CS42L42_PAGE_1B + 0x7A)
+#define CS42L42_M_SHORT_DET_SHIFT	0
+#define CS42L42_M_SHORT_DET_MASK	(1 << \
+					CS42L42_M_SHORT_DET_SHIFT)
+#define CS42L42_M_SHORT_RLS_SHIFT	1
+#define CS42L42_M_SHORT_RLS_MASK	(1 << \
+					CS42L42_M_SHORT_RLS_SHIFT)
+#define CS42L42_M_HSBIAS_HIZ_SHIFT	2
+#define CS42L42_M_HSBIAS_HIZ_MASK	(1 << \
+					CS42L42_M_HSBIAS_HIZ_SHIFT)
+#define CS42L42_M_DETECT_FT_SHIFT	6
+#define CS42L42_M_DETECT_FT_MASK	(1 << \
+					CS42L42_M_DETECT_FT_SHIFT)
+#define CS42L42_M_DETECT_TF_SHIFT	7
+#define CS42L42_M_DETECT_TF_MASK	(1 << \
+					CS42L42_M_DETECT_TF_SHIFT)
+#define CS42L42_DET_INT_VAL2_MASK	(CS42L42_M_SHORT_DET_MASK | \
+					CS42L42_M_SHORT_RLS_MASK | \
+					CS42L42_M_HSBIAS_HIZ_MASK | \
+					CS42L42_M_DETECT_FT_MASK | \
+					CS42L42_M_DETECT_TF_MASK)
+
+/* Page 0x1C Headset Bias Registers */
+#define CS42L42_HS_BIAS_CTL		(CS42L42_PAGE_1C + 0x03)
+#define CS42L42_HSBIAS_RAMP_SHIFT	0
+#define CS42L42_HSBIAS_RAMP_MASK	(3 << CS42L42_HSBIAS_RAMP_SHIFT)
+#define CS42L42_HSBIAS_PD_SHIFT		4
+#define CS42L42_HSBIAS_PD_MASK		(1 << CS42L42_HSBIAS_PD_SHIFT)
+#define CS42L42_HSBIAS_CAPLESS_SHIFT	7
+#define CS42L42_HSBIAS_CAPLESS_MASK	(1 << CS42L42_HSBIAS_CAPLESS_SHIFT)
+
+/* Page 0x1D ADC Registers */
+#define CS42L42_ADC_CTL			(CS42L42_PAGE_1D + 0x01)
+#define CS42L42_ADC_NOTCH_DIS_SHIFT		5
+#define CS42L42_ADC_FORCE_WEAK_VCM_SHIFT	4
+#define CS42L42_ADC_INV_SHIFT			2
+#define CS42L42_ADC_DIG_BOOST_SHIFT		0
+
+#define CS42L42_ADC_VOLUME		(CS42L42_PAGE_1D + 0x03)
+#define CS42L42_ADC_VOL_SHIFT		0
+
+#define CS42L42_ADC_WNF_HPF_CTL		(CS42L42_PAGE_1D + 0x04)
+#define CS42L42_ADC_WNF_CF_SHIFT	4
+#define CS42L42_ADC_WNF_EN_SHIFT	3
+#define CS42L42_ADC_HPF_CF_SHIFT	1
+#define CS42L42_ADC_HPF_EN_SHIFT	0
+
+/* Page 0x1F DAC Registers */
+#define CS42L42_DAC_CTL1		(CS42L42_PAGE_1F + 0x01)
+#define CS42L42_DACB_INV_SHIFT		1
+#define CS42L42_DACA_INV_SHIFT		0
+
+#define CS42L42_DAC_CTL2		(CS42L42_PAGE_1F + 0x06)
+#define CS42L42_HPOUT_PULLDOWN_SHIFT	4
+#define CS42L42_HPOUT_PULLDOWN_MASK	(15 << CS42L42_HPOUT_PULLDOWN_SHIFT)
+#define CS42L42_HPOUT_LOAD_SHIFT	3
+#define CS42L42_HPOUT_LOAD_MASK		(1 << CS42L42_HPOUT_LOAD_SHIFT)
+#define CS42L42_HPOUT_CLAMP_SHIFT	2
+#define CS42L42_HPOUT_CLAMP_MASK	(1 << CS42L42_HPOUT_CLAMP_SHIFT)
+#define CS42L42_DAC_HPF_EN_SHIFT	1
+#define CS42L42_DAC_HPF_EN_MASK		(1 << CS42L42_DAC_HPF_EN_SHIFT)
+#define CS42L42_DAC_MON_EN_SHIFT	0
+#define CS42L42_DAC_MON_EN_MASK		(1 << CS42L42_DAC_MON_EN_SHIFT)
+
+/* Page 0x20 HP CTL Registers */
+#define CS42L42_HP_CTL			(CS42L42_PAGE_20 + 0x01)
+#define CS42L42_HP_ANA_BMUTE_SHIFT	3
+#define CS42L42_HP_ANA_BMUTE_MASK	(1 << CS42L42_HP_ANA_BMUTE_SHIFT)
+#define CS42L42_HP_ANA_AMUTE_SHIFT	2
+#define CS42L42_HP_ANA_AMUTE_MASK	(1 << CS42L42_HP_ANA_AMUTE_SHIFT)
+#define CS42L42_HP_FULL_SCALE_VOL_SHIFT	1
+#define CS42L42_HP_FULL_SCALE_VOL_MASK	(1 << CS42L42_HP_FULL_SCALE_VOL_SHIFT)
+
+/* Page 0x21 Class H Registers */
+#define CS42L42_CLASSH_CTL		(CS42L42_PAGE_21 + 0x01)
+
+/* Page 0x23 Mixer Volume Registers */
+#define CS42L42_MIXER_CHA_VOL		(CS42L42_PAGE_23 + 0x01)
+#define CS42L42_MIXER_ADC_VOL		(CS42L42_PAGE_23 + 0x02)
+
+#define CS42L42_MIXER_CHB_VOL		(CS42L42_PAGE_23 + 0x03)
+#define CS42L42_MIXER_CH_VOL_SHIFT	0
+#define CS42L42_MIXER_CH_VOL_MASK	(0x3f << CS42L42_MIXER_CH_VOL_SHIFT)
+
+/* Page 0x24 EQ Registers */
+#define CS42L42_EQ_COEF_IN0		(CS42L42_PAGE_24 + 0x01)
+#define CS42L42_EQ_COEF_IN1		(CS42L42_PAGE_24 + 0x02)
+#define CS42L42_EQ_COEF_IN2		(CS42L42_PAGE_24 + 0x03)
+#define CS42L42_EQ_COEF_IN3		(CS42L42_PAGE_24 + 0x04)
+#define CS42L42_EQ_COEF_RW		(CS42L42_PAGE_24 + 0x06)
+#define CS42L42_EQ_COEF_OUT0		(CS42L42_PAGE_24 + 0x07)
+#define CS42L42_EQ_COEF_OUT1		(CS42L42_PAGE_24 + 0x08)
+#define CS42L42_EQ_COEF_OUT2		(CS42L42_PAGE_24 + 0x09)
+#define CS42L42_EQ_COEF_OUT3		(CS42L42_PAGE_24 + 0x0A)
+#define CS42L42_EQ_INIT_STAT		(CS42L42_PAGE_24 + 0x0B)
+#define CS42L42_EQ_START_FILT		(CS42L42_PAGE_24 + 0x0C)
+#define CS42L42_EQ_MUTE_CTL		(CS42L42_PAGE_24 + 0x0E)
+
+/* Page 0x25 Audio Port Registers */
+#define CS42L42_SP_RX_CH_SEL		(CS42L42_PAGE_25 + 0x01)
+
+#define CS42L42_SP_RX_ISOC_CTL		(CS42L42_PAGE_25 + 0x02)
+#define CS42L42_SP_RX_RSYNC_SHIFT	6
+#define CS42L42_SP_RX_RSYNC_MASK	(1 << CS42L42_SP_RX_RSYNC_SHIFT)
+#define CS42L42_SP_RX_NSB_POS_SHIFT	3
+#define CS42L42_SP_RX_NSB_POS_MASK	(7 << CS42L42_SP_RX_NSB_POS_SHIFT)
+#define CS42L42_SP_RX_NFS_NSBB_SHIFT	2
+#define CS42L42_SP_RX_NFS_NSBB_MASK	(1 << CS42L42_SP_RX_NFS_NSBB_SHIFT)
+#define CS42L42_SP_RX_ISOC_MODE_SHIFT	0
+#define CS42L42_SP_RX_ISOC_MODE_MASK	(3 << CS42L42_SP_RX_ISOC_MODE_SHIFT)
+
+#define CS42L42_SP_RX_FS		(CS42L42_PAGE_25 + 0x03)
+#define CS42l42_SPDIF_CH_SEL		(CS42L42_PAGE_25 + 0x04)
+#define CS42L42_SP_TX_ISOC_CTL		(CS42L42_PAGE_25 + 0x05)
+#define CS42L42_SP_TX_FS		(CS42L42_PAGE_25 + 0x06)
+#define CS42L42_SPDIF_SW_CTL1		(CS42L42_PAGE_25 + 0x07)
+
+/* Page 0x26 SRC Registers */
+#define CS42L42_SRC_SDIN_FS		(CS42L42_PAGE_26 + 0x01)
+#define CS42L42_SRC_SDIN_FS_SHIFT	0
+#define CS42L42_SRC_SDIN_FS_MASK	(0x1f << CS42L42_SRC_SDIN_FS_SHIFT)
+
+#define CS42L42_SRC_SDOUT_FS		(CS42L42_PAGE_26 + 0x09)
+
+/* Page 0x28 S/PDIF Registers */
+#define CS42L42_SPDIF_CTL1		(CS42L42_PAGE_28 + 0x01)
+#define CS42L42_SPDIF_CTL2		(CS42L42_PAGE_28 + 0x02)
+#define CS42L42_SPDIF_CTL3		(CS42L42_PAGE_28 + 0x03)
+#define CS42L42_SPDIF_CTL4		(CS42L42_PAGE_28 + 0x04)
+
+/* Page 0x29 Serial Port TX Registers */
+#define CS42L42_ASP_TX_SZ_EN		(CS42L42_PAGE_29 + 0x01)
+#define CS42L42_ASP_TX_CH_EN		(CS42L42_PAGE_29 + 0x02)
+#define CS42L42_ASP_TX_CH_AP_RES	(CS42L42_PAGE_29 + 0x03)
+#define CS42L42_ASP_TX_CH1_BIT_MSB	(CS42L42_PAGE_29 + 0x04)
+#define CS42L42_ASP_TX_CH1_BIT_LSB	(CS42L42_PAGE_29 + 0x05)
+#define CS42L42_ASP_TX_HIZ_DLY_CFG	(CS42L42_PAGE_29 + 0x06)
+#define CS42L42_ASP_TX_CH2_BIT_MSB	(CS42L42_PAGE_29 + 0x0A)
+#define CS42L42_ASP_TX_CH2_BIT_LSB	(CS42L42_PAGE_29 + 0x0B)
+
+/* Page 0x2A Serial Port RX Registers */
+#define CS42L42_ASP_RX_DAI0_EN		(CS42L42_PAGE_2A + 0x01)
+#define CS42L42_ASP_RX0_CH_EN_SHIFT	2
+#define CS42L42_ASP_RX0_CH_EN_MASK	(0xf << CS42L42_ASP_RX0_CH_EN_SHIFT)
+#define CS42L42_ASP_RX0_CH1_EN		1
+#define CS42L42_ASP_RX0_CH2_EN		2
+#define CS42L42_ASP_RX0_CH3_EN		4
+#define CS42L42_ASP_RX0_CH4_EN		8
+
+#define CS42L42_ASP_RX_DAI0_CH1_AP_RES	(CS42L42_PAGE_2A + 0x02)
+#define CS42L42_ASP_RX_DAI0_CH1_BIT_MSB	(CS42L42_PAGE_2A + 0x03)
+#define CS42L42_ASP_RX_DAI0_CH1_BIT_LSB	(CS42L42_PAGE_2A + 0x04)
+#define CS42L42_ASP_RX_DAI0_CH2_AP_RES	(CS42L42_PAGE_2A + 0x05)
+#define CS42L42_ASP_RX_DAI0_CH2_BIT_MSB	(CS42L42_PAGE_2A + 0x06)
+#define CS42L42_ASP_RX_DAI0_CH2_BIT_LSB	(CS42L42_PAGE_2A + 0x07)
+#define CS42L42_ASP_RX_DAI0_CH3_AP_RES	(CS42L42_PAGE_2A + 0x08)
+#define CS42L42_ASP_RX_DAI0_CH3_BIT_MSB	(CS42L42_PAGE_2A + 0x09)
+#define CS42L42_ASP_RX_DAI0_CH3_BIT_LSB	(CS42L42_PAGE_2A + 0x0A)
+#define CS42L42_ASP_RX_DAI0_CH4_AP_RES	(CS42L42_PAGE_2A + 0x0B)
+#define CS42L42_ASP_RX_DAI0_CH4_BIT_MSB	(CS42L42_PAGE_2A + 0x0C)
+#define CS42L42_ASP_RX_DAI0_CH4_BIT_LSB	(CS42L42_PAGE_2A + 0x0D)
+#define CS42L42_ASP_RX_DAI1_CH1_AP_RES	(CS42L42_PAGE_2A + 0x0E)
+#define CS42L42_ASP_RX_DAI1_CH1_BIT_MSB	(CS42L42_PAGE_2A + 0x0F)
+#define CS42L42_ASP_RX_DAI1_CH1_BIT_LSB	(CS42L42_PAGE_2A + 0x10)
+#define CS42L42_ASP_RX_DAI1_CH2_AP_RES	(CS42L42_PAGE_2A + 0x11)
+#define CS42L42_ASP_RX_DAI1_CH2_BIT_MSB	(CS42L42_PAGE_2A + 0x12)
+#define CS42L42_ASP_RX_DAI1_CH2_BIT_LSB	(CS42L42_PAGE_2A + 0x13)
+
+#define CS42L42_ASP_RX_CH_AP_SHIFT	6
+#define CS42L42_ASP_RX_CH_AP_MASK	(1 << CS42L42_ASP_RX_CH_AP_SHIFT)
+#define CS42L42_ASP_RX_CH_AP_LOW	0
+#define CS42L42_ASP_RX_CH_AP_HI		1
+#define CS42L42_ASP_RX_CH_RES_SHIFT	0
+#define CS42L42_ASP_RX_CH_RES_MASK	(3 << CS42L42_ASP_RX_CH_RES_SHIFT)
+#define CS42L42_ASP_RX_CH_RES_32	3
+#define CS42L42_ASP_RX_CH_RES_16	1
+#define CS42L42_ASP_RX_CH_BIT_ST_SHIFT	0
+#define CS42L42_ASP_RX_CH_BIT_ST_MASK	(0xff << CS42L42_ASP_RX_CH_BIT_ST_SHIFT)
+
+/* Page 0x30 ID Registers */
+#define CS42L42_SUB_REVID		(CS42L42_PAGE_30 + 0x14)
+#define CS42L42_MAX_REGISTER		(CS42L42_PAGE_30 + 0x14)
+
+/* Defines for fracturing values spread across multiple registers */
+#define CS42L42_FRAC0_VAL(val)	((val) & 0x0000ff)
+#define CS42L42_FRAC1_VAL(val)	(((val) & 0x00ff00) >> 8)
+#define CS42L42_FRAC2_VAL(val)	(((val) & 0xff0000) >> 16)
+
+#define CS42L42_NUM_SUPPLIES	5
+
+static const char *const cs42l42_supply_names[CS42L42_NUM_SUPPLIES] = {
+	"VA",
+	"VP",
+	"VCP",
+	"VD_FILT",
+	"VL",
+};
+
+struct  cs42l42_private {
+	struct regmap *regmap;
+	struct snd_soc_codec *codec;
+	struct regulator_bulk_data supplies[CS42L42_NUM_SUPPLIES];
+	struct gpio_desc *reset_gpio;
+	struct completion pdn_done;
+	u32 sclk;
+	u32 srate;
+	u32 swidth;
+	u8 plug_state;
+	u8 hs_type;
+	u8 ts_inv;
+	u8 ts_dbnc_rise;
+	u8 ts_dbnc_fall;
+	u8 btn_det_init_dbnce;
+	u8 btn_det_event_dbnce;
+	u8 bias_thresholds[CS42L42_NUM_BIASES];
+	u8 hs_bias_ramp_rate;
+	u8 hs_bias_ramp_time;
+};
+
+#endif /* __CS42L42_H__ */
diff --git a/sound/soc/codecs/cs42l56.c b/sound/soc/codecs/cs42l56.c
index 54c1768bc818..cb6ca85f1536 100644
--- a/sound/soc/codecs/cs42l56.c
+++ b/sound/soc/codecs/cs42l56.c
@@ -64,8 +64,6 @@ struct  cs42l56_private {
 };
 
 static const struct reg_default cs42l56_reg_defaults[] = {
-	{ 1, 0x56 },	/* r01	- ID 1 */
-	{ 2, 0x04 },	/* r02	- ID 2 */
 	{ 3, 0x7f },	/* r03	- Power Ctl 1 */
 	{ 4, 0xff },	/* r04	- Power Ctl 2 */
 	{ 5, 0x00 },	/* ro5	- Clocking Ctl 1 */
@@ -1262,8 +1260,6 @@ static int cs42l56_i2c_probe(struct i2c_client *i2c_client,
 		return ret;
 	}
 
-	regcache_cache_bypass(cs42l56->regmap, true);
-
 	ret = regmap_read(cs42l56->regmap, CS42L56_CHIP_ID_1, &reg);
 	devid = reg & CS42L56_CHIP_ID_MASK;
 	if (devid != CS42L56_DEVID) {
@@ -1279,23 +1275,25 @@ static int cs42l56_i2c_probe(struct i2c_client *i2c_client,
 	dev_info(&i2c_client->dev, "Alpha Rev %X Metal Rev %X\n",
 		 alpha_rev, metal_rev);
 
-	regcache_cache_bypass(cs42l56->regmap, false);
-
 	if (cs42l56->pdata.ain1a_ref_cfg)
 		regmap_update_bits(cs42l56->regmap, CS42L56_AIN_REFCFG_ADC_MUX,
-				   CS42L56_AIN1A_REF_MASK, 1);
+				   CS42L56_AIN1A_REF_MASK,
+				   CS42L56_AIN1A_REF_MASK);
 
 	if (cs42l56->pdata.ain1b_ref_cfg)
 		regmap_update_bits(cs42l56->regmap, CS42L56_AIN_REFCFG_ADC_MUX,
-				   CS42L56_AIN1B_REF_MASK, 1);
+				   CS42L56_AIN1B_REF_MASK,
+				   CS42L56_AIN1B_REF_MASK);
 
 	if (cs42l56->pdata.ain2a_ref_cfg)
 		regmap_update_bits(cs42l56->regmap, CS42L56_AIN_REFCFG_ADC_MUX,
-				   CS42L56_AIN2A_REF_MASK, 1);
+				   CS42L56_AIN2A_REF_MASK,
+				   CS42L56_AIN2A_REF_MASK);
 
 	if (cs42l56->pdata.ain2b_ref_cfg)
 		regmap_update_bits(cs42l56->regmap, CS42L56_AIN_REFCFG_ADC_MUX,
-				   CS42L56_AIN2B_REF_MASK, 1);
+				   CS42L56_AIN2B_REF_MASK,
+				   CS42L56_AIN2B_REF_MASK);
 
 	if (cs42l56->pdata.micbias_lvl)
 		regmap_update_bits(cs42l56->regmap, CS42L56_GAIN_BIAS_CTL,
diff --git a/sound/soc/codecs/cs42l73.c b/sound/soc/codecs/cs42l73.c
index 71ba5605495f..3df2c473ab88 100644
--- a/sound/soc/codecs/cs42l73.c
+++ b/sound/soc/codecs/cs42l73.c
@@ -1337,8 +1337,6 @@ static int cs42l73_i2c_probe(struct i2c_client *i2c_client,
 		gpio_set_value_cansleep(cs42l73->pdata.reset_gpio, 1);
 	}
 
-	regcache_cache_bypass(cs42l73->regmap, true);
-
 	/* initialize codec */
 	ret = regmap_read(cs42l73->regmap, CS42L73_DEVID_AB, &reg);
 	devid = (reg & 0xFF) << 12;
@@ -1366,8 +1364,6 @@ static int cs42l73_i2c_probe(struct i2c_client *i2c_client,
 	dev_info(&i2c_client->dev,
 		 "Cirrus Logic CS42L73, Revision: %02X\n", reg & 0xFF);
 
-	regcache_cache_bypass(cs42l73->regmap, false);
-
 	ret =  snd_soc_register_codec(&i2c_client->dev,
 			&soc_codec_dev_cs42l73, cs42l73_dai,
 			ARRAY_SIZE(cs42l73_dai));
diff --git a/sound/soc/codecs/cs42xx8.c b/sound/soc/codecs/cs42xx8.c
index b4d87379d2bc..c1785bd4ff19 100644
--- a/sound/soc/codecs/cs42xx8.c
+++ b/sound/soc/codecs/cs42xx8.c
@@ -321,7 +321,6 @@ static struct snd_soc_dai_driver cs42xx8_dai = {
 };
 
 static const struct reg_default cs42xx8_reg[] = {
-	{ 0x01, 0x01 },   /* Chip I.D. and Revision Register */
 	{ 0x02, 0x00 },   /* Power Control */
 	{ 0x03, 0xF0 },   /* Functional Mode */
 	{ 0x04, 0x46 },   /* Interface Formats */
@@ -498,13 +497,6 @@ int cs42xx8_probe(struct device *dev, struct regmap *regmap)
 	/* Make sure hardware reset done */
 	msleep(5);
 
-	/*
-	 * We haven't marked the chip revision as volatile due to
-	 * sharing a register with the right input volume; explicitly
-	 * bypass the cache to read it.
-	 */
-	regcache_cache_bypass(cs42xx8->regmap, true);
-
 	/* Validate the chip ID */
 	ret = regmap_read(cs42xx8->regmap, CS42XX8_CHIPID, &val);
 	if (ret < 0) {
@@ -523,8 +515,6 @@ int cs42xx8_probe(struct device *dev, struct regmap *regmap)
 	dev_info(dev, "found device, revision %X\n",
 			val & CS42XX8_CHIPID_REV_ID_MASK);
 
-	regcache_cache_bypass(cs42xx8->regmap, false);
-
 	cs42xx8_dai.name = cs42xx8->drvdata->name;
 
 	/* Each adc supports stereo input */
diff --git a/sound/soc/codecs/cs47l24.c b/sound/soc/codecs/cs47l24.c
index 5b22564f037c..73559ae864b6 100644
--- a/sound/soc/codecs/cs47l24.c
+++ b/sound/soc/codecs/cs47l24.c
@@ -335,9 +335,11 @@ static const struct snd_kcontrol_new cs47l24_aec_loopback_mux =
 
 static const struct snd_soc_dapm_widget cs47l24_dapm_widgets[] = {
 SND_SOC_DAPM_SUPPLY("SYSCLK", ARIZONA_SYSTEM_CLOCK_1,
-		    ARIZONA_SYSCLK_ENA_SHIFT, 0, NULL, 0),
+		    ARIZONA_SYSCLK_ENA_SHIFT, 0, arizona_clk_ev,
+		    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 SND_SOC_DAPM_SUPPLY("ASYNCCLK", ARIZONA_ASYNC_CLOCK_1,
-		    ARIZONA_ASYNC_CLK_ENA_SHIFT, 0, NULL, 0),
+		    ARIZONA_ASYNC_CLK_ENA_SHIFT, 0, arizona_clk_ev,
+		    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 SND_SOC_DAPM_SUPPLY("OPCLK", ARIZONA_OUTPUT_SYSTEM_CLOCK,
 		    ARIZONA_OPCLK_ENA_SHIFT, 0, NULL, 0),
 SND_SOC_DAPM_SUPPLY("ASYNCOPCLK", ARIZONA_OUTPUT_ASYNC_CLOCK,
@@ -1064,7 +1066,7 @@ static struct snd_soc_dai_driver cs47l24_dai[] = {
 static int cs47l24_open(struct snd_compr_stream *stream)
 {
 	struct snd_soc_pcm_runtime *rtd = stream->private_data;
-	struct cs47l24_priv *priv = snd_soc_codec_get_drvdata(rtd->codec);
+	struct cs47l24_priv *priv = snd_soc_platform_get_drvdata(rtd->platform);
 	struct arizona *arizona = priv->core.arizona;
 	int n_adsp;
 
@@ -1113,8 +1115,8 @@ static irqreturn_t cs47l24_adsp2_irq(int irq, void *data)
 static int cs47l24_codec_probe(struct snd_soc_codec *codec)
 {
 	struct snd_soc_dapm_context *dapm = snd_soc_codec_get_dapm(codec);
+	struct snd_soc_component *component = snd_soc_dapm_to_component(dapm);
 	struct cs47l24_priv *priv = snd_soc_codec_get_drvdata(codec);
-	struct arizona *arizona = priv->core.arizona;
 	int ret;
 
 	priv->core.arizona->dapm = dapm;
@@ -1124,14 +1126,6 @@ static int cs47l24_codec_probe(struct snd_soc_codec *codec)
 	arizona_init_mono(codec);
 	arizona_init_notifiers(codec);
 
-	ret = arizona_request_irq(arizona, ARIZONA_IRQ_DSP_IRQ1,
-				  "ADSP2 Compressed IRQ", cs47l24_adsp2_irq,
-				  priv);
-	if (ret != 0) {
-		dev_err(codec->dev, "Failed to request DSP IRQ: %d\n", ret);
-		return ret;
-	}
-
 	ret = wm_adsp2_codec_probe(&priv->core.adsp[1], codec);
 	if (ret)
 		goto err_adsp2_codec_probe;
@@ -1145,7 +1139,7 @@ static int cs47l24_codec_probe(struct snd_soc_codec *codec)
 	if (ret)
 		goto err_adsp2_codec_probe;
 
-	snd_soc_dapm_disable_pin(dapm, "HAPTICS");
+	snd_soc_component_disable_pin(component, "HAPTICS");
 
 	return 0;
 
@@ -1159,17 +1153,12 @@ err_adsp2_codec_probe:
 static int cs47l24_codec_remove(struct snd_soc_codec *codec)
 {
 	struct cs47l24_priv *priv = snd_soc_codec_get_drvdata(codec);
-	struct arizona *arizona = priv->core.arizona;
 
 	wm_adsp2_codec_remove(&priv->core.adsp[1], codec);
 	wm_adsp2_codec_remove(&priv->core.adsp[2], codec);
 
 	priv->core.arizona->dapm = NULL;
 
-	arizona_free_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, priv);
-
-	arizona_free_spk(codec);
-
 	return 0;
 }
 
@@ -1285,25 +1274,47 @@ static int cs47l24_probe(struct platform_device *pdev)
 	pm_runtime_enable(&pdev->dev);
 	pm_runtime_idle(&pdev->dev);
 
+	ret = arizona_request_irq(arizona, ARIZONA_IRQ_DSP_IRQ1,
+				  "ADSP2 Compressed IRQ", cs47l24_adsp2_irq,
+				  cs47l24);
+	if (ret != 0) {
+		dev_err(&pdev->dev, "Failed to request DSP IRQ: %d\n", ret);
+		return ret;
+	}
+
+	ret = arizona_init_spk_irqs(arizona);
+	if (ret < 0)
+		goto err_dsp_irq;
+
 	ret = snd_soc_register_platform(&pdev->dev, &cs47l24_compr_platform);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to register platform: %d\n", ret);
-		return ret;
+		goto err_spk_irqs;
 	}
 
 	ret = snd_soc_register_codec(&pdev->dev, &soc_codec_dev_cs47l24,
 				      cs47l24_dai, ARRAY_SIZE(cs47l24_dai));
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to register codec: %d\n", ret);
-		snd_soc_unregister_platform(&pdev->dev);
+		goto err_platform;
 	}
 
 	return ret;
+
+err_platform:
+	snd_soc_unregister_platform(&pdev->dev);
+err_spk_irqs:
+	arizona_free_spk_irqs(arizona);
+err_dsp_irq:
+	arizona_free_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, cs47l24);
+
+	return ret;
 }
 
 static int cs47l24_remove(struct platform_device *pdev)
 {
 	struct cs47l24_priv *cs47l24 = platform_get_drvdata(pdev);
+	struct arizona *arizona = cs47l24->core.arizona;
 
 	snd_soc_unregister_platform(&pdev->dev);
 	snd_soc_unregister_codec(&pdev->dev);
@@ -1312,6 +1323,10 @@ static int cs47l24_remove(struct platform_device *pdev)
 	wm_adsp2_remove(&cs47l24->core.adsp[1]);
 	wm_adsp2_remove(&cs47l24->core.adsp[2]);
 
+	arizona_free_spk_irqs(arizona);
+
+	arizona_free_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, cs47l24);
+
 	return 0;
 }
 
diff --git a/sound/soc/codecs/da7219-aad.c b/sound/soc/codecs/da7219-aad.c
index 2b8914dd5990..6274d79c1353 100644
--- a/sound/soc/codecs/da7219-aad.c
+++ b/sound/soc/codecs/da7219-aad.c
@@ -204,10 +204,19 @@ static void da7219_aad_hptest_work(struct work_struct *work)
 	snd_soc_update_bits(codec, DA7219_MIXOUT_R_CTRL,
 			    DA7219_MIXOUT_R_AMP_EN_MASK,
 			    DA7219_MIXOUT_R_AMP_EN_MASK);
-	snd_soc_write(codec, DA7219_HP_L_CTRL,
-		      DA7219_HP_L_AMP_OE_MASK | DA7219_HP_L_AMP_EN_MASK);
-	snd_soc_write(codec, DA7219_HP_R_CTRL,
-		      DA7219_HP_R_AMP_OE_MASK | DA7219_HP_R_AMP_EN_MASK);
+	snd_soc_update_bits(codec, DA7219_HP_L_CTRL,
+			    DA7219_HP_L_AMP_OE_MASK | DA7219_HP_L_AMP_EN_MASK,
+			    DA7219_HP_L_AMP_OE_MASK | DA7219_HP_L_AMP_EN_MASK);
+	snd_soc_update_bits(codec, DA7219_HP_R_CTRL,
+			    DA7219_HP_R_AMP_OE_MASK | DA7219_HP_R_AMP_EN_MASK,
+			    DA7219_HP_R_AMP_OE_MASK | DA7219_HP_R_AMP_EN_MASK);
+	msleep(DA7219_SETTLING_DELAY);
+	snd_soc_update_bits(codec, DA7219_HP_L_CTRL,
+			    DA7219_HP_L_AMP_MUTE_EN_MASK |
+			    DA7219_HP_L_AMP_MIN_GAIN_EN_MASK, 0);
+	snd_soc_update_bits(codec, DA7219_HP_R_CTRL,
+			    DA7219_HP_R_AMP_MUTE_EN_MASK |
+			    DA7219_HP_R_AMP_MIN_GAIN_EN_MASK, 0);
 
 	/*
 	 * If we're running from the internal oscillator then give audio paths
@@ -244,6 +253,7 @@ static void da7219_aad_hptest_work(struct work_struct *work)
 	regcache_mark_dirty(da7219->regmap);
 	regcache_sync_region(da7219->regmap, DA7219_HP_L_CTRL,
 			     DA7219_HP_R_CTRL);
+	msleep(DA7219_SETTLING_DELAY);
 	regcache_sync_region(da7219->regmap, DA7219_MIXOUT_L_CTRL,
 			     DA7219_MIXOUT_R_CTRL);
 	regcache_sync_region(da7219->regmap, DA7219_DROUTING_ST_OUTFILT_1L,
diff --git a/sound/soc/codecs/da7219.c b/sound/soc/codecs/da7219.c
index cf37936bfe3a..99601627f83c 100644
--- a/sound/soc/codecs/da7219.c
+++ b/sound/soc/codecs/da7219.c
@@ -823,6 +823,85 @@ static int da7219_dai_event(struct snd_soc_dapm_widget *w,
 	}
 }
 
+static int da7219_settling_event(struct snd_soc_dapm_widget *w,
+				 struct snd_kcontrol *kcontrol, int event)
+{
+	switch (event) {
+	case SND_SOC_DAPM_POST_PMU:
+	case SND_SOC_DAPM_POST_PMD:
+		msleep(DA7219_SETTLING_DELAY);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int da7219_mixout_event(struct snd_soc_dapm_widget *w,
+			       struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	u8 hp_ctrl, min_gain_mask;
+
+	switch (w->reg) {
+	case DA7219_MIXOUT_L_CTRL:
+		hp_ctrl = DA7219_HP_L_CTRL;
+		min_gain_mask = DA7219_HP_L_AMP_MIN_GAIN_EN_MASK;
+		break;
+	case DA7219_MIXOUT_R_CTRL:
+		hp_ctrl = DA7219_HP_R_CTRL;
+		min_gain_mask = DA7219_HP_R_AMP_MIN_GAIN_EN_MASK;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMD:
+		/* Enable minimum gain on HP to avoid pops */
+		snd_soc_update_bits(codec, hp_ctrl, min_gain_mask,
+				    min_gain_mask);
+
+		msleep(DA7219_MIN_GAIN_DELAY);
+
+		break;
+	case SND_SOC_DAPM_POST_PMU:
+		/* Remove minimum gain on HP */
+		snd_soc_update_bits(codec, hp_ctrl, min_gain_mask, 0);
+
+		break;
+	}
+
+	return 0;
+}
+
+static int da7219_gain_ramp_event(struct snd_soc_dapm_widget *w,
+				  struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	struct da7219_priv *da7219 = snd_soc_codec_get_drvdata(codec);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+	case SND_SOC_DAPM_PRE_PMD:
+		/* Ensure nominal gain ramping for DAPM sequence */
+		da7219->gain_ramp_ctrl =
+			snd_soc_read(codec, DA7219_GAIN_RAMP_CTRL);
+		snd_soc_write(codec, DA7219_GAIN_RAMP_CTRL,
+			      DA7219_GAIN_RAMP_RATE_NOMINAL);
+		break;
+	case SND_SOC_DAPM_POST_PMU:
+	case SND_SOC_DAPM_POST_PMD:
+		/* Restore previous gain ramp settings */
+		snd_soc_write(codec, DA7219_GAIN_RAMP_CTRL,
+			      da7219->gain_ramp_ctrl);
+		break;
+	}
+
+	return 0;
+}
+
 
 /*
  * DAPM Widgets
@@ -907,30 +986,46 @@ static const struct snd_soc_dapm_widget da7219_dapm_widgets[] = {
 			   ARRAY_SIZE(da7219_st_out_filtr_mix_controls)),
 
 	/* DACs */
-	SND_SOC_DAPM_DAC("DACL", NULL, DA7219_DAC_L_CTRL, DA7219_DAC_L_EN_SHIFT,
-			 DA7219_NO_INVERT),
-	SND_SOC_DAPM_DAC("DACR", NULL, DA7219_DAC_R_CTRL, DA7219_DAC_R_EN_SHIFT,
-			 DA7219_NO_INVERT),
+	SND_SOC_DAPM_DAC_E("DACL", NULL, DA7219_DAC_L_CTRL,
+			   DA7219_DAC_L_EN_SHIFT, DA7219_NO_INVERT,
+			   da7219_settling_event,
+			   SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_DAC_E("DACR", NULL, DA7219_DAC_R_CTRL,
+			   DA7219_DAC_R_EN_SHIFT, DA7219_NO_INVERT,
+			   da7219_settling_event,
+			   SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_POST_PMD),
 
 	/* Output PGAs */
-	SND_SOC_DAPM_PGA("Mixout Left PGA", DA7219_MIXOUT_L_CTRL,
-			 DA7219_MIXOUT_L_AMP_EN_SHIFT, DA7219_NO_INVERT,
-			 NULL, 0),
-	SND_SOC_DAPM_PGA("Mixout Right PGA", DA7219_MIXOUT_R_CTRL,
-			 DA7219_MIXOUT_R_AMP_EN_SHIFT, DA7219_NO_INVERT,
-			 NULL, 0),
-	SND_SOC_DAPM_PGA("Headphone Left PGA", DA7219_HP_L_CTRL,
-			 DA7219_HP_L_AMP_EN_SHIFT, DA7219_NO_INVERT, NULL, 0),
-	SND_SOC_DAPM_PGA("Headphone Right PGA", DA7219_HP_R_CTRL,
-			 DA7219_HP_R_AMP_EN_SHIFT, DA7219_NO_INVERT, NULL, 0),
+	SND_SOC_DAPM_PGA_E("Mixout Left PGA", DA7219_MIXOUT_L_CTRL,
+			   DA7219_MIXOUT_L_AMP_EN_SHIFT, DA7219_NO_INVERT,
+			   NULL, 0, da7219_mixout_event,
+			   SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD),
+	SND_SOC_DAPM_PGA_E("Mixout Right PGA", DA7219_MIXOUT_R_CTRL,
+			   DA7219_MIXOUT_R_AMP_EN_SHIFT, DA7219_NO_INVERT,
+			   NULL, 0, da7219_mixout_event,
+			   SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD),
+	SND_SOC_DAPM_SUPPLY_S("Headphone Left PGA", 1, DA7219_HP_L_CTRL,
+			      DA7219_HP_L_AMP_EN_SHIFT, DA7219_NO_INVERT,
+			      da7219_settling_event,
+			      SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S("Headphone Right PGA", 1, DA7219_HP_R_CTRL,
+			      DA7219_HP_R_AMP_EN_SHIFT, DA7219_NO_INVERT,
+			      da7219_settling_event,
+			      SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_POST_PMD),
 
 	/* Output Supplies */
-	SND_SOC_DAPM_SUPPLY("Charge Pump", DA7219_CP_CTRL, DA7219_CP_EN_SHIFT,
-			    DA7219_NO_INVERT, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("Charge Pump", 0, DA7219_CP_CTRL,
+			      DA7219_CP_EN_SHIFT, DA7219_NO_INVERT,
+			      da7219_settling_event,
+			      SND_SOC_DAPM_POST_PMU),
 
 	/* Outputs */
 	SND_SOC_DAPM_OUTPUT("HPL"),
 	SND_SOC_DAPM_OUTPUT("HPR"),
+
+	/* Pre/Post Power */
+	SND_SOC_DAPM_PRE("Pre Power Gain Ramp", da7219_gain_ramp_event),
+	SND_SOC_DAPM_POST("Post Power Gain Ramp", da7219_gain_ramp_event),
 };
 
 
@@ -1003,8 +1098,8 @@ static const struct snd_soc_dapm_route da7219_audio_map[] = {
 	{"Mixout Left PGA", NULL, "DACL"},
 	{"Mixout Right PGA", NULL, "DACR"},
 
-	{"Headphone Left PGA", NULL, "Mixout Left PGA"},
-	{"Headphone Right PGA", NULL, "Mixout Right PGA"},
+	{"HPL", NULL, "Mixout Left PGA"},
+	{"HPR", NULL, "Mixout Right PGA"},
 
 	{"HPL", NULL, "Headphone Left PGA"},
 	{"HPR", NULL, "Headphone Right PGA"},
@@ -1712,6 +1807,14 @@ static int da7219_probe(struct snd_soc_codec *codec)
 			    DA7219_HP_R_AMP_RAMP_EN_MASK,
 			    DA7219_HP_R_AMP_RAMP_EN_MASK);
 
+	/* Default minimum gain on HP to avoid pops during DAPM sequencing */
+	snd_soc_update_bits(codec, DA7219_HP_L_CTRL,
+			    DA7219_HP_L_AMP_MIN_GAIN_EN_MASK,
+			    DA7219_HP_L_AMP_MIN_GAIN_EN_MASK);
+	snd_soc_update_bits(codec, DA7219_HP_R_CTRL,
+			    DA7219_HP_R_AMP_MIN_GAIN_EN_MASK,
+			    DA7219_HP_R_AMP_MIN_GAIN_EN_MASK);
+
 	/* Default infinite tone gen, start/stop by Kcontrol */
 	snd_soc_write(codec, DA7219_TONE_GEN_CYCLES, DA7219_BEEP_CYCLES_MASK);
 
diff --git a/sound/soc/codecs/da7219.h b/sound/soc/codecs/da7219.h
index 66d3bad86739..6baba7455fa1 100644
--- a/sound/soc/codecs/da7219.h
+++ b/sound/soc/codecs/da7219.h
@@ -777,6 +777,10 @@
 #define DA7219_SYS_STAT_CHECK_RETRIES	6
 #define DA7219_SYS_STAT_CHECK_DELAY	50
 
+/* Power up/down Delays */
+#define DA7219_SETTLING_DELAY	40
+#define DA7219_MIN_GAIN_DELAY	30
+
 enum da7219_clk_src {
 	DA7219_CLKSRC_MCLK = 0,
 	DA7219_CLKSRC_MCLK_SQR,
@@ -814,6 +818,7 @@ struct da7219_priv {
 
 	bool master;
 	bool alc_en;
+	u8 gain_ramp_ctrl;
 };
 
 #endif /* __DA7219_H */
diff --git a/sound/soc/codecs/es8328.h b/sound/soc/codecs/es8328.h
index 1a736e72a929..8930322d712b 100644
--- a/sound/soc/codecs/es8328.h
+++ b/sound/soc/codecs/es8328.h
@@ -278,43 +278,6 @@ int es8328_probe(struct device *dev, struct regmap *regmap);
 
 #define ES8328_REG_MAX		0x35
 
-#define ES8328_PLL1		0
-#define ES8328_PLL2		1
-
-/* clock inputs */
-#define ES8328_MCLK		0
-#define ES8328_PCMCLK		1
-
-/* clock divider id's */
-#define ES8328_PCMDIV		0
-#define ES8328_BCLKDIV		1
-#define ES8328_VXCLKDIV		2
-
-/* PCM clock dividers */
-#define ES8328_PCM_DIV_1	(0 << 6)
-#define ES8328_PCM_DIV_3	(2 << 6)
-#define ES8328_PCM_DIV_5_5	(3 << 6)
-#define ES8328_PCM_DIV_2	(4 << 6)
-#define ES8328_PCM_DIV_4	(5 << 6)
-#define ES8328_PCM_DIV_6	(6 << 6)
-#define ES8328_PCM_DIV_8	(7 << 6)
-
-/* BCLK clock dividers */
-#define ES8328_BCLK_DIV_1	(0 << 7)
-#define ES8328_BCLK_DIV_2	(1 << 7)
-#define ES8328_BCLK_DIV_4	(2 << 7)
-#define ES8328_BCLK_DIV_8	(3 << 7)
-
-/* VXCLK clock dividers */
-#define ES8328_VXCLK_DIV_1	(0 << 6)
-#define ES8328_VXCLK_DIV_2	(1 << 6)
-#define ES8328_VXCLK_DIV_4	(2 << 6)
-#define ES8328_VXCLK_DIV_8	(3 << 6)
-#define ES8328_VXCLK_DIV_16	(4 << 6)
-
-#define ES8328_DAI_HIFI		0
-#define ES8328_DAI_VOICE	1
-
 #define ES8328_1536FS		1536
 #define ES8328_1024FS		1024
 #define ES8328_768FS		768
diff --git a/sound/soc/codecs/hdac_hdmi.c b/sound/soc/codecs/hdac_hdmi.c
index c602c4960924..0c6228a0bf95 100644
--- a/sound/soc/codecs/hdac_hdmi.c
+++ b/sound/soc/codecs/hdac_hdmi.c
@@ -1368,7 +1368,7 @@ static int hdac_hdmi_parse_and_map_nid(struct hdac_ext_device *edev,
 	return hdac_hdmi_init_dai_map(edev);
 }
 
-static void hdac_hdmi_eld_notify_cb(void *aptr, int port)
+static void hdac_hdmi_eld_notify_cb(void *aptr, int port, int pipe)
 {
 	struct hdac_ext_device *edev = aptr;
 	struct hdac_hdmi_priv *hdmi = edev->private_data;
diff --git a/sound/soc/codecs/msm8916-wcd-analog.c b/sound/soc/codecs/msm8916-wcd-analog.c
new file mode 100644
index 000000000000..d8e8590746af
--- /dev/null
+++ b/sound/soc/codecs/msm8916-wcd-analog.c
@@ -0,0 +1,890 @@
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/regulator/consumer.h>
+#include <linux/types.h>
+#include <linux/clk.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <sound/soc.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/tlv.h>
+
+#define CDC_D_REVISION1			(0xf000)
+#define CDC_D_PERPH_SUBTYPE		(0xf005)
+#define CDC_D_CDC_RST_CTL		(0xf046)
+#define RST_CTL_DIG_SW_RST_N_MASK	BIT(7)
+#define RST_CTL_DIG_SW_RST_N_RESET	0
+#define RST_CTL_DIG_SW_RST_N_REMOVE_RESET BIT(7)
+
+#define CDC_D_CDC_TOP_CLK_CTL		(0xf048)
+#define TOP_CLK_CTL_A_MCLK_MCLK2_EN_MASK (BIT(2) | BIT(3))
+#define TOP_CLK_CTL_A_MCLK_EN_ENABLE	 BIT(2)
+#define TOP_CLK_CTL_A_MCLK2_EN_ENABLE	BIT(3)
+
+#define CDC_D_CDC_ANA_CLK_CTL		(0xf049)
+#define ANA_CLK_CTL_EAR_HPHR_CLK_EN_MASK BIT(0)
+#define ANA_CLK_CTL_EAR_HPHR_CLK_EN	BIT(0)
+#define ANA_CLK_CTL_EAR_HPHL_CLK_EN	BIT(1)
+#define ANA_CLK_CTL_SPKR_CLK_EN_MASK	BIT(4)
+#define ANA_CLK_CTL_SPKR_CLK_EN	BIT(4)
+#define ANA_CLK_CTL_TXA_CLK25_EN	BIT(5)
+
+#define CDC_D_CDC_DIG_CLK_CTL		(0xf04A)
+#define DIG_CLK_CTL_RXD1_CLK_EN		BIT(0)
+#define DIG_CLK_CTL_RXD2_CLK_EN		BIT(1)
+#define DIG_CLK_CTL_RXD3_CLK_EN		BIT(3)
+#define DIG_CLK_CTL_TXD_CLK_EN		BIT(4)
+#define DIG_CLK_CTL_NCP_CLK_EN_MASK	BIT(6)
+#define DIG_CLK_CTL_NCP_CLK_EN		BIT(6)
+#define DIG_CLK_CTL_RXD_PDM_CLK_EN_MASK	BIT(7)
+#define DIG_CLK_CTL_RXD_PDM_CLK_EN	BIT(7)
+
+#define CDC_D_CDC_CONN_TX1_CTL		(0xf050)
+#define CONN_TX1_SERIAL_TX1_MUX		GENMASK(1, 0)
+#define CONN_TX1_SERIAL_TX1_ADC_1	0x0
+#define CONN_TX1_SERIAL_TX1_RX_PDM_LB	0x1
+#define CONN_TX1_SERIAL_TX1_ZERO	0x2
+
+#define CDC_D_CDC_CONN_TX2_CTL		(0xf051)
+#define CONN_TX2_SERIAL_TX2_MUX		GENMASK(1, 0)
+#define CONN_TX2_SERIAL_TX2_ADC_2	0x0
+#define CONN_TX2_SERIAL_TX2_RX_PDM_LB	0x1
+#define CONN_TX2_SERIAL_TX2_ZERO	0x2
+#define CDC_D_CDC_CONN_HPHR_DAC_CTL	(0xf052)
+#define CDC_D_CDC_CONN_RX1_CTL		(0xf053)
+#define CDC_D_CDC_CONN_RX2_CTL		(0xf054)
+#define CDC_D_CDC_CONN_RX3_CTL		(0xf055)
+#define CDC_D_CDC_CONN_RX_LB_CTL	(0xf056)
+#define CDC_D_SEC_ACCESS		(0xf0D0)
+#define CDC_D_PERPH_RESET_CTL3		(0xf0DA)
+#define CDC_D_PERPH_RESET_CTL4		(0xf0DB)
+#define CDC_A_REVISION1			(0xf100)
+#define CDC_A_REVISION2			(0xf101)
+#define CDC_A_REVISION3			(0xf102)
+#define CDC_A_REVISION4			(0xf103)
+#define CDC_A_PERPH_TYPE		(0xf104)
+#define CDC_A_PERPH_SUBTYPE		(0xf105)
+#define CDC_A_INT_RT_STS		(0xf110)
+#define CDC_A_INT_SET_TYPE		(0xf111)
+#define CDC_A_INT_POLARITY_HIGH		(0xf112)
+#define CDC_A_INT_POLARITY_LOW		(0xf113)
+#define CDC_A_INT_LATCHED_CLR		(0xf114)
+#define CDC_A_INT_EN_SET		(0xf115)
+#define CDC_A_INT_EN_CLR		(0xf116)
+#define CDC_A_INT_LATCHED_STS		(0xf118)
+#define CDC_A_INT_PENDING_STS		(0xf119)
+#define CDC_A_INT_MID_SEL		(0xf11A)
+#define CDC_A_INT_PRIORITY		(0xf11B)
+#define CDC_A_MICB_1_EN			(0xf140)
+#define MICB_1_EN_MICB_ENABLE		BIT(7)
+#define MICB_1_EN_BYP_CAP_MASK		BIT(6)
+#define MICB_1_EN_NO_EXT_BYP_CAP	BIT(6)
+#define MICB_1_EN_EXT_BYP_CAP		0
+#define MICB_1_EN_PULL_DOWN_EN_MASK	BIT(5)
+#define MICB_1_EN_PULL_DOWN_EN_ENABLE	BIT(5)
+#define MICB_1_EN_OPA_STG2_TAIL_CURR_MASK GENMASK(3, 1)
+#define MICB_1_EN_OPA_STG2_TAIL_CURR_1_60UA	(0x4)
+#define MICB_1_EN_PULL_UP_EN_MASK	BIT(4)
+#define MICB_1_EN_TX3_GND_SEL_MASK	BIT(0)
+#define MICB_1_EN_TX3_GND_SEL_TX_GND	0
+
+#define CDC_A_MICB_1_VAL		(0xf141)
+#define MICB_1_VAL_MICB_OUT_VAL_MASK	GENMASK(7, 3)
+#define MICB_1_VAL_MICB_OUT_VAL_V2P70V	((0x16)  << 3)
+#define CDC_A_MICB_1_CTL		(0xf142)
+
+#define MICB_1_CTL_CFILT_REF_SEL_MASK		BIT(1)
+#define MICB_1_CTL_CFILT_REF_SEL_HPF_REF	BIT(1)
+#define MICB_1_CTL_EXT_PRECHARG_EN_MASK		BIT(5)
+#define MICB_1_CTL_EXT_PRECHARG_EN_ENABLE	BIT(5)
+#define MICB_1_CTL_INT_PRECHARG_BYP_MASK	BIT(6)
+#define MICB_1_CTL_INT_PRECHARG_BYP_EXT_PRECHRG_SEL	BIT(6)
+
+#define CDC_A_MICB_1_INT_RBIAS			(0xf143)
+#define MICB_1_INT_TX1_INT_RBIAS_EN_MASK	BIT(7)
+#define MICB_1_INT_TX1_INT_RBIAS_EN_ENABLE	BIT(7)
+#define MICB_1_INT_TX1_INT_RBIAS_EN_DISABLE	0
+
+#define MICB_1_INT_TX1_INT_PULLUP_EN_MASK	BIT(6)
+#define MICB_1_INT_TX1_INT_PULLUP_EN_TX1N_TO_MICBIAS BIT(6)
+#define MICB_1_INT_TX1_INT_PULLUP_EN_TX1N_TO_GND	0
+
+#define MICB_1_INT_TX2_INT_RBIAS_EN_MASK	BIT(4)
+#define MICB_1_INT_TX2_INT_RBIAS_EN_ENABLE	BIT(4)
+#define MICB_1_INT_TX2_INT_RBIAS_EN_DISABLE	0
+#define MICB_1_INT_TX2_INT_PULLUP_EN_MASK	BIT(3)
+#define MICB_1_INT_TX2_INT_PULLUP_EN_TX1N_TO_MICBIAS BIT(3)
+#define MICB_1_INT_TX2_INT_PULLUP_EN_TX1N_TO_GND	0
+
+#define MICB_1_INT_TX3_INT_RBIAS_EN_MASK	BIT(1)
+#define MICB_1_INT_TX3_INT_RBIAS_EN_ENABLE	BIT(1)
+#define MICB_1_INT_TX3_INT_RBIAS_EN_DISABLE	0
+#define MICB_1_INT_TX3_INT_PULLUP_EN_MASK	BIT(0)
+#define MICB_1_INT_TX3_INT_PULLUP_EN_TX1N_TO_MICBIAS BIT(0)
+#define MICB_1_INT_TX3_INT_PULLUP_EN_TX1N_TO_GND	0
+
+#define CDC_A_MICB_2_EN			(0xf144)
+#define CDC_A_TX_1_2_ATEST_CTL_2	(0xf145)
+#define CDC_A_MASTER_BIAS_CTL		(0xf146)
+#define CDC_A_TX_1_EN			(0xf160)
+#define CDC_A_TX_2_EN			(0xf161)
+#define CDC_A_TX_1_2_TEST_CTL_1		(0xf162)
+#define CDC_A_TX_1_2_TEST_CTL_2		(0xf163)
+#define CDC_A_TX_1_2_ATEST_CTL		(0xf164)
+#define CDC_A_TX_1_2_OPAMP_BIAS		(0xf165)
+#define CDC_A_TX_3_EN			(0xf167)
+#define CDC_A_NCP_EN			(0xf180)
+#define CDC_A_NCP_CLK			(0xf181)
+#define CDC_A_NCP_FBCTRL		(0xf183)
+#define CDC_A_NCP_FBCTRL_FB_CLK_INV_MASK	BIT(5)
+#define CDC_A_NCP_FBCTRL_FB_CLK_INV		BIT(5)
+#define CDC_A_NCP_BIAS			(0xf184)
+#define CDC_A_NCP_VCTRL			(0xf185)
+#define CDC_A_NCP_TEST			(0xf186)
+#define CDC_A_NCP_CLIM_ADDR		(0xf187)
+#define CDC_A_RX_CLOCK_DIVIDER		(0xf190)
+#define CDC_A_RX_COM_OCP_CTL		(0xf191)
+#define CDC_A_RX_COM_OCP_COUNT		(0xf192)
+#define CDC_A_RX_COM_BIAS_DAC		(0xf193)
+#define RX_COM_BIAS_DAC_RX_BIAS_EN_MASK		BIT(7)
+#define RX_COM_BIAS_DAC_RX_BIAS_EN_ENABLE	BIT(7)
+#define RX_COM_BIAS_DAC_DAC_REF_EN_MASK		BIT(0)
+#define RX_COM_BIAS_DAC_DAC_REF_EN_ENABLE	BIT(0)
+
+#define CDC_A_RX_HPH_BIAS_PA		(0xf194)
+#define CDC_A_RX_HPH_BIAS_LDO_OCP	(0xf195)
+#define CDC_A_RX_HPH_BIAS_CNP		(0xf196)
+#define CDC_A_RX_HPH_CNP_EN		(0xf197)
+#define CDC_A_RX_HPH_L_PA_DAC_CTL	(0xf19B)
+#define RX_HPA_L_PA_DAC_CTL_DATA_RESET_MASK	BIT(1)
+#define RX_HPA_L_PA_DAC_CTL_DATA_RESET_RESET	BIT(1)
+#define CDC_A_RX_HPH_R_PA_DAC_CTL	(0xf19D)
+#define RX_HPH_R_PA_DAC_CTL_DATA_RESET	BIT(1)
+#define RX_HPH_R_PA_DAC_CTL_DATA_RESET_MASK BIT(1)
+
+#define CDC_A_RX_EAR_CTL			(0xf19E)
+#define RX_EAR_CTL_SPK_VBAT_LDO_EN_MASK		BIT(0)
+#define RX_EAR_CTL_SPK_VBAT_LDO_EN_ENABLE	BIT(0)
+
+#define CDC_A_SPKR_DAC_CTL		(0xf1B0)
+#define SPKR_DAC_CTL_DAC_RESET_MASK	BIT(4)
+#define SPKR_DAC_CTL_DAC_RESET_NORMAL	0
+
+#define CDC_A_SPKR_DRV_CTL		(0xf1B2)
+#define SPKR_DRV_CTL_DEF_MASK		0xEF
+#define SPKR_DRV_CLASSD_PA_EN_MASK	BIT(7)
+#define SPKR_DRV_CLASSD_PA_EN_ENABLE	BIT(7)
+#define SPKR_DRV_CAL_EN			BIT(6)
+#define SPKR_DRV_SETTLE_EN		BIT(5)
+#define SPKR_DRV_FW_EN			BIT(3)
+#define SPKR_DRV_BOOST_SET		BIT(2)
+#define SPKR_DRV_CMFB_SET		BIT(1)
+#define SPKR_DRV_GAIN_SET		BIT(0)
+#define SPKR_DRV_CTL_DEF_VAL (SPKR_DRV_CLASSD_PA_EN_ENABLE | \
+		SPKR_DRV_CAL_EN | SPKR_DRV_SETTLE_EN | \
+		SPKR_DRV_FW_EN | SPKR_DRV_BOOST_SET | \
+		SPKR_DRV_CMFB_SET | SPKR_DRV_GAIN_SET)
+#define CDC_A_SPKR_OCP_CTL		(0xf1B4)
+#define CDC_A_SPKR_PWRSTG_CTL		(0xf1B5)
+#define SPKR_PWRSTG_CTL_DAC_EN_MASK	BIT(0)
+#define SPKR_PWRSTG_CTL_DAC_EN		BIT(0)
+#define SPKR_PWRSTG_CTL_MASK		0xE0
+#define SPKR_PWRSTG_CTL_BBM_MASK	BIT(7)
+#define SPKR_PWRSTG_CTL_BBM_EN		BIT(7)
+#define SPKR_PWRSTG_CTL_HBRDGE_EN_MASK	BIT(6)
+#define SPKR_PWRSTG_CTL_HBRDGE_EN	BIT(6)
+#define SPKR_PWRSTG_CTL_CLAMP_EN_MASK	BIT(5)
+#define SPKR_PWRSTG_CTL_CLAMP_EN	BIT(5)
+
+#define CDC_A_SPKR_DRV_DBG		(0xf1B7)
+#define CDC_A_CURRENT_LIMIT		(0xf1C0)
+#define CDC_A_BOOST_EN_CTL		(0xf1C3)
+#define CDC_A_SLOPE_COMP_IP_ZERO	(0xf1C4)
+#define CDC_A_SEC_ACCESS		(0xf1D0)
+#define CDC_A_PERPH_RESET_CTL3		(0xf1DA)
+#define CDC_A_PERPH_RESET_CTL4		(0xf1DB)
+
+#define MSM8916_WCD_ANALOG_RATES (SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000 |\
+			SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_48000)
+#define MSM8916_WCD_ANALOG_FORMATS (SNDRV_PCM_FMTBIT_S16_LE |\
+				    SNDRV_PCM_FMTBIT_S24_LE)
+
+static const char * const supply_names[] = {
+	"vdd-cdc-io",
+	"vdd-cdc-tx-rx-cx",
+};
+
+struct pm8916_wcd_analog_priv {
+	u16 pmic_rev;
+	u16 codec_version;
+	struct clk *mclk;
+	struct regulator_bulk_data supplies[ARRAY_SIZE(supply_names)];
+	bool micbias1_cap_mode;
+	bool micbias2_cap_mode;
+};
+
+static const char *const adc2_mux_text[] = { "ZERO", "INP2", "INP3" };
+static const char *const rdac2_mux_text[] = { "ZERO", "RX2", "RX1" };
+static const char *const hph_text[] = { "ZERO", "Switch", };
+
+static const struct soc_enum hph_enum = SOC_ENUM_SINGLE_VIRT(
+					ARRAY_SIZE(hph_text), hph_text);
+
+static const struct snd_kcontrol_new hphl_mux = SOC_DAPM_ENUM("HPHL", hph_enum);
+static const struct snd_kcontrol_new hphr_mux = SOC_DAPM_ENUM("HPHR", hph_enum);
+
+/* ADC2 MUX */
+static const struct soc_enum adc2_enum = SOC_ENUM_SINGLE_VIRT(
+			ARRAY_SIZE(adc2_mux_text), adc2_mux_text);
+
+/* RDAC2 MUX */
+static const struct soc_enum rdac2_mux_enum = SOC_ENUM_SINGLE(
+			CDC_D_CDC_CONN_HPHR_DAC_CTL, 0, 3, rdac2_mux_text);
+
+static const struct snd_kcontrol_new spkr_switch[] = {
+	SOC_DAPM_SINGLE("Switch", CDC_A_SPKR_DAC_CTL, 7, 1, 0)
+};
+
+static const struct snd_kcontrol_new rdac2_mux = SOC_DAPM_ENUM(
+					"RDAC2 MUX Mux", rdac2_mux_enum);
+static const struct snd_kcontrol_new tx_adc2_mux = SOC_DAPM_ENUM(
+					"ADC2 MUX Mux", adc2_enum);
+
+/* Analog Gain control 0 dB to +24 dB in 6 dB steps */
+static const DECLARE_TLV_DB_SCALE(analog_gain, 0, 600, 0);
+
+static const struct snd_kcontrol_new pm8916_wcd_analog_snd_controls[] = {
+	SOC_SINGLE_TLV("ADC1 Volume", CDC_A_TX_1_EN, 3, 8, 0, analog_gain),
+	SOC_SINGLE_TLV("ADC2 Volume", CDC_A_TX_2_EN, 3, 8, 0, analog_gain),
+	SOC_SINGLE_TLV("ADC3 Volume", CDC_A_TX_3_EN, 3, 8, 0, analog_gain),
+};
+
+static void pm8916_wcd_analog_micbias_enable(struct snd_soc_codec *codec)
+{
+	snd_soc_update_bits(codec, CDC_A_MICB_1_CTL,
+			    MICB_1_CTL_EXT_PRECHARG_EN_MASK |
+			    MICB_1_CTL_INT_PRECHARG_BYP_MASK,
+			    MICB_1_CTL_INT_PRECHARG_BYP_EXT_PRECHRG_SEL
+			    | MICB_1_CTL_EXT_PRECHARG_EN_ENABLE);
+
+	snd_soc_write(codec, CDC_A_MICB_1_VAL, MICB_1_VAL_MICB_OUT_VAL_V2P70V);
+	/*
+	 * Special headset needs MICBIAS as 2.7V so wait for
+	 * 50 msec for the MICBIAS to reach 2.7 volts.
+	 */
+	msleep(50);
+	snd_soc_update_bits(codec, CDC_A_MICB_1_CTL,
+			    MICB_1_CTL_EXT_PRECHARG_EN_MASK |
+			    MICB_1_CTL_INT_PRECHARG_BYP_MASK, 0);
+
+}
+
+static int pm8916_wcd_analog_enable_micbias_ext(struct snd_soc_codec
+						 *codec, int event,
+						 int reg, u32 cap_mode)
+{
+	switch (event) {
+	case SND_SOC_DAPM_POST_PMU:
+		pm8916_wcd_analog_micbias_enable(codec);
+		snd_soc_update_bits(codec, CDC_A_MICB_1_EN,
+				    MICB_1_EN_BYP_CAP_MASK, cap_mode);
+		break;
+	}
+
+	return 0;
+}
+
+static int pm8916_wcd_analog_enable_micbias_int(struct snd_soc_codec
+						 *codec, int event,
+						 int reg, u32 cap_mode)
+{
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		snd_soc_update_bits(codec, CDC_A_MICB_1_INT_RBIAS,
+				    MICB_1_INT_TX2_INT_RBIAS_EN_MASK,
+				    MICB_1_INT_TX2_INT_RBIAS_EN_ENABLE);
+		snd_soc_update_bits(codec, reg, MICB_1_EN_PULL_DOWN_EN_MASK, 0);
+		snd_soc_update_bits(codec, CDC_A_MICB_1_EN,
+				    MICB_1_EN_OPA_STG2_TAIL_CURR_MASK,
+				    MICB_1_EN_OPA_STG2_TAIL_CURR_1_60UA);
+
+		break;
+	case SND_SOC_DAPM_POST_PMU:
+		pm8916_wcd_analog_micbias_enable(codec);
+		snd_soc_update_bits(codec, CDC_A_MICB_1_EN,
+				    MICB_1_EN_BYP_CAP_MASK, cap_mode);
+		break;
+	}
+
+	return 0;
+}
+
+static int pm8916_wcd_analog_enable_micbias_ext1(struct
+						  snd_soc_dapm_widget
+						  *w, struct snd_kcontrol
+						  *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	struct pm8916_wcd_analog_priv *wcd = snd_soc_codec_get_drvdata(codec);
+
+	return pm8916_wcd_analog_enable_micbias_ext(codec, event, w->reg,
+						     wcd->micbias1_cap_mode);
+}
+
+static int pm8916_wcd_analog_enable_micbias_ext2(struct
+						  snd_soc_dapm_widget
+						  *w, struct snd_kcontrol
+						  *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	struct pm8916_wcd_analog_priv *wcd = snd_soc_codec_get_drvdata(codec);
+
+	return pm8916_wcd_analog_enable_micbias_ext(codec, event, w->reg,
+						     wcd->micbias2_cap_mode);
+
+}
+
+static int pm8916_wcd_analog_enable_micbias_int1(struct
+						  snd_soc_dapm_widget
+						  *w, struct snd_kcontrol
+						  *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	struct pm8916_wcd_analog_priv *wcd = snd_soc_codec_get_drvdata(codec);
+
+	return pm8916_wcd_analog_enable_micbias_int(codec, event, w->reg,
+						     wcd->micbias1_cap_mode);
+}
+
+static int pm8916_wcd_analog_enable_micbias_int2(struct
+						  snd_soc_dapm_widget
+						  *w, struct snd_kcontrol
+						  *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	struct pm8916_wcd_analog_priv *wcd = snd_soc_codec_get_drvdata(codec);
+
+	return pm8916_wcd_analog_enable_micbias_int(codec, event, w->reg,
+						     wcd->micbias2_cap_mode);
+}
+
+static int pm8916_wcd_analog_enable_adc(struct snd_soc_dapm_widget *w,
+					 struct snd_kcontrol *kcontrol,
+					 int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	u16 adc_reg = CDC_A_TX_1_2_TEST_CTL_2;
+	u8 init_bit_shift;
+
+	if (w->reg == CDC_A_TX_1_EN)
+		init_bit_shift = 5;
+	else
+		init_bit_shift = 4;
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		if (w->reg == CDC_A_TX_2_EN)
+			snd_soc_update_bits(codec, CDC_A_MICB_1_CTL,
+					    MICB_1_CTL_CFILT_REF_SEL_MASK,
+					    MICB_1_CTL_CFILT_REF_SEL_HPF_REF);
+		/*
+		 * Add delay of 10 ms to give sufficient time for the voltage
+		 * to shoot up and settle so that the txfe init does not
+		 * happen when the input voltage is changing too much.
+		 */
+		usleep_range(10000, 10010);
+		snd_soc_update_bits(codec, adc_reg, 1 << init_bit_shift,
+				    1 << init_bit_shift);
+		switch (w->reg) {
+		case CDC_A_TX_1_EN:
+			snd_soc_update_bits(codec, CDC_D_CDC_CONN_TX1_CTL,
+					    CONN_TX1_SERIAL_TX1_MUX,
+					    CONN_TX1_SERIAL_TX1_ADC_1);
+			break;
+		case CDC_A_TX_2_EN:
+		case CDC_A_TX_3_EN:
+			snd_soc_update_bits(codec, CDC_D_CDC_CONN_TX2_CTL,
+					    CONN_TX2_SERIAL_TX2_MUX,
+					    CONN_TX2_SERIAL_TX2_ADC_2);
+			break;
+		}
+		break;
+	case SND_SOC_DAPM_POST_PMU:
+		/*
+		 * Add delay of 12 ms before deasserting the init
+		 * to reduce the tx pop
+		 */
+		usleep_range(12000, 12010);
+		snd_soc_update_bits(codec, adc_reg, 1 << init_bit_shift, 0x00);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		switch (w->reg) {
+		case CDC_A_TX_1_EN:
+			snd_soc_update_bits(codec, CDC_D_CDC_CONN_TX1_CTL,
+					    CONN_TX1_SERIAL_TX1_MUX,
+					    CONN_TX1_SERIAL_TX1_ZERO);
+			break;
+		case CDC_A_TX_2_EN:
+			snd_soc_update_bits(codec, CDC_A_MICB_1_CTL,
+					    MICB_1_CTL_CFILT_REF_SEL_MASK, 0);
+		case CDC_A_TX_3_EN:
+			snd_soc_update_bits(codec, CDC_D_CDC_CONN_TX2_CTL,
+					    CONN_TX2_SERIAL_TX2_MUX,
+					    CONN_TX2_SERIAL_TX2_ZERO);
+			break;
+		}
+
+
+		break;
+	}
+	return 0;
+}
+
+static int pm8916_wcd_analog_enable_spk_pa(struct snd_soc_dapm_widget *w,
+					    struct snd_kcontrol *kcontrol,
+					    int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		snd_soc_update_bits(codec, CDC_A_SPKR_PWRSTG_CTL,
+				    SPKR_PWRSTG_CTL_DAC_EN_MASK |
+				    SPKR_PWRSTG_CTL_BBM_MASK |
+				    SPKR_PWRSTG_CTL_HBRDGE_EN_MASK |
+				    SPKR_PWRSTG_CTL_CLAMP_EN_MASK,
+				    SPKR_PWRSTG_CTL_DAC_EN|
+				    SPKR_PWRSTG_CTL_BBM_EN |
+				    SPKR_PWRSTG_CTL_HBRDGE_EN |
+				    SPKR_PWRSTG_CTL_CLAMP_EN);
+
+		snd_soc_update_bits(codec, CDC_A_RX_EAR_CTL,
+				    RX_EAR_CTL_SPK_VBAT_LDO_EN_MASK,
+				    RX_EAR_CTL_SPK_VBAT_LDO_EN_ENABLE);
+		break;
+	case SND_SOC_DAPM_POST_PMU:
+		snd_soc_update_bits(codec, CDC_A_SPKR_DRV_CTL,
+				    SPKR_DRV_CTL_DEF_MASK,
+				    SPKR_DRV_CTL_DEF_VAL);
+		snd_soc_update_bits(codec, w->reg,
+				    SPKR_DRV_CLASSD_PA_EN_MASK,
+				    SPKR_DRV_CLASSD_PA_EN_ENABLE);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		snd_soc_update_bits(codec, CDC_A_SPKR_PWRSTG_CTL,
+				    SPKR_PWRSTG_CTL_DAC_EN_MASK|
+				    SPKR_PWRSTG_CTL_BBM_MASK |
+				    SPKR_PWRSTG_CTL_HBRDGE_EN_MASK |
+				    SPKR_PWRSTG_CTL_CLAMP_EN_MASK, 0);
+
+		snd_soc_update_bits(codec, CDC_A_SPKR_DAC_CTL,
+				    SPKR_DAC_CTL_DAC_RESET_MASK,
+				    SPKR_DAC_CTL_DAC_RESET_NORMAL);
+		snd_soc_update_bits(codec, CDC_A_RX_EAR_CTL,
+				    RX_EAR_CTL_SPK_VBAT_LDO_EN_MASK, 0);
+		break;
+	}
+	return 0;
+}
+
+static const struct reg_default wcd_reg_defaults_2_0[] = {
+	{CDC_A_RX_COM_OCP_CTL, 0xD1},
+	{CDC_A_RX_COM_OCP_COUNT, 0xFF},
+	{CDC_D_SEC_ACCESS, 0xA5},
+	{CDC_D_PERPH_RESET_CTL3, 0x0F},
+	{CDC_A_TX_1_2_OPAMP_BIAS, 0x4F},
+	{CDC_A_NCP_FBCTRL, 0x28},
+	{CDC_A_SPKR_DRV_CTL, 0x69},
+	{CDC_A_SPKR_DRV_DBG, 0x01},
+	{CDC_A_BOOST_EN_CTL, 0x5F},
+	{CDC_A_SLOPE_COMP_IP_ZERO, 0x88},
+	{CDC_A_SEC_ACCESS, 0xA5},
+	{CDC_A_PERPH_RESET_CTL3, 0x0F},
+	{CDC_A_CURRENT_LIMIT, 0x82},
+	{CDC_A_SPKR_DAC_CTL, 0x03},
+	{CDC_A_SPKR_OCP_CTL, 0xE1},
+	{CDC_A_MASTER_BIAS_CTL, 0x30},
+};
+
+static int pm8916_wcd_analog_probe(struct snd_soc_codec *codec)
+{
+	struct pm8916_wcd_analog_priv *priv = dev_get_drvdata(codec->dev);
+	int err, reg;
+
+	err = regulator_bulk_enable(ARRAY_SIZE(priv->supplies), priv->supplies);
+	if (err != 0) {
+		dev_err(codec->dev, "failed to enable regulators (%d)\n", err);
+		return err;
+	}
+
+	snd_soc_codec_set_drvdata(codec, priv);
+	priv->pmic_rev = snd_soc_read(codec, CDC_D_REVISION1);
+	priv->codec_version = snd_soc_read(codec, CDC_D_PERPH_SUBTYPE);
+
+	dev_info(codec->dev, "PMIC REV: %d\t CODEC Version: %d\n",
+		 priv->pmic_rev, priv->codec_version);
+
+	snd_soc_write(codec, CDC_D_PERPH_RESET_CTL4, 0x01);
+	snd_soc_write(codec, CDC_A_PERPH_RESET_CTL4, 0x01);
+
+	for (reg = 0; reg < ARRAY_SIZE(wcd_reg_defaults_2_0); reg++)
+		snd_soc_write(codec, wcd_reg_defaults_2_0[reg].reg,
+			      wcd_reg_defaults_2_0[reg].def);
+
+	return 0;
+}
+
+static int pm8916_wcd_analog_remove(struct snd_soc_codec *codec)
+{
+	struct pm8916_wcd_analog_priv *priv = dev_get_drvdata(codec->dev);
+
+	return regulator_bulk_disable(ARRAY_SIZE(priv->supplies),
+				      priv->supplies);
+}
+
+static const struct snd_soc_dapm_route pm8916_wcd_analog_audio_map[] = {
+
+	{"PDM_RX1", NULL, "PDM Playback"},
+	{"PDM_RX2", NULL, "PDM Playback"},
+	{"PDM_RX3", NULL, "PDM Playback"},
+	{"PDM Capture", NULL, "PDM_TX"},
+
+	/* ADC Connections */
+	{"PDM_TX", NULL, "ADC2"},
+	{"PDM_TX", NULL, "ADC3"},
+	{"ADC2", NULL, "ADC2 MUX"},
+	{"ADC3", NULL, "ADC2 MUX"},
+	{"ADC2 MUX", "INP2", "ADC2_INP2"},
+	{"ADC2 MUX", "INP3", "ADC2_INP3"},
+
+	{"PDM_TX", NULL, "ADC1"},
+	{"ADC1", NULL, "AMIC1"},
+	{"ADC2_INP2", NULL, "AMIC2"},
+	{"ADC2_INP3", NULL, "AMIC3"},
+
+	/* RDAC Connections */
+	{"HPHR DAC", NULL, "RDAC2 MUX"},
+	{"RDAC2 MUX", "RX1", "PDM_RX1"},
+	{"RDAC2 MUX", "RX2", "PDM_RX2"},
+	{"HPHL DAC", NULL, "PDM_RX1"},
+	{"PDM_RX1", NULL, "RXD1_CLK"},
+	{"PDM_RX2", NULL, "RXD2_CLK"},
+	{"PDM_RX3", NULL, "RXD3_CLK"},
+
+	{"PDM_RX1", NULL, "RXD_PDM_CLK"},
+	{"PDM_RX2", NULL, "RXD_PDM_CLK"},
+	{"PDM_RX3", NULL, "RXD_PDM_CLK"},
+
+	{"ADC1", NULL, "TXD_CLK"},
+	{"ADC2", NULL, "TXD_CLK"},
+	{"ADC3", NULL, "TXD_CLK"},
+
+	{"ADC1", NULL, "TXA_CLK25"},
+	{"ADC2", NULL, "TXA_CLK25"},
+	{"ADC3", NULL, "TXA_CLK25"},
+
+	{"PDM_RX1", NULL, "A_MCLK2"},
+	{"PDM_RX2", NULL, "A_MCLK2"},
+	{"PDM_RX3", NULL, "A_MCLK2"},
+
+	{"PDM_TX", NULL, "A_MCLK2"},
+	{"A_MCLK2", NULL, "A_MCLK"},
+
+	/* Headset (RX MIX1 and RX MIX2) */
+	{"HEADPHONE", NULL, "HPHL PA"},
+	{"HEADPHONE", NULL, "HPHR PA"},
+
+	{"HPHL PA", NULL, "EAR_HPHL_CLK"},
+	{"HPHR PA", NULL, "EAR_HPHR_CLK"},
+
+	{"CP", NULL, "NCP_CLK"},
+
+	{"HPHL PA", NULL, "HPHL"},
+	{"HPHR PA", NULL, "HPHR"},
+	{"HPHL PA", NULL, "CP"},
+	{"HPHL PA", NULL, "RX_BIAS"},
+	{"HPHR PA", NULL, "CP"},
+	{"HPHR PA", NULL, "RX_BIAS"},
+	{"HPHL", "Switch", "HPHL DAC"},
+	{"HPHR", "Switch", "HPHR DAC"},
+
+	{"RX_BIAS", NULL, "DAC_REF"},
+
+	{"SPK_OUT", NULL, "SPK PA"},
+	{"SPK PA", NULL, "RX_BIAS"},
+	{"SPK PA", NULL, "SPKR_CLK"},
+	{"SPK PA", NULL, "SPK DAC"},
+	{"SPK DAC", "Switch", "PDM_RX3"},
+
+	{"MIC BIAS Internal1", NULL, "INT_LDO_H"},
+	{"MIC BIAS Internal2", NULL, "INT_LDO_H"},
+	{"MIC BIAS External1", NULL, "INT_LDO_H"},
+	{"MIC BIAS External2", NULL, "INT_LDO_H"},
+	{"MIC BIAS Internal1", NULL, "vdd-micbias"},
+	{"MIC BIAS Internal2", NULL, "vdd-micbias"},
+	{"MIC BIAS External1", NULL, "vdd-micbias"},
+	{"MIC BIAS External2", NULL, "vdd-micbias"},
+};
+
+static const struct snd_soc_dapm_widget pm8916_wcd_analog_dapm_widgets[] = {
+
+	SND_SOC_DAPM_AIF_IN("PDM_RX1", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_IN("PDM_RX2", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_IN("PDM_RX3", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("PDM_TX", NULL, 0, SND_SOC_NOPM, 0, 0),
+
+	SND_SOC_DAPM_INPUT("AMIC1"),
+	SND_SOC_DAPM_INPUT("AMIC3"),
+	SND_SOC_DAPM_INPUT("AMIC2"),
+	SND_SOC_DAPM_OUTPUT("HEADPHONE"),
+
+	/* RX stuff */
+	SND_SOC_DAPM_SUPPLY("INT_LDO_H", SND_SOC_NOPM, 1, 0, NULL, 0),
+
+	SND_SOC_DAPM_PGA("HPHL PA", CDC_A_RX_HPH_CNP_EN, 5, 0, NULL, 0),
+	SND_SOC_DAPM_MUX("HPHL", SND_SOC_NOPM, 0, 0, &hphl_mux),
+	SND_SOC_DAPM_MIXER("HPHL DAC", CDC_A_RX_HPH_L_PA_DAC_CTL, 3, 0, NULL,
+			   0),
+	SND_SOC_DAPM_PGA("HPHR PA", CDC_A_RX_HPH_CNP_EN, 4, 0, NULL, 0),
+	SND_SOC_DAPM_MUX("HPHR", SND_SOC_NOPM, 0, 0, &hphr_mux),
+	SND_SOC_DAPM_MIXER("HPHR DAC", CDC_A_RX_HPH_R_PA_DAC_CTL, 3, 0, NULL,
+			   0),
+	SND_SOC_DAPM_MIXER("SPK DAC", SND_SOC_NOPM, 0, 0,
+			   spkr_switch, ARRAY_SIZE(spkr_switch)),
+
+	/* Speaker */
+	SND_SOC_DAPM_OUTPUT("SPK_OUT"),
+	SND_SOC_DAPM_PGA_E("SPK PA", CDC_A_SPKR_DRV_CTL,
+			   6, 0, NULL, 0,
+			   pm8916_wcd_analog_enable_spk_pa,
+			   SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU |
+			   SND_SOC_DAPM_PRE_PMD | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_REGULATOR_SUPPLY("vdd-micbias", 0, 0),
+	SND_SOC_DAPM_SUPPLY("CP", CDC_A_NCP_EN, 0, 0, NULL, 0),
+
+	SND_SOC_DAPM_SUPPLY("DAC_REF", CDC_A_RX_COM_BIAS_DAC, 0, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("RX_BIAS", CDC_A_RX_COM_BIAS_DAC, 7, 0, NULL, 0),
+
+	/* TX */
+	SND_SOC_DAPM_SUPPLY("MIC BIAS Internal1", CDC_A_MICB_1_EN, 7, 0,
+			    pm8916_wcd_analog_enable_micbias_int1,
+			    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU |
+			    SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY("MIC BIAS Internal2", CDC_A_MICB_2_EN, 7, 0,
+			    pm8916_wcd_analog_enable_micbias_int2,
+			    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU |
+			    SND_SOC_DAPM_POST_PMD),
+
+	SND_SOC_DAPM_SUPPLY("MIC BIAS External1", CDC_A_MICB_1_EN, 7, 0,
+			    pm8916_wcd_analog_enable_micbias_ext1,
+			    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY("MIC BIAS External2", CDC_A_MICB_2_EN, 7, 0,
+			    pm8916_wcd_analog_enable_micbias_ext2,
+			    SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_POST_PMD),
+
+	SND_SOC_DAPM_ADC_E("ADC1", NULL, CDC_A_TX_1_EN, 7, 0,
+			   pm8916_wcd_analog_enable_adc,
+			   SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU |
+			   SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_ADC_E("ADC2_INP2", NULL, CDC_A_TX_2_EN, 7, 0,
+			   pm8916_wcd_analog_enable_adc,
+			   SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU |
+			   SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_ADC_E("ADC2_INP3", NULL, CDC_A_TX_3_EN, 7, 0,
+			   pm8916_wcd_analog_enable_adc,
+			   SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU |
+			   SND_SOC_DAPM_POST_PMD),
+
+	SND_SOC_DAPM_MIXER("ADC2", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("ADC3", SND_SOC_NOPM, 0, 0, NULL, 0),
+
+	SND_SOC_DAPM_MUX("ADC2 MUX", SND_SOC_NOPM, 0, 0, &tx_adc2_mux),
+	SND_SOC_DAPM_MUX("RDAC2 MUX", SND_SOC_NOPM, 0, 0, &rdac2_mux),
+
+	/* Analog path clocks */
+	SND_SOC_DAPM_SUPPLY("EAR_HPHR_CLK", CDC_D_CDC_ANA_CLK_CTL, 0, 0, NULL,
+			    0),
+	SND_SOC_DAPM_SUPPLY("EAR_HPHL_CLK", CDC_D_CDC_ANA_CLK_CTL, 1, 0, NULL,
+			    0),
+	SND_SOC_DAPM_SUPPLY("SPKR_CLK", CDC_D_CDC_ANA_CLK_CTL, 4, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("TXA_CLK25", CDC_D_CDC_ANA_CLK_CTL, 5, 0, NULL, 0),
+
+	/* Digital path clocks */
+
+	SND_SOC_DAPM_SUPPLY("RXD1_CLK", CDC_D_CDC_DIG_CLK_CTL, 0, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("RXD2_CLK", CDC_D_CDC_DIG_CLK_CTL, 1, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("RXD3_CLK", CDC_D_CDC_DIG_CLK_CTL, 2, 0, NULL, 0),
+
+	SND_SOC_DAPM_SUPPLY("TXD_CLK", CDC_D_CDC_DIG_CLK_CTL, 4, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("NCP_CLK", CDC_D_CDC_DIG_CLK_CTL, 6, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("RXD_PDM_CLK", CDC_D_CDC_DIG_CLK_CTL, 7, 0, NULL,
+			    0),
+
+	/* System Clock source */
+	SND_SOC_DAPM_SUPPLY("A_MCLK", CDC_D_CDC_TOP_CLK_CTL, 2, 0, NULL, 0),
+	/* TX ADC and RX DAC Clock source. */
+	SND_SOC_DAPM_SUPPLY("A_MCLK2", CDC_D_CDC_TOP_CLK_CTL, 3, 0, NULL, 0),
+};
+
+static struct regmap *pm8916_get_regmap(struct device *dev)
+{
+	return dev_get_regmap(dev->parent, NULL);
+}
+
+static int pm8916_wcd_analog_startup(struct snd_pcm_substream *substream,
+				      struct snd_soc_dai *dai)
+{
+	snd_soc_update_bits(dai->codec, CDC_D_CDC_RST_CTL,
+			    RST_CTL_DIG_SW_RST_N_MASK,
+			    RST_CTL_DIG_SW_RST_N_REMOVE_RESET);
+
+	return 0;
+}
+
+static void pm8916_wcd_analog_shutdown(struct snd_pcm_substream *substream,
+					 struct snd_soc_dai *dai)
+{
+	snd_soc_update_bits(dai->codec, CDC_D_CDC_RST_CTL,
+			    RST_CTL_DIG_SW_RST_N_MASK, 0);
+}
+
+static struct snd_soc_dai_ops pm8916_wcd_analog_dai_ops = {
+	.startup = pm8916_wcd_analog_startup,
+	.shutdown = pm8916_wcd_analog_shutdown,
+};
+
+static struct snd_soc_dai_driver pm8916_wcd_analog_dai[] = {
+	[0] = {
+	       .name = "pm8916_wcd_analog_pdm_rx",
+	       .id = 0,
+	       .playback = {
+			    .stream_name = "PDM Playback",
+			    .rates = MSM8916_WCD_ANALOG_RATES,
+			    .formats = MSM8916_WCD_ANALOG_FORMATS,
+			    .channels_min = 1,
+			    .channels_max = 3,
+			    },
+	       .ops = &pm8916_wcd_analog_dai_ops,
+	       },
+	[1] = {
+	       .name = "pm8916_wcd_analog_pdm_tx",
+	       .id = 1,
+	       .capture = {
+			   .stream_name = "PDM Capture",
+			   .rates = MSM8916_WCD_ANALOG_RATES,
+			   .formats = MSM8916_WCD_ANALOG_FORMATS,
+			   .channels_min = 1,
+			   .channels_max = 4,
+			   },
+	       .ops = &pm8916_wcd_analog_dai_ops,
+	       },
+};
+
+static struct snd_soc_codec_driver pm8916_wcd_analog = {
+	.probe = pm8916_wcd_analog_probe,
+	.remove = pm8916_wcd_analog_remove,
+	.get_regmap = pm8916_get_regmap,
+	.component_driver = {
+		.controls = pm8916_wcd_analog_snd_controls,
+		.num_controls = ARRAY_SIZE(pm8916_wcd_analog_snd_controls),
+		.dapm_widgets = pm8916_wcd_analog_dapm_widgets,
+		.num_dapm_widgets = ARRAY_SIZE(pm8916_wcd_analog_dapm_widgets),
+		.dapm_routes = pm8916_wcd_analog_audio_map,
+		.num_dapm_routes = ARRAY_SIZE(pm8916_wcd_analog_audio_map),
+	},
+};
+
+static int pm8916_wcd_analog_parse_dt(struct device *dev,
+				       struct pm8916_wcd_analog_priv *priv)
+{
+
+	if (of_property_read_bool(dev->of_node, "qcom,micbias1-ext-cap"))
+		priv->micbias1_cap_mode = MICB_1_EN_EXT_BYP_CAP;
+	else
+		priv->micbias1_cap_mode = MICB_1_EN_NO_EXT_BYP_CAP;
+
+	if (of_property_read_bool(dev->of_node, "qcom,micbias2-ext-cap"))
+		priv->micbias2_cap_mode = MICB_1_EN_EXT_BYP_CAP;
+	else
+		priv->micbias2_cap_mode = MICB_1_EN_NO_EXT_BYP_CAP;
+
+	return 0;
+}
+
+static int pm8916_wcd_analog_spmi_probe(struct platform_device *pdev)
+{
+	struct pm8916_wcd_analog_priv *priv;
+	struct device *dev = &pdev->dev;
+	int ret, i;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	ret = pm8916_wcd_analog_parse_dt(dev, priv);
+	if (ret < 0)
+		return ret;
+
+	priv->mclk = devm_clk_get(dev, "mclk");
+	if (IS_ERR(priv->mclk)) {
+		dev_err(dev, "failed to get mclk\n");
+		return PTR_ERR(priv->mclk);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(supply_names); i++)
+		priv->supplies[i].supply = supply_names[i];
+
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(priv->supplies),
+				    priv->supplies);
+	if (ret) {
+		dev_err(dev, "Failed to get regulator supplies %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(priv->mclk);
+	if (ret < 0) {
+		dev_err(dev, "failed to enable mclk %d\n", ret);
+		return ret;
+	}
+
+	dev_set_drvdata(dev, priv);
+
+	return snd_soc_register_codec(dev, &pm8916_wcd_analog,
+				      pm8916_wcd_analog_dai,
+				      ARRAY_SIZE(pm8916_wcd_analog_dai));
+}
+
+static int pm8916_wcd_analog_spmi_remove(struct platform_device *pdev)
+{
+	struct pm8916_wcd_analog_priv *priv = dev_get_drvdata(&pdev->dev);
+
+	snd_soc_unregister_codec(&pdev->dev);
+	clk_disable_unprepare(priv->mclk);
+
+	return 0;
+}
+
+static const struct of_device_id pm8916_wcd_analog_spmi_match_table[] = {
+	{ .compatible = "qcom,pm8916-wcd-analog-codec", },
+	{ }
+};
+
+static struct platform_driver pm8916_wcd_analog_spmi_driver = {
+	.driver = {
+		   .name = "qcom,pm8916-wcd-spmi-codec",
+		   .of_match_table = pm8916_wcd_analog_spmi_match_table,
+	},
+	.probe = pm8916_wcd_analog_spmi_probe,
+	.remove = pm8916_wcd_analog_spmi_remove,
+};
+
+module_platform_driver(pm8916_wcd_analog_spmi_driver);
+
+MODULE_AUTHOR("Srinivas Kandagatla <srinivas.kandagatla@linaro.org>");
+MODULE_DESCRIPTION("PMIC PM8916 WCD Analog Codec driver");
+MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/codecs/msm8916-wcd-digital.c b/sound/soc/codecs/msm8916-wcd-digital.c
new file mode 100644
index 000000000000..f690442af8c9
--- /dev/null
+++ b/sound/soc/codecs/msm8916-wcd-digital.c
@@ -0,0 +1,923 @@
+/* Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/types.h>
+#include <linux/clk.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+#include <sound/soc.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/tlv.h>
+
+#define LPASS_CDC_CLK_RX_RESET_CTL		(0x000)
+#define LPASS_CDC_CLK_TX_RESET_B1_CTL		(0x004)
+#define CLK_RX_RESET_B1_CTL_TX1_RESET_MASK	BIT(0)
+#define CLK_RX_RESET_B1_CTL_TX2_RESET_MASK	BIT(1)
+#define LPASS_CDC_CLK_DMIC_B1_CTL		(0x008)
+#define DMIC_B1_CTL_DMIC0_CLK_SEL_MASK		GENMASK(3, 1)
+#define DMIC_B1_CTL_DMIC0_CLK_SEL_DIV2		(0x0 << 1)
+#define DMIC_B1_CTL_DMIC0_CLK_SEL_DIV3		(0x1 << 1)
+#define DMIC_B1_CTL_DMIC0_CLK_SEL_DIV4		(0x2 << 1)
+#define DMIC_B1_CTL_DMIC0_CLK_SEL_DIV6		(0x3 << 1)
+#define DMIC_B1_CTL_DMIC0_CLK_SEL_DIV16		(0x4 << 1)
+#define DMIC_B1_CTL_DMIC0_CLK_EN_MASK		BIT(0)
+#define DMIC_B1_CTL_DMIC0_CLK_EN_ENABLE		BIT(0)
+
+#define LPASS_CDC_CLK_RX_I2S_CTL		(0x00C)
+#define RX_I2S_CTL_RX_I2S_MODE_MASK		BIT(5)
+#define RX_I2S_CTL_RX_I2S_MODE_16		BIT(5)
+#define RX_I2S_CTL_RX_I2S_MODE_32		0
+#define RX_I2S_CTL_RX_I2S_FS_RATE_MASK		GENMASK(2, 0)
+#define RX_I2S_CTL_RX_I2S_FS_RATE_F_8_KHZ	0x0
+#define RX_I2S_CTL_RX_I2S_FS_RATE_F_16_KHZ	0x1
+#define RX_I2S_CTL_RX_I2S_FS_RATE_F_32_KHZ	0x2
+#define RX_I2S_CTL_RX_I2S_FS_RATE_F_48_KHZ	0x3
+#define RX_I2S_CTL_RX_I2S_FS_RATE_F_96_KHZ	0x4
+#define RX_I2S_CTL_RX_I2S_FS_RATE_F_192_KHZ	0x5
+#define LPASS_CDC_CLK_TX_I2S_CTL		(0x010)
+#define TX_I2S_CTL_TX_I2S_MODE_MASK		BIT(5)
+#define TX_I2S_CTL_TX_I2S_MODE_16		BIT(5)
+#define TX_I2S_CTL_TX_I2S_MODE_32		0
+#define TX_I2S_CTL_TX_I2S_FS_RATE_MASK		GENMASK(2, 0)
+#define TX_I2S_CTL_TX_I2S_FS_RATE_F_8_KHZ	0x0
+#define TX_I2S_CTL_TX_I2S_FS_RATE_F_16_KHZ	0x1
+#define TX_I2S_CTL_TX_I2S_FS_RATE_F_32_KHZ	0x2
+#define TX_I2S_CTL_TX_I2S_FS_RATE_F_48_KHZ	0x3
+#define TX_I2S_CTL_TX_I2S_FS_RATE_F_96_KHZ	0x4
+#define TX_I2S_CTL_TX_I2S_FS_RATE_F_192_KHZ	0x5
+
+#define LPASS_CDC_CLK_OTHR_RESET_B1_CTL		(0x014)
+#define LPASS_CDC_CLK_TX_CLK_EN_B1_CTL		(0x018)
+#define LPASS_CDC_CLK_OTHR_CTL			(0x01C)
+#define LPASS_CDC_CLK_RX_B1_CTL			(0x020)
+#define LPASS_CDC_CLK_MCLK_CTL			(0x024)
+#define MCLK_CTL_MCLK_EN_MASK			BIT(0)
+#define MCLK_CTL_MCLK_EN_ENABLE			BIT(0)
+#define MCLK_CTL_MCLK_EN_DISABLE		0
+#define LPASS_CDC_CLK_PDM_CTL			(0x028)
+#define LPASS_CDC_CLK_PDM_CTL_PDM_EN_MASK	BIT(0)
+#define LPASS_CDC_CLK_PDM_CTL_PDM_EN		BIT(0)
+#define LPASS_CDC_CLK_PDM_CTL_PDM_CLK_SEL_MASK	BIT(1)
+#define LPASS_CDC_CLK_PDM_CTL_PDM_CLK_SEL_FB	BIT(1)
+#define LPASS_CDC_CLK_PDM_CTL_PDM_CLK_PDM_CLK	0
+
+#define LPASS_CDC_CLK_SD_CTL			(0x02C)
+#define LPASS_CDC_RX1_B1_CTL			(0x040)
+#define LPASS_CDC_RX2_B1_CTL			(0x060)
+#define LPASS_CDC_RX3_B1_CTL			(0x080)
+#define LPASS_CDC_RX1_B2_CTL			(0x044)
+#define LPASS_CDC_RX2_B2_CTL			(0x064)
+#define LPASS_CDC_RX3_B2_CTL			(0x084)
+#define LPASS_CDC_RX1_B3_CTL			(0x048)
+#define LPASS_CDC_RX2_B3_CTL			(0x068)
+#define LPASS_CDC_RX3_B3_CTL			(0x088)
+#define LPASS_CDC_RX1_B4_CTL			(0x04C)
+#define LPASS_CDC_RX2_B4_CTL			(0x06C)
+#define LPASS_CDC_RX3_B4_CTL			(0x08C)
+#define LPASS_CDC_RX1_B5_CTL			(0x050)
+#define LPASS_CDC_RX2_B5_CTL			(0x070)
+#define LPASS_CDC_RX3_B5_CTL			(0x090)
+#define LPASS_CDC_RX1_B6_CTL			(0x054)
+#define RXn_B6_CTL_MUTE_MASK			BIT(0)
+#define RXn_B6_CTL_MUTE_ENABLE			BIT(0)
+#define RXn_B6_CTL_MUTE_DISABLE			0
+#define LPASS_CDC_RX2_B6_CTL			(0x074)
+#define LPASS_CDC_RX3_B6_CTL			(0x094)
+#define LPASS_CDC_RX1_VOL_CTL_B1_CTL		(0x058)
+#define LPASS_CDC_RX2_VOL_CTL_B1_CTL		(0x078)
+#define LPASS_CDC_RX3_VOL_CTL_B1_CTL		(0x098)
+#define LPASS_CDC_RX1_VOL_CTL_B2_CTL		(0x05C)
+#define LPASS_CDC_RX2_VOL_CTL_B2_CTL		(0x07C)
+#define LPASS_CDC_RX3_VOL_CTL_B2_CTL		(0x09C)
+#define LPASS_CDC_TOP_GAIN_UPDATE		(0x0A0)
+#define LPASS_CDC_TOP_CTL			(0x0A4)
+#define TOP_CTL_DIG_MCLK_FREQ_MASK		BIT(0)
+#define TOP_CTL_DIG_MCLK_FREQ_F_12_288MHZ	0
+#define TOP_CTL_DIG_MCLK_FREQ_F_9_6MHZ		BIT(0)
+
+#define LPASS_CDC_DEBUG_DESER1_CTL		(0x0E0)
+#define LPASS_CDC_DEBUG_DESER2_CTL		(0x0E4)
+#define LPASS_CDC_DEBUG_B1_CTL_CFG		(0x0E8)
+#define LPASS_CDC_DEBUG_B2_CTL_CFG		(0x0EC)
+#define LPASS_CDC_DEBUG_B3_CTL_CFG		(0x0F0)
+#define LPASS_CDC_IIR1_GAIN_B1_CTL		(0x100)
+#define LPASS_CDC_IIR2_GAIN_B1_CTL		(0x140)
+#define LPASS_CDC_IIR1_GAIN_B2_CTL		(0x104)
+#define LPASS_CDC_IIR2_GAIN_B2_CTL		(0x144)
+#define LPASS_CDC_IIR1_GAIN_B3_CTL		(0x108)
+#define LPASS_CDC_IIR2_GAIN_B3_CTL		(0x148)
+#define LPASS_CDC_IIR1_GAIN_B4_CTL		(0x10C)
+#define LPASS_CDC_IIR2_GAIN_B4_CTL		(0x14C)
+#define LPASS_CDC_IIR1_GAIN_B5_CTL		(0x110)
+#define LPASS_CDC_IIR2_GAIN_B5_CTL		(0x150)
+#define LPASS_CDC_IIR1_GAIN_B6_CTL		(0x114)
+#define LPASS_CDC_IIR2_GAIN_B6_CTL		(0x154)
+#define LPASS_CDC_IIR1_GAIN_B7_CTL		(0x118)
+#define LPASS_CDC_IIR2_GAIN_B7_CTL		(0x158)
+#define LPASS_CDC_IIR1_GAIN_B8_CTL		(0x11C)
+#define LPASS_CDC_IIR2_GAIN_B8_CTL		(0x15C)
+#define LPASS_CDC_IIR1_CTL			(0x120)
+#define LPASS_CDC_IIR2_CTL			(0x160)
+#define LPASS_CDC_IIR1_GAIN_TIMER_CTL		(0x124)
+#define LPASS_CDC_IIR2_GAIN_TIMER_CTL		(0x164)
+#define LPASS_CDC_IIR1_COEF_B1_CTL		(0x128)
+#define LPASS_CDC_IIR2_COEF_B1_CTL		(0x168)
+#define LPASS_CDC_IIR1_COEF_B2_CTL		(0x12C)
+#define LPASS_CDC_IIR2_COEF_B2_CTL		(0x16C)
+#define LPASS_CDC_CONN_RX1_B1_CTL		(0x180)
+#define LPASS_CDC_CONN_RX1_B2_CTL		(0x184)
+#define LPASS_CDC_CONN_RX1_B3_CTL		(0x188)
+#define LPASS_CDC_CONN_RX2_B1_CTL		(0x18C)
+#define LPASS_CDC_CONN_RX2_B2_CTL		(0x190)
+#define LPASS_CDC_CONN_RX2_B3_CTL		(0x194)
+#define LPASS_CDC_CONN_RX3_B1_CTL		(0x198)
+#define LPASS_CDC_CONN_RX3_B2_CTL		(0x19C)
+#define LPASS_CDC_CONN_TX_B1_CTL		(0x1A0)
+#define LPASS_CDC_CONN_EQ1_B1_CTL		(0x1A8)
+#define LPASS_CDC_CONN_EQ1_B2_CTL		(0x1AC)
+#define LPASS_CDC_CONN_EQ1_B3_CTL		(0x1B0)
+#define LPASS_CDC_CONN_EQ1_B4_CTL		(0x1B4)
+#define LPASS_CDC_CONN_EQ2_B1_CTL		(0x1B8)
+#define LPASS_CDC_CONN_EQ2_B2_CTL		(0x1BC)
+#define LPASS_CDC_CONN_EQ2_B3_CTL		(0x1C0)
+#define LPASS_CDC_CONN_EQ2_B4_CTL		(0x1C4)
+#define LPASS_CDC_CONN_TX_I2S_SD1_CTL		(0x1C8)
+#define LPASS_CDC_TX1_VOL_CTL_TIMER		(0x280)
+#define LPASS_CDC_TX2_VOL_CTL_TIMER		(0x2A0)
+#define LPASS_CDC_TX1_VOL_CTL_GAIN		(0x284)
+#define LPASS_CDC_TX2_VOL_CTL_GAIN		(0x2A4)
+#define LPASS_CDC_TX1_VOL_CTL_CFG		(0x288)
+#define TX_VOL_CTL_CFG_MUTE_EN_MASK		BIT(0)
+#define TX_VOL_CTL_CFG_MUTE_EN_ENABLE		BIT(0)
+
+#define LPASS_CDC_TX2_VOL_CTL_CFG		(0x2A8)
+#define LPASS_CDC_TX1_MUX_CTL			(0x28C)
+#define TX_MUX_CTL_CUT_OFF_FREQ_MASK		GENMASK(5, 4)
+#define TX_MUX_CTL_CUT_OFF_FREQ_SHIFT		4
+#define TX_MUX_CTL_CF_NEG_3DB_4HZ		(0x0 << 4)
+#define TX_MUX_CTL_CF_NEG_3DB_75HZ		(0x1 << 4)
+#define TX_MUX_CTL_CF_NEG_3DB_150HZ		(0x2 << 4)
+#define TX_MUX_CTL_HPF_BP_SEL_MASK		BIT(3)
+#define TX_MUX_CTL_HPF_BP_SEL_BYPASS		BIT(3)
+#define TX_MUX_CTL_HPF_BP_SEL_NO_BYPASS		0
+
+#define LPASS_CDC_TX2_MUX_CTL			(0x2AC)
+#define LPASS_CDC_TX1_CLK_FS_CTL		(0x290)
+#define LPASS_CDC_TX2_CLK_FS_CTL		(0x2B0)
+#define LPASS_CDC_TX1_DMIC_CTL			(0x294)
+#define LPASS_CDC_TX2_DMIC_CTL			(0x2B4)
+#define TXN_DMIC_CTL_CLK_SEL_MASK		GENMASK(2, 0)
+#define TXN_DMIC_CTL_CLK_SEL_DIV2		0x0
+#define TXN_DMIC_CTL_CLK_SEL_DIV3		0x1
+#define TXN_DMIC_CTL_CLK_SEL_DIV4		0x2
+#define TXN_DMIC_CTL_CLK_SEL_DIV6		0x3
+#define TXN_DMIC_CTL_CLK_SEL_DIV16		0x4
+
+#define MSM8916_WCD_DIGITAL_RATES (SNDRV_PCM_RATE_8000 | \
+				   SNDRV_PCM_RATE_16000 | \
+				   SNDRV_PCM_RATE_32000 | \
+				   SNDRV_PCM_RATE_48000)
+#define MSM8916_WCD_DIGITAL_FORMATS (SNDRV_PCM_FMTBIT_S16_LE |\
+				     SNDRV_PCM_FMTBIT_S24_LE)
+
+struct msm8916_wcd_digital_priv {
+	struct clk *ahbclk, *mclk;
+};
+
+static const unsigned long rx_gain_reg[] = {
+	LPASS_CDC_RX1_VOL_CTL_B2_CTL,
+	LPASS_CDC_RX2_VOL_CTL_B2_CTL,
+	LPASS_CDC_RX3_VOL_CTL_B2_CTL,
+};
+
+static const unsigned long tx_gain_reg[] = {
+	LPASS_CDC_TX1_VOL_CTL_GAIN,
+	LPASS_CDC_TX2_VOL_CTL_GAIN,
+};
+
+static const char *const rx_mix1_text[] = {
+	"ZERO", "IIR1", "IIR2", "RX1", "RX2", "RX3"
+};
+
+static const char *const dec_mux_text[] = {
+	"ZERO", "ADC1", "ADC2", "ADC3", "DMIC1", "DMIC2"
+};
+static const char *const rx_mix2_text[] = { "ZERO", "IIR1", "IIR2" };
+static const char *const adc2_mux_text[] = { "ZERO", "INP2", "INP3" };
+
+/* RX1 MIX1 */
+static const struct soc_enum rx_mix1_inp_enum[] = {
+	SOC_ENUM_SINGLE(LPASS_CDC_CONN_RX1_B1_CTL, 0, 6, rx_mix1_text),
+	SOC_ENUM_SINGLE(LPASS_CDC_CONN_RX1_B1_CTL, 3, 6, rx_mix1_text),
+	SOC_ENUM_SINGLE(LPASS_CDC_CONN_RX1_B2_CTL, 0, 6, rx_mix1_text),
+};
+
+/* RX1 MIX2 */
+static const struct soc_enum rx_mix2_inp1_chain_enum = SOC_ENUM_SINGLE(
+				LPASS_CDC_CONN_RX1_B3_CTL, 0, 3, rx_mix2_text);
+
+/* RX2 MIX1 */
+static const struct soc_enum rx2_mix1_inp_enum[] = {
+	SOC_ENUM_SINGLE(LPASS_CDC_CONN_RX2_B1_CTL, 0, 6, rx_mix1_text),
+	SOC_ENUM_SINGLE(LPASS_CDC_CONN_RX2_B1_CTL, 3, 6, rx_mix1_text),
+	SOC_ENUM_SINGLE(LPASS_CDC_CONN_RX2_B1_CTL, 0, 6, rx_mix1_text),
+};
+
+/* RX2 MIX2 */
+static const struct soc_enum rx2_mix2_inp1_chain_enum = SOC_ENUM_SINGLE(
+				LPASS_CDC_CONN_RX2_B3_CTL, 0, 3, rx_mix2_text);
+
+/* RX3 MIX1 */
+static const struct soc_enum rx3_mix1_inp_enum[] = {
+	SOC_ENUM_SINGLE(LPASS_CDC_CONN_RX3_B1_CTL, 0, 6, rx_mix1_text),
+	SOC_ENUM_SINGLE(LPASS_CDC_CONN_RX3_B1_CTL, 3, 6, rx_mix1_text),
+	SOC_ENUM_SINGLE(LPASS_CDC_CONN_RX3_B1_CTL, 0, 6, rx_mix1_text),
+};
+
+/* DEC */
+static const struct soc_enum dec1_mux_enum = SOC_ENUM_SINGLE(
+				LPASS_CDC_CONN_TX_B1_CTL, 0, 6, dec_mux_text);
+static const struct soc_enum dec2_mux_enum = SOC_ENUM_SINGLE(
+				LPASS_CDC_CONN_TX_B1_CTL, 3, 6, dec_mux_text);
+
+/* RDAC2 MUX */
+static const struct snd_kcontrol_new dec1_mux = SOC_DAPM_ENUM(
+				"DEC1 MUX Mux", dec1_mux_enum);
+static const struct snd_kcontrol_new dec2_mux = SOC_DAPM_ENUM(
+				"DEC2 MUX Mux",	dec2_mux_enum);
+static const struct snd_kcontrol_new rx_mix1_inp1_mux = SOC_DAPM_ENUM(
+				"RX1 MIX1 INP1 Mux", rx_mix1_inp_enum[0]);
+static const struct snd_kcontrol_new rx_mix1_inp2_mux = SOC_DAPM_ENUM(
+				"RX1 MIX1 INP2 Mux", rx_mix1_inp_enum[1]);
+static const struct snd_kcontrol_new rx_mix1_inp3_mux = SOC_DAPM_ENUM(
+				"RX1 MIX1 INP3 Mux", rx_mix1_inp_enum[2]);
+static const struct snd_kcontrol_new rx2_mix1_inp1_mux = SOC_DAPM_ENUM(
+				"RX2 MIX1 INP1 Mux", rx2_mix1_inp_enum[0]);
+static const struct snd_kcontrol_new rx2_mix1_inp2_mux = SOC_DAPM_ENUM(
+				"RX2 MIX1 INP2 Mux", rx2_mix1_inp_enum[1]);
+static const struct snd_kcontrol_new rx2_mix1_inp3_mux = SOC_DAPM_ENUM(
+				"RX2 MIX1 INP3 Mux", rx2_mix1_inp_enum[2]);
+static const struct snd_kcontrol_new rx3_mix1_inp1_mux = SOC_DAPM_ENUM(
+				"RX3 MIX1 INP1 Mux", rx3_mix1_inp_enum[0]);
+static const struct snd_kcontrol_new rx3_mix1_inp2_mux = SOC_DAPM_ENUM(
+				"RX3 MIX1 INP2 Mux", rx3_mix1_inp_enum[1]);
+static const struct snd_kcontrol_new rx3_mix1_inp3_mux = SOC_DAPM_ENUM(
+				"RX3 MIX1 INP3 Mux", rx3_mix1_inp_enum[2]);
+
+/* Digital Gain control -38.4 dB to +38.4 dB in 0.3 dB steps */
+static const DECLARE_TLV_DB_SCALE(digital_gain, -3840, 30, 0);
+
+/* Cutoff Freq for High Pass Filter at -3dB */
+static const char * const hpf_cutoff_text[] = {
+	"4Hz", "75Hz", "150Hz",
+};
+
+static SOC_ENUM_SINGLE_DECL(tx1_hpf_cutoff_enum, LPASS_CDC_TX1_MUX_CTL, 4,
+			    hpf_cutoff_text);
+static SOC_ENUM_SINGLE_DECL(tx2_hpf_cutoff_enum, LPASS_CDC_TX2_MUX_CTL, 4,
+			    hpf_cutoff_text);
+
+/* cut off for dc blocker inside rx chain */
+static const char * const dc_blocker_cutoff_text[] = {
+	"4Hz", "75Hz", "150Hz",
+};
+
+static SOC_ENUM_SINGLE_DECL(rx1_dcb_cutoff_enum, LPASS_CDC_RX1_B4_CTL, 0,
+			    dc_blocker_cutoff_text);
+static SOC_ENUM_SINGLE_DECL(rx2_dcb_cutoff_enum, LPASS_CDC_RX2_B4_CTL, 0,
+			    dc_blocker_cutoff_text);
+static SOC_ENUM_SINGLE_DECL(rx3_dcb_cutoff_enum, LPASS_CDC_RX3_B4_CTL, 0,
+			    dc_blocker_cutoff_text);
+
+static const struct snd_kcontrol_new msm8916_wcd_digital_snd_controls[] = {
+	SOC_SINGLE_S8_TLV("RX1 Digital Volume", LPASS_CDC_RX1_VOL_CTL_B2_CTL,
+			  -128, 127, digital_gain),
+	SOC_SINGLE_S8_TLV("RX2 Digital Volume", LPASS_CDC_RX2_VOL_CTL_B2_CTL,
+			  -128, 127, digital_gain),
+	SOC_SINGLE_S8_TLV("RX3 Digital Volume", LPASS_CDC_RX3_VOL_CTL_B2_CTL,
+			  -128, 127, digital_gain),
+	SOC_SINGLE_S8_TLV("TX1 Digital Volume", LPASS_CDC_TX1_VOL_CTL_GAIN,
+			  -128, 127, digital_gain),
+	SOC_SINGLE_S8_TLV("TX2 Digital Volume", LPASS_CDC_TX2_VOL_CTL_GAIN,
+			  -128, 127, digital_gain),
+	SOC_ENUM("TX1 HPF Cutoff", tx1_hpf_cutoff_enum),
+	SOC_ENUM("TX2 HPF Cutoff", tx2_hpf_cutoff_enum),
+	SOC_SINGLE("TX1 HPF Switch", LPASS_CDC_TX1_MUX_CTL, 3, 1, 0),
+	SOC_SINGLE("TX2 HPF Switch", LPASS_CDC_TX2_MUX_CTL, 3, 1, 0),
+	SOC_ENUM("RX1 DCB Cutoff", rx1_dcb_cutoff_enum),
+	SOC_ENUM("RX2 DCB Cutoff", rx2_dcb_cutoff_enum),
+	SOC_ENUM("RX3 DCB Cutoff", rx3_dcb_cutoff_enum),
+	SOC_SINGLE("RX1 DCB Switch", LPASS_CDC_RX1_B5_CTL, 2, 1, 0),
+	SOC_SINGLE("RX2 DCB Switch", LPASS_CDC_RX2_B5_CTL, 2, 1, 0),
+	SOC_SINGLE("RX3 DCB Switch", LPASS_CDC_RX3_B5_CTL, 2, 1, 0),
+	SOC_SINGLE("RX1 Mute Switch", LPASS_CDC_RX1_B6_CTL, 0, 1, 0),
+	SOC_SINGLE("RX2 Mute Switch", LPASS_CDC_RX2_B6_CTL, 0, 1, 0),
+	SOC_SINGLE("RX3 Mute Switch", LPASS_CDC_RX3_B6_CTL, 0, 1, 0),
+};
+
+static int msm8916_wcd_digital_enable_interpolator(
+						struct snd_soc_dapm_widget *w,
+						struct snd_kcontrol *kcontrol,
+						int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+
+	switch (event) {
+	case SND_SOC_DAPM_POST_PMU:
+		/* apply the digital gain after the interpolator is enabled */
+		usleep_range(10000, 10100);
+		snd_soc_write(codec, rx_gain_reg[w->shift],
+			      snd_soc_read(codec, rx_gain_reg[w->shift]));
+		break;
+	}
+	return 0;
+}
+
+static int msm8916_wcd_digital_enable_dec(struct snd_soc_dapm_widget *w,
+					  struct snd_kcontrol *kcontrol,
+					  int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	unsigned int decimator = w->shift + 1;
+	u16 dec_reset_reg, tx_vol_ctl_reg, tx_mux_ctl_reg;
+	u8 dec_hpf_cut_of_freq;
+
+	dec_reset_reg = LPASS_CDC_CLK_TX_RESET_B1_CTL;
+	tx_vol_ctl_reg = LPASS_CDC_TX1_VOL_CTL_CFG + 32 * (decimator - 1);
+	tx_mux_ctl_reg = LPASS_CDC_TX1_MUX_CTL + 32 * (decimator - 1);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		/* Enable TX digital mute */
+		snd_soc_update_bits(codec, tx_vol_ctl_reg,
+				    TX_VOL_CTL_CFG_MUTE_EN_MASK,
+				    TX_VOL_CTL_CFG_MUTE_EN_ENABLE);
+		dec_hpf_cut_of_freq = snd_soc_read(codec, tx_mux_ctl_reg) &
+					TX_MUX_CTL_CUT_OFF_FREQ_MASK;
+		dec_hpf_cut_of_freq >>= TX_MUX_CTL_CUT_OFF_FREQ_SHIFT;
+		if (dec_hpf_cut_of_freq != TX_MUX_CTL_CF_NEG_3DB_150HZ) {
+			/* set cut of freq to CF_MIN_3DB_150HZ (0x1) */
+			snd_soc_update_bits(codec, tx_mux_ctl_reg,
+					    TX_MUX_CTL_CUT_OFF_FREQ_MASK,
+					    TX_MUX_CTL_CF_NEG_3DB_150HZ);
+		}
+		break;
+	case SND_SOC_DAPM_POST_PMU:
+		/* enable HPF */
+		snd_soc_update_bits(codec, tx_mux_ctl_reg,
+				    TX_MUX_CTL_HPF_BP_SEL_MASK,
+				    TX_MUX_CTL_HPF_BP_SEL_NO_BYPASS);
+		/* apply the digital gain after the decimator is enabled */
+		snd_soc_write(codec, tx_gain_reg[w->shift],
+			      snd_soc_read(codec, tx_gain_reg[w->shift]));
+		snd_soc_update_bits(codec, tx_vol_ctl_reg,
+				    TX_VOL_CTL_CFG_MUTE_EN_MASK, 0);
+		break;
+	case SND_SOC_DAPM_PRE_PMD:
+		snd_soc_update_bits(codec, tx_vol_ctl_reg,
+				    TX_VOL_CTL_CFG_MUTE_EN_MASK,
+				    TX_VOL_CTL_CFG_MUTE_EN_ENABLE);
+		snd_soc_update_bits(codec, tx_mux_ctl_reg,
+				    TX_MUX_CTL_HPF_BP_SEL_MASK,
+				    TX_MUX_CTL_HPF_BP_SEL_BYPASS);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		snd_soc_update_bits(codec, dec_reset_reg, 1 << w->shift,
+				    1 << w->shift);
+		snd_soc_update_bits(codec, dec_reset_reg, 1 << w->shift, 0x0);
+		snd_soc_update_bits(codec, tx_mux_ctl_reg,
+				    TX_MUX_CTL_HPF_BP_SEL_MASK,
+				    TX_MUX_CTL_HPF_BP_SEL_BYPASS);
+		snd_soc_update_bits(codec, tx_vol_ctl_reg,
+				    TX_VOL_CTL_CFG_MUTE_EN_MASK, 0);
+		break;
+	}
+
+	return 0;
+}
+
+static int msm8916_wcd_digital_enable_dmic(struct snd_soc_dapm_widget *w,
+					   struct snd_kcontrol *kcontrol,
+					   int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	unsigned int dmic;
+	int ret;
+	/* get dmic number out of widget name */
+	char *dmic_num = strpbrk(w->name, "12");
+
+	if (dmic_num == NULL) {
+		dev_err(codec->dev, "Invalid DMIC\n");
+		return -EINVAL;
+	}
+	ret = kstrtouint(dmic_num, 10, &dmic);
+	if (ret < 0 || dmic > 2) {
+		dev_err(codec->dev, "Invalid DMIC line on the codec\n");
+		return -EINVAL;
+	}
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		snd_soc_update_bits(codec, LPASS_CDC_CLK_DMIC_B1_CTL,
+				    DMIC_B1_CTL_DMIC0_CLK_SEL_MASK,
+				    DMIC_B1_CTL_DMIC0_CLK_SEL_DIV3);
+		switch (dmic) {
+		case 1:
+			snd_soc_update_bits(codec, LPASS_CDC_TX1_DMIC_CTL,
+					    TXN_DMIC_CTL_CLK_SEL_MASK,
+					    TXN_DMIC_CTL_CLK_SEL_DIV3);
+			break;
+		case 2:
+			snd_soc_update_bits(codec, LPASS_CDC_TX2_DMIC_CTL,
+					    TXN_DMIC_CTL_CLK_SEL_MASK,
+					    TXN_DMIC_CTL_CLK_SEL_DIV3);
+			break;
+		}
+		break;
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_dapm_widget msm8916_wcd_digital_dapm_widgets[] = {
+	/*RX stuff */
+	SND_SOC_DAPM_AIF_IN("I2S RX1", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_IN("I2S RX2", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_IN("I2S RX3", NULL, 0, SND_SOC_NOPM, 0, 0),
+
+	SND_SOC_DAPM_OUTPUT("PDM_RX1"),
+	SND_SOC_DAPM_OUTPUT("PDM_RX2"),
+	SND_SOC_DAPM_OUTPUT("PDM_RX3"),
+
+	SND_SOC_DAPM_INPUT("LPASS_PDM_TX"),
+
+	SND_SOC_DAPM_MIXER("RX1 MIX1", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("RX2 MIX1", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("RX3 MIX1", SND_SOC_NOPM, 0, 0, NULL, 0),
+
+	/* Interpolator */
+	SND_SOC_DAPM_MIXER_E("RX1 INT", LPASS_CDC_CLK_RX_B1_CTL, 0, 0, NULL,
+			     0, msm8916_wcd_digital_enable_interpolator,
+			     SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_MIXER_E("RX2 INT", LPASS_CDC_CLK_RX_B1_CTL, 1, 0, NULL,
+			     0, msm8916_wcd_digital_enable_interpolator,
+			     SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_MIXER_E("RX3 INT", LPASS_CDC_CLK_RX_B1_CTL, 2, 0, NULL,
+			     0, msm8916_wcd_digital_enable_interpolator,
+			     SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_MUX("RX1 MIX1 INP1", SND_SOC_NOPM, 0, 0,
+			 &rx_mix1_inp1_mux),
+	SND_SOC_DAPM_MUX("RX1 MIX1 INP2", SND_SOC_NOPM, 0, 0,
+			 &rx_mix1_inp2_mux),
+	SND_SOC_DAPM_MUX("RX1 MIX1 INP3", SND_SOC_NOPM, 0, 0,
+			 &rx_mix1_inp3_mux),
+	SND_SOC_DAPM_MUX("RX2 MIX1 INP1", SND_SOC_NOPM, 0, 0,
+			 &rx2_mix1_inp1_mux),
+	SND_SOC_DAPM_MUX("RX2 MIX1 INP2", SND_SOC_NOPM, 0, 0,
+			 &rx2_mix1_inp2_mux),
+	SND_SOC_DAPM_MUX("RX2 MIX1 INP3", SND_SOC_NOPM, 0, 0,
+			 &rx2_mix1_inp3_mux),
+	SND_SOC_DAPM_MUX("RX3 MIX1 INP1", SND_SOC_NOPM, 0, 0,
+			 &rx3_mix1_inp1_mux),
+	SND_SOC_DAPM_MUX("RX3 MIX1 INP2", SND_SOC_NOPM, 0, 0,
+			 &rx3_mix1_inp2_mux),
+	SND_SOC_DAPM_MUX("RX3 MIX1 INP3", SND_SOC_NOPM, 0, 0,
+			 &rx3_mix1_inp3_mux),
+
+	/* TX */
+	SND_SOC_DAPM_MIXER("ADC1", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("ADC2", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("ADC3", SND_SOC_NOPM, 0, 0, NULL, 0),
+
+	SND_SOC_DAPM_MUX_E("DEC1 MUX", LPASS_CDC_CLK_TX_CLK_EN_B1_CTL, 0, 0,
+			   &dec1_mux, msm8916_wcd_digital_enable_dec,
+			   SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU |
+			   SND_SOC_DAPM_PRE_PMD | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_MUX_E("DEC2 MUX", LPASS_CDC_CLK_TX_CLK_EN_B1_CTL, 1, 0,
+			   &dec2_mux, msm8916_wcd_digital_enable_dec,
+			   SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU |
+			   SND_SOC_DAPM_PRE_PMD | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_AIF_OUT("I2S TX1", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("I2S TX2", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("I2S TX3", NULL, 0, SND_SOC_NOPM, 0, 0),
+
+	/* Digital Mic Inputs */
+	SND_SOC_DAPM_ADC_E("DMIC1", NULL, SND_SOC_NOPM, 0, 0,
+			   msm8916_wcd_digital_enable_dmic,
+			   SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_ADC_E("DMIC2", NULL, SND_SOC_NOPM, 0, 0,
+			   msm8916_wcd_digital_enable_dmic,
+			   SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY("DMIC_CLK", LPASS_CDC_CLK_DMIC_B1_CTL, 0, 0,
+			    NULL, 0),
+	SND_SOC_DAPM_SUPPLY("RX_I2S_CLK", LPASS_CDC_CLK_RX_I2S_CTL,
+			    4, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("TX_I2S_CLK", LPASS_CDC_CLK_TX_I2S_CTL, 4, 0,
+			    NULL, 0),
+
+	SND_SOC_DAPM_SUPPLY("MCLK", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("PDM_CLK", LPASS_CDC_CLK_PDM_CTL, 0, 0, NULL, 0),
+	/* Connectivity Clock */
+	SND_SOC_DAPM_SUPPLY_S("CDC_CONN", -2, LPASS_CDC_CLK_OTHR_CTL, 2, 0,
+			      NULL, 0),
+
+};
+
+static int msm8916_wcd_digital_get_clks(struct platform_device *pdev,
+					struct msm8916_wcd_digital_priv	*priv)
+{
+	struct device *dev = &pdev->dev;
+
+	priv->ahbclk = devm_clk_get(dev, "ahbix-clk");
+	if (IS_ERR(priv->ahbclk)) {
+		dev_err(dev, "failed to get ahbix clk\n");
+		return PTR_ERR(priv->ahbclk);
+	}
+
+	priv->mclk = devm_clk_get(dev, "mclk");
+	if (IS_ERR(priv->mclk)) {
+		dev_err(dev, "failed to get mclk\n");
+		return PTR_ERR(priv->mclk);
+	}
+
+	return 0;
+}
+
+static int msm8916_wcd_digital_codec_probe(struct snd_soc_codec *codec)
+{
+	struct msm8916_wcd_digital_priv *priv = dev_get_drvdata(codec->dev);
+
+	snd_soc_codec_set_drvdata(codec, priv);
+
+	return 0;
+}
+
+static int msm8916_wcd_digital_hw_params(struct snd_pcm_substream *substream,
+					 struct snd_pcm_hw_params *params,
+					 struct snd_soc_dai *dai)
+{
+	u8 tx_fs_rate;
+	u8 rx_fs_rate;
+
+	switch (params_rate(params)) {
+	case 8000:
+		tx_fs_rate = TX_I2S_CTL_TX_I2S_FS_RATE_F_8_KHZ;
+		rx_fs_rate = RX_I2S_CTL_RX_I2S_FS_RATE_F_8_KHZ;
+		break;
+	case 16000:
+		tx_fs_rate = TX_I2S_CTL_TX_I2S_FS_RATE_F_16_KHZ;
+		rx_fs_rate = RX_I2S_CTL_RX_I2S_FS_RATE_F_16_KHZ;
+		break;
+	case 32000:
+		tx_fs_rate = TX_I2S_CTL_TX_I2S_FS_RATE_F_32_KHZ;
+		rx_fs_rate = RX_I2S_CTL_RX_I2S_FS_RATE_F_32_KHZ;
+		break;
+	case 48000:
+		tx_fs_rate = TX_I2S_CTL_TX_I2S_FS_RATE_F_48_KHZ;
+		rx_fs_rate = RX_I2S_CTL_RX_I2S_FS_RATE_F_48_KHZ;
+		break;
+	default:
+		dev_err(dai->codec->dev, "Invalid sampling rate %d\n",
+			params_rate(params));
+		return -EINVAL;
+	}
+
+	switch (substream->stream) {
+	case SNDRV_PCM_STREAM_CAPTURE:
+		snd_soc_update_bits(dai->codec, LPASS_CDC_CLK_TX_I2S_CTL,
+				    TX_I2S_CTL_TX_I2S_FS_RATE_MASK, tx_fs_rate);
+		break;
+	case SNDRV_PCM_STREAM_PLAYBACK:
+		snd_soc_update_bits(dai->codec, LPASS_CDC_CLK_RX_I2S_CTL,
+				    RX_I2S_CTL_RX_I2S_FS_RATE_MASK, rx_fs_rate);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_S16_LE:
+		snd_soc_update_bits(dai->codec, LPASS_CDC_CLK_TX_I2S_CTL,
+				    TX_I2S_CTL_TX_I2S_MODE_MASK,
+				    TX_I2S_CTL_TX_I2S_MODE_16);
+		snd_soc_update_bits(dai->codec, LPASS_CDC_CLK_RX_I2S_CTL,
+				    RX_I2S_CTL_RX_I2S_MODE_MASK,
+				    RX_I2S_CTL_RX_I2S_MODE_16);
+		break;
+	case SNDRV_PCM_FORMAT_S24_LE:
+		snd_soc_update_bits(dai->codec, LPASS_CDC_CLK_TX_I2S_CTL,
+				    TX_I2S_CTL_TX_I2S_MODE_MASK,
+				    TX_I2S_CTL_TX_I2S_MODE_32);
+		snd_soc_update_bits(dai->codec, LPASS_CDC_CLK_RX_I2S_CTL,
+				    RX_I2S_CTL_RX_I2S_MODE_MASK,
+				    RX_I2S_CTL_RX_I2S_MODE_32);
+		break;
+	default:
+		dev_err(dai->dev, "%s: wrong format selected\n", __func__);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_dapm_route msm8916_wcd_digital_audio_map[] = {
+
+	{"I2S RX1",  NULL, "AIF1 Playback"},
+	{"I2S RX2",  NULL, "AIF1 Playback"},
+	{"I2S RX3",  NULL, "AIF1 Playback"},
+
+	{"AIF1 Capture", NULL, "I2S TX1"},
+	{"AIF1 Capture", NULL, "I2S TX2"},
+	{"AIF1 Capture", NULL, "I2S TX3"},
+
+	/* Decimator Inputs */
+	{"DEC1 MUX", "DMIC1", "DMIC1"},
+	{"DEC1 MUX", "DMIC2", "DMIC2"},
+	{"DEC1 MUX", "ADC1", "ADC1"},
+	{"DEC1 MUX", "ADC2", "ADC2"},
+	{"DEC1 MUX", "ADC3", "ADC3"},
+	{"DEC1 MUX", NULL, "CDC_CONN"},
+
+	{"DEC2 MUX", "DMIC1", "DMIC1"},
+	{"DEC2 MUX", "DMIC2", "DMIC2"},
+	{"DEC2 MUX", "ADC1", "ADC1"},
+	{"DEC2 MUX", "ADC2", "ADC2"},
+	{"DEC2 MUX", "ADC3", "ADC3"},
+	{"DEC2 MUX", NULL, "CDC_CONN"},
+
+	{"DMIC1", NULL, "DMIC_CLK"},
+	{"DMIC2", NULL, "DMIC_CLK"},
+
+	{"I2S TX1", NULL, "DEC1 MUX"},
+	{"I2S TX2", NULL, "DEC2 MUX"},
+
+	{"I2S TX1", NULL, "TX_I2S_CLK"},
+	{"I2S TX2", NULL, "TX_I2S_CLK"},
+
+	{"TX_I2S_CLK", NULL, "MCLK"},
+	{"TX_I2S_CLK", NULL, "PDM_CLK"},
+
+	{"ADC1", NULL, "LPASS_PDM_TX"},
+	{"ADC2", NULL, "LPASS_PDM_TX"},
+	{"ADC3", NULL, "LPASS_PDM_TX"},
+
+	{"I2S RX1", NULL, "RX_I2S_CLK"},
+	{"I2S RX2", NULL, "RX_I2S_CLK"},
+	{"I2S RX3", NULL, "RX_I2S_CLK"},
+
+	{"RX_I2S_CLK", NULL, "PDM_CLK"},
+	{"RX_I2S_CLK", NULL, "MCLK"},
+	{"RX_I2S_CLK", NULL, "CDC_CONN"},
+
+	/* RX1 PATH.. */
+	{"PDM_RX1", NULL, "RX1 INT"},
+	{"RX1 INT", NULL, "RX1 MIX1"},
+
+	{"RX1 MIX1", NULL, "RX1 MIX1 INP1"},
+	{"RX1 MIX1", NULL, "RX1 MIX1 INP2"},
+	{"RX1 MIX1", NULL, "RX1 MIX1 INP3"},
+
+	{"RX1 MIX1 INP1", "RX1", "I2S RX1"},
+	{"RX1 MIX1 INP1", "RX2", "I2S RX2"},
+	{"RX1 MIX1 INP1", "RX3", "I2S RX3"},
+
+	{"RX1 MIX1 INP2", "RX1", "I2S RX1"},
+	{"RX1 MIX1 INP2", "RX2", "I2S RX2"},
+	{"RX1 MIX1 INP2", "RX3", "I2S RX3"},
+
+	{"RX1 MIX1 INP3", "RX1", "I2S RX1"},
+	{"RX1 MIX1 INP3", "RX2", "I2S RX2"},
+	{"RX1 MIX1 INP3", "RX3", "I2S RX3"},
+
+	/* RX2 PATH */
+	{"PDM_RX2", NULL, "RX2 INT"},
+	{"RX2 INT", NULL, "RX2 MIX1"},
+
+	{"RX2 MIX1", NULL, "RX2 MIX1 INP1"},
+	{"RX2 MIX1", NULL, "RX2 MIX1 INP2"},
+	{"RX2 MIX1", NULL, "RX2 MIX1 INP3"},
+
+	{"RX2 MIX1 INP1", "RX1", "I2S RX1"},
+	{"RX2 MIX1 INP1", "RX2", "I2S RX2"},
+	{"RX2 MIX1 INP1", "RX3", "I2S RX3"},
+
+	{"RX2 MIX1 INP2", "RX1", "I2S RX1"},
+	{"RX2 MIX1 INP2", "RX2", "I2S RX2"},
+	{"RX2 MIX1 INP2", "RX3", "I2S RX3"},
+
+	{"RX2 MIX1 INP3", "RX1", "I2S RX1"},
+	{"RX2 MIX1 INP3", "RX2", "I2S RX2"},
+	{"RX2 MIX1 INP3", "RX3", "I2S RX3"},
+
+	/* RX3 PATH */
+	{"PDM_RX3", NULL, "RX3 INT"},
+	{"RX3 INT", NULL, "RX3 MIX1"},
+
+	{"RX3 MIX1", NULL, "RX3 MIX1 INP1"},
+	{"RX3 MIX1", NULL, "RX3 MIX1 INP2"},
+	{"RX3 MIX1", NULL, "RX3 MIX1 INP3"},
+
+	{"RX3 MIX1 INP1", "RX1", "I2S RX1"},
+	{"RX3 MIX1 INP1", "RX2", "I2S RX2"},
+	{"RX3 MIX1 INP1", "RX3", "I2S RX3"},
+
+	{"RX3 MIX1 INP2", "RX1", "I2S RX1"},
+	{"RX3 MIX1 INP2", "RX2", "I2S RX2"},
+	{"RX3 MIX1 INP2", "RX3", "I2S RX3"},
+
+	{"RX3 MIX1 INP3", "RX1", "I2S RX1"},
+	{"RX3 MIX1 INP3", "RX2", "I2S RX2"},
+	{"RX3 MIX1 INP3", "RX3", "I2S RX3"},
+
+};
+
+static int msm8916_wcd_digital_startup(struct snd_pcm_substream *substream,
+				       struct snd_soc_dai *dai)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	struct msm8916_wcd_digital_priv *msm8916_wcd;
+	unsigned long mclk_rate;
+
+	msm8916_wcd = snd_soc_codec_get_drvdata(codec);
+	snd_soc_update_bits(codec, LPASS_CDC_CLK_MCLK_CTL,
+			    MCLK_CTL_MCLK_EN_MASK,
+			    MCLK_CTL_MCLK_EN_ENABLE);
+	snd_soc_update_bits(codec, LPASS_CDC_CLK_PDM_CTL,
+			    LPASS_CDC_CLK_PDM_CTL_PDM_CLK_SEL_MASK,
+			    LPASS_CDC_CLK_PDM_CTL_PDM_CLK_SEL_FB);
+
+	mclk_rate = clk_get_rate(msm8916_wcd->mclk);
+	switch (mclk_rate) {
+	case 12288000:
+		snd_soc_update_bits(codec, LPASS_CDC_TOP_CTL,
+				    TOP_CTL_DIG_MCLK_FREQ_MASK,
+				    TOP_CTL_DIG_MCLK_FREQ_F_12_288MHZ);
+		break;
+	case 9600000:
+		snd_soc_update_bits(codec, LPASS_CDC_TOP_CTL,
+				    TOP_CTL_DIG_MCLK_FREQ_MASK,
+				    TOP_CTL_DIG_MCLK_FREQ_F_9_6MHZ);
+		break;
+	default:
+		dev_err(codec->dev, "Invalid mclk rate %ld\n", mclk_rate);
+		break;
+	}
+	return 0;
+}
+
+static void msm8916_wcd_digital_shutdown(struct snd_pcm_substream *substream,
+					 struct snd_soc_dai *dai)
+{
+	snd_soc_update_bits(dai->codec, LPASS_CDC_CLK_PDM_CTL,
+			    LPASS_CDC_CLK_PDM_CTL_PDM_CLK_SEL_MASK, 0);
+}
+
+static struct snd_soc_dai_ops msm8916_wcd_digital_dai_ops = {
+	.startup = msm8916_wcd_digital_startup,
+	.shutdown = msm8916_wcd_digital_shutdown,
+	.hw_params = msm8916_wcd_digital_hw_params,
+};
+
+static struct snd_soc_dai_driver msm8916_wcd_digital_dai[] = {
+	[0] = {
+	       .name = "msm8916_wcd_digital_i2s_rx1",
+	       .id = 0,
+	       .playback = {
+			    .stream_name = "AIF1 Playback",
+			    .rates = MSM8916_WCD_DIGITAL_RATES,
+			    .formats = MSM8916_WCD_DIGITAL_FORMATS,
+			    .channels_min = 1,
+			    .channels_max = 3,
+			    },
+	       .ops = &msm8916_wcd_digital_dai_ops,
+	       },
+	[1] = {
+	       .name = "msm8916_wcd_digital_i2s_tx1",
+	       .id = 1,
+	       .capture = {
+			   .stream_name = "AIF1 Capture",
+			   .rates = MSM8916_WCD_DIGITAL_RATES,
+			   .formats = MSM8916_WCD_DIGITAL_FORMATS,
+			   .channels_min = 1,
+			   .channels_max = 4,
+			   },
+	       .ops = &msm8916_wcd_digital_dai_ops,
+	       },
+};
+
+static struct snd_soc_codec_driver msm8916_wcd_digital = {
+	.probe = msm8916_wcd_digital_codec_probe,
+	.component_driver = {
+		.controls = msm8916_wcd_digital_snd_controls,
+		.num_controls = ARRAY_SIZE(msm8916_wcd_digital_snd_controls),
+		.dapm_widgets = msm8916_wcd_digital_dapm_widgets,
+		.num_dapm_widgets =
+				 ARRAY_SIZE(msm8916_wcd_digital_dapm_widgets),
+		.dapm_routes = msm8916_wcd_digital_audio_map,
+		.num_dapm_routes = ARRAY_SIZE(msm8916_wcd_digital_audio_map),
+	},
+};
+
+static const struct regmap_config msm8916_codec_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = LPASS_CDC_TX2_DMIC_CTL,
+	.cache_type = REGCACHE_FLAT,
+};
+
+static int msm8916_wcd_digital_probe(struct platform_device *pdev)
+{
+	struct msm8916_wcd_digital_priv *priv;
+	struct device *dev = &pdev->dev;
+	void __iomem *base;
+	struct resource *mem_res;
+	struct regmap *digital_map;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(&pdev->dev, mem_res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	digital_map =
+	    devm_regmap_init_mmio(&pdev->dev, base,
+				  &msm8916_codec_regmap_config);
+	if (IS_ERR(digital_map))
+		return PTR_ERR(digital_map);
+
+	ret = msm8916_wcd_digital_get_clks(pdev, priv);
+	if (ret < 0)
+		return ret;
+
+	ret = clk_prepare_enable(priv->ahbclk);
+	if (ret < 0) {
+		dev_err(dev, "failed to enable ahbclk %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(priv->mclk);
+	if (ret < 0) {
+		dev_err(dev, "failed to enable mclk %d\n", ret);
+		return ret;
+	}
+
+	dev_set_drvdata(dev, priv);
+
+	return snd_soc_register_codec(dev, &msm8916_wcd_digital,
+				      msm8916_wcd_digital_dai,
+				      ARRAY_SIZE(msm8916_wcd_digital_dai));
+}
+
+static int msm8916_wcd_digital_remove(struct platform_device *pdev)
+{
+	struct msm8916_wcd_digital_priv *priv = dev_get_drvdata(&pdev->dev);
+
+	snd_soc_unregister_codec(&pdev->dev);
+	clk_disable_unprepare(priv->mclk);
+	clk_disable_unprepare(priv->ahbclk);
+
+	return 0;
+}
+
+static const struct of_device_id msm8916_wcd_digital_match_table[] = {
+	{ .compatible = "qcom,msm8916-wcd-digital-codec" },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(of, msm8916_wcd_digital_match_table);
+
+static struct platform_driver msm8916_wcd_digital_driver = {
+	.driver = {
+		   .name = "msm8916-wcd-digital-codec",
+		   .of_match_table = msm8916_wcd_digital_match_table,
+	},
+	.probe = msm8916_wcd_digital_probe,
+	.remove = msm8916_wcd_digital_remove,
+};
+
+module_platform_driver(msm8916_wcd_digital_driver);
+
+MODULE_AUTHOR("Srinivas Kandagatla <srinivas.kandagatla@linaro.org>");
+MODULE_DESCRIPTION("MSM8916 WCD Digital Codec driver");
+MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/codecs/nau8825.c b/sound/soc/codecs/nau8825.c
index e643be91d762..efe3a44658d5 100644
--- a/sound/soc/codecs/nau8825.c
+++ b/sound/soc/codecs/nau8825.c
@@ -43,6 +43,8 @@
 #define GAIN_AUGMENT 22500
 #define SIDETONE_BASE 207000
 
+/* the maximum frequency of CLK_ADC and CLK_DAC */
+#define CLK_DA_AD_MAX 6144000
 
 static int nau8825_configure_sysclk(struct nau8825 *nau8825,
 		int clk_id, unsigned int freq);
@@ -95,6 +97,27 @@ static const struct nau8825_fll_attr fll_pre_scalar[] = {
 	{ 8, 0x3 },
 };
 
+/* over sampling rate */
+struct nau8825_osr_attr {
+	unsigned int osr;
+	unsigned int clk_src;
+};
+
+static const struct nau8825_osr_attr osr_dac_sel[] = {
+	{ 64, 2 },	/* OSR 64, SRC 1/4 */
+	{ 256, 0 },	/* OSR 256, SRC 1 */
+	{ 128, 1 },	/* OSR 128, SRC 1/2 */
+	{ 0, 0 },
+	{ 32, 3 },	/* OSR 32, SRC 1/8 */
+};
+
+static const struct nau8825_osr_attr osr_adc_sel[] = {
+	{ 32, 3 },	/* OSR 32, SRC 1/8 */
+	{ 64, 2 },	/* OSR 64, SRC 1/4 */
+	{ 128, 1 },	/* OSR 128, SRC 1/2 */
+	{ 256, 0 },	/* OSR 256, SRC 1 */
+};
+
 static const struct reg_default nau8825_reg_defaults[] = {
 	{ NAU8825_REG_ENA_CTRL, 0x00ff },
 	{ NAU8825_REG_IIC_ADDR_SET, 0x0 },
@@ -1179,15 +1202,64 @@ static const struct snd_soc_dapm_route nau8825_dapm_routes[] = {
 	{"HPOR", NULL, "Class G"},
 };
 
+static int nau8825_clock_check(struct nau8825 *nau8825,
+	int stream, int rate, int osr)
+{
+	int osrate;
+
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		if (osr >= ARRAY_SIZE(osr_dac_sel))
+			return -EINVAL;
+		osrate = osr_dac_sel[osr].osr;
+	} else {
+		if (osr >= ARRAY_SIZE(osr_adc_sel))
+			return -EINVAL;
+		osrate = osr_adc_sel[osr].osr;
+	}
+
+	if (!osrate || rate * osr > CLK_DA_AD_MAX) {
+		dev_err(nau8825->dev, "exceed the maximum frequency of CLK_ADC or CLK_DAC\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int nau8825_hw_params(struct snd_pcm_substream *substream,
 				struct snd_pcm_hw_params *params,
 				struct snd_soc_dai *dai)
 {
 	struct snd_soc_codec *codec = dai->codec;
 	struct nau8825 *nau8825 = snd_soc_codec_get_drvdata(codec);
-	unsigned int val_len = 0;
+	unsigned int val_len = 0, osr;
+
+	nau8825_sema_acquire(nau8825, 3 * HZ);
 
-	nau8825_sema_acquire(nau8825, 2 * HZ);
+	/* CLK_DAC or CLK_ADC = OSR * FS
+	 * DAC or ADC clock frequency is defined as Over Sampling Rate (OSR)
+	 * multiplied by the audio sample rate (Fs). Note that the OSR and Fs
+	 * values must be selected such that the maximum frequency is less
+	 * than 6.144 MHz.
+	 */
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		regmap_read(nau8825->regmap, NAU8825_REG_DAC_CTRL1, &osr);
+		osr &= NAU8825_DAC_OVERSAMPLE_MASK;
+		if (nau8825_clock_check(nau8825, substream->stream,
+			params_rate(params), osr))
+			return -EINVAL;
+		regmap_update_bits(nau8825->regmap, NAU8825_REG_CLK_DIVIDER,
+			NAU8825_CLK_DAC_SRC_MASK,
+			osr_dac_sel[osr].clk_src << NAU8825_CLK_DAC_SRC_SFT);
+	} else {
+		regmap_read(nau8825->regmap, NAU8825_REG_ADC_RATE, &osr);
+		osr &= NAU8825_ADC_SYNC_DOWN_MASK;
+		if (nau8825_clock_check(nau8825, substream->stream,
+			params_rate(params), osr))
+			return -EINVAL;
+		regmap_update_bits(nau8825->regmap, NAU8825_REG_CLK_DIVIDER,
+			NAU8825_CLK_ADC_SRC_MASK,
+			osr_adc_sel[osr].clk_src << NAU8825_CLK_ADC_SRC_SFT);
+	}
 
 	switch (params_width(params)) {
 	case 16:
@@ -1221,7 +1293,7 @@ static int nau8825_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
 	struct nau8825 *nau8825 = snd_soc_codec_get_drvdata(codec);
 	unsigned int ctrl1_val = 0, ctrl2_val = 0;
 
-	nau8825_sema_acquire(nau8825, 2 * HZ);
+	nau8825_sema_acquire(nau8825, 3 * HZ);
 
 	switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) {
 	case SND_SOC_DAIFMT_CBM_CFM:
@@ -1774,9 +1846,10 @@ static void nau8825_init_regs(struct nau8825 *nau8825)
 	 * (audible hiss). Set it to something better.
 	 */
 	regmap_update_bits(regmap, NAU8825_REG_ADC_RATE,
-		NAU8825_ADC_SYNC_DOWN_MASK, NAU8825_ADC_SYNC_DOWN_128);
+		NAU8825_ADC_SYNC_DOWN_MASK | NAU8825_ADC_SINC4_EN,
+		NAU8825_ADC_SYNC_DOWN_64);
 	regmap_update_bits(regmap, NAU8825_REG_DAC_CTRL1,
-		NAU8825_DAC_OVERSAMPLE_MASK, NAU8825_DAC_OVERSAMPLE_128);
+		NAU8825_DAC_OVERSAMPLE_MASK, NAU8825_DAC_OVERSAMPLE_64);
 	/* Disable DACR/L power */
 	regmap_update_bits(regmap, NAU8825_REG_CHARGE_PUMP,
 		NAU8825_POWER_DOWN_DACR | NAU8825_POWER_DOWN_DACL,
@@ -1811,6 +1884,9 @@ static void nau8825_init_regs(struct nau8825 *nau8825)
 		NAU8825_DACL_CH_SEL_MASK, NAU8825_DACL_CH_SEL_L);
 	regmap_update_bits(nau8825->regmap, NAU8825_REG_DACR_CTRL,
 		NAU8825_DACL_CH_SEL_MASK, NAU8825_DACL_CH_SEL_R);
+	/* Disable short Frame Sync detection logic */
+	regmap_update_bits(regmap, NAU8825_REG_LEFT_TIME_SLOT,
+		NAU8825_DIS_FS_SHORT_DET, NAU8825_DIS_FS_SHORT_DET);
 }
 
 static const struct regmap_config nau8825_regmap_config = {
@@ -1919,8 +1995,10 @@ static void nau8825_fll_apply(struct nau8825 *nau8825,
 	regmap_update_bits(nau8825->regmap, NAU8825_REG_CLK_DIVIDER,
 		NAU8825_CLK_SRC_MASK | NAU8825_CLK_MCLK_SRC_MASK,
 		NAU8825_CLK_SRC_MCLK | fll_param->mclk_src);
+	/* Make DSP operate at high speed for better performance. */
 	regmap_update_bits(nau8825->regmap, NAU8825_REG_FLL1,
-			NAU8825_FLL_RATIO_MASK, fll_param->ratio);
+		NAU8825_FLL_RATIO_MASK | NAU8825_ICTRL_LATCH_MASK,
+		fll_param->ratio | (0x6 << NAU8825_ICTRL_LATCH_SFT));
 	/* FLL 16-bit fractional input */
 	regmap_write(nau8825->regmap, NAU8825_REG_FLL2, fll_param->fll_frac);
 	/* FLL 10-bit integer input */
@@ -1936,19 +2014,22 @@ static void nau8825_fll_apply(struct nau8825 *nau8825,
 	regmap_update_bits(nau8825->regmap,
 		NAU8825_REG_FLL6, NAU8825_DCO_EN, 0);
 	if (fll_param->fll_frac) {
+		/* set FLL loop filter enable and cutoff frequency at 500Khz */
 		regmap_update_bits(nau8825->regmap, NAU8825_REG_FLL5,
 			NAU8825_FLL_PDB_DAC_EN | NAU8825_FLL_LOOP_FTR_EN |
 			NAU8825_FLL_FTR_SW_MASK,
 			NAU8825_FLL_PDB_DAC_EN | NAU8825_FLL_LOOP_FTR_EN |
 			NAU8825_FLL_FTR_SW_FILTER);
 		regmap_update_bits(nau8825->regmap, NAU8825_REG_FLL6,
-			NAU8825_SDM_EN, NAU8825_SDM_EN);
+			NAU8825_SDM_EN | NAU8825_CUTOFF500,
+			NAU8825_SDM_EN | NAU8825_CUTOFF500);
 	} else {
+		/* disable FLL loop filter and cutoff frequency */
 		regmap_update_bits(nau8825->regmap, NAU8825_REG_FLL5,
 			NAU8825_FLL_PDB_DAC_EN | NAU8825_FLL_LOOP_FTR_EN |
 			NAU8825_FLL_FTR_SW_MASK, NAU8825_FLL_FTR_SW_ACCU);
-		regmap_update_bits(nau8825->regmap,
-			NAU8825_REG_FLL6, NAU8825_SDM_EN, 0);
+		regmap_update_bits(nau8825->regmap, NAU8825_REG_FLL6,
+			NAU8825_SDM_EN | NAU8825_CUTOFF500, 0);
 	}
 }
 
@@ -2014,6 +2095,9 @@ static void nau8825_configure_mclk_as_sysclk(struct regmap *regmap)
 		NAU8825_CLK_SRC_MASK, NAU8825_CLK_SRC_MCLK);
 	regmap_update_bits(regmap, NAU8825_REG_FLL6,
 		NAU8825_DCO_EN, 0);
+	/* Make DSP operate as default setting for power saving. */
+	regmap_update_bits(regmap, NAU8825_REG_FLL1,
+		NAU8825_ICTRL_LATCH_MASK, 0);
 }
 
 static int nau8825_configure_sysclk(struct nau8825 *nau8825, int clk_id,
@@ -2038,7 +2122,7 @@ static int nau8825_configure_sysclk(struct nau8825 *nau8825, int clk_id,
 		 * fered by cross talk process, the driver make the playback
 		 * preparation halted until cross talk process finish.
 		 */
-		nau8825_sema_acquire(nau8825, 2 * HZ);
+		nau8825_sema_acquire(nau8825, 3 * HZ);
 		nau8825_configure_mclk_as_sysclk(regmap);
 		/* MCLK not changed by clock tree */
 		regmap_update_bits(regmap, NAU8825_REG_CLK_DIVIDER,
@@ -2057,10 +2141,13 @@ static int nau8825_configure_sysclk(struct nau8825 *nau8825, int clk_id,
 				NAU8825_DCO_EN, NAU8825_DCO_EN);
 			regmap_update_bits(regmap, NAU8825_REG_CLK_DIVIDER,
 				NAU8825_CLK_SRC_MASK, NAU8825_CLK_SRC_VCO);
-			/* Decrease the VCO frequency for power saving */
+			/* Decrease the VCO frequency and make DSP operate
+			 * as default setting for power saving.
+			 */
 			regmap_update_bits(regmap, NAU8825_REG_CLK_DIVIDER,
 				NAU8825_CLK_MCLK_SRC_MASK, 0xf);
 			regmap_update_bits(regmap, NAU8825_REG_FLL1,
+				NAU8825_ICTRL_LATCH_MASK |
 				NAU8825_FLL_RATIO_MASK, 0x10);
 			regmap_update_bits(regmap, NAU8825_REG_FLL6,
 				NAU8825_SDM_EN, NAU8825_SDM_EN);
@@ -2083,9 +2170,14 @@ static int nau8825_configure_sysclk(struct nau8825 *nau8825, int clk_id,
 		 * fered by cross talk process, the driver make the playback
 		 * preparation halted until cross talk process finish.
 		 */
-		nau8825_sema_acquire(nau8825, 2 * HZ);
+		nau8825_sema_acquire(nau8825, 3 * HZ);
+		/* Higher FLL reference input frequency can only set lower
+		 * gain error, such as 0000 for input reference from MCLK
+		 * 12.288Mhz.
+		 */
 		regmap_update_bits(regmap, NAU8825_REG_FLL3,
-			NAU8825_FLL_CLK_SRC_MASK, NAU8825_FLL_CLK_SRC_MCLK);
+			NAU8825_FLL_CLK_SRC_MASK | NAU8825_GAIN_ERR_MASK,
+			NAU8825_FLL_CLK_SRC_MCLK | 0);
 		/* Release the semaphone. */
 		nau8825_sema_release(nau8825);
 
@@ -2100,9 +2192,17 @@ static int nau8825_configure_sysclk(struct nau8825 *nau8825, int clk_id,
 		 * fered by cross talk process, the driver make the playback
 		 * preparation halted until cross talk process finish.
 		 */
-		nau8825_sema_acquire(nau8825, 2 * HZ);
+		nau8825_sema_acquire(nau8825, 3 * HZ);
+		/* If FLL reference input is from low frequency source,
+		 * higher error gain can apply such as 0xf which has
+		 * the most sensitive gain error correction threshold,
+		 * Therefore, FLL has the most accurate DCO to
+		 * target frequency.
+		 */
 		regmap_update_bits(regmap, NAU8825_REG_FLL3,
-			NAU8825_FLL_CLK_SRC_MASK, NAU8825_FLL_CLK_SRC_BLK);
+			NAU8825_FLL_CLK_SRC_MASK | NAU8825_GAIN_ERR_MASK,
+			NAU8825_FLL_CLK_SRC_BLK |
+			(0xf << NAU8825_GAIN_ERR_SFT));
 		/* Release the semaphone. */
 		nau8825_sema_release(nau8825);
 
@@ -2118,9 +2218,17 @@ static int nau8825_configure_sysclk(struct nau8825 *nau8825, int clk_id,
 		 * fered by cross talk process, the driver make the playback
 		 * preparation halted until cross talk process finish.
 		 */
-		nau8825_sema_acquire(nau8825, 2 * HZ);
+		nau8825_sema_acquire(nau8825, 3 * HZ);
+		/* If FLL reference input is from low frequency source,
+		 * higher error gain can apply such as 0xf which has
+		 * the most sensitive gain error correction threshold,
+		 * Therefore, FLL has the most accurate DCO to
+		 * target frequency.
+		 */
 		regmap_update_bits(regmap, NAU8825_REG_FLL3,
-			NAU8825_FLL_CLK_SRC_MASK, NAU8825_FLL_CLK_SRC_FS);
+			NAU8825_FLL_CLK_SRC_MASK | NAU8825_GAIN_ERR_MASK,
+			NAU8825_FLL_CLK_SRC_FS |
+			(0xf << NAU8825_GAIN_ERR_SFT));
 		/* Release the semaphone. */
 		nau8825_sema_release(nau8825);
 
diff --git a/sound/soc/codecs/nau8825.h b/sound/soc/codecs/nau8825.h
index 1c63e2abafa9..5d1704e73241 100644
--- a/sound/soc/codecs/nau8825.h
+++ b/sound/soc/codecs/nau8825.h
@@ -115,12 +115,20 @@
 #define NAU8825_CLK_SRC_MASK			(1 << NAU8825_CLK_SRC_SFT)
 #define NAU8825_CLK_SRC_VCO			(1 << NAU8825_CLK_SRC_SFT)
 #define NAU8825_CLK_SRC_MCLK			(0 << NAU8825_CLK_SRC_SFT)
+#define NAU8825_CLK_ADC_SRC_SFT		6
+#define NAU8825_CLK_ADC_SRC_MASK		(0x3 << NAU8825_CLK_ADC_SRC_SFT)
+#define NAU8825_CLK_DAC_SRC_SFT		4
+#define NAU8825_CLK_DAC_SRC_MASK		(0x3 << NAU8825_CLK_DAC_SRC_SFT)
 #define NAU8825_CLK_MCLK_SRC_MASK		(0xf << 0)
 
 /* FLL1 (0x04) */
+#define NAU8825_ICTRL_LATCH_SFT	10
+#define NAU8825_ICTRL_LATCH_MASK	(0x7 << NAU8825_ICTRL_LATCH_SFT)
 #define NAU8825_FLL_RATIO_MASK			(0x7f << 0)
 
 /* FLL3 (0x06) */
+#define NAU8825_GAIN_ERR_SFT			12
+#define NAU8825_GAIN_ERR_MASK			(0xf << NAU8825_GAIN_ERR_SFT)
 #define NAU8825_FLL_INTEGER_MASK		(0x3ff << 0)
 #define NAU8825_FLL_CLK_SRC_SFT		10
 #define NAU8825_FLL_CLK_SRC_MASK		(0x3 << NAU8825_FLL_CLK_SRC_SFT)
@@ -144,6 +152,7 @@
 /* FLL6 (0x9) */
 #define NAU8825_DCO_EN				(0x1 << 15)
 #define NAU8825_SDM_EN				(0x1 << 14)
+#define NAU8825_CUTOFF500			(0x1 << 13)
 
 /* HSD_CTRL (0xc) */
 #define NAU8825_HSD_AUTO_MODE	(1 << 6)
@@ -246,6 +255,11 @@
 #define NAU8825_I2S_MS_SLAVE	(0 << NAU8825_I2S_MS_SFT)
 #define NAU8825_I2S_BLK_DIV_MASK	0x7
 
+/* LEFT_TIME_SLOT (0x1e) */
+#define NAU8825_FS_ERR_CMP_SEL_SFT	14
+#define NAU8825_FS_ERR_CMP_SEL_MASK	(0x3 << NAU8825_FS_ERR_CMP_SEL_SFT)
+#define NAU8825_DIS_FS_SHORT_DET	(1 << 13)
+
 /* BIQ_CTRL (0x20) */
 #define NAU8825_BIQ_WRT_SFT   4
 #define NAU8825_BIQ_WRT_EN     (1 << NAU8825_BIQ_WRT_SFT)
@@ -255,6 +269,8 @@
 #define NAU8825_BIQ_PATH_DAC   (1 << NAU8825_BIQ_PATH_SFT)
 
 /* ADC_RATE (0x2b) */
+#define NAU8825_ADC_SINC4_SFT		4
+#define NAU8825_ADC_SINC4_EN		(1 << NAU8825_ADC_SINC4_SFT)
 #define NAU8825_ADC_SYNC_DOWN_SFT	0
 #define NAU8825_ADC_SYNC_DOWN_MASK	0x3
 #define NAU8825_ADC_SYNC_DOWN_32	0
diff --git a/sound/soc/codecs/rl6231.c b/sound/soc/codecs/rl6231.c
index 1dc68ab08a17..7b447d0b173a 100644
--- a/sound/soc/codecs/rl6231.c
+++ b/sound/soc/codecs/rl6231.c
@@ -102,6 +102,7 @@ struct pll_calc_map {
 };
 
 static const struct pll_calc_map pll_preset_table[] = {
+	{19200000,  4096000,  23, 14, 1, false},
 	{19200000,  24576000,  3, 30, 3, false},
 };
 
diff --git a/sound/soc/codecs/rl6347a.c b/sound/soc/codecs/rl6347a.c
index a4b910efbd45..8f571cf8edd4 100644
--- a/sound/soc/codecs/rl6347a.c
+++ b/sound/soc/codecs/rl6347a.c
@@ -51,7 +51,7 @@ int rl6347a_hw_write(void *context, unsigned int reg, unsigned int value)
 	if (ret == 4)
 		return 0;
 	else
-		pr_err("ret=%d\n", ret);
+		dev_err(&client->dev, "I2C error %d\n", ret);
 	if (ret < 0)
 		return ret;
 	else
diff --git a/sound/soc/codecs/rt298.c b/sound/soc/codecs/rt298.c
index 2db8179047ae..7150a407ffd9 100644
--- a/sound/soc/codecs/rt298.c
+++ b/sound/soc/codecs/rt298.c
@@ -326,11 +326,31 @@ static void rt298_jack_detect_work(struct work_struct *work)
 int rt298_mic_detect(struct snd_soc_codec *codec, struct snd_soc_jack *jack)
 {
 	struct rt298_priv *rt298 = snd_soc_codec_get_drvdata(codec);
+	struct snd_soc_dapm_context *dapm;
+	bool hp = false;
+	bool mic = false;
+	int status = 0;
+
+	/* If jack in NULL, disable HS jack */
+	if (!jack) {
+		regmap_update_bits(rt298->regmap, RT298_IRQ_CTRL, 0x2, 0x0);
+		dapm = snd_soc_codec_get_dapm(codec);
+		snd_soc_dapm_disable_pin(dapm, "LDO1");
+		snd_soc_dapm_sync(dapm);
+		return 0;
+	}
 
 	rt298->jack = jack;
+	regmap_update_bits(rt298->regmap, RT298_IRQ_CTRL, 0x2, 0x2);
 
-	/* Send an initial empty report */
-	snd_soc_jack_report(rt298->jack, 0,
+	rt298_jack_detect(rt298, &hp, &mic);
+	if (hp == true)
+		status |= SND_JACK_HEADPHONE;
+
+	if (mic == true)
+		status |= SND_JACK_MICROPHONE;
+
+	snd_soc_jack_report(rt298->jack, status,
 		SND_JACK_MICROPHONE | SND_JACK_HEADPHONE);
 
 	return 0;
diff --git a/sound/soc/codecs/rt5514-spi.c b/sound/soc/codecs/rt5514-spi.c
index 09103aab0cb2..0901e25d6db6 100644
--- a/sound/soc/codecs/rt5514-spi.c
+++ b/sound/soc/codecs/rt5514-spi.c
@@ -20,7 +20,6 @@
 #include <linux/slab.h>
 #include <linux/gpio.h>
 #include <linux/sched.h>
-#include <linux/kthread.h>
 #include <linux/uaccess.h>
 #include <linux/miscdevice.h>
 #include <linux/regulator/consumer.h>
diff --git a/sound/soc/codecs/rt5514.c b/sound/soc/codecs/rt5514.c
index f24b7cfd3a89..b281a46d769d 100644
--- a/sound/soc/codecs/rt5514.c
+++ b/sound/soc/codecs/rt5514.c
@@ -452,6 +452,9 @@ static int rt5514_set_dmic_clk(struct snd_soc_dapm_widget *w,
 			RT5514_CLK_DMIC_OUT_SEL_MASK,
 			idx << RT5514_CLK_DMIC_OUT_SEL_SFT);
 
+	if (rt5514->pdata.dmic_init_delay)
+		msleep(rt5514->pdata.dmic_init_delay);
+
 	return idx;
 }
 
@@ -1073,9 +1076,18 @@ static const struct of_device_id rt5514_of_match[] = {
 MODULE_DEVICE_TABLE(of, rt5514_of_match);
 #endif
 
+static int rt5514_parse_dt(struct rt5514_priv *rt5514, struct device *dev)
+{
+	device_property_read_u32(dev, "realtek,dmic-init-delay-ms",
+		&rt5514->pdata.dmic_init_delay);
+
+	return 0;
+}
+
 static int rt5514_i2c_probe(struct i2c_client *i2c,
 		    const struct i2c_device_id *id)
 {
+	struct rt5514_platform_data *pdata = dev_get_platdata(&i2c->dev);
 	struct rt5514_priv *rt5514;
 	int ret;
 	unsigned int val;
@@ -1087,6 +1099,11 @@ static int rt5514_i2c_probe(struct i2c_client *i2c,
 
 	i2c_set_clientdata(i2c, rt5514);
 
+	if (pdata)
+		rt5514->pdata = *pdata;
+	else if (i2c->dev.of_node)
+		rt5514_parse_dt(rt5514, &i2c->dev);
+
 	rt5514->i2c_regmap = devm_regmap_init_i2c(i2c, &rt5514_i2c_regmap);
 	if (IS_ERR(rt5514->i2c_regmap)) {
 		ret = PTR_ERR(rt5514->i2c_regmap);
diff --git a/sound/soc/codecs/rt5514.h b/sound/soc/codecs/rt5514.h
index 229de0e2c88c..5d343fb6d125 100644
--- a/sound/soc/codecs/rt5514.h
+++ b/sound/soc/codecs/rt5514.h
@@ -13,6 +13,7 @@
 #define __RT5514_H__
 
 #include <linux/clk.h>
+#include <sound/rt5514.h>
 
 #define RT5514_DEVICE_ID			0x10ec5514
 
@@ -243,6 +244,7 @@ enum {
 };
 
 struct rt5514_priv {
+	struct rt5514_platform_data pdata;
 	struct snd_soc_codec *codec;
 	struct regmap *i2c_regmap, *regmap;
 	struct clk *mclk;
diff --git a/sound/soc/codecs/rt5616.c b/sound/soc/codecs/rt5616.c
index d1f273b24991..7d6e0823f98f 100644
--- a/sound/soc/codecs/rt5616.c
+++ b/sound/soc/codecs/rt5616.c
@@ -960,8 +960,7 @@ static int rt5616_hw_params(struct snd_pcm_substream *substream,
 			    struct snd_pcm_hw_params *params,
 			    struct snd_soc_dai *dai)
 {
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_codec *codec = rtd->codec;
+	struct snd_soc_codec *codec = dai->codec;
 	struct rt5616_priv *rt5616 = snd_soc_codec_get_drvdata(codec);
 	unsigned int val_len = 0, val_clk, mask_clk;
 	int pre_div, bclk_ms, frame_size;
diff --git a/sound/soc/codecs/rt5640.c b/sound/soc/codecs/rt5640.c
index 3cc1135fc2cd..e29a6defefa0 100644
--- a/sound/soc/codecs/rt5640.c
+++ b/sound/soc/codecs/rt5640.c
@@ -423,6 +423,8 @@ static const struct snd_kcontrol_new rt5640_snd_controls[] = {
 	SOC_DOUBLE_TLV("ADC Capture Volume", RT5640_ADC_DIG_VOL,
 			RT5640_L_VOL_SFT, RT5640_R_VOL_SFT,
 			127, 0, adc_vol_tlv),
+	SOC_DOUBLE("Mono ADC Capture Switch", RT5640_DUMMY1,
+		RT5640_M_MONO_ADC_L_SFT, RT5640_M_MONO_ADC_R_SFT, 1, 1),
 	SOC_DOUBLE_TLV("Mono ADC Capture Volume", RT5640_ADC_DATA,
 			RT5640_L_VOL_SFT, RT5640_R_VOL_SFT,
 			127, 0, adc_vol_tlv),
@@ -2407,6 +2409,9 @@ static int rt5640_i2c_probe(struct i2c_client *i2c,
 	if (ret != 0)
 		dev_warn(&i2c->dev, "Failed to apply regmap patch: %d\n", ret);
 
+	regmap_update_bits(rt5640->regmap, RT5640_DUMMY1,
+				RT5640_MCLK_DET, RT5640_MCLK_DET);
+
 	if (rt5640->pdata.in1_diff)
 		regmap_update_bits(rt5640->regmap, RT5640_IN1_IN2,
 					RT5640_IN_DF1, RT5640_IN_DF1);
diff --git a/sound/soc/codecs/rt5640.h b/sound/soc/codecs/rt5640.h
index 90c88711c72a..b8a811732a52 100644
--- a/sound/soc/codecs/rt5640.h
+++ b/sound/soc/codecs/rt5640.h
@@ -1970,6 +1970,12 @@
 #define RT5640_ZCD_HP_DIS			(0x0 << 15)
 #define RT5640_ZCD_HP_EN			(0x1 << 15)
 
+/* General Control 1 (0xfa) */
+#define RT5640_M_MONO_ADC_L			(0x1 << 13)
+#define RT5640_M_MONO_ADC_L_SFT			13
+#define RT5640_M_MONO_ADC_R			(0x1 << 12)
+#define RT5640_M_MONO_ADC_R_SFT			12
+#define RT5640_MCLK_DET				(0x1 << 11)
 
 /* Codec Private Register definition */
 /* 3D Speaker Control (0x63) */
diff --git a/sound/soc/codecs/rt5660.c b/sound/soc/codecs/rt5660.c
index 9f0933ced804..76cf76a2e9b6 100644
--- a/sound/soc/codecs/rt5660.c
+++ b/sound/soc/codecs/rt5660.c
@@ -1311,6 +1311,10 @@ static int rt5660_i2c_probe(struct i2c_client *i2c,
 	if (ret != 0)
 		dev_warn(&i2c->dev, "Failed to apply regmap patch: %d\n", ret);
 
+	regmap_update_bits(rt5660->regmap, RT5660_GEN_CTRL1,
+		RT5660_AUTO_DIS_AMP | RT5660_MCLK_DET | RT5660_POW_CLKDET,
+		RT5660_AUTO_DIS_AMP | RT5660_MCLK_DET | RT5660_POW_CLKDET);
+
 	if (rt5660->pdata.dmic1_data_pin) {
 		regmap_update_bits(rt5660->regmap, RT5660_GPIO_CTRL1,
 			RT5660_GP1_PIN_MASK, RT5660_GP1_PIN_DMIC1_SCL);
diff --git a/sound/soc/codecs/rt5660.h b/sound/soc/codecs/rt5660.h
index 6cdb9269ec9e..bba18fb66b6f 100644
--- a/sound/soc/codecs/rt5660.h
+++ b/sound/soc/codecs/rt5660.h
@@ -810,6 +810,9 @@
 /* General Control 1 (0xfa) */
 #define RT5660_PWR_VREF_HP			(0x1 << 11)
 #define RT5660_PWR_VREF_HP_SFT			11
+#define RT5660_AUTO_DIS_AMP			(0x1 << 6)
+#define RT5660_MCLK_DET				(0x1 << 5)
+#define RT5660_POW_CLKDET			(0x1 << 1)
 #define RT5660_DIG_GATE_CTRL			(0x1)
 #define RT5660_DIG_GATE_CTRL_SFT		0
 
diff --git a/sound/soc/codecs/rt5663.c b/sound/soc/codecs/rt5663.c
index 00ff2788879e..a32508d7dcfd 100644
--- a/sound/soc/codecs/rt5663.c
+++ b/sound/soc/codecs/rt5663.c
@@ -1,5 +1,5 @@
 /*
- * rt5663.c  --  RT5668/RT5663 ALSA SoC audio codec driver
+ * rt5663.c  --  RT5663 ALSA SoC audio codec driver
  *
  * Copyright 2016 Realtek Semiconductor Corp.
  * Author: Jack Yu <jack.yu@realtek.com>
@@ -30,12 +30,12 @@
 #include "rt5663.h"
 #include "rl6231.h"
 
-#define RT5668_DEVICE_ID 0x6451
-#define RT5663_DEVICE_ID 0x6406
+#define RT5663_DEVICE_ID_2 0x6451
+#define RT5663_DEVICE_ID_1 0x6406
 
 enum {
-	CODEC_TYPE_RT5668,
-	CODEC_TYPE_RT5663,
+	CODEC_VER_1,
+	CODEC_VER_0,
 };
 
 struct rt5663_priv {
@@ -45,7 +45,7 @@ struct rt5663_priv {
 	struct snd_soc_jack *hs_jack;
 	struct timer_list btn_check_timer;
 
-	int codec_type;
+	int codec_ver;
 	int sysclk;
 	int sysclk_src;
 	int lrck;
@@ -57,7 +57,7 @@ struct rt5663_priv {
 	int jack_type;
 };
 
-static const struct reg_default rt5668_reg[] = {
+static const struct reg_default rt5663_v2_reg[] = {
 	{ 0x0000, 0x0000 },
 	{ 0x0001, 0xc8c8 },
 	{ 0x0002, 0x8080 },
@@ -730,7 +730,7 @@ static bool rt5663_volatile_register(struct device *dev, unsigned int reg)
 	case RT5663_ADC_EQ_1:
 	case RT5663_INT_ST_1:
 	case RT5663_INT_ST_2:
-	case RT5663_GPIO_STA:
+	case RT5663_GPIO_STA1:
 	case RT5663_SIN_GEN_1:
 	case RT5663_IL_CMD_1:
 	case RT5663_IL_CMD_5:
@@ -846,7 +846,7 @@ static bool rt5663_readable_register(struct device *dev, unsigned int reg)
 	case RT5663_INT_ST_2:
 	case RT5663_GPIO_1:
 	case RT5663_GPIO_2:
-	case RT5663_GPIO_STA:
+	case RT5663_GPIO_STA1:
 	case RT5663_SIN_GEN_1:
 	case RT5663_SIN_GEN_2:
 	case RT5663_SIN_GEN_3:
@@ -1036,23 +1036,23 @@ static bool rt5663_readable_register(struct device *dev, unsigned int reg)
 	}
 }
 
-static bool rt5668_volatile_register(struct device *dev, unsigned int reg)
+static bool rt5663_v2_volatile_register(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
 	case RT5663_RESET:
-	case RT5668_CBJ_TYPE_2:
-	case RT5668_PDM_OUT_CTL:
-	case RT5668_PDM_I2C_DATA_CTL1:
-	case RT5668_PDM_I2C_DATA_CTL4:
-	case RT5668_ALC_BK_GAIN:
+	case RT5663_CBJ_TYPE_2:
+	case RT5663_PDM_OUT_CTL:
+	case RT5663_PDM_I2C_DATA_CTL1:
+	case RT5663_PDM_I2C_DATA_CTL4:
+	case RT5663_ALC_BK_GAIN:
 	case RT5663_PLL_2:
 	case RT5663_MICBIAS_1:
 	case RT5663_ADC_EQ_1:
 	case RT5663_INT_ST_1:
-	case RT5668_GPIO_STA:
+	case RT5663_GPIO_STA2:
 	case RT5663_IL_CMD_1:
 	case RT5663_IL_CMD_5:
-	case RT5668_A_JD_CTRL:
+	case RT5663_A_JD_CTRL:
 	case RT5663_JD_CTRL2:
 	case RT5663_VENDOR_ID:
 	case RT5663_VENDOR_ID_1:
@@ -1061,15 +1061,15 @@ static bool rt5668_volatile_register(struct device *dev, unsigned int reg)
 	case RT5663_STO_DRE_5:
 	case RT5663_STO_DRE_6:
 	case RT5663_STO_DRE_7:
-	case RT5668_MONO_DYNA_6:
-	case RT5668_STO1_SIL_DET:
-	case RT5668_MONOL_SIL_DET:
-	case RT5668_MONOR_SIL_DET:
-	case RT5668_STO2_DAC_SIL:
-	case RT5668_MONO_AMP_CAL_ST1:
-	case RT5668_MONO_AMP_CAL_ST2:
-	case RT5668_MONO_AMP_CAL_ST3:
-	case RT5668_MONO_AMP_CAL_ST4:
+	case RT5663_MONO_DYNA_6:
+	case RT5663_STO1_SIL_DET:
+	case RT5663_MONOL_SIL_DET:
+	case RT5663_MONOR_SIL_DET:
+	case RT5663_STO2_DAC_SIL:
+	case RT5663_MONO_AMP_CAL_ST1:
+	case RT5663_MONO_AMP_CAL_ST2:
+	case RT5663_MONO_AMP_CAL_ST3:
+	case RT5663_MONO_AMP_CAL_ST4:
 	case RT5663_HP_IMP_SEN_2:
 	case RT5663_HP_IMP_SEN_3:
 	case RT5663_HP_IMP_SEN_4:
@@ -1083,218 +1083,218 @@ static bool rt5668_volatile_register(struct device *dev, unsigned int reg)
 	case RT5663_HP_CALIB_ST7:
 	case RT5663_HP_CALIB_ST8:
 	case RT5663_HP_CALIB_ST9:
-	case RT5668_HP_CALIB_ST10:
-	case RT5668_HP_CALIB_ST11:
+	case RT5663_HP_CALIB_ST10:
+	case RT5663_HP_CALIB_ST11:
 		return true;
 	default:
 		return false;
 	}
 }
 
-static bool rt5668_readable_register(struct device *dev, unsigned int reg)
+static bool rt5663_v2_readable_register(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
-	case RT5668_LOUT_CTRL:
-	case RT5668_HP_AMP_2:
-	case RT5668_MONO_OUT:
-	case RT5668_MONO_GAIN:
-	case RT5668_AEC_BST:
-	case RT5668_IN1_IN2:
-	case RT5668_IN3_IN4:
-	case RT5668_INL1_INR1:
-	case RT5668_CBJ_TYPE_2:
-	case RT5668_CBJ_TYPE_3:
-	case RT5668_CBJ_TYPE_4:
-	case RT5668_CBJ_TYPE_5:
-	case RT5668_CBJ_TYPE_8:
-	case RT5668_DAC3_DIG_VOL:
-	case RT5668_DAC3_CTRL:
-	case RT5668_MONO_ADC_DIG_VOL:
-	case RT5668_STO2_ADC_DIG_VOL:
-	case RT5668_MONO_ADC_BST_GAIN:
-	case RT5668_STO2_ADC_BST_GAIN:
-	case RT5668_SIDETONE_CTRL:
-	case RT5668_MONO1_ADC_MIXER:
-	case RT5668_STO2_ADC_MIXER:
-	case RT5668_MONO_DAC_MIXER:
-	case RT5668_DAC2_SRC_CTRL:
-	case RT5668_IF_3_4_DATA_CTL:
-	case RT5668_IF_5_DATA_CTL:
-	case RT5668_PDM_OUT_CTL:
-	case RT5668_PDM_I2C_DATA_CTL1:
-	case RT5668_PDM_I2C_DATA_CTL2:
-	case RT5668_PDM_I2C_DATA_CTL3:
-	case RT5668_PDM_I2C_DATA_CTL4:
-	case RT5668_RECMIX1_NEW:
-	case RT5668_RECMIX1L_0:
-	case RT5668_RECMIX1L:
-	case RT5668_RECMIX1R_0:
-	case RT5668_RECMIX1R:
-	case RT5668_RECMIX2_NEW:
-	case RT5668_RECMIX2_L_2:
-	case RT5668_RECMIX2_R:
-	case RT5668_RECMIX2_R_2:
-	case RT5668_CALIB_REC_LR:
-	case RT5668_ALC_BK_GAIN:
-	case RT5668_MONOMIX_GAIN:
-	case RT5668_MONOMIX_IN_GAIN:
-	case RT5668_OUT_MIXL_GAIN:
-	case RT5668_OUT_LMIX_IN_GAIN:
-	case RT5668_OUT_RMIX_IN_GAIN:
-	case RT5668_OUT_RMIX_IN_GAIN1:
-	case RT5668_LOUT_MIXER_CTRL:
-	case RT5668_PWR_VOL:
-	case RT5668_ADCDAC_RST:
-	case RT5668_I2S34_SDP:
-	case RT5668_I2S5_SDP:
-	case RT5668_TDM_5:
-	case RT5668_TDM_6:
-	case RT5668_TDM_7:
-	case RT5668_TDM_8:
-	case RT5668_ASRC_3:
-	case RT5668_ASRC_6:
-	case RT5668_ASRC_7:
-	case RT5668_PLL_TRK_13:
-	case RT5668_I2S_M_CLK_CTL:
-	case RT5668_FDIV_I2S34_M_CLK:
-	case RT5668_FDIV_I2S34_M_CLK2:
-	case RT5668_FDIV_I2S5_M_CLK:
-	case RT5668_FDIV_I2S5_M_CLK2:
-	case RT5668_IRQ_4:
-	case RT5668_GPIO_3:
-	case RT5668_GPIO_4:
-	case RT5668_GPIO_STA:
-	case RT5668_HP_AMP_DET1:
-	case RT5668_HP_AMP_DET2:
-	case RT5668_HP_AMP_DET3:
-	case RT5668_MID_BD_HP_AMP:
-	case RT5668_LOW_BD_HP_AMP:
-	case RT5668_SOF_VOL_ZC2:
-	case RT5668_ADC_STO2_ADJ1:
-	case RT5668_ADC_STO2_ADJ2:
-	case RT5668_A_JD_CTRL:
-	case RT5668_JD1_TRES_CTRL:
-	case RT5668_JD2_TRES_CTRL:
-	case RT5668_JD_CTRL2:
-	case RT5668_DUM_REG_2:
-	case RT5668_DUM_REG_3:
+	case RT5663_LOUT_CTRL:
+	case RT5663_HP_AMP_2:
+	case RT5663_MONO_OUT:
+	case RT5663_MONO_GAIN:
+	case RT5663_AEC_BST:
+	case RT5663_IN1_IN2:
+	case RT5663_IN3_IN4:
+	case RT5663_INL1_INR1:
+	case RT5663_CBJ_TYPE_2:
+	case RT5663_CBJ_TYPE_3:
+	case RT5663_CBJ_TYPE_4:
+	case RT5663_CBJ_TYPE_5:
+	case RT5663_CBJ_TYPE_8:
+	case RT5663_DAC3_DIG_VOL:
+	case RT5663_DAC3_CTRL:
+	case RT5663_MONO_ADC_DIG_VOL:
+	case RT5663_STO2_ADC_DIG_VOL:
+	case RT5663_MONO_ADC_BST_GAIN:
+	case RT5663_STO2_ADC_BST_GAIN:
+	case RT5663_SIDETONE_CTRL:
+	case RT5663_MONO1_ADC_MIXER:
+	case RT5663_STO2_ADC_MIXER:
+	case RT5663_MONO_DAC_MIXER:
+	case RT5663_DAC2_SRC_CTRL:
+	case RT5663_IF_3_4_DATA_CTL:
+	case RT5663_IF_5_DATA_CTL:
+	case RT5663_PDM_OUT_CTL:
+	case RT5663_PDM_I2C_DATA_CTL1:
+	case RT5663_PDM_I2C_DATA_CTL2:
+	case RT5663_PDM_I2C_DATA_CTL3:
+	case RT5663_PDM_I2C_DATA_CTL4:
+	case RT5663_RECMIX1_NEW:
+	case RT5663_RECMIX1L_0:
+	case RT5663_RECMIX1L:
+	case RT5663_RECMIX1R_0:
+	case RT5663_RECMIX1R:
+	case RT5663_RECMIX2_NEW:
+	case RT5663_RECMIX2_L_2:
+	case RT5663_RECMIX2_R:
+	case RT5663_RECMIX2_R_2:
+	case RT5663_CALIB_REC_LR:
+	case RT5663_ALC_BK_GAIN:
+	case RT5663_MONOMIX_GAIN:
+	case RT5663_MONOMIX_IN_GAIN:
+	case RT5663_OUT_MIXL_GAIN:
+	case RT5663_OUT_LMIX_IN_GAIN:
+	case RT5663_OUT_RMIX_IN_GAIN:
+	case RT5663_OUT_RMIX_IN_GAIN1:
+	case RT5663_LOUT_MIXER_CTRL:
+	case RT5663_PWR_VOL:
+	case RT5663_ADCDAC_RST:
+	case RT5663_I2S34_SDP:
+	case RT5663_I2S5_SDP:
+	case RT5663_TDM_6:
+	case RT5663_TDM_7:
+	case RT5663_TDM_8:
+	case RT5663_TDM_9:
+	case RT5663_ASRC_3:
+	case RT5663_ASRC_6:
+	case RT5663_ASRC_7:
+	case RT5663_PLL_TRK_13:
+	case RT5663_I2S_M_CLK_CTL:
+	case RT5663_FDIV_I2S34_M_CLK:
+	case RT5663_FDIV_I2S34_M_CLK2:
+	case RT5663_FDIV_I2S5_M_CLK:
+	case RT5663_FDIV_I2S5_M_CLK2:
+	case RT5663_V2_IRQ_4:
+	case RT5663_GPIO_3:
+	case RT5663_GPIO_4:
+	case RT5663_GPIO_STA2:
+	case RT5663_HP_AMP_DET1:
+	case RT5663_HP_AMP_DET2:
+	case RT5663_HP_AMP_DET3:
+	case RT5663_MID_BD_HP_AMP:
+	case RT5663_LOW_BD_HP_AMP:
+	case RT5663_SOF_VOL_ZC2:
+	case RT5663_ADC_STO2_ADJ1:
+	case RT5663_ADC_STO2_ADJ2:
+	case RT5663_A_JD_CTRL:
+	case RT5663_JD1_TRES_CTRL:
+	case RT5663_JD2_TRES_CTRL:
+	case RT5663_V2_JD_CTRL2:
+	case RT5663_DUM_REG_2:
+	case RT5663_DUM_REG_3:
 	case RT5663_VENDOR_ID:
 	case RT5663_VENDOR_ID_1:
 	case RT5663_VENDOR_ID_2:
-	case RT5668_DACADC_DIG_VOL2:
-	case RT5668_DIG_IN_PIN2:
-	case RT5668_PAD_DRV_CTL1:
-	case RT5668_SOF_RAM_DEPOP:
-	case RT5668_VOL_TEST:
-	case RT5668_TEST_MODE_3:
-	case RT5668_TEST_MODE_4:
+	case RT5663_DACADC_DIG_VOL2:
+	case RT5663_DIG_IN_PIN2:
+	case RT5663_PAD_DRV_CTL1:
+	case RT5663_SOF_RAM_DEPOP:
+	case RT5663_VOL_TEST:
+	case RT5663_TEST_MODE_4:
+	case RT5663_TEST_MODE_5:
 	case RT5663_STO_DRE_9:
-	case RT5668_MONO_DYNA_1:
-	case RT5668_MONO_DYNA_2:
-	case RT5668_MONO_DYNA_3:
-	case RT5668_MONO_DYNA_4:
-	case RT5668_MONO_DYNA_5:
-	case RT5668_MONO_DYNA_6:
-	case RT5668_STO1_SIL_DET:
-	case RT5668_MONOL_SIL_DET:
-	case RT5668_MONOR_SIL_DET:
-	case RT5668_STO2_DAC_SIL:
-	case RT5668_PWR_SAV_CTL1:
-	case RT5668_PWR_SAV_CTL2:
-	case RT5668_PWR_SAV_CTL3:
-	case RT5668_PWR_SAV_CTL4:
-	case RT5668_PWR_SAV_CTL5:
-	case RT5668_PWR_SAV_CTL6:
-	case RT5668_MONO_AMP_CAL1:
-	case RT5668_MONO_AMP_CAL2:
-	case RT5668_MONO_AMP_CAL3:
-	case RT5668_MONO_AMP_CAL4:
-	case RT5668_MONO_AMP_CAL5:
-	case RT5668_MONO_AMP_CAL6:
-	case RT5668_MONO_AMP_CAL7:
-	case RT5668_MONO_AMP_CAL_ST1:
-	case RT5668_MONO_AMP_CAL_ST2:
-	case RT5668_MONO_AMP_CAL_ST3:
-	case RT5668_MONO_AMP_CAL_ST4:
-	case RT5668_MONO_AMP_CAL_ST5:
-	case RT5668_HP_IMP_SEN_13:
-	case RT5668_HP_IMP_SEN_14:
-	case RT5668_HP_IMP_SEN_6:
-	case RT5668_HP_IMP_SEN_7:
-	case RT5668_HP_IMP_SEN_8:
-	case RT5668_HP_IMP_SEN_9:
-	case RT5668_HP_IMP_SEN_10:
-	case RT5668_HP_LOGIC_3:
-	case RT5668_HP_CALIB_ST10:
-	case RT5668_HP_CALIB_ST11:
-	case RT5668_PRO_REG_TBL_4:
-	case RT5668_PRO_REG_TBL_5:
-	case RT5668_PRO_REG_TBL_6:
-	case RT5668_PRO_REG_TBL_7:
-	case RT5668_PRO_REG_TBL_8:
-	case RT5668_PRO_REG_TBL_9:
-	case RT5668_SAR_ADC_INL_1:
-	case RT5668_SAR_ADC_INL_2:
-	case RT5668_SAR_ADC_INL_3:
-	case RT5668_SAR_ADC_INL_4:
-	case RT5668_SAR_ADC_INL_5:
-	case RT5668_SAR_ADC_INL_6:
-	case RT5668_SAR_ADC_INL_7:
-	case RT5668_SAR_ADC_INL_8:
-	case RT5668_SAR_ADC_INL_9:
-	case RT5668_SAR_ADC_INL_10:
-	case RT5668_SAR_ADC_INL_11:
-	case RT5668_SAR_ADC_INL_12:
-	case RT5668_DRC_CTRL_1:
-	case RT5668_DRC1_CTRL_2:
-	case RT5668_DRC1_CTRL_3:
-	case RT5668_DRC1_CTRL_4:
-	case RT5668_DRC1_CTRL_5:
-	case RT5668_DRC1_CTRL_6:
-	case RT5668_DRC1_HD_CTRL_1:
-	case RT5668_DRC1_HD_CTRL_2:
-	case RT5668_DRC1_PRI_REG_1:
-	case RT5668_DRC1_PRI_REG_2:
-	case RT5668_DRC1_PRI_REG_3:
-	case RT5668_DRC1_PRI_REG_4:
-	case RT5668_DRC1_PRI_REG_5:
-	case RT5668_DRC1_PRI_REG_6:
-	case RT5668_DRC1_PRI_REG_7:
-	case RT5668_DRC1_PRI_REG_8:
-	case RT5668_ALC_PGA_CTL_1:
-	case RT5668_ALC_PGA_CTL_2:
-	case RT5668_ALC_PGA_CTL_3:
-	case RT5668_ALC_PGA_CTL_4:
-	case RT5668_ALC_PGA_CTL_5:
-	case RT5668_ALC_PGA_CTL_6:
-	case RT5668_ALC_PGA_CTL_7:
-	case RT5668_ALC_PGA_CTL_8:
-	case RT5668_ALC_PGA_REG_1:
-	case RT5668_ALC_PGA_REG_2:
-	case RT5668_ALC_PGA_REG_3:
-	case RT5668_ADC_EQ_RECOV_1:
-	case RT5668_ADC_EQ_RECOV_2:
-	case RT5668_ADC_EQ_RECOV_3:
-	case RT5668_ADC_EQ_RECOV_4:
-	case RT5668_ADC_EQ_RECOV_5:
-	case RT5668_ADC_EQ_RECOV_6:
-	case RT5668_ADC_EQ_RECOV_7:
-	case RT5668_ADC_EQ_RECOV_8:
-	case RT5668_ADC_EQ_RECOV_9:
-	case RT5668_ADC_EQ_RECOV_10:
-	case RT5668_ADC_EQ_RECOV_11:
-	case RT5668_ADC_EQ_RECOV_12:
-	case RT5668_ADC_EQ_RECOV_13:
-	case RT5668_VID_HIDDEN:
-	case RT5668_VID_CUSTOMER:
-	case RT5668_SCAN_MODE:
-	case RT5668_I2C_BYPA:
+	case RT5663_MONO_DYNA_1:
+	case RT5663_MONO_DYNA_2:
+	case RT5663_MONO_DYNA_3:
+	case RT5663_MONO_DYNA_4:
+	case RT5663_MONO_DYNA_5:
+	case RT5663_MONO_DYNA_6:
+	case RT5663_STO1_SIL_DET:
+	case RT5663_MONOL_SIL_DET:
+	case RT5663_MONOR_SIL_DET:
+	case RT5663_STO2_DAC_SIL:
+	case RT5663_PWR_SAV_CTL1:
+	case RT5663_PWR_SAV_CTL2:
+	case RT5663_PWR_SAV_CTL3:
+	case RT5663_PWR_SAV_CTL4:
+	case RT5663_PWR_SAV_CTL5:
+	case RT5663_PWR_SAV_CTL6:
+	case RT5663_MONO_AMP_CAL1:
+	case RT5663_MONO_AMP_CAL2:
+	case RT5663_MONO_AMP_CAL3:
+	case RT5663_MONO_AMP_CAL4:
+	case RT5663_MONO_AMP_CAL5:
+	case RT5663_MONO_AMP_CAL6:
+	case RT5663_MONO_AMP_CAL7:
+	case RT5663_MONO_AMP_CAL_ST1:
+	case RT5663_MONO_AMP_CAL_ST2:
+	case RT5663_MONO_AMP_CAL_ST3:
+	case RT5663_MONO_AMP_CAL_ST4:
+	case RT5663_MONO_AMP_CAL_ST5:
+	case RT5663_V2_HP_IMP_SEN_13:
+	case RT5663_V2_HP_IMP_SEN_14:
+	case RT5663_V2_HP_IMP_SEN_6:
+	case RT5663_V2_HP_IMP_SEN_7:
+	case RT5663_V2_HP_IMP_SEN_8:
+	case RT5663_V2_HP_IMP_SEN_9:
+	case RT5663_V2_HP_IMP_SEN_10:
+	case RT5663_HP_LOGIC_3:
+	case RT5663_HP_CALIB_ST10:
+	case RT5663_HP_CALIB_ST11:
+	case RT5663_PRO_REG_TBL_4:
+	case RT5663_PRO_REG_TBL_5:
+	case RT5663_PRO_REG_TBL_6:
+	case RT5663_PRO_REG_TBL_7:
+	case RT5663_PRO_REG_TBL_8:
+	case RT5663_PRO_REG_TBL_9:
+	case RT5663_SAR_ADC_INL_1:
+	case RT5663_SAR_ADC_INL_2:
+	case RT5663_SAR_ADC_INL_3:
+	case RT5663_SAR_ADC_INL_4:
+	case RT5663_SAR_ADC_INL_5:
+	case RT5663_SAR_ADC_INL_6:
+	case RT5663_SAR_ADC_INL_7:
+	case RT5663_SAR_ADC_INL_8:
+	case RT5663_SAR_ADC_INL_9:
+	case RT5663_SAR_ADC_INL_10:
+	case RT5663_SAR_ADC_INL_11:
+	case RT5663_SAR_ADC_INL_12:
+	case RT5663_DRC_CTRL_1:
+	case RT5663_DRC1_CTRL_2:
+	case RT5663_DRC1_CTRL_3:
+	case RT5663_DRC1_CTRL_4:
+	case RT5663_DRC1_CTRL_5:
+	case RT5663_DRC1_CTRL_6:
+	case RT5663_DRC1_HD_CTRL_1:
+	case RT5663_DRC1_HD_CTRL_2:
+	case RT5663_DRC1_PRI_REG_1:
+	case RT5663_DRC1_PRI_REG_2:
+	case RT5663_DRC1_PRI_REG_3:
+	case RT5663_DRC1_PRI_REG_4:
+	case RT5663_DRC1_PRI_REG_5:
+	case RT5663_DRC1_PRI_REG_6:
+	case RT5663_DRC1_PRI_REG_7:
+	case RT5663_DRC1_PRI_REG_8:
+	case RT5663_ALC_PGA_CTL_1:
+	case RT5663_ALC_PGA_CTL_2:
+	case RT5663_ALC_PGA_CTL_3:
+	case RT5663_ALC_PGA_CTL_4:
+	case RT5663_ALC_PGA_CTL_5:
+	case RT5663_ALC_PGA_CTL_6:
+	case RT5663_ALC_PGA_CTL_7:
+	case RT5663_ALC_PGA_CTL_8:
+	case RT5663_ALC_PGA_REG_1:
+	case RT5663_ALC_PGA_REG_2:
+	case RT5663_ALC_PGA_REG_3:
+	case RT5663_ADC_EQ_RECOV_1:
+	case RT5663_ADC_EQ_RECOV_2:
+	case RT5663_ADC_EQ_RECOV_3:
+	case RT5663_ADC_EQ_RECOV_4:
+	case RT5663_ADC_EQ_RECOV_5:
+	case RT5663_ADC_EQ_RECOV_6:
+	case RT5663_ADC_EQ_RECOV_7:
+	case RT5663_ADC_EQ_RECOV_8:
+	case RT5663_ADC_EQ_RECOV_9:
+	case RT5663_ADC_EQ_RECOV_10:
+	case RT5663_ADC_EQ_RECOV_11:
+	case RT5663_ADC_EQ_RECOV_12:
+	case RT5663_ADC_EQ_RECOV_13:
+	case RT5663_VID_HIDDEN:
+	case RT5663_VID_CUSTOMER:
+	case RT5663_SCAN_MODE:
+	case RT5663_I2C_BYPA:
 		return true;
 	case RT5663_TDM_1:
 	case RT5663_DEPOP_3:
 	case RT5663_ASRC_11_2:
 	case RT5663_INT_ST_2:
-	case RT5663_GPIO_STA:
+	case RT5663_GPIO_STA1:
 	case RT5663_SIN_GEN_1:
 	case RT5663_SIN_GEN_2:
 	case RT5663_SIN_GEN_3:
@@ -1344,7 +1344,7 @@ static bool rt5668_readable_register(struct device *dev, unsigned int reg)
 }
 
 static const DECLARE_TLV_DB_SCALE(rt5663_hp_vol_tlv, -2400, 150, 0);
-static const DECLARE_TLV_DB_SCALE(rt5668_hp_vol_tlv, -2250, 150, 0);
+static const DECLARE_TLV_DB_SCALE(rt5663_v2_hp_vol_tlv, -2250, 150, 0);
 static const DECLARE_TLV_DB_SCALE(dac_vol_tlv, -6525, 75, 0);
 static const DECLARE_TLV_DB_SCALE(adc_vol_tlv, -1725, 75, 0);
 
@@ -1374,57 +1374,57 @@ static void rt5663_enable_push_button_irq(struct snd_soc_codec *codec,
 
 	if (enable) {
 		snd_soc_update_bits(codec, RT5663_IL_CMD_6,
-			RT5668_EN_4BTN_INL_MASK, RT5668_EN_4BTN_INL_EN);
+			RT5663_EN_4BTN_INL_MASK, RT5663_EN_4BTN_INL_EN);
 		/* reset in-line command */
 		snd_soc_update_bits(codec, RT5663_IL_CMD_6,
-			RT5668_RESET_4BTN_INL_MASK,
-			RT5668_RESET_4BTN_INL_RESET);
+			RT5663_RESET_4BTN_INL_MASK,
+			RT5663_RESET_4BTN_INL_RESET);
 		snd_soc_update_bits(codec, RT5663_IL_CMD_6,
-			RT5668_RESET_4BTN_INL_MASK,
-			RT5668_RESET_4BTN_INL_NOR);
-		switch (rt5663->codec_type) {
-		case CODEC_TYPE_RT5668:
+			RT5663_RESET_4BTN_INL_MASK,
+			RT5663_RESET_4BTN_INL_NOR);
+		switch (rt5663->codec_ver) {
+		case CODEC_VER_1:
 			snd_soc_update_bits(codec, RT5663_IRQ_3,
-				RT5668_EN_IRQ_INLINE_MASK,
-				RT5668_EN_IRQ_INLINE_NOR);
+				RT5663_V2_EN_IRQ_INLINE_MASK,
+				RT5663_V2_EN_IRQ_INLINE_NOR);
 			break;
-		case CODEC_TYPE_RT5663:
+		case CODEC_VER_0:
 			snd_soc_update_bits(codec, RT5663_IRQ_2,
 				RT5663_EN_IRQ_INLINE_MASK,
 				RT5663_EN_IRQ_INLINE_NOR);
 			break;
 		default:
-			dev_err(codec->dev, "Unknown CODEC_TYPE\n");
+			dev_err(codec->dev, "Unknown CODEC Version\n");
 		}
 	} else {
-		switch (rt5663->codec_type) {
-		case CODEC_TYPE_RT5668:
+		switch (rt5663->codec_ver) {
+		case CODEC_VER_1:
 			snd_soc_update_bits(codec, RT5663_IRQ_3,
-				RT5668_EN_IRQ_INLINE_MASK,
-				RT5668_EN_IRQ_INLINE_BYP);
+				RT5663_V2_EN_IRQ_INLINE_MASK,
+				RT5663_V2_EN_IRQ_INLINE_BYP);
 			break;
-		case CODEC_TYPE_RT5663:
+		case CODEC_VER_0:
 			snd_soc_update_bits(codec, RT5663_IRQ_2,
 				RT5663_EN_IRQ_INLINE_MASK,
 				RT5663_EN_IRQ_INLINE_BYP);
 			break;
 		default:
-			dev_err(codec->dev, "Unknown CODEC_TYPE\n");
+			dev_err(codec->dev, "Unknown CODEC Version\n");
 		}
 		snd_soc_update_bits(codec, RT5663_IL_CMD_6,
-			RT5668_EN_4BTN_INL_MASK, RT5668_EN_4BTN_INL_DIS);
+			RT5663_EN_4BTN_INL_MASK, RT5663_EN_4BTN_INL_DIS);
 		/* reset in-line command */
 		snd_soc_update_bits(codec, RT5663_IL_CMD_6,
-			RT5668_RESET_4BTN_INL_MASK,
-			RT5668_RESET_4BTN_INL_RESET);
+			RT5663_RESET_4BTN_INL_MASK,
+			RT5663_RESET_4BTN_INL_RESET);
 		snd_soc_update_bits(codec, RT5663_IL_CMD_6,
-			RT5668_RESET_4BTN_INL_MASK,
-			RT5668_RESET_4BTN_INL_NOR);
+			RT5663_RESET_4BTN_INL_MASK,
+			RT5663_RESET_4BTN_INL_NOR);
 	}
 }
 
 /**
- * rt5668_jack_detect - Detect headset.
+ * rt5663_v2_jack_detect - Detect headset.
  * @codec: SoC audio codec device.
  * @jack_insert: Jack insert or not.
  *
@@ -1433,16 +1433,16 @@ static void rt5663_enable_push_button_irq(struct snd_soc_codec *codec,
  * Returns detect status.
  */
 
-static int rt5668_jack_detect(struct snd_soc_codec *codec, int jack_insert)
+static int rt5663_v2_jack_detect(struct snd_soc_codec *codec, int jack_insert)
 {
 	struct snd_soc_dapm_context *dapm = snd_soc_codec_get_dapm(codec);
-	struct rt5663_priv *rt5668 = snd_soc_codec_get_drvdata(codec);
+	struct rt5663_priv *rt5663 = snd_soc_codec_get_drvdata(codec);
 	int val, i = 0, sleep_time[5] = {300, 150, 100, 50, 30};
 
 	dev_dbg(codec->dev, "%s jack_insert:%d\n", __func__, jack_insert);
 	if (jack_insert) {
-		snd_soc_write(codec, RT5668_CBJ_TYPE_2, 0x8040);
-		snd_soc_write(codec, RT5668_CBJ_TYPE_3, 0x1484);
+		snd_soc_write(codec, RT5663_CBJ_TYPE_2, 0x8040);
+		snd_soc_write(codec, RT5663_CBJ_TYPE_3, 0x1484);
 
 		snd_soc_dapm_force_enable_pin(dapm, "MICBIAS1");
 		snd_soc_dapm_force_enable_pin(dapm, "MICBIAS2");
@@ -1450,12 +1450,12 @@ static int rt5668_jack_detect(struct snd_soc_codec *codec, int jack_insert)
 		snd_soc_dapm_force_enable_pin(dapm, "CBJ Power");
 		snd_soc_dapm_sync(dapm);
 		snd_soc_update_bits(codec, RT5663_RC_CLK,
-			RT5668_DIG_1M_CLK_MASK, RT5668_DIG_1M_CLK_EN);
+			RT5663_DIG_1M_CLK_MASK, RT5663_DIG_1M_CLK_EN);
 		snd_soc_update_bits(codec, RT5663_RECMIX, 0x8, 0x8);
 
 		while (i < 5) {
 			msleep(sleep_time[i]);
-			val = snd_soc_read(codec, RT5668_CBJ_TYPE_2) & 0x0003;
+			val = snd_soc_read(codec, RT5663_CBJ_TYPE_2) & 0x0003;
 			if (val == 0x1 || val == 0x2 || val == 0x3)
 				break;
 			dev_dbg(codec->dev, "%s: MX-0011 val=%x sleep %d\n",
@@ -1466,7 +1466,7 @@ static int rt5668_jack_detect(struct snd_soc_codec *codec, int jack_insert)
 		switch (val) {
 		case 1:
 		case 2:
-			rt5668->jack_type = SND_JACK_HEADSET;
+			rt5663->jack_type = SND_JACK_HEADSET;
 			rt5663_enable_push_button_irq(codec, true);
 			break;
 		default:
@@ -1475,13 +1475,13 @@ static int rt5668_jack_detect(struct snd_soc_codec *codec, int jack_insert)
 			snd_soc_dapm_disable_pin(dapm, "Mic Det Power");
 			snd_soc_dapm_disable_pin(dapm, "CBJ Power");
 			snd_soc_dapm_sync(dapm);
-			rt5668->jack_type = SND_JACK_HEADPHONE;
+			rt5663->jack_type = SND_JACK_HEADPHONE;
 			break;
 		}
 	} else {
 		snd_soc_update_bits(codec, RT5663_RECMIX, 0x8, 0x0);
 
-		if (rt5668->jack_type == SND_JACK_HEADSET) {
+		if (rt5663->jack_type == SND_JACK_HEADSET) {
 			rt5663_enable_push_button_irq(codec, false);
 			snd_soc_dapm_disable_pin(dapm, "MICBIAS1");
 			snd_soc_dapm_disable_pin(dapm, "MICBIAS2");
@@ -1489,11 +1489,11 @@ static int rt5668_jack_detect(struct snd_soc_codec *codec, int jack_insert)
 			snd_soc_dapm_disable_pin(dapm, "CBJ Power");
 			snd_soc_dapm_sync(dapm);
 		}
-		rt5668->jack_type = 0;
+		rt5663->jack_type = 0;
 	}
 
-	dev_dbg(codec->dev, "jack_type = %d\n", rt5668->jack_type);
-	return rt5668->jack_type;
+	dev_dbg(codec->dev, "jack_type = %d\n", rt5663->jack_type);
+	return rt5663->jack_type;
 }
 
 /**
@@ -1514,11 +1514,11 @@ static int rt5663_jack_detect(struct snd_soc_codec *codec, int jack_insert)
 
 	if (jack_insert) {
 		snd_soc_update_bits(codec, RT5663_DIG_MISC,
-			RT5668_DIG_GATE_CTRL_MASK, RT5668_DIG_GATE_CTRL_EN);
+			RT5663_DIG_GATE_CTRL_MASK, RT5663_DIG_GATE_CTRL_EN);
 		snd_soc_update_bits(codec, RT5663_HP_CHARGE_PUMP_1,
-			RT5663_SI_HP_MASK | RT5668_OSW_HP_L_MASK |
-			RT5668_OSW_HP_R_MASK, RT5663_SI_HP_EN |
-			RT5668_OSW_HP_L_DIS | RT5668_OSW_HP_R_DIS);
+			RT5663_SI_HP_MASK | RT5663_OSW_HP_L_MASK |
+			RT5663_OSW_HP_R_MASK, RT5663_SI_HP_EN |
+			RT5663_OSW_HP_L_DIS | RT5663_OSW_HP_R_DIS);
 		snd_soc_update_bits(codec, RT5663_DUMMY_1,
 			RT5663_EMB_CLK_MASK | RT5663_HPA_CPL_BIAS_MASK |
 			RT5663_HPA_CPR_BIAS_MASK, RT5663_EMB_CLK_EN |
@@ -1530,17 +1530,17 @@ static int rt5663_jack_detect(struct snd_soc_codec *codec, int jack_insert)
 			RT5663_PWR_MIC_DET_MASK, RT5663_PWR_MIC_DET_ON);
 		/* BST1 power on for JD */
 		snd_soc_update_bits(codec, RT5663_PWR_ANLG_2,
-			RT5668_PWR_BST1_MASK, RT5668_PWR_BST1_ON);
+			RT5663_PWR_BST1_MASK, RT5663_PWR_BST1_ON);
 		snd_soc_update_bits(codec, RT5663_EM_JACK_TYPE_1,
 			RT5663_CBJ_DET_MASK | RT5663_EXT_JD_MASK |
 			RT5663_POL_EXT_JD_MASK, RT5663_CBJ_DET_EN |
 			RT5663_EXT_JD_EN | RT5663_POL_EXT_JD_EN);
 		snd_soc_update_bits(codec, RT5663_PWR_ANLG_1,
-			RT5668_PWR_MB_MASK | RT5668_LDO1_DVO_MASK |
-			RT5668_AMP_HP_MASK, RT5668_PWR_MB |
-			RT5668_LDO1_DVO_0_9V | RT5668_AMP_HP_3X);
+			RT5663_PWR_MB_MASK | RT5663_LDO1_DVO_MASK |
+			RT5663_AMP_HP_MASK, RT5663_PWR_MB |
+			RT5663_LDO1_DVO_0_9V | RT5663_AMP_HP_3X);
 		snd_soc_update_bits(codec, RT5663_AUTO_1MRC_CLK,
-			RT5668_IRQ_POW_SAV_MASK, RT5668_IRQ_POW_SAV_EN);
+			RT5663_IRQ_POW_SAV_MASK, RT5663_IRQ_POW_SAV_EN);
 		snd_soc_update_bits(codec, RT5663_IRQ_1,
 			RT5663_EN_IRQ_JD1_MASK, RT5663_EN_IRQ_JD1_EN);
 		while (i < 5) {
@@ -1619,13 +1619,13 @@ static bool rt5663_check_jd_status(struct snd_soc_codec *codec)
 	dev_dbg(codec->dev, "%s val=%x\n", __func__, val);
 
 	/* JD1 */
-	switch (rt5663->codec_type) {
-	case CODEC_TYPE_RT5668:
+	switch (rt5663->codec_ver) {
+	case CODEC_VER_1:
 		return !(val & 0x2000);
-	case CODEC_TYPE_RT5663:
+	case CODEC_VER_0:
 		return !(val & 0x1000);
 	default:
-		dev_err(codec->dev, "Unknown CODEC_TYPE\n");
+		dev_err(codec->dev, "Unknown CODEC Version\n");
 	}
 
 	return false;
@@ -1645,15 +1645,16 @@ static void rt5663_jack_detect_work(struct work_struct *work)
 		/* jack in */
 		if (rt5663->jack_type == 0) {
 			/* jack was out, report jack type */
-			switch (rt5663->codec_type) {
-			case CODEC_TYPE_RT5668:
-				report = rt5668_jack_detect(rt5663->codec, 1);
+			switch (rt5663->codec_ver) {
+			case CODEC_VER_1:
+				report = rt5663_v2_jack_detect(
+						rt5663->codec, 1);
 				break;
-			case CODEC_TYPE_RT5663:
+			case CODEC_VER_0:
 				report = rt5663_jack_detect(rt5663->codec, 1);
 				break;
 			default:
-				dev_err(codec->dev, "Unknown CODEC_TYPE\n");
+				dev_err(codec->dev, "Unknown CODEC Version\n");
 			}
 		} else {
 			/* jack is already in, report button event */
@@ -1702,15 +1703,15 @@ static void rt5663_jack_detect_work(struct work_struct *work)
 		}
 	} else {
 		/* jack out */
-		switch (rt5663->codec_type) {
-		case CODEC_TYPE_RT5668:
-			report = rt5668_jack_detect(rt5663->codec, 0);
+		switch (rt5663->codec_ver) {
+		case CODEC_VER_1:
+			report = rt5663_v2_jack_detect(rt5663->codec, 0);
 			break;
-		case CODEC_TYPE_RT5663:
+		case CODEC_VER_0:
 			report = rt5663_jack_detect(rt5663->codec, 0);
 			break;
 		default:
-			dev_err(codec->dev, "Unknown CODEC_TYPE\n");
+			dev_err(codec->dev, "Unknown CODEC Version\n");
 		}
 	}
 	dev_dbg(codec->dev, "%s jack report: 0x%04x\n", __func__, report);
@@ -1722,24 +1723,24 @@ static void rt5663_jack_detect_work(struct work_struct *work)
 static const struct snd_kcontrol_new rt5663_snd_controls[] = {
 	/* DAC Digital Volume */
 	SOC_DOUBLE_TLV("DAC Playback Volume", RT5663_STO1_DAC_DIG_VOL,
-		RT5668_DAC_L1_VOL_SHIFT + 1, RT5668_DAC_R1_VOL_SHIFT + 1,
+		RT5663_DAC_L1_VOL_SHIFT + 1, RT5663_DAC_R1_VOL_SHIFT + 1,
 		87, 0, dac_vol_tlv),
 	/* ADC Digital Volume Control */
 	SOC_DOUBLE("ADC Capture Switch", RT5663_STO1_ADC_DIG_VOL,
-		RT5668_ADC_L_MUTE_SHIFT, RT5668_ADC_R_MUTE_SHIFT, 1, 1),
+		RT5663_ADC_L_MUTE_SHIFT, RT5663_ADC_R_MUTE_SHIFT, 1, 1),
 	SOC_DOUBLE_TLV("ADC Capture Volume", RT5663_STO1_ADC_DIG_VOL,
-		RT5668_ADC_L_VOL_SHIFT + 1, RT5668_ADC_R_VOL_SHIFT + 1,
+		RT5663_ADC_L_VOL_SHIFT + 1, RT5663_ADC_R_VOL_SHIFT + 1,
 		63, 0, adc_vol_tlv),
 };
 
-static const struct snd_kcontrol_new rt5668_specific_controls[] = {
+static const struct snd_kcontrol_new rt5663_v2_specific_controls[] = {
 	/* Headphone Output Volume */
 	SOC_DOUBLE_R_TLV("Headphone Playback Volume", RT5663_HP_LCH_DRE,
-		RT5663_HP_RCH_DRE, RT5668_GAIN_HP_SHIFT, 15, 1,
-		rt5668_hp_vol_tlv),
+		RT5663_HP_RCH_DRE, RT5663_GAIN_HP_SHIFT, 15, 1,
+		rt5663_v2_hp_vol_tlv),
 	/* Mic Boost Volume */
-	SOC_SINGLE_TLV("IN1 Capture Volume", RT5668_AEC_BST,
-		RT5668_GAIN_CBJ_SHIFT, 8, 0, in_bst_tlv),
+	SOC_SINGLE_TLV("IN1 Capture Volume", RT5663_AEC_BST,
+		RT5663_GAIN_CBJ_SHIFT, 8, 0, in_bst_tlv),
 };
 
 static const struct snd_kcontrol_new rt5663_specific_controls[] = {
@@ -1775,15 +1776,15 @@ static int rt5663_is_using_asrc(struct snd_soc_dapm_widget *w,
 	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
 	struct rt5663_priv *rt5663 = snd_soc_codec_get_drvdata(codec);
 
-	if (rt5663->codec_type == CODEC_TYPE_RT5668) {
+	if (rt5663->codec_ver == CODEC_VER_1) {
 		switch (w->shift) {
-		case RT5668_ADC_STO1_ASRC_SHIFT:
-			reg = RT5668_ASRC_3;
-			shift = RT5668_AD_STO1_TRACK_SHIFT;
+		case RT5663_ADC_STO1_ASRC_SHIFT:
+			reg = RT5663_ASRC_3;
+			shift = RT5663_V2_AD_STO1_TRACK_SHIFT;
 			break;
-		case RT5668_DAC_STO1_ASRC_SHIFT:
+		case RT5663_DAC_STO1_ASRC_SHIFT:
 			reg = RT5663_ASRC_2;
-			shift = RT5668_DA_STO1_TRACK_SHIFT;
+			shift = RT5663_DA_STO1_TRACK_SHIFT;
 			break;
 		default:
 			return 0;
@@ -1820,17 +1821,17 @@ static int rt5663_i2s_use_asrc(struct snd_soc_dapm_widget *source,
 
 	da_asrc_en = (snd_soc_read(codec, RT5663_ASRC_2) &
 		RT5663_DA_STO1_TRACK_MASK) ? 1 : 0;
-	switch (rt5663->codec_type) {
-	case CODEC_TYPE_RT5668:
-		ad_asrc_en = (snd_soc_read(codec, RT5668_ASRC_3) &
-			RT5668_AD_STO1_TRACK_MASK) ? 1 : 0;
+	switch (rt5663->codec_ver) {
+	case CODEC_VER_1:
+		ad_asrc_en = (snd_soc_read(codec, RT5663_ASRC_3) &
+			RT5663_V2_AD_STO1_TRACK_MASK) ? 1 : 0;
 		break;
-	case CODEC_TYPE_RT5663:
+	case CODEC_VER_0:
 		ad_asrc_en = (snd_soc_read(codec, RT5663_ASRC_2) &
 			RT5663_AD_STO1_TRACK_MASK) ? 1 : 0;
 		break;
 	default:
-		dev_err(codec->dev, "Unknown CODEC_TYPE\n");
+		dev_err(codec->dev, "Unknown CODEC Version\n");
 		return 1;
 	}
 
@@ -1849,7 +1850,7 @@ static int rt5663_i2s_use_asrc(struct snd_soc_dapm_widget *source,
  * @filter_mask: mask of filters.
  * @clk_src: clock source
  *
- * The ASRC function is for asynchronous MCLK and LRCK. Also, since RT5668 can
+ * The ASRC function is for asynchronous MCLK and LRCK. Also, since RT5663 can
  * only support standard 32fs or 64fs i2s format, ASRC should be enabled to
  * support special i2s clock format such as Intel's 100fs(100 * sampling rate).
  * ASRC function will track i2s clock and generate a corresponding system clock
@@ -1860,7 +1861,7 @@ static int rt5663_i2s_use_asrc(struct snd_soc_dapm_widget *source,
 int rt5663_sel_asrc_clk_src(struct snd_soc_codec *codec,
 		unsigned int filter_mask, unsigned int clk_src)
 {
-	struct rt5663_priv *rt5668 = snd_soc_codec_get_drvdata(codec);
+	struct rt5663_priv *rt5663 = snd_soc_codec_get_drvdata(codec);
 	unsigned int asrc2_mask = 0;
 	unsigned int asrc2_value = 0;
 	unsigned int asrc3_mask = 0;
@@ -1876,22 +1877,22 @@ int rt5663_sel_asrc_clk_src(struct snd_soc_codec *codec,
 	}
 
 	if (filter_mask & RT5663_DA_STEREO_FILTER) {
-		asrc2_mask |= RT5668_DA_STO1_TRACK_MASK;
-		asrc2_value |= clk_src << RT5668_DA_STO1_TRACK_SHIFT;
+		asrc2_mask |= RT5663_DA_STO1_TRACK_MASK;
+		asrc2_value |= clk_src << RT5663_DA_STO1_TRACK_SHIFT;
 	}
 
 	if (filter_mask & RT5663_AD_STEREO_FILTER) {
-		switch (rt5668->codec_type) {
-		case CODEC_TYPE_RT5668:
-			asrc3_mask |= RT5668_AD_STO1_TRACK_MASK;
-			asrc3_value |= clk_src << RT5668_AD_STO1_TRACK_SHIFT;
+		switch (rt5663->codec_ver) {
+		case CODEC_VER_1:
+			asrc3_mask |= RT5663_V2_AD_STO1_TRACK_MASK;
+			asrc3_value |= clk_src << RT5663_V2_AD_STO1_TRACK_SHIFT;
 			break;
-		case CODEC_TYPE_RT5663:
+		case CODEC_VER_0:
 			asrc2_mask |= RT5663_AD_STO1_TRACK_MASK;
 			asrc2_value |= clk_src << RT5663_AD_STO1_TRACK_SHIFT;
 			break;
 		default:
-			dev_err(codec->dev, "Unknown CODEC_TYPE\n");
+			dev_err(codec->dev, "Unknown CODEC Version\n");
 		}
 	}
 
@@ -1900,7 +1901,7 @@ int rt5663_sel_asrc_clk_src(struct snd_soc_codec *codec,
 			asrc2_value);
 
 	if (asrc3_mask)
-		snd_soc_update_bits(codec, RT5668_ASRC_3, asrc3_mask,
+		snd_soc_update_bits(codec, RT5663_ASRC_3, asrc3_mask,
 			asrc3_value);
 
 	return 0;
@@ -1908,82 +1909,82 @@ int rt5663_sel_asrc_clk_src(struct snd_soc_codec *codec,
 EXPORT_SYMBOL_GPL(rt5663_sel_asrc_clk_src);
 
 /* Analog Mixer */
-static const struct snd_kcontrol_new rt5668_recmix1l[] = {
-	SOC_DAPM_SINGLE("BST2 Switch", RT5668_RECMIX1L,
-		RT5668_RECMIX1L_BST2_SHIFT, 1, 1),
-	SOC_DAPM_SINGLE("BST1 CBJ Switch", RT5668_RECMIX1L,
-		RT5668_RECMIX1L_BST1_CBJ_SHIFT, 1, 1),
+static const struct snd_kcontrol_new rt5663_recmix1l[] = {
+	SOC_DAPM_SINGLE("BST2 Switch", RT5663_RECMIX1L,
+		RT5663_RECMIX1L_BST2_SHIFT, 1, 1),
+	SOC_DAPM_SINGLE("BST1 CBJ Switch", RT5663_RECMIX1L,
+		RT5663_RECMIX1L_BST1_CBJ_SHIFT, 1, 1),
 };
 
-static const struct snd_kcontrol_new rt5668_recmix1r[] = {
-	SOC_DAPM_SINGLE("BST2 Switch", RT5668_RECMIX1R,
-		RT5668_RECMIX1R_BST2_SHIFT, 1, 1),
+static const struct snd_kcontrol_new rt5663_recmix1r[] = {
+	SOC_DAPM_SINGLE("BST2 Switch", RT5663_RECMIX1R,
+		RT5663_RECMIX1R_BST2_SHIFT, 1, 1),
 };
 
 /* Digital Mixer */
 static const struct snd_kcontrol_new rt5663_sto1_adc_l_mix[] = {
 	SOC_DAPM_SINGLE("ADC1 Switch", RT5663_STO1_ADC_MIXER,
-			RT5668_M_STO1_ADC_L1_SHIFT, 1, 1),
+			RT5663_M_STO1_ADC_L1_SHIFT, 1, 1),
 	SOC_DAPM_SINGLE("ADC2 Switch", RT5663_STO1_ADC_MIXER,
-			RT5668_M_STO1_ADC_L2_SHIFT, 1, 1),
+			RT5663_M_STO1_ADC_L2_SHIFT, 1, 1),
 };
 
-static const struct snd_kcontrol_new rt5668_sto1_adc_r_mix[] = {
+static const struct snd_kcontrol_new rt5663_sto1_adc_r_mix[] = {
 	SOC_DAPM_SINGLE("ADC1 Switch", RT5663_STO1_ADC_MIXER,
-			RT5668_M_STO1_ADC_R1_SHIFT, 1, 1),
+			RT5663_M_STO1_ADC_R1_SHIFT, 1, 1),
 	SOC_DAPM_SINGLE("ADC2 Switch", RT5663_STO1_ADC_MIXER,
-			RT5668_M_STO1_ADC_R2_SHIFT, 1, 1),
+			RT5663_M_STO1_ADC_R2_SHIFT, 1, 1),
 };
 
 static const struct snd_kcontrol_new rt5663_adda_l_mix[] = {
 	SOC_DAPM_SINGLE("ADC L Switch", RT5663_AD_DA_MIXER,
-			RT5668_M_ADCMIX_L_SHIFT, 1, 1),
+			RT5663_M_ADCMIX_L_SHIFT, 1, 1),
 	SOC_DAPM_SINGLE("DAC L Switch", RT5663_AD_DA_MIXER,
-			RT5668_M_DAC1_L_SHIFT, 1, 1),
+			RT5663_M_DAC1_L_SHIFT, 1, 1),
 };
 
 static const struct snd_kcontrol_new rt5663_adda_r_mix[] = {
 	SOC_DAPM_SINGLE("ADC R Switch", RT5663_AD_DA_MIXER,
-			RT5668_M_ADCMIX_R_SHIFT, 1, 1),
+			RT5663_M_ADCMIX_R_SHIFT, 1, 1),
 	SOC_DAPM_SINGLE("DAC R Switch", RT5663_AD_DA_MIXER,
-			RT5668_M_DAC1_R_SHIFT, 1, 1),
+			RT5663_M_DAC1_R_SHIFT, 1, 1),
 };
 
 static const struct snd_kcontrol_new rt5663_sto1_dac_l_mix[] = {
 	SOC_DAPM_SINGLE("DAC L Switch", RT5663_STO_DAC_MIXER,
-			RT5668_M_DAC_L1_STO_L_SHIFT, 1, 1),
+			RT5663_M_DAC_L1_STO_L_SHIFT, 1, 1),
 	SOC_DAPM_SINGLE("DAC R Switch", RT5663_STO_DAC_MIXER,
-			RT5668_M_DAC_R1_STO_L_SHIFT, 1, 1),
+			RT5663_M_DAC_R1_STO_L_SHIFT, 1, 1),
 };
 
 static const struct snd_kcontrol_new rt5663_sto1_dac_r_mix[] = {
 	SOC_DAPM_SINGLE("DAC L Switch", RT5663_STO_DAC_MIXER,
-			RT5668_M_DAC_L1_STO_R_SHIFT, 1, 1),
+			RT5663_M_DAC_L1_STO_R_SHIFT, 1, 1),
 	SOC_DAPM_SINGLE("DAC R Switch", RT5663_STO_DAC_MIXER,
-			RT5668_M_DAC_R1_STO_R_SHIFT, 1, 1),
+			RT5663_M_DAC_R1_STO_R_SHIFT, 1, 1),
 };
 
 /* Out Switch */
-static const struct snd_kcontrol_new rt5668_hpo_switch =
-	SOC_DAPM_SINGLE_AUTODISABLE("Switch", RT5668_HP_AMP_2,
-		RT5668_EN_DAC_HPO_SHIFT, 1, 0);
+static const struct snd_kcontrol_new rt5663_hpo_switch =
+	SOC_DAPM_SINGLE_AUTODISABLE("Switch", RT5663_HP_AMP_2,
+		RT5663_EN_DAC_HPO_SHIFT, 1, 0);
 
 /* Stereo ADC source */
-static const char * const rt5668_sto1_adc_src[] = {
+static const char * const rt5663_sto1_adc_src[] = {
 	"ADC L", "ADC R"
 };
 
-static SOC_ENUM_SINGLE_DECL(rt5668_sto1_adcl_enum, RT5663_STO1_ADC_MIXER,
-	RT5668_STO1_ADC_L_SRC_SHIFT, rt5668_sto1_adc_src);
+static SOC_ENUM_SINGLE_DECL(rt5663_sto1_adcl_enum, RT5663_STO1_ADC_MIXER,
+	RT5663_STO1_ADC_L_SRC_SHIFT, rt5663_sto1_adc_src);
 
-static const struct snd_kcontrol_new rt5668_sto1_adcl_mux =
-	SOC_DAPM_ENUM("STO1 ADC L Mux", rt5668_sto1_adcl_enum);
+static const struct snd_kcontrol_new rt5663_sto1_adcl_mux =
+	SOC_DAPM_ENUM("STO1 ADC L Mux", rt5663_sto1_adcl_enum);
 
-static SOC_ENUM_SINGLE_DECL(rt5668_sto1_adcr_enum, RT5663_STO1_ADC_MIXER,
-	RT5668_STO1_ADC_R_SRC_SHIFT, rt5668_sto1_adc_src);
+static SOC_ENUM_SINGLE_DECL(rt5663_sto1_adcr_enum, RT5663_STO1_ADC_MIXER,
+	RT5663_STO1_ADC_R_SRC_SHIFT, rt5663_sto1_adc_src);
 
-static const struct snd_kcontrol_new rt5668_sto1_adcr_mux =
-	SOC_DAPM_ENUM("STO1 ADC R Mux", rt5668_sto1_adcr_enum);
+static const struct snd_kcontrol_new rt5663_sto1_adcr_mux =
+	SOC_DAPM_ENUM("STO1 ADC R Mux", rt5663_sto1_adcr_enum);
 
 /* RT5663: Analog DACL1 input source */
 static const char * const rt5663_alg_dacl_src[] = {
@@ -2015,12 +2016,12 @@ static int rt5663_hp_event(struct snd_soc_dapm_widget *w,
 
 	switch (event) {
 	case SND_SOC_DAPM_POST_PMU:
-		if (rt5663->codec_type == CODEC_TYPE_RT5668) {
+		if (rt5663->codec_ver == CODEC_VER_1) {
 			snd_soc_update_bits(codec, RT5663_HP_CHARGE_PUMP_1,
-				RT5668_SEL_PM_HP_SHIFT, RT5668_SEL_PM_HP_HIGH);
+				RT5663_SEL_PM_HP_SHIFT, RT5663_SEL_PM_HP_HIGH);
 			snd_soc_update_bits(codec, RT5663_HP_LOGIC_2,
-				RT5668_HP_SIG_SRC1_MASK,
-				RT5668_HP_SIG_SRC1_SILENCE);
+				RT5663_HP_SIG_SRC1_MASK,
+				RT5663_HP_SIG_SRC1_SILENCE);
 		} else {
 			snd_soc_write(codec, RT5663_DEPOP_2, 0x3003);
 			snd_soc_update_bits(codec, RT5663_DEPOP_1, 0x000b,
@@ -2028,7 +2029,7 @@ static int rt5663_hp_event(struct snd_soc_dapm_widget *w,
 			snd_soc_update_bits(codec, RT5663_DEPOP_1, 0x0030,
 				0x0030);
 			snd_soc_update_bits(codec, RT5663_HP_CHARGE_PUMP_1,
-				RT5668_OVCD_HP_MASK, RT5668_OVCD_HP_DIS);
+				RT5663_OVCD_HP_MASK, RT5663_OVCD_HP_DIS);
 			snd_soc_write(codec, RT5663_HP_CHARGE_PUMP_2, 0x1371);
 			snd_soc_write(codec, RT5663_HP_BIAS, 0xabba);
 			snd_soc_write(codec, RT5663_CHARGE_PUMP_1, 0x2224);
@@ -2041,14 +2042,14 @@ static int rt5663_hp_event(struct snd_soc_dapm_widget *w,
 		break;
 
 	case SND_SOC_DAPM_PRE_PMD:
-		if (rt5663->codec_type == CODEC_TYPE_RT5668) {
+		if (rt5663->codec_ver == CODEC_VER_1) {
 			snd_soc_update_bits(codec, RT5663_HP_LOGIC_2,
-				RT5668_HP_SIG_SRC1_MASK,
-				RT5668_HP_SIG_SRC1_REG);
+				RT5663_HP_SIG_SRC1_MASK,
+				RT5663_HP_SIG_SRC1_REG);
 		} else {
 			snd_soc_update_bits(codec, RT5663_DEPOP_1, 0x3000, 0x0);
 			snd_soc_update_bits(codec, RT5663_HP_CHARGE_PUMP_1,
-				RT5668_OVCD_HP_MASK, RT5668_OVCD_HP_EN);
+				RT5663_OVCD_HP_MASK, RT5663_OVCD_HP_EN);
 			snd_soc_update_bits(codec, RT5663_DEPOP_1, 0x0030, 0x0);
 			snd_soc_update_bits(codec, RT5663_DEPOP_1, 0x000b,
 				0x000b);
@@ -2062,7 +2063,7 @@ static int rt5663_hp_event(struct snd_soc_dapm_widget *w,
 	return 0;
 }
 
-static int rt5668_bst2_power(struct snd_soc_dapm_widget *w,
+static int rt5663_bst2_power(struct snd_soc_dapm_widget *w,
 	struct snd_kcontrol *kcontrol, int event)
 {
 	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
@@ -2070,13 +2071,13 @@ static int rt5668_bst2_power(struct snd_soc_dapm_widget *w,
 	switch (event) {
 	case SND_SOC_DAPM_POST_PMU:
 		snd_soc_update_bits(codec, RT5663_PWR_ANLG_2,
-			RT5668_PWR_BST2_MASK | RT5668_PWR_BST2_OP_MASK,
-			RT5668_PWR_BST2 | RT5668_PWR_BST2_OP);
+			RT5663_PWR_BST2_MASK | RT5663_PWR_BST2_OP_MASK,
+			RT5663_PWR_BST2 | RT5663_PWR_BST2_OP);
 		break;
 
 	case SND_SOC_DAPM_PRE_PMD:
 		snd_soc_update_bits(codec, RT5663_PWR_ANLG_2,
-			RT5668_PWR_BST2_MASK | RT5668_PWR_BST2_OP_MASK, 0);
+			RT5663_PWR_BST2_MASK | RT5663_PWR_BST2_OP_MASK, 0);
 		break;
 
 	default:
@@ -2110,14 +2111,14 @@ static int rt5663_pre_div_power(struct snd_soc_dapm_widget *w,
 }
 
 static const struct snd_soc_dapm_widget rt5663_dapm_widgets[] = {
-	SND_SOC_DAPM_SUPPLY("PLL", RT5663_PWR_ANLG_3, RT5668_PWR_PLL_SHIFT, 0,
+	SND_SOC_DAPM_SUPPLY("PLL", RT5663_PWR_ANLG_3, RT5663_PWR_PLL_SHIFT, 0,
 		NULL, 0),
 
 	/* micbias */
 	SND_SOC_DAPM_MICBIAS("MICBIAS1", RT5663_PWR_ANLG_2,
-		RT5668_PWR_MB1_SHIFT, 0),
+		RT5663_PWR_MB1_SHIFT, 0),
 	SND_SOC_DAPM_MICBIAS("MICBIAS2", RT5663_PWR_ANLG_2,
-		RT5668_PWR_MB2_SHIFT, 0),
+		RT5663_PWR_MB2_SHIFT, 0),
 
 	/* Input Lines */
 	SND_SOC_DAPM_INPUT("IN1P"),
@@ -2125,14 +2126,14 @@ static const struct snd_soc_dapm_widget rt5663_dapm_widgets[] = {
 
 	/* REC Mixer Power */
 	SND_SOC_DAPM_SUPPLY("RECMIX1L Power", RT5663_PWR_ANLG_2,
-		RT5668_PWR_RECMIX1_SHIFT, 0, NULL, 0),
+		RT5663_PWR_RECMIX1_SHIFT, 0, NULL, 0),
 
 	/* ADCs */
 	SND_SOC_DAPM_ADC("ADC L", NULL, SND_SOC_NOPM, 0, 0),
 	SND_SOC_DAPM_SUPPLY("ADC L Power", RT5663_PWR_DIG_1,
-		RT5668_PWR_ADC_L1_SHIFT, 0, NULL, 0),
+		RT5663_PWR_ADC_L1_SHIFT, 0, NULL, 0),
 	SND_SOC_DAPM_SUPPLY("ADC Clock", RT5663_CHOP_ADC,
-		RT5668_CKGEN_ADCC_SHIFT, 0, NULL, 0),
+		RT5663_CKGEN_ADCC_SHIFT, 0, NULL, 0),
 
 	/* ADC Mixer */
 	SND_SOC_DAPM_MIXER("STO1 ADC MIXL", SND_SOC_NOPM,
@@ -2141,10 +2142,10 @@ static const struct snd_soc_dapm_widget rt5663_dapm_widgets[] = {
 
 	/* ADC Filter Power */
 	SND_SOC_DAPM_SUPPLY("STO1 ADC Filter", RT5663_PWR_DIG_2,
-		RT5668_PWR_ADC_S1F_SHIFT, 0, NULL, 0),
+		RT5663_PWR_ADC_S1F_SHIFT, 0, NULL, 0),
 
 	/* Digital Interface */
-	SND_SOC_DAPM_SUPPLY("I2S", RT5663_PWR_DIG_1, RT5668_PWR_I2S1_SHIFT, 0,
+	SND_SOC_DAPM_SUPPLY("I2S", RT5663_PWR_DIG_1, RT5663_PWR_I2S1_SHIFT, 0,
 		NULL, 0),
 	SND_SOC_DAPM_PGA("IF DAC", SND_SOC_NOPM, 0, 0, NULL, 0),
 	SND_SOC_DAPM_PGA("IF1 DAC1 L", SND_SOC_NOPM, 0, 0, NULL, 0),
@@ -2166,7 +2167,7 @@ static const struct snd_soc_dapm_widget rt5663_dapm_widgets[] = {
 
 	/* DAC Mixer */
 	SND_SOC_DAPM_SUPPLY("STO1 DAC Filter", RT5663_PWR_DIG_2,
-		RT5668_PWR_DAC_S1F_SHIFT, 0, NULL, 0),
+		RT5663_PWR_DAC_S1F_SHIFT, 0, NULL, 0),
 	SND_SOC_DAPM_MIXER("STO1 DAC MIXL", SND_SOC_NOPM, 0, 0,
 		rt5663_sto1_dac_l_mix, ARRAY_SIZE(rt5663_sto1_dac_l_mix)),
 	SND_SOC_DAPM_MIXER("STO1 DAC MIXR", SND_SOC_NOPM, 0, 0,
@@ -2174,9 +2175,9 @@ static const struct snd_soc_dapm_widget rt5663_dapm_widgets[] = {
 
 	/* DACs */
 	SND_SOC_DAPM_SUPPLY("STO1 DAC L Power", RT5663_PWR_DIG_1,
-		RT5668_PWR_DAC_L1_SHIFT, 0, NULL, 0),
+		RT5663_PWR_DAC_L1_SHIFT, 0, NULL, 0),
 	SND_SOC_DAPM_SUPPLY("STO1 DAC R Power", RT5663_PWR_DIG_1,
-		RT5668_PWR_DAC_R1_SHIFT, 0, NULL, 0),
+		RT5663_PWR_DAC_R1_SHIFT, 0, NULL, 0),
 	SND_SOC_DAPM_DAC("DAC L", NULL, SND_SOC_NOPM, 0, 0),
 	SND_SOC_DAPM_DAC("DAC R", NULL, SND_SOC_NOPM, 0, 0),
 
@@ -2189,21 +2190,21 @@ static const struct snd_soc_dapm_widget rt5663_dapm_widgets[] = {
 	SND_SOC_DAPM_OUTPUT("HPOR"),
 };
 
-static const struct snd_soc_dapm_widget rt5668_specific_dapm_widgets[] = {
+static const struct snd_soc_dapm_widget rt5663_v2_specific_dapm_widgets[] = {
 	SND_SOC_DAPM_SUPPLY("LDO2", RT5663_PWR_ANLG_3,
-		RT5668_PWR_LDO2_SHIFT, 0, NULL, 0),
-	SND_SOC_DAPM_SUPPLY("Mic Det Power", RT5668_PWR_VOL,
-		RT5668_PWR_MIC_DET_SHIFT, 0, NULL, 0),
+		RT5663_PWR_LDO2_SHIFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("Mic Det Power", RT5663_PWR_VOL,
+		RT5663_V2_PWR_MIC_DET_SHIFT, 0, NULL, 0),
 	SND_SOC_DAPM_SUPPLY("LDO DAC", RT5663_PWR_DIG_1,
-		RT5668_PWR_LDO_DACREF_SHIFT, 0, NULL, 0),
+		RT5663_PWR_LDO_DACREF_SHIFT, 0, NULL, 0),
 
 	/* ASRC */
 	SND_SOC_DAPM_SUPPLY("I2S ASRC", RT5663_ASRC_1,
-		RT5668_I2S1_ASRC_SHIFT, 0, NULL, 0),
+		RT5663_I2S1_ASRC_SHIFT, 0, NULL, 0),
 	SND_SOC_DAPM_SUPPLY("DAC ASRC", RT5663_ASRC_1,
-		RT5668_DAC_STO1_ASRC_SHIFT, 0, NULL, 0),
+		RT5663_DAC_STO1_ASRC_SHIFT, 0, NULL, 0),
 	SND_SOC_DAPM_SUPPLY("ADC ASRC", RT5663_ASRC_1,
-		RT5668_ADC_STO1_ASRC_SHIFT, 0, NULL, 0),
+		RT5663_ADC_STO1_ASRC_SHIFT, 0, NULL, 0),
 
 	/* Input Lines */
 	SND_SOC_DAPM_INPUT("IN2P"),
@@ -2212,51 +2213,51 @@ static const struct snd_soc_dapm_widget rt5668_specific_dapm_widgets[] = {
 	/* Boost */
 	SND_SOC_DAPM_PGA("BST1 CBJ", SND_SOC_NOPM, 0, 0, NULL, 0),
 	SND_SOC_DAPM_SUPPLY("CBJ Power", RT5663_PWR_ANLG_3,
-		RT5668_PWR_CBJ_SHIFT, 0, NULL, 0),
+		RT5663_PWR_CBJ_SHIFT, 0, NULL, 0),
 	SND_SOC_DAPM_PGA("BST2", SND_SOC_NOPM, 0, 0, NULL, 0),
 	SND_SOC_DAPM_SUPPLY("BST2 Power", SND_SOC_NOPM, 0, 0,
-		rt5668_bst2_power, SND_SOC_DAPM_PRE_PMD |
+		rt5663_bst2_power, SND_SOC_DAPM_PRE_PMD |
 		SND_SOC_DAPM_POST_PMU),
 
 	/* REC Mixer */
-	SND_SOC_DAPM_MIXER("RECMIX1L", SND_SOC_NOPM, 0, 0, rt5668_recmix1l,
-		ARRAY_SIZE(rt5668_recmix1l)),
-	SND_SOC_DAPM_MIXER("RECMIX1R", SND_SOC_NOPM, 0, 0, rt5668_recmix1r,
-		ARRAY_SIZE(rt5668_recmix1r)),
+	SND_SOC_DAPM_MIXER("RECMIX1L", SND_SOC_NOPM, 0, 0, rt5663_recmix1l,
+		ARRAY_SIZE(rt5663_recmix1l)),
+	SND_SOC_DAPM_MIXER("RECMIX1R", SND_SOC_NOPM, 0, 0, rt5663_recmix1r,
+		ARRAY_SIZE(rt5663_recmix1r)),
 	SND_SOC_DAPM_SUPPLY("RECMIX1R Power", RT5663_PWR_ANLG_2,
-		RT5668_PWR_RECMIX2_SHIFT, 0, NULL, 0),
+		RT5663_PWR_RECMIX2_SHIFT, 0, NULL, 0),
 
 	/* ADC */
 	SND_SOC_DAPM_ADC("ADC R", NULL, SND_SOC_NOPM, 0, 0),
 	SND_SOC_DAPM_SUPPLY("ADC R Power", RT5663_PWR_DIG_1,
-		RT5668_PWR_ADC_R1_SHIFT, 0, NULL, 0),
+		RT5663_PWR_ADC_R1_SHIFT, 0, NULL, 0),
 
 	/* ADC Mux */
 	SND_SOC_DAPM_PGA("STO1 ADC L1", RT5663_STO1_ADC_MIXER,
-		RT5668_STO1_ADC_L1_SRC_SHIFT, 0, NULL, 0),
+		RT5663_STO1_ADC_L1_SRC_SHIFT, 0, NULL, 0),
 	SND_SOC_DAPM_PGA("STO1 ADC R1", RT5663_STO1_ADC_MIXER,
-		RT5668_STO1_ADC_R1_SRC_SHIFT, 0, NULL, 0),
+		RT5663_STO1_ADC_R1_SRC_SHIFT, 0, NULL, 0),
 	SND_SOC_DAPM_PGA("STO1 ADC L2", RT5663_STO1_ADC_MIXER,
-		RT5668_STO1_ADC_L2_SRC_SHIFT, 1, NULL, 0),
+		RT5663_STO1_ADC_L2_SRC_SHIFT, 1, NULL, 0),
 	SND_SOC_DAPM_PGA("STO1 ADC R2", RT5663_STO1_ADC_MIXER,
-		RT5668_STO1_ADC_R2_SRC_SHIFT, 1, NULL, 0),
+		RT5663_STO1_ADC_R2_SRC_SHIFT, 1, NULL, 0),
 
 	SND_SOC_DAPM_MUX("STO1 ADC L Mux", SND_SOC_NOPM, 0, 0,
-		&rt5668_sto1_adcl_mux),
+		&rt5663_sto1_adcl_mux),
 	SND_SOC_DAPM_MUX("STO1 ADC R Mux", SND_SOC_NOPM, 0, 0,
-		&rt5668_sto1_adcr_mux),
+		&rt5663_sto1_adcr_mux),
 
 	/* ADC Mix */
 	SND_SOC_DAPM_MIXER("STO1 ADC MIXR", SND_SOC_NOPM, 0, 0,
-		rt5668_sto1_adc_r_mix, ARRAY_SIZE(rt5668_sto1_adc_r_mix)),
+		rt5663_sto1_adc_r_mix, ARRAY_SIZE(rt5663_sto1_adc_r_mix)),
 
 	/* Analog DAC Clock */
 	SND_SOC_DAPM_SUPPLY("DAC Clock", RT5663_CHOP_DAC_L,
-		RT5668_CKGEN_DAC1_SHIFT, 0, NULL, 0),
+		RT5663_CKGEN_DAC1_SHIFT, 0, NULL, 0),
 
 	/* Headphone out */
 	SND_SOC_DAPM_SWITCH("HPO Playback", SND_SOC_NOPM, 0, 0,
-		&rt5668_hpo_switch),
+		&rt5663_hpo_switch),
 };
 
 static const struct snd_soc_dapm_widget rt5663_specific_dapm_widgets[] = {
@@ -2267,7 +2268,7 @@ static const struct snd_soc_dapm_widget rt5663_specific_dapm_widgets[] = {
 
 	/* LDO */
 	SND_SOC_DAPM_SUPPLY("LDO ADC", RT5663_PWR_DIG_1,
-		RT5668_PWR_LDO_DACREF_SHIFT, 0, NULL, 0),
+		RT5663_PWR_LDO_DACREF_SHIFT, 0, NULL, 0),
 
 	/* ASRC */
 	SND_SOC_DAPM_SUPPLY("I2S ASRC", RT5663_ASRC_1,
@@ -2341,7 +2342,7 @@ static const struct snd_soc_dapm_route rt5663_dapm_routes[] = {
 	{ "HP Amp", NULL, "DAC R" },
 };
 
-static const struct snd_soc_dapm_route rt5668_specific_dapm_routes[] = {
+static const struct snd_soc_dapm_route rt5663_v2_specific_dapm_routes[] = {
 	{ "MICBIAS1", NULL, "LDO2" },
 	{ "MICBIAS2", NULL, "LDO2" },
 
@@ -2440,26 +2441,26 @@ static int rt5663_hw_params(struct snd_pcm_substream *substream,
 
 	switch (params_width(params)) {
 	case 8:
-		val_len = RT5668_I2S_DL_8;
+		val_len = RT5663_I2S_DL_8;
 		break;
 	case 16:
-		val_len = RT5668_I2S_DL_16;
+		val_len = RT5663_I2S_DL_16;
 		break;
 	case 20:
-		val_len = RT5668_I2S_DL_20;
+		val_len = RT5663_I2S_DL_20;
 		break;
 	case 24:
-		val_len = RT5668_I2S_DL_24;
+		val_len = RT5663_I2S_DL_24;
 		break;
 	default:
 		return -EINVAL;
 	}
 
 	snd_soc_update_bits(codec, RT5663_I2S1_SDP,
-		RT5668_I2S_DL_MASK, val_len);
+		RT5663_I2S_DL_MASK, val_len);
 
 	snd_soc_update_bits(codec, RT5663_ADDA_CLK_1,
-		RT5668_I2S_PD1_MASK, pre_div << RT5668_I2S_PD1_SHIFT);
+		RT5663_I2S_PD1_MASK, pre_div << RT5663_I2S_PD1_SHIFT);
 
 	return 0;
 }
@@ -2473,7 +2474,7 @@ static int rt5663_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt)
 	case SND_SOC_DAIFMT_CBM_CFM:
 		break;
 	case SND_SOC_DAIFMT_CBS_CFS:
-		reg_val |= RT5668_I2S_MS_S;
+		reg_val |= RT5663_I2S_MS_S;
 		break;
 	default:
 		return -EINVAL;
@@ -2483,7 +2484,7 @@ static int rt5663_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt)
 	case SND_SOC_DAIFMT_NB_NF:
 		break;
 	case SND_SOC_DAIFMT_IB_NF:
-		reg_val |= RT5668_I2S_BP_INV;
+		reg_val |= RT5663_I2S_BP_INV;
 		break;
 	default:
 		return -EINVAL;
@@ -2493,20 +2494,20 @@ static int rt5663_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt)
 	case SND_SOC_DAIFMT_I2S:
 		break;
 	case SND_SOC_DAIFMT_LEFT_J:
-		reg_val |= RT5668_I2S_DF_LEFT;
+		reg_val |= RT5663_I2S_DF_LEFT;
 		break;
 	case SND_SOC_DAIFMT_DSP_A:
-		reg_val |= RT5668_I2S_DF_PCM_A;
+		reg_val |= RT5663_I2S_DF_PCM_A;
 		break;
 	case SND_SOC_DAIFMT_DSP_B:
-		reg_val |= RT5668_I2S_DF_PCM_B;
+		reg_val |= RT5663_I2S_DF_PCM_B;
 		break;
 	default:
 		return -EINVAL;
 	}
 
-	snd_soc_update_bits(codec, RT5663_I2S1_SDP, RT5668_I2S_MS_MASK |
-		RT5668_I2S_BP_MASK | RT5668_I2S_DF_MASK, reg_val);
+	snd_soc_update_bits(codec, RT5663_I2S1_SDP, RT5663_I2S_MS_MASK |
+		RT5663_I2S_BP_MASK | RT5663_I2S_DF_MASK, reg_val);
 
 	return 0;
 }
@@ -2535,7 +2536,7 @@ static int rt5663_set_dai_sysclk(struct snd_soc_dai *dai, int clk_id,
 		dev_err(codec->dev, "Invalid clock id (%d)\n", clk_id);
 		return -EINVAL;
 	}
-	snd_soc_update_bits(codec, RT5663_GLB_CLK, RT5668_SCLK_SRC_MASK,
+	snd_soc_update_bits(codec, RT5663_GLB_CLK, RT5663_SCLK_SRC_MASK,
 		reg_val);
 	rt5663->sysclk = freq;
 	rt5663->sysclk_src = clk_id;
@@ -2569,17 +2570,17 @@ static int rt5663_set_dai_pll(struct snd_soc_dai *dai, int pll_id, int source,
 		return 0;
 	}
 
-	switch (rt5663->codec_type) {
-	case CODEC_TYPE_RT5668:
-		mask = RT5668_PLL1_SRC_MASK;
-		shift = RT5668_PLL1_SRC_SHIFT;
+	switch (rt5663->codec_ver) {
+	case CODEC_VER_1:
+		mask = RT5663_V2_PLL1_SRC_MASK;
+		shift = RT5663_V2_PLL1_SRC_SHIFT;
 		break;
-	case CODEC_TYPE_RT5663:
+	case CODEC_VER_0:
 		mask = RT5663_PLL1_SRC_MASK;
 		shift = RT5663_PLL1_SRC_SHIFT;
 		break;
 	default:
-		dev_err(codec->dev, "Unknown CODEC_TYPE\n");
+		dev_err(codec->dev, "Unknown CODEC Version\n");
 		return -EINVAL;
 	}
 
@@ -2607,10 +2608,10 @@ static int rt5663_set_dai_pll(struct snd_soc_dai *dai, int pll_id, int source,
 		pll_code.k_code);
 
 	snd_soc_write(codec, RT5663_PLL_1,
-		pll_code.n_code << RT5668_PLL_N_SHIFT | pll_code.k_code);
+		pll_code.n_code << RT5663_PLL_N_SHIFT | pll_code.k_code);
 	snd_soc_write(codec, RT5663_PLL_2,
-		(pll_code.m_bp ? 0 : pll_code.m_code) << RT5668_PLL_M_SHIFT |
-		pll_code.m_bp << RT5668_PLL_M_BP_SHIFT);
+		(pll_code.m_bp ? 0 : pll_code.m_code) << RT5663_PLL_M_SHIFT |
+		pll_code.m_bp << RT5663_PLL_M_BP_SHIFT);
 
 	rt5663->pll_in = freq_in;
 	rt5663->pll_out = freq_out;
@@ -2627,20 +2628,20 @@ static int rt5663_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
 	unsigned int val = 0, reg;
 
 	if (rx_mask || tx_mask)
-		val |= RT5668_TDM_MODE_TDM;
+		val |= RT5663_TDM_MODE_TDM;
 
 	switch (slots) {
 	case 4:
-		val |= RT5668_TDM_IN_CH_4;
-		val |= RT5668_TDM_OUT_CH_4;
+		val |= RT5663_TDM_IN_CH_4;
+		val |= RT5663_TDM_OUT_CH_4;
 		break;
 	case 6:
-		val |= RT5668_TDM_IN_CH_6;
-		val |= RT5668_TDM_OUT_CH_6;
+		val |= RT5663_TDM_IN_CH_6;
+		val |= RT5663_TDM_OUT_CH_6;
 		break;
 	case 8:
-		val |= RT5668_TDM_IN_CH_8;
-		val |= RT5668_TDM_OUT_CH_8;
+		val |= RT5663_TDM_IN_CH_8;
+		val |= RT5663_TDM_OUT_CH_8;
 		break;
 	case 2:
 		break;
@@ -2650,16 +2651,16 @@ static int rt5663_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
 
 	switch (slot_width) {
 	case 20:
-		val |= RT5668_TDM_IN_LEN_20;
-		val |= RT5668_TDM_OUT_LEN_20;
+		val |= RT5663_TDM_IN_LEN_20;
+		val |= RT5663_TDM_OUT_LEN_20;
 		break;
 	case 24:
-		val |= RT5668_TDM_IN_LEN_24;
-		val |= RT5668_TDM_OUT_LEN_24;
+		val |= RT5663_TDM_IN_LEN_24;
+		val |= RT5663_TDM_OUT_LEN_24;
 		break;
 	case 32:
-		val |= RT5668_TDM_IN_LEN_32;
-		val |= RT5668_TDM_OUT_LEN_32;
+		val |= RT5663_TDM_IN_LEN_32;
+		val |= RT5663_TDM_OUT_LEN_32;
 		break;
 	case 16:
 		break;
@@ -2667,21 +2668,21 @@ static int rt5663_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
 		return -EINVAL;
 	}
 
-	switch (rt5663->codec_type) {
-	case CODEC_TYPE_RT5668:
+	switch (rt5663->codec_ver) {
+	case CODEC_VER_1:
 		reg = RT5663_TDM_2;
 		break;
-	case CODEC_TYPE_RT5663:
+	case CODEC_VER_0:
 		reg = RT5663_TDM_1;
 		break;
 	default:
-		dev_err(codec->dev, "Unknown CODEC_TYPE\n");
+		dev_err(codec->dev, "Unknown CODEC Version\n");
 		return -EINVAL;
 	}
 
-	snd_soc_update_bits(codec, reg, RT5668_TDM_MODE_MASK |
-		RT5668_TDM_IN_CH_MASK | RT5668_TDM_OUT_CH_MASK |
-		RT5668_TDM_IN_LEN_MASK | RT5668_TDM_OUT_LEN_MASK, val);
+	snd_soc_update_bits(codec, reg, RT5663_TDM_MODE_MASK |
+		RT5663_TDM_IN_CH_MASK | RT5663_TDM_OUT_CH_MASK |
+		RT5663_TDM_IN_LEN_MASK | RT5663_TDM_OUT_LEN_MASK, val);
 
 	return 0;
 }
@@ -2694,8 +2695,8 @@ static int rt5663_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio)
 
 	dev_dbg(codec->dev, "%s ratio = %d\n", __func__, ratio);
 
-	if (rt5663->codec_type == CODEC_TYPE_RT5668)
-		reg = RT5668_TDM_8;
+	if (rt5663->codec_ver == CODEC_VER_1)
+		reg = RT5663_TDM_9;
 	else
 		reg = RT5663_TDM_5;
 
@@ -2736,47 +2737,47 @@ static int rt5663_set_bias_level(struct snd_soc_codec *codec,
 	switch (level) {
 	case SND_SOC_BIAS_ON:
 		snd_soc_update_bits(codec, RT5663_PWR_ANLG_1,
-			RT5668_PWR_FV1_MASK | RT5668_PWR_FV2_MASK,
-			RT5668_PWR_FV1 | RT5668_PWR_FV2);
+			RT5663_PWR_FV1_MASK | RT5663_PWR_FV2_MASK,
+			RT5663_PWR_FV1 | RT5663_PWR_FV2);
 		break;
 
 	case SND_SOC_BIAS_PREPARE:
-		if (rt5663->codec_type == CODEC_TYPE_RT5668) {
+		if (rt5663->codec_ver == CODEC_VER_1) {
 			snd_soc_update_bits(codec, RT5663_DIG_MISC,
-				RT5668_DIG_GATE_CTRL_MASK,
-				RT5668_DIG_GATE_CTRL_EN);
+				RT5663_DIG_GATE_CTRL_MASK,
+				RT5663_DIG_GATE_CTRL_EN);
 			snd_soc_update_bits(codec, RT5663_SIG_CLK_DET,
-				RT5668_EN_ANA_CLK_DET_MASK |
-				RT5668_PWR_CLK_DET_MASK,
-				RT5668_EN_ANA_CLK_DET_AUTO |
-				RT5668_PWR_CLK_DET_EN);
+				RT5663_EN_ANA_CLK_DET_MASK |
+				RT5663_PWR_CLK_DET_MASK,
+				RT5663_EN_ANA_CLK_DET_AUTO |
+				RT5663_PWR_CLK_DET_EN);
 		}
 		break;
 
 	case SND_SOC_BIAS_STANDBY:
-		if (rt5663->codec_type == CODEC_TYPE_RT5668)
+		if (rt5663->codec_ver == CODEC_VER_1)
 			snd_soc_update_bits(codec, RT5663_DIG_MISC,
-				RT5668_DIG_GATE_CTRL_MASK,
-				RT5668_DIG_GATE_CTRL_DIS);
+				RT5663_DIG_GATE_CTRL_MASK,
+				RT5663_DIG_GATE_CTRL_DIS);
 		snd_soc_update_bits(codec, RT5663_PWR_ANLG_1,
-			RT5668_PWR_VREF1_MASK | RT5668_PWR_VREF2_MASK |
-			RT5668_PWR_FV1_MASK | RT5668_PWR_FV2_MASK |
-			RT5668_PWR_MB_MASK, RT5668_PWR_VREF1 |
-			RT5668_PWR_VREF2 | RT5668_PWR_MB);
+			RT5663_PWR_VREF1_MASK | RT5663_PWR_VREF2_MASK |
+			RT5663_PWR_FV1_MASK | RT5663_PWR_FV2_MASK |
+			RT5663_PWR_MB_MASK, RT5663_PWR_VREF1 |
+			RT5663_PWR_VREF2 | RT5663_PWR_MB);
 		usleep_range(10000, 10005);
-		if (rt5663->codec_type == CODEC_TYPE_RT5668) {
+		if (rt5663->codec_ver == CODEC_VER_1) {
 			snd_soc_update_bits(codec, RT5663_SIG_CLK_DET,
-				RT5668_EN_ANA_CLK_DET_MASK |
-				RT5668_PWR_CLK_DET_MASK,
-				RT5668_EN_ANA_CLK_DET_DIS |
-				RT5668_PWR_CLK_DET_DIS);
+				RT5663_EN_ANA_CLK_DET_MASK |
+				RT5663_PWR_CLK_DET_MASK,
+				RT5663_EN_ANA_CLK_DET_DIS |
+				RT5663_PWR_CLK_DET_DIS);
 		}
 		break;
 
 	case SND_SOC_BIAS_OFF:
 		snd_soc_update_bits(codec, RT5663_PWR_ANLG_1,
-			RT5668_PWR_VREF1_MASK | RT5668_PWR_VREF2_MASK |
-			RT5668_PWR_FV1 | RT5668_PWR_FV2, 0x0);
+			RT5663_PWR_VREF1_MASK | RT5663_PWR_VREF2_MASK |
+			RT5663_PWR_FV1 | RT5663_PWR_FV2, 0x0);
 		break;
 
 	default:
@@ -2793,18 +2794,18 @@ static int rt5663_probe(struct snd_soc_codec *codec)
 
 	rt5663->codec = codec;
 
-	switch (rt5663->codec_type) {
-	case CODEC_TYPE_RT5668:
+	switch (rt5663->codec_ver) {
+	case CODEC_VER_1:
 		snd_soc_dapm_new_controls(dapm,
-			rt5668_specific_dapm_widgets,
-			ARRAY_SIZE(rt5668_specific_dapm_widgets));
+			rt5663_v2_specific_dapm_widgets,
+			ARRAY_SIZE(rt5663_v2_specific_dapm_widgets));
 		snd_soc_dapm_add_routes(dapm,
-			rt5668_specific_dapm_routes,
-			ARRAY_SIZE(rt5668_specific_dapm_routes));
-		snd_soc_add_codec_controls(codec, rt5668_specific_controls,
-			ARRAY_SIZE(rt5668_specific_controls));
+			rt5663_v2_specific_dapm_routes,
+			ARRAY_SIZE(rt5663_v2_specific_dapm_routes));
+		snd_soc_add_codec_controls(codec, rt5663_v2_specific_controls,
+			ARRAY_SIZE(rt5663_v2_specific_controls));
 		break;
-	case CODEC_TYPE_RT5663:
+	case CODEC_VER_0:
 		snd_soc_dapm_new_controls(dapm,
 			rt5663_specific_dapm_widgets,
 			ARRAY_SIZE(rt5663_specific_dapm_widgets));
@@ -2905,16 +2906,16 @@ static struct snd_soc_codec_driver soc_codec_dev_rt5663 = {
 	}
 };
 
-static const struct regmap_config rt5668_regmap = {
+static const struct regmap_config rt5663_v2_regmap = {
 	.reg_bits = 16,
 	.val_bits = 16,
 	.use_single_rw = true,
 	.max_register = 0x07fa,
-	.volatile_reg = rt5668_volatile_register,
-	.readable_reg = rt5668_readable_register,
+	.volatile_reg = rt5663_v2_volatile_register,
+	.readable_reg = rt5663_v2_readable_register,
 	.cache_type = REGCACHE_RBTREE,
-	.reg_defaults = rt5668_reg,
-	.num_reg_defaults = ARRAY_SIZE(rt5668_reg),
+	.reg_defaults = rt5663_v2_reg,
+	.num_reg_defaults = ARRAY_SIZE(rt5663_v2_reg),
 };
 
 static const struct regmap_config rt5663_regmap = {
@@ -2939,7 +2940,6 @@ static const struct regmap_config temp_regmap = {
 };
 
 static const struct i2c_device_id rt5663_i2c_id[] = {
-	{ "rt5668", 0 },
 	{ "rt5663", 0 },
 	{}
 };
@@ -2947,7 +2947,6 @@ MODULE_DEVICE_TABLE(i2c, rt5663_i2c_id);
 
 #if defined(CONFIG_OF)
 static const struct of_device_id rt5663_of_match[] = {
-	{ .compatible = "realtek,rt5668", },
 	{ .compatible = "realtek,rt5663", },
 	{},
 };
@@ -2956,80 +2955,79 @@ MODULE_DEVICE_TABLE(of, rt5663_of_match);
 
 #ifdef CONFIG_ACPI
 static struct acpi_device_id rt5663_acpi_match[] = {
-	{ "10EC5668", 0},
 	{ "10EC5663", 0},
 	{},
 };
 MODULE_DEVICE_TABLE(acpi, rt5663_acpi_match);
 #endif
 
-static void rt5668_calibrate(struct rt5663_priv *rt5668)
+static void rt5663_v2_calibrate(struct rt5663_priv *rt5663)
 {
-	regmap_write(rt5668->regmap, RT5663_BIAS_CUR_8, 0xa402);
-	regmap_write(rt5668->regmap, RT5663_PWR_DIG_1, 0x0100);
-	regmap_write(rt5668->regmap, RT5663_RECMIX, 0x4040);
-	regmap_write(rt5668->regmap, RT5663_DIG_MISC, 0x0001);
-	regmap_write(rt5668->regmap, RT5663_RC_CLK, 0x0380);
-	regmap_write(rt5668->regmap, RT5663_GLB_CLK, 0x8000);
-	regmap_write(rt5668->regmap, RT5663_ADDA_CLK_1, 0x1000);
-	regmap_write(rt5668->regmap, RT5663_CHOP_DAC_L, 0x3030);
-	regmap_write(rt5668->regmap, RT5663_CALIB_ADC, 0x3c05);
-	regmap_write(rt5668->regmap, RT5663_PWR_ANLG_1, 0xa23e);
+	regmap_write(rt5663->regmap, RT5663_BIAS_CUR_8, 0xa402);
+	regmap_write(rt5663->regmap, RT5663_PWR_DIG_1, 0x0100);
+	regmap_write(rt5663->regmap, RT5663_RECMIX, 0x4040);
+	regmap_write(rt5663->regmap, RT5663_DIG_MISC, 0x0001);
+	regmap_write(rt5663->regmap, RT5663_RC_CLK, 0x0380);
+	regmap_write(rt5663->regmap, RT5663_GLB_CLK, 0x8000);
+	regmap_write(rt5663->regmap, RT5663_ADDA_CLK_1, 0x1000);
+	regmap_write(rt5663->regmap, RT5663_CHOP_DAC_L, 0x3030);
+	regmap_write(rt5663->regmap, RT5663_CALIB_ADC, 0x3c05);
+	regmap_write(rt5663->regmap, RT5663_PWR_ANLG_1, 0xa23e);
 	msleep(40);
-	regmap_write(rt5668->regmap, RT5663_PWR_ANLG_1, 0xf23e);
-	regmap_write(rt5668->regmap, RT5663_HP_CALIB_2, 0x0321);
-	regmap_write(rt5668->regmap, RT5663_HP_CALIB_1, 0xfc00);
+	regmap_write(rt5663->regmap, RT5663_PWR_ANLG_1, 0xf23e);
+	regmap_write(rt5663->regmap, RT5663_HP_CALIB_2, 0x0321);
+	regmap_write(rt5663->regmap, RT5663_HP_CALIB_1, 0xfc00);
 	msleep(500);
 }
 
-static void rt5663_calibrate(struct rt5663_priv *rt5668)
+static void rt5663_calibrate(struct rt5663_priv *rt5663)
 {
 	int value, count;
 
-	regmap_write(rt5668->regmap, RT5663_RC_CLK, 0x0280);
-	regmap_write(rt5668->regmap, RT5663_GLB_CLK, 0x8000);
-	regmap_write(rt5668->regmap, RT5663_DIG_MISC, 0x8001);
-	regmap_write(rt5668->regmap, RT5663_VREF_RECMIX, 0x0032);
-	regmap_write(rt5668->regmap, RT5663_PWR_ANLG_1, 0xa2be);
+	regmap_write(rt5663->regmap, RT5663_RC_CLK, 0x0280);
+	regmap_write(rt5663->regmap, RT5663_GLB_CLK, 0x8000);
+	regmap_write(rt5663->regmap, RT5663_DIG_MISC, 0x8001);
+	regmap_write(rt5663->regmap, RT5663_VREF_RECMIX, 0x0032);
+	regmap_write(rt5663->regmap, RT5663_PWR_ANLG_1, 0xa2be);
 	msleep(20);
-	regmap_write(rt5668->regmap, RT5663_PWR_ANLG_1, 0xf2be);
-	regmap_write(rt5668->regmap, RT5663_PWR_DIG_2, 0x8400);
-	regmap_write(rt5668->regmap, RT5663_CHOP_ADC, 0x3000);
-	regmap_write(rt5668->regmap, RT5663_DEPOP_1, 0x003b);
-	regmap_write(rt5668->regmap, RT5663_PWR_DIG_1, 0x8df8);
-	regmap_write(rt5668->regmap, RT5663_PWR_ANLG_2, 0x0003);
-	regmap_write(rt5668->regmap, RT5663_PWR_ANLG_3, 0x018c);
-	regmap_write(rt5668->regmap, RT5663_ADDA_CLK_1, 0x1111);
-	regmap_write(rt5668->regmap, RT5663_PRE_DIV_GATING_1, 0xffff);
-	regmap_write(rt5668->regmap, RT5663_PRE_DIV_GATING_2, 0xffff);
-	regmap_write(rt5668->regmap, RT5663_DEPOP_2, 0x3003);
-	regmap_write(rt5668->regmap, RT5663_DEPOP_1, 0x003b);
-	regmap_write(rt5668->regmap, RT5663_HP_CHARGE_PUMP_1, 0x1e32);
-	regmap_write(rt5668->regmap, RT5663_HP_CHARGE_PUMP_2, 0x1371);
-	regmap_write(rt5668->regmap, RT5663_DACREF_LDO, 0x3b0b);
-	regmap_write(rt5668->regmap, RT5663_STO_DAC_MIXER, 0x2080);
-	regmap_write(rt5668->regmap, RT5663_BYPASS_STO_DAC, 0x000c);
-	regmap_write(rt5668->regmap, RT5663_HP_BIAS, 0xabba);
-	regmap_write(rt5668->regmap, RT5663_CHARGE_PUMP_1, 0x2224);
-	regmap_write(rt5668->regmap, RT5663_HP_OUT_EN, 0x8088);
-	regmap_write(rt5668->regmap, RT5663_STO_DRE_9, 0x0017);
-	regmap_write(rt5668->regmap, RT5663_STO_DRE_10, 0x0017);
-	regmap_write(rt5668->regmap, RT5663_STO1_ADC_MIXER, 0x4040);
-	regmap_write(rt5668->regmap, RT5663_RECMIX, 0x0005);
-	regmap_write(rt5668->regmap, RT5663_ADDA_RST, 0xc000);
-	regmap_write(rt5668->regmap, RT5663_STO1_HPF_ADJ1, 0x3320);
-	regmap_write(rt5668->regmap, RT5663_HP_CALIB_2, 0x00c9);
-	regmap_write(rt5668->regmap, RT5663_DUMMY_1, 0x004c);
-	regmap_write(rt5668->regmap, RT5663_ANA_BIAS_CUR_1, 0x7766);
-	regmap_write(rt5668->regmap, RT5663_BIAS_CUR_8, 0x4702);
+	regmap_write(rt5663->regmap, RT5663_PWR_ANLG_1, 0xf2be);
+	regmap_write(rt5663->regmap, RT5663_PWR_DIG_2, 0x8400);
+	regmap_write(rt5663->regmap, RT5663_CHOP_ADC, 0x3000);
+	regmap_write(rt5663->regmap, RT5663_DEPOP_1, 0x003b);
+	regmap_write(rt5663->regmap, RT5663_PWR_DIG_1, 0x8df8);
+	regmap_write(rt5663->regmap, RT5663_PWR_ANLG_2, 0x0003);
+	regmap_write(rt5663->regmap, RT5663_PWR_ANLG_3, 0x018c);
+	regmap_write(rt5663->regmap, RT5663_ADDA_CLK_1, 0x1111);
+	regmap_write(rt5663->regmap, RT5663_PRE_DIV_GATING_1, 0xffff);
+	regmap_write(rt5663->regmap, RT5663_PRE_DIV_GATING_2, 0xffff);
+	regmap_write(rt5663->regmap, RT5663_DEPOP_2, 0x3003);
+	regmap_write(rt5663->regmap, RT5663_DEPOP_1, 0x003b);
+	regmap_write(rt5663->regmap, RT5663_HP_CHARGE_PUMP_1, 0x1e32);
+	regmap_write(rt5663->regmap, RT5663_HP_CHARGE_PUMP_2, 0x1371);
+	regmap_write(rt5663->regmap, RT5663_DACREF_LDO, 0x3b0b);
+	regmap_write(rt5663->regmap, RT5663_STO_DAC_MIXER, 0x2080);
+	regmap_write(rt5663->regmap, RT5663_BYPASS_STO_DAC, 0x000c);
+	regmap_write(rt5663->regmap, RT5663_HP_BIAS, 0xabba);
+	regmap_write(rt5663->regmap, RT5663_CHARGE_PUMP_1, 0x2224);
+	regmap_write(rt5663->regmap, RT5663_HP_OUT_EN, 0x8088);
+	regmap_write(rt5663->regmap, RT5663_STO_DRE_9, 0x0017);
+	regmap_write(rt5663->regmap, RT5663_STO_DRE_10, 0x0017);
+	regmap_write(rt5663->regmap, RT5663_STO1_ADC_MIXER, 0x4040);
+	regmap_write(rt5663->regmap, RT5663_RECMIX, 0x0005);
+	regmap_write(rt5663->regmap, RT5663_ADDA_RST, 0xc000);
+	regmap_write(rt5663->regmap, RT5663_STO1_HPF_ADJ1, 0x3320);
+	regmap_write(rt5663->regmap, RT5663_HP_CALIB_2, 0x00c9);
+	regmap_write(rt5663->regmap, RT5663_DUMMY_1, 0x004c);
+	regmap_write(rt5663->regmap, RT5663_ANA_BIAS_CUR_1, 0x7766);
+	regmap_write(rt5663->regmap, RT5663_BIAS_CUR_8, 0x4702);
 	msleep(200);
-	regmap_write(rt5668->regmap, RT5663_HP_CALIB_1, 0x0069);
-	regmap_write(rt5668->regmap, RT5663_HP_CALIB_3, 0x06c2);
-	regmap_write(rt5668->regmap, RT5663_HP_CALIB_1_1, 0x7b00);
-	regmap_write(rt5668->regmap, RT5663_HP_CALIB_1_1, 0xfb00);
+	regmap_write(rt5663->regmap, RT5663_HP_CALIB_1, 0x0069);
+	regmap_write(rt5663->regmap, RT5663_HP_CALIB_3, 0x06c2);
+	regmap_write(rt5663->regmap, RT5663_HP_CALIB_1_1, 0x7b00);
+	regmap_write(rt5663->regmap, RT5663_HP_CALIB_1_1, 0xfb00);
 	count = 0;
 	while (true) {
-		regmap_read(rt5668->regmap, RT5663_HP_CALIB_1_1, &value);
+		regmap_read(rt5663->regmap, RT5663_HP_CALIB_1_1, &value);
 		if (value & 0x8000)
 			usleep_range(10000, 10005);
 		else
@@ -3066,17 +3064,17 @@ static int rt5663_i2c_probe(struct i2c_client *i2c,
 	}
 	regmap_read(regmap, RT5663_VENDOR_ID_2, &val);
 	switch (val) {
-	case RT5668_DEVICE_ID:
-		rt5663->regmap = devm_regmap_init_i2c(i2c, &rt5668_regmap);
-		rt5663->codec_type = CODEC_TYPE_RT5668;
+	case RT5663_DEVICE_ID_2:
+		rt5663->regmap = devm_regmap_init_i2c(i2c, &rt5663_v2_regmap);
+		rt5663->codec_ver = CODEC_VER_1;
 		break;
-	case RT5663_DEVICE_ID:
+	case RT5663_DEVICE_ID_1:
 		rt5663->regmap = devm_regmap_init_i2c(i2c, &rt5663_regmap);
-		rt5663->codec_type = CODEC_TYPE_RT5663;
+		rt5663->codec_ver = CODEC_VER_0;
 		break;
 	default:
 		dev_err(&i2c->dev,
-			"Device with ID register %#x is not rt5663 or rt5668\n",
+			"Device with ID register %#x is not rt5663\n",
 			val);
 		return -ENODEV;
 	}
@@ -3091,11 +3089,11 @@ static int rt5663_i2c_probe(struct i2c_client *i2c,
 	/* reset and calibrate */
 	regmap_write(rt5663->regmap, RT5663_RESET, 0);
 	regcache_cache_bypass(rt5663->regmap, true);
-	switch (rt5663->codec_type) {
-	case CODEC_TYPE_RT5668:
-		rt5668_calibrate(rt5663);
+	switch (rt5663->codec_ver) {
+	case CODEC_VER_1:
+		rt5663_v2_calibrate(rt5663);
 		break;
-	case CODEC_TYPE_RT5663:
+	case CODEC_VER_0:
 		rt5663_calibrate(rt5663);
 		break;
 	default:
@@ -3106,46 +3104,55 @@ static int rt5663_i2c_probe(struct i2c_client *i2c,
 	dev_dbg(&i2c->dev, "calibrate done\n");
 
 	/* GPIO1 as IRQ */
-	regmap_update_bits(rt5663->regmap, RT5663_GPIO_1, RT5668_GP1_PIN_MASK,
-		RT5668_GP1_PIN_IRQ);
+	regmap_update_bits(rt5663->regmap, RT5663_GPIO_1, RT5663_GP1_PIN_MASK,
+		RT5663_GP1_PIN_IRQ);
 	/* 4btn inline command debounce */
 	regmap_update_bits(rt5663->regmap, RT5663_IL_CMD_5,
-		RT5668_4BTN_CLK_DEB_MASK, RT5668_4BTN_CLK_DEB_65MS);
+		RT5663_4BTN_CLK_DEB_MASK, RT5663_4BTN_CLK_DEB_65MS);
 
-	switch (rt5663->codec_type) {
-	case CODEC_TYPE_RT5668:
+	switch (rt5663->codec_ver) {
+	case CODEC_VER_1:
 		regmap_write(rt5663->regmap, RT5663_BIAS_CUR_8, 0xa402);
 		/* JD1 */
 		regmap_update_bits(rt5663->regmap, RT5663_AUTO_1MRC_CLK,
-			RT5668_IRQ_POW_SAV_MASK | RT5668_IRQ_POW_SAV_JD1_MASK,
-			RT5668_IRQ_POW_SAV_EN | RT5668_IRQ_POW_SAV_JD1_EN);
+			RT5663_IRQ_POW_SAV_MASK | RT5663_IRQ_POW_SAV_JD1_MASK,
+			RT5663_IRQ_POW_SAV_EN | RT5663_IRQ_POW_SAV_JD1_EN);
 		regmap_update_bits(rt5663->regmap, RT5663_PWR_ANLG_2,
-			RT5668_PWR_JD1_MASK, RT5668_PWR_JD1);
+			RT5663_PWR_JD1_MASK, RT5663_PWR_JD1);
 		regmap_update_bits(rt5663->regmap, RT5663_IRQ_1,
-			RT5668_EN_CB_JD_MASK, RT5668_EN_CB_JD_EN);
+			RT5663_EN_CB_JD_MASK, RT5663_EN_CB_JD_EN);
 
 		regmap_update_bits(rt5663->regmap, RT5663_HP_LOGIC_2,
-			RT5668_HP_SIG_SRC1_MASK, RT5668_HP_SIG_SRC1_REG);
+			RT5663_HP_SIG_SRC1_MASK, RT5663_HP_SIG_SRC1_REG);
 		regmap_update_bits(rt5663->regmap, RT5663_RECMIX,
-			RT5668_VREF_BIAS_MASK | RT5668_CBJ_DET_MASK |
-			RT5668_DET_TYPE_MASK, RT5668_VREF_BIAS_REG |
-			RT5668_CBJ_DET_EN | RT5668_DET_TYPE_QFN);
+			RT5663_VREF_BIAS_MASK | RT5663_CBJ_DET_MASK |
+			RT5663_DET_TYPE_MASK, RT5663_VREF_BIAS_REG |
+			RT5663_CBJ_DET_EN | RT5663_DET_TYPE_QFN);
 		/* Set GPIO4 and GPIO8 as input for combo jack */
 		regmap_update_bits(rt5663->regmap, RT5663_GPIO_2,
-			RT5668_GP4_PIN_CONF_MASK, RT5668_GP4_PIN_CONF_INPUT);
-		regmap_update_bits(rt5663->regmap, RT5668_GPIO_3,
-			RT5668_GP8_PIN_CONF_MASK, RT5668_GP8_PIN_CONF_INPUT);
+			RT5663_GP4_PIN_CONF_MASK, RT5663_GP4_PIN_CONF_INPUT);
+		regmap_update_bits(rt5663->regmap, RT5663_GPIO_3,
+			RT5663_GP8_PIN_CONF_MASK, RT5663_GP8_PIN_CONF_INPUT);
 		regmap_update_bits(rt5663->regmap, RT5663_PWR_ANLG_1,
-			RT5668_LDO1_DVO_MASK | RT5668_AMP_HP_MASK,
-			RT5668_LDO1_DVO_0_9V | RT5668_AMP_HP_3X);
+			RT5663_LDO1_DVO_MASK | RT5663_AMP_HP_MASK,
+			RT5663_LDO1_DVO_0_9V | RT5663_AMP_HP_3X);
 			break;
-	case CODEC_TYPE_RT5663:
+	case CODEC_VER_0:
+		regmap_update_bits(rt5663->regmap, RT5663_DIG_MISC,
+			RT5663_DIG_GATE_CTRL_MASK, RT5663_DIG_GATE_CTRL_EN);
+		regmap_update_bits(rt5663->regmap, RT5663_AUTO_1MRC_CLK,
+			RT5663_IRQ_POW_SAV_MASK, RT5663_IRQ_POW_SAV_EN);
+		regmap_update_bits(rt5663->regmap, RT5663_IRQ_1,
+			RT5663_EN_IRQ_JD1_MASK, RT5663_EN_IRQ_JD1_EN);
+		regmap_update_bits(rt5663->regmap, RT5663_GPIO_1,
+			RT5663_GPIO1_TYPE_MASK, RT5663_GPIO1_TYPE_EN);
 		regmap_write(rt5663->regmap, RT5663_VREF_RECMIX, 0x0032);
 		regmap_write(rt5663->regmap, RT5663_PWR_ANLG_1, 0xa2be);
 		msleep(20);
 		regmap_write(rt5663->regmap, RT5663_PWR_ANLG_1, 0xf2be);
 		regmap_update_bits(rt5663->regmap, RT5663_GPIO_2,
-			RT5663_GP1_PIN_CONF_MASK, RT5663_GP1_PIN_CONF_OUTPUT);
+			RT5663_GP1_PIN_CONF_MASK | RT5663_SEL_GPIO1_MASK,
+			RT5663_GP1_PIN_CONF_OUTPUT | RT5663_SEL_GPIO1_EN);
 		/* DACREF LDO control */
 		regmap_update_bits(rt5663->regmap, RT5663_DACREF_LDO, 0x3e0e,
 			0x3a0a);
diff --git a/sound/soc/codecs/rt5663.h b/sound/soc/codecs/rt5663.h
index 2cc8f28080f6..d77fae619f2f 100644
--- a/sound/soc/codecs/rt5663.h
+++ b/sound/soc/codecs/rt5663.h
@@ -18,655 +18,652 @@
 #define RT5663_VENDOR_ID_1			0x00fe
 #define RT5663_VENDOR_ID_2			0x00ff
 
-#define RT5668_LOUT_CTRL			0x0001
-#define RT5668_HP_AMP_2				0x0003
-#define RT5668_MONO_OUT				0x0004
-#define RT5668_MONO_GAIN			0x0007
-
-#define RT5668_AEC_BST				0x000b
-#define RT5668_IN1_IN2				0x000c
-#define RT5668_IN3_IN4				0x000d
-#define RT5668_INL1_INR1			0x000f
-#define RT5668_CBJ_TYPE_2			0x0011
-#define RT5668_CBJ_TYPE_3			0x0012
-#define RT5668_CBJ_TYPE_4			0x0013
-#define RT5668_CBJ_TYPE_5			0x0014
-#define RT5668_CBJ_TYPE_8			0x0017
+#define RT5663_LOUT_CTRL			0x0001
+#define RT5663_HP_AMP_2				0x0003
+#define RT5663_MONO_OUT				0x0004
+#define RT5663_MONO_GAIN			0x0007
+
+#define RT5663_AEC_BST				0x000b
+#define RT5663_IN1_IN2				0x000c
+#define RT5663_IN3_IN4				0x000d
+#define RT5663_INL1_INR1			0x000f
+#define RT5663_CBJ_TYPE_2			0x0011
+#define RT5663_CBJ_TYPE_3			0x0012
+#define RT5663_CBJ_TYPE_4			0x0013
+#define RT5663_CBJ_TYPE_5			0x0014
+#define RT5663_CBJ_TYPE_8			0x0017
 
 /* I/O - ADC/DAC/DMIC */
-#define RT5668_DAC3_DIG_VOL			0x001a
-#define RT5668_DAC3_CTRL			0x001b
-#define RT5668_MONO_ADC_DIG_VOL			0x001d
-#define RT5668_STO2_ADC_DIG_VOL			0x001e
-#define RT5668_MONO_ADC_BST_GAIN		0x0020
-#define RT5668_STO2_ADC_BST_GAIN		0x0021
-#define RT5668_SIDETONE_CTRL			0x0024
+#define RT5663_DAC3_DIG_VOL			0x001a
+#define RT5663_DAC3_CTRL			0x001b
+#define RT5663_MONO_ADC_DIG_VOL			0x001d
+#define RT5663_STO2_ADC_DIG_VOL			0x001e
+#define RT5663_MONO_ADC_BST_GAIN		0x0020
+#define RT5663_STO2_ADC_BST_GAIN		0x0021
+#define RT5663_SIDETONE_CTRL			0x0024
 /* Mixer - D-D */
-#define RT5668_MONO1_ADC_MIXER			0x0027
-#define RT5668_STO2_ADC_MIXER			0x0028
-#define RT5668_MONO_DAC_MIXER			0x002b
-#define RT5668_DAC2_SRC_CTRL			0x002e
-#define RT5668_IF_3_4_DATA_CTL			0x002f
-#define RT5668_IF_5_DATA_CTL			0x0030
-#define RT5668_PDM_OUT_CTL			0x0031
-#define RT5668_PDM_I2C_DATA_CTL1		0x0032
-#define RT5668_PDM_I2C_DATA_CTL2		0x0033
-#define RT5668_PDM_I2C_DATA_CTL3		0x0034
-#define RT5668_PDM_I2C_DATA_CTL4		0x0035
+#define RT5663_MONO1_ADC_MIXER			0x0027
+#define RT5663_STO2_ADC_MIXER			0x0028
+#define RT5663_MONO_DAC_MIXER			0x002b
+#define RT5663_DAC2_SRC_CTRL			0x002e
+#define RT5663_IF_3_4_DATA_CTL			0x002f
+#define RT5663_IF_5_DATA_CTL			0x0030
+#define RT5663_PDM_OUT_CTL			0x0031
+#define RT5663_PDM_I2C_DATA_CTL1		0x0032
+#define RT5663_PDM_I2C_DATA_CTL2		0x0033
+#define RT5663_PDM_I2C_DATA_CTL3		0x0034
+#define RT5663_PDM_I2C_DATA_CTL4		0x0035
 
 /*Mixer - Analog*/
-#define RT5668_RECMIX1_NEW			0x003a
-#define RT5668_RECMIX1L_0			0x003b
-#define RT5668_RECMIX1L				0x003c
-#define RT5668_RECMIX1R_0			0x003d
-#define RT5668_RECMIX1R				0x003e
-#define RT5668_RECMIX2_NEW			0x003f
-#define RT5668_RECMIX2_L_2			0x0041
-#define RT5668_RECMIX2_R			0x0042
-#define RT5668_RECMIX2_R_2			0x0043
-#define RT5668_CALIB_REC_LR			0x0044
-#define RT5668_ALC_BK_GAIN			0x0049
-#define RT5668_MONOMIX_GAIN			0x004a
-#define RT5668_MONOMIX_IN_GAIN			0x004b
-#define RT5668_OUT_MIXL_GAIN			0x004d
-#define RT5668_OUT_LMIX_IN_GAIN			0x004e
-#define RT5668_OUT_RMIX_IN_GAIN			0x004f
-#define RT5668_OUT_RMIX_IN_GAIN1		0x0050
-#define RT5668_LOUT_MIXER_CTRL			0x0052
+#define RT5663_RECMIX1_NEW			0x003a
+#define RT5663_RECMIX1L_0			0x003b
+#define RT5663_RECMIX1L				0x003c
+#define RT5663_RECMIX1R_0			0x003d
+#define RT5663_RECMIX1R				0x003e
+#define RT5663_RECMIX2_NEW			0x003f
+#define RT5663_RECMIX2_L_2			0x0041
+#define RT5663_RECMIX2_R			0x0042
+#define RT5663_RECMIX2_R_2			0x0043
+#define RT5663_CALIB_REC_LR			0x0044
+#define RT5663_ALC_BK_GAIN			0x0049
+#define RT5663_MONOMIX_GAIN			0x004a
+#define RT5663_MONOMIX_IN_GAIN			0x004b
+#define RT5663_OUT_MIXL_GAIN			0x004d
+#define RT5663_OUT_LMIX_IN_GAIN			0x004e
+#define RT5663_OUT_RMIX_IN_GAIN			0x004f
+#define RT5663_OUT_RMIX_IN_GAIN1		0x0050
+#define RT5663_LOUT_MIXER_CTRL			0x0052
 /* Power */
-#define RT5668_PWR_VOL				0x0067
+#define RT5663_PWR_VOL				0x0067
 
-#define RT5668_ADCDAC_RST			0x006d
+#define RT5663_ADCDAC_RST			0x006d
 /* Format - ADC/DAC */
-#define RT5668_I2S34_SDP			0x0071
-#define RT5668_I2S5_SDP				0x0072
-/* Format - TDM Control */
-#define RT5668_TDM_5				0x007c
-#define RT5668_TDM_6				0x007d
-#define RT5668_TDM_7				0x007e
-#define RT5668_TDM_8				0x007f
+#define RT5663_I2S34_SDP			0x0071
+#define RT5663_I2S5_SDP				0x0072
 
 /* Function - Analog */
-#define RT5668_ASRC_3				0x0085
-#define RT5668_ASRC_6				0x0088
-#define RT5668_ASRC_7				0x0089
-#define RT5668_PLL_TRK_13			0x0099
-#define RT5668_I2S_M_CLK_CTL			0x00a0
-#define RT5668_FDIV_I2S34_M_CLK			0x00a1
-#define RT5668_FDIV_I2S34_M_CLK2		0x00a2
-#define RT5668_FDIV_I2S5_M_CLK			0x00a3
-#define RT5668_FDIV_I2S5_M_CLK2			0x00a4
+#define RT5663_ASRC_3				0x0085
+#define RT5663_ASRC_6				0x0088
+#define RT5663_ASRC_7				0x0089
+#define RT5663_PLL_TRK_13			0x0099
+#define RT5663_I2S_M_CLK_CTL			0x00a0
+#define RT5663_FDIV_I2S34_M_CLK			0x00a1
+#define RT5663_FDIV_I2S34_M_CLK2		0x00a2
+#define RT5663_FDIV_I2S5_M_CLK			0x00a3
+#define RT5663_FDIV_I2S5_M_CLK2			0x00a4
 
 /* Function - Digital */
-#define RT5668_IRQ_4				0x00b9
-#define RT5668_GPIO_3				0x00c2
-#define RT5668_GPIO_4				0x00c3
-#define RT5668_GPIO_STA				0x00c4
-#define RT5668_HP_AMP_DET1			0x00d0
-#define RT5668_HP_AMP_DET2			0x00d1
-#define RT5668_HP_AMP_DET3			0x00d2
-#define RT5668_MID_BD_HP_AMP			0x00d3
-#define RT5668_LOW_BD_HP_AMP			0x00d4
-#define RT5668_SOF_VOL_ZC2			0x00da
-#define RT5668_ADC_STO2_ADJ1			0x00ee
-#define RT5668_ADC_STO2_ADJ2			0x00ef
+#define RT5663_V2_IRQ_4				0x00b9
+#define RT5663_GPIO_3				0x00c2
+#define RT5663_GPIO_4				0x00c3
+#define RT5663_GPIO_STA2			0x00c4
+#define RT5663_HP_AMP_DET1			0x00d0
+#define RT5663_HP_AMP_DET2			0x00d1
+#define RT5663_HP_AMP_DET3			0x00d2
+#define RT5663_MID_BD_HP_AMP			0x00d3
+#define RT5663_LOW_BD_HP_AMP			0x00d4
+#define RT5663_SOF_VOL_ZC2			0x00da
+#define RT5663_ADC_STO2_ADJ1			0x00ee
+#define RT5663_ADC_STO2_ADJ2			0x00ef
 /* General Control */
-#define RT5668_A_JD_CTRL			0x00f0
-#define RT5668_JD1_TRES_CTRL			0x00f1
-#define RT5668_JD2_TRES_CTRL			0x00f2
-#define RT5668_JD_CTRL2				0x00f7
-#define RT5668_DUM_REG_2			0x00fb
-#define RT5668_DUM_REG_3			0x00fc
-
-
-#define RT5668_DACADC_DIG_VOL2			0x0101
-#define RT5668_DIG_IN_PIN2			0x0133
-#define RT5668_PAD_DRV_CTL1			0x0136
-#define RT5668_SOF_RAM_DEPOP			0x0138
-#define RT5668_VOL_TEST				0x013f
-#define RT5668_TEST_MODE_3			0x0147
-#define RT5668_TEST_MODE_4			0x0148
-#define RT5668_MONO_DYNA_1			0x0170
-#define RT5668_MONO_DYNA_2			0x0171
-#define RT5668_MONO_DYNA_3			0x0172
-#define RT5668_MONO_DYNA_4			0x0173
-#define RT5668_MONO_DYNA_5			0x0174
-#define RT5668_MONO_DYNA_6			0x0175
-#define RT5668_STO1_SIL_DET			0x0190
-#define RT5668_MONOL_SIL_DET			0x0191
-#define RT5668_MONOR_SIL_DET			0x0192
-#define RT5668_STO2_DAC_SIL			0x0193
-#define RT5668_PWR_SAV_CTL1			0x0194
-#define RT5668_PWR_SAV_CTL2			0x0195
-#define RT5668_PWR_SAV_CTL3			0x0196
-#define RT5668_PWR_SAV_CTL4			0x0197
-#define RT5668_PWR_SAV_CTL5			0x0198
-#define RT5668_PWR_SAV_CTL6			0x0199
-#define RT5668_MONO_AMP_CAL1			0x01a0
-#define RT5668_MONO_AMP_CAL2			0x01a1
-#define RT5668_MONO_AMP_CAL3			0x01a2
-#define RT5668_MONO_AMP_CAL4			0x01a3
-#define RT5668_MONO_AMP_CAL5			0x01a4
-#define RT5668_MONO_AMP_CAL6			0x01a5
-#define RT5668_MONO_AMP_CAL7			0x01a6
-#define RT5668_MONO_AMP_CAL_ST1			0x01a7
-#define RT5668_MONO_AMP_CAL_ST2			0x01a8
-#define RT5668_MONO_AMP_CAL_ST3			0x01a9
-#define RT5668_MONO_AMP_CAL_ST4			0x01aa
-#define RT5668_MONO_AMP_CAL_ST5			0x01ab
-#define RT5668_HP_IMP_SEN_13			0x01b9
-#define RT5668_HP_IMP_SEN_14			0x01ba
-#define RT5668_HP_IMP_SEN_6			0x01bb
-#define RT5668_HP_IMP_SEN_7			0x01bc
-#define RT5668_HP_IMP_SEN_8			0x01bd
-#define RT5668_HP_IMP_SEN_9			0x01be
-#define RT5668_HP_IMP_SEN_10			0x01bf
-#define RT5668_HP_LOGIC_3			0x01dc
-#define RT5668_HP_CALIB_ST10			0x01f3
-#define RT5668_HP_CALIB_ST11			0x01f4
-#define RT5668_PRO_REG_TBL_4			0x0203
-#define RT5668_PRO_REG_TBL_5			0x0204
-#define RT5668_PRO_REG_TBL_6			0x0205
-#define RT5668_PRO_REG_TBL_7			0x0206
-#define RT5668_PRO_REG_TBL_8			0x0207
-#define RT5668_PRO_REG_TBL_9			0x0208
-#define RT5668_SAR_ADC_INL_1			0x0210
-#define RT5668_SAR_ADC_INL_2			0x0211
-#define RT5668_SAR_ADC_INL_3			0x0212
-#define RT5668_SAR_ADC_INL_4			0x0213
-#define RT5668_SAR_ADC_INL_5			0x0214
-#define RT5668_SAR_ADC_INL_6			0x0215
-#define RT5668_SAR_ADC_INL_7			0x0216
-#define RT5668_SAR_ADC_INL_8			0x0217
-#define RT5668_SAR_ADC_INL_9			0x0218
-#define RT5668_SAR_ADC_INL_10			0x0219
-#define RT5668_SAR_ADC_INL_11			0x021a
-#define RT5668_SAR_ADC_INL_12			0x021b
-#define RT5668_DRC_CTRL_1			0x02ff
-#define RT5668_DRC1_CTRL_2			0x0301
-#define RT5668_DRC1_CTRL_3			0x0302
-#define RT5668_DRC1_CTRL_4			0x0303
-#define RT5668_DRC1_CTRL_5			0x0304
-#define RT5668_DRC1_CTRL_6			0x0305
-#define RT5668_DRC1_HD_CTRL_1			0x0306
-#define RT5668_DRC1_HD_CTRL_2			0x0307
-#define RT5668_DRC1_PRI_REG_1			0x0310
-#define RT5668_DRC1_PRI_REG_2			0x0311
-#define RT5668_DRC1_PRI_REG_3			0x0312
-#define RT5668_DRC1_PRI_REG_4			0x0313
-#define RT5668_DRC1_PRI_REG_5			0x0314
-#define RT5668_DRC1_PRI_REG_6			0x0315
-#define RT5668_DRC1_PRI_REG_7			0x0316
-#define RT5668_DRC1_PRI_REG_8			0x0317
-#define RT5668_ALC_PGA_CTL_1			0x0330
-#define RT5668_ALC_PGA_CTL_2			0x0331
-#define RT5668_ALC_PGA_CTL_3			0x0332
-#define RT5668_ALC_PGA_CTL_4			0x0333
-#define RT5668_ALC_PGA_CTL_5			0x0334
-#define RT5668_ALC_PGA_CTL_6			0x0335
-#define RT5668_ALC_PGA_CTL_7			0x0336
-#define RT5668_ALC_PGA_CTL_8			0x0337
-#define RT5668_ALC_PGA_REG_1			0x0338
-#define RT5668_ALC_PGA_REG_2			0x0339
-#define RT5668_ALC_PGA_REG_3			0x033a
-#define RT5668_ADC_EQ_RECOV_1			0x03c0
-#define RT5668_ADC_EQ_RECOV_2			0x03c1
-#define RT5668_ADC_EQ_RECOV_3			0x03c2
-#define RT5668_ADC_EQ_RECOV_4			0x03c3
-#define RT5668_ADC_EQ_RECOV_5			0x03c4
-#define RT5668_ADC_EQ_RECOV_6			0x03c5
-#define RT5668_ADC_EQ_RECOV_7			0x03c6
-#define RT5668_ADC_EQ_RECOV_8			0x03c7
-#define RT5668_ADC_EQ_RECOV_9			0x03c8
-#define RT5668_ADC_EQ_RECOV_10			0x03c9
-#define RT5668_ADC_EQ_RECOV_11			0x03ca
-#define RT5668_ADC_EQ_RECOV_12			0x03cb
-#define RT5668_ADC_EQ_RECOV_13			0x03cc
-#define RT5668_VID_HIDDEN			0x03fe
-#define RT5668_VID_CUSTOMER			0x03ff
-#define RT5668_SCAN_MODE			0x07f0
-#define RT5668_I2C_BYPA				0x07fa
+#define RT5663_A_JD_CTRL			0x00f0
+#define RT5663_JD1_TRES_CTRL			0x00f1
+#define RT5663_JD2_TRES_CTRL			0x00f2
+#define RT5663_V2_JD_CTRL2			0x00f7
+#define RT5663_DUM_REG_2			0x00fb
+#define RT5663_DUM_REG_3			0x00fc
+
+
+#define RT5663_DACADC_DIG_VOL2			0x0101
+#define RT5663_DIG_IN_PIN2			0x0133
+#define RT5663_PAD_DRV_CTL1			0x0136
+#define RT5663_SOF_RAM_DEPOP			0x0138
+#define RT5663_VOL_TEST				0x013f
+#define RT5663_MONO_DYNA_1			0x0170
+#define RT5663_MONO_DYNA_2			0x0171
+#define RT5663_MONO_DYNA_3			0x0172
+#define RT5663_MONO_DYNA_4			0x0173
+#define RT5663_MONO_DYNA_5			0x0174
+#define RT5663_MONO_DYNA_6			0x0175
+#define RT5663_STO1_SIL_DET			0x0190
+#define RT5663_MONOL_SIL_DET			0x0191
+#define RT5663_MONOR_SIL_DET			0x0192
+#define RT5663_STO2_DAC_SIL			0x0193
+#define RT5663_PWR_SAV_CTL1			0x0194
+#define RT5663_PWR_SAV_CTL2			0x0195
+#define RT5663_PWR_SAV_CTL3			0x0196
+#define RT5663_PWR_SAV_CTL4			0x0197
+#define RT5663_PWR_SAV_CTL5			0x0198
+#define RT5663_PWR_SAV_CTL6			0x0199
+#define RT5663_MONO_AMP_CAL1			0x01a0
+#define RT5663_MONO_AMP_CAL2			0x01a1
+#define RT5663_MONO_AMP_CAL3			0x01a2
+#define RT5663_MONO_AMP_CAL4			0x01a3
+#define RT5663_MONO_AMP_CAL5			0x01a4
+#define RT5663_MONO_AMP_CAL6			0x01a5
+#define RT5663_MONO_AMP_CAL7			0x01a6
+#define RT5663_MONO_AMP_CAL_ST1			0x01a7
+#define RT5663_MONO_AMP_CAL_ST2			0x01a8
+#define RT5663_MONO_AMP_CAL_ST3			0x01a9
+#define RT5663_MONO_AMP_CAL_ST4			0x01aa
+#define RT5663_MONO_AMP_CAL_ST5			0x01ab
+#define RT5663_V2_HP_IMP_SEN_13			0x01b9
+#define RT5663_V2_HP_IMP_SEN_14			0x01ba
+#define RT5663_V2_HP_IMP_SEN_6			0x01bb
+#define RT5663_V2_HP_IMP_SEN_7			0x01bc
+#define RT5663_V2_HP_IMP_SEN_8			0x01bd
+#define RT5663_V2_HP_IMP_SEN_9			0x01be
+#define RT5663_V2_HP_IMP_SEN_10			0x01bf
+#define RT5663_HP_LOGIC_3			0x01dc
+#define RT5663_HP_CALIB_ST10			0x01f3
+#define RT5663_HP_CALIB_ST11			0x01f4
+#define RT5663_PRO_REG_TBL_4			0x0203
+#define RT5663_PRO_REG_TBL_5			0x0204
+#define RT5663_PRO_REG_TBL_6			0x0205
+#define RT5663_PRO_REG_TBL_7			0x0206
+#define RT5663_PRO_REG_TBL_8			0x0207
+#define RT5663_PRO_REG_TBL_9			0x0208
+#define RT5663_SAR_ADC_INL_1			0x0210
+#define RT5663_SAR_ADC_INL_2			0x0211
+#define RT5663_SAR_ADC_INL_3			0x0212
+#define RT5663_SAR_ADC_INL_4			0x0213
+#define RT5663_SAR_ADC_INL_5			0x0214
+#define RT5663_SAR_ADC_INL_6			0x0215
+#define RT5663_SAR_ADC_INL_7			0x0216
+#define RT5663_SAR_ADC_INL_8			0x0217
+#define RT5663_SAR_ADC_INL_9			0x0218
+#define RT5663_SAR_ADC_INL_10			0x0219
+#define RT5663_SAR_ADC_INL_11			0x021a
+#define RT5663_SAR_ADC_INL_12			0x021b
+#define RT5663_DRC_CTRL_1			0x02ff
+#define RT5663_DRC1_CTRL_2			0x0301
+#define RT5663_DRC1_CTRL_3			0x0302
+#define RT5663_DRC1_CTRL_4			0x0303
+#define RT5663_DRC1_CTRL_5			0x0304
+#define RT5663_DRC1_CTRL_6			0x0305
+#define RT5663_DRC1_HD_CTRL_1			0x0306
+#define RT5663_DRC1_HD_CTRL_2			0x0307
+#define RT5663_DRC1_PRI_REG_1			0x0310
+#define RT5663_DRC1_PRI_REG_2			0x0311
+#define RT5663_DRC1_PRI_REG_3			0x0312
+#define RT5663_DRC1_PRI_REG_4			0x0313
+#define RT5663_DRC1_PRI_REG_5			0x0314
+#define RT5663_DRC1_PRI_REG_6			0x0315
+#define RT5663_DRC1_PRI_REG_7			0x0316
+#define RT5663_DRC1_PRI_REG_8			0x0317
+#define RT5663_ALC_PGA_CTL_1			0x0330
+#define RT5663_ALC_PGA_CTL_2			0x0331
+#define RT5663_ALC_PGA_CTL_3			0x0332
+#define RT5663_ALC_PGA_CTL_4			0x0333
+#define RT5663_ALC_PGA_CTL_5			0x0334
+#define RT5663_ALC_PGA_CTL_6			0x0335
+#define RT5663_ALC_PGA_CTL_7			0x0336
+#define RT5663_ALC_PGA_CTL_8			0x0337
+#define RT5663_ALC_PGA_REG_1			0x0338
+#define RT5663_ALC_PGA_REG_2			0x0339
+#define RT5663_ALC_PGA_REG_3			0x033a
+#define RT5663_ADC_EQ_RECOV_1			0x03c0
+#define RT5663_ADC_EQ_RECOV_2			0x03c1
+#define RT5663_ADC_EQ_RECOV_3			0x03c2
+#define RT5663_ADC_EQ_RECOV_4			0x03c3
+#define RT5663_ADC_EQ_RECOV_5			0x03c4
+#define RT5663_ADC_EQ_RECOV_6			0x03c5
+#define RT5663_ADC_EQ_RECOV_7			0x03c6
+#define RT5663_ADC_EQ_RECOV_8			0x03c7
+#define RT5663_ADC_EQ_RECOV_9			0x03c8
+#define RT5663_ADC_EQ_RECOV_10			0x03c9
+#define RT5663_ADC_EQ_RECOV_11			0x03ca
+#define RT5663_ADC_EQ_RECOV_12			0x03cb
+#define RT5663_ADC_EQ_RECOV_13			0x03cc
+#define RT5663_VID_HIDDEN			0x03fe
+#define RT5663_VID_CUSTOMER			0x03ff
+#define RT5663_SCAN_MODE			0x07f0
+#define RT5663_I2C_BYPA				0x07fa
 
 /* Headphone Amp Control 2 (0x0003) */
-#define RT5668_EN_DAC_HPO_MASK			(0x1 << 14)
-#define RT5668_EN_DAC_HPO_SHIFT			14
-#define RT5668_EN_DAC_HPO_DIS			(0x0 << 14)
-#define RT5668_EN_DAC_HPO_EN			(0x1 << 14)
+#define RT5663_EN_DAC_HPO_MASK			(0x1 << 14)
+#define RT5663_EN_DAC_HPO_SHIFT			14
+#define RT5663_EN_DAC_HPO_DIS			(0x0 << 14)
+#define RT5663_EN_DAC_HPO_EN			(0x1 << 14)
 
 /*Headphone Amp L/R Analog Gain and Digital NG2 Gain Control (0x0005 0x0006)*/
-#define RT5668_GAIN_HP				(0x1f << 8)
-#define RT5668_GAIN_HP_SHIFT			8
+#define RT5663_GAIN_HP				(0x1f << 8)
+#define RT5663_GAIN_HP_SHIFT			8
 
 /* AEC BST Control (0x000b) */
-#define RT5668_GAIN_CBJ_MASK			(0xf << 8)
-#define RT5668_GAIN_CBJ_SHIFT			8
+#define RT5663_GAIN_CBJ_MASK			(0xf << 8)
+#define RT5663_GAIN_CBJ_SHIFT			8
 
 /* IN1 Control / MIC GND REF (0x000c) */
-#define RT5668_IN1_DF_MASK			(0x1 << 15)
-#define RT5668_IN1_DF_SHIFT			15
+#define RT5663_IN1_DF_MASK			(0x1 << 15)
+#define RT5663_IN1_DF_SHIFT			15
 
 /* Combo Jack and Type Detection Control 1 (0x0010) */
-#define RT5668_CBJ_DET_MASK			(0x1 << 15)
-#define RT5668_CBJ_DET_SHIFT			15
-#define RT5668_CBJ_DET_DIS			(0x0 << 15)
-#define RT5668_CBJ_DET_EN			(0x1 << 15)
-#define RT5668_DET_TYPE_MASK			(0x1 << 12)
-#define RT5668_DET_TYPE_SHIFT			12
-#define RT5668_DET_TYPE_WLCSP			(0x0 << 12)
-#define RT5668_DET_TYPE_QFN			(0x1 << 12)
-#define RT5668_VREF_BIAS_MASK			(0x1 << 6)
-#define RT5668_VREF_BIAS_SHIFT			6
-#define RT5668_VREF_BIAS_FSM			(0x0 << 6)
-#define RT5668_VREF_BIAS_REG			(0x1 << 6)
+#define RT5663_CBJ_DET_MASK			(0x1 << 15)
+#define RT5663_CBJ_DET_SHIFT			15
+#define RT5663_CBJ_DET_DIS			(0x0 << 15)
+#define RT5663_CBJ_DET_EN			(0x1 << 15)
+#define RT5663_DET_TYPE_MASK			(0x1 << 12)
+#define RT5663_DET_TYPE_SHIFT			12
+#define RT5663_DET_TYPE_WLCSP			(0x0 << 12)
+#define RT5663_DET_TYPE_QFN			(0x1 << 12)
+#define RT5663_VREF_BIAS_MASK			(0x1 << 6)
+#define RT5663_VREF_BIAS_SHIFT			6
+#define RT5663_VREF_BIAS_FSM			(0x0 << 6)
+#define RT5663_VREF_BIAS_REG			(0x1 << 6)
 
 /* REC Left Mixer Control 2 (0x003c) */
-#define RT5668_RECMIX1L_BST1_CBJ		(0x1 << 7)
-#define RT5668_RECMIX1L_BST1_CBJ_SHIFT		7
-#define RT5668_RECMIX1L_BST2			(0x1 << 4)
-#define RT5668_RECMIX1L_BST2_SHIFT		4
+#define RT5663_RECMIX1L_BST1_CBJ		(0x1 << 7)
+#define RT5663_RECMIX1L_BST1_CBJ_SHIFT		7
+#define RT5663_RECMIX1L_BST2			(0x1 << 4)
+#define RT5663_RECMIX1L_BST2_SHIFT		4
 
 /* REC Right Mixer Control 2 (0x003e) */
-#define RT5668_RECMIX1R_BST2			(0x1 << 4)
-#define RT5668_RECMIX1R_BST2_SHIFT		4
+#define RT5663_RECMIX1R_BST2			(0x1 << 4)
+#define RT5663_RECMIX1R_BST2_SHIFT		4
 
 /* DAC1 Digital Volume (0x0019) */
-#define RT5668_DAC_L1_VOL_MASK			(0xff << 8)
-#define RT5668_DAC_L1_VOL_SHIFT			8
-#define RT5668_DAC_R1_VOL_MASK			(0xff)
-#define RT5668_DAC_R1_VOL_SHIFT			0
+#define RT5663_DAC_L1_VOL_MASK			(0xff << 8)
+#define RT5663_DAC_L1_VOL_SHIFT			8
+#define RT5663_DAC_R1_VOL_MASK			(0xff)
+#define RT5663_DAC_R1_VOL_SHIFT			0
 
 /* ADC Digital Volume Control (0x001c) */
-#define RT5668_ADC_L_MUTE_MASK			(0x1 << 15)
-#define RT5668_ADC_L_MUTE_SHIFT			15
-#define RT5668_ADC_L_VOL_MASK			(0x7f << 8)
-#define RT5668_ADC_L_VOL_SHIFT			8
-#define RT5668_ADC_R_MUTE_MASK			(0x1 << 7)
-#define RT5668_ADC_R_MUTE_SHIFT			7
-#define RT5668_ADC_R_VOL_MASK			(0x7f)
-#define RT5668_ADC_R_VOL_SHIFT			0
+#define RT5663_ADC_L_MUTE_MASK			(0x1 << 15)
+#define RT5663_ADC_L_MUTE_SHIFT			15
+#define RT5663_ADC_L_VOL_MASK			(0x7f << 8)
+#define RT5663_ADC_L_VOL_SHIFT			8
+#define RT5663_ADC_R_MUTE_MASK			(0x1 << 7)
+#define RT5663_ADC_R_MUTE_SHIFT			7
+#define RT5663_ADC_R_VOL_MASK			(0x7f)
+#define RT5663_ADC_R_VOL_SHIFT			0
 
 /* Stereo ADC Mixer Control (0x0026) */
-#define RT5668_M_STO1_ADC_L1			(0x1 << 15)
-#define RT5668_M_STO1_ADC_L1_SHIFT		15
-#define RT5668_M_STO1_ADC_L2			(0x1 << 14)
-#define RT5668_M_STO1_ADC_L2_SHIFT		14
-#define RT5668_STO1_ADC_L1_SRC			(0x1 << 13)
-#define RT5668_STO1_ADC_L1_SRC_SHIFT		13
-#define RT5668_STO1_ADC_L2_SRC			(0x1 << 12)
-#define RT5668_STO1_ADC_L2_SRC_SHIFT		12
-#define RT5668_STO1_ADC_L_SRC			(0x3 << 10)
-#define RT5668_STO1_ADC_L_SRC_SHIFT		10
-#define RT5668_M_STO1_ADC_R1			(0x1 << 7)
-#define RT5668_M_STO1_ADC_R1_SHIFT		7
-#define RT5668_M_STO1_ADC_R2			(0x1 << 6)
-#define RT5668_M_STO1_ADC_R2_SHIFT		6
-#define RT5668_STO1_ADC_R1_SRC			(0x1 << 5)
-#define RT5668_STO1_ADC_R1_SRC_SHIFT		5
-#define RT5668_STO1_ADC_R2_SRC			(0x1 << 4)
-#define RT5668_STO1_ADC_R2_SRC_SHIFT		4
-#define RT5668_STO1_ADC_R_SRC			(0x3 << 2)
-#define RT5668_STO1_ADC_R_SRC_SHIFT		2
+#define RT5663_M_STO1_ADC_L1			(0x1 << 15)
+#define RT5663_M_STO1_ADC_L1_SHIFT		15
+#define RT5663_M_STO1_ADC_L2			(0x1 << 14)
+#define RT5663_M_STO1_ADC_L2_SHIFT		14
+#define RT5663_STO1_ADC_L1_SRC			(0x1 << 13)
+#define RT5663_STO1_ADC_L1_SRC_SHIFT		13
+#define RT5663_STO1_ADC_L2_SRC			(0x1 << 12)
+#define RT5663_STO1_ADC_L2_SRC_SHIFT		12
+#define RT5663_STO1_ADC_L_SRC			(0x3 << 10)
+#define RT5663_STO1_ADC_L_SRC_SHIFT		10
+#define RT5663_M_STO1_ADC_R1			(0x1 << 7)
+#define RT5663_M_STO1_ADC_R1_SHIFT		7
+#define RT5663_M_STO1_ADC_R2			(0x1 << 6)
+#define RT5663_M_STO1_ADC_R2_SHIFT		6
+#define RT5663_STO1_ADC_R1_SRC			(0x1 << 5)
+#define RT5663_STO1_ADC_R1_SRC_SHIFT		5
+#define RT5663_STO1_ADC_R2_SRC			(0x1 << 4)
+#define RT5663_STO1_ADC_R2_SRC_SHIFT		4
+#define RT5663_STO1_ADC_R_SRC			(0x3 << 2)
+#define RT5663_STO1_ADC_R_SRC_SHIFT		2
 
 /* ADC Mixer to DAC Mixer Control (0x0029) */
-#define RT5668_M_ADCMIX_L			(0x1 << 15)
-#define RT5668_M_ADCMIX_L_SHIFT			15
-#define RT5668_M_DAC1_L				(0x1 << 14)
-#define RT5668_M_DAC1_L_SHIFT			14
-#define RT5668_M_ADCMIX_R			(0x1 << 7)
-#define RT5668_M_ADCMIX_R_SHIFT			7
-#define RT5668_M_DAC1_R				(0x1 << 6)
-#define RT5668_M_DAC1_R_SHIFT			6
+#define RT5663_M_ADCMIX_L			(0x1 << 15)
+#define RT5663_M_ADCMIX_L_SHIFT			15
+#define RT5663_M_DAC1_L				(0x1 << 14)
+#define RT5663_M_DAC1_L_SHIFT			14
+#define RT5663_M_ADCMIX_R			(0x1 << 7)
+#define RT5663_M_ADCMIX_R_SHIFT			7
+#define RT5663_M_DAC1_R				(0x1 << 6)
+#define RT5663_M_DAC1_R_SHIFT			6
 
 /* Stereo DAC Mixer Control (0x002a) */
-#define RT5668_M_DAC_L1_STO_L			(0x1 << 15)
-#define RT5668_M_DAC_L1_STO_L_SHIFT		15
-#define RT5668_M_DAC_R1_STO_L			(0x1 << 13)
-#define RT5668_M_DAC_R1_STO_L_SHIFT		13
-#define RT5668_M_DAC_L1_STO_R			(0x1 << 7)
-#define RT5668_M_DAC_L1_STO_R_SHIFT		7
-#define RT5668_M_DAC_R1_STO_R			(0x1 << 5)
-#define RT5668_M_DAC_R1_STO_R_SHIFT		5
+#define RT5663_M_DAC_L1_STO_L			(0x1 << 15)
+#define RT5663_M_DAC_L1_STO_L_SHIFT		15
+#define RT5663_M_DAC_R1_STO_L			(0x1 << 13)
+#define RT5663_M_DAC_R1_STO_L_SHIFT		13
+#define RT5663_M_DAC_L1_STO_R			(0x1 << 7)
+#define RT5663_M_DAC_L1_STO_R_SHIFT		7
+#define RT5663_M_DAC_R1_STO_R			(0x1 << 5)
+#define RT5663_M_DAC_R1_STO_R_SHIFT		5
 
 /* Power Management for Digital 1 (0x0061) */
-#define RT5668_PWR_I2S1				(0x1 << 15)
-#define RT5668_PWR_I2S1_SHIFT			15
-#define RT5668_PWR_DAC_L1			(0x1 << 11)
-#define RT5668_PWR_DAC_L1_SHIFT			11
-#define RT5668_PWR_DAC_R1			(0x1 << 10)
-#define RT5668_PWR_DAC_R1_SHIFT			10
-#define RT5668_PWR_LDO_DACREF_MASK		(0x1 << 8)
-#define RT5668_PWR_LDO_DACREF_SHIFT		8
-#define RT5668_PWR_LDO_DACREF_ON		(0x1 << 8)
-#define RT5668_PWR_LDO_DACREF_DOWN		(0x0 << 8)
-#define RT5668_PWR_LDO_SHIFT			8
-#define RT5668_PWR_ADC_L1			(0x1 << 4)
-#define RT5668_PWR_ADC_L1_SHIFT			4
-#define RT5668_PWR_ADC_R1			(0x1 << 3)
-#define RT5668_PWR_ADC_R1_SHIFT			3
+#define RT5663_PWR_I2S1				(0x1 << 15)
+#define RT5663_PWR_I2S1_SHIFT			15
+#define RT5663_PWR_DAC_L1			(0x1 << 11)
+#define RT5663_PWR_DAC_L1_SHIFT			11
+#define RT5663_PWR_DAC_R1			(0x1 << 10)
+#define RT5663_PWR_DAC_R1_SHIFT			10
+#define RT5663_PWR_LDO_DACREF_MASK		(0x1 << 8)
+#define RT5663_PWR_LDO_DACREF_SHIFT		8
+#define RT5663_PWR_LDO_DACREF_ON		(0x1 << 8)
+#define RT5663_PWR_LDO_DACREF_DOWN		(0x0 << 8)
+#define RT5663_PWR_LDO_SHIFT			8
+#define RT5663_PWR_ADC_L1			(0x1 << 4)
+#define RT5663_PWR_ADC_L1_SHIFT			4
+#define RT5663_PWR_ADC_R1			(0x1 << 3)
+#define RT5663_PWR_ADC_R1_SHIFT			3
 
 /* Power Management for Digital 2 (0x0062) */
-#define RT5668_PWR_ADC_S1F			(0x1 << 15)
-#define RT5668_PWR_ADC_S1F_SHIFT		15
-#define RT5668_PWR_DAC_S1F			(0x1 << 10)
-#define RT5668_PWR_DAC_S1F_SHIFT		10
+#define RT5663_PWR_ADC_S1F			(0x1 << 15)
+#define RT5663_PWR_ADC_S1F_SHIFT		15
+#define RT5663_PWR_DAC_S1F			(0x1 << 10)
+#define RT5663_PWR_DAC_S1F_SHIFT		10
 
 /* Power Management for Analog 1 (0x0063) */
-#define RT5668_PWR_VREF1			(0x1 << 15)
-#define RT5668_PWR_VREF1_MASK			(0x1 << 15)
-#define RT5668_PWR_VREF1_SHIFT			15
-#define RT5668_PWR_FV1				(0x1 << 14)
-#define RT5668_PWR_FV1_MASK			(0x1 << 14)
-#define RT5668_PWR_FV1_SHIFT			14
-#define RT5668_PWR_VREF2			(0x1 << 13)
-#define RT5668_PWR_VREF2_MASK			(0x1 << 13)
-#define RT5668_PWR_VREF2_SHIFT			13
-#define RT5668_PWR_FV2				(0x1 << 12)
-#define RT5668_PWR_FV2_MASK			(0x1 << 12)
-#define RT5668_PWR_FV2_SHIFT			12
-#define RT5668_PWR_MB				(0x1 << 9)
-#define RT5668_PWR_MB_MASK			(0x1 << 9)
-#define RT5668_PWR_MB_SHIFT			9
-#define RT5668_AMP_HP_MASK			(0x3 << 2)
-#define RT5668_AMP_HP_SHIFT			2
-#define RT5668_AMP_HP_1X			(0x0 << 2)
-#define RT5668_AMP_HP_3X			(0x1 << 2)
-#define RT5668_AMP_HP_5X			(0x3 << 2)
-#define RT5668_LDO1_DVO_MASK			(0x3)
-#define RT5668_LDO1_DVO_SHIFT			0
-#define RT5668_LDO1_DVO_0_9V			(0x0)
-#define RT5668_LDO1_DVO_1_0V			(0x1)
-#define RT5668_LDO1_DVO_1_2V			(0x2)
-#define RT5668_LDO1_DVO_1_4V			(0x3)
+#define RT5663_PWR_VREF1			(0x1 << 15)
+#define RT5663_PWR_VREF1_MASK			(0x1 << 15)
+#define RT5663_PWR_VREF1_SHIFT			15
+#define RT5663_PWR_FV1				(0x1 << 14)
+#define RT5663_PWR_FV1_MASK			(0x1 << 14)
+#define RT5663_PWR_FV1_SHIFT			14
+#define RT5663_PWR_VREF2			(0x1 << 13)
+#define RT5663_PWR_VREF2_MASK			(0x1 << 13)
+#define RT5663_PWR_VREF2_SHIFT			13
+#define RT5663_PWR_FV2				(0x1 << 12)
+#define RT5663_PWR_FV2_MASK			(0x1 << 12)
+#define RT5663_PWR_FV2_SHIFT			12
+#define RT5663_PWR_MB				(0x1 << 9)
+#define RT5663_PWR_MB_MASK			(0x1 << 9)
+#define RT5663_PWR_MB_SHIFT			9
+#define RT5663_AMP_HP_MASK			(0x3 << 2)
+#define RT5663_AMP_HP_SHIFT			2
+#define RT5663_AMP_HP_1X			(0x0 << 2)
+#define RT5663_AMP_HP_3X			(0x1 << 2)
+#define RT5663_AMP_HP_5X			(0x3 << 2)
+#define RT5663_LDO1_DVO_MASK			(0x3)
+#define RT5663_LDO1_DVO_SHIFT			0
+#define RT5663_LDO1_DVO_0_9V			(0x0)
+#define RT5663_LDO1_DVO_1_0V			(0x1)
+#define RT5663_LDO1_DVO_1_2V			(0x2)
+#define RT5663_LDO1_DVO_1_4V			(0x3)
 
 /* Power Management for Analog 2 (0x0064) */
-#define RT5668_PWR_BST1				(0x1 << 15)
-#define RT5668_PWR_BST1_MASK			(0x1 << 15)
-#define RT5668_PWR_BST1_SHIFT			15
-#define RT5668_PWR_BST1_OFF			(0x0 << 15)
-#define RT5668_PWR_BST1_ON			(0x1 << 15)
-#define RT5668_PWR_BST2				(0x1 << 14)
-#define RT5668_PWR_BST2_MASK			(0x1 << 14)
-#define RT5668_PWR_BST2_SHIFT			14
-#define RT5668_PWR_MB1				(0x1 << 11)
-#define RT5668_PWR_MB1_SHIFT			11
-#define RT5668_PWR_MB2				(0x1 << 10)
-#define RT5668_PWR_MB2_SHIFT			10
-#define RT5668_PWR_BST2_OP			(0x1 << 6)
-#define RT5668_PWR_BST2_OP_MASK			(0x1 << 6)
-#define RT5668_PWR_BST2_OP_SHIFT		6
-#define RT5668_PWR_JD1				(0x1 << 3)
-#define RT5668_PWR_JD1_MASK			(0x1 << 3)
-#define RT5668_PWR_JD1_SHIFT			3
-#define RT5668_PWR_JD2				(0x1 << 2)
-#define RT5668_PWR_JD2_MASK			(0x1 << 2)
-#define RT5668_PWR_JD2_SHIFT			2
-#define RT5668_PWR_RECMIX1			(0x1 << 1)
-#define RT5668_PWR_RECMIX1_SHIFT		1
-#define RT5668_PWR_RECMIX2			(0x1)
-#define RT5668_PWR_RECMIX2_SHIFT		0
+#define RT5663_PWR_BST1				(0x1 << 15)
+#define RT5663_PWR_BST1_MASK			(0x1 << 15)
+#define RT5663_PWR_BST1_SHIFT			15
+#define RT5663_PWR_BST1_OFF			(0x0 << 15)
+#define RT5663_PWR_BST1_ON			(0x1 << 15)
+#define RT5663_PWR_BST2				(0x1 << 14)
+#define RT5663_PWR_BST2_MASK			(0x1 << 14)
+#define RT5663_PWR_BST2_SHIFT			14
+#define RT5663_PWR_MB1				(0x1 << 11)
+#define RT5663_PWR_MB1_SHIFT			11
+#define RT5663_PWR_MB2				(0x1 << 10)
+#define RT5663_PWR_MB2_SHIFT			10
+#define RT5663_PWR_BST2_OP			(0x1 << 6)
+#define RT5663_PWR_BST2_OP_MASK			(0x1 << 6)
+#define RT5663_PWR_BST2_OP_SHIFT		6
+#define RT5663_PWR_JD1				(0x1 << 3)
+#define RT5663_PWR_JD1_MASK			(0x1 << 3)
+#define RT5663_PWR_JD1_SHIFT			3
+#define RT5663_PWR_JD2				(0x1 << 2)
+#define RT5663_PWR_JD2_MASK			(0x1 << 2)
+#define RT5663_PWR_JD2_SHIFT			2
+#define RT5663_PWR_RECMIX1			(0x1 << 1)
+#define RT5663_PWR_RECMIX1_SHIFT		1
+#define RT5663_PWR_RECMIX2			(0x1)
+#define RT5663_PWR_RECMIX2_SHIFT		0
 
 /* Power Management for Analog 3 (0x0065) */
-#define RT5668_PWR_CBJ_MASK			(0x1 << 9)
-#define RT5668_PWR_CBJ_SHIFT			9
-#define RT5668_PWR_CBJ_OFF			(0x0 << 9)
-#define RT5668_PWR_CBJ_ON			(0x1 << 9)
-#define RT5668_PWR_PLL				(0x1 << 6)
-#define RT5668_PWR_PLL_SHIFT			6
-#define RT5668_PWR_LDO2				(0x1 << 2)
-#define RT5668_PWR_LDO2_SHIFT			2
+#define RT5663_PWR_CBJ_MASK			(0x1 << 9)
+#define RT5663_PWR_CBJ_SHIFT			9
+#define RT5663_PWR_CBJ_OFF			(0x0 << 9)
+#define RT5663_PWR_CBJ_ON			(0x1 << 9)
+#define RT5663_PWR_PLL				(0x1 << 6)
+#define RT5663_PWR_PLL_SHIFT			6
+#define RT5663_PWR_LDO2				(0x1 << 2)
+#define RT5663_PWR_LDO2_SHIFT			2
 
 /* Power Management for Volume (0x0067) */
-#define RT5668_PWR_MIC_DET			(0x1 << 5)
-#define RT5668_PWR_MIC_DET_SHIFT		5
+#define RT5663_V2_PWR_MIC_DET			(0x1 << 5)
+#define RT5663_V2_PWR_MIC_DET_SHIFT		5
 
 /* MCLK and System Clock Detection Control (0x006b) */
-#define RT5668_EN_ANA_CLK_DET_MASK		(0x1 << 15)
-#define RT5668_EN_ANA_CLK_DET_SHIFT		15
-#define RT5668_EN_ANA_CLK_DET_DIS		(0x0 << 15)
-#define RT5668_EN_ANA_CLK_DET_AUTO		(0x1 << 15)
-#define RT5668_PWR_CLK_DET_MASK			(0x1)
-#define RT5668_PWR_CLK_DET_SHIFT		0
-#define RT5668_PWR_CLK_DET_DIS			(0x0)
-#define RT5668_PWR_CLK_DET_EN			(0x1)
+#define RT5663_EN_ANA_CLK_DET_MASK		(0x1 << 15)
+#define RT5663_EN_ANA_CLK_DET_SHIFT		15
+#define RT5663_EN_ANA_CLK_DET_DIS		(0x0 << 15)
+#define RT5663_EN_ANA_CLK_DET_AUTO		(0x1 << 15)
+#define RT5663_PWR_CLK_DET_MASK			(0x1)
+#define RT5663_PWR_CLK_DET_SHIFT		0
+#define RT5663_PWR_CLK_DET_DIS			(0x0)
+#define RT5663_PWR_CLK_DET_EN			(0x1)
 
 /* I2S1 Audio Serial Data Port Control (0x0070) */
-#define RT5668_I2S_MS_MASK			(0x1 << 15)
-#define RT5668_I2S_MS_SHIFT			15
-#define RT5668_I2S_MS_M				(0x0 << 15)
-#define RT5668_I2S_MS_S				(0x1 << 15)
-#define RT5668_I2S_BP_MASK			(0x1 << 8)
-#define RT5668_I2S_BP_SHIFT			8
-#define RT5668_I2S_BP_NOR			(0x0 << 8)
-#define RT5668_I2S_BP_INV			(0x1 << 8)
-#define RT5668_I2S_DL_MASK			(0x3 << 4)
-#define RT5668_I2S_DL_SHIFT			4
-#define RT5668_I2S_DL_16			(0x0 << 4)
-#define RT5668_I2S_DL_20			(0x1 << 4)
-#define RT5668_I2S_DL_24			(0x2 << 4)
-#define RT5668_I2S_DL_8				(0x3 << 4)
-#define RT5668_I2S_DF_MASK			(0x7)
-#define RT5668_I2S_DF_SHIFT			0
-#define RT5668_I2S_DF_I2S			(0x0)
-#define RT5668_I2S_DF_LEFT			(0x1)
-#define RT5668_I2S_DF_PCM_A			(0x2)
-#define RT5668_I2S_DF_PCM_B			(0x3)
-#define RT5668_I2S_DF_PCM_A_N			(0x6)
-#define RT5668_I2S_DF_PCM_B_N			(0x7)
+#define RT5663_I2S_MS_MASK			(0x1 << 15)
+#define RT5663_I2S_MS_SHIFT			15
+#define RT5663_I2S_MS_M				(0x0 << 15)
+#define RT5663_I2S_MS_S				(0x1 << 15)
+#define RT5663_I2S_BP_MASK			(0x1 << 8)
+#define RT5663_I2S_BP_SHIFT			8
+#define RT5663_I2S_BP_NOR			(0x0 << 8)
+#define RT5663_I2S_BP_INV			(0x1 << 8)
+#define RT5663_I2S_DL_MASK			(0x3 << 4)
+#define RT5663_I2S_DL_SHIFT			4
+#define RT5663_I2S_DL_16			(0x0 << 4)
+#define RT5663_I2S_DL_20			(0x1 << 4)
+#define RT5663_I2S_DL_24			(0x2 << 4)
+#define RT5663_I2S_DL_8				(0x3 << 4)
+#define RT5663_I2S_DF_MASK			(0x7)
+#define RT5663_I2S_DF_SHIFT			0
+#define RT5663_I2S_DF_I2S			(0x0)
+#define RT5663_I2S_DF_LEFT			(0x1)
+#define RT5663_I2S_DF_PCM_A			(0x2)
+#define RT5663_I2S_DF_PCM_B			(0x3)
+#define RT5663_I2S_DF_PCM_A_N			(0x6)
+#define RT5663_I2S_DF_PCM_B_N			(0x7)
 
 /* ADC/DAC Clock Control 1 (0x0073) */
-#define RT5668_I2S_PD1_MASK			(0x7 << 12)
-#define RT5668_I2S_PD1_SHIFT			12
-#define RT5668_M_I2S_DIV_MASK			(0x7 << 8)
-#define RT5668_M_I2S_DIV_SHIFT			8
-#define RT5668_CLK_SRC_MASK			(0x3 << 4)
-#define RT5668_CLK_SRC_MCLK			(0x0 << 4)
-#define RT5668_CLK_SRC_PLL_OUT			(0x1 << 4)
-#define RT5668_CLK_SRC_DIV			(0x2 << 4)
-#define RT5668_CLK_SRC_RC			(0x3 << 4)
-#define RT5668_DAC_OSR_MASK			(0x3 << 2)
-#define RT5668_DAC_OSR_SHIFT			2
-#define RT5668_DAC_OSR_128			(0x0 << 2)
-#define RT5668_DAC_OSR_64			(0x1 << 2)
-#define RT5668_DAC_OSR_32			(0x2 << 2)
-#define RT5668_ADC_OSR_MASK			(0x3)
-#define RT5668_ADC_OSR_SHIFT			0
-#define RT5668_ADC_OSR_128			(0x0)
-#define RT5668_ADC_OSR_64			(0x1)
-#define RT5668_ADC_OSR_32			(0x2)
+#define RT5663_I2S_PD1_MASK			(0x7 << 12)
+#define RT5663_I2S_PD1_SHIFT			12
+#define RT5663_M_I2S_DIV_MASK			(0x7 << 8)
+#define RT5663_M_I2S_DIV_SHIFT			8
+#define RT5663_CLK_SRC_MASK			(0x3 << 4)
+#define RT5663_CLK_SRC_MCLK			(0x0 << 4)
+#define RT5663_CLK_SRC_PLL_OUT			(0x1 << 4)
+#define RT5663_CLK_SRC_DIV			(0x2 << 4)
+#define RT5663_CLK_SRC_RC			(0x3 << 4)
+#define RT5663_DAC_OSR_MASK			(0x3 << 2)
+#define RT5663_DAC_OSR_SHIFT			2
+#define RT5663_DAC_OSR_128			(0x0 << 2)
+#define RT5663_DAC_OSR_64			(0x1 << 2)
+#define RT5663_DAC_OSR_32			(0x2 << 2)
+#define RT5663_ADC_OSR_MASK			(0x3)
+#define RT5663_ADC_OSR_SHIFT			0
+#define RT5663_ADC_OSR_128			(0x0)
+#define RT5663_ADC_OSR_64			(0x1)
+#define RT5663_ADC_OSR_32			(0x2)
 
 /* TDM1 control 1 (0x0078) */
-#define RT5668_TDM_MODE_MASK			(0x1 << 15)
-#define RT5668_TDM_MODE_SHIFT			15
-#define RT5668_TDM_MODE_I2S			(0x0 << 15)
-#define RT5668_TDM_MODE_TDM			(0x1 << 15)
-#define RT5668_TDM_IN_CH_MASK			(0x3 << 10)
-#define RT5668_TDM_IN_CH_SHIFT			10
-#define RT5668_TDM_IN_CH_2			(0x0 << 10)
-#define RT5668_TDM_IN_CH_4			(0x1 << 10)
-#define RT5668_TDM_IN_CH_6			(0x2 << 10)
-#define RT5668_TDM_IN_CH_8			(0x3 << 10)
-#define RT5668_TDM_OUT_CH_MASK			(0x3 << 8)
-#define RT5668_TDM_OUT_CH_SHIFT			8
-#define RT5668_TDM_OUT_CH_2			(0x0 << 8)
-#define RT5668_TDM_OUT_CH_4			(0x1 << 8)
-#define RT5668_TDM_OUT_CH_6			(0x2 << 8)
-#define RT5668_TDM_OUT_CH_8			(0x3 << 8)
-#define RT5668_TDM_IN_LEN_MASK			(0x3 << 6)
-#define RT5668_TDM_IN_LEN_SHIFT			6
-#define RT5668_TDM_IN_LEN_16			(0x0 << 6)
-#define RT5668_TDM_IN_LEN_20			(0x1 << 6)
-#define RT5668_TDM_IN_LEN_24			(0x2 << 6)
-#define RT5668_TDM_IN_LEN_32			(0x3 << 6)
-#define RT5668_TDM_OUT_LEN_MASK			(0x3 << 4)
-#define RT5668_TDM_OUT_LEN_SHIFT		4
-#define RT5668_TDM_OUT_LEN_16			(0x0 << 4)
-#define RT5668_TDM_OUT_LEN_20			(0x1 << 4)
-#define RT5668_TDM_OUT_LEN_24			(0x2 << 4)
-#define RT5668_TDM_OUT_LEN_32			(0x3 << 4)
+#define RT5663_TDM_MODE_MASK			(0x1 << 15)
+#define RT5663_TDM_MODE_SHIFT			15
+#define RT5663_TDM_MODE_I2S			(0x0 << 15)
+#define RT5663_TDM_MODE_TDM			(0x1 << 15)
+#define RT5663_TDM_IN_CH_MASK			(0x3 << 10)
+#define RT5663_TDM_IN_CH_SHIFT			10
+#define RT5663_TDM_IN_CH_2			(0x0 << 10)
+#define RT5663_TDM_IN_CH_4			(0x1 << 10)
+#define RT5663_TDM_IN_CH_6			(0x2 << 10)
+#define RT5663_TDM_IN_CH_8			(0x3 << 10)
+#define RT5663_TDM_OUT_CH_MASK			(0x3 << 8)
+#define RT5663_TDM_OUT_CH_SHIFT			8
+#define RT5663_TDM_OUT_CH_2			(0x0 << 8)
+#define RT5663_TDM_OUT_CH_4			(0x1 << 8)
+#define RT5663_TDM_OUT_CH_6			(0x2 << 8)
+#define RT5663_TDM_OUT_CH_8			(0x3 << 8)
+#define RT5663_TDM_IN_LEN_MASK			(0x3 << 6)
+#define RT5663_TDM_IN_LEN_SHIFT			6
+#define RT5663_TDM_IN_LEN_16			(0x0 << 6)
+#define RT5663_TDM_IN_LEN_20			(0x1 << 6)
+#define RT5663_TDM_IN_LEN_24			(0x2 << 6)
+#define RT5663_TDM_IN_LEN_32			(0x3 << 6)
+#define RT5663_TDM_OUT_LEN_MASK			(0x3 << 4)
+#define RT5663_TDM_OUT_LEN_SHIFT		4
+#define RT5663_TDM_OUT_LEN_16			(0x0 << 4)
+#define RT5663_TDM_OUT_LEN_20			(0x1 << 4)
+#define RT5663_TDM_OUT_LEN_24			(0x2 << 4)
+#define RT5663_TDM_OUT_LEN_32			(0x3 << 4)
 
 /* Global Clock Control (0x0080) */
-#define RT5668_SCLK_SRC_MASK			(0x3 << 14)
-#define RT5668_SCLK_SRC_SHIFT			14
-#define RT5668_SCLK_SRC_MCLK			(0x0 << 14)
-#define RT5668_SCLK_SRC_PLL1			(0x1 << 14)
-#define RT5668_SCLK_SRC_RCCLK			(0x2 << 14)
-#define RT5668_PLL1_SRC_MASK			(0x7 << 8)
-#define RT5668_PLL1_SRC_SHIFT			8
-#define RT5668_PLL1_SRC_MCLK			(0x0 << 8)
-#define RT5668_PLL1_SRC_BCLK1			(0x1 << 8)
-#define RT5668_PLL1_PD_MASK			(0x1 << 4)
-#define RT5668_PLL1_PD_SHIFT			4
-
-#define RT5668_PLL_INP_MAX			40000000
-#define RT5668_PLL_INP_MIN			256000
+#define RT5663_SCLK_SRC_MASK			(0x3 << 14)
+#define RT5663_SCLK_SRC_SHIFT			14
+#define RT5663_SCLK_SRC_MCLK			(0x0 << 14)
+#define RT5663_SCLK_SRC_PLL1			(0x1 << 14)
+#define RT5663_SCLK_SRC_RCCLK			(0x2 << 14)
+#define RT5663_PLL1_SRC_MASK			(0x7 << 11)
+#define RT5663_PLL1_SRC_SHIFT			11
+#define RT5663_PLL1_SRC_MCLK			(0x0 << 11)
+#define RT5663_PLL1_SRC_BCLK1			(0x1 << 11)
+#define RT5663_V2_PLL1_SRC_MASK			(0x7 << 8)
+#define RT5663_V2_PLL1_SRC_SHIFT		8
+#define RT5663_V2_PLL1_SRC_MCLK			(0x0 << 8)
+#define RT5663_V2_PLL1_SRC_BCLK1		(0x1 << 8)
+#define RT5663_PLL1_PD_MASK			(0x1 << 4)
+#define RT5663_PLL1_PD_SHIFT			4
+
+#define RT5663_PLL_INP_MAX			40000000
+#define RT5663_PLL_INP_MIN			256000
 /* PLL M/N/K Code Control 1 (0x0081) */
-#define RT5668_PLL_N_MAX			0x001ff
-#define RT5668_PLL_N_MASK			(RT5668_PLL_N_MAX << 7)
-#define RT5668_PLL_N_SHIFT			7
-#define RT5668_PLL_K_MAX			0x001f
-#define RT5668_PLL_K_MASK			(RT5668_PLL_K_MAX)
-#define RT5668_PLL_K_SHIFT			0
+#define RT5663_PLL_N_MAX			0x001ff
+#define RT5663_PLL_N_MASK			(RT5663_PLL_N_MAX << 7)
+#define RT5663_PLL_N_SHIFT			7
+#define RT5663_PLL_K_MAX			0x001f
+#define RT5663_PLL_K_MASK			(RT5663_PLL_K_MAX)
+#define RT5663_PLL_K_SHIFT			0
 
 /* PLL M/N/K Code Control 2 (0x0082) */
-#define RT5668_PLL_M_MAX			0x00f
-#define RT5668_PLL_M_MASK			(RT5668_PLL_M_MAX << 12)
-#define RT5668_PLL_M_SHIFT			12
-#define RT5668_PLL_M_BP				(0x1 << 11)
-#define RT5668_PLL_M_BP_SHIFT			11
+#define RT5663_PLL_M_MAX			0x00f
+#define RT5663_PLL_M_MASK			(RT5663_PLL_M_MAX << 12)
+#define RT5663_PLL_M_SHIFT			12
+#define RT5663_PLL_M_BP				(0x1 << 11)
+#define RT5663_PLL_M_BP_SHIFT			11
 
 /* PLL tracking mode 1 (0x0083) */
-#define RT5668_I2S1_ASRC_MASK			(0x1 << 13)
-#define RT5668_I2S1_ASRC_SHIFT			13
-#define RT5668_DAC_STO1_ASRC_MASK		(0x1 << 12)
-#define RT5668_DAC_STO1_ASRC_SHIFT		12
-#define RT5668_ADC_STO1_ASRC_MASK		(0x1 << 4)
-#define RT5668_ADC_STO1_ASRC_SHIFT		4
+#define RT5663_V2_I2S1_ASRC_MASK			(0x1 << 13)
+#define RT5663_V2_I2S1_ASRC_SHIFT			13
+#define RT5663_V2_DAC_STO1_ASRC_MASK		(0x1 << 12)
+#define RT5663_V2_DAC_STO1_ASRC_SHIFT		12
+#define RT5663_V2_ADC_STO1_ASRC_MASK		(0x1 << 4)
+#define RT5663_V2_ADC_STO1_ASRC_SHIFT		4
 
 /* PLL tracking mode 2 (0x0084)*/
-#define RT5668_DA_STO1_TRACK_MASK		(0x7 << 12)
-#define RT5668_DA_STO1_TRACK_SHIFT		12
-#define RT5668_DA_STO1_TRACK_SYSCLK		(0x0 << 12)
-#define RT5668_DA_STO1_TRACK_I2S1		(0x1 << 12)
+#define RT5663_DA_STO1_TRACK_MASK		(0x7 << 12)
+#define RT5663_DA_STO1_TRACK_SHIFT		12
+#define RT5663_DA_STO1_TRACK_SYSCLK		(0x0 << 12)
+#define RT5663_DA_STO1_TRACK_I2S1		(0x1 << 12)
 
 /* PLL tracking mode 3 (0x0085)*/
-#define RT5668_AD_STO1_TRACK_MASK		(0x7 << 12)
-#define RT5668_AD_STO1_TRACK_SHIFT		12
-#define RT5668_AD_STO1_TRACK_SYSCLK		(0x0 << 12)
-#define RT5668_AD_STO1_TRACK_I2S1		(0x1 << 12)
+#define RT5663_V2_AD_STO1_TRACK_MASK		(0x7 << 12)
+#define RT5663_V2_AD_STO1_TRACK_SHIFT		12
+#define RT5663_V2_AD_STO1_TRACK_SYSCLK		(0x0 << 12)
+#define RT5663_V2_AD_STO1_TRACK_I2S1		(0x1 << 12)
 
 /* HPOUT Charge pump control 1 (0x0091) */
-#define RT5668_OSW_HP_L_MASK			(0x1 << 11)
-#define RT5668_OSW_HP_L_SHIFT			11
-#define RT5668_OSW_HP_L_EN			(0x1 << 11)
-#define RT5668_OSW_HP_L_DIS			(0x0 << 11)
-#define RT5668_OSW_HP_R_MASK			(0x1 << 10)
-#define RT5668_OSW_HP_R_SHIFT			10
-#define RT5668_OSW_HP_R_EN			(0x1 << 10)
-#define RT5668_OSW_HP_R_DIS			(0x0 << 10)
-#define RT5668_SEL_PM_HP_MASK			(0x3 << 8)
-#define RT5668_SEL_PM_HP_SHIFT			8
-#define RT5668_SEL_PM_HP_0_6			(0x0 << 8)
-#define RT5668_SEL_PM_HP_0_9			(0x1 << 8)
-#define RT5668_SEL_PM_HP_1_8			(0x2 << 8)
-#define RT5668_SEL_PM_HP_HIGH			(0x3 << 8)
-#define RT5668_OVCD_HP_MASK			(0x1 << 2)
-#define RT5668_OVCD_HP_SHIFT			2
-#define RT5668_OVCD_HP_EN			(0x1 << 2)
-#define RT5668_OVCD_HP_DIS			(0x0 << 2)
+#define RT5663_OSW_HP_L_MASK			(0x1 << 11)
+#define RT5663_OSW_HP_L_SHIFT			11
+#define RT5663_OSW_HP_L_EN			(0x1 << 11)
+#define RT5663_OSW_HP_L_DIS			(0x0 << 11)
+#define RT5663_OSW_HP_R_MASK			(0x1 << 10)
+#define RT5663_OSW_HP_R_SHIFT			10
+#define RT5663_OSW_HP_R_EN			(0x1 << 10)
+#define RT5663_OSW_HP_R_DIS			(0x0 << 10)
+#define RT5663_SEL_PM_HP_MASK			(0x3 << 8)
+#define RT5663_SEL_PM_HP_SHIFT			8
+#define RT5663_SEL_PM_HP_0_6			(0x0 << 8)
+#define RT5663_SEL_PM_HP_0_9			(0x1 << 8)
+#define RT5663_SEL_PM_HP_1_8			(0x2 << 8)
+#define RT5663_SEL_PM_HP_HIGH			(0x3 << 8)
+#define RT5663_OVCD_HP_MASK			(0x1 << 2)
+#define RT5663_OVCD_HP_SHIFT			2
+#define RT5663_OVCD_HP_EN			(0x1 << 2)
+#define RT5663_OVCD_HP_DIS			(0x0 << 2)
 
 /* RC Clock Control (0x0094) */
-#define RT5668_DIG_25M_CLK_MASK			(0x1 << 9)
-#define RT5668_DIG_25M_CLK_SHIFT		9
-#define RT5668_DIG_25M_CLK_DIS			(0x0 << 9)
-#define RT5668_DIG_25M_CLK_EN			(0x1 << 9)
-#define RT5668_DIG_1M_CLK_MASK			(0x1 << 8)
-#define RT5668_DIG_1M_CLK_SHIFT			8
-#define RT5668_DIG_1M_CLK_DIS			(0x0 << 8)
-#define RT5668_DIG_1M_CLK_EN			(0x1 << 8)
+#define RT5663_DIG_25M_CLK_MASK			(0x1 << 9)
+#define RT5663_DIG_25M_CLK_SHIFT		9
+#define RT5663_DIG_25M_CLK_DIS			(0x0 << 9)
+#define RT5663_DIG_25M_CLK_EN			(0x1 << 9)
+#define RT5663_DIG_1M_CLK_MASK			(0x1 << 8)
+#define RT5663_DIG_1M_CLK_SHIFT			8
+#define RT5663_DIG_1M_CLK_DIS			(0x0 << 8)
+#define RT5663_DIG_1M_CLK_EN			(0x1 << 8)
 
 /* Auto Turn On 1M RC CLK (0x009f) */
-#define RT5668_IRQ_POW_SAV_MASK			(0x1 << 15)
-#define RT5668_IRQ_POW_SAV_SHIFT		15
-#define RT5668_IRQ_POW_SAV_DIS			(0x0 << 15)
-#define RT5668_IRQ_POW_SAV_EN			(0x1 << 15)
-#define RT5668_IRQ_POW_SAV_JD1_MASK		(0x1 << 14)
-#define RT5668_IRQ_POW_SAV_JD1_SHIFT		14
-#define RT5668_IRQ_POW_SAV_JD1_DIS		(0x0 << 14)
-#define RT5668_IRQ_POW_SAV_JD1_EN		(0x1 << 14)
+#define RT5663_IRQ_POW_SAV_MASK			(0x1 << 15)
+#define RT5663_IRQ_POW_SAV_SHIFT		15
+#define RT5663_IRQ_POW_SAV_DIS			(0x0 << 15)
+#define RT5663_IRQ_POW_SAV_EN			(0x1 << 15)
+#define RT5663_IRQ_POW_SAV_JD1_MASK		(0x1 << 14)
+#define RT5663_IRQ_POW_SAV_JD1_SHIFT		14
+#define RT5663_IRQ_POW_SAV_JD1_DIS		(0x0 << 14)
+#define RT5663_IRQ_POW_SAV_JD1_EN		(0x1 << 14)
 
 /* IRQ Control 1 (0x00b6) */
-#define RT5668_EN_CB_JD_MASK			(0x1 << 3)
-#define RT5668_EN_CB_JD_SHIFT			3
-#define RT5668_EN_CB_JD_EN			(0x1 << 3)
-#define RT5668_EN_CB_JD_DIS			(0x0 << 3)
+#define RT5663_EN_CB_JD_MASK			(0x1 << 3)
+#define RT5663_EN_CB_JD_SHIFT			3
+#define RT5663_EN_CB_JD_EN			(0x1 << 3)
+#define RT5663_EN_CB_JD_DIS			(0x0 << 3)
 
 /* IRQ Control 3 (0x00b8) */
-#define RT5668_EN_IRQ_INLINE_MASK		(0x1 << 6)
-#define RT5668_EN_IRQ_INLINE_SHIFT		6
-#define RT5668_EN_IRQ_INLINE_BYP		(0x0 << 6)
-#define RT5668_EN_IRQ_INLINE_NOR		(0x1 << 6)
+#define RT5663_V2_EN_IRQ_INLINE_MASK		(0x1 << 6)
+#define RT5663_V2_EN_IRQ_INLINE_SHIFT		6
+#define RT5663_V2_EN_IRQ_INLINE_BYP		(0x0 << 6)
+#define RT5663_V2_EN_IRQ_INLINE_NOR		(0x1 << 6)
 
 /* GPIO Control 1 (0x00c0) */
-#define RT5668_GP1_PIN_MASK			(0x1 << 15)
-#define RT5668_GP1_PIN_SHIFT			15
-#define RT5668_GP1_PIN_GPIO1			(0x0 << 15)
-#define RT5668_GP1_PIN_IRQ			(0x1 << 15)
+#define RT5663_GP1_PIN_MASK			(0x1 << 15)
+#define RT5663_GP1_PIN_SHIFT			15
+#define RT5663_GP1_PIN_GPIO1			(0x0 << 15)
+#define RT5663_GP1_PIN_IRQ			(0x1 << 15)
 
 /* GPIO Control 2 (0x00c1) */
-#define RT5668_GP4_PIN_CONF_MASK		(0x1 << 5)
-#define RT5668_GP4_PIN_CONF_SHIFT		5
-#define RT5668_GP4_PIN_CONF_INPUT		(0x0 << 5)
-#define RT5668_GP4_PIN_CONF_OUTPUT		(0x1 << 5)
+#define RT5663_GP4_PIN_CONF_MASK		(0x1 << 5)
+#define RT5663_GP4_PIN_CONF_SHIFT		5
+#define RT5663_GP4_PIN_CONF_INPUT		(0x0 << 5)
+#define RT5663_GP4_PIN_CONF_OUTPUT		(0x1 << 5)
 
 /* GPIO Control 2 (0x00c2) */
-#define RT5668_GP8_PIN_CONF_MASK		(0x1 << 13)
-#define RT5668_GP8_PIN_CONF_SHIFT		13
-#define RT5668_GP8_PIN_CONF_INPUT		(0x0 << 13)
-#define RT5668_GP8_PIN_CONF_OUTPUT		(0x1 << 13)
+#define RT5663_GP8_PIN_CONF_MASK		(0x1 << 13)
+#define RT5663_GP8_PIN_CONF_SHIFT		13
+#define RT5663_GP8_PIN_CONF_INPUT		(0x0 << 13)
+#define RT5663_GP8_PIN_CONF_OUTPUT		(0x1 << 13)
 
 /* 4 Buttons Inline Command Function 1 (0x00df) */
-#define RT5668_4BTN_CLK_DEB_MASK		(0x3 << 2)
-#define RT5668_4BTN_CLK_DEB_SHIFT		2
-#define RT5668_4BTN_CLK_DEB_8MS			(0x0 << 2)
-#define RT5668_4BTN_CLK_DEB_16MS		(0x1 << 2)
-#define RT5668_4BTN_CLK_DEB_32MS		(0x2 << 2)
-#define RT5668_4BTN_CLK_DEB_65MS		(0x3 << 2)
+#define RT5663_4BTN_CLK_DEB_MASK		(0x3 << 2)
+#define RT5663_4BTN_CLK_DEB_SHIFT		2
+#define RT5663_4BTN_CLK_DEB_8MS			(0x0 << 2)
+#define RT5663_4BTN_CLK_DEB_16MS		(0x1 << 2)
+#define RT5663_4BTN_CLK_DEB_32MS		(0x2 << 2)
+#define RT5663_4BTN_CLK_DEB_65MS		(0x3 << 2)
 
 /* Inline Command Function 6 (0x00e0) */
-#define RT5668_EN_4BTN_INL_MASK			(0x1 << 15)
-#define RT5668_EN_4BTN_INL_SHIFT		15
-#define RT5668_EN_4BTN_INL_DIS			(0x0 << 15)
-#define RT5668_EN_4BTN_INL_EN			(0x1 << 15)
-#define RT5668_RESET_4BTN_INL_MASK		(0x1 << 14)
-#define RT5668_RESET_4BTN_INL_SHIFT		14
-#define RT5668_RESET_4BTN_INL_RESET		(0x0 << 14)
-#define RT5668_RESET_4BTN_INL_NOR		(0x1 << 14)
+#define RT5663_EN_4BTN_INL_MASK			(0x1 << 15)
+#define RT5663_EN_4BTN_INL_SHIFT		15
+#define RT5663_EN_4BTN_INL_DIS			(0x0 << 15)
+#define RT5663_EN_4BTN_INL_EN			(0x1 << 15)
+#define RT5663_RESET_4BTN_INL_MASK		(0x1 << 14)
+#define RT5663_RESET_4BTN_INL_SHIFT		14
+#define RT5663_RESET_4BTN_INL_RESET		(0x0 << 14)
+#define RT5663_RESET_4BTN_INL_NOR		(0x1 << 14)
 
 /* Digital Misc Control (0x00fa) */
-#define RT5668_DIG_GATE_CTRL_MASK		0x1
-#define RT5668_DIG_GATE_CTRL_SHIFT		(0)
-#define RT5668_DIG_GATE_CTRL_DIS		0x0
-#define RT5668_DIG_GATE_CTRL_EN			0x1
+#define RT5663_DIG_GATE_CTRL_MASK		0x1
+#define RT5663_DIG_GATE_CTRL_SHIFT		(0)
+#define RT5663_DIG_GATE_CTRL_DIS		0x0
+#define RT5663_DIG_GATE_CTRL_EN			0x1
 
 /* Chopper and Clock control for DAC L (0x013a)*/
-#define RT5668_CKXEN_DAC1_MASK			(0x1 << 13)
-#define RT5668_CKXEN_DAC1_SHIFT			13
-#define RT5668_CKGEN_DAC1_MASK			(0x1 << 12)
-#define RT5668_CKGEN_DAC1_SHIFT			12
+#define RT5663_CKXEN_DAC1_MASK			(0x1 << 13)
+#define RT5663_CKXEN_DAC1_SHIFT			13
+#define RT5663_CKGEN_DAC1_MASK			(0x1 << 12)
+#define RT5663_CKGEN_DAC1_SHIFT			12
 
 /* Chopper and Clock control for ADC (0x013b)*/
-#define RT5668_CKXEN_ADCC_MASK			(0x1 << 13)
-#define RT5668_CKXEN_ADCC_SHIFT			13
-#define RT5668_CKGEN_ADCC_MASK			(0x1 << 12)
-#define RT5668_CKGEN_ADCC_SHIFT			12
+#define RT5663_CKXEN_ADCC_MASK			(0x1 << 13)
+#define RT5663_CKXEN_ADCC_SHIFT			13
+#define RT5663_CKGEN_ADCC_MASK			(0x1 << 12)
+#define RT5663_CKGEN_ADCC_SHIFT			12
 
 /* HP Behavior Logic Control 2 (0x01db) */
-#define RT5668_HP_SIG_SRC1_MASK			(0x3)
-#define RT5668_HP_SIG_SRC1_SHIFT		0
-#define RT5668_HP_SIG_SRC1_HP_DC		(0x0)
-#define RT5668_HP_SIG_SRC1_HP_CALIB		(0x1)
-#define RT5668_HP_SIG_SRC1_REG			(0x2)
-#define RT5668_HP_SIG_SRC1_SILENCE		(0x3)
+#define RT5663_HP_SIG_SRC1_MASK			(0x3)
+#define RT5663_HP_SIG_SRC1_SHIFT		0
+#define RT5663_HP_SIG_SRC1_HP_DC		(0x0)
+#define RT5663_HP_SIG_SRC1_HP_CALIB		(0x1)
+#define RT5663_HP_SIG_SRC1_REG			(0x2)
+#define RT5663_HP_SIG_SRC1_SILENCE		(0x3)
 
 /* RT5663 specific register */
 #define RT5663_HP_OUT_EN			0x0002
@@ -707,6 +704,10 @@
 #define RT5663_TDM_3				0x0079
 #define RT5663_TDM_4				0x007a
 #define RT5663_TDM_5				0x007b
+#define RT5663_TDM_6				0x007c
+#define RT5663_TDM_7				0x007d
+#define RT5663_TDM_8				0x007e
+#define RT5663_TDM_9				0x007f
 #define RT5663_GLB_CLK				0x0080
 #define RT5663_PLL_1				0x0081
 #define RT5663_PLL_2				0x0082
@@ -739,7 +740,7 @@
 #define RT5663_INT_ST_2				0x00bf
 #define RT5663_GPIO_1				0x00c0
 #define RT5663_GPIO_2				0x00c1
-#define RT5663_GPIO_STA				0x00c5
+#define RT5663_GPIO_STA1			0x00c5
 #define RT5663_SIN_GEN_1			0x00cb
 #define RT5663_SIN_GEN_2			0x00cc
 #define RT5663_SIN_GEN_3			0x00cd
@@ -800,6 +801,8 @@
 #define RT5663_TEST_MODE_1			0x0144
 #define RT5663_TEST_MODE_2			0x0145
 #define RT5663_TEST_MODE_3			0x0146
+#define RT5663_TEST_MODE_4			0x0147
+#define RT5663_TEST_MODE_5			0x0148
 #define RT5663_STO_DRE_1			0x0160
 #define RT5663_STO_DRE_2			0x0161
 #define RT5663_STO_DRE_3			0x0162
@@ -921,19 +924,19 @@
 #define RT5663_ADC_EQ_POST_VOL_L		0x03f2
 #define RT5663_ADC_EQ_POST_VOL_R		0x03f3
 
-/* RT5663: RECMIX Control (0x0010) */
+/* RECMIX Control (0x0010) */
 #define RT5663_RECMIX1_BST1_MASK		(0x1)
 #define RT5663_RECMIX1_BST1_SHIFT		0
 #define RT5663_RECMIX1_BST1_ON			(0x0)
 #define RT5663_RECMIX1_BST1_OFF			(0x1)
 
-/* RT5663: Bypass Stereo1 DAC Mixer Control (0x002d) */
+/* Bypass Stereo1 DAC Mixer Control (0x002d) */
 #define RT5663_DACL1_SRC_MASK			(0x1 << 3)
 #define RT5663_DACL1_SRC_SHIFT			3
 #define RT5663_DACR1_SRC_MASK			(0x1 << 2)
 #define RT5663_DACR1_SRC_SHIFT			2
 
-/* RT5663: TDM control 2 (0x0078) */
+/* TDM control 2 (0x0078) */
 #define RT5663_DATA_SWAP_ADCDAT1_MASK		(0x3 << 14)
 #define RT5663_DATA_SWAP_ADCDAT1_SHIFT		14
 #define RT5663_DATA_SWAP_ADCDAT1_LR		(0x0 << 14)
@@ -941,7 +944,7 @@
 #define RT5663_DATA_SWAP_ADCDAT1_LL		(0x2 << 14)
 #define RT5663_DATA_SWAP_ADCDAT1_RR		(0x3 << 14)
 
-/* RT5663: TDM control 5 (0x007b) */
+/* TDM control 5 (0x007b) */
 #define RT5663_TDM_LENGTN_MASK			(0x3)
 #define RT5663_TDM_LENGTN_SHIFT			0
 #define RT5663_TDM_LENGTN_16			(0x0)
@@ -949,17 +952,6 @@
 #define RT5663_TDM_LENGTN_24			(0x2)
 #define RT5663_TDM_LENGTN_32			(0x3)
 
-/* RT5663: Global Clock Control (0x0080) */
-#define RT5663_SCLK_SRC_MASK			(0x3 << 14)
-#define RT5663_SCLK_SRC_SHIFT			14
-#define RT5663_SCLK_SRC_MCLK			(0x0 << 14)
-#define RT5663_SCLK_SRC_PLL1			(0x1 << 14)
-#define RT5663_SCLK_SRC_RCCLK			(0x2 << 14)
-#define RT5663_PLL1_SRC_MASK			(0x7 << 11)
-#define RT5663_PLL1_SRC_SHIFT			11
-#define RT5663_PLL1_SRC_MCLK			(0x0 << 11)
-#define RT5663_PLL1_SRC_BCLK1			(0x1 << 11)
-
 /* PLL tracking mode 1 (0x0083) */
 #define RT5663_I2S1_ASRC_MASK			(0x1 << 11)
 #define RT5663_I2S1_ASRC_SHIFT			11
@@ -978,37 +970,47 @@
 #define RT5663_AD_STO1_TRACK_SYSCLK		(0x0)
 #define RT5663_AD_STO1_TRACK_I2S1		(0x1)
 
-/* RT5663: HPOUT Charge pump control 1 (0x0091) */
+/* HPOUT Charge pump control 1 (0x0091) */
 #define RT5663_SI_HP_MASK			(0x1 << 12)
 #define RT5663_SI_HP_SHIFT			12
 #define RT5663_SI_HP_EN				(0x1 << 12)
 #define RT5663_SI_HP_DIS			(0x0 << 12)
 
-/* RT5663: GPIO Control 2 (0x00b6) */
+/* GPIO Control 2 (0x00b6) */
 #define RT5663_GP1_PIN_CONF_MASK		(0x1 << 2)
 #define RT5663_GP1_PIN_CONF_SHIFT		2
 #define RT5663_GP1_PIN_CONF_OUTPUT		(0x1 << 2)
 #define RT5663_GP1_PIN_CONF_INPUT		(0x0 << 2)
 
-/* RT5663: GPIO Control 2 (0x00b7) */
+/* GPIO Control 2 (0x00b7) */
 #define RT5663_EN_IRQ_INLINE_MASK		(0x1 << 3)
 #define RT5663_EN_IRQ_INLINE_SHIFT		3
 #define RT5663_EN_IRQ_INLINE_NOR		(0x1 << 3)
 #define RT5663_EN_IRQ_INLINE_BYP		(0x0 << 3)
 
-/* RT5663: IRQ Control 1 (0x00c1) */
+/* GPIO Control 1 (0x00c0) */
+#define RT5663_GPIO1_TYPE_MASK			(0x1 << 15)
+#define RT5663_GPIO1_TYPE_SHIFT			15
+#define RT5663_GPIO1_TYPE_EN			(0x1 << 15)
+#define RT5663_GPIO1_TYPE_DIS			(0x0 << 15)
+
+/* IRQ Control 1 (0x00c1) */
 #define RT5663_EN_IRQ_JD1_MASK			(0x1 << 6)
 #define RT5663_EN_IRQ_JD1_SHIFT			6
 #define RT5663_EN_IRQ_JD1_EN			(0x1 << 6)
 #define RT5663_EN_IRQ_JD1_DIS			(0x0 << 6)
+#define RT5663_SEL_GPIO1_MASK			(0x1 << 2)
+#define RT5663_SEL_GPIO1_SHIFT			6
+#define RT5663_SEL_GPIO1_EN			(0x1 << 2)
+#define RT5663_SEL_GPIO1_DIS			(0x0 << 2)
 
-/* RT5663: Inline Command Function 2 (0x00dc) */
+/* Inline Command Function 2 (0x00dc) */
 #define RT5663_PWR_MIC_DET_MASK			(0x1)
 #define RT5663_PWR_MIC_DET_SHIFT		0
 #define RT5663_PWR_MIC_DET_ON			(0x1)
 #define RT5663_PWR_MIC_DET_OFF			(0x0)
 
-/* RT5663: Embeeded Jack and Type Detection Control 1 (0x00e6)*/
+/* Embeeded Jack and Type Detection Control 1 (0x00e6)*/
 #define RT5663_CBJ_DET_MASK			(0x1 << 15)
 #define RT5663_CBJ_DET_SHIFT			15
 #define RT5663_CBJ_DET_DIS			(0x0 << 15)
@@ -1022,17 +1024,17 @@
 #define RT5663_POL_EXT_JD_EN			(0x1 << 10)
 #define RT5663_POL_EXT_JD_DIS			(0x0 << 10)
 
-/* RT5663: DACREF LDO Control (0x0112)*/
+/* DACREF LDO Control (0x0112)*/
 #define RT5663_PWR_LDO_DACREFL_MASK		(0x1 << 9)
 #define RT5663_PWR_LDO_DACREFL_SHIFT		9
 #define RT5663_PWR_LDO_DACREFR_MASK		(0x1 << 1)
 #define RT5663_PWR_LDO_DACREFR_SHIFT		1
 
-/* RT5663: Stereo Dynamic Range Enhancement Control 9 (0x0168, 0x0169)*/
+/* Stereo Dynamic Range Enhancement Control 9 (0x0168, 0x0169)*/
 #define RT5663_DRE_GAIN_HP_MASK			(0x1f)
 #define RT5663_DRE_GAIN_HP_SHIFT		0
 
-/* RT5663: Combo Jack Control (0x0250) */
+/* Combo Jack Control (0x0250) */
 #define RT5663_INBUF_CBJ_BST1_MASK		(0x1 << 11)
 #define RT5663_INBUF_CBJ_BST1_SHIFT		11
 #define RT5663_INBUF_CBJ_BST1_ON		(0x1 << 11)
@@ -1042,11 +1044,11 @@
 #define RT5663_CBJ_SENSE_BST1_L			(0x1 << 10)
 #define RT5663_CBJ_SENSE_BST1_R			(0x0 << 10)
 
-/* RT5663: Combo Jack Control (0x0251) */
+/* Combo Jack Control (0x0251) */
 #define RT5663_GAIN_BST1_MASK			(0xf)
 #define RT5663_GAIN_BST1_SHIFT			0
 
-/* RT5663: Dummy register 1 (0x02fa) */
+/* Dummy register 1 (0x02fa) */
 #define RT5663_EMB_CLK_MASK			(0x1 << 9)
 #define RT5663_EMB_CLK_SHIFT			9
 #define RT5663_EMB_CLK_EN			(0x1 << 9)
diff --git a/sound/soc/codecs/rt5665.c b/sound/soc/codecs/rt5665.c
new file mode 100644
index 000000000000..324461e985b3
--- /dev/null
+++ b/sound/soc/codecs/rt5665.c
@@ -0,0 +1,4874 @@
+/*
+ * rt5665.c  --  RT5665/RT5658 ALSA SoC audio codec driver
+ *
+ * Copyright 2016 Realtek Semiconductor Corp.
+ * Author: Bard Liao <bardliao@realtek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+#include <linux/pm.h>
+#include <linux/i2c.h>
+#include <linux/platform_device.h>
+#include <linux/spi/spi.h>
+#include <linux/acpi.h>
+#include <linux/gpio.h>
+#include <linux/of_gpio.h>
+#include <linux/regulator/consumer.h>
+#include <linux/mutex.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/jack.h>
+#include <sound/soc.h>
+#include <sound/soc-dapm.h>
+#include <sound/initval.h>
+#include <sound/tlv.h>
+#include <sound/rt5665.h>
+
+#include "rl6231.h"
+#include "rt5665.h"
+
+#define RT5665_NUM_SUPPLIES 3
+
+static const char *rt5665_supply_names[RT5665_NUM_SUPPLIES] = {
+	"AVDD",
+	"MICVDD",
+	"VBAT",
+};
+
+struct rt5665_priv {
+	struct snd_soc_codec *codec;
+	struct rt5665_platform_data pdata;
+	struct regmap *regmap;
+	struct gpio_desc *gpiod_ldo1_en;
+	struct gpio_desc *gpiod_reset;
+	struct snd_soc_jack *hs_jack;
+	struct regulator_bulk_data supplies[RT5665_NUM_SUPPLIES];
+	struct delayed_work jack_detect_work;
+	struct delayed_work calibrate_work;
+	struct delayed_work jd_check_work;
+	struct mutex calibrate_mutex;
+
+	int sysclk;
+	int sysclk_src;
+	int lrck[RT5665_AIFS];
+	int bclk[RT5665_AIFS];
+	int master[RT5665_AIFS];
+	int id;
+
+	int pll_src;
+	int pll_in;
+	int pll_out;
+
+	int jack_type;
+	int irq_work_delay_time;
+	unsigned int sar_adc_value;
+};
+
+static const struct reg_default rt5665_reg[] = {
+	{0x0000, 0x0000},
+	{0x0001, 0xc8c8},
+	{0x0002, 0x8080},
+	{0x0003, 0x8000},
+	{0x0004, 0xc80a},
+	{0x0005, 0x0000},
+	{0x0006, 0x0000},
+	{0x0007, 0x0000},
+	{0x000a, 0x0000},
+	{0x000b, 0x0000},
+	{0x000c, 0x0000},
+	{0x000d, 0x0000},
+	{0x000f, 0x0808},
+	{0x0010, 0x4040},
+	{0x0011, 0x0000},
+	{0x0012, 0x1404},
+	{0x0013, 0x1000},
+	{0x0014, 0xa00a},
+	{0x0015, 0x0404},
+	{0x0016, 0x0404},
+	{0x0017, 0x0011},
+	{0x0018, 0xafaf},
+	{0x0019, 0xafaf},
+	{0x001a, 0xafaf},
+	{0x001b, 0x0011},
+	{0x001c, 0x2f2f},
+	{0x001d, 0x2f2f},
+	{0x001e, 0x2f2f},
+	{0x001f, 0x0000},
+	{0x0020, 0x0000},
+	{0x0021, 0x0000},
+	{0x0022, 0x5757},
+	{0x0023, 0x0039},
+	{0x0026, 0xc0c0},
+	{0x0027, 0xc0c0},
+	{0x0028, 0xc0c0},
+	{0x0029, 0x8080},
+	{0x002a, 0xaaaa},
+	{0x002b, 0xaaaa},
+	{0x002c, 0xaba8},
+	{0x002d, 0x0000},
+	{0x002e, 0x0000},
+	{0x002f, 0x0000},
+	{0x0030, 0x0000},
+	{0x0031, 0x5000},
+	{0x0032, 0x0000},
+	{0x0033, 0x0000},
+	{0x0034, 0x0000},
+	{0x0035, 0x0000},
+	{0x003a, 0x0000},
+	{0x003b, 0x0000},
+	{0x003c, 0x00ff},
+	{0x003d, 0x0000},
+	{0x003e, 0x00ff},
+	{0x003f, 0x0000},
+	{0x0040, 0x0000},
+	{0x0041, 0x00ff},
+	{0x0042, 0x0000},
+	{0x0043, 0x00ff},
+	{0x0044, 0x0c0c},
+	{0x0049, 0xc00b},
+	{0x004a, 0x0000},
+	{0x004b, 0x031f},
+	{0x004d, 0x0000},
+	{0x004e, 0x001f},
+	{0x004f, 0x0000},
+	{0x0050, 0x001f},
+	{0x0052, 0xf000},
+	{0x0061, 0x0000},
+	{0x0062, 0x0000},
+	{0x0063, 0x003e},
+	{0x0064, 0x0000},
+	{0x0065, 0x0000},
+	{0x0066, 0x003f},
+	{0x0067, 0x0000},
+	{0x006b, 0x0000},
+	{0x006d, 0xff00},
+	{0x006e, 0x2808},
+	{0x006f, 0x000a},
+	{0x0070, 0x8000},
+	{0x0071, 0x8000},
+	{0x0072, 0x8000},
+	{0x0073, 0x7000},
+	{0x0074, 0x7770},
+	{0x0075, 0x0002},
+	{0x0076, 0x0001},
+	{0x0078, 0x00f0},
+	{0x0079, 0x0000},
+	{0x007a, 0x0000},
+	{0x007b, 0x0000},
+	{0x007c, 0x0000},
+	{0x007d, 0x0123},
+	{0x007e, 0x4500},
+	{0x007f, 0x8003},
+	{0x0080, 0x0000},
+	{0x0081, 0x0000},
+	{0x0082, 0x0000},
+	{0x0083, 0x0000},
+	{0x0084, 0x0000},
+	{0x0085, 0x0000},
+	{0x0086, 0x0008},
+	{0x0087, 0x0000},
+	{0x0088, 0x0000},
+	{0x0089, 0x0000},
+	{0x008a, 0x0000},
+	{0x008b, 0x0000},
+	{0x008c, 0x0003},
+	{0x008e, 0x0060},
+	{0x008f, 0x1000},
+	{0x0091, 0x0c26},
+	{0x0092, 0x0073},
+	{0x0093, 0x0000},
+	{0x0094, 0x0080},
+	{0x0098, 0x0000},
+	{0x0099, 0x0000},
+	{0x009a, 0x0007},
+	{0x009f, 0x0000},
+	{0x00a0, 0x0000},
+	{0x00a1, 0x0002},
+	{0x00a2, 0x0001},
+	{0x00a3, 0x0002},
+	{0x00a4, 0x0001},
+	{0x00ae, 0x2040},
+	{0x00af, 0x0000},
+	{0x00b6, 0x0000},
+	{0x00b7, 0x0000},
+	{0x00b8, 0x0000},
+	{0x00b9, 0x0000},
+	{0x00ba, 0x0002},
+	{0x00bb, 0x0000},
+	{0x00be, 0x0000},
+	{0x00c0, 0x0000},
+	{0x00c1, 0x0aaa},
+	{0x00c2, 0xaa80},
+	{0x00c3, 0x0003},
+	{0x00c4, 0x0000},
+	{0x00d0, 0x0000},
+	{0x00d1, 0x2244},
+	{0x00d3, 0x3300},
+	{0x00d4, 0x2200},
+	{0x00d9, 0x0809},
+	{0x00da, 0x0000},
+	{0x00db, 0x0008},
+	{0x00dc, 0x00c0},
+	{0x00dd, 0x6724},
+	{0x00de, 0x3131},
+	{0x00df, 0x0008},
+	{0x00e0, 0x4000},
+	{0x00e1, 0x3131},
+	{0x00e2, 0x600c},
+	{0x00ea, 0xb320},
+	{0x00eb, 0x0000},
+	{0x00ec, 0xb300},
+	{0x00ed, 0x0000},
+	{0x00ee, 0xb320},
+	{0x00ef, 0x0000},
+	{0x00f0, 0x0201},
+	{0x00f1, 0x0ddd},
+	{0x00f2, 0x0ddd},
+	{0x00f6, 0x0000},
+	{0x00f7, 0x0000},
+	{0x00f8, 0x0000},
+	{0x00fa, 0x0000},
+	{0x00fb, 0x0000},
+	{0x00fc, 0x0000},
+	{0x00fd, 0x0000},
+	{0x00fe, 0x10ec},
+	{0x00ff, 0x6451},
+	{0x0100, 0xaaaa},
+	{0x0101, 0x000a},
+	{0x010a, 0xaaaa},
+	{0x010b, 0xa0a0},
+	{0x010c, 0xaeae},
+	{0x010d, 0xaaaa},
+	{0x010e, 0xaaaa},
+	{0x010f, 0xaaaa},
+	{0x0110, 0xe002},
+	{0x0111, 0xa402},
+	{0x0112, 0xaaaa},
+	{0x0113, 0x2000},
+	{0x0117, 0x0f00},
+	{0x0125, 0x0410},
+	{0x0132, 0x0000},
+	{0x0133, 0x0000},
+	{0x0137, 0x5540},
+	{0x0138, 0x3700},
+	{0x0139, 0x79a1},
+	{0x013a, 0x2020},
+	{0x013b, 0x2020},
+	{0x013c, 0x2005},
+	{0x013f, 0x0000},
+	{0x0145, 0x0002},
+	{0x0146, 0x0000},
+	{0x0147, 0x0000},
+	{0x0148, 0x0000},
+	{0x0150, 0x0000},
+	{0x0160, 0x4eff},
+	{0x0161, 0x0080},
+	{0x0162, 0x0200},
+	{0x0163, 0x0800},
+	{0x0164, 0x0000},
+	{0x0165, 0x0000},
+	{0x0166, 0x0000},
+	{0x0167, 0x000f},
+	{0x0170, 0x4e87},
+	{0x0171, 0x0080},
+	{0x0172, 0x0200},
+	{0x0173, 0x0800},
+	{0x0174, 0x00ff},
+	{0x0175, 0x0000},
+	{0x0190, 0x413d},
+	{0x0191, 0x4139},
+	{0x0192, 0x4135},
+	{0x0193, 0x413d},
+	{0x0194, 0x0000},
+	{0x0195, 0x0000},
+	{0x0196, 0x0000},
+	{0x0197, 0x0000},
+	{0x0198, 0x0000},
+	{0x0199, 0x0000},
+	{0x01a0, 0x1e64},
+	{0x01a1, 0x06a3},
+	{0x01a2, 0x0000},
+	{0x01a3, 0x0000},
+	{0x01a4, 0x0000},
+	{0x01a5, 0x0000},
+	{0x01a6, 0x0000},
+	{0x01a7, 0x8000},
+	{0x01a8, 0x0000},
+	{0x01a9, 0x0000},
+	{0x01aa, 0x0000},
+	{0x01ab, 0x0000},
+	{0x01b5, 0x0000},
+	{0x01b6, 0x01c3},
+	{0x01b7, 0x02a0},
+	{0x01b8, 0x03e9},
+	{0x01b9, 0x1389},
+	{0x01ba, 0xc351},
+	{0x01bb, 0x0009},
+	{0x01bc, 0x0018},
+	{0x01bd, 0x002a},
+	{0x01be, 0x004c},
+	{0x01bf, 0x0097},
+	{0x01c0, 0x433d},
+	{0x01c1, 0x0000},
+	{0x01c2, 0x0000},
+	{0x01c3, 0x0000},
+	{0x01c4, 0x0000},
+	{0x01c5, 0x0000},
+	{0x01c6, 0x0000},
+	{0x01c7, 0x0000},
+	{0x01c8, 0x40af},
+	{0x01c9, 0x0702},
+	{0x01ca, 0x0000},
+	{0x01cb, 0x0000},
+	{0x01cc, 0x5757},
+	{0x01cd, 0x5757},
+	{0x01ce, 0x5757},
+	{0x01cf, 0x5757},
+	{0x01d0, 0x5757},
+	{0x01d1, 0x5757},
+	{0x01d2, 0x5757},
+	{0x01d3, 0x5757},
+	{0x01d4, 0x5757},
+	{0x01d5, 0x5757},
+	{0x01d6, 0x003c},
+	{0x01da, 0x0000},
+	{0x01db, 0x0000},
+	{0x01dc, 0x0000},
+	{0x01de, 0x7c00},
+	{0x01df, 0x0320},
+	{0x01e0, 0x06a1},
+	{0x01e1, 0x0000},
+	{0x01e2, 0x0000},
+	{0x01e3, 0x0000},
+	{0x01e4, 0x0000},
+	{0x01e6, 0x0001},
+	{0x01e7, 0x0000},
+	{0x01e8, 0x0000},
+	{0x01ea, 0xbf3f},
+	{0x01eb, 0x0000},
+	{0x01ec, 0x0000},
+	{0x01ed, 0x0000},
+	{0x01ee, 0x0000},
+	{0x01ef, 0x0000},
+	{0x01f0, 0x0000},
+	{0x01f1, 0x0000},
+	{0x01f2, 0x0000},
+	{0x01f3, 0x0000},
+	{0x01f4, 0x0000},
+	{0x0200, 0x0000},
+	{0x0201, 0x0000},
+	{0x0202, 0x0000},
+	{0x0203, 0x0000},
+	{0x0204, 0x0000},
+	{0x0205, 0x0000},
+	{0x0206, 0x0000},
+	{0x0207, 0x0000},
+	{0x0208, 0x0000},
+	{0x0210, 0x60b1},
+	{0x0211, 0xa005},
+	{0x0212, 0x024c},
+	{0x0213, 0xf7ff},
+	{0x0214, 0x024c},
+	{0x0215, 0x0102},
+	{0x0216, 0x00a3},
+	{0x0217, 0x0048},
+	{0x0218, 0xa2c0},
+	{0x0219, 0x0400},
+	{0x021a, 0x00c8},
+	{0x021b, 0x00c0},
+	{0x02ff, 0x0110},
+	{0x0300, 0x001f},
+	{0x0301, 0x032c},
+	{0x0302, 0x5f21},
+	{0x0303, 0x4000},
+	{0x0304, 0x4000},
+	{0x0305, 0x06d5},
+	{0x0306, 0x8000},
+	{0x0307, 0x0700},
+	{0x0310, 0x4560},
+	{0x0311, 0xa4a8},
+	{0x0312, 0x7418},
+	{0x0313, 0x0000},
+	{0x0314, 0x0006},
+	{0x0315, 0xffff},
+	{0x0316, 0xc400},
+	{0x0317, 0x0000},
+	{0x0330, 0x00a6},
+	{0x0331, 0x04c3},
+	{0x0332, 0x27c8},
+	{0x0333, 0xbf50},
+	{0x0334, 0x0045},
+	{0x0335, 0x0007},
+	{0x0336, 0x7418},
+	{0x0337, 0x0501},
+	{0x0338, 0x0000},
+	{0x0339, 0x0010},
+	{0x033a, 0x1010},
+	{0x03c0, 0x7e00},
+	{0x03c1, 0x8000},
+	{0x03c2, 0x8000},
+	{0x03c3, 0x8000},
+	{0x03c4, 0x8000},
+	{0x03c5, 0x8000},
+	{0x03c6, 0x8000},
+	{0x03c7, 0x8000},
+	{0x03c8, 0x8000},
+	{0x03c9, 0x8000},
+	{0x03ca, 0x8000},
+	{0x03cb, 0x8000},
+	{0x03cc, 0x8000},
+	{0x03d0, 0x0000},
+	{0x03d1, 0x0000},
+	{0x03d2, 0x0000},
+	{0x03d3, 0x0000},
+	{0x03d4, 0x2000},
+	{0x03d5, 0x2000},
+	{0x03d6, 0x0000},
+	{0x03d7, 0x0000},
+	{0x03d8, 0x2000},
+	{0x03d9, 0x2000},
+	{0x03da, 0x2000},
+	{0x03db, 0x2000},
+	{0x03dc, 0x0000},
+	{0x03dd, 0x0000},
+	{0x03de, 0x0000},
+	{0x03df, 0x2000},
+	{0x03e0, 0x0000},
+	{0x03e1, 0x0000},
+	{0x03e2, 0x0000},
+	{0x03e3, 0x0000},
+	{0x03e4, 0x0000},
+	{0x03e5, 0x0000},
+	{0x03e6, 0x0000},
+	{0x03e7, 0x0000},
+	{0x03e8, 0x0000},
+	{0x03e9, 0x0000},
+	{0x03ea, 0x0000},
+	{0x03eb, 0x0000},
+	{0x03ec, 0x0000},
+	{0x03ed, 0x0000},
+	{0x03ee, 0x0000},
+	{0x03ef, 0x0000},
+	{0x03f0, 0x0800},
+	{0x03f1, 0x0800},
+	{0x03f2, 0x0800},
+	{0x03f3, 0x0800},
+};
+
+static bool rt5665_volatile_register(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case RT5665_RESET:
+	case RT5665_EJD_CTRL_2:
+	case RT5665_GPIO_STA:
+	case RT5665_INT_ST_1:
+	case RT5665_IL_CMD_1:
+	case RT5665_4BTN_IL_CMD_1:
+	case RT5665_PSV_IL_CMD_1:
+	case RT5665_AJD1_CTRL:
+	case RT5665_JD_CTRL_3:
+	case RT5665_STO_NG2_CTRL_1:
+	case RT5665_SAR_IL_CMD_4:
+	case RT5665_DEVICE_ID:
+	case RT5665_STO1_DAC_SIL_DET ... RT5665_STO2_DAC_SIL_DET:
+	case RT5665_MONO_AMP_CALIB_STA1 ... RT5665_MONO_AMP_CALIB_STA6:
+	case RT5665_HP_IMP_SENS_CTRL_12 ... RT5665_HP_IMP_SENS_CTRL_15:
+	case RT5665_HP_CALIB_STA_1 ... RT5665_HP_CALIB_STA_11:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool rt5665_readable_register(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case RT5665_RESET:
+	case RT5665_VENDOR_ID:
+	case RT5665_VENDOR_ID_1:
+	case RT5665_DEVICE_ID:
+	case RT5665_LOUT:
+	case RT5665_HP_CTRL_1:
+	case RT5665_HP_CTRL_2:
+	case RT5665_MONO_OUT:
+	case RT5665_HPL_GAIN:
+	case RT5665_HPR_GAIN:
+	case RT5665_MONO_GAIN:
+	case RT5665_CAL_BST_CTRL:
+	case RT5665_CBJ_BST_CTRL:
+	case RT5665_IN1_IN2:
+	case RT5665_IN3_IN4:
+	case RT5665_INL1_INR1_VOL:
+	case RT5665_EJD_CTRL_1:
+	case RT5665_EJD_CTRL_2:
+	case RT5665_EJD_CTRL_3:
+	case RT5665_EJD_CTRL_4:
+	case RT5665_EJD_CTRL_5:
+	case RT5665_EJD_CTRL_6:
+	case RT5665_EJD_CTRL_7:
+	case RT5665_DAC2_CTRL:
+	case RT5665_DAC2_DIG_VOL:
+	case RT5665_DAC1_DIG_VOL:
+	case RT5665_DAC3_DIG_VOL:
+	case RT5665_DAC3_CTRL:
+	case RT5665_STO1_ADC_DIG_VOL:
+	case RT5665_MONO_ADC_DIG_VOL:
+	case RT5665_STO2_ADC_DIG_VOL:
+	case RT5665_STO1_ADC_BOOST:
+	case RT5665_MONO_ADC_BOOST:
+	case RT5665_STO2_ADC_BOOST:
+	case RT5665_HP_IMP_GAIN_1:
+	case RT5665_HP_IMP_GAIN_2:
+	case RT5665_STO1_ADC_MIXER:
+	case RT5665_MONO_ADC_MIXER:
+	case RT5665_STO2_ADC_MIXER:
+	case RT5665_AD_DA_MIXER:
+	case RT5665_STO1_DAC_MIXER:
+	case RT5665_MONO_DAC_MIXER:
+	case RT5665_STO2_DAC_MIXER:
+	case RT5665_A_DAC1_MUX:
+	case RT5665_A_DAC2_MUX:
+	case RT5665_DIG_INF2_DATA:
+	case RT5665_DIG_INF3_DATA:
+	case RT5665_PDM_OUT_CTRL:
+	case RT5665_PDM_DATA_CTRL_1:
+	case RT5665_PDM_DATA_CTRL_2:
+	case RT5665_PDM_DATA_CTRL_3:
+	case RT5665_PDM_DATA_CTRL_4:
+	case RT5665_REC1_GAIN:
+	case RT5665_REC1_L1_MIXER:
+	case RT5665_REC1_L2_MIXER:
+	case RT5665_REC1_R1_MIXER:
+	case RT5665_REC1_R2_MIXER:
+	case RT5665_REC2_GAIN:
+	case RT5665_REC2_L1_MIXER:
+	case RT5665_REC2_L2_MIXER:
+	case RT5665_REC2_R1_MIXER:
+	case RT5665_REC2_R2_MIXER:
+	case RT5665_CAL_REC:
+	case RT5665_ALC_BACK_GAIN:
+	case RT5665_MONOMIX_GAIN:
+	case RT5665_MONOMIX_IN_GAIN:
+	case RT5665_OUT_L_GAIN:
+	case RT5665_OUT_L_MIXER:
+	case RT5665_OUT_R_GAIN:
+	case RT5665_OUT_R_MIXER:
+	case RT5665_LOUT_MIXER:
+	case RT5665_PWR_DIG_1:
+	case RT5665_PWR_DIG_2:
+	case RT5665_PWR_ANLG_1:
+	case RT5665_PWR_ANLG_2:
+	case RT5665_PWR_ANLG_3:
+	case RT5665_PWR_MIXER:
+	case RT5665_PWR_VOL:
+	case RT5665_CLK_DET:
+	case RT5665_HPF_CTRL1:
+	case RT5665_DMIC_CTRL_1:
+	case RT5665_DMIC_CTRL_2:
+	case RT5665_I2S1_SDP:
+	case RT5665_I2S2_SDP:
+	case RT5665_I2S3_SDP:
+	case RT5665_ADDA_CLK_1:
+	case RT5665_ADDA_CLK_2:
+	case RT5665_I2S1_F_DIV_CTRL_1:
+	case RT5665_I2S1_F_DIV_CTRL_2:
+	case RT5665_TDM_CTRL_1:
+	case RT5665_TDM_CTRL_2:
+	case RT5665_TDM_CTRL_3:
+	case RT5665_TDM_CTRL_4:
+	case RT5665_TDM_CTRL_5:
+	case RT5665_TDM_CTRL_6:
+	case RT5665_TDM_CTRL_7:
+	case RT5665_TDM_CTRL_8:
+	case RT5665_GLB_CLK:
+	case RT5665_PLL_CTRL_1:
+	case RT5665_PLL_CTRL_2:
+	case RT5665_ASRC_1:
+	case RT5665_ASRC_2:
+	case RT5665_ASRC_3:
+	case RT5665_ASRC_4:
+	case RT5665_ASRC_5:
+	case RT5665_ASRC_6:
+	case RT5665_ASRC_7:
+	case RT5665_ASRC_8:
+	case RT5665_ASRC_9:
+	case RT5665_ASRC_10:
+	case RT5665_DEPOP_1:
+	case RT5665_DEPOP_2:
+	case RT5665_HP_CHARGE_PUMP_1:
+	case RT5665_HP_CHARGE_PUMP_2:
+	case RT5665_MICBIAS_1:
+	case RT5665_MICBIAS_2:
+	case RT5665_ASRC_12:
+	case RT5665_ASRC_13:
+	case RT5665_ASRC_14:
+	case RT5665_RC_CLK_CTRL:
+	case RT5665_I2S_M_CLK_CTRL_1:
+	case RT5665_I2S2_F_DIV_CTRL_1:
+	case RT5665_I2S2_F_DIV_CTRL_2:
+	case RT5665_I2S3_F_DIV_CTRL_1:
+	case RT5665_I2S3_F_DIV_CTRL_2:
+	case RT5665_EQ_CTRL_1:
+	case RT5665_EQ_CTRL_2:
+	case RT5665_IRQ_CTRL_1:
+	case RT5665_IRQ_CTRL_2:
+	case RT5665_IRQ_CTRL_3:
+	case RT5665_IRQ_CTRL_4:
+	case RT5665_IRQ_CTRL_5:
+	case RT5665_IRQ_CTRL_6:
+	case RT5665_INT_ST_1:
+	case RT5665_GPIO_CTRL_1:
+	case RT5665_GPIO_CTRL_2:
+	case RT5665_GPIO_CTRL_3:
+	case RT5665_GPIO_CTRL_4:
+	case RT5665_GPIO_STA:
+	case RT5665_HP_AMP_DET_CTRL_1:
+	case RT5665_HP_AMP_DET_CTRL_2:
+	case RT5665_MID_HP_AMP_DET:
+	case RT5665_LOW_HP_AMP_DET:
+	case RT5665_SV_ZCD_1:
+	case RT5665_SV_ZCD_2:
+	case RT5665_IL_CMD_1:
+	case RT5665_IL_CMD_2:
+	case RT5665_IL_CMD_3:
+	case RT5665_IL_CMD_4:
+	case RT5665_4BTN_IL_CMD_1:
+	case RT5665_4BTN_IL_CMD_2:
+	case RT5665_4BTN_IL_CMD_3:
+	case RT5665_PSV_IL_CMD_1:
+	case RT5665_ADC_STO1_HP_CTRL_1:
+	case RT5665_ADC_STO1_HP_CTRL_2:
+	case RT5665_ADC_MONO_HP_CTRL_1:
+	case RT5665_ADC_MONO_HP_CTRL_2:
+	case RT5665_ADC_STO2_HP_CTRL_1:
+	case RT5665_ADC_STO2_HP_CTRL_2:
+	case RT5665_AJD1_CTRL:
+	case RT5665_JD1_THD:
+	case RT5665_JD2_THD:
+	case RT5665_JD_CTRL_1:
+	case RT5665_JD_CTRL_2:
+	case RT5665_JD_CTRL_3:
+	case RT5665_DIG_MISC:
+	case RT5665_DUMMY_2:
+	case RT5665_DUMMY_3:
+	case RT5665_DAC_ADC_DIG_VOL1:
+	case RT5665_DAC_ADC_DIG_VOL2:
+	case RT5665_BIAS_CUR_CTRL_1:
+	case RT5665_BIAS_CUR_CTRL_2:
+	case RT5665_BIAS_CUR_CTRL_3:
+	case RT5665_BIAS_CUR_CTRL_4:
+	case RT5665_BIAS_CUR_CTRL_5:
+	case RT5665_BIAS_CUR_CTRL_6:
+	case RT5665_BIAS_CUR_CTRL_7:
+	case RT5665_BIAS_CUR_CTRL_8:
+	case RT5665_BIAS_CUR_CTRL_9:
+	case RT5665_BIAS_CUR_CTRL_10:
+	case RT5665_VREF_REC_OP_FB_CAP_CTRL:
+	case RT5665_CHARGE_PUMP_1:
+	case RT5665_DIG_IN_CTRL_1:
+	case RT5665_DIG_IN_CTRL_2:
+	case RT5665_PAD_DRIVING_CTRL:
+	case RT5665_SOFT_RAMP_DEPOP:
+	case RT5665_PLL:
+	case RT5665_CHOP_DAC:
+	case RT5665_CHOP_ADC:
+	case RT5665_CALIB_ADC_CTRL:
+	case RT5665_VOL_TEST:
+	case RT5665_TEST_MODE_CTRL_1:
+	case RT5665_TEST_MODE_CTRL_2:
+	case RT5665_TEST_MODE_CTRL_3:
+	case RT5665_TEST_MODE_CTRL_4:
+	case RT5665_BASSBACK_CTRL:
+	case RT5665_STO_NG2_CTRL_1:
+	case RT5665_STO_NG2_CTRL_2:
+	case RT5665_STO_NG2_CTRL_3:
+	case RT5665_STO_NG2_CTRL_4:
+	case RT5665_STO_NG2_CTRL_5:
+	case RT5665_STO_NG2_CTRL_6:
+	case RT5665_STO_NG2_CTRL_7:
+	case RT5665_STO_NG2_CTRL_8:
+	case RT5665_MONO_NG2_CTRL_1:
+	case RT5665_MONO_NG2_CTRL_2:
+	case RT5665_MONO_NG2_CTRL_3:
+	case RT5665_MONO_NG2_CTRL_4:
+	case RT5665_MONO_NG2_CTRL_5:
+	case RT5665_MONO_NG2_CTRL_6:
+	case RT5665_STO1_DAC_SIL_DET:
+	case RT5665_MONOL_DAC_SIL_DET:
+	case RT5665_MONOR_DAC_SIL_DET:
+	case RT5665_STO2_DAC_SIL_DET:
+	case RT5665_SIL_PSV_CTRL1:
+	case RT5665_SIL_PSV_CTRL2:
+	case RT5665_SIL_PSV_CTRL3:
+	case RT5665_SIL_PSV_CTRL4:
+	case RT5665_SIL_PSV_CTRL5:
+	case RT5665_SIL_PSV_CTRL6:
+	case RT5665_MONO_AMP_CALIB_CTRL_1:
+	case RT5665_MONO_AMP_CALIB_CTRL_2:
+	case RT5665_MONO_AMP_CALIB_CTRL_3:
+	case RT5665_MONO_AMP_CALIB_CTRL_4:
+	case RT5665_MONO_AMP_CALIB_CTRL_5:
+	case RT5665_MONO_AMP_CALIB_CTRL_6:
+	case RT5665_MONO_AMP_CALIB_CTRL_7:
+	case RT5665_MONO_AMP_CALIB_STA1:
+	case RT5665_MONO_AMP_CALIB_STA2:
+	case RT5665_MONO_AMP_CALIB_STA3:
+	case RT5665_MONO_AMP_CALIB_STA4:
+	case RT5665_MONO_AMP_CALIB_STA6:
+	case RT5665_HP_IMP_SENS_CTRL_01:
+	case RT5665_HP_IMP_SENS_CTRL_02:
+	case RT5665_HP_IMP_SENS_CTRL_03:
+	case RT5665_HP_IMP_SENS_CTRL_04:
+	case RT5665_HP_IMP_SENS_CTRL_05:
+	case RT5665_HP_IMP_SENS_CTRL_06:
+	case RT5665_HP_IMP_SENS_CTRL_07:
+	case RT5665_HP_IMP_SENS_CTRL_08:
+	case RT5665_HP_IMP_SENS_CTRL_09:
+	case RT5665_HP_IMP_SENS_CTRL_10:
+	case RT5665_HP_IMP_SENS_CTRL_11:
+	case RT5665_HP_IMP_SENS_CTRL_12:
+	case RT5665_HP_IMP_SENS_CTRL_13:
+	case RT5665_HP_IMP_SENS_CTRL_14:
+	case RT5665_HP_IMP_SENS_CTRL_15:
+	case RT5665_HP_IMP_SENS_CTRL_16:
+	case RT5665_HP_IMP_SENS_CTRL_17:
+	case RT5665_HP_IMP_SENS_CTRL_18:
+	case RT5665_HP_IMP_SENS_CTRL_19:
+	case RT5665_HP_IMP_SENS_CTRL_20:
+	case RT5665_HP_IMP_SENS_CTRL_21:
+	case RT5665_HP_IMP_SENS_CTRL_22:
+	case RT5665_HP_IMP_SENS_CTRL_23:
+	case RT5665_HP_IMP_SENS_CTRL_24:
+	case RT5665_HP_IMP_SENS_CTRL_25:
+	case RT5665_HP_IMP_SENS_CTRL_26:
+	case RT5665_HP_IMP_SENS_CTRL_27:
+	case RT5665_HP_IMP_SENS_CTRL_28:
+	case RT5665_HP_IMP_SENS_CTRL_29:
+	case RT5665_HP_IMP_SENS_CTRL_30:
+	case RT5665_HP_IMP_SENS_CTRL_31:
+	case RT5665_HP_IMP_SENS_CTRL_32:
+	case RT5665_HP_IMP_SENS_CTRL_33:
+	case RT5665_HP_IMP_SENS_CTRL_34:
+	case RT5665_HP_LOGIC_CTRL_1:
+	case RT5665_HP_LOGIC_CTRL_2:
+	case RT5665_HP_LOGIC_CTRL_3:
+	case RT5665_HP_CALIB_CTRL_1:
+	case RT5665_HP_CALIB_CTRL_2:
+	case RT5665_HP_CALIB_CTRL_3:
+	case RT5665_HP_CALIB_CTRL_4:
+	case RT5665_HP_CALIB_CTRL_5:
+	case RT5665_HP_CALIB_CTRL_6:
+	case RT5665_HP_CALIB_CTRL_7:
+	case RT5665_HP_CALIB_CTRL_9:
+	case RT5665_HP_CALIB_CTRL_10:
+	case RT5665_HP_CALIB_CTRL_11:
+	case RT5665_HP_CALIB_STA_1:
+	case RT5665_HP_CALIB_STA_2:
+	case RT5665_HP_CALIB_STA_3:
+	case RT5665_HP_CALIB_STA_4:
+	case RT5665_HP_CALIB_STA_5:
+	case RT5665_HP_CALIB_STA_6:
+	case RT5665_HP_CALIB_STA_7:
+	case RT5665_HP_CALIB_STA_8:
+	case RT5665_HP_CALIB_STA_9:
+	case RT5665_HP_CALIB_STA_10:
+	case RT5665_HP_CALIB_STA_11:
+	case RT5665_PGM_TAB_CTRL1:
+	case RT5665_PGM_TAB_CTRL2:
+	case RT5665_PGM_TAB_CTRL3:
+	case RT5665_PGM_TAB_CTRL4:
+	case RT5665_PGM_TAB_CTRL5:
+	case RT5665_PGM_TAB_CTRL6:
+	case RT5665_PGM_TAB_CTRL7:
+	case RT5665_PGM_TAB_CTRL8:
+	case RT5665_PGM_TAB_CTRL9:
+	case RT5665_SAR_IL_CMD_1:
+	case RT5665_SAR_IL_CMD_2:
+	case RT5665_SAR_IL_CMD_3:
+	case RT5665_SAR_IL_CMD_4:
+	case RT5665_SAR_IL_CMD_5:
+	case RT5665_SAR_IL_CMD_6:
+	case RT5665_SAR_IL_CMD_7:
+	case RT5665_SAR_IL_CMD_8:
+	case RT5665_SAR_IL_CMD_9:
+	case RT5665_SAR_IL_CMD_10:
+	case RT5665_SAR_IL_CMD_11:
+	case RT5665_SAR_IL_CMD_12:
+	case RT5665_DRC1_CTRL_0:
+	case RT5665_DRC1_CTRL_1:
+	case RT5665_DRC1_CTRL_2:
+	case RT5665_DRC1_CTRL_3:
+	case RT5665_DRC1_CTRL_4:
+	case RT5665_DRC1_CTRL_5:
+	case RT5665_DRC1_CTRL_6:
+	case RT5665_DRC1_HARD_LMT_CTRL_1:
+	case RT5665_DRC1_HARD_LMT_CTRL_2:
+	case RT5665_DRC1_PRIV_1:
+	case RT5665_DRC1_PRIV_2:
+	case RT5665_DRC1_PRIV_3:
+	case RT5665_DRC1_PRIV_4:
+	case RT5665_DRC1_PRIV_5:
+	case RT5665_DRC1_PRIV_6:
+	case RT5665_DRC1_PRIV_7:
+	case RT5665_DRC1_PRIV_8:
+	case RT5665_ALC_PGA_CTRL_1:
+	case RT5665_ALC_PGA_CTRL_2:
+	case RT5665_ALC_PGA_CTRL_3:
+	case RT5665_ALC_PGA_CTRL_4:
+	case RT5665_ALC_PGA_CTRL_5:
+	case RT5665_ALC_PGA_CTRL_6:
+	case RT5665_ALC_PGA_CTRL_7:
+	case RT5665_ALC_PGA_CTRL_8:
+	case RT5665_ALC_PGA_STA_1:
+	case RT5665_ALC_PGA_STA_2:
+	case RT5665_ALC_PGA_STA_3:
+	case RT5665_EQ_AUTO_RCV_CTRL1:
+	case RT5665_EQ_AUTO_RCV_CTRL2:
+	case RT5665_EQ_AUTO_RCV_CTRL3:
+	case RT5665_EQ_AUTO_RCV_CTRL4:
+	case RT5665_EQ_AUTO_RCV_CTRL5:
+	case RT5665_EQ_AUTO_RCV_CTRL6:
+	case RT5665_EQ_AUTO_RCV_CTRL7:
+	case RT5665_EQ_AUTO_RCV_CTRL8:
+	case RT5665_EQ_AUTO_RCV_CTRL9:
+	case RT5665_EQ_AUTO_RCV_CTRL10:
+	case RT5665_EQ_AUTO_RCV_CTRL11:
+	case RT5665_EQ_AUTO_RCV_CTRL12:
+	case RT5665_EQ_AUTO_RCV_CTRL13:
+	case RT5665_ADC_L_EQ_LPF1_A1:
+	case RT5665_R_EQ_LPF1_A1:
+	case RT5665_L_EQ_LPF1_H0:
+	case RT5665_R_EQ_LPF1_H0:
+	case RT5665_L_EQ_BPF1_A1:
+	case RT5665_R_EQ_BPF1_A1:
+	case RT5665_L_EQ_BPF1_A2:
+	case RT5665_R_EQ_BPF1_A2:
+	case RT5665_L_EQ_BPF1_H0:
+	case RT5665_R_EQ_BPF1_H0:
+	case RT5665_L_EQ_BPF2_A1:
+	case RT5665_R_EQ_BPF2_A1:
+	case RT5665_L_EQ_BPF2_A2:
+	case RT5665_R_EQ_BPF2_A2:
+	case RT5665_L_EQ_BPF2_H0:
+	case RT5665_R_EQ_BPF2_H0:
+	case RT5665_L_EQ_BPF3_A1:
+	case RT5665_R_EQ_BPF3_A1:
+	case RT5665_L_EQ_BPF3_A2:
+	case RT5665_R_EQ_BPF3_A2:
+	case RT5665_L_EQ_BPF3_H0:
+	case RT5665_R_EQ_BPF3_H0:
+	case RT5665_L_EQ_BPF4_A1:
+	case RT5665_R_EQ_BPF4_A1:
+	case RT5665_L_EQ_BPF4_A2:
+	case RT5665_R_EQ_BPF4_A2:
+	case RT5665_L_EQ_BPF4_H0:
+	case RT5665_R_EQ_BPF4_H0:
+	case RT5665_L_EQ_HPF1_A1:
+	case RT5665_R_EQ_HPF1_A1:
+	case RT5665_L_EQ_HPF1_H0:
+	case RT5665_R_EQ_HPF1_H0:
+	case RT5665_L_EQ_PRE_VOL:
+	case RT5665_R_EQ_PRE_VOL:
+	case RT5665_L_EQ_POST_VOL:
+	case RT5665_R_EQ_POST_VOL:
+	case RT5665_SCAN_MODE_CTRL:
+	case RT5665_I2C_MODE:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const DECLARE_TLV_DB_SCALE(hp_vol_tlv, -2250, 150, 0);
+static const DECLARE_TLV_DB_SCALE(mono_vol_tlv, -1400, 150, 0);
+static const DECLARE_TLV_DB_SCALE(out_vol_tlv, -4650, 150, 0);
+static const DECLARE_TLV_DB_SCALE(dac_vol_tlv, -65625, 375, 0);
+static const DECLARE_TLV_DB_SCALE(in_vol_tlv, -3450, 150, 0);
+static const DECLARE_TLV_DB_SCALE(adc_vol_tlv, -17625, 375, 0);
+static const DECLARE_TLV_DB_SCALE(adc_bst_tlv, 0, 1200, 0);
+static const DECLARE_TLV_DB_SCALE(in_bst_tlv, -1200, 75, 0);
+
+/* {0, +20, +24, +30, +35, +40, +44, +50, +52} dB */
+static const DECLARE_TLV_DB_RANGE(bst_tlv,
+	0, 0, TLV_DB_SCALE_ITEM(0, 0, 0),
+	1, 1, TLV_DB_SCALE_ITEM(2000, 0, 0),
+	2, 2, TLV_DB_SCALE_ITEM(2400, 0, 0),
+	3, 5, TLV_DB_SCALE_ITEM(3000, 500, 0),
+	6, 6, TLV_DB_SCALE_ITEM(4400, 0, 0),
+	7, 7, TLV_DB_SCALE_ITEM(5000, 0, 0),
+	8, 8, TLV_DB_SCALE_ITEM(5200, 0, 0)
+);
+
+/* Interface data select */
+static const char * const rt5665_data_select[] = {
+	"L/R", "R/L", "L/L", "R/R"
+};
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if1_1_01_adc_enum,
+	RT5665_TDM_CTRL_2, RT5665_I2S1_1_DS_ADC_SLOT01_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if1_1_23_adc_enum,
+	RT5665_TDM_CTRL_2, RT5665_I2S1_1_DS_ADC_SLOT23_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if1_1_45_adc_enum,
+	RT5665_TDM_CTRL_2, RT5665_I2S1_1_DS_ADC_SLOT45_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if1_1_67_adc_enum,
+	RT5665_TDM_CTRL_2, RT5665_I2S1_1_DS_ADC_SLOT67_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if1_2_01_adc_enum,
+	RT5665_TDM_CTRL_2, RT5665_I2S1_2_DS_ADC_SLOT01_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if1_2_23_adc_enum,
+	RT5665_TDM_CTRL_2, RT5665_I2S1_2_DS_ADC_SLOT23_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if1_2_45_adc_enum,
+	RT5665_TDM_CTRL_2, RT5665_I2S1_2_DS_ADC_SLOT45_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if1_2_67_adc_enum,
+	RT5665_TDM_CTRL_2, RT5665_I2S1_2_DS_ADC_SLOT67_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if2_1_dac_enum,
+	RT5665_DIG_INF2_DATA, RT5665_IF2_1_DAC_SEL_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if2_1_adc_enum,
+	RT5665_DIG_INF2_DATA, RT5665_IF2_1_ADC_SEL_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if2_2_dac_enum,
+	RT5665_DIG_INF2_DATA, RT5665_IF2_2_DAC_SEL_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if2_2_adc_enum,
+	RT5665_DIG_INF2_DATA, RT5665_IF2_2_ADC_SEL_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if3_dac_enum,
+	RT5665_DIG_INF3_DATA, RT5665_IF3_DAC_SEL_SFT, rt5665_data_select);
+
+static const SOC_ENUM_SINGLE_DECL(rt5665_if3_adc_enum,
+	RT5665_DIG_INF3_DATA, RT5665_IF3_ADC_SEL_SFT, rt5665_data_select);
+
+static const struct snd_kcontrol_new rt5665_if1_1_01_adc_swap_mux =
+	SOC_DAPM_ENUM("IF1_1 01 ADC Swap Mux", rt5665_if1_1_01_adc_enum);
+
+static const struct snd_kcontrol_new rt5665_if1_1_23_adc_swap_mux =
+	SOC_DAPM_ENUM("IF1_1 23 ADC Swap Mux", rt5665_if1_1_23_adc_enum);
+
+static const struct snd_kcontrol_new rt5665_if1_1_45_adc_swap_mux =
+	SOC_DAPM_ENUM("IF1_1 45 ADC Swap Mux", rt5665_if1_1_45_adc_enum);
+
+static const struct snd_kcontrol_new rt5665_if1_1_67_adc_swap_mux =
+	SOC_DAPM_ENUM("IF1_1 67 ADC Swap Mux", rt5665_if1_1_67_adc_enum);
+
+static const struct snd_kcontrol_new rt5665_if1_2_01_adc_swap_mux =
+	SOC_DAPM_ENUM("IF1_2 01 ADC Swap Mux", rt5665_if1_2_01_adc_enum);
+
+static const struct snd_kcontrol_new rt5665_if1_2_23_adc_swap_mux =
+	SOC_DAPM_ENUM("IF1_2 23 ADC1 Swap Mux", rt5665_if1_2_23_adc_enum);
+
+static const struct snd_kcontrol_new rt5665_if1_2_45_adc_swap_mux =
+	SOC_DAPM_ENUM("IF1_2 45 ADC1 Swap Mux", rt5665_if1_2_45_adc_enum);
+
+static const struct snd_kcontrol_new rt5665_if1_2_67_adc_swap_mux =
+	SOC_DAPM_ENUM("IF1_2 67 ADC1 Swap Mux", rt5665_if1_2_67_adc_enum);
+
+static const struct snd_kcontrol_new rt5665_if2_1_dac_swap_mux =
+	SOC_DAPM_ENUM("IF2_1 DAC Swap Source", rt5665_if2_1_dac_enum);
+
+static const struct snd_kcontrol_new rt5665_if2_1_adc_swap_mux =
+	SOC_DAPM_ENUM("IF2_1 ADC Swap Source", rt5665_if2_1_adc_enum);
+
+static const struct snd_kcontrol_new rt5665_if2_2_dac_swap_mux =
+	SOC_DAPM_ENUM("IF2_2 DAC Swap Source", rt5665_if2_2_dac_enum);
+
+static const struct snd_kcontrol_new rt5665_if2_2_adc_swap_mux =
+	SOC_DAPM_ENUM("IF2_2 ADC Swap Source", rt5665_if2_2_adc_enum);
+
+static const struct snd_kcontrol_new rt5665_if3_dac_swap_mux =
+	SOC_DAPM_ENUM("IF3 DAC Swap Source", rt5665_if3_dac_enum);
+
+static const struct snd_kcontrol_new rt5665_if3_adc_swap_mux =
+	SOC_DAPM_ENUM("IF3 ADC Swap Source", rt5665_if3_adc_enum);
+
+static int rt5665_hp_vol_put(struct snd_kcontrol *kcontrol,
+		struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_codec *codec = snd_soc_kcontrol_codec(kcontrol);
+	int ret = snd_soc_put_volsw(kcontrol, ucontrol);
+
+	if (snd_soc_read(codec, RT5665_STO_NG2_CTRL_1) & RT5665_NG2_EN) {
+		snd_soc_update_bits(codec, RT5665_STO_NG2_CTRL_1,
+			RT5665_NG2_EN_MASK, RT5665_NG2_DIS);
+		snd_soc_update_bits(codec, RT5665_STO_NG2_CTRL_1,
+			RT5665_NG2_EN_MASK, RT5665_NG2_EN);
+	}
+
+	return ret;
+}
+
+static int rt5665_mono_vol_put(struct snd_kcontrol *kcontrol,
+		struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_codec *codec = snd_soc_kcontrol_codec(kcontrol);
+	int ret = snd_soc_put_volsw(kcontrol, ucontrol);
+
+	if (snd_soc_read(codec, RT5665_MONO_NG2_CTRL_1) & RT5665_NG2_EN) {
+		snd_soc_update_bits(codec, RT5665_MONO_NG2_CTRL_1,
+			RT5665_NG2_EN_MASK, RT5665_NG2_DIS);
+		snd_soc_update_bits(codec, RT5665_MONO_NG2_CTRL_1,
+			RT5665_NG2_EN_MASK, RT5665_NG2_EN);
+	}
+
+	return ret;
+}
+
+/**
+ * rt5665_sel_asrc_clk_src - select ASRC clock source for a set of filters
+ * @codec: SoC audio codec device.
+ * @filter_mask: mask of filters.
+ * @clk_src: clock source
+ *
+ * The ASRC function is for asynchronous MCLK and LRCK. Also, since RT5665 can
+ * only support standard 32fs or 64fs i2s format, ASRC should be enabled to
+ * support special i2s clock format such as Intel's 100fs(100 * sampling rate).
+ * ASRC function will track i2s clock and generate a corresponding system clock
+ * for codec. This function provides an API to select the clock source for a
+ * set of filters specified by the mask. And the codec driver will turn on ASRC
+ * for these filters if ASRC is selected as their clock source.
+ */
+int rt5665_sel_asrc_clk_src(struct snd_soc_codec *codec,
+		unsigned int filter_mask, unsigned int clk_src)
+{
+	unsigned int asrc2_mask = 0;
+	unsigned int asrc2_value = 0;
+	unsigned int asrc3_mask = 0;
+	unsigned int asrc3_value = 0;
+
+	switch (clk_src) {
+	case RT5665_CLK_SEL_SYS:
+	case RT5665_CLK_SEL_I2S1_ASRC:
+	case RT5665_CLK_SEL_I2S2_ASRC:
+	case RT5665_CLK_SEL_I2S3_ASRC:
+	case RT5665_CLK_SEL_SYS2:
+	case RT5665_CLK_SEL_SYS3:
+	case RT5665_CLK_SEL_SYS4:
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	if (filter_mask & RT5665_DA_STEREO1_FILTER) {
+		asrc2_mask |= RT5665_DA_STO1_CLK_SEL_MASK;
+		asrc2_value = (asrc2_value & ~RT5665_DA_STO1_CLK_SEL_MASK)
+			| (clk_src << RT5665_DA_STO1_CLK_SEL_SFT);
+	}
+
+	if (filter_mask & RT5665_DA_STEREO2_FILTER) {
+		asrc2_mask |= RT5665_DA_STO2_CLK_SEL_MASK;
+		asrc2_value = (asrc2_value & ~RT5665_DA_STO2_CLK_SEL_MASK)
+			| (clk_src << RT5665_DA_STO2_CLK_SEL_SFT);
+	}
+
+	if (filter_mask & RT5665_DA_MONO_L_FILTER) {
+		asrc2_mask |= RT5665_DA_MONOL_CLK_SEL_MASK;
+		asrc2_value = (asrc2_value & ~RT5665_DA_MONOL_CLK_SEL_MASK)
+			| (clk_src << RT5665_DA_MONOL_CLK_SEL_SFT);
+	}
+
+	if (filter_mask & RT5665_DA_MONO_R_FILTER) {
+		asrc2_mask |= RT5665_DA_MONOR_CLK_SEL_MASK;
+		asrc2_value = (asrc2_value & ~RT5665_DA_MONOR_CLK_SEL_MASK)
+			| (clk_src << RT5665_DA_MONOR_CLK_SEL_SFT);
+	}
+
+	if (filter_mask & RT5665_AD_STEREO1_FILTER) {
+		asrc3_mask |= RT5665_AD_STO1_CLK_SEL_MASK;
+		asrc3_value = (asrc2_value & ~RT5665_AD_STO1_CLK_SEL_MASK)
+			| (clk_src << RT5665_AD_STO1_CLK_SEL_SFT);
+	}
+
+	if (filter_mask & RT5665_AD_STEREO2_FILTER) {
+		asrc3_mask |= RT5665_AD_STO2_CLK_SEL_MASK;
+		asrc3_value = (asrc2_value & ~RT5665_AD_STO2_CLK_SEL_MASK)
+			| (clk_src << RT5665_AD_STO2_CLK_SEL_SFT);
+	}
+
+	if (filter_mask & RT5665_AD_MONO_L_FILTER) {
+		asrc3_mask |= RT5665_AD_MONOL_CLK_SEL_MASK;
+		asrc3_value = (asrc3_value & ~RT5665_AD_MONOL_CLK_SEL_MASK)
+			| (clk_src << RT5665_AD_MONOL_CLK_SEL_SFT);
+	}
+
+	if (filter_mask & RT5665_AD_MONO_R_FILTER)  {
+		asrc3_mask |= RT5665_AD_MONOR_CLK_SEL_MASK;
+		asrc3_value = (asrc3_value & ~RT5665_AD_MONOR_CLK_SEL_MASK)
+			| (clk_src << RT5665_AD_MONOR_CLK_SEL_SFT);
+	}
+
+	if (asrc2_mask)
+		snd_soc_update_bits(codec, RT5665_ASRC_2,
+			asrc2_mask, asrc2_value);
+
+	if (asrc3_mask)
+		snd_soc_update_bits(codec, RT5665_ASRC_3,
+			asrc3_mask, asrc3_value);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(rt5665_sel_asrc_clk_src);
+
+static int rt5665_button_detect(struct snd_soc_codec *codec)
+{
+	int btn_type, val;
+
+	val = snd_soc_read(codec, RT5665_4BTN_IL_CMD_1);
+	btn_type = val & 0xfff0;
+	snd_soc_write(codec, RT5665_4BTN_IL_CMD_1, val);
+
+	return btn_type;
+}
+
+static void rt5665_enable_push_button_irq(struct snd_soc_codec *codec,
+	bool enable)
+{
+	if (enable) {
+		snd_soc_write(codec, RT5665_4BTN_IL_CMD_1, 0x000b);
+		snd_soc_write(codec, RT5665_IL_CMD_1, 0x0048);
+		snd_soc_update_bits(codec, RT5665_4BTN_IL_CMD_2,
+				RT5665_4BTN_IL_MASK | RT5665_4BTN_IL_RST_MASK,
+				RT5665_4BTN_IL_EN | RT5665_4BTN_IL_NOR);
+		snd_soc_update_bits(codec, RT5665_IRQ_CTRL_3,
+				RT5665_IL_IRQ_MASK, RT5665_IL_IRQ_EN);
+	} else {
+		snd_soc_update_bits(codec, RT5665_IRQ_CTRL_3,
+				RT5665_IL_IRQ_MASK, RT5665_IL_IRQ_DIS);
+		snd_soc_update_bits(codec, RT5665_4BTN_IL_CMD_2,
+				RT5665_4BTN_IL_MASK, RT5665_4BTN_IL_DIS);
+		snd_soc_update_bits(codec, RT5665_4BTN_IL_CMD_2,
+				RT5665_4BTN_IL_RST_MASK, RT5665_4BTN_IL_RST);
+	}
+}
+
+/**
+ * rt5665_headset_detect - Detect headset.
+ * @codec: SoC audio codec device.
+ * @jack_insert: Jack insert or not.
+ *
+ * Detect whether is headset or not when jack inserted.
+ *
+ * Returns detect status.
+ */
+static int rt5665_headset_detect(struct snd_soc_codec *codec, int jack_insert)
+{
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+	struct snd_soc_dapm_context *dapm = snd_soc_codec_get_dapm(codec);
+	unsigned int sar_hs_type, val;
+
+	if (jack_insert) {
+		snd_soc_dapm_force_enable_pin(dapm, "MICBIAS1");
+		snd_soc_dapm_sync(dapm);
+
+		regmap_update_bits(rt5665->regmap, RT5665_MICBIAS_2, 0x100,
+			0x100);
+
+		regmap_read(rt5665->regmap, RT5665_GPIO_STA, &val);
+		if (val & 0x4) {
+			regmap_update_bits(rt5665->regmap, RT5665_EJD_CTRL_1,
+				0x100, 0);
+
+			regmap_read(rt5665->regmap, RT5665_GPIO_STA, &val);
+			while (val & 0x4) {
+				usleep_range(10000, 15000);
+				regmap_read(rt5665->regmap, RT5665_GPIO_STA,
+					&val);
+			}
+		}
+
+		regmap_update_bits(rt5665->regmap, RT5665_EJD_CTRL_1,
+			0x180, 0x180);
+		regmap_write(rt5665->regmap, RT5665_EJD_CTRL_3, 0x3424);
+		regmap_write(rt5665->regmap, RT5665_SAR_IL_CMD_1, 0xa291);
+
+		rt5665->sar_adc_value = snd_soc_read(rt5665->codec,
+			RT5665_SAR_IL_CMD_4) & 0x7ff;
+
+		sar_hs_type = rt5665->pdata.sar_hs_type ?
+			rt5665->pdata.sar_hs_type : 729;
+
+		if (rt5665->sar_adc_value > sar_hs_type) {
+			rt5665->jack_type = SND_JACK_HEADSET;
+			rt5665_enable_push_button_irq(codec, true);
+			} else {
+			rt5665->jack_type = SND_JACK_HEADPHONE;
+			regmap_write(rt5665->regmap, RT5665_SAR_IL_CMD_1,
+				0x2291);
+			regmap_update_bits(rt5665->regmap, RT5665_MICBIAS_2,
+				0x100, 0);
+			snd_soc_dapm_disable_pin(dapm, "MICBIAS1");
+			snd_soc_dapm_sync(dapm);
+		}
+	} else {
+		regmap_write(rt5665->regmap, RT5665_SAR_IL_CMD_1, 0x2291);
+		regmap_update_bits(rt5665->regmap, RT5665_MICBIAS_2, 0x100, 0);
+		snd_soc_dapm_disable_pin(dapm, "MICBIAS1");
+		snd_soc_dapm_sync(dapm);
+		if (rt5665->jack_type == SND_JACK_HEADSET)
+			rt5665_enable_push_button_irq(codec, false);
+		rt5665->jack_type = 0;
+	}
+
+	dev_dbg(codec->dev, "jack_type = %d\n", rt5665->jack_type);
+	return rt5665->jack_type;
+}
+
+static irqreturn_t rt5665_irq(int irq, void *data)
+{
+	struct rt5665_priv *rt5665 = data;
+
+	mod_delayed_work(system_power_efficient_wq,
+			   &rt5665->jack_detect_work, msecs_to_jiffies(250));
+
+	return IRQ_HANDLED;
+}
+
+static void rt5665_jd_check_handler(struct work_struct *work)
+{
+	struct rt5665_priv *rt5665 = container_of(work, struct rt5665_priv,
+		calibrate_work.work);
+
+	if (snd_soc_read(rt5665->codec, RT5665_AJD1_CTRL) & 0x0010) {
+		/* jack out */
+		rt5665->jack_type = rt5665_headset_detect(rt5665->codec, 0);
+
+		snd_soc_jack_report(rt5665->hs_jack, rt5665->jack_type,
+				SND_JACK_HEADSET |
+				SND_JACK_BTN_0 | SND_JACK_BTN_1 |
+				SND_JACK_BTN_2 | SND_JACK_BTN_3);
+	} else {
+		schedule_delayed_work(&rt5665->jd_check_work, 500);
+	}
+}
+
+int rt5665_set_jack_detect(struct snd_soc_codec *codec,
+	struct snd_soc_jack *hs_jack)
+{
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+
+	switch (rt5665->pdata.jd_src) {
+	case RT5665_JD1:
+		regmap_update_bits(rt5665->regmap, RT5665_GPIO_CTRL_1,
+			RT5665_GP1_PIN_MASK, RT5665_GP1_PIN_IRQ);
+		regmap_update_bits(rt5665->regmap, RT5665_RC_CLK_CTRL,
+				0xc000, 0xc000);
+		regmap_update_bits(rt5665->regmap, RT5665_PWR_ANLG_2,
+			RT5665_PWR_JD1, RT5665_PWR_JD1);
+		regmap_update_bits(rt5665->regmap, RT5665_IRQ_CTRL_1, 0x8, 0x8);
+		break;
+
+	case RT5665_JD_NULL:
+		break;
+
+	default:
+		dev_warn(codec->dev, "Wrong JD source\n");
+		break;
+	}
+
+	rt5665->hs_jack = hs_jack;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(rt5665_set_jack_detect);
+
+static void rt5665_jack_detect_handler(struct work_struct *work)
+{
+	struct rt5665_priv *rt5665 =
+		container_of(work, struct rt5665_priv, jack_detect_work.work);
+	int val, btn_type;
+
+	while (!rt5665->codec) {
+		pr_debug("%s codec = null\n", __func__);
+		usleep_range(10000, 15000);
+	}
+
+	while (!rt5665->codec->component.card->instantiated) {
+		pr_debug("%s\n", __func__);
+		usleep_range(10000, 15000);
+	}
+
+	mutex_lock(&rt5665->calibrate_mutex);
+
+	val = snd_soc_read(rt5665->codec, RT5665_AJD1_CTRL) & 0x0010;
+	if (!val) {
+		/* jack in */
+		if (rt5665->jack_type == 0) {
+			/* jack was out, report jack type */
+			rt5665->jack_type =
+				rt5665_headset_detect(rt5665->codec, 1);
+		} else {
+			/* jack is already in, report button event */
+			rt5665->jack_type = SND_JACK_HEADSET;
+			btn_type = rt5665_button_detect(rt5665->codec);
+			/**
+			 * rt5665 can report three kinds of button behavior,
+			 * one click, double click and hold. However,
+			 * currently we will report button pressed/released
+			 * event. So all the three button behaviors are
+			 * treated as button pressed.
+			 */
+			switch (btn_type) {
+			case 0x8000:
+			case 0x4000:
+			case 0x2000:
+				rt5665->jack_type |= SND_JACK_BTN_0;
+				break;
+			case 0x1000:
+			case 0x0800:
+			case 0x0400:
+				rt5665->jack_type |= SND_JACK_BTN_1;
+				break;
+			case 0x0200:
+			case 0x0100:
+			case 0x0080:
+				rt5665->jack_type |= SND_JACK_BTN_2;
+				break;
+			case 0x0040:
+			case 0x0020:
+			case 0x0010:
+				rt5665->jack_type |= SND_JACK_BTN_3;
+				break;
+			case 0x0000: /* unpressed */
+				break;
+			default:
+				btn_type = 0;
+				dev_err(rt5665->codec->dev,
+					"Unexpected button code 0x%04x\n",
+					btn_type);
+				break;
+			}
+		}
+	} else {
+		/* jack out */
+		rt5665->jack_type = rt5665_headset_detect(rt5665->codec, 0);
+	}
+
+	snd_soc_jack_report(rt5665->hs_jack, rt5665->jack_type,
+			SND_JACK_HEADSET |
+			    SND_JACK_BTN_0 | SND_JACK_BTN_1 |
+			    SND_JACK_BTN_2 | SND_JACK_BTN_3);
+
+	if (rt5665->jack_type & (SND_JACK_BTN_0 | SND_JACK_BTN_1 |
+		SND_JACK_BTN_2 | SND_JACK_BTN_3))
+		schedule_delayed_work(&rt5665->jd_check_work, 0);
+	else
+		cancel_delayed_work_sync(&rt5665->jd_check_work);
+
+	mutex_unlock(&rt5665->calibrate_mutex);
+}
+
+static const struct snd_kcontrol_new rt5665_snd_controls[] = {
+	/* Headphone Output Volume */
+	SOC_DOUBLE_R_EXT_TLV("Headphone Playback Volume", RT5665_HPL_GAIN,
+		RT5665_HPR_GAIN, RT5665_G_HP_SFT, 15, 1, snd_soc_get_volsw,
+		rt5665_hp_vol_put, hp_vol_tlv),
+
+	/* Mono Output Volume */
+	SOC_SINGLE_EXT_TLV("Mono Playback Volume", RT5665_MONO_GAIN,
+		RT5665_L_VOL_SFT, 15, 1, snd_soc_get_volsw,
+		rt5665_mono_vol_put, mono_vol_tlv),
+
+	/* Output Volume */
+	SOC_DOUBLE_TLV("OUT Playback Volume", RT5665_LOUT, RT5665_L_VOL_SFT,
+		RT5665_R_VOL_SFT, 39, 1, out_vol_tlv),
+
+	/* DAC Digital Volume */
+	SOC_DOUBLE_TLV("DAC1 Playback Volume", RT5665_DAC1_DIG_VOL,
+		RT5665_L_VOL_SFT, RT5665_R_VOL_SFT, 175, 0, dac_vol_tlv),
+	SOC_DOUBLE_TLV("DAC2 Playback Volume", RT5665_DAC2_DIG_VOL,
+		RT5665_L_VOL_SFT, RT5665_R_VOL_SFT, 175, 0, dac_vol_tlv),
+	SOC_DOUBLE("DAC2 Playback Switch", RT5665_DAC2_CTRL,
+		RT5665_M_DAC2_L_VOL_SFT, RT5665_M_DAC2_R_VOL_SFT, 1, 1),
+
+	/* IN1/IN2/IN3/IN4 Volume */
+	SOC_SINGLE_TLV("IN1 Boost Volume", RT5665_IN1_IN2,
+		RT5665_BST1_SFT, 69, 0, in_bst_tlv),
+	SOC_SINGLE_TLV("IN2 Boost Volume", RT5665_IN1_IN2,
+		RT5665_BST2_SFT, 69, 0, in_bst_tlv),
+	SOC_SINGLE_TLV("IN3 Boost Volume", RT5665_IN3_IN4,
+		RT5665_BST3_SFT, 69, 0, in_bst_tlv),
+	SOC_SINGLE_TLV("IN4 Boost Volume", RT5665_IN3_IN4,
+		RT5665_BST4_SFT, 69, 0, in_bst_tlv),
+	SOC_SINGLE_TLV("CBJ Boost Volume", RT5665_CBJ_BST_CTRL,
+		RT5665_BST_CBJ_SFT, 8, 0, bst_tlv),
+
+	/* INL/INR Volume Control */
+	SOC_DOUBLE_TLV("IN Capture Volume", RT5665_INL1_INR1_VOL,
+		RT5665_INL_VOL_SFT, RT5665_INR_VOL_SFT, 31, 1, in_vol_tlv),
+
+	/* ADC Digital Volume Control */
+	SOC_DOUBLE("STO1 ADC Capture Switch", RT5665_STO1_ADC_DIG_VOL,
+		RT5665_L_MUTE_SFT, RT5665_R_MUTE_SFT, 1, 1),
+	SOC_DOUBLE_TLV("STO1 ADC Capture Volume", RT5665_STO1_ADC_DIG_VOL,
+		RT5665_L_VOL_SFT, RT5665_R_VOL_SFT, 127, 0, adc_vol_tlv),
+	SOC_DOUBLE("Mono ADC Capture Switch", RT5665_MONO_ADC_DIG_VOL,
+		RT5665_L_MUTE_SFT, RT5665_R_MUTE_SFT, 1, 1),
+	SOC_DOUBLE_TLV("Mono ADC Capture Volume", RT5665_MONO_ADC_DIG_VOL,
+		RT5665_L_VOL_SFT, RT5665_R_VOL_SFT, 127, 0, adc_vol_tlv),
+	SOC_DOUBLE("STO2 ADC Capture Switch", RT5665_STO2_ADC_DIG_VOL,
+		RT5665_L_MUTE_SFT, RT5665_R_MUTE_SFT, 1, 1),
+	SOC_DOUBLE_TLV("STO2 ADC Capture Volume", RT5665_STO2_ADC_DIG_VOL,
+		RT5665_L_VOL_SFT, RT5665_R_VOL_SFT, 127, 0, adc_vol_tlv),
+
+	/* ADC Boost Volume Control */
+	SOC_DOUBLE_TLV("STO1 ADC Boost Gain Volume", RT5665_STO1_ADC_BOOST,
+		RT5665_STO1_ADC_L_BST_SFT, RT5665_STO1_ADC_R_BST_SFT,
+		3, 0, adc_bst_tlv),
+
+	SOC_DOUBLE_TLV("Mono ADC Boost Gain Volume", RT5665_MONO_ADC_BOOST,
+		RT5665_MONO_ADC_L_BST_SFT, RT5665_MONO_ADC_R_BST_SFT,
+		3, 0, adc_bst_tlv),
+
+	SOC_DOUBLE_TLV("STO2 ADC Boost Gain Volume", RT5665_STO2_ADC_BOOST,
+		RT5665_STO2_ADC_L_BST_SFT, RT5665_STO2_ADC_R_BST_SFT,
+		3, 0, adc_bst_tlv),
+};
+
+/**
+ * set_dmic_clk - Set parameter of dmic.
+ *
+ * @w: DAPM widget.
+ * @kcontrol: The kcontrol of this widget.
+ * @event: Event id.
+ *
+ * Choose dmic clock between 1MHz and 3MHz.
+ * It is better for clock to approximate 3MHz.
+ */
+static int set_dmic_clk(struct snd_soc_dapm_widget *w,
+	struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+	int pd, idx = -EINVAL;
+
+	pd = rl6231_get_pre_div(rt5665->regmap,
+		RT5665_ADDA_CLK_1, RT5665_I2S_PD1_SFT);
+	idx = rl6231_calc_dmic_clk(rt5665->sysclk / pd);
+
+	if (idx < 0)
+		dev_err(codec->dev, "Failed to set DMIC clock\n");
+	else {
+		snd_soc_update_bits(codec, RT5665_DMIC_CTRL_1,
+			RT5665_DMIC_CLK_MASK, idx << RT5665_DMIC_CLK_SFT);
+	}
+	return idx;
+}
+
+static int rt5665_charge_pump_event(struct snd_soc_dapm_widget *w,
+	struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		snd_soc_update_bits(codec, RT5665_HP_CHARGE_PUMP_1,
+			RT5665_PM_HP_MASK | RT5665_OSW_L_MASK,
+			RT5665_PM_HP_HV | RT5665_OSW_L_EN);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		snd_soc_update_bits(codec, RT5665_HP_CHARGE_PUMP_1,
+			RT5665_PM_HP_MASK | RT5665_OSW_L_MASK,
+			RT5665_PM_HP_LV | RT5665_OSW_L_DIS);
+		break;
+	default:
+		return 0;
+	}
+
+	return 0;
+}
+
+static int is_sys_clk_from_pll(struct snd_soc_dapm_widget *w,
+			 struct snd_soc_dapm_widget *sink)
+{
+	unsigned int val;
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+
+	val = snd_soc_read(codec, RT5665_GLB_CLK);
+	val &= RT5665_SCLK_SRC_MASK;
+	if (val == RT5665_SCLK_SRC_PLL1)
+		return 1;
+	else
+		return 0;
+}
+
+static int is_using_asrc(struct snd_soc_dapm_widget *w,
+			 struct snd_soc_dapm_widget *sink)
+{
+	unsigned int reg, shift, val;
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+
+	switch (w->shift) {
+	case RT5665_ADC_MONO_R_ASRC_SFT:
+		reg = RT5665_ASRC_3;
+		shift = RT5665_AD_MONOR_CLK_SEL_SFT;
+		break;
+	case RT5665_ADC_MONO_L_ASRC_SFT:
+		reg = RT5665_ASRC_3;
+		shift = RT5665_AD_MONOL_CLK_SEL_SFT;
+		break;
+	case RT5665_ADC_STO1_ASRC_SFT:
+		reg = RT5665_ASRC_3;
+		shift = RT5665_AD_STO1_CLK_SEL_SFT;
+		break;
+	case RT5665_ADC_STO2_ASRC_SFT:
+		reg = RT5665_ASRC_3;
+		shift = RT5665_AD_STO2_CLK_SEL_SFT;
+		break;
+	case RT5665_DAC_MONO_R_ASRC_SFT:
+		reg = RT5665_ASRC_2;
+		shift = RT5665_DA_MONOR_CLK_SEL_SFT;
+		break;
+	case RT5665_DAC_MONO_L_ASRC_SFT:
+		reg = RT5665_ASRC_2;
+		shift = RT5665_DA_MONOL_CLK_SEL_SFT;
+		break;
+	case RT5665_DAC_STO1_ASRC_SFT:
+		reg = RT5665_ASRC_2;
+		shift = RT5665_DA_STO1_CLK_SEL_SFT;
+		break;
+	case RT5665_DAC_STO2_ASRC_SFT:
+		reg = RT5665_ASRC_2;
+		shift = RT5665_DA_STO2_CLK_SEL_SFT;
+		break;
+	default:
+		return 0;
+	}
+
+	val = (snd_soc_read(codec, reg) >> shift) & 0xf;
+	switch (val) {
+	case RT5665_CLK_SEL_I2S1_ASRC:
+	case RT5665_CLK_SEL_I2S2_ASRC:
+	case RT5665_CLK_SEL_I2S3_ASRC:
+		/* I2S_Pre_Div1 should be 1 in asrc mode */
+		snd_soc_update_bits(codec, RT5665_ADDA_CLK_1,
+			RT5665_I2S_PD1_MASK, RT5665_I2S_PD1_2);
+		return 1;
+	default:
+		return 0;
+	}
+
+}
+
+/* Digital Mixer */
+static const struct snd_kcontrol_new rt5665_sto1_adc_l_mix[] = {
+	SOC_DAPM_SINGLE("ADC1 Switch", RT5665_STO1_ADC_MIXER,
+			RT5665_M_STO1_ADC_L1_SFT, 1, 1),
+	SOC_DAPM_SINGLE("ADC2 Switch", RT5665_STO1_ADC_MIXER,
+			RT5665_M_STO1_ADC_L2_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_sto1_adc_r_mix[] = {
+	SOC_DAPM_SINGLE("ADC1 Switch", RT5665_STO1_ADC_MIXER,
+			RT5665_M_STO1_ADC_R1_SFT, 1, 1),
+	SOC_DAPM_SINGLE("ADC2 Switch", RT5665_STO1_ADC_MIXER,
+			RT5665_M_STO1_ADC_R2_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_sto2_adc_l_mix[] = {
+	SOC_DAPM_SINGLE("ADC1 Switch", RT5665_STO2_ADC_MIXER,
+			RT5665_M_STO2_ADC_L1_SFT, 1, 1),
+	SOC_DAPM_SINGLE("ADC2 Switch", RT5665_STO2_ADC_MIXER,
+			RT5665_M_STO2_ADC_L2_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_sto2_adc_r_mix[] = {
+	SOC_DAPM_SINGLE("ADC1 Switch", RT5665_STO2_ADC_MIXER,
+			RT5665_M_STO2_ADC_R1_SFT, 1, 1),
+	SOC_DAPM_SINGLE("ADC2 Switch", RT5665_STO2_ADC_MIXER,
+			RT5665_M_STO2_ADC_R2_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_mono_adc_l_mix[] = {
+	SOC_DAPM_SINGLE("ADC1 Switch", RT5665_MONO_ADC_MIXER,
+			RT5665_M_MONO_ADC_L1_SFT, 1, 1),
+	SOC_DAPM_SINGLE("ADC2 Switch", RT5665_MONO_ADC_MIXER,
+			RT5665_M_MONO_ADC_L2_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_mono_adc_r_mix[] = {
+	SOC_DAPM_SINGLE("ADC1 Switch", RT5665_MONO_ADC_MIXER,
+			RT5665_M_MONO_ADC_R1_SFT, 1, 1),
+	SOC_DAPM_SINGLE("ADC2 Switch", RT5665_MONO_ADC_MIXER,
+			RT5665_M_MONO_ADC_R2_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_dac_l_mix[] = {
+	SOC_DAPM_SINGLE("Stereo ADC Switch", RT5665_AD_DA_MIXER,
+			RT5665_M_ADCMIX_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC1 Switch", RT5665_AD_DA_MIXER,
+			RT5665_M_DAC1_L_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_dac_r_mix[] = {
+	SOC_DAPM_SINGLE("Stereo ADC Switch", RT5665_AD_DA_MIXER,
+			RT5665_M_ADCMIX_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC1 Switch", RT5665_AD_DA_MIXER,
+			RT5665_M_DAC1_R_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_sto1_dac_l_mix[] = {
+	SOC_DAPM_SINGLE("DAC L1 Switch", RT5665_STO1_DAC_MIXER,
+			RT5665_M_DAC_L1_STO_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC R1 Switch", RT5665_STO1_DAC_MIXER,
+			RT5665_M_DAC_R1_STO_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC L2 Switch", RT5665_STO1_DAC_MIXER,
+			RT5665_M_DAC_L2_STO_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC R2 Switch", RT5665_STO1_DAC_MIXER,
+			RT5665_M_DAC_R2_STO_L_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_sto1_dac_r_mix[] = {
+	SOC_DAPM_SINGLE("DAC L1 Switch", RT5665_STO1_DAC_MIXER,
+			RT5665_M_DAC_L1_STO_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC R1 Switch", RT5665_STO1_DAC_MIXER,
+			RT5665_M_DAC_R1_STO_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC L2 Switch", RT5665_STO1_DAC_MIXER,
+			RT5665_M_DAC_L2_STO_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC R2 Switch", RT5665_STO1_DAC_MIXER,
+			RT5665_M_DAC_R2_STO_R_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_sto2_dac_l_mix[] = {
+	SOC_DAPM_SINGLE("DAC L1 Switch", RT5665_STO2_DAC_MIXER,
+			RT5665_M_DAC_L1_STO2_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC L2 Switch", RT5665_STO2_DAC_MIXER,
+			RT5665_M_DAC_L2_STO2_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC L3 Switch", RT5665_STO2_DAC_MIXER,
+			RT5665_M_DAC_L3_STO2_L_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_sto2_dac_r_mix[] = {
+	SOC_DAPM_SINGLE("DAC R1 Switch", RT5665_STO2_DAC_MIXER,
+			RT5665_M_DAC_R1_STO2_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC R2 Switch", RT5665_STO2_DAC_MIXER,
+			RT5665_M_DAC_R2_STO2_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC R3 Switch", RT5665_STO2_DAC_MIXER,
+			RT5665_M_DAC_R3_STO2_R_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_mono_dac_l_mix[] = {
+	SOC_DAPM_SINGLE("DAC L1 Switch", RT5665_MONO_DAC_MIXER,
+			RT5665_M_DAC_L1_MONO_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC R1 Switch", RT5665_MONO_DAC_MIXER,
+			RT5665_M_DAC_R1_MONO_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC L2 Switch", RT5665_MONO_DAC_MIXER,
+			RT5665_M_DAC_L2_MONO_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC R2 Switch", RT5665_MONO_DAC_MIXER,
+			RT5665_M_DAC_R2_MONO_L_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_mono_dac_r_mix[] = {
+	SOC_DAPM_SINGLE("DAC L1 Switch", RT5665_MONO_DAC_MIXER,
+			RT5665_M_DAC_L1_MONO_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC R1 Switch", RT5665_MONO_DAC_MIXER,
+			RT5665_M_DAC_R1_MONO_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC L2 Switch", RT5665_MONO_DAC_MIXER,
+			RT5665_M_DAC_L2_MONO_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("DAC R2 Switch", RT5665_MONO_DAC_MIXER,
+			RT5665_M_DAC_R2_MONO_R_SFT, 1, 1),
+};
+
+/* Analog Input Mixer */
+static const struct snd_kcontrol_new rt5665_rec1_l_mix[] = {
+	SOC_DAPM_SINGLE("CBJ Switch", RT5665_REC1_L2_MIXER,
+			RT5665_M_CBJ_RM1_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("INL Switch", RT5665_REC1_L2_MIXER,
+			RT5665_M_INL_RM1_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("INR Switch", RT5665_REC1_L2_MIXER,
+			RT5665_M_INR_RM1_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST4 Switch", RT5665_REC1_L2_MIXER,
+			RT5665_M_BST4_RM1_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST3 Switch", RT5665_REC1_L2_MIXER,
+			RT5665_M_BST3_RM1_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST2 Switch", RT5665_REC1_L2_MIXER,
+			RT5665_M_BST2_RM1_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST1 Switch", RT5665_REC1_L2_MIXER,
+			RT5665_M_BST1_RM1_L_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_rec1_r_mix[] = {
+	SOC_DAPM_SINGLE("MONOVOL Switch", RT5665_REC1_R2_MIXER,
+			RT5665_M_AEC_REF_RM1_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("INR Switch", RT5665_REC1_R2_MIXER,
+			RT5665_M_INR_RM1_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST4 Switch", RT5665_REC1_R2_MIXER,
+			RT5665_M_BST4_RM1_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST3 Switch", RT5665_REC1_R2_MIXER,
+			RT5665_M_BST3_RM1_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST2 Switch", RT5665_REC1_R2_MIXER,
+			RT5665_M_BST2_RM1_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST1 Switch", RT5665_REC1_R2_MIXER,
+			RT5665_M_BST1_RM1_R_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_rec2_l_mix[] = {
+	SOC_DAPM_SINGLE("INL Switch", RT5665_REC2_L2_MIXER,
+			RT5665_M_INL_RM2_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("INR Switch", RT5665_REC2_L2_MIXER,
+			RT5665_M_INR_RM2_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("CBJ Switch", RT5665_REC2_L2_MIXER,
+			RT5665_M_CBJ_RM2_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST4 Switch", RT5665_REC2_L2_MIXER,
+			RT5665_M_BST4_RM2_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST3 Switch", RT5665_REC2_L2_MIXER,
+			RT5665_M_BST3_RM2_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST2 Switch", RT5665_REC2_L2_MIXER,
+			RT5665_M_BST2_RM2_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST1 Switch", RT5665_REC2_L2_MIXER,
+			RT5665_M_BST1_RM2_L_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_rec2_r_mix[] = {
+	SOC_DAPM_SINGLE("MONOVOL Switch", RT5665_REC2_R2_MIXER,
+			RT5665_M_MONOVOL_RM2_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("INL Switch", RT5665_REC2_R2_MIXER,
+			RT5665_M_INL_RM2_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("INR Switch", RT5665_REC2_R2_MIXER,
+			RT5665_M_INR_RM2_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST4 Switch", RT5665_REC2_R2_MIXER,
+			RT5665_M_BST4_RM2_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST3 Switch", RT5665_REC2_R2_MIXER,
+			RT5665_M_BST3_RM2_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST2 Switch", RT5665_REC2_R2_MIXER,
+			RT5665_M_BST2_RM2_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST1 Switch", RT5665_REC2_R2_MIXER,
+			RT5665_M_BST1_RM2_R_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_monovol_mix[] = {
+	SOC_DAPM_SINGLE("DAC L2 Switch", RT5665_MONOMIX_IN_GAIN,
+			RT5665_M_DAC_L2_MM_SFT, 1, 1),
+	SOC_DAPM_SINGLE("RECMIX2L Switch", RT5665_MONOMIX_IN_GAIN,
+			RT5665_M_RECMIC2L_MM_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST1 Switch", RT5665_MONOMIX_IN_GAIN,
+			RT5665_M_BST1_MM_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST2 Switch", RT5665_MONOMIX_IN_GAIN,
+			RT5665_M_BST2_MM_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST3 Switch", RT5665_MONOMIX_IN_GAIN,
+			RT5665_M_BST3_MM_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_out_l_mix[] = {
+	SOC_DAPM_SINGLE("DAC L2 Switch", RT5665_OUT_L_MIXER,
+			RT5665_M_DAC_L2_OM_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("INL Switch", RT5665_OUT_L_MIXER,
+			RT5665_M_IN_L_OM_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST1 Switch", RT5665_OUT_L_MIXER,
+			RT5665_M_BST1_OM_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST2 Switch", RT5665_OUT_L_MIXER,
+			RT5665_M_BST2_OM_L_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST3 Switch", RT5665_OUT_L_MIXER,
+			RT5665_M_BST3_OM_L_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_out_r_mix[] = {
+	SOC_DAPM_SINGLE("DAC R2 Switch", RT5665_OUT_R_MIXER,
+			RT5665_M_DAC_R2_OM_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("INR Switch", RT5665_OUT_R_MIXER,
+			RT5665_M_IN_R_OM_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST2 Switch", RT5665_OUT_R_MIXER,
+			RT5665_M_BST2_OM_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST3 Switch", RT5665_OUT_R_MIXER,
+			RT5665_M_BST3_OM_R_SFT, 1, 1),
+	SOC_DAPM_SINGLE("BST4 Switch", RT5665_OUT_R_MIXER,
+			RT5665_M_BST4_OM_R_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_mono_mix[] = {
+	SOC_DAPM_SINGLE("DAC L2 Switch", RT5665_MONOMIX_IN_GAIN,
+			RT5665_M_DAC_L2_MA_SFT, 1, 1),
+	SOC_DAPM_SINGLE("MONOVOL Switch", RT5665_MONOMIX_IN_GAIN,
+			RT5665_M_MONOVOL_MA_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_lout_l_mix[] = {
+	SOC_DAPM_SINGLE("DAC L2 Switch", RT5665_LOUT_MIXER,
+			RT5665_M_DAC_L2_LM_SFT, 1, 1),
+	SOC_DAPM_SINGLE("OUTVOL L Switch", RT5665_LOUT_MIXER,
+			RT5665_M_OV_L_LM_SFT, 1, 1),
+};
+
+static const struct snd_kcontrol_new rt5665_lout_r_mix[] = {
+	SOC_DAPM_SINGLE("DAC R2 Switch", RT5665_LOUT_MIXER,
+			RT5665_M_DAC_R2_LM_SFT, 1, 1),
+	SOC_DAPM_SINGLE("OUTVOL R Switch", RT5665_LOUT_MIXER,
+			RT5665_M_OV_R_LM_SFT, 1, 1),
+};
+
+/*DAC L2, DAC R2*/
+/*MX-17 [6:4], MX-17 [2:0]*/
+static const char * const rt5665_dac2_src[] = {
+	"IF1 DAC2", "IF2_1 DAC", "IF2_2 DAC", "IF3 DAC", "Mono ADC MIX"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_dac_l2_enum, RT5665_DAC2_CTRL,
+	RT5665_DAC_L2_SEL_SFT, rt5665_dac2_src);
+
+static const struct snd_kcontrol_new rt5665_dac_l2_mux =
+	SOC_DAPM_ENUM("Digital DAC L2 Source", rt5665_dac_l2_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_dac_r2_enum, RT5665_DAC2_CTRL,
+	RT5665_DAC_R2_SEL_SFT, rt5665_dac2_src);
+
+static const struct snd_kcontrol_new rt5665_dac_r2_mux =
+	SOC_DAPM_ENUM("Digital DAC R2 Source", rt5665_dac_r2_enum);
+
+/*DAC L3, DAC R3*/
+/*MX-1B [6:4], MX-1B [2:0]*/
+static const char * const rt5665_dac3_src[] = {
+	"IF1 DAC2", "IF2_1 DAC", "IF2_2 DAC", "IF3 DAC", "STO2 ADC MIX"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_dac_l3_enum, RT5665_DAC3_CTRL,
+	RT5665_DAC_L3_SEL_SFT, rt5665_dac3_src);
+
+static const struct snd_kcontrol_new rt5665_dac_l3_mux =
+	SOC_DAPM_ENUM("Digital DAC L3 Source", rt5665_dac_l3_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_dac_r3_enum, RT5665_DAC3_CTRL,
+	RT5665_DAC_R3_SEL_SFT, rt5665_dac3_src);
+
+static const struct snd_kcontrol_new rt5665_dac_r3_mux =
+	SOC_DAPM_ENUM("Digital DAC R3 Source", rt5665_dac_r3_enum);
+
+/* STO1 ADC1 Source */
+/* MX-26 [13] [5] */
+static const char * const rt5665_sto1_adc1_src[] = {
+	"DD Mux", "ADC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto1_adc1l_enum, RT5665_STO1_ADC_MIXER,
+	RT5665_STO1_ADC1L_SRC_SFT, rt5665_sto1_adc1_src);
+
+static const struct snd_kcontrol_new rt5665_sto1_adc1l_mux =
+	SOC_DAPM_ENUM("Stereo1 ADC1L Source", rt5665_sto1_adc1l_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto1_adc1r_enum, RT5665_STO1_ADC_MIXER,
+	RT5665_STO1_ADC1R_SRC_SFT, rt5665_sto1_adc1_src);
+
+static const struct snd_kcontrol_new rt5665_sto1_adc1r_mux =
+	SOC_DAPM_ENUM("Stereo1 ADC1L Source", rt5665_sto1_adc1r_enum);
+
+/* STO1 ADC Source */
+/* MX-26 [11:10] [3:2] */
+static const char * const rt5665_sto1_adc_src[] = {
+	"ADC1 L", "ADC1 R", "ADC2 L", "ADC2 R"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto1_adcl_enum, RT5665_STO1_ADC_MIXER,
+	RT5665_STO1_ADCL_SRC_SFT, rt5665_sto1_adc_src);
+
+static const struct snd_kcontrol_new rt5665_sto1_adcl_mux =
+	SOC_DAPM_ENUM("Stereo1 ADCL Source", rt5665_sto1_adcl_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto1_adcr_enum, RT5665_STO1_ADC_MIXER,
+	RT5665_STO1_ADCR_SRC_SFT, rt5665_sto1_adc_src);
+
+static const struct snd_kcontrol_new rt5665_sto1_adcr_mux =
+	SOC_DAPM_ENUM("Stereo1 ADCR Source", rt5665_sto1_adcr_enum);
+
+/* STO1 ADC2 Source */
+/* MX-26 [12] [4] */
+static const char * const rt5665_sto1_adc2_src[] = {
+	"DAC MIX", "DMIC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto1_adc2l_enum, RT5665_STO1_ADC_MIXER,
+	RT5665_STO1_ADC2L_SRC_SFT, rt5665_sto1_adc2_src);
+
+static const struct snd_kcontrol_new rt5665_sto1_adc2l_mux =
+	SOC_DAPM_ENUM("Stereo1 ADC2L Source", rt5665_sto1_adc2l_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto1_adc2r_enum, RT5665_STO1_ADC_MIXER,
+	RT5665_STO1_ADC2R_SRC_SFT, rt5665_sto1_adc2_src);
+
+static const struct snd_kcontrol_new rt5665_sto1_adc2r_mux =
+	SOC_DAPM_ENUM("Stereo1 ADC2R Source", rt5665_sto1_adc2r_enum);
+
+/* STO1 DMIC Source */
+/* MX-26 [8] */
+static const char * const rt5665_sto1_dmic_src[] = {
+	"DMIC1", "DMIC2"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto1_dmic_enum, RT5665_STO1_ADC_MIXER,
+	RT5665_STO1_DMIC_SRC_SFT, rt5665_sto1_dmic_src);
+
+static const struct snd_kcontrol_new rt5665_sto1_dmic_mux =
+	SOC_DAPM_ENUM("Stereo1 DMIC Mux", rt5665_sto1_dmic_enum);
+
+/* MX-26 [9] */
+static const char * const rt5665_sto1_dd_l_src[] = {
+	"STO2 DAC", "MONO DAC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto1_dd_l_enum, RT5665_STO1_ADC_MIXER,
+	RT5665_STO1_DD_L_SRC_SFT, rt5665_sto1_dd_l_src);
+
+static const struct snd_kcontrol_new rt5665_sto1_dd_l_mux =
+	SOC_DAPM_ENUM("Stereo1 DD L Source", rt5665_sto1_dd_l_enum);
+
+/* MX-26 [1:0] */
+static const char * const rt5665_sto1_dd_r_src[] = {
+	"STO2 DAC", "MONO DAC", "AEC REF"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto1_dd_r_enum, RT5665_STO1_ADC_MIXER,
+	RT5665_STO1_DD_R_SRC_SFT, rt5665_sto1_dd_r_src);
+
+static const struct snd_kcontrol_new rt5665_sto1_dd_r_mux =
+	SOC_DAPM_ENUM("Stereo1 DD R Source", rt5665_sto1_dd_r_enum);
+
+/* MONO ADC L2 Source */
+/* MX-27 [12] */
+static const char * const rt5665_mono_adc_l2_src[] = {
+	"DAC MIXL", "DMIC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_mono_adc_l2_enum, RT5665_MONO_ADC_MIXER,
+	RT5665_MONO_ADC_L2_SRC_SFT, rt5665_mono_adc_l2_src);
+
+static const struct snd_kcontrol_new rt5665_mono_adc_l2_mux =
+	SOC_DAPM_ENUM("Mono ADC L2 Source", rt5665_mono_adc_l2_enum);
+
+
+/* MONO ADC L1 Source */
+/* MX-27 [13] */
+static const char * const rt5665_mono_adc_l1_src[] = {
+	"DD Mux", "ADC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_mono_adc_l1_enum, RT5665_MONO_ADC_MIXER,
+	RT5665_MONO_ADC_L1_SRC_SFT, rt5665_mono_adc_l1_src);
+
+static const struct snd_kcontrol_new rt5665_mono_adc_l1_mux =
+	SOC_DAPM_ENUM("Mono ADC L1 Source", rt5665_mono_adc_l1_enum);
+
+/* MX-27 [9][1]*/
+static const char * const rt5665_mono_dd_src[] = {
+	"STO2 DAC", "MONO DAC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_mono_dd_l_enum, RT5665_MONO_ADC_MIXER,
+	RT5665_MONO_DD_L_SRC_SFT, rt5665_mono_dd_src);
+
+static const struct snd_kcontrol_new rt5665_mono_dd_l_mux =
+	SOC_DAPM_ENUM("Mono DD L Source", rt5665_mono_dd_l_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_mono_dd_r_enum, RT5665_MONO_ADC_MIXER,
+	RT5665_MONO_DD_R_SRC_SFT, rt5665_mono_dd_src);
+
+static const struct snd_kcontrol_new rt5665_mono_dd_r_mux =
+	SOC_DAPM_ENUM("Mono DD R Source", rt5665_mono_dd_r_enum);
+
+/* MONO ADC L Source, MONO ADC R Source*/
+/* MX-27 [11:10], MX-27 [3:2] */
+static const char * const rt5665_mono_adc_src[] = {
+	"ADC1 L", "ADC1 R", "ADC2 L", "ADC2 R"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_mono_adc_l_enum, RT5665_MONO_ADC_MIXER,
+	RT5665_MONO_ADC_L_SRC_SFT, rt5665_mono_adc_src);
+
+static const struct snd_kcontrol_new rt5665_mono_adc_l_mux =
+	SOC_DAPM_ENUM("Mono ADC L Source", rt5665_mono_adc_l_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_mono_adcr_enum, RT5665_MONO_ADC_MIXER,
+	RT5665_MONO_ADC_R_SRC_SFT, rt5665_mono_adc_src);
+
+static const struct snd_kcontrol_new rt5665_mono_adc_r_mux =
+	SOC_DAPM_ENUM("Mono ADC R Source", rt5665_mono_adcr_enum);
+
+/* MONO DMIC L Source */
+/* MX-27 [8] */
+static const char * const rt5665_mono_dmic_l_src[] = {
+	"DMIC1 L", "DMIC2 L"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_mono_dmic_l_enum, RT5665_MONO_ADC_MIXER,
+	RT5665_MONO_DMIC_L_SRC_SFT, rt5665_mono_dmic_l_src);
+
+static const struct snd_kcontrol_new rt5665_mono_dmic_l_mux =
+	SOC_DAPM_ENUM("Mono DMIC L Source", rt5665_mono_dmic_l_enum);
+
+/* MONO ADC R2 Source */
+/* MX-27 [4] */
+static const char * const rt5665_mono_adc_r2_src[] = {
+	"DAC MIXR", "DMIC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_mono_adc_r2_enum, RT5665_MONO_ADC_MIXER,
+	RT5665_MONO_ADC_R2_SRC_SFT, rt5665_mono_adc_r2_src);
+
+static const struct snd_kcontrol_new rt5665_mono_adc_r2_mux =
+	SOC_DAPM_ENUM("Mono ADC R2 Source", rt5665_mono_adc_r2_enum);
+
+/* MONO ADC R1 Source */
+/* MX-27 [5] */
+static const char * const rt5665_mono_adc_r1_src[] = {
+	"DD Mux", "ADC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_mono_adc_r1_enum, RT5665_MONO_ADC_MIXER,
+	RT5665_MONO_ADC_R1_SRC_SFT, rt5665_mono_adc_r1_src);
+
+static const struct snd_kcontrol_new rt5665_mono_adc_r1_mux =
+	SOC_DAPM_ENUM("Mono ADC R1 Source", rt5665_mono_adc_r1_enum);
+
+/* MONO DMIC R Source */
+/* MX-27 [0] */
+static const char * const rt5665_mono_dmic_r_src[] = {
+	"DMIC1 R", "DMIC2 R"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_mono_dmic_r_enum, RT5665_MONO_ADC_MIXER,
+	RT5665_MONO_DMIC_R_SRC_SFT, rt5665_mono_dmic_r_src);
+
+static const struct snd_kcontrol_new rt5665_mono_dmic_r_mux =
+	SOC_DAPM_ENUM("Mono DMIC R Source", rt5665_mono_dmic_r_enum);
+
+
+/* STO2 ADC1 Source */
+/* MX-28 [13] [5] */
+static const char * const rt5665_sto2_adc1_src[] = {
+	"DD Mux", "ADC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto2_adc1l_enum, RT5665_STO2_ADC_MIXER,
+	RT5665_STO2_ADC1L_SRC_SFT, rt5665_sto2_adc1_src);
+
+static const struct snd_kcontrol_new rt5665_sto2_adc1l_mux =
+	SOC_DAPM_ENUM("Stereo2 ADC1L Source", rt5665_sto2_adc1l_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto2_adc1r_enum, RT5665_STO2_ADC_MIXER,
+	RT5665_STO2_ADC1R_SRC_SFT, rt5665_sto2_adc1_src);
+
+static const struct snd_kcontrol_new rt5665_sto2_adc1r_mux =
+	SOC_DAPM_ENUM("Stereo2 ADC1L Source", rt5665_sto2_adc1r_enum);
+
+/* STO2 ADC Source */
+/* MX-28 [11:10] [3:2] */
+static const char * const rt5665_sto2_adc_src[] = {
+	"ADC1 L", "ADC1 R", "ADC2 L"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto2_adcl_enum, RT5665_STO2_ADC_MIXER,
+	RT5665_STO2_ADCL_SRC_SFT, rt5665_sto2_adc_src);
+
+static const struct snd_kcontrol_new rt5665_sto2_adcl_mux =
+	SOC_DAPM_ENUM("Stereo2 ADCL Source", rt5665_sto2_adcl_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto2_adcr_enum, RT5665_STO2_ADC_MIXER,
+	RT5665_STO2_ADCR_SRC_SFT, rt5665_sto2_adc_src);
+
+static const struct snd_kcontrol_new rt5665_sto2_adcr_mux =
+	SOC_DAPM_ENUM("Stereo2 ADCR Source", rt5665_sto2_adcr_enum);
+
+/* STO2 ADC2 Source */
+/* MX-28 [12] [4] */
+static const char * const rt5665_sto2_adc2_src[] = {
+	"DAC MIX", "DMIC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto2_adc2l_enum, RT5665_STO2_ADC_MIXER,
+	RT5665_STO2_ADC2L_SRC_SFT, rt5665_sto2_adc2_src);
+
+static const struct snd_kcontrol_new rt5665_sto2_adc2l_mux =
+	SOC_DAPM_ENUM("Stereo2 ADC2L Source", rt5665_sto2_adc2l_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto2_adc2r_enum, RT5665_STO2_ADC_MIXER,
+	RT5665_STO2_ADC2R_SRC_SFT, rt5665_sto2_adc2_src);
+
+static const struct snd_kcontrol_new rt5665_sto2_adc2r_mux =
+	SOC_DAPM_ENUM("Stereo2 ADC2R Source", rt5665_sto2_adc2r_enum);
+
+/* STO2 DMIC Source */
+/* MX-28 [8] */
+static const char * const rt5665_sto2_dmic_src[] = {
+	"DMIC1", "DMIC2"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto2_dmic_enum, RT5665_STO2_ADC_MIXER,
+	RT5665_STO2_DMIC_SRC_SFT, rt5665_sto2_dmic_src);
+
+static const struct snd_kcontrol_new rt5665_sto2_dmic_mux =
+	SOC_DAPM_ENUM("Stereo2 DMIC Source", rt5665_sto2_dmic_enum);
+
+/* MX-28 [9] */
+static const char * const rt5665_sto2_dd_l_src[] = {
+	"STO2 DAC", "MONO DAC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto2_dd_l_enum, RT5665_STO2_ADC_MIXER,
+	RT5665_STO2_DD_L_SRC_SFT, rt5665_sto2_dd_l_src);
+
+static const struct snd_kcontrol_new rt5665_sto2_dd_l_mux =
+	SOC_DAPM_ENUM("Stereo2 DD L Source", rt5665_sto2_dd_l_enum);
+
+/* MX-28 [1] */
+static const char * const rt5665_sto2_dd_r_src[] = {
+	"STO2 DAC", "MONO DAC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_sto2_dd_r_enum, RT5665_STO2_ADC_MIXER,
+	RT5665_STO2_DD_R_SRC_SFT, rt5665_sto2_dd_r_src);
+
+static const struct snd_kcontrol_new rt5665_sto2_dd_r_mux =
+	SOC_DAPM_ENUM("Stereo2 DD R Source", rt5665_sto2_dd_r_enum);
+
+/* DAC R1 Source, DAC L1 Source*/
+/* MX-29 [11:10], MX-29 [9:8]*/
+static const char * const rt5665_dac1_src[] = {
+	"IF1 DAC1", "IF2_1 DAC", "IF2_2 DAC", "IF3 DAC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_dac_r1_enum, RT5665_AD_DA_MIXER,
+	RT5665_DAC1_R_SEL_SFT, rt5665_dac1_src);
+
+static const struct snd_kcontrol_new rt5665_dac_r1_mux =
+	SOC_DAPM_ENUM("DAC R1 Source", rt5665_dac_r1_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_dac_l1_enum, RT5665_AD_DA_MIXER,
+	RT5665_DAC1_L_SEL_SFT, rt5665_dac1_src);
+
+static const struct snd_kcontrol_new rt5665_dac_l1_mux =
+	SOC_DAPM_ENUM("DAC L1 Source", rt5665_dac_l1_enum);
+
+/* DAC Digital Mixer L Source, DAC Digital Mixer R Source*/
+/* MX-2D [13:12], MX-2D [9:8]*/
+static const char * const rt5665_dig_dac_mix_src[] = {
+	"Stereo1 DAC Mixer", "Stereo2 DAC Mixer", "Mono DAC Mixer"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_dig_dac_mixl_enum, RT5665_A_DAC1_MUX,
+	RT5665_DAC_MIX_L_SFT, rt5665_dig_dac_mix_src);
+
+static const struct snd_kcontrol_new rt5665_dig_dac_mixl_mux =
+	SOC_DAPM_ENUM("DAC Digital Mixer L Source", rt5665_dig_dac_mixl_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_dig_dac_mixr_enum, RT5665_A_DAC1_MUX,
+	RT5665_DAC_MIX_R_SFT, rt5665_dig_dac_mix_src);
+
+static const struct snd_kcontrol_new rt5665_dig_dac_mixr_mux =
+	SOC_DAPM_ENUM("DAC Digital Mixer R Source", rt5665_dig_dac_mixr_enum);
+
+/* Analog DAC L1 Source, Analog DAC R1 Source*/
+/* MX-2D [5:4], MX-2D [1:0]*/
+static const char * const rt5665_alg_dac1_src[] = {
+	"Stereo1 DAC Mixer", "DAC1", "DMIC1"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_alg_dac_l1_enum, RT5665_A_DAC1_MUX,
+	RT5665_A_DACL1_SFT, rt5665_alg_dac1_src);
+
+static const struct snd_kcontrol_new rt5665_alg_dac_l1_mux =
+	SOC_DAPM_ENUM("Analog DAC L1 Source", rt5665_alg_dac_l1_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_alg_dac_r1_enum, RT5665_A_DAC1_MUX,
+	RT5665_A_DACR1_SFT, rt5665_alg_dac1_src);
+
+static const struct snd_kcontrol_new rt5665_alg_dac_r1_mux =
+	SOC_DAPM_ENUM("Analog DAC R1 Source", rt5665_alg_dac_r1_enum);
+
+/* Analog DAC LR Source, Analog DAC R2 Source*/
+/* MX-2E [5:4], MX-2E [0]*/
+static const char * const rt5665_alg_dac2_src[] = {
+	"Mono DAC Mixer", "DAC2"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_alg_dac_l2_enum, RT5665_A_DAC2_MUX,
+	RT5665_A_DACL2_SFT, rt5665_alg_dac2_src);
+
+static const struct snd_kcontrol_new rt5665_alg_dac_l2_mux =
+	SOC_DAPM_ENUM("Analog DAC L2 Source", rt5665_alg_dac_l2_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_alg_dac_r2_enum, RT5665_A_DAC2_MUX,
+	RT5665_A_DACR2_SFT, rt5665_alg_dac2_src);
+
+static const struct snd_kcontrol_new rt5665_alg_dac_r2_mux =
+	SOC_DAPM_ENUM("Analog DAC R2 Source", rt5665_alg_dac_r2_enum);
+
+/* Interface2 ADC Data Input*/
+/* MX-2F [14:12] */
+static const char * const rt5665_if2_1_adc_in_src[] = {
+	"STO1 ADC", "STO2 ADC", "MONO ADC", "IF1 DAC1",
+	"IF1 DAC2", "IF2_2 DAC", "IF3 DAC", "DAC1 MIX"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_if2_1_adc_in_enum, RT5665_DIG_INF2_DATA,
+	RT5665_IF3_ADC_IN_SFT, rt5665_if2_1_adc_in_src);
+
+static const struct snd_kcontrol_new rt5665_if2_1_adc_in_mux =
+	SOC_DAPM_ENUM("IF2_1 ADC IN Source", rt5665_if2_1_adc_in_enum);
+
+/* MX-2F [6:4] */
+static const char * const rt5665_if2_2_adc_in_src[] = {
+	"STO1 ADC", "STO2 ADC", "MONO ADC", "IF1 DAC1",
+	"IF1 DAC2", "IF2_1 DAC", "IF3 DAC", "DAC1 MIX"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_if2_2_adc_in_enum, RT5665_DIG_INF2_DATA,
+	RT5665_IF2_2_ADC_IN_SFT, rt5665_if2_2_adc_in_src);
+
+static const struct snd_kcontrol_new rt5665_if2_2_adc_in_mux =
+	SOC_DAPM_ENUM("IF2_1 ADC IN Source", rt5665_if2_2_adc_in_enum);
+
+/* Interface3 ADC Data Input*/
+/* MX-30 [6:4] */
+static const char * const rt5665_if3_adc_in_src[] = {
+	"STO1 ADC", "STO2 ADC", "MONO ADC", "IF1 DAC1",
+	"IF1 DAC2", "IF2_1 DAC", "IF2_2 DAC", "DAC1 MIX"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_if3_adc_in_enum, RT5665_DIG_INF3_DATA,
+	RT5665_IF3_ADC_IN_SFT, rt5665_if3_adc_in_src);
+
+static const struct snd_kcontrol_new rt5665_if3_adc_in_mux =
+	SOC_DAPM_ENUM("IF3 ADC IN Source", rt5665_if3_adc_in_enum);
+
+/* PDM 1 L/R*/
+/* MX-31 [11:10] [9:8] */
+static const char * const rt5665_pdm_src[] = {
+	"Stereo1 DAC", "Stereo2 DAC", "Mono DAC"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_pdm_l_enum, RT5665_PDM_OUT_CTRL,
+	RT5665_PDM1_L_SFT, rt5665_pdm_src);
+
+static const struct snd_kcontrol_new rt5665_pdm_l_mux =
+	SOC_DAPM_ENUM("PDM L Source", rt5665_pdm_l_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_pdm_r_enum, RT5665_PDM_OUT_CTRL,
+	RT5665_PDM1_R_SFT, rt5665_pdm_src);
+
+static const struct snd_kcontrol_new rt5665_pdm_r_mux =
+	SOC_DAPM_ENUM("PDM R Source", rt5665_pdm_r_enum);
+
+
+/* I2S1 TDM ADCDAT Source */
+/* MX-7a[10] */
+static const char * const rt5665_if1_1_adc1_data_src[] = {
+	"STO1 ADC", "IF2_1 DAC",
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_if1_1_adc1_data_enum, RT5665_TDM_CTRL_3,
+	RT5665_IF1_ADC1_SEL_SFT, rt5665_if1_1_adc1_data_src);
+
+static const struct snd_kcontrol_new rt5665_if1_1_adc1_mux =
+	SOC_DAPM_ENUM("IF1_1 ADC1 Source", rt5665_if1_1_adc1_data_enum);
+
+/* MX-7a[9] */
+static const char * const rt5665_if1_1_adc2_data_src[] = {
+	"STO2 ADC", "IF2_2 DAC",
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_if1_1_adc2_data_enum, RT5665_TDM_CTRL_3,
+	RT5665_IF1_ADC2_SEL_SFT, rt5665_if1_1_adc2_data_src);
+
+static const struct snd_kcontrol_new rt5665_if1_1_adc2_mux =
+	SOC_DAPM_ENUM("IF1_1 ADC2 Source", rt5665_if1_1_adc2_data_enum);
+
+/* MX-7a[8] */
+static const char * const rt5665_if1_1_adc3_data_src[] = {
+	"MONO ADC", "IF3 DAC",
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_if1_1_adc3_data_enum, RT5665_TDM_CTRL_3,
+	RT5665_IF1_ADC3_SEL_SFT, rt5665_if1_1_adc3_data_src);
+
+static const struct snd_kcontrol_new rt5665_if1_1_adc3_mux =
+	SOC_DAPM_ENUM("IF1_1 ADC3 Source", rt5665_if1_1_adc3_data_enum);
+
+/* MX-7b[10] */
+static const char * const rt5665_if1_2_adc1_data_src[] = {
+	"STO1 ADC", "IF1 DAC",
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_if1_2_adc1_data_enum, RT5665_TDM_CTRL_4,
+	RT5665_IF1_ADC1_SEL_SFT, rt5665_if1_2_adc1_data_src);
+
+static const struct snd_kcontrol_new rt5665_if1_2_adc1_mux =
+	SOC_DAPM_ENUM("IF1_2 ADC1 Source", rt5665_if1_2_adc1_data_enum);
+
+/* MX-7b[9] */
+static const char * const rt5665_if1_2_adc2_data_src[] = {
+	"STO2 ADC", "IF2_1 DAC",
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_if1_2_adc2_data_enum, RT5665_TDM_CTRL_4,
+	RT5665_IF1_ADC2_SEL_SFT, rt5665_if1_2_adc2_data_src);
+
+static const struct snd_kcontrol_new rt5665_if1_2_adc2_mux =
+	SOC_DAPM_ENUM("IF1_2 ADC2 Source", rt5665_if1_2_adc2_data_enum);
+
+/* MX-7b[8] */
+static const char * const rt5665_if1_2_adc3_data_src[] = {
+	"MONO ADC", "IF2_2 DAC",
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_if1_2_adc3_data_enum, RT5665_TDM_CTRL_4,
+	RT5665_IF1_ADC3_SEL_SFT, rt5665_if1_2_adc3_data_src);
+
+static const struct snd_kcontrol_new rt5665_if1_2_adc3_mux =
+	SOC_DAPM_ENUM("IF1_2 ADC3 Source", rt5665_if1_2_adc3_data_enum);
+
+/* MX-7b[7] */
+static const char * const rt5665_if1_2_adc4_data_src[] = {
+	"DAC1", "IF3 DAC",
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_if1_2_adc4_data_enum, RT5665_TDM_CTRL_4,
+	RT5665_IF1_ADC4_SEL_SFT, rt5665_if1_2_adc4_data_src);
+
+static const struct snd_kcontrol_new rt5665_if1_2_adc4_mux =
+	SOC_DAPM_ENUM("IF1_2 ADC4 Source", rt5665_if1_2_adc4_data_enum);
+
+/* MX-7a[4:0] MX-7b[4:0] */
+static const char * const rt5665_tdm_adc_data_src[] = {
+	"1234", "1243", "1324",	"1342", "1432", "1423",
+	"2134", "2143", "2314",	"2341", "2431", "2413",
+	"3124", "3142", "3214", "3241", "3412", "3421",
+	"4123", "4132", "4213", "4231", "4312", "4321"
+};
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_tdm1_adc_data_enum, RT5665_TDM_CTRL_3,
+	RT5665_TDM_ADC_SEL_SFT, rt5665_tdm_adc_data_src);
+
+static const struct snd_kcontrol_new rt5665_tdm1_adc_mux =
+	SOC_DAPM_ENUM("TDM1 ADC Mux", rt5665_tdm1_adc_data_enum);
+
+static const SOC_ENUM_SINGLE_DECL(
+	rt5665_tdm2_adc_data_enum, RT5665_TDM_CTRL_4,
+	RT5665_TDM_ADC_SEL_SFT, rt5665_tdm_adc_data_src);
+
+static const struct snd_kcontrol_new rt5665_tdm2_adc_mux =
+	SOC_DAPM_ENUM("TDM2 ADCDAT Source", rt5665_tdm2_adc_data_enum);
+
+/* Out Volume Switch */
+static const struct snd_kcontrol_new monovol_switch =
+	SOC_DAPM_SINGLE("Switch", RT5665_MONO_OUT, RT5665_VOL_L_SFT, 1, 1);
+
+static const struct snd_kcontrol_new outvol_l_switch =
+	SOC_DAPM_SINGLE("Switch", RT5665_LOUT, RT5665_VOL_L_SFT, 1, 1);
+
+static const struct snd_kcontrol_new outvol_r_switch =
+	SOC_DAPM_SINGLE("Switch", RT5665_LOUT, RT5665_VOL_R_SFT, 1, 1);
+
+/* Out Switch */
+static const struct snd_kcontrol_new mono_switch =
+	SOC_DAPM_SINGLE("Switch", RT5665_MONO_OUT, RT5665_L_MUTE_SFT, 1, 1);
+
+static const struct snd_kcontrol_new hpo_switch =
+	SOC_DAPM_SINGLE_AUTODISABLE("Switch", RT5665_HP_CTRL_2,
+					RT5665_VOL_L_SFT, 1, 0);
+
+static const struct snd_kcontrol_new lout_l_switch =
+	SOC_DAPM_SINGLE("Switch", RT5665_LOUT, RT5665_L_MUTE_SFT, 1, 1);
+
+static const struct snd_kcontrol_new lout_r_switch =
+	SOC_DAPM_SINGLE("Switch", RT5665_LOUT, RT5665_R_MUTE_SFT, 1, 1);
+
+static const struct snd_kcontrol_new pdm_l_switch =
+	SOC_DAPM_SINGLE("Switch", RT5665_PDM_OUT_CTRL,
+			RT5665_M_PDM1_L_SFT, 1,	1);
+
+static const struct snd_kcontrol_new pdm_r_switch =
+	SOC_DAPM_SINGLE("Switch", RT5665_PDM_OUT_CTRL,
+			RT5665_M_PDM1_R_SFT, 1,	1);
+
+static int rt5665_mono_event(struct snd_soc_dapm_widget *w,
+	struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		snd_soc_update_bits(codec, RT5665_MONO_NG2_CTRL_1,
+			RT5665_NG2_EN_MASK, RT5665_NG2_EN);
+		snd_soc_update_bits(codec, RT5665_MONO_AMP_CALIB_CTRL_1, 0x40,
+			0x0);
+		snd_soc_update_bits(codec, RT5665_MONO_OUT, 0x10, 0x10);
+		snd_soc_update_bits(codec, RT5665_MONO_OUT, 0x20, 0x20);
+		break;
+
+	case SND_SOC_DAPM_POST_PMD:
+		snd_soc_update_bits(codec, RT5665_MONO_OUT, 0x20, 0);
+		snd_soc_update_bits(codec, RT5665_MONO_OUT, 0x10, 0);
+		snd_soc_update_bits(codec, RT5665_MONO_AMP_CALIB_CTRL_1, 0x40,
+			0x40);
+		snd_soc_update_bits(codec, RT5665_MONO_NG2_CTRL_1,
+			RT5665_NG2_EN_MASK, RT5665_NG2_DIS);
+		break;
+
+	default:
+		return 0;
+	}
+
+	return 0;
+
+}
+
+static int rt5665_hp_event(struct snd_soc_dapm_widget *w,
+	struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		snd_soc_update_bits(codec, RT5665_STO_NG2_CTRL_1,
+			RT5665_NG2_EN_MASK, RT5665_NG2_EN);
+		snd_soc_write(codec, RT5665_HP_LOGIC_CTRL_2, 0x0003);
+		break;
+
+	case SND_SOC_DAPM_POST_PMD:
+		snd_soc_write(codec, RT5665_HP_LOGIC_CTRL_2, 0x0002);
+		snd_soc_update_bits(codec, RT5665_STO_NG2_CTRL_1,
+			RT5665_NG2_EN_MASK, RT5665_NG2_DIS);
+		break;
+
+	default:
+		return 0;
+	}
+
+	return 0;
+
+}
+
+static int rt5665_lout_event(struct snd_soc_dapm_widget *w,
+	struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+
+	switch (event) {
+	case SND_SOC_DAPM_POST_PMU:
+		snd_soc_update_bits(codec, RT5665_DEPOP_1,
+			RT5665_PUMP_EN, RT5665_PUMP_EN);
+		break;
+
+	case SND_SOC_DAPM_PRE_PMD:
+		snd_soc_update_bits(codec, RT5665_DEPOP_1,
+			RT5665_PUMP_EN, 0);
+		break;
+
+	default:
+		return 0;
+	}
+
+	return 0;
+
+}
+
+static int set_dmic_power(struct snd_soc_dapm_widget *w,
+	struct snd_kcontrol *kcontrol, int event)
+{
+	switch (event) {
+	case SND_SOC_DAPM_POST_PMU:
+		/*Add delay to avoid pop noise*/
+		msleep(150);
+		break;
+
+	default:
+		return 0;
+	}
+
+	return 0;
+}
+
+static int rt5655_set_verf(struct snd_soc_dapm_widget *w,
+	struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_codec *codec = snd_soc_dapm_to_codec(w->dapm);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		switch (w->shift) {
+		case RT5665_PWR_VREF1_BIT:
+			snd_soc_update_bits(codec, RT5665_PWR_ANLG_1,
+				RT5665_PWR_FV1, 0);
+			break;
+
+		case RT5665_PWR_VREF2_BIT:
+			snd_soc_update_bits(codec, RT5665_PWR_ANLG_1,
+				RT5665_PWR_FV2, 0);
+			break;
+
+		case RT5665_PWR_VREF3_BIT:
+			snd_soc_update_bits(codec, RT5665_PWR_ANLG_1,
+				RT5665_PWR_FV3, 0);
+			break;
+
+		default:
+			break;
+		}
+		break;
+
+	case SND_SOC_DAPM_POST_PMU:
+		usleep_range(15000, 20000);
+		switch (w->shift) {
+		case RT5665_PWR_VREF1_BIT:
+			snd_soc_update_bits(codec, RT5665_PWR_ANLG_1,
+				RT5665_PWR_FV1, RT5665_PWR_FV1);
+			break;
+
+		case RT5665_PWR_VREF2_BIT:
+			snd_soc_update_bits(codec, RT5665_PWR_ANLG_1,
+				RT5665_PWR_FV2, RT5665_PWR_FV2);
+			break;
+
+		case RT5665_PWR_VREF3_BIT:
+			snd_soc_update_bits(codec, RT5665_PWR_ANLG_1,
+				RT5665_PWR_FV3, RT5665_PWR_FV3);
+			break;
+
+		default:
+			break;
+		}
+		break;
+
+	default:
+		return 0;
+	}
+
+	return 0;
+}
+
+
+static const struct snd_soc_dapm_widget rt5665_dapm_widgets[] = {
+	SND_SOC_DAPM_SUPPLY("LDO2", RT5665_PWR_ANLG_3, RT5665_PWR_LDO2_BIT, 0,
+		NULL, 0),
+	SND_SOC_DAPM_SUPPLY("PLL", RT5665_PWR_ANLG_3, RT5665_PWR_PLL_BIT, 0,
+		NULL, 0),
+	SND_SOC_DAPM_SUPPLY("Mic Det Power", RT5665_PWR_VOL,
+		RT5665_PWR_MIC_DET_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("Vref1", RT5665_PWR_ANLG_1, RT5665_PWR_VREF1_BIT, 0,
+		rt5655_set_verf, SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU),
+	SND_SOC_DAPM_SUPPLY("Vref2", RT5665_PWR_ANLG_1, RT5665_PWR_VREF2_BIT, 0,
+		rt5655_set_verf, SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU),
+	SND_SOC_DAPM_SUPPLY("Vref3", RT5665_PWR_ANLG_1, RT5665_PWR_VREF3_BIT, 0,
+		rt5655_set_verf, SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMU),
+
+	/* ASRC */
+	SND_SOC_DAPM_SUPPLY_S("I2S1 ASRC", 1, RT5665_ASRC_1,
+		RT5665_I2S1_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2S2 ASRC", 1, RT5665_ASRC_1,
+		RT5665_I2S2_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2S3 ASRC", 1, RT5665_ASRC_1,
+		RT5665_I2S3_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("DAC STO1 ASRC", 1, RT5665_ASRC_1,
+		RT5665_DAC_STO1_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("DAC STO2 ASRC", 1, RT5665_ASRC_1,
+		RT5665_DAC_STO2_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("DAC Mono L ASRC", 1, RT5665_ASRC_1,
+		RT5665_DAC_MONO_L_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("DAC Mono R ASRC", 1, RT5665_ASRC_1,
+		RT5665_DAC_MONO_R_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("ADC STO1 ASRC", 1, RT5665_ASRC_1,
+		RT5665_ADC_STO1_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("ADC Mono L ASRC", 1, RT5665_ASRC_1,
+		RT5665_ADC_MONO_L_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("ADC Mono R ASRC", 1, RT5665_ASRC_1,
+		RT5665_ADC_MONO_R_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("DMIC STO1 ASRC", 1, RT5665_ASRC_1,
+		RT5665_DMIC_STO1_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("DMIC STO2 ASRC", 1, RT5665_ASRC_1,
+		RT5665_DMIC_STO2_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("DMIC MONO L ASRC", 1, RT5665_ASRC_1,
+		RT5665_DMIC_MONO_L_ASRC_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("DMIC MONO R ASRC", 1, RT5665_ASRC_1,
+		RT5665_DMIC_MONO_R_ASRC_SFT, 0, NULL, 0),
+
+	/* Input Side */
+	SND_SOC_DAPM_SUPPLY("MICBIAS1", RT5665_PWR_ANLG_2, RT5665_PWR_MB1_BIT,
+		0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("MICBIAS2", RT5665_PWR_ANLG_2, RT5665_PWR_MB2_BIT,
+		0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("MICBIAS3", RT5665_PWR_ANLG_2, RT5665_PWR_MB3_BIT,
+		0, NULL, 0),
+
+	/* Input Lines */
+	SND_SOC_DAPM_INPUT("DMIC L1"),
+	SND_SOC_DAPM_INPUT("DMIC R1"),
+	SND_SOC_DAPM_INPUT("DMIC L2"),
+	SND_SOC_DAPM_INPUT("DMIC R2"),
+
+	SND_SOC_DAPM_INPUT("IN1P"),
+	SND_SOC_DAPM_INPUT("IN1N"),
+	SND_SOC_DAPM_INPUT("IN2P"),
+	SND_SOC_DAPM_INPUT("IN2N"),
+	SND_SOC_DAPM_INPUT("IN3P"),
+	SND_SOC_DAPM_INPUT("IN3N"),
+	SND_SOC_DAPM_INPUT("IN4P"),
+	SND_SOC_DAPM_INPUT("IN4N"),
+
+	SND_SOC_DAPM_PGA("DMIC1", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("DMIC2", SND_SOC_NOPM, 0, 0, NULL, 0),
+
+	SND_SOC_DAPM_SUPPLY("DMIC CLK", SND_SOC_NOPM, 0, 0,
+		set_dmic_clk, SND_SOC_DAPM_PRE_PMU),
+	SND_SOC_DAPM_SUPPLY("DMIC1 Power", RT5665_DMIC_CTRL_1,
+		RT5665_DMIC_1_EN_SFT, 0, set_dmic_power, SND_SOC_DAPM_POST_PMU),
+	SND_SOC_DAPM_SUPPLY("DMIC2 Power", RT5665_DMIC_CTRL_1,
+		RT5665_DMIC_2_EN_SFT, 0, set_dmic_power, SND_SOC_DAPM_POST_PMU),
+
+	/* Boost */
+	SND_SOC_DAPM_PGA("BST1", SND_SOC_NOPM,
+		0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("BST2", SND_SOC_NOPM,
+		0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("BST3", SND_SOC_NOPM,
+		0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("BST4", SND_SOC_NOPM,
+		0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("BST1 CBJ", SND_SOC_NOPM,
+		0, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("BST1 Power", RT5665_PWR_ANLG_2,
+		RT5665_PWR_BST1_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("BST2 Power", RT5665_PWR_ANLG_2,
+		RT5665_PWR_BST2_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("BST3 Power", RT5665_PWR_ANLG_2,
+		RT5665_PWR_BST3_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("BST4 Power", RT5665_PWR_ANLG_2,
+		RT5665_PWR_BST4_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("BST1P Power", RT5665_PWR_ANLG_2,
+		RT5665_PWR_BST1_P_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("BST2P Power", RT5665_PWR_ANLG_2,
+		RT5665_PWR_BST2_P_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("BST3P Power", RT5665_PWR_ANLG_2,
+		RT5665_PWR_BST3_P_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("BST4P Power", RT5665_PWR_ANLG_2,
+		RT5665_PWR_BST4_P_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("CBJ Power", RT5665_PWR_ANLG_3,
+		RT5665_PWR_CBJ_BIT, 0, NULL, 0),
+
+
+	/* Input Volume */
+	SND_SOC_DAPM_PGA("INL VOL", RT5665_PWR_VOL, RT5665_PWR_IN_L_BIT,
+		0, NULL, 0),
+	SND_SOC_DAPM_PGA("INR VOL", RT5665_PWR_VOL, RT5665_PWR_IN_R_BIT,
+		0, NULL, 0),
+
+	/* REC Mixer */
+	SND_SOC_DAPM_MIXER("RECMIX1L", SND_SOC_NOPM, 0, 0, rt5665_rec1_l_mix,
+		ARRAY_SIZE(rt5665_rec1_l_mix)),
+	SND_SOC_DAPM_MIXER("RECMIX1R", SND_SOC_NOPM, 0, 0, rt5665_rec1_r_mix,
+		ARRAY_SIZE(rt5665_rec1_r_mix)),
+	SND_SOC_DAPM_MIXER("RECMIX2L", SND_SOC_NOPM, 0, 0, rt5665_rec2_l_mix,
+		ARRAY_SIZE(rt5665_rec2_l_mix)),
+	SND_SOC_DAPM_MIXER("RECMIX2R", SND_SOC_NOPM, 0, 0, rt5665_rec2_r_mix,
+		ARRAY_SIZE(rt5665_rec2_r_mix)),
+	SND_SOC_DAPM_SUPPLY("RECMIX1L Power", RT5665_PWR_ANLG_2,
+		RT5665_PWR_RM1_L_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("RECMIX1R Power", RT5665_PWR_ANLG_2,
+		RT5665_PWR_RM1_R_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("RECMIX2L Power", RT5665_PWR_MIXER,
+		RT5665_PWR_RM2_L_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("RECMIX2R Power", RT5665_PWR_MIXER,
+		RT5665_PWR_RM2_R_BIT, 0, NULL, 0),
+
+	/* ADCs */
+	SND_SOC_DAPM_ADC("ADC1 L", NULL, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_ADC("ADC1 R", NULL, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_ADC("ADC2 L", NULL, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_ADC("ADC2 R", NULL, SND_SOC_NOPM, 0, 0),
+
+	SND_SOC_DAPM_SUPPLY("ADC1 L Power", RT5665_PWR_DIG_1,
+		RT5665_PWR_ADC_L1_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("ADC1 R Power", RT5665_PWR_DIG_1,
+		RT5665_PWR_ADC_R1_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("ADC2 L Power", RT5665_PWR_DIG_1,
+		RT5665_PWR_ADC_L2_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("ADC2 R Power", RT5665_PWR_DIG_1,
+		RT5665_PWR_ADC_R2_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("ADC1 clock", RT5665_CHOP_ADC,
+		RT5665_CKGEN_ADC1_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("ADC2 clock", RT5665_CHOP_ADC,
+		RT5665_CKGEN_ADC2_SFT, 0, NULL, 0),
+
+	/* ADC Mux */
+	SND_SOC_DAPM_MUX("Stereo1 DMIC L Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto1_dmic_mux),
+	SND_SOC_DAPM_MUX("Stereo1 DMIC R Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto1_dmic_mux),
+	SND_SOC_DAPM_MUX("Stereo1 ADC L1 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto1_adc1l_mux),
+	SND_SOC_DAPM_MUX("Stereo1 ADC R1 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto1_adc1r_mux),
+	SND_SOC_DAPM_MUX("Stereo1 ADC L2 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto1_adc2l_mux),
+	SND_SOC_DAPM_MUX("Stereo1 ADC R2 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto1_adc2r_mux),
+	SND_SOC_DAPM_MUX("Stereo1 ADC L Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto1_adcl_mux),
+	SND_SOC_DAPM_MUX("Stereo1 ADC R Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto1_adcr_mux),
+	SND_SOC_DAPM_MUX("Stereo1 DD L Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto1_dd_l_mux),
+	SND_SOC_DAPM_MUX("Stereo1 DD R Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto1_dd_r_mux),
+	SND_SOC_DAPM_MUX("Mono ADC L2 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_mono_adc_l2_mux),
+	SND_SOC_DAPM_MUX("Mono ADC R2 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_mono_adc_r2_mux),
+	SND_SOC_DAPM_MUX("Mono ADC L1 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_mono_adc_l1_mux),
+	SND_SOC_DAPM_MUX("Mono ADC R1 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_mono_adc_r1_mux),
+	SND_SOC_DAPM_MUX("Mono DMIC L Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_mono_dmic_l_mux),
+	SND_SOC_DAPM_MUX("Mono DMIC R Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_mono_dmic_r_mux),
+	SND_SOC_DAPM_MUX("Mono ADC L Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_mono_adc_l_mux),
+	SND_SOC_DAPM_MUX("Mono ADC R Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_mono_adc_r_mux),
+	SND_SOC_DAPM_MUX("Mono DD L Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_mono_dd_l_mux),
+	SND_SOC_DAPM_MUX("Mono DD R Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_mono_dd_r_mux),
+	SND_SOC_DAPM_MUX("Stereo2 DMIC L Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto2_dmic_mux),
+	SND_SOC_DAPM_MUX("Stereo2 DMIC R Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto2_dmic_mux),
+	SND_SOC_DAPM_MUX("Stereo2 ADC L1 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto2_adc1l_mux),
+	SND_SOC_DAPM_MUX("Stereo2 ADC R1 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto2_adc1r_mux),
+	SND_SOC_DAPM_MUX("Stereo2 ADC L2 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto2_adc2l_mux),
+	SND_SOC_DAPM_MUX("Stereo2 ADC R2 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto2_adc2r_mux),
+	SND_SOC_DAPM_MUX("Stereo2 ADC L Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto2_adcl_mux),
+	SND_SOC_DAPM_MUX("Stereo2 ADC R Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto2_adcr_mux),
+	SND_SOC_DAPM_MUX("Stereo2 DD L Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto2_dd_l_mux),
+	SND_SOC_DAPM_MUX("Stereo2 DD R Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_sto2_dd_r_mux),
+	/* ADC Mixer */
+	SND_SOC_DAPM_SUPPLY("ADC Stereo1 Filter", RT5665_PWR_DIG_2,
+		RT5665_PWR_ADC_S1F_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("ADC Stereo2 Filter", RT5665_PWR_DIG_2,
+		RT5665_PWR_ADC_S2F_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("Stereo1 ADC MIXL", RT5665_STO1_ADC_DIG_VOL,
+		RT5665_L_MUTE_SFT, 1, rt5665_sto1_adc_l_mix,
+		ARRAY_SIZE(rt5665_sto1_adc_l_mix)),
+	SND_SOC_DAPM_MIXER("Stereo1 ADC MIXR", RT5665_STO1_ADC_DIG_VOL,
+		RT5665_R_MUTE_SFT, 1, rt5665_sto1_adc_r_mix,
+		ARRAY_SIZE(rt5665_sto1_adc_r_mix)),
+	SND_SOC_DAPM_MIXER("Stereo2 ADC MIXL", RT5665_STO2_ADC_DIG_VOL,
+		RT5665_L_MUTE_SFT, 1, rt5665_sto2_adc_l_mix,
+		ARRAY_SIZE(rt5665_sto2_adc_l_mix)),
+	SND_SOC_DAPM_MIXER("Stereo2 ADC MIXR", RT5665_STO2_ADC_DIG_VOL,
+		RT5665_R_MUTE_SFT, 1, rt5665_sto2_adc_r_mix,
+		ARRAY_SIZE(rt5665_sto2_adc_r_mix)),
+	SND_SOC_DAPM_SUPPLY("ADC Mono Left Filter", RT5665_PWR_DIG_2,
+		RT5665_PWR_ADC_MF_L_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("Mono ADC MIXL", RT5665_MONO_ADC_DIG_VOL,
+		RT5665_L_MUTE_SFT, 1, rt5665_mono_adc_l_mix,
+		ARRAY_SIZE(rt5665_mono_adc_l_mix)),
+	SND_SOC_DAPM_SUPPLY("ADC Mono Right Filter", RT5665_PWR_DIG_2,
+		RT5665_PWR_ADC_MF_R_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("Mono ADC MIXR", RT5665_MONO_ADC_DIG_VOL,
+		RT5665_R_MUTE_SFT, 1, rt5665_mono_adc_r_mix,
+		ARRAY_SIZE(rt5665_mono_adc_r_mix)),
+
+	/* ADC PGA */
+	SND_SOC_DAPM_PGA("Stereo1 ADC MIX", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("Stereo2 ADC MIX", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("Mono ADC MIX", SND_SOC_NOPM, 0, 0, NULL, 0),
+
+	/* Digital Interface */
+	SND_SOC_DAPM_SUPPLY("I2S1_1", RT5665_PWR_DIG_1, RT5665_PWR_I2S1_1_BIT,
+		0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("I2S1_2", RT5665_PWR_DIG_1, RT5665_PWR_I2S1_2_BIT,
+		0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("I2S2_1", RT5665_PWR_DIG_1, RT5665_PWR_I2S2_1_BIT,
+		0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("I2S2_2", RT5665_PWR_DIG_1, RT5665_PWR_I2S2_2_BIT,
+		0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("I2S3", RT5665_PWR_DIG_1, RT5665_PWR_I2S3_BIT,
+		0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF1 DAC1", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF1 DAC2", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF1 DAC3", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF1 DAC1 L", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF1 DAC1 R", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF1 DAC2 L", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF1 DAC2 R", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF1 DAC3 L", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF1 DAC3 R", SND_SOC_NOPM, 0, 0, NULL, 0),
+
+	SND_SOC_DAPM_PGA("IF2_1 DAC", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF2_2 DAC", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF2_1 DAC L", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF2_1 DAC R", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF2_2 DAC L", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF2_2 DAC R", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF2_1 ADC", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF2_2 ADC", SND_SOC_NOPM, 0, 0, NULL, 0),
+
+	SND_SOC_DAPM_PGA("IF3 DAC", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF3 DAC L", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF3 DAC R", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF3 ADC", SND_SOC_NOPM, 0, 0, NULL, 0),
+
+	/* Digital Interface Select */
+	SND_SOC_DAPM_MUX("IF1_1_ADC1 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_if1_1_adc1_mux),
+	SND_SOC_DAPM_MUX("IF1_1_ADC2 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_if1_1_adc2_mux),
+	SND_SOC_DAPM_MUX("IF1_1_ADC3 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_if1_1_adc3_mux),
+	SND_SOC_DAPM_PGA("IF1_1_ADC4", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MUX("IF1_2_ADC1 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_if1_2_adc1_mux),
+	SND_SOC_DAPM_MUX("IF1_2_ADC2 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_if1_2_adc2_mux),
+	SND_SOC_DAPM_MUX("IF1_2_ADC3 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_if1_2_adc3_mux),
+	SND_SOC_DAPM_MUX("IF1_2_ADC4 Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_if1_2_adc4_mux),
+	SND_SOC_DAPM_MUX("TDM1 slot 01 Data Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_tdm1_adc_mux),
+	SND_SOC_DAPM_MUX("TDM1 slot 23 Data Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_tdm1_adc_mux),
+	SND_SOC_DAPM_MUX("TDM1 slot 45 Data Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_tdm1_adc_mux),
+	SND_SOC_DAPM_MUX("TDM1 slot 67 Data Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_tdm1_adc_mux),
+	SND_SOC_DAPM_MUX("TDM2 slot 01 Data Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_tdm2_adc_mux),
+	SND_SOC_DAPM_MUX("TDM2 slot 23 Data Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_tdm2_adc_mux),
+	SND_SOC_DAPM_MUX("TDM2 slot 45 Data Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_tdm2_adc_mux),
+	SND_SOC_DAPM_MUX("TDM2 slot 67 Data Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_tdm2_adc_mux),
+	SND_SOC_DAPM_MUX("IF2_1 ADC Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_if2_1_adc_in_mux),
+	SND_SOC_DAPM_MUX("IF2_2 ADC Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_if2_2_adc_in_mux),
+	SND_SOC_DAPM_MUX("IF3 ADC Mux", SND_SOC_NOPM, 0, 0,
+		&rt5665_if3_adc_in_mux),
+	SND_SOC_DAPM_MUX("IF1_1 0 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_1_01_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_1 1 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_1_01_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_1 2 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_1_23_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_1 3 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_1_23_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_1 4 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_1_45_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_1 5 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_1_45_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_1 6 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_1_67_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_1 7 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_1_67_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_2 0 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_2_01_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_2 1 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_2_01_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_2 2 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_2_23_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_2 3 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_2_23_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_2 4 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_2_45_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_2 5 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_2_45_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_2 6 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_2_67_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF1_2 7 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if1_2_67_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF2_1 DAC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if2_1_dac_swap_mux),
+	SND_SOC_DAPM_MUX("IF2_1 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if2_1_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF2_2 DAC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if2_2_dac_swap_mux),
+	SND_SOC_DAPM_MUX("IF2_2 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if2_2_adc_swap_mux),
+	SND_SOC_DAPM_MUX("IF3 DAC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if3_dac_swap_mux),
+	SND_SOC_DAPM_MUX("IF3 ADC Swap Mux", SND_SOC_NOPM, 0, 0,
+			&rt5665_if3_adc_swap_mux),
+
+	/* Audio Interface */
+	SND_SOC_DAPM_AIF_OUT("AIF1_1TX slot 0", "AIF1_1 Capture",
+				0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_1TX slot 1", "AIF1_1 Capture",
+				1, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_1TX slot 2", "AIF1_1 Capture",
+				2, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_1TX slot 3", "AIF1_1 Capture",
+				3, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_1TX slot 4", "AIF1_1 Capture",
+				4, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_1TX slot 5", "AIF1_1 Capture",
+				5, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_1TX slot 6", "AIF1_1 Capture",
+				6, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_1TX slot 7", "AIF1_1 Capture",
+				7, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_2TX slot 0", "AIF1_2 Capture",
+				0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_2TX slot 1", "AIF1_2 Capture",
+				1, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_2TX slot 2", "AIF1_2 Capture",
+				2, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_2TX slot 3", "AIF1_2 Capture",
+				3, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_2TX slot 4", "AIF1_2 Capture",
+				4, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_2TX slot 5", "AIF1_2 Capture",
+				5, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_2TX slot 6", "AIF1_2 Capture",
+				6, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF1_2TX slot 7", "AIF1_2 Capture",
+				7, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF2_1TX", "AIF2_1 Capture",
+				0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF2_2TX", "AIF2_2 Capture",
+				0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AIF3TX", "AIF3 Capture",
+				0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_IN("AIF1RX", "AIF1 Playback",
+				0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_IN("AIF2_1RX", "AIF2_1 Playback",
+				0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_IN("AIF2_2RX", "AIF2_2 Playback",
+				0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_IN("AIF3RX", "AIF3 Playback",
+				0, SND_SOC_NOPM, 0, 0),
+
+	/* Output Side */
+	/* DAC mixer before sound effect  */
+	SND_SOC_DAPM_MIXER("DAC1 MIXL", SND_SOC_NOPM, 0, 0,
+		rt5665_dac_l_mix, ARRAY_SIZE(rt5665_dac_l_mix)),
+	SND_SOC_DAPM_MIXER("DAC1 MIXR", SND_SOC_NOPM, 0, 0,
+		rt5665_dac_r_mix, ARRAY_SIZE(rt5665_dac_r_mix)),
+
+	/* DAC channel Mux */
+	SND_SOC_DAPM_MUX("DAC L1 Mux", SND_SOC_NOPM, 0, 0, &rt5665_dac_l1_mux),
+	SND_SOC_DAPM_MUX("DAC R1 Mux", SND_SOC_NOPM, 0, 0, &rt5665_dac_r1_mux),
+	SND_SOC_DAPM_MUX("DAC L2 Mux", SND_SOC_NOPM, 0, 0, &rt5665_dac_l2_mux),
+	SND_SOC_DAPM_MUX("DAC R2 Mux", SND_SOC_NOPM, 0, 0, &rt5665_dac_r2_mux),
+	SND_SOC_DAPM_MUX("DAC L3 Mux", SND_SOC_NOPM, 0, 0, &rt5665_dac_l3_mux),
+	SND_SOC_DAPM_MUX("DAC R3 Mux", SND_SOC_NOPM, 0, 0, &rt5665_dac_r3_mux),
+
+	SND_SOC_DAPM_MUX("DAC L1 Source", SND_SOC_NOPM, 0, 0,
+		&rt5665_alg_dac_l1_mux),
+	SND_SOC_DAPM_MUX("DAC R1 Source", SND_SOC_NOPM, 0, 0,
+		&rt5665_alg_dac_r1_mux),
+	SND_SOC_DAPM_MUX("DAC L2 Source", SND_SOC_NOPM, 0, 0,
+		&rt5665_alg_dac_l2_mux),
+	SND_SOC_DAPM_MUX("DAC R2 Source", SND_SOC_NOPM, 0, 0,
+		&rt5665_alg_dac_r2_mux),
+
+	/* DAC Mixer */
+	SND_SOC_DAPM_SUPPLY("DAC Stereo1 Filter", RT5665_PWR_DIG_2,
+		RT5665_PWR_DAC_S1F_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("DAC Stereo2 Filter", RT5665_PWR_DIG_2,
+		RT5665_PWR_DAC_S2F_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("DAC Mono Left Filter", RT5665_PWR_DIG_2,
+		RT5665_PWR_DAC_MF_L_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("DAC Mono Right Filter", RT5665_PWR_DIG_2,
+		RT5665_PWR_DAC_MF_R_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("Stereo1 DAC MIXL", SND_SOC_NOPM, 0, 0,
+		rt5665_sto1_dac_l_mix, ARRAY_SIZE(rt5665_sto1_dac_l_mix)),
+	SND_SOC_DAPM_MIXER("Stereo1 DAC MIXR", SND_SOC_NOPM, 0, 0,
+		rt5665_sto1_dac_r_mix, ARRAY_SIZE(rt5665_sto1_dac_r_mix)),
+	SND_SOC_DAPM_MIXER("Stereo2 DAC MIXL", SND_SOC_NOPM, 0, 0,
+		rt5665_sto2_dac_l_mix, ARRAY_SIZE(rt5665_sto2_dac_l_mix)),
+	SND_SOC_DAPM_MIXER("Stereo2 DAC MIXR", SND_SOC_NOPM, 0, 0,
+		rt5665_sto2_dac_r_mix, ARRAY_SIZE(rt5665_sto2_dac_r_mix)),
+	SND_SOC_DAPM_MIXER("Mono DAC MIXL", SND_SOC_NOPM, 0, 0,
+		rt5665_mono_dac_l_mix, ARRAY_SIZE(rt5665_mono_dac_l_mix)),
+	SND_SOC_DAPM_MIXER("Mono DAC MIXR", SND_SOC_NOPM, 0, 0,
+		rt5665_mono_dac_r_mix, ARRAY_SIZE(rt5665_mono_dac_r_mix)),
+	SND_SOC_DAPM_MUX("DAC MIXL", SND_SOC_NOPM, 0, 0,
+		&rt5665_dig_dac_mixl_mux),
+	SND_SOC_DAPM_MUX("DAC MIXR", SND_SOC_NOPM, 0, 0,
+		&rt5665_dig_dac_mixr_mux),
+
+	/* DACs */
+	SND_SOC_DAPM_DAC("DAC L1", NULL, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_DAC("DAC R1", NULL, SND_SOC_NOPM, 0, 0),
+
+	SND_SOC_DAPM_SUPPLY("DAC L2 Power", RT5665_PWR_DIG_1,
+		RT5665_PWR_DAC_L2_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("DAC R2 Power", RT5665_PWR_DIG_1,
+		RT5665_PWR_DAC_R2_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_DAC("DAC L2", NULL, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_DAC("DAC R2", NULL, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_PGA("DAC1 MIX", SND_SOC_NOPM, 0, 0, NULL, 0),
+
+	SND_SOC_DAPM_SUPPLY_S("DAC 1 Clock", 1, RT5665_CHOP_DAC,
+		RT5665_CKGEN_DAC1_SFT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("DAC 2 Clock", 1, RT5665_CHOP_DAC,
+		RT5665_CKGEN_DAC2_SFT, 0, NULL, 0),
+
+	/* OUT Mixer */
+	SND_SOC_DAPM_MIXER("MONOVOL MIX", RT5665_PWR_MIXER, RT5665_PWR_MM_BIT,
+		0, rt5665_monovol_mix, ARRAY_SIZE(rt5665_monovol_mix)),
+	SND_SOC_DAPM_MIXER("OUT MIXL", RT5665_PWR_MIXER, RT5665_PWR_OM_L_BIT,
+		0, rt5665_out_l_mix, ARRAY_SIZE(rt5665_out_l_mix)),
+	SND_SOC_DAPM_MIXER("OUT MIXR", RT5665_PWR_MIXER, RT5665_PWR_OM_R_BIT,
+		0, rt5665_out_r_mix, ARRAY_SIZE(rt5665_out_r_mix)),
+
+	/* Output Volume */
+	SND_SOC_DAPM_SWITCH("MONOVOL", RT5665_PWR_VOL, RT5665_PWR_MV_BIT, 0,
+		&monovol_switch),
+	SND_SOC_DAPM_SWITCH("OUTVOL L", RT5665_PWR_VOL, RT5665_PWR_OV_L_BIT, 0,
+		&outvol_l_switch),
+	SND_SOC_DAPM_SWITCH("OUTVOL R", RT5665_PWR_VOL, RT5665_PWR_OV_R_BIT, 0,
+		&outvol_r_switch),
+
+	/* MONO/HPO/LOUT */
+	SND_SOC_DAPM_MIXER("Mono MIX", SND_SOC_NOPM, 0,	0, rt5665_mono_mix,
+		ARRAY_SIZE(rt5665_mono_mix)),
+	SND_SOC_DAPM_MIXER("LOUT L MIX", SND_SOC_NOPM, 0, 0, rt5665_lout_l_mix,
+		ARRAY_SIZE(rt5665_lout_l_mix)),
+	SND_SOC_DAPM_MIXER("LOUT R MIX", SND_SOC_NOPM, 0, 0, rt5665_lout_r_mix,
+		ARRAY_SIZE(rt5665_lout_r_mix)),
+	SND_SOC_DAPM_PGA_S("Mono Amp", 1, RT5665_PWR_ANLG_1, RT5665_PWR_MA_BIT,
+		0, rt5665_mono_event, SND_SOC_DAPM_POST_PMD |
+		SND_SOC_DAPM_PRE_PMU),
+	SND_SOC_DAPM_PGA_S("HP Amp", 1, SND_SOC_NOPM, 0, 0, rt5665_hp_event,
+		SND_SOC_DAPM_POST_PMD | SND_SOC_DAPM_PRE_PMU),
+	SND_SOC_DAPM_PGA_S("LOUT Amp", 1, RT5665_PWR_ANLG_1,
+		RT5665_PWR_LM_BIT, 0, rt5665_lout_event,
+		SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD |
+		SND_SOC_DAPM_POST_PMD | SND_SOC_DAPM_PRE_PMU),
+
+	SND_SOC_DAPM_SUPPLY("Charge Pump", SND_SOC_NOPM, 0, 0,
+		rt5665_charge_pump_event, SND_SOC_DAPM_PRE_PMU |
+		SND_SOC_DAPM_POST_PMD),
+
+	SND_SOC_DAPM_SWITCH("Mono Playback", SND_SOC_NOPM, 0, 0,
+		&mono_switch),
+	SND_SOC_DAPM_SWITCH("HPO Playback", SND_SOC_NOPM, 0, 0,
+		&hpo_switch),
+	SND_SOC_DAPM_SWITCH("LOUT L Playback", SND_SOC_NOPM, 0, 0,
+		&lout_l_switch),
+	SND_SOC_DAPM_SWITCH("LOUT R Playback", SND_SOC_NOPM, 0, 0,
+		&lout_r_switch),
+	SND_SOC_DAPM_SWITCH("PDM L Playback", SND_SOC_NOPM, 0, 0,
+		&pdm_l_switch),
+	SND_SOC_DAPM_SWITCH("PDM R Playback", SND_SOC_NOPM, 0, 0,
+		&pdm_r_switch),
+
+	/* PDM */
+	SND_SOC_DAPM_SUPPLY("PDM Power", RT5665_PWR_DIG_2,
+		RT5665_PWR_PDM1_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_MUX("PDM L Mux", SND_SOC_NOPM,
+		0, 1, &rt5665_pdm_l_mux),
+	SND_SOC_DAPM_MUX("PDM R Mux", SND_SOC_NOPM,
+		0, 1, &rt5665_pdm_r_mux),
+
+	/* CLK DET */
+	SND_SOC_DAPM_SUPPLY("CLKDET SYS", RT5665_CLK_DET, RT5665_SYS_CLK_DET,
+		0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("CLKDET HP", RT5665_CLK_DET, RT5665_HP_CLK_DET,
+		0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("CLKDET MONO", RT5665_CLK_DET, RT5665_MONO_CLK_DET,
+		0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("CLKDET LOUT", RT5665_CLK_DET, RT5665_LOUT_CLK_DET,
+		0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("CLKDET", RT5665_CLK_DET, RT5665_POW_CLK_DET,
+		0, NULL, 0),
+
+	/* Output Lines */
+	SND_SOC_DAPM_OUTPUT("HPOL"),
+	SND_SOC_DAPM_OUTPUT("HPOR"),
+	SND_SOC_DAPM_OUTPUT("LOUTL"),
+	SND_SOC_DAPM_OUTPUT("LOUTR"),
+	SND_SOC_DAPM_OUTPUT("MONOOUT"),
+	SND_SOC_DAPM_OUTPUT("PDML"),
+	SND_SOC_DAPM_OUTPUT("PDMR"),
+};
+
+static const struct snd_soc_dapm_route rt5665_dapm_routes[] = {
+	/*PLL*/
+	{"ADC Stereo1 Filter", NULL, "PLL", is_sys_clk_from_pll},
+	{"ADC Stereo2 Filter", NULL, "PLL", is_sys_clk_from_pll},
+	{"ADC Mono Left Filter", NULL, "PLL", is_sys_clk_from_pll},
+	{"ADC Mono Right Filter", NULL, "PLL", is_sys_clk_from_pll},
+	{"DAC Stereo1 Filter", NULL, "PLL", is_sys_clk_from_pll},
+	{"DAC Stereo2 Filter", NULL, "PLL", is_sys_clk_from_pll},
+	{"DAC Mono Left Filter", NULL, "PLL", is_sys_clk_from_pll},
+	{"DAC Mono Right Filter", NULL, "PLL", is_sys_clk_from_pll},
+
+	/*ASRC*/
+	{"ADC Stereo1 Filter", NULL, "ADC STO1 ASRC", is_using_asrc},
+	{"ADC Mono Left Filter", NULL, "ADC Mono L ASRC", is_using_asrc},
+	{"ADC Mono Right Filter", NULL, "ADC Mono R ASRC", is_using_asrc},
+	{"DAC Mono Left Filter", NULL, "DAC Mono L ASRC", is_using_asrc},
+	{"DAC Mono Right Filter", NULL, "DAC Mono R ASRC", is_using_asrc},
+	{"DAC Stereo1 Filter", NULL, "DAC STO1 ASRC", is_using_asrc},
+	{"DAC Stereo2 Filter", NULL, "DAC STO2 ASRC", is_using_asrc},
+
+	/*Vref*/
+	{"Mic Det Power", NULL, "Vref2"},
+	{"MICBIAS1", NULL, "Vref1"},
+	{"MICBIAS1", NULL, "Vref2"},
+	{"MICBIAS2", NULL, "Vref1"},
+	{"MICBIAS2", NULL, "Vref2"},
+	{"MICBIAS3", NULL, "Vref1"},
+	{"MICBIAS3", NULL, "Vref2"},
+
+	{"Stereo1 DMIC L Mux", NULL, "DMIC STO1 ASRC"},
+	{"Stereo1 DMIC R Mux", NULL, "DMIC STO1 ASRC"},
+	{"Stereo2 DMIC L Mux", NULL, "DMIC STO2 ASRC"},
+	{"Stereo2 DMIC R Mux", NULL, "DMIC STO2 ASRC"},
+	{"Mono DMIC L Mux", NULL, "DMIC MONO L ASRC"},
+	{"Mono DMIC R Mux", NULL, "DMIC MONO R ASRC"},
+
+	{"I2S1_1", NULL, "I2S1 ASRC"},
+	{"I2S1_2", NULL, "I2S1 ASRC"},
+	{"I2S2_1", NULL, "I2S2 ASRC"},
+	{"I2S2_2", NULL, "I2S2 ASRC"},
+	{"I2S3", NULL, "I2S3 ASRC"},
+
+	{"CLKDET SYS", NULL, "CLKDET"},
+	{"CLKDET HP", NULL, "CLKDET"},
+	{"CLKDET MONO", NULL, "CLKDET"},
+	{"CLKDET LOUT", NULL, "CLKDET"},
+
+	{"IN1P", NULL, "LDO2"},
+	{"IN2P", NULL, "LDO2"},
+	{"IN3P", NULL, "LDO2"},
+	{"IN4P", NULL, "LDO2"},
+
+	{"DMIC1", NULL, "DMIC L1"},
+	{"DMIC1", NULL, "DMIC R1"},
+	{"DMIC2", NULL, "DMIC L2"},
+	{"DMIC2", NULL, "DMIC R2"},
+
+	{"BST1", NULL, "IN1P"},
+	{"BST1", NULL, "IN1N"},
+	{"BST1", NULL, "BST1 Power"},
+	{"BST1", NULL, "BST1P Power"},
+	{"BST2", NULL, "IN2P"},
+	{"BST2", NULL, "IN2N"},
+	{"BST2", NULL, "BST2 Power"},
+	{"BST2", NULL, "BST2P Power"},
+	{"BST3", NULL, "IN3P"},
+	{"BST3", NULL, "IN3N"},
+	{"BST3", NULL, "BST3 Power"},
+	{"BST3", NULL, "BST3P Power"},
+	{"BST4", NULL, "IN4P"},
+	{"BST4", NULL, "IN4N"},
+	{"BST4", NULL, "BST4 Power"},
+	{"BST4", NULL, "BST4P Power"},
+	{"BST1 CBJ", NULL, "IN1P"},
+	{"BST1 CBJ", NULL, "IN1N"},
+	{"BST1 CBJ", NULL, "CBJ Power"},
+	{"CBJ Power", NULL, "Vref2"},
+
+	{"INL VOL", NULL, "IN3P"},
+	{"INR VOL", NULL, "IN3N"},
+
+	{"RECMIX1L", "CBJ Switch", "BST1 CBJ"},
+	{"RECMIX1L", "INL Switch", "INL VOL"},
+	{"RECMIX1L", "INR Switch", "INR VOL"},
+	{"RECMIX1L", "BST4 Switch", "BST4"},
+	{"RECMIX1L", "BST3 Switch", "BST3"},
+	{"RECMIX1L", "BST2 Switch", "BST2"},
+	{"RECMIX1L", "BST1 Switch", "BST1"},
+	{"RECMIX1L", NULL, "RECMIX1L Power"},
+
+	{"RECMIX1R", "MONOVOL Switch", "MONOVOL"},
+	{"RECMIX1R", "INR Switch", "INR VOL"},
+	{"RECMIX1R", "BST4 Switch", "BST4"},
+	{"RECMIX1R", "BST3 Switch", "BST3"},
+	{"RECMIX1R", "BST2 Switch", "BST2"},
+	{"RECMIX1R", "BST1 Switch", "BST1"},
+	{"RECMIX1R", NULL, "RECMIX1R Power"},
+
+	{"RECMIX2L", "CBJ Switch", "BST1 CBJ"},
+	{"RECMIX2L", "INL Switch", "INL VOL"},
+	{"RECMIX2L", "INR Switch", "INR VOL"},
+	{"RECMIX2L", "BST4 Switch", "BST4"},
+	{"RECMIX2L", "BST3 Switch", "BST3"},
+	{"RECMIX2L", "BST2 Switch", "BST2"},
+	{"RECMIX2L", "BST1 Switch", "BST1"},
+	{"RECMIX2L", NULL, "RECMIX2L Power"},
+
+	{"RECMIX2R", "MONOVOL Switch", "MONOVOL"},
+	{"RECMIX2R", "INL Switch", "INL VOL"},
+	{"RECMIX2R", "INR Switch", "INR VOL"},
+	{"RECMIX2R", "BST4 Switch", "BST4"},
+	{"RECMIX2R", "BST3 Switch", "BST3"},
+	{"RECMIX2R", "BST2 Switch", "BST2"},
+	{"RECMIX2R", "BST1 Switch", "BST1"},
+	{"RECMIX2R", NULL, "RECMIX2R Power"},
+
+	{"ADC1 L", NULL, "RECMIX1L"},
+	{"ADC1 L", NULL, "ADC1 L Power"},
+	{"ADC1 L", NULL, "ADC1 clock"},
+	{"ADC1 R", NULL, "RECMIX1R"},
+	{"ADC1 R", NULL, "ADC1 R Power"},
+	{"ADC1 R", NULL, "ADC1 clock"},
+
+	{"ADC2 L", NULL, "RECMIX2L"},
+	{"ADC2 L", NULL, "ADC2 L Power"},
+	{"ADC2 L", NULL, "ADC2 clock"},
+	{"ADC2 R", NULL, "RECMIX2R"},
+	{"ADC2 R", NULL, "ADC2 R Power"},
+	{"ADC2 R", NULL, "ADC2 clock"},
+
+	{"DMIC L1", NULL, "DMIC CLK"},
+	{"DMIC L1", NULL, "DMIC1 Power"},
+	{"DMIC R1", NULL, "DMIC CLK"},
+	{"DMIC R1", NULL, "DMIC1 Power"},
+	{"DMIC L2", NULL, "DMIC CLK"},
+	{"DMIC L2", NULL, "DMIC2 Power"},
+	{"DMIC R2", NULL, "DMIC CLK"},
+	{"DMIC R2", NULL, "DMIC2 Power"},
+
+	{"Stereo1 DMIC L Mux", "DMIC1", "DMIC L1"},
+	{"Stereo1 DMIC L Mux", "DMIC2", "DMIC L2"},
+
+	{"Stereo1 DMIC R Mux", "DMIC1", "DMIC R1"},
+	{"Stereo1 DMIC R Mux", "DMIC2", "DMIC R2"},
+
+	{"Mono DMIC L Mux", "DMIC1 L", "DMIC L1"},
+	{"Mono DMIC L Mux", "DMIC2 L", "DMIC L2"},
+
+	{"Mono DMIC R Mux", "DMIC1 R", "DMIC R1"},
+	{"Mono DMIC R Mux", "DMIC2 R", "DMIC R2"},
+
+	{"Stereo2 DMIC L Mux", "DMIC1", "DMIC L1"},
+	{"Stereo2 DMIC L Mux", "DMIC2", "DMIC L2"},
+
+	{"Stereo2 DMIC R Mux", "DMIC1", "DMIC R1"},
+	{"Stereo2 DMIC R Mux", "DMIC2", "DMIC R2"},
+
+	{"Stereo1 ADC L Mux", "ADC1 L", "ADC1 L"},
+	{"Stereo1 ADC L Mux", "ADC1 R", "ADC1 R"},
+	{"Stereo1 ADC L Mux", "ADC2 L", "ADC2 L"},
+	{"Stereo1 ADC L Mux", "ADC2 R", "ADC2 R"},
+	{"Stereo1 ADC R Mux", "ADC1 L", "ADC1 L"},
+	{"Stereo1 ADC R Mux", "ADC1 R", "ADC1 R"},
+	{"Stereo1 ADC R Mux", "ADC2 L", "ADC2 L"},
+	{"Stereo1 ADC R Mux", "ADC2 R", "ADC2 R"},
+
+	{"Stereo1 DD L Mux", "STO2 DAC", "Stereo2 DAC MIXL"},
+	{"Stereo1 DD L Mux", "MONO DAC", "Mono DAC MIXL"},
+
+	{"Stereo1 DD R Mux", "STO2 DAC", "Stereo2 DAC MIXR"},
+	{"Stereo1 DD R Mux", "MONO DAC", "Mono DAC MIXR"},
+
+	{"Stereo1 ADC L1 Mux", "ADC", "Stereo1 ADC L Mux"},
+	{"Stereo1 ADC L1 Mux", "DD Mux", "Stereo1 DD L Mux"},
+	{"Stereo1 ADC L2 Mux", "DMIC", "Stereo1 DMIC L Mux"},
+	{"Stereo1 ADC L2 Mux", "DAC MIX", "DAC MIXL"},
+
+	{"Stereo1 ADC R1 Mux", "ADC", "Stereo1 ADC R Mux"},
+	{"Stereo1 ADC R1 Mux", "DD Mux", "Stereo1 DD R Mux"},
+	{"Stereo1 ADC R2 Mux", "DMIC", "Stereo1 DMIC R Mux"},
+	{"Stereo1 ADC R2 Mux", "DAC MIX", "DAC MIXR"},
+
+	{"Mono ADC L Mux", "ADC1 L", "ADC1 L"},
+	{"Mono ADC L Mux", "ADC1 R", "ADC1 R"},
+	{"Mono ADC L Mux", "ADC2 L", "ADC2 L"},
+	{"Mono ADC L Mux", "ADC2 R", "ADC2 R"},
+
+	{"Mono ADC R Mux", "ADC1 L", "ADC1 L"},
+	{"Mono ADC R Mux", "ADC1 R", "ADC1 R"},
+	{"Mono ADC R Mux", "ADC2 L", "ADC2 L"},
+	{"Mono ADC R Mux", "ADC2 R", "ADC2 R"},
+
+	{"Mono DD L Mux", "STO2 DAC", "Stereo2 DAC MIXL"},
+	{"Mono DD L Mux", "MONO DAC", "Mono DAC MIXL"},
+
+	{"Mono DD R Mux", "STO2 DAC", "Stereo2 DAC MIXR"},
+	{"Mono DD R Mux", "MONO DAC", "Mono DAC MIXR"},
+
+	{"Mono ADC L2 Mux", "DMIC", "Mono DMIC L Mux"},
+	{"Mono ADC L2 Mux", "DAC MIXL", "DAC MIXL"},
+	{"Mono ADC L1 Mux", "DD Mux", "Mono DD L Mux"},
+	{"Mono ADC L1 Mux", "ADC",  "Mono ADC L Mux"},
+
+	{"Mono ADC R1 Mux", "DD Mux", "Mono DD R Mux"},
+	{"Mono ADC R1 Mux", "ADC", "Mono ADC R Mux"},
+	{"Mono ADC R2 Mux", "DMIC", "Mono DMIC R Mux"},
+	{"Mono ADC R2 Mux", "DAC MIXR", "DAC MIXR"},
+
+	{"Stereo2 ADC L Mux", "ADC1 L", "ADC1 L"},
+	{"Stereo2 ADC L Mux", "ADC2 L", "ADC2 L"},
+	{"Stereo2 ADC L Mux", "ADC1 R", "ADC1 R"},
+	{"Stereo2 ADC R Mux", "ADC1 L", "ADC1 L"},
+	{"Stereo2 ADC R Mux", "ADC2 L", "ADC2 L"},
+	{"Stereo2 ADC R Mux", "ADC1 R", "ADC1 R"},
+
+	{"Stereo2 DD L Mux", "STO2 DAC", "Stereo2 DAC MIXL"},
+	{"Stereo2 DD L Mux", "MONO DAC", "Mono DAC MIXL"},
+
+	{"Stereo2 DD R Mux", "STO2 DAC", "Stereo2 DAC MIXR"},
+	{"Stereo2 DD R Mux", "MONO DAC", "Mono DAC MIXR"},
+
+	{"Stereo2 ADC L1 Mux", "ADC", "Stereo2 ADC L Mux"},
+	{"Stereo2 ADC L1 Mux", "DD Mux", "Stereo2 DD L Mux"},
+	{"Stereo2 ADC L2 Mux", "DMIC", "Stereo2 DMIC L Mux"},
+	{"Stereo2 ADC L2 Mux", "DAC MIX", "DAC MIXL"},
+
+	{"Stereo2 ADC R1 Mux", "ADC", "Stereo2 ADC R Mux"},
+	{"Stereo2 ADC R1 Mux", "DD Mux", "Stereo2 DD R Mux"},
+	{"Stereo2 ADC R2 Mux", "DMIC", "Stereo2 DMIC R Mux"},
+	{"Stereo2 ADC R2 Mux", "DAC MIX", "DAC MIXR"},
+
+	{"Stereo1 ADC MIXL", "ADC1 Switch", "Stereo1 ADC L1 Mux"},
+	{"Stereo1 ADC MIXL", "ADC2 Switch", "Stereo1 ADC L2 Mux"},
+	{"Stereo1 ADC MIXL", NULL, "ADC Stereo1 Filter"},
+
+	{"Stereo1 ADC MIXR", "ADC1 Switch", "Stereo1 ADC R1 Mux"},
+	{"Stereo1 ADC MIXR", "ADC2 Switch", "Stereo1 ADC R2 Mux"},
+	{"Stereo1 ADC MIXR", NULL, "ADC Stereo1 Filter"},
+
+	{"Mono ADC MIXL", "ADC1 Switch", "Mono ADC L1 Mux"},
+	{"Mono ADC MIXL", "ADC2 Switch", "Mono ADC L2 Mux"},
+	{"Mono ADC MIXL", NULL, "ADC Mono Left Filter"},
+
+	{"Mono ADC MIXR", "ADC1 Switch", "Mono ADC R1 Mux"},
+	{"Mono ADC MIXR", "ADC2 Switch", "Mono ADC R2 Mux"},
+	{"Mono ADC MIXR", NULL, "ADC Mono Right Filter"},
+
+	{"Stereo2 ADC MIXL", "ADC1 Switch", "Stereo2 ADC L1 Mux"},
+	{"Stereo2 ADC MIXL", "ADC2 Switch", "Stereo2 ADC L2 Mux"},
+	{"Stereo2 ADC MIXL", NULL, "ADC Stereo2 Filter"},
+
+	{"Stereo2 ADC MIXR", "ADC1 Switch", "Stereo2 ADC R1 Mux"},
+	{"Stereo2 ADC MIXR", "ADC2 Switch", "Stereo2 ADC R2 Mux"},
+	{"Stereo2 ADC MIXR", NULL, "ADC Stereo2 Filter"},
+
+	{"Stereo1 ADC MIX", NULL, "Stereo1 ADC MIXL"},
+	{"Stereo1 ADC MIX", NULL, "Stereo1 ADC MIXR"},
+	{"Stereo2 ADC MIX", NULL, "Stereo2 ADC MIXL"},
+	{"Stereo2 ADC MIX", NULL, "Stereo2 ADC MIXR"},
+	{"Mono ADC MIX", NULL, "Mono ADC MIXL"},
+	{"Mono ADC MIX", NULL, "Mono ADC MIXR"},
+
+	{"IF1_1_ADC1 Mux", "STO1 ADC", "Stereo1 ADC MIX"},
+	{"IF1_1_ADC1 Mux", "IF2_1 DAC", "IF2_1 DAC"},
+	{"IF1_1_ADC2 Mux", "STO2 ADC", "Stereo2 ADC MIX"},
+	{"IF1_1_ADC2 Mux", "IF2_2 DAC", "IF2_2 DAC"},
+	{"IF1_1_ADC3 Mux", "MONO ADC", "Mono ADC MIX"},
+	{"IF1_1_ADC3 Mux", "IF3 DAC", "IF3 DAC"},
+	{"IF1_1_ADC4", NULL, "DAC1 MIX"},
+
+	{"IF1_2_ADC1 Mux", "STO1 ADC", "Stereo1 ADC MIX"},
+	{"IF1_2_ADC1 Mux", "IF1 DAC", "IF1 DAC1"},
+	{"IF1_2_ADC2 Mux", "STO2 ADC", "Stereo2 ADC MIX"},
+	{"IF1_2_ADC2 Mux", "IF2_1 DAC", "IF2_1 DAC"},
+	{"IF1_2_ADC3 Mux", "MONO ADC", "Mono ADC MIX"},
+	{"IF1_2_ADC3 Mux", "IF2_2 DAC", "IF2_2 DAC"},
+	{"IF1_2_ADC4 Mux", "DAC1", "DAC1 MIX"},
+	{"IF1_2_ADC4 Mux", "IF3 DAC", "IF3 DAC"},
+
+	{"TDM1 slot 01 Data Mux", "1234", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 01 Data Mux", "1243", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 01 Data Mux", "1324", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 01 Data Mux", "1342", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 01 Data Mux", "1432", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 01 Data Mux", "1423", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 01 Data Mux", "2134", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 01 Data Mux", "2143", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 01 Data Mux", "2314", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 01 Data Mux", "2341", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 01 Data Mux", "2431", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 01 Data Mux", "2413", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 01 Data Mux", "3124", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 01 Data Mux", "3142", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 01 Data Mux", "3214", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 01 Data Mux", "3241", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 01 Data Mux", "3412", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 01 Data Mux", "3421", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 01 Data Mux", "4123", "IF1_1_ADC4"},
+	{"TDM1 slot 01 Data Mux", "4132", "IF1_1_ADC4"},
+	{"TDM1 slot 01 Data Mux", "4213", "IF1_1_ADC4"},
+	{"TDM1 slot 01 Data Mux", "4231", "IF1_1_ADC4"},
+	{"TDM1 slot 01 Data Mux", "4312", "IF1_1_ADC4"},
+	{"TDM1 slot 01 Data Mux", "4321", "IF1_1_ADC4"},
+	{"TDM1 slot 01 Data Mux", NULL, "I2S1_1"},
+
+	{"TDM1 slot 23 Data Mux", "1234", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 23 Data Mux", "1243", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 23 Data Mux", "1324", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 23 Data Mux", "1342", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 23 Data Mux", "1432", "IF1_1_ADC4"},
+	{"TDM1 slot 23 Data Mux", "1423", "IF1_1_ADC4"},
+	{"TDM1 slot 23 Data Mux", "2134", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 23 Data Mux", "2143", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 23 Data Mux", "2314", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 23 Data Mux", "2341", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 23 Data Mux", "2431", "IF1_1_ADC4"},
+	{"TDM1 slot 23 Data Mux", "2413", "IF1_1_ADC4"},
+	{"TDM1 slot 23 Data Mux", "3124", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 23 Data Mux", "3142", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 23 Data Mux", "3214", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 23 Data Mux", "3241", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 23 Data Mux", "3412", "IF1_1_ADC4"},
+	{"TDM1 slot 23 Data Mux", "3421", "IF1_1_ADC4"},
+	{"TDM1 slot 23 Data Mux", "4123", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 23 Data Mux", "4132", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 23 Data Mux", "4213", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 23 Data Mux", "4231", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 23 Data Mux", "4312", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 23 Data Mux", "4321", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 23 Data Mux", NULL, "I2S1_1"},
+
+	{"TDM1 slot 45 Data Mux", "1234", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 45 Data Mux", "1243", "IF1_1_ADC4"},
+	{"TDM1 slot 45 Data Mux", "1324", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 45 Data Mux", "1342", "IF1_1_ADC4"},
+	{"TDM1 slot 45 Data Mux", "1432", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 45 Data Mux", "1423", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 45 Data Mux", "2134", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 45 Data Mux", "2143", "IF1_1_ADC4"},
+	{"TDM1 slot 45 Data Mux", "2314", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 45 Data Mux", "2341", "IF1_1_ADC4"},
+	{"TDM1 slot 45 Data Mux", "2431", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 45 Data Mux", "2413", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 45 Data Mux", "3124", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 45 Data Mux", "3142", "IF1_1_ADC4"},
+	{"TDM1 slot 45 Data Mux", "3214", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 45 Data Mux", "3241", "IF1_1_ADC4"},
+	{"TDM1 slot 45 Data Mux", "3412", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 45 Data Mux", "3421", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 45 Data Mux", "4123", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 45 Data Mux", "4132", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 45 Data Mux", "4213", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 45 Data Mux", "4231", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 45 Data Mux", "4312", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 45 Data Mux", "4321", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 45 Data Mux", NULL, "I2S1_1"},
+
+	{"TDM1 slot 67 Data Mux", "1234", "IF1_1_ADC4"},
+	{"TDM1 slot 67 Data Mux", "1243", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 67 Data Mux", "1324", "IF1_1_ADC4"},
+	{"TDM1 slot 67 Data Mux", "1342", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 67 Data Mux", "1432", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 67 Data Mux", "1423", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 67 Data Mux", "2134", "IF1_1_ADC4"},
+	{"TDM1 slot 67 Data Mux", "2143", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 67 Data Mux", "2314", "IF1_1_ADC4"},
+	{"TDM1 slot 67 Data Mux", "2341", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 67 Data Mux", "2431", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 67 Data Mux", "2413", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 67 Data Mux", "3124", "IF1_1_ADC4"},
+	{"TDM1 slot 67 Data Mux", "3142", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 67 Data Mux", "3214", "IF1_1_ADC4"},
+	{"TDM1 slot 67 Data Mux", "3241", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 67 Data Mux", "3412", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 67 Data Mux", "3421", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 67 Data Mux", "4123", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 67 Data Mux", "4132", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 67 Data Mux", "4213", "IF1_1_ADC3 Mux"},
+	{"TDM1 slot 67 Data Mux", "4231", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 67 Data Mux", "4312", "IF1_1_ADC2 Mux"},
+	{"TDM1 slot 67 Data Mux", "4321", "IF1_1_ADC1 Mux"},
+	{"TDM1 slot 67 Data Mux", NULL, "I2S1_1"},
+
+
+	{"TDM2 slot 01 Data Mux", "1234", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 01 Data Mux", "1243", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 01 Data Mux", "1324", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 01 Data Mux", "1342", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 01 Data Mux", "1432", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 01 Data Mux", "1423", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 01 Data Mux", "2134", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 01 Data Mux", "2143", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 01 Data Mux", "2314", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 01 Data Mux", "2341", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 01 Data Mux", "2431", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 01 Data Mux", "2413", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 01 Data Mux", "3124", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 01 Data Mux", "3142", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 01 Data Mux", "3214", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 01 Data Mux", "3241", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 01 Data Mux", "3412", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 01 Data Mux", "3421", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 01 Data Mux", "4123", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 01 Data Mux", "4132", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 01 Data Mux", "4213", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 01 Data Mux", "4231", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 01 Data Mux", "4312", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 01 Data Mux", "4321", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 01 Data Mux", NULL, "I2S1_2"},
+
+	{"TDM2 slot 23 Data Mux", "1234", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 23 Data Mux", "1243", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 23 Data Mux", "1324", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 23 Data Mux", "1342", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 23 Data Mux", "1432", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 23 Data Mux", "1423", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 23 Data Mux", "2134", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 23 Data Mux", "2143", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 23 Data Mux", "2314", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 23 Data Mux", "2341", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 23 Data Mux", "2431", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 23 Data Mux", "2413", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 23 Data Mux", "3124", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 23 Data Mux", "3142", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 23 Data Mux", "3214", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 23 Data Mux", "3241", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 23 Data Mux", "3412", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 23 Data Mux", "3421", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 23 Data Mux", "4123", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 23 Data Mux", "4132", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 23 Data Mux", "4213", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 23 Data Mux", "4231", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 23 Data Mux", "4312", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 23 Data Mux", "4321", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 23 Data Mux", NULL, "I2S1_2"},
+
+	{"TDM2 slot 45 Data Mux", "1234", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 45 Data Mux", "1243", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 45 Data Mux", "1324", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 45 Data Mux", "1342", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 45 Data Mux", "1432", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 45 Data Mux", "1423", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 45 Data Mux", "2134", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 45 Data Mux", "2143", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 45 Data Mux", "2314", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 45 Data Mux", "2341", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 45 Data Mux", "2431", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 45 Data Mux", "2413", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 45 Data Mux", "3124", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 45 Data Mux", "3142", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 45 Data Mux", "3214", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 45 Data Mux", "3241", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 45 Data Mux", "3412", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 45 Data Mux", "3421", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 45 Data Mux", "4123", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 45 Data Mux", "4132", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 45 Data Mux", "4213", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 45 Data Mux", "4231", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 45 Data Mux", "4312", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 45 Data Mux", "4321", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 45 Data Mux", NULL, "I2S1_2"},
+
+	{"TDM2 slot 67 Data Mux", "1234", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 67 Data Mux", "1243", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 67 Data Mux", "1324", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 67 Data Mux", "1342", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 67 Data Mux", "1432", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 67 Data Mux", "1423", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 67 Data Mux", "2134", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 67 Data Mux", "2143", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 67 Data Mux", "2314", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 67 Data Mux", "2341", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 67 Data Mux", "2431", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 67 Data Mux", "2413", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 67 Data Mux", "3124", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 67 Data Mux", "3142", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 67 Data Mux", "3214", "IF1_2_ADC4 Mux"},
+	{"TDM2 slot 67 Data Mux", "3241", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 67 Data Mux", "3412", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 67 Data Mux", "3421", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 67 Data Mux", "4123", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 67 Data Mux", "4132", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 67 Data Mux", "4213", "IF1_2_ADC3 Mux"},
+	{"TDM2 slot 67 Data Mux", "4231", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 67 Data Mux", "4312", "IF1_2_ADC2 Mux"},
+	{"TDM2 slot 67 Data Mux", "4321", "IF1_2_ADC1 Mux"},
+	{"TDM2 slot 67 Data Mux", NULL, "I2S1_2"},
+
+	{"IF1_1 0 ADC Swap Mux", "L/R", "TDM1 slot 01 Data Mux"},
+	{"IF1_1 0 ADC Swap Mux", "L/L", "TDM1 slot 01 Data Mux"},
+	{"IF1_1 1 ADC Swap Mux", "R/L", "TDM1 slot 01 Data Mux"},
+	{"IF1_1 1 ADC Swap Mux", "R/R", "TDM1 slot 01 Data Mux"},
+	{"IF1_1 2 ADC Swap Mux", "L/R", "TDM1 slot 23 Data Mux"},
+	{"IF1_1 2 ADC Swap Mux", "R/L", "TDM1 slot 23 Data Mux"},
+	{"IF1_1 3 ADC Swap Mux", "L/L", "TDM1 slot 23 Data Mux"},
+	{"IF1_1 3 ADC Swap Mux", "R/R", "TDM1 slot 23 Data Mux"},
+	{"IF1_1 4 ADC Swap Mux", "L/R", "TDM1 slot 45 Data Mux"},
+	{"IF1_1 4 ADC Swap Mux", "R/L", "TDM1 slot 45 Data Mux"},
+	{"IF1_1 5 ADC Swap Mux", "L/L", "TDM1 slot 45 Data Mux"},
+	{"IF1_1 5 ADC Swap Mux", "R/R", "TDM1 slot 45 Data Mux"},
+	{"IF1_1 6 ADC Swap Mux", "L/R", "TDM1 slot 67 Data Mux"},
+	{"IF1_1 6 ADC Swap Mux", "R/L", "TDM1 slot 67 Data Mux"},
+	{"IF1_1 7 ADC Swap Mux", "L/L", "TDM1 slot 67 Data Mux"},
+	{"IF1_1 7 ADC Swap Mux", "R/R", "TDM1 slot 67 Data Mux"},
+	{"IF1_2 0 ADC Swap Mux", "L/R", "TDM2 slot 01 Data Mux"},
+	{"IF1_2 0 ADC Swap Mux", "R/L", "TDM2 slot 01 Data Mux"},
+	{"IF1_2 1 ADC Swap Mux", "L/L", "TDM2 slot 01 Data Mux"},
+	{"IF1_2 1 ADC Swap Mux", "R/R", "TDM2 slot 01 Data Mux"},
+	{"IF1_2 2 ADC Swap Mux", "L/R", "TDM2 slot 23 Data Mux"},
+	{"IF1_2 2 ADC Swap Mux", "R/L", "TDM2 slot 23 Data Mux"},
+	{"IF1_2 3 ADC Swap Mux", "L/L", "TDM2 slot 23 Data Mux"},
+	{"IF1_2 3 ADC Swap Mux", "R/R", "TDM2 slot 23 Data Mux"},
+	{"IF1_2 4 ADC Swap Mux", "L/R", "TDM2 slot 45 Data Mux"},
+	{"IF1_2 4 ADC Swap Mux", "R/L", "TDM2 slot 45 Data Mux"},
+	{"IF1_2 5 ADC Swap Mux", "L/L", "TDM2 slot 45 Data Mux"},
+	{"IF1_2 5 ADC Swap Mux", "R/R", "TDM2 slot 45 Data Mux"},
+	{"IF1_2 6 ADC Swap Mux", "L/R", "TDM2 slot 67 Data Mux"},
+	{"IF1_2 6 ADC Swap Mux", "R/L", "TDM2 slot 67 Data Mux"},
+	{"IF1_2 7 ADC Swap Mux", "L/L", "TDM2 slot 67 Data Mux"},
+	{"IF1_2 7 ADC Swap Mux", "R/R", "TDM2 slot 67 Data Mux"},
+
+	{"IF2_1 ADC Mux", "STO1 ADC", "Stereo1 ADC MIX"},
+	{"IF2_1 ADC Mux", "STO2 ADC", "Stereo2 ADC MIX"},
+	{"IF2_1 ADC Mux", "MONO ADC", "Mono ADC MIX"},
+	{"IF2_1 ADC Mux", "IF1 DAC1", "IF1 DAC1"},
+	{"IF2_1 ADC Mux", "IF1 DAC2", "IF1 DAC2"},
+	{"IF2_1 ADC Mux", "IF2_2 DAC", "IF2_2 DAC"},
+	{"IF2_1 ADC Mux", "IF3 DAC", "IF3 DAC"},
+	{"IF2_1 ADC Mux", "DAC1 MIX", "DAC1 MIX"},
+	{"IF2_1 ADC", NULL, "IF2_1 ADC Mux"},
+	{"IF2_1 ADC", NULL, "I2S2_1"},
+
+	{"IF2_2 ADC Mux", "STO1 ADC", "Stereo1 ADC MIX"},
+	{"IF2_2 ADC Mux", "STO2 ADC", "Stereo2 ADC MIX"},
+	{"IF2_2 ADC Mux", "MONO ADC", "Mono ADC MIX"},
+	{"IF2_2 ADC Mux", "IF1 DAC1", "IF1 DAC1"},
+	{"IF2_2 ADC Mux", "IF1 DAC2", "IF1 DAC2"},
+	{"IF2_2 ADC Mux", "IF2_1 DAC", "IF2_1 DAC"},
+	{"IF2_2 ADC Mux", "IF3 DAC", "IF3 DAC"},
+	{"IF2_2 ADC Mux", "DAC1 MIX", "DAC1 MIX"},
+	{"IF2_2 ADC", NULL, "IF2_2 ADC Mux"},
+	{"IF2_2 ADC", NULL, "I2S2_2"},
+
+	{"IF3 ADC Mux", "STO1 ADC", "Stereo1 ADC MIX"},
+	{"IF3 ADC Mux", "STO2 ADC", "Stereo2 ADC MIX"},
+	{"IF3 ADC Mux", "MONO ADC", "Mono ADC MIX"},
+	{"IF3 ADC Mux", "IF1 DAC1", "IF1 DAC1"},
+	{"IF3 ADC Mux", "IF1 DAC2", "IF1 DAC2"},
+	{"IF3 ADC Mux", "IF2_1 DAC", "IF2_1 DAC"},
+	{"IF3 ADC Mux", "IF2_2 DAC", "IF2_2 DAC"},
+	{"IF3 ADC Mux", "DAC1 MIX", "DAC1 MIX"},
+	{"IF3 ADC", NULL, "IF3 ADC Mux"},
+	{"IF3 ADC", NULL, "I2S3"},
+
+	{"AIF1_1TX slot 0", NULL, "IF1_1 0 ADC Swap Mux"},
+	{"AIF1_1TX slot 1", NULL, "IF1_1 1 ADC Swap Mux"},
+	{"AIF1_1TX slot 2", NULL, "IF1_1 2 ADC Swap Mux"},
+	{"AIF1_1TX slot 3", NULL, "IF1_1 3 ADC Swap Mux"},
+	{"AIF1_1TX slot 4", NULL, "IF1_1 4 ADC Swap Mux"},
+	{"AIF1_1TX slot 5", NULL, "IF1_1 5 ADC Swap Mux"},
+	{"AIF1_1TX slot 6", NULL, "IF1_1 6 ADC Swap Mux"},
+	{"AIF1_1TX slot 7", NULL, "IF1_1 7 ADC Swap Mux"},
+	{"AIF1_2TX slot 0", NULL, "IF1_2 0 ADC Swap Mux"},
+	{"AIF1_2TX slot 1", NULL, "IF1_2 1 ADC Swap Mux"},
+	{"AIF1_2TX slot 2", NULL, "IF1_2 2 ADC Swap Mux"},
+	{"AIF1_2TX slot 3", NULL, "IF1_2 3 ADC Swap Mux"},
+	{"AIF1_2TX slot 4", NULL, "IF1_2 4 ADC Swap Mux"},
+	{"AIF1_2TX slot 5", NULL, "IF1_2 5 ADC Swap Mux"},
+	{"AIF1_2TX slot 6", NULL, "IF1_2 6 ADC Swap Mux"},
+	{"AIF1_2TX slot 7", NULL, "IF1_2 7 ADC Swap Mux"},
+	{"IF2_1 ADC Swap Mux", "L/R", "IF2_1 ADC"},
+	{"IF2_1 ADC Swap Mux", "R/L", "IF2_1 ADC"},
+	{"IF2_1 ADC Swap Mux", "L/L", "IF2_1 ADC"},
+	{"IF2_1 ADC Swap Mux", "R/R", "IF2_1 ADC"},
+	{"AIF2_1TX", NULL, "IF2_1 ADC Swap Mux"},
+	{"IF2_2 ADC Swap Mux", "L/R", "IF2_2 ADC"},
+	{"IF2_2 ADC Swap Mux", "R/L", "IF2_2 ADC"},
+	{"IF2_2 ADC Swap Mux", "L/L", "IF2_2 ADC"},
+	{"IF2_2 ADC Swap Mux", "R/R", "IF2_2 ADC"},
+	{"AIF2_2TX", NULL, "IF2_2 ADC Swap Mux"},
+	{"IF3 ADC Swap Mux", "L/R", "IF3 ADC"},
+	{"IF3 ADC Swap Mux", "R/L", "IF3 ADC"},
+	{"IF3 ADC Swap Mux", "L/L", "IF3 ADC"},
+	{"IF3 ADC Swap Mux", "R/R", "IF3 ADC"},
+	{"AIF3TX", NULL, "IF3 ADC Swap Mux"},
+
+	{"IF1 DAC1", NULL, "AIF1RX"},
+	{"IF1 DAC2", NULL, "AIF1RX"},
+	{"IF1 DAC3", NULL, "AIF1RX"},
+	{"IF2_1 DAC Swap Mux", "L/R", "AIF2_1RX"},
+	{"IF2_1 DAC Swap Mux", "R/L", "AIF2_1RX"},
+	{"IF2_1 DAC Swap Mux", "L/L", "AIF2_1RX"},
+	{"IF2_1 DAC Swap Mux", "R/R", "AIF2_1RX"},
+	{"IF2_2 DAC Swap Mux", "L/R", "AIF2_2RX"},
+	{"IF2_2 DAC Swap Mux", "R/L", "AIF2_2RX"},
+	{"IF2_2 DAC Swap Mux", "L/L", "AIF2_2RX"},
+	{"IF2_2 DAC Swap Mux", "R/R", "AIF2_2RX"},
+	{"IF2_1 DAC", NULL, "IF2_1 DAC Swap Mux"},
+	{"IF2_2 DAC", NULL, "IF2_2 DAC Swap Mux"},
+	{"IF3 DAC Swap Mux", "L/R", "AIF3RX"},
+	{"IF3 DAC Swap Mux", "R/L", "AIF3RX"},
+	{"IF3 DAC Swap Mux", "L/L", "AIF3RX"},
+	{"IF3 DAC Swap Mux", "R/R", "AIF3RX"},
+	{"IF3 DAC", NULL, "IF3 DAC Swap Mux"},
+
+	{"IF1 DAC1", NULL, "I2S1_1"},
+	{"IF1 DAC2", NULL, "I2S1_1"},
+	{"IF1 DAC3", NULL, "I2S1_1"},
+	{"IF2_1 DAC", NULL, "I2S2_1"},
+	{"IF2_2 DAC", NULL, "I2S2_2"},
+	{"IF3 DAC", NULL, "I2S3"},
+
+	{"IF1 DAC1 L", NULL, "IF1 DAC1"},
+	{"IF1 DAC1 R", NULL, "IF1 DAC1"},
+	{"IF1 DAC2 L", NULL, "IF1 DAC2"},
+	{"IF1 DAC2 R", NULL, "IF1 DAC2"},
+	{"IF1 DAC3 L", NULL, "IF1 DAC3"},
+	{"IF1 DAC3 R", NULL, "IF1 DAC3"},
+	{"IF2_1 DAC L", NULL, "IF2_1 DAC"},
+	{"IF2_1 DAC R", NULL, "IF2_1 DAC"},
+	{"IF2_2 DAC L", NULL, "IF2_2 DAC"},
+	{"IF2_2 DAC R", NULL, "IF2_2 DAC"},
+	{"IF3 DAC L", NULL, "IF3 DAC"},
+	{"IF3 DAC R", NULL, "IF3 DAC"},
+
+	{"DAC L1 Mux", "IF1 DAC1", "IF1 DAC1 L"},
+	{"DAC L1 Mux", "IF2_1 DAC", "IF2_1 DAC L"},
+	{"DAC L1 Mux", "IF2_2 DAC", "IF2_2 DAC L"},
+	{"DAC L1 Mux", "IF3 DAC", "IF3 DAC L"},
+	{"DAC L1 Mux", NULL, "DAC Stereo1 Filter"},
+
+	{"DAC R1 Mux", "IF1 DAC1", "IF1 DAC1 R"},
+	{"DAC R1 Mux", "IF2_1 DAC", "IF2_1 DAC R"},
+	{"DAC R1 Mux", "IF2_2 DAC", "IF2_2 DAC R"},
+	{"DAC R1 Mux", "IF3 DAC", "IF3 DAC R"},
+	{"DAC R1 Mux", NULL, "DAC Stereo1 Filter"},
+
+	{"DAC1 MIXL", "Stereo ADC Switch", "Stereo1 ADC MIXL"},
+	{"DAC1 MIXL", "DAC1 Switch", "DAC L1 Mux"},
+	{"DAC1 MIXR", "Stereo ADC Switch", "Stereo1 ADC MIXR"},
+	{"DAC1 MIXR", "DAC1 Switch", "DAC R1 Mux"},
+
+	{"DAC1 MIX", NULL, "DAC1 MIXL"},
+	{"DAC1 MIX", NULL, "DAC1 MIXR"},
+
+	{"DAC L2 Mux", "IF1 DAC2", "IF1 DAC2 L"},
+	{"DAC L2 Mux", "IF2_1 DAC", "IF2_1 DAC L"},
+	{"DAC L2 Mux", "IF2_2 DAC", "IF2_2 DAC L"},
+	{"DAC L2 Mux", "IF3 DAC", "IF3 DAC L"},
+	{"DAC L2 Mux", "Mono ADC MIX", "Mono ADC MIXL"},
+	{"DAC L2 Mux", NULL, "DAC Mono Left Filter"},
+
+	{"DAC R2 Mux", "IF1 DAC2", "IF1 DAC2 R"},
+	{"DAC R2 Mux", "IF2_1 DAC", "IF2_1 DAC R"},
+	{"DAC R2 Mux", "IF2_2 DAC", "IF2_2 DAC R"},
+	{"DAC R2 Mux", "IF3 DAC", "IF3 DAC R"},
+	{"DAC R2 Mux", "Mono ADC MIX", "Mono ADC MIXR"},
+	{"DAC R2 Mux", NULL, "DAC Mono Right Filter"},
+
+	{"DAC L3 Mux", "IF1 DAC2", "IF1 DAC2 L"},
+	{"DAC L3 Mux", "IF2_1 DAC", "IF2_1 DAC L"},
+	{"DAC L3 Mux", "IF2_2 DAC", "IF2_2 DAC L"},
+	{"DAC L3 Mux", "IF3 DAC", "IF3 DAC L"},
+	{"DAC L3 Mux", "STO2 ADC MIX", "Stereo2 ADC MIXL"},
+	{"DAC L3 Mux", NULL, "DAC Stereo2 Filter"},
+
+	{"DAC R3 Mux", "IF1 DAC2", "IF1 DAC2 R"},
+	{"DAC R3 Mux", "IF2_1 DAC", "IF2_1 DAC R"},
+	{"DAC R3 Mux", "IF2_2 DAC", "IF2_2 DAC R"},
+	{"DAC R3 Mux", "IF3 DAC", "IF3 DAC R"},
+	{"DAC R3 Mux", "STO2 ADC MIX", "Stereo2 ADC MIXR"},
+	{"DAC R3 Mux", NULL, "DAC Stereo2 Filter"},
+
+	{"Stereo1 DAC MIXL", "DAC L1 Switch", "DAC1 MIXL"},
+	{"Stereo1 DAC MIXL", "DAC R1 Switch", "DAC1 MIXR"},
+	{"Stereo1 DAC MIXL", "DAC L2 Switch", "DAC L2 Mux"},
+	{"Stereo1 DAC MIXL", "DAC R2 Switch", "DAC R2 Mux"},
+
+	{"Stereo1 DAC MIXR", "DAC R1 Switch", "DAC1 MIXR"},
+	{"Stereo1 DAC MIXR", "DAC L1 Switch", "DAC1 MIXL"},
+	{"Stereo1 DAC MIXR", "DAC L2 Switch", "DAC L2 Mux"},
+	{"Stereo1 DAC MIXR", "DAC R2 Switch", "DAC R2 Mux"},
+
+	{"Stereo2 DAC MIXL", "DAC L1 Switch", "DAC1 MIXL"},
+	{"Stereo2 DAC MIXL", "DAC L2 Switch", "DAC L2 Mux"},
+	{"Stereo2 DAC MIXL", "DAC L3 Switch", "DAC L3 Mux"},
+
+	{"Stereo2 DAC MIXR", "DAC R1 Switch", "DAC1 MIXR"},
+	{"Stereo2 DAC MIXR", "DAC R2 Switch", "DAC R2 Mux"},
+	{"Stereo2 DAC MIXR", "DAC R3 Switch", "DAC R3 Mux"},
+
+	{"Mono DAC MIXL", "DAC L1 Switch", "DAC1 MIXL"},
+	{"Mono DAC MIXL", "DAC R1 Switch", "DAC1 MIXR"},
+	{"Mono DAC MIXL", "DAC L2 Switch", "DAC L2 Mux"},
+	{"Mono DAC MIXL", "DAC R2 Switch", "DAC R2 Mux"},
+	{"Mono DAC MIXR", "DAC L1 Switch", "DAC1 MIXL"},
+	{"Mono DAC MIXR", "DAC R1 Switch", "DAC1 MIXR"},
+	{"Mono DAC MIXR", "DAC L2 Switch", "DAC L2 Mux"},
+	{"Mono DAC MIXR", "DAC R2 Switch", "DAC R2 Mux"},
+
+	{"DAC MIXL", "Stereo1 DAC Mixer", "Stereo1 DAC MIXL"},
+	{"DAC MIXL", "Stereo2 DAC Mixer", "Stereo2 DAC MIXL"},
+	{"DAC MIXL", "Mono DAC Mixer", "Mono DAC MIXL"},
+	{"DAC MIXR", "Stereo1 DAC Mixer", "Stereo1 DAC MIXR"},
+	{"DAC MIXR", "Stereo2 DAC Mixer", "Stereo2 DAC MIXR"},
+	{"DAC MIXR", "Mono DAC Mixer", "Mono DAC MIXR"},
+
+	{"DAC L1 Source", "DAC1", "DAC1 MIXL"},
+	{"DAC L1 Source", "Stereo1 DAC Mixer", "Stereo1 DAC MIXL"},
+	{"DAC L1 Source", "DMIC1", "DMIC L1"},
+	{"DAC R1 Source", "DAC1", "DAC1 MIXR"},
+	{"DAC R1 Source", "Stereo1 DAC Mixer", "Stereo1 DAC MIXR"},
+	{"DAC R1 Source", "DMIC1", "DMIC R1"},
+
+	{"DAC L2 Source", "DAC2", "DAC L2 Mux"},
+	{"DAC L2 Source", "Mono DAC Mixer", "Mono DAC MIXL"},
+	{"DAC L2 Source", NULL, "DAC L2 Power"},
+	{"DAC R2 Source", "DAC2", "DAC R2 Mux"},
+	{"DAC R2 Source", "Mono DAC Mixer", "Mono DAC MIXR"},
+	{"DAC R2 Source", NULL, "DAC R2 Power"},
+
+	{"DAC L1", NULL, "DAC L1 Source"},
+	{"DAC R1", NULL, "DAC R1 Source"},
+	{"DAC L2", NULL, "DAC L2 Source"},
+	{"DAC R2", NULL, "DAC R2 Source"},
+
+	{"DAC L1", NULL, "DAC 1 Clock"},
+	{"DAC R1", NULL, "DAC 1 Clock"},
+	{"DAC L2", NULL, "DAC 2 Clock"},
+	{"DAC R2", NULL, "DAC 2 Clock"},
+
+	{"MONOVOL MIX", "DAC L2 Switch", "DAC L2"},
+	{"MONOVOL MIX", "RECMIX2L Switch", "RECMIX2L"},
+	{"MONOVOL MIX", "BST1 Switch", "BST1"},
+	{"MONOVOL MIX", "BST2 Switch", "BST2"},
+	{"MONOVOL MIX", "BST3 Switch", "BST3"},
+
+	{"OUT MIXL", "DAC L2 Switch", "DAC L2"},
+	{"OUT MIXL", "INL Switch", "INL VOL"},
+	{"OUT MIXL", "BST1 Switch", "BST1"},
+	{"OUT MIXL", "BST2 Switch", "BST2"},
+	{"OUT MIXL", "BST3 Switch", "BST3"},
+	{"OUT MIXR", "DAC R2 Switch", "DAC R2"},
+	{"OUT MIXR", "INR Switch", "INR VOL"},
+	{"OUT MIXR", "BST2 Switch", "BST2"},
+	{"OUT MIXR", "BST3 Switch", "BST3"},
+	{"OUT MIXR", "BST4 Switch", "BST4"},
+
+	{"MONOVOL", "Switch", "MONOVOL MIX"},
+	{"Mono MIX", "DAC L2 Switch", "DAC L2"},
+	{"Mono MIX", "MONOVOL Switch", "MONOVOL"},
+	{"Mono Amp", NULL, "Mono MIX"},
+	{"Mono Amp", NULL, "Vref2"},
+	{"Mono Amp", NULL, "CLKDET SYS"},
+	{"Mono Amp", NULL, "CLKDET MONO"},
+	{"Mono Playback", "Switch", "Mono Amp"},
+	{"MONOOUT", NULL, "Mono Playback"},
+
+	{"HP Amp", NULL, "DAC L1"},
+	{"HP Amp", NULL, "DAC R1"},
+	{"HP Amp", NULL, "Charge Pump"},
+	{"HP Amp", NULL, "CLKDET SYS"},
+	{"HP Amp", NULL, "CLKDET HP"},
+	{"HP Amp", NULL, "CBJ Power"},
+	{"HP Amp", NULL, "Vref2"},
+	{"HPO Playback", "Switch", "HP Amp"},
+	{"HPOL", NULL, "HPO Playback"},
+	{"HPOR", NULL, "HPO Playback"},
+
+	{"OUTVOL L", "Switch", "OUT MIXL"},
+	{"OUTVOL R", "Switch", "OUT MIXR"},
+	{"LOUT L MIX", "DAC L2 Switch", "DAC L2"},
+	{"LOUT L MIX", "OUTVOL L Switch", "OUTVOL L"},
+	{"LOUT R MIX", "DAC R2 Switch", "DAC R2"},
+	{"LOUT R MIX", "OUTVOL R Switch", "OUTVOL R"},
+	{"LOUT Amp", NULL, "LOUT L MIX"},
+	{"LOUT Amp", NULL, "LOUT R MIX"},
+	{"LOUT Amp", NULL, "Vref1"},
+	{"LOUT Amp", NULL, "Vref2"},
+	{"LOUT Amp", NULL, "CLKDET SYS"},
+	{"LOUT Amp", NULL, "CLKDET LOUT"},
+	{"LOUT L Playback", "Switch", "LOUT Amp"},
+	{"LOUT R Playback", "Switch", "LOUT Amp"},
+	{"LOUTL", NULL, "LOUT L Playback"},
+	{"LOUTR", NULL, "LOUT R Playback"},
+
+	{"PDM L Mux", "Mono DAC", "Mono DAC MIXL"},
+	{"PDM L Mux", "Stereo1 DAC", "Stereo1 DAC MIXL"},
+	{"PDM L Mux", "Stereo2 DAC", "Stereo2 DAC MIXL"},
+	{"PDM L Mux", NULL, "PDM Power"},
+	{"PDM R Mux", "Mono DAC", "Mono DAC MIXR"},
+	{"PDM R Mux", "Stereo1 DAC", "Stereo1 DAC MIXR"},
+	{"PDM R Mux", "Stereo2 DAC", "Stereo2 DAC MIXR"},
+	{"PDM R Mux", NULL, "PDM Power"},
+	{"PDM L Playback", "Switch", "PDM L Mux"},
+	{"PDM R Playback", "Switch", "PDM R Mux"},
+	{"PDML", NULL, "PDM L Playback"},
+	{"PDMR", NULL, "PDM R Playback"},
+};
+
+static int rt5665_hw_params(struct snd_pcm_substream *substream,
+	struct snd_pcm_hw_params *params, struct snd_soc_dai *dai)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+	unsigned int val_len = 0, val_clk, mask_clk, val_bits = 0x0100;
+	int pre_div, frame_size;
+
+	rt5665->lrck[dai->id] = params_rate(params);
+	pre_div = rl6231_get_clk_info(rt5665->sysclk, rt5665->lrck[dai->id]);
+	if (pre_div < 0) {
+		dev_err(codec->dev, "Unsupported clock setting %d for DAI %d\n",
+			rt5665->lrck[dai->id], dai->id);
+		return -EINVAL;
+	}
+	frame_size = snd_soc_params_to_frame_size(params);
+	if (frame_size < 0) {
+		dev_err(codec->dev, "Unsupported frame size: %d\n", frame_size);
+		return -EINVAL;
+	}
+
+	dev_dbg(dai->dev, "lrck is %dHz and pre_div is %d for iis %d\n",
+				rt5665->lrck[dai->id], pre_div, dai->id);
+
+	switch (params_width(params)) {
+	case 16:
+		val_bits = 0x0100;
+		break;
+	case 20:
+		val_len |= RT5665_I2S_DL_20;
+		val_bits = 0x1300;
+		break;
+	case 24:
+		val_len |= RT5665_I2S_DL_24;
+		val_bits = 0x2500;
+		break;
+	case 8:
+		val_len |= RT5665_I2S_DL_8;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (dai->id) {
+	case RT5665_AIF1_1:
+	case RT5665_AIF1_2:
+		mask_clk = RT5665_I2S_PD1_MASK;
+		val_clk = pre_div << RT5665_I2S_PD1_SFT;
+		snd_soc_update_bits(codec, RT5665_I2S1_SDP,
+			RT5665_I2S_DL_MASK, val_len);
+		break;
+	case RT5665_AIF2_1:
+	case RT5665_AIF2_2:
+		mask_clk = RT5665_I2S_PD2_MASK;
+		val_clk = pre_div << RT5665_I2S_PD2_SFT;
+		snd_soc_update_bits(codec, RT5665_I2S2_SDP,
+			RT5665_I2S_DL_MASK, val_len);
+		break;
+	case RT5665_AIF3:
+		mask_clk = RT5665_I2S_PD3_MASK;
+		val_clk = pre_div << RT5665_I2S_PD3_SFT;
+		snd_soc_update_bits(codec, RT5665_I2S3_SDP,
+			RT5665_I2S_DL_MASK, val_len);
+		break;
+	default:
+		dev_err(codec->dev, "Invalid dai->id: %d\n", dai->id);
+		return -EINVAL;
+	}
+
+	snd_soc_update_bits(codec, RT5665_ADDA_CLK_1, mask_clk, val_clk);
+	snd_soc_update_bits(codec, RT5665_STO1_DAC_SIL_DET, 0x3700, val_bits);
+
+	switch (rt5665->lrck[dai->id]) {
+	case 192000:
+		snd_soc_update_bits(codec, RT5665_ADDA_CLK_1,
+			RT5665_DAC_OSR_MASK | RT5665_ADC_OSR_MASK,
+			RT5665_DAC_OSR_32 | RT5665_ADC_OSR_32);
+		break;
+	case 96000:
+		snd_soc_update_bits(codec, RT5665_ADDA_CLK_1,
+			RT5665_DAC_OSR_MASK | RT5665_ADC_OSR_MASK,
+			RT5665_DAC_OSR_64 | RT5665_ADC_OSR_64);
+		break;
+	default:
+		snd_soc_update_bits(codec, RT5665_ADDA_CLK_1,
+			RT5665_DAC_OSR_MASK | RT5665_ADC_OSR_MASK,
+			RT5665_DAC_OSR_128 | RT5665_ADC_OSR_128);
+		break;
+	}
+
+	return 0;
+}
+
+static int rt5665_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+	unsigned int reg_val = 0;
+
+	switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) {
+	case SND_SOC_DAIFMT_CBM_CFM:
+		rt5665->master[dai->id] = 1;
+		break;
+	case SND_SOC_DAIFMT_CBS_CFS:
+		reg_val |= RT5665_I2S_MS_S;
+		rt5665->master[dai->id] = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
+	case SND_SOC_DAIFMT_NB_NF:
+		break;
+	case SND_SOC_DAIFMT_IB_NF:
+		reg_val |= RT5665_I2S_BP_INV;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+		break;
+	case SND_SOC_DAIFMT_LEFT_J:
+		reg_val |= RT5665_I2S_DF_LEFT;
+		break;
+	case SND_SOC_DAIFMT_DSP_A:
+		reg_val |= RT5665_I2S_DF_PCM_A;
+		break;
+	case SND_SOC_DAIFMT_DSP_B:
+		reg_val |= RT5665_I2S_DF_PCM_B;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (dai->id) {
+	case RT5665_AIF1_1:
+	case RT5665_AIF1_2:
+		snd_soc_update_bits(codec, RT5665_I2S1_SDP,
+			RT5665_I2S_MS_MASK | RT5665_I2S_BP_MASK |
+			RT5665_I2S_DF_MASK, reg_val);
+		break;
+	case RT5665_AIF2_1:
+	case RT5665_AIF2_2:
+		snd_soc_update_bits(codec, RT5665_I2S2_SDP,
+			RT5665_I2S_MS_MASK | RT5665_I2S_BP_MASK |
+			RT5665_I2S_DF_MASK, reg_val);
+		break;
+	case RT5665_AIF3:
+		snd_soc_update_bits(codec, RT5665_I2S3_SDP,
+			RT5665_I2S_MS_MASK | RT5665_I2S_BP_MASK |
+			RT5665_I2S_DF_MASK, reg_val);
+		break;
+	default:
+		dev_err(codec->dev, "Invalid dai->id: %d\n", dai->id);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int rt5665_set_dai_sysclk(struct snd_soc_dai *dai,
+		int clk_id, unsigned int freq, int dir)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+	unsigned int reg_val = 0;
+
+	if (freq == rt5665->sysclk && clk_id == rt5665->sysclk_src)
+		return 0;
+
+	switch (clk_id) {
+	case RT5665_SCLK_S_MCLK:
+		reg_val |= RT5665_SCLK_SRC_MCLK;
+		break;
+	case RT5665_SCLK_S_PLL1:
+		reg_val |= RT5665_SCLK_SRC_PLL1;
+		break;
+	case RT5665_SCLK_S_RCCLK:
+		reg_val |= RT5665_SCLK_SRC_RCCLK;
+		break;
+	default:
+		dev_err(codec->dev, "Invalid clock id (%d)\n", clk_id);
+		return -EINVAL;
+	}
+	snd_soc_update_bits(codec, RT5665_GLB_CLK,
+		RT5665_SCLK_SRC_MASK, reg_val);
+	rt5665->sysclk = freq;
+	rt5665->sysclk_src = clk_id;
+
+	dev_dbg(dai->dev, "Sysclk is %dHz and clock id is %d\n", freq, clk_id);
+
+	return 0;
+}
+
+static int rt5665_set_dai_pll(struct snd_soc_dai *dai, int pll_id, int Source,
+			unsigned int freq_in, unsigned int freq_out)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+	struct rl6231_pll_code pll_code;
+	int ret;
+
+	if (Source == rt5665->pll_src && freq_in == rt5665->pll_in &&
+	    freq_out == rt5665->pll_out)
+		return 0;
+
+	if (!freq_in || !freq_out) {
+		dev_dbg(codec->dev, "PLL disabled\n");
+
+		rt5665->pll_in = 0;
+		rt5665->pll_out = 0;
+		snd_soc_update_bits(codec, RT5665_GLB_CLK,
+			RT5665_SCLK_SRC_MASK, RT5665_SCLK_SRC_MCLK);
+		return 0;
+	}
+
+	switch (Source) {
+	case RT5665_PLL1_S_MCLK:
+		snd_soc_update_bits(codec, RT5665_GLB_CLK,
+			RT5665_PLL1_SRC_MASK, RT5665_PLL1_SRC_MCLK);
+		break;
+	case RT5665_PLL1_S_BCLK1:
+		snd_soc_update_bits(codec, RT5665_GLB_CLK,
+				RT5665_PLL1_SRC_MASK, RT5665_PLL1_SRC_BCLK1);
+		break;
+	case RT5665_PLL1_S_BCLK2:
+		snd_soc_update_bits(codec, RT5665_GLB_CLK,
+				RT5665_PLL1_SRC_MASK, RT5665_PLL1_SRC_BCLK2);
+		break;
+	case RT5665_PLL1_S_BCLK3:
+		snd_soc_update_bits(codec, RT5665_GLB_CLK,
+				RT5665_PLL1_SRC_MASK, RT5665_PLL1_SRC_BCLK3);
+		break;
+	default:
+		dev_err(codec->dev, "Unknown PLL Source %d\n", Source);
+		return -EINVAL;
+	}
+
+	ret = rl6231_pll_calc(freq_in, freq_out, &pll_code);
+	if (ret < 0) {
+		dev_err(codec->dev, "Unsupport input clock %d\n", freq_in);
+		return ret;
+	}
+
+	dev_dbg(codec->dev, "bypass=%d m=%d n=%d k=%d\n",
+		pll_code.m_bp, (pll_code.m_bp ? 0 : pll_code.m_code),
+		pll_code.n_code, pll_code.k_code);
+
+	snd_soc_write(codec, RT5665_PLL_CTRL_1,
+		pll_code.n_code << RT5665_PLL_N_SFT | pll_code.k_code);
+	snd_soc_write(codec, RT5665_PLL_CTRL_2,
+		(pll_code.m_bp ? 0 : pll_code.m_code) << RT5665_PLL_M_SFT |
+		pll_code.m_bp << RT5665_PLL_M_BP_SFT);
+
+	rt5665->pll_in = freq_in;
+	rt5665->pll_out = freq_out;
+	rt5665->pll_src = Source;
+
+	return 0;
+}
+
+static int rt5665_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
+			unsigned int rx_mask, int slots, int slot_width)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	unsigned int val = 0;
+
+	if (rx_mask || tx_mask)
+		val |= RT5665_I2S1_MODE_TDM;
+
+	switch (slots) {
+	case 4:
+		val |= RT5665_TDM_IN_CH_4;
+		val |= RT5665_TDM_OUT_CH_4;
+		break;
+	case 6:
+		val |= RT5665_TDM_IN_CH_6;
+		val |= RT5665_TDM_OUT_CH_6;
+		break;
+	case 8:
+		val |= RT5665_TDM_IN_CH_8;
+		val |= RT5665_TDM_OUT_CH_8;
+		break;
+	case 2:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (slot_width) {
+	case 20:
+		val |= RT5665_TDM_IN_LEN_20;
+		val |= RT5665_TDM_OUT_LEN_20;
+		break;
+	case 24:
+		val |= RT5665_TDM_IN_LEN_24;
+		val |= RT5665_TDM_OUT_LEN_24;
+		break;
+	case 32:
+		val |= RT5665_TDM_IN_LEN_32;
+		val |= RT5665_TDM_OUT_LEN_32;
+		break;
+	case 16:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	snd_soc_update_bits(codec, RT5665_TDM_CTRL_1,
+		RT5665_I2S1_MODE_MASK | RT5665_TDM_IN_CH_MASK |
+		RT5665_TDM_OUT_CH_MASK | RT5665_TDM_IN_LEN_MASK |
+		RT5665_TDM_OUT_LEN_MASK, val);
+
+	return 0;
+}
+
+static int rt5665_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+
+	dev_dbg(codec->dev, "%s ratio=%d\n", __func__, ratio);
+
+	rt5665->bclk[dai->id] = ratio;
+
+	if (ratio == 64) {
+		switch (dai->id) {
+		case RT5665_AIF2_1:
+		case RT5665_AIF2_2:
+			snd_soc_update_bits(codec, RT5665_ADDA_CLK_1,
+				RT5665_I2S_BCLK_MS2_MASK,
+				RT5665_I2S_BCLK_MS2_64);
+			break;
+		case RT5665_AIF3:
+			snd_soc_update_bits(codec, RT5665_ADDA_CLK_1,
+				RT5665_I2S_BCLK_MS3_MASK,
+				RT5665_I2S_BCLK_MS3_64);
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int rt5665_set_bias_level(struct snd_soc_codec *codec,
+			enum snd_soc_bias_level level)
+{
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+
+	switch (level) {
+	case SND_SOC_BIAS_PREPARE:
+		regmap_update_bits(rt5665->regmap, RT5665_DIG_MISC,
+			RT5665_DIG_GATE_CTRL, RT5665_DIG_GATE_CTRL);
+		break;
+
+	case SND_SOC_BIAS_STANDBY:
+		regmap_update_bits(rt5665->regmap, RT5665_PWR_DIG_1,
+			RT5665_PWR_LDO,	RT5665_PWR_LDO);
+		regmap_update_bits(rt5665->regmap, RT5665_PWR_ANLG_1,
+			RT5665_PWR_MB, RT5665_PWR_MB);
+		regmap_update_bits(rt5665->regmap, RT5665_DIG_MISC,
+			RT5665_DIG_GATE_CTRL, 0);
+		break;
+	case SND_SOC_BIAS_OFF:
+		regmap_update_bits(rt5665->regmap, RT5665_PWR_DIG_1,
+			RT5665_PWR_LDO, 0);
+		regmap_update_bits(rt5665->regmap, RT5665_PWR_ANLG_1,
+			RT5665_PWR_MB, 0);
+		break;
+
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int rt5665_probe(struct snd_soc_codec *codec)
+{
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+
+	rt5665->codec = codec;
+
+	schedule_delayed_work(&rt5665->calibrate_work, msecs_to_jiffies(100));
+
+	return 0;
+}
+
+static int rt5665_remove(struct snd_soc_codec *codec)
+{
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+
+	regmap_write(rt5665->regmap, RT5665_RESET, 0);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int rt5665_suspend(struct snd_soc_codec *codec)
+{
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+
+	regcache_cache_only(rt5665->regmap, true);
+	regcache_mark_dirty(rt5665->regmap);
+	return 0;
+}
+
+static int rt5665_resume(struct snd_soc_codec *codec)
+{
+	struct rt5665_priv *rt5665 = snd_soc_codec_get_drvdata(codec);
+
+	regcache_cache_only(rt5665->regmap, false);
+	regcache_sync(rt5665->regmap);
+
+	return 0;
+}
+#else
+#define rt5665_suspend NULL
+#define rt5665_resume NULL
+#endif
+
+#define RT5665_STEREO_RATES SNDRV_PCM_RATE_8000_192000
+#define RT5665_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE | \
+		SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S8)
+
+static const struct snd_soc_dai_ops rt5665_aif_dai_ops = {
+	.hw_params = rt5665_hw_params,
+	.set_fmt = rt5665_set_dai_fmt,
+	.set_sysclk = rt5665_set_dai_sysclk,
+	.set_tdm_slot = rt5665_set_tdm_slot,
+	.set_pll = rt5665_set_dai_pll,
+	.set_bclk_ratio = rt5665_set_bclk_ratio,
+};
+
+static struct snd_soc_dai_driver rt5665_dai[] = {
+	{
+		.name = "rt5665-aif1_1",
+		.id = RT5665_AIF1_1,
+		.playback = {
+			.stream_name = "AIF1 Playback",
+			.channels_min = 1,
+			.channels_max = 8,
+			.rates = RT5665_STEREO_RATES,
+			.formats = RT5665_FORMATS,
+		},
+		.capture = {
+			.stream_name = "AIF1_1 Capture",
+			.channels_min = 1,
+			.channels_max = 8,
+			.rates = RT5665_STEREO_RATES,
+			.formats = RT5665_FORMATS,
+		},
+		.ops = &rt5665_aif_dai_ops,
+	},
+	{
+		.name = "rt5665-aif1_2",
+		.id = RT5665_AIF1_2,
+		.capture = {
+			.stream_name = "AIF1_2 Capture",
+			.channels_min = 1,
+			.channels_max = 8,
+			.rates = RT5665_STEREO_RATES,
+			.formats = RT5665_FORMATS,
+		},
+		.ops = &rt5665_aif_dai_ops,
+	},
+	{
+		.name = "rt5665-aif2_1",
+		.id = RT5665_AIF2_1,
+		.playback = {
+			.stream_name = "AIF2_1 Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = RT5665_STEREO_RATES,
+			.formats = RT5665_FORMATS,
+		},
+		.capture = {
+			.stream_name = "AIF2_1 Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = RT5665_STEREO_RATES,
+			.formats = RT5665_FORMATS,
+		},
+		.ops = &rt5665_aif_dai_ops,
+	},
+	{
+		.name = "rt5665-aif2_2",
+		.id = RT5665_AIF2_2,
+		.playback = {
+			.stream_name = "AIF2_2 Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = RT5665_STEREO_RATES,
+			.formats = RT5665_FORMATS,
+		},
+		.capture = {
+			.stream_name = "AIF2_2 Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = RT5665_STEREO_RATES,
+			.formats = RT5665_FORMATS,
+		},
+		.ops = &rt5665_aif_dai_ops,
+	},
+	{
+		.name = "rt5665-aif3",
+		.id = RT5665_AIF3,
+		.playback = {
+			.stream_name = "AIF3 Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = RT5665_STEREO_RATES,
+			.formats = RT5665_FORMATS,
+		},
+		.capture = {
+			.stream_name = "AIF3 Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = RT5665_STEREO_RATES,
+			.formats = RT5665_FORMATS,
+		},
+		.ops = &rt5665_aif_dai_ops,
+	},
+};
+
+static struct snd_soc_codec_driver soc_codec_dev_rt5665 = {
+	.probe = rt5665_probe,
+	.remove = rt5665_remove,
+	.suspend = rt5665_suspend,
+	.resume = rt5665_resume,
+	.set_bias_level = rt5665_set_bias_level,
+	.idle_bias_off = true,
+	.component_driver = {
+		.controls = rt5665_snd_controls,
+		.num_controls = ARRAY_SIZE(rt5665_snd_controls),
+		.dapm_widgets = rt5665_dapm_widgets,
+		.num_dapm_widgets = ARRAY_SIZE(rt5665_dapm_widgets),
+		.dapm_routes = rt5665_dapm_routes,
+		.num_dapm_routes = ARRAY_SIZE(rt5665_dapm_routes),
+	}
+};
+
+
+static const struct regmap_config rt5665_regmap = {
+	.reg_bits = 16,
+	.val_bits = 16,
+	.max_register = 0x0400,
+	.volatile_reg = rt5665_volatile_register,
+	.readable_reg = rt5665_readable_register,
+	.cache_type = REGCACHE_RBTREE,
+	.reg_defaults = rt5665_reg,
+	.num_reg_defaults = ARRAY_SIZE(rt5665_reg),
+	.use_single_rw = true,
+};
+
+static const struct i2c_device_id rt5665_i2c_id[] = {
+	{"rt5665", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, rt5665_i2c_id);
+
+static int rt5665_parse_dt(struct rt5665_priv *rt5665, struct device *dev)
+{
+	rt5665->pdata.in1_diff = of_property_read_bool(dev->of_node,
+					"realtek,in1-differential");
+	rt5665->pdata.in2_diff = of_property_read_bool(dev->of_node,
+					"realtek,in2-differential");
+	rt5665->pdata.in3_diff = of_property_read_bool(dev->of_node,
+					"realtek,in3-differential");
+	rt5665->pdata.in4_diff = of_property_read_bool(dev->of_node,
+					"realtek,in4-differential");
+
+	of_property_read_u32(dev->of_node, "realtek,dmic1-data-pin",
+		&rt5665->pdata.dmic1_data_pin);
+	of_property_read_u32(dev->of_node, "realtek,dmic2-data-pin",
+		&rt5665->pdata.dmic2_data_pin);
+	of_property_read_u32(dev->of_node, "realtek,jd-src",
+		&rt5665->pdata.jd_src);
+
+	rt5665->pdata.ldo1_en = of_get_named_gpio(dev->of_node,
+		"realtek,ldo1-en-gpios", 0);
+
+	return 0;
+}
+
+static void rt5665_calibrate(struct rt5665_priv *rt5665)
+{
+	int value, count;
+
+	mutex_lock(&rt5665->calibrate_mutex);
+
+	regcache_cache_bypass(rt5665->regmap, true);
+
+	regmap_write(rt5665->regmap, RT5665_RESET, 0);
+	regmap_write(rt5665->regmap, RT5665_BIAS_CUR_CTRL_8, 0xa602);
+	regmap_write(rt5665->regmap, RT5665_HP_CHARGE_PUMP_1, 0x0c26);
+	regmap_write(rt5665->regmap, RT5665_MONOMIX_IN_GAIN, 0x021f);
+	regmap_write(rt5665->regmap, RT5665_MONO_OUT, 0x480a);
+	regmap_write(rt5665->regmap, RT5665_PWR_MIXER, 0x083f);
+	regmap_write(rt5665->regmap, RT5665_PWR_DIG_1, 0x0180);
+	regmap_write(rt5665->regmap, RT5665_EJD_CTRL_1, 0x4040);
+	regmap_write(rt5665->regmap, RT5665_HP_LOGIC_CTRL_2, 0x0000);
+	regmap_write(rt5665->regmap, RT5665_DIG_MISC, 0x0001);
+	regmap_write(rt5665->regmap, RT5665_MICBIAS_2, 0x0380);
+	regmap_write(rt5665->regmap, RT5665_GLB_CLK, 0x8000);
+	regmap_write(rt5665->regmap, RT5665_ADDA_CLK_1, 0x1000);
+	regmap_write(rt5665->regmap, RT5665_CHOP_DAC, 0x3030);
+	regmap_write(rt5665->regmap, RT5665_CALIB_ADC_CTRL, 0x3c05);
+	regmap_write(rt5665->regmap, RT5665_PWR_ANLG_1, 0xaa3e);
+	usleep_range(15000, 20000);
+	regmap_write(rt5665->regmap, RT5665_PWR_ANLG_1, 0xfe7e);
+	regmap_write(rt5665->regmap, RT5665_HP_CALIB_CTRL_2, 0x0321);
+
+	regmap_write(rt5665->regmap, RT5665_HP_CALIB_CTRL_1, 0xfc00);
+	count = 0;
+	while (true) {
+		regmap_read(rt5665->regmap, RT5665_HP_CALIB_STA_1, &value);
+		if (value & 0x8000)
+			usleep_range(10000, 10005);
+		else
+			break;
+
+		if (count > 60) {
+			pr_err("HP Calibration Failure\n");
+			regmap_write(rt5665->regmap, RT5665_RESET, 0);
+			regcache_cache_bypass(rt5665->regmap, false);
+			goto out_unlock;
+		}
+
+		count++;
+	}
+
+	regmap_write(rt5665->regmap, RT5665_MONO_AMP_CALIB_CTRL_1, 0x9e24);
+	count = 0;
+	while (true) {
+		regmap_read(rt5665->regmap, RT5665_MONO_AMP_CALIB_STA1, &value);
+		if (value & 0x8000)
+			usleep_range(10000, 10005);
+		else
+			break;
+
+		if (count > 60) {
+			pr_err("MONO Calibration Failure\n");
+			regmap_write(rt5665->regmap, RT5665_RESET, 0);
+			regcache_cache_bypass(rt5665->regmap, false);
+			goto out_unlock;
+		}
+
+		count++;
+	}
+
+	regmap_write(rt5665->regmap, RT5665_RESET, 0);
+	regcache_cache_bypass(rt5665->regmap, false);
+
+	regcache_mark_dirty(rt5665->regmap);
+	regcache_sync(rt5665->regmap);
+
+	regmap_write(rt5665->regmap, RT5665_BIAS_CUR_CTRL_8, 0xa602);
+	regmap_write(rt5665->regmap, RT5665_ASRC_8, 0x0120);
+
+out_unlock:
+	mutex_unlock(&rt5665->calibrate_mutex);
+}
+
+static void rt5665_calibrate_handler(struct work_struct *work)
+{
+	struct rt5665_priv *rt5665 = container_of(work, struct rt5665_priv,
+		calibrate_work.work);
+
+	while (!rt5665->codec->component.card->instantiated) {
+		pr_debug("%s\n", __func__);
+		usleep_range(10000, 15000);
+	}
+
+	rt5665_calibrate(rt5665);
+}
+
+static int rt5665_i2c_probe(struct i2c_client *i2c,
+		    const struct i2c_device_id *id)
+{
+	struct rt5665_platform_data *pdata = dev_get_platdata(&i2c->dev);
+	struct rt5665_priv *rt5665;
+	int i, ret;
+	unsigned int val;
+
+	rt5665 = devm_kzalloc(&i2c->dev, sizeof(struct rt5665_priv),
+		GFP_KERNEL);
+
+	if (rt5665 == NULL)
+		return -ENOMEM;
+
+	i2c_set_clientdata(i2c, rt5665);
+
+	if (pdata)
+		rt5665->pdata = *pdata;
+	else
+		rt5665_parse_dt(rt5665, &i2c->dev);
+
+	for (i = 0; i < ARRAY_SIZE(rt5665->supplies); i++)
+		rt5665->supplies[i].supply = rt5665_supply_names[i];
+
+	ret = devm_regulator_bulk_get(&i2c->dev, ARRAY_SIZE(rt5665->supplies),
+				      rt5665->supplies);
+	if (ret != 0) {
+		dev_err(&i2c->dev, "Failed to request supplies: %d\n", ret);
+		return ret;
+	}
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(rt5665->supplies),
+				    rt5665->supplies);
+	if (ret != 0) {
+		dev_err(&i2c->dev, "Failed to enable supplies: %d\n", ret);
+		return ret;
+	}
+
+	if (gpio_is_valid(rt5665->pdata.ldo1_en)) {
+		if (devm_gpio_request_one(&i2c->dev, rt5665->pdata.ldo1_en,
+					  GPIOF_OUT_INIT_HIGH, "rt5665"))
+			dev_err(&i2c->dev, "Fail gpio_request gpio_ldo\n");
+	}
+
+	/* Sleep for 300 ms miniumum */
+	usleep_range(300000, 350000);
+
+	rt5665->regmap = devm_regmap_init_i2c(i2c, &rt5665_regmap);
+	if (IS_ERR(rt5665->regmap)) {
+		ret = PTR_ERR(rt5665->regmap);
+		dev_err(&i2c->dev, "Failed to allocate register map: %d\n",
+			ret);
+		return ret;
+	}
+
+	regmap_read(rt5665->regmap, RT5665_DEVICE_ID, &val);
+	if (val != DEVICE_ID) {
+		dev_err(&i2c->dev,
+			"Device with ID register %x is not rt5665\n", val);
+		return -ENODEV;
+	}
+
+	regmap_read(rt5665->regmap, RT5665_RESET, &val);
+	switch (val) {
+	case 0x0:
+		rt5665->id = CODEC_5666;
+		break;
+	case 0x6:
+		rt5665->id = CODEC_5668;
+		break;
+	case 0x3:
+	default:
+		rt5665->id = CODEC_5665;
+		break;
+	}
+
+	regmap_write(rt5665->regmap, RT5665_RESET, 0);
+
+	/* line in diff mode*/
+	if (rt5665->pdata.in1_diff)
+		regmap_update_bits(rt5665->regmap, RT5665_IN1_IN2,
+			RT5665_IN1_DF_MASK, RT5665_IN1_DF_MASK);
+	if (rt5665->pdata.in2_diff)
+		regmap_update_bits(rt5665->regmap, RT5665_IN1_IN2,
+			RT5665_IN2_DF_MASK, RT5665_IN2_DF_MASK);
+	if (rt5665->pdata.in3_diff)
+		regmap_update_bits(rt5665->regmap, RT5665_IN3_IN4,
+			RT5665_IN3_DF_MASK, RT5665_IN3_DF_MASK);
+	if (rt5665->pdata.in4_diff)
+		regmap_update_bits(rt5665->regmap, RT5665_IN3_IN4,
+			RT5665_IN4_DF_MASK, RT5665_IN4_DF_MASK);
+
+	/* DMIC pin*/
+	if (rt5665->pdata.dmic1_data_pin != RT5665_DMIC1_NULL ||
+		rt5665->pdata.dmic2_data_pin != RT5665_DMIC2_NULL) {
+		regmap_update_bits(rt5665->regmap, RT5665_GPIO_CTRL_2,
+			RT5665_GP9_PIN_MASK, RT5665_GP9_PIN_DMIC1_SCL);
+		regmap_update_bits(rt5665->regmap, RT5665_GPIO_CTRL_1,
+				RT5665_GP8_PIN_MASK, RT5665_GP8_PIN_DMIC2_SCL);
+		switch (rt5665->pdata.dmic1_data_pin) {
+		case RT5665_DMIC1_DATA_IN2N:
+			regmap_update_bits(rt5665->regmap, RT5665_DMIC_CTRL_1,
+				RT5665_DMIC_1_DP_MASK, RT5665_DMIC_1_DP_IN2N);
+			break;
+
+		case RT5665_DMIC1_DATA_GPIO4:
+			regmap_update_bits(rt5665->regmap, RT5665_DMIC_CTRL_1,
+				RT5665_DMIC_1_DP_MASK, RT5665_DMIC_1_DP_GPIO4);
+			regmap_update_bits(rt5665->regmap, RT5665_GPIO_CTRL_1,
+				RT5665_GP4_PIN_MASK, RT5665_GP4_PIN_DMIC1_SDA);
+			break;
+
+		default:
+			dev_dbg(&i2c->dev, "no DMIC1\n");
+			break;
+		}
+
+		switch (rt5665->pdata.dmic2_data_pin) {
+		case RT5665_DMIC2_DATA_IN2P:
+			regmap_update_bits(rt5665->regmap, RT5665_DMIC_CTRL_1,
+				RT5665_DMIC_2_DP_MASK, RT5665_DMIC_2_DP_IN2P);
+			break;
+
+		case RT5665_DMIC2_DATA_GPIO5:
+			regmap_update_bits(rt5665->regmap,
+				RT5665_DMIC_CTRL_1,
+				RT5665_DMIC_2_DP_MASK,
+				RT5665_DMIC_2_DP_GPIO5);
+			regmap_update_bits(rt5665->regmap, RT5665_GPIO_CTRL_1,
+				RT5665_GP5_PIN_MASK, RT5665_GP5_PIN_DMIC2_SDA);
+			break;
+
+		default:
+			dev_dbg(&i2c->dev, "no DMIC2\n");
+			break;
+
+		}
+	}
+
+	regmap_write(rt5665->regmap, RT5665_HP_LOGIC_CTRL_2, 0x0002);
+	regmap_update_bits(rt5665->regmap, RT5665_EJD_CTRL_1,
+		0xf000 | RT5665_VREF_POW_MASK, 0xd000 | RT5665_VREF_POW_REG);
+	/* Work around for pow_pump */
+	regmap_update_bits(rt5665->regmap, RT5665_STO1_DAC_SIL_DET,
+		RT5665_DEB_STO_DAC_MASK, RT5665_DEB_80_MS);
+
+	regmap_update_bits(rt5665->regmap, RT5665_HP_CHARGE_PUMP_1,
+		RT5665_PM_HP_MASK, RT5665_PM_HP_HV);
+
+	/* Set GPIO4,8 as input for combo jack */
+	if (rt5665->id == CODEC_5666) {
+		regmap_update_bits(rt5665->regmap, RT5665_GPIO_CTRL_2,
+			RT5665_GP4_PF_MASK, RT5665_GP4_PF_IN);
+		regmap_update_bits(rt5665->regmap, RT5665_GPIO_CTRL_3,
+			RT5665_GP8_PF_MASK, RT5665_GP8_PF_IN);
+	}
+
+	/* Enhance performance*/
+	regmap_update_bits(rt5665->regmap, RT5665_PWR_ANLG_1,
+		RT5665_HP_DRIVER_MASK | RT5665_LDO1_DVO_MASK,
+		RT5665_HP_DRIVER_5X | RT5665_LDO1_DVO_09);
+
+	INIT_DELAYED_WORK(&rt5665->jack_detect_work,
+				rt5665_jack_detect_handler);
+	INIT_DELAYED_WORK(&rt5665->calibrate_work,
+				rt5665_calibrate_handler);
+	INIT_DELAYED_WORK(&rt5665->jd_check_work,
+				rt5665_jd_check_handler);
+
+	mutex_init(&rt5665->calibrate_mutex);
+
+	if (i2c->irq) {
+		ret = devm_request_threaded_irq(&i2c->dev, i2c->irq, NULL,
+			rt5665_irq, IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING
+			| IRQF_ONESHOT, "rt5665", rt5665);
+		if (ret)
+			dev_err(&i2c->dev, "Failed to reguest IRQ: %d\n", ret);
+
+	}
+
+	return snd_soc_register_codec(&i2c->dev, &soc_codec_dev_rt5665,
+			rt5665_dai, ARRAY_SIZE(rt5665_dai));
+}
+
+static int rt5665_i2c_remove(struct i2c_client *i2c)
+{
+	snd_soc_unregister_codec(&i2c->dev);
+
+	return 0;
+}
+
+static void rt5665_i2c_shutdown(struct i2c_client *client)
+{
+	struct rt5665_priv *rt5665 = i2c_get_clientdata(client);
+
+	regmap_write(rt5665->regmap, RT5665_RESET, 0);
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id rt5665_of_match[] = {
+	{.compatible = "realtek,rt5665"},
+	{.compatible = "realtek,rt5666"},
+	{.compatible = "realtek,rt5668"},
+	{},
+};
+MODULE_DEVICE_TABLE(of, rt5665_of_match);
+#endif
+
+#ifdef CONFIG_ACPI
+static struct acpi_device_id rt5665_acpi_match[] = {
+	{"10EC5665", 0,},
+	{"10EC5666", 0,},
+	{"10EC5668", 0,},
+	{},
+};
+MODULE_DEVICE_TABLE(acpi, rt5665_acpi_match);
+#endif
+
+struct i2c_driver rt5665_i2c_driver = {
+	.driver = {
+		.name = "rt5665",
+		.of_match_table = of_match_ptr(rt5665_of_match),
+		.acpi_match_table = ACPI_PTR(rt5665_acpi_match),
+	},
+	.probe = rt5665_i2c_probe,
+	.remove = rt5665_i2c_remove,
+	.shutdown = rt5665_i2c_shutdown,
+	.id_table = rt5665_i2c_id,
+};
+module_i2c_driver(rt5665_i2c_driver);
+
+MODULE_DESCRIPTION("ASoC RT5665 driver");
+MODULE_AUTHOR("Bard Liao <bardliao@realtek.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/codecs/rt5665.h b/sound/soc/codecs/rt5665.h
new file mode 100644
index 000000000000..12f7080a0d3c
--- /dev/null
+++ b/sound/soc/codecs/rt5665.h
@@ -0,0 +1,1990 @@
+/*
+ * rt5665.h  --  RT5665/RT5658 ALSA SoC audio driver
+ *
+ * Copyright 2016 Realtek Microelectronics
+ * Author: Bard Liao <bardliao@realtek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __RT5665_H__
+#define __RT5665_H__
+
+#include <sound/rt5665.h>
+
+#define DEVICE_ID 0x6451
+
+/* Info */
+#define RT5665_RESET				0x0000
+#define RT5665_VENDOR_ID			0x00fd
+#define RT5665_VENDOR_ID_1			0x00fe
+#define RT5665_DEVICE_ID			0x00ff
+/*  I/O - Output */
+#define RT5665_LOUT				0x0001
+#define RT5665_HP_CTRL_1			0x0002
+#define RT5665_HP_CTRL_2			0x0003
+#define RT5665_MONO_OUT				0x0004
+#define RT5665_HPL_GAIN				0x0005
+#define RT5665_HPR_GAIN				0x0006
+#define RT5665_MONO_GAIN			0x0007
+
+/* I/O - Input */
+#define RT5665_CAL_BST_CTRL			0x000a
+#define RT5665_CBJ_BST_CTRL			0x000b
+#define RT5665_IN1_IN2				0x000c
+#define RT5665_IN3_IN4				0x000d
+#define RT5665_INL1_INR1_VOL			0x000f
+/* I/O - Speaker */
+#define RT5665_EJD_CTRL_1			0x0010
+#define RT5665_EJD_CTRL_2			0x0011
+#define RT5665_EJD_CTRL_3			0x0012
+#define RT5665_EJD_CTRL_4			0x0013
+#define RT5665_EJD_CTRL_5			0x0014
+#define RT5665_EJD_CTRL_6			0x0015
+#define RT5665_EJD_CTRL_7			0x0016
+/* I/O - ADC/DAC/DMIC */
+#define RT5665_DAC2_CTRL			0x0017
+#define RT5665_DAC2_DIG_VOL			0x0018
+#define RT5665_DAC1_DIG_VOL			0x0019
+#define RT5665_DAC3_DIG_VOL			0x001a
+#define RT5665_DAC3_CTRL			0x001b
+#define RT5665_STO1_ADC_DIG_VOL			0x001c
+#define RT5665_MONO_ADC_DIG_VOL			0x001d
+#define RT5665_STO2_ADC_DIG_VOL			0x001e
+#define RT5665_STO1_ADC_BOOST			0x001f
+#define RT5665_MONO_ADC_BOOST			0x0020
+#define RT5665_STO2_ADC_BOOST			0x0021
+#define RT5665_HP_IMP_GAIN_1			0x0022
+#define RT5665_HP_IMP_GAIN_2			0x0023
+/* Mixer - D-D */
+#define RT5665_STO1_ADC_MIXER			0x0026
+#define RT5665_MONO_ADC_MIXER			0x0027
+#define RT5665_STO2_ADC_MIXER			0x0028
+#define RT5665_AD_DA_MIXER			0x0029
+#define RT5665_STO1_DAC_MIXER			0x002a
+#define RT5665_MONO_DAC_MIXER			0x002b
+#define RT5665_STO2_DAC_MIXER			0x002c
+#define RT5665_A_DAC1_MUX			0x002d
+#define RT5665_A_DAC2_MUX			0x002e
+#define RT5665_DIG_INF2_DATA			0x002f
+#define RT5665_DIG_INF3_DATA			0x0030
+/* Mixer - PDM */
+#define RT5665_PDM_OUT_CTRL			0x0031
+#define RT5665_PDM_DATA_CTRL_1			0x0032
+#define RT5665_PDM_DATA_CTRL_2			0x0033
+#define RT5665_PDM_DATA_CTRL_3			0x0034
+#define RT5665_PDM_DATA_CTRL_4			0x0035
+/* Mixer - ADC */
+#define RT5665_REC1_GAIN			0x003a
+#define RT5665_REC1_L1_MIXER			0x003b
+#define RT5665_REC1_L2_MIXER			0x003c
+#define RT5665_REC1_R1_MIXER			0x003d
+#define RT5665_REC1_R2_MIXER			0x003e
+#define RT5665_REC2_GAIN			0x003f
+#define RT5665_REC2_L1_MIXER			0x0040
+#define RT5665_REC2_L2_MIXER			0x0041
+#define RT5665_REC2_R1_MIXER			0x0042
+#define RT5665_REC2_R2_MIXER			0x0043
+#define RT5665_CAL_REC				0x0044
+/* Mixer - DAC */
+#define RT5665_ALC_BACK_GAIN			0x0049
+#define RT5665_MONOMIX_GAIN			0x004a
+#define RT5665_MONOMIX_IN_GAIN			0x004b
+#define RT5665_OUT_L_GAIN			0x004d
+#define RT5665_OUT_L_MIXER			0x004e
+#define RT5665_OUT_R_GAIN			0x004f
+#define RT5665_OUT_R_MIXER			0x0050
+#define RT5665_LOUT_MIXER			0x0052
+/* Power */
+#define RT5665_PWR_DIG_1			0x0061
+#define RT5665_PWR_DIG_2			0x0062
+#define RT5665_PWR_ANLG_1			0x0063
+#define RT5665_PWR_ANLG_2			0x0064
+#define RT5665_PWR_ANLG_3			0x0065
+#define RT5665_PWR_MIXER			0x0066
+#define RT5665_PWR_VOL				0x0067
+/* Clock Detect */
+#define RT5665_CLK_DET				0x006b
+/* Filter */
+#define RT5665_HPF_CTRL1			0x006d
+/* DMIC */
+#define RT5665_DMIC_CTRL_1			0x006e
+#define RT5665_DMIC_CTRL_2			0x006f
+/* Format - ADC/DAC */
+#define RT5665_I2S1_SDP				0x0070
+#define RT5665_I2S2_SDP				0x0071
+#define RT5665_I2S3_SDP				0x0072
+#define RT5665_ADDA_CLK_1			0x0073
+#define RT5665_ADDA_CLK_2			0x0074
+#define RT5665_I2S1_F_DIV_CTRL_1		0x0075
+#define RT5665_I2S1_F_DIV_CTRL_2		0x0076
+/* Format - TDM Control */
+#define RT5665_TDM_CTRL_1			0x0078
+#define RT5665_TDM_CTRL_2			0x0079
+#define RT5665_TDM_CTRL_3			0x007a
+#define RT5665_TDM_CTRL_4			0x007b
+#define RT5665_TDM_CTRL_5			0x007c
+#define RT5665_TDM_CTRL_6			0x007d
+#define RT5665_TDM_CTRL_7			0x007e
+#define RT5665_TDM_CTRL_8			0x007f
+/* Function - Analog */
+#define RT5665_GLB_CLK				0x0080
+#define RT5665_PLL_CTRL_1			0x0081
+#define RT5665_PLL_CTRL_2			0x0082
+#define RT5665_ASRC_1				0x0083
+#define RT5665_ASRC_2				0x0084
+#define RT5665_ASRC_3				0x0085
+#define RT5665_ASRC_4				0x0086
+#define RT5665_ASRC_5				0x0087
+#define RT5665_ASRC_6				0x0088
+#define RT5665_ASRC_7				0x0089
+#define RT5665_ASRC_8				0x008a
+#define RT5665_ASRC_9				0x008b
+#define RT5665_ASRC_10				0x008c
+#define RT5665_DEPOP_1				0x008e
+#define RT5665_DEPOP_2				0x008f
+#define RT5665_HP_CHARGE_PUMP_1			0x0091
+#define RT5665_HP_CHARGE_PUMP_2			0x0092
+#define RT5665_MICBIAS_1			0x0093
+#define RT5665_MICBIAS_2			0x0094
+#define RT5665_ASRC_12				0x0098
+#define RT5665_ASRC_13				0x0099
+#define RT5665_ASRC_14				0x009a
+#define RT5665_RC_CLK_CTRL			0x009f
+#define RT5665_I2S_M_CLK_CTRL_1			0x00a0
+#define RT5665_I2S2_F_DIV_CTRL_1		0x00a1
+#define RT5665_I2S2_F_DIV_CTRL_2		0x00a2
+#define RT5665_I2S3_F_DIV_CTRL_1		0x00a3
+#define RT5665_I2S3_F_DIV_CTRL_2		0x00a4
+/* Function - Digital */
+#define RT5665_EQ_CTRL_1			0x00ae
+#define RT5665_EQ_CTRL_2			0x00af
+#define RT5665_IRQ_CTRL_1			0x00b6
+#define RT5665_IRQ_CTRL_2			0x00b7
+#define RT5665_IRQ_CTRL_3			0x00b8
+#define RT5665_IRQ_CTRL_4			0x00b9
+#define RT5665_IRQ_CTRL_5			0x00ba
+#define RT5665_IRQ_CTRL_6			0x00bb
+#define RT5665_INT_ST_1				0x00be
+#define RT5665_GPIO_CTRL_1			0x00c0
+#define RT5665_GPIO_CTRL_2			0x00c1
+#define RT5665_GPIO_CTRL_3			0x00c2
+#define RT5665_GPIO_CTRL_4			0x00c3
+#define RT5665_GPIO_STA				0x00c4
+#define RT5665_HP_AMP_DET_CTRL_1		0x00d0
+#define RT5665_HP_AMP_DET_CTRL_2		0x00d1
+#define RT5665_MID_HP_AMP_DET			0x00d3
+#define RT5665_LOW_HP_AMP_DET			0x00d4
+#define RT5665_SV_ZCD_1				0x00d9
+#define RT5665_SV_ZCD_2				0x00da
+#define RT5665_IL_CMD_1				0x00db
+#define RT5665_IL_CMD_2				0x00dc
+#define RT5665_IL_CMD_3				0x00dd
+#define RT5665_IL_CMD_4				0x00de
+#define RT5665_4BTN_IL_CMD_1			0x00df
+#define RT5665_4BTN_IL_CMD_2			0x00e0
+#define RT5665_4BTN_IL_CMD_3			0x00e1
+#define RT5665_PSV_IL_CMD_1			0x00e2
+
+#define RT5665_ADC_STO1_HP_CTRL_1		0x00ea
+#define RT5665_ADC_STO1_HP_CTRL_2		0x00eb
+#define RT5665_ADC_MONO_HP_CTRL_1		0x00ec
+#define RT5665_ADC_MONO_HP_CTRL_2		0x00ed
+#define RT5665_ADC_STO2_HP_CTRL_1		0x00ee
+#define RT5665_ADC_STO2_HP_CTRL_2		0x00ef
+#define RT5665_AJD1_CTRL			0x00f0
+#define RT5665_JD1_THD				0x00f1
+#define RT5665_JD2_THD				0x00f2
+#define RT5665_JD_CTRL_1			0x00f6
+#define RT5665_JD_CTRL_2			0x00f7
+#define RT5665_JD_CTRL_3			0x00f8
+/* General Control */
+#define RT5665_DIG_MISC				0x00fa
+#define RT5665_DUMMY_2				0x00fb
+#define RT5665_DUMMY_3				0x00fc
+
+#define RT5665_DAC_ADC_DIG_VOL1			0x0100
+#define RT5665_DAC_ADC_DIG_VOL2			0x0101
+#define RT5665_BIAS_CUR_CTRL_1			0x010a
+#define RT5665_BIAS_CUR_CTRL_2			0x010b
+#define RT5665_BIAS_CUR_CTRL_3			0x010c
+#define RT5665_BIAS_CUR_CTRL_4			0x010d
+#define RT5665_BIAS_CUR_CTRL_5			0x010e
+#define RT5665_BIAS_CUR_CTRL_6			0x010f
+#define RT5665_BIAS_CUR_CTRL_7			0x0110
+#define RT5665_BIAS_CUR_CTRL_8			0x0111
+#define RT5665_BIAS_CUR_CTRL_9			0x0112
+#define RT5665_BIAS_CUR_CTRL_10			0x0113
+#define RT5665_VREF_REC_OP_FB_CAP_CTRL		0x0117
+#define RT5665_CHARGE_PUMP_1			0x0125
+#define RT5665_DIG_IN_CTRL_1			0x0132
+#define RT5665_DIG_IN_CTRL_2			0x0133
+#define RT5665_PAD_DRIVING_CTRL			0x0137
+#define RT5665_SOFT_RAMP_DEPOP			0x0138
+#define RT5665_PLL				0x0139
+#define RT5665_CHOP_DAC				0x013a
+#define RT5665_CHOP_ADC				0x013b
+#define RT5665_CALIB_ADC_CTRL			0x013c
+#define RT5665_VOL_TEST				0x013f
+#define RT5665_TEST_MODE_CTRL_1			0x0145
+#define RT5665_TEST_MODE_CTRL_2			0x0146
+#define RT5665_TEST_MODE_CTRL_3			0x0147
+#define RT5665_TEST_MODE_CTRL_4			0x0148
+#define RT5665_BASSBACK_CTRL			0x0150
+#define RT5665_STO_NG2_CTRL_1			0x0160
+#define RT5665_STO_NG2_CTRL_2			0x0161
+#define RT5665_STO_NG2_CTRL_3			0x0162
+#define RT5665_STO_NG2_CTRL_4			0x0163
+#define RT5665_STO_NG2_CTRL_5			0x0164
+#define RT5665_STO_NG2_CTRL_6			0x0165
+#define RT5665_STO_NG2_CTRL_7			0x0166
+#define RT5665_STO_NG2_CTRL_8			0x0167
+#define RT5665_MONO_NG2_CTRL_1			0x0170
+#define RT5665_MONO_NG2_CTRL_2			0x0171
+#define RT5665_MONO_NG2_CTRL_3			0x0172
+#define RT5665_MONO_NG2_CTRL_4			0x0173
+#define RT5665_MONO_NG2_CTRL_5			0x0174
+#define RT5665_MONO_NG2_CTRL_6			0x0175
+#define RT5665_STO1_DAC_SIL_DET			0x0190
+#define RT5665_MONOL_DAC_SIL_DET		0x0191
+#define RT5665_MONOR_DAC_SIL_DET		0x0192
+#define RT5665_STO2_DAC_SIL_DET			0x0193
+#define RT5665_SIL_PSV_CTRL1			0x0194
+#define RT5665_SIL_PSV_CTRL2			0x0195
+#define RT5665_SIL_PSV_CTRL3			0x0196
+#define RT5665_SIL_PSV_CTRL4			0x0197
+#define RT5665_SIL_PSV_CTRL5			0x0198
+#define RT5665_SIL_PSV_CTRL6			0x0199
+#define RT5665_MONO_AMP_CALIB_CTRL_1		0x01a0
+#define RT5665_MONO_AMP_CALIB_CTRL_2		0x01a1
+#define RT5665_MONO_AMP_CALIB_CTRL_3		0x01a2
+#define RT5665_MONO_AMP_CALIB_CTRL_4		0x01a3
+#define RT5665_MONO_AMP_CALIB_CTRL_5		0x01a4
+#define RT5665_MONO_AMP_CALIB_CTRL_6		0x01a5
+#define RT5665_MONO_AMP_CALIB_CTRL_7		0x01a6
+#define RT5665_MONO_AMP_CALIB_STA1		0x01a7
+#define RT5665_MONO_AMP_CALIB_STA2		0x01a8
+#define RT5665_MONO_AMP_CALIB_STA3		0x01a9
+#define RT5665_MONO_AMP_CALIB_STA4		0x01aa
+#define RT5665_MONO_AMP_CALIB_STA6		0x01ab
+#define RT5665_HP_IMP_SENS_CTRL_01		0x01b5
+#define RT5665_HP_IMP_SENS_CTRL_02		0x01b6
+#define RT5665_HP_IMP_SENS_CTRL_03		0x01b7
+#define RT5665_HP_IMP_SENS_CTRL_04		0x01b8
+#define RT5665_HP_IMP_SENS_CTRL_05		0x01b9
+#define RT5665_HP_IMP_SENS_CTRL_06		0x01ba
+#define RT5665_HP_IMP_SENS_CTRL_07		0x01bb
+#define RT5665_HP_IMP_SENS_CTRL_08		0x01bc
+#define RT5665_HP_IMP_SENS_CTRL_09		0x01bd
+#define RT5665_HP_IMP_SENS_CTRL_10		0x01be
+#define RT5665_HP_IMP_SENS_CTRL_11		0x01bf
+#define RT5665_HP_IMP_SENS_CTRL_12		0x01c0
+#define RT5665_HP_IMP_SENS_CTRL_13		0x01c1
+#define RT5665_HP_IMP_SENS_CTRL_14		0x01c2
+#define RT5665_HP_IMP_SENS_CTRL_15		0x01c3
+#define RT5665_HP_IMP_SENS_CTRL_16		0x01c4
+#define RT5665_HP_IMP_SENS_CTRL_17		0x01c5
+#define RT5665_HP_IMP_SENS_CTRL_18		0x01c6
+#define RT5665_HP_IMP_SENS_CTRL_19		0x01c7
+#define RT5665_HP_IMP_SENS_CTRL_20		0x01c8
+#define RT5665_HP_IMP_SENS_CTRL_21		0x01c9
+#define RT5665_HP_IMP_SENS_CTRL_22		0x01ca
+#define RT5665_HP_IMP_SENS_CTRL_23		0x01cb
+#define RT5665_HP_IMP_SENS_CTRL_24		0x01cc
+#define RT5665_HP_IMP_SENS_CTRL_25		0x01cd
+#define RT5665_HP_IMP_SENS_CTRL_26		0x01ce
+#define RT5665_HP_IMP_SENS_CTRL_27		0x01cf
+#define RT5665_HP_IMP_SENS_CTRL_28		0x01d0
+#define RT5665_HP_IMP_SENS_CTRL_29		0x01d1
+#define RT5665_HP_IMP_SENS_CTRL_30		0x01d2
+#define RT5665_HP_IMP_SENS_CTRL_31		0x01d3
+#define RT5665_HP_IMP_SENS_CTRL_32		0x01d4
+#define RT5665_HP_IMP_SENS_CTRL_33		0x01d5
+#define RT5665_HP_IMP_SENS_CTRL_34		0x01d6
+#define RT5665_HP_LOGIC_CTRL_1			0x01da
+#define RT5665_HP_LOGIC_CTRL_2			0x01db
+#define RT5665_HP_LOGIC_CTRL_3			0x01dc
+#define RT5665_HP_CALIB_CTRL_1			0x01de
+#define RT5665_HP_CALIB_CTRL_2			0x01df
+#define RT5665_HP_CALIB_CTRL_3			0x01e0
+#define RT5665_HP_CALIB_CTRL_4			0x01e1
+#define RT5665_HP_CALIB_CTRL_5			0x01e2
+#define RT5665_HP_CALIB_CTRL_6			0x01e3
+#define RT5665_HP_CALIB_CTRL_7			0x01e4
+#define RT5665_HP_CALIB_CTRL_9			0x01e6
+#define RT5665_HP_CALIB_CTRL_10			0x01e7
+#define RT5665_HP_CALIB_CTRL_11			0x01e8
+#define RT5665_HP_CALIB_STA_1			0x01ea
+#define RT5665_HP_CALIB_STA_2			0x01eb
+#define RT5665_HP_CALIB_STA_3			0x01ec
+#define RT5665_HP_CALIB_STA_4			0x01ed
+#define RT5665_HP_CALIB_STA_5			0x01ee
+#define RT5665_HP_CALIB_STA_6			0x01ef
+#define RT5665_HP_CALIB_STA_7			0x01f0
+#define RT5665_HP_CALIB_STA_8			0x01f1
+#define RT5665_HP_CALIB_STA_9			0x01f2
+#define RT5665_HP_CALIB_STA_10			0x01f3
+#define RT5665_HP_CALIB_STA_11			0x01f4
+#define RT5665_PGM_TAB_CTRL1			0x0200
+#define RT5665_PGM_TAB_CTRL2			0x0201
+#define RT5665_PGM_TAB_CTRL3			0x0202
+#define RT5665_PGM_TAB_CTRL4			0x0203
+#define RT5665_PGM_TAB_CTRL5			0x0204
+#define RT5665_PGM_TAB_CTRL6			0x0205
+#define RT5665_PGM_TAB_CTRL7			0x0206
+#define RT5665_PGM_TAB_CTRL8			0x0207
+#define RT5665_PGM_TAB_CTRL9			0x0208
+#define RT5665_SAR_IL_CMD_1			0x0210
+#define RT5665_SAR_IL_CMD_2			0x0211
+#define RT5665_SAR_IL_CMD_3			0x0212
+#define RT5665_SAR_IL_CMD_4			0x0213
+#define RT5665_SAR_IL_CMD_5			0x0214
+#define RT5665_SAR_IL_CMD_6			0x0215
+#define RT5665_SAR_IL_CMD_7			0x0216
+#define RT5665_SAR_IL_CMD_8			0x0217
+#define RT5665_SAR_IL_CMD_9			0x0218
+#define RT5665_SAR_IL_CMD_10			0x0219
+#define RT5665_SAR_IL_CMD_11			0x021a
+#define RT5665_SAR_IL_CMD_12			0x021b
+#define RT5665_DRC1_CTRL_0			0x02ff
+#define RT5665_DRC1_CTRL_1			0x0300
+#define RT5665_DRC1_CTRL_2			0x0301
+#define RT5665_DRC1_CTRL_3			0x0302
+#define RT5665_DRC1_CTRL_4			0x0303
+#define RT5665_DRC1_CTRL_5			0x0304
+#define RT5665_DRC1_CTRL_6			0x0305
+#define RT5665_DRC1_HARD_LMT_CTRL_1		0x0306
+#define RT5665_DRC1_HARD_LMT_CTRL_2		0x0307
+#define RT5665_DRC1_PRIV_1			0x0310
+#define RT5665_DRC1_PRIV_2			0x0311
+#define RT5665_DRC1_PRIV_3			0x0312
+#define RT5665_DRC1_PRIV_4			0x0313
+#define RT5665_DRC1_PRIV_5			0x0314
+#define RT5665_DRC1_PRIV_6			0x0315
+#define RT5665_DRC1_PRIV_7			0x0316
+#define RT5665_DRC1_PRIV_8			0x0317
+#define RT5665_ALC_PGA_CTRL_1			0x0330
+#define RT5665_ALC_PGA_CTRL_2			0x0331
+#define RT5665_ALC_PGA_CTRL_3			0x0332
+#define RT5665_ALC_PGA_CTRL_4			0x0333
+#define RT5665_ALC_PGA_CTRL_5			0x0334
+#define RT5665_ALC_PGA_CTRL_6			0x0335
+#define RT5665_ALC_PGA_CTRL_7			0x0336
+#define RT5665_ALC_PGA_CTRL_8			0x0337
+#define RT5665_ALC_PGA_STA_1			0x0338
+#define RT5665_ALC_PGA_STA_2			0x0339
+#define RT5665_ALC_PGA_STA_3			0x033a
+#define RT5665_EQ_AUTO_RCV_CTRL1		0x03c0
+#define RT5665_EQ_AUTO_RCV_CTRL2		0x03c1
+#define RT5665_EQ_AUTO_RCV_CTRL3		0x03c2
+#define RT5665_EQ_AUTO_RCV_CTRL4		0x03c3
+#define RT5665_EQ_AUTO_RCV_CTRL5		0x03c4
+#define RT5665_EQ_AUTO_RCV_CTRL6		0x03c5
+#define RT5665_EQ_AUTO_RCV_CTRL7		0x03c6
+#define RT5665_EQ_AUTO_RCV_CTRL8		0x03c7
+#define RT5665_EQ_AUTO_RCV_CTRL9		0x03c8
+#define RT5665_EQ_AUTO_RCV_CTRL10		0x03c9
+#define RT5665_EQ_AUTO_RCV_CTRL11		0x03ca
+#define RT5665_EQ_AUTO_RCV_CTRL12		0x03cb
+#define RT5665_EQ_AUTO_RCV_CTRL13		0x03cc
+#define RT5665_ADC_L_EQ_LPF1_A1			0x03d0
+#define RT5665_R_EQ_LPF1_A1			0x03d1
+#define RT5665_L_EQ_LPF1_H0			0x03d2
+#define RT5665_R_EQ_LPF1_H0			0x03d3
+#define RT5665_L_EQ_BPF1_A1			0x03d4
+#define RT5665_R_EQ_BPF1_A1			0x03d5
+#define RT5665_L_EQ_BPF1_A2			0x03d6
+#define RT5665_R_EQ_BPF1_A2			0x03d7
+#define RT5665_L_EQ_BPF1_H0			0x03d8
+#define RT5665_R_EQ_BPF1_H0			0x03d9
+#define RT5665_L_EQ_BPF2_A1			0x03da
+#define RT5665_R_EQ_BPF2_A1			0x03db
+#define RT5665_L_EQ_BPF2_A2			0x03dc
+#define RT5665_R_EQ_BPF2_A2			0x03dd
+#define RT5665_L_EQ_BPF2_H0			0x03de
+#define RT5665_R_EQ_BPF2_H0			0x03df
+#define RT5665_L_EQ_BPF3_A1			0x03e0
+#define RT5665_R_EQ_BPF3_A1			0x03e1
+#define RT5665_L_EQ_BPF3_A2			0x03e2
+#define RT5665_R_EQ_BPF3_A2			0x03e3
+#define RT5665_L_EQ_BPF3_H0			0x03e4
+#define RT5665_R_EQ_BPF3_H0			0x03e5
+#define RT5665_L_EQ_BPF4_A1			0x03e6
+#define RT5665_R_EQ_BPF4_A1			0x03e7
+#define RT5665_L_EQ_BPF4_A2			0x03e8
+#define RT5665_R_EQ_BPF4_A2			0x03e9
+#define RT5665_L_EQ_BPF4_H0			0x03ea
+#define RT5665_R_EQ_BPF4_H0			0x03eb
+#define RT5665_L_EQ_HPF1_A1			0x03ec
+#define RT5665_R_EQ_HPF1_A1			0x03ed
+#define RT5665_L_EQ_HPF1_H0			0x03ee
+#define RT5665_R_EQ_HPF1_H0			0x03ef
+#define RT5665_L_EQ_PRE_VOL			0x03f0
+#define RT5665_R_EQ_PRE_VOL			0x03f1
+#define RT5665_L_EQ_POST_VOL			0x03f2
+#define RT5665_R_EQ_POST_VOL			0x03f3
+#define RT5665_SCAN_MODE_CTRL			0x07f0
+#define RT5665_I2C_MODE				0x07fa
+
+
+
+/* global definition */
+#define RT5665_L_MUTE				(0x1 << 15)
+#define RT5665_L_MUTE_SFT			15
+#define RT5665_VOL_L_MUTE			(0x1 << 14)
+#define RT5665_VOL_L_SFT			14
+#define RT5665_R_MUTE				(0x1 << 7)
+#define RT5665_R_MUTE_SFT			7
+#define RT5665_VOL_R_MUTE			(0x1 << 6)
+#define RT5665_VOL_R_SFT			6
+#define RT5665_L_VOL_MASK			(0x3f << 8)
+#define RT5665_L_VOL_SFT			8
+#define RT5665_R_VOL_MASK			(0x3f)
+#define RT5665_R_VOL_SFT			0
+
+/*Headphone Amp L/R Analog Gain and Digital NG2 Gain Control (0x0005 0x0006)*/
+#define RT5665_G_HP				(0xf << 8)
+#define RT5665_G_HP_SFT				8
+#define RT5665_G_STO_DA_DMIX			(0xf)
+#define RT5665_G_STO_DA_SFT			0
+
+/* CBJ Control (0x000b) */
+#define RT5665_BST_CBJ_MASK			(0xf << 8)
+#define RT5665_BST_CBJ_SFT			8
+
+/* IN1/IN2 Control (0x000c) */
+#define RT5665_IN1_DF_MASK			(0x1 << 15)
+#define RT5665_IN1_DF				15
+#define RT5665_BST1_MASK			(0x7f << 8)
+#define RT5665_BST1_SFT				8
+#define RT5665_IN2_DF_MASK			(0x1 << 7)
+#define RT5665_IN2_DF				7
+#define RT5665_BST2_MASK			(0x7f)
+#define RT5665_BST2_SFT				0
+
+/* IN3/IN4 Control (0x000d) */
+#define RT5665_IN3_DF_MASK			(0x1 << 15)
+#define RT5665_IN3_DF				15
+#define RT5665_BST3_MASK			(0x7f << 8)
+#define RT5665_BST3_SFT				8
+#define RT5665_IN4_DF_MASK			(0x1 << 7)
+#define RT5665_IN4_DF				7
+#define RT5665_BST4_MASK			(0x7f)
+#define RT5665_BST4_SFT				0
+
+/* INL and INR Volume Control (0x000f) */
+#define RT5665_INL_VOL_MASK			(0x1f << 8)
+#define RT5665_INL_VOL_SFT			8
+#define RT5665_INR_VOL_MASK			(0x1f)
+#define RT5665_INR_VOL_SFT			0
+
+/* Embeeded Jack and Type Detection Control 1 (0x0010) */
+#define RT5665_EMB_JD_EN			(0x1 << 15)
+#define RT5665_EMB_JD_EN_SFT			15
+#define RT5665_JD_MODE				(0x1 << 13)
+#define RT5665_JD_MODE_SFT			13
+#define RT5665_POLA_EXT_JD_MASK			(0x1 << 11)
+#define RT5665_POLA_EXT_JD_LOW			(0x1 << 11)
+#define RT5665_POLA_EXT_JD_HIGH			(0x0 << 11)
+#define RT5665_EXT_JD_DIG			(0x1 << 9)
+#define RT5665_POL_FAST_OFF_MASK		(0x1 << 8)
+#define RT5665_POL_FAST_OFF_HIGH		(0x1 << 8)
+#define RT5665_POL_FAST_OFF_LOW			(0x0 << 8)
+#define RT5665_VREF_POW_MASK			(0x1 << 6)
+#define RT5665_VREF_POW_FSM			(0x0 << 6)
+#define RT5665_VREF_POW_REG			(0x1 << 6)
+#define RT5665_MB1_PATH_MASK			(0x1 << 5)
+#define RT5665_CTRL_MB1_REG			(0x1 << 5)
+#define RT5665_CTRL_MB1_FSM			(0x0 << 5)
+#define RT5665_MB2_PATH_MASK			(0x1 << 4)
+#define RT5665_CTRL_MB2_REG			(0x1 << 4)
+#define RT5665_CTRL_MB2_FSM			(0x0 << 4)
+#define RT5665_TRIG_JD_MASK			(0x1 << 3)
+#define RT5665_TRIG_JD_HIGH			(0x1 << 3)
+#define RT5665_TRIG_JD_LOW			(0x0 << 3)
+
+/* Embeeded Jack and Type Detection Control 2 (0x0011) */
+#define RT5665_EXT_JD_SRC			(0x7 << 4)
+#define RT5665_EXT_JD_SRC_SFT			4
+#define RT5665_EXT_JD_SRC_GPIO_JD1		(0x0 << 4)
+#define RT5665_EXT_JD_SRC_GPIO_JD2		(0x1 << 4)
+#define RT5665_EXT_JD_SRC_JD1_1			(0x2 << 4)
+#define RT5665_EXT_JD_SRC_JD1_2			(0x3 << 4)
+#define RT5665_EXT_JD_SRC_JD2			(0x4 << 4)
+#define RT5665_EXT_JD_SRC_JD3			(0x5 << 4)
+#define RT5665_EXT_JD_SRC_MANUAL		(0x6 << 4)
+
+/* Combo Jack and Type Detection Control 4 (0x0013) */
+#define RT5665_SEL_SHT_MID_TON_MASK		(0x3 << 12)
+#define RT5665_SEL_SHT_MID_TON_2		(0x0 << 12)
+#define RT5665_SEL_SHT_MID_TON_3		(0x1 << 12)
+#define RT5665_CBJ_JD_TEST_MASK			(0x1 << 6)
+#define RT5665_CBJ_JD_TEST_NORM			(0x0 << 6)
+#define RT5665_CBJ_JD_TEST_MODE			(0x1 << 6)
+
+/* Slience Detection Control (0x0015) */
+#define RT5665_SIL_DET_MASK			(0x1 << 15)
+#define RT5665_SIL_DET_DIS			(0x0 << 15)
+#define RT5665_SIL_DET_EN			(0x1 << 15)
+
+/* DAC2 Control (0x0017) */
+#define RT5665_M_DAC2_L_VOL			(0x1 << 13)
+#define RT5665_M_DAC2_L_VOL_SFT			13
+#define RT5665_M_DAC2_R_VOL			(0x1 << 12)
+#define RT5665_M_DAC2_R_VOL_SFT			12
+#define RT5665_DAC_L2_SEL_MASK			(0x7 << 4)
+#define RT5665_DAC_L2_SEL_SFT			4
+#define RT5665_DAC_R2_SEL_MASK			(0x7 << 0)
+#define RT5665_DAC_R2_SEL_SFT			0
+
+/* Sidetone Control (0x0018) */
+#define RT5665_ST_SEL_MASK			(0x7 << 9)
+#define RT5665_ST_SEL_SFT			9
+#define RT5665_ST_EN				(0x1 << 6)
+#define RT5665_ST_EN_SFT			6
+
+/* DAC1 Digital Volume (0x0019) */
+#define RT5665_DAC_L1_VOL_MASK			(0xff << 8)
+#define RT5665_DAC_L1_VOL_SFT			8
+#define RT5665_DAC_R1_VOL_MASK			(0xff)
+#define RT5665_DAC_R1_VOL_SFT			0
+
+/* DAC2 Digital Volume (0x001a) */
+#define RT5665_DAC_L2_VOL_MASK			(0xff << 8)
+#define RT5665_DAC_L2_VOL_SFT			8
+#define RT5665_DAC_R2_VOL_MASK			(0xff)
+#define RT5665_DAC_R2_VOL_SFT			0
+
+/* DAC3 Control (0x001b) */
+#define RT5665_M_DAC3_L_VOL			(0x1 << 13)
+#define RT5665_M_DAC3_L_VOL_SFT			13
+#define RT5665_M_DAC3_R_VOL			(0x1 << 12)
+#define RT5665_M_DAC3_R_VOL_SFT			12
+#define RT5665_DAC_L3_SEL_MASK			(0x7 << 4)
+#define RT5665_DAC_L3_SEL_SFT			4
+#define RT5665_DAC_R3_SEL_MASK			(0x7 << 0)
+#define RT5665_DAC_R3_SEL_SFT			0
+
+/* ADC Digital Volume Control (0x001c) */
+#define RT5665_ADC_L_VOL_MASK			(0x7f << 8)
+#define RT5665_ADC_L_VOL_SFT			8
+#define RT5665_ADC_R_VOL_MASK			(0x7f)
+#define RT5665_ADC_R_VOL_SFT			0
+
+/* Mono ADC Digital Volume Control (0x001d) */
+#define RT5665_MONO_ADC_L_VOL_MASK		(0x7f << 8)
+#define RT5665_MONO_ADC_L_VOL_SFT		8
+#define RT5665_MONO_ADC_R_VOL_MASK		(0x7f)
+#define RT5665_MONO_ADC_R_VOL_SFT		0
+
+/* Stereo1 ADC Boost Gain Control (0x001f) */
+#define RT5665_STO1_ADC_L_BST_MASK		(0x3 << 14)
+#define RT5665_STO1_ADC_L_BST_SFT		14
+#define RT5665_STO1_ADC_R_BST_MASK		(0x3 << 12)
+#define RT5665_STO1_ADC_R_BST_SFT		12
+
+/* Mono ADC Boost Gain Control (0x0020) */
+#define RT5665_MONO_ADC_L_BST_MASK		(0x3 << 14)
+#define RT5665_MONO_ADC_L_BST_SFT		14
+#define RT5665_MONO_ADC_R_BST_MASK		(0x3 << 12)
+#define RT5665_MONO_ADC_R_BST_SFT		12
+
+/* Stereo1 ADC Boost Gain Control (0x001f) */
+#define RT5665_STO2_ADC_L_BST_MASK		(0x3 << 14)
+#define RT5665_STO2_ADC_L_BST_SFT		14
+#define RT5665_STO2_ADC_R_BST_MASK		(0x3 << 12)
+#define RT5665_STO2_ADC_R_BST_SFT		12
+
+/* Stereo1 ADC Mixer Control (0x0026) */
+#define RT5665_M_STO1_ADC_L1			(0x1 << 15)
+#define RT5665_M_STO1_ADC_L1_SFT		15
+#define RT5665_M_STO1_ADC_L2			(0x1 << 14)
+#define RT5665_M_STO1_ADC_L2_SFT		14
+#define RT5665_STO1_ADC1L_SRC_MASK		(0x1 << 13)
+#define RT5665_STO1_ADC1L_SRC_SFT		13
+#define RT5665_STO1_ADC1_SRC_ADC		(0x1 << 13)
+#define RT5665_STO1_ADC1_SRC_DACMIX		(0x0 << 13)
+#define RT5665_STO1_ADC2L_SRC_MASK		(0x1 << 12)
+#define RT5665_STO1_ADC2L_SRC_SFT		12
+#define RT5665_STO1_ADCL_SRC_MASK		(0x3 << 10)
+#define RT5665_STO1_ADCL_SRC_SFT		10
+#define RT5665_STO1_DD_L_SRC_MASK		(0x1 << 9)
+#define RT5665_STO1_DD_L_SRC_SFT		9
+#define RT5665_STO1_DMIC_SRC_MASK		(0x1 << 8)
+#define RT5665_STO1_DMIC_SRC_SFT		8
+#define RT5665_STO1_DMIC_SRC_DMIC2		(0x1 << 8)
+#define RT5665_STO1_DMIC_SRC_DMIC1		(0x0 << 8)
+#define RT5665_M_STO1_ADC_R1			(0x1 << 7)
+#define RT5665_M_STO1_ADC_R1_SFT		7
+#define RT5665_M_STO1_ADC_R2			(0x1 << 6)
+#define RT5665_M_STO1_ADC_R2_SFT		6
+#define RT5665_STO1_ADC1R_SRC_MASK		(0x1 << 5)
+#define RT5665_STO1_ADC1R_SRC_SFT		5
+#define RT5665_STO1_ADC2R_SRC_MASK		(0x1 << 4)
+#define RT5665_STO1_ADC2R_SRC_SFT		4
+#define RT5665_STO1_ADCR_SRC_MASK		(0x3 << 2)
+#define RT5665_STO1_ADCR_SRC_SFT		2
+#define RT5665_STO1_DD_R_SRC_MASK		(0x3)
+#define RT5665_STO1_DD_R_SRC_SFT		0
+
+
+/* Mono1 ADC Mixer control (0x0027) */
+#define RT5665_M_MONO_ADC_L1			(0x1 << 15)
+#define RT5665_M_MONO_ADC_L1_SFT		15
+#define RT5665_M_MONO_ADC_L2			(0x1 << 14)
+#define RT5665_M_MONO_ADC_L2_SFT		14
+#define RT5665_MONO_ADC_L1_SRC_MASK		(0x1 << 13)
+#define RT5665_MONO_ADC_L1_SRC_SFT		13
+#define RT5665_MONO_ADC_L2_SRC_MASK		(0x1 << 12)
+#define RT5665_MONO_ADC_L2_SRC_SFT		12
+#define RT5665_MONO_ADC_L_SRC_MASK		(0x3 << 10)
+#define RT5665_MONO_ADC_L_SRC_SFT		10
+#define RT5665_MONO_DD_L_SRC_MASK		(0x1 << 9)
+#define RT5665_MONO_DD_L_SRC_SFT		9
+#define RT5665_MONO_DMIC_L_SRC_MASK		(0x1 << 8)
+#define RT5665_MONO_DMIC_L_SRC_SFT		8
+#define RT5665_M_MONO_ADC_R1			(0x1 << 7)
+#define RT5665_M_MONO_ADC_R1_SFT		7
+#define RT5665_M_MONO_ADC_R2			(0x1 << 6)
+#define RT5665_M_MONO_ADC_R2_SFT		6
+#define RT5665_MONO_ADC_R1_SRC_MASK		(0x1 << 5)
+#define RT5665_MONO_ADC_R1_SRC_SFT		5
+#define RT5665_MONO_ADC_R2_SRC_MASK		(0x1 << 4)
+#define RT5665_MONO_ADC_R2_SRC_SFT		4
+#define RT5665_MONO_ADC_R_SRC_MASK		(0x3 << 2)
+#define RT5665_MONO_ADC_R_SRC_SFT		2
+#define RT5665_MONO_DD_R_SRC_MASK		(0x1 << 1)
+#define RT5665_MONO_DD_R_SRC_SFT		1
+#define RT5665_MONO_DMIC_R_SRC_MASK		0x1
+#define RT5665_MONO_DMIC_R_SRC_SFT		0
+
+/* Stereo2 ADC Mixer Control (0x0028) */
+#define RT5665_M_STO2_ADC_L1			(0x1 << 15)
+#define RT5665_M_STO2_ADC_L1_UN			(0x0 << 15)
+#define RT5665_M_STO2_ADC_L1_SFT		15
+#define RT5665_M_STO2_ADC_L2			(0x1 << 14)
+#define RT5665_M_STO2_ADC_L2_SFT		14
+#define RT5665_STO2_ADC1L_SRC_MASK		(0x1 << 13)
+#define RT5665_STO2_ADC1L_SRC_SFT		13
+#define RT5665_STO2_ADC1_SRC_ADC		(0x1 << 13)
+#define RT5665_STO2_ADC1_SRC_DACMIX		(0x0 << 13)
+#define RT5665_STO2_ADC2L_SRC_MASK		(0x1 << 12)
+#define RT5665_STO2_ADC2L_SRC_SFT		12
+#define RT5665_STO2_ADCL_SRC_MASK		(0x3 << 10)
+#define RT5665_STO2_ADCL_SRC_SFT		10
+#define RT5665_STO2_DD_L_SRC_MASK		(0x1 << 9)
+#define RT5665_STO2_DD_L_SRC_SFT		9
+#define RT5665_STO2_DMIC_SRC_MASK		(0x1 << 8)
+#define RT5665_STO2_DMIC_SRC_SFT		8
+#define RT5665_STO2_DMIC_SRC_DMIC2		(0x1 << 8)
+#define RT5665_STO2_DMIC_SRC_DMIC1		(0x0 << 8)
+#define RT5665_M_STO2_ADC_R1			(0x1 << 7)
+#define RT5665_M_STO2_ADC_R1_UN			(0x0 << 7)
+#define RT5665_M_STO2_ADC_R1_SFT		7
+#define RT5665_M_STO2_ADC_R2			(0x1 << 6)
+#define RT5665_M_STO2_ADC_R2_SFT		6
+#define RT5665_STO2_ADC1R_SRC_MASK		(0x1 << 5)
+#define RT5665_STO2_ADC1R_SRC_SFT		5
+#define RT5665_STO2_ADC2R_SRC_MASK		(0x1 << 4)
+#define RT5665_STO2_ADC2R_SRC_SFT		4
+#define RT5665_STO2_ADCR_SRC_MASK		(0x3 << 2)
+#define RT5665_STO2_ADCR_SRC_SFT		2
+#define RT5665_STO2_DD_R_SRC_MASK		(0x1 << 1)
+#define RT5665_STO2_DD_R_SRC_SFT		1
+
+/* ADC Mixer to DAC Mixer Control (0x0029) */
+#define RT5665_M_ADCMIX_L			(0x1 << 15)
+#define RT5665_M_ADCMIX_L_SFT			15
+#define RT5665_M_DAC1_L				(0x1 << 14)
+#define RT5665_M_DAC1_L_SFT			14
+#define RT5665_DAC1_R_SEL_MASK			(0x3 << 10)
+#define RT5665_DAC1_R_SEL_SFT			10
+#define RT5665_DAC1_L_SEL_MASK			(0x3 << 8)
+#define RT5665_DAC1_L_SEL_SFT			8
+#define RT5665_M_ADCMIX_R			(0x1 << 7)
+#define RT5665_M_ADCMIX_R_SFT			7
+#define RT5665_M_DAC1_R				(0x1 << 6)
+#define RT5665_M_DAC1_R_SFT			6
+
+/* Stereo1 DAC Mixer Control (0x002a) */
+#define RT5665_M_DAC_L1_STO_L			(0x1 << 15)
+#define RT5665_M_DAC_L1_STO_L_SFT		15
+#define RT5665_G_DAC_L1_STO_L_MASK		(0x1 << 14)
+#define RT5665_G_DAC_L1_STO_L_SFT		14
+#define RT5665_M_DAC_R1_STO_L			(0x1 << 13)
+#define RT5665_M_DAC_R1_STO_L_SFT		13
+#define RT5665_G_DAC_R1_STO_L_MASK		(0x1 << 12)
+#define RT5665_G_DAC_R1_STO_L_SFT		12
+#define RT5665_M_DAC_L2_STO_L			(0x1 << 11)
+#define RT5665_M_DAC_L2_STO_L_SFT		11
+#define RT5665_G_DAC_L2_STO_L_MASK		(0x1 << 10)
+#define RT5665_G_DAC_L2_STO_L_SFT		10
+#define RT5665_M_DAC_R2_STO_L			(0x1 << 9)
+#define RT5665_M_DAC_R2_STO_L_SFT		9
+#define RT5665_G_DAC_R2_STO_L_MASK		(0x1 << 8)
+#define RT5665_G_DAC_R2_STO_L_SFT		8
+#define RT5665_M_DAC_L1_STO_R			(0x1 << 7)
+#define RT5665_M_DAC_L1_STO_R_SFT		7
+#define RT5665_G_DAC_L1_STO_R_MASK		(0x1 << 6)
+#define RT5665_G_DAC_L1_STO_R_SFT		6
+#define RT5665_M_DAC_R1_STO_R			(0x1 << 5)
+#define RT5665_M_DAC_R1_STO_R_SFT		5
+#define RT5665_G_DAC_R1_STO_R_MASK		(0x1 << 4)
+#define RT5665_G_DAC_R1_STO_R_SFT		4
+#define RT5665_M_DAC_L2_STO_R			(0x1 << 3)
+#define RT5665_M_DAC_L2_STO_R_SFT		3
+#define RT5665_G_DAC_L2_STO_R_MASK		(0x1 << 2)
+#define RT5665_G_DAC_L2_STO_R_SFT		2
+#define RT5665_M_DAC_R2_STO_R			(0x1 << 1)
+#define RT5665_M_DAC_R2_STO_R_SFT		1
+#define RT5665_G_DAC_R2_STO_R_MASK		(0x1)
+#define RT5665_G_DAC_R2_STO_R_SFT		0
+
+/* Mono DAC Mixer Control (0x002b) */
+#define RT5665_M_DAC_L1_MONO_L			(0x1 << 15)
+#define RT5665_M_DAC_L1_MONO_L_SFT		15
+#define RT5665_G_DAC_L1_MONO_L_MASK		(0x1 << 14)
+#define RT5665_G_DAC_L1_MONO_L_SFT		14
+#define RT5665_M_DAC_R1_MONO_L			(0x1 << 13)
+#define RT5665_M_DAC_R1_MONO_L_SFT		13
+#define RT5665_G_DAC_R1_MONO_L_MASK		(0x1 << 12)
+#define RT5665_G_DAC_R1_MONO_L_SFT		12
+#define RT5665_M_DAC_L2_MONO_L			(0x1 << 11)
+#define RT5665_M_DAC_L2_MONO_L_SFT		11
+#define RT5665_G_DAC_L2_MONO_L_MASK		(0x1 << 10)
+#define RT5665_G_DAC_L2_MONO_L_SFT		10
+#define RT5665_M_DAC_R2_MONO_L			(0x1 << 9)
+#define RT5665_M_DAC_R2_MONO_L_SFT		9
+#define RT5665_G_DAC_R2_MONO_L_MASK		(0x1 << 8)
+#define RT5665_G_DAC_R2_MONO_L_SFT		8
+#define RT5665_M_DAC_L1_MONO_R			(0x1 << 7)
+#define RT5665_M_DAC_L1_MONO_R_SFT		7
+#define RT5665_G_DAC_L1_MONO_R_MASK		(0x1 << 6)
+#define RT5665_G_DAC_L1_MONO_R_SFT		6
+#define RT5665_M_DAC_R1_MONO_R			(0x1 << 5)
+#define RT5665_M_DAC_R1_MONO_R_SFT		5
+#define RT5665_G_DAC_R1_MONO_R_MASK		(0x1 << 4)
+#define RT5665_G_DAC_R1_MONO_R_SFT		4
+#define RT5665_M_DAC_L2_MONO_R			(0x1 << 3)
+#define RT5665_M_DAC_L2_MONO_R_SFT		3
+#define RT5665_G_DAC_L2_MONO_R_MASK		(0x1 << 2)
+#define RT5665_G_DAC_L2_MONO_R_SFT		2
+#define RT5665_M_DAC_R2_MONO_R			(0x1 << 1)
+#define RT5665_M_DAC_R2_MONO_R_SFT		1
+#define RT5665_G_DAC_R2_MONO_R_MASK		(0x1)
+#define RT5665_G_DAC_R2_MONO_R_SFT		0
+
+/* Stereo2 DAC Mixer Control (0x002c) */
+#define RT5665_M_DAC_L1_STO2_L			(0x1 << 15)
+#define RT5665_M_DAC_L1_STO2_L_SFT		15
+#define RT5665_G_DAC_L1_STO2_L_MASK		(0x1 << 14)
+#define RT5665_G_DAC_L1_STO2_L_SFT		14
+#define RT5665_M_DAC_L2_STO2_L			(0x1 << 13)
+#define RT5665_M_DAC_L2_STO2_L_SFT		13
+#define RT5665_G_DAC_L2_STO2_L_MASK		(0x1 << 12)
+#define RT5665_G_DAC_L2_STO2_L_SFT		12
+#define RT5665_M_DAC_L3_STO2_L			(0x1 << 11)
+#define RT5665_M_DAC_L3_STO2_L_SFT		11
+#define RT5665_G_DAC_L3_STO2_L_MASK		(0x1 << 10)
+#define RT5665_G_DAC_L3_STO2_L_SFT		10
+#define RT5665_M_ST_DAC_L1			(0x1 << 9)
+#define RT5665_M_ST_DAC_L1_SFT			9
+#define RT5665_M_ST_DAC_R1			(0x1 << 8)
+#define RT5665_M_ST_DAC_R1_SFT			8
+#define RT5665_M_DAC_R1_STO2_R			(0x1 << 7)
+#define RT5665_M_DAC_R1_STO2_R_SFT		7
+#define RT5665_G_DAC_R1_STO2_R_MASK		(0x1 << 6)
+#define RT5665_G_DAC_R1_STO2_R_SFT		6
+#define RT5665_M_DAC_R2_STO2_R			(0x1 << 5)
+#define RT5665_M_DAC_R2_STO2_R_SFT		5
+#define RT5665_G_DAC_R2_STO2_R_MASK		(0x1 << 4)
+#define RT5665_G_DAC_R2_STO2_R_SFT		4
+#define RT5665_M_DAC_R3_STO2_R			(0x1 << 3)
+#define RT5665_M_DAC_R3_STO2_R_SFT		3
+#define RT5665_G_DAC_R3_STO2_R_MASK		(0x1 << 2)
+#define RT5665_G_DAC_R3_STO2_R_SFT		2
+
+/* Analog DAC1 Input Source Control (0x002d) */
+#define RT5665_DAC_MIX_L_MASK			(0x3 << 12)
+#define RT5665_DAC_MIX_L_SFT			12
+#define RT5665_DAC_MIX_R_MASK			(0x3 << 8)
+#define RT5665_DAC_MIX_R_SFT			8
+#define RT5665_DAC_L1_SRC_MASK			(0x3 << 4)
+#define RT5665_A_DACL1_SFT			4
+#define RT5665_DAC_R1_SRC_MASK			(0x3)
+#define RT5665_A_DACR1_SFT			0
+
+/* Analog DAC Input Source Control (0x002e) */
+#define RT5665_A_DACL2_SEL			(0x1 << 4)
+#define RT5665_A_DACL2_SFT			4
+#define RT5665_A_DACR2_SEL			(0x1 << 0)
+#define RT5665_A_DACR2_SFT			0
+
+/* Digital Interface Data Control (0x002f) */
+#define RT5665_IF2_1_ADC_IN_MASK		(0x7 << 12)
+#define RT5665_IF2_1_ADC_IN_SFT			12
+#define RT5665_IF2_1_DAC_SEL_MASK		(0x3 << 10)
+#define RT5665_IF2_1_DAC_SEL_SFT		10
+#define RT5665_IF2_1_ADC_SEL_MASK		(0x3 << 8)
+#define RT5665_IF2_1_ADC_SEL_SFT		8
+#define RT5665_IF2_2_ADC_IN_MASK		(0x7 << 4)
+#define RT5665_IF2_2_ADC_IN_SFT			4
+#define RT5665_IF2_2_DAC_SEL_MASK		(0x3 << 2)
+#define RT5665_IF2_2_DAC_SEL_SFT		2
+#define RT5665_IF2_2_ADC_SEL_MASK		(0x3 << 0)
+#define RT5665_IF2_2_ADC_SEL_SFT		0
+
+/* Digital Interface Data Control (0x0030) */
+#define RT5665_IF3_ADC_IN_MASK			(0x7 << 4)
+#define RT5665_IF3_ADC_IN_SFT			4
+#define RT5665_IF3_DAC_SEL_MASK			(0x3 << 2)
+#define RT5665_IF3_DAC_SEL_SFT			2
+#define RT5665_IF3_ADC_SEL_MASK			(0x3 << 0)
+#define RT5665_IF3_ADC_SEL_SFT			0
+
+/* PDM Output Control (0x0031) */
+#define RT5665_M_PDM1_L				(0x1 << 14)
+#define RT5665_M_PDM1_L_SFT			14
+#define RT5665_M_PDM1_R				(0x1 << 12)
+#define RT5665_M_PDM1_R_SFT			12
+#define RT5665_PDM1_L_MASK			(0x3 << 10)
+#define RT5665_PDM1_L_SFT			10
+#define RT5665_PDM1_R_MASK			(0x3 << 8)
+#define RT5665_PDM1_R_SFT			8
+#define RT5665_PDM1_BUSY			(0x1 << 6)
+#define RT5665_PDM_PATTERN			(0x1 << 5)
+#define RT5665_PDM_GAIN				(0x1 << 4)
+#define RT5665_LRCK_PDM_PI2C			(0x1 << 3)
+#define RT5665_PDM_DIV_MASK			(0x3)
+
+/*S/PDIF Output Control (0x0036) */
+#define RT5665_SPDIF_SEL_MASK			(0x3 << 0)
+#define RT5665_SPDIF_SEL_SFT			0
+
+/* REC Left Mixer Control 2 (0x003c) */
+#define RT5665_M_CBJ_RM1_L			(0x1 << 7)
+#define RT5665_M_CBJ_RM1_L_SFT			7
+#define RT5665_M_BST1_RM1_L			(0x1 << 5)
+#define RT5665_M_BST1_RM1_L_SFT			5
+#define RT5665_M_BST2_RM1_L			(0x1 << 4)
+#define RT5665_M_BST2_RM1_L_SFT			4
+#define RT5665_M_BST3_RM1_L			(0x1 << 3)
+#define RT5665_M_BST3_RM1_L_SFT			3
+#define RT5665_M_BST4_RM1_L			(0x1 << 2)
+#define RT5665_M_BST4_RM1_L_SFT			2
+#define RT5665_M_INL_RM1_L			(0x1 << 1)
+#define RT5665_M_INL_RM1_L_SFT			1
+#define RT5665_M_INR_RM1_L			(0x1)
+#define RT5665_M_INR_RM1_L_SFT			0
+
+/* REC Right Mixer Control 2 (0x003e) */
+#define RT5665_M_AEC_REF_RM1_R			(0x1 << 7)
+#define RT5665_M_AEC_REF_RM1_R_SFT		7
+#define RT5665_M_BST1_RM1_R			(0x1 << 5)
+#define RT5665_M_BST1_RM1_R_SFT			5
+#define RT5665_M_BST2_RM1_R			(0x1 << 4)
+#define RT5665_M_BST2_RM1_R_SFT			4
+#define RT5665_M_BST3_RM1_R			(0x1 << 3)
+#define RT5665_M_BST3_RM1_R_SFT			3
+#define RT5665_M_BST4_RM1_R			(0x1 << 2)
+#define RT5665_M_BST4_RM1_R_SFT			2
+#define RT5665_M_INR_RM1_R			(0x1 << 1)
+#define RT5665_M_INR_RM1_R_SFT			1
+#define RT5665_M_MONOVOL_RM1_R			(0x1)
+#define RT5665_M_MONOVOL_RM1_R_SFT		0
+
+/* REC Mixer 2 Left Control 2 (0x0041) */
+#define RT5665_M_CBJ_RM2_L			(0x1 << 7)
+#define RT5665_M_CBJ_RM2_L_SFT			7
+#define RT5665_M_BST1_RM2_L			(0x1 << 5)
+#define RT5665_M_BST1_RM2_L_SFT			5
+#define RT5665_M_BST2_RM2_L			(0x1 << 4)
+#define RT5665_M_BST2_RM2_L_SFT			4
+#define RT5665_M_BST3_RM2_L			(0x1 << 3)
+#define RT5665_M_BST3_RM2_L_SFT			3
+#define RT5665_M_BST4_RM2_L			(0x1 << 2)
+#define RT5665_M_BST4_RM2_L_SFT			2
+#define RT5665_M_INL_RM2_L			(0x1 << 1)
+#define RT5665_M_INL_RM2_L_SFT			1
+#define RT5665_M_INR_RM2_L			(0x1)
+#define RT5665_M_INR_RM2_L_SFT			0
+
+/* REC Mixer 2 Right Control 2 (0x0043) */
+#define RT5665_M_MONOVOL_RM2_R			(0x1 << 7)
+#define RT5665_M_MONOVOL_RM2_R_SFT		7
+#define RT5665_M_BST1_RM2_R			(0x1 << 5)
+#define RT5665_M_BST1_RM2_R_SFT			5
+#define RT5665_M_BST2_RM2_R			(0x1 << 4)
+#define RT5665_M_BST2_RM2_R_SFT			4
+#define RT5665_M_BST3_RM2_R			(0x1 << 3)
+#define RT5665_M_BST3_RM2_R_SFT			3
+#define RT5665_M_BST4_RM2_R			(0x1 << 2)
+#define RT5665_M_BST4_RM2_R_SFT			2
+#define RT5665_M_INL_RM2_R			(0x1 << 1)
+#define RT5665_M_INL_RM2_R_SFT			1
+#define RT5665_M_INR_RM2_R			(0x1)
+#define RT5665_M_INR_RM2_R_SFT			0
+
+/* SPK Left Mixer Control (0x0046) */
+#define RT5665_M_BST3_SM_L			(0x1 << 4)
+#define RT5665_M_BST3_SM_L_SFT			4
+#define RT5665_M_IN_R_SM_L			(0x1 << 3)
+#define RT5665_M_IN_R_SM_L_SFT			3
+#define RT5665_M_IN_L_SM_L			(0x1 << 2)
+#define RT5665_M_IN_L_SM_L_SFT			2
+#define RT5665_M_BST1_SM_L			(0x1 << 1)
+#define RT5665_M_BST1_SM_L_SFT			1
+#define RT5665_M_DAC_L2_SM_L			(0x1)
+#define RT5665_M_DAC_L2_SM_L_SFT		0
+
+/* SPK Right Mixer Control (0x0047) */
+#define RT5665_M_BST3_SM_R			(0x1 << 4)
+#define RT5665_M_BST3_SM_R_SFT			4
+#define RT5665_M_IN_R_SM_R			(0x1 << 3)
+#define RT5665_M_IN_R_SM_R_SFT			3
+#define RT5665_M_IN_L_SM_R			(0x1 << 2)
+#define RT5665_M_IN_L_SM_R_SFT			2
+#define RT5665_M_BST4_SM_R			(0x1 << 1)
+#define RT5665_M_BST4_SM_R_SFT			1
+#define RT5665_M_DAC_R2_SM_R			(0x1)
+#define RT5665_M_DAC_R2_SM_R_SFT		0
+
+/* SPO Amp Input and Gain Control (0x0048) */
+#define RT5665_M_DAC_L2_SPKOMIX			(0x1 << 13)
+#define RT5665_M_DAC_L2_SPKOMIX_SFT		13
+#define RT5665_M_SPKVOLL_SPKOMIX		(0x1 << 12)
+#define RT5665_M_SPKVOLL_SPKOMIX_SFT		12
+#define RT5665_M_DAC_R2_SPKOMIX			(0x1 << 9)
+#define RT5665_M_DAC_R2_SPKOMIX_SFT		9
+#define RT5665_M_SPKVOLR_SPKOMIX		(0x1 << 8)
+#define RT5665_M_SPKVOLR_SPKOMIX_SFT		8
+
+/* MONOMIX Input and Gain Control (0x004b) */
+#define RT5665_G_MONOVOL_MA			(0x1 << 10)
+#define RT5665_G_MONOVOL_MA_SFT			10
+#define RT5665_M_MONOVOL_MA			(0x1 << 9)
+#define RT5665_M_MONOVOL_MA_SFT			9
+#define RT5665_M_DAC_L2_MA			(0x1 << 8)
+#define RT5665_M_DAC_L2_MA_SFT			8
+#define RT5665_M_BST3_MM			(0x1 << 4)
+#define RT5665_M_BST3_MM_SFT			4
+#define RT5665_M_BST2_MM			(0x1 << 3)
+#define RT5665_M_BST2_MM_SFT			3
+#define RT5665_M_BST1_MM			(0x1 << 2)
+#define RT5665_M_BST1_MM_SFT			2
+#define RT5665_M_RECMIC2L_MM			(0x1 << 1)
+#define RT5665_M_RECMIC2L_MM_SFT		1
+#define RT5665_M_DAC_L2_MM			(0x1)
+#define RT5665_M_DAC_L2_MM_SFT			0
+
+/* Output Left Mixer Control 1 (0x004d) */
+#define RT5665_G_BST3_OM_L_MASK			(0x7 << 12)
+#define RT5665_G_BST3_OM_L_SFT			12
+#define RT5665_G_BST2_OM_L_MASK			(0x7 << 9)
+#define RT5665_G_BST2_OM_L_SFT			9
+#define RT5665_G_BST1_OM_L_MASK			(0x7 << 6)
+#define RT5665_G_BST1_OM_L_SFT			6
+#define RT5665_G_IN_L_OM_L_MASK			(0x7 << 3)
+#define RT5665_G_IN_L_OM_L_SFT			3
+#define RT5665_G_DAC_L2_OM_L_MASK		(0x7 << 0)
+#define RT5665_G_DAC_L2_OM_L_SFT		0
+
+/* Output Left Mixer Input Control (0x004e) */
+#define RT5665_M_BST3_OM_L			(0x1 << 4)
+#define RT5665_M_BST3_OM_L_SFT			4
+#define RT5665_M_BST2_OM_L			(0x1 << 3)
+#define RT5665_M_BST2_OM_L_SFT			3
+#define RT5665_M_BST1_OM_L			(0x1 << 2)
+#define RT5665_M_BST1_OM_L_SFT			2
+#define RT5665_M_IN_L_OM_L			(0x1 << 1)
+#define RT5665_M_IN_L_OM_L_SFT			1
+#define RT5665_M_DAC_L2_OM_L			(0x1)
+#define RT5665_M_DAC_L2_OM_L_SFT		0
+
+/* Output Right Mixer Input Control (0x0050) */
+#define RT5665_M_BST4_OM_R			(0x1 << 4)
+#define RT5665_M_BST4_OM_R_SFT			4
+#define RT5665_M_BST3_OM_R			(0x1 << 3)
+#define RT5665_M_BST3_OM_R_SFT			3
+#define RT5665_M_BST2_OM_R			(0x1 << 2)
+#define RT5665_M_BST2_OM_R_SFT			2
+#define RT5665_M_IN_R_OM_R			(0x1 << 1)
+#define RT5665_M_IN_R_OM_R_SFT			1
+#define RT5665_M_DAC_R2_OM_R			(0x1)
+#define RT5665_M_DAC_R2_OM_R_SFT		0
+
+/* LOUT Mixer Control (0x0052) */
+#define RT5665_M_DAC_L2_LM			(0x1 << 15)
+#define RT5665_M_DAC_L2_LM_SFT			15
+#define RT5665_M_DAC_R2_LM			(0x1 << 14)
+#define RT5665_M_DAC_R2_LM_SFT			14
+#define RT5665_M_OV_L_LM			(0x1 << 13)
+#define RT5665_M_OV_L_LM_SFT			13
+#define RT5665_M_OV_R_LM			(0x1 << 12)
+#define RT5665_M_OV_R_LM_SFT			12
+#define RT5665_LOUT_BST_SFT			11
+#define RT5665_LOUT_DF				(0x1 << 11)
+#define RT5665_LOUT_DF_SFT			11
+
+/* Power Management for Digital 1 (0x0061) */
+#define RT5665_PWR_I2S1_1			(0x1 << 15)
+#define RT5665_PWR_I2S1_1_BIT			15
+#define RT5665_PWR_I2S1_2			(0x1 << 14)
+#define RT5665_PWR_I2S1_2_BIT			14
+#define RT5665_PWR_I2S2_1			(0x1 << 13)
+#define RT5665_PWR_I2S2_1_BIT			13
+#define RT5665_PWR_I2S2_2			(0x1 << 12)
+#define RT5665_PWR_I2S2_2_BIT			12
+#define RT5665_PWR_DAC_L1			(0x1 << 11)
+#define RT5665_PWR_DAC_L1_BIT			11
+#define RT5665_PWR_DAC_R1			(0x1 << 10)
+#define RT5665_PWR_DAC_R1_BIT			10
+#define RT5665_PWR_I2S3				(0x1 << 9)
+#define RT5665_PWR_I2S3_BIT			9
+#define RT5665_PWR_LDO				(0x1 << 8)
+#define RT5665_PWR_LDO_BIT			8
+#define RT5665_PWR_DAC_L2			(0x1 << 7)
+#define RT5665_PWR_DAC_L2_BIT			7
+#define RT5665_PWR_DAC_R2			(0x1 << 6)
+#define RT5665_PWR_DAC_R2_BIT			6
+#define RT5665_PWR_ADC_L1			(0x1 << 4)
+#define RT5665_PWR_ADC_L1_BIT			4
+#define RT5665_PWR_ADC_R1			(0x1 << 3)
+#define RT5665_PWR_ADC_R1_BIT			3
+#define RT5665_PWR_ADC_L2			(0x1 << 2)
+#define RT5665_PWR_ADC_L2_BIT			2
+#define RT5665_PWR_ADC_R2			(0x1 << 1)
+#define RT5665_PWR_ADC_R2_BIT			1
+
+/* Power Management for Digital 2 (0x0062) */
+#define RT5665_PWR_ADC_S1F			(0x1 << 15)
+#define RT5665_PWR_ADC_S1F_BIT			15
+#define RT5665_PWR_ADC_S2F			(0x1 << 14)
+#define RT5665_PWR_ADC_S2F_BIT			14
+#define RT5665_PWR_ADC_MF_L			(0x1 << 13)
+#define RT5665_PWR_ADC_MF_L_BIT			13
+#define RT5665_PWR_ADC_MF_R			(0x1 << 12)
+#define RT5665_PWR_ADC_MF_R_BIT			12
+#define RT5665_PWR_DAC_S2F			(0x1 << 11)
+#define RT5665_PWR_DAC_S2F_BIT			11
+#define RT5665_PWR_DAC_S1F			(0x1 << 10)
+#define RT5665_PWR_DAC_S1F_BIT			10
+#define RT5665_PWR_DAC_MF_L			(0x1 << 9)
+#define RT5665_PWR_DAC_MF_L_BIT			9
+#define RT5665_PWR_DAC_MF_R			(0x1 << 8)
+#define RT5665_PWR_DAC_MF_R_BIT			8
+#define RT5665_PWR_PDM1				(0x1 << 7)
+#define RT5665_PWR_PDM1_BIT			7
+
+/* Power Management for Analog 1 (0x0063) */
+#define RT5665_PWR_VREF1			(0x1 << 15)
+#define RT5665_PWR_VREF1_BIT			15
+#define RT5665_PWR_FV1				(0x1 << 14)
+#define RT5665_PWR_FV1_BIT			14
+#define RT5665_PWR_VREF2			(0x1 << 13)
+#define RT5665_PWR_VREF2_BIT			13
+#define RT5665_PWR_FV2				(0x1 << 12)
+#define RT5665_PWR_FV2_BIT			12
+#define RT5665_PWR_VREF3			(0x1 << 11)
+#define RT5665_PWR_VREF3_BIT			11
+#define RT5665_PWR_FV3				(0x1 << 10)
+#define RT5665_PWR_FV3_BIT			10
+#define RT5665_PWR_MB				(0x1 << 9)
+#define RT5665_PWR_MB_BIT			9
+#define RT5665_PWR_LM				(0x1 << 8)
+#define RT5665_PWR_LM_BIT			8
+#define RT5665_PWR_BG				(0x1 << 7)
+#define RT5665_PWR_BG_BIT			7
+#define RT5665_PWR_MA				(0x1 << 6)
+#define RT5665_PWR_MA_BIT			6
+#define RT5665_PWR_HA_L				(0x1 << 5)
+#define RT5665_PWR_HA_L_BIT			5
+#define RT5665_PWR_HA_R				(0x1 << 4)
+#define RT5665_PWR_HA_R_BIT			4
+#define RT5665_HP_DRIVER_MASK			(0x3 << 2)
+#define RT5665_HP_DRIVER_1X			(0x0 << 2)
+#define RT5665_HP_DRIVER_3X			(0x1 << 2)
+#define RT5665_HP_DRIVER_5X			(0x2 << 2)
+#define RT5665_LDO1_DVO_MASK			(0x3)
+#define RT5665_LDO1_DVO_09			(0x0)
+#define RT5665_LDO1_DVO_10			(0x1)
+#define RT5665_LDO1_DVO_12			(0x2)
+#define RT5665_LDO1_DVO_14			(0x3)
+
+/* Power Management for Analog 2 (0x0064) */
+#define RT5665_PWR_BST1				(0x1 << 15)
+#define RT5665_PWR_BST1_BIT			15
+#define RT5665_PWR_BST2				(0x1 << 14)
+#define RT5665_PWR_BST2_BIT			14
+#define RT5665_PWR_BST3				(0x1 << 13)
+#define RT5665_PWR_BST3_BIT			13
+#define RT5665_PWR_BST4				(0x1 << 12)
+#define RT5665_PWR_BST4_BIT			12
+#define RT5665_PWR_MB1				(0x1 << 11)
+#define RT5665_PWR_MB1_PWR_DOWN			(0x0 << 11)
+#define RT5665_PWR_MB1_BIT			11
+#define RT5665_PWR_MB2				(0x1 << 10)
+#define RT5665_PWR_MB2_PWR_DOWN			(0x0 << 10)
+#define RT5665_PWR_MB2_BIT			10
+#define RT5665_PWR_MB3				(0x1 << 9)
+#define RT5665_PWR_MB3_BIT			9
+#define RT5665_PWR_BST1_P			(0x1 << 7)
+#define RT5665_PWR_BST1_P_BIT			7
+#define RT5665_PWR_BST2_P			(0x1 << 6)
+#define RT5665_PWR_BST2_P_BIT			6
+#define RT5665_PWR_BST3_P			(0x1 << 5)
+#define RT5665_PWR_BST3_P_BIT			5
+#define RT5665_PWR_BST4_P			(0x1 << 4)
+#define RT5665_PWR_BST4_P_BIT			4
+#define RT5665_PWR_JD1				(0x1 << 3)
+#define RT5665_PWR_JD1_BIT			3
+#define RT5665_PWR_JD2				(0x1 << 2)
+#define RT5665_PWR_JD2_BIT			2
+#define RT5665_PWR_RM1_L			(0x1 << 1)
+#define RT5665_PWR_RM1_L_BIT			1
+#define RT5665_PWR_RM1_R			(0x1)
+#define RT5665_PWR_RM1_R_BIT			0
+
+/* Power Management for Analog 3 (0x0065) */
+#define RT5665_PWR_CBJ				(0x1 << 9)
+#define RT5665_PWR_CBJ_BIT			9
+#define RT5665_PWR_BST_L			(0x1 << 8)
+#define RT5665_PWR_BST_L_BIT			8
+#define RT5665_PWR_BST_R			(0x1 << 7)
+#define RT5665_PWR_BST_R_BIT			7
+#define RT5665_PWR_PLL				(0x1 << 6)
+#define RT5665_PWR_PLL_BIT			6
+#define RT5665_PWR_LDO2				(0x1 << 2)
+#define RT5665_PWR_LDO2_BIT			2
+#define RT5665_PWR_SVD				(0x1 << 1)
+#define RT5665_PWR_SVD_BIT			1
+
+/* Power Management for Mixer (0x0066) */
+#define RT5665_PWR_RM2_L			(0x1 << 15)
+#define RT5665_PWR_RM2_L_BIT			15
+#define RT5665_PWR_RM2_R			(0x1 << 14)
+#define RT5665_PWR_RM2_R_BIT			14
+#define RT5665_PWR_OM_L				(0x1 << 13)
+#define RT5665_PWR_OM_L_BIT			13
+#define RT5665_PWR_OM_R				(0x1 << 12)
+#define RT5665_PWR_OM_R_BIT			12
+#define RT5665_PWR_MM				(0x1 << 11)
+#define RT5665_PWR_MM_BIT			11
+#define RT5665_PWR_AEC_REF			(0x1 << 6)
+#define RT5665_PWR_AEC_REF_BIT			6
+#define RT5665_PWR_STO1_DAC_L			(0x1 << 5)
+#define RT5665_PWR_STO1_DAC_L_BIT		5
+#define RT5665_PWR_STO1_DAC_R			(0x1 << 4)
+#define RT5665_PWR_STO1_DAC_R_BIT		4
+#define RT5665_PWR_MONO_DAC_L			(0x1 << 3)
+#define RT5665_PWR_MONO_DAC_L_BIT		3
+#define RT5665_PWR_MONO_DAC_R			(0x1 << 2)
+#define RT5665_PWR_MONO_DAC_R_BIT		2
+#define RT5665_PWR_STO2_DAC_L			(0x1 << 1)
+#define RT5665_PWR_STO2_DAC_L_BIT		1
+#define RT5665_PWR_STO2_DAC_R			(0x1)
+#define RT5665_PWR_STO2_DAC_R_BIT		0
+
+/* Power Management for Volume (0x0067) */
+#define RT5665_PWR_OV_L				(0x1 << 13)
+#define RT5665_PWR_OV_L_BIT			13
+#define RT5665_PWR_OV_R				(0x1 << 12)
+#define RT5665_PWR_OV_R_BIT			12
+#define RT5665_PWR_IN_L				(0x1 << 9)
+#define RT5665_PWR_IN_L_BIT			9
+#define RT5665_PWR_IN_R				(0x1 << 8)
+#define RT5665_PWR_IN_R_BIT			8
+#define RT5665_PWR_MV				(0x1 << 7)
+#define RT5665_PWR_MV_BIT			7
+#define RT5665_PWR_MIC_DET			(0x1 << 5)
+#define RT5665_PWR_MIC_DET_BIT			5
+
+/* (0x006b) */
+#define RT5665_SYS_CLK_DET			15
+#define RT5665_HP_CLK_DET			14
+#define RT5665_MONO_CLK_DET			13
+#define RT5665_LOUT_CLK_DET			12
+#define RT5665_POW_CLK_DET			0
+
+/* Digital Microphone Control 1 (0x006e) */
+#define RT5665_DMIC_1_EN_MASK			(0x1 << 15)
+#define RT5665_DMIC_1_EN_SFT			15
+#define RT5665_DMIC_1_DIS			(0x0 << 15)
+#define RT5665_DMIC_1_EN			(0x1 << 15)
+#define RT5665_DMIC_2_EN_MASK			(0x1 << 14)
+#define RT5665_DMIC_2_EN_SFT			14
+#define RT5665_DMIC_2_DIS			(0x0 << 14)
+#define RT5665_DMIC_2_EN			(0x1 << 14)
+#define RT5665_DMIC_2_DP_MASK			(0x1 << 9)
+#define RT5665_DMIC_2_DP_SFT			9
+#define RT5665_DMIC_2_DP_GPIO5			(0x0 << 9)
+#define RT5665_DMIC_2_DP_IN2P			(0x1 << 9)
+#define RT5665_DMIC_CLK_MASK			(0x7 << 5)
+#define RT5665_DMIC_CLK_SFT			5
+#define RT5665_DMIC_1_DP_MASK			(0x1 << 1)
+#define RT5665_DMIC_1_DP_SFT			1
+#define RT5665_DMIC_1_DP_GPIO4			(0x0 << 1)
+#define RT5665_DMIC_1_DP_IN2N			(0x1 << 1)
+
+
+/* Digital Microphone Control 1 (0x006f) */
+#define RT5665_DMIC_2L_LH_MASK			(0x1 << 3)
+#define RT5665_DMIC_2L_LH_SFT			3
+#define RT5665_DMIC_2L_LH_RISING		(0x0 << 3)
+#define RT5665_DMIC_2L_LH_FALLING		(0x1 << 3)
+#define RT5665_DMIC_2R_LH_MASK			(0x1 << 2)
+#define RT5665_DMIC_2R_LH_SFT			2
+#define RT5665_DMIC_2R_LH_RISING		(0x0 << 2)
+#define RT5665_DMIC_2R_LH_FALLING		(0x1 << 2)
+#define RT5665_DMIC_1L_LH_MASK			(0x1 << 1)
+#define RT5665_DMIC_1L_LH_SFT			1
+#define RT5665_DMIC_1L_LH_RISING		(0x0 << 1)
+#define RT5665_DMIC_1L_LH_FALLING		(0x1 << 1)
+#define RT5665_DMIC_1R_LH_MASK			(0x1 << 0)
+#define RT5665_DMIC_1R_LH_SFT			0
+#define RT5665_DMIC_1R_LH_RISING		(0x0)
+#define RT5665_DMIC_1R_LH_FALLING		(0x1)
+
+/* I2S1/2/3 Audio Serial Data Port Control (0x0070 0x0071 0x0072) */
+#define RT5665_I2S_MS_MASK			(0x1 << 15)
+#define RT5665_I2S_MS_SFT			15
+#define RT5665_I2S_MS_M				(0x0 << 15)
+#define RT5665_I2S_MS_S				(0x1 << 15)
+#define RT5665_I2S_PIN_CFG_MASK			(0x1 << 14)
+#define RT5665_I2S_PIN_CFG_SFT			14
+#define RT5665_I2S_CLK_SEL_MASK			(0x1 << 11)
+#define RT5665_I2S_CLK_SEL_SFT			11
+#define RT5665_I2S_BP_MASK			(0x1 << 8)
+#define RT5665_I2S_BP_SFT			8
+#define RT5665_I2S_BP_NOR			(0x0 << 8)
+#define RT5665_I2S_BP_INV			(0x1 << 8)
+#define RT5665_I2S_DL_MASK			(0x3 << 4)
+#define RT5665_I2S_DL_SFT			4
+#define RT5665_I2S_DL_16			(0x0 << 4)
+#define RT5665_I2S_DL_20			(0x1 << 4)
+#define RT5665_I2S_DL_24			(0x2 << 4)
+#define RT5665_I2S_DL_8				(0x3 << 4)
+#define RT5665_I2S_DF_MASK			(0x7)
+#define RT5665_I2S_DF_SFT			0
+#define RT5665_I2S_DF_I2S			(0x0)
+#define RT5665_I2S_DF_LEFT			(0x1)
+#define RT5665_I2S_DF_PCM_A			(0x2)
+#define RT5665_I2S_DF_PCM_B			(0x3)
+#define RT5665_I2S_DF_PCM_A_N			(0x6)
+#define RT5665_I2S_DF_PCM_B_N			(0x7)
+
+/* ADC/DAC Clock Control 1 (0x0073) */
+#define RT5665_I2S_PD1_MASK			(0x7 << 12)
+#define RT5665_I2S_PD1_SFT			12
+#define RT5665_I2S_PD1_1			(0x0 << 12)
+#define RT5665_I2S_PD1_2			(0x1 << 12)
+#define RT5665_I2S_PD1_3			(0x2 << 12)
+#define RT5665_I2S_PD1_4			(0x3 << 12)
+#define RT5665_I2S_PD1_6			(0x4 << 12)
+#define RT5665_I2S_PD1_8			(0x5 << 12)
+#define RT5665_I2S_PD1_12			(0x6 << 12)
+#define RT5665_I2S_PD1_16			(0x7 << 12)
+#define RT5665_I2S_M_PD2_MASK			(0x7 << 8)
+#define RT5665_I2S_M_PD2_SFT			8
+#define RT5665_I2S_M_PD2_1			(0x0 << 8)
+#define RT5665_I2S_M_PD2_2			(0x1 << 8)
+#define RT5665_I2S_M_PD2_3			(0x2 << 8)
+#define RT5665_I2S_M_PD2_4			(0x3 << 8)
+#define RT5665_I2S_M_PD2_6			(0x4 << 8)
+#define RT5665_I2S_M_PD2_8			(0x5 << 8)
+#define RT5665_I2S_M_PD2_12			(0x6 << 8)
+#define RT5665_I2S_M_PD2_16			(0x7 << 8)
+#define RT5665_I2S_CLK_SRC_MASK			(0x3 << 4)
+#define RT5665_I2S_CLK_SRC_SFT			4
+#define RT5665_I2S_CLK_SRC_MCLK			(0x0 << 4)
+#define RT5665_I2S_CLK_SRC_PLL1			(0x1 << 4)
+#define RT5665_I2S_CLK_SRC_RCCLK		(0x2 << 4)
+#define RT5665_DAC_OSR_MASK			(0x3 << 2)
+#define RT5665_DAC_OSR_SFT			2
+#define RT5665_DAC_OSR_128			(0x0 << 2)
+#define RT5665_DAC_OSR_64			(0x1 << 2)
+#define RT5665_DAC_OSR_32			(0x2 << 2)
+#define RT5665_ADC_OSR_MASK			(0x3)
+#define RT5665_ADC_OSR_SFT			0
+#define RT5665_ADC_OSR_128			(0x0)
+#define RT5665_ADC_OSR_64			(0x1)
+#define RT5665_ADC_OSR_32			(0x2)
+
+/* ADC/DAC Clock Control 2 (0x0074) */
+#define RT5665_I2S_BCLK_MS2_MASK		(0x1 << 15)
+#define RT5665_I2S_BCLK_MS2_SFT			15
+#define RT5665_I2S_BCLK_MS2_32			(0x0 << 15)
+#define RT5665_I2S_BCLK_MS2_64			(0x1 << 15)
+#define RT5665_I2S_PD2_MASK			(0x7 << 12)
+#define RT5665_I2S_PD2_SFT			12
+#define RT5665_I2S_PD2_1			(0x0 << 12)
+#define RT5665_I2S_PD2_2			(0x1 << 12)
+#define RT5665_I2S_PD2_3			(0x2 << 12)
+#define RT5665_I2S_PD2_4			(0x3 << 12)
+#define RT5665_I2S_PD2_6			(0x4 << 12)
+#define RT5665_I2S_PD2_8			(0x5 << 12)
+#define RT5665_I2S_PD2_12			(0x6 << 12)
+#define RT5665_I2S_PD2_16			(0x7 << 12)
+#define RT5665_I2S_BCLK_MS3_MASK		(0x1 << 11)
+#define RT5665_I2S_BCLK_MS3_SFT			11
+#define RT5665_I2S_BCLK_MS3_32			(0x0 << 11)
+#define RT5665_I2S_BCLK_MS3_64			(0x1 << 11)
+#define RT5665_I2S_PD3_MASK			(0x7 << 8)
+#define RT5665_I2S_PD3_SFT			8
+#define RT5665_I2S_PD3_1			(0x0 << 8)
+#define RT5665_I2S_PD3_2			(0x1 << 8)
+#define RT5665_I2S_PD3_3			(0x2 << 8)
+#define RT5665_I2S_PD3_4			(0x3 << 8)
+#define RT5665_I2S_PD3_6			(0x4 << 8)
+#define RT5665_I2S_PD3_8			(0x5 << 8)
+#define RT5665_I2S_PD3_12			(0x6 << 8)
+#define RT5665_I2S_PD3_16			(0x7 << 8)
+#define RT5665_I2S_PD4_MASK			(0x7 << 4)
+#define RT5665_I2S_PD4_SFT			4
+#define RT5665_I2S_PD4_1			(0x0 << 4)
+#define RT5665_I2S_PD4_2			(0x1 << 4)
+#define RT5665_I2S_PD4_3			(0x2 << 4)
+#define RT5665_I2S_PD4_4			(0x3 << 4)
+#define RT5665_I2S_PD4_6			(0x4 << 4)
+#define RT5665_I2S_PD4_8			(0x5 << 4)
+#define RT5665_I2S_PD4_12			(0x6 << 4)
+#define RT5665_I2S_PD4_16			(0x7 << 4)
+
+/* TDM control 1 (0x0078) */
+#define RT5665_I2S1_MODE_MASK			(0x1 << 15)
+#define RT5665_I2S1_MODE_I2S			(0x0 << 15)
+#define RT5665_I2S1_MODE_TDM			(0x1 << 15)
+#define RT5665_TDM_IN_CH_MASK			(0x3 << 10)
+#define RT5665_TDM_IN_CH_2			(0x0 << 10)
+#define RT5665_TDM_IN_CH_4			(0x1 << 10)
+#define RT5665_TDM_IN_CH_6			(0x2 << 10)
+#define RT5665_TDM_IN_CH_8			(0x3 << 10)
+#define RT5665_TDM_OUT_CH_MASK			(0x3 << 8)
+#define RT5665_TDM_OUT_CH_2			(0x0 << 8)
+#define RT5665_TDM_OUT_CH_4			(0x1 << 8)
+#define RT5665_TDM_OUT_CH_6			(0x2 << 8)
+#define RT5665_TDM_OUT_CH_8			(0x3 << 8)
+#define RT5665_TDM_IN_LEN_MASK			(0x3 << 6)
+#define RT5665_TDM_IN_LEN_16			(0x0 << 6)
+#define RT5665_TDM_IN_LEN_20			(0x1 << 6)
+#define RT5665_TDM_IN_LEN_24			(0x2 << 6)
+#define RT5665_TDM_IN_LEN_32			(0x3 << 6)
+#define RT5665_TDM_OUT_LEN_MASK			(0x3 << 4)
+#define RT5665_TDM_OUT_LEN_16			(0x0 << 4)
+#define RT5665_TDM_OUT_LEN_20			(0x1 << 4)
+#define RT5665_TDM_OUT_LEN_24			(0x2 << 4)
+#define RT5665_TDM_OUT_LEN_32			(0x3 << 4)
+
+
+/* TDM control 2 (0x0079) */
+#define RT5665_I2S1_1_DS_ADC_SLOT01_SFT		14
+#define RT5665_I2S1_1_DS_ADC_SLOT23_SFT		12
+#define RT5665_I2S1_1_DS_ADC_SLOT45_SFT		10
+#define RT5665_I2S1_1_DS_ADC_SLOT67_SFT		8
+#define RT5665_I2S1_2_DS_ADC_SLOT01_SFT		6
+#define RT5665_I2S1_2_DS_ADC_SLOT23_SFT		4
+#define RT5665_I2S1_2_DS_ADC_SLOT45_SFT		2
+#define RT5665_I2S1_2_DS_ADC_SLOT67_SFT		0
+
+/* TDM control 3/4 (0x007a) (0x007b) */
+#define RT5665_IF1_ADC1_SEL_SFT			10
+#define RT5665_IF1_ADC2_SEL_SFT			9
+#define RT5665_IF1_ADC3_SEL_SFT			8
+#define RT5665_IF1_ADC4_SEL_SFT			7
+#define RT5665_TDM_ADC_SEL_SFT			0
+#define RT5665_TDM_ADC_CTRL_MASK		(0x1f << 0)
+#define RT5665_TDM_ADC_DATA_06			(0x6 << 0)
+
+/* Global Clock Control (0x0080) */
+#define RT5665_SCLK_SRC_MASK			(0x3 << 14)
+#define RT5665_SCLK_SRC_SFT			14
+#define RT5665_SCLK_SRC_MCLK			(0x0 << 14)
+#define RT5665_SCLK_SRC_PLL1			(0x1 << 14)
+#define RT5665_SCLK_SRC_RCCLK			(0x2 << 14)
+#define RT5665_PLL1_SRC_MASK			(0x7 << 8)
+#define RT5665_PLL1_SRC_SFT			8
+#define RT5665_PLL1_SRC_MCLK			(0x0 << 8)
+#define RT5665_PLL1_SRC_BCLK1			(0x1 << 8)
+#define RT5665_PLL1_SRC_BCLK2			(0x2 << 8)
+#define RT5665_PLL1_SRC_BCLK3			(0x3 << 8)
+#define RT5665_PLL1_PD_MASK			(0x7 << 4)
+#define RT5665_PLL1_PD_SFT			4
+
+
+#define RT5665_PLL_INP_MAX			40000000
+#define RT5665_PLL_INP_MIN			256000
+/* PLL M/N/K Code Control 1 (0x0081) */
+#define RT5665_PLL_N_MAX			0x001ff
+#define RT5665_PLL_N_MASK			(RT5665_PLL_N_MAX << 7)
+#define RT5665_PLL_N_SFT			7
+#define RT5665_PLL_K_MAX			0x001f
+#define RT5665_PLL_K_MASK			(RT5665_PLL_K_MAX)
+#define RT5665_PLL_K_SFT			0
+
+/* PLL M/N/K Code Control 2 (0x0082) */
+#define RT5665_PLL_M_MAX			0x00f
+#define RT5665_PLL_M_MASK			(RT5665_PLL_M_MAX << 12)
+#define RT5665_PLL_M_SFT			12
+#define RT5665_PLL_M_BP				(0x1 << 11)
+#define RT5665_PLL_M_BP_SFT			11
+#define RT5665_PLL_K_BP				(0x1 << 10)
+#define RT5665_PLL_K_BP_SFT			10
+
+/* PLL tracking mode 1 (0x0083) */
+#define RT5665_I2S3_ASRC_MASK			(0x1 << 15)
+#define RT5665_I2S3_ASRC_SFT			15
+#define RT5665_I2S2_ASRC_MASK			(0x1 << 14)
+#define RT5665_I2S2_ASRC_SFT			14
+#define RT5665_I2S1_ASRC_MASK			(0x1 << 13)
+#define RT5665_I2S1_ASRC_SFT			13
+#define RT5665_DAC_STO1_ASRC_MASK		(0x1 << 12)
+#define RT5665_DAC_STO1_ASRC_SFT		12
+#define RT5665_DAC_STO2_ASRC_MASK		(0x1 << 11)
+#define RT5665_DAC_STO2_ASRC_SFT		11
+#define RT5665_DAC_MONO_L_ASRC_MASK		(0x1 << 10)
+#define RT5665_DAC_MONO_L_ASRC_SFT		10
+#define RT5665_DAC_MONO_R_ASRC_MASK		(0x1 << 9)
+#define RT5665_DAC_MONO_R_ASRC_SFT		9
+#define RT5665_DMIC_STO1_ASRC_MASK		(0x1 << 8)
+#define RT5665_DMIC_STO1_ASRC_SFT		8
+#define RT5665_DMIC_STO2_ASRC_MASK		(0x1 << 7)
+#define RT5665_DMIC_STO2_ASRC_SFT		7
+#define RT5665_DMIC_MONO_L_ASRC_MASK		(0x1 << 6)
+#define RT5665_DMIC_MONO_L_ASRC_SFT		6
+#define RT5665_DMIC_MONO_R_ASRC_MASK		(0x1 << 5)
+#define RT5665_DMIC_MONO_R_ASRC_SFT		5
+#define RT5665_ADC_STO1_ASRC_MASK		(0x1 << 4)
+#define RT5665_ADC_STO1_ASRC_SFT		4
+#define RT5665_ADC_STO2_ASRC_MASK		(0x1 << 3)
+#define RT5665_ADC_STO2_ASRC_SFT		3
+#define RT5665_ADC_MONO_L_ASRC_MASK		(0x1 << 2)
+#define RT5665_ADC_MONO_L_ASRC_SFT		2
+#define RT5665_ADC_MONO_R_ASRC_MASK		(0x1 << 1)
+#define RT5665_ADC_MONO_R_ASRC_SFT		1
+
+/* PLL tracking mode 2 (0x0084)*/
+#define RT5665_DA_STO1_CLK_SEL_MASK		(0x7 << 12)
+#define RT5665_DA_STO1_CLK_SEL_SFT		12
+#define RT5665_DA_STO2_CLK_SEL_MASK		(0x7 << 8)
+#define RT5665_DA_STO2_CLK_SEL_SFT		8
+#define RT5665_DA_MONOL_CLK_SEL_MASK		(0x7 << 4)
+#define RT5665_DA_MONOL_CLK_SEL_SFT		4
+#define RT5665_DA_MONOR_CLK_SEL_MASK		(0x7)
+#define RT5665_DA_MONOR_CLK_SEL_SFT		0
+
+/* PLL tracking mode 3 (0x0085)*/
+#define RT5665_AD_STO1_CLK_SEL_MASK		(0x7 << 12)
+#define RT5665_AD_STO1_CLK_SEL_SFT		12
+#define RT5665_AD_STO2_CLK_SEL_MASK		(0x7 << 8)
+#define RT5665_AD_STO2_CLK_SEL_SFT		8
+#define RT5665_AD_MONOL_CLK_SEL_MASK		(0x7 << 4)
+#define RT5665_AD_MONOL_CLK_SEL_SFT		4
+#define RT5665_AD_MONOR_CLK_SEL_MASK		(0x7)
+#define RT5665_AD_MONOR_CLK_SEL_SFT		0
+
+/* ASRC Control 4 (0x0086) */
+#define RT5665_I2S1_RATE_MASK			(0xf << 12)
+#define RT5665_I2S1_RATE_SFT			12
+#define RT5665_I2S2_RATE_MASK			(0xf << 8)
+#define RT5665_I2S2_RATE_SFT			8
+#define RT5665_I2S3_RATE_MASK			(0xf << 4)
+#define RT5665_I2S3_RATE_SFT			4
+
+/* Depop Mode Control 1 (0x008e) */
+#define RT5665_PUMP_EN				(0x1 << 3)
+
+/* Depop Mode Control 2 (0x8f) */
+#define RT5665_DEPOP_MASK			(0x1 << 13)
+#define RT5665_DEPOP_SFT			13
+#define RT5665_DEPOP_AUTO			(0x0 << 13)
+#define RT5665_DEPOP_MAN			(0x1 << 13)
+#define RT5665_RAMP_MASK			(0x1 << 12)
+#define RT5665_RAMP_SFT				12
+#define RT5665_RAMP_DIS				(0x0 << 12)
+#define RT5665_RAMP_EN				(0x1 << 12)
+#define RT5665_BPS_MASK				(0x1 << 11)
+#define RT5665_BPS_SFT				11
+#define RT5665_BPS_DIS				(0x0 << 11)
+#define RT5665_BPS_EN				(0x1 << 11)
+#define RT5665_FAST_UPDN_MASK			(0x1 << 10)
+#define RT5665_FAST_UPDN_SFT			10
+#define RT5665_FAST_UPDN_DIS			(0x0 << 10)
+#define RT5665_FAST_UPDN_EN			(0x1 << 10)
+#define RT5665_MRES_MASK			(0x3 << 8)
+#define RT5665_MRES_SFT				8
+#define RT5665_MRES_15MO			(0x0 << 8)
+#define RT5665_MRES_25MO			(0x1 << 8)
+#define RT5665_MRES_35MO			(0x2 << 8)
+#define RT5665_MRES_45MO			(0x3 << 8)
+#define RT5665_VLO_MASK				(0x1 << 7)
+#define RT5665_VLO_SFT				7
+#define RT5665_VLO_3V				(0x0 << 7)
+#define RT5665_VLO_32V				(0x1 << 7)
+#define RT5665_DIG_DP_MASK			(0x1 << 6)
+#define RT5665_DIG_DP_SFT			6
+#define RT5665_DIG_DP_DIS			(0x0 << 6)
+#define RT5665_DIG_DP_EN			(0x1 << 6)
+#define RT5665_DP_TH_MASK			(0x3 << 4)
+#define RT5665_DP_TH_SFT			4
+
+/* Depop Mode Control 3 (0x90) */
+#define RT5665_CP_SYS_MASK			(0x7 << 12)
+#define RT5665_CP_SYS_SFT			12
+#define RT5665_CP_FQ1_MASK			(0x7 << 8)
+#define RT5665_CP_FQ1_SFT			8
+#define RT5665_CP_FQ2_MASK			(0x7 << 4)
+#define RT5665_CP_FQ2_SFT			4
+#define RT5665_CP_FQ3_MASK			(0x7)
+#define RT5665_CP_FQ3_SFT			0
+#define RT5665_CP_FQ_1_5_KHZ			0
+#define RT5665_CP_FQ_3_KHZ			1
+#define RT5665_CP_FQ_6_KHZ			2
+#define RT5665_CP_FQ_12_KHZ			3
+#define RT5665_CP_FQ_24_KHZ			4
+#define RT5665_CP_FQ_48_KHZ			5
+#define RT5665_CP_FQ_96_KHZ			6
+#define RT5665_CP_FQ_192_KHZ			7
+
+/* HPOUT charge pump 1 (0x0091) */
+#define RT5665_OSW_L_MASK			(0x1 << 11)
+#define RT5665_OSW_L_SFT			11
+#define RT5665_OSW_L_DIS			(0x0 << 11)
+#define RT5665_OSW_L_EN				(0x1 << 11)
+#define RT5665_OSW_R_MASK			(0x1 << 10)
+#define RT5665_OSW_R_SFT			10
+#define RT5665_OSW_R_DIS			(0x0 << 10)
+#define RT5665_OSW_R_EN				(0x1 << 10)
+#define RT5665_PM_HP_MASK			(0x3 << 8)
+#define RT5665_PM_HP_SFT			8
+#define RT5665_PM_HP_LV				(0x0 << 8)
+#define RT5665_PM_HP_MV				(0x1 << 8)
+#define RT5665_PM_HP_HV				(0x2 << 8)
+#define RT5665_IB_HP_MASK			(0x3 << 6)
+#define RT5665_IB_HP_SFT			6
+#define RT5665_IB_HP_125IL			(0x0 << 6)
+#define RT5665_IB_HP_25IL			(0x1 << 6)
+#define RT5665_IB_HP_5IL			(0x2 << 6)
+#define RT5665_IB_HP_1IL			(0x3 << 6)
+
+/* PV detection and SPK gain control (0x92) */
+#define RT5665_PVDD_DET_MASK			(0x1 << 15)
+#define RT5665_PVDD_DET_SFT			15
+#define RT5665_PVDD_DET_DIS			(0x0 << 15)
+#define RT5665_PVDD_DET_EN			(0x1 << 15)
+#define RT5665_SPK_AG_MASK			(0x1 << 14)
+#define RT5665_SPK_AG_SFT			14
+#define RT5665_SPK_AG_DIS			(0x0 << 14)
+#define RT5665_SPK_AG_EN			(0x1 << 14)
+
+/* Micbias Control1 (0x93) */
+#define RT5665_MIC1_BS_MASK			(0x1 << 15)
+#define RT5665_MIC1_BS_SFT			15
+#define RT5665_MIC1_BS_9AV			(0x0 << 15)
+#define RT5665_MIC1_BS_75AV			(0x1 << 15)
+#define RT5665_MIC2_BS_MASK			(0x1 << 14)
+#define RT5665_MIC2_BS_SFT			14
+#define RT5665_MIC2_BS_9AV			(0x0 << 14)
+#define RT5665_MIC2_BS_75AV			(0x1 << 14)
+#define RT5665_MIC1_CLK_MASK			(0x1 << 13)
+#define RT5665_MIC1_CLK_SFT			13
+#define RT5665_MIC1_CLK_DIS			(0x0 << 13)
+#define RT5665_MIC1_CLK_EN			(0x1 << 13)
+#define RT5665_MIC2_CLK_MASK			(0x1 << 12)
+#define RT5665_MIC2_CLK_SFT			12
+#define RT5665_MIC2_CLK_DIS			(0x0 << 12)
+#define RT5665_MIC2_CLK_EN			(0x1 << 12)
+#define RT5665_MIC1_OVCD_MASK			(0x1 << 11)
+#define RT5665_MIC1_OVCD_SFT			11
+#define RT5665_MIC1_OVCD_DIS			(0x0 << 11)
+#define RT5665_MIC1_OVCD_EN			(0x1 << 11)
+#define RT5665_MIC1_OVTH_MASK			(0x3 << 9)
+#define RT5665_MIC1_OVTH_SFT			9
+#define RT5665_MIC1_OVTH_600UA			(0x0 << 9)
+#define RT5665_MIC1_OVTH_1500UA			(0x1 << 9)
+#define RT5665_MIC1_OVTH_2000UA			(0x2 << 9)
+#define RT5665_MIC2_OVCD_MASK			(0x1 << 8)
+#define RT5665_MIC2_OVCD_SFT			8
+#define RT5665_MIC2_OVCD_DIS			(0x0 << 8)
+#define RT5665_MIC2_OVCD_EN			(0x1 << 8)
+#define RT5665_MIC2_OVTH_MASK			(0x3 << 6)
+#define RT5665_MIC2_OVTH_SFT			6
+#define RT5665_MIC2_OVTH_600UA			(0x0 << 6)
+#define RT5665_MIC2_OVTH_1500UA			(0x1 << 6)
+#define RT5665_MIC2_OVTH_2000UA			(0x2 << 6)
+#define RT5665_PWR_MB_MASK			(0x1 << 5)
+#define RT5665_PWR_MB_SFT			5
+#define RT5665_PWR_MB_PD			(0x0 << 5)
+#define RT5665_PWR_MB_PU			(0x1 << 5)
+
+/* Micbias Control2 (0x94) */
+#define RT5665_PWR_CLK25M_MASK			(0x1 << 9)
+#define RT5665_PWR_CLK25M_SFT			9
+#define RT5665_PWR_CLK25M_PD			(0x0 << 9)
+#define RT5665_PWR_CLK25M_PU			(0x1 << 9)
+#define RT5665_PWR_CLK1M_MASK			(0x1 << 8)
+#define RT5665_PWR_CLK1M_SFT			8
+#define RT5665_PWR_CLK1M_PD			(0x0 << 8)
+#define RT5665_PWR_CLK1M_PU			(0x1 << 8)
+
+
+/* EQ Control 1 (0x00b0) */
+#define RT5665_EQ_SRC_DAC			(0x0 << 15)
+#define RT5665_EQ_SRC_ADC			(0x1 << 15)
+#define RT5665_EQ_UPD				(0x1 << 14)
+#define RT5665_EQ_UPD_BIT			14
+#define RT5665_EQ_CD_MASK			(0x1 << 13)
+#define RT5665_EQ_CD_SFT			13
+#define RT5665_EQ_CD_DIS			(0x0 << 13)
+#define RT5665_EQ_CD_EN				(0x1 << 13)
+#define RT5665_EQ_DITH_MASK			(0x3 << 8)
+#define RT5665_EQ_DITH_SFT			8
+#define RT5665_EQ_DITH_NOR			(0x0 << 8)
+#define RT5665_EQ_DITH_LSB			(0x1 << 8)
+#define RT5665_EQ_DITH_LSB_1			(0x2 << 8)
+#define RT5665_EQ_DITH_LSB_2			(0x3 << 8)
+
+/* IRQ Control 1 (0x00b7) */
+#define RT5665_JD1_1_EN_MASK			(0x1 << 15)
+#define RT5665_JD1_1_EN_SFT			15
+#define RT5665_JD1_1_DIS			(0x0 << 15)
+#define RT5665_JD1_1_EN				(0x1 << 15)
+#define RT5665_JD1_2_EN_MASK			(0x1 << 12)
+#define RT5665_JD1_2_EN_SFT			12
+#define RT5665_JD1_2_DIS			(0x0 << 12)
+#define RT5665_JD1_2_EN				(0x1 << 12)
+
+/* IRQ Control 2 (0x00b8) */
+#define RT5665_IL_IRQ_MASK			(0x1 << 6)
+#define RT5665_IL_IRQ_DIS			(0x0 << 6)
+#define RT5665_IL_IRQ_EN			(0x1 << 6)
+
+/* IRQ Control 5 (0x00ba) */
+#define RT5665_IRQ_JD_EN			(0x1 << 3)
+#define RT5665_IRQ_JD_EN_SFT			3
+
+/* GPIO Control 1 (0x00c0) */
+#define RT5665_GP1_PIN_MASK			(0x1 << 15)
+#define RT5665_GP1_PIN_SFT			15
+#define RT5665_GP1_PIN_GPIO1			(0x0 << 15)
+#define RT5665_GP1_PIN_IRQ			(0x1 << 15)
+#define RT5665_GP2_PIN_MASK			(0x3 << 13)
+#define RT5665_GP2_PIN_SFT			13
+#define RT5665_GP2_PIN_GPIO2			(0x0 << 13)
+#define RT5665_GP2_PIN_BCLK2			(0x1 << 13)
+#define RT5665_GP2_PIN_PDM_SCL			(0x2 << 13)
+#define RT5665_GP3_PIN_MASK			(0x3 << 11)
+#define RT5665_GP3_PIN_SFT			11
+#define RT5665_GP3_PIN_GPIO3			(0x0 << 11)
+#define RT5665_GP3_PIN_LRCK2			(0x1 << 11)
+#define RT5665_GP3_PIN_PDM_SDA			(0x2 << 11)
+#define RT5665_GP4_PIN_MASK			(0x3 << 9)
+#define RT5665_GP4_PIN_SFT			9
+#define RT5665_GP4_PIN_GPIO4			(0x0 << 9)
+#define RT5665_GP4_PIN_DACDAT2_1		(0x1 << 9)
+#define RT5665_GP4_PIN_DMIC1_SDA		(0x2 << 9)
+#define RT5665_GP5_PIN_MASK			(0x3 << 7)
+#define RT5665_GP5_PIN_SFT			7
+#define RT5665_GP5_PIN_GPIO5			(0x0 << 7)
+#define RT5665_GP5_PIN_ADCDAT2_1		(0x1 << 7)
+#define RT5665_GP5_PIN_DMIC2_SDA		(0x2 << 7)
+#define RT5665_GP6_PIN_MASK			(0x3 << 5)
+#define RT5665_GP6_PIN_SFT			5
+#define RT5665_GP6_PIN_GPIO6			(0x0 << 5)
+#define RT5665_GP6_PIN_BCLK3			(0x0 << 5)
+#define RT5665_GP6_PIN_PDM_SCL			(0x1 << 5)
+#define RT5665_GP7_PIN_MASK			(0x3 << 3)
+#define RT5665_GP7_PIN_SFT			3
+#define RT5665_GP7_PIN_GPIO7			(0x0 << 3)
+#define RT5665_GP7_PIN_LRCK3			(0x1 << 3)
+#define RT5665_GP7_PIN_PDM_SDA			(0x2 << 3)
+#define RT5665_GP8_PIN_MASK			(0x3 << 1)
+#define RT5665_GP8_PIN_SFT			1
+#define RT5665_GP8_PIN_GPIO8			(0x0 << 1)
+#define RT5665_GP8_PIN_DACDAT3			(0x1 << 1)
+#define RT5665_GP8_PIN_DMIC2_SCL		(0x2 << 1)
+#define RT5665_GP8_PIN_DACDAT2_2		(0x3 << 1)
+
+
+/* GPIO Control 2 (0x00c1)*/
+#define RT5665_GP9_PIN_MASK			(0x3 << 14)
+#define RT5665_GP9_PIN_SFT			14
+#define RT5665_GP9_PIN_GPIO9			(0x0 << 14)
+#define RT5665_GP9_PIN_ADCDAT3			(0x1 << 14)
+#define RT5665_GP9_PIN_DMIC1_SCL		(0x2 << 14)
+#define RT5665_GP9_PIN_ADCDAT2_2		(0x3 << 14)
+#define RT5665_GP10_PIN_MASK			(0x3 << 12)
+#define RT5665_GP10_PIN_SFT			12
+#define RT5665_GP10_PIN_GPIO10			(0x0 << 12)
+#define RT5665_GP10_PIN_ADCDAT1_2		(0x1 << 12)
+#define RT5665_GP10_PIN_LPD			(0x2 << 12)
+#define RT5665_GP1_PF_MASK			(0x1 << 11)
+#define RT5665_GP1_PF_IN			(0x0 << 11)
+#define RT5665_GP1_PF_OUT			(0x1 << 11)
+#define RT5665_GP1_OUT_MASK			(0x1 << 10)
+#define RT5665_GP1_OUT_H			(0x0 << 10)
+#define RT5665_GP1_OUT_L			(0x1 << 10)
+#define RT5665_GP2_PF_MASK			(0x1 << 9)
+#define RT5665_GP2_PF_IN			(0x0 << 9)
+#define RT5665_GP2_PF_OUT			(0x1 << 9)
+#define RT5665_GP2_OUT_MASK			(0x1 << 8)
+#define RT5665_GP2_OUT_H			(0x0 << 8)
+#define RT5665_GP2_OUT_L			(0x1 << 8)
+#define RT5665_GP3_PF_MASK			(0x1 << 7)
+#define RT5665_GP3_PF_IN			(0x0 << 7)
+#define RT5665_GP3_PF_OUT			(0x1 << 7)
+#define RT5665_GP3_OUT_MASK			(0x1 << 6)
+#define RT5665_GP3_OUT_H			(0x0 << 6)
+#define RT5665_GP3_OUT_L			(0x1 << 6)
+#define RT5665_GP4_PF_MASK			(0x1 << 5)
+#define RT5665_GP4_PF_IN			(0x0 << 5)
+#define RT5665_GP4_PF_OUT			(0x1 << 5)
+#define RT5665_GP4_OUT_MASK			(0x1 << 4)
+#define RT5665_GP4_OUT_H			(0x0 << 4)
+#define RT5665_GP4_OUT_L			(0x1 << 4)
+#define RT5665_GP5_PF_MASK			(0x1 << 3)
+#define RT5665_GP5_PF_IN			(0x0 << 3)
+#define RT5665_GP5_PF_OUT			(0x1 << 3)
+#define RT5665_GP5_OUT_MASK			(0x1 << 2)
+#define RT5665_GP5_OUT_H			(0x0 << 2)
+#define RT5665_GP5_OUT_L			(0x1 << 2)
+#define RT5665_GP6_PF_MASK			(0x1 << 1)
+#define RT5665_GP6_PF_IN			(0x0 << 1)
+#define RT5665_GP6_PF_OUT			(0x1 << 1)
+#define RT5665_GP6_OUT_MASK			(0x1)
+#define RT5665_GP6_OUT_H			(0x0)
+#define RT5665_GP6_OUT_L			(0x1)
+
+
+/* GPIO Control 3 (0x00c2) */
+#define RT5665_GP7_PF_MASK			(0x1 << 15)
+#define RT5665_GP7_PF_IN			(0x0 << 15)
+#define RT5665_GP7_PF_OUT			(0x1 << 15)
+#define RT5665_GP7_OUT_MASK			(0x1 << 14)
+#define RT5665_GP7_OUT_H			(0x0 << 14)
+#define RT5665_GP7_OUT_L			(0x1 << 14)
+#define RT5665_GP8_PF_MASK			(0x1 << 13)
+#define RT5665_GP8_PF_IN			(0x0 << 13)
+#define RT5665_GP8_PF_OUT			(0x1 << 13)
+#define RT5665_GP8_OUT_MASK			(0x1 << 12)
+#define RT5665_GP8_OUT_H			(0x0 << 12)
+#define RT5665_GP8_OUT_L			(0x1 << 12)
+#define RT5665_GP9_PF_MASK			(0x1 << 11)
+#define RT5665_GP9_PF_IN			(0x0 << 11)
+#define RT5665_GP9_PF_OUT			(0x1 << 11)
+#define RT5665_GP9_OUT_MASK			(0x1 << 10)
+#define RT5665_GP9_OUT_H			(0x0 << 10)
+#define RT5665_GP9_OUT_L			(0x1 << 10)
+#define RT5665_GP10_PF_MASK			(0x1 << 9)
+#define RT5665_GP10_PF_IN			(0x0 << 9)
+#define RT5665_GP10_PF_OUT			(0x1 << 9)
+#define RT5665_GP10_OUT_MASK			(0x1 << 8)
+#define RT5665_GP10_OUT_H			(0x0 << 8)
+#define RT5665_GP10_OUT_L			(0x1 << 8)
+#define RT5665_GP11_PF_MASK			(0x1 << 7)
+#define RT5665_GP11_PF_IN			(0x0 << 7)
+#define RT5665_GP11_PF_OUT			(0x1 << 7)
+#define RT5665_GP11_OUT_MASK			(0x1 << 6)
+#define RT5665_GP11_OUT_H			(0x0 << 6)
+#define RT5665_GP11_OUT_L			(0x1 << 6)
+
+/* Soft volume and zero cross control 1 (0x00d9) */
+#define RT5665_SV_MASK				(0x1 << 15)
+#define RT5665_SV_SFT				15
+#define RT5665_SV_DIS				(0x0 << 15)
+#define RT5665_SV_EN				(0x1 << 15)
+#define RT5665_OUT_SV_MASK			(0x1 << 13)
+#define RT5665_OUT_SV_SFT			13
+#define RT5665_OUT_SV_DIS			(0x0 << 13)
+#define RT5665_OUT_SV_EN			(0x1 << 13)
+#define RT5665_HP_SV_MASK			(0x1 << 12)
+#define RT5665_HP_SV_SFT			12
+#define RT5665_HP_SV_DIS			(0x0 << 12)
+#define RT5665_HP_SV_EN				(0x1 << 12)
+#define RT5665_ZCD_DIG_MASK			(0x1 << 11)
+#define RT5665_ZCD_DIG_SFT			11
+#define RT5665_ZCD_DIG_DIS			(0x0 << 11)
+#define RT5665_ZCD_DIG_EN			(0x1 << 11)
+#define RT5665_ZCD_MASK				(0x1 << 10)
+#define RT5665_ZCD_SFT				10
+#define RT5665_ZCD_PD				(0x0 << 10)
+#define RT5665_ZCD_PU				(0x1 << 10)
+#define RT5665_SV_DLY_MASK			(0xf)
+#define RT5665_SV_DLY_SFT			0
+
+/* Soft volume and zero cross control 2 (0x00da) */
+#define RT5665_ZCD_HP_MASK			(0x1 << 15)
+#define RT5665_ZCD_HP_SFT			15
+#define RT5665_ZCD_HP_DIS			(0x0 << 15)
+#define RT5665_ZCD_HP_EN			(0x1 << 15)
+
+/* 4 Button Inline Command Control 2 (0x00e0) */
+#define RT5665_4BTN_IL_MASK			(0x1 << 15)
+#define RT5665_4BTN_IL_EN			(0x1 << 15)
+#define RT5665_4BTN_IL_DIS			(0x0 << 15)
+#define RT5665_4BTN_IL_RST_MASK			(0x1 << 14)
+#define RT5665_4BTN_IL_NOR			(0x1 << 14)
+#define RT5665_4BTN_IL_RST			(0x0 << 14)
+
+/* Analog JD Control 1 (0x00f0) */
+#define RT5665_JD1_MODE_MASK			(0x3 << 0)
+#define RT5665_JD1_MODE_0			(0x0 << 0)
+#define RT5665_JD1_MODE_1			(0x1 << 0)
+#define RT5665_JD1_MODE_2			(0x2 << 0)
+
+/* Jack Detect Control 3 (0x00f8) */
+#define RT5665_JD_TRI_HPO_SEL_MASK		(0x7)
+#define RT5665_JD_TRI_HPO_SEL_SFT		(0)
+#define RT5665_JD_HPO_GPIO_JD1			(0x0)
+#define RT5665_JD_HPO_JD1_1			(0x1)
+#define RT5665_JD_HPO_JD1_2			(0x2)
+#define RT5665_JD_HPO_JD2			(0x3)
+#define RT5665_JD_HPO_GPIO_JD2			(0x4)
+#define RT5665_JD_HPO_JD3			(0x5)
+#define RT5665_JD_HPO_JD_D			(0x6)
+
+/* Digital Misc Control (0x00fa) */
+#define RT5665_AM_MASK				(0x1 << 7)
+#define RT5665_AM_EN				(0x1 << 7)
+#define RT5665_AM_DIS				(0x1 << 7)
+#define RT5665_DIG_GATE_CTRL			0x1
+#define RT5665_DIG_GATE_CTRL_SFT		(0)
+
+/* Chopper and Clock control for ADC (0x011c)*/
+#define RT5665_M_RF_DIG_MASK			(0x1 << 12)
+#define RT5665_M_RF_DIG_SFT			12
+#define RT5665_M_RI_DIG				(0x1 << 11)
+
+/* Chopper and Clock control for DAC (0x013a)*/
+#define RT5665_CKXEN_DAC1_MASK			(0x1 << 13)
+#define RT5665_CKXEN_DAC1_SFT			13
+#define RT5665_CKGEN_DAC1_MASK			(0x1 << 12)
+#define RT5665_CKGEN_DAC1_SFT			12
+#define RT5665_CKXEN_DAC2_MASK			(0x1 << 5)
+#define RT5665_CKXEN_DAC2_SFT			5
+#define RT5665_CKGEN_DAC2_MASK			(0x1 << 4)
+#define RT5665_CKGEN_DAC2_SFT			4
+
+/* Chopper and Clock control for ADC (0x013b)*/
+#define RT5665_CKXEN_ADC1_MASK			(0x1 << 13)
+#define RT5665_CKXEN_ADC1_SFT			13
+#define RT5665_CKGEN_ADC1_MASK			(0x1 << 12)
+#define RT5665_CKGEN_ADC1_SFT			12
+#define RT5665_CKXEN_ADC2_MASK			(0x1 << 5)
+#define RT5665_CKXEN_ADC2_SFT			5
+#define RT5665_CKGEN_ADC2_MASK			(0x1 << 4)
+#define RT5665_CKGEN_ADC2_SFT			4
+
+/* Volume test (0x013f)*/
+#define RT5665_SEL_CLK_VOL_MASK			(0x1 << 15)
+#define RT5665_SEL_CLK_VOL_EN			(0x1 << 15)
+#define RT5665_SEL_CLK_VOL_DIS			(0x0 << 15)
+
+/* Test Mode Control 1 (0x0145) */
+#define RT5665_AD2DA_LB_MASK			(0x1 << 9)
+#define RT5665_AD2DA_LB_SFT			9
+
+/* Stereo Noise Gate Control 1 (0x0160) */
+#define RT5665_NG2_EN_MASK			(0x1 << 15)
+#define RT5665_NG2_EN				(0x1 << 15)
+#define RT5665_NG2_DIS				(0x0 << 15)
+
+/* Stereo1 DAC Silence Detection Control (0x0190) */
+#define RT5665_DEB_STO_DAC_MASK			(0x7 << 4)
+#define RT5665_DEB_80_MS			(0x0 << 4)
+
+/* SAR ADC Inline Command Control 1 (0x0210) */
+#define RT5665_SAR_BUTT_DET_MASK		(0x1 << 15)
+#define RT5665_SAR_BUTT_DET_EN			(0x1 << 15)
+#define RT5665_SAR_BUTT_DET_DIS			(0x0 << 15)
+#define RT5665_SAR_BUTDET_MODE_MASK		(0x1 << 14)
+#define RT5665_SAR_BUTDET_POW_SAV		(0x1 << 14)
+#define RT5665_SAR_BUTDET_POW_NORM		(0x0 << 14)
+#define RT5665_SAR_BUTDET_RST_MASK		(0x1 << 13)
+#define RT5665_SAR_BUTDET_RST_NORMAL		(0x1 << 13)
+#define RT5665_SAR_BUTDET_RST			(0x0 << 13)
+#define RT5665_SAR_POW_MASK			(0x1 << 12)
+#define RT5665_SAR_POW_EN			(0x1 << 12)
+#define RT5665_SAR_POW_DIS			(0x0 << 12)
+#define RT5665_SAR_RST_MASK			(0x1 << 11)
+#define RT5665_SAR_RST_NORMAL			(0x1 << 11)
+#define RT5665_SAR_RST				(0x0 << 11)
+#define RT5665_SAR_BYPASS_MASK			(0x1 << 10)
+#define RT5665_SAR_BYPASS_EN			(0x1 << 10)
+#define RT5665_SAR_BYPASS_DIS			(0x0 << 10)
+#define RT5665_SAR_SEL_MB1_MASK			(0x1 << 9)
+#define RT5665_SAR_SEL_MB1_SEL			(0x1 << 9)
+#define RT5665_SAR_SEL_MB1_NOSEL		(0x0 << 9)
+#define RT5665_SAR_SEL_MB2_MASK			(0x1 << 8)
+#define RT5665_SAR_SEL_MB2_SEL			(0x1 << 8)
+#define RT5665_SAR_SEL_MB2_NOSEL		(0x0 << 8)
+#define RT5665_SAR_SEL_MODE_MASK		(0x1 << 7)
+#define RT5665_SAR_SEL_MODE_CMP			(0x1 << 7)
+#define RT5665_SAR_SEL_MODE_ADC			(0x0 << 7)
+#define RT5665_SAR_SEL_MB1_MB2_MASK		(0x1 << 5)
+#define RT5665_SAR_SEL_MB1_MB2_AUTO		(0x1 << 5)
+#define RT5665_SAR_SEL_MB1_MB2_MANU		(0x0 << 5)
+#define RT5665_SAR_SEL_SIGNAL_MASK		(0x1 << 4)
+#define RT5665_SAR_SEL_SIGNAL_AUTO		(0x1 << 4)
+#define RT5665_SAR_SEL_SIGNAL_MANU		(0x0 << 4)
+
+/* System Clock Source */
+enum {
+	RT5665_SCLK_S_MCLK,
+	RT5665_SCLK_S_PLL1,
+	RT5665_SCLK_S_RCCLK,
+};
+
+/* PLL1 Source */
+enum {
+	RT5665_PLL1_S_MCLK,
+	RT5665_PLL1_S_BCLK1,
+	RT5665_PLL1_S_BCLK2,
+	RT5665_PLL1_S_BCLK3,
+	RT5665_PLL1_S_BCLK4,
+};
+
+enum {
+	RT5665_AIF1_1,
+	RT5665_AIF1_2,
+	RT5665_AIF2_1,
+	RT5665_AIF2_2,
+	RT5665_AIF3,
+	RT5665_AIFS
+};
+
+enum {
+	CODEC_5665,
+	CODEC_5666,
+	CODEC_5668,
+};
+
+/* filter mask */
+enum {
+	RT5665_DA_STEREO1_FILTER = 0x1,
+	RT5665_DA_STEREO2_FILTER = (0x1 << 1),
+	RT5665_DA_MONO_L_FILTER = (0x1 << 2),
+	RT5665_DA_MONO_R_FILTER = (0x1 << 3),
+	RT5665_AD_STEREO1_FILTER = (0x1 << 4),
+	RT5665_AD_STEREO2_FILTER = (0x1 << 5),
+	RT5665_AD_MONO_L_FILTER = (0x1 << 6),
+	RT5665_AD_MONO_R_FILTER = (0x1 << 7),
+};
+
+enum {
+	RT5665_CLK_SEL_SYS,
+	RT5665_CLK_SEL_I2S1_ASRC,
+	RT5665_CLK_SEL_I2S2_ASRC,
+	RT5665_CLK_SEL_I2S3_ASRC,
+	RT5665_CLK_SEL_SYS2,
+	RT5665_CLK_SEL_SYS3,
+	RT5665_CLK_SEL_SYS4,
+};
+
+int rt5665_sel_asrc_clk_src(struct snd_soc_codec *codec,
+		unsigned int filter_mask, unsigned int clk_src);
+int rt5665_set_jack_detect(struct snd_soc_codec *codec,
+	struct snd_soc_jack *hs_jack);
+
+#endif /* __RT5665_H__ */
diff --git a/sound/soc/codecs/rt5670.c b/sound/soc/codecs/rt5670.c
index 49caf1393aeb..97bafac3bc15 100644
--- a/sound/soc/codecs/rt5670.c
+++ b/sound/soc/codecs/rt5670.c
@@ -2618,7 +2618,7 @@ static int rt5670_set_bias_level(struct snd_soc_codec *codec,
 				RT5670_OSW_L_DIS | RT5670_OSW_R_DIS);
 			snd_soc_update_bits(codec, RT5670_DIG_MISC, 0x1, 0x1);
 			snd_soc_update_bits(codec, RT5670_PWR_ANLG1,
-				RT5670_LDO_SEL_MASK, 0x3);
+				RT5670_LDO_SEL_MASK, 0x5);
 		}
 		break;
 	case SND_SOC_BIAS_STANDBY:
@@ -2626,7 +2626,7 @@ static int rt5670_set_bias_level(struct snd_soc_codec *codec,
 				RT5670_PWR_VREF1 | RT5670_PWR_VREF2 |
 				RT5670_PWR_FV1 | RT5670_PWR_FV2, 0);
 		snd_soc_update_bits(codec, RT5670_PWR_ANLG1,
-				RT5670_LDO_SEL_MASK, 0x1);
+				RT5670_LDO_SEL_MASK, 0x3);
 		break;
 	case SND_SOC_BIAS_OFF:
 		if (rt5670->pdata.jd_mode)
@@ -2813,6 +2813,7 @@ MODULE_DEVICE_TABLE(i2c, rt5670_i2c_id);
 #ifdef CONFIG_ACPI
 static const struct acpi_device_id rt5670_acpi_match[] = {
 	{ "10EC5670", 0},
+	{ "10EC5672", 0},
 	{ },
 };
 MODULE_DEVICE_TABLE(acpi, rt5670_acpi_match);
@@ -2826,6 +2827,13 @@ static const struct dmi_system_id dmi_platform_intel_braswell[] = {
 			DMI_MATCH(DMI_BOARD_NAME, "Braswell CRB"),
 		},
 	},
+	{
+		.ident = "Dell Wyse 3040",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Wyse 3040"),
+		},
+	},
 	{}
 };
 
@@ -2889,6 +2897,9 @@ static int rt5670_i2c_probe(struct i2c_client *i2c,
 	if (ret != 0)
 		dev_warn(&i2c->dev, "Failed to apply regmap patch: %d\n", ret);
 
+	regmap_update_bits(rt5670->regmap, RT5670_DIG_MISC,
+				 RT5670_MCLK_DET, RT5670_MCLK_DET);
+
 	if (rt5670->pdata.in2_diff)
 		regmap_update_bits(rt5670->regmap, RT5670_IN2,
 					RT5670_IN_DF2, RT5670_IN_DF2);
@@ -2903,7 +2914,6 @@ static int rt5670_i2c_probe(struct i2c_client *i2c,
 				   RT5670_GP1_PIN_MASK, RT5670_GP1_PIN_IRQ);
 		regmap_update_bits(rt5670->regmap, RT5670_GPIO_CTRL2,
 				   RT5670_GP1_PF_MASK, RT5670_GP1_PF_OUT);
-		regmap_update_bits(rt5670->regmap, RT5670_DIG_MISC, 0x8, 0x8);
 	}
 
 	if (rt5670->pdata.jd_mode) {
diff --git a/sound/soc/codecs/rt5670.h b/sound/soc/codecs/rt5670.h
index 3f1b0f1df809..5ba485cae4e6 100644
--- a/sound/soc/codecs/rt5670.h
+++ b/sound/soc/codecs/rt5670.h
@@ -1914,6 +1914,7 @@ enum {
 #define RT5670_IF1_ADC1_IN2_SFT			11
 #define RT5670_IF1_ADC2_IN1_SEL			(0x1 << 10)
 #define RT5670_IF1_ADC2_IN1_SFT			10
+#define RT5670_MCLK_DET				(0x1 << 3)
 
 /* General Control2 (0xfb) */
 #define RT5670_RXDC_SRC_MASK			(0x1 << 7)
diff --git a/sound/soc/codecs/rt5677-spi.c b/sound/soc/codecs/rt5677-spi.c
index 91879ea95415..ebd0f7c5ad3b 100644
--- a/sound/soc/codecs/rt5677-spi.c
+++ b/sound/soc/codecs/rt5677-spi.c
@@ -20,7 +20,6 @@
 #include <linux/slab.h>
 #include <linux/gpio.h>
 #include <linux/sched.h>
-#include <linux/kthread.h>
 #include <linux/uaccess.h>
 #include <linux/miscdevice.h>
 #include <linux/regulator/consumer.h>
diff --git a/sound/soc/codecs/stac9766.c b/sound/soc/codecs/stac9766.c
index 27f30d352867..9de7fe8af255 100644
--- a/sound/soc/codecs/stac9766.c
+++ b/sound/soc/codecs/stac9766.c
@@ -18,6 +18,7 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/device.h>
+#include <linux/regmap.h>
 #include <sound/core.h>
 #include <sound/pcm.h>
 #include <sound/ac97_codec.h>
@@ -26,31 +27,56 @@
 #include <sound/soc.h>
 #include <sound/tlv.h>
 
-#include "stac9766.h"
-
 #define STAC9766_VENDOR_ID 0x83847666
 #define STAC9766_VENDOR_ID_MASK 0xffffffff
 
-/*
- * STAC9766 register cache
- */
-static const u16 stac9766_reg[] = {
-	0x6A90, 0x8000, 0x8000, 0x8000, /* 6 */
-	0x0000, 0x0000, 0x8008, 0x8008, /* e */
-	0x8808, 0x8808, 0x8808, 0x8808, /* 16 */
-	0x8808, 0x0000, 0x8000, 0x0000, /* 1e */
-	0x0000, 0x0000, 0x0000, 0x000f, /* 26 */
-	0x0a05, 0x0400, 0xbb80, 0x0000, /* 2e */
-	0x0000, 0xbb80, 0x0000, 0x0000, /* 36 */
-	0x0000, 0x2000, 0x0000, 0x0100, /* 3e */
-	0x0000, 0x0000, 0x0080, 0x0000, /* 46 */
-	0x0000, 0x0000, 0x0003, 0xffff, /* 4e */
-	0x0000, 0x0000, 0x0000, 0x0000, /* 56 */
-	0x4000, 0x0000, 0x0000, 0x0000, /* 5e */
-	0x1201, 0xFFFF, 0xFFFF, 0x0000, /* 66 */
-	0x0000, 0x0000, 0x0000, 0x0000, /* 6e */
-	0x0000, 0x0000, 0x0000, 0x0006, /* 76 */
-	0x0000, 0x0000, 0x0000, 0x0000, /* 7e */
+#define AC97_STAC_DA_CONTROL 0x6A
+#define AC97_STAC_ANALOG_SPECIAL 0x6E
+#define AC97_STAC_STEREO_MIC 0x78
+
+static const struct reg_default stac9766_reg_defaults[] = {
+	{ 0x02, 0x8000 },
+	{ 0x04, 0x8000 },
+	{ 0x06, 0x8000 },
+	{ 0x0a, 0x0000 },
+	{ 0x0c, 0x8008 },
+	{ 0x0e, 0x8008 },
+	{ 0x10, 0x8808 },
+	{ 0x12, 0x8808 },
+	{ 0x14, 0x8808 },
+	{ 0x16, 0x8808 },
+	{ 0x18, 0x8808 },
+	{ 0x1a, 0x0000 },
+	{ 0x1c, 0x8000 },
+	{ 0x20, 0x0000 },
+	{ 0x22, 0x0000 },
+	{ 0x28, 0x0a05 },
+	{ 0x2c, 0xbb80 },
+	{ 0x32, 0xbb80 },
+	{ 0x3a, 0x2000 },
+	{ 0x3e, 0x0100 },
+	{ 0x4c, 0x0300 },
+	{ 0x4e, 0xffff },
+	{ 0x50, 0x0000 },
+	{ 0x52, 0x0000 },
+	{ 0x54, 0x0000 },
+	{ 0x6a, 0x0000 },
+	{ 0x6e, 0x1000 },
+	{ 0x72, 0x0000 },
+	{ 0x78, 0x0000 },
+};
+
+static const struct regmap_config stac9766_regmap_config = {
+	.reg_bits = 16,
+	.reg_stride = 2,
+	.val_bits = 16,
+	.max_register = 0x78,
+	.cache_type = REGCACHE_RBTREE,
+
+	.volatile_reg = regmap_ac97_default_volatile,
+
+	.reg_defaults = stac9766_reg_defaults,
+	.num_reg_defaults = ARRAY_SIZE(stac9766_reg_defaults),
 };
 
 static const char *stac9766_record_mux[] = {"Mic", "CD", "Video", "AUX",
@@ -139,71 +165,22 @@ static const struct snd_kcontrol_new stac9766_snd_ac97_controls[] = {
 	SOC_ENUM("Pop Bypass Mux", stac9766_popbypass_enum),
 };
 
-static int stac9766_ac97_write(struct snd_soc_codec *codec, unsigned int reg,
-			       unsigned int val)
-{
-	struct snd_ac97 *ac97 = snd_soc_codec_get_drvdata(codec);
-	u16 *cache = codec->reg_cache;
-
-	if (reg > AC97_STAC_PAGE0) {
-		stac9766_ac97_write(codec, AC97_INT_PAGING, 0);
-		soc_ac97_ops->write(ac97, reg, val);
-		stac9766_ac97_write(codec, AC97_INT_PAGING, 1);
-		return 0;
-	}
-	if (reg / 2 >= ARRAY_SIZE(stac9766_reg))
-		return -EIO;
-
-	soc_ac97_ops->write(ac97, reg, val);
-	cache[reg / 2] = val;
-	return 0;
-}
-
-static unsigned int stac9766_ac97_read(struct snd_soc_codec *codec,
-				       unsigned int reg)
-{
-	struct snd_ac97 *ac97 = snd_soc_codec_get_drvdata(codec);
-	u16 val = 0, *cache = codec->reg_cache;
-
-	if (reg > AC97_STAC_PAGE0) {
-		stac9766_ac97_write(codec, AC97_INT_PAGING, 0);
-		val = soc_ac97_ops->read(ac97, reg - AC97_STAC_PAGE0);
-		stac9766_ac97_write(codec, AC97_INT_PAGING, 1);
-		return val;
-	}
-	if (reg / 2 >= ARRAY_SIZE(stac9766_reg))
-		return -EIO;
-
-	if (reg == AC97_RESET || reg == AC97_GPIO_STATUS ||
-		reg == AC97_INT_PAGING || reg == AC97_VENDOR_ID1 ||
-		reg == AC97_VENDOR_ID2) {
-
-		val = soc_ac97_ops->read(ac97, reg);
-		return val;
-	}
-	return cache[reg / 2];
-}
-
 static int ac97_analog_prepare(struct snd_pcm_substream *substream,
 			       struct snd_soc_dai *dai)
 {
 	struct snd_soc_codec *codec = dai->codec;
 	struct snd_pcm_runtime *runtime = substream->runtime;
-	unsigned short reg, vra;
-
-	vra = stac9766_ac97_read(codec, AC97_EXTENDED_STATUS);
+	unsigned short reg;
 
-	vra |= 0x1; /* enable variable rate audio */
-	vra &= ~0x4; /* disable SPDIF output */
-
-	stac9766_ac97_write(codec, AC97_EXTENDED_STATUS, vra);
+	/* enable variable rate audio, disable SPDIF output */
+	snd_soc_update_bits(codec, AC97_EXTENDED_STATUS, 0x5, 0x1);
 
 	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
 		reg = AC97_PCM_FRONT_DAC_RATE;
 	else
 		reg = AC97_PCM_LR_ADC_RATE;
 
-	return stac9766_ac97_write(codec, reg, runtime->rate);
+	return snd_soc_write(codec, reg, runtime->rate);
 }
 
 static int ac97_digital_prepare(struct snd_pcm_substream *substream,
@@ -211,18 +188,16 @@ static int ac97_digital_prepare(struct snd_pcm_substream *substream,
 {
 	struct snd_soc_codec *codec = dai->codec;
 	struct snd_pcm_runtime *runtime = substream->runtime;
-	unsigned short reg, vra;
-
-	stac9766_ac97_write(codec, AC97_SPDIF, 0x2002);
+	unsigned short reg;
 
-	vra = stac9766_ac97_read(codec, AC97_EXTENDED_STATUS);
-	vra |= 0x5; /* Enable VRA and SPDIF out */
+	snd_soc_write(codec, AC97_SPDIF, 0x2002);
 
-	stac9766_ac97_write(codec, AC97_EXTENDED_STATUS, vra);
+	/* Enable VRA and SPDIF out */
+	snd_soc_update_bits(codec, AC97_EXTENDED_STATUS, 0x5, 0x5);
 
 	reg = AC97_PCM_FRONT_DAC_RATE;
 
-	return stac9766_ac97_write(codec, reg, runtime->rate);
+	return snd_soc_write(codec, reg, runtime->rate);
 }
 
 static int stac9766_set_bias_level(struct snd_soc_codec *codec,
@@ -232,11 +207,11 @@ static int stac9766_set_bias_level(struct snd_soc_codec *codec,
 	case SND_SOC_BIAS_ON: /* full On */
 	case SND_SOC_BIAS_PREPARE: /* partial On */
 	case SND_SOC_BIAS_STANDBY: /* Off, with power */
-		stac9766_ac97_write(codec, AC97_POWERDOWN, 0x0000);
+		snd_soc_write(codec, AC97_POWERDOWN, 0x0000);
 		break;
 	case SND_SOC_BIAS_OFF: /* Off, without power */
 		/* disable everything including AC link */
-		stac9766_ac97_write(codec, AC97_POWERDOWN, 0xffff);
+		snd_soc_write(codec, AC97_POWERDOWN, 0xffff);
 		break;
 	}
 	return 0;
@@ -300,21 +275,34 @@ static struct snd_soc_dai_driver stac9766_dai[] = {
 static int stac9766_codec_probe(struct snd_soc_codec *codec)
 {
 	struct snd_ac97 *ac97;
+	struct regmap *regmap;
+	int ret;
 
 	ac97 = snd_soc_new_ac97_codec(codec, STAC9766_VENDOR_ID,
 			STAC9766_VENDOR_ID_MASK);
 	if (IS_ERR(ac97))
 		return PTR_ERR(ac97);
 
+	regmap = regmap_init_ac97(ac97, &stac9766_regmap_config);
+	if (IS_ERR(regmap)) {
+		ret = PTR_ERR(regmap);
+		goto err_free_ac97;
+	}
+
+	snd_soc_codec_init_regmap(codec, regmap);
 	snd_soc_codec_set_drvdata(codec, ac97);
 
 	return 0;
+err_free_ac97:
+	snd_soc_free_ac97_codec(ac97);
+	return ret;
 }
 
 static int stac9766_codec_remove(struct snd_soc_codec *codec)
 {
 	struct snd_ac97 *ac97 = snd_soc_codec_get_drvdata(codec);
 
+	snd_soc_codec_exit_regmap(codec);
 	snd_soc_free_ac97_codec(ac97);
 	return 0;
 }
@@ -324,17 +312,11 @@ static struct snd_soc_codec_driver soc_codec_dev_stac9766 = {
 		.controls		= stac9766_snd_ac97_controls,
 		.num_controls		= ARRAY_SIZE(stac9766_snd_ac97_controls),
 	},
-	.write = stac9766_ac97_write,
-	.read = stac9766_ac97_read,
 	.set_bias_level = stac9766_set_bias_level,
 	.suspend_bias_off = true,
 	.probe = stac9766_codec_probe,
 	.remove = stac9766_codec_remove,
 	.resume = stac9766_codec_resume,
-	.reg_cache_size = ARRAY_SIZE(stac9766_reg),
-	.reg_word_size = sizeof(u16),
-	.reg_cache_step = 2,
-	.reg_cache_default = stac9766_reg,
 };
 
 static int stac9766_probe(struct platform_device *pdev)
diff --git a/sound/soc/codecs/stac9766.h b/sound/soc/codecs/stac9766.h
deleted file mode 100644
index c726f907e2c0..000000000000
--- a/sound/soc/codecs/stac9766.h
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * stac9766.h  --  STAC9766 Soc Audio driver
- */
-
-#ifndef _STAC9766_H
-#define _STAC9766_H
-
-#define AC97_STAC_PAGE0 0x1000
-#define AC97_STAC_DA_CONTROL (AC97_STAC_PAGE0 | 0x6A)
-#define AC97_STAC_ANALOG_SPECIAL (AC97_STAC_PAGE0 | 0x6E)
-#define AC97_STAC_STEREO_MIC 0x78
-
-/* STAC9766 DAI ID's */
-#define STAC9766_DAI_AC97_ANALOG		0
-#define STAC9766_DAI_AC97_DIGITAL		1
-
-#endif
diff --git a/sound/soc/codecs/sti-sas.c b/sound/soc/codecs/sti-sas.c
index d6e00c77edcd..62c618765224 100644
--- a/sound/soc/codecs/sti-sas.c
+++ b/sound/soc/codecs/sti-sas.c
@@ -14,28 +14,8 @@
 #include <sound/soc.h>
 #include <sound/soc-dapm.h>
 
-/* chipID supported */
-#define CHIPID_STIH416 0
-#define CHIPID_STIH407 1
-
 /* DAC definitions */
 
-/* stih416 DAC registers */
-/* sysconf 2517: Audio-DAC-Control */
-#define STIH416_AUDIO_DAC_CTRL 0x00000814
-/* sysconf 2519: Audio-Gue-Control */
-#define STIH416_AUDIO_GLUE_CTRL 0x0000081C
-
-#define STIH416_DAC_NOT_STANDBY	0x3
-#define STIH416_DAC_SOFTMUTE	0x4
-#define STIH416_DAC_ANA_NOT_PWR	0x5
-#define STIH416_DAC_NOT_PNDBG	0x6
-
-#define STIH416_DAC_NOT_STANDBY_MASK	BIT(STIH416_DAC_NOT_STANDBY)
-#define STIH416_DAC_SOFTMUTE_MASK	BIT(STIH416_DAC_SOFTMUTE)
-#define STIH416_DAC_ANA_NOT_PWR_MASK	BIT(STIH416_DAC_ANA_NOT_PWR)
-#define STIH416_DAC_NOT_PNDBG_MASK	BIT(STIH416_DAC_NOT_PNDBG)
-
 /* stih407 DAC registers */
 /* sysconf 5041: Audio-Gue-Control */
 #define STIH407_AUDIO_GLUE_CTRL 0x000000A4
@@ -63,14 +43,9 @@ enum {
 	STI_SAS_DAI_ANALOG_OUT,
 };
 
-static const struct reg_default stih416_sas_reg_defaults[] = {
-	{ STIH407_AUDIO_GLUE_CTRL, 0x00000040 },
-	{ STIH407_AUDIO_DAC_CTRL, 0x000000000 },
-};
-
 static const struct reg_default stih407_sas_reg_defaults[] = {
-	{ STIH416_AUDIO_DAC_CTRL, 0x000000000 },
-	{ STIH416_AUDIO_GLUE_CTRL, 0x00000040 },
+	{ STIH407_AUDIO_DAC_CTRL, 0x000000000 },
+	{ STIH407_AUDIO_GLUE_CTRL, 0x00000040 },
 };
 
 struct sti_dac_audio {
@@ -89,7 +64,6 @@ struct sti_spdif_audio {
 
 /* device data structure */
 struct sti_sas_dev_data {
-	const int chipid; /* IC version */
 	const struct regmap_config *regmap;
 	const struct snd_soc_dai_ops *dac_ops;  /* DAC function callbacks */
 	const struct snd_soc_dapm_widget *dapm_widgets; /* dapms declaration */
@@ -150,51 +124,27 @@ static int  sti_sas_init_sas_registers(struct snd_soc_codec *codec,
 		ret = snd_soc_update_bits(codec, STIH407_AUDIO_GLUE_CTRL,
 					  SPDIF_BIPHASE_IDLE_MASK, 0);
 	if (ret < 0) {
-		dev_err(codec->dev, "Failed to update SPDIF registers");
+		dev_err(codec->dev, "Failed to update SPDIF registers\n");
 		return ret;
 	}
 
 	/* Init DAC configuration */
-	switch (data->dev_data->chipid) {
-	case CHIPID_STIH407:
-		/* init configuration */
-		ret =  snd_soc_update_bits(codec, STIH407_AUDIO_DAC_CTRL,
-					   STIH407_DAC_STANDBY_MASK,
-					   STIH407_DAC_STANDBY_MASK);
-
-		if (!ret)
-			ret = snd_soc_update_bits(codec, STIH407_AUDIO_DAC_CTRL,
-						  STIH407_DAC_STANDBY_ANA_MASK,
-						  STIH407_DAC_STANDBY_ANA_MASK);
-		if (!ret)
-			ret = snd_soc_update_bits(codec, STIH407_AUDIO_DAC_CTRL,
-						  STIH407_DAC_SOFTMUTE_MASK,
-						  STIH407_DAC_SOFTMUTE_MASK);
-		break;
-	case CHIPID_STIH416:
-		ret =  snd_soc_update_bits(codec, STIH416_AUDIO_DAC_CTRL,
-					   STIH416_DAC_NOT_STANDBY_MASK, 0);
-		if (!ret)
-			ret =  snd_soc_update_bits(codec,
-						   STIH416_AUDIO_DAC_CTRL,
-						   STIH416_DAC_ANA_NOT_PWR, 0);
-		if (!ret)
-			ret =  snd_soc_update_bits(codec,
-						   STIH416_AUDIO_DAC_CTRL,
-						   STIH416_DAC_NOT_PNDBG_MASK,
-						   0);
-		if (!ret)
-			ret =  snd_soc_update_bits(codec,
-						   STIH416_AUDIO_DAC_CTRL,
-						   STIH416_DAC_SOFTMUTE_MASK,
-						   STIH416_DAC_SOFTMUTE_MASK);
-		break;
-	default:
-		return -EINVAL;
-	}
+	/* init configuration */
+	ret =  snd_soc_update_bits(codec, STIH407_AUDIO_DAC_CTRL,
+				   STIH407_DAC_STANDBY_MASK,
+				   STIH407_DAC_STANDBY_MASK);
+
+	if (!ret)
+		ret = snd_soc_update_bits(codec, STIH407_AUDIO_DAC_CTRL,
+					  STIH407_DAC_STANDBY_ANA_MASK,
+					  STIH407_DAC_STANDBY_ANA_MASK);
+	if (!ret)
+		ret = snd_soc_update_bits(codec, STIH407_AUDIO_DAC_CTRL,
+					  STIH407_DAC_SOFTMUTE_MASK,
+					  STIH407_DAC_SOFTMUTE_MASK);
 
 	if (ret < 0) {
-		dev_err(codec->dev, "Failed to update DAC registers");
+		dev_err(codec->dev, "Failed to update DAC registers\n");
 		return ret;
 	}
 
@@ -217,37 +167,6 @@ static int sti_sas_dac_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
 	return 0;
 }
 
-static int stih416_dac_probe(struct snd_soc_dai *dai)
-{
-	struct snd_soc_codec *codec = dai->codec;
-	struct sti_sas_data *drvdata = dev_get_drvdata(codec->dev);
-	struct sti_dac_audio *dac = &drvdata->dac;
-
-	/* Get reset control */
-	dac->rst = devm_reset_control_get(codec->dev, "dac_rst");
-	if (IS_ERR(dac->rst)) {
-		dev_err(dai->codec->dev,
-			"%s: ERROR: DAC reset control not defined !\n",
-			__func__);
-		dac->rst = NULL;
-		return -EFAULT;
-	}
-	/* Put the DAC into reset */
-	reset_control_assert(dac->rst);
-
-	return 0;
-}
-
-static const struct snd_soc_dapm_widget stih416_sas_dapm_widgets[] = {
-	SND_SOC_DAPM_PGA("DAC bandgap", STIH416_AUDIO_DAC_CTRL,
-			 STIH416_DAC_NOT_PNDBG_MASK, 0, NULL, 0),
-	SND_SOC_DAPM_OUT_DRV("DAC standby ana", STIH416_AUDIO_DAC_CTRL,
-			     STIH416_DAC_ANA_NOT_PWR, 0, NULL, 0),
-	SND_SOC_DAPM_DAC("DAC standby",  "dac_p", STIH416_AUDIO_DAC_CTRL,
-			 STIH416_DAC_NOT_STANDBY, 0),
-	SND_SOC_DAPM_OUTPUT("DAC Output"),
-};
-
 static const struct snd_soc_dapm_widget stih407_sas_dapm_widgets[] = {
 	SND_SOC_DAPM_OUT_DRV("DAC standby ana", STIH407_AUDIO_DAC_CTRL,
 			     STIH407_DAC_STANDBY_ANA, 1, NULL, 0),
@@ -256,30 +175,11 @@ static const struct snd_soc_dapm_widget stih407_sas_dapm_widgets[] = {
 	SND_SOC_DAPM_OUTPUT("DAC Output"),
 };
 
-static const struct snd_soc_dapm_route stih416_sas_route[] = {
-	{"DAC Output", NULL, "DAC bandgap"},
-	{"DAC Output", NULL, "DAC standby ana"},
-	{"DAC standby ana", NULL, "DAC standby"},
-};
-
 static const struct snd_soc_dapm_route stih407_sas_route[] = {
 	{"DAC Output", NULL, "DAC standby ana"},
 	{"DAC standby ana", NULL, "DAC standby"},
 };
 
-static int stih416_sas_dac_mute(struct snd_soc_dai *dai, int mute, int stream)
-{
-	struct snd_soc_codec *codec = dai->codec;
-
-	if (mute) {
-		return snd_soc_update_bits(codec, STIH416_AUDIO_DAC_CTRL,
-					    STIH416_DAC_SOFTMUTE_MASK,
-					    STIH416_DAC_SOFTMUTE_MASK);
-	} else {
-		return snd_soc_update_bits(codec, STIH416_AUDIO_DAC_CTRL,
-					    STIH416_DAC_SOFTMUTE_MASK, 0);
-	}
-}
 
 static int stih407_sas_dac_mute(struct snd_soc_dai *dai, int mute, int stream)
 {
@@ -392,13 +292,13 @@ static int sti_sas_prepare(struct snd_pcm_substream *substream,
 	switch (dai->id) {
 	case STI_SAS_DAI_SPDIF_OUT:
 		if ((drvdata->spdif.mclk / runtime->rate) != 128) {
-			dev_err(codec->dev, "unexpected mclk-fs ratio");
+			dev_err(codec->dev, "unexpected mclk-fs ratio\n");
 			return -EINVAL;
 		}
 		break;
 	case STI_SAS_DAI_ANALOG_OUT:
 		if ((drvdata->dac.mclk / runtime->rate) != 256) {
-			dev_err(codec->dev, "unexpected mclk-fs ratio");
+			dev_err(codec->dev, "unexpected mclk-fs ratio\n");
 			return -EINVAL;
 		}
 		break;
@@ -407,13 +307,6 @@ static int sti_sas_prepare(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static const struct snd_soc_dai_ops stih416_dac_ops = {
-	.set_fmt = sti_sas_dac_set_fmt,
-	.mute_stream = stih416_sas_dac_mute,
-	.prepare = sti_sas_prepare,
-	.set_sysclk = sti_sas_set_sysclk,
-};
-
 static const struct snd_soc_dai_ops stih407_dac_ops = {
 	.set_fmt = sti_sas_dac_set_fmt,
 	.mute_stream = stih407_sas_dac_mute,
@@ -434,31 +327,7 @@ static const struct regmap_config stih407_sas_regmap = {
 	.reg_write = sti_sas_write_reg,
 };
 
-static const struct regmap_config stih416_sas_regmap = {
-	.reg_bits = 32,
-	.val_bits = 32,
-
-	.max_register = STIH416_AUDIO_DAC_CTRL,
-	.reg_defaults = stih416_sas_reg_defaults,
-	.num_reg_defaults = ARRAY_SIZE(stih416_sas_reg_defaults),
-	.volatile_reg = sti_sas_volatile_register,
-	.cache_type = REGCACHE_RBTREE,
-	.reg_read = sti_sas_read_reg,
-	.reg_write = sti_sas_write_reg,
-};
-
-static const struct sti_sas_dev_data stih416_data = {
-	.chipid = CHIPID_STIH416,
-	.regmap = &stih416_sas_regmap,
-	.dac_ops = &stih416_dac_ops,
-	.dapm_widgets = stih416_sas_dapm_widgets,
-	.num_dapm_widgets = ARRAY_SIZE(stih416_sas_dapm_widgets),
-	.dapm_routes =	stih416_sas_route,
-	.num_dapm_routes = ARRAY_SIZE(stih416_sas_route),
-};
-
 static const struct sti_sas_dev_data stih407_data = {
-	.chipid = CHIPID_STIH407,
 	.regmap = &stih407_sas_regmap,
 	.dac_ops = &stih407_dac_ops,
 	.dapm_widgets = stih407_sas_dapm_widgets,
@@ -533,10 +402,6 @@ static struct snd_soc_codec_driver sti_sas_driver = {
 
 static const struct of_device_id sti_sas_dev_match[] = {
 	{
-		.compatible = "st,stih416-sas-codec",
-		.data = &stih416_data,
-	},
-	{
 		.compatible = "st,stih407-sas-codec",
 		.data = &stih407_data,
 	},
@@ -558,7 +423,7 @@ static int sti_sas_driver_probe(struct platform_device *pdev)
 	/* Populate data structure depending on compatibility */
 	of_id = of_match_node(sti_sas_dev_match, pnode);
 	if (!of_id->data) {
-		dev_err(&pdev->dev, "data associated to device is missing");
+		dev_err(&pdev->dev, "data associated to device is missing\n");
 		return -EINVAL;
 	}
 
@@ -584,10 +449,6 @@ static int sti_sas_driver_probe(struct platform_device *pdev)
 	}
 	drvdata->spdif.regmap = drvdata->dac.regmap;
 
-	/* Set DAC dai probe */
-	if (drvdata->dev_data->chipid == CHIPID_STIH416)
-		sti_sas_dai[STI_SAS_DAI_ANALOG_OUT].probe = stih416_dac_probe;
-
 	sti_sas_dai[STI_SAS_DAI_ANALOG_OUT].ops = drvdata->dev_data->dac_ops;
 
 	/* Set dapms*/
diff --git a/sound/soc/codecs/tlv320aic31xx.c b/sound/soc/codecs/tlv320aic31xx.c
index be1a64bfd320..f8a90ba8cd71 100644
--- a/sound/soc/codecs/tlv320aic31xx.c
+++ b/sound/soc/codecs/tlv320aic31xx.c
@@ -1253,6 +1253,8 @@ static const struct of_device_id tlv320aic31xx_of_match[] = {
 	{ .compatible = "ti,tlv320aic3110" },
 	{ .compatible = "ti,tlv320aic3120" },
 	{ .compatible = "ti,tlv320aic3111" },
+	{ .compatible = "ti,tlv320dac3100" },
+	{ .compatible = "ti,tlv320dac3101" },
 	{},
 };
 MODULE_DEVICE_TABLE(of, tlv320aic31xx_of_match);
@@ -1379,6 +1381,7 @@ static const struct i2c_device_id aic31xx_i2c_id[] = {
 	{ "tlv320aic3120", AIC3120 },
 	{ "tlv320aic3111", AIC3111 },
 	{ "tlv320dac3100", DAC3100 },
+	{ "tlv320dac3101", DAC3101 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, aic31xx_i2c_id);
diff --git a/sound/soc/codecs/tlv320aic31xx.h b/sound/soc/codecs/tlv320aic31xx.h
index 5acd5b69fb83..730fb2058869 100644
--- a/sound/soc/codecs/tlv320aic31xx.h
+++ b/sound/soc/codecs/tlv320aic31xx.h
@@ -32,6 +32,7 @@ enum aic31xx_type {
 	AIC3120 = AIC31XX_MINIDSP_BIT,
 	AIC3111 = (AIC31XX_STEREO_CLASS_D_BIT | AIC31XX_MINIDSP_BIT),
 	DAC3100 = DAC31XX_BIT,
+	DAC3101 = DAC31XX_BIT | AIC31XX_STEREO_CLASS_D_BIT,
 };
 
 struct aic31xx_pdata {
diff --git a/sound/soc/codecs/tlv320aic3x.c b/sound/soc/codecs/tlv320aic3x.c
index 5a8d96ec058c..8877b74b0510 100644
--- a/sound/soc/codecs/tlv320aic3x.c
+++ b/sound/soc/codecs/tlv320aic3x.c
@@ -157,7 +157,7 @@ static int snd_soc_dapm_put_volsw_aic3x(struct snd_kcontrol *kcontrol,
 	unsigned int mask = (1 << fls(max)) - 1;
 	unsigned int invert = mc->invert;
 	unsigned short val;
-	struct snd_soc_dapm_update update;
+	struct snd_soc_dapm_update update = { 0 };
 	int connect, change;
 
 	val = (ucontrol->value.integer.value[0] & mask);
diff --git a/sound/soc/codecs/uda1380.c b/sound/soc/codecs/uda1380.c
index 533e3bb444e4..2918fdb95e58 100644
--- a/sound/soc/codecs/uda1380.c
+++ b/sound/soc/codecs/uda1380.c
@@ -698,25 +698,10 @@ static int uda1380_probe(struct snd_soc_codec *codec)
 	codec->hw_write = (hw_write_t)i2c_master_send;
 	codec->control_data = uda1380->control_data;
 
-	if (!pdata)
-		return -EINVAL;
-
-	if (gpio_is_valid(pdata->gpio_reset)) {
-		ret = gpio_request_one(pdata->gpio_reset, GPIOF_OUT_INIT_LOW,
-				       "uda1380 reset");
-		if (ret)
-			goto err_out;
-	}
-
-	if (gpio_is_valid(pdata->gpio_power)) {
-		ret = gpio_request_one(pdata->gpio_power, GPIOF_OUT_INIT_LOW,
-				   "uda1380 power");
-		if (ret)
-			goto err_free_gpio;
-	} else {
+	if (!gpio_is_valid(pdata->gpio_power)) {
 		ret = uda1380_reset(codec);
 		if (ret)
-			goto err_free_gpio;
+			return ret;
 	}
 
 	INIT_WORK(&uda1380->work, uda1380_flush_work);
@@ -733,28 +718,10 @@ static int uda1380_probe(struct snd_soc_codec *codec)
 	}
 
 	return 0;
-
-err_free_gpio:
-	if (gpio_is_valid(pdata->gpio_reset))
-		gpio_free(pdata->gpio_reset);
-err_out:
-	return ret;
-}
-
-/* power down chip */
-static int uda1380_remove(struct snd_soc_codec *codec)
-{
-	struct uda1380_platform_data *pdata =codec->dev->platform_data;
-
-	gpio_free(pdata->gpio_reset);
-	gpio_free(pdata->gpio_power);
-
-	return 0;
 }
 
 static struct snd_soc_codec_driver soc_codec_dev_uda1380 = {
 	.probe =	uda1380_probe,
-	.remove =	uda1380_remove,
 	.read =		uda1380_read_reg_cache,
 	.write =	uda1380_write,
 	.set_bias_level = uda1380_set_bias_level,
@@ -775,18 +742,35 @@ static struct snd_soc_codec_driver soc_codec_dev_uda1380 = {
 	},
 };
 
-#if IS_ENABLED(CONFIG_I2C)
 static int uda1380_i2c_probe(struct i2c_client *i2c,
 			     const struct i2c_device_id *id)
 {
+	struct uda1380_platform_data *pdata = i2c->dev.platform_data;
 	struct uda1380_priv *uda1380;
 	int ret;
 
+	if (!pdata)
+		return -EINVAL;
+
 	uda1380 = devm_kzalloc(&i2c->dev, sizeof(struct uda1380_priv),
 			       GFP_KERNEL);
 	if (uda1380 == NULL)
 		return -ENOMEM;
 
+	if (gpio_is_valid(pdata->gpio_reset)) {
+		ret = devm_gpio_request_one(&i2c->dev, pdata->gpio_reset,
+			GPIOF_OUT_INIT_LOW, "uda1380 reset");
+		if (ret)
+			return ret;
+	}
+
+	if (gpio_is_valid(pdata->gpio_power)) {
+		ret = devm_gpio_request_one(&i2c->dev, pdata->gpio_power,
+			GPIOF_OUT_INIT_LOW, "uda1380 power");
+		if (ret)
+			return ret;
+	}
+
 	i2c_set_clientdata(i2c, uda1380);
 	uda1380->control_data = i2c;
 
@@ -815,27 +799,8 @@ static struct i2c_driver uda1380_i2c_driver = {
 	.remove =   uda1380_i2c_remove,
 	.id_table = uda1380_i2c_id,
 };
-#endif
 
-static int __init uda1380_modinit(void)
-{
-	int ret = 0;
-#if IS_ENABLED(CONFIG_I2C)
-	ret = i2c_add_driver(&uda1380_i2c_driver);
-	if (ret != 0)
-		pr_err("Failed to register UDA1380 I2C driver: %d\n", ret);
-#endif
-	return ret;
-}
-module_init(uda1380_modinit);
-
-static void __exit uda1380_exit(void)
-{
-#if IS_ENABLED(CONFIG_I2C)
-	i2c_del_driver(&uda1380_i2c_driver);
-#endif
-}
-module_exit(uda1380_exit);
+module_i2c_driver(uda1380_i2c_driver);
 
 MODULE_AUTHOR("Giorgio Padrin");
 MODULE_DESCRIPTION("Audio support for codec Philips UDA1380");
diff --git a/sound/soc/codecs/uda1380.h b/sound/soc/codecs/uda1380.h
index 942e3927c72b..69a326ac3c1a 100644
--- a/sound/soc/codecs/uda1380.h
+++ b/sound/soc/codecs/uda1380.h
@@ -72,8 +72,4 @@
 #define R22_SKIP_DCFIL	0x0002
 #define R23_AGC_EN	0x0001
 
-#define UDA1380_DAI_DUPLEX	0 /* playback and capture on single DAI */
-#define UDA1380_DAI_PLAYBACK	1 /* playback DAI */
-#define UDA1380_DAI_CAPTURE	2 /* capture DAI */
-
 #endif /* _UDA1380_H */
diff --git a/sound/soc/codecs/wm2200.c b/sound/soc/codecs/wm2200.c
index 606bf88abfc4..d83dab57a1d1 100644
--- a/sound/soc/codecs/wm2200.c
+++ b/sound/soc/codecs/wm2200.c
@@ -999,7 +999,7 @@ static DECLARE_TLV_DB_SCALE(in_tlv, -6300, 100, 0);
 static DECLARE_TLV_DB_SCALE(digital_tlv, -6400, 50, 0);
 static DECLARE_TLV_DB_SCALE(out_tlv, -6400, 100, 0);
 
-static const char *wm2200_mixer_texts[] = {
+static const char * const wm2200_mixer_texts[] = {
 	"None",
 	"Tone Generator",
 	"AEC Loopback",
@@ -1033,7 +1033,7 @@ static const char *wm2200_mixer_texts[] = {
 	"DSP2.6",
 };
 
-static int wm2200_mixer_values[] = {
+static unsigned int wm2200_mixer_values[] = {
 	0x00,
 	0x04,   /* Tone */
 	0x08,   /* AEC */
diff --git a/sound/soc/codecs/wm5102.c b/sound/soc/codecs/wm5102.c
index 93876c6d48ee..e7ab37d0dd32 100644
--- a/sound/soc/codecs/wm5102.c
+++ b/sound/soc/codecs/wm5102.c
@@ -607,6 +607,9 @@ static int wm5102_sysclk_ev(struct snd_soc_dapm_widget *w,
 		break;
 	case SND_SOC_DAPM_PRE_PMD:
 		break;
+	case SND_SOC_DAPM_PRE_PMU:
+	case SND_SOC_DAPM_POST_PMD:
+		return arizona_clk_ev(w, kcontrol, event);
 	default:
 		return 0;
 	}
@@ -1077,9 +1080,11 @@ static const struct snd_kcontrol_new wm5102_aec_loopback_mux =
 static const struct snd_soc_dapm_widget wm5102_dapm_widgets[] = {
 SND_SOC_DAPM_SUPPLY("SYSCLK", ARIZONA_SYSTEM_CLOCK_1, ARIZONA_SYSCLK_ENA_SHIFT,
 		    0, wm5102_sysclk_ev,
-		    SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD),
+		    SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD |
+		    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 SND_SOC_DAPM_SUPPLY("ASYNCCLK", ARIZONA_ASYNC_CLOCK_1,
-		    ARIZONA_ASYNC_CLK_ENA_SHIFT, 0, NULL, 0),
+		    ARIZONA_ASYNC_CLK_ENA_SHIFT, 0, arizona_clk_ev,
+		    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 SND_SOC_DAPM_SUPPLY("OPCLK", ARIZONA_OUTPUT_SYSTEM_CLOCK,
 		    ARIZONA_OPCLK_ENA_SHIFT, 0, NULL, 0),
 SND_SOC_DAPM_SUPPLY("ASYNCOPCLK", ARIZONA_OUTPUT_ASYNC_CLOCK,
@@ -1903,7 +1908,7 @@ static struct snd_soc_dai_driver wm5102_dai[] = {
 static int wm5102_open(struct snd_compr_stream *stream)
 {
 	struct snd_soc_pcm_runtime *rtd = stream->private_data;
-	struct wm5102_priv *priv = snd_soc_codec_get_drvdata(rtd->codec);
+	struct wm5102_priv *priv = snd_soc_platform_get_drvdata(rtd->platform);
 
 	return wm_adsp_compr_open(&priv->core.adsp[0], stream);
 }
@@ -1926,18 +1931,10 @@ static irqreturn_t wm5102_adsp2_irq(int irq, void *data)
 static int wm5102_codec_probe(struct snd_soc_codec *codec)
 {
 	struct snd_soc_dapm_context *dapm = snd_soc_codec_get_dapm(codec);
+	struct snd_soc_component *component = snd_soc_dapm_to_component(dapm);
 	struct wm5102_priv *priv = snd_soc_codec_get_drvdata(codec);
-	struct arizona *arizona = priv->core.arizona;
 	int ret;
 
-	ret = arizona_request_irq(arizona, ARIZONA_IRQ_DSP_IRQ1,
-				  "ADSP2 Compressed IRQ", wm5102_adsp2_irq,
-				  priv);
-	if (ret != 0) {
-		dev_err(codec->dev, "Failed to request DSP IRQ: %d\n", ret);
-		return ret;
-	}
-
 	ret = wm_adsp2_codec_probe(&priv->core.adsp[0], codec);
 	if (ret)
 		return ret;
@@ -1949,8 +1946,9 @@ static int wm5102_codec_probe(struct snd_soc_codec *codec)
 
 	arizona_init_spk(codec);
 	arizona_init_gpio(codec);
+	arizona_init_notifiers(codec);
 
-	snd_soc_dapm_disable_pin(dapm, "HAPTICS");
+	snd_soc_component_disable_pin(component, "HAPTICS");
 
 	priv->core.arizona->dapm = dapm;
 
@@ -1965,16 +1963,11 @@ err_adsp2_codec_probe:
 static int wm5102_codec_remove(struct snd_soc_codec *codec)
 {
 	struct wm5102_priv *priv = snd_soc_codec_get_drvdata(codec);
-	struct arizona *arizona = priv->core.arizona;
 
 	wm_adsp2_codec_remove(&priv->core.adsp[0], codec);
 
 	priv->core.arizona->dapm = NULL;
 
-	arizona_free_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, priv);
-
-	arizona_free_spk(codec);
-
 	return 0;
 }
 
@@ -2092,25 +2085,47 @@ static int wm5102_probe(struct platform_device *pdev)
 	pm_runtime_enable(&pdev->dev);
 	pm_runtime_idle(&pdev->dev);
 
+	ret = arizona_request_irq(arizona, ARIZONA_IRQ_DSP_IRQ1,
+				  "ADSP2 Compressed IRQ", wm5102_adsp2_irq,
+				  wm5102);
+	if (ret != 0) {
+		dev_err(&pdev->dev, "Failed to request DSP IRQ: %d\n", ret);
+		return ret;
+	}
+
+	ret = arizona_init_spk_irqs(arizona);
+	if (ret < 0)
+		goto err_dsp_irq;
+
 	ret = snd_soc_register_platform(&pdev->dev, &wm5102_compr_platform);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to register platform: %d\n", ret);
-		return ret;
+		goto err_spk_irqs;
 	}
 
 	ret = snd_soc_register_codec(&pdev->dev, &soc_codec_dev_wm5102,
 				      wm5102_dai, ARRAY_SIZE(wm5102_dai));
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to register codec: %d\n", ret);
-		snd_soc_unregister_platform(&pdev->dev);
+		goto err_platform;
 	}
 
 	return ret;
+
+err_platform:
+	snd_soc_unregister_platform(&pdev->dev);
+err_spk_irqs:
+	arizona_free_spk_irqs(arizona);
+err_dsp_irq:
+	arizona_free_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, wm5102);
+
+	return ret;
 }
 
 static int wm5102_remove(struct platform_device *pdev)
 {
 	struct wm5102_priv *wm5102 = platform_get_drvdata(pdev);
+	struct arizona *arizona = wm5102->core.arizona;
 
 	snd_soc_unregister_platform(&pdev->dev);
 	snd_soc_unregister_codec(&pdev->dev);
@@ -2118,6 +2133,10 @@ static int wm5102_remove(struct platform_device *pdev)
 
 	wm_adsp2_remove(&wm5102->core.adsp[0]);
 
+	arizona_free_spk_irqs(arizona);
+
+	arizona_free_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, wm5102);
+
 	return 0;
 }
 
diff --git a/sound/soc/codecs/wm5110.c b/sound/soc/codecs/wm5110.c
index 06bae3b23fce..585fc706c1b0 100644
--- a/sound/soc/codecs/wm5110.c
+++ b/sound/soc/codecs/wm5110.c
@@ -183,7 +183,9 @@ static int wm5110_sysclk_ev(struct snd_soc_dapm_widget *w,
 				regmap_write_async(regmap, patch[i].reg,
 						   patch[i].def);
 		break;
-
+	case SND_SOC_DAPM_PRE_PMU:
+	case SND_SOC_DAPM_POST_PMD:
+		return arizona_clk_ev(w, kcontrol, event);
 	default:
 		break;
 	}
@@ -1073,9 +1075,11 @@ static const struct snd_kcontrol_new wm5110_output_anc_src[] = {
 
 static const struct snd_soc_dapm_widget wm5110_dapm_widgets[] = {
 SND_SOC_DAPM_SUPPLY("SYSCLK", ARIZONA_SYSTEM_CLOCK_1, ARIZONA_SYSCLK_ENA_SHIFT,
-		    0, wm5110_sysclk_ev, SND_SOC_DAPM_POST_PMU),
+		    0, wm5110_sysclk_ev, SND_SOC_DAPM_POST_PMU |
+		    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 SND_SOC_DAPM_SUPPLY("ASYNCCLK", ARIZONA_ASYNC_CLOCK_1,
-		    ARIZONA_ASYNC_CLK_ENA_SHIFT, 0, NULL, 0),
+		    ARIZONA_ASYNC_CLK_ENA_SHIFT, 0, arizona_clk_ev,
+		    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 SND_SOC_DAPM_SUPPLY("OPCLK", ARIZONA_OUTPUT_SYSTEM_CLOCK,
 		    ARIZONA_OPCLK_ENA_SHIFT, 0, NULL, 0),
 SND_SOC_DAPM_SUPPLY("ASYNCOPCLK", ARIZONA_OUTPUT_ASYNC_CLOCK,
@@ -2220,7 +2224,7 @@ static struct snd_soc_dai_driver wm5110_dai[] = {
 static int wm5110_open(struct snd_compr_stream *stream)
 {
 	struct snd_soc_pcm_runtime *rtd = stream->private_data;
-	struct wm5110_priv *priv = snd_soc_codec_get_drvdata(rtd->codec);
+	struct wm5110_priv *priv = snd_soc_platform_get_drvdata(rtd->platform);
 	struct arizona *arizona = priv->core.arizona;
 	int n_adsp;
 
@@ -2269,8 +2273,8 @@ static irqreturn_t wm5110_adsp2_irq(int irq, void *data)
 static int wm5110_codec_probe(struct snd_soc_codec *codec)
 {
 	struct snd_soc_dapm_context *dapm = snd_soc_codec_get_dapm(codec);
+	struct snd_soc_component *component = snd_soc_dapm_to_component(dapm);
 	struct wm5110_priv *priv = snd_soc_codec_get_drvdata(codec);
-	struct arizona *arizona = priv->core.arizona;
 	int i, ret;
 
 	priv->core.arizona->dapm = dapm;
@@ -2280,14 +2284,6 @@ static int wm5110_codec_probe(struct snd_soc_codec *codec)
 	arizona_init_mono(codec);
 	arizona_init_notifiers(codec);
 
-	ret = arizona_request_irq(arizona, ARIZONA_IRQ_DSP_IRQ1,
-				  "ADSP2 Compressed IRQ", wm5110_adsp2_irq,
-				  priv);
-	if (ret != 0) {
-		dev_err(codec->dev, "Failed to request DSP IRQ: %d\n", ret);
-		return ret;
-	}
-
 	for (i = 0; i < WM5110_NUM_ADSP; ++i) {
 		ret = wm_adsp2_codec_probe(&priv->core.adsp[i], codec);
 		if (ret)
@@ -2300,7 +2296,7 @@ static int wm5110_codec_probe(struct snd_soc_codec *codec)
 	if (ret)
 		goto err_adsp2_codec_probe;
 
-	snd_soc_dapm_disable_pin(dapm, "HAPTICS");
+	snd_soc_component_disable_pin(component, "HAPTICS");
 
 	return 0;
 
@@ -2308,15 +2304,12 @@ err_adsp2_codec_probe:
 	for (--i; i >= 0; --i)
 		wm_adsp2_codec_remove(&priv->core.adsp[i], codec);
 
-	arizona_free_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, priv);
-
 	return ret;
 }
 
 static int wm5110_codec_remove(struct snd_soc_codec *codec)
 {
 	struct wm5110_priv *priv = snd_soc_codec_get_drvdata(codec);
-	struct arizona *arizona = priv->core.arizona;
 	int i;
 
 	for (i = 0; i < WM5110_NUM_ADSP; ++i)
@@ -2324,10 +2317,6 @@ static int wm5110_codec_remove(struct snd_soc_codec *codec)
 
 	priv->core.arizona->dapm = NULL;
 
-	arizona_free_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, priv);
-
-	arizona_free_spk(codec);
-
 	return 0;
 }
 
@@ -2449,25 +2438,47 @@ static int wm5110_probe(struct platform_device *pdev)
 	pm_runtime_enable(&pdev->dev);
 	pm_runtime_idle(&pdev->dev);
 
+	ret = arizona_request_irq(arizona, ARIZONA_IRQ_DSP_IRQ1,
+				  "ADSP2 Compressed IRQ", wm5110_adsp2_irq,
+				  wm5110);
+	if (ret != 0) {
+		dev_err(&pdev->dev, "Failed to request DSP IRQ: %d\n", ret);
+		return ret;
+	}
+
+	ret = arizona_init_spk_irqs(arizona);
+	if (ret < 0)
+		goto err_dsp_irq;
+
 	ret = snd_soc_register_platform(&pdev->dev, &wm5110_compr_platform);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to register platform: %d\n", ret);
-		return ret;
+		goto err_spk_irqs;
 	}
 
 	ret = snd_soc_register_codec(&pdev->dev, &soc_codec_dev_wm5110,
 				      wm5110_dai, ARRAY_SIZE(wm5110_dai));
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to register codec: %d\n", ret);
-		snd_soc_unregister_platform(&pdev->dev);
+		goto err_platform;
 	}
 
 	return ret;
+
+err_platform:
+	snd_soc_unregister_platform(&pdev->dev);
+err_spk_irqs:
+	arizona_free_spk_irqs(arizona);
+err_dsp_irq:
+	arizona_free_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, wm5110);
+
+	return ret;
 }
 
 static int wm5110_remove(struct platform_device *pdev)
 {
 	struct wm5110_priv *wm5110 = platform_get_drvdata(pdev);
+	struct arizona *arizona = wm5110->core.arizona;
 	int i;
 
 	snd_soc_unregister_platform(&pdev->dev);
@@ -2477,6 +2488,10 @@ static int wm5110_remove(struct platform_device *pdev)
 	for (i = 0; i < WM5110_NUM_ADSP; i++)
 		wm_adsp2_remove(&wm5110->core.adsp[i]);
 
+	arizona_free_spk_irqs(arizona);
+
+	arizona_free_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, wm5110);
+
 	return 0;
 }
 
diff --git a/sound/soc/codecs/wm8523.c b/sound/soc/codecs/wm8523.c
index deb2e075428e..6d0a2723bfde 100644
--- a/sound/soc/codecs/wm8523.c
+++ b/sound/soc/codecs/wm8523.c
@@ -446,7 +446,6 @@ static const struct regmap_config wm8523_regmap = {
 	.volatile_reg = wm8523_volatile_register,
 };
 
-#if IS_ENABLED(CONFIG_I2C)
 static int wm8523_i2c_probe(struct i2c_client *i2c,
 			    const struct i2c_device_id *id)
 {
@@ -543,29 +542,8 @@ static struct i2c_driver wm8523_i2c_driver = {
 	.remove =   wm8523_i2c_remove,
 	.id_table = wm8523_i2c_id,
 };
-#endif
 
-static int __init wm8523_modinit(void)
-{
-	int ret;
-#if IS_ENABLED(CONFIG_I2C)
-	ret = i2c_add_driver(&wm8523_i2c_driver);
-	if (ret != 0) {
-		printk(KERN_ERR "Failed to register WM8523 I2C driver: %d\n",
-		       ret);
-	}
-#endif
-	return 0;
-}
-module_init(wm8523_modinit);
-
-static void __exit wm8523_exit(void)
-{
-#if IS_ENABLED(CONFIG_I2C)
-	i2c_del_driver(&wm8523_i2c_driver);
-#endif
-}
-module_exit(wm8523_exit);
+module_i2c_driver(wm8523_i2c_driver);
 
 MODULE_DESCRIPTION("ASoC WM8523 driver");
 MODULE_AUTHOR("Mark Brown <broonie@opensource.wolfsonmicro.com>");
diff --git a/sound/soc/codecs/wm8580.c b/sound/soc/codecs/wm8580.c
index faa7287a5253..910801dddd64 100644
--- a/sound/soc/codecs/wm8580.c
+++ b/sound/soc/codecs/wm8580.c
@@ -1,5 +1,5 @@
 /*
- * wm8580.c  --  WM8580 ALSA Soc Audio driver
+ * wm8580.c  --  WM8580 and WM8581 ALSA Soc Audio driver
  *
  * Copyright 2008-12 Wolfson Microelectronics PLC.
  *
@@ -12,6 +12,9 @@
  *  The WM8580 is a multichannel codec with S/PDIF support, featuring six
  *  DAC channels and two ADC channels.
  *
+ *  The WM8581 is a multichannel codec with S/PDIF support, featuring eight
+ *  DAC channels and two ADC channels.
+ *
  *  Currently only the primary audio interface is supported - S/PDIF and
  *  the secondary audio interfaces are not.
  */
@@ -65,6 +68,8 @@
 #define WM8580_DIGITAL_ATTENUATION_DACR2     0x17
 #define WM8580_DIGITAL_ATTENUATION_DACL3     0x18
 #define WM8580_DIGITAL_ATTENUATION_DACR3     0x19
+#define WM8581_DIGITAL_ATTENUATION_DACL4     0x1A
+#define WM8581_DIGITAL_ATTENUATION_DACR4     0x1B
 #define WM8580_MASTER_DIGITAL_ATTENUATION    0x1C
 #define WM8580_ADC_CONTROL1                  0x1D
 #define WM8580_SPDTXCHAN0                    0x1E
@@ -236,12 +241,17 @@ static const char *wm8580_supply_names[WM8580_NUM_SUPPLIES] = {
 	"PVDD",
 };
 
+struct wm8580_driver_data {
+	int num_dacs;
+};
+
 /* codec private data */
 struct wm8580_priv {
 	struct regmap *regmap;
 	struct regulator_bulk_data supplies[WM8580_NUM_SUPPLIES];
 	struct pll_state a;
 	struct pll_state b;
+	const struct wm8580_driver_data *drvdata;
 	int sysclk[2];
 };
 
@@ -306,6 +316,19 @@ SOC_DOUBLE("Capture Switch", WM8580_ADC_CONTROL1, 0, 1, 1, 1),
 SOC_SINGLE("Capture High-Pass Filter Switch", WM8580_ADC_CONTROL1, 4, 1, 0),
 };
 
+static const struct snd_kcontrol_new wm8581_snd_controls[] = {
+SOC_DOUBLE_R_EXT_TLV("DAC4 Playback Volume",
+		     WM8581_DIGITAL_ATTENUATION_DACL4,
+		     WM8581_DIGITAL_ATTENUATION_DACR4,
+		     0, 0xff, 0, snd_soc_get_volsw, wm8580_out_vu, dac_tlv),
+
+SOC_SINGLE("DAC4 Deemphasis Switch", WM8580_DAC_CONTROL3, 3, 1, 0),
+
+SOC_DOUBLE("DAC4 Invert Switch", WM8580_DAC_CONTROL4,  8, 7, 1, 0),
+
+SOC_SINGLE("DAC4 Switch", WM8580_DAC_CONTROL5, 3, 1, 1),
+};
+
 static const struct snd_soc_dapm_widget wm8580_dapm_widgets[] = {
 SND_SOC_DAPM_DAC("DAC1", "Playback", WM8580_PWRDN1, 2, 1),
 SND_SOC_DAPM_DAC("DAC2", "Playback", WM8580_PWRDN1, 3, 1),
@@ -324,6 +347,13 @@ SND_SOC_DAPM_INPUT("AINL"),
 SND_SOC_DAPM_INPUT("AINR"),
 };
 
+static const struct snd_soc_dapm_widget wm8581_dapm_widgets[] = {
+SND_SOC_DAPM_DAC("DAC4", "Playback", WM8580_PWRDN1, 5, 1),
+
+SND_SOC_DAPM_OUTPUT("VOUT4L"),
+SND_SOC_DAPM_OUTPUT("VOUT4R"),
+};
+
 static const struct snd_soc_dapm_route wm8580_dapm_routes[] = {
 	{ "VOUT1L", NULL, "DAC1" },
 	{ "VOUT1R", NULL, "DAC1" },
@@ -338,6 +368,11 @@ static const struct snd_soc_dapm_route wm8580_dapm_routes[] = {
 	{ "ADC", NULL, "AINR" },
 };
 
+static const struct snd_soc_dapm_route wm8581_dapm_routes[] = {
+	{ "VOUT4L", NULL, "DAC4" },
+	{ "VOUT4R", NULL, "DAC4" },
+};
+
 /* PLL divisors */
 struct _pll_div {
 	u32 prescale:1;
@@ -815,10 +850,21 @@ static int wm8580_set_bias_level(struct snd_soc_codec *codec,
 	return 0;
 }
 
+static int wm8580_playback_startup(struct snd_pcm_substream *substream,
+			   struct snd_soc_dai *dai)
+{
+	struct snd_soc_codec *codec = dai->codec;
+	struct wm8580_priv *wm8580 = snd_soc_codec_get_drvdata(codec);
+
+	return snd_pcm_hw_constraint_minmax(substream->runtime,
+		SNDRV_PCM_HW_PARAM_CHANNELS, 1, wm8580->drvdata->num_dacs * 2);
+}
+
 #define WM8580_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE |\
 			SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S32_LE)
 
 static const struct snd_soc_dai_ops wm8580_dai_ops_playback = {
+	.startup	= wm8580_playback_startup,
 	.set_sysclk	= wm8580_set_sysclk,
 	.hw_params	= wm8580_paif_hw_params,
 	.set_fmt	= wm8580_set_paif_dai_fmt,
@@ -842,7 +888,6 @@ static struct snd_soc_dai_driver wm8580_dai[] = {
 		.playback = {
 			.stream_name = "Playback",
 			.channels_min = 1,
-			.channels_max = 6,
 			.rates = SNDRV_PCM_RATE_8000_192000,
 			.formats = WM8580_FORMATS,
 		},
@@ -865,8 +910,22 @@ static struct snd_soc_dai_driver wm8580_dai[] = {
 static int wm8580_probe(struct snd_soc_codec *codec)
 {
 	struct wm8580_priv *wm8580 = snd_soc_codec_get_drvdata(codec);
+	struct snd_soc_dapm_context *dapm = snd_soc_codec_get_dapm(codec);
 	int ret = 0;
 
+	switch (wm8580->drvdata->num_dacs) {
+	case 4:
+		snd_soc_add_codec_controls(codec, wm8581_snd_controls,
+					ARRAY_SIZE(wm8581_snd_controls));
+		snd_soc_dapm_new_controls(dapm, wm8581_dapm_widgets,
+					ARRAY_SIZE(wm8581_dapm_widgets));
+		snd_soc_dapm_add_routes(dapm, wm8581_dapm_routes,
+					ARRAY_SIZE(wm8581_dapm_routes));
+		break;
+	default:
+		break;
+	}
+
 	ret = regulator_bulk_enable(ARRAY_SIZE(wm8580->supplies),
 				    wm8580->supplies);
 	if (ret != 0) {
@@ -914,12 +973,6 @@ static const struct snd_soc_codec_driver soc_codec_dev_wm8580 = {
 	},
 };
 
-static const struct of_device_id wm8580_of_match[] = {
-	{ .compatible = "wlf,wm8580" },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, wm8580_of_match);
-
 static const struct regmap_config wm8580_regmap = {
 	.reg_bits = 7,
 	.val_bits = 9,
@@ -932,10 +985,25 @@ static const struct regmap_config wm8580_regmap = {
 	.volatile_reg = wm8580_volatile,
 };
 
-#if IS_ENABLED(CONFIG_I2C)
+static const struct wm8580_driver_data wm8580_data = {
+	.num_dacs = 3,
+};
+
+static const struct wm8580_driver_data wm8581_data = {
+	.num_dacs = 4,
+};
+
+static const struct of_device_id wm8580_of_match[] = {
+	{ .compatible = "wlf,wm8580", .data = &wm8580_data },
+	{ .compatible = "wlf,wm8581", .data = &wm8581_data },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, wm8580_of_match);
+
 static int wm8580_i2c_probe(struct i2c_client *i2c,
 			    const struct i2c_device_id *id)
 {
+	const struct of_device_id *of_id;
 	struct wm8580_priv *wm8580;
 	int ret, i;
 
@@ -960,6 +1028,15 @@ static int wm8580_i2c_probe(struct i2c_client *i2c,
 
 	i2c_set_clientdata(i2c, wm8580);
 
+	of_id = of_match_device(wm8580_of_match, &i2c->dev);
+	if (of_id)
+		wm8580->drvdata = of_id->data;
+
+	if (!wm8580->drvdata) {
+		dev_err(&i2c->dev, "failed to find driver data\n");
+		return -EINVAL;
+	}
+
 	ret =  snd_soc_register_codec(&i2c->dev,
 			&soc_codec_dev_wm8580, wm8580_dai, ARRAY_SIZE(wm8580_dai));
 
@@ -973,7 +1050,8 @@ static int wm8580_i2c_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id wm8580_i2c_id[] = {
-	{ "wm8580", 0 },
+	{ "wm8580", (kernel_ulong_t)&wm8580_data },
+	{ "wm8581", (kernel_ulong_t)&wm8581_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, wm8580_i2c_id);
@@ -987,31 +1065,10 @@ static struct i2c_driver wm8580_i2c_driver = {
 	.remove =   wm8580_i2c_remove,
 	.id_table = wm8580_i2c_id,
 };
-#endif
 
-static int __init wm8580_modinit(void)
-{
-	int ret = 0;
-
-#if IS_ENABLED(CONFIG_I2C)
-	ret = i2c_add_driver(&wm8580_i2c_driver);
-	if (ret != 0) {
-		pr_err("Failed to register WM8580 I2C driver: %d\n", ret);
-	}
-#endif
-
-	return ret;
-}
-module_init(wm8580_modinit);
-
-static void __exit wm8580_exit(void)
-{
-#if IS_ENABLED(CONFIG_I2C)
-	i2c_del_driver(&wm8580_i2c_driver);
-#endif
-}
-module_exit(wm8580_exit);
+module_i2c_driver(wm8580_i2c_driver);
 
 MODULE_DESCRIPTION("ASoC WM8580 driver");
 MODULE_AUTHOR("Mark Brown <broonie@opensource.wolfsonmicro.com>");
+MODULE_AUTHOR("Matt Flax <flatmax@flatmax.org>");
 MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/wm8753.h b/sound/soc/codecs/wm8753.h
index 94edac144bcb..8b39e3677ac8 100644
--- a/sound/soc/codecs/wm8753.h
+++ b/sound/soc/codecs/wm8753.h
@@ -112,7 +112,4 @@
 #define WM8753_VXCLK_DIV_8	(3 << 6)
 #define WM8753_VXCLK_DIV_16	(4 << 6)
 
-#define WM8753_DAI_HIFI		0
-#define WM8753_DAI_VOICE		1
-
 #endif
diff --git a/sound/soc/codecs/wm8978.h b/sound/soc/codecs/wm8978.h
index 6ae43495b7cf..0dcf6868dff6 100644
--- a/sound/soc/codecs/wm8978.h
+++ b/sound/soc/codecs/wm8978.h
@@ -78,8 +78,8 @@ enum wm8978_clk_id {
 };
 
 enum wm8978_sysclk_src {
+	WM8978_MCLK = 0,
 	WM8978_PLL,
-	WM8978_MCLK
 };
 
 #endif	/* __WM8978_H__ */
diff --git a/sound/soc/codecs/wm8997.c b/sound/soc/codecs/wm8997.c
index 2f2821b3382f..ee0c8639c743 100644
--- a/sound/soc/codecs/wm8997.c
+++ b/sound/soc/codecs/wm8997.c
@@ -108,6 +108,9 @@ static int wm8997_sysclk_ev(struct snd_soc_dapm_widget *w,
 		break;
 	case SND_SOC_DAPM_PRE_PMD:
 		break;
+	case SND_SOC_DAPM_PRE_PMU:
+	case SND_SOC_DAPM_POST_PMD:
+		return arizona_clk_ev(w, kcontrol, event);
 	default:
 		return 0;
 	}
@@ -408,9 +411,11 @@ static const struct snd_kcontrol_new wm8997_aec_loopback_mux =
 static const struct snd_soc_dapm_widget wm8997_dapm_widgets[] = {
 SND_SOC_DAPM_SUPPLY("SYSCLK", ARIZONA_SYSTEM_CLOCK_1, ARIZONA_SYSCLK_ENA_SHIFT,
 		    0, wm8997_sysclk_ev,
-		    SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD),
+		    SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD |
+		    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 SND_SOC_DAPM_SUPPLY("ASYNCCLK", ARIZONA_ASYNC_CLOCK_1,
-		    ARIZONA_ASYNC_CLK_ENA_SHIFT, 0, NULL, 0),
+		    ARIZONA_ASYNC_CLK_ENA_SHIFT, 0, arizona_clk_ev,
+		    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 SND_SOC_DAPM_SUPPLY("OPCLK", ARIZONA_OUTPUT_SYSTEM_CLOCK,
 		    ARIZONA_OPCLK_ENA_SHIFT, 0, NULL, 0),
 SND_SOC_DAPM_SUPPLY("ASYNCOPCLK", ARIZONA_OUTPUT_ASYNC_CLOCK,
@@ -1055,11 +1060,13 @@ static struct snd_soc_dai_driver wm8997_dai[] = {
 static int wm8997_codec_probe(struct snd_soc_codec *codec)
 {
 	struct snd_soc_dapm_context *dapm = snd_soc_codec_get_dapm(codec);
+	struct snd_soc_component *component = snd_soc_dapm_to_component(dapm);
 	struct wm8997_priv *priv = snd_soc_codec_get_drvdata(codec);
 
 	arizona_init_spk(codec);
+	arizona_init_notifiers(codec);
 
-	snd_soc_dapm_disable_pin(dapm, "HAPTICS");
+	snd_soc_component_disable_pin(component, "HAPTICS");
 
 	priv->core.arizona->dapm = dapm;
 
@@ -1072,8 +1079,6 @@ static int wm8997_codec_remove(struct snd_soc_codec *codec)
 
 	priv->core.arizona->dapm = NULL;
 
-	arizona_free_spk(codec);
-
 	return 0;
 }
 
@@ -1119,7 +1124,7 @@ static int wm8997_probe(struct platform_device *pdev)
 {
 	struct arizona *arizona = dev_get_drvdata(pdev->dev.parent);
 	struct wm8997_priv *wm8997;
-	int i;
+	int i, ret;
 
 	wm8997 = devm_kzalloc(&pdev->dev, sizeof(struct wm8997_priv),
 			      GFP_KERNEL);
@@ -1159,15 +1164,33 @@ static int wm8997_probe(struct platform_device *pdev)
 	pm_runtime_enable(&pdev->dev);
 	pm_runtime_idle(&pdev->dev);
 
-	return snd_soc_register_codec(&pdev->dev, &soc_codec_dev_wm8997,
-				      wm8997_dai, ARRAY_SIZE(wm8997_dai));
+	ret = arizona_init_spk_irqs(arizona);
+	if (ret < 0)
+		return ret;
+
+	ret = snd_soc_register_codec(&pdev->dev, &soc_codec_dev_wm8997,
+				     wm8997_dai, ARRAY_SIZE(wm8997_dai));
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Failed to register codec: %d\n", ret);
+		goto err_spk_irqs;
+	}
+
+err_spk_irqs:
+	arizona_free_spk_irqs(arizona);
+
+	return ret;
 }
 
 static int wm8997_remove(struct platform_device *pdev)
 {
+	struct wm8997_priv *wm8997 = platform_get_drvdata(pdev);
+	struct arizona *arizona = wm8997->core.arizona;
+
 	snd_soc_unregister_codec(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
+	arizona_free_spk_irqs(arizona);
+
 	return 0;
 }
 
diff --git a/sound/soc/codecs/wm8998.c b/sound/soc/codecs/wm8998.c
index bcc2e1060a6c..3694f5958d86 100644
--- a/sound/soc/codecs/wm8998.c
+++ b/sound/soc/codecs/wm8998.c
@@ -541,9 +541,11 @@ static const struct snd_kcontrol_new wm8998_aec_loopback_mux[] = {
 
 static const struct snd_soc_dapm_widget wm8998_dapm_widgets[] = {
 SND_SOC_DAPM_SUPPLY("SYSCLK", ARIZONA_SYSTEM_CLOCK_1,
-		    ARIZONA_SYSCLK_ENA_SHIFT, 0, NULL, 0),
+		    ARIZONA_SYSCLK_ENA_SHIFT, 0, arizona_clk_ev,
+		    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 SND_SOC_DAPM_SUPPLY("ASYNCCLK", ARIZONA_ASYNC_CLOCK_1,
-		    ARIZONA_ASYNC_CLK_ENA_SHIFT, 0, NULL, 0),
+		    ARIZONA_ASYNC_CLK_ENA_SHIFT, 0, arizona_clk_ev,
+		    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 SND_SOC_DAPM_SUPPLY("OPCLK", ARIZONA_OUTPUT_SYSTEM_CLOCK,
 		    ARIZONA_OPCLK_ENA_SHIFT, 0, NULL, 0),
 SND_SOC_DAPM_SUPPLY("ASYNCOPCLK", ARIZONA_OUTPUT_ASYNC_CLOCK,
@@ -1318,13 +1320,15 @@ static int wm8998_codec_probe(struct snd_soc_codec *codec)
 {
 	struct wm8998_priv *priv = snd_soc_codec_get_drvdata(codec);
 	struct snd_soc_dapm_context *dapm = snd_soc_codec_get_dapm(codec);
+	struct snd_soc_component *component = snd_soc_dapm_to_component(dapm);
 
 	priv->core.arizona->dapm = dapm;
 
 	arizona_init_spk(codec);
 	arizona_init_gpio(codec);
+	arizona_init_notifiers(codec);
 
-	snd_soc_dapm_disable_pin(dapm, "HAPTICS");
+	snd_soc_component_disable_pin(component, "HAPTICS");
 
 	return 0;
 }
@@ -1335,8 +1339,6 @@ static int wm8998_codec_remove(struct snd_soc_codec *codec)
 
 	priv->core.arizona->dapm = NULL;
 
-	arizona_free_spk(codec);
-
 	return 0;
 }
 
@@ -1385,7 +1387,7 @@ static int wm8998_probe(struct platform_device *pdev)
 {
 	struct arizona *arizona = dev_get_drvdata(pdev->dev.parent);
 	struct wm8998_priv *wm8998;
-	int i;
+	int i, ret;
 
 	wm8998 = devm_kzalloc(&pdev->dev, sizeof(struct wm8998_priv),
 			      GFP_KERNEL);
@@ -1417,15 +1419,35 @@ static int wm8998_probe(struct platform_device *pdev)
 	pm_runtime_enable(&pdev->dev);
 	pm_runtime_idle(&pdev->dev);
 
-	return snd_soc_register_codec(&pdev->dev, &soc_codec_dev_wm8998,
-				      wm8998_dai, ARRAY_SIZE(wm8998_dai));
+	ret = arizona_init_spk_irqs(arizona);
+	if (ret < 0)
+		return ret;
+
+	ret = snd_soc_register_codec(&pdev->dev, &soc_codec_dev_wm8998,
+				     wm8998_dai, ARRAY_SIZE(wm8998_dai));
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Failed to register codec: %d\n", ret);
+		goto err_spk_irqs;
+	}
+
+	return ret;
+
+err_spk_irqs:
+	arizona_free_spk_irqs(arizona);
+
+	return ret;
 }
 
 static int wm8998_remove(struct platform_device *pdev)
 {
+	struct wm8998_priv *wm8998 = platform_get_drvdata(pdev);
+	struct arizona *arizona = wm8998->core.arizona;
+
 	snd_soc_unregister_codec(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
+	arizona_free_spk_irqs(arizona);
+
 	return 0;
 }
 
diff --git a/sound/soc/codecs/wm9081.c b/sound/soc/codecs/wm9081.c
index 856867ec2813..6febef337dd2 100644
--- a/sound/soc/codecs/wm9081.c
+++ b/sound/soc/codecs/wm9081.c
@@ -1304,7 +1304,6 @@ static const struct regmap_config wm9081_regmap = {
 	.cache_type = REGCACHE_RBTREE,
 };
 
-#if IS_ENABLED(CONFIG_I2C)
 static int wm9081_i2c_probe(struct i2c_client *i2c,
 			    const struct i2c_device_id *id)
 {
@@ -1384,7 +1383,6 @@ static struct i2c_driver wm9081_i2c_driver = {
 	.remove =   wm9081_i2c_remove,
 	.id_table = wm9081_i2c_id,
 };
-#endif
 
 module_i2c_driver(wm9081_i2c_driver);
 
diff --git a/sound/soc/codecs/wm9705.c b/sound/soc/codecs/wm9705.c
index dcdd055db57b..f6d5c0f2aea5 100644
--- a/sound/soc/codecs/wm9705.c
+++ b/sound/soc/codecs/wm9705.c
@@ -14,37 +14,58 @@
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/device.h>
+#include <linux/regmap.h>
 #include <sound/core.h>
 #include <sound/pcm.h>
 #include <sound/ac97_codec.h>
 #include <sound/initval.h>
 #include <sound/soc.h>
 
-#include "wm9705.h"
-
 #define WM9705_VENDOR_ID 0x574d4c05
 #define WM9705_VENDOR_ID_MASK 0xffffffff
 
-/*
- * WM9705 register cache
- */
-static const u16 wm9705_reg[] = {
-	0x6150, 0x8000, 0x8000, 0x8000, /* 0x0  */
-	0x0000, 0x8000, 0x8008, 0x8008, /* 0x8  */
-	0x8808, 0x8808, 0x8808, 0x8808, /* 0x10 */
-	0x8808, 0x0000, 0x8000, 0x0000, /* 0x18 */
-	0x0000, 0x0000, 0x0000, 0x000f, /* 0x20 */
-	0x0605, 0x0000, 0xbb80, 0x0000, /* 0x28 */
-	0x0000, 0xbb80, 0x0000, 0x0000, /* 0x30 */
-	0x0000, 0x2000, 0x0000, 0x0000, /* 0x38 */
-	0x0000, 0x0000, 0x0000, 0x0000, /* 0x40 */
-	0x0000, 0x0000, 0x0000, 0x0000, /* 0x48 */
-	0x0000, 0x0000, 0x0000, 0x0000, /* 0x50 */
-	0x0000, 0x0000, 0x0000, 0x0000, /* 0x58 */
-	0x0000, 0x0000, 0x0000, 0x0000, /* 0x60 */
-	0x0000, 0x0000, 0x0000, 0x0000, /* 0x68 */
-	0x0000, 0x0808, 0x0000, 0x0006, /* 0x70 */
-	0x0000, 0x0000, 0x574d, 0x4c05, /* 0x78 */
+static const struct reg_default wm9705_reg_defaults[] = {
+	{ 0x02, 0x8000 },
+	{ 0x04, 0x8000 },
+	{ 0x06, 0x8000 },
+	{ 0x0a, 0x8000 },
+	{ 0x0c, 0x8008 },
+	{ 0x0e, 0x8008 },
+	{ 0x10, 0x8808 },
+	{ 0x12, 0x8808 },
+	{ 0x14, 0x8808 },
+	{ 0x16, 0x8808 },
+	{ 0x18, 0x8808 },
+	{ 0x1a, 0x0000 },
+	{ 0x1c, 0x8000 },
+	{ 0x20, 0x0000 },
+	{ 0x22, 0x0000 },
+	{ 0x26, 0x000f },
+	{ 0x28, 0x0605 },
+	{ 0x2a, 0x0000 },
+	{ 0x2c, 0xbb80 },
+	{ 0x32, 0xbb80 },
+	{ 0x34, 0x2000 },
+	{ 0x5a, 0x0000 },
+	{ 0x5c, 0x0000 },
+	{ 0x72, 0x0808 },
+	{ 0x74, 0x0000 },
+	{ 0x76, 0x0006 },
+	{ 0x78, 0x0000 },
+	{ 0x7a, 0x0000 },
+};
+
+static const struct regmap_config wm9705_regmap_config = {
+	.reg_bits = 16,
+	.reg_stride = 2,
+	.val_bits = 16,
+	.max_register = 0x7e,
+	.cache_type = REGCACHE_RBTREE,
+
+	.volatile_reg = regmap_ac97_default_volatile,
+
+	.reg_defaults = wm9705_reg_defaults,
+	.num_reg_defaults = ARRAY_SIZE(wm9705_reg_defaults),
 };
 
 static const struct snd_kcontrol_new wm9705_snd_ac97_controls[] = {
@@ -203,57 +224,20 @@ static const struct snd_soc_dapm_route wm9705_audio_map[] = {
 	{"Right ADC", NULL, "ADC PGA"},
 };
 
-/* We use a register cache to enhance read performance. */
-static unsigned int ac97_read(struct snd_soc_codec *codec, unsigned int reg)
-{
-	struct snd_ac97 *ac97 = snd_soc_codec_get_drvdata(codec);
-	u16 *cache = codec->reg_cache;
-
-	switch (reg) {
-	case AC97_RESET:
-	case AC97_VENDOR_ID1:
-	case AC97_VENDOR_ID2:
-		return soc_ac97_ops->read(ac97, reg);
-	default:
-		reg = reg >> 1;
-
-		if (reg >= (ARRAY_SIZE(wm9705_reg)))
-			return -EIO;
-
-		return cache[reg];
-	}
-}
-
-static int ac97_write(struct snd_soc_codec *codec, unsigned int reg,
-	unsigned int val)
-{
-	struct snd_ac97 *ac97 = snd_soc_codec_get_drvdata(codec);
-	u16 *cache = codec->reg_cache;
-
-	soc_ac97_ops->write(ac97, reg, val);
-	reg = reg >> 1;
-	if (reg < (ARRAY_SIZE(wm9705_reg)))
-		cache[reg] = val;
-
-	return 0;
-}
-
 static int ac97_prepare(struct snd_pcm_substream *substream,
 			struct snd_soc_dai *dai)
 {
 	struct snd_soc_codec *codec = dai->codec;
 	int reg;
-	u16 vra;
 
-	vra = ac97_read(codec, AC97_EXTENDED_STATUS);
-	ac97_write(codec, AC97_EXTENDED_STATUS, vra | 0x1);
+	snd_soc_update_bits(codec, AC97_EXTENDED_STATUS, 0x1, 0x1);
 
 	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
 		reg = AC97_PCM_FRONT_DAC_RATE;
 	else
 		reg = AC97_PCM_LR_ADC_RATE;
 
-	return ac97_write(codec, reg, substream->runtime->rate);
+	return snd_soc_write(codec, reg, substream->runtime->rate);
 }
 
 #define WM9705_AC97_RATES (SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_11025 | \
@@ -299,9 +283,9 @@ static struct snd_soc_dai_driver wm9705_dai[] = {
 #ifdef CONFIG_PM
 static int wm9705_soc_suspend(struct snd_soc_codec *codec)
 {
-	struct snd_ac97 *ac97 = snd_soc_codec_get_drvdata(codec);
-
-	soc_ac97_ops->write(ac97, AC97_POWERDOWN, 0xffff);
+	regcache_cache_bypass(codec->component.regmap, true);
+	snd_soc_write(codec, AC97_POWERDOWN, 0xffff);
+	regcache_cache_bypass(codec->component.regmap, false);
 
 	return 0;
 }
@@ -309,17 +293,14 @@ static int wm9705_soc_suspend(struct snd_soc_codec *codec)
 static int wm9705_soc_resume(struct snd_soc_codec *codec)
 {
 	struct snd_ac97 *ac97 = snd_soc_codec_get_drvdata(codec);
-	int i, ret;
-	u16 *cache = codec->reg_cache;
+	int ret;
 
 	ret = snd_ac97_reset(ac97, true, WM9705_VENDOR_ID,
 		WM9705_VENDOR_ID_MASK);
 	if (ret < 0)
 		return ret;
 
-	for (i = 2; i < ARRAY_SIZE(wm9705_reg) << 1; i += 2) {
-		soc_ac97_ops->write(ac97, i, cache[i>>1]);
-	}
+	regcache_sync(codec->component.regmap);
 
 	return 0;
 }
@@ -331,6 +312,8 @@ static int wm9705_soc_resume(struct snd_soc_codec *codec)
 static int wm9705_soc_probe(struct snd_soc_codec *codec)
 {
 	struct snd_ac97 *ac97;
+	struct regmap *regmap;
+	int ret;
 
 	ac97 = snd_soc_new_ac97_codec(codec, WM9705_VENDOR_ID,
 		WM9705_VENDOR_ID_MASK);
@@ -339,15 +322,26 @@ static int wm9705_soc_probe(struct snd_soc_codec *codec)
 		return PTR_ERR(ac97);
 	}
 
+	regmap = regmap_init_ac97(ac97, &wm9705_regmap_config);
+	if (IS_ERR(regmap)) {
+		ret = PTR_ERR(regmap);
+		goto err_free_ac97_codec;
+	}
+
 	snd_soc_codec_set_drvdata(codec, ac97);
+	snd_soc_codec_init_regmap(codec, regmap);
 
 	return 0;
+err_free_ac97_codec:
+	snd_soc_free_ac97_codec(ac97);
+	return ret;
 }
 
 static int wm9705_soc_remove(struct snd_soc_codec *codec)
 {
 	struct snd_ac97 *ac97 = snd_soc_codec_get_drvdata(codec);
 
+	snd_soc_codec_exit_regmap(codec);
 	snd_soc_free_ac97_codec(ac97);
 	return 0;
 }
@@ -357,12 +351,6 @@ static const struct snd_soc_codec_driver soc_codec_dev_wm9705 = {
 	.remove = 	wm9705_soc_remove,
 	.suspend =	wm9705_soc_suspend,
 	.resume =	wm9705_soc_resume,
-	.read = ac97_read,
-	.write = ac97_write,
-	.reg_cache_size = ARRAY_SIZE(wm9705_reg),
-	.reg_word_size = sizeof(u16),
-	.reg_cache_step = 2,
-	.reg_cache_default = wm9705_reg,
 
 	.component_driver = {
 		.controls		= wm9705_snd_ac97_controls,
diff --git a/sound/soc/codecs/wm9705.h b/sound/soc/codecs/wm9705.h
deleted file mode 100644
index 23ea9ce47359..000000000000
--- a/sound/soc/codecs/wm9705.h
+++ /dev/null
@@ -1,11 +0,0 @@
-/*
- * wm9705.h  --  WM9705 Soc Audio driver
- */
-
-#ifndef _WM9705_H
-#define _WM9705_H
-
-#define WM9705_DAI_AC97_HIFI	0
-#define WM9705_DAI_AC97_AUX	1
-
-#endif
diff --git a/sound/soc/codecs/wm9712.c b/sound/soc/codecs/wm9712.c
index 557709eac698..1a3e1797994a 100644
--- a/sound/soc/codecs/wm9712.c
+++ b/sound/soc/codecs/wm9712.c
@@ -15,13 +15,13 @@
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/device.h>
+#include <linux/regmap.h>
 #include <sound/core.h>
 #include <sound/pcm.h>
 #include <sound/ac97_codec.h>
 #include <sound/initval.h>
 #include <sound/soc.h>
 #include <sound/tlv.h>
-#include "wm9712.h"
 
 #define WM9712_VENDOR_ID 0x574d4c12
 #define WM9712_VENDOR_ID_MASK 0xffffffff
@@ -32,31 +32,66 @@ struct wm9712_priv {
 	struct mutex lock;
 };
 
-static unsigned int ac97_read(struct snd_soc_codec *codec,
-	unsigned int reg);
-static int ac97_write(struct snd_soc_codec *codec,
-	unsigned int reg, unsigned int val);
+static const struct reg_default wm9712_reg_defaults[] = {
+	{ 0x02, 0x8000 },
+	{ 0x04, 0x8000 },
+	{ 0x06, 0x8000 },
+	{ 0x08, 0x0f0f },
+	{ 0x0a, 0xaaa0 },
+	{ 0x0c, 0xc008 },
+	{ 0x0e, 0x6808 },
+	{ 0x10, 0xe808 },
+	{ 0x12, 0xaaa0 },
+	{ 0x14, 0xad00 },
+	{ 0x16, 0x8000 },
+	{ 0x18, 0xe808 },
+	{ 0x1a, 0x3000 },
+	{ 0x1c, 0x8000 },
+	{ 0x20, 0x0000 },
+	{ 0x22, 0x0000 },
+	{ 0x26, 0x000f },
+	{ 0x28, 0x0605 },
+	{ 0x2a, 0x0410 },
+	{ 0x2c, 0xbb80 },
+	{ 0x2e, 0xbb80 },
+	{ 0x32, 0xbb80 },
+	{ 0x34, 0x2000 },
+	{ 0x4c, 0xf83e },
+	{ 0x4e, 0xffff },
+	{ 0x50, 0x0000 },
+	{ 0x52, 0x0000 },
+	{ 0x56, 0xf83e },
+	{ 0x58, 0x0008 },
+	{ 0x5c, 0x0000 },
+	{ 0x60, 0xb032 },
+	{ 0x62, 0x3e00 },
+	{ 0x64, 0x0000 },
+	{ 0x76, 0x0006 },
+	{ 0x78, 0x0001 },
+	{ 0x7a, 0x0000 },
+};
 
-/*
- * WM9712 register cache
- */
-static const u16 wm9712_reg[] = {
-	0x6174, 0x8000, 0x8000, 0x8000, /*  6 */
-	0x0f0f, 0xaaa0, 0xc008, 0x6808, /*  e */
-	0xe808, 0xaaa0, 0xad00, 0x8000, /* 16 */
-	0xe808, 0x3000, 0x8000, 0x0000, /* 1e */
-	0x0000, 0x0000, 0x0000, 0x000f, /* 26 */
-	0x0405, 0x0410, 0xbb80, 0xbb80, /* 2e */
-	0x0000, 0xbb80, 0x0000, 0x0000, /* 36 */
-	0x0000, 0x2000, 0x0000, 0x0000, /* 3e */
-	0x0000, 0x0000, 0x0000, 0x0000, /* 46 */
-	0x0000, 0x0000, 0xf83e, 0xffff, /* 4e */
-	0x0000, 0x0000, 0x0000, 0xf83e, /* 56 */
-	0x0008, 0x0000, 0x0000, 0x0000, /* 5e */
-	0xb032, 0x3e00, 0x0000, 0x0000, /* 66 */
-	0x0000, 0x0000, 0x0000, 0x0000, /* 6e */
-	0x0000, 0x0000, 0x0000, 0x0006, /* 76 */
-	0x0001, 0x0000, 0x574d, 0x4c12, /* 7e */
+static bool wm9712_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case AC97_REC_GAIN:
+		return true;
+	default:
+		return regmap_ac97_default_volatile(dev, reg);
+	}
+}
+
+static const struct regmap_config wm9712_regmap_config = {
+	.reg_bits = 16,
+	.reg_stride = 2,
+	.val_bits = 16,
+	.max_register = 0x7e,
+	.cache_type = REGCACHE_RBTREE,
+
+	.volatile_reg = wm9712_volatile_reg,
+
+	.reg_defaults = wm9712_reg_defaults,
+	.num_reg_defaults = ARRAY_SIZE(wm9712_reg_defaults),
 };
 
 #define HPL_MIXER	0x0
@@ -187,7 +222,7 @@ static int wm9712_hp_mixer_put(struct snd_kcontrol *kcontrol,
 	struct soc_mixer_control *mc =
 		(struct soc_mixer_control *)kcontrol->private_value;
 	unsigned int mixer, mask, shift, old;
-	struct snd_soc_dapm_update update;
+	struct snd_soc_dapm_update update = { 0 };
 	bool change;
 
 	mixer = mc->shift >> 8;
@@ -485,75 +520,36 @@ static const struct snd_soc_dapm_route wm9712_audio_map[] = {
 	{"ROUT2", NULL, "Speaker PGA"},
 };
 
-static unsigned int ac97_read(struct snd_soc_codec *codec,
-	unsigned int reg)
-{
-	struct wm9712_priv *wm9712 = snd_soc_codec_get_drvdata(codec);
-	u16 *cache = codec->reg_cache;
-
-	if (reg == AC97_RESET || reg == AC97_GPIO_STATUS ||
-		reg == AC97_VENDOR_ID1 || reg == AC97_VENDOR_ID2 ||
-		reg == AC97_REC_GAIN)
-		return soc_ac97_ops->read(wm9712->ac97, reg);
-	else {
-		reg = reg >> 1;
-
-		if (reg >= (ARRAY_SIZE(wm9712_reg)))
-			return -EIO;
-
-		return cache[reg];
-	}
-}
-
-static int ac97_write(struct snd_soc_codec *codec, unsigned int reg,
-	unsigned int val)
-{
-	struct wm9712_priv *wm9712 = snd_soc_codec_get_drvdata(codec);
-	u16 *cache = codec->reg_cache;
-
-	soc_ac97_ops->write(wm9712->ac97, reg, val);
-	reg = reg >> 1;
-	if (reg < (ARRAY_SIZE(wm9712_reg)))
-		cache[reg] = val;
-
-	return 0;
-}
-
 static int ac97_prepare(struct snd_pcm_substream *substream,
 			struct snd_soc_dai *dai)
 {
 	struct snd_soc_codec *codec = dai->codec;
 	int reg;
-	u16 vra;
 	struct snd_pcm_runtime *runtime = substream->runtime;
 
-	vra = ac97_read(codec, AC97_EXTENDED_STATUS);
-	ac97_write(codec, AC97_EXTENDED_STATUS, vra | 0x1);
+	snd_soc_update_bits(codec, AC97_EXTENDED_STATUS, 0x1, 0x1);
 
 	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
 		reg = AC97_PCM_FRONT_DAC_RATE;
 	else
 		reg = AC97_PCM_LR_ADC_RATE;
 
-	return ac97_write(codec, reg, runtime->rate);
+	return snd_soc_write(codec, reg, runtime->rate);
 }
 
 static int ac97_aux_prepare(struct snd_pcm_substream *substream,
 			    struct snd_soc_dai *dai)
 {
 	struct snd_soc_codec *codec = dai->codec;
-	u16 vra, xsle;
 	struct snd_pcm_runtime *runtime = substream->runtime;
 
-	vra = ac97_read(codec, AC97_EXTENDED_STATUS);
-	ac97_write(codec, AC97_EXTENDED_STATUS, vra | 0x1);
-	xsle = ac97_read(codec, AC97_PCI_SID);
-	ac97_write(codec, AC97_PCI_SID, xsle | 0x8000);
+	snd_soc_update_bits(codec, AC97_EXTENDED_STATUS, 0x1, 0x1);
+	snd_soc_update_bits(codec, AC97_PCI_SID, 0x8000, 0x8000);
 
 	if (substream->stream != SNDRV_PCM_STREAM_PLAYBACK)
 		return -ENODEV;
 
-	return ac97_write(codec, AC97_PCM_SURR_DAC_RATE, runtime->rate);
+	return snd_soc_write(codec, AC97_PCM_SURR_DAC_RATE, runtime->rate);
 }
 
 #define WM9712_AC97_RATES (SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_11025 |\
@@ -605,12 +601,12 @@ static int wm9712_set_bias_level(struct snd_soc_codec *codec,
 	case SND_SOC_BIAS_PREPARE:
 		break;
 	case SND_SOC_BIAS_STANDBY:
-		ac97_write(codec, AC97_POWERDOWN, 0x0000);
+		snd_soc_write(codec, AC97_POWERDOWN, 0x0000);
 		break;
 	case SND_SOC_BIAS_OFF:
 		/* disable everything including AC link */
-		ac97_write(codec, AC97_EXTENDED_MSTATUS, 0xffff);
-		ac97_write(codec, AC97_POWERDOWN, 0xffff);
+		snd_soc_write(codec, AC97_EXTENDED_MSTATUS, 0xffff);
+		snd_soc_write(codec, AC97_POWERDOWN, 0xffff);
 		break;
 	}
 	return 0;
@@ -619,8 +615,7 @@ static int wm9712_set_bias_level(struct snd_soc_codec *codec,
 static int wm9712_soc_resume(struct snd_soc_codec *codec)
 {
 	struct wm9712_priv *wm9712 = snd_soc_codec_get_drvdata(codec);
-	int i, ret;
-	u16 *cache = codec->reg_cache;
+	int ret;
 
 	ret = snd_ac97_reset(wm9712->ac97, true, WM9712_VENDOR_ID,
 		WM9712_VENDOR_ID_MASK);
@@ -629,15 +624,8 @@ static int wm9712_soc_resume(struct snd_soc_codec *codec)
 
 	snd_soc_codec_force_bias_level(codec, SND_SOC_BIAS_STANDBY);
 
-	if (ret == 0) {
-		/* Sync reg_cache with the hardware after cold reset */
-		for (i = 2; i < ARRAY_SIZE(wm9712_reg) << 1; i += 2) {
-			if (i == AC97_INT_PAGING || i == AC97_POWERDOWN ||
-			    (i > 0x58 && i != 0x5c))
-				continue;
-			soc_ac97_ops->write(wm9712->ac97, i, cache[i>>1]);
-		}
-	}
+	if (ret == 0)
+		regcache_sync(codec->component.regmap);
 
 	return ret;
 }
@@ -645,6 +633,7 @@ static int wm9712_soc_resume(struct snd_soc_codec *codec)
 static int wm9712_soc_probe(struct snd_soc_codec *codec)
 {
 	struct wm9712_priv *wm9712 = snd_soc_codec_get_drvdata(codec);
+	struct regmap *regmap;
 	int ret;
 
 	wm9712->ac97 = snd_soc_new_ac97_codec(codec, WM9712_VENDOR_ID,
@@ -655,16 +644,28 @@ static int wm9712_soc_probe(struct snd_soc_codec *codec)
 		return ret;
 	}
 
+	regmap = regmap_init_ac97(wm9712->ac97, &wm9712_regmap_config);
+	if (IS_ERR(regmap)) {
+		ret = PTR_ERR(regmap);
+		goto err_free_ac97_codec;
+	}
+
+	snd_soc_codec_init_regmap(codec, regmap);
+
 	/* set alc mux to none */
-	ac97_write(codec, AC97_VIDEO, ac97_read(codec, AC97_VIDEO) | 0x3000);
+	snd_soc_update_bits(codec, AC97_VIDEO, 0x3000, 0x3000);
 
 	return 0;
+err_free_ac97_codec:
+	snd_soc_free_ac97_codec(wm9712->ac97);
+	return ret;
 }
 
 static int wm9712_soc_remove(struct snd_soc_codec *codec)
 {
 	struct wm9712_priv *wm9712 = snd_soc_codec_get_drvdata(codec);
 
+	snd_soc_codec_exit_regmap(codec);
 	snd_soc_free_ac97_codec(wm9712->ac97);
 	return 0;
 }
@@ -673,14 +674,8 @@ static const struct snd_soc_codec_driver soc_codec_dev_wm9712 = {
 	.probe = 	wm9712_soc_probe,
 	.remove = 	wm9712_soc_remove,
 	.resume =	wm9712_soc_resume,
-	.read = ac97_read,
-	.write = ac97_write,
 	.set_bias_level = wm9712_set_bias_level,
 	.suspend_bias_off = true,
-	.reg_cache_size = ARRAY_SIZE(wm9712_reg),
-	.reg_word_size = sizeof(u16),
-	.reg_cache_step = 2,
-	.reg_cache_default = wm9712_reg,
 
 	.component_driver = {
 		.controls		= wm9712_snd_ac97_controls,
diff --git a/sound/soc/codecs/wm9712.h b/sound/soc/codecs/wm9712.h
deleted file mode 100644
index fb69c3aa4ed0..000000000000
--- a/sound/soc/codecs/wm9712.h
+++ /dev/null
@@ -1,11 +0,0 @@
-/*
- * wm9712.h  --  WM9712 Soc Audio driver
- */
-
-#ifndef _WM9712_H
-#define _WM9712_H
-
-#define WM9712_DAI_AC97_HIFI	0
-#define WM9712_DAI_AC97_AUX		1
-
-#endif
diff --git a/sound/soc/codecs/wm9713.c b/sound/soc/codecs/wm9713.c
index e4301ddb1b84..7e4822185feb 100644
--- a/sound/soc/codecs/wm9713.c
+++ b/sound/soc/codecs/wm9713.c
@@ -231,7 +231,7 @@ static int wm9713_hp_mixer_put(struct snd_kcontrol *kcontrol,
 	struct soc_mixer_control *mc =
 		(struct soc_mixer_control *)kcontrol->private_value;
 	unsigned int mixer, mask, shift, old;
-	struct snd_soc_dapm_update update;
+	struct snd_soc_dapm_update update = { 0 };
 	bool change;
 
 	mixer = mc->shift >> 8;
diff --git a/sound/soc/codecs/wm9713.h b/sound/soc/codecs/wm9713.h
index 53df11b1f727..7ecffc563016 100644
--- a/sound/soc/codecs/wm9713.h
+++ b/sound/soc/codecs/wm9713.h
@@ -41,8 +41,4 @@
 #define WM9713_PCMBCLK_DIV_8	(3 << 9)
 #define WM9713_PCMBCLK_DIV_16	(4 << 9)
 
-#define WM9713_DAI_AC97_HIFI	0
-#define WM9713_DAI_AC97_AUX		1
-#define WM9713_DAI_PCM_VOICE	2
-
 #endif
diff --git a/sound/soc/codecs/wm_adsp.c b/sound/soc/codecs/wm_adsp.c
index b943dde8dbe5..593b7d1aed46 100644
--- a/sound/soc/codecs/wm_adsp.c
+++ b/sound/soc/codecs/wm_adsp.c
@@ -162,6 +162,16 @@
 
 #define ADSP_MAX_STD_CTRL_SIZE               512
 
+#define WM_ADSP_ACKED_CTL_TIMEOUT_MS         100
+#define WM_ADSP_ACKED_CTL_N_QUICKPOLLS       10
+#define WM_ADSP_ACKED_CTL_MIN_VALUE          0
+#define WM_ADSP_ACKED_CTL_MAX_VALUE          0xFFFFFF
+
+/*
+ * Event control messages
+ */
+#define WM_ADSP_FW_EVENT_SHUTDOWN            0x000001
+
 struct wm_adsp_buf {
 	struct list_head list;
 	void *buf;
@@ -177,7 +187,7 @@ static struct wm_adsp_buf *wm_adsp_buf_alloc(const void *src, size_t len,
 
 	buf->buf = vmalloc(len);
 	if (!buf->buf) {
-		vfree(buf);
+		kfree(buf);
 		return NULL;
 	}
 	memcpy(buf->buf, src, len);
@@ -441,11 +451,29 @@ struct wm_coeff_ctl {
 	unsigned int offset;
 	size_t len;
 	unsigned int set:1;
-	struct snd_kcontrol *kcontrol;
 	struct soc_bytes_ext bytes_ext;
 	unsigned int flags;
+	unsigned int type;
 };
 
+static const char *wm_adsp_mem_region_name(unsigned int type)
+{
+	switch (type) {
+	case WMFW_ADSP1_PM:
+		return "PM";
+	case WMFW_ADSP1_DM:
+		return "DM";
+	case WMFW_ADSP2_XM:
+		return "XM";
+	case WMFW_ADSP2_YM:
+		return "YM";
+	case WMFW_ADSP1_ZM:
+		return "ZM";
+	default:
+		return NULL;
+	}
+}
+
 #ifdef CONFIG_DEBUG_FS
 static void wm_adsp_debugfs_save_wmfwname(struct wm_adsp *dsp, const char *s)
 {
@@ -727,6 +755,24 @@ static inline struct wm_coeff_ctl *bytes_ext_to_ctl(struct soc_bytes_ext *ext)
 	return container_of(ext, struct wm_coeff_ctl, bytes_ext);
 }
 
+static int wm_coeff_base_reg(struct wm_coeff_ctl *ctl, unsigned int *reg)
+{
+	const struct wm_adsp_alg_region *alg_region = &ctl->alg_region;
+	struct wm_adsp *dsp = ctl->dsp;
+	const struct wm_adsp_region *mem;
+
+	mem = wm_adsp_find_region(dsp, alg_region->type);
+	if (!mem) {
+		adsp_err(dsp, "No base for region %x\n",
+			 alg_region->type);
+		return -EINVAL;
+	}
+
+	*reg = wm_adsp_region_to_reg(mem, ctl->alg_region.base + ctl->offset);
+
+	return 0;
+}
+
 static int wm_coeff_info(struct snd_kcontrol *kctl,
 			 struct snd_ctl_elem_info *uinfo)
 {
@@ -734,30 +780,94 @@ static int wm_coeff_info(struct snd_kcontrol *kctl,
 		(struct soc_bytes_ext *)kctl->private_value;
 	struct wm_coeff_ctl *ctl = bytes_ext_to_ctl(bytes_ext);
 
-	uinfo->type = SNDRV_CTL_ELEM_TYPE_BYTES;
-	uinfo->count = ctl->len;
+	switch (ctl->type) {
+	case WMFW_CTL_TYPE_ACKED:
+		uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
+		uinfo->value.integer.min = WM_ADSP_ACKED_CTL_MIN_VALUE;
+		uinfo->value.integer.max = WM_ADSP_ACKED_CTL_MAX_VALUE;
+		uinfo->value.integer.step = 1;
+		uinfo->count = 1;
+		break;
+	default:
+		uinfo->type = SNDRV_CTL_ELEM_TYPE_BYTES;
+		uinfo->count = ctl->len;
+		break;
+	}
+
 	return 0;
 }
 
+static int wm_coeff_write_acked_control(struct wm_coeff_ctl *ctl,
+					unsigned int event_id)
+{
+	struct wm_adsp *dsp = ctl->dsp;
+	u32 val = cpu_to_be32(event_id);
+	unsigned int reg;
+	int i, ret;
+
+	ret = wm_coeff_base_reg(ctl, &reg);
+	if (ret)
+		return ret;
+
+	adsp_dbg(dsp, "Sending 0x%x to acked control alg 0x%x %s:0x%x\n",
+		 event_id, ctl->alg_region.alg,
+		 wm_adsp_mem_region_name(ctl->alg_region.type), ctl->offset);
+
+	ret = regmap_raw_write(dsp->regmap, reg, &val, sizeof(val));
+	if (ret) {
+		adsp_err(dsp, "Failed to write %x: %d\n", reg, ret);
+		return ret;
+	}
+
+	/*
+	 * Poll for ack, we initially poll at ~1ms intervals for firmwares
+	 * that respond quickly, then go to ~10ms polls. A firmware is unlikely
+	 * to ack instantly so we do the first 1ms delay before reading the
+	 * control to avoid a pointless bus transaction
+	 */
+	for (i = 0; i < WM_ADSP_ACKED_CTL_TIMEOUT_MS;) {
+		switch (i) {
+		case 0 ... WM_ADSP_ACKED_CTL_N_QUICKPOLLS - 1:
+			usleep_range(1000, 2000);
+			i++;
+			break;
+		default:
+			usleep_range(10000, 20000);
+			i += 10;
+			break;
+		}
+
+		ret = regmap_raw_read(dsp->regmap, reg, &val, sizeof(val));
+		if (ret) {
+			adsp_err(dsp, "Failed to read %x: %d\n", reg, ret);
+			return ret;
+		}
+
+		if (val == 0) {
+			adsp_dbg(dsp, "Acked control ACKED at poll %u\n", i);
+			return 0;
+		}
+	}
+
+	adsp_warn(dsp, "Acked control @0x%x alg:0x%x %s:0x%x timed out\n",
+		  reg, ctl->alg_region.alg,
+		  wm_adsp_mem_region_name(ctl->alg_region.type),
+		  ctl->offset);
+
+	return -ETIMEDOUT;
+}
+
 static int wm_coeff_write_control(struct wm_coeff_ctl *ctl,
 				  const void *buf, size_t len)
 {
-	struct wm_adsp_alg_region *alg_region = &ctl->alg_region;
-	const struct wm_adsp_region *mem;
 	struct wm_adsp *dsp = ctl->dsp;
 	void *scratch;
 	int ret;
 	unsigned int reg;
 
-	mem = wm_adsp_find_region(dsp, alg_region->type);
-	if (!mem) {
-		adsp_err(dsp, "No base for region %x\n",
-			 alg_region->type);
-		return -EINVAL;
-	}
-
-	reg = ctl->alg_region.base + ctl->offset;
-	reg = wm_adsp_region_to_reg(mem, reg);
+	ret = wm_coeff_base_reg(ctl, &reg);
+	if (ret)
+		return ret;
 
 	scratch = kmemdup(buf, len, GFP_KERNEL | GFP_DMA);
 	if (!scratch)
@@ -823,25 +933,41 @@ static int wm_coeff_tlv_put(struct snd_kcontrol *kctl,
 	return ret;
 }
 
+static int wm_coeff_put_acked(struct snd_kcontrol *kctl,
+			      struct snd_ctl_elem_value *ucontrol)
+{
+	struct soc_bytes_ext *bytes_ext =
+		(struct soc_bytes_ext *)kctl->private_value;
+	struct wm_coeff_ctl *ctl = bytes_ext_to_ctl(bytes_ext);
+	unsigned int val = ucontrol->value.integer.value[0];
+	int ret;
+
+	if (val == 0)
+		return 0;	/* 0 means no event */
+
+	mutex_lock(&ctl->dsp->pwr_lock);
+
+	if (ctl->enabled)
+		ret = wm_coeff_write_acked_control(ctl, val);
+	else
+		ret = -EPERM;
+
+	mutex_unlock(&ctl->dsp->pwr_lock);
+
+	return ret;
+}
+
 static int wm_coeff_read_control(struct wm_coeff_ctl *ctl,
 				 void *buf, size_t len)
 {
-	struct wm_adsp_alg_region *alg_region = &ctl->alg_region;
-	const struct wm_adsp_region *mem;
 	struct wm_adsp *dsp = ctl->dsp;
 	void *scratch;
 	int ret;
 	unsigned int reg;
 
-	mem = wm_adsp_find_region(dsp, alg_region->type);
-	if (!mem) {
-		adsp_err(dsp, "No base for region %x\n",
-			 alg_region->type);
-		return -EINVAL;
-	}
-
-	reg = ctl->alg_region.base + ctl->offset;
-	reg = wm_adsp_region_to_reg(mem, reg);
+	ret = wm_coeff_base_reg(ctl, &reg);
+	if (ret)
+		return ret;
 
 	scratch = kmalloc(len, GFP_KERNEL | GFP_DMA);
 	if (!scratch)
@@ -918,6 +1044,21 @@ static int wm_coeff_tlv_get(struct snd_kcontrol *kctl,
 	return ret;
 }
 
+static int wm_coeff_get_acked(struct snd_kcontrol *kcontrol,
+			      struct snd_ctl_elem_value *ucontrol)
+{
+	/*
+	 * Although it's not useful to read an acked control, we must satisfy
+	 * user-side assumptions that all controls are readable and that a
+	 * write of the same value should be filtered out (it's valid to send
+	 * the same event number again to the firmware). We therefore return 0,
+	 * meaning "no event" so valid event numbers will always be a change
+	 */
+	ucontrol->value.integer.value[0] = 0;
+
+	return 0;
+}
+
 struct wmfw_ctl_work {
 	struct wm_adsp *dsp;
 	struct wm_coeff_ctl *ctl;
@@ -967,30 +1108,35 @@ static int wmfw_add_ctl(struct wm_adsp *dsp, struct wm_coeff_ctl *ctl)
 	kcontrol = kzalloc(sizeof(*kcontrol), GFP_KERNEL);
 	if (!kcontrol)
 		return -ENOMEM;
-	kcontrol->iface = SNDRV_CTL_ELEM_IFACE_MIXER;
 
 	kcontrol->name = ctl->name;
 	kcontrol->info = wm_coeff_info;
-	kcontrol->get = wm_coeff_get;
-	kcontrol->put = wm_coeff_put;
 	kcontrol->iface = SNDRV_CTL_ELEM_IFACE_MIXER;
 	kcontrol->tlv.c = snd_soc_bytes_tlv_callback;
 	kcontrol->private_value = (unsigned long)&ctl->bytes_ext;
+	kcontrol->access = wmfw_convert_flags(ctl->flags, ctl->len);
 
-	ctl->bytes_ext.max = ctl->len;
-	ctl->bytes_ext.get = wm_coeff_tlv_get;
-	ctl->bytes_ext.put = wm_coeff_tlv_put;
+	switch (ctl->type) {
+	case WMFW_CTL_TYPE_ACKED:
+		kcontrol->get = wm_coeff_get_acked;
+		kcontrol->put = wm_coeff_put_acked;
+		break;
+	default:
+		kcontrol->get = wm_coeff_get;
+		kcontrol->put = wm_coeff_put;
 
-	kcontrol->access = wmfw_convert_flags(ctl->flags, ctl->len);
+		ctl->bytes_ext.max = ctl->len;
+		ctl->bytes_ext.get = wm_coeff_tlv_get;
+		ctl->bytes_ext.put = wm_coeff_tlv_put;
+		break;
+	}
 
-	ret = snd_soc_add_card_controls(dsp->card, kcontrol, 1);
+	ret = snd_soc_add_codec_controls(dsp->codec, kcontrol, 1);
 	if (ret < 0)
 		goto err_kcontrol;
 
 	kfree(kcontrol);
 
-	ctl->kcontrol = snd_soc_card_get_kcontrol(dsp->card, ctl->name);
-
 	return 0;
 
 err_kcontrol:
@@ -1035,6 +1181,27 @@ static int wm_coeff_sync_controls(struct wm_adsp *dsp)
 	return 0;
 }
 
+static void wm_adsp_signal_event_controls(struct wm_adsp *dsp,
+					  unsigned int event)
+{
+	struct wm_coeff_ctl *ctl;
+	int ret;
+
+	list_for_each_entry(ctl, &dsp->ctl_list, list) {
+		if (ctl->type != WMFW_CTL_TYPE_HOSTEVENT)
+			continue;
+
+		if (!ctl->enabled)
+			continue;
+
+		ret = wm_coeff_write_acked_control(ctl, event);
+		if (ret)
+			adsp_warn(dsp,
+				  "Failed to send 0x%x event to alg 0x%x (%d)\n",
+				  event, ctl->alg_region.alg, ret);
+	}
+}
+
 static void wm_adsp_ctl_work(struct work_struct *work)
 {
 	struct wmfw_ctl_work *ctl_work = container_of(work,
@@ -1056,34 +1223,16 @@ static int wm_adsp_create_control(struct wm_adsp *dsp,
 				  const struct wm_adsp_alg_region *alg_region,
 				  unsigned int offset, unsigned int len,
 				  const char *subname, unsigned int subname_len,
-				  unsigned int flags)
+				  unsigned int flags, unsigned int type)
 {
 	struct wm_coeff_ctl *ctl;
 	struct wmfw_ctl_work *ctl_work;
 	char name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN];
-	char *region_name;
+	const char *region_name;
 	int ret;
 
-	if (flags & WMFW_CTL_FLAG_SYS)
-		return 0;
-
-	switch (alg_region->type) {
-	case WMFW_ADSP1_PM:
-		region_name = "PM";
-		break;
-	case WMFW_ADSP1_DM:
-		region_name = "DM";
-		break;
-	case WMFW_ADSP2_XM:
-		region_name = "XM";
-		break;
-	case WMFW_ADSP2_YM:
-		region_name = "YM";
-		break;
-	case WMFW_ADSP1_ZM:
-		region_name = "ZM";
-		break;
-	default:
+	region_name = wm_adsp_mem_region_name(alg_region->type);
+	if (!region_name) {
 		adsp_err(dsp, "Unknown region type: %d\n", alg_region->type);
 		return -EINVAL;
 	}
@@ -1139,6 +1288,7 @@ static int wm_adsp_create_control(struct wm_adsp *dsp,
 	ctl->dsp = dsp;
 
 	ctl->flags = flags;
+	ctl->type = type;
 	ctl->offset = offset;
 	ctl->len = len;
 	ctl->cache = kzalloc(ctl->len, GFP_KERNEL);
@@ -1149,6 +1299,9 @@ static int wm_adsp_create_control(struct wm_adsp *dsp,
 
 	list_add(&ctl->list, &dsp->ctl_list);
 
+	if (flags & WMFW_CTL_FLAG_SYS)
+		return 0;
+
 	ctl_work = kzalloc(sizeof(*ctl_work), GFP_KERNEL);
 	if (!ctl_work) {
 		ret = -ENOMEM;
@@ -1308,6 +1461,21 @@ static inline void wm_coeff_parse_coeff(struct wm_adsp *dsp, const u8 **data,
 	adsp_dbg(dsp, "\tALSA control len: %#x\n", blk->len);
 }
 
+static int wm_adsp_check_coeff_flags(struct wm_adsp *dsp,
+				const struct wm_coeff_parsed_coeff *coeff_blk,
+				unsigned int f_required,
+				unsigned int f_illegal)
+{
+	if ((coeff_blk->flags & f_illegal) ||
+	    ((coeff_blk->flags & f_required) != f_required)) {
+		adsp_err(dsp, "Illegal flags 0x%x for control type 0x%x\n",
+			 coeff_blk->flags, coeff_blk->ctl_type);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int wm_adsp_parse_coeff(struct wm_adsp *dsp,
 			       const struct wmfw_region *region)
 {
@@ -1324,6 +1492,28 @@ static int wm_adsp_parse_coeff(struct wm_adsp *dsp,
 		switch (coeff_blk.ctl_type) {
 		case SNDRV_CTL_ELEM_TYPE_BYTES:
 			break;
+		case WMFW_CTL_TYPE_ACKED:
+			if (coeff_blk.flags & WMFW_CTL_FLAG_SYS)
+				continue;	/* ignore */
+
+			ret = wm_adsp_check_coeff_flags(dsp, &coeff_blk,
+						WMFW_CTL_FLAG_VOLATILE |
+						WMFW_CTL_FLAG_WRITEABLE |
+						WMFW_CTL_FLAG_READABLE,
+						0);
+			if (ret)
+				return -EINVAL;
+			break;
+		case WMFW_CTL_TYPE_HOSTEVENT:
+			ret = wm_adsp_check_coeff_flags(dsp, &coeff_blk,
+						WMFW_CTL_FLAG_SYS |
+						WMFW_CTL_FLAG_VOLATILE |
+						WMFW_CTL_FLAG_WRITEABLE |
+						WMFW_CTL_FLAG_READABLE,
+						0);
+			if (ret)
+				return -EINVAL;
+			break;
 		default:
 			adsp_err(dsp, "Unknown control type: %d\n",
 				 coeff_blk.ctl_type);
@@ -1338,7 +1528,8 @@ static int wm_adsp_parse_coeff(struct wm_adsp *dsp,
 					     coeff_blk.len,
 					     coeff_blk.name,
 					     coeff_blk.name_len,
-					     coeff_blk.flags);
+					     coeff_blk.flags,
+					     coeff_blk.ctl_type);
 		if (ret < 0)
 			adsp_err(dsp, "Failed to create control: %.*s, %d\n",
 				 coeff_blk.name_len, coeff_blk.name, ret);
@@ -1491,23 +1682,11 @@ static int wm_adsp_load(struct wm_adsp *dsp)
 			reg = offset;
 			break;
 		case WMFW_ADSP1_PM:
-			region_name = "PM";
-			reg = wm_adsp_region_to_reg(mem, offset);
-			break;
 		case WMFW_ADSP1_DM:
-			region_name = "DM";
-			reg = wm_adsp_region_to_reg(mem, offset);
-			break;
 		case WMFW_ADSP2_XM:
-			region_name = "XM";
-			reg = wm_adsp_region_to_reg(mem, offset);
-			break;
 		case WMFW_ADSP2_YM:
-			region_name = "YM";
-			reg = wm_adsp_region_to_reg(mem, offset);
-			break;
 		case WMFW_ADSP1_ZM:
-			region_name = "ZM";
+			region_name = wm_adsp_mem_region_name(type);
 			reg = wm_adsp_region_to_reg(mem, offset);
 			break;
 		default:
@@ -1750,7 +1929,8 @@ static int wm_adsp1_setup_algs(struct wm_adsp *dsp)
 				len -= be32_to_cpu(adsp1_alg[i].dm);
 				len *= 4;
 				wm_adsp_create_control(dsp, alg_region, 0,
-						       len, NULL, 0, 0);
+						     len, NULL, 0, 0,
+						     SNDRV_CTL_ELEM_TYPE_BYTES);
 			} else {
 				adsp_warn(dsp, "Missing length info for region DM with ID %x\n",
 					  be32_to_cpu(adsp1_alg[i].alg.id));
@@ -1770,7 +1950,8 @@ static int wm_adsp1_setup_algs(struct wm_adsp *dsp)
 				len -= be32_to_cpu(adsp1_alg[i].zm);
 				len *= 4;
 				wm_adsp_create_control(dsp, alg_region, 0,
-						       len, NULL, 0, 0);
+						     len, NULL, 0, 0,
+						     SNDRV_CTL_ELEM_TYPE_BYTES);
 			} else {
 				adsp_warn(dsp, "Missing length info for region ZM with ID %x\n",
 					  be32_to_cpu(adsp1_alg[i].alg.id));
@@ -1861,7 +2042,8 @@ static int wm_adsp2_setup_algs(struct wm_adsp *dsp)
 				len -= be32_to_cpu(adsp2_alg[i].xm);
 				len *= 4;
 				wm_adsp_create_control(dsp, alg_region, 0,
-						       len, NULL, 0, 0);
+						     len, NULL, 0, 0,
+						     SNDRV_CTL_ELEM_TYPE_BYTES);
 			} else {
 				adsp_warn(dsp, "Missing length info for region XM with ID %x\n",
 					  be32_to_cpu(adsp2_alg[i].alg.id));
@@ -1881,7 +2063,8 @@ static int wm_adsp2_setup_algs(struct wm_adsp *dsp)
 				len -= be32_to_cpu(adsp2_alg[i].ym);
 				len *= 4;
 				wm_adsp_create_control(dsp, alg_region, 0,
-						       len, NULL, 0, 0);
+						     len, NULL, 0, 0,
+						     SNDRV_CTL_ELEM_TYPE_BYTES);
 			} else {
 				adsp_warn(dsp, "Missing length info for region YM with ID %x\n",
 					  be32_to_cpu(adsp2_alg[i].alg.id));
@@ -1901,7 +2084,8 @@ static int wm_adsp2_setup_algs(struct wm_adsp *dsp)
 				len -= be32_to_cpu(adsp2_alg[i].zm);
 				len *= 4;
 				wm_adsp_create_control(dsp, alg_region, 0,
-						       len, NULL, 0, 0);
+						     len, NULL, 0, 0,
+						     SNDRV_CTL_ELEM_TYPE_BYTES);
 			} else {
 				adsp_warn(dsp, "Missing length info for region ZM with ID %x\n",
 					  be32_to_cpu(adsp2_alg[i].alg.id));
@@ -2114,7 +2298,7 @@ int wm_adsp1_event(struct snd_soc_dapm_widget *w,
 	int ret;
 	unsigned int val;
 
-	dsp->card = codec->component.card;
+	dsp->codec = codec;
 
 	mutex_lock(&dsp->pwr_lock);
 
@@ -2325,8 +2509,6 @@ int wm_adsp2_early_event(struct snd_soc_dapm_widget *w,
 	struct wm_adsp *dsp = &dsps[w->shift];
 	struct wm_coeff_ctl *ctl;
 
-	dsp->card = codec->component.card;
-
 	switch (event) {
 	case SND_SOC_DAPM_PRE_PMU:
 		wm_adsp2_set_dspclk(dsp, freq);
@@ -2393,14 +2575,22 @@ int wm_adsp2_event(struct snd_soc_dapm_widget *w,
 
 		mutex_lock(&dsp->pwr_lock);
 
-		if (wm_adsp_fw[dsp->fw].num_caps != 0)
+		if (wm_adsp_fw[dsp->fw].num_caps != 0) {
 			ret = wm_adsp_buffer_init(dsp);
+			if (ret < 0) {
+				mutex_unlock(&dsp->pwr_lock);
+				goto err;
+			}
+		}
 
 		mutex_unlock(&dsp->pwr_lock);
 
 		break;
 
 	case SND_SOC_DAPM_PRE_PMD:
+		/* Tell the firmware to cleanup */
+		wm_adsp_signal_event_controls(dsp, WM_ADSP_FW_EVENT_SHUTDOWN);
+
 		/* Log firmware state, it can be useful for analysis */
 		wm_adsp2_show_fw_status(dsp);
 
@@ -2441,6 +2631,8 @@ EXPORT_SYMBOL_GPL(wm_adsp2_event);
 
 int wm_adsp2_codec_probe(struct wm_adsp *dsp, struct snd_soc_codec *codec)
 {
+	dsp->codec = codec;
+
 	wm_adsp2_init_debugfs(dsp, codec);
 
 	return snd_soc_add_codec_controls(codec,
diff --git a/sound/soc/codecs/wm_adsp.h b/sound/soc/codecs/wm_adsp.h
index 362dd7ce60d8..411d062c13f2 100644
--- a/sound/soc/codecs/wm_adsp.h
+++ b/sound/soc/codecs/wm_adsp.h
@@ -44,7 +44,7 @@ struct wm_adsp {
 	int type;
 	struct device *dev;
 	struct regmap *regmap;
-	struct snd_soc_card *card;
+	struct snd_soc_codec *codec;
 
 	int base;
 	int sysclk_reg;
@@ -110,18 +110,17 @@ int wm_adsp2_early_event(struct snd_soc_dapm_widget *w,
 int wm_adsp2_event(struct snd_soc_dapm_widget *w,
 		   struct snd_kcontrol *kcontrol, int event);
 
-extern int wm_adsp_compr_open(struct wm_adsp *dsp,
-			      struct snd_compr_stream *stream);
-extern int wm_adsp_compr_free(struct snd_compr_stream *stream);
-extern int wm_adsp_compr_set_params(struct snd_compr_stream *stream,
-				    struct snd_compr_params *params);
-extern int wm_adsp_compr_get_caps(struct snd_compr_stream *stream,
-				  struct snd_compr_caps *caps);
-extern int wm_adsp_compr_trigger(struct snd_compr_stream *stream, int cmd);
-extern int wm_adsp_compr_handle_irq(struct wm_adsp *dsp);
-extern int wm_adsp_compr_pointer(struct snd_compr_stream *stream,
-				 struct snd_compr_tstamp *tstamp);
-extern int wm_adsp_compr_copy(struct snd_compr_stream *stream,
-			      char __user *buf, size_t count);
+int wm_adsp_compr_open(struct wm_adsp *dsp, struct snd_compr_stream *stream);
+int wm_adsp_compr_free(struct snd_compr_stream *stream);
+int wm_adsp_compr_set_params(struct snd_compr_stream *stream,
+			     struct snd_compr_params *params);
+int wm_adsp_compr_get_caps(struct snd_compr_stream *stream,
+			   struct snd_compr_caps *caps);
+int wm_adsp_compr_trigger(struct snd_compr_stream *stream, int cmd);
+int wm_adsp_compr_handle_irq(struct wm_adsp *dsp);
+int wm_adsp_compr_pointer(struct snd_compr_stream *stream,
+			  struct snd_compr_tstamp *tstamp);
+int wm_adsp_compr_copy(struct snd_compr_stream *stream,
+		       char __user *buf, size_t count);
 
 #endif
diff --git a/sound/soc/codecs/wmfw.h b/sound/soc/codecs/wmfw.h
index 7613d60d62ea..ec78b9da020f 100644
--- a/sound/soc/codecs/wmfw.h
+++ b/sound/soc/codecs/wmfw.h
@@ -26,6 +26,10 @@
 #define WMFW_CTL_FLAG_WRITEABLE   0x0002
 #define WMFW_CTL_FLAG_READABLE    0x0001
 
+/* Non-ALSA coefficient types start at 0x1000 */
+#define WMFW_CTL_TYPE_ACKED       0x1000 /* acked control */
+#define WMFW_CTL_TYPE_HOSTEVENT   0x1001 /* event control */
+
 struct wmfw_header {
 	char magic[4];
 	__le32 len;
diff --git a/sound/soc/fsl/Kconfig b/sound/soc/fsl/Kconfig
index 19bdcac71775..37f9b6201918 100644
--- a/sound/soc/fsl/Kconfig
+++ b/sound/soc/fsl/Kconfig
@@ -40,6 +40,7 @@ config SND_SOC_FSL_SPDIF
 	select REGMAP_MMIO
 	select SND_SOC_IMX_PCM_DMA if SND_IMX_SOC != n
 	select SND_SOC_IMX_PCM_FIQ if SND_IMX_SOC != n && (MXC_TZIC || MXC_AVIC)
+	select BITREVERSE
 	help
 	  Say Y if you want to add Sony/Philips Digital Interface (SPDIF)
 	  support for the Freescale CPUs.
diff --git a/sound/soc/fsl/efika-audio-fabric.c b/sound/soc/fsl/efika-audio-fabric.c
index b2acd3293ea8..f200d1cfc4bd 100644
--- a/sound/soc/fsl/efika-audio-fabric.c
+++ b/sound/soc/fsl/efika-audio-fabric.c
@@ -27,7 +27,6 @@
 
 #include "mpc5200_dma.h"
 #include "mpc5200_psc_ac97.h"
-#include "../codecs/stac9766.h"
 
 #define DRV_NAME "efika-audio-fabric"
 
diff --git a/sound/soc/fsl/fsl-asoc-card.c b/sound/soc/fsl/fsl-asoc-card.c
index dffd549a0e2a..9998aea23597 100644
--- a/sound/soc/fsl/fsl-asoc-card.c
+++ b/sound/soc/fsl/fsl-asoc-card.c
@@ -183,7 +183,7 @@ static int fsl_asoc_card_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static struct snd_soc_ops fsl_asoc_card_ops = {
+static const struct snd_soc_ops fsl_asoc_card_ops = {
 	.hw_params = fsl_asoc_card_hw_params,
 };
 
diff --git a/sound/soc/fsl/imx-wm8962.c b/sound/soc/fsl/imx-wm8962.c
index 201a70d1027a..1b60958e2080 100644
--- a/sound/soc/fsl/imx-wm8962.c
+++ b/sound/soc/fsl/imx-wm8962.c
@@ -61,7 +61,7 @@ static int imx_hifi_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static struct snd_soc_ops imx_hifi_ops = {
+static const struct snd_soc_ops imx_hifi_ops = {
 	.hw_params = imx_hifi_hw_params,
 };
 
diff --git a/sound/soc/generic/simple-card-utils.c b/sound/soc/generic/simple-card-utils.c
index 1cb39309f5d5..cf026252cd4a 100644
--- a/sound/soc/generic/simple-card-utils.c
+++ b/sound/soc/generic/simple-card-utils.c
@@ -1,5 +1,5 @@
 /*
- * simple-card-core.c
+ * simple-card-utils.c
  *
  * Copyright (c) 2016 Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
  *
@@ -195,9 +195,6 @@ EXPORT_SYMBOL_GPL(asoc_simple_card_init_dai);
 
 int asoc_simple_card_canonicalize_dailink(struct snd_soc_dai_link *dai_link)
 {
-	if (!dai_link->cpu_dai_name || !dai_link->codec_dai_name)
-		return -EINVAL;
-
 	/* Assumes platform == cpu */
 	if (!dai_link->platform_of_node)
 		dai_link->platform_of_node = dai_link->cpu_of_node;
diff --git a/sound/soc/generic/simple-card.c b/sound/soc/generic/simple-card.c
index f608f8d23f3d..a385ff6bfa4b 100644
--- a/sound/soc/generic/simple-card.c
+++ b/sound/soc/generic/simple-card.c
@@ -174,7 +174,7 @@ err:
 	return ret;
 }
 
-static struct snd_soc_ops asoc_simple_card_ops = {
+static const struct snd_soc_ops asoc_simple_card_ops = {
 	.startup = asoc_simple_card_startup,
 	.shutdown = asoc_simple_card_shutdown,
 	.hw_params = asoc_simple_card_hw_params,
diff --git a/sound/soc/generic/simple-scu-card.c b/sound/soc/generic/simple-scu-card.c
index b9973a56bcb0..bb86ee042490 100644
--- a/sound/soc/generic/simple-scu-card.c
+++ b/sound/soc/generic/simple-scu-card.c
@@ -22,7 +22,7 @@
 #include <sound/soc-dai.h>
 #include <sound/simple_card_utils.h>
 
-struct asoc_simple_card_priv {
+struct simple_card_data {
 	struct snd_soc_card snd_card;
 	struct snd_soc_codec_conf codec_conf;
 	struct asoc_simple_dai *dai_props;
@@ -42,7 +42,7 @@ struct asoc_simple_card_priv {
 static int asoc_simple_card_startup(struct snd_pcm_substream *substream)
 {
 	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct asoc_simple_card_priv *priv =	snd_soc_card_get_drvdata(rtd->card);
+	struct simple_card_data *priv =	snd_soc_card_get_drvdata(rtd->card);
 	struct asoc_simple_dai *dai_props =
 		simple_priv_to_props(priv, rtd->num);
 
@@ -52,21 +52,21 @@ static int asoc_simple_card_startup(struct snd_pcm_substream *substream)
 static void asoc_simple_card_shutdown(struct snd_pcm_substream *substream)
 {
 	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct asoc_simple_card_priv *priv =	snd_soc_card_get_drvdata(rtd->card);
+	struct simple_card_data *priv =	snd_soc_card_get_drvdata(rtd->card);
 	struct asoc_simple_dai *dai_props =
 		simple_priv_to_props(priv, rtd->num);
 
 	clk_disable_unprepare(dai_props->clk);
 }
 
-static struct snd_soc_ops asoc_simple_card_ops = {
+static const struct snd_soc_ops asoc_simple_card_ops = {
 	.startup = asoc_simple_card_startup,
 	.shutdown = asoc_simple_card_shutdown,
 };
 
 static int asoc_simple_card_dai_init(struct snd_soc_pcm_runtime *rtd)
 {
-	struct asoc_simple_card_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+	struct simple_card_data *priv = snd_soc_card_get_drvdata(rtd->card);
 	struct snd_soc_dai *dai;
 	struct snd_soc_dai_link *dai_link;
 	struct asoc_simple_dai *dai_props;
@@ -84,7 +84,7 @@ static int asoc_simple_card_dai_init(struct snd_soc_pcm_runtime *rtd)
 static int asoc_simple_card_be_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
 					struct snd_pcm_hw_params *params)
 {
-	struct asoc_simple_card_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+	struct simple_card_data *priv = snd_soc_card_get_drvdata(rtd->card);
 	struct snd_interval *rate = hw_param_interval(params,
 						      SNDRV_PCM_HW_PARAM_RATE);
 	struct snd_interval *channels = hw_param_interval(params,
@@ -101,8 +101,8 @@ static int asoc_simple_card_be_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
 	return 0;
 }
 
-static int asoc_simple_card_parse_links(struct device_node *np,
-					struct asoc_simple_card_priv *priv,
+static int asoc_simple_card_dai_link_of(struct device_node *np,
+					struct simple_card_data *priv,
 					unsigned int daifmt,
 					int idx, bool is_fe)
 {
@@ -195,22 +195,35 @@ static int asoc_simple_card_parse_links(struct device_node *np,
 	return 0;
 }
 
-static int asoc_simple_card_dai_link_of(struct device_node *node,
-				 struct asoc_simple_card_priv *priv)
+static int asoc_simple_card_parse_of(struct device_node *node,
+				     struct simple_card_data *priv)
+
 {
 	struct device *dev = simple_priv_to_dev(priv);
 	struct device_node *np;
 	unsigned int daifmt = 0;
-	int ret, i;
 	bool is_fe;
+	int ret, i;
+
+	if (!node)
+		return -EINVAL;
+
+	ret = snd_soc_of_parse_audio_routing(&priv->snd_card, PREFIX "routing");
+	if (ret < 0)
+		return ret;
+
+	/* sampling rate convert */
+	of_property_read_u32(node, PREFIX "convert-rate", &priv->convert_rate);
+
+	/* channels transfer */
+	of_property_read_u32(node, PREFIX "convert-channels", &priv->convert_channels);
 
 	/* find 1st codec */
 	np = of_get_child_by_name(node, PREFIX "codec");
 	if (!np)
 		return -ENODEV;
 
-	ret = asoc_simple_card_parse_daifmt(dev, node, np,
-					    PREFIX, &daifmt);
+	ret = asoc_simple_card_parse_daifmt(dev, node, np, PREFIX, &daifmt);
 	if (ret < 0)
 		return ret;
 
@@ -220,58 +233,12 @@ static int asoc_simple_card_dai_link_of(struct device_node *node,
 		if (strcmp(np->name, PREFIX "cpu") == 0)
 			is_fe = true;
 
-		ret = asoc_simple_card_parse_links(np, priv, daifmt, i, is_fe);
+		ret = asoc_simple_card_dai_link_of(np, priv, daifmt, i, is_fe);
 		if (ret < 0)
 			return ret;
 		i++;
 	}
 
-	return 0;
-}
-
-static int asoc_simple_card_parse_of(struct device_node *node,
-			      struct asoc_simple_card_priv *priv,
-			      struct device *dev)
-{
-	struct asoc_simple_dai *props;
-	struct snd_soc_dai_link *links;
-	int ret;
-	int num;
-
-	if (!node)
-		return -EINVAL;
-
-	num = of_get_child_count(node);
-	props = devm_kzalloc(dev, sizeof(*props) * num, GFP_KERNEL);
-	links = devm_kzalloc(dev, sizeof(*links) * num, GFP_KERNEL);
-	if (!props || !links)
-		return -ENOMEM;
-
-	priv->dai_props	= props;
-	priv->dai_link	= links;
-
-	/* Init snd_soc_card */
-	priv->snd_card.owner			= THIS_MODULE;
-	priv->snd_card.dev			= dev;
-	priv->snd_card.dai_link			= priv->dai_link;
-	priv->snd_card.num_links		= num;
-	priv->snd_card.codec_conf		= &priv->codec_conf;
-	priv->snd_card.num_configs		= 1;
-
-	ret = snd_soc_of_parse_audio_routing(&priv->snd_card, PREFIX "routing");
-	if (ret < 0)
-		return ret;
-
-	/* sampling rate convert */
-	of_property_read_u32(node, PREFIX "convert-rate", &priv->convert_rate);
-
-	/* channels transfer */
-	of_property_read_u32(node, PREFIX "convert-channels", &priv->convert_channels);
-
-	ret = asoc_simple_card_dai_link_of(node, priv);
-	if (ret < 0)
-		return ret;
-
 	ret = asoc_simple_card_parse_card_name(&priv->snd_card, PREFIX);
 	if (ret < 0)
 		return ret;
@@ -286,17 +253,37 @@ static int asoc_simple_card_parse_of(struct device_node *node,
 
 static int asoc_simple_card_probe(struct platform_device *pdev)
 {
-	struct asoc_simple_card_priv *priv;
-	struct device_node *np = pdev->dev.of_node;
+	struct simple_card_data *priv;
+	struct snd_soc_dai_link *dai_link;
+	struct asoc_simple_dai *dai_props;
 	struct device *dev = &pdev->dev;
-	int ret;
+	struct device_node *np = pdev->dev.of_node;
+	int num, ret;
 
 	/* Allocate the private data */
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	ret = asoc_simple_card_parse_of(np, priv, dev);
+	num = of_get_child_count(np);
+
+	dai_props = devm_kzalloc(dev, sizeof(*dai_props) * num, GFP_KERNEL);
+	dai_link  = devm_kzalloc(dev, sizeof(*dai_link)  * num, GFP_KERNEL);
+	if (!dai_props || !dai_link)
+		return -ENOMEM;
+
+	priv->dai_props				= dai_props;
+	priv->dai_link				= dai_link;
+
+	/* Init snd_soc_card */
+	priv->snd_card.owner			= THIS_MODULE;
+	priv->snd_card.dev			= dev;
+	priv->snd_card.dai_link			= priv->dai_link;
+	priv->snd_card.num_links		= num;
+	priv->snd_card.codec_conf		= &priv->codec_conf;
+	priv->snd_card.num_configs		= 1;
+
+	ret = asoc_simple_card_parse_of(np, priv);
 	if (ret < 0) {
 		if (ret != -EPROBE_DEFER)
 			dev_err(dev, "parse error %d\n", ret);
diff --git a/sound/soc/intel/atom/sst-atom-controls.c b/sound/soc/intel/atom/sst-atom-controls.c
index 0838478c4c3f..c7b3cbf92faf 100644
--- a/sound/soc/intel/atom/sst-atom-controls.c
+++ b/sound/soc/intel/atom/sst-atom-controls.c
@@ -937,7 +937,7 @@ int send_ssp_cmd(struct snd_soc_dai *dai, const char *id, bool enable)
 	struct sst_data *drv = snd_soc_dai_get_drvdata(dai);
 	int ssp_id;
 
-	dev_info(dai->dev, "Enter: enable=%d port_name=%s\n", enable, id);
+	dev_dbg(dai->dev, "Enter: enable=%d port_name=%s\n", enable, id);
 
 	if (strcmp(id, "ssp0-port") == 0)
 		ssp_id = SSP_MODEM;
diff --git a/sound/soc/intel/atom/sst-mfld-platform-pcm.c b/sound/soc/intel/atom/sst-mfld-platform-pcm.c
index 25c6d87c818e..f5a8050351b5 100644
--- a/sound/soc/intel/atom/sst-mfld-platform-pcm.c
+++ b/sound/soc/intel/atom/sst-mfld-platform-pcm.c
@@ -771,6 +771,9 @@ static int sst_soc_prepare(struct device *dev)
 	struct sst_data *drv = dev_get_drvdata(dev);
 	struct snd_soc_pcm_runtime *rtd;
 
+	if (!drv->soc_card)
+		return 0;
+
 	/* suspend all pcms first */
 	snd_soc_suspend(drv->soc_card->dev);
 	snd_soc_poweroff(drv->soc_card->dev);
@@ -793,6 +796,9 @@ static void sst_soc_complete(struct device *dev)
 	struct sst_data *drv = dev_get_drvdata(dev);
 	struct snd_soc_pcm_runtime *rtd;
 
+	if (!drv->soc_card)
+		return;
+
 	/* restart SSPs */
 	list_for_each_entry(rtd, &drv->soc_card->rtd_list, list) {
 		struct snd_soc_dai *dai = rtd->cpu_dai;
diff --git a/sound/soc/intel/atom/sst/sst.c b/sound/soc/intel/atom/sst/sst.c
index 9b6e27385dc9..f9ba71315e33 100644
--- a/sound/soc/intel/atom/sst/sst.c
+++ b/sound/soc/intel/atom/sst/sst.c
@@ -27,6 +27,7 @@
 #include <linux/pm_qos.h>
 #include <linux/async.h>
 #include <linux/acpi.h>
+#include <linux/sysfs.h>
 #include <sound/core.h>
 #include <sound/soc.h>
 #include <asm/platform_sst_audio.h>
@@ -242,6 +243,32 @@ int sst_alloc_drv_context(struct intel_sst_drv **ctx,
 }
 EXPORT_SYMBOL_GPL(sst_alloc_drv_context);
 
+static ssize_t firmware_version_show(struct device *dev,
+			    struct device_attribute *attr, char *buf)
+{
+	struct intel_sst_drv *ctx = dev_get_drvdata(dev);
+
+	if (ctx->fw_version.type == 0 && ctx->fw_version.major == 0 &&
+	    ctx->fw_version.minor == 0 && ctx->fw_version.build == 0)
+		return sprintf(buf, "FW not yet loaded\n");
+	else
+		return sprintf(buf, "v%02x.%02x.%02x.%02x\n",
+			       ctx->fw_version.type, ctx->fw_version.major,
+			       ctx->fw_version.minor, ctx->fw_version.build);
+
+}
+
+DEVICE_ATTR_RO(firmware_version);
+
+static const struct attribute *sst_fw_version_attrs[] = {
+	&dev_attr_firmware_version.attr,
+	NULL,
+};
+
+static const struct attribute_group sst_fw_version_attr_group = {
+	.attrs = (struct attribute **)sst_fw_version_attrs,
+};
+
 int sst_context_init(struct intel_sst_drv *ctx)
 {
 	int ret = 0, i;
@@ -315,8 +342,19 @@ int sst_context_init(struct intel_sst_drv *ctx)
 		dev_err(ctx->dev, "Firmware download failed:%d\n", ret);
 		goto do_free_mem;
 	}
+
+	ret = sysfs_create_group(&ctx->dev->kobj,
+				 &sst_fw_version_attr_group);
+	if (ret) {
+		dev_err(ctx->dev,
+			"Unable to create sysfs\n");
+		goto err_sysfs;
+	}
+
 	sst_register(ctx->dev);
 	return 0;
+err_sysfs:
+	sysfs_remove_group(&ctx->dev->kobj, &sst_fw_version_attr_group);
 
 do_free_mem:
 	destroy_workqueue(ctx->post_msg_wq);
@@ -330,6 +368,7 @@ void sst_context_cleanup(struct intel_sst_drv *ctx)
 	pm_runtime_disable(ctx->dev);
 	sst_unregister(ctx->dev);
 	sst_set_fw_state_locked(ctx, SST_SHUTDOWN);
+	sysfs_remove_group(&ctx->dev->kobj, &sst_fw_version_attr_group);
 	flush_scheduled_work();
 	destroy_workqueue(ctx->post_msg_wq);
 	pm_qos_remove_request(ctx->qos);
diff --git a/sound/soc/intel/atom/sst/sst.h b/sound/soc/intel/atom/sst/sst.h
index 3f493862e98d..5c9a51cc77aa 100644
--- a/sound/soc/intel/atom/sst/sst.h
+++ b/sound/soc/intel/atom/sst/sst.h
@@ -436,6 +436,7 @@ struct intel_sst_drv {
 	 */
 	char firmware_name[FW_NAME_SIZE];
 
+	struct snd_sst_fw_version fw_version;
 	struct sst_fw_save	*fw_save;
 };
 
diff --git a/sound/soc/intel/atom/sst/sst_acpi.c b/sound/soc/intel/atom/sst/sst_acpi.c
index 0a88537ca58a..f4d92bbc5373 100644
--- a/sound/soc/intel/atom/sst/sst_acpi.c
+++ b/sound/soc/intel/atom/sst/sst_acpi.c
@@ -452,6 +452,8 @@ static struct sst_acpi_mach sst_acpi_bytcr[] = {
 static struct sst_acpi_mach sst_acpi_chv[] = {
 	{"10EC5670", "cht-bsw-rt5672", "intel/fw_sst_22a8.bin", "cht-bsw", NULL,
 						&chv_platform_data },
+	{"10EC5672", "cht-bsw-rt5672", "intel/fw_sst_22a8.bin", "cht-bsw", NULL,
+						&chv_platform_data },
 	{"10EC5645", "cht-bsw-rt5645", "intel/fw_sst_22a8.bin", "cht-bsw", NULL,
 						&chv_platform_data },
 	{"10EC5650", "cht-bsw-rt5645", "intel/fw_sst_22a8.bin", "cht-bsw", NULL,
diff --git a/sound/soc/intel/atom/sst/sst_ipc.c b/sound/soc/intel/atom/sst/sst_ipc.c
index bfc889950bb2..374bb61c596d 100644
--- a/sound/soc/intel/atom/sst/sst_ipc.c
+++ b/sound/soc/intel/atom/sst/sst_ipc.c
@@ -236,6 +236,17 @@ static void process_fw_init(struct intel_sst_drv *sst_drv_ctx,
 		retval = init->result;
 		goto ret;
 	}
+	dev_info(sst_drv_ctx->dev, "FW Version %02x.%02x.%02x.%02x\n",
+			init->fw_version.type, init->fw_version.major,
+			init->fw_version.minor, init->fw_version.build);
+	dev_dbg(sst_drv_ctx->dev, "Build date %s Time %s\n",
+			init->build_info.date, init->build_info.time);
+
+	/* Save FW version */
+	sst_drv_ctx->fw_version.type = init->fw_version.type;
+	sst_drv_ctx->fw_version.major = init->fw_version.major;
+	sst_drv_ctx->fw_version.minor = init->fw_version.minor;
+	sst_drv_ctx->fw_version.build = init->fw_version.build;
 
 ret:
 	sst_wake_up_block(sst_drv_ctx, retval, FW_DWNL_ID, 0 , NULL, 0);
diff --git a/sound/soc/intel/atom/sst/sst_stream.c b/sound/soc/intel/atom/sst/sst_stream.c
index 4ccc80e5e8cc..51bdeeecb7c8 100644
--- a/sound/soc/intel/atom/sst/sst_stream.c
+++ b/sound/soc/intel/atom/sst/sst_stream.c
@@ -104,7 +104,7 @@ int sst_alloc_stream_mrfld(struct intel_sst_drv *sst_drv_ctx, void *params)
 	sst_init_stream(&sst_drv_ctx->streams[str_id], alloc_param.codec_type,
 			str_id, alloc_param.operation, 0);
 
-	dev_info(sst_drv_ctx->dev, "Alloc for str %d pipe %#x\n",
+	dev_dbg(sst_drv_ctx->dev, "Alloc for str %d pipe %#x\n",
 			str_id, pipe_id);
 	ret = sst_prepare_and_post_msg(sst_drv_ctx, task_id, IPC_CMD,
 			IPC_IA_ALLOC_STREAM_MRFLD, pipe_id, sizeof(alloc_param),
@@ -415,7 +415,7 @@ int sst_free_stream(struct intel_sst_drv *sst_drv_ctx, int str_id)
 		str_info->status = STREAM_UN_INIT;
 		mutex_unlock(&str_info->lock);
 
-		dev_info(sst_drv_ctx->dev, "Free for str %d pipe %#x\n",
+		dev_dbg(sst_drv_ctx->dev, "Free for str %d pipe %#x\n",
 				str_id, str_info->pipe_id);
 		retval = sst_prepare_and_post_msg(sst_drv_ctx, str_info->task_id, IPC_CMD,
 				IPC_IA_FREE_STREAM_MRFLD, str_info->pipe_id, 0,
diff --git a/sound/soc/intel/baytrail/sst-baytrail-ipc.c b/sound/soc/intel/baytrail/sst-baytrail-ipc.c
index 7ab14ce65a73..260447da32b8 100644
--- a/sound/soc/intel/baytrail/sst-baytrail-ipc.c
+++ b/sound/soc/intel/baytrail/sst-baytrail-ipc.c
@@ -23,7 +23,6 @@
 #include <linux/slab.h>
 #include <linux/delay.h>
 #include <linux/platform_device.h>
-#include <linux/kthread.h>
 #include <linux/firmware.h>
 #include <linux/io.h>
 #include <asm/div64.h>
@@ -338,7 +337,7 @@ static irqreturn_t sst_byt_irq_thread(int irq, void *context)
 	spin_unlock_irqrestore(&sst->spinlock, flags);
 
 	/* continue to send any remaining messages... */
-	kthread_queue_work(&ipc->kworker, &ipc->kwork);
+	schedule_work(&ipc->kwork);
 
 	return IRQ_HANDLED;
 }
diff --git a/sound/soc/intel/boards/bdw-rt5677.c b/sound/soc/intel/boards/bdw-rt5677.c
index 547e6705bf6d..53c6b4cbb1e1 100644
--- a/sound/soc/intel/boards/bdw-rt5677.c
+++ b/sound/soc/intel/boards/bdw-rt5677.c
@@ -156,7 +156,7 @@ static int bdw_rt5677_hw_params(struct snd_pcm_substream *substream,
 	return ret;
 }
 
-static struct snd_soc_ops bdw_rt5677_ops = {
+static const struct snd_soc_ops bdw_rt5677_ops = {
 	.hw_params = bdw_rt5677_hw_params,
 };
 
diff --git a/sound/soc/intel/boards/broadwell.c b/sound/soc/intel/boards/broadwell.c
index 7486a0022fde..4d7e9decfa92 100644
--- a/sound/soc/intel/boards/broadwell.c
+++ b/sound/soc/intel/boards/broadwell.c
@@ -126,7 +126,7 @@ static int broadwell_rt286_hw_params(struct snd_pcm_substream *substream,
 	return ret;
 }
 
-static struct snd_soc_ops broadwell_rt286_ops = {
+static const struct snd_soc_ops broadwell_rt286_ops = {
 	.hw_params = broadwell_rt286_hw_params,
 };
 
@@ -220,10 +220,12 @@ static struct snd_soc_dai_link broadwell_rt286_dais[] = {
 };
 
 static int broadwell_suspend(struct snd_soc_card *card){
-	struct snd_soc_codec *codec;
+	struct snd_soc_component *component;
+
+	list_for_each_entry(component, &card->component_dev_list, card_list) {
+		if (!strcmp(component->name, "i2c-INT343A:00")) {
+			struct snd_soc_codec *codec = snd_soc_component_to_codec(component);
 
-	list_for_each_entry(codec, &card->codec_dev_list, card_list) {
-		if (!strcmp(codec->component.name, "i2c-INT343A:00")) {
 			dev_dbg(codec->dev, "disabling jack detect before going to suspend.\n");
 			rt286_mic_detect(codec, NULL);
 			break;
@@ -233,10 +235,12 @@ static int broadwell_suspend(struct snd_soc_card *card){
 }
 
 static int broadwell_resume(struct snd_soc_card *card){
-	struct snd_soc_codec *codec;
+	struct snd_soc_component *component;
+
+	list_for_each_entry(component, &card->component_dev_list, card_list) {
+		if (!strcmp(component->name, "i2c-INT343A:00")) {
+			struct snd_soc_codec *codec = snd_soc_component_to_codec(component);
 
-	list_for_each_entry(codec, &card->codec_dev_list, card_list) {
-		if (!strcmp(codec->component.name, "i2c-INT343A:00")) {
 			dev_dbg(codec->dev, "enabling jack detect for resume.\n");
 			rt286_mic_detect(codec, &broadwell_headset);
 			break;
diff --git a/sound/soc/intel/boards/bxt_da7219_max98357a.c b/sound/soc/intel/boards/bxt_da7219_max98357a.c
index 865a21e557cc..1b4330cd2739 100644
--- a/sound/soc/intel/boards/bxt_da7219_max98357a.c
+++ b/sound/soc/intel/boards/bxt_da7219_max98357a.c
@@ -30,6 +30,7 @@
 #define BXT_DIALOG_CODEC_DAI	"da7219-hifi"
 #define BXT_MAXIM_CODEC_DAI	"HiFi"
 #define DUAL_CHANNEL		2
+#define QUAD_CHANNEL		4
 
 static struct snd_soc_jack broxton_headset;
 
@@ -182,6 +183,16 @@ static struct snd_pcm_hw_constraint_list constraints_channels = {
 	.mask = 0,
 };
 
+static unsigned int channels_quad[] = {
+	QUAD_CHANNEL,
+};
+
+static struct snd_pcm_hw_constraint_list constraints_channels_quad = {
+	.count = ARRAY_SIZE(channels_quad),
+	.list = channels_quad,
+	.mask = 0,
+};
+
 static int bxt_fe_startup(struct snd_pcm_substream *substream)
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
@@ -248,7 +259,7 @@ static int broxton_da7219_hw_free(struct snd_pcm_substream *substream)
 	return ret;
 }
 
-static struct snd_soc_ops broxton_da7219_ops = {
+static const struct snd_soc_ops broxton_da7219_ops = {
 	.hw_params = broxton_da7219_hw_params,
 	.hw_free = broxton_da7219_hw_free,
 };
@@ -258,7 +269,10 @@ static int broxton_dmic_fixup(struct snd_soc_pcm_runtime *rtd,
 {
 	struct snd_interval *channels = hw_param_interval(params,
 						SNDRV_PCM_HW_PARAM_CHANNELS);
-	channels->min = channels->max = DUAL_CHANNEL;
+	if (params_channels(params) == 2)
+		channels->min = channels->max = 2;
+	else
+		channels->min = channels->max = 4;
 
 	return 0;
 }
@@ -267,9 +281,9 @@ static int broxton_dmic_startup(struct snd_pcm_substream *substream)
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
 
-	runtime->hw.channels_max = DUAL_CHANNEL;
+	runtime->hw.channels_min = runtime->hw.channels_max = QUAD_CHANNEL;
 	snd_pcm_hw_constraint_list(runtime, 0, SNDRV_PCM_HW_PARAM_CHANNELS,
-			&constraints_channels);
+			&constraints_channels_quad);
 
 	return snd_pcm_hw_constraint_list(substream->runtime, 0,
 			SNDRV_PCM_HW_PARAM_RATE, &constraints_rates);
@@ -295,7 +309,7 @@ static int broxton_refcap_startup(struct snd_pcm_substream *substream)
 			&constraints_16000);
 };
 
-static struct snd_soc_ops broxton_refcap_ops = {
+static const struct snd_soc_ops broxton_refcap_ops = {
 	.startup = broxton_refcap_startup,
 };
 
@@ -348,7 +362,7 @@ static struct snd_soc_dai_link broxton_dais[] = {
 		.dynamic = 1,
 		.ops = &broxton_refcap_ops,
 	},
-	[BXT_DPCM_AUDIO_DMIC_CP]
+	[BXT_DPCM_AUDIO_DMIC_CP] =
 	{
 		.name = "Bxt Audio DMIC cap",
 		.stream_name = "dmiccap",
diff --git a/sound/soc/intel/boards/bxt_rt298.c b/sound/soc/intel/boards/bxt_rt298.c
index d610bdca1608..1309405b3808 100644
--- a/sound/soc/intel/boards/bxt_rt298.c
+++ b/sound/soc/intel/boards/bxt_rt298.c
@@ -181,7 +181,7 @@ static int broxton_rt298_hw_params(struct snd_pcm_substream *substream,
 	return ret;
 }
 
-static struct snd_soc_ops broxton_rt298_ops = {
+static const struct snd_soc_ops broxton_rt298_ops = {
 	.hw_params = broxton_rt298_hw_params,
 };
 
@@ -230,7 +230,7 @@ static int broxton_dmic_startup(struct snd_pcm_substream *substream)
 				SNDRV_PCM_HW_PARAM_RATE, &constraints_rates);
 }
 
-static struct snd_soc_ops broxton_dmic_ops = {
+static const struct snd_soc_ops broxton_dmic_ops = {
 	.startup = broxton_dmic_startup,
 };
 
diff --git a/sound/soc/intel/boards/bytcr_rt5640.c b/sound/soc/intel/boards/bytcr_rt5640.c
index bff77a1f27fc..507a86a5eafe 100644
--- a/sound/soc/intel/boards/bytcr_rt5640.c
+++ b/sound/soc/intel/boards/bytcr_rt5640.c
@@ -57,9 +57,7 @@ struct byt_rt5640_private {
 	struct clk *mclk;
 };
 
-static unsigned long byt_rt5640_quirk = BYT_RT5640_DMIC1_MAP |
-					BYT_RT5640_DMIC_EN |
-					BYT_RT5640_MCLK_EN;
+static unsigned long byt_rt5640_quirk = BYT_RT5640_MCLK_EN;
 
 static void log_quirks(struct device *dev)
 {
@@ -597,11 +595,11 @@ static int byt_rt5640_aif1_startup(struct snd_pcm_substream *substream)
 			SNDRV_PCM_HW_PARAM_RATE, 48000);
 }
 
-static struct snd_soc_ops byt_rt5640_aif1_ops = {
+static const struct snd_soc_ops byt_rt5640_aif1_ops = {
 	.startup = byt_rt5640_aif1_startup,
 };
 
-static struct snd_soc_ops byt_rt5640_be_ssp2_ops = {
+static const struct snd_soc_ops byt_rt5640_be_ssp2_ops = {
 	.hw_params = byt_rt5640_aif1_hw_params,
 };
 
@@ -689,6 +687,10 @@ static bool is_valleyview(void)
 	return true;
 }
 
+struct acpi_chan_package {   /* ACPICA seems to require 64 bit integers */
+	u64 aif_value;       /* 1: AIF1, 2: AIF2 */
+	u64 mclock_value;    /* usually 25MHz (0x17d7940), ignored */
+};
 
 static int snd_byt_rt5640_mc_probe(struct platform_device *pdev)
 {
@@ -698,6 +700,7 @@ static int snd_byt_rt5640_mc_probe(struct platform_device *pdev)
 	int i;
 	int dai_index;
 	struct byt_rt5640_private *priv;
+	bool is_bytcr = false;
 
 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_ATOMIC);
 	if (!priv)
@@ -734,10 +737,61 @@ static int snd_byt_rt5640_mc_probe(struct platform_device *pdev)
 		struct sst_platform_info *p_info = mach->pdata;
 		const struct sst_res_info *res_info = p_info->res_info;
 
-		/* TODO: use CHAN package info from BIOS to detect AIF1/AIF2 */
-		if (res_info->acpi_ipc_irq_index == 0) {
+		if (res_info->acpi_ipc_irq_index == 0)
+			is_bytcr = true;
+	}
+
+	if (is_bytcr) {
+		/*
+		 * Baytrail CR platforms may have CHAN package in BIOS, try
+		 * to find relevant routing quirk based as done on Windows
+		 * platforms. We have to read the information directly from the
+		 * BIOS, at this stage the card is not created and the links
+		 * with the codec driver/pdata are non-existent
+		 */
+
+		struct acpi_chan_package chan_package;
+
+		/* format specified: 2 64-bit integers */
+		struct acpi_buffer format = {sizeof("NN"), "NN"};
+		struct acpi_buffer state = {0, NULL};
+		struct sst_acpi_package_context pkg_ctx;
+		bool pkg_found = false;
+
+		state.length = sizeof(chan_package);
+		state.pointer = &chan_package;
+
+		pkg_ctx.name = "CHAN";
+		pkg_ctx.length = 2;
+		pkg_ctx.format = &format;
+		pkg_ctx.state = &state;
+		pkg_ctx.data_valid = false;
+
+		pkg_found = sst_acpi_find_package_from_hid(mach->id, &pkg_ctx);
+		if (pkg_found) {
+			if (chan_package.aif_value == 1) {
+				dev_info(&pdev->dev, "BIOS Routing: AIF1 connected\n");
+				byt_rt5640_quirk |= BYT_RT5640_SSP0_AIF1;
+			} else  if (chan_package.aif_value == 2) {
+				dev_info(&pdev->dev, "BIOS Routing: AIF2 connected\n");
+				byt_rt5640_quirk |= BYT_RT5640_SSP0_AIF2;
+			} else {
+				dev_info(&pdev->dev, "BIOS Routing isn't valid, ignored\n");
+				pkg_found = false;
+			}
+		}
+
+		if (!pkg_found) {
+			/* no BIOS indications, assume SSP0-AIF2 connection */
 			byt_rt5640_quirk |= BYT_RT5640_SSP0_AIF2;
 		}
+
+		/* change defaults for Baytrail-CR capture */
+		byt_rt5640_quirk |= BYT_RT5640_IN1_MAP;
+		byt_rt5640_quirk |= BYT_RT5640_DIFF_MIC;
+	} else {
+		byt_rt5640_quirk |= (BYT_RT5640_DMIC1_MAP |
+				BYT_RT5640_DMIC_EN);
 	}
 
 	/* check quirks before creating card */
diff --git a/sound/soc/intel/boards/bytcr_rt5651.c b/sound/soc/intel/boards/bytcr_rt5651.c
index 35f591eab3c9..2d24dc04b597 100644
--- a/sound/soc/intel/boards/bytcr_rt5651.c
+++ b/sound/soc/intel/boards/bytcr_rt5651.c
@@ -219,11 +219,11 @@ static int byt_rt5651_aif1_startup(struct snd_pcm_substream *substream)
 			&constraints_48000);
 }
 
-static struct snd_soc_ops byt_rt5651_aif1_ops = {
+static const struct snd_soc_ops byt_rt5651_aif1_ops = {
 	.startup = byt_rt5651_aif1_startup,
 };
 
-static struct snd_soc_ops byt_rt5651_be_ssp2_ops = {
+static const struct snd_soc_ops byt_rt5651_be_ssp2_ops = {
 	.hw_params = byt_rt5651_aif1_hw_params,
 };
 
diff --git a/sound/soc/intel/boards/cht_bsw_max98090_ti.c b/sound/soc/intel/boards/cht_bsw_max98090_ti.c
index cdcced9f32b6..742bc0d4e681 100644
--- a/sound/soc/intel/boards/cht_bsw_max98090_ti.c
+++ b/sound/soc/intel/boards/cht_bsw_max98090_ti.c
@@ -204,11 +204,11 @@ static int cht_max98090_headset_init(struct snd_soc_component *component)
 	return ts3a227e_enable_jack_detect(component, &ctx->jack);
 }
 
-static struct snd_soc_ops cht_aif1_ops = {
+static const struct snd_soc_ops cht_aif1_ops = {
 	.startup = cht_aif1_startup,
 };
 
-static struct snd_soc_ops cht_be_ssp2_ops = {
+static const struct snd_soc_ops cht_be_ssp2_ops = {
 	.hw_params = cht_aif1_hw_params,
 };
 
diff --git a/sound/soc/intel/boards/cht_bsw_rt5645.c b/sound/soc/intel/boards/cht_bsw_rt5645.c
index 56056ed7fcfd..f504a0e18f91 100644
--- a/sound/soc/intel/boards/cht_bsw_rt5645.c
+++ b/sound/soc/intel/boards/cht_bsw_rt5645.c
@@ -44,6 +44,7 @@ struct cht_acpi_card {
 struct cht_mc_private {
 	struct snd_soc_jack jack;
 	struct cht_acpi_card *acpi_card;
+	char codec_name[16];
 };
 
 static inline struct snd_soc_dai *cht_get_codec_dai(struct snd_soc_card *card)
@@ -250,11 +251,11 @@ static int cht_aif1_startup(struct snd_pcm_substream *substream)
 			SNDRV_PCM_HW_PARAM_RATE, 48000);
 }
 
-static struct snd_soc_ops cht_aif1_ops = {
+static const struct snd_soc_ops cht_aif1_ops = {
 	.startup = cht_aif1_startup,
 };
 
-static struct snd_soc_ops cht_be_ssp2_ops = {
+static const struct snd_soc_ops cht_be_ssp2_ops = {
 	.hw_params = cht_aif1_hw_params,
 };
 
@@ -354,7 +355,6 @@ static int snd_cht_mc_probe(struct platform_device *pdev)
 	int i;
 	struct cht_mc_private *drv;
 	struct snd_soc_card *card = snd_soc_cards[0].soc_card;
-	char codec_name[16];
 	struct sst_acpi_mach *mach;
 	const char *i2c_name = NULL;
 	int dai_index = 0;
@@ -374,12 +374,12 @@ static int snd_cht_mc_probe(struct platform_device *pdev)
 	}
 	card->dev = &pdev->dev;
 	mach = card->dev->platform_data;
-	sprintf(codec_name, "i2c-%s:00", drv->acpi_card->codec_id);
+	sprintf(drv->codec_name, "i2c-%s:00", drv->acpi_card->codec_id);
 
 	/* set correct codec name */
 	for (i = 0; i < ARRAY_SIZE(cht_dailink); i++)
 		if (!strcmp(card->dai_link[i].codec_name, "i2c-10EC5645:00")) {
-			card->dai_link[i].codec_name = kstrdup(codec_name, GFP_KERNEL);
+			card->dai_link[i].codec_name = drv->codec_name;
 			dai_index = i;
 		}
 
diff --git a/sound/soc/intel/boards/cht_bsw_rt5672.c b/sound/soc/intel/boards/cht_bsw_rt5672.c
index df9d254baa18..e4d46d4360d7 100644
--- a/sound/soc/intel/boards/cht_bsw_rt5672.c
+++ b/sound/soc/intel/boards/cht_bsw_rt5672.c
@@ -25,12 +25,14 @@
 #include <sound/jack.h>
 #include "../../codecs/rt5670.h"
 #include "../atom/sst-atom-controls.h"
+#include "../common/sst-acpi.h"
 
 /* The platform clock #3 outputs 19.2Mhz clock to codec as I2S MCLK */
 #define CHT_PLAT_CLK_3_HZ	19200000
 #define CHT_CODEC_DAI	"rt5670-aif1"
 
 static struct snd_soc_jack cht_bsw_headset;
+static char cht_bsw_codec_name[16];
 
 /* Headset jack detection DAPM pins */
 static struct snd_soc_jack_pin cht_bsw_headset_pins[] = {
@@ -225,11 +227,11 @@ static int cht_aif1_startup(struct snd_pcm_substream *substream)
 			SNDRV_PCM_HW_PARAM_RATE, 48000);
 }
 
-static struct snd_soc_ops cht_aif1_ops = {
+static const struct snd_soc_ops cht_aif1_ops = {
 	.startup = cht_aif1_startup,
 };
 
-static struct snd_soc_ops cht_be_ssp2_ops = {
+static const struct snd_soc_ops cht_be_ssp2_ops = {
 	.hw_params = cht_aif1_hw_params,
 };
 
@@ -292,10 +294,12 @@ static struct snd_soc_dai_link cht_dailink[] = {
 
 static int cht_suspend_pre(struct snd_soc_card *card)
 {
-	struct snd_soc_codec *codec;
+	struct snd_soc_component *component;
+
+	list_for_each_entry(component, &card->component_dev_list, card_list) {
+		if (!strcmp(component->name, "i2c-10EC5670:00")) {
+			struct snd_soc_codec *codec = snd_soc_component_to_codec(component);
 
-	list_for_each_entry(codec, &card->codec_dev_list, card_list) {
-		if (!strcmp(codec->component.name, "i2c-10EC5670:00")) {
 			dev_dbg(codec->dev, "disabling jack detect before going to suspend.\n");
 			rt5670_jack_suspend(codec);
 			break;
@@ -306,10 +310,12 @@ static int cht_suspend_pre(struct snd_soc_card *card)
 
 static int cht_resume_post(struct snd_soc_card *card)
 {
-	struct snd_soc_codec *codec;
+	struct snd_soc_component *component;
+
+	list_for_each_entry(component, &card->component_dev_list, card_list) {
+		if (!strcmp(component->name, "i2c-10EC5670:00")) {
+			struct snd_soc_codec *codec = snd_soc_component_to_codec(component);
 
-	list_for_each_entry(codec, &card->codec_dev_list, card_list) {
-		if (!strcmp(codec->component.name, "i2c-10EC5670:00")) {
 			dev_dbg(codec->dev, "enabling jack detect for resume.\n");
 			rt5670_jack_resume(codec);
 			break;
@@ -335,9 +341,33 @@ static struct snd_soc_card snd_soc_card_cht = {
 	.resume_post = cht_resume_post,
 };
 
+#define RT5672_I2C_DEFAULT	"i2c-10EC5670:00"
+
 static int snd_cht_mc_probe(struct platform_device *pdev)
 {
 	int ret_val = 0;
+	struct sst_acpi_mach *mach = pdev->dev.platform_data;
+	const char *i2c_name;
+	int i;
+
+	strcpy(cht_bsw_codec_name, RT5672_I2C_DEFAULT);
+
+	/* fixup codec name based on HID */
+	if (mach) {
+		i2c_name = sst_acpi_find_name_from_hid(mach->id);
+		if (i2c_name) {
+			snprintf(cht_bsw_codec_name, sizeof(cht_bsw_codec_name),
+				 "i2c-%s", i2c_name);
+			for (i = 0; i < ARRAY_SIZE(cht_dailink); i++) {
+				if (!strcmp(cht_dailink[i].codec_name,
+					    RT5672_I2C_DEFAULT)) {
+					cht_dailink[i].codec_name =
+						cht_bsw_codec_name;
+					break;
+				}
+			}
+		}
+	}
 
 	/* register the soc card */
 	snd_soc_card_cht.dev = &pdev->dev;
diff --git a/sound/soc/intel/boards/haswell.c b/sound/soc/intel/boards/haswell.c
index 863f1d5e2a2c..5e1ea0371c90 100644
--- a/sound/soc/intel/boards/haswell.c
+++ b/sound/soc/intel/boards/haswell.c
@@ -81,7 +81,7 @@ static int haswell_rt5640_hw_params(struct snd_pcm_substream *substream,
 	return ret;
 }
 
-static struct snd_soc_ops haswell_rt5640_ops = {
+static const struct snd_soc_ops haswell_rt5640_ops = {
 	.hw_params = haswell_rt5640_hw_params,
 };
 
diff --git a/sound/soc/intel/boards/mfld_machine.c b/sound/soc/intel/boards/mfld_machine.c
index 34f46c72a0e2..4e08885f37aa 100644
--- a/sound/soc/intel/boards/mfld_machine.c
+++ b/sound/soc/intel/boards/mfld_machine.c
@@ -81,9 +81,9 @@ static struct snd_soc_jack_zone mfld_zones[] = {
 };
 
 /* sound card controls */
-static const char *headset_switch_text[] = {"Earpiece", "Headset"};
+static const char * const headset_switch_text[] = {"Earpiece", "Headset"};
 
-static const char *lo_text[] = {"Vibra", "Headset", "IHF", "None"};
+static const char * const lo_text[] = {"Vibra", "Headset", "IHF", "None"};
 
 static const struct soc_enum headset_enum =
 	SOC_ENUM_SINGLE_EXT(2, headset_switch_text);
diff --git a/sound/soc/intel/boards/skl_nau88l25_max98357a.c b/sound/soc/intel/boards/skl_nau88l25_max98357a.c
index 25db5be7fdfa..fddd1cd12f13 100644
--- a/sound/soc/intel/boards/skl_nau88l25_max98357a.c
+++ b/sound/soc/intel/boards/skl_nau88l25_max98357a.c
@@ -332,7 +332,7 @@ static int skylake_nau8825_hw_params(struct snd_pcm_substream *substream,
 	return ret;
 }
 
-static struct snd_soc_ops skylake_nau8825_ops = {
+static const struct snd_soc_ops skylake_nau8825_ops = {
 	.hw_params = skylake_nau8825_hw_params,
 };
 
@@ -382,7 +382,7 @@ static int skylake_dmic_startup(struct snd_pcm_substream *substream)
 			SNDRV_PCM_HW_PARAM_RATE, &constraints_rates);
 }
 
-static struct snd_soc_ops skylake_dmic_ops = {
+static const struct snd_soc_ops skylake_dmic_ops = {
 	.startup = skylake_dmic_startup,
 };
 
@@ -416,7 +416,7 @@ static int skylake_refcap_startup(struct snd_pcm_substream *substream)
 				&constraints_16000);
 }
 
-static struct snd_soc_ops skylaye_refcap_ops = {
+static const struct snd_soc_ops skylaye_refcap_ops = {
 	.startup = skylake_refcap_startup,
 };
 
diff --git a/sound/soc/intel/boards/skl_nau88l25_ssm4567.c b/sound/soc/intel/boards/skl_nau88l25_ssm4567.c
index 69c5d5da4e86..8ab865ee0cad 100644
--- a/sound/soc/intel/boards/skl_nau88l25_ssm4567.c
+++ b/sound/soc/intel/boards/skl_nau88l25_ssm4567.c
@@ -394,7 +394,7 @@ static int skylake_nau8825_hw_params(struct snd_pcm_substream *substream,
 	return ret;
 }
 
-static struct snd_soc_ops skylake_nau8825_ops = {
+static const struct snd_soc_ops skylake_nau8825_ops = {
 	.hw_params = skylake_nau8825_hw_params,
 };
 
@@ -430,7 +430,7 @@ static int skylake_dmic_startup(struct snd_pcm_substream *substream)
 			SNDRV_PCM_HW_PARAM_RATE, &constraints_rates);
 }
 
-static struct snd_soc_ops skylake_dmic_ops = {
+static const struct snd_soc_ops skylake_dmic_ops = {
 	.startup = skylake_dmic_startup,
 };
 
@@ -464,7 +464,7 @@ static int skylake_refcap_startup(struct snd_pcm_substream *substream)
 			&constraints_16000);
 }
 
-static struct snd_soc_ops skylaye_refcap_ops = {
+static const struct snd_soc_ops skylaye_refcap_ops = {
 	.startup = skylake_refcap_startup,
 };
 
diff --git a/sound/soc/intel/boards/skl_rt286.c b/sound/soc/intel/boards/skl_rt286.c
index 88c61e8cb87f..dc5c3611a6ff 100644
--- a/sound/soc/intel/boards/skl_rt286.c
+++ b/sound/soc/intel/boards/skl_rt286.c
@@ -250,7 +250,7 @@ static int skylake_rt286_hw_params(struct snd_pcm_substream *substream,
 	return ret;
 }
 
-static struct snd_soc_ops skylake_rt286_ops = {
+static const struct snd_soc_ops skylake_rt286_ops = {
 	.hw_params = skylake_rt286_hw_params,
 };
 
@@ -289,7 +289,7 @@ static int skylake_dmic_startup(struct snd_pcm_substream *substream)
 			SNDRV_PCM_HW_PARAM_RATE, &constraints_rates);
 }
 
-static struct snd_soc_ops skylake_dmic_ops = {
+static const struct snd_soc_ops skylake_dmic_ops = {
 	.startup = skylake_dmic_startup,
 };
 
diff --git a/sound/soc/intel/common/sst-acpi.h b/sound/soc/intel/common/sst-acpi.h
index 012742299dd5..214e000667ae 100644
--- a/sound/soc/intel/common/sst-acpi.h
+++ b/sound/soc/intel/common/sst-acpi.h
@@ -15,14 +15,29 @@
 #include <linux/stddef.h>
 #include <linux/acpi.h>
 
-/* translation fron HID to I2C name, needed for DAI codec_name */
+struct sst_acpi_package_context {
+	char *name;           /* package name */
+	int length;           /* number of elements */
+	struct acpi_buffer *format;
+	struct acpi_buffer *state;
+	bool data_valid;
+};
+
 #if IS_ENABLED(CONFIG_ACPI)
+/* translation fron HID to I2C name, needed for DAI codec_name */
 const char *sst_acpi_find_name_from_hid(const u8 hid[ACPI_ID_LEN]);
+bool sst_acpi_find_package_from_hid(const u8 hid[ACPI_ID_LEN],
+				    struct sst_acpi_package_context *ctx);
 #else
 static inline const char *sst_acpi_find_name_from_hid(const u8 hid[ACPI_ID_LEN])
 {
 	return NULL;
 }
+static inline bool sst_acpi_find_package_from_hid(const u8 hid[ACPI_ID_LEN],
+					   struct sst_acpi_package_context *ctx)
+{
+	return false;
+}
 #endif
 
 /* acpi match */
diff --git a/sound/soc/intel/common/sst-ipc.c b/sound/soc/intel/common/sst-ipc.c
index 6c672ac79cce..62f3a8e0ec87 100644
--- a/sound/soc/intel/common/sst-ipc.c
+++ b/sound/soc/intel/common/sst-ipc.c
@@ -26,7 +26,6 @@
 #include <linux/sched.h>
 #include <linux/delay.h>
 #include <linux/platform_device.h>
-#include <linux/kthread.h>
 #include <sound/asound.h>
 
 #include "sst-dsp.h"
@@ -109,10 +108,9 @@ static int ipc_tx_message(struct sst_generic_ipc *ipc, u64 header,
 		ipc->ops.tx_data_copy(msg, tx_data, tx_bytes);
 
 	list_add_tail(&msg->list, &ipc->tx_list);
+	schedule_work(&ipc->kwork);
 	spin_unlock_irqrestore(&ipc->dsp->spinlock, flags);
 
-	kthread_queue_work(&ipc->kworker, &ipc->kwork);
-
 	if (wait)
 		return tx_wait_done(ipc, msg, rx_data);
 	else
@@ -156,42 +154,56 @@ free_mem:
 	return -ENOMEM;
 }
 
-static void ipc_tx_msgs(struct kthread_work *work)
+static void ipc_tx_msgs(struct work_struct *work)
 {
 	struct sst_generic_ipc *ipc =
 		container_of(work, struct sst_generic_ipc, kwork);
 	struct ipc_message *msg;
-	unsigned long flags;
 
-	spin_lock_irqsave(&ipc->dsp->spinlock, flags);
+	spin_lock_irq(&ipc->dsp->spinlock);
 
-	if (list_empty(&ipc->tx_list) || ipc->pending) {
-		spin_unlock_irqrestore(&ipc->dsp->spinlock, flags);
-		return;
-	}
-
-	/* if the DSP is busy, we will TX messages after IRQ.
-	 * also postpone if we are in the middle of procesing completion irq*/
-	if (ipc->ops.is_dsp_busy && ipc->ops.is_dsp_busy(ipc->dsp)) {
-		dev_dbg(ipc->dev, "ipc_tx_msgs dsp busy\n");
-		spin_unlock_irqrestore(&ipc->dsp->spinlock, flags);
-		return;
-	}
+	while (!list_empty(&ipc->tx_list) && !ipc->pending) {
+		/* if the DSP is busy, we will TX messages after IRQ.
+		 * also postpone if we are in the middle of processing
+		 * completion irq
+		 */
+		if (ipc->ops.is_dsp_busy && ipc->ops.is_dsp_busy(ipc->dsp)) {
+			dev_dbg(ipc->dev, "ipc_tx_msgs dsp busy\n");
+			break;
+		}
 
-	msg = list_first_entry(&ipc->tx_list, struct ipc_message, list);
-	list_move(&msg->list, &ipc->rx_list);
+		msg = list_first_entry(&ipc->tx_list, struct ipc_message, list);
+		list_move(&msg->list, &ipc->rx_list);
 
-	if (ipc->ops.tx_msg != NULL)
-		ipc->ops.tx_msg(ipc, msg);
+		if (ipc->ops.tx_msg != NULL)
+			ipc->ops.tx_msg(ipc, msg);
+	}
 
-	spin_unlock_irqrestore(&ipc->dsp->spinlock, flags);
+	spin_unlock_irq(&ipc->dsp->spinlock);
 }
 
 int sst_ipc_tx_message_wait(struct sst_generic_ipc *ipc, u64 header,
 	void *tx_data, size_t tx_bytes, void *rx_data, size_t rx_bytes)
 {
-	return ipc_tx_message(ipc, header, tx_data, tx_bytes,
+	int ret;
+
+	/*
+	 * DSP maybe in lower power active state, so
+	 * check if the DSP supports DSP lp On method
+	 * if so invoke that before sending IPC
+	 */
+	if (ipc->ops.check_dsp_lp_on)
+		if (ipc->ops.check_dsp_lp_on(ipc->dsp, true))
+			return -EIO;
+
+	ret = ipc_tx_message(ipc, header, tx_data, tx_bytes,
 		rx_data, rx_bytes, 1);
+
+	if (ipc->ops.check_dsp_lp_on)
+		if (ipc->ops.check_dsp_lp_on(ipc->dsp, false))
+			return -EIO;
+
+	return ret;
 }
 EXPORT_SYMBOL_GPL(sst_ipc_tx_message_wait);
 
@@ -203,6 +215,14 @@ int sst_ipc_tx_message_nowait(struct sst_generic_ipc *ipc, u64 header,
 }
 EXPORT_SYMBOL_GPL(sst_ipc_tx_message_nowait);
 
+int sst_ipc_tx_message_nopm(struct sst_generic_ipc *ipc, u64 header,
+	void *tx_data, size_t tx_bytes, void *rx_data, size_t rx_bytes)
+{
+	return ipc_tx_message(ipc, header, tx_data, tx_bytes,
+		rx_data, rx_bytes, 1);
+}
+EXPORT_SYMBOL_GPL(sst_ipc_tx_message_nopm);
+
 struct ipc_message *sst_ipc_reply_find_msg(struct sst_generic_ipc *ipc,
 	u64 header)
 {
@@ -280,19 +300,7 @@ int sst_ipc_init(struct sst_generic_ipc *ipc)
 	if (ret < 0)
 		return -ENOMEM;
 
-	/* start the IPC message thread */
-	kthread_init_worker(&ipc->kworker);
-	ipc->tx_thread = kthread_run(kthread_worker_fn,
-					&ipc->kworker, "%s",
-					dev_name(ipc->dev));
-	if (IS_ERR(ipc->tx_thread)) {
-		dev_err(ipc->dev, "error: failed to create message TX task\n");
-		ret = PTR_ERR(ipc->tx_thread);
-		kfree(ipc->msg);
-		return ret;
-	}
-
-	kthread_init_work(&ipc->kwork, ipc_tx_msgs);
+	INIT_WORK(&ipc->kwork, ipc_tx_msgs);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(sst_ipc_init);
@@ -301,8 +309,7 @@ void sst_ipc_fini(struct sst_generic_ipc *ipc)
 {
 	int i;
 
-	if (ipc->tx_thread)
-		kthread_stop(ipc->tx_thread);
+	cancel_work_sync(&ipc->kwork);
 
 	if (ipc->msg) {
 		for (i = 0; i < IPC_EMPTY_LIST_SIZE; i++) {
diff --git a/sound/soc/intel/common/sst-ipc.h b/sound/soc/intel/common/sst-ipc.h
index ceb7e468a3fa..7ed42a640ad6 100644
--- a/sound/soc/intel/common/sst-ipc.h
+++ b/sound/soc/intel/common/sst-ipc.h
@@ -23,7 +23,6 @@
 #include <linux/list.h>
 #include <linux/workqueue.h>
 #include <linux/sched.h>
-#include <linux/kthread.h>
 
 #define IPC_MAX_MAILBOX_BYTES	256
 
@@ -52,6 +51,7 @@ struct sst_plat_ipc_ops {
 	void (*tx_data_copy)(struct ipc_message *, char *, size_t);
 	u64  (*reply_msg_match)(u64 header, u64 *mask);
 	bool (*is_dsp_busy)(struct sst_dsp *dsp);
+	int (*check_dsp_lp_on)(struct sst_dsp *dsp, bool state);
 };
 
 /* SST generic IPC data */
@@ -65,8 +65,7 @@ struct sst_generic_ipc {
 	struct list_head empty_list;
 	wait_queue_head_t wait_txq;
 	struct task_struct *tx_thread;
-	struct kthread_worker kworker;
-	struct kthread_work kwork;
+	struct work_struct kwork;
 	bool pending;
 	struct ipc_message *msg;
 	int tx_data_max_size;
@@ -81,6 +80,9 @@ int sst_ipc_tx_message_wait(struct sst_generic_ipc *ipc, u64 header,
 int sst_ipc_tx_message_nowait(struct sst_generic_ipc *ipc, u64 header,
 	void *tx_data, size_t tx_bytes);
 
+int sst_ipc_tx_message_nopm(struct sst_generic_ipc *ipc, u64 header,
+	void *tx_data, size_t tx_bytes, void *rx_data, size_t rx_bytes);
+
 struct ipc_message *sst_ipc_reply_find_msg(struct sst_generic_ipc *ipc,
 	u64 header);
 
diff --git a/sound/soc/intel/common/sst-match-acpi.c b/sound/soc/intel/common/sst-match-acpi.c
index 789843307a49..1070f3ad23e5 100644
--- a/sound/soc/intel/common/sst-match-acpi.c
+++ b/sound/soc/intel/common/sst-match-acpi.c
@@ -77,5 +77,62 @@ struct sst_acpi_mach *sst_acpi_find_machine(struct sst_acpi_mach *machines)
 }
 EXPORT_SYMBOL_GPL(sst_acpi_find_machine);
 
+static acpi_status sst_acpi_find_package(acpi_handle handle, u32 level,
+					void *context, void **ret)
+{
+	struct acpi_device *adev;
+	acpi_status status = AE_OK;
+	struct sst_acpi_package_context *pkg_ctx = context;
+
+	pkg_ctx->data_valid = false;
+
+	if (acpi_bus_get_device(handle, &adev))
+		return AE_OK;
+
+	if (adev->status.present && adev->status.functional) {
+		struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER, NULL};
+		union acpi_object  *myobj = NULL;
+
+		status = acpi_evaluate_object_typed(handle, pkg_ctx->name,
+						NULL, &buffer,
+						ACPI_TYPE_PACKAGE);
+		if (ACPI_FAILURE(status))
+			return AE_OK;
+
+		myobj = buffer.pointer;
+		if (!myobj || myobj->package.count != pkg_ctx->length) {
+			kfree(buffer.pointer);
+			return AE_OK;
+		}
+
+		status = acpi_extract_package(myobj,
+					pkg_ctx->format, pkg_ctx->state);
+		if (ACPI_FAILURE(status)) {
+			kfree(buffer.pointer);
+			return AE_OK;
+		}
+
+		kfree(buffer.pointer);
+		pkg_ctx->data_valid = true;
+		return AE_CTRL_TERMINATE;
+	}
+
+	return AE_OK;
+}
+
+bool sst_acpi_find_package_from_hid(const u8 hid[ACPI_ID_LEN],
+				struct sst_acpi_package_context *ctx)
+{
+	acpi_status status;
+
+	status = acpi_get_devices(hid, sst_acpi_find_package, ctx, NULL);
+
+	if (ACPI_FAILURE(status) || !ctx->data_valid)
+		return false;
+
+	return true;
+}
+EXPORT_SYMBOL_GPL(sst_acpi_find_package_from_hid);
+
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("Intel Common ACPI Match module");
diff --git a/sound/soc/intel/haswell/sst-haswell-ipc.c b/sound/soc/intel/haswell/sst-haswell-ipc.c
index e432a31fd9f2..a3459d1682a6 100644
--- a/sound/soc/intel/haswell/sst-haswell-ipc.c
+++ b/sound/soc/intel/haswell/sst-haswell-ipc.c
@@ -26,7 +26,6 @@
 #include <linux/delay.h>
 #include <linux/sched.h>
 #include <linux/platform_device.h>
-#include <linux/kthread.h>
 #include <linux/firmware.h>
 #include <linux/dma-mapping.h>
 #include <linux/debugfs.h>
@@ -818,7 +817,7 @@ static irqreturn_t hsw_irq_thread(int irq, void *context)
 	spin_unlock_irqrestore(&sst->spinlock, flags);
 
 	/* continue to send any remaining messages... */
-	kthread_queue_work(&ipc->kworker, &ipc->kwork);
+	schedule_work(&ipc->kwork);
 
 	return IRQ_HANDLED;
 }
diff --git a/sound/soc/intel/skylake/bxt-sst.c b/sound/soc/intel/skylake/bxt-sst.c
index 1d251d59bcb9..1f9f33d34000 100644
--- a/sound/soc/intel/skylake/bxt-sst.c
+++ b/sound/soc/intel/skylake/bxt-sst.c
@@ -43,6 +43,9 @@
 
 #define BXT_ADSP_FW_BIN_HDR_OFFSET 0x2000
 
+/* Delay before scheduling D0i3 entry */
+#define BXT_D0I3_DELAY 5000
+
 static unsigned int bxt_get_errorcode(struct sst_dsp *ctx)
 {
 	 return sst_dsp_shim_read(ctx, BXT_ADSP_ERROR_CODE);
@@ -288,6 +291,141 @@ sst_load_base_firmware_failed:
 	return ret;
 }
 
+/*
+ * Decide the D0i3 state that can be targeted based on the usecase
+ * ref counts and DSP state
+ *
+ * Decision Matrix:  (X= dont care; state = target state)
+ *
+ * DSP state != SKL_DSP_RUNNING ; state = no d0i3
+ *
+ * DSP state == SKL_DSP_RUNNING , the following matrix applies
+ * non_d0i3 >0; streaming =X; non_streaming =X; state = no d0i3
+ * non_d0i3 =X; streaming =0; non_streaming =0; state = no d0i3
+ * non_d0i3 =0; streaming >0; non_streaming =X; state = streaming d0i3
+ * non_d0i3 =0; streaming =0; non_streaming =X; state = non-streaming d0i3
+ */
+static int bxt_d0i3_target_state(struct sst_dsp *ctx)
+{
+	struct skl_sst *skl = ctx->thread_context;
+	struct skl_d0i3_data *d0i3 = &skl->d0i3;
+
+	if (skl->cores.state[SKL_DSP_CORE0_ID] != SKL_DSP_RUNNING)
+		return SKL_DSP_D0I3_NONE;
+
+	if (d0i3->non_d0i3)
+		return SKL_DSP_D0I3_NONE;
+	else if (d0i3->streaming)
+		return SKL_DSP_D0I3_STREAMING;
+	else if (d0i3->non_streaming)
+		return SKL_DSP_D0I3_NON_STREAMING;
+	else
+		return SKL_DSP_D0I3_NONE;
+}
+
+static void bxt_set_dsp_D0i3(struct work_struct *work)
+{
+	int ret;
+	struct skl_ipc_d0ix_msg msg;
+	struct skl_sst *skl = container_of(work,
+			struct skl_sst, d0i3.work.work);
+	struct sst_dsp *ctx = skl->dsp;
+	struct skl_d0i3_data *d0i3 = &skl->d0i3;
+	int target_state;
+
+	dev_dbg(ctx->dev, "In %s:\n", __func__);
+
+	/* D0i3 entry allowed only if core 0 alone is running */
+	if (skl_dsp_get_enabled_cores(ctx) !=  SKL_DSP_CORE0_MASK) {
+		dev_warn(ctx->dev,
+				"D0i3 allowed when only core0 running:Exit\n");
+		return;
+	}
+
+	target_state = bxt_d0i3_target_state(ctx);
+	if (target_state == SKL_DSP_D0I3_NONE)
+		return;
+
+	msg.instance_id = 0;
+	msg.module_id = 0;
+	msg.wake = 1;
+	msg.streaming = 0;
+	if (target_state == SKL_DSP_D0I3_STREAMING)
+		msg.streaming = 1;
+
+	ret =  skl_ipc_set_d0ix(&skl->ipc, &msg);
+
+	if (ret < 0) {
+		dev_err(ctx->dev, "Failed to set DSP to D0i3 state\n");
+		return;
+	}
+
+	/* Set Vendor specific register D0I3C.I3 to enable D0i3*/
+	if (skl->update_d0i3c)
+		skl->update_d0i3c(skl->dev, true);
+
+	d0i3->state = target_state;
+	skl->cores.state[SKL_DSP_CORE0_ID] = SKL_DSP_RUNNING_D0I3;
+}
+
+static int bxt_schedule_dsp_D0i3(struct sst_dsp *ctx)
+{
+	struct skl_sst *skl = ctx->thread_context;
+	struct skl_d0i3_data *d0i3 = &skl->d0i3;
+
+	/* Schedule D0i3 only if the usecase ref counts are appropriate */
+	if (bxt_d0i3_target_state(ctx) != SKL_DSP_D0I3_NONE) {
+
+		dev_dbg(ctx->dev, "%s: Schedule D0i3\n", __func__);
+
+		schedule_delayed_work(&d0i3->work,
+				msecs_to_jiffies(BXT_D0I3_DELAY));
+	}
+
+	return 0;
+}
+
+static int bxt_set_dsp_D0i0(struct sst_dsp *ctx)
+{
+	int ret;
+	struct skl_ipc_d0ix_msg msg;
+	struct skl_sst *skl = ctx->thread_context;
+
+	dev_dbg(ctx->dev, "In %s:\n", __func__);
+
+	/* First Cancel any pending attempt to put DSP to D0i3 */
+	cancel_delayed_work_sync(&skl->d0i3.work);
+
+	/* If DSP is currently in D0i3, bring it to D0i0 */
+	if (skl->cores.state[SKL_DSP_CORE0_ID] != SKL_DSP_RUNNING_D0I3)
+		return 0;
+
+	dev_dbg(ctx->dev, "Set DSP to D0i0\n");
+
+	msg.instance_id = 0;
+	msg.module_id = 0;
+	msg.streaming = 0;
+	msg.wake = 0;
+
+	if (skl->d0i3.state == SKL_DSP_D0I3_STREAMING)
+		msg.streaming = 1;
+
+	/* Clear Vendor specific register D0I3C.I3 to disable D0i3*/
+	if (skl->update_d0i3c)
+		skl->update_d0i3c(skl->dev, false);
+
+	ret =  skl_ipc_set_d0ix(&skl->ipc, &msg);
+	if (ret < 0) {
+		dev_err(ctx->dev, "Failed to set DSP to D0i0\n");
+		return ret;
+	}
+
+	skl->cores.state[SKL_DSP_CORE0_ID] = SKL_DSP_RUNNING;
+	skl->d0i3.state = SKL_DSP_D0I3_NONE;
+
+	return 0;
+}
+
 static int bxt_set_dsp_D0(struct sst_dsp *ctx, unsigned int core_id)
 {
 	struct skl_sst *skl = ctx->thread_context;
@@ -414,6 +552,8 @@ static int bxt_set_dsp_D3(struct sst_dsp *ctx, unsigned int core_id)
 static struct skl_dsp_fw_ops bxt_fw_ops = {
 	.set_state_D0 = bxt_set_dsp_D0,
 	.set_state_D3 = bxt_set_dsp_D3,
+	.set_state_D0i3 = bxt_schedule_dsp_D0i3,
+	.set_state_D0i0 = bxt_set_dsp_D0i0,
 	.load_fw = bxt_load_base_firmware,
 	.get_fw_errcode = bxt_get_errorcode,
 	.load_library = bxt_load_library,
@@ -470,10 +610,15 @@ int bxt_sst_dsp_init(struct device *dev, void __iomem *mmio_base, int irq,
 	if (ret)
 		return ret;
 
+	/* set the D0i3 check */
+	skl->ipc.ops.check_dsp_lp_on = skl_ipc_check_D0i0;
+
 	skl->cores.count = 2;
 	skl->boot_complete = false;
 	init_waitqueue_head(&skl->boot_wait);
 	skl->is_first_boot = true;
+	INIT_DELAYED_WORK(&skl->d0i3.work, bxt_set_dsp_D0i3);
+	skl->d0i3.state = SKL_DSP_D0I3_NONE;
 
 	if (dsp)
 		*dsp = skl;
diff --git a/sound/soc/intel/skylake/skl-messages.c b/sound/soc/intel/skylake/skl-messages.c
index 805b7f2173f3..e79cbcf6e462 100644
--- a/sound/soc/intel/skylake/skl-messages.c
+++ b/sound/soc/intel/skylake/skl-messages.c
@@ -294,6 +294,33 @@ int skl_free_dsp(struct skl *skl)
 	return 0;
 }
 
+/*
+ * In the case of "suspend_active" i.e, the Audio IP being active
+ * during system suspend, immediately excecute any pending D0i3 work
+ * before suspending. This is needed for the IP to work in low power
+ * mode during system suspend. In the case of normal suspend, cancel
+ * any pending D0i3 work.
+ */
+int skl_suspend_late_dsp(struct skl *skl)
+{
+	struct skl_sst *ctx = skl->skl_sst;
+	struct delayed_work *dwork;
+
+	if (!ctx)
+		return 0;
+
+	dwork = &ctx->d0i3.work;
+
+	if (dwork->work.func) {
+		if (skl->supend_active)
+			flush_delayed_work(dwork);
+		else
+			cancel_delayed_work_sync(dwork);
+	}
+
+	return 0;
+}
+
 int skl_suspend_dsp(struct skl *skl)
 {
 	struct skl_sst *ctx = skl->skl_sst;
@@ -500,16 +527,14 @@ static void skl_setup_cpr_gateway_cfg(struct skl_sst *ctx,
 int skl_dsp_set_dma_control(struct skl_sst *ctx, struct skl_module_cfg *mconfig)
 {
 	struct skl_dma_control *dma_ctrl;
-	struct skl_i2s_config_blob config_blob;
 	struct skl_ipc_large_config_msg msg = {0};
 	int err = 0;
 
 
 	/*
-	 * if blob size is same as capablity size, then no dma control
-	 * present so return
+	 * if blob size zero, then return
 	 */
-	if (mconfig->formats_config.caps_size == sizeof(config_blob))
+	if (mconfig->formats_config.caps_size == 0)
 		return 0;
 
 	msg.large_param_id = DMA_CONTROL_ID;
@@ -523,7 +548,7 @@ int skl_dsp_set_dma_control(struct skl_sst *ctx, struct skl_module_cfg *mconfig)
 	dma_ctrl->node_id = skl_get_node_id(ctx, mconfig);
 
 	/* size in dwords */
-	dma_ctrl->config_length = sizeof(config_blob) / 4;
+	dma_ctrl->config_length = mconfig->formats_config.caps_size / 4;
 
 	memcpy(dma_ctrl->config_data, mconfig->formats_config.caps,
 				mconfig->formats_config.caps_size);
@@ -531,7 +556,6 @@ int skl_dsp_set_dma_control(struct skl_sst *ctx, struct skl_module_cfg *mconfig)
 	err = skl_ipc_set_large_config(&ctx->ipc, &msg, (u32 *)dma_ctrl);
 
 	kfree(dma_ctrl);
-
 	return err;
 }
 
@@ -1042,7 +1066,8 @@ int skl_create_pipeline(struct skl_sst *ctx, struct skl_pipe *pipe)
 	dev_dbg(ctx->dev, "%s: pipe_id = %d\n", __func__, pipe->ppl_id);
 
 	ret = skl_ipc_create_pipeline(&ctx->ipc, pipe->memory_pages,
-				pipe->pipe_priority, pipe->ppl_id);
+				pipe->pipe_priority, pipe->ppl_id,
+				pipe->lp_mode);
 	if (ret < 0) {
 		dev_err(ctx->dev, "Failed to create pipeline\n");
 		return ret;
diff --git a/sound/soc/intel/skylake/skl-pcm.c b/sound/soc/intel/skylake/skl-pcm.c
index 58c728662600..84b5101e6ca6 100644
--- a/sound/soc/intel/skylake/skl-pcm.c
+++ b/sound/soc/intel/skylake/skl-pcm.c
@@ -144,6 +144,8 @@ static int skl_pcm_open(struct snd_pcm_substream *substream,
 	struct hdac_ext_stream *stream;
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	struct skl_dma_params *dma_params;
+	struct skl *skl = get_skl_ctx(dai->dev);
+	struct skl_module_cfg *mconfig;
 
 	dev_dbg(dai->dev, "%s: %s\n", __func__, dai->name);
 
@@ -177,6 +179,9 @@ static int skl_pcm_open(struct snd_pcm_substream *substream,
 	skl_set_suspend_active(substream, dai, true);
 	snd_pcm_set_sync(substream);
 
+	mconfig = skl_tplg_fe_get_cpr_module(dai, substream->stream);
+	skl_tplg_d0i3_get(skl, mconfig->d0i3_caps);
+
 	return 0;
 }
 
@@ -302,6 +307,7 @@ static void skl_pcm_close(struct snd_pcm_substream *substream,
 	struct hdac_ext_bus *ebus = dev_get_drvdata(dai->dev);
 	struct skl_dma_params *dma_params = NULL;
 	struct skl *skl = ebus_to_skl(ebus);
+	struct skl_module_cfg *mconfig;
 
 	dev_dbg(dai->dev, "%s: %s\n", __func__, dai->name);
 
@@ -325,6 +331,9 @@ static void skl_pcm_close(struct snd_pcm_substream *substream,
 		skl->skl_sst->miscbdcg_disabled = false;
 	}
 
+	mconfig = skl_tplg_fe_get_cpr_module(dai, substream->stream);
+	skl_tplg_d0i3_put(skl, mconfig->d0i3_caps);
+
 	kfree(dma_params);
 }
 
@@ -1031,10 +1040,24 @@ static snd_pcm_uframes_t skl_platform_pcm_pointer
 			(struct snd_pcm_substream *substream)
 {
 	struct hdac_ext_stream *hstream = get_hdac_ext_stream(substream);
+	struct hdac_ext_bus *ebus = get_bus_ctx(substream);
 	unsigned int pos;
 
-	/* use the position buffer as default */
-	pos = snd_hdac_stream_get_pos_posbuf(hdac_stream(hstream));
+	/*
+	 * Use DPIB for Playback stream as the periodic DMA Position-in-
+	 * Buffer Writes may be scheduled at the same time or later than
+	 * the MSI and does not guarantee to reflect the Position of the
+	 * last buffer that was transferred. Whereas DPIB register in
+	 * HAD space reflects the actual data that is transferred.
+	 * Use the position buffer for capture, as DPIB write gets
+	 * completed earlier than the actual data written to the DDR.
+	 */
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		pos = readl(ebus->bus.remap_addr + AZX_REG_VS_SDXDPIB_XBASE +
+				(AZX_REG_VS_SDXDPIB_XINTERVAL *
+				hdac_stream(hstream)->index));
+	else
+		pos = snd_hdac_stream_get_pos_posbuf(hdac_stream(hstream));
 
 	if (pos >= hdac_stream(hstream)->bufsize)
 		pos = 0;
@@ -1197,6 +1220,7 @@ static int skl_platform_soc_probe(struct snd_soc_platform *platform)
 			return ret;
 		}
 		skl_populate_modules(skl);
+		skl->skl_sst->update_d0i3c = skl_update_d0i3c;
 	}
 	pm_runtime_mark_last_busy(platform->dev);
 	pm_runtime_put_autosuspend(platform->dev);
diff --git a/sound/soc/intel/skylake/skl-sst-cldma.c b/sound/soc/intel/skylake/skl-sst-cldma.c
index efa2532114ba..c9f6d87381db 100644
--- a/sound/soc/intel/skylake/skl-sst-cldma.c
+++ b/sound/soc/intel/skylake/skl-sst-cldma.c
@@ -17,7 +17,6 @@
 
 #include <linux/device.h>
 #include <linux/mm.h>
-#include <linux/kthread.h>
 #include <linux/delay.h>
 #include "../common/sst-dsp.h"
 #include "../common/sst-dsp-priv.h"
diff --git a/sound/soc/intel/skylake/skl-sst-dsp.h b/sound/soc/intel/skylake/skl-sst-dsp.h
index b9e71d051fb1..7c272ba0f4b5 100644
--- a/sound/soc/intel/skylake/skl-sst-dsp.h
+++ b/sound/soc/intel/skylake/skl-sst-dsp.h
@@ -126,11 +126,21 @@ struct sst_dsp_device;
 #define SKL_ADSPCS_CPA_SHIFT		24
 #define SKL_ADSPCS_CPA_MASK(cm)		((cm) << SKL_ADSPCS_CPA_SHIFT)
 
+/* DSP Core state */
 enum skl_dsp_states {
 	SKL_DSP_RUNNING = 1,
+	/* Running in D0i3 state; can be in streaming or non-streaming D0i3 */
+	SKL_DSP_RUNNING_D0I3, /* Running in D0i3 state*/
 	SKL_DSP_RESET,
 };
 
+/* D0i3 substates */
+enum skl_dsp_d0i3_states {
+	SKL_DSP_D0I3_NONE = -1, /* No D0i3 */
+	SKL_DSP_D0I3_NON_STREAMING = 0,
+	SKL_DSP_D0I3_STREAMING = 1,
+};
+
 struct skl_dsp_fw_ops {
 	int (*load_fw)(struct sst_dsp  *ctx);
 	/* FW module parser/loader */
@@ -139,6 +149,8 @@ struct skl_dsp_fw_ops {
 	int (*parse_fw)(struct sst_dsp *ctx);
 	int (*set_state_D0)(struct sst_dsp *ctx, unsigned int core_id);
 	int (*set_state_D3)(struct sst_dsp *ctx, unsigned int core_id);
+	int (*set_state_D0i3)(struct sst_dsp *ctx);
+	int (*set_state_D0i0)(struct sst_dsp *ctx);
 	unsigned int (*get_fw_errcode)(struct sst_dsp *ctx);
 	int (*load_mod)(struct sst_dsp *ctx, u16 mod_id, u8 *mod_name);
 	int (*unload_mod)(struct sst_dsp *ctx, u16 mod_id);
diff --git a/sound/soc/intel/skylake/skl-sst-ipc.c b/sound/soc/intel/skylake/skl-sst-ipc.c
index 797cf4053235..e1391dfbc9e9 100644
--- a/sound/soc/intel/skylake/skl-sst-ipc.c
+++ b/sound/soc/intel/skylake/skl-sst-ipc.c
@@ -81,6 +81,11 @@
 #define IPC_INSTANCE_ID(x)		(((x) & IPC_INSTANCE_ID_MASK) \
 					<< IPC_INSTANCE_ID_SHIFT)
 
+#define IPC_PPL_LP_MODE_SHIFT           0
+#define IPC_PPL_LP_MODE_MASK            0x1
+#define IPC_PPL_LP_MODE(x)              (((x) & IPC_PPL_LP_MODE_MASK) \
+					<< IPC_PPL_LP_MODE_SHIFT)
+
 /* Set pipeline state message */
 #define IPC_PPL_STATE_SHIFT		0
 #define IPC_PPL_STATE_MASK		0x1F
@@ -172,6 +177,17 @@
 					<< IPC_INITIAL_BLOCK_SHIFT)
 #define IPC_INITIAL_BLOCK_CLEAR		~(IPC_INITIAL_BLOCK_MASK \
 					  << IPC_INITIAL_BLOCK_SHIFT)
+/* Set D0ix IPC extension register */
+#define IPC_D0IX_WAKE_SHIFT		0
+#define IPC_D0IX_WAKE_MASK		0x1
+#define IPC_D0IX_WAKE(x)		(((x) & IPC_D0IX_WAKE_MASK) \
+					<< IPC_D0IX_WAKE_SHIFT)
+
+#define IPC_D0IX_STREAMING_SHIFT	1
+#define IPC_D0IX_STREAMING_MASK		0x1
+#define IPC_D0IX_STREAMING(x)		(((x) & IPC_D0IX_STREAMING_MASK) \
+					<< IPC_D0IX_STREAMING_SHIFT)
+
 
 enum skl_ipc_msg_target {
 	IPC_FW_GEN_MSG = 0,
@@ -258,7 +274,8 @@ enum skl_ipc_module_msg {
 	IPC_MOD_LARGE_CONFIG_SET = 4,
 	IPC_MOD_BIND = 5,
 	IPC_MOD_UNBIND = 6,
-	IPC_MOD_SET_DX = 7
+	IPC_MOD_SET_DX = 7,
+	IPC_MOD_SET_D0IX = 8
 };
 
 static void skl_ipc_tx_data_copy(struct ipc_message *msg, char *tx_data,
@@ -289,6 +306,23 @@ static void skl_ipc_tx_msg(struct sst_generic_ipc *ipc, struct ipc_message *msg)
 		header->primary | SKL_ADSP_REG_HIPCI_BUSY);
 }
 
+int skl_ipc_check_D0i0(struct sst_dsp *dsp, bool state)
+{
+	int ret;
+
+	/* check D0i3 support */
+	if (!dsp->fw_ops.set_state_D0i0)
+		return 0;
+
+	/* Attempt D0i0 or D0i3 based on state */
+	if (state)
+		ret = dsp->fw_ops.set_state_D0i0(dsp);
+	else
+		ret = dsp->fw_ops.set_state_D0i3(dsp);
+
+	return ret;
+}
+
 static struct ipc_message *skl_ipc_reply_get_msg(struct sst_generic_ipc *ipc,
 				u64 ipc_header)
 {
@@ -464,7 +498,7 @@ irqreturn_t skl_dsp_irq_thread_handler(int irq, void *context)
 	skl_ipc_int_enable(dsp);
 
 	/* continue to send any remaining messages... */
-	kthread_queue_work(&ipc->kworker, &ipc->kwork);
+	schedule_work(&ipc->kwork);
 
 	return IRQ_HANDLED;
 }
@@ -547,7 +581,7 @@ void skl_ipc_free(struct sst_generic_ipc *ipc)
 }
 
 int skl_ipc_create_pipeline(struct sst_generic_ipc *ipc,
-		u16 ppl_mem_size, u8 ppl_type, u8 instance_id)
+		u16 ppl_mem_size, u8 ppl_type, u8 instance_id, u8 lp_mode)
 {
 	struct skl_ipc_header header = {0};
 	u64 *ipc_header = (u64 *)(&header);
@@ -560,6 +594,8 @@ int skl_ipc_create_pipeline(struct sst_generic_ipc *ipc,
 	header.primary |= IPC_PPL_TYPE(ppl_type);
 	header.primary |= IPC_PPL_MEM_SIZE(ppl_mem_size);
 
+	header.extension = IPC_PPL_LP_MODE(lp_mode);
+
 	dev_dbg(ipc->dev, "In %s header=%d\n", __func__, header.primary);
 	ret = sst_ipc_tx_message_wait(ipc, *ipc_header, NULL, 0, NULL, 0);
 	if (ret < 0) {
@@ -931,3 +967,32 @@ int skl_sst_ipc_load_library(struct sst_generic_ipc *ipc,
 	return ret;
 }
 EXPORT_SYMBOL_GPL(skl_sst_ipc_load_library);
+
+int skl_ipc_set_d0ix(struct sst_generic_ipc *ipc, struct skl_ipc_d0ix_msg *msg)
+{
+	struct skl_ipc_header header = {0};
+	u64 *ipc_header = (u64 *)(&header);
+	int ret;
+
+	header.primary = IPC_MSG_TARGET(IPC_MOD_MSG);
+	header.primary |= IPC_MSG_DIR(IPC_MSG_REQUEST);
+	header.primary |= IPC_GLB_TYPE(IPC_MOD_SET_D0IX);
+	header.primary |= IPC_MOD_INSTANCE_ID(msg->instance_id);
+	header.primary |= IPC_MOD_ID(msg->module_id);
+
+	header.extension = IPC_D0IX_WAKE(msg->wake);
+	header.extension |= IPC_D0IX_STREAMING(msg->streaming);
+
+	dev_dbg(ipc->dev, "In %s primary=%x ext=%x\n", __func__,
+			header.primary,	header.extension);
+
+	/*
+	 * Use the nopm IPC here as we dont want it checking for D0iX
+	 */
+	ret = sst_ipc_tx_message_nopm(ipc, *ipc_header, NULL, 0, NULL, 0);
+	if (ret < 0)
+		dev_err(ipc->dev, "ipc: set d0ix failed, err %d\n", ret);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(skl_ipc_set_d0ix);
diff --git a/sound/soc/intel/skylake/skl-sst-ipc.h b/sound/soc/intel/skylake/skl-sst-ipc.h
index 0334ed4af031..cc40341233fa 100644
--- a/sound/soc/intel/skylake/skl-sst-ipc.h
+++ b/sound/soc/intel/skylake/skl-sst-ipc.h
@@ -16,7 +16,6 @@
 #ifndef __SKL_IPC_H
 #define __SKL_IPC_H
 
-#include <linux/kthread.h>
 #include <linux/irqreturn.h>
 #include "../common/sst-ipc.h"
 
@@ -53,6 +52,23 @@ struct skl_dsp_cores {
 	int usage_count[SKL_DSP_CORES_MAX];
 };
 
+/**
+ * skl_d0i3_data: skl D0i3 counters data struct
+ *
+ * @streaming: Count of usecases that can attempt streaming D0i3
+ * @non_streaming: Count of usecases that can attempt non-streaming D0i3
+ * @non_d0i3: Count of usecases that cannot attempt D0i3
+ * @state: current state
+ * @work: D0i3 worker thread
+ */
+struct skl_d0i3_data {
+	int streaming;
+	int non_streaming;
+	int non_d0i3;
+	enum skl_dsp_d0i3_states state;
+	struct delayed_work work;
+};
+
 struct skl_sst {
 	struct device *dev;
 	struct sst_dsp *dsp;
@@ -83,6 +99,11 @@ struct skl_sst {
 
 	/* tplg manifest */
 	struct skl_dfw_manifest manifest;
+
+	/* Callback to update D0i3C register */
+	void (*update_d0i3c)(struct device *dev, bool enable);
+
+	struct skl_d0i3_data d0i3;
 };
 
 struct skl_ipc_init_instance_msg {
@@ -111,6 +132,13 @@ struct skl_ipc_large_config_msg {
 	u32 param_data_size;
 };
 
+struct skl_ipc_d0ix_msg {
+	u32 module_id;
+	u32 instance_id;
+	u8 streaming;
+	u8 wake;
+};
+
 #define SKL_IPC_BOOT_MSECS		3000
 
 #define SKL_IPC_D3_MASK	0
@@ -119,7 +147,7 @@ struct skl_ipc_large_config_msg {
 irqreturn_t skl_dsp_irq_thread_handler(int irq, void *context);
 
 int skl_ipc_create_pipeline(struct sst_generic_ipc *sst_ipc,
-		u16 ppl_mem_size, u8 ppl_type, u8 instance_id);
+		u16 ppl_mem_size, u8 ppl_type, u8 instance_id, u8 lp_mode);
 
 int skl_ipc_delete_pipeline(struct sst_generic_ipc *sst_ipc, u8 instance_id);
 
@@ -155,6 +183,11 @@ int skl_ipc_get_large_config(struct sst_generic_ipc *ipc,
 int skl_sst_ipc_load_library(struct sst_generic_ipc *ipc,
 			u8 dma_id, u8 table_id);
 
+int skl_ipc_set_d0ix(struct sst_generic_ipc *ipc,
+		struct skl_ipc_d0ix_msg *msg);
+
+int skl_ipc_check_D0i0(struct sst_dsp *dsp, bool state);
+
 void skl_ipc_int_enable(struct sst_dsp *dsp);
 void skl_ipc_op_int_enable(struct sst_dsp *ctx);
 void skl_ipc_op_int_disable(struct sst_dsp *ctx);
diff --git a/sound/soc/intel/skylake/skl-sst-utils.c b/sound/soc/intel/skylake/skl-sst-utils.c
index 8dc03039b311..ea162fbf68e5 100644
--- a/sound/soc/intel/skylake/skl-sst-utils.c
+++ b/sound/soc/intel/skylake/skl-sst-utils.c
@@ -179,7 +179,7 @@ static inline int skl_getid_32(struct uuid_module *module, u64 *val,
 		index = ffz(mask_val);
 		pvt_id = index + word1_mask + word2_mask;
 		if (pvt_id <= (max_inst - 1)) {
-			*val |= 1 << (index + word1_mask);
+			*val |= 1ULL << (index + word1_mask);
 			return pvt_id;
 		}
 	}
diff --git a/sound/soc/intel/skylake/skl-topology.c b/sound/soc/intel/skylake/skl-topology.c
index b5b1934d8550..bd313c907b20 100644
--- a/sound/soc/intel/skylake/skl-topology.c
+++ b/sound/soc/intel/skylake/skl-topology.c
@@ -36,6 +36,44 @@
 #define SKL_IN_DIR_BIT_MASK		BIT(0)
 #define SKL_PIN_COUNT_MASK		GENMASK(7, 4)
 
+void skl_tplg_d0i3_get(struct skl *skl, enum d0i3_capability caps)
+{
+	struct skl_d0i3_data *d0i3 =  &skl->skl_sst->d0i3;
+
+	switch (caps) {
+	case SKL_D0I3_NONE:
+		d0i3->non_d0i3++;
+		break;
+
+	case SKL_D0I3_STREAMING:
+		d0i3->streaming++;
+		break;
+
+	case SKL_D0I3_NON_STREAMING:
+		d0i3->non_streaming++;
+		break;
+	}
+}
+
+void skl_tplg_d0i3_put(struct skl *skl, enum d0i3_capability caps)
+{
+	struct skl_d0i3_data *d0i3 =  &skl->skl_sst->d0i3;
+
+	switch (caps) {
+	case SKL_D0I3_NONE:
+		d0i3->non_d0i3--;
+		break;
+
+	case SKL_D0I3_STREAMING:
+		d0i3->streaming--;
+		break;
+
+	case SKL_D0I3_NON_STREAMING:
+		d0i3->non_streaming--;
+		break;
+	}
+}
+
 /*
  * SKL DSP driver modelling uses only few DAPM widgets so for rest we will
  * ignore. This helpers checks if the SKL driver handles this widget type
@@ -1519,6 +1557,10 @@ static int skl_tplg_fill_pipe_tkn(struct device *dev,
 		pipe->memory_pages = tkn_val;
 		break;
 
+	case SKL_TKN_U32_PMODE:
+		pipe->lp_mode = tkn_val;
+		break;
+
 	default:
 		dev_err(dev, "Token not handled %d\n", tkn);
 		return -EINVAL;
@@ -1826,6 +1868,10 @@ static int skl_tplg_get_token(struct device *dev,
 		mconfig->converter = tkn_elem->value;
 		break;
 
+	case SKL_TKL_U32_D0I3_CAPS:
+		mconfig->d0i3_caps = tkn_elem->value;
+		break;
+
 	case SKL_TKN_U32_PIPE_ID:
 		ret = skl_tplg_add_pipe(dev,
 				mconfig, skl, tkn_elem);
@@ -1841,6 +1887,7 @@ static int skl_tplg_get_token(struct device *dev,
 	case SKL_TKN_U32_PIPE_CONN_TYPE:
 	case SKL_TKN_U32_PIPE_PRIORITY:
 	case SKL_TKN_U32_PIPE_MEM_PGS:
+	case SKL_TKN_U32_PMODE:
 		if (is_pipe_exists) {
 			ret = skl_tplg_fill_pipe_tkn(dev, mconfig->pipe,
 					tkn_elem->token, tkn_elem->value);
diff --git a/sound/soc/intel/skylake/skl-topology.h b/sound/soc/intel/skylake/skl-topology.h
index a519360f42a6..08d39280b07b 100644
--- a/sound/soc/intel/skylake/skl-topology.h
+++ b/sound/soc/intel/skylake/skl-topology.h
@@ -113,23 +113,6 @@ struct skl_cpr_gtw_cfg {
 	u32 config_data[1];
 } __packed;
 
-struct skl_i2s_config_blob {
-	u32 gateway_attrib;
-	u32 tdm_ts_group[8];
-	u32 ssc0;
-	u32 ssc1;
-	u32 sscto;
-	u32 sspsp;
-	u32 sstsa;
-	u32 ssrsa;
-	u32 ssc2;
-	u32 sspsp2;
-	u32 ssc3;
-	u32 ssioc;
-	u32 mdivc;
-	u32 mdivr;
-} __packed;
-
 struct skl_dma_control {
 	u32 node_id;
 	u32 config_length;
@@ -279,6 +262,7 @@ struct skl_pipe {
 	u8 pipe_priority;
 	u16 conn_type;
 	u32 memory_pages;
+	u8 lp_mode;
 	struct skl_pipe_params *p_params;
 	enum skl_pipe_state state;
 	struct list_head w_list;
@@ -293,6 +277,12 @@ enum skl_module_state {
 	SKL_MODULE_UNLOADED = 4,
 };
 
+enum d0i3_capability {
+	SKL_D0I3_NONE = 0,
+	SKL_D0I3_STREAMING = 1,
+	SKL_D0I3_NON_STREAMING = 2,
+};
+
 struct skl_module_cfg {
 	u8 guid[16];
 	struct skl_module_inst_id id;
@@ -319,6 +309,7 @@ struct skl_module_cfg {
 	u32 converter;
 	u32 vbus_id;
 	u32 mem_pages;
+	enum d0i3_capability d0i3_caps;
 	struct skl_module_pin *m_in_pin;
 	struct skl_module_pin *m_out_pin;
 	enum skl_module_type m_type;
@@ -361,6 +352,9 @@ struct skl_module_cfg *skl_tplg_fe_get_cpr_module(
 int skl_tplg_update_pipe_params(struct device *dev,
 		struct skl_module_cfg *mconfig, struct skl_pipe_params *params);
 
+void skl_tplg_d0i3_get(struct skl *skl, enum d0i3_capability caps);
+void skl_tplg_d0i3_put(struct skl *skl, enum d0i3_capability caps);
+
 int skl_create_pipeline(struct skl_sst *ctx, struct skl_pipe *pipe);
 
 int skl_run_pipe(struct skl_sst *ctx, struct skl_pipe *pipe);
diff --git a/sound/soc/intel/skylake/skl.c b/sound/soc/intel/skylake/skl.c
index 06fa5e85dd0e..da5db5098274 100644
--- a/sound/soc/intel/skylake/skl.c
+++ b/sound/soc/intel/skylake/skl.c
@@ -26,6 +26,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/platform_device.h>
 #include <linux/firmware.h>
+#include <linux/delay.h>
 #include <sound/pcm.h>
 #include "../common/sst-acpi.h"
 #include <sound/hda_register.h>
@@ -109,6 +110,52 @@ static int skl_init_chip(struct hdac_bus *bus, bool full_reset)
 	return ret;
 }
 
+void skl_update_d0i3c(struct device *dev, bool enable)
+{
+	struct pci_dev *pci = to_pci_dev(dev);
+	struct hdac_ext_bus *ebus = pci_get_drvdata(pci);
+	struct hdac_bus *bus = ebus_to_hbus(ebus);
+	u8 reg;
+	int timeout = 50;
+
+	reg = snd_hdac_chip_readb(bus, VS_D0I3C);
+	/* Do not write to D0I3C until command in progress bit is cleared */
+	while ((reg & AZX_REG_VS_D0I3C_CIP) && --timeout) {
+		udelay(10);
+		reg = snd_hdac_chip_readb(bus, VS_D0I3C);
+	}
+
+	/* Highly unlikely. But if it happens, flag error explicitly */
+	if (!timeout) {
+		dev_err(bus->dev, "Before D0I3C update: D0I3C CIP timeout\n");
+		return;
+	}
+
+	if (enable)
+		reg = reg | AZX_REG_VS_D0I3C_I3;
+	else
+		reg = reg & (~AZX_REG_VS_D0I3C_I3);
+
+	snd_hdac_chip_writeb(bus, VS_D0I3C, reg);
+
+	timeout = 50;
+	/* Wait for cmd in progress to be cleared before exiting the function */
+	reg = snd_hdac_chip_readb(bus, VS_D0I3C);
+	while ((reg & AZX_REG_VS_D0I3C_CIP) && --timeout) {
+		udelay(10);
+		reg = snd_hdac_chip_readb(bus, VS_D0I3C);
+	}
+
+	/* Highly unlikely. But if it happens, flag error explicitly */
+	if (!timeout) {
+		dev_err(bus->dev, "After D0I3C update: D0I3C CIP timeout\n");
+		return;
+	}
+
+	dev_dbg(bus->dev, "D0I3C register = 0x%x\n",
+			snd_hdac_chip_readb(bus, VS_D0I3C));
+}
+
 /* called from IRQ */
 static void skl_stream_update(struct hdac_bus *bus, struct hdac_stream *hstr)
 {
@@ -181,6 +228,15 @@ static int skl_acquire_irq(struct hdac_ext_bus *ebus, int do_disconnect)
 	return 0;
 }
 
+static int skl_suspend_late(struct device *dev)
+{
+	struct pci_dev *pci = to_pci_dev(dev);
+	struct hdac_ext_bus *ebus = pci_get_drvdata(pci);
+	struct skl *skl = ebus_to_skl(ebus);
+
+	return skl_suspend_late_dsp(skl);
+}
+
 #ifdef CONFIG_PM
 static int _skl_suspend(struct hdac_ext_bus *ebus)
 {
@@ -243,7 +299,6 @@ static int skl_suspend(struct device *dev)
 
 		enable_irq_wake(bus->irq);
 		pci_save_state(pci);
-		pci_disable_device(pci);
 	} else {
 		ret = _skl_suspend(ebus);
 		if (ret < 0)
@@ -286,7 +341,6 @@ static int skl_resume(struct device *dev)
 	 */
 	if (skl->supend_active) {
 		pci_restore_state(pci);
-		ret = pci_enable_device(pci);
 		snd_hdac_ext_bus_link_power_up_all(ebus);
 		disable_irq_wake(bus->irq);
 		/*
@@ -345,6 +399,7 @@ static int skl_runtime_resume(struct device *dev)
 static const struct dev_pm_ops skl_pm = {
 	SET_SYSTEM_SLEEP_PM_OPS(skl_suspend, skl_resume)
 	SET_RUNTIME_PM_OPS(skl_runtime_suspend, skl_runtime_resume, NULL)
+	.suspend_late = skl_suspend_late,
 };
 
 /*
diff --git a/sound/soc/intel/skylake/skl.h b/sound/soc/intel/skylake/skl.h
index 5d4fbb094c48..4986e3929dd3 100644
--- a/sound/soc/intel/skylake/skl.h
+++ b/sound/soc/intel/skylake/skl.h
@@ -52,6 +52,9 @@
 #define AZX_PGCTL_LSRMD_MASK		(1 << 4)
 #define AZX_PCIREG_CGCTL		0x48
 #define AZX_CGCTL_MISCBDCGE_MASK	(1 << 6)
+/* D0I3C Register fields */
+#define AZX_REG_VS_D0I3C_CIP      0x1 /* Command in progress */
+#define AZX_REG_VS_D0I3C_I3       0x4 /* D0i3 enable */
 
 struct skl_dsp_resource {
 	u32 max_mcps;
@@ -121,8 +124,11 @@ int skl_get_dmic_geo(struct skl *skl);
 int skl_nhlt_update_topology_bin(struct skl *skl);
 int skl_init_dsp(struct skl *skl);
 int skl_free_dsp(struct skl *skl);
+int skl_suspend_late_dsp(struct skl *skl);
 int skl_suspend_dsp(struct skl *skl);
 int skl_resume_dsp(struct skl *skl);
 void skl_cleanup_resources(struct skl *skl);
 const struct skl_dsp_ops *skl_get_dsp_ops(int pci_id);
+void skl_update_d0i3c(struct device *dev, bool enable);
+
 #endif /* __SOUND_SOC_SKL_H */
diff --git a/sound/soc/kirkwood/armada-370-db.c b/sound/soc/kirkwood/armada-370-db.c
index e0304d544f26..677a48d7b891 100644
--- a/sound/soc/kirkwood/armada-370-db.c
+++ b/sound/soc/kirkwood/armada-370-db.c
@@ -42,7 +42,7 @@ static int a370db_hw_params(struct snd_pcm_substream *substream,
 	return snd_soc_dai_set_sysclk(codec_dai, 0, freq, SND_SOC_CLOCK_IN);
 }
 
-static struct snd_soc_ops a370db_ops = {
+static const struct snd_soc_ops a370db_ops = {
 	.hw_params = a370db_hw_params,
 };
 
diff --git a/sound/soc/mxs/mxs-saif.c b/sound/soc/mxs/mxs-saif.c
index 13631003cb7c..a002ab892772 100644
--- a/sound/soc/mxs/mxs-saif.c
+++ b/sound/soc/mxs/mxs-saif.c
@@ -735,6 +735,11 @@ static int mxs_saif_probe(struct platform_device *pdev)
 	else
 		saif->id = ret;
 
+	if (saif->id >= ARRAY_SIZE(mxs_saif)) {
+		dev_err(&pdev->dev, "get wrong saif id\n");
+		return -EINVAL;
+	}
+
 	/*
 	 * If there is no "fsl,saif-master" phandle, it's a saif
 	 * master.  Otherwise, it's a slave and its phandle points
@@ -749,11 +754,11 @@ static int mxs_saif_probe(struct platform_device *pdev)
 			return ret;
 		else
 			saif->master_id = ret;
-	}
 
-	if (saif->master_id >= ARRAY_SIZE(mxs_saif)) {
-		dev_err(&pdev->dev, "get wrong master id\n");
-		return -EINVAL;
+		if (saif->master_id >= ARRAY_SIZE(mxs_saif)) {
+			dev_err(&pdev->dev, "get wrong master id\n");
+			return -EINVAL;
+		}
 	}
 
 	mxs_saif[saif->id] = saif;
diff --git a/sound/soc/mxs/mxs-sgtl5000.c b/sound/soc/mxs/mxs-sgtl5000.c
index 2b23ffbac6b1..a96276e77332 100644
--- a/sound/soc/mxs/mxs-sgtl5000.c
+++ b/sound/soc/mxs/mxs-sgtl5000.c
@@ -68,7 +68,7 @@ static int mxs_sgtl5000_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static struct snd_soc_ops mxs_sgtl5000_hifi_ops = {
+static const struct snd_soc_ops mxs_sgtl5000_hifi_ops = {
 	.hw_params = mxs_sgtl5000_hw_params,
 };
 
diff --git a/sound/soc/pxa/corgi.c b/sound/soc/pxa/corgi.c
index dcbb7aa9830c..311774e9ca46 100644
--- a/sound/soc/pxa/corgi.c
+++ b/sound/soc/pxa/corgi.c
@@ -244,9 +244,9 @@ static const struct snd_soc_dapm_route corgi_audio_map[] = {
 	{"MICIN", NULL, "Line Jack"},
 };
 
-static const char *jack_function[] = {"Headphone", "Mic", "Line", "Headset",
-	"Off"};
-static const char *spk_function[] = {"On", "Off"};
+static const char * const jack_function[] = {"Headphone", "Mic", "Line",
+	"Headset", "Off"};
+static const char * const spk_function[] = {"On", "Off"};
 static const struct soc_enum corgi_enum[] = {
 	SOC_ENUM_SINGLE_EXT(5, jack_function),
 	SOC_ENUM_SINGLE_EXT(2, spk_function),
diff --git a/sound/soc/pxa/e740_wm9705.c b/sound/soc/pxa/e740_wm9705.c
index 1de876529aa1..086c37a85630 100644
--- a/sound/soc/pxa/e740_wm9705.c
+++ b/sound/soc/pxa/e740_wm9705.c
@@ -22,7 +22,6 @@
 
 #include <asm/mach-types.h>
 
-#include "../codecs/wm9705.h"
 #include "pxa2xx-ac97.h"
 
 
diff --git a/sound/soc/pxa/e750_wm9705.c b/sound/soc/pxa/e750_wm9705.c
index b7eb7cd5df7d..7823278012a6 100644
--- a/sound/soc/pxa/e750_wm9705.c
+++ b/sound/soc/pxa/e750_wm9705.c
@@ -22,7 +22,6 @@
 
 #include <asm/mach-types.h>
 
-#include "../codecs/wm9705.h"
 #include "pxa2xx-ac97.h"
 
 static int e750_spk_amp_event(struct snd_soc_dapm_widget *w,
diff --git a/sound/soc/pxa/e800_wm9712.c b/sound/soc/pxa/e800_wm9712.c
index 41bf71466a7b..07b9c6e17df9 100644
--- a/sound/soc/pxa/e800_wm9712.c
+++ b/sound/soc/pxa/e800_wm9712.c
@@ -21,7 +21,6 @@
 #include <mach/audio.h>
 #include <mach/eseries-gpio.h>
 
-#include "../codecs/wm9712.h"
 #include "pxa2xx-ac97.h"
 
 static int e800_spk_amp_event(struct snd_soc_dapm_widget *w,
diff --git a/sound/soc/pxa/em-x270.c b/sound/soc/pxa/em-x270.c
index 64743a05aeae..966163d1c813 100644
--- a/sound/soc/pxa/em-x270.c
+++ b/sound/soc/pxa/em-x270.c
@@ -30,7 +30,6 @@
 #include <asm/mach-types.h>
 #include <mach/audio.h>
 
-#include "../codecs/wm9712.h"
 #include "pxa2xx-ac97.h"
 
 static struct snd_soc_dai_link em_x270_dai[] = {
diff --git a/sound/soc/pxa/hx4700.c b/sound/soc/pxa/hx4700.c
index ecbf2873b7ff..85483049b916 100644
--- a/sound/soc/pxa/hx4700.c
+++ b/sound/soc/pxa/hx4700.c
@@ -27,8 +27,6 @@
 #include <asm/mach-types.h>
 #include "pxa2xx-i2s.h"
 
-#include "../codecs/ak4641.h"
-
 static struct snd_soc_jack hs_jack;
 
 /* Headphones jack detection DAPM pin */
diff --git a/sound/soc/pxa/magician.c b/sound/soc/pxa/magician.c
index 62b8377a9d2b..2d4d4455fe87 100644
--- a/sound/soc/pxa/magician.c
+++ b/sound/soc/pxa/magician.c
@@ -376,7 +376,7 @@ static const struct snd_soc_dapm_route audio_map[] = {
 	{"VINM", NULL, "Call Mic"},
 };
 
-static const char *input_select[] = {"Call Mic", "Headset Mic"};
+static const char * const input_select[] = {"Call Mic", "Headset Mic"};
 static const struct soc_enum magician_in_sel_enum =
 	SOC_ENUM_SINGLE_EXT(2, input_select);
 
diff --git a/sound/soc/pxa/mioa701_wm9713.c b/sound/soc/pxa/mioa701_wm9713.c
index d1661fa6ee08..0fe0abec8fc4 100644
--- a/sound/soc/pxa/mioa701_wm9713.c
+++ b/sound/soc/pxa/mioa701_wm9713.c
@@ -187,7 +187,7 @@ static int mioa701_wm9713_probe(struct platform_device *pdev)
 	mioa701.dev = &pdev->dev;
 	rc = devm_snd_soc_register_card(&pdev->dev, &mioa701);
 	if (!rc)
-		dev_warn(&pdev->dev, "Be warned that incorrect mixers/muxes setup will"
+		dev_warn(&pdev->dev, "Be warned that incorrect mixers/muxes setup will "
 			 "lead to overheating and possible destruction of your device."
 			 " Do not use without a good knowledge of mio's board design!\n");
 	return rc;
diff --git a/sound/soc/pxa/palm27x.c b/sound/soc/pxa/palm27x.c
index bcc81e920a67..387492d46b6c 100644
--- a/sound/soc/pxa/palm27x.c
+++ b/sound/soc/pxa/palm27x.c
@@ -27,7 +27,6 @@
 #include <mach/audio.h>
 #include <linux/platform_data/asoc-palm27x.h>
 
-#include "../codecs/wm9712.h"
 #include "pxa2xx-ac97.h"
 
 static struct snd_soc_jack hs_jack;
diff --git a/sound/soc/pxa/poodle.c b/sound/soc/pxa/poodle.c
index 4b3b714f5ee7..a879aba0691f 100644
--- a/sound/soc/pxa/poodle.c
+++ b/sound/soc/pxa/poodle.c
@@ -209,8 +209,8 @@ static const struct snd_soc_dapm_route poodle_audio_map[] = {
 	{"MICIN", NULL, "Microphone"},
 };
 
-static const char *jack_function[] = {"Off", "Headphone"};
-static const char *spk_function[] = {"Off", "On"};
+static const char * const jack_function[] = {"Off", "Headphone"};
+static const char * const spk_function[] = {"Off", "On"};
 static const struct soc_enum poodle_enum[] = {
 	SOC_ENUM_SINGLE_EXT(2, jack_function),
 	SOC_ENUM_SINGLE_EXT(2, spk_function),
diff --git a/sound/soc/pxa/pxa-ssp.h b/sound/soc/pxa/pxa-ssp.h
index bc79da221c0d..abf6ec080258 100644
--- a/sound/soc/pxa/pxa-ssp.h
+++ b/sound/soc/pxa/pxa-ssp.h
@@ -9,12 +9,6 @@
 #ifndef _PXA_SSP_H
 #define _PXA_SSP_H
 
-/* pxa DAI SSP IDs */
-#define PXA_DAI_SSP1			0
-#define PXA_DAI_SSP2			1
-#define PXA_DAI_SSP3			2
-#define PXA_DAI_SSP4			3
-
 /* SSP clock sources */
 #define PXA_SSP_CLK_PLL	0
 #define PXA_SSP_CLK_EXT	1
diff --git a/sound/soc/pxa/pxa2xx-i2s.h b/sound/soc/pxa/pxa2xx-i2s.h
index 070f3c6059fe..7e218e2105a9 100644
--- a/sound/soc/pxa/pxa2xx-i2s.h
+++ b/sound/soc/pxa/pxa2xx-i2s.h
@@ -9,9 +9,6 @@
 #ifndef _PXA2XX_I2S_H
 #define _PXA2XX_I2S_H
 
-/* pxa2xx DAI ID's */
-#define PXA2XX_DAI_I2S			0
-
 /* I2S clock */
 #define PXA2XX_I2S_SYSCLK		0
 
diff --git a/sound/soc/pxa/spitz.c b/sound/soc/pxa/spitz.c
index 0e02634c8b7f..07d77cddac60 100644
--- a/sound/soc/pxa/spitz.c
+++ b/sound/soc/pxa/spitz.c
@@ -241,9 +241,9 @@ static const struct snd_soc_dapm_route spitz_audio_map[] = {
 	{"LINPUT1", NULL, "Line Jack"},
 };
 
-static const char *jack_function[] = {"Headphone", "Mic", "Line", "Headset",
-	"Off"};
-static const char *spk_function[] = {"On", "Off"};
+static const char * const jack_function[] = {"Headphone", "Mic", "Line",
+	"Headset", "Off"};
+static const char * const spk_function[] = {"On", "Off"};
 static const struct soc_enum spitz_enum[] = {
 	SOC_ENUM_SINGLE_EXT(5, jack_function),
 	SOC_ENUM_SINGLE_EXT(2, spk_function),
diff --git a/sound/soc/pxa/tosa.c b/sound/soc/pxa/tosa.c
index c508f024ecfb..2e312c62e3c7 100644
--- a/sound/soc/pxa/tosa.c
+++ b/sound/soc/pxa/tosa.c
@@ -31,7 +31,6 @@
 #include <mach/tosa.h>
 #include <mach/audio.h>
 
-#include "../codecs/wm9712.h"
 #include "pxa2xx-ac97.h"
 
 #define TOSA_HP        0
@@ -170,9 +169,9 @@ static const struct snd_soc_dapm_route audio_map[] = {
 	{"Mic Bias", NULL, "Headset Jack"},
 };
 
-static const char *jack_function[] = {"Headphone", "Mic", "Line", "Headset",
-	"Off"};
-static const char *spk_function[] = {"On", "Off"};
+static const char * const jack_function[] = {"Headphone", "Mic", "Line",
+	"Headset", "Off"};
+static const char * const spk_function[] = {"On", "Off"};
 static const struct soc_enum tosa_enum[] = {
 	SOC_ENUM_SINGLE_EXT(5, jack_function),
 	SOC_ENUM_SINGLE_EXT(2, spk_function),
diff --git a/sound/soc/qcom/apq8016_sbc.c b/sound/soc/qcom/apq8016_sbc.c
index 07f91e918b23..d084d7468299 100644
--- a/sound/soc/qcom/apq8016_sbc.c
+++ b/sound/soc/qcom/apq8016_sbc.c
@@ -123,20 +123,15 @@ static struct apq8016_sbc_data *apq8016_sbc_parse_of(struct snd_soc_card *card)
 			return ERR_PTR(-EINVAL);
 		}
 
-		link->codec_of_node = of_parse_phandle(codec, "sound-dai", 0);
-		if (!link->codec_of_node) {
-			dev_err(card->dev, "error getting codec phandle\n");
-			return ERR_PTR(-EINVAL);
-		}
-
 		ret = snd_soc_of_get_dai_name(cpu, &link->cpu_dai_name);
 		if (ret) {
 			dev_err(card->dev, "error getting cpu dai name\n");
 			return ERR_PTR(ret);
 		}
 
-		ret = snd_soc_of_get_dai_name(codec, &link->codec_dai_name);
-		if (ret) {
+		ret = snd_soc_of_get_dai_link_codecs(dev, codec, link);
+
+		if (ret < 0) {
 			dev_err(card->dev, "error getting codec dai name\n");
 			return ERR_PTR(ret);
 		}
diff --git a/sound/soc/qcom/lpass-platform.c b/sound/soc/qcom/lpass-platform.c
index b392e51de94d..dd5bdd0da730 100644
--- a/sound/soc/qcom/lpass-platform.c
+++ b/sound/soc/qcom/lpass-platform.c
@@ -25,8 +25,7 @@
 #include "lpass.h"
 
 struct lpass_pcm_data {
-	int rdma_ch;
-	int wrdma_ch;
+	int dma_ch;
 	int i2s_port;
 };
 
@@ -78,6 +77,9 @@ static int lpass_platform_pcmops_open(struct snd_pcm_substream *substream)
 	dma_ch = 0;
 	if (v->alloc_dma_channel)
 		dma_ch = v->alloc_dma_channel(drvdata, dir);
+	else
+		dma_ch = 0;
+
 	if (dma_ch < 0)
 		return dma_ch;
 
@@ -92,10 +94,7 @@ static int lpass_platform_pcmops_open(struct snd_pcm_substream *substream)
 			return ret;
 	}
 
-	if (dir == SNDRV_PCM_STREAM_PLAYBACK)
-		data->rdma_ch = dma_ch;
-	else
-		data->wrdma_ch = dma_ch;
+	data->dma_ch = dma_ch;
 
 	snd_soc_set_runtime_hwparams(substream, &lpass_platform_pcm_hardware);
 
@@ -122,20 +121,12 @@ static int lpass_platform_pcmops_close(struct snd_pcm_substream *substream)
 		snd_soc_platform_get_drvdata(soc_runtime->platform);
 	struct lpass_variant *v = drvdata->variant;
 	struct lpass_pcm_data *data;
-	int dma_ch, dir = substream->stream;
 
 	data = runtime->private_data;
 	v = drvdata->variant;
-
-	if (dir == SNDRV_PCM_STREAM_PLAYBACK)
-		dma_ch = data->rdma_ch;
-	else
-		dma_ch = data->wrdma_ch;
-
-	drvdata->substream[dma_ch] = NULL;
-
+	drvdata->substream[data->dma_ch] = NULL;
 	if (v->free_dma_channel)
-		v->free_dma_channel(drvdata, dma_ch);
+		v->free_dma_channel(drvdata, data->dma_ch);
 
 	return 0;
 }
@@ -156,10 +147,7 @@ static int lpass_platform_pcmops_hw_params(struct snd_pcm_substream *substream,
 	int bitwidth;
 	int ret, dma_port = pcm_data->i2s_port + v->dmactl_audif_start;
 
-	if (dir ==  SNDRV_PCM_STREAM_PLAYBACK)
-		ch = pcm_data->rdma_ch;
-	else
-		ch = pcm_data->wrdma_ch;
+	ch = pcm_data->dma_ch;
 
 	bitwidth = snd_pcm_format_width(format);
 	if (bitwidth < 0) {
@@ -246,11 +234,7 @@ static int lpass_platform_pcmops_hw_free(struct snd_pcm_substream *substream)
 	unsigned int reg;
 	int ret;
 
-	if (substream->stream ==  SNDRV_PCM_STREAM_PLAYBACK)
-		reg = LPAIF_RDMACTL_REG(v, pcm_data->rdma_ch);
-	else
-		reg = LPAIF_WRDMACTL_REG(v, pcm_data->wrdma_ch);
-
+	reg = LPAIF_DMACTL_REG(v, pcm_data->dma_ch, substream->stream);
 	ret = regmap_write(drvdata->lpaif_map, reg, 0);
 	if (ret)
 		dev_err(soc_runtime->dev, "%s() error writing to rdmactl reg: %d\n",
@@ -270,10 +254,7 @@ static int lpass_platform_pcmops_prepare(struct snd_pcm_substream *substream)
 	struct lpass_variant *v = drvdata->variant;
 	int ret, ch, dir = substream->stream;
 
-	if (dir ==  SNDRV_PCM_STREAM_PLAYBACK)
-		ch = pcm_data->rdma_ch;
-	else
-		ch = pcm_data->wrdma_ch;
+	ch = pcm_data->dma_ch;
 
 	ret = regmap_write(drvdata->lpaif_map,
 			LPAIF_DMABASE_REG(v, ch, dir),
@@ -325,10 +306,7 @@ static int lpass_platform_pcmops_trigger(struct snd_pcm_substream *substream,
 	struct lpass_variant *v = drvdata->variant;
 	int ret, ch, dir = substream->stream;
 
-	if (dir == SNDRV_PCM_STREAM_PLAYBACK)
-		ch = pcm_data->rdma_ch;
-	else
-		ch = pcm_data->wrdma_ch;
+	ch = pcm_data->dma_ch;
 
 	switch (cmd) {
 	case SNDRV_PCM_TRIGGER_START:
@@ -403,10 +381,7 @@ static snd_pcm_uframes_t lpass_platform_pcmops_pointer(
 	unsigned int base_addr, curr_addr;
 	int ret, ch, dir = substream->stream;
 
-	if (dir == SNDRV_PCM_STREAM_PLAYBACK)
-		ch = pcm_data->rdma_ch;
-	else
-		ch = pcm_data->wrdma_ch;
+	ch = pcm_data->dma_ch;
 
 	ret = regmap_read(drvdata->lpaif_map,
 			LPAIF_DMABASE_REG(v, ch, dir), &base_addr);
diff --git a/sound/soc/qcom/storm.c b/sound/soc/qcom/storm.c
index 2d833bffdba0..8fcac2ac3aa6 100644
--- a/sound/soc/qcom/storm.c
+++ b/sound/soc/qcom/storm.c
@@ -58,7 +58,7 @@ static int storm_ops_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static struct snd_soc_ops storm_soc_ops = {
+static const struct snd_soc_ops storm_soc_ops = {
 	.hw_params	= storm_ops_hw_params,
 };
 
diff --git a/sound/soc/rockchip/rk3399_gru_sound.c b/sound/soc/rockchip/rk3399_gru_sound.c
index 9ed735a6cf49..3475c61a5fa0 100644
--- a/sound/soc/rockchip/rk3399_gru_sound.c
+++ b/sound/soc/rockchip/rk3399_gru_sound.c
@@ -38,7 +38,7 @@
 
 #define SOUND_FS	256
 
-unsigned int rt5514_dmic_delay;
+static unsigned int rt5514_dmic_delay;
 
 static struct snd_soc_jack rockchip_sound_jack;
 
@@ -228,15 +228,15 @@ static int rockchip_sound_da7219_init(struct snd_soc_pcm_runtime *rtd)
 	return 0;
 }
 
-static struct snd_soc_ops rockchip_sound_max98357a_ops = {
+static const struct snd_soc_ops rockchip_sound_max98357a_ops = {
 	.hw_params = rockchip_sound_max98357a_hw_params,
 };
 
-static struct snd_soc_ops rockchip_sound_rt5514_ops = {
+static const struct snd_soc_ops rockchip_sound_rt5514_ops = {
 	.hw_params = rockchip_sound_rt5514_hw_params,
 };
 
-static struct snd_soc_ops rockchip_sound_da7219_ops = {
+static const struct snd_soc_ops rockchip_sound_da7219_ops = {
 	.hw_params = rockchip_sound_da7219_hw_params,
 };
 
diff --git a/sound/soc/rockchip/rockchip_max98090.c b/sound/soc/rockchip/rockchip_max98090.c
index e70ffad07184..789d6f1e2b5f 100644
--- a/sound/soc/rockchip/rockchip_max98090.c
+++ b/sound/soc/rockchip/rockchip_max98090.c
@@ -119,7 +119,7 @@ static int rk_aif1_hw_params(struct snd_pcm_substream *substream,
 	return ret;
 }
 
-static struct snd_soc_ops rk_aif1_ops = {
+static const struct snd_soc_ops rk_aif1_ops = {
 	.hw_params = rk_aif1_hw_params,
 };
 
diff --git a/sound/soc/rockchip/rockchip_rt5645.c b/sound/soc/rockchip/rockchip_rt5645.c
index 440a8026346a..9e0c17805807 100644
--- a/sound/soc/rockchip/rockchip_rt5645.c
+++ b/sound/soc/rockchip/rockchip_rt5645.c
@@ -135,7 +135,7 @@ static int rk_init(struct snd_soc_pcm_runtime *runtime)
 				     &headset_jack);
 }
 
-static struct snd_soc_ops rk_aif1_ops = {
+static const struct snd_soc_ops rk_aif1_ops = {
 	.hw_params = rk_aif1_hw_params,
 };
 
diff --git a/sound/soc/samsung/Kconfig b/sound/soc/samsung/Kconfig
index f6023b46c107..7c423151ef7d 100644
--- a/sound/soc/samsung/Kconfig
+++ b/sound/soc/samsung/Kconfig
@@ -1,6 +1,7 @@
 menuconfig SND_SOC_SAMSUNG
 	tristate "ASoC support for Samsung"
-	depends on (PLAT_SAMSUNG || ARCH_EXYNOS)
+	depends on PLAT_SAMSUNG || ARCH_EXYNOS || COMPILE_TEST
+	depends on COMMON_CLK
 	select SND_SOC_GENERIC_DMAENGINE_PCM
 	---help---
 	  Say Y or M if you want to add support for codecs attached to
@@ -22,10 +23,6 @@ config SND_S3C2412_SOC_I2S
 config SND_SAMSUNG_PCM
 	tristate "Samsung PCM interface support"
 
-config SND_SAMSUNG_AC97
-	tristate
-	select SND_SOC_AC97_BUS
-
 config SND_SAMSUNG_SPDIF
 	tristate "Samsung SPDIF transmitter support"
 	select SND_SOC_SPDIF
@@ -53,7 +50,7 @@ config SND_SOC_SAMSUNG_JIVE_WM8750
 
 config SND_SOC_SAMSUNG_SMDK_WM8580
 	tristate "SoC I2S Audio support for WM8580 on SMDK"
-	depends on MACH_SMDK6410 || MACH_SMDKC100 || MACH_SMDKV210 || MACH_SMDKC110
+	depends on MACH_SMDK6410 || COMPILE_TEST
 	depends on I2C
 	select SND_SOC_WM8580
 	select SND_SAMSUNG_I2S
@@ -69,26 +66,6 @@ config SND_SOC_SAMSUNG_SMDK_WM8994
 	help
 		Say Y if you want to add support for SoC audio on the SMDKs.
 
-config SND_SOC_SAMSUNG_SMDK2443_WM9710
-	tristate "SoC AC97 Audio support for SMDK2443 - WM9710"
-	depends on MACH_SMDK2443
-	select AC97_BUS
-	select SND_SOC_AC97_CODEC
-	select SND_SAMSUNG_AC97
-	help
-	  Say Y if you want to add support for SoC audio on smdk2443
-	  with the WM9710.
-
-config SND_SOC_SAMSUNG_LN2440SBC_ALC650
-	tristate "SoC AC97 Audio support for LN2440SBC - ALC650"
-	depends on ARCH_S3C24XX
-	select AC97_BUS
-	select SND_SOC_AC97_CODEC
-	select SND_SAMSUNG_AC97
-	help
-	  Say Y if you want to add support for SoC audio on ln2440sbc
-	  with the ALC650.
-
 config SND_SOC_SAMSUNG_S3C24XX_UDA134X
 	tristate "SoC I2S Audio support UDA134X wired to a S3C24XX"
 	depends on ARCH_S3C24XX
@@ -131,17 +108,10 @@ config SND_SOC_SAMSUNG_RX1950_UDA1380
 	help
 	  This driver provides audio support for HP iPAQ RX1950 PDA.
 
-config SND_SOC_SAMSUNG_SMDK_WM9713
-	tristate "SoC AC97 Audio support for SMDK with WM9713"
-	depends on MACH_SMDK6410 || MACH_SMDKC100 || MACH_SMDKV210 || MACH_SMDKC110
-	select SND_SOC_WM9713
-	select SND_SAMSUNG_AC97
-	help
-	  Say Y if you want to add support for SoC audio on the SMDK.
-
 config SND_SOC_SMARTQ
 	tristate "SoC I2S Audio support for SmartQ board"
-	depends on MACH_SMARTQ && I2C
+	depends on MACH_SMARTQ || COMPILE_TEST
+	depends on I2C
 	select SND_SAMSUNG_I2S
 	select SND_SOC_WM8750
 
@@ -151,15 +121,6 @@ config SND_SOC_SAMSUNG_SMDK_SPDIF
 	help
 	  Say Y if you want to add support for SoC S/PDIF audio on the SMDK.
 
-config SND_SOC_SMDK_WM8580_PCM
-	tristate "SoC PCM Audio support for WM8580 on SMDK"
-	depends on MACH_SMDKV210 || MACH_SMDKC110
-	depends on I2C
-	select SND_SOC_WM8580
-	select SND_SAMSUNG_PCM
-	help
-	  Say Y if you want to add support for SoC audio on the SMDK.
-
 config SND_SOC_SMDK_WM8994_PCM
 	tristate "SoC PCM Audio support for WM8994 on SMDK"
 	depends on I2C=y
@@ -229,4 +190,13 @@ config SND_SOC_ARNDALE_RT5631_ALC5631
         select SND_SAMSUNG_I2S
         select SND_SOC_RT5631
 
+config SND_SOC_SAMSUNG_TM2_WM5110
+	tristate "SoC I2S Audio support for WM5110 on TM2 board"
+	depends on SND_SOC_SAMSUNG && MFD_ARIZONA && I2C && SPI_MASTER
+	select SND_SOC_MAX98504
+	select SND_SOC_WM5110
+	select SND_SAMSUNG_I2S
+	help
+	  Say Y if you want to add support for SoC audio on the TM2 board.
+
 endif #SND_SOC_SAMSUNG
diff --git a/sound/soc/samsung/Makefile b/sound/soc/samsung/Makefile
index 5d03f5ce6916..b5df5e2e3d94 100644
--- a/sound/soc/samsung/Makefile
+++ b/sound/soc/samsung/Makefile
@@ -3,7 +3,6 @@ snd-soc-s3c-dma-objs := dmaengine.o
 snd-soc-idma-objs := idma.o
 snd-soc-s3c24xx-i2s-objs := s3c24xx-i2s.o
 snd-soc-s3c2412-i2s-objs := s3c2412-i2s.o
-snd-soc-ac97-objs := ac97.o
 snd-soc-s3c-i2s-v2-objs := s3c-i2s-v2.o
 snd-soc-samsung-spdif-objs := spdif.o
 snd-soc-pcm-objs := pcm.o
@@ -11,7 +10,6 @@ snd-soc-i2s-objs := i2s.o
 
 obj-$(CONFIG_SND_SOC_SAMSUNG) += snd-soc-s3c-dma.o
 obj-$(CONFIG_SND_S3C24XX_I2S) += snd-soc-s3c24xx-i2s.o
-obj-$(CONFIG_SND_SAMSUNG_AC97) += snd-soc-ac97.o
 obj-$(CONFIG_SND_S3C2412_SOC_I2S) += snd-soc-s3c2412-i2s.o
 obj-$(CONFIG_SND_S3C_I2SV2_SOC) += snd-soc-s3c-i2s-v2.o
 obj-$(CONFIG_SND_SAMSUNG_SPDIF) += snd-soc-samsung-spdif.o
@@ -36,7 +34,6 @@ snd-soc-snow-objs := snow.o
 snd-soc-smdk-wm9713-objs := smdk_wm9713.o
 snd-soc-s3c64xx-smartq-wm8987-objs := smartq_wm8987.o
 snd-soc-smdk-spdif-objs := smdk_spdif.o
-snd-soc-smdk-wm8580pcm-objs := smdk_wm8580pcm.o
 snd-soc-smdk-wm8994pcm-objs := smdk_wm8994pcm.o
 snd-soc-speyside-objs := speyside.o
 snd-soc-tobermory-objs := tobermory.o
@@ -44,11 +41,10 @@ snd-soc-lowland-objs := lowland.o
 snd-soc-littlemill-objs := littlemill.o
 snd-soc-bells-objs := bells.o
 snd-soc-arndale-rt5631-objs := arndale_rt5631.o
+snd-soc-tm2-wm5110-objs := tm2_wm5110.o
 
 obj-$(CONFIG_SND_SOC_SAMSUNG_JIVE_WM8750) += snd-soc-jive-wm8750.o
 obj-$(CONFIG_SND_SOC_SAMSUNG_NEO1973_WM8753) += snd-soc-neo1973-wm8753.o
-obj-$(CONFIG_SND_SOC_SAMSUNG_SMDK2443_WM9710) += snd-soc-smdk2443-wm9710.o
-obj-$(CONFIG_SND_SOC_SAMSUNG_LN2440SBC_ALC650) += snd-soc-ln2440sbc-alc650.o
 obj-$(CONFIG_SND_SOC_SAMSUNG_S3C24XX_UDA134X) += snd-soc-s3c24xx-uda134x.o
 obj-$(CONFIG_SND_SOC_SAMSUNG_SIMTEC) += snd-soc-s3c24xx-simtec.o
 obj-$(CONFIG_SND_SOC_SAMSUNG_SIMTEC_HERMES) += snd-soc-s3c24xx-simtec-hermes.o
@@ -58,10 +54,8 @@ obj-$(CONFIG_SND_SOC_SAMSUNG_RX1950_UDA1380) += snd-soc-rx1950-uda1380.o
 obj-$(CONFIG_SND_SOC_SAMSUNG_SMDK_WM8580) += snd-soc-smdk-wm8580.o
 obj-$(CONFIG_SND_SOC_SAMSUNG_SMDK_WM8994) += snd-soc-smdk-wm8994.o
 obj-$(CONFIG_SND_SOC_SNOW) += snd-soc-snow.o
-obj-$(CONFIG_SND_SOC_SAMSUNG_SMDK_WM9713) += snd-soc-smdk-wm9713.o
 obj-$(CONFIG_SND_SOC_SMARTQ) += snd-soc-s3c64xx-smartq-wm8987.o
 obj-$(CONFIG_SND_SOC_SAMSUNG_SMDK_SPDIF) += snd-soc-smdk-spdif.o
-obj-$(CONFIG_SND_SOC_SMDK_WM8580_PCM) += snd-soc-smdk-wm8580pcm.o
 obj-$(CONFIG_SND_SOC_SMDK_WM8994_PCM) += snd-soc-smdk-wm8994pcm.o
 obj-$(CONFIG_SND_SOC_SPEYSIDE) += snd-soc-speyside.o
 obj-$(CONFIG_SND_SOC_TOBERMORY) += snd-soc-tobermory.o
@@ -69,3 +63,4 @@ obj-$(CONFIG_SND_SOC_LOWLAND) += snd-soc-lowland.o
 obj-$(CONFIG_SND_SOC_LITTLEMILL) += snd-soc-littlemill.o
 obj-$(CONFIG_SND_SOC_BELLS) += snd-soc-bells.o
 obj-$(CONFIG_SND_SOC_ARNDALE_RT5631_ALC5631) += snd-soc-arndale-rt5631.o
+obj-$(CONFIG_SND_SOC_SAMSUNG_TM2_WM5110) += snd-soc-tm2-wm5110.o
diff --git a/sound/soc/samsung/ac97.c b/sound/soc/samsung/ac97.c
deleted file mode 100644
index cbc0023c2bc8..000000000000
--- a/sound/soc/samsung/ac97.c
+++ /dev/null
@@ -1,437 +0,0 @@
-/* sound/soc/samsung/ac97.c
- *
- * ALSA SoC Audio Layer - S3C AC97 Controller driver
- * 	Evolved from s3c2443-ac97.c
- *
- * Copyright (c) 2010 Samsung Electronics Co. Ltd
- *	Author: Jaswinder Singh <jassisinghbrar@gmail.com>
- * 	Credits: Graeme Gregory, Sean Choi
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/io.h>
-#include <linux/delay.h>
-#include <linux/clk.h>
-#include <linux/module.h>
-
-#include <sound/soc.h>
-
-#include "regs-ac97.h"
-#include <linux/platform_data/asoc-s3c.h>
-
-#include "dma.h"
-
-#define AC_CMD_ADDR(x) (x << 16)
-#define AC_CMD_DATA(x) (x & 0xffff)
-
-#define S3C_AC97_DAI_PCM 0
-#define S3C_AC97_DAI_MIC 1
-
-struct s3c_ac97_info {
-	struct clk         *ac97_clk;
-	void __iomem	   *regs;
-	struct mutex       lock;
-	struct completion  done;
-};
-static struct s3c_ac97_info s3c_ac97;
-
-static struct snd_dmaengine_dai_dma_data s3c_ac97_pcm_out = {
-	.addr_width	= 4,
-};
-
-static struct snd_dmaengine_dai_dma_data s3c_ac97_pcm_in = {
-	.addr_width	= 4,
-};
-
-static struct snd_dmaengine_dai_dma_data s3c_ac97_mic_in = {
-	.addr_width	= 4,
-};
-
-static void s3c_ac97_activate(struct snd_ac97 *ac97)
-{
-	u32 ac_glbctrl, stat;
-
-	stat = readl(s3c_ac97.regs + S3C_AC97_GLBSTAT) & 0x7;
-	if (stat == S3C_AC97_GLBSTAT_MAINSTATE_ACTIVE)
-		return; /* Return if already active */
-
-	reinit_completion(&s3c_ac97.done);
-
-	ac_glbctrl = readl(s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	ac_glbctrl = S3C_AC97_GLBCTRL_ACLINKON;
-	writel(ac_glbctrl, s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	msleep(1);
-
-	ac_glbctrl |= S3C_AC97_GLBCTRL_TRANSFERDATAENABLE;
-	writel(ac_glbctrl, s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	msleep(1);
-
-	ac_glbctrl = readl(s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	ac_glbctrl |= S3C_AC97_GLBCTRL_CODECREADYIE;
-	writel(ac_glbctrl, s3c_ac97.regs + S3C_AC97_GLBCTRL);
-
-	if (!wait_for_completion_timeout(&s3c_ac97.done, HZ))
-		pr_err("AC97: Unable to activate!\n");
-}
-
-static unsigned short s3c_ac97_read(struct snd_ac97 *ac97,
-	unsigned short reg)
-{
-	u32 ac_glbctrl, ac_codec_cmd;
-	u32 stat, addr, data;
-
-	mutex_lock(&s3c_ac97.lock);
-
-	s3c_ac97_activate(ac97);
-
-	reinit_completion(&s3c_ac97.done);
-
-	ac_codec_cmd = readl(s3c_ac97.regs + S3C_AC97_CODEC_CMD);
-	ac_codec_cmd = S3C_AC97_CODEC_CMD_READ | AC_CMD_ADDR(reg);
-	writel(ac_codec_cmd, s3c_ac97.regs + S3C_AC97_CODEC_CMD);
-
-	udelay(50);
-
-	ac_glbctrl = readl(s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	ac_glbctrl |= S3C_AC97_GLBCTRL_CODECREADYIE;
-	writel(ac_glbctrl, s3c_ac97.regs + S3C_AC97_GLBCTRL);
-
-	if (!wait_for_completion_timeout(&s3c_ac97.done, HZ))
-		pr_err("AC97: Unable to read!\n");
-
-	stat = readl(s3c_ac97.regs + S3C_AC97_STAT);
-	addr = (stat >> 16) & 0x7f;
-	data = (stat & 0xffff);
-
-	if (addr != reg)
-		pr_err("ac97: req addr = %02x, rep addr = %02x\n",
-			reg, addr);
-
-	mutex_unlock(&s3c_ac97.lock);
-
-	return (unsigned short)data;
-}
-
-static void s3c_ac97_write(struct snd_ac97 *ac97, unsigned short reg,
-	unsigned short val)
-{
-	u32 ac_glbctrl, ac_codec_cmd;
-
-	mutex_lock(&s3c_ac97.lock);
-
-	s3c_ac97_activate(ac97);
-
-	reinit_completion(&s3c_ac97.done);
-
-	ac_codec_cmd = readl(s3c_ac97.regs + S3C_AC97_CODEC_CMD);
-	ac_codec_cmd = AC_CMD_ADDR(reg) | AC_CMD_DATA(val);
-	writel(ac_codec_cmd, s3c_ac97.regs + S3C_AC97_CODEC_CMD);
-
-	udelay(50);
-
-	ac_glbctrl = readl(s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	ac_glbctrl |= S3C_AC97_GLBCTRL_CODECREADYIE;
-	writel(ac_glbctrl, s3c_ac97.regs + S3C_AC97_GLBCTRL);
-
-	if (!wait_for_completion_timeout(&s3c_ac97.done, HZ))
-		pr_err("AC97: Unable to write!\n");
-
-	ac_codec_cmd = readl(s3c_ac97.regs + S3C_AC97_CODEC_CMD);
-	ac_codec_cmd |= S3C_AC97_CODEC_CMD_READ;
-	writel(ac_codec_cmd, s3c_ac97.regs + S3C_AC97_CODEC_CMD);
-
-	mutex_unlock(&s3c_ac97.lock);
-}
-
-static void s3c_ac97_cold_reset(struct snd_ac97 *ac97)
-{
-	pr_debug("AC97: Cold reset\n");
-	writel(S3C_AC97_GLBCTRL_COLDRESET,
-			s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	msleep(1);
-
-	writel(0, s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	msleep(1);
-}
-
-static void s3c_ac97_warm_reset(struct snd_ac97 *ac97)
-{
-	u32 stat;
-
-	stat = readl(s3c_ac97.regs + S3C_AC97_GLBSTAT) & 0x7;
-	if (stat == S3C_AC97_GLBSTAT_MAINSTATE_ACTIVE)
-		return; /* Return if already active */
-
-	pr_debug("AC97: Warm reset\n");
-
-	writel(S3C_AC97_GLBCTRL_WARMRESET, s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	msleep(1);
-
-	writel(0, s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	msleep(1);
-
-	s3c_ac97_activate(ac97);
-}
-
-static irqreturn_t s3c_ac97_irq(int irq, void *dev_id)
-{
-	u32 ac_glbctrl, ac_glbstat;
-
-	ac_glbstat = readl(s3c_ac97.regs + S3C_AC97_GLBSTAT);
-
-	if (ac_glbstat & S3C_AC97_GLBSTAT_CODECREADY) {
-
-		ac_glbctrl = readl(s3c_ac97.regs + S3C_AC97_GLBCTRL);
-		ac_glbctrl &= ~S3C_AC97_GLBCTRL_CODECREADYIE;
-		writel(ac_glbctrl, s3c_ac97.regs + S3C_AC97_GLBCTRL);
-
-		complete(&s3c_ac97.done);
-	}
-
-	ac_glbctrl = readl(s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	ac_glbctrl |= (1<<30); /* Clear interrupt */
-	writel(ac_glbctrl, s3c_ac97.regs + S3C_AC97_GLBCTRL);
-
-	return IRQ_HANDLED;
-}
-
-static struct snd_ac97_bus_ops s3c_ac97_ops = {
-	.read       = s3c_ac97_read,
-	.write      = s3c_ac97_write,
-	.warm_reset = s3c_ac97_warm_reset,
-	.reset      = s3c_ac97_cold_reset,
-};
-
-static int s3c_ac97_trigger(struct snd_pcm_substream *substream, int cmd,
-				struct snd_soc_dai *dai)
-{
-	u32 ac_glbctrl;
-
-	ac_glbctrl = readl(s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	if (substream->stream == SNDRV_PCM_STREAM_CAPTURE)
-		ac_glbctrl &= ~S3C_AC97_GLBCTRL_PCMINTM_MASK;
-	else
-		ac_glbctrl &= ~S3C_AC97_GLBCTRL_PCMOUTTM_MASK;
-
-	switch (cmd) {
-	case SNDRV_PCM_TRIGGER_START:
-	case SNDRV_PCM_TRIGGER_RESUME:
-	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
-		if (substream->stream == SNDRV_PCM_STREAM_CAPTURE)
-			ac_glbctrl |= S3C_AC97_GLBCTRL_PCMINTM_DMA;
-		else
-			ac_glbctrl |= S3C_AC97_GLBCTRL_PCMOUTTM_DMA;
-		break;
-
-	case SNDRV_PCM_TRIGGER_STOP:
-	case SNDRV_PCM_TRIGGER_SUSPEND:
-	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
-		break;
-	}
-
-	writel(ac_glbctrl, s3c_ac97.regs + S3C_AC97_GLBCTRL);
-
-	return 0;
-}
-
-static int s3c_ac97_mic_trigger(struct snd_pcm_substream *substream,
-				    int cmd, struct snd_soc_dai *dai)
-{
-	u32 ac_glbctrl;
-
-	ac_glbctrl = readl(s3c_ac97.regs + S3C_AC97_GLBCTRL);
-	ac_glbctrl &= ~S3C_AC97_GLBCTRL_MICINTM_MASK;
-
-	switch (cmd) {
-	case SNDRV_PCM_TRIGGER_START:
-	case SNDRV_PCM_TRIGGER_RESUME:
-	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
-		ac_glbctrl |= S3C_AC97_GLBCTRL_MICINTM_DMA;
-		break;
-
-	case SNDRV_PCM_TRIGGER_STOP:
-	case SNDRV_PCM_TRIGGER_SUSPEND:
-	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
-		break;
-	}
-
-	writel(ac_glbctrl, s3c_ac97.regs + S3C_AC97_GLBCTRL);
-
-	return 0;
-}
-
-static const struct snd_soc_dai_ops s3c_ac97_dai_ops = {
-	.trigger	= s3c_ac97_trigger,
-};
-
-static const struct snd_soc_dai_ops s3c_ac97_mic_dai_ops = {
-	.trigger	= s3c_ac97_mic_trigger,
-};
-
-static int s3c_ac97_dai_probe(struct snd_soc_dai *dai)
-{
-	snd_soc_dai_init_dma_data(dai, &s3c_ac97_pcm_out, &s3c_ac97_pcm_in);
-
-	return 0;
-}
-
-static int s3c_ac97_mic_dai_probe(struct snd_soc_dai *dai)
-{
-	snd_soc_dai_init_dma_data(dai, NULL, &s3c_ac97_mic_in);
-
-	return 0;
-}
-
-static struct snd_soc_dai_driver s3c_ac97_dai[] = {
-	[S3C_AC97_DAI_PCM] = {
-		.name =	"samsung-ac97",
-		.bus_control = true,
-		.playback = {
-			.stream_name = "AC97 Playback",
-			.channels_min = 2,
-			.channels_max = 2,
-			.rates = SNDRV_PCM_RATE_8000_48000,
-			.formats = SNDRV_PCM_FMTBIT_S16_LE,},
-		.capture = {
-			.stream_name = "AC97 Capture",
-			.channels_min = 2,
-			.channels_max = 2,
-			.rates = SNDRV_PCM_RATE_8000_48000,
-			.formats = SNDRV_PCM_FMTBIT_S16_LE,},
-		.probe = s3c_ac97_dai_probe,
-		.ops = &s3c_ac97_dai_ops,
-	},
-	[S3C_AC97_DAI_MIC] = {
-		.name = "samsung-ac97-mic",
-		.bus_control = true,
-		.capture = {
-			.stream_name = "AC97 Mic Capture",
-			.channels_min = 1,
-			.channels_max = 1,
-			.rates = SNDRV_PCM_RATE_8000_48000,
-			.formats = SNDRV_PCM_FMTBIT_S16_LE,},
-		.probe = s3c_ac97_mic_dai_probe,
-		.ops = &s3c_ac97_mic_dai_ops,
-	},
-};
-
-static const struct snd_soc_component_driver s3c_ac97_component = {
-	.name		= "s3c-ac97",
-};
-
-static int s3c_ac97_probe(struct platform_device *pdev)
-{
-	struct resource *mem_res, *irq_res;
-	struct s3c_audio_pdata *ac97_pdata;
-	int ret;
-
-	ac97_pdata = pdev->dev.platform_data;
-	if (!ac97_pdata || !ac97_pdata->cfg_gpio) {
-		dev_err(&pdev->dev, "cfg_gpio callback not provided!\n");
-		return -EINVAL;
-	}
-
-	/* Check for availability of necessary resource */
-	irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-	if (!irq_res) {
-		dev_err(&pdev->dev, "AC97 IRQ not provided!\n");
-		return -ENXIO;
-	}
-
-	mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	s3c_ac97.regs = devm_ioremap_resource(&pdev->dev, mem_res);
-	if (IS_ERR(s3c_ac97.regs))
-		return PTR_ERR(s3c_ac97.regs);
-
-	s3c_ac97_pcm_out.filter_data = ac97_pdata->dma_playback;
-	s3c_ac97_pcm_out.addr = mem_res->start + S3C_AC97_PCM_DATA;
-	s3c_ac97_pcm_in.filter_data = ac97_pdata->dma_capture;
-	s3c_ac97_pcm_in.addr = mem_res->start + S3C_AC97_PCM_DATA;
-	s3c_ac97_mic_in.filter_data = ac97_pdata->dma_capture_mic;
-	s3c_ac97_mic_in.addr = mem_res->start + S3C_AC97_MIC_DATA;
-
-	init_completion(&s3c_ac97.done);
-	mutex_init(&s3c_ac97.lock);
-
-	s3c_ac97.ac97_clk = devm_clk_get(&pdev->dev, "ac97");
-	if (IS_ERR(s3c_ac97.ac97_clk)) {
-		dev_err(&pdev->dev, "ac97 failed to get ac97_clock\n");
-		ret = -ENODEV;
-		goto err2;
-	}
-	clk_prepare_enable(s3c_ac97.ac97_clk);
-
-	if (ac97_pdata->cfg_gpio(pdev)) {
-		dev_err(&pdev->dev, "Unable to configure gpio\n");
-		ret = -EINVAL;
-		goto err3;
-	}
-
-	ret = request_irq(irq_res->start, s3c_ac97_irq,
-					0, "AC97", NULL);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "ac97: interrupt request failed.\n");
-		goto err4;
-	}
-
-	ret = snd_soc_set_ac97_ops(&s3c_ac97_ops);
-	if (ret != 0) {
-		dev_err(&pdev->dev, "Failed to set AC'97 ops: %d\n", ret);
-		goto err4;
-	}
-
-	ret = samsung_asoc_dma_platform_register(&pdev->dev,
-						 ac97_pdata->dma_filter,
-						 NULL, NULL);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to get register DMA: %d\n", ret);
-		goto err5;
-	}
-
-	ret = devm_snd_soc_register_component(&pdev->dev, &s3c_ac97_component,
-					 s3c_ac97_dai, ARRAY_SIZE(s3c_ac97_dai));
-	if (ret)
-		goto err5;
-
-	return 0;
-err5:
-	free_irq(irq_res->start, NULL);
-err4:
-err3:
-	clk_disable_unprepare(s3c_ac97.ac97_clk);
-err2:
-	snd_soc_set_ac97_ops(NULL);
-	return ret;
-}
-
-static int s3c_ac97_remove(struct platform_device *pdev)
-{
-	struct resource *irq_res;
-
-	irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-	if (irq_res)
-		free_irq(irq_res->start, NULL);
-
-	clk_disable_unprepare(s3c_ac97.ac97_clk);
-	snd_soc_set_ac97_ops(NULL);
-
-	return 0;
-}
-
-static struct platform_driver s3c_ac97_driver = {
-	.probe  = s3c_ac97_probe,
-	.remove = s3c_ac97_remove,
-	.driver = {
-		.name = "samsung-ac97",
-	},
-};
-
-module_platform_driver(s3c_ac97_driver);
-
-MODULE_AUTHOR("Jaswinder Singh, <jassisinghbrar@gmail.com>");
-MODULE_DESCRIPTION("AC97 driver for the Samsung SoC");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:samsung-ac97");
diff --git a/sound/soc/samsung/dmaengine.c b/sound/soc/samsung/dmaengine.c
index 9104c98deeb7..cda656e4afc6 100644
--- a/sound/soc/samsung/dmaengine.c
+++ b/sound/soc/samsung/dmaengine.c
@@ -37,12 +37,8 @@ int samsung_asoc_dma_platform_register(struct device *dev, dma_filter_fn filter,
 	pcm_conf->prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config;
 	pcm_conf->compat_filter_fn = filter;
 
-	if (dev->of_node) {
-		pcm_conf->chan_names[SNDRV_PCM_STREAM_PLAYBACK] = tx;
-		pcm_conf->chan_names[SNDRV_PCM_STREAM_CAPTURE] = rx;
-	} else {
-		flags |= SND_DMAENGINE_PCM_FLAG_CUSTOM_CHANNEL_NAME;
-	}
+	pcm_conf->chan_names[SNDRV_PCM_STREAM_PLAYBACK] = tx;
+	pcm_conf->chan_names[SNDRV_PCM_STREAM_CAPTURE] = rx;
 
 	return devm_snd_dmaengine_pcm_register(dev, pcm_conf, flags);
 }
diff --git a/sound/soc/samsung/i2s.c b/sound/soc/samsung/i2s.c
index 7825bff45ae3..e00974bc5616 100644
--- a/sound/soc/samsung/i2s.c
+++ b/sound/soc/samsung/i2s.c
@@ -1029,12 +1029,13 @@ static int samsung_i2s_dai_probe(struct snd_soc_dai *dai)
 static int samsung_i2s_dai_remove(struct snd_soc_dai *dai)
 {
 	struct i2s_dai *i2s = snd_soc_dai_get_drvdata(dai);
+	unsigned long flags;
 
 	if (!is_secondary(i2s)) {
 		if (i2s->quirks & QUIRK_NEED_RSTCLR) {
-			spin_lock(i2s->lock);
+			spin_lock_irqsave(i2s->lock, flags);
 			writel(0, i2s->addr + I2SCON);
-			spin_unlock(i2s->lock);
+			spin_unlock_irqrestore(i2s->lock, flags);
 		}
 	}
 
@@ -1304,8 +1305,6 @@ static int samsung_i2s_probe(struct platform_device *pdev)
 	}
 	pri_dai->dma_playback.addr = regs_base + I2STXD;
 	pri_dai->dma_capture.addr = regs_base + I2SRXD;
-	pri_dai->dma_playback.chan_name = "tx";
-	pri_dai->dma_capture.chan_name = "rx";
 	pri_dai->dma_playback.addr_width = 4;
 	pri_dai->dma_capture.addr_width = 4;
 	pri_dai->quirks = quirks;
@@ -1330,7 +1329,6 @@ static int samsung_i2s_probe(struct platform_device *pdev)
 		sec_dai->lock = &pri_dai->spinlock;
 		sec_dai->variant_regs = pri_dai->variant_regs;
 		sec_dai->dma_playback.addr = regs_base + I2STXDS;
-		sec_dai->dma_playback.chan_name = "tx-sec";
 
 		if (!np) {
 			sec_dai->dma_playback.filter_data = i2s_pdata->dma_play_sec;
diff --git a/sound/soc/samsung/ln2440sbc_alc650.c b/sound/soc/samsung/ln2440sbc_alc650.c
deleted file mode 100644
index 9342fc270c2b..000000000000
--- a/sound/soc/samsung/ln2440sbc_alc650.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * SoC audio for ln2440sbc
- *
- * Copyright 2007 KonekTel, a.s.
- * Author: Ivan Kuten
- *         ivan.kuten@promwad.com
- *
- * Heavily based on smdk2443_wm9710.c
- * Copyright 2007 Wolfson Microelectronics PLC.
- * Author: Graeme Gregory
- *         graeme.gregory@wolfsonmicro.com or linux@wolfsonmicro.com
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License version 2 as
- *  published by the Free Software Foundation.
- *
- */
-
-#include <linux/module.h>
-#include <sound/soc.h>
-
-static struct snd_soc_card ln2440sbc;
-
-static struct snd_soc_dai_link ln2440sbc_dai[] = {
-{
-	.name = "AC97",
-	.stream_name = "AC97 HiFi",
-	.cpu_dai_name = "samsung-ac97",
-	.codec_dai_name = "ac97-hifi",
-	.codec_name = "ac97-codec",
-	.platform_name = "samsung-ac97",
-},
-};
-
-static struct snd_soc_card ln2440sbc = {
-	.name = "LN2440SBC",
-	.owner = THIS_MODULE,
-	.dai_link = ln2440sbc_dai,
-	.num_links = ARRAY_SIZE(ln2440sbc_dai),
-};
-
-static struct platform_device *ln2440sbc_snd_ac97_device;
-
-static int __init ln2440sbc_init(void)
-{
-	int ret;
-
-	ln2440sbc_snd_ac97_device = platform_device_alloc("soc-audio", -1);
-	if (!ln2440sbc_snd_ac97_device)
-		return -ENOMEM;
-
-	platform_set_drvdata(ln2440sbc_snd_ac97_device, &ln2440sbc);
-	ret = platform_device_add(ln2440sbc_snd_ac97_device);
-
-	if (ret)
-		platform_device_put(ln2440sbc_snd_ac97_device);
-
-	return ret;
-}
-
-static void __exit ln2440sbc_exit(void)
-{
-	platform_device_unregister(ln2440sbc_snd_ac97_device);
-}
-
-module_init(ln2440sbc_init);
-module_exit(ln2440sbc_exit);
-
-/* Module information */
-MODULE_AUTHOR("Ivan Kuten");
-MODULE_DESCRIPTION("ALSA SoC ALC650 LN2440SBC");
-MODULE_LICENSE("GPL");
diff --git a/sound/soc/samsung/pcm.c b/sound/soc/samsung/pcm.c
index c484985812ed..d50a6377c23d 100644
--- a/sound/soc/samsung/pcm.c
+++ b/sound/soc/samsung/pcm.c
@@ -499,13 +499,6 @@ static int s3c_pcm_dev_probe(struct platform_device *pdev)
 
 	pcm_pdata = pdev->dev.platform_data;
 
-	/* Check for availability of necessary resource */
-	mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!mem_res) {
-		dev_err(&pdev->dev, "Unable to get register resource\n");
-		return -ENXIO;
-	}
-
 	if (pcm_pdata && pcm_pdata->cfg_gpio && pcm_pdata->cfg_gpio(pdev)) {
 		dev_err(&pdev->dev, "Unable to configure gpio\n");
 		return -EINVAL;
@@ -519,36 +512,26 @@ static int s3c_pcm_dev_probe(struct platform_device *pdev)
 	/* Default is 128fs */
 	pcm->sclk_per_fs = 128;
 
+	mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	pcm->regs = devm_ioremap_resource(&pdev->dev, mem_res);
+	if (IS_ERR(pcm->regs))
+		return PTR_ERR(pcm->regs);
+
 	pcm->cclk = devm_clk_get(&pdev->dev, "audio-bus");
 	if (IS_ERR(pcm->cclk)) {
-		dev_err(&pdev->dev, "failed to get audio-bus\n");
-		ret = PTR_ERR(pcm->cclk);
-		goto err1;
+		dev_err(&pdev->dev, "failed to get audio-bus clock\n");
+		return PTR_ERR(pcm->cclk);
 	}
 	clk_prepare_enable(pcm->cclk);
 
 	/* record our pcm structure for later use in the callbacks */
 	dev_set_drvdata(&pdev->dev, pcm);
 
-	if (!request_mem_region(mem_res->start,
-				resource_size(mem_res), "samsung-pcm")) {
-		dev_err(&pdev->dev, "Unable to request register region\n");
-		ret = -EBUSY;
-		goto err2;
-	}
-
-	pcm->regs = ioremap(mem_res->start, 0x100);
-	if (pcm->regs == NULL) {
-		dev_err(&pdev->dev, "cannot ioremap registers\n");
-		ret = -ENXIO;
-		goto err3;
-	}
-
 	pcm->pclk = devm_clk_get(&pdev->dev, "pcm");
 	if (IS_ERR(pcm->pclk)) {
-		dev_err(&pdev->dev, "failed to get pcm_clock\n");
-		ret = -ENOENT;
-		goto err4;
+		dev_err(&pdev->dev, "failed to get pcm clock\n");
+		ret = PTR_ERR(pcm->pclk);
+		goto err_dis_cclk;
 	}
 	clk_prepare_enable(pcm->pclk);
 
@@ -569,7 +552,7 @@ static int s3c_pcm_dev_probe(struct platform_device *pdev)
 						 NULL, NULL);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to get register DMA: %d\n", ret);
-		goto err5;
+		goto err_dis_pclk;
 	}
 
 	pm_runtime_enable(&pdev->dev);
@@ -578,36 +561,25 @@ static int s3c_pcm_dev_probe(struct platform_device *pdev)
 					 &s3c_pcm_dai[pdev->id], 1);
 	if (ret != 0) {
 		dev_err(&pdev->dev, "failed to get register DAI: %d\n", ret);
-		goto err6;
+		goto err_dis_pm;
 	}
 
 	return 0;
-err6:
+
+err_dis_pm:
 	pm_runtime_disable(&pdev->dev);
-err5:
+err_dis_pclk:
 	clk_disable_unprepare(pcm->pclk);
-err4:
-	iounmap(pcm->regs);
-err3:
-	release_mem_region(mem_res->start, resource_size(mem_res));
-err2:
+err_dis_cclk:
 	clk_disable_unprepare(pcm->cclk);
-err1:
 	return ret;
 }
 
 static int s3c_pcm_dev_remove(struct platform_device *pdev)
 {
 	struct s3c_pcm_info *pcm = &s3c_pcm[pdev->id];
-	struct resource *mem_res;
 
 	pm_runtime_disable(&pdev->dev);
-
-	iounmap(pcm->regs);
-
-	mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	release_mem_region(mem_res->start, resource_size(mem_res));
-
 	clk_disable_unprepare(pcm->cclk);
 	clk_disable_unprepare(pcm->pclk);
 
diff --git a/sound/soc/samsung/regs-ac97.h b/sound/soc/samsung/regs-ac97.h
deleted file mode 100644
index a71be45bbffc..000000000000
--- a/sound/soc/samsung/regs-ac97.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2006 Simtec Electronics <linux@simtec.co.uk>
- *		http://www.simtec.co.uk/products/SWLINUX/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * S3C2440 AC97 Controller
-*/
-
-#ifndef __SAMSUNG_REGS_AC97_H__
-#define __SAMSUNG_REGS_AC97_H__
-
-#define S3C_AC97_GLBCTRL				(0x00)
-
-#define S3C_AC97_GLBCTRL_CODECREADYIE			(1<<22)
-#define S3C_AC97_GLBCTRL_PCMOUTURIE			(1<<21)
-#define S3C_AC97_GLBCTRL_PCMINORIE			(1<<20)
-#define S3C_AC97_GLBCTRL_MICINORIE			(1<<19)
-#define S3C_AC97_GLBCTRL_PCMOUTTIE			(1<<18)
-#define S3C_AC97_GLBCTRL_PCMINTIE			(1<<17)
-#define S3C_AC97_GLBCTRL_MICINTIE			(1<<16)
-#define S3C_AC97_GLBCTRL_PCMOUTTM_OFF			(0<<12)
-#define S3C_AC97_GLBCTRL_PCMOUTTM_PIO			(1<<12)
-#define S3C_AC97_GLBCTRL_PCMOUTTM_DMA			(2<<12)
-#define S3C_AC97_GLBCTRL_PCMOUTTM_MASK			(3<<12)
-#define S3C_AC97_GLBCTRL_PCMINTM_OFF			(0<<10)
-#define S3C_AC97_GLBCTRL_PCMINTM_PIO			(1<<10)
-#define S3C_AC97_GLBCTRL_PCMINTM_DMA			(2<<10)
-#define S3C_AC97_GLBCTRL_PCMINTM_MASK			(3<<10)
-#define S3C_AC97_GLBCTRL_MICINTM_OFF			(0<<8)
-#define S3C_AC97_GLBCTRL_MICINTM_PIO			(1<<8)
-#define S3C_AC97_GLBCTRL_MICINTM_DMA			(2<<8)
-#define S3C_AC97_GLBCTRL_MICINTM_MASK			(3<<8)
-#define S3C_AC97_GLBCTRL_TRANSFERDATAENABLE		(1<<3)
-#define S3C_AC97_GLBCTRL_ACLINKON			(1<<2)
-#define S3C_AC97_GLBCTRL_WARMRESET			(1<<1)
-#define S3C_AC97_GLBCTRL_COLDRESET			(1<<0)
-
-#define S3C_AC97_GLBSTAT				(0x04)
-
-#define S3C_AC97_GLBSTAT_CODECREADY			(1<<22)
-#define S3C_AC97_GLBSTAT_PCMOUTUR			(1<<21)
-#define S3C_AC97_GLBSTAT_PCMINORI			(1<<20)
-#define S3C_AC97_GLBSTAT_MICINORI			(1<<19)
-#define S3C_AC97_GLBSTAT_PCMOUTTI			(1<<18)
-#define S3C_AC97_GLBSTAT_PCMINTI			(1<<17)
-#define S3C_AC97_GLBSTAT_MICINTI			(1<<16)
-#define S3C_AC97_GLBSTAT_MAINSTATE_IDLE			(0<<0)
-#define S3C_AC97_GLBSTAT_MAINSTATE_INIT			(1<<0)
-#define S3C_AC97_GLBSTAT_MAINSTATE_READY		(2<<0)
-#define S3C_AC97_GLBSTAT_MAINSTATE_ACTIVE		(3<<0)
-#define S3C_AC97_GLBSTAT_MAINSTATE_LP			(4<<0)
-#define S3C_AC97_GLBSTAT_MAINSTATE_WARM			(5<<0)
-
-#define S3C_AC97_CODEC_CMD				(0x08)
-
-#define S3C_AC97_CODEC_CMD_READ				(1<<23)
-
-#define S3C_AC97_STAT					(0x0c)
-#define S3C_AC97_PCM_ADDR				(0x10)
-#define S3C_AC97_PCM_DATA				(0x18)
-#define S3C_AC97_MIC_DATA				(0x1C)
-
-#endif /* __SAMSUNG_REGS_AC97_H__ */
diff --git a/sound/soc/samsung/s3c2412-i2s.c b/sound/soc/samsung/s3c2412-i2s.c
index 0a4718207e6e..6d0b8897fa6c 100644
--- a/sound/soc/samsung/s3c2412-i2s.c
+++ b/sound/soc/samsung/s3c2412-i2s.c
@@ -35,12 +35,10 @@
 #include <linux/platform_data/asoc-s3c.h>
 
 static struct snd_dmaengine_dai_dma_data s3c2412_i2s_pcm_stereo_out = {
-	.chan_name	= "tx",
 	.addr_width	= 4,
 };
 
 static struct snd_dmaengine_dai_dma_data s3c2412_i2s_pcm_stereo_in = {
-	.chan_name	= "rx",
 	.addr_width	= 4,
 };
 
diff --git a/sound/soc/samsung/s3c24xx-i2s.c b/sound/soc/samsung/s3c24xx-i2s.c
index 9052f6a7073e..07f5091b33e8 100644
--- a/sound/soc/samsung/s3c24xx-i2s.c
+++ b/sound/soc/samsung/s3c24xx-i2s.c
@@ -30,15 +30,11 @@
 #include "dma.h"
 #include "s3c24xx-i2s.h"
 
-#include <linux/platform_data/asoc-s3c.h>
-
 static struct snd_dmaengine_dai_dma_data s3c24xx_i2s_pcm_stereo_out = {
-	.chan_name	= "tx",
 	.addr_width	= 2,
 };
 
 static struct snd_dmaengine_dai_dma_data s3c24xx_i2s_pcm_stereo_in = {
-	.chan_name	= "rx",
 	.addr_width	= 2,
 };
 
@@ -58,8 +54,6 @@ static void s3c24xx_snd_txctrl(int on)
 	u32 iiscon;
 	u32 iismod;
 
-	pr_debug("Entered %s\n", __func__);
-
 	iisfcon = readl(s3c24xx_i2s.regs + S3C2410_IISFCON);
 	iiscon  = readl(s3c24xx_i2s.regs + S3C2410_IISCON);
 	iismod  = readl(s3c24xx_i2s.regs + S3C2410_IISMOD);
@@ -103,8 +97,6 @@ static void s3c24xx_snd_rxctrl(int on)
 	u32 iiscon;
 	u32 iismod;
 
-	pr_debug("Entered %s\n", __func__);
-
 	iisfcon = readl(s3c24xx_i2s.regs + S3C2410_IISFCON);
 	iiscon  = readl(s3c24xx_i2s.regs + S3C2410_IISCON);
 	iismod  = readl(s3c24xx_i2s.regs + S3C2410_IISMOD);
@@ -151,8 +143,6 @@ static int s3c24xx_snd_lrsync(void)
 	u32 iiscon;
 	int timeout = 50; /* 5ms */
 
-	pr_debug("Entered %s\n", __func__);
-
 	while (1) {
 		iiscon = readl(s3c24xx_i2s.regs + S3C2410_IISCON);
 		if (iiscon & S3C2410_IISCON_LRINDEX)
@@ -171,8 +161,6 @@ static int s3c24xx_snd_lrsync(void)
  */
 static inline int s3c24xx_snd_is_clkmaster(void)
 {
-	pr_debug("Entered %s\n", __func__);
-
 	return (readl(s3c24xx_i2s.regs + S3C2410_IISMOD) & S3C2410_IISMOD_SLAVE) ? 0:1;
 }
 
@@ -184,8 +172,6 @@ static int s3c24xx_i2s_set_fmt(struct snd_soc_dai *cpu_dai,
 {
 	u32 iismod;
 
-	pr_debug("Entered %s\n", __func__);
-
 	iismod = readl(s3c24xx_i2s.regs + S3C2410_IISMOD);
 	pr_debug("hw_params r: IISMOD: %x \n", iismod);
 
@@ -213,6 +199,7 @@ static int s3c24xx_i2s_set_fmt(struct snd_soc_dai *cpu_dai,
 
 	writel(iismod, s3c24xx_i2s.regs + S3C2410_IISMOD);
 	pr_debug("hw_params w: IISMOD: %x \n", iismod);
+
 	return 0;
 }
 
@@ -223,8 +210,6 @@ static int s3c24xx_i2s_hw_params(struct snd_pcm_substream *substream,
 	struct snd_dmaengine_dai_dma_data *dma_data;
 	u32 iismod;
 
-	pr_debug("Entered %s\n", __func__);
-
 	dma_data = snd_soc_dai_get_dma_data(dai, substream);
 
 	/* Working copies of register */
@@ -246,6 +231,7 @@ static int s3c24xx_i2s_hw_params(struct snd_pcm_substream *substream,
 
 	writel(iismod, s3c24xx_i2s.regs + S3C2410_IISMOD);
 	pr_debug("hw_params w: IISMOD: %x\n", iismod);
+
 	return 0;
 }
 
@@ -254,8 +240,6 @@ static int s3c24xx_i2s_trigger(struct snd_pcm_substream *substream, int cmd,
 {
 	int ret = 0;
 
-	pr_debug("Entered %s\n", __func__);
-
 	switch (cmd) {
 	case SNDRV_PCM_TRIGGER_START:
 	case SNDRV_PCM_TRIGGER_RESUME:
@@ -297,8 +281,6 @@ static int s3c24xx_i2s_set_sysclk(struct snd_soc_dai *cpu_dai,
 {
 	u32 iismod = readl(s3c24xx_i2s.regs + S3C2410_IISMOD);
 
-	pr_debug("Entered %s\n", __func__);
-
 	iismod &= ~S3C2440_IISMOD_MPLL;
 
 	switch (clk_id) {
@@ -323,8 +305,6 @@ static int s3c24xx_i2s_set_clkdiv(struct snd_soc_dai *cpu_dai,
 {
 	u32 reg;
 
-	pr_debug("Entered %s\n", __func__);
-
 	switch (div_id) {
 	case S3C24XX_DIV_BCLK:
 		reg = readl(s3c24xx_i2s.regs + S3C2410_IISMOD) & ~S3C2410_IISMOD_FS_MASK;
@@ -358,8 +338,6 @@ EXPORT_SYMBOL_GPL(s3c24xx_i2s_get_clockrate);
 
 static int s3c24xx_i2s_probe(struct snd_soc_dai *dai)
 {
-	pr_debug("Entered %s\n", __func__);
-
 	snd_soc_dai_init_dma_data(dai, &s3c24xx_i2s_pcm_stereo_out,
 					&s3c24xx_i2s_pcm_stereo_in);
 
@@ -385,8 +363,6 @@ static int s3c24xx_i2s_probe(struct snd_soc_dai *dai)
 #ifdef CONFIG_PM
 static int s3c24xx_i2s_suspend(struct snd_soc_dai *cpu_dai)
 {
-	pr_debug("Entered %s\n", __func__);
-
 	s3c24xx_i2s.iiscon = readl(s3c24xx_i2s.regs + S3C2410_IISCON);
 	s3c24xx_i2s.iismod = readl(s3c24xx_i2s.regs + S3C2410_IISMOD);
 	s3c24xx_i2s.iisfcon = readl(s3c24xx_i2s.regs + S3C2410_IISFCON);
@@ -399,7 +375,6 @@ static int s3c24xx_i2s_suspend(struct snd_soc_dai *cpu_dai)
 
 static int s3c24xx_i2s_resume(struct snd_soc_dai *cpu_dai)
 {
-	pr_debug("Entered %s\n", __func__);
 	clk_prepare_enable(s3c24xx_i2s.iis_clk);
 
 	writel(s3c24xx_i2s.iiscon, s3c24xx_i2s.regs + S3C2410_IISCON);
@@ -414,7 +389,6 @@ static int s3c24xx_i2s_resume(struct snd_soc_dai *cpu_dai)
 #define s3c24xx_i2s_resume NULL
 #endif
 
-
 #define S3C24XX_I2S_RATES \
 	(SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_11025 | SNDRV_PCM_RATE_16000 | \
 	SNDRV_PCM_RATE_22050 | SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_44100 | \
@@ -451,41 +425,28 @@ static const struct snd_soc_component_driver s3c24xx_i2s_component = {
 
 static int s3c24xx_iis_dev_probe(struct platform_device *pdev)
 {
-	int ret = 0;
 	struct resource *res;
-	struct s3c_audio_pdata *pdata = dev_get_platdata(&pdev->dev);
-
-	if (!pdata) {
-		dev_err(&pdev->dev, "missing platform data");
-		return -ENXIO;
-	}
+	int ret;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		dev_err(&pdev->dev, "Can't get IO resource.\n");
-		return -ENOENT;
-	}
 	s3c24xx_i2s.regs = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(s3c24xx_i2s.regs))
 		return PTR_ERR(s3c24xx_i2s.regs);
 
 	s3c24xx_i2s_pcm_stereo_out.addr = res->start + S3C2410_IISFIFO;
-	s3c24xx_i2s_pcm_stereo_out.filter_data = pdata->dma_playback;
 	s3c24xx_i2s_pcm_stereo_in.addr = res->start + S3C2410_IISFIFO;
-	s3c24xx_i2s_pcm_stereo_in.filter_data = pdata->dma_capture;
 
-	ret = samsung_asoc_dma_platform_register(&pdev->dev,
-						 pdata->dma_filter,
+	ret = samsung_asoc_dma_platform_register(&pdev->dev, NULL,
 						 NULL, NULL);
 	if (ret) {
-		pr_err("failed to register the dma: %d\n", ret);
+		dev_err(&pdev->dev, "Failed to register the DMA: %d\n", ret);
 		return ret;
 	}
 
 	ret = devm_snd_soc_register_component(&pdev->dev,
 			&s3c24xx_i2s_component, &s3c24xx_i2s_dai, 1);
 	if (ret)
-		pr_err("failed to register the dai\n");
+		dev_err(&pdev->dev, "Failed to register the DAI\n");
 
 	return ret;
 }
diff --git a/sound/soc/samsung/s3c24xx_uda134x.c b/sound/soc/samsung/s3c24xx_uda134x.c
index 7853fbe6ccc9..81a78940967c 100644
--- a/sound/soc/samsung/s3c24xx_uda134x.c
+++ b/sound/soc/samsung/s3c24xx_uda134x.c
@@ -19,9 +19,15 @@
 #include <sound/s3c24xx_uda134x.h>
 
 #include "regs-iis.h"
-
 #include "s3c24xx-i2s.h"
 
+struct s3c24xx_uda134x {
+	struct clk *xtal;
+	struct clk *pclk;
+	struct mutex clk_lock;
+	int clk_users;
+};
+
 /* #define ENFORCE_RATES 1 */
 /*
   Unfortunately the S3C24XX in master mode has a limited capacity of
@@ -36,15 +42,6 @@
   possible an error will be returned.
 */
 
-static struct clk *xtal;
-static struct clk *pclk;
-/* this is need because we don't have a place where to keep the
- * pointers to the clocks in each substream. We get the clocks only
- * when we are actually using them so we don't block stuff like
- * frequency change or oscillator power-off */
-static int clk_users;
-static DEFINE_MUTEX(clk_lock);
-
 static unsigned int rates[33 * 2];
 #ifdef ENFORCE_RATES
 static struct snd_pcm_hw_constraint_list hw_constraints_rates = {
@@ -57,26 +54,24 @@ static struct snd_pcm_hw_constraint_list hw_constraints_rates = {
 static int s3c24xx_uda134x_startup(struct snd_pcm_substream *substream)
 {
 	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct s3c24xx_uda134x *priv = snd_soc_card_get_drvdata(rtd->card);
 	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
-#ifdef ENFORCE_RATES
-	struct snd_pcm_runtime *runtime = substream->runtime;
-#endif
 	int ret = 0;
 
-	mutex_lock(&clk_lock);
+	mutex_lock(&priv->clk_lock);
 
-	if (clk_users == 0) {
-		xtal = clk_get(rtd->dev, "xtal");
-		if (IS_ERR(xtal)) {
+	if (priv->clk_users == 0) {
+		priv->xtal = clk_get(rtd->dev, "xtal");
+		if (IS_ERR(priv->xtal)) {
 			dev_err(rtd->dev, "%s cannot get xtal\n", __func__);
-			ret = PTR_ERR(xtal);
+			ret = PTR_ERR(priv->xtal);
 		} else {
-			pclk = clk_get(cpu_dai->dev, "iis");
-			if (IS_ERR(pclk)) {
+			priv->pclk = clk_get(cpu_dai->dev, "iis");
+			if (IS_ERR(priv->pclk)) {
 				dev_err(rtd->dev, "%s cannot get pclk\n",
 					__func__);
-				clk_put(xtal);
-				ret = PTR_ERR(pclk);
+				clk_put(priv->xtal);
+				ret = PTR_ERR(priv->pclk);
 			}
 		}
 		if (!ret) {
@@ -85,18 +80,19 @@ static int s3c24xx_uda134x_startup(struct snd_pcm_substream *substream)
 			for (i = 0; i < 2; i++) {
 				int fs = i ? 256 : 384;
 
-				rates[i*33] = clk_get_rate(xtal) / fs;
+				rates[i*33] = clk_get_rate(priv->xtal) / fs;
 				for (j = 1; j < 33; j++)
-					rates[i*33 + j] = clk_get_rate(pclk) /
+					rates[i*33 + j] = clk_get_rate(priv->pclk) /
 						(j * fs);
 			}
 		}
 	}
-	clk_users += 1;
-	mutex_unlock(&clk_lock);
+	priv->clk_users += 1;
+	mutex_unlock(&priv->clk_lock);
+
 	if (!ret) {
 #ifdef ENFORCE_RATES
-		ret = snd_pcm_hw_constraint_list(runtime, 0,
+		ret = snd_pcm_hw_constraint_list(substream->runtime, 0,
 						 SNDRV_PCM_HW_PARAM_RATE,
 						 &hw_constraints_rates);
 		if (ret < 0)
@@ -109,15 +105,18 @@ static int s3c24xx_uda134x_startup(struct snd_pcm_substream *substream)
 
 static void s3c24xx_uda134x_shutdown(struct snd_pcm_substream *substream)
 {
-	mutex_lock(&clk_lock);
-	clk_users -= 1;
-	if (clk_users == 0) {
-		clk_put(xtal);
-		xtal = NULL;
-		clk_put(pclk);
-		pclk = NULL;
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct s3c24xx_uda134x *priv = snd_soc_card_get_drvdata(rtd->card);
+
+	mutex_lock(&priv->clk_lock);
+	priv->clk_users -= 1;
+	if (priv->clk_users == 0) {
+		clk_put(priv->xtal);
+		priv->xtal = NULL;
+		clk_put(priv->pclk);
+		priv->pclk = NULL;
 	}
-	mutex_unlock(&clk_lock);
+	mutex_unlock(&priv->clk_lock);
 }
 
 static int s3c24xx_uda134x_hw_params(struct snd_pcm_substream *substream,
@@ -228,10 +227,18 @@ static struct snd_soc_card snd_soc_s3c24xx_uda134x = {
 static int s3c24xx_uda134x_probe(struct platform_device *pdev)
 {
 	struct snd_soc_card *card = &snd_soc_s3c24xx_uda134x;
+	struct s3c24xx_uda134x *priv;
 	int ret;
 
-	platform_set_drvdata(pdev, card);
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	mutex_init(&priv->clk_lock);
+
 	card->dev = &pdev->dev;
+	platform_set_drvdata(pdev, card);
+	snd_soc_card_set_drvdata(card, priv);
 
 	ret = devm_snd_soc_register_card(&pdev->dev, card);
 	if (ret)
diff --git a/sound/soc/samsung/smdk2443_wm9710.c b/sound/soc/samsung/smdk2443_wm9710.c
deleted file mode 100644
index c390aad68cfb..000000000000
--- a/sound/soc/samsung/smdk2443_wm9710.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * smdk2443_wm9710.c  --  SoC audio for smdk2443
- *
- * Copyright 2007 Wolfson Microelectronics PLC.
- * Author: Graeme Gregory
- *         graeme.gregory@wolfsonmicro.com or linux@wolfsonmicro.com
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- *
- */
-
-#include <linux/module.h>
-#include <sound/soc.h>
-
-static struct snd_soc_card smdk2443;
-
-static struct snd_soc_dai_link smdk2443_dai[] = {
-{
-	.name = "AC97",
-	.stream_name = "AC97 HiFi",
-	.cpu_dai_name = "samsung-ac97",
-	.codec_dai_name = "ac97-hifi",
-	.codec_name = "ac97-codec",
-	.platform_name = "samsung-ac97",
-},
-};
-
-static struct snd_soc_card smdk2443 = {
-	.name = "SMDK2443",
-	.owner = THIS_MODULE,
-	.dai_link = smdk2443_dai,
-	.num_links = ARRAY_SIZE(smdk2443_dai),
-};
-
-static struct platform_device *smdk2443_snd_ac97_device;
-
-static int __init smdk2443_init(void)
-{
-	int ret;
-
-	smdk2443_snd_ac97_device = platform_device_alloc("soc-audio", -1);
-	if (!smdk2443_snd_ac97_device)
-		return -ENOMEM;
-
-	platform_set_drvdata(smdk2443_snd_ac97_device, &smdk2443);
-	ret = platform_device_add(smdk2443_snd_ac97_device);
-
-	if (ret)
-		platform_device_put(smdk2443_snd_ac97_device);
-
-	return ret;
-}
-
-static void __exit smdk2443_exit(void)
-{
-	platform_device_unregister(smdk2443_snd_ac97_device);
-}
-
-module_init(smdk2443_init);
-module_exit(smdk2443_exit);
-
-/* Module information */
-MODULE_AUTHOR("Graeme Gregory, graeme.gregory@wolfsonmicro.com, www.wolfsonmicro.com");
-MODULE_DESCRIPTION("ALSA SoC WM9710 SMDK2443");
-MODULE_LICENSE("GPL");
diff --git a/sound/soc/samsung/smdk_wm8580.c b/sound/soc/samsung/smdk_wm8580.c
index 548bfd993788..de724ce7b955 100644
--- a/sound/soc/samsung/smdk_wm8580.c
+++ b/sound/soc/samsung/smdk_wm8580.c
@@ -14,8 +14,6 @@
 #include <sound/soc.h>
 #include <sound/pcm_params.h>
 
-#include <asm/mach-types.h>
-
 #include "../codecs/wm8580.h"
 #include "i2s.h"
 
@@ -147,7 +145,6 @@ static int smdk_wm8580_init_paiftx(struct snd_soc_pcm_runtime *rtd)
 enum {
 	PRI_PLAYBACK = 0,
 	PRI_CAPTURE,
-	SEC_PLAYBACK,
 };
 
 #define SMDK_DAI_FMT (SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | \
@@ -157,7 +154,7 @@ static struct snd_soc_dai_link smdk_dai[] = {
 	[PRI_PLAYBACK] = { /* Primary Playback i/f */
 		.name = "WM8580 PAIF RX",
 		.stream_name = "Playback",
-		.cpu_dai_name = "samsung-i2s.0",
+		.cpu_dai_name = "samsung-i2s.2",
 		.codec_dai_name = "wm8580-hifi-playback",
 		.platform_name = "samsung-i2s.0",
 		.codec_name = "wm8580.0-001b",
@@ -167,7 +164,7 @@ static struct snd_soc_dai_link smdk_dai[] = {
 	[PRI_CAPTURE] = { /* Primary Capture i/f */
 		.name = "WM8580 PAIF TX",
 		.stream_name = "Capture",
-		.cpu_dai_name = "samsung-i2s.0",
+		.cpu_dai_name = "samsung-i2s.2",
 		.codec_dai_name = "wm8580-hifi-capture",
 		.platform_name = "samsung-i2s.0",
 		.codec_name = "wm8580.0-001b",
@@ -175,23 +172,13 @@ static struct snd_soc_dai_link smdk_dai[] = {
 		.init = smdk_wm8580_init_paiftx,
 		.ops = &smdk_ops,
 	},
-	[SEC_PLAYBACK] = { /* Sec_Fifo Playback i/f */
-		.name = "Sec_FIFO TX",
-		.stream_name = "Playback",
-		.cpu_dai_name = "samsung-i2s-sec",
-		.codec_dai_name = "wm8580-hifi-playback",
-		.platform_name = "samsung-i2s-sec",
-		.codec_name = "wm8580.0-001b",
-		.dai_fmt = SMDK_DAI_FMT,
-		.ops = &smdk_ops,
-	},
 };
 
 static struct snd_soc_card smdk = {
 	.name = "SMDK-I2S",
 	.owner = THIS_MODULE,
 	.dai_link = smdk_dai,
-	.num_links = 2,
+	.num_links = ARRAY_SIZE(smdk_dai),
 
 	.dapm_widgets = smdk_wm8580_dapm_widgets,
 	.num_dapm_widgets = ARRAY_SIZE(smdk_wm8580_dapm_widgets),
@@ -204,17 +191,6 @@ static struct platform_device *smdk_snd_device;
 static int __init smdk_audio_init(void)
 {
 	int ret;
-	char *str;
-
-	if (machine_is_smdkc100()
-			|| machine_is_smdkv210() || machine_is_smdkc110()) {
-		smdk.num_links = 3;
-	} else if (machine_is_smdk6410()) {
-		str = (char *)smdk_dai[PRI_PLAYBACK].cpu_dai_name;
-		str[strlen(str) - 1] = '2';
-		str = (char *)smdk_dai[PRI_CAPTURE].cpu_dai_name;
-		str[strlen(str) - 1] = '2';
-	}
 
 	smdk_snd_device = platform_device_alloc("soc-audio", -1);
 	if (!smdk_snd_device)
diff --git a/sound/soc/samsung/smdk_wm8580pcm.c b/sound/soc/samsung/smdk_wm8580pcm.c
deleted file mode 100644
index a6d223310c67..000000000000
--- a/sound/soc/samsung/smdk_wm8580pcm.c
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- *  sound/soc/samsung/smdk_wm8580pcm.c
- *
- *  Copyright (c) 2011 Samsung Electronics Co. Ltd
- *
- *  This program is free software; you can redistribute  it and/or  modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- */
-#include <linux/module.h>
-#include <sound/soc.h>
-#include <sound/pcm_params.h>
-#include <sound/pcm.h>
-
-#include <asm/mach-types.h>
-
-#include "../codecs/wm8580.h"
-#include "pcm.h"
-
-/*
- * Board Settings:
- *  o '1' means 'ON'
- *  o '0' means 'OFF'
- *  o 'X' means 'Don't care'
- *
- * SMDK6410 Base B/D: CFG1-0000, CFG2-1111
- * SMDKC110, SMDKV210: CFGB11-100100, CFGB12-0000
- */
-
-#define SMDK_WM8580_EXT_OSC 12000000
-#define SMDK_WM8580_EXT_MCLK 4096000
-#define SMDK_WM8580_EXT_VOICE 2048000
-
-static unsigned long mclk_freq;
-static unsigned long xtal_freq;
-
-/*
- * If MCLK clock directly gets from XTAL, we don't have to use PLL
- * to make MCLK, but if XTAL clock source connects with other codec
- * pin (like XTI), we should have to set codec's PLL to make MCLK.
- * Because Samsung SoC does not support pcmcdclk output like I2S.
- */
-
-static int smdk_wm8580_pcm_hw_params(struct snd_pcm_substream *substream,
-			      struct snd_pcm_hw_params *params)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *codec_dai = rtd->codec_dai;
-	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
-	int rfs, ret;
-
-	switch (params_rate(params)) {
-	case 8000:
-		break;
-	default:
-		printk(KERN_ERR "%s:%d Sampling Rate %u not supported!\n",
-		__func__, __LINE__, params_rate(params));
-		return -EINVAL;
-	}
-
-	rfs = mclk_freq / params_rate(params) / 2;
-
-	if (mclk_freq == xtal_freq) {
-		ret = snd_soc_dai_set_sysclk(codec_dai, WM8580_CLKSRC_MCLK,
-						mclk_freq, SND_SOC_CLOCK_IN);
-		if (ret < 0)
-			return ret;
-
-		ret = snd_soc_dai_set_clkdiv(codec_dai, WM8580_MCLK,
-						WM8580_CLKSRC_MCLK);
-		if (ret < 0)
-			return ret;
-	} else {
-		ret = snd_soc_dai_set_sysclk(codec_dai, WM8580_CLKSRC_PLLA,
-						mclk_freq, SND_SOC_CLOCK_IN);
-		if (ret < 0)
-			return ret;
-
-		ret = snd_soc_dai_set_clkdiv(codec_dai, WM8580_MCLK,
-						WM8580_CLKSRC_PLLA);
-		if (ret < 0)
-			return ret;
-
-		ret = snd_soc_dai_set_pll(codec_dai, WM8580_PLLA, 0,
-						xtal_freq, mclk_freq);
-		if (ret < 0)
-			return ret;
-	}
-
-	/* Set PCM source clock on CPU */
-	ret = snd_soc_dai_set_sysclk(cpu_dai, S3C_PCM_CLKSRC_MUX,
-					mclk_freq, SND_SOC_CLOCK_IN);
-	if (ret < 0)
-		return ret;
-
-	/* Set SCLK_DIV for making bclk */
-	ret = snd_soc_dai_set_clkdiv(cpu_dai, S3C_PCM_SCLK_PER_FS, rfs);
-	if (ret < 0)
-		return ret;
-
-	return 0;
-}
-
-static struct snd_soc_ops smdk_wm8580_pcm_ops = {
-	.hw_params = smdk_wm8580_pcm_hw_params,
-};
-
-#define SMDK_DAI_FMT (SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_IB_NF | \
-	SND_SOC_DAIFMT_CBS_CFS)
-
-static struct snd_soc_dai_link smdk_dai[] = {
-	{
-		.name = "WM8580 PAIF PCM RX",
-		.stream_name = "Playback",
-		.cpu_dai_name = "samsung-pcm.0",
-		.codec_dai_name = "wm8580-hifi-playback",
-		.platform_name = "samsung-audio",
-		.codec_name = "wm8580.0-001b",
-		.dai_fmt = SMDK_DAI_FMT,
-		.ops = &smdk_wm8580_pcm_ops,
-	}, {
-		.name = "WM8580 PAIF PCM TX",
-		.stream_name = "Capture",
-		.cpu_dai_name = "samsung-pcm.0",
-		.codec_dai_name = "wm8580-hifi-capture",
-		.platform_name = "samsung-pcm.0",
-		.codec_name = "wm8580.0-001b",
-		.dai_fmt = SMDK_DAI_FMT,
-		.ops = &smdk_wm8580_pcm_ops,
-	},
-};
-
-static struct snd_soc_card smdk_pcm = {
-	.name = "SMDK-PCM",
-	.owner = THIS_MODULE,
-	.dai_link = smdk_dai,
-	.num_links = 2,
-};
-
-/*
- * After SMDKC110 Base Board's Rev is '0.1', 12MHz External OSC(X1)
- * is absent (or not connected), so we connect EXT_VOICE_CLK(OSC4),
- * 2.0484Mhz, directly with MCLK both Codec and SoC.
- */
-static int snd_smdk_probe(struct platform_device *pdev)
-{
-	int ret = 0;
-
-	xtal_freq = SMDK_WM8580_EXT_OSC;
-	mclk_freq = SMDK_WM8580_EXT_MCLK;
-
-	if (machine_is_smdkc110() || machine_is_smdkv210())
-		xtal_freq = mclk_freq = SMDK_WM8580_EXT_VOICE;
-
-	smdk_pcm.dev = &pdev->dev;
-	ret = devm_snd_soc_register_card(&pdev->dev, &smdk_pcm);
-	if (ret)
-		dev_err(&pdev->dev, "snd_soc_register_card failed %d\n", ret);
-
-	return ret;
-}
-
-static struct platform_driver snd_smdk_driver = {
-	.driver = {
-		.name = "samsung-smdk-pcm",
-	},
-	.probe = snd_smdk_probe,
-};
-
-module_platform_driver(snd_smdk_driver);
-
-MODULE_AUTHOR("Sangbeom Kim, <sbkim73@samsung.com>");
-MODULE_DESCRIPTION("ALSA SoC SMDK WM8580 for PCM");
-MODULE_LICENSE("GPL");
diff --git a/sound/soc/samsung/smdk_wm9713.c b/sound/soc/samsung/smdk_wm9713.c
deleted file mode 100644
index 0d20e4ed27aa..000000000000
--- a/sound/soc/samsung/smdk_wm9713.c
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * smdk_wm9713.c  --  SoC audio for SMDK
- *
- * Copyright 2010 Samsung Electronics Co. Ltd.
- * Author: Jaswinder Singh Brar <jassisinghbrar@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or (at your option) any later version.
- *
- */
-
-#include <linux/module.h>
-#include <sound/soc.h>
-
-static struct snd_soc_card smdk;
-
-/*
- * Default CFG switch settings to use this driver:
- *
- *   SMDK6410: Set CFG1 1-3 On, CFG2 1-4 Off
- *   SMDKC100: Set CFG6 1-3 On, CFG7 1   On
- *   SMDKC110: Set CFGB10 1-2 Off, CFGB12 1-3 On
- *   SMDKV210: Set CFGB10 1-2 Off, CFGB12 1-3 On
- *   SMDKV310: Set CFG2 1-2 Off, CFG4 All On, CFG7 All Off, CFG8 1-On
- */
-
-/*
- Playback (HeadPhone):-
-	$ amixer sset 'Headphone' unmute
-	$ amixer sset 'Right Headphone Out Mux' 'Headphone'
-	$ amixer sset 'Left Headphone Out Mux' 'Headphone'
-	$ amixer sset 'Right HP Mixer PCM' unmute
-	$ amixer sset 'Left HP Mixer PCM' unmute
-
- Capture (LineIn):-
-	$ amixer sset 'Right Capture Source' 'Line'
-	$ amixer sset 'Left Capture Source' 'Line'
-*/
-
-static struct snd_soc_dai_link smdk_dai = {
-	.name = "AC97",
-	.stream_name = "AC97 PCM",
-	.platform_name = "samsung-ac97",
-	.cpu_dai_name = "samsung-ac97",
-	.codec_dai_name = "wm9713-hifi",
-	.codec_name = "wm9713-codec",
-};
-
-static struct snd_soc_card smdk = {
-	.name = "SMDK WM9713",
-	.owner = THIS_MODULE,
-	.dai_link = &smdk_dai,
-	.num_links = 1,
-};
-
-static struct platform_device *smdk_snd_wm9713_device;
-static struct platform_device *smdk_snd_ac97_device;
-
-static int __init smdk_init(void)
-{
-	int ret;
-
-	smdk_snd_wm9713_device = platform_device_alloc("wm9713-codec", -1);
-	if (!smdk_snd_wm9713_device)
-		return -ENOMEM;
-
-	ret = platform_device_add(smdk_snd_wm9713_device);
-	if (ret)
-		goto err1;
-
-	smdk_snd_ac97_device = platform_device_alloc("soc-audio", -1);
-	if (!smdk_snd_ac97_device) {
-		ret = -ENOMEM;
-		goto err2;
-	}
-
-	platform_set_drvdata(smdk_snd_ac97_device, &smdk);
-
-	ret = platform_device_add(smdk_snd_ac97_device);
-	if (ret)
-		goto err3;
-
-	return 0;
-
-err3:
-	platform_device_put(smdk_snd_ac97_device);
-err2:
-	platform_device_del(smdk_snd_wm9713_device);
-err1:
-	platform_device_put(smdk_snd_wm9713_device);
-	return ret;
-}
-
-static void __exit smdk_exit(void)
-{
-	platform_device_unregister(smdk_snd_ac97_device);
-	platform_device_unregister(smdk_snd_wm9713_device);
-}
-
-module_init(smdk_init);
-module_exit(smdk_exit);
-
-/* Module information */
-MODULE_AUTHOR("Jaswinder Singh Brar, jassisinghbrar@gmail.com");
-MODULE_DESCRIPTION("ALSA SoC SMDK+WM9713");
-MODULE_LICENSE("GPL");
diff --git a/sound/soc/samsung/tm2_wm5110.c b/sound/soc/samsung/tm2_wm5110.c
new file mode 100644
index 000000000000..5cdf7d19b87f
--- /dev/null
+++ b/sound/soc/samsung/tm2_wm5110.c
@@ -0,0 +1,552 @@
+/*
+ * Copyright (C) 2015 - 2016 Samsung Electronics Co., Ltd.
+ *
+ * Authors: Inha Song <ideal.song@samsung.com>
+ *          Sylwester Nawrocki <s.nawrocki@samsung.com>
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/clk.h>
+#include <linux/gpio.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+
+#include "i2s.h"
+#include "../codecs/wm5110.h"
+
+/*
+ * The source clock is XCLKOUT with its mux set to the external fixed rate
+ * oscillator (XXTI).
+ */
+#define MCLK_RATE	24000000U
+
+#define TM2_DAI_AIF1	0
+#define TM2_DAI_AIF2	1
+
+struct tm2_machine_priv {
+	struct snd_soc_codec *codec;
+	unsigned int sysclk_rate;
+	struct gpio_desc *gpio_mic_bias;
+};
+
+static int tm2_start_sysclk(struct snd_soc_card *card)
+{
+	struct tm2_machine_priv *priv = snd_soc_card_get_drvdata(card);
+	struct snd_soc_codec *codec = priv->codec;
+	int ret;
+
+	ret = snd_soc_codec_set_pll(codec, WM5110_FLL1_REFCLK,
+				    ARIZONA_FLL_SRC_MCLK1,
+				    MCLK_RATE,
+				    priv->sysclk_rate);
+	if (ret < 0) {
+		dev_err(codec->dev, "Failed to set FLL1 source: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_codec_set_pll(codec, WM5110_FLL1,
+				    ARIZONA_FLL_SRC_MCLK1,
+				    MCLK_RATE,
+				    priv->sysclk_rate);
+	if (ret < 0) {
+		dev_err(codec->dev, "Failed to start FLL1: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_codec_set_sysclk(codec, ARIZONA_CLK_SYSCLK,
+				       ARIZONA_CLK_SRC_FLL1,
+				       priv->sysclk_rate,
+				       SND_SOC_CLOCK_IN);
+	if (ret < 0) {
+		dev_err(codec->dev, "Failed to set SYSCLK source: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int tm2_stop_sysclk(struct snd_soc_card *card)
+{
+	struct tm2_machine_priv *priv = snd_soc_card_get_drvdata(card);
+	struct snd_soc_codec *codec = priv->codec;
+	int ret;
+
+	ret = snd_soc_codec_set_pll(codec, WM5110_FLL1, 0, 0, 0);
+	if (ret < 0) {
+		dev_err(codec->dev, "Failed to stop FLL1: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_codec_set_sysclk(codec, ARIZONA_CLK_SYSCLK,
+				       ARIZONA_CLK_SRC_FLL1, 0, 0);
+	if (ret < 0) {
+		dev_err(codec->dev, "Failed to stop SYSCLK: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int tm2_aif1_hw_params(struct snd_pcm_substream *substream,
+				struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_codec *codec = rtd->codec;
+	struct tm2_machine_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+
+	switch (params_rate(params)) {
+	case 4000:
+	case 8000:
+	case 12000:
+	case 16000:
+	case 24000:
+	case 32000:
+	case 48000:
+	case 96000:
+	case 192000:
+		/* Highest possible SYSCLK frequency: 147.456MHz */
+		priv->sysclk_rate = 147456000U;
+		break;
+	case 11025:
+	case 22050:
+	case 44100:
+	case 88200:
+	case 176400:
+		/* Highest possible SYSCLK frequency: 135.4752 MHz */
+		priv->sysclk_rate = 135475200U;
+		break;
+	default:
+		dev_err(codec->dev, "Not supported sample rate: %d\n",
+			params_rate(params));
+		return -EINVAL;
+	}
+
+	return tm2_start_sysclk(rtd->card);
+}
+
+static struct snd_soc_ops tm2_aif1_ops = {
+	.hw_params = tm2_aif1_hw_params,
+};
+
+static int tm2_aif2_hw_params(struct snd_pcm_substream *substream,
+				struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_codec *codec = rtd->codec;
+	unsigned int asyncclk_rate;
+	int ret;
+
+	switch (params_rate(params)) {
+	case 8000:
+	case 12000:
+	case 16000:
+		/* Highest possible ASYNCCLK frequency: 49.152MHz */
+		asyncclk_rate = 49152000U;
+		break;
+	case 11025:
+		/* Highest possible ASYNCCLK frequency: 45.1584 MHz */
+		asyncclk_rate = 45158400U;
+		break;
+	default:
+		dev_err(codec->dev, "Not supported sample rate: %d\n",
+			params_rate(params));
+		return -EINVAL;
+	}
+
+	ret = snd_soc_codec_set_pll(codec, WM5110_FLL2_REFCLK,
+				    ARIZONA_FLL_SRC_MCLK1,
+				    MCLK_RATE,
+				    asyncclk_rate);
+	if (ret < 0) {
+		dev_err(codec->dev, "Failed to set FLL2 source: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_codec_set_pll(codec, WM5110_FLL2,
+				    ARIZONA_FLL_SRC_MCLK1,
+				    MCLK_RATE,
+				    asyncclk_rate);
+	if (ret < 0) {
+		dev_err(codec->dev, "Failed to start FLL2: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_codec_set_sysclk(codec, ARIZONA_CLK_ASYNCCLK,
+				       ARIZONA_CLK_SRC_FLL2,
+				       asyncclk_rate,
+				       SND_SOC_CLOCK_IN);
+	if (ret < 0) {
+		dev_err(codec->dev, "Failed to set ASYNCCLK source: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int tm2_aif2_hw_free(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_codec *codec = rtd->codec;
+	int ret;
+
+	/* disable FLL2 */
+	ret = snd_soc_codec_set_pll(codec, WM5110_FLL2, ARIZONA_FLL_SRC_MCLK1,
+				    0, 0);
+	if (ret < 0)
+		dev_err(codec->dev, "Failed to stop FLL2: %d\n", ret);
+
+	return ret;
+}
+
+static struct snd_soc_ops tm2_aif2_ops = {
+	.hw_params = tm2_aif2_hw_params,
+	.hw_free = tm2_aif2_hw_free,
+};
+
+static int tm2_mic_bias(struct snd_soc_dapm_widget *w,
+				struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_card *card = w->dapm->card;
+	struct tm2_machine_priv *priv = snd_soc_card_get_drvdata(card);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		gpiod_set_value_cansleep(priv->gpio_mic_bias,  1);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		gpiod_set_value_cansleep(priv->gpio_mic_bias,  0);
+		break;
+	}
+
+	return 0;
+}
+
+static int tm2_set_bias_level(struct snd_soc_card *card,
+				struct snd_soc_dapm_context *dapm,
+				enum snd_soc_bias_level level)
+{
+	struct snd_soc_pcm_runtime *rtd;
+
+	rtd = snd_soc_get_pcm_runtime(card, card->dai_link[0].name);
+
+	if (dapm->dev != rtd->codec_dai->dev)
+		return 0;
+
+	switch (level) {
+	case SND_SOC_BIAS_STANDBY:
+		if (card->dapm.bias_level == SND_SOC_BIAS_OFF)
+			tm2_start_sysclk(card);
+		break;
+	case SND_SOC_BIAS_OFF:
+		tm2_stop_sysclk(card);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static struct snd_soc_aux_dev tm2_speaker_amp_dev;
+
+static int tm2_late_probe(struct snd_soc_card *card)
+{
+	struct tm2_machine_priv *priv = snd_soc_card_get_drvdata(card);
+	struct snd_soc_dai_link_component dlc = { 0 };
+	unsigned int ch_map[] = { 0, 1 };
+	struct snd_soc_dai *amp_pdm_dai;
+	struct snd_soc_pcm_runtime *rtd;
+	struct snd_soc_dai *aif1_dai;
+	struct snd_soc_dai *aif2_dai;
+	int ret;
+
+	rtd = snd_soc_get_pcm_runtime(card, card->dai_link[TM2_DAI_AIF1].name);
+	aif1_dai = rtd->codec_dai;
+	priv->codec = rtd->codec;
+
+	ret = snd_soc_dai_set_sysclk(aif1_dai, ARIZONA_CLK_SYSCLK, 0, 0);
+	if (ret < 0) {
+		dev_err(aif1_dai->dev, "Failed to set SYSCLK: %d\n", ret);
+		return ret;
+	}
+
+	rtd = snd_soc_get_pcm_runtime(card, card->dai_link[TM2_DAI_AIF2].name);
+	aif2_dai = rtd->codec_dai;
+
+	ret = snd_soc_dai_set_sysclk(aif2_dai, ARIZONA_CLK_ASYNCCLK, 0, 0);
+	if (ret < 0) {
+		dev_err(aif2_dai->dev, "Failed to set ASYNCCLK: %d\n", ret);
+		return ret;
+	}
+
+	dlc.of_node = tm2_speaker_amp_dev.codec_of_node;
+	amp_pdm_dai = snd_soc_find_dai(&dlc);
+	if (!amp_pdm_dai)
+		return -ENODEV;
+
+	/* Set the MAX98504 V/I sense PDM Tx DAI channel mapping */
+	ret = snd_soc_dai_set_channel_map(amp_pdm_dai, ARRAY_SIZE(ch_map),
+					  ch_map, 0, NULL);
+	if (ret < 0)
+		return ret;
+
+	ret = snd_soc_dai_set_tdm_slot(amp_pdm_dai, 0x3, 0x0, 2, 16);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static const struct snd_kcontrol_new tm2_controls[] = {
+	SOC_DAPM_PIN_SWITCH("HP"),
+	SOC_DAPM_PIN_SWITCH("SPK"),
+	SOC_DAPM_PIN_SWITCH("RCV"),
+	SOC_DAPM_PIN_SWITCH("VPS"),
+	SOC_DAPM_PIN_SWITCH("HDMI"),
+
+	SOC_DAPM_PIN_SWITCH("Main Mic"),
+	SOC_DAPM_PIN_SWITCH("Sub Mic"),
+	SOC_DAPM_PIN_SWITCH("Third Mic"),
+
+	SOC_DAPM_PIN_SWITCH("Headset Mic"),
+};
+
+const struct snd_soc_dapm_widget tm2_dapm_widgets[] = {
+	SND_SOC_DAPM_HP("HP", NULL),
+	SND_SOC_DAPM_SPK("SPK", NULL),
+	SND_SOC_DAPM_SPK("RCV", NULL),
+	SND_SOC_DAPM_LINE("VPS", NULL),
+	SND_SOC_DAPM_LINE("HDMI", NULL),
+
+	SND_SOC_DAPM_MIC("Main Mic", tm2_mic_bias),
+	SND_SOC_DAPM_MIC("Sub Mic", NULL),
+	SND_SOC_DAPM_MIC("Third Mic", NULL),
+
+	SND_SOC_DAPM_MIC("Headset Mic", NULL),
+};
+
+static const struct snd_soc_component_driver tm2_component = {
+	.name	= "tm2-audio",
+};
+
+static struct snd_soc_dai_driver tm2_ext_dai[] = {
+	{
+		.name = "Voice call",
+		.playback = {
+			.channels_min = 1,
+			.channels_max = 4,
+			.rate_min = 8000,
+			.rate_max = 48000,
+			.rates = (SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000 |
+					SNDRV_PCM_RATE_48000),
+			.formats = SNDRV_PCM_FMTBIT_S16_LE,
+		},
+		.capture = {
+			.channels_min = 1,
+			.channels_max = 4,
+			.rate_min = 8000,
+			.rate_max = 48000,
+			.rates = (SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000 |
+					SNDRV_PCM_RATE_48000),
+			.formats = SNDRV_PCM_FMTBIT_S16_LE,
+		},
+	},
+	{
+		.name = "Bluetooth",
+		.playback = {
+			.channels_min = 1,
+			.channels_max = 4,
+			.rate_min = 8000,
+			.rate_max = 16000,
+			.rates = (SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000),
+			.formats = SNDRV_PCM_FMTBIT_S16_LE,
+		},
+		.capture = {
+			.channels_min = 1,
+			.channels_max = 2,
+			.rate_min = 8000,
+			.rate_max = 16000,
+			.rates = (SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000),
+			.formats = SNDRV_PCM_FMTBIT_S16_LE,
+		},
+	},
+};
+
+static struct snd_soc_dai_link tm2_dai_links[] = {
+	{
+		.name		= "WM5110 AIF1",
+		.stream_name	= "HiFi Primary",
+		.codec_dai_name = "wm5110-aif1",
+		.ops		= &tm2_aif1_ops,
+		.dai_fmt	= SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF |
+				  SND_SOC_DAIFMT_CBM_CFM,
+	}, {
+		.name		= "WM5110 Voice",
+		.stream_name	= "Voice call",
+		.codec_dai_name = "wm5110-aif2",
+		.ops		= &tm2_aif2_ops,
+		.dai_fmt	= SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF |
+				  SND_SOC_DAIFMT_CBM_CFM,
+		.ignore_suspend = 1,
+	}, {
+		.name		= "WM5110 BT",
+		.stream_name	= "Bluetooth",
+		.codec_dai_name = "wm5110-aif3",
+		.dai_fmt	= SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF |
+				  SND_SOC_DAIFMT_CBM_CFM,
+		.ignore_suspend = 1,
+	}
+};
+
+static struct snd_soc_card tm2_card = {
+	.owner			= THIS_MODULE,
+
+	.dai_link		= tm2_dai_links,
+	.num_links		= ARRAY_SIZE(tm2_dai_links),
+	.controls		= tm2_controls,
+	.num_controls		= ARRAY_SIZE(tm2_controls),
+	.dapm_widgets		= tm2_dapm_widgets,
+	.num_dapm_widgets	= ARRAY_SIZE(tm2_dapm_widgets),
+	.aux_dev		= &tm2_speaker_amp_dev,
+	.num_aux_devs		= 1,
+
+	.late_probe		= tm2_late_probe,
+	.set_bias_level		= tm2_set_bias_level,
+};
+
+static int tm2_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct snd_soc_card *card = &tm2_card;
+	struct tm2_machine_priv *priv;
+	struct device_node *cpu_dai_node, *codec_dai_node;
+	int ret, i;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	snd_soc_card_set_drvdata(card, priv);
+	card->dev = dev;
+
+	priv->gpio_mic_bias = devm_gpiod_get(dev, "mic-bias",
+						GPIOF_OUT_INIT_LOW);
+	if (IS_ERR(priv->gpio_mic_bias)) {
+		dev_err(dev, "Failed to get mic bias gpio\n");
+		return PTR_ERR(priv->gpio_mic_bias);
+	}
+
+	ret = snd_soc_of_parse_card_name(card, "model");
+	if (ret < 0) {
+		dev_err(dev, "Card name is not specified\n");
+		return ret;
+	}
+
+	ret = snd_soc_of_parse_audio_routing(card, "samsung,audio-routing");
+	if (ret < 0) {
+		dev_err(dev, "Audio routing is not specified or invalid\n");
+		return ret;
+	}
+
+	card->aux_dev[0].codec_of_node = of_parse_phandle(dev->of_node,
+							"audio-amplifier", 0);
+	if (!card->aux_dev[0].codec_of_node) {
+		dev_err(dev, "audio-amplifier property invalid or missing\n");
+		return -EINVAL;
+	}
+
+	cpu_dai_node = of_parse_phandle(dev->of_node, "i2s-controller", 0);
+	if (!cpu_dai_node) {
+		dev_err(dev, "i2s-controllers property invalid or missing\n");
+		ret = -EINVAL;
+		goto amp_node_put;
+	}
+
+	codec_dai_node = of_parse_phandle(dev->of_node, "audio-codec", 0);
+	if (!codec_dai_node) {
+		dev_err(dev, "audio-codec property invalid or missing\n");
+		ret = -EINVAL;
+		goto cpu_dai_node_put;
+	}
+
+	for (i = 0; i < card->num_links; i++) {
+		card->dai_link[i].cpu_dai_name = NULL;
+		card->dai_link[i].cpu_name = NULL;
+		card->dai_link[i].platform_name = NULL;
+		card->dai_link[i].codec_of_node = codec_dai_node;
+		card->dai_link[i].cpu_of_node = cpu_dai_node;
+		card->dai_link[i].platform_of_node = cpu_dai_node;
+	}
+
+	ret = devm_snd_soc_register_component(dev, &tm2_component,
+				tm2_ext_dai, ARRAY_SIZE(tm2_ext_dai));
+	if (ret < 0) {
+		dev_err(dev, "Failed to register component: %d\n", ret);
+		goto codec_dai_node_put;
+	}
+
+	ret = devm_snd_soc_register_card(dev, card);
+	if (ret < 0) {
+		dev_err(dev, "Failed to register card: %d\n", ret);
+		goto codec_dai_node_put;
+	}
+
+codec_dai_node_put:
+	of_node_put(codec_dai_node);
+cpu_dai_node_put:
+	of_node_put(cpu_dai_node);
+amp_node_put:
+	of_node_put(card->aux_dev[0].codec_of_node);
+	return ret;
+}
+
+static int tm2_pm_prepare(struct device *dev)
+{
+	struct snd_soc_card *card = dev_get_drvdata(dev);
+
+	return tm2_stop_sysclk(card);
+}
+
+static void tm2_pm_complete(struct device *dev)
+{
+	struct snd_soc_card *card = dev_get_drvdata(dev);
+
+	tm2_start_sysclk(card);
+}
+
+const struct dev_pm_ops tm2_pm_ops = {
+	.prepare	= tm2_pm_prepare,
+	.suspend	= snd_soc_suspend,
+	.resume		= snd_soc_resume,
+	.complete	= tm2_pm_complete,
+	.freeze		= snd_soc_suspend,
+	.thaw		= snd_soc_resume,
+	.poweroff	= snd_soc_poweroff,
+	.restore	= snd_soc_resume,
+};
+
+static const struct of_device_id tm2_of_match[] = {
+	{ .compatible = "samsung,tm2-audio" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, tm2_of_match);
+
+static struct platform_driver tm2_driver = {
+	.driver = {
+		.name		= "tm2-audio",
+		.pm		= &tm2_pm_ops,
+		.of_match_table	= tm2_of_match,
+	},
+	.probe	= tm2_probe,
+};
+module_platform_driver(tm2_driver);
+
+MODULE_AUTHOR("Inha Song <ideal.song@samsung.com>");
+MODULE_DESCRIPTION("ALSA SoC Exynos TM2 Audio Support");
+MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/sh/Kconfig b/sound/soc/sh/Kconfig
index 6db6405d952f..147ebecfed94 100644
--- a/sound/soc/sh/Kconfig
+++ b/sound/soc/sh/Kconfig
@@ -1,5 +1,5 @@
 menu "SoC Audio support for SuperH"
-	depends on SUPERH || ARCH_SHMOBILE
+	depends on SUPERH || ARCH_SHMOBILE || COMPILE_TEST
 
 config SND_SOC_PCM_SH7760
 	tristate "SoC Audio support for Renesas SH7760"
@@ -37,6 +37,7 @@ config SND_SOC_SH4_SIU
 config SND_SOC_RCAR
 	tristate "R-Car series SRU/SCU/SSIU/SSI support"
 	depends on COMMON_CLK
+	depends on OF || COMPILE_TEST
 	select SND_SIMPLE_CARD
 	select REGMAP_MMIO
 	help
diff --git a/sound/soc/sh/rcar/adg.c b/sound/soc/sh/rcar/adg.c
index 2145957d0229..85a33ac0a5c4 100644
--- a/sound/soc/sh/rcar/adg.c
+++ b/sound/soc/sh/rcar/adg.c
@@ -34,6 +34,9 @@ struct rsnd_adg {
 	struct clk_onecell_data onecell;
 	struct rsnd_mod mod;
 	u32 flags;
+	u32 ckr;
+	u32 rbga;
+	u32 rbgb;
 
 	int rbga_rate_for_441khz; /* RBGA */
 	int rbgb_rate_for_48khz;  /* RBGB */
@@ -316,9 +319,11 @@ int rsnd_adg_ssi_clk_try_start(struct rsnd_mod *ssi_mod, unsigned int rate)
 	struct rsnd_priv *priv = rsnd_mod_to_priv(ssi_mod);
 	struct rsnd_adg *adg = rsnd_priv_to_adg(priv);
 	struct device *dev = rsnd_priv_to_dev(priv);
+	struct rsnd_mod *adg_mod = rsnd_mod_get(adg);
 	struct clk *clk;
 	int i;
 	u32 data;
+	u32 ckr = 0;
 	int sel_table[] = {
 		[CLKA] = 0x1,
 		[CLKB] = 0x2,
@@ -360,15 +365,14 @@ found_clock:
 	rsnd_adg_set_ssi_clk(ssi_mod, data);
 
 	if (!(adg_mode_flags(adg) & LRCLK_ASYNC)) {
-		struct rsnd_mod *adg_mod = rsnd_mod_get(adg);
-		u32 ckr = 0;
-
 		if (0 == (rate % 8000))
 			ckr = 0x80000000;
-
-		rsnd_mod_bset(adg_mod, SSICKR, 0x80000000, ckr);
 	}
 
+	rsnd_mod_bset(adg_mod, BRGCKR, 0x80FF0000, adg->ckr | ckr);
+	rsnd_mod_write(adg_mod, BRRA,  adg->rbga);
+	rsnd_mod_write(adg_mod, BRRB,  adg->rbgb);
+
 	dev_dbg(dev, "ADG: %s[%d] selects 0x%x for %d\n",
 		rsnd_mod_name(ssi_mod), rsnd_mod_id(ssi_mod),
 		data, rate);
@@ -376,6 +380,25 @@ found_clock:
 	return 0;
 }
 
+void rsnd_adg_clk_control(struct rsnd_priv *priv, int enable)
+{
+	struct rsnd_adg *adg = rsnd_priv_to_adg(priv);
+	struct device *dev = rsnd_priv_to_dev(priv);
+	struct clk *clk;
+	int i, ret;
+
+	for_each_rsnd_clk(clk, adg, i) {
+		ret = 0;
+		if (enable)
+			ret = clk_prepare_enable(clk);
+		else
+			clk_disable_unprepare(clk);
+
+		if (ret < 0)
+			dev_warn(dev, "can't use clk %d\n", i);
+	}
+}
+
 static void rsnd_adg_get_clkin(struct rsnd_priv *priv,
 			       struct rsnd_adg *adg)
 {
@@ -387,27 +410,21 @@ static void rsnd_adg_get_clkin(struct rsnd_priv *priv,
 		[CLKC]	= "clk_c",
 		[CLKI]	= "clk_i",
 	};
-	int i, ret;
+	int i;
 
 	for (i = 0; i < CLKMAX; i++) {
 		clk = devm_clk_get(dev, clk_name[i]);
 		adg->clk[i] = IS_ERR(clk) ? NULL : clk;
 	}
 
-	for_each_rsnd_clk(clk, adg, i) {
-		ret = clk_prepare_enable(clk);
-		if (ret < 0)
-			dev_warn(dev, "can't use clk %d\n", i);
-
+	for_each_rsnd_clk(clk, adg, i)
 		dev_dbg(dev, "clk %d : %p : %ld\n", i, clk, clk_get_rate(clk));
-	}
 }
 
 static void rsnd_adg_get_clkout(struct rsnd_priv *priv,
 				struct rsnd_adg *adg)
 {
 	struct clk *clk;
-	struct rsnd_mod *adg_mod = rsnd_mod_get(adg);
 	struct device *dev = rsnd_priv_to_dev(priv);
 	struct device_node *np = dev->of_node;
 	u32 ckr, rbgx, rbga, rbgb;
@@ -532,13 +549,13 @@ static void rsnd_adg_get_clkout(struct rsnd_priv *priv,
 		}
 	}
 
-	rsnd_mod_bset(adg_mod, SSICKR, 0x80FF0000, ckr);
-	rsnd_mod_write(adg_mod, BRRA,  rbga);
-	rsnd_mod_write(adg_mod, BRRB,  rbgb);
+	adg->ckr = ckr;
+	adg->rbga = rbga;
+	adg->rbgb = rbgb;
 
 	for_each_rsnd_clkout(clk, adg, i)
 		dev_dbg(dev, "clkout %d : %p : %ld\n", i, clk, clk_get_rate(clk));
-	dev_dbg(dev, "SSICKR = 0x%08x, BRRA/BRRB = 0x%x/0x%x\n",
+	dev_dbg(dev, "BRGCKR = 0x%08x, BRRA/BRRB = 0x%x/0x%x\n",
 		ckr, rbga, rbgb);
 }
 
@@ -565,16 +582,12 @@ int rsnd_adg_probe(struct rsnd_priv *priv)
 
 	priv->adg = adg;
 
+	rsnd_adg_clk_enable(priv);
+
 	return 0;
 }
 
 void rsnd_adg_remove(struct rsnd_priv *priv)
 {
-	struct rsnd_adg *adg = rsnd_priv_to_adg(priv);
-	struct clk *clk;
-	int i;
-
-	for_each_rsnd_clk(clk, adg, i) {
-		clk_disable_unprepare(clk);
-	}
+	rsnd_adg_clk_disable(priv);
 }
diff --git a/sound/soc/sh/rcar/core.c b/sound/soc/sh/rcar/core.c
index f18141098b50..4bd68de76130 100644
--- a/sound/soc/sh/rcar/core.c
+++ b/sound/soc/sh/rcar/core.c
@@ -306,7 +306,7 @@ u32 rsnd_get_adinr_bit(struct rsnd_mod *mod, struct rsnd_dai_stream *io)
  */
 u32 rsnd_get_dalign(struct rsnd_mod *mod, struct rsnd_dai_stream *io)
 {
-	struct rsnd_mod *ssi = rsnd_io_to_mod_ssi(io);
+	struct rsnd_mod *ssiu = rsnd_io_to_mod_ssiu(io);
 	struct rsnd_mod *target;
 	struct snd_pcm_runtime *runtime = rsnd_io_to_runtime(io);
 	u32 val = 0x76543210;
@@ -315,11 +315,11 @@ u32 rsnd_get_dalign(struct rsnd_mod *mod, struct rsnd_dai_stream *io)
 	if (rsnd_io_is_play(io)) {
 		struct rsnd_mod *src = rsnd_io_to_mod_src(io);
 
-		target = src ? src : ssi;
+		target = src ? src : ssiu;
 	} else {
 		struct rsnd_mod *cmd = rsnd_io_to_mod_cmd(io);
 
-		target = cmd ? cmd : ssi;
+		target = cmd ? cmd : ssiu;
 	}
 
 	mask <<= runtime->channels * 4;
@@ -348,32 +348,28 @@ u32 rsnd_get_dalign(struct rsnd_mod *mod, struct rsnd_dai_stream *io)
 /*
  *	rsnd_dai functions
  */
-#define rsnd_mod_call(idx, io, func, param...)			\
-({								\
-	struct rsnd_priv *priv = rsnd_mod_to_priv(mod);		\
-	struct rsnd_mod *mod = (io)->mod[idx];			\
-	struct device *dev = rsnd_priv_to_dev(priv);		\
-	u32 *status = mod->get_status(io, mod, idx);			\
-	u32 mask = 0xF << __rsnd_mod_shift_##func;			\
-	u8 val  = (*status >> __rsnd_mod_shift_##func) & 0xF;		\
-	u8 add  = ((val + __rsnd_mod_add_##func) & 0xF);		\
-	int ret = 0;							\
-	int call = (val == __rsnd_mod_call_##func) && (mod)->ops->func;	\
-	if (add == 0xF)							\
-		call = 0;						\
-	else								\
-		*status = (*status & ~mask) +				\
-			(add << __rsnd_mod_shift_##func);		\
-	dev_dbg(dev, "%s[%d]\t0x%08x %s\n",				\
-		rsnd_mod_name(mod), rsnd_mod_id(mod),			\
-		*status, call ? #func : "");				\
-	if (call)							\
-		ret = (mod)->ops->func(mod, io, param);			\
-	if (ret)							\
-		dev_dbg(dev, "%s[%d] : rsnd_mod_call error %d\n",	\
-			rsnd_mod_name(mod), rsnd_mod_id(mod), ret);	\
-	ret;								\
-})
+struct rsnd_mod *rsnd_mod_next(int *iterator,
+			       struct rsnd_dai_stream *io,
+			       enum rsnd_mod_type *array,
+			       int array_size)
+{
+	struct rsnd_mod *mod;
+	enum rsnd_mod_type type;
+	int max = array ? array_size : RSND_MOD_MAX;
+
+	for (; *iterator < max; (*iterator)++) {
+		type = (array) ? array[*iterator] : *iterator;
+		mod = io->mod[type];
+		if (!mod)
+			continue;
+
+		(*iterator)++;
+
+		return mod;
+	}
+
+	return NULL;
+}
 
 static enum rsnd_mod_type rsnd_mod_sequence[][RSND_MOD_MAX] = {
 	{
@@ -409,19 +405,49 @@ static enum rsnd_mod_type rsnd_mod_sequence[][RSND_MOD_MAX] = {
 	},
 };
 
-#define rsnd_dai_call(fn, io, param...)				\
-({								\
-	struct rsnd_mod *mod;					\
-	int type, is_play = rsnd_io_is_play(io);		\
-	int ret = 0, i;						\
-	for (i = 0; i < RSND_MOD_MAX; i++) {			\
-		type = rsnd_mod_sequence[is_play][i];		\
-		mod = (io)->mod[type];				\
-		if (!mod)					\
-			continue;				\
-		ret |= rsnd_mod_call(type, io, fn, param);	\
-	}							\
-	ret;							\
+static int rsnd_status_update(u32 *status,
+			      int shift, int add, int timing)
+{
+	u32 mask	= 0xF << shift;
+	u8 val		= (*status >> shift) & 0xF;
+	u8 next_val	= (val + add) & 0xF;
+	int func_call	= (val == timing);
+
+	if (next_val == 0xF) /* underflow case */
+		func_call = 0;
+	else
+		*status = (*status & ~mask) + (next_val << shift);
+
+	return func_call;
+}
+
+#define rsnd_dai_call(fn, io, param...)					\
+({									\
+	struct rsnd_priv *priv = rsnd_io_to_priv(io);			\
+	struct device *dev = rsnd_priv_to_dev(priv);			\
+	struct rsnd_mod *mod;						\
+	int is_play = rsnd_io_is_play(io);				\
+	int ret = 0, i;							\
+	enum rsnd_mod_type *types = rsnd_mod_sequence[is_play];		\
+	for_each_rsnd_mod_arrays(i, mod, io, types, RSND_MOD_MAX) {	\
+		int tmp = 0;						\
+		u32 *status = mod->get_status(io, mod, types[i]);	\
+		int func_call = rsnd_status_update(status,		\
+						__rsnd_mod_shift_##fn,	\
+						__rsnd_mod_add_##fn,	\
+						__rsnd_mod_call_##fn);	\
+		dev_dbg(dev, "%s[%d]\t0x%08x %s\n",			\
+			rsnd_mod_name(mod), rsnd_mod_id(mod), *status,	\
+			(func_call && (mod)->ops->fn) ? #fn : "");	\
+		if (func_call && (mod)->ops->fn)			\
+			tmp = (mod)->ops->fn(mod, io, param);		\
+		if (tmp)						\
+			dev_err(dev, "%s[%d] : %s error %d\n",		\
+				rsnd_mod_name(mod), rsnd_mod_id(mod),	\
+						     #fn, tmp);		\
+		ret |= tmp;						\
+	}								\
+	ret;								\
 })
 
 int rsnd_dai_connect(struct rsnd_mod *mod,
@@ -690,7 +716,33 @@ static int rsnd_soc_set_dai_tdm_slot(struct snd_soc_dai *dai,
 	return 0;
 }
 
+static int rsnd_soc_dai_startup(struct snd_pcm_substream *substream,
+				struct snd_soc_dai *dai)
+{
+	struct rsnd_dai *rdai = rsnd_dai_to_rdai(dai);
+	struct rsnd_dai_stream *io = rsnd_rdai_to_io(rdai, substream);
+
+	/*
+	 * call rsnd_dai_call without spinlock
+	 */
+	return rsnd_dai_call(nolock_start, io, priv);
+}
+
+static void rsnd_soc_dai_shutdown(struct snd_pcm_substream *substream,
+				  struct snd_soc_dai *dai)
+{
+	struct rsnd_dai *rdai = rsnd_dai_to_rdai(dai);
+	struct rsnd_dai_stream *io = rsnd_rdai_to_io(rdai, substream);
+
+	/*
+	 * call rsnd_dai_call without spinlock
+	 */
+	rsnd_dai_call(nolock_stop, io, priv);
+}
+
 static const struct snd_soc_dai_ops rsnd_soc_dai_ops = {
+	.startup	= rsnd_soc_dai_startup,
+	.shutdown	= rsnd_soc_dai_shutdown,
 	.trigger	= rsnd_soc_dai_trigger,
 	.set_fmt	= rsnd_soc_dai_set_fmt,
 	.set_tdm_slot	= rsnd_soc_set_dai_tdm_slot,
@@ -993,7 +1045,11 @@ static int __rsnd_kctrl_new(struct rsnd_mod *mod,
 
 void _rsnd_kctrl_remove(struct rsnd_kctrl_cfg *cfg)
 {
-	snd_ctl_remove(cfg->card, cfg->kctrl);
+	if (cfg->card && cfg->kctrl)
+		snd_ctl_remove(cfg->card, cfg->kctrl);
+
+	cfg->card = NULL;
+	cfg->kctrl = NULL;
 }
 
 int rsnd_kctrl_new_m(struct rsnd_mod *mod,
@@ -1070,8 +1126,8 @@ static int rsnd_pcm_new(struct snd_soc_pcm_runtime *rtd)
 
 	return snd_pcm_lib_preallocate_pages_for_all(
 		rtd->pcm,
-		SNDRV_DMA_TYPE_DEV,
-		rtd->card->snd_card->dev,
+		SNDRV_DMA_TYPE_CONTINUOUS,
+		snd_dma_continuous_data(GFP_KERNEL),
 		PREALLOC_BUFFER, PREALLOC_BUFFER_MAX);
 }
 
@@ -1092,6 +1148,7 @@ static int rsnd_rdai_continuance_probe(struct rsnd_priv *priv,
 	ret = rsnd_dai_call(probe, io, priv);
 	if (ret == -EAGAIN) {
 		struct rsnd_mod *ssi_mod = rsnd_io_to_mod_ssi(io);
+		struct rsnd_mod *mod;
 		int i;
 
 		/*
@@ -1111,8 +1168,8 @@ static int rsnd_rdai_continuance_probe(struct rsnd_priv *priv,
 		 * remove all mod from io
 		 * and, re connect ssi
 		 */
-		for (i = 0; i < RSND_MOD_MAX; i++)
-			rsnd_dai_disconnect((io)->mod[i], io, i);
+		for_each_rsnd_mod(i, mod, io)
+			rsnd_dai_disconnect(mod, io, i);
 		rsnd_dai_connect(ssi_mod, io, RSND_MOD_SSI);
 
 		/*
@@ -1251,9 +1308,33 @@ static int rsnd_remove(struct platform_device *pdev)
 	return ret;
 }
 
+static int rsnd_suspend(struct device *dev)
+{
+	struct rsnd_priv *priv = dev_get_drvdata(dev);
+
+	rsnd_adg_clk_disable(priv);
+
+	return 0;
+}
+
+static int rsnd_resume(struct device *dev)
+{
+	struct rsnd_priv *priv = dev_get_drvdata(dev);
+
+	rsnd_adg_clk_enable(priv);
+
+	return 0;
+}
+
+static struct dev_pm_ops rsnd_pm_ops = {
+	.suspend		= rsnd_suspend,
+	.resume			= rsnd_resume,
+};
+
 static struct platform_driver rsnd_driver = {
 	.driver	= {
 		.name	= "rcar_sound",
+		.pm	= &rsnd_pm_ops,
 		.of_match_table = rsnd_of_match,
 	},
 	.probe		= rsnd_probe,
diff --git a/sound/soc/sh/rcar/dma.c b/sound/soc/sh/rcar/dma.c
index 6bc93cbb3049..1f405c833867 100644
--- a/sound/soc/sh/rcar/dma.c
+++ b/sound/soc/sh/rcar/dma.c
@@ -25,6 +25,10 @@
 
 struct rsnd_dmaen {
 	struct dma_chan		*chan;
+	dma_addr_t		dma_buf;
+	unsigned int		dma_len;
+	unsigned int		dma_period;
+	unsigned int		dma_cnt;
 };
 
 struct rsnd_dmapp {
@@ -34,6 +38,8 @@ struct rsnd_dmapp {
 
 struct rsnd_dma {
 	struct rsnd_mod		mod;
+	struct rsnd_mod		*mod_from;
+	struct rsnd_mod		*mod_to;
 	dma_addr_t		src_addr;
 	dma_addr_t		dst_addr;
 	union {
@@ -56,10 +62,38 @@ struct rsnd_dma_ctrl {
 /*
  *		Audio DMAC
  */
+#define rsnd_dmaen_sync(dmaen, io, i)	__rsnd_dmaen_sync(dmaen, io, i, 1)
+#define rsnd_dmaen_unsync(dmaen, io, i)	__rsnd_dmaen_sync(dmaen, io, i, 0)
+static void __rsnd_dmaen_sync(struct rsnd_dmaen *dmaen, struct rsnd_dai_stream *io,
+			      int i, int sync)
+{
+	struct device *dev = dmaen->chan->device->dev;
+	enum dma_data_direction dir;
+	int is_play = rsnd_io_is_play(io);
+	dma_addr_t buf;
+	int len, max;
+	size_t period;
+
+	len	= dmaen->dma_len;
+	period	= dmaen->dma_period;
+	max	= len / period;
+	i	= i % max;
+	buf	= dmaen->dma_buf + (period * i);
+
+	dir = is_play ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+
+	if (sync)
+		dma_sync_single_for_device(dev, buf, period, dir);
+	else
+		dma_sync_single_for_cpu(dev, buf, period, dir);
+}
+
 static void __rsnd_dmaen_complete(struct rsnd_mod *mod,
 				  struct rsnd_dai_stream *io)
 {
 	struct rsnd_priv *priv = rsnd_mod_to_priv(mod);
+	struct rsnd_dma *dma = rsnd_mod_to_dma(mod);
+	struct rsnd_dmaen *dmaen = rsnd_dma_to_dmaen(dma);
 	bool elapsed = false;
 	unsigned long flags;
 
@@ -76,9 +110,22 @@ static void __rsnd_dmaen_complete(struct rsnd_mod *mod,
 	 */
 	spin_lock_irqsave(&priv->lock, flags);
 
-	if (rsnd_io_is_working(io))
+	if (rsnd_io_is_working(io)) {
+		rsnd_dmaen_unsync(dmaen, io, dmaen->dma_cnt);
+
+		/*
+		 * Next period is already started.
+		 * Let's sync Next Next period
+		 * see
+		 *	rsnd_dmaen_start()
+		 */
+		rsnd_dmaen_sync(dmaen, io, dmaen->dma_cnt + 2);
+
 		elapsed = rsnd_dai_pointer_update(io, io->byte_per_period);
 
+		dmaen->dma_cnt++;
+	}
+
 	spin_unlock_irqrestore(&priv->lock, flags);
 
 	if (elapsed)
@@ -92,6 +139,20 @@ static void rsnd_dmaen_complete(void *data)
 	rsnd_mod_interrupt(mod, __rsnd_dmaen_complete);
 }
 
+static struct dma_chan *rsnd_dmaen_request_channel(struct rsnd_dai_stream *io,
+						   struct rsnd_mod *mod_from,
+						   struct rsnd_mod *mod_to)
+{
+	if ((!mod_from && !mod_to) ||
+	    (mod_from && mod_to))
+		return NULL;
+
+	if (mod_from)
+		return rsnd_mod_dma_req(io, mod_from);
+	else
+		return rsnd_mod_dma_req(io, mod_to);
+}
+
 static int rsnd_dmaen_stop(struct rsnd_mod *mod,
 			   struct rsnd_dai_stream *io,
 			   struct rsnd_priv *priv)
@@ -99,7 +160,66 @@ static int rsnd_dmaen_stop(struct rsnd_mod *mod,
 	struct rsnd_dma *dma = rsnd_mod_to_dma(mod);
 	struct rsnd_dmaen *dmaen = rsnd_dma_to_dmaen(dma);
 
-	dmaengine_terminate_all(dmaen->chan);
+	if (dmaen->chan) {
+		int is_play = rsnd_io_is_play(io);
+
+		dmaengine_terminate_all(dmaen->chan);
+		dma_unmap_single(dmaen->chan->device->dev,
+				 dmaen->dma_buf, dmaen->dma_len,
+				 is_play ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
+	}
+
+	return 0;
+}
+
+static int rsnd_dmaen_nolock_stop(struct rsnd_mod *mod,
+				   struct rsnd_dai_stream *io,
+				   struct rsnd_priv *priv)
+{
+	struct rsnd_dma *dma = rsnd_mod_to_dma(mod);
+	struct rsnd_dmaen *dmaen = rsnd_dma_to_dmaen(dma);
+
+	/*
+	 * DMAEngine release uses mutex lock.
+	 * Thus, it shouldn't be called under spinlock.
+	 * Let's call it under nolock_start
+	 */
+	if (dmaen->chan)
+		dma_release_channel(dmaen->chan);
+
+	dmaen->chan = NULL;
+
+	return 0;
+}
+
+static int rsnd_dmaen_nolock_start(struct rsnd_mod *mod,
+			    struct rsnd_dai_stream *io,
+			    struct rsnd_priv *priv)
+{
+	struct rsnd_dma *dma = rsnd_mod_to_dma(mod);
+	struct rsnd_dmaen *dmaen = rsnd_dma_to_dmaen(dma);
+	struct device *dev = rsnd_priv_to_dev(priv);
+
+	if (dmaen->chan) {
+		dev_err(dev, "it already has dma channel\n");
+		return -EIO;
+	}
+
+	/*
+	 * DMAEngine request uses mutex lock.
+	 * Thus, it shouldn't be called under spinlock.
+	 * Let's call it under nolock_start
+	 */
+	dmaen->chan = rsnd_dmaen_request_channel(io,
+						 dma->mod_from,
+						 dma->mod_to);
+	if (IS_ERR_OR_NULL(dmaen->chan)) {
+		int ret = PTR_ERR(dmaen->chan);
+
+		dmaen->chan = NULL;
+		dev_err(dev, "can't get dma channel\n");
+		return ret;
+	}
 
 	return 0;
 }
@@ -113,12 +233,41 @@ static int rsnd_dmaen_start(struct rsnd_mod *mod,
 	struct snd_pcm_substream *substream = io->substream;
 	struct device *dev = rsnd_priv_to_dev(priv);
 	struct dma_async_tx_descriptor *desc;
+	struct dma_slave_config cfg = {};
+	dma_addr_t buf;
+	size_t len;
+	size_t period;
 	int is_play = rsnd_io_is_play(io);
+	int i;
+	int ret;
+
+	cfg.direction	= is_play ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
+	cfg.src_addr	= dma->src_addr;
+	cfg.dst_addr	= dma->dst_addr;
+	cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+
+	dev_dbg(dev, "%s[%d] %pad -> %pad\n",
+		rsnd_mod_name(mod), rsnd_mod_id(mod),
+		&cfg.src_addr, &cfg.dst_addr);
+
+	ret = dmaengine_slave_config(dmaen->chan, &cfg);
+	if (ret < 0)
+		return ret;
+
+	len	= snd_pcm_lib_buffer_bytes(substream);
+	period	= snd_pcm_lib_period_bytes(substream);
+	buf	= dma_map_single(dmaen->chan->device->dev,
+				 substream->runtime->dma_area,
+				 len,
+				 is_play ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
+	if (dma_mapping_error(dmaen->chan->device->dev, buf)) {
+		dev_err(dev, "dma map failed\n");
+		return -EIO;
+	}
 
 	desc = dmaengine_prep_dma_cyclic(dmaen->chan,
-					 substream->runtime->dma_addr,
-					 snd_pcm_lib_buffer_bytes(substream),
-					 snd_pcm_lib_period_bytes(substream),
+					 buf, len, period,
 					 is_play ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM,
 					 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 
@@ -130,6 +279,19 @@ static int rsnd_dmaen_start(struct rsnd_mod *mod,
 	desc->callback		= rsnd_dmaen_complete;
 	desc->callback_param	= rsnd_mod_get(dma);
 
+	dmaen->dma_buf		= buf;
+	dmaen->dma_len		= len;
+	dmaen->dma_period	= period;
+	dmaen->dma_cnt		= 0;
+
+	/*
+	 * synchronize this and next period
+	 * see
+	 *	__rsnd_dmaen_complete()
+	 */
+	for (i = 0; i < 2; i++)
+		rsnd_dmaen_sync(dmaen, io, i);
+
 	if (dmaengine_submit(desc) < 0) {
 		dev_err(dev, "dmaengine_submit() fail\n");
 		return -EIO;
@@ -143,124 +305,55 @@ static int rsnd_dmaen_start(struct rsnd_mod *mod,
 struct dma_chan *rsnd_dma_request_channel(struct device_node *of_node,
 					  struct rsnd_mod *mod, char *name)
 {
-	struct dma_chan *chan;
+	struct dma_chan *chan = NULL;
 	struct device_node *np;
 	int i = 0;
 
 	for_each_child_of_node(of_node, np) {
-		if (i == rsnd_mod_id(mod))
-			break;
+		if (i == rsnd_mod_id(mod) && (!chan))
+			chan = of_dma_request_slave_channel(np, name);
 		i++;
 	}
 
-	chan = of_dma_request_slave_channel(np, name);
-
-	of_node_put(np);
+	/* It should call of_node_put(), since, it is rsnd_xxx_of_node() */
 	of_node_put(of_node);
 
 	return chan;
 }
 
-static struct dma_chan *rsnd_dmaen_request_channel(struct rsnd_dai_stream *io,
-						   struct rsnd_mod *mod_from,
-						   struct rsnd_mod *mod_to)
-{
-	if ((!mod_from && !mod_to) ||
-	    (mod_from && mod_to))
-		return NULL;
-
-	if (mod_from)
-		return rsnd_mod_dma_req(io, mod_from);
-	else
-		return rsnd_mod_dma_req(io, mod_to);
-}
-
-static int rsnd_dmaen_remove(struct rsnd_mod *mod,
-			      struct rsnd_dai_stream *io,
-			      struct rsnd_priv *priv)
-{
-	struct rsnd_dma *dma = rsnd_mod_to_dma(mod);
-	struct rsnd_dmaen *dmaen = rsnd_dma_to_dmaen(dma);
-
-	if (dmaen->chan)
-		dma_release_channel(dmaen->chan);
-
-	dmaen->chan = NULL;
-
-	return 0;
-}
-
 static int rsnd_dmaen_attach(struct rsnd_dai_stream *io,
-			   struct rsnd_dma *dma, int id,
+			   struct rsnd_dma *dma,
 			   struct rsnd_mod *mod_from, struct rsnd_mod *mod_to)
 {
-	struct rsnd_mod *mod = rsnd_mod_get(dma);
-	struct rsnd_dmaen *dmaen = rsnd_dma_to_dmaen(dma);
 	struct rsnd_priv *priv = rsnd_io_to_priv(io);
 	struct rsnd_dma_ctrl *dmac = rsnd_priv_to_dmac(priv);
-	struct device *dev = rsnd_priv_to_dev(priv);
-	struct dma_slave_config cfg = {};
-	int is_play = rsnd_io_is_play(io);
-	int ret;
-
-	if (dmaen->chan) {
-		dev_err(dev, "it already has dma channel\n");
-		return -EIO;
-	}
-
-	if (dev->of_node) {
-		dmaen->chan = rsnd_dmaen_request_channel(io, mod_from, mod_to);
-	} else {
-		dma_cap_mask_t mask;
-
-		dma_cap_zero(mask);
-		dma_cap_set(DMA_SLAVE, mask);
+	struct dma_chan *chan;
 
-		dmaen->chan = dma_request_channel(mask, shdma_chan_filter,
-						  (void *)(uintptr_t)id);
-	}
-	if (IS_ERR_OR_NULL(dmaen->chan)) {
-		dmaen->chan = NULL;
-		dev_err(dev, "can't get dma channel\n");
-		goto rsnd_dma_channel_err;
+	/* try to get DMAEngine channel */
+	chan = rsnd_dmaen_request_channel(io, mod_from, mod_to);
+	if (IS_ERR_OR_NULL(chan)) {
+		/*
+		 * DMA failed. try to PIO mode
+		 * see
+		 *	rsnd_ssi_fallback()
+		 *	rsnd_rdai_continuance_probe()
+		 */
+		return -EAGAIN;
 	}
 
-	cfg.direction	= is_play ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
-	cfg.src_addr	= dma->src_addr;
-	cfg.dst_addr	= dma->dst_addr;
-	cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
-	cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
-
-	dev_dbg(dev, "%s[%d] %pad -> %pad\n",
-		rsnd_mod_name(mod), rsnd_mod_id(mod),
-		&cfg.src_addr, &cfg.dst_addr);
-
-	ret = dmaengine_slave_config(dmaen->chan, &cfg);
-	if (ret < 0)
-		goto rsnd_dma_attach_err;
+	dma_release_channel(chan);
 
 	dmac->dmaen_num++;
 
 	return 0;
-
-rsnd_dma_attach_err:
-	rsnd_dmaen_remove(mod, io, priv);
-rsnd_dma_channel_err:
-
-	/*
-	 * DMA failed. try to PIO mode
-	 * see
-	 *	rsnd_ssi_fallback()
-	 *	rsnd_rdai_continuance_probe()
-	 */
-	return -EAGAIN;
 }
 
 static struct rsnd_mod_ops rsnd_dmaen_ops = {
 	.name	= "audmac",
+	.nolock_start = rsnd_dmaen_nolock_start,
+	.nolock_stop  = rsnd_dmaen_nolock_stop,
 	.start	= rsnd_dmaen_start,
 	.stop	= rsnd_dmaen_stop,
-	.remove	= rsnd_dmaen_remove,
 };
 
 /*
@@ -394,7 +487,7 @@ static int rsnd_dmapp_start(struct rsnd_mod *mod,
 }
 
 static int rsnd_dmapp_attach(struct rsnd_dai_stream *io,
-			     struct rsnd_dma *dma, int id,
+			     struct rsnd_dma *dma,
 			     struct rsnd_mod *mod_from, struct rsnd_mod *mod_to)
 {
 	struct rsnd_dmapp *dmapp = rsnd_dma_to_dmapp(dma);
@@ -627,7 +720,7 @@ static void rsnd_dma_of_path(struct rsnd_mod *this,
 }
 
 int rsnd_dma_attach(struct rsnd_dai_stream *io, struct rsnd_mod *mod,
-		    struct rsnd_mod **dma_mod, int id)
+		    struct rsnd_mod **dma_mod)
 {
 	struct rsnd_mod *mod_from = NULL;
 	struct rsnd_mod *mod_to = NULL;
@@ -636,7 +729,7 @@ int rsnd_dma_attach(struct rsnd_dai_stream *io, struct rsnd_mod *mod,
 	struct device *dev = rsnd_priv_to_dev(priv);
 	struct rsnd_mod_ops *ops;
 	enum rsnd_mod_type type;
-	int (*attach)(struct rsnd_dai_stream *io, struct rsnd_dma *dma, int id,
+	int (*attach)(struct rsnd_dai_stream *io, struct rsnd_dma *dma,
 		      struct rsnd_mod *mod_from, struct rsnd_mod *mod_to);
 	int is_play = rsnd_io_is_play(io);
 	int ret, dma_id;
@@ -682,9 +775,6 @@ int rsnd_dma_attach(struct rsnd_dai_stream *io, struct rsnd_mod *mod,
 
 		*dma_mod = rsnd_mod_get(dma);
 
-		dma->src_addr = rsnd_dma_addr(io, mod_from, is_play, 1);
-		dma->dst_addr = rsnd_dma_addr(io, mod_to,   is_play, 0);
-
 		ret = rsnd_mod_init(priv, *dma_mod, ops, NULL,
 				    rsnd_mod_get_status, type, dma_id);
 		if (ret < 0)
@@ -695,9 +785,14 @@ int rsnd_dma_attach(struct rsnd_dai_stream *io, struct rsnd_mod *mod,
 			rsnd_mod_name(mod_from), rsnd_mod_id(mod_from),
 			rsnd_mod_name(mod_to),   rsnd_mod_id(mod_to));
 
-		ret = attach(io, dma, id, mod_from, mod_to);
+		ret = attach(io, dma, mod_from, mod_to);
 		if (ret < 0)
 			return ret;
+
+		dma->src_addr = rsnd_dma_addr(io, mod_from, is_play, 1);
+		dma->dst_addr = rsnd_dma_addr(io, mod_to,   is_play, 0);
+		dma->mod_from = mod_from;
+		dma->mod_to   = mod_to;
 	}
 
 	ret = rsnd_dai_connect(*dma_mod, io, type);
diff --git a/sound/soc/sh/rcar/dvc.c b/sound/soc/sh/rcar/dvc.c
index 02d971f69eff..cf8f59cdd8d7 100644
--- a/sound/soc/sh/rcar/dvc.c
+++ b/sound/soc/sh/rcar/dvc.c
@@ -48,8 +48,6 @@ struct rsnd_dvc {
 
 #define rsnd_dvc_get(priv, id) ((struct rsnd_dvc *)(priv->dvc) + id)
 #define rsnd_dvc_nr(priv) ((priv)->dvc_nr)
-#define rsnd_dvc_of_node(priv) \
-	of_get_child_by_name(rsnd_priv_to_dev(priv)->of_node, "rcar_sound,dvc")
 
 #define rsnd_mod_to_dvc(_mod)	\
 	container_of((_mod), struct rsnd_dvc, mod)
diff --git a/sound/soc/sh/rcar/gen.c b/sound/soc/sh/rcar/gen.c
index 7d2fdf8dd188..63b6d3c28021 100644
--- a/sound/soc/sh/rcar/gen.c
+++ b/sound/soc/sh/rcar/gen.c
@@ -211,6 +211,14 @@ static int rsnd_gen2_probe(struct rsnd_priv *priv)
 		RSND_GEN_S_REG(SSI_MODE1,	0x804),
 		RSND_GEN_S_REG(SSI_MODE2,	0x808),
 		RSND_GEN_S_REG(SSI_CONTROL,	0x810),
+		RSND_GEN_S_REG(SSI_SYS_STATUS0,	0x840),
+		RSND_GEN_S_REG(SSI_SYS_STATUS1,	0x844),
+		RSND_GEN_S_REG(SSI_SYS_STATUS2,	0x848),
+		RSND_GEN_S_REG(SSI_SYS_STATUS3,	0x84c),
+		RSND_GEN_S_REG(SSI_SYS_STATUS4,	0x880),
+		RSND_GEN_S_REG(SSI_SYS_STATUS5,	0x884),
+		RSND_GEN_S_REG(SSI_SYS_STATUS6,	0x888),
+		RSND_GEN_S_REG(SSI_SYS_STATUS7,	0x88c),
 
 		/* FIXME: it needs SSI_MODE2/3 in the future */
 		RSND_GEN_M_REG(SSI_BUSIF_MODE,	0x0,	0x80),
@@ -311,7 +319,7 @@ static int rsnd_gen2_probe(struct rsnd_priv *priv)
 	static const struct rsnd_regmap_field_conf conf_adg[] = {
 		RSND_GEN_S_REG(BRRA,		0x00),
 		RSND_GEN_S_REG(BRRB,		0x04),
-		RSND_GEN_S_REG(SSICKR,		0x08),
+		RSND_GEN_S_REG(BRGCKR,		0x08),
 		RSND_GEN_S_REG(AUDIO_CLK_SEL0,	0x0c),
 		RSND_GEN_S_REG(AUDIO_CLK_SEL1,	0x10),
 		RSND_GEN_S_REG(AUDIO_CLK_SEL2,	0x14),
@@ -362,7 +370,7 @@ static int rsnd_gen1_probe(struct rsnd_priv *priv)
 	static const struct rsnd_regmap_field_conf conf_adg[] = {
 		RSND_GEN_S_REG(BRRA,		0x00),
 		RSND_GEN_S_REG(BRRB,		0x04),
-		RSND_GEN_S_REG(SSICKR,		0x08),
+		RSND_GEN_S_REG(BRGCKR,		0x08),
 		RSND_GEN_S_REG(AUDIO_CLK_SEL0,	0x0c),
 		RSND_GEN_S_REG(AUDIO_CLK_SEL1,	0x10),
 	};
diff --git a/sound/soc/sh/rcar/rsnd.h b/sound/soc/sh/rcar/rsnd.h
index a8f61d79333b..b90df77662df 100644
--- a/sound/soc/sh/rcar/rsnd.h
+++ b/sound/soc/sh/rcar/rsnd.h
@@ -43,17 +43,7 @@
  * see gen1/gen2 for detail
  */
 enum rsnd_reg {
-	/* SCU (SRC/SSIU/MIX/CTU/DVC) */
-	RSND_REG_SSI_MODE,		/* Gen2 only */
-	RSND_REG_SSI_MODE0,
-	RSND_REG_SSI_MODE1,
-	RSND_REG_SSI_MODE2,
-	RSND_REG_SSI_CONTROL,
-	RSND_REG_SSI_CTRL,		/* Gen2 only */
-	RSND_REG_SSI_BUSIF_MODE,	/* Gen2 only */
-	RSND_REG_SSI_BUSIF_ADINR,	/* Gen2 only */
-	RSND_REG_SSI_BUSIF_DALIGN,	/* Gen2 only */
-	RSND_REG_SSI_INT_ENABLE,	/* Gen2 only */
+	/* SCU (MIX/CTU/DVC) */
 	RSND_REG_SRC_I_BUSIF_MODE,
 	RSND_REG_SRC_O_BUSIF_MODE,
 	RSND_REG_SRC_ROUTE_MODE0,
@@ -63,29 +53,29 @@ enum rsnd_reg {
 	RSND_REG_SRC_IFSCR,
 	RSND_REG_SRC_IFSVR,
 	RSND_REG_SRC_SRCCR,
-	RSND_REG_SRC_CTRL,		/* Gen2 only */
-	RSND_REG_SRC_BSDSR,		/* Gen2 only */
-	RSND_REG_SRC_BSISR,		/* Gen2 only */
-	RSND_REG_SRC_INT_ENABLE0,	/* Gen2 only */
-	RSND_REG_SRC_BUSIF_DALIGN,	/* Gen2 only */
-	RSND_REG_SRCIN_TIMSEL0,		/* Gen2 only */
-	RSND_REG_SRCIN_TIMSEL1,		/* Gen2 only */
-	RSND_REG_SRCIN_TIMSEL2,		/* Gen2 only */
-	RSND_REG_SRCIN_TIMSEL3,		/* Gen2 only */
-	RSND_REG_SRCIN_TIMSEL4,		/* Gen2 only */
-	RSND_REG_SRCOUT_TIMSEL0,	/* Gen2 only */
-	RSND_REG_SRCOUT_TIMSEL1,	/* Gen2 only */
-	RSND_REG_SRCOUT_TIMSEL2,	/* Gen2 only */
-	RSND_REG_SRCOUT_TIMSEL3,	/* Gen2 only */
-	RSND_REG_SRCOUT_TIMSEL4,	/* Gen2 only */
+	RSND_REG_SRC_CTRL,
+	RSND_REG_SRC_BSDSR,
+	RSND_REG_SRC_BSISR,
+	RSND_REG_SRC_INT_ENABLE0,
+	RSND_REG_SRC_BUSIF_DALIGN,
+	RSND_REG_SRCIN_TIMSEL0,
+	RSND_REG_SRCIN_TIMSEL1,
+	RSND_REG_SRCIN_TIMSEL2,
+	RSND_REG_SRCIN_TIMSEL3,
+	RSND_REG_SRCIN_TIMSEL4,
+	RSND_REG_SRCOUT_TIMSEL0,
+	RSND_REG_SRCOUT_TIMSEL1,
+	RSND_REG_SRCOUT_TIMSEL2,
+	RSND_REG_SRCOUT_TIMSEL3,
+	RSND_REG_SRCOUT_TIMSEL4,
 	RSND_REG_SCU_SYS_STATUS0,
-	RSND_REG_SCU_SYS_STATUS1,	/* Gen2 only */
+	RSND_REG_SCU_SYS_STATUS1,
 	RSND_REG_SCU_SYS_INT_EN0,
-	RSND_REG_SCU_SYS_INT_EN1,	/* Gen2 only */
-	RSND_REG_CMD_CTRL,		/* Gen2 only */
-	RSND_REG_CMD_BUSIF_DALIGN,	/* Gen2 only */
+	RSND_REG_SCU_SYS_INT_EN1,
+	RSND_REG_CMD_CTRL,
+	RSND_REG_CMD_BUSIF_DALIGN,
 	RSND_REG_CMD_ROUTE_SLCT,
-	RSND_REG_CMDOUT_TIMSEL,		/* Gen2 only */
+	RSND_REG_CMDOUT_TIMSEL,
 	RSND_REG_CTU_SWRSR,
 	RSND_REG_CTU_CTUIR,
 	RSND_REG_CTU_ADINR,
@@ -147,18 +137,38 @@ enum rsnd_reg {
 	RSND_REG_DVC_VOL6R,
 	RSND_REG_DVC_VOL7R,
 	RSND_REG_DVC_DVUER,
-	RSND_REG_DVC_VRCTR,		/* Gen2 only */
-	RSND_REG_DVC_VRPDR,		/* Gen2 only */
-	RSND_REG_DVC_VRDBR,		/* Gen2 only */
+	RSND_REG_DVC_VRCTR,
+	RSND_REG_DVC_VRPDR,
+	RSND_REG_DVC_VRDBR,
 
 	/* ADG */
 	RSND_REG_BRRA,
 	RSND_REG_BRRB,
-	RSND_REG_SSICKR,
-	RSND_REG_DIV_EN,		/* Gen2 only */
+	RSND_REG_BRGCKR,
+	RSND_REG_DIV_EN,
 	RSND_REG_AUDIO_CLK_SEL0,
 	RSND_REG_AUDIO_CLK_SEL1,
-	RSND_REG_AUDIO_CLK_SEL2,	/* Gen2 only */
+	RSND_REG_AUDIO_CLK_SEL2,
+
+	/* SSIU */
+	RSND_REG_SSI_MODE,
+	RSND_REG_SSI_MODE0,
+	RSND_REG_SSI_MODE1,
+	RSND_REG_SSI_MODE2,
+	RSND_REG_SSI_CONTROL,
+	RSND_REG_SSI_CTRL,
+	RSND_REG_SSI_BUSIF_MODE,
+	RSND_REG_SSI_BUSIF_ADINR,
+	RSND_REG_SSI_BUSIF_DALIGN,
+	RSND_REG_SSI_INT_ENABLE,
+	RSND_REG_SSI_SYS_STATUS0,
+	RSND_REG_SSI_SYS_STATUS1,
+	RSND_REG_SSI_SYS_STATUS2,
+	RSND_REG_SSI_SYS_STATUS3,
+	RSND_REG_SSI_SYS_STATUS4,
+	RSND_REG_SSI_SYS_STATUS5,
+	RSND_REG_SSI_SYS_STATUS6,
+	RSND_REG_SSI_SYS_STATUS7,
 
 	/* SSI */
 	RSND_REG_SSICR,
@@ -199,7 +209,7 @@ u32 rsnd_get_dalign(struct rsnd_mod *mod, struct rsnd_dai_stream *io);
  *	R-Car DMA
  */
 int rsnd_dma_attach(struct rsnd_dai_stream *io,
-		    struct rsnd_mod *mod, struct rsnd_mod **dma_mod, int id);
+		    struct rsnd_mod *mod, struct rsnd_mod **dma_mod);
 int rsnd_dma_probe(struct rsnd_priv *priv);
 struct dma_chan *rsnd_dma_request_channel(struct device_node *of_node,
 					  struct rsnd_mod *mod, char *name);
@@ -259,6 +269,12 @@ struct rsnd_mod_ops {
 	int (*fallback)(struct rsnd_mod *mod,
 			struct rsnd_dai_stream *io,
 			struct rsnd_priv *priv);
+	int (*nolock_start)(struct rsnd_mod *mod,
+		    struct rsnd_dai_stream *io,
+		    struct rsnd_priv *priv);
+	int (*nolock_stop)(struct rsnd_mod *mod,
+		    struct rsnd_dai_stream *io,
+		    struct rsnd_priv *priv);
 };
 
 struct rsnd_dai_stream;
@@ -278,7 +294,7 @@ struct rsnd_mod {
  *
  * 0xH0000CBA
  *
- * A	0: probe	1: remove
+ * A	0: nolock_start	1: nolock_stop
  * B	0: init		1: quit
  * C	0: start	1: stop
  *
@@ -288,19 +304,23 @@ struct rsnd_mod {
  * H	0: fallback
  * H	0: hw_params
  */
-#define __rsnd_mod_shift_probe		0
-#define __rsnd_mod_shift_remove		0
+#define __rsnd_mod_shift_nolock_start	0
+#define __rsnd_mod_shift_nolock_stop	0
 #define __rsnd_mod_shift_init		4
 #define __rsnd_mod_shift_quit		4
 #define __rsnd_mod_shift_start		8
 #define __rsnd_mod_shift_stop		8
+#define __rsnd_mod_shift_probe		28 /* always called */
+#define __rsnd_mod_shift_remove		28 /* always called */
 #define __rsnd_mod_shift_irq		28 /* always called */
 #define __rsnd_mod_shift_pcm_new	28 /* always called */
 #define __rsnd_mod_shift_fallback	28 /* always called */
 #define __rsnd_mod_shift_hw_params	28 /* always called */
 
-#define __rsnd_mod_add_probe		 1
-#define __rsnd_mod_add_remove		-1
+#define __rsnd_mod_add_probe		0
+#define __rsnd_mod_add_remove		0
+#define __rsnd_mod_add_nolock_start	 1
+#define __rsnd_mod_add_nolock_stop	-1
 #define __rsnd_mod_add_init		 1
 #define __rsnd_mod_add_quit		-1
 #define __rsnd_mod_add_start		 1
@@ -311,7 +331,7 @@ struct rsnd_mod {
 #define __rsnd_mod_add_hw_params	0
 
 #define __rsnd_mod_call_probe		0
-#define __rsnd_mod_call_remove		1
+#define __rsnd_mod_call_remove		0
 #define __rsnd_mod_call_init		0
 #define __rsnd_mod_call_quit		1
 #define __rsnd_mod_call_start		0
@@ -320,6 +340,8 @@ struct rsnd_mod {
 #define __rsnd_mod_call_pcm_new		0
 #define __rsnd_mod_call_fallback	0
 #define __rsnd_mod_call_hw_params	0
+#define __rsnd_mod_call_nolock_start	0
+#define __rsnd_mod_call_nolock_stop	1
 
 #define rsnd_mod_to_priv(mod) ((mod)->priv)
 #define rsnd_mod_id(mod) ((mod) ? (mod)->id : -1)
@@ -346,6 +368,18 @@ void rsnd_mod_interrupt(struct rsnd_mod *mod,
 u32 *rsnd_mod_get_status(struct rsnd_dai_stream *io,
 			 struct rsnd_mod *mod,
 			 enum rsnd_mod_type type);
+struct rsnd_mod *rsnd_mod_next(int *iterator,
+			       struct rsnd_dai_stream *io,
+			       enum rsnd_mod_type *array,
+			       int array_size);
+#define for_each_rsnd_mod(iterator, pos, io)				\
+	for (iterator = 0;						\
+	     (pos = rsnd_mod_next(&iterator, io, NULL, 0));)
+#define for_each_rsnd_mod_arrays(iterator, pos, io, array, size)	\
+	for (iterator = 0;						\
+	     (pos = rsnd_mod_next(&iterator, io, array, size));)
+#define for_each_rsnd_mod_array(iterator, pos, io, array)		\
+	for_each_rsnd_mod_arrays(iterator, pos, io, array, ARRAY_SIZE(array))
 
 void rsnd_parse_connect_common(struct rsnd_dai *rdai,
 		struct rsnd_mod* (*mod_get)(struct rsnd_priv *priv, int id),
@@ -365,6 +399,18 @@ int rsnd_runtime_is_ssi_multi(struct rsnd_dai_stream *io);
 int rsnd_runtime_is_ssi_tdm(struct rsnd_dai_stream *io);
 
 /*
+ * DT
+ */
+#define rsnd_parse_of_node(priv, node)					\
+	of_get_child_by_name(rsnd_priv_to_dev(priv)->of_node, node)
+#define RSND_NODE_DAI	"rcar_sound,dai"
+#define RSND_NODE_SSI	"rcar_sound,ssi"
+#define RSND_NODE_SRC	"rcar_sound,src"
+#define RSND_NODE_CTU	"rcar_sound,ctu"
+#define RSND_NODE_MIX	"rcar_sound,mix"
+#define RSND_NODE_DVC	"rcar_sound,dvc"
+
+/*
  *	R-Car sound DAI
  */
 #define RSND_DAI_NAME_SIZE	16
@@ -382,6 +428,7 @@ struct rsnd_dai_stream {
 };
 #define rsnd_io_to_mod(io, i)	((i) < RSND_MOD_MAX ? (io)->mod[(i)] : NULL)
 #define rsnd_io_to_mod_ssi(io)	rsnd_io_to_mod((io), RSND_MOD_SSI)
+#define rsnd_io_to_mod_ssiu(io)	rsnd_io_to_mod((io), RSND_MOD_SSIU)
 #define rsnd_io_to_mod_ssip(io)	rsnd_io_to_mod((io), RSND_MOD_SSIP)
 #define rsnd_io_to_mod_src(io)	rsnd_io_to_mod((io), RSND_MOD_SRC)
 #define rsnd_io_to_mod_ctu(io)	rsnd_io_to_mod((io), RSND_MOD_CTU)
@@ -428,8 +475,7 @@ int rsnd_dai_pointer_offset(struct rsnd_dai_stream *io, int additional);
 int rsnd_dai_connect(struct rsnd_mod *mod,
 		     struct rsnd_dai_stream *io,
 		     enum rsnd_mod_type type);
-#define rsnd_dai_of_node(priv)						\
-	of_get_child_by_name(rsnd_priv_to_dev(priv)->of_node, "rcar_sound,dai")
+#define rsnd_dai_of_node(priv) rsnd_parse_of_node(priv, RSND_NODE_DAI)
 
 /*
  *	R-Car Gen1/Gen2
@@ -453,6 +499,9 @@ int rsnd_adg_set_src_timesel_gen2(struct rsnd_mod *src_mod,
 				  unsigned int out_rate);
 int rsnd_adg_set_cmd_timsel_gen2(struct rsnd_mod *mod,
 				 struct rsnd_dai_stream *io);
+#define rsnd_adg_clk_enable(priv)	rsnd_adg_clk_control(priv, 1)
+#define rsnd_adg_clk_disable(priv)	rsnd_adg_clk_control(priv, 0)
+void rsnd_adg_clk_control(struct rsnd_priv *priv, int enable);
 
 /*
  *	R-Car sound priv
@@ -606,8 +655,7 @@ u32 rsnd_ssi_multi_slaves_runtime(struct rsnd_dai_stream *io);
 	__rsnd_ssi_is_pin_sharing(rsnd_io_to_mod_ssi(io))
 int __rsnd_ssi_is_pin_sharing(struct rsnd_mod *mod);
 
-#define rsnd_ssi_of_node(priv)						\
-	of_get_child_by_name(rsnd_priv_to_dev(priv)->of_node, "rcar_sound,ssi")
+#define rsnd_ssi_of_node(priv) rsnd_parse_of_node(priv, RSND_NODE_SSI)
 void rsnd_parse_connect_ssi(struct rsnd_dai *rdai,
 			    struct device_node *playback,
 			    struct device_node *capture);
@@ -633,8 +681,7 @@ unsigned int rsnd_src_get_rate(struct rsnd_priv *priv,
 			       struct rsnd_dai_stream *io,
 			       int is_in);
 
-#define rsnd_src_of_node(priv)						\
-	of_get_child_by_name(rsnd_priv_to_dev(priv)->of_node, "rcar_sound,src")
+#define rsnd_src_of_node(priv) rsnd_parse_of_node(priv, RSND_NODE_SRC)
 #define rsnd_parse_connect_src(rdai, playback, capture)			\
 	rsnd_parse_connect_common(rdai, rsnd_src_mod_get,		\
 				  rsnd_src_of_node(rsnd_rdai_to_priv(rdai)), \
@@ -647,8 +694,7 @@ int rsnd_ctu_probe(struct rsnd_priv *priv);
 void rsnd_ctu_remove(struct rsnd_priv *priv);
 int rsnd_ctu_converted_channel(struct rsnd_mod *mod);
 struct rsnd_mod *rsnd_ctu_mod_get(struct rsnd_priv *priv, int id);
-#define rsnd_ctu_of_node(priv)						\
-	of_get_child_by_name(rsnd_priv_to_dev(priv)->of_node, "rcar_sound,ctu")
+#define rsnd_ctu_of_node(priv) rsnd_parse_of_node(priv, RSND_NODE_CTU)
 #define rsnd_parse_connect_ctu(rdai, playback, capture)			\
 	rsnd_parse_connect_common(rdai, rsnd_ctu_mod_get,		\
 				  rsnd_ctu_of_node(rsnd_rdai_to_priv(rdai)), \
@@ -660,8 +706,7 @@ struct rsnd_mod *rsnd_ctu_mod_get(struct rsnd_priv *priv, int id);
 int rsnd_mix_probe(struct rsnd_priv *priv);
 void rsnd_mix_remove(struct rsnd_priv *priv);
 struct rsnd_mod *rsnd_mix_mod_get(struct rsnd_priv *priv, int id);
-#define rsnd_mix_of_node(priv)						\
-	of_get_child_by_name(rsnd_priv_to_dev(priv)->of_node, "rcar_sound,mix")
+#define rsnd_mix_of_node(priv) rsnd_parse_of_node(priv, RSND_NODE_MIX)
 #define rsnd_parse_connect_mix(rdai, playback, capture)			\
 	rsnd_parse_connect_common(rdai, rsnd_mix_mod_get,		\
 				  rsnd_mix_of_node(rsnd_rdai_to_priv(rdai)), \
@@ -673,8 +718,7 @@ struct rsnd_mod *rsnd_mix_mod_get(struct rsnd_priv *priv, int id);
 int rsnd_dvc_probe(struct rsnd_priv *priv);
 void rsnd_dvc_remove(struct rsnd_priv *priv);
 struct rsnd_mod *rsnd_dvc_mod_get(struct rsnd_priv *priv, int id);
-#define rsnd_dvc_of_node(priv)						\
-	of_get_child_by_name(rsnd_priv_to_dev(priv)->of_node, "rcar_sound,dvc")
+#define rsnd_dvc_of_node(priv) rsnd_parse_of_node(priv, RSND_NODE_DVC)
 #define rsnd_parse_connect_dvc(rdai, playback, capture)			\
 	rsnd_parse_connect_common(rdai, rsnd_dvc_mod_get,		\
 				  rsnd_dvc_of_node(rsnd_rdai_to_priv(rdai)), \
diff --git a/sound/soc/sh/rcar/src.c b/sound/soc/sh/rcar/src.c
index 969a5169de25..3a8f65bd1bf9 100644
--- a/sound/soc/sh/rcar/src.c
+++ b/sound/soc/sh/rcar/src.c
@@ -189,6 +189,7 @@ static void rsnd_src_set_convert_rate(struct rsnd_dai_stream *io,
 	struct rsnd_priv *priv = rsnd_mod_to_priv(mod);
 	struct device *dev = rsnd_priv_to_dev(priv);
 	struct snd_pcm_runtime *runtime = rsnd_io_to_runtime(io);
+	int use_src = 0;
 	u32 fin, fout;
 	u32 ifscr, fsrate, adinr;
 	u32 cr, route;
@@ -214,6 +215,8 @@ static void rsnd_src_set_convert_rate(struct rsnd_dai_stream *io,
 		return;
 	}
 
+	use_src = (fin != fout) | rsnd_src_sync_is_enabled(mod);
+
 	/*
 	 *	SRC_ADINR
 	 */
@@ -225,7 +228,7 @@ static void rsnd_src_set_convert_rate(struct rsnd_dai_stream *io,
 	 */
 	ifscr = 0;
 	fsrate = 0;
-	if (fin != fout) {
+	if (use_src) {
 		u64 n;
 
 		ifscr = 1;
@@ -239,7 +242,7 @@ static void rsnd_src_set_convert_rate(struct rsnd_dai_stream *io,
 	 */
 	cr	= 0x00011110;
 	route	= 0x0;
-	if (fin != fout) {
+	if (use_src) {
 		route	= 0x1;
 
 		if (rsnd_src_sync_is_enabled(mod)) {
@@ -327,8 +330,8 @@ static void rsnd_src_status_clear(struct rsnd_mod *mod)
 {
 	u32 val = OUF_SRC(rsnd_mod_id(mod));
 
-	rsnd_mod_bset(mod, SCU_SYS_STATUS0, val, val);
-	rsnd_mod_bset(mod, SCU_SYS_STATUS1, val, val);
+	rsnd_mod_write(mod, SCU_SYS_STATUS0, val);
+	rsnd_mod_write(mod, SCU_SYS_STATUS1, val);
 }
 
 static bool rsnd_src_error_occurred(struct rsnd_mod *mod)
@@ -475,7 +478,7 @@ static int rsnd_src_probe_(struct rsnd_mod *mod,
 			return ret;
 	}
 
-	ret = rsnd_dma_attach(io, mod, &src->dma, 0);
+	ret = rsnd_dma_attach(io, mod, &src->dma);
 
 	return ret;
 }
diff --git a/sound/soc/sh/rcar/ssi.c b/sound/soc/sh/rcar/ssi.c
index 6cb6db005fc4..411bda2387ad 100644
--- a/sound/soc/sh/rcar/ssi.c
+++ b/sound/soc/sh/rcar/ssi.c
@@ -417,11 +417,14 @@ static int rsnd_ssi_hw_params(struct rsnd_mod *mod,
 	int chan = params_channels(params);
 
 	/*
-	 * Already working.
-	 * It will happen if SSI has parent/child connection.
+	 * snd_pcm_ops::hw_params will be called *before*
+	 * snd_soc_dai_ops::trigger. Thus, ssi->usrcnt is 0
+	 * in 1st call.
 	 */
-	if (ssi->usrcnt > 1) {
+	if (ssi->usrcnt) {
 		/*
+		 * Already working.
+		 * It will happen if SSI has parent/child connection.
 		 * it is error if child <-> parent SSI uses
 		 * different channels.
 		 */
@@ -644,10 +647,14 @@ static int rsnd_ssi_common_probe(struct rsnd_mod *mod,
 	if (ret < 0)
 		return ret;
 
-	ret = devm_request_irq(dev, ssi->irq,
-			       rsnd_ssi_interrupt,
-			       IRQF_SHARED,
-			       dev_name(dev), mod);
+	/*
+	 * SSI might be called again as PIO fallback
+	 * It is easy to manual handling for IRQ request/free
+	 */
+	ret = request_irq(ssi->irq,
+			  rsnd_ssi_interrupt,
+			  IRQF_SHARED,
+			  dev_name(dev), mod);
 
 	return ret;
 }
@@ -669,7 +676,6 @@ static int rsnd_ssi_dma_probe(struct rsnd_mod *mod,
 			      struct rsnd_priv *priv)
 {
 	struct rsnd_ssi *ssi = rsnd_mod_to_ssi(mod);
-	int dma_id = 0; /* not needed */
 	int ret;
 
 	/*
@@ -684,7 +690,7 @@ static int rsnd_ssi_dma_probe(struct rsnd_mod *mod,
 		return ret;
 
 	/* SSI probe might be called many times in MUX multi path */
-	ret = rsnd_dma_attach(io, mod, &ssi->dma, dma_id);
+	ret = rsnd_dma_attach(io, mod, &ssi->dma);
 
 	return ret;
 }
@@ -694,11 +700,9 @@ static int rsnd_ssi_dma_remove(struct rsnd_mod *mod,
 			       struct rsnd_priv *priv)
 {
 	struct rsnd_ssi *ssi = rsnd_mod_to_ssi(mod);
-	struct device *dev = rsnd_priv_to_dev(priv);
-	int irq = ssi->irq;
 
 	/* PIO will request IRQ again */
-	devm_free_irq(dev, irq, mod);
+	free_irq(ssi->irq, mod);
 
 	return 0;
 }
diff --git a/sound/soc/sh/rcar/ssiu.c b/sound/soc/sh/rcar/ssiu.c
index 6f9b388ec5a8..4e817c8a18c0 100644
--- a/sound/soc/sh/rcar/ssiu.c
+++ b/sound/soc/sh/rcar/ssiu.c
@@ -33,6 +33,26 @@ static int rsnd_ssiu_init(struct rsnd_mod *mod,
 	u32 mask1, val1;
 	u32 mask2, val2;
 
+	/* clear status */
+	switch (id) {
+	case 0:
+	case 1:
+	case 2:
+	case 3:
+	case 4:
+		rsnd_mod_write(mod, SSI_SYS_STATUS0, 0xf << (id * 4));
+		rsnd_mod_write(mod, SSI_SYS_STATUS2, 0xf << (id * 4));
+		rsnd_mod_write(mod, SSI_SYS_STATUS4, 0xf << (id * 4));
+		rsnd_mod_write(mod, SSI_SYS_STATUS6, 0xf << (id * 4));
+		break;
+	case 9:
+		rsnd_mod_write(mod, SSI_SYS_STATUS1, 0xf << 4);
+		rsnd_mod_write(mod, SSI_SYS_STATUS3, 0xf << 4);
+		rsnd_mod_write(mod, SSI_SYS_STATUS5, 0xf << 4);
+		rsnd_mod_write(mod, SSI_SYS_STATUS7, 0xf << 4);
+		break;
+	}
+
 	/*
 	 * SSI_MODE0
 	 */
diff --git a/sound/soc/soc-compress.c b/sound/soc/soc-compress.c
index bf7b52fce597..bfd71b873ca2 100644
--- a/sound/soc/soc-compress.c
+++ b/sound/soc/soc-compress.c
@@ -30,16 +30,26 @@ static int soc_compr_open(struct snd_compr_stream *cstream)
 {
 	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
 	struct snd_soc_platform *platform = rtd->platform;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int ret = 0;
 
 	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
 
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->startup) {
+		ret = cpu_dai->driver->cops->startup(cstream, cpu_dai);
+		if (ret < 0) {
+			dev_err(cpu_dai->dev, "Compress ASoC: can't open interface %s: %d\n",
+				cpu_dai->name, ret);
+			goto out;
+		}
+	}
+
 	if (platform->driver->compr_ops && platform->driver->compr_ops->open) {
 		ret = platform->driver->compr_ops->open(cstream);
 		if (ret < 0) {
 			pr_err("compress asoc: can't open platform %s\n",
 				platform->component.name);
-			goto out;
+			goto plat_err;
 		}
 	}
 
@@ -60,6 +70,9 @@ static int soc_compr_open(struct snd_compr_stream *cstream)
 machine_err:
 	if (platform->driver->compr_ops && platform->driver->compr_ops->free)
 		platform->driver->compr_ops->free(cstream);
+plat_err:
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->shutdown)
+		cpu_dai->driver->cops->shutdown(cstream, cpu_dai);
 out:
 	mutex_unlock(&rtd->pcm_mutex);
 	return ret;
@@ -70,6 +83,7 @@ static int soc_compr_open_fe(struct snd_compr_stream *cstream)
 	struct snd_soc_pcm_runtime *fe = cstream->private_data;
 	struct snd_pcm_substream *fe_substream = fe->pcm->streams[0].substream;
 	struct snd_soc_platform *platform = fe->platform;
+	struct snd_soc_dai *cpu_dai = fe->cpu_dai;
 	struct snd_soc_dpcm *dpcm;
 	struct snd_soc_dapm_widget_list *list;
 	int stream;
@@ -82,12 +96,22 @@ static int soc_compr_open_fe(struct snd_compr_stream *cstream)
 
 	mutex_lock_nested(&fe->card->mutex, SND_SOC_CARD_CLASS_RUNTIME);
 
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->startup) {
+		ret = cpu_dai->driver->cops->startup(cstream, cpu_dai);
+		if (ret < 0) {
+			dev_err(cpu_dai->dev, "Compress ASoC: can't open interface %s: %d\n",
+				cpu_dai->name, ret);
+			goto out;
+		}
+	}
+
+
 	if (platform->driver->compr_ops && platform->driver->compr_ops->open) {
 		ret = platform->driver->compr_ops->open(cstream);
 		if (ret < 0) {
 			pr_err("compress asoc: can't open platform %s\n",
 				platform->component.name);
-			goto out;
+			goto plat_err;
 		}
 	}
 
@@ -144,6 +168,9 @@ fe_err:
 machine_err:
 	if (platform->driver->compr_ops && platform->driver->compr_ops->free)
 		platform->driver->compr_ops->free(cstream);
+plat_err:
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->shutdown)
+		cpu_dai->driver->cops->shutdown(cstream, cpu_dai);
 out:
 	fe->dpcm[stream].runtime_update = SND_SOC_DPCM_UPDATE_NO;
 	mutex_unlock(&fe->card->mutex);
@@ -210,6 +237,9 @@ static int soc_compr_free(struct snd_compr_stream *cstream)
 	if (platform->driver->compr_ops && platform->driver->compr_ops->free)
 		platform->driver->compr_ops->free(cstream);
 
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->shutdown)
+		cpu_dai->driver->cops->shutdown(cstream, cpu_dai);
+
 	if (cstream->direction == SND_COMPRESS_PLAYBACK) {
 		if (snd_soc_runtime_ignore_pmdown_time(rtd)) {
 			snd_soc_dapm_stream_event(rtd,
@@ -236,6 +266,7 @@ static int soc_compr_free_fe(struct snd_compr_stream *cstream)
 {
 	struct snd_soc_pcm_runtime *fe = cstream->private_data;
 	struct snd_soc_platform *platform = fe->platform;
+	struct snd_soc_dai *cpu_dai = fe->cpu_dai;
 	struct snd_soc_dpcm *dpcm;
 	int stream, ret;
 
@@ -275,6 +306,9 @@ static int soc_compr_free_fe(struct snd_compr_stream *cstream)
 	if (platform->driver->compr_ops && platform->driver->compr_ops->free)
 		platform->driver->compr_ops->free(cstream);
 
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->shutdown)
+		cpu_dai->driver->cops->shutdown(cstream, cpu_dai);
+
 	mutex_unlock(&fe->card->mutex);
 	return 0;
 }
@@ -285,6 +319,7 @@ static int soc_compr_trigger(struct snd_compr_stream *cstream, int cmd)
 	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
 	struct snd_soc_platform *platform = rtd->platform;
 	struct snd_soc_dai *codec_dai = rtd->codec_dai;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int ret = 0;
 
 	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
@@ -295,6 +330,10 @@ static int soc_compr_trigger(struct snd_compr_stream *cstream, int cmd)
 			goto out;
 	}
 
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->trigger)
+		cpu_dai->driver->cops->trigger(cstream, cmd, cpu_dai);
+
+
 	switch (cmd) {
 	case SNDRV_PCM_TRIGGER_START:
 		snd_soc_dai_digital_mute(codec_dai, 0, cstream->direction);
@@ -313,6 +352,7 @@ static int soc_compr_trigger_fe(struct snd_compr_stream *cstream, int cmd)
 {
 	struct snd_soc_pcm_runtime *fe = cstream->private_data;
 	struct snd_soc_platform *platform = fe->platform;
+	struct snd_soc_dai *cpu_dai = fe->cpu_dai;
 	int ret = 0, stream;
 
 	if (cmd == SND_COMPR_TRIGGER_PARTIAL_DRAIN ||
@@ -332,6 +372,12 @@ static int soc_compr_trigger_fe(struct snd_compr_stream *cstream, int cmd)
 
 	mutex_lock_nested(&fe->card->mutex, SND_SOC_CARD_CLASS_RUNTIME);
 
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->trigger) {
+		ret = cpu_dai->driver->cops->trigger(cstream, cmd, cpu_dai);
+		if (ret < 0)
+			goto out;
+	}
+
 	if (platform->driver->compr_ops && platform->driver->compr_ops->trigger) {
 		ret = platform->driver->compr_ops->trigger(cstream, cmd);
 		if (ret < 0)
@@ -368,6 +414,7 @@ static int soc_compr_set_params(struct snd_compr_stream *cstream,
 {
 	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
 	struct snd_soc_platform *platform = rtd->platform;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int ret = 0;
 
 	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
@@ -378,6 +425,12 @@ static int soc_compr_set_params(struct snd_compr_stream *cstream,
 	 * expectation is that platform and machine will configure everything
 	 * for this compress path, like configuring pcm port for codec
 	 */
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->set_params) {
+		ret = cpu_dai->driver->cops->set_params(cstream, params, cpu_dai);
+		if (ret < 0)
+			goto err;
+	}
+
 	if (platform->driver->compr_ops && platform->driver->compr_ops->set_params) {
 		ret = platform->driver->compr_ops->set_params(cstream, params);
 		if (ret < 0)
@@ -416,6 +469,7 @@ static int soc_compr_set_params_fe(struct snd_compr_stream *cstream,
 	struct snd_soc_pcm_runtime *fe = cstream->private_data;
 	struct snd_pcm_substream *fe_substream = fe->pcm->streams[0].substream;
 	struct snd_soc_platform *platform = fe->platform;
+	struct snd_soc_dai *cpu_dai = fe->cpu_dai;
 	int ret = 0, stream;
 
 	if (cstream->direction == SND_COMPRESS_PLAYBACK)
@@ -425,6 +479,12 @@ static int soc_compr_set_params_fe(struct snd_compr_stream *cstream,
 
 	mutex_lock_nested(&fe->card->mutex, SND_SOC_CARD_CLASS_RUNTIME);
 
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->set_params) {
+		ret = cpu_dai->driver->cops->set_params(cstream, params, cpu_dai);
+		if (ret < 0)
+			goto out;
+	}
+
 	if (platform->driver->compr_ops && platform->driver->compr_ops->set_params) {
 		ret = platform->driver->compr_ops->set_params(cstream, params);
 		if (ret < 0)
@@ -469,13 +529,21 @@ static int soc_compr_get_params(struct snd_compr_stream *cstream,
 {
 	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
 	struct snd_soc_platform *platform = rtd->platform;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int ret = 0;
 
 	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
 
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->get_params) {
+		ret = cpu_dai->driver->cops->get_params(cstream, params, cpu_dai);
+		if (ret < 0)
+			goto err;
+	}
+
 	if (platform->driver->compr_ops && platform->driver->compr_ops->get_params)
 		ret = platform->driver->compr_ops->get_params(cstream, params);
 
+err:
 	mutex_unlock(&rtd->pcm_mutex);
 	return ret;
 }
@@ -516,13 +584,21 @@ static int soc_compr_ack(struct snd_compr_stream *cstream, size_t bytes)
 {
 	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
 	struct snd_soc_platform *platform = rtd->platform;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int ret = 0;
 
 	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
 
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->ack) {
+		ret = cpu_dai->driver->cops->ack(cstream, bytes, cpu_dai);
+		if (ret < 0)
+			goto err;
+	}
+
 	if (platform->driver->compr_ops && platform->driver->compr_ops->ack)
 		ret = platform->driver->compr_ops->ack(cstream, bytes);
 
+err:
 	mutex_unlock(&rtd->pcm_mutex);
 	return ret;
 }
@@ -533,9 +609,13 @@ static int soc_compr_pointer(struct snd_compr_stream *cstream,
 	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
 	struct snd_soc_platform *platform = rtd->platform;
 	int ret = 0;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 
 	mutex_lock_nested(&rtd->pcm_mutex, rtd->pcm_subclass);
 
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->pointer)
+		cpu_dai->driver->cops->pointer(cstream, tstamp, cpu_dai);
+
 	if (platform->driver->compr_ops && platform->driver->compr_ops->pointer)
 		ret = platform->driver->compr_ops->pointer(cstream, tstamp);
 
@@ -564,8 +644,15 @@ static int soc_compr_set_metadata(struct snd_compr_stream *cstream,
 {
 	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
 	struct snd_soc_platform *platform = rtd->platform;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int ret = 0;
 
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->set_metadata) {
+		ret = cpu_dai->driver->cops->set_metadata(cstream, metadata, cpu_dai);
+		if (ret < 0)
+			return ret;
+	}
+
 	if (platform->driver->compr_ops && platform->driver->compr_ops->set_metadata)
 		ret = platform->driver->compr_ops->set_metadata(cstream, metadata);
 
@@ -577,8 +664,15 @@ static int soc_compr_get_metadata(struct snd_compr_stream *cstream,
 {
 	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
 	struct snd_soc_platform *platform = rtd->platform;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
 	int ret = 0;
 
+	if (cpu_dai->driver->cops && cpu_dai->driver->cops->get_metadata) {
+		ret = cpu_dai->driver->cops->get_metadata(cstream, metadata, cpu_dai);
+		if (ret < 0)
+			return ret;
+	}
+
 	if (platform->driver->compr_ops && platform->driver->compr_ops->get_metadata)
 		ret = platform->driver->compr_ops->get_metadata(cstream, metadata);
 
diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c
index c0bbcd903261..f1901bb1466e 100644
--- a/sound/soc/soc-core.c
+++ b/sound/soc/soc-core.c
@@ -626,7 +626,7 @@ static void codec2codec_close_delayed_work(struct work_struct *work)
 int snd_soc_suspend(struct device *dev)
 {
 	struct snd_soc_card *card = dev_get_drvdata(dev);
-	struct snd_soc_codec *codec;
+	struct snd_soc_component *component;
 	struct snd_soc_pcm_runtime *rtd;
 	int i;
 
@@ -702,39 +702,39 @@ int snd_soc_suspend(struct device *dev)
 	dapm_mark_endpoints_dirty(card);
 	snd_soc_dapm_sync(&card->dapm);
 
-	/* suspend all CODECs */
-	list_for_each_entry(codec, &card->codec_dev_list, card_list) {
-		struct snd_soc_dapm_context *dapm = snd_soc_codec_get_dapm(codec);
+	/* suspend all COMPONENTs */
+	list_for_each_entry(component, &card->component_dev_list, card_list) {
+		struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component);
 
-		/* If there are paths active then the CODEC will be held with
+		/* If there are paths active then the COMPONENT will be held with
 		 * bias _ON and should not be suspended. */
-		if (!codec->suspended) {
+		if (!component->suspended) {
 			switch (snd_soc_dapm_get_bias_level(dapm)) {
 			case SND_SOC_BIAS_STANDBY:
 				/*
-				 * If the CODEC is capable of idle
+				 * If the COMPONENT is capable of idle
 				 * bias off then being in STANDBY
 				 * means it's doing something,
 				 * otherwise fall through.
 				 */
 				if (dapm->idle_bias_off) {
-					dev_dbg(codec->dev,
+					dev_dbg(component->dev,
 						"ASoC: idle_bias_off CODEC on over suspend\n");
 					break;
 				}
 
 			case SND_SOC_BIAS_OFF:
-				if (codec->driver->suspend)
-					codec->driver->suspend(codec);
-				codec->suspended = 1;
-				if (codec->component.regmap)
-					regcache_mark_dirty(codec->component.regmap);
+				if (component->suspend)
+					component->suspend(component);
+				component->suspended = 1;
+				if (component->regmap)
+					regcache_mark_dirty(component->regmap);
 				/* deactivate pins to sleep state */
-				pinctrl_pm_select_sleep_state(codec->dev);
+				pinctrl_pm_select_sleep_state(component->dev);
 				break;
 			default:
-				dev_dbg(codec->dev,
-					"ASoC: CODEC is on over suspend\n");
+				dev_dbg(component->dev,
+					"ASoC: COMPONENT is on over suspend\n");
 				break;
 			}
 		}
@@ -768,7 +768,7 @@ static void soc_resume_deferred(struct work_struct *work)
 	struct snd_soc_card *card =
 			container_of(work, struct snd_soc_card, deferred_resume_work);
 	struct snd_soc_pcm_runtime *rtd;
-	struct snd_soc_codec *codec;
+	struct snd_soc_component *component;
 	int i;
 
 	/* our power state is still SNDRV_CTL_POWER_D3hot from suspend time,
@@ -794,11 +794,11 @@ static void soc_resume_deferred(struct work_struct *work)
 			cpu_dai->driver->resume(cpu_dai);
 	}
 
-	list_for_each_entry(codec, &card->codec_dev_list, card_list) {
-		if (codec->suspended) {
-			if (codec->driver->resume)
-				codec->driver->resume(codec);
-			codec->suspended = 0;
+	list_for_each_entry(component, &card->component_dev_list, card_list) {
+		if (component->suspended) {
+			if (component->resume)
+				component->resume(component);
+			component->suspended = 0;
 		}
 	}
 
@@ -972,6 +972,48 @@ struct snd_soc_dai *snd_soc_find_dai(
 }
 EXPORT_SYMBOL_GPL(snd_soc_find_dai);
 
+
+/**
+ * snd_soc_find_dai_link - Find a DAI link
+ *
+ * @card: soc card
+ * @id: DAI link ID to match
+ * @name: DAI link name to match, optional
+ * @stream name: DAI link stream name to match, optional
+ *
+ * This function will search all existing DAI links of the soc card to
+ * find the link of the same ID. Since DAI links may not have their
+ * unique ID, so name and stream name should also match if being
+ * specified.
+ *
+ * Return: pointer of DAI link, or NULL if not found.
+ */
+struct snd_soc_dai_link *snd_soc_find_dai_link(struct snd_soc_card *card,
+					       int id, const char *name,
+					       const char *stream_name)
+{
+	struct snd_soc_dai_link *link, *_link;
+
+	lockdep_assert_held(&client_mutex);
+
+	list_for_each_entry_safe(link, _link, &card->dai_link_list, list) {
+		if (link->id != id)
+			continue;
+
+		if (name && (!link->name || strcmp(name, link->name)))
+			continue;
+
+		if (stream_name && (!link->stream_name
+			|| strcmp(stream_name, link->stream_name)))
+			continue;
+
+		return link;
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(snd_soc_find_dai_link);
+
 static bool soc_is_dai_link_bound(struct snd_soc_card *card,
 		struct snd_soc_dai_link *dai_link)
 {
@@ -993,6 +1035,7 @@ static int soc_bind_dai_link(struct snd_soc_card *card,
 	struct snd_soc_dai_link_component cpu_dai_component;
 	struct snd_soc_dai **codec_dais;
 	struct snd_soc_platform *platform;
+	struct device_node *platform_of_node;
 	const char *platform_name;
 	int i;
 
@@ -1042,9 +1085,12 @@ static int soc_bind_dai_link(struct snd_soc_card *card,
 
 	/* find one from the set of registered platforms */
 	list_for_each_entry(platform, &platform_list, list) {
+		platform_of_node = platform->dev->of_node;
+		if (!platform_of_node && platform->dev->parent->of_node)
+			platform_of_node = platform->dev->parent->of_node;
+
 		if (dai_link->platform_of_node) {
-			if (platform->dev->of_node !=
-			    dai_link->platform_of_node)
+			if (platform_of_node != dai_link->platform_of_node)
 				continue;
 		} else {
 			if (strcmp(platform->component.name, platform_name))
@@ -1072,9 +1118,7 @@ static void soc_remove_component(struct snd_soc_component *component)
 	if (!component->card)
 		return;
 
-	/* This is a HACK and will be removed soon */
-	if (component->codec)
-		list_del(&component->codec->card_list);
+	list_del(&component->card_list);
 
 	if (component->remove)
 		component->remove(component);
@@ -1443,10 +1487,7 @@ static int soc_probe_component(struct snd_soc_card *card,
 					component->num_dapm_routes);
 
 	list_add(&dapm->list, &card->dapm_list);
-
-	/* This is a HACK and will be removed soon */
-	if (component->codec)
-		list_add(&component->codec->card_list, &card->codec_dev_list);
+	list_add(&component->card_list, &card->component_dev_list);
 
 	return 0;
 
@@ -1706,7 +1747,8 @@ static int soc_bind_aux_dev(struct snd_soc_card *card, int num)
 	}
 
 	component->init = aux_dev->init;
-	list_add(&component->list_aux, &card->aux_comp_list);
+	component->auxiliary = 1;
+
 	return 0;
 
 err_defer:
@@ -1722,7 +1764,10 @@ static int soc_probe_aux_devices(struct snd_soc_card *card)
 
 	for (order = SND_SOC_COMP_ORDER_FIRST; order <= SND_SOC_COMP_ORDER_LAST;
 		order++) {
-		list_for_each_entry(comp, &card->aux_comp_list, list_aux) {
+		list_for_each_entry(comp, &card->component_dev_list, card_list) {
+			if (!comp->auxiliary)
+				continue;
+
 			if (comp->driver->probe_order == order) {
 				ret = soc_probe_component(card,	comp);
 				if (ret < 0) {
@@ -1746,11 +1791,14 @@ static void soc_remove_aux_devices(struct snd_soc_card *card)
 	for (order = SND_SOC_COMP_ORDER_FIRST; order <= SND_SOC_COMP_ORDER_LAST;
 		order++) {
 		list_for_each_entry_safe(comp, _comp,
-			&card->aux_comp_list, list_aux) {
+			&card->component_dev_list, card_list) {
+
+			if (!comp->auxiliary)
+				continue;
+
 			if (comp->driver->remove_order == order) {
 				soc_remove_component(comp);
-				/* remove it from the card's aux_comp_list */
-				list_del(&comp->list_aux);
+				comp->auxiliary = 0;
 			}
 		}
 	}
@@ -2926,6 +2974,8 @@ static int snd_soc_component_initialize(struct snd_soc_component *component,
 	component->driver = driver;
 	component->probe = component->driver->probe;
 	component->remove = component->driver->remove;
+	component->suspend = component->driver->suspend;
+	component->resume = component->driver->resume;
 
 	dapm = &component->dapm;
 	dapm->dev = dev;
@@ -3275,6 +3325,20 @@ static void snd_soc_codec_drv_remove(struct snd_soc_component *component)
 	codec->driver->remove(codec);
 }
 
+static int snd_soc_codec_drv_suspend(struct snd_soc_component *component)
+{
+	struct snd_soc_codec *codec = snd_soc_component_to_codec(component);
+
+	return codec->driver->suspend(codec);
+}
+
+static int snd_soc_codec_drv_resume(struct snd_soc_component *component)
+{
+	struct snd_soc_codec *codec = snd_soc_component_to_codec(component);
+
+	return codec->driver->resume(codec);
+}
+
 static int snd_soc_codec_drv_write(struct snd_soc_component *component,
 	unsigned int reg, unsigned int val)
 {
@@ -3336,6 +3400,10 @@ int snd_soc_register_codec(struct device *dev,
 		codec->component.probe = snd_soc_codec_drv_probe;
 	if (codec_drv->remove)
 		codec->component.remove = snd_soc_codec_drv_remove;
+	if (codec_drv->suspend)
+		codec->component.suspend = snd_soc_codec_drv_suspend;
+	if (codec_drv->resume)
+		codec->component.resume = snd_soc_codec_drv_resume;
 	if (codec_drv->write)
 		codec->component.write = snd_soc_codec_drv_write;
 	if (codec_drv->read)
@@ -3424,10 +3492,10 @@ found:
 EXPORT_SYMBOL_GPL(snd_soc_unregister_codec);
 
 /* Retrieve a card's name from device tree */
-int snd_soc_of_parse_card_name(struct snd_soc_card *card,
-			       const char *propname)
+int snd_soc_of_parse_card_name_from_node(struct snd_soc_card *card,
+					 struct device_node *np,
+					 const char *propname)
 {
-	struct device_node *np;
 	int ret;
 
 	if (!card->dev) {
@@ -3435,7 +3503,8 @@ int snd_soc_of_parse_card_name(struct snd_soc_card *card,
 		return -EINVAL;
 	}
 
-	np = card->dev->of_node;
+	if (!np)
+		np = card->dev->of_node;
 
 	ret = of_property_read_string_index(np, propname, 0, &card->name);
 	/*
@@ -3452,7 +3521,7 @@ int snd_soc_of_parse_card_name(struct snd_soc_card *card,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(snd_soc_of_parse_card_name);
+EXPORT_SYMBOL_GPL(snd_soc_of_parse_card_name_from_node);
 
 static const struct snd_soc_dapm_widget simple_widgets[] = {
 	SND_SOC_DAPM_MIC("Microphone", NULL),
@@ -3461,14 +3530,17 @@ static const struct snd_soc_dapm_widget simple_widgets[] = {
 	SND_SOC_DAPM_SPK("Speaker", NULL),
 };
 
-int snd_soc_of_parse_audio_simple_widgets(struct snd_soc_card *card,
+int snd_soc_of_parse_audio_simple_widgets_from_node(struct snd_soc_card *card,
+					  struct device_node *np,
 					  const char *propname)
 {
-	struct device_node *np = card->dev->of_node;
 	struct snd_soc_dapm_widget *widgets;
 	const char *template, *wname;
 	int i, j, num_widgets, ret;
 
+	if (!np)
+		np = card->dev->of_node;
+
 	num_widgets = of_property_count_strings(np, propname);
 	if (num_widgets < 0) {
 		dev_err(card->dev,
@@ -3539,7 +3611,7 @@ int snd_soc_of_parse_audio_simple_widgets(struct snd_soc_card *card,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(snd_soc_of_parse_audio_simple_widgets);
+EXPORT_SYMBOL_GPL(snd_soc_of_parse_audio_simple_widgets_from_node);
 
 static int snd_soc_of_get_slot_mask(struct device_node *np,
 				    const char *prop_name,
@@ -3595,15 +3667,18 @@ int snd_soc_of_parse_tdm_slot(struct device_node *np,
 }
 EXPORT_SYMBOL_GPL(snd_soc_of_parse_tdm_slot);
 
-void snd_soc_of_parse_audio_prefix(struct snd_soc_card *card,
+void snd_soc_of_parse_audio_prefix_from_node(struct snd_soc_card *card,
+				   struct device_node *np,
 				   struct snd_soc_codec_conf *codec_conf,
 				   struct device_node *of_node,
 				   const char *propname)
 {
-	struct device_node *np = card->dev->of_node;
 	const char *str;
 	int ret;
 
+	if (!np)
+		np = card->dev->of_node;
+
 	ret = of_property_read_string(np, propname, &str);
 	if (ret < 0) {
 		/* no prefix is not error */
@@ -3613,16 +3688,19 @@ void snd_soc_of_parse_audio_prefix(struct snd_soc_card *card,
 	codec_conf->of_node	= of_node;
 	codec_conf->name_prefix	= str;
 }
-EXPORT_SYMBOL_GPL(snd_soc_of_parse_audio_prefix);
+EXPORT_SYMBOL_GPL(snd_soc_of_parse_audio_prefix_from_node);
 
-int snd_soc_of_parse_audio_routing(struct snd_soc_card *card,
+int snd_soc_of_parse_audio_routing_from_node(struct snd_soc_card *card,
+				   struct device_node *np,
 				   const char *propname)
 {
-	struct device_node *np = card->dev->of_node;
 	int num_routes;
 	struct snd_soc_dapm_route *routes;
 	int i, ret;
 
+	if (!np)
+		np = card->dev->of_node;
+
 	num_routes = of_property_count_strings(np, propname);
 	if (num_routes < 0 || num_routes & 1) {
 		dev_err(card->dev,
@@ -3669,7 +3747,7 @@ int snd_soc_of_parse_audio_routing(struct snd_soc_card *card,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(snd_soc_of_parse_audio_routing);
+EXPORT_SYMBOL_GPL(snd_soc_of_parse_audio_routing_from_node);
 
 unsigned int snd_soc_of_parse_daifmt(struct device_node *np,
 				     const char *prefix,
@@ -3784,7 +3862,7 @@ unsigned int snd_soc_of_parse_daifmt(struct device_node *np,
 }
 EXPORT_SYMBOL_GPL(snd_soc_of_parse_daifmt);
 
-static int snd_soc_get_dai_name(struct of_phandle_args *args,
+int snd_soc_get_dai_name(struct of_phandle_args *args,
 				const char **dai_name)
 {
 	struct snd_soc_component *pos;
@@ -3836,6 +3914,7 @@ static int snd_soc_get_dai_name(struct of_phandle_args *args,
 	mutex_unlock(&client_mutex);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(snd_soc_get_dai_name);
 
 int snd_soc_of_get_dai_name(struct device_node *of_node,
 			    const char **dai_name)
diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c
index 3bbe32ee4630..27dd02e57b31 100644
--- a/sound/soc/soc-dapm.c
+++ b/sound/soc/soc-dapm.c
@@ -330,6 +330,11 @@ static int dapm_kcontrol_data_alloc(struct snd_soc_dapm_widget *widget,
 	case snd_soc_dapm_mixer_named_ctl:
 		mc = (struct soc_mixer_control *)kcontrol->private_value;
 
+		if (mc->autodisable && snd_soc_volsw_is_stereo(mc))
+			dev_warn(widget->dapm->dev,
+				 "ASoC: Unsupported stereo autodisable control '%s'\n",
+				 ctrl_name);
+
 		if (mc->autodisable) {
 			struct snd_soc_dapm_widget template;
 
@@ -723,7 +728,8 @@ static int dapm_connect_mux(struct snd_soc_dapm_context *dapm,
 }
 
 /* set up initial codec paths */
-static void dapm_set_mixer_path_status(struct snd_soc_dapm_path *p, int i)
+static void dapm_set_mixer_path_status(struct snd_soc_dapm_path *p, int i,
+				       int nth_path)
 {
 	struct soc_mixer_control *mc = (struct soc_mixer_control *)
 		p->sink->kcontrol_news[i].private_value;
@@ -736,7 +742,25 @@ static void dapm_set_mixer_path_status(struct snd_soc_dapm_path *p, int i)
 
 	if (reg != SND_SOC_NOPM) {
 		soc_dapm_read(p->sink->dapm, reg, &val);
-		val = (val >> shift) & mask;
+		/*
+		 * The nth_path argument allows this function to know
+		 * which path of a kcontrol it is setting the initial
+		 * status for. Ideally this would support any number
+		 * of paths and channels. But since kcontrols only come
+		 * in mono and stereo variants, we are limited to 2
+		 * channels.
+		 *
+		 * The following code assumes for stereo controls the
+		 * first path is the left channel, and all remaining
+		 * paths are the right channel.
+		 */
+		if (snd_soc_volsw_is_stereo(mc) && nth_path > 0) {
+			if (reg != mc->rreg)
+				soc_dapm_read(p->sink->dapm, mc->rreg, &val);
+			val = (val >> mc->rshift) & mask;
+		} else {
+			val = (val >> shift) & mask;
+		}
 		if (invert)
 			val = max - val;
 		p->connect = !!val;
@@ -749,13 +773,13 @@ static void dapm_set_mixer_path_status(struct snd_soc_dapm_path *p, int i)
 static int dapm_connect_mixer(struct snd_soc_dapm_context *dapm,
 	struct snd_soc_dapm_path *path, const char *control_name)
 {
-	int i;
+	int i, nth_path = 0;
 
 	/* search for mixer kcontrol */
 	for (i = 0; i < path->sink->num_kcontrols; i++) {
 		if (!strcmp(control_name, path->sink->kcontrol_news[i].name)) {
 			path->name = path->sink->kcontrol_news[i].name;
-			dapm_set_mixer_path_status(path, i);
+			dapm_set_mixer_path_status(path, i, nth_path++);
 			return 0;
 		}
 	}
@@ -1626,6 +1650,15 @@ static void dapm_widget_update(struct snd_soc_card *card)
 		dev_err(w->dapm->dev, "ASoC: %s DAPM update failed: %d\n",
 			w->name, ret);
 
+	if (update->has_second_set) {
+		ret = soc_dapm_update_bits(w->dapm, update->reg2,
+					   update->mask2, update->val2);
+		if (ret < 0)
+			dev_err(w->dapm->dev,
+				"ASoC: %s DAPM update failed: %d\n",
+				w->name, ret);
+	}
+
 	for (wi = 0; wi < wlist->num_widgets; wi++) {
 		w = wlist->widgets[wi];
 
@@ -2177,7 +2210,8 @@ EXPORT_SYMBOL_GPL(snd_soc_dapm_mux_update_power);
 
 /* test and update the power status of a mixer or switch widget */
 static int soc_dapm_mixer_update_power(struct snd_soc_card *card,
-				   struct snd_kcontrol *kcontrol, int connect)
+				       struct snd_kcontrol *kcontrol,
+				       int connect, int rconnect)
 {
 	struct snd_soc_dapm_path *path;
 	int found = 0;
@@ -2186,8 +2220,33 @@ static int soc_dapm_mixer_update_power(struct snd_soc_card *card,
 
 	/* find dapm widget path assoc with kcontrol */
 	dapm_kcontrol_for_each_path(path, kcontrol) {
+		/*
+		 * Ideally this function should support any number of
+		 * paths and channels. But since kcontrols only come
+		 * in mono and stereo variants, we are limited to 2
+		 * channels.
+		 *
+		 * The following code assumes for stereo controls the
+		 * first path (when 'found == 0') is the left channel,
+		 * and all remaining paths (when 'found == 1') are the
+		 * right channel.
+		 *
+		 * A stereo control is signified by a valid 'rconnect'
+		 * value, either 0 for unconnected, or >= 0 for connected.
+		 * This is chosen instead of using snd_soc_volsw_is_stereo,
+		 * so that the behavior of snd_soc_dapm_mixer_update_power
+		 * doesn't change even when the kcontrol passed in is
+		 * stereo.
+		 *
+		 * It passes 'connect' as the path connect status for
+		 * the left channel, and 'rconnect' for the right
+		 * channel.
+		 */
+		if (found && rconnect >= 0)
+			soc_dapm_connect_path(path, rconnect, "mixer update");
+		else
+			soc_dapm_connect_path(path, connect, "mixer update");
 		found = 1;
-		soc_dapm_connect_path(path, connect, "mixer update");
 	}
 
 	if (found)
@@ -2205,7 +2264,7 @@ int snd_soc_dapm_mixer_update_power(struct snd_soc_dapm_context *dapm,
 
 	mutex_lock_nested(&card->dapm_mutex, SND_SOC_DAPM_CLASS_RUNTIME);
 	card->update = update;
-	ret = soc_dapm_mixer_update_power(card, kcontrol, connect);
+	ret = soc_dapm_mixer_update_power(card, kcontrol, connect, -1);
 	card->update = NULL;
 	mutex_unlock(&card->dapm_mutex);
 	if (ret > 0)
@@ -3030,22 +3089,28 @@ int snd_soc_dapm_get_volsw(struct snd_kcontrol *kcontrol,
 	int reg = mc->reg;
 	unsigned int shift = mc->shift;
 	int max = mc->max;
+	unsigned int width = fls(max);
 	unsigned int mask = (1 << fls(max)) - 1;
 	unsigned int invert = mc->invert;
-	unsigned int val;
+	unsigned int reg_val, val, rval = 0;
 	int ret = 0;
 
-	if (snd_soc_volsw_is_stereo(mc))
-		dev_warn(dapm->dev,
-			 "ASoC: Control '%s' is stereo, which is not supported\n",
-			 kcontrol->id.name);
-
 	mutex_lock_nested(&card->dapm_mutex, SND_SOC_DAPM_CLASS_RUNTIME);
 	if (dapm_kcontrol_is_powered(kcontrol) && reg != SND_SOC_NOPM) {
-		ret = soc_dapm_read(dapm, reg, &val);
-		val = (val >> shift) & mask;
+		ret = soc_dapm_read(dapm, reg, &reg_val);
+		val = (reg_val >> shift) & mask;
+
+		if (ret == 0 && reg != mc->rreg)
+			ret = soc_dapm_read(dapm, mc->rreg, &reg_val);
+
+		if (snd_soc_volsw_is_stereo(mc))
+			rval = (reg_val >> mc->rshift) & mask;
 	} else {
-		val = dapm_kcontrol_get_value(kcontrol);
+		reg_val = dapm_kcontrol_get_value(kcontrol);
+		val = reg_val & mask;
+
+		if (snd_soc_volsw_is_stereo(mc))
+			rval = (reg_val >> width) & mask;
 	}
 	mutex_unlock(&card->dapm_mutex);
 
@@ -3057,6 +3122,13 @@ int snd_soc_dapm_get_volsw(struct snd_kcontrol *kcontrol,
 	else
 		ucontrol->value.integer.value[0] = val;
 
+	if (snd_soc_volsw_is_stereo(mc)) {
+		if (invert)
+			ucontrol->value.integer.value[1] = max - rval;
+		else
+			ucontrol->value.integer.value[1] = rval;
+	}
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(snd_soc_dapm_get_volsw);
@@ -3080,46 +3152,66 @@ int snd_soc_dapm_put_volsw(struct snd_kcontrol *kcontrol,
 	int reg = mc->reg;
 	unsigned int shift = mc->shift;
 	int max = mc->max;
-	unsigned int mask = (1 << fls(max)) - 1;
+	unsigned int width = fls(max);
+	unsigned int mask = (1 << width) - 1;
 	unsigned int invert = mc->invert;
-	unsigned int val;
-	int connect, change, reg_change = 0;
-	struct snd_soc_dapm_update update;
+	unsigned int val, rval = 0;
+	int connect, rconnect = -1, change, reg_change = 0;
+	struct snd_soc_dapm_update update = { NULL };
 	int ret = 0;
 
-	if (snd_soc_volsw_is_stereo(mc))
-		dev_warn(dapm->dev,
-			 "ASoC: Control '%s' is stereo, which is not supported\n",
-			 kcontrol->id.name);
-
 	val = (ucontrol->value.integer.value[0] & mask);
 	connect = !!val;
 
 	if (invert)
 		val = max - val;
 
+	if (snd_soc_volsw_is_stereo(mc)) {
+		rval = (ucontrol->value.integer.value[1] & mask);
+		rconnect = !!rval;
+		if (invert)
+			rval = max - rval;
+	}
+
 	mutex_lock_nested(&card->dapm_mutex, SND_SOC_DAPM_CLASS_RUNTIME);
 
-	change = dapm_kcontrol_set_value(kcontrol, val);
+	/* This assumes field width < (bits in unsigned int / 2) */
+	if (width > sizeof(unsigned int) * 8 / 2)
+		dev_warn(dapm->dev,
+			 "ASoC: control %s field width limit exceeded\n",
+			 kcontrol->id.name);
+	change = dapm_kcontrol_set_value(kcontrol, val | (rval << width));
 
 	if (reg != SND_SOC_NOPM) {
-		mask = mask << shift;
 		val = val << shift;
+		rval = rval << mc->rshift;
+
+		reg_change = soc_dapm_test_bits(dapm, reg, mask << shift, val);
 
-		reg_change = soc_dapm_test_bits(dapm, reg, mask, val);
+		if (snd_soc_volsw_is_stereo(mc))
+			reg_change |= soc_dapm_test_bits(dapm, mc->rreg,
+							 mask << mc->rshift,
+							 rval);
 	}
 
 	if (change || reg_change) {
 		if (reg_change) {
+			if (snd_soc_volsw_is_stereo(mc)) {
+				update.has_second_set = true;
+				update.reg2 = mc->rreg;
+				update.mask2 = mask << mc->rshift;
+				update.val2 = rval;
+			}
 			update.kcontrol = kcontrol;
 			update.reg = reg;
-			update.mask = mask;
+			update.mask = mask << shift;
 			update.val = val;
 			card->update = &update;
 		}
 		change |= reg_change;
 
-		ret = soc_dapm_mixer_update_power(card, kcontrol, connect);
+		ret = soc_dapm_mixer_update_power(card, kcontrol, connect,
+						  rconnect);
 
 		card->update = NULL;
 	}
@@ -3192,7 +3284,7 @@ int snd_soc_dapm_put_enum_double(struct snd_kcontrol *kcontrol,
 	unsigned int *item = ucontrol->value.enumerated.item;
 	unsigned int val, change, reg_change = 0;
 	unsigned int mask;
-	struct snd_soc_dapm_update update;
+	struct snd_soc_dapm_update update = { NULL };
 	int ret = 0;
 
 	if (item[0] >= e->items)
diff --git a/sound/soc/soc-generic-dmaengine-pcm.c b/sound/soc/soc-generic-dmaengine-pcm.c
index 6cef3977507a..17eb14935577 100644
--- a/sound/soc/soc-generic-dmaengine-pcm.c
+++ b/sound/soc/soc-generic-dmaengine-pcm.c
@@ -263,7 +263,6 @@ static int dmaengine_pcm_new(struct snd_soc_pcm_runtime *rtd)
 	struct dmaengine_pcm *pcm = soc_platform_to_pcm(rtd->platform);
 	const struct snd_dmaengine_pcm_config *config = pcm->config;
 	struct device *dev = rtd->platform->dev;
-	struct snd_dmaengine_dai_dma_data *dma_data;
 	struct snd_pcm_substream *substream;
 	size_t prealloc_buffer_size;
 	size_t max_buffer_size;
@@ -278,19 +277,11 @@ static int dmaengine_pcm_new(struct snd_soc_pcm_runtime *rtd)
 		max_buffer_size = SIZE_MAX;
 	}
 
-
 	for (i = SNDRV_PCM_STREAM_PLAYBACK; i <= SNDRV_PCM_STREAM_CAPTURE; i++) {
 		substream = rtd->pcm->streams[i].substream;
 		if (!substream)
 			continue;
 
-		dma_data = snd_soc_dai_get_dma_data(rtd->cpu_dai, substream);
-
-		if (!pcm->chan[i] &&
-		    (pcm->flags & SND_DMAENGINE_PCM_FLAG_CUSTOM_CHANNEL_NAME))
-			pcm->chan[i] = dma_request_slave_channel(dev,
-				dma_data->chan_name);
-
 		if (!pcm->chan[i] && (pcm->flags & SND_DMAENGINE_PCM_FLAG_COMPAT)) {
 			pcm->chan[i] = dmaengine_pcm_compat_request_channel(rtd,
 				substream);
@@ -359,9 +350,7 @@ static int dmaengine_pcm_request_chan_of(struct dmaengine_pcm *pcm,
 	const char *name;
 	struct dma_chan *chan;
 
-	if ((pcm->flags & (SND_DMAENGINE_PCM_FLAG_NO_DT |
-			   SND_DMAENGINE_PCM_FLAG_CUSTOM_CHANNEL_NAME)) ||
-	    !dev->of_node)
+	if ((pcm->flags & SND_DMAENGINE_PCM_FLAG_NO_DT) || !dev->of_node)
 		return 0;
 
 	if (config && config->dma_dev) {
diff --git a/sound/soc/soc-pcm.c b/sound/soc/soc-pcm.c
index d56a16a0f6fa..e7a1eaa2772f 100644
--- a/sound/soc/soc-pcm.c
+++ b/sound/soc/soc-pcm.c
@@ -2882,7 +2882,7 @@ int snd_soc_platform_trigger(struct snd_pcm_substream *substream,
 EXPORT_SYMBOL_GPL(snd_soc_platform_trigger);
 
 #ifdef CONFIG_DEBUG_FS
-static char *dpcm_state_string(enum snd_soc_dpcm_state state)
+static const char *dpcm_state_string(enum snd_soc_dpcm_state state)
 {
 	switch (state) {
 	case SND_SOC_DPCM_STATE_NEW:
diff --git a/sound/soc/soc-topology.c b/sound/soc/soc-topology.c
index 6b05047a4134..65670b2b408c 100644
--- a/sound/soc/soc-topology.c
+++ b/sound/soc/soc-topology.c
@@ -49,10 +49,68 @@
 #define SOC_TPLG_PASS_GRAPH		5
 #define SOC_TPLG_PASS_PINS		6
 #define SOC_TPLG_PASS_BE_DAI		7
+#define SOC_TPLG_PASS_LINK		8
 
 #define SOC_TPLG_PASS_START	SOC_TPLG_PASS_MANIFEST
-#define SOC_TPLG_PASS_END	SOC_TPLG_PASS_BE_DAI
+#define SOC_TPLG_PASS_END	SOC_TPLG_PASS_LINK
 
+/*
+ * Old version of ABI structs, supported for backward compatibility.
+ */
+
+/* Manifest v4 */
+struct snd_soc_tplg_manifest_v4 {
+	__le32 size;		/* in bytes of this structure */
+	__le32 control_elems;	/* number of control elements */
+	__le32 widget_elems;	/* number of widget elements */
+	__le32 graph_elems;	/* number of graph elements */
+	__le32 pcm_elems;	/* number of PCM elements */
+	__le32 dai_link_elems;	/* number of DAI link elements */
+	struct snd_soc_tplg_private priv;
+} __packed;
+
+/* Stream Capabilities v4 */
+struct snd_soc_tplg_stream_caps_v4 {
+	__le32 size;		/* in bytes of this structure */
+	char name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN];
+	__le64 formats;	/* supported formats SNDRV_PCM_FMTBIT_* */
+	__le32 rates;		/* supported rates SNDRV_PCM_RATE_* */
+	__le32 rate_min;	/* min rate */
+	__le32 rate_max;	/* max rate */
+	__le32 channels_min;	/* min channels */
+	__le32 channels_max;	/* max channels */
+	__le32 periods_min;	/* min number of periods */
+	__le32 periods_max;	/* max number of periods */
+	__le32 period_size_min;	/* min period size bytes */
+	__le32 period_size_max;	/* max period size bytes */
+	__le32 buffer_size_min;	/* min buffer size bytes */
+	__le32 buffer_size_max;	/* max buffer size bytes */
+} __packed;
+
+/* PCM v4 */
+struct snd_soc_tplg_pcm_v4 {
+	__le32 size;		/* in bytes of this structure */
+	char pcm_name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN];
+	char dai_name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN];
+	__le32 pcm_id;		/* unique ID - used to match with DAI link */
+	__le32 dai_id;		/* unique ID - used to match */
+	__le32 playback;	/* supports playback mode */
+	__le32 capture;		/* supports capture mode */
+	__le32 compress;	/* 1 = compressed; 0 = PCM */
+	struct snd_soc_tplg_stream stream[SND_SOC_TPLG_STREAM_CONFIG_MAX]; /* for DAI link */
+	__le32 num_streams;	/* number of streams */
+	struct snd_soc_tplg_stream_caps_v4 caps[2]; /* playback and capture for DAI */
+} __packed;
+
+/* Physical link config v4 */
+struct snd_soc_tplg_link_config_v4 {
+	__le32 size;            /* in bytes of this structure */
+	__le32 id;              /* unique ID - used to match */
+	struct snd_soc_tplg_stream stream[SND_SOC_TPLG_STREAM_CONFIG_MAX]; /* supported configs playback and captrure */
+	__le32 num_streams;     /* number of streams */
+} __packed;
+
+/* topology context */
 struct soc_tplg {
 	const struct firmware *fw;
 
@@ -428,33 +486,41 @@ static void remove_widget(struct snd_soc_component *comp,
 		dobj->ops->widget_unload(comp, dobj);
 
 	/*
-	 * Dynamic Widgets either have 1 enum kcontrol or 1..N mixers.
+	 * Dynamic Widgets either have 1..N enum kcontrols or mixers.
 	 * The enum may either have an array of values or strings.
 	 */
-	if (dobj->widget.kcontrol_enum) {
+	if (dobj->widget.kcontrol_type == SND_SOC_TPLG_TYPE_ENUM) {
 		/* enumerated widget mixer */
-		struct soc_enum *se =
-			(struct soc_enum *)w->kcontrols[0]->private_value;
+		for (i = 0; i < w->num_kcontrols; i++) {
+			struct snd_kcontrol *kcontrol = w->kcontrols[i];
+			struct soc_enum *se =
+				(struct soc_enum *)kcontrol->private_value;
 
-		snd_ctl_remove(card, w->kcontrols[0]);
+			snd_ctl_remove(card, kcontrol);
 
-		kfree(se->dobj.control.dvalues);
-		for (i = 0; i < se->items; i++)
-			kfree(se->dobj.control.dtexts[i]);
+			kfree(se->dobj.control.dvalues);
+			for (i = 0; i < se->items; i++)
+				kfree(se->dobj.control.dtexts[i]);
 
-		kfree(se);
+			kfree(se);
+		}
 		kfree(w->kcontrol_news);
 	} else {
-		/* non enumerated widget mixer */
+		/* volume mixer or bytes controls */
 		for (i = 0; i < w->num_kcontrols; i++) {
 			struct snd_kcontrol *kcontrol = w->kcontrols[i];
-			struct soc_mixer_control *sm =
-			(struct soc_mixer_control *) kcontrol->private_value;
 
-			kfree(w->kcontrols[i]->tlv.p);
+			if (dobj->widget.kcontrol_type
+			    == SND_SOC_TPLG_TYPE_MIXER)
+				kfree(kcontrol->tlv.p);
 
-			snd_ctl_remove(card, w->kcontrols[i]);
-			kfree(sm);
+			snd_ctl_remove(card, kcontrol);
+
+			/* Private value is used as struct soc_mixer_control
+			 * for volume mixers or soc_bytes_ext for bytes
+			 * controls.
+			 */
+			kfree((void *)kcontrol->private_value);
 		}
 		kfree(w->kcontrol_news);
 	}
@@ -474,6 +540,7 @@ static void remove_dai(struct snd_soc_component *comp,
 	if (dobj->ops && dobj->ops->dai_unload)
 		dobj->ops->dai_unload(comp, dobj);
 
+	kfree(dai_drv->name);
 	list_del(&dobj->list);
 	kfree(dai_drv);
 }
@@ -491,6 +558,10 @@ static void remove_link(struct snd_soc_component *comp,
 	if (dobj->ops && dobj->ops->link_unload)
 		dobj->ops->link_unload(comp, dobj);
 
+	kfree(link->name);
+	kfree(link->stream_name);
+	kfree(link->cpu_dai_name);
+
 	list_del(&dobj->list);
 	snd_soc_remove_dai_link(comp->card, link);
 	kfree(link);
@@ -1193,98 +1264,105 @@ err:
 }
 
 static struct snd_kcontrol_new *soc_tplg_dapm_widget_denum_create(
-	struct soc_tplg *tplg)
+	struct soc_tplg *tplg, int num_kcontrols)
 {
 	struct snd_kcontrol_new *kc;
 	struct snd_soc_tplg_enum_control *ec;
 	struct soc_enum *se;
-	int i, err;
-
-	ec = (struct snd_soc_tplg_enum_control *)tplg->pos;
-	tplg->pos += (sizeof(struct snd_soc_tplg_enum_control) +
-		ec->priv.size);
+	int i, j, err;
 
-	/* validate kcontrol */
-	if (strnlen(ec->hdr.name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN) ==
-		SNDRV_CTL_ELEM_ID_NAME_MAXLEN)
-		return NULL;
-
-	kc = kzalloc(sizeof(*kc), GFP_KERNEL);
+	kc = kcalloc(num_kcontrols, sizeof(*kc), GFP_KERNEL);
 	if (kc == NULL)
 		return NULL;
 
-	se = kzalloc(sizeof(*se), GFP_KERNEL);
-	if (se == NULL)
-		goto err;
+	for (i = 0; i < num_kcontrols; i++) {
+		ec = (struct snd_soc_tplg_enum_control *)tplg->pos;
+		/* validate kcontrol */
+		if (strnlen(ec->hdr.name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN) ==
+			    SNDRV_CTL_ELEM_ID_NAME_MAXLEN)
+			return NULL;
 
-	dev_dbg(tplg->dev, " adding DAPM widget enum control %s\n",
-		ec->hdr.name);
+		se = kzalloc(sizeof(*se), GFP_KERNEL);
+		if (se == NULL)
+			goto err;
 
-	kc->name = ec->hdr.name;
-	kc->private_value = (long)se;
-	kc->iface = SNDRV_CTL_ELEM_IFACE_MIXER;
-	kc->access = ec->hdr.access;
+		dev_dbg(tplg->dev, " adding DAPM widget enum control %s\n",
+			ec->hdr.name);
+
+		kc[i].name = ec->hdr.name;
+		kc[i].private_value = (long)se;
+		kc[i].iface = SNDRV_CTL_ELEM_IFACE_MIXER;
+		kc[i].access = ec->hdr.access;
 
-	/* we only support FL/FR channel mapping atm */
-	se->reg = tplc_chan_get_reg(tplg, ec->channel, SNDRV_CHMAP_FL);
-	se->shift_l = tplc_chan_get_shift(tplg, ec->channel, SNDRV_CHMAP_FL);
-	se->shift_r = tplc_chan_get_shift(tplg, ec->channel, SNDRV_CHMAP_FR);
+		/* we only support FL/FR channel mapping atm */
+		se->reg = tplc_chan_get_reg(tplg, ec->channel, SNDRV_CHMAP_FL);
+		se->shift_l = tplc_chan_get_shift(tplg, ec->channel,
+						  SNDRV_CHMAP_FL);
+		se->shift_r = tplc_chan_get_shift(tplg, ec->channel,
+						  SNDRV_CHMAP_FR);
 
-	se->items = ec->items;
-	se->mask = ec->mask;
-	se->dobj.index = tplg->index;
+		se->items = ec->items;
+		se->mask = ec->mask;
+		se->dobj.index = tplg->index;
 
-	switch (ec->hdr.ops.info) {
-	case SND_SOC_TPLG_CTL_ENUM_VALUE:
-	case SND_SOC_TPLG_DAPM_CTL_ENUM_VALUE:
-		err = soc_tplg_denum_create_values(se, ec);
-		if (err < 0) {
-			dev_err(tplg->dev, "ASoC: could not create values for %s\n",
-				ec->hdr.name);
+		switch (ec->hdr.ops.info) {
+		case SND_SOC_TPLG_CTL_ENUM_VALUE:
+		case SND_SOC_TPLG_DAPM_CTL_ENUM_VALUE:
+			err = soc_tplg_denum_create_values(se, ec);
+			if (err < 0) {
+				dev_err(tplg->dev, "ASoC: could not create values for %s\n",
+					ec->hdr.name);
+				goto err_se;
+			}
+			/* fall through to create texts */
+		case SND_SOC_TPLG_CTL_ENUM:
+		case SND_SOC_TPLG_DAPM_CTL_ENUM_DOUBLE:
+		case SND_SOC_TPLG_DAPM_CTL_ENUM_VIRT:
+			err = soc_tplg_denum_create_texts(se, ec);
+			if (err < 0) {
+				dev_err(tplg->dev, "ASoC: could not create texts for %s\n",
+					ec->hdr.name);
+				goto err_se;
+			}
+			break;
+		default:
+			dev_err(tplg->dev, "ASoC: invalid enum control type %d for %s\n",
+				ec->hdr.ops.info, ec->hdr.name);
 			goto err_se;
 		}
-		/* fall through to create texts */
-	case SND_SOC_TPLG_CTL_ENUM:
-	case SND_SOC_TPLG_DAPM_CTL_ENUM_DOUBLE:
-	case SND_SOC_TPLG_DAPM_CTL_ENUM_VIRT:
-		err = soc_tplg_denum_create_texts(se, ec);
+
+		/* map io handlers */
+		err = soc_tplg_kcontrol_bind_io(&ec->hdr, &kc[i], tplg);
+		if (err) {
+			soc_control_err(tplg, &ec->hdr, ec->hdr.name);
+			goto err_se;
+		}
+
+		/* pass control to driver for optional further init */
+		err = soc_tplg_init_kcontrol(tplg, &kc[i],
+			(struct snd_soc_tplg_ctl_hdr *)ec);
 		if (err < 0) {
-			dev_err(tplg->dev, "ASoC: could not create texts for %s\n",
+			dev_err(tplg->dev, "ASoC: failed to init %s\n",
 				ec->hdr.name);
 			goto err_se;
 		}
-		break;
-	default:
-		dev_err(tplg->dev, "ASoC: invalid enum control type %d for %s\n",
-			ec->hdr.ops.info, ec->hdr.name);
-		goto err_se;
-	}
 
-	/* map io handlers */
-	err = soc_tplg_kcontrol_bind_io(&ec->hdr, kc, tplg);
-	if (err) {
-		soc_control_err(tplg, &ec->hdr, ec->hdr.name);
-		goto err_se;
-	}
-
-	/* pass control to driver for optional further init */
-	err = soc_tplg_init_kcontrol(tplg, kc,
-		(struct snd_soc_tplg_ctl_hdr *)ec);
-	if (err < 0) {
-		dev_err(tplg->dev, "ASoC: failed to init %s\n",
-			ec->hdr.name);
-		goto err_se;
+		tplg->pos += (sizeof(struct snd_soc_tplg_enum_control) +
+				ec->priv.size);
 	}
 
 	return kc;
 
 err_se:
-	/* free values and texts */
-	kfree(se->dobj.control.dvalues);
-	for (i = 0; i < ec->items; i++)
-		kfree(se->dobj.control.dtexts[i]);
+	for (; i >= 0; i--) {
+		/* free values and texts */
+		se = (struct soc_enum *)kc[i].private_value;
+		kfree(se->dobj.control.dvalues);
+		for (j = 0; j < ec->items; j++)
+			kfree(se->dobj.control.dtexts[j]);
 
-	kfree(se);
+		kfree(se);
+	}
 err:
 	kfree(kc);
 
@@ -1366,6 +1444,7 @@ static int soc_tplg_dapm_widget_create(struct soc_tplg *tplg,
 	struct snd_soc_dapm_widget template, *widget;
 	struct snd_soc_tplg_ctl_hdr *control_hdr;
 	struct snd_soc_card *card = tplg->comp->card;
+	unsigned int kcontrol_type;
 	int ret = 0;
 
 	if (strnlen(w->name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN) ==
@@ -1406,6 +1485,7 @@ static int soc_tplg_dapm_widget_create(struct soc_tplg *tplg,
 	tplg->pos +=
 		(sizeof(struct snd_soc_tplg_dapm_widget) + w->priv.size);
 	if (w->num_kcontrols == 0) {
+		kcontrol_type = 0;
 		template.num_kcontrols = 0;
 		goto widget;
 	}
@@ -1421,6 +1501,7 @@ static int soc_tplg_dapm_widget_create(struct soc_tplg *tplg,
 	case SND_SOC_TPLG_CTL_VOLSW_XR_SX:
 	case SND_SOC_TPLG_CTL_RANGE:
 	case SND_SOC_TPLG_DAPM_CTL_VOLSW:
+		kcontrol_type = SND_SOC_TPLG_TYPE_MIXER;  /* volume mixer */
 		template.num_kcontrols = w->num_kcontrols;
 		template.kcontrol_news =
 			soc_tplg_dapm_widget_dmixer_create(tplg,
@@ -1435,16 +1516,18 @@ static int soc_tplg_dapm_widget_create(struct soc_tplg *tplg,
 	case SND_SOC_TPLG_DAPM_CTL_ENUM_DOUBLE:
 	case SND_SOC_TPLG_DAPM_CTL_ENUM_VIRT:
 	case SND_SOC_TPLG_DAPM_CTL_ENUM_VALUE:
-		template.dobj.widget.kcontrol_enum = 1;
-		template.num_kcontrols = 1;
+		kcontrol_type = SND_SOC_TPLG_TYPE_ENUM;	/* enumerated mixer */
+		template.num_kcontrols = w->num_kcontrols;
 		template.kcontrol_news =
-			soc_tplg_dapm_widget_denum_create(tplg);
+			soc_tplg_dapm_widget_denum_create(tplg,
+			template.num_kcontrols);
 		if (!template.kcontrol_news) {
 			ret = -ENOMEM;
 			goto hdr_err;
 		}
 		break;
 	case SND_SOC_TPLG_CTL_BYTES:
+		kcontrol_type = SND_SOC_TPLG_TYPE_BYTES; /* bytes control */
 		template.num_kcontrols = w->num_kcontrols;
 		template.kcontrol_news =
 			soc_tplg_dapm_widget_dbytes_create(tplg,
@@ -1481,6 +1564,7 @@ widget:
 	}
 
 	widget->dobj.type = SND_SOC_DOBJ_WIDGET;
+	widget->dobj.widget.kcontrol_type = kcontrol_type;
 	widget->dobj.ops = tplg->ops;
 	widget->dobj.index = tplg->index;
 	kfree(template.sname);
@@ -1589,7 +1673,8 @@ static int soc_tplg_dai_create(struct soc_tplg *tplg,
 	if (dai_drv == NULL)
 		return -ENOMEM;
 
-	dai_drv->name = pcm->dai_name;
+	if (strlen(pcm->dai_name))
+		dai_drv->name = kstrdup(pcm->dai_name, GFP_KERNEL);
 	dai_drv->id = pcm->dai_id;
 
 	if (pcm->playback) {
@@ -1621,8 +1706,31 @@ static int soc_tplg_dai_create(struct soc_tplg *tplg,
 	return snd_soc_register_dai(tplg->comp, dai_drv);
 }
 
+static void set_link_flags(struct snd_soc_dai_link *link,
+		unsigned int flag_mask, unsigned int flags)
+{
+	if (flag_mask & SND_SOC_TPLG_LNK_FLGBIT_SYMMETRIC_RATES)
+		link->symmetric_rates =
+			flags & SND_SOC_TPLG_LNK_FLGBIT_SYMMETRIC_RATES ? 1 : 0;
+
+	if (flag_mask & SND_SOC_TPLG_LNK_FLGBIT_SYMMETRIC_CHANNELS)
+		link->symmetric_channels =
+			flags & SND_SOC_TPLG_LNK_FLGBIT_SYMMETRIC_CHANNELS ?
+			1 : 0;
+
+	if (flag_mask & SND_SOC_TPLG_LNK_FLGBIT_SYMMETRIC_SAMPLEBITS)
+		link->symmetric_samplebits =
+			flags & SND_SOC_TPLG_LNK_FLGBIT_SYMMETRIC_SAMPLEBITS ?
+			1 : 0;
+
+	if (flag_mask & SND_SOC_TPLG_LNK_FLGBIT_VOICE_WAKEUP)
+		link->ignore_suspend =
+		flags & SND_SOC_TPLG_LNK_FLGBIT_VOICE_WAKEUP ?
+		1 : 0;
+}
+
 /* create the FE DAI link */
-static int soc_tplg_link_create(struct soc_tplg *tplg,
+static int soc_tplg_fe_link_create(struct soc_tplg *tplg,
 	struct snd_soc_tplg_pcm *pcm)
 {
 	struct snd_soc_dai_link *link;
@@ -1632,11 +1740,15 @@ static int soc_tplg_link_create(struct soc_tplg *tplg,
 	if (link == NULL)
 		return -ENOMEM;
 
-	link->name = pcm->pcm_name;
-	link->stream_name = pcm->pcm_name;
+	if (strlen(pcm->pcm_name)) {
+		link->name = kstrdup(pcm->pcm_name, GFP_KERNEL);
+		link->stream_name = kstrdup(pcm->pcm_name, GFP_KERNEL);
+	}
 	link->id = pcm->pcm_id;
 
-	link->cpu_dai_name = pcm->dai_name;
+	if (strlen(pcm->dai_name))
+		link->cpu_dai_name = kstrdup(pcm->dai_name, GFP_KERNEL);
+
 	link->codec_name = "snd-soc-dummy";
 	link->codec_dai_name = "snd-soc-dummy-dai";
 
@@ -1644,6 +1756,8 @@ static int soc_tplg_link_create(struct soc_tplg *tplg,
 	link->dynamic = 1;
 	link->dpcm_playback = pcm->playback;
 	link->dpcm_capture = pcm->capture;
+	if (pcm->flag_mask)
+		set_link_flags(link, pcm->flag_mask, pcm->flags);
 
 	/* pass control to component driver for optional further init */
 	ret = soc_tplg_dai_link_load(tplg, link);
@@ -1672,55 +1786,351 @@ static int soc_tplg_pcm_create(struct soc_tplg *tplg,
 	if (ret < 0)
 		return ret;
 
-	return  soc_tplg_link_create(tplg, pcm);
+	return  soc_tplg_fe_link_create(tplg, pcm);
+}
+
+/* copy stream caps from the old version 4 of source */
+static void stream_caps_new_ver(struct snd_soc_tplg_stream_caps *dest,
+				struct snd_soc_tplg_stream_caps_v4 *src)
+{
+	dest->size = sizeof(*dest);
+	memcpy(dest->name, src->name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN);
+	dest->formats = src->formats;
+	dest->rates = src->rates;
+	dest->rate_min = src->rate_min;
+	dest->rate_max = src->rate_max;
+	dest->channels_min = src->channels_min;
+	dest->channels_max = src->channels_max;
+	dest->periods_min = src->periods_min;
+	dest->periods_max = src->periods_max;
+	dest->period_size_min = src->period_size_min;
+	dest->period_size_max = src->period_size_max;
+	dest->buffer_size_min = src->buffer_size_min;
+	dest->buffer_size_max = src->buffer_size_max;
+}
+
+/**
+ * pcm_new_ver - Create the new version of PCM from the old version.
+ * @tplg: topology context
+ * @src: older version of pcm as a source
+ * @pcm: latest version of pcm created from the source
+ *
+ * Support from vesion 4. User should free the returned pcm manually.
+ */
+static int pcm_new_ver(struct soc_tplg *tplg,
+		       struct snd_soc_tplg_pcm *src,
+		       struct snd_soc_tplg_pcm **pcm)
+{
+	struct snd_soc_tplg_pcm *dest;
+	struct snd_soc_tplg_pcm_v4 *src_v4;
+	int i;
+
+	*pcm = NULL;
+
+	if (src->size != sizeof(*src_v4)) {
+		dev_err(tplg->dev, "ASoC: invalid PCM size\n");
+		return -EINVAL;
+	}
+
+	dev_warn(tplg->dev, "ASoC: old version of PCM\n");
+	src_v4 = (struct snd_soc_tplg_pcm_v4 *)src;
+	dest = kzalloc(sizeof(*dest), GFP_KERNEL);
+	if (!dest)
+		return -ENOMEM;
+
+	dest->size = sizeof(*dest);	/* size of latest abi version */
+	memcpy(dest->pcm_name, src_v4->pcm_name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN);
+	memcpy(dest->dai_name, src_v4->dai_name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN);
+	dest->pcm_id = src_v4->pcm_id;
+	dest->dai_id = src_v4->dai_id;
+	dest->playback = src_v4->playback;
+	dest->capture = src_v4->capture;
+	dest->compress = src_v4->compress;
+	dest->num_streams = src_v4->num_streams;
+	for (i = 0; i < dest->num_streams; i++)
+		memcpy(&dest->stream[i], &src_v4->stream[i],
+		       sizeof(struct snd_soc_tplg_stream));
+
+	for (i = 0; i < 2; i++)
+		stream_caps_new_ver(&dest->caps[i], &src_v4->caps[i]);
+
+	*pcm = dest;
+	return 0;
 }
 
 static int soc_tplg_pcm_elems_load(struct soc_tplg *tplg,
 	struct snd_soc_tplg_hdr *hdr)
 {
-	struct snd_soc_tplg_pcm *pcm;
+	struct snd_soc_tplg_pcm *pcm, *_pcm;
 	int count = hdr->count;
-	int i;
+	int i, err;
+	bool abi_match;
 
 	if (tplg->pass != SOC_TPLG_PASS_PCM_DAI)
 		return 0;
 
+	/* check the element size and count */
+	pcm = (struct snd_soc_tplg_pcm *)tplg->pos;
+	if (pcm->size > sizeof(struct snd_soc_tplg_pcm)
+		|| pcm->size < sizeof(struct snd_soc_tplg_pcm_v4)) {
+		dev_err(tplg->dev, "ASoC: invalid size %d for PCM elems\n",
+			pcm->size);
+		return -EINVAL;
+	}
+
 	if (soc_tplg_check_elem_count(tplg,
-		sizeof(struct snd_soc_tplg_pcm), count,
+		pcm->size, count,
 		hdr->payload_size, "PCM DAI")) {
 		dev_err(tplg->dev, "ASoC: invalid count %d for PCM DAI elems\n",
 			count);
 		return -EINVAL;
 	}
 
-	/* create the FE DAIs and DAI links */
-	pcm = (struct snd_soc_tplg_pcm *)tplg->pos;
 	for (i = 0; i < count; i++) {
-		if (pcm->size != sizeof(*pcm)) {
-			dev_err(tplg->dev, "ASoC: invalid pcm size\n");
-			return -EINVAL;
+		pcm = (struct snd_soc_tplg_pcm *)tplg->pos;
+
+		/* check ABI version by size, create a new version of pcm
+		 * if abi not match.
+		 */
+		if (pcm->size == sizeof(*pcm)) {
+			abi_match = true;
+			_pcm = pcm;
+		} else {
+			abi_match = false;
+			err = pcm_new_ver(tplg, pcm, &_pcm);
 		}
 
-		soc_tplg_pcm_create(tplg, pcm);
-		pcm++;
+		/* create the FE DAIs and DAI links */
+		soc_tplg_pcm_create(tplg, _pcm);
+
+		/* offset by version-specific struct size and
+		 * real priv data size
+		 */
+		tplg->pos += pcm->size + _pcm->priv.size;
+
+		if (!abi_match)
+			kfree(_pcm); /* free the duplicated one */
 	}
 
 	dev_dbg(tplg->dev, "ASoC: adding %d PCM DAIs\n", count);
-	tplg->pos += sizeof(struct snd_soc_tplg_pcm) * count;
 
 	return 0;
 }
 
-/* *
- * soc_tplg_be_dai_config - Find and configure an existing BE DAI.
+/**
+ * set_link_hw_format - Set the HW audio format of the physical DAI link.
+ * @tplg: topology context
+ * @cfg: physical link configs.
+ *
+ * Topology context contains a list of supported HW formats (configs) and
+ * a default format ID for the physical link. This function will use this
+ * default ID to choose the HW format to set the link's DAI format for init.
+ */
+static void set_link_hw_format(struct snd_soc_dai_link *link,
+			struct snd_soc_tplg_link_config *cfg)
+{
+	struct snd_soc_tplg_hw_config *hw_config;
+	unsigned char bclk_master, fsync_master;
+	unsigned char invert_bclk, invert_fsync;
+	int i;
+
+	for (i = 0; i < cfg->num_hw_configs; i++) {
+		hw_config = &cfg->hw_config[i];
+		if (hw_config->id != cfg->default_hw_config_id)
+			continue;
+
+		link->dai_fmt = hw_config->fmt & SND_SOC_DAIFMT_FORMAT_MASK;
+
+		/* clock signal polarity */
+		invert_bclk = hw_config->invert_bclk;
+		invert_fsync = hw_config->invert_fsync;
+		if (!invert_bclk && !invert_fsync)
+			link->dai_fmt |= SND_SOC_DAIFMT_NB_NF;
+		else if (!invert_bclk && invert_fsync)
+			link->dai_fmt |= SND_SOC_DAIFMT_NB_IF;
+		else if (invert_bclk && !invert_fsync)
+			link->dai_fmt |= SND_SOC_DAIFMT_IB_NF;
+		else
+			link->dai_fmt |= SND_SOC_DAIFMT_IB_IF;
+
+		/* clock masters */
+		bclk_master = hw_config->bclk_master;
+		fsync_master = hw_config->fsync_master;
+		if (!bclk_master && !fsync_master)
+			link->dai_fmt |= SND_SOC_DAIFMT_CBM_CFM;
+		else if (bclk_master && !fsync_master)
+			link->dai_fmt |= SND_SOC_DAIFMT_CBS_CFM;
+		else if (!bclk_master && fsync_master)
+			link->dai_fmt |= SND_SOC_DAIFMT_CBM_CFS;
+		else
+			link->dai_fmt |= SND_SOC_DAIFMT_CBS_CFS;
+	}
+}
+
+/**
+ * link_new_ver - Create a new physical link config from the old
+ * version of source.
+ * @toplogy: topology context
+ * @src: old version of phyical link config as a source
+ * @link: latest version of physical link config created from the source
+ *
+ * Support from vesion 4. User need free the returned link config manually.
+ */
+static int link_new_ver(struct soc_tplg *tplg,
+			struct snd_soc_tplg_link_config *src,
+			struct snd_soc_tplg_link_config **link)
+{
+	struct snd_soc_tplg_link_config *dest;
+	struct snd_soc_tplg_link_config_v4 *src_v4;
+	int i;
+
+	*link = NULL;
+
+	if (src->size != sizeof(struct snd_soc_tplg_link_config_v4)) {
+		dev_err(tplg->dev, "ASoC: invalid physical link config size\n");
+		return -EINVAL;
+	}
+
+	dev_warn(tplg->dev, "ASoC: old version of physical link config\n");
+
+	src_v4 = (struct snd_soc_tplg_link_config_v4 *)src;
+	dest = kzalloc(sizeof(*dest), GFP_KERNEL);
+	if (!dest)
+		return -ENOMEM;
+
+	dest->size = sizeof(*dest);
+	dest->id = src_v4->id;
+	dest->num_streams = src_v4->num_streams;
+	for (i = 0; i < dest->num_streams; i++)
+		memcpy(&dest->stream[i], &src_v4->stream[i],
+		       sizeof(struct snd_soc_tplg_stream));
+
+	*link = dest;
+	return 0;
+}
+
+/* Find and configure an existing physical DAI link */
+static int soc_tplg_link_config(struct soc_tplg *tplg,
+	struct snd_soc_tplg_link_config *cfg)
+{
+	struct snd_soc_dai_link *link;
+	const char *name, *stream_name;
+	size_t len;
+	int ret;
+
+	len = strnlen(cfg->name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN);
+	if (len == SNDRV_CTL_ELEM_ID_NAME_MAXLEN)
+		return -EINVAL;
+	else if (len)
+		name = cfg->name;
+	else
+		name = NULL;
+
+	len = strnlen(cfg->stream_name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN);
+	if (len == SNDRV_CTL_ELEM_ID_NAME_MAXLEN)
+		return -EINVAL;
+	else if (len)
+		stream_name = cfg->stream_name;
+	else
+		stream_name = NULL;
+
+	link = snd_soc_find_dai_link(tplg->comp->card, cfg->id,
+				     name, stream_name);
+	if (!link) {
+		dev_err(tplg->dev, "ASoC: physical link %s (id %d) not exist\n",
+			name, cfg->id);
+		return -EINVAL;
+	}
+
+	/* hw format */
+	if (cfg->num_hw_configs)
+		set_link_hw_format(link, cfg);
+
+	/* flags */
+	if (cfg->flag_mask)
+		set_link_flags(link, cfg->flag_mask, cfg->flags);
+
+	/* pass control to component driver for optional further init */
+	ret = soc_tplg_dai_link_load(tplg, link);
+	if (ret < 0) {
+		dev_err(tplg->dev, "ASoC: physical link loading failed\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+
+/* Load physical link config elements from the topology context */
+static int soc_tplg_link_elems_load(struct soc_tplg *tplg,
+	struct snd_soc_tplg_hdr *hdr)
+{
+	struct snd_soc_tplg_link_config *link, *_link;
+	int count = hdr->count;
+	int i, ret;
+	bool abi_match;
+
+	if (tplg->pass != SOC_TPLG_PASS_LINK) {
+		tplg->pos += hdr->size + hdr->payload_size;
+		return 0;
+	};
+
+	/* check the element size and count */
+	link = (struct snd_soc_tplg_link_config *)tplg->pos;
+	if (link->size > sizeof(struct snd_soc_tplg_link_config)
+		|| link->size < sizeof(struct snd_soc_tplg_link_config_v4)) {
+		dev_err(tplg->dev, "ASoC: invalid size %d for physical link elems\n",
+			link->size);
+		return -EINVAL;
+	}
+
+	if (soc_tplg_check_elem_count(tplg,
+		link->size, count,
+		hdr->payload_size, "physical link config")) {
+		dev_err(tplg->dev, "ASoC: invalid count %d for physical link elems\n",
+			count);
+		return -EINVAL;
+	}
+
+	/* config physical DAI links */
+	for (i = 0; i < count; i++) {
+		link = (struct snd_soc_tplg_link_config *)tplg->pos;
+		if (link->size == sizeof(*link)) {
+			abi_match = true;
+			_link = link;
+		} else {
+			abi_match = false;
+			ret = link_new_ver(tplg, link, &_link);
+			if (ret < 0)
+				return ret;
+		}
+
+		ret = soc_tplg_link_config(tplg, _link);
+		if (ret < 0)
+			return ret;
+
+		/* offset by version-specific struct size and
+		 * real priv data size
+		 */
+		tplg->pos += link->size + _link->priv.size;
+
+		if (!abi_match)
+			kfree(_link); /* free the duplicated one */
+	}
+
+	return 0;
+}
+
+/**
+ * soc_tplg_dai_config - Find and configure an existing physical DAI.
  * @tplg: topology context
- * @be: topology BE DAI configs.
+ * @d: physical DAI configs.
  *
- * The BE dai should already be registered by the platform driver. The
- * platform driver should specify the BE DAI name and ID for matching.
+ * The physical dai should already be registered by the platform driver.
+ * The platform driver should specify the DAI name and ID for matching.
  */
-static int soc_tplg_be_dai_config(struct soc_tplg *tplg,
-				  struct snd_soc_tplg_be_dai *be)
+static int soc_tplg_dai_config(struct soc_tplg *tplg,
+			       struct snd_soc_tplg_dai *d)
 {
 	struct snd_soc_dai_link_component dai_component = {0};
 	struct snd_soc_dai *dai;
@@ -1729,17 +2139,17 @@ static int soc_tplg_be_dai_config(struct soc_tplg *tplg,
 	struct snd_soc_tplg_stream_caps *caps;
 	int ret;
 
-	dai_component.dai_name = be->dai_name;
+	dai_component.dai_name = d->dai_name;
 	dai = snd_soc_find_dai(&dai_component);
 	if (!dai) {
-		dev_err(tplg->dev, "ASoC: BE DAI %s not registered\n",
-			be->dai_name);
+		dev_err(tplg->dev, "ASoC: physical DAI %s not registered\n",
+			d->dai_name);
 		return -EINVAL;
 	}
 
-	if (be->dai_id != dai->id) {
-		dev_err(tplg->dev, "ASoC: BE DAI %s id mismatch\n",
-			be->dai_name);
+	if (d->dai_id != dai->id) {
+		dev_err(tplg->dev, "ASoC: physical DAI %s id mismatch\n",
+			d->dai_name);
 		return -EINVAL;
 	}
 
@@ -1747,20 +2157,20 @@ static int soc_tplg_be_dai_config(struct soc_tplg *tplg,
 	if (!dai_drv)
 		return -EINVAL;
 
-	if (be->playback) {
+	if (d->playback) {
 		stream = &dai_drv->playback;
-		caps = &be->caps[SND_SOC_TPLG_STREAM_PLAYBACK];
+		caps = &d->caps[SND_SOC_TPLG_STREAM_PLAYBACK];
 		set_stream_info(stream, caps);
 	}
 
-	if (be->capture) {
+	if (d->capture) {
 		stream = &dai_drv->capture;
-		caps = &be->caps[SND_SOC_TPLG_STREAM_CAPTURE];
+		caps = &d->caps[SND_SOC_TPLG_STREAM_CAPTURE];
 		set_stream_info(stream, caps);
 	}
 
-	if (be->flag_mask)
-		set_dai_flags(dai_drv, be->flag_mask, be->flags);
+	if (d->flag_mask)
+		set_dai_flags(dai_drv, d->flag_mask, d->flags);
 
 	/* pass control to component driver for optional further init */
 	ret = soc_tplg_dai_load(tplg, dai_drv);
@@ -1772,10 +2182,11 @@ static int soc_tplg_be_dai_config(struct soc_tplg *tplg,
 	return 0;
 }
 
-static int soc_tplg_be_dai_elems_load(struct soc_tplg *tplg,
-				      struct snd_soc_tplg_hdr *hdr)
+/* load physical DAI elements */
+static int soc_tplg_dai_elems_load(struct soc_tplg *tplg,
+				   struct snd_soc_tplg_hdr *hdr)
 {
-	struct snd_soc_tplg_be_dai *be;
+	struct snd_soc_tplg_dai *dai;
 	int count = hdr->count;
 	int i;
 
@@ -1784,41 +2195,95 @@ static int soc_tplg_be_dai_elems_load(struct soc_tplg *tplg,
 
 	/* config the existing BE DAIs */
 	for (i = 0; i < count; i++) {
-		be = (struct snd_soc_tplg_be_dai *)tplg->pos;
-		if (be->size != sizeof(*be)) {
-			dev_err(tplg->dev, "ASoC: invalid BE DAI size\n");
+		dai = (struct snd_soc_tplg_dai *)tplg->pos;
+		if (dai->size != sizeof(*dai)) {
+			dev_err(tplg->dev, "ASoC: invalid physical DAI size\n");
 			return -EINVAL;
 		}
 
-		soc_tplg_be_dai_config(tplg, be);
-		tplg->pos += (sizeof(*be) + be->priv.size);
+		soc_tplg_dai_config(tplg, dai);
+		tplg->pos += (sizeof(*dai) + dai->priv.size);
 	}
 
 	dev_dbg(tplg->dev, "ASoC: Configure %d BE DAIs\n", count);
 	return 0;
 }
 
+/**
+ * manifest_new_ver - Create a new version of manifest from the old version
+ * of source.
+ * @toplogy: topology context
+ * @src: old version of manifest as a source
+ * @manifest: latest version of manifest created from the source
+ *
+ * Support from vesion 4. Users need free the returned manifest manually.
+ */
+static int manifest_new_ver(struct soc_tplg *tplg,
+			    struct snd_soc_tplg_manifest *src,
+			    struct snd_soc_tplg_manifest **manifest)
+{
+	struct snd_soc_tplg_manifest *dest;
+	struct snd_soc_tplg_manifest_v4 *src_v4;
+
+	*manifest = NULL;
+
+	if (src->size != sizeof(*src_v4)) {
+		dev_err(tplg->dev, "ASoC: invalid manifest size\n");
+		return -EINVAL;
+	}
+
+	dev_warn(tplg->dev, "ASoC: old version of manifest\n");
+
+	src_v4 = (struct snd_soc_tplg_manifest_v4 *)src;
+	dest = kzalloc(sizeof(*dest) + src_v4->priv.size, GFP_KERNEL);
+	if (!dest)
+		return -ENOMEM;
+
+	dest->size = sizeof(*dest);	/* size of latest abi version */
+	dest->control_elems = src_v4->control_elems;
+	dest->widget_elems = src_v4->widget_elems;
+	dest->graph_elems = src_v4->graph_elems;
+	dest->pcm_elems = src_v4->pcm_elems;
+	dest->dai_link_elems = src_v4->dai_link_elems;
+	dest->priv.size = src_v4->priv.size;
+	if (dest->priv.size)
+		memcpy(dest->priv.data, src_v4->priv.data,
+		       src_v4->priv.size);
+
+	*manifest = dest;
+	return 0;
+}
 
 static int soc_tplg_manifest_load(struct soc_tplg *tplg,
 				  struct snd_soc_tplg_hdr *hdr)
 {
-	struct snd_soc_tplg_manifest *manifest;
+	struct snd_soc_tplg_manifest *manifest, *_manifest;
+	bool abi_match;
+	int err;
 
 	if (tplg->pass != SOC_TPLG_PASS_MANIFEST)
 		return 0;
 
 	manifest = (struct snd_soc_tplg_manifest *)tplg->pos;
-	if (manifest->size != sizeof(*manifest)) {
-		dev_err(tplg->dev, "ASoC: invalid manifest size\n");
-		return -EINVAL;
-	}
 
-	tplg->pos += sizeof(struct snd_soc_tplg_manifest);
+	/* check ABI version by size, create a new manifest if abi not match */
+	if (manifest->size == sizeof(*manifest)) {
+		abi_match = true;
+		_manifest = manifest;
+	} else {
+		abi_match = false;
+		err = manifest_new_ver(tplg, manifest, &_manifest);
+		if (err < 0)
+			return err;
+	}
 
+	/* pass control to component driver for optional further init */
 	if (tplg->comp && tplg->ops && tplg->ops->manifest)
-		return tplg->ops->manifest(tplg->comp, manifest);
+		return tplg->ops->manifest(tplg->comp, _manifest);
+
+	if (!abi_match)	/* free the duplicated one */
+		kfree(_manifest);
 
-	dev_err(tplg->dev, "ASoC: Firmware manifest not supported\n");
 	return 0;
 }
 
@@ -1854,7 +2319,9 @@ static int soc_valid_header(struct soc_tplg *tplg,
 		return -EINVAL;
 	}
 
-	if (hdr->abi != SND_SOC_TPLG_ABI_VERSION) {
+	/* Support ABI from version 4 */
+	if (hdr->abi > SND_SOC_TPLG_ABI_VERSION
+		|| hdr->abi < SND_SOC_TPLG_ABI_VERSION_MIN) {
 		dev_err(tplg->dev,
 			"ASoC: pass %d invalid ABI version got 0x%x need 0x%x at offset 0x%lx size 0x%zx.\n",
 			tplg->pass, hdr->abi,
@@ -1902,8 +2369,12 @@ static int soc_tplg_load_header(struct soc_tplg *tplg,
 		return soc_tplg_dapm_widget_elems_load(tplg, hdr);
 	case SND_SOC_TPLG_TYPE_PCM:
 		return soc_tplg_pcm_elems_load(tplg, hdr);
-	case SND_SOC_TPLG_TYPE_BE_DAI:
-		return soc_tplg_be_dai_elems_load(tplg, hdr);
+	case SND_SOC_TPLG_TYPE_DAI:
+		return soc_tplg_dai_elems_load(tplg, hdr);
+	case SND_SOC_TPLG_TYPE_DAI_LINK:
+	case SND_SOC_TPLG_TYPE_BACKEND_LINK:
+		/* physical link configurations */
+		return soc_tplg_link_elems_load(tplg, hdr);
 	case SND_SOC_TPLG_TYPE_MANIFEST:
 		return soc_tplg_manifest_load(tplg, hdr);
 	default:
diff --git a/sound/soc/soc-utils.c b/sound/soc/soc-utils.c
index 393e8f0fe2cc..644d9a9ebfbc 100644
--- a/sound/soc/soc-utils.c
+++ b/sound/soc/soc-utils.c
@@ -58,6 +58,205 @@ int snd_soc_params_to_bclk(struct snd_pcm_hw_params *params)
 }
 EXPORT_SYMBOL_GPL(snd_soc_params_to_bclk);
 
+int snd_soc_component_enable_pin(struct snd_soc_component *component,
+				 const char *pin)
+{
+	struct snd_soc_dapm_context *dapm =
+		snd_soc_component_get_dapm(component);
+	char *full_name;
+	int ret;
+
+	if (!component->name_prefix)
+		return snd_soc_dapm_enable_pin(dapm, pin);
+
+	full_name = kasprintf(GFP_KERNEL, "%s %s", component->name_prefix, pin);
+	if (!full_name)
+		return -ENOMEM;
+
+	ret = snd_soc_dapm_enable_pin(dapm, full_name);
+	kfree(full_name);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(snd_soc_component_enable_pin);
+
+int snd_soc_component_enable_pin_unlocked(struct snd_soc_component *component,
+					  const char *pin)
+{
+	struct snd_soc_dapm_context *dapm =
+		snd_soc_component_get_dapm(component);
+	char *full_name;
+	int ret;
+
+	if (!component->name_prefix)
+		return snd_soc_dapm_enable_pin_unlocked(dapm, pin);
+
+	full_name = kasprintf(GFP_KERNEL, "%s %s", component->name_prefix, pin);
+	if (!full_name)
+		return -ENOMEM;
+
+	ret = snd_soc_dapm_enable_pin_unlocked(dapm, full_name);
+	kfree(full_name);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(snd_soc_component_enable_pin_unlocked);
+
+int snd_soc_component_disable_pin(struct snd_soc_component *component,
+				  const char *pin)
+{
+	struct snd_soc_dapm_context *dapm =
+		snd_soc_component_get_dapm(component);
+	char *full_name;
+	int ret;
+
+	if (!component->name_prefix)
+		return snd_soc_dapm_disable_pin(dapm, pin);
+
+	full_name = kasprintf(GFP_KERNEL, "%s %s", component->name_prefix, pin);
+	if (!full_name)
+		return -ENOMEM;
+
+	ret = snd_soc_dapm_disable_pin(dapm, full_name);
+	kfree(full_name);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(snd_soc_component_disable_pin);
+
+int snd_soc_component_disable_pin_unlocked(struct snd_soc_component *component,
+					   const char *pin)
+{
+	struct snd_soc_dapm_context *dapm =
+		snd_soc_component_get_dapm(component);
+	char *full_name;
+	int ret;
+
+	if (!component->name_prefix)
+		return snd_soc_dapm_disable_pin_unlocked(dapm, pin);
+
+	full_name = kasprintf(GFP_KERNEL, "%s %s", component->name_prefix, pin);
+	if (!full_name)
+		return -ENOMEM;
+
+	ret = snd_soc_dapm_disable_pin_unlocked(dapm, full_name);
+	kfree(full_name);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(snd_soc_component_disable_pin_unlocked);
+
+int snd_soc_component_nc_pin(struct snd_soc_component *component,
+			     const char *pin)
+{
+	struct snd_soc_dapm_context *dapm =
+		snd_soc_component_get_dapm(component);
+	char *full_name;
+	int ret;
+
+	if (!component->name_prefix)
+		return snd_soc_dapm_nc_pin(dapm, pin);
+
+	full_name = kasprintf(GFP_KERNEL, "%s %s", component->name_prefix, pin);
+	if (!full_name)
+		return -ENOMEM;
+
+	ret = snd_soc_dapm_nc_pin(dapm, full_name);
+	kfree(full_name);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(snd_soc_component_nc_pin);
+
+int snd_soc_component_nc_pin_unlocked(struct snd_soc_component *component,
+				      const char *pin)
+{
+	struct snd_soc_dapm_context *dapm =
+		snd_soc_component_get_dapm(component);
+	char *full_name;
+	int ret;
+
+	if (!component->name_prefix)
+		return snd_soc_dapm_nc_pin_unlocked(dapm, pin);
+
+	full_name = kasprintf(GFP_KERNEL, "%s %s", component->name_prefix, pin);
+	if (!full_name)
+		return -ENOMEM;
+
+	ret = snd_soc_dapm_nc_pin_unlocked(dapm, full_name);
+	kfree(full_name);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(snd_soc_component_nc_pin_unlocked);
+
+int snd_soc_component_get_pin_status(struct snd_soc_component *component,
+				     const char *pin)
+{
+	struct snd_soc_dapm_context *dapm =
+		snd_soc_component_get_dapm(component);
+	char *full_name;
+	int ret;
+
+	if (!component->name_prefix)
+		return snd_soc_dapm_get_pin_status(dapm, pin);
+
+	full_name = kasprintf(GFP_KERNEL, "%s %s", component->name_prefix, pin);
+	if (!full_name)
+		return -ENOMEM;
+
+	ret = snd_soc_dapm_get_pin_status(dapm, full_name);
+	kfree(full_name);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(snd_soc_component_get_pin_status);
+
+int snd_soc_component_force_enable_pin(struct snd_soc_component *component,
+				       const char *pin)
+{
+	struct snd_soc_dapm_context *dapm =
+		snd_soc_component_get_dapm(component);
+	char *full_name;
+	int ret;
+
+	if (!component->name_prefix)
+		return snd_soc_dapm_force_enable_pin(dapm, pin);
+
+	full_name = kasprintf(GFP_KERNEL, "%s %s", component->name_prefix, pin);
+	if (!full_name)
+		return -ENOMEM;
+
+	ret = snd_soc_dapm_force_enable_pin(dapm, full_name);
+	kfree(full_name);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(snd_soc_component_force_enable_pin);
+
+int snd_soc_component_force_enable_pin_unlocked(
+					struct snd_soc_component *component,
+					const char *pin)
+{
+	struct snd_soc_dapm_context *dapm =
+		snd_soc_component_get_dapm(component);
+	char *full_name;
+	int ret;
+
+	if (!component->name_prefix)
+		return snd_soc_dapm_force_enable_pin_unlocked(dapm, pin);
+
+	full_name = kasprintf(GFP_KERNEL, "%s %s", component->name_prefix, pin);
+	if (!full_name)
+		return -ENOMEM;
+
+	ret = snd_soc_dapm_force_enable_pin_unlocked(dapm, full_name);
+	kfree(full_name);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(snd_soc_component_force_enable_pin_unlocked);
+
 static const struct snd_pcm_hardware dummy_dma_hardware = {
 	/* Random values to keep userspace happy when checking constraints */
 	.info			= SNDRV_PCM_INFO_INTERLEAVED |
diff --git a/sound/soc/sti/sti_uniperif.c b/sound/soc/sti/sti_uniperif.c
index 549fac349fa0..98eb205a0b62 100644
--- a/sound/soc/sti/sti_uniperif.c
+++ b/sound/soc/sti/sti_uniperif.c
@@ -7,6 +7,7 @@
 
 #include <linux/module.h>
 #include <linux/pinctrl/consumer.h>
+#include <linux/delay.h>
 
 #include "uniperif.h"
 
@@ -97,6 +98,28 @@ static const struct of_device_id snd_soc_sti_match[] = {
 	{},
 };
 
+int  sti_uniperiph_reset(struct uniperif *uni)
+{
+	int count = 10;
+
+	/* Reset uniperipheral uni */
+	SET_UNIPERIF_SOFT_RST_SOFT_RST(uni);
+
+	if (uni->ver < SND_ST_UNIPERIF_VERSION_UNI_PLR_TOP_1_0) {
+		while (GET_UNIPERIF_SOFT_RST_SOFT_RST(uni) && count) {
+			udelay(5);
+			count--;
+		}
+	}
+
+	if (!count) {
+		dev_err(uni->dev, "Failed to reset uniperif\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
 int sti_uniperiph_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
 			       unsigned int rx_mask, int slots,
 			       int slot_width)
@@ -293,7 +316,7 @@ static int sti_uniperiph_dai_suspend(struct snd_soc_dai *dai)
 
 	/* The uniperipheral should be in stopped state */
 	if (uni->state != UNIPERIF_STATE_STOPPED) {
-		dev_err(uni->dev, "%s: invalid uni state( %d)",
+		dev_err(uni->dev, "%s: invalid uni state( %d)\n",
 			__func__, (int)uni->state);
 		return -EBUSY;
 	}
@@ -301,7 +324,7 @@ static int sti_uniperiph_dai_suspend(struct snd_soc_dai *dai)
 	/* Pinctrl: switch pinstate to sleep */
 	ret = pinctrl_pm_select_sleep_state(uni->dev);
 	if (ret)
-		dev_err(uni->dev, "%s: failed to select pinctrl state",
+		dev_err(uni->dev, "%s: failed to select pinctrl state\n",
 			__func__);
 
 	return ret;
@@ -322,7 +345,7 @@ static int sti_uniperiph_dai_resume(struct snd_soc_dai *dai)
 	/* pinctrl: switch pinstate to default */
 	ret = pinctrl_pm_select_default_state(uni->dev);
 	if (ret)
-		dev_err(uni->dev, "%s: failed to select pinctrl state",
+		dev_err(uni->dev, "%s: failed to select pinctrl state\n",
 			__func__);
 
 	return ret;
@@ -366,11 +389,12 @@ static int sti_uniperiph_cpu_dai_of(struct device_node *node,
 	const struct of_device_id *of_id;
 	const struct sti_uniperiph_dev_data *dev_data;
 	const char *mode;
+	int ret;
 
 	/* Populate data structure depending on compatibility */
 	of_id = of_match_node(snd_soc_sti_match, node);
 	if (!of_id->data) {
-		dev_err(dev, "data associated to device is missing");
+		dev_err(dev, "data associated to device is missing\n");
 		return -EINVAL;
 	}
 	dev_data = (struct sti_uniperiph_dev_data *)of_id->data;
@@ -389,7 +413,7 @@ static int sti_uniperiph_cpu_dai_of(struct device_node *node,
 	uni->mem_region = platform_get_resource(priv->pdev, IORESOURCE_MEM, 0);
 
 	if (!uni->mem_region) {
-		dev_err(dev, "Failed to get memory resource");
+		dev_err(dev, "Failed to get memory resource\n");
 		return -ENODEV;
 	}
 
@@ -403,7 +427,7 @@ static int sti_uniperiph_cpu_dai_of(struct device_node *node,
 
 	uni->irq = platform_get_irq(priv->pdev, 0);
 	if (uni->irq < 0) {
-		dev_err(dev, "Failed to get IRQ resource");
+		dev_err(dev, "Failed to get IRQ resource\n");
 		return -ENXIO;
 	}
 
@@ -421,12 +445,15 @@ static int sti_uniperiph_cpu_dai_of(struct device_node *node,
 	dai_data->stream = dev_data->stream;
 
 	if (priv->dai_data.stream == SNDRV_PCM_STREAM_PLAYBACK) {
-		uni_player_init(priv->pdev, uni);
+		ret = uni_player_init(priv->pdev, uni);
 		stream = &dai->playback;
 	} else {
-		uni_reader_init(priv->pdev, uni);
+		ret = uni_reader_init(priv->pdev, uni);
 		stream = &dai->capture;
 	}
+	if (ret < 0)
+		return ret;
+
 	dai->ops = uni->dai_ops;
 
 	stream->stream_name = dai->name;
diff --git a/sound/soc/sti/uniperif.h b/sound/soc/sti/uniperif.h
index 1993c655fb79..d487dd2ef016 100644
--- a/sound/soc/sti/uniperif.h
+++ b/sound/soc/sti/uniperif.h
@@ -1397,6 +1397,8 @@ static inline int sti_uniperiph_get_unip_tdm_frame_size(struct uniperif *uni)
 	return (uni->tdm_slot.slots * uni->tdm_slot.slot_width / 8);
 }
 
+int  sti_uniperiph_reset(struct uniperif *uni);
+
 int sti_uniperiph_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
 			       unsigned int rx_mask, int slots,
 			       int slot_width);
diff --git a/sound/soc/sti/uniperif_player.c b/sound/soc/sti/uniperif_player.c
index ad54d4cf58ad..60ae31a303ab 100644
--- a/sound/soc/sti/uniperif_player.c
+++ b/sound/soc/sti/uniperif_player.c
@@ -6,8 +6,6 @@
  */
 
 #include <linux/clk.h>
-#include <linux/delay.h>
-#include <linux/io.h>
 #include <linux/mfd/syscon.h>
 
 #include <sound/asoundef.h>
@@ -55,25 +53,6 @@ static const struct snd_pcm_hardware uni_player_pcm_hw = {
 	.buffer_bytes_max = 256 * PAGE_SIZE
 };
 
-static inline int reset_player(struct uniperif *player)
-{
-	int count = 10;
-
-	if (player->ver < SND_ST_UNIPERIF_VERSION_UNI_PLR_TOP_1_0) {
-		while (GET_UNIPERIF_SOFT_RST_SOFT_RST(player) && count) {
-			udelay(5);
-			count--;
-		}
-	}
-
-	if (!count) {
-		dev_err(player->dev, "Failed to reset uniperif");
-		return -EIO;
-	}
-
-	return 0;
-}
-
 /*
  * uni_player_irq_handler
  * In case of error audio stream is stopped; stop action is protected via PCM
@@ -97,7 +76,7 @@ static irqreturn_t uni_player_irq_handler(int irq, void *dev_id)
 
 	/* Check for fifo error (underrun) */
 	if (unlikely(status & UNIPERIF_ITS_FIFO_ERROR_MASK(player))) {
-		dev_err(player->dev, "FIFO underflow error detected");
+		dev_err(player->dev, "FIFO underflow error detected\n");
 
 		/* Interrupt is just for information when underflow recovery */
 		if (player->underflow_enabled) {
@@ -119,7 +98,7 @@ static irqreturn_t uni_player_irq_handler(int irq, void *dev_id)
 
 	/* Check for dma error (overrun) */
 	if (unlikely(status & UNIPERIF_ITS_DMA_ERROR_MASK(player))) {
-		dev_err(player->dev, "DMA error detected");
+		dev_err(player->dev, "DMA error detected\n");
 
 		/* Disable interrupt so doesn't continually fire */
 		SET_UNIPERIF_ITM_BCLR_DMA_ERROR(player);
@@ -135,11 +114,14 @@ static irqreturn_t uni_player_irq_handler(int irq, void *dev_id)
 	/* Check for underflow recovery done */
 	if (unlikely(status & UNIPERIF_ITM_UNDERFLOW_REC_DONE_MASK(player))) {
 		if (!player->underflow_enabled) {
-			dev_err(player->dev, "unexpected Underflow recovering");
+			dev_err(player->dev,
+				"unexpected Underflow recovering\n");
 			return -EPERM;
 		}
 		/* Read the underflow recovery duration */
 		tmp = GET_UNIPERIF_STATUS_1_UNDERFLOW_DURATION(player);
+		dev_dbg(player->dev, "Underflow recovered (%d LR clocks max)\n",
+			tmp);
 
 		/* Clear the underflow recovery duration */
 		SET_UNIPERIF_BIT_CONTROL_CLR_UNDERFLOW_DURATION(player);
@@ -153,7 +135,7 @@ static irqreturn_t uni_player_irq_handler(int irq, void *dev_id)
 	/* Check if underflow recovery failed */
 	if (unlikely(status &
 		     UNIPERIF_ITM_UNDERFLOW_REC_FAILED_MASK(player))) {
-		dev_err(player->dev, "Underflow recovery failed");
+		dev_err(player->dev, "Underflow recovery failed\n");
 
 		/* Stop the player */
 		snd_pcm_stream_lock(player->substream);
@@ -336,7 +318,7 @@ static int uni_player_prepare_iec958(struct uniperif *player,
 
 	/* Oversampling must be multiple of 128 as iec958 frame is 32-bits */
 	if ((clk_div % 128) || (clk_div <= 0)) {
-		dev_err(player->dev, "%s: invalid clk_div %d",
+		dev_err(player->dev, "%s: invalid clk_div %d\n",
 			__func__, clk_div);
 		return -EINVAL;
 	}
@@ -359,7 +341,7 @@ static int uni_player_prepare_iec958(struct uniperif *player,
 		SET_UNIPERIF_I2S_FMT_DATA_SIZE_24(player);
 		break;
 	default:
-		dev_err(player->dev, "format not supported");
+		dev_err(player->dev, "format not supported\n");
 		return -EINVAL;
 	}
 
@@ -448,12 +430,12 @@ static int uni_player_prepare_pcm(struct uniperif *player,
 	 * for 16 bits must be a multiple of 64
 	 */
 	if ((slot_width == 32) && (clk_div % 128)) {
-		dev_err(player->dev, "%s: invalid clk_div", __func__);
+		dev_err(player->dev, "%s: invalid clk_div\n", __func__);
 		return -EINVAL;
 	}
 
 	if ((slot_width == 16) && (clk_div % 64)) {
-		dev_err(player->dev, "%s: invalid clk_div", __func__);
+		dev_err(player->dev, "%s: invalid clk_div\n", __func__);
 		return -EINVAL;
 	}
 
@@ -471,7 +453,7 @@ static int uni_player_prepare_pcm(struct uniperif *player,
 		SET_UNIPERIF_I2S_FMT_DATA_SIZE_16(player);
 		break;
 	default:
-		dev_err(player->dev, "subframe format not supported");
+		dev_err(player->dev, "subframe format not supported\n");
 		return -EINVAL;
 	}
 
@@ -491,7 +473,7 @@ static int uni_player_prepare_pcm(struct uniperif *player,
 		break;
 
 	default:
-		dev_err(player->dev, "format not supported");
+		dev_err(player->dev, "format not supported\n");
 		return -EINVAL;
 	}
 
@@ -504,7 +486,7 @@ static int uni_player_prepare_pcm(struct uniperif *player,
 	/* Number of channelsmust be even*/
 	if ((runtime->channels % 2) || (runtime->channels < 2) ||
 	    (runtime->channels > 10)) {
-		dev_err(player->dev, "%s: invalid nb of channels", __func__);
+		dev_err(player->dev, "%s: invalid nb of channels\n", __func__);
 		return -EINVAL;
 	}
 
@@ -762,7 +744,7 @@ static int uni_player_prepare(struct snd_pcm_substream *substream,
 
 	/* The player should be stopped */
 	if (player->state != UNIPERIF_STATE_STOPPED) {
-		dev_err(player->dev, "%s: invalid player state %d", __func__,
+		dev_err(player->dev, "%s: invalid player state %d\n", __func__,
 			player->state);
 		return -EINVAL;
 	}
@@ -791,7 +773,8 @@ static int uni_player_prepare(struct snd_pcm_substream *substream,
 	/* Trigger limit must be an even number */
 	if ((!trigger_limit % 2) || (trigger_limit != 1 && transfer_size % 2) ||
 	    (trigger_limit > UNIPERIF_CONFIG_DMA_TRIG_LIMIT_MASK(player))) {
-		dev_err(player->dev, "invalid trigger limit %d", trigger_limit);
+		dev_err(player->dev, "invalid trigger limit %d\n",
+			trigger_limit);
 		return -EINVAL;
 	}
 
@@ -812,7 +795,7 @@ static int uni_player_prepare(struct snd_pcm_substream *substream,
 		ret = uni_player_prepare_tdm(player, runtime);
 		break;
 	default:
-		dev_err(player->dev, "invalid player type");
+		dev_err(player->dev, "invalid player type\n");
 		return -EINVAL;
 	}
 
@@ -852,16 +835,14 @@ static int uni_player_prepare(struct snd_pcm_substream *substream,
 		SET_UNIPERIF_I2S_FMT_PADDING_SONY_MODE(player);
 		break;
 	default:
-		dev_err(player->dev, "format not supported");
+		dev_err(player->dev, "format not supported\n");
 		return -EINVAL;
 	}
 
 	SET_UNIPERIF_I2S_FMT_NO_OF_SAMPLES_TO_READ(player, 0);
 
-	/* Reset uniperipheral player */
-	SET_UNIPERIF_SOFT_RST_SOFT_RST(player);
 
-	return reset_player(player);
+	return sti_uniperiph_reset(player);
 }
 
 static int uni_player_start(struct uniperif *player)
@@ -870,13 +851,13 @@ static int uni_player_start(struct uniperif *player)
 
 	/* The player should be stopped */
 	if (player->state != UNIPERIF_STATE_STOPPED) {
-		dev_err(player->dev, "%s: invalid player state", __func__);
+		dev_err(player->dev, "%s: invalid player state\n", __func__);
 		return -EINVAL;
 	}
 
 	ret = clk_prepare_enable(player->clk);
 	if (ret) {
-		dev_err(player->dev, "%s: Failed to enable clock", __func__);
+		dev_err(player->dev, "%s: Failed to enable clock\n", __func__);
 		return ret;
 	}
 
@@ -893,10 +874,7 @@ static int uni_player_start(struct uniperif *player)
 		SET_UNIPERIF_ITM_BSET_UNDERFLOW_REC_FAILED(player);
 	}
 
-	/* Reset uniperipheral player */
-	SET_UNIPERIF_SOFT_RST_SOFT_RST(player);
-
-	ret = reset_player(player);
+	ret = sti_uniperiph_reset(player);
 	if (ret < 0) {
 		clk_disable_unprepare(player->clk);
 		return ret;
@@ -938,17 +916,14 @@ static int uni_player_stop(struct uniperif *player)
 
 	/* The player should not be in stopped state */
 	if (player->state == UNIPERIF_STATE_STOPPED) {
-		dev_err(player->dev, "%s: invalid player state", __func__);
+		dev_err(player->dev, "%s: invalid player state\n", __func__);
 		return -EINVAL;
 	}
 
 	/* Turn the player off */
 	SET_UNIPERIF_CTRL_OPERATION_OFF(player);
 
-	/* Soft reset the player */
-	SET_UNIPERIF_SOFT_RST_SOFT_RST(player);
-
-	ret = reset_player(player);
+	ret = sti_uniperiph_reset(player);
 	if (ret < 0)
 		return ret;
 
@@ -973,7 +948,7 @@ int uni_player_resume(struct uniperif *player)
 		ret = regmap_field_write(player->clk_sel, 1);
 		if (ret) {
 			dev_err(player->dev,
-				"%s: Failed to select freq synth clock",
+				"%s: Failed to select freq synth clock\n",
 				__func__);
 			return ret;
 		}
@@ -1070,7 +1045,7 @@ int uni_player_init(struct platform_device *pdev,
 	ret = uni_player_parse_dt_audio_glue(pdev, player);
 
 	if (ret < 0) {
-		dev_err(player->dev, "Failed to parse DeviceTree");
+		dev_err(player->dev, "Failed to parse DeviceTree\n");
 		return ret;
 	}
 
@@ -1085,15 +1060,17 @@ int uni_player_init(struct platform_device *pdev,
 
 	/* Get uniperif resource */
 	player->clk = of_clk_get(pdev->dev.of_node, 0);
-	if (IS_ERR(player->clk))
+	if (IS_ERR(player->clk)) {
+		dev_err(player->dev, "Failed to get clock\n");
 		ret = PTR_ERR(player->clk);
+	}
 
 	/* Select the frequency synthesizer clock */
 	if (player->clk_sel) {
 		ret = regmap_field_write(player->clk_sel, 1);
 		if (ret) {
 			dev_err(player->dev,
-				"%s: Failed to select freq synth clock",
+				"%s: Failed to select freq synth clock\n",
 				__func__);
 			return ret;
 		}
@@ -1105,7 +1082,7 @@ int uni_player_init(struct platform_device *pdev,
 		ret = regmap_field_write(player->valid_sel, player->id);
 		if (ret) {
 			dev_err(player->dev,
-				"%s: unable to connect to tdm bus", __func__);
+				"%s: unable to connect to tdm bus\n", __func__);
 			return ret;
 		}
 	}
@@ -1113,8 +1090,10 @@ int uni_player_init(struct platform_device *pdev,
 	ret = devm_request_irq(&pdev->dev, player->irq,
 			       uni_player_irq_handler, IRQF_SHARED,
 			       dev_name(&pdev->dev), player);
-	if (ret < 0)
+	if (ret < 0) {
+		dev_err(player->dev, "unable to request IRQ %d\n", player->irq);
 		return ret;
+	}
 
 	mutex_init(&player->ctrl_lock);
 
diff --git a/sound/soc/sti/uniperif_reader.c b/sound/soc/sti/uniperif_reader.c
index 0e1c3ee56675..5992c6ab3833 100644
--- a/sound/soc/sti/uniperif_reader.c
+++ b/sound/soc/sti/uniperif_reader.c
@@ -5,10 +5,6 @@
  * License terms:  GNU General Public License (GPL), version 2
  */
 
-#include <linux/clk.h>
-#include <linux/delay.h>
-#include <linux/io.h>
-
 #include <sound/soc.h>
 
 #include "uniperif.h"
@@ -52,7 +48,7 @@ static irqreturn_t uni_reader_irq_handler(int irq, void *dev_id)
 
 	if (reader->state == UNIPERIF_STATE_STOPPED) {
 		/* Unexpected IRQ: do nothing */
-		dev_warn(reader->dev, "unexpected IRQ ");
+		dev_warn(reader->dev, "unexpected IRQ\n");
 		return IRQ_HANDLED;
 	}
 
@@ -62,7 +58,7 @@ static irqreturn_t uni_reader_irq_handler(int irq, void *dev_id)
 
 	/* Check for fifo overflow error */
 	if (unlikely(status & UNIPERIF_ITS_FIFO_ERROR_MASK(reader))) {
-		dev_err(reader->dev, "FIFO error detected");
+		dev_err(reader->dev, "FIFO error detected\n");
 
 		snd_pcm_stream_lock(reader->substream);
 		snd_pcm_stop(reader->substream, SNDRV_PCM_STATE_XRUN);
@@ -105,7 +101,7 @@ static int uni_reader_prepare_pcm(struct snd_pcm_runtime *runtime,
 		SET_UNIPERIF_I2S_FMT_DATA_SIZE_16(reader);
 		break;
 	default:
-		dev_err(reader->dev, "subframe format not supported");
+		dev_err(reader->dev, "subframe format not supported\n");
 		return -EINVAL;
 	}
 
@@ -125,14 +121,14 @@ static int uni_reader_prepare_pcm(struct snd_pcm_runtime *runtime,
 		break;
 
 	default:
-		dev_err(reader->dev, "format not supported");
+		dev_err(reader->dev, "format not supported\n");
 		return -EINVAL;
 	}
 
 	/* Number of channels must be even */
 	if ((runtime->channels % 2) || (runtime->channels < 2) ||
 	    (runtime->channels > 10)) {
-		dev_err(reader->dev, "%s: invalid nb of channels", __func__);
+		dev_err(reader->dev, "%s: invalid nb of channels\n", __func__);
 		return -EINVAL;
 	}
 
@@ -186,11 +182,10 @@ static int uni_reader_prepare(struct snd_pcm_substream *substream,
 	struct uniperif *reader = priv->dai_data.uni;
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	int transfer_size, trigger_limit, ret;
-	int count = 10;
 
 	/* The reader should be stopped */
 	if (reader->state != UNIPERIF_STATE_STOPPED) {
-		dev_err(reader->dev, "%s: invalid reader state %d", __func__,
+		dev_err(reader->dev, "%s: invalid reader state %d\n", __func__,
 			reader->state);
 		return -EINVAL;
 	}
@@ -219,7 +214,8 @@ static int uni_reader_prepare(struct snd_pcm_substream *substream,
 	if ((!trigger_limit % 2) ||
 	    (trigger_limit != 1 && transfer_size % 2) ||
 	    (trigger_limit > UNIPERIF_CONFIG_DMA_TRIG_LIMIT_MASK(reader))) {
-		dev_err(reader->dev, "invalid trigger limit %d", trigger_limit);
+		dev_err(reader->dev, "invalid trigger limit %d\n",
+			trigger_limit);
 		return -EINVAL;
 	}
 
@@ -246,7 +242,7 @@ static int uni_reader_prepare(struct snd_pcm_substream *substream,
 		SET_UNIPERIF_I2S_FMT_PADDING_SONY_MODE(reader);
 		break;
 	default:
-		dev_err(reader->dev, "format not supported");
+		dev_err(reader->dev, "format not supported\n");
 		return -EINVAL;
 	}
 
@@ -287,25 +283,14 @@ static int uni_reader_prepare(struct snd_pcm_substream *substream,
 	}
 
 	/* Reset uniperipheral reader */
-	SET_UNIPERIF_SOFT_RST_SOFT_RST(reader);
-
-	while (GET_UNIPERIF_SOFT_RST_SOFT_RST(reader)) {
-		udelay(5);
-		count--;
-	}
-	if (!count) {
-		dev_err(reader->dev, "Failed to reset uniperif");
-		return -EIO;
-	}
-
-	return 0;
+	return sti_uniperiph_reset(reader);
 }
 
 static int uni_reader_start(struct uniperif *reader)
 {
 	/* The reader should be stopped */
 	if (reader->state != UNIPERIF_STATE_STOPPED) {
-		dev_err(reader->dev, "%s: invalid reader state", __func__);
+		dev_err(reader->dev, "%s: invalid reader state\n", __func__);
 		return -EINVAL;
 	}
 
@@ -325,7 +310,7 @@ static int uni_reader_stop(struct uniperif *reader)
 {
 	/* The reader should not be in stopped state */
 	if (reader->state == UNIPERIF_STATE_STOPPED) {
-		dev_err(reader->dev, "%s: invalid reader state", __func__);
+		dev_err(reader->dev, "%s: invalid reader state\n", __func__);
 		return -EINVAL;
 	}
 
@@ -423,7 +408,7 @@ int uni_reader_init(struct platform_device *pdev,
 			       uni_reader_irq_handler, IRQF_SHARED,
 			       dev_name(&pdev->dev), reader);
 	if (ret < 0) {
-		dev_err(&pdev->dev, "Failed to request IRQ");
+		dev_err(&pdev->dev, "Failed to request IRQ\n");
 		return -EBUSY;
 	}
 
diff --git a/sound/soc/sunxi/Kconfig b/sound/soc/sunxi/Kconfig
index dd2368297fd3..6c344e16aca4 100644
--- a/sound/soc/sunxi/Kconfig
+++ b/sound/soc/sunxi/Kconfig
@@ -9,6 +9,14 @@ config SND_SUN4I_CODEC
 	  Select Y or M to add support for the Codec embedded in the Allwinner
 	  A10 and affiliated SoCs.
 
+config SND_SUN8I_CODEC_ANALOG
+	tristate "Allwinner sun8i Codec Analog Controls Support"
+	depends on MACH_SUN8I || COMPILE_TEST
+	select REGMAP
+	help
+	  Say Y or M if you want to add support for the analog controls for
+	  the codec embedded in newer Allwinner SoCs.
+
 config SND_SUN4I_I2S
 	tristate "Allwinner A10 I2S Support"
 	select SND_SOC_GENERIC_DMAENGINE_PCM
diff --git a/sound/soc/sunxi/Makefile b/sound/soc/sunxi/Makefile
index 604c7b842837..241c0df9ca0c 100644
--- a/sound/soc/sunxi/Makefile
+++ b/sound/soc/sunxi/Makefile
@@ -1,3 +1,4 @@
 obj-$(CONFIG_SND_SUN4I_CODEC) += sun4i-codec.o
 obj-$(CONFIG_SND_SUN4I_I2S) += sun4i-i2s.o
 obj-$(CONFIG_SND_SUN4I_SPDIF) += sun4i-spdif.o
+obj-$(CONFIG_SND_SUN8I_CODEC_ANALOG) += sun8i-codec-analog.o
diff --git a/sound/soc/sunxi/sun4i-codec.c b/sound/soc/sunxi/sun4i-codec.c
index 56ed9472e89f..848af01692a0 100644
--- a/sound/soc/sunxi/sun4i-codec.c
+++ b/sound/soc/sunxi/sun4i-codec.c
@@ -3,6 +3,7 @@
  * Copyright 2014 Jon Smirl <jonsmirl@gmail.com>
  * Copyright 2015 Maxime Ripard <maxime.ripard@free-electrons.com>
  * Copyright 2015 Adam Sampson <ats@offog.org>
+ * Copyright 2016 Chen-Yu Tsai <wens@csie.org>
  *
  * Based on the Allwinner SDK driver, released under the GPL.
  *
@@ -24,10 +25,12 @@
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/of.h>
-#include <linux/of_platform.h>
 #include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_platform.h>
 #include <linux/clk.h>
 #include <linux/regmap.h>
+#include <linux/reset.h>
 #include <linux/gpio/consumer.h>
 
 #include <sound/core.h>
@@ -38,7 +41,7 @@
 #include <sound/initval.h>
 #include <sound/dmaengine_pcm.h>
 
-/* Codec DAC register offsets and bit fields */
+/* Codec DAC digital controls and FIFO registers */
 #define SUN4I_CODEC_DAC_DPC			(0x00)
 #define SUN4I_CODEC_DAC_DPC_EN_DA			(31)
 #define SUN4I_CODEC_DAC_DPC_DVOL			(12)
@@ -55,6 +58,8 @@
 #define SUN4I_CODEC_DAC_FIFOC_FIFO_FLUSH		(0)
 #define SUN4I_CODEC_DAC_FIFOS			(0x08)
 #define SUN4I_CODEC_DAC_TXDATA			(0x0c)
+
+/* Codec DAC side analog signal controls */
 #define SUN4I_CODEC_DAC_ACTL			(0x10)
 #define SUN4I_CODEC_DAC_ACTL_DACAENR			(31)
 #define SUN4I_CODEC_DAC_ACTL_DACAENL			(30)
@@ -69,7 +74,7 @@
 #define SUN4I_CODEC_DAC_TUNE			(0x14)
 #define SUN4I_CODEC_DAC_DEBUG			(0x18)
 
-/* Codec ADC register offsets and bit fields */
+/* Codec ADC digital controls and FIFO registers */
 #define SUN4I_CODEC_ADC_FIFOC			(0x1c)
 #define SUN4I_CODEC_ADC_FIFOC_ADC_FS			(29)
 #define SUN4I_CODEC_ADC_FIFOC_EN_AD			(28)
@@ -81,6 +86,8 @@
 #define SUN4I_CODEC_ADC_FIFOC_FIFO_FLUSH		(0)
 #define SUN4I_CODEC_ADC_FIFOS			(0x20)
 #define SUN4I_CODEC_ADC_RXDATA			(0x24)
+
+/* Codec ADC side analog signal controls */
 #define SUN4I_CODEC_ADC_ACTL			(0x28)
 #define SUN4I_CODEC_ADC_ACTL_ADC_R_EN			(31)
 #define SUN4I_CODEC_ADC_ACTL_ADC_L_EN			(30)
@@ -93,19 +100,141 @@
 #define SUN4I_CODEC_ADC_ACTL_DDE			(3)
 #define SUN4I_CODEC_ADC_DEBUG			(0x2c)
 
-/* Other various ADC registers */
+/* FIFO counters */
 #define SUN4I_CODEC_DAC_TXCNT			(0x30)
 #define SUN4I_CODEC_ADC_RXCNT			(0x34)
+
+/* Calibration register (sun7i only) */
 #define SUN7I_CODEC_AC_DAC_CAL			(0x38)
+
+/* Microphone controls (sun7i only) */
 #define SUN7I_CODEC_AC_MIC_PHONE_CAL		(0x3c)
 
+/*
+ * sun6i specific registers
+ *
+ * sun6i shares the same digital control and FIFO registers as sun4i,
+ * but only the DAC digital controls are at the same offset. The others
+ * have been moved around to accommodate extra analog controls.
+ */
+
+/* Codec DAC digital controls and FIFO registers */
+#define SUN6I_CODEC_ADC_FIFOC			(0x10)
+#define SUN6I_CODEC_ADC_FIFOC_EN_AD			(28)
+#define SUN6I_CODEC_ADC_FIFOS			(0x14)
+#define SUN6I_CODEC_ADC_RXDATA			(0x18)
+
+/* Output mixer and gain controls */
+#define SUN6I_CODEC_OM_DACA_CTRL		(0x20)
+#define SUN6I_CODEC_OM_DACA_CTRL_DACAREN		(31)
+#define SUN6I_CODEC_OM_DACA_CTRL_DACALEN		(30)
+#define SUN6I_CODEC_OM_DACA_CTRL_RMIXEN			(29)
+#define SUN6I_CODEC_OM_DACA_CTRL_LMIXEN			(28)
+#define SUN6I_CODEC_OM_DACA_CTRL_RMIX_MIC1		(23)
+#define SUN6I_CODEC_OM_DACA_CTRL_RMIX_MIC2		(22)
+#define SUN6I_CODEC_OM_DACA_CTRL_RMIX_PHONE		(21)
+#define SUN6I_CODEC_OM_DACA_CTRL_RMIX_PHONEP		(20)
+#define SUN6I_CODEC_OM_DACA_CTRL_RMIX_LINEINR		(19)
+#define SUN6I_CODEC_OM_DACA_CTRL_RMIX_DACR		(18)
+#define SUN6I_CODEC_OM_DACA_CTRL_RMIX_DACL		(17)
+#define SUN6I_CODEC_OM_DACA_CTRL_LMIX_MIC1		(16)
+#define SUN6I_CODEC_OM_DACA_CTRL_LMIX_MIC2		(15)
+#define SUN6I_CODEC_OM_DACA_CTRL_LMIX_PHONE		(14)
+#define SUN6I_CODEC_OM_DACA_CTRL_LMIX_PHONEN		(13)
+#define SUN6I_CODEC_OM_DACA_CTRL_LMIX_LINEINL		(12)
+#define SUN6I_CODEC_OM_DACA_CTRL_LMIX_DACL		(11)
+#define SUN6I_CODEC_OM_DACA_CTRL_LMIX_DACR		(10)
+#define SUN6I_CODEC_OM_DACA_CTRL_RHPIS			(9)
+#define SUN6I_CODEC_OM_DACA_CTRL_LHPIS			(8)
+#define SUN6I_CODEC_OM_DACA_CTRL_RHPPAMUTE		(7)
+#define SUN6I_CODEC_OM_DACA_CTRL_LHPPAMUTE		(6)
+#define SUN6I_CODEC_OM_DACA_CTRL_HPVOL			(0)
+#define SUN6I_CODEC_OM_PA_CTRL			(0x24)
+#define SUN6I_CODEC_OM_PA_CTRL_HPPAEN			(31)
+#define SUN6I_CODEC_OM_PA_CTRL_HPCOM_CTL		(29)
+#define SUN6I_CODEC_OM_PA_CTRL_COMPTEN			(28)
+#define SUN6I_CODEC_OM_PA_CTRL_MIC1G			(15)
+#define SUN6I_CODEC_OM_PA_CTRL_MIC2G			(12)
+#define SUN6I_CODEC_OM_PA_CTRL_LINEING			(9)
+#define SUN6I_CODEC_OM_PA_CTRL_PHONEG			(6)
+#define SUN6I_CODEC_OM_PA_CTRL_PHONEPG			(3)
+#define SUN6I_CODEC_OM_PA_CTRL_PHONENG			(0)
+
+/* Microphone, line out and phone out controls */
+#define SUN6I_CODEC_MIC_CTRL			(0x28)
+#define SUN6I_CODEC_MIC_CTRL_HBIASEN			(31)
+#define SUN6I_CODEC_MIC_CTRL_MBIASEN			(30)
+#define SUN6I_CODEC_MIC_CTRL_MIC1AMPEN			(28)
+#define SUN6I_CODEC_MIC_CTRL_MIC1BOOST			(25)
+#define SUN6I_CODEC_MIC_CTRL_MIC2AMPEN			(24)
+#define SUN6I_CODEC_MIC_CTRL_MIC2BOOST			(21)
+#define SUN6I_CODEC_MIC_CTRL_MIC2SLT			(20)
+#define SUN6I_CODEC_MIC_CTRL_LINEOUTLEN			(19)
+#define SUN6I_CODEC_MIC_CTRL_LINEOUTREN			(18)
+#define SUN6I_CODEC_MIC_CTRL_LINEOUTLSRC		(17)
+#define SUN6I_CODEC_MIC_CTRL_LINEOUTRSRC		(16)
+#define SUN6I_CODEC_MIC_CTRL_LINEOUTVC			(11)
+#define SUN6I_CODEC_MIC_CTRL_PHONEPREG			(8)
+
+/* ADC mixer controls */
+#define SUN6I_CODEC_ADC_ACTL			(0x2c)
+#define SUN6I_CODEC_ADC_ACTL_ADCREN			(31)
+#define SUN6I_CODEC_ADC_ACTL_ADCLEN			(30)
+#define SUN6I_CODEC_ADC_ACTL_ADCRG			(27)
+#define SUN6I_CODEC_ADC_ACTL_ADCLG			(24)
+#define SUN6I_CODEC_ADC_ACTL_RADCMIX_MIC1		(13)
+#define SUN6I_CODEC_ADC_ACTL_RADCMIX_MIC2		(12)
+#define SUN6I_CODEC_ADC_ACTL_RADCMIX_PHONE		(11)
+#define SUN6I_CODEC_ADC_ACTL_RADCMIX_PHONEP		(10)
+#define SUN6I_CODEC_ADC_ACTL_RADCMIX_LINEINR		(9)
+#define SUN6I_CODEC_ADC_ACTL_RADCMIX_OMIXR		(8)
+#define SUN6I_CODEC_ADC_ACTL_RADCMIX_OMIXL		(7)
+#define SUN6I_CODEC_ADC_ACTL_LADCMIX_MIC1		(6)
+#define SUN6I_CODEC_ADC_ACTL_LADCMIX_MIC2		(5)
+#define SUN6I_CODEC_ADC_ACTL_LADCMIX_PHONE		(4)
+#define SUN6I_CODEC_ADC_ACTL_LADCMIX_PHONEN		(3)
+#define SUN6I_CODEC_ADC_ACTL_LADCMIX_LINEINL		(2)
+#define SUN6I_CODEC_ADC_ACTL_LADCMIX_OMIXL		(1)
+#define SUN6I_CODEC_ADC_ACTL_LADCMIX_OMIXR		(0)
+
+/* Analog performance tuning controls */
+#define SUN6I_CODEC_ADDA_TUNE			(0x30)
+
+/* Calibration controls */
+#define SUN6I_CODEC_CALIBRATION			(0x34)
+
+/* FIFO counters */
+#define SUN6I_CODEC_DAC_TXCNT			(0x40)
+#define SUN6I_CODEC_ADC_RXCNT			(0x44)
+
+/* headset jack detection and button support registers */
+#define SUN6I_CODEC_HMIC_CTL			(0x50)
+#define SUN6I_CODEC_HMIC_DATA			(0x54)
+
+/* TODO sun6i DAP (Digital Audio Processing) bits */
+
+/* FIFO counters moved on A23 */
+#define SUN8I_A23_CODEC_DAC_TXCNT		(0x1c)
+#define SUN8I_A23_CODEC_ADC_RXCNT		(0x20)
+
+/* TX FIFO moved on H3 */
+#define SUN8I_H3_CODEC_DAC_TXDATA		(0x20)
+#define SUN8I_H3_CODEC_DAC_DBG			(0x48)
+#define SUN8I_H3_CODEC_ADC_DBG			(0x4c)
+
+/* TODO H3 DAP (Digital Audio Processing) bits */
+
 struct sun4i_codec {
 	struct device	*dev;
 	struct regmap	*regmap;
 	struct clk	*clk_apb;
 	struct clk	*clk_module;
+	struct reset_control *rst;
 	struct gpio_desc *gpio_pa;
 
+	/* ADC_FIFOC register is at different offset on different SoCs */
+	struct regmap_field *reg_adc_fifoc;
+
 	struct snd_dmaengine_dai_dma_data	capture_dma_data;
 	struct snd_dmaengine_dai_dma_data	playback_dma_data;
 };
@@ -134,16 +263,16 @@ static void sun4i_codec_stop_playback(struct sun4i_codec *scodec)
 static void sun4i_codec_start_capture(struct sun4i_codec *scodec)
 {
 	/* Enable ADC DRQ */
-	regmap_update_bits(scodec->regmap, SUN4I_CODEC_ADC_FIFOC,
-			   BIT(SUN4I_CODEC_ADC_FIFOC_ADC_DRQ_EN),
-			   BIT(SUN4I_CODEC_ADC_FIFOC_ADC_DRQ_EN));
+	regmap_field_update_bits(scodec->reg_adc_fifoc,
+				 BIT(SUN4I_CODEC_ADC_FIFOC_ADC_DRQ_EN),
+				 BIT(SUN4I_CODEC_ADC_FIFOC_ADC_DRQ_EN));
 }
 
 static void sun4i_codec_stop_capture(struct sun4i_codec *scodec)
 {
 	/* Disable ADC DRQ */
-	regmap_update_bits(scodec->regmap, SUN4I_CODEC_ADC_FIFOC,
-			   BIT(SUN4I_CODEC_ADC_FIFOC_ADC_DRQ_EN), 0);
+	regmap_field_update_bits(scodec->reg_adc_fifoc,
+				 BIT(SUN4I_CODEC_ADC_FIFOC_ADC_DRQ_EN), 0);
 }
 
 static int sun4i_codec_trigger(struct snd_pcm_substream *substream, int cmd,
@@ -186,24 +315,29 @@ static int sun4i_codec_prepare_capture(struct snd_pcm_substream *substream,
 
 
 	/* Flush RX FIFO */
-	regmap_update_bits(scodec->regmap, SUN4I_CODEC_ADC_FIFOC,
-			   BIT(SUN4I_CODEC_ADC_FIFOC_FIFO_FLUSH),
-			   BIT(SUN4I_CODEC_ADC_FIFOC_FIFO_FLUSH));
+	regmap_field_update_bits(scodec->reg_adc_fifoc,
+				 BIT(SUN4I_CODEC_ADC_FIFOC_FIFO_FLUSH),
+				 BIT(SUN4I_CODEC_ADC_FIFOC_FIFO_FLUSH));
 
 
 	/* Set RX FIFO trigger level */
-	regmap_update_bits(scodec->regmap, SUN4I_CODEC_ADC_FIFOC,
-			   0xf << SUN4I_CODEC_ADC_FIFOC_RX_TRIG_LEVEL,
-			   0x7 << SUN4I_CODEC_ADC_FIFOC_RX_TRIG_LEVEL);
+	regmap_field_update_bits(scodec->reg_adc_fifoc,
+				 0xf << SUN4I_CODEC_ADC_FIFOC_RX_TRIG_LEVEL,
+				 0x7 << SUN4I_CODEC_ADC_FIFOC_RX_TRIG_LEVEL);
 
 	/*
 	 * FIXME: Undocumented in the datasheet, but
 	 *        Allwinner's code mentions that it is related
 	 *        related to microphone gain
 	 */
-	regmap_update_bits(scodec->regmap, SUN4I_CODEC_ADC_ACTL,
-			   0x3 << 25,
-			   0x1 << 25);
+	if (of_device_is_compatible(scodec->dev->of_node,
+				    "allwinner,sun4i-a10-codec") ||
+	    of_device_is_compatible(scodec->dev->of_node,
+				    "allwinner,sun7i-a20-codec")) {
+		regmap_update_bits(scodec->regmap, SUN4I_CODEC_ADC_ACTL,
+				   0x3 << 25,
+				   0x1 << 25);
+	}
 
 	if (of_device_is_compatible(scodec->dev->of_node,
 				    "allwinner,sun7i-a20-codec"))
@@ -213,9 +347,9 @@ static int sun4i_codec_prepare_capture(struct snd_pcm_substream *substream,
 				   0x1 << 8);
 
 	/* Fill most significant bits with valid data MSB */
-	regmap_update_bits(scodec->regmap, SUN4I_CODEC_ADC_FIFOC,
-			   BIT(SUN4I_CODEC_ADC_FIFOC_RX_FIFO_MODE),
-			   BIT(SUN4I_CODEC_ADC_FIFOC_RX_FIFO_MODE));
+	regmap_field_update_bits(scodec->reg_adc_fifoc,
+				 BIT(SUN4I_CODEC_ADC_FIFOC_RX_FIFO_MODE),
+				 BIT(SUN4I_CODEC_ADC_FIFOC_RX_FIFO_MODE));
 
 	return 0;
 }
@@ -342,18 +476,19 @@ static int sun4i_codec_hw_params_capture(struct sun4i_codec *scodec,
 					 unsigned int hwrate)
 {
 	/* Set ADC sample rate */
-	regmap_update_bits(scodec->regmap, SUN4I_CODEC_ADC_FIFOC,
-			   7 << SUN4I_CODEC_ADC_FIFOC_ADC_FS,
-			   hwrate << SUN4I_CODEC_ADC_FIFOC_ADC_FS);
+	regmap_field_update_bits(scodec->reg_adc_fifoc,
+				 7 << SUN4I_CODEC_ADC_FIFOC_ADC_FS,
+				 hwrate << SUN4I_CODEC_ADC_FIFOC_ADC_FS);
 
 	/* Set the number of channels we want to use */
 	if (params_channels(params) == 1)
-		regmap_update_bits(scodec->regmap, SUN4I_CODEC_ADC_FIFOC,
-				   BIT(SUN4I_CODEC_ADC_FIFOC_MONO_EN),
-				   BIT(SUN4I_CODEC_ADC_FIFOC_MONO_EN));
+		regmap_field_update_bits(scodec->reg_adc_fifoc,
+					 BIT(SUN4I_CODEC_ADC_FIFOC_MONO_EN),
+					 BIT(SUN4I_CODEC_ADC_FIFOC_MONO_EN));
 	else
-		regmap_update_bits(scodec->regmap, SUN4I_CODEC_ADC_FIFOC,
-				   BIT(SUN4I_CODEC_ADC_FIFOC_MONO_EN), 0);
+		regmap_field_update_bits(scodec->reg_adc_fifoc,
+					 BIT(SUN4I_CODEC_ADC_FIFOC_MONO_EN),
+					 0);
 
 	return 0;
 }
@@ -502,7 +637,7 @@ static struct snd_soc_dai_driver sun4i_codec_dai = {
 	},
 };
 
-/*** Codec ***/
+/*** sun4i Codec ***/
 static const struct snd_kcontrol_new sun4i_codec_pa_mute =
 	SOC_DAPM_SINGLE("Switch", SUN4I_CODEC_DAC_ACTL,
 			SUN4I_CODEC_DAC_ACTL_PA_MUTE, 1, 0);
@@ -638,6 +773,337 @@ static struct snd_soc_codec_driver sun4i_codec_codec = {
 	},
 };
 
+/*** sun6i Codec ***/
+
+/* mixer controls */
+static const struct snd_kcontrol_new sun6i_codec_mixer_controls[] = {
+	SOC_DAPM_DOUBLE("DAC Playback Switch",
+			SUN6I_CODEC_OM_DACA_CTRL,
+			SUN6I_CODEC_OM_DACA_CTRL_LMIX_DACL,
+			SUN6I_CODEC_OM_DACA_CTRL_RMIX_DACR, 1, 0),
+	SOC_DAPM_DOUBLE("DAC Reversed Playback Switch",
+			SUN6I_CODEC_OM_DACA_CTRL,
+			SUN6I_CODEC_OM_DACA_CTRL_LMIX_DACR,
+			SUN6I_CODEC_OM_DACA_CTRL_RMIX_DACL, 1, 0),
+	SOC_DAPM_DOUBLE("Line In Playback Switch",
+			SUN6I_CODEC_OM_DACA_CTRL,
+			SUN6I_CODEC_OM_DACA_CTRL_LMIX_LINEINL,
+			SUN6I_CODEC_OM_DACA_CTRL_RMIX_LINEINR, 1, 0),
+	SOC_DAPM_DOUBLE("Mic1 Playback Switch",
+			SUN6I_CODEC_OM_DACA_CTRL,
+			SUN6I_CODEC_OM_DACA_CTRL_LMIX_MIC1,
+			SUN6I_CODEC_OM_DACA_CTRL_RMIX_MIC1, 1, 0),
+	SOC_DAPM_DOUBLE("Mic2 Playback Switch",
+			SUN6I_CODEC_OM_DACA_CTRL,
+			SUN6I_CODEC_OM_DACA_CTRL_LMIX_MIC2,
+			SUN6I_CODEC_OM_DACA_CTRL_RMIX_MIC2, 1, 0),
+};
+
+/* ADC mixer controls */
+static const struct snd_kcontrol_new sun6i_codec_adc_mixer_controls[] = {
+	SOC_DAPM_DOUBLE("Mixer Capture Switch",
+			SUN6I_CODEC_ADC_ACTL,
+			SUN6I_CODEC_ADC_ACTL_LADCMIX_OMIXL,
+			SUN6I_CODEC_ADC_ACTL_RADCMIX_OMIXR, 1, 0),
+	SOC_DAPM_DOUBLE("Mixer Reversed Capture Switch",
+			SUN6I_CODEC_ADC_ACTL,
+			SUN6I_CODEC_ADC_ACTL_LADCMIX_OMIXR,
+			SUN6I_CODEC_ADC_ACTL_RADCMIX_OMIXL, 1, 0),
+	SOC_DAPM_DOUBLE("Line In Capture Switch",
+			SUN6I_CODEC_ADC_ACTL,
+			SUN6I_CODEC_ADC_ACTL_LADCMIX_LINEINL,
+			SUN6I_CODEC_ADC_ACTL_RADCMIX_LINEINR, 1, 0),
+	SOC_DAPM_DOUBLE("Mic1 Capture Switch",
+			SUN6I_CODEC_ADC_ACTL,
+			SUN6I_CODEC_ADC_ACTL_LADCMIX_MIC1,
+			SUN6I_CODEC_ADC_ACTL_RADCMIX_MIC1, 1, 0),
+	SOC_DAPM_DOUBLE("Mic2 Capture Switch",
+			SUN6I_CODEC_ADC_ACTL,
+			SUN6I_CODEC_ADC_ACTL_LADCMIX_MIC2,
+			SUN6I_CODEC_ADC_ACTL_RADCMIX_MIC2, 1, 0),
+};
+
+/* headphone controls */
+static const char * const sun6i_codec_hp_src_enum_text[] = {
+	"DAC", "Mixer",
+};
+
+static SOC_ENUM_DOUBLE_DECL(sun6i_codec_hp_src_enum,
+			    SUN6I_CODEC_OM_DACA_CTRL,
+			    SUN6I_CODEC_OM_DACA_CTRL_LHPIS,
+			    SUN6I_CODEC_OM_DACA_CTRL_RHPIS,
+			    sun6i_codec_hp_src_enum_text);
+
+static const struct snd_kcontrol_new sun6i_codec_hp_src[] = {
+	SOC_DAPM_ENUM("Headphone Source Playback Route",
+		      sun6i_codec_hp_src_enum),
+};
+
+/* microphone controls */
+static const char * const sun6i_codec_mic2_src_enum_text[] = {
+	"Mic2", "Mic3",
+};
+
+static SOC_ENUM_SINGLE_DECL(sun6i_codec_mic2_src_enum,
+			    SUN6I_CODEC_MIC_CTRL,
+			    SUN6I_CODEC_MIC_CTRL_MIC2SLT,
+			    sun6i_codec_mic2_src_enum_text);
+
+static const struct snd_kcontrol_new sun6i_codec_mic2_src[] = {
+	SOC_DAPM_ENUM("Mic2 Amplifier Source Route",
+		      sun6i_codec_mic2_src_enum),
+};
+
+/* line out controls */
+static const char * const sun6i_codec_lineout_src_enum_text[] = {
+	"Stereo", "Mono Differential",
+};
+
+static SOC_ENUM_DOUBLE_DECL(sun6i_codec_lineout_src_enum,
+			    SUN6I_CODEC_MIC_CTRL,
+			    SUN6I_CODEC_MIC_CTRL_LINEOUTLSRC,
+			    SUN6I_CODEC_MIC_CTRL_LINEOUTRSRC,
+			    sun6i_codec_lineout_src_enum_text);
+
+static const struct snd_kcontrol_new sun6i_codec_lineout_src[] = {
+	SOC_DAPM_ENUM("Line Out Source Playback Route",
+		      sun6i_codec_lineout_src_enum),
+};
+
+/* volume / mute controls */
+static const DECLARE_TLV_DB_SCALE(sun6i_codec_dvol_scale, -7308, 116, 0);
+static const DECLARE_TLV_DB_SCALE(sun6i_codec_hp_vol_scale, -6300, 100, 1);
+static const DECLARE_TLV_DB_SCALE(sun6i_codec_out_mixer_pregain_scale,
+				  -450, 150, 0);
+static const DECLARE_TLV_DB_RANGE(sun6i_codec_lineout_vol_scale,
+	0, 1, TLV_DB_SCALE_ITEM(TLV_DB_GAIN_MUTE, 0, 1),
+	2, 31, TLV_DB_SCALE_ITEM(-4350, 150, 0),
+);
+static const DECLARE_TLV_DB_RANGE(sun6i_codec_mic_gain_scale,
+	0, 0, TLV_DB_SCALE_ITEM(0, 0, 0),
+	1, 7, TLV_DB_SCALE_ITEM(2400, 300, 0),
+);
+
+static const struct snd_kcontrol_new sun6i_codec_codec_widgets[] = {
+	SOC_SINGLE_TLV("DAC Playback Volume", SUN4I_CODEC_DAC_DPC,
+		       SUN4I_CODEC_DAC_DPC_DVOL, 0x3f, 1,
+		       sun6i_codec_dvol_scale),
+	SOC_SINGLE_TLV("Headphone Playback Volume",
+		       SUN6I_CODEC_OM_DACA_CTRL,
+		       SUN6I_CODEC_OM_DACA_CTRL_HPVOL, 0x3f, 0,
+		       sun6i_codec_hp_vol_scale),
+	SOC_SINGLE_TLV("Line Out Playback Volume",
+		       SUN6I_CODEC_MIC_CTRL,
+		       SUN6I_CODEC_MIC_CTRL_LINEOUTVC, 0x1f, 0,
+		       sun6i_codec_lineout_vol_scale),
+	SOC_DOUBLE("Headphone Playback Switch",
+		   SUN6I_CODEC_OM_DACA_CTRL,
+		   SUN6I_CODEC_OM_DACA_CTRL_LHPPAMUTE,
+		   SUN6I_CODEC_OM_DACA_CTRL_RHPPAMUTE, 1, 0),
+	SOC_DOUBLE("Line Out Playback Switch",
+		   SUN6I_CODEC_MIC_CTRL,
+		   SUN6I_CODEC_MIC_CTRL_LINEOUTLEN,
+		   SUN6I_CODEC_MIC_CTRL_LINEOUTREN, 1, 0),
+	/* Mixer pre-gains */
+	SOC_SINGLE_TLV("Line In Playback Volume",
+		       SUN6I_CODEC_OM_PA_CTRL, SUN6I_CODEC_OM_PA_CTRL_LINEING,
+		       0x7, 0, sun6i_codec_out_mixer_pregain_scale),
+	SOC_SINGLE_TLV("Mic1 Playback Volume",
+		       SUN6I_CODEC_OM_PA_CTRL, SUN6I_CODEC_OM_PA_CTRL_MIC1G,
+		       0x7, 0, sun6i_codec_out_mixer_pregain_scale),
+	SOC_SINGLE_TLV("Mic2 Playback Volume",
+		       SUN6I_CODEC_OM_PA_CTRL, SUN6I_CODEC_OM_PA_CTRL_MIC2G,
+		       0x7, 0, sun6i_codec_out_mixer_pregain_scale),
+
+	/* Microphone Amp boost gains */
+	SOC_SINGLE_TLV("Mic1 Boost Volume", SUN6I_CODEC_MIC_CTRL,
+		       SUN6I_CODEC_MIC_CTRL_MIC1BOOST, 0x7, 0,
+		       sun6i_codec_mic_gain_scale),
+	SOC_SINGLE_TLV("Mic2 Boost Volume", SUN6I_CODEC_MIC_CTRL,
+		       SUN6I_CODEC_MIC_CTRL_MIC2BOOST, 0x7, 0,
+		       sun6i_codec_mic_gain_scale),
+	SOC_DOUBLE_TLV("ADC Capture Volume",
+		       SUN6I_CODEC_ADC_ACTL, SUN6I_CODEC_ADC_ACTL_ADCLG,
+		       SUN6I_CODEC_ADC_ACTL_ADCRG, 0x7, 0,
+		       sun6i_codec_out_mixer_pregain_scale),
+};
+
+static const struct snd_soc_dapm_widget sun6i_codec_codec_dapm_widgets[] = {
+	/* Microphone inputs */
+	SND_SOC_DAPM_INPUT("MIC1"),
+	SND_SOC_DAPM_INPUT("MIC2"),
+	SND_SOC_DAPM_INPUT("MIC3"),
+
+	/* Microphone Bias */
+	SND_SOC_DAPM_SUPPLY("HBIAS", SUN6I_CODEC_MIC_CTRL,
+			    SUN6I_CODEC_MIC_CTRL_HBIASEN, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("MBIAS", SUN6I_CODEC_MIC_CTRL,
+			    SUN6I_CODEC_MIC_CTRL_MBIASEN, 0, NULL, 0),
+
+	/* Mic input path */
+	SND_SOC_DAPM_MUX("Mic2 Amplifier Source Route",
+			 SND_SOC_NOPM, 0, 0, sun6i_codec_mic2_src),
+	SND_SOC_DAPM_PGA("Mic1 Amplifier", SUN6I_CODEC_MIC_CTRL,
+			 SUN6I_CODEC_MIC_CTRL_MIC1AMPEN, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("Mic2 Amplifier", SUN6I_CODEC_MIC_CTRL,
+			 SUN6I_CODEC_MIC_CTRL_MIC2AMPEN, 0, NULL, 0),
+
+	/* Line In */
+	SND_SOC_DAPM_INPUT("LINEIN"),
+
+	/* Digital parts of the ADCs */
+	SND_SOC_DAPM_SUPPLY("ADC Enable", SUN6I_CODEC_ADC_FIFOC,
+			    SUN6I_CODEC_ADC_FIFOC_EN_AD, 0,
+			    NULL, 0),
+
+	/* Analog parts of the ADCs */
+	SND_SOC_DAPM_ADC("Left ADC", "Codec Capture", SUN6I_CODEC_ADC_ACTL,
+			 SUN6I_CODEC_ADC_ACTL_ADCLEN, 0),
+	SND_SOC_DAPM_ADC("Right ADC", "Codec Capture", SUN6I_CODEC_ADC_ACTL,
+			 SUN6I_CODEC_ADC_ACTL_ADCREN, 0),
+
+	/* ADC Mixers */
+	SOC_MIXER_ARRAY("Left ADC Mixer", SND_SOC_NOPM, 0, 0,
+			sun6i_codec_adc_mixer_controls),
+	SOC_MIXER_ARRAY("Right ADC Mixer", SND_SOC_NOPM, 0, 0,
+			sun6i_codec_adc_mixer_controls),
+
+	/* Digital parts of the DACs */
+	SND_SOC_DAPM_SUPPLY("DAC Enable", SUN4I_CODEC_DAC_DPC,
+			    SUN4I_CODEC_DAC_DPC_EN_DA, 0,
+			    NULL, 0),
+
+	/* Analog parts of the DACs */
+	SND_SOC_DAPM_DAC("Left DAC", "Codec Playback",
+			 SUN6I_CODEC_OM_DACA_CTRL,
+			 SUN6I_CODEC_OM_DACA_CTRL_DACALEN, 0),
+	SND_SOC_DAPM_DAC("Right DAC", "Codec Playback",
+			 SUN6I_CODEC_OM_DACA_CTRL,
+			 SUN6I_CODEC_OM_DACA_CTRL_DACAREN, 0),
+
+	/* Mixers */
+	SOC_MIXER_ARRAY("Left Mixer", SUN6I_CODEC_OM_DACA_CTRL,
+			SUN6I_CODEC_OM_DACA_CTRL_LMIXEN, 0,
+			sun6i_codec_mixer_controls),
+	SOC_MIXER_ARRAY("Right Mixer", SUN6I_CODEC_OM_DACA_CTRL,
+			SUN6I_CODEC_OM_DACA_CTRL_RMIXEN, 0,
+			sun6i_codec_mixer_controls),
+
+	/* Headphone output path */
+	SND_SOC_DAPM_MUX("Headphone Source Playback Route",
+			 SND_SOC_NOPM, 0, 0, sun6i_codec_hp_src),
+	SND_SOC_DAPM_OUT_DRV("Headphone Amp", SUN6I_CODEC_OM_PA_CTRL,
+			     SUN6I_CODEC_OM_PA_CTRL_HPPAEN, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("HPCOM Protection", SUN6I_CODEC_OM_PA_CTRL,
+			    SUN6I_CODEC_OM_PA_CTRL_COMPTEN, 0, NULL, 0),
+	SND_SOC_DAPM_REG(snd_soc_dapm_supply, "HPCOM", SUN6I_CODEC_OM_PA_CTRL,
+			 SUN6I_CODEC_OM_PA_CTRL_HPCOM_CTL, 0x3, 0x3, 0),
+	SND_SOC_DAPM_OUTPUT("HP"),
+
+	/* Line Out path */
+	SND_SOC_DAPM_MUX("Line Out Source Playback Route",
+			 SND_SOC_NOPM, 0, 0, sun6i_codec_lineout_src),
+	SND_SOC_DAPM_OUTPUT("LINEOUT"),
+};
+
+static const struct snd_soc_dapm_route sun6i_codec_codec_dapm_routes[] = {
+	/* DAC Routes */
+	{ "Left DAC", NULL, "DAC Enable" },
+	{ "Right DAC", NULL, "DAC Enable" },
+
+	/* Microphone Routes */
+	{ "Mic1 Amplifier", NULL, "MIC1"},
+	{ "Mic2 Amplifier Source Route", "Mic2", "MIC2" },
+	{ "Mic2 Amplifier Source Route", "Mic3", "MIC3" },
+	{ "Mic2 Amplifier", NULL, "Mic2 Amplifier Source Route"},
+
+	/* Left Mixer Routes */
+	{ "Left Mixer", "DAC Playback Switch", "Left DAC" },
+	{ "Left Mixer", "DAC Reversed Playback Switch", "Right DAC" },
+	{ "Left Mixer", "Line In Playback Switch", "LINEIN" },
+	{ "Left Mixer", "Mic1 Playback Switch", "Mic1 Amplifier" },
+	{ "Left Mixer", "Mic2 Playback Switch", "Mic2 Amplifier" },
+
+	/* Right Mixer Routes */
+	{ "Right Mixer", "DAC Playback Switch", "Right DAC" },
+	{ "Right Mixer", "DAC Reversed Playback Switch", "Left DAC" },
+	{ "Right Mixer", "Line In Playback Switch", "LINEIN" },
+	{ "Right Mixer", "Mic1 Playback Switch", "Mic1 Amplifier" },
+	{ "Right Mixer", "Mic2 Playback Switch", "Mic2 Amplifier" },
+
+	/* Left ADC Mixer Routes */
+	{ "Left ADC Mixer", "Mixer Capture Switch", "Left Mixer" },
+	{ "Left ADC Mixer", "Mixer Reversed Capture Switch", "Right Mixer" },
+	{ "Left ADC Mixer", "Line In Capture Switch", "LINEIN" },
+	{ "Left ADC Mixer", "Mic1 Capture Switch", "Mic1 Amplifier" },
+	{ "Left ADC Mixer", "Mic2 Capture Switch", "Mic2 Amplifier" },
+
+	/* Right ADC Mixer Routes */
+	{ "Right ADC Mixer", "Mixer Capture Switch", "Right Mixer" },
+	{ "Right ADC Mixer", "Mixer Reversed Capture Switch", "Left Mixer" },
+	{ "Right ADC Mixer", "Line In Capture Switch", "LINEIN" },
+	{ "Right ADC Mixer", "Mic1 Capture Switch", "Mic1 Amplifier" },
+	{ "Right ADC Mixer", "Mic2 Capture Switch", "Mic2 Amplifier" },
+
+	/* Headphone Routes */
+	{ "Headphone Source Playback Route", "DAC", "Left DAC" },
+	{ "Headphone Source Playback Route", "DAC", "Right DAC" },
+	{ "Headphone Source Playback Route", "Mixer", "Left Mixer" },
+	{ "Headphone Source Playback Route", "Mixer", "Right Mixer" },
+	{ "Headphone Amp", NULL, "Headphone Source Playback Route" },
+	{ "HP", NULL, "Headphone Amp" },
+	{ "HPCOM", NULL, "HPCOM Protection" },
+
+	/* Line Out Routes */
+	{ "Line Out Source Playback Route", "Stereo", "Left Mixer" },
+	{ "Line Out Source Playback Route", "Stereo", "Right Mixer" },
+	{ "Line Out Source Playback Route", "Mono Differential", "Left Mixer" },
+	{ "LINEOUT", NULL, "Line Out Source Playback Route" },
+
+	/* ADC Routes */
+	{ "Left ADC", NULL, "ADC Enable" },
+	{ "Right ADC", NULL, "ADC Enable" },
+	{ "Left ADC", NULL, "Left ADC Mixer" },
+	{ "Right ADC", NULL, "Right ADC Mixer" },
+};
+
+static struct snd_soc_codec_driver sun6i_codec_codec = {
+	.component_driver = {
+		.controls		= sun6i_codec_codec_widgets,
+		.num_controls		= ARRAY_SIZE(sun6i_codec_codec_widgets),
+		.dapm_widgets		= sun6i_codec_codec_dapm_widgets,
+		.num_dapm_widgets	= ARRAY_SIZE(sun6i_codec_codec_dapm_widgets),
+		.dapm_routes		= sun6i_codec_codec_dapm_routes,
+		.num_dapm_routes	= ARRAY_SIZE(sun6i_codec_codec_dapm_routes),
+	},
+};
+
+/* sun8i A23 codec */
+static const struct snd_kcontrol_new sun8i_a23_codec_codec_controls[] = {
+	SOC_SINGLE_TLV("DAC Playback Volume", SUN4I_CODEC_DAC_DPC,
+		       SUN4I_CODEC_DAC_DPC_DVOL, 0x3f, 1,
+		       sun6i_codec_dvol_scale),
+};
+
+static const struct snd_soc_dapm_widget sun8i_a23_codec_codec_widgets[] = {
+	/* Digital parts of the ADCs */
+	SND_SOC_DAPM_SUPPLY("ADC Enable", SUN6I_CODEC_ADC_FIFOC,
+			    SUN6I_CODEC_ADC_FIFOC_EN_AD, 0, NULL, 0),
+	/* Digital parts of the DACs */
+	SND_SOC_DAPM_SUPPLY("DAC Enable", SUN4I_CODEC_DAC_DPC,
+			    SUN4I_CODEC_DAC_DPC_EN_DA, 0, NULL, 0),
+
+};
+
+static struct snd_soc_codec_driver sun8i_a23_codec_codec = {
+	.component_driver = {
+		.controls		= sun8i_a23_codec_codec_controls,
+		.num_controls		= ARRAY_SIZE(sun8i_a23_codec_codec_controls),
+		.dapm_widgets		= sun8i_a23_codec_codec_widgets,
+		.num_dapm_widgets	= ARRAY_SIZE(sun8i_a23_codec_codec_widgets),
+	},
+};
+
 static const struct snd_soc_component_driver sun4i_codec_component = {
 	.name = "sun4i-codec",
 };
@@ -678,45 +1144,6 @@ static struct snd_soc_dai_driver dummy_cpu_dai = {
 	 },
 };
 
-static const struct regmap_config sun4i_codec_regmap_config = {
-	.reg_bits	= 32,
-	.reg_stride	= 4,
-	.val_bits	= 32,
-	.max_register	= SUN4I_CODEC_ADC_RXCNT,
-};
-
-static const struct regmap_config sun7i_codec_regmap_config = {
-	.reg_bits	= 32,
-	.reg_stride	= 4,
-	.val_bits	= 32,
-	.max_register	= SUN7I_CODEC_AC_MIC_PHONE_CAL,
-};
-
-struct sun4i_codec_quirks {
-	const struct regmap_config *regmap_config;
-};
-
-static const struct sun4i_codec_quirks sun4i_codec_quirks = {
-	.regmap_config = &sun4i_codec_regmap_config,
-};
-
-static const struct sun4i_codec_quirks sun7i_codec_quirks = {
-	.regmap_config = &sun7i_codec_regmap_config,
-};
-
-static const struct of_device_id sun4i_codec_of_match[] = {
-	{
-		.compatible = "allwinner,sun4i-a10-codec",
-		.data = &sun4i_codec_quirks,
-	},
-	{
-		.compatible = "allwinner,sun7i-a20-codec",
-		.data = &sun7i_codec_quirks,
-	},
-	{}
-};
-MODULE_DEVICE_TABLE(of, sun4i_codec_of_match);
-
 static struct snd_soc_dai_link *sun4i_codec_create_link(struct device *dev,
 							int *num_links)
 {
@@ -781,6 +1208,259 @@ static struct snd_soc_card *sun4i_codec_create_card(struct device *dev)
 	return card;
 };
 
+static const struct snd_soc_dapm_widget sun6i_codec_card_dapm_widgets[] = {
+	SND_SOC_DAPM_HP("Headphone", NULL),
+	SND_SOC_DAPM_LINE("Line In", NULL),
+	SND_SOC_DAPM_LINE("Line Out", NULL),
+	SND_SOC_DAPM_MIC("Headset Mic", NULL),
+	SND_SOC_DAPM_MIC("Mic", NULL),
+	SND_SOC_DAPM_SPK("Speaker", sun4i_codec_spk_event),
+};
+
+static struct snd_soc_card *sun6i_codec_create_card(struct device *dev)
+{
+	struct snd_soc_card *card;
+	int ret;
+
+	card = devm_kzalloc(dev, sizeof(*card), GFP_KERNEL);
+	if (!card)
+		return ERR_PTR(-ENOMEM);
+
+	card->dai_link = sun4i_codec_create_link(dev, &card->num_links);
+	if (!card->dai_link)
+		return ERR_PTR(-ENOMEM);
+
+	card->dev		= dev;
+	card->name		= "A31 Audio Codec";
+	card->dapm_widgets	= sun6i_codec_card_dapm_widgets;
+	card->num_dapm_widgets	= ARRAY_SIZE(sun6i_codec_card_dapm_widgets);
+	card->fully_routed	= true;
+
+	ret = snd_soc_of_parse_audio_routing(card, "allwinner,audio-routing");
+	if (ret)
+		dev_warn(dev, "failed to parse audio-routing: %d\n", ret);
+
+	return card;
+};
+
+/* Connect digital side enables to analog side widgets */
+static const struct snd_soc_dapm_route sun8i_codec_card_routes[] = {
+	/* ADC Routes */
+	{ "Left ADC", NULL, "ADC Enable" },
+	{ "Right ADC", NULL, "ADC Enable" },
+	{ "Codec Capture", NULL, "Left ADC" },
+	{ "Codec Capture", NULL, "Right ADC" },
+
+	/* DAC Routes */
+	{ "Left DAC", NULL, "DAC Enable" },
+	{ "Right DAC", NULL, "DAC Enable" },
+	{ "Left DAC", NULL, "Codec Playback" },
+	{ "Right DAC", NULL, "Codec Playback" },
+};
+
+static struct snd_soc_aux_dev aux_dev = {
+	.name = "Codec Analog Controls",
+};
+
+static struct snd_soc_card *sun8i_a23_codec_create_card(struct device *dev)
+{
+	struct snd_soc_card *card;
+	int ret;
+
+	card = devm_kzalloc(dev, sizeof(*card), GFP_KERNEL);
+	if (!card)
+		return ERR_PTR(-ENOMEM);
+
+	aux_dev.codec_of_node = of_parse_phandle(dev->of_node,
+						 "allwinner,codec-analog-controls",
+						 0);
+	if (!aux_dev.codec_of_node) {
+		dev_err(dev, "Can't find analog controls for codec.\n");
+		return ERR_PTR(-EINVAL);
+	};
+
+	card->dai_link = sun4i_codec_create_link(dev, &card->num_links);
+	if (!card->dai_link)
+		return ERR_PTR(-ENOMEM);
+
+	card->dev		= dev;
+	card->name		= "A23 Audio Codec";
+	card->dapm_widgets	= sun6i_codec_card_dapm_widgets;
+	card->num_dapm_widgets	= ARRAY_SIZE(sun6i_codec_card_dapm_widgets);
+	card->dapm_routes	= sun8i_codec_card_routes;
+	card->num_dapm_routes	= ARRAY_SIZE(sun8i_codec_card_routes);
+	card->aux_dev		= &aux_dev;
+	card->num_aux_devs	= 1;
+	card->fully_routed	= true;
+
+	ret = snd_soc_of_parse_audio_routing(card, "allwinner,audio-routing");
+	if (ret)
+		dev_warn(dev, "failed to parse audio-routing: %d\n", ret);
+
+	return card;
+};
+
+static struct snd_soc_card *sun8i_h3_codec_create_card(struct device *dev)
+{
+	struct snd_soc_card *card;
+	int ret;
+
+	card = devm_kzalloc(dev, sizeof(*card), GFP_KERNEL);
+	if (!card)
+		return ERR_PTR(-ENOMEM);
+
+	aux_dev.codec_of_node = of_parse_phandle(dev->of_node,
+						 "allwinner,codec-analog-controls",
+						 0);
+	if (!aux_dev.codec_of_node) {
+		dev_err(dev, "Can't find analog controls for codec.\n");
+		return ERR_PTR(-EINVAL);
+	};
+
+	card->dai_link = sun4i_codec_create_link(dev, &card->num_links);
+	if (!card->dai_link)
+		return ERR_PTR(-ENOMEM);
+
+	card->dev		= dev;
+	card->name		= "H3 Audio Codec";
+	card->dapm_widgets	= sun6i_codec_card_dapm_widgets;
+	card->num_dapm_widgets	= ARRAY_SIZE(sun6i_codec_card_dapm_widgets);
+	card->dapm_routes	= sun8i_codec_card_routes;
+	card->num_dapm_routes	= ARRAY_SIZE(sun8i_codec_card_routes);
+	card->aux_dev		= &aux_dev;
+	card->num_aux_devs	= 1;
+	card->fully_routed	= true;
+
+	ret = snd_soc_of_parse_audio_routing(card, "allwinner,audio-routing");
+	if (ret)
+		dev_warn(dev, "failed to parse audio-routing: %d\n", ret);
+
+	return card;
+};
+
+static const struct regmap_config sun4i_codec_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= SUN4I_CODEC_ADC_RXCNT,
+};
+
+static const struct regmap_config sun6i_codec_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= SUN6I_CODEC_HMIC_DATA,
+};
+
+static const struct regmap_config sun7i_codec_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= SUN7I_CODEC_AC_MIC_PHONE_CAL,
+};
+
+static const struct regmap_config sun8i_a23_codec_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= SUN8I_A23_CODEC_ADC_RXCNT,
+};
+
+static const struct regmap_config sun8i_h3_codec_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= SUN8I_H3_CODEC_ADC_DBG,
+};
+
+struct sun4i_codec_quirks {
+	const struct regmap_config *regmap_config;
+	const struct snd_soc_codec_driver *codec;
+	struct snd_soc_card * (*create_card)(struct device *dev);
+	struct reg_field reg_adc_fifoc;	/* used for regmap_field */
+	unsigned int reg_dac_txdata;	/* TX FIFO offset for DMA config */
+	unsigned int reg_adc_rxdata;	/* RX FIFO offset for DMA config */
+	bool has_reset;
+};
+
+static const struct sun4i_codec_quirks sun4i_codec_quirks = {
+	.regmap_config	= &sun4i_codec_regmap_config,
+	.codec		= &sun4i_codec_codec,
+	.create_card	= sun4i_codec_create_card,
+	.reg_adc_fifoc	= REG_FIELD(SUN4I_CODEC_ADC_FIFOC, 0, 31),
+	.reg_dac_txdata	= SUN4I_CODEC_DAC_TXDATA,
+	.reg_adc_rxdata	= SUN4I_CODEC_ADC_RXDATA,
+};
+
+static const struct sun4i_codec_quirks sun6i_a31_codec_quirks = {
+	.regmap_config	= &sun6i_codec_regmap_config,
+	.codec		= &sun6i_codec_codec,
+	.create_card	= sun6i_codec_create_card,
+	.reg_adc_fifoc	= REG_FIELD(SUN6I_CODEC_ADC_FIFOC, 0, 31),
+	.reg_dac_txdata	= SUN4I_CODEC_DAC_TXDATA,
+	.reg_adc_rxdata	= SUN6I_CODEC_ADC_RXDATA,
+	.has_reset	= true,
+};
+
+static const struct sun4i_codec_quirks sun7i_codec_quirks = {
+	.regmap_config	= &sun7i_codec_regmap_config,
+	.codec		= &sun4i_codec_codec,
+	.create_card	= sun4i_codec_create_card,
+	.reg_adc_fifoc	= REG_FIELD(SUN4I_CODEC_ADC_FIFOC, 0, 31),
+	.reg_dac_txdata	= SUN4I_CODEC_DAC_TXDATA,
+	.reg_adc_rxdata	= SUN4I_CODEC_ADC_RXDATA,
+};
+
+static const struct sun4i_codec_quirks sun8i_a23_codec_quirks = {
+	.regmap_config	= &sun8i_a23_codec_regmap_config,
+	.codec		= &sun8i_a23_codec_codec,
+	.create_card	= sun8i_a23_codec_create_card,
+	.reg_adc_fifoc	= REG_FIELD(SUN6I_CODEC_ADC_FIFOC, 0, 31),
+	.reg_dac_txdata	= SUN4I_CODEC_DAC_TXDATA,
+	.reg_adc_rxdata	= SUN6I_CODEC_ADC_RXDATA,
+	.has_reset	= true,
+};
+
+static const struct sun4i_codec_quirks sun8i_h3_codec_quirks = {
+	.regmap_config	= &sun8i_h3_codec_regmap_config,
+	/*
+	 * TODO Share the codec structure with A23 for now.
+	 * This should be split out when adding digital audio
+	 * processing support for the H3.
+	 */
+	.codec		= &sun8i_a23_codec_codec,
+	.create_card	= sun8i_h3_codec_create_card,
+	.reg_adc_fifoc	= REG_FIELD(SUN6I_CODEC_ADC_FIFOC, 0, 31),
+	.reg_dac_txdata	= SUN8I_H3_CODEC_DAC_TXDATA,
+	.reg_adc_rxdata	= SUN6I_CODEC_ADC_RXDATA,
+	.has_reset	= true,
+};
+
+static const struct of_device_id sun4i_codec_of_match[] = {
+	{
+		.compatible = "allwinner,sun4i-a10-codec",
+		.data = &sun4i_codec_quirks,
+	},
+	{
+		.compatible = "allwinner,sun6i-a31-codec",
+		.data = &sun6i_a31_codec_quirks,
+	},
+	{
+		.compatible = "allwinner,sun7i-a20-codec",
+		.data = &sun7i_codec_quirks,
+	},
+	{
+		.compatible = "allwinner,sun8i-a23-codec",
+		.data = &sun8i_a23_codec_quirks,
+	},
+	{
+		.compatible = "allwinner,sun8i-h3-codec",
+		.data = &sun8i_h3_codec_quirks,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, sun4i_codec_of_match);
+
 static int sun4i_codec_probe(struct platform_device *pdev)
 {
 	struct snd_soc_card *card;
@@ -829,6 +1509,14 @@ static int sun4i_codec_probe(struct platform_device *pdev)
 		return PTR_ERR(scodec->clk_module);
 	}
 
+	if (quirks->has_reset) {
+		scodec->rst = devm_reset_control_get(&pdev->dev, NULL);
+		if (IS_ERR(scodec->rst)) {
+			dev_err(&pdev->dev, "Failed to get reset control\n");
+			return PTR_ERR(scodec->rst);
+		}
+	}
+
 	scodec->gpio_pa = devm_gpiod_get_optional(&pdev->dev, "allwinner,pa",
 						  GPIOD_OUT_LOW);
 	if (IS_ERR(scodec->gpio_pa)) {
@@ -838,27 +1526,48 @@ static int sun4i_codec_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	/* reg_field setup */
+	scodec->reg_adc_fifoc = devm_regmap_field_alloc(&pdev->dev,
+							scodec->regmap,
+							quirks->reg_adc_fifoc);
+	if (IS_ERR(scodec->reg_adc_fifoc)) {
+		ret = PTR_ERR(scodec->reg_adc_fifoc);
+		dev_err(&pdev->dev, "Failed to create regmap fields: %d\n",
+			ret);
+		return ret;
+	}
+
 	/* Enable the bus clock */
 	if (clk_prepare_enable(scodec->clk_apb)) {
 		dev_err(&pdev->dev, "Failed to enable the APB clock\n");
 		return -EINVAL;
 	}
 
+	/* Deassert the reset control */
+	if (scodec->rst) {
+		ret = reset_control_deassert(scodec->rst);
+		if (ret) {
+			dev_err(&pdev->dev,
+				"Failed to deassert the reset control\n");
+			goto err_clk_disable;
+		}
+	}
+
 	/* DMA configuration for TX FIFO */
-	scodec->playback_dma_data.addr = res->start + SUN4I_CODEC_DAC_TXDATA;
-	scodec->playback_dma_data.maxburst = 4;
+	scodec->playback_dma_data.addr = res->start + quirks->reg_dac_txdata;
+	scodec->playback_dma_data.maxburst = 8;
 	scodec->playback_dma_data.addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 
 	/* DMA configuration for RX FIFO */
-	scodec->capture_dma_data.addr = res->start + SUN4I_CODEC_ADC_RXDATA;
-	scodec->capture_dma_data.maxburst = 4;
+	scodec->capture_dma_data.addr = res->start + quirks->reg_adc_rxdata;
+	scodec->capture_dma_data.maxburst = 8;
 	scodec->capture_dma_data.addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 
-	ret = snd_soc_register_codec(&pdev->dev, &sun4i_codec_codec,
+	ret = snd_soc_register_codec(&pdev->dev, quirks->codec,
 				     &sun4i_codec_dai, 1);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to register our codec\n");
-		goto err_clk_disable;
+		goto err_assert_reset;
 	}
 
 	ret = devm_snd_soc_register_component(&pdev->dev,
@@ -875,7 +1584,7 @@ static int sun4i_codec_probe(struct platform_device *pdev)
 		goto err_unregister_codec;
 	}
 
-	card = sun4i_codec_create_card(&pdev->dev);
+	card = quirks->create_card(&pdev->dev);
 	if (IS_ERR(card)) {
 		ret = PTR_ERR(card);
 		dev_err(&pdev->dev, "Failed to create our card\n");
@@ -895,6 +1604,9 @@ static int sun4i_codec_probe(struct platform_device *pdev)
 
 err_unregister_codec:
 	snd_soc_unregister_codec(&pdev->dev);
+err_assert_reset:
+	if (scodec->rst)
+		reset_control_assert(scodec->rst);
 err_clk_disable:
 	clk_disable_unprepare(scodec->clk_apb);
 	return ret;
@@ -907,6 +1619,8 @@ static int sun4i_codec_remove(struct platform_device *pdev)
 
 	snd_soc_unregister_card(card);
 	snd_soc_unregister_codec(&pdev->dev);
+	if (scodec->rst)
+		reset_control_assert(scodec->rst);
 	clk_disable_unprepare(scodec->clk_apb);
 
 	return 0;
@@ -926,4 +1640,5 @@ MODULE_DESCRIPTION("Allwinner A10 codec driver");
 MODULE_AUTHOR("Emilio López <emilio@elopez.com.ar>");
 MODULE_AUTHOR("Jon Smirl <jonsmirl@gmail.com>");
 MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>");
+MODULE_AUTHOR("Chen-Yu Tsai <wens@csie.org>");
 MODULE_LICENSE("GPL");
diff --git a/sound/soc/sunxi/sun4i-i2s.c b/sound/soc/sunxi/sun4i-i2s.c
index 687a8f83dbe5..f24d19526603 100644
--- a/sound/soc/sunxi/sun4i-i2s.c
+++ b/sound/soc/sunxi/sun4i-i2s.c
@@ -93,6 +93,9 @@ struct sun4i_i2s {
 	struct clk	*mod_clk;
 	struct regmap	*regmap;
 
+	unsigned int	mclk_freq;
+
+	struct snd_dmaengine_dai_dma_data	capture_dma_data;
 	struct snd_dmaengine_dai_dma_data	playback_dma_data;
 };
 
@@ -157,14 +160,24 @@ static int sun4i_i2s_get_mclk_div(struct sun4i_i2s *i2s,
 }
 
 static int sun4i_i2s_oversample_rates[] = { 128, 192, 256, 384, 512, 768 };
+static bool sun4i_i2s_oversample_is_valid(unsigned int oversample)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(sun4i_i2s_oversample_rates); i++)
+		if (sun4i_i2s_oversample_rates[i] == oversample)
+			return true;
+
+	return false;
+}
 
 static int sun4i_i2s_set_clk_rate(struct sun4i_i2s *i2s,
 				  unsigned int rate,
 				  unsigned int word_size)
 {
-	unsigned int clk_rate;
+	unsigned int oversample_rate, clk_rate;
 	int bclk_div, mclk_div;
-	int ret, i;
+	int ret;
 
 	switch (rate) {
 	case 176400:
@@ -196,21 +209,18 @@ static int sun4i_i2s_set_clk_rate(struct sun4i_i2s *i2s,
 	if (ret)
 		return ret;
 
-	/* Always favor the highest oversampling rate */
-	for (i = (ARRAY_SIZE(sun4i_i2s_oversample_rates) - 1); i >= 0; i--) {
-		unsigned int oversample_rate = sun4i_i2s_oversample_rates[i];
-
-		bclk_div = sun4i_i2s_get_bclk_div(i2s, oversample_rate,
-						  word_size);
-		mclk_div = sun4i_i2s_get_mclk_div(i2s, oversample_rate,
-						  clk_rate,
-						  rate);
+	oversample_rate = i2s->mclk_freq / rate;
+	if (!sun4i_i2s_oversample_is_valid(oversample_rate))
+		return -EINVAL;
 
-		if ((bclk_div >= 0) && (mclk_div >= 0))
-			break;
-	}
+	bclk_div = sun4i_i2s_get_bclk_div(i2s, oversample_rate,
+					  word_size);
+	if (bclk_div < 0)
+		return -EINVAL;
 
-	if ((bclk_div < 0) || (mclk_div < 0))
+	mclk_div = sun4i_i2s_get_mclk_div(i2s, oversample_rate,
+					  clk_rate, rate);
+	if (mclk_div < 0)
 		return -EINVAL;
 
 	regmap_write(i2s->regmap, SUN4I_I2S_CLK_DIV_REG,
@@ -341,6 +351,27 @@ static int sun4i_i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
 	return 0;
 }
 
+static void sun4i_i2s_start_capture(struct sun4i_i2s *i2s)
+{
+	/* Flush RX FIFO */
+	regmap_update_bits(i2s->regmap, SUN4I_I2S_FIFO_CTRL_REG,
+			   SUN4I_I2S_FIFO_CTRL_FLUSH_RX,
+			   SUN4I_I2S_FIFO_CTRL_FLUSH_RX);
+
+	/* Clear RX counter */
+	regmap_write(i2s->regmap, SUN4I_I2S_RX_CNT_REG, 0);
+
+	/* Enable RX Block */
+	regmap_update_bits(i2s->regmap, SUN4I_I2S_CTRL_REG,
+			   SUN4I_I2S_CTRL_RX_EN,
+			   SUN4I_I2S_CTRL_RX_EN);
+
+	/* Enable RX DRQ */
+	regmap_update_bits(i2s->regmap, SUN4I_I2S_DMA_INT_CTRL_REG,
+			   SUN4I_I2S_DMA_INT_CTRL_RX_DRQ_EN,
+			   SUN4I_I2S_DMA_INT_CTRL_RX_DRQ_EN);
+}
+
 static void sun4i_i2s_start_playback(struct sun4i_i2s *i2s)
 {
 	/* Flush TX FIFO */
@@ -362,6 +393,18 @@ static void sun4i_i2s_start_playback(struct sun4i_i2s *i2s)
 			   SUN4I_I2S_DMA_INT_CTRL_TX_DRQ_EN);
 }
 
+static void sun4i_i2s_stop_capture(struct sun4i_i2s *i2s)
+{
+	/* Disable RX Block */
+	regmap_update_bits(i2s->regmap, SUN4I_I2S_CTRL_REG,
+			   SUN4I_I2S_CTRL_RX_EN,
+			   0);
+
+	/* Disable RX DRQ */
+	regmap_update_bits(i2s->regmap, SUN4I_I2S_DMA_INT_CTRL_REG,
+			   SUN4I_I2S_DMA_INT_CTRL_RX_DRQ_EN,
+			   0);
+}
 
 static void sun4i_i2s_stop_playback(struct sun4i_i2s *i2s)
 {
@@ -388,7 +431,7 @@ static int sun4i_i2s_trigger(struct snd_pcm_substream *substream, int cmd,
 		if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
 			sun4i_i2s_start_playback(i2s);
 		else
-			return -EINVAL;
+			sun4i_i2s_start_capture(i2s);
 		break;
 
 	case SNDRV_PCM_TRIGGER_STOP:
@@ -397,7 +440,7 @@ static int sun4i_i2s_trigger(struct snd_pcm_substream *substream, int cmd,
 		if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
 			sun4i_i2s_stop_playback(i2s);
 		else
-			return -EINVAL;
+			sun4i_i2s_stop_capture(i2s);
 		break;
 
 	default:
@@ -447,9 +490,23 @@ static void sun4i_i2s_shutdown(struct snd_pcm_substream *substream,
 	regmap_write(i2s->regmap, SUN4I_I2S_CTRL_REG, 0);
 }
 
+static int sun4i_i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id,
+				unsigned int freq, int dir)
+{
+	struct sun4i_i2s *i2s = snd_soc_dai_get_drvdata(dai);
+
+	if (clk_id != 0)
+		return -EINVAL;
+
+	i2s->mclk_freq = freq;
+
+	return 0;
+}
+
 static const struct snd_soc_dai_ops sun4i_i2s_dai_ops = {
 	.hw_params	= sun4i_i2s_hw_params,
 	.set_fmt	= sun4i_i2s_set_fmt,
+	.set_sysclk	= sun4i_i2s_set_sysclk,
 	.shutdown	= sun4i_i2s_shutdown,
 	.startup	= sun4i_i2s_startup,
 	.trigger	= sun4i_i2s_trigger,
@@ -459,7 +516,9 @@ static int sun4i_i2s_dai_probe(struct snd_soc_dai *dai)
 {
 	struct sun4i_i2s *i2s = snd_soc_dai_get_drvdata(dai);
 
-	snd_soc_dai_init_dma_data(dai, &i2s->playback_dma_data, NULL);
+	snd_soc_dai_init_dma_data(dai,
+				  &i2s->playback_dma_data,
+				  &i2s->capture_dma_data);
 
 	snd_soc_dai_set_drvdata(dai, i2s);
 
@@ -468,6 +527,13 @@ static int sun4i_i2s_dai_probe(struct snd_soc_dai *dai)
 
 static struct snd_soc_dai_driver sun4i_i2s_dai = {
 	.probe = sun4i_i2s_dai_probe,
+	.capture = {
+		.stream_name = "Capture",
+		.channels_min = 2,
+		.channels_max = 2,
+		.rates = SNDRV_PCM_RATE_8000_192000,
+		.formats = SNDRV_PCM_FMTBIT_S16_LE,
+	},
 	.playback = {
 		.stream_name = "Playback",
 		.channels_min = 2,
@@ -630,6 +696,9 @@ static int sun4i_i2s_probe(struct platform_device *pdev)
 	i2s->playback_dma_data.addr = res->start + SUN4I_I2S_FIFO_TX_REG;
 	i2s->playback_dma_data.maxburst = 4;
 
+	i2s->capture_dma_data.addr = res->start + SUN4I_I2S_FIFO_RX_REG;
+	i2s->capture_dma_data.maxburst = 4;
+
 	pm_runtime_enable(&pdev->dev);
 	if (!pm_runtime_enabled(&pdev->dev)) {
 		ret = sun4i_i2s_runtime_resume(&pdev->dev);
diff --git a/sound/soc/sunxi/sun8i-codec-analog.c b/sound/soc/sunxi/sun8i-codec-analog.c
new file mode 100644
index 000000000000..af02290ebe49
--- /dev/null
+++ b/sound/soc/sunxi/sun8i-codec-analog.c
@@ -0,0 +1,665 @@
+/*
+ * This driver supports the analog controls for the internal codec
+ * found in Allwinner's A31s, A23, A33 and H3 SoCs.
+ *
+ * Copyright 2016 Chen-Yu Tsai <wens@csie.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <sound/soc.h>
+#include <sound/soc-dapm.h>
+#include <sound/tlv.h>
+
+/* Codec analog control register offsets and bit fields */
+#define SUN8I_ADDA_HP_VOLC		0x00
+#define SUN8I_ADDA_HP_VOLC_PA_CLK_GATE		7
+#define SUN8I_ADDA_HP_VOLC_HP_VOL		0
+#define SUN8I_ADDA_LOMIXSC		0x01
+#define SUN8I_ADDA_LOMIXSC_MIC1			6
+#define SUN8I_ADDA_LOMIXSC_MIC2			5
+#define SUN8I_ADDA_LOMIXSC_PHONE		4
+#define SUN8I_ADDA_LOMIXSC_PHONEN		3
+#define SUN8I_ADDA_LOMIXSC_LINEINL		2
+#define SUN8I_ADDA_LOMIXSC_DACL			1
+#define SUN8I_ADDA_LOMIXSC_DACR			0
+#define SUN8I_ADDA_ROMIXSC		0x02
+#define SUN8I_ADDA_ROMIXSC_MIC1			6
+#define SUN8I_ADDA_ROMIXSC_MIC2			5
+#define SUN8I_ADDA_ROMIXSC_PHONE		4
+#define SUN8I_ADDA_ROMIXSC_PHONEP		3
+#define SUN8I_ADDA_ROMIXSC_LINEINR		2
+#define SUN8I_ADDA_ROMIXSC_DACR			1
+#define SUN8I_ADDA_ROMIXSC_DACL			0
+#define SUN8I_ADDA_DAC_PA_SRC		0x03
+#define SUN8I_ADDA_DAC_PA_SRC_DACAREN		7
+#define SUN8I_ADDA_DAC_PA_SRC_DACALEN		6
+#define SUN8I_ADDA_DAC_PA_SRC_RMIXEN		5
+#define SUN8I_ADDA_DAC_PA_SRC_LMIXEN		4
+#define SUN8I_ADDA_DAC_PA_SRC_RHPPAMUTE		3
+#define SUN8I_ADDA_DAC_PA_SRC_LHPPAMUTE		2
+#define SUN8I_ADDA_DAC_PA_SRC_RHPIS		1
+#define SUN8I_ADDA_DAC_PA_SRC_LHPIS		0
+#define SUN8I_ADDA_PHONEIN_GCTRL	0x04
+#define SUN8I_ADDA_PHONEIN_GCTRL_PHONEPG	4
+#define SUN8I_ADDA_PHONEIN_GCTRL_PHONENG	0
+#define SUN8I_ADDA_LINEIN_GCTRL		0x05
+#define SUN8I_ADDA_LINEIN_GCTRL_LINEING		4
+#define SUN8I_ADDA_LINEIN_GCTRL_PHONEG		0
+#define SUN8I_ADDA_MICIN_GCTRL		0x06
+#define SUN8I_ADDA_MICIN_GCTRL_MIC1G		4
+#define SUN8I_ADDA_MICIN_GCTRL_MIC2G		0
+#define SUN8I_ADDA_PAEN_HP_CTRL		0x07
+#define SUN8I_ADDA_PAEN_HP_CTRL_HPPAEN		7
+#define SUN8I_ADDA_PAEN_HP_CTRL_LINEOUTEN	7	/* H3 specific */
+#define SUN8I_ADDA_PAEN_HP_CTRL_HPCOM_FC	5
+#define SUN8I_ADDA_PAEN_HP_CTRL_COMPTEN		4
+#define SUN8I_ADDA_PAEN_HP_CTRL_PA_ANTI_POP_CTRL	2
+#define SUN8I_ADDA_PAEN_HP_CTRL_LTRNMUTE	1
+#define SUN8I_ADDA_PAEN_HP_CTRL_RTLNMUTE	0
+#define SUN8I_ADDA_PHONEOUT_CTRL	0x08
+#define SUN8I_ADDA_PHONEOUT_CTRL_PHONEOUTG	5
+#define SUN8I_ADDA_PHONEOUT_CTRL_PHONEOUTEN	4
+#define SUN8I_ADDA_PHONEOUT_CTRL_PHONEOUT_MIC1	3
+#define SUN8I_ADDA_PHONEOUT_CTRL_PHONEOUT_MIC2	2
+#define SUN8I_ADDA_PHONEOUT_CTRL_PHONEOUT_RMIX	1
+#define SUN8I_ADDA_PHONEOUT_CTRL_PHONEOUT_LMIX	0
+#define SUN8I_ADDA_PHONE_GAIN_CTRL	0x09
+#define SUN8I_ADDA_PHONE_GAIN_CTRL_LINEOUT_VOL	3
+#define SUN8I_ADDA_PHONE_GAIN_CTRL_PHONEPREG	0
+#define SUN8I_ADDA_MIC2G_CTRL		0x0a
+#define SUN8I_ADDA_MIC2G_CTRL_MIC2AMPEN		7
+#define SUN8I_ADDA_MIC2G_CTRL_MIC2BOOST		4
+#define SUN8I_ADDA_MIC2G_CTRL_LINEOUTLEN	3
+#define SUN8I_ADDA_MIC2G_CTRL_LINEOUTREN	2
+#define SUN8I_ADDA_MIC2G_CTRL_LINEOUTLSRC	1
+#define SUN8I_ADDA_MIC2G_CTRL_LINEOUTRSRC	0
+#define SUN8I_ADDA_MIC1G_MICBIAS_CTRL	0x0b
+#define SUN8I_ADDA_MIC1G_MICBIAS_CTRL_HMICBIASEN	7
+#define SUN8I_ADDA_MIC1G_MICBIAS_CTRL_MMICBIASEN	6
+#define SUN8I_ADDA_MIC1G_MICBIAS_CTRL_HMICBIAS_MODE	5
+#define SUN8I_ADDA_MIC1G_MICBIAS_CTRL_MIC1AMPEN		3
+#define SUN8I_ADDA_MIC1G_MICBIAS_CTRL_MIC1BOOST		0
+#define SUN8I_ADDA_LADCMIXSC		0x0c
+#define SUN8I_ADDA_LADCMIXSC_MIC1		6
+#define SUN8I_ADDA_LADCMIXSC_MIC2		5
+#define SUN8I_ADDA_LADCMIXSC_PHONE		4
+#define SUN8I_ADDA_LADCMIXSC_PHONEN		3
+#define SUN8I_ADDA_LADCMIXSC_LINEINL		2
+#define SUN8I_ADDA_LADCMIXSC_OMIXRL		1
+#define SUN8I_ADDA_LADCMIXSC_OMIXRR		0
+#define SUN8I_ADDA_RADCMIXSC		0x0d
+#define SUN8I_ADDA_RADCMIXSC_MIC1		6
+#define SUN8I_ADDA_RADCMIXSC_MIC2		5
+#define SUN8I_ADDA_RADCMIXSC_PHONE		4
+#define SUN8I_ADDA_RADCMIXSC_PHONEP		3
+#define SUN8I_ADDA_RADCMIXSC_LINEINR		2
+#define SUN8I_ADDA_RADCMIXSC_OMIXR		1
+#define SUN8I_ADDA_RADCMIXSC_OMIXL		0
+#define SUN8I_ADDA_RES			0x0e
+#define SUN8I_ADDA_RES_MMICBIAS_SEL		4
+#define SUN8I_ADDA_RES_PA_ANTI_POP_CTRL		0
+#define SUN8I_ADDA_ADC_AP_EN		0x0f
+#define SUN8I_ADDA_ADC_AP_EN_ADCREN		7
+#define SUN8I_ADDA_ADC_AP_EN_ADCLEN		6
+#define SUN8I_ADDA_ADC_AP_EN_ADCG		0
+
+/* Analog control register access bits */
+#define ADDA_PR			0x0		/* PRCM base + 0x1c0 */
+#define ADDA_PR_RESET			BIT(28)
+#define ADDA_PR_WRITE			BIT(24)
+#define ADDA_PR_ADDR_SHIFT		16
+#define ADDA_PR_ADDR_MASK		GENMASK(4, 0)
+#define ADDA_PR_DATA_IN_SHIFT		8
+#define ADDA_PR_DATA_IN_MASK		GENMASK(7, 0)
+#define ADDA_PR_DATA_OUT_SHIFT		0
+#define ADDA_PR_DATA_OUT_MASK		GENMASK(7, 0)
+
+/* regmap access bits */
+static int adda_reg_read(void *context, unsigned int reg, unsigned int *val)
+{
+	void __iomem *base = (void __iomem *)context;
+	u32 tmp;
+
+	/* De-assert reset */
+	writel(readl(base) | ADDA_PR_RESET, base);
+
+	/* Clear write bit */
+	writel(readl(base) & ~ADDA_PR_WRITE, base);
+
+	/* Set register address */
+	tmp = readl(base);
+	tmp &= ~(ADDA_PR_ADDR_MASK << ADDA_PR_ADDR_SHIFT);
+	tmp |= (reg & ADDA_PR_ADDR_MASK) << ADDA_PR_ADDR_SHIFT;
+	writel(tmp, base);
+
+	/* Read back value */
+	*val = readl(base) & ADDA_PR_DATA_OUT_MASK;
+
+	return 0;
+}
+
+static int adda_reg_write(void *context, unsigned int reg, unsigned int val)
+{
+	void __iomem *base = (void __iomem *)context;
+	u32 tmp;
+
+	/* De-assert reset */
+	writel(readl(base) | ADDA_PR_RESET, base);
+
+	/* Set register address */
+	tmp = readl(base);
+	tmp &= ~(ADDA_PR_ADDR_MASK << ADDA_PR_ADDR_SHIFT);
+	tmp |= (reg & ADDA_PR_ADDR_MASK) << ADDA_PR_ADDR_SHIFT;
+	writel(tmp, base);
+
+	/* Set data to write */
+	tmp = readl(base);
+	tmp &= ~(ADDA_PR_DATA_IN_MASK << ADDA_PR_DATA_IN_SHIFT);
+	tmp |= (val & ADDA_PR_DATA_IN_MASK) << ADDA_PR_DATA_IN_SHIFT;
+	writel(tmp, base);
+
+	/* Set write bit to signal a write */
+	writel(readl(base) | ADDA_PR_WRITE, base);
+
+	/* Clear write bit */
+	writel(readl(base) & ~ADDA_PR_WRITE, base);
+
+	return 0;
+}
+
+static const struct regmap_config adda_pr_regmap_cfg = {
+	.name		= "adda-pr",
+	.reg_bits	= 5,
+	.reg_stride	= 1,
+	.val_bits	= 8,
+	.reg_read	= adda_reg_read,
+	.reg_write	= adda_reg_write,
+	.fast_io	= true,
+	.max_register	= 24,
+};
+
+/* mixer controls */
+static const struct snd_kcontrol_new sun8i_codec_mixer_controls[] = {
+	SOC_DAPM_DOUBLE_R("DAC Playback Switch",
+			  SUN8I_ADDA_LOMIXSC,
+			  SUN8I_ADDA_ROMIXSC,
+			  SUN8I_ADDA_LOMIXSC_DACL, 1, 0),
+	SOC_DAPM_DOUBLE_R("DAC Reversed Playback Switch",
+			  SUN8I_ADDA_LOMIXSC,
+			  SUN8I_ADDA_ROMIXSC,
+			  SUN8I_ADDA_LOMIXSC_DACR, 1, 0),
+	SOC_DAPM_DOUBLE_R("Line In Playback Switch",
+			  SUN8I_ADDA_LOMIXSC,
+			  SUN8I_ADDA_ROMIXSC,
+			  SUN8I_ADDA_LOMIXSC_LINEINL, 1, 0),
+	SOC_DAPM_DOUBLE_R("Mic1 Playback Switch",
+			  SUN8I_ADDA_LOMIXSC,
+			  SUN8I_ADDA_ROMIXSC,
+			  SUN8I_ADDA_LOMIXSC_MIC1, 1, 0),
+	SOC_DAPM_DOUBLE_R("Mic2 Playback Switch",
+			  SUN8I_ADDA_LOMIXSC,
+			  SUN8I_ADDA_ROMIXSC,
+			  SUN8I_ADDA_LOMIXSC_MIC2, 1, 0),
+};
+
+/* ADC mixer controls */
+static const struct snd_kcontrol_new sun8i_codec_adc_mixer_controls[] = {
+	SOC_DAPM_DOUBLE_R("Mixer Capture Switch",
+			  SUN8I_ADDA_LADCMIXSC,
+			  SUN8I_ADDA_RADCMIXSC,
+			  SUN8I_ADDA_LADCMIXSC_OMIXRL, 1, 0),
+	SOC_DAPM_DOUBLE_R("Mixer Reversed Capture Switch",
+			  SUN8I_ADDA_LADCMIXSC,
+			  SUN8I_ADDA_RADCMIXSC,
+			  SUN8I_ADDA_LADCMIXSC_OMIXRR, 1, 0),
+	SOC_DAPM_DOUBLE_R("Line In Capture Switch",
+			  SUN8I_ADDA_LADCMIXSC,
+			  SUN8I_ADDA_RADCMIXSC,
+			  SUN8I_ADDA_LADCMIXSC_LINEINL, 1, 0),
+	SOC_DAPM_DOUBLE_R("Mic1 Capture Switch",
+			  SUN8I_ADDA_LADCMIXSC,
+			  SUN8I_ADDA_RADCMIXSC,
+			  SUN8I_ADDA_LADCMIXSC_MIC1, 1, 0),
+	SOC_DAPM_DOUBLE_R("Mic2 Capture Switch",
+			  SUN8I_ADDA_LADCMIXSC,
+			  SUN8I_ADDA_RADCMIXSC,
+			  SUN8I_ADDA_LADCMIXSC_MIC2, 1, 0),
+};
+
+/* volume / mute controls */
+static const DECLARE_TLV_DB_SCALE(sun8i_codec_out_mixer_pregain_scale,
+				  -450, 150, 0);
+static const DECLARE_TLV_DB_RANGE(sun8i_codec_mic_gain_scale,
+	0, 0, TLV_DB_SCALE_ITEM(0, 0, 0),
+	1, 7, TLV_DB_SCALE_ITEM(2400, 300, 0),
+);
+
+static const struct snd_kcontrol_new sun8i_codec_common_controls[] = {
+	/* Mixer pre-gains */
+	SOC_SINGLE_TLV("Line In Playback Volume", SUN8I_ADDA_LINEIN_GCTRL,
+		       SUN8I_ADDA_LINEIN_GCTRL_LINEING,
+		       0x7, 0, sun8i_codec_out_mixer_pregain_scale),
+	SOC_SINGLE_TLV("Mic1 Playback Volume", SUN8I_ADDA_MICIN_GCTRL,
+		       SUN8I_ADDA_MICIN_GCTRL_MIC1G,
+		       0x7, 0, sun8i_codec_out_mixer_pregain_scale),
+	SOC_SINGLE_TLV("Mic2 Playback Volume",
+		       SUN8I_ADDA_MICIN_GCTRL, SUN8I_ADDA_MICIN_GCTRL_MIC2G,
+		       0x7, 0, sun8i_codec_out_mixer_pregain_scale),
+
+	/* Microphone Amp boost gains */
+	SOC_SINGLE_TLV("Mic1 Boost Volume", SUN8I_ADDA_MIC1G_MICBIAS_CTRL,
+		       SUN8I_ADDA_MIC1G_MICBIAS_CTRL_MIC1BOOST, 0x7, 0,
+		       sun8i_codec_mic_gain_scale),
+	SOC_SINGLE_TLV("Mic2 Boost Volume", SUN8I_ADDA_MIC2G_CTRL,
+		       SUN8I_ADDA_MIC2G_CTRL_MIC2BOOST, 0x7, 0,
+		       sun8i_codec_mic_gain_scale),
+
+	/* ADC */
+	SOC_SINGLE_TLV("ADC Gain Capture Volume", SUN8I_ADDA_ADC_AP_EN,
+		       SUN8I_ADDA_ADC_AP_EN_ADCG, 0x7, 0,
+		       sun8i_codec_out_mixer_pregain_scale),
+};
+
+static const struct snd_soc_dapm_widget sun8i_codec_common_widgets[] = {
+	/* ADC */
+	SND_SOC_DAPM_ADC("Left ADC", NULL, SUN8I_ADDA_ADC_AP_EN,
+			 SUN8I_ADDA_ADC_AP_EN_ADCLEN, 0),
+	SND_SOC_DAPM_ADC("Right ADC", NULL, SUN8I_ADDA_ADC_AP_EN,
+			 SUN8I_ADDA_ADC_AP_EN_ADCREN, 0),
+
+	/* DAC */
+	SND_SOC_DAPM_DAC("Left DAC", NULL, SUN8I_ADDA_DAC_PA_SRC,
+			 SUN8I_ADDA_DAC_PA_SRC_DACALEN, 0),
+	SND_SOC_DAPM_DAC("Right DAC", NULL, SUN8I_ADDA_DAC_PA_SRC,
+			 SUN8I_ADDA_DAC_PA_SRC_DACAREN, 0),
+	/*
+	 * Due to this component and the codec belonging to separate DAPM
+	 * contexts, we need to manually link the above widgets to their
+	 * stream widgets at the card level.
+	 */
+
+	/* Line In */
+	SND_SOC_DAPM_INPUT("LINEIN"),
+
+	/* Microphone inputs */
+	SND_SOC_DAPM_INPUT("MIC1"),
+	SND_SOC_DAPM_INPUT("MIC2"),
+
+	/* Microphone Bias */
+	SND_SOC_DAPM_SUPPLY("MBIAS", SUN8I_ADDA_MIC1G_MICBIAS_CTRL,
+			    SUN8I_ADDA_MIC1G_MICBIAS_CTRL_MMICBIASEN,
+			    0, NULL, 0),
+
+	/* Mic input path */
+	SND_SOC_DAPM_PGA("Mic1 Amplifier", SUN8I_ADDA_MIC1G_MICBIAS_CTRL,
+			 SUN8I_ADDA_MIC1G_MICBIAS_CTRL_MIC1AMPEN, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("Mic2 Amplifier", SUN8I_ADDA_MIC2G_CTRL,
+			 SUN8I_ADDA_MIC2G_CTRL_MIC2AMPEN, 0, NULL, 0),
+
+	/* Mixers */
+	SND_SOC_DAPM_MIXER("Left Mixer", SUN8I_ADDA_DAC_PA_SRC,
+			   SUN8I_ADDA_DAC_PA_SRC_LMIXEN, 0,
+			   sun8i_codec_mixer_controls,
+			   ARRAY_SIZE(sun8i_codec_mixer_controls)),
+	SND_SOC_DAPM_MIXER("Right Mixer", SUN8I_ADDA_DAC_PA_SRC,
+			   SUN8I_ADDA_DAC_PA_SRC_RMIXEN, 0,
+			   sun8i_codec_mixer_controls,
+			   ARRAY_SIZE(sun8i_codec_mixer_controls)),
+	SND_SOC_DAPM_MIXER("Left ADC Mixer", SUN8I_ADDA_ADC_AP_EN,
+			   SUN8I_ADDA_ADC_AP_EN_ADCLEN, 0,
+			   sun8i_codec_adc_mixer_controls,
+			   ARRAY_SIZE(sun8i_codec_adc_mixer_controls)),
+	SND_SOC_DAPM_MIXER("Right ADC Mixer", SUN8I_ADDA_ADC_AP_EN,
+			   SUN8I_ADDA_ADC_AP_EN_ADCREN, 0,
+			   sun8i_codec_adc_mixer_controls,
+			   ARRAY_SIZE(sun8i_codec_adc_mixer_controls)),
+};
+
+static const struct snd_soc_dapm_route sun8i_codec_common_routes[] = {
+	/* Microphone Routes */
+	{ "Mic1 Amplifier", NULL, "MIC1"},
+	{ "Mic2 Amplifier", NULL, "MIC2"},
+
+	/* Left Mixer Routes */
+	{ "Left Mixer", "DAC Playback Switch", "Left DAC" },
+	{ "Left Mixer", "DAC Reversed Playback Switch", "Right DAC" },
+	{ "Left Mixer", "Line In Playback Switch", "LINEIN" },
+	{ "Left Mixer", "Mic1 Playback Switch", "Mic1 Amplifier" },
+	{ "Left Mixer", "Mic2 Playback Switch", "Mic2 Amplifier" },
+
+	/* Right Mixer Routes */
+	{ "Right Mixer", "DAC Playback Switch", "Right DAC" },
+	{ "Right Mixer", "DAC Reversed Playback Switch", "Left DAC" },
+	{ "Right Mixer", "Line In Playback Switch", "LINEIN" },
+	{ "Right Mixer", "Mic1 Playback Switch", "Mic1 Amplifier" },
+	{ "Right Mixer", "Mic2 Playback Switch", "Mic2 Amplifier" },
+
+	/* Left ADC Mixer Routes */
+	{ "Left ADC Mixer", "Mixer Capture Switch", "Left Mixer" },
+	{ "Left ADC Mixer", "Mixer Reversed Capture Switch", "Right Mixer" },
+	{ "Left ADC Mixer", "Line In Capture Switch", "LINEIN" },
+	{ "Left ADC Mixer", "Mic1 Capture Switch", "Mic1 Amplifier" },
+	{ "Left ADC Mixer", "Mic2 Capture Switch", "Mic2 Amplifier" },
+
+	/* Right ADC Mixer Routes */
+	{ "Right ADC Mixer", "Mixer Capture Switch", "Right Mixer" },
+	{ "Right ADC Mixer", "Mixer Reversed Capture Switch", "Left Mixer" },
+	{ "Right ADC Mixer", "Line In Capture Switch", "LINEIN" },
+	{ "Right ADC Mixer", "Mic1 Capture Switch", "Mic1 Amplifier" },
+	{ "Right ADC Mixer", "Mic2 Capture Switch", "Mic2 Amplifier" },
+
+	/* ADC Routes */
+	{ "Left ADC", NULL, "Left ADC Mixer" },
+	{ "Right ADC", NULL, "Right ADC Mixer" },
+};
+
+/* headphone specific controls, widgets, and routes */
+static const DECLARE_TLV_DB_SCALE(sun8i_codec_hp_vol_scale, -6300, 100, 1);
+static const struct snd_kcontrol_new sun8i_codec_headphone_controls[] = {
+	SOC_SINGLE_TLV("Headphone Playback Volume",
+		       SUN8I_ADDA_HP_VOLC,
+		       SUN8I_ADDA_HP_VOLC_HP_VOL, 0x3f, 0,
+		       sun8i_codec_hp_vol_scale),
+	SOC_DOUBLE("Headphone Playback Switch",
+		   SUN8I_ADDA_DAC_PA_SRC,
+		   SUN8I_ADDA_DAC_PA_SRC_LHPPAMUTE,
+		   SUN8I_ADDA_DAC_PA_SRC_RHPPAMUTE, 1, 0),
+};
+
+static const char * const sun8i_codec_hp_src_enum_text[] = {
+	"DAC", "Mixer",
+};
+
+static SOC_ENUM_DOUBLE_DECL(sun8i_codec_hp_src_enum,
+			    SUN8I_ADDA_DAC_PA_SRC,
+			    SUN8I_ADDA_DAC_PA_SRC_LHPIS,
+			    SUN8I_ADDA_DAC_PA_SRC_RHPIS,
+			    sun8i_codec_hp_src_enum_text);
+
+static const struct snd_kcontrol_new sun8i_codec_hp_src[] = {
+	SOC_DAPM_ENUM("Headphone Source Playback Route",
+		      sun8i_codec_hp_src_enum),
+};
+
+static const struct snd_soc_dapm_widget sun8i_codec_headphone_widgets[] = {
+	SND_SOC_DAPM_MUX("Headphone Source Playback Route",
+			 SND_SOC_NOPM, 0, 0, sun8i_codec_hp_src),
+	SND_SOC_DAPM_OUT_DRV("Headphone Amp", SUN8I_ADDA_PAEN_HP_CTRL,
+			     SUN8I_ADDA_PAEN_HP_CTRL_HPPAEN, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("HPCOM Protection", SUN8I_ADDA_PAEN_HP_CTRL,
+			    SUN8I_ADDA_PAEN_HP_CTRL_COMPTEN, 0, NULL, 0),
+	SND_SOC_DAPM_REG(snd_soc_dapm_supply, "HPCOM", SUN8I_ADDA_PAEN_HP_CTRL,
+			 SUN8I_ADDA_PAEN_HP_CTRL_HPCOM_FC, 0x3, 0x3, 0),
+	SND_SOC_DAPM_OUTPUT("HP"),
+};
+
+static const struct snd_soc_dapm_route sun8i_codec_headphone_routes[] = {
+	{ "Headphone Source Playback Route", "DAC", "Left DAC" },
+	{ "Headphone Source Playback Route", "DAC", "Right DAC" },
+	{ "Headphone Source Playback Route", "Mixer", "Left Mixer" },
+	{ "Headphone Source Playback Route", "Mixer", "Right Mixer" },
+	{ "Headphone Amp", NULL, "Headphone Source Playback Route" },
+	{ "HPCOM", NULL, "HPCOM Protection" },
+	{ "HP", NULL, "Headphone Amp" },
+};
+
+static int sun8i_codec_add_headphone(struct snd_soc_component *cmpnt)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(cmpnt);
+	struct device *dev = cmpnt->dev;
+	int ret;
+
+	ret = snd_soc_add_component_controls(cmpnt,
+					     sun8i_codec_headphone_controls,
+					     ARRAY_SIZE(sun8i_codec_headphone_controls));
+	if (ret) {
+		dev_err(dev, "Failed to add Headphone controls: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_dapm_new_controls(dapm, sun8i_codec_headphone_widgets,
+					ARRAY_SIZE(sun8i_codec_headphone_widgets));
+	if (ret) {
+		dev_err(dev, "Failed to add Headphone DAPM widgets: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_dapm_add_routes(dapm, sun8i_codec_headphone_routes,
+				      ARRAY_SIZE(sun8i_codec_headphone_routes));
+	if (ret) {
+		dev_err(dev, "Failed to add Headphone DAPM routes: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+/* hmic specific widget */
+static const struct snd_soc_dapm_widget sun8i_codec_hmic_widgets[] = {
+	SND_SOC_DAPM_SUPPLY("HBIAS", SUN8I_ADDA_MIC1G_MICBIAS_CTRL,
+			    SUN8I_ADDA_MIC1G_MICBIAS_CTRL_HMICBIASEN,
+			    0, NULL, 0),
+};
+
+static int sun8i_codec_add_hmic(struct snd_soc_component *cmpnt)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(cmpnt);
+	struct device *dev = cmpnt->dev;
+	int ret;
+
+	ret = snd_soc_dapm_new_controls(dapm, sun8i_codec_hmic_widgets,
+					ARRAY_SIZE(sun8i_codec_hmic_widgets));
+	if (ret)
+		dev_err(dev, "Failed to add Mic3 DAPM widgets: %d\n", ret);
+
+	return ret;
+}
+
+/* line out specific controls, widgets and routes */
+static const DECLARE_TLV_DB_RANGE(sun8i_codec_lineout_vol_scale,
+	0, 1, TLV_DB_SCALE_ITEM(TLV_DB_GAIN_MUTE, 0, 1),
+	2, 31, TLV_DB_SCALE_ITEM(-4350, 150, 0),
+);
+static const struct snd_kcontrol_new sun8i_codec_lineout_controls[] = {
+	SOC_SINGLE_TLV("Line Out Playback Volume",
+		       SUN8I_ADDA_PHONE_GAIN_CTRL,
+		       SUN8I_ADDA_PHONE_GAIN_CTRL_LINEOUT_VOL, 0x1f, 0,
+		       sun8i_codec_lineout_vol_scale),
+	SOC_DOUBLE("Line Out Playback Switch",
+		   SUN8I_ADDA_MIC2G_CTRL,
+		   SUN8I_ADDA_MIC2G_CTRL_LINEOUTLEN,
+		   SUN8I_ADDA_MIC2G_CTRL_LINEOUTREN, 1, 0),
+};
+
+static const char * const sun8i_codec_lineout_src_enum_text[] = {
+	"Stereo", "Mono Differential",
+};
+
+static SOC_ENUM_DOUBLE_DECL(sun8i_codec_lineout_src_enum,
+			    SUN8I_ADDA_MIC2G_CTRL,
+			    SUN8I_ADDA_MIC2G_CTRL_LINEOUTLSRC,
+			    SUN8I_ADDA_MIC2G_CTRL_LINEOUTRSRC,
+			    sun8i_codec_lineout_src_enum_text);
+
+static const struct snd_kcontrol_new sun8i_codec_lineout_src[] = {
+	SOC_DAPM_ENUM("Line Out Source Playback Route",
+		      sun8i_codec_lineout_src_enum),
+};
+
+static const struct snd_soc_dapm_widget sun8i_codec_lineout_widgets[] = {
+	SND_SOC_DAPM_MUX("Line Out Source Playback Route",
+			 SND_SOC_NOPM, 0, 0, sun8i_codec_lineout_src),
+	/* It is unclear if this is a buffer or gate, model it as a supply */
+	SND_SOC_DAPM_SUPPLY("Line Out Enable", SUN8I_ADDA_PAEN_HP_CTRL,
+			    SUN8I_ADDA_PAEN_HP_CTRL_LINEOUTEN, 0, NULL, 0),
+	SND_SOC_DAPM_OUTPUT("LINEOUT"),
+};
+
+static const struct snd_soc_dapm_route sun8i_codec_lineout_routes[] = {
+	{ "Line Out Source Playback Route", "Stereo", "Left Mixer" },
+	{ "Line Out Source Playback Route", "Stereo", "Right Mixer" },
+	{ "Line Out Source Playback Route", "Mono Differential", "Left Mixer" },
+	{ "Line Out Source Playback Route", "Mono Differential", "Right Mixer" },
+	{ "LINEOUT", NULL, "Line Out Source Playback Route" },
+	{ "LINEOUT", NULL, "Line Out Enable", },
+};
+
+static int sun8i_codec_add_lineout(struct snd_soc_component *cmpnt)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(cmpnt);
+	struct device *dev = cmpnt->dev;
+	int ret;
+
+	ret = snd_soc_add_component_controls(cmpnt,
+					     sun8i_codec_lineout_controls,
+					     ARRAY_SIZE(sun8i_codec_lineout_controls));
+	if (ret) {
+		dev_err(dev, "Failed to add Line Out controls: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_dapm_new_controls(dapm, sun8i_codec_lineout_widgets,
+					ARRAY_SIZE(sun8i_codec_lineout_widgets));
+	if (ret) {
+		dev_err(dev, "Failed to add Line Out DAPM widgets: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_dapm_add_routes(dapm, sun8i_codec_lineout_routes,
+				      ARRAY_SIZE(sun8i_codec_lineout_routes));
+	if (ret) {
+		dev_err(dev, "Failed to add Line Out DAPM routes: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+struct sun8i_codec_analog_quirks {
+	bool has_headphone;
+	bool has_hmic;
+	bool has_lineout;
+};
+
+static const struct sun8i_codec_analog_quirks sun8i_a23_quirks = {
+	.has_headphone	= true,
+	.has_hmic	= true,
+};
+
+static const struct sun8i_codec_analog_quirks sun8i_h3_quirks = {
+	.has_lineout	= true,
+};
+
+static int sun8i_codec_analog_cmpnt_probe(struct snd_soc_component *cmpnt)
+{
+	struct device *dev = cmpnt->dev;
+	const struct sun8i_codec_analog_quirks *quirks;
+	int ret;
+
+	/*
+	 * This would never return NULL unless someone directly registers a
+	 * platform device matching this driver's name, without specifying a
+	 * device tree node.
+	 */
+	quirks = of_device_get_match_data(dev);
+
+	/* Add controls, widgets, and routes for individual features */
+
+	if (quirks->has_headphone) {
+		ret = sun8i_codec_add_headphone(cmpnt);
+		if (ret)
+			return ret;
+	}
+
+	if (quirks->has_hmic) {
+		ret = sun8i_codec_add_hmic(cmpnt);
+		if (ret)
+			return ret;
+	}
+
+	if (quirks->has_lineout) {
+		ret = sun8i_codec_add_lineout(cmpnt);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_component_driver sun8i_codec_analog_cmpnt_drv = {
+	.controls		= sun8i_codec_common_controls,
+	.num_controls		= ARRAY_SIZE(sun8i_codec_common_controls),
+	.dapm_widgets		= sun8i_codec_common_widgets,
+	.num_dapm_widgets	= ARRAY_SIZE(sun8i_codec_common_widgets),
+	.dapm_routes		= sun8i_codec_common_routes,
+	.num_dapm_routes	= ARRAY_SIZE(sun8i_codec_common_routes),
+	.probe			= sun8i_codec_analog_cmpnt_probe,
+};
+
+static const struct of_device_id sun8i_codec_analog_of_match[] = {
+	{
+		.compatible = "allwinner,sun8i-a23-codec-analog",
+		.data = &sun8i_a23_quirks,
+	},
+	{
+		.compatible = "allwinner,sun8i-h3-codec-analog",
+		.data = &sun8i_h3_quirks,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, sun8i_codec_analog_of_match);
+
+static int sun8i_codec_analog_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	struct regmap *regmap;
+	void __iomem *base;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base)) {
+		dev_err(&pdev->dev, "Failed to map the registers\n");
+		return PTR_ERR(base);
+	}
+
+	regmap = devm_regmap_init(&pdev->dev, NULL, base, &adda_pr_regmap_cfg);
+	if (IS_ERR(regmap)) {
+		dev_err(&pdev->dev, "Failed to create regmap\n");
+		return PTR_ERR(regmap);
+	}
+
+	return devm_snd_soc_register_component(&pdev->dev,
+					       &sun8i_codec_analog_cmpnt_drv,
+					       NULL, 0);
+}
+
+static struct platform_driver sun8i_codec_analog_driver = {
+	.driver = {
+		.name = "sun8i-codec-analog",
+		.of_match_table = sun8i_codec_analog_of_match,
+	},
+	.probe = sun8i_codec_analog_probe,
+};
+module_platform_driver(sun8i_codec_analog_driver);
+
+MODULE_DESCRIPTION("Allwinner internal codec analog controls driver");
+MODULE_AUTHOR("Chen-Yu Tsai <wens@csie.org>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:sun8i-codec-analog");
diff --git a/sound/soc/tegra/tegra_alc5632.c b/sound/soc/tegra/tegra_alc5632.c
index deb597f7c302..eead6e7f205b 100644
--- a/sound/soc/tegra/tegra_alc5632.c
+++ b/sound/soc/tegra/tegra_alc5632.c
@@ -65,7 +65,7 @@ static int tegra_alc5632_asoc_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static struct snd_soc_ops tegra_alc5632_asoc_ops = {
+static const struct snd_soc_ops tegra_alc5632_asoc_ops = {
 	.hw_params = tegra_alc5632_asoc_hw_params,
 };
 
diff --git a/sound/soc/tegra/tegra_max98090.c b/sound/soc/tegra/tegra_max98090.c
index 902da36581d1..a403db6d563e 100644
--- a/sound/soc/tegra/tegra_max98090.c
+++ b/sound/soc/tegra/tegra_max98090.c
@@ -93,7 +93,7 @@ static int tegra_max98090_asoc_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static struct snd_soc_ops tegra_max98090_ops = {
+static const struct snd_soc_ops tegra_max98090_ops = {
 	.hw_params = tegra_max98090_asoc_hw_params,
 };
 
diff --git a/sound/soc/tegra/tegra_rt5640.c b/sound/soc/tegra/tegra_rt5640.c
index e5ef4e9c4ac5..25b9fc03ba62 100644
--- a/sound/soc/tegra/tegra_rt5640.c
+++ b/sound/soc/tegra/tegra_rt5640.c
@@ -76,7 +76,7 @@ static int tegra_rt5640_asoc_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static struct snd_soc_ops tegra_rt5640_ops = {
+static const struct snd_soc_ops tegra_rt5640_ops = {
 	.hw_params = tegra_rt5640_asoc_hw_params,
 };
 
diff --git a/sound/soc/tegra/tegra_rt5677.c b/sound/soc/tegra/tegra_rt5677.c
index 1470873ecde6..ebf58d0e0f10 100644
--- a/sound/soc/tegra/tegra_rt5677.c
+++ b/sound/soc/tegra/tegra_rt5677.c
@@ -93,7 +93,7 @@ static int tegra_rt5677_event_hp(struct snd_soc_dapm_widget *w,
 	return 0;
 }
 
-static struct snd_soc_ops tegra_rt5677_ops = {
+static const struct snd_soc_ops tegra_rt5677_ops = {
 	.hw_params = tegra_rt5677_asoc_hw_params,
 };
 
diff --git a/sound/soc/tegra/tegra_sgtl5000.c b/sound/soc/tegra/tegra_sgtl5000.c
index 1e76869dd488..4bbab098f50b 100644
--- a/sound/soc/tegra/tegra_sgtl5000.c
+++ b/sound/soc/tegra/tegra_sgtl5000.c
@@ -82,7 +82,7 @@ static int tegra_sgtl5000_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static struct snd_soc_ops tegra_sgtl5000_ops = {
+static const struct snd_soc_ops tegra_sgtl5000_ops = {
 	.hw_params = tegra_sgtl5000_hw_params,
 };
 
diff --git a/sound/soc/tegra/tegra_wm8753.c b/sound/soc/tegra/tegra_wm8753.c
index f0cd01dbfc38..bdedd1028569 100644
--- a/sound/soc/tegra/tegra_wm8753.c
+++ b/sound/soc/tegra/tegra_wm8753.c
@@ -89,7 +89,7 @@ static int tegra_wm8753_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static struct snd_soc_ops tegra_wm8753_ops = {
+static const struct snd_soc_ops tegra_wm8753_ops = {
 	.hw_params = tegra_wm8753_hw_params,
 };
 
diff --git a/sound/soc/tegra/tegra_wm8903.c b/sound/soc/tegra/tegra_wm8903.c
index e485278e027a..2013e9c4bba0 100644
--- a/sound/soc/tegra/tegra_wm8903.c
+++ b/sound/soc/tegra/tegra_wm8903.c
@@ -96,7 +96,7 @@ static int tegra_wm8903_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static struct snd_soc_ops tegra_wm8903_ops = {
+static const struct snd_soc_ops tegra_wm8903_ops = {
 	.hw_params = tegra_wm8903_hw_params,
 };
 
diff --git a/sound/soc/tegra/trimslice.c b/sound/soc/tegra/trimslice.c
index 2cea203c4f5f..870f84ab5005 100644
--- a/sound/soc/tegra/trimslice.c
+++ b/sound/soc/tegra/trimslice.c
@@ -74,7 +74,7 @@ static int trimslice_asoc_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
-static struct snd_soc_ops trimslice_asoc_ops = {
+static const struct snd_soc_ops trimslice_asoc_ops = {
 	.hw_params = trimslice_asoc_hw_params,
 };
 
diff --git a/sound/soc/zte/Kconfig b/sound/soc/zte/Kconfig
index c47eb25e441f..6d8a90d36315 100644
--- a/sound/soc/zte/Kconfig
+++ b/sound/soc/zte/Kconfig
@@ -1,17 +1,17 @@
-config ZX296702_SPDIF
-	tristate "ZX296702 spdif"
-	depends on SOC_ZX296702 || COMPILE_TEST
+config ZX_SPDIF
+	tristate "ZTE ZX SPDIF Driver Support"
+	depends on ARCH_ZX || COMPILE_TEST
 	depends on COMMON_CLK
 	select SND_SOC_GENERIC_DMAENGINE_PCM
 	help
 	  Say Y or M if you want to add support for codecs attached to the
-	  zx296702 spdif interface
+	  ZTE ZX SPDIF interface
 
-config ZX296702_I2S
-	tristate "ZX296702 i2s"
-	depends on SOC_ZX296702 || COMPILE_TEST
+config ZX_I2S
+	tristate "ZTE ZX I2S Driver Support"
+	depends on ARCH_ZX || COMPILE_TEST
 	depends on COMMON_CLK
 	select SND_SOC_GENERIC_DMAENGINE_PCM
 	help
 	  Say Y or M if you want to add support for codecs attached to the
-	  zx296702 i2s interface
+	  ZTE ZX I2S interface
diff --git a/sound/soc/zte/Makefile b/sound/soc/zte/Makefile
index 254ed2c8c1a0..77768f5fd10c 100644
--- a/sound/soc/zte/Makefile
+++ b/sound/soc/zte/Makefile
@@ -1,2 +1,2 @@
-obj-$(CONFIG_ZX296702_SPDIF)	+= zx296702-spdif.o
-obj-$(CONFIG_ZX296702_I2S)	+= zx296702-i2s.o
+obj-$(CONFIG_ZX_SPDIF)	+= zx-spdif.o
+obj-$(CONFIG_ZX_I2S)	+= zx-i2s.o
diff --git a/sound/soc/zte/zx296702-i2s.c b/sound/soc/zte/zx-i2s.c
index 1cad93dc1fcf..1cad93dc1fcf 100644
--- a/sound/soc/zte/zx296702-i2s.c
+++ b/sound/soc/zte/zx-i2s.c
diff --git a/sound/soc/zte/zx296702-spdif.c b/sound/soc/zte/zx-spdif.c
index 26265ce4caca..9fa6463ce5d7 100644
--- a/sound/soc/zte/zx296702-spdif.c
+++ b/sound/soc/zte/zx-spdif.c
@@ -71,7 +71,7 @@
 #define ZX_VALID_RIGHT_TRACK		(2 << 0)
 #define ZX_VALID_TRACK_MASK		(3 << 0)
 
-#define ZX_SPDIF_CLK_RAT		(4 * 32)
+#define ZX_SPDIF_CLK_RAT		(2 * 32)
 
 struct zx_spdif_info {
 	struct snd_dmaengine_dai_dma_data	dma_data;
diff --git a/sound/usb/card.c b/sound/usb/card.c
index 2ddc034673a8..f36cb068dad3 100644
--- a/sound/usb/card.c
+++ b/sound/usb/card.c
@@ -206,7 +206,6 @@ static int snd_usb_create_stream(struct snd_usb_audio *chip, int ctrlif, int int
 	if (! snd_usb_parse_audio_interface(chip, interface)) {
 		usb_set_interface(dev, interface, 0); /* reset the current interface */
 		usb_driver_claim_interface(&usb_audio_driver, iface, (void *)-1L);
-		return -EINVAL;
 	}
 
 	return 0;
diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c
index c470251cea4b..a2cdf3370afe 100644
--- a/sound/usb/endpoint.c
+++ b/sound/usb/endpoint.c
@@ -384,6 +384,9 @@ static void snd_complete_urb(struct urb *urb)
 	if (unlikely(atomic_read(&ep->chip->shutdown)))
 		goto exit_clear;
 
+	if (unlikely(!test_bit(EP_FLAG_RUNNING, &ep->flags)))
+		goto exit_clear;
+
 	if (usb_pipeout(ep->pipe)) {
 		retire_outbound_urb(ep, ctx);
 		/* can be stopped during retire callback */
@@ -534,6 +537,11 @@ static int wait_clear_urbs(struct snd_usb_endpoint *ep)
 			alive, ep->ep_num);
 	clear_bit(EP_FLAG_STOPPING, &ep->flags);
 
+	ep->data_subs = NULL;
+	ep->sync_slave = NULL;
+	ep->retire_data_urb = NULL;
+	ep->prepare_data_urb = NULL;
+
 	return 0;
 }
 
@@ -630,10 +638,24 @@ static int data_ep_set_params(struct snd_usb_endpoint *ep,
 
 	ep->datainterval = fmt->datainterval;
 	ep->stride = frame_bits >> 3;
-	ep->silence_value = pcm_format == SNDRV_PCM_FORMAT_U8 ? 0x80 : 0;
 
-	/* assume max. frequency is 25% higher than nominal */
-	ep->freqmax = ep->freqn + (ep->freqn >> 2);
+	switch (pcm_format) {
+	case SNDRV_PCM_FORMAT_U8:
+		ep->silence_value = 0x80;
+		break;
+	case SNDRV_PCM_FORMAT_DSD_U8:
+	case SNDRV_PCM_FORMAT_DSD_U16_LE:
+	case SNDRV_PCM_FORMAT_DSD_U32_LE:
+	case SNDRV_PCM_FORMAT_DSD_U16_BE:
+	case SNDRV_PCM_FORMAT_DSD_U32_BE:
+		ep->silence_value = 0x69;
+		break;
+	default:
+		ep->silence_value = 0;
+	}
+
+	/* assume max. frequency is 50% higher than nominal */
+	ep->freqmax = ep->freqn + (ep->freqn >> 1);
 	/* Round up freqmax to nearest integer in order to calculate maximum
 	 * packet size, which must represent a whole number of frames.
 	 * This is accomplished by adding 0x0.ffff before converting the
@@ -1006,10 +1028,6 @@ void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep)
 
 	if (--ep->use_count == 0) {
 		deactivate_urbs(ep, false);
-		ep->data_subs = NULL;
-		ep->sync_slave = NULL;
-		ep->retire_data_urb = NULL;
-		ep->prepare_data_urb = NULL;
 		set_bit(EP_FLAG_STOPPING, &ep->flags);
 	}
 }
diff --git a/sound/usb/hiface/pcm.c b/sound/usb/hiface/pcm.c
index 2c44139b4041..33db205dd12b 100644
--- a/sound/usb/hiface/pcm.c
+++ b/sound/usb/hiface/pcm.c
@@ -445,6 +445,8 @@ static int hiface_pcm_prepare(struct snd_pcm_substream *alsa_sub)
 
 	mutex_lock(&rt->stream_mutex);
 
+	hiface_pcm_stream_stop(rt);
+
 	sub->dma_off = 0;
 	sub->period_off = 0;
 
diff --git a/sound/usb/line6/driver.h b/sound/usb/line6/driver.h
index 7e3a3aada222..a5c2e9ae5f17 100644
--- a/sound/usb/line6/driver.h
+++ b/sound/usb/line6/driver.h
@@ -98,10 +98,11 @@ struct line6_properties {
 
 	int altsetting;
 
-	unsigned ep_ctrl_r;
-	unsigned ep_ctrl_w;
-	unsigned ep_audio_r;
-	unsigned ep_audio_w;
+	unsigned int ctrl_if;
+	unsigned int ep_ctrl_r;
+	unsigned int ep_ctrl_w;
+	unsigned int ep_audio_r;
+	unsigned int ep_audio_w;
 };
 
 /* Capability bits */
diff --git a/sound/usb/line6/podhd.c b/sound/usb/line6/podhd.c
index 49cd4a65e390..6ab23e5aee71 100644
--- a/sound/usb/line6/podhd.c
+++ b/sound/usb/line6/podhd.c
@@ -153,6 +153,7 @@ static struct line6_pcm_properties podx3_pcm_properties = {
 			    .rats = &podhd_ratden},
 	.bytes_per_channel = 3 /* SNDRV_PCM_FMTBIT_S24_3LE */
 };
+static struct usb_driver podhd_driver;
 
 static void podhd_startup_start_workqueue(unsigned long data);
 static void podhd_startup_workqueue(struct work_struct *work);
@@ -291,8 +292,14 @@ static void podhd_disconnect(struct usb_line6 *line6)
 	struct usb_line6_podhd *pod = (struct usb_line6_podhd *)line6;
 
 	if (pod->line6.properties->capabilities & LINE6_CAP_CONTROL) {
+		struct usb_interface *intf;
+
 		del_timer_sync(&pod->startup_timer);
 		cancel_work_sync(&pod->startup_work);
+
+		intf = usb_ifnum_to_if(line6->usbdev,
+					pod->line6.properties->ctrl_if);
+		usb_driver_release_interface(&podhd_driver, intf);
 	}
 }
 
@@ -304,10 +311,27 @@ static int podhd_init(struct usb_line6 *line6,
 {
 	int err;
 	struct usb_line6_podhd *pod = (struct usb_line6_podhd *) line6;
+	struct usb_interface *intf;
 
 	line6->disconnect = podhd_disconnect;
 
 	if (pod->line6.properties->capabilities & LINE6_CAP_CONTROL) {
+		/* claim the data interface */
+		intf = usb_ifnum_to_if(line6->usbdev,
+					pod->line6.properties->ctrl_if);
+		if (!intf) {
+			dev_err(pod->line6.ifcdev, "interface %d not found\n",
+				pod->line6.properties->ctrl_if);
+			return -ENODEV;
+		}
+
+		err = usb_driver_claim_interface(&podhd_driver, intf, NULL);
+		if (err != 0) {
+			dev_err(pod->line6.ifcdev, "can't claim interface %d, error %d\n",
+				pod->line6.properties->ctrl_if, err);
+			return err;
+		}
+
 		/* create sysfs entries: */
 		err = snd_card_add_dev_attr(line6->card, &podhd_dev_attr_group);
 		if (err < 0)
@@ -406,6 +430,7 @@ static const struct line6_properties podhd_properties_table[] = {
 		.altsetting = 1,
 		.ep_ctrl_r = 0x81,
 		.ep_ctrl_w = 0x01,
+		.ctrl_if = 1,
 		.ep_audio_r = 0x86,
 		.ep_audio_w = 0x02,
 	},
@@ -417,6 +442,7 @@ static const struct line6_properties podhd_properties_table[] = {
 		.altsetting = 1,
 		.ep_ctrl_r = 0x81,
 		.ep_ctrl_w = 0x01,
+		.ctrl_if = 1,
 		.ep_audio_r = 0x86,
 		.ep_audio_w = 0x02,
 	},
diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c
index 2f8c388ef84f..4703caea56b2 100644
--- a/sound/usb/mixer.c
+++ b/sound/usb/mixer.c
@@ -932,9 +932,10 @@ static void volume_control_quirks(struct usb_mixer_elem_info *cval,
 	case USB_ID(0x046d, 0x0826): /* HD Webcam c525 */
 	case USB_ID(0x046d, 0x08ca): /* Logitech Quickcam Fusion */
 	case USB_ID(0x046d, 0x0991):
+	case USB_ID(0x046d, 0x09a2): /* QuickCam Communicate Deluxe/S7500 */
 	/* Most audio usb devices lie about volume resolution.
 	 * Most Logitech webcams have res = 384.
-	 * Proboly there is some logitech magic behind this number --fishor
+	 * Probably there is some logitech magic behind this number --fishor
 	 */
 		if (!strcmp(kctl->id.name, "Mic Capture Volume")) {
 			usb_audio_info(chip,
diff --git a/sound/usb/pcm.c b/sound/usb/pcm.c
index 44d178ee9177..34c6d4f2c0b6 100644
--- a/sound/usb/pcm.c
+++ b/sound/usb/pcm.c
@@ -348,6 +348,16 @@ static int set_sync_ep_implicit_fb_quirk(struct snd_usb_substream *subs,
 
 		alts = &iface->altsetting[1];
 		goto add_sync_ep;
+	case USB_ID(0x2466, 0x8003):
+		ep = 0x86;
+		iface = usb_ifnum_to_if(dev, 2);
+
+		if (!iface || iface->num_altsetting == 0)
+			return -EINVAL;
+
+		alts = &iface->altsetting[1];
+		goto add_sync_ep;
+
 	}
 	if (attr == USB_ENDPOINT_SYNC_ASYNC &&
 	    altsd->bInterfaceClass == USB_CLASS_VENDOR_SPEC &&
@@ -806,17 +816,18 @@ static int snd_usb_pcm_prepare(struct snd_pcm_substream *substream)
 	if (ret < 0)
 		goto unlock;
 
-	iface = usb_ifnum_to_if(subs->dev, subs->cur_audiofmt->iface);
-	alts = &iface->altsetting[subs->cur_audiofmt->altset_idx];
-	ret = snd_usb_init_sample_rate(subs->stream->chip,
-				       subs->cur_audiofmt->iface,
-				       alts,
-				       subs->cur_audiofmt,
-				       subs->cur_rate);
-	if (ret < 0)
-		goto unlock;
-
 	if (subs->need_setup_ep) {
+
+		iface = usb_ifnum_to_if(subs->dev, subs->cur_audiofmt->iface);
+		alts = &iface->altsetting[subs->cur_audiofmt->altset_idx];
+		ret = snd_usb_init_sample_rate(subs->stream->chip,
+					       subs->cur_audiofmt->iface,
+					       alts,
+					       subs->cur_audiofmt,
+					       subs->cur_rate);
+		if (ret < 0)
+			goto unlock;
+
 		ret = configure_endpoint(subs);
 		if (ret < 0)
 			goto unlock;
diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c
index 2782155ae3ce..b3fd2382fdd9 100644
--- a/sound/usb/quirks.c
+++ b/sound/usb/quirks.c
@@ -1165,6 +1165,18 @@ static bool is_marantz_denon_dac(unsigned int id)
 	return false;
 }
 
+/* TEAC UD-501/UD-503/NT-503 USB DACs need a vendor cmd to switch
+ * between PCM/DOP and native DSD mode
+ */
+static bool is_teac_50X_dac(unsigned int id)
+{
+	switch (id) {
+	case USB_ID(0x0644, 0x8043): /* TEAC UD-501/UD-503/NT-503 */
+		return true;
+	}
+	return false;
+}
+
 int snd_usb_select_mode_quirk(struct snd_usb_substream *subs,
 			      struct audioformat *fmt)
 {
@@ -1192,6 +1204,26 @@ int snd_usb_select_mode_quirk(struct snd_usb_substream *subs,
 			break;
 		}
 		mdelay(20);
+	} else if (is_teac_50X_dac(subs->stream->chip->usb_id)) {
+		/* Vendor mode switch cmd is required. */
+		switch (fmt->altsetting) {
+		case 3: /* DSD mode (DSD_U32) requested */
+			err = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), 0,
+					      USB_DIR_OUT|USB_TYPE_VENDOR|USB_RECIP_INTERFACE,
+					      1, 1, NULL, 0);
+			if (err < 0)
+				return err;
+			break;
+
+		case 2: /* PCM or DOP mode (S32) requested */
+		case 1: /* PCM mode (S16) requested */
+			err = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), 0,
+					      USB_DIR_OUT|USB_TYPE_VENDOR|USB_RECIP_INTERFACE,
+					      0, 1, NULL, 0);
+			if (err < 0)
+				return err;
+			break;
+		}
 	}
 	return 0;
 }
@@ -1337,5 +1369,11 @@ u64 snd_usb_interface_dsd_format_quirks(struct snd_usb_audio *chip,
 			return SNDRV_PCM_FMTBIT_DSD_U32_BE;
 	}
 
+	/* TEAC devices with USB DAC functionality */
+	if (is_teac_50X_dac(chip->usb_id)) {
+		if (fp->altsetting == 3)
+			return SNDRV_PCM_FMTBIT_DSD_U32_BE;
+	}
+
 	return 0;
 }
diff --git a/tools/Makefile b/tools/Makefile
index daa8fb3e4363..00caacd3ed92 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -17,6 +17,7 @@ help:
 	@echo '  hv                     - tools used when in Hyper-V clients'
 	@echo '  iio                    - IIO tools'
 	@echo '  kvm_stat               - top-like utility for displaying kvm statistics'
+	@echo '  leds                   - LEDs  tools'
 	@echo '  lguest                 - a minimal 32-bit x86 hypervisor'
 	@echo '  net                    - misc networking tools'
 	@echo '  perf                   - Linux performance measurement and analysis tool'
@@ -56,7 +57,7 @@ acpi: FORCE
 cpupower: FORCE
 	$(call descend,power/$@)
 
-cgroup firewire hv guest spi usb virtio vm net iio gpio objtool: FORCE
+cgroup firewire hv guest spi usb virtio vm net iio gpio objtool leds: FORCE
 	$(call descend,$@)
 
 liblockdep: FORCE
@@ -126,7 +127,7 @@ acpi_clean:
 cpupower_clean:
 	$(call descend,power/cpupower,clean)
 
-cgroup_clean hv_clean firewire_clean lguest_clean spi_clean usb_clean virtio_clean vm_clean net_clean iio_clean gpio_clean objtool_clean:
+cgroup_clean hv_clean firewire_clean lguest_clean spi_clean usb_clean virtio_clean vm_clean net_clean iio_clean gpio_clean objtool_clean leds_clean:
 	$(call descend,$(@:_clean=),clean)
 
 liblockdep_clean:
@@ -164,6 +165,6 @@ clean: acpi_clean cgroup_clean cpupower_clean hv_clean firewire_clean lguest_cle
 		perf_clean selftests_clean turbostat_clean spi_clean usb_clean virtio_clean \
 		vm_clean net_clean iio_clean x86_energy_perf_policy_clean tmon_clean \
 		freefall_clean build_clean libbpf_clean libsubcmd_clean liblockdep_clean \
-		gpio_clean objtool_clean
+		gpio_clean objtool_clean leds_clean
 
 .PHONY: FORCE
diff --git a/tools/gpio/gpio-hammer.c b/tools/gpio/gpio-hammer.c
index 37b3f141053d..f1eab587dfea 100644
--- a/tools/gpio/gpio-hammer.c
+++ b/tools/gpio/gpio-hammer.c
@@ -23,54 +23,31 @@
 #include <getopt.h>
 #include <sys/ioctl.h>
 #include <linux/gpio.h>
+#include "gpio-utils.h"
 
 int hammer_device(const char *device_name, unsigned int *lines, int nlines,
 		  unsigned int loops)
 {
-	struct gpiohandle_request req;
 	struct gpiohandle_data data;
-	char *chrdev_name;
 	char swirr[] = "-\\|/";
 	int fd;
 	int ret;
 	int i, j;
 	unsigned int iteration = 0;
 
-	ret = asprintf(&chrdev_name, "/dev/%s", device_name);
+	memset(&data.values, 0, sizeof(data.values));
+	ret = gpiotools_request_linehandle(device_name, lines, nlines,
+					   GPIOHANDLE_REQUEST_OUTPUT, &data,
+					   "gpio-hammler");
 	if (ret < 0)
-		return -ENOMEM;
+		goto exit_error;
+	else
+		fd = ret;
 
-	fd = open(chrdev_name, 0);
-	if (fd == -1) {
-		ret = -errno;
-		fprintf(stderr, "Failed to open %s\n", chrdev_name);
-		goto exit_close_error;
-	}
-
-	/* Request lines as output */
-	for (i = 0; i < nlines; i++)
-		req.lineoffsets[i] = lines[i];
-	req.flags = GPIOHANDLE_REQUEST_OUTPUT; /* Request as output */
-	strcpy(req.consumer_label, "gpio-hammer");
-	req.lines = nlines;
-	ret = ioctl(fd, GPIO_GET_LINEHANDLE_IOCTL, &req);
-	if (ret == -1) {
-		ret = -errno;
-		fprintf(stderr, "Failed to issue GET LINEHANDLE "
-			"IOCTL (%d)\n",
-			ret);
+	ret = gpiotools_get_values(fd, &data);
+	if (ret < 0)
 		goto exit_close_error;
-	}
 
-	/* Read initial states */
-	ret = ioctl(req.fd, GPIOHANDLE_GET_LINE_VALUES_IOCTL, &data);
-	if (ret == -1) {
-		ret = -errno;
-		fprintf(stderr, "Failed to issue GPIOHANDLE GET LINE "
-			"VALUES IOCTL (%d)\n",
-			ret);
-		goto exit_close_error;
-	}
 	fprintf(stdout, "Hammer lines [");
 	for (i = 0; i < nlines; i++) {
 		fprintf(stdout, "%d", lines[i]);
@@ -92,23 +69,14 @@ int hammer_device(const char *device_name, unsigned int *lines, int nlines,
 		for (i = 0; i < nlines; i++)
 			data.values[i] = !data.values[i];
 
-		ret = ioctl(req.fd, GPIOHANDLE_SET_LINE_VALUES_IOCTL, &data);
-		if (ret == -1) {
-			ret = -errno;
-			fprintf(stderr, "Failed to issue GPIOHANDLE SET LINE "
-				"VALUES IOCTL (%d)\n",
-				ret);
+		ret = gpiotools_set_values(fd, &data);
+		if (ret < 0)
 			goto exit_close_error;
-		}
+
 		/* Re-read values to get status */
-		ret = ioctl(req.fd, GPIOHANDLE_GET_LINE_VALUES_IOCTL, &data);
-		if (ret == -1) {
-			ret = -errno;
-			fprintf(stderr, "Failed to issue GPIOHANDLE GET LINE "
-				"VALUES IOCTL (%d)\n",
-				ret);
+		ret = gpiotools_get_values(fd, &data);
+		if (ret < 0)
 			goto exit_close_error;
-		}
 
 		fprintf(stdout, "[%c] ", swirr[j]);
 		j++;
@@ -132,9 +100,8 @@ int hammer_device(const char *device_name, unsigned int *lines, int nlines,
 	ret = 0;
 
 exit_close_error:
-	if (close(fd) == -1)
-		perror("Failed to close GPIO character device file");
-	free(chrdev_name);
+	gpiotools_release_linehandle(fd);
+exit_error:
 	return ret;
 }
 
diff --git a/tools/gpio/gpio-utils.c b/tools/gpio/gpio-utils.c
index 8208718f2c99..b86a32d90d88 100644
--- a/tools/gpio/gpio-utils.c
+++ b/tools/gpio/gpio-utils.c
@@ -2,10 +2,266 @@
  * GPIO tools - helpers library for the GPIO tools
  *
  * Copyright (C) 2015 Linus Walleij
+ * Copyright (C) 2016 Bamvor Jian Zhang
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published by
  * the Free Software Foundation.
  */
 
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <sys/ioctl.h>
+#include <linux/gpio.h>
 #include "gpio-utils.h"
+
+#define COMSUMER "gpio-utils"
+
+/**
+ * doc: Operation of gpio
+ *
+ * Provide the api of gpiochip for chardev interface. There are two
+ * types of api.  The first one provide as same function as each
+ * ioctl, including request and release for lines of gpio, read/write
+ * the value of gpio. If the user want to do lots of read and write of
+ * lines of gpio, user should use this type of api.
+ *
+ * The second one provide the easy to use api for user. Each of the
+ * following api will request gpio lines, do the operation and then
+ * release these lines.
+ */
+/**
+ * gpiotools_request_linehandle() - request gpio lines in a gpiochip
+ * @device_name:	The name of gpiochip without prefix "/dev/",
+ *			such as "gpiochip0"
+ * @lines:		An array desired lines, specified by offset
+ *			index for the associated GPIO device.
+ * @nline:		The number of lines to request.
+ * @flag:		The new flag for requsted gpio. Reference
+ *			"linux/gpio.h" for the meaning of flag.
+ * @data:		Default value will be set to gpio when flag is
+ *			GPIOHANDLE_REQUEST_OUTPUT.
+ * @consumer_label:	The name of consumer, such as "sysfs",
+ *			"powerkey". This is useful for other users to
+ *			know who is using.
+ *
+ * Request gpio lines through the ioctl provided by chardev. User
+ * could call gpiotools_set_values() and gpiotools_get_values() to
+ * read and write respectively through the returned fd. Call
+ * gpiotools_release_linehandle() to release these lines after that.
+ *
+ * Return:		On success return the fd;
+ *			On failure return the errno.
+ */
+int gpiotools_request_linehandle(const char *device_name, unsigned int *lines,
+				 unsigned int nlines, unsigned int flag,
+				 struct gpiohandle_data *data,
+				 const char *consumer_label)
+{
+	struct gpiohandle_request req;
+	char *chrdev_name;
+	int fd;
+	int i;
+	int ret;
+
+	ret = asprintf(&chrdev_name, "/dev/%s", device_name);
+	if (ret < 0)
+		return -ENOMEM;
+
+	fd = open(chrdev_name, 0);
+	if (fd == -1) {
+		ret = -errno;
+		fprintf(stderr, "Failed to open %s\n", chrdev_name);
+		goto exit_close_error;
+	}
+
+	for (i = 0; i < nlines; i++)
+		req.lineoffsets[i] = lines[i];
+
+	req.flags = flag;
+	strcpy(req.consumer_label, consumer_label);
+	req.lines = nlines;
+	if (flag & GPIOHANDLE_REQUEST_OUTPUT)
+		memcpy(req.default_values, data, sizeof(req.default_values));
+
+	ret = ioctl(fd, GPIO_GET_LINEHANDLE_IOCTL, &req);
+	if (ret == -1) {
+		ret = -errno;
+		fprintf(stderr, "Failed to issue GET LINEHANDLE IOCTL (%d)\n",
+			ret);
+	}
+
+exit_close_error:
+	if (close(fd) == -1)
+		perror("Failed to close GPIO character device file");
+	free(chrdev_name);
+	return ret < 0 ? ret : req.fd;
+}
+/**
+ * gpiotools_set_values(): Set the value of gpio(s)
+ * @fd:			The fd returned by
+ *			gpiotools_request_linehandle().
+ * @data:		The array of values want to set.
+ *
+ * Return:		On success return 0;
+ *			On failure return the errno.
+ */
+int gpiotools_set_values(const int fd, struct gpiohandle_data *data)
+{
+	int ret;
+
+	ret = ioctl(fd, GPIOHANDLE_SET_LINE_VALUES_IOCTL, data);
+	if (ret == -1) {
+		ret = -errno;
+		fprintf(stderr, "Failed to issue %s (%d)\n",
+			"GPIOHANDLE_SET_LINE_VALUES_IOCTL", ret);
+	}
+
+	return ret;
+}
+
+/**
+ * gpiotools_get_values(): Get the value of gpio(s)
+ * @fd:			The fd returned by
+ *			gpiotools_request_linehandle().
+ * @data:		The array of values get from hardware.
+ *
+ * Return:		On success return 0;
+ *			On failure return the errno.
+ */
+int gpiotools_get_values(const int fd, struct gpiohandle_data *data)
+{
+	int ret;
+
+	ret = ioctl(fd, GPIOHANDLE_GET_LINE_VALUES_IOCTL, data);
+	if (ret == -1) {
+		ret = -errno;
+		fprintf(stderr, "Failed to issue %s (%d)\n",
+			"GPIOHANDLE_GET_LINE_VALUES_IOCTL", ret);
+	}
+
+	return ret;
+}
+
+/**
+ * gpiotools_release_linehandle(): Release the line(s) of gpiochip
+ * @fd:			The fd returned by
+ *			gpiotools_request_linehandle().
+ *
+ * Return:		On success return 0;
+ *			On failure return the errno.
+ */
+int gpiotools_release_linehandle(const int fd)
+{
+	int ret;
+
+	ret = close(fd);
+	if (ret == -1) {
+		perror("Failed to close GPIO LINEHANDLE device file");
+		ret = -errno;
+	}
+
+	return ret;
+}
+
+/**
+ * gpiotools_get(): Get value from specific line
+ * @device_name:	The name of gpiochip without prefix "/dev/",
+ *			such as "gpiochip0"
+ * @line:		number of line, such as 2.
+ *
+ * Return:		On success return 0;
+ *			On failure return the errno.
+ */
+int gpiotools_get(const char *device_name, unsigned int line)
+{
+	struct gpiohandle_data data;
+	unsigned int lines[] = {line};
+
+	gpiotools_gets(device_name, lines, 1, &data);
+	return data.values[0];
+}
+
+
+/**
+ * gpiotools_gets(): Get values from specific lines.
+ * @device_name:	The name of gpiochip without prefix "/dev/",
+ *			such as "gpiochip0".
+ * @lines:		An array desired lines, specified by offset
+ *			index for the associated GPIO device.
+ * @nline:		The number of lines to request.
+ * @data:		The array of values get from gpiochip.
+ *
+ * Return:		On success return 0;
+ *			On failure return the errno.
+ */
+int gpiotools_gets(const char *device_name, unsigned int *lines,
+		   unsigned int nlines, struct gpiohandle_data *data)
+{
+	int fd;
+	int ret;
+	int ret_close;
+
+	ret = gpiotools_request_linehandle(device_name, lines, nlines,
+					   GPIOHANDLE_REQUEST_INPUT, data,
+					   COMSUMER);
+	if (ret < 0)
+		return ret;
+
+	fd = ret;
+	ret = gpiotools_get_values(fd, data);
+	ret_close = gpiotools_release_linehandle(fd);
+	return ret < 0 ? ret : ret_close;
+}
+
+/**
+ * gpiotools_set(): Set value to specific line
+ * @device_name:	The name of gpiochip without prefix "/dev/",
+ *			such as "gpiochip0"
+ * @line:		number of line, such as 2.
+ * @value:		The value of gpio, must be 0(low) or 1(high).
+ *
+ * Return:		On success return 0;
+ *			On failure return the errno.
+ */
+int gpiotools_set(const char *device_name, unsigned int line,
+		  unsigned int value)
+{
+	struct gpiohandle_data data;
+	unsigned int lines[] = {line};
+
+	data.values[0] = value;
+	return gpiotools_sets(device_name, lines, 1, &data);
+}
+
+/**
+ * gpiotools_sets(): Set values to specific lines.
+ * @device_name:	The name of gpiochip without prefix "/dev/",
+ *			such as "gpiochip0".
+ * @lines:		An array desired lines, specified by offset
+ *			index for the associated GPIO device.
+ * @nline:		The number of lines to request.
+ * @data:		The array of values set to gpiochip, must be
+ *			0(low) or 1(high).
+ *
+ * Return:		On success return 0;
+ *			On failure return the errno.
+ */
+int gpiotools_sets(const char *device_name, unsigned int *lines,
+		   unsigned int nlines, struct gpiohandle_data *data)
+{
+	int ret;
+
+	ret = gpiotools_request_linehandle(device_name, lines, nlines,
+					   GPIOHANDLE_REQUEST_OUTPUT, data,
+					   COMSUMER);
+	if (ret < 0)
+		return ret;
+
+	return gpiotools_release_linehandle(ret);
+}
diff --git a/tools/gpio/gpio-utils.h b/tools/gpio/gpio-utils.h
index 5f57133b8c04..344ea041f8d4 100644
--- a/tools/gpio/gpio-utils.h
+++ b/tools/gpio/gpio-utils.h
@@ -24,4 +24,20 @@ static inline int check_prefix(const char *str, const char *prefix)
 		strncmp(str, prefix, strlen(prefix)) == 0;
 }
 
+int gpiotools_request_linehandle(const char *device_name, unsigned int *lines,
+				 unsigned int nlines, unsigned int flag,
+				 struct gpiohandle_data *data,
+				 const char *consumer_label);
+int gpiotools_set_values(const int fd, struct gpiohandle_data *data);
+int gpiotools_get_values(const int fd, struct gpiohandle_data *data);
+int gpiotools_release_linehandle(const int fd);
+
+int gpiotools_get(const char *device_name, unsigned int line);
+int gpiotools_gets(const char *device_name, unsigned int *lines,
+		   unsigned int nlines, struct gpiohandle_data *data);
+int gpiotools_set(const char *device_name, unsigned int line,
+		  unsigned int value);
+int gpiotools_sets(const char *device_name, unsigned int *lines,
+		   unsigned int nlines, struct gpiohandle_data *data);
+
 #endif /* _GPIO_UTILS_H_ */
diff --git a/tools/hv/Makefile b/tools/hv/Makefile
index a8c4644022a6..0d1e61b81844 100644
--- a/tools/hv/Makefile
+++ b/tools/hv/Makefile
@@ -1,9 +1,8 @@
 # Makefile for Hyper-V tools
 
 CC = $(CROSS_COMPILE)gcc
-PTHREAD_LIBS = -lpthread
 WARNINGS = -Wall -Wextra
-CFLAGS = $(WARNINGS) -g $(PTHREAD_LIBS) $(shell getconf LFS_CFLAGS)
+CFLAGS = $(WARNINGS) -g $(shell getconf LFS_CFLAGS)
 
 CFLAGS += -D__EXPORTED_HEADERS__ -I../../include/uapi -I../../include
 
diff --git a/tools/hv/hv_fcopy_daemon.c b/tools/hv/hv_fcopy_daemon.c
index fdc9ca4c0356..26ae609a9448 100644
--- a/tools/hv/hv_fcopy_daemon.c
+++ b/tools/hv/hv_fcopy_daemon.c
@@ -18,21 +18,14 @@
 
 
 #include <sys/types.h>
-#include <sys/socket.h>
-#include <sys/poll.h>
-#include <linux/types.h>
-#include <linux/kdev_t.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include <string.h>
-#include <ctype.h>
 #include <errno.h>
 #include <linux/hyperv.h>
 #include <syslog.h>
 #include <sys/stat.h>
 #include <fcntl.h>
-#include <dirent.h>
 #include <getopt.h>
 
 static int target_fd;
diff --git a/tools/hv/hv_kvp_daemon.c b/tools/hv/hv_kvp_daemon.c
index bc7adb84e679..f1758fcbc37d 100644
--- a/tools/hv/hv_kvp_daemon.c
+++ b/tools/hv/hv_kvp_daemon.c
@@ -22,8 +22,6 @@
  */
 
 
-#include <sys/types.h>
-#include <sys/socket.h>
 #include <sys/poll.h>
 #include <sys/utsname.h>
 #include <stdio.h>
@@ -34,7 +32,6 @@
 #include <errno.h>
 #include <arpa/inet.h>
 #include <linux/hyperv.h>
-#include <linux/netlink.h>
 #include <ifaddrs.h>
 #include <netdb.h>
 #include <syslog.h>
@@ -96,13 +93,13 @@ static struct utsname uts_buf;
 
 #define KVP_CONFIG_LOC	"/var/lib/hyperv"
 
+#ifndef KVP_SCRIPTS_PATH
+#define KVP_SCRIPTS_PATH "/usr/libexec/hypervkvpd/"
+#endif
+
 #define MAX_FILE_NAME 100
 #define ENTRIES_PER_BLOCK 50
 
-#ifndef SOL_NETLINK
-#define SOL_NETLINK 270
-#endif
-
 struct kvp_record {
 	char key[HV_KVP_EXCHANGE_MAX_KEY_SIZE];
 	char value[HV_KVP_EXCHANGE_MAX_VALUE_SIZE];
@@ -702,7 +699,7 @@ static char *kvp_mac_to_if_name(char *mac)
 	if (dir == NULL)
 		return NULL;
 
-	snprintf(dev_id, sizeof(dev_id), kvp_net_dir);
+	snprintf(dev_id, sizeof(dev_id), "%s", kvp_net_dir);
 	q = dev_id + strlen(kvp_net_dir);
 
 	while ((entry = readdir(dir)) != NULL) {
@@ -825,7 +822,7 @@ static void kvp_get_ipconfig_info(char *if_name,
 	 * .
 	 */
 
-	sprintf(cmd, "%s",  "hv_get_dns_info");
+	sprintf(cmd, KVP_SCRIPTS_PATH "%s",  "hv_get_dns_info");
 
 	/*
 	 * Execute the command to gather DNS info.
@@ -842,7 +839,7 @@ static void kvp_get_ipconfig_info(char *if_name,
 	 * Enabled: DHCP enabled.
 	 */
 
-	sprintf(cmd, "%s %s", "hv_get_dhcp_info", if_name);
+	sprintf(cmd, KVP_SCRIPTS_PATH "%s %s", "hv_get_dhcp_info", if_name);
 
 	file = popen(cmd, "r");
 	if (file == NULL)
@@ -1348,7 +1345,8 @@ static int kvp_set_ip_info(char *if_name, struct hv_kvp_ipaddr_value *new_val)
 	 * invoke the external script to do its magic.
 	 */
 
-	snprintf(cmd, sizeof(cmd), "%s %s", "hv_set_ifconfig", if_file);
+	snprintf(cmd, sizeof(cmd), KVP_SCRIPTS_PATH "%s %s",
+		 "hv_set_ifconfig", if_file);
 	if (system(cmd)) {
 		syslog(LOG_ERR, "Failed to execute cmd '%s'; error: %d %s",
 				cmd, errno, strerror(errno));
diff --git a/tools/iio/iio_generic_buffer.c b/tools/iio/iio_generic_buffer.c
index f39c0e9c0d5c..f0c6f54a8b2f 100644
--- a/tools/iio/iio_generic_buffer.c
+++ b/tools/iio/iio_generic_buffer.c
@@ -247,6 +247,7 @@ void print_usage(void)
 	fprintf(stderr, "Usage: generic_buffer [options]...\n"
 		"Capture, convert and output data from IIO device buffer\n"
 		"  -a         Auto-activate all available channels\n"
+		"  -A         Force-activate ALL channels\n"
 		"  -c <n>     Do n conversions\n"
 		"  -e         Disable wait for event (new data)\n"
 		"  -g         Use trigger-less mode\n"
@@ -347,16 +348,22 @@ int main(int argc, char **argv)
 	int noevents = 0;
 	int notrigger = 0;
 	char *dummy;
+	bool force_autochannels = false;
 
 	struct iio_channel_info *channels = NULL;
 
 	register_cleanup();
 
-	while ((c = getopt_long(argc, argv, "ac:egl:n:N:t:T:w:", longopts, NULL)) != -1) {
+	while ((c = getopt_long(argc, argv, "aAc:egl:n:N:t:T:w:?", longopts,
+				NULL)) != -1) {
 		switch (c) {
 		case 'a':
 			autochannels = AUTOCHANNELS_ENABLED;
 			break;
+		case 'A':
+			autochannels = AUTOCHANNELS_ENABLED;
+			force_autochannels = true;
+			break;	
 		case 'c':
 			errno = 0;
 			num_loops = strtoul(optarg, &dummy, 10);
@@ -519,15 +526,16 @@ int main(int argc, char **argv)
 			"diag %s\n", dev_dir_name);
 		goto error;
 	}
-	if (num_channels && autochannels == AUTOCHANNELS_ENABLED) {
+	if (num_channels && autochannels == AUTOCHANNELS_ENABLED &&
+	    !force_autochannels) {
 		fprintf(stderr, "Auto-channels selected but some channels "
 			"are already activated in sysfs\n");
 		fprintf(stderr, "Proceeding without activating any channels\n");
 	}
 
-	if (!num_channels && autochannels == AUTOCHANNELS_ENABLED) {
-		fprintf(stderr,
-			"No channels are enabled, enabling all channels\n");
+	if ((!num_channels && autochannels == AUTOCHANNELS_ENABLED) ||
+	    (autochannels == AUTOCHANNELS_ENABLED && force_autochannels)) {
+		fprintf(stderr, "Enabling all channels\n");
 
 		ret = enable_disable_all_channels(dev_dir_name, 1);
 		if (ret) {
diff --git a/tools/include/asm/bug.h b/tools/include/asm/bug.h
index 9e5f4846967f..beda1a884b50 100644
--- a/tools/include/asm/bug.h
+++ b/tools/include/asm/bug.h
@@ -12,6 +12,17 @@
 	unlikely(__ret_warn_on);		\
 })
 
+#define WARN_ON_ONCE(condition) ({			\
+	static int __warned;				\
+	int __ret_warn_once = !!(condition);		\
+							\
+	if (unlikely(__ret_warn_once && !__warned)) {	\
+		__warned = true;			\
+		WARN_ON(1);				\
+	}						\
+	unlikely(__ret_warn_once);			\
+})
+
 #define WARN_ONCE(condition, format...)	({	\
 	static int __warned;			\
 	int __ret_warn_once = !!(condition);	\
diff --git a/tools/include/linux/bitmap.h b/tools/include/linux/bitmap.h
index 43c1c5021e4b..eef41d500e9e 100644
--- a/tools/include/linux/bitmap.h
+++ b/tools/include/linux/bitmap.h
@@ -35,6 +35,32 @@ static inline void bitmap_zero(unsigned long *dst, int nbits)
 	}
 }
 
+static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
+{
+	unsigned int nlongs = BITS_TO_LONGS(nbits);
+	if (!small_const_nbits(nbits)) {
+		unsigned int len = (nlongs - 1) * sizeof(unsigned long);
+		memset(dst, 0xff,  len);
+	}
+	dst[nlongs - 1] = BITMAP_LAST_WORD_MASK(nbits);
+}
+
+static inline int bitmap_empty(const unsigned long *src, unsigned nbits)
+{
+	if (small_const_nbits(nbits))
+		return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
+
+	return find_first_bit(src, nbits) == nbits;
+}
+
+static inline int bitmap_full(const unsigned long *src, unsigned int nbits)
+{
+	if (small_const_nbits(nbits))
+		return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
+
+	return find_first_zero_bit(src, nbits) == nbits;
+}
+
 static inline int bitmap_weight(const unsigned long *src, int nbits)
 {
 	if (small_const_nbits(nbits))
diff --git a/tools/include/linux/types.h b/tools/include/linux/types.h
index 8ebf6278b2ef..c24b3e3ae296 100644
--- a/tools/include/linux/types.h
+++ b/tools/include/linux/types.h
@@ -42,11 +42,7 @@ typedef __s8  s8;
 #else
 #define __bitwise__
 #endif
-#ifdef __CHECK_ENDIAN__
 #define __bitwise __bitwise__
-#else
-#define __bitwise
-#endif
 
 #define __force
 #define __user
diff --git a/tools/include/uapi/linux/hw_breakpoint.h b/tools/include/uapi/linux/hw_breakpoint.h
index b04000a2296a..2b65efd19a46 100644
--- a/tools/include/uapi/linux/hw_breakpoint.h
+++ b/tools/include/uapi/linux/hw_breakpoint.h
@@ -4,7 +4,11 @@
 enum {
 	HW_BREAKPOINT_LEN_1 = 1,
 	HW_BREAKPOINT_LEN_2 = 2,
+	HW_BREAKPOINT_LEN_3 = 3,
 	HW_BREAKPOINT_LEN_4 = 4,
+	HW_BREAKPOINT_LEN_5 = 5,
+	HW_BREAKPOINT_LEN_6 = 6,
+	HW_BREAKPOINT_LEN_7 = 7,
 	HW_BREAKPOINT_LEN_8 = 8,
 };
 
diff --git a/tools/leds/.gitignore b/tools/leds/.gitignore
new file mode 100644
index 000000000000..ac96d9f53dfc
--- /dev/null
+++ b/tools/leds/.gitignore
@@ -0,0 +1 @@
+uledmon
diff --git a/tools/leds/Makefile b/tools/leds/Makefile
new file mode 100644
index 000000000000..c03a79ebf9c8
--- /dev/null
+++ b/tools/leds/Makefile
@@ -0,0 +1,13 @@
+# Makefile for LEDs tools
+
+CC = $(CROSS_COMPILE)gcc
+CFLAGS = -Wall -Wextra -g -I../../include/uapi
+
+all: uledmon
+%: %.c
+	$(CC) $(CFLAGS) -o $@ $^
+
+clean:
+	$(RM) uledmon
+
+.PHONY: all clean
diff --git a/tools/leds/uledmon.c b/tools/leds/uledmon.c
new file mode 100644
index 000000000000..25cbc7acf50a
--- /dev/null
+++ b/tools/leds/uledmon.c
@@ -0,0 +1,63 @@
+/*
+ * uledmon.c
+ *
+ * This program creates a new userspace LED class device and monitors it. A
+ * timestamp and brightness value is printed each time the brightness changes.
+ *
+ * Usage: uledmon <device-name>
+ *
+ * <device-name> is the name of the LED class device to be created. Pressing
+ * CTRL+C will exit.
+ */
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+#include <linux/uleds.h>
+
+int main(int argc, char const *argv[])
+{
+	struct uleds_user_dev uleds_dev;
+	int fd, ret;
+	int brightness;
+	struct timespec ts;
+
+	if (argc != 2) {
+		fprintf(stderr, "Requires <device-name> argument\n");
+		return 1;
+	}
+
+	strncpy(uleds_dev.name, argv[1], LED_MAX_NAME_SIZE);
+	uleds_dev.max_brightness = 100;
+
+	fd = open("/dev/uleds", O_RDWR);
+	if (fd == -1) {
+		perror("Failed to open /dev/uleds");
+		return 1;
+	}
+
+	ret = write(fd, &uleds_dev, sizeof(uleds_dev));
+	if (ret == -1) {
+		perror("Failed to write to /dev/uleds");
+		close(fd);
+		return 1;
+	}
+
+	while (1) {
+		ret = read(fd, &brightness, sizeof(brightness));
+		if (ret == -1) {
+			perror("Failed to read from /dev/uleds");
+			close(fd);
+			return 1;
+		}
+		clock_gettime(CLOCK_MONOTONIC, &ts);
+		printf("[%ld.%09ld] %u\n", ts.tv_sec, ts.tv_nsec, brightness);
+	}
+
+	close(fd);
+
+	return 0;
+}
diff --git a/tools/power/acpi/os_specific/service_layers/osunixxf.c b/tools/power/acpi/os_specific/service_layers/osunixxf.c
index 8d8003c919d4..10648aaf6164 100644
--- a/tools/power/acpi/os_specific/service_layers/osunixxf.c
+++ b/tools/power/acpi/os_specific/service_layers/osunixxf.c
@@ -646,8 +646,12 @@ acpi_os_create_semaphore(u32 max_units,
 	}
 #ifdef __APPLE__
 	{
-		char *semaphore_name = tmpnam(NULL);
+		static int semaphore_count = 0;
+		char semaphore_name[32];
 
+		snprintf(semaphore_name, sizeof(semaphore_name), "acpi_sem_%d",
+			 semaphore_count++);
+		printf("%s\n", semaphore_name);
 		sem =
 		    sem_open(semaphore_name, O_EXCL | O_CREAT, 0755,
 			     initial_units);
@@ -692,10 +696,15 @@ acpi_status acpi_os_delete_semaphore(acpi_handle handle)
 	if (!sem) {
 		return (AE_BAD_PARAMETER);
 	}
-
+#ifdef __APPLE__
+	if (sem_close(sem) == -1) {
+		return (AE_BAD_PARAMETER);
+	}
+#else
 	if (sem_destroy(sem) == -1) {
 		return (AE_BAD_PARAMETER);
 	}
+#endif
 
 	return (AE_OK);
 }
diff --git a/tools/power/acpi/tools/ec/ec_access.c b/tools/power/acpi/tools/ec/ec_access.c
index 6b8aaed44f2c..5f50642386db 100644
--- a/tools/power/acpi/tools/ec/ec_access.c
+++ b/tools/power/acpi/tools/ec/ec_access.c
@@ -46,7 +46,7 @@ void usage(char progname[], int exit_status)
 	puts("\t-b offset          : Read value at byte_offset (in hex)");
 	puts("\t-w offset -v value : Write value at byte_offset");
 	puts("\t-h                 : Print this help\n\n");
-	puts("Offsets and values are in hexadecimal number sytem.");
+	puts("Offsets and values are in hexadecimal number system.");
 	puts("The offset and value must be between 0 and 0xff.");
 	exit(exit_status);
 }
diff --git a/tools/spi/spidev_test.c b/tools/spi/spidev_test.c
index f046b77cfefe..816f119c9b7b 100644
--- a/tools/spi/spidev_test.c
+++ b/tools/spi/spidev_test.c
@@ -315,7 +315,7 @@ static void transfer_file(int fd, char *filename)
 		pabort("can't stat input file");
 
 	tx_fd = open(filename, O_RDONLY);
-	if (fd < 0)
+	if (tx_fd < 0)
 		pabort("can't open input file");
 
 	tx = malloc(sb.st_size);
diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index d08e214ec6e7..be93ab02b490 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -719,14 +719,14 @@ sub set_value {
 
     if ($buildonly && $lvalue =~ /^TEST_TYPE(\[.*\])?$/ && $prvalue ne "build") {
 	# Note if a test is something other than build, then we
-	# will need other manditory options.
+	# will need other mandatory options.
 	if ($prvalue ne "install") {
 	    # for bisect, we need to check BISECT_TYPE
 	    if ($prvalue ne "bisect") {
 		$buildonly = 0;
 	    }
 	} else {
-	    # install still limits some manditory options.
+	    # install still limits some mandatory options.
 	    $buildonly = 2;
 	}
     }
@@ -735,7 +735,7 @@ sub set_value {
 	if ($prvalue ne "install") {
 	    $buildonly = 0;
 	} else {
-	    # install still limits some manditory options.
+	    # install still limits some mandatory options.
 	    $buildonly = 2;
 	}
     }
@@ -3989,7 +3989,7 @@ sub make_min_config {
 		}
 	    }
 
-	    # Save off all the current mandidory configs
+	    # Save off all the current mandatory configs
 	    open (OUT, ">$temp_config")
 		or die "Can't write to $temp_config";
 	    foreach my $config (keys %keep_configs) {
diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile
index f2e07f2fd4b4..3635e4d3eca7 100644
--- a/tools/testing/radix-tree/Makefile
+++ b/tools/testing/radix-tree/Makefile
@@ -1,10 +1,14 @@
 
-CFLAGS += -I. -g -O2 -Wall -D_LGPL_SOURCE
+CFLAGS += -I. -I../../include -g -O2 -Wall -D_LGPL_SOURCE
 LDFLAGS += -lpthread -lurcu
 TARGETS = main
 OFILES = main.o radix-tree.o linux.o test.o tag_check.o find_next_bit.o \
 	 regression1.o regression2.o regression3.o multiorder.o \
-	 iteration_check.o
+	 iteration_check.o benchmark.o
+
+ifdef BENCHMARK
+	CFLAGS += -DBENCHMARK=1
+endif
 
 targets: $(TARGETS)
 
@@ -14,7 +18,12 @@ main:	$(OFILES)
 clean:
 	$(RM) -f $(TARGETS) *.o radix-tree.c
 
-$(OFILES): *.h */*.h ../../../include/linux/radix-tree.h ../../include/linux/*.h
+find_next_bit.o: ../../lib/find_bit.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+
+$(OFILES): *.h */*.h \
+	../../include/linux/*.h \
+	../../../include/linux/radix-tree.h
 
 radix-tree.c: ../../../lib/radix-tree.c
 	sed -e 's/^static //' -e 's/__always_inline //' -e 's/inline //' < $< > $@
diff --git a/tools/testing/radix-tree/benchmark.c b/tools/testing/radix-tree/benchmark.c
new file mode 100644
index 000000000000..215ca86c7605
--- /dev/null
+++ b/tools/testing/radix-tree/benchmark.c
@@ -0,0 +1,98 @@
+/*
+ * benchmark.c:
+ * Author: Konstantin Khlebnikov <koct9i@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+#include <linux/radix-tree.h>
+#include <linux/slab.h>
+#include <linux/errno.h>
+#include <time.h>
+#include "test.h"
+
+#define NSEC_PER_SEC	1000000000L
+
+static long long benchmark_iter(struct radix_tree_root *root, bool tagged)
+{
+	volatile unsigned long sink = 0;
+	struct radix_tree_iter iter;
+	struct timespec start, finish;
+	long long nsec;
+	int l, loops = 1;
+	void **slot;
+
+#ifdef BENCHMARK
+again:
+#endif
+	clock_gettime(CLOCK_MONOTONIC, &start);
+	for (l = 0; l < loops; l++) {
+		if (tagged) {
+			radix_tree_for_each_tagged(slot, root, &iter, 0, 0)
+				sink ^= (unsigned long)slot;
+		} else {
+			radix_tree_for_each_slot(slot, root, &iter, 0)
+				sink ^= (unsigned long)slot;
+		}
+	}
+	clock_gettime(CLOCK_MONOTONIC, &finish);
+
+	nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC +
+	       (finish.tv_nsec - start.tv_nsec);
+
+#ifdef BENCHMARK
+	if (loops == 1 && nsec * 5 < NSEC_PER_SEC) {
+		loops = NSEC_PER_SEC / nsec / 4 + 1;
+		goto again;
+	}
+#endif
+
+	nsec /= loops;
+	return nsec;
+}
+
+static void benchmark_size(unsigned long size, unsigned long step, int order)
+{
+	RADIX_TREE(tree, GFP_KERNEL);
+	long long normal, tagged;
+	unsigned long index;
+
+	for (index = 0 ; index < size ; index += step) {
+		item_insert_order(&tree, index, order);
+		radix_tree_tag_set(&tree, index, 0);
+	}
+
+	tagged = benchmark_iter(&tree, true);
+	normal = benchmark_iter(&tree, false);
+
+	printf("Size %ld, step %6ld, order %d tagged %10lld ns, normal %10lld ns\n",
+		size, step, order, tagged, normal);
+
+	item_kill_tree(&tree);
+	rcu_barrier();
+}
+
+void benchmark(void)
+{
+	unsigned long size[] = {1 << 10, 1 << 20, 0};
+	unsigned long step[] = {1, 2, 7, 15, 63, 64, 65,
+				128, 256, 512, 12345, 0};
+	int c, s;
+
+	printf("starting benchmarks\n");
+	printf("RADIX_TREE_MAP_SHIFT = %d\n", RADIX_TREE_MAP_SHIFT);
+
+	for (c = 0; size[c]; c++)
+		for (s = 0; step[s]; s++)
+			benchmark_size(size[c], step[s], 0);
+
+	for (c = 0; size[c]; c++)
+		for (s = 0; step[s]; s++)
+			benchmark_size(size[c], step[s] << 9, 9);
+}
diff --git a/tools/testing/radix-tree/find_next_bit.c b/tools/testing/radix-tree/find_next_bit.c
deleted file mode 100644
index d1c2178bb2d4..000000000000
--- a/tools/testing/radix-tree/find_next_bit.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/* find_next_bit.c: fallback find next bit implementation
- *
- * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
- * Written by David Howells (dhowells@redhat.com)
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/types.h>
-#include <linux/bitops.h>
-
-#define BITOP_WORD(nr)		((nr) / BITS_PER_LONG)
-
-/*
- * Find the next set bit in a memory region.
- */
-unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
-			    unsigned long offset)
-{
-	const unsigned long *p = addr + BITOP_WORD(offset);
-	unsigned long result = offset & ~(BITS_PER_LONG-1);
-	unsigned long tmp;
-
-	if (offset >= size)
-		return size;
-	size -= result;
-	offset %= BITS_PER_LONG;
-	if (offset) {
-		tmp = *(p++);
-		tmp &= (~0UL << offset);
-		if (size < BITS_PER_LONG)
-			goto found_first;
-		if (tmp)
-			goto found_middle;
-		size -= BITS_PER_LONG;
-		result += BITS_PER_LONG;
-	}
-	while (size & ~(BITS_PER_LONG-1)) {
-		if ((tmp = *(p++)))
-			goto found_middle;
-		result += BITS_PER_LONG;
-		size -= BITS_PER_LONG;
-	}
-	if (!size)
-		return result;
-	tmp = *p;
-
-found_first:
-	tmp &= (~0UL >> (BITS_PER_LONG - size));
-	if (tmp == 0UL)		/* Are any bits set? */
-		return result + size;	/* Nope. */
-found_middle:
-	return result + __ffs(tmp);
-}
diff --git a/tools/testing/radix-tree/iteration_check.c b/tools/testing/radix-tree/iteration_check.c
index 9adb8e7415a6..7572b7ed930e 100644
--- a/tools/testing/radix-tree/iteration_check.c
+++ b/tools/testing/radix-tree/iteration_check.c
@@ -16,35 +16,50 @@
 #include <pthread.h>
 #include "test.h"
 
-#define NUM_THREADS 4
-#define TAG 0
+#define NUM_THREADS	5
+#define MAX_IDX		100
+#define TAG		0
+#define NEW_TAG		1
+
 static pthread_mutex_t tree_lock = PTHREAD_MUTEX_INITIALIZER;
 static pthread_t threads[NUM_THREADS];
-RADIX_TREE(tree, GFP_KERNEL);
-bool test_complete;
+static unsigned int seeds[3];
+static RADIX_TREE(tree, GFP_KERNEL);
+static bool test_complete;
+static int max_order;
 
 /* relentlessly fill the tree with tagged entries */
 static void *add_entries_fn(void *arg)
 {
-	int pgoff;
+	rcu_register_thread();
 
 	while (!test_complete) {
-		for (pgoff = 0; pgoff < 100; pgoff++) {
+		unsigned long pgoff;
+		int order;
+
+		for (pgoff = 0; pgoff < MAX_IDX; pgoff++) {
 			pthread_mutex_lock(&tree_lock);
-			if (item_insert(&tree, pgoff) == 0)
-				item_tag_set(&tree, pgoff, TAG);
+			for (order = max_order; order >= 0; order--) {
+				if (item_insert_order(&tree, pgoff, order)
+						== 0) {
+					item_tag_set(&tree, pgoff, TAG);
+					break;
+				}
+			}
 			pthread_mutex_unlock(&tree_lock);
 		}
 	}
 
+	rcu_unregister_thread();
+
 	return NULL;
 }
 
 /*
  * Iterate over the tagged entries, doing a radix_tree_iter_retry() as we find
  * things that have been removed and randomly resetting our iteration to the
- * next chunk with radix_tree_iter_next().  Both radix_tree_iter_retry() and
- * radix_tree_iter_next() cause radix_tree_next_slot() to be called with a
+ * next chunk with radix_tree_iter_resume().  Both radix_tree_iter_retry() and
+ * radix_tree_iter_resume() cause radix_tree_next_slot() to be called with a
  * NULL 'slot' variable.
  */
 static void *tagged_iteration_fn(void *arg)
@@ -52,17 +67,12 @@ static void *tagged_iteration_fn(void *arg)
 	struct radix_tree_iter iter;
 	void **slot;
 
+	rcu_register_thread();
+
 	while (!test_complete) {
 		rcu_read_lock();
 		radix_tree_for_each_tagged(slot, &tree, &iter, 0, TAG) {
-			void *entry;
-			int i;
-
-			/* busy wait to let removals happen */
-			for (i = 0; i < 1000000; i++)
-				;
-
-			entry = radix_tree_deref_slot(slot);
+			void *entry = radix_tree_deref_slot(slot);
 			if (unlikely(!entry))
 				continue;
 
@@ -71,20 +81,26 @@ static void *tagged_iteration_fn(void *arg)
 				continue;
 			}
 
-			if (rand() % 50 == 0)
-				slot = radix_tree_iter_next(&iter);
+			if (rand_r(&seeds[0]) % 50 == 0) {
+				slot = radix_tree_iter_resume(slot, &iter);
+				rcu_read_unlock();
+				rcu_barrier();
+				rcu_read_lock();
+			}
 		}
 		rcu_read_unlock();
 	}
 
+	rcu_unregister_thread();
+
 	return NULL;
 }
 
 /*
  * Iterate over the entries, doing a radix_tree_iter_retry() as we find things
  * that have been removed and randomly resetting our iteration to the next
- * chunk with radix_tree_iter_next().  Both radix_tree_iter_retry() and
- * radix_tree_iter_next() cause radix_tree_next_slot() to be called with a
+ * chunk with radix_tree_iter_resume().  Both radix_tree_iter_retry() and
+ * radix_tree_iter_resume() cause radix_tree_next_slot() to be called with a
  * NULL 'slot' variable.
  */
 static void *untagged_iteration_fn(void *arg)
@@ -92,17 +108,12 @@ static void *untagged_iteration_fn(void *arg)
 	struct radix_tree_iter iter;
 	void **slot;
 
+	rcu_register_thread();
+
 	while (!test_complete) {
 		rcu_read_lock();
 		radix_tree_for_each_slot(slot, &tree, &iter, 0) {
-			void *entry;
-			int i;
-
-			/* busy wait to let removals happen */
-			for (i = 0; i < 1000000; i++)
-				;
-
-			entry = radix_tree_deref_slot(slot);
+			void *entry = radix_tree_deref_slot(slot);
 			if (unlikely(!entry))
 				continue;
 
@@ -111,12 +122,18 @@ static void *untagged_iteration_fn(void *arg)
 				continue;
 			}
 
-			if (rand() % 50 == 0)
-				slot = radix_tree_iter_next(&iter);
+			if (rand_r(&seeds[1]) % 50 == 0) {
+				slot = radix_tree_iter_resume(slot, &iter);
+				rcu_read_unlock();
+				rcu_barrier();
+				rcu_read_lock();
+			}
 		}
 		rcu_read_unlock();
 	}
 
+	rcu_unregister_thread();
+
 	return NULL;
 }
 
@@ -126,47 +143,71 @@ static void *untagged_iteration_fn(void *arg)
  */
 static void *remove_entries_fn(void *arg)
 {
+	rcu_register_thread();
+
 	while (!test_complete) {
 		int pgoff;
 
-		pgoff = rand() % 100;
+		pgoff = rand_r(&seeds[2]) % MAX_IDX;
 
 		pthread_mutex_lock(&tree_lock);
 		item_delete(&tree, pgoff);
 		pthread_mutex_unlock(&tree_lock);
 	}
 
+	rcu_unregister_thread();
+
+	return NULL;
+}
+
+static void *tag_entries_fn(void *arg)
+{
+	rcu_register_thread();
+
+	while (!test_complete) {
+		tag_tagged_items(&tree, &tree_lock, 0, MAX_IDX, 10, TAG,
+					NEW_TAG);
+	}
+	rcu_unregister_thread();
 	return NULL;
 }
 
 /* This is a unit test for a bug found by the syzkaller tester */
-void iteration_test(void)
+void iteration_test(unsigned order, unsigned test_duration)
 {
 	int i;
 
-	printf("Running iteration tests for 10 seconds\n");
+	printf("Running %siteration tests for %d seconds\n",
+			order > 0 ? "multiorder " : "", test_duration);
 
-	srand(time(0));
+	max_order = order;
 	test_complete = false;
 
+	for (i = 0; i < 3; i++)
+		seeds[i] = rand();
+
 	if (pthread_create(&threads[0], NULL, tagged_iteration_fn, NULL)) {
-		perror("pthread_create");
+		perror("create tagged iteration thread");
 		exit(1);
 	}
 	if (pthread_create(&threads[1], NULL, untagged_iteration_fn, NULL)) {
-		perror("pthread_create");
+		perror("create untagged iteration thread");
 		exit(1);
 	}
 	if (pthread_create(&threads[2], NULL, add_entries_fn, NULL)) {
-		perror("pthread_create");
+		perror("create add entry thread");
 		exit(1);
 	}
 	if (pthread_create(&threads[3], NULL, remove_entries_fn, NULL)) {
-		perror("pthread_create");
+		perror("create remove entry thread");
+		exit(1);
+	}
+	if (pthread_create(&threads[4], NULL, tag_entries_fn, NULL)) {
+		perror("create tag entry thread");
 		exit(1);
 	}
 
-	sleep(10);
+	sleep(test_duration);
 	test_complete = true;
 
 	for (i = 0; i < NUM_THREADS; i++) {
diff --git a/tools/testing/radix-tree/linux.c b/tools/testing/radix-tree/linux.c
index 154823737b20..d31ea7c9abec 100644
--- a/tools/testing/radix-tree/linux.c
+++ b/tools/testing/radix-tree/linux.c
@@ -1,14 +1,26 @@
 #include <stdlib.h>
 #include <string.h>
 #include <malloc.h>
+#include <pthread.h>
 #include <unistd.h>
 #include <assert.h>
 
 #include <linux/mempool.h>
+#include <linux/poison.h>
 #include <linux/slab.h>
+#include <linux/radix-tree.h>
 #include <urcu/uatomic.h>
 
 int nr_allocated;
+int preempt_count;
+
+struct kmem_cache {
+	pthread_mutex_t lock;
+	int size;
+	int nr_objs;
+	void *objs;
+	void (*ctor)(void *);
+};
 
 void *mempool_alloc(mempool_t *pool, int gfp_mask)
 {
@@ -33,19 +45,59 @@ mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
 
 void *kmem_cache_alloc(struct kmem_cache *cachep, int flags)
 {
-	void *ret = malloc(cachep->size);
-	if (cachep->ctor)
-		cachep->ctor(ret);
+	struct radix_tree_node *node;
+
+	if (flags & __GFP_NOWARN)
+		return NULL;
+
+	pthread_mutex_lock(&cachep->lock);
+	if (cachep->nr_objs) {
+		cachep->nr_objs--;
+		node = cachep->objs;
+		cachep->objs = node->private_data;
+		pthread_mutex_unlock(&cachep->lock);
+		node->private_data = NULL;
+	} else {
+		pthread_mutex_unlock(&cachep->lock);
+		node = malloc(cachep->size);
+		if (cachep->ctor)
+			cachep->ctor(node);
+	}
+
 	uatomic_inc(&nr_allocated);
-	return ret;
+	return node;
 }
 
 void kmem_cache_free(struct kmem_cache *cachep, void *objp)
 {
 	assert(objp);
 	uatomic_dec(&nr_allocated);
-	memset(objp, 0, cachep->size);
-	free(objp);
+	pthread_mutex_lock(&cachep->lock);
+	if (cachep->nr_objs > 10) {
+		memset(objp, POISON_FREE, cachep->size);
+		free(objp);
+	} else {
+		struct radix_tree_node *node = objp;
+		cachep->nr_objs++;
+		node->private_data = cachep->objs;
+		cachep->objs = node;
+	}
+	pthread_mutex_unlock(&cachep->lock);
+}
+
+void *kmalloc(size_t size, gfp_t gfp)
+{
+	void *ret = malloc(size);
+	uatomic_inc(&nr_allocated);
+	return ret;
+}
+
+void kfree(void *p)
+{
+	if (!p)
+		return;
+	uatomic_dec(&nr_allocated);
+	free(p);
 }
 
 struct kmem_cache *
@@ -54,7 +106,10 @@ kmem_cache_create(const char *name, size_t size, size_t offset,
 {
 	struct kmem_cache *ret = malloc(sizeof(*ret));
 
+	pthread_mutex_init(&ret->lock, NULL);
 	ret->size = size;
+	ret->nr_objs = 0;
+	ret->objs = NULL;
 	ret->ctor = ctor;
 	return ret;
 }
diff --git a/tools/testing/radix-tree/linux/bitops.h b/tools/testing/radix-tree/linux/bitops.h
index 71d58427ab60..a13e9bc76eec 100644
--- a/tools/testing/radix-tree/linux/bitops.h
+++ b/tools/testing/radix-tree/linux/bitops.h
@@ -2,9 +2,14 @@
 #define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
 
 #include <linux/types.h>
+#include <linux/bitops/find.h>
+#include <linux/bitops/hweight.h>
+#include <linux/kernel.h>
 
-#define BITOP_MASK(nr)		(1UL << ((nr) % BITS_PER_LONG))
-#define BITOP_WORD(nr)		((nr) / BITS_PER_LONG)
+#define BIT_MASK(nr)		(1UL << ((nr) % BITS_PER_LONG))
+#define BIT_WORD(nr)		((nr) / BITS_PER_LONG)
+#define BITS_PER_BYTE		8
+#define BITS_TO_LONGS(nr)	DIV_ROUND_UP(nr, BITS_PER_BYTE * sizeof(long))
 
 /**
  * __set_bit - Set a bit in memory
@@ -17,16 +22,16 @@
  */
 static inline void __set_bit(int nr, volatile unsigned long *addr)
 {
-	unsigned long mask = BITOP_MASK(nr);
-	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
+	unsigned long mask = BIT_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 
 	*p  |= mask;
 }
 
 static inline void __clear_bit(int nr, volatile unsigned long *addr)
 {
-	unsigned long mask = BITOP_MASK(nr);
-	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
+	unsigned long mask = BIT_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 
 	*p &= ~mask;
 }
@@ -42,8 +47,8 @@ static inline void __clear_bit(int nr, volatile unsigned long *addr)
  */
 static inline void __change_bit(int nr, volatile unsigned long *addr)
 {
-	unsigned long mask = BITOP_MASK(nr);
-	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
+	unsigned long mask = BIT_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 
 	*p ^= mask;
 }
@@ -59,8 +64,8 @@ static inline void __change_bit(int nr, volatile unsigned long *addr)
  */
 static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
 {
-	unsigned long mask = BITOP_MASK(nr);
-	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
+	unsigned long mask = BIT_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 	unsigned long old = *p;
 
 	*p = old | mask;
@@ -78,8 +83,8 @@ static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
  */
 static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
 {
-	unsigned long mask = BITOP_MASK(nr);
-	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
+	unsigned long mask = BIT_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 	unsigned long old = *p;
 
 	*p = old & ~mask;
@@ -90,8 +95,8 @@ static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
 static inline int __test_and_change_bit(int nr,
 					    volatile unsigned long *addr)
 {
-	unsigned long mask = BITOP_MASK(nr);
-	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
+	unsigned long mask = BIT_MASK(nr);
+	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);
 	unsigned long old = *p;
 
 	*p = old ^ mask;
@@ -105,7 +110,7 @@ static inline int __test_and_change_bit(int nr,
  */
 static inline int test_bit(int nr, const volatile unsigned long *addr)
 {
-	return 1UL & (addr[BITOP_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
+	return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
 }
 
 /**
@@ -147,4 +152,9 @@ unsigned long find_next_bit(const unsigned long *addr,
 			    unsigned long size,
 			    unsigned long offset);
 
+static inline unsigned long hweight_long(unsigned long w)
+{
+	return sizeof(w) == 4 ? hweight32(w) : hweight64(w);
+}
+
 #endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */
diff --git a/tools/testing/radix-tree/linux/bitops/non-atomic.h b/tools/testing/radix-tree/linux/bitops/non-atomic.h
index 46a825cf2ae1..6a1bcb9d2c4a 100644
--- a/tools/testing/radix-tree/linux/bitops/non-atomic.h
+++ b/tools/testing/radix-tree/linux/bitops/non-atomic.h
@@ -3,7 +3,6 @@
 
 #include <asm/types.h>
 
-#define BITOP_MASK(nr)		(1UL << ((nr) % BITS_PER_LONG))
 #define BITOP_WORD(nr)		((nr) / BITS_PER_LONG)
 
 /**
@@ -17,7 +16,7 @@
  */
 static inline void __set_bit(int nr, volatile unsigned long *addr)
 {
-	unsigned long mask = BITOP_MASK(nr);
+	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
 
 	*p  |= mask;
@@ -25,7 +24,7 @@ static inline void __set_bit(int nr, volatile unsigned long *addr)
 
 static inline void __clear_bit(int nr, volatile unsigned long *addr)
 {
-	unsigned long mask = BITOP_MASK(nr);
+	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
 
 	*p &= ~mask;
@@ -42,7 +41,7 @@ static inline void __clear_bit(int nr, volatile unsigned long *addr)
  */
 static inline void __change_bit(int nr, volatile unsigned long *addr)
 {
-	unsigned long mask = BITOP_MASK(nr);
+	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
 
 	*p ^= mask;
@@ -59,7 +58,7 @@ static inline void __change_bit(int nr, volatile unsigned long *addr)
  */
 static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
 {
-	unsigned long mask = BITOP_MASK(nr);
+	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
 	unsigned long old = *p;
 
@@ -78,7 +77,7 @@ static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
  */
 static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
 {
-	unsigned long mask = BITOP_MASK(nr);
+	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
 	unsigned long old = *p;
 
@@ -90,7 +89,7 @@ static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
 static inline int __test_and_change_bit(int nr,
 					    volatile unsigned long *addr)
 {
-	unsigned long mask = BITOP_MASK(nr);
+	unsigned long mask = BIT_MASK(nr);
 	unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
 	unsigned long old = *p;
 
diff --git a/tools/testing/radix-tree/linux/bug.h b/tools/testing/radix-tree/linux/bug.h
index ccbe444977df..23b8ed52f8c8 100644
--- a/tools/testing/radix-tree/linux/bug.h
+++ b/tools/testing/radix-tree/linux/bug.h
@@ -1 +1 @@
-#define WARN_ON_ONCE(x)		assert(x)
+#include "asm/bug.h"
diff --git a/tools/testing/radix-tree/linux/cpu.h b/tools/testing/radix-tree/linux/cpu.h
index 7cf412103205..a45530d78107 100644
--- a/tools/testing/radix-tree/linux/cpu.h
+++ b/tools/testing/radix-tree/linux/cpu.h
@@ -1,21 +1 @@
-
-#define hotcpu_notifier(a, b)
-
-#define CPU_ONLINE		0x0002 /* CPU (unsigned)v is up */
-#define CPU_UP_PREPARE		0x0003 /* CPU (unsigned)v coming up */
-#define CPU_UP_CANCELED		0x0004 /* CPU (unsigned)v NOT coming up */
-#define CPU_DOWN_PREPARE	0x0005 /* CPU (unsigned)v going down */
-#define CPU_DOWN_FAILED		0x0006 /* CPU (unsigned)v NOT going down */
-#define CPU_DEAD		0x0007 /* CPU (unsigned)v dead */
-#define CPU_POST_DEAD		0x0009 /* CPU (unsigned)v dead, cpu_hotplug
-					* lock is dropped */
-#define CPU_BROKEN		0x000C /* CPU (unsigned)v did not die properly,
-					* perhaps due to preemption. */
-#define CPU_TASKS_FROZEN	0x0010
-
-#define CPU_ONLINE_FROZEN	(CPU_ONLINE | CPU_TASKS_FROZEN)
-#define CPU_UP_PREPARE_FROZEN	(CPU_UP_PREPARE | CPU_TASKS_FROZEN)
-#define CPU_UP_CANCELED_FROZEN  (CPU_UP_CANCELED | CPU_TASKS_FROZEN)
-#define CPU_DOWN_PREPARE_FROZEN (CPU_DOWN_PREPARE | CPU_TASKS_FROZEN)
-#define CPU_DOWN_FAILED_FROZEN  (CPU_DOWN_FAILED | CPU_TASKS_FROZEN)
-#define CPU_DEAD_FROZEN		(CPU_DEAD | CPU_TASKS_FROZEN)
+#define cpuhp_setup_state_nocalls(a, b, c, d)	(0)
diff --git a/tools/testing/radix-tree/linux/gfp.h b/tools/testing/radix-tree/linux/gfp.h
index 5201b915f631..5b09b2ce6c33 100644
--- a/tools/testing/radix-tree/linux/gfp.h
+++ b/tools/testing/radix-tree/linux/gfp.h
@@ -3,8 +3,24 @@
 
 #define __GFP_BITS_SHIFT 26
 #define __GFP_BITS_MASK ((gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
-#define __GFP_WAIT 1
-#define __GFP_ACCOUNT 0
-#define __GFP_NOWARN 0
+
+#define __GFP_HIGH		0x20u
+#define __GFP_IO		0x40u
+#define __GFP_FS		0x80u
+#define __GFP_NOWARN		0x200u
+#define __GFP_ATOMIC		0x80000u
+#define __GFP_ACCOUNT		0x100000u
+#define __GFP_DIRECT_RECLAIM	0x400000u
+#define __GFP_KSWAPD_RECLAIM	0x2000000u
+
+#define __GFP_RECLAIM		(__GFP_DIRECT_RECLAIM|__GFP_KSWAPD_RECLAIM)
+
+#define GFP_ATOMIC		(__GFP_HIGH|__GFP_ATOMIC|__GFP_KSWAPD_RECLAIM)
+#define GFP_KERNEL		(__GFP_RECLAIM | __GFP_IO | __GFP_FS)
+
+static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags)
+{
+	return !!(gfp_flags & __GFP_DIRECT_RECLAIM);
+}
 
 #endif
diff --git a/tools/testing/radix-tree/linux/kernel.h b/tools/testing/radix-tree/linux/kernel.h
index be98a47b4e1b..9b43b4975d83 100644
--- a/tools/testing/radix-tree/linux/kernel.h
+++ b/tools/testing/radix-tree/linux/kernel.h
@@ -8,9 +8,14 @@
 #include <limits.h>
 
 #include "../../include/linux/compiler.h"
+#include "../../include/linux/err.h"
 #include "../../../include/linux/kconfig.h"
 
+#ifdef BENCHMARK
+#define RADIX_TREE_MAP_SHIFT	6
+#else
 #define RADIX_TREE_MAP_SHIFT	3
+#endif
 
 #ifndef NULL
 #define NULL	0
@@ -43,4 +48,17 @@ static inline int in_interrupt(void)
 {
 	return 0;
 }
+
+/*
+ * This looks more complex than it should be. But we need to
+ * get the type for the ~ right in round_down (it needs to be
+ * as wide as the result!), and we want to evaluate the macro
+ * arguments just once each.
+ */
+#define __round_mask(x, y) ((__typeof__(x))((y)-1))
+#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
+#define round_down(x, y) ((x) & ~__round_mask(x, y))
+
+#define xchg(ptr, x)	uatomic_xchg(ptr, x)
+
 #endif /* _KERNEL_H */
diff --git a/tools/testing/radix-tree/linux/notifier.h b/tools/testing/radix-tree/linux/notifier.h
deleted file mode 100644
index 70e4797d5a46..000000000000
--- a/tools/testing/radix-tree/linux/notifier.h
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef _NOTIFIER_H
-#define _NOTIFIER_H
-
-struct notifier_block;
-
-#define NOTIFY_OK              0x0001          /* Suits me */
-
-#endif
diff --git a/tools/testing/radix-tree/linux/preempt.h b/tools/testing/radix-tree/linux/preempt.h
index 6210672e3baa..65c04c226965 100644
--- a/tools/testing/radix-tree/linux/preempt.h
+++ b/tools/testing/radix-tree/linux/preempt.h
@@ -1,4 +1,4 @@
-/* */
+extern int preempt_count;
 
-#define preempt_disable() do { } while (0)
-#define preempt_enable() do { } while (0)
+#define preempt_disable()	uatomic_inc(&preempt_count)
+#define preempt_enable()	uatomic_dec(&preempt_count)
diff --git a/tools/testing/radix-tree/linux/slab.h b/tools/testing/radix-tree/linux/slab.h
index 6d5a34770fd4..e40337f41a38 100644
--- a/tools/testing/radix-tree/linux/slab.h
+++ b/tools/testing/radix-tree/linux/slab.h
@@ -7,15 +7,8 @@
 #define SLAB_PANIC 2
 #define SLAB_RECLAIM_ACCOUNT    0x00020000UL            /* Objects are reclaimable */
 
-static inline int gfpflags_allow_blocking(gfp_t mask)
-{
-	return 1;
-}
-
-struct kmem_cache {
-	int size;
-	void (*ctor)(void *);
-};
+void *kmalloc(size_t size, gfp_t);
+void kfree(void *);
 
 void *kmem_cache_alloc(struct kmem_cache *cachep, int flags);
 void kmem_cache_free(struct kmem_cache *cachep, void *objp);
diff --git a/tools/testing/radix-tree/linux/types.h b/tools/testing/radix-tree/linux/types.h
index faa0b6ff9ca8..8491d89873bb 100644
--- a/tools/testing/radix-tree/linux/types.h
+++ b/tools/testing/radix-tree/linux/types.h
@@ -6,8 +6,6 @@
 #define __rcu
 #define __read_mostly
 
-#define BITS_PER_LONG (sizeof(long) * 8)
-
 static inline void INIT_LIST_HEAD(struct list_head *list)
 {
 	list->next = list;
diff --git a/tools/testing/radix-tree/main.c b/tools/testing/radix-tree/main.c
index daa9010693e8..f7e9801a6754 100644
--- a/tools/testing/radix-tree/main.c
+++ b/tools/testing/radix-tree/main.c
@@ -67,7 +67,6 @@ void big_gang_check(bool long_run)
 
 	for (i = 0; i < (long_run ? 1000 : 3); i++) {
 		__big_gang_check();
-		srand(time(0));
 		printf("%d ", i);
 		fflush(stdout);
 	}
@@ -206,8 +205,7 @@ void copy_tag_check(void)
 	}
 
 //	printf("\ncopying tags...\n");
-	cur = start;
-	tagged = radix_tree_range_tag_if_tagged(&tree, &cur, end, ITEMS, 0, 1);
+	tagged = tag_tagged_items(&tree, NULL, start, end, ITEMS, 0, 1);
 
 //	printf("checking copied tags\n");
 	assert(tagged == count);
@@ -215,16 +213,13 @@ void copy_tag_check(void)
 
 	/* Copy tags in several rounds */
 //	printf("\ncopying tags...\n");
-	cur = start;
-	do {
-		tmp = rand() % (count/10+2);
-		tagged = radix_tree_range_tag_if_tagged(&tree, &cur, end, tmp, 0, 2);
-	} while (tmp == tagged);
+	tmp = rand() % (count / 10 + 2);
+	tagged = tag_tagged_items(&tree, NULL, start, end, tmp, 0, 2);
+	assert(tagged == count);
 
 //	printf("%lu %lu %lu\n", tagged, tmp, count);
 //	printf("checking copied tags\n");
 	check_copied_tags(&tree, start, end, idx, ITEMS, 0, 2);
-	assert(tagged < tmp);
 	verify_tag_consistency(&tree, 0);
 	verify_tag_consistency(&tree, 1);
 	verify_tag_consistency(&tree, 2);
@@ -240,7 +235,7 @@ static void __locate_check(struct radix_tree_root *tree, unsigned long index,
 
 	item_insert_order(tree, index, order);
 	item = item_lookup(tree, index);
-	index2 = radix_tree_locate_item(tree, item);
+	index2 = find_item(tree, item);
 	if (index != index2) {
 		printf("index %ld order %d inserted; found %ld\n",
 			index, order, index2);
@@ -274,17 +269,17 @@ static void locate_check(void)
 			     index += (1UL << order)) {
 				__locate_check(&tree, index + offset, order);
 			}
-			if (radix_tree_locate_item(&tree, &tree) != -1)
+			if (find_item(&tree, &tree) != -1)
 				abort();
 
 			item_kill_tree(&tree);
 		}
 	}
 
-	if (radix_tree_locate_item(&tree, &tree) != -1)
+	if (find_item(&tree, &tree) != -1)
 		abort();
 	__locate_check(&tree, -1, 0);
-	if (radix_tree_locate_item(&tree, &tree) != -1)
+	if (find_item(&tree, &tree) != -1)
 		abort();
 	item_kill_tree(&tree);
 }
@@ -293,50 +288,80 @@ static void single_thread_tests(bool long_run)
 {
 	int i;
 
-	printf("starting single_thread_tests: %d allocated\n", nr_allocated);
+	printf("starting single_thread_tests: %d allocated, preempt %d\n",
+		nr_allocated, preempt_count);
 	multiorder_checks();
-	printf("after multiorder_check: %d allocated\n", nr_allocated);
+	rcu_barrier();
+	printf("after multiorder_check: %d allocated, preempt %d\n",
+		nr_allocated, preempt_count);
 	locate_check();
-	printf("after locate_check: %d allocated\n", nr_allocated);
+	rcu_barrier();
+	printf("after locate_check: %d allocated, preempt %d\n",
+		nr_allocated, preempt_count);
 	tag_check();
-	printf("after tag_check: %d allocated\n", nr_allocated);
+	rcu_barrier();
+	printf("after tag_check: %d allocated, preempt %d\n",
+		nr_allocated, preempt_count);
 	gang_check();
-	printf("after gang_check: %d allocated\n", nr_allocated);
+	rcu_barrier();
+	printf("after gang_check: %d allocated, preempt %d\n",
+		nr_allocated, preempt_count);
 	add_and_check();
-	printf("after add_and_check: %d allocated\n", nr_allocated);
+	rcu_barrier();
+	printf("after add_and_check: %d allocated, preempt %d\n",
+		nr_allocated, preempt_count);
 	dynamic_height_check();
-	printf("after dynamic_height_check: %d allocated\n", nr_allocated);
+	rcu_barrier();
+	printf("after dynamic_height_check: %d allocated, preempt %d\n",
+		nr_allocated, preempt_count);
 	big_gang_check(long_run);
-	printf("after big_gang_check: %d allocated\n", nr_allocated);
+	rcu_barrier();
+	printf("after big_gang_check: %d allocated, preempt %d\n",
+		nr_allocated, preempt_count);
 	for (i = 0; i < (long_run ? 2000 : 3); i++) {
 		copy_tag_check();
 		printf("%d ", i);
 		fflush(stdout);
 	}
-	printf("after copy_tag_check: %d allocated\n", nr_allocated);
+	rcu_barrier();
+	printf("after copy_tag_check: %d allocated, preempt %d\n",
+		nr_allocated, preempt_count);
 }
 
 int main(int argc, char **argv)
 {
 	bool long_run = false;
 	int opt;
+	unsigned int seed = time(NULL);
 
-	while ((opt = getopt(argc, argv, "l")) != -1) {
+	while ((opt = getopt(argc, argv, "ls:")) != -1) {
 		if (opt == 'l')
 			long_run = true;
+		else if (opt == 's')
+			seed = strtoul(optarg, NULL, 0);
 	}
 
+	printf("random seed %u\n", seed);
+	srand(seed);
+
 	rcu_register_thread();
 	radix_tree_init();
 
 	regression1_test();
 	regression2_test();
 	regression3_test();
-	iteration_test();
+	iteration_test(0, 10);
+	iteration_test(7, 20);
 	single_thread_tests(long_run);
 
-	sleep(1);
-	printf("after sleep(1): %d allocated\n", nr_allocated);
+	/* Free any remaining preallocated nodes */
+	radix_tree_cpu_dead(0);
+
+	benchmark();
+
+	rcu_barrier();
+	printf("after rcu_barrier: %d allocated, preempt %d\n",
+		nr_allocated, preempt_count);
 	rcu_unregister_thread();
 
 	exit(0);
diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c
index d1be94667a30..f79812a5e070 100644
--- a/tools/testing/radix-tree/multiorder.c
+++ b/tools/testing/radix-tree/multiorder.c
@@ -26,7 +26,6 @@ static void __multiorder_tag_test(int index, int order)
 {
 	RADIX_TREE(tree, GFP_KERNEL);
 	int base, err, i;
-	unsigned long first = 0;
 
 	/* our canonical entry */
 	base = index & ~((1 << order) - 1);
@@ -60,7 +59,7 @@ static void __multiorder_tag_test(int index, int order)
 		assert(!radix_tree_tag_get(&tree, i, 1));
 	}
 
-	assert(radix_tree_range_tag_if_tagged(&tree, &first, ~0UL, 10, 0, 1) == 1);
+	assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 1);
 	assert(radix_tree_tag_clear(&tree, index, 0));
 
 	for_each_index(i, base, order) {
@@ -76,8 +75,27 @@ static void __multiorder_tag_test(int index, int order)
 	item_kill_tree(&tree);
 }
 
+static void __multiorder_tag_test2(unsigned order, unsigned long index2)
+{
+	RADIX_TREE(tree, GFP_KERNEL);
+	unsigned long index = (1 << order);
+	index2 += index;
+
+	assert(item_insert_order(&tree, 0, order) == 0);
+	assert(item_insert(&tree, index2) == 0);
+
+	assert(radix_tree_tag_set(&tree, 0, 0));
+	assert(radix_tree_tag_set(&tree, index2, 0));
+
+	assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 2);
+
+	item_kill_tree(&tree);
+}
+
 static void multiorder_tag_tests(void)
 {
+	int i, j;
+
 	/* test multi-order entry for indices 0-7 with no sibling pointers */
 	__multiorder_tag_test(0, 3);
 	__multiorder_tag_test(5, 3);
@@ -117,6 +135,10 @@ static void multiorder_tag_tests(void)
 	__multiorder_tag_test(300, 8);
 
 	__multiorder_tag_test(0x12345678UL, 8);
+
+	for (i = 1; i < 10; i++)
+		for (j = 0; j < (10 << i); j++)
+			__multiorder_tag_test2(i, j);
 }
 
 static void multiorder_check(unsigned long index, int order)
@@ -125,7 +147,7 @@ static void multiorder_check(unsigned long index, int order)
 	unsigned long min = index & ~((1UL << order) - 1);
 	unsigned long max = min + (1UL << order);
 	void **slot;
-	struct item *item2 = item_create(min);
+	struct item *item2 = item_create(min, order);
 	RADIX_TREE(tree, GFP_KERNEL);
 
 	printf("Multiorder index %ld, order %d\n", index, order);
@@ -231,11 +253,14 @@ void multiorder_iteration(void)
 		radix_tree_for_each_slot(slot, &tree, &iter, j) {
 			int height = order[i] / RADIX_TREE_MAP_SHIFT;
 			int shift = height * RADIX_TREE_MAP_SHIFT;
-			int mask = (1 << order[i]) - 1;
+			unsigned long mask = (1UL << order[i]) - 1;
+			struct item *item = *slot;
 
-			assert(iter.index >= (index[i] &~ mask));
-			assert(iter.index <= (index[i] | mask));
+			assert((iter.index | mask) == (index[i] | mask));
 			assert(iter.shift == shift);
+			assert(!radix_tree_is_internal_node(item));
+			assert((item->index | mask) == (index[i] | mask));
+			assert(item->order == order[i]);
 			i++;
 		}
 	}
@@ -248,7 +273,6 @@ void multiorder_tagged_iteration(void)
 	RADIX_TREE(tree, GFP_KERNEL);
 	struct radix_tree_iter iter;
 	void **slot;
-	unsigned long first = 0;
 	int i, j;
 
 	printf("Multiorder tagged iteration test\n");
@@ -269,7 +293,7 @@ void multiorder_tagged_iteration(void)
 		assert(radix_tree_tag_set(&tree, tag_index[i], 1));
 
 	for (j = 0; j < 256; j++) {
-		int mask, k;
+		int k;
 
 		for (i = 0; i < TAG_ENTRIES; i++) {
 			for (k = i; index[k] < tag_index[i]; k++)
@@ -279,18 +303,22 @@ void multiorder_tagged_iteration(void)
 		}
 
 		radix_tree_for_each_tagged(slot, &tree, &iter, j, 1) {
+			unsigned long mask;
+			struct item *item = *slot;
 			for (k = i; index[k] < tag_index[i]; k++)
 				;
-			mask = (1 << order[k]) - 1;
+			mask = (1UL << order[k]) - 1;
 
-			assert(iter.index >= (tag_index[i] &~ mask));
-			assert(iter.index <= (tag_index[i] | mask));
+			assert((iter.index | mask) == (tag_index[i] | mask));
+			assert(!radix_tree_is_internal_node(item));
+			assert((item->index | mask) == (tag_index[i] | mask));
+			assert(item->order == order[k]);
 			i++;
 		}
 	}
 
-	radix_tree_range_tag_if_tagged(&tree, &first, ~0UL,
-					MT_NUM_ENTRIES, 1, 2);
+	assert(tag_tagged_items(&tree, NULL, 0, ~0UL, TAG_ENTRIES, 1, 2) ==
+				TAG_ENTRIES);
 
 	for (j = 0; j < 256; j++) {
 		int mask, k;
@@ -303,19 +331,21 @@ void multiorder_tagged_iteration(void)
 		}
 
 		radix_tree_for_each_tagged(slot, &tree, &iter, j, 2) {
+			struct item *item = *slot;
 			for (k = i; index[k] < tag_index[i]; k++)
 				;
 			mask = (1 << order[k]) - 1;
 
-			assert(iter.index >= (tag_index[i] &~ mask));
-			assert(iter.index <= (tag_index[i] | mask));
+			assert((iter.index | mask) == (tag_index[i] | mask));
+			assert(!radix_tree_is_internal_node(item));
+			assert((item->index | mask) == (tag_index[i] | mask));
+			assert(item->order == order[k]);
 			i++;
 		}
 	}
 
-	first = 1;
-	radix_tree_range_tag_if_tagged(&tree, &first, ~0UL,
-					MT_NUM_ENTRIES, 1, 0);
+	assert(tag_tagged_items(&tree, NULL, 1, ~0UL, MT_NUM_ENTRIES * 2, 1, 0)
+			== TAG_ENTRIES);
 	i = 0;
 	radix_tree_for_each_tagged(slot, &tree, &iter, 0, 0) {
 		assert(iter.index == tag_index[i]);
@@ -325,6 +355,261 @@ void multiorder_tagged_iteration(void)
 	item_kill_tree(&tree);
 }
 
+static void multiorder_join1(unsigned long index,
+				unsigned order1, unsigned order2)
+{
+	unsigned long loc;
+	void *item, *item2 = item_create(index + 1, order1);
+	RADIX_TREE(tree, GFP_KERNEL);
+
+	item_insert_order(&tree, index, order2);
+	item = radix_tree_lookup(&tree, index);
+	radix_tree_join(&tree, index + 1, order1, item2);
+	loc = find_item(&tree, item);
+	if (loc == -1)
+		free(item);
+	item = radix_tree_lookup(&tree, index + 1);
+	assert(item == item2);
+	item_kill_tree(&tree);
+}
+
+static void multiorder_join2(unsigned order1, unsigned order2)
+{
+	RADIX_TREE(tree, GFP_KERNEL);
+	struct radix_tree_node *node;
+	void *item1 = item_create(0, order1);
+	void *item2;
+
+	item_insert_order(&tree, 0, order2);
+	radix_tree_insert(&tree, 1 << order2, (void *)0x12UL);
+	item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL);
+	assert(item2 == (void *)0x12UL);
+	assert(node->exceptional == 1);
+
+	radix_tree_join(&tree, 0, order1, item1);
+	item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL);
+	assert(item2 == item1);
+	assert(node->exceptional == 0);
+	item_kill_tree(&tree);
+}
+
+/*
+ * This test revealed an accounting bug for exceptional entries at one point.
+ * Nodes were being freed back into the pool with an elevated exception count
+ * by radix_tree_join() and then radix_tree_split() was failing to zero the
+ * count of exceptional entries.
+ */
+static void multiorder_join3(unsigned int order)
+{
+	RADIX_TREE(tree, GFP_KERNEL);
+	struct radix_tree_node *node;
+	void **slot;
+	struct radix_tree_iter iter;
+	unsigned long i;
+
+	for (i = 0; i < (1 << order); i++) {
+		radix_tree_insert(&tree, i, (void *)0x12UL);
+	}
+
+	radix_tree_join(&tree, 0, order, (void *)0x16UL);
+	rcu_barrier();
+
+	radix_tree_split(&tree, 0, 0);
+
+	radix_tree_for_each_slot(slot, &tree, &iter, 0) {
+		radix_tree_iter_replace(&tree, &iter, slot, (void *)0x12UL);
+	}
+
+	__radix_tree_lookup(&tree, 0, &node, NULL);
+	assert(node->exceptional == node->count);
+
+	item_kill_tree(&tree);
+}
+
+static void multiorder_join(void)
+{
+	int i, j, idx;
+
+	for (idx = 0; idx < 1024; idx = idx * 2 + 3) {
+		for (i = 1; i < 15; i++) {
+			for (j = 0; j < i; j++) {
+				multiorder_join1(idx, i, j);
+			}
+		}
+	}
+
+	for (i = 1; i < 15; i++) {
+		for (j = 0; j < i; j++) {
+			multiorder_join2(i, j);
+		}
+	}
+
+	for (i = 3; i < 10; i++) {
+		multiorder_join3(i);
+	}
+}
+
+static void check_mem(unsigned old_order, unsigned new_order, unsigned alloc)
+{
+	struct radix_tree_preload *rtp = &radix_tree_preloads;
+	if (rtp->nr != 0)
+		printf("split(%u %u) remaining %u\n", old_order, new_order,
+							rtp->nr);
+	/*
+	 * Can't check for equality here as some nodes may have been
+	 * RCU-freed while we ran.  But we should never finish with more
+	 * nodes allocated since they should have all been preloaded.
+	 */
+	if (nr_allocated > alloc)
+		printf("split(%u %u) allocated %u %u\n", old_order, new_order,
+							alloc, nr_allocated);
+}
+
+static void __multiorder_split(int old_order, int new_order)
+{
+	RADIX_TREE(tree, GFP_ATOMIC);
+	void **slot;
+	struct radix_tree_iter iter;
+	unsigned alloc;
+
+	radix_tree_preload(GFP_KERNEL);
+	assert(item_insert_order(&tree, 0, old_order) == 0);
+	radix_tree_preload_end();
+
+	/* Wipe out the preloaded cache or it'll confuse check_mem() */
+	radix_tree_cpu_dead(0);
+
+	radix_tree_tag_set(&tree, 0, 2);
+
+	radix_tree_split_preload(old_order, new_order, GFP_KERNEL);
+	alloc = nr_allocated;
+	radix_tree_split(&tree, 0, new_order);
+	check_mem(old_order, new_order, alloc);
+	radix_tree_for_each_slot(slot, &tree, &iter, 0) {
+		radix_tree_iter_replace(&tree, &iter, slot,
+					item_create(iter.index, new_order));
+	}
+	radix_tree_preload_end();
+
+	item_kill_tree(&tree);
+}
+
+static void __multiorder_split2(int old_order, int new_order)
+{
+	RADIX_TREE(tree, GFP_KERNEL);
+	void **slot;
+	struct radix_tree_iter iter;
+	struct radix_tree_node *node;
+	void *item;
+
+	__radix_tree_insert(&tree, 0, old_order, (void *)0x12);
+
+	item = __radix_tree_lookup(&tree, 0, &node, NULL);
+	assert(item == (void *)0x12);
+	assert(node->exceptional > 0);
+
+	radix_tree_split(&tree, 0, new_order);
+	radix_tree_for_each_slot(slot, &tree, &iter, 0) {
+		radix_tree_iter_replace(&tree, &iter, slot,
+					item_create(iter.index, new_order));
+	}
+
+	item = __radix_tree_lookup(&tree, 0, &node, NULL);
+	assert(item != (void *)0x12);
+	assert(node->exceptional == 0);
+
+	item_kill_tree(&tree);
+}
+
+static void __multiorder_split3(int old_order, int new_order)
+{
+	RADIX_TREE(tree, GFP_KERNEL);
+	void **slot;
+	struct radix_tree_iter iter;
+	struct radix_tree_node *node;
+	void *item;
+
+	__radix_tree_insert(&tree, 0, old_order, (void *)0x12);
+
+	item = __radix_tree_lookup(&tree, 0, &node, NULL);
+	assert(item == (void *)0x12);
+	assert(node->exceptional > 0);
+
+	radix_tree_split(&tree, 0, new_order);
+	radix_tree_for_each_slot(slot, &tree, &iter, 0) {
+		radix_tree_iter_replace(&tree, &iter, slot, (void *)0x16);
+	}
+
+	item = __radix_tree_lookup(&tree, 0, &node, NULL);
+	assert(item == (void *)0x16);
+	assert(node->exceptional > 0);
+
+	item_kill_tree(&tree);
+
+	__radix_tree_insert(&tree, 0, old_order, (void *)0x12);
+
+	item = __radix_tree_lookup(&tree, 0, &node, NULL);
+	assert(item == (void *)0x12);
+	assert(node->exceptional > 0);
+
+	radix_tree_split(&tree, 0, new_order);
+	radix_tree_for_each_slot(slot, &tree, &iter, 0) {
+		if (iter.index == (1 << new_order))
+			radix_tree_iter_replace(&tree, &iter, slot,
+						(void *)0x16);
+		else
+			radix_tree_iter_replace(&tree, &iter, slot, NULL);
+	}
+
+	item = __radix_tree_lookup(&tree, 1 << new_order, &node, NULL);
+	assert(item == (void *)0x16);
+	assert(node->count == node->exceptional);
+	do {
+		node = node->parent;
+		if (!node)
+			break;
+		assert(node->count == 1);
+		assert(node->exceptional == 0);
+	} while (1);
+
+	item_kill_tree(&tree);
+}
+
+static void multiorder_split(void)
+{
+	int i, j;
+
+	for (i = 3; i < 11; i++)
+		for (j = 0; j < i; j++) {
+			__multiorder_split(i, j);
+			__multiorder_split2(i, j);
+			__multiorder_split3(i, j);
+		}
+}
+
+static void multiorder_account(void)
+{
+	RADIX_TREE(tree, GFP_KERNEL);
+	struct radix_tree_node *node;
+	void **slot;
+
+	item_insert_order(&tree, 0, 5);
+
+	__radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12);
+	__radix_tree_lookup(&tree, 0, &node, NULL);
+	assert(node->count == node->exceptional * 2);
+	radix_tree_delete(&tree, 1 << 5);
+	assert(node->exceptional == 0);
+
+	__radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12);
+	__radix_tree_lookup(&tree, 1 << 5, &node, &slot);
+	assert(node->count == node->exceptional * 2);
+	__radix_tree_replace(&tree, node, slot, NULL, NULL, NULL);
+	assert(node->exceptional == 0);
+
+	item_kill_tree(&tree);
+}
+
 void multiorder_checks(void)
 {
 	int i;
@@ -342,4 +627,9 @@ void multiorder_checks(void)
 	multiorder_tag_tests();
 	multiorder_iteration();
 	multiorder_tagged_iteration();
+	multiorder_join();
+	multiorder_split();
+	multiorder_account();
+
+	radix_tree_cpu_dead(0);
 }
diff --git a/tools/testing/radix-tree/rcupdate.c b/tools/testing/radix-tree/rcupdate.c
deleted file mode 100644
index 31a2d14225d6..000000000000
--- a/tools/testing/radix-tree/rcupdate.c
+++ /dev/null
@@ -1,86 +0,0 @@
-#include <linux/rcupdate.h>
-#include <pthread.h>
-#include <stdio.h>
-#include <assert.h>
-
-static pthread_mutex_t rculock = PTHREAD_MUTEX_INITIALIZER;
-static struct rcu_head *rcuhead_global = NULL;
-static __thread int nr_rcuhead = 0;
-static __thread struct rcu_head *rcuhead = NULL;
-static __thread struct rcu_head *rcutail = NULL;
-
-static pthread_cond_t rcu_worker_cond = PTHREAD_COND_INITIALIZER;
-
-/* switch to urcu implementation when it is merged. */
-void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *head))
-{
-	head->func = func;
-	head->next = rcuhead;
-	rcuhead = head;
-	if (!rcutail)
-		rcutail = head;
-	nr_rcuhead++;
-	if (nr_rcuhead >= 1000) {
-		int signal = 0;
-
-		pthread_mutex_lock(&rculock);
-		if (!rcuhead_global)
-			signal = 1;
-		rcutail->next = rcuhead_global;
-		rcuhead_global = head;
-		pthread_mutex_unlock(&rculock);
-
-		nr_rcuhead = 0;
-		rcuhead = NULL;
-		rcutail = NULL;
-
-		if (signal) {
-			pthread_cond_signal(&rcu_worker_cond);
-		}
-	}
-}
-
-static void *rcu_worker(void *arg)
-{
-	struct rcu_head *r;
-
-	rcupdate_thread_init();
-
-	while (1) {
-		pthread_mutex_lock(&rculock);
-		while (!rcuhead_global) {
-			pthread_cond_wait(&rcu_worker_cond, &rculock);
-		}
-		r = rcuhead_global;
-		rcuhead_global = NULL;
-
-		pthread_mutex_unlock(&rculock);
-
-		synchronize_rcu();
-
-		while (r) {
-			struct rcu_head *tmp = r->next;
-			r->func(r);
-			r = tmp;
-		}
-	}
-
-	rcupdate_thread_exit();
-
-	return NULL;
-}
-
-static pthread_t worker_thread;
-void rcupdate_init(void)
-{
-	pthread_create(&worker_thread, NULL, rcu_worker, NULL);
-}
-
-void rcupdate_thread_init(void)
-{
-	rcu_register_thread();
-}
-void rcupdate_thread_exit(void)
-{
-	rcu_unregister_thread();
-}
diff --git a/tools/testing/radix-tree/regression2.c b/tools/testing/radix-tree/regression2.c
index 63bf347aaf33..a41325d7a170 100644
--- a/tools/testing/radix-tree/regression2.c
+++ b/tools/testing/radix-tree/regression2.c
@@ -50,6 +50,7 @@
 #include <stdio.h>
 
 #include "regression.h"
+#include "test.h"
 
 #define PAGECACHE_TAG_DIRTY     0
 #define PAGECACHE_TAG_WRITEBACK 1
@@ -90,7 +91,7 @@ void regression2_test(void)
 	/* 1. */
 	start = 0;
 	end = max_slots - 2;
-	radix_tree_range_tag_if_tagged(&mt_tree, &start, end, 1,
+	tag_tagged_items(&mt_tree, NULL, start, end, 1,
 				PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE);
 
 	/* 2. */
diff --git a/tools/testing/radix-tree/regression3.c b/tools/testing/radix-tree/regression3.c
index 1f06ed73d0a8..b594841fae85 100644
--- a/tools/testing/radix-tree/regression3.c
+++ b/tools/testing/radix-tree/regression3.c
@@ -5,7 +5,7 @@
  * In following radix_tree_next_slot current chunk size becomes zero.
  * This isn't checked and it tries to dereference null pointer in slot.
  *
- * Helper radix_tree_iter_next reset slot to NULL and next_index to index + 1,
+ * Helper radix_tree_iter_resume reset slot to NULL and next_index to index + 1,
  * for tagger iteraction it also must reset cached tags in iterator to abort
  * next radix_tree_next_slot and go to slow-path into radix_tree_next_chunk.
  *
@@ -88,7 +88,7 @@ void regression3_test(void)
 		printf("slot %ld %p\n", iter.index, *slot);
 		if (!iter.index) {
 			printf("next at %ld\n", iter.index);
-			slot = radix_tree_iter_next(&iter);
+			slot = radix_tree_iter_resume(slot, &iter);
 		}
 	}
 
@@ -96,7 +96,7 @@ void regression3_test(void)
 		printf("contig %ld %p\n", iter.index, *slot);
 		if (!iter.index) {
 			printf("next at %ld\n", iter.index);
-			slot = radix_tree_iter_next(&iter);
+			slot = radix_tree_iter_resume(slot, &iter);
 		}
 	}
 
@@ -106,7 +106,7 @@ void regression3_test(void)
 		printf("tagged %ld %p\n", iter.index, *slot);
 		if (!iter.index) {
 			printf("next at %ld\n", iter.index);
-			slot = radix_tree_iter_next(&iter);
+			slot = radix_tree_iter_resume(slot, &iter);
 		}
 	}
 
diff --git a/tools/testing/radix-tree/tag_check.c b/tools/testing/radix-tree/tag_check.c
index b0ac05741750..fd98c132207a 100644
--- a/tools/testing/radix-tree/tag_check.c
+++ b/tools/testing/radix-tree/tag_check.c
@@ -23,7 +23,7 @@ __simple_checks(struct radix_tree_root *tree, unsigned long index, int tag)
 	item_tag_set(tree, index, tag);
 	ret = item_tag_get(tree, index, tag);
 	assert(ret != 0);
-	ret = radix_tree_range_tag_if_tagged(tree, &first, ~0UL, 10, tag, !tag);
+	ret = tag_tagged_items(tree, NULL, first, ~0UL, 10, tag, !tag);
 	assert(ret == 1);
 	ret = item_tag_get(tree, index, !tag);
 	assert(ret != 0);
@@ -51,6 +51,7 @@ void simple_checks(void)
 	verify_tag_consistency(&tree, 1);
 	printf("before item_kill_tree: %d allocated\n", nr_allocated);
 	item_kill_tree(&tree);
+	rcu_barrier();
 	printf("after item_kill_tree: %d allocated\n", nr_allocated);
 }
 
@@ -319,10 +320,13 @@ static void single_check(void)
 	assert(ret == 0);
 	verify_tag_consistency(&tree, 0);
 	verify_tag_consistency(&tree, 1);
-	ret = radix_tree_range_tag_if_tagged(&tree, &first, 10, 10, 0, 1);
+	ret = tag_tagged_items(&tree, NULL, first, 10, 10, 0, 1);
 	assert(ret == 1);
 	ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 0, BATCH, 1);
 	assert(ret == 1);
+	item_tag_clear(&tree, 0, 0);
+	ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 0, BATCH, 0);
+	assert(ret == 0);
 	item_kill_tree(&tree);
 }
 
@@ -331,12 +335,16 @@ void tag_check(void)
 	single_check();
 	extend_checks();
 	contract_checks();
+	rcu_barrier();
 	printf("after extend_checks: %d allocated\n", nr_allocated);
 	__leak_check();
 	leak_check();
+	rcu_barrier();
 	printf("after leak_check: %d allocated\n", nr_allocated);
 	simple_checks();
+	rcu_barrier();
 	printf("after simple_checks: %d allocated\n", nr_allocated);
 	thrash_tags();
+	rcu_barrier();
 	printf("after thrash_tags: %d allocated\n", nr_allocated);
 }
diff --git a/tools/testing/radix-tree/test.c b/tools/testing/radix-tree/test.c
index a6e8099eaf4f..e5726e373646 100644
--- a/tools/testing/radix-tree/test.c
+++ b/tools/testing/radix-tree/test.c
@@ -24,21 +24,29 @@ int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag)
 	return radix_tree_tag_get(root, index, tag);
 }
 
-int __item_insert(struct radix_tree_root *root, struct item *item,
-			unsigned order)
+int __item_insert(struct radix_tree_root *root, struct item *item)
 {
-	return __radix_tree_insert(root, item->index, order, item);
+	return __radix_tree_insert(root, item->index, item->order, item);
 }
 
 int item_insert(struct radix_tree_root *root, unsigned long index)
 {
-	return __item_insert(root, item_create(index), 0);
+	return __item_insert(root, item_create(index, 0));
 }
 
 int item_insert_order(struct radix_tree_root *root, unsigned long index,
 			unsigned order)
 {
-	return __item_insert(root, item_create(index), order);
+	return __item_insert(root, item_create(index, order));
+}
+
+void item_sanity(struct item *item, unsigned long index)
+{
+	unsigned long mask;
+	assert(!radix_tree_is_internal_node(item));
+	assert(item->order < BITS_PER_LONG);
+	mask = (1UL << item->order) - 1;
+	assert((item->index | mask) == (index | mask));
 }
 
 int item_delete(struct radix_tree_root *root, unsigned long index)
@@ -46,18 +54,19 @@ int item_delete(struct radix_tree_root *root, unsigned long index)
 	struct item *item = radix_tree_delete(root, index);
 
 	if (item) {
-		assert(item->index == index);
+		item_sanity(item, index);
 		free(item);
 		return 1;
 	}
 	return 0;
 }
 
-struct item *item_create(unsigned long index)
+struct item *item_create(unsigned long index, unsigned int order)
 {
 	struct item *ret = malloc(sizeof(*ret));
 
 	ret->index = index;
+	ret->order = order;
 	return ret;
 }
 
@@ -66,8 +75,8 @@ void item_check_present(struct radix_tree_root *root, unsigned long index)
 	struct item *item;
 
 	item = radix_tree_lookup(root, index);
-	assert(item != 0);
-	assert(item->index == index);
+	assert(item != NULL);
+	item_sanity(item, index);
 }
 
 struct item *item_lookup(struct radix_tree_root *root, unsigned long index)
@@ -80,7 +89,7 @@ void item_check_absent(struct radix_tree_root *root, unsigned long index)
 	struct item *item;
 
 	item = radix_tree_lookup(root, index);
-	assert(item == 0);
+	assert(item == NULL);
 }
 
 /*
@@ -142,6 +151,62 @@ void item_full_scan(struct radix_tree_root *root, unsigned long start,
 	assert(nfound == 0);
 }
 
+/* Use the same pattern as tag_pages_for_writeback() in mm/page-writeback.c */
+int tag_tagged_items(struct radix_tree_root *root, pthread_mutex_t *lock,
+			unsigned long start, unsigned long end, unsigned batch,
+			unsigned iftag, unsigned thentag)
+{
+	unsigned long tagged = 0;
+	struct radix_tree_iter iter;
+	void **slot;
+
+	if (batch == 0)
+		batch = 1;
+
+	if (lock)
+		pthread_mutex_lock(lock);
+	radix_tree_for_each_tagged(slot, root, &iter, start, iftag) {
+		if (iter.index > end)
+			break;
+		radix_tree_iter_tag_set(root, &iter, thentag);
+		tagged++;
+		if ((tagged % batch) != 0)
+			continue;
+		slot = radix_tree_iter_resume(slot, &iter);
+		if (lock) {
+			pthread_mutex_unlock(lock);
+			rcu_barrier();
+			pthread_mutex_lock(lock);
+		}
+	}
+	if (lock)
+		pthread_mutex_unlock(lock);
+
+	return tagged;
+}
+
+/* Use the same pattern as find_swap_entry() in mm/shmem.c */
+unsigned long find_item(struct radix_tree_root *root, void *item)
+{
+	struct radix_tree_iter iter;
+	void **slot;
+	unsigned long found = -1;
+	unsigned long checked = 0;
+
+	radix_tree_for_each_slot(slot, root, &iter, 0) {
+		if (*slot == item) {
+			found = iter.index;
+			break;
+		}
+		checked++;
+		if ((checked % 4) != 0)
+			continue;
+		slot = radix_tree_iter_resume(slot, &iter);
+	}
+
+	return found;
+}
+
 static int verify_node(struct radix_tree_node *slot, unsigned int tag,
 			int tagged)
 {
@@ -200,9 +265,16 @@ void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag)
 
 void item_kill_tree(struct radix_tree_root *root)
 {
+	struct radix_tree_iter iter;
+	void **slot;
 	struct item *items[32];
 	int nfound;
 
+	radix_tree_for_each_slot(slot, root, &iter, 0) {
+		if (radix_tree_exceptional_entry(*slot))
+			radix_tree_delete(root, iter.index);
+	}
+
 	while ((nfound = radix_tree_gang_lookup(root, (void **)items, 0, 32))) {
 		int i;
 
diff --git a/tools/testing/radix-tree/test.h b/tools/testing/radix-tree/test.h
index 217fb2403f09..056a23b56467 100644
--- a/tools/testing/radix-tree/test.h
+++ b/tools/testing/radix-tree/test.h
@@ -5,11 +5,11 @@
 
 struct item {
 	unsigned long index;
+	unsigned int order;
 };
 
-struct item *item_create(unsigned long index);
-int __item_insert(struct radix_tree_root *root, struct item *item,
-			unsigned order);
+struct item *item_create(unsigned long index, unsigned int order);
+int __item_insert(struct radix_tree_root *root, struct item *item);
 int item_insert(struct radix_tree_root *root, unsigned long index);
 int item_insert_order(struct radix_tree_root *root, unsigned long index,
 			unsigned order);
@@ -25,9 +25,15 @@ void item_full_scan(struct radix_tree_root *root, unsigned long start,
 			unsigned long nr, int chunk);
 void item_kill_tree(struct radix_tree_root *root);
 
+int tag_tagged_items(struct radix_tree_root *, pthread_mutex_t *,
+			unsigned long start, unsigned long end, unsigned batch,
+			unsigned iftag, unsigned thentag);
+unsigned long find_item(struct radix_tree_root *, void *item);
+
 void tag_check(void);
 void multiorder_checks(void);
-void iteration_test(void);
+void iteration_test(unsigned order, unsigned duration);
+void benchmark(void);
 
 struct item *
 item_tag_set(struct radix_tree_root *root, unsigned long index, int tag);
@@ -40,7 +46,14 @@ void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag);
 extern int nr_allocated;
 
 /* Normally private parts of lib/radix-tree.c */
+struct radix_tree_node *entry_to_node(void *ptr);
 void radix_tree_dump(struct radix_tree_root *root);
 int root_tag_get(struct radix_tree_root *root, unsigned int tag);
 unsigned long node_maxindex(struct radix_tree_node *);
 unsigned long shift_maxindex(unsigned int shift);
+int radix_tree_cpu_dead(unsigned int cpu);
+struct radix_tree_preload {
+	unsigned nr;
+	struct radix_tree_node *nodes;
+};
+extern struct radix_tree_preload radix_tree_preloads;
diff --git a/tools/testing/selftests/.gitignore b/tools/testing/selftests/.gitignore
new file mode 100644
index 000000000000..f0600d20ce7d
--- /dev/null
+++ b/tools/testing/selftests/.gitignore
@@ -0,0 +1 @@
+kselftest
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index a3144a3de3a8..71b05891a6a1 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -7,6 +7,7 @@ TARGETS += exec
 TARGETS += firmware
 TARGETS += ftrace
 TARGETS += futex
+TARGETS += gpio
 TARGETS += ipc
 TARGETS += kcmp
 TARGETS += lib
@@ -24,6 +25,7 @@ TARGETS += seccomp
 TARGETS += sigaltstack
 TARGETS += size
 TARGETS += static_keys
+TARGETS += sync
 TARGETS += sysctl
 ifneq (1, $(quicktest))
 TARGETS += timers
diff --git a/tools/testing/selftests/breakpoints/Makefile b/tools/testing/selftests/breakpoints/Makefile
index 74e533fd4bc5..61b79e8df1f4 100644
--- a/tools/testing/selftests/breakpoints/Makefile
+++ b/tools/testing/selftests/breakpoints/Makefile
@@ -5,6 +5,9 @@ ARCH ?= $(shell echo $(uname_M) | sed -e s/i.86/x86/ -e s/x86_64/x86/)
 ifeq ($(ARCH),x86)
 TEST_PROGS := breakpoint_test
 endif
+ifeq ($(ARCH),aarch64)
+TEST_PROGS := breakpoint_test_arm64
+endif
 
 TEST_PROGS += step_after_suspend_test
 
@@ -13,4 +16,4 @@ all: $(TEST_PROGS)
 include ../lib.mk
 
 clean:
-	rm -fr breakpoint_test step_after_suspend_test
+	rm -fr breakpoint_test breakpoint_test_arm64 step_after_suspend_test
diff --git a/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c b/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c
new file mode 100644
index 000000000000..3897e996541e
--- /dev/null
+++ b/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c
@@ -0,0 +1,236 @@
+/*
+ * Copyright (C) 2016 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * Original Code by Pavel Labath <labath@google.com>
+ *
+ * Code modified by Pratyush Anand <panand@redhat.com>
+ * for testing different byte select for each access size.
+ *
+ */
+
+#define _GNU_SOURCE
+
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/ptrace.h>
+#include <sys/param.h>
+#include <sys/uio.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <elf.h>
+#include <errno.h>
+#include <signal.h>
+
+#include "../kselftest.h"
+
+static volatile uint8_t var[96] __attribute__((__aligned__(32)));
+
+static void child(int size, int wr)
+{
+	volatile uint8_t *addr = &var[32 + wr];
+
+	if (ptrace(PTRACE_TRACEME, 0, NULL, NULL) != 0) {
+		perror("ptrace(PTRACE_TRACEME) failed");
+		_exit(1);
+	}
+
+	if (raise(SIGSTOP) != 0) {
+		perror("raise(SIGSTOP) failed");
+		_exit(1);
+	}
+
+	if ((uintptr_t) addr % size) {
+		perror("Wrong address write for the given size\n");
+		_exit(1);
+	}
+	switch (size) {
+	case 1:
+		*addr = 47;
+		break;
+	case 2:
+		*(uint16_t *)addr = 47;
+		break;
+	case 4:
+		*(uint32_t *)addr = 47;
+		break;
+	case 8:
+		*(uint64_t *)addr = 47;
+		break;
+	case 16:
+		__asm__ volatile ("stp x29, x30, %0" : "=m" (addr[0]));
+		break;
+	case 32:
+		__asm__ volatile ("stp q29, q30, %0" : "=m" (addr[0]));
+		break;
+	}
+
+	_exit(0);
+}
+
+static bool set_watchpoint(pid_t pid, int size, int wp)
+{
+	const volatile uint8_t *addr = &var[32 + wp];
+	const int offset = (uintptr_t)addr % 8;
+	const unsigned int byte_mask = ((1 << size) - 1) << offset;
+	const unsigned int type = 2; /* Write */
+	const unsigned int enable = 1;
+	const unsigned int control = byte_mask << 5 | type << 3 | enable;
+	struct user_hwdebug_state dreg_state;
+	struct iovec iov;
+
+	memset(&dreg_state, 0, sizeof(dreg_state));
+	dreg_state.dbg_regs[0].addr = (uintptr_t)(addr - offset);
+	dreg_state.dbg_regs[0].ctrl = control;
+	iov.iov_base = &dreg_state;
+	iov.iov_len = offsetof(struct user_hwdebug_state, dbg_regs) +
+				sizeof(dreg_state.dbg_regs[0]);
+	if (ptrace(PTRACE_SETREGSET, pid, NT_ARM_HW_WATCH, &iov) == 0)
+		return true;
+
+	if (errno == EIO) {
+		printf("ptrace(PTRACE_SETREGSET, NT_ARM_HW_WATCH) "
+			"not supported on this hardware\n");
+		ksft_exit_skip();
+	}
+	perror("ptrace(PTRACE_SETREGSET, NT_ARM_HW_WATCH) failed");
+	return false;
+}
+
+static bool run_test(int wr_size, int wp_size, int wr, int wp)
+{
+	int status;
+	siginfo_t siginfo;
+	pid_t pid = fork();
+	pid_t wpid;
+
+	if (pid < 0) {
+		perror("fork() failed");
+		return false;
+	}
+	if (pid == 0)
+		child(wr_size, wr);
+
+	wpid = waitpid(pid, &status, __WALL);
+	if (wpid != pid) {
+		perror("waitpid() failed");
+		return false;
+	}
+	if (!WIFSTOPPED(status)) {
+		printf("child did not stop\n");
+		return false;
+	}
+	if (WSTOPSIG(status) != SIGSTOP) {
+		printf("child did not stop with SIGSTOP\n");
+		return false;
+	}
+
+	if (!set_watchpoint(pid, wp_size, wp))
+		return false;
+
+	if (ptrace(PTRACE_CONT, pid, NULL, NULL) < 0) {
+		perror("ptrace(PTRACE_SINGLESTEP) failed");
+		return false;
+	}
+
+	alarm(3);
+	wpid = waitpid(pid, &status, __WALL);
+	if (wpid != pid) {
+		perror("waitpid() failed");
+		return false;
+	}
+	alarm(0);
+	if (WIFEXITED(status)) {
+		printf("child did not single-step\t");
+		return false;
+	}
+	if (!WIFSTOPPED(status)) {
+		printf("child did not stop\n");
+		return false;
+	}
+	if (WSTOPSIG(status) != SIGTRAP) {
+		printf("child did not stop with SIGTRAP\n");
+		return false;
+	}
+	if (ptrace(PTRACE_GETSIGINFO, pid, NULL, &siginfo) != 0) {
+		perror("ptrace(PTRACE_GETSIGINFO)");
+		return false;
+	}
+	if (siginfo.si_code != TRAP_HWBKPT) {
+		printf("Unexpected si_code %d\n", siginfo.si_code);
+		return false;
+	}
+
+	kill(pid, SIGKILL);
+	wpid = waitpid(pid, &status, 0);
+	if (wpid != pid) {
+		perror("waitpid() failed");
+		return false;
+	}
+	return true;
+}
+
+static void sigalrm(int sig)
+{
+}
+
+int main(int argc, char **argv)
+{
+	int opt;
+	bool succeeded = true;
+	struct sigaction act;
+	int wr, wp, size;
+	bool result;
+
+	act.sa_handler = sigalrm;
+	sigemptyset(&act.sa_mask);
+	act.sa_flags = 0;
+	sigaction(SIGALRM, &act, NULL);
+	for (size = 1; size <= 32; size = size*2) {
+		for (wr = 0; wr <= 32; wr = wr + size) {
+			for (wp = wr - size; wp <= wr + size; wp = wp + size) {
+				printf("Test size = %d write offset = %d watchpoint offset = %d\t", size, wr, wp);
+				result = run_test(size, MIN(size, 8), wr, wp);
+				if ((result && wr == wp) || (!result && wr != wp)) {
+					printf("[OK]\n");
+					ksft_inc_pass_cnt();
+				} else {
+					printf("[FAILED]\n");
+					ksft_inc_fail_cnt();
+					succeeded = false;
+				}
+			}
+		}
+	}
+
+	for (size = 1; size <= 32; size = size*2) {
+		printf("Test size = %d write offset = %d watchpoint offset = -8\t", size, -size);
+
+		if (run_test(size, 8, -size, -8)) {
+			printf("[OK]\n");
+			ksft_inc_pass_cnt();
+		} else {
+			printf("[FAILED]\n");
+			ksft_inc_fail_cnt();
+			succeeded = false;
+		}
+	}
+
+	ksft_print_cnts();
+	if (succeeded)
+		ksft_exit_pass();
+	else
+		ksft_exit_fail();
+}
diff --git a/tools/testing/selftests/drivers/gpu/i915.sh b/tools/testing/selftests/drivers/gpu/i915.sh
new file mode 100755
index 000000000000..d407f0fa1e3a
--- /dev/null
+++ b/tools/testing/selftests/drivers/gpu/i915.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+# Runs hardware independent tests for i915 (drivers/gpu/drm/i915)
+
+if ! /sbin/modprobe -q -r i915; then
+	echo "drivers/gpu/i915: [SKIP]"
+	exit 77
+fi
+
+if /sbin/modprobe -q i915 mock_selftests=-1; then
+	echo "drivers/gpu/i915: ok"
+else
+	echo "drivers/gpu/i915: [FAIL]"
+	exit 1
+fi
diff --git a/tools/testing/selftests/ftrace/.gitignore b/tools/testing/selftests/ftrace/.gitignore
new file mode 100644
index 000000000000..98d8a5a63049
--- /dev/null
+++ b/tools/testing/selftests/ftrace/.gitignore
@@ -0,0 +1 @@
+logs
diff --git a/tools/testing/selftests/ftrace/ftracetest b/tools/testing/selftests/ftrace/ftracetest
index 4c6a0bf8ba79..52e3c4df28d6 100755
--- a/tools/testing/selftests/ftrace/ftracetest
+++ b/tools/testing/selftests/ftrace/ftracetest
@@ -13,7 +13,8 @@ echo "Usage: ftracetest [options] [testcase(s)] [testcase-directory(s)]"
 echo " Options:"
 echo "		-h|--help  Show help message"
 echo "		-k|--keep  Keep passed test logs"
-echo "		-v|--verbose Show all stdout messages in testcases"
+echo "		-v|--verbose Increase verbosity of test messages"
+echo "		-vv        Alias of -v -v (Show all results in stdout)"
 echo "		-d|--debug Debug mode (trace all shell commands)"
 exit $1
 }
@@ -54,8 +55,9 @@ parse_opts() { # opts
       KEEP_LOG=1
       shift 1
     ;;
-    --verbose|-v)
-      VERBOSE=1
+    --verbose|-v|-vv)
+      VERBOSE=$((VERBOSE + 1))
+      [ $1 == '-vv' ] && VERBOSE=$((VERBOSE + 1))
       shift 1
     ;;
     --debug|-d)
@@ -228,7 +230,7 @@ trap 'SIG_RESULT=$XFAIL' $SIG_XFAIL
 
 __run_test() { # testfile
   # setup PID and PPID, $$ is not updated.
-  (cd $TRACING_DIR; read PID _ < /proc/self/stat ; set -e; set -x; . $1)
+  (cd $TRACING_DIR; read PID _ < /proc/self/stat; set -e; set -x; initialize_ftrace; . $1)
   [ $? -ne 0 ] && kill -s $SIG_FAIL $SIG_PID
 }
 
@@ -236,10 +238,11 @@ __run_test() { # testfile
 run_test() { # testfile
   local testname=`basename $1`
   local testlog=`mktemp $LOG_DIR/${testname}-log.XXXXXX`
+  export TMPDIR=`mktemp -d /tmp/ftracetest-dir.XXXXXX`
   testcase $1
   echo "execute: "$1 > $testlog
   SIG_RESULT=0
-  if [ $VERBOSE -ne 0 ]; then
+  if [ $VERBOSE -ge 2 ]; then
     __run_test $1 2>> $testlog | tee -a $testlog
   else
     __run_test $1 >> $testlog 2>&1
@@ -249,9 +252,10 @@ run_test() { # testfile
     # Remove test log if the test was done as it was expected.
     [ $KEEP_LOG -eq 0 ] && rm $testlog
   else
-    catlog $testlog
+    [ $VERBOSE -ge 1 ] && catlog $testlog
     TOTAL_RESULT=1
   fi
+  rm -rf $TMPDIR
 }
 
 # load in the helper functions
diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-glob.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-glob.tc
new file mode 100644
index 000000000000..9dcd0ca1f49c
--- /dev/null
+++ b/tools/testing/selftests/ftrace/test.d/ftrace/func-filter-glob.tc
@@ -0,0 +1,49 @@
+#!/bin/sh
+# description: ftrace - function glob filters
+
+# Make sure that function glob matching filter works.
+
+if ! grep -q function available_tracers; then
+    echo "no function tracer configured"
+    exit_unsupported
+fi
+
+disable_tracing
+clear_trace
+
+# filter by ?, schedule is always good
+if ! echo "sch?dule" > set_ftrace_filter; then
+    # test for powerpc 64
+    if ! echo ".sch?dule" > set_ftrace_filter; then
+	fail "can not enable schedule filter"
+    fi
+    cat set_ftrace_filter | grep '^.schedule$'
+else
+    cat set_ftrace_filter | grep '^schedule$'
+fi
+
+ftrace_filter_check() { # glob grep
+  echo "$1" > set_ftrace_filter
+  cut -f1 -d" " set_ftrace_filter > $TMPDIR/actual
+  cut -f1 -d" " available_filter_functions | grep "$2" > $TMPDIR/expected
+  DIFF=`diff $TMPDIR/actual $TMPDIR/expected`
+  test -z "$DIFF"
+}
+
+# filter by *, front match
+ftrace_filter_check '*schedule' '^.*schedule$'
+
+# filter by *, middle match
+ftrace_filter_check '*schedule*' '^.*schedule.*$'
+
+# filter by *, end match
+ftrace_filter_check 'schedule*' '^schedule.*$'
+
+# filter by *, both side match
+ftrace_filter_check 'sch*ule' '^sch.*ule$'
+
+# filter by char class.
+ftrace_filter_check '[Ss]y[Ss]_*' '^[Ss]y[Ss]_.*$'
+
+echo > set_ftrace_filter
+enable_tracing
diff --git a/tools/testing/selftests/ftrace/test.d/functions b/tools/testing/selftests/ftrace/test.d/functions
index c37262f6c269..91de1a8e4f19 100644
--- a/tools/testing/selftests/ftrace/test.d/functions
+++ b/tools/testing/selftests/ftrace/test.d/functions
@@ -23,3 +23,31 @@ reset_trigger() { # reset all current setting triggers
     done
 }
 
+reset_events_filter() { # reset all current setting filters
+    grep -v ^none events/*/*/filter |
+    while read line; do
+	echo 0 > `echo $line | cut -f1 -d:`
+    done
+}
+
+disable_events() {
+    echo 0 > events/enable
+}
+
+initialize_ftrace() { # Reset ftrace to initial-state
+# As the initial state, ftrace will be set to nop tracer,
+# no events, no triggers, no filters, no function filters,
+# no probes, and tracing on.
+    disable_tracing
+    reset_tracer
+    reset_trigger
+    reset_events_filter
+    disable_events
+    echo > set_event_pid	# event tracer is always on
+    [ -f set_ftrace_filter ] && echo | tee set_ftrace_*
+    [ -f set_graph_function ] && echo | tee set_graph_*
+    [ -f stack_trace_filter ] && echo > stack_trace_filter
+    [ -f kprobe_events ] && echo > kprobe_events
+    [ -f uprobe_events ] && echo > uprobe_events
+    enable_tracing
+}
diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_type.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_type.tc
new file mode 100644
index 000000000000..0a78705b43b2
--- /dev/null
+++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_type.tc
@@ -0,0 +1,37 @@
+#!/bin/sh
+# description: Kprobes event arguments with types
+
+[ -f kprobe_events ] || exit_unsupported # this is configurable
+
+grep "x8/16/32/64" README > /dev/null || exit_unsupported # version issue
+
+echo 0 > events/enable
+echo > kprobe_events
+enable_tracing
+
+echo 'p:testprobe _do_fork $stack0:s32 $stack0:u32 $stack0:x32 $stack0:b8@4/32' > kprobe_events
+grep testprobe kprobe_events
+test -d events/kprobes/testprobe
+
+echo 1 > events/kprobes/testprobe/enable
+( echo "forked")
+echo 0 > events/kprobes/testprobe/enable
+ARGS=`tail -n 1 trace | sed -e 's/.* arg1=\(.*\) arg2=\(.*\) arg3=\(.*\) arg4=\(.*\)/\1 \2 \3 \4/'`
+
+check_types() {
+  X1=`printf "%x" $1 | tail -c 8`
+  X2=`printf "%x" $2`
+  X3=`printf "%x" $3`
+  test $X1 = $X2
+  test $X2 = $X3
+  test 0x$X3 = $3
+
+  B4=`printf "%x" $4`
+  B3=`echo -n $X3 | tail -c 3 | head -c 2`
+  test $B3 = $B4
+}
+check_types $ARGS
+
+echo "-:testprobe" >> kprobe_events
+clear_trace
+test -d events/kprobes/testprobe && exit 1 || exit 0
diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist-mod.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist-mod.tc
index 0bf5085281f3..400e98b64948 100644
--- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist-mod.tc
+++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist-mod.tc
@@ -56,7 +56,7 @@ echo "Test histogram with syscall modifier"
 
 echo 'hist:keys=id.syscall' > events/raw_syscalls/sys_exit/trigger
 for i in `seq 1 10` ; do ( echo "forked" > /dev/null); done
-grep "id: sys_" events/raw_syscalls/sys_exit/hist > /dev/null || \
+grep "id: \(unknown_\|sys_\)" events/raw_syscalls/sys_exit/hist > /dev/null || \
     fail "syscall modifier on raw_syscalls/sys_exit did not work"
 
 
diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-snapshot.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-snapshot.tc
index f84b80d551a2..ed94f0c4e0e4 100644
--- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-snapshot.tc
+++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-snapshot.tc
@@ -23,6 +23,11 @@ if [ ! -f events/sched/sched_process_fork/trigger ]; then
     exit_unsupported
 fi
 
+if [ ! -f snapshot ]; then
+    echo "snapshot is not supported"
+    exit_unsupported
+fi
+
 reset_tracer
 do_reset
 
diff --git a/tools/testing/selftests/gpio/Makefile b/tools/testing/selftests/gpio/Makefile
new file mode 100644
index 000000000000..205e4d10e085
--- /dev/null
+++ b/tools/testing/selftests/gpio/Makefile
@@ -0,0 +1,23 @@
+
+TEST_PROGS := gpio-mockup.sh
+TEST_FILES := gpio-mockup-sysfs.sh $(BINARIES)
+BINARIES := gpio-mockup-chardev
+
+include ../lib.mk
+
+all: $(BINARIES)
+
+clean:
+	$(RM) $(BINARIES)
+
+CFLAGS += -O2 -g -std=gnu99 -Wall -I../../../../usr/include/
+LDLIBS += -lmount -I/usr/include/libmount
+
+$(BINARIES): ../../../gpio/gpio-utils.o ../../../../usr/include/linux/gpio.h
+
+../../../gpio/gpio-utils.o:
+	make ARCH=$(ARCH) CROSS_COMPILE=$(CROSS_COMPILE) -C ../../../gpio
+
+../../../../usr/include/linux/gpio.h:
+	make -C ../../../.. headers_install INSTALL_HDR_PATH=$(shell pwd)/../../../../usr/
+
diff --git a/tools/testing/selftests/gpio/gpio-mockup-chardev.c b/tools/testing/selftests/gpio/gpio-mockup-chardev.c
new file mode 100644
index 000000000000..667e916fa7cc
--- /dev/null
+++ b/tools/testing/selftests/gpio/gpio-mockup-chardev.c
@@ -0,0 +1,324 @@
+/*
+ * GPIO chardev test helper
+ *
+ * Copyright (C) 2016 Bamvor Jian Zhang
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+
+#define _GNU_SOURCE
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <sys/ioctl.h>
+#include <libmount.h>
+#include <err.h>
+#include <dirent.h>
+#include <linux/gpio.h>
+#include "../../../gpio/gpio-utils.h"
+
+#define CONSUMER	"gpio-selftest"
+#define	GC_NUM		10
+enum direction {
+	OUT,
+	IN
+};
+
+static int get_debugfs(char **path)
+{
+	struct libmnt_context *cxt;
+	struct libmnt_table *tb;
+	struct libmnt_iter *itr = NULL;
+	struct libmnt_fs *fs;
+	int found = 0;
+
+	cxt = mnt_new_context();
+	if (!cxt)
+		err(EXIT_FAILURE, "libmount context allocation failed");
+
+	itr = mnt_new_iter(MNT_ITER_FORWARD);
+	if (!itr)
+		err(EXIT_FAILURE, "failed to initialize libmount iterator");
+
+	if (mnt_context_get_mtab(cxt, &tb))
+		err(EXIT_FAILURE, "failed to read mtab");
+
+	while (mnt_table_next_fs(tb, itr, &fs) == 0) {
+		const char *type = mnt_fs_get_fstype(fs);
+
+		if (!strcmp(type, "debugfs")) {
+			found = 1;
+			break;
+		}
+	}
+	if (found)
+		asprintf(path, "%s/gpio", mnt_fs_get_target(fs));
+
+	mnt_free_iter(itr);
+	mnt_free_context(cxt);
+
+	if (!found)
+		return -1;
+
+	return 0;
+}
+
+static int gpio_debugfs_get(const char *consumer, int *dir, int *value)
+{
+	char *debugfs;
+	FILE *f;
+	char *line = NULL;
+	size_t len = 0;
+	char *cur;
+	int found = 0;
+
+	if (get_debugfs(&debugfs) != 0)
+		err(EXIT_FAILURE, "debugfs is not mounted");
+
+	f = fopen(debugfs, "r");
+	if (!f)
+		err(EXIT_FAILURE, "read from gpio debugfs failed");
+
+	/*
+	 * gpio-2   (                    |gpio-selftest               ) in  lo
+	 */
+	while (getline(&line, &len, f) != -1) {
+		cur = strstr(line, consumer);
+		if (cur == NULL)
+			continue;
+
+		cur = strchr(line, ')');
+		if (!cur)
+			continue;
+
+		cur += 2;
+		if (!strncmp(cur, "out", 3)) {
+			*dir = OUT;
+			cur += 4;
+		} else if (!strncmp(cur, "in", 2)) {
+			*dir = IN;
+			cur += 4;
+		}
+
+		if (!strncmp(cur, "hi", 2))
+			*value = 1;
+		else if (!strncmp(cur, "lo", 2))
+			*value = 0;
+
+		found = 1;
+		break;
+	}
+	free(debugfs);
+	fclose(f);
+	free(line);
+
+	if (!found)
+		return -1;
+
+	return 0;
+}
+
+static struct gpiochip_info *list_gpiochip(const char *gpiochip_name, int *ret)
+{
+	struct gpiochip_info *cinfo;
+	struct gpiochip_info *current;
+	const struct dirent *ent;
+	DIR *dp;
+	char *chrdev_name;
+	int fd;
+	int i = 0;
+
+	cinfo = calloc(sizeof(struct gpiochip_info) * 4, GC_NUM + 1);
+	if (!cinfo)
+		err(EXIT_FAILURE, "gpiochip_info allocation failed");
+
+	current = cinfo;
+	dp = opendir("/dev");
+	if (!dp) {
+		*ret = -errno;
+		goto error_out;
+	} else {
+		*ret = 0;
+	}
+
+	while (ent = readdir(dp), ent) {
+		if (check_prefix(ent->d_name, "gpiochip")) {
+			*ret = asprintf(&chrdev_name, "/dev/%s", ent->d_name);
+			if (*ret < 0)
+				goto error_out;
+
+			fd = open(chrdev_name, 0);
+			if (fd == -1) {
+				*ret = -errno;
+				fprintf(stderr, "Failed to open %s\n",
+					chrdev_name);
+				goto error_close_dir;
+			}
+			*ret = ioctl(fd, GPIO_GET_CHIPINFO_IOCTL, current);
+			if (*ret == -1) {
+				perror("Failed to issue CHIPINFO IOCTL\n");
+				goto error_close_dir;
+			}
+			close(fd);
+			if (strcmp(current->label, gpiochip_name) == 0
+			    || check_prefix(current->label, gpiochip_name)) {
+				*ret = 0;
+				current++;
+				i++;
+			}
+		}
+	}
+
+	if ((!*ret && i == 0) || *ret < 0) {
+		free(cinfo);
+		cinfo = NULL;
+	}
+	if (!*ret && i > 0) {
+		cinfo = realloc(cinfo, sizeof(struct gpiochip_info) * 4 * i);
+		*ret = i;
+	}
+
+error_close_dir:
+	closedir(dp);
+error_out:
+	if (*ret < 0)
+		err(EXIT_FAILURE, "list gpiochip failed: %s", strerror(*ret));
+
+	return cinfo;
+}
+
+int gpio_pin_test(struct gpiochip_info *cinfo, int line, int flag, int value)
+{
+	struct gpiohandle_data data;
+	unsigned int lines[] = {line};
+	int fd;
+	int debugfs_dir = IN;
+	int debugfs_value = 0;
+	int ret;
+
+	data.values[0] = value;
+	ret = gpiotools_request_linehandle(cinfo->name, lines, 1, flag, &data,
+					   CONSUMER);
+	if (ret < 0)
+		goto fail_out;
+	else
+		fd = ret;
+
+	ret = gpio_debugfs_get(CONSUMER, &debugfs_dir, &debugfs_value);
+	if (ret) {
+		ret = -EINVAL;
+		goto fail_out;
+	}
+	if (flag & GPIOHANDLE_REQUEST_INPUT) {
+		if (debugfs_dir != IN) {
+			errno = -EINVAL;
+			ret = -errno;
+		}
+	} else if (flag & GPIOHANDLE_REQUEST_OUTPUT) {
+		if (flag & GPIOHANDLE_REQUEST_ACTIVE_LOW)
+			debugfs_value = !debugfs_value;
+
+		if (!(debugfs_dir == OUT && value == debugfs_value))
+			errno = -EINVAL;
+		ret = -errno;
+
+	}
+	gpiotools_release_linehandle(fd);
+
+fail_out:
+	if (ret)
+		err(EXIT_FAILURE, "gpio<%s> line<%d> test flag<0x%x> value<%d>",
+		    cinfo->name, line, flag, value);
+
+	return ret;
+}
+
+void gpio_pin_tests(struct gpiochip_info *cinfo, unsigned int line)
+{
+	printf("line<%d>", line);
+	gpio_pin_test(cinfo, line, GPIOHANDLE_REQUEST_OUTPUT, 0);
+	printf(".");
+	gpio_pin_test(cinfo, line, GPIOHANDLE_REQUEST_OUTPUT, 1);
+	printf(".");
+	gpio_pin_test(cinfo, line,
+		      GPIOHANDLE_REQUEST_OUTPUT | GPIOHANDLE_REQUEST_ACTIVE_LOW,
+		      0);
+	printf(".");
+	gpio_pin_test(cinfo, line,
+		      GPIOHANDLE_REQUEST_OUTPUT | GPIOHANDLE_REQUEST_ACTIVE_LOW,
+		      1);
+	printf(".");
+	gpio_pin_test(cinfo, line, GPIOHANDLE_REQUEST_INPUT, 0);
+	printf(".");
+}
+
+/*
+ * ./gpio-mockup-chardev gpio_chip_name_prefix is_valid_gpio_chip
+ * Return 0 if successful or exit with EXIT_FAILURE if test failed.
+ * gpio_chip_name_prefix: The prefix of gpiochip you want to test. E.g.
+ *			  gpio-mockup
+ * is_valid_gpio_chip:	  Whether the gpio_chip is valid. 1 means valid,
+ *			  0 means invalid which could not be found by
+ *			  list_gpiochip.
+ */
+int main(int argc, char *argv[])
+{
+	char *prefix;
+	int valid;
+	struct gpiochip_info *cinfo;
+	struct gpiochip_info *current;
+	int i;
+	int ret;
+
+	if (argc < 3) {
+		printf("Usage: %s prefix is_valid", argv[0]);
+		exit(EXIT_FAILURE);
+	}
+
+	prefix = argv[1];
+	valid = strcmp(argv[2], "true") == 0 ? 1 : 0;
+
+	printf("Test gpiochip %s: ", prefix);
+	cinfo = list_gpiochip(prefix, &ret);
+	if (!cinfo) {
+		if (!valid && ret == 0) {
+			printf("Invalid test successful\n");
+			ret = 0;
+			goto out;
+		} else {
+			ret = -EINVAL;
+			goto out;
+		}
+	} else if (cinfo && !valid) {
+		ret = -EINVAL;
+		goto out;
+	}
+	current = cinfo;
+	for (i = 0; i < ret; i++) {
+		gpio_pin_tests(current, 0);
+		gpio_pin_tests(current, current->lines - 1);
+		gpio_pin_tests(current, random() % current->lines);
+		current++;
+	}
+	ret = 0;
+	printf("successful\n");
+
+out:
+	if (ret)
+		fprintf(stderr, "gpio<%s> test failed\n", prefix);
+
+	if (cinfo)
+		free(cinfo);
+
+	if (ret)
+		exit(EXIT_FAILURE);
+
+	return ret;
+}
diff --git a/tools/testing/selftests/gpio/gpio-mockup-sysfs.sh b/tools/testing/selftests/gpio/gpio-mockup-sysfs.sh
new file mode 100755
index 000000000000..085d7a39899c
--- /dev/null
+++ b/tools/testing/selftests/gpio/gpio-mockup-sysfs.sh
@@ -0,0 +1,134 @@
+
+is_consistent()
+{
+	val=
+
+	active_low_sysfs=`cat $GPIO_SYSFS/gpio$nr/active_low`
+	val_sysfs=`cat $GPIO_SYSFS/gpio$nr/value`
+	dir_sysfs=`cat $GPIO_SYSFS/gpio$nr/direction`
+
+	gpio_this_debugfs=`cat $GPIO_DEBUGFS |grep "gpio-$nr" | sed "s/(.*)//g"`
+	dir_debugfs=`echo $gpio_this_debugfs | awk '{print $2}'`
+	val_debugfs=`echo $gpio_this_debugfs | awk '{print $3}'`
+	if [ $val_debugfs = "lo" ]; then
+		val=0
+	elif [ $val_debugfs = "hi" ]; then
+		val=1
+	fi
+
+	if [ $active_low_sysfs = "1" ]; then
+		if [ $val = "0" ]; then
+			val="1"
+		else
+			val="0"
+		fi
+	fi
+
+	if [ $val_sysfs = $val ] && [ $dir_sysfs = $dir_debugfs ]; then
+		echo -n "."
+	else
+		echo "test fail, exit"
+		die
+	fi
+}
+
+test_pin_logic()
+{
+	nr=$1
+	direction=$2
+	active_low=$3
+	value=$4
+
+	echo $direction > $GPIO_SYSFS/gpio$nr/direction
+	echo $active_low > $GPIO_SYSFS/gpio$nr/active_low
+	if [ $direction = "out" ]; then
+		echo $value > $GPIO_SYSFS/gpio$nr/value
+	fi
+	is_consistent $nr
+}
+
+test_one_pin()
+{
+	nr=$1
+
+	echo -n "test pin<$nr>"
+
+	echo $nr > $GPIO_SYSFS/export 2>/dev/null
+
+	if [ X$? != X0 ]; then
+		echo "test GPIO pin $nr failed"
+		die
+	fi
+
+	#"Checking if the sysfs is consistent with debugfs: "
+	is_consistent $nr
+
+	#"Checking the logic of active_low: "
+	test_pin_logic $nr out 1 1
+	test_pin_logic $nr out 1 0
+	test_pin_logic $nr out 0 1
+	test_pin_logic $nr out 0 0
+
+	#"Checking the logic of direction: "
+	test_pin_logic $nr in 1 1
+	test_pin_logic $nr out 1 0
+	test_pin_logic $nr low 0 1
+	test_pin_logic $nr high 0 0
+
+	echo $nr > $GPIO_SYSFS/unexport
+
+	echo "successful"
+}
+
+test_one_pin_fail()
+{
+	nr=$1
+
+	echo $nr > $GPIO_SYSFS/export 2>/dev/null
+
+	if [ X$? != X0 ]; then
+		echo "test invalid pin $nr successful"
+	else
+		echo "test invalid pin $nr failed"
+		echo $nr > $GPIO_SYSFS/unexport 2>/dev/null
+		die
+	fi
+}
+
+list_chip()
+{
+	echo `ls -d $GPIO_DRV_SYSFS/gpiochip* 2>/dev/null`
+}
+
+test_chip()
+{
+	chip=$1
+	name=`basename $chip`
+	base=`cat $chip/base`
+	ngpio=`cat $chip/ngpio`
+	printf "%-10s %-5s %-5s\n" $name $base $ngpio
+	if [ $ngpio = "0" ]; then
+		echo "number of gpio is zero is not allowed".
+	fi
+	test_one_pin $base
+	test_one_pin $(($base + $ngpio - 1))
+	test_one_pin $((( RANDOM % $ngpio )  + $base ))
+}
+
+test_chips_sysfs()
+{
+       gpiochip=`list_chip $module`
+       if [ X"$gpiochip" = X ]; then
+               if [ X"$valid" = Xfalse ]; then
+                       echo "successful"
+               else
+                       echo "fail"
+                       die
+               fi
+       else
+               for chip in $gpiochip; do
+                       test_chip $chip
+               done
+       fi
+}
+
diff --git a/tools/testing/selftests/gpio/gpio-mockup.sh b/tools/testing/selftests/gpio/gpio-mockup.sh
new file mode 100755
index 000000000000..b183439e058e
--- /dev/null
+++ b/tools/testing/selftests/gpio/gpio-mockup.sh
@@ -0,0 +1,201 @@
+#!/bin/bash
+
+#exit status
+#1: run as non-root user
+#2: sysfs/debugfs not mount
+#3: insert module fail when gpio-mockup is a module.
+#4: other reason.
+
+SYSFS=
+GPIO_SYSFS=
+GPIO_DRV_SYSFS=
+DEBUGFS=
+GPIO_DEBUGFS=
+dev_type=
+module=
+
+usage()
+{
+	echo "Usage:"
+	echo "$0 [-f] [-m name] [-t type]"
+	echo "-f:  full test. It maybe conflict with existence gpio device."
+	echo "-m:  module name, default name is gpio-mockup. It could also test"
+	echo "     other gpio device."
+	echo "-t:  interface type: chardev(char device) and sysfs(being"
+	echo "     deprecated). The first one is default"
+	echo ""
+	echo "$0 -h"
+	echo "This usage"
+}
+
+prerequisite()
+{
+	msg="skip all tests:"
+	if [ $UID != 0 ]; then
+		echo $msg must be run as root >&2
+		exit 1
+	fi
+	SYSFS=`mount -t sysfs | head -1 | awk '{ print $3 }'`
+	if [ ! -d "$SYSFS" ]; then
+		echo $msg sysfs is not mounted >&2
+		exit 2
+	fi
+	GPIO_SYSFS=`echo $SYSFS/class/gpio`
+	GPIO_DRV_SYSFS=`echo $SYSFS/devices/platform/$module/gpio`
+	DEBUGFS=`mount -t debugfs | head -1 | awk '{ print $3 }'`
+	if [ ! -d "$DEBUGFS" ]; then
+		echo $msg debugfs is not mounted >&2
+		exit 2
+	fi
+	GPIO_DEBUGFS=`echo $DEBUGFS/gpio`
+	source gpio-mockup-sysfs.sh
+}
+
+try_insert_module()
+{
+	if [ -d "$GPIO_DRV_SYSFS" ]; then
+		echo "$GPIO_DRV_SYSFS exist. Skip insert module"
+	else
+		modprobe -q $module $1
+		if [ X$? != X0 ]; then
+			echo $msg insmod $module failed >&2
+			exit 3
+		fi
+	fi
+}
+
+remove_module()
+{
+	modprobe -r -q $module
+}
+
+die()
+{
+	remove_module
+	exit 4
+}
+
+test_chips()
+{
+	if [ X$dev_type = Xsysfs ]; then
+		echo "WARNING: sysfs ABI of gpio is going to deprecated."
+		test_chips_sysfs $*
+	else
+		$BASE/gpio-mockup-chardev $*
+	fi
+}
+
+gpio_test()
+{
+	param=$1
+	valid=$2
+
+	if [ X"$param" = X ]; then
+		die
+	fi
+	try_insert_module "gpio_mockup_ranges=$param"
+	echo -n "GPIO $module test with ranges: <"
+	echo "$param>: "
+	printf "%-10s %s\n" $param
+	test_chips $module $valid
+	remove_module
+}
+
+BASE=`dirname $0`
+
+dev_type=
+TEMP=`getopt -o fhm:t: -n '$0' -- "$@"`
+
+if [ "$?" != "0" ]; then
+        echo "Parameter process failed, Terminating..." >&2
+        exit 1
+fi
+
+# Note the quotes around `$TEMP': they are essential!
+eval set -- "$TEMP"
+
+while true; do
+	case $1 in
+	-f)
+		full_test=true
+		shift
+		;;
+	-h)
+		usage
+		exit
+		;;
+	-m)
+		module=$2
+		shift 2
+		;;
+	-t)
+		dev_type=$2
+		shift 2
+		;;
+	--)
+		shift
+		break
+		;;
+	*)
+		echo "Internal error!"
+		exit 1
+		;;
+	esac
+done
+
+if [ X"$module" = X ]; then
+	module="gpio-mockup"
+fi
+
+if [ X$dev_type != Xsysfs ]; then
+	dev_type="chardev"
+fi
+
+prerequisite
+
+echo "1.  Test dynamic allocation of gpio successful means insert gpiochip and"
+echo "    manipulate gpio pin successful"
+gpio_test "-1,32" true
+gpio_test "-1,32,-1,32" true
+gpio_test "-1,32,-1,32,-1,32" true
+if [ X$full_test = Xtrue ]; then
+	gpio_test "-1,32,32,64" true
+	gpio_test "-1,32,40,64,-1,5" true
+	gpio_test "-1,32,32,64,-1,32" true
+	gpio_test "0,32,32,64,-1,32,-1,32" true
+	gpio_test "-1,32,-1,32,0,32,32,64" true
+	echo "2.  Do basic test: successful means insert gpiochip and"
+	echo "    manipulate gpio pin successful"
+	gpio_test "0,32" true
+	gpio_test "0,32,32,64" true
+	gpio_test "0,32,40,64,64,96" true
+fi
+echo "3.  Error test: successful means insert gpiochip failed"
+echo "3.1 Test number of gpio overflow"
+#Currently: The max number of gpio(1024) is defined in arm architecture.
+gpio_test "-1,32,-1,1024" false
+if [ X$full_test = Xtrue ]; then
+	echo "3.2 Test zero line of gpio"
+	gpio_test "0,0" false
+	echo "3.3 Test range overlap"
+	echo "3.3.1 Test corner case"
+	gpio_test "0,32,0,1" false
+	gpio_test "0,32,32,64,32,40" false
+	gpio_test "0,32,35,64,35,45" false
+	gpio_test "0,32,31,32" false
+	gpio_test "0,32,32,64,36,37" false
+	gpio_test "0,32,35,64,34,36" false
+	echo "3.3.2 Test inserting invalid second gpiochip"
+	gpio_test "0,32,30,35" false
+	gpio_test "0,32,1,5" false
+	gpio_test "10,32,9,14" false
+	gpio_test "10,32,30,35" false
+	echo "3.3.3 Test others"
+	gpio_test "0,32,40,56,39,45" false
+	gpio_test "0,32,40,56,30,33" false
+	gpio_test "0,32,40,56,30,41" false
+	gpio_test "0,32,40,56,20,21" false
+fi
+
+echo GPIO test PASS
+
diff --git a/tools/testing/selftests/nsfs/.gitignore b/tools/testing/selftests/nsfs/.gitignore
new file mode 100644
index 000000000000..2ab2c824ce86
--- /dev/null
+++ b/tools/testing/selftests/nsfs/.gitignore
@@ -0,0 +1,2 @@
+owner
+pidns
diff --git a/tools/testing/selftests/powerpc/Makefile b/tools/testing/selftests/powerpc/Makefile
index db54a33f850f..c2c4211ba58b 100644
--- a/tools/testing/selftests/powerpc/Makefile
+++ b/tools/testing/selftests/powerpc/Makefile
@@ -8,7 +8,7 @@ ifeq ($(ARCH),powerpc)
 
 GIT_VERSION = $(shell git describe --always --long --dirty || echo "unknown")
 
-CFLAGS := -std=gnu99 -Wall -O2 -Wall -Werror -DGIT_VERSION='"$(GIT_VERSION)"' -I$(CURDIR) $(CFLAGS)
+CFLAGS := -std=gnu99 -Wall -O2 -Wall -Werror -DGIT_VERSION='"$(GIT_VERSION)"' -I$(CURDIR)/include $(CFLAGS)
 
 export CFLAGS
 
@@ -26,7 +26,8 @@ SUB_DIRS = alignment		\
 	   syscalls		\
 	   tm			\
 	   vphn         \
-	   math
+	   math		\
+	   ptrace
 
 endif
 
diff --git a/tools/testing/selftests/powerpc/benchmarks/.gitignore b/tools/testing/selftests/powerpc/benchmarks/.gitignore
index bce49ebd869e..04dc1e6ef2ce 100644
--- a/tools/testing/selftests/powerpc/benchmarks/.gitignore
+++ b/tools/testing/selftests/powerpc/benchmarks/.gitignore
@@ -1,4 +1,5 @@
 gettimeofday
 context_switch
 mmap_bench
-futex_bench
\ No newline at end of file
+futex_bench
+null_syscall
diff --git a/tools/testing/selftests/powerpc/benchmarks/Makefile b/tools/testing/selftests/powerpc/benchmarks/Makefile
index a9adfb7de78f..545077f98f72 100644
--- a/tools/testing/selftests/powerpc/benchmarks/Makefile
+++ b/tools/testing/selftests/powerpc/benchmarks/Makefile
@@ -1,4 +1,4 @@
-TEST_PROGS := gettimeofday context_switch mmap_bench futex_bench
+TEST_PROGS := gettimeofday context_switch mmap_bench futex_bench null_syscall
 
 CFLAGS += -O2
 
diff --git a/tools/testing/selftests/powerpc/benchmarks/context_switch.c b/tools/testing/selftests/powerpc/benchmarks/context_switch.c
index a36883ad48a4..778f5fbfd784 100644
--- a/tools/testing/selftests/powerpc/benchmarks/context_switch.c
+++ b/tools/testing/selftests/powerpc/benchmarks/context_switch.c
@@ -28,7 +28,7 @@
 #ifdef __powerpc__
 #include <altivec.h>
 #endif
-#include "../utils.h"
+#include "utils.h"
 
 static unsigned int timeout = 30;
 
diff --git a/tools/testing/selftests/powerpc/benchmarks/null_syscall.c b/tools/testing/selftests/powerpc/benchmarks/null_syscall.c
new file mode 100644
index 000000000000..ecc14d68e101
--- /dev/null
+++ b/tools/testing/selftests/powerpc/benchmarks/null_syscall.c
@@ -0,0 +1,157 @@
+/*
+ * Test null syscall performance
+ *
+ * Copyright (C) 2009-2015 Anton Blanchard, IBM
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#define NR_LOOPS 10000000
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <signal.h>
+
+static volatile int soak_done;
+unsigned long long clock_frequency;
+unsigned long long timebase_frequency;
+double timebase_multiplier;
+
+static inline unsigned long long mftb(void)
+{
+	unsigned long low;
+
+	asm volatile("mftb %0" : "=r" (low));
+
+	return low;
+}
+
+static void sigalrm_handler(int unused)
+{
+	soak_done = 1;
+}
+
+/*
+ * Use a timer instead of busy looping on clock_gettime() so we don't
+ * pollute profiles with glibc and VDSO hits.
+ */
+static void cpu_soak_usecs(unsigned long usecs)
+{
+	struct itimerval val;
+
+	memset(&val, 0, sizeof(val));
+	val.it_value.tv_usec = usecs;
+
+	signal(SIGALRM, sigalrm_handler);
+	setitimer(ITIMER_REAL, &val, NULL);
+
+	while (1) {
+		if (soak_done)
+			break;
+	}
+
+	signal(SIGALRM, SIG_DFL);
+}
+
+/*
+ * This only works with recent kernels where cpufreq modifies
+ * /proc/cpuinfo dynamically.
+ */
+static void get_proc_frequency(void)
+{
+	FILE *f;
+	char line[128];
+	char *p, *end;
+	unsigned long v;
+	double d;
+	char *override;
+
+	/* Try to get out of low power/low frequency mode */
+	cpu_soak_usecs(0.25 * 1000000);
+
+	f = fopen("/proc/cpuinfo", "r");
+	if (f == NULL)
+		return;
+
+	timebase_frequency = 0;
+
+	while (fgets(line, sizeof(line), f) != NULL) {
+		if (strncmp(line, "timebase", 8) == 0) {
+			p = strchr(line, ':');
+			if (p != NULL) {
+				v = strtoull(p + 1, &end, 0);
+				if (end != p + 1)
+					timebase_frequency = v;
+			}
+		}
+
+		if (((strncmp(line, "clock", 5) == 0) ||
+		     (strncmp(line, "cpu MHz", 7) == 0))) {
+			p = strchr(line, ':');
+			if (p != NULL) {
+				d = strtod(p + 1, &end);
+				if (end != p + 1) {
+					/* Find fastest clock frequency */
+					if ((d * 1000000ULL) > clock_frequency)
+						clock_frequency = d * 1000000ULL;
+				}
+			}
+		}
+	}
+
+	fclose(f);
+
+	override = getenv("FREQUENCY");
+	if (override)
+		clock_frequency = strtoull(override, NULL, 10);
+
+	if (timebase_frequency)
+		timebase_multiplier = (double)clock_frequency
+					/ timebase_frequency;
+	else
+		timebase_multiplier = 1;
+}
+
+static void do_null_syscall(unsigned long nr)
+{
+	unsigned long i;
+
+	for (i = 0; i < nr; i++)
+		getppid();
+}
+
+#define TIME(A, STR) \
+
+int main(void)
+{
+	unsigned long tb_start, tb_now;
+	struct timespec tv_start, tv_now;
+	unsigned long long elapsed_ns, elapsed_tb;
+
+	get_proc_frequency();
+
+	clock_gettime(CLOCK_MONOTONIC, &tv_start);
+	tb_start = mftb();
+
+	do_null_syscall(NR_LOOPS);
+
+	clock_gettime(CLOCK_MONOTONIC, &tv_now);
+	tb_now = mftb();
+
+	elapsed_ns = (tv_now.tv_sec - tv_start.tv_sec) * 1000000000ULL +
+			(tv_now.tv_nsec - tv_start.tv_nsec);
+	elapsed_tb = tb_now - tb_start;
+
+	printf("%10.2f ns %10.2f cycles\n", (float)elapsed_ns / NR_LOOPS,
+			(float)elapsed_tb * timebase_multiplier / NR_LOOPS);
+
+	return 0;
+}
diff --git a/tools/testing/selftests/powerpc/copyloops/asm/ppc_asm.h b/tools/testing/selftests/powerpc/copyloops/asm/ppc_asm.h
index 50ae7d2091ce..80d34a9ffff4 100644
--- a/tools/testing/selftests/powerpc/copyloops/asm/ppc_asm.h
+++ b/tools/testing/selftests/powerpc/copyloops/asm/ppc_asm.h
@@ -25,6 +25,8 @@
 
 #define PPC_MTOCRF(A, B)	mtocrf A, B
 
+#define EX_TABLE(x, y)
+
 FUNC_START(enter_vmx_usercopy)
 	li	r3,1
 	blr
diff --git a/tools/testing/selftests/powerpc/copyloops/validate.c b/tools/testing/selftests/powerpc/copyloops/validate.c
index 1750ff57ee58..7fb436f82d16 100644
--- a/tools/testing/selftests/powerpc/copyloops/validate.c
+++ b/tools/testing/selftests/powerpc/copyloops/validate.c
@@ -3,7 +3,7 @@
 #include <stdlib.h>
 #include <stdbool.h>
 
-#include "../utils.h"
+#include "utils.h"
 
 #define MAX_LEN 8192
 #define MAX_OFFSET 16
diff --git a/tools/testing/selftests/powerpc/dscr/dscr.h b/tools/testing/selftests/powerpc/dscr/dscr.h
index a36af1b2c2bb..18ea223bd398 100644
--- a/tools/testing/selftests/powerpc/dscr/dscr.h
+++ b/tools/testing/selftests/powerpc/dscr/dscr.h
@@ -28,8 +28,6 @@
 
 #include "utils.h"
 
-#define SPRN_DSCR	0x11	/* Privilege state SPR */
-#define SPRN_DSCR_USR	0x03	/* Problem state SPR */
 #define THREADS		100	/* Max threads */
 #define COUNT		100	/* Max iterations */
 #define DSCR_MAX	16	/* Max DSCR value */
@@ -48,14 +46,14 @@ inline unsigned long get_dscr(void)
 {
 	unsigned long ret;
 
-	asm volatile("mfspr %0,%1" : "=r" (ret): "i" (SPRN_DSCR));
+	asm volatile("mfspr %0,%1" : "=r" (ret) : "i" (SPRN_DSCR_PRIV));
 
 	return ret;
 }
 
 inline void set_dscr(unsigned long val)
 {
-	asm volatile("mtspr %1,%0" : : "r" (val), "i" (SPRN_DSCR));
+	asm volatile("mtspr %1,%0" : : "r" (val), "i" (SPRN_DSCR_PRIV));
 }
 
 /* Problem state DSCR access */
@@ -63,14 +61,14 @@ inline unsigned long get_dscr_usr(void)
 {
 	unsigned long ret;
 
-	asm volatile("mfspr %0,%1" : "=r" (ret): "i" (SPRN_DSCR_USR));
+	asm volatile("mfspr %0,%1" : "=r" (ret) : "i" (SPRN_DSCR));
 
 	return ret;
 }
 
 inline void set_dscr_usr(unsigned long val)
 {
-	asm volatile("mtspr %1,%0" : : "r" (val), "i" (SPRN_DSCR_USR));
+	asm volatile("mtspr %1,%0" : : "r" (val), "i" (SPRN_DSCR));
 }
 
 /* Default DSCR access */
diff --git a/tools/testing/selftests/powerpc/basic_asm.h b/tools/testing/selftests/powerpc/include/basic_asm.h
index 3349a0704d1a..12eaddf72e66 100644
--- a/tools/testing/selftests/powerpc/basic_asm.h
+++ b/tools/testing/selftests/powerpc/include/basic_asm.h
@@ -4,12 +4,12 @@
 #include <ppc-asm.h>
 #include <asm/unistd.h>
 
-#define LOAD_REG_IMMEDIATE(reg,expr) \
-	lis	reg,(expr)@highest;	\
-	ori	reg,reg,(expr)@higher;	\
-	rldicr	reg,reg,32,31;	\
-	oris	reg,reg,(expr)@high;	\
-	ori	reg,reg,(expr)@l;
+#define LOAD_REG_IMMEDIATE(reg, expr) \
+	lis	reg, (expr)@highest;	\
+	ori	reg, reg, (expr)@higher;	\
+	rldicr	reg, reg, 32, 31;	\
+	oris	reg, reg, (expr)@high;	\
+	ori	reg, reg, (expr)@l;
 
 /*
  * Note: These macros assume that variables being stored on the stack are
@@ -20,7 +20,8 @@
 #define STACK_FRAME_MIN_SIZE 32
 #define STACK_FRAME_TOC_POS  24
 #define __STACK_FRAME_PARAM(_param)  (32 + ((_param)*8))
-#define __STACK_FRAME_LOCAL(_num_params,_var_num)  ((STACK_FRAME_PARAM(_num_params)) + ((_var_num)*8))
+#define __STACK_FRAME_LOCAL(_num_params, _var_num)  \
+	((STACK_FRAME_PARAM(_num_params)) + ((_var_num)*8))
 #else
 #define STACK_FRAME_MIN_SIZE 112
 #define STACK_FRAME_TOC_POS  40
@@ -30,14 +31,16 @@
  * Caveat: if a function passed more than 8 doublewords, the caller will have
  * made more space... which would render the 112 incorrect.
  */
-#define __STACK_FRAME_LOCAL(_num_params,_var_num)  (112 + ((_var_num)*8))
+#define __STACK_FRAME_LOCAL(_num_params, _var_num)  \
+	(112 + ((_var_num)*8))
 #endif
 
 /* Parameter x saved to the stack */
 #define STACK_FRAME_PARAM(var)    __STACK_FRAME_PARAM(var)
 
 /* Local variable x saved to the stack after x parameters */
-#define STACK_FRAME_LOCAL(num_params,var)    __STACK_FRAME_LOCAL(num_params,var)
+#define STACK_FRAME_LOCAL(num_params, var)    \
+	__STACK_FRAME_LOCAL(num_params, var)
 #define STACK_FRAME_LR_POS   16
 #define STACK_FRAME_CR_POS   8
 
@@ -53,18 +56,18 @@
  */
 #define PUSH_BASIC_STACK(_extra) \
 	mflr	r0; \
-	std	r0,STACK_FRAME_LR_POS(%r1); \
-	stdu	%r1,-(_extra + STACK_FRAME_MIN_SIZE)(%r1); \
+	std	r0, STACK_FRAME_LR_POS(%r1); \
+	stdu	%r1, -(_extra + STACK_FRAME_MIN_SIZE)(%r1); \
 	mfcr	r0; \
-	stw	r0,STACK_FRAME_CR_POS(%r1); \
-	std	%r2,STACK_FRAME_TOC_POS(%r1);
+	stw	r0, STACK_FRAME_CR_POS(%r1); \
+	std	%r2, STACK_FRAME_TOC_POS(%r1);
 
 #define POP_BASIC_STACK(_extra) \
-	ld	%r2,STACK_FRAME_TOC_POS(%r1); \
-	lwz	r0,STACK_FRAME_CR_POS(%r1); \
+	ld	%r2, STACK_FRAME_TOC_POS(%r1); \
+	lwz	r0, STACK_FRAME_CR_POS(%r1); \
 	mtcr	r0; \
-	addi	%r1,%r1,(_extra + STACK_FRAME_MIN_SIZE); \
-	ld	r0,STACK_FRAME_LR_POS(%r1); \
+	addi	%r1, %r1, (_extra + STACK_FRAME_MIN_SIZE); \
+	ld	r0, STACK_FRAME_LR_POS(%r1); \
 	mtlr	r0;
 
 #endif /* _SELFTESTS_POWERPC_BASIC_ASM_H */
diff --git a/tools/testing/selftests/powerpc/fpu_asm.h b/tools/testing/selftests/powerpc/include/fpu_asm.h
index 6a387d255e27..6a387d255e27 100644
--- a/tools/testing/selftests/powerpc/fpu_asm.h
+++ b/tools/testing/selftests/powerpc/include/fpu_asm.h
diff --git a/tools/testing/selftests/powerpc/gpr_asm.h b/tools/testing/selftests/powerpc/include/gpr_asm.h
index f6f38852d3a0..f6f38852d3a0 100644
--- a/tools/testing/selftests/powerpc/gpr_asm.h
+++ b/tools/testing/selftests/powerpc/include/gpr_asm.h
diff --git a/tools/testing/selftests/powerpc/instructions.h b/tools/testing/selftests/powerpc/include/instructions.h
index 0fb0bd3b28c9..0fb0bd3b28c9 100644
--- a/tools/testing/selftests/powerpc/instructions.h
+++ b/tools/testing/selftests/powerpc/include/instructions.h
diff --git a/tools/testing/selftests/powerpc/include/reg.h b/tools/testing/selftests/powerpc/include/reg.h
new file mode 100644
index 000000000000..4afdebcce4cd
--- /dev/null
+++ b/tools/testing/selftests/powerpc/include/reg.h
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2014, Michael Ellerman, IBM Corp.
+ * Licensed under GPLv2.
+ */
+
+#ifndef _SELFTESTS_POWERPC_REG_H
+#define _SELFTESTS_POWERPC_REG_H
+
+#define __stringify_1(x)        #x
+#define __stringify(x)          __stringify_1(x)
+
+#define mfspr(rn)	({unsigned long rval; \
+			 asm volatile("mfspr %0," _str(rn) \
+				    : "=r" (rval)); rval; })
+#define mtspr(rn, v)	asm volatile("mtspr " _str(rn) ",%0" : \
+				    : "r" ((unsigned long)(v)) \
+				    : "memory")
+
+#define mb()		asm volatile("sync" : : : "memory");
+
+#define SPRN_MMCR2     769
+#define SPRN_MMCRA     770
+#define SPRN_MMCR0     779
+#define   MMCR0_PMAO   0x00000080
+#define   MMCR0_PMAE   0x04000000
+#define   MMCR0_FC     0x80000000
+#define SPRN_EBBHR     804
+#define SPRN_EBBRR     805
+#define SPRN_BESCR     806     /* Branch event status & control register */
+#define SPRN_BESCRS    800     /* Branch event status & control set (1 bits set to 1) */
+#define SPRN_BESCRSU   801     /* Branch event status & control set upper */
+#define SPRN_BESCRR    802     /* Branch event status & control REset (1 bits set to 0) */
+#define SPRN_BESCRRU   803     /* Branch event status & control REset upper */
+
+#define BESCR_PMEO     0x1     /* PMU Event-based exception Occurred */
+#define BESCR_PME      (0x1ul << 32) /* PMU Event-based exception Enable */
+
+#define SPRN_PMC1      771
+#define SPRN_PMC2      772
+#define SPRN_PMC3      773
+#define SPRN_PMC4      774
+#define SPRN_PMC5      775
+#define SPRN_PMC6      776
+
+#define SPRN_SIAR      780
+#define SPRN_SDAR      781
+#define SPRN_SIER      768
+
+#define SPRN_TEXASR     0x82    /* Transaction Exception and Status Register */
+#define SPRN_TFIAR      0x81    /* Transaction Failure Inst Addr    */
+#define SPRN_TFHAR      0x80    /* Transaction Failure Handler Addr */
+#define SPRN_TAR        0x32f	/* Target Address Register */
+
+#define SPRN_DSCR_PRIV 0x11	/* Privilege State DSCR */
+#define SPRN_DSCR      0x03	/* Data Stream Control Register */
+#define SPRN_PPR       896	/* Program Priority Register */
+
+/* TEXASR register bits */
+#define TEXASR_FC	0xFE00000000000000
+#define TEXASR_FP	0x0100000000000000
+#define TEXASR_DA	0x0080000000000000
+#define TEXASR_NO	0x0040000000000000
+#define TEXASR_FO	0x0020000000000000
+#define TEXASR_SIC	0x0010000000000000
+#define TEXASR_NTC	0x0008000000000000
+#define TEXASR_TC	0x0004000000000000
+#define TEXASR_TIC	0x0002000000000000
+#define TEXASR_IC	0x0001000000000000
+#define TEXASR_IFC	0x0000800000000000
+#define TEXASR_ABT	0x0000000100000000
+#define TEXASR_SPD	0x0000000080000000
+#define TEXASR_HV	0x0000000020000000
+#define TEXASR_PR	0x0000000010000000
+#define TEXASR_FS	0x0000000008000000
+#define TEXASR_TE	0x0000000004000000
+#define TEXASR_ROT	0x0000000002000000
+
+/* Vector Instructions */
+#define VSX_XX1(xs, ra, rb)	(((xs) & 0x1f) << 21 | ((ra) << 16) |  \
+				 ((rb) << 11) | (((xs) >> 5)))
+#define STXVD2X(xs, ra, rb)	.long (0x7c000798 | VSX_XX1((xs), (ra), (rb)))
+#define LXVD2X(xs, ra, rb)	.long (0x7c000698 | VSX_XX1((xs), (ra), (rb)))
+
+#define ASM_LOAD_GPR_IMMED(_asm_symbol_name_immed) \
+		"li 14, %[" #_asm_symbol_name_immed "];" \
+		"li 15, %[" #_asm_symbol_name_immed "];" \
+		"li 16, %[" #_asm_symbol_name_immed "];" \
+		"li 17, %[" #_asm_symbol_name_immed "];" \
+		"li 18, %[" #_asm_symbol_name_immed "];" \
+		"li 19, %[" #_asm_symbol_name_immed "];" \
+		"li 20, %[" #_asm_symbol_name_immed "];" \
+		"li 21, %[" #_asm_symbol_name_immed "];" \
+		"li 22, %[" #_asm_symbol_name_immed "];" \
+		"li 23, %[" #_asm_symbol_name_immed "];" \
+		"li 24, %[" #_asm_symbol_name_immed "];" \
+		"li 25, %[" #_asm_symbol_name_immed "];" \
+		"li 26, %[" #_asm_symbol_name_immed "];" \
+		"li 27, %[" #_asm_symbol_name_immed "];" \
+		"li 28, %[" #_asm_symbol_name_immed "];" \
+		"li 29, %[" #_asm_symbol_name_immed "];" \
+		"li 30, %[" #_asm_symbol_name_immed "];" \
+		"li 31, %[" #_asm_symbol_name_immed "];"
+
+#define ASM_LOAD_FPR_SINGLE_PRECISION(_asm_symbol_name_addr) \
+		"lfs 0, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 1, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 2, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 3, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 4, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 5, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 6, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 7, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 8, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 9, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 10, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 11, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 12, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 13, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 14, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 15, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 16, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 17, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 18, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 19, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 20, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 21, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 22, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 23, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 24, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 25, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 26, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 27, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 28, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 29, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 30, 0(%[" #_asm_symbol_name_addr "]);" \
+		"lfs 31, 0(%[" #_asm_symbol_name_addr "]);"
+
+#ifndef __ASSEMBLER__
+void store_gpr(unsigned long *addr);
+void load_gpr(unsigned long *addr);
+void load_fpr_single_precision(float *addr);
+void store_fpr_single_precision(float *addr);
+#endif /* end of __ASSEMBLER__ */
+
+#endif /* _SELFTESTS_POWERPC_REG_H */
diff --git a/tools/testing/selftests/powerpc/subunit.h b/tools/testing/selftests/powerpc/include/subunit.h
index 9c6c4e901ab6..9c6c4e901ab6 100644
--- a/tools/testing/selftests/powerpc/subunit.h
+++ b/tools/testing/selftests/powerpc/include/subunit.h
diff --git a/tools/testing/selftests/powerpc/utils.h b/tools/testing/selftests/powerpc/include/utils.h
index 53405e8a52ab..53405e8a52ab 100644
--- a/tools/testing/selftests/powerpc/utils.h
+++ b/tools/testing/selftests/powerpc/include/utils.h
diff --git a/tools/testing/selftests/powerpc/vmx_asm.h b/tools/testing/selftests/powerpc/include/vmx_asm.h
index 2eaaeca9cf1d..2eaaeca9cf1d 100644
--- a/tools/testing/selftests/powerpc/vmx_asm.h
+++ b/tools/testing/selftests/powerpc/include/vmx_asm.h
diff --git a/tools/testing/selftests/powerpc/vsx_asm.h b/tools/testing/selftests/powerpc/include/vsx_asm.h
index d828bfb6ef2d..d828bfb6ef2d 100644
--- a/tools/testing/selftests/powerpc/vsx_asm.h
+++ b/tools/testing/selftests/powerpc/include/vsx_asm.h
diff --git a/tools/testing/selftests/powerpc/lib/reg.S b/tools/testing/selftests/powerpc/lib/reg.S
new file mode 100644
index 000000000000..0dc44f0da065
--- /dev/null
+++ b/tools/testing/selftests/powerpc/lib/reg.S
@@ -0,0 +1,397 @@
+/*
+ * test helper assembly functions
+ *
+ * Copyright (C) 2016 Simon Guo, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#include <ppc-asm.h>
+#include "reg.h"
+
+
+/* Non volatile GPR - unsigned long buf[18] */
+FUNC_START(load_gpr)
+	ld	14, 0*8(3)
+	ld	15, 1*8(3)
+	ld	16, 2*8(3)
+	ld	17, 3*8(3)
+	ld	18, 4*8(3)
+	ld	19, 5*8(3)
+	ld	20, 6*8(3)
+	ld	21, 7*8(3)
+	ld	22, 8*8(3)
+	ld	23, 9*8(3)
+	ld	24, 10*8(3)
+	ld	25, 11*8(3)
+	ld	26, 12*8(3)
+	ld	27, 13*8(3)
+	ld	28, 14*8(3)
+	ld	29, 15*8(3)
+	ld	30, 16*8(3)
+	ld	31, 17*8(3)
+	blr
+FUNC_END(load_gpr)
+
+FUNC_START(store_gpr)
+	std	14, 0*8(3)
+	std	15, 1*8(3)
+	std	16, 2*8(3)
+	std	17, 3*8(3)
+	std	18, 4*8(3)
+	std	19, 5*8(3)
+	std	20, 6*8(3)
+	std	21, 7*8(3)
+	std	22, 8*8(3)
+	std	23, 9*8(3)
+	std	24, 10*8(3)
+	std	25, 11*8(3)
+	std	26, 12*8(3)
+	std	27, 13*8(3)
+	std	28, 14*8(3)
+	std	29, 15*8(3)
+	std	30, 16*8(3)
+	std	31, 17*8(3)
+	blr
+FUNC_END(store_gpr)
+
+/* Single Precision Float - float buf[32] */
+FUNC_START(load_fpr_single_precision)
+	lfs 0, 0*4(3)
+	lfs 1, 1*4(3)
+	lfs 2, 2*4(3)
+	lfs 3, 3*4(3)
+	lfs 4, 4*4(3)
+	lfs 5, 5*4(3)
+	lfs 6, 6*4(3)
+	lfs 7, 7*4(3)
+	lfs 8, 8*4(3)
+	lfs 9, 9*4(3)
+	lfs 10, 10*4(3)
+	lfs 11, 11*4(3)
+	lfs 12, 12*4(3)
+	lfs 13, 13*4(3)
+	lfs 14, 14*4(3)
+	lfs 15, 15*4(3)
+	lfs 16, 16*4(3)
+	lfs 17, 17*4(3)
+	lfs 18, 18*4(3)
+	lfs 19, 19*4(3)
+	lfs 20, 20*4(3)
+	lfs 21, 21*4(3)
+	lfs 22, 22*4(3)
+	lfs 23, 23*4(3)
+	lfs 24, 24*4(3)
+	lfs 25, 25*4(3)
+	lfs 26, 26*4(3)
+	lfs 27, 27*4(3)
+	lfs 28, 28*4(3)
+	lfs 29, 29*4(3)
+	lfs 30, 30*4(3)
+	lfs 31, 31*4(3)
+	blr
+FUNC_END(load_fpr_single_precision)
+
+/* Single Precision Float - float buf[32] */
+FUNC_START(store_fpr_single_precision)
+	stfs 0, 0*4(3)
+	stfs 1, 1*4(3)
+	stfs 2, 2*4(3)
+	stfs 3, 3*4(3)
+	stfs 4, 4*4(3)
+	stfs 5, 5*4(3)
+	stfs 6, 6*4(3)
+	stfs 7, 7*4(3)
+	stfs 8, 8*4(3)
+	stfs 9, 9*4(3)
+	stfs 10, 10*4(3)
+	stfs 11, 11*4(3)
+	stfs 12, 12*4(3)
+	stfs 13, 13*4(3)
+	stfs 14, 14*4(3)
+	stfs 15, 15*4(3)
+	stfs 16, 16*4(3)
+	stfs 17, 17*4(3)
+	stfs 18, 18*4(3)
+	stfs 19, 19*4(3)
+	stfs 20, 20*4(3)
+	stfs 21, 21*4(3)
+	stfs 22, 22*4(3)
+	stfs 23, 23*4(3)
+	stfs 24, 24*4(3)
+	stfs 25, 25*4(3)
+	stfs 26, 26*4(3)
+	stfs 27, 27*4(3)
+	stfs 28, 28*4(3)
+	stfs 29, 29*4(3)
+	stfs 30, 30*4(3)
+	stfs 31, 31*4(3)
+	blr
+FUNC_END(store_fpr_single_precision)
+
+/* VMX/VSX registers - unsigned long buf[128] */
+FUNC_START(loadvsx)
+	lis	4, 0
+	LXVD2X	(0,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(1,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(2,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(3,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(4,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(5,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(6,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(7,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(8,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(9,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(10,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(11,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(12,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(13,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(14,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(15,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(16,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(17,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(18,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(19,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(20,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(21,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(22,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(23,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(24,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(25,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(26,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(27,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(28,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(29,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(30,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(31,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(32,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(33,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(34,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(35,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(36,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(37,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(38,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(39,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(40,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(41,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(42,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(43,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(44,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(45,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(46,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(47,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(48,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(49,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(50,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(51,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(52,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(53,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(54,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(55,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(56,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(57,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(58,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(59,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(60,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(61,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(62,(4),(3))
+	addi	4, 4, 16
+	LXVD2X	(63,(4),(3))
+	blr
+FUNC_END(loadvsx)
+
+FUNC_START(storevsx)
+	lis	4, 0
+	STXVD2X	(0,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(1,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(2,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(3,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(4,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(5,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(6,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(7,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(8,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(9,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(10,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(11,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(12,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(13,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(14,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(15,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(16,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(17,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(18,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(19,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(20,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(21,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(22,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(23,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(24,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(25,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(26,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(27,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(28,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(29,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(30,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(31,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(32,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(33,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(34,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(35,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(36,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(37,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(38,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(39,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(40,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(41,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(42,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(43,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(44,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(45,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(46,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(47,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(48,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(49,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(50,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(51,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(52,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(53,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(54,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(55,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(56,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(57,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(58,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(59,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(60,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(61,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(62,(4),(3))
+	addi	4, 4, 16
+	STXVD2X	(63,(4),(3))
+	blr
+FUNC_END(storevsx)
diff --git a/tools/testing/selftests/powerpc/math/fpu_asm.S b/tools/testing/selftests/powerpc/math/fpu_asm.S
index 241f067a510f..8a04bb117b69 100644
--- a/tools/testing/selftests/powerpc/math/fpu_asm.S
+++ b/tools/testing/selftests/powerpc/math/fpu_asm.S
@@ -7,8 +7,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
-#include "../basic_asm.h"
-#include "../fpu_asm.h"
+#include "basic_asm.h"
+#include "fpu_asm.h"
 
 FUNC_START(check_fpu)
 	mr r4,r3
diff --git a/tools/testing/selftests/powerpc/math/vmx_asm.S b/tools/testing/selftests/powerpc/math/vmx_asm.S
index fd74da488625..cb1e5ae1be99 100644
--- a/tools/testing/selftests/powerpc/math/vmx_asm.S
+++ b/tools/testing/selftests/powerpc/math/vmx_asm.S
@@ -7,8 +7,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
-#include "../basic_asm.h"
-#include "../vmx_asm.h"
+#include "basic_asm.h"
+#include "vmx_asm.h"
 
 # Should be safe from C, only touches r4, r5 and v0,v1,v2
 FUNC_START(check_vmx)
diff --git a/tools/testing/selftests/powerpc/math/vsx_asm.S b/tools/testing/selftests/powerpc/math/vsx_asm.S
index a110dd882d5e..8f431f6abc49 100644
--- a/tools/testing/selftests/powerpc/math/vsx_asm.S
+++ b/tools/testing/selftests/powerpc/math/vsx_asm.S
@@ -7,8 +7,8 @@
  * 2 of the License, or (at your option) any later version.
  */
 
-#include "../basic_asm.h"
-#include "../vsx_asm.h"
+#include "basic_asm.h"
+#include "vsx_asm.h"
 
 #long check_vsx(vector int *r3);
 #This function wraps storeing VSX regs to the end of an array and a
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/.gitignore b/tools/testing/selftests/powerpc/pmu/ebb/.gitignore
index 44b7df14a936..42bddbed8b64 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/.gitignore
+++ b/tools/testing/selftests/powerpc/pmu/ebb/.gitignore
@@ -20,5 +20,3 @@ back_to_back_ebbs_test
 lost_exception_test
 no_handler_test
 cycles_with_mmcr2_test
-ebb_lmr
-ebb_lmr_regs
\ No newline at end of file
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/Makefile b/tools/testing/selftests/powerpc/pmu/ebb/Makefile
index 6b0453e60d53..8d2279c4bb4b 100644
--- a/tools/testing/selftests/powerpc/pmu/ebb/Makefile
+++ b/tools/testing/selftests/powerpc/pmu/ebb/Makefile
@@ -14,7 +14,7 @@ TEST_PROGS := reg_access_test event_attributes_test cycles_test	\
 	 fork_cleanup_test ebb_on_child_test			\
 	 ebb_on_willing_child_test back_to_back_ebbs_test	\
 	 lost_exception_test no_handler_test			\
-	 cycles_with_mmcr2_test ebb_lmr ebb_lmr_regs
+	 cycles_with_mmcr2_test
 
 all: $(TEST_PROGS)
 
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/ebb_lmr.c b/tools/testing/selftests/powerpc/pmu/ebb/ebb_lmr.c
deleted file mode 100644
index c47ebd55ba4d..000000000000
--- a/tools/testing/selftests/powerpc/pmu/ebb/ebb_lmr.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Copyright 2016, Jack Miller, IBM Corp.
- * Licensed under GPLv2.
- */
-
-#include <stdlib.h>
-#include <stdio.h>
-
-#include "ebb.h"
-#include "ebb_lmr.h"
-
-#define SIZE		(32 * 1024 * 1024)	/* 32M */
-#define LM_SIZE		0	/* Smallest encoding, 32M */
-
-#define SECTIONS	64	/* 1 per bit in LMSER */
-#define SECTION_SIZE	(SIZE / SECTIONS)
-#define SECTION_LONGS   (SECTION_SIZE / sizeof(long))
-
-static unsigned long *test_mem;
-
-static int lmr_count = 0;
-
-void ebb_lmr_handler(void)
-{
-	lmr_count++;
-}
-
-void ldmx_full_section(unsigned long *mem, int section)
-{
-	unsigned long *ptr;
-	int i;
-
-	for (i = 0; i < SECTION_LONGS; i++) {
-		ptr = &mem[(SECTION_LONGS * section) + i];
-		ldmx((unsigned long) &ptr);
-		ebb_lmr_reset();
-	}
-}
-
-unsigned long section_masks[] = {
-	0x8000000000000000,
-	0xFF00000000000000,
-	0x0000000F70000000,
-	0x8000000000000001,
-	0xF0F0F0F0F0F0F0F0,
-	0x0F0F0F0F0F0F0F0F,
-	0x0
-};
-
-int ebb_lmr_section_test(unsigned long *mem)
-{
-	unsigned long *mask = section_masks;
-	int i;
-
-	for (; *mask; mask++) {
-		mtspr(SPRN_LMSER, *mask);
-		printf("Testing mask 0x%016lx\n", mfspr(SPRN_LMSER));
-
-		for (i = 0; i < 64; i++) {
-			lmr_count = 0;
-			ldmx_full_section(mem, i);
-			if (*mask & (1UL << (63 - i)))
-				FAIL_IF(lmr_count != SECTION_LONGS);
-			else
-				FAIL_IF(lmr_count);
-		}
-	}
-
-	return 0;
-}
-
-int ebb_lmr(void)
-{
-	int i;
-
-	SKIP_IF(!lmr_is_supported());
-
-	setup_ebb_handler(ebb_lmr_handler);
-
-	ebb_global_enable();
-
-	FAIL_IF(posix_memalign((void **)&test_mem, SIZE, SIZE) != 0);
-
-	mtspr(SPRN_LMSER, 0);
-
-	FAIL_IF(mfspr(SPRN_LMSER) != 0);
-
-	mtspr(SPRN_LMRR, ((unsigned long)test_mem | LM_SIZE));
-
-	FAIL_IF(mfspr(SPRN_LMRR) != ((unsigned long)test_mem | LM_SIZE));
-
-	/* Read every single byte to ensure we get no false positives */
-	for (i = 0; i < SECTIONS; i++)
-		ldmx_full_section(test_mem, i);
-
-	FAIL_IF(lmr_count != 0);
-
-	/* Turn on the first section */
-
-	mtspr(SPRN_LMSER, (1UL << 63));
-	FAIL_IF(mfspr(SPRN_LMSER) != (1UL << 63));
-
-	/* Enable LM (BESCR) */
-
-	mtspr(SPRN_BESCR, mfspr(SPRN_BESCR) | BESCR_LME);
-	FAIL_IF(!(mfspr(SPRN_BESCR) & BESCR_LME));
-
-	ldmx((unsigned long)&test_mem);
-
-	FAIL_IF(lmr_count != 1);	// exactly one exception
-	FAIL_IF(mfspr(SPRN_BESCR) & BESCR_LME);	// LM now disabled
-	FAIL_IF(!(mfspr(SPRN_BESCR) & BESCR_LMEO));	// occurred bit set
-
-	printf("Simple LMR EBB OK\n");
-
-	/* This shouldn't cause an EBB since it's been disabled */
-	ldmx((unsigned long)&test_mem);
-	FAIL_IF(lmr_count != 1);
-
-	printf("LMR disable on EBB OK\n");
-
-	ebb_lmr_reset();
-
-	/* This should cause an EBB or reset is broken */
-	ldmx((unsigned long)&test_mem);
-	FAIL_IF(lmr_count != 2);
-
-	printf("LMR reset EBB OK\n");
-
-	ebb_lmr_reset();
-
-	return ebb_lmr_section_test(test_mem);
-}
-
-int main(void)
-{
-	int ret = test_harness(ebb_lmr, "ebb_lmr");
-
-	if (test_mem)
-		free(test_mem);
-
-	return ret;
-}
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/ebb_lmr.h b/tools/testing/selftests/powerpc/pmu/ebb/ebb_lmr.h
deleted file mode 100644
index ef50abd557cd..000000000000
--- a/tools/testing/selftests/powerpc/pmu/ebb/ebb_lmr.h
+++ /dev/null
@@ -1,39 +0,0 @@
-#ifndef _SELFTESTS_POWERPC_PMU_EBB_LMR_H
-#define _SELFTESTS_POWERPC_PMU_EBB_LMR_H
-
-#include "reg.h"
-
-#ifndef PPC_FEATURE2_ARCH_3_00
-#define PPC_FEATURE2_ARCH_3_00 0x00800000
-#endif
-
-#define lmr_is_supported() have_hwcap2(PPC_FEATURE2_ARCH_3_00)
-
-static inline void ebb_lmr_reset(void)
-{
-	unsigned long bescr = mfspr(SPRN_BESCR);
-	bescr &= ~(BESCR_LMEO);
-	bescr |= BESCR_LME;
-	mtspr(SPRN_BESCR, bescr);
-}
-
-#define LDMX(t, a, b)\
-	(0x7c00026a |				\
-	 (((t) & 0x1f) << 21) |			\
-	 (((a) & 0x1f) << 16) |			\
-	 (((b) & 0x1f) << 11))
-
-static inline unsigned long ldmx(unsigned long address)
-{
-	unsigned long ret;
-
-	asm volatile ("mr 9, %1\r\n"
-		      ".long " __stringify(LDMX(9, 0, 9)) "\r\n"
-		      "mr %0, 9\r\n":"=r"(ret)
-		      :"r"(address)
-		      :"r9");
-
-	return ret;
-}
-
-#endif
diff --git a/tools/testing/selftests/powerpc/pmu/ebb/ebb_lmr_regs.c b/tools/testing/selftests/powerpc/pmu/ebb/ebb_lmr_regs.c
deleted file mode 100644
index aff4241fd88a..000000000000
--- a/tools/testing/selftests/powerpc/pmu/ebb/ebb_lmr_regs.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2016, Jack Miller, IBM Corp.
- * Licensed under GPLv2.
- */
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <unistd.h>
-
-#include "ebb.h"
-#include "ebb_lmr.h"
-
-#define CHECKS 10000
-
-int ebb_lmr_regs(void)
-{
-	int i;
-
-	SKIP_IF(!lmr_is_supported());
-
-	ebb_global_enable();
-
-	for (i = 0; i < CHECKS; i++) {
-		mtspr(SPRN_LMRR, i << 25);	// skip size and rsvd bits
-		mtspr(SPRN_LMSER, i);
-
-		FAIL_IF(mfspr(SPRN_LMRR) != (i << 25));
-		FAIL_IF(mfspr(SPRN_LMSER) != i);
-	}
-
-	return 0;
-}
-
-int main(void)
-{
-	return test_harness(ebb_lmr_regs, "ebb_lmr_regs");
-}
diff --git a/tools/testing/selftests/powerpc/pmu/lib.c b/tools/testing/selftests/powerpc/pmu/lib.c
index 8b992fa5b478..5bf5dd40822b 100644
--- a/tools/testing/selftests/powerpc/pmu/lib.c
+++ b/tools/testing/selftests/powerpc/pmu/lib.c
@@ -193,9 +193,9 @@ bool require_paranoia_below(int level)
 	long current;
 	char *end, buf[16];
 	FILE *f;
-	int rc;
+	bool rc;
 
-	rc = -1;
+	rc = false;
 
 	f = fopen(PARANOID_PATH, "r");
 	if (!f) {
@@ -218,7 +218,7 @@ bool require_paranoia_below(int level)
 	if (current >= level)
 		goto out_close;
 
-	rc = 0;
+	rc = true;
 out_close:
 	fclose(f);
 out:
diff --git a/tools/testing/selftests/powerpc/primitives/asm/firmware.h b/tools/testing/selftests/powerpc/primitives/asm/firmware.h
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/tools/testing/selftests/powerpc/primitives/asm/firmware.h
diff --git a/tools/testing/selftests/powerpc/primitives/asm/ppc_asm.h b/tools/testing/selftests/powerpc/primitives/asm/ppc_asm.h
new file mode 120000
index 000000000000..66c8193224e9
--- /dev/null
+++ b/tools/testing/selftests/powerpc/primitives/asm/ppc_asm.h
@@ -0,0 +1 @@
+../../../../../../arch/powerpc/include/asm/ppc_asm.h
\ No newline at end of file
diff --git a/tools/testing/selftests/powerpc/primitives/asm/processor.h b/tools/testing/selftests/powerpc/primitives/asm/processor.h
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/tools/testing/selftests/powerpc/primitives/asm/processor.h
diff --git a/tools/testing/selftests/powerpc/primitives/linux/stringify.h b/tools/testing/selftests/powerpc/primitives/linux/stringify.h
new file mode 100644
index 000000000000..e69de29bb2d1
--- /dev/null
+++ b/tools/testing/selftests/powerpc/primitives/linux/stringify.h
diff --git a/tools/testing/selftests/powerpc/primitives/load_unaligned_zeropad.c b/tools/testing/selftests/powerpc/primitives/load_unaligned_zeropad.c
index 6cae06117b55..ed3239bbfae2 100644
--- a/tools/testing/selftests/powerpc/primitives/load_unaligned_zeropad.c
+++ b/tools/testing/selftests/powerpc/primitives/load_unaligned_zeropad.c
@@ -73,20 +73,23 @@ extern char __stop___ex_table[];
 #error implement UCONTEXT_NIA
 #endif
 
-static int segv_error;
+struct extbl_entry {
+	int insn;
+	int fixup;
+};
 
 static void segv_handler(int signr, siginfo_t *info, void *ptr)
 {
 	ucontext_t *uc = (ucontext_t *)ptr;
 	unsigned long addr = (unsigned long)info->si_addr;
 	unsigned long *ip = &UCONTEXT_NIA(uc);
-	unsigned long *ex_p = (unsigned long *)__start___ex_table;
+	struct extbl_entry *entry = (struct extbl_entry *)__start___ex_table;
 
-	while (ex_p < (unsigned long *)__stop___ex_table) {
+	while (entry < (struct extbl_entry *)__stop___ex_table) {
 		unsigned long insn, fixup;
 
-		insn = *ex_p++;
-		fixup = *ex_p++;
+		insn  = (unsigned long)&entry->insn + entry->insn;
+		fixup = (unsigned long)&entry->fixup + entry->fixup;
 
 		if (insn == *ip) {
 			*ip = fixup;
@@ -95,7 +98,7 @@ static void segv_handler(int signr, siginfo_t *info, void *ptr)
 	}
 
 	printf("No exception table match for NIA %lx ADDR %lx\n", *ip, addr);
-	segv_error++;
+	abort();
 }
 
 static void setup_segv_handler(void)
@@ -119,8 +122,10 @@ static int do_one_test(char *p, int page_offset)
 
 	got = load_unaligned_zeropad(p);
 
-	if (should != got)
+	if (should != got) {
 		printf("offset %u load_unaligned_zeropad returned 0x%lx, should be 0x%lx\n", page_offset, got, should);
+		return 1;
+	}
 
 	return 0;
 }
@@ -145,8 +150,6 @@ static int test_body(void)
 	for (i = 0; i < page_size; i++)
 		FAIL_IF(do_one_test(mem_region+i, i));
 
-	FAIL_IF(segv_error);
-
 	return 0;
 }
 
diff --git a/tools/testing/selftests/powerpc/ptrace/.gitignore b/tools/testing/selftests/powerpc/ptrace/.gitignore
new file mode 100644
index 000000000000..349acfafc95b
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/.gitignore
@@ -0,0 +1,10 @@
+ptrace-gpr
+ptrace-tm-gpr
+ptrace-tm-spd-gpr
+ptrace-tar
+ptrace-tm-tar
+ptrace-tm-spd-tar
+ptrace-vsx
+ptrace-tm-vsx
+ptrace-tm-spd-vsx
+ptrace-tm-spr
diff --git a/tools/testing/selftests/powerpc/ptrace/Makefile b/tools/testing/selftests/powerpc/ptrace/Makefile
new file mode 100644
index 000000000000..fe6bc60dfc60
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/Makefile
@@ -0,0 +1,14 @@
+TEST_PROGS := ptrace-gpr ptrace-tm-gpr ptrace-tm-spd-gpr \
+              ptrace-tar ptrace-tm-tar ptrace-tm-spd-tar ptrace-vsx ptrace-tm-vsx \
+              ptrace-tm-spd-vsx ptrace-tm-spr
+
+include ../../lib.mk
+
+all: $(TEST_PROGS)
+
+CFLAGS += -m64 -I../../../../../usr/include -I../tm -mhtm
+
+$(TEST_PROGS): ../harness.c ../utils.c ../lib/reg.S ptrace.h
+
+clean:
+	rm -f $(TEST_PROGS) *.o
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-gpr.c b/tools/testing/selftests/powerpc/ptrace/ptrace-gpr.c
new file mode 100644
index 000000000000..0b4ebcc2f485
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-gpr.c
@@ -0,0 +1,123 @@
+/*
+ * Ptrace test for GPR/FPR registers
+ *
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#include "ptrace.h"
+#include "ptrace-gpr.h"
+#include "reg.h"
+
+/* Tracer and Tracee Shared Data */
+int shm_id;
+int *cptr, *pptr;
+
+float a = FPR_1;
+float b = FPR_2;
+float c = FPR_3;
+
+void gpr(void)
+{
+	unsigned long gpr_buf[18];
+	float fpr_buf[32];
+
+	cptr = (int *)shmat(shm_id, NULL, 0);
+
+	asm __volatile__(
+		ASM_LOAD_GPR_IMMED(gpr_1)
+		ASM_LOAD_FPR_SINGLE_PRECISION(flt_1)
+		:
+		: [gpr_1]"i"(GPR_1), [flt_1] "r" (&a)
+		: "memory", "r6", "r7", "r8", "r9", "r10",
+		"r11", "r12", "r13", "r14", "r15", "r16", "r17",
+		"r18", "r19", "r20", "r21", "r22", "r23", "r24",
+		"r25", "r26", "r27", "r28", "r29", "r30", "r31"
+		);
+
+	cptr[1] = 1;
+
+	while (!cptr[0])
+		asm volatile("" : : : "memory");
+
+	shmdt((void *)cptr);
+	store_gpr(gpr_buf);
+	store_fpr_single_precision(fpr_buf);
+
+	if (validate_gpr(gpr_buf, GPR_3))
+		exit(1);
+
+	if (validate_fpr_float(fpr_buf, c))
+		exit(1);
+
+	exit(0);
+}
+
+int trace_gpr(pid_t child)
+{
+	unsigned long gpr[18];
+	unsigned long fpr[32];
+
+	FAIL_IF(start_trace(child));
+	FAIL_IF(show_gpr(child, gpr));
+	FAIL_IF(validate_gpr(gpr, GPR_1));
+	FAIL_IF(show_fpr(child, fpr));
+	FAIL_IF(validate_fpr(fpr, FPR_1_REP));
+	FAIL_IF(write_gpr(child, GPR_3));
+	FAIL_IF(write_fpr(child, FPR_3_REP));
+	FAIL_IF(stop_trace(child));
+
+	return TEST_PASS;
+}
+
+int ptrace_gpr(void)
+{
+	pid_t pid;
+	int ret, status;
+
+	shm_id = shmget(IPC_PRIVATE, sizeof(int) * 2, 0777|IPC_CREAT);
+	pid = fork();
+	if (pid < 0) {
+		perror("fork() failed");
+		return TEST_FAIL;
+	}
+	if (pid == 0)
+		gpr();
+
+	if (pid) {
+		pptr = (int *)shmat(shm_id, NULL, 0);
+		while (!pptr[1])
+			asm volatile("" : : : "memory");
+
+		ret = trace_gpr(pid);
+		if (ret) {
+			kill(pid, SIGTERM);
+			shmdt((void *)pptr);
+			shmctl(shm_id, IPC_RMID, NULL);
+			return TEST_FAIL;
+		}
+
+		pptr[0] = 1;
+		shmdt((void *)pptr);
+
+		ret = wait(&status);
+		shmctl(shm_id, IPC_RMID, NULL);
+		if (ret != pid) {
+			printf("Child's exit status not captured\n");
+			return TEST_FAIL;
+		}
+
+		return (WIFEXITED(status) && WEXITSTATUS(status)) ? TEST_FAIL :
+			TEST_PASS;
+	}
+
+	return TEST_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	return test_harness(ptrace_gpr, "ptrace_gpr");
+}
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-gpr.h b/tools/testing/selftests/powerpc/ptrace/ptrace-gpr.h
new file mode 100644
index 000000000000..e30fef63824c
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-gpr.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#define GPR_1	1
+#define GPR_2	2
+#define GPR_3	3
+#define GPR_4	4
+
+#define FPR_1	0.001
+#define FPR_2	0.002
+#define FPR_3	0.003
+#define FPR_4	0.004
+
+#define FPR_1_REP 0x3f50624de0000000
+#define FPR_2_REP 0x3f60624de0000000
+#define FPR_3_REP 0x3f689374c0000000
+#define FPR_4_REP 0x3f70624de0000000
+
+/* Buffer must have 18 elements */
+int validate_gpr(unsigned long *gpr, unsigned long val)
+{
+	int i, found = 1;
+
+	for (i = 0; i < 18; i++) {
+		if (gpr[i] != val) {
+			printf("GPR[%d]: %lx Expected: %lx\n",
+				i+14, gpr[i], val);
+			found = 0;
+		}
+	}
+
+	if (!found)
+		return TEST_FAIL;
+	return TEST_PASS;
+}
+
+/* Buffer must have 32 elements */
+int validate_fpr(unsigned long *fpr, unsigned long val)
+{
+	int i, found = 1;
+
+	for (i = 0; i < 32; i++) {
+		if (fpr[i] != val) {
+			printf("FPR[%d]: %lx Expected: %lx\n", i, fpr[i], val);
+			found = 0;
+		}
+	}
+
+	if (!found)
+		return TEST_FAIL;
+	return TEST_PASS;
+}
+
+/* Buffer must have 32 elements */
+int validate_fpr_float(float *fpr, float val)
+{
+	int i, found = 1;
+
+	for (i = 0; i < 32; i++) {
+		if (fpr[i] != val) {
+			printf("FPR[%d]: %f Expected: %f\n", i, fpr[i], val);
+			found = 0;
+		}
+	}
+
+	if (!found)
+		return TEST_FAIL;
+	return TEST_PASS;
+}
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-tar.c b/tools/testing/selftests/powerpc/ptrace/ptrace-tar.c
new file mode 100644
index 000000000000..f9b5069db89b
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-tar.c
@@ -0,0 +1,135 @@
+/*
+ * Ptrace test for TAR, PPR, DSCR registers
+ *
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#include "ptrace.h"
+#include "ptrace-tar.h"
+
+/* Tracer and Tracee Shared Data */
+int shm_id;
+int *cptr;
+int *pptr;
+
+void tar(void)
+{
+	unsigned long reg[3];
+	int ret;
+
+	cptr = (int *)shmat(shm_id, NULL, 0);
+	printf("%-30s TAR: %u PPR: %lx DSCR: %u\n",
+			user_write, TAR_1, PPR_1, DSCR_1);
+
+	mtspr(SPRN_TAR, TAR_1);
+	mtspr(SPRN_PPR, PPR_1);
+	mtspr(SPRN_DSCR, DSCR_1);
+
+	cptr[2] = 1;
+
+	/* Wait on parent */
+	while (!cptr[0])
+		asm volatile("" : : : "memory");
+
+	reg[0] = mfspr(SPRN_TAR);
+	reg[1] = mfspr(SPRN_PPR);
+	reg[2] = mfspr(SPRN_DSCR);
+
+	printf("%-30s TAR: %lu PPR: %lx DSCR: %lu\n",
+			user_read, reg[0], reg[1], reg[2]);
+
+	/* Unblock the parent now */
+	cptr[1] = 1;
+	shmdt((int *)cptr);
+
+	ret = validate_tar_registers(reg, TAR_2, PPR_2, DSCR_2);
+	if (ret)
+		exit(1);
+	exit(0);
+}
+
+int trace_tar(pid_t child)
+{
+	unsigned long reg[3];
+
+	FAIL_IF(start_trace(child));
+	FAIL_IF(show_tar_registers(child, reg));
+	printf("%-30s TAR: %lu PPR: %lx DSCR: %lu\n",
+			ptrace_read_running, reg[0], reg[1], reg[2]);
+
+	FAIL_IF(validate_tar_registers(reg, TAR_1, PPR_1, DSCR_1));
+	FAIL_IF(stop_trace(child));
+	return TEST_PASS;
+}
+
+int trace_tar_write(pid_t child)
+{
+	FAIL_IF(start_trace(child));
+	FAIL_IF(write_tar_registers(child, TAR_2, PPR_2, DSCR_2));
+	printf("%-30s TAR: %u PPR: %lx DSCR: %u\n",
+			ptrace_write_running, TAR_2, PPR_2, DSCR_2);
+
+	FAIL_IF(stop_trace(child));
+	return TEST_PASS;
+}
+
+int ptrace_tar(void)
+{
+	pid_t pid;
+	int ret, status;
+
+	shm_id = shmget(IPC_PRIVATE, sizeof(int) * 3, 0777|IPC_CREAT);
+	pid = fork();
+	if (pid < 0) {
+		perror("fork() failed");
+		return TEST_FAIL;
+	}
+
+	if (pid == 0)
+		tar();
+
+	if (pid) {
+		pptr = (int *)shmat(shm_id, NULL, 0);
+		pptr[0] = 0;
+		pptr[1] = 0;
+
+		while (!pptr[2])
+			asm volatile("" : : : "memory");
+		ret = trace_tar(pid);
+		if (ret)
+			return ret;
+
+		ret = trace_tar_write(pid);
+		if (ret)
+			return ret;
+
+		/* Unblock the child now */
+		pptr[0] = 1;
+
+		/* Wait on child */
+		while (!pptr[1])
+			asm volatile("" : : : "memory");
+
+		shmdt((int *)pptr);
+
+		ret = wait(&status);
+		shmctl(shm_id, IPC_RMID, NULL);
+		if (ret != pid) {
+			printf("Child's exit status not captured\n");
+			return TEST_PASS;
+		}
+
+		return (WIFEXITED(status) && WEXITSTATUS(status)) ? TEST_FAIL :
+			TEST_PASS;
+	}
+	return TEST_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	return test_harness(ptrace_tar, "ptrace_tar");
+}
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-tar.h b/tools/testing/selftests/powerpc/ptrace/ptrace-tar.h
new file mode 100644
index 000000000000..aed0aac716d2
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-tar.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#define TAR_1   10
+#define TAR_2   20
+#define TAR_3   30
+#define TAR_4   40
+#define TAR_5   50
+
+#define DSCR_1  100
+#define DSCR_2  200
+#define DSCR_3  300
+#define DSCR_4  400
+#define DSCR_5  500
+
+#define PPR_1   0x4000000000000         /* or 31,31,31*/
+#define PPR_2   0x8000000000000         /* or 1,1,1 */
+#define PPR_3   0xc000000000000         /* or 6,6,6 */
+#define PPR_4   0x10000000000000        /* or 2,2,2 */
+
+char *user_read = "[User Read (Running)]";
+char *user_write = "[User Write (Running)]";
+char *ptrace_read_running = "[Ptrace Read (Running)]";
+char *ptrace_write_running = "[Ptrace Write (Running)]";
+char *ptrace_read_ckpt = "[Ptrace Read (Checkpointed)]";
+char *ptrace_write_ckpt = "[Ptrace Write (Checkpointed)]";
+
+int validate_tar_registers(unsigned long *reg, unsigned long tar,
+				unsigned long ppr, unsigned long dscr)
+{
+	int match = 1;
+
+	if (reg[0] != tar)
+		match = 0;
+
+	if (reg[1] != ppr)
+		match = 0;
+
+	if (reg[2] != dscr)
+		match = 0;
+
+	if (!match)
+		return TEST_FAIL;
+	return TEST_PASS;
+}
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-tm-gpr.c b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-gpr.c
new file mode 100644
index 000000000000..59206b96e98a
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-gpr.c
@@ -0,0 +1,158 @@
+/*
+ * Ptrace test for GPR/FPR registers in TM context
+ *
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#include "ptrace.h"
+#include "ptrace-gpr.h"
+#include "tm.h"
+
+/* Tracer and Tracee Shared Data */
+int shm_id;
+unsigned long *cptr, *pptr;
+
+float a = FPR_1;
+float b = FPR_2;
+float c = FPR_3;
+
+void tm_gpr(void)
+{
+	unsigned long gpr_buf[18];
+	unsigned long result, texasr;
+	float fpr_buf[32];
+
+	printf("Starting the child\n");
+	cptr = (unsigned long *)shmat(shm_id, NULL, 0);
+
+trans:
+	cptr[1] = 0;
+	asm __volatile__(
+		ASM_LOAD_GPR_IMMED(gpr_1)
+		ASM_LOAD_FPR_SINGLE_PRECISION(flt_1)
+		"1: ;"
+		"tbegin.;"
+		"beq 2f;"
+		ASM_LOAD_GPR_IMMED(gpr_2)
+		ASM_LOAD_FPR_SINGLE_PRECISION(flt_2)
+		"tsuspend.;"
+		"li 7, 1;"
+		"stw 7, 0(%[cptr1]);"
+		"tresume.;"
+		"b .;"
+
+		"tend.;"
+		"li 0, 0;"
+		"ori %[res], 0, 0;"
+		"b 3f;"
+
+		/* Transaction abort handler */
+		"2: ;"
+		"li 0, 1;"
+		"ori %[res], 0, 0;"
+		"mfspr %[texasr], %[sprn_texasr];"
+
+		"3: ;"
+		: [res] "=r" (result), [texasr] "=r" (texasr)
+		: [gpr_1]"i"(GPR_1), [gpr_2]"i"(GPR_2),
+		[sprn_texasr] "i" (SPRN_TEXASR), [flt_1] "r" (&a),
+		[flt_2] "r" (&b), [cptr1] "r" (&cptr[1])
+		: "memory", "r7", "r8", "r9", "r10",
+		"r11", "r12", "r13", "r14", "r15", "r16",
+		"r17", "r18", "r19", "r20", "r21", "r22",
+		"r23", "r24", "r25", "r26", "r27", "r28",
+		"r29", "r30", "r31"
+		);
+
+	if (result) {
+		if (!cptr[0])
+			goto trans;
+
+		shmdt((void *)cptr);
+		store_gpr(gpr_buf);
+		store_fpr_single_precision(fpr_buf);
+
+		if (validate_gpr(gpr_buf, GPR_3))
+			exit(1);
+
+		if (validate_fpr_float(fpr_buf, c))
+			exit(1);
+
+		exit(0);
+	}
+	shmdt((void *)cptr);
+	exit(1);
+}
+
+int trace_tm_gpr(pid_t child)
+{
+	unsigned long gpr[18];
+	unsigned long fpr[32];
+
+	FAIL_IF(start_trace(child));
+	FAIL_IF(show_gpr(child, gpr));
+	FAIL_IF(validate_gpr(gpr, GPR_2));
+	FAIL_IF(show_fpr(child, fpr));
+	FAIL_IF(validate_fpr(fpr, FPR_2_REP));
+	FAIL_IF(show_ckpt_fpr(child, fpr));
+	FAIL_IF(validate_fpr(fpr, FPR_1_REP));
+	FAIL_IF(show_ckpt_gpr(child, gpr));
+	FAIL_IF(validate_gpr(gpr, GPR_1));
+	FAIL_IF(write_ckpt_gpr(child, GPR_3));
+	FAIL_IF(write_ckpt_fpr(child, FPR_3_REP));
+
+	pptr[0] = 1;
+	FAIL_IF(stop_trace(child));
+
+	return TEST_PASS;
+}
+
+int ptrace_tm_gpr(void)
+{
+	pid_t pid;
+	int ret, status;
+
+	SKIP_IF(!have_htm());
+	shm_id = shmget(IPC_PRIVATE, sizeof(int) * 2, 0777|IPC_CREAT);
+	pid = fork();
+	if (pid < 0) {
+		perror("fork() failed");
+		return TEST_FAIL;
+	}
+	if (pid == 0)
+		tm_gpr();
+
+	if (pid) {
+		pptr = (unsigned long *)shmat(shm_id, NULL, 0);
+
+		while (!pptr[1])
+			asm volatile("" : : : "memory");
+		ret = trace_tm_gpr(pid);
+		if (ret) {
+			kill(pid, SIGTERM);
+			return TEST_FAIL;
+		}
+
+		shmdt((void *)pptr);
+
+		ret = wait(&status);
+		shmctl(shm_id, IPC_RMID, NULL);
+		if (ret != pid) {
+			printf("Child's exit status not captured\n");
+			return TEST_FAIL;
+		}
+
+		return (WIFEXITED(status) && WEXITSTATUS(status)) ? TEST_FAIL :
+			TEST_PASS;
+	}
+	return TEST_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	return test_harness(ptrace_tm_gpr, "ptrace_tm_gpr");
+}
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-tm-spd-gpr.c b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-spd-gpr.c
new file mode 100644
index 000000000000..327fa943c7f3
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-spd-gpr.c
@@ -0,0 +1,169 @@
+/*
+ * Ptrace test for GPR/FPR registers in TM Suspend context
+ *
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#include "ptrace.h"
+#include "ptrace-gpr.h"
+#include "tm.h"
+
+/* Tracer and Tracee Shared Data */
+int shm_id;
+int *cptr, *pptr;
+
+float a = FPR_1;
+float b = FPR_2;
+float c = FPR_3;
+float d = FPR_4;
+
+__attribute__((used)) void wait_parent(void)
+{
+	cptr[2] = 1;
+	while (!cptr[1])
+		asm volatile("" : : : "memory");
+}
+
+void tm_spd_gpr(void)
+{
+	unsigned long gpr_buf[18];
+	unsigned long result, texasr;
+	float fpr_buf[32];
+
+	cptr = (int *)shmat(shm_id, NULL, 0);
+
+trans:
+	cptr[2] = 0;
+	asm __volatile__(
+		ASM_LOAD_GPR_IMMED(gpr_1)
+		ASM_LOAD_FPR_SINGLE_PRECISION(flt_1)
+
+		"1: ;"
+		"tbegin.;"
+		"beq 2f;"
+
+		ASM_LOAD_GPR_IMMED(gpr_2)
+		"tsuspend.;"
+		ASM_LOAD_GPR_IMMED(gpr_4)
+		ASM_LOAD_FPR_SINGLE_PRECISION(flt_4)
+
+		"bl wait_parent;"
+		"tresume.;"
+		"tend.;"
+		"li 0, 0;"
+		"ori %[res], 0, 0;"
+		"b 3f;"
+
+		/* Transaction abort handler */
+		"2: ;"
+		"li 0, 1;"
+		"ori %[res], 0, 0;"
+		"mfspr %[texasr], %[sprn_texasr];"
+
+		"3: ;"
+		: [res] "=r" (result), [texasr] "=r" (texasr)
+		: [gpr_1]"i"(GPR_1), [gpr_2]"i"(GPR_2), [gpr_4]"i"(GPR_4),
+		[sprn_texasr] "i" (SPRN_TEXASR), [flt_1] "r" (&a),
+		[flt_2] "r" (&b), [flt_4] "r" (&d)
+		: "memory", "r5", "r6", "r7",
+		"r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15",
+		"r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
+		"r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31"
+		);
+
+	if (result) {
+		if (!cptr[0])
+			goto trans;
+
+		shmdt((void *)cptr);
+		store_gpr(gpr_buf);
+		store_fpr_single_precision(fpr_buf);
+
+		if (validate_gpr(gpr_buf, GPR_3))
+			exit(1);
+
+		if (validate_fpr_float(fpr_buf, c))
+			exit(1);
+		exit(0);
+	}
+	shmdt((void *)cptr);
+	exit(1);
+}
+
+int trace_tm_spd_gpr(pid_t child)
+{
+	unsigned long gpr[18];
+	unsigned long fpr[32];
+
+	FAIL_IF(start_trace(child));
+	FAIL_IF(show_gpr(child, gpr));
+	FAIL_IF(validate_gpr(gpr, GPR_4));
+	FAIL_IF(show_fpr(child, fpr));
+	FAIL_IF(validate_fpr(fpr, FPR_4_REP));
+	FAIL_IF(show_ckpt_fpr(child, fpr));
+	FAIL_IF(validate_fpr(fpr, FPR_1_REP));
+	FAIL_IF(show_ckpt_gpr(child, gpr));
+	FAIL_IF(validate_gpr(gpr, GPR_1));
+	FAIL_IF(write_ckpt_gpr(child, GPR_3));
+	FAIL_IF(write_ckpt_fpr(child, FPR_3_REP));
+
+	pptr[0] = 1;
+	pptr[1] = 1;
+	FAIL_IF(stop_trace(child));
+	return TEST_PASS;
+}
+
+int ptrace_tm_spd_gpr(void)
+{
+	pid_t pid;
+	int ret, status;
+
+	SKIP_IF(!have_htm());
+	shm_id = shmget(IPC_PRIVATE, sizeof(int) * 3, 0777|IPC_CREAT);
+	pid = fork();
+	if (pid < 0) {
+		perror("fork() failed");
+		return TEST_FAIL;
+	}
+
+	if (pid == 0)
+		tm_spd_gpr();
+
+	if (pid) {
+		pptr = (int *)shmat(shm_id, NULL, 0);
+		pptr[0] = 0;
+		pptr[1] = 0;
+
+		while (!pptr[2])
+			asm volatile("" : : : "memory");
+		ret = trace_tm_spd_gpr(pid);
+		if (ret) {
+			kill(pid, SIGTERM);
+			shmdt((void *)pptr);
+			shmctl(shm_id, IPC_RMID, NULL);
+			return TEST_FAIL;
+		}
+
+		shmdt((void *)pptr);
+
+		ret = wait(&status);
+		shmctl(shm_id, IPC_RMID, NULL);
+		if (ret != pid) {
+			printf("Child's exit status not captured\n");
+			return TEST_FAIL;
+		}
+
+		return (WIFEXITED(status) && WEXITSTATUS(status)) ? TEST_FAIL :
+			TEST_PASS;
+	}
+	return TEST_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	return test_harness(ptrace_tm_spd_gpr, "ptrace_tm_spd_gpr");
+}
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-tm-spd-tar.c b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-spd-tar.c
new file mode 100644
index 000000000000..b3c061dc9512
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-spd-tar.c
@@ -0,0 +1,174 @@
+/*
+ * Ptrace test for TAR, PPR, DSCR registers in the TM Suspend context
+ *
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#include "ptrace.h"
+#include "tm.h"
+#include "ptrace-tar.h"
+
+int shm_id;
+int *cptr, *pptr;
+
+__attribute__((used)) void wait_parent(void)
+{
+	cptr[2] = 1;
+	while (!cptr[1])
+		asm volatile("" : : : "memory");
+}
+
+void tm_spd_tar(void)
+{
+	unsigned long result, texasr;
+	unsigned long regs[3];
+	int ret;
+
+	cptr = (int *)shmat(shm_id, NULL, 0);
+
+trans:
+	cptr[2] = 0;
+	asm __volatile__(
+		"li	4, %[tar_1];"
+		"mtspr %[sprn_tar],  4;"	/* TAR_1 */
+		"li	4, %[dscr_1];"
+		"mtspr %[sprn_dscr], 4;"	/* DSCR_1 */
+		"or     31,31,31;"		/* PPR_1*/
+
+		"1: ;"
+		"tbegin.;"
+		"beq 2f;"
+
+		"li	4, %[tar_2];"
+		"mtspr %[sprn_tar],  4;"	/* TAR_2 */
+		"li	4, %[dscr_2];"
+		"mtspr %[sprn_dscr], 4;"	/* DSCR_2 */
+		"or     1,1,1;"			/* PPR_2 */
+
+		"tsuspend.;"
+		"li	4, %[tar_3];"
+		"mtspr %[sprn_tar],  4;"	/* TAR_3 */
+		"li	4, %[dscr_3];"
+		"mtspr %[sprn_dscr], 4;"	/* DSCR_3 */
+		"or     6,6,6;"			/* PPR_3 */
+		"bl wait_parent;"
+		"tresume.;"
+
+		"tend.;"
+		"li 0, 0;"
+		"ori %[res], 0, 0;"
+		"b 3f;"
+
+		/* Transaction abort handler */
+		"2: ;"
+		"li 0, 1;"
+		"ori %[res], 0, 0;"
+		"mfspr %[texasr], %[sprn_texasr];"
+
+		"3: ;"
+
+		: [res] "=r" (result), [texasr] "=r" (texasr)
+		: [val] "r" (cptr[1]), [sprn_dscr]"i"(SPRN_DSCR),
+		[sprn_tar]"i"(SPRN_TAR), [sprn_ppr]"i"(SPRN_PPR),
+		[sprn_texasr]"i"(SPRN_TEXASR), [tar_1]"i"(TAR_1),
+		[dscr_1]"i"(DSCR_1), [tar_2]"i"(TAR_2), [dscr_2]"i"(DSCR_2),
+		[tar_3]"i"(TAR_3), [dscr_3]"i"(DSCR_3)
+		: "memory", "r0", "r1", "r3", "r4", "r5", "r6"
+		);
+
+	/* TM failed, analyse */
+	if (result) {
+		if (!cptr[0])
+			goto trans;
+
+		regs[0] = mfspr(SPRN_TAR);
+		regs[1] = mfspr(SPRN_PPR);
+		regs[2] = mfspr(SPRN_DSCR);
+
+		shmdt(&cptr);
+		printf("%-30s TAR: %lu PPR: %lx DSCR: %lu\n",
+				user_read, regs[0], regs[1], regs[2]);
+
+		ret = validate_tar_registers(regs, TAR_4, PPR_4, DSCR_4);
+		if (ret)
+			exit(1);
+		exit(0);
+	}
+	shmdt(&cptr);
+	exit(1);
+}
+
+int trace_tm_spd_tar(pid_t child)
+{
+	unsigned long regs[3];
+
+	FAIL_IF(start_trace(child));
+	FAIL_IF(show_tar_registers(child, regs));
+	printf("%-30s TAR: %lu PPR: %lx DSCR: %lu\n",
+			ptrace_read_running, regs[0], regs[1], regs[2]);
+
+	FAIL_IF(validate_tar_registers(regs, TAR_3, PPR_3, DSCR_3));
+	FAIL_IF(show_tm_checkpointed_state(child, regs));
+	printf("%-30s TAR: %lu PPR: %lx DSCR: %lu\n",
+			ptrace_read_ckpt, regs[0], regs[1], regs[2]);
+
+	FAIL_IF(validate_tar_registers(regs, TAR_1, PPR_1, DSCR_1));
+	FAIL_IF(write_ckpt_tar_registers(child, TAR_4, PPR_4, DSCR_4));
+	printf("%-30s TAR: %u PPR: %lx DSCR: %u\n",
+			ptrace_write_ckpt, TAR_4, PPR_4, DSCR_4);
+
+	pptr[0] = 1;
+	pptr[1] = 1;
+	FAIL_IF(stop_trace(child));
+	return TEST_PASS;
+}
+
+int ptrace_tm_spd_tar(void)
+{
+	pid_t pid;
+	int ret, status;
+
+	SKIP_IF(!have_htm());
+	shm_id = shmget(IPC_PRIVATE, sizeof(int) * 3, 0777|IPC_CREAT);
+	pid = fork();
+	if (pid == 0)
+		tm_spd_tar();
+
+	pptr = (int *)shmat(shm_id, NULL, 0);
+	pptr[0] = 0;
+	pptr[1] = 0;
+
+	if (pid) {
+		while (!pptr[2])
+			asm volatile("" : : : "memory");
+		ret = trace_tm_spd_tar(pid);
+		if (ret) {
+			kill(pid, SIGTERM);
+			shmdt(&pptr);
+			shmctl(shm_id, IPC_RMID, NULL);
+			return TEST_FAIL;
+		}
+
+		shmdt(&pptr);
+
+		ret = wait(&status);
+		shmctl(shm_id, IPC_RMID, NULL);
+		if (ret != pid) {
+			printf("Child's exit status not captured\n");
+			return TEST_FAIL;
+		}
+
+		return (WIFEXITED(status) && WEXITSTATUS(status)) ? TEST_FAIL :
+			TEST_PASS;
+	}
+	return TEST_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	return test_harness(ptrace_tm_spd_tar, "ptrace_tm_spd_tar");
+}
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-tm-spd-vsx.c b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-spd-vsx.c
new file mode 100644
index 000000000000..0df3c23b7888
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-spd-vsx.c
@@ -0,0 +1,185 @@
+/*
+ * Ptrace test for VMX/VSX registers in the TM Suspend context
+ *
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#include "ptrace.h"
+#include "tm.h"
+#include "ptrace-vsx.h"
+
+int shm_id;
+int *cptr, *pptr;
+
+unsigned long fp_load[VEC_MAX];
+unsigned long fp_load_new[VEC_MAX];
+unsigned long fp_store[VEC_MAX];
+unsigned long fp_load_ckpt[VEC_MAX];
+unsigned long fp_load_ckpt_new[VEC_MAX];
+
+__attribute__((used)) void load_vsx(void)
+{
+	loadvsx(fp_load, 0);
+}
+
+__attribute__((used)) void load_vsx_new(void)
+{
+	loadvsx(fp_load_new, 0);
+}
+
+__attribute__((used)) void load_vsx_ckpt(void)
+{
+	loadvsx(fp_load_ckpt, 0);
+}
+
+__attribute__((used)) void wait_parent(void)
+{
+	cptr[2] = 1;
+	while (!cptr[1])
+		asm volatile("" : : : "memory");
+}
+
+void tm_spd_vsx(void)
+{
+	unsigned long result, texasr;
+	int ret;
+
+	cptr = (int *)shmat(shm_id, NULL, 0);
+
+trans:
+	cptr[2] = 0;
+	asm __volatile__(
+		"bl load_vsx_ckpt;"
+
+		"1: ;"
+		"tbegin.;"
+		"beq 2f;"
+
+		"bl load_vsx_new;"
+		"tsuspend.;"
+		"bl load_vsx;"
+		"bl wait_parent;"
+		"tresume.;"
+
+		"tend.;"
+		"li 0, 0;"
+		"ori %[res], 0, 0;"
+		"b 3f;"
+
+		"2: ;"
+		"li 0, 1;"
+		"ori %[res], 0, 0;"
+		"mfspr %[texasr], %[sprn_texasr];"
+
+		"3: ;"
+		: [res] "=r" (result), [texasr] "=r" (texasr)
+		: [fp_load] "r" (fp_load), [fp_load_ckpt] "r" (fp_load_ckpt),
+		[sprn_texasr] "i"  (SPRN_TEXASR)
+		: "memory", "r0", "r1", "r2", "r3", "r4",
+		"r8", "r9", "r10", "r11"
+		);
+
+	if (result) {
+		if (!cptr[0])
+			goto trans;
+		shmdt((void *)cptr);
+
+		storevsx(fp_store, 0);
+		ret = compare_vsx_vmx(fp_store, fp_load_ckpt_new);
+		if (ret)
+			exit(1);
+		exit(0);
+	}
+	shmdt((void *)cptr);
+	exit(1);
+}
+
+int trace_tm_spd_vsx(pid_t child)
+{
+	unsigned long vsx[VSX_MAX];
+	unsigned long vmx[VMX_MAX + 2][2];
+
+	FAIL_IF(start_trace(child));
+	FAIL_IF(show_vsx(child, vsx));
+	FAIL_IF(validate_vsx(vsx, fp_load));
+	FAIL_IF(show_vmx(child, vmx));
+	FAIL_IF(validate_vmx(vmx, fp_load));
+	FAIL_IF(show_vsx_ckpt(child, vsx));
+	FAIL_IF(validate_vsx(vsx, fp_load_ckpt));
+	FAIL_IF(show_vmx_ckpt(child, vmx));
+	FAIL_IF(validate_vmx(vmx, fp_load_ckpt));
+
+	memset(vsx, 0, sizeof(vsx));
+	memset(vmx, 0, sizeof(vmx));
+
+	load_vsx_vmx(fp_load_ckpt_new, vsx, vmx);
+
+	FAIL_IF(write_vsx_ckpt(child, vsx));
+	FAIL_IF(write_vmx_ckpt(child, vmx));
+
+	pptr[0] = 1;
+	pptr[1] = 1;
+	FAIL_IF(stop_trace(child));
+
+	return TEST_PASS;
+}
+
+int ptrace_tm_spd_vsx(void)
+{
+	pid_t pid;
+	int ret, status, i;
+
+	SKIP_IF(!have_htm());
+	shm_id = shmget(IPC_PRIVATE, sizeof(int) * 3, 0777|IPC_CREAT);
+
+	for (i = 0; i < 128; i++) {
+		fp_load[i] = 1 + rand();
+		fp_load_new[i] = 1 + 2 * rand();
+		fp_load_ckpt[i] = 1 + 3 * rand();
+		fp_load_ckpt_new[i] = 1 + 4 * rand();
+	}
+
+	pid = fork();
+	if (pid < 0) {
+		perror("fork() failed");
+		return TEST_FAIL;
+	}
+
+	if (pid == 0)
+		tm_spd_vsx();
+
+	if (pid) {
+		pptr = (int *)shmat(shm_id, NULL, 0);
+		while (!pptr[2])
+			asm volatile("" : : : "memory");
+
+		ret = trace_tm_spd_vsx(pid);
+		if (ret) {
+			kill(pid, SIGKILL);
+			shmdt((void *)pptr);
+			shmctl(shm_id, IPC_RMID, NULL);
+			return TEST_FAIL;
+		}
+
+		shmdt((void *)pptr);
+		ret = wait(&status);
+		shmctl(shm_id, IPC_RMID, NULL);
+		if (ret != pid) {
+			printf("Child's exit status not captured\n");
+			return TEST_FAIL;
+		}
+
+		return (WIFEXITED(status) && WEXITSTATUS(status)) ? TEST_FAIL :
+			TEST_PASS;
+	}
+	return TEST_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	return test_harness(ptrace_tm_spd_vsx, "ptrace_tm_spd_vsx");
+}
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-tm-spr.c b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-spr.c
new file mode 100644
index 000000000000..94e57cb89769
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-spr.c
@@ -0,0 +1,168 @@
+/*
+ * Ptrace test TM SPR registers
+ *
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#include "ptrace.h"
+#include "tm.h"
+
+/* Tracee and tracer shared data */
+struct shared {
+	int flag;
+	struct tm_spr_regs regs;
+};
+unsigned long tfhar;
+
+int shm_id;
+struct shared *cptr, *pptr;
+
+int shm_id1;
+int *cptr1, *pptr1;
+
+#define TM_KVM_SCHED   0xe0000001ac000001
+int validate_tm_spr(struct tm_spr_regs *regs)
+{
+	FAIL_IF(regs->tm_tfhar != tfhar);
+	FAIL_IF((regs->tm_texasr == TM_KVM_SCHED) && (regs->tm_tfiar != 0));
+
+	return TEST_PASS;
+}
+
+void tm_spr(void)
+{
+	unsigned long result, texasr;
+	int ret;
+
+	cptr = (struct shared *)shmat(shm_id, NULL, 0);
+	cptr1 = (int *)shmat(shm_id1, NULL, 0);
+
+trans:
+	cptr1[0] = 0;
+	asm __volatile__(
+		"1: ;"
+		/* TM failover handler should follow "tbegin.;" */
+		"mflr 31;"
+		"bl 4f;"	/* $ = TFHAR - 12 */
+		"4: ;"
+		"mflr %[tfhar];"
+		"mtlr 31;"
+
+		"tbegin.;"
+		"beq 2f;"
+
+		"tsuspend.;"
+		"li 8, 1;"
+		"sth 8, 0(%[cptr1]);"
+		"tresume.;"
+		"b .;"
+
+		"tend.;"
+		"li 0, 0;"
+		"ori %[res], 0, 0;"
+		"b 3f;"
+
+		"2: ;"
+
+		"li 0, 1;"
+		"ori %[res], 0, 0;"
+		"mfspr %[texasr], %[sprn_texasr];"
+
+		"3: ;"
+		: [tfhar] "=r" (tfhar), [res] "=r" (result),
+		[texasr] "=r" (texasr), [cptr1] "=r" (cptr1)
+		: [sprn_texasr] "i"  (SPRN_TEXASR)
+		: "memory", "r0", "r1", "r2", "r3", "r4",
+		"r8", "r9", "r10", "r11", "r31"
+		);
+
+	/* There are 2 32bit instructions before tbegin. */
+	tfhar += 12;
+
+	if (result) {
+		if (!cptr->flag)
+			goto trans;
+
+		ret = validate_tm_spr((struct tm_spr_regs *)&cptr->regs);
+		shmdt((void *)cptr);
+		shmdt((void *)cptr1);
+		if (ret)
+			exit(1);
+		exit(0);
+	}
+	shmdt((void *)cptr);
+	shmdt((void *)cptr1);
+	exit(1);
+}
+
+int trace_tm_spr(pid_t child)
+{
+	FAIL_IF(start_trace(child));
+	FAIL_IF(show_tm_spr(child, (struct tm_spr_regs *)&pptr->regs));
+
+	printf("TFHAR: %lx TEXASR: %lx TFIAR: %lx\n", pptr->regs.tm_tfhar,
+				pptr->regs.tm_texasr, pptr->regs.tm_tfiar);
+
+	pptr->flag = 1;
+	FAIL_IF(stop_trace(child));
+
+	return TEST_PASS;
+}
+
+int ptrace_tm_spr(void)
+{
+	pid_t pid;
+	int ret, status;
+
+	SKIP_IF(!have_htm());
+	shm_id = shmget(IPC_PRIVATE, sizeof(struct shared), 0777|IPC_CREAT);
+	shm_id1 = shmget(IPC_PRIVATE, sizeof(int), 0777|IPC_CREAT);
+	pid = fork();
+	if (pid < 0) {
+		perror("fork() failed");
+		return TEST_FAIL;
+	}
+
+	if (pid == 0)
+		tm_spr();
+
+	if (pid) {
+		pptr = (struct shared *)shmat(shm_id, NULL, 0);
+		pptr1 = (int *)shmat(shm_id1, NULL, 0);
+
+		while (!pptr1[0])
+			asm volatile("" : : : "memory");
+		ret = trace_tm_spr(pid);
+		if (ret) {
+			kill(pid, SIGKILL);
+			shmdt((void *)pptr);
+			shmdt((void *)pptr1);
+			shmctl(shm_id, IPC_RMID, NULL);
+			shmctl(shm_id1, IPC_RMID, NULL);
+			return TEST_FAIL;
+		}
+
+		shmdt((void *)pptr);
+		shmdt((void *)pptr1);
+		ret = wait(&status);
+		shmctl(shm_id, IPC_RMID, NULL);
+		shmctl(shm_id1, IPC_RMID, NULL);
+		if (ret != pid) {
+			printf("Child's exit status not captured\n");
+			return TEST_FAIL;
+		}
+
+		return (WIFEXITED(status) && WEXITSTATUS(status)) ? TEST_FAIL :
+			TEST_PASS;
+	}
+	return TEST_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	return test_harness(ptrace_tm_spr, "ptrace_tm_spr");
+}
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-tm-tar.c b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-tar.c
new file mode 100644
index 000000000000..48b462f75023
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-tar.c
@@ -0,0 +1,160 @@
+/*
+ * Ptrace test for TAR, PPR, DSCR registers in the TM context
+ *
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#include "ptrace.h"
+#include "tm.h"
+#include "ptrace-tar.h"
+
+int shm_id;
+unsigned long *cptr, *pptr;
+
+
+void tm_tar(void)
+{
+	unsigned long result, texasr;
+	unsigned long regs[3];
+	int ret;
+
+	cptr = (unsigned long *)shmat(shm_id, NULL, 0);
+
+trans:
+	cptr[1] = 0;
+	asm __volatile__(
+		"li	4, %[tar_1];"
+		"mtspr %[sprn_tar],  4;"	/* TAR_1 */
+		"li	4, %[dscr_1];"
+		"mtspr %[sprn_dscr], 4;"	/* DSCR_1 */
+		"or     31,31,31;"		/* PPR_1*/
+
+		"1: ;"
+		"tbegin.;"
+		"beq 2f;"
+
+		"li	4, %[tar_2];"
+		"mtspr %[sprn_tar],  4;"	/* TAR_2 */
+		"li	4, %[dscr_2];"
+		"mtspr %[sprn_dscr], 4;"	/* DSCR_2 */
+		"or     1,1,1;"			/* PPR_2 */
+		"tsuspend.;"
+		"li 0, 1;"
+		"stw 0, 0(%[cptr1]);"
+		"tresume.;"
+		"b .;"
+
+		"tend.;"
+		"li 0, 0;"
+		"ori %[res], 0, 0;"
+		"b 3f;"
+
+		/* Transaction abort handler */
+		"2: ;"
+		"li 0, 1;"
+		"ori %[res], 0, 0;"
+		"mfspr %[texasr], %[sprn_texasr];"
+
+		"3: ;"
+
+		: [res] "=r" (result), [texasr] "=r" (texasr)
+		: [sprn_dscr]"i"(SPRN_DSCR), [sprn_tar]"i"(SPRN_TAR),
+		[sprn_ppr]"i"(SPRN_PPR), [sprn_texasr]"i"(SPRN_TEXASR),
+		[tar_1]"i"(TAR_1), [dscr_1]"i"(DSCR_1), [tar_2]"i"(TAR_2),
+		[dscr_2]"i"(DSCR_2), [cptr1] "r" (&cptr[1])
+		: "memory", "r0", "r1", "r3", "r4", "r5", "r6"
+		);
+
+	/* TM failed, analyse */
+	if (result) {
+		if (!cptr[0])
+			goto trans;
+
+		regs[0] = mfspr(SPRN_TAR);
+		regs[1] = mfspr(SPRN_PPR);
+		regs[2] = mfspr(SPRN_DSCR);
+
+		shmdt(&cptr);
+		printf("%-30s TAR: %lu PPR: %lx DSCR: %lu\n",
+				user_read, regs[0], regs[1], regs[2]);
+
+		ret = validate_tar_registers(regs, TAR_4, PPR_4, DSCR_4);
+		if (ret)
+			exit(1);
+		exit(0);
+	}
+	shmdt(&cptr);
+	exit(1);
+}
+
+int trace_tm_tar(pid_t child)
+{
+	unsigned long regs[3];
+
+	FAIL_IF(start_trace(child));
+	FAIL_IF(show_tar_registers(child, regs));
+	printf("%-30s TAR: %lu PPR: %lx DSCR: %lu\n",
+			ptrace_read_running, regs[0], regs[1], regs[2]);
+
+	FAIL_IF(validate_tar_registers(regs, TAR_2, PPR_2, DSCR_2));
+	FAIL_IF(show_tm_checkpointed_state(child, regs));
+	printf("%-30s TAR: %lu PPR: %lx DSCR: %lu\n",
+			ptrace_read_ckpt, regs[0], regs[1], regs[2]);
+
+	FAIL_IF(validate_tar_registers(regs, TAR_1, PPR_1, DSCR_1));
+	FAIL_IF(write_ckpt_tar_registers(child, TAR_4, PPR_4, DSCR_4));
+	printf("%-30s TAR: %u PPR: %lx DSCR: %u\n",
+			ptrace_write_ckpt, TAR_4, PPR_4, DSCR_4);
+
+	pptr[0] = 1;
+	FAIL_IF(stop_trace(child));
+	return TEST_PASS;
+}
+
+int ptrace_tm_tar(void)
+{
+	pid_t pid;
+	int ret, status;
+
+	SKIP_IF(!have_htm());
+	shm_id = shmget(IPC_PRIVATE, sizeof(int) * 2, 0777|IPC_CREAT);
+	pid = fork();
+	if (pid == 0)
+		tm_tar();
+
+	pptr = (unsigned long *)shmat(shm_id, NULL, 0);
+	pptr[0] = 0;
+
+	if (pid) {
+		while (!pptr[1])
+			asm volatile("" : : : "memory");
+		ret = trace_tm_tar(pid);
+		if (ret) {
+			kill(pid, SIGTERM);
+			shmdt(&pptr);
+			shmctl(shm_id, IPC_RMID, NULL);
+			return TEST_FAIL;
+		}
+		shmdt(&pptr);
+
+		ret = wait(&status);
+		shmctl(shm_id, IPC_RMID, NULL);
+		if (ret != pid) {
+			printf("Child's exit status not captured\n");
+			return TEST_FAIL;
+		}
+
+		return (WIFEXITED(status) && WEXITSTATUS(status)) ? TEST_FAIL :
+			TEST_PASS;
+	}
+	return TEST_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	return test_harness(ptrace_tm_tar, "ptrace_tm_tar");
+}
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-tm-vsx.c b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-vsx.c
new file mode 100644
index 000000000000..b4081e2b22d5
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-tm-vsx.c
@@ -0,0 +1,168 @@
+/*
+ * Ptrace test for VMX/VSX registers in the TM context
+ *
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#include "ptrace.h"
+#include "tm.h"
+#include "ptrace-vsx.h"
+
+int shm_id;
+unsigned long *cptr, *pptr;
+
+unsigned long fp_load[VEC_MAX];
+unsigned long fp_store[VEC_MAX];
+unsigned long fp_load_ckpt[VEC_MAX];
+unsigned long fp_load_ckpt_new[VEC_MAX];
+
+__attribute__((used)) void load_vsx(void)
+{
+	loadvsx(fp_load, 0);
+}
+
+__attribute__((used)) void load_vsx_ckpt(void)
+{
+	loadvsx(fp_load_ckpt, 0);
+}
+
+void tm_vsx(void)
+{
+	unsigned long result, texasr;
+	int ret;
+
+	cptr = (unsigned long *)shmat(shm_id, NULL, 0);
+
+trans:
+	cptr[1] = 0;
+	asm __volatile__(
+		"bl load_vsx_ckpt;"
+
+		"1: ;"
+		"tbegin.;"
+		"beq 2f;"
+
+		"bl load_vsx;"
+		"tsuspend.;"
+		"li 7, 1;"
+		"stw 7, 0(%[cptr1]);"
+		"tresume.;"
+		"b .;"
+
+		"tend.;"
+		"li 0, 0;"
+		"ori %[res], 0, 0;"
+		"b 3f;"
+
+		"2: ;"
+		"li 0, 1;"
+		"ori %[res], 0, 0;"
+		"mfspr %[texasr], %[sprn_texasr];"
+
+		"3: ;"
+		: [res] "=r" (result), [texasr] "=r" (texasr)
+		: [fp_load] "r" (fp_load), [fp_load_ckpt] "r" (fp_load_ckpt),
+		[sprn_texasr] "i"  (SPRN_TEXASR), [cptr1] "r" (&cptr[1])
+		: "memory", "r0", "r1", "r2", "r3", "r4",
+		"r7", "r8", "r9", "r10", "r11"
+		);
+
+	if (result) {
+		if (!cptr[0])
+			goto trans;
+
+		shmdt((void *)cptr);
+		storevsx(fp_store, 0);
+		ret = compare_vsx_vmx(fp_store, fp_load_ckpt_new);
+		if (ret)
+			exit(1);
+		exit(0);
+	}
+	shmdt((void *)cptr);
+	exit(1);
+}
+
+int trace_tm_vsx(pid_t child)
+{
+	unsigned long vsx[VSX_MAX];
+	unsigned long vmx[VMX_MAX + 2][2];
+
+	FAIL_IF(start_trace(child));
+	FAIL_IF(show_vsx(child, vsx));
+	FAIL_IF(validate_vsx(vsx, fp_load));
+	FAIL_IF(show_vmx(child, vmx));
+	FAIL_IF(validate_vmx(vmx, fp_load));
+	FAIL_IF(show_vsx_ckpt(child, vsx));
+	FAIL_IF(validate_vsx(vsx, fp_load_ckpt));
+	FAIL_IF(show_vmx_ckpt(child, vmx));
+	FAIL_IF(validate_vmx(vmx, fp_load_ckpt));
+	memset(vsx, 0, sizeof(vsx));
+	memset(vmx, 0, sizeof(vmx));
+
+	load_vsx_vmx(fp_load_ckpt_new, vsx, vmx);
+
+	FAIL_IF(write_vsx_ckpt(child, vsx));
+	FAIL_IF(write_vmx_ckpt(child, vmx));
+	pptr[0] = 1;
+	FAIL_IF(stop_trace(child));
+	return TEST_PASS;
+}
+
+int ptrace_tm_vsx(void)
+{
+	pid_t pid;
+	int ret, status, i;
+
+	SKIP_IF(!have_htm());
+	shm_id = shmget(IPC_PRIVATE, sizeof(int) * 2, 0777|IPC_CREAT);
+
+	for (i = 0; i < 128; i++) {
+		fp_load[i] = 1 + rand();
+		fp_load_ckpt[i] = 1 + 2 * rand();
+		fp_load_ckpt_new[i] = 1 + 3 * rand();
+	}
+
+	pid = fork();
+	if (pid < 0) {
+		perror("fork() failed");
+		return TEST_FAIL;
+	}
+
+	if (pid == 0)
+		tm_vsx();
+
+	if (pid) {
+		pptr = (unsigned long *)shmat(shm_id, NULL, 0);
+		while (!pptr[1])
+			asm volatile("" : : : "memory");
+
+		ret = trace_tm_vsx(pid);
+		if (ret) {
+			kill(pid, SIGKILL);
+			shmdt((void *)pptr);
+			shmctl(shm_id, IPC_RMID, NULL);
+			return TEST_FAIL;
+		}
+
+		shmdt((void *)pptr);
+		ret = wait(&status);
+		shmctl(shm_id, IPC_RMID, NULL);
+		if (ret != pid) {
+			printf("Child's exit status not captured\n");
+			return TEST_FAIL;
+		}
+
+		return (WIFEXITED(status) && WEXITSTATUS(status)) ? TEST_FAIL :
+			TEST_PASS;
+	}
+	return TEST_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	return test_harness(ptrace_tm_vsx, "ptrace_tm_vsx");
+}
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-vsx.c b/tools/testing/selftests/powerpc/ptrace/ptrace-vsx.c
new file mode 100644
index 000000000000..04084ee7d27b
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-vsx.c
@@ -0,0 +1,117 @@
+/*
+ * Ptrace test for VMX/VSX registers
+ *
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#include "ptrace.h"
+#include "ptrace-vsx.h"
+
+/* Tracer and Tracee Shared Data */
+int shm_id;
+int *cptr, *pptr;
+
+unsigned long fp_load[VEC_MAX];
+unsigned long fp_load_new[VEC_MAX];
+unsigned long fp_store[VEC_MAX];
+
+void vsx(void)
+{
+	int ret;
+
+	cptr = (int *)shmat(shm_id, NULL, 0);
+	loadvsx(fp_load, 0);
+	cptr[1] = 1;
+
+	while (!cptr[0])
+		asm volatile("" : : : "memory");
+	shmdt((void *) cptr);
+
+	storevsx(fp_store, 0);
+	ret = compare_vsx_vmx(fp_store, fp_load_new);
+	if (ret)
+		exit(1);
+	exit(0);
+}
+
+int trace_vsx(pid_t child)
+{
+	unsigned long vsx[VSX_MAX];
+	unsigned long vmx[VMX_MAX + 2][2];
+
+	FAIL_IF(start_trace(child));
+	FAIL_IF(show_vsx(child, vsx));
+	FAIL_IF(validate_vsx(vsx, fp_load));
+	FAIL_IF(show_vmx(child, vmx));
+	FAIL_IF(validate_vmx(vmx, fp_load));
+
+	memset(vsx, 0, sizeof(vsx));
+	memset(vmx, 0, sizeof(vmx));
+	load_vsx_vmx(fp_load_new, vsx, vmx);
+
+	FAIL_IF(write_vsx(child, vsx));
+	FAIL_IF(write_vmx(child, vmx));
+	FAIL_IF(stop_trace(child));
+
+	return TEST_PASS;
+}
+
+int ptrace_vsx(void)
+{
+	pid_t pid;
+	int ret, status, i;
+
+	shm_id = shmget(IPC_PRIVATE, sizeof(int) * 2, 0777|IPC_CREAT);
+
+	for (i = 0; i < VEC_MAX; i++)
+		fp_load[i] = i + rand();
+
+	for (i = 0; i < VEC_MAX; i++)
+		fp_load_new[i] = i + 2 * rand();
+
+	pid = fork();
+	if (pid < 0) {
+		perror("fork() failed");
+		return TEST_FAIL;
+	}
+
+	if (pid == 0)
+		vsx();
+
+	if (pid) {
+		pptr = (int *)shmat(shm_id, NULL, 0);
+		while (!pptr[1])
+			asm volatile("" : : : "memory");
+
+		ret = trace_vsx(pid);
+		if (ret) {
+			kill(pid, SIGTERM);
+			shmdt((void *)pptr);
+			shmctl(shm_id, IPC_RMID, NULL);
+			return TEST_FAIL;
+		}
+
+		pptr[0] = 1;
+		shmdt((void *)pptr);
+
+		ret = wait(&status);
+		shmctl(shm_id, IPC_RMID, NULL);
+		if (ret != pid) {
+			printf("Child's exit status not captured\n");
+			return TEST_FAIL;
+		}
+
+		return (WIFEXITED(status) && WEXITSTATUS(status)) ? TEST_FAIL :
+			TEST_PASS;
+	}
+	return TEST_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+	return test_harness(ptrace_vsx, "ptrace_vsx");
+}
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-vsx.h b/tools/testing/selftests/powerpc/ptrace/ptrace-vsx.h
new file mode 100644
index 000000000000..f4e4b427c9d9
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace-vsx.h
@@ -0,0 +1,127 @@
+/*
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#define VEC_MAX 128
+#define VSX_MAX 32
+#define VMX_MAX 32
+
+/*
+ * unsigned long vsx[32]
+ * unsigned long load[128]
+ */
+int validate_vsx(unsigned long *vsx, unsigned long *load)
+{
+	int i;
+
+	for (i = 0; i < VSX_MAX; i++) {
+		if (vsx[i] != load[2 * i + 1]) {
+			printf("vsx[%d]: %lx load[%d] %lx\n",
+					i, vsx[i], 2 * i + 1, load[2 * i + 1]);
+			return TEST_FAIL;
+		}
+	}
+	return TEST_PASS;
+}
+
+/*
+ * unsigned long vmx[32][2]
+ * unsigned long load[128]
+ */
+int validate_vmx(unsigned long vmx[][2], unsigned long *load)
+{
+	int i;
+
+	for (i = 0; i < VMX_MAX; i++) {
+		#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+		if ((vmx[i][0] != load[64 + 2 * i]) ||
+				(vmx[i][1] != load[65 + 2 * i])) {
+			printf("vmx[%d][0]: %lx load[%d] %lx\n",
+					i, vmx[i][0], 64 + 2 * i,
+					load[64 + 2 * i]);
+			printf("vmx[%d][1]: %lx load[%d] %lx\n",
+					i, vmx[i][1], 65 + 2 * i,
+					load[65 + 2 * i]);
+			return TEST_FAIL;
+		}
+		#else  /*
+			* In LE each value pair is stored in an
+			* alternate manner.
+			*/
+		if ((vmx[i][0] != load[65 + 2 * i]) ||
+				(vmx[i][1] != load[64 + 2 * i])) {
+			printf("vmx[%d][0]: %lx load[%d] %lx\n",
+					i, vmx[i][0], 65 + 2 * i,
+					load[65 + 2 * i]);
+			printf("vmx[%d][1]: %lx load[%d] %lx\n",
+					i, vmx[i][1], 64 + 2 * i,
+					load[64 + 2 * i]);
+			return TEST_FAIL;
+		}
+		#endif
+	}
+	return TEST_PASS;
+}
+
+/*
+ * unsigned long store[128]
+ * unsigned long load[128]
+ */
+int compare_vsx_vmx(unsigned long *store, unsigned long *load)
+{
+	int i;
+
+	for (i = 0; i < VSX_MAX; i++) {
+		if (store[1 + 2 * i] != load[1 + 2 * i]) {
+			printf("store[%d]: %lx load[%d] %lx\n",
+					1 + 2 * i, store[i],
+					1 + 2 * i, load[i]);
+			return TEST_FAIL;
+		}
+	}
+
+	#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	for (i = 64; i < VEC_MAX; i++) {
+		if (store[i] != load[i]) {
+			printf("store[%d]: %lx load[%d] %lx\n",
+					i, store[i], i, load[i]);
+			return TEST_FAIL;
+		}
+	}
+	#else	/* In LE each value pair is stored in an alternate manner */
+	for (i = 64; i < VEC_MAX; i++) {
+		if (!(i % 2) && (store[i] != load[i+1])) {
+			printf("store[%d]: %lx load[%d] %lx\n",
+					i, store[i], i+1, load[i+1]);
+			return TEST_FAIL;
+		}
+		if ((i % 2) && (store[i] != load[i-1])) {
+			printf("here store[%d]: %lx load[%d] %lx\n",
+					i, store[i], i-1, load[i-1]);
+			return TEST_FAIL;
+		}
+	}
+	#endif
+	return TEST_PASS;
+}
+
+void load_vsx_vmx(unsigned long *load, unsigned long *vsx,
+		unsigned long vmx[][2])
+{
+	int i;
+
+	for (i = 0; i < VSX_MAX; i++)
+		vsx[i] = load[1 + 2 * i];
+
+	for (i = 0; i < VMX_MAX; i++) {
+		vmx[i][0] = load[64 + 2 * i];
+		vmx[i][1] = load[65 + 2 * i];
+	}
+}
+
+void loadvsx(void *p, int tmp);
+void storevsx(void *p, int tmp);
diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace.h b/tools/testing/selftests/powerpc/ptrace/ptrace.h
new file mode 100644
index 000000000000..19fb825270a1
--- /dev/null
+++ b/tools/testing/selftests/powerpc/ptrace/ptrace.h
@@ -0,0 +1,711 @@
+/*
+ * Ptrace interface test helper functions
+ *
+ * Copyright (C) 2015 Anshuman Khandual, IBM Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#include <inttypes.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <malloc.h>
+#include <errno.h>
+#include <time.h>
+#include <sys/ptrace.h>
+#include <sys/ioctl.h>
+#include <sys/uio.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/signal.h>
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <sys/user.h>
+#include <linux/elf.h>
+#include <linux/types.h>
+#include <linux/auxvec.h>
+#include "reg.h"
+#include "utils.h"
+
+#define TEST_PASS 0
+#define TEST_FAIL 1
+
+struct fpr_regs {
+	unsigned long fpr[32];
+	unsigned long fpscr;
+};
+
+struct tm_spr_regs {
+	unsigned long tm_tfhar;
+	unsigned long tm_texasr;
+	unsigned long tm_tfiar;
+};
+
+#ifndef NT_PPC_TAR
+#define NT_PPC_TAR	0x103
+#define NT_PPC_PPR	0x104
+#define NT_PPC_DSCR	0x105
+#define NT_PPC_EBB	0x106
+#define NT_PPC_PMU	0x107
+#define NT_PPC_TM_CGPR	0x108
+#define NT_PPC_TM_CFPR	0x109
+#define NT_PPC_TM_CVMX	0x10a
+#define NT_PPC_TM_CVSX	0x10b
+#define NT_PPC_TM_SPR	0x10c
+#define NT_PPC_TM_CTAR	0x10d
+#define NT_PPC_TM_CPPR	0x10e
+#define NT_PPC_TM_CDSCR	0x10f
+#endif
+
+/* Basic ptrace operations */
+int start_trace(pid_t child)
+{
+	int ret;
+
+	ret = ptrace(PTRACE_ATTACH, child, NULL, NULL);
+	if (ret) {
+		perror("ptrace(PTRACE_ATTACH) failed");
+		return TEST_FAIL;
+	}
+	ret = waitpid(child, NULL, 0);
+	if (ret != child) {
+		perror("waitpid() failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+int stop_trace(pid_t child)
+{
+	int ret;
+
+	ret = ptrace(PTRACE_DETACH, child, NULL, NULL);
+	if (ret) {
+		perror("ptrace(PTRACE_DETACH) failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+int cont_trace(pid_t child)
+{
+	int ret;
+
+	ret = ptrace(PTRACE_CONT, child, NULL, NULL);
+	if (ret) {
+		perror("ptrace(PTRACE_CONT) failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+/* TAR, PPR, DSCR */
+int show_tar_registers(pid_t child, unsigned long *out)
+{
+	struct iovec iov;
+	unsigned long *reg;
+	int ret;
+
+	reg = malloc(sizeof(unsigned long));
+	if (!reg) {
+		perror("malloc() failed");
+		return TEST_FAIL;
+	}
+	iov.iov_base = (u64 *) reg;
+	iov.iov_len = sizeof(unsigned long);
+
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_TAR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		goto fail;
+	}
+	if (out)
+		out[0] = *reg;
+
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_PPR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		goto fail;
+	}
+	if (out)
+		out[1] = *reg;
+
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_DSCR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		goto fail;
+	}
+	if (out)
+		out[2] = *reg;
+
+	free(reg);
+	return TEST_PASS;
+fail:
+	free(reg);
+	return TEST_FAIL;
+}
+
+int write_tar_registers(pid_t child, unsigned long tar,
+		unsigned long ppr, unsigned long dscr)
+{
+	struct iovec iov;
+	unsigned long *reg;
+	int ret;
+
+	reg = malloc(sizeof(unsigned long));
+	if (!reg) {
+		perror("malloc() failed");
+		return TEST_FAIL;
+	}
+
+	iov.iov_base = (u64 *) reg;
+	iov.iov_len = sizeof(unsigned long);
+
+	*reg = tar;
+	ret = ptrace(PTRACE_SETREGSET, child, NT_PPC_TAR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_SETREGSET) failed");
+		goto fail;
+	}
+
+	*reg = ppr;
+	ret = ptrace(PTRACE_SETREGSET, child, NT_PPC_PPR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_SETREGSET) failed");
+		goto fail;
+	}
+
+	*reg = dscr;
+	ret = ptrace(PTRACE_SETREGSET, child, NT_PPC_DSCR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_SETREGSET) failed");
+		goto fail;
+	}
+
+	free(reg);
+	return TEST_PASS;
+fail:
+	free(reg);
+	return TEST_FAIL;
+}
+
+int show_tm_checkpointed_state(pid_t child, unsigned long *out)
+{
+	struct iovec iov;
+	unsigned long *reg;
+	int ret;
+
+	reg = malloc(sizeof(unsigned long));
+	if (!reg) {
+		perror("malloc() failed");
+		return TEST_FAIL;
+	}
+
+	iov.iov_base = (u64 *) reg;
+	iov.iov_len = sizeof(unsigned long);
+
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_TM_CTAR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		goto fail;
+	}
+	if (out)
+		out[0] = *reg;
+
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_TM_CPPR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		goto fail;
+	}
+	if (out)
+		out[1] = *reg;
+
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_TM_CDSCR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		goto fail;
+	}
+	if (out)
+		out[2] = *reg;
+
+	free(reg);
+	return TEST_PASS;
+
+fail:
+	free(reg);
+	return TEST_FAIL;
+}
+
+int write_ckpt_tar_registers(pid_t child, unsigned long tar,
+		unsigned long ppr, unsigned long dscr)
+{
+	struct iovec iov;
+	unsigned long *reg;
+	int ret;
+
+	reg = malloc(sizeof(unsigned long));
+	if (!reg) {
+		perror("malloc() failed");
+		return TEST_FAIL;
+	}
+
+	iov.iov_base = (u64 *) reg;
+	iov.iov_len = sizeof(unsigned long);
+
+	*reg = tar;
+	ret = ptrace(PTRACE_SETREGSET, child, NT_PPC_TM_CTAR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		goto fail;
+	}
+
+	*reg = ppr;
+	ret = ptrace(PTRACE_SETREGSET, child, NT_PPC_TM_CPPR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		goto fail;
+	}
+
+	*reg = dscr;
+	ret = ptrace(PTRACE_SETREGSET, child, NT_PPC_TM_CDSCR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		goto fail;
+	}
+
+	free(reg);
+	return TEST_PASS;
+fail:
+	free(reg);
+	return TEST_FAIL;
+}
+
+/* FPR */
+int show_fpr(pid_t child, unsigned long *fpr)
+{
+	struct fpr_regs *regs;
+	int ret, i;
+
+	regs = (struct fpr_regs *) malloc(sizeof(struct fpr_regs));
+	ret = ptrace(PTRACE_GETFPREGS, child, NULL, regs);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+
+	if (fpr) {
+		for (i = 0; i < 32; i++)
+			fpr[i] = regs->fpr[i];
+	}
+	return TEST_PASS;
+}
+
+int write_fpr(pid_t child, unsigned long val)
+{
+	struct fpr_regs *regs;
+	int ret, i;
+
+	regs = (struct fpr_regs *) malloc(sizeof(struct fpr_regs));
+	ret = ptrace(PTRACE_GETFPREGS, child, NULL, regs);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+
+	for (i = 0; i < 32; i++)
+		regs->fpr[i] = val;
+
+	ret = ptrace(PTRACE_SETFPREGS, child, NULL, regs);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+int show_ckpt_fpr(pid_t child, unsigned long *fpr)
+{
+	struct fpr_regs *regs;
+	struct iovec iov;
+	int ret, i;
+
+	regs = (struct fpr_regs *) malloc(sizeof(struct fpr_regs));
+	iov.iov_base = regs;
+	iov.iov_len = sizeof(struct fpr_regs);
+
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_TM_CFPR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+
+	if (fpr) {
+		for (i = 0; i < 32; i++)
+			fpr[i] = regs->fpr[i];
+	}
+
+	return TEST_PASS;
+}
+
+int write_ckpt_fpr(pid_t child, unsigned long val)
+{
+	struct fpr_regs *regs;
+	struct iovec iov;
+	int ret, i;
+
+	regs = (struct fpr_regs *) malloc(sizeof(struct fpr_regs));
+	iov.iov_base = regs;
+	iov.iov_len = sizeof(struct fpr_regs);
+
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_TM_CFPR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+
+	for (i = 0; i < 32; i++)
+		regs->fpr[i] = val;
+
+	ret = ptrace(PTRACE_SETREGSET, child, NT_PPC_TM_CFPR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+/* GPR */
+int show_gpr(pid_t child, unsigned long *gpr)
+{
+	struct pt_regs *regs;
+	int ret, i;
+
+	regs = (struct pt_regs *) malloc(sizeof(struct pt_regs));
+	if (!regs) {
+		perror("malloc() failed");
+		return TEST_FAIL;
+	}
+
+	ret = ptrace(PTRACE_GETREGS, child, NULL, regs);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+
+	if (gpr) {
+		for (i = 14; i < 32; i++)
+			gpr[i-14] = regs->gpr[i];
+	}
+
+	return TEST_PASS;
+}
+
+int write_gpr(pid_t child, unsigned long val)
+{
+	struct pt_regs *regs;
+	int i, ret;
+
+	regs = (struct pt_regs *) malloc(sizeof(struct pt_regs));
+	if (!regs) {
+		perror("malloc() failed");
+		return TEST_FAIL;
+	}
+
+	ret = ptrace(PTRACE_GETREGS, child, NULL, regs);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+
+	for (i = 14; i < 32; i++)
+		regs->gpr[i] = val;
+
+	ret = ptrace(PTRACE_SETREGS, child, NULL, regs);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+int show_ckpt_gpr(pid_t child, unsigned long *gpr)
+{
+	struct pt_regs *regs;
+	struct iovec iov;
+	int ret, i;
+
+	regs = (struct pt_regs *) malloc(sizeof(struct pt_regs));
+	if (!regs) {
+		perror("malloc() failed");
+		return TEST_FAIL;
+	}
+
+	iov.iov_base = (u64 *) regs;
+	iov.iov_len = sizeof(struct pt_regs);
+
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_TM_CGPR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+
+	if (gpr) {
+		for (i = 14; i < 32; i++)
+			gpr[i-14] = regs->gpr[i];
+	}
+
+	return TEST_PASS;
+}
+
+int write_ckpt_gpr(pid_t child, unsigned long val)
+{
+	struct pt_regs *regs;
+	struct iovec iov;
+	int ret, i;
+
+	regs = (struct pt_regs *) malloc(sizeof(struct pt_regs));
+	if (!regs) {
+		perror("malloc() failed\n");
+		return TEST_FAIL;
+	}
+	iov.iov_base = (u64 *) regs;
+	iov.iov_len = sizeof(struct pt_regs);
+
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_TM_CGPR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+
+	for (i = 14; i < 32; i++)
+		regs->gpr[i] = val;
+
+	ret = ptrace(PTRACE_SETREGSET, child, NT_PPC_TM_CGPR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+/* VMX */
+int show_vmx(pid_t child, unsigned long vmx[][2])
+{
+	int ret;
+
+	ret = ptrace(PTRACE_GETVRREGS, child, 0, vmx);
+	if (ret) {
+		perror("ptrace(PTRACE_GETVRREGS) failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+int show_vmx_ckpt(pid_t child, unsigned long vmx[][2])
+{
+	unsigned long regs[34][2];
+	struct iovec iov;
+	int ret;
+
+	iov.iov_base = (u64 *) regs;
+	iov.iov_len = sizeof(regs);
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_TM_CVMX, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET, NT_PPC_TM_CVMX) failed");
+		return TEST_FAIL;
+	}
+	memcpy(vmx, regs, sizeof(regs));
+	return TEST_PASS;
+}
+
+
+int write_vmx(pid_t child, unsigned long vmx[][2])
+{
+	int ret;
+
+	ret = ptrace(PTRACE_SETVRREGS, child, 0, vmx);
+	if (ret) {
+		perror("ptrace(PTRACE_SETVRREGS) failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+int write_vmx_ckpt(pid_t child, unsigned long vmx[][2])
+{
+	unsigned long regs[34][2];
+	struct iovec iov;
+	int ret;
+
+	memcpy(regs, vmx, sizeof(regs));
+	iov.iov_base = (u64 *) regs;
+	iov.iov_len = sizeof(regs);
+	ret = ptrace(PTRACE_SETREGSET, child, NT_PPC_TM_CVMX, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_SETREGSET, NT_PPC_TM_CVMX) failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+/* VSX */
+int show_vsx(pid_t child, unsigned long *vsx)
+{
+	int ret;
+
+	ret = ptrace(PTRACE_GETVSRREGS, child, 0, vsx);
+	if (ret) {
+		perror("ptrace(PTRACE_GETVSRREGS) failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+int show_vsx_ckpt(pid_t child, unsigned long *vsx)
+{
+	unsigned long regs[32];
+	struct iovec iov;
+	int ret;
+
+	iov.iov_base = (u64 *) regs;
+	iov.iov_len = sizeof(regs);
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_TM_CVSX, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET, NT_PPC_TM_CVSX) failed");
+		return TEST_FAIL;
+	}
+	memcpy(vsx, regs, sizeof(regs));
+	return TEST_PASS;
+}
+
+int write_vsx(pid_t child, unsigned long *vsx)
+{
+	int ret;
+
+	ret = ptrace(PTRACE_SETVSRREGS, child, 0, vsx);
+	if (ret) {
+		perror("ptrace(PTRACE_SETVSRREGS) failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+int write_vsx_ckpt(pid_t child, unsigned long *vsx)
+{
+	unsigned long regs[32];
+	struct iovec iov;
+	int ret;
+
+	memcpy(regs, vsx, sizeof(regs));
+	iov.iov_base = (u64 *) regs;
+	iov.iov_len = sizeof(regs);
+	ret = ptrace(PTRACE_SETREGSET, child, NT_PPC_TM_CVSX, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_SETREGSET, NT_PPC_TM_CVSX) failed");
+		return TEST_FAIL;
+	}
+	return TEST_PASS;
+}
+
+/* TM SPR */
+int show_tm_spr(pid_t child, struct tm_spr_regs *out)
+{
+	struct tm_spr_regs *regs;
+	struct iovec iov;
+	int ret;
+
+	regs = (struct tm_spr_regs *) malloc(sizeof(struct tm_spr_regs));
+	if (!regs) {
+		perror("malloc() failed");
+		return TEST_FAIL;
+	}
+
+	iov.iov_base = (u64 *) regs;
+	iov.iov_len = sizeof(struct tm_spr_regs);
+
+	ret = ptrace(PTRACE_GETREGSET, child, NT_PPC_TM_SPR, &iov);
+	if (ret) {
+		perror("ptrace(PTRACE_GETREGSET) failed");
+		return TEST_FAIL;
+	}
+
+	if (out)
+		memcpy(out, regs, sizeof(struct tm_spr_regs));
+
+	return TEST_PASS;
+}
+
+
+
+/* Analyse TEXASR after TM failure */
+inline unsigned long get_tfiar(void)
+{
+	unsigned long ret;
+
+	asm volatile("mfspr %0,%1" : "=r" (ret) : "i" (SPRN_TFIAR));
+	return ret;
+}
+
+void analyse_texasr(unsigned long texasr)
+{
+	printf("TEXASR: %16lx\t", texasr);
+
+	if (texasr & TEXASR_FP)
+		printf("TEXASR_FP  ");
+
+	if (texasr & TEXASR_DA)
+		printf("TEXASR_DA  ");
+
+	if (texasr & TEXASR_NO)
+		printf("TEXASR_NO  ");
+
+	if (texasr & TEXASR_FO)
+		printf("TEXASR_FO  ");
+
+	if (texasr & TEXASR_SIC)
+		printf("TEXASR_SIC  ");
+
+	if (texasr & TEXASR_NTC)
+		printf("TEXASR_NTC  ");
+
+	if (texasr & TEXASR_TC)
+		printf("TEXASR_TC  ");
+
+	if (texasr & TEXASR_TIC)
+		printf("TEXASR_TIC  ");
+
+	if (texasr & TEXASR_IC)
+		printf("TEXASR_IC  ");
+
+	if (texasr & TEXASR_IFC)
+		printf("TEXASR_IFC  ");
+
+	if (texasr & TEXASR_ABT)
+		printf("TEXASR_ABT  ");
+
+	if (texasr & TEXASR_SPD)
+		printf("TEXASR_SPD  ");
+
+	if (texasr & TEXASR_HV)
+		printf("TEXASR_HV  ");
+
+	if (texasr & TEXASR_PR)
+		printf("TEXASR_PR  ");
+
+	if (texasr & TEXASR_FS)
+		printf("TEXASR_FS  ");
+
+	if (texasr & TEXASR_TE)
+		printf("TEXASR_TE  ");
+
+	if (texasr & TEXASR_ROT)
+		printf("TEXASR_ROT  ");
+
+	printf("TFIAR :%lx\n", get_tfiar());
+}
+
+void store_gpr(unsigned long *addr);
+void store_fpr(float *addr);
diff --git a/tools/testing/selftests/powerpc/reg.h b/tools/testing/selftests/powerpc/reg.h
deleted file mode 100644
index fddf368ed82f..000000000000
--- a/tools/testing/selftests/powerpc/reg.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2014, Michael Ellerman, IBM Corp.
- * Licensed under GPLv2.
- */
-
-#ifndef _SELFTESTS_POWERPC_REG_H
-#define _SELFTESTS_POWERPC_REG_H
-
-#define __stringify_1(x)        #x
-#define __stringify(x)          __stringify_1(x)
-
-#define mfspr(rn)	({unsigned long rval; \
-			 asm volatile("mfspr %0," _str(rn) \
-				    : "=r" (rval)); rval; })
-#define mtspr(rn, v)	asm volatile("mtspr " _str(rn) ",%0" : \
-				    : "r" ((unsigned long)(v)) \
-				    : "memory")
-
-#define mb()		asm volatile("sync" : : : "memory");
-
-#define SPRN_MMCR2     769
-#define SPRN_MMCRA     770
-#define SPRN_MMCR0     779
-#define   MMCR0_PMAO   0x00000080
-#define   MMCR0_PMAE   0x04000000
-#define   MMCR0_FC     0x80000000
-#define SPRN_EBBHR     804
-#define SPRN_EBBRR     805
-#define SPRN_BESCR     806     /* Branch event status & control register */
-#define SPRN_BESCRS    800     /* Branch event status & control set (1 bits set to 1) */
-#define SPRN_BESCRSU   801     /* Branch event status & control set upper */
-#define SPRN_BESCRR    802     /* Branch event status & control REset (1 bits set to 0) */
-#define SPRN_BESCRRU   803     /* Branch event status & control REset upper */
-
-#define BESCR_PMEO     0x1     /* PMU Event-based exception Occurred */
-#define BESCR_PME      (0x1ul << 32) /* PMU Event-based exception Enable */
-#define BESCR_LME      (0x1ul << 34) /* Load Monitor Enable */
-#define BESCR_LMEO     (0x1ul << 2)  /* Load Monitor Exception Occurred */
-
-#define SPRN_LMRR      813     /* Load Monitor Region Register */
-#define SPRN_LMSER     814     /* Load Monitor Section Enable Register */
-
-#define SPRN_PMC1      771
-#define SPRN_PMC2      772
-#define SPRN_PMC3      773
-#define SPRN_PMC4      774
-#define SPRN_PMC5      775
-#define SPRN_PMC6      776
-
-#define SPRN_SIAR      780
-#define SPRN_SDAR      781
-#define SPRN_SIER      768
-
-#define SPRN_TEXASR     0x82
-#define SPRN_TFIAR      0x81    /* Transaction Failure Inst Addr    */
-#define SPRN_TFHAR      0x80    /* Transaction Failure Handler Addr */
-#define TEXASR_FS       0x08000000
-#define SPRN_TAR        0x32f
-
-#endif /* _SELFTESTS_POWERPC_REG_H */
diff --git a/tools/testing/selftests/powerpc/signal/signal.S b/tools/testing/selftests/powerpc/signal/signal.S
index 7043d521df0a..322f2f1fc327 100644
--- a/tools/testing/selftests/powerpc/signal/signal.S
+++ b/tools/testing/selftests/powerpc/signal/signal.S
@@ -7,7 +7,7 @@
  * 2 of the License, or (at your option) any later version.
  */
 
-#include "../basic_asm.h"
+#include "basic_asm.h"
 
 /* long signal_self(pid_t pid, int sig); */
 FUNC_START(signal_self)
diff --git a/tools/testing/selftests/powerpc/stringloops/memcmp.c b/tools/testing/selftests/powerpc/stringloops/memcmp.c
index 17417dd70708..30b1222380ca 100644
--- a/tools/testing/selftests/powerpc/stringloops/memcmp.c
+++ b/tools/testing/selftests/powerpc/stringloops/memcmp.c
@@ -1,7 +1,7 @@
 #include <malloc.h>
 #include <stdlib.h>
 #include <string.h>
-#include "../utils.h"
+#include "utils.h"
 
 #define SIZE 256
 #define ITERATIONS 10000
diff --git a/tools/testing/selftests/powerpc/tm/tm-signal.S b/tools/testing/selftests/powerpc/tm/tm-signal.S
index 4e13e8b3a96f..506a4ebaf3ae 100644
--- a/tools/testing/selftests/powerpc/tm/tm-signal.S
+++ b/tools/testing/selftests/powerpc/tm/tm-signal.S
@@ -7,11 +7,11 @@
  * 2 of the License, or (at your option) any later version.
  */
 
-#include "../basic_asm.h"
-#include "../gpr_asm.h"
-#include "../fpu_asm.h"
-#include "../vmx_asm.h"
-#include "../vsx_asm.h"
+#include "basic_asm.h"
+#include "gpr_asm.h"
+#include "fpu_asm.h"
+#include "vmx_asm.h"
+#include "vsx_asm.h"
 
 /*
  * Large caveat here being that the caller cannot expect the
diff --git a/tools/testing/selftests/powerpc/tm/tm.h b/tools/testing/selftests/powerpc/tm/tm.h
index 2c8da74304e7..0ffff04433c5 100644
--- a/tools/testing/selftests/powerpc/tm/tm.h
+++ b/tools/testing/selftests/powerpc/tm/tm.h
@@ -10,7 +10,7 @@
 #include <asm/cputable.h>
 #include <stdbool.h>
 
-#include "../utils.h"
+#include "utils.h"
 
 static inline bool have_htm(void)
 {
diff --git a/tools/testing/selftests/sigaltstack/.gitignore b/tools/testing/selftests/sigaltstack/.gitignore
new file mode 100644
index 000000000000..35897b0a3f44
--- /dev/null
+++ b/tools/testing/selftests/sigaltstack/.gitignore
@@ -0,0 +1 @@
+sas
diff --git a/tools/testing/selftests/sync/.gitignore b/tools/testing/selftests/sync/.gitignore
new file mode 100644
index 000000000000..f5091e7792f2
--- /dev/null
+++ b/tools/testing/selftests/sync/.gitignore
@@ -0,0 +1 @@
+sync_test
diff --git a/tools/testing/selftests/sync/Makefile b/tools/testing/selftests/sync/Makefile
new file mode 100644
index 000000000000..87ac400507c0
--- /dev/null
+++ b/tools/testing/selftests/sync/Makefile
@@ -0,0 +1,24 @@
+CFLAGS += -O2 -g -std=gnu89 -pthread -Wall -Wextra
+CFLAGS += -I../../../../usr/include/
+LDFLAGS += -pthread
+
+TEST_PROGS = sync_test
+
+all: $(TEST_PROGS)
+
+include ../lib.mk
+
+OBJS = sync_test.o sync.o
+
+TESTS += sync_alloc.o
+TESTS += sync_fence.o
+TESTS += sync_merge.o
+TESTS += sync_wait.o
+TESTS += sync_stress_parallelism.o
+TESTS += sync_stress_consumer.o
+TESTS += sync_stress_merge.o
+
+sync_test: $(OBJS) $(TESTS)
+
+clean:
+	$(RM) sync_test $(OBJS) $(TESTS)
diff --git a/tools/testing/selftests/sync/sw_sync.h b/tools/testing/selftests/sync/sw_sync.h
new file mode 100644
index 000000000000..e2cfc6bad83e
--- /dev/null
+++ b/tools/testing/selftests/sync/sw_sync.h
@@ -0,0 +1,46 @@
+/*
+ *  sw_sync abstraction
+ *
+ *  Copyright 2015-2016 Collabora Ltd.
+ *
+ *  Based on the implementation from the Android Open Source Project,
+ *
+ *  Copyright 2013 Google, Inc
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ *  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *  OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SELFTESTS_SW_SYNC_H
+#define SELFTESTS_SW_SYNC_H
+
+/*
+ * sw_sync is mainly intended for testing and should not be compiled into
+ * production kernels
+ */
+
+int sw_sync_timeline_create(void);
+int sw_sync_timeline_is_valid(int fd);
+int sw_sync_timeline_inc(int fd, unsigned int count);
+void sw_sync_timeline_destroy(int fd);
+
+int sw_sync_fence_create(int fd, const char *name, unsigned int value);
+int sw_sync_fence_is_valid(int fd);
+void sw_sync_fence_destroy(int fd);
+
+#endif
diff --git a/tools/testing/selftests/sync/sync.c b/tools/testing/selftests/sync/sync.c
new file mode 100644
index 000000000000..f3d599f249b9
--- /dev/null
+++ b/tools/testing/selftests/sync/sync.c
@@ -0,0 +1,221 @@
+/*
+ *  sync / sw_sync abstraction
+ *  Copyright 2015-2016 Collabora Ltd.
+ *
+ *  Based on the implementation from the Android Open Source Project,
+ *
+ *  Copyright 2012 Google, Inc
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ *  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *  OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <fcntl.h>
+#include <malloc.h>
+#include <poll.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "sync.h"
+#include "sw_sync.h"
+
+#include <linux/sync_file.h>
+
+
+/* SW_SYNC ioctls */
+struct sw_sync_create_fence_data {
+	__u32	value;
+	char	name[32];
+	__s32	fence;
+};
+
+#define SW_SYNC_IOC_MAGIC		'W'
+#define SW_SYNC_IOC_CREATE_FENCE	_IOWR(SW_SYNC_IOC_MAGIC, 0,\
+					      struct sw_sync_create_fence_data)
+#define SW_SYNC_IOC_INC			_IOW(SW_SYNC_IOC_MAGIC, 1, __u32)
+
+
+int sync_wait(int fd, int timeout)
+{
+	struct pollfd fds;
+
+	fds.fd = fd;
+	fds.events = POLLIN | POLLERR;
+
+	return poll(&fds, 1, timeout);
+}
+
+int sync_merge(const char *name, int fd1, int fd2)
+{
+	struct sync_merge_data data = {};
+	int err;
+
+	data.fd2 = fd2;
+	strncpy(data.name, name, sizeof(data.name) - 1);
+	data.name[sizeof(data.name) - 1] = '\0';
+
+	err = ioctl(fd1, SYNC_IOC_MERGE, &data);
+	if (err < 0)
+		return err;
+
+	return data.fence;
+}
+
+static struct sync_file_info *sync_file_info(int fd)
+{
+	struct sync_file_info *info;
+	struct sync_fence_info *fence_info;
+	int err, num_fences;
+
+	info = calloc(1, sizeof(*info));
+	if (info == NULL)
+		return NULL;
+
+	err = ioctl(fd, SYNC_IOC_FILE_INFO, info);
+	if (err < 0) {
+		free(info);
+		return NULL;
+	}
+
+	num_fences = info->num_fences;
+
+	if (num_fences) {
+		info->flags = 0;
+		info->num_fences = num_fences;
+
+		fence_info = calloc(num_fences, sizeof(*fence_info));
+		if (!fence_info) {
+			free(info);
+			return NULL;
+		}
+
+		info->sync_fence_info = (uint64_t)fence_info;
+
+		err = ioctl(fd, SYNC_IOC_FILE_INFO, info);
+		if (err < 0) {
+			free(fence_info);
+			free(info);
+			return NULL;
+		}
+	}
+
+	return info;
+}
+
+static void sync_file_info_free(struct sync_file_info *info)
+{
+	free((void *)info->sync_fence_info);
+	free(info);
+}
+
+int sync_fence_size(int fd)
+{
+	int count;
+	struct sync_file_info *info = sync_file_info(fd);
+
+	if (!info)
+		return 0;
+
+	count = info->num_fences;
+
+	sync_file_info_free(info);
+
+	return count;
+}
+
+int sync_fence_count_with_status(int fd, int status)
+{
+	unsigned int i, count = 0;
+	struct sync_fence_info *fence_info = NULL;
+	struct sync_file_info *info = sync_file_info(fd);
+
+	if (!info)
+		return -1;
+
+	fence_info = (struct sync_fence_info *)info->sync_fence_info;
+	for (i = 0 ; i < info->num_fences ; i++) {
+		if (fence_info[i].status == status)
+			count++;
+	}
+
+	sync_file_info_free(info);
+
+	return count;
+}
+
+int sw_sync_timeline_create(void)
+{
+	return open("/sys/kernel/debug/sync/sw_sync", O_RDWR);
+}
+
+int sw_sync_timeline_inc(int fd, unsigned int count)
+{
+	__u32 arg = count;
+
+	return ioctl(fd, SW_SYNC_IOC_INC, &arg);
+}
+
+int sw_sync_timeline_is_valid(int fd)
+{
+	int status;
+
+	if (fd == -1)
+		return 0;
+
+	status = fcntl(fd, F_GETFD, 0);
+	return (status >= 0);
+}
+
+void sw_sync_timeline_destroy(int fd)
+{
+	if (sw_sync_timeline_is_valid(fd))
+		close(fd);
+}
+
+int sw_sync_fence_create(int fd, const char *name, unsigned int value)
+{
+	struct sw_sync_create_fence_data data = {};
+	int err;
+
+	data.value = value;
+	strncpy(data.name, name, sizeof(data.name) - 1);
+	data.name[sizeof(data.name) - 1] = '\0';
+
+	err = ioctl(fd, SW_SYNC_IOC_CREATE_FENCE, &data);
+	if (err < 0)
+		return err;
+
+	return data.fence;
+}
+
+int sw_sync_fence_is_valid(int fd)
+{
+	/* Same code! */
+	return sw_sync_timeline_is_valid(fd);
+}
+
+void sw_sync_fence_destroy(int fd)
+{
+	if (sw_sync_fence_is_valid(fd))
+		close(fd);
+}
diff --git a/tools/testing/selftests/sync/sync.h b/tools/testing/selftests/sync/sync.h
new file mode 100644
index 000000000000..fb7156148350
--- /dev/null
+++ b/tools/testing/selftests/sync/sync.h
@@ -0,0 +1,40 @@
+/*
+ *  sync abstraction
+ *  Copyright 2015-2016 Collabora Ltd.
+ *
+ *  Based on the implementation from the Android Open Source Project,
+ *
+ *  Copyright 2012 Google, Inc
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ *  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *  OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SELFTESTS_SYNC_H
+#define SELFTESTS_SYNC_H
+
+#define FENCE_STATUS_ERROR	(-1)
+#define FENCE_STATUS_ACTIVE	(0)
+#define FENCE_STATUS_SIGNALED	(1)
+
+int sync_wait(int fd, int timeout);
+int sync_merge(const char *name, int fd1, int fd2);
+int sync_fence_size(int fd);
+int sync_fence_count_with_status(int fd, int status);
+
+#endif
diff --git a/tools/testing/selftests/sync/sync_alloc.c b/tools/testing/selftests/sync/sync_alloc.c
new file mode 100644
index 000000000000..66a28afc05dc
--- /dev/null
+++ b/tools/testing/selftests/sync/sync_alloc.c
@@ -0,0 +1,74 @@
+/*
+ *  sync allocation tests
+ *  Copyright 2015-2016 Collabora Ltd.
+ *
+ *  Based on the implementation from the Android Open Source Project,
+ *
+ *  Copyright 2012 Google, Inc
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ *  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *  OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "sync.h"
+#include "sw_sync.h"
+#include "synctest.h"
+
+int test_alloc_timeline(void)
+{
+	int timeline, valid;
+
+	timeline = sw_sync_timeline_create();
+	valid = sw_sync_timeline_is_valid(timeline);
+	ASSERT(valid, "Failure allocating timeline\n");
+
+	sw_sync_timeline_destroy(timeline);
+	return 0;
+}
+
+int test_alloc_fence(void)
+{
+	int timeline, fence, valid;
+
+	timeline = sw_sync_timeline_create();
+	valid = sw_sync_timeline_is_valid(timeline);
+	ASSERT(valid, "Failure allocating timeline\n");
+
+	fence = sw_sync_fence_create(timeline, "allocFence", 1);
+	valid = sw_sync_fence_is_valid(fence);
+	ASSERT(valid, "Failure allocating fence\n");
+
+	sw_sync_fence_destroy(fence);
+	sw_sync_timeline_destroy(timeline);
+	return 0;
+}
+
+int test_alloc_fence_negative(void)
+{
+	int fence, timeline;
+
+	timeline = sw_sync_timeline_create();
+	ASSERT(timeline > 0, "Failure allocating timeline\n");
+
+	fence = sw_sync_fence_create(-1, "fence", 1);
+	ASSERT(fence < 0, "Success allocating negative fence\n");
+
+	sw_sync_fence_destroy(fence);
+	sw_sync_timeline_destroy(timeline);
+	return 0;
+}
diff --git a/tools/testing/selftests/sync/sync_fence.c b/tools/testing/selftests/sync/sync_fence.c
new file mode 100644
index 000000000000..13f175287da3
--- /dev/null
+++ b/tools/testing/selftests/sync/sync_fence.c
@@ -0,0 +1,132 @@
+/*
+ *  sync fence tests with one timeline
+ *  Copyright 2015-2016 Collabora Ltd.
+ *
+ *  Based on the implementation from the Android Open Source Project,
+ *
+ *  Copyright 2012 Google, Inc
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ *  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *  OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "sync.h"
+#include "sw_sync.h"
+#include "synctest.h"
+
+int test_fence_one_timeline_wait(void)
+{
+	int fence, valid, ret;
+	int timeline = sw_sync_timeline_create();
+
+	valid = sw_sync_timeline_is_valid(timeline);
+	ASSERT(valid, "Failure allocating timeline\n");
+
+	fence = sw_sync_fence_create(timeline, "allocFence", 5);
+	valid = sw_sync_fence_is_valid(fence);
+	ASSERT(valid, "Failure allocating fence\n");
+
+	/* Wait on fence until timeout */
+	ret = sync_wait(fence, 0);
+	ASSERT(ret == 0, "Failure waiting on fence until timeout\n");
+
+	/* Advance timeline from 0 -> 1 */
+	ret = sw_sync_timeline_inc(timeline, 1);
+	ASSERT(ret == 0, "Failure advancing timeline\n");
+
+	/* Wait on fence until timeout */
+	ret = sync_wait(fence, 0);
+	ASSERT(ret == 0, "Failure waiting on fence until timeout\n");
+
+	/* Signal the fence */
+	ret = sw_sync_timeline_inc(timeline, 4);
+	ASSERT(ret == 0, "Failure signaling the fence\n");
+
+	/* Wait successfully */
+	ret = sync_wait(fence, 0);
+	ASSERT(ret > 0, "Failure waiting on fence\n");
+
+	/* Go even further, and confirm wait still succeeds */
+	ret = sw_sync_timeline_inc(timeline, 10);
+	ASSERT(ret == 0, "Failure going further\n");
+	ret = sync_wait(fence, 0);
+	ASSERT(ret > 0, "Failure waiting ahead\n");
+
+	sw_sync_fence_destroy(fence);
+	sw_sync_timeline_destroy(timeline);
+
+	return 0;
+}
+
+int test_fence_one_timeline_merge(void)
+{
+	int a, b, c, d, valid;
+	int timeline = sw_sync_timeline_create();
+
+	/* create fence a,b,c and then merge them all into fence d */
+	a = sw_sync_fence_create(timeline, "allocFence", 1);
+	b = sw_sync_fence_create(timeline, "allocFence", 2);
+	c = sw_sync_fence_create(timeline, "allocFence", 3);
+
+	valid = sw_sync_fence_is_valid(a) &&
+		sw_sync_fence_is_valid(b) &&
+		sw_sync_fence_is_valid(c);
+	ASSERT(valid, "Failure allocating fences\n");
+
+	d = sync_merge("mergeFence", b, a);
+	d = sync_merge("mergeFence", c, d);
+	valid = sw_sync_fence_is_valid(d);
+	ASSERT(valid, "Failure merging fences\n");
+
+	/* confirm all fences have one active point (even d) */
+	ASSERT(sync_fence_count_with_status(a, FENCE_STATUS_ACTIVE) == 1,
+	       "a has too many active fences!\n");
+	ASSERT(sync_fence_count_with_status(a, FENCE_STATUS_ACTIVE) == 1,
+	       "b has too many active fences!\n");
+	ASSERT(sync_fence_count_with_status(a, FENCE_STATUS_ACTIVE) == 1,
+	       "c has too many active fences!\n");
+	ASSERT(sync_fence_count_with_status(a, FENCE_STATUS_ACTIVE) == 1,
+	       "d has too many active fences!\n");
+
+	/* confirm that d is not signaled until the max of a,b,c */
+	sw_sync_timeline_inc(timeline, 1);
+	ASSERT(sync_fence_count_with_status(a, FENCE_STATUS_SIGNALED) == 1,
+	       "a did not signal!\n");
+	ASSERT(sync_fence_count_with_status(d, FENCE_STATUS_ACTIVE) == 1,
+	       "d signaled too early!\n");
+
+	sw_sync_timeline_inc(timeline, 1);
+	ASSERT(sync_fence_count_with_status(b, FENCE_STATUS_SIGNALED) == 1,
+	       "b did not signal!\n");
+	ASSERT(sync_fence_count_with_status(d, FENCE_STATUS_ACTIVE) == 1,
+	       "d signaled too early!\n");
+
+	sw_sync_timeline_inc(timeline, 1);
+	ASSERT(sync_fence_count_with_status(c, FENCE_STATUS_SIGNALED) == 1,
+	       "c did not signal!\n");
+	ASSERT(sync_fence_count_with_status(d, FENCE_STATUS_ACTIVE) == 0 &&
+	       sync_fence_count_with_status(d, FENCE_STATUS_SIGNALED) == 1,
+	       "d did not signal!\n");
+
+	sw_sync_fence_destroy(d);
+	sw_sync_fence_destroy(c);
+	sw_sync_fence_destroy(b);
+	sw_sync_fence_destroy(a);
+	sw_sync_timeline_destroy(timeline);
+	return 0;
+}
diff --git a/tools/testing/selftests/sync/sync_merge.c b/tools/testing/selftests/sync/sync_merge.c
new file mode 100644
index 000000000000..8914d43395c7
--- /dev/null
+++ b/tools/testing/selftests/sync/sync_merge.c
@@ -0,0 +1,60 @@
+/*
+ *  sync fence merge tests
+ *  Copyright 2015-2016 Collabora Ltd.
+ *
+ *  Based on the implementation from the Android Open Source Project,
+ *
+ *  Copyright 2012 Google, Inc
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ *  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *  OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "sync.h"
+#include "sw_sync.h"
+#include "synctest.h"
+
+int test_fence_merge_same_fence(void)
+{
+	int fence, valid, merged;
+	int timeline = sw_sync_timeline_create();
+
+	valid = sw_sync_timeline_is_valid(timeline);
+	ASSERT(valid, "Failure allocating timeline\n");
+
+	fence = sw_sync_fence_create(timeline, "allocFence", 5);
+	valid = sw_sync_fence_is_valid(fence);
+	ASSERT(valid, "Failure allocating fence\n");
+
+	merged = sync_merge("mergeFence", fence, fence);
+	valid = sw_sync_fence_is_valid(fence);
+	ASSERT(valid, "Failure merging fence\n");
+
+	ASSERT(sync_fence_count_with_status(merged, FENCE_STATUS_SIGNALED) == 0,
+	       "fence signaled too early!\n");
+
+	sw_sync_timeline_inc(timeline, 5);
+	ASSERT(sync_fence_count_with_status(merged, FENCE_STATUS_SIGNALED) == 1,
+	       "fence did not signal!\n");
+
+	sw_sync_fence_destroy(merged);
+	sw_sync_fence_destroy(fence);
+	sw_sync_timeline_destroy(timeline);
+
+	return 0;
+}
diff --git a/tools/testing/selftests/sync/sync_stress_consumer.c b/tools/testing/selftests/sync/sync_stress_consumer.c
new file mode 100644
index 000000000000..d9eff8d524f7
--- /dev/null
+++ b/tools/testing/selftests/sync/sync_stress_consumer.c
@@ -0,0 +1,185 @@
+/*
+ *  sync stress test: producer/consumer
+ *  Copyright 2015-2016 Collabora Ltd.
+ *
+ *  Based on the implementation from the Android Open Source Project,
+ *
+ *  Copyright 2012 Google, Inc
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ *  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *  OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <pthread.h>
+
+#include "sync.h"
+#include "sw_sync.h"
+#include "synctest.h"
+
+/* IMPORTANT NOTE: if you see this test failing on your system, it may be
+ * due to a shortage of file descriptors. Please ensure your system has
+ * a sensible limit for this test to finish correctly.
+ */
+
+/* Returns 1 on error, 0 on success */
+static int busy_wait_on_fence(int fence)
+{
+	int error, active;
+
+	do {
+		error = sync_fence_count_with_status(fence, FENCE_STATUS_ERROR);
+		ASSERT(error == 0, "Error occurred on fence\n");
+		active = sync_fence_count_with_status(fence,
+						      FENCE_STATUS_ACTIVE);
+	} while (active);
+
+	return 0;
+}
+
+static struct {
+	int iterations;
+	int threads;
+	int counter;
+	int consumer_timeline;
+	int *producer_timelines;
+	pthread_mutex_t lock;
+} test_data_mpsc;
+
+static int mpsc_producer_thread(void *d)
+{
+	int id = (long)d;
+	int fence, valid, i;
+	int *producer_timelines = test_data_mpsc.producer_timelines;
+	int consumer_timeline = test_data_mpsc.consumer_timeline;
+	int iterations = test_data_mpsc.iterations;
+
+	for (i = 0; i < iterations; i++) {
+		fence = sw_sync_fence_create(consumer_timeline, "fence", i);
+		valid = sw_sync_fence_is_valid(fence);
+		ASSERT(valid, "Failure creating fence\n");
+
+		/*
+		 * Wait for the consumer to finish. Use alternate
+		 * means of waiting on the fence
+		 */
+
+		if ((iterations + id) % 8 != 0) {
+			ASSERT(sync_wait(fence, -1) > 0,
+			       "Failure waiting on fence\n");
+		} else {
+			ASSERT(busy_wait_on_fence(fence) == 0,
+			       "Failure waiting on fence\n");
+		}
+
+		/*
+		 * Every producer increments the counter, the consumer
+		 * checks and erases it
+		 */
+		pthread_mutex_lock(&test_data_mpsc.lock);
+		test_data_mpsc.counter++;
+		pthread_mutex_unlock(&test_data_mpsc.lock);
+
+		ASSERT(sw_sync_timeline_inc(producer_timelines[id], 1) == 0,
+		       "Error advancing producer timeline\n");
+
+		sw_sync_fence_destroy(fence);
+	}
+
+	return 0;
+}
+
+static int mpcs_consumer_thread(void)
+{
+	int fence, merged, tmp, valid, it, i;
+	int *producer_timelines = test_data_mpsc.producer_timelines;
+	int consumer_timeline = test_data_mpsc.consumer_timeline;
+	int iterations = test_data_mpsc.iterations;
+	int n = test_data_mpsc.threads;
+
+	for (it = 1; it <= iterations; it++) {
+		fence = sw_sync_fence_create(producer_timelines[0], "name", it);
+		for (i = 1; i < n; i++) {
+			tmp = sw_sync_fence_create(producer_timelines[i],
+						   "name", it);
+			merged = sync_merge("name", tmp, fence);
+			sw_sync_fence_destroy(tmp);
+			sw_sync_fence_destroy(fence);
+			fence = merged;
+		}
+
+		valid = sw_sync_fence_is_valid(fence);
+		ASSERT(valid, "Failure merging fences\n");
+
+		/*
+		 * Make sure we see an increment from every producer thread.
+		 * Vary the means by which we wait.
+		 */
+		if (iterations % 8 != 0) {
+			ASSERT(sync_wait(fence, -1) > 0,
+			       "Producers did not increment as expected\n");
+		} else {
+			ASSERT(busy_wait_on_fence(fence) == 0,
+			       "Producers did not increment as expected\n");
+		}
+
+		ASSERT(test_data_mpsc.counter == n * it,
+		       "Counter value mismatch!\n");
+
+		/* Release the producer threads */
+		ASSERT(sw_sync_timeline_inc(consumer_timeline, 1) == 0,
+		       "Failure releasing producer threads\n");
+
+		sw_sync_fence_destroy(fence);
+	}
+
+	return 0;
+}
+
+int test_consumer_stress_multi_producer_single_consumer(void)
+{
+	int iterations = 1 << 12;
+	int n = 5;
+	long i, ret;
+	int producer_timelines[n];
+	int consumer_timeline;
+	pthread_t threads[n];
+
+	consumer_timeline = sw_sync_timeline_create();
+	for (i = 0; i < n; i++)
+		producer_timelines[i] = sw_sync_timeline_create();
+
+	test_data_mpsc.producer_timelines = producer_timelines;
+	test_data_mpsc.consumer_timeline = consumer_timeline;
+	test_data_mpsc.iterations = iterations;
+	test_data_mpsc.threads = n;
+	test_data_mpsc.counter = 0;
+	pthread_mutex_init(&test_data_mpsc.lock, NULL);
+
+	for (i = 0; i < n; i++) {
+		pthread_create(&threads[i], NULL, (void * (*)(void *))
+			       mpsc_producer_thread, (void *)i);
+	}
+
+	/* Consumer thread runs here */
+	ret = mpcs_consumer_thread();
+
+	for (i = 0; i < n; i++)
+		pthread_join(threads[i], NULL);
+
+	return ret;
+}
diff --git a/tools/testing/selftests/sync/sync_stress_merge.c b/tools/testing/selftests/sync/sync_stress_merge.c
new file mode 100644
index 000000000000..99e83ef45fbf
--- /dev/null
+++ b/tools/testing/selftests/sync/sync_stress_merge.c
@@ -0,0 +1,115 @@
+/*
+ *  sync stress test: merging
+ *  Copyright 2015-2016 Collabora Ltd.
+ *
+ *  Based on the implementation from the Android Open Source Project,
+ *
+ *  Copyright 2012 Google, Inc
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ *  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *  OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include "sync.h"
+#include "sw_sync.h"
+#include "synctest.h"
+
+int test_merge_stress_random_merge(void)
+{
+	int i, size, ret;
+	int timeline_count = 32;
+	int merge_count = 1024 * 32;
+	int timelines[timeline_count];
+	int fence_map[timeline_count];
+	int fence, tmpfence, merged, valid;
+	int timeline, timeline_offset, sync_point;
+
+	srand(time(NULL));
+
+	for (i = 0; i < timeline_count; i++)
+		timelines[i] = sw_sync_timeline_create();
+
+	fence = sw_sync_fence_create(timelines[0], "fence", 0);
+	valid = sw_sync_fence_is_valid(fence);
+	ASSERT(valid, "Failure creating fence\n");
+
+	memset(fence_map, -1, sizeof(fence_map));
+	fence_map[0] = 0;
+
+	/*
+	 * Randomly create sync_points out of a fixed set of timelines,
+	 * and merge them together
+	 */
+	for (i = 0; i < merge_count; i++) {
+		/* Generate sync_point. */
+		timeline_offset = rand() % timeline_count;
+		timeline = timelines[timeline_offset];
+		sync_point = rand();
+
+		/* Keep track of the latest sync_point in each timeline. */
+		if (fence_map[timeline_offset] == -1)
+			fence_map[timeline_offset] = sync_point;
+		else if (fence_map[timeline_offset] < sync_point)
+			fence_map[timeline_offset] = sync_point;
+
+		/* Merge */
+		tmpfence = sw_sync_fence_create(timeline, "fence", sync_point);
+		merged = sync_merge("merge", tmpfence, fence);
+		sw_sync_fence_destroy(tmpfence);
+		sw_sync_fence_destroy(fence);
+		fence = merged;
+
+		valid = sw_sync_fence_is_valid(merged);
+		ASSERT(valid, "Failure creating fence i\n");
+	}
+
+	size = 0;
+	for (i = 0; i < timeline_count; i++)
+		if (fence_map[i] != -1)
+			size++;
+
+	/* Confirm our map matches the fence. */
+	ASSERT(sync_fence_size(fence) == size,
+	       "Quantity of elements not matching\n");
+
+	/* Trigger the merged fence */
+	for (i = 0; i < timeline_count; i++) {
+		if (fence_map[i] != -1) {
+			ret = sync_wait(fence, 0);
+			ASSERT(ret == 0,
+			       "Failure waiting on fence until timeout\n");
+			/* Increment the timeline to the last sync_point */
+			sw_sync_timeline_inc(timelines[i], fence_map[i]);
+		}
+	}
+
+	/* Check that the fence is triggered. */
+	ret = sync_wait(fence, 0);
+	ASSERT(ret > 0, "Failure triggering fence\n");
+
+	sw_sync_fence_destroy(fence);
+
+	for (i = 0; i < timeline_count; i++)
+		sw_sync_timeline_destroy(timelines[i]);
+
+	return 0;
+}
diff --git a/tools/testing/selftests/sync/sync_stress_parallelism.c b/tools/testing/selftests/sync/sync_stress_parallelism.c
new file mode 100644
index 000000000000..e6c9be671dfc
--- /dev/null
+++ b/tools/testing/selftests/sync/sync_stress_parallelism.c
@@ -0,0 +1,111 @@
+/*
+ *  sync stress test: parallelism
+ *  Copyright 2015-2016 Collabora Ltd.
+ *
+ *  Based on the implementation from the Android Open Source Project,
+ *
+ *  Copyright 2012 Google, Inc
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ *  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *  OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <pthread.h>
+
+#include "sync.h"
+#include "sw_sync.h"
+#include "synctest.h"
+
+static struct {
+	int iterations;
+	int timeline;
+	int counter;
+} test_data_two_threads;
+
+static int test_stress_two_threads_shared_timeline_thread(void *d)
+{
+	int thread_id = (long)d;
+	int timeline = test_data_two_threads.timeline;
+	int iterations = test_data_two_threads.iterations;
+	int fence, valid, ret, i;
+
+	for (i = 0; i < iterations; i++) {
+		fence = sw_sync_fence_create(timeline, "fence",
+					     i * 2 + thread_id);
+		valid = sw_sync_fence_is_valid(fence);
+		ASSERT(valid, "Failure allocating fence\n");
+
+		/* Wait on the prior thread to complete */
+		ret = sync_wait(fence, -1);
+		ASSERT(ret > 0, "Problem occurred on prior thread\n");
+
+		/*
+		 * Confirm the previous thread's writes are visible
+		 * and then increment
+		 */
+		ASSERT(test_data_two_threads.counter == i * 2 + thread_id,
+		       "Counter got damaged!\n");
+		test_data_two_threads.counter++;
+
+		/* Kick off the other thread */
+		ret = sw_sync_timeline_inc(timeline, 1);
+		ASSERT(ret == 0, "Advancing timeline failed\n");
+
+		sw_sync_fence_destroy(fence);
+	}
+
+	return 0;
+}
+
+int test_stress_two_threads_shared_timeline(void)
+{
+	pthread_t a, b;
+	int valid;
+	int timeline = sw_sync_timeline_create();
+
+	valid = sw_sync_timeline_is_valid(timeline);
+	ASSERT(valid, "Failure allocating timeline\n");
+
+	test_data_two_threads.iterations = 1 << 16;
+	test_data_two_threads.counter = 0;
+	test_data_two_threads.timeline = timeline;
+
+	/*
+	 * Use a single timeline to synchronize two threads
+	 * hammmering on the same counter.
+	 */
+
+	pthread_create(&a, NULL, (void *(*)(void *))
+		       test_stress_two_threads_shared_timeline_thread,
+		       (void *)0);
+	pthread_create(&b, NULL, (void *(*)(void *))
+		       test_stress_two_threads_shared_timeline_thread,
+		       (void *)1);
+
+	pthread_join(a, NULL);
+	pthread_join(b, NULL);
+
+	/* make sure the threads did not trample on one another */
+	ASSERT(test_data_two_threads.counter ==
+	       test_data_two_threads.iterations * 2,
+	       "Counter has unexpected value\n");
+
+	sw_sync_timeline_destroy(timeline);
+
+	return 0;
+}
diff --git a/tools/testing/selftests/sync/sync_test.c b/tools/testing/selftests/sync/sync_test.c
new file mode 100644
index 000000000000..9ea08d9f0b13
--- /dev/null
+++ b/tools/testing/selftests/sync/sync_test.c
@@ -0,0 +1,79 @@
+/*
+ *  sync test runner
+ *  Copyright 2015-2016 Collabora Ltd.
+ *
+ *  Based on the implementation from the Android Open Source Project,
+ *
+ *  Copyright 2012 Google, Inc
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ *  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *  OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include "synctest.h"
+
+static int run_test(int (*test)(void), char *name)
+{
+	int result;
+	pid_t childpid;
+
+	fflush(stdout);
+	childpid = fork();
+
+	if (childpid) {
+		waitpid(childpid, &result, 0);
+		if (WIFEXITED(result))
+			return WEXITSTATUS(result);
+		return 1;
+	}
+
+	printf("[RUN]\tExecuting %s\n", name);
+	exit(test());
+}
+
+int main(void)
+{
+	int err = 0;
+
+	printf("[RUN]\tTesting sync framework\n");
+
+	err += RUN_TEST(test_alloc_timeline);
+	err += RUN_TEST(test_alloc_fence);
+	err += RUN_TEST(test_alloc_fence_negative);
+
+	err += RUN_TEST(test_fence_one_timeline_wait);
+	err += RUN_TEST(test_fence_one_timeline_merge);
+	err += RUN_TEST(test_fence_merge_same_fence);
+	err += RUN_TEST(test_fence_multi_timeline_wait);
+	err += RUN_TEST(test_stress_two_threads_shared_timeline);
+	err += RUN_TEST(test_consumer_stress_multi_producer_single_consumer);
+	err += RUN_TEST(test_merge_stress_random_merge);
+
+	if (err)
+		printf("[FAIL]\tsync errors: %d\n", err);
+	else
+		printf("[OK]\tsync\n");
+
+	return !!err;
+}
diff --git a/tools/testing/selftests/sync/sync_wait.c b/tools/testing/selftests/sync/sync_wait.c
new file mode 100644
index 000000000000..d69b752f6550
--- /dev/null
+++ b/tools/testing/selftests/sync/sync_wait.c
@@ -0,0 +1,91 @@
+/*
+ *  sync fence wait tests
+ *  Copyright 2015-2016 Collabora Ltd.
+ *
+ *  Based on the implementation from the Android Open Source Project,
+ *
+ *  Copyright 2012 Google, Inc
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ *  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *  OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "sync.h"
+#include "sw_sync.h"
+#include "synctest.h"
+
+int test_fence_multi_timeline_wait(void)
+{
+	int timelineA, timelineB, timelineC;
+	int fenceA, fenceB, fenceC, merged;
+	int valid, active, signaled, ret;
+
+	timelineA = sw_sync_timeline_create();
+	timelineB = sw_sync_timeline_create();
+	timelineC = sw_sync_timeline_create();
+
+	fenceA = sw_sync_fence_create(timelineA, "fenceA", 5);
+	fenceB = sw_sync_fence_create(timelineB, "fenceB", 5);
+	fenceC = sw_sync_fence_create(timelineC, "fenceC", 5);
+
+	merged = sync_merge("mergeFence", fenceB, fenceA);
+	merged = sync_merge("mergeFence", fenceC, merged);
+
+	valid = sw_sync_fence_is_valid(merged);
+	ASSERT(valid, "Failure merging fence from various timelines\n");
+
+	/* Confirm fence isn't signaled */
+	active = sync_fence_count_with_status(merged, FENCE_STATUS_ACTIVE);
+	ASSERT(active == 3, "Fence signaled too early!\n");
+
+	ret = sync_wait(merged, 0);
+	ASSERT(ret == 0,
+	       "Failure waiting on fence until timeout\n");
+
+	ret = sw_sync_timeline_inc(timelineA, 5);
+	active = sync_fence_count_with_status(merged, FENCE_STATUS_ACTIVE);
+	signaled = sync_fence_count_with_status(merged, FENCE_STATUS_SIGNALED);
+	ASSERT(active == 2 && signaled == 1,
+	       "Fence did not signal properly!\n");
+
+	ret = sw_sync_timeline_inc(timelineB, 5);
+	active = sync_fence_count_with_status(merged, FENCE_STATUS_ACTIVE);
+	signaled = sync_fence_count_with_status(merged, FENCE_STATUS_SIGNALED);
+	ASSERT(active == 1 && signaled == 2,
+	       "Fence did not signal properly!\n");
+
+	ret = sw_sync_timeline_inc(timelineC, 5);
+	active = sync_fence_count_with_status(merged, FENCE_STATUS_ACTIVE);
+	signaled = sync_fence_count_with_status(merged, FENCE_STATUS_SIGNALED);
+	ASSERT(active == 0 && signaled == 3,
+	       "Fence did not signal properly!\n");
+
+	/* confirm you can successfully wait */
+	ret = sync_wait(merged, 100);
+	ASSERT(ret > 0, "Failure waiting on signaled fence\n");
+
+	sw_sync_fence_destroy(merged);
+	sw_sync_fence_destroy(fenceC);
+	sw_sync_fence_destroy(fenceB);
+	sw_sync_fence_destroy(fenceA);
+	sw_sync_timeline_destroy(timelineC);
+	sw_sync_timeline_destroy(timelineB);
+	sw_sync_timeline_destroy(timelineA);
+
+	return 0;
+}
diff --git a/tools/testing/selftests/sync/synctest.h b/tools/testing/selftests/sync/synctest.h
new file mode 100644
index 000000000000..e7d1d57dba7a
--- /dev/null
+++ b/tools/testing/selftests/sync/synctest.h
@@ -0,0 +1,66 @@
+/*
+ *  sync tests
+ *  Copyright 2015-2016 Collabora Ltd.
+ *
+ *  Based on the implementation from the Android Open Source Project,
+ *
+ *  Copyright 2012 Google, Inc
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ *  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ *  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ *  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *  OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SELFTESTS_SYNCTEST_H
+#define SELFTESTS_SYNCTEST_H
+
+#include <stdio.h>
+
+#define ASSERT(cond, msg) do { \
+	if (!(cond)) { \
+		printf("[ERROR]\t%s", (msg)); \
+		return 1; \
+	} \
+} while (0)
+
+#define RUN_TEST(x) run_test((x), #x)
+
+/* Allocation tests */
+int test_alloc_timeline(void);
+int test_alloc_fence(void);
+int test_alloc_fence_negative(void);
+
+/* Fence tests with one timeline */
+int test_fence_one_timeline_wait(void);
+int test_fence_one_timeline_merge(void);
+
+/* Fence merge tests */
+int test_fence_merge_same_fence(void);
+
+/* Fence wait tests */
+int test_fence_multi_timeline_wait(void);
+
+/* Stress test - parallelism */
+int test_stress_two_threads_shared_timeline(void);
+
+/* Stress test - consumer */
+int test_consumer_stress_multi_producer_single_consumer(void);
+
+/* Stress test - merging */
+int test_merge_stress_random_merge(void);
+
+#endif
diff --git a/tools/testing/selftests/timers/.gitignore b/tools/testing/selftests/timers/.gitignore
index 68f3fc71ac44..cc986621f512 100644
--- a/tools/testing/selftests/timers/.gitignore
+++ b/tools/testing/selftests/timers/.gitignore
@@ -17,3 +17,4 @@ skew_consistency
 threadtest
 valid-adjtimex
 adjtick
+set-tz
diff --git a/tools/usb/usbip/.gitignore b/tools/usb/usbip/.gitignore
index 9aad9e30a8ba..03b892c8bd8c 100644
--- a/tools/usb/usbip/.gitignore
+++ b/tools/usb/usbip/.gitignore
@@ -2,6 +2,7 @@ Makefile
 Makefile.in
 aclocal.m4
 autom4te.cache/
+compile
 config.guess
 config.h
 config.h.in
@@ -21,7 +22,10 @@ src/Makefile.in
 stamp-h1
 libsrc/libusbip.la
 libsrc/libusbip_la-names.lo
+libsrc/libusbip_la-sysfs_utils.lo
 libsrc/libusbip_la-usbip_common.lo
+libsrc/libusbip_la-usbip_device_driver.lo
+libsrc/libusbip_la-usbip_host_common.lo
 libsrc/libusbip_la-usbip_host_driver.lo
 libsrc/libusbip_la-vhci_driver.lo
 src/usbip
diff --git a/tools/usb/usbip/src/usbipd.c b/tools/usb/usbip/src/usbipd.c
index a0972dea9e6c..009afb4a3aae 100644
--- a/tools/usb/usbip/src/usbipd.c
+++ b/tools/usb/usbip/src/usbipd.c
@@ -398,13 +398,6 @@ static int listen_all_addrinfo(struct addrinfo *ai_head, int sockfdlist[],
 		 * (see do_standalone_mode()) */
 		usbip_net_set_v6only(sock);
 
-		if (sock >= FD_SETSIZE) {
-			err("FD_SETSIZE: %s: sock=%d, max=%d",
-			    ai_buf, sock, FD_SETSIZE);
-			close(sock);
-			continue;
-		}
-
 		ret = bind(sock, ai->ai_addr, ai->ai_addrlen);
 		if (ret < 0) {
 			err("bind: %s: %d (%s)",
diff --git a/tools/virtio/linux/compiler.h b/tools/virtio/linux/compiler.h
index 845960e1cbf2..c9ccfd42ec13 100644
--- a/tools/virtio/linux/compiler.h
+++ b/tools/virtio/linux/compiler.h
@@ -4,6 +4,6 @@
 #define WRITE_ONCE(var, val) \
 	(*((volatile typeof(val) *)(&(var))) = (val))
 
-#define READ_ONCE(var) (*((volatile typeof(val) *)(&(var))))
+#define READ_ONCE(var) (*((volatile typeof(var) *)(&(var))))
 
 #endif
diff --git a/tools/virtio/linux/uaccess.h b/tools/virtio/linux/uaccess.h
index 0a578fe18653..fa05d01b2c90 100644
--- a/tools/virtio/linux/uaccess.h
+++ b/tools/virtio/linux/uaccess.h
@@ -1,8 +1,9 @@
 #ifndef UACCESS_H
 #define UACCESS_H
-extern void *__user_addr_min, *__user_addr_max;
 
-#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
+#include <linux/compiler.h>
+
+extern void *__user_addr_min, *__user_addr_max;
 
 static inline void __chk_user_ptr(const volatile void *p, size_t size)
 {
@@ -13,7 +14,7 @@ static inline void __chk_user_ptr(const volatile void *p, size_t size)
 ({								\
 	typeof(ptr) __pu_ptr = (ptr);				\
 	__chk_user_ptr(__pu_ptr, sizeof(*__pu_ptr));		\
-	ACCESS_ONCE(*(__pu_ptr)) = x;				\
+	WRITE_ONCE(*(__pu_ptr), x);				\
 	0;							\
 })
 
@@ -21,7 +22,7 @@ static inline void __chk_user_ptr(const volatile void *p, size_t size)
 ({								\
 	typeof(ptr) __pu_ptr = (ptr);				\
 	__chk_user_ptr(__pu_ptr, sizeof(*__pu_ptr));		\
-	x = ACCESS_ONCE(*(__pu_ptr));				\
+	x = READ_ONCE(*(__pu_ptr));				\
 	0;							\
 })
 
diff --git a/usr/Kconfig b/usr/Kconfig
index 572dcf7b6a44..6278f135256d 100644
--- a/usr/Kconfig
+++ b/usr/Kconfig
@@ -98,3 +98,130 @@ config RD_LZ4
 	help
 	  Support loading of a LZ4 encoded initial ramdisk or cpio buffer
 	  If unsure, say N.
+
+choice
+	prompt "Built-in initramfs compression mode"
+	depends on INITRAMFS_SOURCE!=""
+	optional
+	help
+	  This option allows you to decide by which algorithm the builtin
+	  initramfs will be compressed.  Several compression algorithms are
+	  available, which differ in efficiency, compression and
+	  decompression speed.  Compression speed is only relevant
+	  when building a kernel.  Decompression speed is relevant at
+	  each boot. Also the memory usage during decompression may become
+	  relevant on memory constrained systems. This is usually based on the
+	  dictionary size of the algorithm with algorithms like XZ and LZMA
+	  featuring large dictionary sizes.
+
+	  High compression options are mostly useful for users who are
+	  low on RAM, since it reduces the memory consumption during
+	  boot.
+
+	  Keep in mind that your build system needs to provide the appropriate
+	  compression tool to compress the generated initram cpio file for
+	  embedding.
+
+	  If in doubt, select 'None'
+
+config INITRAMFS_COMPRESSION_NONE
+	bool "None"
+	help
+	  Do not compress the built-in initramfs at all. This may sound wasteful
+	  in space, but, you should be aware that the built-in initramfs will be
+	  compressed at a later stage anyways along with the rest of the kernel,
+	  on those architectures that support this. However, not compressing the
+	  initramfs may lead to slightly higher memory consumption during a
+	  short time at boot, while both the cpio image and the unpacked
+	  filesystem image will be present in memory simultaneously
+
+config INITRAMFS_COMPRESSION_GZIP
+	bool "Gzip"
+	depends on RD_GZIP
+	help
+	  Use the old and well tested gzip compression algorithm. Gzip provides
+	  a good balance between compression ratio and decompression speed and
+	  has a reasonable compression speed. It is also more likely to be
+	  supported by your build system as the gzip tool is present by default
+	  on most distros.
+
+config INITRAMFS_COMPRESSION_BZIP2
+	bool "Bzip2"
+	depends on RD_BZIP2
+	help
+	  It's compression ratio and speed is intermediate. Decompression speed
+	  is slowest among the choices. The initramfs size is about 10% smaller
+	  with bzip2, in comparison to gzip. Bzip2 uses a large amount of
+	  memory. For modern kernels you will need at least 8MB RAM or more for
+	  booting.
+
+	  If you choose this, keep in mind that you need to have the bzip2 tool
+	  available to be able to compress the initram.
+
+config INITRAMFS_COMPRESSION_LZMA
+	bool "LZMA"
+	depends on RD_LZMA
+	help
+	  This algorithm's compression ratio is best but has a large dictionary
+	  size which might cause issues in memory constrained systems.
+	  Decompression speed is between the other choices. Compression is
+	  slowest. The initramfs size is about 33% smaller with LZMA in
+	  comparison to gzip.
+
+	  If you choose this, keep in mind that you may need to install the xz
+	  or lzma tools to be able to compress the initram.
+
+config INITRAMFS_COMPRESSION_XZ
+	bool "XZ"
+	depends on RD_XZ
+	help
+	  XZ uses the LZMA2 algorithm and has a large dictionary which may cause
+	  problems on memory constrained systems. The initramfs size is about
+	  30% smaller with XZ in comparison to gzip. Decompression speed is
+	  better than that of bzip2 but worse than gzip and LZO. Compression is
+	  slow.
+
+	  If you choose this, keep in mind that you may need to install the xz
+	  tool to be able to compress the initram.
+
+config INITRAMFS_COMPRESSION_LZO
+	bool "LZO"
+	depends on RD_LZO
+	help
+	  It's compression ratio is the second poorest amongst the choices. The
+	  kernel size is about 10% bigger than gzip. Despite that, it's
+	  decompression speed is the second fastest and it's compression speed
+	  is quite fast too.
+
+	  If you choose this, keep in mind that you may need to install the lzop
+	  tool to be able to compress the initram.
+
+config INITRAMFS_COMPRESSION_LZ4
+	bool "LZ4"
+	depends on RD_LZ4
+	help
+	  It's compression ratio is the poorest amongst the choices. The kernel
+	  size is about 15% bigger than gzip; however its decompression speed
+	  is the fastest.
+
+	  If you choose this, keep in mind that most distros don't provide lz4
+	  by default which could cause a build failure.
+
+endchoice
+
+config INITRAMFS_COMPRESSION
+	string
+	default ""      if INITRAMFS_COMPRESSION_NONE
+	default ".gz"   if INITRAMFS_COMPRESSION_GZIP
+	default ".bz2"  if INITRAMFS_COMPRESSION_BZIP2
+	default ".lzma" if INITRAMFS_COMPRESSION_LZMA
+	default ".xz"   if INITRAMFS_COMPRESSION_XZ
+	default ".lzo"  if INITRAMFS_COMPRESSION_LZO
+	default ".lz4"  if INITRAMFS_COMPRESSION_LZ4
+	default ".gz"   if RD_GZIP
+	default ".lz4"  if RD_LZ4
+	default ".lzo"  if RD_LZO
+	default ".xz"   if RD_XZ
+	default ".lzma" if RD_LZMA
+	default ".bz2"  if RD_BZIP2
+	default ""
diff --git a/usr/Makefile b/usr/Makefile
index e767f019accf..17a513268325 100644
--- a/usr/Makefile
+++ b/usr/Makefile
@@ -5,25 +5,7 @@
 klibcdirs:;
 PHONY += klibcdirs
 
-
-# Bzip2
-suffix_$(CONFIG_RD_BZIP2)  = .bz2
-
-# Lzma
-suffix_$(CONFIG_RD_LZMA)   = .lzma
-
-# XZ
-suffix_$(CONFIG_RD_XZ)     = .xz
-
-# Lzo
-suffix_$(CONFIG_RD_LZO)    = .lzo
-
-# Lz4
-suffix_$(CONFIG_RD_LZ4)    = .lz4
-
-# Gzip
-suffix_$(CONFIG_RD_GZIP)   = .gz
-
+suffix_y = $(CONFIG_INITRAMFS_COMPRESSION)
 AFLAGS_initramfs_data.o += -DINITRAMFS_IMAGE="usr/initramfs_data.cpio$(suffix_y)"
 
 # Generate builtin.o based on initramfs_data.o
diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c
index 27a1f6341d41..ae95fc0e3214 100644
--- a/virt/kvm/arm/arch_timer.c
+++ b/virt/kvm/arm/arch_timer.c
@@ -425,6 +425,11 @@ int kvm_timer_hyp_init(void)
 	info = arch_timer_get_kvm_info();
 	timecounter = &info->timecounter;
 
+	if (!timecounter->cc) {
+		kvm_err("kvm_arch_timer: uninitialized timecounter\n");
+		return -ENODEV;
+	}
+
 	if (info->virtual_irq <= 0) {
 		kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
 			info->virtual_irq);
@@ -498,17 +503,7 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu)
 	if (ret)
 		return ret;
 
-
-	/*
-	 * There is a potential race here between VCPUs starting for the first
-	 * time, which may be enabling the timer multiple times.  That doesn't
-	 * hurt though, because we're just setting a variable to the same
-	 * variable that it already was.  The important thing is that all
-	 * VCPUs have the enabled variable set, before entering the guest, if
-	 * the arch timers are enabled.
-	 */
-	if (timecounter)
-		timer->enabled = 1;
+	timer->enabled = 1;
 
 	return 0;
 }
diff --git a/virt/kvm/arm/vgic/vgic-its.c b/virt/kvm/arm/vgic/vgic-its.c
index 4660a7d04eea..8c2b3cdcb2c5 100644
--- a/virt/kvm/arm/vgic/vgic-its.c
+++ b/virt/kvm/arm/vgic/vgic-its.c
@@ -632,21 +632,22 @@ static bool vgic_its_check_id(struct vgic_its *its, u64 baser, int id)
 	int index;
 	u64 indirect_ptr;
 	gfn_t gfn;
+	int esz = GITS_BASER_ENTRY_SIZE(baser);
 
 	if (!(baser & GITS_BASER_INDIRECT)) {
 		phys_addr_t addr;
 
-		if (id >= (l1_tbl_size / GITS_BASER_ENTRY_SIZE(baser)))
+		if (id >= (l1_tbl_size / esz))
 			return false;
 
-		addr = BASER_ADDRESS(baser) + id * GITS_BASER_ENTRY_SIZE(baser);
+		addr = BASER_ADDRESS(baser) + id * esz;
 		gfn = addr >> PAGE_SHIFT;
 
 		return kvm_is_visible_gfn(its->dev->kvm, gfn);
 	}
 
 	/* calculate and check the index into the 1st level */
-	index = id / (SZ_64K / GITS_BASER_ENTRY_SIZE(baser));
+	index = id / (SZ_64K / esz);
 	if (index >= (l1_tbl_size / sizeof(u64)))
 		return false;
 
@@ -670,8 +671,8 @@ static bool vgic_its_check_id(struct vgic_its *its, u64 baser, int id)
 	indirect_ptr &= GENMASK_ULL(51, 16);
 
 	/* Find the address of the actual entry */
-	index = id % (SZ_64K / GITS_BASER_ENTRY_SIZE(baser));
-	indirect_ptr += index * GITS_BASER_ENTRY_SIZE(baser);
+	index = id % (SZ_64K / esz);
+	indirect_ptr += index * esz;
 	gfn = indirect_ptr >> PAGE_SHIFT;
 
 	return kvm_is_visible_gfn(its->dev->kvm, gfn);
diff --git a/virt/kvm/arm/vgic/vgic-kvm-device.c b/virt/kvm/arm/vgic/vgic-kvm-device.c
index ce1f4ed9daf4..fbe87a63d250 100644
--- a/virt/kvm/arm/vgic/vgic-kvm-device.c
+++ b/virt/kvm/arm/vgic/vgic-kvm-device.c
@@ -221,11 +221,9 @@ int kvm_register_vgic_device(unsigned long type)
 		ret = kvm_register_device_ops(&kvm_arm_vgic_v3_ops,
 					      KVM_DEV_TYPE_ARM_VGIC_V3);
 
-#ifdef CONFIG_KVM_ARM_VGIC_V3_ITS
 		if (ret)
 			break;
 		ret = kvm_vgic_register_its_device();
-#endif
 		break;
 	}
 
diff --git a/virt/kvm/arm/vgic/vgic-mmio-v2.c b/virt/kvm/arm/vgic/vgic-mmio-v2.c
index b44b359cbbad..78e34bc4d89b 100644
--- a/virt/kvm/arm/vgic/vgic-mmio-v2.c
+++ b/virt/kvm/arm/vgic/vgic-mmio-v2.c
@@ -129,6 +129,7 @@ static void vgic_mmio_write_target(struct kvm_vcpu *vcpu,
 				   unsigned long val)
 {
 	u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
+	u8 cpu_mask = GENMASK(atomic_read(&vcpu->kvm->online_vcpus) - 1, 0);
 	int i;
 
 	/* GICD_ITARGETSR[0-7] are read-only */
@@ -141,7 +142,7 @@ static void vgic_mmio_write_target(struct kvm_vcpu *vcpu,
 
 		spin_lock(&irq->irq_lock);
 
-		irq->targets = (val >> (i * 8)) & 0xff;
+		irq->targets = (val >> (i * 8)) & cpu_mask;
 		target = irq->targets ? __ffs(irq->targets) : 0;
 		irq->target_vcpu = kvm_get_vcpu(vcpu->kvm, target);
 
diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/virt/kvm/arm/vgic/vgic-mmio-v3.c
index 0d3c76a4208b..50f42f0f8c4f 100644
--- a/virt/kvm/arm/vgic/vgic-mmio-v3.c
+++ b/virt/kvm/arm/vgic/vgic-mmio-v3.c
@@ -42,7 +42,6 @@ u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len,
 	return reg | ((u64)val << lower);
 }
 
-#ifdef CONFIG_KVM_ARM_VGIC_V3_ITS
 bool vgic_has_its(struct kvm *kvm)
 {
 	struct vgic_dist *dist = &kvm->arch.vgic;
@@ -52,7 +51,6 @@ bool vgic_has_its(struct kvm *kvm)
 
 	return dist->has_its;
 }
-#endif
 
 static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu,
 					    gpa_t addr, unsigned int len)
diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h
index 9d9e014765a2..859f65c6e056 100644
--- a/virt/kvm/arm/vgic/vgic.h
+++ b/virt/kvm/arm/vgic/vgic.h
@@ -84,37 +84,11 @@ int vgic_v3_probe(const struct gic_kvm_info *info);
 int vgic_v3_map_resources(struct kvm *kvm);
 int vgic_register_redist_iodevs(struct kvm *kvm, gpa_t dist_base_address);
 
-#ifdef CONFIG_KVM_ARM_VGIC_V3_ITS
 int vgic_register_its_iodevs(struct kvm *kvm);
 bool vgic_has_its(struct kvm *kvm);
 int kvm_vgic_register_its_device(void);
 void vgic_enable_lpis(struct kvm_vcpu *vcpu);
 int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi);
-#else
-static inline int vgic_register_its_iodevs(struct kvm *kvm)
-{
-	return -ENODEV;
-}
-
-static inline bool vgic_has_its(struct kvm *kvm)
-{
-	return false;
-}
-
-static inline int kvm_vgic_register_its_device(void)
-{
-	return -ENODEV;
-}
-
-static inline void vgic_enable_lpis(struct kvm_vcpu *vcpu)
-{
-}
-
-static inline int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
-{
-	return -ENODEV;
-}
-#endif
 
 int kvm_register_vgic_device(unsigned long type);
 int vgic_lazy_init(struct kvm *kvm);
diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c
index efeceb0a222d..3815e940fbea 100644
--- a/virt/kvm/async_pf.c
+++ b/virt/kvm/async_pf.c
@@ -76,16 +76,20 @@ static void async_pf_execute(struct work_struct *work)
 	struct kvm_vcpu *vcpu = apf->vcpu;
 	unsigned long addr = apf->addr;
 	gva_t gva = apf->gva;
+	int locked = 1;
 
 	might_sleep();
 
 	/*
 	 * This work is run asynchromously to the task which owns
 	 * mm and might be done in another context, so we must
-	 * use FOLL_REMOTE.
+	 * access remotely.
 	 */
-	__get_user_pages_unlocked(NULL, mm, addr, 1, NULL,
-			FOLL_WRITE | FOLL_REMOTE);
+	down_read(&mm->mmap_sem);
+	get_user_pages_remote(NULL, mm, addr, 1, FOLL_WRITE, NULL, NULL,
+			&locked);
+	if (locked)
+		up_read(&mm->mmap_sem);
 
 	kvm_async_page_present_sync(vcpu, apf);
 
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index c55f5d6a7767..de102cae7125 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -70,16 +70,19 @@ MODULE_AUTHOR("Qumranet");
 MODULE_LICENSE("GPL");
 
 /* Architectures should define their poll value according to the halt latency */
-static unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
+unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
 module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR);
+EXPORT_SYMBOL_GPL(halt_poll_ns);
 
 /* Default doubles per-vcpu halt_poll_ns. */
-static unsigned int halt_poll_ns_grow = 2;
+unsigned int halt_poll_ns_grow = 2;
 module_param(halt_poll_ns_grow, uint, S_IRUGO | S_IWUSR);
+EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
 
 /* Default resets per-vcpu halt_poll_ns . */
-static unsigned int halt_poll_ns_shrink;
+unsigned int halt_poll_ns_shrink;
 module_param(halt_poll_ns_shrink, uint, S_IRUGO | S_IWUSR);
+EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
 
 /*
  * Ordering of locks:
@@ -595,7 +598,7 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
 		stat_data->kvm = kvm;
 		stat_data->offset = p->offset;
 		kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
-		if (!debugfs_create_file(p->name, 0444,
+		if (!debugfs_create_file(p->name, 0644,
 					 kvm->debugfs_dentry,
 					 stat_data,
 					 stat_fops_per_vm[p->kind]))
@@ -1415,13 +1418,12 @@ static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
 		npages = get_user_page_nowait(addr, write_fault, page);
 		up_read(&current->mm->mmap_sem);
 	} else {
-		unsigned int flags = FOLL_TOUCH | FOLL_HWPOISON;
+		unsigned int flags = FOLL_HWPOISON;
 
 		if (write_fault)
 			flags |= FOLL_WRITE;
 
-		npages = __get_user_pages_unlocked(current, current->mm, addr, 1,
-						   page, flags);
+		npages = get_user_pages_unlocked(addr, 1, page, flags);
 	}
 	if (npages != 1)
 		return npages;
@@ -3669,11 +3671,23 @@ static int vm_stat_get_per_vm(void *data, u64 *val)
 	return 0;
 }
 
+static int vm_stat_clear_per_vm(void *data, u64 val)
+{
+	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
+
+	if (val)
+		return -EINVAL;
+
+	*(ulong *)((void *)stat_data->kvm + stat_data->offset) = 0;
+
+	return 0;
+}
+
 static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file)
 {
 	__simple_attr_check_format("%llu\n", 0ull);
 	return kvm_debugfs_open(inode, file, vm_stat_get_per_vm,
-				NULL, "%llu\n");
+				vm_stat_clear_per_vm, "%llu\n");
 }
 
 static const struct file_operations vm_stat_get_per_vm_fops = {
@@ -3699,11 +3713,26 @@ static int vcpu_stat_get_per_vm(void *data, u64 *val)
 	return 0;
 }
 
+static int vcpu_stat_clear_per_vm(void *data, u64 val)
+{
+	int i;
+	struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
+	struct kvm_vcpu *vcpu;
+
+	if (val)
+		return -EINVAL;
+
+	kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
+		*(u64 *)((void *)vcpu + stat_data->offset) = 0;
+
+	return 0;
+}
+
 static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file)
 {
 	__simple_attr_check_format("%llu\n", 0ull);
 	return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm,
-				 NULL, "%llu\n");
+				 vcpu_stat_clear_per_vm, "%llu\n");
 }
 
 static const struct file_operations vcpu_stat_get_per_vm_fops = {
@@ -3738,7 +3767,26 @@ static int vm_stat_get(void *_offset, u64 *val)
 	return 0;
 }
 
-DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
+static int vm_stat_clear(void *_offset, u64 val)
+{
+	unsigned offset = (long)_offset;
+	struct kvm *kvm;
+	struct kvm_stat_data stat_tmp = {.offset = offset};
+
+	if (val)
+		return -EINVAL;
+
+	spin_lock(&kvm_lock);
+	list_for_each_entry(kvm, &vm_list, vm_list) {
+		stat_tmp.kvm = kvm;
+		vm_stat_clear_per_vm((void *)&stat_tmp, 0);
+	}
+	spin_unlock(&kvm_lock);
+
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
 
 static int vcpu_stat_get(void *_offset, u64 *val)
 {
@@ -3758,7 +3806,27 @@ static int vcpu_stat_get(void *_offset, u64 *val)
 	return 0;
 }
 
-DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
+static int vcpu_stat_clear(void *_offset, u64 val)
+{
+	unsigned offset = (long)_offset;
+	struct kvm *kvm;
+	struct kvm_stat_data stat_tmp = {.offset = offset};
+
+	if (val)
+		return -EINVAL;
+
+	spin_lock(&kvm_lock);
+	list_for_each_entry(kvm, &vm_list, vm_list) {
+		stat_tmp.kvm = kvm;
+		vcpu_stat_clear_per_vm((void *)&stat_tmp, 0);
+	}
+	spin_unlock(&kvm_lock);
+
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
+			"%llu\n");
 
 static const struct file_operations *stat_fops[] = {
 	[KVM_STAT_VCPU] = &vcpu_stat_fops,
@@ -3776,7 +3844,7 @@ static int kvm_init_debug(void)
 
 	kvm_debugfs_num_entries = 0;
 	for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) {
-		if (!debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
+		if (!debugfs_create_file(p->name, 0644, kvm_debugfs_dir,
 					 (void *)(long)p->offset,
 					 stat_fops[p->kind]))
 			goto out_dir;
diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c
index 1dd087da6f31..d32f239eb471 100644
--- a/virt/kvm/vfio.c
+++ b/virt/kvm/vfio.c
@@ -60,6 +60,19 @@ static void kvm_vfio_group_put_external_user(struct vfio_group *vfio_group)
 	symbol_put(vfio_group_put_external_user);
 }
 
+static void kvm_vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm)
+{
+	void (*fn)(struct vfio_group *, struct kvm *);
+
+	fn = symbol_get(vfio_group_set_kvm);
+	if (!fn)
+		return;
+
+	fn(group, kvm);
+
+	symbol_put(vfio_group_set_kvm);
+}
+
 static bool kvm_vfio_group_is_coherent(struct vfio_group *vfio_group)
 {
 	long (*fn)(struct vfio_group *, unsigned long);
@@ -159,6 +172,8 @@ static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg)
 
 		mutex_unlock(&kv->lock);
 
+		kvm_vfio_group_set_kvm(vfio_group, dev->kvm);
+
 		kvm_vfio_update_coherency(dev);
 
 		return 0;
@@ -196,6 +211,8 @@ static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg)
 
 		mutex_unlock(&kv->lock);
 
+		kvm_vfio_group_set_kvm(vfio_group, NULL);
+
 		kvm_vfio_group_put_external_user(vfio_group);
 
 		kvm_vfio_update_coherency(dev);
@@ -240,6 +257,7 @@ static void kvm_vfio_destroy(struct kvm_device *dev)
 	struct kvm_vfio_group *kvg, *tmp;
 
 	list_for_each_entry_safe(kvg, tmp, &kv->group_list, node) {
+		kvm_vfio_group_set_kvm(kvg->vfio_group, NULL);
 		kvm_vfio_group_put_external_user(kvg->vfio_group);
 		list_del(&kvg->node);
 		kfree(kvg);